From 44983fd083a9517afc3868d21d665e436eea5e08 Mon Sep 17 00:00:00 2001
From: Djordje Antic <djoantic@amd.com>
Date: Tue, 8 Jul 2025 04:09:36 -0500
Subject: [PATCH 1/6] Squashed 'external/llvm-project/' changes from
 e8215edf65d2..659e9cd04732
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

659e9cd04732 merge main into amd-staging (#2991)
dacd132ef399 Fix Unknown immediate size X86MCCodeEmitter::encodeInstruction (#2990)
8f5727a2a2f8 merge main into amd-staging
3afeb1805010 Fix Unknown immediate size X86MCCodeEmitter::encodeInstruction
80c3b03f99af [Comgr][NFC] Refactor remove dir for windows
3c76a054ac4c [CIR] Implement functional cast to ComplexType (#147147)
b8f5cbb4ffbf [OpenACC][CIR] 'cache' construct lowering (#146915)
59b1d631d7c8 [llvm][SandBoxIR] Fix Windows+clang compiler warning (#146905)
28649f211742 [Clang] accept @tparam on variable template partial specializations (#147219)
68309adef359 [NFC] Clean up poison folding in simplifyBinaryIntrinsic (#147259)
8ae8d3183272 [Offload] Add liboffload unit tests for shared/local memory (#147040)
0f48bafafc38 [ExtractAPI] Include tilde in destructor name (#146001)
1121034dd1c4 Triple: Record default exception handling type
b7c4ac2db4cd NFC, use structured binding to simplify the code.
6aa33ee971f6 [AArch64] arm64-bitfield-extract.ll - regenerate test checks
5f2e88a1254e [NFC][TableGen] Rename `CodeGenTarget` instruction accessors (#146767)
17d6aa01ec0c [ARM] Fix expansion of ABS in a call sequence (#147270)
599b9def29a8 [AArch64] sve-vscale.ll - cleanup test checks
7c2182a132c7 NFC, use structured binding to simplify the code in SourceManager.cpp.
543f94831245 [AMDGPU] Preserve exact flag for lshr (#146744)
d3d8ef7e416c [PowerPC] licm-xxsplti.ll - regenerate test checks
45d679218aeb [clang][OpenMP] Fix signed v unsigned comparison
fa859ed8c6bf [AMDGPU][NFC] Fix typo "store" -> "load" in comment for AMDGPUTLI::performLoadCombine (#147298)
c80fa2364bee [DAG] SDPatternMatch m_Zero/m_One/m_AllOnes have inconsistent undef h… (#147044)
acb4fff866f9 [NFC] Fix GCC warning on enum/non-enum ternary thanks to 438863a0
1a1a11f7096d [clang][OpenMP] Issue a warning when parsing future directive spelling (#146933)
ea62de5b1dc0 [mlir] NFC - refactor id builder and avoid leaking impl details (#146922)
3277f623440f [SDAG] Remove invalid check (NFC) (#146899)
747496269a44 [Hexagon] build-vector-float-type.ll - regenerate test checks
6190d407e09c [Hexagon] vect-vshifts.ll - regenerate test checks
9ebe6f9a1f9e [lldb][test] Fix libstdc++ std::variant formatter tests for valueless variants (#147283)
16dad11f503e [ValueTracking] Have sub and xor in KnownNonZero take the same exact path (#146975)
c3c3919dc2d3 Revert "[DenseMap] Do not align pointer sentinel values (NFC) (#146595)"
60630310e9a4 [clang-cl] Support /std:clatest (#147284)
0a6283696981 [mlir][gpu][transforms] Add support for mapping to lanes (#146912)
46e3ec0244c4 [BOLT][NFCI] Report perf script time (#147232)
3d6407965d5a [clang][OpenMP] Use DirectiveNameParser to parse directive names (#146779)
20864c4379f3 [LAA] Strip outdated comment in isDependent (NFC) (#146367)
9d9d68fd3b3e AMDGPU: Fix using underaligned atomicrmw in atomic scan tests
b5401624e1f1 DAG: Add RTLIB::getPOW helper (#147274)
db389bda9b28 Revert "[clang][modules-driver] Add scanner to detect C++20 module presence" (#147286)
2bb72924c245 [clang] Remove source range from CXXOperatorCallExpr (#147028)
6855573700c3 [flang][OpenMP] Fix parallel-firstprivate-clause-scalar.f90 test (#146932)
95862d089735 [clang] Fix manual memory management with SmallVector in ConceptRef (#147231)
3c86b747286f [AMDGPU] add_i1.ll / sub_i1.ll - regenerate test checks
49dc163d15ca [NVPTX] mulwide.ll - regenerate test checks
c4386031ac01 [RISCV] Use cached SubtargetInfo in AsmPrinter (NFC) (#147269)
e14e98290e71 [lldb][test] Combine libstdc++ and libc++ std::shared_ptr tests into generic test (#147141)
22357fe33a8a [analyzer] Avoid unnecessary super region invalidation in `CStringChecker` (#146212)
1113224f9444 [VectorCombine] Account for IRBuilder simplification in translateExt.
6daf2b956d0b [MLIR][Linalg] Remove elemwise_unary and elemwise_binary (#147082)
0aab8e4ccc18 [MLIR][LLVM][NFC] Remove unused argument from parseOptionalLLVMKeyword (#147058)
8315167a76e4 [Driver][RISCV] Fix incorrect compiler-rt path override in BareMetal toolchain after RISCVToolChain removal (#146849)
ae441382aec9 [MIPS] setcc-se.ll - regenerate test checks
8a5458c79ac7 [C23] Fix typeof handling in enum declarations (#146394)
40fb90efed7b [lldb][Formatters] Add shared/weak count to libstdc++ std::shared_ptr summary (#147166)
87632451165a [X86] pr134607.ll - regenerate test checks
fb845f93c003 [LAA] Hoist setting condition for RT-checks (#128045)
e7bcd3f7c743 ARM: Remove fast flags from ldexp calls test
f34de0a14fa9 [clang-tidy] [NFC] Potential dereference of nullptr. (#143145)
c995c5035567 [KeyInstr] Add bitcode support (#147260)
7a6435bec590 [DenseMap] Do not align pointer sentinel values (NFC) (#146595)
c19c71b90593 [lldb][test] Split out libc++ std::string tests that check corrupted strings (#147252)
912ab5241379 [lldb][test] Adjust import-std-module shared_ptr/weak_ptr tests
84e54515bc4e [AArch64] Add support for -mcpu=gb10. (#146515)
e2aa8781e18a [RISCV] Fix out-of-bounds offsets in fixed vector splice tests. NFC (#147247)
ded142671663 [clang][modules-driver] Add scanner to detect C++20 module presence (#145220)
de7ff1f2248e [mlir][spirv][nfc] Add StorageImageReadWithoutFormat capability to the Target test (#145949)
b6b2561337f5 [mlir][spirv] Add support for Aligned memory operand in CoopMatrix memory operations (#145480)
52383956f8b7 [DAG] Replace DAGCombiner::ConstantFoldBITCASTofBUILD_VECTOR with SelectionDAG::FoldConstantBuildVector (#147037)
465f2b05712d DAG: Fix asserting in error case for frexp softening (#147236)
7cd179612d7a [libclc] Fix typo in OpenCL header math/sincos.h (#147244)
b1a8c8a32c4b [flang][test] Fix REQUIRES and options for a few x86 specific tests (#146872)
3e934dded0d9 [flang][test] Fix test REQUIRES and options for aint.f90 (#146870)
ea685890b8b2 [libclc] Reduce include usage in OpenCL builtins (#146840)
08defcb6d3cd DAG: Fix asserting in error case for powi softening (#147237)
074ccde3b092 [lldb][Formatter] Consolidate libstdc++ and libc++ unique_ptr formatter tests into generic test (#147031)
0860f7aaeb42 [clangd][NFC] fix broken documentation link (#147088)
30836781e801 [X86] Don't use `rbp` when it's reserved (#146638)
8a221a585ca7 [Flang][OpenMP] Push context when parsing DECLARE VARIANT (#147075)
5271f9fba9dd [Flang][Doc] NFC: Minor fix for headings (#147077)
1c8283a30c6e [BasicTTIImpl] Add cost entries for ldexp, [l]lround (#146373)
c50415bb82a7 [AArch64] Do not promote scalable constants to global variables (#146926)
e430581e66ff [libc][bazel] Add missing header dependencies.
49d7c53756bc [lldb][test] Combine libstdc++ and libc++ std::map tests into generic test (#147174)
6fec6a98c015 [lldb][Formatters] Make libc++ and libstdc++ std::shared_ptr formatters consistent with each other (#147165)
d06956e24afa [lldb][Formatters] Use container summary helper for libstdc++ formatters (#147140)
7fea83e3143e [clang] NFC, use LocalLocOffsetTable in getFIleIDLocal.
784bd61fc497 [clang] Speedup getFileIDLocal with a separate offset table. (#146604)
b7d4735a0e2a [lldb][test] Combine libstdc++ and libc++ vector<bool> tests into generic test (#147137)
d8ef15637935 DAG: Remove verifyReturnAddressArgumentIsConstant (#147240)
dc14cd059418 [Comgr] Add test for get_version() API
fb628238c540 [Comgr] Add test for status_string() API
69c5925aee39 [Comgr] Add test for OpenCL 2.0
ca634f3762d3 [Comgr] Add test for DataAction APIs
19afd27eb8ef [Flang] Fix ACOSD and ASIND (fixes issue #145593) (#145656)
721adc18b77a [CodeGen] Fix stale comment about register coalescing order (#146762)
a5af8745039f [clang-format][NFC] Use `empty()` instead of comparing size() to 0 or 1
0fbaeafd7ff7 [CodeGen][NPM] Allow nested MF pass managers for -passes (#128852)
7f90a5c2d8d1 MCAsmBackend: Simplify FT_Data/FT_Relaxable code
d733c8e5c584 [Orc] Fix error handling in `ORCPlatformSupport::initialize` (#144637)
91cc33f3219c [TargetLowering] hasAndNotCompare should be checking for X, not Y (#146935)
1b136824bb81 [DebugInfo][LICM] Fix debug value loss caused by hoisting the cmp instructions (#146640)
3f8af4f55129 [NFC][AMDGPU] Pre-commit srl preserve flag test (#146747)
3f33e7ba5b08 [RISCV] Add Andes XAndesVSIntLoad (Andes Vector INT4 Load) extension (#147005)
9d8a1bec2b55 [Sema] Remove an unnecessary cast (NFC) (#147203)
be4cd9f4da98 [mlir] Remove unused includes (NFC) (#147206)
7f9bacd58e86 [IR] Remove an unnecessary cast (NFC) (#147205)
15653108b11e [Serialization] Remove an unnecessary cast (NFC) (#147204)
29b2b2263f9e [TargetParser] Use StringRef::consume_front (NFC) (#147202)
eca05fde844b [X86] Switch operands order for FMINIMUMNUM/FMAXIMUMNUM (#147193)
dc5d353f3a0a AArch64: Fix some missing llvm.frexp test coverage (#146691)
9865d7cb635f ARM: Add missing test coverage for windows frexp libcalls (#146690)
5a8e60e7249b [mlir] Use `llvm::fill` instead of `std::fill`(NFC) (#146889)
718e647a0c8f [mlir] Fix Wparentheses warning (#146893)
7b0f70a9b26a WebAssembly: Stop changing MCAsmInfo's ExceptionsType based on flags (#146343)
408e87184f82 MCAssembler: Merge fragmentNeedsRelaxation into relaxInstruction
fa9cd473284f [NFC][libclc] Rename __CLC_FUNCTION to either FUNCTION or __IMPL_FUNCTION (#146999)
eeba57a860e5 LoongArch: Remove unused relaxInstruction
5a70a9949c5a MCA: Remove unused MAB
24ba6d753de6 MCAsmBackend: Use assert for unreachable relaxInstruction dump code
3af5317c8938 M68k: Simplify relaxation code and reduce MCInst uses
01c97b4953e8 MCA: Remove relaxInstruction call
d476c1db5120 merge main into amd-staging (#2978)
2235e20885b4 MCAsmBackend: Remove redundant relaxInstruction overrides
51f4e2cda2b3 [Bitcode][NFC] Add abbrev for FUNC_CODE_DEBUG_LOC (#147211)
5ccbea9f480d [clang-format][NFC] Replace size() with empty() (#147164)
6a9a16da7a38 [VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). (#142322)
6d77b8261f39 [LV] Add tests where LCSSA phis could be re-used during SCEV expansion.
c35fbb5460b4 [VPlan] Use VPReductionPHIRecipe::isOrdered instead of CM (NFC).
6d67794d164e X86MCCodeEmitter: Work around gcc/value-range.cc bug in GCC 13.3.0
c5fff132d019 [LV] Add LVL::getRecurrenceDescriptor (NFC).
fcf1e4834ed4 merge main into amd-staging
cf0604723158 [LV] Remove AddedAnyChecks, check directly instead (NFC).
1828381ed259 [clang][bytecode] Fix APValue generation for RValueReferenceType (#147207)
e26f8ba8b6ed [NFC] Add `_LOWLEVEL_` before dwarf low-level library header guards (#147172)
494253f5d6d3 [DebugInfo] [DWARF] Fix C to CMake comment (#147199)
923a3cc160a1 [LLD] Fix crash on parsing ':ALIGN' in linker script (#146723)
9c7320e5d370 [clang][bytecode] Fix visiting for-range loop variable (#147188)
aa9e02cc10f8 [AArch64] Add lrint and lround costmodel tests. NFC
0d1e5ab2fd9c [libc++] P2655R3 common_reference_t of reference_wrapper Should Be a Reference Type (#141408)
f1549befc125 [AArch64] Add ldexp cost-model tests. NFC
ec9eefcef560 [clang][bytecode] Fix a crash in overflow builtins (#147189)
f63674cbf6d2 merge main into amd-staging (#2975)
db4e927f9f02 [clang][bytecode] Misc union fixes (#146824)
6c153e50f8b2 [clang-tidy] Fix false positives with template in `misc-unconventional-assign-operator` check (#143292)
ab92c68c5c38 [libc][NFC] Fix func name in comment. (#147183)
13f7a1fb5904 [libc++] LWG 3987 provide iterator.range access from flat_{map,set} (#137524)
fc75dda7f6da merge main into amd-staging
ca297cd6e4ad [MLIR] Add support for IntArrayProp<I32Prop> (#146685)
6c257f7d0d33 [PowerPC] fp-branch.ll - regenerate checks
5f44b5a3054d [libc++][doc][NFC] update release notes on P3372R3 (#147161)
c358979d7f45 [libc++] fix atomic::wait memory order (#146267)
ba7d78ac4597 [DAG] foldABSToABD - fallback to value tracking if the (ABS (SUB LHS, RHS)) operands aren't extended (#147053)
7ef0f9a216b1 merge main into amd-staging (#2974)
bdbaa9636d27 merge main into amd-staging
9f66ebe42715 MC: Eliminate redundant fragment relaxation
6f87e5812d91 merge main into amd-staging (#2973)
aec88832df5e MC: Remove unneeded MCFixupKind casts
d575f802104f MCFixup: Make MCFixupKind a type alias
878ce210e30f [clang-format] Propagate `LeadingEmptyLinesAffected` when joining lines (#146761)
c43efcb039dd [MLIR][MemRef]-Add basic folding for memref ViewOp (#146237)
5a4002349786 MCAsmBackend: Reduce FK_NONE uses
83401ed6a233 [clang][bytecode] Narrow allocated single-array Pointer (#147160)
178e18263f1a [InstCombine] Add tests for moving exts across identity shuffles.
6433030f6984 [Hexagon] Use range-based for loops (NFC) (#147157)
16435a87b6a7 [CodeGen] Remove an unnecessary cast (NFC) (#147155)
945ed2ed5eea [MLIR][SCF] fix loop pipelining pass use of uninitialized value (#146991)
30298f91367f MC: Simplify fragment code. NFC
9a934611c36f merge main into amd-staging
602791c0c1a3 Revert "[AMDGPU] SelectionDAG divergence tracking should take into ac… (#2971)
244e053b6c39 MC: Remove llvm/MC/MCFixupKindInfo.h
152c9d577c41 MC: Sink FKF_IsAlignedDownTo32Bits to needed targets
3f4be893e3ba AArch64AsmBackend: Remove redundant PCRelFlagVal
430c0376c8a0 [Sema] Remove an unnecessary cast (NFC) (#147154)
fb2c7610e831 [clang][bytecode] Fix comparing pointers pointing to base classes (#146285)
38b8ef16f76f [mlir] Remove unused includes (NFC) (#147158)
b94a9bc741e5 ARM: Improve branch fixup test
7d65cb1952dc [ARM] Copy (SELECT_CC setgt, iN lhs, -1, 1, -1) -> (OR (ASR lhs, N-1), 1 from AArch64 to ARM (#146561)
3448e9c075aa [AArch64][GlobalISel] Fix lowering of i64->f32 itofp. (#132703)
9eb4fc700636 [libc++] constexpr `flat_set` (#140360)
113ea3d15be0 [gn build] Port ca697096aa6f
f17a6387daf0 [gn build] Port 1121d64ba96d
ca697096aa6f [libc++] Granularize <locale> (#146650)
1121d64ba96d [libc++][NFC] Merge add_{r,l}value_reference.h (#147022)
bdbac2bb4954 [libc++] default some special members in map and set (#147081)
22f8ceded4e1 [libc++] Add the thread safety annotations unconditionally (#117497)
c9d9c3e34984 [InstCombine] Fold `icmp pred X + K, Y -> icmp pred2 X, Y` if both X and Y is divisible by K (#147130)
0bc6d60b9b53 [llvm] Fix typos in documentation (#147129)
15f07db6a541 [clang] Fix a typo in documentation (#147128)
2d71692bcfe9 Revert "[AMDGPU] SelectionDAG divergence tracking should take into account Target divergency. (#144947)"
e64289baa00f [clang][Sema] Unify getPrototypeLoc helpers in SemaCodeComplete and clangd (#143345)
0a067dc10700 [Attributor] Swap range metadata to attribute for calls. (#108835)
1941d40a97bf merge main into amd-staging (#2970)
9d994d1c08c4 [bazel] Port b9b2661f72ac5f9d4f23d9bb83131aa3d46020b9
b726713e4fcb merge main into amd-staging
a38cf8573890 [MachineLICM] Let targets decide if copy instructions are cheap (#146599)
fb13be06e18d MC: Generalize evaluateTargetFixup
1947eb93974a merge main into amd-staging (#2967)
cd39eaeb376e MC: Merge MCFixupKindInfo.h into MCAsmBackend.h
a49a5dba5d82 test: Update mc-dump.s
65e11f600db3 [Clang][AArch64] Remove redundant tune args to the backend (#146896)
3d6f4fb5f7ac [Clang][Sema] Do not perform error recovery for invalid member using-declaration in C++20+ mode (#147003)
9372f4050a94 [LoongArch] Pre-commit for optimizing bitcast extracted fp elements. NFC
32f83d38bd62 MCFixup: Remove FK_PCRel_
158fa4ae8389 AMDGPU: Replace deprecated FK_PCRel_ with FK_Data_ fixup and PCRel flag
49c91ec6f179 Hexagon: Remove dead deprecated FK_PCRel_4 use
abaa55d937e5 COFF: Replace deprecated FK_PCRel_ with FK_Data_ fixup and PCRel flag
970ed59e031d CSKY: Remove unused deprecated FK_PCRel_4
a26f8f234843 BPF: Replace deprecated FK_PCRel_ with FK_Data_ fixup and PCRel flag
66952f7b2d61 M68k: Remove unused deprecated FK_PCRel_
c20379198c7f X86: Replace deprecated FK_PCRel_ with FK_Data_ fixup and PCRel flag
22c590bd3c75 [RISCV][ISel] Optimize setcc with mask test idioms (#147015)
ae3d3135277a VE: Remove dead FK_PCRel_* code
c05593b1908e M68: Replace FK_PCRel_ with FK_Data_+PCRel
5760c064bbe7 M68,test: Remove CHECK lines for MCAsmStreamer's fixup output
45509a8c6d2d M68,test: Remove CHECK lines for MCAsmStreamer's fixup output
6ec27f1b8a5a MCFixup: Remove FKF_IsPCRel
74fcff4905ff CSKY: Remove unneeded MCFixupKindInfo::FKF_IsPCRel
d85536c5e362 CSKYMCCodeEmitter: Set PCRel at fixup creation
0e9571df982f M68kMCCodeEmitter: Standardize how fixups are appended
e0384a7c5c17 merge main into amd-staging (#2965)
09372335c51b X86: Test fixup/relocation with -filetype=obj
3e2e99bc6778 [Analysis] Remove traces of BlockEdgesAdder (#147102)
f78d74f9c86f [llvm] Use llvm::fill (NFC) (#147099)
57cb9265bf48 [Analysis] Drop const from a return type (NFC) (#147098)
21fb7e68ef05 [Driver] Use range-based for loops (NFC) (#146987)
add2c58e6560 [Index] Remove an unnecessary cast (NFC) (#146944)
755b4732cf81 VEMCCodeEmitter: Set PCRel at fixup creation
7b7dc151a75b AVRMCCodeEmitter: Set PCRel at fixup creation
3544eba44a33 MSP430MCCodeEmitter: Set PCRel at fixup creation
b418e73becc1 AMDGPUMCCodeEmitter: Set PCRel at fixup creation
21cf7bd4ec51 BPFMCCodeEmitter: Set PCRel at fixup creation
56cf2cb0eadd HexagonMCCodeEmitter: Set PCRel at fixup creation
def731a787da MipsMCCodeEmitter: Set PCRel at fixup creation
adc566006d65 merge main into amd-staging
5b7f1c17d943 BOLT: Replace deprecated MCFixupKindInfo::FKF_IsPCRel with MCFixup::isPCRel
3c5d03a86a04 SystemZMCCodeEmitter: Standardize how fixups are appended
637a38211824 SystemZMCCodeEmitter: Remove unneeded MCFixupKindInfo::FKF_IsPCRel
8bb4e534284d PPCMCCodeEmitter: Set PCRel at fixup creation
6670fe2ed498 ARMMCCodeEmitter: Set PCRel at fixup creation
73c03b92cec6 XtensaMCCodeEmitter: Set PCRel at fixup creation
2bfc488d34ef X86MCCodeEmitter: Remove unneeded MCFixupKindInfo::FKF_IsPCRel
955c04862e99 AArch64MCCodeEmitter: Standardize how fixups are appended
43397e5fe3de LoongArchMCCodeEmitter: Set PCRel at fixup creation
b9257a34a9e0 RISCVMCCodeEmitter: Set PCRel at fixup creation
7d500b115db5 SPARC: Remove unneeded MCFixupKindInfo::FKF_IsPCRel
83fbd8690911 SPARCMCCodeEmitter: Set PCRel at fixup creation
d337e09e73b3 SystemZMCCodeEmitter: Set PCRel at fixup creation
eb84af49a1bf AArch64MCCodeEmitter: Set PCRel at fixup creation
07ae19c132e1 [RISCV] Move performCombineVMergeAndVOps to RISCVVectorPeephole (#144076)
07286b1fcdec [InstCombine] Propagate poison pow[i], [us]add, [us]sub and [us]mul (#146750)
6db02dc4311b [Clang] Introduce `--offload-targets` for `-fopenmp-targets` (#146594)
454e4e3e29d8 [mlir][AffineExpr] Order arguments in the commutative affine exprs (#146895)
1f3f9874b025 [VPlan] Fix crash when narrowing interleave-groups with reuse.
ea600576a6f9 MCFixup: Add PCRel to ctor parameter and set it in  X86MCCodeEmitter
fa9adbfda967 [mlir] Remove unused includes (NFC) (#147101)
ed0ee3a419f3 [mlir] Use llvm::fill (NFC) (#147100)
1be685ac2395 [mlir] Remove unnecessary casts (NFC) (#147097)
e2510b189dae [clang] SourceManager: Cache offsets for LastFileIDLookup to speed up getFileID (#146782)
a7cf78fd5261 merge main into amd-staging
372752c2ddd0 MCFixup: Remove unused Loc argument
20b3ab5683c6 MCFixup: Remove unused Loc argument
777391a2164b MCFixup: Improve location accuracy and remove MCFixup::Loc
833839ba65fd [ASTMatchers][NFC] improve 'isInteger' docs to show matching of unsigned (#147012)
8740ff822d46 [mlir][docs][python] Fix up testing docs (#147092)
249acb8e4913 MC: Update tests that check column information
b38de6c18ed7 [NFCI][LLVM] Adopt `ArrayRef::consume_front()` in a few places (#146793)
7b517cf743f1 MCParser: Add SMLoc to expressions
06922c463a57 [PowerPC] Pre-commit test case to ensure that milicode is not used for memcmp, but instead a library call is used on AIX OS. (#146948)
3c0058c8e835 Manual update of LLVM_MAIN_REVISION to 543426 (#2962)
6a948145aa7a [HLSL][RootSignature] Update `setDefaultFlags` to account for Root Signature Version (#145828)
b0473c599b04 [InstCombine] Pull extract through broadcast (#143380)
cc6a864788ff [RISCV][VLOPT] Add support for vfrec7.v (#146918)
2472cdcb69d2 [InstCombine] Refine nuw propagation in `OptimizePointerDifference` (#147059)
fd5ed046fdca [Flang][OpenMP][NFC] Remove flag toggling deprecated no hlfir flow in map-types-and-sizes.f90 (#146995)
6b19cdcefacb [Offload][amdgpu] Map `INVALID_CODE_OBJECT` to `INVALID_BINARY` (#147070)
0008af882dc3 [SROA] Allow as zext<i1> index when unfolding GEP select (#146929)
a4f31ccfa0ef [mlir-tblgen] Emit named operand indices (#146839)
c08ef07e8b70 Manual update of LLVM_MAIN_REVISION to 543426
a1789497f9f5 [gn build] Port 56e3fc4c42d1
56e3fc4c42d1 [NFC][HLSL][RootSignature] Split up `HLSLRootSignatureUtils` (#146124)
a438c6099786 [X86][FP16] Do not customize WidenLowerNode for half if VLX not enabled (#146994)
e8543213e5fc [IR] Use llvm::for_each (NFC) (#146989)
60fca53ad64c [llvm] Use llvm::fill (NFC) (#146988)
a465e35908b6 [Sema] Remove an unnecessary cast (NFC) (#146985)
f46c1d6bcc3b [PowerPC] Fix a warning
5224a17aa7cb [FuzzMutate] Prevent the mutator from generating illegal memory operations (#144885)
32946eb124e8 [lldb][Formatters] Fix weak reference count for std::shared_ptr/std::weak_ptr (#147033)
9fcea2e4656a [ARM] Add neon vector support for roundeven
258c048f643f [clang][bytecode] Fix copy constructors for empty unions (#147050)
45909ec469ce [PowePC] using MTVSRBMI instruction instead of constant pool in power10+ (#144084)
e5cd9bdfea90 [NFC][TableGen] Delete extra spaces in comments (#147004)
79d5db4613d4 [NFC][CodeGen] Improve comments in Target.td (#147006)
740da004af5e [DAG] Fix static analyzer warning about mismatched argument comments in isConstOrConstSplat. NFC.
ed17bf1e4c54 [flang] Fix tests broken by #146734 (#147055)
5eb195fa90c7 [mlir][Vector] Fold vector.constant_mask to SplatElementsAttr (#146724)
0b4941a30f9a [mlir][xevm] Add missing static value utils dependency (#147051)
af2bb8f82605 [Clang] Correctly handle allocations in the condition of a `if constexpr` (#146890)
ad20dc0a87fc [mlir][Transforms] Add `ApplyConversionAction` for profiling purposes (#146208)
b9b2661f72ac [MLIR][Dialect] Add XeVM dialect (#144811)
274e798a9832 [flang] use set_union instead of merge in added DerivedTypeCache (#147024)
0ba59587fa98 [flang] Optimize assignments of multidimensional arrays (#146408)
d17a248fc614 [libcxx][test] Correct syntax of lit directives in some tests (#146886)
6faccd73128a merge main into amd-staging (#2961)
4a6d78e66c56 [MLIR][ODS] Improve diagnostic for using a property as an anchor
d0db199bcc0e [PowerPC] anyext_srl.ll - regenerate checks
ec752c67669f [LoongArch] Pre-commit tests for optimizing insert bitcast fp element
ea5ee2e74347 [mlir][OpenMP] Don't allow firstprivate for simd (#146734)
3099b7eb5da7 [Passes] Move LoopInterchange into optimization pipeline (#145503)
03cfba484fff [MLIR][IRDL][CMake] CMake fixes for cross-compilation (#145672)
043789519a11 [X86] combineShiftToPMULH - convert matching to use SDPatternMatch. NFC.
222e795347c1 [libclc] Fix target dependency
be63b2309a6a [OpenCL] Add decls for cl_intel_subgroup_local_block_io (#146656)
81e6552a3d68 [libclc] Make library output directories explicit (#146833)
1f8e2a4b1068 [libc++][NFC] Remove __remove_uncvref (#140531)
8f6a964c89c3 [libc++] Fix tests broken on the Buildkite CI (#146733)
a77446313367 [libc++][NFC] Fixed some wrongly spelled `_LIBCPP_STD_VER` in comments (#147008)
a34b1755e23c [DLCov] Origin-Tracking: Add debugify support (#143594)
de3c8410d87f [debuginfo][coro] Emit debug info labels for coroutine resume points (#141937)
2fcf1f8279b8 [MsDemangle] Use LLVM style RTTI for AST nodes (#143410)
d74c9ef8370c [lldb][test] Skip TestFrameVarDILGlobalVariableLookup.py on older AArch64 macOS version
0106585499c4 Revert "[flang][OpenMP] Allocate `reduction` init temps on the stack for GPUs (#146667)"
cb286da0011b merge main into amd-staging
c7d3b818963a [bazel] Port 0ceb0c377a3e67b70965c32e7f21fe8b33f555d2
a89e232058a2 [lldb][DataFormatter] Format libstdc++ unique_ptr like we do libc++ (#146909)
25f05c02afb9 [llvm][release] On release page, explain package types and verification (#138144)
6181a0689736 [NFC][LoopFuse] Regenerate LoopFusion tests using UTC (#146902)
7e3e2e1b8c6f MCParser: Add SMLoc to expressions
7ee2c72a8e5b [AArch64] Mark aarch64_set_fpmr as IntrWriteMem (#146353)
d84df61c001a [flang] Fix x86 REQUIRES in a couple of tests (#146869)
3e78afff0d8e [flang] Fix Windows bot failure caused by #146667 (#147002)
1d5d1256487c [ConstantFolding] Consolidate poison propagation for intrinsics (#146878)
25bf90eaede4 [MLIR][Docs] Fix typo in addTypeAttributeConversion comment (#146977)
06c988cefd91 [RISCV][VLOPT] Add support for vwsll.vx/vv. (#146998)
5a9e6babd81f [clang-tidy] fix false negatives with type aliases in `cppcoreguidlines-pro-bounds-pointer-arithmetic` check (#139430)
59e812f2eed8 [lldb][test] Skip pointer to std::unordered_map formatter tests on older Clang versions
f724d31a37ee [RISCV] Trim line to 80 chars in RISCVInstrInfoXAndes.td. NFC.
671870be9132 [RISCV] Fold funct7 into class for XAndesVBFHCvt instructions. NFC.
1194532bbae6 merge main into amd-staging (#2959)
d90e4b1699b6 merge main into amd-staging (#2958)
a550fef9061f [llvm] Use llvm::fill instead of std::fill(NFC) (#146911)
872eac7af005 [RISCV] Move vendor clang intrinsics tests to seperate directory. NFC. (#146862)
0612c4bbd336 [Driver][Test] pgo-sample-use-profi.c: Specify targets to fix test (#146996)
8fe205d31695 [Serialization] Remove an unnecessary cast (NFC) (#146986)
afd20aaca5fd [clang-scan-deps] Fix "unterminated conditional directive" bug (#146645)
85aaaf6e7409 [clang-tidy] EndSourceFile() for preprocessor before diagnostic client (#145784)
8c9e0c6c61f6 [flang][OpenMP] Allocate `reduction` init temps on the stack for GPUs (#146667)
ddcccc4afaff [OpenCL] Add decls for cl_intel_subgroups_char, cl_intel_subgroups_long (#146655)
724dfdc948a8 [OpenCL] Add declarations for version 1.1 of cl_intel_subgroups_short (#146647)
038431b2ba10 merge main into amd-staging
61529d9e36fa [RISCV] Remove implied extension Zvfhmin for XAndesVPackFPH (#146861)
5b8304d6b90c Revert "[AMDGPU] Re-apply: Implement vop3p complex pattern optmization for gisel" (#146982)
48e89a8cf4b8 [mlir][transform] Fix Wtype-limits warning (#146898)
db03c2776365 [AMDGPU] Re-apply: Implement vop3p complex pattern optmization for gisel (#136262)
965b94428c1b [TableGen] Use emplace_back and const correct two reference arguments. NFC
0db0c79ab45f [mlir] Use range-based for loops (NFC) (#146946)
60d1c4e257ed [llvm] Use range-based for loops (NFC) (#146945)
c3947a63690d [AMDGPU][NFC] Remove a redundant test (#146888)
a60d6c808b46 merge main into amd-staging
ba9c262a9711 [mlir] Use llvm::any_of and llvm::all_of (NFC) (#146947)
7e04bfbf18c9 Revert "[lldb][RPC] Upstream lldb-rpc-gen tool" (#146969)
0a02c28e2912 [clang-doc] add namespace references to VarInfo (#146964)
96f7f22825d0 [mlir][examples] Use {} instead of std::nullopt (NFC) (#146949)
2910c24638fc [Modules] Record side effect info in EvaluatedStmt  (#146468)
9bfb347ea0a0 [lldb][RPC] Upstream lldb-rpc-gen tool (#138031)
4482bc19a56b merge main into amd-staging (#2957)
45083dc4d224 [lldb][framework] Correctly place framework files in framework with script (#146425)
1f7effc88774 [mlir][acc][flang] Use SymbolRefAttr for func_name in ACC routine (#146951)
0ceb0c377a3e [NFC][HLSL][DirectX] Let `HLSLRootSignature` reuse the `dxbc` defined enums (#145986)
1725cc025e8f [lldb] Remove unused `hardware` argument (NFC)
2b7e3f6fa6fb [flang] Unify derived types in distinct module files (#146759)
dd3214d5a6b5 [flang] Fix handling of identifier in column 1 of free form continuat… (#146430)
0b98b27930da Use the root directory as the SDK root on POSIX platforms
17cb0a73a054 [lldb][docs] Fix another broken markdown link
7007a50a62ba merge main into amd-staging (#2956)
eda3161c3565 AMDGPU: Implement tensor load and store instructions for gfx1250 (#146636)
1995fd9ac6be [lldb] Extract DW_OP_deref evaluation code (NFC) (#146801)
2aa0f0a3bd54 [AArch64] Add option -msve-streaming-vector-bits= . (#144611)
3c13257f32f5 [RISCV] Rename XTHeadVdot instructions to match their mnemonic. NFC (#146953)
d0d84c4150a3 [RISCV] Add SF_ to SiFive instructions in RISCVGenInstrInfo.inc. NFC (#146939)
e35cf02e54c7 [RISCV] Pass RISCVSubtarget to translateSetCCForBranch. NFC
ec350657893d [ARM] Add neon vector support for rint
e18c5dec6dc3 [X86/AArch64] Autogen two additional tests
6efa3dfb7b21 [VPlan] Handle interleave groups with trivially narrow operands.
bc89380179c4 [flang][preprocessor] fix use of bitwise-and for logical-and (#146758)
67d6679c91e1 [flang][prescanner] fix invalid check (#146613)
08ed9e17bf87 [LV] Add tests for narrowing interleave groups with single scalar ops.
be762b7b7d60 [RISCV] Efficiently lower (select cond, u, rot[r/l](u, rot.amt)) using zicond extension (#143768)
e4fb69d4e4ab merge main into amd-staging
ea88634764f8 [Support] Remove an unnecessary cast (NFC) (#146810)
0afbf1721346 [LAA,LV] Add tests with early-exit loops and deref assumptions.
48ef55ce3e85 [CGP] Update tests to use autogen scripts, and refresh check lines
a3dea0addd0e [gn build] Port 795b17d0b821
bfd457588af9 Revert "[AMDGPU] SelectionDAG divergence tracking should take into account Target divergency. (#144947)"
378f248934d6 [lldb] Add SB API to make a breakpoint a hardware breakpoint (#146602)
1e76f012db3c Fix the build after 8ac7210b7f0ad49ae7809bf6a9faf2f7433384b0 made TTI mandatory
994501abe75c [Clang] Fix evaluation context of lambdas appearing in discarded statements (#146857)
6cadd3524dc3 merge main into amd-staging
1e541c1c8904 merge main into amd-staging (#2955)
d66254e7fe21 Revert " Move python binding tests to lit framework (#146844)" (#146931)
a0e846277023 [compiler-rt][sanitizer_common] Improve handling of env vars for iOS simulator tests (#146721)
53baee94cce6 [RISCV] Prune unnecessary LMUL from SiFive sf.vfwmacc.4x4x4 pseudo instruction. NFC
8d3dbdb00f8f [lldb] fix incorrect logging message (#146903)
2f75f658b18a [lldb] Take a sledgehammer approach to resizing the statusline (#146578)
795b17d0b821 [Frontend][OpenMP] Implement directive name parser (#146776)
1aa39690aa73 [CIR] Upstream UnaryDeref support for ComplexType (#146757)
a695d6b9a726 [HashRecognize] Check TC against bitwidth of LHSAux (#144881)
77f0f812b419 [Sema][ObjC] Treat unknown selector messages as unrecoverable errors under ARC (#146803)
4cd9c894d1ef [DebugInfo] Remove an unnecessary cast (NFC) (#146809)
8ac7210b7f0a [AMDGPU] SelectionDAG divergence tracking should take into account Target divergency. (#144947)
b6e113a149bf Drop timestamp in generated source code (#146822)
e8f491fc973e [mlir] Remove unused includes (NFC) (#146812)
61a0653cc610 [VPlan] Fix first-order splices without header mask not using EVL (#146672)
21c4fbd4235a [RISCV][VLOPT] Pre-commit test for adding support for vfrec7.v
e93a34662f14 [CIR] Upstream SubstNonTypeTemplateParmExpr support for ComplexType (#146755)
59d641a2dab4 [CIR] Upstream support for SubstNonTypeTemplateParmExpr (#146751)
a24490792235 [clang] Use range-based for loops (NFC) (#146811)
5df748dd329e [Sema] Remove an unnecessary cast (NFC) (#146808)
0f717044ff1a [X86] lowerX86FPLogicOp - use MVT::changeVectorElementTypeToInteger(). NFC.
46b4ca8e9869 [NFC][PowerPC] Pre-commit testcases for locking down the xxsel instructions for ternary patterns (#146764)
6c23e9e82c36 [RISCV] Fix crash when trying to remove segment (#146524)
4b9f622ca94d [CIR] Bit manipulation builtin functions (#146529)
f019c890087a [X86] foldXorTruncShiftIntoCmp - pull out repeated SDLoc. NFC.
51ff8f2f7e4c [X86] foldXor1SetCC - pull out repeated SDLoc. NFC.
a282c6858067 [X86] combineX86AddSub - pull out repeated getOperand() call. NFC.
23104a74e79f [CIR] Add nothrow attribute to the call operation (#145178)
3ba161bb7530 [NFC][LLVM][CodeGen][SVE] Add merging unary operation tests.
2b49d36c081f [flang][cmake] Separate FLANG_INCLUDE_TOOLS from FLANG_BUILD_TOOLS (#145005)
a88e286aefbf [lldb][test] Consolidate libstdc++ and libc++ vector formatter tests into generic test (#146885)
34f124b06ffd [mlir][toy] Use `make_early_inc_range` when erasing ops during iteration (#146892)
522d743545e6 [lldb][test] Run generic set formatter test-case on specified STL (#146882)
6440b1028220 [lldb] Remove PRIuFT macro and use LLDB_LOG instead (#146867)
10dc4e122ace [lldb][test] Turn std::chrono libcxx test generic (#146873)
b0444b0473c0 [lldb][test] Turn std::atomic libcxx test generic (#146843)
31bdd5fd39cb [lldb][test] XFAIL TestIRInterpreter.py on Windows again
03aa5c0a9e71 [LLDB] Optimize identifier lookup in DIL (#146094)
f2ccda7e8972 Revert "[mlir][vector][nfc] Rename `populateVectorTransferCollapseInnerMostContiguousDimsPatterns` (#145228)"
438863a09e2d [OpenACC][Sema] Implement warning for 'cache' invalid var ref
d4e57c6d45e5 Revert "[mlgo][regalloc] Fix after PR #131837" (#146897)
c71bbd50a10d [analyzer] Correct Z3 test cases, fix exposed crashes (#146597)
4f047bc5955c  [DLCov] Origin-Tracking: Collect stack traces in DebugLoc (#146678)
30eb97c5848d [X86] commuteSelect - update to use SDPatternMatch. NFC. (#146868)
23216b4af5ba [mlir][SCF][NFC] Remove duplicate `getConstantTripCount` implementation (#146347)
d805707ee435 [clang][test] Correct UNSUPPORTED syntax in print-ranges.cpp
8171f474d642 [llvm][dsymutil][test] Fix REQUIRES syntax
651c994febea [llvm][test] Fix REQUIRES in extractelement-insertpoint.ll
59731f5cfac6 [llvm][llc][test] Fix REQUIRES in regalloc pipeline test
57b55b6401e3 merge main into amd-staging
b7059ebafe8a [LSR] Strip dead code (NFC) (#146109)
0a656d8e57c9 [Bitcode] Add abbreviations for additional instructions (#146825)
791bb606b5bd [X86] fp128-select.ll - remove CFI noise
c5f47c6fd2cd [lldb-dap] Prevent using an implicit `step-in`. (#143644)
8763ac3252f2 [flang] fix skip-external-rtti-definition for ppc (#146826)
4868d66282b2 [flang] improve DITypeAttr caching with recursive derived types (#146543)
b5f5a7699c1d [OpenMP] Make headers parse standalone. NFC
2638fa1be63c [ORC] Add cloneToContext: Clone Module to a given ThreadSafeContext (#146852)
2532bde03889 [clang][python][test] Move python binding tests to lit framework (#146844)
76274eb2b343 [PHIElimination] Revert #131837 #146320 #146337 (#146850)
51485ab46d33 [mlir][bufferization] Do not access operation after it was replaced (#146860)
f36ad98b2740 [RISCV] Remove unneeded blank line between the features for XAndesVBFHCvt and XAndesVPackFPH. NFC.
a2505cf1e3f2 [LangRef] Revisit attributes semantics after opaque ptr migration (NFC)
5f62c791061d [C++20] [Modules] Use current named module to do module local lookup
ca0b56639cf9 [NFC] [C++] [Modules] Mark P1979 as implemented and add test (#146841)
ec25a0568cfb [VPlan] Don't convert VPWidenSelectRecipes to vp.select in EVL transform (#146695)
789127036dc9 [lldb][Windows] Fix compilation warnings
d8023e7cf8fe [lldb] Use correct size when dumping DWARF64 DW_FORM_ref_addr (#146686)
776300235778 [MLIR/Utils] Add missing dep on Arith dialect (#146834)
4e8843ee870e [lldb][test] Disable TestIRInterpreter.py on x86_64 Windows
da6c7e6dd2c2 [lldb][mcp] Skip MCPUnixSocketCommandTestCase if remote (#146807)
41fd13c63509 [ORC] Fix file header comment formatting. NFC.
cbc2ac5db8ea [WebAssembly] Fold TargetGlobalAddress with added offset (#145829)
a17c598145b0 [lldb][test] Enable TestExpressionInSyscall.py on Windows
c79fcfee4125 [DAG] combineVSelectWithAllOnesOrZeros - reusing existing VT. NFC.
dd495f16f7a0 [lldb][test] Correct TestIRInterpreter
563bea91222f [lldb][test] Enable TestIRInterpreter on Windows
d56c06e6c9f0 [flang][debug] Generate DISubprogramAttr for omp::TargetOp. (#146532)
4017dc06e3b5 [lldb][Commands][NFC] image lookup: remove unused local variable (#146554)
58d84a615e91 [lldb][DWARF][NFC] Reduce scope of ref_addr_size variable (#146557)
94d823157827 [LLDB] Mark TestCallStdStringFunction as XPASS on Windows (#146835)
431507db52c8 [libcxxabi][cmake] Account for LIBCXXABI_TARGET_SUBDIR in test config (#138527)
44031237284e Revert "[win][aarch64] Always reserve frame pointers for Arm64 Windows" (#146836)
6a50aa77821d [lldb] Add linux test for the three-arg version of getProcFile (#146500)
032966ff5679 [RISCV] Added the MIPS prefetch extensions for MIPS RV64 P8700. (#145647)
bd6cd92984e7 [MLIR][OpenMP][NFC] Use header guards for tblgen'd definitions (#146684)
2dba752f0575 [RISCV] Use template version isInt<N> instead. NFC.
fcdb91e11346 [VPlan] Remove redundant debug location setting in VPInterleaveRecipe::execute. nfc (#146670)
b478c38c19f9 MCAsmBackend: Replace FKF_IsPCRel with isPCRel()
4acdb8e14ef7 [VectorCombine] Scalarize extracts of ZExt if profitable. (#142976)
3efa461d45a1 [libcxx] Avoid hash key in __hash_table::find() if it is empty. (#126837)
3e370452fd23 [VPlan] Early assert for unsupported interleaved access features. nfc (#146669)
85d09de5fa19 [libclc] Add prepare-<triple> targets (#146700)
0bfa0bcd79f7 [ORC] Replace ThreadSafeContext::getContext with withContextDo. (#146819)
9234d077522e MCAssembler: Optimize PCRel fixups
6504c96b1d86 [clang][analyzer] Fix the false positive ArgInitializedness warning on unnamed bit-field (#145066)
1f8f477bd038 [ARM] Add neon vector support for trunc
65537539e129 [llvm][ADT] Add wrappers to `std::fill` (#146681)
b59763a7db7f MCAsmBackend: Simplify shouldForceRelocation overrides
dd2891535d4b MCAsmBackend: Merge addReloc into applyFixup (#146820)
119705e5ad2b [lldb][test] Synchronize `__compressed_pair_padding` with libc++ (#142516)
d67013a2b442 [Flang][AArch64][VecLib] Add libmvec support for Flang/AArch64 (#146453)
31e85f987d58 MCAsmBackend: Make some target overrides out-of-line
283f53ac6fcd [RISCV] Add isel patterns for generating XAndesPerf branch immediate instructions (#145147)
cd65d4e3c9e2 merge main into amd-staging (#2952)
e481c3a1f0bf [openmp] - Add openmp_INCLUDE_DIR to openmp-config.cmake (#2950)
4cf53cd266b9 [msan] Fix "Add optional flag to improve instrumentation of disjoint OR (#145990)" (#146799)
922dde3c6403 MCAssembler: Simplify fixup handling
c9babbc2065d Pre-commit PhaseOrdering/always-inline-alloca-promotion.ll
a25887077bd5 ELFObjectWriter: Optimize isInSymtab
9382a95e654e [NFC][TableGen] Adopt `ArrayRef::consume_front()` in `PrintMessage` (#146775)
4f1ba5a06ed6 [RISCV] Add additional gather/scatter addressing coverage
797d519d9915 [RISCV] Check whether plain type is supported for permutation intrinsics instead of its float type (#146657)
cc801b6570df [clang] [modules] Add err_main_in_named_module (#146635)
13fddeaf2c49 [BPF] Emit proper error message for insns with tied operands (#146778)
854e951365a5 Revert "[mlir][OpenMP] implement SIMD reduction (#146671)"
220a00239696 [SDAG] Prefer scalar for prefix of vector GEP expansion (#146719)
44bed1af0fb6 [RISCV] Add negative pre-defined macro test for XAndesVBFHCvt
e718ce00374d Revert "[flang][cuda] Do not create global for derived-type with allocatable device components (#146780)"
a5350785db62 Revert "[flang][cuda] Allocate derived-type with CUDA componement in managed memory (#146797)"
eaac126c9fe9 Port ZCFS improvements (#2829)
a068ed288af1 [lldb][mcp] Fix unix domain socket protocol server addresses (#146603)
f01017ca9711 [docs] Refresh Developer Policy text (#136198)
4e26801ea91f clang: Add requires webassembly to a test
5e31d4cdf2f5 [RISCV][VLOPT] Support v[f]slide1up.v{x,f} (#146716)
717899ce8664 [CIR] Upstream get_bitfield operation to load bit-field members from structs (#145971)
925588cd001a [flang][cuda] Allocate derived-type with CUDA componement in managed memory (#146797)
e873ce31ae0e [flang][cuda] Do not create global for derived-type with allocatable device components (#146780)
0d7e64f5d2b4 [ASan][Windows] Honor asan config flags on windows when set through the user function (#122990)
6550f28977b9 [RISCV][VLOPT] Support vslide{up,down} (#146710)
c6abab2875ef [RISCV][VLOPT] Add support for vfmerge.vfm and vfmv.v.f (#146692)
6ab7e52dd80d WebAssembly: Move validation of EH flags to TargetMachine construct time (#146634)
8dbf6a42c84d merge main into amd-staging (#2948)
d4331344ac45 [Clang][Driver][SamplePGO] Introduce -fno_sample_profile_use_profi flag for SamplePGO (#145957)
c863e0ebc2e5 Revert "Move python binding tests to lit framework (#146486)" (#146789)
c2347170f40d Reapply "[Clang,debuginfo] added vtt parameter in destructor DISubroutineType (#130674)" (#145697)
a74c7d877637 [win][aarch64] Always reserve frame pointers for Arm64 Windows (#146582)
551d6ddaa381 [clang] [test] Add a missing requirement for a test
497060fae5c3 [lld][WebAssembly] Add missing relocation types to the --compress-relocations path (#144578)
272576543246 [LLVM][ADT] Add `consume_front` and `consume_back` to ArrayRef (#146741)
7fc50e92a59c clang: Fix parsing of seh exception model (#146643)
734d31a464e2 clang-format ELF.cpp
6ff3b4370017 [TableGen] More generically handle tied source operands in CompressInstEmitter. (#146183)
8eb61d9154c5 [SHT_LLVM_BB_ADDR_MAP] Cleanup old version support code from ELF.cpp.
274152c5fa9f [mlir][vector][spirv] Lower `vector.to_elements` to SPIR-V (#146618)
a880c8e670be [NFC][TableGen] Add accessors for various instruction subclasses (#146615)
d45762187252 [lldb] Fixing warnings / win32 builds in MainLoop. (#146632)
ae9990ed9651 [libc++] Remove dead code from <type_traits> (#143854)
3ee0f97b950a [libc++] Avoid including vector in <functional> (#144310)
b173c12b5454 [libc++] Make libcxx/selftest a top-level test directory (#144852)
954810d27b85 merge main into amd-staging
0a822f82de91 [PowerPC] Fix ppc-reduce-cr-ops mishandling of subregister uses (#144405)
1f7ba23422da [NFC] [MSAN] replace (void) with [[maybe_unused]] (#146617)
81ba1d421ec7 offload fixes for Name -> getName
6ecb6a8a8cd5 [mlir][vector][nfc] Rename `populateVectorTransferCollapseInnerMostContiguousDimsPatterns` (#145228)
ab0fa6c5dc72 [lldb] Fix else-after-return in Breakpoint (NFC)
bec6a124bb79 merge main into amd-staging
c03acc899e6c Regen 3 lit tests
a63f57262898 [mlir][bufferization] Return BufferLikeType in BufferizableOpInterface (#144867)
2ed7b22f0c9e [HLSL] Add `-Gis` option to clang-dxc (#146448)
218fd692617b [BOLT] Decouple new segment creation from PHDR rewrite. NFCI (#146111)
da01257c3a16 [Test] Account for spaces in paths in the new dtlto/files.test (#146749)
e3edc1bd873f asan: refactor new/delete interceptor macros (#146696)
00e071d69062 [lldb] remove do-nothing defaults in case statements, unbreak gcc CI bots.
77d95911a3a4 Revert "Fix wcpncpy() return value; add test." (#146752)
988876cdd918 Fix wcpncpy() return value; add test.
dfc5987faccb [bazel] Add missing dep after 242996efee4aaa30dd0ddb4359675b8bc103294a
832cd2df0b56 [SPIRV] Add option to add all KHR extensions (#145535)
5a3d107252b9 [DAG] Refactor X86 combineVSelectWithAllOnesOrZeros fold into a generic DAG Combine (#145298)
358348b7922e merge main into amd-staging
a4d517dc3808 [SystemZ][z/OS] Fix error about const char in Text Encoding (#146727)
5e6e51bbbec4 [Sema] Remove an unnecessary cast (NFC) (#146703)
aa8e1bc0e989 [X86] Add BLEND/UNPCK shuffles to canCreateUndefOrPoisonForTargetNode/isGuaranteedNotToBeUndefOrPoisonForTargetNode (#146728)
bf016b930b5c [mlir][spirv] Add support for SPV_ARM_tensors (#144667)
1626867ccdf4 [lldb] Fix warnings
6b623a662270 [SHT_LLVM_BB_ADDR_MAP] Remove support for versions 1 and 0 (SHT_LLVM_BB_ADDR_MAP_V0). (#146186)
cbfc10260cfe Fix lld crash caused by dynamic bit offset patch (#146701)
50f40a5327ad [libc] Fix internal alignment in allcoator (#146738)
e9be5286e14e [CIR] Implement NotEqualOp for ComplexType (#146129)
6a97b56ce5c5 [MLIR][AMDGPU] Redirect transfer read to masked load lowering (#146705)
dfcef35ff1d3 [lldb][NFC][MachO] Clean up LC_THREAD reading code, remove i386 corefile (#146480)
ed27f18e324d __sys builtin support for AArch64 (#146456)
de4f283d141c [Xtensa] Fix XtensaELFObjectWriter. (#146731)
641ad52b6ab4 [AMDGPU][MC] Fix disassembly for v_permlane16_swap_b32 for GFX950 (#146600)
2fe0feb84875 [LLVM][AsmWriter] Print a comment for unknown intrinsics (#146726)
b62826cc054c [InstrEmitter] Use AddOperand in EmitCopyToRegClassNode. (#146637)
191583c6a550 [mlir] Remove unused includes (NFC) (#146709)
bbf4402635c5 [Analysis] Remove a redundant control flow statement (NFC) (#146708)
0f1accd80642 [clangd] Drop const from a return type (NFC) (#146707)
4461de2e5139 [StaticAnalyzer] Remove unnecessary casts (NFC) (#146706)
e717e503cae1 [AMDGPU] Fix comment on DelayInfo::advance (#146718)
3c0571a74971 [Offload] Add missing license header to Common.td (#146737)
338fd8b12ce6 [SimplifyCFG] Transform switch to select when common bits uniquely identify one case (#145233)
68173c809159 [HLSL][SPRIV] Handle signed RWBuffer correctly (#144774)
1c305f711f86 [HashRecognize][NFC] Add a missing "phi" in a comment (#146702)
7e1dc7e8d4b7 [nfc] Rename API with typo in BranchProbabilityInfo (#146447)
16b75c819dde [mlir][OpenMP] implement SIMD reduction (#146671)
7931a8f10298 [VectorCombine] Scalarize vector intrinsics with scalar arguments (#146530)
05ebb363c901 [libc++] Re-add a script to trigger the libc++ Buildkite pipeline (#146730)
7d52b0983e0b [Offload] Add `MAX_WORK_GROUP_SIZE` device info query (#143718)
ab42c4abe86c [NFC] Replace "omp.default.mapper" with llvm::omp::OmpDefaultMapperName.
1841b021c6bf [CIR] Add OptInfo attribute (#146261)
ce99db9d1b8a [CIR] Add mlir-translate to CMake (#146608)
d04d2557255a [Clang] Properly use `CommaJoined` for `--offload-arch` (#146687)
d74d4ffa3342 [Bitcode] Extract common BitCodeAbbrevOps (NFC)
b1774222c761 [flang] Emit `fir.global` in the global address space (#146653)
3b4e79398de5 [DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO (#142757)
e32439249d8a [lldb][test] Consolidate generic and libcxx std::deque formatter tests (#146697)
3cb28e9dee41 [CIR] Untie Type and Attribute definitions (#146663)
242996efee4a [Clang][DLCov][NFCish] Fix debugloc coverage tracking macro in Clang (#146521)
03919ef8d9d9 [Clang][OpenCL] Declare cl_amd_media_ops/cl_amd_media_ops2 builtins with -fdeclare-opencl-builtins (#143507)
4db8ce725138 [clang-fuzzer] Fix latent race condition in build (#146119)
e288561e6ba4 [CIR] Clean up FPAttr (#146662)
3dc09fbf29d5 [Xtensa] Implement THREADPTR and DFPAccel Xtensa Options. (#145543)
2dc44b3a7ba1 [InstCombine] Fix multi-use handling for multi-GEP rewrite (#146689)
ad715beca184 [AMDGPU] Remove HasSampler variable. NFC. (#146682)
5a8d096ae344 [clang-tidy] Fix false positive for cppcoreguidelines-pro-bounds-pointer-arithmetic (#127394)
cbfd0d68ea50 [HashRecognize] Rewrite arePHIsIntertwined (#144878)
8dcdc0ff1f41 [CIR] Clean up IntAttr (#146661)
38ad6b1983e3 [mlir][Affine] Fix crash in affine-loop-fusion pass by guarding against an empty op list (#144841)
5491576a1623 [CIR] Introduce IntTypeInterface to allow uniform integer types handling (#146660)
24828c8c45d7 [libc] Efficiently implement `aligned_alloc` for AMDGPU (#146585)
3f386876ec16 merge main into amd-staging (#2940)
bca79ec0d21e [mlir][linalg] Use `ub.poison` in linalg vectorizer instead of `0` for some transfer ops (#146544)
ebcf7f91ffa0 [SystemZ][HLASM] Emit END instruction (#146110)
bbcebec3af5a [DAG] Refactor X86 combineVSelectWithAllOnesOrZeros fold into a generic DAG Combine (#145298)
d78036f06ea9 Only restart failed libc++ jobs, not cancelled ones. (#146397)
8b3cc4d1ed06 [llvm][docs] Remove LLDB DWARF64 release note
a9ee1797b716 Remove helper function and use target agnostic needConversion function (#146680)
fc00256b2b4d [lldb][test][NFC] Rename libcxx unordered_map tests to unordered_map-iterator
2b03efc7fba8 [AMDGPU] Use isImage. NFC. (#146677)
dbe441e71611 X86: Avoid some uses of getPointerTy (#146306)
4f02965ae246 [Offload] Store kernel name in GenericKernelTy (#142799)
e63de82d904c [LLD][COFF] Disallow importing DllMain from import libraries (#146610)
acb52a8a98f0 [Offload] Improve liboffload documentation (#142403)
4e213159af9e [SPIRV] Add FloatControl2 capability (#144371)
9c0743fbc5ba [GlobalISel] Allow expansion of urem by constant in prelegalizer (#145914)
0aafeb8ba1b1 Reland [TSan] Clarify and enforce shadow end alignment  (#146676)
c0e9084b1ce1 [AMDGPU] Add a debug option `-amdgpu-snop-padding` for `GCNHazardRecognizer` (#146587)
9eac5f72f629 Revert "[TSan] Clarify and enforce shadow end alignment" (#146674)
6ec9b1b36672 [MLIR] Remove spurious space when printing `prop-dict` (#145962)
f575b18fdc83 [LV] Add support for partial reductions without a binary op (#133922)
dea4f3213d84 [libc] Use is aligned builtin instead of ptrtoint (#146402)
5fe63ae9a3fb [lldb-dap] Fix flaky test TestDAP_server (#145231)
585b41c2ecc0 TargetOptions: Look up frame-pointer attribute once (#146639)
35626e97d886 [DLCov] Origin-Tracking: Enable collecting and symbolizing stack traces (#143591)
a2c9f7dbcc8b [Xtensa] Implement lowering SELECT_CC/BRCC for Xtensa FP Option. (#145544)
7cc8fe2a2cd4 [LLVM][AArch64] Relax SVE/SME codegen predicates. (#145322)
38200e94f142 [DAG] visitFREEZE - always allow freezing multiple operands (#145939)
4c7a706589ca [LLDB] Simplify libstdc++ string summaries (#146562)
40275a4ee312 [lldb][test] Add tests for formatting pointers to std::unordered_map
bc8dad1c7ef0 [VPlan] Emit VPVectorEndPointerRecipe for reverse interleave pointer adjustment (#144864)
6e1e89ee380e [SLP] Avoid -passes=instcombine stages in SLP tests (#146257)
7ead20db2842 [lldb] Use llvm::erase_if (NFC) (#146624)
82c0a53763bc [LoongArch] Pre-commit for optimizing insert extracted pair elements
1b7cbe1f871e [flang][OpenMP] Create unique reduction decls for different logical kinds (#146558)
651c5208f859 VPlanRecipes.cpp - fix "'llvm::VPExpressionRecipe::computeCost': not all control paths return a value" MSVC warning. NFC.
85bc8684173b [AArch64][TTI] Reduce cost for splatting whole first vector segment (SVE) (#145701)
a75587d2718f [clang][python][test] Move python binding tests to lit framework (#146486)
2c1900860c08 [lld][LoongArch] Support TLSDESC GD/LD to IE/LE (#123715)
f1cc0b607b03 [IR] Introduce `dead_on_return` attribute
d5608d675131 MC,test: Improve section group test
647aa6616fa5 [mlir][SPIRVToLLVM] Set valid insertion point after op erasure (#146551)
83272a484939 [InstCombine] Fold icmp of gep chain with base (#144065)
0588e8188c64 [Serialization] Use the SourceLocation::UIntTy instead of the raw type for the offset, NFC
6c9be27b526f [mlir][tensor] Fold identity `reshape` of 0d-tensors (#146375)
9262ac3ee4d5 Revert "ELFObjectWriter: Optimize isInSymtab"
c577bc6fea42 merge main into amd-staging
eac1a1d3a8db MCAssembler: Consistently place MCFragment parameter before MCFixup
85be60d358eb merge main into amd-staging (#2936)
b68e8f1de71a [mlir][linalg] Allow promotion to use the original subview size (#144334)
3c6cade485d4 MCObjectStreamer: De-virtualize emitInstToFragment
f4b938b7c0ec [TableGen] Use range-based for loops (NFC) (#146626)
b809d5e2ac8d [ProfileData] Use lambdas instead of std::bind (NFC) (#146625)
838b91d7f683 [clangd] Drop const from a return type (NFC) (#146623)
7b4dbb4f37d8 [Sema] Remove an unnecessary cast (NFC) (#146622)
2723a6d9928c [LLVM][Cygwin] Enable dynamic linking of libLLVM (#146440)
984c78f27d14 [clang][bytecode] Add back missing initialize call (#146589)
c9bfdae62002 [RISCV] Use uint64_t for Insn in getInstruction32 and getInstruction16. NFC (#146619)
75d12330ef05 merge main into amd-staging
f1a4bb62452d [RFC][NFC][AMDGPU] Remove explicit value assignments from `AMDGPU::GPUKind` (#146567)
36f4a9b9750f merge main into amd-staging (#2935)
a8a9a7f95a69 [WebAssembly] Fix inline assembly with vector types (#146574)
2a702cdc38f9 Driver: Avoid llvm::sys::path::append if resource directory absolute.
aa1d9a4c3104 [MLIR][Affine] Enhance simplifyAdd for AffineExpr mod (#146492)
eb07f0d4a933 [Analysis] Use range-based for loops (NFC) (#146466)
2599a9aeb543 [clang] [modules] Implement P3618R0: Allow attaching main to the global module (#146461)
3deed4211a46 [docs] Add clang release notes for LoongArch (#146481)
a87b27fd5161 [lldb] Fix the hardware breakpoint decorator (#146609)
7502af89fc0e clang: Forward exception_model flag for bitcode inputs (#146342)
b0e6faae0842 [libclc] Add missing clc_lgamma_r with generic address space pointer arg (#146495)
93fe52f19e99 [libclc] Add __clc_nan implementation with signed nancode argument (#146485)
c71ec37aebe6 merge main into amd-staging
2b16af8df2ee [Offload][cmake] Add GPU test job limit for AMDGPU buildbot cmake cache (#146611)
6b7e1b97f4bb [CI] Use Github Native Groups in monolithic-* scripts
e89458d3985c [lldb] Fix PipeTest name collision in unit tests
e9c9f8f3748a [WebAssembly] Fold any/alltrue (setcc x, 0, eq/ne) to [not] any/alltrue x (#144741)
4a8c1f7d127a [WebAssembly] [Backend] Wasm optimize illegal bitmask (#145627)
ae2104897c7d [SelectionDAG] Fix NaN regression in fma dag-combine. (#146592)
475cd8dfaf84 [NVPTX] Further cleanup call isel (#146411)
5ed852f7f728 [mlir][arith] Add `arith::ConstantIntOp` constructor (#144638)
863e17a5be77 [VPlan] Make Phi operand for VPReductionPHIRecipe optional (NFC).
f393211454a2 [Reland][IPO] Added attributor for identifying invariant loads (#146584)
d99b14623f43 AMDGPU: Implement tensor_save and tensor_stop for gfx1250 (#146590)
bcbc4407120c [VPlan] Add missing VPWidenSelectto VPRecipeWithIRFlags::classof (NFC).
42578e8586e6 [mlir][linalg] Use hasPureTensorSemantics in TransposeMatmul methods. (#146438)
b6ffe2cb88a2 merge main into amd-staging (#2931)
5d849d3a9085 [mlir][xegpu] Fix seg-fault caused by setting a null attribute (#146002)
829f2f2448f5 [VectorCombine] Mark function as changed if shuffle is created.
ac76e4d8a9ae [Serialization] Use SourceLocation::UIntTy for the offset type, NFC
c56c349d3946 [clang-tidy] Switch misc-confusable-identifiers check to a faster algorithm. (#130369)
a06117142632 [AsmParser] Remove unnecessary casts (NFC) (#146549)
650d0151c623 [clang] Improve getFileIDLocal binary search. (#146510)
777d6b5de90b [VectorCombine] Use InstSimplifyFolder to simplify instrs on creation. (#146350)
6b3d2b629cc3 [VPlan] Add VPExpressionRecipe, replacing extended reduction recipes. (#144281)
7da8ed8d339a Fix missing/outdated pass options in PassRegistry.def (#146160)
d48035942050 [mlir][sparse] Do not access operation after it was replaced (#146546)
119292c40b0f [IR2Vec] Add out-of-place arithmetic operators to Embedding class (#145118)
efe0deae3fc6 [NFC][IR2Vec] Increasing tolerance in `approximatelyEquals()` of `Embedding`  (#145117)
6fafa46f8c1b [CIR] Upstream PackIndexingExpr for ScalarExpr (#146239)
e11d354d64fb [CIR] Upstream GenericSelectionExpr (#146211)
67f1eb5c1702 [CIR] Comma Operator for ComplexType (#146204)
b54e02a40b3c [clang][bytecode] Check pointer data type for bitcast eligibility (#146552)
8dbfe83681ed [CIR] Fix mlir::ValueRange init from ArrayRef warning (#146577)
491c79f76396 [CIR] Implement ArraySubscript for ComplexType (#146283)
5035d20dcbce AMDGPU: Implement ds_atomic_async_barrier_arrive_b64/ds_atomic_barrier_arrive_rtn_b64 (#146409)
3edae0d5e5b6 [lldb] Use EL_REFRESH to refresh Editline
35b80031f43f [NFC] Formatting PassRegistry.def (#144139)
9aa80c32e9a5 merge main into amd-staging
c17e88f07e26 [Analysis] Remove a workaround for ancient GCC (NFC) (#146550)
e79e22c9af97 [AMDGPU] Remove an unnecessary cast (NFC) (#146548)
72a9c3cced89 [Analysis] Remove an unnecessary cast (NFC) (#146547)
98e6d5cd47d4 [lldb] Fix warnings
d79c284a2fef [flang] Fix a warning
a79404e50cf5 [lldb] Correct an assertion name.
c8450ea226ed AMDGPU: Remove the unnecessary needAlias argument in DS_Real_gfx12 (#146439)
21d5017a05d1 [AMDGPU] Refactor FLAT_Load_Pseudo class (#146539)
b697b801b13d [X86] freeze-vector.ll - fix cut+pasta typo in frozen build vector tests
a34a02481246 [AMDGPU][SIInsertWaitCnts] skip meta instructions early  (#145720)
13e6ea7b4c29 [bazel] Add missing dependency for 08cf6ae537852d39f93f76575fff62ea26e21ed1
9fe42c6a0adf Reland "[flang] add option to generate runtime type info as external (#145901)" (#2923)
01a6c0853314 [PHIElimination] Verify reappropriated COPY is of similar register class, update livevars. (#146337)
ad5522764f52 merge main into amd-staging (#2924)
ba116a8bed9a [flang][OpenMP] Split check-omp-structure.cpp into smaller files, NFC (#146359)
1a7b7e24bcc1 [lldb] Adding pipe support to lldb_private::MainLoopWindows. (#145621)
9d3875203294 Refactor sequential asserter and add unit tests  (#2910)
47e28d9cd1c0 [libc] wcscspn implementation (#146158)
04c614327c28 [InstCombine] Pull vector reverse through intrinsics (#146384)
4dab0b230060 [AMDGPU] Fix wrong reverse operations for `v_cmpx_le_u32` (#146398)
2b36406d5c95 [LLVM][AMDGPU] Remove compiler flag for enabling vectorization of i8s (#2912)
569ca0f69876 [mlir][async] Erase op later to preserve insertion point (#146516)
0c124be33f8d [lldb][NFC] Inline ResolveSDKPathFromDebugInfo in one of its call site (#146062)
69b69cbcb451 [flang][tco] Add -emit-final-mlir flag (#146533)
3cff3d882b02 [Offload] Add skeleton for offload conformance tests (#146391)
6b1c92cbcb76 [X86] legalize-sub-zero.ll - regenerate test checks
86077c41a789 [flang][OpenMP] Rewrite min/max with more than 2 arguments (#146423)
6e3465cd0f23 [AArch64] Fix ldp rename through a bundle (#146415)
5332534b9cb6 [ARM] Add neon vector support for ceil
cf9374933d2e [Modularize] Make `Location::operator bool` explicit
d7b8b65e239f [LLVM][TableGen][DecoderEmitter] Add wrapper struct for `bit_value_t` (#146248)
e99da2b7a95a [mlir] Remove unused includes (NFC) (#146467)
f4cdb89b47d7 [mlir] Remove unnecessary casts (NFC) (#146465)
7622bf9d1270 [IR] Remove an unnecessary cast (NFC) (#146464)
bb080107e401 [CodeGen] Remove unnecessary casts (NFC) (#146463)
11ecd4742b74 [LLDB] Update DIL to pass current 'frame var' tests. (#145055)
ace5108f373e feat(linalg): add a way to pass controlFn to `foldIntoPackUnpackPatterns` (#143685)
f9413e1754a5 [clang][test] Remove duplication from gcc toolchain test (NFC) (#146487)
42e779692027 [ARM] Add a comment about fixupImmediateBr updaing ImmBranches. NFC
bab9d4c2d78a [NFC][AMDGPU] Pre-commit a test case that shows wrong reverse operation is used for V_CMPX_LE_U32 (#146527)
a6592ddf4ed9 [AArch64] Mark neon.stN intrinsics as writeonly (#145289)
1fe993c25196 [clang][bytecode] Allocate operator new data as array (#146471)
8cfd9b88215a [MLIR] Make generic skip packing init operand when not used in DataLayoutPropagation (#146139)
08cf6ae53785 [mlir][memref] Add a new `ReifyResultShapes` pass (#145927)
3355cca9380e [NFC][AMDGPU] Auto generate check lines for some test cases (#146400)
bedd7ddb7fb5 [InstCombine] Fix use after free
1a253e213daf [NFC][Offload] Fix possible edge cases in offload-tblgen (#146511)
da0828b1e93c [lldb] Enable support for DWARF64 format handling (#145645)
857815f3fa26 [OpenACC][CIR] Implement 'rest' of update clause lowering (#146414)
e44fbea0a17f [FunctionAttrs] Handle ConstantRange overflow in memset initializes inference (#145739)
3702d64801c8 [mlir] Reapply 141423 mlir-query combinators plus fix (#146156)
771ee8e38785 [bazel] Add mising dependency for 698ec8c7ba6c47cfec6112d1c93b481fc9ac7973
72f87d2d69b2 [DAG] canCreateUndefOrPoison - remove isGuaranteedNotToBeUndefOrPoison check for insert/extract vector element indices (#146514)
524f0903064c [alpha.webkit.UncountedCallArgsChecker] Treat CFEqual as a safe function (#146369)
6ebe0912a763 xfail: clang/test/CodeGenCXX/bitfield-access-empty.cpp
6731f151ea15 [clang][bytecode] Remove unused InRange function (#146509)
d9d9ab869817 [clang][CodeComplete] skip explicit obj param in code completion string (#146258)
59eadf748e74 merge main into amd-staging
e5c7169bebc4 fixup! [offload][omptest] Add unit tests for sequential asserter
b8b749455143 [InstCombine] Rewrite multi-use GEPs when simplifying comparison (#146100)
84282ea562b8 Use `replaceWithAddrOfOrASCast` (amd-staging only util)
5d3c1dc43d9d Reland "[flang] add option to generate runtime type info as external (#145901)"
9b13dfdfbc6d [LV] Use vscale for tuning to improve branch weight estimates (#144733)
15ab4bb5c8a7 [Hexagon] Implement shouldConvertConstantLoadToIntImm (#146452)
67b79468fbba [RISCV] Factor out getKillRegState in copyPhysReg (NFC) (#146454)
89fe42926293 [DAG] canCreateUndefOrPoison - remove isGuaranteedNotToBeUndefOrPoison check for shift nodes (#146502)
56841565db77 [DAG] canCreateUndefOrPoison - add handling for CTTZ/CTLZ_ZERO_UNDEF nodes (#146501)
4a2fa0847fb0 [VPlan] Support VPWidenIntOrFpInductionRecipes with EVL tail folding (#144666)
ee372f5f5f5d fixup! [offload][omptest] Refactor asserters to enable testing
a6339d0e5854 [clang]Fix Handle structs exceeding 1EB size limit (#146032)
90e20d4f42c3 [Clang][Bytecode] Implement P1061 structured binding pack (#146474)
fd46e409a91f [X86] detectZextAbsDiff - use m_SpecificVectorElementVT matcher. NFC. (#146498)
c0e62dd982bc Apply llvm#127838: [flang][Lower][OpenMP] Don't read moldarg for static sized array (#2918)
37d30d9e219a [mlir][tblgen] Fix test definition names to reflect expected valid results (NFC) (#146243)
6d6b36439f38 [Clang][AArch64] Move definitions of FP8 Neon loads & stores (#146352)
f581ef5b6687 [mlir][gpu] Add gpu.rotate operation (#142796)
a97826a13b7c [DAG] canCreateUndefOrPoison - explicitly state the AssertSext/Zext/Align/NoFPClass can create poison. NFC. (#146493)
43d2486e27fd [mlir][tblgen] Fix region and successor references in custom directives (#146242)
4e30f8101e4b [DAG] visitFREEZE - remove isGuaranteedNotToBeUndefOrPoison assertion (#146490)
2ee884a9db29 [clang-doc] Remove the unused clangd header.
8fe25f30714c [PowerPC] Remove chenzheng1030 from active maintainers (#145519)
cd60247a0fca [llvm][docs] Document how to ask for things other than commit access (#146340)
5c08aeac85e5 [clang] Don't use raw source location in DeclarationName, NFC (#146412)
698ec8c7ba6c [mlir][tosa] Require signless types in validation and add corresponding conversion pass (#144367)
cd10ded69787 [Clang] Remove AArch64TargetInfo::setArchFeatures (#146107)
a6bb895efe47 [RISCV] Fix typo in header of llvm/test/CodeGen/RISCV/rvv/select-fp.ll. NFC.
fadd3f9661b7 merge main into amd-staging (#2917)
84c849e85b0f [InstCombine] Combine interleaved recurrences. (#143878)
102c22cb2c10 [FatLTO] Disable analysis verification in pipeline test (NFC)
5fa4eb1dfd8f [Clang] Verify data layout consistency (#144720)
cb806510914e [RISCV] Merge AllBFloatVectors into AllFloatVectors. NFC.
b5f4fd03c2cc [RISCV] Remove unused variable AllFloatVectorsExceptFP16. NFC.
67be4fe3d5fd Fix MSVC implicit double -> float truncation warning. NFC.
3cc200f46b9c DynamicAPInt: Support APInt constructor. (#146301)
65cb0eae58d2 [Flang][OpenMP] Add Semantics support for Nested OpenMPLoopConstructs (#145917)
372e332a57ed [X86] freeze-unary.ll - remove unnecessary freeze from CTLZ/CTTZ_ZERO_UNDEF tests
ab6316e21965 [Coroutines] Work on intrinsic IDs instead of names (NFCI) (#145518)
9beb467d9213 MC: Store fragment content and fixups out-of-line
7e830f76711f [Clang] Partially fix m68k alignments (#144740)
c5e4546a0119 [DwarfDebug] Slightly optimize computeKeyInstructions() (NFC) (#146357)
72b40f7c6437 [lldb] Delete unused DWARFDataExtractor methods (#146356)
54b801af0040 Apply llvm#127838: [flang][Lower][OpenMP] Don't read moldarg for static sized array
e066cc433226 merge main into amd-staging
84e82746c3ff [X86][llvm-exegesis] fix LNLP performance counter binding issue (#146348)
393aebf5c218 CodeView: Move MCCVDefRangeFragment storage to MCContext/MCFragment. NFC (#146462)
67b740bd73a6 [X86] Add diagnostic for fp128 inline assemble for 32-bit (#146458)
e94c6091c93a [lldb][Mach-O] Fix several bugs in x86_64 Mach-O corefile (#146460)
bbe4357fd0ca merge main into amd-staging (#2915)
ce159d20e52a [RISCV] Put REQUIRES: riscv-registered-target in the first line of the file. NFC.
393a75ebb735 [mlir][Vector] Add constant folding for vector.from_elements operation (#145849)
0a69c83421ac [NFC][IR2Vec] Remove unreachable code and simplify invalid mode test (#146459)
66cc167dfa5b [LoongArch] Add tests for inserting extracted integer elements. NFC
8cf3b989a933 [Analysis] Fix a warning
a2dc64cd0332 [NFC][MLGO] Convert notes to proper RST note directives in MLGO.rst (#146450)
943804881689 [IR2Vec] Simplifying creation of Embedder (#143999)
24c4bba076b5 [bazel] Quick fix for #145584 (Core deps ProfileData/InstrProf.h)
6323541a2a56 [LLD][ELF] Skip non-SHF_ALLOC sections when checking max VA and max VA difference in relaxOnce() (#145863)
6620db50fef7 [NFC][MLGO] Fix indentation in MLGO.rst (#146444)
70db191efa47 [RISCV] Add LLVM IR intrinsicis vrgatherei16/vslideup/vslidedown support for Zvfbfmin (#146312)
4e1a97870498 [flang][acc] Fix dialect dependencies (#146431)
6a929ffafff9 merge main into amd-staging
a4f637d988bd [MLGO] Fix feature iteration using FeatureMap.size() instead of NumberOfFeatures (#146436)
8d9cdb65f090 [Clang][LoongArch] Fixed incorrect _BitInt(N>64) alignment (#145297)
ad7d6759913b [BOLT] Refactor mapCodeSections(). NFC (#146434)
a3c816542147 [flang][acc] Fix dialect dependencies (#146431)
d2500e639b64 [pgo] add means to specify "unknown" MD_prof (#145578)
d3cc2598aabb merge main into amd-staging (#2911)
12409a18f609 [DirectX] Implement `memcpy` in DXIL CBuffer Access pass  (#144436)
92b50959da32 [NFC][TableGen] Capitalize `to` in `UseFnTableInDecodetoMCInst`. (#146419)
f8d4d5384378 Remove compiler flag for enabling vectorization of i8s
b5ef184f506a merge main into amd-staging (#2905)
ff4faaa660e1 [mlir][examples] Fix deprecation warnings for `ArrayRef(std::nullopt)` (#146420)
f16983f7d0f8 [flang][acc] Ensure fir.class is handled in type categorization (#146174)
6896d8a05dab [RISCV][MC] Support Base P non-GPR pair instructions (#137927)
59a7185dd9d6 [VPlan] Truncate/Extend ComputeReductionResult at construction (NFC). (#141860)
d599bdeaa49d [AMDGPU] Autogen checks for test (#146421)
1eb795413df3 [lldb] Correctly restore the cursor column after resizing the statusline (#146132)
0d1392e979ba [MachineOutliner] Remove LOHs from outlined candidates (#143617)
f4cecfe1bbdd [flang][cuda] Bring PARAMETER arrays into the GPU module (#146416)
56739f586653 [Analysis] Fix a warning
0745eb501df3 [IR2Vec] Scale embeddings once in vocab analysis instead of repetitive scaling (#143986)
56ef00a59d7e [X86][GlobalISel] Fix RegBank issue for G_FABS (#145674)
eb904e857af1 [lldb] Fix link syntax in docs/use/mcp.md
dcf8ec921863 Reland "[MLGO][IR2Vec] Integrating IR2Vec with MLInliner (#143479)" (#145664)
026aae704799 [VPlan] Infer reduction result types w/o accessing underlying phis.(NFC)
f205e354ae3e [CIR] Upstream GenericSelectionExpr for ComplexType (#146265)
81c9a1a4e510 [libc][obvious] Fixed typos in some wchar headers (#146413)
4c7d3e9315a5 [STLForwardCompat] Implement llvm::type_identity (#146390)
95409b376ea4 [offload][omptest] Add unit tests for sequential asserter
4bf0c6b5f8e2 [libc][bazel] Added wchar functions to bazel (#146126)
163a7e1b4fad [mlir][LLVMIR][NFC] Remove duplicate getUnnamedAddrAttrName uses in op printers (#146090)
93849a39c432 [flang] Check for ultimate ALLOCATABLE component in LOCAL_INIT() (#145800)
16813de178b4 merge main into amd-staging
7da3f0d9a93d [offload][omptest] Refactor asserters to enable testing
a68e4470c16b [clang-doc] serialize friends (#146165)
96b9b2e21de4 [Clang] Fix '-Wformat-overflow' FP when floats had field-width and plus prefix (#144274)
90da61634a4a [flang] Use outermost fir.dummy_scope for TBAA of local allocations. (#146006)
0f291e5787a5 [clang-tidy] Add flag to specify an alternative to std::move in cppcoreguidelines-rvalue-reference-param-not-moved (#138757)
5f91b697bcd8 [MLIR] Always emit setPropertiesFromParsedAttr() when hasCustomAssemblyFormat is set (#145973)
46628718c05a [IR][PGO] Verify the structure of `VP` metadata. (#145584)
878d3594ed74 [mlir][vector] Avoid setting padding by default to `0` in `vector.transfer_read` prefer `ub.poison` (#146088)
6a57af8d0367 [clang-tidy] Add flag to specify an alternative to std::forward (#138755)
999b216135b5 Revert "[flang] add option to generate runtime type info as external (#146071)"
ba84d0c8d762 [clang-doc] Precommit friends test (#146164)
233078fd8d70 [nfc][asan] clang-format for #145087
536ba87726d8 [libomptarget] Add a test for OMP_TARGET_OFFLOAD=disabled (#146385)
5548f4d5ef6a [LLDB][NFC] Refactor code extracting timestamp from StructuredData (#145954)
125dbe103e4f [OpenACC][CIR] 'update' construct lowering + a few clauses (#146378)
2d825cc3af4e [flang] Add new hints to expected warnings (#146399)
b2b20eeb5412 tsan: Support free_sized and free_aligned_sized from C23 (#144531)
01d0171e7a95 Address formatting mistake in libc++ test
36cf51d16e02 [clang][bytecode] Classify variable initializer, not the decl (#146338)
0494f934345f [Basic] Drop const from a return type (NFC) (#146382)
6a83a84ac52a [IR] Remove an unnecessary cast (NFC) (#146381)
7a33b709b1e2 [libc] wcstok implementation (#145989)
790bc5bc724e [flang][NFC] Remove inadvertently added source file (#146395)
de7c2f29405f [HLSL] Remove dead code in Type.cpp [NFC] (#146365)
407542b3ecbe [flang] Process pointer component default initializers sooner (#145601)
f3d57590bfae [flang] Skip over fixed form spaces when prescanning exponents & kind… (#145347)
a93d843ab31a [flang] Don't warn on (0.,0.)**(nonzero noninteger) (#145179)
348002e1117b [flang] Check definability for logical INQUIRE specifiers (#144797)
dccc0266f423 [flang][runtime] Allow INQUIRE(IOLENGTH=) in the presence of defined I/O (#144541)
83b462af17ca [flang][CLI] Have the CLI hint the flag to disable a warning (#144767)
efc561c061bc [CodeGen] Remove an unnecessary cast (NFC) (#146380)
529508c187e9 [DAG] canCreateUndefOrPoison - add handling for CTTZ/CTLZ nodes (#146361)
04395be63078 MC: Merge MCFragment.h into MCSection.h
a8f460d1dcc2 [libc] implement sigsetjmp for thumb/thumb2/armv7-a (#138147)
c79b68541be0 Restrict tests in Transforms/InstCombine/SystemZ to SystemZ (NFC) (#146363)
6bd9ff04afdf [ARM] Add neon vector support for round
a1d83311c892 [AArch64] Change IssueWidth to 5 in AArch64SchedNeoverseN2.td (#145717)
18a0675082de [bazel] Add gdb-scripts filegroup in llvm and mlir (#146358)
16dc6f64ded3 [lldb] Document MCP support in LLDB (#145935)
25e5010a302f [lldb] Delete unused variables in TestLoongArchEmulator.cpp.
4aaab693148e [libc++] Fix wait_on_destruct.pass.cpp hanging sometimes (#146240)
199c6ecb5cd0 [RISCV] Add coverage for missed scalarization of gather/scatter base pointers
8a5b97a7205d Revert "[clang] [modules] Add err_main_in_named_module (#146247)"
545cdca48835 [SCCP] Improve worklist management (#146321)
6f7370ced630 [InstCombine] Pull vector reverse through fneg (#146349)
425359a83d22 [experimental] Ask copilot to check perf profile degradation (#146271)
00cdaa5c395c [clang][bytecode] Add `Descriptor::hasTrivialDtor()` (#146286)
53102a395f3b [ExtractAPI] Format pointer types correctly (#146182)
5ab3114bd12c Expand annotation check for -Wunique-object-duplication on Windows. (#145944)
c00c5a389899 [X86] Add test coverage to show failure to push freeze through CTLZ/CTTZ nodes (+ZERO_UNDEF variants)
44ec3e8f9c3e [clang][AST] Fix AST IgnoreUnlessSpelledInSource traversal nullptr dereference (#146103)
1cdc7f872410 [clang] [Sema] Suggest [[noreturn]] for void functions that always throw (#146234)
d1ba2692ee52 [LLVM][Clang] Enable strict mode for `getTrailingObjects` (#144930)
b0ff473340e3 [LLVM] Change `ModulePass::skipModule` to take a const reference (#146168)
05cc70fbfaa3 merge main into amd-staging
67e73ba605ea [Offload] Refactor device/platform info queries (#146345)
619f7afd716c [LAA] Clean up APInt-overflow related code (#140048)
8c7719183558 [InstCombine] smin(smax(X, -1), 1) -> scmp(X, 0) and smax(smin(X, 1), -1) -> scmp(X, 0) (#145736)
10445acfa64a [libc] Efficiently implement 'realloc' for AMDGPU devices (#145960)
d7e23bef6ad4 [lldb][RISCV] fix LR/SC atomic sequence handling in lldb-server (#146072)
a99fee6989a6 [OpenACC][CIR] Implement 'exit data' construct + clauses (#146167)
51dfe28f8754 [clang] odr-checker fix for conversion operators (#146153)
d19d7e03a7cf merge main into amd-staging (#2898)
d4fdfc3aa718 [DA] Improve code in getSplitIteration (NFC) (#146137)
183acdd27985 [GlobalOpt] Revert global widening transform (#144652)
f01a7936bea4 [VPlan] Replace all uses of VF when EVL tail folding. NFCI (#146339)
652630b3c918 [ISel/RISCV] Fix fixed-vector [l]lrint lowering (#145898)
b9e4679976e2 [DAG] canCreateUndefOrPoison - add handling for ADD/SUB/MUL overflow nodes (#146322)
eb1a80bfd3a0 [DAG] Implement SDPatternMatch `m_SpecificScalarVT` and `m_SpecificVectorElementVT` matchers (#144996)
749c7c5dc49a [AMDGPU] Add support for `v_cvt_f16_bf8` on gfx1250 (#146305)
a99c964d7f40 [AMDGPU] Add support for `v_cvt_f16_fp8` on gfx1250 (#146302)
e810c639febc [gn build] Port f8cb7987c64d
a4647b21f2cb [gn] port b42c8831d57a
b29fea6eebae [KeyInstr][Clang][NFC] Don't set -dwarf-use-key-instructions (#144115)
be75ded3fea9 [KeyInstr][Clang] Copy ctor/assignment operator source atoms (#144346)
003145d0c8d5 [Offload] Implement `olShutDown` (#144055)
6e6c61d696c9 [InstCombine] Add explicit data layout to test
7a3e555353c5 [Clang][Sema] Require `BaseClass::` (not other classes) in member using-declaration in C++98 mode (#143492)
c3f8dd122822 [NFC][analyzer] Use %clang_analyze_cc1 consistently (#145895)
3a7d60860d1a [FatLTO] Relax checks for fatlto pipeline test
5186d4aeeeca [NFC] [clang] Fix a test regression
33c8d5c686ea [Clang][AArch64] Add FP8 variants of Neon store intrinsics (#145346)
a72a0f415d59 [Clang][AArch64] Add mfloat8_t variants of Neon load intrinsics (#145666)
5e732c09b213 [CLANG][AArch64] Add mfloat8_t support for more SVE load intrinsics (#145383)
d7a3bdffb9a3 [PassBuilder][FatLTO] Expose FatLTO pipeline via pipeline string (#146048)
f4128428af0d [PHIElimination] Fix bug around $noreg assignment (#146320)
de6b8cdc4112 [EarlyCSE] Add support for writeonly call CSE (#145474)
f8cb7987c64d [RISCV] Integrate RISCV target in baremetal toolchain object and deprecate RISCVToolchain object (#121831)
24fd3437c311 [lldb][test] TestMetadataRegisters.py: add missing include
47ecd18f24d3 Global string alignment (#142346)
d0c1ea928c5c [InstCombine] Pull unary shuffles through fneg/fabs (#144933)
f226852265e6 [X86] Add test coverage to show failure to push freeze through ssubo/usubo nodes (+carry variants)
338dee0742ce [NFC][libclc] Refactor _CLC_*_VECTORIZE macros to functions in .inc files (#145678)
1dfc3e846154 [X86] Add test coverage to show failure to push freeze through saddo/uaddo nodes (+carry variants)
2e7aa7ead680 [mlir][tosa] Add custom operand getters for select op (#145921)
473769ec9b2f [clang] [modules] Add err_main_in_named_module (#146247)
b42c8831d57a [clang] Build the Z3 mock module via CMake (#146284)
af82e14c4a35 [KeyInstr] Fully support mixed key/non-key inlining modes (#144103)
7ec494e9bde8 [gn build] Port a64db49371f0
96fc0b3a329c [gn build] Port a3a60e03e2bf
22064ce53eb5 [gn build] Port 3f531552e66f
5c5bcd622192 [gn build] Port 34b2e934ea9e
6a120bf02ec0 [Hexagon] Remove unused flag from HexagonGenExtract(NFC) (#146310)
179cc202bda3 [libc++] Add missing _LIBCPP_NODEBUG in <__type_traits/invoke.h>
9fbde32a8c75 Revert "CodeView: Move MCCVDefRangeFragment storage to MCContext/MCFragment"
e0864b56b016 [libc++] Fix <charconv> not exporting std::errc (#146268)
5220e4f2bfb6 Speculative test tweak after ed25d473fb1b7
b563e763065d [AArch64] Improve scalar and Neon popcount with SVE CNT. (#143870)
597ee882a557 [Driver] Add support for `-m` flag to linker job of Baremetal toolchain (#134442)
faefe7cf7daf [flang] add option to generate runtime type info as external (#146071)
834c410d9df3 [Target] Prevent copying in loop variables (NFC)
8602204d9fc4 [mlir][tensor] Relax input type requirement on `tensor.splat` (#145893)
57f7e14b5779 [LoongArch] Pre-commit test for _BitInt(N)
7354123c34e6 [KeyInstr] Use DISubprogram's is-key-instructions-on flag at DWARF emission (#144104)
ed391f174b90 [Driver] Forward sysroot from Driver to linker in BareMetal ToolChain Object (#132808)
140e1894f245 [KeyInstr] Add DISubprogram::keyInstructions bit (#144107)
629126ed44bd [clang][bytecode][NFC] Remove some dead code (#146287)
2d94c08d03ad Reland [Driver] Add option to force undefined symbols during linking in BareMetal toolchain object.  (#146304)
bb982e733cfc AMDGPU: support s_monitor_sleep on gfx1250 (#146293)
a17f63590ac0 Reland  [Driver] Fix link order of BareMetal toolchain object (#146291)
59eaffe93aec [RISCV] Remove required features zvfhmin from vslideup/vslidedown
baa8aff47198 merge main into amd-staging (#2897)
42d94afffe64 [Target] Use range-based for loops (NFC) (#146277)
72c0fc230504 [LTO] Remove an unnecessary cast (NFC) (#146275)
678ccd5e536d [IR] Remove an unnecessary cast (NFC) (#146274)
d4b5905a2591 Add`final` specifier to the classop (#145977)
e47d4010d341 CodeView: Move MCCVDefRangeFragment storage to MCContext/MCFragment
9a6e0688b04f [mlgo][regalloc] Fix after PR #131837 (#146297)
2105da73cab4 Revert "[PHIElimination] Reuse existing COPY in predecessor basic block (#131837)"
ff7c5c829497 merge main into amd-staging
613222ec332b [DAGCombiner] Remove `UnsafeFPMath` usage in `visitFSUBForFMACombine` etc. (#145637)
2de51345fb3d MCFragment: Add addFixup to replace getFixups().push_back()
dc6b029aa107 merge main into amd-staging
cd075a40133d MCObjectStreamer: Deduplicate emitInstToData
38df9ebdc6db [docs] Fix a typo in llvm/docs/CMake.rst (#138639)
75f359dc5643 [MC] Remove an unnecessary cast (NFC) (#146276)
b62b58d1bbbf [OpenMP] Fix crash with duplicate mapping on target directive (#146136)
1f5f38192090 AMDGPU: Implement intrinsic/builtins for gfx1250 load transpose instructions (#146289)
b822a32659c3 [VPlan] Fix crash when trying to narrow interleave group storing const.
347a5f4f3d8d [gn] port 23f1ba3ee456 (DebugInfo/DWARF/LowLevel)
d8081297bd88 [gn build] Port 23f1ba3ee456
d0e8d84424fd [gn] port 3fdf46ad60f1 (RuntimeLibcalls.inc)
ed25d473fb1b Reland [Driver] Add support for crtbegin.o, crtend.o and libgloss lib to BareMetal toolchain object  (#144649)
43f2a3cd2df9 [gn] port e34e02128ec5eb8
8943036ec37b Fix UAF in ARMConstantIslandPass.
fa058647fe3a [clang-format] Make EndsInComma in ContinuationIndenter consistent (#146256)
a3a60e03e2bf [clang-tidy] add new check: modernize-use-scoped-lock (#126434)
40525fb4c78a [NFC][docs][coro] Fix syntax & typos (#146282)
b5cd49eff02b [mlir] Remove unused includes (NFC) (#146278)
f5c62ee0fa04 [PHIElimination] Reuse existing COPY in predecessor basic block (#131837)
37ffe896a5cd Reland [Driver] Add support for GCC installation detection in Baremetal toolchain (#145390)
5fe40b9df3b9 MCExpr: Make print private
8d099271c6ce MCInt: Change dump functions to accept MCContext instead of MCRegiserInfo
713839729c97 [Clang] Add __builtin_invoke and use it in libc++ (#116709)
43ab5bb92115 [CodeGen] Use std::tie to implement a comparison functor (NFC) (#146252)
d2d5203bf48a [clangd] Consistent precedence between `--header-insertion` and `HeaderInsertion` (#146235)
2f6a927099e8 merge main into amd-staging (#2892)
f5ed863176dd Revert "[VPlan] Allow derived IVs and scalar-steps in narrowing interleave."
267945b665cf [libc++][NFC] Refactored equality comparisons tests for `pair` and `expected` (#145668)
f21f2b483c4f [InstCombine] Create Icmp in canonical form (NFC) (#146266)
2787759ef2e4 [VPlan] Allow derived IVs and scalar-steps in narrowing interleave.
dd02fb3a5145 [AArch64] Fix stale +zcm target feature to +zcm-gpr64 (#146260)
1649e37f2eb4 merge main into amd-staging
dcc9e36b1826 [ARM] Add neon vector support for floor (#142559)
c5a49fb62d41 [LAA] Add tests with 128 bit inductions and 128 bit pointers.
20fbbd76755c [LV] Add support for cmp reductions with decreasing IVs. (#140451)
8bd6d36a4413 [ARM] Override hasAndNotCompare (#145441)
dc1a79a97814 MC: Make save-temp-labels imply UseNamesOnTempLabels
b54337d76cc7 MC: Enhance mc-dump output
bab13b379c2c merge main into amd-staging (#2891)
842f4f711d3a [Target] Prevent copying in loop variables (NFC)
2661d595793c MC: Remove post-relaxation and Symbol printing from mc-dump output
279e808b7511 MC: Make mc-dump output compact
2248cdfa74ec [Arm] Fix UAF in ARMConstantIslandPass (#146232)
c9cdc33dd697 [clang] Remove unused includes (NFC) (#146254)
9cf251d9d8cc [Target] Use range-based for loops (NFC) (#146253)
72e24f3022f3 [MC] Remove an unnecessary cast (NFC) (#146251)
8a4b6cd8d80d [IR] Remove an unnecessary cast (NFC) (#146250)
0d0daef6ee95 [GlobalISel] Remove an unnecessary cast (NFC) (#146249)
6f2cec219a52 merge main into amd-staging
68d83fae70b2 [llvm-profdata] Resolve tilde for weighted input filenames (#146206)
25d52fbf96ec [PowerPC] Prevent copying in loop variables (NFC)
6c2aa37fb1fd MCAsmStreamer: Print symbol equated to a register
1108cf64196a ELFObjectWriter: Optimize isInSymtab
3cc78a806778 [NFC][AMDGPU] Fix the instruction order in some test cases to make downstream easier (#146214)
aa8afadd251c MCSymbolRefExpr: Migrate away from deprecated VariantKind
53e665e50134 merge main into amd-staging (#2889)
2ed04016120f CSKY: Migrate to the new relocation specifier representation
f771d08a2476 [clang-format] Fix a bug in `ReflowComments: Always` (#146202)
d2e8e55e02f7 CSKY: Replace deprecated MCExpr::print with MCAsmInfo::printExpr
bdb299a67eb8 [VPlan] Simplify code in single scalar transform code (NFC).
a365abd544fd MCExpr: Remove error-prone operator<<
e6b25288eb0a MCExpr: Migrate away from operator<<
5ffdd9480d80 [CodeGenPrepare] Filter out unrecreatable addresses from memory optimization (#143566)
3ea7fc733970 attempt to calm down restarter job
fb24b4d46a0a [BOLT] Push code to higher addresses under options (#146180)
f90025ebd930 [llvm] Compare std::optional<T> to values directly (NFC) (#146222)
bad5a740e114 [PowerPC] Use range-based for loops (NFC) (#146221)
402baea0a9ff [modularize] Use std::tie to implement operator< (NFC) (#146220)
dea8217a7a09 [ADT] Use llvm::has_single_bit (NFC) (#146219)
289c066ac474 [ExecutionEngine] Remove unnecessary casts (NFC) (#146218)
c101415820f2 [DebugInfo] Remove an unnecessary cast (NFC) (#146217)
25d7fd35b203 [Analysis] Remove unnecessary casts (NFC) (#146216)
c57c5f53a32a [flang] Fix warnings
303bc0df6ade SPIRV: Replace deprecated MCExpr::print with MCAsmInfo::printExpr
f96832a52019 BPF: Replace deprecated MCExpr::print with MCAsmInfo::printExpr
55fd1ac63839 merge main into amd-staging
80222f0aa44c [bazel] Port 3f531552e66f242d4bfc0163be848f9e51259d80
0255ae81ba4e AsmMatcher: Use getAsmInfo()
e878b7e3492b MCParsedAsmOperand::print: Add MCAsmInfo parameter
33c265ddf7f3 [SimplifyCFG] Use indexType from data layout in switch to table conversion (#146207)
344b5b7f9e5b [flang][OpenMP] Move lowering of ATOMIC to separate file, NFC (#146225)
ff075d0ac22e MCParser: Remove operator<< const MCParsedAsmOperand &MO
2327f3f1d5aa merge main into amd-staging (#2887)
194953649494 [VPlan] Also visit VPBBs outside loop region when unrolling by VF.
d93aff42c230 MC: Migrate away from operator<< MCExpr
a9295ad4d2d3 AArch64: Replace deprecated MCExpr::print with MCAsmInfo::printExpr
94c1506548ec Further restrict the restarters ability to restart manually canceled jobs
169b4bea8774 Attempt to fix over-keen restarter.
372c80821777 [X86] canCreateUndefOrPoisonForTargetNode - PCMPEQ/PCMPGT don't create poison/undef (#146116)
bf8b346e0566 merge main into amd-staging
fd47c265dda7 merge main into amd-staging (#2886)
3d21f54e42e0 merge main into amd-staging (#2885)
b8f122812e0a [docs][coroutines] Revamp "Debugging C++ coroutines" (#142651)
34b2e934ea9e [libc++] Introduce `__product_iterator_traits` and optimise `flat_map::insert` (#139454)
feb61f5b0529 [Target] Prevent copying in loop variables (NFC)
8ecee5098e34 merge main into amd-staging
8c1aca180cf7 Revert "Reapply "[NFC][DebugInfo][DWARF] Create new low-level dwarf library (#… (#145959) (#146112)"
5ea29f77b9f8 [DA] Let getConstantPart return optional APInt (NFC) (#146135)
cadf26b68865 merge main into amd-staging
04cd0f270227 [LSR] Clean up code using SCEVPatternMatch (NFC) (#145556)
7ff9669a2e80 [ISel/RISCV] Refactor isPromotedOpNeedingSplit (NFC) (#146059)
be9502b095d2 [LoopInterchange] Modernize loops (NFC) (#146105)
1e2ddc8a3d4a [CostModel/RISCV] Add tests for ldexp, [l]lround (#146108)
93ae443ef53f Merge amd-debug into amd-staging (#2883)
b6515ae6843d [AArch64] Align 0-cycle reg-mov model of GPR64, GPR32 reg classes (#146051)
69b8e59f29a3 [ValueTracking] Forward-declare class instead of including header (NFC)
0ba456fcc6b1 [Clang][LoongArch] Match GCC behaviour when parsing FPRs in asm clobbers (#138391)
c83862451184 merge main into amd-staging (#2884)
e34e02128ec5 [clang] Fix tests requiring Z3 headers in standalone builds (#146200)
3c4e7308028e [Sema][clangd] add noexcept to override functions during code completion (#75937)
8351752dbc40 [clang-tidy] Fix false positives in `readability-redundant-inline-specifier` (#135391)
18e9cfd56b0a Hexagon: Remove getVariantKind
b9ca4c5fab40 MCExpr: Make COFF-specific VK_SECREL target-specific
3f531552e66f [REAPPLY][Clang-Repl] Add support for out-of-process execution. #110418 (#144064)
ac9204de7d4f MCExpr: Remove VK_None
c5479dc09f25 merge main into amd-staging
e121f72c945f X86: Rename X86MCExpr::VK_ to X86::S_
1a94bcf90f26 [llvm] Remove unused includes (NFC) (#146199)
094a7087b83b [Target] Use range-based for loops (NFC) (#146198)
56da4a962855 [llvm] Use std::tie to implement comparison functors (NFC) (#146197)
f90af1c9e83b [MC] Drop const from a return type (NFC) (#146196)
7f6d7391cfff [Basic] Use StringRef::drop_back (NFC) (#146195)
eff28bdd46b1 [AMDGPU] Use StringRef::consume_back (NFC) (#146194)
c73906ec691d MCParser: Reduce VK_None uses
80625c16f067 [MLIR][Affine] Fix memref replacement in affine-data-copy-generate (#139016)
28d4cc6d7b16 MC: Reduce MCSymbolRefExpr::VK_None uses
4d7510c33525 [RISCV] Remove untested code from SelectAddrRegRegScale. (#146185)
0e558239b825 LoongArch: Remove unneeded VK_None check
5aa3e6baa0b0 MC: Reduce MCSymbolRefExpr::VK_None uses
109b7d965c58 MC: Remove unneeded VK_None argument to MCSymbolRefExpr::create calls
c6bd0207141f Revert "[Hexagon] NFC: Reduce the amount of version-specific code" (#146193)
54a7d53227ae [BOLT] Fix program-header.test
163ad0bcdae5 [bazel] Add lldb deps for #144627 (#146192)
3923dd4484b8 Fix uncaught-exception.test. (#146190)
72060f1cfd56 Fix uncaught-exception.test. (#146181)
a64db49371f0 [lldb][Mach-O] Allow "process metadata" LC_NOTE to supply registers (#144627)
67a5fc8e12dc Update references from llvm-beanz to llvm (#146177)
389fd7e888f8 xfail two mlir tests
32180cf9f9eb [CIR] Upstream support for operator assign (#145979)
48a9d5bc30d1 Resolve conflicts
66cf7844fb6c Merge branch 'amd-debug' into amd-staging
570b95218c13 [flang][cuda] Do not generate section checks in device context (#146170)
6d817810daa1 [clang-doc] serialize IsBuiltIn and IsTemplate for types (#146149)
71d4c9ce70e7 cmake: Allow CLANG_RESOURCE_DIR to be absolute.
5ecd21a46ab1 [NFC] Make unwind table internal data structures more accessible (#145804)
c43282ab69d7 [RelLookupTableConverter] Drop unnamed_addr for GVs in entries to avoid generating GOTPCREL relocations (#146068)
c7c79183febc Fix libomptarget build with flang disabled (#2879)
51d642d8289f Manual update of LLVM_MAIN_REVISION to 542578 (#2882)
75175e723085 [flang][cuda] Inline this_thread_block() calls (#146144)
9a93de58f7f3 [TableGen] Simplify copying OperandMap entries for tied operands in CompressInstEmitter. NFC
457c9aec9e84 [TableGen] Simplify how we calculate NumMIOperands in addDagOperandMapping. NFC
8bc61cbfde28 [Clang][ByteCode][NFC] Avoid copies by using move in Disasm.cpp (#146127)
f58caed1bc50 [libc] Changed CharacterConverter returns (#146130)
171aa34e43bc [DirectX] Add static sampler support to root signature (#143422)
442f99d7698a [lldb] Fix evaluating expressions without JIT in an object context (#145599)
9f5061d4f004 [lldb] Fix a typo in documentation (#146115)
b2f504ff15a1 [flang][cuda] Inline this_warp() calls (#146134)
231076898cc0 merge main into amd-staging (#2878)
68239b76f139 [gtest] Fix building on OpenBSD/sparc64 (#145225)
29b105483510 [mlir][linalg] Update pack and unpack documentation (#143903)
33d20828d1ff [OpenACC][CIR] Implement enter-data + clause lowering (#146146)
8d2034cf68b5 [clang] Add flag fallow-runtime-check-skip-hot-cutoff (#145999)
23be14b22200 [HLSL][SPIRV] Boolean in a RawBuffer should be i32 and Boolean vector in a RawBuffer should be <N x i32> (#144929)
f20ef8520ddc [CIR][NFC] Fix init llvm::ArrayRef warning (#146147)
98f7d756e334 [FileCheck] Improve printing variables with escapes (#145865)
929fea8c807d Manual update of LLVM_MAIN_REVISION to 542578
526701f8dc54 [LV] Add additional tests for narrowing interleave groups.
e9c9adcefed1 [Hexagon] NFC: Reduce the amount of version-specific code (#145812)
21993f0a47f0 [lldb][NFC] Switch IRMemoryMap::Malloc to return llvm::Expected (#146016)
254c26d02276 Add missing dependency. (#146140)
17328f36f6ca [BOLT][test] Fix NFC mismatches in perf2bolt tests (#146148)
23daa31341a6 [llvm] Don't preserve analysis results after EmbedBitcodePass (#146118)
d34214a85e3b [libc] Add and use 'cpp::launder' to guard placement new (#146123)
1eacdddc0c44 [SandboxVec][SeedCollector][NFC] Replace cl::opt flags with constructor args (#143206)
2e98979ef4bf Fix libomptarget build with flang disabled
52040b44f606 [ADT] Remove MutableArrayRef(std::array) (NFC) (#146114)
9d6cbc3c2092 [ADT] Deprecate MutableArrayRef(std::nullopt) (#146113)
71bc606e95d6 [LowerAllowCheckPass] allow to specify runtime.check hotness (#145998)
6c2e91285510 AMDGPU: Add alias tests for load transpose instructions for gfx1250 (#146117)
5dbd8772fed2 [DirectX] add support for i64 buffer load/stores (#145047)
23f1ba3ee456 Reapply "[NFC][DebugInfo][DWARF] Create new low-level dwarf library (#… (#145959) (#146112)
a4be46e0e5bc [Clang][ByteCode][NFC] Misc minor performance fixes (#145988)
836dd77c45ee merge main into amd-staging
32ef4ceec03d [InstrRef] Skip clobbered EntryValue register recovery (#142478)
e4d8e06f83f8 [OpenACC][CIR] Implement present/deviceptr/attach lowering for data
778b6a21ec27 [DirectX] Allow vector Allocas to be transformed into arrays (#145972)
a19d37044b35 [libc++] P2944R3: Constrained comparisions - `tuple` (#145677)
a3fcfacc35e7 [TableGen] Simplify verifyDagOpCount in CompressInstEmitter.cpp. NFC
c11ea449e59c [ValueTracking] Add `matchSimpleBinaryIntrinsicRecurrence` helper
ac7e3910350a [libc] Implemented wcsnlen (#145610)
af2bf2e544b5 MC: Remove unneeded MCSymbolRefExpr::VariantKind calls
74cabdb806ae [CIR] Add basic support for operator new (#145802)
3e9fd4966d73 [CIR] Add support for constructor aliases (#145792)
d1d8ef2ad9c2 [PassBuilder] Treat pipeline aliases as normal passes (#146038) (#2871)
f80d1b5d9dad merge main into amd-staging (#2875)
2557f9946373 [OpenACC][CIR] Implement 'no_create' lowering for data
f03782dd67d3 [NVPTX] Fixup v2i8 parameter and return lowering (#145585)
7613c2412a25 [LoopInterchange] Use ArrayRef in more places (NFC) (#146077)
4df059484c13 Merge llvm/main into amd-debug
d00c83ef226f [BOLT] Skip creation of new segments (#146023)
2282d4faa011 [ISel/RISCV] Improve code in lowerFCOPYSIGN (NFC) (#146061)
9df1c81fee5b [SelectionDAG] Combine range metadata when loads are CSEd. (#146026)
375af75efb2e [RISCV] Simplify the check for when to call EmitLoweredCascadedSelect. NFC (#145930)
b76bc185a4f2 [OpenACC][CIR] Add copy/etc clause lowering for 'data'.
e933cfcfb2d2 [NVPTX] Fixup NVPTXPrologEpilogPass for opt-bisect-limit (#144136)
37b0b0f7d2b6 [lldb] Add class property for the version string (#145974)
a460aa1071c9 [Sema] Fix a warning
ec150a9944b2 [PhaseOrdering] Add test for memset DSE (NFC)
fd24e8237087 Merge llvm/main into amd-debug
794edd187cee [clang] Suppress noreturn warning if last statement in a function is a throw (#145166)
07f1502b867e [CIR] Implement SizedTypeInterface to make isSized hookable (#146045)
61c0a94a9047 [CIR] Refactor type interfaces (#146044)
dc6d2b841f22 Revert "[flang][OpenMP] Move lowering of ATOMIC to separate file, NFC" (#146091)
3b7b95f78e24 [VPlan] Support VPWidenSelectRecipe in narrowToSingleScalar.
1df3e0a275bd [lldb] Use llvm::is_contained (NFC) (#146012)
f0311f447a23 [ADT] Remove a constructor (NFC) (#146010)
3876e887d00f [MLIR][ArmSVE] Add an ArmSVE dialect operation mapping to `bfmmla` (#145064)
da2969b1051f [LLDB] Update DIL to handle smart pointers; add more tests. (#143786)
772009ce4aff [PS5][Driver] Allow selection of CRT with `-L` (#145869)
3463aba45f61 [OpenACC][CIR] Implement copyin/copyout/create lowering for compute/c… (#145976)
08d747c1ef65 [AMDGPU] Fix bad removal of s_delay_alu (#145728)
39f19f2f1fb8 [Offload] Store device info tree in device handle (#145913)
102cf1b99971 [Offload] Make CUDA Driver Version a string (#146049)
3e8437cf8e8f [openmp] - Rework openmp-config for portability (#2856)
dcea5f1f38eb [TargetLowering] Fold (a | b) ==/!= b -> (a & ~b) ==/!= 0 when and-not exists (#145368)
e9acd14d27d5 [HeterogeneousDwarf] Fix asserts during expression emission (#2860)
5ecf16f5e107 merge main into amd-staging
f329689ec091 [AMDGPU] add_i128.ll - regenerate test checks
2b48ce7034a8 [docs] Add documentation for LLVM Qualification Group (#145331)
2cbcec483293 [Clang][NFC] - Move attr-cpuspecific-cpus test over to Sema (#146065)
f59d270867cc [cmake] Ignore pstl in LLVM_ENABLE_PROJECTS (#146070)
302ed97b583f [flang][OpenMP] Move lowering of ATOMIC to separate file, NFC (#146067)
ec1c73b2ecb3 [compiler-rt][ARM] Only use bxaut when the target has pacbti (#146057)
d8ca77e2b96e [Clang][Sema] Allow qualified type names in `swift_name` attribute
541f33e0751d [mlir][linalg] Prevent hoisting of transfer pairs in the presence of aliases (#145235)
7e2e03012143 GlobalISel: Replace use of report_fatal_error (#145866)
37e2d104994d Revert "[flang] add option to generate runtime type info as external" (#146064)
91f10df794d3 [Flang][OpenMP] Skip implicit mapping of named constants (#145966)
c8ea114741ce AMDGPU: Introduce a pass to replace VGPR MFMAs with AGPR (#145024)
bc1a6a2a9377 AMDGPU: Add baseline tests for VGPR MFMA rewriting pass (#145023)
cf3d136c2253 [AArch64] Do not generate ld1IndexPost when inserting into lane 0 of a zero vector (#145723)
e9805235bf68 [libc++] Move libcxx/test/libcxx/extensions to libcxx/test/extensions and update the tests (#145476)
3666b42257c9 [PassBuilder] Treat pipeline aliases as normal passes (#146038)
5fdcb35aaa4e [InferAlignment] Add tests with GEP recurrences.
e816817bbb28 [flang] add option to generate runtime type info as external (#145901)
f0f46e25ecbf [bazel] Port 3fdf46ad60f1747e8ea1cced2e67dd36f29fc43b
b989c76f3974 [flang][NFC] switch ValueRange(nullopt) to ValueRange{} after #146011 (#146043)
3f00cff5c787 [lldb][test] Disable TestLocationsAfterRebuild.py on Windows
7dde6027a079 [DAG] canCreateUndefOrPoison - add handling for ISD::SELECT (#146046)
0158ca21a23b Prevent a crash when a global variable has debug metadata (#145918)
793667017c64 [NFC][LLVM] Use DL consistently throughout AArch64ISelLowering.cpp.
aed8f1992a3b [NFC][mlir][memref] refine debug message about memref::SubViewOp. (#145470)
c3e08c9b893b [MLIR] Replace getVoidPtrType with getPtrType in ConvertToLLVMPattern (#145657)
1fb786ea9387 [clang][scandeps] Improve handling of rawstrings. (#139504)
7f223d121d74 [PassBuilder] Treat pipeline aliases as normal passes (#146038)
613804cca986 [LV] Improve code using [[maybe_unused]] (NFC) (#137138)
08f074a59f3d [TTI] getInstructionCost - consistently treat all undef/poison shuffle masks as free (#146039)
5f2b9dd90dcc Re-land "[-Wunterminated-string-initialization] Handle C string literals ending with explicit '\0' (#143487)"
bf2b14acf326 [LV] Enable auto-vectorisation of loops with uncountable exits (#133099)
2c90c0b90cbd [lldb] Extract debug server location code (#145706)
7255c3aee3eb DAG: Check libcall function is supported before emission (#144314)
ddb8493ca757 [LV] Fix test issue caused by #145877 (#146041)
b4d3283ab79c AArch64: Add libcall impl declarations for __arm_sc* memory functions (#144977)
779f7243c8bb XCore: Declare libcalls used for align 4 memcpy (#144976)
f38773e980e2 Hexagon: Add libcall declarations for special memcpy (#144975)
4243e502c13e ARM: Add runtime libcall definitions for aebi memory functions (#144974)
b88e1f6a79cb TableGen: Generate enum for runtime libcall implementations (#144973)
3fdf46ad60f1 TableGen: Add runtime libcall backend (#144972)
d163ab332349 [libc++] Remove a bunch of unnecessary type indirections from __tree (#145295)
c15f4225419f [RISCV] Remove required features zvfhmin/zvfbfmin from plain f16/bf16 intrinsics (#145891)
6f43754e9caa [LV] Disable interleaving via hints for uncountable early exit loops (#145877)
2529de5c935a [ADT] Deprecate ArrayRef(std::nullopt) (#146011)
a40a4c552b92 [MC] MCSectionGOFF.h - fix GCC Wparentheses operator precedence warning around assert message. NFC.
ec62dee70349 [VPlan] Handle FirstActiveLane when unrolling. (#145394)
045b82736702 [GlobalISel] Use-Vector-Truncate Opt Needs Elt Type Check (#146003)
8a839ea79123 [analyzer][NFC] Fix clang-tidy warning in Malloc and UnixApi checkers (#145719)
205dcf7146ce PowerPC: Remove redundant MCSymbolRefExpr::VariantKind casts
96c1611163d3 [mlir][linalg] fix OuterUnitDims linalg.pack decomposition pattern (#141613)
23384cd58171 [OpenMP][test][NFC] Temporarily disabling failing test
950d281eb2d4 [RISCV] Add ISel patterns for Qualcomm uC Xqcicm extension (#145643)
bc90166a50c4 [TSan] Clarify and enforce shadow end alignment (#144648)
7726103d1ee1 WebAssembly: Merge MCExpr into MCAsmInfo
a277d24ddb7c [ProfileData] Use llvm::count (NFC) (#146013)
c7b34b0b44ae [mlir] Use a new constructor of ArrayRef (NFC) (#146009)
26ec66dc1825 [llvm] Use a new constructor of ArrayRef (NFC) (#146008)
8f71650baaee [clang] Use a new constructor of ArrayRef (NFC) (#146007)
786ccb2c0e8c [LV] Directly check if memory or SCEV check blocks are used (NFCI).
eb9d22b24cb8 VE: Merge MCExpr into MCAsmInfo
afe6af14ff9b [msan] Add optional flag to improve instrumentation of disjoint OR (#145990)
56b2c7d98875 MC: Rename initializeVariantKinds to initializeAtSpecifiers
917932244714 Revert "[llvm][EmbedBitcodePass] Prevent modifying the module with ThinLTO" (#145987)
d829636f5d71 [C++20] [Modules] Don't mark namespace decl as module local declaration
47292428785f AMDGPU: Add MC layer support for load transpose instructions for gfx1250 (#146024)
7dfcf489fd93 PowerPC: Separate ELF and XCOFF @ specifiers
905f987d2f44 merge main into amd-staging (#2865)
60a18d611907 [LowerAllowCheckPass] fix pipeline printing (#146000)
0cde5a85690c [gn build] Port 4f97780a7a55
61739d76f009 AMDGPU: Trivial doc fixes (#146021)
8ed064b979d5 [bazel] Add targets for transform.debug python extension (#146022)
207925ebe780 Xtensa: Move MCExpr into MCAsmInfo
ab1e4d55d866 [clang-doc] refactor BitcodeReader::readSubBlock (#145835)
4f97780a7a55 LoongArch: Move MCExpr into MCAsmInfo
569fcac4584a [LoongArch] Pre-commit tests for optimizing insert extracted fp elements
1b2843bae062 [bazel] Port #145995 (#146014)
96ec1c29f236 [RISCV] Add nds.bfos and nds.bfoz for the short forward branch optimization. (#145836)
0515449f6dcb [mlir][tensor][memref] Enhance collapse(expand(src)) canonicalization pattern. (#145995)
30e519e1ad18 [LoongArch] Fix xvshuf instructions lowering (#145868)
d58b0f23d062 AMDGPU/MC: Try harder to evaluate absolute MC expressions (#145146)
54e2f5ac9c45 [clang][docs] Fix typo in -fapinotes-modules option. (#145907)
13da48ddb31e [lldb][NFC] Remove unused ConstString includes in Utility (#145983)
f2f17e563df9 [clang-format][NFC] Remove `\brief` from comments (#145853)
ad6b597875b9 [ORC] Fix EPCGenericMemoryAccessTest write-ptrs implementation after f93df5ebd99
0247f88ab6f6 merge main into amd-staging
2e4aa7855b99 merge main into amd-staging (#2863)
9a0a9764f38a [Clang][AArch64] _interlockedbittestand{set,reset}64_{acq,rel,nf} support for AArch64 (#145980)
ec48d15b2003 Fix a bug in the breakpoint ID verifier in CommandObjectBreakpoint. (#145994)
c3811c8474f4 [lldb][scripts] Use named args in versioning script (#145993)
89513a0cb4b8 merge main into amd-staging (#2862)
f93df5ebd992 [ORC] Add read operations to orc::MemoryAccess. (#145834)
76f3cc9e0453 [lldb] Fix another race condition in Target::GetExecutableModule (#145991)
018548ddffea [objcopy][coff] Place section name first in strtab (#145266)
3df36a2b1872 [AMDGPU] Enable vectorization of i8 values. (#134934)
9e4981cf1189 [NFCI][msan] Add test for "disjoint" OR (#145982)
45639186cd54 merge main into amd-staging
c73e5e3e209c [lldb][scripts] Fix bugs in framework fix script (#145961)
4e154ccaf17f [bazel] Add missing deps needed for parse_headers after #145313 (#145978)
772eb075674b [VPlan] Clarify comments after #145340 (NFC).
aeec2c6e4893 [VPlan] Speed up VPSlotTracker by using ModuleSlotTracker (#139881)
e811383f3353 [gn build] Port d7b936b633f3
ca0565aa0b4c [gn build] Port 33872f12187b
8f62c0c3dadb [gn build] Port 0faa181434cf
6116f5acc615 [gn build] Port 0a3c5c42a178
063cee7bdeef [AMDGPU][MC] Allow opsel for v_max_i16 etc in GFX10 (#143982)
9727781b5727 locally land: [Clang] Determine offloading architectures at Toolchain creation (#145799)
1dfdd1e6dea1 [mlir][emitC] Add support to emitter for `classop`, `fieldop` and `getfieldop` (#145605)
fb138a675e4a [gn] port 40cc4379cda6e0d
64f16f48965e [gn] port e175ecff93628
30c61119b932 [MergeICmps] Fix miss-compile in MergeICmps in presence of blockaddresses (#145925)
a9ed84b61808 [clang] ms-abi: member pointer inheritance model lock-down fix (#145958)
d699fbd2033e [OpenACC][Docs] Add a release note for Clang 21 (#145938)
5d03e7a2047c Revert "[NFC][DebugInfo][DWARF] Create new low-level dwarf library (#… (#145959)
0f1fd8735ca1 [gn] fix `gn gen` after 2a907f40bcb1f26c
7849590a5fe4 merge main into amd-staging
df5097dd946e [Offload] Add default for HSA agent type to silence warning (#145943)
938cdb30f16f [flang] Migrate away from std::nullopt (NFC) (#145928)
a0c5f1992d21 [SystemZ][zOS] disable _Float16 support on z/OS (#145532)
283c2e8d7c9f [flang][semantics] fix issue with equality of min/max in module files (#145824)
e25db2f6b30d [CostModel] getInstructionCost - match SK_InsertSubvector shuffle patterns before SK_Select (#145920)
72ffa799c847 [NVPTX] tryStoreParam - remove default-only switch statement. NFC. (#145948)
70233c61d659 Add minimum count threshold for indirect call promotion (#145282)
4308292d1e3f [BOLT] Refactor NewTextSegmentAddress handling (#145950)
b0500f56c8b6 Revert "[clang][python][test] Move python binding tests to lit framework" (#145951)
0ef27186c931 [tests] Additional coverage for gather/scatter address optimizations
947e072744ae [CodeGen/RISCV] Add fixed-vector [l]lround tests (#145926)
232c2921e19a Reland [mlir][OpenMP] Use correct debug location with link clause. (#145889)
cbf781f0bdf2 [NFC][DebugInfo][DWARF] Create new low-level dwarf library (#145081)
f2991bf79109 [AMDGPU] Convert 64-bit sra to 32-bit if shift amt >= 32 (#144421)
442c417749d2 [libc][bazel] Support generating public libc headers in Bazel builds. (#145381)
66ec14171ede [libc][bazel] Add bazel targets for libc/include/... tests. (#141150)
3c5b17fb980f merge main into amd-staging (#2851)
edf0d0da435c [OpenACc][CIR] Fix tests failed because of crossed-patches
6b9e6f066f84 [Hexagon] Remove emplace workaround (NFC) (#145718)
592f203ee6d3 [X86] abds-vector-128.ll - regenerate VPTERNLOG asm comment
30d861f2e873 [clang-scan-deps] Enable test P1689.cppm on Windows (#145857)
655d04859be6 [GOFF] Add writing of text records (#137235)
e880cf74217d Reland "[lldb][target] Add progress report for wait-attaching to proc… (#145111)
4c8fab399b2e [libc++] Avoid type-punning between __hash_value_type and pair (#143501)
7842e9eada2e [mlir][Vector] Lower `vector.to_elements` to LLVM (#145766)
abf8e25ac764 [DAG] Add `SDPatternMatch::m_Load` (#145481)
1d60d91bf3ae [HLSL][SPIRV] Add option to add all KHR extensions (#145536)
0a2b6f6c1cc7 [OpenACC] Fix 'copyout' allowed modifiers alwaysin vs alwaysout
0b8a656ba110 [lldb-dap] Fix source references (#144364)
5c310d1ef018 Default transcript dumping in "statistics dump" to false (#145436)
066a14d4d483 [clang-doc] refactor JSONGenerator array usage (#145595)
e0c7136713d2 [LowerAllowCheck] preserve CFG analyses (#145816)
9a44f55c00f7 [sanitizers] do not accept out of bounds -fsanitize-skip-hot-cutoff (#145806)
32aa80c362bf [NFCI] [LowerAllowCheck] also test lower-allow-check-percentile-cutoff-hot (#145822)
8a6519677c08 [RISCV] Move CascadeSelect test into float/half/double-select-fcmp.ll. NFC
d83457eb83d8 [NFC][lldb-dap] Add missing header guard for DAPError and ProgressEvent. (#145915)
48e893792a33 [Clang] Determine offloading architectures at Toolchain creation (#145799)
928a7e6cb933 [AArch64] Use 0-cycle reg2reg MOVs for FPR32, FPR16, FPR8 (#144152)
f0c1a9a85d18 [DAG] Add SDPatternMatch::m_Result to match a specific SDValue result (#145775)
720d7e09b322 [CIR][NFC] Fix an unused variable warning (#145922)
69bbf2135e7d [OpenACC][CIR] Implement 'modifier-list' lowering (#145770)
2e3995981cb3 [HLSL][RootSignature] Use "stringified" version for diagnostic output of punctuator tokens (#145827)
aeea062dd43e [lldb][DataFormatter] Unwrap reference type when formatting std::unordered_map (#145872)
e0b83ca8a409 [MLIR][ArmNeon] Add a couple of negative tests for BFMMLA with scalable dimensions (#145882)
9d91b07e1e8b [CIR] Implement EqualOp for ComplexType (#145769)
07e3c859b0cb [FileCheck][NFC] Pre-commit test for printing escaped variables (#145906)
5a996c6ed760 Add triple target to buildtype.ll in X86 (#145764)
33872f12187b [GOFF] Add writing of section symbols (#133799)
6bdfecaea837 [clang] Revise relnote for int->enum conversion. (#145755)
04b60d29ba7f Disable HWSan for thread_creae_failure.pass.
abc2c3a53805 [mlir] Use llvm::is_contained instead of llvm::all_of (NFC) (#145845)
3d5903c4d8a6 [llvm] Use llvm::is_contained (NFC) (#145844)
31122446c90e [clang] Use llvm::is_contained instead of llvm::all_of (NFC) (#145843)
70dce3d987ce [mlir] Migrate away from std::nullopt (NFC) (#145842)
87729bcbb817 [clangd] Migrate away from std::nullopt (NFC) (#145841)
a13cf846e0c6 [ASTMatchers] Migrate away from ArrayRef(std::nullopt) (NFC) (#145840)
620dbf1fec47 [llvm] Use llvm::interleaved (NFC) (#145839)
7eec132f9c0d [lldb] Fix source/Host/common/Editline.cpp formatting (NFC)
775a69b237d0 [OpenMP] Fix comma -> semicolon (#145900)
08964d69b069 [HLSL][SPIRV] Handle `uint` type for spec constant (#145577)
4233ca1e4e53 [OMPIRBuilder] Fix use of uninitialized variable. (#145883)
5bbe1536dfa6 Cap IntRange::Width to MaxWidth (#145356)
0f87c78f647c [GitHub] Add PR filters for VectorCombine patches (#145896)
d13e223a89b0 [MLIR][AArch64] Add integration test for lowering of `vector.contract` to Neon FEAT_I8MM (#144699)
1a60c74c131a [CostModel][X86] SK_InsertSubvector inserted into the lowest subvector should be treated as SK_Select blend (#145892)
817da3e996a1 merge main into amd-staging
06a4394b5036 Disable HWSan for thread_creae_failure.pass.
2bece2194a12 [mlir][mesh] resubmitting #144079 (#145897)
5bf43634c450 [mlir][affine] NFC Rename SimplifyAffineMinMax -> SimplifyAffineMinMa… (#145905)
90c9cc2c9810 [clang][python][test] Move python binding tests to lit framework (#145855)
58b7d200aa51 [libc] Fix the risc-v half precison float feature check (#145894)
67302b2e6f81 [AMDGPU][NFC] rename some constants for readability (#145870)
4ce96987e5e5 merge main into amd-staging (#2846)
d05634d5cdc5 [VectorCombine] Fold bitwise operations of bitcasts into bitcast of bitwise operation (#137322)
cfdc4c4a5b67 Revert "[flang][OpenMP] Move lowering of ATOMIC to separate file, NFC (#144960)"
597ffb1187dd Revert "[flang][OpenMP] Fix namespace nesting after PR144960"
a1d5b9d1cda4 [mlir][affine] Wrap SimplifyAffineMinMax in a pass (#145741)
90f3147424a3 [Sema] Cut the deep_recursion.c test in half to fix tests in asan mode
5b76cdba5aac [VPlan] Handle AnyOf when unrolling. (#145340)
fe4b4033edf2 [X86] lowerShuffleAsVTRUNC - use combineConcatVectorOps to catch more "cheap" concats (#145876)
cc1eae6ea2de [flang][OpenMP] Remove experimental warning (#144915)
6a5469bb815d [bazel] Fixes for e5a8c51c9dc85a7b463a4570942e3e5e1cb70e0b
afc6c2bb9b43 [Clang] Allow the use of [[gnu::visibility]] with #pragma clang attribute (#145653)
d59e0ba80b85 [lldb][docs] Document qWatchpointSupportInfo (#145709)
e5a8c51c9dc8 [mlir][tensor] Make tensor::PadOp a ReifyRankedShapedTypeOpInterface (#145867)
af7166a3f126 [AArch64] Ensure the LR is preserved if we must call __arm_get_current_vg (#145760)
35a0c18b88e0 [GOFF] Introduce GOFFWriter class (#131216)
40cc4379cda6 [NFC][analyzer] Remove Z3-as-constraint-manager hacks from lit test code (#145731)
2a907f40bcb1 [gn] sync dbb3e7d1162d (ish)
0b6ddb02efdc [clang] NFC: Add alias for std::pair<FileID, unsigned> used in SourceLocation (#145711)
548e8e92e98d [X86] [clang] Add missing check line for diamondrapids (NFC) (#145542)
635acfbfca47 [LLVM][AArch64] Relax SVE/SME codegen predicates for crypto and bitperm instructions. (#145696)
3287c1c17655 [XeGPU] Move targetinfo constants to their own header file
b77114b723eb [lldb] Remove child_process_inherit argument from Pipe (#145516)
f64d5df3c252 [LLVM][SME] Only FADD/FSUB are enabled by either sme-f16f16 or sme-f8f16. (#145704)
6307b496f8ba [AMDGPU] Add `GCNRPTarget` to track register pressure against a target  (#145765)
d41f5fe9e40e merge main into amd-staging
95149011ea01 Guard against self-assignment; NFC (#145743)
21def215b53e [Utils] Drop const from a return type (NFC) (#145838)
9ae41f017d8e [lldb][Darwin] revert change to lang_opts.BuiltinHeadersInSystemModules (#145864)
d3fd7921d4d7 Clarify some code based on static analysis complaints; NFC (#145679)
a19ddff98013 [LoongArch] Pre-commit test for fixing xvshuf instructions. NFC
30de98c283f5 Revert "[-Wunterminated-string-initialization] Handle C string literals ending with explicit '\0' (#143487)"
0cc10383f4bc [X86] avx512-vbroadcast.ll - regenerate VPTERNLOG asm comment
0905767610d6 [libc++] Address remaining comments from #130145
163871c2d269 [Support] Remove workarounds for building with mingw.org toolchains (#145683)
aa2402931908 [VPlan] Unroll VPReplicateRecipe by VF. (#142433)
696c0f92e0fe [libc++] Don't try to wait on a thread that hasn't started in std::async, take 2 (#130145)
4f1b362ffa9c merge main into amd-staging (#2839)
13daf656561d [flang] handle common block used as BIND(C) module variables (#145669)
a2265423d0ba [MLIR][ArmNeon] Add an ArmNeon operation which maps to `bfmmla` (#145038)
7a5af4f6b850 [BOLT] Gadget scanner: detect untrusted LR before tail call (#137224)
2e196a0ed14c [DOCS][BUILDBOT] Bump recommended worker version (#144853)
8b3e345002f3 [ADT] Clean up TinyPtrVector
e8f85cf51fb9 [ArrayRef] Bring MutableArrayRef's constructor in line with ArrayRef
d144eb1d8cd3 [Clang] Back out the source location workaround for CXXConstructExpr (#145260)
741e1ca3b331 [AMDGPU] upgrade test to use auto-generated llc checks (#145832)
a16126844750 [InstCombine] Add test for freeze of GEP with recurrence offset (NFC) (#145541)
e9e25f02e6e1 [mlir][vector] Restrict vector.insert/vector.extract to disallow 0-d vectors (#121458)
d2ca10ab75bd [AArch64] Match indexed forms of fmul/fmla/fmls (#144892)
9903c1936a5d [-Wunterminated-string-initialization] Handle C string literals ending with explicit '\0' (#143487)
1b914e15cf21 [lldb][test] Remove debug code from TestCommandScriptAlias.py
249f074b2253 [BOLT][AArch64] Make gs-pacret-autiasp.s deterministic (#145527)
d681c73a04bc [BOLT] Create marker for source changes in nfc-mode testing. (#142931)
a8998fa06261 [mlir][vector][nfc] Move vector.splat test (#145699)
7289b6789ef3 [libc] Improve memcpy for ARM Cortex-M supporting unaligned accesses. (#144872)
0e268791c561 [SpeculativeJIT] Add missing include
f070c74e3543 [AMDGPU][SDAG] Tests for target-specific ISD::PTRADD combines (#143672)
78b7f6a03d0f [Sema] Avoid deep recursion in AnalyzeImplicitConversions (#145734)
c225d6dee011 [AMDGPU][SDAG] Handle ISD::PTRADD in SelectionDAGAddressAnalysis (#142778)
616816468bd2 [AMDGPU][SDAG] Add test for ISD::PTRADD handling in SelectionDAGAddressAnalysis (#142777)
215e61c08847 [AMDGPU][SDAG] Add ISD::PTRADD DAG combines (#142739)
28aa871dd434 AArch64: Remove AArch64MCExpr.h
8b55129d46c9 [llvm-debuginfo-analyzer] Fix incorrect REQUIRES in WebAssembly test case (#145848)
7e67307fea97 AArch64: Remove deprecated AArch64MCExpr::Specifier
402480325b4e [AMDGPU] More tests for S_WAIT_XCNT insertion (#145681)
c91cbafad211 TargetLibraryInfo: Delete default TargetLibraryInfoImpl constructor (#145826)
418c5de19bbd PowerPC: Move PPCMCExpr into PPCMCAsmInfo
96ed2abadf90 [DebugInfo] Specify x86_64 triple for test (#145797)
d46a69cab4e3 [PredicateInfo] Use BumpPtrAllocator for predicates (NFC) (#145692)
5fb0ae1a5b58 [LegacyPM] Remove unused getAdjustedAnalysisPointer() method (NFC) (#145738)
9ccf613b3426 [mlir][Transforms][NFC] Store per-pattern IR modifications in separate state (#145319)
a98e01f3c5ab Regenerate newly added test
fff720d6419b Triple: Forward declare Twine and remove include (#145685)
39bb67391567 Merge llvm/main into amd-debug
c8243251cb25 [RISCV] Remove separate immediate condition codes from RISCVCC. NFC (#145762)
a6e524276e2c [Modules] Add merged Files to UsedModuleFiles
ce8f1600d042 [RISCV] Use SelectAddrRegRegScale for Xqcisls instructions. (#145608)
49847148d4b1 [BOLT] Fix density for jump-through functions (#145619)
4cb8308ee9cb AMDGPU: Avoid report_fatal_error for unsupported ds_ordered_count (#145172)
8b7fc6487d56 [llvm-debuginfo-analyzer] Fix crash with WebAssembly dead code (#141616)
3ee8dabcc016 [CMake] Fix for #144598: Move LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS backwards
cca44e030ddd [RISCV] Explicitly check for supported opcodes in optimizeCondBranch. NFC (#145622)
a440aa079dcc merge main into amd-staging
b6bca4a3fd19 merge main into amd-staging (#2838)
5b5e95ceb8d6 [RISCV] Add more test cases for Xqcisls scaled load store instructions. NFC
ae05dbbb04ef [NFC][AMDGPU] Add a test that can cause backend crash (#145777)
117ed65c0720 [clang-format][NFC] Remove a redundant check for null
25a6f7ae21cb merge main into amd-staging (#2837)
3f3282cee87f [AMDGPU] Adding AMDGPU dialect wrapper for ROCDL transpose loads. (#145395)
e2cc82b1777c [NFC] [LowerAllowCheck] use InstIterator (#145817)
13e06403b4ed [GlobalISel] Remove dead code. (NFC) (#145811)
16e712e7c308 [NVPTX] Allow directly storing immediates to improve readability (#145552)
46c8cc7e5172 [libc] Enable all f16 math functions on the GPU (#145813)
2f9eee849f7d [RISCV] Implement intrinsics for XAndesVBFHCVT (#145634)
5f69f2e6513b merge main into amd-staging (#2833)
0529a346007c [clang][Preprocessor] Handle the first pp-token in EnterMainSourceFile (#145244)
63b14b3f2bf6 [RISCV] Add missing required features for Zvfbfmin intrinsics (#145646)
09b52653781f [RISCV] Decrease the capacity of SmallVector to 6. NFC. (#145650)
36fbc6a8d23c [MLIR][XeGPU] Remove the transpose attribute from Gather/Scatter ops and Cleanup the documents (#145389)
0faa181434cf [ORC] Extract MemoryAccess from ExecutorProcessControl, break up header. (#145671)
d36b0f738261 merge main into amd-staging
b413bea4355e [NFC] [sanitizer] get rid of references to fsanitize-top-hot (#145810)
967540bbd127 Revert "Non constant size and offset in DWARF (#141106)"
c539ec0db53a [mlir][vector] Add support for vector extract/insert_strided_slice in vector distribution.   (#145421)
2b2bd51f3b35 [flang][cuda] Inline this_grid call for cooperative groups (#145796)
f63bc84b0d2a [lldb] Fix FindProcessImpl() for iOS simulators (#139174)
19797500d85c [NFC] [sanitizer] fix inaccurate comment (#145809)
a5baf5302adf Add modified runtimes back to runtimes_to_{test|build} (#145805)
82cbd68504e6 [NFC][PGO] Use constants rather than free strings for metadata labels (#145721)
8ef76c7496b5 merge main into amd-staging
00f6d6ab66a7 [NFC][CodingStandard] Require``[[maybe_unused]]`` for unused variables in asserts (#142850)
3d5149062288 [RISCV] Fix typo in the description of xsfvfwmaccqqq. (#145771)
e1f224b99a91 AMDGPU: Handle folding vector splats of inline split f64 inline immediates (#140878)
472c9141f9d0 AMDGPU: Fix tracking subreg defs when folding through reg_sequence (#140608)
7381d816f31c [lldb][NFC] remove the ResolveSDKPathFromDebugInfo method (#145744)
ff0dcc461474 [MLIR][Linalg] Harden parsing Linalg named ops (#145337)
ac29858e2d7f AMDGPU: Remove global_atomic_ordered_add_b64 from gfx1250 support (#145781)
79da5fecdb70 [CIR][NFC] Remove duplicate code (#145790)
bd86fc925238 [libc] Error fixes for mbrtowc and wcrtomb (#145785)
17f91a26fff2 [AMDGPU] Add gfx1250 run lines to gfx12_dasm_vop3_dpp8.txt. NFC. (#145778)
c72507d43149 [FIX] Add type definition into test case `clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl`
38aec4f1f422 [DirectX] Limit GEP transformations to Arrays (#145758)
a8ef75e75898 [DirectX] Strip `dx.rootsignatures` metadata during `dxil-prepare` (#145746)
ce1c1a0e6d6b [AMDGPU] Add support for `v_cvt_pk_f16_bf8` on gfx1250 (#145753)
ff23ee40d6af [AMDGPU] Add support for `v_cvt_pk_f16_fp8` on gfx1250 (#145747)
6fd182a3bb28 [RISCV] Support fixed vector vp.reverse/splice with Zvfhmin/Zvfbfmin. (#145596)
e8abdfc88ffe [lldb] Make MCP server instance global (#145616)
2db0289abe17 [libc] Implemented wctomb (#145554)
695264590b9a merge main into amd-staging
0c359d747daa Revert "Move python binding tests to lit framework (#142948)" (#145774)
62f8281e0842 [IR][PGO] Verify invalid `MD_prof` metadata on instructions (#145576)
fcbfff193e96 AMDGPU: Remove floating point dot4 and dot2 from gfx1250 support (#145751)
bc905417d953 [libclang/python][NFC] Fix formatting in test_file (#145773)
4a31f7f8ea77  [ConstantFold] Special case atan +/-0.0 (#143962)
473f992c1f14 [AMDGPU] Add the support for `v_cvt_f32_bf16` on gfx1250 (#145632)
ad0fba211cc9 [llvm] annotate remaining CodeGen and CodeGenTypes library interfaces for DLL export (#145361)
f8b906ef0f7b [CIR][NFC] Sort the 'Misc' section of MissingFeatures (#145750)
b6b08117bdb4 [AMDGPU] Simplify S_WAIT_XCNT insertion. NFC. (#145682)
7dfcced079e3 [clang][python][test] Move python binding tests to lit framework (#142948)
3b8ac7a22c64 [OpenMP] Fix various alignment issues (#142376)
5781d526ecf2 [llvm] annotate remaining ExecutionEngine, Frontend, and TargetParser interfaces for DLL export (#145367)
1abe1aa7b2b4 [llvm] annotate remaining Analysis library interfaces for DLL export (#145359)
d9a7b1647984 InterlockedAdd_*, InterlockedAdd64_* support for AArch64 (#145607)
aec88679db5f   [OpenMP] avoid segv for a lock that has already been destroyed (#145625)
7aeb1a1fd8bd Fix Solaris buildbot (#145737)
a75279e4a5b5 Revert "[mlir][OpenMP] Use correct debug location with link clause." (#145768)
1c56d101a214 [libc] Added isValidState to CharacterConverter class to ensure a provided mbstate is valid (#145564)
20f56d140909 [libc++abi] Fix C++ demangling for _BitInt type (#143466)
c873e5f87d84 [mlir][TilingInterface] Handle multi operand consumer fusion. (#145193)
28f6f870617f [mlir] Migrate away from std::nullopt (NFC) (#145523)
d760f9738787 [Clang] Implement diagnostics for why `std::is_standard_layout` is false (#144161)
70333de6cfa7 [NVPTX] Consolidate and cleanup various NVPTXISD nodes (NFC) (#145581)
5f8e7ed5a38c [mlir][linalg][nfc] Split hoisting tests into dedicated test functions (#145234)
369cbcc1a21a [lldb] include `LLVMTargetParser` in `LINK_COMPONENTS` (#145606)
626be98c3544 [mlir] Remove dependency on LinalgTransformOps.h from #144657 (#145749)
3b90597c2cea Non constant size and offset in DWARF (#141106)
487581b826a5 [lldb][darwin] force BuiltinHeadersInSystemModules to be always false (#144913)
357297c0f283 [bolt] Fix the build after 0556a2aa187b86c28a9441aec3e98b9780a2c9ee
1e45ea12db5e [CIR] Add support for function linkage and visibility (#145600)
1276a5b36849 [CIR] Upstream support for builtin_vectorelements (#144877)
e4da49f0183b [CIR] Upstream __imag__ for ComplexType (#144262)
585ed213a859 [NFC][Clang][Preprocessor] Refine the implementation of isNextPPTokenOneOf (#145546)
0556a2aa187b [ArrayRef] Provide constructors from any type with a data() member (#145735)
df79c40c988a [clang-tidy Fix a warning
2a35414e9888 [Transforms] Use range-based for loops (NFC) (#145252)
86026ee623cd [clang-tidy] Warn about misuse of sizeof operator in loops. (#143205)
532c15a718d1 [mlir][linalg] Fix module dependency issue due to unused import (#145727)
b4f4af7ebb91 [flang][OpenMP] Verify that N in -fopenmp-version=N is valid (#145725)
5a194c1fd97b [msan] Sharpen instrumentation of Intrinsic::{ctlz,cttz} (#145609)
9b307ebdd051 [bazel] Update deps from #144657 (#145726)
dd47b845a62c [clang-format] Handle Trailing Whitespace After Line Continuation (P2223R2) (#145243)
9a7720ad2f96 [CIR] Upstream __real__ for ComplexType (#144261)
3e337bc30802 [Offload] Add a stub unloadBinaryImpl for host device (#145716)
029823a84de9 [Clang][NFC] Clean up OpenMP offload toolchain generation (#145549)
a76448c27de2 [Clang] Fix replaceability/relocatability computation (#145655)
c92b580d6308 [mlir][ArmSME] Remove `ConvertIllegalShapeCastOpsToTransposes` (#139706)
c08502defe47 [MLIR][Transform] expose transform.debug extension in Python (#145550)
46ee7f1908f4 [libclc] Avoid out-of-range float-to-int. (#145698)
8202c94cec41 [CostModel][X86] getMaskedMemoryOpCost - widening masks must compute the cost of the full width insert_subvector across multiple legal vectors (#145693)
81e07996aa8b [AMDGPU][SIInsertWaitcnts] don't crash when printing messages at end of block (#145694)
60285d98efa6 [asan] Implement address sanitizer on AIX: build configuration (#139583)
c3b160bcaa7d [RISCV] Remove -mattr=+no-rvc-hints (#145138)
2dfcc30e3800 [AArch64] Add tests for inefficient LD1lanePost. NFC
77a3ae5845da [flang][OpenMP] Remove recognition of versions 3.0 and older (#145708)
287b24e1899e [asan] Implement address sanitizer on AIX: address descriptions (#138891)
ea1e18157133 [MLIR][AArch64] Lower `vector.contract` with mixed signed/unsigned arguments to Neon FEAT_I8MM (#144698)
36819eaed124 [llvm-objdump] Support --symbolize-operand on AArch64
6bc708728735 merge main into amd-staging (#2826)
10edc3df99f7 [RISCV] Try to optimize `vp.splice` to `vslide1up`. (#144871)
020fefb6af01 AMDGPU: Avoid report_fatal_error in image intrinsic lowering (#145201)
75de95ddd28f [Option] Use llvm::interlaved (NFC) (#145642)
817c097887e0 [Analysis] Drop const from a return type (NFC) (#145641)
050f6283eeaa [AMDGPU] Drop const from return types (NFC) (#145640)
fb6882719a51 Triple: Remove workaround for gcc 4.0.3 (#145660)
c9910d5ef8b7 [LangRef] Clarify definition of fragments (#145582)
096cad59dff5 [Serialization] Fix a warning
ae947567ae6f merge main into amd-staging (#2824)
b8e8ff3dd669 [Clang] Implement diagnostics for why is_empty is false (#145044)
a945fb1481e0 [clang] NFC, move the SourceLocationEncoding.h header guard to the begining of the file
e64f5dd94847 Revert "[clang][dataflow] Expose simple access to child StorageLocation presence." (#145710)
b8bda9d446e7 [clang][analyzer] Correctly handle lambda-converted function pointers (#144906)
022e1e99f3b0 [mlir][memref]: Fix Bug in GlobalOp Verifier (#144900)
d76fdf7f536c [clang-c] introduce queries on GCC-style inline assembly statements (#143424)
fd3cc204de96 [SelectionDAG] Fold undemanded operand to UNDEF for VECTOR_SHUFFLE (#145524)
41457bc9d2b5 [NFC][AsmPrinter] Rework/add comments (#145587)
2bf0ff3254a9 [X86] var-permute-256.ll - add missing check prefixes for AVX512F/DQ only targets
0870c8838ba2 [Offload] Add an `unloadBinary` interface to PluginInterface (#143873)
e90ab0e34277 [Serialization] Remove delta encoding optimization (#145670)
183ff08b32ff [libclc] Add missing deps for LLVM_ENABLE_RUNTIMES. (#145684)
e99d8bb0dcc5 [PowerPC] eliminate RLWINM instruction following LBARX as possible (#144089)
31bf9348fa10 [MLIR][NVVM] Add an explicit mask operand to elect.sync (#145509)
77618a9253d2 Delete copy constructor/assignment; NFC (#145689)
01174ff2babe Remove unneeded checks for null; NFC (#145686)
d16f42d1e227 [mlir][linalg] Constrain the parameters m, r in Winograd ops (#144657)
4bb5e48fb933 [LoongArch] Add codegen support for ILP32D calling convention (#141539)
d7b936b633f3 [OpenMP] Add directive spellings introduced in OpenMP 6.0 (#141772)
006037675c10 [mlir][OpenMP] Use correct debug location with link clause. (#145026)
3ef796b981f1 [Clang] Diagnose unsatisfied `std::is_assignable`. (#144836)
d3ed84ed6717 [Utils][mlir] Fix interaction between CodeExtractor and OpenMPIRBuilder (#145051)
2db3cc4f3470 [AArch64][CostModel] Lower cost of dupq (SVE2.1) (#144918)
bb7242477c2f [MC] Use StringTable to reduce dynamic relocations (#144202)
8231dd71cb7d [flang][OpenMP] Skip runtime mapping with no offload targets (#145594)
7ff0d28f2e1a Triple: Remove redundant member initializers (#145661)
daedad8b3d56 merge main into amd-staging
7fcd2ca2efc7 [lldb-dap][test] Import skipIfAsan decorator
c941bee75d25 [lldb] Fix qEcho message handling. (#145675)
97fdc237ddda [NFC] [Modules] Adding profiling points to template lookups in serializer
8e4fb4beada4 [IA] Remove recursive [de]interleaving support (#143875)
14e280e44e05 [lldb-dap][test] Skip TestDAP_runInTerminal/TestDAP_restart_runInTerminal on ASAN
0036beeeefd1 [bazel] Port e175ecff936287823b5443d7b2d443fc6569f31f
46b97a836a89 Fix signed/unsigned comparison warning. NFC.
62d36857b542 RuntimeLibcalls: Cleanup darwin bzero configuration (#145639)
9e8152b51491 RuntimeLibcalls: Cleanup darwin exp10 case (#145638)
b991b21874ed Triple: Add BridgeOS to isOSDarwin (#145636)
acab82b4c344 Regen CodeGen/AMDGPU/global-load-xcnt.ll
54953b922d11 Handle vendor string in match for clang version string (#145612)
630d55cce45f [AsmPrinter] Always emit global equivalents if there is non-global uses (#145648)
892a263106af [flang] Add release note for compiler-rt dependency on AArch64 (#145547)
6f47f4b9e58f [MIPS][MSA] compare_float.ll - regenerate checks
9fde72e72987 [AArch64] Use `-filetype=null` in test (NFC) (#145662)
992f0d12255a [Clang][SPIRV][AMDGPU] Override `supportsLibCall` for AMDGCNSPIRV (#143814)
eeb206d688b7 [SystemZ] vec-max-min-zerosplat.ll - regenerate checks
a8a2c6fa882d [BOLT] Gadget scanner: fix LR to be safe in leaf functions without CFG (#141824)
36a060a4e51b [ORC] Sink DynamicLibrary.h header include into ExecutorProcessControl.cpp.
d223832a5888 [TableGen] Use getValue instead of getInt for enum attributes (#144030)
1060d8910f12 [TTI] improveShuffleKindFromMask - pull out repeated Mask.size() calls. NFC.
4efb61850b59 [C++20] [Modules] Handling template declare with debug info
22ee837ec0ae [flang][NFC] do not copy fields in fir::RecordType::getTypeList (#145530)
5d2ece14378e [X86] Allow znver3/4/5 targets to use double-shift instructions by default (#132720)
3a3d1bf4a39b [lldb][AArch64] Handle core file tag segments missing tag data (#145338)
97b8cec789fe [lldb][lldb-dap] Skip runInTerminalWithObjectEnv test on Windows
20a72083fdff [BOLT] Gadget scanner: improve handling of unreachable basic blocks (#136183)
29f8dca0853c [bazel] Add missing dependency for e93a0d0d1ed790173e23247fa2833cdac30b8268
32febe60f364 [PowerPC] Support for Packed BCD conversion builtins (#142723)
0f01cd528914 [X86][GlobalIsel] support G_FABS (#136718)
f2f4b557aa7a [libc++] P2944R3: Constrained comparisions - `variant` (#141396)
88297cca8f8c [AMDGPU] Correct -mcpu processor name in test (#145493)
d0469d1d3c31 [mlir] Move WalkResult to Support (#145649)
ad87d951c94f [clang] Fix __builtin_mul_overflow for big _BitInts (#145497)
ecff028a9662 [LV] Update test after 4ac4726d00.
3b251cd675fb [MLIR] Legalize certain `vector.transfer_read` ops of scalable vectors (#143146)
4f9adb688941 [mlir][vector] Relax operand type restrictions for `vector.splat` (#145517)
17c5c19902a8 [Verifier] Always verify all assume bundles. (#145586)
4ac4726d0064 [VPlan] Format some print forms.NFC (#144644)
d12fb1fc3714 [WebAssembly] Refactor PerformSETCCCombine (#144875)
d32239b461ed [ORC] Move UnsupportedExecutorProcessControl into unittests.
aeaf319b8c22 [flang] avoid useless rebox of polymorphic scalars (#145507)
31545ca5f467 [Docs] Incorporate the recent discussed changes to release process (#145285)
237b8de2c0d9 [IfConversion] Fix bug related to !HasFallThrough (#145471)
7c38ee26d441 [FunctionAttrs][IR] Fix memory attr inference for volatile mem intrinsics (#122926)
5238f06f7234 [IfConversion] Pre-commit testcase for !HasFallThrough bug. NFC
956bab0381e9 [TSan] Add 2 test cases related to incomplete shadow cleanup in unmap (#145472)
4f4e4b547720 merge main into amd-staging
d95f46ca84d7 [IR] Fix incorrect writeonly on llvm.allow.ubsan/runtime.check (#145492)
01b288fe6a1e [clang-format] Improve QualifierAlignment in guessing macros (#145468)
34bfa4e7898c [RISCV] Fix incorrect chapter number in comment in RISCVSchedSiFive7.td. NFC.
c3c923c8d62c [X86][GlobalISel] Enable SINCOS with libcall mapping (#142438)
6f186bdd8808 merge main into amd-staging (#2819)
a0ce3e691c19 [C++20] [Modules] Avoid crash with calls to (this auto) syntax
0512d119fdf0 [lldb] Clean up GDBRemoteCommunication::StartDebugserverProcess (#145021)
888f84f72c07 [ARM] Return the correct chain when expanding READ_REGISTER (#145237)
13a9b86f6279 [NFC][libclc] Replace and delete _CLC_DEFINE_UNARY/BINARY/TERNARY_BUILTIN macros (#145458)
2550a637a127 AMDGPU: Remove LDS-direct(param)-loads and VINTERP ops from gfx1250 support (#145631)
494a74882b26 Reapply "ELF: Add branch-to-branch optimization."
08b8d467d425 [AMDGPU][GFX1250] Insert S_WAIT_XCNT for SMEM and VMEM load-stores (#145566)
deb346419264 [RISCV] Add Tied operands in Xqcicm instructions and changes to handle a single tied operand in source DAG and instruction (#145538)
7ccc59143136 Merge llvm/main into amd-debug
901e1390c977 [SelectionDAG] Remove UnsafeFPMath check in `visitFADDForFMACombine` (#127770)
948cc91188e5 Reapply "[Win/X86] Make _m_prefetch[w] builtins to avoid winnt.h conflicts (#115099)" (#138360)
77af8bff97a0 [mlir]Moves the StateStack to IR folder from Support folder. (#145598)
505906bff6dd [AMDGPU][True16][CodeGen] do not legalize t16 operand during user scan (#145450)
c85466dcd485 Reapply "[msan] Automatically print shadow for failing outlined checks" (#145611) (#145615)
7269d1d6b4fd merge main into amd-staging
3de2af3ef56c AMDGPU: Remove export and related instructions from gfx1250 support (#145624)
215b12f963b1 Merge llvm/main into amd-debug
30922f740e47 Move relocation specifier constants to AArch64::
ed5f8f238d68 [LLVM][DecoderEmitter] Add option to use function table in decodeToMCInst (#144814)
f608716b0986 [LLVM][TableGen] Minor cleanup in CGIOperandList  (#142721)
dbb3e7d1162d [clang] Register all LLVM targets in AllClangUnitTest main (#144428)
3140333872f6 [-Wunsafe-buffer-usage] Support span creation from std::initializer_list and begin/end pairs (#145311)
e6d6bb52980e AMDGPU: Remove s_waitcnt from gfx1250 support (#145620)
c0a362a09c1a Merge llvm/main into amd-debug
d362b43a1f12 merge main into amd-staging (#2811)
c04e804a3117 [lldb] Eliminate check for HasLoadedSections (NFC) (#145366)
a451fff1ad43 [llvm] fix extern cl::opt definitions for DLL export (#145374)
e93a0d0d1ed7 [HLSL][RootSignature] Add `fdx-rootsignature-version` option to specify root signature version (#144813)
52fbefb281ea [clang][clang-scan-deps] Add named modules to format 'experimental-full' (#145221)
87ce754e9847  [llvm] annotate remaining BinaryFormat and DebugInfo library interfaces for DLL export (#145364)
3574560b6b3c   [llvm] annotate remaining Support and ADT library interfaces for DLL export (#145354)
5ae00a7ce228 [llvm] annotate remaining Telemetry and ToolDrivers interfaces for DLL export (#145369)
c84d620d4d3f [llvm] annotate WindowsDriver and WIndowsManifest interfaces for DLL export (#145567)
f37d944152ab [AArch64][SME] Use reportFatalUsageError rather than assert (NFC) (#145491)
5eb5f0d8760c Revert "[msan] Automatically print shadow for failing outlined checks" (#145611)
d06c2efd67e5 [AMDGPU] Support v_lshl_add_u64 in gfx1250 (#145591)
51d1385cdf5e [clang Dependency Scanning] Enhance File Caching Diagnostics (#144105)
48a21e69159a [RISCV] Fix a correctness issue in optimizeCondBranch. Prevent optimizing compare with x0. NFC (#145440)
32451484113e Merge llvm/main into amd-debug
9702d37062a3 [RISCV] Support scalable vector vp.reverse/splice with Zvfhmin/Zvfbfmin. (#145588)
9e3bb5bb91ce Reland "[lldb] Add count for number of DWO files loaded in statistics #144424" (#145572)
46d33b6102f2 [AMDGPU][MC] Allow dpp in v_pk_fmac_f16 for GFX9 and GFX10 (#144782)
1b71ea411a9d [msan] Automatically print shadow for failing outlined checks (#145107)
6bab2c50d68d [Comgr][NFC] Update comment for logging APIs
a93eb14e8588 [LTO][Legacy] Add new C APIs to query undefined symbols in assembly (#145413)
0a3c5c42a178 Add support for Windows Secure Hot-Patching (redo) (#145565)
ebdd5a04f5d6 AMDGPU: Add checks to verify that gfx1250 no longer supports vimage instructions (#145602)
7e3af676312b [lldb] Add more tests MCP protocol types
770d0b028aa8 [DirectX] Fix order of `v2::DescriptorRange` (#145555)
fe8a26263a12 AMDGPU: Remove Formatted MUBUF instructions from gfx1250 support (#145590)
aa3c5d0297e4 Revert "[lldb][RISCV] fix LR/SC atomic sequence handling in lldb-server" (#145597)
830b2c842e78 [LV] Replace redundant ExtractLastElement of reduction result (NFC).
40bbfd3b28c3 Merge llvm/main into amd-debug
405ed40b9181 merge main into amd-staging
3a71884ab1a5 [flang][OpenMP] Map device pointers on host device as well (#145562)
3413aa83f356 Revert "[AMDGPU] Implement IR variant of isFMAFasterThanFMulAndFAdd (… (#145580)
e615544e2bd1 [mlir][EmitC] Add pass to wrap a func in class (#141158)
e1cd450c8f7c [NVPTX] fold movs into loads and stores (#144581)
3187d4da24ad [Verifier] Add additional tests for dereferenceable assumptions.
c3e25e7fc463 [VPlan] Add VPInst::getNumOperandsForOpcode, use to verify in ctor (NFC) (#142284)
b171ebbfffcd [lldb] Defend against infinite recursion in GetClassDescriptor (#145396)
2b5141189f7b [bazel] Add missing comma to BUILD.bazel (#145573)
fe0568389d68 [AMDGPU] Require aligned VGPRs for gfx1250 (#145561)
f624ba2d9d21 [OpenMP][clang] 6.0: parsing/sema for num_threads 'strict' modifier (#145490)
4d21da002a05 [mlir] Return vectorized values instead of replacing (#144158)
280f60ed6d99 [CIR] Fix NYI AAPCS bit-fields by skipping unsupported case (#145560)
f875bab8ecee [HeterogeneousDwarf] Sync with amd-staging and update tests (#2713)
ce4d21494764 AMDGPU: Remove MTBUF instructions from gfx1250 support (#145563)
6e1ed0f1b153 Merge llvm/main into amd-debug
cd46354dbd10 [WebAssembly] Enable a limited amount of stackification for debug code (#136510)
de569ad6b845 [bazel] Add //mlir:IR to Support for #144897 (#145569)
5ff6c8e84372 [RISCV] Add Zvfh scalable vector vp.splice tests. NFC
b8ea65025dc6 [clang-doc] document global variables (#145070)
4a682e16f54d merge main into amd-staging (#2799)
de2ec228c03c [analyzer] Fix crash when modelling 'getline' function in checkers (#145229)
9e33cb22f991 Fix `pragma-comment.c` test failing on false match (#145455)
52fee553a541 [DWARF] Fix redundant DW_AT_ prefix in HANDLE_DW_AT definition for `INTEL_other_endian` (#145336)
ffdfe94a2b41 [bazel] Add //mlir:DialectUtils to TestTransforms for #145376 (#145557)
21423c7fdcee [DirectX] Make firstbithigh intrinsic use first arg as overload type (#145401)
52b27c2bd627 [libc++] Refactor basic_filebuf::overflow() (#144793)
23b0564800f6 [lldb] Fix AppleObjCDeclVendor for classes which have no methods (#145452)
0b8f3cc6b7dc [libc++] Fix no-localisation CI failure on <complex> (#145213)
90828e00a09c [CIR] Restore the underscore in dso_local (#145551)
8f7f48a97ea5 [flang][OpenMP][NFC] remove globals with mlir::StateStack (#144898)
c74ed8a0d3ca [AMDGPU][Scheduler] Support for rematerializing SGPRs and AGPRs (#140036)
09b43a5a862f [clang][dataflow] Expose simple access to child StorageLocation presence. (#145520)
2b938769c60f [X86][CodeGen] Add Scheduling Model for LNL-P (#139446)
f8ed45611b75 Remove PSTL from the top-level monorepo (#143717)
8763d2968d47 [CIR] Add support for bitfields in unions (#145096)
9291ad1c9691 [CIR] Add support for __builtin_expect (#144726)
4b4782bc868b Revert "Add support for Windows Secure Hot-Patching" (#145553)
0c50971460cb [mlir] Walk nested non-symbol table ops in symbol dce (#143353)
418b409df853 [CIR] Add support for member initialization from constructors (#144583)
7150b2c76a71 [RISCV] Optimize vp.splice with 0 offset. (#145533)
0e9e1ce14625 [analyzer][NFC] Make SymExpr::classof methods constexpr (#145526)
1ae786da8e48 [HeterogeneousDwarf] Sync with amd-staging and update tests
8b0d112478ce [Clang][Preprocessor] Expand UCNs in macro concatenation (#145351)
353f75410a19 [LLDB] Add SI_USER and SI_KERNEL to Linux signals (#144800)
7377410ddaf2 [FuzzMutate] Properly handle intrinsics and avoid illegal code genertion (#145495)
f18cfb9108fd [InstrProf] Factor out getRecord() and use NamedInstrProfRecord (#145417)
3bc1fc649324 [lldb][RISCV] fix LR/SC atomic sequence handling in lldb-server (#127505)
7d2293d1d953 [InstCombine] KnownBits::isNonNegative should recognize `b - a` after `a <= b` (#145105)
a84891698a2a [instcombine] Scalarize operands of vector geps if possible (#145402)
cb4fb3aa1888 [NFC] Move areCompatibleSveTypes etc. from ASTContext to SemaARM. (#145429)
cc756716cf69 [mlir][NFC] Move LLVM::ModuleTranslation::SaveStack to a shared header (#144897)
410b5cd787de Merge llvm/main into amd-debug
f6973baf289a [BOLT][NFC] Split out parsePerfData (#145248)
85d250c96ee1 Use the Windows SDK arguments over the environment (#144805)
d80766152c49 [libc++] Fix bitset conversion functions and refactor constructor (#121348)
46ada0d91bec [Comgr] Remove deprecated action from tests
fcd82f9cf3b6 MathExtras: Make some templates constexpr NFC (#145522)
c8a9579ff9f6 [mlir][xegpu] Add support for distributing `gpu.barrier` (#145434)
70bedc74b91a [CIR] Upstream ChooseExpr for ComplexType (#145163)
b3c53025aa16 [CIR][NFC] Merge vector type invalid tests and update names (#145238)
26d318e4a943 Add support for Windows Secure Hot-Patching (#138972)
e175ecff9362 [llvm] revisions to `LLVM_ABI` export macro definitions (#144598)
6a8899cee713 [HLSL][RootSignature] Enable resource range analysis for remaining `RootElement`s (#145109)
b0c63071fb92 Port #140158 whilst upstream review continues (#2701)
559218f7ee78 [NFC][LLVM][SVE2p2] Add RUN lines to fully verify codegen predicates.
da5c442550a3 Reland "[llvm-cov] Export decision coverage to output json (#144335)" (#145325)
bb2bd5fb01c1 Revert "[mlir] Improve mlir-query by adding matcher combinators" (#145534)
4359e55838cd [Offload] Properly report errors when jit compiling (#145498)
b0ef9125347c Revert "[mlir][mesh] adding option for traversal order in sharding propagation" (#145531)
43d042b350af Revert "[clang][scan-deps] Add option to disable caching stat failures" (#145528)
469cbd454389 merge main into amd-staging
b581f9d056ba [HLSL] Add option for VK layouts (#145327)
4847bd5ae40d [mlir][acc] Add support for data clause modifiers (#144806)
117c76ac01e3 Merge llvm/main into amd-debug
2767ff49955f [CIR][NFC] Upstream computeVolatileBitfields (#145414)
fba63e3281e7 [NFC][TableGen] Use ArrayRef instead of const vector reference (#145323)
12ba75145efe [NFC][LLVM][SME] Seperate streaming mode only intrinsic tests.
bd96918f01cf [LLVM][Support][Cygwin] Add threading support for Cygwin host (#145314)
7a4b3925598c [Support] Remove an outdated MinGW workaround (#145294)
08513281bd14 [BOLT][test] Drop toolname from X86/perf2bolt-spe.test (#145515)
63f30d7d820c [mlir] Migrate away from {TypeRange,ValueRange}(std::nullopt) (NFC) (#145445)
6dad1e87fb7a [mlir][transform][Linalg] NFC - DCE unused options in PadTilingInterfaceOptions
8fec6d11777a llvm-c: Introduce 'LLVMDISubprogramReplaceType' (#143461)
68f09370f99f [Module] Use getDeclarationIfExists() (NFC)
75cf82684971 [KeyInstr][Clang] Fix atomic ops atoms test
b77c7138a872 [lld][BP] Fix duplicate section size measurment (#145384)
3e98d2b0311e [lldb] Fix windows build for #145293
9186df9b088f [InlineCost] Simplify extractvalue across callsite (#145054)
e04c938cc08a [analyzer][NFC] Add xrefs to a test case that has poor git blame (#145501)
6fe8543a2a15 [analyzer][docs] Mention perfetto for visualizing trace JSONs (#145500)
db4dc88d06a5 [X86] combineEXTRACT_SUBVECTOR - remove unnecessary bitcast handling. (#145496)
ddc6dca2f20f Merge llvm/main into amd-debug
9f3931b659f6 [AMDGPU] Fold fmed3 when inputs include infinity (#144824)
478583214410 [Offload] Fix cmake warning (#145488)
9aebfde1e70c [flang] Allow `cycle` in `target teams distribute [simd]` (#145462)
352baa386c10 [RemoveDIs] Resolve RemoveRedundantDbgInstrs fwd scan FIXME (#144718)
825ad86aea50 [DAG] Fold nested add(add(reduce(a), b), add(reduce(c), d)) (#115150)
db72f6cbe634 [RemoveDIs][NFC] Remove dbg intrinsic handling code from AssignmentTrackingAnalysis (#144674)
8f4da2cbf055 [mlir][affine] Fix min simplification in makeComposedAffineApply (#145376)
1dc46d45fcf7 [RemoveDIs] Fix rotten --implicit-check-not lines (#144711)
ddecfa696c49 [KeyInstr][Clang] Atomic ops atoms (#141624)
fa5d7c926f5f [lldb][lldb-dap] Fix runInTerminal test program on Windows
b8769104f1ef [LAA] Address follow-up suggestions for #128061.
1a264b0ad80d Merge llvm/main into amd-debug
650b1aeebcc0 Merge llvm/main into amd-debug
1345ebe538be Merge llvm/main into amd-debug
4a2fe9f27eaa Merge llvm/main into amd-debug
8a9f87ae4b0a Merge llvm/main into amd-debug
9dfbc0fdd44f Merge llvm/main into amd-debug
70f4831afe38 Merge llvm/main into amd-debug
747f3ee87f91 Merge llvm/main into amd-debug
4254168a99c8 Merge llvm/main into amd-debug
cf2c577ab085 Merge llvm/main into amd-debug
2cdffed73491 Merge llvm/main into amd-debug
15f67a54aa09 Merge llvm/main into amd-debug
3a3ed2fd97ec Merge llvm/main into amd-debug
9ccb2911a595 Merge llvm/main into amd-debug
eadabc9da156 Merge llvm/main into amd-debug
9bbc3fde50a3 Merge llvm/main into amd-debug
0986883e851f Merge llvm/main into amd-debug
fdac2dc3e48f Merge llvm/main into amd-debug
0b89fb6f39dc Merge llvm/main into amd-debug
46b09acf8305 Merge llvm/main into amd-debug
e39db390ff8b Merge llvm/main into amd-debug
fc55abd89359 Merge llvm/main into amd-debug
3474df1a036e Merge llvm/main into amd-debug
7df99d54b589 Merge llvm/main into amd-debug
366b77707a04 Merge llvm/main into amd-debug
b998a63efb8c Merge llvm/main into amd-debug
95bef3d7e2f6 Merge llvm/main into amd-debug
8d0bc8fe619d Merge llvm/main into amd-debug
b5ed79793bba Merge llvm/main into amd-debug
ce4e3c273019 Merge llvm/main into amd-debug
d342b9af1e22 Merge llvm/main into amd-debug
082da87f544c Merge llvm/main into amd-debug
5a4777092703 Merge llvm/main into amd-debug
fbccea81bd78 Merge llvm/main into amd-debug
3487fa14673a Merge llvm/main into amd-debug
681a25b2ce97 Merge llvm/main into amd-debug
0ccc64d91f0d Merge llvm/main into amd-debug
670172fbad90 Merge llvm/main into amd-debug
8612d7cfe104 Merge llvm/main into amd-debug
ae7abf56f2a9 Merge llvm/main into amd-debug
c5a8fbb19206 Merge llvm/main into amd-debug
b93bb66ea606 Merge llvm/main into amd-debug
838a01eed273 Merge llvm/main into amd-debug
70245f929b63 Merge llvm/main into amd-debug
75402d1787d3 Merge llvm/main into amd-debug
c60f83b8398e Merge llvm/main into amd-debug
9c82511e6301 Merge llvm/main into amd-debug
24117b9e71bd Merge llvm/main into amd-debug
2626c9c24de6 Merge llvm/main into amd-debug
92eb39edd3bd Merge llvm/main into amd-debug
e450ca8f9280 Merge llvm/main into amd-debug
4647974cc553 Merge llvm/main into amd-debug
5117cba3aec2 Merge llvm/main into amd-debug
85d609540f71 Merge llvm/main into amd-debug
b184a2d7561b Merge llvm/main into amd-debug
7c7afe3b1e18 Merge llvm/main into amd-debug
2e6346c5dc2a Merge llvm/main into amd-debug
3cd2da974633 Merge llvm/main into amd-debug
16051b608dc3 Merge llvm/main into amd-debug
6db01f1f1ed5 Merge llvm/main into amd-debug
56c35ee52d05 Merge llvm/main into amd-debug
8629854c7ca7 Merge llvm/main into amd-debug
eab7353c2ac5 Merge llvm/main into amd-debug
7499660c065e Merge llvm/main into amd-debug
faa72dce78a2 Merge llvm/main into amd-debug
e9d566650336 Merge llvm/main into amd-debug
f52ee1afd65a Merge llvm/main into amd-debug
4fe7a37e9d09 Merge llvm/main into amd-debug
11fd1ab11a8d Merge llvm/main into amd-debug
e8a0817a5071 Merge llvm/main into amd-debug
5cfc4769e3e6 Merge llvm/main into amd-debug
7e1626624ebc Merge llvm/main into amd-debug
46ed5eb4c789 Merge llvm/main into amd-debug
c07a39b4382c Merge llvm/main into amd-debug
d0bf5b22e242 Merge llvm/main into amd-debug
ca7a15639746 Merge llvm/main into amd-debug

git-subtree-dir: external/llvm-project
git-subtree-split: 659e9cd047321f679ad8fc0587edec71eb3bd0db
---
 .ci/compute_projects.py                       |    1 +
 .ci/compute_projects_test.py                  |   12 +-
 .ci/monolithic-linux.sh                       |   25 +-
 .ci/monolithic-windows.sh                     |    8 +-
 .github/copilot-instructions.md               |    4 +
 .github/new-prs-labeler.yml                   |    4 +
 .github/workflows/hlsl-test-all.yaml          |    4 +-
 .../libcxx-restart-preempted-jobs.yaml        |   97 +-
 amd/comgr/include/amd_comgr.h.in              |    6 -
 amd/comgr/src/comgr-compiler.cpp              |   23 +-
 amd/comgr/test-lit/CMakeLists.txt             |    5 +-
 .../compile-minimal-test-cached-bad-dir.cl    |    2 +-
 .../compile-minimal-test-cached-bad-policy.cl |    2 +-
 .../compile-minimal-test-cached.cl            |   16 +-
 amd/comgr/test-lit/comgr-sources/common.h     |   11 +
 ...inimal-test.c => compile-opencl-minimal.c} |   12 +-
 .../test-lit/comgr-sources/data-action.c      |  129 +
 .../test-lit/comgr-sources/get-version.c      |   25 +
 .../test-lit/comgr-sources/status-string.c    |   42 +
 amd/comgr/test-lit/compile-minimal-test.cl    |    2 +-
 amd/comgr/test-lit/compile-opencl-2.cl        |   12 +
 amd/comgr/test-lit/data-action.c              |    2 +
 amd/comgr/test-lit/get-version.c              |    2 +
 amd/comgr/test-lit/status-string.c            |    2 +
 amd/comgr/test-lit/time-statistics.cl         |    2 +-
 amd/comgr/test/CMakeLists.txt                 |    1 -
 ...mpile_source_with_device_libs_to_bc_test.c |   28 +-
 ...rce_with_device_libs_to_bc_with_vfs_test.c |   28 +-
 amd/comgr/test/options_test.c                 |   66 -
 bolt/include/bolt/Core/BinaryContext.h        |   10 +-
 bolt/include/bolt/Core/BinarySection.h        |    5 -
 bolt/include/bolt/Core/MCPlusBuilder.h        |    2 +-
 bolt/include/bolt/Profile/DataAggregator.h    |    6 +-
 bolt/include/bolt/Rewrite/RewriteInstance.h   |    6 +
 bolt/lib/Core/BinaryFunction.cpp              |    5 +-
 bolt/lib/Passes/BinaryPasses.cpp              |   33 +-
 bolt/lib/Passes/Instrumentation.cpp           |    3 +-
 bolt/lib/Passes/PAuthGadgetScanner.cpp        |  220 +-
 bolt/lib/Profile/DataAggregator.cpp           |  149 +-
 bolt/lib/Rewrite/RewriteInstance.cpp          |  436 +-
 .../Target/AArch64/AArch64MCPlusBuilder.cpp   |   42 +-
 bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp  |   19 +-
 bolt/lib/Target/X86/X86MCPlusBuilder.cpp      |   91 +-
 bolt/test/X86/zero-density.s                  |   32 +
 .../AArch64/gs-pacret-autiasp.s               |  105 +-
 .../AArch64/gs-pauth-authentication-oracles.s |   20 -
 .../binary-analysis/AArch64/gs-pauth-calls.s  |   84 +
 .../AArch64/gs-pauth-debug-output.s           |   32 +-
 .../AArch64/gs-pauth-signing-oracles.s        |   27 -
 .../AArch64/gs-pauth-tail-calls.s             |  597 ++
 bolt/test/code-at-high-address.c              |   24 +
 bolt/test/perf2bolt/X86/perf2bolt-spe.test    |    2 +-
 bolt/test/program-header.test                 |   14 +
 bolt/unittests/Core/BinaryContext.cpp         |   32 +-
 bolt/unittests/Core/MemoryMaps.cpp            |   16 +-
 bolt/utils/llvm-bolt-wrapper.py               |    2 +-
 bolt/utils/nfc-check-setup.py                 |   39 +
 clang-tools-extra/clang-doc/BitcodeReader.cpp |  236 +-
 clang-tools-extra/clang-doc/BitcodeReader.h   |    7 +
 clang-tools-extra/clang-doc/BitcodeWriter.cpp |   75 +-
 clang-tools-extra/clang-doc/BitcodeWriter.h   |   18 +-
 clang-tools-extra/clang-doc/HTMLGenerator.cpp |    6 +
 .../clang-doc/HTMLMustacheGenerator.cpp       |    3 +
 clang-tools-extra/clang-doc/JSONGenerator.cpp |  287 +-
 clang-tools-extra/clang-doc/MDGenerator.cpp   |    8 +
 clang-tools-extra/clang-doc/Mapper.cpp        |    6 +
 clang-tools-extra/clang-doc/Mapper.h          |    1 +
 .../clang-doc/Representation.cpp              |   32 +
 clang-tools-extra/clang-doc/Representation.h  |   33 +-
 clang-tools-extra/clang-doc/Serialize.cpp     |   81 +
 clang-tools-extra/clang-doc/Serialize.h       |    4 +
 clang-tools-extra/clang-doc/YAMLGenerator.cpp |    2 +
 .../ClangTidyDiagnosticConsumer.cpp           |   19 +
 .../clang-tidy/ClangTidyDiagnosticConsumer.h  |   10 +
 .../MisleadingSetterOfReferenceCheck.cpp      |    2 +
 .../bugprone/SizeofExpressionCheck.cpp        |   38 +-
 .../bugprone/SizeofExpressionCheck.h          |    1 +
 .../MissingStdForwardCheck.cpp                |   11 +-
 .../MissingStdForwardCheck.h                  |    7 +-
 .../ProBoundsPointerArithmeticCheck.cpp       |   16 +-
 .../ProBoundsPointerArithmeticCheck.h         |    3 +
 .../RvalueReferenceParamNotMovedCheck.cpp     |    8 +-
 .../RvalueReferenceParamNotMovedCheck.h       |    1 +
 .../misc/ConfusableIdentifierCheck.cpp        |  235 +-
 .../misc/ConfusableIdentifierCheck.h          |   25 +-
 .../UnconventionalAssignOperatorCheck.cpp     |    7 +-
 .../clang-tidy/modernize/CMakeLists.txt       |    1 +
 .../modernize/ModernizeTidyModule.cpp         |    3 +
 .../modernize/UseScopedLockCheck.cpp          |  311 +
 .../clang-tidy/modernize/UseScopedLockCheck.h |   54 +
 .../RedundantInlineSpecifierCheck.cpp         |    7 +-
 clang-tools-extra/clangd/AST.cpp              |    2 +-
 clang-tools-extra/clangd/InlayHints.cpp       |   51 +-
 .../clangd/refactor/tweaks/DefineInline.cpp   |    2 +-
 .../refactor/tweaks/ExtractVariable.cpp       |    6 +-
 clang-tools-extra/clangd/tool/ClangdMain.cpp  |   10 +-
 .../clangd/unittests/CodeCompleteTests.cpp    |   30 +
 clang-tools-extra/docs/ReleaseNotes.rst       |   35 +
 .../checks/bugprone/sizeof-expression.rst     |    9 +
 .../cppcoreguidelines/missing-std-forward.rst |    7 +
 .../rvalue-reference-param-not-moved.rst      |    4 +
 .../docs/clang-tidy/checks/list.rst           |    1 +
 .../checks/modernize/use-scoped-lock.rst      |  101 +
 .../docs/clangd/DeveloperDocumentation.rst    |    2 +-
 clang-tools-extra/modularize/Modularize.cpp   |    4 +-
 .../test/clang-doc/json/class-template.cpp    |    2 +-
 .../test/clang-doc/json/class.cpp             |   47 +-
 .../test/clang-doc/json/function-requires.cpp |    2 +-
 .../test/clang-doc/json/method-template.cpp   |    2 +-
 .../test/clang-doc/json/namespace.cpp         |   46 +-
 .../test/clang-doc/json/nested-namespace.cpp  |   36 +
 .../clang-tidy/checkers/Inputs/Headers/mutex  |   33 +
 .../checkers/bugprone/sizeof-expression.cpp   |   63 +
 .../missing-std-forward-custom-function.cpp   |   42 +
 .../pro-bounds-pointer-arithmetic-pr36489.cpp |    9 +
 .../pro-bounds-pointer-arithmetic.cpp         |   51 +
 ...erence-param-not-moved-custom-function.cpp |   47 +
 .../checkers/misc/confusable-identifiers.cpp  |   59 +
 .../misc/unconventional-assign-operator.cpp   |   13 +
 ...e-lock-warn-on-using-and-typedef-false.cpp |   31 +
 ...scoped-lock-warn-on-single-locks-false.cpp |  102 +
 .../checkers/modernize/use-scoped-lock.cpp    |  471 ++
 .../redundant-inline-specifier.cpp            |   10 +
 .../unittests/clang-doc/BitcodeTest.cpp       |    4 +
 clang/CMakeLists.txt                          |    6 +
 .../bindings/python/tests/cindex/test_file.py |    1 +
 clang/docs/APINotes.rst                       |    2 +-
 clang/docs/ClangFormatStyleOptions.rst        |   13 +-
 clang/docs/DebuggingCoroutines.rst            | 1347 ++--
 clang/docs/LanguageExtensions.rst             |  113 +-
 clang/docs/ReleaseNotes.rst                   |   87 +-
 clang/docs/Toolchain.rst                      |    5 +
 .../analyzer/developer-docs/DebugChecks.rst   |   13 +-
 .../PerformanceInvestigation.rst              |    2 +-
 clang/docs/coro-async-task-continuations.png  |  Bin 0 -> 858961 bytes
 clang/docs/coro-generator-suspended.png       |  Bin 0 -> 670202 bytes
 clang/docs/coro-generator-variables.png       |  Bin 0 -> 774476 bytes
 clang/include/clang-c/Index.h                 |  125 +-
 clang/include/clang/AST/ASTContext.h          |    9 -
 clang/include/clang/AST/ASTNodeTraverser.h    |    2 +-
 clang/include/clang/AST/Decl.h                |   21 +-
 clang/include/clang/AST/DeclarationName.h     |   18 +-
 clang/include/clang/AST/ExprCXX.h             |    8 +-
 clang/include/clang/AST/ExternalASTSource.h   |   15 -
 clang/include/clang/AST/ODRHash.h             |   10 +-
 clang/include/clang/AST/OpenMPClause.h        |   66 +-
 clang/include/clang/AST/Type.h                |    7 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |    4 +-
 clang/include/clang/Analysis/CFG.h            |    7 +-
 clang/include/clang/Basic/AMDGPUTypes.def     |    8 +
 clang/include/clang/Basic/Attr.td             |    8 +
 clang/include/clang/Basic/BuiltinHeaders.def  |    1 +
 clang/include/clang/Basic/Builtins.def        |    1 +
 clang/include/clang/Basic/Builtins.td         |   42 +
 clang/include/clang/Basic/BuiltinsAArch64.def |    8 +
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |   31 +-
 clang/include/clang/Basic/BuiltinsPPC.def     |    6 +
 clang/include/clang/Basic/BuiltinsX86.td      |   12 +-
 clang/include/clang/Basic/CodeGenOptions.h    |    9 +
 .../include/clang/Basic/DiagnosticASTKinds.td |    2 +
 .../clang/Basic/DiagnosticDriverKinds.td      |    3 +
 clang/include/clang/Basic/DiagnosticGroups.td |    4 +-
 .../clang/Basic/DiagnosticParseKinds.td       |    3 +
 .../clang/Basic/DiagnosticSemaKinds.td        |   54 +-
 clang/include/clang/Basic/LangOptions.def     |    3 +
 clang/include/clang/Basic/LangOptions.h       |    5 +
 clang/include/clang/Basic/Module.h            |    5 +
 clang/include/clang/Basic/OpenACCKinds.h      |   19 +-
 clang/include/clang/Basic/OpenMPKinds.def     |    7 +
 clang/include/clang/Basic/OpenMPKinds.h       |    6 +
 clang/include/clang/Basic/Sanitizers.h        |    3 +-
 clang/include/clang/Basic/SourceLocation.h    |    6 +-
 clang/include/clang/Basic/SourceManager.h     |   35 +-
 clang/include/clang/Basic/TargetInfo.h        |   10 +-
 clang/include/clang/Basic/arm_neon.td         |   54 +-
 clang/include/clang/Basic/arm_sve.td          |   12 +-
 .../include/clang/Basic/riscv_andes_vector.td |   46 +
 clang/include/clang/Basic/riscv_vector.td     |  217 +-
 .../clang/Basic/riscv_vector_common.td        |   10 +-
 .../CIR/Dialect/Builder/CIRBaseBuilder.h      |   27 +-
 clang/include/clang/CIR/Dialect/IR/CIRAttrs.h |   22 +-
 .../include/clang/CIR/Dialect/IR/CIRAttrs.td  |  183 +-
 .../include/clang/CIR/Dialect/IR/CIRDialect.h |    1 +
 .../clang/CIR/Dialect/IR/CIRDialect.td        |    4 +
 clang/include/clang/CIR/Dialect/IR/CIROps.td  |  377 +-
 clang/include/clang/CIR/Dialect/IR/CIRTypes.h |   11 +-
 .../include/clang/CIR/Dialect/IR/CIRTypes.td  |   76 +-
 .../CIR/Interfaces/CIRFPTypeInterface.td      |   56 -
 .../clang/CIR/Interfaces/CIROpInterfaces.td   |    6 +-
 ...RFPTypeInterface.h => CIRTypeInterfaces.h} |   10 +-
 .../clang/CIR/Interfaces/CIRTypeInterfaces.td |  123 +
 .../clang/CIR/Interfaces/CMakeLists.txt       |    2 +-
 clang/include/clang/CIR/LoweringHelpers.h     |   14 +
 clang/include/clang/CIR/MissingFeatures.h     |  157 +-
 clang/include/clang/Driver/CommonArgs.h       |    2 +
 clang/include/clang/Driver/Driver.h           |   14 +-
 clang/include/clang/Driver/Options.td         |   85 +-
 clang/include/clang/Format/Format.h           |   28 +-
 clang/include/clang/Interpreter/Interpreter.h |    7 +-
 .../clang/Interpreter/RemoteJITUtils.h        |   38 +
 .../include/clang/Lex/LexHLSLRootSignature.h  |    4 +
 clang/include/clang/Lex/Lexer.h               |   10 +-
 clang/include/clang/Lex/Preprocessor.h        |   27 +-
 clang/include/clang/Lex/Token.h               |    9 +-
 clang/include/clang/Lex/TokenLexer.h          |    5 +-
 .../clang/Parse/ParseHLSLRootSignature.h      |   46 +-
 clang/include/clang/Parse/Parser.h            |   60 +-
 clang/include/clang/Sema/DeclSpec.h           |    4 +-
 clang/include/clang/Sema/HeuristicResolver.h  |    7 +
 clang/include/clang/Sema/Initialization.h     |    7 +-
 .../clang/Sema/MultiplexExternalSemaSource.h  |    2 -
 clang/include/clang/Sema/Scope.h              |   14 +-
 clang/include/clang/Sema/Sema.h               |   15 +-
 clang/include/clang/Sema/SemaAMDGPU.h         |    9 +
 clang/include/clang/Sema/SemaARM.h            |    9 +
 clang/include/clang/Sema/SemaOpenACC.h        |   19 +
 clang/include/clang/Sema/SemaOpenMP.h         |    8 +-
 clang/include/clang/Serialization/ASTReader.h |   30 +-
 .../clang/Serialization/ASTRecordReader.h     |   11 +-
 .../clang/Serialization/ASTRecordWriter.h     |   11 +-
 clang/include/clang/Serialization/ASTWriter.h |   13 +-
 .../Serialization/SourceLocationEncoding.h    |  120 +-
 .../Core/PathSensitive/CallEvent.h            |    2 +
 .../Core/PathSensitive/ExplodedGraph.h        |    5 +-
 .../Core/PathSensitive/SMTConv.h              |    3 +
 .../Core/PathSensitive/SymExpr.h              |    6 +-
 .../Core/PathSensitive/SymbolManager.h        |  100 +-
 .../DependencyScanningFilesystem.h            |   60 +-
 .../DependencyScanningService.h               |    6 +-
 .../DependencyScanningTool.h                  |   14 +
 .../clang/Tooling/Transformer/Transformer.h   |   13 +-
 clang/lib/APINotes/APINotesWriter.cpp         |    5 +-
 clang/lib/AST/ASTContext.cpp                  |   98 +-
 clang/lib/AST/ASTImporter.cpp                 |    8 +-
 clang/lib/AST/ByteCode/Compiler.cpp           |   16 +-
 clang/lib/AST/ByteCode/Compiler.h             |   18 +-
 clang/lib/AST/ByteCode/Descriptor.cpp         |   15 +
 clang/lib/AST/ByteCode/Descriptor.h           |    3 +
 clang/lib/AST/ByteCode/Disasm.cpp             |    8 +-
 clang/lib/AST/ByteCode/FunctionPointer.cpp    |    2 -
 clang/lib/AST/ByteCode/Interp.h               |   54 +-
 clang/lib/AST/ByteCode/InterpBuiltin.cpp      |   58 +-
 .../lib/AST/ByteCode/InterpBuiltinBitCast.cpp |    7 +-
 clang/lib/AST/ByteCode/InterpFrame.cpp        |    1 -
 clang/lib/AST/ByteCode/Pointer.cpp            |   80 +-
 clang/lib/AST/ByteCode/Pointer.h              |    3 +
 clang/lib/AST/Comment.cpp                     |    7 +
 clang/lib/AST/CommentLexer.cpp                |    2 +-
 clang/lib/AST/Decl.cpp                        |   43 +-
 clang/lib/AST/DeclBase.cpp                    |    2 +
 clang/lib/AST/DeclObjC.cpp                    |    1 -
 clang/lib/AST/Expr.cpp                        |    6 +-
 clang/lib/AST/ExprCXX.cpp                     |    4 +-
 clang/lib/AST/ExprConcepts.cpp                |    1 -
 clang/lib/AST/ExprObjC.cpp                    |    1 -
 clang/lib/AST/ItaniumMangle.cpp               |    3 +-
 clang/lib/AST/ODRHash.cpp                     |   29 +-
 clang/lib/AST/OpenMPClause.cpp                |    5 +
 clang/lib/AST/RawCommentList.cpp              |    7 +-
 clang/lib/AST/RecordLayoutBuilder.cpp         |    7 +
 clang/lib/AST/StmtOpenACC.cpp                 |    2 +-
 clang/lib/AST/StmtPrinter.cpp                 |    1 -
 clang/lib/AST/TextNodeDumper.cpp              |   12 +-
 clang/lib/AST/Type.cpp                        |    9 -
 clang/lib/AST/TypeLoc.cpp                     |    6 +-
 clang/lib/ASTMatchers/Dynamic/Marshallers.h   |    2 +-
 clang/lib/Analysis/CFG.cpp                    |    8 +-
 clang/lib/Analysis/CFGStmtMap.cpp             |    4 +-
 clang/lib/Analysis/CallGraph.cpp              |    5 +-
 .../lib/Analysis/FlowSensitive/CNFFormula.cpp |    5 +-
 clang/lib/Analysis/LiveVariables.cpp          |   10 +-
 clang/lib/Analysis/PathDiagnostic.cpp         |    8 +-
 clang/lib/Analysis/ReachableCode.cpp          |    5 +-
 clang/lib/Analysis/RetainSummaryManager.cpp   |    3 +-
 clang/lib/Analysis/UnsafeBufferUsage.cpp      |   40 +-
 clang/lib/Basic/Attributes.cpp                |    6 +-
 clang/lib/Basic/Diagnostic.cpp                |    9 +-
 clang/lib/Basic/DiagnosticIDs.cpp             |    8 +-
 clang/lib/Basic/FileManager.cpp               |    2 +-
 clang/lib/Basic/OpenMPKinds.cpp               |   21 +-
 clang/lib/Basic/Sanitizers.cpp                |    3 +-
 clang/lib/Basic/Sarif.cpp                     |    2 +-
 clang/lib/Basic/SourceLocation.cpp            |    4 +-
 clang/lib/Basic/SourceManager.cpp             |  160 +-
 clang/lib/Basic/Targets/AArch64.cpp           |  102 +-
 clang/lib/Basic/Targets/AArch64.h             |    4 +-
 clang/lib/Basic/Targets/LoongArch.cpp         |   72 +-
 clang/lib/Basic/Targets/LoongArch.h           |    1 +
 clang/lib/Basic/Targets/M68k.cpp              |    1 +
 clang/lib/Basic/Targets/PPC.cpp               |    6 +
 clang/lib/Basic/Targets/RISCV.cpp             |    4 +-
 clang/lib/Basic/Targets/RISCV.h               |    2 +-
 clang/lib/Basic/Targets/SPIR.cpp              |    5 +
 clang/lib/Basic/Targets/SPIR.h                |    1 +
 clang/lib/Basic/Targets/SystemZ.h             |    4 +-
 clang/lib/CIR/CodeGen/CIRGenBuilder.cpp       |    5 +-
 clang/lib/CIR/CodeGen/CIRGenBuilder.h         |   35 +-
 clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp       |   90 +
 clang/lib/CIR/CodeGen/CIRGenCXX.cpp           |    2 +-
 clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp       |  229 +-
 clang/lib/CIR/CodeGen/CIRGenCall.cpp          |   52 +-
 clang/lib/CIR/CodeGen/CIRGenCall.h            |    3 +-
 clang/lib/CIR/CodeGen/CIRGenClass.cpp         |  176 +-
 clang/lib/CIR/CodeGen/CIRGenExpr.cpp          |   88 +-
 clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp   |   60 +-
 clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp  |   16 +-
 clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp    |  107 +-
 clang/lib/CIR/CodeGen/CIRGenFunction.cpp      |   33 +-
 clang/lib/CIR/CodeGen/CIRGenFunction.h        |   91 +-
 clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp |   89 +-
 clang/lib/CIR/CodeGen/CIRGenModule.cpp        |  199 +-
 clang/lib/CIR/CodeGen/CIRGenModule.h          |   31 +-
 clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp       |  131 +
 clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp |  435 +-
 .../CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp |   50 +-
 clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp   |   58 +-
 .../lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp |    1 +
 clang/lib/CIR/CodeGen/CIRGenTypeCache.h       |   18 +-
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp         |    4 +-
 clang/lib/CIR/CodeGen/CIRGenValue.h           |   34 +
 clang/lib/CIR/CodeGen/CMakeLists.txt          |    1 +
 clang/lib/CIR/Dialect/IR/CIRAttrs.cpp         |  123 +-
 clang/lib/CIR/Dialect/IR/CIRDialect.cpp       |  205 +-
 clang/lib/CIR/Dialect/IR/CIRTypes.cpp         |   13 +-
 .../Dialect/Transforms/CIRCanonicalize.cpp    |    5 +-
 ...ypeInterface.cpp => CIRTypeInterfaces.cpp} |    6 +-
 clang/lib/CIR/Interfaces/CMakeLists.txt       |    4 +-
 .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp |  309 +-
 .../CIR/Lowering/DirectToLLVM/LowerToLLVM.h   |   92 +-
 clang/lib/CIR/Lowering/LoweringHelpers.cpp    |   40 +-
 clang/lib/CodeGen/BackendUtil.cpp             |   21 +-
 clang/lib/CodeGen/CGAtomic.cpp                |   17 +-
 clang/lib/CodeGen/CGBuiltin.cpp               |   22 +-
 clang/lib/CodeGen/CGCall.cpp                  |    7 +
 clang/lib/CodeGen/CGClass.cpp                 |    9 +-
 clang/lib/CodeGen/CGDebugInfo.cpp             |   56 +-
 clang/lib/CodeGen/CGDebugInfo.h               |    7 +-
 clang/lib/CodeGen/CGExprScalar.cpp            |   19 +-
 clang/lib/CodeGen/CGHLSLRuntime.cpp           |   25 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp      |    4 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |    2 +-
 clang/lib/CodeGen/CGVTables.cpp               |    4 +-
 clang/lib/CodeGen/CodeGenFunction.cpp         |   10 +-
 clang/lib/CodeGen/CodeGenFunction.h           |    3 +
 clang/lib/CodeGen/CodeGenModule.cpp           |   98 +
 clang/lib/CodeGen/CodeGenModule.h             |    5 +
 clang/lib/CodeGen/CodeGenPGO.cpp              |    3 +-
 clang/lib/CodeGen/CodeGenTypes.cpp            |    3 +
 clang/lib/CodeGen/ConstantInitBuilder.cpp     |    2 +-
 clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp   |   38 +-
 clang/lib/CodeGen/TargetBuiltins/ARM.cpp      |   84 +-
 clang/lib/CodeGen/TargetBuiltins/X86.cpp      |   11 +
 clang/lib/CodeGen/Targets/RISCV.cpp           |    2 +-
 clang/lib/CodeGen/Targets/SPIR.cpp            |   32 +-
 .../linux/DirectoryWatcher-linux.cpp          |    5 -
 clang/lib/Driver/CMakeLists.txt               |    1 -
 clang/lib/Driver/Driver.cpp                   |  245 +-
 clang/lib/Driver/OffloadBundler.cpp           |    6 +-
 clang/lib/Driver/SanitizerArgs.cpp            |    2 +-
 clang/lib/Driver/ToolChain.cpp                |    2 +-
 clang/lib/Driver/ToolChains/Arch/AArch64.cpp  |   27 +-
 .../lib/Driver/ToolChains/Arch/LoongArch.cpp  |   10 +-
 clang/lib/Driver/ToolChains/BareMetal.cpp     |  333 +-
 clang/lib/Driver/ToolChains/BareMetal.h       |   29 +-
 clang/lib/Driver/ToolChains/Clang.cpp         |   69 +-
 clang/lib/Driver/ToolChains/CommonArgs.cpp    |   74 +-
 clang/lib/Driver/ToolChains/Flang.cpp         |    9 +-
 clang/lib/Driver/ToolChains/Gnu.cpp           |   72 -
 clang/lib/Driver/ToolChains/HIPAMD.cpp        |   11 +-
 clang/lib/Driver/ToolChains/HIPSPV.cpp        |   19 +-
 clang/lib/Driver/ToolChains/HLSL.cpp          |   40 +-
 clang/lib/Driver/ToolChains/MSVC.cpp          |   19 +-
 clang/lib/Driver/ToolChains/Managarm.cpp      |    1 -
 clang/lib/Driver/ToolChains/PS4CPU.cpp        |    6 +-
 .../lib/Driver/ToolChains/RISCVToolchain.cpp  |  231 -
 clang/lib/Driver/ToolChains/RISCVToolchain.h  |   67 -
 clang/lib/Edit/Commit.cpp                     |    9 +-
 clang/lib/ExtractAPI/DeclarationFragments.cpp |   40 +-
 clang/lib/Format/BreakableToken.cpp           |    3 +-
 clang/lib/Format/ContinuationIndenter.cpp     |   17 +-
 clang/lib/Format/ContinuationIndenter.h       |   15 +-
 clang/lib/Format/Format.cpp                   |    2 +-
 clang/lib/Format/FormatTokenLexer.cpp         |   45 +-
 clang/lib/Format/MacroExpander.cpp            |    2 +-
 clang/lib/Format/QualifierAlignmentFixer.cpp  |   25 +-
 clang/lib/Format/TokenAnnotator.cpp           |    7 +-
 clang/lib/Format/UnwrappedLineFormatter.cpp   |    9 +-
 clang/lib/Format/UnwrappedLineParser.cpp      |   14 +-
 clang/lib/Format/WhitespaceManager.cpp        |    8 +-
 clang/lib/Format/WhitespaceManager.h          |    2 +-
 clang/lib/Frontend/ASTUnit.cpp                |    4 +-
 clang/lib/Frontend/CompilerInvocation.cpp     |   72 +-
 clang/lib/Frontend/FrontendAction.cpp         |    8 +-
 clang/lib/Frontend/SARIFDiagnostic.cpp        |    4 +-
 clang/lib/Frontend/TextDiagnostic.cpp         |    6 +-
 .../ExecuteCompilerInvocation.cpp             |    1 -
 clang/lib/Headers/intrin.h                    |   23 +-
 clang/lib/Headers/opencl-c.h                  |  334 +-
 clang/lib/Headers/prfchwintrin.h              |   23 +-
 clang/lib/Headers/xmmintrin.h                 |    5 +-
 clang/lib/Index/CommentToXML.cpp              |    2 +-
 clang/lib/Index/IndexBody.cpp                 |    2 +-
 clang/lib/Index/USRGeneration.cpp             |    2 +-
 clang/lib/Interpreter/CMakeLists.txt          |    1 +
 clang/lib/Interpreter/Interpreter.cpp         |   52 +-
 clang/lib/Interpreter/RemoteJITUtils.cpp      |  267 +
 clang/lib/Lex/DependencyDirectivesScanner.cpp |   52 +-
 clang/lib/Lex/Lexer.cpp                       |   35 +-
 clang/lib/Lex/MacroInfo.cpp                   |    7 +-
 clang/lib/Lex/PPDirectives.cpp                |    7 +-
 clang/lib/Lex/PPMacroExpansion.cpp            |   22 +-
 clang/lib/Lex/Preprocessor.cpp                |   21 +-
 clang/lib/Lex/TokenLexer.cpp                  |   11 +
 clang/lib/Parse/ParseDeclCXX.cpp              |    4 +-
 clang/lib/Parse/ParseExpr.cpp                 |  196 +-
 clang/lib/Parse/ParseExprCXX.cpp              |   18 +-
 clang/lib/Parse/ParseHLSLRootSignature.cpp    |   63 +-
 clang/lib/Parse/ParseOpenACC.cpp              |   17 +-
 clang/lib/Parse/ParseOpenMP.cpp               |  243 +-
 clang/lib/Parse/ParseStmt.cpp                 |    2 +-
 clang/lib/Parse/ParseStmtAsm.cpp              |    9 +-
 clang/lib/Parse/ParseTemplate.cpp             |    4 +-
 clang/lib/Parse/Parser.cpp                    |    5 +-
 clang/lib/Rewrite/Rewriter.cpp                |    2 +-
 clang/lib/Sema/AnalysisBasedWarnings.cpp      |   24 +-
 clang/lib/Sema/CodeCompleteConsumer.cpp       |    4 +-
 clang/lib/Sema/HeuristicResolver.cpp          |   57 +
 clang/lib/Sema/JumpDiagnostics.cpp            |    7 +-
 .../lib/Sema/MultiplexExternalSemaSource.cpp  |    8 -
 clang/lib/Sema/OpenCLBuiltins.td              |   32 +
 clang/lib/Sema/Scope.cpp                      |    3 +-
 clang/lib/Sema/Sema.cpp                       |   30 +-
 clang/lib/Sema/SemaAMDGPU.cpp                 |   77 +
 clang/lib/Sema/SemaARM.cpp                    |  123 +-
 clang/lib/Sema/SemaAttr.cpp                   |    3 +-
 clang/lib/Sema/SemaCast.cpp                   |  209 +-
 clang/lib/Sema/SemaChecking.cpp               |  162 +-
 clang/lib/Sema/SemaCodeComplete.cpp           |   88 +-
 clang/lib/Sema/SemaConcept.cpp                |    4 +-
 clang/lib/Sema/SemaDecl.cpp                   |   65 +-
 clang/lib/Sema/SemaDeclAttr.cpp               |   48 +
 clang/lib/Sema/SemaDeclCXX.cpp                |  103 +-
 clang/lib/Sema/SemaExpr.cpp                   |   51 +-
 clang/lib/Sema/SemaExprCXX.cpp                |    9 +-
 clang/lib/Sema/SemaHLSL.cpp                   |   74 +-
 clang/lib/Sema/SemaInit.cpp                   |   69 +-
 clang/lib/Sema/SemaModule.cpp                 |    8 +-
 clang/lib/Sema/SemaObjCProperty.cpp           |    2 +-
 clang/lib/Sema/SemaOpenACC.cpp                |   68 +-
 clang/lib/Sema/SemaOpenACCClause.cpp          |    4 +-
 clang/lib/Sema/SemaOpenMP.cpp                 |   76 +-
 clang/lib/Sema/SemaOverload.cpp               |   37 +-
 clang/lib/Sema/SemaPPC.cpp                    |    4 +
 clang/lib/Sema/SemaRISCV.cpp                  |   23 +-
 clang/lib/Sema/SemaSPIRV.cpp                  |    2 +-
 clang/lib/Sema/SemaSwift.cpp                  |   17 +-
 clang/lib/Sema/SemaTemplate.cpp               |    6 +-
 clang/lib/Sema/SemaType.cpp                   |    4 +-
 clang/lib/Sema/SemaTypeTraits.cpp             |  376 +-
 clang/lib/Sema/TreeTransform.h                |   28 +-
 clang/lib/Serialization/ASTReader.cpp         |   67 +-
 clang/lib/Serialization/ASTReaderDecl.cpp     |    5 +-
 clang/lib/Serialization/ASTReaderStmt.cpp     |    2 +-
 clang/lib/Serialization/ASTWriter.cpp         |   59 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |    5 +-
 clang/lib/Serialization/ASTWriterStmt.cpp     |    4 +-
 .../Serialization/TemplateArgumentHasher.cpp  |    2 +
 .../Checkers/CStringChecker.cpp               |   48 +-
 .../Checkers/CallAndMessageChecker.cpp        |    2 +
 .../Checkers/LocalizationChecker.cpp          |    3 +-
 .../StaticAnalyzer/Checkers/MallocChecker.cpp |   75 +-
 .../Checkers/UnixAPIChecker.cpp               |   50 +-
 .../WebKit/RawPtrRefCallArgsChecker.cpp       |    2 +-
 .../Core/BugReporterVisitors.cpp              |    2 +-
 clang/lib/StaticAnalyzer/Core/CallEvent.cpp   |   12 +
 .../StaticAnalyzer/Core/HTMLDiagnostics.cpp   |    4 +-
 clang/lib/Tooling/Core/Replacement.cpp        |    7 +-
 .../DependencyScanningFilesystem.cpp          |   68 +-
 .../DependencyScanningService.cpp             |    3 +-
 .../DependencyScanningTool.cpp                |    2 +
 .../DependencyScanningWorker.cpp              |    3 +-
 .../DependencyScanning/ModuleDepCollector.cpp |    5 +-
 .../lib/Tooling/Refactoring/AtomicChange.cpp  |    3 +-
 .../Refactoring/Rename/USRLocFinder.cpp       |    3 +-
 clang/lib/Tooling/Transformer/SourceCode.cpp  |    4 +-
 clang/test/AST/ByteCode/builtin-functions.cpp |    8 +
 .../libcxx/rvalue-reference-param.cpp         |  122 +
 .../libcxx/tuple-decompose-for-range.cpp      |  264 +
 clang/test/AST/ByteCode/literals.cpp          |   12 +
 clang/test/AST/ByteCode/new-delete.cpp        |   36 +
 clang/test/AST/ByteCode/placement-new.cpp     |   21 +
 clang/test/AST/ByteCode/unions.cpp            |  120 +-
 clang/test/AST/HLSL/RootSignatures-AST.hlsl   |   31 +-
 .../test/AST/HLSL/vk.spec-constant.usage.hlsl |   11 +
 clang/test/AST/absurdly_big_struct.cpp        |   13 +
 clang/test/AST/ast-dump-expr.cpp              |   10 +
 .../AST/ast-print-openacc-cache-construct.cpp |    3 +
 .../ast-print-openacc-combined-construct.cpp  |    4 +-
 clang/test/Analysis/ArrayDelete.cpp           |    2 +-
 .../WebKit/call-args-safe-functions.mm        |   12 +
 .../Checkers/WebKit/objc-mock-types.h         |    1 +
 .../Analysis/DeleteWithNonVirtualDtor.cpp     |    2 +-
 clang/test/Analysis/PR37855.c                 |   12 +-
 clang/test/Analysis/PR38208.c                 |    5 +
 clang/test/Analysis/aggrinit-cfg-output.cpp   |    2 +-
 clang/test/Analysis/analyzer_test.py          |   53 -
 clang/test/Analysis/bool-assignment.c         |    4 -
 clang/test/Analysis/call-and-message.c        |   27 +-
 clang/test/Analysis/call-and-message.cpp      |   16 +
 clang/test/Analysis/compound-literals.c       |    2 +-
 clang/test/Analysis/cstring-addrspace.c       |    3 +-
 .../cstring-should-not-invalidate.cpp         |  132 +
 .../test/Analysis/ctu-cxxdefaultinitexpr.cpp  |    2 +-
 clang/test/Analysis/ctu-implicit.c            |    2 +-
 clang/test/Analysis/ctu-import-threshold.c    |    4 +-
 .../Analysis/ctu-lookup-name-with-space.cpp   |    2 +-
 clang/test/Analysis/ctu-main.c                |    4 +-
 clang/test/Analysis/ctu-on-demand-parsing.c   |    2 +-
 .../test/Analysis/ctu-test-import-failure.cpp |    2 +-
 .../exploded-graph-rewriter/dynamic_types.cpp |    7 +-
 .../exploded-graph-rewriter/escapes.c         |    7 +-
 .../initializers_under_construction.cpp       |    7 +-
 .../l_name_starts_with_l.cpp                  |    7 +-
 .../Analysis/exploded-graph-rewriter/macros.c |    7 +-
 .../objects_under_construction.cpp            |    9 +-
 .../win_path_forbidden_chars.cpp              |    7 +-
 .../getline-unixapi-invalid-signatures.c      |  127 +
 clang/test/Analysis/gmalloc.c                 |    2 +-
 ...dle_constructors_for_default_arguments.cpp |    2 +-
 .../handle_constructors_with_new_array.cpp    |    2 +-
 clang/test/Analysis/initialization.c          |    2 +-
 clang/test/Analysis/initialization.cpp        |    2 +-
 .../Analysis/lambda-convert-to-func-ptr.cpp   |   21 +
 clang/test/Analysis/lifetime-cfg-output.cpp   |    2 +-
 clang/test/Analysis/lit.local.cfg             |   12 +-
 clang/test/Analysis/loopexit-cfg-output.cpp   |    2 +-
 clang/test/Analysis/method-arg-decay.m        |    2 +-
 clang/test/Analysis/null-deref-static.m       |    2 +-
 clang/test/Analysis/objc-encode.m             |    2 +-
 clang/test/Analysis/openmp-unsupported.c      |    2 +-
 .../pointer-escape-on-conservative-calls.c    |    2 +-
 clang/test/Analysis/print-ranges.cpp          |    2 +-
 clang/test/Analysis/ptr-arith.c               |   17 -
 clang/test/Analysis/reference.cpp             |   16 -
 clang/test/Analysis/scopes-cfg-output.cpp     |    2 +-
 .../test/Analysis/taint-diagnostic-visitor.c  |    2 +-
 .../Analysis/temporaries-callback-order.cpp   |    2 +-
 .../Analysis/unary-sym-expr-z3-refutation.c   |    2 +-
 clang/test/Analysis/undef-call.c              |    2 +-
 .../Analysis/z3-crosscheck-max-attempts.cpp   |   15 +-
 clang/test/Analysis/z3-crosscheck.c           |    8 +-
 clang/test/Analysis/z3/D83660.c               |   11 +-
 .../test/Analysis/z3/crosscheck-statistics.c  |    2 -
 clang/test/Analysis/z3/enabled.c              |    3 -
 clang/test/Analysis/z3/pretty-dump.c          |    2 +-
 clang/test/CIR/CodeGen/align-load.c           |    6 +-
 clang/test/CIR/CodeGen/align-store.c          |    2 +-
 clang/test/CIR/CodeGen/array.cpp              |   26 +-
 clang/test/CIR/CodeGen/assign-operator.cpp    |  144 +
 clang/test/CIR/CodeGen/basic.c                |   48 +-
 clang/test/CIR/CodeGen/basic.cpp              |   18 +-
 clang/test/CIR/CodeGen/binassign.c            |    2 +-
 clang/test/CIR/CodeGen/binop.cpp              |   56 +-
 clang/test/CIR/CodeGen/bitfield-union.c       |   30 +
 clang/test/CIR/CodeGen/bitfields.c            |   58 +
 clang/test/CIR/CodeGen/bitfields.cpp          |   28 +
 clang/test/CIR/CodeGen/bitfields_be.c         |   29 +-
 clang/test/CIR/CodeGen/builtin_bit.cpp        |  327 +
 clang/test/CIR/CodeGen/builtin_call.cpp       |   60 +-
 clang/test/CIR/CodeGen/builtin_printf.cpp     |   18 +-
 clang/test/CIR/CodeGen/call.c                 |   36 +-
 clang/test/CIR/CodeGen/call.cpp               |   47 +-
 clang/test/CIR/CodeGen/cast.cpp               |   14 +-
 clang/test/CIR/CodeGen/class.cpp              |    6 +-
 clang/test/CIR/CodeGen/cmp.cpp                |   30 +-
 clang/test/CIR/CodeGen/comma.c                |    2 +-
 clang/test/CIR/CodeGen/complex-builtins.cpp   |   35 +
 clang/test/CIR/CodeGen/complex.cpp            |  422 +-
 clang/test/CIR/CodeGen/compound_assign.cpp    |    2 +-
 clang/test/CIR/CodeGen/ctor-alias.cpp         |   75 +
 clang/test/CIR/CodeGen/ctor.cpp               |  126 +-
 clang/test/CIR/CodeGen/deferred-defs.cpp      |    2 +-
 clang/test/CIR/CodeGen/dso-local.c            |   32 +-
 clang/test/CIR/CodeGen/forrange.cpp           |   10 +-
 clang/test/CIR/CodeGen/generic-selection.c    |   24 +
 clang/test/CIR/CodeGen/if.cpp                 |   24 +-
 clang/test/CIR/CodeGen/inline-cxx-func.cpp    |    4 +-
 clang/test/CIR/CodeGen/int-to-bool.cpp        |   16 +-
 clang/test/CIR/CodeGen/linkage-spec.cpp       |   28 +-
 clang/test/CIR/CodeGen/local-vars.cpp         |    2 +-
 clang/test/CIR/CodeGen/loop.cpp               |   40 +-
 clang/test/CIR/CodeGen/member-functions.cpp   |    6 +-
 clang/test/CIR/CodeGen/namespace.cpp          |   16 +-
 clang/test/CIR/CodeGen/new.cpp                |  158 +
 .../CIR/CodeGen/non-type-template-param.cpp   |   27 +
 clang/test/CIR/CodeGen/nullptr-init.cpp       |    2 +-
 clang/test/CIR/CodeGen/opt-info-attr.cpp      |   22 +
 clang/test/CIR/CodeGen/pack-indexing.cpp      |   49 +
 clang/test/CIR/CodeGen/static-vars.c          |   12 +-
 clang/test/CIR/CodeGen/static-vars.cpp        |   12 +-
 clang/test/CIR/CodeGen/string-literals.c      |   22 +-
 clang/test/CIR/CodeGen/string-literals.cpp    |    2 +-
 clang/test/CIR/CodeGen/struct.c               |   20 +-
 clang/test/CIR/CodeGen/struct.cpp             |   10 +-
 clang/test/CIR/CodeGen/switch.cpp             |   94 +-
 clang/test/CIR/CodeGen/switch_flat_op.cpp     |    4 +-
 clang/test/CIR/CodeGen/ternary.cpp            |   12 +-
 clang/test/CIR/CodeGen/typedef.c              |    4 +-
 clang/test/CIR/CodeGen/unary.cpp              |   74 +-
 clang/test/CIR/CodeGen/union.c                |   28 +-
 clang/test/CIR/CodeGen/vector-ext.cpp         |   15 +
 clang/test/CIR/CodeGen/vector.cpp             |   15 +
 clang/test/CIR/CodeGenOpenACC/cache.c         |  132 +
 clang/test/CIR/CodeGenOpenACC/combined-copy.c |   79 +-
 .../combined-copyin-copyout-create.c          |  160 +
 clang/test/CIR/CodeGenOpenACC/combined.cpp    |    4 +-
 clang/test/CIR/CodeGenOpenACC/compute-copy.c  |   63 +-
 .../compute-copyin-copyout-create.c           |  128 +
 .../data-copy-copyin-copyout-create.c         |  218 +
 clang/test/CIR/CodeGenOpenACC/data.c          |   50 +-
 clang/test/CIR/CodeGenOpenACC/enter-data.c    |  125 +
 clang/test/CIR/CodeGenOpenACC/exit-data.c     |  134 +
 clang/test/CIR/CodeGenOpenACC/host_data.c     |    2 +-
 clang/test/CIR/CodeGenOpenACC/init.c          |    2 +-
 clang/test/CIR/CodeGenOpenACC/kernels.c       |    4 +-
 clang/test/CIR/CodeGenOpenACC/loop.cpp        |    2 +-
 clang/test/CIR/CodeGenOpenACC/parallel.c      |    4 +-
 clang/test/CIR/CodeGenOpenACC/serial.c        |    4 +-
 clang/test/CIR/CodeGenOpenACC/set.c           |    2 +-
 clang/test/CIR/CodeGenOpenACC/shutdown.c      |    2 +-
 clang/test/CIR/CodeGenOpenACC/update.c        |  178 +
 clang/test/CIR/CodeGenOpenACC/wait.c          |    2 +-
 clang/test/CIR/IR/array.cir                   |    6 +-
 clang/test/CIR/IR/binassign.cir               |    2 +-
 clang/test/CIR/IR/call.cir                    |   14 +-
 clang/test/CIR/IR/cast.cir                    |    4 +-
 clang/test/CIR/IR/cmp.cir                     |   10 +-
 clang/test/CIR/IR/func.cir                    |   18 +-
 clang/test/CIR/IR/invalid-call.cir            |   12 +-
 clang/test/CIR/IR/invalid-complex.cir         |   24 +
 clang/test/CIR/IR/invalid-func.cir            |   11 +
 clang/test/CIR/IR/invalid-opt-info.cir        |   13 +
 .../IR/invalid-vector-create-wrong-size.cir   |   16 -
 .../IR/invalid-vector-create-wrong-type.cir   |   17 -
 ...invalid-vector-shift-wrong-result-type.cir |   17 -
 ...alid-vector-shuffle-dyn-wrong-operands.cir |   19 -
 .../IR/invalid-vector-shuffle-wrong-index.cir |   16 -
 .../test/CIR/IR/invalid-vector-zero-size.cir  |   10 -
 clang/test/CIR/IR/invalid-vector.cir          |  105 +-
 clang/test/CIR/IR/ternary.cir                 |    2 +-
 clang/test/CIR/IR/unary.cir                   |    4 +-
 clang/test/CIR/IR/vector.cir                  |   16 +-
 clang/test/CIR/Lowering/array.cpp             |   18 +-
 clang/test/CIR/Transforms/canonicalize.cir    |   12 +-
 .../CIR/Transforms/complex-create-fold.cir    |    2 +-
 .../test/CIR/Transforms/complex-imag-fold.cir |   23 +
 .../test/CIR/Transforms/complex-real-fold.cir |   23 +
 clang/test/CIR/Transforms/hoist-allocas.cir   |    6 +-
 clang/test/CIR/Transforms/if.cir              |    4 +-
 clang/test/CIR/Transforms/loop.cir            |    6 +-
 clang/test/CIR/Transforms/scope.cir           |    6 +-
 clang/test/CIR/Transforms/select.cir          |   10 +-
 clang/test/CIR/Transforms/switch.cir          |   20 +-
 clang/test/CIR/Transforms/ternary-fold.cir    |    8 +-
 clang/test/CIR/Transforms/ternary.cir         |    4 +-
 clang/test/CIR/Transforms/vector-cmp-fold.cir |   24 +-
 .../CIR/Transforms/vector-create-fold.cir     |    2 +-
 .../vector-shuffle-dynamic-fold.cir           |    4 +-
 .../CIR/Transforms/vector-shuffle-fold.cir    |    6 +-
 clang/test/CIR/Transforms/vector-splat.cir    |    4 +-
 .../CIR/Transforms/vector-ternary-fold.cir    |    2 +-
 clang/test/CIR/func-linkage.cpp               |   51 +
 clang/test/CIR/func-simple.cpp                |   20 +-
 clang/test/CIR/global-var-linkage.cpp         |    2 +-
 clang/test/CIR/mlprint.c                      |    2 +-
 clang/test/CMakeLists.txt                     |   27 +-
 .../basic/basic.start/basic.start.main/p3.cpp |    5 +-
 .../CXX/class.access/class.access.dcl/p1.cpp  |   11 +-
 .../basic.namespace/namespace.udecl/p3.cpp    |   25 +-
 .../basic.namespace/namespace.udecl/p4.cpp    |   12 +-
 clang/test/CXX/drs/cwg4xx.cpp                 |    9 +
 .../CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp  |   22 +
 clang/test/ClangScanDeps/P1689.cppm           |   21 +-
 .../modules-full-named-modules.cppm           |  351 +
 clang/test/ClangScanDeps/raw-strings.cpp      |  148 +
 clang/test/CodeCompletion/overrides.cpp       |   14 +
 .../skip-explicit-object-parameter.cpp        |   14 +
 .../test/CodeGen/AArch64/attr-fp8-function.c  |   37 +
 clang/test/CodeGen/AArch64/fmv-dependencies.c |    2 +-
 clang/test/CodeGen/AArch64/neon-intrinsics.c  |  389 +
 clang/test/CodeGen/AArch64/neon-ldst-one.c    |  357 +
 .../AArch64/sve-intrinsics/acle_sve_ld1ro.c   |   15 +
 .../AArch64/sve-intrinsics/acle_sve_ld1rq.c   |   15 +
 .../AArch64/sve-intrinsics/acle_sve_ldff1.c   |   32 +
 .../AArch64/sve-intrinsics/acle_sve_ldnf1.c   |   32 +
 clang/test/CodeGen/LoongArch/bitint.c         |  122 +
 .../LoongArch/inline-asm-gcc-regs-error.c     |    9 -
 .../CodeGen/LoongArch/inline-asm-gcc-regs.c   |   62 +-
 .../CodeGen/PowerPC/builtins-bcd-transform.c  |   79 +
 .../non-policy/non-overloaded/nds_vd4dots.c   |    0
 .../non-policy/non-overloaded/nds_vd4dotsu.c  |    0
 .../non-policy/non-overloaded/nds_vd4dotu.c   |    0
 .../non-overloaded/nds_vfncvtbf16s.c          |  108 +
 .../non-policy/non-overloaded/nds_vfpmadb.c   |    1 -
 .../non-policy/non-overloaded/nds_vfpmadt.c   |    1 -
 .../non-overloaded/nds_vfwcvtsbf16.c          |   58 +
 .../non-policy/overloaded/nds_vd4dots.c       |    0
 .../non-policy/overloaded/nds_vd4dotsu.c      |    0
 .../non-policy/overloaded/nds_vd4dotu.c       |    0
 .../non-policy/overloaded/nds_vfncvtbf16s.c   |  108 +
 .../non-policy/overloaded/nds_vfpmadb.c       |    1 -
 .../non-policy/overloaded/nds_vfpmadt.c       |    1 -
 .../non-policy/overloaded/nds_vfwcvtsbf16.c   |   58 +
 .../policy/non-overloaded/nds_vd4dots.c       |    0
 .../policy/non-overloaded/nds_vd4dotsu.c      |    0
 .../policy/non-overloaded/nds_vd4dotu.c       |    0
 .../policy/non-overloaded/nds_vfncvtbf16s.c   |  108 +
 .../policy/non-overloaded/nds_vfpmadb.c       |    1 -
 .../policy/non-overloaded/nds_vfpmadt.c       |    1 -
 .../policy/non-overloaded/nds_vfwcvtsbf16.c   |   58 +
 .../policy/overloaded/nds_vd4dots.c           |    0
 .../policy/overloaded/nds_vd4dotsu.c          |    0
 .../policy/overloaded/nds_vd4dotu.c           |    0
 .../policy/overloaded/nds_vfncvtbf16s.c       |  120 +
 .../policy/overloaded/nds_vfpmadb.c           |    1 -
 .../policy/overloaded/nds_vfpmadt.c           |    1 -
 .../policy/overloaded/nds_vfwcvtsbf16.c       |   63 +
 .../non-overloaded/sf_vfnrclip_x_f_qf.c       |    0
 .../non-overloaded/sf_vfnrclip_x_f_qf_rm.c    |    0
 .../non-overloaded/sf_vfnrclip_xu_f_qf.c      |    0
 .../non-overloaded/sf_vfnrclip_xu_f_qf_rm.c   |    0
 .../non-overloaded/sf_vfwmacc_4x4x4.c         |    0
 .../non-overloaded/sf_vqmacc_2x8x2.c          |    0
 .../non-overloaded/sf_vqmacc_4x8x4.c          |    0
 .../non-overloaded/sf_vqmaccsu_2x8x2.c        |    0
 .../non-overloaded/sf_vqmaccsu_4x8x4.c        |    0
 .../non-overloaded/sf_vqmaccu_2x8x2.c         |    0
 .../non-overloaded/sf_vqmaccu_4x8x4.c         |    0
 .../non-overloaded/sf_vqmaccus_2x8x2.c        |    0
 .../non-overloaded/sf_vqmaccus_4x8x4.c        |    0
 .../overloaded/sf_vfnrclip_x_f_qf.c           |    0
 .../overloaded/sf_vfnrclip_x_f_qf_rm.c        |    0
 .../overloaded/sf_vfnrclip_xu_f_qf.c          |    0
 .../overloaded/sf_vfnrclip_xu_f_qf_rm.c       |    0
 .../non-policy/overloaded/sf_vfwmacc_4x4x4.c  |    0
 .../non-policy/overloaded/sf_vqmacc_2x8x2.c   |    0
 .../non-policy/overloaded/sf_vqmacc_4x8x4.c   |    0
 .../non-policy/overloaded/sf_vqmaccsu_2x8x2.c |    0
 .../non-policy/overloaded/sf_vqmaccsu_4x8x4.c |    0
 .../non-policy/overloaded/sf_vqmaccu_2x8x2.c  |    0
 .../non-policy/overloaded/sf_vqmaccu_4x8x4.c  |    0
 .../non-policy/overloaded/sf_vqmaccus_2x8x2.c |    0
 .../non-policy/overloaded/sf_vqmaccus_4x8x4.c |    0
 .../non-overloaded/sf_vfnrclip_x_f_qf.c       |    0
 .../non-overloaded/sf_vfnrclip_x_f_qf_rm.c    |    0
 .../non-overloaded/sf_vfnrclip_xu_f_qf.c      |    0
 .../non-overloaded/sf_vfnrclip_xu_f_qf_rm.c   |    0
 .../policy/non-overloaded/sf_vfwmacc_4x4x4.c  |    0
 .../policy/non-overloaded/sf_vqmacc_2x8x2.c   |    0
 .../policy/non-overloaded/sf_vqmacc_4x8x4.c   |    0
 .../policy/non-overloaded/sf_vqmaccsu_2x8x2.c |    0
 .../policy/non-overloaded/sf_vqmaccsu_4x8x4.c |    0
 .../policy/non-overloaded/sf_vqmaccu_2x8x2.c  |    0
 .../policy/non-overloaded/sf_vqmaccu_4x8x4.c  |    0
 .../policy/non-overloaded/sf_vqmaccus_2x8x2.c |    0
 .../policy/non-overloaded/sf_vqmaccus_4x8x4.c |    0
 .../policy/overloaded/sf_vfnrclip_x_f_qf.c    |    0
 .../policy/overloaded/sf_vfnrclip_x_f_qf_rm.c |    0
 .../policy/overloaded/sf_vfnrclip_xu_f_qf.c   |    0
 .../overloaded/sf_vfnrclip_xu_f_qf_rm.c       |    0
 .../policy/overloaded/sf_vfwmacc_4x4x4.c      |    0
 .../policy/overloaded/sf_vqmacc_2x8x2.c       |    0
 .../policy/overloaded/sf_vqmacc_4x8x4.c       |    0
 .../policy/overloaded/sf_vqmaccsu_2x8x2.c     |    0
 .../policy/overloaded/sf_vqmaccsu_4x8x4.c     |    0
 .../policy/overloaded/sf_vqmaccu_2x8x2.c      |    0
 .../policy/overloaded/sf_vqmaccu_4x8x4.c      |    0
 .../policy/overloaded/sf_vqmaccus_2x8x2.c     |    0
 .../policy/overloaded/sf_vqmaccus_4x8x4.c     |    0
 ...asm-exception-model-flag-parse-ir-input.ll |   17 +
 .../wasm-invalid-exception-kinds.ll           |    8 +
 .../CodeGen/X86/ms-secure-hotpatch-bad-file.c |   18 +
 .../CodeGen/X86/ms-secure-hotpatch-cpp.cpp    |   24 +
 .../CodeGen/X86/ms-secure-hotpatch-eh.cpp     |   26 +
 .../CodeGen/X86/ms-secure-hotpatch-globals.c  |  135 +
 .../test/CodeGen/X86/ms-secure-hotpatch-lto.c |   26 +
 clang/test/CodeGen/X86/ms-secure-hotpatch.c   |   23 +
 .../CodeGen/amdgpu-builtin-is-invocable.c     |   28 +-
 .../CodeGen/amdgpu-builtin-processor-is.c     |   29 +-
 .../amdgpu-feature-builtins-invalid-use.cpp   |   53 +-
 .../test/CodeGen/arm-neon-directed-rounding.c |   42 +-
 .../arm-sve-vector-bits-vscale-range.c        |   80 +-
 .../test/CodeGen/arm-v8.2a-neon-intrinsics.c  |   24 +-
 .../test/CodeGen/arm64-microsoft-intrinsics.c |   60 +
 clang/test/CodeGen/arm64-microsoft-sys.c      |   65 +
 clang/test/CodeGen/attr-target-mv.c           |    4 +
 clang/test/CodeGen/bitfield-access-pad.c      |   52 +-
 clang/test/CodeGen/bittest-intrin.c           |  129 +-
 clang/test/CodeGen/builtins-overflow.c        |   12 +
 clang/test/CodeGen/debug-label-inline.c       |    2 +-
 clang/test/CodeGen/debug-label.c              |    2 +-
 clang/test/CodeGen/ext-int-cc.c               |   12 +-
 .../fallow-runtime-check-skip-hot-cutoff.c    |   15 +
 clang/test/CodeGen/logb_scalbn.c              |  873 +++
 clang/test/CodeGen/pragma-comment.c           |    3 +
 clang/test/CodeGen/visibility.c               |   30 +-
 clang/test/CodeGenCUDA/bf16.cu                |    6 +-
 clang/test/CodeGenCXX/auto-var-init.cpp       |    1 +
 .../test/CodeGenCXX/bitfield-access-empty.cpp |    1 +
 clang/test/CodeGenCXX/builtin-invoke.cpp      |   61 +
 clang/test/CodeGenCXX/builtins-eh-wasm.cpp    |    3 +-
 .../debug-info-dtor-implicit-args.cpp         |   24 +
 .../microsoft-abi-member-pointers.cpp         |    7 +
 clang/test/CodeGenCXX/visibility.cpp          |   40 +-
 clang/test/CodeGenCXX/wasm-eh.cpp             |    6 +-
 .../AppendStructuredBuffer-elementtype.hlsl   |    3 +
 .../ConsumeStructuredBuffer-elementtype.hlsl  |    3 +
 .../builtins/RWBuffer-elementtype.hlsl        |   10 +-
 .../builtins/RWBuffer-subscript.hlsl          |    8 +-
 .../RWStructuredBuffer-elementtype.hlsl       |    5 +-
 .../StructuredBuffer-elementtype.hlsl         |    5 +-
 .../vk-features/vk.spec-constant.hlsl         |   45 +-
 clang/test/CodeGenOpenCL/amdgpu-features.cl   |    2 +-
 ...ins-amdgcn-gfx1250-async-load-store-lds.cl |   24 +
 .../builtins-amdgcn-gfx1250-load-tr.cl        |  130 +
 ...iltins-amdgcn-gfx1250-tensor-load-store.cl |   46 +
 .../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl  |  129 +
 clang/test/DebugInfo/KeyInstructions/atomic.c |  166 +
 clang/test/DebugInfo/KeyInstructions/flag.cpp |   20 +-
 .../init-member-memcopyable-2.cpp             |   59 +
 .../init-member-memcopyable.cpp               |   62 +
 clang/test/Driver/HLSL/dxc_fvk_layout.hlsl    |    8 +
 .../aarch64-none-elf/include/c++/8.2.1/.keep  |    0
 .../aarch64-none-elf/lib/.keep                |    0
 .../aarch64-none-elf/lib/crt0.o               |    0
 .../bin/aarch64-none-elf-ld                   |    1 +
 .../lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o |    0
 .../lib/gcc/aarch64-none-elf/8.2.1/crtend.o   |    0
 .../aarch64-none-elf/lib/crt0.o               |    0
 .../aarch64-none-elf/lib/crtbegin.o           |    0
 .../aarch64-none-elf/lib/crtend.o             |    0
 .../bin/aarch64-none-elf-ld                   |    1 +
 .../armv6m-none-eabi/include/c++/8.2.1/.keep  |    0
 .../armv6m-none-eabi/lib/.keep                |    0
 .../armv6m-none-eabi/lib/crt0.o               |    0
 .../bin/armv6m-none-eabi-ld                   |    1 +
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o |    0
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtend.o   |    0
 .../armv6m-none-eabi/lib/crt0.o               |    0
 .../armv6m-none-eabi/lib/crtbegin.o           |    0
 .../armv6m-none-eabi/lib/crtend.o             |    0
 .../bin/armv6m-none-eabi-ld                   |    1 +
 clang/test/Driver/aarch64-gnutools.c          |    4 +
 .../Driver/aarch64-implied-sve-features.c     |    4 +-
 clang/test/Driver/aarch64-sve-vector-bits.c   |    4 +
 clang/test/Driver/aarch64-toolchain-extra.c   |   35 +
 clang/test/Driver/aarch64-toolchain.c         |  159 +
 .../Driver/amdgpu-openmp-sanitize-options.c   |    4 +-
 clang/test/Driver/arm-gnutools.c              |    6 +
 clang/test/Driver/arm-toolchain-extra.c       |   36 +
 clang/test/Driver/arm-toolchain.c             |  160 +
 clang/test/Driver/autocomplete.c              |    1 +
 clang/test/Driver/baremetal-multilib.yaml     |    3 +-
 clang/test/Driver/baremetal-sysroot.cpp       |    8 +-
 .../test/Driver/baremetal-undefined-symbols.c |   10 +
 clang/test/Driver/baremetal.cpp               |  147 +-
 clang/test/Driver/check-no-multlib-warning.c  |   10 +
 clang/test/Driver/cl-include.c                |    5 +
 clang/test/Driver/cl-options.c                |    6 +
 clang/test/Driver/dxc_fspv_extension.hlsl     |   11 +-
 clang/test/Driver/dxc_hlsl-rootsig-ver.hlsl   |   16 +
 .../fallow-runtime-check-skip-hot-cutoff.c    |    6 +
 clang/test/Driver/fsanitize.c                 |   16 +-
 clang/test/Driver/gcc-toolchain.cpp           |    5 -
 clang/test/Driver/hip-options.hip             |    2 +-
 clang/test/Driver/ir-exception-model.c        |   14 +
 clang/test/Driver/openmp-offload-gpu.c        |    2 +-
 clang/test/Driver/pgo-sample-use-profi.c      |   21 +-
 .../aarch64-fujitsu-monaka.c                  |    2 +-
 .../print-enabled-extensions/aarch64-gb10.c   |   69 +
 .../print-enabled-extensions/aarch64-grace.c  |    2 +-
 .../aarch64-olympus.c                         |    2 +-
 .../riscv-andes-a25.c                         |    2 +-
 .../riscv-andes-a45.c                         |    2 +-
 .../riscv-andes-ax25.c                        |    2 +-
 .../riscv-andes-ax45.c                        |    2 +-
 .../riscv-andes-ax45mpv.c                     |    2 +-
 .../riscv-andes-n45.c                         |    2 +-
 .../riscv-andes-nx45.c                        |    2 +-
 .../riscv-sifive-p870.c                       |    2 +-
 .../riscv-sifive-x390.c                       |    2 +-
 .../Driver/print-libgcc-file-name-clangrt.c   |    9 +
 .../print-supported-extensions-aarch64.c      |    3 +-
 .../Driver/print-supported-extensions-riscv.c |    4 +-
 clang/test/Driver/ps5-linker.c                |   22 +-
 clang/test/Driver/riscv-args.c                |    6 -
 clang/test/Driver/riscv-cpus.c                |    3 +
 clang/test/Driver/riscv32-toolchain-extra.c   |    7 +-
 clang/test/Driver/riscv32-toolchain.c         |   26 +-
 clang/test/Driver/riscv64-toolchain-extra.c   |    7 +-
 clang/test/Driver/riscv64-toolchain.c         |   20 +-
 clang/test/Driver/sanitizer-ld.c              |    2 +-
 .../ExtractAPI/constructor_destructor.cpp     |    2 +-
 clang/test/ExtractAPI/global_record.c         |    4 +-
 .../test/ExtractAPI/global_record_multifile.c |    4 +-
 clang/test/ExtractAPI/macro_undefined.c       |    4 +-
 clang/test/ExtractAPI/pointers.c              |  388 +
 clang/test/Index/inline-assembly.c            |   46 +
 clang/test/Interpreter/out-of-process.cpp     |   88 +
 ...a-attribute-supported-attributes-list.test |    1 +
 .../Misc/target-invalid-cpu-note/aarch64.c    |    1 +
 clang/test/Modules/ModulesLocalNamespace.cppm |   47 +
 clang/test/Modules/mingw-exceptions.cppm      |    5 +
 .../Modules/no-transitive-decl-change-4.cppm  |   81 +
 clang/test/Modules/odr_hash.cpp               |   23 +
 clang/test/Modules/pr118137.cppm              |   24 +
 clang/test/Modules/pr145975.cppm              |   58 +
 clang/test/Modules/template-declare.cppm      |   39 +
 .../var-init-side-effects-modulemap.cpp       |   51 +
 clang/test/OpenMP/declare_mapper_messages.c   |   63 +-
 clang/test/OpenMP/declare_target_messages.cpp |  106 +-
 .../declare_variant_clauses_messages.cpp      |    2 +-
 .../test/OpenMP/openmp-6-future-spellings.cpp |   55 +
 clang/test/OpenMP/parallel_ast_print.cpp      |   42 +-
 .../OpenMP/parallel_num_threads_messages.cpp  |   73 +-
 clang/test/OpenMP/target_data_ast_print.cpp   |    8 +-
 clang/test/OpenMP/target_map_messages.cpp     |   59 +-
 clang/test/Options/Gis.hlsl                   |   13 +
 clang/test/Parser/c23-typeof.m                |    7 +
 clang/test/Parser/c2x-typeof.c                |   10 +
 .../ParserOpenACC/parse-cache-construct.c     |   40 +-
 .../ParserOpenACC/parse-cache-construct.cpp   |   18 +-
 .../Preprocessor/aarch64-target-features.c    |   16 +-
 clang/test/Preprocessor/arm-target-features.c |    5 +
 .../Preprocessor/macro_paste_identifier_ucn.c |   10 +
 .../riscv-target-features-andes.c             |   10 +
 clang/test/Sema/Float16.c                     |    1 +
 clang/test/Sema/attr-arm-sve-vector-bits.c    |   31 +-
 .../{CodeGen => Sema}/attr-cpuspecific-cpus.c |    4 +-
 clang/test/Sema/attr-nonstring_safe.c         |   51 +
 clang/test/Sema/attr-visibility.c             |    6 +
 clang/test/Sema/builtins-bcd-transform.c      |   30 +
 clang/test/Sema/builtins-microsoft-arm64.c    |    9 +
 clang/test/Sema/deep_recursion.c              |  209 +
 .../Sema/implicit-int-conversion-on-int.c     |    8 +
 clang/test/Sema/offsetof-64.c                 |    4 +-
 .../rvv-andes-required-features-invalid.c     |   20 +
 .../test/Sema/rvv-required-features-invalid.c |    7 +
 clang/test/Sema/warn-documentation.cpp        |   12 +
 .../Sema/warn-format-overflow-truncation.c    |   14 +
 clang/test/SemaCXX/Float16.cpp                |    2 +-
 clang/test/SemaCXX/P1979.cpp                  |   38 +
 ...rch64-streaming-sve-vector-conversions.cpp |   69 +
 clang/test/SemaCXX/builtin-invoke.cpp         |  240 +
 .../test/SemaCXX/cxx2a-constexpr-dynalloc.cpp |   64 +
 clang/test/SemaCXX/cxx2c-binding-pack.cpp     |   30 +-
 .../SemaCXX/cxx2c-trivially-relocatable.cpp   |   36 +
 clang/test/SemaCXX/modules.cppm               |    4 +
 clang/test/SemaCXX/nested-name-spec.cpp       |   40 +-
 .../type-traits-unsatisfied-diags-std.cpp     |  158 +
 .../SemaCXX/type-traits-unsatisfied-diags.cpp |  279 +
 .../test/SemaCXX/unique_object_duplication.h  |   45 +-
 ...ffer-usage-in-container-span-construct.cpp |   60 +
 .../SemaCXX/wmissing-noreturn-suggestion.cpp  |   23 +
 clang/test/SemaCXX/wreturn-always-throws.cpp  |   46 +
 .../amdgpu-feature-builtins-cant-jump.hip     |   62 +
 ...feature-builtins-return-type-deduction.hip |   31 +
 clang/test/SemaHLSL/RootSignature-err.hlsl    |    4 +
 .../RootSignature-resource-ranges-err.hlsl    |   43 +
 .../RootSignature-resource-ranges.hlsl        |    9 +
 clang/test/SemaObjCXX/arc-0x.mm               |    7 +-
 clang/test/SemaObjCXX/attr-swift_name-cxx.mm  |   36 +
 .../test/SemaOpenACC/cache-construct-ast.cpp  |   91 +-
 clang/test/SemaOpenACC/cache-construct.cpp    |   21 +-
 .../SemaOpenACC/cache-warn-invalid-varloc.cpp |   29 +
 .../combined-construct-copyout-ast.cpp        |   10 +-
 .../combined-construct-copyout-clause.c       |    8 +-
 .../compute-construct-copyout-clause.c        |    8 +-
 .../compute-construct-varlist-ast.cpp         |   12 +-
 .../data-construct-copyout-ast.cpp            |    6 +-
 .../data-construct-copyout-clause.c           |   12 +-
 clang/test/SemaOpenACC/declare-construct.cpp  |    9 +-
 clang/test/SemaOpenCL/amd-media-ops.cl        |  103 +
 .../builtins-amdgcn-error-gfx1250-param.cl    |   23 +
 clang/test/lit.cfg.py                         |   31 +-
 clang/test/lit.site.cfg.py.in                 |    4 +-
 clang/tools/c-index-test/c-index-test.c       |   50 +
 .../clang-fuzzer/handle-llvm/CMakeLists.txt   |    1 +
 .../clang-installapi/ClangInstallAPI.cpp      |    4 +-
 clang/tools/clang-installapi/Options.cpp      |   12 +-
 clang/tools/clang-repl/CMakeLists.txt         |    2 +
 clang/tools/clang-repl/ClangRepl.cpp          |  123 +-
 clang/tools/clang-scan-deps/ClangScanDeps.cpp |   34 +-
 clang/tools/clang-scan-deps/Opts.td           |    1 -
 clang/tools/libclang/CIndex.cpp               |  114 +-
 clang/tools/libclang/CIndexHigh.cpp           |    6 +-
 clang/tools/libclang/CXIndexDataConsumer.cpp  |    6 +-
 clang/tools/libclang/CXSourceLocation.cpp     |    4 +-
 clang/tools/libclang/Indexing.cpp             |    4 +-
 clang/tools/libclang/libclang.map             |    9 +
 clang/unittests/AST/ASTTraverserTest.cpp      |   75 +
 clang/unittests/AllClangUnitTests.cpp         |   28 +
 clang/unittests/CMakeLists.txt                |    1 +
 clang/unittests/Format/FormatTest.cpp         |   29 +-
 clang/unittests/Format/FormatTestBase.h       |    2 +-
 clang/unittests/Format/FormatTestComments.cpp |    2 +-
 .../unittests/Format/FormatTestSelective.cpp  |    5 +
 clang/unittests/Format/QualifierFixerTest.cpp |    7 +-
 clang/unittests/Index/IndexTests.cpp          |    4 +-
 .../Lex/DependencyDirectivesScannerTest.cpp   |   31 +
 clang/unittests/Lex/LexerTest.cpp             |   41 +-
 .../Parse/ParseHLSLRootSignatureTest.cpp      |  324 +-
 .../unittests/Sema/HeuristicResolverTest.cpp  |   80 +
 .../SourceLocationEncodingTest.cpp            |   58 -
 .../DependencyScannerTest.cpp                 |   55 +-
 .../DependencyScanningFilesystemTest.cpp      |   78 +-
 clang/utils/TableGen/ClangAttrEmitter.cpp     |   10 +-
 .../TableGen/ClangDiagnosticsEmitter.cpp      |    3 +-
 clang/www/cxx_status.html                     |    4 +-
 cmake/Modules/GetClangResourceDir.cmake       |    3 +-
 compiler-rt/cmake/Modules/AddCompilerRT.cmake |   18 +
 compiler-rt/lib/asan/CMakeLists.txt           |   13 +
 .../lib/asan/asan.link_with_main_exec.txt     |  115 +
 compiler-rt/lib/asan/asan_allocator.cpp       |   12 +-
 compiler-rt/lib/asan/asan_allocator.h         |    1 +
 .../lib/asan/asan_cxx.link_with_main_exec.txt |   21 +
 compiler-rt/lib/asan/asan_descriptions.cpp    |   22 +-
 compiler-rt/lib/asan/asan_flags.cpp           |   32 +-
 compiler-rt/lib/asan/asan_internal.h          |    1 +
 compiler-rt/lib/asan/asan_new_delete.cpp      |  206 +-
 compiler-rt/lib/asan/asan_rtl.cpp             |   52 +-
 .../lib/asan/tests/asan_noinst_test.cpp       |    4 +-
 compiler-rt/lib/builtins/arm/aeabi_cdcmp.S    |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_cfcmp.S    |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_dcmp.S     |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_fcmp.S     |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_idivmod.S  |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_ldivmod.S  |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_uidivmod.S |    2 +-
 compiler-rt/lib/builtins/arm/aeabi_uldivmod.S |    2 +-
 compiler-rt/lib/builtins/assembly.h           |    6 +
 compiler-rt/lib/msan/msan.cpp                 |   66 +-
 .../lib/msan/msan_interface_internal.h        |    2 +
 .../lib/tsan/rtl/tsan_interceptors_posix.cpp  |   41 +
 .../lib/tsan/rtl/tsan_interface_java.cpp      |    1 -
 compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp  |   22 +-
 compiler-rt/lib/tsan/rtl/tsan_rtl.cpp         |   21 +-
 compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp  |   18 +-
 compiler-rt/lib/tsan/rtl/tsan_sync.cpp        |   14 +
 .../Windows/alloc_dealloc_mismatch.cpp        |   29 +
 .../test/fuzzer/uncaught-exception.test       |    6 +-
 .../msan_print_shadow_on_outlined_check.cpp   |   42 +
 .../TestCases/Linux/free_aligned_sized.c      |    2 +-
 .../TestCases/Linux/free_sized.c              |    2 +-
 .../ios_commands/iossim_env.py                |   12 +-
 .../ios_commands/iossim_run.py                |   12 +-
 .../test/sanitizer_common/lit.common.cfg.py   |    2 +-
 compiler-rt/test/tsan/java_heap_init2.cpp     |   33 +
 compiler-rt/test/tsan/munmap_clear_shadow.c   |   63 +
 cross-project-tests/CMakeLists.txt            |    7 +
 cross-project-tests/dtlto/README.md           |    3 +
 cross-project-tests/dtlto/ld-dtlto.c          |   41 +
 cross-project-tests/dtlto/lit.local.cfg       |    5 +
 flang-rt/lib/runtime/descriptor-io.cpp        |   63 +-
 flang/CMakeLists.txt                          |    3 +-
 flang/docs/Extensions.md                      |   14 +
 flang/docs/OpenMP-declare-target.md           |   20 +-
 flang/docs/OpenMP-descriptor-management.md    |   11 +-
 flang/docs/ReleaseNotes.md                    |    7 +
 flang/include/flang/Evaluate/complex.h        |    2 +-
 flang/include/flang/Evaluate/tools.h          |   11 +
 flang/include/flang/Lower/AbstractConverter.h |    7 +-
 flang/include/flang/Lower/LoweringOptions.def |    4 +
 flang/include/flang/Lower/SymbolMap.h         |    4 +-
 .../flang/Optimizer/Builder/FIRBuilder.h      |    4 +-
 .../flang/Optimizer/Builder/IntrinsicCall.h   |    5 +-
 .../include/flang/Optimizer/CodeGen/CodeGen.h |    3 +
 .../include/flang/Optimizer/Dialect/FIROps.td |   10 +-
 .../flang/Optimizer/Dialect/FIRTypes.td       |    3 +-
 .../Optimizer/Dialect/Support/KindMapping.h   |    2 +-
 .../flang/Optimizer/Passes/CommandLineOpts.h  |   13 +
 flang/include/flang/Optimizer/Support/Utils.h |   26 -
 flang/include/flang/Parser/message.h          |    6 +-
 flang/include/flang/Parser/parse-tree.h       |    5 +-
 .../flang/Semantics/runtime-type-info.h       |    4 +
 .../include/flang/Support/Fortran-features.h  |    7 +-
 flang/include/flang/Support/OpenMP-features.h |   14 +-
 flang/lib/Evaluate/fold-implementation.h      |  105 +-
 flang/lib/Evaluate/tools.cpp                  |    5 +
 flang/lib/Evaluate/type.cpp                   |   47 +-
 flang/lib/Frontend/CompilerInvocation.cpp     |   50 +-
 flang/lib/Frontend/FrontendAction.cpp         |   13 +-
 flang/lib/Frontend/FrontendActions.cpp        |    4 -
 flang/lib/Lower/Allocatable.cpp               |   25 +-
 flang/lib/Lower/Bridge.cpp                    |   24 +-
 flang/lib/Lower/CallInterface.cpp             |   10 +-
 flang/lib/Lower/ConvertCall.cpp               |    4 +-
 flang/lib/Lower/ConvertConstant.cpp           |    7 +-
 flang/lib/Lower/ConvertExprToHLFIR.cpp        |   14 +-
 flang/lib/Lower/ConvertVariable.cpp           |   46 +-
 flang/lib/Lower/HostAssociations.cpp          |    4 +-
 flang/lib/Lower/Mangler.cpp                   |    6 +-
 flang/lib/Lower/OpenACC.cpp                   |    7 +-
 flang/lib/Lower/OpenMP/Atomic.cpp             |  281 +-
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |    2 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  118 +-
 .../Lower/OpenMP/PrivateReductionUtils.cpp    |   32 +-
 .../lib/Lower/OpenMP/PrivateReductionUtils.h  |    4 +-
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp |   26 +-
 flang/lib/Lower/SymbolMap.cpp                 |    4 +
 flang/lib/Optimizer/Builder/Character.cpp     |    2 +-
 flang/lib/Optimizer/Builder/IntrinsicCall.cpp |  202 +-
 flang/lib/Optimizer/Builder/MutableBox.cpp    |    2 +-
 .../Optimizer/Builder/PPCIntrinsicCall.cpp    |    3 +-
 flang/lib/Optimizer/CodeGen/CodeGen.cpp       |  116 +-
 flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp |   11 +-
 flang/lib/Optimizer/CodeGen/TypeConverter.cpp |    4 +-
 flang/lib/Optimizer/Dialect/FIRType.cpp       |    6 +-
 .../HLFIR/Transforms/ConvertToFIR.cpp         |    5 +-
 .../Transforms/OptimizedBufferization.cpp     |   57 +-
 .../OpenACC/FIROpenACCTypeInterfaces.cpp      |   37 +-
 .../lib/Optimizer/Passes/CommandLineOpts.cpp  |    6 +
 flang/lib/Optimizer/Passes/Pipelines.cpp      |    1 +
 .../lib/Optimizer/Transforms/AddAliasTags.cpp |   30 +-
 .../lib/Optimizer/Transforms/AddDebugInfo.cpp |  117 +-
 .../Optimizer/Transforms/CUFDeviceGlobal.cpp  |   10 +-
 .../Transforms/DebugTypeGenerator.cpp         |  236 +-
 .../Optimizer/Transforms/DebugTypeGenerator.h |   74 +-
 .../Transforms/PolymorphicOpConversion.cpp    |   95 +-
 flang/lib/Parser/message.cpp                  |   31 +-
 flang/lib/Parser/openmp-parsers.cpp           |    2 +-
 flang/lib/Parser/preprocessor.cpp             |    8 +-
 flang/lib/Parser/prescan.cpp                  |   54 +-
 flang/lib/Parser/prescan.h                    |   17 +-
 flang/lib/Parser/token-sequence.cpp           |    3 +-
 flang/lib/Parser/unparse.cpp                  |    7 +-
 flang/lib/Semantics/CMakeLists.txt            |    8 +-
 flang/lib/Semantics/canonicalize-omp.cpp      |   97 +-
 .../lib/Semantics/check-directive-structure.h |    2 +
 flang/lib/Semantics/check-io.cpp              |    2 +
 flang/lib/Semantics/check-omp-atomic.cpp      | 1295 ++++
 flang/lib/Semantics/check-omp-loop.cpp        |  671 ++
 .../lib/Semantics/check-omp-metadirective.cpp |  548 ++
 flang/lib/Semantics/check-omp-structure.cpp   | 2820 +------
 flang/lib/Semantics/check-omp-structure.h     |    8 -
 flang/lib/Semantics/openmp-utils.cpp          |  393 +
 flang/lib/Semantics/openmp-utils.h            |   66 +
 flang/lib/Semantics/resolve-directives.cpp    |  108 +-
 flang/lib/Semantics/resolve-names.cpp         |   27 +-
 flang/lib/Semantics/semantics.cpp             |    4 +-
 flang/lib/Support/Fortran-features.cpp        |   14 +-
 flang/module/cooperative_groups.f90           |   56 +
 .../test/Driver/bbc-openmp-version-macro.f90  |   11 +-
 .../Driver/flang-openmp-version-macro.f90     |    8 -
 flang/test/Driver/fopenmp-version.F90         |   25 +
 flang/test/Driver/fveclib-codegen.f90         |    4 +
 flang/test/Driver/fveclib.f90                 |    3 +-
 flang/test/Driver/omp-driver-offload.f90      |    6 +-
 flang/test/Driver/tco-emit-final-mlir.fir     |   19 +
 flang/test/Evaluate/fold-dim.f90              |    2 +-
 flang/test/Evaluate/fold-nearest.f90          |   22 +-
 flang/test/Evaluate/fold-out_of_range.f90     |   80 +-
 flang/test/Evaluate/fold-unsigned.f90         |    2 +-
 flang/test/Evaluate/folding03.f90             |   76 +-
 flang/test/Evaluate/folding04.f90             |   22 +-
 flang/test/Evaluate/folding05.f90             |  Bin 10993 -> 11212 bytes
 flang/test/Evaluate/folding06.f90             |    4 +-
 flang/test/Evaluate/folding14.f90             |    4 +-
 flang/test/Evaluate/folding28.f90             |    2 +-
 flang/test/Fir/CUDA/cuda-device-global.f90    |   13 +
 .../OpenACC/openacc-type-categories-class.f90 |   46 +
 .../test/Fir/convert-nontemporal-to-llvm.fir  |   18 +-
 flang/test/Fir/convert-to-llvm.fir            |    4 +-
 flang/test/Fir/simd-nontemporal.fir           |   12 +-
 flang/test/HLFIR/declare-codegen.fir          |   18 +
 flang/test/HLFIR/opt-scalar-assign.fir        |   50 +-
 .../OpenMP/map-types-and-sizes.f90            |   36 +-
 ...u-target-desc-cast-to-global-addrspace.f90 |   18 +
 ...ug-cyclic-derived-type-caching-complex.f90 |  116 +
 ...bug-cyclic-derived-type-caching-simple.f90 |   39 +
 .../skip-external-rtti-definition.F90         |   47 +
 flang/test/Lower/CUDA/cuda-cooperative.cuf    |   99 +
 flang/test/Lower/CUDA/cuda-runtime-check.cuf  |   38 +-
 flang/test/Lower/HLFIR/custom-intrinsic.f90   |   16 +-
 flang/test/Lower/Intrinsics/acosd.f90         |   14 +-
 flang/test/Lower/Intrinsics/asind.f90         |   14 +-
 .../Lower/Intrinsics/ieee_class_queries.f90   |    2 +-
 flang/test/Lower/Intrinsics/product.f90       |    4 +-
 flang/test/Lower/Intrinsics/sum.f90           |    4 +-
 flang/test/Lower/OpenMP/atomic-update.f90     |   12 +-
 .../OpenMP/minmax-optional-parameters.f90     |   68 +
 ...nested-loop-transformation-construct01.f90 |   20 +
 ...nested-loop-transformation-construct02.f90 |   20 +
 .../parallel-firstprivate-clause-scalar.f90   |   20 +-
 .../OpenMP/reduction-array-intrinsic.f90      |    8 +-
 .../Lower/OpenMP/target-data-if-false.f90     |   25 +
 .../OpenMP/target-data-skip-mapper-calls.f90  |   30 +
 ...oop-reduction-allocatable-array-minmax.f90 |   58 +-
 .../wsloop-reduction-logical-and-byref.f90    |    8 +-
 .../OpenMP/wsloop-reduction-logical-and.f90   |    8 +-
 .../wsloop-reduction-logical-eqv-byref.f90    |    8 +-
 .../OpenMP/wsloop-reduction-logical-eqv.f90   |    8 +-
 .../OpenMP/wsloop-reduction-logical-kinds.f90 |   24 +
 .../wsloop-reduction-logical-neqv-byref.f90   |    8 +-
 .../OpenMP/wsloop-reduction-logical-neqv.f90  |    8 +-
 .../wsloop-reduction-logical-or-byref.f90     |    8 +-
 .../OpenMP/wsloop-reduction-logical-or.f90    |    8 +-
 .../OpenMP/wsloop-reduction-max-byref.f90     |   10 +-
 .../Lower/OpenMP/wsloop-reduction-max.f90     |   10 +-
 .../OpenMP/wsloop-reduction-min-byref.f90     |   10 +-
 .../Lower/OpenMP/wsloop-reduction-min.f90     |   10 +-
 flang/test/Lower/complex-operations.f90       |    4 +-
 flang/test/Lower/math-lowering/aint.f90       |    8 +-
 flang/test/Lower/select-type-2.f90            |    8 +-
 flang/test/Lower/select-type.f90              |   72 +-
 .../variable-common-viewed-as-module-var.f90  |   37 +
 flang/test/Parser/OpenMP/declare-variant.f90  |   27 +
 .../loop-transformation-construct01.f90       |   74 +
 .../loop-transformation-construct02.f90       |   85 +
 flang/test/Parser/issue-146362.1.f90          |    5 +
 flang/test/Parser/issue-146362.2.f90          |   18 +
 flang/test/Preprocessing/bug1077.F90          |    7 +
 flang/test/Preprocessing/bug518.F             |    2 +-
 flang/test/Preprocessing/pp111.F90            |    2 +-
 flang/test/Preprocessing/pp112.F90            |    2 +-
 flang/test/Preprocessing/pp115.F90            |    2 +-
 flang/test/Preprocessing/pp116.F90            |    2 +-
 flang/test/Semantics/OpenACC/acc-branch.f90   |    2 +-
 flang/test/Semantics/OpenACC/acc-data.f90     |    2 +-
 .../OpenACC/acc-declare-validity.f90          |    2 +-
 flang/test/Semantics/OpenACC/acc-serial.f90   |    6 +-
 .../Semantics/OpenMP/allocate-align01.f90     |    2 +-
 flang/test/Semantics/OpenMP/allocate01.f90    |    2 +-
 .../Semantics/OpenMP/clause-validity01.f90    |    4 +-
 .../Semantics/OpenMP/combined-constructs.f90  |    8 +-
 flang/test/Semantics/OpenMP/copying.f90       |    8 +-
 .../OpenMP/declarative-directive01.f90        |    8 +-
 .../Semantics/OpenMP/declare-target01.f90     |   26 +-
 .../Semantics/OpenMP/declare-target02.f90     |   30 +-
 .../Semantics/OpenMP/declare-target03.f90     |    2 +-
 .../Semantics/OpenMP/declare-target06.f90     |    2 +-
 flang/test/Semantics/OpenMP/deprecation.f90   |   12 +-
 .../Semantics/OpenMP/do05-positivecase.f90    |   14 +
 .../loop-transformation-construct01.f90       |  100 +
 flang/test/Semantics/OpenMP/nested-target.f90 |   14 +-
 flang/test/Semantics/OpenMP/requires04.f90    |    2 +-
 flang/test/Semantics/OpenMP/requires05.f90    |    2 +-
 flang/test/Semantics/OpenMP/single03.f90      |    2 +-
 flang/test/Semantics/OpenMP/single04.f90      |   12 +-
 flang/test/Semantics/OpenMP/target01.f90      |    8 +-
 .../test/Semantics/OpenMP/threadprivate03.f90 |    2 +-
 .../test/Semantics/OpenMP/use_device_ptr1.f90 |    2 +-
 flang/test/Semantics/allocate09.f90           |    4 +-
 flang/test/Semantics/argshape01.f90           |    4 +-
 flang/test/Semantics/assign02.f90             |   14 +-
 flang/test/Semantics/assign09.f90             |    4 +-
 flang/test/Semantics/associate01.f90          |    2 +-
 flang/test/Semantics/associated.f90           |   14 +-
 flang/test/Semantics/bind-c02.f90             |    4 +-
 flang/test/Semantics/bind-c04.f90             |    2 +-
 flang/test/Semantics/bind-c06.f90             |    6 +-
 flang/test/Semantics/bind-c11.f90             |    2 +-
 flang/test/Semantics/bind-c13.f90             |    4 +-
 flang/test/Semantics/bind-c17.f90             |    2 +-
 flang/test/Semantics/bindings01.f90           |    6 +-
 flang/test/Semantics/bindings03.f90           |    4 +-
 flang/test/Semantics/block-data01.f90         |    2 +-
 .../test/Semantics/boz-literal-constants.f90  |    2 +-
 flang/test/Semantics/bug1046.f90              |   15 +
 flang/test/Semantics/bug1056.f90              |   13 +
 flang/test/Semantics/bug1092.F90              |   26 +
 flang/test/Semantics/bug144453.f90            |   10 +
 flang/test/Semantics/c7108.f90                |    4 +-
 flang/test/Semantics/c_f_pointer.f90          |    8 +-
 flang/test/Semantics/c_loc01.f90              |    6 +-
 flang/test/Semantics/call01.f90               |    6 +-
 flang/test/Semantics/call02.f90               |    6 +-
 flang/test/Semantics/call03.f90               |   12 +-
 flang/test/Semantics/call05.f90               |    4 +-
 flang/test/Semantics/call07.f90               |    6 +-
 flang/test/Semantics/call09.f90               |    2 +-
 flang/test/Semantics/call10.f90               |    2 +-
 flang/test/Semantics/call14.f90               |    4 +-
 flang/test/Semantics/call24.f90               |    2 +-
 flang/test/Semantics/call27.f90               |    4 +-
 flang/test/Semantics/call30.f90               |   30 +-
 flang/test/Semantics/call31.f90               |    4 +-
 flang/test/Semantics/call33.f90               |    2 +-
 flang/test/Semantics/call34.f90               |    6 +-
 flang/test/Semantics/call35.f90               |    4 +-
 flang/test/Semantics/call36.f90               |    2 +-
 flang/test/Semantics/call37.f90               |   14 +-
 flang/test/Semantics/call38.f90               |    4 +-
 flang/test/Semantics/call41.f90               |    2 +-
 flang/test/Semantics/call42.f90               |   12 +-
 flang/test/Semantics/call43.f90               |    6 +-
 flang/test/Semantics/call44.f90               |    4 +-
 flang/test/Semantics/case01.f90               |   12 +-
 flang/test/Semantics/common-blocks.f90        |    4 +-
 flang/test/Semantics/contiguous01.f90         |   16 +-
 flang/test/Semantics/cuf01.cuf                |    8 +-
 flang/test/Semantics/cuf03.cuf                |   10 +-
 flang/test/Semantics/cuf04.cuf                |    4 +-
 flang/test/Semantics/cuf09.cuf                |    2 +-
 flang/test/Semantics/data06.f90               |    4 +-
 flang/test/Semantics/data17.f90               |    4 +-
 flang/test/Semantics/declarations04.f90       |    2 +-
 flang/test/Semantics/declarations05.f90       |    2 +-
 flang/test/Semantics/declarations07.f90       |    4 +-
 flang/test/Semantics/deferred01.f90           |    2 +-
 flang/test/Semantics/definable02.f90          |    4 +-
 flang/test/Semantics/dim01.f90                |   16 +-
 flang/test/Semantics/dosemantics02.f90        |    4 +-
 flang/test/Semantics/dosemantics03.f90        |   36 +-
 flang/test/Semantics/dosemantics12.f90        |    6 +-
 flang/test/Semantics/expr-errors05.f90        |   14 +-
 flang/test/Semantics/expr-errors06.f90        |    2 +-
 flang/test/Semantics/final03.f90              |    2 +-
 flang/test/Semantics/forall01.f90             |   22 +-
 flang/test/Semantics/forall02.f90             |    6 +-
 .../Semantics/function-result-extent-max.f90  |   30 +
 flang/test/Semantics/generic03.f90            |    2 +-
 flang/test/Semantics/generic06.f90            |    4 +-
 flang/test/Semantics/global01.f90             |    4 +-
 flang/test/Semantics/ichar01.f90              |    4 +-
 flang/test/Semantics/ignore_tkr01.f90         |   10 +-
 flang/test/Semantics/int-literals.f90         |    4 +-
 flang/test/Semantics/intrinsics02.f90         |    2 +-
 flang/test/Semantics/kinds05b.f90             |    4 +-
 flang/test/Semantics/label18.f90              |    4 +-
 flang/test/Semantics/local-vs-global.f90      |   32 +-
 flang/test/Semantics/long-name.f90            |    6 +-
 flang/test/Semantics/modfile43.f90            |    2 +-
 flang/test/Semantics/null-init.f90            |    2 +-
 flang/test/Semantics/null01.f90               |    6 +-
 flang/test/Semantics/pointer01.f90            |    2 +-
 flang/test/Semantics/procinterface02.f90      |    2 +-
 flang/test/Semantics/procinterface04.f90      |    4 +-
 flang/test/Semantics/resolve05.f90            |    8 +-
 flang/test/Semantics/resolve108.f90           |    4 +-
 flang/test/Semantics/resolve11.f90            |    4 +-
 flang/test/Semantics/resolve114.f90           |   20 +-
 flang/test/Semantics/resolve118.f90           |    2 +-
 flang/test/Semantics/resolve17.f90            |    6 +-
 flang/test/Semantics/resolve18.f90            |    8 +-
 flang/test/Semantics/resolve20.f90            |    4 +-
 flang/test/Semantics/resolve24.f90            |    8 +-
 flang/test/Semantics/resolve30.f90            |    2 +-
 flang/test/Semantics/resolve31.f90            |    8 +-
 flang/test/Semantics/resolve35.f90            |   10 +-
 flang/test/Semantics/resolve37.f90            |   14 +-
 flang/test/Semantics/resolve45.f90            |    6 +-
 flang/test/Semantics/resolve46.f90            |    4 +-
 flang/test/Semantics/resolve55.f90            |   20 +
 flang/test/Semantics/resolve58.f90            |    8 +-
 flang/test/Semantics/resolve59.f90            |    6 +-
 flang/test/Semantics/resolve60.f90            |    2 +-
 flang/test/Semantics/resolve61.f90            |    2 +-
 flang/test/Semantics/resolve65.f90            |    4 +-
 flang/test/Semantics/resolve67.f90            |    4 +-
 flang/test/Semantics/resolve69.f90            |    6 +-
 flang/test/Semantics/resolve77.f90            |    4 +-
 flang/test/Semantics/resolve78.f90            |   10 +-
 flang/test/Semantics/resolve79.f90            |   10 +-
 flang/test/Semantics/resolve80.f90            |   12 +-
 flang/test/Semantics/resolve81.f90            |   38 +-
 flang/test/Semantics/resolve82.f90            |   20 +-
 flang/test/Semantics/resolve83.f90            |   12 +-
 flang/test/Semantics/resolve85.f90            |   10 +-
 flang/test/Semantics/resolve90.f90            |    2 +-
 flang/test/Semantics/resolve99.f90            |    4 +-
 flang/test/Semantics/separate-mp02.f90        |    2 +-
 flang/test/Semantics/spec-expr.f90            |    2 +-
 flang/test/Semantics/stmt-func01.f90          |   16 +-
 flang/test/Semantics/stmt-func02.f90          |    6 +-
 flang/test/Semantics/structconst03.f90        |    2 +-
 flang/test/Semantics/structconst04.f90        |    2 +-
 flang/test/Semantics/structconst08.f90        |   10 +-
 flang/test/Semantics/structconst10.f90        |    8 +-
 flang/test/Semantics/symbol15.f90             |   19 +-
 flang/test/Semantics/transfer01.f90           |   10 +-
 flang/test/Semantics/undef-result01.f90       |    8 +-
 flang/test/Semantics/windows.f90              |    2 +-
 .../test/Transforms/debug-omp-target-op-1.fir |   40 +
 .../test/Transforms/debug-omp-target-op-2.fir |   53 +
 flang/test/Transforms/tbaa-for-local-vars.fir |   92 +
 .../Transforms/tbaa-with-dummy-scope2.fir     |    9 +-
 flang/test/lib/OpenACC/CMakeLists.txt         |    4 +
 .../lib/OpenACC/TestOpenACCInterfaces.cpp     |    9 +
 flang/tools/bbc/bbc.cpp                       |    1 +
 flang/tools/f18/CMakeLists.txt                |   16 +-
 flang/tools/tco/tco.cpp                       |   11 +-
 .../unittests/Common/FortranFeaturesTest.cpp  |    8 +
 libc/config/gpu/amdgpu/entrypoints.txt        |   18 +
 libc/config/gpu/nvptx/entrypoints.txt         |   18 +
 libc/config/linux/arm/entrypoints.txt         |    2 +
 libc/config/linux/x86_64/entrypoints.txt      |    4 +
 libc/include/llvm-libc-types/jmp_buf.h        |    2 +-
 libc/include/wchar.yaml                       |   29 +
 libc/src/__support/CPP/new.h                  |    8 +
 libc/src/__support/GPU/allocator.cpp          |   59 +-
 libc/src/__support/GPU/allocator.h            |    2 +
 libc/src/__support/macros/optimization.h      |    4 +-
 .../macros/properties/cpu_features.h          |    2 +-
 libc/src/__support/wchar/CMakeLists.txt       |    3 +
 .../__support/wchar/character_converter.cpp   |   25 +-
 .../src/__support/wchar/character_converter.h |    1 +
 libc/src/__support/wchar/mbrtowc.cpp          |    7 +-
 libc/src/__support/wchar/wcrtomb.cpp          |    5 +-
 libc/src/setjmp/arm/CMakeLists.txt            |   16 +
 libc/src/setjmp/arm/sigsetjmp.cpp             |   65 +
 libc/src/stdlib/gpu/aligned_alloc.cpp         |   18 +-
 libc/src/stdlib/gpu/realloc.cpp               |   20 +-
 libc/src/string/memory_utils/CMakeLists.txt   |    1 +
 .../string/memory_utils/arm/inline_memcpy.h   |  217 +
 libc/src/string/memory_utils/inline_memcpy.h  |    3 +
 libc/src/string/memory_utils/utils.h          |    2 +-
 libc/src/wchar/CMakeLists.txt                 |   45 +
 libc/src/wchar/mbrtowc.cpp                    |    2 +-
 libc/src/wchar/wcpncpy.h                      |    2 +-
 libc/src/wchar/wcrtomb.cpp                    |    2 +-
 libc/src/wchar/wcscspn.cpp                    |   34 +
 libc/src/wchar/wcscspn.h                      |   22 +
 libc/src/wchar/wcsnlen.cpp                    |   25 +
 libc/src/wchar/wcsnlen.h                      |   22 +
 libc/src/wchar/wcsspn.h                       |    2 +-
 libc/src/wchar/wcstok.cpp                     |   50 +
 libc/src/wchar/wcstok.h                       |   22 +
 libc/src/wchar/wctomb.cpp                     |   35 +
 libc/src/wchar/wctomb.h                       |   22 +
 .../integration/src/stdlib/gpu/CMakeLists.txt |   30 +
 .../src/stdlib/gpu/aligned_alloc.cpp          |   29 +
 .../integration/src/stdlib/gpu/malloc.cpp     |    1 +
 .../integration/src/stdlib/gpu/realloc.cpp    |   44 +
 .../src/__support/wchar/utf32_to_8_test.cpp   |   42 +
 .../src/__support/wchar/utf8_to_32_test.cpp   |   11 +-
 libc/test/src/math/tanpif16_test.cpp          |    2 +-
 libc/test/src/wchar/CMakeLists.txt            |   47 +
 libc/test/src/wchar/mbrtowc_test.cpp          |   63 +-
 libc/test/src/wchar/wcrtomb_test.cpp          |   39 +-
 libc/test/src/wchar/wcscspn_test.cpp          |   67 +
 libc/test/src/wchar/wcsnlen_test.cpp          |   54 +
 libc/test/src/wchar/wcstok_test.cpp           |  181 +
 libc/test/src/wchar/wctomb_test.cpp           |   73 +
 libclc/CMakeLists.txt                         |    3 +
 libclc/clc/include/clc/clcmacro.h             |  218 -
 libclc/clc/include/clc/common/clc_degrees.h   |    4 +-
 libclc/clc/include/clc/common/clc_radians.h   |    4 +-
 libclc/clc/include/clc/common/clc_sign.h      |    4 +-
 libclc/clc/include/clc/common/clc_step.h      |    4 +-
 .../clc/include/clc/geometric/binary_decl.inc |    4 +-
 .../clc/include/clc/geometric/binary_def.inc  |    6 +-
 .../clc/include/clc/geometric/clc_distance.h  |    4 +-
 libclc/clc/include/clc/geometric/clc_dot.h    |    4 +-
 .../include/clc/geometric/clc_fast_distance.h |    5 +-
 .../include/clc/geometric/clc_fast_length.h   |    5 +-
 .../clc/geometric/clc_fast_normalize.h        |    5 +-
 libclc/clc/include/clc/geometric/clc_length.h |    4 +-
 .../clc/include/clc/geometric/clc_normalize.h |    4 +-
 .../clc/include/clc/geometric/unary_decl.inc  |    2 +-
 .../clc/include/clc/geometric/unary_def.inc   |    6 +-
 libclc/clc/include/clc/integer/clc_add_sat.h  |    4 +-
 libclc/clc/include/clc/integer/clc_clz.h      |    4 +-
 libclc/clc/include/clc/integer/clc_ctz.h      |    4 +-
 libclc/clc/include/clc/integer/clc_hadd.h     |    4 +-
 libclc/clc/include/clc/integer/clc_mad24.h    |    4 +-
 libclc/clc/include/clc/integer/clc_mad_sat.h  |    4 +-
 libclc/clc/include/clc/integer/clc_mul24.h    |    4 +-
 libclc/clc/include/clc/integer/clc_mul_hi.h   |    4 +-
 libclc/clc/include/clc/integer/clc_popcount.h |    4 +-
 libclc/clc/include/clc/integer/clc_rhadd.h    |    4 +-
 libclc/clc/include/clc/integer/clc_rotate.h   |    4 +-
 libclc/clc/include/clc/integer/clc_sub_sat.h  |    4 +-
 .../include/clc/internal/math/clc_sw_fma.h    |    4 +-
 .../binary_decl_with_scalar_second_arg.inc    |    8 +-
 .../include/clc/math/binary_def_via_fp32.inc  |    6 +-
 libclc/clc/include/clc/math/clc_acos.h        |    4 +-
 libclc/clc/include/clc/math/clc_acosh.h       |    4 +-
 libclc/clc/include/clc/math/clc_acospi.h      |    4 +-
 libclc/clc/include/clc/math/clc_asin.h        |    4 +-
 libclc/clc/include/clc/math/clc_asinh.h       |    4 +-
 libclc/clc/include/clc/math/clc_asinpi.h      |    4 +-
 libclc/clc/include/clc/math/clc_atan.h        |    4 +-
 libclc/clc/include/clc/math/clc_atan2.h       |    4 +-
 libclc/clc/include/clc/math/clc_atan2pi.h     |    4 +-
 libclc/clc/include/clc/math/clc_atanh.h       |    4 +-
 libclc/clc/include/clc/math/clc_atanpi.h      |    4 +-
 libclc/clc/include/clc/math/clc_cbrt.inc      |    4 +-
 libclc/clc/include/clc/math/clc_ceil.h        |    4 +-
 libclc/clc/include/clc/math/clc_copysign.h    |    4 +-
 libclc/clc/include/clc/math/clc_cos.h         |    4 +-
 libclc/clc/include/clc/math/clc_cosh.h        |    4 +-
 libclc/clc/include/clc/math/clc_cospi.h       |    4 +-
 libclc/clc/include/clc/math/clc_erf.h         |    4 +-
 libclc/clc/include/clc/math/clc_erfc.h        |    4 +-
 libclc/clc/include/clc/math/clc_exp.h         |    4 +-
 libclc/clc/include/clc/math/clc_exp10.h       |    4 +-
 libclc/clc/include/clc/math/clc_exp2.h        |    4 +-
 libclc/clc/include/clc/math/clc_exp_helper.h  |    2 -
 libclc/clc/include/clc/math/clc_expm1.h       |    4 +-
 libclc/clc/include/clc/math/clc_fabs.h        |    4 +-
 libclc/clc/include/clc/math/clc_fdim.h        |    4 +-
 libclc/clc/include/clc/math/clc_floor.h       |    4 +-
 libclc/clc/include/clc/math/clc_fma.h         |    4 +-
 libclc/clc/include/clc/math/clc_fmax.h        |    4 +-
 libclc/clc/include/clc/math/clc_fmin.h        |    4 +-
 libclc/clc/include/clc/math/clc_fmod.h        |    4 +-
 libclc/clc/include/clc/math/clc_fract.h       |    4 +-
 libclc/clc/include/clc/math/clc_frexp.h       |    4 +-
 libclc/clc/include/clc/math/clc_half_cos.h    |    5 +-
 libclc/clc/include/clc/math/clc_half_divide.h |    5 +-
 libclc/clc/include/clc/math/clc_half_exp.h    |    5 +-
 libclc/clc/include/clc/math/clc_half_exp10.h  |    5 +-
 libclc/clc/include/clc/math/clc_half_exp2.h   |    5 +-
 libclc/clc/include/clc/math/clc_half_log.h    |    5 +-
 libclc/clc/include/clc/math/clc_half_log10.h  |    5 +-
 libclc/clc/include/clc/math/clc_half_log2.h   |    5 +-
 libclc/clc/include/clc/math/clc_half_powr.h   |    5 +-
 libclc/clc/include/clc/math/clc_half_recip.h  |    5 +-
 libclc/clc/include/clc/math/clc_half_rsqrt.h  |    5 +-
 libclc/clc/include/clc/math/clc_half_sin.h    |    5 +-
 libclc/clc/include/clc/math/clc_half_sqrt.h   |    5 +-
 libclc/clc/include/clc/math/clc_half_tan.h    |    5 +-
 libclc/clc/include/clc/math/clc_hypot.h       |    4 +-
 libclc/clc/include/clc/math/clc_ilogb.h       |    4 +-
 libclc/clc/include/clc/math/clc_lgamma.h      |    4 +-
 libclc/clc/include/clc/math/clc_lgamma_r.h    |    4 +-
 libclc/clc/include/clc/math/clc_log.h         |    4 +-
 libclc/clc/include/clc/math/clc_log10.h       |    4 +-
 libclc/clc/include/clc/math/clc_log1p.h       |    4 +-
 libclc/clc/include/clc/math/clc_log2.h        |    4 +-
 libclc/clc/include/clc/math/clc_logb.h        |    4 +-
 libclc/clc/include/clc/math/clc_mad.h         |    4 +-
 libclc/clc/include/clc/math/clc_maxmag.h      |    4 +-
 libclc/clc/include/clc/math/clc_minmag.h      |    4 +-
 libclc/clc/include/clc/math/clc_modf.h        |    4 +-
 libclc/clc/include/clc/math/clc_nan.h         |    4 +-
 libclc/clc/include/clc/math/clc_nan.inc       |    1 +
 libclc/clc/include/clc/math/clc_native_cos.h  |    5 +-
 .../clc/include/clc/math/clc_native_divide.h  |    5 +-
 libclc/clc/include/clc/math/clc_native_exp.h  |    5 +-
 .../clc/include/clc/math/clc_native_exp10.h   |    5 +-
 libclc/clc/include/clc/math/clc_native_exp2.h |    5 +-
 libclc/clc/include/clc/math/clc_native_log.h  |    5 +-
 .../clc/include/clc/math/clc_native_log10.h   |    5 +-
 libclc/clc/include/clc/math/clc_native_log2.h |    5 +-
 libclc/clc/include/clc/math/clc_native_powr.h |    5 +-
 .../clc/include/clc/math/clc_native_recip.h   |    5 +-
 .../clc/include/clc/math/clc_native_rsqrt.h   |    5 +-
 libclc/clc/include/clc/math/clc_native_sin.h  |    5 +-
 libclc/clc/include/clc/math/clc_native_sqrt.h |    5 +-
 libclc/clc/include/clc/math/clc_native_tan.h  |    5 +-
 libclc/clc/include/clc/math/clc_nextafter.h   |    4 +-
 libclc/clc/include/clc/math/clc_pow.h         |    4 +-
 libclc/clc/include/clc/math/clc_pown.h        |    6 +-
 libclc/clc/include/clc/math/clc_powr.h        |    4 +-
 libclc/clc/include/clc/math/clc_remainder.h   |    4 +-
 libclc/clc/include/clc/math/clc_remquo.h      |    4 +-
 libclc/clc/include/clc/math/clc_rint.h        |    4 +-
 libclc/clc/include/clc/math/clc_rootn.h       |    6 +-
 libclc/clc/include/clc/math/clc_round.h       |    4 +-
 libclc/clc/include/clc/math/clc_rsqrt.h       |    4 +-
 libclc/clc/include/clc/math/clc_sin.h         |    4 +-
 libclc/clc/include/clc/math/clc_sincos.h      |    4 +-
 .../clc/include/clc/math/clc_sincos_helpers.h |    4 -
 libclc/clc/include/clc/math/clc_sinh.h        |    4 +-
 libclc/clc/include/clc/math/clc_sinpi.h       |    4 +-
 libclc/clc/include/clc/math/clc_sqrt.h        |    4 +-
 libclc/clc/include/clc/math/clc_tan.h         |    4 +-
 libclc/clc/include/clc/math/clc_tanh.h        |    4 +-
 libclc/clc/include/clc/math/clc_tanpi.h       |    4 +-
 libclc/clc/include/clc/math/clc_tgamma.h      |    4 +-
 libclc/clc/include/clc/math/clc_trunc.h       |    4 +-
 libclc/clc/include/clc/math/gentype.inc       |    4 +
 libclc/clc/include/clc/math/remquo_decl.inc   |   20 +-
 libclc/clc/include/clc/math/unary_decl.inc    |    2 +-
 .../clc/math/unary_decl_with_int_ptr.inc      |   16 +-
 .../clc/math/unary_decl_with_int_return.inc   |    2 +-
 .../include/clc/math/unary_decl_with_ptr.inc  |   16 +-
 .../include/clc/math/unary_def_via_fp32.inc   |    4 +-
 .../clc/math/unary_def_with_int_ptr.inc       |   12 +-
 .../clc/math/unary_def_with_int_return.inc    |    6 +-
 .../include/clc/math/unary_def_with_ptr.inc   |   12 +-
 libclc/clc/include/clc/misc/clc_shuffle.h     |    4 +-
 libclc/clc/include/clc/misc/clc_shuffle2.h    |    4 +-
 libclc/clc/include/clc/misc/shuffle2_decl.inc |   16 +-
 libclc/clc/include/clc/misc/shuffle2_def.inc  |   12 +-
 libclc/clc/include/clc/misc/shuffle_decl.inc  |    8 +-
 libclc/clc/include/clc/misc/shuffle_def.inc   |   12 +-
 .../include/clc/relational/binary_decl.inc    |    3 +-
 .../clc/include/clc/relational/clc_isfinite.h |    4 +-
 .../include/clc/relational/clc_isgreater.h    |    4 +-
 .../clc/relational/clc_isgreaterequal.h       |    4 +-
 .../clc/include/clc/relational/clc_isless.h   |    4 +-
 .../include/clc/relational/clc_islessequal.h  |    4 +-
 .../clc/relational/clc_islessgreater.h        |    4 +-
 .../clc/include/clc/relational/clc_isnormal.h |    4 +-
 .../include/clc/relational/clc_isnotequal.h   |    4 +-
 .../include/clc/relational/clc_isordered.h    |    4 +-
 .../include/clc/relational/clc_isunordered.h  |    4 +-
 .../clc/include/clc/relational/clc_signbit.h  |    4 +-
 .../clc/include/clc/relational/unary_decl.inc |    2 +-
 libclc/clc/include/clc/shared/binary_decl.inc |    4 +-
 .../binary_decl_with_int_second_arg.inc       |    3 +-
 .../binary_decl_with_scalar_second_arg.inc    |    8 +-
 libclc/clc/include/clc/shared/binary_def.inc  |    6 +-
 .../clc/shared/binary_def_scalarize.inc       |  131 +
 .../binary_def_with_int_second_arg.inc        |    6 +-
 .../binary_def_with_scalar_second_arg.inc     |    8 +-
 .../clc/include/clc/shared/ternary_decl.inc   |    5 +-
 libclc/clc/include/clc/shared/ternary_def.inc |    6 +-
 .../clc/shared/ternary_def_scalarize.inc      |  153 +
 libclc/clc/include/clc/shared/unary_decl.inc  |    2 +-
 libclc/clc/include/clc/shared/unary_def.inc   |    6 +-
 .../clc/shared/unary_def_scalarize.inc        |   97 +
 libclc/clc/lib/amdgcn/math/clc_fmax.cl        |    8 +-
 libclc/clc/lib/amdgcn/math/clc_fmin.cl        |    8 +-
 .../clc/lib/amdgcn/math/clc_ldexp_override.cl |   17 +-
 libclc/clc/lib/amdgpu/math/clc_native_exp2.cl |   11 +-
 libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl   |    5 +-
 libclc/clc/lib/clspv/math/clc_sw_fma.cl       |    6 +-
 libclc/clc/lib/generic/integer/clc_add_sat.cl |    2 +-
 libclc/clc/lib/generic/integer/clc_clz.cl     |   11 +-
 libclc/clc/lib/generic/integer/clc_ctz.cl     |   11 +-
 .../clc/lib/generic/integer/clc_popcount.cl   |    2 +-
 libclc/clc/lib/generic/integer/clc_sub_sat.cl |    2 +-
 libclc/clc/lib/generic/math/clc_ceil.cl       |    2 +-
 libclc/clc/lib/generic/math/clc_copysign.cl   |   28 +-
 libclc/clc/lib/generic/math/clc_erf.cl        |   14 +-
 libclc/clc/lib/generic/math/clc_erfc.cl       |   14 +-
 libclc/clc/lib/generic/math/clc_fabs.cl       |    2 +-
 libclc/clc/lib/generic/math/clc_floor.cl      |    2 +-
 libclc/clc/lib/generic/math/clc_fmax.cl       |   28 +-
 libclc/clc/lib/generic/math/clc_fmin.cl       |   27 +-
 libclc/clc/lib/generic/math/clc_fmod.cl       |   18 +-
 libclc/clc/lib/generic/math/clc_half_cos.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_half_exp.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_half_exp10.cl |    2 +-
 libclc/clc/lib/generic/math/clc_half_exp2.cl  |    2 +-
 libclc/clc/lib/generic/math/clc_half_log.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_half_log10.cl |    2 +-
 libclc/clc/lib/generic/math/clc_half_log2.cl  |    2 +-
 libclc/clc/lib/generic/math/clc_half_powr.cl  |    2 +-
 libclc/clc/lib/generic/math/clc_half_rsqrt.cl |    2 +-
 libclc/clc/lib/generic/math/clc_half_sin.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_half_sqrt.cl  |    2 +-
 libclc/clc/lib/generic/math/clc_half_tan.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_ldexp.cl      |   12 +-
 libclc/clc/lib/generic/math/clc_lgamma_r.cl   |    7 +
 libclc/clc/lib/generic/math/clc_log.cl        |   10 +-
 libclc/clc/lib/generic/math/clc_log10.cl      |   12 +-
 libclc/clc/lib/generic/math/clc_log2.cl       |   12 +-
 libclc/clc/lib/generic/math/clc_nan.inc       |    4 +
 libclc/clc/lib/generic/math/clc_native_cos.cl |    2 +-
 libclc/clc/lib/generic/math/clc_native_exp.cl |    2 +-
 .../clc/lib/generic/math/clc_native_exp2.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_native_log.cl |    2 +-
 .../clc/lib/generic/math/clc_native_log10.cl  |    2 +-
 .../clc/lib/generic/math/clc_native_log2.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_native_sin.cl |    2 +-
 .../clc/lib/generic/math/clc_native_sqrt.cl   |    2 +-
 libclc/clc/lib/generic/math/clc_pow.inc       |   13 +-
 libclc/clc/lib/generic/math/clc_pown.inc      |   13 +-
 libclc/clc/lib/generic/math/clc_powr.inc      |   13 +-
 libclc/clc/lib/generic/math/clc_remainder.cl  |   19 +-
 libclc/clc/lib/generic/math/clc_rint.cl       |    2 +-
 libclc/clc/lib/generic/math/clc_rootn.inc     |   13 +-
 libclc/clc/lib/generic/math/clc_round.cl      |    2 +-
 .../lib/generic/math/clc_sincos_helpers.cl    |    4 -
 libclc/clc/lib/generic/math/clc_sw_fma.cl     |    6 +-
 libclc/clc/lib/generic/math/clc_tgamma.cl     |   14 +-
 libclc/clc/lib/generic/math/clc_trunc.cl      |    2 +-
 libclc/clc/lib/r600/math/clc_fmax.cl          |   15 +-
 libclc/clc/lib/r600/math/clc_fmin.cl          |   15 +-
 libclc/clc/lib/r600/math/clc_native_rsqrt.cl  |    5 +-
 .../clc/lib/r600/math/clc_rsqrt_override.cl   |   12 +-
 libclc/cmake/modules/AddLibclc.cmake          |  117 +-
 .../include/clc/opencl/atomic/atom_add.h      |    6 +-
 .../include/clc/opencl/atomic/atom_and.h      |    6 +-
 .../clc/opencl/atomic/atom_decl_int32.inc     |    4 +-
 .../clc/opencl/atomic/atom_decl_int64.inc     |    4 +-
 .../include/clc/opencl/atomic/atom_max.h      |    6 +-
 .../include/clc/opencl/atomic/atom_min.h      |    6 +-
 .../include/clc/opencl/atomic/atom_or.h       |    6 +-
 .../include/clc/opencl/atomic/atom_sub.h      |    6 +-
 .../include/clc/opencl/atomic/atom_xchg.h     |    6 +-
 .../include/clc/opencl/atomic/atom_xor.h      |    6 +-
 .../include/clc/opencl/atomic/atomic_add.h    |    2 +-
 .../include/clc/opencl/atomic/atomic_and.h    |    2 +-
 .../include/clc/opencl/atomic/atomic_decl.inc |    4 +-
 .../include/clc/opencl/atomic/atomic_max.h    |    2 +-
 .../include/clc/opencl/atomic/atomic_min.h    |    2 +-
 .../include/clc/opencl/atomic/atomic_or.h     |    2 +-
 .../include/clc/opencl/atomic/atomic_sub.h    |    2 +-
 .../include/clc/opencl/atomic/atomic_xchg.h   |    6 +-
 .../include/clc/opencl/atomic/atomic_xor.h    |    2 +-
 libclc/opencl/include/clc/opencl/clc.h        |   14 +-
 .../include/clc/opencl/common/degrees.h       |    4 +-
 .../include/clc/opencl/common/radians.h       |    4 +-
 .../opencl/include/clc/opencl/common/sign.h   |    4 +-
 .../include/clc/opencl/geometric/distance.h   |    4 +-
 .../opencl/include/clc/opencl/geometric/dot.h |    4 +-
 .../clc/opencl/geometric/fast_distance.h      |    5 +-
 .../clc/opencl/geometric/fast_length.h        |    5 +-
 .../clc/opencl/geometric/fast_normalize.h     |    5 +-
 .../include/clc/opencl/geometric/length.h     |    4 +-
 .../include/clc/opencl/geometric/normalize.h  |    4 +-
 .../opencl/include/clc/opencl/integer/abs.h   |    7 +
 .../include/clc/opencl/integer/abs_diff.h     |    7 +
 .../include/clc/opencl/integer/add_sat.h      |   11 +-
 .../opencl/include/clc/opencl/integer/clz.h   |   11 +-
 .../opencl/include/clc/opencl/integer/ctz.h   |   11 +-
 .../opencl/include/clc/opencl/integer/hadd.h  |   11 +-
 .../opencl/include/clc/opencl/integer/mad24.h |   11 +-
 .../include/clc/opencl/integer/mad_hi.h       |   11 +-
 .../include/clc/opencl/integer/mad_sat.h      |   11 +-
 .../opencl/include/clc/opencl/integer/mul24.h |   11 +-
 .../include/clc/opencl/integer/mul_hi.h       |   11 +-
 .../include/clc/opencl/integer/popcount.h     |   11 +-
 .../opencl/include/clc/opencl/integer/rhadd.h |    9 +-
 .../include/clc/opencl/integer/rotate.h       |   11 +-
 .../include/clc/opencl/integer/sub_sat.h      |    9 +-
 .../include/clc/opencl/integer/upsample.h     |    7 +
 libclc/opencl/include/clc/opencl/math/acos.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/acosh.h |    4 +-
 .../opencl/include/clc/opencl/math/acospi.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/asin.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/asinh.h |    4 +-
 .../opencl/include/clc/opencl/math/asinpi.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/atan.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/atan2.h |    4 +-
 .../opencl/include/clc/opencl/math/atan2pi.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/atanh.h |    4 +-
 .../opencl/include/clc/opencl/math/atanpi.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/cbrt.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/ceil.h  |    4 +-
 .../opencl/include/clc/opencl/math/copysign.h |    4 +-
 libclc/opencl/include/clc/opencl/math/cos.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/cosh.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/cospi.h |    4 +-
 libclc/opencl/include/clc/opencl/math/erf.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/erfc.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/exp.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/exp10.h |    4 +-
 libclc/opencl/include/clc/opencl/math/exp2.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/expm1.h |    4 +-
 libclc/opencl/include/clc/opencl/math/fabs.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/fdim.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/floor.h |    4 +-
 libclc/opencl/include/clc/opencl/math/fma.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/fmax.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/fmin.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/fmod.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/fract.h |    4 +-
 libclc/opencl/include/clc/opencl/math/frexp.h |    4 +-
 .../opencl/include/clc/opencl/math/half_cos.h |    5 +-
 .../include/clc/opencl/math/half_divide.h     |    4 +-
 .../opencl/include/clc/opencl/math/half_exp.h |    5 +-
 .../include/clc/opencl/math/half_exp10.h      |    5 +-
 .../include/clc/opencl/math/half_exp2.h       |    5 +-
 .../opencl/include/clc/opencl/math/half_log.h |    5 +-
 .../include/clc/opencl/math/half_log10.h      |    5 +-
 .../include/clc/opencl/math/half_log2.h       |    5 +-
 .../include/clc/opencl/math/half_powr.h       |    4 +-
 .../include/clc/opencl/math/half_recip.h      |    5 +-
 .../include/clc/opencl/math/half_rsqrt.h      |    5 +-
 .../opencl/include/clc/opencl/math/half_sin.h |    5 +-
 .../include/clc/opencl/math/half_sqrt.h       |    5 +-
 .../opencl/include/clc/opencl/math/half_tan.h |    5 +-
 libclc/opencl/include/clc/opencl/math/hypot.h |    4 +-
 libclc/opencl/include/clc/opencl/math/ilogb.h |    4 +-
 libclc/opencl/include/clc/opencl/math/ldexp.h |    5 +
 .../opencl/include/clc/opencl/math/ldexp.inc  |    4 +-
 .../opencl/include/clc/opencl/math/lgamma.h   |    4 +-
 .../opencl/include/clc/opencl/math/lgamma_r.h |    4 +-
 libclc/opencl/include/clc/opencl/math/log.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/log10.h |    4 +-
 libclc/opencl/include/clc/opencl/math/log1p.h |    4 +-
 libclc/opencl/include/clc/opencl/math/log2.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/logb.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/mad.h   |    4 +-
 .../opencl/include/clc/opencl/math/maxmag.h   |    4 +-
 .../opencl/include/clc/opencl/math/minmag.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/modf.h  |    4 +-
 .../include/clc/opencl/math/native_cos.h      |    5 +-
 .../include/clc/opencl/math/native_divide.h   |    4 +-
 .../include/clc/opencl/math/native_exp.h      |    5 +-
 .../include/clc/opencl/math/native_exp10.h    |    5 +-
 .../include/clc/opencl/math/native_exp2.h     |    5 +-
 .../include/clc/opencl/math/native_log.h      |    5 +-
 .../include/clc/opencl/math/native_log10.h    |    5 +-
 .../include/clc/opencl/math/native_log2.h     |    5 +-
 .../include/clc/opencl/math/native_powr.h     |    4 +-
 .../include/clc/opencl/math/native_recip.h    |    5 +-
 .../include/clc/opencl/math/native_rsqrt.h    |    5 +-
 .../include/clc/opencl/math/native_sin.h      |    5 +-
 .../include/clc/opencl/math/native_sqrt.h     |    5 +-
 .../include/clc/opencl/math/native_tan.h      |    5 +-
 .../include/clc/opencl/math/nextafter.h       |    4 +-
 libclc/opencl/include/clc/opencl/math/pow.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/pown.h  |    6 +-
 libclc/opencl/include/clc/opencl/math/powr.h  |    4 +-
 .../include/clc/opencl/math/remainder.h       |    4 +-
 .../opencl/include/clc/opencl/math/remquo.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/rint.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/rootn.h |    6 +-
 libclc/opencl/include/clc/opencl/math/round.h |    4 +-
 libclc/opencl/include/clc/opencl/math/rsqrt.h |    4 +-
 libclc/opencl/include/clc/opencl/math/sin.h   |    4 +-
 .../opencl/include/clc/opencl/math/sincos.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/sinh.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/sinpi.h |    4 +-
 libclc/opencl/include/clc/opencl/math/sqrt.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/tan.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/tanh.h  |    4 +-
 libclc/opencl/include/clc/opencl/math/tanpi.h |    4 +-
 .../opencl/include/clc/opencl/math/tgamma.h   |    4 +-
 libclc/opencl/include/clc/opencl/math/trunc.h |    4 +-
 .../opencl/include/clc/opencl/misc/shuffle.h  |    4 +-
 .../opencl/include/clc/opencl/misc/shuffle2.h |    4 +-
 .../opencl/include/clc/opencl/opencl-base.h   |   26 +
 .../include/clc/opencl/relational/isfinite.h  |    4 +-
 .../include/clc/opencl/relational/isgreater.h |    4 +-
 .../clc/opencl/relational/isgreaterequal.h    |    4 +-
 .../include/clc/opencl/relational/isless.h    |    4 +-
 .../clc/opencl/relational/islessequal.h       |    4 +-
 .../clc/opencl/relational/islessgreater.h     |    4 +-
 .../include/clc/opencl/relational/isnormal.h  |    4 +-
 .../clc/opencl/relational/isnotequal.h        |    4 +-
 .../include/clc/opencl/relational/isordered.h |    4 +-
 .../clc/opencl/relational/isunordered.h       |    4 +-
 .../include/clc/opencl/relational/signbit.h   |    4 +-
 libclc/opencl/lib/clspv/math/fma.cl           |    8 +-
 libclc/opencl/lib/generic/common/degrees.cl   |   18 +-
 libclc/opencl/lib/generic/common/radians.cl   |   18 +-
 libclc/opencl/lib/generic/integer/abs.cl      |    2 +-
 libclc/opencl/lib/generic/integer/abs_diff.cl |    2 +-
 libclc/opencl/lib/generic/integer/add_sat.cl  |    2 +-
 libclc/opencl/lib/generic/integer/clz.cl      |    2 +-
 libclc/opencl/lib/generic/integer/ctz.cl      |    2 +-
 libclc/opencl/lib/generic/integer/hadd.cl     |    2 +-
 libclc/opencl/lib/generic/integer/mad24.cl    |    2 +-
 libclc/opencl/lib/generic/integer/mad_hi.cl   |    2 +-
 libclc/opencl/lib/generic/integer/mad_sat.cl  |    2 +-
 libclc/opencl/lib/generic/integer/mul24.cl    |    2 +-
 libclc/opencl/lib/generic/integer/mul_hi.cl   |    2 +-
 libclc/opencl/lib/generic/integer/popcount.cl |    2 +-
 libclc/opencl/lib/generic/integer/rhadd.cl    |    2 +-
 libclc/opencl/lib/generic/integer/rotate.cl   |    2 +-
 libclc/opencl/lib/generic/integer/sub_sat.cl  |    2 +-
 libclc/opencl/lib/generic/integer/upsample.cl |    2 +-
 libclc/opencl/lib/generic/math/fma.cl         |   18 +-
 libclc/opencl/lib/generic/math/ldexp.cl       |   21 +-
 libclc/opencl/lib/generic/math/mad.cl         |   18 +-
 libclc/opencl/lib/generic/math/nextafter.cl   |   24 +-
 libclc/opencl/lib/generic/math/pown.cl        |    2 +-
 libclc/opencl/lib/generic/math/rootn.cl       |    2 +-
 .../lib/generic/relational/binary_def.inc     |    4 +-
 .../lib/generic/relational/unary_def.inc      |    4 +-
 libclc/opencl/lib/spirv/math/fma.cl           |    8 +-
 libcxx/.clang-format                          |    9 +-
 libcxx/docs/FeatureTestMacroTable.rst         |    4 +
 libcxx/docs/ReleaseNotes/21.rst               |    6 +-
 libcxx/docs/Status/Cxx23Papers.csv            |    2 +-
 libcxx/docs/Status/Cxx2cPapers.csv            |    2 +-
 libcxx/docs/UserDocumentation.rst             |    5 -
 libcxx/include/CMakeLists.txt                 |   13 +-
 libcxx/include/__chrono/formatter.h           |    4 +-
 libcxx/include/__concepts/common_with.h       |    2 +-
 libcxx/include/__config                       |   71 +-
 libcxx/include/__coroutine/coroutine_handle.h |    2 +-
 libcxx/include/__coroutine/coroutine_traits.h |    2 +-
 .../__coroutine/noop_coroutine_handle.h       |    2 +-
 .../include/__coroutine/trivial_awaitables.h  |    2 +-
 libcxx/include/__filesystem/path.h            |    1 -
 libcxx/include/__filesystem/u8path.h          |    7 +-
 .../include/__flat_map/key_value_iterator.h   |   28 +
 libcxx/include/__flat_map/utils.h             |   22 +-
 libcxx/include/__flat_set/flat_set.h          |  278 +-
 libcxx/include/__flat_set/utils.h             |    8 +-
 .../__functional/boyer_moore_searcher.h       |   12 +-
 .../include/__functional/reference_wrapper.h  |   32 +-
 libcxx/include/__hash_table                   |  114 +-
 libcxx/include/__iterator/product_iterator.h  |   76 +
 libcxx/include/__locale_dir/check_grouping.h  |   31 +
 libcxx/include/__locale_dir/get_c_locale.h    |   40 +
 libcxx/include/__locale_dir/messages.h        |  143 +
 libcxx/include/__locale_dir/money.h           |  873 +++
 libcxx/include/__locale_dir/num.h             | 1072 +++
 libcxx/include/__locale_dir/scan_keyword.h    |  143 +
 libcxx/include/__locale_dir/time.h            |  766 ++
 libcxx/include/__locale_dir/wbuffer_convert.h |  430 ++
 libcxx/include/__locale_dir/wstring_convert.h |  254 +
 libcxx/include/__memory/shared_ptr.h          |    2 +-
 libcxx/include/__memory/unique_ptr.h          |    2 +-
 libcxx/include/__mutex/lock_guard.h           |   10 +-
 libcxx/include/__mutex/mutex.h                |    8 +-
 libcxx/include/__ostream/basic_ostream.h      |    4 +-
 libcxx/include/__ostream/print.h              |    2 +-
 libcxx/include/__ranges/zip_view.h            |   22 +
 libcxx/include/__thread/thread.h              |    2 +-
 libcxx/include/__tree                         |  208 +-
 .../__type_traits/add_lvalue_reference.h      |   54 -
 libcxx/include/__type_traits/add_pointer.h    |   16 +-
 ...add_rvalue_reference.h => add_reference.h} |   44 +-
 .../include/__type_traits/common_reference.h  |   11 +-
 libcxx/include/__type_traits/copy_cvref.h     |    3 +-
 libcxx/include/__type_traits/decay.h          |   44 +-
 .../has_unique_object_representation.h        |    8 +-
 libcxx/include/__type_traits/invoke.h         |  167 +-
 libcxx/include/__type_traits/is_array.h       |   24 +-
 libcxx/include/__type_traits/is_assignable.h  |    3 +-
 .../include/__type_traits/is_bounded_array.h  |   21 +-
 libcxx/include/__type_traits/is_const.h       |   20 +-
 .../include/__type_traits/is_constructible.h  |    3 +-
 .../__type_traits/is_core_convertible.h       |    9 +
 .../__type_traits/is_nothrow_assignable.h     |    3 +-
 .../__type_traits/is_nothrow_constructible.h  |    3 +-
 libcxx/include/__type_traits/is_pointer.h     |   33 -
 libcxx/include/__type_traits/is_swappable.h   |    2 +-
 .../__type_traits/is_trivially_assignable.h   |    3 +-
 .../is_trivially_constructible.h              |    3 +-
 libcxx/include/__type_traits/is_volatile.h    |   20 +-
 .../__type_traits/remove_all_extents.h        |   22 +-
 libcxx/include/__type_traits/remove_cvref.h   |    4 -
 libcxx/include/__type_traits/remove_extent.h  |   22 +-
 libcxx/include/bitset                         |  184 +-
 libcxx/include/complex                        |    1 +
 libcxx/include/experimental/iterator          |    5 +
 libcxx/include/experimental/memory            |    2 +-
 libcxx/include/flat_map                       |    9 +
 libcxx/include/flat_set                       |    9 +
 libcxx/include/fstream                        |   77 +-
 libcxx/include/functional                     |    5 +
 libcxx/include/future                         |   13 +-
 libcxx/include/iomanip                        |    9 +-
 libcxx/include/istream                        |    7 +-
 libcxx/include/locale                         | 3488 +--------
 libcxx/include/map                            |   45 +-
 libcxx/include/module.modulemap.in            |   38 +-
 libcxx/include/mutex                          |   11 +-
 libcxx/include/ostream                        |    5 +
 libcxx/include/set                            |   16 +-
 libcxx/include/shared_mutex                   |   56 +-
 libcxx/include/sstream                        |    2 +-
 libcxx/include/string                         |   22 +-
 libcxx/include/tuple                          |    6 +
 libcxx/include/type_traits                    |    3 +-
 libcxx/include/unordered_map                  |  153 +-
 libcxx/include/variant                        |   31 +
 libcxx/include/version                        |    4 +
 libcxx/modules/std/functional.inc             |    4 +
 libcxx/src/atomic.cpp                         |    7 +-
 .../associative_container_benchmarks.h        |   56 +-
 .../containers/associative/flat_map.bench.cpp |    2 +-
 .../associative/flat_multimap.bench.cpp       |    2 +-
 .../associative/unordered_set.bench.cpp       |    2 +
 libcxx/test/configs/cmake-bridge.cfg.in       |    3 +
 .../gnu/hash/specializations.pass.cpp         |   31 +
 .../gnu/hash/specializations.verify.cpp}      |    7 +-
 .../gnu/hash_map/const_iterator.verify.cpp}   |   13 +-
 libcxx/test/extensions/gnu/lit.local.cfg      |    5 +
 .../flat.set/insert.temporary.pass.cpp        |    5 +-
 .../flat.set/insert_range.pass.cpp            |   18 +-
 .../extensions/hash/specializations.pass.cpp  |   36 -
 .../ios.base.cons/dtor.uninitialized.pass.cpp |    2 +-
 .../iterators/product_iterator.pass.cpp       |   66 +
 .../locale.categories/__scan_keyword.pass.cpp |    5 +-
 ...ad_safety_annotations_not_enabled.pass.cpp |    5 +-
 .../thread_safety_lock_guard.pass.cpp         |   12 +-
 .../thread_safety_lock_unlock.pass.cpp        |   12 +-
 .../thread_safety_missing_unlock.verify.cpp   |   16 +-
 ...thread_safety_requires_capability.pass.cpp |   12 +-
 .../thread_safety.verify.cpp                  |   13 +-
 .../thread_safety.verify.cpp                  |   17 +-
 .../test/libcxx/transitive_includes/cxx26.csv |   18 -
 .../func.require/bullet_1_2_3.pass.cpp        |    5 +-
 .../conditional-compile-flags.sh.cpp          |    0
 .../additional_compile_flags/lit.local.cfg    |    0
 .../substitutes-in-compile-flags.sh.cpp       |    0
 .../substitutes-in-run.sh.cpp                 |    0
 .../compile-error.compile.fail.cpp            |    0
 .../compile-success.compile.fail.cpp          |    0
 .../compile-error.compile.pass.cpp            |    0
 .../compile-success.compile.pass.cpp          |    0
 .../link-error.compile.pass.cpp               |    0
 .../run-error.compile.pass.cpp                |    0
 .../compile-error.compile.pass.mm             |    0
 .../compile-success.compile.pass.mm           |    0
 .../link-error.compile.pass.mm                |    0
 .../compile.pass.mm/run-error.compile.pass.mm |    0
 .../build_run.sh.cpp                          |    0
 .../convenience_substitutions/verify.sh.cpp   |    0
 .../test/{libcxx => }/selftest/dsl/dsl.sh.py  |    2 +-
 .../{libcxx => }/selftest/dsl/lit.local.cfg   |    0
 libcxx/test/selftest/file_dependencies/a.txt  |    0
 .../absolute-and-relative-paths.sh.cpp        |    0
 .../test/selftest/file_dependencies/dir/b.txt |    0
 .../substitute-in-dependencies.sh.cpp         |    0
 .../selftest/gen.cpp/empty.gen.cpp            |    0
 .../{libcxx => }/selftest/gen.cpp/one.gen.cpp |    0
 .../{libcxx => }/selftest/gen.cpp/two.gen.cpp |    0
 .../link.fail.cpp/compile-error.link.fail.cpp |    0
 .../link.fail.cpp/link-error.link.fail.cpp    |    0
 .../link.fail.cpp/link-success.link.fail.cpp  |    0
 .../link.pass.cpp/compile-error.link.pass.cpp |    0
 .../link.pass.cpp/link-error.link.pass.cpp    |    0
 .../link.pass.cpp/link-success.link.pass.cpp  |    0
 .../link.pass.cpp/run-error.link.pass.cpp     |    0
 .../link.pass.mm/compile-error.link.pass.mm   |    0
 .../link.pass.mm/link-error.link.pass.mm      |    0
 .../link.pass.mm/link-success.link.pass.mm    |    0
 .../link.pass.mm/run-error.link.pass.mm       |    0
 .../test/{libcxx => selftest}/lit.local.cfg   |    0
 .../selftest/modules/no-modules.sh.cpp        |    4 +
 .../modules/std-and-std.compat-module.sh.cpp  |    4 +
 .../selftest/modules/std-module.sh.cpp        |    4 +
 .../selftest/modules/std.compat-module.sh.cpp |    4 +
 .../selftest/pass.cpp/compile-error.pass.cpp  |    0
 .../selftest/pass.cpp/link-error.pass.cpp     |    0
 .../selftest/pass.cpp/run-error.pass.cpp      |    0
 .../selftest/pass.cpp/run-success.pass.cpp    |    0
 .../selftest/pass.cpp/werror.pass.cpp         |    0
 .../selftest/pass.mm/compile-error.pass.mm    |    0
 .../selftest/pass.mm/link-error.pass.mm       |    0
 .../selftest/pass.mm/no-arc.pass.mm           |    0
 .../selftest/pass.mm/run-error.pass.mm        |    0
 .../selftest/pass.mm/run-success.pass.mm      |    0
 .../pass.mm/use-objective-cxx.pass.mm         |    0
 .../selftest/remote-substitutions.sh.cpp      |    0
 .../selftest/sh.cpp/run-error.sh.cpp          |    0
 .../selftest/sh.cpp/run-success.sh.cpp        |    0
 .../selftest/sh.cpp/substitutions.sh.cpp      |    0
 .../selftest/sh.cpp/werror.sh.cpp             |    0
 .../selftest/shell-no-escape-builtins.sh.cpp  |    0
 .../selftest/stdin-is-piped.sh.cpp            |    0
 .../selftest/test_macros.pass.cpp             |    0
 .../selftest/tmpdir-exists.sh.cpp             |    0
 .../no-diagnostics-unmarked.verify.cpp        |    0
 .../verify.cpp/no-diagnostics.verify.cpp      |    0
 .../selftest/verify.cpp/no-werror.verify.cpp  |    0
 .../verify.cpp/right-diagnostic.verify.cpp    |    0
 .../verify.cpp/wrong-diagnostic.verify.cpp    |    0
 .../insert_iter_iter.pass.cpp                 |   23 +-
 .../flat.set/flat.set.capacity/empty.pass.cpp |   14 +-
 .../flat.set.capacity/max_size.pass.cpp       |    7 +-
 .../flat.set/flat.set.capacity/size.pass.cpp  |   17 +-
 .../flat.set/flat.set.cons/alloc.pass.cpp     |   26 +-
 .../assign_initializer_list.pass.cpp          |   14 +-
 .../flat.set/flat.set.cons/compare.pass.cpp   |   80 +-
 .../flat.set.cons/containers.pass.cpp         |  106 +-
 .../flat.set/flat.set.cons/copy.pass.cpp      |   21 +-
 .../flat.set.cons/copy_alloc.pass.cpp         |   47 +-
 .../flat.set.cons/copy_assign.pass.cpp        |   27 +-
 .../flat.set/flat.set.cons/default.pass.cpp   |   42 +-
 .../flat.set.cons/dtor_noexcept.pass.cpp      |   28 +-
 .../flat.set.cons/initializer_list.pass.cpp   |  121 +-
 .../flat.set/flat.set.cons/iter_iter.pass.cpp |   98 +-
 .../flat.set/flat.set.cons/move.pass.cpp      |   64 +-
 .../flat.set.cons/move_alloc.pass.cpp         |   51 +-
 .../flat.set.cons/move_assign.pass.cpp        |   51 +-
 .../flat.set/flat.set.cons/range.pass.cpp     |   93 +-
 .../flat.set.cons/sorted_container.pass.cpp   |   91 +-
 .../sorted_initializer_list.pass.cpp          |  122 +-
 .../flat.set.cons/sorted_iter_iter.pass.cpp   |   94 +-
 .../flat.set.erasure/erase_if.pass.cpp        |   26 +-
 .../flat.set.iterators/iterator.pass.cpp      |   14 +-
 .../iterator_comparison.pass.cpp              |   14 +-
 .../reverse_iterator.pass.cpp                 |   52 +-
 .../flat.set.modifiers/clear.pass.cpp         |   15 +-
 .../flat.set.modifiers/emplace.pass.cpp       |   21 +-
 .../flat.set.modifiers/emplace_hint.pass.cpp  |   21 +-
 .../flat.set.modifiers/erase_iter.pass.cpp    |   14 +-
 .../erase_iter_iter.pass.cpp                  |   14 +-
 .../flat.set.modifiers/erase_key.pass.cpp     |   14 +-
 .../erase_key_transparent.pass.cpp            |   29 +-
 .../flat.set.modifiers/extract.pass.cpp       |   14 +-
 .../flat.set.modifiers/insert_cv.pass.cpp     |   14 +-
 .../insert_initializer_list.pass.cpp          |   14 +-
 .../insert_iter_cv.pass.cpp                   |   14 +-
 .../insert_iter_iter.pass.cpp                 |   14 +-
 .../insert_iter_rv.pass.cpp                   |   18 +-
 .../flat.set.modifiers/insert_range.pass.cpp  |   16 +-
 .../flat.set.modifiers/insert_rv.pass.cpp     |   18 +-
 .../insert_sorted_initializer_list.pass.cpp   |   14 +-
 .../insert_sorted_iter_iter.pass.cpp          |   14 +-
 .../insert_transparent.pass.cpp               |   14 +-
 .../flat.set.modifiers/replace.pass.cpp       |   14 +-
 .../flat.set.modifiers/swap_free.pass.cpp     |   15 +-
 .../flat.set.modifiers/swap_member.pass.cpp   |   14 +-
 .../flat.set/flat.set.observers/comp.pass.cpp |    9 +-
 .../flat.set.operations/contains.pass.cpp     |   14 +-
 .../contains_transparent.pass.cpp             |   14 +-
 .../flat.set.operations/count.pass.cpp        |   14 +-
 .../count_transparent.pass.cpp                |   14 +-
 .../flat.set.operations/equal_range.pass.cpp  |   14 +-
 .../equal_range_transparent.pass.cpp          |   14 +-
 .../flat.set.operations/find.pass.cpp         |   14 +-
 .../find_transparent.pass.cpp                 |   14 +-
 .../flat.set.operations/lower_bound.pass.cpp  |   14 +-
 .../lower_bound_transparent.pass.cpp          |   14 +-
 .../flat.set.operations/upper_bound.pass.cpp  |   14 +-
 .../upper_bound_transparent.pass.cpp          |   14 +-
 .../container.adaptors/flat.set/helpers.h     |    2 +-
 .../flat.set/incomplete_type.pass.cpp         |   11 +-
 .../flat.set/op_compare.pass.cpp              |   16 +-
 .../container.adaptors/flat_helpers.h         |   15 +-
 .../unord.set.cnstr/assign_copy.pass.cpp      |   14 +
 .../fstreams/filebuf.virtuals/setbuf.pass.cpp |    2 +-
 .../ext.manip/get_money.pass.cpp              |    3 +-
 .../ext.manip/get_time.pass.cpp               |    4 +-
 .../ext.manip/put_time.pass.cpp               |    3 +-
 .../istream.unformatted/sync.pass.cpp         |    2 +-
 .../mandatory_inclusions.gen.py               |  180 +
 .../functional.version.compile.pass.cpp       |   33 +
 .../type_traits.version.compile.pass.cpp      |   33 +
 .../version.version.compile.pass.cpp          |   66 +
 .../get_long_double_fr_FR.pass.cpp            |    2 +-
 .../put_long_double_fr_FR.pass.cpp            |    2 +-
 .../grouping.pass.cpp                         |    2 +-
 .../facet.num.get.members/get_double.pass.cpp |    2 +-
 .../facet.num.get.members/get_float.pass.cpp  |    2 +-
 .../get_long_double.pass.cpp                  |    2 +-
 .../locale.time.get.byname/get_date.pass.cpp  |    4 +-
 .../get_monthname.pass.cpp                    |    4 +-
 .../locale.time.get.byname/get_one.pass.cpp   |    4 +-
 .../locale.time.get.byname/get_time.pass.cpp  |    4 +-
 .../get_weekday.pass.cpp                      |    4 +-
 .../locale.time.get.byname/get_year.pass.cpp  |    4 +-
 .../locale.time.get.members/get_date.pass.cpp |    4 +-
 .../locale.time.get.members/get_many.pass.cpp |    4 +-
 .../get_monthname.pass.cpp                    |    4 +-
 .../locale.time.get.members/get_one.pass.cpp  |    4 +-
 .../locale.time.get.members/get_time.pass.cpp |    4 +-
 .../get_weekday.pass.cpp                      |    4 +-
 .../locale.time.get.members/get_year.pass.cpp |    4 +-
 .../tuple_element.compile.pass.cpp            |    2 +-
 .../complex.tuple/tuple_size.compile.pass.cpp |    2 +-
 .../thread_create_failure.pass.cpp            |   61 +
 .../futures.async/wait_on_destruct.pass.cpp   |   42 +
 .../equality/equality.T2.pass.cpp             |    8 +-
 .../equality/equality.other_expected.pass.cpp |   22 +-
 .../equality/equality.unexpected.pass.cpp     |    8 +-
 .../equality/equality.other_expected.pass.cpp |   22 +-
 .../equality/equality.unexpected.pass.cpp     |    8 +-
 libcxx/test/std/utilities/expected/types.h    |   13 -
 .../refwrap/common_reference.compile.pass.cpp |  152 +
 .../common_reference.compile.pass.cpp         |   58 +-
 .../bitset.members/to_ullong.pass.cpp         |   68 +-
 .../bitset.members/to_ulong.pass.cpp          |   66 +-
 .../tuple/tuple.tuple/tuple.rel/eq.pass.cpp   |   25 +-
 .../size_incompatible_comparison.verify.cpp   |    3 +
 .../pairs/pairs.spec/comparison.pass.cpp      |   15 +-
 .../variant/variant.relops/relops.pass.cpp    |   49 +
 .../relops_bool_conv.verify.cpp               |   11 +-
 libcxx/test/support/concat_macros.h           |    1 +
 libcxx/test/support/test_comparisons.h        |   41 +
 .../header_exportable_declarations.cpp        |    2 +
 libcxx/utils/ci/buildkite-pipeline-trigger.sh |   34 +
 libcxx/utils/gdb/libcxx/printers.py           |   14 +-
 .../generate_feature_test_macro_components.py |   10 +
 libcxx/utils/libcxx/test/features.py          |   21 +-
 libcxx/utils/libcxx/test/params.py            |    1 -
 libcxxabi/CMakeLists.txt                      |    4 +
 libcxxabi/src/demangle/ItaniumDemangle.h      |    4 +-
 libcxxabi/test/DemangleTestCases.inc          |    1 +
 libcxxabi/test/configs/cmake-bridge.cfg.in    |    2 +-
 lld/COFF/Config.h                             |    1 +
 lld/COFF/Driver.cpp                           |   16 +-
 lld/COFF/InputFiles.cpp                       |   40 +-
 lld/ELF/Arch/AArch64.cpp                      |   58 +
 lld/ELF/Arch/LoongArch.cpp                    |  151 +-
 lld/ELF/Arch/TargetImpl.h                     |   93 +
 lld/ELF/Arch/X86_64.cpp                       |   70 +
 lld/ELF/Config.h                              |    5 +
 lld/ELF/Driver.cpp                            |    7 +
 lld/ELF/InputSection.cpp                      |    6 +-
 lld/ELF/LTO.cpp                               |    7 +
 lld/ELF/Options.td                            |   16 +-
 lld/ELF/Relocations.cpp                       |   46 +-
 lld/ELF/Relocations.h                         |    1 +
 lld/ELF/ScriptParser.cpp                      |    2 +
 lld/ELF/Target.h                              |    1 +
 lld/docs/DTLTO.rst                            |   42 +
 lld/docs/ReleaseNotes.rst                     |    4 +
 lld/docs/index.rst                            |    1 +
 lld/docs/ld.lld.1                             |    9 +-
 .../lld/Common/BPSectionOrdererBase.inc       |    4 +-
 lld/test/COFF/exported-dllmain.test           |   57 +
 lld/test/COFF/winsysroot.test                 |    4 +
 lld/test/ELF/aarch64-branch-to-branch.s       |   84 +
 lld/test/ELF/dtlto/files.test                 |   99 +
 lld/test/ELF/dtlto/options.test               |   40 +
 lld/test/ELF/dtlto/partitions.test            |   40 +
 lld/test/ELF/linkerscript/align-section.test  |   21 +-
 lld/test/ELF/loongarch-relax-tlsdesc.s        |  256 +-
 lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s   |   59 +-
 lld/test/ELF/loongarch-tlsdesc.s              |   65 +-
 lld/test/ELF/lto/lazy-debug.ll                |  107 +
 lld/test/ELF/x86-64-branch-to-branch.s        |  135 +
 lld/test/MachO/icf-safe-thunks-dwarf.ll       |    4 +-
 lld/test/MachO/icf-safe-thunks.ll             |    4 +-
 lld/test/lit.cfg.py                           |    1 +
 lld/test/wasm/compress-relocs.ll              |   26 -
 lld/test/wasm/compress-relocs.s               |  129 +
 lld/test/wasm/compress-relocs64.s             |   92 +
 lld/wasm/InputChunks.cpp                      |   46 +-
 .../bindings/interface/SBDebuggerExtensions.i |   13 +
 lldb/cmake/modules/LLDBFramework.cmake        |    3 +-
 lldb/docs/index.rst                           |    1 +
 lldb/docs/resources/contributing.rst          |    2 +-
 lldb/docs/resources/lldbgdbremote.md          |   17 +
 lldb/docs/use/mcp.md                          |   85 +
 lldb/include/lldb/API/SBBreakpoint.h          |    5 +
 lldb/include/lldb/API/SBStatisticsOptions.h   |    3 +-
 lldb/include/lldb/Breakpoint/Breakpoint.h     |    2 +
 .../lldb/Breakpoint/BreakpointLocation.h      |    9 +-
 .../lldb/Breakpoint/BreakpointResolver.h      |    3 +-
 lldb/include/lldb/Core/Debugger.h             |    8 +-
 lldb/include/lldb/Core/EmulateInstruction.h   |   61 +
 lldb/include/lldb/Core/IOHandler.h            |    4 +
 lldb/include/lldb/Core/ProtocolServer.h       |    5 +-
 lldb/include/lldb/Core/Statusline.h           |    4 +-
 .../lldb/DataFormatters/FormattersHelpers.h   |    8 +
 .../include/lldb/Expression/DWARFExpression.h |    4 +-
 lldb/include/lldb/Expression/IRMemoryMap.h    |   10 +-
 lldb/include/lldb/Host/Editline.h             |    2 +
 lldb/include/lldb/Host/PipeBase.h             |   10 +-
 lldb/include/lldb/Host/posix/HostInfoPosix.h  |    2 +
 lldb/include/lldb/Host/posix/PipePosix.h      |   10 +-
 .../lldb/Host/windows/MainLoopWindows.h       |   19 +-
 lldb/include/lldb/Host/windows/PipeWindows.h  |   13 +-
 lldb/include/lldb/Symbol/ObjectFile.h         |   17 +-
 lldb/include/lldb/Symbol/SymbolFile.h         |    8 +
 lldb/include/lldb/Target/MemoryTagManager.h   |   12 +-
 lldb/include/lldb/Target/Statistics.h         |    8 +-
 lldb/include/lldb/Utility/Broadcaster.h       |    2 +-
 lldb/include/lldb/Utility/Event.h             |    1 -
 lldb/include/lldb/Utility/IOObject.h          |    7 +-
 lldb/include/lldb/ValueObject/DILEval.h       |    6 +-
 lldb/include/lldb/lldb-forward.h              |    2 +-
 lldb/include/lldb/lldb-private-interfaces.h   |    3 +-
 lldb/include/lldb/lldb-types.h                |    2 +-
 .../Python/lldbsuite/test/builders/builder.py |   26 +-
 .../Python/lldbsuite/test/gdbclientutils.py   |   10 +
 .../Python/lldbsuite/test/lldbutil.py         |   87 +
 .../Python/lldbsuite/test/make/Makefile.rules |    4 +
 .../compressed_pair.h                         |   26 +-
 .../test/tools/lldb-dap/lldbdap_testcase.py   |    7 +-
 lldb/scripts/framework-header-fix.py          |    8 +-
 lldb/scripts/version-header-fix.py            |   29 +-
 lldb/source/API/CMakeLists.txt                |    2 +-
 lldb/source/API/SBBreakpoint.cpp              |   12 +
 lldb/source/Breakpoint/Breakpoint.cpp         |  112 +-
 lldb/source/Breakpoint/BreakpointIDList.cpp   |    4 +-
 lldb/source/Breakpoint/BreakpointLocation.cpp |   49 +-
 .../Breakpoint/BreakpointLocationList.cpp     |    5 +-
 lldb/source/Breakpoint/BreakpointOptions.cpp  |    6 +-
 .../Breakpoint/BreakpointResolverName.cpp     |   17 +-
 lldb/source/Breakpoint/Watchpoint.cpp         |    6 +-
 lldb/source/Breakpoint/WatchpointOptions.cpp  |    4 +-
 .../Commands/CommandObjectBreakpoint.cpp      |    5 +-
 .../Commands/CommandObjectProtocolServer.cpp  |   63 +-
 lldb/source/Commands/CommandObjectStats.cpp   |   14 +-
 lldb/source/Commands/CommandObjectTarget.cpp  |    5 +-
 lldb/source/Commands/Options.td               |   16 +-
 lldb/source/Core/Debugger.cpp                 |   30 +-
 lldb/source/Core/EmulateInstruction.cpp       |   92 +
 lldb/source/Core/FormatEntity.cpp             |    4 +-
 lldb/source/Core/IOHandler.cpp                |    7 +
 lldb/source/Core/ProtocolServer.cpp           |   34 +-
 lldb/source/Core/Statusline.cpp               |   42 +-
 lldb/source/Core/Telemetry.cpp                |   32 +-
 .../DataFormatters/FormattersHelpers.cpp      |   25 +
 lldb/source/Expression/DWARFExpression.cpp    |  139 +-
 lldb/source/Expression/IRExecutionUnit.cpp    |   19 +-
 lldb/source/Expression/IRMemoryMap.cpp        |   50 +-
 lldb/source/Expression/LLVMUserExpression.cpp |   40 +-
 lldb/source/Expression/Materializer.cpp       |  106 +-
 lldb/source/Host/common/Editline.cpp          |   37 +-
 lldb/source/Host/common/File.cpp              |    4 +
 lldb/source/Host/common/JSONTransport.cpp     |    3 +-
 lldb/source/Host/common/Socket.cpp            |    5 +-
 lldb/source/Host/macosx/objcxx/Host.mm        |   37 +-
 .../posix/ConnectionFileDescriptorPosix.cpp   |   31 +-
 lldb/source/Host/posix/HostInfoPosix.cpp      |    6 +
 lldb/source/Host/posix/MainLoopPosix.cpp      |    2 +-
 lldb/source/Host/posix/PipePosix.cpp          |   31 +-
 .../Host/posix/ProcessLauncherPosixFork.cpp   |    3 +-
 lldb/source/Host/windows/MainLoopWindows.cpp  |  193 +-
 lldb/source/Host/windows/PipeWindows.cpp      |   23 +-
 lldb/source/Interpreter/ScriptInterpreter.cpp |    2 +-
 .../Instruction/ARM/EmulateInstructionARM.cpp |   13 +
 .../Instruction/ARM/EmulateInstructionARM.h   |   18 +
 .../LoongArch/EmulateInstructionLoongArch.cpp |  122 +-
 .../LoongArch/EmulateInstructionLoongArch.h   |    2 -
 .../RISCV/EmulateInstructionRISCV.cpp         |  143 +-
 .../RISCV/EmulateInstructionRISCV.h           |   37 +-
 .../Language/CPlusPlus/CPlusPlusLanguage.cpp  |  155 +-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     |  122 +-
 .../Plugins/Language/CPlusPlus/LibCxx.h       |    4 +-
 .../Language/CPlusPlus/LibCxxUnorderedMap.cpp |    6 +-
 .../Plugins/Language/CPlusPlus/LibStdcpp.cpp  |  190 +-
 .../Plugins/Language/CPlusPlus/LibStdcpp.h    |    6 +-
 .../CPlusPlus/LibStdcppUniquePointer.cpp      |   26 +-
 .../AppleObjCClassDescriptorV2.cpp            |    5 +-
 .../AppleObjCRuntime/AppleObjCRuntimeV2.cpp   |   21 +-
 .../AppleObjCRuntime/AppleObjCRuntimeV2.h     |    5 +
 .../ObjectFile/Mach-O/ObjectFileMachO.cpp     |  324 +-
 .../ObjectFile/Mach-O/ObjectFileMachO.h       |    2 +
 .../Platform/MacOSX/PlatformDarwin.cpp        |   50 +-
 .../Process/FreeBSD/NativeProcessFreeBSD.cpp  |   12 +-
 .../Process/Linux/NativeProcessLinux.cpp      |   12 +-
 .../Process/MacOSX-Kernel/CMakeLists.txt      |    1 -
 .../MacOSX-Kernel/RegisterContextKDP_i386.cpp |  114 -
 .../MacOSX-Kernel/RegisterContextKDP_i386.h   |   38 -
 .../Process/MacOSX-Kernel/ThreadKDP.cpp       |    5 -
 .../Plugins/Process/Utility/CMakeLists.txt    |    2 -
 .../Plugins/Process/Utility/LinuxSignals.cpp  |    4 +
 .../Utility/MemoryTagManagerAArch64MTE.cpp    |   10 +-
 .../Utility/MemoryTagManagerAArch64MTE.h      |    2 +-
 .../NativeProcessSoftwareSingleStep.cpp       |  108 +-
 .../Utility/NativeProcessSoftwareSingleStep.h |    3 +-
 .../Utility/RegisterContextDarwin_i386.cpp    |  958 ---
 .../Utility/RegisterContextDarwin_i386.h      |  208 -
 .../Utility/RegisterContextMach_i386.cpp      |   60 -
 .../Utility/RegisterContextMach_i386.h        |   35 -
 .../gdb-remote/GDBRemoteClientBase.cpp        |    9 +-
 .../Process/gdb-remote/GDBRemoteClientBase.h  |    6 +-
 .../gdb-remote/GDBRemoteCommunication.cpp     |  249 +-
 .../gdb-remote/GDBRemoteCommunication.h       |   14 +-
 .../GDBRemoteCommunicationClient.cpp          |    6 +-
 .../GDBRemoteCommunicationServerPlatform.cpp  |   44 +-
 .../GDBRemoteCommunicationServerPlatform.h    |    4 +-
 .../Process/gdb-remote/ProcessGDBRemote.cpp   |   66 +-
 .../Plugins/Process/mach-core/CMakeLists.txt  |    1 +
 .../mach-core/RegisterContextUnifiedCore.cpp  |  312 +
 .../mach-core/RegisterContextUnifiedCore.h    |   57 +
 .../Process/mach-core/ThreadMachCore.cpp      |   56 +-
 lldb/source/Plugins/Protocol/MCP/Protocol.h   |    2 +
 .../Protocol/MCP/ProtocolServerMCP.cpp        |   30 +-
 .../Plugins/Protocol/MCP/ProtocolServerMCP.h  |    6 +-
 lldb/source/Plugins/Protocol/MCP/Tool.cpp     |  109 +-
 lldb/source/Plugins/Protocol/MCP/Tool.h       |   24 +-
 .../SymbolFile/DWARF/DWARFDataExtractor.cpp   |   12 +-
 .../SymbolFile/DWARF/DWARFDataExtractor.h     |    8 -
 .../SymbolFile/DWARF/DWARFFormValue.cpp       |   37 +-
 .../Plugins/SymbolFile/DWARF/DWARFUnit.cpp    |   15 +-
 .../Plugins/SymbolFile/DWARF/DWARFUnit.h      |    3 +
 .../SymbolFile/DWARF/SymbolFileDWARF.cpp      |   29 +
 .../SymbolFile/DWARF/SymbolFileDWARF.h        |    5 +
 lldb/source/Symbol/SaveCoreOptions.cpp        |    2 +-
 lldb/source/Target/Process.cpp                |    2 +-
 lldb/source/Target/StackFrame.cpp             |    1 +
 lldb/source/Target/Statistics.cpp             |   12 +-
 lldb/source/Target/Target.cpp                 |   10 +-
 lldb/source/Utility/IOObject.cpp              |    9 +
 lldb/source/Utility/SelectHelper.cpp          |    2 +-
 lldb/source/ValueObject/DILEval.cpp           |  191 +-
 lldb/source/ValueObject/DILParser.cpp         |   26 +-
 .../TestCallStdStringFunction.py              |    1 -
 .../TestExpressionInSyscall.py                |   13 +-
 ...estSharedPtrDbgInfoContentFromStdModule.py |    2 +-
 .../shared_ptr/TestSharedPtrFromStdModule.py  |    6 +-
 .../TestDbgInfoContentWeakPtrFromStdModule.py |    4 +-
 .../weak_ptr/TestWeakPtrFromStdModule.py      |    6 +-
 .../ir-interpreter/TestIRInterpreter.py       |   14 +-
 .../frame/var-dil/basics/BitField}/Makefile   |    1 -
 .../BitField/TestFrameVarDILBitField.py       |   43 +
 .../frame/var-dil/basics/BitField/main.cpp    |   42 +
 .../basics/GlobalVariableLookup/Makefile      |    2 +-
 .../TestFrameVarDILGlobalVariableLookup.py    |   23 +-
 .../basics/GlobalVariableLookup/extern.cpp    |   10 +
 .../basics/GlobalVariableLookup/main.cpp      |    3 +
 .../var-dil/basics/Indirection}/Makefile      |    2 -
 .../Indirection/TestFrameVarDILIndirection.py |   46 +
 .../frame/var-dil/basics/Indirection/main.cpp |   12 +
 .../MemberOf/TestFrameVarDILMemberOf.py       |   19 +-
 .../TestFrameVarDILMemberOfAnonymousMember.py |   35 +-
 .../basics/PointerDereference/Makefile        |    3 +
 .../TestFrameVarDILPointerDereference.py      |   48 +
 .../basics/PointerDereference/main.cpp        |   34 +
 .../frame/var-dil/basics/QualifiedId/Makefile |    3 +
 .../QualifiedId/TestFrameVarDILQualifiedId.py |   31 +
 .../frame/var-dil/basics/QualifiedId/main.cpp |   16 +
 .../basics/SyntheticDereference/Makefile      |    3 +
 .../TestFrameVarDILSyntheticDereference.py    |   34 +
 .../basics/SyntheticDereference/main.cpp      |   26 +
 .../wrapPtrSynthProvider.py                   |   29 +
 .../TestPlatformLaunchGDBServer.py            |   69 +-
 .../commands/protocol/TestMCPUnixSocket.py    |   34 +
 .../commands/statistics/basic/TestStats.py    |  203 +-
 .../API/commands/statistics/basic/baz.cpp     |   12 +
 .../API/commands/statistics/basic/third.cpp   |    7 +
 .../after_rebuild/Makefile                    |    1 +
 .../TestLocationsAfterRebuild.py              |   59 +
 .../breakpoint_locations/after_rebuild/main.c |    1 +
 .../after_rebuild/second_main.c               |    1 +
 .../after_rebuild/third_main.c                |    1 +
 .../breakpoint/hardware_breakpoints/base.py   |   10 +
 .../TestHWBreakMultiThread.py                 |    7 +-
 .../TestRequireHWBreakpoints.py               |    8 +-
 .../simple_hw_breakpoints/Makefile            |    7 +
 .../TestSimpleHWBreakpoints.py                |   50 +
 .../simple_hw_breakpoints/main.c              |    7 +
 .../TestWriteMemoryWithHWBreakpoint.py        |    5 +-
 .../generic/atomic/Makefile                   |    3 +
 .../atomic/TestDataFormatterStdAtomic.py}     |   17 +-
 .../{libcxx => generic}/atomic/main.cpp       |   26 +-
 .../{libcxx => generic}/chrono/Makefile       |    4 +-
 .../chrono/TestDataFormatterStdChrono.py}     |   14 +-
 .../{libcxx => generic}/chrono/main.cpp       |    0
 .../deque/TestDataFormatterGenericDeque.py    |   74 +-
 .../data-formatter-stl/generic/deque/main.cpp |   25 +-
 .../data-formatter-stl/generic/map/Makefile   |    3 +
 .../map/TestDataFormatterStdMap.py}           |   27 +-
 .../data-formatter-stl/generic/map/main.cpp   |   80 +
 .../set/TestDataFormatterGenericSet.py        |    2 +-
 .../generic/shared_ptr/Makefile               |    3 +
 .../TestDataFormatterStdSharedPtr.py}         |   76 +-
 .../generic/shared_ptr/main.cpp               |   38 +
 .../generic/unique_ptr/Makefile               |    4 +
 .../TestDataFormatterStdUniquePtr.py          |  157 +
 .../{libcxx => generic}/unique_ptr/main.cpp   |   25 +-
 .../{libstdcpp/map => generic/vbool}/Makefile |    3 +-
 .../vbool/TestDataFormatterStdVBool.py        |   33 +-
 .../data-formatter-stl/generic/vbool/main.cpp |   65 +
 .../generic/vector/Makefile                   |    3 +
 .../vector/TestDataFormatterStdVector.py}     |   43 +-
 .../generic/vector/main.cpp                   |   40 +
 .../libcxx-simulators/invalid-vector/main.cpp |    2 +-
 .../TestDataFormatterLibcxxStringSimulator.py |    2 +-
 .../libcxx-simulators/string/main.cpp         |    6 +-
 ...stDataFormatterLibcxxUniquePtrSimulator.py |    2 +-
 .../libcxx-simulators/unique_ptr/main.cpp     |    3 +-
 .../data-formatter-stl/libcxx/atomic/Makefile |    5 -
 .../deque/TestDataFormatterLibcxxDeque.py     |   87 -
 .../data-formatter-stl/libcxx/deque/main.cpp  |   30 -
 .../libcxx/{vbool => invalid-string}/Makefile |    3 +-
 .../TestDataFormatterLibcxxInvalidString.py   |   38 +
 .../libcxx/invalid-string/main.cpp            |  110 +
 .../data-formatter-stl/libcxx/map/main.cpp    |   82 -
 .../libcxx/shared_ptr/Makefile                |    8 -
 .../libcxx/shared_ptr/main.cpp                |   18 -
 .../string/TestDataFormatterLibcxxString.py   |   21 -
 .../data-formatter-stl/libcxx/string/main.cpp |  103 -
 .../libcxx/unique_ptr/Makefile                |    8 -
 .../TestDataFormatterLibcxxUniquePtr.py       |  117 -
 .../{map => unordered_map-iterator}/Makefile  |    0
 .../TestDataFormatterLibccUnorderedMap.py     |   63 +
 .../libcxx/unordered_map-iterator/main.cpp    |   53 +
 .../libcxx/unordered_map/Makefile             |    6 -
 .../libcxx/unordered_map/main.cpp             |   28 -
 .../vbool/TestDataFormatterLibcxxVBool.py     |   76 -
 .../data-formatter-stl/libcxx/vbool/main.cpp  |   65 -
 .../data-formatter-stl/libcxx/vector/Makefile |    6 -
 .../data-formatter-stl/libcxx/vector/main.cpp |   41 -
 .../libstdcpp/map/TestDataFormatterStdMap.py  |  301 -
 .../data-formatter-stl/libstdcpp/map/main.cpp |   55 -
 .../libstdcpp/smart_ptr/Makefile              |    6 -
 .../smart_ptr/TestDataFormatterStdSmartPtr.py |   52 -
 .../libstdcpp/smart_ptr/main.cpp              |   26 -
 .../TestDataFormatterStdUniquePtr.py          |  134 -
 .../TestDataFormatterInvalidStdUniquePtr.py   |    4 -
 .../libstdcpp/unique_ptr/invalid/main.cpp     |   11 -
 .../libstdcpp/unique_ptr/main.cpp             |   35 -
 .../TestDataFormatterLibStdcxxVariant.py      |   11 +-
 .../libstdcpp/variant/main.cpp                |   25 +-
 .../libstdcpp/vbool/Makefile                  |    6 -
 .../libstdcpp/vbool/main.cpp                  |   63 -
 .../libstdcpp/vector/Makefile                 |    6 -
 .../vector/TestDataFormatterStdVector.py      |  220 -
 .../libstdcpp/vector/main.cpp                 |   31 -
 .../gdb_remote_client/TestGDBRemoteClient.py  |   72 +
 .../postmortem/elf-core/expr/TestExpr.py      |   52 +
 .../elf-core/expr/linux-x86_64.core           |  Bin 0 -> 40960 bytes
 .../postmortem/elf-core/expr/linux-x86_64.out |  Bin 0 -> 10816 bytes
 .../postmortem/elf-core/expr/main.cpp         |   15 +
 .../TestProgressReporting.py                  |   22 +
 .../statusline/TestStatusline.py              |   21 +-
 .../API/lang/cpp/char8_t/TestCxxChar8_t.py    |    4 +-
 .../lang/objc/class-without-methods/Makefile  |    5 +
 .../lang/objc/class-without-methods/Point.h   |    4 +
 .../lang/objc/class-without-methods/Point.m   |    7 +
 .../TestObjCClassWithoutMethods.py            |   11 +
 .../lang/objc/class-without-methods/main.m    |    7 +
 .../TestAArch64LinuxMTEMemoryTagCoreFile.py   |   23 +
 .../aarch64/mte_core_file/core.mte.notags     |  Bin 0 -> 32768 bytes
 .../API/linux/aarch64/mte_core_file/main.c    |    7 +-
 .../lc-note/additional-registers/Makefile     |   11 +
 .../TestMetadataRegisters.py                  |  150 +
 .../additional-registers/add-lcnote.cpp       |  385 +
 .../lc-note/additional-registers/main.c       |   11 +
 lldb/test/API/macosx/simulator/Makefile       |    5 +-
 .../macosx/simulator/TestSimulatorPlatform.py |   42 +-
 lldb/test/API/macosx/simulator/hello.c        |    5 -
 lldb/test/API/macosx/simulator/hello.cpp      |   14 +
 .../python_api/debugger/TestDebuggerAPI.py    |    8 +
 lldb/test/API/riscv/step/Makefile             |    3 +
 lldb/test/API/riscv/step/TestSoftwareStep.py  |   79 +
 lldb/test/API/riscv/step/branch.c             |   22 +
 .../step/incomplete_sequence_without_lr.c     |   22 +
 .../step/incomplete_sequence_without_sc.c     |   21 +
 lldb/test/API/riscv/step/main.c               |   22 +
 .../TestDAP_breakpointAssembly.py             |   14 +-
 .../tools/lldb-dap/console/TestDAP_console.py |    4 +-
 .../restart/TestDAP_restart_runInTerminal.py  |    4 +-
 .../runInTerminal/TestDAP_runInTerminal.py    |    5 +-
 .../API/tools/lldb-dap/runInTerminal/main.c   |    4 +
 .../tools/lldb-dap/server/TestDAP_server.py   |    4 +-
 .../API/tools/lldb-dap/step/TestDAP_step.py   |   38 +
 lldb/test/API/tools/lldb-dap/step/main.cpp    |   14 +-
 lldb/test/API/tools/lldb-dap/step/other.h     |    7 +
 .../lldb-server/TestAppleSimulatorOSType.py   |   72 +-
 .../Shell/Scripts/Inputs/Main/SBAddress.h     |    1 +
 .../Shell/Scripts/TestFrameworkFixScript.test |    1 +
 .../Shell/Scripts/TestVersionFixScript.test   |    2 +-
 lldb/tools/lldb-dap/Breakpoint.cpp            |    4 +-
 lldb/tools/lldb-dap/DAP.cpp                   |   81 +
 lldb/tools/lldb-dap/DAP.h                     |   42 +-
 lldb/tools/lldb-dap/DAPError.h                |    5 +
 .../Handler/DisassembleRequestHandler.cpp     |   13 +-
 .../Handler/LocationsRequestHandler.cpp       |   22 +-
 .../lldb-dap/Handler/SourceRequestHandler.cpp |   32 +-
 .../Handler/StackTraceRequestHandler.cpp      |    2 +-
 lldb/tools/lldb-dap/JSONUtils.cpp             |   14 +-
 lldb/tools/lldb-dap/JSONUtils.h               |    5 +-
 lldb/tools/lldb-dap/ProgressEvent.h           |   13 +-
 .../lldb-dap/Protocol/ProtocolRequests.h      |    2 +-
 .../tools/lldb-dap/Protocol/ProtocolTypes.cpp |    2 +-
 lldb/tools/lldb-dap/Protocol/ProtocolTypes.h  |    3 +-
 lldb/tools/lldb-dap/ProtocolUtils.cpp         |   73 +-
 lldb/tools/lldb-dap/ProtocolUtils.h           |   19 +-
 lldb/tools/lldb-dap/SourceBreakpoint.cpp      |    8 +-
 lldb/tools/lldb-dap/tool/lldb-dap.cpp         |    2 -
 lldb/tools/lldb-server/lldb-gdbserver.cpp     |    6 +-
 lldb/tools/lldb-server/lldb-platform.cpp      |   43 +-
 lldb/tools/lldb-test/lldb-test.cpp            |   10 +-
 lldb/unittests/Core/CommunicationTest.cpp     |    3 +-
 lldb/unittests/DAP/TestBase.cpp               |    2 +-
 lldb/unittests/DAP/TestBase.h                 |    2 +-
 lldb/unittests/Host/CMakeLists.txt            |    4 +-
 lldb/unittests/Host/FileTest.cpp              |   16 +-
 lldb/unittests/Host/HostTest.cpp              |    3 +-
 lldb/unittests/Host/JSONTransportTest.cpp     |    4 +-
 lldb/unittests/Host/MainLoopTest.cpp          |   72 +-
 lldb/unittests/Host/PipeTest.cpp              |   25 +-
 lldb/unittests/Host/SocketTest.cpp            |    4 +-
 lldb/unittests/Host/posix/SupportTest.cpp     |   19 +-
 .../LoongArch/TestLoongArchEmulator.cpp       |   42 +-
 .../MemoryTagManagerAArch64MTETest.cpp        |   51 +-
 .../Protocol/ProtocolMCPServerTest.cpp        |   21 +-
 lldb/unittests/Protocol/ProtocolMCPTest.cpp   |   97 +
 .../unittests/SymbolFile/DWARF/CMakeLists.txt |    1 +
 .../SymbolFile/DWARF/DWARF64UnitTest.cpp      |   86 +
 .../TestingSupport/Host/PipeTestUtilities.h   |    6 +-
 llvm/CMakeLists.txt                           |   19 +-
 llvm/Maintainers.md                           |    5 +-
 llvm/cmake/modules/AddLLVM.cmake              |    6 +-
 llvm/cmake/modules/HandleLLVMOptions.cmake    |    2 +-
 llvm/docs/AMDGPUUsage.rst                     |   45 +-
 llvm/docs/AliasAnalysis.rst                   |   14 -
 llvm/docs/CMake.rst                           |    6 +-
 llvm/docs/CodingStandards.rst                 |   21 +-
 llvm/docs/CommandGuide/llvm-objdump.rst       |    2 +-
 llvm/docs/DeveloperPolicy.rst                 |  319 +-
 llvm/docs/Extensions.rst                      |   21 -
 llvm/docs/GitHub.rst                          |    2 +
 llvm/docs/HowToAddABuilder.rst                |    7 +-
 llvm/docs/HowToReleaseLLVM.rst                |   14 +-
 llvm/docs/HowToUpdateDebugInfo.rst            |   38 +-
 llvm/docs/InterfaceExportAnnotations.rst      |    7 +-
 llvm/docs/LangRef.rst                         |   52 +-
 llvm/docs/MLGO.rst                            |   52 +-
 llvm/docs/QualGroup.rst                       |   60 +
 llvm/docs/RISCVUsage.rst                      |    6 +
 llvm/docs/ReleaseNotes.md                     |    3 +
 llvm/docs/SPIRVUsage.rst                      |    9 +
 llvm/docs/SourceLevelDebugging.rst            |   57 +-
 llvm/docs/index.rst                           |    4 +
 .../BuildingAJIT/Chapter1/KaleidoscopeJIT.h   |    1 +
 .../BuildingAJIT/Chapter2/KaleidoscopeJIT.h   |    1 +
 .../BuildingAJIT/Chapter3/KaleidoscopeJIT.h   |    1 +
 .../BuildingAJIT/Chapter4/KaleidoscopeJIT.h   |    1 +
 .../Kaleidoscope/include/KaleidoscopeJIT.h    |    1 +
 .../LLJITWithExecutorProcessControl.cpp       |    1 +
 .../LLJITWithThinLTOSummaries.cpp             |    4 +-
 .../OrcV2CBindingsBasicUsage.c                |   10 +-
 .../OrcV2CBindingsDumpObjects.c               |    5 +-
 .../OrcV2CBindingsIRTransforms.c              |    4 +-
 .../OrcV2CBindingsLazy/OrcV2CBindingsLazy.c   |   10 +-
 .../OrcV2CBindingsMCJITLikeMemoryManager.c    |   11 +-
 .../OrcV2CBindingsRemovableCode.c             |   11 +-
 .../OrcV2CBindingsVeryLazy.c                  |   10 +-
 .../SpeculativeJIT/SpeculativeJIT.cpp         |    1 +
 llvm/include/llvm-c/DebugInfo.h               |   10 +
 llvm/include/llvm-c/Orc.h                     |   26 +-
 llvm/include/llvm-c/lto.h                     |   17 +-
 llvm/include/llvm/ADT/APFloat.h               |   25 +-
 llvm/include/llvm/ADT/APInt.h                 |    2 +-
 llvm/include/llvm/ADT/ArrayRef.h              |  112 +-
 llvm/include/llvm/ADT/BitVector.h             |    4 +-
 llvm/include/llvm/ADT/Bitset.h                |    2 +-
 llvm/include/llvm/ADT/DynamicAPInt.h          |   11 +-
 llvm/include/llvm/ADT/STLExtras.h             |    6 +
 llvm/include/llvm/ADT/STLForwardCompat.h      |   66 +-
 llvm/include/llvm/ADT/SlowDynamicAPInt.h      |   17 +-
 llvm/include/llvm/ADT/SmallPtrSet.h           |    2 +-
 llvm/include/llvm/ADT/TinyPtrVector.h         |   64 +-
 .../llvm/Analysis/BlockFrequencyInfoImpl.h    |   38 +-
 .../llvm/Analysis/BranchProbabilityInfo.h     |    4 +-
 llvm/include/llvm/Analysis/DXILResource.h     |    4 +-
 .../Analysis/FunctionPropertiesAnalysis.h     |   25 +-
 llvm/include/llvm/Analysis/IR2Vec.h           |   69 +-
 llvm/include/llvm/Analysis/IVDescriptors.h    |   42 +-
 llvm/include/llvm/Analysis/InlineAdvisor.h    |    4 +
 .../llvm/Analysis/InlineModelFeatureMaps.h    |   16 +-
 .../llvm/Analysis/LoopAccessAnalysis.h        |    4 -
 llvm/include/llvm/Analysis/MLInlineAdvisor.h  |    1 +
 .../Analysis/ScalarEvolutionPatternMatch.h    |    3 +
 .../include/llvm/Analysis/TargetLibraryInfo.h |    8 +-
 .../llvm/Analysis/TargetTransformInfo.h       |   18 +-
 .../llvm/Analysis/TargetTransformInfoImpl.h   |   18 +-
 llvm/include/llvm/Analysis/ValueTracking.h    |   23 +-
 llvm/include/llvm/BinaryFormat/DXContainer.h  |   63 +-
 .../BinaryFormat/DXContainerConstants.def     |  206 +-
 llvm/include/llvm/BinaryFormat/Dwarf.def      |   13 +-
 llvm/include/llvm/BinaryFormat/Dwarf.h        |    1 +
 llvm/include/llvm/BinaryFormat/ELF.h          |    4 +-
 llvm/include/llvm/BinaryFormat/GOFF.h         |   85 +
 llvm/include/llvm/BinaryFormat/Wasm.h         |    4 +-
 llvm/include/llvm/Bitcode/BitcodeReader.h     |    2 +-
 llvm/include/llvm/Bitcode/LLVMBitCodes.h      |    1 +
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   18 +-
 .../llvm/CodeGen/GlobalISel/CallLowering.h    |    9 +
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |   13 +-
 .../llvm/CodeGen/GlobalISel/LegalizerHelper.h |    5 +
 llvm/include/llvm/CodeGen/MachineOperand.h    |    2 +-
 llvm/include/llvm/CodeGen/Passes.h            |    3 +
 llvm/include/llvm/CodeGen/PseudoSourceValue.h |    4 +-
 .../include/llvm/CodeGen/RuntimeLibcallUtil.h |    4 +
 llvm/include/llvm/CodeGen/SDPatternMatch.h    |   91 +-
 llvm/include/llvm/CodeGen/SelectionDAG.h      |    5 +
 llvm/include/llvm/CodeGen/SelectionDAGISel.h  |    6 +-
 llvm/include/llvm/CodeGen/SelectionDAGNodes.h |   18 +
 llvm/include/llvm/CodeGen/TargetLowering.h    |   19 +-
 .../CodeGen/TargetLoweringObjectFileImpl.h    |    5 +
 llvm/include/llvm/CodeGenTypes/LowLevelType.h |    5 +-
 .../llvm/CodeGenTypes/MachineValueType.h      |    9 +-
 llvm/include/llvm/Config/llvm-config.h.cmake  |    5 +-
 .../DebugInfo/CodeView/CodeViewSymbols.def    |    2 +
 .../llvm/DebugInfo/CodeView/SymbolRecord.h    |   27 +-
 .../llvm/DebugInfo/DWARF/DWARFCFIPrinter.h    |    8 +-
 .../llvm/DebugInfo/DWARF/DWARFCFIProgram.h    |   22 +-
 .../DWARF/DWARFDataExtractorSimple.h          |    6 +-
 .../llvm/DebugInfo/DWARF/DWARFDebugFrame.h    |    2 +
 .../llvm/DebugInfo/DWARF/DWARFExpression.h    |   25 +-
 .../DebugInfo/LogicalView/Core/LVObject.h     |    8 +
 .../LogicalView/Core/LVSourceLanguage.h       |    3 +-
 .../LogicalView/Readers/LVDWARFReader.h       |    9 +
 llvm/include/llvm/Demangle/ItaniumDemangle.h  |    4 +-
 .../llvm/Demangle/MicrosoftDemangleNodes.h    |  179 +-
 llvm/include/llvm/ExecutionEngine/Orc/COFF.h  |    5 +-
 .../Orc/EPCGenericMemoryAccess.h              |  131 +-
 .../Orc/ExecutorProcessControl.h              |  201 +-
 .../Orc/InProcessMemoryAccess.h               |   68 +
 llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h |    3 +-
 .../llvm/ExecutionEngine/Orc/MemoryAccess.h   |  195 +
 .../Orc/SelfExecutorProcessControl.h          |   76 +
 .../ExecutionEngine/Orc/Shared/OrcRTBridge.h  |   10 +-
 .../Orc/Shared/TargetProcessControlTypes.h    |    4 +-
 .../ExecutionEngine/Orc/SymbolStringPool.h    |    8 +-
 .../ExecutionEngine/Orc/ThreadSafeModule.h    |   78 +-
 .../llvm/ExecutionEngine/RuntimeDyld.h        |    2 +-
 .../llvm/Frontend/Directive/Spelling.h        |    4 +-
 .../llvm/Frontend/Driver/CodeGenOptions.h     |    2 +-
 .../llvm/Frontend/HLSL/HLSLRootSignature.h    |  195 +-
 .../Frontend/HLSL/RootSignatureMetadata.h     |   56 +
 ...tureUtils.h => RootSignatureValidations.h} |   71 +-
 .../Frontend/OpenMP/DirectiveNameParser.h     |   77 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |   50 +-
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |    8 +-
 llvm/include/llvm/IR/Argument.h               |    3 +
 llvm/include/llvm/IR/Attributes.td            |   13 +
 llvm/include/llvm/IR/CMakeLists.txt           |    3 +
 llvm/include/llvm/IR/DIBuilder.h              |   95 +-
 llvm/include/llvm/IR/DebugInfoMetadata.h      |  600 +-
 llvm/include/llvm/IR/DebugLoc.h               |   49 +-
 llvm/include/llvm/IR/Function.h               |    5 +-
 llvm/include/llvm/IR/IRBuilder.h              |    8 +-
 llvm/include/llvm/IR/Instruction.h            |    2 +-
 llvm/include/llvm/IR/Intrinsics.td            |    4 +-
 llvm/include/llvm/IR/IntrinsicsAArch64.td     |   16 +-
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |   83 +
 llvm/include/llvm/IR/IntrinsicsARM.td         |   32 +-
 llvm/include/llvm/IR/IntrinsicsPowerPC.td     |    8 +
 llvm/include/llvm/IR/LegacyPassManagers.h     |    3 +-
 llvm/include/llvm/IR/OptBisect.h              |   12 +-
 llvm/include/llvm/IR/PatternMatch.h           |    7 +
 llvm/include/llvm/IR/ProfDataUtils.h          |   23 +
 llvm/include/llvm/IR/RuntimeLibcalls.def      |  701 --
 llvm/include/llvm/IR/RuntimeLibcalls.h        |   85 +-
 llvm/include/llvm/IR/RuntimeLibcalls.td       | 1661 +++++
 llvm/include/llvm/IR/RuntimeLibcallsImpl.td   |   32 +
 llvm/include/llvm/IR/Type.h                   |    2 +-
 llvm/include/llvm/InitializePasses.h          |    1 +
 llvm/include/llvm/LTO/legacy/LTOModule.h      |    8 +
 llvm/include/llvm/LinkAllPasses.h             |    6 +-
 .../llvm/MC/DXContainerRootSignature.h        |    2 +-
 llvm/include/llvm/MC/MCAsmBackend.h           |   43 +-
 llvm/include/llvm/MC/MCAsmInfo.h              |    7 +-
 llvm/include/llvm/MC/MCAssembler.h            |   13 +-
 llvm/include/llvm/MC/MCCodeView.h             |    5 +
 llvm/include/llvm/MC/MCContext.h              |   17 +-
 llvm/include/llvm/MC/MCDXContainerStreamer.h  |    3 -
 llvm/include/llvm/MC/MCExpr.h                 |   42 +-
 llvm/include/llvm/MC/MCFixup.h                |   29 +-
 llvm/include/llvm/MC/MCFixupKindInfo.h        |   47 -
 llvm/include/llvm/MC/MCFragment.h             |  576 --
 llvm/include/llvm/MC/MCGOFFAttributes.h       |   92 +
 llvm/include/llvm/MC/MCGOFFObjectWriter.h     |   20 +
 llvm/include/llvm/MC/MCGOFFStreamer.h         |    7 +-
 llvm/include/llvm/MC/MCInst.h                 |   11 +-
 llvm/include/llvm/MC/MCMachObjectWriter.h     |    2 -
 llvm/include/llvm/MC/MCObjectFileInfo.h       |    4 -
 llvm/include/llvm/MC/MCObjectStreamer.h       |    5 +-
 .../llvm/MC/MCParser/MCParsedAsmOperand.h     |   12 +-
 llvm/include/llvm/MC/MCSPIRVStreamer.h        |    3 -
 llvm/include/llvm/MC/MCSchedule.h             |   12 +-
 llvm/include/llvm/MC/MCSection.h              |  578 +-
 llvm/include/llvm/MC/MCSectionGOFF.h          |  100 +-
 llvm/include/llvm/MC/MCStreamer.h             |    4 +-
 llvm/include/llvm/MC/MCSymbol.h               |    2 +-
 llvm/include/llvm/MC/MCSymbolGOFF.h           |   26 +
 llvm/include/llvm/MC/MCWasmStreamer.h         |    2 -
 llvm/include/llvm/MC/MCWinCOFFStreamer.h      |    2 -
 llvm/include/llvm/MC/MCXCOFFStreamer.h        |    1 -
 llvm/include/llvm/MCA/CodeEmitter.h           |    3 +-
 llvm/include/llvm/Object/DXContainer.h        |   26 +-
 llvm/include/llvm/Object/ELF.h                |    2 +-
 .../include/llvm/ObjectYAML/DXContainerYAML.h |   12 +-
 llvm/include/llvm/Pass.h                      |    7 +-
 llvm/include/llvm/PassAnalysisSupport.h       |   24 +-
 .../ProfileData/Coverage/CoverageMapping.h    |   10 +
 .../llvm/ProfileData/InstrProfReader.h        |    2 +-
 llvm/include/llvm/ProfileData/SampleProf.h    |    4 +-
 .../llvm/ProfileData/SampleProfWriter.h       |    4 +-
 llvm/include/llvm/SandboxIR/Value.h           |    1 +
 llvm/include/llvm/Support/Compiler.h          |   45 +-
 llvm/include/llvm/Support/ConvertUTF.h        |    6 +-
 llvm/include/llvm/Support/Error.h             |    2 +-
 llvm/include/llvm/Support/FileSystem.h        |    2 +-
 llvm/include/llvm/Support/JSON.h              |    4 +-
 llvm/include/llvm/Support/KnownFPClass.h      |   12 +-
 llvm/include/llvm/Support/MathExtras.h        |   20 +-
 llvm/include/llvm/Support/Path.h              |   10 +-
 llvm/include/llvm/Support/PrettyStackTrace.h  |    2 +-
 llvm/include/llvm/Support/ProgramStack.h      |    7 +-
 llvm/include/llvm/Support/Signals.h           |   35 +
 llvm/include/llvm/Support/TextEncoding.h      |    8 +-
 llvm/include/llvm/Support/Timer.h             |    2 +-
 llvm/include/llvm/Support/TrailingObjects.h   |   55 +-
 llvm/include/llvm/TableGen/Record.h           |   16 +-
 llvm/include/llvm/TableGen/SetTheory.h        |    6 +-
 llvm/include/llvm/TableGen/StringMatcher.h    |    2 +-
 .../include/llvm/Target/GlobalISel/Combine.td |   14 +-
 llvm/include/llvm/Target/Target.td            |    6 +-
 llvm/include/llvm/Target/TargetMachine.h      |   10 +
 .../llvm/TargetParser/SubtargetFeature.h      |    2 +-
 llvm/include/llvm/TargetParser/TargetParser.h |  150 +-
 llvm/include/llvm/TargetParser/Triple.h       |   20 +-
 llvm/include/llvm/Telemetry/Telemetry.h       |   10 +-
 .../Testing/Demangle/DemangleTestCases.inc    |    1 +
 .../ToolDrivers/llvm-dlltool/DlltoolDriver.h  |    4 +-
 .../llvm/ToolDrivers/llvm-lib/LibDriver.h     |    5 +-
 .../llvm/Transforms/HipStdPar/HipStdPar.h     |    7 +
 llvm/include/llvm/Transforms/IPO/Attributor.h |   41 +
 .../Instrumentation/LowerAllowCheckPass.h     |    1 +
 .../llvm/Transforms/Utils/PredicateInfo.h     |   12 +-
 .../Vectorize/LoopVectorizationLegality.h     |    8 +
 .../SandboxVectorizer/SeedCollector.h         |    3 +-
 llvm/include/llvm/WindowsDriver/MSVCPaths.h   |   72 +-
 .../WindowsManifest/WindowsManifestMerger.h   |   14 +-
 llvm/include/llvm/XRay/InstrumentationMap.h   |    2 +-
 llvm/include/llvm/XRay/Trace.h                |    2 +-
 llvm/include/module.modulemap                 |    1 -
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      |    4 +
 llvm/lib/Analysis/BranchProbabilityInfo.cpp   |    7 +-
 llvm/lib/Analysis/ConstantFolding.cpp         |   47 +-
 llvm/lib/Analysis/DependenceAnalysis.cpp      |  170 +-
 .../Analysis/DevelopmentModeInlineAdvisor.cpp |   10 +-
 .../Analysis/FunctionPropertiesAnalysis.cpp   |  111 +-
 llvm/lib/Analysis/HashRecognize.cpp           |  103 +-
 llvm/lib/Analysis/IR2Vec.cpp                  |  171 +-
 llvm/lib/Analysis/IVDescriptors.cpp           |   61 +-
 .../IndirectCallPromotionAnalysis.cpp         |   11 +-
 llvm/lib/Analysis/InlineAdvisor.cpp           |   29 +
 llvm/lib/Analysis/InlineCost.cpp              |   15 +-
 llvm/lib/Analysis/InstructionSimplify.cpp     |   13 +-
 llvm/lib/Analysis/LazyValueInfo.cpp           |    8 +-
 llvm/lib/Analysis/LoopAccessAnalysis.cpp      |  103 +-
 llvm/lib/Analysis/LoopPass.cpp                |    2 +-
 llvm/lib/Analysis/MLInlineAdvisor.cpp         |   37 +-
 llvm/lib/Analysis/RegionPass.cpp              |    2 +-
 llvm/lib/Analysis/TargetLibraryInfo.cpp       |    8 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  165 +-
 .../models/seedEmbeddingVocab75D.json         |  133 +-
 llvm/lib/AsmParser/LLParser.cpp               |  149 +-
 llvm/lib/BinaryFormat/DXContainer.cpp         |   69 +
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp     |   32 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  106 +-
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp     |  179 +-
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp    |   41 +-
 llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp |   28 +-
 llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h   |    2 +
 llvm/lib/CodeGen/AsmPrinter/DIE.cpp           |    5 +-
 .../AsmPrinter/DbgEntityHistoryCalculator.cpp |   18 +-
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |    9 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |   50 +-
 .../CodeGen/AsmPrinter/DwarfExpression.cpp    |    2 +
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp     |   18 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h       |    3 +-
 llvm/lib/CodeGen/AsmPrinter/WinException.cpp  |    6 +-
 .../CodeGen/AssignmentTrackingAnalysis.cpp    |  320 +-
 llvm/lib/CodeGen/BranchFolding.cpp            |    7 +
 llvm/lib/CodeGen/CMakeLists.txt               |    1 +
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |    7 +
 llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp |    6 +-
 llvm/lib/CodeGen/FaultMaps.cpp                |    8 -
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp |   43 +-
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |    2 +-
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    |   86 +-
 llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp  |    2 +-
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |    2 +-
 llvm/lib/CodeGen/GlobalMergeFunctions.cpp     |    1 +
 llvm/lib/CodeGen/IfConversion.cpp             |   35 +-
 llvm/lib/CodeGen/InterleavedAccessPass.cpp    |  199 +-
 .../LiveDebugValues/InstrRefBasedImpl.cpp     |   29 +-
 llvm/lib/CodeGen/MachineDomTreeUpdater.cpp    |    9 +-
 llvm/lib/CodeGen/MachineDominators.cpp        |   30 +-
 llvm/lib/CodeGen/MachineLICM.cpp              |    6 +-
 llvm/lib/CodeGen/MachineLoopInfo.cpp          |    7 +-
 llvm/lib/CodeGen/MachineOutliner.cpp          |   17 +
 llvm/lib/CodeGen/MachinePassManager.cpp       |   15 +-
 llvm/lib/CodeGen/MachinePostDominators.cpp    |   33 +-
 llvm/lib/CodeGen/MachineScheduler.cpp         |   28 +-
 llvm/lib/CodeGen/RegAllocScore.cpp            |   17 +-
 llvm/lib/CodeGen/RegisterCoalescer.cpp        |    8 +-
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  434 +-
 .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp |    9 +-
 .../SelectionDAG/LegalizeFloatTypes.cpp       |   36 +-
 .../SelectionDAG/ResourcePriorityQueue.cpp    |    4 +-
 .../SelectionDAG/ScheduleDAGRRList.cpp        |    6 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  157 +-
 .../SelectionDAGAddressAnalysis.cpp           |    6 +-
 .../SelectionDAG/SelectionDAGBuilder.cpp      |   33 +-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |   75 +-
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |    9 +-
 .../CodeGen/TargetLoweringObjectFileImpl.cpp  |   78 +-
 llvm/lib/CodeGen/TargetOptionsImpl.cpp        |    7 +-
 llvm/lib/CodeGen/TargetPassConfig.cpp         |    3 +
 llvm/lib/CodeGen/WindowsSecureHotPatching.cpp |  617 ++
 .../Parallel/DWARFLinkerCompileUnit.cpp       |    6 +-
 llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp  |    7 +
 .../CodeView/SymbolRecordMapping.cpp          |    7 +
 .../DebugInfo/DWARF/LowLevel/CMakeLists.txt   |   14 +
 llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp      |    9 +-
 .../DebugInfo/LogicalView/Core/LVScope.cpp    |    2 +-
 .../LogicalView/Readers/LVDWARFReader.cpp     |   15 +-
 .../DebugInfo/PDB/Native/GSIStreamBuilder.cpp |    2 +-
 llvm/lib/DebugInfo/PDB/UDTLayout.cpp          |    1 -
 llvm/lib/ExecutionEngine/ExecutionEngine.cpp  |    9 +-
 .../ExecutionEngine/JITLink/ELF_aarch64.cpp   |    2 +-
 .../ExecutionEngine/JITLink/MachO_arm64.cpp   |    2 +-
 .../JITLink/XCOFFLinkGraphBuilder.cpp         |    2 +-
 llvm/lib/ExecutionEngine/JITLink/aarch64.cpp  |    2 +-
 llvm/lib/ExecutionEngine/JITLink/ppc64.cpp    |    2 +-
 llvm/lib/ExecutionEngine/JITLink/x86_64.cpp   |    2 +-
 llvm/lib/ExecutionEngine/Orc/CMakeLists.txt   |    2 +
 .../Orc/ExecutorProcessControl.cpp            |  217 +-
 .../Orc/InProcessMemoryAccess.cpp             |  134 +
 llvm/lib/ExecutionEngine/Orc/LLJIT.cpp        |   16 +-
 .../ExecutionEngine/Orc/OrcV2CBindings.cpp    |    6 +-
 llvm/lib/ExecutionEngine/Orc/SectCreate.cpp   |    3 +-
 .../Orc/SelfExecutorProcessControl.cpp        |  172 +
 .../Orc/Shared/OrcRTBridge.cpp                |   19 +-
 .../ExecutionEngine/Orc/SimpleRemoteEPC.cpp   |   10 +-
 llvm/lib/ExecutionEngine/Orc/Speculation.cpp  |    2 -
 .../Orc/TargetProcess/OrcRTBootstrap.cpp      |  100 +-
 .../ExecutionEngine/Orc/ThreadSafeModule.cpp  |   81 +-
 llvm/lib/FileCheck/FileCheck.cpp              |   54 +-
 llvm/lib/FileCheck/FileCheckImpl.h            |   24 +-
 llvm/lib/Frontend/HLSL/CMakeLists.txt         |    5 +-
 llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp  |  244 +
 .../Frontend/HLSL/HLSLRootSignatureUtils.cpp  |  577 --
 .../Frontend/HLSL/RootSignatureMetadata.cpp   |  194 +
 .../HLSL/RootSignatureValidations.cpp         |   84 +
 llvm/lib/Frontend/OpenMP/CMakeLists.txt       |    1 +
 .../Frontend/OpenMP/DirectiveNameParser.cpp   |   83 +
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |   53 +-
 llvm/lib/FuzzMutate/IRMutator.cpp             |   49 +-
 llvm/lib/FuzzMutate/RandomIRBuilder.cpp       |   88 +-
 llvm/lib/IR/AsmWriter.cpp                     |   12 +-
 llvm/lib/IR/Attributes.cpp                    |    3 +-
 llvm/lib/IR/AutoUpgrade.cpp                   |    6 +
 llvm/lib/IR/DIBuilder.cpp                     |  110 +-
 llvm/lib/IR/DebugInfo.cpp                     |   17 +-
 llvm/lib/IR/DebugInfoMetadata.cpp             |  139 +-
 llvm/lib/IR/DebugLoc.cpp                      |   26 +-
 llvm/lib/IR/Function.cpp                      |   14 +-
 llvm/lib/IR/IRBuilder.cpp                     |    2 +-
 llvm/lib/IR/Instructions.cpp                  |    6 +-
 llvm/lib/IR/LLVMContextImpl.h                 |  119 +-
 llvm/lib/IR/LegacyPassManager.cpp             |    6 +-
 llvm/lib/IR/MDBuilder.cpp                     |    9 +-
 llvm/lib/IR/Metadata.cpp                      |    7 +-
 llvm/lib/IR/Module.cpp                        |   44 +-
 llvm/lib/IR/OptBisect.cpp                     |   14 +-
 llvm/lib/IR/Pass.cpp                          |    8 +-
 llvm/lib/IR/ProfDataUtils.cpp                 |   52 +-
 llvm/lib/IR/RuntimeLibcalls.cpp               |  680 +-
 llvm/lib/IR/Type.cpp                          |    2 +-
 llvm/lib/IR/Verifier.cpp                      |  112 +-
 llvm/lib/IR/ZOSLibcallNames.def               |  100 -
 llvm/lib/LTO/LTO.cpp                          |   12 +-
 llvm/lib/MC/CMakeLists.txt                    |    1 +
 llvm/lib/MC/ConstantPools.cpp                 |    2 +-
 llvm/lib/MC/DXContainerRootSignature.cpp      |   45 +-
 llvm/lib/MC/ELFObjectWriter.cpp               |   49 +-
 llvm/lib/MC/GOFFObjectWriter.cpp              |  344 +-
 llvm/lib/MC/MCAsmBackend.cpp                  |   36 +-
 llvm/lib/MC/MCAsmInfo.cpp                     |   19 +-
 llvm/lib/MC/MCAsmStreamer.cpp                 |   36 +-
 llvm/lib/MC/MCAssembler.cpp                   |  351 +-
 llvm/lib/MC/MCCodeView.cpp                    |   25 +-
 llvm/lib/MC/MCContext.cpp                     |   48 +-
 llvm/lib/MC/MCDXContainerStreamer.cpp         |    3 -
 llvm/lib/MC/MCDisassembler/Disassembler.h     |    2 +-
 llvm/lib/MC/MCDwarf.cpp                       |   10 +-
 llvm/lib/MC/MCELFStreamer.cpp                 |   14 +-
 llvm/lib/MC/MCExpr.cpp                        |   25 +-
 llvm/lib/MC/MCFragment.cpp                    |  186 +-
 llvm/lib/MC/MCGOFFStreamer.cpp                |   20 +
 llvm/lib/MC/MCInst.cpp                        |   31 +-
 llvm/lib/MC/MCMachOStreamer.cpp               |   20 -
 llvm/lib/MC/MCObjectFileInfo.cpp              |   58 +-
 llvm/lib/MC/MCObjectStreamer.cpp              |   66 +-
 llvm/lib/MC/MCObjectWriter.cpp                |    1 -
 llvm/lib/MC/MCParser/AsmParser.cpp            |   40 +-
 llvm/lib/MC/MCParser/MCAsmParser.cpp          |    4 +-
 llvm/lib/MC/MCParser/MasmParser.cpp           |    2 +-
 llvm/lib/MC/MCPseudoProbe.cpp                 |    6 +-
 llvm/lib/MC/MCRegisterInfo.cpp                |    2 +-
 llvm/lib/MC/MCSPIRVStreamer.cpp               |   15 -
 llvm/lib/MC/MCSchedule.cpp                    |    1 +
 llvm/lib/MC/MCSection.cpp                     |   67 +-
 llvm/lib/MC/MCSectionELF.cpp                  |    3 +-
 llvm/lib/MC/MCSectionGOFF.cpp                 |  143 +
 llvm/lib/MC/MCStreamer.cpp                    |    8 +-
 llvm/lib/MC/MCSubtargetInfo.cpp               |    1 +
 llvm/lib/MC/MCSymbol.cpp                      |    1 -
 llvm/lib/MC/MCWasmStreamer.cpp                |   21 -
 llvm/lib/MC/MCWin64EH.cpp                     |    2 +-
 llvm/lib/MC/MCWinCOFFStreamer.cpp             |   28 +-
 llvm/lib/MC/MCXCOFFStreamer.cpp               |   22 +-
 llvm/lib/MC/MachObjectWriter.cpp              |   13 +-
 llvm/lib/MC/WasmObjectWriter.cpp              |    4 +-
 llvm/lib/MC/WinCOFFObjectWriter.cpp           |    7 +-
 llvm/lib/MC/XCOFFObjectWriter.cpp             |   10 +-
 llvm/lib/MCA/CodeEmitter.cpp                  |    6 +-
 llvm/lib/MCA/InstrBuilder.cpp                 |    3 +-
 llvm/lib/ObjCopy/COFF/COFFWriter.cpp          |    7 +-
 llvm/lib/Object/ArchiveWriter.cpp             |    4 +-
 llvm/lib/Object/ELF.cpp                       |   60 +-
 llvm/lib/Object/ELFObjectFile.cpp             |    3 +-
 llvm/lib/Object/IRSymtab.cpp                  |    8 +-
 llvm/lib/Object/OffloadBundle.cpp             |    3 +-
 llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp   |    5 +
 llvm/lib/ObjectYAML/DXContainerYAML.cpp       |  124 +-
 llvm/lib/ObjectYAML/ELFYAML.cpp               |    1 -
 llvm/lib/Option/Arg.cpp                       |   17 +-
 llvm/lib/Passes/PassBuilder.cpp               |  245 +-
 llvm/lib/Passes/PassBuilderPipelines.cpp      |    7 +-
 llvm/lib/Passes/PassRegistry.def              |  123 +-
 llvm/lib/Passes/StandardInstrumentations.cpp  |    2 +-
 llvm/lib/ProfileData/InstrProf.cpp            |   25 +-
 llvm/lib/ProfileData/InstrProfReader.cpp      |    6 +-
 llvm/lib/Support/Signals.cpp                  |  206 +-
 llvm/lib/Support/TextEncoding.cpp             |    2 +-
 llvm/lib/Support/Unix/Signals.inc             |   15 +
 llvm/lib/Support/Unix/Threading.inc           |   15 +-
 llvm/lib/Support/Windows/Process.inc          |    5 -
 llvm/lib/Support/Windows/Signals.inc          |   65 +-
 llvm/lib/Support/Z3Solver.cpp                 |    3 +-
 llvm/lib/TableGen/DetailedRecordsBackend.cpp  |    4 +-
 llvm/lib/TableGen/Error.cpp                   |   12 +-
 llvm/lib/TableGen/Main.cpp                    |    4 +-
 llvm/lib/TableGen/Record.cpp                  |   12 +-
 llvm/lib/TableGen/StringMatcher.cpp           |    9 +-
 llvm/lib/TableGen/StringToOffsetTable.cpp     |    6 +-
 llvm/lib/TableGen/TGLexer.cpp                 |   38 +-
 llvm/lib/TableGen/TGLexer.h                   |   58 +-
 llvm/lib/TableGen/TGParser.cpp                |   48 +-
 llvm/lib/TableGen/TGParser.h                  |    6 +-
 llvm/lib/Target/AArch64/AArch64.td            |   12 +-
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp |   14 +-
 llvm/lib/Target/AArch64/AArch64Features.td    |   20 +-
 .../Target/AArch64/AArch64FrameLowering.cpp   |   10 +-
 .../Target/AArch64/AArch64ISelLowering.cpp    | 1488 ++--
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |   89 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   51 +-
 .../AArch64/AArch64LoadStoreOptimizer.cpp     |    4 +-
 .../lib/Target/AArch64/AArch64MCInstLower.cpp |   83 +-
 .../AArch64/AArch64MachineFunctionInfo.h      |   14 +
 .../Target/AArch64/AArch64PerfectShuffle.h    |   42 +
 llvm/lib/Target/AArch64/AArch64Processors.td  |   39 +-
 .../Target/AArch64/AArch64PromoteConstant.cpp |    4 +
 .../lib/Target/AArch64/AArch64SMEInstrInfo.td |    2 +
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   98 +-
 .../Target/AArch64/AArch64SchedNeoverseN2.td  |    2 +-
 .../AArch64/AArch64SelectionDAGInfo.cpp       |   16 +-
 .../Target/AArch64/AArch64TargetMachine.cpp   |   12 +-
 .../lib/Target/AArch64/AArch64TargetMachine.h |    3 +
 .../AArch64/AArch64TargetObjectFile.cpp       |   10 +-
 .../AArch64/AArch64TargetTransformInfo.cpp    |   48 +-
 .../AArch64/AsmParser/AArch64AsmParser.cpp    |  321 +-
 .../AArch64ExternalSymbolizer.cpp             |   20 +-
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   16 +-
 .../MCTargetDesc/AArch64AsmBackend.cpp        |  114 +-
 .../MCTargetDesc/AArch64ELFObjectWriter.cpp   |  241 +-
 .../MCTargetDesc/AArch64InstPrinter.cpp       |   10 +
 .../AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp |  151 +-
 .../AArch64/MCTargetDesc/AArch64MCAsmInfo.h   |  159 +-
 .../MCTargetDesc/AArch64MCCodeEmitter.cpp     |   94 +-
 .../AArch64/MCTargetDesc/AArch64MCExpr.cpp    |    5 +-
 .../AArch64/MCTargetDesc/AArch64MCExpr.h      |  174 -
 .../MCTargetDesc/AArch64MachObjectWriter.cpp  |   40 +-
 .../MCTargetDesc/AArch64TargetStreamer.h      |    2 +-
 .../AArch64WinCOFFObjectWriter.cpp            |   30 +-
 llvm/lib/Target/AMDGPU/AMDGPU.h               |   11 +
 llvm/lib/Target/AMDGPU/AMDGPU.td              |   30 +-
 .../AMDGPU/AMDGPUExpandFeaturePredicates.cpp  |   48 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |  118 +-
 .../Target/AMDGPU/AMDGPUInsertDelayAlu.cpp    |   19 +-
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp     |   47 +-
 llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def |    3 +
 .../AMDGPU/AMDGPURegBankLegalizeHelper.cpp    |    9 +-
 .../AMDGPU/AMDGPURegBankLegalizeHelper.h      |    6 +-
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |   42 +
 .../AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp      |  304 +
 .../Target/AMDGPU/AMDGPUSearchableTables.td   |    6 +
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h      |    1 +
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |   11 +-
 .../AMDGPU/AMDGPUTargetTransformInfo.cpp      |   59 +-
 .../Target/AMDGPU/AMDGPUTargetTransformInfo.h |   15 +-
 .../AMDGPU/AsmParser/AMDGPUAsmParser.cpp      |   54 +-
 llvm/lib/Target/AMDGPU/BUFInstructions.td     |   53 +-
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |    1 +
 llvm/lib/Target/AMDGPU/DSDIRInstructions.td   |    4 +-
 llvm/lib/Target/AMDGPU/DSInstructions.td      |   76 +-
 .../Disassembler/AMDGPUDisassembler.cpp       |   15 +-
 llvm/lib/Target/AMDGPU/EXPInstructions.td     |    1 +
 llvm/lib/Target/AMDGPU/FLATInstructions.td    |  179 +-
 .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp |    7 +-
 llvm/lib/Target/AMDGPU/GCNRegPressure.cpp     |   63 +
 llvm/lib/Target/AMDGPU/GCNRegPressure.h       |   97 +-
 llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp   |  259 +-
 llvm/lib/Target/AMDGPU/GCNSchedStrategy.h     |    2 -
 llvm/lib/Target/AMDGPU/GCNSubtarget.cpp       |    7 -
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |   24 +-
 .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp  |   20 +-
 .../MCTargetDesc/AMDGPUELFObjectWriter.cpp    |    2 -
 .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp |   10 +-
 .../AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp   |    4 +-
 .../MCTargetDesc/AMDGPUMCCodeEmitter.cpp      |   24 +-
 .../AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp |    2 +-
 llvm/lib/Target/AMDGPU/MIMGInstructions.td    |   94 +
 llvm/lib/Target/AMDGPU/SIDefines.h            |    6 +-
 llvm/lib/Target/AMDGPU/SIFoldOperands.cpp     |  530 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  159 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.h       |    4 +-
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  185 +-
 llvm/lib/Target/AMDGPU/SIInstrFormats.td      |   12 +-
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |   45 +-
 llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp     |    4 +
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |    4 +
 llvm/lib/Target/AMDGPU/SIRegisterInfo.h       |    4 +
 llvm/lib/Target/AMDGPU/SIRegisterInfo.td      |    4 +
 .../Target/AMDGPU/SIShrinkInstructions.cpp    |    5 +-
 llvm/lib/Target/AMDGPU/SOPInstructions.td     |   14 +
 .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp    |   33 +
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h |   20 +-
 .../Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp |    2 +-
 llvm/lib/Target/AMDGPU/VOP1Instructions.td    |   75 +-
 llvm/lib/Target/AMDGPU/VOP2Instructions.td    |   19 +-
 llvm/lib/Target/AMDGPU/VOP3Instructions.td    |   27 +-
 llvm/lib/Target/AMDGPU/VOPCInstructions.td    |    2 +-
 llvm/lib/Target/AMDGPU/VOPInstructions.td     |   48 +-
 llvm/lib/Target/ARC/ARCMCInstLower.cpp        |    3 +-
 .../ARC/MCTargetDesc/ARCInstPrinter.cpp       |    2 +-
 llvm/lib/Target/ARM/ARMAsmPrinter.cpp         |    2 +-
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp      |    4 +-
 llvm/lib/Target/ARM/ARMConstantIslandPass.cpp |    4 +-
 llvm/lib/Target/ARM/ARMFrameLowering.cpp      |    4 +-
 llvm/lib/Target/ARM/ARMISelLowering.cpp       |  290 +-
 llvm/lib/Target/ARM/ARMISelLowering.h         |    5 +
 llvm/lib/Target/ARM/ARMInstrNEON.td           |   12 +-
 llvm/lib/Target/ARM/ARMInstrVFP.td            |    7 +-
 llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp   |   18 +-
 .../lib/Target/ARM/AsmParser/ARMAsmParser.cpp |   17 +-
 .../Target/ARM/MCTargetDesc/ARMAsmBackend.cpp |  181 +-
 .../Target/ARM/MCTargetDesc/ARMAsmBackend.h   |    5 +-
 .../ARM/MCTargetDesc/ARMELFStreamer.cpp       |    6 +-
 .../Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp  |   12 +-
 .../Target/ARM/MCTargetDesc/ARMMCAsmInfo.h    |    2 +
 .../ARM/MCTargetDesc/ARMMCCodeEmitter.cpp     |  156 +-
 .../ARM/MCTargetDesc/ARMMachObjectWriter.cpp  |    6 +-
 .../MCTargetDesc/ARMWinCOFFObjectWriter.cpp   |   13 +-
 .../lib/Target/AVR/AsmParser/AVRAsmParser.cpp |   10 +-
 .../Target/AVR/MCTargetDesc/AVRAsmBackend.cpp |   26 +-
 .../Target/AVR/MCTargetDesc/AVRAsmBackend.h   |    3 -
 .../AVR/MCTargetDesc/AVRInstPrinter.cpp       |    4 +-
 .../Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp  |    2 +-
 .../AVR/MCTargetDesc/AVRMCCodeEmitter.cpp     |   26 +-
 .../AVR/MCTargetDesc/AVRMCELFStreamer.cpp     |    6 +-
 .../lib/Target/BPF/AsmParser/BPFAsmParser.cpp |    8 +-
 .../Target/BPF/MCTargetDesc/BPFAsmBackend.cpp |   12 +-
 .../BPF/MCTargetDesc/BPFELFObjectWriter.cpp   |    5 +-
 .../BPF/MCTargetDesc/BPFInstPrinter.cpp       |   24 +-
 .../BPF/MCTargetDesc/BPFMCCodeEmitter.cpp     |   13 +-
 llvm/lib/Target/CMakeLists.txt                |    2 +-
 .../Target/CSKY/AsmParser/CSKYAsmParser.cpp   |   52 +-
 llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp       |   25 +-
 llvm/lib/Target/CSKY/CSKYISelLowering.cpp     |    3 -
 llvm/lib/Target/CSKY/CSKYMCInstLower.cpp      |   22 +-
 .../Target/CSKY/MCTargetDesc/CMakeLists.txt   |    1 -
 .../CSKY/MCTargetDesc/CSKYAsmBackend.cpp      |   47 +-
 .../Target/CSKY/MCTargetDesc/CSKYAsmBackend.h |    5 +-
 .../CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp |   59 +-
 .../CSKY/MCTargetDesc/CSKYInstPrinter.cpp     |    8 +-
 .../CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp       |   53 +-
 .../Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h  |   28 +
 .../CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp   |  225 +-
 .../CSKY/MCTargetDesc/CSKYMCCodeEmitter.h     |  188 -
 .../Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp   |   76 -
 .../lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h |   75 -
 .../CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp    |    1 -
 .../CSKY/MCTargetDesc/CSKYTargetStreamer.cpp  |    9 +-
 .../CSKY/MCTargetDesc/CSKYTargetStreamer.h    |    8 +-
 llvm/lib/Target/DirectX/DXILCBufferAccess.cpp |  270 +-
 .../Target/DirectX/DXILDataScalarization.cpp  |   19 +-
 .../Target/DirectX/DXILIntrinsicExpansion.cpp |  132 +-
 llvm/lib/Target/DirectX/DXILPrepare.cpp       |   11 +
 llvm/lib/Target/DirectX/DXILRootSignature.cpp |  231 +-
 llvm/lib/Target/DirectX/DXILRootSignature.h   |    1 +
 .../DirectX/DirectXTargetTransformInfo.cpp    |    2 +
 .../Hexagon/AsmParser/HexagonAsmParser.cpp    |    4 +-
 llvm/lib/Target/Hexagon/HexagonBlockRanges.h  |    2 +-
 .../Hexagon/HexagonConstPropagation.cpp       |    3 +-
 .../Target/Hexagon/HexagonFrameLowering.cpp   |    5 +-
 llvm/lib/Target/Hexagon/HexagonGenExtract.cpp |    3 -
 .../Target/Hexagon/HexagonISelLowering.cpp    |   19 +-
 llvm/lib/Target/Hexagon/HexagonISelLowering.h |    5 +
 .../Target/Hexagon/HexagonISelLoweringHVX.cpp |   17 +-
 .../lib/Target/Hexagon/HexagonOptAddrMode.cpp |    7 +-
 .../Hexagon/HexagonSelectionDAGInfo.cpp       |   10 +-
 .../MCTargetDesc/HexagonAsmBackend.cpp        |  305 +-
 .../MCTargetDesc/HexagonELFObjectWriter.cpp   |    2 -
 .../Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp |    4 +-
 .../MCTargetDesc/HexagonMCCodeEmitter.cpp     |   36 +-
 .../Hexagon/MCTargetDesc/HexagonMCExpr.h      |    5 -
 .../Target/Lanai/AsmParser/LanaiAsmParser.cpp |   17 +-
 llvm/lib/Target/Lanai/LanaiISelLowering.cpp   |   10 +-
 .../Lanai/MCTargetDesc/LanaiAsmBackend.cpp    |   10 +-
 .../Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp |    3 +-
 .../AsmParser/LoongArchAsmParser.cpp          |    5 +-
 .../LoongArch/LoongArchFloat64InstrInfo.td    |   16 +
 .../LoongArch/LoongArchISelLowering.cpp       |  314 +-
 .../Target/LoongArch/LoongArchISelLowering.h  |    4 +
 .../Target/LoongArch/LoongArchInstrInfo.td    |   12 +
 .../Target/LoongArch/LoongArchMCInstLower.cpp |    2 +-
 .../LoongArch/MCTargetDesc/CMakeLists.txt     |    1 -
 .../MCTargetDesc/LoongArchAsmBackend.cpp      |   64 +-
 .../MCTargetDesc/LoongArchAsmBackend.h        |    9 +-
 .../MCTargetDesc/LoongArchMCAsmInfo.cpp       |  179 +-
 .../MCTargetDesc/LoongArchMCAsmInfo.h         |   21 +-
 .../MCTargetDesc/LoongArchMCCodeEmitter.cpp   |   24 +-
 .../MCTargetDesc/LoongArchMCExpr.cpp          |  199 -
 .../LoongArch/MCTargetDesc/LoongArchMCExpr.h  |   43 -
 .../Target/M68k/AsmParser/M68kAsmParser.cpp   |    5 +-
 llvm/lib/Target/M68k/M68kInstrInfo.cpp        |    4 +-
 .../M68k/MCTargetDesc/M68kAsmBackend.cpp      |   85 +-
 .../M68k/MCTargetDesc/M68kELFObjectWriter.cpp |    3 -
 .../Target/M68k/MCTargetDesc/M68kFixupKinds.h |   21 -
 .../M68k/MCTargetDesc/M68kMCAsmInfo.cpp       |    4 +-
 .../M68k/MCTargetDesc/M68kMCCodeEmitter.cpp   |   14 +-
 .../MSP430/AsmParser/MSP430AsmParser.cpp      |   10 +-
 .../MSP430/MCTargetDesc/MSP430AsmBackend.cpp  |   16 +-
 .../MCTargetDesc/MSP430MCCodeEmitter.cpp      |   23 +-
 llvm/lib/Target/MSP430/MSP430ISelLowering.cpp |   77 +-
 .../Target/Mips/AsmParser/MipsAsmParser.cpp   |   12 +-
 .../Mips/MCTargetDesc/MipsAsmBackend.cpp      |  218 +-
 .../Target/Mips/MCTargetDesc/MipsAsmBackend.h |    3 -
 .../Mips/MCTargetDesc/MipsMCCodeEmitter.cpp   |   75 +-
 .../Mips/MCTargetDesc/MipsMCTargetDesc.cpp    |    2 +-
 .../Mips/MCTargetDesc/MipsTargetStreamer.cpp  |   24 +-
 llvm/lib/Target/Mips/Mips16ISelLowering.cpp   |   89 +-
 llvm/lib/Target/Mips/MipsISelLowering.cpp     |    3 -
 .../NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp   |   22 +
 .../NVPTX/MCTargetDesc/NVPTXInstPrinter.h     |    2 +
 llvm/lib/Target/NVPTX/NVPTX.h                 |    1 +
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp   |  279 +-
 llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h     |    3 +-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp   |  808 ++-
 llvm/lib/Target/NVPTX/NVPTXISelLowering.h     |   53 +-
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp      |   20 +-
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.h        |    3 -
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td       |  381 +-
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      |    2 +
 .../Target/NVPTX/NVPTXPrologEpilogPass.cpp    |   84 +-
 llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp   |   15 +-
 llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp  |    1 +
 .../Target/PowerPC/AsmParser/PPCAsmParser.cpp |   13 +-
 .../PowerPC/MCTargetDesc/CMakeLists.txt       |    1 -
 .../PowerPC/MCTargetDesc/PPCAsmBackend.cpp    |  163 +-
 .../MCTargetDesc/PPCELFObjectWriter.cpp       |    2 +-
 .../PowerPC/MCTargetDesc/PPCELFStreamer.cpp   |    6 +-
 .../PowerPC/MCTargetDesc/PPCInstPrinter.cpp   |    2 +-
 .../PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp     |   86 +-
 .../PowerPC/MCTargetDesc/PPCMCAsmInfo.h       |   11 +
 .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp |   80 +-
 .../Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp |   77 -
 .../Target/PowerPC/MCTargetDesc/PPCMCExpr.h   |   35 -
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp  |    8 +-
 .../PowerPC/MCTargetDesc/PPCTargetStreamer.h  |    2 +-
 .../MCTargetDesc/PPCXCOFFObjectWriter.cpp     |    2 +-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |   29 +-
 llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp   |    1 -
 llvm/lib/Target/PowerPC/PPCFrameLowering.cpp  |   20 +-
 llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp   |   15 +
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |   83 +-
 llvm/lib/Target/PowerPC/PPCInstrAltivec.td    |   12 +-
 llvm/lib/Target/PowerPC/PPCMCInstLower.cpp    |    5 +-
 .../Target/PowerPC/PPCReduceCRLogicals.cpp    |   33 +-
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |   33 +-
 .../RISCV/Disassembler/RISCVDisassembler.cpp  |   14 +-
 .../RISCV/MCTargetDesc/RISCVAsmBackend.cpp    |   79 +-
 .../RISCV/MCTargetDesc/RISCVAsmBackend.h      |    8 +-
 .../Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |    3 +
 .../RISCV/MCTargetDesc/RISCVFixupKinds.h      |   20 +-
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp |   44 +-
 llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp     |   27 +-
 .../Target/RISCV/RISCVExpandPseudoInsts.cpp   |   20 +-
 llvm/lib/Target/RISCV/RISCVFeatures.td        |   29 +-
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp   |  273 +-
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h     |    2 +-
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  238 +-
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp  |    5 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  488 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.h        |   21 +-
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |   15 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoC.td      |   10 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoP.td      |  163 +
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |   67 +-
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |    2 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |    6 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td |  152 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |    9 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td  |   38 +
 llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td    |  112 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td |   38 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td   |   78 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td  |    4 +-
 llvm/lib/Target/RISCV/RISCVInstrPredicates.td |    2 +
 llvm/lib/Target/RISCV/RISCVLateBranchOpt.cpp  |    2 +-
 llvm/lib/Target/RISCV/RISCVProcessors.td      |    3 +-
 .../RISCV/RISCVRedundantCopyElimination.cpp   |   10 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |   12 +-
 llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp    |   20 +
 llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp |  155 +-
 .../SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp   |   18 +-
 llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp       |   38 +-
 llvm/lib/Target/SPIRV/SPIRVBuiltins.td        |    1 +
 llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp    |   11 +-
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp |    3 +-
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |   34 +
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h   |   15 +-
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |   62 +-
 llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp |   28 +-
 .../lib/Target/SPIRV/SPIRVSymbolicOperands.td |    5 +-
 .../Target/Sparc/AsmParser/SparcAsmParser.cpp |    9 +-
 .../Sparc/MCTargetDesc/SparcAsmBackend.cpp    |   28 +-
 .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp |   38 +-
 llvm/lib/Target/Sparc/SparcISelLowering.cpp   |  116 +-
 .../SystemZ/AsmParser/SystemZAsmParser.cpp    |   35 +-
 .../MCTargetDesc/SystemZHLASMAsmStreamer.cpp  |    7 +-
 .../MCTargetDesc/SystemZHLASMAsmStreamer.h    |   10 +-
 .../MCTargetDesc/SystemZInstPrinterCommon.cpp |    4 +-
 .../MCTargetDesc/SystemZMCAsmBackend.cpp      |   14 +-
 .../SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp |    6 +-
 .../MCTargetDesc/SystemZMCCodeEmitter.cpp     |   15 +-
 .../SystemZ/MCTargetDesc/SystemZMCFixups.h    |   40 +-
 .../MCTargetDesc/SystemZMCTargetDesc.cpp      |    2 +-
 .../MCTargetDesc/SystemZTargetStreamer.cpp    |    7 +
 .../MCTargetDesc/SystemZTargetStreamer.h      |    4 +
 llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp |   10 +-
 .../Target/SystemZ/SystemZISelLowering.cpp    |   13 +-
 llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp  |   34 +-
 .../lib/Target/VE/MCTargetDesc/CMakeLists.txt |    1 -
 .../Target/VE/MCTargetDesc/VEAsmBackend.cpp   |  106 +-
 .../VE/MCTargetDesc/VEELFObjectWriter.cpp     |    6 +-
 .../Target/VE/MCTargetDesc/VEMCAsmInfo.cpp    |   42 +-
 llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h |   25 +
 .../VE/MCTargetDesc/VEMCCodeEmitter.cpp       |   19 +-
 llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp  |   60 -
 llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h    |   50 -
 llvm/lib/Target/VE/VEAsmPrinter.cpp           |    2 +-
 llvm/lib/Target/VE/VEISelLowering.cpp         |   15 +-
 llvm/lib/Target/VE/VEMCInstLower.cpp          |    2 +-
 .../AsmParser/WebAssemblyAsmParser.cpp        |    4 +-
 .../AsmParser/WebAssemblyAsmTypeCheck.cpp     |    2 +-
 .../Disassembler/WebAssemblyDisassembler.cpp  |    2 +-
 .../MCTargetDesc/WebAssemblyAsmBackend.cpp    |   10 +-
 .../MCTargetDesc/WebAssemblyInstPrinter.cpp   |    2 +-
 .../MCTargetDesc/WebAssemblyMCAsmInfo.cpp     |   12 +-
 .../MCTargetDesc/WebAssemblyMCAsmInfo.h       |   12 +
 .../MCTargetDesc/WebAssemblyMCCodeEmitter.cpp |   14 +-
 .../MCTargetDesc/WebAssemblyMCExpr.h          |   29 -
 .../MCTargetDesc/WebAssemblyMCTargetDesc.cpp  |   29 -
 .../MCTargetDesc/WebAssemblyMCTargetDesc.h    |    7 -
 .../WebAssemblyTargetStreamer.cpp             |    5 +-
 .../WebAssemblyWasmObjectWriter.cpp           |    2 +-
 llvm/lib/Target/WebAssembly/WebAssembly.h     |    2 +-
 .../WebAssembly/WebAssemblyAsmPrinter.cpp     |   15 +-
 .../WebAssembly/WebAssemblyAsmPrinter.h       |    2 +-
 .../WebAssembly/WebAssemblyCFGStackify.cpp    |    2 +-
 .../WebAssembly/WebAssemblyISelDAGToDAG.cpp   |   14 +-
 .../WebAssembly/WebAssemblyISelLowering.cpp   |  211 +-
 .../WebAssembly/WebAssemblyLateEHPrepare.cpp  |    2 +-
 .../WebAssembly/WebAssemblyMCInstLower.cpp    |    6 +-
 .../WebAssembly/WebAssemblyRegStackify.cpp    |  138 +-
 .../WebAssembly/WebAssemblyRegisterInfo.td    |    4 +-
 .../WebAssembly/WebAssemblyTargetMachine.cpp  |  153 +-
 .../WebAssembly/WebAssemblyTargetMachine.h    |    9 +
 .../lib/Target/X86/AsmParser/X86AsmParser.cpp |    5 +-
 llvm/lib/Target/X86/AsmParser/X86Operand.h    |    3 +-
 .../lib/Target/X86/GISel/X86LegalizerInfo.cpp |   13 +-
 .../Target/X86/GISel/X86RegisterBankInfo.cpp  |    7 +-
 .../Target/X86/MCTargetDesc/CMakeLists.txt    |    1 +
 .../Target/X86/MCTargetDesc/X86AsmBackend.cpp |   74 +-
 .../X86/MCTargetDesc/X86ELFObjectWriter.cpp   |  128 +-
 .../MCTargetDesc/X86EncodingOptimization.cpp  |    6 +-
 .../Target/X86/MCTargetDesc/X86MCAsmInfo.cpp  |   67 +-
 .../Target/X86/MCTargetDesc/X86MCAsmInfo.h    |   38 +
 .../X86/MCTargetDesc/X86MCCodeEmitter.cpp     |  148 +-
 llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h  |   45 -
 .../X86/MCTargetDesc/X86MachObjectWriter.cpp  |   25 +-
 .../MCTargetDesc/X86WinCOFFObjectWriter.cpp   |   22 +-
 llvm/lib/Target/X86/X86.td                    |    7 +-
 llvm/lib/Target/X86/X86ISelLowering.cpp       |  273 +-
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp   |   26 +-
 llvm/lib/Target/X86/X86MCInstLower.cpp        |   77 +-
 llvm/lib/Target/X86/X86PfmCounters.td         |   20 +
 llvm/lib/Target/X86/X86PreTileConfig.cpp      |    4 +-
 llvm/lib/Target/X86/X86RegisterInfo.cpp       |    2 +-
 llvm/lib/Target/X86/X86SchedLunarlakeP.td     | 2409 ++++++
 llvm/lib/Target/X86/X86TargetObjectFile.cpp   |   12 +-
 .../lib/Target/X86/X86TargetTransformInfo.cpp |    9 +-
 .../XCore/MCTargetDesc/XCoreInstPrinter.cpp   |    2 +-
 llvm/lib/Target/XCore/XCoreISelLowering.cpp   |   10 +-
 llvm/lib/Target/XCore/XCoreMCInstLower.cpp    |    4 +-
 .../Target/XCore/XCoreSelectionDAGInfo.cpp    |   13 +-
 .../Xtensa/AsmParser/XtensaAsmParser.cpp      |    6 +-
 .../Xtensa/MCTargetDesc/XtensaAsmBackend.cpp  |   47 +-
 .../MCTargetDesc/XtensaELFObjectWriter.cpp    |    5 +-
 .../Xtensa/MCTargetDesc/XtensaInstPrinter.cpp |    2 +-
 .../Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp   |    1 -
 .../Xtensa/MCTargetDesc/XtensaMCAsmInfo.h     |    9 +
 .../MCTargetDesc/XtensaMCCodeEmitter.cpp      |   39 +-
 .../Target/Xtensa/MCTargetDesc/XtensaMCExpr.h |   33 -
 .../MCTargetDesc/XtensaMCTargetDesc.cpp       |   23 +-
 llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp   |    2 +-
 llvm/lib/Target/Xtensa/XtensaFeatures.td      |   10 +
 llvm/lib/Target/Xtensa/XtensaISelLowering.cpp |  174 +-
 llvm/lib/Target/Xtensa/XtensaISelLowering.h   |    9 +
 llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp    |   27 +
 llvm/lib/Target/Xtensa/XtensaInstrInfo.td     |   31 +-
 llvm/lib/Target/Xtensa/XtensaOperators.td     |   15 +-
 llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp |    5 +-
 llvm/lib/Target/Xtensa/XtensaRegisterInfo.td  |   11 +-
 llvm/lib/Target/Xtensa/XtensaSubtarget.h      |    2 +-
 llvm/lib/TargetParser/AArch64TargetParser.cpp |    9 +-
 llvm/lib/TargetParser/CSKYTargetParser.cpp    |    1 +
 llvm/lib/TargetParser/Host.cpp                |   36 +-
 .../TargetParser/LoongArchTargetParser.cpp    |    1 +
 llvm/lib/TargetParser/TargetParser.cpp        |    1 +
 llvm/lib/TargetParser/Triple.cpp              |   66 +-
 llvm/lib/TargetParser/Unix/Host.inc           |    2 +-
 .../lib/Transforms/Coroutines/CoroCleanup.cpp |   10 +-
 llvm/lib/Transforms/Coroutines/CoroEarly.cpp  |   11 +-
 llvm/lib/Transforms/Coroutines/CoroElide.cpp  |    2 +-
 llvm/lib/Transforms/Coroutines/CoroFrame.cpp  |    5 +-
 llvm/lib/Transforms/Coroutines/CoroInternal.h |    3 +-
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  |   33 +-
 llvm/lib/Transforms/Coroutines/Coroutines.cpp |   95 +-
 llvm/lib/Transforms/Coroutines/SpillUtils.cpp |    8 +-
 llvm/lib/Transforms/HipStdPar/HipStdPar.cpp   |  118 +
 llvm/lib/Transforms/IPO/Attributor.cpp        |    9 +
 .../Transforms/IPO/AttributorAttributes.cpp   |  424 +-
 llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp  |    8 +-
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp     |   20 +-
 llvm/lib/Transforms/IPO/GlobalOpt.cpp         |  173 +-
 llvm/lib/Transforms/IPO/StripSymbols.cpp      |    3 +-
 .../InstCombine/InstCombineAddSub.cpp         |   21 +-
 .../InstCombine/InstCombineAndOrXor.cpp       |    2 +-
 .../InstCombine/InstCombineCalls.cpp          |   78 +-
 .../InstCombine/InstCombineCompares.cpp       |  129 +-
 .../InstCombine/InstCombineInternal.h         |   15 +
 .../InstCombine/InstCombineVectorOps.cpp      |   66 +-
 .../InstCombine/InstructionCombining.cpp      |  158 +-
 .../Instrumentation/LowerAllowCheckPass.cpp   |   73 +-
 .../Instrumentation/MemorySanitizer.cpp       |  138 +-
 .../Instrumentation/PGOInstrumentation.cpp    |   51 +-
 .../Scalar/DeadStoreElimination.cpp           |    6 +-
 llvm/lib/Transforms/Scalar/EarlyCSE.cpp       |   18 +-
 llvm/lib/Transforms/Scalar/LICM.cpp           |    8 +-
 llvm/lib/Transforms/Scalar/LoopDeletion.cpp   |    3 +-
 .../lib/Transforms/Scalar/LoopInterchange.cpp |   43 +-
 .../Transforms/Scalar/LoopStrengthReduce.cpp  |   94 +-
 .../Scalar/LowerMatrixIntrinsics.cpp          |    2 +-
 llvm/lib/Transforms/Scalar/MergeICmps.cpp     |    4 +
 llvm/lib/Transforms/Scalar/NewGVN.cpp         |    7 +-
 llvm/lib/Transforms/Scalar/Reassociate.cpp    |   10 +-
 llvm/lib/Transforms/Scalar/SROA.cpp           |   31 +-
 llvm/lib/Transforms/Utils/BasicBlockUtils.cpp |   23 +-
 llvm/lib/Transforms/Utils/CodeExtractor.cpp   |   91 +-
 llvm/lib/Transforms/Utils/Debugify.cpp        |   88 +-
 llvm/lib/Transforms/Utils/InlineFunction.cpp  |   20 +-
 llvm/lib/Transforms/Utils/Local.cpp           |   18 +-
 llvm/lib/Transforms/Utils/LoopUtils.cpp       |   10 +-
 llvm/lib/Transforms/Utils/PredicateInfo.cpp   |   25 +-
 .../Utils/RelLookupTableConverter.cpp         |   37 +-
 llvm/lib/Transforms/Utils/SCCPSolver.cpp      |  172 +-
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     |   54 +-
 .../Transforms/Vectorize/LoopVectorize.cpp    |  368 +-
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   21 +-
 .../Passes/SeedCollection.cpp                 |   11 +-
 .../SandboxVectorizer/SeedCollector.cpp       |   12 +-
 .../Transforms/Vectorize/VPRecipeBuilder.h    |   15 +-
 llvm/lib/Transforms/Vectorize/VPlan.cpp       |   46 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |  409 +-
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   14 +-
 llvm/lib/Transforms/Vectorize/VPlanHelpers.h  |    6 +
 .../Transforms/Vectorize/VPlanPatternMatch.h  |   20 +
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  627 +-
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  329 +-
 .../Transforms/Vectorize/VPlanTransforms.h    |   10 +-
 llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp |  104 +-
 llvm/lib/Transforms/Vectorize/VPlanUtils.h    |    3 +-
 llvm/lib/Transforms/Vectorize/VPlanValue.h    |    4 +-
 .../Transforms/Vectorize/VPlanVerifier.cpp    |   32 +-
 .../Transforms/Vectorize/VectorCombine.cpp    |  234 +-
 llvm/lib/XRay/FDRTraceWriter.cpp              |    6 +-
 llvm/runtimes/CMakeLists.txt                  |    7 +
 llvm/test/Analysis/BasicAA/cs-cs-arm.ll       |    2 +-
 llvm/test/Analysis/BasicAA/intrinsics-arm.ll  |    2 +-
 llvm/test/Analysis/CostModel/AArch64/ldexp.ll |   98 +
 llvm/test/Analysis/CostModel/AArch64/lrint.ll |  254 +
 .../segmented-shufflevector-patterns.ll       |  101 +
 .../CostModel/AArch64/shuffle-other.ll        |   10 +-
 .../Analysis/CostModel/AArch64/sve-ldexp.ll   |   58 +
 .../Analysis/CostModel/AArch64/sve-lrint.ll   |  218 +
 llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll  |  100 +-
 llvm/test/Analysis/CostModel/AMDGPU/load.ll   |   66 +-
 .../CostModel/AMDGPU/shufflevector.ll         |  448 +-
 llvm/test/Analysis/CostModel/AMDGPU/store.ll  |   66 +-
 llvm/test/Analysis/CostModel/ARM/shuffle.ll   |    4 +-
 llvm/test/Analysis/CostModel/RISCV/exp.ll     |  109 +
 llvm/test/Analysis/CostModel/RISCV/fround.ll  |  194 +
 .../CostModel/RISCV/shuffle-exact-vlen.ll     |   24 +-
 .../RISCV/shuffle-insert_subvector.ll         |  120 +-
 .../CostModel/RISCV/shuffle-select.ll         |    8 +-
 .../CostModel/X86/alternate-shuffle-cost.ll   |   65 +-
 .../CostModel/X86/shuffle-insert_subvector.ll |  164 +-
 .../Analysis/CostModel/X86/shuffle-select.ll  |  113 +-
 .../HashRecognize/cyclic-redundancy-check.ll  |  240 +-
 .../IR2Vec/Inputs/dummy_2D_vocab.json         |   11 +
 .../IR2Vec/Inputs/dummy_3D_vocab.json         |   18 +-
 .../IR2Vec/Inputs/dummy_5D_vocab.json         |   24 +-
 .../IR2Vec/Inputs/incorrect_vocab1.json       |   11 +
 .../IR2Vec/Inputs/incorrect_vocab2.json       |   12 +
 .../IR2Vec/Inputs/incorrect_vocab3.json       |   12 +
 .../IR2Vec/Inputs/incorrect_vocab4.json       |   16 +
 llvm/test/Analysis/IR2Vec/basic.ll            |   14 +-
 llvm/test/Analysis/IR2Vec/dbg-inst.ll         |   13 +
 llvm/test/Analysis/IR2Vec/unreachable.ll      |   42 +
 llvm/test/Analysis/IR2Vec/vocab-test.ll       |   20 +
 .../dependences-i128-inductions.ll            |  151 +
 .../early-exit-runtime-checks.ll              |  120 +
 ...untime-checks-after-dependence-analysis.ll |   34 +-
 .../UniformityAnalysis/AMDGPU/intrinsics.ll   |   72 +
 .../aarch64-intrinsics-attributes.ll          |   10 +
 llvm/test/Bitcode/attributes.ll               |    5 +
 llvm/test/Bitcode/debug-loc-again.ll          |    4 +-
 .../Bitcode/function-encoding-rel-operands.ll |    2 +-
 .../AArch64/GlobalISel/inline-memcpy.mir      |    2 +-
 .../AArch64/GlobalISel/inline-memmove.mir     |    2 +-
 .../AArch64/GlobalISel/inline-memset.mir      |    2 +-
 ...legalizer-combiner-use-vector-truncate.mir |   52 +
 .../AArch64/PHIElimination-reuse-copy.mir     |  197 +
 .../AArch64/aarch64-bitwisenot-fold.ll        |   48 +
 llvm/test/CodeGen/AArch64/abd-combine.ll      |   53 +-
 .../CodeGen/AArch64/arm64-bitfield-extract.ll |   40 +-
 .../CodeGen/AArch64/arm64-fp-contract-zero.ll |   10 +-
 .../AArch64/arm64-indexed-vector-ldst.ll      |  115 +
 .../AArch64/arm64-zero-cycle-regmov-fpr.ll    |  103 +
 .../AArch64/arm64-zero-cycle-regmov-gpr.ll    |   54 +
 .../AArch64/arm64-zero-cycle-regmov-gpr32.ll  |   45 -
 llvm/test/CodeGen/AArch64/arm64-zip.ll        |    2 +-
 llvm/test/CodeGen/AArch64/arm64ec-builtins.ll |    9 +-
 llvm/test/CodeGen/AArch64/bf16_fast_math.ll   |   36 +-
 llvm/test/CodeGen/AArch64/cmp-select-sign.ll  |    6 +-
 llvm/test/CodeGen/AArch64/concatbinop.ll      |    2 +-
 llvm/test/CodeGen/AArch64/cpa-selectiondag.ll |  107 +-
 llvm/test/CodeGen/AArch64/double_reduct.ll    |  149 +-
 .../CodeGen/AArch64/exp10-libcall-names.ll    |   16 +-
 .../test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll |    1 +
 llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll     |    1 +
 llvm/test/CodeGen/AArch64/itofp.ll            | 1506 ++--
 llvm/test/CodeGen/AArch64/llvm.frexp.ll       | 1345 ++++
 .../AArch64/lr-reserved-for-ra-live-in.ll     |    2 +-
 llvm/test/CodeGen/AArch64/machine-combiner.ll |    8 +-
 .../CodeGen/AArch64/machine-outliner-loh.ll   |   74 +
 .../CodeGen/AArch64/machine-outliner-loh.mir  |   50 +
 llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll |    2 +
 llvm/test/CodeGen/AArch64/memsize-remarks.ll  |    4 +-
 .../CodeGen/AArch64/merge-scoped-aa-store.ll  |   29 +-
 .../neon-partial-reduce-dot-product.ll        |    1 -
 .../no-promote-scalabale-const-to-global.ll   |   20 +
 llvm/test/CodeGen/AArch64/popcount.ll         |  302 +-
 llvm/test/CodeGen/AArch64/pr58431.ll          |    8 +-
 llvm/test/CodeGen/AArch64/ptradd.ll           |   82 +-
 .../AArch64/ptrauth-arm64-tls-dynamics.ll     |    2 +-
 llvm/test/CodeGen/AArch64/rem-by-const.ll     | 3616 +++++++++
 llvm/test/CodeGen/AArch64/sat-add.ll          |    8 +-
 llvm/test/CodeGen/AArch64/select_cc.ll        |    2 +-
 .../CodeGen/AArch64/selectcc-to-shiftand.ll   |    3 -
 .../AArch64/sme-must-save-lr-for-vg.ll        |   49 +
 .../CodeGen/AArch64/sme2-intrinsics-fclamp.ll |   85 +
 .../CodeGen/AArch64/sme2-intrinsics-qcvtn.ll  |   40 +-
 .../CodeGen/AArch64/sme2-intrinsics-sclamp.ll |  110 +
 .../CodeGen/AArch64/sme2-intrinsics-uclamp.ll |  110 +
 .../CodeGen/AArch64/stp-opt-with-renaming.mir |   29 +
 .../CodeGen/AArch64/streaming-func-no-sme.ll  |    7 +
 llvm/test/CodeGen/AArch64/sve-abd.ll          |    3 +-
 .../AArch64/sve-fixed-length-bit-counting.ll  |   48 +-
 .../CodeGen/AArch64/sve-indexed-arithmetic.ll |  366 +
 .../CodeGen/AArch64/sve-intrinsics-fexpa.ll   |    4 +-
 .../test/CodeGen/AArch64/sve-merging-unary.ll | 1748 +++++
 llvm/test/CodeGen/AArch64/sve-vscale.ll       |   37 +-
 ...ve2-intrinsics-binary-narrowing-add-sub.ll |    1 +
 .../sve2-intrinsics-binary-narrowing-shr.ll   |    1 +
 .../sve2-intrinsics-bit-permutation.ll        |    5 +-
 .../AArch64/sve2-intrinsics-complex-dot.ll    |    1 +
 ...ntrinsics-contiguous-conflict-detection.ll |    1 +
 .../CodeGen/AArch64/sve2-intrinsics-crypto.ll |   52 +-
 .../AArch64/sve2-intrinsics-faminmax.ll       |    1 +
 .../AArch64/sve2-intrinsics-fp-converts.ll    |    1 +
 ...sve2-intrinsics-fp-int-binary-logarithm.ll |    1 +
 .../sve2-intrinsics-fp-widening-mul-acc.ll    |    1 +
 .../AArch64/sve2-intrinsics-int-mul-lane.ll   |    1 +
 .../CodeGen/AArch64/sve2-intrinsics-luti.ll   |    4 +-
 ...-intrinsics-non-widening-pairwise-arith.ll |    1 +
 .../sve2-intrinsics-polynomial-arithmetic.ll  |    1 +
 .../CodeGen/AArch64/sve2-intrinsics-psel.ll   |    2 +
 .../CodeGen/AArch64/sve2-intrinsics-rax1.ll   |   20 +
 .../CodeGen/AArch64/sve2-intrinsics-revd.ll   |    2 +
 .../CodeGen/AArch64/sve2-intrinsics-sm4.ll    |   33 +
 .../sve2-intrinsics-unary-narrowing.ll        |    1 +
 .../sve2-intrinsics-uniform-complex-arith.ll  |    1 +
 .../AArch64/sve2-intrinsics-while-reversed.ll |    1 +
 .../CodeGen/AArch64/sve2-intrinsics-while.ll  |    1 +
 ...2-intrinsics-widening-complex-int-arith.ll |    1 +
 .../AArch64/sve2-intrinsics-widening-dsp.ll   |    1 +
 ...sve2-intrinsics-widening-pairwise-arith.ll |    1 +
 .../AArch64/sve2p1-intrinsics-bfmlsl.ll       |    5 +-
 .../CodeGen/AArch64/sve2p1-intrinsics-cntp.ll |    1 +
 .../CodeGen/AArch64/sve2p1-intrinsics-dots.ll |    2 +
 .../CodeGen/AArch64/sve2p1-intrinsics-dupq.ll |    5 +-
 .../CodeGen/AArch64/sve2p1-intrinsics-extq.ll |    5 +-
 .../AArch64/sve2p1-intrinsics-fclamp.ll       |   96 +-
 .../AArch64/sve2p1-intrinsics-fp-reduce.ll    |    2 +
 .../AArch64/sve2p1-intrinsics-int-reduce.ll   |    2 +
 .../AArch64/sve2p1-intrinsics-loads.ll        |    5 +-
 .../sve2p1-intrinsics-multivec-loads.ll       |    5 +-
 .../sve2p1-intrinsics-multivec-stores.ll      |    5 +-
 .../AArch64/sve2p1-intrinsics-pmov-to-pred.ll |    3 +
 .../sve2p1-intrinsics-pmov-to-vector.ll       |    3 +
 .../sve2p1-intrinsics-predicate-as-counter.ll |    1 +
 .../AArch64/sve2p1-intrinsics-qcvtn.ll        |   57 +
 .../AArch64/sve2p1-intrinsics-qrshr.ll        |    3 +
 .../AArch64/sve2p1-intrinsics-sclamp.ll       |  122 +-
 .../AArch64/sve2p1-intrinsics-stores.ll       |    5 +-
 .../AArch64/sve2p1-intrinsics-uclamp.ll       |  122 +-
 .../AArch64/sve2p1-intrinsics-while-pn.ll     |    2 +-
 .../AArch64/sve2p1-intrinsics-while-pp.ll     |    3 +
 .../CodeGen/AArch64/sve2p1-vector-shuffles.ll |   63 +-
 .../test/CodeGen/AArch64/sve2p2-intrinsics.ll |   45 +
 .../AArch64/switch-cases-to-branch-and.ll     |    3 +-
 llvm/test/CodeGen/AArch64/tbl-loops.ll        |   12 +-
 .../AArch64/vecreduce-propagate-sd-flags.ll   |    2 +-
 .../test/CodeGen/AArch64/vselect-constants.ll |    4 +-
 llvm/test/CodeGen/AArch64/vselect-ext.ll      |  140 +-
 .../CodeGen/AArch64/zeroing-forms-abs-neg.ll  |    2 +
 .../AArch64/zeroing-forms-counts-not.ll       |    2 +
 .../test/CodeGen/AArch64/zeroing-forms-ext.ll |    2 +
 .../AArch64/zeroing-forms-fcvt-bfcvt.ll       |    2 +
 .../AArch64/zeroing-forms-fcvtlt-fcvtx.ll     |    2 +
 .../CodeGen/AArch64/zeroing-forms-fcvtzsu.ll  |    2 +
 .../CodeGen/AArch64/zeroing-forms-flogb.ll    |    2 +
 .../zeroing-forms-frint-frecpx-fsqrt.ll       |    2 +
 .../test/CodeGen/AArch64/zeroing-forms-rev.ll |    2 +
 ...eroing-forms-urecpe-ursqrte-sqabs-sqneg.ll |    6 +-
 .../CodeGen/AArch64/zeroing-forms-uscvtf.ll   |    2 +
 .../AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll  |    2 +
 .../AMDGPU/GlobalISel/global-value.illegal.ll |    2 +-
 .../GlobalISel/inst-select-ashr.s16.mir       |   10 +-
 .../GlobalISel/inst-select-lshr.s16.mir       |   10 +-
 .../inst-select-pattern-smed3.s16.mir         |   42 +-
 .../inst-select-pattern-umed3.s16.mir         |   42 +-
 .../AMDGPU/GlobalISel/inst-select-shl.s16.mir |   50 +-
 ...-select-stacksave-stackrestore.invalid.mir |    2 +-
 .../GlobalISel/legalize-atomicrmw-nand.mir    |    2 +-
 .../legalize-atomicrmw-xchg-flat.mir          |    2 +-
 .../AMDGPU/GlobalISel/legalize-jump-table.mir |    2 +-
 .../llvm.amdgcn.image.gather4.o.dim.ll        |    2 +-
 .../GlobalISel/llvm.amdgcn.intersect_ray.ll   |    2 +-
 ....amdgcn.raw.buffer.atomic.fadd-with-ret.ll |    2 +-
 ...gcn.raw.ptr.buffer.atomic.fadd-with-ret.ll |    2 +-
 ...dgcn.struct.buffer.atomic.fadd-with-ret.ll |    2 +-
 ....struct.ptr.buffer.atomic.fadd-with-ret.ll |    2 +-
 llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll    |   18 +-
 .../GlobalISel/regbankselect-illegal-copy.mir |    4 +-
 .../AMDGPU/GlobalISel/unsupported-load.ll     |    2 +-
 .../AMDGPU/GlobalISel/unsupported-ptr-add.ll  |    2 +-
 .../CodeGen/AMDGPU/GlobalISel/urem.i32.ll     |  104 +-
 .../CodeGen/AMDGPU/GlobalISel/urem.i64.ll     |  595 +-
 llvm/test/CodeGen/AMDGPU/add_i1.ll            |  225 +-
 llvm/test/CodeGen/AMDGPU/add_i128.ll          |   99 +-
 .../AMDGPU/alloc-aligned-tuples-gfx1250.mir   |  148 +
 .../amdgpu-expand-feature-predicates.ll       |  169 +-
 .../CodeGen/AMDGPU/amdgpu-snop-padding.mir    |  132 +
 llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll     |  180 +-
 .../AMDGPU/atomic_optimizations_buffer.ll     |   13 +
 .../atomic_optimizations_global_pointer.ll    |   18 +-
 .../atomic_optimizations_local_pointer.ll     |    3 +
 .../AMDGPU/atomic_optimizations_raw_buffer.ll |   11 +
 .../atomic_optimizations_struct_buffer.ll     |   11 +
 llvm/test/CodeGen/AMDGPU/bf16-conversions.ll  |    4 +-
 .../buffer-fat-pointer-atomicrmw-fadd.ll      |   13 +
 .../buffer-fat-pointer-atomicrmw-fmax.ll      |   18 +
 .../buffer-fat-pointer-atomicrmw-fmin.ll      |   18 +
 llvm/test/CodeGen/AMDGPU/cmp_shrink.mir       |   13 +
 .../CodeGen/AMDGPU/combine-add-zext-xor.ll    |   20 +-
 .../CodeGen/AMDGPU/combine-and-sext-bool.ll   |  132 +-
 .../CodeGen/AMDGPU/convergence-laneops.ll     |    2 +-
 llvm/test/CodeGen/AMDGPU/div_i128.ll          |   64 +-
 .../CodeGen/AMDGPU/divergence-driven-ctpop.ll |    2 +-
 .../test/CodeGen/AMDGPU/dynamic_stackalloc.ll |    2 -
 .../CodeGen/AMDGPU/extract-subvector-16bit.ll |    5 -
 llvm/test/CodeGen/AMDGPU/extract-subvector.ll |   10 -
 llvm/test/CodeGen/AMDGPU/fdot2.ll             |  237 +-
 .../AMDGPU/fix-sgpr-copies-f16-true16.mir     |   26 +
 .../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll     |   36 +-
 .../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll     |   58 +-
 .../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll     |   58 +-
 .../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll     |   40 +-
 llvm/test/CodeGen/AMDGPU/fma-combine.ll       |  342 +-
 llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll  |   18 +-
 llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll  |   18 +-
 .../AMDGPU/fmul-2-combine-multi-use.ll        |   10 +-
 llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll |    6 +-
 llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll |   95 +-
 .../fneg-fabs-divergence-driven-isel.ll       |    6 +-
 llvm/test/CodeGen/AMDGPU/fneg.ll              |    4 +-
 llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll   |    1 +
 .../AMDGPU/fp-min-max-image-atomics.ll        |    2 +-
 llvm/test/CodeGen/AMDGPU/fract-match.ll       |   13 +-
 .../CodeGen/AMDGPU/global-atomicrmw-fadd.ll   |  128 +-
 .../CodeGen/AMDGPU/global-atomicrmw-fmax.ll   |   58 +-
 .../CodeGen/AMDGPU/global-atomicrmw-fmin.ll   |   58 +-
 .../CodeGen/AMDGPU/global-atomicrmw-fsub.ll   |   40 +-
 llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll  |  605 ++
 .../AMDGPU/global_atomics_scan_fadd.ll        | 5090 +++++--------
 .../AMDGPU/global_atomics_scan_fmax.ll        | 4988 ++++---------
 .../AMDGPU/global_atomics_scan_fmin.ll        | 4988 ++++---------
 .../AMDGPU/global_atomics_scan_fsub.ll        | 5104 +++++--------
 llvm/test/CodeGen/AMDGPU/icmp.i16.ll          | 1640 ++++-
 .../CodeGen/AMDGPU/indirect-addressing-si.ll  |    7 +-
 ...class-vgpr-mfma-to-agpr-negative-tests.mir |  313 +
 ...class-vgpr-mfma-to-av-with-load-source.mir |  358 +
 .../inflate-reg-class-vgpr-mfma-to-av.mir     |  190 +
 .../insert_waitcnt_for_precise_memory.ll      |    1 +
 .../CodeGen/AMDGPU/integer-mad-patterns.ll    |   33 +-
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |    4 +
 .../AMDGPU/llvm.amdgcn.buffer.load.lds.err.ll |    8 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll |  278 +
 .../AMDGPU/llvm.amdgcn.cvt.f32.fp8.err.ll     |   18 +-
 ...dgcn.ds.atomic.async.barrier.arrive.b64.ll |   21 +
 ...amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll |   27 +
 .../llvm.amdgcn.ds.gws.sema.release.all.ll    |    2 +-
 .../llvm.amdgcn.ds.ordered.add-unsupported.ll |   22 +
 .../AMDGPU/llvm.amdgcn.dual_intersect_ray.ll  |    2 +-
 .../AMDGPU/llvm.amdgcn.global.load.lds.err.ll |    4 +-
 .../llvm.amdgcn.global.load.lds.gfx950.ll     |    2 +-
 ...llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll |   10 +-
 .../AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll    |  228 +-
 .../AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll |   12 +-
 .../AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll |    3 +-
 .../AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll  |    2 +-
 .../AMDGPU/llvm.amdgcn.load.to.lds.gfx950.ll  |    2 +-
 .../AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll |  322 +
 .../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll |   41 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.permlane.ll    |   80 +-
 .../AMDGPU/llvm.amdgcn.permlane.ptr.ll        |   16 +
 .../AMDGPU/llvm.amdgcn.permlane16.swap.ll     |    2 +-
 .../AMDGPU/llvm.amdgcn.permlane32.swap.ll     |    2 +-
 ...mdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll |    8 +-
 ...mdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll |    1 +
 ...m.amdgcn.raw.ptr.buffer.load.lds.gfx950.ll |    2 +-
 .../AMDGPU/llvm.amdgcn.s.monitor.sleep.ll     |   20 +
 ....amdgcn.struct.buffer.load.format.v3f16.ll |    2 +
 ...cn.struct.ptr.buffer.atomic.fadd.v2bf16.ll |    2 +
 ...gcn.struct.ptr.buffer.atomic.fadd_nortn.ll |    2 +
 ...mdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll |    2 +
 ...mdgcn.struct.ptr.buffer.atomic.fmax.f32.ll |    2 +
 ...mdgcn.struct.ptr.buffer.atomic.fmin.f32.ll |    2 +
 ...mdgcn.struct.ptr.buffer.load.lds.gfx950.ll |    2 +-
 .../AMDGPU/llvm.amdgcn.tensor.load.store.ll   |  245 +
 .../CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll     |    2 +-
 .../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll   |   14 +
 .../AMDGPU/llvm.amdgcn.writelane.ptr.ll       |   11 +
 .../CodeGen/AMDGPU/llvm.fptrunc.round.err.ll  |    6 +-
 llvm/test/CodeGen/AMDGPU/llvm.mulo.ll         |   13 +-
 .../CodeGen/AMDGPU/local-atomicrmw-fadd.ll    |    4 +-
 .../CodeGen/AMDGPU/local-atomicrmw-fmax.ll    |    2 +
 .../CodeGen/AMDGPU/local-atomicrmw-fmin.ll    |    2 +
 .../CodeGen/AMDGPU/local-atomicrmw-fsub.ll    |    2 +
 .../local-stack-alloc-block-sp-reference.ll   |    2 -
 ...ine-scheduler-sink-trivial-remats-attr.mir |  554 +-
 .../machine-scheduler-sink-trivial-remats.mir |  366 +-
 llvm/test/CodeGen/AMDGPU/mad-combine.ll       |  191 +-
 llvm/test/CodeGen/AMDGPU/mad_64_32.ll         |    8 +-
 llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll   |   18 +-
 .../AMDGPU/memory-legalizer-global-agent.ll   |   26 -
 .../AMDGPU/memory-legalizer-global-system.ll  |   26 -
 llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll   |   18 +-
 llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll |  425 +-
 llvm/test/CodeGen/AMDGPU/packed-op-sel.ll     |    1 -
 .../CodeGen/AMDGPU/prevent-fmul-hoist-ir.ll   |  108 +-
 .../AMDGPU/promote-constOffset-to-imm.ll      |   33 +-
 .../AMDGPU/ptradd-sdag-optimizations.ll       |  374 +-
 .../AMDGPU/ptradd-sdag-undef-poison.ll        |   49 +
 llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll       |    6 +-
 llvm/test/CodeGen/AMDGPU/rem_i128.ll          |   64 +-
 .../AMDGPU/rewrite-vgpr-mfma-to-agpr.ll       |  195 +
 llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll      | 1956 +++++
 llvm/test/CodeGen/AMDGPU/rsq.f32.ll           | 2194 +-----
 .../AMDGPU/select-flags-to-fmin-fmax.ll       |    6 +
 .../CodeGen/AMDGPU/sgpr-regalloc-flags.ll     |    4 +
 .../AMDGPU/shufflevector.v3bf16.v2bf16.ll     |   16 +-
 .../AMDGPU/shufflevector.v3bf16.v3bf16.ll     |   15 +-
 .../AMDGPU/shufflevector.v3f16.v2f16.ll       |   16 +-
 .../AMDGPU/shufflevector.v3f16.v3f16.ll       |   15 +-
 .../AMDGPU/si-fold-operands-subreg-imm.mir    |    4 +-
 .../test/CodeGen/AMDGPU/srl64_reduce_flags.ll |   41 +
 .../AMDGPU/stacksave_stackrestore.invalid.ll  |    4 +-
 .../CodeGen/AMDGPU/store-to-constant-error.ll |    4 +-
 llvm/test/CodeGen/AMDGPU/sub_i1.ll            |  226 +-
 .../AMDGPU/undef-handling-crash-in-ra.ll      |  193 +
 .../CodeGen/AMDGPU/unsupported-cs-chain.ll    |    2 +-
 .../CodeGen/AMDGPU/unsupported-image-a16.ll   |    2 +-
 .../CodeGen/AMDGPU/unsupported-image-g16.ll   |    4 +-
 .../AMDGPU/unsupported-image-sample.ll        |   19 +-
 llvm/test/CodeGen/AMDGPU/valu-i1.ll           |  355 +-
 .../CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll  |  215 +-
 llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll |    7 +
 .../test/CodeGen/AMDGPU/vector-reduce-smax.ll |   23 +-
 .../test/CodeGen/AMDGPU/vector-reduce-smin.ll |   23 +-
 .../test/CodeGen/AMDGPU/vector-reduce-umax.ll |   23 +-
 .../test/CodeGen/AMDGPU/vector-reduce-umin.ll |   23 +-
 .../test/CodeGen/AMDGPU/vni8-across-blocks.ll | 2499 ++-----
 llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll |  279 -
 llvm/test/CodeGen/AMDGPU/wait-xcnt.mir        |  968 +++
 llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir |   20 +
 .../CodeGen/AMDGPU/widen-vselect-and-mask.ll  |   64 +-
 llvm/test/CodeGen/ARM/Windows/frexp.ll        |   28 +
 llvm/test/CodeGen/ARM/cmp-select-sign.ll      |  171 +
 llvm/test/CodeGen/ARM/fp16_fast_math.ll       |   20 +-
 llvm/test/CodeGen/ARM/fpclamptosat.ll         |  357 +-
 llvm/test/CodeGen/ARM/fpclamptosat_vec.ll     |   14 +-
 .../CodeGen/ARM/frexp-soften-libcall-error.ll |    8 +
 llvm/test/CodeGen/ARM/iabs.ll                 |   25 +
 .../ARM/ifcvt_unanalyzable_fallthrough.mir    |  114 +
 llvm/test/CodeGen/ARM/ldexp.ll                |   12 +-
 llvm/test/CodeGen/ARM/llvm.frexp.ll           |   37 +
 .../CodeGen/ARM/minnum-maxnum-intrinsics.ll   |    2 +-
 .../CodeGen/ARM/powi-soften-libcall-error.ll  |    8 +
 llvm/test/CodeGen/ARM/special-reg.ll          |   12 +-
 llvm/test/CodeGen/ARM/usat-with-shift.ll      |   64 +-
 llvm/test/CodeGen/ARM/usat.ll                 |  122 +-
 llvm/test/CodeGen/ARM/vrint.ll                |  336 +-
 llvm/test/CodeGen/ARM/vrintn.ll               |   29 +-
 llvm/test/CodeGen/DirectX/BufferLoadDouble.ll |    2 +-
 llvm/test/CodeGen/DirectX/BufferLoadInt64.ll  |   48 +
 llvm/test/CodeGen/DirectX/BufferStoreInt64.ll |   38 +
 .../CodeGen/DirectX/CBufferAccess/memcpy.ll   |  216 +
 ...gnature-StaticSamplers-Invalid-AddressU.ll |   19 +
 ...gnature-StaticSamplers-Invalid-AddressV.ll |   19 +
 ...gnature-StaticSamplers-Invalid-AddressW.ll |   19 +
 ...ture-StaticSamplers-Invalid-BorderColor.ll |   19 +
 ...e-StaticSamplers-Invalid-ComparisonFunc.ll |   19 +
 ...Signature-StaticSamplers-Invalid-Filter.ll |   19 +
 ...re-StaticSamplers-Invalid-MaxAnisotropy.ll |   19 +
 ...Signature-StaticSamplers-Invalid-MaxLod.ll |   19 +
 ...Signature-StaticSamplers-Invalid-MinLod.ll |   19 +
 ...ature-StaticSamplers-Invalid-MinLopBias.ll |   19 +
 ...re-StaticSamplers-Invalid-RegisterSpace.ll |   19 +
 ...e-StaticSamplers-Invalid-ShaderRegister.ll |   19 +
 ...StaticSamplers-Invalid-ShaderVisibility.ll |   19 +
 .../RootSignature-StaticSamplers.ll           |   42 +
 llvm/test/CodeGen/DirectX/firstbithigh.ll     |   92 +-
 llvm/test/CodeGen/DirectX/firstbitlow.ll      |   18 +-
 .../DirectX/issue-145408-gep-struct-fix.ll    |   17 +
 llvm/test/CodeGen/DirectX/llc-pipeline.ll     |    2 +-
 llvm/test/CodeGen/DirectX/scalarize-alloca.ll |   19 +-
 .../CodeGen/DirectX/strip-rootsignatures.ll   |   18 +
 .../MIRDebugify/locations-and-values.mir      |    2 +-
 .../autohvx/build-vector-float-type.ll        |  619 +-
 llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll   |   57 +
 .../test/CodeGen/Hexagon/vect/vect-vshifts.ll |  197 +-
 .../CodeGen/LoongArch/calling-conv-ilp32d.ll  |  193 +
 .../LoongArch/inline-asm-constraint-f.ll      |   14 +-
 .../ir-instruction/double-convert.ll          |   14 +-
 .../ir-instruction/load-store-atomic.ll       |   40 +-
 .../ir-instruction/bitcast-extract-element.ll |   28 +
 .../lasx/ir-instruction/fix-xvshuf.ll         |   30 +
 .../ir-instruction/insert-bitcast-element.ll  |   28 +
 .../ir-instruction/insert-extract-element.ll  |   98 +
 .../insert-extract-pair-elements.ll           |  132 +
 llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll |    3 +-
 .../ir-instruction/bitcast-extract-element.ll |   26 +
 .../ir-instruction/insert-bitcast-element.ll  |   28 +
 .../ir-instruction/insert-extract-element.ll  |   76 +
 llvm/test/CodeGen/LoongArch/lsx/vselect.ll    |    4 +-
 .../floating-point-immediate-operands.mir     |    4 +-
 .../MSP430/powi-soften-libcall-error.ll       |   16 +
 llvm/test/CodeGen/Mips/llvm-ir/load.ll        |  432 +-
 llvm/test/CodeGen/Mips/llvm-ir/store.ll       |  252 +-
 llvm/test/CodeGen/Mips/msa/compare_float.ll   |  595 +-
 llvm/test/CodeGen/Mips/setcc-se.ll            |  463 +-
 llvm/test/CodeGen/NVPTX/access-non-generic.ll |    3 +-
 llvm/test/CodeGen/NVPTX/alias.ll              |    3 +-
 llvm/test/CodeGen/NVPTX/and-or-setcc.ll       |    4 +-
 llvm/test/CodeGen/NVPTX/atomics.ll            |    5 +-
 llvm/test/CodeGen/NVPTX/bf16-instructions.ll  |  671 +-
 .../NVPTX/bf16x2-instructions-approx.ll       |   34 +-
 .../test/CodeGen/NVPTX/bf16x2-instructions.ll |  417 +-
 llvm/test/CodeGen/NVPTX/byval-const-global.ll |    6 +-
 .../CodeGen/NVPTX/call-with-alloca-buffer.ll  |    3 +-
 llvm/test/CodeGen/NVPTX/chain-different-as.ll |    9 +-
 llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll       | 1080 +--
 llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll       | 1080 +--
 llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll       | 1080 +--
 llvm/test/CodeGen/NVPTX/cmpxchg.ll            |  240 +-
 llvm/test/CodeGen/NVPTX/combine-mad.ll        |    7 +-
 .../CodeGen/NVPTX/compute-ptx-value-vts.ll    |   35 +-
 .../test/CodeGen/NVPTX/convergent-mir-call.ll |   10 +-
 .../CodeGen/NVPTX/convert-call-to-indirect.ll |   43 +-
 llvm/test/CodeGen/NVPTX/convert-fp-i8.ll      |   10 +-
 llvm/test/CodeGen/NVPTX/convert-int-sm20.ll   |   73 +-
 llvm/test/CodeGen/NVPTX/demote-vars.ll        |    3 +-
 llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll |   12 +-
 llvm/test/CodeGen/NVPTX/elect.ll              |    2 +-
 llvm/test/CodeGen/NVPTX/extractelement.ll     |   23 +-
 llvm/test/CodeGen/NVPTX/f16-instructions.ll   |   55 +-
 llvm/test/CodeGen/NVPTX/f16x2-instructions.ll |  150 +-
 llvm/test/CodeGen/NVPTX/fexp2.ll              |  125 +-
 llvm/test/CodeGen/NVPTX/flog2.ll              |   80 +-
 llvm/test/CodeGen/NVPTX/fma-relu-contract.ll  |  568 +-
 .../CodeGen/NVPTX/fma-relu-fma-intrinsic.ll   |  417 +-
 .../NVPTX/fma-relu-instruction-flag.ll        |  851 ++-
 llvm/test/CodeGen/NVPTX/fma.ll                |   19 +-
 llvm/test/CodeGen/NVPTX/forward-ld-param.ll   |   12 +-
 llvm/test/CodeGen/NVPTX/fp128-storage-type.ll |    6 +-
 .../test/CodeGen/NVPTX/frameindex-lifetime.ll |   51 +
 llvm/test/CodeGen/NVPTX/frem.ll               |   37 +-
 llvm/test/CodeGen/NVPTX/i1-icmp.ll            |   80 +-
 llvm/test/CodeGen/NVPTX/i1-load-lower.ll      |    5 +-
 llvm/test/CodeGen/NVPTX/i1-select.ll          |   30 +-
 llvm/test/CodeGen/NVPTX/i128-ld-st.ll         |    7 +-
 llvm/test/CodeGen/NVPTX/i128.ll               |  587 +-
 llvm/test/CodeGen/NVPTX/i16x2-instructions.ll |  224 +-
 llvm/test/CodeGen/NVPTX/i8x2-instructions.ll  |   18 +-
 llvm/test/CodeGen/NVPTX/i8x4-instructions.ll  |   74 +-
 llvm/test/CodeGen/NVPTX/idioms.ll             |  127 +-
 llvm/test/CodeGen/NVPTX/indirect_byval.ll     |   16 +-
 llvm/test/CodeGen/NVPTX/jump-table.ll         |   37 +-
 llvm/test/CodeGen/NVPTX/ldg-invariant-256.ll  |   10 +-
 llvm/test/CodeGen/NVPTX/ldg-invariant.ll      |   23 +-
 llvm/test/CodeGen/NVPTX/ldparam-v4.ll         |    5 +-
 llvm/test/CodeGen/NVPTX/ldu-i8.ll             |   17 +-
 llvm/test/CodeGen/NVPTX/ldu-ldg.ll            |    5 +-
 llvm/test/CodeGen/NVPTX/load-store-vectors.ll |   56 +-
 .../NVPTX/load-with-non-coherent-cache.ll     |  654 +-
 llvm/test/CodeGen/NVPTX/local-stack-frame.ll  |   49 +-
 llvm/test/CodeGen/NVPTX/lower-alloca.ll       |    4 +-
 .../CodeGen/NVPTX/lower-args-gridconstant.ll  |   44 +-
 llvm/test/CodeGen/NVPTX/lower-args.ll         |   18 +-
 llvm/test/CodeGen/NVPTX/lower-byval-args.ll   |   34 +-
 llvm/test/CodeGen/NVPTX/math-intrins.ll       |  356 +-
 llvm/test/CodeGen/NVPTX/misched_func_call.ll  |   18 +-
 llvm/test/CodeGen/NVPTX/mulwide.ll            |  294 +-
 .../NVPTX/naked-fn-with-frame-pointer.ll      |   20 +-
 .../CodeGen/NVPTX/nvvm-reflect-arch-O0.ll     |   98 +-
 llvm/test/CodeGen/NVPTX/param-add.ll          |   37 +-
 llvm/test/CodeGen/NVPTX/param-align.ll        |   22 +-
 llvm/test/CodeGen/NVPTX/param-load-store.ll   |  200 +-
 llvm/test/CodeGen/NVPTX/param-overalign.ll    |   89 +-
 .../CodeGen/NVPTX/param-vectorize-device.ll   |  110 +-
 llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll      |   15 +-
 llvm/test/CodeGen/NVPTX/pr13291-i1-store.ll   |    6 +-
 .../CodeGen/NVPTX/proxy-reg-erasure-ptx.ll    |   16 +-
 llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir |    8 +-
 .../CodeGen/NVPTX/reduction-intrinsics.ll     |  288 +-
 llvm/test/CodeGen/NVPTX/reg-types.ll          |   89 +-
 llvm/test/CodeGen/NVPTX/sext-setcc.ll         |    7 +-
 llvm/test/CodeGen/NVPTX/shift-opt.ll          |   34 +-
 llvm/test/CodeGen/NVPTX/sqrt-approx.ll        |   16 +-
 llvm/test/CodeGen/NVPTX/st-param-imm.ll       |  564 +-
 llvm/test/CodeGen/NVPTX/store-undef.ll        |   12 +-
 llvm/test/CodeGen/NVPTX/tex-read-cuda.ll      |    6 +-
 llvm/test/CodeGen/NVPTX/tid-range.ll          |   17 +-
 .../NVPTX/unaligned-param-load-store.ll       |  797 +-
 ...unfold-masked-merge-vector-variablemask.ll |   74 +-
 llvm/test/CodeGen/NVPTX/unreachable.ll        |   20 +-
 llvm/test/CodeGen/NVPTX/variadics-backend.ll  |  114 +-
 llvm/test/CodeGen/NVPTX/vector-loads.ll       |  315 +-
 llvm/test/CodeGen/NVPTX/vector-returns.ll     |  320 +-
 .../GlobalISel/fconstant-unsupported.ll       |    2 +-
 .../CodeGen/PowerPC/PR35812-neg-cmpxchg.ll    |    2 -
 .../aix32-vector-vararg-caller-split.ll       |    8 +-
 llvm/test/CodeGen/PowerPC/all-atomics.ll      |   32 +-
 llvm/test/CodeGen/PowerPC/anyext_srl.ll       |   11 +-
 .../CodeGen/PowerPC/atomics-regression.ll     |   68 -
 .../CodeGen/PowerPC/builtins-bcd-transform.ll |   91 +
 llvm/test/CodeGen/PowerPC/crreduce-reg.mir    |   66 +
 llvm/test/CodeGen/PowerPC/fma-combine.ll      |   41 +-
 llvm/test/CodeGen/PowerPC/fmf-propagation.ll  |    4 +-
 llvm/test/CodeGen/PowerPC/fp-branch.ll        |   15 +-
 llvm/test/CodeGen/PowerPC/licm-xxsplti.ll     |  262 +-
 llvm/test/CodeGen/PowerPC/loop-comment.ll     |    1 -
 llvm/test/CodeGen/PowerPC/memintr32.ll        |   39 +
 llvm/test/CodeGen/PowerPC/memintr64.ll        |   55 +
 llvm/test/CodeGen/PowerPC/mtvsrbmi.ll         |   23 +-
 llvm/test/CodeGen/PowerPC/nofpexcept.ll       |   36 +-
 llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll   |  417 +-
 llvm/test/CodeGen/PowerPC/recipest.ll         |    9 +-
 llvm/test/CodeGen/PowerPC/sat-add.ll          |   21 +-
 .../PowerPC/vsx-fma-mutate-trivial-copy.ll    |    4 +-
 .../CodeGen/PowerPC/xxeval-vselect-x-and.ll   |  186 +-
 .../CodeGen/PowerPC/xxeval-vselect-x-b.ll     |  145 +-
 .../CodeGen/PowerPC/xxeval-vselect-x-c.ll     |  136 +-
 .../CodeGen/PowerPC/xxeval-vselect-x-or.ll    |  246 +
 .../CodeGen/PowerPC/xxeval-vselect-x-xor.ll   |  176 +-
 .../RISCV/GlobalISel/float-intrinsics.ll      |   34 +-
 llvm/test/CodeGen/RISCV/attributes.ll         |   11 +-
 llvm/test/CodeGen/RISCV/branch-opt.mir        |  308 +
 llvm/test/CodeGen/RISCV/double-select-fcmp.ll |   45 +
 llvm/test/CodeGen/RISCV/features-info.ll      |    5 +-
 llvm/test/CodeGen/RISCV/float-convert.ll      |    8 +-
 llvm/test/CodeGen/RISCV/float-intrinsics.ll   |   46 +-
 llvm/test/CodeGen/RISCV/float-select-fcmp.ll  |   39 +
 llvm/test/CodeGen/RISCV/fpclamptosat.ll       |   88 +-
 llvm/test/CodeGen/RISCV/half-select-fcmp.ll   |   84 +
 llvm/test/CodeGen/RISCV/i32-icmp.ll           |   54 +
 llvm/test/CodeGen/RISCV/i64-icmp.ll           |   52 +
 .../RISCV/intrinsic-cttz-elts-vscale.ll       |   36 +-
 .../test/CodeGen/RISCV/overflow-intrinsics.ll |    7 +-
 llvm/test/CodeGen/RISCV/pr145363.ll           |   40 +
 llvm/test/CodeGen/RISCV/rv32xandesperf.ll     |  164 +
 llvm/test/CodeGen/RISCV/rv64-float-convert.ll |   24 +-
 llvm/test/CodeGen/RISCV/rvv/abd.ll            |   10 +-
 .../RISCV/rvv/combine-reduce-add-to-vcpop.ll  |   67 +-
 llvm/test/CodeGen/RISCV/rvv/copyprop.mir      |    2 +-
 .../rvv/fixed-vectors-deinterleave-load.ll    |   67 -
 .../RISCV/rvv/fixed-vectors-fp-buildvec.ll    |   36 +-
 .../RISCV/rvv/fixed-vectors-int-buildvec.ll   |  396 +-
 .../rvv/fixed-vectors-interleave-store.ll     |   34 -
 .../rvv/fixed-vectors-interleaved-access.ll   |   14 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-llrint.ll |  931 +--
 .../RISCV/rvv/fixed-vectors-llround.ll        | 1674 +++++
 .../CodeGen/RISCV/rvv/fixed-vectors-lrint.ll  |  781 +-
 .../CodeGen/RISCV/rvv/fixed-vectors-lround.ll | 1594 ++++
 .../RISCV/rvv/fixed-vectors-mask-buildvec.ll  |   80 +-
 .../RISCV/rvv/fixed-vectors-masked-gather.ll  |  266 +-
 .../fixed-vectors-shuffle-deinterleave2.ll    |  192 +-
 .../fixed-vectors-shuffle-int-interleave.ll   |   55 +-
 .../RISCV/rvv/fixed-vectors-shuffle-int.ll    |   23 +-
 ...t.ll => fixed-vectors-vp-reverse-float.ll} |   33 +-
 ...int.ll => fixed-vectors-vp-reverse-int.ll} |    0
 .../RISCV/rvv/fixed-vectors-vp-splice.ll      |  197 +-
 .../RISCV/rvv/fixed-vectors-vselect.ll        |   24 +-
 llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll |  216 +
 .../RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll |   45 -
 llvm/test/CodeGen/RISCV/rvv/select-fp.ll      |    4 +-
 llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll    |   96 +-
 .../RISCV/rvv/vector-deinterleave-fixed.ll    |   22 +-
 .../RISCV/rvv/vector-deinterleave-load.ll     |   67 -
 .../RISCV/rvv/vector-interleave-store.ll      |   34 -
 .../CodeGen/RISCV/rvv/vector-tuple-align.ll   |    8 +-
 .../test/CodeGen/RISCV/rvv/vfma-vp-combine.ll |    6 +-
 llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll  |  288 +-
 .../CodeGen/RISCV/rvv/vmerge-peephole.mir     |   57 +
 llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir      |    8 +-
 .../CodeGen/RISCV/rvv/vp-reverse-float.ll     |  177 +-
 .../RISCV/rvv/vp-splice-mask-fixed-vectors.ll |   40 +-
 llvm/test/CodeGen/RISCV/rvv/vp-splice.ll      |  204 +-
 .../RISCV/rvv/vp-vector-interleaved-access.ll |  368 +-
 llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll   |  209 +-
 llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll |   35 +-
 .../test/CodeGen/RISCV/rvv/vsetvli-insert.mir |   42 +
 llvm/test/CodeGen/RISCV/rvv/vslidedown.ll     |  344 +-
 llvm/test/CodeGen/RISCV/rvv/vslideup.ll       |  344 +-
 llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll   |   14 +-
 llvm/test/CodeGen/RISCV/select-cond.ll        |  204 +
 .../CodeGen/RISCV/select-optimize-multiple.ll |   42 -
 .../CodeGen/RISCV/short-forward-branch-opt.ll | 1361 +++-
 .../RISCV/wide-scalar-shift-legalization.ll   | 1901 +++--
 llvm/test/CodeGen/RISCV/xmips-cbop.ll         |   51 +
 llvm/test/CodeGen/RISCV/xqcicm.ll             |  679 ++
 llvm/test/CodeGen/RISCV/xqcisls.ll            |  311 +
 llvm/test/CodeGen/RISCV/zicond-opts.ll        |   82 +-
 .../CodeGen/SPIRV/capability-FloatControl2.ll |   22 +
 .../enable-all-extensions-but-one.ll          |    2 +
 .../hlsl-resources/MixedBufferLoadStore.ll    |   36 +
 .../hlsl-resources/SignedBufferLoadStore.ll   |  137 +
 .../hlsl-resources/UnsignedBufferLoadStore.ll |  137 +
 llvm/test/CodeGen/SystemZ/pr60413.ll          |  112 +-
 .../CodeGen/SystemZ/vec-max-min-zerosplat.ll  |   70 +-
 .../CodeGen/SystemZ/zos-ada-relocations.ll    |    4 +-
 llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll    |    1 +
 llvm/test/CodeGen/SystemZ/zos-intrinsics.ll   |    4 +-
 llvm/test/CodeGen/SystemZ/zos-landingpad.ll   |    4 +-
 .../ppa1.ll => CodeGen/SystemZ/zos-ppa1.ll}   |    3 +-
 llvm/test/CodeGen/SystemZ/zos-ppa2.ll         |    4 +-
 llvm/test/CodeGen/SystemZ/zos-section-1.ll    |  129 +
 llvm/test/CodeGen/SystemZ/zos-section-2.ll    |  175 +
 llvm/test/CodeGen/SystemZ/zos-simple-test.ll  |    8 +-
 llvm/test/CodeGen/Thumb2/mve-doublereduct.ll  |  162 +-
 .../CodeGen/Thumb2/mve-vselect-constants.ll   |    8 +-
 llvm/test/CodeGen/WebAssembly/PR40172.ll      |    6 +-
 llvm/test/CodeGen/WebAssembly/PR41841.ll      |   16 +-
 .../WebAssembly/cfg-stackify-eh-legacy.mir    |    2 +-
 .../WebAssembly/debug-code-stackification.ll  |   89 +
 llvm/test/CodeGen/WebAssembly/eh-lsda.ll      |    4 +-
 .../CodeGen/WebAssembly/exception-legacy.ll   |    2 +-
 .../CodeGen/WebAssembly/exception-legacy.mir  |    2 +-
 .../CodeGen/WebAssembly/function-info.mir     |    2 +-
 llvm/test/CodeGen/WebAssembly/inline-asm.ll   |   11 +
 llvm/test/CodeGen/WebAssembly/offset.ll       |   10 +
 llvm/test/CodeGen/WebAssembly/pr51651.ll      |   14 +-
 .../WebAssembly/signext-zeroext-callsite.ll   |   60 +-
 llvm/test/CodeGen/WebAssembly/simd-bitmask.ll |  226 +-
 .../WebAssembly/simd-illegal-bitmask.ll       |  596 ++
 .../WebAssembly/simd-setcc-reductions.ll      |  135 +
 .../WebAssembly/simd-vecreduce-bool.ll        |    6 +-
 .../CodeGen/WebAssembly/suboptimal-compare.ll |   25 +-
 .../X86/GlobalISel/avoid-matchtable-crash.mir |    2 +-
 .../CodeGen/X86/GlobalISel/llvm.sincos.mir    |  189 +
 llvm/test/CodeGen/X86/abds-neg.ll             |   36 +-
 llvm/test/CodeGen/X86/abds-vector-128.ll      |    2 +-
 .../CodeGen/X86/apx/kmov-domain-assignment.ll |    8 +-
 llvm/test/CodeGen/X86/avg.ll                  |  177 +-
 llvm/test/CodeGen/X86/avx512-vbroadcast.ll    |    2 +-
 llvm/test/CodeGen/X86/avx512fp16-cvt.ll       |   18 +
 .../test/CodeGen/X86/change-unsafe-fp-math.ll |   34 +-
 .../CodeGen/X86/dag-combiner-fma-folding.ll   |   34 +
 .../X86/div-rem-pair-recomposition-signed.ll  |   39 +-
 .../div-rem-pair-recomposition-unsigned.ll    |   12 +-
 llvm/test/CodeGen/X86/fdiv.ll                 |    4 +-
 .../CodeGen/X86/fminimumnum-fmaximumnum.ll    |  252 +-
 llvm/test/CodeGen/X86/fp-intrinsics-flags.ll  |    2 +-
 llvm/test/CodeGen/X86/fp128-select.ll         |   24 +-
 .../CodeGen/X86/frame-pointer-reserved.ll     |   25 +
 llvm/test/CodeGen/X86/freeze-binary.ll        |  120 +
 llvm/test/CodeGen/X86/freeze-unary.ll         |  134 +
 llvm/test/CodeGen/X86/freeze-vector.ll        |  138 +-
 llvm/test/CodeGen/X86/isel-fabs-x87.ll        |   79 +-
 llvm/test/CodeGen/X86/isel-fabs.ll            |   66 +-
 llvm/test/CodeGen/X86/legalize-sub-zero.ll    |   22 +-
 llvm/test/CodeGen/X86/llvm.sincos.ll          |   92 +-
 .../CodeGen/X86/ms-secure-hotpatch-attr.ll    |   38 +
 .../X86/ms-secure-hotpatch-bad-file.ll        |   16 +
 ...ms-secure-hotpatch-direct-global-access.ll |   39 +
 .../X86/ms-secure-hotpatch-functions-file.ll  |   39 +
 .../X86/ms-secure-hotpatch-functions-list.ll  |   38 +
 llvm/test/CodeGen/X86/pr134607.ll             |   19 +-
 llvm/test/CodeGen/X86/pr43157.ll              |    3 +-
 .../test/CodeGen/X86/setcc-non-simple-type.ll |    4 +-
 llvm/test/CodeGen/X86/shift-i512.ll           |  194 +-
 llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll    |   20 +-
 llvm/test/CodeGen/X86/sqrt-fastmath.ll        |   13 +-
 .../statepoint-cmp-sunk-past-statepoint.ll    |  223 +-
 llvm/test/CodeGen/X86/unpredictable-brcond.ll |    2 +-
 .../X86/urem-seteq-vec-tautological.ll        |   17 +-
 llvm/test/CodeGen/X86/var-permute-256.ll      |  216 +-
 .../X86/vec-strict-fptoint-128-fp16.ll        |  158 +-
 llvm/test/CodeGen/X86/vec_int_to_fp.ll        |   24 +-
 llvm/test/CodeGen/X86/vector-compress.ll      |   42 +-
 .../vector-constrained-fp-intrinsics-flags.ll |    6 +-
 .../vector-interleaved-load-i16-stride-4.ll   | 1462 ++--
 .../vector-interleaved-load-i8-stride-8.ll    | 6448 ++++++++---------
 .../CodeGen/X86/x86-64-double-shifts-var.ll   |   64 +-
 llvm/test/CodeGen/X86/x86-access-to-global.ll |   27 +
 llvm/test/CodeGen/Xtensa/float-arith.ll       |   21 +-
 llvm/test/CodeGen/Xtensa/invalid-tls.ll       |   17 +
 llvm/test/CodeGen/Xtensa/select-cc-fp.ll      |  457 ++
 llvm/test/CodeGen/Xtensa/threadptr.ll         |   18 +
 ...-dwarf-diop-diexpression-address-spaces.ll |   12 +
 .../DebugInfo/AMDGPU/reg-sequence-salvage.ll  |    8 +-
 .../codegenprepare/sunk-addr.ll               |    2 +-
 .../dse/dse-after-memcpyopt-merge.ll          |    2 +-
 .../assignment-tracking/dse/shorten-offset.ll |    2 +-
 .../inline/shared-alloca.ll                   |    2 +-
 .../do-not-remove-redundant-dbg.ll            |    2 +-
 .../instcombine/remove-redundant-dbg.ll       |    2 +-
 .../loop-vectorize/remove-redundant-dbg.ll    |    2 +-
 .../assignment-tracking/mem2reg/phi.ll        |    2 +-
 .../mem2reg/single-block-alloca.ll            |    2 +-
 .../mem2reg/single-store-alloca.ll            |    2 +-
 .../mem2reg/store-to-part-of-alloca.ll        |    2 +-
 .../Generic/assignment-tracking/optnone.ll    |    2 +-
 .../remove-redundant-fwd-scan-linked.ll       |    2 +-
 .../assignment-tracking/remove-redundant.ll   |    2 +-
 .../assignment-tracking/salvage-value.ll      |    2 +-
 .../sroa/alloca-single-slice.ll               |    2 +-
 .../assignment-tracking/sroa/complex.ll       |    2 +-
 .../assignment-tracking/sroa/fail-fragment.ll |    2 +-
 .../assignment-tracking/sroa/memcpy.ll        |    2 +-
 .../sroa/remove-redundant-dbg.ll              |    2 +-
 .../assignment-tracking/sroa/rewrite.ll       |    2 +-
 .../sroa/split-pre-fragmented-store-2.ll      |    2 +-
 .../sroa/split-pre-fragmented-store.ll        |    2 +-
 .../Generic/assignment-tracking/sroa/store.ll |    2 +-
 .../assignment-tracking/sroa/user-memcpy.ll   |    2 +-
 .../assignment-tracking/track-assignments.ll  |    2 +-
 llvm/test/DebugInfo/Generic/debug-label.ll    |    4 +-
 .../Generic/inline-alloca-ordering.ll         |    2 +-
 .../DebugInfo/Generic/inline-dbg-values.ll    |    2 +-
 .../KeyInstructions/Generic/inline-nodbg.ll   |    4 +-
 .../KeyInstructions/Generic/inline.ll         |    6 +-
 .../Generic/jump-threading-2-bbs.ll           |    3 +-
 .../Generic/jump-threading-basic.ll           |    3 +-
 ...-threading-dup-cond-br-on-phi-into-pred.ll |    3 +-
 .../KeyInstructions/Generic/link-two-modes.ll |   67 +
 .../KeyInstructions/Generic/loop-rotate.ll    |    3 +-
 .../Generic/loop-unroll-runtime.ll            |    3 +-
 .../KeyInstructions/Generic/loop-unroll.ll    |    2 +-
 .../KeyInstructions/Generic/loop-unswitch.ll  |    3 +-
 .../KeyInstructions/Generic/parse.ll          |    3 +-
 .../KeyInstructions/Generic/roundtrip.ll      |   48 +
 .../Generic/simplifycfg-branch-fold.ll        |    4 +-
 .../Generic/simplifycfg-thread-phi.ll         |    4 +-
 .../KeyInstructions/Generic/verify.ll         |   22 +
 .../X86/cgp-break-critical-edge.ll            |    2 +-
 .../KeyInstructions/X86/dwarf-basic-ranks.ll  |    2 +-
 .../KeyInstructions/X86/dwarf-basic.ll        |    4 +-
 .../X86/dwarf-buoy-multi-key.mir              |    2 +-
 .../KeyInstructions/X86/dwarf-buoy.mir        |    2 +-
 .../KeyInstructions/X86/dwarf-calls.ll        |    2 +-
 .../X86/dwarf-inline-modes.mir                |   93 +
 .../KeyInstructions/X86/dwarf-ranks-blocks.ll |    2 +-
 .../DebugInfo/KeyInstructions/debugify.ll     |    3 +
 .../entry_value_gets_propagated_aarch64.mir   |   84 +
 .../X86/entry_value_clobbered_stack_copy.mir  |   52 +
 .../X86/entry_value_gets_propagated_X86.mir   |   98 +
 .../X86/debug-line-in-one-fragment.ll         |   12 +-
 .../assignment-tracking/X86/large-type.ll     |    2 +
 llvm/test/DebugInfo/debug_frame_symbol.ll     |    2 +-
 llvm/test/Feature/intrinsics.ll               |    5 +-
 llvm/test/FileCheck/var-escape.txt            |   29 +
 .../AddressSanitizer/calls-only-smallfn.ll    |    2 +-
 .../AddressSanitizer/calls-only.ll            |    2 +-
 .../BoundsChecking/runtimes.ll                |    2 +-
 .../MemorySanitizer/count-zeroes.ll           |   88 +-
 .../Instrumentation/MemorySanitizer/or.ll     |   57 +
 .../MemorySanitizer/with-call-type-size.ll    |   21 +-
 llvm/test/LTO/AArch64/module-asm.ll           |   21 +
 .../directive-arch_extension-negative.s       |    2 +-
 llvm/test/MC/AArch64/SVE2/aesd.s              |    2 +-
 llvm/test/MC/AArch64/SVE2/aese.s              |    2 +-
 llvm/test/MC/AArch64/SVE2/aesimc.s            |    4 +-
 llvm/test/MC/AArch64/SVE2/aesmc.s             |    4 +-
 llvm/test/MC/AArch64/SVE2/bdep.s              |    8 +-
 llvm/test/MC/AArch64/SVE2/bext.s              |    8 +-
 llvm/test/MC/AArch64/SVE2/bgrp.s              |    8 +-
 .../MC/AArch64/SVE2/directive-arch-negative.s |    2 +-
 .../SVE2/directive-arch_extension-negative.s  |   12 +-
 .../MC/AArch64/SVE2/directive-cpu-negative.s  |    6 +-
 llvm/test/MC/AArch64/SVE2/pmullb-128.s        |    2 +-
 llvm/test/MC/AArch64/SVE2/pmullt-128.s        |    2 +-
 llvm/test/MC/AArch64/SVE2/rax1.s              |    6 +-
 llvm/test/MC/AArch64/SVE2p1/aesd.s            |   12 +-
 llvm/test/MC/AArch64/SVE2p1/aesdimc.s         |   12 +-
 llvm/test/MC/AArch64/SVE2p1/aese.s            |   12 +-
 llvm/test/MC/AArch64/SVE2p1/aesemc.s          |   12 +-
 llvm/test/MC/AArch64/SVE2p1/pmlal.s           |    6 +-
 llvm/test/MC/AArch64/SVE2p1/pmull.s           |    6 +-
 .../MC/AArch64/coff-relocations-branch26.s    |    2 +-
 llvm/test/MC/AArch64/coff-relocations-diags.s |    6 +-
 llvm/test/MC/AArch64/ilp32-diagnostics.s      |   56 +-
 llvm/test/MC/AMDGPU/gfx10_asm_vop2.s          |   12 +
 llvm/test/MC/AMDGPU/gfx10_asm_vop3.s          |   24 +
 llvm/test/MC/AMDGPU/gfx10_unsupported_sdwa.s  |    3 +
 llvm/test/MC/AMDGPU/gfx1250_asm_ds.s          |   26 +
 llvm/test/MC/AMDGPU/gfx1250_asm_load_tr.s     |  219 +
 llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s        |   12 +
 llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s |  569 ++
 llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s   |    7 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s       |   22 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s |   37 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s      |   34 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s  |   25 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s |   89 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s        |   98 +
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s |   83 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s  |  103 +
 .../MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s  |   39 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s   |   59 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_err.s    |   36 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop3-fake16.s |   17 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop3.s        |   17 +
 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s    |   11 +
 .../gfx1250_asm_vop3_from_vop1-fake16.s       |  155 +
 .../MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s    |  158 +
 .../gfx1250_asm_vop3_from_vop1_dpp16-fake16.s |  103 +
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s |  115 +
 .../gfx1250_asm_vop3_from_vop1_dpp8-fake16.s  |   75 +
 .../AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s  |   83 +
 llvm/test/MC/AMDGPU/gfx950_asm_features.s     |   24 +
 llvm/test/MC/AMDGPU/gfx9_asm_vop2_features.s  |   91 +
 llvm/test/MC/AMDGPU/max-branch-distance.s     |    2 +-
 .../Relocations/thumb-branch-out-of-range.s   |   71 +
 llvm/test/MC/ARM/Relocations/thumb-branch.s   |   21 +
 .../test/MC/ARM/Windows/branch-reloc-offset.s |    8 +-
 llvm/test/MC/ARM/Windows/invalid-relocation.s |    2 +-
 .../ARM/arm-memory-instructions-immediate.s   |    2 +-
 llvm/test/MC/ARM/elf-thumbfunc-reloc.s        |    9 +-
 llvm/test/MC/ARM/lower-upper-errors-2.s       |    6 +-
 llvm/test/MC/ARM/pcrel-ldrd-diff-section.s    |    4 +-
 llvm/test/MC/ARM/pcrel-vldr-diff-section.s    |    6 +-
 llvm/test/MC/ARM/quad-relocation.s            |    2 +-
 .../ARM/t2-modified-immediate-fixup-error1.s  |    2 +-
 llvm/test/MC/ARM/thumb-branches.s             |   71 -
 llvm/test/MC/ARM/thumb1-relax-adr.s           |    2 +-
 llvm/test/MC/ARM/thumb1-relax-bcc.s           |    2 +-
 llvm/test/MC/ARM/thumb1-relax-br.s            |    2 +-
 llvm/test/MC/ARM/thumb1-relax-ldrlit.s        |    2 +-
 llvm/test/MC/ARM/thumb2-branch-ranges.s       |   10 +-
 llvm/test/MC/BPF/bad-tied.s                   |   12 +
 .../Disassembler/AMDGPU/gfx10_vop2_dpp16.txt  |    7 +
 .../Disassembler/AMDGPU/gfx10_vop2_dpp8.txt   |    3 +
 .../MC/Disassembler/AMDGPU/gfx10_vop3.txt     |   24 +
 .../Disassembler/AMDGPU/gfx1250_dasm_ds.txt   |   19 +
 .../AMDGPU/gfx1250_dasm_load_tr.txt           |  103 +
 .../Disassembler/AMDGPU/gfx1250_dasm_sopp.txt |    9 +
 .../AMDGPU/gfx1250_dasm_vflat.txt             |   16 +
 .../AMDGPU/gfx1250_dasm_vimage.txt            |   25 +
 .../Disassembler/AMDGPU/gfx1250_dasm_vop1.txt |  109 +
 .../AMDGPU/gfx1250_dasm_vop1_dpp16.txt        |   86 +
 .../AMDGPU/gfx1250_dasm_vop1_dpp8.txt         |   53 +
 .../Disassembler/AMDGPU/gfx1250_dasm_vop3.txt |   21 +
 .../AMDGPU/gfx1250_dasm_vop3_from_vop1.txt    |  105 +
 .../gfx1250_dasm_vop3_from_vop1_dpp16.txt     |  115 +
 .../gfx1250_dasm_vop3_from_vop1_dpp8.txt      |   87 +
 .../AMDGPU/gfx12_dasm_vop3_dpp8.txt           |  108 +-
 llvm/test/MC/Disassembler/AMDGPU/gfx950.txt   |   36 +
 .../AMDGPU/gfx9_vop2_features.txt             |   92 +
 .../MC/Disassembler/RISCV/c_lui_disasm.txt    |   68 -
 llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt |   19 +
 llvm/test/MC/Disassembler/Xtensa/fp.txt       |  208 +-
 .../test/MC/Disassembler/Xtensa/threadptr.txt |   11 +
 llvm/test/MC/ELF/bad-expr.s                   |    2 +-
 llvm/test/MC/ELF/comdat.s                     |  100 -
 llvm/test/MC/ELF/group.s                      |   59 +
 llvm/test/MC/ELF/mc-dump.s                    |   53 +
 llvm/test/MC/ELF/relocation-alias.s           |    6 +-
 llvm/test/MC/ELF/weakref.s                    |    2 +-
 llvm/test/MC/GOFF/empty-goff.s                |   10 +-
 .../Relocations/fixups-diagnostics.s          |   12 +-
 llvm/test/MC/M68k/Control/branch-pc-rel.s     |    2 -
 llvm/test/MC/M68k/Control/bsr.s               |    2 -
 llvm/test/MC/M68k/Control/bsr32.s             |    3 -
 llvm/test/MC/M68k/Control/call-pc-rel.s       |    4 -
 llvm/test/MC/M68k/Relaxations/PIC/branch.s    |    5 +-
 llvm/test/MC/M68k/Relaxations/PIC/branch32.s  |    6 +-
 llvm/test/MC/M68k/Relaxations/branch32.s      |    6 +-
 llvm/test/MC/M68k/Relocations/PIC/data-abs.s  |    6 +-
 .../MC/M68k/Relocations/PIC/data-gotoff.s     |    5 +-
 .../MC/M68k/Relocations/PIC/data-gotpcrel.s   |    4 +-
 .../MC/M68k/Relocations/PIC/data-pc-rel.s     |    5 +-
 llvm/test/MC/M68k/Relocations/PIC/text-plt.s  |    4 +-
 llvm/test/MC/M68k/Relocations/data-abs.s      |    6 +-
 llvm/test/MC/M68k/Relocations/data-gotoff.s   |    5 +-
 llvm/test/MC/M68k/Relocations/data-gotpcrel.s |    4 +-
 llvm/test/MC/M68k/Relocations/data-pc-rel.s   |    5 +-
 llvm/test/MC/M68k/Relocations/text-plt.s      |    4 +-
 llvm/test/MC/M68k/Relocations/tls.s           |    7 +-
 llvm/test/MC/M68k/pc-rel.s                    |    2 -
 llvm/test/MC/Mips/reloc-directive-bad-obj.s   |   18 +-
 llvm/test/MC/Mips/unsupported-relocation.s    |    2 +-
 llvm/test/MC/PowerPC/ppc64-abs-reloc.s        |    3 -
 .../Relocations/data-directive-specifier.s    |   12 +-
 llvm/test/MC/RISCV/Relocations/expr-err.s     |   40 +-
 llvm/test/MC/RISCV/Relocations/mc-dump.s      |   20 +
 llvm/test/MC/RISCV/fixups-diagnostics.s       |    6 +-
 .../MC/RISCV/option-exact-long-jump-disable.s |    6 +-
 llvm/test/MC/RISCV/pcrel-lo12-invalid.s       |    6 +-
 llvm/test/MC/RISCV/rv32-relaxation-xqci.s     |    4 +-
 llvm/test/MC/RISCV/rv32c-invalid.s            |   11 +-
 llvm/test/MC/RISCV/rv32p-invalid.s            |   19 +
 llvm/test/MC/RISCV/rv32p-valid.s              |   71 +-
 llvm/test/MC/RISCV/rv64p-invalid.s            |   21 +
 llvm/test/MC/RISCV/rv64p-valid.s              |  107 +-
 llvm/test/MC/RISCV/rvv/xsfvfwmacc.s           |    4 +-
 .../MC/RISCV/xandesperf-fixups-diagnostics.s  |    4 +-
 llvm/test/MC/RISCV/xandesvsintload-valid.s    |   39 +
 llvm/test/MC/RISCV/xmips-invalid.s            |   11 +-
 llvm/test/MC/RISCV/xmips-valid.s              |   18 +-
 llvm/test/MC/RISCV/xqcicm-valid.s             |    6 +
 llvm/test/MC/VE/data-reloc-error.s            |    6 +-
 llvm/test/MC/X86/Relocations/x86-16.s         |   13 +
 llvm/test/MC/X86/Relocations/x86-32.s         |   17 +
 llvm/test/MC/X86/Relocations/x86-64.s         |  102 +
 llvm/test/MC/X86/avx-64-att.s                 |   13 -
 llvm/test/MC/X86/fma4-att.s                   |   60 -
 llvm/test/MC/X86/gotpcrel-non-globals.ll      |   36 +
 llvm/test/MC/X86/macho-reloc-errors-x86.s     |    6 +-
 llvm/test/MC/X86/macho-reloc-errors-x86_64.s  |   16 +-
 llvm/test/MC/X86/pltoff.s                     |    2 +-
 llvm/test/MC/X86/register-assignment.s        |    4 +-
 llvm/test/MC/X86/unused_reg_var_assign.s      |    3 -
 llvm/test/MC/X86/x86-16.s                     |   11 -
 llvm/test/MC/X86/x86-32-coverage.s            |    6 -
 llvm/test/MC/X86/x86-32.s                     |   20 +-
 llvm/test/MC/X86/x86-64.s                     |   72 +-
 llvm/test/MC/X86/x86_64-encoding.s            |    6 -
 .../Xtensa/Relocations/fixups-diagnostics.s   |    8 +-
 llvm/test/MC/Xtensa/dfpaccel.s                |   53 +
 llvm/test/MC/Xtensa/threadptr.s               |   21 +
 llvm/test/Other/fatlto.ll                     |    7 +
 llvm/test/Other/pipeline-alias-errors.ll      |   20 +
 .../TableGen/CompressWriteLatencyEntry.td     |    8 +-
 llvm/test/TableGen/DecoderEmitterFnTable.td   |   84 +
 .../FixedLenDecoderEmitter/conflict.td        |    2 +-
 llvm/test/TableGen/InvalidMCSchedClassDesc.td |   22 +-
 llvm/test/TableGen/RuntimeLibcallEmitter.td   |   77 +
 .../Attributor/AMDGPU/tag-invariant-loads.ll  |  431 ++
 .../Attributor/IPConstantProp/PR16052.ll      |    8 +-
 .../Attributor/dereferenceable-1.ll           |    1 -
 llvm/test/Transforms/Attributor/range.ll      |   45 +-
 .../Attributor/value-simplify-local-remote.ll |    8 -
 .../Transforms/Attributor/value-simplify.ll   |    4 +-
 .../callsite-split-preserve-debug.ll          |    2 +-
 .../X86/gather-scatter-opt-inseltpoison.ll    |   66 +
 .../CodeGenPrepare/X86/sink-addr-recreate.ll  |  106 +
 .../CodeGenPrepare/X86/sink-addrmode-base.ll  |  630 +-
 .../X86/sink-addrmode-two-phi.ll              |   13 +-
 .../Coroutines/coro-debug-dbg.values.ll       |    2 +-
 llvm/test/Transforms/Coroutines/coro-debug.ll |   32 +-
 .../Coroutines/coro-split-dbg-labels.ll       |  147 +
 ...-lieftime.ll => coro-split-no-lifetime.ll} |    0
 llvm/test/Transforms/DCE/dbg-value-removal.ll |   60 +-
 .../Transforms/DeadStoreElimination/simple.ll |   35 +-
 llvm/test/Transforms/EarlyCSE/writeonly.ll    |  148 +-
 .../test/Transforms/EmbedBitcode/embed-wpd.ll |    7 +-
 .../Transforms/FunctionAttrs/initializes.ll   |   20 +-
 llvm/test/Transforms/FunctionAttrs/nosync.ll  |    2 +-
 .../GlobalOpt/ARM/arm-widen-dest-non-array.ll |   39 -
 .../GlobalOpt/ARM/arm-widen-global-dest.ll    |   28 -
 .../GlobalOpt/ARM/arm-widen-non-byte-array.ll |   22 -
 .../ARM/arm-widen-non-const-global.ll         |   21 -
 .../ARM/arm-widen-string-multi-use.ll         |   33 -
 .../GlobalOpt/ARM/arm-widen-strings-1.ll      |   21 -
 .../GlobalOpt/ARM/arm-widen-strings-2.ll      |   21 -
 .../arm-widen-strings-lengths-dont-match.ll   |   26 -
 .../arm-widen-strings-more-than-64-bytes.ll   |   28 -
 .../ARM/arm-widen-strings-ptrtoint.ll         |   54 -
 .../ARM/arm-widen-strings-struct-test.ll      |   45 -
 .../ARM/arm-widen-strings-volatile.ll         |   29 -
 .../Transforms/GlobalOpt/ARM/lit.local.cfg    |    2 -
 .../assumption-cache-invalidation.ll          |    2 +-
 .../InferAlignment/gep-recurrence.ll          |  574 ++
 .../Inline/simplify-crosscallsite.ll          |   67 +
 .../Transforms/InstCombine/AMDGPU/fmed3.ll    |   90 +
 .../InstCombine/SystemZ/lit.local.cfg         |    2 +
 .../SystemZ/printf-opt-alignment.ll           |   17 +
 .../InstCombine/binop-recurrence.ll           | 1408 ++++
 .../Transforms/InstCombine/compare-3way.ll    |  114 +-
 .../test/Transforms/InstCombine/ctpop-cttz.ll |   21 +
 .../InstCombine/debuginfo-variables.ll        |    9 +-
 .../fold-phi-arg-gep-to-phi-negative.ll       |    6 +-
 .../InstCombine/fold-shuffle-ext.ll           |  121 +
 llvm/test/Transforms/InstCombine/freeze.ll    |   98 +
 .../Transforms/InstCombine/getelementptr.ll   |   14 +-
 llvm/test/Transforms/InstCombine/icmp-gep.ll  |   92 +
 llvm/test/Transforms/InstCombine/icmp.ll      |  102 +
 .../matrix-multiplication-negation.ll         |    7 +-
 llvm/test/Transforms/InstCombine/pr39908.ll   |   12 +-
 .../sub-after-sle-is-non-negative.ll          |  153 +
 llvm/test/Transforms/InstCombine/sub-gep.ll   |  133 +
 .../vec_demanded_elts-inseltpoison.ll         |   24 +-
 .../InstCombine/vec_demanded_elts.ll          |   24 +-
 .../vec_extract_through_broadcast.ll          |   52 +
 .../InstCombine/vec_shuffle-inseltpoison.ll   |    9 +-
 .../Transforms/InstCombine/vec_shuffle.ll     |   45 +-
 .../Transforms/InstCombine/vector-reverse.ll  |  153 +-
 .../InstCombine/vector_gep1-inseltpoison.ll   |    2 +-
 .../Transforms/InstCombine/vector_gep1.ll     |    2 +-
 .../vscale_extractelement-inseltpoison.ll     |   13 +-
 .../InstCombine/vscale_extractelement.ll      |   13 +-
 .../InstSimplify/ConstProp/atan-intrinsic.ll  |    1 -
 .../InstSimplify/ConstProp/calls.ll           |   12 +
 llvm/test/Transforms/InstSimplify/call.ll     |   24 +-
 .../InstSimplify/fold-intrinsics.ll           |   45 +-
 .../InstSimplify/saturating-add-sub.ll        |   32 +-
 .../AArch64/sve-deinterleave4.ll              |   90 +-
 .../AArch64/sve-interleave4.ll                |   17 +-
 .../RISCV/interleaved-accesses.ll             |  196 -
 .../Transforms/LICM/salvage-hoisted-minmax.ll |   59 +
 llvm/test/Transforms/LoopDeletion/diundef.ll  |    2 +-
 llvm/test/Transforms/LoopFusion/four_loops.ll |   85 +-
 .../Transforms/LoopFusion/guarded_peel.ll     |   93 +-
 .../LoopFusion/guarded_unsafeblock_peel.ll    |   55 +-
 .../Transforms/LoopFusion/hoist_preheader.ll  |   25 +-
 .../test/Transforms/LoopFusion/inner_loops.ll |   72 +-
 llvm/test/Transforms/LoopFusion/loop_nest.ll  |   68 +-
 .../Transforms/LoopFusion/nonadjacent_peel.ll |   67 +-
 llvm/test/Transforms/LoopFusion/peel.ll       |  132 +-
 .../Transforms/LoopFusion/sink_preheader.ll   |   20 +-
 .../LoopInterchange/position-in-pipeline.ll   |   48 +
 llvm/test/Transforms/LoopRotate/dbgvalue.ll   |    2 +-
 .../LoopRotate/delete-dbg-values.ll           |    2 +-
 .../LoopVectorize/AArch64/check-prof-info.ll  |    2 +-
 .../LoopVectorize/AArch64/early_exit_costs.ll |    2 +-
 .../LoopVectorize/AArch64/licm-calls.ll       |   72 +
 .../AArch64/partial-reduce-chained.ll         |  466 ++
 .../partial-reduce-dot-product-mixed.ll       |   24 +-
 .../AArch64/partial-reduce-dot-product.ll     |  586 +-
 .../LoopVectorize/AArch64/partial-reduce.ll   |  918 +++
 .../AArch64/simple_early_exit.ll              |    2 +-
 .../AArch64/single-early-exit-interleave.ll   |   55 +-
 .../AArch64/sve-interleaved-accesses.ll       |   24 +-
 ...-narrow-interleave-to-widen-memory-cost.ll |  307 +
 ...-interleave-to-widen-memory-derived-ivs.ll |  303 +
 ...sform-narrow-interleave-to-widen-memory.ll |  385 +
 .../LoopVectorize/AArch64/vplan-printing.ll   |    3 +-
 .../AMDGPU/assert-vplan-cost-model.ll         |    2 +-
 .../LoopVectorize/ARM/mve-reductions.ll       |    4 +-
 .../RISCV/evl-compatible-loops.ll             |   56 +-
 .../LoopVectorize/RISCV/inloop-reduction.ll   |    2 +-
 .../RISCV/interleaved-masked-access.ll        |  326 +-
 .../RISCV/only-compute-cost-for-vplan-vfs.ll  |   10 +-
 .../LoopVectorize/RISCV/preserve-dbg-loc.ll   |    2 +-
 .../RISCV/riscv-vector-reverse-output.ll      |   48 +-
 .../RISCV/riscv-vector-reverse.ll             |  148 +-
 .../RISCV/type-info-cache-evl-crash.ll        |    2 +-
 ...ize-force-tail-with-evl-cast-intrinsics.ll |   45 +-
 ...rize-force-tail-with-evl-cond-reduction.ll |  271 +-
 ...ce-tail-with-evl-fixed-order-recurrence.ll |  149 +
 ...rize-force-tail-with-evl-gather-scatter.ll |   54 +-
 ...vectorize-force-tail-with-evl-reduction.ll |   12 +-
 ...-force-tail-with-evl-reverse-load-store.ll |   21 +-
 ...orize-force-tail-with-evl-uniform-store.ll |    3 +-
 .../RISCV/vplan-vp-call-intrinsics.ll         |   27 +-
 .../RISCV/vplan-vp-cast-intrinsics.ll         |   61 +-
 .../RISCV/vplan-vp-intrinsics-reduction.ll    |   18 +-
 .../RISCV/vplan-vp-intrinsics.ll              |    5 +-
 .../RISCV/vplan-vp-select-intrinsics.ll       |   77 -
 .../LoopVectorize/X86/cost-model.ll           |    3 +-
 .../X86/fixed-order-recurrence.ll             |    6 -
 ...terleave-ptradd-with-replicated-operand.ll |   51 +-
 ...6-sunk-instruction-used-outside-of-loop.ll |    2 +
 ...able-info-from-assumption-variable-size.ll |  407 --
 .../LoopVectorize/early_exit_legality.ll      |    2 +-
 .../early_exit_store_legality.ll              |   31 +-
 .../epilog-vectorization-reductions.ll        |    6 +-
 ...-order-recurrence-sink-replicate-region.ll |    3 +-
 .../LoopVectorize/first-order-recurrence.ll   |   16 +-
 .../invariant-store-vectorization-2.ll        |    6 +-
 .../LoopVectorize/iv-select-cmp-decreasing.ll |  930 ++-
 .../LoopVectorize/iv_outside_user.ll          |    5 -
 .../LoopVectorize/load-deref-pred-align.ll    |    2 +
 .../LoopVectorize/multi_early_exit.ll         |    2 +-
 .../multi_early_exit_live_outs.ll             |    2 +-
 .../LoopVectorize/narrow-to-single-scalar.ll  |   69 +
 .../LoopVectorize/reduction-small-size.ll     |    9 +-
 .../reuse-lcssa-phi-scev-expansion.ll         |  198 +
 .../scalable-reduction-inloop.ll              |    4 +-
 .../single-early-exit-deref-assumptions.ll    |   96 +
 .../single-early-exit-interleave-hint.ll      |   89 +
 .../single-early-exit-interleave.ll           |  385 +-
 .../LoopVectorize/single_early_exit.ll        |    2 +-
 .../single_early_exit_live_outs.ll            |    2 +-
 .../single_early_exit_with_outer_loop.ll      |    2 +-
 .../LoopVectorize/struct-return-replicate.ll  |   96 +-
 .../uncountable-early-exit-vplan.ll           |    2 +-
 .../Transforms/LoopVectorize/uniform-blend.ll |    4 +
 .../LoopVectorize/unsupported_early_exit.ll   |    2 +-
 ...or-loop-backedge-elimination-early-exit.ll |   29 +-
 .../vplan-printing-reductions.ll              |   23 +-
 .../MergeICmps/X86/blockaddresses.ll          |   87 +
 .../PGOProfile/indirect_call_promotion.ll     |   14 +
 .../PhaseOrdering/X86/blendv-select.ll        |  160 +-
 .../test/Transforms/PhaseOrdering/X86/hadd.ll |   16 +-
 .../test/Transforms/PhaseOrdering/X86/hsub.ll |   16 +-
 .../always-inline-alloca-promotion.ll         |   91 +
 .../PhaseOrdering/loop-access-checks.ll       |    3 +-
 .../Transforms/PhaseOrdering/memset-dse.ll    |   30 +
 .../RelLookupTableConverter/unnamed_addr.ll   |   45 +
 .../SLPVectorizer/AArch64/gather-cost.ll      |   40 +-
 .../SLPVectorizer/AArch64/gather-reduce.ll    |  136 +-
 .../SLPVectorizer/AArch64/getelementptr.ll    |   60 +-
 .../SLPVectorizer/AArch64/getelementptr2.ll   |   26 +-
 .../SLPVectorizer/AArch64/loadorder.ll        |  357 +-
 .../AArch64/transpose-inseltpoison.ll         |    6 +-
 .../SLPVectorizer/AArch64/transpose.ll        |    6 +-
 .../AMDGPU/add_sub_sat-inseltpoison.ll        |  132 +-
 .../SLPVectorizer/AMDGPU/add_sub_sat.ll       |  152 +-
 .../Transforms/SLPVectorizer/AMDGPU/i8.ll     |  199 +-
 .../SLPVectorizer/AMDGPU/min_max.ll           |   58 +-
 .../AMDGPU/phi-result-use-order.ll            |   34 +-
 .../SLPVectorizer/AMDGPU/reduction.ll         |  127 +-
 .../SLPVectorizer/AMDGPU/vectorize-i8.ll      |  504 +-
 .../WebAssembly/no-vectorize-rotate.ll        |   12 +-
 .../X86/alternate-calls-inseltpoison.ll       |  104 +-
 .../SLPVectorizer/X86/alternate-calls.ll      |  116 +-
 .../X86/alternate-cast-inseltpoison.ll        |   80 +-
 .../SLPVectorizer/X86/alternate-cast.ll       |   80 +-
 .../X86/alternate-fp-inseltpoison.ll          |  123 +-
 .../SLPVectorizer/X86/alternate-fp.ll         |  123 +-
 .../X86/alternate-int-inseltpoison.ll         |  239 +-
 .../SLPVectorizer/X86/alternate-int.ll        |  239 +-
 .../X86/blending-shuffle-inseltpoison.ll      |   34 +-
 .../SLPVectorizer/X86/blending-shuffle.ll     |   44 +-
 .../X86/cmp_commute-inseltpoison.ll           |    4 +-
 .../SLPVectorizer/X86/cmp_commute.ll          |    4 +-
 .../X86/extracts-non-extendable.ll            |   26 +-
 .../SLPVectorizer/X86/hadd-inseltpoison.ll    |   78 +-
 .../test/Transforms/SLPVectorizer/X86/hadd.ll |   90 +-
 .../SLPVectorizer/X86/hsub-inseltpoison.ll    |   72 +-
 .../test/Transforms/SLPVectorizer/X86/hsub.ll |   72 +-
 .../SLPVectorizer/X86/minimum-sizes.ll        |   74 +-
 .../SLPVectorizer/X86/operandorder.ll         |   86 +-
 .../Transforms/SLPVectorizer/X86/pr40522.ll   |   34 +-
 .../Transforms/SLPVectorizer/X86/pr46983.ll   |   30 +-
 .../Transforms/SLPVectorizer/X86/pr47623.ll   |   26 +-
 .../SLPVectorizer/X86/pr47629-inseltpoison.ll |  479 +-
 .../Transforms/SLPVectorizer/X86/pr47629.ll   |  479 +-
 .../Transforms/SLPVectorizer/X86/pr47642.ll   |   10 +-
 .../Transforms/SLPVectorizer/X86/pr49081.ll   |    7 +-
 ...masked-loads-consecutive-loads-same-ptr.ll |   10 +-
 .../X86/reverse_extract_elements.ll           |    2 +-
 .../extractelement-insertpoint.ll             |    2 +-
 .../SLPVectorizer/revec-shufflevector.ll      |   93 +-
 llvm/test/Transforms/SROA/select-gep.ll       |   41 +
 .../RISCV/switch-of-powers-of-two.ll          |    3 +-
 .../RISCV/switch_to_lookup_table-rv64.ll      |   12 +-
 .../SimplifyCFG/X86/disable-lookup-table.ll   |    3 +-
 .../SimplifyCFG/X86/switch-covered-bug.ll     |    4 +-
 .../X86/switch-of-powers-of-two.ll            |    3 +-
 .../SimplifyCFG/X86/switch-table-bug.ll       |    4 +-
 .../X86/switch-to-lookup-bitcast.ll           |    3 +-
 .../SimplifyCFG/X86/switch-to-lookup-gep.ll   |    3 +-
 .../X86/switch-to-lookup-globals.ll           |    3 +-
 .../X86/switch-to-lookup-large-types.ll       |    5 +-
 .../SimplifyCFG/X86/switch_to_lookup_table.ll |   60 +-
 .../SimplifyCFG/preserve-branchweights.ll     |   44 +-
 .../Transforms/SimplifyCFG/rangereduce.ll     |   15 +-
 .../switch-dead-default-lookup-table.ll       |    2 +-
 .../SimplifyCFG/switch-to-select-two-case.ll  |  441 ++
 .../Transforms/SimplifyCFG/switch_mask.ll     |    3 +-
 .../test/Transforms/Util/Debugify/loc-only.ll |    2 +-
 .../trivial-auto-var-init-crash-20210521.ll   |    2 +-
 .../VectorCombine/AArch64/ext-extract.ll      |  128 +-
 ...load-extract-insert-store-scalarization.ll |    7 +-
 .../load-extractelement-scalarization.ll      |    3 +-
 .../VectorCombine/AArch64/shrink-types.ll     |   29 +-
 .../AArch64/shuffletoidentity.ll              |    6 +-
 .../AArch64/vecreduce-shuffle.ll              |   18 +-
 .../RISCV/vpintrin-scalarization.ll           |   23 +-
 .../VectorCombine/X86/extract-binop.ll        |   19 +
 .../VectorCombine/X86/extract-fneg-insert.ll  |    8 +-
 .../X86/load-extractelement-scalarization.ll  |    3 +-
 .../VectorCombine/X86/load-inseltpoison.ll    |    5 +-
 .../test/Transforms/VectorCombine/X86/load.ll |   11 +-
 .../Transforms/VectorCombine/X86/pr114901.ll  |    6 +-
 .../VectorCombine/X86/shuffle-of-selects.ll   |   10 +-
 .../VectorCombine/intrinsic-scalarize.ll      |    6 +-
 .../lower-builtin-allow-check-pipeline.ll     |    3 +
 .../lower-builtin-allow-check-remarks.ll      |   12 +
 llvm/test/Verifier/assume-bundles.ll          |    4 +
 llvm/test/Verifier/branch-weight.ll           |  151 +
 llvm/test/Verifier/dead-on-return.ll          |    7 +
 llvm/test/Verifier/value-profile.ll           |   38 +
 .../Inputs/nvptx-basic.ll.expected            |    8 +-
 .../X86/global_downgraded_to_static.c         |    2 +-
 llvm/test/tools/dxil-dis/shuffle.ll           |    2 +-
 llvm/test/tools/llc/new-pm/pipeline.mir       |    8 +
 .../llc/new-pm/x86_64-regalloc-pipeline.mir   |    2 +-
 .../Inputs/binary-formats.canonical.json      |    2 +-
 .../test/tools/llvm-cov/mcdc-export-json.test |    8 +-
 .../WebAssembly/wasm-32bit-tombstone.s        |  107 +
 .../AArch64/template_ref_ptr.ll               |    2 +-
 .../macho-gsym-callsite-info-obj.test         |    6 +-
 .../AArch64/Neoverse/N2-sve-instructions.s    |   18 +-
 .../llvm-mca/AArch64/Neoverse/N2-writeback.s  | 4014 +++++-----
 .../X86/LunarlakeP/independent-load-stores.s  |  148 +
 .../LunarlakeP/partially-overlapping-groups.s |   21 +
 .../llvm-mca/X86/LunarlakeP/resources-aes.s   |   68 +
 .../llvm-mca/X86/LunarlakeP/resources-avx1.s  | 2428 +++++++
 .../llvm-mca/X86/LunarlakeP/resources-avx2.s  | 1092 +++
 .../X86/LunarlakeP/resources-avxgfni.s        |   82 +
 .../X86/LunarlakeP/resources-avxvnni.s        |   96 +
 .../llvm-mca/X86/LunarlakeP/resources-bmi2.s  |  152 +
 .../X86/LunarlakeP/resources-clflushopt.s     |   44 +
 .../llvm-mca/X86/LunarlakeP/resources-clwb.s  |   44 +
 .../llvm-mca/X86/LunarlakeP/resources-cmov.s  |  334 +
 .../X86/LunarlakeP/resources-cmpxchg.s        |   53 +
 .../llvm-mca/X86/LunarlakeP/resources-f16c.s  |   68 +
 .../llvm-mca/X86/LunarlakeP/resources-fma.s   |  712 ++
 .../X86/LunarlakeP/resources-fsgsbase.s       |   68 +
 .../llvm-mca/X86/LunarlakeP/resources-gfni.s  |   61 +
 .../llvm-mca/X86/LunarlakeP/resources-lea.s   |  448 ++
 .../llvm-mca/X86/LunarlakeP/resources-lzcnt.s |   61 +
 .../llvm-mca/X86/LunarlakeP/resources-mmx.s   |  404 ++
 .../llvm-mca/X86/LunarlakeP/resources-movbe.s |   61 +
 .../X86/LunarlakeP/resources-pclmul.s         |   47 +
 .../X86/LunarlakeP/resources-popcnt.s         |   61 +
 .../X86/LunarlakeP/resources-prefetchw.s      |   47 +
 .../X86/LunarlakeP/resources-rdrand.s         |   50 +
 .../X86/LunarlakeP/resources-rdseed.s         |   50 +
 .../llvm-mca/X86/LunarlakeP/resources-sse1.s  |  472 ++
 .../llvm-mca/X86/LunarlakeP/resources-sse2.s  |  971 +++
 .../llvm-mca/X86/LunarlakeP/resources-sse3.s  |  115 +
 .../llvm-mca/X86/LunarlakeP/resources-sse41.s |  377 +
 .../llvm-mca/X86/LunarlakeP/resources-sse42.s |  110 +
 .../llvm-mca/X86/LunarlakeP/resources-ssse3.s |  264 +
 .../llvm-mca/X86/LunarlakeP/resources-vaes.s  |   68 +
 .../X86/LunarlakeP/resources-vpclmulqdq.s     |   47 +
 .../X86/LunarlakeP/resources-x86_32.s         |   89 +
 .../X86/LunarlakeP/resources-x86_64.s         | 2886 ++++++++
 .../llvm-mca/X86/LunarlakeP/resources-x87.s   |  532 ++
 .../llvm-mca/X86/LunarlakeP/resources-xsave.s |   60 +
 .../llvm-mca/X86/LunarlakeP/zero-idioms.s     |  502 ++
 llvm/test/tools/llvm-mca/X86/show-encoding.s  |    2 +-
 .../tools/llvm-objcopy/COFF/string-table.test |   34 +
 .../symbolize-operands-executable.yaml        |   67 +
 .../AArch64/symbolize-operands-relocatable.s  |   79 +
 ...ddrmap-disassemble-symbolize-operands.yaml |   34 +-
 .../Inputs/expected-origin.html               |  124 +
 .../Inputs/expected-skipped.html              |    9 +
 .../Inputs/origin.json                        |    1 +
 .../llvm-original-di-preservation/basic.test  |    4 +
 llvm/tools/lli/lli.cpp                        |    4 +-
 llvm/tools/llvm-c-test/debuginfo.c            |    8 +-
 llvm/tools/llvm-cov/CoverageExporterJson.cpp  |    6 +-
 llvm/tools/llvm-exegesis/lib/Analysis.cpp     |    9 +-
 .../llvm-exegesis/lib/MCInstrDescView.cpp     |   12 +-
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      |    1 +
 llvm/tools/llvm-lto/llvm-lto.cpp              |    2 +
 llvm/tools/llvm-mc/Disassembler.cpp           |    2 +-
 .../llvm-mca/Views/BottleneckAnalysis.cpp     |    3 +-
 .../llvm-mca/Views/RegisterFileStatistics.cpp |    4 +-
 llvm/tools/llvm-mca/Views/TimelineView.cpp    |    6 +-
 llvm/tools/llvm-ml/Disassembler.cpp           |    2 +-
 llvm/tools/llvm-objdump/llvm-objdump.cpp      |    3 +-
 .../llvm-pdbutil/MinimalSymbolDumper.cpp      |    8 +
 llvm/tools/llvm-profdata/llvm-profdata.cpp    |    5 +-
 llvm/tools/llvm-profgen/ProfileGenerator.cpp  |    2 -
 llvm/tools/llvm-remarkutil/RemarkCounter.cpp  |   17 +-
 llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp        |    5 +-
 llvm/tools/lto/lto.cpp                        |    9 +
 llvm/tools/lto/lto.exports                    |    2 +
 llvm/unittests/ADT/ArrayRefTest.cpp           |   71 +-
 llvm/unittests/ADT/DynamicAPIntTest.cpp       |   10 +
 llvm/unittests/ADT/STLExtrasTest.cpp          |   11 +
 llvm/unittests/ADT/STLForwardCompatTest.cpp   |   19 +
 llvm/unittests/Analysis/AliasAnalysisTest.cpp |    3 +-
 .../Analysis/BasicAliasAnalysisTest.cpp       |    3 +-
 llvm/unittests/Analysis/DDGTest.cpp           |    2 +-
 .../FunctionPropertiesAnalysisTest.cpp        |  118 +-
 llvm/unittests/Analysis/IR2VecTest.cpp        |  120 +-
 llvm/unittests/Analysis/IVDescriptorsTest.cpp |    2 +-
 llvm/unittests/Analysis/LoadsTest.cpp         |    2 +-
 llvm/unittests/Analysis/LoopInfoTest.cpp      |    2 +-
 llvm/unittests/Analysis/LoopNestTest.cpp      |    2 +-
 .../Analysis/MemoryProfileInfoTest.cpp        |    3 +-
 llvm/unittests/Analysis/MemorySSATest.cpp     |    3 +-
 .../Analysis/ScalarEvolutionTest.cpp          |    3 +-
 .../Analysis/TargetLibraryInfoTest.cpp        |    2 +-
 .../unittests/Analysis/UnrollAnalyzerTest.cpp |    2 +-
 llvm/unittests/Analysis/ValueTrackingTest.cpp |   59 +-
 .../CodeGen/SelectionDAGPatternMatchTest.cpp  |  121 +
 .../EPCGenericJITLinkMemoryManagerTest.cpp    |    1 +
 .../Orc/EPCGenericMemoryAccessTest.cpp        |  365 +-
 ...ecutionSessionWrapperFunctionCallsTest.cpp |    3 +-
 .../Orc/ObjectLinkingLayerTest.cpp            |    2 +
 .../ExecutionEngine/Orc/OrcTestCommon.h       |   42 +
 .../Orc/RTDyldObjectLinkingLayerTest.cpp      |   42 +-
 .../Orc/ReOptimizeLayerTest.cpp               |    5 +-
 .../Orc/SharedMemoryMapperTest.cpp            |    1 +
 .../Orc/ThreadSafeModuleTest.cpp              |  123 +-
 llvm/unittests/FileCheck/FileCheckTest.cpp    |    8 +-
 llvm/unittests/Frontend/CMakeLists.txt        |    2 +
 .../Frontend/HLSLRootSignatureDumpTest.cpp    |  136 +-
 .../Frontend/HLSLRootSignatureRangesTest.cpp  |    2 +-
 .../OpenMPDirectiveNameParserTest.cpp         |  171 +
 .../Frontend/OpenMPDirectiveNameTest.cpp      |   96 +
 llvm/unittests/FuzzMutate/StrategiesTest.cpp  |   68 +-
 llvm/unittests/IR/BasicBlockDbgInfoTest.cpp   |    2 +-
 llvm/unittests/IR/DebugInfoTest.cpp           |   29 +
 .../unittests/IR/DebugTypeODRUniquingTest.cpp |    6 +-
 llvm/unittests/IR/IRBuilderTest.cpp           |    8 +-
 llvm/unittests/IR/LegacyPassManagerTest.cpp   |    3 +-
 llvm/unittests/IR/MetadataTest.cpp            |   76 +-
 llvm/unittests/Object/DXContainerTest.cpp     |    8 +-
 llvm/unittests/Object/ELFObjectFileTest.cpp   |    7 +-
 .../ObjectYAML/DXContainerYAMLTest.cpp        |    4 +-
 llvm/unittests/ProfileData/InstrProfTest.cpp  |   20 +-
 llvm/unittests/SandboxIR/UtilsTest.cpp        |    2 +-
 llvm/unittests/Support/Base64Test.cpp         |    2 +-
 llvm/unittests/Support/ParallelTest.cpp       |    6 +-
 .../Support/RISCVAttributeParserTest.cpp      |    2 +-
 .../unittests/Support/TrailingObjectsTest.cpp |    7 +-
 .../Target/RISCV/RISCVInstrInfoTest.cpp       |    6 +-
 llvm/unittests/TargetParser/Host.cpp          |    8 +
 .../TargetParser/RISCVISAInfoTest.cpp         |    2 +
 .../TargetParser/TargetParserTest.cpp         |   13 +-
 llvm/unittests/TargetParser/TripleTest.cpp    |  190 +
 .../Instrumentation/MemProfUseTest.cpp        |    8 +-
 .../Transforms/Utils/BasicBlockUtilsTest.cpp  |    4 +-
 .../Transforms/Utils/CloningTest.cpp          |    4 +-
 .../Transforms/Utils/CodeMoverUtilsTest.cpp   |    8 +-
 .../Utils/LoopRotationUtilsTest.cpp           |    4 +-
 .../Transforms/Utils/LoopUtilsTest.cpp        |    2 +-
 .../Utils/ScalarEvolutionExpanderTest.cpp     |    3 +-
 .../Transforms/Utils/UnrollLoopTest.cpp       |    2 +-
 .../SandboxVectorizer/DependencyGraphTest.cpp |   15 +-
 .../SandboxVectorizer/LegalityTest.cpp        |    9 +-
 .../SandboxVectorizer/SchedulerTest.cpp       |   15 +-
 .../SandboxVectorizer/SeedCollectorTest.cpp   |   29 +-
 .../SandboxVectorizer/VecUtilsTest.cpp        |   10 +-
 .../Transforms/Vectorize/VPlanSlpTest.cpp     |   13 +-
 .../Transforms/Vectorize/VPlanTest.cpp        |   39 +-
 .../Transforms/Vectorize/VPlanTestBase.h      |   11 +-
 llvm/utils/TableGen/AsmMatcherEmitter.cpp     |   28 +-
 llvm/utils/TableGen/AsmWriterEmitter.cpp      |    2 +-
 llvm/utils/TableGen/Basic/CMakeLists.txt      |    1 +
 .../TableGen/Basic/RuntimeLibcallsEmitter.cpp |  345 +
 llvm/utils/TableGen/CodeEmitterGen.cpp        |   11 +-
 llvm/utils/TableGen/CodeGenMapTable.cpp       |    2 +-
 .../TableGen/Common/CodeGenDAGPatterns.cpp    |    7 +-
 .../TableGen/Common/CodeGenInstruction.cpp    |   47 +-
 .../TableGen/Common/CodeGenInstruction.h      |   15 +-
 .../TableGen/Common/CodeGenRegisters.cpp      |    2 +-
 .../utils/TableGen/Common/CodeGenSchedule.cpp |   34 +-
 llvm/utils/TableGen/Common/CodeGenTarget.cpp  |   28 +-
 llvm/utils/TableGen/Common/CodeGenTarget.h    |   47 +-
 .../GlobalISel/GlobalISelMatchTable.cpp       |    2 +-
 .../TableGen/Common/VarLenCodeEmitterGen.cpp  |    2 +-
 llvm/utils/TableGen/CompressInstEmitter.cpp   |  198 +-
 llvm/utils/TableGen/DAGISelMatcherEmitter.cpp |    2 +-
 llvm/utils/TableGen/DAGISelMatcherOpt.cpp     |    4 +-
 llvm/utils/TableGen/DecoderEmitter.cpp        |  334 +-
 llvm/utils/TableGen/DisassemblerEmitter.cpp   |    5 +-
 llvm/utils/TableGen/InstrDocsEmitter.cpp      |    2 +-
 llvm/utils/TableGen/InstrInfoEmitter.cpp      |   37 +-
 llvm/utils/TableGen/SubtargetEmitter.cpp      |   22 +-
 llvm/utils/TableGen/X86FoldTablesEmitter.cpp  |    2 +-
 .../utils/TableGen/X86InstrMappingEmitter.cpp |    3 +-
 llvm/utils/TableGen/X86MnemonicTables.cpp     |    2 +-
 .../utils/gn/secondary/bolt/lib/Core/BUILD.gn |    1 +
 .../gn/secondary/bolt/lib/Rewrite/BUILD.gn    |    1 +
 .../clang-tidy/modernize/BUILD.gn             |    1 +
 .../gn/secondary/clang/lib/Driver/BUILD.gn    |    1 -
 .../secondary/clang/lib/Interpreter/BUILD.gn  |    1 +
 llvm/utils/gn/secondary/clang/test/BUILD.gn   |   11 +-
 .../gn/secondary/clang/unittests/BUILD.gn     |    7 +
 .../gn/secondary/libcxx/include/BUILD.gn      |   13 +-
 .../source/Plugins/Process/Utility/BUILD.gn   |    2 -
 .../source/Plugins/Process/mach-core/BUILD.gn |    1 +
 .../llvm/include/llvm/Config/BUILD.gn         |    1 +
 .../secondary/llvm/include/llvm/IR/BUILD.gn   |    9 +
 .../llvm/lib/CodeGen/AsmPrinter/BUILD.gn      |    1 +
 .../gn/secondary/llvm/lib/CodeGen/BUILD.gn    |    1 +
 .../llvm/lib/DWARFLinker/Classic/BUILD.gn     |    1 +
 .../llvm/lib/DWARFLinker/Parallel/BUILD.gn    |    1 +
 .../llvm/lib/DebugInfo/DWARF/BUILD.gn         |    4 +-
 .../lib/DebugInfo/DWARF/LowLevel/BUILD.gn     |   12 +
 .../llvm/lib/DebugInfo/LogicalView/BUILD.gn   |    1 +
 .../llvm/lib/ExecutionEngine/Orc/BUILD.gn     |    2 +
 .../secondary/llvm/lib/Frontend/HLSL/BUILD.gn |    4 +-
 .../llvm/lib/Frontend/OpenMP/BUILD.gn         |    1 +
 llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn  |    1 +
 .../secondary/llvm/lib/ProfileData/BUILD.gn   |    1 +
 .../secondary/llvm/lib/Target/AMDGPU/BUILD.gn |    1 +
 .../Target/LoongArch/MCTargetDesc/BUILD.gn    |    1 -
 .../lib/Target/PowerPC/MCTargetDesc/BUILD.gn  |    1 -
 .../gn/secondary/llvm/tools/dsymutil/BUILD.gn |    1 +
 .../llvm/tools/llvm-dwarfdump/BUILD.gn        |    1 +
 .../llvm/tools/llvm-dwarfutil/BUILD.gn        |    1 +
 .../llvm/tools/llvm-objdump/BUILD.gn          |    1 +
 .../llvm/unittests/DebugInfo/DWARF/BUILD.gn   |    1 +
 .../llvm/unittests/Frontend/BUILD.gn          |    2 +
 .../llvm/utils/TableGen/Basic/BUILD.gn        |    1 +
 llvm/utils/lit/lit/llvm/config.py             |    3 +-
 llvm/utils/llvm-original-di-preservation.py   |   36 +-
 llvm/utils/release/github-upload-release.py   |   43 +-
 mlir/cmake/modules/IRDLToCpp.cmake            |    2 +-
 mlir/docs/Bindings/Python.md                  |    7 +-
 mlir/docs/Tutorials/transform/Ch1.md          |   44 +-
 mlir/docs/Tutorials/transform/Ch2.md          |    4 +-
 mlir/docs/Tutorials/transform/Ch4.md          |   18 +-
 mlir/docs/Tutorials/transform/ChH.md          |    2 +-
 mlir/examples/toy/Ch2/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch2/mlir/MLIRGen.cpp        |    2 +-
 mlir/examples/toy/Ch3/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch3/mlir/MLIRGen.cpp        |    2 +-
 mlir/examples/toy/Ch4/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch4/mlir/MLIRGen.cpp        |    2 +-
 mlir/examples/toy/Ch5/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch5/mlir/MLIRGen.cpp        |    2 +-
 mlir/examples/toy/Ch6/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp    |    2 +-
 mlir/examples/toy/Ch6/mlir/MLIRGen.cpp        |    2 +-
 mlir/examples/toy/Ch7/include/toy/Ops.td      |    2 +-
 mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp    |    2 +-
 mlir/examples/toy/Ch7/mlir/MLIRGen.cpp        |    6 +-
 .../mlir/Conversion/LLVMCommon/Pattern.h      |    4 +
 mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td |   34 +
 .../mlir/Dialect/AMDGPU/Transforms/Passes.h   |    6 +-
 .../mlir/Dialect/AMDGPU/Transforms/Passes.td  |    4 +-
 .../mlir/Dialect/Affine/IR/AffineOps.td       |   25 +-
 mlir/include/mlir/Dialect/Affine/Passes.td    |   17 +
 mlir/include/mlir/Dialect/Affine/Utils.h      |   14 +-
 mlir/include/mlir/Dialect/Arith/IR/Arith.h    |    9 +
 mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td  |   28 +
 .../include/mlir/Dialect/ArmNeon/Transforms.h |    2 +-
 mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td |   29 +
 .../include/mlir/Dialect/Async/IR/AsyncOps.td |    2 +-
 .../IR/BufferizableOpInterface.h              |    2 +-
 .../IR/BufferizableOpInterface.td             |    2 +-
 .../Bufferization/IR/BufferizationOps.td      |    6 +-
 .../IR/BufferizationTypeInterfaces.h          |    1 -
 .../IR/UnstructuredControlFlow.h              |    4 +-
 mlir/include/mlir/Dialect/EmitC/IR/EmitC.td   |   97 +
 .../mlir/Dialect/EmitC/Transforms/Passes.h    |    1 +
 .../mlir/Dialect/EmitC/Transforms/Passes.td   |   38 +
 .../Dialect/EmitC/Transforms/Transforms.h     |    4 +
 mlir/include/mlir/Dialect/Func/IR/FuncOps.td  |    2 +-
 .../Dialect/GPU/IR/GPUDeviceMappingAttr.td    |   24 +
 mlir/include/mlir/Dialect/GPU/IR/GPUOps.td    |   52 +-
 .../mlir/Dialect/GPU/TransformOps/Utils.h     |   40 +-
 .../mlir/Dialect/LLVMIR/CMakeLists.txt        |   10 +
 .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td |    3 +-
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   |    2 +-
 mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td   |   14 +-
 .../include/mlir/Dialect/LLVMIR/XeVMDialect.h |   28 +
 mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td   |  561 ++
 mlir/include/mlir/Dialect/Linalg/IR/Linalg.h  |   18 +
 .../mlir/Dialect/Linalg/IR/LinalgBase.td      |    3 +-
 .../mlir/Dialect/Linalg/IR/LinalgEnums.td     |   15 +
 .../Dialect/Linalg/IR/LinalgInterfaces.td     |    2 +-
 .../Linalg/IR/LinalgNamedStructuredOps.yaml   |  114 -
 .../mlir/Dialect/Linalg/IR/LinalgOps.td       |   18 +-
 .../Dialect/Linalg/IR/LinalgRelayoutOps.td    |   76 +-
 .../Dialect/Linalg/IR/LinalgStructuredOps.td  |   50 +-
 .../Linalg/TransformOps/LinalgTransformOps.td |    5 +-
 .../Dialect/Linalg/Transforms/Transforms.h    |   61 +-
 .../mlir/Dialect/MLProgram/IR/MLProgramOps.td |    4 +-
 .../mlir/Dialect/MemRef/IR/MemRefOps.td       |    1 +
 .../mlir/Dialect/MemRef/Transforms/Passes.td  |   62 +
 .../Dialect/MemRef/Transforms/Transforms.h    |    2 +-
 mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h   |    2 +-
 .../mlir/Dialect/OpenACC/OpenACCOps.td        |  179 +-
 .../Dialect/OpenMP/OpenMPClauseOperands.h     |    6 +-
 .../mlir/Dialect/OpenMP/OpenMPDialect.h       |    3 +-
 .../mlir/Dialect/OpenMP/OpenMPInterfaces.h    |    3 +-
 .../mlir/Dialect/OpenMP/OpenMPOpsAttributes.h |   18 +
 .../mlir/Dialect/OpenMP/OpenMPOpsEnums.h      |   19 +
 mlir/include/mlir/Dialect/SCF/IR/SCFOps.td    |    8 +-
 .../SCF/Transforms/TileUsingInterface.h       |   22 +-
 mlir/include/mlir/Dialect/SMT/IR/SMTOps.td    |    2 +-
 .../mlir/Dialect/SPIRV/IR/SPIRVBase.td        |   34 +-
 .../SPIRV/IR/SPIRVCooperativeMatrixOps.td     |   16 +-
 .../mlir/Dialect/SPIRV/IR/SPIRVTypes.h        |   44 +-
 .../include/mlir/Dialect/Shape/IR/ShapeOps.td |    4 +-
 .../mlir/Dialect/Tensor/IR/TensorOps.td       |    7 +-
 .../Dialect/Tensor/Transforms/Transforms.h    |   14 +-
 mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h |   18 +-
 mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td  |   13 +-
 .../mlir/Dialect/Tosa/Transforms/Passes.td    |   14 +
 .../DebugExtension/DebugExtensionOps.td       |    4 +-
 .../mlir/Dialect/Utils/ReshapeOpsUtils.h      |   38 +-
 mlir/include/mlir/Dialect/Vector/IR/Vector.td |    5 +-
 .../mlir/Dialect/Vector/IR/VectorOps.td       |   44 +-
 .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td |  132 +-
 .../mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h   |   30 +
 .../mlir/Dialect/XeGPU/Utils/XeGPUUtils.h     |   14 -
 .../mlir/ExecutionEngine/MemRefUtils.h        |    8 +-
 mlir/include/mlir/IR/EnumAttr.td              |    7 +-
 mlir/include/mlir/IR/Matchers.h               |    2 +-
 mlir/include/mlir/IR/ODSSupport.h             |    3 +
 mlir/include/mlir/IR/OpDefinition.h           |    2 +-
 mlir/include/mlir/IR/OpImplementation.h       |    3 +
 mlir/include/mlir/IR/PatternMatch.h           |    5 +-
 mlir/include/mlir/IR/Unit.h                   |    2 +-
 mlir/include/mlir/IR/Visitors.h               |   36 +-
 mlir/include/mlir/InitAllDialects.h           |    4 +-
 .../mlir/Interfaces/InferTypeOpInterface.td   |    5 +-
 .../mlir/Interfaces/TilingInterface.td        |   55 +-
 mlir/include/mlir/Support/StateStack.h        |  128 +
 mlir/include/mlir/Support/WalkResult.h        |   59 +
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |   72 +-
 mlir/include/mlir/Tools/PDLL/AST/Nodes.h      |    4 +-
 .../mlir/Transforms/DialectConversion.h       |   13 +-
 mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp  |    1 -
 mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp |    1 -
 .../Analysis/FlatLinearValueConstraints.cpp   |    2 -
 .../Analysis/Presburger/IntegerRelation.cpp   |   12 +-
 mlir/lib/Analysis/Presburger/PWMAFunction.cpp |    2 +-
 mlir/lib/Analysis/Presburger/Simplex.cpp      |    1 -
 mlir/lib/Analysis/Presburger/Utils.cpp        |    1 -
 mlir/lib/Analysis/SliceAnalysis.cpp           |    1 -
 mlir/lib/AsmParser/AttributeParser.cpp        |    2 +-
 mlir/lib/Bindings/Python/IRAttributes.cpp     |    2 +-
 mlir/lib/CAPI/Dialect/Linalg.cpp              |    2 +-
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           |   79 +-
 .../AffineToStandard/AffineToStandard.cpp     |    1 -
 .../Conversion/ArithToLLVM/ArithToLLVM.cpp    |    1 -
 .../Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp  |    1 -
 .../Conversion/AsyncToLLVM/AsyncToLLVM.cpp    |    1 -
 .../BufferizationToMemRef.cpp                 |    3 +-
 .../ComplexToLLVM/ComplexToLLVM.cpp           |    1 -
 .../ComplexToLibm/ComplexToLibm.cpp           |    1 -
 .../ComplexToSPIRV/ComplexToSPIRV.cpp         |    2 -
 .../ComplexToStandard/ComplexToStandard.cpp   |    2 -
 .../ControlFlowToLLVM/ControlFlowToLLVM.cpp   |    3 -
 .../ControlFlowToSCF/ControlFlowToSCF.cpp     |    1 -
 .../ControlFlowToSPIRV/ControlFlowToSPIRV.cpp |    3 -
 .../ConvertToLLVM/ConvertToLLVMPass.cpp       |    2 -
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp |   11 -
 .../Conversion/FuncToSPIRV/FuncToSPIRV.cpp    |    3 -
 .../GPUCommon/GPUToLLVMConversion.cpp         |    4 -
 .../GPUToNVVM/LowerGpuOpsToNVVMOps.cpp        |    1 -
 .../GPUToROCDL/LowerGpuOpsToROCDLOps.cpp      |    3 -
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp |   52 +-
 .../Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp  |    1 -
 .../Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp  |    3 -
 .../Conversion/IndexToSPIRV/IndexToSPIRV.cpp  |    1 -
 mlir/lib/Conversion/LLVMCommon/Pattern.cpp    |    9 +-
 .../Conversion/LLVMCommon/TypeConverter.cpp   |    1 -
 .../LinalgToStandard/LinalgToStandard.cpp     |    1 -
 mlir/lib/Conversion/MathToLibm/MathToLibm.cpp |    1 -
 .../Conversion/MathToROCDL/MathToROCDL.cpp    |    2 -
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    |    2 -
 .../Conversion/MemRefToLLVM/MemRefToLLVM.cpp  |   12 +-
 .../MemRefToSPIRV/MemRefToSPIRV.cpp           |    1 -
 .../Conversion/OpenACCToSCF/OpenACCToSCF.cpp  |    2 -
 .../Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp  |    1 -
 .../PDLToPDLInterp/PDLToPDLInterp.cpp         |   10 +-
 .../PDLToPDLInterp/PredicateTree.cpp          |    3 -
 .../SCFToControlFlow/SCFToControlFlow.cpp     |    2 -
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |    3 -
 mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp     |    1 -
 mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp |    6 -
 .../Conversion/SCFToOpenMP/SCFToOpenMP.cpp    |    1 -
 mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp |    2 -
 .../Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp  |    1 -
 .../ConvertLaunchFuncToLLVMCalls.cpp          |    7 +-
 .../Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp    |    3 -
 .../ConvertShapeConstraints.cpp               |    2 -
 .../TensorToLinalg/TensorToLinalg.cpp         |    3 -
 .../TensorToSPIRV/TensorToSPIRV.cpp           |    3 -
 .../Conversion/TosaToArith/TosaToArith.cpp    |    1 -
 .../TosaToArith/TosaToArithPass.cpp           |    3 -
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  |   18 +-
 .../TosaToLinalg/TosaToLinalgNamed.cpp        |    8 -
 .../TosaToLinalg/TosaToLinalgNamedPass.cpp    |    2 -
 .../TosaToLinalg/TosaToLinalgPass.cpp         |    2 -
 .../TosaToMLProgram/TosaToMLProgramPass.cpp   |    1 -
 mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp   |    2 -
 .../Conversion/TosaToSCF/TosaToSCFPass.cpp    |    2 -
 .../Conversion/TosaToTensor/TosaToTensor.cpp  |    1 -
 .../TosaToTensor/TosaToTensorPass.cpp         |    2 -
 .../Conversion/VectorToGPU/VectorToGPU.cpp    |    4 -
 .../VectorToLLVM/ConvertVectorToLLVM.cpp      |   35 +-
 .../VectorToLLVM/ConvertVectorToLLVMPass.cpp  |    3 +-
 .../Conversion/VectorToSCF/VectorToSCF.cpp    |   16 +-
 .../VectorToSPIRV/VectorToSPIRV.cpp           |   49 +-
 .../VectorToSPIRV/VectorToSPIRVPass.cpp       |    1 -
 .../VectorToXeGPU/VectorToXeGPU.cpp           |    1 -
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |   34 +
 .../Dialect/AMDGPU/Transforms/CMakeLists.txt  |    2 +-
 .../AMDGPU/Transforms/MaskedloadToLoad.cpp    |  167 +
 .../AMDGPU/Transforms/TransferReadToLoad.cpp  |  239 -
 .../Affine/Analysis/AffineAnalysis.cpp        |    4 +-
 .../Affine/Analysis/AffineStructures.cpp      |    2 +-
 mlir/lib/Dialect/Affine/Analysis/Utils.cpp    |    2 +-
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp      |  119 +-
 .../Dialect/Affine/Transforms/LoopFusion.cpp  |    9 +-
 .../Dialect/Affine/Transforms/LoopTiling.cpp  |    7 +-
 .../Transforms/PipelineDataTransfer.cpp       |   17 +-
 .../Transforms/SimplifyAffineMinMax.cpp       |   95 +-
 .../Affine/Transforms/SuperVectorize.cpp      |    3 +-
 mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp   |   22 +-
 mlir/lib/Dialect/Affine/Utils/Utils.cpp       |   55 +-
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp        |   16 +
 .../BufferizableOpInterfaceImpl.cpp           |    8 +-
 .../ArmNeonVectorTransformOps.cpp             |    2 +-
 .../Dialect/ArmNeon/Transforms/CMakeLists.txt |    2 +-
 ... => LowerContractionToNeonI8MMPattern.cpp} |  178 +-
 .../ArmSME/Transforms/VectorLegalization.cpp  |  171 +-
 .../Transforms/LegalizeForLLVMExport.cpp      |    3 +-
 .../Transforms/LegalizeVectorStorage.cpp      |  153 +-
 .../LowerContractionToSVEI8MMPattern.cpp      |    7 +-
 mlir/lib/Dialect/Async/IR/Async.cpp           |    2 +-
 .../Async/Transforms/AsyncToAsyncRuntime.cpp  |    2 +-
 .../IR/BufferizableOpInterface.cpp            |   27 +-
 .../Bufferization/IR/BufferizationOps.cpp     |    5 +-
 .../FuncBufferizableOpInterfaceImpl.cpp       |   15 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           |   45 +-
 .../Dialect/EmitC/Transforms/CMakeLists.txt   |    1 +
 .../EmitC/Transforms/WrapFuncInClass.cpp      |  112 +
 mlir/lib/Dialect/Func/IR/FuncOps.cpp          |    2 +-
 mlir/lib/Dialect/GPU/IR/GPUDialect.cpp        |   65 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |   48 +-
 mlir/lib/Dialect/GPU/TransformOps/Utils.cpp   |  175 +-
 .../GPU/Transforms/DecomposeMemRefs.cpp       |    3 +-
 mlir/lib/Dialect/LLVMIR/CMakeLists.txt        |   23 +
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    |   47 +-
 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp    |  366 +
 mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp  |    8 -
 .../Dialect/Linalg/IR/LinalgInterfaces.cpp    |    2 -
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      |  241 +-
 .../Linalg/IR/ValueBoundsOpInterfaceImpl.cpp  |    1 -
 .../Linalg/TransformOps/DialectExtension.cpp  |    2 -
 .../Linalg/TransformOps/LinalgMatchOps.cpp    |    1 -
 .../TransformOps/LinalgTransformOps.cpp       |   29 +-
 .../Linalg/Transforms/BlockPackMatmul.cpp     |    1 -
 .../Transforms/BubbleUpExtractSlice.cpp       |    3 -
 .../BufferizableOpInterfaceImpl.cpp           |    2 -
 .../Linalg/Transforms/ConstantFold.cpp        |    2 -
 .../Transforms/ConvertConv2DToImg2Col.cpp     |    1 -
 .../Transforms/ConvertToDestinationStyle.cpp  |    2 -
 .../Transforms/DataLayoutPropagation.cpp      |   37 +-
 ...DecomposeGenericByUnfoldingPermutation.cpp |    2 -
 .../Dialect/Linalg/Transforms/Detensorize.cpp |    3 -
 .../Linalg/Transforms/DropUnitDims.cpp        |    3 -
 .../Linalg/Transforms/ElementwiseToLinalg.cpp |    1 -
 .../Transforms/EliminateEmptyTensors.cpp      |    1 -
 .../Linalg/Transforms/FoldIntoElementwise.cpp |    1 -
 .../FusePadOpWithLinalgProducer.cpp           |    1 -
 mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp |   11 -
 .../Linalg/Transforms/Generalization.cpp      |    5 -
 .../Linalg/Transforms/HoistPadding.cpp        |    1 -
 .../Dialect/Linalg/Transforms/Hoisting.cpp    |   52 +-
 .../Dialect/Linalg/Transforms/Interchange.cpp |    5 -
 mlir/lib/Dialect/Linalg/Transforms/Loops.cpp  |    1 -
 .../Transforms/MeshShardingInterfaceImpl.cpp  |    3 -
 .../Transforms/PackAndUnpackPatterns.cpp      |   84 +-
 .../Linalg/Transforms/PadTilingInterface.cpp  |    5 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |   17 +-
 .../Dialect/Linalg/Transforms/Specialize.cpp  |    2 -
 mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp |    6 -
 .../Linalg/Transforms/TilingInterfaceImpl.cpp |  191 +-
 .../Dialect/Linalg/Transforms/Transforms.cpp  |   36 +-
 .../Linalg/Transforms/TransposeConv2D.cpp     |    6 -
 .../Linalg/Transforms/TransposeMatmul.cpp     |    5 +-
 .../Linalg/Transforms/Vectorization.cpp       |  169 +-
 .../Linalg/Transforms/WinogradConv2D.cpp      |  153 +-
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp       |    3 -
 mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp      |   49 +-
 .../Dialect/MemRef/Transforms/CMakeLists.txt  |    1 +
 .../MemRef/Transforms/NormalizeMemRefs.cpp    |    9 +-
 .../MemRef/Transforms/ReifyResultShapes.cpp   |  159 +
 .../Mesh/Transforms/ShardingPropagation.cpp   |    4 +-
 mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp       |  220 +-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   20 +
 mlir/lib/Dialect/SCF/IR/SCF.cpp               |    2 +-
 .../BufferizableOpInterfaceImpl.cpp           |   61 +-
 .../Dialect/SCF/Transforms/LoopPipelining.cpp |   29 +-
 .../SCF/Transforms/TileUsingInterface.cpp     |  216 +-
 mlir/lib/Dialect/SCF/Utils/Utils.cpp          |   15 +-
 .../Dialect/SPIRV/IR/CooperativeMatrixOps.cpp |   28 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp    |   75 +-
 mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp        |    6 +
 mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp      |  113 +-
 .../Transforms/LowerABIAttributesPass.cpp     |    4 +-
 mlir/lib/Dialect/Shape/IR/Shape.cpp           |    2 +-
 .../Transforms/SparseTensorConversion.cpp     |    2 +-
 .../Transforms/Utils/IterationGraphSorter.cpp |    4 +-
 .../Transforms/Utils/LoopEmitter.cpp          |    2 +-
 mlir/lib/Dialect/Tensor/IR/TensorOps.cpp      |   29 +-
 .../BufferizableOpInterfaceImpl.cpp           |   52 +-
 .../SwapExtractSliceWithProducerPatterns.cpp  |   57 +-
 .../Dialect/Tosa/IR/TosaCanonicalizations.cpp |   10 +-
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp          |    6 +-
 .../Dialect/Tosa/Transforms/CMakeLists.txt    |    1 +
 .../TosaConvertIntegerTypeToSignless.cpp      |  139 +
 .../Tosa/Transforms/TosaMakeBroadcastable.cpp |    6 +-
 .../Tosa/Transforms/TosaProfileCompliance.cpp |    4 +-
 .../Tosa/Transforms/TosaReduceTransposes.cpp  |    3 +-
 .../Tosa/Transforms/TosaValidation.cpp        |    9 +-
 .../DebugExtension/DebugExtensionOps.cpp      |   13 +-
 .../lib/Dialect/Transform/IR/TransformOps.cpp |    9 +-
 mlir/lib/Dialect/Utils/CMakeLists.txt         |    1 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp      |  138 +-
 .../Vector/Transforms/VectorDistribute.cpp    |  227 +-
 .../Transforms/VectorTransferOpTransforms.cpp |    3 +-
 mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp |    3 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp    |   35 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        |   33 +-
 .../XeGPU/Transforms/XeGPUPropagateLayout.cpp |   71 +-
 .../Transforms/XeGPUSubgroupDistribute.cpp    |   35 +-
 .../Dialect/XeGPU/Transforms/XeGPUUnroll.cpp  |   65 +-
 .../Transforms/XeGPUWgToSgDistribute.cpp      |   14 +-
 mlir/lib/IR/AffineExpr.cpp                    |   50 +-
 mlir/lib/IR/AffineMap.cpp                     |    4 -
 mlir/lib/IR/AsmPrinter.cpp                    |   17 +-
 mlir/lib/IR/Block.cpp                         |    2 -
 mlir/lib/IR/Builders.cpp                      |    3 -
 mlir/lib/IR/BuiltinAttributes.cpp             |    2 -
 mlir/lib/IR/BuiltinDialect.cpp                |    2 -
 mlir/lib/IR/BuiltinTypeInterfaces.cpp         |    2 -
 mlir/lib/IR/BuiltinTypes.cpp                  |    2 -
 mlir/lib/IR/Diagnostics.cpp                   |    2 +-
 mlir/lib/IR/Dialect.cpp                       |    3 -
 mlir/lib/IR/Dominance.cpp                     |    1 -
 mlir/lib/IR/MLIRContext.cpp                   |    4 -
 mlir/lib/IR/ODSSupport.cpp                    |    5 +
 mlir/lib/IR/Operation.cpp                     |   18 +-
 mlir/lib/IR/OperationSupport.cpp              |    1 -
 mlir/lib/IR/TypeRange.cpp                     |    1 -
 mlir/lib/IR/Unit.cpp                          |    1 -
 mlir/lib/Pass/Pass.cpp                        |    2 +-
 mlir/lib/Query/Matcher/VariantValue.cpp       |    4 +-
 mlir/lib/Reducer/Tester.cpp                   |    2 +-
 mlir/lib/Support/CMakeLists.txt               |    1 +
 .../lib/Support/StateStack.cpp                |   12 +-
 mlir/lib/TableGen/Argument.cpp                |    1 -
 mlir/lib/TableGen/Attribute.cpp               |    1 -
 mlir/lib/TableGen/Class.cpp                   |    2 -
 mlir/lib/TableGen/Interfaces.cpp              |    2 -
 mlir/lib/TableGen/Operator.cpp                |    2 -
 mlir/lib/TableGen/Predicate.cpp               |    1 -
 mlir/lib/TableGen/Property.cpp                |    1 -
 mlir/lib/TableGen/Trait.cpp                   |    2 -
 mlir/lib/Target/Cpp/TranslateToCpp.cpp        |   61 +-
 mlir/lib/Target/LLVM/ModuleToObject.cpp       |    3 -
 mlir/lib/Target/LLVM/NVVM/Target.cpp          |    3 -
 mlir/lib/Target/LLVM/ROCDL/Utils.cpp          |    1 -
 mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp |    1 -
 mlir/lib/Target/LLVMIR/DebugImporter.cpp      |    4 -
 mlir/lib/Target/LLVMIR/DebugTranslation.cpp   |    4 +-
 .../LLVMIR/LLVMIRToLLVMTranslation.cpp        |    4 -
 .../LLVMIR/LLVMToLLVMIRTranslation.cpp        |    1 -
 .../Dialect/NVVM/LLVMIRToNVVMTranslation.cpp  |    1 -
 .../Dialect/NVVM/NVVMToLLVMIRTranslation.cpp  |    1 -
 .../OpenACC/OpenACCToLLVMIRTranslation.cpp    |    2 -
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      |   58 +-
 .../SPIRV/SPIRVToLLVMIRTranslation.cpp        |    2 -
 .../Dialect/VCIX/VCIXToLLVMIRTranslation.cpp  |    1 -
 .../lib/Target/LLVMIR/LLVMImportInterface.cpp |    1 -
 .../LLVMIR/LoopAnnotationTranslation.cpp      |    2 +-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       |    1 -
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  |   22 +-
 mlir/lib/Target/LLVMIR/TypeToLLVM.cpp         |    1 -
 mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp       |    1 -
 .../SPIRV/Deserialization/DeserializeOps.cpp  |    1 +
 .../SPIRV/Deserialization/Deserializer.cpp    |   51 +
 .../SPIRV/Deserialization/Deserializer.h      |    2 +
 mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp    |    1 -
 .../SPIRV/Serialization/Serialization.cpp     |    1 -
 .../Target/SPIRV/Serialization/Serializer.cpp |   48 +
 .../Target/SPIRV/TranslateRegistration.cpp    |    4 -
 mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp       |    5 +-
 mlir/lib/Tools/PDLL/Parser/Parser.cpp         |    9 +-
 .../Transforms/LoopInvariantCodeMotion.cpp    |    1 -
 mlir/lib/Transforms/RemoveDeadValues.cpp      |    2 -
 mlir/lib/Transforms/StripDebugInfo.cpp        |    1 -
 mlir/lib/Transforms/SymbolDCE.cpp             |   49 +-
 mlir/lib/Transforms/Utils/CFGToSCF.cpp        |    2 -
 .../Transforms/Utils/DialectConversion.cpp    |  190 +-
 .../Utils/GreedyPatternRewriteDriver.cpp      |    2 -
 mlir/lib/Transforms/Utils/Inliner.cpp         |    4 +-
 mlir/lib/Transforms/Utils/InliningUtils.cpp   |    1 -
 mlir/lib/Transforms/Utils/RegionUtils.cpp     |    3 -
 mlir/lib/Transforms/ViewOpGraph.cpp           |    1 -
 mlir/python/CMakeLists.txt                    |    9 +
 .../dialects/TransformDebugExtensionOps.td    |   19 +
 .../linalg/opdsl/ops/core_named_ops.py        |   31 -
 mlir/python/mlir/dialects/transform/debug.py  |   81 +
 .../AMDGPUToROCDL/transpose_load.mlir         |   56 +
 .../AMDGPUToROCDL/transpose_load_reject.mlir  |   17 +
 mlir/test/Conversion/GPUToSPIRV/rotate.mlir   |  102 +
 .../vector-to-llvm-interface.mlir             |   40 +-
 .../VectorToSPIRV/vector-to-spirv.mlir        |   35 +
 mlir/test/Dialect/AMDGPU/invalid.mlir         |   56 +
 ...d-to-load.mlir => maskedload-to-load.mlir} |   78 +-
 mlir/test/Dialect/AMDGPU/ops.mlir             |    7 +
 .../Affine/SuperVectorize/vectorize_1d.mlir   |   12 +-
 .../vectorize_affine_apply.mlir               |    6 +-
 .../test/Dialect/Affine/affine-data-copy.mlir |   48 +
 .../Dialect/Affine/decompose-affine-ops.mlir  |   18 +-
 mlir/test/Dialect/Affine/loop-fusion-4.mlir   |   77 +
 .../Dialect/Affine/simplify-min-max-ops.mlir  |   60 +
 .../Dialect/Affine/simplify-structures.mlir   |    4 +-
 .../transform-op-simplify-min-max-ops.mlir    |    2 +-
 mlir/test/Dialect/ArmNeon/invalid.mlir        |   60 +
 .../Dialect/ArmNeon/lower-to-arm-neon.mlir    |   80 +-
 mlir/test/Dialect/ArmNeon/roundtrip.mlir      |   12 +
 .../Dialect/ArmSME/vector-legalization.mlir   |  101 +-
 mlir/test/Dialect/ArmSVE/invalid.mlir         |   60 +
 .../ArmSVE/legalize-transfer-read.mlir        |  257 +
 mlir/test/Dialect/ArmSVE/roundtrip.mlir       |   10 +
 .../Transforms/one-shot-bufferize.mlir        |   23 +-
 .../EmitC/wrap_emitc_func_in_class.mlir       |   40 +
 .../wrap_emitc_func_in_class_noAttr.mlir      |   17 +
 mlir/test/Dialect/GPU/invalid.mlir            |   78 +
 mlir/test/Dialect/GPU/ops.mlir                |    4 +
 mlir/test/Dialect/GPU/transform-gpu.mlir      |   64 +
 mlir/test/Dialect/LLVMIR/invalid.mlir         |   45 +-
 mlir/test/Dialect/LLVMIR/xevm.mlir            |   95 +
 .../Linalg/data-layout-propagation.mlir       |   52 +-
 mlir/test/Dialect/Linalg/decompose-pack.mlir  |   45 +
 .../test/Dialect/Linalg/decompose-unpack.mlir |   34 +
 .../generalize-named-polymorphic-ops.mlir     |  111 -
 mlir/test/Dialect/Linalg/hoisting.mlir        |  495 +-
 mlir/test/Dialect/Linalg/invalid.mlir         |   74 +-
 mlir/test/Dialect/Linalg/library-calls.mlir   |   40 -
 .../Dialect/Linalg/match-ops-interpreter.mlir |    6 +-
 mlir/test/Dialect/Linalg/named-ops.mlir       |   95 +
 .../Linalg/one-shot-bufferize-analysis.mlir   |   38 -
 mlir/test/Dialect/Linalg/pad_fusion.mlir      |    6 +-
 .../Dialect/Linalg/promotion_options.mlir     |   56 +
 .../resolve-shaped-type-result-dims.mlir      |    4 +-
 mlir/test/Dialect/Linalg/roundtrip.mlir       |   24 +-
 .../transform-op-bufferize-to-allocation.mlir |    2 +-
 .../transform-op-fuse-into-containing.mlir    |   43 +-
 .../Dialect/Linalg/transform-op-fuse.mlir     |   72 +-
 .../Linalg/transform-op-generalize.mlir       |    4 +-
 ...-rewrite-in-destination-passing-style.mlir |    6 +-
 .../transform-tile-and-winograd-rewrite.mlir  |   24 +-
 .../Linalg/transform-tile-winograd.mlir       |   36 +-
 .../Linalg/transform-winograd-conv2d.mlir     |   24 +-
 .../vectorization/extract-with-patterns.mlir  |   39 +-
 .../Dialect/Linalg/vectorization/extract.mlir |    6 +-
 .../linalg-ops-with-patterns.mlir             |   14 +-
 .../Linalg/vectorization/linalg-ops.mlir      |   74 +-
 .../Linalg/winograd-conv2d-rewrite.mlir       |    6 +-
 mlir/test/Dialect/Linalg/winograd-conv2d.mlir |   42 +-
 mlir/test/Dialect/MemRef/canonicalize.mlir    |   39 +
 mlir/test/Dialect/MemRef/invalid.mlir         |   16 +-
 mlir/test/Dialect/MemRef/ops.mlir             |    3 +
 mlir/test/Dialect/OpenACC/invalid.mlir        |   12 +
 mlir/test/Dialect/OpenACC/ops.mlir            |   25 +
 mlir/test/Dialect/OpenMP/invalid.mlir         |   33 +
 mlir/test/Dialect/SCF/canonicalize.mlir       |    8 +-
 mlir/test/Dialect/SCF/loop-pipelining.mlir    |   36 +
 .../SPIRV/IR/khr-cooperative-matrix-ops.mlir  |   34 +-
 mlir/test/Dialect/SPIRV/IR/types.mlir         |   51 +
 .../SparseTensor/sparse_vector_peeled.mlir    |    2 +-
 mlir/test/Dialect/Tensor/bufferize.mlir       |   17 +-
 mlir/test/Dialect/Tensor/canonicalize.mlir    |   29 +
 .../Tensor/fold-into-pack-and-unpack.mlir     |   61 +
 mlir/test/Dialect/Tensor/invalid.mlir         |    7 +-
 mlir/test/Dialect/Tensor/ops.mlir             |   12 +-
 mlir/test/Dialect/Tensor/reify-shapes.mlir    |   31 +
 .../value-bounds-op-interface-impl.mlir       |   17 +
 mlir/test/Dialect/Tosa/invalid.mlir           |    2 +
 ...tosa-convert-integer-type-to-signless.mlir |   73 +
 .../Dialect/Tosa/tosa-validation-valid.mlir   |   31 +
 mlir/test/Dialect/Vector/canonicalize.mlir    |   67 +-
 .../Vector/canonicalize/vector-transpose.mlir |   64 +-
 mlir/test/Dialect/Vector/invalid.mlir         |   47 +-
 mlir/test/Dialect/Vector/ops.mlir             |   23 +-
 .../Dialect/Vector/vector-mem-transforms.mlir |    4 +-
 .../Vector/vector-transfer-flatten.mlir       |   24 +-
 .../Vector/vector-warp-distribute.mlir        |   84 +-
 mlir/test/Dialect/XeGPU/invalid.mlir          |    2 +-
 mlir/test/Dialect/XeGPU/ops.mlir              |   32 +-
 mlir/test/Dialect/XeGPU/propagate-layout.mlir |   27 +-
 .../Dialect/XeGPU/subgroup-distribute.mlir    |   19 +
 mlir/test/Dialect/XeGPU/xegpu-blocking.mlir   |   46 +-
 .../Dialect/XeGPU/xegpu-unroll-patterns.mlir  |   56 +-
 .../XeGPU/xegpu-wg-to-sg-elemwise.mlir        |   19 +
 .../transform/Ch1/invalidation-1.mlir         |   22 +-
 .../transform/Ch1/invalidation-2.mlir         |   14 +-
 .../test/Examples/transform/Ch1/sequence.mlir |   20 +-
 .../test/Examples/transform/Ch2/sequence.mlir |   18 +-
 .../test/Examples/transform/Ch3/sequence.mlir |   18 +-
 .../test/Examples/transform/Ch4/multiple.mlir |   20 +-
 .../test/Examples/transform/Ch4/sequence.mlir |   14 +-
 mlir/test/IR/affine-map.mlir                  |    2 +-
 mlir/test/IR/attribute.mlir                   |    4 +
 mlir/test/IR/properties.mlir                  |    6 +-
 mlir/test/IR/region.mlir                      |    7 +
 .../Dialect/Transform/match_matmul.mlir       |    2 +-
 .../CPU/ArmNeon/vector-contract-i8mm.mlir     |  336 +
 .../transfer-read-scalable-non-trailing.mlir  |   79 +
 .../tile-and-fuse-consumer.mlir               |  301 +-
 .../Target/LLVMIR/Import/import-failure.ll    |   16 -
 mlir/test/Target/LLVMIR/arm-neon.mlir         |   13 +
 mlir/test/Target/LLVMIR/arm-sve.mlir          |   12 +
 mlir/test/Target/LLVMIR/nvvm/elect.mlir       |   20 +
 mlir/test/Target/LLVMIR/nvvmir.mlir           |    9 -
 .../Target/LLVMIR/omptarget-debug-empty.mlir  |   27 +
 .../LLVMIR/omptarget-debug-loop-loc.mlir      |    4 +-
 .../LLVMIR/omptarget-debug-map-link-loc.mlir  |   43 +
 .../Target/LLVMIR/omptarget-debug-nowait.mlir |    5 +-
 .../Target/LLVMIR/omptarget-debug-var-1.mlir  |   13 +-
 .../Target/LLVMIR/omptarget-debug-var-2.mlir  |   13 +-
 mlir/test/Target/LLVMIR/omptarget-debug.mlir  |    8 +-
 mlir/test/Target/LLVMIR/omptarget-debug2.mlir |    8 +-
 mlir/test/Target/LLVMIR/omptarget-llvm.mlir   |  348 +-
 .../LLVMIR/omptarget-parallel-llvm-debug.mlir |    8 +-
 .../LLVMIR/omptarget-parallel-llvm.mlir       |    2 +-
 mlir/test/Target/LLVMIR/omptarget-wsloop.mlir |    2 +-
 .../LLVMIR/omptargetdata-nowait-llvm.mlir     |   42 +-
 .../LLVMIR/openmp-data-target-device.mlir     |    2 +-
 .../openmp-nested-task-target-parallel.mlir   |   62 +
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |    4 +-
 mlir/test/Target/SPIRV/image-ops.mlir         |    2 +-
 .../SPIRV/khr-cooperative-matrix-ops.mlir     |   14 +
 mlir/test/Target/SPIRV/tensorARM.mlir         |   66 +
 .../make-composed-folded-affine-apply.mlir    |   77 +
 mlir/test/Transforms/test-legalizer.mlir      |   16 +-
 mlir/test/Transforms/test-symbol-dce.mlir     |   35 +
 .../FuncToLLVM/TestConvertCallOp.cpp          |    2 +-
 .../Dialect/Linalg/TestLinalgTransforms.cpp   |   28 +-
 mlir/test/lib/Dialect/Test/TestEnumDefs.td    |    8 +-
 .../test/lib/Dialect/Test/TestFormatUtils.cpp |   23 +
 mlir/test/lib/Dialect/Test/TestFormatUtils.h  |   18 +
 mlir/test/lib/Dialect/Test/TestOpDefs.cpp     |   34 +
 mlir/test/lib/Dialect/Test/TestOps.td         |   90 +-
 mlir/test/lib/Dialect/Test/TestPatterns.cpp   |    4 +-
 .../TestTilingInterfaceTransformOps.cpp       |   50 +-
 .../TestTilingInterfaceTransformOps.td        |    7 +-
 .../test/lib/Transforms/TestTransformsOps.cpp |   31 +-
 mlir/test/lib/Transforms/TestTransformsOps.td |   27 +
 .../test-linalg-ods-yaml-gen.yaml             |   25 +-
 mlir/test/mlir-tblgen/op-decl-and-defs.td     |    1 +
 mlir/test/mlir-tblgen/op-format-spec.td       |   19 +-
 mlir/test/mlir-tblgen/op-format.mlir          |   10 +-
 mlir/test/mlir-tblgen/op-format.td            |   20 +
 mlir/test/mlir-tblgen/op-operand.td           |   11 +
 mlir/test/mlir-translate/emitc_classops.mlir  |   51 +
 mlir/test/python/dialects/linalg/ops.py       |   36 -
 .../python/dialects/transform_debug_ext.py    |   45 +
 .../integration/dialects/linalg/opsrun.py     |  121 -
 .../mlir-linalg-ods-yaml-gen.cpp              |   17 +-
 mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp   |    8 +-
 mlir/tools/mlir-tblgen/EnumsGen.cpp           |    6 +-
 mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp   |   11 +
 mlir/tools/mlir-tblgen/OpFormatGen.cpp        |   41 +-
 .../FileLineColLocBreakpointManagerTest.cpp   |    6 +-
 .../Dialect/SPIRV/DeserializationTest.cpp     |    6 +-
 mlir/unittests/IR/AffineExprTest.cpp          |   23 +
 mlir/unittests/IR/OperationSupportTest.cpp    |   29 +-
 mlir/unittests/IR/ValueTest.cpp               |   12 +-
 mlir/unittests/Pass/AnalysisManagerTest.cpp   |   10 +-
 mlir/unittests/Pass/PassManagerTest.cpp       |   10 +-
 .../Transforms/DialectConversion.cpp          |    6 +-
 offload/DeviceRTL/CMakeLists.txt              |    2 +-
 offload/cmake/caches/AMDGPULibcBot.cmake      |    1 +
 offload/liboffload/API/Common.td              |   15 +-
 offload/liboffload/API/Device.td              |    3 +-
 offload/liboffload/API/Program.td             |    4 +-
 offload/liboffload/API/README.md              |  105 +-
 offload/liboffload/README.md                  |   38 +-
 offload/liboffload/src/Helpers.hpp            |   54 +-
 offload/liboffload/src/OffloadImpl.cpp        |  229 +-
 offload/libomptarget/OpenMP/Mapping.cpp       |   24 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |   26 +-
 offload/plugins-nextgen/common/CMakeLists.txt |    2 +-
 .../common/include/PluginInterface.h          |   16 +-
 .../common/src/PluginInterface.cpp            |   87 +-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |   31 +-
 offload/plugins-nextgen/host/src/rtl.cpp      |    6 +
 offload/test/mapping/duplicate_mappings_1.cpp |   27 +
 offload/test/mapping/duplicate_mappings_2.cpp |   29 +
 .../test/offloading/disable_default_device.c  |   22 +
 offload/test/ompTest/CMakeLists.txt           |    2 +-
 offload/test/ompTest/include/AssertMacros.h   |   13 +
 offload/test/ompTest/include/Logging.h        |   24 +-
 .../test/ompTest/include/OmptAssertEvent.h    |    2 +-
 offload/test/ompTest/include/OmptAsserter.h   |  126 +-
 .../ompTest/include/OmptTesterGoogleTest.h    |   15 +-
 .../ompTest/include/OmptTesterStandalone.h    |   15 +-
 .../ompTest/src/InternalEventOperators.cpp    |    6 +-
 offload/test/ompTest/src/Logging.cpp          |   37 +-
 offload/test/ompTest/src/OmptAsserter.cpp     |  157 +-
 offload/test/ompTest/src/OmptTester.cpp       |    8 -
 .../test/ompTest/src/OmptTesterStandalone.cpp |    4 +-
 .../test/ompTest/test/asserter-seq-test.cpp   |  358 +
 offload/tools/offload-tblgen/APIGen.cpp       |    3 +-
 offload/tools/offload-tblgen/MiscGen.cpp      |    2 +-
 offload/tools/offload-tblgen/PrintGen.cpp     |    5 +
 offload/tools/offload-tblgen/RecordTypes.hpp  |   12 +-
 offload/unittests/CMakeLists.txt              |   31 +-
 offload/unittests/Conformance/CMakeLists.txt  |    8 +
 .../Conformance/device_code/CMakeLists.txt    |    4 +
 .../unittests/Conformance/device_code/sin.c   |    4 +
 offload/unittests/Conformance/sin.cpp         |    8 +
 .../OffloadAPI/device/olGetDeviceInfo.cpp     |    9 +
 .../OffloadAPI/device/olGetDeviceInfoSize.cpp |    8 +
 .../OffloadAPI/device_code/CMakeLists.txt     |   12 +-
 .../OffloadAPI/device_code/localmem.c         |   11 +
 .../device_code/localmem_reduction.c          |   16 +
 .../OffloadAPI/device_code/localmem_static.c  |   17 +
 offload/unittests/OffloadAPI/init/olInit.cpp  |   12 +
 .../OffloadAPI/kernel/olLaunchKernel.cpp      |  101 +-
 openmp/CMakeLists.txt                         |    4 +
 openmp/runtime/openmp-config.cmake.in         |   11 +-
 openmp/runtime/src/CMakeLists.txt             |   10 +-
 openmp/runtime/src/kmp_alloc.cpp              |    8 +-
 openmp/runtime/src/kmp_csupport.cpp           |    2 +-
 openmp/runtime/src/kmp_lock.cpp               |    2 +
 openmp/runtime/src/kmp_tasking.cpp            |    2 +-
 .../test/ompt/misc/lock_double_destroy.cpp    |   43 +
 openmp/runtime/tools/message-converter.py     |    4 +-
 pstl/.clang-format                            |   20 -
 pstl/CMakeLists.txt                           |  103 -
 pstl/CREDITS.txt                              |   21 -
 pstl/LICENSE.TXT                              |  307 -
 pstl/README.md                                |   35 -
 pstl/cmake/ParallelSTLConfig.cmake.in         |   19 -
 pstl/docs/ReleaseNotes.rst                    |   40 -
 pstl/include/__pstl_algorithm                 |   15 -
 pstl/include/__pstl_config_site.in            |   17 -
 pstl/include/__pstl_execution                 |   15 -
 pstl/include/__pstl_numeric                   |   15 -
 pstl/include/pstl/internal/algorithm_fwd.h    | 1202 ---
 pstl/include/pstl/internal/algorithm_impl.h   | 3819 ----------
 pstl/include/pstl/internal/execution_defs.h   |  100 -
 pstl/include/pstl/internal/execution_impl.h   |  105 -
 .../pstl/internal/glue_algorithm_defs.h       |  558 --
 .../pstl/internal/glue_algorithm_impl.h       | 1108 ---
 .../pstl/internal/glue_execution_defs.h       |   55 -
 pstl/include/pstl/internal/glue_memory_defs.h |   85 -
 pstl/include/pstl/internal/glue_memory_impl.h |  352 -
 .../include/pstl/internal/glue_numeric_defs.h |  124 -
 .../include/pstl/internal/glue_numeric_impl.h |  232 -
 pstl/include/pstl/internal/memory_impl.h      |  112 -
 pstl/include/pstl/internal/numeric_fwd.h      |  139 -
 pstl/include/pstl/internal/numeric_impl.h     |  383 -
 pstl/include/pstl/internal/omp/parallel_for.h |   64 -
 .../pstl/internal/omp/parallel_for_each.h     |   59 -
 .../pstl/internal/omp/parallel_invoke.h       |   50 -
 .../pstl/internal/omp/parallel_merge.h        |   98 -
 .../pstl/internal/omp/parallel_reduce.h       |   73 -
 .../include/pstl/internal/omp/parallel_scan.h |  136 -
 .../omp/parallel_stable_partial_sort.h        |   33 -
 .../pstl/internal/omp/parallel_stable_sort.h  |  160 -
 .../internal/omp/parallel_transform_reduce.h  |  113 -
 .../internal/omp/parallel_transform_scan.h    |   32 -
 pstl/include/pstl/internal/omp/util.h         |  173 -
 pstl/include/pstl/internal/parallel_backend.h |   37 -
 .../pstl/internal/parallel_backend_omp.h      |   58 -
 .../pstl/internal/parallel_backend_serial.h   |  137 -
 .../pstl/internal/parallel_backend_tbb.h      | 1296 ----
 .../pstl/internal/parallel_backend_utils.h    |  263 -
 pstl/include/pstl/internal/parallel_impl.h    |   90 -
 pstl/include/pstl/internal/pstl_config.h      |  204 -
 .../pstl/internal/unseq_backend_simd.h        |  862 ---
 pstl/include/pstl/internal/utils.h            |  177 -
 pstl/test/CMakeLists.txt                      |   41 -
 ...eader_inclusion_order_algorithm_0.pass.cpp |   21 -
 ...eader_inclusion_order_algorithm_1.pass.cpp |   21 -
 .../header_inclusion_order_memory_0.pass.cpp  |   21 -
 .../header_inclusion_order_memory_1.pass.cpp  |   21 -
 .../header_inclusion_order_numeric_0.pass.cpp |   21 -
 .../header_inclusion_order_numeric_1.pass.cpp |   21 -
 pstl/test/pstl/version.pass.cpp               |   20 -
 .../alg.merge/inplace_merge.pass.cpp          |  159 -
 .../std/algorithms/alg.merge/merge.pass.cpp   |  113 -
 .../alg.copy/copy_if.pass.cpp                 |  147 -
 .../alg.partitions/is_partitioned.pass.cpp    |  101 -
 .../alg.partitions/partition.pass.cpp         |  178 -
 .../alg.partitions/partition_copy.pass.cpp    |  116 -
 .../alg.reverse/reverse.pass.cpp              |  104 -
 .../alg.reverse/reverse_copy.pass.cpp         |  130 -
 .../copy_move.pass.cpp                        |  197 -
 .../alg.modifying.operations/fill.pass.cpp    |  100 -
 .../generate.pass.cpp                         |  104 -
 .../alg.modifying.operations/remove.pass.cpp  |  161 -
 .../remove_copy.pass.cpp                      |   91 -
 .../alg.modifying.operations/replace.pass.cpp |  160 -
 .../replace_copy.pass.cpp                     |  105 -
 .../alg.modifying.operations/rotate.pass.cpp  |  176 -
 .../rotate_copy.pass.cpp                      |  146 -
 .../swap_ranges.pass.cpp                      |  133 -
 .../transform_binary.pass.cpp                 |  122 -
 .../transform_unary.pass.cpp                  |   91 -
 .../alg.modifying.operations/unique.pass.cpp  |  163 -
 .../unique_copy_equal.pass.cpp                |  135 -
 .../alg.nonmodifying/adjacent_find.pass.cpp   |  114 -
 .../alg.nonmodifying/all_of.pass.cpp          |  117 -
 .../alg.nonmodifying/any_of.pass.cpp          |  103 -
 .../alg.nonmodifying/count.pass.cpp           |  108 -
 .../alg.nonmodifying/equal.pass.cpp           |  168 -
 .../algorithms/alg.nonmodifying/find.pass.cpp |   96 -
 .../alg.nonmodifying/find_end.pass.cpp        |  123 -
 .../alg.nonmodifying/find_first_of.pass.cpp   |  112 -
 .../alg.nonmodifying/find_if.pass.cpp         |  109 -
 .../alg.nonmodifying/for_each.pass.cpp        |  102 -
 .../alg.nonmodifying/mismatch.pass.cpp        |  132 -
 .../alg.nonmodifying/none_of.pass.cpp         |  101 -
 .../alg.nonmodifying/nth_element.pass.cpp     |  175 -
 .../alg.nonmodifying/search_n.pass.cpp        |  109 -
 .../alg.heap.operations/is_heap.pass.cpp      |  146 -
 .../lexicographical_compare.pass.cpp          |  175 -
 .../alg.min.max/minmax_element.pass.cpp       |  192 -
 .../alg.set.operations/includes.pass.cpp      |  106 -
 .../alg.set.operations/set.pass.cpp           |  280 -
 .../algorithms/alg.sorting/is_sorted.pass.cpp |  100 -
 .../alg.sorting/partial_sort.pass.cpp         |  149 -
 .../alg.sorting/partial_sort_copy.pass.cpp    |  196 -
 .../std/algorithms/alg.sorting/sort.pass.cpp  |  247 -
 pstl/test/std/lit.local.cfg                   |    2 -
 .../numeric.ops/adjacent_difference.pass.cpp  |  170 -
 .../std/numerics/numeric.ops/reduce.pass.cpp  |  114 -
 .../std/numerics/numeric.ops/scan.fail.cpp    |   28 -
 .../std/numerics/numeric.ops/scan.pass.cpp    |  201 -
 .../numeric.ops/transform_reduce.pass.cpp     |  129 -
 .../numeric.ops/transform_scan.pass.cpp       |  177 -
 .../uninitialized_construct.pass.cpp          |  123 -
 .../uninitialized_copy_move.pass.cpp          |  143 -
 .../uninitialized_fill_destroy.pass.cpp       |   93 -
 pstl/test/support/pstl_test_config.h          |   52 -
 pstl/test/support/stdlib/algorithm            |   26 -
 pstl/test/support/stdlib/execution            |   50 -
 pstl/test/support/stdlib/memory               |   26 -
 pstl/test/support/stdlib/numeric              |   26 -
 pstl/test/support/utils.h                     | 1318 ----
 third-party/unittest/googletest/src/gtest.cc  |    1 +
 utils/bazel/WORKSPACE                         |    9 +
 .../llvm-project-overlay/clang/BUILD.bazel    |    5 +
 .../llvm-project-overlay/libc/BUILD.bazel     |  336 +
 .../libc/libc_build_rules.bzl                 |   30 +
 .../libc/test/UnitTest/BUILD.bazel            |    3 +
 .../libc/test/include/BUILD.bazel             |  362 +
 .../libc/test/libc_test_rules.bzl             |   46 +-
 .../libc/test/src/wchar/BUILD.bazel           |  224 +
 .../llvm-project-overlay/lldb/BUILD.bazel     |    2 +
 .../lldb/source/Plugins/BUILD.bazel           |    1 +
 .../llvm-project-overlay/llvm/BUILD.bazel     |   19 +-
 .../llvm-project-overlay/llvm/config.bzl      |    1 +
 .../llvm/include/llvm/Config/llvm-config.h    |    3 +
 .../llvm-project-overlay/mlir/BUILD.bazel     |   79 +-
 .../mlir/python/BUILD.bazel                   |   26 +
 .../mlir/test/BUILD.bazel                     |    1 +
 utils/bazel/llvm_configs/llvm-config.h.cmake  |    3 +
 utils/bazel/third_party_build/pyyaml.BUILD    |   16 +
 5577 files changed, 198669 insertions(+), 119665 deletions(-)
 create mode 100644 .github/copilot-instructions.md
 rename amd/comgr/test-lit/comgr-sources/{compile-minimal-test.c => compile-opencl-minimal.c} (92%)
 create mode 100644 amd/comgr/test-lit/comgr-sources/data-action.c
 create mode 100644 amd/comgr/test-lit/comgr-sources/get-version.c
 create mode 100644 amd/comgr/test-lit/comgr-sources/status-string.c
 create mode 100644 amd/comgr/test-lit/compile-opencl-2.cl
 create mode 100644 amd/comgr/test-lit/data-action.c
 create mode 100644 amd/comgr/test-lit/get-version.c
 create mode 100644 amd/comgr/test-lit/status-string.c
 delete mode 100644 amd/comgr/test/options_test.c
 create mode 100644 bolt/test/X86/zero-density.s
 create mode 100644 bolt/test/binary-analysis/AArch64/gs-pauth-tail-calls.s
 create mode 100644 bolt/test/code-at-high-address.c
 create mode 100644 bolt/test/program-header.test
 create mode 100644 clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst
 create mode 100644 clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp
 create mode 100644 clang/docs/coro-async-task-continuations.png
 create mode 100644 clang/docs/coro-generator-suspended.png
 create mode 100644 clang/docs/coro-generator-variables.png
 delete mode 100644 clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.td
 rename clang/include/clang/CIR/Interfaces/{CIRFPTypeInterface.h => CIRTypeInterfaces.h} (62%)
 create mode 100644 clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
 create mode 100644 clang/include/clang/Interpreter/RemoteJITUtils.h
 create mode 100644 clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp
 rename clang/lib/CIR/Interfaces/{CIRFPTypeInterface.cpp => CIRTypeInterfaces.cpp} (73%)
 delete mode 100644 clang/lib/Driver/ToolChains/RISCVToolchain.cpp
 delete mode 100644 clang/lib/Driver/ToolChains/RISCVToolchain.h
 create mode 100644 clang/lib/Interpreter/RemoteJITUtils.cpp
 create mode 100644 clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp
 create mode 100644 clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp
 create mode 100644 clang/test/AST/absurdly_big_struct.cpp
 create mode 100644 clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm
 delete mode 100644 clang/test/Analysis/analyzer_test.py
 create mode 100644 clang/test/Analysis/cstring-should-not-invalidate.cpp
 create mode 100644 clang/test/Analysis/getline-unixapi-invalid-signatures.c
 create mode 100644 clang/test/Analysis/lambda-convert-to-func-ptr.cpp
 delete mode 100644 clang/test/Analysis/z3/enabled.c
 create mode 100644 clang/test/CIR/CodeGen/assign-operator.cpp
 create mode 100644 clang/test/CIR/CodeGen/bitfield-union.c
 create mode 100644 clang/test/CIR/CodeGen/builtin_bit.cpp
 create mode 100644 clang/test/CIR/CodeGen/complex-builtins.cpp
 create mode 100644 clang/test/CIR/CodeGen/ctor-alias.cpp
 create mode 100644 clang/test/CIR/CodeGen/generic-selection.c
 create mode 100644 clang/test/CIR/CodeGen/new.cpp
 create mode 100644 clang/test/CIR/CodeGen/non-type-template-param.cpp
 create mode 100644 clang/test/CIR/CodeGen/opt-info-attr.cpp
 create mode 100644 clang/test/CIR/CodeGen/pack-indexing.cpp
 create mode 100644 clang/test/CIR/CodeGenOpenACC/cache.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/combined-copyin-copyout-create.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/compute-copyin-copyout-create.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/data-copy-copyin-copyout-create.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/enter-data.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/exit-data.c
 create mode 100644 clang/test/CIR/CodeGenOpenACC/update.c
 create mode 100644 clang/test/CIR/IR/invalid-func.cir
 create mode 100644 clang/test/CIR/IR/invalid-opt-info.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-create-wrong-size.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-create-wrong-type.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-shift-wrong-result-type.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-shuffle-dyn-wrong-operands.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir
 delete mode 100644 clang/test/CIR/IR/invalid-vector-zero-size.cir
 create mode 100644 clang/test/CIR/Transforms/complex-imag-fold.cir
 create mode 100644 clang/test/CIR/Transforms/complex-real-fold.cir
 create mode 100644 clang/test/CIR/func-linkage.cpp
 create mode 100644 clang/test/ClangScanDeps/modules-full-named-modules.cppm
 create mode 100644 clang/test/ClangScanDeps/raw-strings.cpp
 create mode 100644 clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
 create mode 100644 clang/test/CodeGen/AArch64/attr-fp8-function.c
 create mode 100644 clang/test/CodeGen/LoongArch/bitint.c
 create mode 100644 clang/test/CodeGen/PowerPC/builtins-bcd-transform.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/non-overloaded/nds_vd4dots.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/non-overloaded/nds_vd4dotsu.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/non-overloaded/nds_vd4dotu.c (100%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/non-overloaded/nds_vfpmadb.c (99%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/non-overloaded/nds_vfpmadt.c (99%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/overloaded/nds_vd4dots.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/overloaded/nds_vd4dotsu.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/overloaded/nds_vd4dotu.c (100%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/overloaded/nds_vfpmadb.c (99%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/non-policy/overloaded/nds_vfpmadt.c (99%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/non-overloaded/nds_vd4dots.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/non-overloaded/nds_vd4dotsu.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/non-overloaded/nds_vd4dotu.c (100%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/non-overloaded/nds_vfpmadb.c (99%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/non-overloaded/nds_vfpmadt.c (99%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/overloaded/nds_vd4dots.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/overloaded/nds_vd4dotsu.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/overloaded/nds_vd4dotu.c (100%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/overloaded/nds_vfpmadb.c (99%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => andes-intrinsics}/policy/overloaded/nds_vfpmadt.c (99%)
 create mode 100644 clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmacc_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmacc_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vfnrclip_x_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vfwmacc_4x4x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmacc_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmacc_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccsu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccsu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccus_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/non-policy/overloaded/sf_vqmaccus_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vfnrclip_x_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vfwmacc_4x4x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmacc_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmacc_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccsu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccsu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccus_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/non-overloaded/sf_vqmaccus_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vfnrclip_x_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vfnrclip_xu_f_qf.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vfwmacc_4x4x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmacc_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmacc_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccsu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccsu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccu_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccu_4x8x4.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccus_2x8x2.c (100%)
 rename clang/test/CodeGen/RISCV/{rvv-intrinsics-autogenerated => sifive-intrinsics}/policy/overloaded/sf_vqmaccus_4x8x4.c (100%)
 create mode 100644 clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll
 create mode 100644 clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch-bad-file.c
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch-cpp.cpp
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch-eh.cpp
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch-globals.c
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch-lto.c
 create mode 100644 clang/test/CodeGen/X86/ms-secure-hotpatch.c
 create mode 100644 clang/test/CodeGen/arm64-microsoft-sys.c
 create mode 100644 clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c
 create mode 100644 clang/test/CodeGenCXX/builtin-invoke.cpp
 create mode 100644 clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
 create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl
 create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl
 create mode 100644 clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl
 create mode 100644 clang/test/DebugInfo/KeyInstructions/atomic.c
 create mode 100644 clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp
 create mode 100644 clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp
 create mode 100644 clang/test/Driver/HLSL/dxc_fvk_layout.hlsl
 rename libcxx/test/libcxx/selftest/file_dependencies/a.txt => clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep (100%)
 rename libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt => clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep (100%)
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
 create mode 100755 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
 create mode 100755 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
 create mode 100755 clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
 create mode 100755 clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
 create mode 100644 clang/test/Driver/aarch64-gnutools.c
 create mode 100644 clang/test/Driver/aarch64-toolchain-extra.c
 create mode 100644 clang/test/Driver/aarch64-toolchain.c
 create mode 100644 clang/test/Driver/arm-gnutools.c
 create mode 100644 clang/test/Driver/arm-toolchain-extra.c
 create mode 100644 clang/test/Driver/arm-toolchain.c
 create mode 100644 clang/test/Driver/baremetal-undefined-symbols.c
 create mode 100644 clang/test/Driver/check-no-multlib-warning.c
 create mode 100644 clang/test/Driver/dxc_hlsl-rootsig-ver.hlsl
 create mode 100644 clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c
 create mode 100644 clang/test/Driver/ir-exception-model.c
 create mode 100644 clang/test/Driver/print-enabled-extensions/aarch64-gb10.c
 delete mode 100644 clang/test/Driver/riscv-args.c
 create mode 100644 clang/test/ExtractAPI/pointers.c
 create mode 100644 clang/test/Index/inline-assembly.c
 create mode 100644 clang/test/Interpreter/out-of-process.cpp
 create mode 100644 clang/test/Modules/ModulesLocalNamespace.cppm
 create mode 100644 clang/test/Modules/mingw-exceptions.cppm
 create mode 100644 clang/test/Modules/no-transitive-decl-change-4.cppm
 create mode 100644 clang/test/Modules/pr118137.cppm
 create mode 100644 clang/test/Modules/pr145975.cppm
 create mode 100644 clang/test/Modules/template-declare.cppm
 create mode 100644 clang/test/Modules/var-init-side-effects-modulemap.cpp
 create mode 100644 clang/test/OpenMP/openmp-6-future-spellings.cpp
 create mode 100644 clang/test/Options/Gis.hlsl
 create mode 100644 clang/test/Parser/c23-typeof.m
 create mode 100644 clang/test/Preprocessor/macro_paste_identifier_ucn.c
 rename clang/test/{CodeGen => Sema}/attr-cpuspecific-cpus.c (97%)
 create mode 100644 clang/test/Sema/attr-nonstring_safe.c
 create mode 100644 clang/test/Sema/builtins-bcd-transform.c
 create mode 100644 clang/test/Sema/deep_recursion.c
 create mode 100644 clang/test/Sema/rvv-andes-required-features-invalid.c
 create mode 100644 clang/test/SemaCXX/P1979.cpp
 create mode 100644 clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp
 create mode 100644 clang/test/SemaCXX/builtin-invoke.cpp
 create mode 100644 clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
 create mode 100644 clang/test/SemaCXX/wreturn-always-throws.cpp
 create mode 100644 clang/test/SemaHIP/amdgpu-feature-builtins-cant-jump.hip
 create mode 100644 clang/test/SemaHIP/amdgpu-feature-builtins-return-type-deduction.hip
 create mode 100644 clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp
 create mode 100644 clang/test/SemaOpenCL/amd-media-ops.cl
 create mode 100644 clang/unittests/AllClangUnitTests.cpp
 create mode 100644 compiler-rt/lib/asan/asan.link_with_main_exec.txt
 create mode 100644 compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt
 create mode 100644 compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp
 create mode 100644 compiler-rt/test/msan/msan_print_shadow_on_outlined_check.cpp
 create mode 100644 compiler-rt/test/tsan/java_heap_init2.cpp
 create mode 100644 compiler-rt/test/tsan/munmap_clear_shadow.c
 create mode 100644 cross-project-tests/dtlto/README.md
 create mode 100644 cross-project-tests/dtlto/ld-dtlto.c
 create mode 100644 cross-project-tests/dtlto/lit.local.cfg
 create mode 100644 flang/lib/Semantics/check-omp-atomic.cpp
 create mode 100644 flang/lib/Semantics/check-omp-loop.cpp
 create mode 100644 flang/lib/Semantics/check-omp-metadirective.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.cpp
 create mode 100644 flang/lib/Semantics/openmp-utils.h
 create mode 100644 flang/module/cooperative_groups.f90
 create mode 100644 flang/test/Driver/fopenmp-version.F90
 create mode 100644 flang/test/Driver/tco-emit-final-mlir.fir
 create mode 100644 flang/test/Fir/OpenACC/openacc-type-categories-class.f90
 create mode 100644 flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
 create mode 100644 flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
 create mode 100644 flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
 create mode 100644 flang/test/Integration/skip-external-rtti-definition.F90
 create mode 100644 flang/test/Lower/CUDA/cuda-cooperative.cuf
 create mode 100644 flang/test/Lower/OpenMP/minmax-optional-parameters.f90
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
 create mode 100644 flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
 create mode 100644 flang/test/Lower/OpenMP/target-data-if-false.f90
 create mode 100644 flang/test/Lower/OpenMP/target-data-skip-mapper-calls.f90
 create mode 100644 flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
 create mode 100644 flang/test/Lower/variable-common-viewed-as-module-var.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct01.f90
 create mode 100644 flang/test/Parser/OpenMP/loop-transformation-construct02.f90
 create mode 100644 flang/test/Parser/issue-146362.1.f90
 create mode 100644 flang/test/Parser/issue-146362.2.f90
 create mode 100644 flang/test/Preprocessing/bug1077.F90
 create mode 100644 flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
 create mode 100644 flang/test/Semantics/bug1046.f90
 create mode 100644 flang/test/Semantics/bug1056.f90
 create mode 100644 flang/test/Semantics/bug1092.F90
 create mode 100644 flang/test/Semantics/bug144453.f90
 create mode 100644 flang/test/Semantics/function-result-extent-max.f90
 create mode 100644 flang/test/Transforms/debug-omp-target-op-1.fir
 create mode 100644 flang/test/Transforms/debug-omp-target-op-2.fir
 create mode 100644 flang/test/Transforms/tbaa-for-local-vars.fir
 create mode 100644 libc/src/setjmp/arm/sigsetjmp.cpp
 create mode 100644 libc/src/string/memory_utils/arm/inline_memcpy.h
 create mode 100644 libc/src/wchar/wcscspn.cpp
 create mode 100644 libc/src/wchar/wcscspn.h
 create mode 100644 libc/src/wchar/wcsnlen.cpp
 create mode 100644 libc/src/wchar/wcsnlen.h
 create mode 100644 libc/src/wchar/wcstok.cpp
 create mode 100644 libc/src/wchar/wcstok.h
 create mode 100644 libc/src/wchar/wctomb.cpp
 create mode 100644 libc/src/wchar/wctomb.h
 create mode 100644 libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp
 create mode 100644 libc/test/integration/src/stdlib/gpu/realloc.cpp
 create mode 100644 libc/test/src/wchar/wcscspn_test.cpp
 create mode 100644 libc/test/src/wchar/wcsnlen_test.cpp
 create mode 100644 libc/test/src/wchar/wcstok_test.cpp
 create mode 100644 libc/test/src/wchar/wctomb_test.cpp
 rename libclc/clc/include/clc/{math => shared}/binary_decl_with_int_second_arg.inc (74%)
 create mode 100644 libclc/clc/include/clc/shared/binary_def_scalarize.inc
 rename libclc/clc/include/clc/{math => shared}/binary_def_with_int_second_arg.inc (80%)
 create mode 100644 libclc/clc/include/clc/shared/ternary_def_scalarize.inc
 create mode 100644 libclc/clc/include/clc/shared/unary_def_scalarize.inc
 create mode 100644 libclc/opencl/include/clc/opencl/opencl-base.h
 create mode 100644 libcxx/include/__iterator/product_iterator.h
 create mode 100644 libcxx/include/__locale_dir/check_grouping.h
 create mode 100644 libcxx/include/__locale_dir/get_c_locale.h
 create mode 100644 libcxx/include/__locale_dir/messages.h
 create mode 100644 libcxx/include/__locale_dir/money.h
 create mode 100644 libcxx/include/__locale_dir/num.h
 create mode 100644 libcxx/include/__locale_dir/scan_keyword.h
 create mode 100644 libcxx/include/__locale_dir/time.h
 create mode 100644 libcxx/include/__locale_dir/wbuffer_convert.h
 create mode 100644 libcxx/include/__locale_dir/wstring_convert.h
 delete mode 100644 libcxx/include/__type_traits/add_lvalue_reference.h
 rename libcxx/include/__type_traits/{add_rvalue_reference.h => add_reference.h} (51%)
 create mode 100644 libcxx/test/extensions/gnu/hash/specializations.pass.cpp
 rename libcxx/test/{libcxx/extensions/hash/specializations.compile.fail.cpp => extensions/gnu/hash/specializations.verify.cpp} (70%)
 rename libcxx/test/{libcxx/extensions/hash_map/const_iterator.compile.fail.cpp => extensions/gnu/hash_map/const_iterator.verify.cpp} (55%)
 create mode 100644 libcxx/test/extensions/gnu/lit.local.cfg
 delete mode 100644 libcxx/test/libcxx/extensions/hash/specializations.pass.cpp
 create mode 100644 libcxx/test/libcxx/iterators/product_iterator.pass.cpp
 rename libcxx/test/{libcxx => }/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/additional_compile_flags/lit.local.cfg (100%)
 rename libcxx/test/{libcxx => }/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/additional_compile_flags/substitutes-in-run.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.fail.cpp/compile-error.compile.fail.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.fail.cpp/compile-success.compile.fail.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.cpp/compile-error.compile.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.cpp/compile-success.compile.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.cpp/link-error.compile.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.cpp/run-error.compile.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.mm/compile-error.compile.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.mm/compile-success.compile.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.mm/link-error.compile.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/compile.pass.mm/run-error.compile.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/convenience_substitutions/build_run.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/convenience_substitutions/verify.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/dsl/dsl.sh.py (99%)
 rename libcxx/test/{libcxx => }/selftest/dsl/lit.local.cfg (100%)
 create mode 100644 libcxx/test/selftest/file_dependencies/a.txt
 rename libcxx/test/{libcxx => }/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp (100%)
 create mode 100644 libcxx/test/selftest/file_dependencies/dir/b.txt
 rename libcxx/test/{libcxx => }/selftest/file_dependencies/substitute-in-dependencies.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/gen.cpp/empty.gen.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/gen.cpp/one.gen.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/gen.cpp/two.gen.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.fail.cpp/compile-error.link.fail.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.fail.cpp/link-error.link.fail.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.fail.cpp/link-success.link.fail.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.cpp/compile-error.link.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.cpp/link-error.link.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.cpp/link-success.link.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.cpp/run-error.link.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.mm/compile-error.link.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.mm/link-error.link.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.mm/link-success.link.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/link.pass.mm/run-error.link.pass.mm (100%)
 rename libcxx/test/{libcxx => selftest}/lit.local.cfg (100%)
 rename libcxx/test/{libcxx => }/selftest/modules/no-modules.sh.cpp (73%)
 rename libcxx/test/{libcxx => }/selftest/modules/std-and-std.compat-module.sh.cpp (82%)
 rename libcxx/test/{libcxx => }/selftest/modules/std-module.sh.cpp (82%)
 rename libcxx/test/{libcxx => }/selftest/modules/std.compat-module.sh.cpp (82%)
 rename libcxx/test/{libcxx => }/selftest/pass.cpp/compile-error.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.cpp/link-error.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.cpp/run-error.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.cpp/run-success.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.cpp/werror.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/compile-error.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/link-error.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/no-arc.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/run-error.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/run-success.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/pass.mm/use-objective-cxx.pass.mm (100%)
 rename libcxx/test/{libcxx => }/selftest/remote-substitutions.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/sh.cpp/run-error.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/sh.cpp/run-success.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/sh.cpp/substitutions.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/sh.cpp/werror.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/shell-no-escape-builtins.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/stdin-is-piped.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/test_macros.pass.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/tmpdir-exists.sh.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/verify.cpp/no-diagnostics.verify.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/verify.cpp/no-werror.verify.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/verify.cpp/right-diagnostic.verify.cpp (100%)
 rename libcxx/test/{libcxx => }/selftest/verify.cpp/wrong-diagnostic.verify.cpp (100%)
 create mode 100644 libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py
 create mode 100644 libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
 create mode 100644 libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
 create mode 100644 libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp
 create mode 100755 libcxx/utils/ci/buildkite-pipeline-trigger.sh
 create mode 100644 lld/ELF/Arch/TargetImpl.h
 create mode 100644 lld/docs/DTLTO.rst
 create mode 100644 lld/test/COFF/exported-dllmain.test
 create mode 100644 lld/test/ELF/aarch64-branch-to-branch.s
 create mode 100644 lld/test/ELF/dtlto/files.test
 create mode 100644 lld/test/ELF/dtlto/options.test
 create mode 100644 lld/test/ELF/dtlto/partitions.test
 create mode 100644 lld/test/ELF/lto/lazy-debug.ll
 create mode 100644 lld/test/ELF/x86-64-branch-to-branch.s
 delete mode 100644 lld/test/wasm/compress-relocs.ll
 create mode 100644 lld/test/wasm/compress-relocs.s
 create mode 100644 lld/test/wasm/compress-relocs64.s
 create mode 100644 lldb/docs/use/mcp.md
 delete mode 100644 lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
 delete mode 100644 lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
 delete mode 100644 lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp
 delete mode 100644 lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
 delete mode 100644 lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
 delete mode 100644 lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
 create mode 100644 lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp
 create mode 100644 lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h
 rename lldb/test/API/{functionalities/data-formatter/data-formatter-stl/libcxx/deque => commands/frame/var-dil/basics/BitField}/Makefile (75%)
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/BitField/TestFrameVarDILBitField.py
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/BitField/main.cpp
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp
 rename lldb/test/API/{functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr => commands/frame/var-dil/basics/Indirection}/Makefile (70%)
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/Indirection/TestFrameVarDILIndirection.py
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/Indirection/main.cpp
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/PointerDereference/Makefile
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/PointerDereference/TestFrameVarDILPointerDereference.py
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/PointerDereference/main.cpp
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/QualifiedId/Makefile
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/QualifiedId/TestFrameVarDILQualifiedId.py
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/QualifiedId/main.cpp
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/Makefile
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/TestFrameVarDILSyntheticDereference.py
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/main.cpp
 create mode 100644 lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/wrapPtrSynthProvider.py
 create mode 100644 lldb/test/API/commands/protocol/TestMCPUnixSocket.py
 create mode 100644 lldb/test/API/commands/statistics/basic/baz.cpp
 create mode 100644 lldb/test/API/commands/statistics/basic/third.cpp
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/Makefile
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/TestLocationsAfterRebuild.py
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/main.c
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/second_main.c
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/third_main.c
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py
 create mode 100644 lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx/atomic/TestLibCxxAtomic.py => generic/atomic/TestDataFormatterStdAtomic.py} (89%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx => generic}/atomic/main.cpp (51%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx => generic}/chrono/Makefile (82%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx/chrono/TestDataFormatterLibcxxChrono.py => generic/chrono/TestDataFormatterStdChrono.py} (98%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx => generic}/chrono/main.cpp (100%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx/map/TestDataFormatterLibccMap.py => generic/map/TestDataFormatterStdMap.py} (94%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py => generic/shared_ptr/TestDataFormatterStdSharedPtr.py} (54%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx => generic}/unique_ptr/main.cpp (58%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libstdcpp/map => generic/vbool}/Makefile (72%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libstdcpp => generic}/vbool/TestDataFormatterStdVBool.py (81%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/{libcxx/vector/TestDataFormatterLibcxxVector.py => generic/vector/TestDataFormatterStdVector.py} (84%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/{vbool => invalid-string}/Makefile (98%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/{map => unordered_map-iterator}/Makefile (100%)
 rename lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/{unordered_map => unordered_map-iterator}/TestDataFormatterLibccUnorderedMap.py (51%)
 create mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
 delete mode 100644 lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
 create mode 100644 lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py
 create mode 100644 lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core
 create mode 100755 lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out
 create mode 100644 lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp
 create mode 100644 lldb/test/API/lang/objc/class-without-methods/Makefile
 create mode 100644 lldb/test/API/lang/objc/class-without-methods/Point.h
 create mode 100644 lldb/test/API/lang/objc/class-without-methods/Point.m
 create mode 100644 lldb/test/API/lang/objc/class-without-methods/TestObjCClassWithoutMethods.py
 create mode 100644 lldb/test/API/lang/objc/class-without-methods/main.m
 create mode 100644 lldb/test/API/linux/aarch64/mte_core_file/core.mte.notags
 create mode 100644 lldb/test/API/macosx/lc-note/additional-registers/Makefile
 create mode 100644 lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py
 create mode 100644 lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp
 create mode 100644 lldb/test/API/macosx/lc-note/additional-registers/main.c
 delete mode 100644 lldb/test/API/macosx/simulator/hello.c
 create mode 100644 lldb/test/API/macosx/simulator/hello.cpp
 create mode 100644 lldb/test/API/riscv/step/Makefile
 create mode 100644 lldb/test/API/riscv/step/TestSoftwareStep.py
 create mode 100644 lldb/test/API/riscv/step/branch.c
 create mode 100644 lldb/test/API/riscv/step/incomplete_sequence_without_lr.c
 create mode 100644 lldb/test/API/riscv/step/incomplete_sequence_without_sc.c
 create mode 100644 lldb/test/API/riscv/step/main.c
 create mode 100644 lldb/test/API/tools/lldb-dap/step/other.h
 create mode 100644 lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp
 create mode 100644 llvm/docs/QualGroup.rst
 create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h
 create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/MemoryAccess.h
 create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h
 create mode 100644 llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
 rename llvm/include/llvm/Frontend/HLSL/{HLSLRootSignatureUtils.h => RootSignatureValidations.h} (50%)
 create mode 100644 llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h
 delete mode 100644 llvm/include/llvm/IR/RuntimeLibcalls.def
 create mode 100644 llvm/include/llvm/IR/RuntimeLibcalls.td
 create mode 100644 llvm/include/llvm/IR/RuntimeLibcallsImpl.td
 delete mode 100644 llvm/include/llvm/MC/MCFixupKindInfo.h
 delete mode 100644 llvm/include/llvm/MC/MCFragment.h
 create mode 100644 llvm/include/llvm/MC/MCGOFFAttributes.h
 create mode 100644 llvm/lib/CodeGen/WindowsSecureHotPatching.cpp
 create mode 100644 llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt
 create mode 100644 llvm/lib/ExecutionEngine/Orc/InProcessMemoryAccess.cpp
 create mode 100644 llvm/lib/ExecutionEngine/Orc/SelfExecutorProcessControl.cpp
 create mode 100644 llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
 delete mode 100644 llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
 create mode 100644 llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
 create mode 100644 llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
 create mode 100644 llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp
 delete mode 100644 llvm/lib/IR/ZOSLibcallNames.def
 create mode 100644 llvm/lib/MC/MCSectionGOFF.cpp
 delete mode 100644 llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
 create mode 100644 llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
 delete mode 100644 llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
 delete mode 100644 llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
 delete mode 100644 llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
 delete mode 100644 llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
 delete mode 100644 llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
 delete mode 100644 llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
 delete mode 100644 llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
 create mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoP.td
 delete mode 100644 llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
 delete mode 100644 llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
 delete mode 100644 llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCExpr.h
 create mode 100644 llvm/lib/Target/X86/X86SchedLunarlakeP.td
 delete mode 100644 llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/ldexp.ll
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/lrint.ll
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll
 create mode 100644 llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll
 create mode 100644 llvm/test/Analysis/CostModel/RISCV/exp.ll
 create mode 100644 llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
 create mode 100644 llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json
 create mode 100644 llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json
 create mode 100644 llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json
 create mode 100644 llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json
 create mode 100644 llvm/test/Analysis/IR2Vec/dbg-inst.ll
 create mode 100644 llvm/test/Analysis/IR2Vec/unreachable.ll
 create mode 100644 llvm/test/Analysis/IR2Vec/vocab-test.ll
 create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-use-vector-truncate.mir
 create mode 100644 llvm/test/CodeGen/AArch64/PHIElimination-reuse-copy.mir
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
 create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-loh.mir
 create mode 100644 llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll
 create mode 100644 llvm/test/CodeGen/AArch64/rem-by-const.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-fclamp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-sclamp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-uclamp.ll
 create mode 100644 llvm/test/CodeGen/AArch64/streaming-func-no-sme.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-indexed-arithmetic.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve-merging-unary.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-rax1.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add-unsupported.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/ptradd-sdag-undef-poison.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
 create mode 100644 llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir
 create mode 100644 llvm/test/CodeGen/ARM/Windows/frexp.ll
 create mode 100644 llvm/test/CodeGen/ARM/cmp-select-sign.ll
 create mode 100644 llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll
 create mode 100644 llvm/test/CodeGen/ARM/ifcvt_unanalyzable_fallthrough.mir
 create mode 100644 llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll
 create mode 100644 llvm/test/CodeGen/DirectX/BufferLoadInt64.ll
 create mode 100644 llvm/test/CodeGen/DirectX/BufferStoreInt64.ll
 create mode 100644 llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll
 create mode 100644 llvm/test/CodeGen/DirectX/issue-145408-gep-struct-fix.ll
 create mode 100644 llvm/test/CodeGen/DirectX/strip-rootsignatures.ll
 create mode 100644 llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/calling-conv-ilp32d.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fix-xvshuf.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll
 create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
 create mode 100644 llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/builtins-bcd-transform.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/crreduce-reg.mir
 create mode 100644 llvm/test/CodeGen/PowerPC/memintr32.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/memintr64.ll
 create mode 100644 llvm/test/CodeGen/RISCV/pr145363.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llround.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lround.ll
 rename llvm/test/CodeGen/RISCV/rvv/{fixed-vectors-vp-reverser-float.ll => fixed-vectors-vp-reverse-float.ll} (71%)
 rename llvm/test/CodeGen/RISCV/rvv/{fixed-vectors-vp-reverser-int.ll => fixed-vectors-vp-reverse-int.ll} (100%)
 delete mode 100644 llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
 create mode 100644 llvm/test/CodeGen/RISCV/xmips-cbop.ll
 create mode 100644 llvm/test/CodeGen/RISCV/xqcicm.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll
 rename llvm/test/{MC/GOFF/ppa1.ll => CodeGen/SystemZ/zos-ppa1.ll} (97%)
 create mode 100644 llvm/test/CodeGen/SystemZ/zos-section-1.ll
 create mode 100644 llvm/test/CodeGen/SystemZ/zos-section-2.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/debug-code-stackification.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll
 create mode 100644 llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll
 create mode 100644 llvm/test/CodeGen/X86/GlobalISel/llvm.sincos.mir
 create mode 100644 llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
 create mode 100644 llvm/test/CodeGen/X86/frame-pointer-reserved.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-secure-hotpatch-attr.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-secure-hotpatch-bad-file.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-secure-hotpatch-direct-global-access.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-file.ll
 create mode 100644 llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-list.ll
 create mode 100644 llvm/test/CodeGen/X86/x86-access-to-global.ll
 create mode 100644 llvm/test/CodeGen/Xtensa/invalid-tls.ll
 create mode 100644 llvm/test/CodeGen/Xtensa/select-cc-fp.ll
 create mode 100644 llvm/test/CodeGen/Xtensa/threadptr.ll
 create mode 100644 llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll
 create mode 100644 llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll
 create mode 100644 llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll
 create mode 100644 llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir
 create mode 100644 llvm/test/DebugInfo/MIR/AArch64/entry_value_gets_propagated_aarch64.mir
 create mode 100644 llvm/test/DebugInfo/MIR/X86/entry_value_clobbered_stack_copy.mir
 create mode 100644 llvm/test/DebugInfo/MIR/X86/entry_value_gets_propagated_X86.mir
 create mode 100644 llvm/test/FileCheck/var-escape.txt
 create mode 100644 llvm/test/Instrumentation/MemorySanitizer/or.ll
 create mode 100644 llvm/test/LTO/AArch64/module-asm.ll
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_load_tr.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop1_err.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
 create mode 100644 llvm/test/MC/AMDGPU/gfx9_asm_vop2_features.s
 create mode 100644 llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s
 create mode 100644 llvm/test/MC/ARM/Relocations/thumb-branch.s
 delete mode 100644 llvm/test/MC/ARM/thumb-branches.s
 create mode 100644 llvm/test/MC/BPF/bad-tied.s
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_load_tr.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx9_vop2_features.txt
 create mode 100644 llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt
 create mode 100644 llvm/test/MC/Disassembler/Xtensa/threadptr.txt
 delete mode 100644 llvm/test/MC/ELF/comdat.s
 create mode 100644 llvm/test/MC/ELF/group.s
 create mode 100644 llvm/test/MC/ELF/mc-dump.s
 create mode 100644 llvm/test/MC/RISCV/Relocations/mc-dump.s
 create mode 100644 llvm/test/MC/RISCV/rv32p-invalid.s
 create mode 100644 llvm/test/MC/RISCV/rv64p-invalid.s
 create mode 100644 llvm/test/MC/RISCV/xandesvsintload-valid.s
 create mode 100644 llvm/test/MC/X86/Relocations/x86-16.s
 create mode 100644 llvm/test/MC/X86/Relocations/x86-32.s
 create mode 100644 llvm/test/MC/X86/Relocations/x86-64.s
 create mode 100644 llvm/test/MC/X86/gotpcrel-non-globals.ll
 delete mode 100644 llvm/test/MC/X86/unused_reg_var_assign.s
 create mode 100644 llvm/test/MC/Xtensa/dfpaccel.s
 create mode 100644 llvm/test/MC/Xtensa/threadptr.s
 create mode 100644 llvm/test/Other/fatlto.ll
 create mode 100644 llvm/test/Other/pipeline-alias-errors.ll
 create mode 100644 llvm/test/TableGen/DecoderEmitterFnTable.td
 create mode 100644 llvm/test/TableGen/RuntimeLibcallEmitter.td
 create mode 100644 llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll
 create mode 100644 llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll
 rename llvm/test/Transforms/Coroutines/{coro-split-no-lieftime.ll => coro-split-no-lifetime.ll} (100%)
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
 delete mode 100644 llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InferAlignment/gep-recurrence.ll
 create mode 100644 llvm/test/Transforms/Inline/simplify-crosscallsite.ll
 create mode 100644 llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg
 create mode 100644 llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll
 create mode 100644 llvm/test/Transforms/InstCombine/binop-recurrence.ll
 create mode 100644 llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll
 create mode 100644 llvm/test/Transforms/InstCombine/sub-after-sle-is-non-negative.ll
 create mode 100644 llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
 create mode 100644 llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll
 create mode 100644 llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
 delete mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
 delete mode 100644 llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
 create mode 100644 llvm/test/Transforms/MergeICmps/X86/blockaddresses.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/memset-dse.ll
 create mode 100644 llvm/test/Transforms/lower-builtin-allow-check-pipeline.ll
 create mode 100644 llvm/test/Verifier/branch-weight.ll
 create mode 100644 llvm/test/Verifier/dead-on-return.ll
 create mode 100644 llvm/test/Verifier/value-profile.ll
 create mode 100644 llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/wasm-32bit-tombstone.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/independent-load-stores.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/partially-overlapping-groups.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-aes.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx1.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx2.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxgfni.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxvnni.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-bmi2.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clflushopt.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clwb.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmov.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmpxchg.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-f16c.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fma.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fsgsbase.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-gfni.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lea.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lzcnt.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-mmx.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-movbe.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-pclmul.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-popcnt.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-prefetchw.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdrand.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdseed.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse1.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse2.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse3.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse41.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse42.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-ssse3.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vaes.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vpclmulqdq.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_32.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_64.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x87.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-xsave.s
 create mode 100644 llvm/test/tools/llvm-mca/X86/LunarlakeP/zero-idioms.s
 create mode 100644 llvm/test/tools/llvm-objcopy/COFF/string-table.test
 create mode 100644 llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-executable.yaml
 create mode 100644 llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-relocatable.s
 create mode 100644 llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
 create mode 100644 llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
 create mode 100644 llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
 create mode 100644 llvm/unittests/Frontend/OpenMPDirectiveNameTest.cpp
 create mode 100644 llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
 create mode 100644 llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h
 create mode 100644 mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td
 create mode 100644 mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
 create mode 100644 mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
 create mode 100644 mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h
 create mode 100644 mlir/include/mlir/Support/StateStack.h
 create mode 100644 mlir/include/mlir/Support/WalkResult.h
 create mode 100644 mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
 delete mode 100644 mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
 rename mlir/lib/Dialect/ArmNeon/Transforms/{LowerContractionToSMMLAPattern.cpp => LowerContractionToNeonI8MMPattern.cpp} (59%)
 create mode 100644 mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
 create mode 100644 mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
 create mode 100644 mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
 create mode 100644 mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp
 rename pstl/include/__pstl_memory => mlir/lib/Support/StateStack.cpp (61%)
 create mode 100644 mlir/python/mlir/dialects/TransformDebugExtensionOps.td
 create mode 100644 mlir/python/mlir/dialects/transform/debug.py
 create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
 create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
 create mode 100644 mlir/test/Conversion/GPUToSPIRV/rotate.mlir
 rename mlir/test/Dialect/AMDGPU/{transfer-read-to-load.mlir => maskedload-to-load.mlir} (56%)
 create mode 100644 mlir/test/Dialect/Affine/simplify-min-max-ops.mlir
 create mode 100644 mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
 create mode 100644 mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir
 create mode 100644 mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir
 create mode 100644 mlir/test/Dialect/LLVMIR/xevm.mlir
 create mode 100644 mlir/test/Dialect/Tensor/reify-shapes.mlir
 create mode 100644 mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir
 create mode 100644 mlir/test/Dialect/Tosa/tosa-validation-valid.mlir
 create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/ArmNeon/vector-contract-i8mm.mlir
 create mode 100644 mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/transfer-read-scalable-non-trailing.mlir
 create mode 100644 mlir/test/Target/LLVMIR/nvvm/elect.mlir
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
 create mode 100644 mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
 create mode 100644 mlir/test/Target/LLVMIR/openmp-nested-task-target-parallel.mlir
 create mode 100644 mlir/test/Target/SPIRV/tensorARM.mlir
 create mode 100644 mlir/test/Transforms/make-composed-folded-affine-apply.mlir
 create mode 100644 mlir/test/mlir-translate/emitc_classops.mlir
 create mode 100644 mlir/test/python/dialects/transform_debug_ext.py
 create mode 100644 offload/test/mapping/duplicate_mappings_1.cpp
 create mode 100644 offload/test/mapping/duplicate_mappings_2.cpp
 create mode 100644 offload/test/offloading/disable_default_device.c
 create mode 100644 offload/test/ompTest/test/asserter-seq-test.cpp
 create mode 100644 offload/unittests/Conformance/CMakeLists.txt
 create mode 100644 offload/unittests/Conformance/device_code/CMakeLists.txt
 create mode 100644 offload/unittests/Conformance/device_code/sin.c
 create mode 100644 offload/unittests/Conformance/sin.cpp
 create mode 100644 offload/unittests/OffloadAPI/device_code/localmem.c
 create mode 100644 offload/unittests/OffloadAPI/device_code/localmem_reduction.c
 create mode 100644 offload/unittests/OffloadAPI/device_code/localmem_static.c
 create mode 100644 openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
 delete mode 100644 pstl/.clang-format
 delete mode 100644 pstl/CMakeLists.txt
 delete mode 100644 pstl/CREDITS.txt
 delete mode 100644 pstl/LICENSE.TXT
 delete mode 100644 pstl/README.md
 delete mode 100644 pstl/cmake/ParallelSTLConfig.cmake.in
 delete mode 100644 pstl/docs/ReleaseNotes.rst
 delete mode 100644 pstl/include/__pstl_algorithm
 delete mode 100644 pstl/include/__pstl_config_site.in
 delete mode 100644 pstl/include/__pstl_execution
 delete mode 100644 pstl/include/__pstl_numeric
 delete mode 100644 pstl/include/pstl/internal/algorithm_fwd.h
 delete mode 100644 pstl/include/pstl/internal/algorithm_impl.h
 delete mode 100644 pstl/include/pstl/internal/execution_defs.h
 delete mode 100644 pstl/include/pstl/internal/execution_impl.h
 delete mode 100644 pstl/include/pstl/internal/glue_algorithm_defs.h
 delete mode 100644 pstl/include/pstl/internal/glue_algorithm_impl.h
 delete mode 100644 pstl/include/pstl/internal/glue_execution_defs.h
 delete mode 100644 pstl/include/pstl/internal/glue_memory_defs.h
 delete mode 100644 pstl/include/pstl/internal/glue_memory_impl.h
 delete mode 100644 pstl/include/pstl/internal/glue_numeric_defs.h
 delete mode 100644 pstl/include/pstl/internal/glue_numeric_impl.h
 delete mode 100644 pstl/include/pstl/internal/memory_impl.h
 delete mode 100644 pstl/include/pstl/internal/numeric_fwd.h
 delete mode 100644 pstl/include/pstl/internal/numeric_impl.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_for.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_for_each.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_invoke.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_merge.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_reduce.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_scan.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_stable_partial_sort.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_stable_sort.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_transform_reduce.h
 delete mode 100644 pstl/include/pstl/internal/omp/parallel_transform_scan.h
 delete mode 100644 pstl/include/pstl/internal/omp/util.h
 delete mode 100644 pstl/include/pstl/internal/parallel_backend.h
 delete mode 100644 pstl/include/pstl/internal/parallel_backend_omp.h
 delete mode 100644 pstl/include/pstl/internal/parallel_backend_serial.h
 delete mode 100644 pstl/include/pstl/internal/parallel_backend_tbb.h
 delete mode 100644 pstl/include/pstl/internal/parallel_backend_utils.h
 delete mode 100644 pstl/include/pstl/internal/parallel_impl.h
 delete mode 100644 pstl/include/pstl/internal/pstl_config.h
 delete mode 100644 pstl/include/pstl/internal/unseq_backend_simd.h
 delete mode 100644 pstl/include/pstl/internal/utils.h
 delete mode 100644 pstl/test/CMakeLists.txt
 delete mode 100644 pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
 delete mode 100644 pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
 delete mode 100644 pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
 delete mode 100644 pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
 delete mode 100644 pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
 delete mode 100644 pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
 delete mode 100644 pstl/test/pstl/version.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.merge/merge.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
 delete mode 100644 pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
 delete mode 100644 pstl/test/std/lit.local.cfg
 delete mode 100644 pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
 delete mode 100644 pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
 delete mode 100644 pstl/test/std/numerics/numeric.ops/scan.fail.cpp
 delete mode 100644 pstl/test/std/numerics/numeric.ops/scan.pass.cpp
 delete mode 100644 pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
 delete mode 100644 pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
 delete mode 100644 pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
 delete mode 100644 pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
 delete mode 100644 pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
 delete mode 100644 pstl/test/support/pstl_test_config.h
 delete mode 100644 pstl/test/support/stdlib/algorithm
 delete mode 100644 pstl/test/support/stdlib/execution
 delete mode 100644 pstl/test/support/stdlib/memory
 delete mode 100644 pstl/test/support/stdlib/numeric
 delete mode 100644 pstl/test/support/utils.h
 create mode 100644 utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel
 create mode 100644 utils/bazel/third_party_build/pyyaml.BUILD

diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index 80e4e0221b3d..c3cf714ce6c1 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -77,6 +77,7 @@
 DEPENDENT_RUNTIMES_TO_TEST = {
     "clang": {"compiler-rt"},
     "clang-tools-extra": {"libc"},
+    "libc": {"libc"},
     ".ci": {"compiler-rt", "libc"},
 }
 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index 5efac2636698..6299931e1ec3 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -187,7 +187,7 @@ def test_top_level_file(self):
         self.assertEqual(env_variables["runtimes_check_targets"], "")
         self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
-    def test_exclude_runtiems_in_projects(self):
+    def test_exclude_libcxx_in_projects(self):
         env_variables = compute_projects.get_env_variables(
             ["libcxx/CMakeLists.txt"], "Linux"
         )
@@ -197,6 +197,16 @@ def test_exclude_runtiems_in_projects(self):
         self.assertEqual(env_variables["runtimes_check_targets"], "")
         self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
+    def test_include_libc_in_runtimes(self):
+        env_variables = compute_projects.get_env_variables(
+            ["libc/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(env_variables["projects_to_build"], "clang;lld")
+        self.assertEqual(env_variables["project_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_to_build"], "libc")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
+
     def test_exclude_docs(self):
         env_variables = compute_projects.get_env_variables(
             ["llvm/docs/CIBestPractices.rst"], "Linux"
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 89447963b852..8d1faab13986 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -56,7 +56,7 @@ runtime_targets_needs_reconfig="${5}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
-echo "--- cmake"
+echo "::group::cmake"
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
@@ -85,38 +85,49 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON \
       -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
 
-echo "--- ninja"
+echo "::endgroup::"
+echo "::group::ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
+echo "::endgroup::"
+
 if [[ "${runtime_targets}" != "" ]]; then
-  echo "--- ninja runtimes"
+  echo "::group::ninja runtimes"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets}
+
+  echo "::endgroup::"
 fi
 
 # Compiling runtimes with just-built Clang and running their tests
 # as an additional testing for Clang.
 if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
-  echo "--- cmake runtimes C++26"
+  echo "::group::cmake runtimes C++26"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="std=c++26" \
     -D LIBCXXABI_TEST_PARAMS="std=c++26" \
     "${BUILD_DIR}"
 
-  echo "--- ninja runtimes C++26"
+  echo "::endgroup::"
+  echo "::group::ninja runtimes C++26"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
 
-  echo "--- cmake runtimes clang modules"
+  echo "::endgroup::"
+  echo "::group::cmake runtimes clang modules"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
     -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
     "${BUILD_DIR}"
 
-  echo "--- ninja runtimes clang modules"
+  echo "::endgroup::"
+  echo "::group::ninja runtimes clang modules"
 
   ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
+
+  echo "::endgroup::"
 fi
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index dc2913830e92..c27111bf5aa1 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -46,7 +46,7 @@ trap at-exit EXIT
 projects="${1}"
 targets="${2}"
 
-echo "--- cmake"
+echo "::group::cmake"
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
 export CC=cl
@@ -78,6 +78,10 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_PARALLEL_COMPILE_JOBS=${MAX_PARALLEL_COMPILE_JOBS} \
       -D LLVM_PARALLEL_LINK_JOBS=${MAX_PARALLEL_LINK_JOBS}
 
-echo "--- ninja"
+echo "::endgroup::"
+echo "::group::ninja"
+
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
+
+echo "::endgroup"
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
new file mode 100644
index 000000000000..03748938700e
--- /dev/null
+++ b/.github/copilot-instructions.md
@@ -0,0 +1,4 @@
+When performing a code review, pay close attention to code modifying a function's
+control flow. Could the change result in the corruption of performance profile
+data? Could the change result in invalid debug information, in particular for
+branches and calls?
diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index 2f8d5745668d..863090af9af7 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -632,6 +632,10 @@ llvm:instcombine:
   - llvm/test/Transforms/InstCombine/**
   - llvm/test/Transforms/InstSimplify/**
 
+llvm:vectorcombine:
+  - llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+  - llvm/test/Transforms/VectorCombine/**
+
 clangd:
   - clang-tools-extra/clangd/**
 
diff --git a/.github/workflows/hlsl-test-all.yaml b/.github/workflows/hlsl-test-all.yaml
index 93a1c6d2662d..b6530fe11b84 100644
--- a/.github/workflows/hlsl-test-all.yaml
+++ b/.github/workflows/hlsl-test-all.yaml
@@ -43,13 +43,13 @@ jobs:
       - name: Checkout OffloadTest
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-test-suite
+          repository: llvm/offload-test-suite
           ref: main
           path: OffloadTest
       - name: Checkout Golden Images
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          repository: llvm-beanz/offload-golden-images
+          repository: llvm/offload-golden-images
           ref: main
           path: golden-images
       - name: Setup Windows
diff --git a/.github/workflows/libcxx-restart-preempted-jobs.yaml b/.github/workflows/libcxx-restart-preempted-jobs.yaml
index 9706f0459922..accb84efb5c9 100644
--- a/.github/workflows/libcxx-restart-preempted-jobs.yaml
+++ b/.github/workflows/libcxx-restart-preempted-jobs.yaml
@@ -20,7 +20,7 @@ permissions:
 
 jobs:
   restart:
-    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled')
+    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure')
     name: "Restart Job"
     permissions:
       statuses: read
@@ -32,7 +32,10 @@ jobs:
         uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
         with:
           script: |
-            const failure_regex = /Process completed with exit code 1./
+            // The "The run was canceled by" message comes from a user manually canceling a workflow
+            // the "higher priority" message comes from github canceling a workflow because the user updated the change.
+            // And the "exit code 1" message indicates a genuine failure.
+            const failure_regex = /(Process completed with exit code 1.)/
             const preemption_regex = /(The runner has received a shutdown signal)|(The operation was canceled)/
 
             const wf_run = context.payload.workflow_run
@@ -74,7 +77,7 @@ jobs:
                 console.log('Check run was not completed. Skipping.');
                 continue;
               }
-              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
+              if (check_run.conclusion != 'failure') {
                 console.log('Check run had conclusion: ' + check_run.conclusion + '. Skipping.');
                 continue;
               }
@@ -153,91 +156,3 @@ jobs:
                 run_id: context.payload.workflow_run.id
               })
             await create_check_run('success', 'Restarted workflow run due to preempted job')
-
-  restart-test:
-    if: github.repository_owner == 'llvm' && (github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled') && github.event.actor.login == 'ldionne' # TESTING ONLY
-    name: "Restart Job (test)"
-    permissions:
-      statuses: read
-      checks: write
-      actions: write
-    runs-on: ubuntu-24.04
-    steps:
-      - name: "Restart Job (test)"
-        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
-        with:
-          script: |
-            const FAILURE_REGEX = /Process completed with exit code 1./
-            const PREEMPTION_REGEX = /(The runner has received a shutdown signal)|(The operation was canceled)/
-
-            function log(msg) {
-              core.notice(msg)
-            }
-
-            const wf_run = context.payload.workflow_run
-            log(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
-
-            log('Listing check runs for suite')
-            const check_suites = await github.rest.checks.listForSuite({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              check_suite_id: context.payload.workflow_run.check_suite_id,
-              per_page: 100 // FIXME: We don't have 100 check runs yet, but we should handle this better.
-            })
-
-            preemptions = [];
-            legitimate_failures = [];
-            for (check_run of check_suites.data.check_runs) {
-              log(`Checking check run: ${check_run.id}`);
-              if (check_run.status != 'completed') {
-                log('Check run was not completed. Skipping.');
-                continue;
-              }
-
-              if (check_run.conclusion != 'failure' && check_run.conclusion != 'cancelled') {
-                log(`Check run had conclusion: ${check_run.conclusion}. Skipping.`);
-                continue;
-              }
-
-              annotations = await github.rest.checks.listAnnotations({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                check_run_id: check_run.id
-              })
-
-              preemption_annotation = annotations.data.find(function(annotation) {
-                return annotation.annotation_level == 'failure' &&
-                       annotation.message.match(PREEMPTION_REGEX) != null;
-              });
-              if (preemption_annotation != null) {
-                log(`Found preemption message: ${preemption_annotation.message}`);
-                preemptions.push(check_run);
-                break;
-              }
-
-              failure_annotation = annotations.data.find(function(annotation) {
-                return annotation.annotation_level == 'failure' &&
-                       annotation.message.match(FAILURE_REGEX) != null;
-              });
-              if (failure_annotation != null) {
-                log(`Found legitimate failure annotation: ${failure_annotation.message}`);
-                legitimate_failures.push(check_run);
-                break;
-              }
-            }
-
-            if (preemptions) {
-              log('Found some preempted jobs');
-              if (legitimate_failures) {
-                log('Also found some legitimate failures, so not restarting the workflow.');
-              } else {
-                log('Did not find any legitimate failures. Restarting workflow.');
-                await github.rest.actions.reRunWorkflowFailedJobs({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  run_id: context.payload.workflow_run.id
-                })
-              }
-            } else {
-              log('Did not find any preempted jobs. Not restarting the workflow.');
-            }
diff --git a/amd/comgr/include/amd_comgr.h.in b/amd/comgr/include/amd_comgr.h.in
index 3c7df1c0901e..3f2f75586d61 100644
--- a/amd/comgr/include/amd_comgr.h.in
+++ b/amd/comgr/include/amd_comgr.h.in
@@ -1563,9 +1563,6 @@ amd_comgr_action_info_get_working_directory_path(
  *
  * @retval ::AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT @p
  * action_info is an invalid action info object.
- *
- * @retval ::AMD_COMGR_STATUS_ERROR_OUT_OF_RESOURCES
- * Unable to update action info object as out of resources.
  */
 amd_comgr_status_t AMD_COMGR_API
 amd_comgr_action_info_set_logging(
@@ -1584,9 +1581,6 @@ amd_comgr_action_info_set_logging(
  *
  * @retval ::AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT @p
  * action_info is an invalid action info object. @p logging is NULL.
- *
- * @retval ::AMD_COMGR_STATUS_ERROR_OUT_OF_RESOURCES
- * Unable to update the data object as out of resources.
  */
 amd_comgr_status_t AMD_COMGR_API
 amd_comgr_action_info_get_logging(
diff --git a/amd/comgr/src/comgr-compiler.cpp b/amd/comgr/src/comgr-compiler.cpp
index 756513246f25..5e69d9b9cfab 100644
--- a/amd/comgr/src/comgr-compiler.cpp
+++ b/amd/comgr/src/comgr-compiler.cpp
@@ -810,10 +810,16 @@ amd_comgr_status_t AMDGPUCompiler::createTmpDirs() {
   return AMD_COMGR_STATUS_SUCCESS;
 }
 
+amd_comgr_status_t AMDGPUCompiler::removeTmpDirs() {
+  if (TmpDir.empty()) {
+    return AMD_COMGR_STATUS_SUCCESS;
+  }
+  ProfilePoint Point("RemoveDir");
+
+#ifdef _WIN32
 // On windows fs::remove_directories takes huge time so use fs::remove.
-amd_comgr_status_t removeDirectory(const StringRef DirName) {
   std::error_code EC;
-  for (fs::directory_iterator Dir(DirName, EC), DirEnd; Dir != DirEnd && !EC;
+  for (fs::directory_iterator Dir(TmpDir, EC), DirEnd; Dir != DirEnd && !EC;
        Dir.increment(EC)) {
     const StringRef Path = Dir->path();
 
@@ -843,25 +849,16 @@ amd_comgr_status_t removeDirectory(const StringRef DirName) {
     }
   }
 
-  if (fs::remove(DirName)) {
+  if (fs::remove(TmpDir)) {
     return AMD_COMGR_STATUS_ERROR;
   }
 
   return AMD_COMGR_STATUS_SUCCESS;
-}
-
-amd_comgr_status_t AMDGPUCompiler::removeTmpDirs() {
-  if (TmpDir.empty()) {
-    return AMD_COMGR_STATUS_SUCCESS;
-  }
-  ProfilePoint Point("RemoveDir");
-#ifndef _WIN32
+#else
   if (fs::remove_directories(TmpDir)) {
     return AMD_COMGR_STATUS_ERROR;
   }
   return AMD_COMGR_STATUS_SUCCESS;
-#else
-  return removeDirectory(TmpDir);
 #endif
 }
 
diff --git a/amd/comgr/test-lit/CMakeLists.txt b/amd/comgr/test-lit/CMakeLists.txt
index b7466b21be51..73966d7f3072 100644
--- a/amd/comgr/test-lit/CMakeLists.txt
+++ b/amd/comgr/test-lit/CMakeLists.txt
@@ -40,8 +40,11 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 
 add_comgr_lit_binary(source-to-bc-with-dev-libs c)
 add_comgr_lit_binary(spirv-translator c)
-add_comgr_lit_binary(compile-minimal-test c)
+add_comgr_lit_binary(compile-opencl-minimal c)
 add_comgr_lit_binary(spirv-to-reloc c)
 add_comgr_lit_binary(unbundle c)
+add_comgr_lit_binary(get-version c)
+add_comgr_lit_binary(status-string c)
+add_comgr_lit_binary(data-action c)
 
 add_dependencies(check-comgr test-lit)
diff --git a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-dir.cl b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-dir.cl
index b8b9c48b227d..df6aa7f068f8 100644
--- a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-dir.cl
+++ b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-dir.cl
@@ -6,7 +6,7 @@
 // RUN: AMD_COMGR_CACHE_DIR=%t.txt \
 // RUN:   AMD_COMGR_EMIT_VERBOSE_LOGS=1 \
 // RUN:   AMD_COMGR_REDIRECT_LOGS=%t.log \
-// RUN:     compile-minimal-test %S/../compile-minimal-test.cl %t.bin
+// RUN:     compile-opencl-minimal %S/../compile-minimal-test.cl %t.bin 1.2
 // RUN: llvm-objdump -d %t.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: FileCheck --check-prefix=BAD %s < %t.log
 // BAD: Failed to open cache file
diff --git a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-policy.cl b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-policy.cl
index f5962b8ac62f..c1f4bf35d5af 100644
--- a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-policy.cl
+++ b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached-bad-policy.cl
@@ -6,7 +6,7 @@
 // RUN:   AMD_COMGR_CACHE_POLICY="foo=2h" \
 // RUN:   AMD_COMGR_EMIT_VERBOSE_LOGS=1 \
 // RUN:   AMD_COMGR_REDIRECT_LOGS=%t.log \
-// RUN:     compile-minimal-test %S/../compile-minimal-test.cl %t.bin
+// RUN:     compile-opencl-minimal %S/../compile-minimal-test.cl %t.bin 1.2
 // RUN: llvm-objdump -d %t.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: FileCheck --check-prefix=BAD %s < %t.log
 // BAD: when parsing the cache policy: Unknown key: 'foo'
diff --git a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached.cl b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached.cl
index 06471b1d6088..546d6dfd4864 100644
--- a/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached.cl
+++ b/amd/comgr/test-lit/cache-tests/compile-minimal-test-cached.cl
@@ -1,16 +1,16 @@
 // RUN: rm -fr %t.cache
 //
 // RUN: unset AMD_COMGR_CACHE
-// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-minimal-test \
-// RUN:    %S/../compile-minimal-test.cl %t.bin
+// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-opencl-minimal \
+// RUN:    %S/../compile-minimal-test.cl %t.bin 1.2
 // RUN: llvm-objdump -d %t.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: [ -d %t.cache ]
 //
 // RUN: rm -fr %t.cache
 //
 // RUN: export AMD_COMGR_CACHE=0
-// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-minimal-test \
-// RUN:    %S/../compile-minimal-test.cl %t.bin
+// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-opencl-minimal \
+// RUN:    %S/../compile-minimal-test.cl %t.bin 1.2
 // RUN: llvm-objdump -d %t.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: [ ! -d %t.cache ]
 //
@@ -19,8 +19,8 @@
 // COM: Run once and check that the cache directory exists and it has more than
 // COM     1 element (one for the cache tag, one or more for the cached
 // COM:    commands)
-// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-minimal-test \
-// RUN:    %S/../compile-minimal-test.cl %t_a.bin
+// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-opencl-minimal \
+// RUN:    %S/../compile-minimal-test.cl %t_a.bin 1.2
 // RUN: llvm-objdump -d %t_a.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: COUNT_BEFORE=$(ls "%t.cache" | wc -l)
 
@@ -28,8 +28,8 @@
 // COM:    elements for src->bc since we currently not support it.
 // RUN: [ 3 -eq $COUNT_BEFORE ]
 //
-// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-minimal-test \
-// RUN:    %S/../compile-minimal-test.cl %t_b.bin
+// RUN: AMD_COMGR_CACHE_DIR=%t.cache compile-opencl-minimal \
+// RUN:    %S/../compile-minimal-test.cl %t_b.bin 1.2
 // RUN: llvm-objdump -d %t_b.bin | FileCheck %S/../compile-minimal-test.cl
 // RUN: COUNT_AFTER=$(ls "%t.cache" | wc -l)
 // RUN: [ $COUNT_AFTER = $COUNT_BEFORE ]
diff --git a/amd/comgr/test-lit/comgr-sources/common.h b/amd/comgr/test-lit/comgr-sources/common.h
index 8f95bfcc2eae..130d25f685da 100644
--- a/amd/comgr/test-lit/comgr-sources/common.h
+++ b/amd/comgr/test-lit/comgr-sources/common.h
@@ -38,6 +38,17 @@
     }                                                                          \
   } while (false)
 
+#define fail_amd_comgr_(call)                                                  \
+  do {                                                                         \
+    amd_comgr_status_t status = amd_comgr_##call;                              \
+    if (status == AMD_COMGR_STATUS_SUCCESS) {                                  \
+      const char *reason = "";                                                 \
+      amd_comgr_status_string(status, &reason);                                \
+      fail(#call " expected fail: %s\n  file, line: %s, %d\n", reason,         \
+           __FILE__, __LINE__);                                                \
+    }                                                                          \
+  } while (false)
+
 static void fail(const char *format, ...) {
   va_list ap;
   va_start(ap, format);
diff --git a/amd/comgr/test-lit/comgr-sources/compile-minimal-test.c b/amd/comgr/test-lit/comgr-sources/compile-opencl-minimal.c
similarity index 92%
rename from amd/comgr/test-lit/comgr-sources/compile-minimal-test.c
rename to amd/comgr/test-lit/comgr-sources/compile-opencl-minimal.c
index d7a33045b482..a5a69167a7dc 100644
--- a/amd/comgr/test-lit/comgr-sources/compile-minimal-test.c
+++ b/amd/comgr/test-lit/comgr-sources/compile-opencl-minimal.c
@@ -1,4 +1,4 @@
-//===- compile-minimal-test.c ---------------------------------------------===//
+//===- compile-opencl-minimal.c -------------------------------------------===//
 //
 // Part of Comgr, under the Apache License v2.0 with LLVM Exceptions. See
 // amd/comgr/LICENSE.TXT in this repository for license information.
@@ -26,6 +26,16 @@ int main(int argc, char *argv[]) {
 
   SizeSource = setBuf(argv[1], &BufSource);
 
+  amd_comgr_language_t OpenCLVersion;
+  if (strcmp(argv[3], "1.2")) {
+    OpenCLVersion = AMD_COMGR_LANGUAGE_OPENCL_1_2;
+  }
+  else if (strcmp(argv[3], "2.0")) {
+    OpenCLVersion = AMD_COMGR_LANGUAGE_OPENCL_1_2;
+  }
+  else
+    fail("unsupported OCL version: %s", argv[3]);
+
   amd_comgr_(create_data_set(&DataSetIn));
   amd_comgr_(create_data(AMD_COMGR_DATA_KIND_SOURCE, &DataSource));
   amd_comgr_(set_data(DataSource, SizeSource, BufSource));
diff --git a/amd/comgr/test-lit/comgr-sources/data-action.c b/amd/comgr/test-lit/comgr-sources/data-action.c
new file mode 100644
index 000000000000..13cb24c65890
--- /dev/null
+++ b/amd/comgr/test-lit/comgr-sources/data-action.c
@@ -0,0 +1,129 @@
+//===- compile-minimal-test.c ---------------------------------------------===//
+//
+// Part of Comgr, under the Apache License v2.0 with LLVM Exceptions. See
+// amd/comgr/LICENSE.TXT in this repository for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "amd_comgr.h"
+#include "common.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+  amd_comgr_action_info_t DataAction;
+  amd_comgr_(create_action_info(&DataAction));
+
+  // ---- set_language, get_language
+  amd_comgr_language_t Language;
+  amd_comgr_(
+      action_info_set_language(DataAction, AMD_COMGR_LANGUAGE_NONE));
+  amd_comgr_(action_info_get_language(DataAction, &Language));
+  if (Language != AMD_COMGR_LANGUAGE_NONE)
+    fail("AMD_COMGR_LANGUAGE_NONE not returned!");
+
+  amd_comgr_(
+      action_info_set_language(DataAction, AMD_COMGR_LANGUAGE_OPENCL_1_2));
+  amd_comgr_(action_info_get_language(DataAction, &Language));
+  if (Language != AMD_COMGR_LANGUAGE_OPENCL_1_2)
+    fail("AMD_COMGR_LANGUAGE_OPENCL_1_2 not returned!");
+
+  amd_comgr_(
+    action_info_set_language(DataAction, AMD_COMGR_LANGUAGE_OPENCL_2_0));
+  amd_comgr_(action_info_get_language(DataAction, &Language));
+  if (Language != AMD_COMGR_LANGUAGE_OPENCL_2_0)
+    fail("AMD_COMGR_LANGUAGE_OPENCL_2_0 not returned!");
+
+  amd_comgr_(
+    action_info_set_language(DataAction, AMD_COMGR_LANGUAGE_HIP));
+  amd_comgr_(action_info_get_language(DataAction, &Language));
+  if (Language != AMD_COMGR_LANGUAGE_HIP)
+    fail("AMD_COMGR_LANGUAGE_HIP not returned!");
+
+  amd_comgr_(
+    action_info_set_language(DataAction, AMD_COMGR_LANGUAGE_LLVM_IR));
+  amd_comgr_(action_info_get_language(DataAction, &Language));
+  if (Language != AMD_COMGR_LANGUAGE_LLVM_IR)
+    fail("AMD_COMGR_LANGUAGE_LLVM_IR not returned!");
+
+  // ---- set_isa_name, get_isa_name
+  // Tested in comgr/test/get_data_isa_name_test.c
+
+  // ---- set_option_list, get_option_list_count, get_option_list_item
+  const char *Options[] = {"foo", "bar", "bazqux", "aaaaaaaaaaaaaaaaaaaaa"};
+  size_t OptionsCount = sizeof(Options) / sizeof(Options[0]);
+
+  amd_comgr_(action_info_set_option_list(DataAction, Options, OptionsCount));
+
+  size_t ActualCount;
+  amd_comgr_(action_info_get_option_list_count(DataAction, &ActualCount));
+
+  if (OptionsCount != ActualCount) {
+    fail("incorrect option count: expected %zu, saw %zu", OptionsCount,
+         ActualCount);
+  }
+
+  size_t Size;
+  for (size_t I = 0; I < OptionsCount; ++I) {
+    amd_comgr_(action_info_get_option_list_item(DataAction, I, &Size, NULL));
+
+    char *Option = calloc(Size, sizeof(char));
+    amd_comgr_(action_info_get_option_list_item(DataAction, I, &Size, Option));
+
+    if (strcmp(Options[I], Option)) {
+      fail("incorrect option string: expected '%s', saw '%s'", Options[I],
+           Option);
+    }
+    free(Option);
+  }
+
+  fail_amd_comgr_(action_info_get_option_list_item(DataAction, OptionsCount,
+                                                   &Size, NULL));
+  fail_amd_comgr_(action_info_get_option_list_count(DataAction, NULL));
+  fail_amd_comgr_(action_info_get_option_list_item(DataAction, 0, NULL, NULL));
+
+  // ---- set_bundle_entry_ids, get_bundle_entry_id_count, get_bundle_entry_id
+  // Tested in comgr/test/unbundle-hip-test.c
+
+  // ---- set_working_directory_path, get_working_directory_path
+  const char *Path = "/path/to/my/directory";
+  amd_comgr_(action_info_set_working_directory_path(DataAction, Path));
+
+  amd_comgr_(action_info_get_working_directory_path(DataAction, &Size,
+                                                    NULL));
+  char *GetPath = calloc(Size, sizeof(char));
+  amd_comgr_(action_info_get_working_directory_path(DataAction, &Size,
+                                                    GetPath));
+
+  if (strcmp(Path, GetPath))
+    fail("incorrect path string: expected '%s', saw '%s'", Path, GetPath);
+  free(GetPath);
+
+  // ---- set_logging, get_logging
+  amd_comgr_(action_info_set_logging(DataAction, true));
+
+  bool GetLogging;
+  amd_comgr_(action_info_get_logging(DataAction, &GetLogging));
+
+  if (!GetLogging)
+    fail("incorrect logging boolean: expected 'true', saw 'false'");
+
+  amd_comgr_(action_info_set_logging(DataAction, false));
+  amd_comgr_(action_info_get_logging(DataAction, &GetLogging));
+
+  if (GetLogging)
+    fail("incorrect logging boolean: expected 'false', saw 'true'");
+
+  // ---- set_device_lib_linking
+  amd_comgr_(action_info_set_device_lib_linking(DataAction, true));
+  amd_comgr_(action_info_set_device_lib_linking(DataAction, false));
+
+  // ---- set_vfs
+  amd_comgr_(action_info_set_vfs(DataAction, true));
+  amd_comgr_(action_info_set_vfs(DataAction, false));
+
+  amd_comgr_(destroy_action_info(DataAction));
+  return 0;
+}
diff --git a/amd/comgr/test-lit/comgr-sources/get-version.c b/amd/comgr/test-lit/comgr-sources/get-version.c
new file mode 100644
index 000000000000..2af958848239
--- /dev/null
+++ b/amd/comgr/test-lit/comgr-sources/get-version.c
@@ -0,0 +1,25 @@
+//===- get-version.c ------------------------------------------------------===//
+//
+// Part of Comgr, under the Apache License v2.0 with LLVM Exceptions. See
+// amd/comgr/LICENSE.TXT in this repository for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "amd_comgr.h"
+#include "common.h"
+
+int main(int argc, char *argv[]) {
+
+  size_t *Major = malloc(sizeof(size_t));
+  size_t *Minor = malloc(sizeof(size_t));
+
+  amd_comgr_get_version(Major, Minor);
+
+  if (*Major != 3 || *Minor != 0)
+    fail("incorrect version: expected 3.0, saw %d, %d", *Major, *Minor);
+
+  free(Major);
+  free(Minor);
+  return 0;
+}
diff --git a/amd/comgr/test-lit/comgr-sources/status-string.c b/amd/comgr/test-lit/comgr-sources/status-string.c
new file mode 100644
index 000000000000..ffad82be9c9a
--- /dev/null
+++ b/amd/comgr/test-lit/comgr-sources/status-string.c
@@ -0,0 +1,42 @@
+//===- status-string.c ----------------------------------------------------===//
+//
+// Part of Comgr, under the Apache License v2.0 with LLVM Exceptions. See
+// amd/comgr/LICENSE.TXT in this repository for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "amd_comgr.h"
+#include "common.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+  const char *StatusString = malloc(sizeof(char) * 100);
+  amd_comgr_(status_string(AMD_COMGR_STATUS_SUCCESS, &StatusString));
+  if (strcmp(StatusString, "SUCCESS"))
+    fail("incorrect status: expected 'SUCCESS', saw '%s'", StatusString);
+
+  amd_comgr_(status_string(AMD_COMGR_STATUS_ERROR, &StatusString));
+  if (strcmp(StatusString, "ERROR"))
+    fail("incorrect status: expected 'ERROR', saw '%s'", StatusString);
+
+  amd_comgr_(status_string(AMD_COMGR_STATUS_ERROR_INVALID_ARGUMENT,
+                           &StatusString));
+  if (strcmp(StatusString, "INVALID_ARGUMENT")) {
+    fail("incorrect status: expected 'INVALID_ARGUMENT', saw '%s'",
+         StatusString);
+  }
+
+  amd_comgr_(status_string(AMD_COMGR_STATUS_ERROR_OUT_OF_RESOURCES,
+                           &StatusString));
+  if (strcmp(StatusString, "OUT_OF_RESOURCES")) {
+    fail("incorrect status: expected 'OUT_OF_RESOURCES', saw '%s'",
+         StatusString);
+  }
+
+  fail_amd_comgr_(status_string(-1, &StatusString));
+  free(StatusString);
+  return 0;
+}
diff --git a/amd/comgr/test-lit/compile-minimal-test.cl b/amd/comgr/test-lit/compile-minimal-test.cl
index 460692d1ce61..3f09758519d4 100644
--- a/amd/comgr/test-lit/compile-minimal-test.cl
+++ b/amd/comgr/test-lit/compile-minimal-test.cl
@@ -1,6 +1,6 @@
 // COM: Run Comgr binary to compile OpenCL source into LLVM IR Bitcode,
 // COM: and, then generate an executable
-// RUN: compile-minimal-test %s %t.bin
+// RUN: compile-opencl-minimal %s %t.bin 1.2
 
 // COM: Dissasemble
 // RUN: llvm-objdump -d %t.bin | FileCheck %s
diff --git a/amd/comgr/test-lit/compile-opencl-2.cl b/amd/comgr/test-lit/compile-opencl-2.cl
new file mode 100644
index 000000000000..67d7164f69b4
--- /dev/null
+++ b/amd/comgr/test-lit/compile-opencl-2.cl
@@ -0,0 +1,12 @@
+// COM: Run Comgr binary to compile OpenCL source into LLVM IR Bitcode,
+// COM: and, then generate an executable
+// RUN: compile-opencl-minimal %s %t.bin 2.0
+
+// COM: Dissasemble
+// RUN: llvm-objdump -d %t.bin | FileCheck %s
+// CHECK: <add>:
+// CHECK: s_endpgm
+
+void kernel add(__global float *A, __global float *B, __global float *C) {
+    *C = *A + *B;
+}
diff --git a/amd/comgr/test-lit/data-action.c b/amd/comgr/test-lit/data-action.c
new file mode 100644
index 000000000000..0df24cc3148c
--- /dev/null
+++ b/amd/comgr/test-lit/data-action.c
@@ -0,0 +1,2 @@
+// COM: Run Comgr binary test data action APIs
+// RUN: data-action
diff --git a/amd/comgr/test-lit/get-version.c b/amd/comgr/test-lit/get-version.c
new file mode 100644
index 000000000000..d5b3cf63d2fa
--- /dev/null
+++ b/amd/comgr/test-lit/get-version.c
@@ -0,0 +1,2 @@
+// COM: Test Comgr get_version() API
+// RUN: get-version
diff --git a/amd/comgr/test-lit/status-string.c b/amd/comgr/test-lit/status-string.c
new file mode 100644
index 000000000000..6e04df4d8ca1
--- /dev/null
+++ b/amd/comgr/test-lit/status-string.c
@@ -0,0 +1,2 @@
+// COM: Check the Comgr status string API
+// RUN: status-string
diff --git a/amd/comgr/test-lit/time-statistics.cl b/amd/comgr/test-lit/time-statistics.cl
index 5881e3787984..2e546b2c434e 100644
--- a/amd/comgr/test-lit/time-statistics.cl
+++ b/amd/comgr/test-lit/time-statistics.cl
@@ -1,5 +1,5 @@
 // COM: Check for any runtime errors with the Comgr Profilier
-// RUN: AMD_COMGR_TIME_STATISTICS=1 compile-minimal-test %s %t.bin
+// RUN: AMD_COMGR_TIME_STATISTICS=1 compile-opencl-minimal %s %t.bin 1.2
 // RUN: test -f PerfStatsLog.txt
 
 void kernel add(__global float *A, __global float *B, __global float *C) {
diff --git a/amd/comgr/test/CMakeLists.txt b/amd/comgr/test/CMakeLists.txt
index 4d6d15e32ed1..1899e3cb2540 100644
--- a/amd/comgr/test/CMakeLists.txt
+++ b/amd/comgr/test/CMakeLists.txt
@@ -227,7 +227,6 @@ add_comgr_test(link_test c)
 add_comgr_test(isa_name_parsing_test c)
 add_comgr_test(get_data_isa_name_test c)
 add_comgr_test(include_subdirectory_test c)
-add_comgr_test(options_test c)
 add_comgr_test(demangle_test c)
 add_comgr_test(fail_to_build_driver c)
 add_comgr_test(file_map c)
diff --git a/amd/comgr/test/compile_source_with_device_libs_to_bc_test.c b/amd/comgr/test/compile_source_with_device_libs_to_bc_test.c
index 78dd46e9b118..802d726cb95b 100644
--- a/amd/comgr/test/compile_source_with_device_libs_to_bc_test.c
+++ b/amd/comgr/test/compile_source_with_device_libs_to_bc_test.c
@@ -16,8 +16,8 @@ int main(int argc, char *argv[]) {
   char *BufSource;
   size_t SizeSource;
   amd_comgr_data_t DataSource;
-  amd_comgr_data_set_t DataSetIn, DataSetPch, DataSetBc, DataSetLinked,
-      DataSetReloc, DataSetExec;
+  amd_comgr_data_set_t DataSetIn, DataSetBc, DataSetLinked, DataSetReloc,
+      DataSetExec;
   amd_comgr_action_info_t DataAction;
   amd_comgr_status_t Status;
   const char *CodeGenOptions[] = {"-mcode-object-version=5", "-mllvm",
@@ -48,25 +48,6 @@ int main(int argc, char *argv[]) {
                                               "amdgcn-amd-amdhsa--gfx900");
   checkError(Status, "amd_comgr_action_info_set_isa_name");
 
-  Status = amd_comgr_create_data_set(&DataSetPch);
-  checkError(Status, "amd_comgr_create_data_set");
-
-  Status = amd_comgr_do_action(AMD_COMGR_ACTION_ADD_PRECOMPILED_HEADERS,
-                               DataAction, DataSetIn, DataSetPch);
-  checkError(Status, "amd_comgr_do_action");
-
-  size_t Count;
-  Status = amd_comgr_action_data_count(
-      DataSetPch, AMD_COMGR_DATA_KIND_PRECOMPILED_HEADER, &Count);
-  checkError(Status, "amd_comgr_action_data_count");
-
-  if (Count != 0) {
-    printf("AMD_COMGR_ACTION_ADD_PRECOMPILED_HEADERS Failed: "
-           "produced %zu precompiled header objects (expected 0)\n",
-           Count);
-    exit(1);
-  }
-
   Status = amd_comgr_create_data_set(&DataSetBc);
   checkError(Status, "amd_comgr_create_data_set");
 
@@ -76,9 +57,10 @@ int main(int argc, char *argv[]) {
 
   Status = amd_comgr_do_action(
       AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC, DataAction,
-      DataSetPch, DataSetBc);
+      DataSetIn, DataSetBc);
   checkError(Status, "amd_comgr_do_action");
 
+  size_t Count;
   Status =
       amd_comgr_action_data_count(DataSetBc, AMD_COMGR_DATA_KIND_BC, &Count);
   checkError(Status, "amd_comgr_action_data_count");
@@ -151,8 +133,6 @@ int main(int argc, char *argv[]) {
   checkError(Status, "amd_comgr_release_data");
   Status = amd_comgr_destroy_data_set(DataSetIn);
   checkError(Status, "amd_comgr_destroy_data_set");
-  Status = amd_comgr_destroy_data_set(DataSetPch);
-  checkError(Status, "amd_comgr_destroy_data_set");
   Status = amd_comgr_destroy_data_set(DataSetBc);
   checkError(Status, "amd_comgr_destroy_data_set");
   Status = amd_comgr_destroy_data_set(DataSetLinked);
diff --git a/amd/comgr/test/compile_source_with_device_libs_to_bc_with_vfs_test.c b/amd/comgr/test/compile_source_with_device_libs_to_bc_with_vfs_test.c
index 1d355017d6d9..1c98243f6305 100644
--- a/amd/comgr/test/compile_source_with_device_libs_to_bc_with_vfs_test.c
+++ b/amd/comgr/test/compile_source_with_device_libs_to_bc_with_vfs_test.c
@@ -16,8 +16,8 @@ int main(int argc, char *argv[]) {
   char *BufSource;
   size_t SizeSource;
   amd_comgr_data_t DataSource;
-  amd_comgr_data_set_t DataSetIn, DataSetPch, DataSetBc, DataSetLinked,
-      DataSetReloc, DataSetExec;
+  amd_comgr_data_set_t DataSetIn, DataSetBc, DataSetLinked, DataSetReloc,
+      DataSetExec;
   amd_comgr_action_info_t DataAction;
   amd_comgr_status_t Status;
   const char *CodeGenOptions[] = {"-mcode-object-version=5", "-mllvm",
@@ -51,25 +51,6 @@ int main(int argc, char *argv[]) {
   Status = amd_comgr_action_info_set_vfs(DataAction, true);
   checkError(Status, "amd_comgr_action_info_set_vfs");
 
-  Status = amd_comgr_create_data_set(&DataSetPch);
-  checkError(Status, "amd_comgr_create_data_set");
-
-  Status = amd_comgr_do_action(AMD_COMGR_ACTION_ADD_PRECOMPILED_HEADERS,
-                               DataAction, DataSetIn, DataSetPch);
-  checkError(Status, "amd_comgr_do_action");
-
-  size_t Count;
-  Status = amd_comgr_action_data_count(
-      DataSetPch, AMD_COMGR_DATA_KIND_PRECOMPILED_HEADER, &Count);
-  checkError(Status, "amd_comgr_action_data_count");
-
-  if (Count != 0) {
-    printf("AMD_COMGR_ACTION_ADD_PRECOMPILED_HEADERS Failed: "
-           "produced %zu precompiled header objects (expected 0)\n",
-           Count);
-    exit(1);
-  }
-
   Status = amd_comgr_create_data_set(&DataSetBc);
   checkError(Status, "amd_comgr_create_data_set");
 
@@ -79,9 +60,10 @@ int main(int argc, char *argv[]) {
 
   Status = amd_comgr_do_action(
       AMD_COMGR_ACTION_COMPILE_SOURCE_WITH_DEVICE_LIBS_TO_BC, DataAction,
-      DataSetPch, DataSetBc);
+      DataSetIn, DataSetBc);
   checkError(Status, "amd_comgr_do_action");
 
+  size_t Count;
   Status =
       amd_comgr_action_data_count(DataSetBc, AMD_COMGR_DATA_KIND_BC, &Count);
   checkError(Status, "amd_comgr_action_data_count");
@@ -154,8 +136,6 @@ int main(int argc, char *argv[]) {
   checkError(Status, "amd_comgr_release_data");
   Status = amd_comgr_destroy_data_set(DataSetIn);
   checkError(Status, "amd_comgr_destroy_data_set");
-  Status = amd_comgr_destroy_data_set(DataSetPch);
-  checkError(Status, "amd_comgr_destroy_data_set");
   Status = amd_comgr_destroy_data_set(DataSetBc);
   checkError(Status, "amd_comgr_destroy_data_set");
   Status = amd_comgr_destroy_data_set(DataSetLinked);
diff --git a/amd/comgr/test/options_test.c b/amd/comgr/test/options_test.c
deleted file mode 100644
index 9d54ffff8466..000000000000
--- a/amd/comgr/test/options_test.c
+++ /dev/null
@@ -1,66 +0,0 @@
-//===- options_test.c -----------------------------------------------------===//
-//
-// Part of Comgr, under the Apache License v2.0 with LLVM Exceptions. See
-// amd/comgr/LICENSE.TXT in this repository for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "amd_comgr.h"
-#include "common.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-void testList(amd_comgr_action_info_t ActionInfo, const char *Options[],
-              size_t Count) {
-  size_t ActualCount;
-  amd_comgr_status_t Status;
-
-  Status = amd_comgr_action_info_set_option_list(ActionInfo, Options, Count);
-  checkError(Status, "amd_comgr_action_info_set_option_list");
-
-  Status =
-      amd_comgr_action_info_get_option_list_count(ActionInfo, &ActualCount);
-  checkError(Status, "amd_comgr_action_info_get_option_list_count");
-
-  if (Count != ActualCount) {
-    fail("incorrect option count: expected %zu, saw %zu", Count, ActualCount);
-  }
-
-  for (size_t I = 0; I < Count; ++I) {
-    size_t Size;
-    Status =
-        amd_comgr_action_info_get_option_list_item(ActionInfo, I, &Size, NULL);
-    checkError(Status, "amd_comgr_action_info_get_option_list_item");
-    char *Option = calloc(Size, sizeof(char));
-    Status = amd_comgr_action_info_get_option_list_item(ActionInfo, I, &Size,
-                                                        Option);
-    checkError(Status, "amd_comgr_action_info_get_option_list_item");
-    if (strcmp(Options[I], Option)) {
-      fail("incorrect option string: expected '%s', saw '%s'", Options[I],
-           Option);
-    }
-    free(Option);
-  }
-}
-
-int main(int argc, char *argv[]) {
-  amd_comgr_action_info_t ActionInfo;
-  amd_comgr_status_t Status;
-
-  Status = amd_comgr_create_action_info(&ActionInfo);
-  checkError(Status, "amd_comgr_create_action_info");
-
-  const char *Options[] = {"foo", "bar", "bazqux", "aaaaaaaaaaaaaaaaaaaaa"};
-  size_t OptionsCount = sizeof(Options) / sizeof(Options[0]);
-
-  for (size_t I = 0; I <= OptionsCount; ++I) {
-    for (size_t J = 0; I + J <= OptionsCount; ++J) {
-      testList(ActionInfo, Options + I, J);
-    }
-  }
-
-  Status = amd_comgr_destroy_action_info(ActionInfo);
-  checkError(Status, "amd_comgr_destroy_action_info");
-}
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index 64e524e2f13d..91ecf89da618 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -73,14 +73,15 @@ struct SegmentInfo {
   uint64_t FileSize;          /// Size in file.
   uint64_t Alignment;         /// Alignment of the segment.
   bool IsExecutable;          /// Is the executable bit set on the Segment?
+  bool IsWritable;            /// Is the segment writable.
 
   void print(raw_ostream &OS) const {
     OS << "SegmentInfo { Address: 0x" << Twine::utohexstr(Address)
        << ", Size: 0x" << Twine::utohexstr(Size) << ", FileOffset: 0x"
        << Twine::utohexstr(FileOffset) << ", FileSize: 0x"
        << Twine::utohexstr(FileSize) << ", Alignment: 0x"
-       << Twine::utohexstr(Alignment) << ", " << (IsExecutable ? "x" : " ")
-       << "}";
+       << Twine::utohexstr(Alignment) << ", " << (IsExecutable ? "x" : "")
+       << (IsWritable ? "w" : "") << " }";
   };
 };
 
@@ -333,9 +334,14 @@ class BinaryContext {
                                   std::optional<StringRef> Source,
                                   unsigned CUID, unsigned DWARFVersion);
 
+  /// Input file segment info
+  ///
   /// [start memory address] -> [segment info] mapping.
   std::map<uint64_t, SegmentInfo> SegmentMapInfo;
 
+  /// Newly created segments.
+  std::vector<SegmentInfo> NewSegments;
+
   /// Symbols that are expected to be undefined in MCContext during emission.
   std::unordered_set<MCSymbol *> UndefinedSymbols;
 
diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h
index ad2fed2cf27e..154a8d12de5c 100644
--- a/bolt/include/bolt/Core/BinarySection.h
+++ b/bolt/include/bolt/Core/BinarySection.h
@@ -523,11 +523,6 @@ inline uint8_t *copyByteArray(const uint8_t *Data, uint64_t Size) {
   return Array;
 }
 
-inline uint8_t *copyByteArray(StringRef Buffer) {
-  return copyByteArray(reinterpret_cast<const uint8_t *>(Buffer.data()),
-                       Buffer.size());
-}
-
 inline uint8_t *copyByteArray(ArrayRef<char> Buffer) {
   return copyByteArray(reinterpret_cast<const uint8_t *>(Buffer.data()),
                        Buffer.size());
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index 804100db8079..b99cf8b40ef5 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -1393,7 +1393,7 @@ class MCPlusBuilder {
       return getTargetSymbol(BinaryExpr->getLHS());
 
     auto *SymbolRefExpr = dyn_cast<const MCSymbolRefExpr>(Expr);
-    if (SymbolRefExpr && SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None)
+    if (SymbolRefExpr && SymbolRefExpr->getSpecifier() == 0)
       return &SymbolRefExpr->getSymbol();
 
     return nullptr;
diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index cc28a06c151e..c7400b0d45ca 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -236,8 +236,7 @@ class DataAggregator : public DataReader {
 
   /// Launch a perf subprocess with given args and save output for later
   /// parsing.
-  void launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
-                         const char *ArgsString, bool Wait);
+  void launchPerfProcess(StringRef Name, PerfProcessInfo &PPI, StringRef Args);
 
   /// Delete all temporary files created to hold the output generated by spawned
   /// subprocesses during the aggregation job
@@ -374,6 +373,9 @@ class DataAggregator : public DataReader {
   /// Parse a single pair of binary full path and associated build-id
   std::optional<std::pair<StringRef, StringRef>> parseNameBuildIDPair();
 
+  /// Coordinate reading and parsing of perf.data file
+  void parsePerfData(BinaryContext &BC);
+
   /// Coordinate reading and parsing of pre-aggregated file
   ///
   /// The regular perf2bolt aggregation job is to read perf output directly.
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 94dd06e91d6b..91d62a78de39 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -202,6 +202,9 @@ class RewriteInstance {
   /// Map code sections generated by BOLT.
   void mapCodeSections(BOLTLinker::SectionMapper MapSection);
 
+  /// Map code without relocating sections.
+  void mapCodeSectionsInPlace(BOLTLinker::SectionMapper MapSection);
+
   /// Map the rest of allocatable sections.
   void mapAllocatableSections(BOLTLinker::SectionMapper MapSection);
 
@@ -297,6 +300,9 @@ class RewriteInstance {
     return FUNC(ELF64BE);                                                      \
   }
 
+  /// Update loadable segment information based on new sections.
+  void updateSegmentInfo();
+
   /// Patch ELF book-keeping info.
   void patchELFPHDRTable();
 
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index b998d7160aae..eec68ff5a5fc 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -3328,10 +3328,7 @@ void BinaryFunction::duplicateConstantIslands() {
 
         // Update instruction reference
         Operand = MCOperand::createExpr(BC.MIB->getTargetExprFor(
-            Inst,
-            MCSymbolRefExpr::create(ColdSymbol, MCSymbolRefExpr::VK_None,
-                                    *BC.Ctx),
-            *BC.Ctx, 0));
+            Inst, MCSymbolRefExpr::create(ColdSymbol, *BC.Ctx), *BC.Ctx, 0));
         ++OpNum;
       }
     }
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index e356481bbdc7..5d44e1a1a490 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -1765,27 +1765,26 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
 
   if (opts::ShowDensity) {
     double Density = 0.0;
-    // Sorted by the density in descending order.
-    llvm::stable_sort(FuncDensityList,
-                      [&](const std::pair<double, uint64_t> &A,
-                          const std::pair<double, uint64_t> &B) {
-                        if (A.first != B.first)
-                          return A.first > B.first;
-                        return A.second < B.second;
-                      });
+    llvm::sort(FuncDensityList);
 
     uint64_t AccumulatedSamples = 0;
-    uint32_t I = 0;
     assert(opts::ProfileDensityCutOffHot <= 1000000 &&
            "The cutoff value is greater than 1000000(100%)");
-    while (AccumulatedSamples <
-               TotalSampleCount *
-                   static_cast<float>(opts::ProfileDensityCutOffHot) /
-                   1000000 &&
-           I < FuncDensityList.size()) {
-      AccumulatedSamples += FuncDensityList[I].second;
-      Density = FuncDensityList[I].first;
-      I++;
+    // Subtract samples in zero-density functions (no fall-throughs) from
+    // TotalSampleCount (not used anywhere below).
+    for (const auto [CurDensity, CurSamples] : FuncDensityList) {
+      if (CurDensity != 0.0)
+        break;
+      TotalSampleCount -= CurSamples;
+    }
+    const uint64_t CutoffSampleCount =
+        1.f * TotalSampleCount * opts::ProfileDensityCutOffHot / 1000000;
+    // Process functions in decreasing density order
+    for (const auto [CurDensity, CurSamples] : llvm::reverse(FuncDensityList)) {
+      if (AccumulatedSamples >= CutoffSampleCount)
+        break;
+      AccumulatedSamples += CurSamples;
+      Density = CurDensity;
     }
     if (Density == 0.0) {
       BC.errs() << "BOLT-WARNING: the output profile is empty or the "
diff --git a/bolt/lib/Passes/Instrumentation.cpp b/bolt/lib/Passes/Instrumentation.cpp
index fbf889279f1c..c2f876f0dff9 100644
--- a/bolt/lib/Passes/Instrumentation.cpp
+++ b/bolt/lib/Passes/Instrumentation.cpp
@@ -666,8 +666,7 @@ Error Instrumentation::runOnFunctions(BinaryContext &BC) {
       auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); };
       const auto LEA = std::find_if(
           std::next(llvm::find_if(reverse(BB), IsLEA)), BB.rend(), IsLEA);
-      LEA->getOperand(4).setExpr(
-          MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *BC.Ctx));
+      LEA->getOperand(4).setExpr(MCSymbolRefExpr::create(Target, *BC.Ctx));
     } else {
       BC.errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";
     }
diff --git a/bolt/lib/Passes/PAuthGadgetScanner.cpp b/bolt/lib/Passes/PAuthGadgetScanner.cpp
index 95e831fe9c8c..f928dd49edb2 100644
--- a/bolt/lib/Passes/PAuthGadgetScanner.cpp
+++ b/bolt/lib/Passes/PAuthGadgetScanner.cpp
@@ -82,6 +82,22 @@ namespace PAuthGadgetScanner {
   dbgs() << "\n";
 }
 
+// Iterates over BinaryFunction's instructions like a range-based for loop:
+//
+// iterateOverInstrs(BF, [&](MCInstReference Inst) {
+//   // loop body
+// });
+template <typename T> static void iterateOverInstrs(BinaryFunction &BF, T Fn) {
+  if (BF.hasCFG()) {
+    for (BinaryBasicBlock &BB : BF)
+      for (int64_t I = 0, E = BB.size(); I < E; ++I)
+        Fn(MCInstInBBReference(&BB, I));
+  } else {
+    for (auto I : BF.instrs())
+      Fn(MCInstInBFReference(&BF, I.first));
+  }
+}
+
 // This class represents mapping from a set of arbitrary physical registers to
 // consecutive array indexes.
 class TrackedRegisters {
@@ -342,6 +358,29 @@ class SrcSafetyAnalysis {
     return S;
   }
 
+  /// Computes a reasonably pessimistic estimation of the register state when
+  /// the previous instruction is not known for sure. Takes the set of registers
+  /// which are trusted at function entry and removes all registers that can be
+  /// clobbered inside this function.
+  SrcState computePessimisticState(BinaryFunction &BF) {
+    BitVector ClobberedRegs(NumRegs);
+    iterateOverInstrs(BF, [&](MCInstReference Inst) {
+      BC.MIB->getClobberedRegs(Inst, ClobberedRegs);
+
+      // If this is a call instruction, no register is safe anymore, unless
+      // it is a tail call. Ignore tail calls for the purpose of estimating the
+      // worst-case scenario, assuming no instructions are executed in the
+      // caller after this point anyway.
+      if (BC.MIB->isCall(Inst) && !BC.MIB->isTailCall(Inst))
+        ClobberedRegs.set();
+    });
+
+    SrcState S = createEntryState();
+    S.SafeToDerefRegs.reset(ClobberedRegs);
+    S.TrustedRegs.reset(ClobberedRegs);
+    return S;
+  }
+
   BitVector getClobberedRegs(const MCInst &Point) const {
     BitVector Clobbered(NumRegs);
     // Assume a call can clobber all registers, including callee-saved
@@ -545,6 +584,10 @@ class DataflowSrcSafetyAnalysis
   using SrcSafetyAnalysis::BC;
   using SrcSafetyAnalysis::computeNext;
 
+  // Pessimistic initial state for basic blocks without any predecessors
+  // (not needed for most functions, thus initialized lazily).
+  SrcState PessimisticState;
+
 public:
   DataflowSrcSafetyAnalysis(BinaryFunction &BF,
                             MCPlusBuilder::AllocatorIdTy AllocId,
@@ -585,6 +628,18 @@ class DataflowSrcSafetyAnalysis
     if (BB.isEntryPoint())
       return createEntryState();
 
+    // If a basic block without any predecessors is found in an optimized code,
+    // this likely means that some CFG edges were not detected. Pessimistically
+    // assume any register that can ever be clobbered in this function to be
+    // unsafe before this basic block.
+    // Warn about this fact in FunctionAnalysis::findUnsafeUses(), as it likely
+    // means imprecise CFG information.
+    if (BB.pred_empty()) {
+      if (PessimisticState.empty())
+        PessimisticState = computePessimisticState(*BB.getParent());
+      return PessimisticState;
+    }
+
     return SrcState();
   }
 
@@ -682,19 +737,14 @@ template <typename StateTy> class CFGUnawareAnalysis {
 //
 // Then, a function can be split into a number of disjoint contiguous sequences
 // of instructions without labels in between. These sequences can be processed
-// the same way basic blocks are processed by data-flow analysis, assuming
-// pessimistically that all registers are unsafe at the start of each sequence.
+// the same way basic blocks are processed by data-flow analysis, with the same
+// pessimistic estimation of the initial state at the start of each sequence
+// (except the first instruction of the function).
 class CFGUnawareSrcSafetyAnalysis : public SrcSafetyAnalysis,
                                     public CFGUnawareAnalysis<SrcState> {
   using SrcSafetyAnalysis::BC;
   BinaryFunction &BF;
 
-  /// Creates a state with all registers marked unsafe (not to be confused
-  /// with empty state).
-  SrcState createUnsafeState() const {
-    return SrcState(NumRegs, RegsToTrackInstsFor.getNumTrackedRegisters());
-  }
-
 public:
   CFGUnawareSrcSafetyAnalysis(BinaryFunction &BF,
                               MCPlusBuilder::AllocatorIdTy AllocId,
@@ -704,6 +754,7 @@ class CFGUnawareSrcSafetyAnalysis : public SrcSafetyAnalysis,
   }
 
   void run() override {
+    const SrcState DefaultState = computePessimisticState(BF);
     SrcState S = createEntryState();
     for (auto &I : BF.instrs()) {
       MCInst &Inst = I.second;
@@ -718,7 +769,7 @@ class CFGUnawareSrcSafetyAnalysis : public SrcSafetyAnalysis,
         LLVM_DEBUG({
           traceInst(BC, "Due to label, resetting the state before", Inst);
         });
-        S = createUnsafeState();
+        S = DefaultState;
       }
 
       // Attach the state *before* this instruction executes.
@@ -1268,6 +1319,90 @@ shouldReportReturnGadget(const BinaryContext &BC, const MCInstReference &Inst,
   return make_gadget_report(RetKind, Inst, *RetReg);
 }
 
+/// While BOLT already marks some of the branch instructions as tail calls,
+/// this function tries to detect less obvious cases, assuming false positives
+/// are acceptable as long as there are not too many of them.
+///
+/// It is possible that not all the instructions classified as tail calls by
+/// this function are safe to be considered as such for the purpose of code
+/// transformations performed by BOLT. The intention of this function is to
+/// spot some of actually missed tail calls (and likely a number of unrelated
+/// indirect branch instructions) as long as this doesn't increase the amount
+/// of false positive reports unacceptably.
+static bool shouldAnalyzeTailCallInst(const BinaryContext &BC,
+                                      const BinaryFunction &BF,
+                                      const MCInstReference &Inst) {
+  // Some BC.MIB->isXYZ(Inst) methods simply delegate to MCInstrDesc::isXYZ()
+  // (such as isBranch at the time of writing this comment), some don't (such
+  // as isCall). For that reason, call MCInstrDesc's methods explicitly when
+  // it is important.
+  const MCInstrDesc &Desc =
+      BC.MII->get(static_cast<const MCInst &>(Inst).getOpcode());
+  // Tail call should be a branch (but not necessarily an indirect one).
+  if (!Desc.isBranch())
+    return false;
+
+  // Always analyze the branches already marked as tail calls by BOLT.
+  if (BC.MIB->isTailCall(Inst))
+    return true;
+
+  // Try to also check the branches marked as "UNKNOWN CONTROL FLOW" - the
+  // below is a simplified condition from BinaryContext::printInstruction.
+  bool IsUnknownControlFlow =
+      BC.MIB->isIndirectBranch(Inst) && !BC.MIB->getJumpTable(Inst);
+
+  if (BF.hasCFG() && IsUnknownControlFlow)
+    return true;
+
+  return false;
+}
+
+static std::optional<PartialReport<MCPhysReg>>
+shouldReportUnsafeTailCall(const BinaryContext &BC, const BinaryFunction &BF,
+                           const MCInstReference &Inst, const SrcState &S) {
+  static const GadgetKind UntrustedLRKind(
+      "untrusted link register found before tail call");
+
+  if (!shouldAnalyzeTailCallInst(BC, BF, Inst))
+    return std::nullopt;
+
+  // Not only the set of registers returned by getTrustedLiveInRegs() can be
+  // seen as a reasonable target-independent _approximation_ of "the LR", these
+  // are *exactly* those registers used by SrcSafetyAnalysis to initialize the
+  // set of trusted registers on function entry.
+  // Thus, this function basically checks that the precondition expected to be
+  // imposed by a function call instruction (which is hardcoded into the target-
+  // specific getTrustedLiveInRegs() function) is also respected on tail calls.
+  SmallVector<MCPhysReg> RegsToCheck = BC.MIB->getTrustedLiveInRegs();
+  LLVM_DEBUG({
+    traceInst(BC, "Found tail call inst", Inst);
+    traceRegMask(BC, "Trusted regs", S.TrustedRegs);
+  });
+
+  // In musl on AArch64, the _start function sets LR to zero and calls the next
+  // stage initialization function at the end, something along these lines:
+  //
+  //   _start:
+  //     mov     x30, #0
+  //     ; ... other initialization ...
+  //     b       _start_c ; performs "exit" system call at some point
+  //
+  // As this would produce a false positive for every executable linked with
+  // such libc, ignore tail calls performed by ELF entry function.
+  if (BC.StartFunctionAddress &&
+      *BC.StartFunctionAddress == Inst.getFunction()->getAddress()) {
+    LLVM_DEBUG({ dbgs() << "  Skipping tail call in ELF entry function.\n"; });
+    return std::nullopt;
+  }
+
+  // Returns at most one report per instruction - this is probably OK...
+  for (auto Reg : RegsToCheck)
+    if (!S.TrustedRegs[Reg])
+      return make_gadget_report(UntrustedLRKind, Inst, Reg);
+
+  return std::nullopt;
+}
+
 static std::optional<PartialReport<MCPhysReg>>
 shouldReportCallGadget(const BinaryContext &BC, const MCInstReference &Inst,
                        const SrcState &S) {
@@ -1344,17 +1479,6 @@ shouldReportAuthOracle(const BinaryContext &BC, const MCInstReference &Inst,
   return make_gadget_report(AuthOracleKind, Inst, *AuthReg);
 }
 
-template <typename T> static void iterateOverInstrs(BinaryFunction &BF, T Fn) {
-  if (BF.hasCFG()) {
-    for (BinaryBasicBlock &BB : BF)
-      for (int64_t I = 0, E = BB.size(); I < E; ++I)
-        Fn(MCInstInBBReference(&BB, I));
-  } else {
-    for (auto I : BF.instrs())
-      Fn(MCInstInBFReference(&BF, I.first));
-  }
-}
-
 static SmallVector<MCPhysReg>
 collectRegsToTrack(ArrayRef<PartialReport<MCPhysReg>> Reports) {
   SmallSet<MCPhysReg, 4> RegsToTrack;
@@ -1375,17 +1499,60 @@ void FunctionAnalysisContext::findUnsafeUses(
     BF.dump();
   });
 
+  bool UnreachableBBReported = false;
+  if (BF.hasCFG()) {
+    // Warn on basic blocks being unreachable according to BOLT (at most once
+    // per BinaryFunction), as this likely means the CFG reconstructed by BOLT
+    // is imprecise. A basic block can be
+    // * reachable from an entry basic block - a hopefully correct non-empty
+    //   state is propagated to that basic block sooner or later. All basic
+    //   blocks are expected to belong to this category under normal conditions.
+    // * reachable from a "directly unreachable" BB (a basic block that has no
+    //   direct predecessors and this is not because it is an entry BB) - *some*
+    //   non-empty state is propagated to this basic block sooner or later, as
+    //   the initial state of directly unreachable basic blocks is
+    //   pessimistically initialized to "all registers are unsafe"
+    //   - a warning can be printed for the "directly unreachable" basic block
+    // * neither reachable from an entry nor from a "directly unreachable" BB
+    //   (such as if this BB is in an isolated loop of basic blocks) - the final
+    //   state is computed to be empty for this basic block
+    //   - a warning can be printed for this basic block
+    for (BinaryBasicBlock &BB : BF) {
+      MCInst *FirstInst = BB.getFirstNonPseudoInstr();
+      // Skip empty basic block early for simplicity.
+      if (!FirstInst)
+        continue;
+
+      bool IsDirectlyUnreachable = BB.pred_empty() && !BB.isEntryPoint();
+      bool HasNoStateComputed = Analysis->getStateBefore(*FirstInst).empty();
+      if (!IsDirectlyUnreachable && !HasNoStateComputed)
+        continue;
+
+      // Arbitrarily attach the report to the first instruction of BB.
+      // This is printed as "[message] in function [name], basic block ...,
+      // at address ..." when the issue is reported to the user.
+      Reports.push_back(make_generic_report(
+          MCInstReference::get(FirstInst, BF),
+          "Warning: possibly imprecise CFG, the analysis quality may be "
+          "degraded in this function. According to BOLT, unreachable code is "
+          "found" /* in function [name]... */));
+      UnreachableBBReported = true;
+      break; // One warning per function.
+    }
+  }
+  // FIXME: Warn the user about imprecise analysis when the function has no CFG
+  //        information at all.
+
   iterateOverInstrs(BF, [&](MCInstReference Inst) {
     if (BC.MIB->isCFI(Inst))
       return;
 
     const SrcState &S = Analysis->getStateBefore(Inst);
-
-    // If non-empty state was never propagated from the entry basic block
-    // to Inst, assume it to be unreachable and report a warning.
     if (S.empty()) {
-      Reports.push_back(
-          make_generic_report(Inst, "Warning: unreachable instruction found"));
+      LLVM_DEBUG(
+          { traceInst(BC, "Instruction has no state, skipping", Inst); });
+      assert(UnreachableBBReported && "Should be reported at least once");
+      (void)UnreachableBBReported;
       return;
     }
 
@@ -1395,6 +1562,9 @@ void FunctionAnalysisContext::findUnsafeUses(
     if (PacRetGadgetsOnly)
       return;
 
+    if (auto Report = shouldReportUnsafeTailCall(BC, BF, Inst, S))
+      Reports.push_back(*Report);
+
     if (auto Report = shouldReportCallGadget(BC, Inst, S))
       Reports.push_back(*Report);
     if (auto Report = shouldReportSigningOracle(BC, Inst, S))
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 5c8af3710720..c60591c02b6c 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -197,34 +197,27 @@ void DataAggregator::start() {
 
   if (opts::BasicAggregation) {
     launchPerfProcess("events without LBR", MainEventsPPI,
-                      "script -F pid,event,ip",
-                      /*Wait = */ false);
+                      "script -F pid,event,ip");
   } else if (!opts::ITraceAggregation.empty()) {
     // Disable parsing memory profile from trace data, unless requested by user.
     if (!opts::ParseMemProfile.getNumOccurrences())
       opts::ParseMemProfile = false;
-
-    std::string ItracePerfScriptArgs = llvm::formatv(
-        "script -F pid,brstack --itrace={0}", opts::ITraceAggregation);
     launchPerfProcess("branch events with itrace", MainEventsPPI,
-                      ItracePerfScriptArgs.c_str(),
-                      /*Wait = */ false);
+                      "script -F pid,brstack --itrace=" +
+                          opts::ITraceAggregation);
   } else {
-    launchPerfProcess("branch events", MainEventsPPI, "script -F pid,brstack",
-                      /*Wait = */ false);
+    launchPerfProcess("branch events", MainEventsPPI, "script -F pid,brstack");
   }
 
   if (opts::ParseMemProfile)
-    launchPerfProcess("mem events", MemEventsPPI, "script -F pid,event,addr,ip",
-                      /*Wait = */ false);
+    launchPerfProcess("mem events", MemEventsPPI,
+                      "script -F pid,event,addr,ip");
 
   launchPerfProcess("process events", MMapEventsPPI,
-                    "script --show-mmap-events --no-itrace",
-                    /*Wait = */ false);
+                    "script --show-mmap-events --no-itrace");
 
   launchPerfProcess("task events", TaskEventsPPI,
-                    "script --show-task-events --no-itrace",
-                    /*Wait = */ false);
+                    "script --show-task-events --no-itrace");
 }
 
 void DataAggregator::abort() {
@@ -246,13 +239,13 @@ void DataAggregator::abort() {
 }
 
 void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
-                                       const char *ArgsString, bool Wait) {
+                                       StringRef Args) {
   SmallVector<StringRef, 4> Argv;
 
   outs() << "PERF2BOLT: spawning perf job to read " << Name << '\n';
   Argv.push_back(PerfPath.data());
 
-  StringRef(ArgsString).split(Argv, ' ');
+  Args.split(Argv, ' ');
   Argv.push_back("-f");
   Argv.push_back("-i");
   Argv.push_back(Filename.c_str());
@@ -286,64 +279,45 @@ void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,
            << "\n";
   });
 
-  if (Wait)
-    PPI.PI.ReturnCode = sys::ExecuteAndWait(PerfPath.data(), Argv,
-                                            /*envp*/ std::nullopt, Redirects);
-  else
-    PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,
-                                Redirects);
+  PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,
+                              Redirects);
 }
 
 void DataAggregator::processFileBuildID(StringRef FileBuildID) {
+  auto WarningCallback = [](int ReturnCode, StringRef ErrBuf) {
+    errs() << "PERF-ERROR: return code " << ReturnCode << "\n" << ErrBuf;
+  };
+
   PerfProcessInfo BuildIDProcessInfo;
-  launchPerfProcess("buildid list",
-                    BuildIDProcessInfo,
-                    "buildid-list",
-                    /*Wait = */true);
-
-  if (BuildIDProcessInfo.PI.ReturnCode != 0) {
-    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-        MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StderrPath.data());
-    StringRef ErrBuf = (*MB)->getBuffer();
-
-    errs() << "PERF-ERROR: return code " << BuildIDProcessInfo.PI.ReturnCode
-           << '\n';
-    errs() << ErrBuf;
+  launchPerfProcess("buildid list", BuildIDProcessInfo, "buildid-list");
+  if (prepareToParse("buildid", BuildIDProcessInfo, WarningCallback))
     return;
-  }
 
-  ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
-      MemoryBuffer::getFileOrSTDIN(BuildIDProcessInfo.StdoutPath.data());
-  if (std::error_code EC = MB.getError()) {
-    errs() << "Cannot open " << BuildIDProcessInfo.StdoutPath.data() << ": "
-           << EC.message() << "\n";
+  std::optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
+  if (FileName && *FileName == sys::path::filename(BC->getFilename())) {
+    outs() << "PERF2BOLT: matched build-id and file name\n";
     return;
   }
 
-  FileBuf = std::move(*MB);
-  ParsingBuf = FileBuf->getBuffer();
-
-  std::optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);
-  if (!FileName) {
-    if (hasAllBuildIDs()) {
-      errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
-                "This indicates the input binary supplied for data aggregation "
-                "is not the same recorded by perf when collecting profiling "
-                "data, or there were no samples recorded for the binary. "
-                "Use -ignore-build-id option to override.\n";
-      if (!opts::IgnoreBuildID)
-        abort();
-    } else {
-      errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
-                "data was recorded without it\n";
-      return;
-    }
-  } else if (*FileName != llvm::sys::path::filename(BC->getFilename())) {
+  if (FileName) {
     errs() << "PERF2BOLT-WARNING: build-id matched a different file name\n";
     BuildIDBinaryName = std::string(*FileName);
-  } else {
-    outs() << "PERF2BOLT: matched build-id and file name\n";
+    return;
+  }
+
+  if (!hasAllBuildIDs()) {
+    errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "
+              "data was recorded without it\n";
+    return;
   }
+
+  errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "
+            "This indicates the input binary supplied for data aggregation "
+            "is not the same recorded by perf when collecting profiling "
+            "data, or there were no samples recorded for the binary. "
+            "Use -ignore-build-id option to override.\n";
+  if (!opts::IgnoreBuildID)
+    abort();
 }
 
 bool DataAggregator::checkPerfDataMagic(StringRef FileName) {
@@ -432,7 +406,8 @@ int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
   std::string Error;
   outs() << "PERF2BOLT: waiting for perf " << Name
          << " collection to finish...\n";
-  sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error);
+  std::optional<sys::ProcessStatistics> PS;
+  sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error, &PS);
 
   if (!Error.empty()) {
     errs() << "PERF-ERROR: " << PerfPath << ": " << Error << "\n";
@@ -440,6 +415,15 @@ int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
     exit(1);
   }
 
+  LLVM_DEBUG({
+    const float UserSec = 1.f * PS->UserTime.count() / 1e6;
+    const float TotalSec = 1.f * PS->TotalTime.count() / 1e6;
+    const float PeakGiB = 1.f * PS->PeakMemory / (1 << 20);
+    dbgs() << formatv("Finished in {0:f2}s user time, {1:f2}s total time, "
+                      "{2:f2} GiB peak RSS\n",
+                      UserSec, TotalSec, PeakGiB);
+  });
+
   if (PI.ReturnCode != 0) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorMB =
         MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data());
@@ -466,9 +450,7 @@ int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,
   return PI.ReturnCode;
 }
 
-Error DataAggregator::preprocessProfile(BinaryContext &BC) {
-  this->BC = &BC;
-
+void DataAggregator::parsePerfData(BinaryContext &BC) {
   auto ErrorCallback = [](int ReturnCode, StringRef ErrBuf) {
     errs() << "PERF-ERROR: return code " << ReturnCode << "\n" << ErrBuf;
     exit(1);
@@ -481,11 +463,6 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
       ErrorCallback(ReturnCode, ErrBuf);
   };
 
-  if (opts::ReadPreAggregated) {
-    parsePreAggregated();
-    goto heatmap;
-  }
-
   if (std::optional<StringRef> FileBuildID = BC.getFileBuildID()) {
     outs() << "BOLT-INFO: binary build-id is:     " << *FileBuildID << "\n";
     processFileBuildID(*FileBuildID);
@@ -534,22 +511,28 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
              << '\n';
 
   deleteTempFiles();
+}
 
-heatmap:
-  // Sort parsed traces for faster processing.
-  llvm::sort(Traces, llvm::less_first());
+Error DataAggregator::preprocessProfile(BinaryContext &BC) {
+  this->BC = &BC;
 
-  if (!opts::HeatmapMode)
-    return Error::success();
+  if (opts::ReadPreAggregated) {
+    parsePreAggregated();
+  } else {
+    parsePerfData(BC);
+  }
 
-  if (std::error_code EC = printLBRHeatMap())
-    return errorCodeToError(EC);
+  // Sort parsed traces for faster processing.
+  llvm::sort(Traces, llvm::less_first());
 
-  if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Optional)
-    return Error::success();
+  if (opts::HeatmapMode) {
+    if (std::error_code EC = printLBRHeatMap())
+      return errorCodeToError(EC);
+    if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
+      exit(0);
+  }
 
-  assert(opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive);
-  exit(0);
+  return Error::success();
 }
 
 Error DataAggregator::readProfile(BinaryContext &BC) {
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 93bd93b6cb98..96045a916232 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -547,9 +547,14 @@ Error RewriteInstance::discoverStorage() {
       NextAvailableOffset = std::max(NextAvailableOffset,
                                      Phdr.p_offset + Phdr.p_filesz);
 
-      BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
-          Phdr.p_vaddr,  Phdr.p_memsz, Phdr.p_offset,
-          Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
+      BC->SegmentMapInfo[Phdr.p_vaddr] =
+          SegmentInfo{Phdr.p_vaddr,
+                      Phdr.p_memsz,
+                      Phdr.p_offset,
+                      Phdr.p_filesz,
+                      Phdr.p_align,
+                      (Phdr.p_flags & ELF::PF_X) != 0,
+                      (Phdr.p_flags & ELF::PF_W) != 0};
       if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
           Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
         BC->IsLinuxKernel = true;
@@ -626,6 +631,9 @@ Error RewriteInstance::discoverStorage() {
     NextAvailableAddress += BC->PageAlign;
   }
 
+  NewTextSegmentAddress = NextAvailableAddress;
+  NewTextSegmentOffset = NextAvailableOffset;
+
   if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
     // This is where the black magic happens. Creating PHDR table in a segment
     // other than that containing ELF header is tricky. Some loaders and/or
@@ -652,6 +660,8 @@ Error RewriteInstance::discoverStorage() {
 
     PHDRTableAddress = NextAvailableAddress;
     PHDRTableOffset = NextAvailableOffset;
+    NewTextSegmentAddress = NextAvailableAddress;
+    NewTextSegmentOffset = NextAvailableOffset;
 
     // Reserve space for 3 extra pheaders.
     unsigned Phnum = Obj.getHeader().e_phnum;
@@ -664,14 +674,12 @@ Error RewriteInstance::discoverStorage() {
 
     NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
     NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
-  }
 
-  // Align at cache line.
-  NextAvailableAddress = alignTo(NextAvailableAddress, 64);
-  NextAvailableOffset = alignTo(NextAvailableOffset, 64);
+    // Align at cache line.
+    NextAvailableAddress = alignTo(NextAvailableAddress, 64);
+    NextAvailableOffset = alignTo(NextAvailableOffset, 64);
+  }
 
-  NewTextSegmentAddress = NextAvailableAddress;
-  NewTextSegmentOffset = NextAvailableOffset;
   BC->LayoutStartAddress = NextAvailableAddress;
 
   // Tools such as objcopy can strip section contents but leave header
@@ -3853,111 +3861,138 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
 }
 
 void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
-  if (BC->HasRelocations) {
-    // Map sections for functions with pre-assigned addresses.
-    for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
-      const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
-      if (!OutputAddress)
-        continue;
-
-      ErrorOr<BinarySection &> FunctionSection =
-          InjectedFunction->getCodeSection();
-      assert(FunctionSection && "function should have section");
-      FunctionSection->setOutputAddress(OutputAddress);
-      MapSection(*FunctionSection, OutputAddress);
-      InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
-      InjectedFunction->setImageSize(FunctionSection->getOutputSize());
-    }
+  if (!BC->HasRelocations) {
+    mapCodeSectionsInPlace(MapSection);
+    return;
+  }
 
-    // Populate the list of sections to be allocated.
-    std::vector<BinarySection *> CodeSections = getCodeSections();
+  // Map sections for functions with pre-assigned addresses.
+  for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
+    const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
+    if (!OutputAddress)
+      continue;
 
-    // Remove sections that were pre-allocated (patch sections).
-    llvm::erase_if(CodeSections, [](BinarySection *Section) {
-      return Section->getOutputAddress();
-    });
-    LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
-      for (const BinarySection *Section : CodeSections)
-        dbgs() << Section->getName() << '\n';
-    );
+    ErrorOr<BinarySection &> FunctionSection =
+        InjectedFunction->getCodeSection();
+    assert(FunctionSection && "function should have section");
+    FunctionSection->setOutputAddress(OutputAddress);
+    MapSection(*FunctionSection, OutputAddress);
+    InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
+    InjectedFunction->setImageSize(FunctionSection->getOutputSize());
+  }
 
-    uint64_t PaddingSize = 0; // size of padding required at the end
+  // Populate the list of sections to be allocated.
+  std::vector<BinarySection *> CodeSections = getCodeSections();
 
-    // Allocate sections starting at a given Address.
-    auto allocateAt = [&](uint64_t Address) {
-      const char *LastNonColdSectionName = BC->HasWarmSection
-                                               ? BC->getWarmCodeSectionName()
-                                               : BC->getMainCodeSectionName();
-      for (BinarySection *Section : CodeSections) {
-        Address = alignTo(Address, Section->getAlignment());
-        Section->setOutputAddress(Address);
-        Address += Section->getOutputSize();
-
-        // Hugify: Additional huge page from right side due to
-        // weird ASLR mapping addresses (4KB aligned)
-        if (opts::Hugify && !BC->HasFixedLoadAddress &&
-            Section->getName() == LastNonColdSectionName)
-          Address = alignTo(Address, Section->getAlignment());
-      }
+  // Remove sections that were pre-allocated (patch sections).
+  llvm::erase_if(CodeSections, [](BinarySection *Section) {
+    return Section->getOutputAddress();
+  });
+  LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
+             for (const BinarySection *Section : CodeSections) dbgs()
+             << Section->getName() << '\n';);
 
-      // Make sure we allocate enough space for huge pages.
-      ErrorOr<BinarySection &> TextSection =
-          BC->getUniqueSectionByName(LastNonColdSectionName);
-      if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
-        uint64_t HotTextEnd =
-            TextSection->getOutputAddress() + TextSection->getOutputSize();
-        HotTextEnd = alignTo(HotTextEnd, BC->PageAlign);
-        if (HotTextEnd > Address) {
-          PaddingSize = HotTextEnd - Address;
-          Address = HotTextEnd;
-        }
-      }
-      return Address;
-    };
+  uint64_t PaddingSize = 0; // size of padding required at the end
 
-    // Check if we can fit code in the original .text
-    bool AllocationDone = false;
-    if (opts::UseOldText) {
-      const uint64_t CodeSize =
-          allocateAt(BC->OldTextSectionAddress) - BC->OldTextSectionAddress;
+  // Allocate sections starting at a given Address.
+  auto allocateAt = [&](uint64_t Address) {
+    const char *LastNonColdSectionName = BC->HasWarmSection
+                                             ? BC->getWarmCodeSectionName()
+                                             : BC->getMainCodeSectionName();
+    for (BinarySection *Section : CodeSections) {
+      Address = alignTo(Address, Section->getAlignment());
+      Section->setOutputAddress(Address);
+      Address += Section->getOutputSize();
+
+      // Hugify: Additional huge page from right side due to
+      // weird ASLR mapping addresses (4KB aligned)
+      if (opts::Hugify && !BC->HasFixedLoadAddress &&
+          Section->getName() == LastNonColdSectionName)
+        Address = alignTo(Address, Section->getAlignment());
+    }
 
-      if (CodeSize <= BC->OldTextSectionSize) {
-        BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
-                   << Twine::utohexstr(opts::AlignText) << " alignment\n";
-        AllocationDone = true;
-      } else {
-        BC->errs()
-            << "BOLT-WARNING: original .text too small to fit the new code"
-            << " using 0x" << Twine::utohexstr(opts::AlignText)
-            << " alignment. " << CodeSize << " bytes needed, have "
-            << BC->OldTextSectionSize << " bytes available.\n";
-        opts::UseOldText = false;
+    // Make sure we allocate enough space for huge pages.
+    ErrorOr<BinarySection &> TextSection =
+        BC->getUniqueSectionByName(LastNonColdSectionName);
+    if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
+      uint64_t HotTextEnd =
+          TextSection->getOutputAddress() + TextSection->getOutputSize();
+      HotTextEnd = alignTo(HotTextEnd, BC->PageAlign);
+      if (HotTextEnd > Address) {
+        PaddingSize = HotTextEnd - Address;
+        Address = HotTextEnd;
       }
     }
+    return Address;
+  };
 
-    if (!AllocationDone)
-      NextAvailableAddress = allocateAt(NextAvailableAddress);
+  // Try to allocate sections before the \p Address and return an address for
+  // the allocation of the first section, or 0 if [0, Address) range is not
+  // big enough to fit all sections.
+  auto allocateBefore = [&](uint64_t Address) -> uint64_t {
+    for (BinarySection *Section : llvm::reverse(CodeSections)) {
+      if (Section->getOutputSize() > Address)
+        return 0;
+      Address -= Section->getOutputSize();
+      Address = alignDown(Address, Section->getAlignment());
+      Section->setOutputAddress(Address);
+    }
+    return Address;
+  };
 
-    // Do the mapping for ORC layer based on the allocation.
-    for (BinarySection *Section : CodeSections) {
-      LLVM_DEBUG(
-          dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
-                 << Twine::utohexstr(Section->getAllocAddress()) << " to 0x"
-                 << Twine::utohexstr(Section->getOutputAddress()) << '\n');
-      MapSection(*Section, Section->getOutputAddress());
-      Section->setOutputFileOffset(
-          getFileOffsetForAddress(Section->getOutputAddress()));
+  // Check if we can fit code in the original .text
+  bool AllocationDone = false;
+  if (opts::UseOldText) {
+    uint64_t StartAddress;
+    uint64_t EndAddress;
+    if (opts::HotFunctionsAtEnd) {
+      EndAddress = BC->OldTextSectionAddress + BC->OldTextSectionSize;
+      StartAddress = allocateBefore(EndAddress);
+    } else {
+      StartAddress = BC->OldTextSectionAddress;
+      EndAddress = allocateAt(BC->OldTextSectionAddress);
+    }
+
+    const uint64_t CodeSize = EndAddress - StartAddress;
+    if (CodeSize <= BC->OldTextSectionSize) {
+      BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
+                 << Twine::utohexstr(opts::AlignText) << " alignment";
+      if (StartAddress != BC->OldTextSectionAddress)
+        BC->outs() << " at 0x" << Twine::utohexstr(StartAddress);
+      BC->outs() << '\n';
+      AllocationDone = true;
+    } else {
+      BC->errs() << "BOLT-WARNING: original .text too small to fit the new code"
+                 << " using 0x" << Twine::utohexstr(opts::AlignText)
+                 << " alignment. " << CodeSize << " bytes needed, have "
+                 << BC->OldTextSectionSize << " bytes available.\n";
+      opts::UseOldText = false;
     }
+  }
 
-    // Check if we need to insert a padding section for hot text.
-    if (PaddingSize && !opts::UseOldText)
-      BC->outs() << "BOLT-INFO: padding code to 0x"
-                 << Twine::utohexstr(NextAvailableAddress)
-                 << " to accommodate hot text\n";
+  if (!AllocationDone)
+    NextAvailableAddress = allocateAt(NextAvailableAddress);
 
-    return;
+  // Do the mapping for ORC layer based on the allocation.
+  for (BinarySection *Section : CodeSections) {
+    LLVM_DEBUG(dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
+                      << Twine::utohexstr(Section->getAllocAddress())
+                      << " to 0x"
+                      << Twine::utohexstr(Section->getOutputAddress()) << '\n');
+    MapSection(*Section, Section->getOutputAddress());
+    Section->setOutputFileOffset(
+        getFileOffsetForAddress(Section->getOutputAddress()));
   }
 
+  // Check if we need to insert a padding section for hot text.
+  if (PaddingSize && !opts::UseOldText)
+    BC->outs() << "BOLT-INFO: padding code to 0x"
+               << Twine::utohexstr(NextAvailableAddress)
+               << " to accommodate hot text\n";
+}
+
+void RewriteInstance::mapCodeSectionsInPlace(
+    BOLTLinker::SectionMapper MapSection) {
   // Processing in non-relocation mode.
   uint64_t NewTextSectionStartAddress = NextAvailableAddress;
 
@@ -4133,13 +4168,8 @@ void RewriteInstance::mapAllocatableSections(
     }
 
     if (SType == ST_READONLY) {
-      if (PHDRTableAddress) {
-        // Segment size includes the size of the PHDR area.
-        NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
-      } else if (NewTextSegmentAddress) {
-        // Existing PHDR table would be updated.
+      if (NewTextSegmentAddress)
         NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
-      }
     } else if (SType == ST_READWRITE) {
       NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
       // Restore NextAvailableAddress if no new writable sections
@@ -4157,6 +4187,74 @@ void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
     Function->updateOutputValues(Linker);
 }
 
+void RewriteInstance::updateSegmentInfo() {
+  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
+  // last segments size based on the NextAvailableAddress variable.
+  if (!NewWritableSegmentSize) {
+    if (NewTextSegmentAddress)
+      NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
+  } else {
+    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
+  }
+
+  if (NewTextSegmentSize) {
+    SegmentInfo TextSegment = {NewTextSegmentAddress,
+                               NewTextSegmentSize,
+                               NewTextSegmentOffset,
+                               NewTextSegmentSize,
+                               BC->PageAlign,
+                               true,
+                               false};
+    if (!opts::Instrument) {
+      BC->NewSegments.push_back(TextSegment);
+    } else {
+      ErrorOr<BinarySection &> Sec =
+          BC->getUniqueSectionByName(".bolt.instr.counters");
+      assert(Sec && "expected one and only one `.bolt.instr.counters` section");
+      const uint64_t Addr = Sec->getOutputAddress();
+      const uint64_t Offset = Sec->getOutputFileOffset();
+      const uint64_t Size = Sec->getOutputSize();
+      assert(Addr > TextSegment.Address &&
+             Addr + Size < TextSegment.Address + TextSegment.Size &&
+             "`.bolt.instr.counters` section is expected to be included in the "
+             "new text segment");
+
+      // Set correct size for the previous header since we are breaking the
+      // new text segment into three segments.
+      uint64_t Delta = Addr - TextSegment.Address;
+      TextSegment.Size = Delta;
+      TextSegment.FileSize = Delta;
+      BC->NewSegments.push_back(TextSegment);
+
+      // Create RW segment that includes the `.bolt.instr.counters` section.
+      SegmentInfo RWSegment = {Addr,  Size, Offset, Size, BC->RegularPageSize,
+                               false, true};
+      BC->NewSegments.push_back(RWSegment);
+
+      // Create RX segment that includes all RX sections from runtime library.
+      const uint64_t AddrRX = alignTo(Addr + Size, BC->RegularPageSize);
+      const uint64_t OffsetRX = alignTo(Offset + Size, BC->RegularPageSize);
+      const uint64_t SizeRX =
+          NewTextSegmentSize - (AddrRX - TextSegment.Address);
+      SegmentInfo RXSegment = {
+          AddrRX, SizeRX, OffsetRX, SizeRX, BC->RegularPageSize, true, false};
+      BC->NewSegments.push_back(RXSegment);
+    }
+  }
+
+  if (NewWritableSegmentSize) {
+    SegmentInfo DataSegmentInfo = {
+        NewWritableSegmentAddress,
+        NewWritableSegmentSize,
+        getFileOffsetForAddress(NewWritableSegmentAddress),
+        NewWritableSegmentSize,
+        BC->RegularPageSize,
+        false,
+        true};
+    BC->NewSegments.push_back(DataSegmentInfo);
+  }
+}
+
 void RewriteInstance::patchELFPHDRTable() {
   auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);
   const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
@@ -4183,110 +4281,36 @@ void RewriteInstance::patchELFPHDRTable() {
   if (opts::Instrument)
     Phnum += 2;
 
-  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
-  // last segments size based on the NextAvailableAddress variable.
-  if (!NewWritableSegmentSize) {
-    if (PHDRTableAddress)
-      NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
-    else if (NewTextSegmentAddress)
-      NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
-  } else {
-    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
+  if (BC->NewSegments.empty()) {
+    BC->outs() << "BOLT-INFO: not adding new segments\n";
+    return;
   }
 
   const uint64_t SavedPos = OS.tell();
   OS.seek(PHDRTableOffset);
 
-  auto createNewPhdrs = [&]() {
-    SmallVector<ELF64LEPhdrTy, 3> NewPhdrs;
-    ELF64LEPhdrTy NewPhdr;
-    NewPhdr.p_type = ELF::PT_LOAD;
-    if (PHDRTableAddress) {
-      NewPhdr.p_offset = PHDRTableOffset;
-      NewPhdr.p_vaddr = PHDRTableAddress;
-      NewPhdr.p_paddr = PHDRTableAddress;
-    } else {
-      NewPhdr.p_offset = NewTextSegmentOffset;
-      NewPhdr.p_vaddr = NewTextSegmentAddress;
-      NewPhdr.p_paddr = NewTextSegmentAddress;
-    }
-    NewPhdr.p_filesz = NewTextSegmentSize;
-    NewPhdr.p_memsz = NewTextSegmentSize;
-    NewPhdr.p_flags = ELF::PF_X | ELF::PF_R;
-    NewPhdr.p_align = BC->PageAlign;
-
-    if (!opts::Instrument) {
-      NewPhdrs.push_back(NewPhdr);
-    } else {
-      ErrorOr<BinarySection &> Sec =
-          BC->getUniqueSectionByName(".bolt.instr.counters");
-      assert(Sec && "expected one and only one `.bolt.instr.counters` section");
-      const uint64_t Addr = Sec->getOutputAddress();
-      const uint64_t Offset = Sec->getOutputFileOffset();
-      const uint64_t Size = Sec->getOutputSize();
-      assert(Addr > NewPhdr.p_vaddr &&
-             Addr + Size < NewPhdr.p_vaddr + NewPhdr.p_memsz &&
-             "`.bolt.instr.counters` section is expected to be included in the "
-             "new text sgement");
-
-      // Set correct size for the previous header since we are breaking the
-      // new text segment into three segments.
-      uint64_t Delta = Addr - NewPhdr.p_vaddr;
-      NewPhdr.p_filesz = Delta;
-      NewPhdr.p_memsz = Delta;
-      NewPhdrs.push_back(NewPhdr);
-
-      // Create a program header for a RW segment that includes the
-      // `.bolt.instr.counters` section only.
-      ELF64LEPhdrTy NewPhdrRWSegment;
-      NewPhdrRWSegment.p_type = ELF::PT_LOAD;
-      NewPhdrRWSegment.p_offset = Offset;
-      NewPhdrRWSegment.p_vaddr = Addr;
-      NewPhdrRWSegment.p_paddr = Addr;
-      NewPhdrRWSegment.p_filesz = Size;
-      NewPhdrRWSegment.p_memsz = Size;
-      NewPhdrRWSegment.p_flags = ELF::PF_R | ELF::PF_W;
-      NewPhdrRWSegment.p_align = BC->RegularPageSize;
-      NewPhdrs.push_back(NewPhdrRWSegment);
-
-      // Create a program header for a RX segment that includes all the RX
-      // sections from runtime library.
-      ELF64LEPhdrTy NewPhdrRXSegment;
-      NewPhdrRXSegment.p_type = ELF::PT_LOAD;
-      const uint64_t AddrRX = alignTo(Addr + Size, BC->RegularPageSize);
-      const uint64_t OffsetRX = alignTo(Offset + Size, BC->RegularPageSize);
-      const uint64_t SizeRX = NewTextSegmentSize - (AddrRX - NewPhdr.p_paddr);
-      NewPhdrRXSegment.p_offset = OffsetRX;
-      NewPhdrRXSegment.p_vaddr = AddrRX;
-      NewPhdrRXSegment.p_paddr = AddrRX;
-      NewPhdrRXSegment.p_filesz = SizeRX;
-      NewPhdrRXSegment.p_memsz = SizeRX;
-      NewPhdrRXSegment.p_flags = ELF::PF_X | ELF::PF_R;
-      NewPhdrRXSegment.p_align = BC->RegularPageSize;
-      NewPhdrs.push_back(NewPhdrRXSegment);
-    }
-
-    return NewPhdrs;
+  auto createPhdr = [](const SegmentInfo &SI) {
+    ELF64LEPhdrTy Phdr;
+    Phdr.p_type = ELF::PT_LOAD;
+    Phdr.p_offset = SI.FileOffset;
+    Phdr.p_vaddr = SI.Address;
+    Phdr.p_paddr = SI.Address;
+    Phdr.p_filesz = SI.FileSize;
+    Phdr.p_memsz = SI.Size;
+    Phdr.p_flags = ELF::PF_R;
+    if (SI.IsExecutable)
+      Phdr.p_flags |= ELF::PF_X;
+    if (SI.IsWritable)
+      Phdr.p_flags |= ELF::PF_W;
+    Phdr.p_align = SI.Alignment;
+
+    return Phdr;
   };
 
   auto writeNewSegmentPhdrs = [&]() {
-    if (PHDRTableAddress || NewTextSegmentSize) {
-      SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
-      OS.write(reinterpret_cast<const char *>(NewPhdrs.data()),
-               sizeof(ELF64LE::Phdr) * NewPhdrs.size());
-    }
-
-    if (NewWritableSegmentSize) {
-      ELF64LEPhdrTy NewPhdr;
-      NewPhdr.p_type = ELF::PT_LOAD;
-      NewPhdr.p_offset = getFileOffsetForAddress(NewWritableSegmentAddress);
-      NewPhdr.p_vaddr = NewWritableSegmentAddress;
-      NewPhdr.p_paddr = NewWritableSegmentAddress;
-      NewPhdr.p_filesz = NewWritableSegmentSize;
-      NewPhdr.p_memsz = NewWritableSegmentSize;
-      NewPhdr.p_align = BC->RegularPageSize;
-      NewPhdr.p_flags = ELF::PF_R | ELF::PF_W;
-      OS.write(reinterpret_cast<const char *>(&NewPhdr), sizeof(NewPhdr));
+    for (const SegmentInfo &SI : BC->NewSegments) {
+      ELF64LEPhdrTy Phdr = createPhdr(SI);
+      OS.write(reinterpret_cast<const char *>(&Phdr), sizeof(Phdr));
     }
   };
 
@@ -4322,11 +4346,9 @@ void RewriteInstance::patchELFPHDRTable() {
     case ELF::PT_GNU_STACK:
       if (opts::UseGnuStack) {
         // Overwrite the header with the new segment header.
-        assert(!opts::Instrument);
-        SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
-        assert(NewPhdrs.size() == 1 &&
-               "expect exactly one program header was created");
-        NewPhdr = NewPhdrs[0];
+        assert(BC->NewSegments.size() == 1 &&
+               "Expected exactly one new segment");
+        NewPhdr = createPhdr(BC->NewSegments.front());
         ModdedGnuStack = true;
       }
       break;
@@ -5951,8 +5973,10 @@ void RewriteInstance::rewriteFile() {
     addBATSection();
 
   // Patch program header table.
-  if (!BC->IsLinuxKernel)
+  if (!BC->IsLinuxKernel) {
+    updateSegmentInfo();
     patchELFPHDRTable();
+  }
 
   // Finalize memory image of section string table.
   finalizeSectionStringTable();
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index e6e0aeba3457..973261765f95 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -22,7 +22,6 @@
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegister.h"
@@ -1081,7 +1080,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
 
     if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
         RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
-      return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx);
+      return MCSpecifierExpr::create(Expr, AArch64::S_ABS, Ctx);
     } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
                RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
                RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
@@ -1089,7 +1088,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
       // Never emit a GOT reloc, we handled this in
       // RewriteInstance::readRelocations().
-      return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
+      return MCSpecifierExpr::create(Expr, AArch64::S_ABS_PAGE, Ctx);
     } else {
       switch (RelType) {
       case ELF::R_AARCH64_ADD_ABS_LO12_NC:
@@ -1103,18 +1102,18 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       case ELF::R_AARCH64_TLSDESC_LD64_LO12:
       case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
       case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
-        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64::S_LO12, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G3:
-        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G3, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G3, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G2:
       case ELF::R_AARCH64_MOVW_UABS_G2_NC:
-        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G2_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G2_NC, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G1:
       case ELF::R_AARCH64_MOVW_UABS_G1_NC:
-        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G1_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G1_NC, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G0:
       case ELF::R_AARCH64_MOVW_UABS_G0_NC:
-        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G0_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G0_NC, Ctx);
       default:
         break;
       }
@@ -1206,8 +1205,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
       OI = Inst.begin() + 2;
     }
 
-    *OI = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    *OI = MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   /// Matches indirect branch patterns in AArch64 related to a jump table (JT),
@@ -1633,8 +1631,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           .addImm(0));
     Code.emplace_back(MCInstBuilder(AArch64::Bcc)
                           .addImm(AArch64CC::EQ)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Code;
   }
 
@@ -1656,8 +1653,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           .addImm(0));
     Code.emplace_back(MCInstBuilder(AArch64::Bcc)
                           .addImm(AArch64CC::NE)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Code;
   }
 
@@ -1957,8 +1953,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);
     Inst.clear();
     Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-        Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        *Ctx, 0)));
+        Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
     if (IsTailCall)
       convertJmpToTailCall(Inst);
   }
@@ -2028,7 +2023,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(AArch64::MOVZXi);
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createExpr(
-        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G3, *Ctx)));
+        MCSpecifierExpr::create(Target, AArch64::S_ABS_G3, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x30));
     Seq.emplace_back(Inst);
 
@@ -2037,7 +2032,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createExpr(
-        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G2_NC, *Ctx)));
+        MCSpecifierExpr::create(Target, AArch64::S_ABS_G2_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x20));
     Seq.emplace_back(Inst);
 
@@ -2046,7 +2041,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createExpr(
-        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G1_NC, *Ctx)));
+        MCSpecifierExpr::create(Target, AArch64::S_ABS_G1_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x10));
     Seq.emplace_back(Inst);
 
@@ -2055,7 +2050,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createExpr(
-        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G0_NC, *Ctx)));
+        MCSpecifierExpr::create(Target, AArch64::S_ABS_G0_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0));
     Seq.emplace_back(Inst);
 
@@ -2228,9 +2223,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
                           MCContext *Ctx) const override {
     Inst.setOpcode(AArch64::B);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-        Inst, MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx),
-        *Ctx, 0)));
+    Inst.addOperand(MCOperand::createExpr(
+        getTargetExprFor(Inst, MCSymbolRefExpr::create(TBB, *Ctx), *Ctx, 0)));
   }
 
   bool shouldRecordCodeRelocation(uint32_t RelType) const override {
@@ -2562,7 +2556,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     else if (Fixup.getKind() ==
              MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))
       RelType = ELF::R_AARCH64_JUMP26;
-    else if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
+    else if (Fixup.isPCRel()) {
       switch (FKI.TargetSize) {
       default:
         return std::nullopt;
diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
index c7d664ab09d4..10b4913b6ab7 100644
--- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
+++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp
@@ -171,8 +171,8 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     (void)Result;
     assert(Result && "unimplemented branch");
 
-    Inst.getOperand(SymOpIndex) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(SymOpIndex) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   IndirectBranchType analyzeIndirectBranch(
@@ -233,8 +233,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(RISCV::JAL);
     Inst.clear();
     Inst.addOperand(MCOperand::createReg(RISCV::X0));
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx)));
   }
 
   StringRef getTrapFillValue() const override {
@@ -246,8 +245,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Inst.setOpcode(Opcode);
     Inst.clear();
     Inst.addOperand(MCOperand::createExpr(MCSpecifierExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        ELF::R_RISCV_CALL_PLT, *Ctx)));
+        MCSymbolRefExpr::create(Target, *Ctx), ELF::R_RISCV_CALL_PLT, *Ctx)));
   }
 
   void createCall(MCInst &Inst, const MCSymbol *Target,
@@ -563,8 +561,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     Insts.emplace_back(MCInstBuilder(RISCV::BEQ)
                            .addReg(RegNo)
                            .addReg(RegTmp)
-                           .addExpr(MCSymbolRefExpr::create(
-                               Target, MCSymbolRefExpr::VK_None, *Ctx)));
+                           .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     return Insts;
   }
 
@@ -663,14 +660,12 @@ class RISCVMCPlusBuilder : public MCPlusBuilder {
     if (IsTailCall) {
       Inst.addOperand(MCOperand::createReg(RISCV::X0));
       Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-          *Ctx, 0)));
+          Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
       convertJmpToTailCall(Inst);
     } else {
       Inst.addOperand(MCOperand::createReg(RISCV::X1));
       Inst.addOperand(MCOperand::createExpr(getTargetExprFor(
-          Inst, MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-          *Ctx, 0)));
+          Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));
     }
   }
 
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index b909d7fb6bf2..09d2322a29ca 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -18,7 +18,6 @@
 #include "bolt/Core/MCPlusBuilder.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -72,9 +71,9 @@ static InstructionListType createIncMemory(const MCSymbol *Target,
   Insts.back().addOperand(MCOperand::createImm(1));               // ScaleAmt
   Insts.back().addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
 
-  Insts.back().addOperand(MCOperand::createExpr(
-      MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None,
-                              *Ctx))); // Displacement
+  Insts.back().addOperand(
+      MCOperand::createExpr(MCSymbolRefExpr::create(Target,
+                                                    *Ctx))); // Displacement
   Insts.back().addOperand(
       MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   return Insts;
@@ -1625,9 +1624,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.insert(Inst.begin(),
                 MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
     Inst.insert(Inst.begin(),
-                MCOperand::createExpr(                  // Displacement
-                    MCSymbolRefExpr::create(TargetLocation,
-                                            MCSymbolRefExpr::VK_None, *Ctx)));
+                MCOperand::createExpr( // Displacement
+                    MCSymbolRefExpr::create(TargetLocation, *Ctx)));
     Inst.insert(Inst.begin(),
                 MCOperand::createReg(X86::NoRegister)); // IndexReg
     Inst.insert(Inst.begin(),
@@ -2420,8 +2418,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                           .addReg(RegNo)
                           .addImm(Imm));
     Code.emplace_back(MCInstBuilder(X86::JCC_1)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx))
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx))
                           .addImm(X86::COND_E));
     return Code;
   }
@@ -2432,8 +2429,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     InstructionListType Code;
     Code.emplace_back(MCInstBuilder(X86::CMP64ri8).addReg(RegNo).addImm(Imm));
     Code.emplace_back(MCInstBuilder(X86::JCC_1)
-                          .addExpr(MCSymbolRefExpr::create(
-                              Target, MCSymbolRefExpr::VK_None, *Ctx))
+                          .addExpr(MCSymbolRefExpr::create(Target, *Ctx))
                           .addImm(X86::COND_NE));
     return Code;
   }
@@ -2447,7 +2443,7 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     const uint64_t RelOffset = Fixup.getOffset();
 
     uint32_t RelType;
-    if (FKI.Flags & MCFixupKindInfo::FKF_IsPCRel) {
+    if (Fixup.isPCRel()) {
       switch (FKI.TargetSize) {
       default:
         return std::nullopt;
@@ -2738,24 +2734,23 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.clear();
     Inst.setOpcode(X86::JMP_1);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx)));
   }
 
   void createLongUncondBranch(MCInst &Inst, const MCSymbol *Target,
                               MCContext *Ctx) const override {
     Inst.setOpcode(X86::JMP_4);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
   }
 
   void createCall(MCInst &Inst, const MCSymbol *Target,
                   MCContext *Ctx) override {
     Inst.setOpcode(X86::CALL64pcrel32);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
   }
 
   void createTailCall(MCInst &Inst, const MCSymbol *Target,
@@ -2779,8 +2774,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                         MCContext *Ctx) const override {
     Inst.setOpcode(X86::JCC_1);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     Inst.addOperand(MCOperand::createImm(CC));
   }
 
@@ -2788,8 +2783,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                             MCContext *Ctx) const override {
     Inst.setOpcode(X86::JCC_4);
     Inst.clear();
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     Inst.addOperand(MCOperand::createImm(CC));
   }
 
@@ -2798,8 +2793,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     unsigned InvCC = getInvertedCondCode(getCondCode(Inst));
     assert(InvCC != X86::COND_INVALID && "invalid branch instruction");
     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(InvCC);
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   bool replaceBranchCondition(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx,
@@ -2807,8 +2802,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     if (CC == X86::COND_INVALID)
       return false;
     Inst.getOperand(Info->get(Inst.getOpcode()).NumOperands - 1).setImm(CC);
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
     return true;
   }
 
@@ -2846,8 +2841,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                            MCContext *Ctx) const override {
     assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&
            "Invalid instruction");
-    Inst.getOperand(0) = MCOperand::createExpr(
-        MCSymbolRefExpr::create(TBB, MCSymbolRefExpr::VK_None, *Ctx));
+    Inst.getOperand(0) =
+        MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));
   }
 
   MCPhysReg getX86R11() const override { return X86::R11; }
@@ -2894,8 +2889,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
                         bool IsTailCall) override {
     Inst.clear();
     Inst.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     if (IsTailCall)
       setTailCall(Inst);
   }
@@ -2905,8 +2900,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Seq.clear();
     MCInst Inst;
     Inst.setOpcode(X86::JMP_1);
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx)));
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Target, *Ctx)));
     if (IsTailCall)
       setTailCall(Inst);
     Seq.emplace_back(Inst);
@@ -3332,8 +3327,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         Target.addOperand(MCOperand::createReg(FuncAddrReg));
         if (Targets[i].first) {
           // Is this OK?
-          Target.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-              Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+          Target.addOperand(MCOperand::createExpr(
+              MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
         } else {
           const uint64_t Addr = Targets[i].second;
           // Immediate address is out of sign extended 32 bit range.
@@ -3409,8 +3404,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         Je.clear();
         Je.setOpcode(X86::JCC_1);
         if (Targets[i].first)
-          Je.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-              Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+          Je.addOperand(MCOperand::createExpr(
+              MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
         else
           Je.addOperand(MCOperand::createImm(Targets[i].second));
 
@@ -3422,8 +3417,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
         // Jump to next compare if target addresses don't match.
         Jne.clear();
         Jne.setOpcode(X86::JCC_1);
-        Jne.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-            NextTarget, MCSymbolRefExpr::VK_None, *Ctx)));
+        Jne.addOperand(
+            MCOperand::createExpr(MCSymbolRefExpr::create(NextTarget, *Ctx)));
         Jne.addOperand(MCOperand::createImm(X86::COND_NE));
 
         // Call specific target directly.
@@ -3442,8 +3437,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
           CallOrJmp.setOpcode(IsTailCall ? X86::JMP_4 : X86::CALL64pcrel32);
 
           if (Targets[i].first)
-            CallOrJmp.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-                Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+            CallOrJmp.addOperand(MCOperand::createExpr(
+                MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
           else
             CallOrJmp.addOperand(MCOperand::createImm(Targets[i].second));
         }
@@ -3545,8 +3540,8 @@ class X86MCPlusBuilder : public MCPlusBuilder {
 
       // Jump to target if indices match
       JEInst.setOpcode(X86::JCC_1);
-      JEInst.addOperand(MCOperand::createExpr(MCSymbolRefExpr::create(
-          Targets[i].first, MCSymbolRefExpr::VK_None, *Ctx)));
+      JEInst.addOperand(MCOperand::createExpr(
+          MCSymbolRefExpr::create(Targets[i].first, *Ctx)));
       JEInst.addOperand(MCOperand::createImm(X86::COND_E));
     }
 
@@ -3571,9 +3566,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
-                                *Ctx)));                    // Displacement
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Src,
+                                                      *Ctx))); // Displacement
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   }
 
@@ -3585,9 +3580,9 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createReg(X86::RIP));        // BaseReg
     Inst.addOperand(MCOperand::createImm(1));               // ScaleAmt
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // IndexReg
-    Inst.addOperand(MCOperand::createExpr(
-        MCSymbolRefExpr::create(Src, MCSymbolRefExpr::VK_None,
-                                *Ctx)));                    // Displacement
+    Inst.addOperand(
+        MCOperand::createExpr(MCSymbolRefExpr::create(Src,
+                                                      *Ctx))); // Displacement
     Inst.addOperand(MCOperand::createReg(X86::NoRegister)); // AddrSegmentReg
   }
 };
diff --git a/bolt/test/X86/zero-density.s b/bolt/test/X86/zero-density.s
new file mode 100644
index 000000000000..7843804ffed8
--- /dev/null
+++ b/bolt/test/X86/zero-density.s
@@ -0,0 +1,32 @@
+## Check that trampoline functions are excluded from density computation.
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: link_fdata %s %t %t.preagg PREAGG
+# RUN: llvm-strip -NLjmp %t
+# RUN: perf2bolt %t -p %t.preagg --pa -o %t.fdata | FileCheck %s
+# CHECK: Functions with density >= {{.*}} account for 99.00% total sample counts.
+# CHECK-NOT: the output profile is empty or the --profile-density-cutoff-hot option is set too low.
+
+  .text
+  .globl trampoline
+trampoline:
+  mov main,%rax
+  jmpq *%rax
+.size trampoline,.-trampoline
+# PREAGG: f #trampoline# #trampoline# 2
+
+	.globl main
+main:
+	.cfi_startproc
+	vmovaps %zmm31,%zmm3
+
+	add    $0x4,%r9
+	add    $0x40,%r10
+	dec    %r14
+Ljmp:
+	jne    main
+# PREAGG: T #Ljmp# #main# #Ljmp# 10
+	ret
+	.cfi_endproc
+.size main,.-main
diff --git a/bolt/test/binary-analysis/AArch64/gs-pacret-autiasp.s b/bolt/test/binary-analysis/AArch64/gs-pacret-autiasp.s
index 284f0bea607a..8e991fade2c8 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pacret-autiasp.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pacret-autiasp.s
@@ -18,6 +18,10 @@ f1:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   ret
@@ -40,6 +44,10 @@ f_intermediate_overwrite1:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   autiasp
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
@@ -63,6 +71,10 @@ f_intermediate_overwrite2:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: mov     x30, x0
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autiasp
@@ -102,6 +114,10 @@ f_intermediate_overwrite3:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: mov     w30, w0
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autiasp
@@ -126,6 +142,10 @@ f_nonx30_ret:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: mov     x16, x30
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   mov     x16, x30
@@ -215,7 +235,7 @@ f_callclobbered_calleesaved:
         .globl  f_unreachable_instruction
         .type   f_unreachable_instruction,@function
 f_unreachable_instruction:
-// CHECK-LABEL: GS-PAUTH: Warning: unreachable instruction found in function f_unreachable_instruction, basic block {{[0-9a-zA-Z.]+}}, at address
+// CHECK-LABEL: GS-PAUTH: Warning: possibly imprecise CFG, the analysis quality may be degraded in this function. According to BOLT, unreachable code is found in function f_unreachable_instruction, basic block {{[0-9a-zA-Z.]+}}, at address
 // CHECK-NEXT:    The instruction is     {{[0-9a-f]+}}:       add     x0, x1, x2
 // CHECK-NOT:   instructions that write to the affected registers after any authentication are:
         b       1f
@@ -224,20 +244,33 @@ f_unreachable_instruction:
         ret
         .size f_unreachable_instruction, .-f_unreachable_instruction
 
-// Expected false positive: without CFG, the state is reset to all-unsafe
-// after an unconditional branch.
+// Without CFG, the state is reset at labels, assuming every register that can
+// be clobbered in the function was actually clobbered.
 
-        .globl  state_is_reset_after_indirect_branch_nocfg
-        .type   state_is_reset_after_indirect_branch_nocfg,@function
-state_is_reset_after_indirect_branch_nocfg:
-// CHECK-LABEL: GS-PAUTH: non-protected ret found in function state_is_reset_after_indirect_branch_nocfg, at address
-// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:         ret
+        .globl  lr_untouched_nocfg
+        .type   lr_untouched_nocfg,@function
+lr_untouched_nocfg:
+// CHECK-NOT: lr_untouched_nocfg
+        adr     x2, 1f
+        br      x2
+1:
+        ret
+        .size lr_untouched_nocfg, .-lr_untouched_nocfg
+
+        .globl  lr_clobbered_nocfg
+        .type   lr_clobbered_nocfg,@function
+lr_clobbered_nocfg:
+// CHECK-LABEL: GS-PAUTH: non-protected ret found in function lr_clobbered_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      ret
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
         adr     x2, 1f
         br      x2
 1:
+        b       2f
+        bl      g   // never executed, but affects the expected worst-case scenario
+2:
         ret
-        .size state_is_reset_after_indirect_branch_nocfg, .-state_is_reset_after_indirect_branch_nocfg
+        .size lr_clobbered_nocfg, .-lr_clobbered_nocfg
 
 /// Now do a basic sanity check on every different Authentication instruction:
 
@@ -312,6 +345,10 @@ f_autia1716:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autia1716
@@ -334,6 +371,10 @@ f_autib1716:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autib1716
@@ -356,6 +397,10 @@ f_autiax12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autia   x12, sp
@@ -378,6 +423,10 @@ f_autibx12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autib   x12, sp
@@ -429,6 +478,10 @@ f_autdax12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autda   x12, sp
@@ -451,6 +504,10 @@ f_autdbx12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autdb   x12, sp
@@ -502,6 +559,10 @@ f_autizax12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autiza  x12
@@ -524,6 +585,10 @@ f_autizbx12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autizb  x12
@@ -575,6 +640,10 @@ f_autdzax12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autdza  x12
@@ -597,6 +666,10 @@ f_autdzbx12:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autdzb  x12
@@ -855,6 +928,10 @@ f_autia171615:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autia171615
@@ -877,6 +954,10 @@ f_autib171615:
 // CHECK-NEXT:    The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:    1. {{[0-9a-f]+}}: ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT: {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT: {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT: {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT: {{[0-9a-f]+}}:   bl      g
 // CHECK-NEXT: {{[0-9a-f]+}}:   add     x0, x0, #0x3
 // CHECK-NEXT: {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
 // CHECK-NEXT: {{[0-9a-f]+}}:   autib171615
@@ -884,3 +965,9 @@ f_autib171615:
         ret
         .size f_autib171615, .-f_autib171615
 
+        .globl  g
+        .type   g,@function
+g:
+        nop
+        ret
+        .size g, .-g
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
index 717bf40df3d0..c314bc7cfe5a 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
@@ -491,10 +491,6 @@ good_address_arith_multi_bb:
         ret
         .size good_address_arith_multi_bb, .-good_address_arith_multi_bb
 
-// FIXME: Most *_nocfg test cases contain paciasp+autiasp instructions even if
-//        LR is not spilled - this is a workaround for RET instructions being
-//        reported as non-protected, because LR state is reset at every label.
-
         .globl  good_ret_nocfg
         .type   good_ret_nocfg,@function
 good_ret_nocfg:
@@ -541,14 +537,12 @@ good_branch_nocfg:
         .type   good_load_other_reg_nocfg,@function
 good_load_other_reg_nocfg:
 // CHECK-NOT: good_load_other_reg_nocfg
-        paciasp
         adr     x2, 1f
         br      x2
 1:
         autia   x0, x1
         ldr     x2, [x0]
 
-        autiasp
         ret
         .size good_load_other_reg_nocfg, .-good_load_other_reg_nocfg
 
@@ -556,14 +550,12 @@ good_load_other_reg_nocfg:
         .type   good_load_same_reg_nocfg,@function
 good_load_same_reg_nocfg:
 // CHECK-NOT: good_load_same_reg_nocfg
-        paciasp
         adr     x2, 1f
         br      x2
 1:
         autia   x0, x1
         ldr     x0, [x0]
 
-        autiasp
         ret
         .size good_load_same_reg_nocfg, .-good_load_same_reg_nocfg
 
@@ -575,13 +567,11 @@ bad_unchecked_nocfg:
 // CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unchecked_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
 // CHECK-NEXT:  The 0 instructions that leak the affected registers are:
-        paciasp
         adr     x2, 1f
         br      x2
 1:
         autia   x0, x1
 
-        autiasp
         ret
         .size bad_unchecked_nocfg, .-bad_unchecked_nocfg
 
@@ -615,7 +605,6 @@ bad_unknown_usage_read_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
 // CHECK-NEXT:  The 1 instructions that leak the affected registers are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     x3, x0, x1
-        paciasp
         adr     x2, 1f
         br      x2
 1:
@@ -623,7 +612,6 @@ bad_unknown_usage_read_nocfg:
         mul     x3, x0, x1
         ldr     x2, [x0]
 
-        autiasp
         ret
         .size bad_unknown_usage_read_nocfg, .-bad_unknown_usage_read_nocfg
 
@@ -634,7 +622,6 @@ bad_unknown_usage_subreg_read_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
 // CHECK-NEXT:  The 1 instructions that leak the affected registers are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     w3, w0, w1
-        paciasp
         adr     x2, 1f
         br      x2
 1:
@@ -642,7 +629,6 @@ bad_unknown_usage_subreg_read_nocfg:
         mul     w3, w0, w1
         ldr     x2, [x0]
 
-        autiasp
         ret
         .size bad_unknown_usage_subreg_read_nocfg, .-bad_unknown_usage_subreg_read_nocfg
 
@@ -653,7 +639,6 @@ bad_unknown_usage_update_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
 // CHECK-NEXT:  The 1 instructions that leak the affected registers are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      movk    x0, #0x2a, lsl #16
-        paciasp
         adr     x2, 1f
         br      x2
 1:
@@ -661,7 +646,6 @@ bad_unknown_usage_update_nocfg:
         movk    x0, #42, lsl #16  // does not overwrite x0 completely
         ldr     x2, [x0]
 
-        autiasp
         ret
         .size bad_unknown_usage_update_nocfg, .-bad_unknown_usage_update_nocfg
 
@@ -669,14 +653,12 @@ bad_unknown_usage_update_nocfg:
         .type   good_overwrite_with_constant_nocfg,@function
 good_overwrite_with_constant_nocfg:
 // CHECK-NOT: good_overwrite_with_constant_nocfg
-        paciasp
         adr     x2, 1f
         br      x2
 1:
         autia   x0, x1
         mov     x0, #42
 
-        autiasp
         ret
         .size good_overwrite_with_constant_nocfg, .-good_overwrite_with_constant_nocfg
 
@@ -684,7 +666,6 @@ good_overwrite_with_constant_nocfg:
         .type   good_address_arith_nocfg,@function
 good_address_arith_nocfg:
 // CHECK-NOT: good_address_arith_nocfg
-        paciasp
         adr     x2, 1f
         br      x2
 1:
@@ -698,7 +679,6 @@ good_address_arith_nocfg:
         mov     x1, #0
         mov     x2, #0
 
-        autiasp
         ret
         .size good_address_arith_nocfg, .-good_address_arith_nocfg
 
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-calls.s b/bolt/test/binary-analysis/AArch64/gs-pauth-calls.s
index c79c5926a05c..fb0bc7cff237 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-calls.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-calls.s
@@ -1428,6 +1428,90 @@ printed_instrs_nocfg:
         br      x0
         .size   printed_instrs_nocfg, .-printed_instrs_nocfg
 
+// Test handling of unreachable basic blocks.
+//
+// Basic blocks without any predecessors were observed in real-world optimized
+// code. At least sometimes they were actually reachable via jump table, which
+// was not detected, but the function was processed as if its CFG was
+// reconstructed successfully.
+//
+// As a more predictable model example, let's use really unreachable code
+// for testing.
+
+        .globl  bad_unreachable_call
+        .type   bad_unreachable_call,@function
+bad_unreachable_call:
+// CHECK-LABEL: GS-PAUTH: Warning: possibly imprecise CFG, the analysis quality may be degraded in this function. According to BOLT, unreachable code is found in function bad_unreachable_call, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      blr     x0
+// CHECK-NOT:   instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: non-protected call found in function bad_unreachable_call, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      blr     x0
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        b       1f
+        // unreachable basic block:
+        blr     x0
+
+1:      // reachable basic block:
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size bad_unreachable_call, .-bad_unreachable_call
+
+        .globl  good_unreachable_call
+        .type   good_unreachable_call,@function
+good_unreachable_call:
+// CHECK-NOT: non-protected call{{.*}}good_unreachable_call
+// CHECK-LABEL: GS-PAUTH: Warning: possibly imprecise CFG, the analysis quality may be degraded in this function. According to BOLT, unreachable code is found in function good_unreachable_call, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NOT: instructions that write to the affected registers after any authentication are:
+// CHECK-NOT: non-protected call{{.*}}good_unreachable_call
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        b       1f
+        // unreachable basic block:
+        autia   x0, x1
+        blr     x0      // <-- this call is definitely protected provided at least
+                        //     basic block boundaries are detected correctly
+
+1:      // reachable basic block:
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size good_unreachable_call, .-good_unreachable_call
+
+        .globl  unreachable_loop_of_bbs
+        .type   unreachable_loop_of_bbs,@function
+unreachable_loop_of_bbs:
+// CHECK-NOT: unreachable basic blocks{{.*}}unreachable_loop_of_bbs
+// CHECK-NOT: non-protected call{{.*}}unreachable_loop_of_bbs
+// CHECK-LABEL: GS-PAUTH: Warning: possibly imprecise CFG, the analysis quality may be degraded in this function. According to BOLT, unreachable code is found in function unreachable_loop_of_bbs, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      blr     x0
+// CHECK-NOT: unreachable basic blocks{{.*}}unreachable_loop_of_bbs
+// CHECK-NOT: non-protected call{{.*}}unreachable_loop_of_bbs
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+        b       .Lreachable_epilogue_bb
+
+.Lfirst_unreachable_bb:
+        blr     x0      // <-- this call is not analyzed
+        b       .Lsecond_unreachable_bb
+.Lsecond_unreachable_bb:
+        blr     x1      // <-- this call is not analyzed
+        b       .Lfirst_unreachable_bb
+
+.Lreachable_epilogue_bb:
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size unreachable_loop_of_bbs, .-unreachable_loop_of_bbs
+
         .globl  main
         .type   main,@function
 main:
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
index fbb96a63d41e..b1cec7f92ad0 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
@@ -199,8 +199,8 @@ nocfg:
 // CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(  br      x0, src-state<SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI , TrustedRegs: LR W0 W30 X0 W0_HI W30_HI , Insts: >)
 // CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI , TrustedRegs: LR W0 W30 X0 W0_HI W30_HI , Insts: >)
 // CHECK-NEXT:   Due to label, resetting the state before:     00000000:       ret # Offset: 8
-// CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(  ret     x30, src-state<SafeToDerefRegs: , TrustedRegs: , Insts: >)
-// CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: , TrustedRegs: , Insts: >)
+// CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(  ret     x30, src-state<SafeToDerefRegs: LR W30 W30_HI , TrustedRegs: LR W30 W30_HI , Insts: >)
+// CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: LR W30 W30_HI , TrustedRegs: LR W30 W30_HI , Insts: >)
 // CHECK-NEXT: After src register safety analysis:
 // CHECK-NEXT: Binary Function "nocfg"  {
 // CHECK-NEXT:   Number      : 3
@@ -223,33 +223,7 @@ nocfg:
 // PAUTH-NEXT:     SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI{{[ \t]*$}}
 // CHECK-NEXT:   Found RET inst:     00000000:         ret # Offset: 8 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: >
 // CHECK-NEXT:     RetReg: LR
-// CHECK-NEXT:     SafeToDerefRegs:{{[ \t]*$}}
-// CHECK-EMPTY:
-// CHECK-NEXT: Running detailed src register safety analysis...
-// CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(   adr     x0, __ENTRY_nocfg@0x[[ENTRY_ADDR]], src-state<SafeToDerefRegs: LR W30 W30_HI , TrustedRegs: LR W30 W30_HI , Insts: [0]()>)
-// CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI , TrustedRegs: LR W0 W30 X0 W0_HI W30_HI , Insts: [0]()>)
-// CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(  br      x0, src-state<SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI , TrustedRegs: LR W0 W30 X0 W0_HI W30_HI , Insts: [0]()>)
-// CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI , TrustedRegs: LR W0 W30 X0 W0_HI W30_HI , Insts: [0]()>)
-// CHECK-NEXT:   Due to label, resetting the state before:     00000000:       ret # Offset: 8
-// CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(  ret     x30, src-state<SafeToDerefRegs: , TrustedRegs: , Insts: [0]()>)
-// CHECK-NEXT:     .. result: (src-state<SafeToDerefRegs: , TrustedRegs: , Insts: [0]()>)
-// CHECK-NEXT: After detailed src register safety analysis:
-// CHECK-NEXT: Binary Function "nocfg"  {
-// CHECK-NEXT:   Number      : 3
-// ...
-// CHECK:        Secondary Entry Points : __ENTRY_nocfg@0x[[ENTRY_ADDR]]
-// CHECK-NEXT: }
-// CHECK-NEXT: .{{[A-Za-z0-9]+}}:
-// CHECK-NEXT:     00000000:   adr     x0, __ENTRY_nocfg@0x[[ENTRY_ADDR]] # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: [0]()>
-// CHECK-NEXT:     00000004:   br      x0 # UNKNOWN CONTROL FLOW # Offset: 4 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: [0]()>
-// CHECK-NEXT: __ENTRY_nocfg@0x[[ENTRY_ADDR]] (Entry Point):
-// CHECK-NEXT: .{{[A-Za-z0-9]+}}:
-// CHECK-NEXT:     00000008:   ret # Offset: 8 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: [0]()>
-// CHECK-NEXT: DWARF CFI Instructions:
-// CHECK-NEXT:     <empty>
-// CHECK-NEXT: End of Function "nocfg"
-// CHECK-EMPTY:
-// CHECK-NEXT:   Attaching clobbering info to:     00000000:   ret # Offset: 8 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: [0]()>
+// CHECK-NEXT:     SafeToDerefRegs: LR W30 W30_HI{{[ \t]*$}}
 
         .globl  auth_oracle
         .type   auth_oracle,@function
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s b/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
index 334a4108d8ab..3a4d383ec5bc 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
@@ -505,21 +505,16 @@ bad_one_auted_one_checked_multi_bb:
 // * untrusted: not even s-t-d - from arg and from memory
 // * untrusted: subreg clobbered - between address materialization and use, between auth and check, between check and use
 // * untrusted: first checked then auted, auted then auted, checked then checked
-//
-// Note that it is important to sign and authenticate LR, as it is not kept
-// safe-to-dereference across unconditional branches.
 
         .globl  good_sign_addr_mat_nocfg
         .type   good_sign_addr_mat_nocfg,@function
 good_sign_addr_mat_nocfg:
 // CHECK-NOT: good_sign_addr_mat_nocfg
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         adr     x0, sym
         pacda   x0, x1
-        autiasp
         ret
         .size good_sign_addr_mat_nocfg, .-good_sign_addr_mat_nocfg
 
@@ -527,14 +522,12 @@ good_sign_addr_mat_nocfg:
         .type   good_sign_auted_checked_ldr_nocfg,@function
 good_sign_auted_checked_ldr_nocfg:
 // CHECK-NOT: good_sign_auted_checked_ldr_nocfg
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         autda   x0, x2
         ldr     x2, [x0]
         pacda   x0, x1
-        autiasp
         ret
         .size good_sign_auted_checked_ldr_nocfg, .-good_sign_auted_checked_ldr_nocfg
 
@@ -544,13 +537,11 @@ bad_sign_authed_unchecked_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_sign_authed_unchecked_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         autda   x0, x2
         pacda   x0, x1
-        autiasp
         ret
         .size bad_sign_authed_unchecked_nocfg, .-bad_sign_authed_unchecked_nocfg
 
@@ -560,13 +551,11 @@ bad_sign_checked_not_auted_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_sign_checked_not_auted_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         ldr     x2, [x0]
         pacda   x0, x1
-        autiasp
         ret
         .size bad_sign_checked_not_auted_nocfg, .-bad_sign_checked_not_auted_nocfg
 
@@ -576,12 +565,10 @@ bad_sign_plain_arg_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_sign_plain_arg_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         pacda   x0, x1
-        autiasp
         ret
         .size bad_sign_plain_arg_nocfg, .-bad_sign_plain_arg_nocfg
 
@@ -592,13 +579,11 @@ bad_sign_plain_mem_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldr     x0, [x1]
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         ldr     x0, [x1]
         pacda   x0, x1
-        autiasp
         ret
         .size bad_sign_plain_mem_nocfg, .-bad_sign_plain_mem_nocfg
 
@@ -609,14 +594,12 @@ bad_clobber_between_addr_mat_and_use_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mov     w0, w4
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         adr     x0, sym
         mov     w0, w4
         pacda   x0, x1
-        autiasp
         ret
         .size bad_clobber_between_addr_mat_and_use_nocfg, .-bad_clobber_between_addr_mat_and_use_nocfg
 
@@ -627,7 +610,6 @@ bad_clobber_between_auted_and_checked_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mov     w0, w4
-        paciasp
         adr     x3, 1f
         br      x3
 1:
@@ -635,7 +617,6 @@ bad_clobber_between_auted_and_checked_nocfg:
         mov     w0, w4
         ldr     x2, [x0]
         pacda   x0, x1
-        autiasp
         ret
         .size bad_clobber_between_auted_and_checked_nocfg, .-bad_clobber_between_auted_and_checked_nocfg
 
@@ -646,7 +627,6 @@ bad_clobber_between_checked_and_used_nocfg:
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
 // CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mov     w0, w4
-        paciasp
         adr     x3, 1f
         br      x3
 1:
@@ -654,7 +634,6 @@ bad_clobber_between_checked_and_used_nocfg:
         ldr     x2, [x0]
         mov     w0, w4
         pacda   x0, x1
-        autiasp
         ret
         .size bad_clobber_between_checked_and_used_nocfg, .-bad_clobber_between_checked_and_used_nocfg
 
@@ -664,14 +643,12 @@ bad_transition_check_then_auth_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_transition_check_then_auth_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         ldr     x2, [x0]
         autda   x0, x2
         pacda   x0, x1
-        autiasp
         ret
         .size bad_transition_check_then_auth_nocfg, .-bad_transition_check_then_auth_nocfg
 
@@ -681,14 +658,12 @@ bad_transition_auth_then_auth_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_transition_auth_then_auth_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         autda   x0, x2
         autda   x0, x2
         pacda   x0, x1
-        autiasp
         ret
         .size bad_transition_auth_then_auth_nocfg, .-bad_transition_auth_then_auth_nocfg
 
@@ -698,14 +673,12 @@ bad_transition_check_then_check_nocfg:
 // CHECK-LABEL: GS-PAUTH: signing oracle found in function bad_transition_check_then_check_nocfg, at address
 // CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:     pacda   x0, x1
 // CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
-        paciasp
         adr     x3, 1f
         br      x3
 1:
         ldr     x2, [x0]
         ldr     x2, [x0]
         pacda   x0, x1
-        autiasp
         ret
         .size bad_transition_check_then_check_nocfg, .-bad_transition_check_then_check_nocfg
 
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-tail-calls.s b/bolt/test/binary-analysis/AArch64/gs-pauth-tail-calls.s
new file mode 100644
index 000000000000..2d3c2f1a632c
--- /dev/null
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-tail-calls.s
@@ -0,0 +1,597 @@
+// RUN: %clang %cflags -Wl,--entry=_custom_start -march=armv8.3-a %s -o %t.exe
+// RUN: llvm-bolt-binary-analysis --scanners=pacret %t.exe 2>&1 | FileCheck -check-prefix=PACRET %s
+// RUN: llvm-bolt-binary-analysis --scanners=pauth  %t.exe 2>&1 | FileCheck %s
+
+// PACRET-NOT: untrusted link register found before tail call
+
+        .text
+
+        .globl  callee
+        .type   callee,@function
+callee:
+        ret
+        .size callee, .-callee
+
+        .globl  good_direct_tailcall_no_clobber
+        .type   good_direct_tailcall_no_clobber,@function
+good_direct_tailcall_no_clobber:
+// CHECK-NOT: good_direct_tailcall_no_clobber
+        b       callee
+        .size good_direct_tailcall_no_clobber, .-good_direct_tailcall_no_clobber
+
+        .globl  good_plt_tailcall_no_clobber
+        .type   good_plt_tailcall_no_clobber,@function
+good_plt_tailcall_no_clobber:
+// CHECK-NOT: good_plt_tailcall_no_clobber
+        b       callee_ext
+        .size good_plt_tailcall_no_clobber, .-good_plt_tailcall_no_clobber
+
+        .globl  good_indirect_tailcall_no_clobber
+        .type   good_indirect_tailcall_no_clobber,@function
+good_indirect_tailcall_no_clobber:
+// CHECK-NOT: good_indirect_tailcall_no_clobber
+        autia   x0, x1
+        br      x0
+        .size good_indirect_tailcall_no_clobber, .-good_indirect_tailcall_no_clobber
+
+        .globl  bad_direct_tailcall_not_auted
+        .type   bad_direct_tailcall_not_auted,@function
+bad_direct_tailcall_not_auted:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_not_auted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   b       callee # TAILCALL
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        b       callee
+        .size bad_direct_tailcall_not_auted, .-bad_direct_tailcall_not_auted
+
+        .globl  bad_plt_tailcall_not_auted
+        .type   bad_plt_tailcall_not_auted,@function
+bad_plt_tailcall_not_auted:
+// FIXME: Calls via PLT are disassembled incorrectly. Nevertheless, they are
+//        still detected as tail calls.
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_plt_tailcall_not_auted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_not_auted # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   b       bad_indirect_tailcall_not_auted # TAILCALL
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        b       callee_ext
+        .size bad_plt_tailcall_not_auted, .-bad_plt_tailcall_not_auted
+
+        .globl  bad_indirect_tailcall_not_auted
+        .type   bad_indirect_tailcall_not_auted,@function
+bad_indirect_tailcall_not_auted:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_indirect_tailcall_not_auted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   br      x0 # TAILCALL
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_not_auted, .-bad_indirect_tailcall_not_auted
+
+        .globl  bad_direct_tailcall_untrusted
+        .type   bad_direct_tailcall_untrusted,@function
+bad_direct_tailcall_untrusted:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_direct_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autiasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   b       callee # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        b       callee
+        .size bad_direct_tailcall_untrusted, .-bad_direct_tailcall_untrusted
+
+        .globl  bad_plt_tailcall_untrusted
+        .type   bad_plt_tailcall_untrusted,@function
+bad_plt_tailcall_untrusted:
+// FIXME: Calls via PLT are disassembled incorrectly. Nevertheless, they are
+//        still detected as tail calls.
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_plt_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_untrusted # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_plt_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_untrusted # TAILCALL
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autiasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   b       bad_indirect_tailcall_untrusted # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        b       callee_ext
+        .size bad_plt_tailcall_untrusted, .-bad_plt_tailcall_untrusted
+
+        .globl  bad_indirect_tailcall_untrusted
+        .type   bad_indirect_tailcall_untrusted,@function
+bad_indirect_tailcall_untrusted:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_indirect_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_indirect_tailcall_untrusted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      br      x0 # TAILCALL
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autiasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   br      x0 # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_untrusted, .-bad_indirect_tailcall_untrusted
+
+        .globl  good_direct_tailcall_trusted
+        .type   good_direct_tailcall_trusted,@function
+good_direct_tailcall_trusted:
+// CHECK-NOT: good_direct_tailcall_trusted
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b       callee
+        .size good_direct_tailcall_trusted, .-good_direct_tailcall_trusted
+
+        .globl  good_plt_tailcall_trusted
+        .type   good_plt_tailcall_trusted,@function
+good_plt_tailcall_trusted:
+// CHECK-NOT: good_plt_tailcall_trusted
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b       callee_ext
+        .size good_plt_tailcall_trusted, .-good_plt_tailcall_trusted
+
+        .globl  good_indirect_tailcall_trusted
+        .type   good_indirect_tailcall_trusted,@function
+good_indirect_tailcall_trusted:
+// CHECK-NOT: good_indirect_tailcall_trusted
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        autia   x0, x1
+        br      x0
+        .size good_indirect_tailcall_trusted, .-good_indirect_tailcall_trusted
+
+        .globl  good_direct_tailcall_no_clobber_multi_bb
+        .type   good_direct_tailcall_no_clobber_multi_bb,@function
+good_direct_tailcall_no_clobber_multi_bb:
+// CHECK-NOT: good_direct_tailcall_no_clobber_multi_bb
+        b 1f
+1:
+        b       callee
+        .size good_direct_tailcall_no_clobber_multi_bb, .-good_direct_tailcall_no_clobber_multi_bb
+
+        .globl  good_indirect_tailcall_no_clobber_multi_bb
+        .type   good_indirect_tailcall_no_clobber_multi_bb,@function
+good_indirect_tailcall_no_clobber_multi_bb:
+// CHECK-NOT: good_indirect_tailcall_no_clobber_multi_bb
+        autia   x0, x1
+        b 1f
+1:
+        br      x0
+        .size good_indirect_tailcall_no_clobber_multi_bb_multi_bb, .-good_indirect_tailcall_no_clobber_multi_bb_multi_bb
+
+        .globl  bad_direct_tailcall_not_auted_multi_bb
+        .type   bad_direct_tailcall_not_auted_multi_bb,@function
+bad_direct_tailcall_not_auted_multi_bb:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_not_auted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        cbz     x3, 1f
+        autiasp
+        ldr     w2, [x30]
+1:
+        b       callee
+        .size bad_direct_tailcall_not_auted_multi_bb, .-bad_direct_tailcall_not_auted_multi_bb
+
+        .globl  bad_indirect_tailcall_not_auted_multi_bb
+        .type   bad_indirect_tailcall_not_auted_multi_bb,@function
+bad_indirect_tailcall_not_auted_multi_bb:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_indirect_tailcall_not_auted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # UNKNOWN CONTROL FLOW
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        cbz     x3, 1f
+        autiasp
+        ldr     w2, [x30]
+1:
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_not_auted_multi_bb, .-bad_indirect_tailcall_not_auted_multi_bb
+
+        .globl  bad_direct_tailcall_untrusted_multi_bb
+        .type   bad_direct_tailcall_untrusted_multi_bb,@function
+bad_direct_tailcall_untrusted_multi_bb:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_untrusted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_direct_tailcall_untrusted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      b       callee # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        cbz     x3, 1f
+        ldr     w2, [x30]
+1:
+        b       callee
+        .size bad_direct_tailcall_untrusted_multi_bb, .-bad_direct_tailcall_untrusted_multi_bb
+
+        .globl  bad_indirect_tailcall_untrusted_multi_bb
+        .type   bad_indirect_tailcall_untrusted_multi_bb,@function
+bad_indirect_tailcall_untrusted_multi_bb:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_indirect_tailcall_untrusted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # UNKNOWN CONTROL FLOW
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_indirect_tailcall_untrusted_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        cbz     x3, 1f
+        ldr     w2, [x30]
+1:
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_untrusted_multi_bb, .-bad_indirect_tailcall_untrusted_multi_bb
+
+        .globl  good_direct_tailcall_trusted_multi_bb
+        .type   good_direct_tailcall_trusted_multi_bb,@function
+good_direct_tailcall_trusted_multi_bb:
+// CHECK-NOT: good_direct_tailcall_trusted_multi_bb
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b 1f
+1:
+        b       callee
+        .size good_direct_tailcall_trusted_multi_bb, .-good_direct_tailcall_trusted_multi_bb
+
+        .globl  good_indirect_tailcall_trusted_multi_bb
+        .type   good_indirect_tailcall_trusted_multi_bb,@function
+good_indirect_tailcall_trusted_multi_bb:
+// CHECK-NOT: good_indirect_tailcall_trusted_multi_bb
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b 1f
+1:
+        autia   x0, x1
+        br      x0
+        .size good_indirect_tailcall_trusted_multi_bb, .-good_indirect_tailcall_trusted_multi_bb
+
+        .globl  good_direct_tailcall_no_clobber_nocfg
+        .type   good_direct_tailcall_no_clobber_nocfg,@function
+good_direct_tailcall_no_clobber_nocfg:
+// CHECK-NOT: good_direct_tailcall_no_clobber_nocfg
+        adr     x3, 1f
+        br      x3
+1:
+        b       callee
+        .size good_direct_tailcall_no_clobber_nocfg, .-good_direct_tailcall_no_clobber_nocfg
+
+        .globl  good_plt_tailcall_no_clobber_nocfg
+        .type   good_plt_tailcall_no_clobber_nocfg,@function
+good_plt_tailcall_no_clobber_nocfg:
+// CHECK-NOT: good_plt_tailcall_no_clobber_nocfg
+        adr     x3, 1f
+        br      x3
+1:
+        b       callee_ext
+        .size good_plt_tailcall_no_clobber_nocfg, .-good_plt_tailcall_no_clobber_nocfg
+
+        .globl  good_indirect_tailcall_no_clobber_nocfg
+        .type   good_indirect_tailcall_no_clobber_nocfg,@function
+good_indirect_tailcall_no_clobber_nocfg:
+// CHECK-NOT: good_indirect_tailcall_no_clobber_nocfg
+        adr     x3, 1f
+        br      x3
+1:
+        autia   x0, x1
+        br      x0
+        .size good_indirect_tailcall_no_clobber_nocfg, .-good_indirect_tailcall_no_clobber_nocfg
+
+        .globl  bad_direct_tailcall_not_auted_nocfg
+        .type   bad_direct_tailcall_not_auted_nocfg,@function
+bad_direct_tailcall_not_auted_nocfg:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_not_auted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        b       callee
+        .size bad_direct_tailcall_not_auted_nocfg, .-bad_direct_tailcall_not_auted_nocfg
+
+        .globl  bad_plt_tailcall_not_auted_nocfg
+        .type   bad_plt_tailcall_not_auted_nocfg,@function
+bad_plt_tailcall_not_auted_nocfg:
+// FIXME: Calls via PLT are disassembled incorrectly. Nevertheless, they are
+//        still detected as tail calls.
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_plt_tailcall_not_auted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_not_auted_nocfg # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        b       callee_ext
+        .size bad_plt_tailcall_not_auted_nocfg, .-bad_plt_tailcall_not_auted_nocfg
+
+        .globl  bad_indirect_tailcall_not_auted_nocfg
+        .type   bad_indirect_tailcall_not_auted_nocfg,@function
+bad_indirect_tailcall_not_auted_nocfg:
+// Known false positive: ignoring UNKNOWN CONTROL FLOW without CFG.
+// CHECK-NOT: bad_indirect_tailcall_not_auted_nocfg
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_not_auted_nocfg, .-bad_indirect_tailcall_not_auted_nocfg
+
+        .globl  bad_direct_tailcall_untrusted_nocfg
+        .type   bad_direct_tailcall_untrusted_nocfg,@function
+bad_direct_tailcall_untrusted_nocfg:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_direct_tailcall_untrusted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_direct_tailcall_untrusted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      b       callee # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        b       callee
+        .size bad_direct_tailcall_untrusted_nocfg, .-bad_direct_tailcall_untrusted_nocfg
+
+        .globl  bad_plt_tailcall_untrusted_nocfg
+        .type   bad_plt_tailcall_untrusted_nocfg,@function
+bad_plt_tailcall_untrusted_nocfg:
+// FIXME: Calls via PLT are disassembled incorrectly. Nevertheless, they are
+//        still detected as tail calls.
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_plt_tailcall_untrusted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_untrusted_nocfg # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_plt_tailcall_untrusted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      b       bad_indirect_tailcall_untrusted_nocfg # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        b       callee_ext
+        .size bad_plt_tailcall_untrusted_nocfg, .-bad_plt_tailcall_untrusted_nocfg
+
+        .globl  bad_indirect_tailcall_untrusted_nocfg
+        .type   bad_indirect_tailcall_untrusted_nocfg,@function
+bad_indirect_tailcall_untrusted_nocfg:
+// Known false negative: ignoring UNKNOWN CONTROL FLOW without CFG.
+// Authentication oracle is found by a generic checker, though.
+// CHECK-NOT: untrusted link register{{.*}}bad_indirect_tailcall_untrusted_nocfg
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_indirect_tailcall_untrusted_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+// CHECK-NOT: untrusted link register{{.*}}bad_indirect_tailcall_untrusted_nocfg
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        autia   x0, x1
+        br      x0
+        .size bad_indirect_tailcall_untrusted_nocfg, .-bad_indirect_tailcall_untrusted_nocfg
+
+        .globl  good_direct_tailcall_trusted_nocfg
+        .type   good_direct_tailcall_trusted_nocfg,@function
+good_direct_tailcall_trusted_nocfg:
+// CHECK-NOT: good_direct_tailcall_trusted_nocfg
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b       callee
+        .size good_direct_tailcall_trusted_nocfg, .-good_direct_tailcall_trusted_nocfg
+
+        .globl  good_plt_tailcall_trusted_nocfg
+        .type   good_plt_tailcall_trusted_nocfg,@function
+good_plt_tailcall_trusted_nocfg:
+// CHECK-NOT: good_plt_tailcall_trusted_nocfg
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        b       callee_ext
+        .size good_plt_tailcall_trusted_nocfg, .-good_plt_tailcall_trusted_nocfg
+
+        .globl  good_indirect_tailcall_trusted_nocfg
+        .type   good_indirect_tailcall_trusted_nocfg,@function
+good_indirect_tailcall_trusted_nocfg:
+// CHECK-NOT: good_indirect_tailcall_trusted_nocfg
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        adr     x3, 1f
+        br      x3
+1:
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        ldr     w2, [x30]
+        autia   x0, x1
+        br      x0
+        .size good_indirect_tailcall_trusted_nocfg, .-good_indirect_tailcall_trusted_nocfg
+
+// Check Armv8.3-a fused auth+branch instructions.
+
+        .globl  good_indirect_tailcall_no_clobber_v83
+        .type   good_indirect_tailcall_no_clobber_v83,@function
+good_indirect_tailcall_no_clobber_v83:
+// CHECK-NOT: good_indirect_tailcall_no_clobber_v83
+        braa    x0, x1
+        .size good_indirect_tailcall_no_clobber_v83, .-good_indirect_tailcall_no_clobber_v83
+
+        .globl  bad_indirect_tailcall_untrusted_v83
+        .type   bad_indirect_tailcall_untrusted_v83,@function
+bad_indirect_tailcall_untrusted_v83:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_indirect_tailcall_untrusted_v83, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      braa    x0, x1 # TAILCALL
+// CHECK-NEXT:  The 0 instructions that write to the affected registers after any authentication are:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_indirect_tailcall_untrusted_v83, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      braa    x0, x1 # TAILCALL
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autiasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   braa    x0, x1 # TAILCALL
+        paciasp
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        autiasp
+        braa    x0, x1
+        .size bad_indirect_tailcall_untrusted_v83, .-bad_indirect_tailcall_untrusted_v83
+
+// Make sure ELF entry function does not generate false positive reports.
+// Additionally, check that the correct entry point is read from ELF header.
+
+        .globl  _start
+        .type   _start,@function
+_start:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function _start, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      b       callee # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mov     x30, #0x0
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   mov     x30, #0x0
+// CHECK-NEXT:  {{[0-9a-f]+}}:   b       callee # TAILCALL
+        mov     x30, #0
+        b       callee
+        .size   _start, .-_start
+
+        .globl  _custom_start
+        .type   _custom_start,@function
+_custom_start:
+// CHECK-NOT: _custom_start
+        mov     x30, #0
+        b       callee
+        .size   _custom_start, .-_custom_start
+
+// Test two issues being reported for the same instruction.
+
+        .globl  bad_non_protected_indirect_tailcall_not_auted
+        .type   bad_non_protected_indirect_tailcall_not_auted,@function
+bad_non_protected_indirect_tailcall_not_auted:
+// CHECK-LABEL: GS-PAUTH: untrusted link register found before tail call in function bad_non_protected_indirect_tailcall_not_auted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x0, [x1]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   br      x0 # TAILCALL
+// CHECK-LABEL: GS-PAUTH: non-protected call found in function bad_non_protected_indirect_tailcall_not_auted, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      br      x0 # TAILCALL
+// CHECK-NEXT:  The 1 instructions that write to the affected registers after any authentication are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      ldr     x0, [x1]
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x0, [x1]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   br      x0 # TAILCALL
+        stp     x29, x30, [sp, #-0x10]!
+        ldp     x29, x30, [sp], #0x10
+        ldr     x0, [x1]
+        br      x0
+        .size bad_non_protected_indirect_tailcall_not_auted, .-bad_non_protected_indirect_tailcall_not_auted
+
+        .globl  main
+        .type   main,@function
+main:
+        mov x0, 0
+        ret
+        .size   main, .-main
diff --git a/bolt/test/code-at-high-address.c b/bolt/test/code-at-high-address.c
new file mode 100644
index 000000000000..fa33c1eb342d
--- /dev/null
+++ b/bolt/test/code-at-high-address.c
@@ -0,0 +1,24 @@
+// Check that llvm-bolt pushes code to higher addresses under
+// --hot-functions-at-end when rewriting code in-place.
+
+// REQUIRES: system-linux
+
+// RUN: %clang %cflags -O0 %s -o %t -no-pie -Wl,-q -falign-functions=64 \
+// RUN:   -nostartfiles -nostdlib -ffreestanding
+// RUN: llvm-bolt %t -o %t.bolt --use-old-text --align-functions=1 \
+// RUN:   --no-huge-pages --align-text=1 --use-gnu-stack --hot-functions-at-end \
+// RUN:   | FileCheck %s --check-prefix=CHECK-BOLT
+// RUN: llvm-readelf --sections %t.bolt | FileCheck %s
+
+// CHECK-BOLT: using original .text for new code with 0x1 alignment at {{.*}}
+
+// As .text is pushed higher, preceding .bolt.org.text should have non-zero
+// size.
+// CHECK: .bolt.org.text PROGBITS
+// CHECK-NOT: {{ 000000 }}
+// CHECK-SAME: AX
+// CHECK-NEXT: .text PROGBITS
+
+int foo() { return 0; }
+
+int main() { return foo(); }
diff --git a/bolt/test/perf2bolt/X86/perf2bolt-spe.test b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
index 101bd3789a18..e981aef553b3 100644
--- a/bolt/test/perf2bolt/X86/perf2bolt-spe.test
+++ b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
@@ -6,4 +6,4 @@ RUN: %clang %cflags %p/../../Inputs/asm_foo.s %p/../../Inputs/asm_main.c -o %t.e
 RUN: touch %t.empty.perf.data
 RUN: not perf2bolt -p %t.empty.perf.data -o %t.perf.boltdata --spe --pa %t.exe 2>&1 | FileCheck %s
 
-CHECK: perf2bolt{{.*}} -spe is available only on AArch64.
+CHECK: -spe is available only on AArch64.
diff --git a/bolt/test/program-header.test b/bolt/test/program-header.test
new file mode 100644
index 000000000000..f5490394eb6d
--- /dev/null
+++ b/bolt/test/program-header.test
@@ -0,0 +1,14 @@
+# Check that llvm-bolt does not add new segments when writing code in-place.
+
+REQUIRES: system-linux
+
+RUN: %clang %cflags %p/Inputs/hello.c -o %t -no-pie -Wl,-q -nostartfiles \
+RUN:   -nostdlib -ffreestanding
+RUN: llvm-bolt %t -o %t.bolt --use-old-text --align-functions=1 \
+RUN:   --no-huge-pages --align-text=1 --use-gnu-stack \
+RUN:   | FileCheck %s --check-prefix=CHECK-BOLT
+RUN: llvm-readelf -WS %t.bolt | FileCheck %s
+
+CHECK-BOLT: not adding new segments
+
+CHECK-NOT: .bolt.org.eh_frame_hdr
diff --git a/bolt/unittests/Core/BinaryContext.cpp b/bolt/unittests/Core/BinaryContext.cpp
index 3fa0851d01e5..d7374b323c91 100644
--- a/bolt/unittests/Core/BinaryContext.cpp
+++ b/bolt/unittests/Core/BinaryContext.cpp
@@ -199,13 +199,13 @@ TEST_P(BinaryContextTester, BaseAddress) {
   // Check that  base address calculation is correct for a binary with the
   // following segment layout:
   BC->SegmentMapInfo[0] =
-      SegmentInfo{0, 0x10e8c2b4, 0, 0x10e8c2b4, 0x1000, true};
-  BC->SegmentMapInfo[0x10e8d2b4] =
-      SegmentInfo{0x10e8d2b4, 0x3952faec, 0x10e8c2b4, 0x3952faec, 0x1000, true};
-  BC->SegmentMapInfo[0x4a3bddc0] =
-      SegmentInfo{0x4a3bddc0, 0x148e828, 0x4a3bbdc0, 0x148e828, 0x1000, true};
-  BC->SegmentMapInfo[0x4b84d5e8] =
-      SegmentInfo{0x4b84d5e8, 0x294f830, 0x4b84a5e8, 0x3d3820, 0x1000, true};
+      SegmentInfo{0, 0x10e8c2b4, 0, 0x10e8c2b4, 0x1000, true, false};
+  BC->SegmentMapInfo[0x10e8d2b4] = SegmentInfo{
+      0x10e8d2b4, 0x3952faec, 0x10e8c2b4, 0x3952faec, 0x1000, true, false};
+  BC->SegmentMapInfo[0x4a3bddc0] = SegmentInfo{
+      0x4a3bddc0, 0x148e828, 0x4a3bbdc0, 0x148e828, 0x1000, true, false};
+  BC->SegmentMapInfo[0x4b84d5e8] = SegmentInfo{
+      0x4b84d5e8, 0x294f830, 0x4b84a5e8, 0x3d3820, 0x1000, true, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0x7f13f5556000, 0x10e8c000);
@@ -220,13 +220,14 @@ TEST_P(BinaryContextTester, BaseAddress2) {
   // Check that base address calculation is correct for a binary if the
   // alignment in ELF file are different from pagesize.
   // The segment layout is as follows:
-  BC->SegmentMapInfo[0] = SegmentInfo{0, 0x2177c, 0, 0x2177c, 0x10000, true};
+  BC->SegmentMapInfo[0] =
+      SegmentInfo{0, 0x2177c, 0, 0x2177c, 0x10000, true, false};
   BC->SegmentMapInfo[0x31860] =
-      SegmentInfo{0x31860, 0x370, 0x21860, 0x370, 0x10000, true};
+      SegmentInfo{0x31860, 0x370, 0x21860, 0x370, 0x10000, true, false};
   BC->SegmentMapInfo[0x41c20] =
-      SegmentInfo{0x41c20, 0x1f8, 0x21c20, 0x1f8, 0x10000, true};
+      SegmentInfo{0x41c20, 0x1f8, 0x21c20, 0x1f8, 0x10000, true, false};
   BC->SegmentMapInfo[0x54e18] =
-      SegmentInfo{0x54e18, 0x51, 0x24e18, 0x51, 0x10000, true};
+      SegmentInfo{0x54e18, 0x51, 0x24e18, 0x51, 0x10000, true, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0xaaaaea444000, 0x21000);
@@ -242,13 +243,14 @@ TEST_P(BinaryContextTester, BaseAddressSegmentsSmallerThanAlignment) {
   // when multiple segments are close together in the ELF file (closer
   // than the required alignment in the process space).
   // See https://github.com/llvm/llvm-project/issues/109384
-  BC->SegmentMapInfo[0] = SegmentInfo{0, 0x1d1c, 0, 0x1d1c, 0x10000, false};
+  BC->SegmentMapInfo[0] =
+      SegmentInfo{0, 0x1d1c, 0, 0x1d1c, 0x10000, false, false};
   BC->SegmentMapInfo[0x11d40] =
-      SegmentInfo{0x11d40, 0x11e0, 0x1d40, 0x11e0, 0x10000, true};
+      SegmentInfo{0x11d40, 0x11e0, 0x1d40, 0x11e0, 0x10000, true, false};
   BC->SegmentMapInfo[0x22f20] =
-      SegmentInfo{0x22f20, 0x10e0, 0x2f20, 0x1f0, 0x10000, false};
+      SegmentInfo{0x22f20, 0x10e0, 0x2f20, 0x1f0, 0x10000, false, false};
   BC->SegmentMapInfo[0x33110] =
-      SegmentInfo{0x33110, 0x89, 0x3110, 0x88, 0x10000, false};
+      SegmentInfo{0x33110, 0x89, 0x3110, 0x88, 0x10000, false, false};
 
   std::optional<uint64_t> BaseAddress =
       BC->getBaseAddressForMapping(0xaaaaaaab1000, 0x1000);
diff --git a/bolt/unittests/Core/MemoryMaps.cpp b/bolt/unittests/Core/MemoryMaps.cpp
index b0cab5431bdd..8eb8f8ae529b 100644
--- a/bolt/unittests/Core/MemoryMaps.cpp
+++ b/bolt/unittests/Core/MemoryMaps.cpp
@@ -100,10 +100,10 @@ TEST_P(MemoryMapsTester, ParseMultipleSegments) {
       "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
       Pid, Filename);
 
-  BC->SegmentMapInfo[0x11da000] =
-      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
-  BC->SegmentMapInfo[0x31d0000] =
-      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true};
+  BC->SegmentMapInfo[0x11da000] = SegmentInfo{
+      0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true, false};
+  BC->SegmentMapInfo[0x31d0000] = SegmentInfo{
+      0x31d0000, 0x51ac82c, 0x31d0000, 0x3000000, 0x200000, true, false};
 
   DataAggregator DA("");
   BC->setFilename(Filename);
@@ -131,12 +131,12 @@ TEST_P(MemoryMapsTester, MultipleSegmentsMismatchedBaseAddress) {
       "[0xabc2000000(0x8000000) @ 0x31d0000 103:01 1573523 0]: r-xp {1}\n",
       Pid, Filename);
 
-  BC->SegmentMapInfo[0x11da000] =
-      SegmentInfo{0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true};
+  BC->SegmentMapInfo[0x11da000] = SegmentInfo{
+      0x11da000, 0x10da000, 0x11ca000, 0x10da000, 0x10000, true, false};
   // Using '0x31d0fff' FileOffset which triggers a different base address
   // for this second text segment.
-  BC->SegmentMapInfo[0x31d0000] =
-      SegmentInfo{0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true};
+  BC->SegmentMapInfo[0x31d0000] = SegmentInfo{
+      0x31d0000, 0x51ac82c, 0x31d0fff, 0x3000000, 0x200000, true, false};
 
   DataAggregator DA("");
   BC->setFilename(Filename);
diff --git a/bolt/utils/llvm-bolt-wrapper.py b/bolt/utils/llvm-bolt-wrapper.py
index b9d6fad825e7..b913394bce65 100755
--- a/bolt/utils/llvm-bolt-wrapper.py
+++ b/bolt/utils/llvm-bolt-wrapper.py
@@ -79,7 +79,7 @@ def get_cfg(key):
 
 
 # perf2bolt mode
-PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id"]
+PERF2BOLT_MODE = ["-aggregate-only", "-ignore-build-id", "-show-density"]
 
 # boltdiff mode
 BOLTDIFF_MODE = ["-diff-only", "-o", "/dev/null"]
diff --git a/bolt/utils/nfc-check-setup.py b/bolt/utils/nfc-check-setup.py
index 710b18350585..275ac7b886d0 100755
--- a/bolt/utils/nfc-check-setup.py
+++ b/bolt/utils/nfc-check-setup.py
@@ -7,6 +7,29 @@
 import sys
 import textwrap
 
+def get_relevant_bolt_changes(dir: str) -> str:
+    # Return a list of bolt source changes that are relevant to testing.
+    all_changes = subprocess.run(
+        shlex.split("git show HEAD --name-only --pretty=''"),
+        cwd=dir,
+        text=True,
+        stdout=subprocess.PIPE,
+    )
+    keep_bolt = subprocess.run(
+        shlex.split("grep '^bolt'"),
+        input=all_changes.stdout,
+        text=True,
+        stdout=subprocess.PIPE,
+    )
+    keep_relevant = subprocess.run(
+        shlex.split(
+            "grep -v -e '^bolt/docs' -e '^bolt/utils/docker' -e '^bolt/utils/dot2html'"
+        ),
+        input=keep_bolt.stdout,
+        text=True,
+        stdout=subprocess.PIPE,
+    )
+    return keep_relevant.stdout
 
 def get_git_ref_or_rev(dir: str) -> str:
     # Run 'git symbolic-ref -q --short HEAD || git rev-parse --short HEAD'
@@ -36,6 +59,12 @@ def main():
         default=os.getcwd(),
         help="Path to BOLT build directory, default is current " "directory",
     )
+    parser.add_argument(
+        "--check-bolt-sources",
+        default=False,
+        action="store_true",
+        help="Create a marker file (.llvm-bolt.changes) if any relevant BOLT sources are modified",
+    )
     parser.add_argument(
         "--switch-back",
         default=False,
@@ -71,6 +100,16 @@ def main():
     # memorize the old hash for logging
     old_ref = get_git_ref_or_rev(source_dir)
 
+    if args.check_bolt_sources:
+        marker = f"{args.build_dir}/.llvm-bolt.changes"
+        if os.path.exists(marker):
+            os.remove(marker)
+        file_changes = get_relevant_bolt_changes(source_dir)
+        # Create a marker file if any relevant BOLT source files changed.
+        if len(file_changes) > 0:
+            print(f"BOLT source changes were found:\n{file_changes}")
+            open(marker, "a").close()
+
     # determine whether a stash is needed
     stash = subprocess.run(
         shlex.split("git status --porcelain"),
diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 66852931226b..f756ae6d897c 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -93,6 +93,8 @@ static llvm::Error decodeRecord(const Record &R, InfoType &Field,
   case InfoType::IT_enum:
   case InfoType::IT_typedef:
   case InfoType::IT_concept:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     Field = IT;
     return llvm::Error::success();
   }
@@ -110,6 +112,7 @@ static llvm::Error decodeRecord(const Record &R, FieldId &Field,
   case FieldId::F_child_namespace:
   case FieldId::F_child_record:
   case FieldId::F_concept:
+  case FieldId::F_friend:
   case FieldId::F_default:
     Field = F;
     return llvm::Error::success();
@@ -282,7 +285,15 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
 
 static llvm::Error parseRecord(const Record &R, unsigned ID,
                                llvm::StringRef Blob, TypeInfo *I) {
-  return llvm::Error::success();
+  switch (ID) {
+  case TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid field for TypeInfo");
+  }
 }
 
 static llvm::Error parseRecord(const Record &R, unsigned ID,
@@ -292,6 +303,10 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
     return decodeRecord(R, I->Name, Blob);
   case FIELD_DEFAULT_VALUE:
     return decodeRecord(R, I->DefaultValue, Blob);
+  case FIELD_TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case FIELD_TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for TypeInfo");
@@ -307,6 +322,10 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
     return decodeRecord(R, I->Access, Blob);
   case MEMBER_TYPE_IS_STATIC:
     return decodeRecord(R, I->IsStatic, Blob);
+  case MEMBER_TYPE_IS_BUILTIN:
+    return decodeRecord(R, I->IsBuiltIn, Blob);
+  case MEMBER_TYPE_IS_TEMPLATE:
+    return decodeRecord(R, I->IsTemplate, Blob);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid field for MemberTypeInfo");
@@ -416,6 +435,32 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
                                  "invalid field for ConstraintInfo");
 }
 
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, VarInfo *I) {
+  switch (ID) {
+  case VAR_USR:
+    return decodeRecord(R, I->USR, Blob);
+  case VAR_NAME:
+    return decodeRecord(R, I->Name, Blob);
+  case VAR_DEFLOCATION:
+    return decodeRecord(R, I->DefLoc, Blob);
+  case VAR_IS_STATIC:
+    return decodeRecord(R, I->IsStatic, Blob);
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "invalid field for VarInfo");
+  }
+}
+
+static llvm::Error parseRecord(const Record &R, unsigned ID, StringRef Blob,
+                               FriendInfo *F) {
+  if (ID == FRIEND_IS_CLASS) {
+    return decodeRecord(R, F->IsClass, Blob);
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "invalid field for Friend");
+}
+
 template <typename T> static llvm::Expected<CommentInfo *> getCommentInfo(T I) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain CommentInfo");
@@ -458,6 +503,10 @@ template <> llvm::Expected<CommentInfo *> getCommentInfo(ConceptInfo *I) {
   return &I->Description.emplace_back();
 }
 
+template <> Expected<CommentInfo *> getCommentInfo(VarInfo *I) {
+  return &I->Description.emplace_back();
+}
+
 // When readSubBlock encounters a TypeInfo sub-block, it calls addTypeInfo on
 // the parent block to set it. The template specializations define what to do
 // for each supported parent block.
@@ -487,6 +536,18 @@ template <> llvm::Error addTypeInfo(FunctionInfo *I, FieldTypeInfo &&T) {
   return llvm::Error::success();
 }
 
+template <> llvm::Error addTypeInfo(FriendInfo *I, FieldTypeInfo &&T) {
+  if (!I->Params)
+    I->Params.emplace();
+  I->Params->emplace_back(std::move(T));
+  return llvm::Error::success();
+}
+
+template <> llvm::Error addTypeInfo(FriendInfo *I, TypeInfo &&T) {
+  I->ReturnType.emplace(std::move(T));
+  return llvm::Error::success();
+}
+
 template <> llvm::Error addTypeInfo(EnumInfo *I, TypeInfo &&T) {
   I->BaseType = std::move(T);
   return llvm::Error::success();
@@ -497,12 +558,28 @@ template <> llvm::Error addTypeInfo(TypedefInfo *I, TypeInfo &&T) {
   return llvm::Error::success();
 }
 
+template <> llvm::Error addTypeInfo(VarInfo *I, TypeInfo &&T) {
+  I->Type = std::move(T);
+  return llvm::Error::success();
+}
+
 template <typename T>
 static llvm::Error addReference(T I, Reference &&R, FieldId F) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain Reference");
 }
 
+template <> llvm::Error addReference(VarInfo *I, Reference &&R, FieldId F) {
+  switch (F) {
+  case FieldId::F_namespace:
+    I->Namespace.emplace_back(std::move(R));
+    return llvm::Error::success();
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "VarInfo cannot contain this Reference");
+  }
+}
+
 template <> llvm::Error addReference(TypeInfo *I, Reference &&R, FieldId F) {
   switch (F) {
   case FieldId::F_type:
@@ -624,6 +701,16 @@ llvm::Error addReference(ConstraintInfo *I, Reference &&R, FieldId F) {
       "ConstraintInfo cannot contain this Reference");
 }
 
+template <>
+llvm::Error addReference(FriendInfo *Friend, Reference &&R, FieldId F) {
+  if (F == FieldId::F_friend) {
+    Friend->Ref = std::move(R);
+    return llvm::Error::success();
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "Friend cannot contain this Reference");
+}
+
 template <typename T, typename ChildInfoType>
 static void addChild(T I, ChildInfoType &&R) {
   llvm::errs() << "invalid child type for info";
@@ -643,6 +730,9 @@ template <> void addChild(NamespaceInfo *I, TypedefInfo &&R) {
 template <> void addChild(NamespaceInfo *I, ConceptInfo &&R) {
   I->Children.Concepts.emplace_back(std::move(R));
 }
+template <> void addChild(NamespaceInfo *I, VarInfo &&R) {
+  I->Children.Variables.emplace_back(std::move(R));
+}
 
 // Record children:
 template <> void addChild(RecordInfo *I, FunctionInfo &&R) {
@@ -654,6 +744,9 @@ template <> void addChild(RecordInfo *I, EnumInfo &&R) {
 template <> void addChild(RecordInfo *I, TypedefInfo &&R) {
   I->Children.Typedefs.emplace_back(std::move(R));
 }
+template <> void addChild(RecordInfo *I, FriendInfo &&R) {
+  I->Friends.emplace_back(std::move(R));
+}
 
 // Other types of children:
 template <> void addChild(EnumInfo *I, EnumValueInfo &&R) {
@@ -695,6 +788,9 @@ template <> void addTemplate(FunctionInfo *I, TemplateInfo &&P) {
 template <> void addTemplate(ConceptInfo *I, TemplateInfo &&P) {
   I->Template = std::move(P);
 }
+template <> void addTemplate(FriendInfo *I, TemplateInfo &&P) {
+  I->Template.emplace(std::move(P));
+}
 
 // Template specializations go only into template records.
 template <typename T>
@@ -770,11 +866,39 @@ llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
   }
 }
 
-// TODO: Create a helper that can receive a function to reduce repetition for
-// most blocks.
+// TODO: fix inconsistentent returning of errors in add callbacks.
+// Once that's fixed, we only need one handleSubBlock.
+template <typename InfoType, typename T, typename Callback>
+llvm::Error ClangDocBitcodeReader::handleSubBlock(unsigned ID, T Parent,
+                                                  Callback Function) {
+  InfoType Info;
+  if (auto Err = readBlock(ID, &Info))
+    return Err;
+  Function(Parent, std::move(Info));
+  return llvm::Error::success();
+}
+
+template <typename InfoType, typename T, typename Callback>
+llvm::Error ClangDocBitcodeReader::handleTypeSubBlock(unsigned ID, T Parent,
+                                                      Callback Function) {
+  InfoType Info;
+  if (auto Err = readBlock(ID, &Info))
+    return Err;
+  if (auto Err = Function(Parent, std::move(Info)))
+    return Err;
+  return llvm::Error::success();
+}
+
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
   llvm::TimeTraceScope("Reducing infos", "readSubBlock");
+
+  static auto CreateAddFunc = [](auto AddFunc) {
+    return [AddFunc](auto Parent, auto Child) {
+      return AddFunc(Parent, std::move(Child));
+    };
+  };
+
   switch (ID) {
   // Blocks can only have certain types of sub blocks.
   case BI_COMMENT_BLOCK_ID: {
@@ -786,28 +910,16 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
     return llvm::Error::success();
   }
   case BI_TYPE_BLOCK_ID: {
-    TypeInfo TI;
-    if (auto Err = readBlock(ID, &TI))
-      return Err;
-    if (auto Err = addTypeInfo(I, std::move(TI)))
-      return Err;
-    return llvm::Error::success();
+    return handleTypeSubBlock<TypeInfo>(
+        ID, I, CreateAddFunc(addTypeInfo<T, TypeInfo>));
   }
   case BI_FIELD_TYPE_BLOCK_ID: {
-    FieldTypeInfo TI;
-    if (auto Err = readBlock(ID, &TI))
-      return Err;
-    if (auto Err = addTypeInfo(I, std::move(TI)))
-      return Err;
-    return llvm::Error::success();
+    return handleTypeSubBlock<FieldTypeInfo>(
+        ID, I, CreateAddFunc(addTypeInfo<T, FieldTypeInfo>));
   }
   case BI_MEMBER_TYPE_BLOCK_ID: {
-    MemberTypeInfo TI;
-    if (auto Err = readBlock(ID, &TI))
-      return Err;
-    if (auto Err = addTypeInfo(I, std::move(TI)))
-      return Err;
-    return llvm::Error::success();
+    return handleTypeSubBlock<MemberTypeInfo>(
+        ID, I, CreateAddFunc(addTypeInfo<T, MemberTypeInfo>));
   }
   case BI_REFERENCE_BLOCK_ID: {
     Reference R;
@@ -818,74 +930,50 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
     return llvm::Error::success();
   }
   case BI_FUNCTION_BLOCK_ID: {
-    FunctionInfo F;
-    if (auto Err = readBlock(ID, &F))
-      return Err;
-    addChild(I, std::move(F));
-    return llvm::Error::success();
+    return handleSubBlock<FunctionInfo>(
+        ID, I, CreateAddFunc(addChild<T, FunctionInfo>));
   }
   case BI_BASE_RECORD_BLOCK_ID: {
-    BaseRecordInfo BR;
-    if (auto Err = readBlock(ID, &BR))
-      return Err;
-    addChild(I, std::move(BR));
-    return llvm::Error::success();
+    return handleSubBlock<BaseRecordInfo>(
+        ID, I, CreateAddFunc(addChild<T, BaseRecordInfo>));
   }
   case BI_ENUM_BLOCK_ID: {
-    EnumInfo E;
-    if (auto Err = readBlock(ID, &E))
-      return Err;
-    addChild(I, std::move(E));
-    return llvm::Error::success();
+    return handleSubBlock<EnumInfo>(ID, I,
+                                    CreateAddFunc(addChild<T, EnumInfo>));
   }
   case BI_ENUM_VALUE_BLOCK_ID: {
-    EnumValueInfo EV;
-    if (auto Err = readBlock(ID, &EV))
-      return Err;
-    addChild(I, std::move(EV));
-    return llvm::Error::success();
+    return handleSubBlock<EnumValueInfo>(
+        ID, I, CreateAddFunc(addChild<T, EnumValueInfo>));
   }
   case BI_TEMPLATE_BLOCK_ID: {
-    TemplateInfo TI;
-    if (auto Err = readBlock(ID, &TI))
-      return Err;
-    addTemplate(I, std::move(TI));
-    return llvm::Error::success();
+    return handleSubBlock<TemplateInfo>(ID, I, CreateAddFunc(addTemplate<T>));
   }
   case BI_TEMPLATE_SPECIALIZATION_BLOCK_ID: {
-    TemplateSpecializationInfo TSI;
-    if (auto Err = readBlock(ID, &TSI))
-      return Err;
-    addTemplateSpecialization(I, std::move(TSI));
-    return llvm::Error::success();
+    return handleSubBlock<TemplateSpecializationInfo>(
+        ID, I, CreateAddFunc(addTemplateSpecialization<T>));
   }
   case BI_TEMPLATE_PARAM_BLOCK_ID: {
-    TemplateParamInfo TPI;
-    if (auto Err = readBlock(ID, &TPI))
-      return Err;
-    addTemplateParam(I, std::move(TPI));
-    return llvm::Error::success();
+    return handleSubBlock<TemplateParamInfo>(
+        ID, I, CreateAddFunc(addTemplateParam<T>));
   }
   case BI_TYPEDEF_BLOCK_ID: {
-    TypedefInfo TI;
-    if (auto Err = readBlock(ID, &TI))
-      return Err;
-    addChild(I, std::move(TI));
-    return llvm::Error::success();
+    return handleSubBlock<TypedefInfo>(ID, I,
+                                       CreateAddFunc(addChild<T, TypedefInfo>));
   }
   case BI_CONSTRAINT_BLOCK_ID: {
-    ConstraintInfo CI;
-    if (auto Err = readBlock(ID, &CI))
-      return Err;
-    addConstraint(I, std::move(CI));
-    return llvm::Error::success();
+    return handleSubBlock<ConstraintInfo>(ID, I,
+                                          CreateAddFunc(addConstraint<T>));
   }
   case BI_CONCEPT_BLOCK_ID: {
-    ConceptInfo CI;
-    if (auto Err = readBlock(ID, &CI))
-      return Err;
-    addChild(I, std::move(CI));
-    return llvm::Error::success();
+    return handleSubBlock<ConceptInfo>(ID, I,
+                                       CreateAddFunc(addChild<T, ConceptInfo>));
+  }
+  case BI_VAR_BLOCK_ID: {
+    return handleSubBlock<VarInfo>(ID, I, CreateAddFunc(addChild<T, VarInfo>));
+  }
+  case BI_FRIEND_BLOCK_ID: {
+    return handleSubBlock<FriendInfo>(ID, I,
+                                      CreateAddFunc(addChild<T, FriendInfo>));
   }
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -996,6 +1084,10 @@ ClangDocBitcodeReader::readBlockToInfo(unsigned ID) {
     return createInfo<ConceptInfo>(ID);
   case BI_FUNCTION_BLOCK_ID:
     return createInfo<FunctionInfo>(ID);
+  case BI_VAR_BLOCK_ID:
+    return createInfo<VarInfo>(ID);
+  case BI_FRIEND_BLOCK_ID:
+    return createInfo<FriendInfo>(ID);
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "cannot create info");
@@ -1035,6 +1127,8 @@ ClangDocBitcodeReader::readBitcode() {
     case BI_ENUM_BLOCK_ID:
     case BI_TYPEDEF_BLOCK_ID:
     case BI_CONCEPT_BLOCK_ID:
+    case BI_VAR_BLOCK_ID:
+    case BI_FRIEND_BLOCK_ID:
     case BI_FUNCTION_BLOCK_ID: {
       auto InfoOrErr = readBlockToInfo(ID);
       if (!InfoOrErr)
diff --git a/clang-tools-extra/clang-doc/BitcodeReader.h b/clang-tools-extra/clang-doc/BitcodeReader.h
index a046ee2f7a24..4947721f0a06 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.h
+++ b/clang-tools-extra/clang-doc/BitcodeReader.h
@@ -64,6 +64,13 @@ class ClangDocBitcodeReader {
   // Helper function to set up the appropriate type of Info.
   llvm::Expected<std::unique_ptr<Info>> readBlockToInfo(unsigned ID);
 
+  template <typename InfoType, typename T, typename CallbackFunction>
+  llvm::Error handleSubBlock(unsigned ID, T Parent, CallbackFunction Function);
+
+  template <typename InfoType, typename T, typename CallbackFunction>
+  llvm::Error handleTypeSubBlock(unsigned ID, T Parent,
+                                 CallbackFunction Function);
+
   llvm::BitstreamCursor &Stream;
   std::optional<llvm::BitstreamBlockInfo> BlockInfo;
   FieldId CurrentReferenceField;
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index b7308c012786..3cc0d4ad332f 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -130,7 +130,9 @@ static const llvm::IndexedMap<llvm::StringRef, BlockIdToIndexFunctor>
           {BI_TEMPLATE_SPECIALIZATION_BLOCK_ID, "TemplateSpecializationBlock"},
           {BI_TEMPLATE_PARAM_BLOCK_ID, "TemplateParamBlock"},
           {BI_CONSTRAINT_BLOCK_ID, "ConstraintBlock"},
-          {BI_CONCEPT_BLOCK_ID, "ConceptBlock"}};
+          {BI_CONCEPT_BLOCK_ID, "ConceptBlock"},
+          {BI_VAR_BLOCK_ID, "VarBlock"},
+          {BI_FRIEND_BLOCK_ID, "FriendBlock"}};
       assert(Inits.size() == BlockIdCount);
       for (const auto &Init : Inits)
         BlockIdNameMap[Init.first] = Init.second;
@@ -160,9 +162,15 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {COMMENT_ARG, {"Arg", &genStringAbbrev}},
           {FIELD_TYPE_NAME, {"Name", &genStringAbbrev}},
           {FIELD_DEFAULT_VALUE, {"DefaultValue", &genStringAbbrev}},
+          {FIELD_TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {FIELD_TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
           {MEMBER_TYPE_NAME, {"Name", &genStringAbbrev}},
           {MEMBER_TYPE_ACCESS, {"Access", &genIntAbbrev}},
           {MEMBER_TYPE_IS_STATIC, {"IsStatic", &genBoolAbbrev}},
+          {MEMBER_TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {MEMBER_TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
+          {TYPE_IS_BUILTIN, {"IsBuiltin", &genBoolAbbrev}},
+          {TYPE_IS_TEMPLATE, {"IsTemplate", &genBoolAbbrev}},
           {NAMESPACE_USR, {"USR", &genSymbolIdAbbrev}},
           {NAMESPACE_NAME, {"Name", &genStringAbbrev}},
           {NAMESPACE_PATH, {"Path", &genStringAbbrev}},
@@ -213,7 +221,13 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {CONCEPT_IS_TYPE, {"IsType", &genBoolAbbrev}},
           {CONCEPT_CONSTRAINT_EXPRESSION,
            {"ConstraintExpression", &genStringAbbrev}},
-          {CONSTRAINT_EXPRESSION, {"Expression", &genStringAbbrev}}};
+          {CONSTRAINT_EXPRESSION, {"Expression", &genStringAbbrev}},
+          {VAR_USR, {"USR", &genSymbolIdAbbrev}},
+          {VAR_NAME, {"Name", &genStringAbbrev}},
+          {VAR_DEFLOCATION, {"DefLocation", &genLocationAbbrev}},
+          {VAR_IS_STATIC, {"IsStatic", &genBoolAbbrev}},
+          {FRIEND_IS_CLASS, {"IsClass", &genBoolAbbrev}}};
+
       assert(Inits.size() == RecordIdCount);
       for (const auto &Init : Inits) {
         RecordIdNameMap[Init.first] = Init.second;
@@ -233,12 +247,15 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
           COMMENT_PARAMNAME, COMMENT_CLOSENAME, COMMENT_SELFCLOSING,
           COMMENT_EXPLICIT, COMMENT_ATTRKEY, COMMENT_ATTRVAL, COMMENT_ARG}},
         // Type Block
-        {BI_TYPE_BLOCK_ID, {}},
+        {BI_TYPE_BLOCK_ID, {TYPE_IS_BUILTIN, TYPE_IS_TEMPLATE}},
         // FieldType Block
-        {BI_FIELD_TYPE_BLOCK_ID, {FIELD_TYPE_NAME, FIELD_DEFAULT_VALUE}},
+        {BI_FIELD_TYPE_BLOCK_ID,
+         {FIELD_TYPE_NAME, FIELD_DEFAULT_VALUE, FIELD_TYPE_IS_BUILTIN,
+          FIELD_TYPE_IS_TEMPLATE}},
         // MemberType Block
         {BI_MEMBER_TYPE_BLOCK_ID,
-         {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS, MEMBER_TYPE_IS_STATIC}},
+         {MEMBER_TYPE_NAME, MEMBER_TYPE_ACCESS, MEMBER_TYPE_IS_STATIC,
+          MEMBER_TYPE_IS_BUILTIN, MEMBER_TYPE_IS_TEMPLATE}},
         // Enum Block
         {BI_ENUM_BLOCK_ID,
          {ENUM_USR, ENUM_NAME, ENUM_DEFLOCATION, ENUM_LOCATION, ENUM_SCOPED}},
@@ -277,7 +294,9 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
          {CONCEPT_USR, CONCEPT_NAME, CONCEPT_IS_TYPE,
           CONCEPT_CONSTRAINT_EXPRESSION}},
         // Constraint Block
-        {BI_CONSTRAINT_BLOCK_ID, {CONSTRAINT_EXPRESSION}}};
+        {BI_CONSTRAINT_BLOCK_ID, {CONSTRAINT_EXPRESSION}},
+        {BI_VAR_BLOCK_ID, {VAR_NAME, VAR_USR, VAR_DEFLOCATION, VAR_IS_STATIC}},
+        {BI_FRIEND_BLOCK_ID, {FRIEND_IS_CLASS}}};
 
 // AbbreviationMap
 
@@ -460,9 +479,24 @@ void ClangDocBitcodeWriter::emitBlock(const Reference &R, FieldId Field) {
   emitRecord((unsigned)Field, REFERENCE_FIELD);
 }
 
+void ClangDocBitcodeWriter::emitBlock(const FriendInfo &R) {
+  StreamSubBlockGuard Block(Stream, BI_FRIEND_BLOCK_ID);
+  emitBlock(R.Ref, FieldId::F_friend);
+  emitRecord(R.IsClass, FRIEND_IS_CLASS);
+  if (R.Template)
+    emitBlock(*R.Template);
+  if (R.Params)
+    for (const auto &P : *R.Params)
+      emitBlock(P);
+  if (R.ReturnType)
+    emitBlock(*R.ReturnType);
+}
+
 void ClangDocBitcodeWriter::emitBlock(const TypeInfo &T) {
   StreamSubBlockGuard Block(Stream, BI_TYPE_BLOCK_ID);
   emitBlock(T.Type, FieldId::F_type);
+  emitRecord(T.IsBuiltIn, TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, TYPE_IS_TEMPLATE);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const TypedefInfo &T) {
@@ -484,6 +518,8 @@ void ClangDocBitcodeWriter::emitBlock(const FieldTypeInfo &T) {
   emitBlock(T.Type, FieldId::F_type);
   emitRecord(T.Name, FIELD_TYPE_NAME);
   emitRecord(T.DefaultValue, FIELD_DEFAULT_VALUE);
+  emitRecord(T.IsBuiltIn, FIELD_TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, FIELD_TYPE_IS_TEMPLATE);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
@@ -492,6 +528,9 @@ void ClangDocBitcodeWriter::emitBlock(const MemberTypeInfo &T) {
   emitRecord(T.Name, MEMBER_TYPE_NAME);
   emitRecord(T.Access, MEMBER_TYPE_ACCESS);
   emitRecord(T.IsStatic, MEMBER_TYPE_IS_STATIC);
+  emitRecord(T.IsBuiltIn, MEMBER_TYPE_IS_BUILTIN);
+  emitRecord(T.IsTemplate, MEMBER_TYPE_IS_TEMPLATE);
+  emitRecord(T.IsTemplate, MEMBER_TYPE_IS_TEMPLATE);
   for (const auto &CI : T.Description)
     emitBlock(CI);
 }
@@ -540,6 +579,8 @@ void ClangDocBitcodeWriter::emitBlock(const NamespaceInfo &I) {
     emitBlock(C);
   for (const auto &C : I.Children.Concepts)
     emitBlock(C);
+  for (const auto &C : I.Children.Variables)
+    emitBlock(C);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const EnumInfo &I) {
@@ -603,6 +644,8 @@ void ClangDocBitcodeWriter::emitBlock(const RecordInfo &I) {
     emitBlock(C);
   if (I.Template)
     emitBlock(*I.Template);
+  for (const auto &C : I.Friends)
+    emitBlock(C);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const BaseRecordInfo &I) {
@@ -682,6 +725,20 @@ void ClangDocBitcodeWriter::emitBlock(const ConstraintInfo &C) {
   emitBlock(C.ConceptRef, FieldId::F_concept);
 }
 
+void ClangDocBitcodeWriter::emitBlock(const VarInfo &I) {
+  StreamSubBlockGuard Block(Stream, BI_VAR_BLOCK_ID);
+  emitRecord(I.USR, VAR_USR);
+  emitRecord(I.Name, VAR_NAME);
+  for (const auto &N : I.Namespace)
+    emitBlock(N, FieldId::F_namespace);
+  for (const auto &CI : I.Description)
+    emitBlock(CI);
+  if (I.DefLoc)
+    emitRecord(*I.DefLoc, VAR_DEFLOCATION);
+  emitRecord(I.IsStatic, VAR_IS_STATIC);
+  emitBlock(I.Type);
+}
+
 bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   switch (I->IT) {
   case InfoType::IT_namespace:
@@ -702,6 +759,12 @@ bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   case InfoType::IT_concept:
     emitBlock(*static_cast<clang::doc::ConceptInfo *>(I));
     break;
+  case InfoType::IT_variable:
+    emitBlock(*static_cast<VarInfo *>(I));
+    break;
+  case InfoType::IT_friend:
+    emitBlock(*static_cast<FriendInfo *>(I));
+    break;
   case InfoType::IT_default:
     llvm::errs() << "Unexpected info, unable to write.\n";
     return true;
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.h b/clang-tools-extra/clang-doc/BitcodeWriter.h
index 4d0c0c07805e..d09ec4ca3400 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.h
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.h
@@ -69,6 +69,8 @@ enum BlockId {
   BI_CONSTRAINT_BLOCK_ID,
   BI_TYPEDEF_BLOCK_ID,
   BI_CONCEPT_BLOCK_ID,
+  BI_VAR_BLOCK_ID,
+  BI_FRIEND_BLOCK_ID,
   BI_LAST,
   BI_FIRST = BI_VERSION_BLOCK_ID
 };
@@ -95,11 +97,17 @@ enum RecordId {
   COMMENT_ATTRKEY,
   COMMENT_ATTRVAL,
   COMMENT_ARG,
+  TYPE_IS_BUILTIN,
+  TYPE_IS_TEMPLATE,
   FIELD_TYPE_NAME,
   FIELD_DEFAULT_VALUE,
+  FIELD_TYPE_IS_BUILTIN,
+  FIELD_TYPE_IS_TEMPLATE,
   MEMBER_TYPE_NAME,
   MEMBER_TYPE_ACCESS,
   MEMBER_TYPE_IS_STATIC,
+  MEMBER_TYPE_IS_BUILTIN,
+  MEMBER_TYPE_IS_TEMPLATE,
   NAMESPACE_USR,
   NAMESPACE_NAME,
   NAMESPACE_PATH,
@@ -142,6 +150,11 @@ enum RecordId {
   CONCEPT_IS_TYPE,
   CONCEPT_CONSTRAINT_EXPRESSION,
   CONSTRAINT_EXPRESSION,
+  VAR_USR,
+  VAR_NAME,
+  VAR_DEFLOCATION,
+  VAR_IS_STATIC,
+  FRIEND_IS_CLASS,
   RI_LAST,
   RI_FIRST = VERSION
 };
@@ -158,7 +171,8 @@ enum class FieldId {
   F_type,
   F_child_namespace,
   F_child_record,
-  F_concept
+  F_concept,
+  F_friend
 };
 
 class ClangDocBitcodeWriter {
@@ -190,6 +204,8 @@ class ClangDocBitcodeWriter {
   void emitBlock(const ConceptInfo &T);
   void emitBlock(const ConstraintInfo &T);
   void emitBlock(const Reference &B, FieldId F);
+  void emitBlock(const FriendInfo &R);
+  void emitBlock(const VarInfo &B);
 
 private:
   class AbbreviationMap {
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index 935bbfee7a9b..8294ff911855 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -986,6 +986,8 @@ llvm::Error HTMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
         genHTML(*static_cast<clang::doc::TypedefInfo *>(I), CDCtx, InfoTitle);
     break;
   case InfoType::IT_concept:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
@@ -1015,6 +1017,10 @@ static std::string getRefType(InfoType IT) {
     return "typedef";
   case InfoType::IT_concept:
     return "concept";
+  case InfoType::IT_variable:
+    return "variable";
+  case InfoType::IT_friend:
+    return "friend";
   }
   llvm_unreachable("Unknown InfoType");
 }
diff --git a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index 81ba99c21e37..7aeaa1b7cf67 100644
--- a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -587,6 +587,9 @@ Error MustacheHTMLGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
     break;
   case InfoType::IT_concept:
     break;
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
+    break;
   case InfoType::IT_default:
     return createStringError(inconvertibleErrorCode(), "unexpected InfoType");
   }
diff --git a/clang-tools-extra/clang-doc/JSONGenerator.cpp b/clang-tools-extra/clang-doc/JSONGenerator.cpp
index 8a37621597c6..0e1a0cc347e4 100644
--- a/clang-tools-extra/clang-doc/JSONGenerator.cpp
+++ b/clang-tools-extra/clang-doc/JSONGenerator.cpp
@@ -22,22 +22,30 @@ class JSONGenerator : public Generator {
 
 const char *JSONGenerator::Format = "json";
 
-static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl);
-static void serializeInfo(const EnumInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl);
 static void serializeInfo(const ConstraintInfo &I, Object &Obj);
+static void serializeInfo(const RecordInfo &I, Object &Obj,
+                          const std::optional<StringRef> &RepositoryUrl);
+
+static void serializeReference(const Reference &Ref, Object &ReferenceObj);
+
+template <typename Container, typename SerializationFunc>
+static void serializeArray(const Container &Records, Object &Obj,
+                           const std::string &Key,
+                           SerializationFunc SerializeInfo);
 
 // Convenience lambda to pass to serializeArray.
 // If a serializeInfo needs a RepositoryUrl, create a local lambda that captures
 // the optional.
-static auto SerializeInfoLambda = [](const ConstraintInfo &Info,
-                                     Object &Object) {
+static auto SerializeInfoLambda = [](const auto &Info, Object &Object) {
   serializeInfo(Info, Object);
 };
+static auto SerializeReferenceLambda = [](const auto &Ref, Object &Object) {
+  serializeReference(Ref, Object);
+};
 
-static json::Object serializeLocation(const Location &Loc,
-                                      std::optional<StringRef> RepositoryUrl) {
+static json::Object
+serializeLocation(const Location &Loc,
+                  const std::optional<StringRef> &RepositoryUrl) {
   Object LocationObj = Object();
   LocationObj["LineNumber"] = Loc.StartLineNumber;
   LocationObj["Filename"] = Loc.Filename;
@@ -159,8 +167,9 @@ static json::Value serializeComment(const CommentInfo &I) {
   llvm_unreachable("Unknown comment kind encountered.");
 }
 
-static void serializeCommonAttributes(const Info &I, json::Object &Obj,
-                                      std::optional<StringRef> RepositoryUrl) {
+static void
+serializeCommonAttributes(const Info &I, json::Object &Obj,
+                          const std::optional<StringRef> &RepositoryUrl) {
   Obj["Name"] = I.Name;
   Obj["USR"] = toHex(toStringRef(I.USR));
 
@@ -198,67 +207,28 @@ static void serializeReference(const Reference &Ref, Object &ReferenceObj) {
   ReferenceObj["USR"] = toHex(toStringRef(Ref.USR));
 }
 
-static void serializeReference(const SmallVector<Reference, 4> &References,
-                               Object &Obj, std::string Key) {
-  json::Value ReferencesArray = Array();
-  json::Array &ReferencesArrayRef = *ReferencesArray.getAsArray();
-  ReferencesArrayRef.reserve(References.size());
-  for (const auto &Reference : References) {
-    json::Value ReferenceVal = Object();
-    auto &ReferenceObj = *ReferenceVal.getAsObject();
-    serializeReference(Reference, ReferenceObj);
-    ReferencesArrayRef.push_back(ReferenceVal);
-  }
-  Obj[Key] = ReferencesArray;
-}
-
 // Although namespaces and records both have ScopeChildren, they serialize them
 // differently. Only enums, records, and typedefs are handled here.
-static void serializeCommonChildren(const ScopeChildren &Children,
-                                    json::Object &Obj,
-                                    std::optional<StringRef> RepositoryUrl) {
-  if (!Children.Enums.empty()) {
-    json::Value EnumsArray = Array();
-    auto &EnumsArrayRef = *EnumsArray.getAsArray();
-    EnumsArrayRef.reserve(Children.Enums.size());
-    for (const auto &Enum : Children.Enums) {
-      json::Value EnumVal = Object();
-      auto &EnumObj = *EnumVal.getAsObject();
-      serializeInfo(Enum, EnumObj, RepositoryUrl);
-      EnumsArrayRef.push_back(EnumVal);
-    }
-    Obj["Enums"] = EnumsArray;
-  }
+static void
+serializeCommonChildren(const ScopeChildren &Children, json::Object &Obj,
+                        const std::optional<StringRef> &RepositoryUrl) {
+  static auto SerializeInfo = [&RepositoryUrl](const auto &Info,
+                                               Object &Object) {
+    serializeInfo(Info, Object, RepositoryUrl);
+  };
 
-  if (!Children.Typedefs.empty()) {
-    json::Value TypedefsArray = Array();
-    auto &TypedefsArrayRef = *TypedefsArray.getAsArray();
-    TypedefsArrayRef.reserve(Children.Typedefs.size());
-    for (const auto &Typedef : Children.Typedefs) {
-      json::Value TypedefVal = Object();
-      auto &TypedefObj = *TypedefVal.getAsObject();
-      serializeInfo(Typedef, TypedefObj, RepositoryUrl);
-      TypedefsArrayRef.push_back(TypedefVal);
-    }
-    Obj["Typedefs"] = TypedefsArray;
-  }
+  if (!Children.Enums.empty())
+    serializeArray(Children.Enums, Obj, "Enums", SerializeInfo);
 
-  if (!Children.Records.empty()) {
-    json::Value RecordsArray = Array();
-    auto &RecordsArrayRef = *RecordsArray.getAsArray();
-    RecordsArrayRef.reserve(Children.Records.size());
-    for (const auto &Record : Children.Records) {
-      json::Value RecordVal = Object();
-      auto &RecordObj = *RecordVal.getAsObject();
-      serializeReference(Record, RecordObj);
-      RecordsArrayRef.push_back(RecordVal);
-    }
-    Obj["Records"] = RecordsArray;
-  }
+  if (!Children.Typedefs.empty())
+    serializeArray(Children.Typedefs, Obj, "Typedefs", SerializeInfo);
+
+  if (!Children.Records.empty())
+    serializeArray(Children.Records, Obj, "Records", SerializeReferenceLambda);
 }
 
-template <typename T, typename SerializationFunc>
-static void serializeArray(const std::vector<T> &Records, Object &Obj,
+template <typename Container, typename SerializationFunc>
+static void serializeArray(const Container &Records, Object &Obj,
                            const std::string &Key,
                            SerializationFunc SerializeInfo) {
   json::Value RecordsArray = Array();
@@ -278,6 +248,16 @@ static void serializeInfo(const ConstraintInfo &I, Object &Obj) {
   Obj["Expression"] = I.ConstraintExpr;
 }
 
+static void serializeInfo(const ArrayRef<TemplateParamInfo> &Params,
+                          Object &Obj) {
+  json::Value ParamsArray = Array();
+  auto &ParamsArrayRef = *ParamsArray.getAsArray();
+  ParamsArrayRef.reserve(Params.size());
+  for (const auto &Param : Params)
+    ParamsArrayRef.push_back(Param.Contents);
+  Obj["Parameters"] = ParamsArray;
+}
+
 static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
   json::Value TemplateVal = Object();
   auto &TemplateObj = *TemplateVal.getAsObject();
@@ -287,25 +267,13 @@ static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
     auto &TemplateSpecializationObj = *TemplateSpecializationVal.getAsObject();
     TemplateSpecializationObj["SpecializationOf"] =
         toHex(toStringRef(Template.Specialization->SpecializationOf));
-    if (!Template.Specialization->Params.empty()) {
-      json::Value ParamsArray = Array();
-      auto &ParamsArrayRef = *ParamsArray.getAsArray();
-      ParamsArrayRef.reserve(Template.Specialization->Params.size());
-      for (const auto &Param : Template.Specialization->Params)
-        ParamsArrayRef.push_back(Param.Contents);
-      TemplateSpecializationObj["Parameters"] = ParamsArray;
-    }
+    if (!Template.Specialization->Params.empty())
+      serializeInfo(Template.Specialization->Params, TemplateSpecializationObj);
     TemplateObj["Specialization"] = TemplateSpecializationVal;
   }
 
-  if (!Template.Params.empty()) {
-    json::Value ParamsArray = Array();
-    auto &ParamsArrayRef = *ParamsArray.getAsArray();
-    ParamsArrayRef.reserve(Template.Params.size());
-    for (const auto &Param : Template.Params)
-      ParamsArrayRef.push_back(Param.Contents);
-    TemplateObj["Parameters"] = ParamsArray;
-  }
+  if (!Template.Params.empty())
+    serializeInfo(Template.Params, TemplateObj);
 
   if (!Template.Constraints.empty())
     serializeArray(Template.Constraints, TemplateObj, "Constraints",
@@ -315,7 +283,7 @@ static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
 }
 
 static void serializeInfo(const ConceptInfo &I, Object &Obj,
-                          std::optional<StringRef> RepositoryUrl) {
+                          const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
   Obj["IsType"] = I.IsType;
   Obj["ConstraintExpression"] = I.ConstraintExpression;
@@ -330,8 +298,13 @@ static void serializeInfo(const TypeInfo &I, Object &Obj) {
   Obj["IsBuiltIn"] = I.IsBuiltIn;
 }
 
+static void serializeInfo(const FieldTypeInfo &I, Object &Obj) {
+  Obj["Name"] = I.Name;
+  Obj["Type"] = I.Type.Name;
+}
+
 static void serializeInfo(const FunctionInfo &F, json::Object &Obj,
-                          std::optional<StringRef> RepositoryURL) {
+                          const std::optional<StringRef> &RepositoryURL) {
   serializeCommonAttributes(F, Obj, RepositoryURL);
   Obj["IsStatic"] = F.IsStatic;
 
@@ -339,26 +312,23 @@ static void serializeInfo(const FunctionInfo &F, json::Object &Obj,
   serializeInfo(F.ReturnType, ReturnTypeObj);
   Obj["ReturnType"] = std::move(ReturnTypeObj);
 
-  if (!F.Params.empty()) {
-    json::Value ParamsArray = json::Array();
-    auto &ParamsArrayRef = *ParamsArray.getAsArray();
-    ParamsArrayRef.reserve(F.Params.size());
-    for (const auto &Param : F.Params) {
-      json::Value ParamVal = Object();
-      auto &ParamObj = *ParamVal.getAsObject();
-      ParamObj["Name"] = Param.Name;
-      ParamObj["Type"] = Param.Type.Name;
-      ParamsArrayRef.push_back(ParamVal);
-    }
-    Obj["Params"] = ParamsArray;
-  }
+  if (!F.Params.empty())
+    serializeArray(F.Params, Obj, "Params", SerializeInfoLambda);
 
   if (F.Template)
     serializeInfo(F.Template.value(), Obj);
 }
 
+static void serializeInfo(const EnumValueInfo &I, Object &Obj) {
+  Obj["Name"] = I.Name;
+  if (!I.ValueExpr.empty())
+    Obj["ValueExpr"] = I.ValueExpr;
+  else
+    Obj["Value"] = I.Value;
+}
+
 static void serializeInfo(const EnumInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl) {
+                          const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
   Obj["Scoped"] = I.Scoped;
 
@@ -371,26 +341,12 @@ static void serializeInfo(const EnumInfo &I, json::Object &Obj,
     Obj["BaseType"] = BaseTypeVal;
   }
 
-  if (!I.Members.empty()) {
-    json::Value MembersArray = Array();
-    auto &MembersArrayRef = *MembersArray.getAsArray();
-    MembersArrayRef.reserve(I.Members.size());
-    for (const auto &Member : I.Members) {
-      json::Value MemberVal = Object();
-      auto &MemberObj = *MemberVal.getAsObject();
-      MemberObj["Name"] = Member.Name;
-      if (!Member.ValueExpr.empty())
-        MemberObj["ValueExpr"] = Member.ValueExpr;
-      else
-        MemberObj["Value"] = Member.Value;
-      MembersArrayRef.push_back(MemberVal);
-    }
-    Obj["Members"] = MembersArray;
-  }
+  if (!I.Members.empty())
+    serializeArray(I.Members, Obj, "Members", SerializeInfoLambda);
 }
 
 static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl) {
+                          const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
   Obj["TypeDeclaration"] = I.TypeDeclaration;
   Obj["IsUsing"] = I.IsUsing;
@@ -400,8 +356,32 @@ static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
   Obj["Underlying"] = TypeVal;
 }
 
+static void serializeInfo(const BaseRecordInfo &I, Object &Obj,
+                          const std::optional<StringRef> &RepositoryUrl) {
+  serializeInfo(static_cast<const RecordInfo &>(I), Obj, RepositoryUrl);
+  Obj["IsVirtual"] = I.IsVirtual;
+  Obj["Access"] = getAccessSpelling(I.Access);
+  Obj["IsParent"] = I.IsParent;
+}
+
+static void serializeInfo(const FriendInfo &I, Object &Obj) {
+  auto FriendRef = Object();
+  serializeReference(I.Ref, FriendRef);
+  Obj["Reference"] = std::move(FriendRef);
+  Obj["IsClass"] = I.IsClass;
+  if (I.Template)
+    serializeInfo(I.Template.value(), Obj);
+  if (I.Params)
+    serializeArray(I.Params.value(), Obj, "Params", SerializeInfoLambda);
+  if (I.ReturnType) {
+    auto ReturnTypeObj = Object();
+    serializeInfo(I.ReturnType.value(), ReturnTypeObj);
+    Obj["ReturnType"] = std::move(ReturnTypeObj);
+  }
+}
+
 static void serializeInfo(const RecordInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl) {
+                          const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
   Obj["FullName"] = I.FullName;
   Obj["TagType"] = getTagType(I.TagType);
@@ -454,71 +434,60 @@ static void serializeInfo(const RecordInfo &I, json::Object &Obj,
       Obj["ProtectedMembers"] = ProtectedMembersArray;
   }
 
-  if (!I.Bases.empty()) {
-    json::Value BasesArray = Array();
-    json::Array &BasesArrayRef = *BasesArray.getAsArray();
-    BasesArrayRef.reserve(I.Bases.size());
-    for (const auto &BaseInfo : I.Bases) {
-      json::Value BaseInfoVal = Object();
-      auto &BaseInfoObj = *BaseInfoVal.getAsObject();
-      serializeInfo(BaseInfo, BaseInfoObj, RepositoryUrl);
-      BaseInfoObj["IsVirtual"] = BaseInfo.IsVirtual;
-      BaseInfoObj["Access"] = getAccessSpelling(BaseInfo.Access);
-      BaseInfoObj["IsParent"] = BaseInfo.IsParent;
-      BasesArrayRef.push_back(BaseInfoVal);
-    }
-    Obj["Bases"] = BasesArray;
-  }
+  if (!I.Bases.empty())
+    serializeArray(
+        I.Bases, Obj, "Bases",
+        [&RepositoryUrl](const BaseRecordInfo &Base, Object &BaseObj) {
+          serializeInfo(Base, BaseObj, RepositoryUrl);
+        });
 
   if (!I.Parents.empty())
-    serializeReference(I.Parents, Obj, "Parents");
+    serializeArray(I.Parents, Obj, "Parents", SerializeReferenceLambda);
 
   if (!I.VirtualParents.empty())
-    serializeReference(I.VirtualParents, Obj, "VirtualParents");
+    serializeArray(I.VirtualParents, Obj, "VirtualParents",
+                   SerializeReferenceLambda);
 
   if (I.Template)
     serializeInfo(I.Template.value(), Obj);
 
+  if (!I.Friends.empty())
+    serializeArray(I.Friends, Obj, "Friends", SerializeInfoLambda);
+
   serializeCommonChildren(I.Children, Obj, RepositoryUrl);
 }
 
+static void serializeInfo(const VarInfo &I, json::Object &Obj,
+                          const std::optional<StringRef> &RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+  Obj["IsStatic"] = I.IsStatic;
+  auto TypeObj = Object();
+  serializeInfo(I.Type, TypeObj);
+  Obj["Type"] = std::move(TypeObj);
+}
+
 static void serializeInfo(const NamespaceInfo &I, json::Object &Obj,
-                          std::optional<StringRef> RepositoryUrl) {
+                          const std::optional<StringRef> &RepositoryUrl) {
   serializeCommonAttributes(I, Obj, RepositoryUrl);
 
-  if (!I.Children.Namespaces.empty()) {
-    json::Value NamespacesArray = Array();
-    auto &NamespacesArrayRef = *NamespacesArray.getAsArray();
-    NamespacesArrayRef.reserve(I.Children.Namespaces.size());
-    for (auto &Namespace : I.Children.Namespaces) {
-      json::Value NamespaceVal = Object();
-      auto &NamespaceObj = *NamespaceVal.getAsObject();
-      serializeReference(Namespace, NamespaceObj);
-      NamespacesArrayRef.push_back(NamespaceVal);
-    }
-    Obj["Namespaces"] = NamespacesArray;
-  }
+  if (!I.Children.Namespaces.empty())
+    serializeArray(I.Children.Namespaces, Obj, "Namespaces",
+                   SerializeReferenceLambda);
 
-  auto SerializeInfo = [RepositoryUrl](const auto &Info, Object &Object) {
+  static auto SerializeInfo = [&RepositoryUrl](const auto &Info,
+                                               Object &Object) {
     serializeInfo(Info, Object, RepositoryUrl);
   };
 
-  if (!I.Children.Functions.empty()) {
-    json::Value FunctionsArray = Array();
-    auto &FunctionsArrayRef = *FunctionsArray.getAsArray();
-    FunctionsArrayRef.reserve(I.Children.Functions.size());
-    for (const auto &Function : I.Children.Functions) {
-      json::Value FunctionVal = Object();
-      auto &FunctionObj = *FunctionVal.getAsObject();
-      serializeInfo(Function, FunctionObj, RepositoryUrl);
-      FunctionsArrayRef.push_back(FunctionVal);
-    }
-    Obj["Functions"] = FunctionsArray;
-  }
+  if (!I.Children.Functions.empty())
+    serializeArray(I.Children.Functions, Obj, "Functions", SerializeInfo);
 
   if (!I.Children.Concepts.empty())
     serializeArray(I.Children.Concepts, Obj, "Concepts", SerializeInfo);
 
+  if (!I.Children.Variables.empty())
+    serializeArray(I.Children.Variables, Obj, "Variables", SerializeInfo);
+
   serializeCommonChildren(I.Children, Obj, RepositoryUrl);
 }
 
@@ -573,6 +542,8 @@ Error JSONGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
   case InfoType::IT_enum:
   case InfoType::IT_function:
   case InfoType::IT_typedef:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return createStringError(inconvertibleErrorCode(), "unexpected info type");
diff --git a/clang-tools-extra/clang-doc/MDGenerator.cpp b/clang-tools-extra/clang-doc/MDGenerator.cpp
index 6e68e09cfa2a..6f16f5bd2f52 100644
--- a/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -375,6 +375,12 @@ static llvm::Error genIndex(ClangDocContext &CDCtx) {
       case InfoType::IT_concept:
         Type = "Concept";
         break;
+      case InfoType::IT_variable:
+        Type = "Variable";
+        break;
+      case InfoType::IT_friend:
+        Type = "Friend";
+        break;
       case InfoType::IT_default:
         Type = "Other";
       }
@@ -468,6 +474,8 @@ llvm::Error MDGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     genMarkdown(CDCtx, *static_cast<clang::doc::TypedefInfo *>(I), OS);
     break;
   case InfoType::IT_concept:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return createStringError(llvm::inconvertibleErrorCode(),
diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp
index 6021e17b4696..497b80cf9746 100644
--- a/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/clang-tools-extra/clang-doc/Mapper.cpp
@@ -138,6 +138,12 @@ bool MapASTVisitor::VisitConceptDecl(const ConceptDecl *D) {
   return mapDecl(D, true);
 }
 
+bool MapASTVisitor::VisitVarDecl(const VarDecl *D) {
+  if (D->isCXXClassMember())
+    return true;
+  return mapDecl(D, D->isThisDeclarationADefinition());
+}
+
 comments::FullComment *
 MapASTVisitor::getComment(const NamedDecl *D, const ASTContext &Context) const {
   RawComment *Comment = Context.getRawCommentForDeclNoCache(D);
diff --git a/clang-tools-extra/clang-doc/Mapper.h b/clang-tools-extra/clang-doc/Mapper.h
index 04dc5450c8ba..322df6d594b3 100644
--- a/clang-tools-extra/clang-doc/Mapper.h
+++ b/clang-tools-extra/clang-doc/Mapper.h
@@ -42,6 +42,7 @@ class MapASTVisitor : public clang::RecursiveASTVisitor<MapASTVisitor>,
   bool VisitTypedefDecl(const TypedefDecl *D);
   bool VisitTypeAliasDecl(const TypeAliasDecl *D);
   bool VisitConceptDecl(const ConceptDecl *D);
+  bool VisitVarDecl(const VarDecl *D);
 
 private:
   template <typename T> bool mapDecl(const T *D, bool IsDefinition);
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 286aeeea1001..422a76d99e5b 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -145,6 +145,10 @@ mergeInfos(std::vector<std::unique_ptr<Info>> &Values) {
     return reduce<TypedefInfo>(Values);
   case InfoType::IT_concept:
     return reduce<ConceptInfo>(Values);
+  case InfoType::IT_variable:
+    return reduce<VarInfo>(Values);
+  case InfoType::IT_friend:
+    return reduce<FriendInfo>(Values);
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected info type");
@@ -245,6 +249,15 @@ void Reference::merge(Reference &&Other) {
     Path = Other.Path;
 }
 
+bool FriendInfo::mergeable(const FriendInfo &Other) {
+  return Ref.USR == Other.Ref.USR && Ref.Name == Other.Ref.Name;
+}
+
+void FriendInfo::merge(FriendInfo &&Other) {
+  assert(mergeable(Other));
+  Ref.merge(std::move(Other.Ref));
+}
+
 void Info::mergeBase(Info &&Other) {
   assert(mergeable(Other));
   if (USR == EmptySID)
@@ -291,6 +304,7 @@ void NamespaceInfo::merge(NamespaceInfo &&Other) {
   reduceChildren(Children.Enums, std::move(Other.Children.Enums));
   reduceChildren(Children.Typedefs, std::move(Other.Children.Typedefs));
   reduceChildren(Children.Concepts, std::move(Other.Children.Concepts));
+  reduceChildren(Children.Variables, std::move(Other.Children.Variables));
   mergeBase(std::move(Other));
 }
 
@@ -310,6 +324,8 @@ void RecordInfo::merge(RecordInfo &&Other) {
     Parents = std::move(Other.Parents);
   if (VirtualParents.empty())
     VirtualParents = std::move(Other.VirtualParents);
+  if (Friends.empty())
+    Friends = std::move(Other.Friends);
   // Reduce children if necessary.
   reduceChildren(Children.Records, std::move(Other.Children.Records));
   reduceChildren(Children.Functions, std::move(Other.Children.Functions));
@@ -368,6 +384,15 @@ void ConceptInfo::merge(ConceptInfo &&Other) {
   SymbolInfo::merge(std::move(Other));
 }
 
+void VarInfo::merge(VarInfo &&Other) {
+  assert(mergeable(Other));
+  if (!IsStatic)
+    IsStatic = Other.IsStatic;
+  if (Type.Type.USR == EmptySID && Type.Type.Name == "")
+    Type = std::move(Other.Type);
+  SymbolInfo::merge(std::move(Other));
+}
+
 BaseRecordInfo::BaseRecordInfo() : RecordInfo() {}
 
 BaseRecordInfo::BaseRecordInfo(SymbolID USR, StringRef Name, StringRef Path,
@@ -407,6 +432,12 @@ llvm::SmallString<16> Info::extractName() const {
   case InfoType::IT_concept:
     return llvm::SmallString<16>("@nonymous_concept_" +
                                  toHex(llvm::toStringRef(USR)));
+  case InfoType::IT_variable:
+    return llvm::SmallString<16>("@nonymous_variable_" +
+                                 toHex(llvm::toStringRef(USR)));
+  case InfoType::IT_friend:
+    return llvm::SmallString<16>("@nonymous_friend_" +
+                                 toHex(llvm::toStringRef(USR)));
   case InfoType::IT_default:
     return llvm::SmallString<16>("@nonymous_" + toHex(llvm::toStringRef(USR)));
   }
@@ -473,6 +504,7 @@ void ScopeChildren::sort() {
   llvm::sort(Enums.begin(), Enums.end());
   llvm::sort(Typedefs.begin(), Typedefs.end());
   llvm::sort(Concepts.begin(), Concepts.end());
+  llvm::sort(Variables.begin(), Variables.end());
 }
 } // namespace doc
 } // namespace clang
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index b23069f2bd32..fe5cc48069d5 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -36,6 +36,7 @@ struct FunctionInfo;
 struct Info;
 struct TypedefInfo;
 struct ConceptInfo;
+struct VarInfo;
 
 enum class InfoType {
   IT_default,
@@ -44,7 +45,9 @@ enum class InfoType {
   IT_function,
   IT_enum,
   IT_typedef,
-  IT_concept
+  IT_concept,
+  IT_variable,
+  IT_friend
 };
 
 enum class CommentKind {
@@ -169,6 +172,7 @@ struct ScopeChildren {
   std::vector<EnumInfo> Enums;
   std::vector<TypedefInfo> Typedefs;
   std::vector<ConceptInfo> Concepts;
+  std::vector<VarInfo> Variables;
 
   void sort();
 };
@@ -376,6 +380,31 @@ struct SymbolInfo : public Info {
   bool IsStatic = false;
 };
 
+struct FriendInfo : SymbolInfo {
+  FriendInfo() : SymbolInfo(InfoType::IT_friend) {}
+  FriendInfo(SymbolID USR) : SymbolInfo(InfoType::IT_friend, USR) {}
+  FriendInfo(const InfoType IT, const SymbolID &USR,
+             const StringRef Name = StringRef())
+      : SymbolInfo(IT, USR, Name) {}
+  bool mergeable(const FriendInfo &Other);
+  void merge(FriendInfo &&Other);
+
+  Reference Ref;
+  std::optional<TemplateInfo> Template;
+  std::optional<TypeInfo> ReturnType;
+  std::optional<SmallVector<FieldTypeInfo, 4>> Params;
+  bool IsClass = false;
+};
+
+struct VarInfo : SymbolInfo {
+  VarInfo() : SymbolInfo(InfoType::IT_variable) {}
+  explicit VarInfo(SymbolID USR) : SymbolInfo(InfoType::IT_variable, USR) {}
+
+  void merge(VarInfo &&I);
+
+  TypeInfo Type;
+};
+
 // TODO: Expand to allow for documenting templating and default args.
 // Info for functions.
 struct FunctionInfo : public SymbolInfo {
@@ -442,6 +471,8 @@ struct RecordInfo : public SymbolInfo {
       Bases; // List of base/parent records; this includes inherited methods and
              // attributes
 
+  std::vector<FriendInfo> Friends;
+
   ScopeChildren Children;
 };
 
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 5f3e5c37fa34..6cc372ce98a6 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -8,8 +8,10 @@
 
 #include "Serialize.h"
 #include "BitcodeWriter.h"
+
 #include "clang/AST/Attr.h"
 #include "clang/AST/Comment.h"
+#include "clang/AST/DeclFriend.h"
 #include "clang/Index/USRGeneration.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/StringExtras.h"
@@ -401,6 +403,9 @@ std::string serialize(std::unique_ptr<Info> &I) {
     return serialize(*static_cast<FunctionInfo *>(I.get()));
   case InfoType::IT_concept:
     return serialize(*static_cast<ConceptInfo *>(I.get()));
+  case InfoType::IT_variable:
+    return serialize(*static_cast<VarInfo *>(I.get()));
+  case InfoType::IT_friend:
   case InfoType::IT_typedef:
   case InfoType::IT_default:
     return "";
@@ -508,6 +513,10 @@ static void InsertChild(ScopeChildren &Scope, ConceptInfo Info) {
   Scope.Concepts.push_back(std::move(Info));
 }
 
+static void InsertChild(ScopeChildren &Scope, VarInfo Info) {
+  Scope.Variables.push_back(std::move(Info));
+}
+
 // Creates a parent of the correct type for the given child and inserts it into
 // that parent.
 //
@@ -549,6 +558,8 @@ static std::unique_ptr<Info> makeAndInsertIntoParent(ChildType Child) {
   case InfoType::IT_function:
   case InfoType::IT_typedef:
   case InfoType::IT_concept:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   }
   llvm_unreachable("Invalid reference type for parent namespace");
@@ -940,6 +951,55 @@ emitInfo(const NamespaceDecl *D, const FullComment *FC, Location Loc,
   return {std::move(NSI), makeAndInsertIntoParent<const NamespaceInfo &>(*NSI)};
 }
 
+static void parseFriends(RecordInfo &RI, const CXXRecordDecl *D) {
+  if (!D->hasDefinition() || !D->hasFriends())
+    return;
+
+  for (const FriendDecl *FD : D->friends()) {
+    if (FD->isUnsupportedFriend())
+      continue;
+
+    FriendInfo F(InfoType::IT_friend, getUSRForDecl(FD));
+    const auto *ActualDecl = FD->getFriendDecl();
+    if (!ActualDecl) {
+      const auto *FriendTypeInfo = FD->getFriendType();
+      if (!FriendTypeInfo)
+        continue;
+      ActualDecl = FriendTypeInfo->getType()->getAsCXXRecordDecl();
+
+      if (!ActualDecl)
+        continue;
+      F.IsClass = true;
+    }
+
+    if (const auto *ActualTD = dyn_cast_or_null<TemplateDecl>(ActualDecl)) {
+      if (isa<RecordDecl>(ActualTD->getTemplatedDecl()))
+        F.IsClass = true;
+      F.Template.emplace();
+      for (const auto *Param : ActualTD->getTemplateParameters()->asArray())
+        F.Template->Params.emplace_back(
+            getSourceCode(Param, Param->getSourceRange()));
+      ActualDecl = ActualTD->getTemplatedDecl();
+    }
+
+    if (auto *FuncDecl = dyn_cast_or_null<FunctionDecl>(ActualDecl)) {
+      FunctionInfo TempInfo;
+      parseParameters(TempInfo, FuncDecl);
+      F.Params.emplace();
+      F.Params = std::move(TempInfo.Params);
+      F.ReturnType = getTypeInfoForType(FuncDecl->getReturnType(),
+                                        FuncDecl->getLangOpts());
+    }
+
+    F.Ref =
+        Reference(getUSRForDecl(ActualDecl), ActualDecl->getNameAsString(),
+                  InfoType::IT_default, ActualDecl->getQualifiedNameAsString(),
+                  getInfoRelativePath(ActualDecl));
+
+    RI.Friends.push_back(std::move(F));
+  }
+}
+
 std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const RecordDecl *D, const FullComment *FC, Location Loc,
          bool PublicOnly) {
@@ -963,6 +1023,7 @@ emitInfo(const RecordDecl *D, const FullComment *FC, Location Loc,
     // TODO: remove first call to parseBases, that function should be deleted
     parseBases(*RI, C);
     parseBases(*RI, C, /*IsFileInRootDir=*/true, PublicOnly, /*IsParent=*/true);
+    parseFriends(*RI, C);
   }
   RI->Path = getInfoRelativePath(RI->Namespace);
 
@@ -1164,6 +1225,26 @@ emitInfo(const ConceptDecl *D, const FullComment *FC, const Location &Loc,
   return {nullptr, makeAndInsertIntoParent<ConceptInfo &&>(std::move(Concept))};
 }
 
+std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
+emitInfo(const VarDecl *D, const FullComment *FC, const Location &Loc,
+         bool PublicOnly) {
+  VarInfo Var;
+  bool IsInAnonymousNamespace = false;
+  populateSymbolInfo(Var, D, FC, Loc, IsInAnonymousNamespace);
+  if (!shouldSerializeInfo(PublicOnly, IsInAnonymousNamespace, D))
+    return {};
+
+  if (D->getStorageClass() == StorageClass::SC_Static)
+    Var.IsStatic = true;
+  Var.Type =
+      getTypeInfoForType(D->getType(), D->getASTContext().getPrintingPolicy());
+
+  if (!shouldSerializeInfo(PublicOnly, IsInAnonymousNamespace, D))
+    return {};
+
+  return {nullptr, makeAndInsertIntoParent<VarInfo &&>(std::move(Var))};
+}
+
 } // namespace serialize
 } // namespace doc
 } // namespace clang
diff --git a/clang-tools-extra/clang-doc/Serialize.h b/clang-tools-extra/clang-doc/Serialize.h
index 497b09bb339f..06c4d64c5149 100644
--- a/clang-tools-extra/clang-doc/Serialize.h
+++ b/clang-tools-extra/clang-doc/Serialize.h
@@ -72,6 +72,10 @@ std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const ConceptDecl *D, const FullComment *FC, const Location &Loc,
          bool PublicOnly);
 
+std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
+emitInfo(const VarDecl *D, const FullComment *FC, const Location &Loc,
+         bool PublicOnly);
+
 // Function to hash a given USR value for storage.
 // As USRs (Unified Symbol Resolution) could be large, especially for functions
 // with long type arguments, we use 160-bits SHA1(USR) values to
diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index f95887104698..eeccdd804b66 100644
--- a/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -409,6 +409,8 @@ llvm::Error YAMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     InfoYAML << *static_cast<clang::doc::TypedefInfo *>(I);
     break;
   case InfoType::IT_concept:
+  case InfoType::IT_variable:
+  case InfoType::IT_friend:
     break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index a71813a103df..c35f0b941c60 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -358,8 +358,27 @@ getFixIt(const tooling::Diagnostic &Diagnostic, bool AnyFix) {
 
 } // namespace clang::tidy
 
+void ClangTidyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
+                                                  const Preprocessor *PP) {
+  DiagnosticConsumer::BeginSourceFile(LangOpts, PP);
+
+  assert(!InSourceFile);
+  InSourceFile = true;
+}
+
+void ClangTidyDiagnosticConsumer::EndSourceFile() {
+  assert(InSourceFile);
+  InSourceFile = false;
+
+  DiagnosticConsumer::EndSourceFile();
+}
+
 void ClangTidyDiagnosticConsumer::HandleDiagnostic(
     DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
+  // A diagnostic should not be reported outside of a
+  // BeginSourceFile()/EndSourceFile() pair if it has a source location.
+  assert(InSourceFile || Info.getLocation().isInvalid());
+
   if (LastErrorWasIgnored && DiagLevel == DiagnosticsEngine::Note)
     return;
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index a8851e794f24..6e7cb7bb10e5 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -292,6 +292,11 @@ class ClangTidyDiagnosticConsumer : public DiagnosticConsumer {
   void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                         const Diagnostic &Info) override;
 
+  void BeginSourceFile(const LangOptions &LangOpts,
+                       const Preprocessor *PP = nullptr) override;
+
+  void EndSourceFile() override;
+
   // Retrieve the diagnostics that were captured.
   std::vector<ClangTidyError> take();
 
@@ -326,6 +331,11 @@ class ClangTidyDiagnosticConsumer : public DiagnosticConsumer {
   bool LastErrorRelatesToUserCode = false;
   bool LastErrorPassesLineFilter = false;
   bool LastErrorWasIgnored = false;
+  /// Tracks whether we're currently inside a
+  /// `BeginSourceFile()/EndSourceFile()` pair. Outside of a source file, we
+  /// should only receive diagnostics that have to source location, such as
+  /// command-line warnings.
+  bool InSourceFile = false;
 };
 
 } // end namespace tidy
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
index 4aba5831e677..23de8d971898 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisleadingSetterOfReferenceCheck.cpp
@@ -50,6 +50,8 @@ void MisleadingSetterOfReferenceCheck::check(
     const MatchFinder::MatchResult &Result) {
   const auto *Found = Result.Nodes.getNodeAs<CXXMethodDecl>("bad-set-function");
   const auto *Member = Result.Nodes.getNodeAs<FieldDecl>("member");
+  assert(Found != nullptr);
+  assert(Member != nullptr);
 
   diag(Found->getBeginLoc(),
        "function '%0' can be mistakenly used in order to change the "
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
index 9eeba867f521..88d2f2c388d0 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
@@ -72,7 +72,9 @@ SizeofExpressionCheck::SizeofExpressionCheck(StringRef Name,
           Options.get("WarnOnSizeOfPointerToAggregate", true)),
       WarnOnSizeOfPointer(Options.get("WarnOnSizeOfPointer", false)),
       WarnOnOffsetDividedBySizeOf(
-          Options.get("WarnOnOffsetDividedBySizeOf", true)) {}
+          Options.get("WarnOnOffsetDividedBySizeOf", true)),
+      WarnOnSizeOfInLoopTermination(
+          Options.get("WarnOnSizeOfInLoopTermination", true)) {}
 
 void SizeofExpressionCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "WarnOnSizeOfConstant", WarnOnSizeOfConstant);
@@ -86,6 +88,8 @@ void SizeofExpressionCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "WarnOnSizeOfPointer", WarnOnSizeOfPointer);
   Options.store(Opts, "WarnOnOffsetDividedBySizeOf",
                 WarnOnOffsetDividedBySizeOf);
+  Options.store(Opts, "WarnOnSizeOfInLoopTermination",
+                WarnOnSizeOfInLoopTermination);
 }
 
 void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
@@ -93,6 +97,13 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
   // Some of the checks should not match in template code to avoid false
   // positives if sizeof is applied on template argument.
 
+  auto LoopCondExpr =
+      [](const ast_matchers::internal::Matcher<Stmt> &InnerMatcher) {
+        return stmt(anyOf(forStmt(hasCondition(InnerMatcher)),
+                          whileStmt(hasCondition(InnerMatcher)),
+                          doStmt(hasCondition(InnerMatcher))));
+      };
+
   const auto IntegerExpr = ignoringParenImpCasts(integerLiteral());
   const auto ConstantExpr = ignoringParenImpCasts(
       anyOf(integerLiteral(), unaryOperator(hasUnaryOperand(IntegerExpr)),
@@ -130,6 +141,14 @@ void SizeofExpressionCheck::registerMatchers(MatchFinder *Finder) {
                        this);
   }
 
+  if (WarnOnSizeOfInLoopTermination) {
+    auto CondExpr = binaryOperator(
+        allOf(has(SizeOfExpr.bind("sizeof-expr")), isComparisonOperator()));
+    Finder->addMatcher(LoopCondExpr(anyOf(CondExpr, hasDescendant(CondExpr)))
+                           .bind("loop-expr"),
+                       this);
+  }
+
   // Detect sizeof(kPtr) where kPtr is 'const char* kPtr = "abc"';
   const auto CharPtrType = pointerType(pointee(isAnyCharacter()));
   const auto ConstStrLiteralDecl =
@@ -349,6 +368,23 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
     diag(E->getBeginLoc(),
          "suspicious usage of 'sizeof(char*)'; do you mean 'strlen'?")
         << E->getSourceRange();
+  } else if (Result.Nodes.getNodeAs<Stmt>("loop-expr")) {
+    auto *SizeofArgTy = Result.Nodes.getNodeAs<Type>("sizeof-arg-type");
+    if (const auto member = dyn_cast<MemberPointerType>(SizeofArgTy))
+      SizeofArgTy = member->getPointeeType().getTypePtr();
+
+    const auto *SzOfExpr = Result.Nodes.getNodeAs<Expr>("sizeof-expr");
+
+    if (const auto type = dyn_cast<ArrayType>(SizeofArgTy)) {
+      // check if the array element size is larger than one. If true,
+      // the size of the array is higher than the number of elements
+      CharUnits sSize = Ctx.getTypeSizeInChars(type->getElementType());
+      if (!sSize.isOne()) {
+        diag(SzOfExpr->getBeginLoc(),
+             "suspicious usage of 'sizeof' in the loop")
+            << SzOfExpr->getSourceRange();
+      }
+    }
   } else if (const auto *E = Result.Nodes.getNodeAs<Expr>("sizeof-pointer")) {
     diag(E->getBeginLoc(), "suspicious usage of 'sizeof()' on an expression "
                            "of pointer type")
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
index fbd62cb80fb2..e979b4723cf2 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.h
@@ -32,6 +32,7 @@ class SizeofExpressionCheck : public ClangTidyCheck {
   const bool WarnOnSizeOfPointerToAggregate;
   const bool WarnOnSizeOfPointer;
   const bool WarnOnOffsetDividedBySizeOf;
+  const bool WarnOnSizeOfInLoopTermination;
 };
 
 } // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
index cf299609e646..268b51f76a2c 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.cpp
@@ -120,7 +120,7 @@ void MissingStdForwardCheck::registerMatchers(MatchFinder *Finder) {
                           equalsBoundNode("param"), equalsBoundNode("var")))))),
                 CapturedInLambda)),
       callee(unresolvedLookupExpr(hasAnyDeclaration(
-          namedDecl(hasUnderlyingDecl(hasName("::std::forward")))))),
+          namedDecl(hasUnderlyingDecl(hasName(ForwardFunction)))))),
 
       unless(anyOf(hasAncestor(typeLoc()),
                    hasAncestor(expr(hasUnevaluatedContext())))));
@@ -149,4 +149,13 @@ void MissingStdForwardCheck::check(const MatchFinder::MatchResult &Result) {
       << Param;
 }
 
+MissingStdForwardCheck::MissingStdForwardCheck(StringRef Name,
+                                               ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      ForwardFunction(Options.get("ForwardFunction", "::std::forward")) {}
+
+void MissingStdForwardCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "ForwardFunction", ForwardFunction);
+}
+
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
index 5995ad588855..f833b8031f8a 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/MissingStdForwardCheck.h
@@ -21,8 +21,7 @@ namespace clang::tidy::cppcoreguidelines {
 /// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/missing-std-forward.html
 class MissingStdForwardCheck : public ClangTidyCheck {
 public:
-  MissingStdForwardCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+  MissingStdForwardCheck(StringRef Name, ClangTidyContext *Context);
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
@@ -31,6 +30,10 @@ class MissingStdForwardCheck : public ClangTidyCheck {
   std::optional<TraversalKind> getCheckTraversalKind() const override {
     return TK_IgnoreUnlessSpelledInSource;
   }
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
+private:
+  const StringRef ForwardFunction;
 };
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
index a1494a095f5b..9ac7b9e057e3 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
@@ -15,16 +15,13 @@ using namespace clang::ast_matchers;
 namespace clang::tidy::cppcoreguidelines {
 
 void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
-  if (!getLangOpts().CPlusPlus)
-    return;
-
   const auto AllPointerTypes =
-      anyOf(hasType(pointerType()),
+      anyOf(hasType(hasUnqualifiedDesugaredType(pointerType())),
             hasType(autoType(
                 hasDeducedType(hasUnqualifiedDesugaredType(pointerType())))),
             hasType(decltypeType(hasUnderlyingType(pointerType()))));
 
-  // Flag all operators +, -, +=, -=, ++, -- that result in a pointer
+  // Flag all operators +, -, +=, -= that result in a pointer
   Finder->addMatcher(
       binaryOperator(
           hasAnyOperatorName("+", "-", "+=", "-="), AllPointerTypes,
@@ -32,8 +29,12 @@ void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
           .bind("expr"),
       this);
 
+  // Flag all operators ++, -- that result in a pointer
   Finder->addMatcher(
-      unaryOperator(hasAnyOperatorName("++", "--"), hasType(pointerType()))
+      unaryOperator(hasAnyOperatorName("++", "--"),
+                    hasType(hasUnqualifiedDesugaredType(pointerType())),
+                    unless(hasUnaryOperand(
+                        ignoringImpCasts(declRefExpr(to(isImplicit()))))))
           .bind("expr"),
       this);
 
@@ -42,7 +43,8 @@ void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
       arraySubscriptExpr(
           hasBase(ignoringImpCasts(
               anyOf(AllPointerTypes,
-                    hasType(decayedType(hasDecayedType(pointerType())))))))
+                    hasType(decayedType(hasDecayedType(pointerType())))))),
+          hasIndex(hasType(isInteger())))
           .bind("expr"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
index 67f7d1bf9dd6..3466c72a769e 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
@@ -23,6 +23,9 @@ class ProBoundsPointerArithmeticCheck : public ClangTidyCheck {
 public:
   ProBoundsPointerArithmeticCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
index 8c386d5bc794..272152644d7d 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.cpp
@@ -42,9 +42,9 @@ void RvalueReferenceParamNotMovedCheck::registerMatchers(MatchFinder *Finder) {
   StatementMatcher MoveCallMatcher =
       callExpr(
           argumentCountIs(1),
-          anyOf(callee(functionDecl(hasName("::std::move"))),
+          anyOf(callee(functionDecl(hasName(MoveFunction))),
                 callee(unresolvedLookupExpr(hasAnyDeclaration(
-                    namedDecl(hasUnderlyingDecl(hasName("::std::move"))))))),
+                    namedDecl(hasUnderlyingDecl(hasName(MoveFunction))))))),
           hasArgument(
               0, argumentOf(
                      AllowPartialMove,
@@ -122,7 +122,8 @@ RvalueReferenceParamNotMovedCheck::RvalueReferenceParamNotMovedCheck(
       AllowPartialMove(Options.get("AllowPartialMove", false)),
       IgnoreUnnamedParams(Options.get("IgnoreUnnamedParams", false)),
       IgnoreNonDeducedTemplateTypes(
-          Options.get("IgnoreNonDeducedTemplateTypes", false)) {}
+          Options.get("IgnoreNonDeducedTemplateTypes", false)),
+      MoveFunction(Options.get("MoveFunction", "::std::move")) {}
 
 void RvalueReferenceParamNotMovedCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
@@ -130,6 +131,7 @@ void RvalueReferenceParamNotMovedCheck::storeOptions(
   Options.store(Opts, "IgnoreUnnamedParams", IgnoreUnnamedParams);
   Options.store(Opts, "IgnoreNonDeducedTemplateTypes",
                 IgnoreNonDeducedTemplateTypes);
+  Options.store(Opts, "MoveFunction", MoveFunction);
 }
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
index d8c3d2bd4ba0..950c0206745d 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/RvalueReferenceParamNotMovedCheck.h
@@ -32,6 +32,7 @@ class RvalueReferenceParamNotMovedCheck : public ClangTidyCheck {
   const bool AllowPartialMove;
   const bool IgnoreUnnamedParams;
   const bool IgnoreNonDeducedTemplateTypes;
+  const StringRef MoveFunction;
 };
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
index 763947698052..79ae5ee98182 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.cpp
@@ -1,5 +1,4 @@
-//===--- ConfusableIdentifierCheck.cpp -
-// clang-tidy--------------------------===//
+//===--- ConfusableIdentifierCheck.cpp - clang-tidy -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,6 +8,7 @@
 
 #include "ConfusableIdentifierCheck.h"
 
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ConvertUTF.h"
@@ -88,99 +88,98 @@ static llvm::SmallString<64U> skeleton(StringRef Name) {
   return Skeleton;
 }
 
-static bool mayShadowImpl(const DeclContext *DC0, const DeclContext *DC1) {
-  return DC0 && DC0 == DC1;
-}
-
-static bool mayShadowImpl(const NamedDecl *ND0, const NamedDecl *ND1) {
-  return isa<TemplateTypeParmDecl>(ND0) || isa<TemplateTypeParmDecl>(ND1);
-}
-
-static bool isMemberOf(const ConfusableIdentifierCheck::ContextInfo *DC0,
-                       const ConfusableIdentifierCheck::ContextInfo *DC1) {
-  return llvm::is_contained(DC1->Bases, DC0->PrimaryContext);
-}
-
-static bool enclosesContext(const ConfusableIdentifierCheck::ContextInfo *DC0,
-                            const ConfusableIdentifierCheck::ContextInfo *DC1) {
-  if (DC0->PrimaryContext == DC1->PrimaryContext)
-    return true;
-
-  return llvm::is_contained(DC0->PrimaryContexts, DC1->PrimaryContext) ||
-         llvm::is_contained(DC1->PrimaryContexts, DC0->PrimaryContext);
-}
-
-static bool mayShadow(const NamedDecl *ND0,
-                      const ConfusableIdentifierCheck::ContextInfo *DC0,
-                      const NamedDecl *ND1,
-                      const ConfusableIdentifierCheck::ContextInfo *DC1) {
-
-  if (!DC0->Bases.empty() && !DC1->Bases.empty()) {
-    // if any of the declaration is a non-private member of the other
-    // declaration, it's shadowed by the former
-
-    if (ND1->getAccess() != AS_private && isMemberOf(DC1, DC0))
-      return true;
+namespace {
+struct Entry {
+  const NamedDecl *ND;
+  const Decl *Parent;
+  bool FromDerivedClass;
+};
+} // namespace
 
-    if (ND0->getAccess() != AS_private && isMemberOf(DC0, DC1))
+// Map from a context to the declarations in that context with the current
+// skeleton. At most one entry per distinct identifier is tracked. The
+// context is usually a `DeclContext`, but can also be a template declaration
+// that has no corresponding context, such as an alias template or variable
+// template.
+using DeclsWithinContextMap =
+    llvm::DenseMap<const Decl *, llvm::SmallVector<Entry, 1>>;
+
+static bool addToContext(DeclsWithinContextMap &DeclsWithinContext,
+                         const Decl *Context, Entry E) {
+  auto &Decls = DeclsWithinContext[Context];
+  if (!Decls.empty() &&
+      Decls.back().ND->getIdentifier() == E.ND->getIdentifier()) {
+    // Already have a declaration with this identifier in this context. Don't
+    // track another one. This means that if an outer name is confusable with an
+    // inner name, we'll only diagnose the outer name once, pointing at the
+    // first inner declaration with that name.
+    if (Decls.back().FromDerivedClass && !E.FromDerivedClass) {
+      // Prefer the declaration that's not from the derived class, because that
+      // conflicts with more declarations.
+      Decls.back() = E;
       return true;
-  }
-
-  if (!mayShadowImpl(DC0->NonTransparentContext, DC1->NonTransparentContext) &&
-      !mayShadowImpl(ND0, ND1))
+    }
     return false;
-
-  return enclosesContext(DC0, DC1);
+  }
+  Decls.push_back(E);
+  return true;
 }
 
-const ConfusableIdentifierCheck::ContextInfo *
-ConfusableIdentifierCheck::getContextInfo(const DeclContext *DC) {
-  const DeclContext *PrimaryContext = DC->getPrimaryContext();
-  auto [It, Inserted] = ContextInfos.try_emplace(PrimaryContext);
-  if (!Inserted)
-    return &It->second;
-
-  ContextInfo &Info = It->second;
-  Info.PrimaryContext = PrimaryContext;
-  Info.NonTransparentContext = PrimaryContext;
+static void addToEnclosingContexts(DeclsWithinContextMap &DeclsWithinContext,
+                                   const Decl *Parent, const NamedDecl *ND) {
+  const Decl *Outer = Parent;
+  while (Outer) {
+    if (const auto *NS = dyn_cast<NamespaceDecl>(Outer))
+      Outer = NS->getCanonicalDecl();
+
+    if (!addToContext(DeclsWithinContext, Outer, {ND, Parent, false}))
+      return;
+
+    if (const auto *RD = dyn_cast<CXXRecordDecl>(Outer)) {
+      RD = RD->getDefinition();
+      if (RD) {
+        RD->forallBases([&](const CXXRecordDecl *Base) {
+          addToContext(DeclsWithinContext, Base, {ND, Parent, true});
+          return true;
+        });
+      }
+    }
 
-  while (Info.NonTransparentContext->isTransparentContext()) {
-    Info.NonTransparentContext = Info.NonTransparentContext->getParent();
-    if (!Info.NonTransparentContext)
+    auto *OuterDC = Outer->getDeclContext();
+    if (!OuterDC)
       break;
+    Outer = cast_or_null<Decl>(OuterDC->getNonTransparentContext());
   }
+}
+
+void ConfusableIdentifierCheck::check(
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  const auto *ND = Result.Nodes.getNodeAs<NamedDecl>("nameddecl");
+  if (!ND)
+    return;
 
-  if (Info.NonTransparentContext)
-    Info.NonTransparentContext =
-        Info.NonTransparentContext->getPrimaryContext();
+  addDeclToCheck(ND,
+                 cast<Decl>(ND->getDeclContext()->getNonTransparentContext()));
 
-  while (DC) {
-    if (!isa<LinkageSpecDecl>(DC) && !isa<ExportDecl>(DC))
-      Info.PrimaryContexts.push_back(DC->getPrimaryContext());
-    DC = DC->getParent();
+  // Associate template parameters with this declaration of this template.
+  if (const auto *TD = dyn_cast<TemplateDecl>(ND)) {
+    for (const NamedDecl *Param : *TD->getTemplateParameters())
+      addDeclToCheck(Param, TD->getTemplatedDecl());
   }
 
-  if (const auto *RD = dyn_cast<CXXRecordDecl>(PrimaryContext)) {
-    RD = RD->getDefinition();
-    if (RD) {
-      Info.Bases.push_back(RD);
-      RD->forallBases([&](const CXXRecordDecl *Base) {
-        Info.Bases.push_back(Base);
-        return false;
-      });
-    }
+  // Associate function parameters with this declaration of this function.
+  if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
+    for (const NamedDecl *Param : FD->parameters())
+      addDeclToCheck(Param, ND);
   }
-
-  return &Info;
 }
 
-void ConfusableIdentifierCheck::check(
-    const ast_matchers::MatchFinder::MatchResult &Result) {
-  const auto *ND = Result.Nodes.getNodeAs<NamedDecl>("nameddecl");
-  if (!ND)
+void ConfusableIdentifierCheck::addDeclToCheck(const NamedDecl *ND,
+                                               const Decl *Parent) {
+  if (!ND || !Parent)
     return;
 
-  IdentifierInfo *NDII = ND->getIdentifier();
+  const IdentifierInfo *NDII = ND->getIdentifier();
   if (!NDII)
     return;
 
@@ -188,34 +187,78 @@ void ConfusableIdentifierCheck::check(
   if (NDName.empty())
     return;
 
-  const ContextInfo *Info = getContextInfo(ND->getDeclContext());
+  NameToDecls[NDII].push_back({ND, Parent});
+}
+
+void ConfusableIdentifierCheck::onEndOfTranslationUnit() {
+  llvm::StringMap<llvm::SmallVector<const IdentifierInfo *, 1>> SkeletonToNames;
+  // Compute the skeleton for each identifier.
+  for (auto &[Ident, Decls] : NameToDecls) {
+    SkeletonToNames[skeleton(Ident->getName())].push_back(Ident);
+  }
 
-  llvm::SmallVector<Entry> &Mapped = Mapper[skeleton(NDName)];
-  for (const Entry &E : Mapped) {
-    if (!mayShadow(ND, Info, E.Declaration, E.Info))
+  // Visit each skeleton with more than one identifier.
+  for (auto &[Skel, Idents] : SkeletonToNames) {
+    if (Idents.size() < 2) {
       continue;
+    }
 
-    const IdentifierInfo *ONDII = E.Declaration->getIdentifier();
-    StringRef ONDName = ONDII->getName();
-    if (ONDName == NDName)
-      continue;
+    // Find the declaration contexts that transitively contain each identifier.
+    DeclsWithinContextMap DeclsWithinContext;
+    for (const IdentifierInfo *II : Idents) {
+      for (auto [ND, Parent] : NameToDecls[II]) {
+        addToEnclosingContexts(DeclsWithinContext, Parent, ND);
+      }
+    }
 
-    diag(ND->getLocation(), "%0 is confusable with %1") << ND << E.Declaration;
-    diag(E.Declaration->getLocation(), "other declaration found here",
-         DiagnosticIDs::Note);
+    // Check to see if any declaration is declared in a context that
+    // transitively contains another declaration with a different identifier but
+    // the same skeleton.
+    for (const IdentifierInfo *II : Idents) {
+      for (auto [OuterND, OuterParent] : NameToDecls[II]) {
+        for (Entry Inner : DeclsWithinContext[OuterParent]) {
+          // Don't complain if the identifiers are the same.
+          if (OuterND->getIdentifier() == Inner.ND->getIdentifier())
+            continue;
+
+          // Don't complain about a derived-class name shadowing a base class
+          // private member.
+          if (OuterND->getAccess() == AS_private && Inner.FromDerivedClass)
+            continue;
+
+          // If the declarations are in the same context, only diagnose the
+          // later one.
+          if (OuterParent == Inner.Parent &&
+              Inner.ND->getASTContext()
+                  .getSourceManager()
+                  .isBeforeInTranslationUnit(Inner.ND->getLocation(),
+                                             OuterND->getLocation()))
+            continue;
+
+          diag(Inner.ND->getLocation(), "%0 is confusable with %1")
+              << Inner.ND << OuterND;
+          diag(OuterND->getLocation(), "other declaration found here",
+               DiagnosticIDs::Note);
+        }
+      }
+    }
   }
 
-  Mapped.push_back({ND, Info});
-}
-
-void ConfusableIdentifierCheck::onEndOfTranslationUnit() {
-  Mapper.clear();
-  ContextInfos.clear();
+  NameToDecls.clear();
 }
 
 void ConfusableIdentifierCheck::registerMatchers(
     ast_matchers::MatchFinder *Finder) {
-  Finder->addMatcher(ast_matchers::namedDecl().bind("nameddecl"), this);
+  // Parameter declarations sometimes use the translation unit or some outer
+  // enclosing context as their `DeclContext`, instead of their parent, so
+  // we handle them specially in `check`.
+  auto AnyParamDecl = ast_matchers::anyOf(
+      ast_matchers::parmVarDecl(), ast_matchers::templateTypeParmDecl(),
+      ast_matchers::nonTypeTemplateParmDecl(),
+      ast_matchers::templateTemplateParmDecl());
+  Finder->addMatcher(ast_matchers::namedDecl(ast_matchers::unless(AnyParamDecl))
+                         .bind("nameddecl"),
+                     this);
 }
 
 } // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
index f3b0c8ed0030..9cce6cce6768 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableIdentifierCheck.h
@@ -1,5 +1,4 @@
-//===--- ConfusableIdentifierCheck.h - clang-tidy
-//-------------------------------*- C++ -*-===//
+//===--- ConfusableIdentifierCheck.h - clang-tidy ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,7 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_CONFUSABLE_IDENTIFIER_CHECK_H
 
 #include "../ClangTidyCheck.h"
-#include <unordered_map>
+#include "llvm/ADT/DenseMap.h"
 
 namespace clang::tidy::misc {
 
@@ -31,23 +30,13 @@ class ConfusableIdentifierCheck : public ClangTidyCheck {
     return TK_IgnoreUnlessSpelledInSource;
   }
 
-  struct ContextInfo {
-    const DeclContext *PrimaryContext;
-    const DeclContext *NonTransparentContext;
-    llvm::SmallVector<const DeclContext *> PrimaryContexts;
-    llvm::SmallVector<const CXXRecordDecl *> Bases;
-  };
-
 private:
-  struct Entry {
-    const NamedDecl *Declaration;
-    const ContextInfo *Info;
-  };
-
-  const ContextInfo *getContextInfo(const DeclContext *DC);
+  void addDeclToCheck(const NamedDecl *ND, const Decl *Parent);
 
-  llvm::StringMap<llvm::SmallVector<Entry>> Mapper;
-  std::unordered_map<const DeclContext *, ContextInfo> ContextInfos;
+  llvm::DenseMap<
+      const IdentifierInfo *,
+      llvm::SmallVector<std::pair<const NamedDecl *, const Decl *>, 1>>
+      NameToDecls;
 };
 
 } // namespace clang::tidy::misc
diff --git a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
index afc4897eeb2a..3fdaf9239f6a 100644
--- a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
@@ -66,8 +66,11 @@ void UnconventionalAssignOperatorCheck::registerMatchers(
                                 hasArgument(0, cxxThisExpr())),
             cxxOperatorCallExpr(
                 hasOverloadedOperatorName("="),
-                hasArgument(
-                    0, unaryOperator(hasOperatorName("*"),
+                hasArgument(0, unaryOperator(hasOperatorName("*"),
+                                             hasUnaryOperand(cxxThisExpr())))),
+            binaryOperator(
+                hasOperatorName("="),
+                hasLHS(unaryOperator(hasOperatorName("*"),
                                      hasUnaryOperand(cxxThisExpr())))))))));
   const auto IsGoodAssign = cxxMethodDecl(IsAssign, HasGoodReturnType);
 
diff --git a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
index bab1167fb15f..619a27b2f9bb 100644
--- a/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/modernize/CMakeLists.txt
@@ -42,6 +42,7 @@ add_clang_library(clangTidyModernizeModule STATIC
   UseNullptrCheck.cpp
   UseOverrideCheck.cpp
   UseRangesCheck.cpp
+  UseScopedLockCheck.cpp
   UseStartsEndsWithCheck.cpp
   UseStdFormatCheck.cpp
   UseStdNumbersCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
index 0cf59b6e0216..fdf38bc4b630 100644
--- a/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ModernizeTidyModule.cpp
@@ -43,6 +43,7 @@
 #include "UseNullptrCheck.h"
 #include "UseOverrideCheck.h"
 #include "UseRangesCheck.h"
+#include "UseScopedLockCheck.h"
 #include "UseStartsEndsWithCheck.h"
 #include "UseStdFormatCheck.h"
 #include "UseStdNumbersCheck.h"
@@ -80,6 +81,8 @@ class ModernizeModule : public ClangTidyModule {
     CheckFactories.registerCheck<UseIntegerSignComparisonCheck>(
         "modernize-use-integer-sign-comparison");
     CheckFactories.registerCheck<UseRangesCheck>("modernize-use-ranges");
+    CheckFactories.registerCheck<UseScopedLockCheck>(
+        "modernize-use-scoped-lock");
     CheckFactories.registerCheck<UseStartsEndsWithCheck>(
         "modernize-use-starts-ends-with");
     CheckFactories.registerCheck<UseStdFormatCheck>("modernize-use-std-format");
diff --git a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
new file mode 100644
index 000000000000..9c2fc9e06fb4
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
@@ -0,0 +1,311 @@
+//===--- UseScopedLockCheck.cpp - clang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UseScopedLockCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::modernize {
+
+static bool isLockGuardDecl(const NamedDecl *Decl) {
+  return Decl->getDeclName().isIdentifier() &&
+         Decl->getName() == "lock_guard" && Decl->isInStdNamespace();
+}
+
+static bool isLockGuard(const QualType &Type) {
+  if (const auto *Record = Type->getAs<RecordType>())
+    if (const RecordDecl *Decl = Record->getDecl())
+      return isLockGuardDecl(Decl);
+
+  if (const auto *TemplateSpecType = Type->getAs<TemplateSpecializationType>())
+    if (const TemplateDecl *Decl =
+            TemplateSpecType->getTemplateName().getAsTemplateDecl())
+      return isLockGuardDecl(Decl);
+
+  return false;
+}
+
+static llvm::SmallVector<const VarDecl *>
+getLockGuardsFromDecl(const DeclStmt *DS) {
+  llvm::SmallVector<const VarDecl *> LockGuards;
+
+  for (const Decl *Decl : DS->decls()) {
+    if (const auto *VD = dyn_cast<VarDecl>(Decl)) {
+      const QualType Type =
+          VD->getType().getCanonicalType().getUnqualifiedType();
+      if (isLockGuard(Type))
+        LockGuards.push_back(VD);
+    }
+  }
+
+  return LockGuards;
+}
+
+// Scans through the statements in a block and groups consecutive
+// 'std::lock_guard' variable declarations together.
+static llvm::SmallVector<llvm::SmallVector<const VarDecl *>>
+findLocksInCompoundStmt(const CompoundStmt *Block,
+                        const ast_matchers::MatchFinder::MatchResult &Result) {
+  // store groups of consecutive 'std::lock_guard' declarations
+  llvm::SmallVector<llvm::SmallVector<const VarDecl *>> LockGuardGroups;
+  llvm::SmallVector<const VarDecl *> CurrentLockGuardGroup;
+
+  auto AddAndClearCurrentGroup = [&]() {
+    if (!CurrentLockGuardGroup.empty()) {
+      LockGuardGroups.push_back(CurrentLockGuardGroup);
+      CurrentLockGuardGroup.clear();
+    }
+  };
+
+  for (const Stmt *Stmt : Block->body()) {
+    if (const auto *DS = dyn_cast<DeclStmt>(Stmt)) {
+      llvm::SmallVector<const VarDecl *> LockGuards = getLockGuardsFromDecl(DS);
+
+      if (!LockGuards.empty()) {
+        CurrentLockGuardGroup.append(LockGuards);
+        continue;
+      }
+    }
+    AddAndClearCurrentGroup();
+  }
+
+  AddAndClearCurrentGroup();
+
+  return LockGuardGroups;
+}
+
+static TemplateSpecializationTypeLoc
+getTemplateLockGuardTypeLoc(const TypeSourceInfo *SourceInfo) {
+  const TypeLoc Loc = SourceInfo->getTypeLoc();
+
+  const auto ElaboratedLoc = Loc.getAs<ElaboratedTypeLoc>();
+  if (!ElaboratedLoc)
+    return {};
+
+  return ElaboratedLoc.getNamedTypeLoc().getAs<TemplateSpecializationTypeLoc>();
+}
+
+// Find the exact source range of the 'lock_guard' token
+static SourceRange getLockGuardRange(const TypeSourceInfo *SourceInfo) {
+  const TypeLoc LockGuardTypeLoc = SourceInfo->getTypeLoc();
+
+  return SourceRange(LockGuardTypeLoc.getBeginLoc(),
+                     LockGuardTypeLoc.getEndLoc());
+}
+
+// Find the exact source range of the 'lock_guard' name token
+static SourceRange getLockGuardNameRange(const TypeSourceInfo *SourceInfo) {
+  const TemplateSpecializationTypeLoc TemplateLoc =
+      getTemplateLockGuardTypeLoc(SourceInfo);
+  if (!TemplateLoc)
+    return {};
+
+  return SourceRange(TemplateLoc.getTemplateNameLoc(),
+                     TemplateLoc.getLAngleLoc().getLocWithOffset(-1));
+}
+
+const static StringRef UseScopedLockMessage =
+    "use 'std::scoped_lock' instead of 'std::lock_guard'";
+
+UseScopedLockCheck::UseScopedLockCheck(StringRef Name,
+                                       ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      WarnOnSingleLocks(Options.get("WarnOnSingleLocks", true)),
+      WarnOnUsingAndTypedef(Options.get("WarnOnUsingAndTypedef", true)) {}
+
+void UseScopedLockCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "WarnOnSingleLocks", WarnOnSingleLocks);
+  Options.store(Opts, "WarnOnUsingAndTypedef", WarnOnUsingAndTypedef);
+}
+
+void UseScopedLockCheck::registerMatchers(MatchFinder *Finder) {
+  const auto LockGuardClassDecl =
+      namedDecl(hasName("lock_guard"), isInStdNamespace());
+
+  const auto LockGuardType = qualType(anyOf(
+      hasUnqualifiedDesugaredType(
+          recordType(hasDeclaration(LockGuardClassDecl))),
+      elaboratedType(namesType(hasUnqualifiedDesugaredType(
+          templateSpecializationType(hasDeclaration(LockGuardClassDecl)))))));
+
+  const auto LockVarDecl = varDecl(hasType(LockGuardType));
+
+  if (WarnOnSingleLocks) {
+    Finder->addMatcher(
+        compoundStmt(
+            unless(isExpansionInSystemHeader()),
+            has(declStmt(has(LockVarDecl)).bind("lock-decl-single")),
+            unless(has(declStmt(unless(equalsBoundNode("lock-decl-single")),
+                                has(LockVarDecl))))),
+        this);
+  }
+
+  Finder->addMatcher(
+      compoundStmt(unless(isExpansionInSystemHeader()),
+                   has(declStmt(has(LockVarDecl)).bind("lock-decl-multiple")),
+                   has(declStmt(unless(equalsBoundNode("lock-decl-multiple")),
+                                has(LockVarDecl))))
+          .bind("block-multiple"),
+      this);
+
+  if (WarnOnUsingAndTypedef) {
+    // Match 'typedef std::lock_guard<std::mutex> Lock'
+    Finder->addMatcher(typedefDecl(unless(isExpansionInSystemHeader()),
+                                   hasUnderlyingType(LockGuardType))
+                           .bind("lock-guard-typedef"),
+                       this);
+
+    // Match 'using Lock = std::lock_guard<std::mutex>'
+    Finder->addMatcher(
+        typeAliasDecl(
+            unless(isExpansionInSystemHeader()),
+            hasType(elaboratedType(namesType(templateSpecializationType(
+                hasDeclaration(LockGuardClassDecl))))))
+            .bind("lock-guard-using-alias"),
+        this);
+
+    // Match 'using std::lock_guard'
+    Finder->addMatcher(
+        usingDecl(unless(isExpansionInSystemHeader()),
+                  hasAnyUsingShadowDecl(hasTargetDecl(LockGuardClassDecl)))
+            .bind("lock-guard-using-decl"),
+        this);
+  }
+}
+
+void UseScopedLockCheck::check(const MatchFinder::MatchResult &Result) {
+  if (const auto *DS = Result.Nodes.getNodeAs<DeclStmt>("lock-decl-single")) {
+    llvm::SmallVector<const VarDecl *> Decls = getLockGuardsFromDecl(DS);
+    diagOnMultipleLocks({Decls}, Result);
+    return;
+  }
+
+  if (const auto *Compound =
+          Result.Nodes.getNodeAs<CompoundStmt>("block-multiple")) {
+    diagOnMultipleLocks(findLocksInCompoundStmt(Compound, Result), Result);
+    return;
+  }
+
+  if (const auto *Typedef =
+          Result.Nodes.getNodeAs<TypedefDecl>("lock-guard-typedef")) {
+    diagOnSourceInfo(Typedef->getTypeSourceInfo(), Result);
+    return;
+  }
+
+  if (const auto *UsingAlias =
+          Result.Nodes.getNodeAs<TypeAliasDecl>("lock-guard-using-alias")) {
+    diagOnSourceInfo(UsingAlias->getTypeSourceInfo(), Result);
+    return;
+  }
+
+  if (const auto *Using =
+          Result.Nodes.getNodeAs<UsingDecl>("lock-guard-using-decl")) {
+    diagOnUsingDecl(Using, Result);
+  }
+}
+
+void UseScopedLockCheck::diagOnSingleLock(
+    const VarDecl *LockGuard, const MatchFinder::MatchResult &Result) {
+  auto Diag = diag(LockGuard->getBeginLoc(), UseScopedLockMessage);
+
+  const SourceRange LockGuardTypeRange =
+      getLockGuardRange(LockGuard->getTypeSourceInfo());
+
+  if (LockGuardTypeRange.isInvalid())
+    return;
+
+  // Create Fix-its only if we can find the constructor call to properly handle
+  // 'std::lock_guard l(m, std::adopt_lock)' case.
+  const auto *CtorCall = dyn_cast<CXXConstructExpr>(LockGuard->getInit());
+  if (!CtorCall)
+    return;
+
+  if (CtorCall->getNumArgs() == 1) {
+    Diag << FixItHint::CreateReplacement(LockGuardTypeRange,
+                                         "std::scoped_lock");
+    return;
+  }
+
+  if (CtorCall->getNumArgs() == 2) {
+    const Expr *const *CtorArgs = CtorCall->getArgs();
+
+    const Expr *MutexArg = CtorArgs[0];
+    const Expr *AdoptLockArg = CtorArgs[1];
+
+    const StringRef MutexSourceText = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(MutexArg->getSourceRange()),
+        *Result.SourceManager, Result.Context->getLangOpts());
+    const StringRef AdoptLockSourceText = Lexer::getSourceText(
+        CharSourceRange::getTokenRange(AdoptLockArg->getSourceRange()),
+        *Result.SourceManager, Result.Context->getLangOpts());
+
+    Diag << FixItHint::CreateReplacement(LockGuardTypeRange, "std::scoped_lock")
+         << FixItHint::CreateReplacement(
+                SourceRange(MutexArg->getBeginLoc(), AdoptLockArg->getEndLoc()),
+                (llvm::Twine(AdoptLockSourceText) + ", " + MutexSourceText)
+                    .str());
+    return;
+  }
+
+  llvm_unreachable("Invalid argument number of std::lock_guard constructor");
+}
+
+void UseScopedLockCheck::diagOnMultipleLocks(
+    const llvm::SmallVector<llvm::SmallVector<const VarDecl *>> &LockGroups,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  for (const llvm::SmallVector<const VarDecl *> &Group : LockGroups) {
+    if (Group.size() == 1) {
+      if (WarnOnSingleLocks)
+        diagOnSingleLock(Group[0], Result);
+    } else {
+      diag(Group[0]->getBeginLoc(),
+           "use single 'std::scoped_lock' instead of multiple "
+           "'std::lock_guard'");
+
+      for (const VarDecl *Lock : llvm::drop_begin(Group))
+        diag(Lock->getLocation(), "additional 'std::lock_guard' declared here",
+             DiagnosticIDs::Note);
+    }
+  }
+}
+
+void UseScopedLockCheck::diagOnSourceInfo(
+    const TypeSourceInfo *LockGuardSourceInfo,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  const TypeLoc TL = LockGuardSourceInfo->getTypeLoc();
+
+  if (const auto ElaboratedTL = TL.getAs<ElaboratedTypeLoc>()) {
+    auto Diag = diag(ElaboratedTL.getBeginLoc(), UseScopedLockMessage);
+
+    const SourceRange LockGuardRange =
+        getLockGuardNameRange(LockGuardSourceInfo);
+    if (LockGuardRange.isInvalid())
+      return;
+
+    Diag << FixItHint::CreateReplacement(LockGuardRange, "scoped_lock");
+  }
+}
+
+void UseScopedLockCheck::diagOnUsingDecl(
+    const UsingDecl *UsingDecl,
+    const ast_matchers::MatchFinder::MatchResult &Result) {
+  diag(UsingDecl->getLocation(), UseScopedLockMessage)
+      << FixItHint::CreateReplacement(UsingDecl->getLocation(), "scoped_lock");
+}
+
+} // namespace clang::tidy::modernize
diff --git a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h
new file mode 100644
index 000000000000..a5697805c15c
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.h
@@ -0,0 +1,54 @@
+//===--- UseScopedLockCheck.h - clang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
+
+#include "../ClangTidyCheck.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Stmt.h"
+#include <optional>
+
+namespace clang::tidy::modernize {
+
+/// Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+/// alternative ``std::scoped_lock``.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/modernize/use-scoped-lock.html
+class UseScopedLockCheck : public ClangTidyCheck {
+public:
+  UseScopedLockCheck(StringRef Name, ClangTidyContext *Context);
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus17;
+  }
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TK_IgnoreUnlessSpelledInSource;
+  }
+
+private:
+  void diagOnSingleLock(const VarDecl *LockGuard,
+                        const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnMultipleLocks(
+      const llvm::SmallVector<llvm::SmallVector<const VarDecl *>> &LockGroups,
+      const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnSourceInfo(const TypeSourceInfo *LockGuardSourceInfo,
+                        const ast_matchers::MatchFinder::MatchResult &Result);
+  void diagOnUsingDecl(const UsingDecl *UsingDecl,
+                       const ast_matchers::MatchFinder::MatchResult &Result);
+
+  const bool WarnOnSingleLocks;
+  const bool WarnOnUsingAndTypedef;
+};
+
+} // namespace clang::tidy::modernize
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MODERNIZE_USESCOPEDLOCKCHECK_H
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
index 969bf2d1add6..7f1882c775c5 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantInlineSpecifierCheck.cpp
@@ -72,11 +72,13 @@ static SourceLocation getInlineTokenLocation(SourceRange RangeLocation,
 }
 
 void RedundantInlineSpecifierCheck::registerMatchers(MatchFinder *Finder) {
+  const auto IsPartOfRecordDecl = hasAncestor(recordDecl());
   Finder->addMatcher(
       functionDecl(isInlineSpecified(),
-                   anyOf(isConstexpr(), isDeleted(), isDefaulted(),
+                   anyOf(isConstexpr(), isDeleted(),
+                         allOf(isDefaulted(), IsPartOfRecordDecl),
                          isInternalLinkage(StrictMode),
-                         allOf(isDefinition(), hasAncestor(recordDecl()))))
+                         allOf(isDefinition(), IsPartOfRecordDecl)))
           .bind("fun_decl"),
       this);
 
@@ -88,7 +90,6 @@ void RedundantInlineSpecifierCheck::registerMatchers(MatchFinder *Finder) {
         this);
 
   if (getLangOpts().CPlusPlus17) {
-    const auto IsPartOfRecordDecl = hasAncestor(recordDecl());
     Finder->addMatcher(
         varDecl(
             isInlineSpecified(),
diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index 3b991e5e9013..e27423652781 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -440,7 +440,7 @@ QualType declaredType(const TypeDecl *D) {
     if (const auto *Args = CTSD->getTemplateArgsAsWritten())
       return Context.getTemplateSpecializationType(
           TemplateName(CTSD->getSpecializedTemplate()), Args->arguments(),
-          /*CanonicalArgs=*/std::nullopt);
+          /*CanonicalArgs=*/{});
   return Context.getTypeDeclType(D);
 }
 
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index 20a238612a7e..197c62c40dcf 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -33,7 +33,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/ADT/identity.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -339,53 +338,6 @@ QualType maybeDesugar(ASTContext &AST, QualType QT) {
   return QT;
 }
 
-// Given a callee expression `Fn`, if the call is through a function pointer,
-// try to find the declaration of the corresponding function pointer type,
-// so that we can recover argument names from it.
-// FIXME: This function is mostly duplicated in SemaCodeComplete.cpp; unify.
-static FunctionProtoTypeLoc getPrototypeLoc(Expr *Fn) {
-  TypeLoc Target;
-  Expr *NakedFn = Fn->IgnoreParenCasts();
-  if (const auto *T = NakedFn->getType().getTypePtr()->getAs<TypedefType>()) {
-    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
-  } else if (const auto *DR = dyn_cast<DeclRefExpr>(NakedFn)) {
-    const auto *D = DR->getDecl();
-    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
-      Target = VD->getTypeSourceInfo()->getTypeLoc();
-    }
-  }
-
-  if (!Target)
-    return {};
-
-  // Unwrap types that may be wrapping the function type
-  while (true) {
-    if (auto P = Target.getAs<PointerTypeLoc>()) {
-      Target = P.getPointeeLoc();
-      continue;
-    }
-    if (auto A = Target.getAs<AttributedTypeLoc>()) {
-      Target = A.getModifiedLoc();
-      continue;
-    }
-    if (auto P = Target.getAs<ParenTypeLoc>()) {
-      Target = P.getInnerLoc();
-      continue;
-    }
-    break;
-  }
-
-  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
-    // In some edge cases the AST can contain a "trivial" FunctionProtoTypeLoc
-    // which has null parameters. Avoid these as they don't contain useful
-    // information.
-    if (llvm::all_of(F.getParams(), llvm::identity<ParmVarDecl *>()))
-      return F;
-  }
-
-  return {};
-}
-
 ArrayRef<const ParmVarDecl *>
 maybeDropCxxExplicitObjectParameters(ArrayRef<const ParmVarDecl *> Params) {
   if (!Params.empty() && Params.front()->isExplicitObjectParameter())
@@ -514,7 +466,8 @@ class InlayHintVisitor : public RecursiveASTVisitor<InlayHintVisitor> {
       Callee.Decl = FD;
     else if (const auto *FTD = dyn_cast<FunctionTemplateDecl>(CalleeDecls[0]))
       Callee.Decl = FTD->getTemplatedDecl();
-    else if (FunctionProtoTypeLoc Loc = getPrototypeLoc(E->getCallee()))
+    else if (FunctionProtoTypeLoc Loc =
+                 Resolver->getFunctionProtoTypeLoc(E->getCallee()))
       Callee.Loc = Loc;
     else
       return true;
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
index d6556bba1472..c9704492bf1c 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
@@ -341,7 +341,7 @@ const FunctionDecl *findTarget(const FunctionDecl *FD) {
 
 // Returns the beginning location for a FunctionDecl. Returns location of
 // template keyword for templated functions.
-const SourceLocation getBeginLoc(const FunctionDecl *FD) {
+SourceLocation getBeginLoc(const FunctionDecl *FD) {
   // Include template parameter list.
   if (auto *FTD = FD->getDescribedFunctionTemplate())
     return FTD->getBeginLoc();
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
index 90dac3b76c64..c74250ccbe9e 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
@@ -359,9 +359,9 @@ struct ParsedBinaryOperator {
 //    +   c <- End
 //   / \
 //  a   b <- Start
-const SourceRange getBinaryOperatorRange(const SelectionTree::Node &N,
-                                         const SourceManager &SM,
-                                         const LangOptions &LangOpts) {
+SourceRange getBinaryOperatorRange(const SelectionTree::Node &N,
+                                   const SourceManager &SM,
+                                   const LangOptions &LangOpts) {
   // If N is not a suitable binary operator, bail out.
   ParsedBinaryOperator Op;
   if (!Op.parse(N.ignoreImplicit()) || !Op.associative() ||
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index b11d194da04d..f287439f10ca 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -668,7 +668,6 @@ class FlagsConfigProvider : public config::Provider {
     std::optional<Config::ExternalIndexSpec> IndexSpec;
     std::optional<Config::BackgroundPolicy> BGPolicy;
     std::optional<Config::ArgumentListsPolicy> ArgumentLists;
-    std::optional<Config::HeaderInsertionPolicy> HeaderInsertionPolicy;
 
     // If --compile-commands-dir arg was invoked, check value and override
     // default path.
@@ -713,11 +712,6 @@ class FlagsConfigProvider : public config::Provider {
       BGPolicy = Config::BackgroundPolicy::Skip;
     }
 
-    // If CLI has set never, use that regardless of what the config files have
-    if (HeaderInsertion == Config::HeaderInsertionPolicy::NeverInsert) {
-      HeaderInsertionPolicy = Config::HeaderInsertionPolicy::NeverInsert;
-    }
-
     if (std::optional<bool> Enable = shouldEnableFunctionArgSnippets()) {
       ArgumentLists = *Enable ? Config::ArgumentListsPolicy::FullPlaceholders
                               : Config::ArgumentListsPolicy::Delimiters;
@@ -732,8 +726,8 @@ class FlagsConfigProvider : public config::Provider {
         C.Index.Background = *BGPolicy;
       if (ArgumentLists)
         C.Completion.ArgumentLists = *ArgumentLists;
-      if (HeaderInsertionPolicy)
-        C.Completion.HeaderInsertion = *HeaderInsertionPolicy;
+      if (HeaderInsertion.getNumOccurrences())
+        C.Completion.HeaderInsertion = HeaderInsertion;
       if (AllScopesCompletion.getNumOccurrences())
         C.Completion.AllScopes = AllScopesCompletion;
 
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 311f0d98904a..b7c64c7a0674 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -4363,6 +4363,36 @@ TEST(CompletionTest, PreambleFromDifferentTarget) {
   EXPECT_THAT(Result.Completions, Not(testing::IsEmpty()));
   EXPECT_THAT(Signatures.signatures, Not(testing::IsEmpty()));
 }
+
+TEST(CompletionTest, SkipExplicitObjectParameter) {
+  Annotations Code(R"cpp(
+    struct A {
+      void foo(this auto&& self, int arg); 
+    };
+
+    int main() {
+      A a {};
+      a.^
+    }
+  )cpp");
+
+  auto TU = TestTU::withCode(Code.code());
+  TU.ExtraArgs = {"-std=c++23"};
+
+  auto Preamble = TU.preamble();
+  ASSERT_TRUE(Preamble);
+
+  CodeCompleteOptions Opts{};
+
+  MockFS FS;
+  auto Inputs = TU.inputs(FS);
+  auto Result = codeComplete(testPath(TU.Filename), Code.point(),
+                             Preamble.get(), Inputs, Opts);
+
+  EXPECT_THAT(Result.Completions,
+              ElementsAre(AllOf(named("foo"), signature("(int arg)"),
+                                snippetSuffix("(${1:int arg})"))));
+}
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index fc51f3c9329a..e021d6350694 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -142,6 +142,12 @@ New checks
   Finds unscoped (non-class) ``enum`` declarations and suggests using
   ``enum class`` instead.
 
+- New :doc:`modernize-use-scoped-lock
+  <clang-tidy/checks/modernize/use-scoped-lock>` check.
+
+  Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+  alternative ``std::scoped_lock``.
+
 - New :doc:`portability-avoid-pragma-once
   <clang-tidy/checks/portability/avoid-pragma-once>` check.
 
@@ -173,6 +179,11 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/optional-value-conversion>` check to detect
   conversion in argument of ``std::make_optional``.
 
+- Improved :doc: `bugprone-sizeof-expression
+  <clang-tidy/checks/bugprone/bugprone-sizeof-expression>` check by adding
+  `WarnOnSizeOfInLoopTermination` option to detect misuses of ``sizeof``
+  expression in loop conditions.
+
 - Improved :doc:`bugprone-string-constructor
   <clang-tidy/checks/bugprone/string-constructor>` check to find suspicious
   calls of ``std::string`` constructor with char pointer, start position and
@@ -201,6 +212,21 @@ Changes in existing checks
   <clang-tidy/checks/cppcoreguidelines/avoid-goto>` check by adding the option
   `IgnoreMacros` to ignore ``goto`` labels defined in macros.
 
+- Improved :doc:`cppcoreguidelines-missing-std-forward
+  <clang-tidy/checks/cppcoreguidelines/missing-std-forward>` check by adding a
+  flag to specify the function used for forwarding instead of ``std::forward``.
+   
+- Improved :doc:`cppcoreguidelines-pro-bounds-pointer-arithmetic
+  <clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic>` check by
+  fixing false positives when calling indexing operators that do not perform
+  pointer arithmetic in template, for example ``std::map::operator[]`` and
+  when pointer arithmetic was used through type aliases.
+
+- Improved :doc:`cppcoreguidelines-rvalue-reference-param-not-moved
+  <clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved>` check
+  by adding a flag to specify the function used for moving instead of
+  ``std::move``.
+
 - Improved :doc:`cppcoreguidelines-special-member-functions
   <clang-tidy/checks/cppcoreguidelines/special-member-functions>` check by
   adding the option `IgnoreMacros` to ignore classes defined in macros.
@@ -240,6 +266,11 @@ Changes in existing checks
   <clang-tidy/checks/misc/redundant-expression>` check by providing additional
   examples and fixing some macro related false positives.
 
+- Improved :doc:`misc-unconventional-assign-operator
+  <clang-tidy/checks/misc/unconventional-assign-operator>` check by fixing
+  false positives when copy assignment operator function in a template class
+  returns the result of another assignment to ``*this`` (``return *this=...``).
+
 - Improved :doc:`misc-unused-using-decls
   <clang-tidy/checks/misc/unused-using-decls>` check by fixing false positives
   on ``operator""`` with template parameters.
@@ -308,6 +339,10 @@ Changes in existing checks
   <clang-tidy/checks/readability/qualified-auto>` check by adding the option
   `AllowedTypes`, that excludes specified types from adding qualifiers.
 
+- Improved :doc:`readability-redundant-inline-specifier
+  <clang-tidy/checks/readability/redundant-inline-specifier>` check by fixing
+  false positives on out-of-line explicitly defaulted functions.
+
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check by fixing
   some false positives involving smart pointers to arrays.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
index 29edb26ed7aa..04824cc1fe0e 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/sizeof-expression.rst
@@ -316,3 +316,12 @@ Options
    When `true`, the check will warn on pointer arithmetic where the
    element count is obtained from a division with ``sizeof(...)``,
    e.g., ``Ptr + Bytes / sizeof(*T)``. Default is `true`.
+
+.. option:: WarnOnSizeOfInLoopTermination
+
+   When `true`, the check will warn about incorrect use of sizeof expression
+   in loop termination condition. The warning triggers if the ``sizeof``
+   expression appears to be incorrectly used to determine the number of
+   array/buffer elements.
+   e.g, ``long arr[10]; for(int i = 0; i < sizeof(arr); i++) { ... }``. Default
+   is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
index 0c311b59a5d5..62e38fcd3b9d 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/missing-std-forward.rst
@@ -35,6 +35,13 @@ Example:
     f(1, 2); // Incorrect - may not invoke the desired qualified function operator
   }
 
+Options
+-------
+
+.. option:: ForwardFunction
+
+   Specify the function used for forwarding. Default is `::std::forward`.
+
 This check implements `F.19
 <http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-forward>`_
 from the C++ Core Guidelines.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
index ffa3a9d61e48..2fea9f16b3bb 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/rvalue-reference-param-not-moved.rst
@@ -79,6 +79,10 @@ Options
       T other = std::forward<T>(t);
     }
 
+.. option:: MoveFunction
+
+   Specify the function used for moving. Default is `::std::move`.
+
 This check implements `F.18
 <http://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#f18-for-will-move-from-parameters-pass-by-x-and-stdmove-the-parameter>`_
 from the C++ Core Guidelines.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index ccb78ee45e9c..5098582d0c42 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -312,6 +312,7 @@ Clang-Tidy Checks
    :doc:`modernize-use-nullptr <modernize/use-nullptr>`, "Yes"
    :doc:`modernize-use-override <modernize/use-override>`, "Yes"
    :doc:`modernize-use-ranges <modernize/use-ranges>`, "Yes"
+   :doc:`modernize-use-scoped-lock <modernize/use-scoped-lock>`, "Yes"
    :doc:`modernize-use-starts-ends-with <modernize/use-starts-ends-with>`, "Yes"
    :doc:`modernize-use-std-format <modernize/use-std-format>`, "Yes"
    :doc:`modernize-use-std-numbers <modernize/use-std-numbers>`, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst
new file mode 100644
index 000000000000..d184f1aefd80
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-scoped-lock.rst
@@ -0,0 +1,101 @@
+.. title:: clang-tidy - modernize-use-scoped-lock
+
+modernize-use-scoped-lock
+=========================
+
+Finds uses of ``std::lock_guard`` and suggests replacing them with C++17's
+alternative ``std::scoped_lock``.
+
+Fix-its are provided for single declarations of ``std::lock_guard`` and warning
+is emitted for multiple declarations of ``std::lock_guard`` that can be
+replaced with a single declaration of ``std::scoped_lock``.
+
+Examples
+--------
+
+Single ``std::lock_guard`` declaration:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock_guard<std::mutex> L(M);
+
+
+Transforms to:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::scoped_lock L(M);
+
+Single ``std::lock_guard`` declaration with ``std::adopt_lock``:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock(M);
+  std::lock_guard<std::mutex> L(M, std::adopt_lock);
+
+
+Transforms to:
+
+.. code-block:: c++
+
+  std::mutex M;
+  std::lock(M);
+  std::scoped_lock L(std::adopt_lock, M);
+
+Multiple ``std::lock_guard`` declarations only emit warnings:
+
+.. code-block:: c++
+
+  std::mutex M1, M2;
+  std::lock(M1, M2);
+  std::lock_guard Lock1(M, std::adopt_lock); // warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  std::lock_guard Lock2(M, std::adopt_lock); // note: additional 'std::lock_guard' declared here
+
+
+Limitations
+-----------
+
+The check will not emit warnings if ``std::lock_guard`` is used implicitly via
+``template`` parameter:
+
+.. code-block:: c++
+
+  template <template <typename> typename Lock>
+  void TemplatedLock() {
+    std::mutex M;
+    Lock<std::mutex> L(M); // no warning
+  }
+
+  void instantiate() {
+    TemplatedLock<std::lock_guard>();
+  }
+
+
+Options
+-------
+
+.. option:: WarnOnSingleLocks
+
+  When `true`, the check will warn on single ``std::lock_guard`` declarations.
+  Set this option to `false` if you want to get warnings only on multiple
+  ``std::lock_guard`` declarations that can be replaced with a single
+  ``std::scoped_lock``. Default is `true`.
+
+.. option:: WarnOnUsingAndTypedef
+
+  When `true`, the check will emit warnings if ``std::lock_guard`` is used
+  in ``using`` or ``typedef`` context. Default is `true`.
+
+  .. code-block:: c++
+
+    template <typename T>
+    using Lock = std::lock_guard<T>; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    
+    using LockMutex = std::lock_guard<std::mutex>; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    
+    typedef std::lock_guard<std::mutex> LockDef; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+
+    using std::lock_guard; // warning: use 'std::scoped_lock' instead of 'std::lock_guard'
\ No newline at end of file
diff --git a/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst b/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
index aacf3675f979..520c1b50733a 100644
--- a/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
+++ b/clang-tools-extra/docs/clangd/DeveloperDocumentation.rst
@@ -1,3 +1,3 @@
 :orphan:
 :template: clangd_redirect.html
-:redirect_target: https://clangd.llvm.org/design.html
+:redirect_target: https://clangd.llvm.org/design/
diff --git a/clang-tools-extra/modularize/Modularize.cpp b/clang-tools-extra/modularize/Modularize.cpp
index 2a90c5e3f678..376ad0c7875b 100644
--- a/clang-tools-extra/modularize/Modularize.cpp
+++ b/clang-tools-extra/modularize/Modularize.cpp
@@ -395,7 +395,7 @@ struct Location {
     Column = SM.getColumnNumber(Decomposed.first, Decomposed.second);
   }
 
-  operator bool() const { return File != nullptr; }
+  explicit operator bool() const { return File != nullptr; }
 
   friend bool operator==(const Location &X, const Location &Y) {
     return X.File == Y.File && X.Line == Y.Line && X.Column == Y.Column;
@@ -459,7 +459,7 @@ struct HeaderEntry {
     return !(X == Y);
   }
   friend bool operator<(const HeaderEntry &X, const HeaderEntry &Y) {
-    return X.Loc < Y.Loc || (X.Loc == Y.Loc && X.Name < Y.Name);
+    return std::tie(X.Loc, X.Name) < std::tie(Y.Loc, Y.Name);
   }
   friend bool operator>(const HeaderEntry &X, const HeaderEntry &Y) {
     return Y < X;
diff --git a/clang-tools-extra/test/clang-doc/json/class-template.cpp b/clang-tools-extra/test/clang-doc/json/class-template.cpp
index e3ca086d1d9a..fb9c4c2f21c2 100644
--- a/clang-tools-extra/test/clang-doc/json/class-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-template.cpp
@@ -17,7 +17,7 @@ template<typename T> struct MyClass {
 // CHECK-NEXT:    ], 
 // CHECK-NEXT:    "ReturnType": {
 // CHECK-NEXT:      "IsBuiltIn": false,
-// CHECK-NEXT:      "IsTemplate": false,
+// CHECK-NEXT:      "IsTemplate": true,
 // CHECK-NEXT:      "Name": "T",
 // CHECK-NEXT:      "QualName": "T"
 // CHECK-NEXT:      "USR": "0000000000000000000000000000000000000000"
diff --git a/clang-tools-extra/test/clang-doc/json/class.cpp b/clang-tools-extra/test/clang-doc/json/class.cpp
index 9ae1f14e2d2a..ae47da75edcc 100644
--- a/clang-tools-extra/test/clang-doc/json/class.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class.cpp
@@ -23,6 +23,9 @@ struct MyClass {
   typedef int MyTypedef;
   
   class NestedClass;
+  
+  friend struct Foo;
+  template<typename T> friend void friendFunction(int);
 protected:
   int protectedMethod();
 
@@ -86,6 +89,44 @@ struct MyClass {
 // CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
 // CHECK-NEXT:      }
 // CHECK-NEXT:    ],
+// CHECK-NEXT:    "Friends": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsClass": false,
+// CHECK-NEXT:        "Params": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Name": "",
+// CHECK-NEXT:            "Type": "int"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
+// CHECK-NEXT:        "Reference": {
+// CHECK-NEXT:          "Name": "friendFunction",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "friendFunction",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "ReturnType": {
+// CHECK-NEXT:          "IsBuiltIn": true,
+// CHECK-NEXT:          "IsTemplate": false,
+// CHECK-NEXT:          "Name": "void",
+// CHECK-NEXT:          "QualName": "void",
+// CHECK-NEXT:          "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "Template": {
+// CHECK-NEXT:          "Parameters": [
+// CHECK-NEXT:            "typename T"
+// CHECK-NEXT:          ]
+// CHECK-NEXT:        }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "IsClass": true,
+// CHECK-NEXT:        "Reference": {
+// CHECK-NEXT:          "Name": "Foo",
+// CHECK-NEXT:          "Path": "GlobalNamespace",
+// CHECK-NEXT:          "QualName": "Foo",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
 // COM:           FIXME: FullName is not emitted correctly.
 // CHECK-NEXT:    "FullName": "",
 // CHECK-NEXT:    "IsTypedef": false,
@@ -107,7 +148,7 @@ struct MyClass {
 // CHECK-NEXT:         "GlobalNamespace"
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "ReturnType": {
-// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsBuiltIn": true,
 // CHECK-NEXT:         "IsTemplate": false,
 // CHECK-NEXT:         "Name": "int",
 // CHECK-NEXT:         "QualName": "int",
@@ -137,7 +178,7 @@ struct MyClass {
 // CHECK-NEXT:          }
 // CHECK-NEXT:        ],
 // CHECK-NEXT:        "ReturnType": {
-// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsBuiltIn": true,
 // CHECK-NEXT:          "IsTemplate": false,
 // CHECK-NEXT:          "Name": "int",
 // CHECK-NEXT:          "QualName": "int",
@@ -184,7 +225,7 @@ struct MyClass {
 // CHECK-NEXT:        "TypeDeclaration": "",
 // CHECK-NEXT:        "USR": "{{[0-9A-F]*}}",
 // CHECK-NEXT:        "Underlying": {
-// CHECK-NEXT:          "IsBuiltIn": false,
+// CHECK-NEXT:          "IsBuiltIn": true,
 // CHECK-NEXT:          "IsTemplate": false,
 // CHECK-NEXT:          "Name": "int",
 // CHECK-NEXT:          "QualName": "int",
diff --git a/clang-tools-extra/test/clang-doc/json/function-requires.cpp b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
index 99eb2bdb898f..4e8432e088c4 100644
--- a/clang-tools-extra/test/clang-doc/json/function-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
@@ -23,7 +23,7 @@ template<Incrementable T> Incrementable auto incrementTwo(T t);
 // CHECK-NEXT:        }
 // CHECK-NEXT:      ],
 // CHECK-NEXT:      "ReturnType": {
-// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsBuiltIn": true,
 // CHECK-NEXT:        "IsTemplate": false,
 // CHECK-NEXT:        "Name": "void",
 // CHECK-NEXT:        "QualName": "void",
diff --git a/clang-tools-extra/test/clang-doc/json/method-template.cpp b/clang-tools-extra/test/clang-doc/json/method-template.cpp
index c51a2706d1c2..ea9110d6c2d1 100644
--- a/clang-tools-extra/test/clang-doc/json/method-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/method-template.cpp
@@ -27,7 +27,7 @@ struct MyClass {
 // CHECK-NEXT:          ],
 // CHECK-NEXT:          "ReturnType": {
 // CHECK-NEXT:            "IsBuiltIn": false,
-// CHECK-NEXT:            "IsTemplate": false,
+// CHECK-NEXT:            "IsTemplate": true,
 // CHECK-NEXT:            "Name": "T",
 // CHECK-NEXT:            "QualName": "T",
 // CHECK-NEXT:            "USR": "0000000000000000000000000000000000000000"
diff --git a/clang-tools-extra/test/clang-doc/json/namespace.cpp b/clang-tools-extra/test/clang-doc/json/namespace.cpp
index 248d47351bd3..6e4fc6938d85 100644
--- a/clang-tools-extra/test/clang-doc/json/namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/json/namespace.cpp
@@ -9,7 +9,6 @@ void myFunction(int Param);
 namespace NestedNamespace {
 } // namespace NestedNamespace
 
-// FIXME: Global variables are not mapped or serialized.
 static int Global;
 
 enum Color {
@@ -25,7 +24,7 @@ typedef int MyTypedef;
 // CHECK-NEXT:      {
 // CHECK-NEXT:        "Location": {
 // CHECK-NEXT:          "Filename": "{{.*}}namespace.cpp",
-// CHECK-NEXT:          "LineNumber": 15
+// CHECK-NEXT:          "LineNumber": 14
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "Members": [
 // CHECK-NEXT:          {
@@ -57,7 +56,7 @@ typedef int MyTypedef;
 // CHECK-NEXT:         }
 // CHECK-NEXT:       ],
 // CHECK-NEXT:       "ReturnType": {
-// CHECK-NEXT:         "IsBuiltIn": false,
+// CHECK-NEXT:         "IsBuiltIn": true,
 // CHECK-NEXT:         "IsTemplate": false,
 // CHECK-NEXT:         "Name": "void",
 // CHECK-NEXT:         "QualName": "void",
@@ -88,13 +87,13 @@ typedef int MyTypedef;
 // CHECK-NEXT:      "IsUsing": false,
 // CHECK-NEXT:      "Location": {
 // CHECK-NEXT:        "Filename": "{{.*}}namespace.cpp",
-// CHECK-NEXT:        "LineNumber": 21
+// CHECK-NEXT:        "LineNumber": 20
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "Name": "MyTypedef",
 // CHECK-NEXT:      "TypeDeclaration": "",
 // CHECK-NEXT:      "USR": "{{[0-9A-F]*}}",
 // CHECK-NEXT:      "Underlying": {
-// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsBuiltIn": true,
 // CHECK-NEXT:        "IsTemplate": false,
 // CHECK-NEXT:        "Name": "int",
 // CHECK-NEXT:        "QualName": "int",
@@ -103,23 +102,22 @@ typedef int MyTypedef;
 // CHECK-NEXT:      }
 // CHECK-NEXT:    ],
 // CHECK-NEXT:    "USR": "0000000000000000000000000000000000000000"
-// CHECK-NOT:    "Variables": [
-// CHECK-NOT:      {
-// CHECK-NOT:        "IsStatic": true,
-// CHECK-NOT:        "Location": {
-// CHECK-NOT:          "Filename": "{{.*}}namespace.cpp",
-// CHECK-NOT:          "LineNumber": 13
-// CHECK-NOT:        },
-// CHECK-NOT:        "Name": "Global",
-// CHECK-NOT:        "Type": {
-// COM:                FIXME: IsBuiltIn emits as its default value
-// CHECK-NOT:          "IsBuiltIn": false,
-// CHECK-NOT:          "IsTemplate": false,
-// CHECK-NOT:          "Name": "int",
-// CHECK-NOT:          "QualName": "int",
-// CHECK-NOT:          "USR": "0000000000000000000000000000000000000000"
-// CHECK-NOT:        },
-// CHECK-NOT:        "USR": "{{[0-9A-F]*}}"
-// CHECK-NOT:      }
-// CHECK-NOT:    ]
+// CHECK-NEXT:   "Variables": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "IsStatic": true,
+// CHECK-NEXT:       "Location": {
+// CHECK-NEXT:         "Filename": "{{.*}}namespace.cpp",
+// CHECK-NEXT:         "LineNumber": 12
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "Name": "Global",
+// CHECK-NEXT:       "Type": {
+// CHECK-NEXT:         "IsBuiltIn": true,
+// CHECK-NEXT:         "IsTemplate": false,
+// CHECK-NEXT:         "Name": "int",
+// CHECK-NEXT:         "QualName": "int",
+// CHECK-NEXT:         "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ]
 // CHECK-NEXT:  }
diff --git a/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
new file mode 100644
index 000000000000..9b176feb67a0
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
@@ -0,0 +1,36 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/nested/index.json --check-prefix=NESTED
+// RUN: FileCheck %s < %t/nested/inner/index.json --check-prefix=INNER
+
+namespace nested {
+  int Global;
+  namespace inner {
+    int InnerGlobal;
+  } // namespace inner
+} // namespace nested
+
+// NESTED:       "Variables": [
+// NESTED-NEXT:    {
+// NESTED-NEXT:      "IsStatic": false,
+// NESTED-NEXT:      "Location": {
+// NESTED-NEXT:        "Filename": "{{.*}}nested-namespace.cpp",
+// NESTED-NEXT:        "LineNumber": 7
+// NESTED-NEXT:      },
+// NESTED-NEXT:      "Name": "Global",
+// NESTED-NEXT:      "Namespace": [
+// NESTED-NEXT:        "nested"
+// NESTED-NEXT:      ],
+
+// INNER:       "Variables": [
+// INNER-NEXT:    {
+// INNER-NEXT:      "IsStatic": false,
+// INNER-NEXT:      "Location": {
+// INNER-NEXT:        "Filename": "{{.*}}nested-namespace.cpp",
+// INNER-NEXT:        "LineNumber": 9
+// INNER-NEXT:      },
+// INNER-NEXT:      "Name": "InnerGlobal",
+// INNER-NEXT:      "Namespace": [
+// INNER-NEXT:        "inner",
+// INNER-NEXT:        "nested"
+// INNER-NEXT:      ],
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex
new file mode 100644
index 000000000000..a0030a2b9da0
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/mutex
@@ -0,0 +1,33 @@
+#ifndef _MUTEX_
+#define _MUTEX_
+
+namespace std {
+
+struct mutex {
+  void lock() {}
+  void unlock() {}
+};
+
+template<class Lockable1, class Lockable2, class... LockableN >
+void lock(Lockable1& lock1, Lockable2& lock2, LockableN&... lockn );
+
+struct adopt_lock_t { };
+std::adopt_lock_t adopt_lock {};
+
+template <typename Mutex>
+struct lock_guard {
+  lock_guard(Mutex &m) { }
+  lock_guard(Mutex &m, std::adopt_lock_t t) {}
+  lock_guard(const lock_guard&) = delete;
+};
+
+template <typename... MutexTypes>
+struct scoped_lock {
+  scoped_lock(MutexTypes&... m) {}
+  scoped_lock(std::adopt_lock_t t, MutexTypes&... m) {}
+  scoped_lock(const scoped_lock&) = delete;
+};
+
+} // namespace std
+
+#endif // _MUTEX_
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
index 5e6f394152e9..33cf1cbea837 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.cpp
@@ -164,6 +164,69 @@ int Test2(MyConstChar* A) {
   return sum;
 }
 
+struct A {
+   int array[10];
+};
+
+struct B {
+  struct A a;
+};
+
+void loop_access_elements(int num, struct B b) {
+  struct A arr[10];
+  char buf[20];
+
+  // CHECK-MESSAGES: :[[@LINE+1]]:22: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression]
+  for(int i = 0; i < sizeof(arr); i++) {
+    struct A a = arr[i];
+  }
+
+  // Loop warning should not trigger here, even though this code is incorrect
+  // CHECK-MESSAGES: :[[@LINE+2]]:22: warning: suspicious usage of 'sizeof(K)'; did you mean 'K'? [bugprone-sizeof-expression]
+  // CHECK-MESSAGES: :[[@LINE+1]]:32: warning: suspicious usage of 'sizeof(...)/sizeof(...)'; numerator is not a multiple of denominator [bugprone-sizeof-expression] 
+  for(int i = 0; i < sizeof(10)/sizeof(A); i++) {
+    struct A a = arr[i];
+  }
+    
+  // Should not warn here
+  for(int i = 0; i < sizeof(arr)/sizeof(A); i++) {}
+
+  // Should not warn here
+  for (int i = 0; i < 10; i++) {
+    if (sizeof(arr) != 0) {
+
+    }
+  }
+
+  for (int i = 0; i < 10; i++) {
+    // CHECK-MESSAGES: :[[@LINE+1]]:25: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression]
+    for (int j = 0; j < sizeof(arr); j++) {
+    }
+  }
+
+  // CHECK-MESSAGES: :[[@LINE+1]]:22: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression]
+  for(int j = 0; j < sizeof(b.a.array); j++) {}
+  
+  // Should not warn here
+  for(int i = 0; i < sizeof(buf); i++) {} 
+
+  // Should not warn here
+  for(int i = 0; i < (sizeof(arr) << 3); i++) {}
+  
+  int i = 0;
+  // CHECK-MESSAGES: :[[@LINE+1]]:14: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression]
+  while(i <= sizeof(arr)) {i++;}
+   
+  i = 0;
+  do {
+    i++;
+  // CHECK-MESSAGES: :[[@LINE+1]]:16: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression] 
+  } while(i <= sizeof(arr));
+
+  // CHECK-MESSAGES: :[[@LINE+1]]:29: warning: suspicious usage of 'sizeof' in the loop [bugprone-sizeof-expression]
+  for(int i = 0, j = 0; i < sizeof(arr) && j < sizeof(buf); i++, j++) {}
+}
+
 template <int T>
 int Foo() { int A[T]; return sizeof(T); }
 // CHECK-MESSAGES: :[[@LINE-1]]:30: warning: suspicious usage of 'sizeof(K)'
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp
new file mode 100644
index 000000000000..7ccac1a10015
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/missing-std-forward-custom-function.cpp
@@ -0,0 +1,42 @@
+// RUN: %check_clang_tidy -std=c++14 %s cppcoreguidelines-missing-std-forward %t -- \
+// RUN: -config="{CheckOptions: {cppcoreguidelines-missing-std-forward.ForwardFunction: custom_forward}}" -- -fno-delayed-template-parsing
+
+// NOLINTBEGIN
+namespace std {
+
+template <typename T> struct remove_reference      { using type = T; };
+template <typename T> struct remove_reference<T&>  { using type = T; };
+template <typename T> struct remove_reference<T&&> { using type = T; };
+
+template <typename T> using remove_reference_t = typename remove_reference<T>::type;
+
+template <typename T> constexpr T &&forward(remove_reference_t<T> &t) noexcept;
+template <typename T> constexpr T &&forward(remove_reference_t<T> &&t) noexcept;
+template <typename T> constexpr remove_reference_t<T> &&move(T &&x);
+
+} // namespace std
+// NOLINTEND
+
+template<class T>
+constexpr decltype(auto) custom_forward(std::remove_reference_t<T>& tmp) noexcept
+{
+  return static_cast<T&&>(tmp);
+}
+
+template<class T>
+constexpr decltype(auto) custom_forward(std::remove_reference_t<T>&& tmp) noexcept
+{
+  return static_cast<T&&>(tmp);
+}
+
+template<class T>
+void forward_with_std(T&& t) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: forwarding reference parameter 't' is never forwarded inside the function body [cppcoreguidelines-missing-std-forward]
+
+  T other{std::forward<T>(t)};
+}
+
+template<class T>
+void move_with_custom(T&& t) {
+  T other{custom_forward<T>(t)};
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
index faab2a1ccf0e..b3d3fab9c540 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic-pr36489.cpp
@@ -1,11 +1,14 @@
 // RUN: %check_clang_tidy -std=c++14-or-later %s cppcoreguidelines-pro-bounds-pointer-arithmetic %t
 
 // Fix PR36489 and detect auto-deduced value correctly.
+typedef char* charPtr;
+
 char *getPtr();
 auto getPtrAuto() { return getPtr(); }
 decltype(getPtr()) getPtrDeclType();
 decltype(auto) getPtrDeclTypeAuto() { return getPtr(); }
 auto getPtrWithTrailingReturnType() -> char *;
+charPtr getCharPtr() { return getPtr(); }
 
 void auto_deduction_binary() {
   auto p1 = getPtr() + 1;
@@ -28,6 +31,10 @@ void auto_deduction_binary() {
   // CHECK-MESSAGES: :[[@LINE-1]]:31: warning: do not use pointer arithmetic
   auto *p9 = getPtrDeclTypeAuto() + 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:35: warning: do not use pointer arithmetic
+  auto p10 = getCharPtr() + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:27: warning: do not use pointer 
+  auto* p11 = getCharPtr() + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:28: warning: do not use pointer arithmetic
 }
 
 void auto_deduction_subscript() {
@@ -50,4 +57,6 @@ void auto_deduction_subscript() {
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
   auto p8 = getPtrDeclTypeAuto()[9];
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
+  auto p9 = getCharPtr()[10];
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: do not use pointer arithmetic
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
index 7cbc6ddf96ab..aed6080471e1 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
@@ -4,10 +4,13 @@ enum E {
   ENUM_LITERAL = 1
 };
 
+typedef int* IntPtr;
+
 int i = 4;
 int j = 1;
 int *p = 0;
 int *q = 0;
+IntPtr ip = 0;
 
 void fail() {
   q = p + 4;
@@ -50,6 +53,32 @@ void fail() {
 
   i = p[1];
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+
+  p = ip + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: do not use pointer arithmetic
+  ip++;
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use pointer arithmetic
+  i = ip[1];
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+}
+
+template <typename T>
+void template_fail() {
+  T* p;
+  T* q;
+
+  p = q + 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: do not use pointer arithmetic
+  q = p - 1;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: do not use pointer arithmetic
+  p++;
+  // CHECK-MESSAGES: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
+  i = p[1];
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
+}
+
+void instantiate() {
+  template_fail<int>();
 }
 
 struct S {
@@ -87,3 +116,25 @@ void okay() {
 
   for(int ii : a) ; // OK, pointer arithmetic generated by compiler
 }
+
+namespace gh126424 {
+
+namespace std {
+template <typename, typename>
+class pair {};
+
+template <typename Key, typename Value>
+class map {
+  public:
+   using value_type = pair<Key, Value>;
+   value_type& operator[](const Key& key);
+   value_type& operator[](Key&& key);
+ };
+}
+
+template <typename R>
+int f(std::map<R*, int>& map, R* r) {
+  return map[r]; // OK
+}
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp
new file mode 100644
index 000000000000..4ed29824496f
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/rvalue-reference-param-not-moved-custom-function.cpp
@@ -0,0 +1,47 @@
+// RUN: %check_clang_tidy -std=c++11 %s cppcoreguidelines-rvalue-reference-param-not-moved %t -- \
+// RUN: -config="{CheckOptions: {cppcoreguidelines-rvalue-reference-param-not-moved.AllowPartialMove: true, cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams: true, cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreNonDeducedTemplateTypes: true, cppcoreguidelines-rvalue-reference-param-not-moved.MoveFunction: custom_move}}" -- -fno-delayed-template-parsing
+
+// NOLINTBEGIN
+namespace std {
+template <typename>
+struct remove_reference;
+
+template <typename _Tp> struct remove_reference { typedef _Tp type; };
+template <typename _Tp> struct remove_reference<_Tp&> { typedef _Tp type; };
+template <typename _Tp> struct remove_reference<_Tp&&> { typedef _Tp type; };
+
+template <typename _Tp>
+constexpr typename std::remove_reference<_Tp>::type &&move(_Tp &&__t) noexcept;
+
+template <typename _Tp>
+constexpr _Tp &&
+forward(typename remove_reference<_Tp>::type &__t) noexcept;
+
+}
+// NOLINTEND
+
+
+struct Obj {
+  Obj();
+  Obj(const Obj&);
+  Obj& operator=(const Obj&);
+  Obj(Obj&&);
+  Obj& operator=(Obj&&);
+  void member() const;
+};
+
+template<class T>
+constexpr typename std::remove_reference<T>::type&& custom_move(T&& x) noexcept
+{
+    return static_cast<typename std::remove_reference<T>::type&&>(x);
+}
+
+void move_with_std(Obj&& o) {
+  // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: rvalue reference parameter 'o' is never moved from inside the function body [cppcoreguidelines-rvalue-reference-param-not-moved]
+  Obj other{std::move(o)};
+}
+
+void move_with_custom(Obj&& o) {
+  Obj other{custom_move(o)};
+}
+
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
index cdfed7edb431..fb7eeed7e715 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/confusable-identifiers.cpp
@@ -74,6 +74,47 @@ template <typename t1, typename tl>
 // CHECK-MESSAGES: :[[#@LINE-2]]:20: note: other declaration found here
 void f9();
 
+namespace different_contexts {
+  // No warning for names in unrelated contexts.
+  template <typename u1> void different_templates_1();
+  template <typename ul> void different_templates_2();
+  namespace inner {
+    int ul;
+  }
+}
+
+namespace same_template {
+  template <typename u1, typename ul> using T = int;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:35: warning: 'ul' is confusable with 'u1' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:22: note: other declaration found here
+
+  template <typename v1, typename vl> int n;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:35: warning: 'vl' is confusable with 'v1' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:22: note: other declaration found here
+
+  template <typename w1> int wl;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:22: warning: 'w1' is confusable with 'wl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-2]]:30: note: other declaration found here
+
+  int xl;
+  template <typename x1> int x;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:22: warning: 'x1' is confusable with 'xl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-3]]:7: note: other declaration found here
+}
+
+namespace f10 {
+int il;
+namespace inner {
+  int i1;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:7: warning: 'i1' is confusable with 'il' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-4]]:5: note: other declaration found here
+  int j1;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:7: warning: 'j1' is confusable with 'jl' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE+2]]:5: note: other declaration found here
+}
+int jl;
+}
+
 struct Base0 {
   virtual void mO0();
 
@@ -103,3 +144,21 @@ struct Derived1 : Base1 {
 
   long mI1(); // no warning: mII is private
 };
+
+struct Base2 {
+  long nO0;
+
+private:
+  long nII;
+};
+
+struct Mid2 : Base0, Base1, Base2 {
+};
+
+struct Derived2 : Mid2 {
+  long nOO;
+  // CHECK-MESSAGES: :[[#@LINE-1]]:8: warning: 'nOO' is confusable with 'nO0' [misc-confusable-identifiers]
+  // CHECK-MESSAGES: :[[#@LINE-12]]:8: note: other declaration found here
+
+  long nI1(); // no warning: mII is private
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
index 74a22a7c083f..28b53ae4af63 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
@@ -163,3 +163,16 @@ struct TemplateTypeAlias {
   Alias3<TypeAlias::Alias> &operator=(double) { return *this; }
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: operator=() should return 'TemplateTypeAlias&' [misc-unconventional-assign-operator]
 };
+
+namespace gh143237 {
+template<typename T>
+struct TemplateAssignment {
+  explicit TemplateAssignment(int) {
+  }
+
+  TemplateAssignment& operator=(int n) {
+    // No warning
+    return *this = TemplateAssignment(n);
+  }
+};
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp
new file mode 100644
index 000000000000..77df84575653
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scope-lock-warn-on-using-and-typedef-false.cpp
@@ -0,0 +1,31 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- \
+// RUN:   -config="{CheckOptions: {modernize-use-scoped-lock.WarnOnUsingAndTypedef: false}}" \
+// RUN:   -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+template <typename T>
+using Lock = std::lock_guard<T>;
+
+using LockM = std::lock_guard<std::mutex>;
+
+typedef std::lock_guard<std::mutex> LockDef;
+
+void PositiveUsingDecl() {
+  using std::lock_guard;
+
+  using LockMFun = std::lock_guard<std::mutex>;
+  
+  typedef std::lock_guard<std::mutex> LockDefFun;
+}
+
+template <typename T>
+void PositiveUsingDeclTemplate() {
+  using std::lock_guard;
+
+  using LockFunT = std::lock_guard<T>;
+
+  using LockMFunT = std::lock_guard<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFunT;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp
new file mode 100644
index 000000000000..7b30cdd75c80
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock-warn-on-single-locks-false.cpp
@@ -0,0 +1,102 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- \
+// RUN:   -config="{CheckOptions: {modernize-use-scoped-lock.WarnOnSingleLocks: false}}" \
+// RUN:   -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+void Positive() {
+  std::mutex m;
+
+  {
+    std::lock_guard<std::mutex> l1(m);
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m), l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:40: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:47: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:33: note: additional 'std::lock_guard' declared here
+  }
+  
+  { 
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  } 
+}
+
+void Negative() {
+  std::mutex m;
+  {
+    std::lock_guard<std::mutex> l(m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l(m, std::adopt_lock);
+  }
+
+  { 
+    std::lock_guard<std::mutex> l3(m);
+    int a = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+  }
+}
+
+void PositiveInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+}
+
+
+void NegativeInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l3(m3);
+}
+
+template <typename T>
+void PositiveTemplated() {
+  std::mutex m1, m2;
+  
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here  
+}
+
+template <typename T>
+void NegativeTemplated() {
+  std::mutex m1, m2, m3;
+  std::lock_guard<std::mutex> l(m1);
+}
+
+template <typename Mutex>
+void PositiveTemplatedMutex() {
+  Mutex m1, m2;
+
+  std::lock_guard<Mutex> l1(m1);
+  std::lock_guard<Mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:26: note: additional 'std::lock_guard' declared here
+}
+
+template <typename Mutex>
+void NegativeTemplatedMutex() {
+  Mutex m1;
+  std::lock_guard<Mutex> l(m1);
+}
+
+struct NegativeClass {
+  void Negative() {
+    std::lock_guard<std::mutex> l(m1);
+  }
+  
+  std::mutex m1;
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp
new file mode 100644
index 000000000000..45eabdbece68
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-scoped-lock.cpp
@@ -0,0 +1,471 @@
+// RUN: %check_clang_tidy -std=c++17-or-later %s modernize-use-scoped-lock %t -- -- -isystem %clang_tidy_headers -fno-delayed-template-parsing
+
+#include <mutex>
+
+void Positive() {
+  std::mutex m;
+  {
+    std::lock_guard<std::mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(std::adopt_lock, m);
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m);
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:40: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m), l2(m), l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:40: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:47: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:33: note: additional 'std::lock_guard' declared here
+  }
+  
+  { 
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m);
+    int b = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l4(std::adopt_lock, m);
+  } 
+}
+
+
+std::mutex p_m1;
+void PositiveShortFunction() {
+  std::lock_guard<std::mutex> l(p_m1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(p_m1);
+}
+
+
+void PositiveNested() {
+  std::mutex m1;
+  if (true) {
+    std::lock_guard<std::mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m1);
+    {
+      std::lock_guard<std::mutex> l2(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l2(m1);
+      {
+        std::lock_guard<std::mutex> l3(m1);
+        std::lock_guard<std::mutex> l4(m1);
+        // CHECK-MESSAGES: :[[@LINE-2]]:9: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+        // CHECK-MESSAGES: :[[@LINE-2]]:37: note: additional 'std::lock_guard' declared here
+      }
+      {
+        std::lock_guard<std::mutex> l2(m1);
+        // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+        // CHECK-FIXES: std::scoped_lock l2(m1);
+      }
+    }
+  }
+  std::lock_guard<std::mutex> l(m1);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(m1);
+}
+
+
+void PositiveInsideArg(std::mutex &m1, std::mutex &m2, std::mutex &m3) {
+  std::lock_guard<std::mutex> l1(m1);
+  std::lock_guard<std::mutex> l2(m2);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+  int a = 0;
+  std::lock_guard<std::mutex> l3(m3);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l3(m3);
+}
+
+
+void PositiveInsideConditional() {
+  std::mutex m1;
+  if (true) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  } else {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+
+  while (true) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+
+  for (int i = 0; i < 10; ++i) {
+    std::lock_guard<std::mutex> l1(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m1);
+  }
+}
+
+void PositiveLambda() {
+  std::mutex m;
+  std::lock_guard<std::mutex> l1(m);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l1(m);
+  auto lambda1 = [&]() {
+    std::lock_guard<std::mutex> l1(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m);
+  };
+
+  std::lock_guard<std::mutex> l3(m);
+  std::lock_guard<std::mutex> l4(m);
+  // CHECK-MESSAGES: :[[@LINE-2]]:3: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+  // CHECK-MESSAGES: :[[@LINE-2]]:31: note: additional 'std::lock_guard' declared here
+  auto lamda2 = [&]() {
+    std::lock_guard<std::mutex> l3(m);
+    std::lock_guard<std::mutex> l4(m);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  };
+
+  auto lamda3 = [&]() {
+    std::lock(m, m);
+    std::lock_guard<std::mutex> l1(m, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m);
+    int b = 0;
+    std::lock_guard<std::mutex> l4(m, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l4(std::adopt_lock, m);
+  };
+
+  auto lamda4 = [&]() {
+    std::lock_guard<std::mutex> l1(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l1(m);
+    int a = 0;
+    std::lock_guard<std::mutex> l2(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l2(m);
+  };
+}
+
+template <typename T>
+void PositiveTemplated() {
+  std::mutex m1, m2, m3;
+  {
+    std::lock_guard<std::mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m1);
+  }
+
+  {
+    std::lock_guard<std::mutex> l1(m1);
+    std::lock_guard<std::mutex> l2(m2);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock(m1, m2);
+    std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+    std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:33: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<std::mutex> l3(m3);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l3(m3);
+  }
+}
+
+
+template <typename Mutex>
+void PositiveTemplatedMutex() {
+  Mutex m1, m2, m3;
+  {
+    std::lock_guard<Mutex> l(m1);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+
+  {
+    std::lock_guard<Mutex> l1(m1);
+    std::lock_guard<Mutex> l2(m2);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:28: note: additional 'std::lock_guard' declared here
+  }
+
+  {
+    std::lock(m1, m2);
+    std::lock_guard<Mutex> l1(m1, std::adopt_lock);
+    std::lock_guard<Mutex> l2(m2, std::adopt_lock);
+    // CHECK-MESSAGES: :[[@LINE-2]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-2]]:28: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    std::lock_guard<Mutex> l3(m3);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+}
+
+
+template <template <typename> typename Lock>
+void NegativeTemplate() {
+  std::mutex m1, m2;
+  {
+    Lock<std::mutex> l(m1);
+  }
+
+  {
+    Lock<std::mutex> l1(m1);
+    Lock<std::mutex> l2(m2);
+  }
+}
+
+void instantiate() {
+  NegativeTemplate<std::lock_guard>();
+}
+
+
+struct PositiveClass {
+  void Positive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock_guard<std::mutex> l1(m1);
+      std::lock_guard<std::mutex> l2(m2);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+  
+  std::mutex m1;
+  std::mutex m2;
+  std::mutex m3;
+};
+
+
+template <typename T>
+struct PositiveTemplatedClass {
+  void Positive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+
+  template <typename... Ts>
+  void TemplatedPositive() {
+    {
+      std::lock_guard<std::mutex> l(m1);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l(m1);
+    }
+
+    {
+      std::lock(m1, m2);  
+      std::lock_guard<std::mutex> l1(m1, std::adopt_lock);
+      std::lock_guard<std::mutex> l2(m2, std::adopt_lock);
+      // CHECK-MESSAGES: :[[@LINE-2]]:7: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+      // CHECK-MESSAGES: :[[@LINE-2]]:35: note: additional 'std::lock_guard' declared here
+      int a = 0;
+      std::lock_guard<std::mutex> l3(m3);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+      // CHECK-FIXES: std::scoped_lock l3(m3);
+    }
+  }
+  
+  std::mutex m1;
+  std::mutex m2;
+  std::mutex m3;
+};
+
+
+template <typename T>
+using Lock = std::lock_guard<T>;
+// CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: using Lock = std::scoped_lock<T>;
+
+using LockM = std::lock_guard<std::mutex>;
+// CHECK-MESSAGES: :[[@LINE-1]]:15: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: using LockM = std::scoped_lock<std::mutex>;
+
+typedef std::lock_guard<std::mutex> LockDef;
+// CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+// CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDef;
+
+
+void PositiveUsingDecl() {
+  using std::lock_guard;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using std::scoped_lock;
+
+  using LockMFun = std::lock_guard<std::mutex>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockMFun = std::scoped_lock<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFun;
+  // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDefFun;
+}
+
+template <typename T>
+void PositiveUsingDeclTemplate() {
+  using std::lock_guard;
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using std::scoped_lock;
+
+  std::mutex m;
+  lock_guard<std::mutex> l(m);
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: std::scoped_lock l(m);
+
+  using LockFunT = std::lock_guard<T>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockFunT = std::scoped_lock<T>;
+
+  using LockMFunT = std::lock_guard<std::mutex>;
+  // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: using LockMFunT = std::scoped_lock<std::mutex>;
+
+  typedef std::lock_guard<std::mutex> LockDefFunT;
+  // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  // CHECK-FIXES: typedef std::scoped_lock<std::mutex> LockDefFunT;
+}
+
+void PositiveInUsingTypedefs() {
+  std::mutex m;
+
+  {
+    Lock<std::mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    LockM l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    LockDef l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l(m);
+  }
+
+  {
+    std::lock(m, m);
+    Lock<std::mutex> l1(m, std::adopt_lock);
+    LockM l2(m, std::adopt_lock);
+    LockDef l3(m), l4(m);
+    // CHECK-MESSAGES: :[[@LINE-3]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:11: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-3]]:13: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:20: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    LockDef l5(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+    // CHECK-FIXES: std::scoped_lock l5(m);
+  }
+}
+
+template <typename Mutex>
+void PositiveInUsingTypedefsTemplated() {
+  Mutex m;
+
+  {
+    Lock<Mutex> l(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+
+  {
+    std::lock(m, m);
+    Lock<Mutex> l1(m, std::adopt_lock);
+    LockM l2(m, std::adopt_lock);
+    LockDef l3(m), l4(m);
+    // CHECK-MESSAGES: :[[@LINE-3]]:5: warning: use single 'std::scoped_lock' instead of multiple 'std::lock_guard'
+    // CHECK-MESSAGES: :[[@LINE-3]]:11: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-3]]:13: note: additional 'std::lock_guard' declared here
+    // CHECK-MESSAGES: :[[@LINE-4]]:20: note: additional 'std::lock_guard' declared here
+    int a = 0;
+    LockDef l5(m);
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use 'std::scoped_lock' instead of 'std::lock_guard'
+  }
+}
+
+// Non-STD lock_guard.
+template <typename Mutex>
+struct lock_guard {
+  lock_guard(Mutex &m) { }
+  lock_guard(const lock_guard& ) = delete;
+};
+
+void NegativeNonStdLockGuard() {
+  std::mutex m;
+  {
+    lock_guard<std::mutex> l(m);
+  }
+
+  {
+    lock_guard<std::mutex> l1(m);
+    lock_guard<std::mutex> l2(m);
+  }
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
index 14f9e88f7e72..882ce640b13c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/redundant-inline-specifier.cpp
@@ -149,3 +149,13 @@ class A
   // CHECK-FIXES-STRICT: static float test4;
 };
 }
+
+namespace ns {
+class B
+{
+public:
+  ~B();
+};
+
+inline B::~B() = default;
+}
diff --git a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
index a38dfd303660..d88b0c91bb9a 100644
--- a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
@@ -39,6 +39,10 @@ static std::string writeInfo(Info *I) {
     return writeInfo(*static_cast<TypedefInfo *>(I));
   case InfoType::IT_concept:
     return writeInfo(*static_cast<ConceptInfo *>(I));
+  case InfoType::IT_variable:
+    return writeInfo(*static_cast<VarInfo *>(I));
+  case InfoType::IT_friend:
+    return writeInfo(*static_cast<FriendInfo *>(I));
   case InfoType::IT_default:
     return "";
   }
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index e0da146b6c5a..cc408dfe1767 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -150,6 +150,12 @@ if(CLANG_BUILT_STANDALONE)
     endif()
 
     umbrella_lit_testsuite_begin(check-all)
+
+    # If LLVM was built with Z3, check if we still have the headers around.
+    # They are used by a few tests.
+    if (LLVM_WITH_Z3)
+      find_package(Z3 4.8.9)
+    endif()
   endif() # LLVM_INCLUDE_TESTS
 endif() # standalone
 
diff --git a/clang/bindings/python/tests/cindex/test_file.py b/clang/bindings/python/tests/cindex/test_file.py
index a8c1dbf55854..2be9b9e33261 100644
--- a/clang/bindings/python/tests/cindex/test_file.py
+++ b/clang/bindings/python/tests/cindex/test_file.py
@@ -9,6 +9,7 @@
 
 inputs_dir = os.path.join(os.path.dirname(__file__), "INPUTS")
 
+
 class TestFile(unittest.TestCase):
     def test_file(self):
         index = Index.create()
diff --git a/clang/docs/APINotes.rst b/clang/docs/APINotes.rst
index d20c4f9b5ba8..e5ec154237b0 100644
--- a/clang/docs/APINotes.rst
+++ b/clang/docs/APINotes.rst
@@ -42,7 +42,7 @@ PrivateHeaders directory as well, though it does not need an additional
 "_private" suffix on its name.
 
 Clang will search for API notes files next to module maps only when passed the
-``-fapi-notes-modules`` option.
+``-fapinotes-modules`` option.
 
 
 Limitations
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 548c73af6587..c61c80883170 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -1702,9 +1702,9 @@ the configuration (without a prefix: ``Auto``).
 .. _AllowAllArgumentsOnNextLine:
 
 **AllowAllArgumentsOnNextLine** (``Boolean``) :versionbadge:`clang-format 9` :ref:`¶ <AllowAllArgumentsOnNextLine>`
-  If a function call or braced initializer list doesn't fit on a
-  line, allow putting all arguments onto the next line, even if
-  ``BinPackArguments`` is ``false``.
+  If a function call or braced initializer list doesn't fit on a line, allow
+  putting all arguments onto the next line, even if ``BinPackArguments`` is
+  ``false``.
 
   .. code-block:: c++
 
@@ -5528,8 +5528,7 @@ the configuration (without a prefix: ``Auto``).
 .. _ReferenceAlignment:
 
 **ReferenceAlignment** (``ReferenceAlignmentStyle``) :versionbadge:`clang-format 13` :ref:`¶ <ReferenceAlignment>`
-  Reference alignment style (overrides ``PointerAlignment`` for
-  references).
+  Reference alignment style (overrides ``PointerAlignment`` for references).
 
   Possible values:
 
@@ -6885,8 +6884,8 @@ the configuration (without a prefix: ``Auto``).
 .. _TypenameMacros:
 
 **TypenameMacros** (``List of Strings``) :versionbadge:`clang-format 9` :ref:`¶ <TypenameMacros>`
-  A vector of macros that should be interpreted as type declarations
-  instead of as function calls.
+  A vector of macros that should be interpreted as type declarations instead
+  of as function calls.
 
   These are expected to be macros of the form:
 
diff --git a/clang/docs/DebuggingCoroutines.rst b/clang/docs/DebuggingCoroutines.rst
index 80df32134072..8702152747c6 100644
--- a/clang/docs/DebuggingCoroutines.rst
+++ b/clang/docs/DebuggingCoroutines.rst
@@ -8,288 +8,735 @@ Debugging C++ Coroutines
 Introduction
 ============
 
-For performance and other architectural reasons, the C++ Coroutines feature in
-the Clang compiler is implemented in two parts of the compiler.  Semantic
-analysis is performed in Clang, and Coroutine construction and optimization
-takes place in the LLVM middle-end.
+Coroutines in C++ were introduced in C++20, and the user experience for
+debugging them can still be challenging. This document guides you how to most
+efficiently debug coroutines and how to navigate existing shortcomings in
+debuggers and compilers.
+
+Coroutines are generally used either as generators or for asynchronous
+programming. In this document, we will discuss both use cases. Even if you are
+using coroutines for asynchronous programming, you should still read the
+generators section, as it will introduce foundational debugging techniques also
+applicable to the debugging of asynchronous programs.
+
+Both compilers (clang, gcc, ...) and debuggers (lldb, gdb, ...) are
+still improving their support for coroutines. As such, we recommend using the
+latest available version of your toolchain.
+
+This document focuses on clang and lldb. The screenshots show
+`lldb-dap <https://marketplace.visualstudio.com/items?itemName=llvm-vs-code-extensions.lldb-dap>`_
+in combination with VS Code. The same techniques can also be used in other
+IDEs.
+
+Debugging clang-compiled binaries with gdb is possible, but requires more
+scripting. This guide comes with a basic GDB script for coroutine debugging.
+
+This guide will first showcase the more polished, bleeding-edge experience, but
+will also show you how to debug coroutines with older toolchains. In general,
+the older your toolchain, the deeper you will have to dive into the
+implementation details of coroutines (such as their ABI). The further down in
+this document you go, the more low-level, technical the content will become. If
+you are on an up-to-date toolchain, you will hopefully be able to stop reading
+earlier.
+
+Debugging generators
+====================
+
+One of the two major use cases for coroutines in C++ are generators, i.e.,
+functions which can produce values via ``co_yield``. Values are produced
+lazily, on-demand. For this purpose, every time a new value is requested, the
+coroutine gets resumed. As soon as it reaches a ``co_yield`` and thereby
+returns the requested value, the coroutine is suspended again.
+
+This logic is encapsulated in a ``generator`` type similar to this one:
 
-However, this design forces us to generate insufficient debugging information.
-Typically, the compiler generates debug information in the Clang frontend, as
-debug information is highly language specific. However, this is not possible
-for Coroutine frames because the frames are constructed in the LLVM middle-end.
-
-To mitigate this problem, the LLVM middle end attempts to generate some debug
-information, which is unfortunately incomplete, since much of the language
-specific information is missing in the middle end.
+.. code-block:: c++
 
-This document describes how to use this debug information to better debug
-coroutines.
+  // generator.hpp
+  #include <coroutine>
 
-Terminology
-===========
+  // `generator` is a stripped down, minimal generator type.
+  template<typename T>
+  struct generator {
+    struct promise_type {
+      T current_value{};
 
-Due to the recent nature of C++20 Coroutines, the terminology used to describe
-the concepts of Coroutines is not settled.  This section defines a common,
-understandable terminology to be used consistently throughout this document.
+      auto get_return_object() {
+        return std::coroutine_handle<promise_type>::from_promise(*this);
+      }
+      auto initial_suspend() { return std::suspend_always(); }
+      auto final_suspend() noexcept { return std::suspend_always(); }
+      auto return_void() { return std::suspend_always(); }
+      void unhandled_exception() { __builtin_unreachable(); }
+      auto yield_value(T v) {
+        current_value = v;
+        return std::suspend_always();
+      }
+    };
 
-coroutine type
---------------
+    generator(std::coroutine_handle<promise_type> h) : hdl(h) { hdl.resume(); }
+    ~generator() { hdl.destroy(); }
 
-A `coroutine function` is any function that contains any of the Coroutine
-Keywords `co_await`, `co_yield`, or `co_return`.  A `coroutine type` is a
-possible return type of one of these `coroutine functions`.  `Task` and
-`Generator` are commonly referred to coroutine types.
+    generator<T>& operator++() { hdl.resume(); return *this; } // resume the coroutine
+    T operator*() const { return hdl.promise().current_value; }
 
-coroutine
----------
+    private:
+    std::coroutine_handle<promise_type> hdl;
+  };
 
-By technical definition, a `coroutine` is a suspendable function. However,
-programmers typically use `coroutine` to refer to an individual instance.
-For example:
+We can then use this ``generator`` class to print the Fibonacci sequence:
 
 .. code-block:: c++
 
-  std::vector<Task> Coros; // Task is a coroutine type.
-  for (int i = 0; i < 3; i++)
-    Coros.push_back(CoroTask()); // CoroTask is a coroutine function, which
-                                 // would return a coroutine type 'Task'.
+  #include "generator.hpp"
+  #include <iostream>
 
-In practice, we typically say "`Coros` contains 3 coroutines" in the above
-example, though this is not strictly correct.  More technically, this should
-say "`Coros` contains 3 coroutine instances" or "Coros contains 3 coroutine
-objects."
+  generator<int> fibonacci() {
+    co_yield 0;
+    int prev = 0;
+    co_yield 1;
+    int current = 1;
+    while (true) {
+      int next = current + prev;
+      co_yield next;
+      prev = current;
+      current = next;
+    }
+  }
 
-In this document, we follow the common practice of using `coroutine` to refer
-to an individual `coroutine instance`, since the terms `coroutine instance` and
-`coroutine object` aren't sufficiently defined in this case.
+  template<typename T>
+  void print10Elements(generator<T>& gen) {
+    for (unsigned i = 0; i < 10; ++i) {
+      std::cerr << *gen << "\n";
+      ++gen;
+    }
+  }
 
-coroutine frame
----------------
+  int main() {
+    std::cerr << "Fibonacci sequence - here we go\n";
+    generator<int> fib = fibonacci();
+    for (unsigned i = 0; i < 5; ++i) {
+      ++fib;
+    }
+    print10Elements(fib);
+  }
 
-The C++ Standard uses `coroutine state` to describe the allocated storage. In
-the compiler, we use `coroutine frame` to describe the generated data structure
-that contains the necessary information.
+To compile this code, use ``clang++ --std=c++23 generator-example.cpp -g``.
 
-The structure of coroutine frames
-=================================
+Breakpoints inside the generators
+---------------------------------
 
-The structure of coroutine frames is defined as:
+We can set breakpoints inside coroutines just as we set them in regular
+functions. For VS Code, that means clicking next the line number in the editor.
+In the ``lldb`` CLI or in ``gdb``, you can use ``b`` to set a breakpoint.
 
-.. code-block:: c++
+Inspecting variables in a coroutine
+-----------------------------------
 
-  struct {
-    void (*__r)(); // function pointer to the `resume` function
-    void (*__d)(); // function pointer to the `destroy` function
-    promise_type; // the corresponding `promise_type`
-    ... // Any other needed information
-  }
+If you hit a breakpoint inside the ``fibonacci`` function, you should be able
+to inspect all local variables (``prev``, ``current``, ``next``) just like in
+a regular function.
 
-In the debugger, the function's name is obtainable from the address of the
-function. And the name of `resume` function is equal to the name of the
-coroutine function. So the name of the coroutine is obtainable once the
-address of the coroutine is known.
+.. image:: ./coro-generator-variables.png
 
-Print promise_type
-==================
+Note the two additional variables ``__promise`` and ``__coro_frame``. Those
+show the internal state of the coroutine. They are not relevant for our
+generator example, but will be relevant for asynchronous programming described
+in the next section.
 
-Every coroutine has a `promise_type`, which defines the behavior
-for the corresponding coroutine. In other words, if two coroutines have the
-same `promise_type`, they should behave in the same way.
-To print a `promise_type` in a debugger when stopped at a breakpoint inside a
-coroutine, printing the `promise_type` can be done by:
+Stepping out of a coroutine
+---------------------------
 
-.. parsed-literal::
+When single-stepping, you will notice that the debugger will leave the
+``fibonacci`` function as soon as you hit a ``co_yield`` statement. You might
+find yourself inside some standard library code. After stepping out of the
+library code, you will be back in the ``main`` function.
 
-  print __promise
+Stepping into a coroutine
+-------------------------
 
-It is also possible to print the `promise_type` of a coroutine from the address
-of the coroutine frame. For example, if the address of a coroutine frame is
-0x416eb0, and the type of the `promise_type` is `task::promise_type`, printing
-the `promise_type` can be done by:
+If you stop at ``++fib`` and try to step into the generator, you will first
+find yourself inside ``operator++``. Stepping into the ``handle.resume()`` will
+not work by default.
 
-.. parsed-literal::
+This is because lldb does not step into functions from the standard library by
+default. To make this work, you first need to run ``settings set
+target.process.thread.step-avoid-regexp ""``. You can do so from the "Debug
+Console" towards the bottom of the screen. With that setting change, you can
+step through ``coroutine_handle::resume`` and into your generator.
 
-  print (task::promise_type)*(0x416eb0+0x10)
+You might find yourself at the top of the coroutine at first, instead of at
+your previous suspension point. In that case, single-step and you will arrive
+at the previously suspended ``co_yield`` statement.
 
-This is possible because the `promise_type` is guaranteed by the ABI to be at a
-16 bit offset from the coroutine frame.
 
-Note that there is also an ABI independent method:
+Inspecting a suspended coroutine
+--------------------------------
 
-.. parsed-literal::
+The ``print10Elements`` function receives an opaque ``generator`` type. Let's
+assume we are suspended at the ``++gen;`` line, and want to inspect the
+generator and its internal state.
 
-  print std::coroutine_handle<task::promise_type>::from_address((void*)0x416eb0).promise()
+To do so, we can simply look into the ``gen.hdl`` variable. LLDB comes with a
+pretty printer for ``std::coroutine_handle`` which will show us the internal
+state of the coroutine. For GDB, you will have to use the ``show-coro-frame``
+command provided by the :ref:`gdb-script`.
 
-The functions `from_address(void*)` and `promise()` are often small enough to
-be removed during optimization, so this method may not be possible.
+.. image:: ./coro-generator-suspended.png
 
-Print coroutine frames
-======================
+We can see two function pointers ``resume`` and ``destroy``. These pointers
+point to the resume / destroy functions. By inspecting those function pointers,
+we can see that our ``generator`` is actually backed by our ``fibonacci``
+coroutine. When using VS Code + lldb-dap, you can Cmd+Click on the function
+address (``0x555...`` in the screenshot) to directly jump to the function
+definition backing your coroutine handle.
 
-LLVM generates the debug information for the coroutine frame in the LLVM middle
-end, which permits printing of the coroutine frame in the debugger. Much like
-the `promise_type`, when stopped at a breakpoint inside a coroutine we can
-print the coroutine frame by:
+Next, we see the ``promise``. In our case, this reveals the current value of
+our generator.
 
-.. parsed-literal::
+The ``coro_frame`` member represents the internal state of the coroutine. It
+contains our internal coroutine state ``prev``, ``current``, ``next``.
+Furthermore, it contains many internal, compiler-specific members, which are
+named based on their type. These represent temporary values which the compiler
+decided to spill across suspension points, but which were not declared in our
+original source code and hence have no proper user-provided name.
 
-  print __coro_frame
+Tracking the exact suspension point
+-----------------------------------
 
+Among the compiler-generated members, the ``__coro_index`` is particularly
+important. This member identifies the suspension point at which the coroutine
+is currently suspended.
 
-Just as printing the `promise_type` is possible from the coroutine address,
-printing the details of the coroutine frame from an address is also possible:
+However, it is non-trivial to map this number back to a source code location.
+The compiler emits debug info labels for the suspension points. This allows us
+to map the suspension point index back to a source code location. In gdb, we
+can use the ``info line`` command to get the source code location of the
+suspension point.
 
 ::
 
-  (gdb) # Get the address of coroutine frame
-  (gdb) print/x *0x418eb0
-  $1 = 0x4019e0
-  (gdb) # Get the linkage name for the coroutine
-  (gdb) x 0x4019e0
-  0x4019e0 <_ZL9coro_taski>:  0xe5894855
-  (gdb) # Turn off the demangler temporarily to avoid the debugger misunderstanding the name.
-  (gdb) set demangle-style none
-  (gdb) # The coroutine frame type is 'linkage_name.coro_frame_ty'
-  (gdb) print  ('_ZL9coro_taski.coro_frame_ty')*(0x418eb0)
-  $2 = {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {...}, ...}
+  (gdb) info line -function coro_task -label __coro_resume_2
+  Line 45 of "llvm-example.cpp" starts at address 0x1b1b <_ZL9coro_taski.resume+555> and ends at 0x1b46 <_ZL9coro_taski.resume+598>.
+  Line 45 of "llvm-example.cpp" starts at address 0x201b <_ZL9coro_taski.destroy+555> and ends at 0x2046 <_ZL9coro_taski.destroy+598>.
+  Line 45 of "llvm-example.cpp" starts at address 0x253b <_ZL9coro_taski.cleanup+555> and ends at 0x2566 <_ZL9coro_taski.cleanup+598>.
+
+LLDB does not support looking up labels. Furthermore, those labels are only emitted
+starting with clang 21.0.
+
+For simple cases, you might still be able to guess the suspension point correctly.
+Alternatively, you might also want to modify your coroutine library to store
+the line number of the current suspension point in the promise:
+
+.. code-block:: c++
 
-The above is possible because:
+  // For all promise_types we need a new `_coro_return_address` variable:
+  class promise_type {
+    ...
+    void* _coro_return_address = nullptr;
+  };
 
-(1) The name of the debug type of the coroutine frame is the `linkage_name`,
-plus the `.coro_frame_ty` suffix because each coroutine function shares the
-same coroutine type.
+  // For all the awaiter types we need:
+  class awaiter {
+    ...
+    template <typename Promise>
+    __attribute__((noinline)) auto await_suspend(std::coroutine_handle<Promise> handle) {
+          ...
+          handle.promise()._coro_return_address = __builtin_return_address(0);
+    }
+  };
 
-(2) The coroutine function name is accessible from the address of the coroutine
-frame.
+This stores the return address of ``await_suspend`` within the promise.
+Thereby, we can read it back from the promise of a suspended coroutine, and map
+it to an exact source code location. For a complete example, see the ``task``
+type used below for asynchronous programming.
 
-The above commands can be simplified by placing them in debug scripts.
+Alternatively, we can modify the C++ code to store the line number in the
+promise type. We can use a ``std::source_location`` to get the line number of
+the await and store it inside the ``promise_type``. In the debugger, we can
+then read the line number from the promise of the suspended coroutine.
 
-Examples to print coroutine frames
-----------------------------------
+.. code-block:: c++
+
+  // For all the awaiter types we need:
+  class awaiter {
+    ...
+    template <typename Promise>
+    void await_suspend(std::coroutine_handle<Promise> handle,
+                       std::source_location sl = std::source_location::current()) {
+          ...
+          handle.promise().line_number = sl.line();
+    }
+  };
+
+The downside of both approaches is that they come at the price of additional
+runtime cost. In particular the second approach increases binary size, since it
+requires additional ``std::source_location`` objects, and those source
+locations are not stripped by split-dwarf. Whether the first approach is worth
+the additional runtime cost is a trade-off you need to make yourself.
 
-The print examples below use the following definition:
+Async stack traces
+==================
+
+Besides generators, the second common use case for coroutines in C++ is
+asynchronous programming, usually involving libraries such as stdexec, folly,
+cppcoro, boost::asio, or similar libraries. Some of those libraries already
+provide custom debugging support, so in addition to this guide, you might want
+to check out their documentation.
+
+When using coroutines for asynchronous programming, your library usually
+provides you some ``task`` type. This type usually looks similar to this:
 
 .. code-block:: c++
 
+  // async-task-library.hpp
   #include <coroutine>
-  #include <iostream>
+  #include <utility>
 
-  struct task{
+  struct task {
     struct promise_type {
       task get_return_object() { return std::coroutine_handle<promise_type>::from_promise(*this); }
-      std::suspend_always initial_suspend() { return {}; }
-      std::suspend_always final_suspend() noexcept { return {}; }
-      void return_void() noexcept {}
+      auto initial_suspend() { return std::suspend_always{}; }
+
       void unhandled_exception() noexcept {}
 
-      int count = 0;
-    };
+      auto final_suspend() noexcept {
+        struct FinalSuspend {
+          std::coroutine_handle<> continuation;
+          auto await_ready() noexcept { return false; }
+          auto await_suspend(std::coroutine_handle<> handle) noexcept {
+            return continuation;
+          }
+          void await_resume() noexcept {}
+        };
+        return FinalSuspend{continuation};
+      }
 
-    void resume() noexcept {
-      handle.resume();
-    }
+      void return_value(int res) { result = res; }
+
+      std::coroutine_handle<> continuation = std::noop_coroutine();
+      int result = 0;
+      #ifndef NDEBUG
+      void* _coro_suspension_point_addr = nullptr;
+      #endif
+    };
 
-    task(std::coroutine_handle<promise_type> hdl) : handle(hdl) {}
+    task(std::coroutine_handle<promise_type> handle) : handle(handle) {}
     ~task() {
       if (handle)
         handle.destroy();
     }
 
-    std::coroutine_handle<> handle;
-  };
-
-  class await_counter : public std::suspend_always {
-    public:
-      template<class PromiseType>
-      void await_suspend(std::coroutine_handle<PromiseType> handle) noexcept {
-          handle.promise().count++;
+    struct Awaiter {
+      std::coroutine_handle<promise_type> handle;
+      auto await_ready() { return false; }
+
+      template <typename P>
+      #ifndef NDEBUG
+      __attribute__((noinline))
+      #endif
+      auto await_suspend(std::coroutine_handle<P> continuation) {
+        handle.promise().continuation = continuation;
+        #ifndef NDEBUG
+        continuation.promise()._coro_suspension_point_addr = __builtin_return_address(0);
+        #endif
+        return handle;
       }
+      int await_resume() {
+        return handle.promise().result;
+      }
+    };
+
+    auto operator co_await() {
+      return Awaiter{handle};
+    }
+
+    int syncStart() {
+      handle.resume();
+      return handle.promise().result;
+    }
+
+  private:
+    std::coroutine_handle<promise_type> handle;
   };
 
-  static task coro_task(int v) {
-    int a = v;
-    co_await await_counter{};
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
-    co_await await_counter{};
-    a++;
-    std::cout << a << "\n";
-    a++;
-    std::cout << a << "\n";
+Note how the ``task::promise_type`` has a member variable
+``std::coroutine_handle<> continuation``. This is the handle of the coroutine
+that will be resumed when the current coroutine is finished executing (see
+``final_suspend``). In a sense, this is the "return address" of the coroutine.
+It is set inside ``operator co_await`` when another coroutine calls our
+generator and awaits for the next value to be produced.
+
+The result value is returned via the ``int result`` member. It is written in
+``return_value`` and read by ``Awaiter::await_resume``. Usually, the result
+type of a task is a template argument. For simplicity's sake, we hard-coded the
+``int`` type in this example.
+
+Stack traces of in-flight coroutines
+------------------------------------
+
+Let's assume you have the following program and set a breakpoint inside the
+``write_output`` function. There are multiple call paths through which this
+function could have been reached. How can we find out said call path?
+
+.. code-block:: c++
+
+  #include <iostream>
+  #include <string_view>
+  #include "async-task-library.hpp"
+
+  static task write_output(std::string_view contents) {
+    std::cout << contents << "\n";
+    co_return contents.size();
+  }
+
+  static task greet() {
+    int bytes_written = 0;
+    bytes_written += co_await write_output("Hello");
+    bytes_written += co_await write_output("World");
+    co_return bytes_written;
   }
 
   int main() {
-    task t = coro_task(43);
-    t.resume();
-    t.resume();
-    t.resume();
+    int bytes_written = greet().syncStart();
+    std::cout << "Bytes written: " << bytes_written << "\n";
     return 0;
   }
 
-In debug mode (`O0` + `g`), the printing result would be:
+To do so, let's break inside ``write_output``. We can understand our call-stack
+by looking into the special ``__promise`` variable. This artificial variable is
+generated by the compiler and points to the ``promise_type`` instance
+corresponding to the currently in-flight coroutine. In this case, the
+``__promise`` variable contains the ``continuation`` which points to our
+caller. That caller again contains a ``promise`` with a ``continuation`` which
+points to our caller's caller.
 
-.. parsed-literal::
+.. image:: ./coro-async-task-continuations.png
 
-  {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {count = 1}, v = 43, a = 45, __coro_index = 1 '\001', struct_std__suspend_always_0 = {__int_8 = 0 '\000'},
-    class_await_counter_1 = {__int_8 = 0 '\000'}, class_await_counter_2 = {__int_8 = 0 '\000'}, struct_std__suspend_always_3 = {__int_8 = 0 '\000'}}
+We can figure out the involved coroutine functions and their current suspension
+points as discussed above in the "Inspecting a suspended coroutine" section.
 
-In the above, the values of `v` and `a` are clearly expressed, as are the
-temporary values for `await_counter` (`class_await_counter_1` and
-`class_await_counter_2`) and `std::suspend_always` (
-`struct_std__suspend_always_0` and `struct_std__suspend_always_3`). The index
-of the current suspension point of the coroutine is emitted as `__coro_index`.
-In the above example, the `__coro_index` value of `1` means the coroutine
-stopped at the second suspend point (Note that `__coro_index` is zero indexed)
-which is the first `co_await await_counter{};` in `coro_task`. Note that the
-first initial suspend point is the compiler generated
-`co_await promise_type::initial_suspend()`.
+When using LLDB's CLI, the command ``p --ptr-depth 4 __promise`` might also be
+useful to automatically dereference all the pointers up to the given depth.
 
-However, when optimizations are enabled, the printed result changes drastically:
+To get a flat representation of that call stack, we can use a debugger script,
+such as the one shown in the :ref:`lldb-script` section. With that
+script, we can run ``coro bt`` to get the following stack trace:
 
-.. parsed-literal::
+.. code-block::
 
-  {__resume_fn = 0x401280 <coro_task(int)>, __destroy_fn = 0x401390 <coro_task(int)>, __promise = {count = 1}, __int_32_0 = 43, __coro_index = 1 '\001'}
+  (lldb) coro bt
+  frame #0: write_output(std::basic_string_view<char, std::char_traits<char>>) at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:6:16
+  [async] frame #1: greet() at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:12:20
+  [async] frame #2: std::__n4861::coroutine_handle<std::__n4861::noop_coroutine_promise>::__frame::__dummy_resume_destroy() at /usr/include/c++/14/coroutine:298, suspension point unknown
+  frame #3: std::__n4861::coroutine_handle<task::promise_type>::resume() const at /usr/include/c++/14/coroutine:242:29
+  frame #4: task::syncStart() at /home/avogelsgesang/Documents/corotest/async-task-library.hpp:78:14
+  frame #5: main at /home/avogelsgesang/Documents/corotest/async-task-example.cpp:18:11
+  frame #6: __libc_start_call_main at sysdeps/nptl/libc_start_call_main.h:58:16
+  frame #7: __libc_start_main_impl at csu/libc-start.c:360:3
+  frame #8: _start at :4294967295
 
-Unused values are optimized out, as well as the name of the local variable `a`.
-The only information remained is the value of a 32-bit integer. In this simple
-case, it seems to be pretty clear that `__int_32_0` represents `a`. However, it
-is not true.
+Note how the frames #1 and #2 are async frames.
 
-An important note with optimization is that the value of a variable may not
-properly express the intended value in the source code.  For example:
+The ``coro bt`` command already includes logic to identify the exact suspension
+point of each frame based on the ``_coro_suspension_point_addr`` stored inside
+the promise.
+
+Stack traces of suspended coroutines
+------------------------------------
+
+Usually, while a coroutine is waiting for, e.g., an in-flight network request,
+the suspended ``coroutine_handle`` is stored within the work queues inside the
+IO scheduler. As soon as we get hold of the coroutine handle, we can backtrace
+it by using ``coro bt <coro_handle>`` where ``<coro_handle>`` is an expression
+evaluating to the coroutine handle of the suspended coroutine.
+
+Keeping track of all existing coroutines
+----------------------------------------
+
+Usually, we should be able to get hold of all currently suspended coroutines by
+inspecting the worker queues of the IO scheduler. In cases where this is not
+possible, we can use the following approach to keep track of all currently
+suspended coroutines.
+
+One such solution is to store the list of in-flight coroutines in a collection:
+
+.. code-block:: c++
+
+  inline std::unordered_set<std::coroutine_handle<void>> inflight_coroutines;
+  inline std::mutex inflight_coroutines_mutex;
+
+  class promise_type {
+  public:
+      promise_type() {
+          std::unique_lock<std::mutex> lock(inflight_coroutines_mutex);
+          inflight_coroutines.insert(std::coroutine_handle<promise_type>::from_promise(*this));
+      }
+      ~promise_type() {
+          std::unique_lock<std::mutex> lock(inflight_coroutines_mutex);
+          inflight_coroutines.erase(std::coroutine_handle<promise_type>::from_promise(*this));
+      }
+  };
+
+With this in place, it is possible to inspect ``inflight_coroutines`` from the
+debugger, and rely on LLDB's ``std::coroutine_handle`` pretty-printer to
+inspect the coroutines.
+
+This technique will track *all* coroutines, also the ones which are currently
+awaiting another coroutine, though. To identify just the "roots" of our
+in-flight coroutines, we can use the ``coro in-flight inflight_coroutines``
+command provided by the :ref:`lldb-script`.
+
+Please note that the above is expensive from a runtime performance perspective,
+and requires locking to prevent data races. As such, it is not recommended to
+use this approach in production code.
+
+Known issues & workarounds for older LLDB versions
+==================================================
+
+LLDB before 21.0 did not yet show the ``__coro_frame`` inside
+``coroutine_handle``. To inspect the coroutine frame, you had to use the
+approach described in the :ref:`devirtualization` section.
+
+LLDB before 18.0 was hiding the ``__promise`` and ``__coro_frame``
+variable by default. The variables are still present, but they need to be
+explicitly added to the "watch" pane in VS Code or requested via
+``print __promise`` and ``print __coro_frame`` from the debugger console.
+
+LLDB before 16.0 did not yet provide a pretty-printer for
+``std::coroutine_handle``. To inspect the coroutine handle, you had to manually
+use the approach described in the :ref:`devirtualization`
+section.
+
+Toolchain Implementation Details
+================================
+
+This section covers the ABI, as well as additional compiler-specific behavior.
+The ABI is followed by all compilers, on all major systems, including Windows,
+Linux and macOS. Different compilers emit different debug information, though.
+
+Ramp, resume and destroy functions
+----------------------------------
+
+Every coroutine is split into three parts:
+
+* The ramp function allocates the coroutine frame and initializes it, usually
+  copying over all variables into the coroutine frame
+* The resume function continues the coroutine from its previous suspension point
+* The destroy function destroys and deallocates the coroutine frame
+* The cleanup function destroys the coroutine frame but does not deallocate it.
+  It is used when the coroutine's allocation was elided thanks to
+  `Heap Allocation Elision (HALO) <https://www.open-std.org/JTC1/SC22/WG21/docs/papers/2018/p0981r0.html>`_
+
+The ramp function is called by the coroutine's caller, and available under the
+original function name used in the C++ source code. The resume function is
+called via ``std::coroutine_handle::resume``. The destroy function is called
+via ``std::coroutine_handle::destroy``.
+
+Information between the three functions is passed via the coroutine frame, a
+compiler-synthesized struct that contains all necessary internal state. The
+resume function knows where to resume execution by reading the suspension point
+index from the coroutine frame. Similarly, the destroy function relies on the
+suspension point index to know which variables are currently in scope and need
+to be destructed.
+
+Usually, the destroy function calls all destructors and deallocates the
+coroutine frame. When a coroutine frame was elided thanks to HALO, only the
+destructors need to be called, but the coroutine frame must not be deallocated.
+In those cases, the cleanup function is used instead of the destroy function.
+
+For coroutines allocated with ``[[clang::coro_await_elidable]]``, clang also
+generates a ``.noalloc`` variant of the ramp function, which does not allocate
+the coroutine frame by itself, but instead expects the caller to allocate the
+coroutine frame and pass it to the ramp function.
+
+When trying to intercept all creations of new coroutines in the debugger, you
+hence might have to set breakpoints in the ramp function and its ``.noalloc``
+variant.
+
+Artificial ``__promise`` and ``__coro_frame`` variables
+-------------------------------------------------------
+
+Inside all coroutine functions, clang / LLVM synthesize a ``__promise`` and
+``__coro_frame`` variable. These variables are used to store the coroutine's
+state. When inside the coroutine function, those can be used to directly
+inspect the promise and the coroutine frame of the own function.
+
+The ABI of a coroutine
+----------------------
+
+A ``std::coroutine_handle`` essentially only holds a pointer to a coroutine
+frame. It resembles the following struct:
+
+.. code-block:: c++
+
+  template<typename promise_type>
+  struct coroutine_handle {
+    void* __coroutine_frame = nullptr;
+  };
+
+The structure of coroutine frames is defined as
+
+.. code-block:: c++
+
+  struct my_coroutine_frame {
+    void (*__resume)(coroutine_frame*); // function pointer to the `resume` function
+    void (*__destroy)(coroutine_frame*); // function pointer to the `destroy` function
+    promise_type promise; // the corresponding `promise_type`
+    ... // Internal coroutine state
+  }
+
+For each coroutine, the compiler synthesizes a different coroutine type,
+storing all necessary internal state. The actual coroutine type is type-erased
+behind the ``std::coroutine_handle``.
+
+However, all coroutine frames always contain the ``resume`` and ``destroy``
+functions as their first two members. As such, we can read the function
+pointers from the coroutine frame and then obtain the function's name from its
+address.
+
+The promise is guaranteed to be at a 16 byte offset from the coroutine frame.
+If we have a coroutine handle at address 0x416eb0, we can hence reinterpret-cast
+the promise as follows:
+
+.. code-block:: text
+
+  print (task::promise_type)*(0x416eb0+16)
+
+Implementation in clang / LLVM
+------------------------------
+
+The C++ Coroutines feature in the Clang compiler is implemented in two parts of
+the compiler. Semantic analysis is performed in Clang, and Coroutine
+construction and optimization takes place in the LLVM middle-end.
+
+For each coroutine function, the frontend generates a single corresponding
+LLVM-IR function. This function uses special ``llvm.coro.suspend`` intrinsics
+to mark the suspension points of the coroutine. The middle end first optimizes
+this function and applies, e.g., constant propagation across the whole,
+non-split coroutine.
+
+CoroSplit then splits the function into ramp, resume and destroy functions.
+This pass also moves stack-local variables which are alive across suspension
+points into the coroutine frame. Most of the heavy lifting to preserve debugging
+information is done in this pass. This pass needs to rewrite all variable
+locations to point into the coroutine frame.
+
+Afterwards, a couple of additional optimizations are applied, before code
+gets emitted, but none of them are really interesting regarding debugging
+information.
+
+For more details on the IR representation of coroutines and the relevant
+optimization passes, see `Coroutines in LLVM <https://llvm.org/docs/Coroutines.html>`_.
+
+Emitting debug information inside ``CoroSplit`` forces us to generate
+insufficient debugging information. Usually, the compiler generates debug
+information in the frontend, as debug information is highly language specific.
+However, this is not possible for coroutine frames because the frames are
+constructed in the LLVM middle-end.
+
+To mitigate this problem, the LLVM middle end attempts to generate some debug
+information, which is unfortunately incomplete, since much of the language
+specific information is missing in the middle end.
+
+.. _devirtualization:
+
+Devirtualization of coroutine handles
+-------------------------------------
+
+Figuring out the promise type and the coroutine frame type of a coroutine
+handle requires inspecting the ``resume`` and ``destroy`` function pointers.
+There are two possible approaches to do so:
+
+1. clang always names the type by appending ``.coro_frame_ty`` to the
+   linkage name of the ramp function.
+2. Both clang and GCC add the function-local ``__promise`` and
+   ``__coro_frame`` variables to the resume and destroy functions.
+   We can lookup their types and thereby get the types of promise
+   and coroutine frame.
+
+In gdb, one can use the following approach to devirtualize coroutine type,
+assuming we have a ``std::coroutine_handle`` is at address 0x418eb0:
+
+::
+
+  (gdb) # Get the address of coroutine frame
+  (gdb) print/x *0x418eb0
+  $1 = 0x4019e0
+  (gdb) # Get the linkage name for the coroutine
+  (gdb) x 0x4019e0
+  0x4019e0 <_ZL9coro_taski>:  0xe5894855
+  (gdb) # Turn off the demangler temporarily to avoid the debugger misunderstanding the name.
+  (gdb) set demangle-style none
+  (gdb) # The coroutine frame type is 'linkage_name.coro_frame_ty'
+  (gdb) print  ('_ZL9coro_taski.coro_frame_ty')*(0x418eb0)
+  $2 = {__resume_fn = 0x4019e0 <coro_task(int)>, __destroy_fn = 0x402000 <coro_task(int)>, __promise = {...}, ...}
+
+In practice, one would use the ``show-coro-frame`` command provided by the
+:ref:`gdb-script`.
+
+LLDB comes with devirtualization support out of the box, as part of the
+pretty-printer for ``std::coroutine_handle``. Internally, this pretty-printer
+uses the second approach. We look up the types in the destroy function and not
+the resume function because the resume function pointer will be set to a
+nullptr as soon as a coroutine reaches its final suspension point. If we used
+the resume function, devirtualization would hence fail for all coroutines that
+have reached their final suspension point.
+
+Interpreting the coroutine frame in optimized builds
+----------------------------------------------------
+
+The ``__coro_frame`` variable usually refers to the coroutine frame of an
+*in-flight* coroutine. This means, the coroutine is currently executing.
+However, the compiler only guarantees the coroutine frame to be in a consistent
+state while the coroutine is suspended. As such, the variables inside the
+``__coro_frame`` variable might be outdated, in particular when optimizations
+are enabled.
+
+Furthermore, when optimizations are enabled, the compiler will layout the
+coroutine frame more aggressively. Unused values are optimized out, and the
+state will usually contain only the minimal information required to reconstruct
+the coroutine's state.
+
+clang / LLVM usually use variables like ``__int_32_0`` to represent this
+optimized storage. Those values usually do not directly correspond to variables
+in the source code.
+
+When compiling the program
 
 .. code-block:: c++
 
   static task coro_task(int v) {
     int a = v;
-    co_await await_counter{};
+    co_await some_other_task();
     a++; // __int_32_0 is 43 here
     std::cout << a << "\n";
     a++; // __int_32_0 is still 43 here
     std::cout << a << "\n";
     a++; // __int_32_0 is still 43 here!
     std::cout << a << "\n";
-    co_await await_counter{};
+    co_await some_other_task();
     a++; // __int_32_0 is still 43 here!!
     std::cout << a << "\n";
     a++; // Why is __int_32_0 still 43 here?
     std::cout << a << "\n";
   }
 
-When debugging step-by-step, the value of `__int_32_0` seemingly does not
-change, despite being frequently incremented, and instead is always `43`.
+clang creates a single entry ``__int_32_0`` in the coroutine state.
+
+Intuitively, one might assume that ``__int_32_0`` represents the value of the
+local variable ``a``. However, inspecting ``__int_32_0`` in the debugger while
+single-stepping will reveal that the value of ``__int_32_0`` stays constant,
+despite ``a`` being frequently incremented.
+
 While this might be surprising, this is a result of the optimizer recognizing
-that it can eliminate most of the load/store operations. The above code gets
-optimized to the equivalent of:
+that it can eliminate most of the load/store operations.
+The above code gets optimized to the equivalent of:
 
 .. code-block:: c++
 
   static task coro_task(int v) {
-    store v to __int_32_0 in the frame
+    store v into __int_32_0 in the frame
     co_await await_counter{};
     a = load __int_32_0
     std::cout << a+1 << "\n";
@@ -301,173 +748,280 @@ optimized to the equivalent of:
     std::cout << a+5 << "\n";
   }
 
-It should now be obvious why the value of `__int_32_0` remains unchanged
-throughout the function. It is important to recognize that `__int_32_0`
-does not directly correspond to `a`, but is instead a variable generated
-to assist the compiler in code generation. The variables in an optimized
-coroutine frame should not be thought of as directly representing the
-variables in the C++ source.
+It should now be obvious why the value of ``__int_32_0`` remains unchanged
+throughout the function. It is important to recognize that ``__int_32_0`` does
+not directly correspond to ``a``, but is instead a variable generated to assist
+the compiler in code generation. The variables in an optimized coroutine frame
+should not be thought of as directly representing the variables in the C++
+source.
 
-Get the suspended points
-========================
-
-An important requirement for debugging coroutines is to understand suspended
-points, which are where the coroutine is currently suspended and awaiting.
-
-For simple cases like the above, inspecting the value of the `__coro_index`
-variable in the coroutine frame works well.
-
-However, it is not quite so simple in really complex situations. In these
-cases, it is necessary to use the coroutine libraries to insert the
-line-number.
-
-For example:
 
-.. code-block:: c++
+Resources
+=========
 
-  // For all the promise_type we want:
-  class promise_type {
-    ...
-  +  unsigned line_number = 0xffffffff;
-  };
+.. _lldb-script:
 
-  #include <source_location>
+LLDB Debugger Script
+--------------------
 
-  // For all the awaiter types we need:
-  class awaiter {
-    ...
-    template <typename Promise>
-    void await_suspend(std::coroutine_handle<Promise> handle,
-                       std::source_location sl = std::source_location::current()) {
-          ...
-          handle.promise().line_number = sl.line();
-    }
-  };
+The following script provides the ``coro bt`` and ``coro in-flight`` commands
+discussed above. It can be loaded into LLDB using ``command script import
+lldb_coro_debugging.py``. To load this by default, add this command to your
+``~/.lldbinit`` file.
 
-In this case, we use `std::source_location` to store the line number of the
-await inside the `promise_type`.  Since we can locate the coroutine function
-from the address of the coroutine, we can identify suspended points this way
-as well.
+Note that this script requires LLDB 21.0 or newer.
 
-The downside here is that this comes at the price of additional runtime cost.
-This is consistent with the C++ philosophy of "Pay for what you use".
-
-Get the asynchronous stack
-==========================
-
-Another important requirement to debug a coroutine is to print the asynchronous
-stack to identify the asynchronous caller of the coroutine.  As many
-implementations of coroutine types store `std::coroutine_handle<> continuation`
-in the promise type, identifying the caller should be trivial.  The
-`continuation` is typically the awaiting coroutine for the current coroutine.
-That is, the asynchronous parent.
-
-Since the `promise_type` is obtainable from the address of a coroutine and
-contains the corresponding continuation (which itself is a coroutine with a
-`promise_type`), it should be trivial to print the entire asynchronous stack.
-
-This logic should be quite easily captured in a debugger script.
-
-Examples to print asynchronous stack
-------------------------------------
-
-Here is an example to print the asynchronous stack for the normal task implementation.
+.. code-block:: python
 
-.. code-block:: c++
+  # lldb_coro_debugging.py
+  import lldb
+  from lldb.plugins.parsed_cmd import ParsedCommand
+
+  def _get_first_var_path(v, paths):
+      """
+      Tries multiple variable paths via `GetValueForExpressionPath`
+      and returns the first one that succeeds, or None if none succeed.
+      """
+      for path in paths:
+          var = v.GetValueForExpressionPath(path)
+          if var.error.Success():
+              return var
+      return None
+
+
+  def _print_async_bt(coro_hdl, result, *, curr_idx, start, limit, continuation_paths, prefix=""):
+      """
+      Prints a backtrace for an async coroutine stack starting from `coro_hdl`,
+      using the given `continuation_paths` to get the next coroutine from the promise.
+      """
+      target = coro_hdl.GetTarget()
+      while curr_idx < limit and coro_hdl is not None and coro_hdl.error.Success():
+          # Print the stack frame, if in range
+          if curr_idx >= start:
+              # Figure out the function name
+              destroy_func_var = coro_hdl.GetValueForExpressionPath(".destroy")
+              destroy_addr = target.ResolveLoadAddress(destroy_func_var.GetValueAsAddress())
+              func_name = destroy_addr.function.name
+              # Figure out the line entry to show
+              suspension_addr_var = coro_hdl.GetValueForExpressionPath(".promise._coro_suspension_point_addr")
+              if suspension_addr_var.error.Success():
+                  line_entry = target.ResolveLoadAddress(suspension_addr_var.GetValueAsAddress()).line_entry
+                  print(f"{prefix} frame #{curr_idx}: {func_name} at {line_entry}", file=result)
+              else:
+                  # We don't know the exact line, print the suspension point ID, so we at least show
+                  # the id of the current suspension point
+                  suspension_point_var = coro_hdl.GetValueForExpressionPath(".coro_frame.__coro_index")
+                  if suspension_point_var.error.Success():
+                      suspension_point = suspension_point_var.GetValueAsUnsigned()
+                  else:
+                      suspension_point = "unknown"
+                  line_entry = destroy_addr.line_entry
+                  print(f"{prefix} frame #{curr_idx}: {func_name} at {line_entry}, suspension point {suspension_point}", file=result)
+          # Move to the next stack frame
+          curr_idx += 1
+          promise_var = coro_hdl.GetChildMemberWithName("promise")
+          coro_hdl = _get_first_var_path(promise_var, continuation_paths)
+      return curr_idx
+
+  def _print_combined_bt(frame, result, *, unfiltered, curr_idx, start, limit, continuation_paths):
+      """
+      Prints a backtrace starting from `frame`, interleaving async coroutine frames
+      with regular frames.
+      """
+      while curr_idx < limit and frame.IsValid():
+          if curr_idx >= start and (unfiltered or not frame.IsHidden()):
+              print(f"frame #{curr_idx}: {frame.name} at {frame.line_entry}", file=result)
+          curr_idx += 1
+          coro_var = _get_first_var_path(frame.GetValueForVariablePath("__promise"), continuation_paths)
+          if coro_var:
+              curr_idx = _print_async_bt(coro_var, result,
+                  curr_idx=curr_idx, start=start, limit=limit,
+                  continuation_paths=continuation_paths, prefix="[async]")
+          frame = frame.parent
+
+
+  class CoroBacktraceCommand(ParsedCommand):
+      def get_short_help(self):
+          return "Create a backtrace for C++-20 coroutines"
+
+      def get_flags(self):
+          return lldb.eCommandRequiresFrame | lldb.eCommandProcessMustBePaused
+
+      def setup_command_definition(self):
+          ov_parser = self.get_parser()
+          ov_parser.add_option(
+              "e",
+              "continuation-expr",
+              help = (
+                  "Semi-colon-separated list of expressions evaluated against the promise object"
+                  "to get the next coroutine (e.g. `.continuation;.coro_parent`)"
+              ),
+              value_type = lldb.eArgTypeNone,
+              dest = "continuation_expr_arg",
+              default = ".continuation",
+          )
+          ov_parser.add_option(
+              "c",
+              "count",
+              help = "How many frames to display (0 for all)",
+              value_type = lldb.eArgTypeCount,
+              dest = "count_arg",
+              default = 20,
+          )
+          ov_parser.add_option(
+              "s",
+              "start",
+              help = "Frame in which to start the backtrace",
+              value_type = lldb.eArgTypeIndex,
+              dest = "frame_index_arg",
+              default = 0,
+          )
+          ov_parser.add_option(
+              "u",
+              "unfiltered",
+              help = "Do not filter out frames according to installed frame recognizers",
+              value_type = lldb.eArgTypeBoolean,
+              dest = "unfiltered_arg",
+              default = False,
+          )
+          ov_parser.add_argument_set([
+              ov_parser.make_argument_element(
+                  lldb.eArgTypeExpression,
+                  repeat="optional"
+              )
+          ])
+
+      def __call__(self, debugger, args_array, exe_ctx, result):
+          ov_parser = self.get_parser()
+          continuation_paths = ov_parser.continuation_expr_arg.split(";")
+          count = ov_parser.count_arg
+          if count == 0:
+              count = 99999
+          frame_index = ov_parser.frame_index_arg
+          unfiltered = ov_parser.unfiltered_arg
+
+          frame = exe_ctx.GetFrame()
+          if not frame.IsValid():
+              result.SetError("invalid frame")
+              return
 
-  // debugging-example.cpp
-  #include <coroutine>
-  #include <iostream>
-  #include <utility>
+          if len(args_array) > 1:
+              result.SetError("At most one expression expected")
+              return
+          elif len(args_array) == 1:
+              expr = args_array.GetItemAtIndex(0).GetStringValue(9999)
+              coro_hdl = frame.EvaluateExpression(expr)
+              if not coro_hdl.error.Success():
+                  result.AppendMessage(
+                      f'error: expression failed {expr} => {coro_hdl.error}'
+                  )
+                  result.SetError(f"Expression `{expr}` failed to evaluate")
+                  return
+              _print_async_bt(coro_hdl, result,
+                  curr_idx = 0, start = frame_index, limit = frame_index + count,
+                  continuation_paths = continuation_paths)
+          else:
+              _print_combined_bt(frame, result, unfiltered=unfiltered,
+                  curr_idx = 0, start = frame_index, limit = frame_index + count,
+                  continuation_paths = continuation_paths)
+
+
+  class CoroInflightCommand(ParsedCommand):
+      def get_short_help(self):
+          return "Identify all in-flight coroutines"
+
+      def get_flags(self):
+          return lldb.eCommandRequiresTarget | lldb.eCommandProcessMustBePaused
+
+      def setup_command_definition(self):
+          ov_parser = self.get_parser()
+          ov_parser.add_option(
+              "e",
+              "continuation-expr",
+              help = (
+                  "Semi-colon-separated list of expressions evaluated against the promise object"
+                  "to get the next coroutine (e.g. `.continuation;.coro_parent`)"
+              ),
+              value_type = lldb.eArgTypeNone,
+              dest = "continuation_expr_arg",
+              default = ".continuation",
+          )
+          ov_parser.add_option(
+              "c",
+              "count",
+              help = "How many frames to display (0 for all)",
+              value_type = lldb.eArgTypeCount,
+              dest = "count_arg",
+              default = 5,
+          )
+          ov_parser.add_argument_set([
+              ov_parser.make_argument_element(
+                  lldb.eArgTypeExpression,
+                  repeat="plus"
+              )
+          ])
+
+      def __call__(self, debugger, args_array, exe_ctx, result):
+          ov_parser = self.get_parser()
+          continuation_paths = ov_parser.continuation_expr_arg.split(";")
+          count = ov_parser.count_arg
+
+          # Collect all coroutine_handles from the provided containers
+          all_coros = []
+          for entry in args_array:
+              expr = entry.GetStringValue(9999)
+              if exe_ctx.frame.IsValid():
+                  coro_container = exe_ctx.frame.EvaluateExpression(expr)
+              else:
+                  coro_container = exe_ctx.target.EvaluateExpression(expr)
+              if not coro_container.error.Success():
+                  result.AppendMessage(
+                      f'error: expression failed {expr} => {coro_container.error}'
+                  )
+                  result.SetError(f"Expression `{expr}` failed to evaluate")
+                  return
+              for entry in coro_container.children:
+                  if "coroutine_handle" not in entry.GetType().name:
+                      result.SetError(f"Found entry of type {entry.GetType().name} in {expr},"
+                                      "  expected a coroutine handle")
+                      return
+                  all_coros.append(entry)
 
-  struct task {
-    struct promise_type {
-      task get_return_object();
-      std::suspend_always initial_suspend() { return {}; }
+          # Remove all coroutines that are currently waiting for other coroutines to finish
+          coro_roots = {c.GetChildMemberWithName("coro_frame").GetValueAsAddress(): c for c in all_coros}
+          for coro_hdl in all_coros:
+              parent_coro = _get_first_var_path(coro_hdl.GetChildMemberWithName("promise"), continuation_paths)
+              parent_addr = parent_coro.GetChildMemberWithName("coro_frame").GetValueAsAddress()
+              if parent_addr in coro_roots:
+                  del coro_roots[parent_addr]
 
-      void unhandled_exception() noexcept {}
+          # Print all remaining coroutines
+          for addr, root_hdl in coro_roots.items():
+              print(f"coroutine root 0x{addr:x}", file=result)
+              _print_async_bt(root_hdl, result,
+                              curr_idx=0, start=0, limit=count,
+                              continuation_paths=continuation_paths, prefix="    ")
 
-      struct FinalSuspend {
-        std::coroutine_handle<> continuation;
-        auto await_ready() noexcept { return false; }
-        auto await_suspend(std::coroutine_handle<> handle) noexcept {
-          return continuation;
-        }
-        void await_resume() noexcept {}
-      };
-      FinalSuspend final_suspend() noexcept { return {continuation}; }
 
-      void return_value(int res) { result = res; }
+  def __lldb_init_module(debugger, internal_dict):
+      debugger.HandleCommand("command container add -h 'Debugging utilities for C++20 coroutines' coro")
+      debugger.HandleCommand(f"command script add -o -p -c {__name__}.CoroBacktraceCommand coro bt")
+      debugger.HandleCommand(f"command script add -o -p -c {__name__}.CoroInflightCommand coro in-flight")
+      print("Coro debugging utilities installed. Use `help coro` to see available commands.")
 
-      std::coroutine_handle<> continuation = std::noop_coroutine();
-      int result = 0;
-    };
+  if __name__ == '__main__':
+      print("This script should be loaded from LLDB using `command script import <filename>`")
 
-    task(std::coroutine_handle<promise_type> handle) : handle(handle) {}
-    ~task() {
-      if (handle)
-        handle.destroy();
-    }
+.. _gdb-script:
 
-    auto operator co_await() {
-      struct Awaiter {
-        std::coroutine_handle<promise_type> handle;
-        auto await_ready() { return false; }
-        auto await_suspend(std::coroutine_handle<> continuation) {
-          handle.promise().continuation = continuation;
-          return handle;
-        }
-        int await_resume() {
-          int ret = handle.promise().result;
-          handle.destroy();
-          return ret;
-        }
-      };
-      return Awaiter{std::exchange(handle, nullptr)};
-    }
+GDB Debugger Script
+-------------------
 
-    int syncStart() {
-      handle.resume();
-      return handle.promise().result;
-    }
+For GDB, the following script provides a couple of useful commands:
 
-  private:
-    std::coroutine_handle<promise_type> handle;
-  };
-
-  task task::promise_type::get_return_object() {
-    return std::coroutine_handle<promise_type>::from_promise(*this);
-  }
-
-  namespace detail {
-  template <int N>
-  task chain_fn() {
-    co_return N + co_await chain_fn<N - 1>();
-  }
-
-  template <>
-  task chain_fn<0>() {
-    // This is the default breakpoint.
-    __builtin_debugtrap();
-    co_return 0;
-  }
-  }  // namespace detail
-
-  task chain() {
-    co_return co_await detail::chain_fn<30>();
-  }
-
-  int main() {
-    std::cout << chain().syncStart() << "\n";
-    return 0;
-  }
-
-In the example, the ``task`` coroutine holds a ``continuation`` field,
-which would be resumed once the ``task`` completes.
-In another word, the ``continuation`` is the asynchronous caller for the ``task``.
-Just like the normal function returns to its caller when the function completes.
-
-So we can use the ``continuation`` field to construct the asynchronous stack:
+* ``async-bt`` to print the stack trace of a coroutine
+* ``show-coro-frame`` to print the coroutine frame, similar to
+  LLDB's builtin pretty-printer for coroutine frames
 
 .. code-block:: python
 
@@ -599,7 +1153,7 @@ So we can use the ``continuation`` field to construct the asynchronous stack:
           addr = int(argv[0], 16)
           block = gdb.block_for_pc(long(cast_addr2long_pointer(addr).dereference()))
           if block is None:
-              print "block " + str(addr) + "  is none."
+              print "block " + str(addr) + " is None."
               return
 
           # Disable demangling since gdb will treat names starting with `_Z`(The marker for Itanium ABI) specially.
@@ -614,128 +1168,17 @@ So we can use the ``continuation`` field to construct the asynchronous stack:
 
   ShowCoroFrame()
 
-Then let's run:
-
-.. code-block:: text
-
-  $ clang++ -std=c++20 -g debugging-example.cpp -o debugging-example
-  $ gdb ./debugging-example
-  (gdb) # We've already set the breakpoint.
-  (gdb) r
-  Program received signal SIGTRAP, Trace/breakpoint trap.
-  detail::chain_fn<0> () at debugging-example2.cpp:73
-  73	  co_return 0;
-  (gdb) # Executes the debugging scripts
-  (gdb) source debugging-helper.py
-  (gdb) # Print the asynchronous stack
-  (gdb) async-bt
-  #0 0x401c40 in detail::chain_fn<0>() ['frame_addr = 0x441860', 'promise_addr = 0x441870', 'continuation_addr = 0x441870'] at debugging-example.cpp:71
-  #1 0x4022d0 in detail::chain_fn<1>() ['frame_addr = 0x441810', 'promise_addr = 0x441820', 'continuation_addr = 0x441820'] at debugging-example.cpp:66
-  #2 0x403060 in detail::chain_fn<2>() ['frame_addr = 0x4417c0', 'promise_addr = 0x4417d0', 'continuation_addr = 0x4417d0'] at debugging-example.cpp:66
-  #3 0x403df0 in detail::chain_fn<3>() ['frame_addr = 0x441770', 'promise_addr = 0x441780', 'continuation_addr = 0x441780'] at debugging-example.cpp:66
-  #4 0x404b80 in detail::chain_fn<4>() ['frame_addr = 0x441720', 'promise_addr = 0x441730', 'continuation_addr = 0x441730'] at debugging-example.cpp:66
-  #5 0x405910 in detail::chain_fn<5>() ['frame_addr = 0x4416d0', 'promise_addr = 0x4416e0', 'continuation_addr = 0x4416e0'] at debugging-example.cpp:66
-  #6 0x4066a0 in detail::chain_fn<6>() ['frame_addr = 0x441680', 'promise_addr = 0x441690', 'continuation_addr = 0x441690'] at debugging-example.cpp:66
-  #7 0x407430 in detail::chain_fn<7>() ['frame_addr = 0x441630', 'promise_addr = 0x441640', 'continuation_addr = 0x441640'] at debugging-example.cpp:66
-  #8 0x4081c0 in detail::chain_fn<8>() ['frame_addr = 0x4415e0', 'promise_addr = 0x4415f0', 'continuation_addr = 0x4415f0'] at debugging-example.cpp:66
-  #9 0x408f50 in detail::chain_fn<9>() ['frame_addr = 0x441590', 'promise_addr = 0x4415a0', 'continuation_addr = 0x4415a0'] at debugging-example.cpp:66
-  #10 0x409ce0 in detail::chain_fn<10>() ['frame_addr = 0x441540', 'promise_addr = 0x441550', 'continuation_addr = 0x441550'] at debugging-example.cpp:66
-  #11 0x40aa70 in detail::chain_fn<11>() ['frame_addr = 0x4414f0', 'promise_addr = 0x441500', 'continuation_addr = 0x441500'] at debugging-example.cpp:66
-  #12 0x40b800 in detail::chain_fn<12>() ['frame_addr = 0x4414a0', 'promise_addr = 0x4414b0', 'continuation_addr = 0x4414b0'] at debugging-example.cpp:66
-  #13 0x40c590 in detail::chain_fn<13>() ['frame_addr = 0x441450', 'promise_addr = 0x441460', 'continuation_addr = 0x441460'] at debugging-example.cpp:66
-  #14 0x40d320 in detail::chain_fn<14>() ['frame_addr = 0x441400', 'promise_addr = 0x441410', 'continuation_addr = 0x441410'] at debugging-example.cpp:66
-  #15 0x40e0b0 in detail::chain_fn<15>() ['frame_addr = 0x4413b0', 'promise_addr = 0x4413c0', 'continuation_addr = 0x4413c0'] at debugging-example.cpp:66
-  #16 0x40ee40 in detail::chain_fn<16>() ['frame_addr = 0x441360', 'promise_addr = 0x441370', 'continuation_addr = 0x441370'] at debugging-example.cpp:66
-  #17 0x40fbd0 in detail::chain_fn<17>() ['frame_addr = 0x441310', 'promise_addr = 0x441320', 'continuation_addr = 0x441320'] at debugging-example.cpp:66
-  #18 0x410960 in detail::chain_fn<18>() ['frame_addr = 0x4412c0', 'promise_addr = 0x4412d0', 'continuation_addr = 0x4412d0'] at debugging-example.cpp:66
-  #19 0x4116f0 in detail::chain_fn<19>() ['frame_addr = 0x441270', 'promise_addr = 0x441280', 'continuation_addr = 0x441280'] at debugging-example.cpp:66
-  #20 0x412480 in detail::chain_fn<20>() ['frame_addr = 0x441220', 'promise_addr = 0x441230', 'continuation_addr = 0x441230'] at debugging-example.cpp:66
-  #21 0x413210 in detail::chain_fn<21>() ['frame_addr = 0x4411d0', 'promise_addr = 0x4411e0', 'continuation_addr = 0x4411e0'] at debugging-example.cpp:66
-  #22 0x413fa0 in detail::chain_fn<22>() ['frame_addr = 0x441180', 'promise_addr = 0x441190', 'continuation_addr = 0x441190'] at debugging-example.cpp:66
-  #23 0x414d30 in detail::chain_fn<23>() ['frame_addr = 0x441130', 'promise_addr = 0x441140', 'continuation_addr = 0x441140'] at debugging-example.cpp:66
-  #24 0x415ac0 in detail::chain_fn<24>() ['frame_addr = 0x4410e0', 'promise_addr = 0x4410f0', 'continuation_addr = 0x4410f0'] at debugging-example.cpp:66
-  #25 0x416850 in detail::chain_fn<25>() ['frame_addr = 0x441090', 'promise_addr = 0x4410a0', 'continuation_addr = 0x4410a0'] at debugging-example.cpp:66
-  #26 0x4175e0 in detail::chain_fn<26>() ['frame_addr = 0x441040', 'promise_addr = 0x441050', 'continuation_addr = 0x441050'] at debugging-example.cpp:66
-  #27 0x418370 in detail::chain_fn<27>() ['frame_addr = 0x440ff0', 'promise_addr = 0x441000', 'continuation_addr = 0x441000'] at debugging-example.cpp:66
-  #28 0x419100 in detail::chain_fn<28>() ['frame_addr = 0x440fa0', 'promise_addr = 0x440fb0', 'continuation_addr = 0x440fb0'] at debugging-example.cpp:66
-  #29 0x419e90 in detail::chain_fn<29>() ['frame_addr = 0x440f50', 'promise_addr = 0x440f60', 'continuation_addr = 0x440f60'] at debugging-example.cpp:66
-  #30 0x41ac20 in detail::chain_fn<30>() ['frame_addr = 0x440f00', 'promise_addr = 0x440f10', 'continuation_addr = 0x440f10'] at debugging-example.cpp:66
-  #31 0x41b9b0 in chain() ['frame_addr = 0x440eb0', 'promise_addr = 0x440ec0', 'continuation_addr = 0x440ec0'] at debugging-example.cpp:77
-
-Now we get the complete asynchronous stack!
-It is also possible to print other asynchronous stack which doesn't live in the top of the stack.
-We can make it by passing the address of the corresponding coroutine frame to ``async-bt`` command.
-
-By the debugging scripts, we can print any coroutine frame too as long as we know the address.
-For example, we can print the coroutine frame for ``detail::chain_fn<18>()`` in the above example.
-From the log record, we know the address of the coroutine frame is ``0x4412c0`` in the run. Then we can:
-
-.. code-block:: text
-
-  (gdb) show-coro-frame 0x4412c0
-  {
-    __resume_fn = 0x410960 <detail::chain_fn<18>()>,
-    __destroy_fn = 0x410d60 <detail::chain_fn<18>()>,
-    __promise = {
-      continuation = {
-        _M_fr_ptr = 0x441270
-      },
-      result = 0
-    },
-    struct_Awaiter_0 = {
-      struct_std____n4861__coroutine_handle_0 = {
-        struct_std____n4861__coroutine_handle = {
-          PointerType = 0x441310
-        }
-      }
-    },
-    struct_task_1 = {
-      struct_std____n4861__coroutine_handle_0 = {
-        struct_std____n4861__coroutine_handle = {
-          PointerType = 0x0
-        }
-      }
-    },
-    struct_task__promise_type__FinalSuspend_2 = {
-      struct_std____n4861__coroutine_handle = {
-        PointerType = 0x0
-      }
-    },
-    __coro_index = 1 '\001',
-    struct_std____n4861__suspend_always_3 = {
-      __int_8 = 0 '\000'
-    }
-
-
-Get the living coroutines
-=========================
-
-Another useful task when debugging coroutines is to enumerate the list of
-living coroutines, which is often done with threads.  While technically
-possible, this task is not recommended in production code as it is costly at
-runtime. One such solution is to store the list of currently running coroutines
-in a collection:
-
-.. code-block:: c++
+Further Reading
+---------------
 
-  inline std::unordered_set<void*> lived_coroutines;
-  // For all promise_type we want to record
-  class promise_type {
-  public:
-      promise_type() {
-          // Note to avoid data races
-          lived_coroutines.insert(std::coroutine_handle<promise_type>::from_promise(*this).address());
-      }
-      ~promise_type() {
-          // Note to avoid data races
-          lived_coroutines.erase(std::coroutine_handle<promise_type>::from_promise(*this).address());
-      }
-  };
+The authors of the Folly libraries wrote a blog post series on how they debug coroutines:
 
-In the above code snippet, we save the address of every lived coroutine in the
-`lived_coroutines` `unordered_set`. As before, once we know the address of the
-coroutine we can derive the function, `promise_type`, and other members of the
-frame. Thus, we could print the list of lived coroutines from that collection.
+* `Async stack traces in folly: Introduction <https://developers.facebook.com/blog/post/2021/09/16/async-stack-traces-folly-Introduction/>`_
+* `Async stack traces in folly: Synchronous and asynchronous stack traces <https://developers.facebook.com/blog/post/2021/09/23/async-stack-traces-folly-synchronous-asynchronous-stack-traces/>`_
+* `Async stack traces in folly: Forming an async stack from individual frames <https://developers.facebook.com/blog/post/2021/09/30/async-stack-traces-folly-forming-async-stack-individual-frames/>`_
+* `Async Stack Traces for C++ Coroutines in Folly: Walking the async stack <https://developers.facebook.com/blog/post/2021/10/14/async-stack-traces-c-plus-plus-coroutines-folly-walking-async-stack/>`_
+* `Async stack traces in folly: Improving debugging in the developer lifecycle <https://developers.facebook.com/blog/post/2021/10/21/async-stack-traces-folly-improving-debugging-developer-lifecycle/>`_
 
-Please note that the above is expensive from a storage perspective, and requires
-some level of locking (not pictured) on the collection to prevent data races.
+Besides some topics also covered here (stack traces from the debugger), Folly's blog post series also covers
+more additional topics, such as capturing async stack traces in performance profiles via eBPF filters
+and printing async stack traces on crashes.
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index cb188bc02b35..db4b911b134e 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -3798,6 +3798,17 @@ Trivially relocates ``count`` objects of relocatable, complete type ``T``
 from ``src`` to ``dest`` and returns ``dest``.
 This builtin is used to implement ``std::trivially_relocate``.
 
+``__builtin_invoke``
+--------------------
+
+**Syntax**:
+
+.. code-block:: c++
+
+  template <class Callee, class... Args>
+  decltype(auto) __builtin_invoke(Callee&& callee, Args&&... args);
+
+``__builtin_invoke`` is equivalent to ``std::invoke``.
 
 ``__builtin_preserve_access_index``
 -----------------------------------
@@ -5027,12 +5038,8 @@ a functional mechanism for programatically querying:
 
 .. code-block:: c
 
-  // When used as the predicate for a control structure
-  bool __builtin_amdgcn_processor_is(const char*);
-  bool __builtin_amdgcn_is_invocable(builtin_name);
-  // Otherwise
-  void __builtin_amdgcn_processor_is(const char*);
-  void __builtin_amdgcn_is_invocable(void);
+  __amdgpu_feature_predicate_t __builtin_amdgcn_processor_is(const char*);
+  __amdgpu_feature_predicate_t __builtin_amdgcn_is_invocable(builtin_name);
 
 **Example of use**:
 
@@ -5051,7 +5058,7 @@ a functional mechanism for programatically querying:
   while (__builtin_amdgcn_processor_is("gfx1101")) *p += x;
 
   do {
-    *p -= x;
+    break;
   } while (__builtin_amdgcn_processor_is("gfx1010"));
 
   for (; __builtin_amdgcn_processor_is("gfx1201"); ++*p) break;
@@ -5062,7 +5069,7 @@ a functional mechanism for programatically querying:
     __builtin_amdgcn_s_ttracedata_imm(1);
 
   do {
-    *p -= x;
+    break;
   } while (
       __builtin_amdgcn_is_invocable(__builtin_amdgcn_global_load_tr_b64_i32));
 
@@ -5071,17 +5078,50 @@ a functional mechanism for programatically querying:
 
 **Description**:
 
-When used as the predicate value of the following control structures:
+The builtins return a value of type ``__amdgpu_feature_predicate_t``, which is a
+target specific type that behaves as if its C++ definition was the following:
 
 .. code-block:: c++
 
-  if (...)
-  while (...)
-  do { } while (...)
-  for (...)
+  struct __amdgpu_feature_predicate_t {
+    __amdgpu_feature_predicate_t() = delete;
+    __amdgpu_feature_predicate_t(const __amdgpu_feature_predicate_t&) = delete;
+    __amdgpu_feature_predicate_t(__amdgpu_feature_predicate_t&&) = delete;
+
+    explicit
+    operator bool() const noexcept;
+  };
+
+The builtins can be used in C as well, wherein the
+``__amdgpu_feature_predicate_t`` type behaves as an opaque, forward declared
+type with conditional automated conversion to ``_Bool`` when used as the
+predicate argument to a control structure:
 
-be it directly, or as arguments to logical operators such as ``!, ||, &&``, the
-builtins return a boolean value that:
+.. code-block:: c
+
+  struct __amdgpu_feature_predicate_t ret();     // Error
+  void arg(struct __amdgpu_feature_predicate_t); // Error
+  void local() {
+    struct __amdgpu_feature_predicate_t x;       // Error
+    struct __amdgpu_feature_predicate_t y =
+        __builtin_amdgcn_processor_is("gfx900"); // Error
+  }
+  void valid_use() {
+    _Bool x = (_Bool)__builtin_amdgcn_processor_is("gfx900"); // OK
+    if (__builtin_amdgcn_processor_is("gfx900"))       // Implicit cast to _Bool
+      return;
+    for (; __builtin_amdgcn_processor_is("gfx900");)   // Implicit cast to _Bool
+      break;
+    while (__builtin_amdgcn_processor_is("gfx900"))    // Implicit cast to _Bool
+      break;
+    do {
+      break;
+    } while (__builtin_amdgcn_processor_is("gfx900")); // Implicit cast to _Bool
+
+    __builtin_amdgcn_processor_is("gfx900") ? x : !x;
+  }
+
+The boolean interpretation of the predicate values returned by the builtins:
 
 * indicates whether the current target matches the argument; the argument MUST
   be a string literal and a valid AMDGPU target
@@ -5089,45 +5129,14 @@ builtins return a boolean value that:
   by the current target; the argument MUST be either a generic or AMDGPU
   specific builtin name
 
-Outside of these contexts, the builtins have a ``void`` returning signature
-which prevents their misuse.
-
-**Example of invalid use**:
-
-.. code-block:: c++
-
-  void kernel(int* p, int x, bool (*pfn)(bool), const char* str) {
-    if (__builtin_amdgcn_processor_is("not_an_amdgcn_gfx_id")) return;
-    else if (__builtin_amdgcn_processor_is(str)) __builtin_trap();
-
-    bool a = __builtin_amdgcn_processor_is("gfx906");
-    const bool b = !__builtin_amdgcn_processor_is("gfx906");
-    const bool c = !__builtin_amdgcn_processor_is("gfx906");
-    bool d = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    bool e = !__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    const auto f =
-        !__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_wait_event_export_ready)
-        || __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    const auto g =
-        !__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_wait_event_export_ready)
-        || !__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    __builtin_amdgcn_processor_is("gfx1201")
-      ? __builtin_amdgcn_s_sleep_var(x) : __builtin_amdgcn_s_sleep(42);
-    if (pfn(__builtin_amdgcn_processor_is("gfx1200")))
-      __builtin_amdgcn_s_sleep_var(x);
-
-    if (__builtin_amdgcn_is_invocable("__builtin_amdgcn_s_sleep_var")) return;
-    else if (__builtin_amdgcn_is_invocable(x)) __builtin_trap();
-  }
-
 When invoked while compiling for a concrete target, the builtins are evaluated
 early by Clang, and never produce any CodeGen effects / have no observable
-side-effects in IR. Conversely, when compiling for AMDGCN flavoured SPIR-v,
-which is an abstract target, a series of predicate values are implicitly
-created. These predicates get resolved when finalizing the compilation process
-for a concrete target, and shall reflect the latter's identity and features.
-Thus, it is possible to author high-level code, in e.g. HIP, that is target
-adaptive in a dynamic fashion, contrary to macro based mechanisms.
++side-effects in IR. Conversely, when compiling for AMDGCN flavoured SPIR-v,
++which is an abstract target, a series of predicate values are implicitly
++created. These predicates get resolved when finalizing the compilation process
++for a concrete target, and shall reflect the latter's identity and features.
++Thus, it is possible to author high-level code, in e.g. HIP, that is target
++adaptive in a dynamic fashion, contrary to macro based mechanisms.
 
 ARM/AArch64 Language Extensions
 -------------------------------
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 0d7173d6b4a4..beade861f23a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -66,9 +66,15 @@ C++ Specific Potentially Breaking Changes
 - A workaround for libstdc++4.7 has been removed. Note that 4.8.3 remains the oldest
   supported libstdc++ version.
 - Added ``!nonnull/!align`` metadata to load of references for better codegen.
-- Checking for int->enum conversions in constant expressions is more strict;
-  in particular, ``const E x = (E)-1;`` is not treated as a constant if it's
-  out of range. This impacts old versions of Boost.  (#GH143034)
+- Checking for integer to enum conversions in constant expressions is more
+  strict; in particular, ``const E x = (E)-1;`` is not treated as a constant
+  if it's out of range. The Boost numeric_conversion library is impacted by
+  this; it was fixed in Boost 1.81. (#GH143034)
+
+- Fully implemented `CWG400 Using-declarations and the `
+  `"struct hack" <https://wg21.link/CWG400>`_. Invalid member using-declaration
+  whose nested-name-specifier doesn't refer to a base class such as
+  ``using CurrentClass::Foo;`` is now rejected in C++98 mode.
 
 ABI Changes in This Version
 ---------------------------
@@ -129,6 +135,8 @@ C++2c Feature Support
 
 - Implemented `P2719R4 Type-aware allocation and deallocation functions <https://wg21.link/P2719>`_.
 
+- Implemented `P3618R0 Allow attaching main to the global module <https://wg21.link/P3618>`_.
+
 C++23 Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
 
@@ -222,6 +230,7 @@ C Language Changes
 
     char buf1[3] = "foo"; // -Wunterminated-string-initialization
     char buf2[3] = "flarp"; // -Wexcess-initializers
+    char buf3[3] = "fo\0";  // This is fine, no warning.
 
   This diagnostic can be suppressed by adding the new ``nonstring`` attribute
   to the field or variable being initialized. #GH137705
@@ -240,6 +249,8 @@ C Language Changes
 - The ``[[clang::assume()]]`` attribute is now correctly recognized in C. The
   ``__attribute__((assume()))`` form has always been supported, so the fix is
   specific to the attribute syntax used.
+- The ``clang-cl`` driver now recognizes ``/std:clatest`` and sets the language
+  mode to C23. (#GH147233)
 
 C2y Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -297,6 +308,8 @@ C23 Feature Support
   which clarified how Clang is handling underspecified object declarations.
 - Clang now accepts single variadic parameter in type-name. It's a part of
   `WG14 N2975 <https://open-std.org/JTC1/SC22/WG14/www/docs/n2975.pdf>`_
+- Fixed a bug with handling the type operand form of ``typeof`` when it is used
+  to specify a fixed underlying type for an enumeration. #GH146351
 
 C11 Feature Support
 ^^^^^^^^^^^^^^^^^^^
@@ -325,10 +338,14 @@ Non-comprehensive list of changes in this release
   different than before.
 - Fixed a crash when a VLA with an invalid size expression was used within a
   ``sizeof`` or ``typeof`` expression. (#GH138444)
+- ``__builtin_invoke`` has been added to improve the compile time of ``std::invoke``.
 - Deprecation warning is emitted for the deprecated ``__reference_binds_to_temporary`` intrinsic.
   ``__reference_constructs_from_temporary`` should be used instead. (#GH44056)
 - Added `__builtin_get_vtable_pointer` to directly load the primary vtable pointer from a
   polymorphic object.
+- ``libclang`` receives a family of new bindings to query basic facts about
+  GCC-style inline assembly blocks, including whether the block is ``volatile``
+  and its template string following the LLVM IR ``asm`` format. (#GH143424)
 - Clang no longer rejects reinterpret_cast conversions between indirect
   ARC-managed pointers and other pointer types. The prior behavior was overly
   strict and inconsistent with the ARC specification.
@@ -660,6 +677,22 @@ Improvements to Clang's diagnostics
   #GH69470, #GH59391, #GH58172, #GH46215, #GH45915, #GH45891, #GH44490,
   #GH36703, #GH32903, #GH23312, #GH69874.
 
+- Fixed false positives in ``-Wformat-truncation`` and ``-Wformat-overflow``
+  diagnostics when floating-point numbers had both width field and plus or space
+  prefix specified. (#GH143951)
+
+- A warning is now emitted when ``main`` is attached to a named module,
+  which can be turned off with ``-Wno-main-attached-to-named-module``. (#GH146247)
+
+- Clang now avoids issuing `-Wreturn-type` warnings in some cases where
+  the final statement of a non-void function is a `throw` expression, or
+  a call to a function that is trivially known to always throw (i.e., its
+  body consists solely of a `throw` statement). This avoids certain
+  false positives in exception-heavy code, though only simple patterns
+  are currently recognized.
+
+- Clang now accepts ``@tparam`` comments on variable template partial
+  specializations. (#GH144775)
 
 Improvements to Clang's time-trace
 ----------------------------------
@@ -738,6 +771,7 @@ Bug Fixes in This Version
 - Fixed incorrect token location when emitting diagnostics for tokens expanded from macros. (#GH143216)
 - Fixed an infinite recursion when checking constexpr destructors. (#GH141789)
 - Fixed a crash when a malformed using declaration appears in a ``constexpr`` function. (#GH144264)
+- Fixed a bug when use unicode character name in macro concatenation. (#GH145240)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -887,6 +921,8 @@ Bug Fixes to C++ Support
 - Fixed a crash when constant evaluating some explicit object member assignment operators. (#GH142835)
 - Fixed an access checking bug when substituting into concepts (#GH115838)
 - Fix a bug where private access specifier of overloaded function not respected. (#GH107629)
+- Correctly handle allocations in the condition of a ``if constexpr``.(#GH120197) (#GH134820)
+- Fixed a crash when handling invalid member using-declaration in C++20+ mode. (#GH63254)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -895,6 +931,9 @@ Bug Fixes to AST Handling
 - Fixed a malformed printout of ``CXXParenListInitExpr`` in certain contexts.
 - Fixed a malformed printout of certain calling convention function attributes. (#GH143160)
 - Fixed dependency calculation for TypedefTypes (#GH89774)
+- The ODR checker now correctly hashes the names of conversion operators. (#GH143152)
+- Fixed the right parenthesis source location of ``CXXTemporaryObjectExpr``. (#GH143711)
+- Fixed a crash when performing an ``IgnoreUnlessSpelledInSource`` traversal of ASTs containing ``catch(...)`` statements. (#GH146103)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -961,6 +1000,8 @@ Arm and AArch64 Support
 
 - For AArch64, added support for generating executable-only code sections by using the
   ``-mexecute-only`` or ``-mpure-code`` compiler flags. (#GH125688)
+- Added ``-msve-streaming-vector-bits=`` flag, which allows specifying the
+  SVE vector width in streaming mode.
 
 Android Support
 ^^^^^^^^^^^^^^^
@@ -979,6 +1020,23 @@ Windows Support
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
 
+- Add support for OHOS on loongarch64.
+
+- Add target attribute support for function. Supported formats include:
+  * `arch=<arch>` strings - specifies architecture features for a function (equivalent to `-march=<arch>`).
+  * `tune=<cpu>` strings - specifies the tune CPU for a function (equivalent to `-mtune`).
+  * `<feature>`/`no-<feature>` - enables/disables specific features.
+
+- Add support for the `_Float16` type. And fix incorrect ABI lowering of `_Float16`
+  in the case of structs containing fp16 that are eligible for passing via `GPR+FPR`
+  or `FPR+FPR`. Also fix `int16` -> `__fp16` conversion code gen, which uses generic LLVM
+  IR rather than `llvm.convert.to.fp16` intrinsics.
+
+- Add support for the `__bf16` type.
+
+- Fix incorrect _BitInt(N>64) alignment. Now consistently uses 16-byte alignment for all
+  `_BitInt(N)` where N > 64.
+
 RISC-V Support
 ^^^^^^^^^^^^^^
 
@@ -1082,14 +1140,29 @@ Static Analyzer
 New features
 ^^^^^^^^^^^^
 
-A new flag - `-static-libclosure` was introduced to support statically linking
-the runtime for the Blocks extension on Windows. This flag currently only
-changes the code generation, and even then, only on Windows. This does not
-impact the linker behaviour like the other `-static-*` flags.
+- A new flag - `-static-libclosure` was introduced to support statically linking
+  the runtime for the Blocks extension on Windows. This flag currently only
+  changes the code generation, and even then, only on Windows. This does not
+  impact the linker behaviour like the other `-static-*` flags.
+- OpenACC support, enabled via `-fopenacc` has reached a level of completeness
+  to finally be at least notionally usable. Currently, the OpenACC 3.4
+  specification has been completely implemented for Sema and AST creation, so
+  nodes will show up in the AST after having been properly checked. Lowering is
+  currently a work in progress, with compute, loop, and combined constructs
+  partially implemented, plus a handful of data and executable constructs
+  implemented. Lowering will only work in Clang-IR mode (so only with a compiler
+  built with Clang-IR enabled, and with `-fclangir` used on the command line).
+  However, note that the Clang-IR implementation status is also quite partial,
+  so frequent 'not yet implemented' diagnostics should be expected.  Also, the
+  ACC MLIR dialect does not currently implement any lowering to LLVM-IR, so no
+  code generation is possible for OpenACC.
 
 Crash and bug fixes
 ^^^^^^^^^^^^^^^^^^^
 
+- Fixed a crash in ``UnixAPIMisuseChecker`` and ``MallocChecker`` when analyzing
+  code with non-standard ``getline`` or ``getdelim`` function signatures. (#GH144884)
+
 Improvements
 ^^^^^^^^^^^^
 
diff --git a/clang/docs/Toolchain.rst b/clang/docs/Toolchain.rst
index 958199eb7a2e..d56b21d74c7e 100644
--- a/clang/docs/Toolchain.rst
+++ b/clang/docs/Toolchain.rst
@@ -347,3 +347,8 @@ workarounds for issues discovered in libstdc++, and these are removed
 as fixed libstdc++ becomes sufficiently old.
 
 You can instruct Clang to use libstdc++ with the ``-stdlib=libstdc++`` flag.
+
+GCC Installation
+=================
+Users can point to their GCC installation by using the ``-gcc-toolchain`` or by
+using ``-gcc-install-dir`` flag.
diff --git a/clang/docs/analyzer/developer-docs/DebugChecks.rst b/clang/docs/analyzer/developer-docs/DebugChecks.rst
index f03a6ecf2683..767ef6565d33 100644
--- a/clang/docs/analyzer/developer-docs/DebugChecks.rst
+++ b/clang/docs/analyzer/developer-docs/DebugChecks.rst
@@ -317,15 +317,10 @@ ExprInspection checks
   The value can be represented either as a range set or as a concrete integer.
   For the rest of the types function prints ``n/a`` (aka not available).
 
-  **Note:** This function will print nothing for clang built with Z3 constraint manager.
-  This may cause crashes of your tests. To manage this use one of the test constraining
-  techniques:
-
-  * llvm-lit commands ``REQUIRES no-z3`` or ``UNSUPPORTED z3`` `See for details. <https://llvm.org/docs/TestingGuide.html#constraining-test-execution>`_
-
-  * a preprocessor directive ``#ifndef ANALYZER_CM_Z3``
-
-  * a clang command argument ``-analyzer-constraints=range``
+  **Note:** This function will print nothing when clang uses Z3 as the
+  constraint manager (which is an unsupported and badly broken analysis mode
+  that's distinct from the supported and stable "Z3 refutation" aka "Z3
+  crosscheck" mode).
 
   Example usage::
 
diff --git a/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst b/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst
index ca3a56828209..5d662cfb65be 100644
--- a/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst
+++ b/clang/docs/analyzer/developer-docs/PerformanceInvestigation.rst
@@ -10,7 +10,7 @@ Performance analysis using ``-ftime-trace``
 
 You can add the ``-ftime-trace=file.json`` option to break down the analysis time into individual entry points and steps within each entry point.
 You can explore the generated JSON file in a Chromium browser using the ``chrome://tracing`` URL,
-or using `speedscope <https://speedscope.app>`_.
+or using `perfetto <https://ui.perfetto.dev>`_ or `speedscope <https://speedscope.app>`_.
 Once you narrow down to specific analysis steps you are interested in, you can more effectively employ heavier profilers,
 such as `Perf <https://perfwiki.github.io/main/>`_ and `Callgrind <https://valgrind.org/docs/manual/cl-manual.html>`_.
 
diff --git a/clang/docs/coro-async-task-continuations.png b/clang/docs/coro-async-task-continuations.png
new file mode 100644
index 0000000000000000000000000000000000000000..0a8894d8fdb1dde9650c31dcf54be01f62982980
GIT binary patch
literal 858961
zcmeFYcT^MK*FFk@qM~92siCN#QUs(!1_eO`K|lpWh%^x?5v7-miZm$!QL#Z(5X3?+
zDo9kMm(ZlDQ9uF&3`r<SNM?Q${d~)Me}CV#?!D_}lgwl?=gcYl?DOnr?<AkFwcNN~
zYQ2Dfz(%X1<_-b^Yr6#m)~s2(2D#$j{n<c3V1u=vnb`>|Gc)-Um$B#l0(=Anjwau6
z5^;8H70-2bP!L{gZhCC$_~y*5rpMN*U0qO2I(Br`jjdjXPn6}QtS@#mFMaw_{-*5a
zG7l&5#@}beMfdHedVVafTPGMY>{PEu)(eMLNcX`mgYe%uMgrZVYzMzhFV>kB@3Ir$
zHcOJqzTKf1x=PVhP&Pu)y53a%m-&%+A?GzC#O!cA!KVQw1}at1FpNpLdt#S_z*X}@
zkM`Uay{v#<EAV|+MjJyw{+I$5tz-A%^Gk(+%tImy#SUW|b_6+$9o$jh>$^>&A)d8P
zz&jL;xhr_pLT|!_eSO1ov&5cN%E>2GuMLTL625+IJ)A^293T|<<59>nw9%YMPAbu_
zVB<vwTQbghhc!k)B<WYL4731i%fhu2lPYT_OA}6+IEru0OO>3Qvs#)su={i(D{hyX
z;#SW<ffHv{XDggM&lf1}a`(ZHterS}Py54-h*V?cyMv_WwQNC;&C-V-skbJtrPb`U
zkr3G%cn<2;3YOE@<Y|)h(wy;O2={PXd8?)n^Se!UumO*IrT!I3<Qb%QB+S_I8Q#zL
zW(1FO&+E|nvx=WjCJ8Yb*KOJCZz`ieO5aGWvD(`*VX<MTWxHmZv#R{v<Xs=3eX*|&
zx`}V^!xFWf(3_3I6dgNKrkb6YTH}XLzWtQDmK6&%IF|-x-q;2_R=R~3MI{|kS-<vV
zV|vz+TP)`tc&ZhX>vhsA*vQZJB+=xzM%%j6jPskDTbW)WO_P+5jYYl9DfyB1!TSI@
zLlt-7b{i^2W39;9%>vG*g8owXHk4m|Q}wc5^m|d%bpce1>6*=giMaFU%snx9VRx@X
zf~;+dTQbDbnKbH-YcVaL?5BMObos(}hF2FL^s{5S5$l81*9sZTyZ03Cl3b7%G`^O~
zLdnrRc6``*`P```I2w1A*L6vL+jo-6n)BX0BC5&PJ_~rR)l9mOBK-sXMnY~=39i&|
zJm9OBd^&CEM&S+JPfDLQ9ya>fwJZ2ZKvK5XA)|h=8Is)yp8fIl<AVLa@zG|X&r`L7
zg??PE{Sc}DM&r@&j`FWK;vE-{jXjX7Fkt)yXLNJh>Sr7BSoQ%4>HVdBmt;@cPjZ^M
z&tr~Dcj$$e`c84bqRKUH1P-0B4%iYVV3e5iS^eB`XOnXt&1Gv|tu}h`Id1=NEAmb0
zdujzFLcnwFkj@yeyv?R9=i(#7ociY+hIt?3wuVx8dB{7PN16AV$3DoKeIBzCPFHg7
zvh6AsP+3NOq0Ai*A=h6&;97r$35tAJiNIoMnY6Q0!2(Nf8|*&)q&?aw9i(<q;1QsD
zqJ?ovrnqkNkL<WjWOVGg4IS4urZ!8y`54<JCVc(G=5=GHTkoxXPr5-pyQbj!{d00E
zt9G9=a29I&xTkc-$!p)v0nbH_|JbE3s(x+%k2P;Zo$N#+1mAr$nwLDZdFx@bok?iV
z>&}WxN!p(*Vlqs3$s`}Tm2q;HP^SEP?>$<my@yUK{%R4=ml{r#v9MK?$k3hG#MnG-
z?yf{QvEh_u=n19%^|Hz3_M4;<nx1Uy-Vn0>%vE*M?592#cE1ySZD#j0yIcB#5oguh
zM}dL_O-;uRK`*bfs#~p_dNn0=rS~@-D?9c|-2VoNEj5rH>VXPY?E2vHM(FHEg$QHy
z4~0J}=({5f+~Ws-7SjD$W_8lA#p5j-)-4I!t->Ej&p34Nn2O@<t<~E$ZRz?DUKBT<
zesq^s@_>0bOMTCliX`nM>m=!<=p^N&z|YN}<+dj0+m$OvW+__5ILH+xRi;;_gggql
z(|?F#dRpmWMo#OY3Geb7XOF7yrtLy0VRi#bV8I0k)xEj5kKWAC$|+ARPwPGU)@J%~
zOu<F_*G1p^^)HHt%2lPFu{JS#ZyB3cP;kCr@`?ECJ*OYOJY7)jG*>rbR)wD#nF*UY
zySP0xp5w46JEPoBdro}PbK&$s;Ea|UzWVm*<3>lKO;;X&?7r0h!*8w5<b{(L`u)y+
ziLff{R-2aLm`4e|zm2<!yZ`-}RZnH_#6^aVum9-_aszeGhVvS`rJn65y7RKi+`n*O
z^=rz=uE70srh!kER?WR0DXc_kA4o8WYu|ieQoCBS>wsZMlHq`%agJ)v)|_vKuMNMw
z8+oVxZr!`Sko_U}nQCp#f*o1S9UkglzkR=Z<D||XOx3nI7(g!_@o-9fKPX*HDOB#=
zKQNjXZbr*OTLUKmS1=y;9*PzE`k_mH<lKvo7IE5Z+!{g?qBlxx(2%PKS=fZ<PjWm`
ze{N`aouHOrV$o#r(ZV4w)S@u;UF!WNT$4-F%QU=X_=5vBF*cv~ygZ)%ko&RMC)nBG
zmDYgY>6)S^en~~2LQ?!*PWx*6S431Ct}r>j+*vdkzzku2ne3eWCca19OYQuberN7^
zt1@rb^Jj{yjD6YP+eY^FTzqos{g*5LCjPmh*Fw`4O{5>Bm2_@@c<dpy+T67|uzGII
z=>D+hIWOG`+~U9aME8I1bPphYtPd3SxY4|}HK}b&jZD9+{wDS&olRo0_tSr-$EPo*
zYgnF1wUK-4Iq^1={OD}X<ww;9-(;Ve><c;h^LdkfNzUA()m=Nf1}*z6MACA#-+3(T
zmxz|gY`4fBE#5F#RxIlF(2e-+ZEfq<i{A{tseN_)Hdsscq?YClSr1W6*2+P0SBxJR
zqodwMPDcJ-=C-$g+&}7HILOo-@G|$RPK%rkoa|iwKKAW*1H)=@c((7i|4jSx>xI|}
z-`~m$qraC^z1myHX2<Fl!Wr3Y6ayqJPIQgsuKlp;y3kFbGeT8D<3jC1DB=67eT2UV
zhl<p$A+1GkIlOkAsLPhiDZZb#&A*8J_T=}Iio<q^MTw0GcM?t3*NA4v>!lxQPDppt
zS#rPOb7R+2^j33*qSKM@`114zTSIp}*eSciZ42+<`I8+T_pX$s4q25Md6660&1xyN
z(%5Y!Nm$#*%K5uaD%aU<IC}Q}5ISh$jmx<DZpmHASyH#GTGzFXW<@uNq+_(}l^S)P
zsr;H0rmpMTbN1ea1Dd{LJyr`=Tq_ZWbw$ZVMvlaJCH5U|?m_nvPGyjwQ|amX!&Ay<
z)n>OxrkuW+`z*S!#0Jd$nNzkePrFaA;$Ve!$Byh2j8D9GU|SSH_WpaNT0Lb=UpsXJ
zJ)PxJbJtsH1}4K1jLOzPhiJdd%GjHf+d7W#d_2nME}Nqdea|Qq0vBAj>)s@67h}dU
z_A>yR<M*P@Js3FHT6uRmtWUxYJiYnkeEWu&gmV^5o4)(Z^XNy7_p5w#wTuG8m@-TQ
z(aDgGjk&Hmctefo?<MiJ@%u%|U3QPiZ$4c3_;ocNTn`l$9L+m=I>i6U;HTp$^GC*x
zyiT#yaIayxmX)$RlO3)6TC;aw-z@>kEnXQWy|?bmu2#R|S{-Z<{MMV@xubsni_;fh
zt2(nIz6_lTed+Qt#fj*q?bjLakllOSzsP@w|7iQCcBuiESFZKHts~E~)7HZ`O{g0O
zHQZrb+@*#uX*g+rJ;}g*!wY^D5zINYkFGQOUhlK6*;+G+jeV!)G5QuWL_CSr8*(q_
zQ8%@wFTJ?;qVLP^FJFu{hO56k<iA}fr`93NZCJLn_Q}ALuy(o0+*|L@m3Y!nD-O%w
zzM`s$Y=ITixkK%7R58s>I|da(`>5}z+SF0$TT&yN66Jz|7A_ixe5SXg@98VJnfJ~z
zTQ*P5_0p^5h>!H#ZN1yvl(lWqLAm`g%l&U02kk5gK_=V6Cb`3pD+;@UyJP`cZ2k0g
zN_>OFumn}(t=kL#;6Q_gm(-HPGvCfo)oU?9Z^nY3l?B?2mc6WZvU3-G6SibX<2;kz
zqBj!MM6Qe7|LgQh;LkZq_~qYq95uLx6i)fB^mTuINhERM+RwN)av(_V`#peL@c7Vh
z*Rl-%8!c{(qnkq$*3H)oD+`~IY;sQl;kfVplKqjt@N_VwE~B3H_0&S)0D+1bsWWC}
zlQC-F#@`vax4JLZyE81{&h#2}?8E4%QF3+dvCHgOx(H9~6lxeniao|%m}T~B<ZJj~
z0*Sgf(Uq_gy_y=Qwru(eZuNo3#v}Pc^HCxKmAeFX&VM~BTVjY>+F>4AFlY7y@c1HG
zUh+opgeW|nSNnt9sV1-#B=lkZOrN7b(o@0B6v6JEJQ3;#mtxx2B31JU-60vCj+)*^
z?!%{sS<sK=ZgG`UZXddrFua&JRjfY=K)j2aKF(I>j~^G<hn%k!5EhgYSdE+sA}_R{
z^uNz71XTot{#;)rAdu)MApFlgw#bqHmw~+aW&R$8ZY2naAfL7(uZYL1{&n}-?#Dv^
zI$y*0Yy=KDnps&PM@O&AK0ZNLE?`5fq_jkl3+pZ&b-5xSAikUb60~yIHILLk<ag3J
z#QC_5p%*q#-NPH}>7yPIc!^(+fN_K&avJCp;vpXq7!Y*DFv3Lf&mD%yIsdYTqWqs*
zLi|k>osXZ8H^W}`k=Io}pngCRy<T2k-uSZjc|!+ti+_qEpG*`lgoIo&)X)eI4_6P@
zR>xlU)zCCBFwi)lrJ<#zhTNfcB{C?)BSJ0ciqhXo{#B2;&lRuBewRZ0utD<tdObX`
zp&=%Uiu?!t`|oc%eIoq+^Q53F|8xuKpa%aN4Ndg}8vm{vDQe8WYIwpg!Y9DR+%FIr
zGvpa)O`U^U#(xU@Utj&_k^fcH`9DRqv~+a;yXb#?^*@WAy5e)$3>%0%Gz9&h3Hzt;
ze}DN;L1PVm-~VeY{ton?tH?;B*Bfj6d)CnFwIAs5r&7kx-0mcDMAVG`C&X7emA^;i
zTtvwY$YV?j2$%|3nIAeCAvpU!`c>fYDZ<-8r#o|(-v)SjWq;gxQ_uE*im-6<rSJ{A
z(i8%>JZ1~+3csqdR#-83yZNQ9ibwb3e<vKgdGx8N+|3MonJ1>AhHus-Z4SLY+mf{F
zn{j{)kxbiH`+_^QI2TH+m~^RHj{F+k^_KFTxzX<%Tk1-x0qEbM+59#7d#vt572(73
z|NH;HBk=#*5h&JLg}=WCels08tFu|fEWg@iPyF*l0*;ZD_#B0bw2J+aMR>Q62rf0h
za309=_c^PqZAldiU;u2)z?y1-x3*316YHwSqRucKgJ3^IJa(v2TNlUPUH;aR1$V-e
zVD{pep&vZW)f*AOWtBrGffw&j>=6_5uH`C`J2aUe>>hF%Y45)Hzp$BTv6GFAkpI<c
z#ORcn7LlG`B~9O|JkxCHPCv_Z+yu{-Eb6J<NIr`ymKeOV$f$&nR*-$(zu~vRnA+5@
z(<pU|V91y^E!K{-tiSJ-X@yk%jK~8k1!t939u6k^-wd6IdXgSP6+0;oDxOxd)ovaG
zRkIUJ*WNH)`?YE9*WkdQHQR&kopcnDjCiHDG16-A;cLbeP;D=_m!~Z?w!5G|0Zu)>
zq*wBBfLwmM<?U$(kQl3qGXq!IvdN97KX~OY5hORxHR>D*Tp^_%U$I-kM|VWIov7}t
z%7O#o9)c%ojyOl;teN%5AHz#-9BfZ=q&KH-yf+^Lxdy~Ps#es!I|`11i^@Y*E41=|
zS9k<<m|x+>dsAM6Z%$MnpJfo)n89sJnaBvwNS7vNj)3P6p4CvYl_-N#X-{ML{W-f=
zn0sP%^UG~(1Q=e@p_`?}7a8fBMvbR;E;s|HuyH=JM!)P6wTkRukAM%t>-nRb4fi+|
z$L>cf!lO_hR2wr@>aNs?ib%eHlXzaHbW7|$okl_<VSMD?>wH!%(~^SkLbbb>rK`<v
zYb5^_&Kk#9Gm2PVQG@1r`f~YsLr|uAqgny0;*a#b_k3UTu2~#Zi*A+5dngG`HCvk@
z_VtH@Vrf`LknRR?-!`vrQ)lOVb9{uvSL74x=H_cX(iZ23o$luwx0TC%xBu>ZIw3Yd
zn%zhIu}}TSuCOr~NYvI-&^F^u@*N+|eB<h2Teqbe<hyhG!p1Dxl<p2R_ybhc{S9cw
z$R?1b@#dzeesa#eTa2V<i}NES!}hjPd|h<|3KA_ITV?C1T##?<>1R^6RMAzgHLFxI
zH=hV!fBEKW)==`^y?Ey{VkNv+6|Xj|wD=owfBw@1&tw1lnExjS4ZO=z0@h#Ky!P4^
zGXcu8K8NH@fi}%`>5e!DZ`10`^vIEnL$cn-+&B`(AI^UsVK^%nS6cl9loWLHy>bU;
zv1Qt&*Uo@!0+&{<es&x6AS|kwBDMqnwwg!eg;(DQCdo4M<}$GGefmXYjRwMQ+-o?`
zoOk4m8K=n|2jPcsPVi>wUQN4~Lz#se+u#Y10klV5I{vDUE0&~+HaDIe8H1My!&{q7
zd^_;fX7xPZrkA`LM8fGJoaE3MoY@P^eWz|N77A}V6mYcJ__e2HuC5B41;1^BeBD>`
zq*v`1{(9;3mmMYYtH#f>ll>qR>cHbPt-XO;BWFsLM%1#}d{LtX1wt;9RzKr1R%1Db
zy!Mo?gJ(%q`EG9t{`3vIf`_i&7U}gJvdD=}j_h5+oKS&OpncG1r|ls2+s7rs7|n-t
z2_v|3`SuuzFFq!v>SvW3G1}ss^;`=&Wh=IpBo2B>@)vRi@siuQZH<@|QV2#+BK9KP
z36a5aIcJIDtg#YF(b(0vJl0EAdqiLCz-4-Ko%`qb#q#}7H?dnT9<e+M!k2Ct+Bje*
z*9(TupCFM1cZ&^M;(ldthO$FC?TDG!eL<18EhQZ+I1RvxKkujZ<0{~ZJ4TbUgF49N
zPt*nFk&`Q8M^z^t{GF0OxY748mDeA8nqYYMryH{du7)q}A6etKqVdvS;ecS_68Vo{
zVg&O^{h0KXxhbdDm_N6rhdn*+XXmO^d){}x3a<PkKSUod;>OT<c9QM$#q3dV1gzW<
z7+1+l{8F{~-c`pH+UqJ<9b5YKj_K7SQs1Dp^SOh1lV4k!MxQHcGHwmtZ=iM>KdOt#
z4@#O)duRt={~}Df`w6UmIg52M8d*cNP$Y96#<g<opc<a<(uO%=d+x<LL9Hg4u{?N}
zY!RXv64Lz1nV)?M?u?Cx;>0R$0(_Y~H3l6^))|vh#10$d?sAXvH;Xx9BMO-bT0f9p
zf6JJf&*G0f-=1Dsy46Uqw&|3!=bDVJc1}uUccD^Vcc^q~9-%KcekOamORZA4)Vk%I
zQlsYKeB*)4<?2G?pt`%tcg<pFt=~r?oBM^t2?gxK(A#}tTH3n#=Y+1MnY(w`_-Sis
z6p77c7*7ONq+e#mbzIjQGCil1DWUB+($Q``V`j8VEOPW!o(v>k@nfUz=Aok#jUBgj
znT%VU?oF*`-*u@fi9m~LMBGMU-K~+D+Rr-c>hjT*W{ixQPQ$jJVp^i#|MyY<pBO+V
zS@2SF&^F-%Q&R&L0*RlGTYukOlV#P+IB9&zdt!IQE{49rN+vv1rNwB)&&?bfJVSnG
zT$T?@tvRO<PdwTYrZ?iWG{|X(JE(s#g)3L{H@*>LZSyu=Z_wxIhM5S9ebAY16$qHc
z+yqko2tDO5mPlM~9wY58INSV?s%3wg?n@7+;D6N}DrOE|x1^LbtXNY@x-dCJL(nar
zIrv6Q-RSUJ{vL7Y>W@!C!#i)g;vJ~$c@eGMz|w9zU?g(Z_n5S~i{#7`TXe)LW$gP{
zo4^VUKTZQ`<wsADO$Bh;hAI}_ojB$DywCPSOB7>Zw7Fk*<<=x(?uSJ1VO|elkH!hG
zR3tVDmtZ(2ocf<SAZ0i%cK(810ULGN1`_#yNhKX$m;m$DiXH;5Y&KtNI5kw<dB&HT
zV0_myF#v}-2A~!n6o3{2F$1i$hziMcpUCjViq7qFsF^0;eL;TNGBfhM3yJUrvAn?h
z<m_IOZ>0&lO0&HnuYOizMS9Moatm|CArQYkm;P}{ezax3Z+|w=2h1}`)%E;Sdqk5+
zble5ZGHTHW942278);LoE&Ct!|5KA1QWJ-rTCGgfJwF`s-}Sth%lxE`9+LBxj{KF%
zDJRZiD-(MxI%w9@6d@Ap9Cq^@75b+^hg^q!^s4WF8aMmKvT0PCyJaS8{@`zms(tt^
zBJ?ax3^u7gO#knCs!SY^hK|MsD3v_2$1G*@(v}X4B(Y=n@?w5SqGmTfjr|7x)1V9c
zYiFBu9V22&tERRp{jje6Y*trp#{5=#kJWk-?7G=6V!5$Lz2`iJ(f#HU&)`f%OS>qF
z9UA-39%ntT$B}DyJOzntF5dEH%CXgqO<tX~L}-*70VQhB<!lE#cBY2eb5H#dpSmM6
zeQdm29;<>02Z(UnM^vC(x`60&X0zwE96&sJ=eW%*0>W9+(mUs#{)OT>K+<jwH{2jP
zS!CrrT~DRsB=`O22gqiAJOpwM7_;a?6_@o|U%!evc=p!Mvuuxf^rd(DG7hryw@viF
zo}qngLgx$Xqul7WjY0-XPi$ou6a;?Q-wm85n7*|*r-=Tccy~ZH5s3XT(onRsgibLz
zL5bdTq#D0C4FpvD1Q$3Y=pQ6Ps>`m1yEHB6%|BT+e>SUHW+LTP7p8qUOZ)Dd5NMWb
zfIHvMu4U@2OxBZ7E8Hg}6zAV>Njo^)XCzGOx+JBv-7DN*mt&w$(LZ=cUkytezzWUU
zd2BZt!7$d0bLe7$4m6Pc-CQ_gZ_h8&s|~9sHdZZKe5!rANlHvnUy_^`nbYlThiNm^
zjI_A#$W(g*r~5sU=?|?OH*PhmDZ`n7Z9k5#2ecF>&e$E37Aud?JMnk6eu3iTBuu~E
zt?S1jiIeBgj)0q*5zo!~(Z{I0(2)9_v66KOV<ZE04Jdbc)AE-tR0V#D`-T0I>BP55
z?e|4VQ|KP#qc4Sq2B(<W2JW9xOKfWbu~cq8W#}*gP$OuUqB&f@RUS|4Vb|HQ^RdvF
zlpEEz^KgIkmcD!0QOx}2Mu3s3l^MK07Ve73dzf@paNxMir#q)!XKfbvzB(l;MC$hL
z8JF3GzR2!GU)4U}S!L46_j7g^bGjPdJ$ynao9!d90)RPg<GGOhu0eK+E64ZIAPwMF
z5C_NuCbBU_kq!v9y&v2TtE2OIzSYS~<l#LxfxA`Lj&BEL(%}@M_zGf|zp+aY#33Yn
zlPHdI;OdklkClvR1UzP4p>IT-L2?dY2&R&WA<!AF7r*bX649{xSV5>v;D(RrUUoFv
z|LkXWmbG1d%9d!0b0+LXHmN*@$|Hv#UGd|gVjrjU0{&F8=#i`0(r&7&^tVe)Mg_Zw
ziKUF5>x!?R%qA^A7dmG#5v$T3ZO`c`!ZkM<r}t@SBt{1{I(*jwM?NNQzrQ)=d%kSw
zer?`Q0<Y8m=3@Dha<Sy#DXIp7umcfBqR~}uDVvX>erGijm>d=8H@AoF?=ITZTfn;6
zMaTh+z%b~uvS|*{cf*HOWPMH>8m2NyXmEz99D~f&4+SxrL@d-z5kuaV!B{x5oCX?%
z=S*<Z2zRm<tFW-W__PlP%As3I+_<U%UGI1E)4^R`v-SsiO}wfDIlf4=uhDpEbWtId
zn|IbHEjIjDi4(wxwcxdQrRqH;r+JuwLY5(>(*(`6@an`ret0Vaa66*&S0;8!r?TBv
zz!lo8SqiYJ`9&k9y)kTX2d)5FSJp-<km|o>yWsK~uY@t0q~xV(ZW-%XtOb8QK5ypt
zFFpDQh1NyJm0?vq0z5|86^LdcTkGYVR9r}o-fyjEIov4tS+3#0PGK=Tzmcta)oE-u
zdI_W02{6pNVsS3!Bjwdh+s*K@w?%I+S(yH~cJ|=JTullXgKd+QSWpplADI|MLhhU|
zMKU3#cdZoIk5V{$@!3h$9euz;o*bk5D2~kiA|mE(z|)rd35;j()E*l7=OL6sE3A%c
z>TrL`6Crm)My_V`W|{D77WBbc=^;H!t?!jxT6<4&b;eI1cCqoy6t{+ziD+Pn+lm!^
zu+DcoxC3Onc+)B6*p8Q8&|pNLVS6J%Td|R>z3DHC;47%w$_4&dD4T$3)F$Wl{+)3Q
zt8oqX04LL<gPI#C8yiL^SP+91wDPo(o^J6~<+P<R1@zTrWb{#6uhym{gte(}pK0}H
zRc0=miAzX^Dv?ea|6s<gjl3P{z*~@RHj5<SMjnaAvW7eDfzSmVlWA@ra7}m1Py&d+
zpHcP1*Qg%la&<mmwM6g<tc1nE?Vwt4R&b#oozmKcLX5PX+8@3AV?uT}&2n{CJ|ntG
z{KqzW3`_sVKpN|<yiM^V5Es~@6p>|{G3B;#%JGS2hK~WLy{Snj7c3a7wwcklge_&S
zJVW`mWQ_nk`OrBdu+1#$5ia7Xw)xfUZ8;{Zq~6=E+Ndh8cgKFbIDTxEd3yQ{^xaJD
zNw@y_d*MGFL}Z_#T-uZbps9t@bFUwkyOzh?#x>&{c<D~V(_(-5qE`5wGT*1Ves$>t
zH+ZM0m&ySI&y5AlnvY1pH*=~_ab5nAx3ij@6S3YawCfw4-y|YV7IsBLw}*nMkS*Qw
zc&x<7ts|)eo-tZ@&8L|DigoHUV!x$0s6MnE6bGfvN^K26)nIXdGs6;=Bf)$<TisR)
zZV>4q7~{y{&u5D+THnPpz~<>f6GJ88m}%c#zmKH^D|rc)ZXOUlJ+ACV5e#~j2jH~=
z3xmE9bX%b2>2P{lO2{l;rWvm)@rW_Vn-=X!x7l=cDKXpF8+3@gQx#MLyUbW-cdh1)
zEBwmm+(&syt@KG;wpGpnITEUkq~!`44`GPzFIDK&^XbXq4}RQ|r8x5SqprwYunEKB
z11nfa4|e4`a-F!fk5JVQDOJ1*-tq*6IEMGp_?U2$h~Qpg3Xv|v*VG@PhG1vd4`_&e
zC0R0+@L|m-EWhtdyoPclc+AV9CWZ>_03{G1xN$q(L;5)tyn|Hy@_2aDlJDH%!f)^_
zIKW9BP9@_X5UZ>00Iex8lrSnZ-*}0DOyKW>oH^I1oV5Z`A>(V4l=fO|N)x>rg%Vef
z+>m2zJ{+&5Za8|>50xVl3Sn1s%<^j&M)6Cr9N84n&5;@9mONope;%AUeF%MUCXsCL
zJqhPSR^{nML(}930}3m|qbs8DyvQ{!sT{6^)PvymKV%7C=a*0<d%i3Cd9_3^q>lK$
z|B-v+>HGza2x*MQgrO2<^A|MOQ9;xh$BykDJ0CFQOJ1hUA6lRr#kAGVl4&z`93JtV
z)gZ^xZ7Rj~+Eoq8b~n%W30D&~C5$Udiai+UYg~Oew>~3OO25hGdp|T&ZbNp9Nsmw2
zKWU{Pu_<!VRYVLQb=)s;S)a#Hh~oxx{1Vk2fJ8Y;-~)*L9`ktxQEpPG?+C}AHyzy-
zjaaA&aKWemAtvk~-!j2*oa2Cqe|+M*Sc(|?!!2$x3qUr~FSG{;*Ing=BnuJ%K1aln
z?L+P_WMPpKXvcBG0c$#jj-|tOrwe{vnp)X9Xhk7(fkTHW<jB}m=qZBbM&FDsyc)-H
z8TE^W>JeSmusww0-Qu{QGBlqDXBSaLpu8~+c7~Z8j+vkFu7%c%=T6;TyY~0q+D5d(
z0Z&Ugq0a~0RH|J8oxsQmS;sUSVT&v6)6N{h%U_NdN$;3Ye5=Bnu_tDGzw))2XpVG6
zr?6jI{+T{Xng;b#3=Qs~%C8W3JNzRgTu2W-;lkL&0XWqI!d5@#(o*B7*SE5{w7lRJ
zgw>}@{MLSTC-W5+*<jWF5KWJGSJA!qR>96(gFFYW0b;NtSrz(&ZTG|x1?Ee1*CmWj
zRsFxfsv#HxeGog^bSsCrL;!}(B0pcW!CG)TYp)pl7NN)jeyFLPq^IL`gB(fAkoCgC
z?CoyF#``7A4)oGL#3>xBbX>Y7)*$RZ!o6yRPCNZ}Tl-xnc3auE{v(1!M(h4TT<t<%
z6A8%d9b~*=g-!d;E=W^IbwMWq9<6Rq__3uf0wA%QFdhf+QX1-Vmr+oPn}^*{<<c+1
zRfDtGk5(}sUM>!DwC}dSs?k7wT<91_KzixKva7=ggP4N3|Dy%A-8qFB1t%AKAy=+F
zw~A%O6M}kq@iB;HD~Vp?M{h1i|6SZh;17vr60!}e1+UgLCxD@lR{g=de*xp*n6ev+
z)!v1Pg}zb9uWq=~m?HJ<x^i{7G~#_`S~0j#L6j>bgt0APGcY-9IiS~#0!IAun$5UO
zae#Bz3QF*j@0A?ryKpw|kmqC1H|>uth1Q@WixO2_6BSB)y4TRud5jS<<8%F-*-abH
zEyKHeqq}iZytYfKGqox_!*}fkC>eRWJZ;6qnyW$OmT(Yz<s7p_=3uz)5Y~Mg8?HzJ
zM&QZC(BUz{5;5{M1!ls(qYKzD7k0@Zj>04G{Qfx@4~d<f{lH)1ziSKr(pJ7qHb}Sp
zBTjV6+zzmd;NBVi*)U<U;(c%@J~9$JNP?4OHKL%I@tmtwK7%rgKhWuOD%?y3AokZY
zxu*fLTCs(dt!C(vxT`<r{EDLvo96XCZcM#xh@x87mu)l<ellMs&mtUX>u&j?qa&*-
zANgE~B;<nMltzeRdKt0%W-K?2pKX7#A#HIc|2ucc7OLcYoXO{W6kvCEc9(Q5BoeDH
zWiFL;A&zLDcsa2lni=^U@cfP}d_U3G2u@D>3>PMYRd11nZ0Xwam-?lsG=haI+#()j
zf&ffk;fHIg;{U}z1_)qIAj%N@fCeLTPqgLm3N@Qwi-TJ3T&4ORgA)nr4|&>XuD;m8
zrttKfzm4R;_T-(Gv%NIAI2=>&1;!x1M@~3KGWtPDh)$R2c<`Z&@F#K7i}R)3MTrNa
zB--D(dsUK?fWW&}k^MAh3*OW(-=lAn%#0?Se@Ihh2ZK4XwC5O03o??0(lrwXNxVkM
z$O>Jtg#$1Wi_jm!RSg38>~j*2%+o_m7e7juLZmRA5Y5G|(CWn1$G8}z7yptTN=Ynq
zmh1LMzAf8HFnLrT04zgomcEtyYnfcg<36MfY8(S7zpHrK?p#HE)_arCb7qlFCR*|<
zANTj1zkfT?Fjyh&^xCyUFW!yaxH^=P-t=yb<@jBU((Y%S^-_=Lma=>&EJYSKU$*r1
zklmaAsBZ=%vO5pQYyqr!ErzXJk~P4z0#sDPD%zn>Fn5qZ)#du4iaLKPcX}22PJ<bP
z9Fh0|kcxHj9e*Ir9I~R)V#Z34dA&lA9{hR_%lic)F*7n#AFlaB0e4Uy^NIhzn#zA~
z&yMGqJGAIu4_(D(kG!dwcN4<xDF24b&qHpTuR(lUL?H;-I~LaEXW(|h+T%~a_ia%2
zTg(F2cOjcM_i8bE`iMyZ#RF~$Gs!GhruB)@?(9DFu3TXD-kXH$9*5N};!F=*2`(*@
z7Wh>A+iP?vu|7lVfbGc7svz#-&Y2$`LYYqW2zpH2QclvBKvwQvv-|0H7AzKwoF<-P
zT<(%w;e19nf-8xM^Mv3Q$XS6>1KoN3z6xpD@0|SC!0Dq@THTmfStra##o`D?*p7%Q
z>iP~au%YD23OA3HN5VLJ{R>|GJ+|YV5rpQKIvbvjgRMG3^2orxTEw~VcjCsoT~MO<
zfK%O26KzV8<iY~HwU_LdPS*Wg5%GW9;$3I_+asM&z28p1_=3fU6qNeDi?16Z4xCO3
zOHI<<kbV+Pz2jk67LcrU*}^NoM9VECKjZRG<;{sLFPm5IB8Zy4pBWB&1i4VN!C7Fb
z3`c<6cs);Weav3)E7y|O5zk11i|K^(P+^bQ8b08}(vdkBR-W?xs03@%5dSB5xCU91
zs6b#=6R}9=*8S!8RngTvN)MFBE951H3L;ECO3sbMuH}WhSPwb=V~`RNcEX7+5q&+_
zf&i-4U*kEP<#5M2^)NqV!cc99+V{t=I=gg_nM&jaK&PES)|z@ppoAqf+#Sh6c?ypU
z(^kKs-4D7$>(+pWz*#R7x(IFC(;Qq^^1J=k0^8NrJb!X-my+Lwyq=6~l#Gt5_>%<X
z`bv+h19roKxluDy4Gkq7QxOHTf;kodPf~*MsQC=Q=#O>a%@e$#bXaxK?uD3#Wh4rs
zJb^1&abH}z0N#7JC)^u;4RguAk(CS3=H>ujX=Dhn?g+Yc<n|$k>!(Qm7-5XyMj&p7
zcfgAX=>UX6)7&`J)uY1_sS3%m&+|IsUe7=JLZSTuG|Gr2xc_LzoR89|!=MQ*>s>f@
z<3$dojs(wGkX@#o#2H;$kf}N&qrG0~bbfaf&))K(SnL!vQVPmk>I5g=-ij2gzm-MQ
z_0yDZ%s8O9kuKGKXpH=`Z~nq#F>_hoL3CSQ-(xv<2e}Adsc&1*O35NVA;owB&@x#l
z+)!l3PDPD8L%EOm8QB7Jg*|wiX28cTA7eaG14OkFYM_eGaMUCLYM?$_28Up8IHklh
zR`wTs1}t1AB3%#*7CKl=SR#+%|3FCmnAp<lxyLMjm>vBU_-2<}@);j{mtjRBBN6)F
z<;3B1{z{G|z`sC4n8S8iA<b%xf+@gF;-vlyB&x!XYnP1Dxqfw5a_M4*T?}rcvkTsr
z$6isv%EFJc!j{e3CZ5o_KPd2(c7_I%k{$803Z+%7y6)4d)dF6<NiF`}QmfbGX(rzZ
zTym4nUp#j#7c$=5)qXtH&^w*Oi!PFatQJaS6@9!F3EVS8!vW6Es=0hljwsVK_m@3<
zGWG&A<4hIVKicI0TT#Fs&c|OCWMTv=9`?zBPl0eJu!7~0hoEdYdg0o9)H0m%9cF-R
zA{PVEEV=+>Uj|m(C<Gqj@Po0?)s3d(zQ&E1e@O*uiW^;p2C=aJAE|X)0w#Y_6ej4{
z&nx-H-KSFF#<5ovJZ&>gn#a9XmUD`GO=faKCWZ($1@K1T;q!3TS77Jj6Xvlk0`8}J
zr*s6`^(8l6IC$XFr=35uotnKbVpcwi4&&=Y&%CFicg~@C{S9^)?-zq1w||Z04pqmb
zu5%#P6{NFwal$(Z@9YPHIaqI~c0pTjA#U}Yo;AQK;B!=wCbsQA&mF%t2tOL+{L6YH
zA=K=M&02eQks<Bz9>{3)Q^fg?y2Z=fxWKe>#JNV4;Uz!LpiQtU<!{ND0W)O1i`1V>
zMKT!pZk8|cgL^B*`PDjymc_%8s5YVd>HbCkBr<Sz4xXSlccu{Mt+NHeSn8BEb{BNc
zHt!XcPFyf(=izrsdj{n394T_1GC{)zE!w|<dcDZKW(8NK*Yoy2ce>CvTI6{!&;I_P
z^M`oug|kaf<a(2uZb$jJ*vXokq{JP0sv04`GGxJ1=gof-O4FLPdmp*zm~fQ#NG`O+
zSDITr*Jp={xImQS73L6ABKyOYn80h?<n!bD<ORFOHoN5N9=cU~n*?t5V`=yYNSs%8
z1Aj<DDc|NkX{|h@`|--o{G}K8{q{<Ai!KP9&8h7|E$oGS^!6fFO%>`-yjQN?!FL|L
zM2$E5Aw!d>nSeVy`T;X^8rkemaSfX9#X>Qxm4q%-L`N4Y2e|7TJ?MT0+4s}j=1uc!
zE^<3rZZyg?%obIddfUUwqJMAU<iReDKuabI{cUG-HtE2C{za(m3eM}M+}!b;n$Wh1
zlC5R>@@aj>MS)T`5<DNqS_mls>iX^Z#w}03l^Y)LOqEpBK=~}X+T|qJ-BeB1w6{bt
z?talb6*#I!>n-UkMRr!&L!e8>Fd65aH0(zq+7VfWV0BO4caY_aEak@RAlbnMJhY^O
zKpBH5#Oa&_U6kg43a8;6U=_ln@x`6=hsGSYAi;CMzoZc6R{agsR|^iTns7f{PXaky
zk=kxu4%f?l|En#fS3flA^etmtk5<DA>=l_T#1F4SSBfmVrrxOXazTxsdCGgvOAMQ`
z7$88YnqN2Yri8QY0iWi>a^<rTuWLHdEt$8AcrJ1)ZPBg;Hr#z%L`*To8nO)vq^Gx@
z_w5X*umLCDeC)A(XfCNG%eH%pa*;b}^`%6nFK@x<bdx57U=DD(4ry8l3&)Enwgj^M
zyAH#<CQXUs^6@gG99|zzjuSdEP5y*}wrv<Xyi~)|SN+SW<?^6GNW1ye95DjH?PCbp
z@Ylf}!!E5FQ+oa{C#;LZY~}~iY&q;`B#e~>8^q`;s9)h_jM*iRMtnF{ZnJ(^1Froc
zqW+_gy<D1h&6*Ame}>Pkb4yrkmV{LYfXhwn(u4&)$1rO?>-Y9~9{e;nY{Ztq7K**&
z6=edE=yqdEGld)L7a=cpKF5-f+A(@7rnTnRM1qxHzuA|8s}@RfqfOa%a_BUa(Dj2_
z3@M4oqe^+-Au9LHr3|&#%{Nse@Vz8Y#i?qV-(p|*uBLrCCxPrBc~0(WSReGsI6UK<
zIuZGJ3rMl!MNR|jAU;;0#UNl%1I~u|hEBOn1=<Te^im6W!U{*jSC9PJ`-SO>)gQ*+
zMs(1OK)cydbpGaY+bAdjSZRmKi31p9IQUsmMolta^(?RScKzBP`z==~7ySuntay{*
zvh&VHd<<0+#hsSB<PH%?!Vy8<NH$fs9{^sl9n3?yZAJ%wZZTPSvq+pX=8bb@Qx@~v
zbi_0mKSR6naIC#%6Kd_Xfp27C!F!`Q2e+qPmq^><0MuV-|Dm~5=I;J0qgQupsDJa3
zTCEmTA?DkXX5-E~X)<k6uZ;_P>@X~=%=itCjJAWai|sF=WZt=-`b}F<J|8RzIuIlG
zByjnM$$s99j!#|s@NHd+u1_D}u^DpyWHWRCulw|P-;0fzWFL4W$8V|T+@3IG9aWbq
z&0pYM`)u=?i1<6<^euA$Q|@~BB{2muv@(UT6AhItuQJKNLY5a@?N#0%jj}ua$>tv%
ziKHR8aII#ACCxH55Q@hgglKOYDw)mY=g%4;mgN0vpqHJ8=$#l+#EH)Jh!I@ns|hYY
zLqNy8Wnmh;=#f8l(8Lyqnl<vN2^nGcC8J*{EIK5FO=mmFIk0w7GcCV|QGeCG?%i^A
z^rGj>)(e}x7FnLHD`P)*lRTg1YJ5gQ&{AJjvg;NXDvW)^Hr%SK9H2?i@^BOM_WcmI
zPbHiF9DG$6q-xhFgdWMa2cpt@N!;czN~9DoH9Ws>imU^MmQZS6s?s{U@m&j~oj2g?
zAC3(3DO^62ReCLx0f<T=aT&d>;NxcqJK)b#qo$pO_;~aRD`uK4eRe4aTrA2T+FC^E
zEi*nv0ZTjt#2u)1yUy220^}BGl2rFM6X`bzp$_iGhJM8ku4uZSekG1~<qkr%&SnTh
zgryH&PTz~z>j|U5Q3Xg>SANaT>AGbg<QgjEJpwX+M2%q<o-z+|T$rAry^BET@JtyY
zs=3^_c79nl<9Dx}36A|{MJx}!kov1AVx?EAe>AY5KE`pfCRf{sdcYKe-fQv9{$jt>
zNPoz=x{23`WSlBmgOIWBN?VZCw>%s>bNo$avzPScUCfmQ{AGkC|GBfJA!SpO$&&Qq
zOJC37UIevFAsBs2DdLp5Q&D9nlzE*->YPl#pVuNRy=XJ6_6%$~boLWJxrgs}`PRaG
zXwE0SC4JpQffTSrP(l5_{SaR^+mRmrM{0tR%)wMs0!Y%oAi?KEL!c=nU^Cv-h5CGE
zJiOj_2e{%^yRvLW1T6qK(*eN3<4XX{+|9g}4Jo}|Y{&B+g+Oh)-{<n0>D>~NsaE&<
z?a_N7FR&SFnEdwr+H*mcWASNC=I(I^N-|&d^?dj#+gfg%S9Wn%ep{tm&VpltalR6v
zyl1~gB>U&DIKh!w%Xb67DcQ=BMaR>*oW=kGZlcJMcl#&#8OEKWs<D^!91Ji@gYqWv
zHltq*(tY{?#$2BUP=^Fu^8LA_GjRTxJ*yB7n6;CsGAm6`!A}^?@ieMiqFNzWm`!90
zoxikm`DQRllF7)J1kVGxNXFSEk{`S!*&q<ki9N#KsQsV(4WuG8Awm>t$Qrw*Qr3I<
zJIF=y2S{q}M#SMywiZt}V39nD!s-+Mrrjf>8wsrC_Ifz_3EP5)zkH8oHI52lJPLuc
zoX+A?h`%15l-V5GGAsc`vMq-Fb5c*eW$c7q(LvCL^Vu@N@}~yQ0I}Hpo=<C(3^Y<#
zGoIZFFZ*ET?mp5|Le<&eU1p`y$2eJ~bI~Cw%E!}m!jdQT<qYX_*kkg;p#CRWXk&Gm
zkl9{*$M^!C86M+Uqt-+4?Wb{nzP}RH|K1wzuX6$}KemU_+~sNDB9tm^*vTb1z^_br
z{D3l`u|k5*3nO8F()$m*z^|YyXl>jJ)x6qK*BneI+0!i%W?gjE9MJxt4eoNTgjIEW
zpizh#r2;t*td6!sGHX9B#y~>26Du@N$2lUB=_WnPcMC0}FTgG_tNrtkRK3SYiq@7P
zVgLBp8-*lbl68G&Tq12!zBKbDAWm@LYI^;|#ib!`bp0>mWz4|O#SyZSBj&qg$X2-7
zI6vzZxICQIfpVYxcx5Y0eI$uzXN{m@yY<rB*;Lxm*mNd1f5@MAWiD`D<J>pf8`rjQ
zFFdX{bxgZvkX4AQdwdv}38)~S43*ESGnP{E;V4p7c$NN-5d3esCv@}Wy$wcPey_oc
z=+i)KfBdWQg$A)4bG}}&?N6`SOY)n<lKW=JiSXi7p&bWb16*Pt!HAz#hKDMuc(b$x
zD)CQvZ4$(%ASrGmh?6tPt3uQ$rRtJ6<uB#_=e;3%J#LGQvovQIvFT!HvMOX)cY4oJ
z4wovm2H|%2Y`|Zk&gVLCL#xK@00igNXD(x7njRcZCnrm{XE|6(;}_FEa$@N~HrC>M
zQG~qzd-!F_Nt1!1M6Gv^`#MlKZ_~Cj)I!`Kx3LLo*-0_y)w5Gdyq=X@o;CXxc*uqk
zrhvJ(SEHRxU!`*bGj>7v+*O4`X9Y474OtV<HZq=Wim6PC3Y#t@ogs?GZ<lPjaln*O
z{Sx(ds<1=j;|B^W=q_$jOtz4qNqazK%24n^<=@#o_5Kj$8XP%A)&nPNaPTy+f-t?c
zgP1=iUFJK;L^z>}bbvoPVT}k2^jBuZW(`jKVG5e-G!Rc2`0G6}eV^m1H_H8&-6H1z
z)y+s`aoYypQ0U&493C~Y36m#RuiQk~LP;fC)B0%PGs;!q1dF#4Oa|;;xf=fFUh8;R
z1TMhe_X}h%vQvT&_xGs}D0T63D-5xbm&a<j<^}5xNMOFRF2C|km%18y*RO(h;b7a7
z&W1G!Pv|O0tiPG>MSm12!hXZ+Mq2RZ@r$>6u)H&yd_s-FCiZ#}aKE}Kh$11>$^7`v
z^gf_0AhOF0wweSTcw<04c2f85{RUP(Jc}S`0uITM878~1$paWFt#SqYg8~&k8KWUU
z9pW>COkyI*HHWxqTr<~XQJGEVVv)3V4Z6?`U?;C2lp`9umdn{R#zIsFk%pDIH|Z_4
zVKzocR>3YfuCNm>w`~el+A{}Og75&cI-_?iVP<78rZ)*!i}fI34Iji585H4~bU2!f
z-zx>Fu7(H;Fh81I?uDWNP+L0^exsd%PgXs%hgq@9Xx>uZH-lFctjl*-^-#~6uOFO*
z!sMg`I;9l$em*=FpLwPR_jaH?rN^{>DifvjG-o?*EgE~vUat9o)Z?hzmd7ycJ1MF&
z{dl>EC$XHjvLbuiLtcaZ8(}oBI&&nEtInIc49)1cQ7xNP?9jcjOcXnO8bwF_+MBgR
z=iwuMKLld^;=0bjDjK|$PWOH!RMbfD28RjgibXvdmdXoXknlxv@EL#TLUMo*-zSB1
zV55@6F4X8ak{;skdjd&!b4Du<A(J0nv^RP^^@1@hfdmB_);u1gjnR6Lgx$Xc-q^OA
z#2Qy``m%5w{V=jO)({-s7MY3utANP&WH?tBk^}ets$|g^l$8ZXVhZ>k`lg)z5ODq|
znu#jYhP=_r$-pn8R_;#^x%}qo!Q%Vv_3ZsA{RZ;c^%+;}2hqF*qEBlwxqREbHC-tl
za`i_sYeUDm6Zc<;u{xQze(4<9R31xbyt{O81M5!U`jT-)v9v%hxq3HFPaecC-gfTm
z1Ts3mQuT+!XR7Bc4QEi12yfRQB1Z{T;$#ZS7J`$8IZ&{i7w!>kwTO2%#P%U08k){t
zAki|zrJxEQlrKC?PTjUN_D(yHH@5`mA)6qZ&`9G4(Bv~<w=q(WQz{*${|jN+zlI;@
zlkMR-fF+<ci7+-mYOM|b3Xki*u%i*hOzdabBrT5B`;Q=cHVW0w78Pgp;8WRS_|-3$
zFt465K?L_BdCUPq<L>s{NHm=?8=4(EWx*ArQ4i9^xCn>1vJf3idCpqd#mN+Vg2iwT
zNi+4^pn}P5&2V0q{Un-K42J?UNNm+>t95&xEwl1l`9a(H(~oX6)!?3V#7mgh8rRMD
z>?|A;v=<vrG}bGcs=)QMT@xv{d4Zm}^9+~TE$vq6DAq%Ab)DB2?;BrG5^|p~ZIfMe
zBn}`n0f6m%z1XKa6Rw`#iEtu)61WwaXP^nlQMu9&%qWRHv2<@+h;L#yT)7Qpb)nuO
zmyqA<6kEbkP)o3_geJTY$GUt$5d~EQ`hw04hl|7gRcjz$a|RNOE*wUXnD+VRM%1OZ
z-zVj)-!AGY^>Q8tU*Nfc+>O^8fkuGUKSmrTu&@XxCOr^Q33nwSFa1iye=Zk~nBQm_
z3k|S+-L-STyYHX<NkQh*WzBVZEGz}@gBRFFmyf?m^p#Opgw`-qK7;*wB*p~iElUdr
zqn9wvgz-n|3}8H>#Vn8KQV@vYrf{opoHRPVyhz#Zq4#ibI;WNSYO&;(Y$3|c3l-{2
zbKW~zT}@on|M~t`YwP!$+j<V{tY!Y#?Xf}5Tm`SurOVPeJhIpSs#L?fSDAi0+fLVO
zIp^m=!9V32Aj|1Qk+&J+F4V$KO_dbfOd?^a3fGo;Dso<bX>aY9#7Du8=3B7u-wg)N
zgnvawdh#B=7bu&{A##2x$Wp-DxXI|#9LdD;Ug8jjg?bNF-j}Z8r#uv@A%GT89tKBY
zaY82i4Fzv;e2YizK<GqMgg!*TMInze#UxU&vF8yxNvbDdh8#yg%9Rkh$Utd~dLntg
zC!0T_eA(*t20P2)zGs3Ia5?pFTE0QCV!fx^W=py$r)P{LDR$|jd#Z`m;_kt1;6Ci<
zhg@fiW~=$|5~FE|B9mSLHeG&%^Zr<)0eRZA$bYMX9mM7;h+D?<=A9D5QqDF_$n@hk
zBn6}NKzew39?m<pe1J1O{wN@h%zjR0E79&MDC62UutrP;31wc}>t!g1Wg{_PZ-2X0
z*N^ueM(=zG^yP12tCcMJ3(tG3twci3f<@T2pjP_6m9C;C6<*sNa1g<6g}CE|U&H|B
zP$8nMtjDpelEoCD$gp}<f)=wh&;+i+t^Dqz)jH;Knho}u<yQ0)vCuT;doYW%)*DPg
zNtRvOc&*oOkQjSskOM@)J@}i0X&mxAf%holJhD64(=k+RdWo9`n-FOyEcTp-MHnbo
zgWudce_~f+1bR)n565@mt=IRRo9g+&I>36LWX%M4#M{=*sJIZxKlmeP#Gq_6DL~(f
z_35CR{oMa23t@=I%X{-C_bC5H{ub8v88xzTZ-_fINb@tXd@k{DAR&Buxy}P<pFg~G
zM9vH7f^x`c-Z_e)(w%-g!`|lv;}yuBg+hX~O-;s#J*3#-Uc*T8RbAE;p&u0^N4Zy2
z>yvbde6!O1V8@}jCIOud8zTb}?cABweR#taCDFL6vvD?zJcUtP*)<nndszQG2cL2S
z@~Y)o@+y@G@5<nlUvJITE;Nwxg7LgS>Cr*XFN0&qk5{IBuU6A;RRv+FHL<TzZd)(o
zKY_2oG<XO}(y0^O%TI`#TL#BD$nOzcV+_#VpOulg$*FOu5d2mH594}y$_Up{`RotZ
zQTs@YkAn2~BdME60!k?wypBJK_+2GT429T*ss&(~@%;>bx^*b@faLah^Ei@8r;Z3e
zo9-oZki}!pMM2)g+8!R>WFeoT5n;C#<m4E0Q?Y5l+^sn;uAS_axF&Wc=pB`Q6YQ^|
z@kUX@-Di;42Qr#ng!*p%dgN#HUcp@x(@MhxgJD>Ox8$^DuUGxEmXG9Pag(dp9c;R(
zpfBdIG2ym^e!U82>$FOuR!qj41KHxkl}cjnA-61n1RJ@h$9eNg{<D@wUdpnIePn~V
zUNKmu*}w>n<@i~u&$aAa9MwIA>d$fl>N2L)dE7Hy!AxXB8RRsxjf#`!PIBz7!^P~l
z=3tJbXh=U7N!8QpBMxzkmx-(S#I%1xX*-hLVmk?jBUHN`)O=3~t{qxM{j_t_U>^h+
z{ovBJTH@);R%2u?ZF(FYp84bPi>;kNkkoI4mt@v=^9Pka*h2tl9B$UVr%d?rB$Zox
zu7_R21PNe=?<Q!DtK-#!&?U{m6fxB!<ZJJDdJU0cm!ih-n&}49Y?1d~#8{4G4I!R3
z3R@T;>^XLV_5fH!w<u`nE{#4TS0~3(*eJ>7Mq=usJ~xwS4J@YaI8y@7=!40teIjyz
zy;0MYuU3Oh!i(y;`EQqZ>DTmrkku|)qg$U(KjKQ_Hc5O=iGS7i;kIvEdqG;F;jbsr
z{Tisy3C-EE*nM(m-zfkyP6I6^xP`gik_|{Eoo2v<fjg@?J|D5@oc}}Fx5qR6zwbL0
zMMfuQBbAgWgqTesNi1}HJ1j}avE(pjBNass5tW!6ql1)?!{$(km}Ac7Jm#<&GdurY
z`kdaM?_Zza<FQ9cvxmK2_w&B5`?{|C-h@K)8z)i;E&@fqw^_!naLbLi&SZR~x0S@7
z0N<SpwC64`j}X=lajOHWq`5b0tUvlraY9x`V|jLC?Jodp6=pB*3%II<zUe9dXxy{t
zkKr8uD03$#f%Bn~pRhX5C1Jk0-vGLoMiO@0vrLl6ac?eRBYvfdr2VNeO9ge{1#>Rs
zJpJ7uVm-zte*HoF`=vbkl|Ll+AOH>8T#hZv`~X{zT;iONYc>bQXckSH92P2z|FqX^
zDq82G4526!rUjsNz6-m^QI3BHx_|@<!&6^d9n4K|1z*9ytKXZOK)489(@l$5F<9!<
z;`SaP+)K62oVqV_jizLP*_eI<xG!29aCiIgSumY5X!bQxJknlF*^2wSzA@S*%kk0c
zkq<i85fa+Liq+#iwhA|mwZF?0xW;Sv)|~B*&?bb1h9-0;zB#SAFBZ2J^*anm#K!C2
z`gJ}|t>rp!b35*6-f-Ve{0kpuHKPX7p1H$Ioq5A8W6f9phL4_A<COuo!q}Ue>qj#v
z#-L;pg3kA^0^^<{4VWgkepar9<4^@ab;`lGOGdujOccNIx9tMv3w{wK@_V7<?m`JY
zCb3RAIGIib;20i=T%&*wyZP-nVg9EDfM7W8_=FAoAmFj}1uQ_vI_@Hlf5rWcrBHyW
z1Oe2|srXJHKafIsO$W=wQCpA^KVD?5ajd{uDa$bRF4*JnUIwApsx`aj2v3PrR*qrY
zgMPN580}yQNY}@I?E=U;aV?~m^CRFp_~e=e3}9UPKS#{G9FUZzcUsL7s-rrEyFyGb
z^j8V2ip-C6U5uB5I?_wjP$J%1rAE==snA};G31a2yRNXlyC5n##xpt{;VXy0(Vn{@
zcs|moa`>PybNabAaHzUOK`dnI!ah6P-GpvXB4sC(eSUDsuz$5sDI(9C>(EzD0Fs7N
zuX9geFDEnIZQuldYE3tNp%2ypHes(BcQ~&S^TFs@O88xrLtpL~J`iv){5rCki<Ooz
z9p2oEKV!>)lx4sVzsMQpeR&Ogf*~yb+imd=*F`A`Isjk@-W+LoU=#a?VDZ}e0hfvY
zV#CO<VH=Y7)^cUO^18k)u=}UhzA((1m-qw6jMgLg;i<fsLNvPr;n0&7Jg@HJVOw^G
zZ^}h8&KHkpR3E@29;G)kb(eD|`9b(DehQeb)_?r9IV^%Pk9}2M=FMyJ=m4K#dPX18
z)jm6%VV%B#*cx`EQ*7B&N;WdEGvEb?E>*-yo%srOUkFY1mMg1|Ll3nlZuP;ihdgc&
z*RL`1l}}35tQXqKmDycRJ5&7|gylBldd=dQ5W4ZLdrw(KF}-l)^w)^j_ziz!m#GjQ
z4#RqRg&4+LKb63;f)^7@lMh>Gs&^&2xaaXdaaTH2Rs<15EQP4wO~sbB@@CujWJtwD
zEMRV&{8R#TuAuOre1i8req6{^jv0Wv{DGmwgHQ0v5sx;BR7JOr;|R{l;YU0NNE1MT
zY;tISblwicVrC|cbX7q!d(#O16uVj7{%Y?0<C*s8I9~E$4yO5~q!RDcv?}8ki0c2s
zKTgQSVZbC15y;FNZ=8AQEa}zbl%bJ_u+P~5@BWblXYO0`Rs&K)UvOJT7iFZ0*|zX{
zI=_TwHL(kX^eyxm#WCTU>FJ7`8KsHOMM5kwLnImXx~;{ad}ZIY3)DNR4~H+dQV)3V
zcIb&ZMR{gj9nm#$`i+a3?C?+T2eX!3Et(z7&Hc6a$<&C~#J7RJh7+8WNw9=1S<bA?
zp1%M-t<)v6R}GX^n#p;~-@mcInG;t{uW`308ITQN<tcb+F8P!sNU!yZI(@b|n3jvC
zDidM8>t-@~fWyFp{ZES!RP&F0{v~7X-_kmT=~M}AzVN6ij3G&GPh6Td<|{%0HBn3d
zBIn!tj7d<<S>V#c*&wXL2Zmj1=T%CUbuF_rbJG#04c+262bk*lnmi|z=cZqKH^x@-
zZq8&$Pcvem<60B=eOr*CnVkGWA95#m&Ez)UY;PHw-<uoQ3(qwA5Tcpts}5^IOouQL
z8-Q40mk@S_Nq$`g9)~MmgLrd)VG)ydL?}PRWaAxtFs+#hehA|vaWx5F(-#p3kw>J)
zh4Zvu-Eq^a5tGOai!~ZTU0Vo4JmEGm6VIGA0ddIJ7A#9-PMuY4zVEVEEKG&|>~t==
z`P1)2i4Er#pwd~b8Ef4s)uwldbICWH(v<pvZ@`iLt6)0(uq1lxIi=nYB>1o)CN$~Y
z`#Zc`bSN;Tz@u6<&&%+?^kQBy3;Byg%!W6&xdab*l8n9i!+?13*c7n!BkH1q>UEm{
zXE=}&7(P?{b#P7ldEj3qqzM4E{?pcAiQx^AFkk#3aRfvhJ|4&7eE8=KsOYB6y~3We
zRCVB>3PBgm_x)uK*~a89*s)=#!l$q<B+wuz=VRCDEVBaoFKPS!Q0a$8EFez7L{h3r
zC*0*O-yxB+=3Y}CcYuycOhgZ<lAD+*F^kbYElff_6EWi;Pojk9Y12~k(ZlnWnX4h1
zy~7?Kz%D<L4$AZ%Mt8SiRGZ=1D3R~Z+L^~!?JcPhR(ZI99a6~hVR%+Dv4&;o_R`!3
zaZKZZ&&CgXW*+%OX0hbju>S`v8q#alVgbYCO1M4jwID;@=V~L}BQs@gjnBN{Y|NjU
z-BlP<N*y_2BE{7>f{^p(QWK`~;$1v@6E$xdY7HqFX!xq(ksEV?uhAP5(Jf}J*(IzG
z>dp%lev0BjEhc2{5z-lY;?xN&kNS`4BqU*|f<LakieKE@kLQ0QmGO_33WsizT)-9A
z_9$2fLnc4>KcjGTDF8kI<QUOAyY-qJ)R`RDX@kVq{~)>k{8=&y5B@$0ceDAl34;&1
z^d4+6Yf4h*^1S~%5g4#`UMUO?&f<wG^}Au#dLAWy#tg~wH(oMcgd{9hzn)OWHHj<t
z-a$Sjz43M@ho;sK{0!#>Iyv-35zoMAO)JkrOiD<wavLAfyDesAX1NfOr^)=rm|_WP
z{UM0^YhG_fo%{TF2c{S*i)Hm2O~f=Y9Stg+axhBM2jsi!F9__VxqCi{A54#)U<!>p
zGfDw9I$JwW!bNh4_U!gChu}6*(ANEwfqrzaDtMp2+_eN}ZbCDT3~Yc%+OpK02*mN4
zd%P5qtr8cfLnWG+IKB8rx<&~5-GiXsmqASgV%Z}E<WNu6`fGK_#5MwoGX>`NHbd7-
z8IYC7a8&o^m(8O{QYJj?mz#26^Tmpntr;4ypv<~g-H8~^lmZzBJkO`?x363Q#?x{2
zLfUwRcF^<zcEZf&!jXPfqxzgT;AucGq<n9T=mIlXl&;NT=#QIHHatRI-;OiNG$VF0
zy`j|!p_@}>gTP>mI<)@F84{rb+jL>*d!@&pVKY{22Qd>*2W8@oG1Vd0cWwT?pkRbS
zWUVsFv|vV=T=0wa<wO{*MGgFl1`eBh6Ensbj|SmZxfBdvfk=I1z!@vyvr72u!$jVC
zh%JTE*t0smhPoM6d9g9-F@?y@B2xIrgOwL1rd*@*{5U+UQPv7C3bjDt&x@-nD6eUV
zyU^i-W-JZkw68q7y*hG*Ziod-s0bFa0PQtfShvI9o%up2pK@o39@Elgehxhzxm{2%
z@^(NT?HUXx;i4h}qRa23LwX0XG|!vA9b!~Zmml#S#p*y>4?g#F5G{iA>aLgi9#;Mp
z)Jr&hH%oqK09)f8`ZLgLc5J@;S0Sp=0M#fCV}F6M-(_*8)iFPXL688_w1_^KGcIZX
z=22q|KrjyV)?LZrHXRW*%F}QwHznTsl-s8}8e4l+{`WOVlB(mF3x*ewDdJAICXpjk
z$`#+H5{&hG63_V*fL{%qhv}5Q#F-IC!YQ>G;mhAh$@9i*SMcj!OSpK<G6}s*#4Y*y
zhqLt-88)DFPfnja?hC1U1Gtz(VmP6WEs&Q)UvnhFvY#3aP3MA<<lmnQihYOQB9vK4
zq?P4{RFk9yuJEiWI)Z<_H_=a-^yvK0u%JWDyc;*c3lo-O`D$B3uWkEuAnM@r7IDqh
zNUTwvOeo2KR^KQLTY{`jtfck74KIcslVzQ11<wGKlp6eG+L8ge!iFnv7`A_7-gsxw
zTp`%E<xPSPT?%g)NQq#KGQ5cEzQpx*EN>pmgV~64=l#QaGV~!O818}rCA8VNJf6Et
zNLA{8eO!@?@_??s%m7*%!T^d~h$hdTBp^FxHQ>nB>cTiZK5a%D!z9!M<!s9Uod+({
zh67xRNJu2|9SzWY{6aOFJe~{$p|#!=gYV*W{2{hlF!lLyaz#hDup5kZn%ZdDDxADr
z#lMc3?h=)!4`cPgp_WK$qIm|liItHRa^E0A<(;6wxTmImV7*23okg16)|>T(i*03D
zI@-=I!-d7ZN}o9oWe~+Rd1OmbcQLu?nN`|$_u+!YDCwuHuHy+ISTsuw#MLxs4ZxMi
zF)<{op4BpcW|}{<7`EQb8?Y{n(;+Z3iOkG)f8MOWAI9s}?YRFv%7AAEs|55rhxp>w
ztnv2%(wNW8@PJIo`SU>h2^gp3PpJE=U*^@MV1ECkEPYEkHU=}@N8|^>BK2tybP0vQ
zcWOGd75LSq+}V!nAHTDB!b{%Vb<1*=RguArOFiJz8UUvKTyGDpTWU=OE|Rmsdg~Rw
ze&4?HjW*i!%%g6duQ^tK?|qHPSsY!w)I}LsM^fVye@=~_pWF>{%WJpR@aEFsnlT$n
z8C*FvqZ0+rtmHQ}xoeFvkf1}}irPr>L^x*+;>9W|V&f>lBOJ<1fiVx1Alto_b-9`4
z)=biig3>epd)%l7Xh@I4p|#^%v<FaKG7(+sW9%^ouLsO71+#xzDkC2ta*+XHm1n0I
z^mrm=y=cnsK5=0+0>%J&rURnVDym-_93oL;`DuYC7*VS$P{IJGdKIGqqj{|m1$@Z=
z0~nVw7ZZb{8vEJLB<BeKMZ{-!=+g``MP>>>H)RF1FC?9bbQQcOP>=-yZCU~C9(jg*
z%oR+kNYu8?&Bxw7+ujIA_rms6<Y^8a-+l8`E}C7G2rbuDrY9QxG^0LcRXpz}PYk+b
zz2xZ+7vk6QwYl~4gcG%M&r4A5m;$TRbDIl;{>R6-k<>WBIHJ8D)jqpiP2u6uoMsHi
zvXC-GG<=NYuTihu+`x^oMMw60`o-X<sxF*mE+crQ5@27c5aCpdb8<Q02o)MNkZ-Cy
zY*mX90EGU-%H1W3WJ_+VeL43GxA&wd<<!+%zgi-Hvy>{9bo*^M{Z^yjRUzk2xb3Wz
zcA^O!3Z-s5vnec&Hez~3pFLqa3_vSjhfVo!cxJ?SlxD`^w$iIGGCq)8D9N1j7}ICz
z5RPBUVjb6G+YBy^Y|OlbbECi;QD8J1s606cRz6|~wU+B-`2TlQ{6wPW*&FTs=%s$Y
zEc$y7?j*nnViqh55IIc$lo}A6u<@fNM1;d5vPIp<&H{Ko11iv{KoJDyZnva;g9G6p
zytEFpg`ofrxj1IGeHuMg)n0qUa6Je-V+R|<Wcz^0+B;50wdtss-Z&p=B4kk;$vRz7
zd=wkuS@=|EKiD1EL3L()N%LtFnp_CPhVQ!F^2M*uA1j7-5^|lGy|5EX@iy1iRy{I$
zwE;1%CAqUX$CTvUv-)eEpwE2>LX8U(c}MWHBN<+$wwLTS7hL_XkGg9r0nJJwJ~-Ap
zd{`i*mI3UMKi;)}wY{e?eHDd)w}$Z0e3*L$V>|Hv!^OwsvJ>^j3{?DILTN~Bq|vg1
zY3n`$SD0c5tSzHJxn)+ZZvYJ%1@N&8sOCHSu5f=lh?Yboz4ICJJA8bg?d*NqR#~K7
z<iR8-*_AEg8yCMGAl=+frKPZLcvd`x&{W8ksX5Sl-3KWzx)VK6Z-6n7vq*rIG?uXs
zors>cJjf|4!rm#u8nI{>_-iuwjUNd9q6dtPVgEpIel$Zk)JBI(yZ_fpvY9&q(!@rB
z`kgamYUeDlp$m>gRyT%Q!l(03{zMbjAbKWayT;k577C*lV``l4^xl9zDZT_ODXsrp
z#2JHK04<dv`2t9(pee2>c`jGcft(~*lu;ArtHA8Cko#*P#Hp;_paZMlmJ12>KYa3M
zhEc55$;hXAI4N3uPn$YU;8X#2{8^01%P@Ay>KbWcMkRcI=$n-TiL4@kco5Fb&BEpK
zib5|?uD;EMan|$wWHh|7Xfk#q9?0k7F0^OqoK-CQ8#cTn8U(Ghwhj9K$cIuw9+7pU
zA;endBma#@2K+CWbpnp=ObP3!!1sY)D5K{a*rH>`vJN;t>7xK-WKXt)oyVB5$lBCO
zM<kMR;KIR@^AzwRAffvqC!%A}WG5m<);*rpXVt2v8mI{P`P8rg4<DlGMig(?jcPj`
zZ!s=%g?bj*lzuB?s!&3(jML$t`cy)~_F7(zSm^nm!+waTH*OE@bPqjqn(~$y>|SSW
z%=*xC?9h>vnxzD{bTkW!4$Jye7m4gw{WY`IW3;piA1Lq6)m&h^$^ZUXH61Taj99$I
z5<i}y&70Kb{lv10u&l>c&Bg^Wu!-pi0emGG&-XYqa)ippAkWq+sDMzvlPwd><#LoF
zk~l5*uMFm*Y1FRf>KzX2;~#~Pu{i2aGZ${_i{kR2j*t&noib~D4edpTiC;I~mU^%V
zmiJqR@B;NS^1HYowRBVlC+<m}v#ipO1qO3zS052%;2B0mq;&TUq#9Xe<{9h#LKh*_
zlACIIH@>$B3(;w20y8rpw=h}{mbXTDiB=}>!SfdIys{YO$t!Pi{`a>IJag52d<Ez+
zX18dE)`w%})6w$^g@`xIQLFx#f*AG|6qb@Huq+Tjs%MKjeU_uBhYJ2+i&B#fEF^)q
z?`uPiN96+IBn5Vec#)eheCHZap4?=EbhTQiIL&NOyf={xVvSyuR}az^6*?;wDUQo#
zeHOWtU7_@GC8bdLJ{F>p4i_M%XaBl0CSU)(e8~*9VH1IDby_F{->d}+M}irbYP^-6
z(k&W6J<`0QMl3x)=!@Ba&MV+o6!;_}Z;8lz142`yGvFwpJ^z2{4!Mu|q8V~)by^*U
zT;-5pd<^o(5tkLXx-;daGOnK#7jBQIkl6ecV+?J|LIFQ#FvAl-vOiEjD*fde;juMw
zrpBl_dA38aCi!J?c^;bmHo7+)w7`~GKP5mr!!0W^dzsn?zm4fBBF`oX<4+EGe7Ntv
z-LzX?rS1ONK{Rf^{Q~MGb>&d_q9~=s60;5JmHOB<UT3}pJ#@x}Y&lj`d9<zZ1;DNn
z&r)-QtGvUYOBuNN3?A>D3%$5xqs0>X`NJaizwWn`AW!srEn}=2*`1PArivkDhWHuw
zTKu0(kiJWhmolxE2>CqG-vkaFk7^f%Qlor%cN5C(fW2h@E*L<&3Lii*(=kl`DlHw)
zsaDq;(=h?r5<<as;i6*-QhKdU0V_Pg5WzWvG1xdX>a)+4gpeJF+RVuLa^z;?ovD^N
zHNSz(0DDh&D8PPhS4-y@N+JQiH1Q@f^<+6BG^?KIBj0SA6_b9mR$GkP)OIMXkZfk^
z;E&kRl3wi7ma9B{m13j4kn~t{N4n1UP?aH-=|ooh5gFysqsCCO>Ox;4ggkDR#wuLA
z++RyT&FW$quSpORiMWywGD7c4>^}NuBi_F+Qu}OH`0^lte+dITN(}Y?WQxkK(&PQj
zg|Ka(n4K{jaRPr2s5<x4;|a#g1E`_Gh@r-k@X-a_@{&J>%{P9!6g4kLk)m4QLS#31
z*zoOTY%ZQ5Sc5UGHqVG?cPQ^L2Bo(5^jB~14d+K1QNW7%SpCP)XRO+OQxl`3iK}_-
zcM_)TVRsp${78O5Qudg{jDXr9fujRBJ$q%VfDbZPXF4V8aJyUAD?nT4+DZZE+zwDg
zO9^6k4FJR)Kmf7l&<7CMS7Po0V-vf{iis@gvnF_GAZ3lw%WDq7t!Bb_V+>q6>+gGd
z$DgQC2b!V5s~T9EMn<8@l9%XNs`W!kIFVgG1Hx=HiqhTx4}#QE7lNZEcNuWtx8{oA
z5#)LmrsBp?e`MS26qw{T^<S_4StWoN!H}+q;FoZRO1P7H2YK;5jXjMaoDn$HPvDed
zA_G*gxL(a)`4|bn>Q$Z~vs77b?ae3~R36T+3k&UogUNmVSZ9{ka3PrY6@Am*k_>t7
zuduY!w>**iK814HA(eQ3C!|SWBT3tbyHD;&kW>vWptVEz`j*{;ZneJTi#VNcWkM!8
zCeyx!9k{vugqi0nX?w$Zy<24&^Olcj{jB6_IIM<jxvq=YXpliO6o?y5lo^bQ`QH!B
z6$L@)bu6#2v0ik^>D7D=y|4W7E%4ufML@Vp-4>*Fcfct$>wKu0{NIm|lcdPT2`T_f
z1~Gg9n$N(`tA&x)sI^cK15{}|&K00+r*5_tZ}EX)oHtTKwH~Gl$c6WXixJx>unU(#
zOFYkJ7B4zPvUd|xkxwYmjrHjc>&nfkqqA0W>5wutYvfwgqCv-=EOB-9(1|DN5AWys
z)ytkzIQ2$ZVBSDpn5%ijth2D-C@ew+A%I#yuRAXkD%->6-});i>Wqg-P!m}bD_vK|
z!E{6aChywx+esECeC{})2)ZNIKhGB$<LB!|nYy?C>nvQdh-}kk5Hb8mSaG_H2kYz0
zcei)68~?527!Nj&MC4EG+)>K6gt5BX!JK;Vy~ne`sIGR6e?4tH%{)?^0LQR;_%r^T
zCVx(gUQ~q*2yfcRV2#L(vw^(J;G-#U@{Am1Ydk}GZfaC`<|)cy^$lAzRW)=jhEs~C
z$BQG&-T__<_O-NrUaJF)tJ(@#k0#(c5ORN19q>#`$Qupe%4^fpO3~U`E1sJ7zim}k
zCVP?&)Pd}pEq1lYm~MZnU1xMJF=9!rq~dcgsAS5b1eGh{=W(i42c#WDEykp(14=OP
znKi#za>BlZh_Tfyh8qY%t-AtU>ojvX{`VYH`9zHBkpW%!0`+E}GD`OtWBsmhv5<Wg
z_@BJGxpj7%NCAHkCB@!kbe0GwogY_>?f-Ynre!R|7z?M|s-kif@N`E!U8bNEhYW*4
zOi%=hk|3U6*g@eJF4l8}wYmZd1)!Aub)?KKlwF5<s<%_Sw*pgTC~t16Z!{4)hPkc?
z;8BWZJh$Z?riPzglN@qmwAHLI1mIz5DTR{P@_N;#8mx|KIjv-O{xsg}5GB38s`Frg
zz!_i_rTx&H$8f62Ty+Qg^u*Vv7fr}l(jI*Q;0(D?bMTDZHL9t1?YdneP^g`{CkSUv
z7~*)UIG?%pg@bM88^-=Udwe`71TZs!`W#S<7Qv~VL45!3cMCl2qq2}o&IG(1H*e!#
z6(*oJ_3wl&Re4hkGH(SRg3M^ZnS5Rkh6UGt`DmS3(2U4lfkw(W04cXxK+4zU^#o6V
z5UL$n!vMd>7E5k;ru-U?*{!2QPt>VRALnji*X3#s$r`vbeKLUmZK?^gc~0WSB&S!|
zX+?Y&!-=bjhP!TO91@Z`+u0%x%#++0Nf=E(A+#O;IaRoBBtd>gH5xr~@BVlcHjMof
z|GPp9?p&_-JEa0bu2}vnU6?y}T3c@nH}r#m4PeF33$+t)Bohj=3;>q}1BBr0|01>m
zR;4OQS>ffFs>jkb0FO7k=8@qyR(<yW1j%k8&dEada<%`v1lY{l8(^9==VGDyzm5;k
z-Cq|K3L!=q@PDp6EkV=Y;gPq|Q-A2zvdIwci10WRNUb$Q)5C<+(em&su0CJDHN9%#
zHT+|(s~@8n;Df(ENQS1GDY+|C4qYvjY)yEIo_!vjWv2}ER+z=9*S|Syc3Q}0v(~^j
zdXm%o_T=`b*X3!5@1{?0iPon@N^~8p>|H$+ryqY)@4?FCcAZ0N4#+hy-62s?`RpK#
z(o-VwwD)*UA|n0`_VYx3=0$E^61Nu)9y;51inr3!xaAchlu=FOT7mhMiw?ejZ`K0b
zr2tNEBfdQa%b9Km$Qj%YK3nha=L{&_mGWZLBdI_gaxnPA&Ti6(B~5u5?*BPzPWB>|
z;~XpOnib`xR%cD9{NzF*seXRNPcURF=QPn+XiPYJEL>njoXV?9HK}wn#VkwLS)*%G
zOkeUKN!*D8{`k`KCM;F914(!6nk)s;DG7^6w!DpDnNs6d?O=2%-|4<Y#f39jMPY50
zVXpgQW%4g%TSlmf32kYiZhy~76YQypG?rm^$LMbxx8%~;r&F8VWiSUH8#Tu$B7vb+
zxtXGvd+!Uew5BkoY};F4LdC$n;`8^p<s!~J0mB2#n44`_O0}sl<Zmh1>8OY@f1}wS
z$_Rvo7<=*hz5agnHk;Y6Qk8rjyJ8n_l*k=zHv^g6`+AV*^PfXEzEibs%d0{dcOH!D
z0}v(bd}A1QjVi9^qQAnczxb8Q!_?@GbGI&oUXT(Z&6RxDR+NQ&siD|N9g#$acxv+&
zSztqq_Dy6>$)~jW7riisQxn<gJ(!+CN_e2tX_<KwnT{UHPR4vZ*^IdqmT*}4fPAKq
zt8zi9WKx8n(CO-+=`BdWB4;FVeG#rKKeH-yMy*uIg}$NF%$qZZLaJFk9b05s@7dm-
zYfAK#m|pKE@$`oZ?)HgU1ZIvlD+hsYHXd8qfTJ2*ZNL8Q(mbepZaNO|gDDlT;jt2e
z)8qb#B6a2;s1NW18K7<DAj~R~<?tnA6BrGXHnod<WlXgAf4WA;J%TqrZK^fHFw(ij
zJ%2CgBZ?xiKeL#>JfO4;JTn8K`*b|0N0+xv9Y<nRqnCG|on|hBX}l-|CtSUr4Wizy
zhzwR9O5XPKo#uj14CVtCy6IQ7FJR4=@QyaFfr6+SalyccjJ?Gk@AOkz>M(O*+uDe7
zU-rbx`I62|33S-}JW+rD$8@K$?M2h;&e6-V5vpZwklxkL$}g44Dr6BkbkJKzle}`9
zL1F4oheQc6sPb84Y9HZ18sLx%zWFx#;<Xl4U}1dhXC(Om`@$Jz8nTWGkj&ozmIT2$
z;vYUdT;TKf;0frHnj;uEe>FfeXTvue2wn@Asg_Z~X)T#nzs-<LLY!q8@MnoWV}RbX
zI8&thw|q3dCPsy2QmQF9Yre<p%In-3l8OIn46hSBQ~)F@#_U84cnq2dp|#8s`D_gS
zsZ2BfP%Dz;h?{ZL2S|Gd{gBb1{zn6=cM`)#W`h8fJ&Wh0^FK2Qd?w6)j5BM1RW1wr
zM}zWOLkL5LbBo%s{8bF^ynB}U0Bl2)e|@zL&uLS4HIY$XFUPMK@S0|IVXL|X3m)_h
zXRry(dIKIpZ8U<%n(uz3bJU9|yk*K5s_#lf-#79#aSXLy)Id9YuuY4O{V?6%fCX4U
z-vV(xK39(idyxgQO(d?Q&`d`JRVcNBts}6=8osHLIoU+Wp0YjZbXoYf2jN?1y>ww?
z*4bH5V6DmwpkndfB9zuMs{H6Gjr!!KJM{}AzG^{GHSJY~)}xyB6ZyQ6*!mHPodNZ?
z6d~us&U+PVzP)dFNojIwhW)HXSd+U5kZz`JfLQ(<Ms`=E9!iG<CHS{0$3I3>U8Mk{
z19{S-QP@shyU3mXsu+vTMY95;W_j~6p6I2`1|{|Bf7M}mAuI!lt>CJ>6z?-w=tT7$
z^T&w`ahRVr<%Kb7l=OEQ75|6@M*~U(CEQ>W<gCXfY<343aAqoekUFvbJ9h8JrjD>w
z(&KZ4_4jf#{z!@{w4-!@!s(!lB)wmq`}IQxcafWdp~02EKVYxb0AOd7(}>vs8Tg~7
z6bv@ipmHYA9SpU*fft$=s>5pm05HT^!gDJOxK_iX8y6Kefi~%BT^xcl*&c!>g(xpf
z5!@7=syL6a8;>*YK7K?vjb5|F+*>_~T|G%U{?*O>Zt_|Q_bWbl0)y*n(_QQ`h~C(<
zyctHc+09<1{Q~BF$AnNj#^5-=el4KwCwqIrh@48@#G3@mNnt%^O|xlc$X1_pOpWJ#
z$nvgu?s#qoI`=)0>pQie<Qmek@<zafKKd4nUNc*`W9UK>2UY<puO`Ug0uStV*}bj7
za7FsiaL7~3>k|{wp~&Z63iN4J)~iBIiqS($6I$}=gk8!kue`ET>j(NSv{<`k@se0x
zjmLsl;!)H_HyAXZ_iOXseMMvo7W}QF0w5A_-S`trqPrumqUdAX_N*0vuSxgYNQwu5
z2mND+FTa`U?XvnR6FfqNJ;E=x;e1y2AXfGa!?dYUAk_8ML|zw+ytBO>{dgYquVh#0
z`=?GApdPnqe|9=JNwDTV$J4Ip$5ZmYLn@OZdP@{Oy5B*@#u;!XUH_zsbU{I0%}ND^
zUEz;8vG8_f9FJxdV!|5pBO|_Be1|beyph?Ta5T*}4ZtgA85m9?e`Qt;JfZ!=Nz;b^
z6SKORJZiM^F(e8nb7?HdIkV7~^K6iTrZD{6Q3M8x@Q^>b1maUc{=7LbH#dXGt|bm{
zwg*jg+o>wn3bU1j;gS4Zn5|d*cw<C%c>jT^Fyn|R1N77^fj0=N=FU`KshQl<f8^FM
z1rTz-RNy_&Pk5kbJ%Z5%dWXl;=Xu|;8@m+HS0{~eQ^rjN)+t~B8wAMrQs5x*l<+Oj
zLZr{Oua3eJLnOxOe)lzer)%NI#*R{jp46n-6s7-0y5|ttGPDaG?7#5zmgm@BXkJ2c
z7!ue~a*f0lrMG2k4p~b0!YX$Ekg=GXOur}~OS^JO0F<*hJ+VbX@W9thL`W22T_ZeT
zR~%%?4Yr{tHH50Cb!~-V60Epuc2Hj_7BkR>`c(UM>0wN(ayR@MH?KqR<y_2MTVVfM
zHW=9fd{;J=H4)9vn*W9u*W0!IZ}rc~P6S64#|Id%YIQo<30GS6@TP#9;E!8g!1=x7
zUMdOeo0UON$mnwxd3V02s}NZqh*cADsajo%F_)MA)A<J66<Nmfc}G|M`Ro4Jnx2z^
zw1*TpfZu#KEE<xbyp=?M{PY^^;n<yKr3`>RV-6#JD>9on6h4pQukGv5ti4RdbD5L(
zY&bjH@$3-3pR@DH&03+gJE{;$FB(QPv3h~m+ciIPr2<VA#RQNJ$fKqa+;^Cu)dL9H
z0n1k4H)DO+A!B5FS085IsiE+#r5?MpN_i4CgIaV$9}N%NHqO!J{lE_dqvov<VH5a<
zCT-#;{#d-VCWo1~$>V2S!L#}@@VtILYreiZN|{qUg5?1{#CfgK8tOys5zY_%hD=57
z1YDoy-DD|^fpSFA<D%vGNR9{Z7xv^6dcoHnGM+rPmlPgmg1SCCDuU;e!qLmylUQ6B
zV{482_^3#mLr;C}7WQ3AU9fv7KQdK6^jrV<X#7|7#vGe^0?G<4aRu7b9?|>o(Y2Js
zb!XO0kjt*&v^h%*n3dDtyIw>dPY^*4%}0*}s;fsVKl!L}0le$MA;;zi{n?#cwX?<6
z<W1Sb{ZNN=U*Lo09sxSEV<GRQ+pJxhc8FpeYtg?L#oE^+a}E)jYGnjfqx6;?7*Zwq
zC|`5V7JT~m4024!Is03KHvgbaCf|a=CGxq$e2eB7bNr^_Q=Ckl0Q1EY`6t$vL-66N
zZm@MX1)TZg|0*8fC^iL;8D{D@XEOnt2dh0KqCZ6GJlTYTB&=T%`xaTtd;S1$EQGMF
zPjDJ$&whW*x!=x8587CjCPXX;8fYVbT9@PRd1CwVJh~|U%ynn4O|0&;UkHWdh47b&
zqo`Tnpgw|GGdj&cN<~?1k*^cKnXjhvx)EW$TZDKQ+r?d|_m{Bm1{p_AaL7l}gZ>0}
z2AG))K50+9W)K_&RBMA94%(;*--xK$cL#l33{1dPkEDr9Y$_Me5UO!%bQYkE9LV0G
z9NUY|OOml2pIS)}Lbl+hpSu%}(|LkHEmWbWkhQ`w9SxAqy!E+j%FIrR?09KJtFk;G
z#>99iz5`sE<QTq_)eNb4`c=RwQ*~r~WKG>i!Mv<sAPt-vtT7#w89VN8Os`%OuhM>a
zv3q0s4j^gm%2Cd2iTPF1{6UfoOx<wib0r-@J3&OPlf5hIuQz~xXr-KYG2uU#{JORf
zhb3IW!qC()cUNZ(@JMaqC9iSeCk#Ar^RS}$OQex4uN>*^qg7M91j<H2yGO)t4`1E_
zhFc=a?^!7OGT~8(2Q+j4Mxr?X3ZIMsibSuM`}e=|SD#b@_*{m+c>0Xc4jCujPZ(4E
zk40ek&I+uRxvIw5J^^LXwE495Hkb9!X6;Vp6Eb@e_GyJ+=vOdwL2=I7AC-0d)*r^t
zm;ZWa74vzLLCwStV6B`uZd|-v#_xIqb|ubHt(G&RO&kd%m}Ky|d=HAv`1JeYHNJw_
zpw-o#!Z#Q8xo$hA@+7iB;Gkw}j>`R<L&pSr@wqCM@e0Rc?@JoB7#=cw^7d`Esi|r3
zo<ZjeXjzFUpZyP{j%~ZvvXx1sd_W&AB%<F{6<3A9NIm|eOCy2H5n~m_@5b0QB=myg
z1dhG@#U`H<C8{jTn}YFjt#j@gMm25I%NH+A_?4<r`^t26UF_#bsA;yVGngkJ1QW20
zB9F_dq#l@5?z9{cq|F7q&4esXDqc^IHRx7-zJn>lSJgjYJa8JGgX@jEV3x6%R0zyf
zBr0<kuX+}~N2f}+&iw^D+~c<}6CF0=9#bW`YirlFoNZHtXV<0?y9zb4-|qU<EABaD
zajfg%nd*B%_swr7`$iq+&sp_e!JTj+cNJ=DxDTCGt!%+JJ+N7OTd0)m%Pbsoxi{B&
z2EkN^XU=+Y-7wc7!3iR-s;d>EZqYhVO&}ia*j_EEyt@e`FLg%pP(#JdQSXUwUl$sG
z^?5h!I@153@Ny6j9WVXm)9^-=Ye_@)G2~+8G1uO`uDzFECp>li`FV%y*s~oIMOzjc
zh_mmv>CRrcwbkvN+ReEKQLb%P!i}AV9Z|XZAmQ?ZRll1)`OQE2q@Up5#|d)%q3hjP
zjC9t;CA|1Dd{0DC(6UZ?E>uJ;BVy%;OiZa;b3xK4pgi!i6y(dNEW1r0i=W%HG&o1$
zLU)93%y;LHx?Mk!(6CMKgwQN`>9oM)uQIP5`?GV_THp59u*mO%hU4&V4Ug5cUgUHJ
zs0B=&;a)5r&I?)mAr>!x;_z$a?8|Su4<66~MN6%s{W5N1hrgJ0nx+@y>sCRDYo#|H
zo}e(Hu>$Zt*n8bpUbC9LV}R=}Lh9idY^KPeaO+-r@@<&EJ`uHA=H70tU3*bF+c{+P
zgd&Es^5I@=<((gUuI$eVd3UBHK%`aW!lB*v)*-5$RBF?vNw3=z$v1slwEtJs(y{j=
z!Nb1uf>Ms@?O#mhHczTOoq%5dQa3RvW+;?kdPQYi<Q3VDarAX@^Wk^%j)ot^O|FX<
z7Om8@C&cXQoYQ;aR>y*g&rM2QD6ruO*;fk1(o$7hIze_|AC=zRIHheypUsqfM;@4g
zB4}l8dy6(eWOaqq6KOAw-Z^;BfiUz&sf*L%?KJn!g{RSD+gdrTd&?Sq6I&&@XGfV?
zrk+B&ws_8#AG7oOBBJtB+=p9)(^&Rt5q*Ms5mwS)hvV++lZzabt2bslf1G~uHVYCa
z7IgW&$pXUYH>~P1SZVOUkCOF6ji4YOTGf<G_)-vY@<*Io`GR}m*;g+=)6*uhKRkL=
zs%H7{RP@$IygMbsotQY0jwZT7!y~$P<IoB3#z5_(+Zfu8dzKxm_Ce%+>Sd~i>+jg^
zz);>fhHLIT=J)H<F~eu!<+8HP<w*Xdg*4+YC-63Ci4N}Jkw$;4)1KuBR8Z&*KT^E^
zRoCkSx3F(q7YrmN8$$++BxLmt`5e#23_vEEf2^<`(OtP!J6uO!otSt|N`I7(aaQ6n
z3;SGW6xF-$HPNS_{|L~ZVxQEH#f>I;Na5eeHg4?M?sfcux?2AS$Mmy$4|Y?$CPiLl
z1viDQhb9`4-v-nPiauyo4%a-7h1NQ!xZX*Qc!zwpJmg)n1H)$%husOr3_p>pH?<gR
zC*uMui0JSm0s{(Mpc4z1HSlw)>e#4>dAnuBz5f1|2|!Oe+kf|6!prUZ4rGRri(I?&
ztmZXXN6W_xku`2#(GmNmA>6eduT!6#JcIqV@4Bnx`W^!~T<mcl(&MzM4oN6O44D&N
z_UX$p0{#7#P+GFyF=(Tau@jnMmKTx`%K+!W^F~NJe1V2M+;H}VTX)F$@|+1>kH$wk
zCVt2uBrwc2#NFKT(%SZ)>e%y)%MBhbC#LHzE7j%@_SI1o@~WjgH;57Q`RQpLqM$&?
z#&~r8U6)$`DPSwT*Jl_w9ecF!oV$+CGx$sbM%nysEkJ5ExTia(y(2wOh*TRFyQe2)
zn_Z;rs@{1q@zUj<ZF(m6O#`n&{0ZOPE<vsW_Gs*kOPjhBU%)<o&)T?9(m73nph9Yw
zU9Eke#@Db`w_OTJuA2P&VOGEKO*tp8Ti0x7y;Z87o`Gp&zpC=K1n;M5*W5%Dtr_iq
z-XnS3FaEDjddKafEURgG0v3lS_QyWB;cEqQRD6BtL5;1Vkx^7$nbq{JjP5tKG7@2*
ze>m&-<ivTtCdVkdnYRRX9&~&qOyB8qp`6zjb1~3DbhtYoeHGFaA|uq^m3!uaTZNMH
zx-v{(LS7%@Gx8qcMZNR-=<1y{1;nuI?~kYh5Gdx<3tsn;iQnmIxxZcSF7g~7O--)M
zVg~gk<o15E%n&sBe0<FO>6EaYngW6->*D;)h{JrG-taW>ql5vhyA&>yV08VeY*1aA
z>snTBWA>ZIeO|X#>ttoGthkQYMCUEj`&|;J9QW*nt(?9}8*dH0x40ppmlgESJnJfE
zn#COI$EB^uc+A;m4c-$Hf0zZ&xg*qGusZ)O!EziZygP>ds(m*AI<^0nEcy1C!0BYm
zt*34H>lOGN06!CQ!r7@%PViH-?ER6@si%)yx<-0^cjh$BJ^lpTSgrJF`=#qgqt5J?
zrE`Iw7=02M^!u8}meY?b4mgEhaY4FYJFzeRHHOD@9XA!KOOwet{w7@}i~A121-yar
zW6$>zzx=)?c2iIp2^<ux@A_Q>)4FYXqvQMUHAoe!b}FK%^%J>ZuIIc-hJhv&3<ZKJ
zrhl^0aVNCU)_6&8PtWg1Qs<@BJ5#T(oPRDx9o{;xB_bdDJ}LhI;Djg~-v4k3i1Ue>
z)eeXB)YBt4YJ<3A@5~;6<fA+&C@AO^xkd+Agd16}`cDc0!Pz+>W*Aiv7Mp46U~>7I
zje6&rm{5Ca=-6yi^r0OATduyiye9jbeRAWD3eW6UTBtxT@Ud!sg21BQh60`PPgW*X
zd7rJ$Rr?$&f#!*YeYSZ5BcGj*bNP*5%Kg@|)i>6srC_lIpDA9Inr|&<vH;ubpro;+
zsH7Fy*>xkrvs)<Is=MQjmH64FeB4~1s>*a9T%bF~NZ_EhLUH0HU|o!*OzzrD%;%<C
z%@1*Oa9qB)7<X~Qy^&?9zJNXM;KB9wp@B1cUdS4pHrCNo-ZklPj1ffNYfDk6y|ght
z_n|St#<=Ne{yswNwIAtdji+#CGW<lZr5A47+0Vz%pkA7U1Xl*HbO(%1y8l?;ibszp
zNl&C|s5Rtej!CiTiPRr=4s=VPTRgZw?&K>xzaykzP_sAd<yS+eKq8I$g%~99)1%rf
zCT0nwBz=aWchF)n%5azdUTlKTalNhEllDucPCoyl3O$Yw{oU|sE^B7uVR$P=)oF32
z<WE{K6pGjpC9QU5Kke6pZM(XAGa)^!?T2>%_003?`3UAry5o`H9(nc7dp)~)#A7V-
zUI1ub-0PO+jrWi3q=%2N7r(&wU4^WC-xU=x>RfCYX_1qxW?3MK;IsN0rxIZ<5hUC0
zP2;q+xQCBh!|y2ilLMVn$x#ny?lz=dH(vZ?RwUQZUVQ0Y_vJ%L=VBz}x^E<hFOPgv
z)ou_M9|i;mb;r4#LAM4!Pf{+IpIIbO+HTbo3~(z?FOvXla{4VJ-!^+bf~4nv##h?&
zv)#>DBeOiu*dQsTqk@T)7pT27WU@LRV94YKZtb23>`?2Rels03BQ+kfquW_bF9wVb
zWx<M9vI09!M!ofYK5YhOT?i$z$NEpZGifsu(O}N?xT+c<N7y!D&N$(RbbyFr)W^+}
z(wDmo0Eg1n+eT3ZmQtRQPf5PEkXTbSs4zt2!}^mH0qHGK+S%ak-Gy03&%5lhE;f)$
z$)#<llww$M)l%=CU4UnDQV6UH?n--<?^Cr~DQ`8w@^LKhk9lJB;vUGSD9qR7Vm!Hx
znOOXEk77t(3of$9e8tK|RAt_L+r@iQbi2Ra`+m^A-iN5m#gec_%w_}D2sa0&Zg_+S
zf6{5m>YZ-@wi01l%e7glC#SK849MUw2e>|2rJyIxw#lY}SFY^zOi(_#@Ab)B8K2x-
z?ag!Lgl?%lHrS$9ua|lC!pVo7XSAPPyhT7BbfR-$jUTM#t?iK^@kXos+Q5ZlUj0i`
zm!w?PimesFjKe#lV)P_cX_ua8xE0>dV{TXdtK<F4xP}nSm$Hdu&oi~YZO7J~9Upd8
z$H!6Wpb~Z3`FmrKghiV@CQ(-vuPeH}{=QJ+FQ)wqrKMTve(fqP$uh@kU?UEwS(_M5
z<sKy2<JGHIJLPS&G%$9%w`ZKy(@Yoxa-_*4^C^*6G*X|Ve4M}oNY=hvr#{&Yr%H`Z
zt@Xe`aeI$$*94{=5+iqcYPD=RQk~R3sq1)VVCnh5SX5y*oi7booa#Y;S)8f@@NCcC
zb7fytcz8Y=KLIe~1y`Q|IP?*~tYO5T$O_td43&B12kY2odkI>W90rormCTOMDMjYl
za=R<WX1|<f+*X;Ec5fPMQ`Ko#jZw7!jL_Kme3$3BW5@!)c3=~I4=7wbHU*3W?7`1(
z2?=I+>dZV~bYb`mOySX8UyMAk5y7FeFFii@9c_g%f5L^PFHZxKQ{WSc{?qm1|ES+P
z|I}~K|Ek}i5PwuulxgD=-35O8kxs8YhI_5eWei2c-Zd&-#zK3g=gjDNV!i_wz3E0u
zWsmvUyzWa+K62jKqCM5%#^tF5mL;6$tH1@g9=A(7=iillhzW?qaR6jH|7>4xNNmd~
zOWBGPH=(^3Z7!tzfCZ&yT$9x(31jt0x^6kRyIe1L@DtZ9@TW4J1{xT+kZHSj#s{-l
zjOCaGBx9Tfes7KVl-Q!Ik;Bx5^>wA|<OH*(;G%SSQD3L=HC3VZ1ZS)4LAQbIfb{`8
zSH5$wr0S9|Vt?VW+NTOpLaxOU_(MYjUTz$_)cprm+(%i|mYy#S?u1&|ET=wR;kD9Q
zkDj`K<<j9t!VHC7Y6SC^duOHsVxOcmz0@u5p7(+_N_oYauh!*?HyRZ3bs@(qj!!9E
zmX&S7ehQMsBOC@$ae{YHKbG=j!{={_yZJqO5_a8s8Y>%h2H1U2@&O?;3o}yx#h>Z^
zbhl%ucUJpf=~6~;uc#_v_x6V(_ftHEaC_^XWYmmQJ~TJmFTd~Ly^uRXHX$&xS`QTJ
z<EKMn(otJ*XIg-onOgoa6zIGf0Z8^+EG44p(PmVd^ryRodC3|F``kVvbM1_hZ{Dl-
zTGLv*#B&3|Hvm|Axcsc|Jw^G?Rhxl#<-Y>&<Z|WIxh|c{AD!en45#947=iIW{L!l!
zuD@O!FS@n9IG7*ncjkb5rfG`B_qMMp*+lf{LsQ*5li%y~<DMz_RjNvNT<j<qj6G4V
zbb4`fbYp}e6E^IRkEkRUzdLuc+<3WCjq3dKUpX&3S4>ZeMFCjI5e;dRU7shaD&3;W
zyV@@;MxI0USNmq|@+8Alfn?0_t{wVfA=28FYiXtk)m(q6lz3%IRZB3gXZmZv$5c8$
zXvNdzWx}_8;Ot7-k!P?|N=}0Imy_q!*uAbj&)jd6e#e_)&Cpi!!W2`E=wRJJq$Kn4
z;Kj=sWDPg9;VZ1Ut_0WuHrSja%FLBCG1q^%!=2d=d3Hk$5zw97a8tJCpn&1|bMy^S
z0sHX#Ted&g6IE1NY{QB?rx0dzs=MoF3+qZP$1N^ee0#W179{8RIjm9p!&Dp%4hDdM
z^=hFFw6hRgMe}b?)NThmx6**=!UnIB+V!gR&QH#&SVe=wuXT|2vW6&gMcL^@Lx{A8
zte2*YLBNfBMLCAQX7#VTCO`2zX7;-&>2*JZe!i0P?SMsXFb++w^lhBCq)2T|jJ;rS
z)8WC$gXx1gPvkDiPaI5dzleP+;Pp_qjP?A|L`;R21hHx!o6$Rz91YQ38Qs;;Dc#t(
zaJ<!~>Rh)<M)ToeBlh7GorBNxDh+FR+=gQJBMo0Fe%b7Qb>ki0XcJG3i>+CRT%E?v
z<-`dYXi5VgvxV8Sd;3u<Jv@7q((vrSQ>KeO_t5R0dqOt8q2GeJViHP5W%nH72`~EI
zUFr9K|BVFDGYSbN>jRx{kG>vwMf$!AE|H=Rmo)#Sdb_0tx!RBdhsr4fAd%X7<aq_e
zJz*_I>J;VyRM(?Jq91{YxLUdoHl*fgC;eFD8(^PljF;1GNY!u<qHDhB*7jErj8#^D
z9dLSRGMeD$7p&!6r3c86Y7^_j{)x#4>0fkS+B$+FCU;@aJ_LRRHE3fU;fE|JwBvPm
z`Dw`D4#8tbuCD?ripnB@FQa_p>t?Zhjz_oqFW;YN_L9ulT7f}+VT0Te5wo-EDBD{V
zZic{%3uA3ICIa_Fy=XxueOkJB^oG^p=j8$Wo=K3Ti+0aMtsCH(?ehn9k+Z9So9>Cv
zvqnkd_q(kgy|pgP41V?Mu0y%sjTK4ergQJ3fktqMP`d)4xl$8CYSi>joXG!BQmH>n
zz=DA-=|%UVvp(p)m!j6Zlla#zfEf?QXbAk00TC7W<LIqW*u4A(y#$jvrttf6E#;)Q
z@aJ8_5t01y`skkUScWDEyRfVVW}dCui^NC|X1@+FQ4H#}6NFSw{A8yQKsd|iY^@7N
zqYBa(@L-hr?wbqGtWpl<&)qja83qie2_^4U0S~R#zkWRTTs@O1mqfz9GRiy>dq`8t
z{UIAH)vF%|l3vN0;b!gmC|K@d7W*63exkqZf?zW+UU$rNT=B&1i$(+|3Sb4p{#E1e
zTX;@H9%^8SZvg`)2LZc-a9b#I`rb#+sX(3^n0+<tuYGS>du_jUhPv+&n_Ddh+SE`2
zO|F%(^#MV9pCm2w{o45T_oTq(jj2SCX3o$D>8^hMJ=QAs3B!bF?1L%y<!m&=lqXH@
z_CBMUOaW%c=qD%gtCz|hBj+v?g7bf!gf0%<Hg0~kb-Wdb?9M4^Prax|6mp+mjQC4x
znxo(RMG2s7INNpps8^Y;El}&jM51OpLy!J?ENTZ*X|7Y3T6qY0ho=g$PLRHBGgnd+
zSnE9X)<G_!wag0PE_Up*!k43!JILDnNuBG$e6e!r1%zAf(=cW#7}bVHT`0iL-dZY>
z<j)l;e)*|i#7Sy}p)YAGF$+clqz~;d81YH^sUj2yNio;+vbpM>dGDT&f;+2Q%X#ns
zP;!Ik?X^w%9MXK9_urn<%4<5LG}M~_BA2RG=f(PTby!un<xASZp7yQRmuSh~n<1{}
zXVS()^?qp2goOOqYnu`yxg9Zhr~1=^ZT0sqbI(_g!Ct{1t_woO9Y$pCmi}@%cjs`?
z!*toG8Ss}|kF*JBRY+gQA-(FSU|Fq`Gdo-pO@d=ZGUt?T=2lONz2Z7_-MCr$BG2&m
zz|q4^GMx!lc*ABsV|sOBwZBI8)-QiVYWka9TOV0@BuKagYn$)<b?e864C;;qiFby2
zw_DH2gSG7{5H+s`c=JrTz((CGN4LG3{pyc$&kT$HCseCAbA8~G>(TNj0krc=h3_s5
znCG}kC;xEW`?l&GfSIbZUXGpo2vE)jKEMs0*km8O8%~FNFh7RW9<e}I0rsjuEAZEW
znE1xi1!A*<I3Vr*r7|g|%3XbMNJw)%eSvU#2i%rVzJkBqmw6NwdRzV*&<yXqvy~Pr
zLez}3rk>KnnlMHjE`6}y%i{w^{3FS6y@q$ETlBF7JYX6rtz6R{Y`-@IDPKA=qOz@Y
z{B^|z5i1gBl|UG^Jq6u({XlPD)b$_xf(~r9#fYMBmmI%}ylT_NL!?!X)%B17_hjN<
zgT)2+16Lnvqg0D-5fCT1A8ak9Jtd(V<8?u8!Y4L+?|tUi@O{%j8^I{BXuI3LwO>{G
z_+q(S+OfJD2OaNDZyWxLyH*H0>KwB6WjbNKhBOUo*s?zj(iI*XyTz%wCI)UapLe<V
zS0>}tKAErfv3vSm!hItJQlFQL>CtO2??L;fGJ$UYfP`-bG)0Ac0?0oHXOkIn{exJ+
z;MZTLj|wJO<oN8btEibd8pStz=G9Y>zXkc+YUw0$ebB@DJABC2gx+huR}HJ4Bs7?l
zWD$PkxtLxJNLBANnc41-?wPE;;cL3Lk2KcsP~7oXBN-_oANPplZ4pU4eiq<akEH=N
z9rl6y8t&!~Tw4ZE71H$u1An31tw*butBynz{^TW28p!k68~k5sHk_>;WT|aY6!Wj!
zZ~i*+v2tJXy$xOpWAyDao1?dA&SIy_HgW!%jflKWwxC5$E4BK|GwLWC7$UMb@LbX>
zQ0VC{&!Iq9o0mHIYjx6~GdJ;Z9=@Bm)_cgM=19&__h$Q#KKnWYa_0s^4+-t45vUa2
zc}Mz?qwT5HC;yMGuMCT-?fwNMC5LX90i;1Bq;u$!1}W(pLPeyzySqa|KpF(4TV!a4
z5Tsig1c|fpdEWo|zvsHn7r*dfF8AK+UTgj8cC%|tC$Tt>iRxy07GtdXL|p<;x6x6F
zZa*r{pX=SWLb3YSeKuK)`-xfoa(Q`akqy!Nh9QY-gZo{Z+al&aC!c(OOWLNXRoG5p
z#|kTvd%OJ<?y^tvJL9v{#M?!EW_3Q^CrgpQ#R@T;D2<MnjxFezw*)S#Mw4F3B{<n0
zUyfyllfIif9@8TWyD99EqI%c54a@0&a!<(+H%-zSDS6sHZW!{`EsHqRN#q%Bhbdzw
zY`%$Amtx#fQ|F5i8Yo0Dg>fbTg)xhSyMy}9tH1IAA}HDPe8x;8D(byi%wzV^aV@C`
z#Dmt)8kljsyx$qC<<Xyfa=d>Q1ufX%=ElE2KSf)y>G*VWW^ZV*<g|NKGl?+}24ZMt
zO|&5=Hxx`c(5LpOom=n1RS^=1Tl3o4c)C}5+Jk-B1k`od3huun?+;Ss+|e~T8uclN
zT5kEhHpyeLu!q&_8eV@{NrPO-OASD=xDn*MuR8rZ3)l0df(VDeRDH#%<5DxP#wWRQ
zYg%C6>g4%QNQJ5pgGd#n5{Zpc%P{C8<$N=Z0dcVynq6@~Q#s`NvwRzi=t=Uhr9Nt)
z%d3#~G^00)cF5syk8rl=q3irl{eO;n@SDRI$EK#H?2lRDWWDH^cm`%5yDY~hRvn_4
z5Gox}PN&)xR%hAhS1VZ|ew!BWI;BuH8x<N+DRU*aLMgjNBXG3fD3oiP_htR?^C9U>
zmujnh8mau1m;Qqvj&A`8n)+eD%l<xd@u+KELHQAl@fh18oE7Xh$j_V>TcCnq{QgvG
zxtw0P;~}|4+e0rrI;imVQJ4hBd)DbI5Z8$OEZETlHIv(CM>zt?40q$J{K+6|YGuNE
zyj_obTWXI(<4s9Rwm$}4UWsMw3QOdt`|IC7CFCJz%2e%aiC=!M0p=wj@%!!p(+<HJ
z<c+sw<RJXvxmU&ZVQRjnmpNR4<Cg-xK4>Qt{w#{QIA@<|38>z>SzS+nLP0BhNsljf
z_QHQZjae;!NTzSE0;t-j4^?~A5qViBp7b#X1GMMPeRPFBmw9M#;ib;+R^5KR3-zv+
zd;YQKUj9z~__GOi#O4F|7s)3ZOw;anWU%+ZmiQ0_#L|!XGcE_6D1<dg3A>+-%MZD=
z{$=eBK5u_e*_lSoWPk|3`z=&O!|{6F@M4$}_F}!Dd0yZ;8Qi+6TCQqWsf0?p(?cRD
zkMTU&GEHDEKS#9bSaE5$&%8oD&QtSjO1;Tkr-GreiM8m_y_DOup5qL&$JsG>p-!Gc
zWA4M$x9VNi_vt^2i)1BvK_}sz0I=ei+2XH7)fokqrU_Kt-3}5D`z8EX@G4G~iLOpo
z!kBMcWAyYBwt7?1OTxW8(YwzIgKu9cpC&q0vc+iK)dEpdMc6T4ZY-z@mgC)3!dF-i
zcOp5nN7-RyOZ9td1we=w@nIRyhvdhiq3KP|k<V$R0!52b*>;(JlqH?;{HU?FYT<EO
zb6r^^dr@x4b@Q{#W!220+utmxrysYs%`EyBB46TBS)Lr4>Yd!5c&y-^J;lIAuwi8!
z9sTz92X>FgEZ30jjPBxbLxR~rPgZwlflRj?@I#4C7mk!B?NW`ruYzC8E|x8c5_-?{
zmOkHH<mVRkdiSS44ff|h2uYlje^wF)Z9PB6>UzdSUaNEgQ^Y1fcKLWt)ZV62Idoz1
z#q>>9>B#w4X{t%wo;CP?lXi4-NE`lY*z_jpAl)KX(XhzKZO*Fiu$xn-g01$);{>aQ
z8fiWfQq>lZ!9o_pcLY{JgF=JzjSfAmrP-Qws#92-g?%(l{Dl$=9`vxi?hCP9_<J|y
zCz1n%wk$<&7aW_>#GfPVLk3ajV<Qf9IXr^<GJq+4i`5?45;8~^j{<%5>KJpRKI-Z6
z)#&>Pt%ct~@u$FunB)lNz^T_kQI`NiCj<RurU-WNgtzng+`iLLxjB?@Xat`2k}_c;
z)UpIP9d%?W3Q^W(M9VF}*?llZXANO~?JG^rfY7Rtc7WY(u4#SM6-Z1GUR5P{KRsNM
zW-<?9!}7TY{8Ok+oP-(0l!idv*SbGNuQtr}{G=<m1>G8NlnSO8{RHtnGTGYkX*Z=e
z`nG)g>%H=3gnN<WD1wyr&{}N$(*uOP)lcQDvXHM~+v^p3igPSG%cbGru`HdsIv?Em
zZnwO&N&nSPpA4q4d>IArAt3tT8}YIKK^D14R6*vWL#b+;YGy?y6jiS!30O1glCXr4
zSN10iT6cO!Wi(I_95kAg!V&P{qr{_qM8(jlA95i!iXHAm1c`zLG&A1i#~T`b@O%hq
z>ci$gG2;dED1qxV@&9bA>%A_)%M#et^uprvmS|4M<ZAgDSZA0g6d-5QH|a_6^AdZ4
z29wWYfvu6BxxMEWIJZ>p>#Ml#pVJTSzK?TcPa7c3qkz_x%ZDosUS`@W^4a8iSVR6u
zvTXj5x$rU+vp5oen8M$Jwpjp}L$()-4?rg1jY3vRpPr1V#2`kZFi<J0eswdvx;35P
zw(iA{l=qzK)7fF%mC5*W_LHPcCD(H<a<12}Q{0hor%Z)P6btr^%MsCdrz0083dx&V
z!Khv9uRd2dxhsjojmZ$e@|Yj{Hmcz{({atuF^xoJJw|T+GhC9y7DYFS6sbXZYPM-8
zRuegrCP3EjyJGp7V<c@;c@q8!W_dVUlj`kpNfa^N2~oY`b`g<s$n-3l!q*$EFx)Ho
z<!kx4&%=1E39jh97Uqm&i*qEXnfD9tj1YOUt;mMN48!E+O0R`+lBn15PE>03<-ZP>
z%(uj7Xhx_bi%~l#XCF_!m2DI@{Uqe|$K|K5g>=Uqk2WuA%9&o5-^XLeH`&`_uk&wL
z4dz<z8jk=7s+%>9bH4b6yi+sWiLU?fIrwbZT+&@E_LJiYjs-^5%mla2C3e<%9BOg+
z@)!B7(9-tt_T$EjRsR39U=n0fgBDh}IpxWemU^S7dGojhGo!B_44eU+=p4eDk3*zi
ziVk2z^#EQVzvlGjoO&llcn0r)uGbQ1`zb1Y62XrB(iaa_jt(Xj?Y(4|ThSG`S2`v#
zhuRdon_1E&t0740NlmvoTQ7)o21NP+j7lGGw(4Aw%nU(hL?wk+NkZrogGuWR(gTUX
zwWyTsv!_$9X$#PN+FA^qsPLC&VRVddkfIb}7OzSuQDS0dB#Y5V^XgCuF-4{r3<8m(
zq_jS$C_0Mb`i{$zm3;BJ0t|1uk=}ApG-ZyR$0)cKK?KM=5Ow!wNj!v#3XJ8VV!QNw
zQf##guNBimYMGQ%g8`k=ryZ=zeSQWpP38Oo)5b#z&I=0}!;bhgf@AsK>W2mO0YlK!
z%3@t78^kiQfQFex&%HKC#9L=yG9)Y<mGqZ6X?nf-^}|6{OZD%?F#+-zcevEA@q39;
z*29JUM%DFVK&89DGy%Q1-n@}+9sk3@or$&nyxKv$o`A|O;>|x5E8vO0X5C(I1Mjy|
zB!7My1Z5c_v~9iSm0YV3Ky$uhe)_zc?2q_nW=KnjKbElfrlB+>Z#<Hq--{3h*F55#
z5%=T=Rps-GkZ`$d^=jiLl7@ug{rilU%MMVC)zg=je0R%Ip4>qHW(|v;)eA!EFpjSV
ziYr%C=+<=TPIrwr3U1v$^qv>Es3wcBj%PW7{C&xNDJ)IE_vSaWH(0x8OK{owH{Ce^
z+5|tO5EPV`vzq0TEq#lOFKxBg!bEuEHOdwSY^y+@`?5>t4&6yq&E)L?M4aTI)C0AV
z_w7A^6CFTkP7GrXk1>~hY(}#UzMi7no^(FGpZ$F|c>j5n*pX_Gv>K|ir((u#?2BI~
zkaPK?HMQ0SzLNcRRMFUTg;Y(nstb1^<=FpG=Szvb`*njq@?~p?#bQWqZ$(4RC`xVb
z=CnF0`7q|k{`F(2T(@B18;c;%ogut6bLtayk=B#)H<d~dpR2;?{Inc~b3~VTdB2?L
z&Py-E;I&o;aqV?qCVKm@;v>~m6&?@dkBTXj7NNp;$QBFRrM2ZWFNWUIx3&!K6(`A$
zjVq2t4914Id2e^ecj!+`Pnlk<n_A8NEM(x8XkbXTGHg=VFttdD)x1vBYrD~~{w3-*
zN!;wxTF*{e^Wo3{Gs!IyHPd~vP;d>x(bpBlM<kEH!8%vTta};N`u!ZC*d!2YV=YD#
z>tn7qQGK~_f!!Zs{C)v^9qqO@c2u&>yH0`4s!6TK<7`UwI&?;`x_H66@7AKf+hrXM
zZZMsB>SZh{5dEWy;EyaHqLp<C$)1Nmr1Q;82@%G?cLqU)!BgJN&;g0%_G1#xtB?O~
z+Uk=2vZP{F3kGXe#~Is?jX4*~f+eR6q#47roLbB47j-t|vBLqn|1p=IJrW9(PY@vL
zCJWItnB5a4eA22W1fn81IO1pl&ui*bTN(I0M`|PjMQ8(Xy6I>|xpv%MD1-FMf};x^
zePG%sPQD;F3H5N>t}mc)0iek%=a&=PS^YU^#)s}nc1HVpSFdBWA4N$1euX@%#E(h|
z(XD9-PL9%)p|UWCg~xp9?B)Z#CCFV5l>`$)D`0CSA2IoG9CoM5BU}p=h&^ONT)K4G
zUM9mQktJ%D0epPIxmuKFHy2>9U8Z&PHsn%jP&41$E>yr7#>Oxbv9ZGpCiD$2Djb=G
zHEA`FpZPb%4=0k>IAYPy?KOiAg)m=31Ol8t_nC{}Exi>$!F}%Cc&lA<S;a*7I``e%
zG!I?KSi{z>MFclgjqQER8(P4#PWLpK(EvqN&kx<!sIfSHM&DIB9H{|j@Y?GTNP^;W
zk)c_6n|mo}E>X_`K1z_MRYVnZ)prR)guBoXDB9lLT$LMrSN@7hxg?Uh-s^=nVa5~)
zAW;=U4AokA0n!0~qK%dFSMIqwNX{u72Zp6hIS<7anlnWY-#VsPpL@{N)fmJ9_KZWt
z<=No`Tl#QXK<Bgq6rpis`1X2+=Zgz*pf+Xx)~43~&_8=9_oZz`E^(1wCUF2vYWCXu
z=189=ZWVI<A(_NvSKrtAmaZBFLz?D*dTKm2AKqnOGjAcVVWoV4AgDz*#yTK`St#F4
zKe^vFxJO7~_-GDKq~L49G`auq7!vMi1mL;~4ZfvME@Acpl7|gEu1dca1g^vq&ovXh
zTJ`@Rj9m0=4b1JYjqj38EGdPmS=)dmSX*ja$_+6bworDPmNPh?vRbf9a@<kk7GUZs
z<+2nJ-wXP3Gd9#@H!@Q_U*))q-fv;9M`YUqj(19zL9zoUjE&=)D?|-rR+PYD_9|}l
z0lj$j*59L1Ge2{R%;>k?1|FNu{$L0|C(EbNf~>$!mM8Mn1Xv3b%Bh)&;My7vo6%bg
z0oJ}6wWG7oO+>l&pL$5QQher#`NXelrOMkf(HZG+WD!oac6<{pwDKv>^hHzKbAV#}
z3z&1EA_CXy^KN0KY89UdZS@7MsUm+d0)&Q|P0?X=2qvBP1wHKT>)Z3$QfB`JX{1E-
zs9{6rukYLej(oEF6RG^dZq}??F;XFk0)261tYM}u``n$Vl~F8xx7Gy5jWYe1WU&-S
zKq#j!ee*gNc6(&_3$Dk3*Nr6$qyeD{cy3XIn0y>dn^>o`yw)oAb%qBuVk5Kd7mb=^
zo8L&~U}2p}OFQphl{dcoz>dX7iThS?C76hC9Q8yump~^*_P5gM$_*U=6SG3#Wg0~j
zhb@(y*Y%DGg9i4^KtYZrGpH(>mj9^uKpi008lzxv##o4(Ag=$rxN@7;gOP%i4g7>2
zRz&KIml|r;>|EE|Aoq0(x{v%8f|o4vegxT^CNP$1OtSDNr7S7u%j70p#gGx~L#Dz_
zdV>?Af12XOEp7qoGX)`VWNfvga{kUP9H~&nvRUg})}uV85vlI2gr?~_jR05GV~SfK
zYp>ANgtKcK$jxv?0NiI{c@%ehOP^Q5J&BPY=4^MRK|LV|17iCbep^g!y2~_Hqmu-I
zt|qaVILaCUn9z&M23k~L;@Q=W)z**u{_+oty|PJ1HDv6F0^g%;@jmiP?YuDUJpHgr
zDzqmx-+SPv7sztm^T5zF_dP)2F9keQr$pR?9_nM|hrgI{C?+4gKZrBiRYvY(+sjbv
zko2B6$Sj6puH|bOxaHUVFv4MKguh?Fi~dkpM4A-Ud7V^b?9BkI9Nz;f{G0hxdxL@D
z`UVCK%$#ama7=@)#Nhs{L>1ykS>!0ME?&GGKZTbMMvnnu<M`IK^O?2A&-RY5F(N%n
z3T-eq&}_h_p3|8NRLPR;5wMDYm8tmUxVs9i`$eS>bTGjO^Y%5LINyG(<k6Jd7u22+
zwUYNKACRyx^A5kRr?`scS^VwX>V#dIvr4M=dN0_Wfu6?BJ<BALUdCW0n5QwiH`F)b
zIAQ*kpG&0$cB#QiwBHDvWmfmgW@Ssr<Xs<UHi{u*-!Bgd%VCgx<@0_x;<}HwhG)su
z_vd&BE~4A6ttCWDYq8tz)nfO~`6}WKecc)&S)M1RLO&%x%23deR)$>uW>Z4o`N$Dx
zwm{d05TG@6LG=_uiD#L6yb;b2tH66ZAaMuMmmKeQl2^$?+ifDH67F`Mc@o9@5@!fl
zBS)Gr=jYM9IO=M~=5v6p;>Ql`ET)#=869Ex4-<50q=l6u+M%f6t_$V|m)u0R_5Izg
zR*P>F6J+1lE4Y?`jc$kwrf2a!jK)m?`+oYOIf0j+YnRm?ICimYs6gGZj|fqto+=2i
zg-tWSx9|-I=snia+B$Q|j6<`UIKIrXap27z$c7`m9t4LqJnM00_YEWbzBMXxID*^u
z1K60{e#Yap;AOyz6_Z}Zze2phrv6AW2nwV1v~~YxxRe<FWEX=A7GbrFkt(V6?SdN;
z@v03VOriVqgxH5@OIaI?hm;)hv@7?|dNJDbEdTJYip@?<C5J0Tr@(1*?>;d`hkSnR
z#1ViVC|mHNY>L0x5zreTqfl$&J$hGpd0WI|5q6~mF*blCF^##N+xXvllhS(F30%h=
z*fe7hA)}ik9kN2iK_J{`ADRjnH1ScNhTBJSG3H@VyGUNQkG#tufxtj9yP;H0gE4qo
z_yL<*dDq(XCmF%DQWtj0*^-l_(n}^-$UYz5F$JJQunc`R6@^f6mx;hjM@zRraGyb>
zsFmZ91X<wX1HA5r-Tww;TO-pub8!>vHQTn7q96zvm&^vUk*<NeRhPtUM10{Qz&yDu
zozH;D7m4bictNVu&V53$n2VuZ`Ss<=gR5;awW#qdp_<4x7<y0+lpt|{wWDZH<>{$K
zaL|{T+ZDP_tg*+n?zq=0iB-*3ClgU~x&mcMbo+%-+}@g@?_XV!)u!)SC25QE`5Ma6
z(d;<Dl<d;z_UH_gIqzi-l&tjbw4J{pX32-1Yx9+dQu6mBnKFkF4_y;uHXtJ<ZJ{RW
z`2iJ8-&jag;CY?EjOOhi-*iAexX?bK=Xcd$5n-G-rpJs;wZ0KFbv;1)4k5*dew~oU
zc~efuYIQx>ab0gJ1(wVIT*ThwBK*m&?I?FBN&3FvH8U$FpMpwM(wKJSw5JYLGDY!k
zN(Vd1maZlMDmpH==?-~o$}%&3<+*|(A3}t`0LCE~7IaD=j_G~$JfTAK&8+CI5q@9o
zv*L6ao+q~uDXW8u>6Nti5_F?&Zi-GF_Uz1e3uR-jh~TFzp+-yOSbPfCwJ9~dUQ`pJ
zANZ;_sU#_v_uDo@=JcAKy&c^zt^bxHb%0YS(*36v*1;Ir;1C!onQ$U>@;kllZx{V9
z)(JXMP6Usv;kiFtBo6quQ8O<f+qVu<{)0S=#byif9;cfnRf`BnA6Ie>!DrW<iH0&P
zGvq?50tZn0Az{uM;2(L{hqhurfQq$R7{#TuJIzsSb1;ABC>vbBZ8)<X%9Mac&jFbR
zdMYN_9FMZLV(3UzjC+#tZ}}=tl$e|>o&{@xI7E+7sAbodd9Gw4V9x>>>^PxPiU@3W
z-z!3r>0w*FN$cl>fZ7}Zapmhx%W`i2Y*`^fX#0DqQ^3qEpo8ycwn{2y&|?95Ps#M!
z0|C#82e1jco1*AtJo6{FUc;)=^s0u{f{~EO?!pH;MU5%e_*g@YltVwhwNW--syz2H
zE{5%qN6@8}kQtRI1OuEZX;I5H`}|Y8MjkRp_^HrzY7H92LXW}tjQ6#XjdJSgs0epC
zOCd<ZmcGx$MB-)zzf=IzK6~#$ocWM_Q~N}VRs_nCDfrs_*hF3Kks$%#U%$N|<!mIh
z{h*qaI@%U}B>wq)#!#m+9ng?LM3)fgC?V2dJ6EQF%?I{lr%ltXWdq=4^3HF=@AD$5
zKisYL$0#pIYixq?3XwKJ*z6!%dezj04_dzSN^>)W=Ie<}$|&R8^r|v$oAo_^g&W9G
zl0{g2@iyT&^HWonKN^NlX~G91a#R5&sw((a-T%d02B4^>mCE<9`SJu%TfP{oJddh`
zEFrUEcORkJGFWx#n{o{@Yy<wf^Ej(Hh7uYv#U>R~yl?z9cPVMknj8FF{Y78S)lFUP
zwDQtb*T&PwYH#`Ru3ao|2PzUd=YShJU-*@50~L4_O6e%jAB>8gRQGdE(Y{V8qY_n&
zy{;z$%>AeMOpDEs?IWFqG>bB`h}xB&$n`pCu1f-PKV^7-Vfiv|J3ngh!;Jzrp!?XF
zz|Z5^SEj!^3$C8;-s%0zpe(S$GMuJrZeLJCR8sh|&IrRw`K+@}v+M;nR&%iL3Y9u)
zPD;{xnyp73&vJ78daDZeTrUoi4~FORJ&8uRxl336HhdxtT<`NF=6*A=iam$1pH##-
zdYz)b<z_>#`?+uX)0>;iZ+jHF<#0Exu&^98S~|JtMFEiQ#P1`90VdUF+Rv7(hlu~w
zHfmEZS|1M^+)YUIROXH1b1(jhY!Lg8ESLea=2fRTIkq0=0<5Oyi_?!*@j%J74|7_B
zydwwzq!l@IDu0F&Z>G=ps>?qd8Mg7EFDmE#l910u4KBe#sAY((lX%jD6Oc4PsLr!7
z6@0=nj~zvp()4Di-jQr|$GJ#reP<z@ez8(BIl&%y@h^x;z*O(RdvTW+53zjOIr6`3
zqa=G&bs>BT?SScy#gNCi)?D@_`LLF`Asr-DGI|c!zg|L-*2-Tu$-l6~yMAwjg1fd>
zZ=&JL!t0Cq>U=PS%!J&Xj$XhXedCuxek~(ee~ZgXH_{xMIW<VEr3u^}5&t5#tQ*%;
z*8UIRlA^c+=zVmmxsbilM0Kk3T%xYP+SAY4P6_@8Z|UEI=Pp~A*q**GjMHX#)q5VR
zEt8+{{pFH`w_gxakR_HuU8#100gE93HU7pKBX>{*f0J~R3U^1B3Wd{-g7XSKb=-L(
zfHYfWLuf5R%@sPh9{wYeOUKd#9gmv)0tm$+ma;!tABrHh*KzlxT5TvMMZwkJ6J!LO
zh@Vj88_ho{R~v))KOP)<`cMT(7sMlNE1{?|)#(GxuU9Z8=e}U>Afi+ygK$)6D|=!H
z=zVW{MlyJnt*qEUz}e#pWHc%?{yH0Eh*Mc6`vru3K?&F2{wtZ{K#6%W#?nf*+(?4P
zzXu4W07*?&JDgqERJXKf7yx&QIb;1!vzA?ndq2B9<QSi2Wi3D3%JcKBR+HE9U1y+E
z#GfNbAw6;T>b=WjZq52y8gG-fj0PL{cKhoR*cLt5j7<g(acLQ&7N}+NJp^cBE4^_F
zuQAL8s?PVKP+2jAT~L$dP2WMj^;VD`xS)^OfHIPwUBTFL5Xr%ASdP@c1MlM-d(!#t
zfTW1&#q~7Lm>_{&MWzPBp?M8I&4uObG<YSaL-Q|Dfc)gc+ULc(s-sWR`#McDsq_x%
zTLF>0n7_*e!92lLvMMs{c8{^3sJ3|YSl07GdI~F_Zh_TIi}}$#V(7XQA96pD#C1SD
zBjHHhWMZ{H=r{5tK$4ersj{^S=$@IPt}lKM$1VV<N#t{^&x%VoLCAGj5dT5mh%YVN
z<?VEnG4nNACd=fxnf$!j&;NuRO|glq@236e$Hz&_J_)TEpgoSLvlWpouoEbyd!E%_
zGEVC7f+4McO0>aMcXct>`;FP}IEU0=iqnV|oerFwr%(4?HwSBaeuZp*5NwCIju(VZ
zrd|mSPCef((IPlD6tqxt->gXMuJAaZGBvFJ`r|?Oi!7h-1?jv;lkQi~c*cYq&&i62
z>`?^JvQHcyn*bz$^>%Zld`Xho0XWT*kPnM~g+J|5O#qkzSKhTq_G*QwJp;k9HL>D@
zSNi@oW?2nI5Mb>8mIeqybv0hpS&XDQ`cWA`KBoP^>vJmCdLS_{jslVy7zaPa*rSBT
zqgFNmeQI{}JE3o7j^`;s31ZZr%IM}$3XvMcnZuZ)njhfOnHo9>Y)&W%Li2>m;x(4|
znB_)?n7stKo0W_gl=Q+8qr5(?3<3Rw#J}k<Ku_4AajRcrliN(dxKgyz;T63}wa|s1
znf}peS77#w*2+sU42@t}l=TNDiS?P(%q);Qw*hU)7gx>)?_wExg4HE%U0J&oE6Pqv
zNKs<==Vtz3bYn&}U?$*&kH#}^vs^zuNR^bCGyg%7ha6o%TZ_s)s2O|_<_|ryTFN5)
z>i7AgWr@CzzHu}~LFH^k?s=hOp(eOp8z6-N3It_{vw1NQLSJsAx{I1=|Kd%~8%#iR
zIkk?OIfKa`DLFY`H~V*YAEOMABeEs_5rr8EsH{vK!fH1=AK|Mk8?**$1MaTX`{hNV
zNvsrLlShqra7_S@xoBlz@+F@0On!N4vihjz4nNL6nZ!ILg;};NE!?yVmjcffM&p6y
zIPvXz;7RMMsVz>l{GEjWmLozcIn8*t?yf|cO8PVBUD&xg-7w+zpl2Sa993!+uwG@i
zqnH|2s#YYHEmi12D@%I*lL{gk31oA1ly?&)Cc@dTI-6T-)91$vjG5!_6ol>@#^u{<
z%6YU(eJdDBM(5?#*P`Y$>2~?#y)?`pQEQXcGXa7GvfH&KMKf<b#}#6qF!fj}61BN^
zn&d6h;i?KL>1;SWN*$QU<94fj<~V7;Td@+gzkS!eGjFxNI&o25Y~MNAqO89iD~(+c
zP!X{XdusX3EJ2d!mN&UqlQp(Tu!3*+tnir~B|FW;&s7Jg?>YO&ZhXJ5PrV(kC7!>t
zmv`#MG7O_x76aqZ@QBJ4u*&6`dD~I9s6oCw>njU3LFKX%Sn#f|vA<$pnakP=z3MQU
zuiTIL7o#+V`pZ(}j)|<t%S@cjafS48LdaXR<>-}jJ$|o*^(;PKc%t1jBUA;v8S3mm
zb!6?Oc`n^p;JaT6mZD8O6BkTBkjZlU7XVG)<O_r}CZTFB!FMYF>QYdJX1;(TLE?{=
z<nOL$J3=2ki5vCYFpLwR;Rxy26b<N~psw7IkT5$0!EH#rWd*_ZDC$pb(8M4&(c~aM
z8`-NKy1TJR9CIu_DAtgZkhDbZ_7iggO*^ACK=$d_gO(s^3Br5|W9oAzZbIj{OI6xb
z;wCe0VAVv=%MgGyGL6}?r)myR@(Ltkf+&b}dNYTd3?^qF!U<X&uY(6kBRR~QFFj6%
zn3WAf4EzL~Ye!Awe<Vm+G&5+elZ()u#P>dEE47VdC1n_9MaJ{T%=W+u*6t`oCo;N}
z$GP<j1!fY5Uo90`wq0)LS+p*0WRBK61A-HnG3dNq<|h(_q!9_)|Dc$k-h02+qJ9%i
z(vM#&lgHd(g~fN#x*rKg5?OkAvgmR*9_O39*>96rC<2;82~33^a<;D#2824IpNc#D
zhWQhO9$h7n+3f<<tNQ)k7#M|UHmH~q1s7XBzRp!V+_j6^o`iX9Dl&b~@BwdO6AtJv
z)Ep5B_`v_fWDO7n(1bl8DFdX14ZcA-%MM9T7#(~hILHcMNp5RzBMaq%p}y>{A8LS`
zi1C0eGowsz>l=Y>Y^<2|fQ6N){(hq<=I|;}Z^#m(ZkRx@Fsd5zI@w3%tm`qSE6`LA
zYiH>VuxM$-;RPDcX-e#F%di31<OGwt$gS;c`pzfed8;6|v-LdU1!0yVhg)U&yDBGb
z)g}chF2P+Ev%tDfidr+h<yyHsz)ea*UF*n0D2iE~%!D|eXJqaT?A!|g_>a~K?2={D
zN+JRvF3FP<^_+dBG|pr`wZ@^2{^F}L6VJ?~^Emvgfoe=XCA$GUAA*~=D$&pDoVQrB
ztK-E0bFWPd4at&H*)FnSFS65oAd90HF-FM{nO2Iq&leCry<$MxyS*NU`dqv;*^`kZ
z_60H@IsN@PK9=l1zp8oy{={Jc_uCSUDT{+Ae@c)>2IrdN%cY?goAnd>KGvJ-{|!G3
zkS7auRx_o{O<zbq#%0IBS!0N>Ui1xd=jF=16Qu7CPIgjzo)*>Ecl@Fx>zU*LsoFo%
z^_PtB%Ir1l;_h!+e^<b3Hk2Fx)$(6Kc)~1FovPYSgY`CFl*LgDPDX%4s$}WqO^5e+
zF9D-Z-rI3Q`EH`~FzPo7*S^<4>R0iCBQp@<`Gp}Hkhe1ZE>+K?b(e-K39un)DA9M(
z6QIKQGDM#ZgXF0}`ZZ~m&Ff{rvG3P5jlX<_C6<lbqso~402}<8*z!niuw-yaW=JLg
zhdNd<dN4k6`eov|QxZY#PuoiaLLIaDM#JcgoOl0MN2kaf<)fvgg)OaFQstJI^q4#K
z#S5o)*DcDBv!EIh5cWP?;@+EWA&ut!B9G&J_l)J>$?vBaW@Sv!nFn>eX2NOjVs+au
zVg6=?46;KIR)RCVm}0}Ay3pZuqKQD`xXeC3n^;qX$Ck))2I(JgN%-&L3nkVB^G>7(
zSs_X=C3}3|Z`y+896(?LP<Ge=xTL>}=OL9YT|C54%(p5w&^VASA4_dODUgtv>vJ*J
z(1-w({!{jbxd6}`OF)f|OPLHC%TwT<vo-KGIf8O0##CrUNKP(Y#+^qw*Mu-)Y6M?N
zk&a&#(EP(7<_lI_n|RPUW;Nj~PtgSq1E{Eeh2ZseQ4e4WHDXH|B1Dwc8^*lO1rO8=
zn)M}G(yMEwBAUKjK3(qny*P9AbI7GFBzIps2^c~slZ{HLA680!0MrR}$WsXR==S3N
zlY=1^o(;o|I64Yzs!<^UuFm5x8?kw35<I44D0ockvl_u{+5OKaBTS48unqXm&fqag
z-DdnG?KsKX%o|fCNC_bJAE+tv1P%&A!SAsHt?o4q%8%v!=@?#b2u<&VVclSyERGRq
z^j8xob8~{^b6SM6ueCmKTj8BSwZU+3HL}}=Y59JaDn0i-r71x3iS3Vvazn*gwAAOW
zfrQb7O22u_G;z$e3GXSnuP4Pb5?0;<MtohNR08P7V2RA&U2QpIVgH_Bemldsb~O0B
zhKWy+eO*91Y%ElP%*%eV&tDu^9vQM<oxMXIQ)gCmvt$q`xhqB_;a)Z0PJL)^jpxH}
z{3dZ>OGOzc{kO9+3wch3yKLeu1k+#>F-YU^;HlO_8J1uia=D5j6SyJ8i3rwO-yR3E
zi)@>29r!)(VymT4ByN($<?=J{{lTi9;+$suTY<RJXx=^ovx=X2!4+o{`>Ec<2{zKv
zc&BG0vMX&J29!>|jQ0@@=5h0;^8c|#9k@}D2`qf^i%hmSw#eoTor$LST@uAIy3;r-
z5KLw*PDZ*|$HC!IR=f7mM{cR--&wSczH^ljMAWhpeuR39MrQPrF*BPenF;=24F3H5
z{o7VI55RxadP97+i(Io2f05jNwVBmZ6em8<x#hs{@~I@Lf`R`^8=r=ae(-d+wMla3
zL&%h!7lCT*_9GZp;C|DB6DR5GdMLK5J$Q@&EueG?@$0c!G=V66$k7=!6jbP;fY{0n
zDXEln`gx;3A9W;j4~$*K2lCqLP+Hlg0=!j%w-&TS%xWs9L;~!3)i|YjzOAk3Z85|t
zr<>oEqb6VJvk;^7LSj-pfCL#jWyCk6`!K+opI4rqJ7|$hAa(*tN+y2$48V<Ig*MYA
z87Z*V%te^{V@*lV>0>1&$G(g=_JW5SYMD|7>8GS}U)=zNyc#1m%*7|%G8k{x#I3Cg
z(5nG<ucNH@##*LoEgFtRwUSD|znR`87OL<tSK~&hGoj$lavp-&jV7Tz?92x~aI-Xr
zxNwvUYOCAVRYX3)ROv?8Zll`*{QMyg1O=boAaC|E1+R5+I1@C1LfqSZh-2KXMTiS9
zg4rGZ!JvzdxqK<KBKD0`eF7-x?b(z(<c)q~Rk2`!qm*^oRElSo3Z8jNVaXxbg=Pj6
zfb9qJxFYtFbymcZs{Ik6nSx<pLDH>g88m-|a#gGv?s~liLM>}X(XgJb&}m}Z$dR0U
zO!%n`8QF&vfQdrk27mkzcpcOM=;kGWOpj&<z^fa6pd7x#j4LLHylU^V@zfm#*}5Mk
z-+#}cp`aHL=l+Yz{=@P+{9h(f>GES8!u6eyOnwH{z^Og(io-EFmH08b(<q$?BKV{u
z&)A=olszLJcQn~hvpeqB)X=zP87Gs*Vf9hi32TQ9`c<-B*?8B<&;c0gy6IZeIb!%i
zi{?7Bqu-yayQN0Ct)Ek*Th@1*Js~zHYJNV)-sM0b*`gCgiq{G`AItttgF|4$j=7z+
zW%B&k03M3%1cPji6dhKR>kH@GZ_S$Xd|Y!OcCs{%6G?u%#mx8Hs~E<BiYPhF&(*H}
zV$ec85YOtnY}3*AfV)n2l}M1S(?JC$-?(4mWy}(rw~OmyY*~#351}fxBc;pEfC_r7
zJMXjUBSHrFNG`94QJcG6i_>wPbySS1xXWO;%W(3F*@RkARDvxzwMDc%wUSbm{BES4
zL9}MQnUA~mOw>x|pVY9t+v?hN+N;H1V<59Xfc<t{BnImiqnFA{6l+>kd<Vf7!yQ{|
z`UU*e>_=;j`%K`!)?1TW`v(f@A&tWYb>jQYLmI`3|06sF+ZuY;yQ{6z(sY_!2+);r
zqTr39eaq8=z)}7YE_ry4FZ456OHC0R5TTH0JhH9E8>VsuxJbUDC4im1IrvUd-w3dw
z=RM-kj>1WtO%o<{+3BzyFC=AaB(U9XS@!)CQV$nzYp`AJPG_yb{HBGN3nwqDIUs!7
z#E7vA#^WJlcN2jaI-kM<P*^dd8KG~?q;jb`Q-b3c^o(B1`g^C7P`w^#+K|<*J(%+b
z)U?;2lyHaVQK~aftVn;-o2|#tnB%O?lA3wkI?`yc|JC0Kn*#c}08}V31=dy!^T|h&
z!fFDKK{=P9ix79zD^zL#z}$Wh08Wi9YIV@0%7cscEz3BEBoCS2wcYN{fJJ~fCQ^3x
z4e(o*MF;vK0(oTRrGx<=K!<b+6j0*;4jU{=qh<n0_hqSHKTj!#7{*>K0|oc{lYE_F
z-G*#xS2dGG05<mJ-1f8W*E!pNo^HHJY@XHuYCa+LgIdnZQKLCeQc-#b>pm<#1Lzkc
z?1yGq{{Eq`Dgb!*Q^P?Cjl{D0$=z~c&ldSIGd<5H*7CR;kevwiHY7U?u?}(MzA`~I
z8+{DrZ*Kz?jQr!38<l<_>KP9Hw?8;G;=vz`=#Jp`!<+ft=lsIh2&DFHlSlE}UPBQ^
zMsTv5>k1zIx=yL*c>#x6BUORwHKLR2=FzULOtG>_Q*y;8LF^NCF${}hHH(-z#yVCX
z%^M57yK<Sajd<?4WZYeTzbmGLrwMDZF8M>l0#?|;-+2{x98`pPP_RVmkQGoug?Gww
z1Y9lIt(Aq>%bRa?<Rdvz?-4akR;J8YV)It&<>%@&<_Cx`j%l>-W+X1k){vSxSEaI7
zxZ)&<2IAMZHF|n#7h*BJ;;5+52{*k?ze{%r+T&6)ohVN1o8UX>Rb$4%O$k19+$i-h
zzz0(_A>-7|_cceGeiRg;I8=mix!2g`(2W2^q7W9o?^C(qC;c^^8@oqa_m`S2I|(Uf
zfQJNbYosJk^OPexdcf_z&WF5VV+?cII_t>X`xw6)#{iS>(*Rw{r+&JCs62h*RMDDY
z1~{gII?X?&^b*{us^WChabD%D>L@A(S5<97+TJrC;^k;^k!fa5uo3hF_RKxK*8?MY
zRKB9O@Iv0;=)F%J;vih%enX4OI{ojs#Q>7n+$YUD0}iV}7_k;V3l4k;lvz=A_SE`#
z$FW`Gjy;Y-sH%&Q4z7a61LS{W1BRC_JT=0tb8?ITTnEMXuF>~9Xq=trsyYGU4Gp`o
zn6u*t`RqpwEzzkkMUGdZ7C(awFuP?;_)zyx_=Jjyt1;MrqGVt~<d>wMwu*Azt>F6!
z-s0EYiX=(~nPSSp7NlL1IoccQ$Jznnh3{hjLchUq<*Ov$^NVyi5vJOM*6-D~T+v$8
z%%*UrCW)=Q`;};ZDj_tnxeZZCNX>$rfyl-Q6w_SeH2{NhR%74r&^4{RYNPBQ15^iN
zE7>-|$KKmAbi+(Mh}woNF`I*Un`ojZDwvPZaTE-15PQc!Dy^OH`wf?3iRA=xUdu0)
zudoS6E#kyd9hHJ{Kxrp~4z9NFKLf(+Z!`8X%6Km_nO0F|MY>my0rS68TZy-xFM0!|
z3NDH8d)J{T4jHiomT>x@BNvLicN@X*5ni|`FAkMJW#1OcEV{<$aM2!*kzru?rrM6O
zZ0`P;CaL8V#mhWqIez(u^8x)SUP?iwMDMi0yAQfjOC?#dfN3a9eUZcLr6^>g1=J5E
zoL82YDS9S~_;He$?_U2!?RY^hD5~eSz|QJfS(gcX1R&JRqE(M9nmGRBSB|EIEbK05
zKT#CQ!u<T5jBwSm{nL}L3qN}5&JQWW=o}_@74&+k_vpMH45Aob4Tf~dRkpqcVso{v
zh@R#a!iI76b4uQK08rhU%yd%qMn-cFp!s>_YT)1+PIBE8&tOG1btMUr*K>(#Fg0PL
z7o;zy4et>vv1F|0gx5Q2<jnHx<Z}(zGaZd)14W{4L(2FI1g3Qt<~kAkGFjL9BHORz
zo5&3<1A;%e6P-0x#{%V;U>Ro?)SX<_jDz9G&lerzm4HvlCwLYok-5rUil7anb>?u=
z?+|dJsTB~toJc$hv-;TWqpTOqtamaueq3iWneVc_B-0J{>OGF3APvQ#BxoWR^lx*-
zLc|=)Enhe8b?@@;bqD@`B7VRR5kFXeBYvWvAOFEeKwzZKQ_`0!A2uBJ1cDnKW`gx{
zO-WO4-|4ttsekMl7IF}w6}|jx<_Dz1MW9#X_=FE;_`FRX05gRavNr3_7cm=d{H$N6
zfWQ%G<!s+wKWF|kytX3O5Of7HE~EmmJF{b4SD?E(Xpm6UN7Q;G9hwb3AqAj8G)-im
zbJSD7tTbsOV*(AbchsE-((dk-SWpQ#T_o=dK5(@M!CczYMG_SBaYl(Y`@bOiF==x9
zt(8mdL(q5zNnsg1=E+Q~+7^%Apxnp(Ue?;X&wB|OL)7|XsW%=1-(!S#XF8Z7#jE?r
z#rMrFL8pRj1@)0f(riukIm$CmUc=E?-8v@8g)jHt7`xDLnS2p6o79-~T1wybfP;kO
zEy<R+VM9sG-nqse8!6>j(ey*wghw32k0V}ndC4ri)7jN>G~_&_3%9w(34Gd<s=u)=
zpRo9d0?TZnQmNJWmg0pIUXC$*3uP-~TN8S{tF~?j?CBTNV{JFDjTwwp^UZL+z<~gx
z7HIVdL;eX((!q@2;4B0h9+ds)3be|*K2#|=WrjWSvg4!Wf&1LU1`z;g!F`Dqx*(=N
zbmvRir{CupPjZ_iLb7vijVuj~+DoOK{MHlX>waiultn(sRTr8%F25drpWZ1`5|4|>
zei;Y&`DmvieE?Jkp!g4?ZT3F99}$04;9aaStIA8xa%>Hdv0nh(+{}YbT}+ZSGPDav
z)UKy$0(L~R0RrwA4G9xZM0A(m5EUyA2S}hAuJhwxrHQ#tVcaXsNE`s!R(mp5F&T9?
zlkn?O(;nGTN;~qW90WK<%rcI@8<_*@p0F4rGrK_FfSztIh}Wifk}AsGl1GID=@(QO
z<^%E+G~SkuT5v%-z}~=lu9xLK)PDY{wjfFjt2Hz2JT7;CP+|=Nr%7~@&5u}9eCDy8
zUt_~Kx&O@l1-XQ6)YCUelS||#FE2gQ>9E*%^*&*#LnlVGJ_?7WPl#4(*8NvjoA&)v
z`;V(7jg~v1PY+A}E2_<}=&R)y<ctSbB_}g3jtnyn=A-5x5FEsZ@a0y)c7A0O?2l_L
zx}x<0m6@~rYYv^nKZ5@SPs!*zW0Z6GwyYVjePPN8WOuFDp)*WepXY`HIii9^8{rTS
zbz&eWr?-B_X8;EjRC;QlZK;tNvh^jL{l|NB@w%l*$*-uHOOf)Xd1T{G;hsT~(L#Hb
z^>4@&4Mal-!Wu|&7+K#XrR`tH!n%^lkhdBp)v)EMUqdPRQAL}$-|WmiMDp0=byU;(
z6ueN7*~UYqH0w$aUj>|3#TB}&>#Bt-Qr2F8_)QrD)fTf~ST2S#CA`vWzgB+UydkAP
zUy6=2-TCuIYF+T^7hjXDzfm4VAbL%fh2a^<Y(Hi)bZ7W+!cj{e2DmNp2uT2h1!i{p
z0Q@Ea0=+@AcOh1g?(bPm2@ph7aXQBY7)Yc3f|#r5;TV!sDELS;c=Ot!R7eK0w_{|b
z{=x~aKS}A&WGXu<{ah>ku7^#VV*2c_4ezJyvKz)tFUm?)A`!l9Ye7Qvx_O?xqiAAR
zrN(PTdGqE?JdlBy+9tab#B`@s78X+MlPb^Q?YH@l5ZRVbKDMSi76j2W7BIgUe<ZqF
z>9;d1VHNS<PEZ^#QY%dQSUVKfXm+a{cQ?VreeY&e_uWrwZp?jJUO3v(R<wVB`KQxH
zSm?h#od=nrAc`XxDPr@%PyfI{Uh{d#34#**nMc^gsyKiA$^?dez$KgeoC`u!!}1lF
zLc>Vdb&IPwhGpMvd(@wTE?rRpl~6Z>VVgpDagP8lmk$Q=ps(okw;5GT><<rw%sgSR
zLqN#OrgIHTQrBTeWT@f$<Oc`q(b==Ya1C{_bM44<l%eDNAiT%W)r1<{jr$w;ec84h
zzLbCI5VKmgJ|(9{VmM)_kxLgE>x54EIB%>Z)!*Jbo&(5s!)^Kgii&84XHsauFm-Rh
z>JzdvSz5|DC)^$OPtah#E~-6vyhe5X`}1X^8CgTd%xVby=_}?j;O*{*7|QklQ2<Jy
zbiho1*+{XHa*!f%`59jFXCAhOpb1(2SR@~}3C7I5PA9{^K+#K1J?DYHl(<1^K}rph
z%n*QjXu6IAo#L!)jX+*{Ltm$YncLI_`-uA&N)zb1fI&pyldvn_lbwMIMqT-L#guLP
zE2eK><ecd=A>z{<&yoi^Tr|J-1+}_iKeA7@g&@IyW`v_h<E!TKWx;34C7No2USZd~
zGv^!%b%TwKVo6r|InaqDPv<?8&QTL)4cjEF=e(H=yZc>8CmpGNRoVd?fMH>u!Tfdr
zXZZDugiobUKiaspIiQrA`BfL9MI8sceMZb8t^Z^JJQTt|UqY-TGo+I!eKC17^!Gt8
zxaFfsAX0CJmbw`KkKl(84o|+?%Cv04yilo5LR*!0!zaa82hGd5RTpo#<^8Lq=+RCk
zY7!;IKLhDCJwX1Oj$+k9;{gEM>?p2%sZy^CdUv8invkm{{x5u(EdsoO_3qUa5&{;W
z^|)NFC#ma>?aqGyfJ<f&wE-rr08GCPy=P0Lw;K?)v4~wQrx>y|IY5>kI)*1d3PGWV
zen}#MVtkeRLSj|Zd<B}+;;syfX{&2_)oz36+I9c)b`1#f>}t!`F67^wNd2&{kIM_|
z8Mfcxhj7*-p4HShqgj2^D{Q#^i6Mdc{D-R+XHB!=dq5R~{h-#-(ssThYnE?X^4xn0
zI2kMh51%?Zf=AE$FM6P#ETS5Iv4lzjnmX73N-0u+JpCO?HhP30cCzf70A~6ne?hFX
z5;{7Lb*8DEf!fBBJNr@<+Fayv=)2@;I%Cu}`%=6PK#p9#_ToNWRXXQ@!H}|JK7+_(
zUXjV(X5@rh4Ncgd8MwvL4J(wAi839DVH!)Ds{DzQI-chg14_RBJZj>;*`p&xPD3o8
z-7y9<cUZz>*Xg8T36JRkSOs&+7+Hef@NeeEJ1b^gW~n?kW6TC1HY{RZo7<>tmfA8Y
zS-4+?mI|7mL5#&WnIlP*;PQ7keE<O-m}PuO%{Q*LVB|;W#Z|1rf8?a-g4<@*Gs5K}
z*T?pl_lWj2Faf?tGPVUc5xl0ny=!8m@&Q-JgV0Fvd2BtYfxZ=J1<3DI9=EGI5GxS1
z9E@)uK<eUACN}rwAYBFm<kaZ*u$b#d$V=67C?8y7i@N`O3|0i>qV6sTr?vw^kmu$|
zhE+qWIzOn_o!}J^M5~Umz^lUmlNxdBg{gPXxP43G{`4Vh@82)j^(o2%tX{QcJyCfV
zGT!js7MqkqVL791*AK<=Z`S5Pj-2tW-$_zl@LqLWb#?o}0V)M4A?_ukh~a}OB=q5)
zlz>6y9~#e4Gd*5Mv#**7VOL&8VMsofkTgOa<UK<A`KZ#NI*m(UFsYE+L~7)N_^Mi&
z>Mbhq`P6IuE?RM+Vr)GrfSPSA2#bFv?pmuT0bK*#OW&2tmo?u_tDh$LY0jnLmG`Sg
znu<8hZ<u=4R`@2YU30>ABRKIyG;nD-PY9ocmZ>4}vcg!b>KY7uVhlVR6xHvLbv@1-
znKj>0(zQ+vIgd1LJD;bdiL>5Ssh`*n2w>cm$Yk|1FHbPbPN~u1_4gOF6WBiK<;?c|
zVUlXyu=Y1w^xql>Cf-M&SH~{jIGcgA6@1b6;+6pPQ$~xNJ8$_hN!09H?yTu-Rfs6=
zr@aGANiJ66$H;{w-K+NMOSwFU-p7^)z%8(}?yGP5qbVYYQ6u_Gwir5vz7yTUz5(Xm
zVQ}Z<f{`+;N9g^p#W(gpnsV=tMP)w)h3J#!xfkAVq?xK=c*Oo{)t_<fIuH0KYEKdK
zXwK*6c%Yf$b!kG+%AX@3#3KGqoo~XastNBwhT>H`<9jfk_uSc}1@J!1TNM8A0r%9Y
zTG%|s2i>fab!^WpW8egAZij#W;$9Py<n%H;4ao+H770UQ|0jk1&}6+R#@RI4M9c=c
zzSlPW^>NwM30pC@@s3uLI44}{R7y`60UD4x?Lx<JRI+I5-5YD+H=Q5TGAQ5eO?V-8
zz0P#vyf>w*71<*f{zSoEs;dcY1cfm_d01}AeF~rE?46nI%}nS`5qqu2&U`x(YhvW?
zj>BoPtbbQnzx3BL+b!t=*g{IPei^>w30=LStyY`r_(tbwygRWuYNnwmiYm1GA8Fjb
z!pMgdT8EQ{olJg*?n2q9YkCxMtAGCV53ul1p3V$N0oJ|Ypnx!rk8|k7&P5yb|AXN@
z{C$n!DQf0>d2Yb^>?rz^q^2n@-WvZH<tMp9rT77X6~Qt3Pg2WMNbFdez)j*y!&-ed
zWx8L;fxf!i$|4q8v^3Whd5zqDC)_5Fd<b^tMTG$c>U>J6u|*MHtp%9dSo1sf*H)<7
z?(X8E;-f@dkOp}rI$Uy`s0-D2$BuA*B3!@~t!#g6w69iH6e+?jVqa^9RAD!au5t1M
zkqT`AuJK%7b3YqZZ3Um(*|x=Pk<Fhz^Bt6Uzq5@zxBbEP8;9(ZI9kW>YeIZ%F@eY&
zKr|5^_+K-N4>7WDa&6<IXMht-JTg#y;h$sc-_r}o6>h^MNFmJNchH@W<t4fK-_I9#
zI+Gya=~xt=&K>(N&^w{51w85$1{`6<M;K!ueE&MfxV~cc_JH{shCm#jBPl931tHt8
zILWNuc^jM=AQJve)-vx4&~jx?=}TgSUc~un?$L>0wo=zgDVg@m1M}WI)ApQiX4s8G
z?8GGXRL2M6q63n-SNWOYs{F_|3VH|0@17=9;?9qe*sf=^AVtq5s-El6%@@ZxZs>4#
zC;`s^%Q4)W&H3Fl-94t2H|umnqTlp_Z^gIy7Y3&T?Ob0PLbMpiZsr-rkfQ$ksSqJM
zt~J2s>S_RNrX4E~KL7tdmLN=)>yhCUcCYmdRiC5etK@&ds)uiejNug^5C;JFIGyj4
zBznm0NRVpeY>cchUP%K89`dcWI^AnBz}-5jcWbLhOM`dSiQY@J`;#%;uRy%EVk+&+
zP1113^{{{^B2C@3R#n7Y%R$D&wd($jPf3`nH@J>-xo=+B#HuE?x`c}|7{()bQlXa-
zfh2k^44x>+Mtfm}%ph_mm8{8aybGfhJ12_Y-e!z1{c#=F$A~B}xhsd7hE=6;2_uSP
z9%&eemxrPDps~_u#sm@5A)s+zP{oFeA!91<QDS}`PyMg|c0hYrXN_5pk{9Rc*Q05@
zxtFv5{V_azGIf}+#lfoCC9q5=4i+%%sU?9aH!5^q=Izj&o*Mk$@d!9NC;t{!UoAXt
zV0csLh$&g-2I)2O&qwsh=dvluo}ia3>~!_zs;y^5xB*d;jO25n>hw;bXUB205Q%N&
zspkv*?zU_Bsuz{`Cy2CIm<G2`zs0Q<cN<UE$?rt`d4i8N3zdHRFYn7O#JJzQR5x7+
z>jI;w2YsL61qWko2WvDTRjANhBcOSr7wbWHk3|^s?PS6p`P1jqS2W*Qg}Gg&(F`MM
z?#8up&T2mqspgy@68wNuX;+@+V4@&zxB(okwFV)<dt~9O87=uXiO3d0$n~G;@2~5~
z47ER&hRXfg^?N42*?6I>4>?v|>pv^te}+V7`TXYzujb!YyCKZNIw80kEM#0p2wJVB
z+5h$2fe#89lN==m7!vZ$4os=hzQl*gBaul`qYI*kx1bZ4^^y><20sI?NzP}FMHVPe
ztG+3akr>{FoW!-?Bmw`@>|s#NbhsWY0?G>8f&atUSI0%UXluiOgfu7(1EL}#C0#>_
zs7MIXF@#8WBSR}7F(^uxA|TQ*Gz>j-BPr6|F~m^ci*EPX=iYO_d;bxE;RmzcmCsu1
zS@3oBhgz6n$XQ>y(z)QKwuV(SOzi<oKnulr@lEk<DUvtp;A-Ft^5P8acUFyJ*hgyB
z7>-g*N4ko5^lKMTT!{C(Z&IG5W9%QH$<x{z^f|x3%X4vKt&NQ1#>8@aomwe}hGh2Y
z2NOlpXUvkT6<80do=r6r<^SSZGuxSM;rqop@h@rhX;F;Ylyo;Xhf*~G3?oSo2DNxV
zspbC^RxK?ydpAO&@<wm2LA8*?UKfAEj%6r37O=N}`Kqh9K=i6u_H~oc6o%vE<E)*<
z5MgckqkSLoIZ!pT6&`CRkAIzuBB6MCC#U<MiP}~Zx=|!XC^XFx;`=O@YkW{-BhDgL
zGEuF7?<<(&Ae$YI+~rFQI*>RI-hZ?Mk5U+VBZ*Re`_<rFgS%|_=x&yvoAsys$Lf!n
zjaA)1Ag#cdNX+~38f}GT)~C!^kIjfwi)rx1LQa8QRkf_=LITVk9nq@+@0fj9`;6pV
zM?V{$SwvsNaUBn~k1$$9YctR%o|-Bm3%M>3Y)!L#+>{n}ZXl(`Cg{Jt$w7dC7|10)
z`CgodJ}9vnyFb;FtpcThc>YTeKu&)fq>RVm`*XC+5(sXgB6Y@1Z+L-BVt=yEo6P>7
z2`b>??*U>1cBJ-Zi*7Rml)QUd&*_HGM4J^<t3AlKZ#L7UDKmn084(<mBruN7+Z@O~
zxer=kwFl=jY}yxyiThq-kZ=WkQWrr;1UBq+ZUUo~3dp&?q0VJNac_hXsH|U{sKQ4N
zOq7@kh^uNpxAU`*c7Pscd#h4MJ?&Zr&Lc)$2|;C1Q6#E~bJ&hH-Ub#DyJX^r#y>v=
zClen>gFmi>wUJ_y?(NeO$uh%Sz9>kIYYgl6Eda-#c6ngq<Q-Pg5cH_=kD-2Lv75|u
zz@}9B5w$V}A<aJ}!sze92K!Or$Z>@f;2Q_H9RGCc{t|ALI7m4u+xyLEv1?9V*^?(W
zlK{m82;kRy)1~(>-wsIY`K6MLE_)bWlETUoSf4pyYcJ8XV%;DHe7mP3s4czVzF5a#
zsu1ZaaE`#`tK1M&KhVUs(`UI)`E48i+;iXcb}0@Y^6?UMLgEHb4=zl=ca>K7#s$Lc
zY%b927%-{#ZFSxj%vJ@WEjnxu0$)n|$?8NmAo23OR@>H8rP^2zzC2L@=p&h=2_AT}
ztitnGb5nrv2wyeS)7m!9RfX!CiRBo?O7rUHv}CcM=M`VxE{Yv1jEt*GjW3L3hQqm%
z+<?VPhW}Qp)KfenC`_heovdu&_J0)Ukbd@E3#w6?^)EyHP1_pFHr*V5s?O`dTkb1&
zqq~rnZKnEcuUBeoLWq$3PuJ#`SeI`Do~!%SY}8`&<xtp&R$647QWBO2_AF^gmUN;A
z-as%|I+@`9tbCec;{J<3F~_TY8O%TZH-hbhgj_f##P-d!B(Co846%Ph#KkTKB2j(-
zfkfg1$imccde+YSSX+S0R8wWH92utTTGJuJWT5BTpC;(pk-!+3$sF}4BL~<tcLhC@
z2Fw}8QgP=dh1mnh3+Oymhn1X)nY_b0vQ-LnR<aP&u3U%E6Mz*eJYHxb^?vYTFc&K0
zIGblK-S8}fCg%ywU0VX!cJCF%)K5DWPC%mB@b9rN?K%h`dnaO3h$Pmtt2E0I0<IVQ
z(`X%V#fbbLQoNuy%PLu%w2-<Sn&7G3SJN)F(nwdTmi%``#bW{ThgNxK=edUH@NMj7
zHWnT`Y>vh%sZ$(B&@@p<J!MI@l-Lm4sC2WOG|kEE?LKZjKD|{ql62gR%&xi0BczVJ
z1mMWY7V4m9IOCX8@y1_QFT+?Vq`Dnz`0hvv1b-c9l=9MfMK6UOeGWYi(LfWSCOF5i
zEz?GC^-7D9*Hc*}+>170Tz<<KK<bM1=27cnKh5-R`h84DZQS!1W#B}?Q7gOHDB_%h
zjXrEKAkyKB5~Qf$oG<Y~S-*^c9Y*ifE}1c21GN|<y)bTq<13(X)*VK}6RGp>CJBT#
zLC~oK12AntE9PRsQrAMQ7vyuck1j1m{+<B-tF>_XAwo8-_Dk%+f&Gw#*aYdGVVw_5
zyWGyePo%`8<?V;9b;k;+e;4B_ket%n&LD&is72+QsYI|Hu1@1TY6~M{Nx`xm2vlcb
z+8qM6po{Xn>`p&(+W7|rss`6<6`p0QCt}`q*2h^y+wNR@<O)Ppd{=^lB6)zDRBSEv
zzR6Mm@IRqrh1eQzAe5SZVNNWjDJc!Z(|FaZBw5JHOPhw(|DtcmsNK3g!J-T<EvHo`
zsEqlXdIitmL77GQ^uTQf&~u^X^c`dHz*P<aSn^z@OD#2X6>t+5UU~mS_@$zA8vHbR
zHuYLR?c){Bi|}~blsQw}ghX$mtdO98%Txc0W&X-LL~${9+cSHUEm8}A1yCTv_a*oS
zPw_}~gFmPzw?Mq6qOf?tZJZ2c#PT4fd?$5hkXup>H^py1BP<2jdBx09h2#J{){>5Q
z`sDG&XSw=R^l``DY5>b|uneAyq0SgglvVC7J9JjHPsQ~M!Fm6PM`2);leQjx&UhZl
z>We_{$s6ge3?WQ{<YOi`<~0f)&&pxpFu&hm>&vFWDpmmRipkvVe52-Kk!}Nv9%+9D
zRU^cVma#wbLOg?$pkAv!oR1w^G)shhi}jt*vRwu)4Sr^4916Y<d((mg>ULr8dvH_#
z->vJveU>)DHXRgi==&BH@Ry7<Sf-m3?n*mdS4fMdXd|~+e0X@HF9gtqz$3h&=-?ND
zU!NLE;O|hqz=5qkC%f=FN93q<#fXsXJ{I?Mq`Z>@KOMZy9WVfZlF<-bg7azKHoy}x
z!)#1rCC+kv0ZusZGaa9|rSeRx&+X4%oOQx6!$~G-wI=LcMuL1KKVsV1{MckeloQ4U
z9I+!HwjWjk4I=d1EZ7U2x?;G=(VtS$?JYqL17Bl()m(~RgGtA-;AoyitE_v;aZ0d-
z)Z$Z9y>mgH@p;Abq8zb_V9YiD+jcVoljRO?Eh_$*SpUOf{%D7E0YEUoU%UAM6ZDs$
z=)?V@E$NBBCOYG;Uh2e8CZ#SozMe=C3Ld5VnTQc<=bA+zxSll<cVoWgmb5~dg*x3&
ztg{!2o{q$K6sIs*shBo^X6rcGcYK;J%7o}sM2(GlmZMs*!Me50fC51Ga}uK{Q+t}t
z^r~2nxyWpn;LHZ1F}(K~dw5U_CLsN>$g*MJ<`sIgAfJQ~th5y5jfAvxp)uCi-Z%V8
zoK<zQ*FUM$Nn9MglN^I<ScY3QMPfRE4HGxy2Z~eYwz_q-heImObmJKXwn-LpaKL3B
z`0rQ%R385+ru|n>%;;|z=tpJl)t$W``Uma;__x2T+)SH{U{l4zmP-?beCvX@zV#?X
z@uUPTT$8^4iG&+#<I9bTW*aN20%<d#Qzyh#cI_mTT}E7J8(=d&HvtIGCWqA@HtR(B
zcI=nBt*~DpSP?nOp2xvZnH0}&z<Dmoagn~#bKrQW1*A+0jj+D=VU1EWJIC3XUCFQg
zMgp<fLPqqp1n4~xER$PASXE-42-ad1W)>`_cPjfC)@Ph5;+}pQ&<@QzYq)L9W~)2!
zHoPy?gg*VTqeb7q9KlTDWg<7{%@k>_o2{0<9*jV~j-KBAXZ`k%MEP%#c-;pXJWg79
zG9`JObUgZJ-h!>~<BOR5{YiSl0!Lg7Dd>%~vrQ5@z#}&j-};2n0rVj_wcbn0`!XjC
z6_;kdA8~}eFOLt~Wei!r3s)kQ_j@L3Sq2B<G#aQt-eoD79e?o^;$aSQSqV#PWo<}Z
zcw%8Q^jJk~|Cw*8dJItH4>`7_tXe<e>oa^uqI6r_VLjb+0$a)J%Tw8C)&<pS%xBZb
z9H0od2lbB?>f2K~M!PXuiE1t{v^dUP-l+xWe-CT9xvL_B+|YM1?&LNb8%XL~lKv7*
zdQLGOB3&|fd2gN}NgC(oztv~|!Djl@DZa+msM*)0yp4wc1%#w+v8wX0X8!bnkr}5J
zvGvhjW`DK)v+EwYq4+PTIk&O3FW9e}`%B2i3wdCRSp+gjKVg!+%P<ctEKgA9C<?Xb
zz}3bw$?(m9HY|rvpv}aG(~I7pp7os?8YjMqc(}3=YB+>SsuT|ruH!c2k&(+h-_9mD
z*xdVc{_MH;Xv$l)^(z^VC>jE`o39Q$XaU{jVkOkDdUU1H*SB{;eC+VW;!p9z9<JzI
z^l{R9>4>4()>M2~vgnfViE3R^^|5VxaL+RoC7afb91WNg$B;rHh50=p=T9SRrVNNE
zkFT?z+irbT{wdElti^TOA#snA0sjkyGJp=otJmj3x7N%P)w$TaVowQtmXmnbhYsh>
zN*(Wi4J<(06tNzBUtvRg_LY;)9XS@dt*_gt4-5CMj`gr5^@J*jnB9~G`Af5q6X5<&
zAJW#yEn>0wf1DfO8$A{I{lc_03X9m~fk<C*qygEBdo1L%X1D-@6x^Ze7C3CL0p^3N
zj8%1b2JR6%Je$E`!JzkJ$`6-XMNUBv40`-eao3uB(;!I%CP@rjveuHVoiTjQvjz5J
zvpjV;;6+*3`mU6|RAraFp|fTAZP<gl6vN&+)M$(;&hX<O9FG3VDVjIla+T)ll&#*2
zYtl#cFKu7>ZqtwAeWRT6-dlExZ_P}7K{2%nlk!w|X-=P!|4q{J(1TnVMc8b2^!F!O
zf+-@dCfiH$>SfJ|fz<Iy@NN_QgsWjk?EfDrTuP{l*x&!XvF>qcu{#|x-{c>urK#Xz
zZZzx*YTjo03Z<j+0$z#;mX=8~#!t}OYhX*77Cw4(%cTN)e}U#}6Nw+E-`Bao2oaZY
z;7`3bhcPSd*J7vfaUc%K%HUCG_%YA1wq&;;CxL%HoK<RnX0v*$i;!b2T^wEvsVWV9
z=58>YCis)u*ZLD1>TrMjv^*uZd%I&Yw1l@|@MY>_r-~!B*7E%(d_D!0f+gtQcB3@S
z3q6&DA~UA@+F%3l5=q<9;El+^8wWd|AZm;3k)FIS!;dW}oEd`iatp|I7V9++LX$th
zHTL+ps4RNI{!a>GO-@U2oUy6C<=$;#`uT5d!l$r!_{Q>OPAp@HisWq!@O54XKueTh
z`1G4-XLu?O1Y7N3Mx;&9i1Fh6mm9tTe68OX=s&p4<tc_~M^hK^v^FLkrSr453y&A-
zMIWh^8wzRdgwL45)`vCSNrz;4_7W3^e?&5vlVC6}T({Mf-3oOi;!)zFnB|Z#*~1s6
zay(0ZJu{(gRyZ#8cS1SmGlkB>Kbj~n$#RhfC7?uU)`w9Axm%bvjbZl8bv>%;(h!;Q
z6z9?{!ZIPQ^99D8TV%*}i@<4o|CeE>1X(4s6w1<+l>f7#{4%0RKWX0waTt%<DH7{&
zQM^A!#pW-ad`*38WT|h;V^ZpGS&)eLb#=8idOM_oIlyDhwLV8!PM~S*Hhe%!dT2k_
zW|enw*ZDcRe`vr@=38f64smm_8Tx|f9bD|wuGA)1yDLIt9a|YM3|msVr*_!z#$IQt
zazG92sJy%&)<?1SNqx|#7BXiWN@d6Sb@3@Fdv^dXRD5$mR{P$KBgCjk$fxVHv&+>+
zOJN2^mYo>CfK;T8?GSqb7QGhoP6(@(&Xn#EL-hAI`4MGZ_AtF&ZAGXpV)#?3k0!$&
zE14Urf20jxyf2ACyzj;T>PC#Z*!pPbG9j8<<+?wl&vaXYCI6TkZ4LqPhi$dPoWo8u
zT2K@7ni&xC2CFJLaHfFGzP(lMc={(&g245d*z;=xyiWl{fF!5^wFwh2VxM-&b$TSH
z`bS6{-9U_nf8=S<j#p<AuUv^eE*(rDIuAhaxGIM`M+JKAGA4;53FbySL~O%jj}^tw
zL)O%UwFE`teD4ep@{v(F%|0zd5JiR6$6g`Iudc)@#2QZ+p&=G8n$`j(R+AX*n0Gm1
z7VF<A8xV5o*skA}Gh9DX1<SDaRPGYoT8jT}mAp_@L}3lX4D*b9=@GHMf$!O$4AtZ;
zf+TC~`uB!EO%r7&5cuB}$dt|>8N$7woh_`V;yc%E(n$SB<8lcI_7!|MS`}-N2T0T^
zvY!m_+5VSe@(3F2@|O0VNx69MT~8EM5fC9~=U0PuS|oZqi#3s6AIAk3TuFs`^!$?`
zyJ(v=OczzihUutdi8T$6bv6#iY1oWIujmCc?n-Ib)exiTO5Ru->7V19_M{KZC~AYN
z(F@Ov)O%N_n?ZqLo1v7rP9KT*vfMEDKn<vQY+NQj&A!4)3zzj<%ZprlaYPOx<g@#N
z&>RTgd-IT%)YA|*8Fu_C^!GePVp$k!yBvu$^RXdtz2+pUwHDSpFCio;)wr_m5ftqp
z&+zBH{twZ}l)@kRhzAf(?3#rc4@~w_0>9j_CN<!L>pz%g7`A2>6jj;p<M>Ob`QlpP
zCg_NoJcCTj)#Lcl)U!%~n6cbWq7=w0utQ)jv)jV>-q^SdI~k*~=F-IHsDZmZVF^8f
ze3+rE1r`(rG2Clq`V_jx|AxeOwJ!lj+~kZh*o)bjJZD*M3T;tRU_ZeXD#p5KJPW!_
zz@3SUl^DjWIo&M-9S9+6iIpIjepOsLgD-xyC_fL07!Pq~9%k2sw%-fasoo#ym6*SU
zm#T&s)SdD+%iX&XY~BA5;w?G)`sE%?&GZ#>;PSgc`0GD!?~fb#+qeD+*u1lVPS|C`
zM^2trAU=A*x_l<_UvXNtmTN>7;{W-n<cDbnAfj`091?@(TZEQ>KS2u~2pb;4eOL<b
zvuy*%Wi!wtP$>h~?-~+~mYNWcAgUz1%@b!hI63CzV)+&si#vmO;Uh!iB<<rX%j2t}
zOq`xqhi|{Bn*SKw*D>XpfG@SmHeB|R)1ODES;WOUF7Q}&%0u@w!Z;}jF&m_!lM2BP
zAge^TJmj_sq-N(|#ubes6s6GH{S<q=X39-0UWF{<9)4{e`;bZcJ~Fp}$Mh&JEG96i
zd*8<H=siwDEaQHsZpx@lelzB1*&5dB;1Z5m?zzHLguY^Q=9ITQ>S>%81MfZ<HHDXK
z0Zab5Z~k|=6$5fpzLZ;683LUZJC~1$?=G?cwC1;D?H;Dl`CrkSG(6B-nN^K6%{fGu
zY}o790k8PT<PV(fb3mEAgitTarC=W(&Cs=JNSlWv;o6M~7rB}>Si0VH{M6vH4%*Xe
zt(KwbrI4DSVn{Q+C?UpNJv|e-OGm5zwcvwSe5>PbRA7@Qzwr8{rX`Qk2^Xxj;%CEG
zM)<|rd>@Y!8o%@Ni;E_#B0!1YvgMFStz!ic&{wet>fP}q-%X4i_6q-gG9T05EMg+z
z7Nk3mvLhLJr$pkr_y#_!-zl<W#=Kf09=Ug8m~luUD@J>Cwbx?>N?~!?#wghWvPX$)
z19=3w<v+c?4_(e!+8M}piT`oR|Fp!kPV(*Q<!?OVrJXOC2T1_wK9is3<JLHcX|4D%
zRwxBA6!e_u<95Fv877vT0grxJ%+;!Cics9U^}O|48RJ&y7Hg+!+7pZi^WxB;Le4&O
zM(nrjtXm3?SPD&83y+H7T0~yk4ftBQuUo7DrA>prCn6VQtNKu;#=~m+$Ij0wzDhP)
zDo6TB<A}a_b?SZefeg#u4+(yBoR&jY<|UPQ7rK76O1vWu&NsKfGmQhq7Uyzg9L`*b
z3RXQpw>N#%o~qQX>k4Lb9FmPotV|B}?1EybOrcV9E9rW5n+oeuEhchGFPLw)w__1t
z-(laRJ$DTV!Sa|5k?RVSas<s)9Ph{nhs(l%OT}{ftfYxM^S4rWGMMzndMEE{`s!Yp
za^f1kAtlRf+Z6#ZMZG93dblfQzqZESj`GTRIi<w!znH|*hh=>DmWC@(da!nh_vK}r
z`=fDxvdMDRY4?7XK9`C3uwy(>dbo7pok4#1TZzE&!X?F6{o1Yz0rskS#grr27=Kmu
z_b;yQcA3H#6sx{_tSHQ;96j9pwppC;d^1}`UFpkx4Z&9k>{GpFl6rA{kbhB7{p=K2
zOUWEyp*6MC<%<P==Q0cE?43bEgmp=tO4i5PO`5aIFEU)J_@jdwLGa$*jHIs!5u^8L
z5S5>SA(k6%BEC1so32Ua2JzhTXx~mcoOTXhcTbIs$rrUPR}K+tjDzg5?z8<A*?%8x
z%!Hy?OQfkr(`9W*EYB%O+Aw4K%u|MWT01K#>|smMN*iHB_EJtdLF}#i3!vniKSvGj
zoK;D|ys`Ak-j~^YJKc!JXY%XI3su@K9>>ZKZ;CXMONu@2h&JfWBXc8Nn@H%TVc_C3
zV}y(&1{27NNvw^S(HIPGnHNN84A@)mMFR6WCG>!rd-UHY5SaABGyVBrB@S4;3mD`>
zV-~g&_3J6T!SUZMdXk}>=T#R^l3F!&v@<sd(2flN*ZC68JA6cjDMWGn?PUEf*xCTX
zpDN37z7gQ(ECQtY)j9_<6bGBgq>`w443n5)2=v*^%c60cwOe{)Yd*gM_AtRU+=Rg5
zk#+AvoQM}RZl7lir9e_LHwa+oa!oYS^O8rYj8aY?YOqbw8S+nF!86#wVDc{iNlXU(
z^vbGkr=w~6mIU44NEJpL?A^cM8>Pj7Tx`l>%o#CH5;>()4{tJjQVWel=fo1Nk5s-6
z6xF}S?)UgR`?^S8m2{LY1J@4Y{0*`I9(HU7NYb5p&G6Qj5`4?yqITW)W9Kk04^i#S
zZ?scSkEjY#z5`x|Hb|tXKGlPd!}$|i{&&bue#<choMh_p@u<X>w6o^Abo9Grlbz3I
z<Q7#&H&H`>JJ4T({H5H>_i<0T>2VXd#T|i~DJ_gV>eOUyXbraR%5XM-XZ$ikaQIP9
zen;jH)_*;iLVz}W|8kiTU~F?+*;Qk2{&1!yT>O`dHZr=Dq1IxE&T*8q9B6Z|^SYlo
z3%-ogRc|cX!d;>cN89F^*0{M*^7SZ}ZF$D5+p=INDo=$N9pT75Hbib3!XAru?Rz+-
zkLeRL9ClZ3PHTeK$Lc!WD%K^O%ZEZoG#7bXTfr**hH{^kLp&{#$+?JLu#QS<i-<iS
zcL_pPVHwHUbSr?jEm?Cu*l?|WM8mNAW_vwKFd0nH=BjYkVIEla77?MZsl|?a#Qe=j
zISE?zkrX8j;=E&8{!~wG9PvJzcZ34wxc;}<`den@I{|~=c^sbocgX&iDEiC@jtm3{
z1Pt6}O{)B*V_8(+L!TtCjGVYK%w_b>w?_{qSm%#v9|My!-=I@_oTtBa7!bXYnXD1e
zNwb`74y`4~xMb$ALdPSO-xk^HnT&L4Mh0KHpEEWffAkIl$UA{;z+f#G(jd*BAeaa~
zfj3r&z%GtOL<@LRnl84QWHZQgxSotjJntY^(}dZozA`NXc}jGej4!rVh{x3^;z@4u
zeI>_~XN@Y5SD6{JD>bbukdxXSQOZkZBGDFTcZLz$>#GNDv|XpxP*$66%S)zTek)x;
zOSAO2o-8rV6V&0|Q@g1^JUz5@WqI_S7wjaGkIcr58`V*&19)d*cIs-+;_){5q&;3R
zlJOB;s|XU4NS4}uJSqUBkGUQ}>JLAMwS>jUq^-<G)2jcS8UD+`{)%E2Ibj<HWl3>%
zfYpcd;XVO9<(%XYW*8AN>-d1cte!^lW*qM=X$cbqim*BL|5d-`<`QgI2)VrseDM}H
z^rac>R&!M=J?M)6;<QA$-d@oiume8%<_R06{4}xAASoGE5j^$OU(KD-_c2fNJF-MH
z;^X-!ZENh0@Q-snp_)>zr{66pLb;d6!v|4Gw9c^b=&+_^`W$ecXJ33ex*zSnIrcna
zFh&St8`eHSdJojr&;W`T36tr{V`A`Hv7VBjXf40C0a6FONfl*H*apeoOirW9S`O>3
zK^#rAjvTM4gpuX3<4v;Ve6Qf~f)$C|yXwx8FU@#&db=`#&yI5{bR-%H399vKEa-3(
zHX(4nlYg1(-yx}r5W6l6&p$J<N4HCm9GE`8S;sq}wpCD2zy-I~1$sp}@&LWlndOYE
zTRj=!-+>I9!tcc@x9#NtU6a_dsv-9nK;JOJB_HQLU269fV7m0oGcJk{>LnaQ9L69F
zYrkBNA0z&m)1$y@tY=rMkxthOvlGb}WDPv7WK8nul6jMhGGkQ3v{CU4Zm`lsIjF7;
zVptd|q+yB(adVzkNitDJB`3}`)0@{83zk=oTX79&s&>aSF5G1C>^n5t^$P);rg*zq
zHU#tOVnBgBj*AXSH282O%56ZCeXOV9eQJY_RvrQEramb4h4kDlvEoWTDU;Bqsc2cH
zljVHQoR4v>d@2q)sh+A+p6?SynoC98PYd(j!7XLd+V8kj{t7JyJ~vD+>-ztiKe<o<
zp@jsbM3an&vtsGHMexYaxGLi(6g)L>0kE)|%+ef?L5_?{PoBo!lggy~BmVF^AON1Q
z697)OKh8_henx3JWV;q^YgTX~F9rOH7=<*JzCxVtF7JBVsMGh~-|}FC{a&5z5_#xH
zOJI58M=pKp&_cd;JDq2&=Saglp1;M5!UoIOJ=Dy>ET-M!DH>``tr<nPTiu#>{TVK*
zf=8E*H@ADUFHa?mTz!!p?pzMpYlw`Jb2JEd48cFng%|jk-8(1?`XYDT`2&%Y@dD^;
zI**$R8A>}O>~jGbmmBIkc7j)k=*FS;tMglU^_|)eyOeSrM~8M~x&Q|2)t>LP7y`b<
zEcEMuNVuq_Xt=XPB0XgBK)xvsml<$`Q;w}nqjFW(uz7R=l~YtPditLdQSJxzTb}^~
zZ;Yl<C&fjHfNl|He%z#FEGP$N0d#e(^Gd%Z`a5<%MSQ-@QGY#)1U9S{m&PDM4!2L1
zkt*6a{_kXeOD^x&;|_Qmds?Xe%Cz(}cRfy&$LC1<6$NMt4%ZEpfMJQ|Wrgqnm3S(!
zbK|o#NYM}v%(<M>bVYkA0qqq@WtxOssA5Wo2+}!ocLhQqA;{&$w3dQgt;kru(Fy9X
zawctD5klw6I~xaz(^npVX+kGkaA;w$JE1-Pqq96mqkV}jIykAtKAV~>(fHsNiP0n~
zu^q;YN$vGjGj12sB@WhyVBOg`8d+xVok8}TkIJcz%;(jXKm8ZGSOB=CN*gd<8l3-Q
z&?O@O8f~0ZeDD&6>TqP9Y-USh_L%lZ0Fx7P{}L6*|29D(4vBbwSE`hhV?p*9_x%i|
z$+D+oe_}-*#U0>c?~8&lAnZ25d@HcXHERIYMmb}~Pc4R&x<d=(v<j(wK{fLVv|09y
zqIo>v2C?}&^5UY5>RDSNRf20`kkKT3SkUpaDGAJ2IhQrL)?o=W|6R?zFur3TC8djO
z9ODfS<uW^SJ_4e?^7|4$4}<u4Xj((RD?{cAt2d(NQgAT~guJ&CIp5WrFokzToR0Sa
zn2V3Dbzed-b^i!v)FAkH*;E*g?!X*5u?)+w^Y|bnIxNEg@CDeBq6_7w{-s}GDke|=
zMxXvz9H24Wa=-Pfn2H9da;mVfLwU=TD+?f|k%Rel=NL;MnyvtJCQsmyRY`3Af1(w)
zK$koV>RHN40Fj(S)!D&BptO1bfCcjgTsl0rwD^zkV|eV+IkK_V*MryryKm;|BbRqm
z^`;-^sUKba1RalH4VJ)>646shT^}Gv<;vajL^qMb{l&H^$Zd{zRp#1eeptS{PX>b(
zvNf}fX0vbkJ31R>`ILw)3IjdK;HX|#iyXZ1v)0fu1}zN#<J-pE><`ug;tPq~Ye=uU
zorQO2qQV{)hYRVg5m6U)wJ@3)ZXi%P(ca_X{dl88S@PR#!JdU<TEh5T><QEjWS_Wi
zkY?EB9yj!0MniPCz=^iNmVkMwSEeyD*$N2vSLEOp(cx?P?;a0Vll*tacYQtxbsyx<
z7vOi3HdB2ShDMjPxCfF5eVv^P4osu?8%o`ez^VptA?KBW49lJ*(KAYmwnb(BGR?1t
zr3#mb(61cD@MEogd$1lyBi~ml$j#8c4z{!fKQfl`8#yGA0itdAP=CmI*|<e#LO4U>
z_b0T$&PFgRF)~-ZC3$N*lRQ26pn~sNTX3Hmw3xU|Eku~z4Shcq-RU^DZ=P%G_}&(u
z8vALGJlirs^j#X33gs%Njtkn(3R48qXq%1fm^G>|rgE79n1bO5??0R$jBQg}WQjzw
z461>lS5og&SDz1$Jme1YdhZj?<vc4#>J%-PJS>_>1a?{pOI8rErNm7LmyHekRroEi
z-_Hd9L+Snqkmj$#um5paHXg{pPx^6yWkyLjMtjU$f<PdR{BLm)6Jrw!oJuB3zH+-h
z6i%Q%K4jbZq}yh}V;OLHeT+cq*2I=%*WEhnT`xZU)2H$ku>^?lCW1idbiXxHiPWEO
z`|5tf!&K@|w}N3VyVG;|_qR!CmsrY?qiEsaA3+jBoGAU~_Y6|ajEgRIh^}}*8^9G*
zh+)$o$0j^$=@Y^z2$xCfjto&W-M6T~Ktji~SfdYAF8vFkC{0u>JHX@wl|bEpXg4eR
z)pm+;d{5QXaDL~-G@PfB9203g7}-s8<D8%}FE}!;S5%D5d8{1I8TC>EhO(g8Gcx@-
zFO&6Jy2Q~ZXmD&tF+BVM&2Oo-u=8d9r*b*=FR6`%hOOX%Od0*(6AV5vCq(m|Abuc-
zK;5)b{d~25iJ0%T^l>>qfq?1iQ1c7T`@cI@3CU#4GdCTBUY$8%SyWE-(pJ?3<_d`r
zE{C^M)I*t+L8fNR9#_Jh+a&AZNK>7>2g*1PXmNd=SclV6&~!_@!g;ebUyUW#sO-6P
zmlVUK))c}8X?B*)h=nj^I(b1z>!FLCPYr_CHiM#q+X+E1Y?eBzk^Be<e%Q=5pZrm*
zEcc#u>+PZ-zE(xQ!gr}Lr`@Xzts=)35lzr%<t_PALdv;oT3KdY7{t{od`HI3V$-Bj
zi|?T_OBBld_x%^c{|*K~g#`)aL;q2ye<UJR#J@ASPG$Y4!sN7bz~*0^)3@=`k1&@L
z#DY?N#U_WzpHvqY1bF!iHy{WLD)Uhi5DNoo?i&S>?ygG|9x3(U`WCuLUb;gw-%sL7
z=&fx~VL!VefGzH$n-mgVw-@|+jl4BErF7TyCitvY0g~x)$|hP^Gvg@v(vyhyz5b~I
zk*Pw?el(tR&Pfuhh!AwCsoau-?dUi)5rS#-W_|m2e~yi(F^+3j@2~9RT*GQC$Bh)h
zTD<a-)TGN_`RQYbW4J_slA?qVO3`;VyMY#eR^JkKlWM2OpBA;u06oZ)XKF2CHG`df
zg=<5hPRbqlJlal0INh1?(#|iv-*r85Df*qQc~hFM{>l(s_&~tA{iDm%IKX{fW$${h
za1Wf>pyq4bWg`z4#EAuxsP+M-VJ(4c(3Svlvhq%$=5{gY9(N4yZ2%UtAh$p@uY^1X
zUP%nPMUkoh!Oix-@;jXzk39(GszMPa^d}><hyc`29YpeTRFUIkwJh)S$*W%$`%A=3
z7J$Vzdbeq%z2>a<A9b4YeGEAzBT|qCDQYt8IN`SilB*CS^@kK%Q6g19HIaqx6HdHq
zhl}ohHrxvL?s`@P@o8=|D9qe5{vp;p$gWwPcS`0hN<QKj33eq%F%52FXqz7TKAOGx
z=KUAkK|(P;RNNhWkjT9!QD5OaT*$~uhxU7m-P{rKbMR#SVZNUU;jH`FGO5uo4W0X(
zL;&cjD=y6Oi$0OhJf*@?I+<w*dWBzM10^cl+9oHt%GNyCs=@^7z75>3KGSjk>1nPh
zS9Se&HmBEX^cSqSDeqS&1FXO*A9Az&2EXc`1l0qa53#f2O;21J6^ybkbLDOVKc-tC
z|BnRN?Yw2&wHn}R)3P6JKrb>>AMtmFeL?WSR~-LI8Y`zGkP(8Xwh3_Y-bzz}x@Q?V
zMXxY}Ku~OMtQxGKR~O5>gncu(4%6(cSE}`KbxA#OFh*=WKb9jc(~*L7TD?|*JkOl&
zg}C>a!Dh)tsTW(`fYl;(z$v&zJ=djNY>ws>@}2BPJClxd!BCU!hKET&lMMQZ_c=i5
zKq0}7dO*P*wi#q#K^F*4TQOzUb;a(?#N%BKscbXAAE}H>grGwleZn6nv|C*f5c1CJ
zJ?*>hGM6lV-uw}Hw37H8rJrq1va1b-E?K}(?d?fg-3kDxBD3rUfGYaLTL=7fd%tSg
zv{&RtvGKsF5N}=?y-HA5YT8~t9^<;(wES98O~QArV1*<wi+HL++i&o@(BwwF{L`NW
zV;tc)Iu#Damnlik;o5cQ??vC<{Z6Cl9uK8!vJ}p;O&=2KY5n*HpX8g^y=}BE%@**#
zc!SCJaQWjXKajist1|*IS?Df4Oay1trhkTTxu5{z`E^Ex_!Q)ckLrt+lVJ={;>(i2
zYqeRB<lWS5YkG$2bI%{{%={L^c-6E`>0Usjl!>A1crgUIPvQJYTx%)Nwr{o>FPQ~j
zjKWz|XJh|*gNZSQFLU#As76+X$~~W?2N*QqPrPY4Pev(}Np}`0jZoX}#Nw`<$MkdF
z;Tc@k-^5dbbAF88>Wm_42(Ah@=3KMpPvx7PWlQoQhu-9OGPq}mAO<S3)aJiz+~7?;
za7#}AojFU_J)Pp4l9)SSnJR?<=|Rb*yD#YhCy*>-bb7R7Iq>E2^-p+|-9IXw#iK=v
zbBBIIGo0v77HTko%Qb95-kV%#uR2D#a{Dl;AYY8*B$WU#U*XSMcw#LTlX3vs9NF=8
zuoOEcnIOJb*jYRL(`e6S8ec^*dPKiKw|&9fSLkfV9J;lu(E@a^4#;V*`oDS79ILh!
zYx=8tw!Z9W#g746-gLYzMMpEcaPvj!K;<DBpS(YVpOM0H4pmssCxb|TQ$*ya=){;N
z%wk}UpB?Pv$Rz6)Fyn4AHH>LCQ4eP=Pa;^26%22Ptb?uT4pv(50Epb&9Hw|2sf{>m
zGo#>+3;jUkJj?w7d6QiFj_aZ^GtDMpmmCR8C@Xv~^R|#l|Au+#j5*9^ImBmv=1Y7a
zvQpB2zk4eQZOk3zw46SXC11!8NW#Q`j-Vnsv)E|~u37~0_PgRzbg9&E2T<hZ?@YY9
z*R%`r%R$=7_|=I^lo9TB0Z0}-O-{Ahc^43?Ctb=+@-Oom4%`aAN?<2f^Jaoal6sb^
zLFL{rT#WN@jYSg>KBfPAEcMl)J14G_NUn34H0;+=?;aR+crj!=jvmQ<0y$V_TlqO*
z$JoBhl#LD;@x5<bK4L6s-3A;A1w+!=JX@V8+xYQeM_mYv*xF&jVn!C2=4OIn$OChE
z1*0@vdWSphe5+-%&$q1MmmVGS5GW%VlO)ZT0u@9Y;owiNWBux{_~Edq0DD9M29Y2j
zwl?$I-Gh-<lFz><Vzq27l6c-SBWK_oH&JtB99=H$b8^(D&=zLLY?G1dT!a+NgG-s3
z5m%D$s-@DpoB$SrN(f~oYxKz0LN%Pq&3t-GBn-!jn~=x$o7kIP`y8rZuFv4ZJ<5mi
zlZT+t<~7$&5RAh-dabz)j9tO0@MoV>Oo0g!h$-V}mt`t-B9NhKNm=t6VwUVm4NqzL
z$+-6PZh;fcK;7~Gqq~;zQ}DaIf||fPA8$9|`S44iG#llmZ>aBhgC*WC{aNNEaF|Te
z$KI@oWq3mLch=uyOdk(8Z>C?*V_NNCtqP`h8Yqo(wX?GazlKdIxy)9O_u^c9(7x`l
zXdzkCr1K~==Vl16dZt~{u(o+1;zhrl<vGk<vsLc!=d+EDfvYJrb!W{8wB3nDQy4SQ
z9N6fdD0gBiIrtbMJ@{5)j!^gAV*wlL;>95j%gKo0$Lhj5?0MBQVwN*<VFSthTgvX;
zr|wl3J@~t81~;qHlSO>=y$(0UGqQ6ICG5OKUP>;Vd#-#@y!a5~#vp=&h11xH$6%v+
zR=crAq5S3U<x`LO_E#kWyF^PzDec@Kiwl7#r`zQ7qV@RFEYTKOuc`c$-hrpZy~bkB
z=QFk{l5ln{vOZ?XFa_xzb;zf+kQqmYv(9JO``Dmn>?Pbz5!qD1zB*HU>$4^>lt_oy
z{!WH`czVpvC|VAT(0<WJ=)@wbjTI)_w~?b4?6U&jY%Ctv7Vx=>6{V+o1wYYelvW{^
zP&-8Mdqny|8nUQ|3i>696Ec5yZB35!5f>_UuOV2BV5h5-x*FN+^`exrHM9s)G&<7O
ziePdZo#`6w5cu!FGXPE%?h*UCGkeazLwNAbA8vjCaOow_Fr7=}&YXPOggEFCQmT+j
z=@ARB_V1_NaD&znFi%y5w43#afzWLDSa}OD=a5!d!|1VZX*oWaqDS(>x3zm^=&;Vn
zGt0)av0}$E(HtBVbI<$Eq)WHF5=`^jT(gFjir&U<F=As6`cBAam)mp4@bgw}<MH9t
zN4pwKqf66usmJ9PYh<18ECIL7P*C)94K6WvS`Dn${6wvN=}xH%Rad#Fm_;F`8(v31
zI##Wfl&hHvO~=jZ%)e-8EeQ{Io3%Fv4@Qjqc(0V;LimRZz~{KOb6~Rk=im=ZVx887
zi}Kd3@>k=p*1F%oG^J`cGhNnS$L2ZQq$l8!NuxW6Rk;ROzWk--zfbl&KM3#!z@u|y
zo|lkD4Ymg!<LEI<IvAJcm^9opR@^UgIfr}1PurVf_eKWK@%F8{${l`RP0oM_#;Ycx
zw5`2Kh|zX4Z_QUbQcs|+&d;WPpkg8U;6%P<#^Ek2IejT)AoY7<jjkD!C)1228Pszr
z*GwcRjw-bGi{Z&QQTr5MY^;t}14)v9Jkpt>2@qP+8*a*+jPZW2y`2H6>;eYe_CLwp
zn?qck%Y`Od7n5?~J1)pbnw0L8G_Cwc+|mUb#9&|JsB2sL{hNZY>)IxNJtI(5coel<
z1KK)9qvQ@W39pH0GAo@csi}a{fb>^Y@zUW){yC5---Am(_Or~O(g2s(hxJ+jmoc(;
zn*!A?4}2{4ra}`20Llsa>PAtAt=dD<6Yf+T);^ocPQ}3zvSVr8SAlfjht3aj_`fAv
zG3yMRJgnK<9CmB`@_nF$)W_FroXGGaOFx3`u<Kj_gU$vkITzG@rU00S@UEE}(%tby
zy>=lAec1{tQ#`3#l2;aoEig?fmQRiD?faLeh}@D~x-Oo@1U=msaHK2`#k*ZoaJ~>b
zk_Clb6uV3@krCgsxK<PC*<Uh<Jz*<o@{#9gB4Wi<s?Mf)HR8c1O<a1b5%=iJjJ>Dy
z=LoYc@Lu;LevkLXA5Y`Z)?j(y5gZ;bJnrjoPitxAb0OVm9_zJY1J4o6MZF|HU3FdN
z3}-_OMW-A2z*b%@%SpDDj9dSxoC~Z&jz_SuSE1*hjIwJMAt8$|elRXKYX`@%hEg#-
zp=5u`&EDO-9?ejzf3M7GxA`4@f2_R8(jXRl(r`@~fNvp&%k4>=t+E7zy>!6I`ixY2
z#%H;jt|4ZUi=_bOzLG8lyOJCzB88qdhi!D>Bvuyg<@qUyBvZg0I{rf;L<j<|x_HtH
z+v(4O96TMX6f>vdUXokTs|H;{51gzZ{k9|tPbb|f7t1Pu%2Z1h*Q7mJ_BUd7?en|K
z>B){Q+B_H4Z<kKM906QT5k@Y{ciI{8$JSk(ZLN%37s%VT0YnEeM?M-I$4jYojQyF6
zTR{+VP<I1ZlE=2<dt`0Lr29$t%ID~*_|2+?g4*@T@v-H0@vK@nMql^U*@~QQV;y%L
z%NeG+^(ONe&<13c5jAP(ZsT7^2v`aVt#7^a0$U45S?E$&2*FlSD}vxxo~hzJ63G3S
z?sWJ)B^sZF_bzCaS9P(a#I!U1skJNDEzdY4A5`>Stga%CDD24GG>1FkZ^Tx|B;hZs
z-5WmdH%9yRje*jWBDO-i-O`M8UGF~yZu_{>p^*od10!kn0FcoAY0UX0WE3yu<n*e&
z!!9Xvk5~;>>zcjK_!^gw!xeH*ynWGq4dJ>ZT&p>pEv#Bb0GTA<@+lk5wIs0#x6Xpw
zO^Gdjia{^<@{*mlo5Ec72I!%S-HRQ;J8JtyT&;MyBhjQHg`sR=FYhyQzm+f1)}?Nx
z^~soClc41}i(m&1b9=0PME7UoA7^w0^LSpf7H3~m9CrLU%$|kpiPv8q&-35<;;owM
zy-(ppZ_QY=+&pUzn+tK|M(we*DVgH`Cwz*ArEnoePqQTV!mF4l^QZM&pwPOa-~z+~
zqgxN^4;wDM#Vvi<y-M}pd2q6<!0sjPe(47S)nHvnF34pbwqRC>OMjmWw*SuMSTt2}
z<aC}hDQF?uJ=Cl<ww6P3sV{wx#F(e`@abv40<$vzh0EiFlir*Gxx*hZ%vFbx0>&S|
z4<x7yKeo=RvmC!KwP=}4A`G?rc-VDpF1B?v@?{Bp-v28*m?O7B9xSFXY;;};H*O*_
zyl6*E8U}0dd#@K|taJwV(l(XY`dR_HLd1OJjyZm8uflWD7atL%mFzC7E1qiIn}uVV
z{9rI#x&)(}lOYEmlTy-ut!L0VpRaSFEPQ6EnSd@AT^X+HRZ`@adD$@}IFPWG_ii|K
zry0?S6n_ygrrQZBM){r)Aw8a9=2}eQkF+Q)QZ6zE2TyMS<lDJVq3(Gun+&91-{bkC
z^gbM;Ufll4Ql_!A<C`$nvD0_yWIZoIDPSnH2bimrrk|N^BC5gG#hl{Ls|K^x2MZj7
z5Jx8Lq)<;Xvs6dawt`LROs6rd0<WhPGu$p0A*!8P79L!udO*hxH-RRiv%~BZm6o0l
zvy+LEuP^651JBv+sTN@NBNXiQk~}8-<*`KLf}%6OFPbEM3Hwq&mgl3Bx7&9E^=S>x
zlS)m?Pihb)Vu>*+f2qk(Su_{890r~}EsgaF*MlZk4V#B!x~*p61tx|trgXC|N7s{~
zfH(ko6l%v!C;`a7BXi%laDskSTIDGXaqhp+l*vG8U{YX7zvbz#Nr7b||Nh&3j6g49
zWxI*YUaTb=VE8d`YQLq_+9*=?7rHX+89IgwnhgJ?S0zYH98AwU;8#AWN&p-ga-LjR
znJuVcr+R8PG9<X6?k06}12%8|s@r`h0Dth~k`L?ITrpTvynUZO*LAe=Wos<+M<L%u
zNbSSTI_H<VaiO#fcD3qeWFRFT57dLK>z<3jsorL~0vG4|sOWsluD!R;=fVcChQ}-N
z_L84+9tpUDy0?5B%)Aa{#4~oNYFs~THTLLzIN#E%d+b}YTmajhMn$}m_3CggSMGha
zkZ&2-ojO$i=5P$`b6~#0Q&)vFW*~Y(Zt=#4nx(n|r#r9w=Qz-GV{iNbbDeOy{R_yg
zrSNEC_!)ID=|1%nWxNWkrgOqB@6iCf%?z1W-?<fD4nR2J(?4SSv9!c21x)0)o=_^%
z4nh-O!~$i8NibAeQKB+*PPNyn0O=i6%ncc4kC77D?>rq{Ph^1Z_Rm6BF{5G+!?f$N
zM)nGv3w`3JFo;E$ANuo}t@yA<vhP1#rF!}KYV`dAsu!~$^?>kvo}HsU9w@=61a_2q
z*JDN72YM>@I?C(woNGjP-CT4@n=xITKl0)Fl<wQ|v~0r|GiDTp%~5<}0_vl4L$9aq
zl=iNkRE~x+fk5IC5Q8p@!1tTif3;&?^%{3c{8a?9041MOrb)^#oM%6x-h#^FlBFu+
zawt-X-ShHt_2nt||C4?wR^!q?x*sKzX;)=i@FHwUDvJAfv3~`X<1^d|i4h-kALZXT
zNh!#3oU4@YW8=R#&Drm)i45Mf-onpR@jPd>JRD(8yq!P3*G9WH770{JK%w%b-C9X$
z?_6?ZMQGfAUAn195^zg8RTq(Z^p2(mD`67OTeiZQ9z%ozT4&3OD}-|>_v@jVz?sX`
z*fs7mElhi8-J?CQwkp_aES1;MTkI(CB*QkP)U3&TWuSE+8!RS1>9mnYGGW09B%ByJ
zzjdH>@M%tDl=zYvZZ`Nf8;8WeF>pJ>A<E(*<v#Y%T>^o1X?{*HfZD+QHcBfFRZ^)T
zeo0s&ILzBuwvCBp%!&JzB}^dhMW5jL({Q75B+N;7iXJhfkPDA2C>>gHW^oc<7LT=>
z-tk>^vx*9qJI5D8&pcz<zrH@7wjkYR3|q9SOA_|64SRe!i;#4Wi-_{tMnx4lDqg_!
z9!OjWoARCwsGio1#+Dn_{#twLbexVKr8{iir914XjOs6k*h<m}1tldJsYJP<uA%Zw
z4WQSCKec4`wT_4yWl^VYX0@;-`K4F?p555oqb>Q3cQKF?fRu!jGU7~sd5$wSUNzhV
z0L9`l@Ht*snYuW4FL<2nF&k=$*<V??Jb_oW+rgPrRC_SaxdBk=g+d!^H*D?K7F5hH
z7sZ8tI5asO@P;%0>c)33CkK9IB*?WO_<$RyDQ2DjCu6bBEqXW(WGHd6asU-m^yNvu
z<#d~(7V4G9D!OGkIAv1jJ8(+?4gT76dxO$qGsV?FY;E-HR~GAq3Y&^6xa+Lr8Lxks
zSTbO*d}iV0u~To~#uLUYYCB+M{bd9;Q;b-pw-$bD=(|$6a)HD^3{^?=#5=qaZIVgP
zmpwy&xYFg|(zo%S)E+Vb)ol5brNv||<_c%cg6u<uG=GVbk$0)edYk+2_tRCtDVr55
zDz&q%R!c4RQbi-m_a(|glg(LrS;*feP*~7fKm0s{_E|kSe6+V!O%=<>1_P+bpAWi6
zLO7+~MocJfv{&rR4PbA<9LIfnvjjj(r+WwcX!q3NWq8@8+NJul^bs>PHN5obaZZVb
z95s+-UgrNiuoXodWq!xm=6%34%XjOx8;J&?=ex1kt(ySD^$K0D$3x8ImQO6?Y(<4k
zgqtxKd^t|2o$saySzHPv<J`sfU|Wjnjr#rsv1c5?*GwkrfZJg{aok!@i<&WKSsu0R
z))?<NCYJ+7(wv)6v=Xo1d(YL-0)U54h-*<if6mAJ2wK$kEhz=d5^kZjM5#pj3f@R<
zb*5@lppo#_O}1-J*1|yGIQfnbcR)+5v<$s*5|!U(MNzY3<MJK>n_uJ0yj&p{+i<o;
zmO8<9dnWH(6+;)3ZTT(*Vli|)(OjfiDW&!OH_A%h`}}Ww;!`JWw=<C9le}cF0RUdJ
z9Ut<m^I`||sDPqo;POUkg4kWfSnfRT`CWYL++Q6l{^8!|T|B?M+NMI9I&K2{e4bzV
zxT}|!0JNfV<(m#0X}ECYQt#3UFa~@1GEOiEN%j*U3TrAm5=QPstXy)E*Nmh%b#PqQ
zd1G?fq*zLFR{rj@pAJ?U(H|@^I46Q0HSXuT*XC4lV1RLt`&`<6&TNrT8RXf9y~=U5
zHRhnaF1ZJ|{9&BveAABXcNa+VN8{yeg)W;arIGH+gOS$tiCKPKpvr-p(E&?y;be7R
zbO#VwlENE%nY~%xM|A2&C_TOV<=N{vCp`yKUyZe@nFEKY#5XXzhXR?+J^WI~jST$p
zeu^uBG|UT#t?Ig%f(~aLpZIWjP{e$^Qu!+%!DntX-=ZxrH~Jo>7#}jdXb<o-koz28
z(J{XdzCb4<c7&=k#XR4X?#rhc*WB!vPN|$ODNuC2=$8YTd*?pl1a$`<N_;?KBGvbl
z6(v&(@cE&JvlpJHtd^*QM4`sIi+A8mhJ0jYKUnD&fq<NX6!1^0Lm}NM_vv&XS*8|1
z^oXNeatR(z<q;kM3|etMJ7G{_k%K&Yn6=l6B2*U!UV7!(HYx*@_D2Nl9%MhJpzp~%
zu(ye7mf%b?I3z$ee@YkI3`&)j(uE)#ELzDL-bAXzLk&Tmz{m_(lzi@MuYOlA?PqWv
zx+pq5DH^sm$Fn4Rj$fJ-eO$N#Y@RW7TFH%|7|DVC5bJMR%7MqNpukYUhlN+d+nw%=
zTk7C`CXw!=n4lD(=@!8nbnjnG``*jAz}TihtcLkM7mj|gYM<-2Hs}-w-11NhX$lVX
z?5AG~yO~JOLw$PvwA&iC6Q9(Yc^O)N=D~rBZk|qFEu4n$h7Dhh@!;!wu<QEV^8D$!
zq><aJJkOBNc$t@VJFqf({>sA$a37>m;U>`aCdIgkTLdG9@vJp1?JHlpVKxr)pAdod
zU&I^R*E#}lfN`+cRUpT&r<Df^fSJn&$`RYkGocJQo=0p}tz`lUTz1v*@i(p7yN{7^
zLbkf{2-d|o)5ymohOcbLfqAh(<6x$k3coM7p$~69tV36SU2@nrAG#pc)O%&+HLE$1
zRhnfRe_JW#{q0;aQOOz$AKm@>+LbBcbHPbiQ>o<&n|>@2AudV?q~fz1J(%)pQ%C82
za?NPv3;C$`SU~GH8EE~M9pU=pU5knRL~hYmwo1SDr8e^D`MAjv&EXSLiiHm6x~TK*
zx#+#)8A{A3g<G=mA)Wi%8Q6vR7R>OR)T`Hzz4Cl<BFi?#T~+;p%gyVVu4yufS44z7
zsC%5D>k+>1{13!<!kXZsE_*-XM&D37|LuzPuDL-$spGeke3P=D$4<WXE{v=!F^Kny
z%$ySl9Lap1DfK<u>Y<CCr&}C-E6u-GP%@ZYes6+ecglHFN$afQY%)-2-0I9_h%i|A
z$fSJIK>@QWUZ=A1EEe#=XV-B3UoO*<3dxaM;D!BoOQ*%w*VU0T|5?-<Yx8iAH7pQv
zgG?HUo2L)bJ=+$H85UW`&;{oy2e9J#$i(AFHG4Kdc}ZW5DU@@&b_IiHX?@HXO;H{8
z4y2<LODz()_RNl-R1Dy`nx5S<%FQAtlEX5m*0VA}2Umf`DDzJtVQKPDun?8xD1&fO
zEcpQfQ_T2%W#x%nP0`W#D@AYe(3ZfuX_q<71pTqg1Q_3*11b)m&6~jHm}+ln6hl!y
z%;w0r+^3QnSWTV2dJ|X~`xznLw~go2Cxf4nt)8`LMx5$`6N<W=-lRj0$h;u^J-)Rp
zbE%bQa3AAwhc$L|fJ?tz*x(k@7ag+ka4uqwrEg!1r2lgC5NH)n;RgdFH}B1X>*&wl
zK`Us(e!SYM{jQwK<JqqGO+lw-fy>@9EwVS>JS9Gk0q!>bB3kLaulr(8a^h&pxPk$0
zXeRc=*SkJ68+f4ZiXEzf#oh^F+p-DBaVA-;d;(EjX5G2@R6&S9w`3zdK{yGn&ruUN
z6S#X+l+bmliB?(pMSLZnx<6mSpxLUIAeaidg|x9J-CciFxXR>Q-~MTnR07Mb4~_YZ
zG^x9r0$V8)Pxij9s!Z$&TK+$tzB``k|Bttjk?hJ|Q7R#O9L`ZhQdA;aB`euE)^QGz
zk#R)G7DZ+6O*YwkJC1{6AIHHt);ag%`@8r4<FDg!IOqL-eO|BU>p2D#PxP&sUy^qs
z*8T~?zYht9mG*fsKR&Ksj-YlcfEiP&Q2Y7kkC9(7zA3884JR6Q<)TA!B7^1E2F%_1
zDLHv?xgG7O$Wb0T13Mv_<mM&*wQoi=$w#WrN0{CZ>!Aa`Q|Lw!?Fd{-<&1So2(F5}
z(c6aku4PsmrB3*ot9pG7M(7L2>26&_rA(#~^bzkeXvSU^03YzD{+;LxfwtcGt;$V+
z@G9@KUA2Pw4*PpOK;v(s$-`^A=ZbrnV(J$3q{~Uxi4g&M0%jgE{hxR@`NI4it*gMP
z7vA)Zc?{W^NSyR0js4RH2UR#(MC|Wq_nM41C+_zoy}Hyq;XU!FN0s#?>|ScwMjIV(
zfr5JyC~qqw?o1g#!{L6dGBPXi&?3NEk5BOGk0xm<1FO5xK>5U)z6ap<d|@@ee_MVu
zKH?IYTn@MEJ=*7*cxAb`<E6HGBsYTBe*YT@NUL9m8kd=Z75adqH{=8wbVZ-B%O3+f
zNp_iW9l8IC;lp~KEUo`Ni502lyZA2Re{ctCz0iKA$m7DtV#D9BJpCWHqLvLfYoI5Y
zx4LCv$T#BlmGLV>k5im$9i)DdaTKyt8W5q^@LuqEvU6BA<FGd*N_)dSj<h$TUj!9U
zBa<*AM;k;{g=-Jxn7(N}!Pl)Ciw)-2pl#QFACv7&{nxdqJKk+)NIjt(wNI@Lg>tee
z-QcSe43r-?&NihHjVvK6OVi6)g8Mkt2#2vs);;0CCX+ykOS?80lg9tVa&Mf2y{_zQ
zsdsbG+Dp{Y)1!$i5Xrvm9Nd1#z>nN*{J-?0!~NZgs?C^5;T(R4Q1vozGWA+L#wu+)
z`LGK5DDs<&m1r`*mG6sb=c3j>HN|m*;YD-Twmg?|<B(47vEuALA@T(`X_9}Bj6cza
zZ5ul5CeI!@NN5NT7BlvxG=A_FIULZd5w-BSXBW;Fwd8bj4#?8MaezF?8&;G*jZzi5
zS@<CkklSiryZBjrTK%`n1aQov23(_Q18tRtwi;)FV`k`cP_|ogS8#Slo5X6qT}p*j
zXAWrNe*bea_qu+!$Bu?)QtoI^ZJUV|nF=#*XSvH+&ZYr2+a%pQiuZXm=VrFpt1W=F
zgKuC$qo@VV2!k*AvaQymS^h{cUb@|kxwn7eHT2%-t9yJg_W)g0ynSx#x4$WI_~(xh
zn+N-{^X7$LIG&5mb6J%ph!#IImzMG|XSCi2*RZZ{gl?;RV`<~u<<^aOD#(Av->lrj
zV$Y}8=B#tiRZ=s}i~^9S)ElSxv!piun&RdIbb$b!e8T?@SyWNjnSzYtyz?U`V5k^-
za39|M_v*ZE!j5rojC!UAz_$quNs4ZBmYY`+*ih7(R`$^5_k1z!yHSugpo!mHgvKzU
z)L$*1p>KNZ_TvEMb+69_UUsc_XKtwh=kKx$`Uj{~hJ{1<p7XDEoyCQI07mA0mGdw$
zzdlFO7el~)m7#ehC&l?4Np=6X{*w&yUM}zHS(@z{kWFvDr@?IkEOt~9801t7*Vd#E
z2Jp>~iN3TUL)p4lbMu;rNqd799&}e(EyZc5s%G7F=fD5LKA$;H#}`9B!9tKzo)-c;
zdI}C(uv{X8>TPGEly3tR7^Muh>Wz2~$=m(!rVj<0yzx8iL=~OP*Yle@6&v?(I-o7$
z_%f7KOCI(ZacT5;Y#J+=B$vFqPC4{Ru3f7fyS$x582B(fbX=~M7RYYjp#ot3Ro8}Q
zLBx>qHRu4&t%p)ocsw^HuIdxgUU-v+XS|dgDV8(gza9hDkpm9r<5W3ef08!t-AO~R
zDs{MR`Og%3xmTX6-$RX%e@>54l=-XSE5JwTF%2-+M}P?C1TZ0K;D@&0a;1%(D7{=-
zv)f%0K;gqRYBc)|K%b5irG7B(o>vZ$y6sN0K)pk&j$-MU+-tVYFal`1zKwf(Bz0Jz
z*#hrHuz>sJNA8jGk0^Y@zd|iL44N(<FWnPk;r!g2IHpyfO1B}qlKKc?ZJy}wzXsCc
zQ_;2}^tJ|ZMlgXj^ndt#`z6->;I)C*CB%Dr%-(quP_0-17s0loG=;^_|5EJ5g(N|K
z-oq{%{tv<4FkO%ewC(76cl&Il8A|?Z1(#T&g8QIYdVQPFUbhCycNtb%bFcJXry$R6
zIlzf6n_-YM|FPA^R6NmmP{BmSbl9=X_#POSBMa1w&Q9#!qMTTD%JNzNR<~KI+jISn
zaj_79M&;GUcy8BV6&i~FG6jz4fh3{?Az%wY6R<`(ZK6!bWKf?>k8L4+RE>31Z({dO
zaR|tn<@DWro1XSNP~DF@<VIpc(NB~*Eh&EM?7iQqnR878)S)9LKcF*Doj<6oGK9le
zA%JuCag;K?jbp?%#mUHt-v^Kh5_hGfSLI_su&1H7{!2VVoo;)rZI%r<`&zEM6&$7I
z8!it1v2<u!Np>7A=@gi`%SHhPp@tzTL|+IV9V)1ICQk4S!bZyWKLo^&j;&QMNT1}8
zsVRWtuJ3-Hl?Ai>7zG0%i|_-B&@nE>M9D_dVh(&Z;AjA_YHzb<C3$*TC`qMK0C(QP
zWJiBhfcIZV{~=jj7Kb8?%*o<(QgG#K?grfbbN2Oas-pueu<vM!y*OI!4Z7vi8ekBv
z%W3X<Z+$6h-S>&}&7!*PT7Xn0ptCIUZ2f;CG4*paFu~2eV|GrDnqNKr(lFY>I5SfI
zA&|A&cytGW{=_LsOgamcahS01dUD2Hg8;L3Vx5rr_S4oPiNi<^8D9XC|B%ma|6d!u
z(S^)095>FBh!N;1LP0Y8-_&APHl{x_Vu9IIbZ|#)L$T?w@0-O7R{Q$RWT~)dX(}dM
z)G`ajn+@NPx?%A1ER4>9(5EQ%WbK47fa{X)x72Y=)8nvz+%y#Dr6ctZsgx|^Y5fE+
z*Gfg$wEScAo6;(keLehVt<Q?$_h9e)HVWZyTf4+dQTugkZ=YVBW92l)``I-Ah?6Y)
z69k9l5H#JbK^(|MmEU*0*WkHyyC)&7?jPLmsIJNXt*Kp5`oK%YQ#T_x-{{oVbuL<*
zGhhaL9&&dfeN<gX?!@)8V~06{GK&RQ&3gU(-U7(YzF$)!k$~P=pzexvsnDRShyxT5
zHpk(nU9T#w60}%2)&D1E1e0V!GBhuD43$XJx&(m99be^#@XZq{N3XTZF}iT;&BO|>
zPC?#U4-v7$Fm$HX%6+VQzW$x8Ib&072NxgXJ07f(b${Q{)Tl9w@67$I@1GaGMTD#P
zaB4Ae-Z%S9?l>19W|2bPyKsMw<DTGa^OVfQ#&`pA?zaA=k{F9$1FAc2jjKm7Rr#;`
z-aGfQ@Iw}rQoW3L6$o)Xuu<ZJmv0>X7fXuLkxy#zY<m0r_8~#)9>AA=z49^ugIa-C
z&aSEXGkQ4+0YZsO2dX(iW$WcxLjC#;;UB@zCw1tv7<UH{WDj+ktIh%a-=woYqw5m^
z^jOdQ8#?DNKxGm~8BumFxzpUgFms#v!8~x8avO;AbzeTF01KmS-LnvP+Vj*+MX+V@
zZsm&4)Uw5x74fD@@t<v(<%%QC^fC-)TP#{CCV9~s2to}o1P;@IcLGDqKzP=0@L{)p
zPF`W{AGRWQAj16x4JTZO_Ff*uXKuqX(Gvgp<qPM6{y9LwmL#O+)>;oi=X4+_M8wOb
zGBV6sl@-_z0B3AF%>$vbAp&_~wpnlDyZsuJJC>6_oTpkhWbQjXw$>qTw_nOq0SpA+
z+3kg<;Q%^?SOM^}d4PKNTmR?rcAd*FU1Lcwy<E>d_7I?lTc!SA2Zzquu!9^=mG>#k
z{790uy8-Upohs!h1(HXN>ZW5NaOY!E9W=DgV3TM=s=i7o<vULGtsW^<<vWdaBNmh+
zk7O_GX8FIb6(iFr@4uv1UqNYP#1tN{d5t(BYU&-#xn91f0UjAg*s7DEN8)KF<Nt$N
zkgldC405l)4!d9X7OUX5o}gaq*T&y5xhv(r{3~~gAD_VQK5mLA)3OklE8l7M<2jAD
za0t~b%Zj!0X3DpxNk+ph1P{W4hK)e`=GsjFv1ORklDLnSoN^&Miw+jwETp*eS3}sN
zkpBQ22x0ccj?{cV7XU`mv_32x0)LQez7g#Jz!GPcD?9V2sm8Ie1f<#9v(3~0Q>5+|
zj{H4x&GkyP67|b-b5|{_JYH)=8io>I0T70wpg%UpS<n4xhTCb$cT7;j-k2W0<FgDt
z45OKmcyp-tX;128jJsYxM_uFSouBhkokwF5(?!;T%qy8Tv3Ja0(Zsqow<23VCE(kJ
z6z9#SM#Gx5B=13cQ(2#`<0IYZvl`PykNO9c;{X*_^?njWEXBCZ?R^-2mE`&g_&x6r
z8=&+jW9BBh%61*y`n^A3y~b<;XgPD5<g*+r{7<j^IQx40nrrO#z(s&uho(N8d~FRV
z4juxl%FT&8kp$tgJzhO3CJ!|2&CQ)g&C*8zS7mec`F2QK6tlC|-gVDKx}|85Xr*f<
zanHrMZw9UuAx<4DvZ;JXulk^;n}yHC+>29sUI~aiXSWYA&1*vg;NNDVdJD8W9W?&3
zqb#Bg@ZyZBVTW@weMee75;xDsBz~^fdbB1N7%~?%Y4bb3&siJj%A~}H$2l@cZC_ES
z5kfk#e#x30m`b1jqOjMe+5xO!1pOt>z>wM)JE0-<oZq^+UfK8#H!%jeO%jcRL6V%q
zb5pxC&p$+Mil9u#vGiL`=fMxp0b-1foXAU|K`@o-$MfuPF&ia!707?HiZBEjA8g*`
zCz$b4$cV*|1VjA0mHPrT60-sP5t?Vfd1$i2VrW<Z>OtqVK-)A2v!8YX2lMs!rE2b&
zZK^4rN}TSWxctxtNPD)uJfg|BHuQDNiVi)cj_p5?oDC^@L;b2GN+&TMEwoiB%K|fD
zd|UN>b`^bm!~4?pPW2VF7gT)@)2m8=1{o=_c%*(xr$^n~uy>2$Zo!9>4ePxm>WuEG
z3+xP3PcF2&Q!Y&~%Uzy+H{m~$uFaQWlvHn<y$$SvcsLk-UZ<iK9?I`p;tl{_+mon0
z5~v6#k2y?N@)}xG(mh;`cE6zBtt-ZMIA0$OU?XPC?}R7<qjGVO7^Le2$ZAXjQw|4M
zX0YFd1|yfmE4|J}+zx9S+?6wpPK~@EyL|wy^Eu+_VU}*5r1JF?9#KxE^L$~cVKTIH
zZo5azNh~9-+%ZQ=(j*zZyv&}LF>f(k1=33%{i}S1*Yo1HO;cGU@Z}we`}|)0{nzVW
zE-IbSFY;?^b9w3g-1c4`-G+#+2Y(A4XBeWYjJxW6?;R~OezNb&*ljuqpLOMPc`#um
zmQ(Ir0cCECD>&N5z&d?3{hY@$j@NoC_$77MTUPX>MeREhHNWimFomtsu!?N!m&J*g
z$h97RR3Cw99M=g%L@(SHM7OoO(93lhiw$`Xxjz7k!U*F)cp;-%6XEq2i0DKA&}j6H
z!KZ|*2+4Iq34iR}kVl!_b()5C$xjW?Ev%b+jp-2<?2yPhVBg}m^(cD|nn{yvg#D!b
z1aJ-(#6565OB(rdhD4jZQFqeG<bGGcP8as@tP!J=qf4V@XZgJWfYbLeA^pJeu3=e#
zFy;W*gI%|-vs(r>!@8;+nUbtdXQ1JFi?DUfd!eCV8mgcNuap?%`W*;%&dKRs-t$qm
zee=lc8%t-y(6(q`2+_}Zb^H?}tKW8wo$s=^mC4+TSA4@_6i(rh508Mue@j$E|LpL`
z)MOA+u`iH(?}#4ydG9vEI{mi28VPB8K$qVbKa;uD$ihq)7@~vkdc+LPILn@%+LcZu
z?XH@GQ3;9~C5bJVhZ)DSSfd-zi)!d#j(_Mu?K3TGG=Tp>>G+byQ3pvq!eu*TL@o-k
zpS@P{bL*5Ty>=Pu(8r|M^CRWc+IXI%i6rhR6YBt(h@dF`H|iB;lA8j~d5k<!Gz94%
z5S6}TA(X>0Yy^;H`vT(rO7BE)N&7NuzjwZAjmTw1YuN1FAJhUrXF2bE;+Ft4WjpVF
z$Arw8#8FaUV%nreAh;R+;;)_He5cSA5f~Xw#|Z1(z{ykN(Pz8CeeTbKaXUVriS#>}
zw5SblAhatOm!ZmhJ}}^Wz?Ett^TY*wqbGM1VTF(Oer8};w@9Fnf$+(%w?z#HGVL*D
z0&2}jKayUr<5jeLo4ou(<US778#Ez}GEmN_7a;MPenv2+@8{e7lmc*cL$G_}h6QfD
zVn!UG0^_Lb(tWm-Twon`oO00j?Jhrx{C4N7O9AVS;Xiq~b5_Lg@+}WPT=ns!weH9m
zHshO+RdA#e;NCNozq-WT|CrT&+Rq=pI_qVdWj=r+rcmU>{fQcQ`rHFn^ZAsyg(;hy
zMh>Qo-^^gr!n$^xIpP4fw96gh%s|yQD9!gi1ZerL)?AS-6dJ6*UF7;-&o_Insq3F%
zrs2T5cArfC4CM9001+WOD#!;@aDA)(D|2Z@>(vf)sGgq3+X`XQ0gQVUumEXfcL@8*
zToSe)-nxy3g+0*b@{pWN<FNjGxbyRd1dM*euA-DX4^mK3prOm}XzS?;w0;*e6XHHB
zE{*F&Jtu08Hn$^{^l347E+gKr{LS6Vn-I<6)5gC2LN_;lv{&$~a-8@>MapBT__iM8
z&CvDQ1!Uh*f6urNrZw<u0cJ^rGfRpl*#Y>+FUGBB04y}xPMI@=RGcF)#G|xR!0!hU
zAXWVu^Y~FyofAnD7-Bj*x$p(j_XAip#XhDwAI#EA|CUq7vTr@H4e8EzjZX1Ra`#gP
zxuqqlt9AElc$pMKg^x*$#kQUsmF|57^m5b%hKqO2X?Rwsk}f=F9FF`ktV6{_!S*nI
zF;_YruWB?Lb3R~c0xwszFza;zN%PMvl$=(0E7db0G+ID2!DnP4g3}@-r;Q}kIakhp
zRVm?(Qj7*j^cbp1wm-8Ck=pn)01{!Anz(;;`Af*=&3#}sW287RnN$!DAepYRuYnw^
z0_V0l=#10YlTf23O>9o<xS5N^Pdj3fEpBG^oZs|_Z>1Hsp>V8E%9)JrL{T{#n*%dv
z_z`{EARg2CQgP&X`bvR&Gy+g^st54)YYX_iwaz)4jAm}ID1H=tS^d3_VinBzV>b1U
zml<r<>0uNuC{1+GH7<q7Ht*%l@=8%Pm<XEBUf%^U(Vw;5{iv&8+1<t#?C4P{=}xPp
z!gf~|mIoZ<;ql=n=))+M<Me<HPyppM(QVmnK5pYkzbm;#-X8eLkS95FYq)oUvU%*V
zU6DBLR6gLdAvV&siD*}Ze7UMN=%=WYTIt{IxQoe5PrRz;?NYn<gGGK(<@*kTFDr>A
z*jCv`>Xp}kQ3k(I)qlwK)#+ljoxjvq&aVLM2VI*P0KFdmnJC%1sD|^=SHaa2c*vd?
zN`p{@$wR;%LgcLHeegmZTXy9~=<c!d!E<kO8O3l@|HKs#pJLciOAri(*asOWHCd#o
zvhSI;@0#S3f2|Fvxb&AmsF>;wc69e+jj7JB@AgP3wr84wIV^!JvZyA*0(Lx{T$<RB
zKY>Je!xW0REl2oH`oz0)CzXgX+QPHI<_>sk@kM<dFczdl5v!&>?~tRqgTdXHc;BTs
zOv6KVF5$4b)tgJ63)(+av#=`M+JEqN#O0<_u2<%iYaagy*hN{mVE@MxzbB6KYE?fY
z^nJXNkCBtog`9V-t4AU=%Vc8gl&%A70urM~-*1EA<3?hR*53?Cvf0rPCwzVtnJ^51
zykXqI^V!iVe}KIU_+0Rk6_A=U#zId@QAh>$+Gx!x+$=z4<IRY7DhH^%$rE6ZVDS-}
zS*YBVRx~uT^@8iH;>Gu2KQ4PJQMG*~$Lj68j^vBB*66a6FeSty0ei^`_S3W?>0RD(
z16}e3l|xnn$IJ2d&_+DTx-1=2Ns4#O0cSP&+VoP4n;Hc=>1JiRzkiOflYSi8F0?YP
zhq^Ffc6Bi;Orl-s{l^zn*>fu66P$Gc>jsr8opg1C!m3gHCo~L?cb(<@)Hsa?cH9%X
z5p&APMep&sNF>=@XIf4G{q>o*Ij~S&koga&F`uHBGmqK1#%M;<v~Cz9?e`XVl+K=8
zxN);Z#Ghp2RF)FSZhyvRz<ugXH<6-9aYj`KbVq49GrI=Hf(31B){H4Xjgr2LMw<gJ
zU}HrL+}I$6rr9%|r!Mgk4k$00=9$L;iZWN-jjt&3FQ~1qG;h8d0$_2p&-2<|1MhaS
z8<S?q>a2>`2@=fzKv@Hx*iz5Q_hYTH-NDCk8Z2sZ4J-0Rzh_T|$E+Sl(oyxXash6!
z6?U6S5^9u+x(~23oP9#-jhR=bgrbMfV}{{(bjdOkfQiopiPH06&z6GlVOh@MP*huQ
z>9-!m-0-k?1f139ZX*3D=104V0Zj~>%toB>wQ?Zq%4h{h$W`b|166Iu{yRjRZdl>=
z#b2d12k~;l6>f0=O$_DX8I@xMaXNu8v`wGsi?#s#XR1X0Vk?laC6NXwfZT<L%>5S1
zym|U%YtON2-@jPT$U%{X4C{E6doBzC(<@*M<^<a%v`7(909+;E6_m=GSwG<B(RdkV
zwHurGHRlo5?;O$`t(BM2l<0A>wODxJ_|#ZrFdyF*HSFwi(&fIff?Fs86Z^~e*a}$=
z66R2Z!_Vp-Q@1EmCAv>6LL$@wI>oVQqf~qgg}RY~uQuvti6#h~cgwmQ#s};tHox5m
z`5*0dh*)rA33N;H%Rv5^V?E;D=vk@YUmuI5lL=`d-k~pjsFUe7n=nC3SSV>)KBh*+
zomv)1Nz~O>r#x}MvOR5!%e@ShMJG;aur(q|Y2WtAC#ped^S^;wikf>InX~=pzZqE3
z@PMh^8_iiZUR7~De3}+xb-Mo20m#H^cwVDX2)Y33LL~ts_l)y?BPA4a^8K1O0TEkm
z2Y{$iotoCbG6ZY{=zbaAazW*9Lnyll4AB1g$ag4hU}`UEx*fr8Ec^BWq;Z}I{_ID&
z^m6)Kir_8xzaRoWI@t2xe~jPt89}`WPF3S%4ICAX(9rva8Ti;^T$7_wEx9VYsjTba
zUj=DgR{2O{PuQ<A3fv*t<MfKiv)YY!@2ff$)~-<I2wrjBR<14WYsneJ7!Pd^X>qDR
z+7r^_7M4fC$lajZP}I^f8$vJ?0NUCM-A5OBULH;rj3#LVWM2vFe<ld?XY9qr_@~#-
ztSQsBWC<log07t2BEJN8n2LR*bFhrPGEJ(ZLJ_0~Tvl(w3N|KLJJ{A24XP%v!glwf
zKjAxMoc(<+lI}*Y7WLH(ExP!_uE3&NL(Sdq>@|n5VBQ#vw?!LQz3-dD46QU>b&$<Z
ztc#oI5_7(>EPS^-vP8Y(hUz?3vj2HeG4dpv=^JHb4ubH)B>pGPwAV4;hf6F}>iO=1
z^{EaLiOwxjF8wX-b=vi5VFMPKM=L;~gX~trH=YT;jtr_V)Ihy|N54S@I@z&O6CC-d
z9tV^Y;5u1Vr>IpWS%r)u_}%;DCqyMxq7r}!!UlLiH!tOv_5XQ=eKQSg@W+)dv;y#b
zilg)M(kn78p9G!d#@iW50MJefOqvA{6z2zzUYp^>Acx(CH6qdW4RH^3_|PHkTD(p#
zoDffhn$j^^+NVKYz*jxGFo8T7z62mYpm}`sBf&1n7O=LTcv-+6iWVKLKZ_GTV;toz
zm+(G5&4L0$a7-MD>eHy8l@T{4L;V#O8y7HJ=)P7=#@74hmXA4)B$`dVIvber4>|0O
z|DLfwI+!pj&pz?uI|UIZ?rn+g=vg5pfR%9jnvFxEdaEC(X|oVjf73r@ZdQ;!kakx<
zl;?+DCOV7WgXp1SqMw!$80-5{UF7yqAwB)hQK)JqzJpgWVPunHVqN^Hg=r@j0Ggtn
zIbWIx7mI2ZzTL<IJ=5UJ<mhM2e`7<c3gZBK#SEc=Gc;ynKhIB~&wW@&%n<2`22ET~
zG=JV;w*|n|V^8!Kh0oV|TnVq?t_8XvLqNX*Vgb#4MfH&MI+VF?y=GFS{nR<R<DW^c
zsRgq@B2ccaOl#|TMcZCp=Ls4KA*3$iNXgAcEK;7TuXdzL2h{kE`@80*P=|WVGb0Pf
zxsz|XX@9n_4$MeJrQDV7t3Sc6p&u{~pH|I3ol=}8PZ`!7lot%IZ#cyWSd=Qwxy>2I
zJI$wg45vcKIm2|vETT0IBRPiXzK6^pT)oFWcmd^U>9V<Fa*;;H(wd&em~50-CwuQN
zWiS*xO8>f(rP1n|3*vZ&7gYB=gJRYPZCU{FKm_iQ$Xm<?dno`HZLV?ciOU~)CeqGn
zQgY!H-icQv)y(1!U@#gOqj)*RM3DM1zLU$*{J&4Y(SkH!ftRxk=(;>bU!j$9-}mG8
z8H(f6pVeFKB<Lk$ql6NQF)X5)di1EKDf~lpr}CsXL{mw7^N|_ZB7n7(X#A<kbMsZU
z`T_cg_P**5$KZ);HPy_|8wYlMLtb<{CJ?pB%y?^&6FfCFRP1Qe4q~>i?Ms*ZZr7TW
zp$DkGoI)(@CjE21UXNJGMZvJ&KAiqk!$~h2SFLS0m)n?nVG4#@b+6{sIUhQjk`~nF
zDuch_9!38>qu417Ov|YGBc1a2-(2m(k~aq5wzT2boUHu)O}$zx@-&g)UHhZn?_#-n
zO9Ioq&rJb1cj2$REw~A<*4@d*d?h2Wjld0fuTbCnLja4*28Ia!BRqWHNUg!Ofdh+?
z2jB-~c^gtBVpW|ydDFa$CG!x{F7l+a_J7VJNi5r2tu71m;A=(a^a4&kUN=dkTTmh|
zx6^D+MrPxb>Z#W6-jE8EX5JkJ6wV&6vIkD~%HRry@N#-M2}F0a)Si)C_(hd;SBsy-
z+RM~Dp&E&{%ZCFNL$}kp+B~^K1#9;Uo*l6%-Yi+o{3ysGqg^9(HFB`_9N@zYxH1h~
zv-k>*eob7;!Z@mVOpA4#07PTS=#hj_%+xGUAFbqm>RGlO9r)#|)pV~hK0vvruk!VG
z^Nq_ZDjUnb0R!$p&>2phF6!m7;6oX;QQlYQ4A|{gSzeGX=+|uF)iEl|*atmia`A^h
z^R8Gx4fHX>%knRV%KtE;4k#eS8;qcqoeh7fe7BKUM*E*1fUEiGe&v{!v3Oe|z^`t=
zZ}^iJ$|^i%Z-tA@dLK5kz@MHD3%+NSd;tf-J=d;G17i@UZ$tPql<=>YLpRa_8)+|X
z<r#&yy0Y3=KkHDKbUy5(XjqCYx2vs8oxL`_ApqXSR-p4@k;6%7P=`h<NF8&M3$c!S
zr<mOBSdjDk9j98klEwdru8Ctnsd@^5l-E%+dBm=M>}S5S9F?7o1lQI&{N+;o?2TK|
zVy{!-`!INqiRyY&n5`IRn9QRjPF{p;^%vLnD=%Mgs*_~;IUj%8BKQYWR{|*bKMUFF
zXRVTMFz}+wji$R;TLCN6c%i{C&2sUJR4~clS)+MA=TObo=ojE?Ophl6Kc%#_UI)rn
zta_^?45qa*^vItLOv+yp0T21)UuwZxdZjO%M&Nu3Xj+67?S!i!818|GBESP6JvY_;
z5vJ~(V&DcS>?>i#MJ!|LblVkJP#Mu=4d;&f3}Elqkn69W!ET9?)>=e>?*Mc_l&@*S
z(BmdI<<#w(L)@A^gyi1ZxE<*<5*@05>ce+fQEp14ds+Fe!;4%<6Y+H})e07V{kJ>`
z0mmm6!I{k)+<{-8!X@vrp)MRo2k%V%6rC>7U=o%8guAhi%ZbDrU-wGJ$o^f+&9MBm
z(?C_4;$gqfmAi|70&}?|nD#<!P(9UJEU8xgUBJ1w5@bRA;e%V<s*{rqp9i!&(G(%2
zB)ZZEAjsq|toSjt@L=R8LPzRZy?W2O#N9+Bn0yheRG;7Ys@{foq0e|Je7EK}Ql1MN
zYTJ(i+=1S0e$Hbl5}t2mYlIN8Z-T4b)2Q!rDZpde?2P?yU#GN6;SVL516LL!qzc34
z;B-|sME;d~O*4glwf*RKp^q}B4r=Iro&hCtw{Vls{%t9qNJC23EY<?+oF)1!atSvx
z8s<q!au{wui4;FL|LJlp?k6ha@HdUlO;J{^4f^*rBa!8lx&s|mXNJSpPY4|ddAMC8
z{lJ4|{(8d>(}2rT4VkF;Ll)u&&xR!b`>dJCEx~t4W}`_Sx`(j2G3e?a=VwWuu~n^g
z?sZdqG4);KuvYc<-_tLR59MGPJM&B#jKaB|pJAwX(Ha$;fDt3s99F#BN=H0IGkG^<
zBuK@vTa9KsB`xTqluVhGI7qtBo9RkIPKPI?-_xJs4>Q3E1QQfrd8FUlia)yfvrKOQ
zQ%n5MXl1}46m_)XD!C#P>Tg{Tl*Sj<7%6PiAfTH<NiOayw%pchF_$#2^ABv3q(7>D
zxGBl}ca<^HCk#Kx8L*>K3@jujJGJzt-6bQ<CitdCmE5NPsYW^iR*Y*5Qk_EuI(dc2
zeFrvzYvO&u%~gT8vVLZ{T(23<ZeTljJ29M7;^hr0rXTdqdp^&Stm+f%&M^UG3;;I%
z7@;gQdvoc&jSt%U7Y&c(f%mgG$m$umfcfEJ5E}t~uhU!hG(`t!aHzyM1vfY`hsDpF
z(X&7;7!byG&{2U4Dydu)h)@PO|FPR{3)Jj_El07yk|7Jfp>^`l`)IjO$oXPo%L5&_
z$;2kLCD7rR3$0WPn18bZbhfedkv9TEJPS*YegO5OAVc1|#)h&=iWuQnoNY*LNNSEC
zXWz?2uM8f{xL7@jNcpvG-2ULRK5UUAp8w+|s~ztQsZT^Fmwtk{CETs>D-F;5Ml}On
zc+S^Y(N?KZq}st`hpTc(!<0y>=@!rmv4b`ScP)F&C*{9ihDHXw#h|u@2EVQbH)aX_
z-ALj6cv7mvO&^{Evl5C<357v<_3-}1aWp*eQ_mI#u|3ch5^;iY0N6eG&bx_<0}Lml
zxnM)Ka3NAW$hQ2^x?V6)G|)Gskex3`W-Z&%1z@{zK}b6@h&17%l!6>MUA9wo4lsyo
zVzPd6B54X|;bHWE4UdC<<O=SsEa$l6@)0?Zbn>T=G=3X}Ih&$CO=#%ABTR>WS6z>J
znpcv^*X<*0#t5liq;FhG^#>1jh#4bR5KHNo=xa+(XRIa8x?9u&JU*8tUdap3X^=0U
zlT=qWq9XFPPnwl9xuc;=z;@dOrhq;cM)utLL@bY;ECc8b`?U`(#0K{&JQ$I2`vlRt
zXxBjVTY`*jp10w*rXT%6Q#8h{$eB5Yz3Q7lxws_C0&ex6Uk7{+&tJ?LeL&*tmXwI$
z0bSpuS=qk3v}CI<?%od&SCb~w41YH{7ZYkf(zjx@8?^~Oy8u%HQ4x0u<<*U|dk%Ya
z>v;9R3BR1!yQht*wLA3<tiybD44_%Dv))>kr*lekrqj(dvDr;_7!ZjD1k|;gLlV7P
zB|nilc>@q>3!iz0`1!TN=(mkxs;|vemL>_qM`oLY<(sJ+@C>*&tTEBhPffvd8){89
zuX~<f22VGN@hqyD=UGcbsa^xLZ-N+N>OU96La*G0i^saSfGYyf*Xy}}@f8m7V>b+?
zPZUWj$NKtZMRiVpl|x1VVw#3_Vv`Pl&ZtBObr+Yi+fN!M_`Qm|v0q;lz&`<+lPHV6
z9(;m>6YCnlkTmGDg9HNBrltU|H-<~{_9F8)2wdB6;54^TyUco0F)7y66IpTtH1HZ_
zP;Vwz+886ZNLejiOXx?yQNCc7{%ENxW9(0npg+(w^eTCPUcQqFaUJQaVeR>*yw}MI
zq2Z-dRvxWL5n(a(Y3p-)laI6<@3b4dU#b_zag|2QY<ei|4&%AZQg8`XNE$U_YjuhB
z&Z5{-HjODiM2{6iU$T<@Guor?opq#=BgpEcOM`i__fwkw-EmQ7UT_-_p__~xKUy7^
zh4=$>$NixK{T|Tp3g9#}Z4dkMR_5mp<Mu5BU#_8_4N1%l0an#F)20JMtd=jmWrj7U
zsd)Zb7N@$usXh{5-StQ(ec^sSC#_Depo+lU3vx4NYFm}F{OcE?!KX;ZILi}^mT(iF
z5wN8RWax={6wGqr<n_yF&~|l4!j!ovVTu5>Q*A5^a+5uP&3gn9z%-xsq&=^GwJZWD
zCW<XP26*%+Z|j=^)n`?T4s${kZP$BytBs?xUDG8bC8!d(z*c6nTwa_xrB@YKAb`*#
zA;<CvaG+1DcDn*7y=l**2qE`XrzoqOIZRdCK9}YjJht`i{Fic_^ldu*GZLUaJ}eCF
z3xbFhPKRv>Z6#B#VsvOJ^E$IF9nU<AKO*!T270hrh!x=`?RGN7)b2z$dJ+HA>!b+)
zTR^!@^H!!mw(h&#Vq#&Tv(?Kv4KkOhn2H#ycCk5%P48W9oT!|v=_7pQ0oj`^{U@gf
z>H!hatdO$-%UI7!lt+NNoB8&|rXNxeq66;3#_<B+%5?I{<dh(bDyyFQ=OxjFk6Aj7
z1AMhN0g$4>d7MbB9tQG#QikuDjQu|^fB+Mtc;dWnX3Dm$z2^B6?%HRX&w^=+sr4o`
z+lA+EcAQTww14cL4GxFjVCH_f2Zo0ab<$+Rhr(_Kf$^VXZ!&8}g<aI7ei!CyW3O(@
z!(;DzZnpqcNKS)05|s@-|L8I0uKQ=$jhO=5lHzehhDz%^r2#5BUzYCNWVNVrXz{Q1
za^<%bH5JRt(D#)aHIFh)8W%I&_Z;U-_*J*Gl^z7yk(=pyvaa_eYpcGVNDx!SEtM{Y
z<t9dL;O=n3mn^o>4jUdaNu5{PJ(yK(Z*AYFtI-;!;Rt-%u`Suq$XG7M*iGdZj6!X7
zD&dcNQT#E|{~e)m$QSk2jjy;14G+)RrMhYPbjYR;RhRY%Py{b(UTVx1nw68`etrqG
zyJOa=)Uj<usAC$|`+ASf-gdG20Tol5E?BqvSHKQ`BBM0^nYUZoiS3fV&9xT45;syK
z_L!sS3WE1T#K;XV{0teWu7WoIyv(A{XQb6b@5DTrlvdo<9EY<+`k&J4rFdwiqA+zT
zr%7O3x1aT_a*9uhhyL*_lW0j0vq~5`THQWn-Q}xZu|QNet9|3~ltE7!0;^$DrLF4w
zVsyW1?rEcuS6a>5KV9v$&<a8u#&UiWEIWSEbjU){KODfp;CJG_^ZT{Ik>Z_=o5+9J
z0>|PuxoL?BGZU82;Cu0uqLangAxfP_-CqF?#3VB)!OFkcB~c^#P;eQ&KV(6i_b~{(
zv)@pQ8SL<z*xyM$UdQNxD^)iPJE1pxCki%trgGsWv#|Y2LOQ?~<T>Uqbx~d$<pjut
z9{LAi=DSjqXXL~3Z7hFzQnw0&oPjn7_|fTF%69d3V8{et-Q<~mihs5GpuI47E*M8L
z7Z2jf-bs^=JsmS3q{R>l;o4fA@(tl+y%xBFD1bqk5Q++buR!(OCr8Q-7xdRBh1ql7
z7wCPx&#}G+944fX=JqC^a&=~SukXm~IVo7LAYm6acVwF?4!LE)@>$55Fiy|OYVdTM
ztFts1U#zhcnuxAec|Uy^^#uOIjQr&bm?G2t`(m8Un-2HbQOR!%+WuQm<^+XEzs$Wc
z$&Wvyd88{3waY~iQA~qWxN3O6V1pAji`nXp_h7e8xZAXc>m{p+A|~rFg}Ld;TH0=v
z)gAO99o?v4AGY^E7Nz9Vb3r6yz#Kc4cnJ_i4E^XOel^fr`-T-sRe5^p2U|CX^@Sx-
zdQQ%%+C+Z6tlrCkA#omejn$Mp9fvGliYgZ`7mIyc<m+^71&H$JHkK$N^WXo4OMBf~
z_4INUg=<nV`CziYPrkD%Z5INXs&nWz7ORg+C0F$$qpJ=`Jm2U8ZS&39=;$Tq8^#@(
zQ~4sr^0FYvtj|`!!g+S6gJH|NB{{AEhL4bWNX1mWyNsNmL4I&xL_Wgbhx$d5y*Zs~
z4}VCqv7XnD?WAyY-Z0e%tP5M2S;U)mKZ+%YSbZ>V&`m8;nt5wXFX!Srl5){&;hNR)
z^aRgG{w)&es$_757qU5o_(Dp>O*o*l{U&e^+m9d8j)o))W|8kMw=`sYHxM4&v0TdL
z@ouuVm<#5hlJcACN;}cXTnl-+zQRFuB3Niz@u%#EvnWp^mJgsS@~4o}2cPK~SgX*>
zZHyN<Hj3j(S05GOEC31^fe{Dk(gyE-Sy{S*aw0X-Pw}FU|DbYt+s7d=2<^)V&PY2z
zRJ<LFIZb37te?2AT1{Gvz{qJ=)P(gAo#=x%dHzi|0!|Ld(r|ufuD^}*o6rC536jx*
zwcuE>qh2GwLUw!Tyc2SF58#YJi;Hzzs=zzryMsZg+Po5FK<7uiRD>WaDs9)>k}|@O
z;K(v34*Nxs^elc+8mR2D^s^IZfe6jNl9H*v9!w+M7AZZ}eYnAg@BV^;PU6-Y5LEJD
z&T#VQXO!{76m<@-M`$fJ`u5X{D0xywQrHJS%@+`w<SDT87xfWKsf>Hu_A>(6f=G=j
z{4YO%E->XY@b1{H@y#6oL2ALBI3w#Ypqn5m_MR3y`d|VVq%vG5gZMh^wkp-ofP9CL
zE8Y~=37Q|8jM|^>gb&J~=;c_Pr(QsRf!s?#x?eL8xY^x;5tM}j3gPmC#q)T|<tSx0
zIJfCTul&hLmQ;Mv>8V*qId?~MC*R?$dkZeZ2x?yL#CdK4k12v@!HB(0<)ET>&<wxs
zx_(I{c<I(mfn)1jRU+c32X)(P4z>@qZq0l}Oj3le{wN*z)1Wfq|B67poF_;zs@W4y
z^!$L47eMq{_vlhcpKONSNIr||Is$D=<enZt5}xoX<R$N)RTu~j%o)oJ6d5;}vm!&A
zKb~(g!dh1|-ci8`{kiP*SPESySodr<Wi3`$(m}wJ*QLefn2iE3W&Ax(lTh+7u_#56
z@-6Ir2&=tbA)%%+fyoSeq4S(8DRzs~P3M}qsG;_gX33A?kT;i(V9gP}r}=%q`5Vi*
zwE+j#zp8uRb2RIww|XCd-k^C2*0~u3<UvO}b8{lguPww9GCD}2G0gEUM1HaQa0?lE
zaOgE7^&QL~q|D*wUe0#ofm<&pwYm!Wpqer3;DSL)6szF5-)XW?0pq<kk;a-$4qdB`
zY$eFMjxdb^eJ6$ccit%3Sv%@Sz2YE@5EnKuCnaKu(oO~&*%Assp6Y`pU5Vo{!4$gz
zV4jLYnt#B*eVk^1ils@`miyM%sjY;c-fIzv2>!JIb*3B@QB2PSuTcMnv#PVC@4s#U
zqPeWM;JO>?<JW?ap-JvkU6IbNY{H6Kni&<Cj{Q=SZo_<;wz_={pZz2=(5+{#Aw6bU
zl*??J1wd6UP&GyhZHg`(Oh>&*g#Zq>9sjPled|4daqJ_;%hJbYg}cL@gBj6y{PiHw
zK}~nVsx<%VyZzxU-lef|b?kyS2+4fdm}S80{qMxkdsh1c^OAbijjz@{mg3sm`BBAI
zco#~Q`G<q1M2VDI|CiATB2N0n`ZoNU$-T^SI6u^J@__6KMo;!^-DN>KN&dBCjqLn*
zlii+l=M86>CA(MM!YL&ea_V5Ma)s&%?BtS{yKAVEBKo5sNMGpxRvTU_bJiulG<JY*
zn<3!8H^dx&MhWG=2Gv%MVnWf;)lfh2tBpa{(Eod2`EceUDiL6TvV09<0arhJ9m>2&
z|B>bGLUqsuO>W8cPEaQgrT}3Zt8P6H2)6TlBA*;*6=JrM(>%B93H@7*RaIM?)+~WH
z78fSXf1mP0CIrwl;5;kJcIgcG*oAH^p7X@O#1F4n1<1j0fy=M=K?F5LD^&a|k1h|(
zXI_~;rc>U;wB;-$WerM>Gu%lA<jGCuli}7`E!G7XrmLg|fP?1({Y{|!GP?tRxx0#X
z2Ep#rSc~z9f;(=38^cklu!Ji8r`l9ZsDgA-n;29VwR-Km2=IaFc><8zhm*<h-xI{}
z=x2!?kJ;>3THJUe)LW$&6>m44*6OG-&vj%2x$q<j-_4R4sB9P%Q{kg(P?XkQZQOFq
zfXlsX8F7*xjVmhY!<lU>gnaEcwMMJO!P)J71;4<(G99$1>(S{UO4MK8HZo{_K4s=V
zmM8aQSUuOdVjcHBgemc79VDy%s7UgrE|lMg#=Pnfu~uYdR#>+=GZzwK>eZ`@vWzD8
zDH6?F0d3*0lM$;_V-&I;HZt--k9qTG$7XpSar$pJ>`FFr>k6lZq2ys=&%cVqA_*37
z@#!>l)**D4WX&w8twz}?UEiTzSMG4?slydAE$?~38)tpD4nPM^Wyyy|?Djd)K4(AB
zZ$<9BrDFQ#mMs&^1((v-JgUH|aPcZ#>X9E+&vc_pesYCV&zInQ=^3pa)x@z6bs|<h
zDzjD3Z*R5Q^+A-N${Vg8u4n^U=f{iRWg#WK$oa{NsntIHOC1%>+K`TKB8}3BEd_-Y
zv4pMFptO=@SET`aB$IM?P}@l)I*3{DT^kq^ca=)tFbqwTblK^~zqWkGvTUUKJC0LK
z;C*B!MqTJ*+qaTEF#!yf4IDGs8B&uhsCFMBaPxHV@Xb&DX0Zh5sx_Q&SMNKHuO#ZA
zVrDN2*+978X*x8Nv$DAa-`~#husza?Os--JV^uZdectv$ltGS=s6I099}N1SueLwB
zvfQG>wq3pf#m9jR8fHF&w&zJ5iqXgHpwqn(5OQU48=_Zz^UEN<cA~*~lrUd%I088x
z0hlI{v%g(83aS($$A1G6zW^IF^03|uYcsdSJooSf;63gTbh(|z|JCY&w#ru)B~A~b
zrP+cG+=R8vyf4SrW#?PeReB`4%@mp;Vrs52j{vAFzxk*#pfpRl?j|D(##vA?iJjJd
zv7A=gnI)b{7M>BRuSov6mAbk2hHoZWf2oDDNZ&G-9%R;LAs(6HdF?J6jct%Rp}2OV
zmrab%RR5h?&HUl&%akuu^M2Tn-Lw(J(?ZJH6OJT!X9r{F99Tm|3lIW?09EcbLu8CY
z{Y*0GPCcs+z*67uW$q4fgZ^?;l!+93lo<}1Tfq2ry`Y!Fm%K2iS#pEU1xLJwH;M-4
z@ZSHOf}~fc!8gS1-^&W*D)HT4?{^m|$;nP#3Bd(5(mlRDM?S`(^4*sI#eXV&H}>b5
zW}>cCL;1;J2IcVAWpmg{9Y@=@o93F!ActfB9VZt*GY@HFvqpoGR;IVS7irQgLG!C|
z`wMJ49*)4(G!pQ*@3ZMPBfzq>E;w2J{bV5yVItT*div!O2zRhkx|Y6e-NE&uk4WEj
zd-S5{l`w-;)e!RzteR7Q+RZt7x#$9uvK--I0U%2&8@lAmx5!l2QQ<NylmqI*`>|$S
z@<3N5Azc;`Q=LzEkiM)mJd32XFomX264O|b>d_EVAnqX_2G6zdaNGL*HNfyE9y5Mp
zaHo;WC2PU6Rpevs-0PmgpJZs%w-1gQy9XdyP?suJd0_3E+XT+I1shmYamK5hQEQ>M
za(?ti9KPZbsM;QNe=mO0mY8`(+(?c_!?I}ByB$kQ0i6tx5UtW3E;$1L6Hh5a0KFug
z8KT-ElCoWM+VLx0am(&>fEeEzjTrm3aX3{qaElYRhy+@f%4RXjk>fwNqT?RvT>t$h
zM8gRov;)$lVjH8ZMCxcN%4m&OmrF=J-Nwex6dn(hJYLzG+(yn8)Nh|k%|vUq{!pB+
z`fcumA7kRS#SgdJU~MA)#abkM?G&gf-7M{-uThiSp7$iik5E<#%at3ruyy4jJ*vJ@
z7LET~Q@jg8aChOv^oMs8Ztj3QnvgdZX+hAxsZlA<Zj@Gjde1>GCuYbuK3@9$s-&^|
zn^ebBxK2X$!nSN=w#Jg^pq6$Zulg6MS9ccCydL&eWBdq0wnp(($<<wvNRXoS7pWDm
z=yiO=VL0bjiFIm?icw0RO0T|To7==g$<vab`oal>UwvAmyse%}^IlID@6o$`<KuY4
zCfP-j)NFV)oJ+rwH4Q~qOof5RZ#ps3oPI+o$bU?8>wa=CG^1qjo*rH(Q`>he(gDeY
zXFauM#;(DNvz^@Z@&zeMii8c_jDuKH=07ya1O0m}-9TZ{n|6Ipiw-^2#7Ta1ZSCg2
z&LGDa3OrP>Eh1uTY<k5W>^Ui}_-HAARwtonk=G~PnvuzI;!YnmL_d50_VdDDEokal
z?>SpPwyk0jpkB8AK6w?Ke$$;lCYbaw_=BIQzGyV?x`oa;;A!ySo7f#I$zHv9b!q`s
zv*Au2H<?Sj|1_^JstK6b)tj@0uH6`Hn+vP<8@9M=tjOgZ7&!NuOn`Kf(S66sNaMgw
zs<j8<n*5cf*pI{B><r2=J)+;Du!)#(b>8z*RDSFYYTz$$2-0=h8a3CGY!H3RPjIhm
z>weirFA+k_uC=K~4OPjm{oXnCy}cpi-9*E)J^ATXnCtn7G_%XlICKdUz2-l-7yVcD
zAV?ceSHzrV7ITJ#s@}hWZx1t@Z)|7T@Cg*`_S%8V)^@#V1Wnu*0_hYyYy*hbp$V%u
z>ZOk6q!{GpSRc*79s~@aNypa<<8V<hmNW;vzbh+Eva^RBmbxi>2Mz^j@bB$9yj-Av
z7Ro;1HeXtJ8ft`0^%vh3Vmj*qePwBqzG(~fJs8%isu{3LL%CoVON{G#wwE0!8;A*Q
zu=})4`TIr}|C$pDtbW$BRZ5F#6ET|XzLT8zwWn#P#Qj)G3u@16b9aF`BM(p9M4ir|
z;#vGxHD<P+w^1Bbty=6H6rXGQ`JM=X4wvFmB>JbMbq=?^nr1dNtx-H5FVR4PQG6$o
zQAn3J8y?V)PCc1`t)8Hc%u1EniP~7dvc+jQtNqG9H%AV8Z^-h{OMDh%lZbHErAI;<
z?s_!ny7xr`l^w6B5Y<ZO7(&zHNP0YsnjUa%ak$-beLLWqv%gY*A%mYJD<!EUJUz7X
zrDxCXs9p?fs03mCG@*!!%if{?bFgi2f}8`q8;0I5Y=v*C6Jov647}bb=ftIGqbyF^
zSrT;{KbW?EH5ecm7zvc*)UR%3@OZcp+{6;(izujN9Y;L@kGTz(y2ey8oj&h~?IdlY
z1S+o{Es_`w+H!$yvh-Zxgugbgt9f7S-JRurvXO3S)U^6i>x;y362UOh(f~kBTmV{W
zO`TvBcbOV4N-RNJ3RThVHRiPOPh<ZSF<9!|$=~xw`;JS?JREhpTqAgTl4;QUP8FVU
z2r!2wp^*>5{b$A-HnYGp*=!G~`s`J&`bq%+ieX80PQvj!bS+qujm+&6R7Pr#R9P=C
zTJK17J4J)ILQT~aE`PX8V|EdeUy;|RC+U7|CFjYd-IPxAgTBB5LL04`+aRF8yf9Bq
zj=Z%Z2ln-xmJMO?J)JpB<5ZvrU*5bSLBsWxHKM!5`^arrGQrf0)69^cAJg{wbw_qb
zD0Q#lRqIh(>rSo<z{+{$HT*bI+p1$tWO73vtZU9f!)f_y_2$Zt9Hf4)zUpbWHOvVE
z8(q`CqltJ&u%`Y*`s@205Z980O$Z7t+m~mhKf1<dP0am^x<_hd+4C0DX1Bk3HSh*_
zK8!>7Jz_vo*2%4tWpA^-InBQzm@!#Zgb%&sa_pw@WpF4;JqQJ`;z%uNCR<XUfeEFW
zBN7z<MS&svSqNm-v%O51O)u7`B@8Yay?1SJ=iVi}_BCdB*9(AVC|~cs{O`uq&ak@6
zGiH&BL26{p`c9F-$VBnQyK^)=dOwb)57%EdnF7cI@`F*=!EmU6ul)#RMX|}`RWOk~
zdTq)sW6npS#74m@<MwPxU$~}~qj><*%kiCEpBoqxDXT&wWod<{sfU_~QE}XRl>=OX
z#*CBH&cxjMrJbWo(581$y{@Ho{cZceitYx39P3ro$%>*E*-Lq9z@q-5*ZE_CmJ4k2
z+DBg=0<50<ii_89bco|BgxTaly3^KFl-6MZ>ZqWfzrDWG+#CIg+4CJ({3S5c?TG}^
zUU0<wubru64$&-#NqjaQu*NB<3CNXUkh}I}|NgymG7Kor>{LLM&=)QDOwqszMVA|-
zuJIl^7-xR|chPD3AWo&`)x|IA5wUa7@-;=vX%^b^NW3|~OlZ-t+DYDB74t$|rfV`#
zm~}HSL5xh7ViwbSwvj5ucT5`XsC$D_6ytRVPEwAPaw~$)|Iz+B;J@vWHV0&=L{6{7
zxKK6+$obhTsShDdMXvR$SrHfO%7ivYg&nG%y+0-Ndo`8`PYf>;Wl<bLGeR(z9pd9`
zdIar*%s?(W^!dOp&^Yt0SAX?x!g2#eTZo_UJro@bIIe2kFW747kH=hPkb}9L%mGEq
zwy->7Cs^#mN}rcP*Ug|9&W6<oeO@}&B@tY8jgO5c8NPW4oj`VLgH!w5a#?gPE&jmX
z84`|ZAu(O@2_Xvfv&gJe%PaEZeKb*AacNi>AqBwRhGH%Lp$Fhjhw<2^lAW#_(6pLE
zigR6wUE^!^2|~Rh;s;;-(Whr)4wR*j0|_qv>vL&jn1ky672F-|#vQK?b7oL+F-H$9
z_5nyryonH=F@`F!1g|x!K<8i6r5_5>sk%KdA*|;n#On*38t0UZL(eeU2{L+S86s8h
zey7Yt5VRhk9naE4B9*@VlGvH!@`$%m^(lEFg7d5H#JsA%j;*i^_5-a>CZ2ET&K;Z8
z=QjW!7UrpPJtY7R`4e84XJakl&bV;l?~_ceeC5E#nyVyQdaHE&y+L#<-F#Q#x|21j
z6F=?r<_*%)Z*wQNa@5u?Rncm<lV$8-b#tgPJF+@@$_u_Chmhau)knNhmOAZbp->|(
z0%*#L3vCM*xiDh-WKv%zwe7|K;q1MGn!vj@PzX^#km^dPp$Y6NB_g5s5?lpDRzX2p
zlpdrv0~kPxKq8_@uNG_+DS{w1Bp_W7LI5eDDImQHcz@B|{oZfB_s-n8e>p=L;qW`{
zInQ}cM7i{oLEPDlHa3QFh8LpME^Z!-kqF(DzWaC7EQhJ+W|-=7fxN=Z^A|=iMeQT|
zg;|0~qL>zjxZ_UXxV35@DN=efY@ffGMCm6UgS!oO!Vcy{J4C7JIi=U!hKfHgdYRS-
zs_=So0YpSM*Gf(LY0UIaDD8EK_!O%g&#sNX>8lpD+a$tptv(CxOi$e~v$ZMY4R1UC
zp)S4zqZD{x80Y)dFt;(A_p(N|_@DLA^lfnObpxmp$VUr)avH@}>s9%x+QudllWd^^
zdnyyZ!5Jew-#`UoRl-;J!VzDIU#}aV<X*CcrpL|PRXOh@$Gf3Dew!(sH|{fHD6ITr
zFAA>t^|3M`a5Bo-TD03MB01i`xEOVBe*%8saHY)21l=6)Df2@8-fDYB*q*+I$Vy9#
z)D#$t%hyI*@Zj3S6@Zw`^!J6!*&Dm)4+R{b1D=(yWql}8B;{^4t$;2*d;PKWLptAH
zID(ZAT2-e0S+mQ~I3P?n=sINS*&;bf18lR(E&SDdmPJIBOS@Pc7tMAE8(noHjJNy%
zcj2p?r=?w}Kbi*fU0*wP(e}F@8{cSE%^<HR2C#zCL?4S!&Eow7?$XvCl9|2oOzk}G
z0~O2P*^V+EEbV-2`@u4mwjE1KBQ<R`eh~%Lmt)m-*Y`eAsQ732j}^Fbp6DN2=gtTb
zg-(-PpfhSXr`gd>8FKA}S-a`E%5Z4o+(VDCVZWu(>HB6s>Nm{I0zI`=M31VQ(0M~9
zXSDe2*3t+46892sVhe<@%Z*v3RbUw}z=zCzkh2f}0=tQ+-Fm+y%XN@ss~&U8uC7*R
zYGN&YEGp;6-mH3Q^#rT9moon`+~IsV<yILi-^6z#Yn43qM)FpvEg{A2yUAlXJy=W{
zMOA`c1_TZbJ!;m`Au;a!O66gN@G#4q@PnPjC+njzC4SQnEdv6MyjytF9`bFwJrNF6
zQk>Ohud3Mb)Mu==GSs)PR=3V4(>{!-lCba@EMeBFL|5f)vxF|t7|Bqo`9Y5*D{@ui
z5k@(Hvl0^`4i(T8{U~aU^r5XaaRtcT`Vc>M6_0IP>d#O;zD9fct;!oy_Py)3vHfSk
ztWD^g)xFidw^+Z9iv4xF?mvF)5|)JeoM6mmVLy~VNNfrYiPaD94S9dDxT#q=a%*-|
z`DQ>F{P?KIttDH0&}Q;i8Y_p3$YS|?GwNV4?MF%{=YyJ)uF)v<Yb)<xe%P?wd$T5@
zJ~*h%E1A9_sQl<Hb|Pb1T6q*XI5BrC49mP=lp8ne!!F=o`MPGPu>GAn1i)+fcD+wP
zcj)BF!|v`wf5lH7Q#leV#I()AVwd7-(R`@EGv+Fn-)3o&zK)4QW)RnfWnC0InjA#T
z1A6igUCc79D31DYxZ5hi2!qg8Qd_e?w~wr&r1_evFJE$&rDSkgPY#i@!#eD0vM8YA
zac_PqFV^r^OB-)YOO2JA)ZHo=)!Vzg0Pt3$UzcV*+l1VHkED>93S>i|>UVcK-L)J~
zA9lTS^}9;oEICzgIGC#)iI{UghsC&;`Fd@5n@lN3znhEK^tHR#&bZ-7x>B~G5Ax#%
z3%ibnYxO;rGalMU85`wqXnc*}4K=SCo&$wOP)uw?+i#{MFI~=jMm{lu#%ilizd6c(
zvLv!yaBQ2@we~)dk5w4W?NTz^Yv_GzA&lh=R^g2=$NX0#M{lhGY~Hk7_~U(Shw&aP
zUMKRVq+st-in?RN9j>L*(~qCP=QNs{(R!J-Q&J?-eO(Yxyxx4PKbWs4{W)5TtZnCM
zp6mWWd0W4~h}V|p9GVnbWp}rvc&23uQJW-8CbMQ26o~HkYZj^$-UB4!saF&4gi`yp
z+)<IO@}QN;rYvv*E<QOb(?wFG_~xp{^zUY_a#l6XTu7Usr46e`o7l1(7C(LatH21>
zT@GaU#mn{UmLE*xcV4_PIb1s5QR&jBpNmUqJ|uJ}{=%MinQRST5R4Yi{pR}HtlTOW
z=mf{a`_!OgpRIRt@Htl&8d?n5PVl&_&*D8k3sz+a^27)F`|3u->m?Z+WxO{k{D3aw
za`gFwV4cb-4*Snr50JRSp(A-lHCFl^SE_w(6#NM>F_0RLYv_;||Nb$T^-QgqZlyj&
zmk?hWI5s+lT&kU287(vsmy{a&9;`iF<&E}5)DN3Jpgiedha4Q=c-z=<eNnz$%S`2k
z&}PQ7zogAxpC9@0;^uyi&nRb|mUoPnEFA*t`sRWC>@%ss!!DYqcYhoqZ$8O!W`V?g
zws_>P#--;n*aP1Jmii>_u!1-9DCI`{Hpan{D{Ra^84Qy}ZI*{yGkCVUpwDFvw9pBo
zYWsCWtmp-uQzB8zf}LY%jm!9oioow_C&{n*aEFCNZM;wIe<RJi1340=9og1unB(sY
z*w$N`WAeY%E$KE)axbvJ-kiiE@7>zQ&o5oUBWAC1Da^iAtUMNQbK1Xio^itIkFLn&
zTWS8f0x=i9sBE}{(cH|BIP&e@oY!mFol&#nIoCERH3y$3%e^#l*WjIg-NN;mk!6|t
zzw15@-mWA3uJO*L{pry%?L5~^M)i`~scX)Uely#yD7u3De9KaGEvx(Yt=BbA=C>72
zZiP=;uswdWTTnB9T26CmsogL~ke$EQJwi3#rq<8=O6Bm!HxB#zH`a`b%sGk|sy)tL
zc^7-`dY{DBfMEW&x*YeRHM4Eu6foXWEjpW<COk6hcDeZNr!gbxK~-CecG6YRBCjSP
z@zz#8T2jCwGITw=YC2x^kb3=N#pt8rX;SzCXQ5kK$DS8(hOP}dNl?)4LJJ#)d~9`H
zC%S$UJfs;qnVOTHn$UPtAzIB1s;SDgMtzE`51Y~{NksJY64vwv-@?jolE=pl5~i<z
zBA<oe8{AMb!;rhh7&1d7@*rXOEiznKK&ePuDW`U3uq&7+2En1%KLPrhmGKbKPP0X!
z!U{4<b0?G=Hi%ddR+LgLSInte9(lBX5;wCjv@4bdO%H({7R@fVF@_4*M_P?5KB=BB
z93BnHd@$iC+8qWri)}m$uYBrxPSVfoV$3y-5LUa>)rB%ikKtgm`0E5z2ErW?pFv{{
z^sCki;8=d_BpH@~%5^9rdOzYZ)$>MKK8V%r-*O{trRLj)%`vBe=H#*|S@dNsO?at9
z{jN@*Ilg!N$30rZX5VPkXjm1OX0{i*$`JYNur0(qo*b<^$#13t64QNkUQkUu7ge|h
zO&{H&RtQ-JW(BW{Nk|XTgqSPl&nJAjL(~1H!%N$8(=Va@Na>EYJuSj|sNQpm7x-QH
za@x$Hc+cQ%k15}$vKl_}T2f`2UAHpF)ZcNtRP~U%cx{5#eDA?*K=Kc)>(e-XH-60c
zt<o8$9cgZ3s1p<@S1_N=6?4M+^^NUyj(Kct^U8(P0~-AwK;c$jUnu4k-ysNxdVsfB
z))p(vO498}5n;Y6F{8EjoosIqrj^cz8MS$5PU8+sZib9-PoJ${dAD2mG>MHw%Uc^y
zF`2$woPEXfV5o(bX6l;Rea@J}eET)4n8^Ku4-ZeOU%&4Heuuds*N4fkj*KzIHvZrk
zW8SV$eh_iA#vv34_w-}`+6RLcY|XU-&-WVx(XHwq6#PK9Att%Rxi>&)+LC7RZ%>hR
zZ*5pnKO9xytT}(B7yBpXb6|t*ggQIaezo>sqxfdO?{rlnkOIwLFkJ_9?4BoQ2v;Yc
zCRQ;yI+qX*FV&AU9JBMS<@8Mmx;bmUxDW&*@J7K}$3F1YE#H|51grUQPtR2wb~W3a
zkz;x`b;DXq!QJkYiqSe%Vs+q@N5;LrG~OsgtZJ?2=Y!3+(m11@GE*o?*M4aA{fDyf
z?@ZGT%>=<4ZQs<#wkhxj8Aa5CEt*WT*f))dFY>b8*o~Y=E$v1DkxsuaX!!07&KD$h
zhuH-$<0hmYu&?B`J?~b4U@JkR5Uo3wF2{PNEb_ZeCsn-1#&R||Towy!W4=cpe#}y}
ziLg3_;o?psGq@)52<C3cHsALA@Fv1p=?n3%XR-&ep;QUGaaW9cMl%8+13O1a4bjL_
z{=P?*I~#d~_o3(u5~Menuw5^iWkMadyvHt2^5%5^X{Xb2Dsmlh>Q_cwBu?q+8AD@L
zj$uE6^K5WZkC@1@-SWi@K|$aAcvaO@7VlE;1Y{-gqviO<bN#H~eBZ9^US;hN+JIAE
zK+&9QjokN{GxZ1?LGKa=P8zrp?y~-SCQNJkfrXl_&FyzU_g8&C%`@nuPaT(wcq4-z
zo3#BqR_X4lwGNcL_k_adw5ObK<VX@x*lGoer)A!DU~@G6xS^JGZ1#)@F1m`oJ+5Ay
z1QPqmrD@+ULm@Sxx{a#j^g;0*t^m{o@mLe9RDhr^NutNTnd6Vl9!P;FG(DDa6Vn*O
zYrEj5Q#b6Hv4>2F?D);wrMud?-Xxd5J7<t{K03CQRb5Wz*2Q9Lte?%Xt7T4|g~^oO
zlpoXDJIUH<%cHnVhH${`A5s*N?hih8zIOZA+6{H?Ic6M=*Tu88W^cjTG6fy>@|#+j
zq8+ZXEjKW0=y<RfkPY<7j&FG%wZ)`mT?HeJR`Xr+GnenfXRF@{U)l;Q76QY7teR-y
zDaZ*z!TfP)phi<#QXo+Cx}{s|+rhLW8$o35ghya_2ZDSo;N3+lX06&W??=iX6LktS
z`7Q@Z9?JPdr)<cndE+A4#1=xLbqXie#81%uo*4&P0w8gvb@VLPeYPdRd--7bdr6t?
z`Yc<(kQz|iS7V!b&z>QJH46b+&3)#j?c!reJK6EeqX}Ea^<}Hs;<#EwtG43Z+x3RC
zYPiD}QcjJPz1pjoO+jhzP~J3~zF+#YZmUwT-K=Zhpn7RD!wGusfEf$5D2~Doj{6rA
zQ7U;fZsmoK6P_Z}3iku~9ZbiMzvL%w8DkAN1EzkwtlxcUE7yq^yDuRs{k>}1lVhQY
zY{J1JnXs8CI5*UCcWji#3f0+T-r1!E3aGy-zq2J?XZgGDL{%a^Mq{<3jy+j#8d-@O
z5JCrZ`@hXD-P^8{#tGC|)=vJ`Ie%|ZVj3t)2dV-EWHqrK@3PpX@Aq6(<oy8Y*e}d|
zO||i+6DyL(RYhJmuiH~kL;P+~nLo|m*?h<t8qLBOkT>M~$>9z;aI|jouFvT7*zcSk
zGo-B@+4ht{;ioz4>#NHs?WZHwfz+Mp@M|{TswQUFZuM5bGswNeP+xjqrw?)?`W;BS
zV!1JrCZavdhS_Y7v0T21@Vt~l$Rh@h+fzKG9`h-=ekU3lF1R=)*b}3CJEzpGOrP3l
z+L&T>cV|LIA)1Svnt(R56P$H1#bhZ5usjy?D3?e<R`N?|X^~!cnrbb?f^yzPf@Fvi
z^p>j>oW6c=FX+W2G?A8{xR~#l_&T1HgJgJ4JCeZWR2%9A8?4J3(^As2U3~rp%rm6d
z@}=biSeQEy>moXC*f<BR1)0s~Uj(~@dj<zmDW{9xJ4d$CQoRfLXEz@+&0lVCZdv^P
z(vu_^n9I-JpIx2_)CL-J--I5Mo77x%HFIbAW8r-B*-E;ucW2gX55}DPEFF@NJ<$os
z21o@N!cmnTLmxByzXMcE<!jO(`yFvxkZ*F+`AU5AV)ow7Na^1|medtfaDkW<2b++K
z6cLa3@Xy{cnIx#rA^A$GA|G9QS{0e1o-<6~g2_uu_FZ~;S&5)Hg^=F2(HQ6ct2SqO
zA97lT4^H9<7!-&5E^Y?>fu(UMFNku(gOx5X2%UrS;2J?WJ1!cS>yV;R`Hjw<E-Mq`
zLsSQ`i4m@yz+%~ndVaOi?<bDlgjJ}>^_P8*L~`hTjGpT#O1|&hx)urF(@B2i%*YrX
z&vCy&VYGcbNvM5u<k`Y%YiPzBkM8&Fbnq>4e(<4s(ZLmf%~kYi-dUV1D`UN`@Izi{
z{MHeIvl=*OnSSj;|5|9mk7AjHKSOrEJX4-}ST)R!r0D64|KRf34{A#MWAC!=_FhcC
z`T*jaGqb?KO|c82dq1e2$0j(hvj$AZ*EFhh*wn2)EKZ=R_Lu7RbKU<8089eaoRGmy
zC45X@q#?N>M}sm!4_zQ~qT}evoV_XOXY9XSKl_<B#8|kWE}=4ZO^jR~H&%Oihkd-F
z#rcAy$>eOHDa*rJryb366BWKQY0i!xSqM=Pt>wd>HU6<@Y6o^Oof26=9lQoYlN)Qo
zlx={n#_5RPW08cg9mU1*xRk^@j!{bk>_P{ICGJ7f7ZF!l({*xHt0%oeWw-j93`K&D
zuCnc+reefGsu%`V$oYIR8l{5Yd$X(73$AsT$E14)^CXye6oZa5G|u_F-_$cBl=ZFs
zmoNASIjm>z;P<xQ*hV2}7vh=1W}bH%QA77*b;EX90xAVtt{-q+e=!+#?Unt<w-}M#
z-t0;?v`8;rOZ=W(wjtUXi#85vWESL%IOu2DERRD8S`SwY@jIWwf6s5?dq4lAQ$Vz$
z$EZX5<FL~?J8ieSmD^X{%0Nn6+}LJ$R^~JxbDhUU<-7%-*;SCIzgjs(8Hj=?%Y!TB
zPW6PCfGO3pFfigCTK~MUc<4p-`CR_*#vSi_>2sSy5r3>s%ZFEIf}?o_Qf-$HMnc&U
zv26ScryoJ<&ADz!bEldE?LHx(_heX7CRSsklREKZl=qp&<ozu9AnozqCkdsB5rjp1
z(*?cTng#-Ltw8;-Sa>6qYf`^xBH#7?vJN_+Wh@_^bkZp>y7^&Ncdy<4Cp!$wCVgF9
zx>n-Gt%qEKc(Ar%)cNCV{VX<UVLllES0uQe*7cMjo><+5fTq!&S4<yGGBnOy^F;^#
zW{jJn&1Bdb%vAe~JY|bnnbga5B9DQE{PJKsXG?|F{soBQm5DXRQl#lytua|KAKyHj
zAhgZQQ|nx9eBIa&#<d$+;7$B|8mA<olg`fA^{V<=V!n0#>*mUgF4z@<(fBrva8so0
z!_prj78>Rg-#KS#Tmk#;ypPCtnUePD7Xj7`a295o{-^qcG8}+nbF@0A=c^m~vR;ha
z_dQH9<VThv<41a)=AxbnUq8OXI9HNntpo2JsI+0imKw$m%WbEPQ7HL+DOfKVfP{4N
z&%QXS+MBGst7e*5b=)J?`TSnno!ofDoH{<y_jM;e*Vk6~M*UI9`AZ2Cj#OUhI%GLN
z^D)2e9_Pl*=dySUHXaP=P#iKMdy<Vc3*8l0xAQ63K9jj30CBif(>OsL?rR@U+#7XF
zGt=HB259xCP(7&qMJB?WKWu}E+$c%hVS`;R(*?RXzc2wCWx;&c<8KRf_(Gp1>nM(b
zX@oxQ4h@jG{(3qv<C*s8{lbkbQTz55sjxlY700pAt;K|EKx+Z5btE61ng||XDiS=L
zOb*D=WGCj6#&SGBw?~K&Rs)u)KLB;eD;m1wHr;>}T~Ur@%GoKPE3}m%tQ@p7Uh1dC
z6U|0h$ZkLzUvz@wBzrmz5U_Q2$_`#Qr*^#Yhj>)_0|R?0?t3z5%s1Q6bx`sZgT=^L
zn}NT*J+5J!h1lIC*^@vR6qT(Cp2}LnI$r^}hiEfia2_a}d8~d8k`PLKP#vKo+Ec>R
zSy3@!$9Da`j2AnXTbyM11jK;jQ0Nh~7jxn7EFna#TBbDxqcN-g*byk!#X?akFb}D|
z7PsQ+ULazPrXX{6ItxT@-9;3ZCY^N1srX|~h6+nwfw95Y?6psbE&tWR%G-Wc5%Hb)
zOqefT&eYbc$Gu&=x2!Dw)Ct;EcX#-$)t+Zh2|c8gkH`HeXR(c!?Iv@b<-A9*)b#Fm
z>B`mAfG!*rF-%0o^4mqd8r#ELEL~(Hc&RDEC!j*Sk0g7}3%x8q54@djb2}DxaGQcs
z2DZuTr<ULJbxGJ=b8;I@3zDq$$^_NshyG!WyEkg#-$z)xX&D#6H`xbpFNb=)V>$B*
zFU%Um{(-*y-Q+27q<YkK1`YpiH9NU?Ryyl18I2<xdf9yXs!87k>sJI5@#H%oW@{t<
z8YoN90a_e84D2eJIRLs|juHIGm?zfwj<@r+#f>Th`}0ujkGBI=084&FGi(^a89_mN
z^7E%PhK5LRX{!S!4Opnd_tt!w45~y)DD+9DOLixa!sRv~xS7Xl#dHjjZE%PC;LEs1
zPYhehljEwa6+@*3i4ArQC?KM-KEW@#%06dNo`(_fpE|8Az14a1)S}U6p~`6;LKk(%
z#WF^14UyuTw02u|XXOOPF7ihtV&N1q7h{X=LzN4DN!<HHQp1~6nT+eQf`eqoCNGw4
z2YK~4@STvuueit;ZnL-zk6F#FC^IWD=L8K@47Dv$SA01bzUjsvbVw9E2@pga7AA4;
zWo}JUk@d{Ui2cI0C86aO{7(AV4$R`mjZZVTM{rN`d!7MikXge=ji15U!2YwS0t@7*
z{VD6Za-oe3vm))1b2|sKVSgdVx&s%}c;$tpaZbyrYXZR&Y<5<gV5;~AyR$kYc|`kz
z>JB7W3SNCWs6)|FT!EkNXn<*AwYEEl(xz&Bh}&`qc%r*WK@;9wZ@cd^bmP&(!20i_
z!_~fe-cLnF-&|Y$%g+!{$ger@t4&wWW)E}S*oMciR=$1z8Vmlf*cTqs5XA@`$tGgd
z*H0~4nD6~7;`Tf)2B+r#PAmqw%8->Z3vkee=Y0QA<==n1x2W4ov;`Jb-lnBx4S6+Q
z<E<PCI?wj=;9!$Prs&-Z<4g^LxlcLGZaWH^GVC$zK>8{9WKL_hDoAO!3VL^7My4@E
zXjheUP4MI{FE64<wJ=fN#AdI=Jzj0nqGS-RJu_ir<9ta;?>tl!vsWIs6BnQA{#bbu
zU9<E~_|+1_Cmihj$XSb*zKBJ`kAu-yYp5QR7bK{O>y?beqEiX1Z<Iv6Nw-st6XG>*
zU$lUZ1gje*!kR1BS_bz>lAOV>6BFezEtK^N-Pyg~KIun0m|<yYV&;>I1xwa5vmFDR
zZM<e0rLU|?^GypLxEEDFrDsi(TeaFTLO#a9-+Xh15#NDNs21E@2Ozjk4m}Fe)9|6_
z20(KtJDS0VUyGUafX8*@fCN-_=k`)$nPG15y!F`Cpml3yhd_bsvFy`Z^nw0}wh?dI
zME#HN^?ZaUoAB)h<`1sVvYeFX4eX7$@3H8Av|aQ>Q$e;j;<S{amyUGjmuArkT3Lw!
zmAy2x(j&wJaD^pNy~pDf8?L^=KdflQV*C)L63!}>Akn|fDw$E(d|XrMfJnNm?H3y`
zC($iRRps=UR&z1Kh^TT<1Rqr*``52IWAFkJc*iaZIY($Qb*@!99HQ<k5JRa%AH2xU
zWEq&MMb)M%@Lo#I^>&pw4<s>v6QoiHoY4B(bzBK%92}X5=Mq*G!`zM+mh%Sil{;yK
zMXJXF3}qG;I+q-;7uFKip)ctDy1bX0Evs^~%}mfcGpxgGY>z9DYf>lo$|d;3`Vi>U
z@13`~ucPSw>f`CZeJ4x|lyE!#8oo*(>X&$Nr(z(TK*-}VUUeq5h<QAMO-`nMyG8yT
z{)vyLnCO=X)c#|3d5Cj%u#@U~kAC|*|M?G>EU#Un-}HGDFZa$Ri0^?SJOnD)RC14Q
z+TIjzc}UklVq}LHy<z=nU)XV|lwPju(jVwCwW5j)IlQwv_|!Xz9`T-6uP=k+S^i;B
zn2~CdS&l$9!=VF4r4Y9&MlT^5AE=82b({d>^u(z-ic*Yv4{O#!7i4&xdy-L}5j!v4
z{yK$iHDtd-@e%G=f+=lMDbiYo2QfRXgdWS(40r!)GtJSNn>Ev9rzY`xya98LbJw3=
zxi|ij+;T@jMAwjDdfN(y@k_X3<CXASRw-cSCb?KY(VoV3!P;Mhi<WHXt<|ObJ{XO`
zV&eE(_(W+%+!xtZxfao3GsbW@z($5DjtAtb-l_~Up5Ztn6#YD2T6PJj8a`)kXZbgy
z!dN76PV8PjGCz$Wre|NtdPX05$<&`6x+j|sO-JZ_$OBJsP7r4t6^DUre0}+rbR2Tj
z2j~MT`5&D9C*JemKC>*8G;?&)s^B4Wc|OfYmCq3LJ*?lJMXV32Vc&hBmcF6;#wa%n
zSj4uBcXwaG|JEkq1(8|hcPpD<?up^OtzbH=wa3t^e76=73X|3nA7m7!^&YQd42`da
zy7^6&UCYSY+iuw=5wk04+xPi2D}vq103qd0wL>Ili`DM)0{n{mTSa*+SExru0e@x(
z{62?XLUWYOy?CiQ-Wwf;EHX;8d*@)YheUg!I$X3}y>V`%x*ix|Cyt<5_tm{jRUskG
zPSEXcO%2zhy*WV4U6Igs44XxX_;SPZ0y4E$`qTQncZmkA4z@s`FPYi;!-qd}RZp^>
z(K}PB7MPm`Gj)CVtNv8@!KE^n8+k;#P}zf0GTJupaP>6^5!|NDe*Hd0!1f9<P?fZ5
zkD)hBalJ2#>q0I#D0V>%K;ULu&Yiu_i{sbR5l^UaC4W_%QQ7!>Yw23Rh&R~>2+uua
z7Fou=PkQrjqUFKGHcYm}XOKfYspZd?`|o;6rWTfAgwg(eEh`|}$LZVzRqV{ePha0%
zMGitZ?JZ=NA(D6lA6B&^His4_pmfX2SF%)Kp}Q27=+7!M4(GcVy<O4zd$2*3shDTn
zs;m_ASThaMvGwiKGb+d#=aD_k0doN9M0Dk<w$Mv`VRip0y*b{PI$tB_8l<$eRBGl{
z>?2XG_$H#v-4M6qxI*5){QHTDhc`u7);AADsw4=;XgP+t?Iq-C?W!7eRx!UpX4Lr_
zn%T}eu&-GNzC^0m&5*OVG1??<H&U>-nwgv9B&{}+EO#l;J~XN?4KStd8o<HyU#sEI
zzP}@J@STRczn%Q>@yWS%QlemcB<Jdes|p|{HV2RR(91^!7_=(2ik>rn>~B4y9|Tp<
zRs0a~)8wlqv-!Z}x}rP|j(Qtk9diN}<iT8dXYozg{_cHZFn7D!KapY%-vBNJTn-rZ
zAN1+g+&V+p^hmZ3*%%JV5A5^rPN#yO#^qPopluC`7vs_=<AQ@3=D619rjEIgQ|ks+
zkHGo}$o!nrW!z1j93*W<hk&mOD^H>f4yNys8pdo89Edr^Z`ciIcHM53wc8qz?^}XD
zlDjWv2n$lTC3c74y{41$xWi&3hsjgNyt(_0p*mS?4kIlvrVFC+qIg%c&Y*<n!q8(a
zsy*bCI(aU1FN#_@^`N84s$9AL`f<nHW)Ar9z@<-O4Y8c)W8W#MWdG{p8&eMK^;zy$
zE(PYQDhB30H(nP5d5-DT8I!*gkR^-8-UR2(Z$=Xwrr(VDFQ0{UmWY7f5;WoL5Jpqd
z0yt!M2h~ty{nnF9Opyek*C1@`4YO~6U3Gw6Ln^Y2ZjET=1n!05w!;C5+%4PxX9NE;
z4wZA+zjY1pjNEQ=c6#d@hPjv!IuCpEPaFoNB+HrN7nj}LC%8?$IL4K{vf%e)H5mMS
z7z()q+#E)-?0eeda!pSC*{{eEsH2IIq$(QOtj(Lf4#fnJO3`&BuL92R{l`3FP1tvh
z6UeD+;@Z9s?QkNEw&8<+ns`8KTKR#p=1gGGT<i^x-OzJ3I%8ed!m0OoOPyi^5|$uB
zzLS)a3yT}vFgt@?)>q7VS#~|kiYcWVc{i<fo^!j;pxDWY6S_@sVH&Bg@`ZrrL2N)T
zd5_XFFhJeIlVP>SDFZ$_qqvz@pjH&`jRCbHXUWGizuuI+tysgM!G|X;k?sOB*IYm@
zEVfOlPNj!jCdR;=#%XYjxD$1C9d|9=lK|@RO0pz9F@pCj*~leZNuEawE^dwZb~j1<
zPQgWIxzH>BU&V=m{VgcHQgj+b2K@dW2;{z1E=)JRTZ$1Den&6Jc|nWdL5~fO-GZ;l
zF6c0EHb4+}+z*baaL&1&%hZi2d~0?zkioj8&b!CmY}niJ_nDZ!TsY#yz<7ZQ%5DAF
z9yQ|gj#0S(z;=>_=bg%#I1M(xnF=IFXbWPubaDIf*Hrt9jFoB?d{OC!yg5w~GT?|w
z0aDQ1ECzv7mZF*GzYoEDnI~lJ^!n;hdRz9q7dzs$kUUneXlh@*B=G#rnd`0=H0Kej
z$L=?m=Bm1W$^<Hm|F4<Y`5r>)jS8U);I*FoZ?83w#j7+yU6r1QX3R}jpc^HpkRTNJ
zJod?^o6Z=jlGIj^QB`v<%b9|cI>DWK1aKSDj)Z*40O6Y^HonB}zBnAZZjKs0$Ty5n
z7d}&3z4iH42Ki!->3<|VZfF2#j+=r{YpGXXZY}D=xF$O;SfjpszEjCw1b+jVAWkPY
z*s!9uH05Z;HsOm>xC+z@GH1cH>vd=fL99de&vC0TmH)ncxDc%|*=l$1jKw8k4kp^@
zLg$nEH8D7PQLA(UsZvb!fT~w>P*nKbCd(Illl1e7W!1>b1&i2nUt5almGj6?IwXht
zB9eU5L>g=pawl7h&BAQ>4v*Ow)ngrjR6|cNojtU?ULfmDVlo!tyow3Xd77B5($!Ak
zvF^#JM+)YdohNlX>Do)+j-(`{!L{$^C+<|}7YiRO=#+(@=*i523gl&2WdXX=#DmK}
z9aa*u#)qS`_XD5m<PX&b`<BV>D!iuK*suIoeq(`Bf}_=!#k##LClZx?IOtXupj%lW
z><b;FK}(3T9)3v?DK37kG-dwn6$(rhYI^-{oiRk;3;*{CABteeA!Nd7f#USsr=$#a
zf5xPUl(!<?b(-1x|JYcy`l_uRXq+e0wIR;8#hG=w7|{1^gq}rn-j>B|L#X4kRS1~=
z6i+Uy5UoYywYFR6p}l;M-u@W<Oq4rXYjduUHWC|e&r3_=)NF#4PU{w;oa0J6p1_eL
z$>_Dgn1jc6BrSfEI>ADGvq((5nWJ`hQ;x!<W8+WlBY#E=lN#`q<5no3p`$<7cl<hw
zLM=mTj2a2BS;i_=OTA4KxS4c|Ux)e~!FW&g$<2@n-goC410G%7Y)7eFI=1^jvS*#u
zD*bOrvqK$P#Za?U_Z=xIaGX5)`!N=!?85B8_^v}I=)ji3brvr@9=>eiKUx6ETkazD
zC|%L+5x+i@&hH!dWTl^}Gt(OlAq^STCkFN+VqD>tMC3V-tzk9t>uRn>;Z3eTlbGSy
z5>S1d(B>38vugG$5;y4S-5!fw=$UaA&xu*MK#jMdN`0Wb#Z&&NBmnvXr5}B^p&L7*
zqfseLq;IMs%+HHqiqY6Kk8_Zr)QGHN!|2z~0vbya`aFcSvV2DzgQt4zl@F4tTT&_F
z#0CyH80<S^whekeQ!ocHYt77pzEmf<uO*GPS!)J_qoFguJsBmSX_YhxS?QaDJv$$f
z(f;BhYmLl;jNF-Bws!}l7c;_eKjNflAO3(C#Sp1&*wEpMT@!qnj)z<e=XlB-V?{(c
zhf!xxF|s?{)P#(_>I)XJhG9qa5;^@l;QNULZG((UYapA@RRnyujIVVkfU0z`c19`7
zMjPzBe7M7BOZn36*PNVEedzmniJYCkU*tCXq_^qsf4hJ$9EbJj1N~exQa?O?)#|~6
zpMm!&6C<kv5x*F=yX$RE@C^ayHWzKqWa3qwJN^yKFs^Z<suhu&6#8TKEG~Oghj~PQ
z&U1;sa4YInAolw{XMS&0qOe!2eEPIU5k{3MLCi;dojefaBO=he%X}X{!lZhbh?>jD
zu|jB%Fgbo%vbv<C<t4J5mkOL(Hzqf*DADm{*9JGQ^~`K$X1#Af*r&(b<Y^m~Z+k%I
z9Yp8>h5buVfhbrEmDQZ-r_B4;!{E4UqIrf1I|w#thRfH_sd6sFIdHg<mX)>Z=o6qx
zMjgKR`8dSLYf`Ye1PPrfH07Kysn$_k?4H7K{hNxZ&>)fH3^TYx;T*llZCbNfX-z+i
zlcA3KD&Taf3Ox;SRn{lj9!*z3q_`6L`}NhIRylqOvf$R>y!Pfs`w_e4>f{rDH)r;n
z-1e3V-^52s9P|O6aqFE{JGf4rrU-NT)t)$L-BsN(u=kV@-MMpx6!11oIZJDRBKoZ>
zg>TnrxwSq{$Dn~+UE=HTCwSx?aOidiZS9`>)qBpAV5e8zjh^VQuZHK=KfYc3*`2(n
zcPGg&K5bNtSg;8(R5O)NLoYN-j7Mo;R5<&+SdgFc>yUzBy5(d}F}Vo&2u?IbyfX1l
zc7$DmWla`rDyFr7P)TX`_4<726H#0{U=`Nk5Vk%?b=1I$EnP^?LGEy`^$lbTyuYJS
zjy|AfWpcT^Xquu)W~r2ZN|va+Rub>+P3gtwzPZ484^d9WBRKD?Zp3X0sjRbTQH->G
zi2~ufyF<$1!e6Hds;VUvQR;m*!QBCj@VrBZe-c6Kkas2w?A*dhD3#KC)q44=Klawz
zA6nYBJ#aG7_-Sx;Xqw7D#q8hR?@u5j2FzQNb;`y$LgiHGlL5(kEAkX?uPa(JJ6xzc
zK)i&D+y0T%_5IxqHt(J483TJ=Q3a9SC&N{0^T-SD*D*B<Ab>ScK!E5x`Eq)rqQPZ4
zM0;@iId_|s%@PV+o}pYB=eCa=XL4pVCxGg=F(uu*MQEL0KVcf+VEm&_&LRsg$zU%@
zy!Q^pJ1oILUrejQ_l(@;9F^89H}gO~FVr7{ea^l1c2My&m2Ag17}Fl`RL<63Lxyx2
zj`N-vI7Kw9;GK)Hy(g~lew9sJ)q}S=i0I=<Y$omWJI&^PG+J(saSZddkRZk~j|$CI
z=*AEcle+J`wG!T<`=mJz-C{Yz%ubWY{8}rRhT%C%z)Y@~l*wk7pNDk*1MK|m`2YOV
z(4^ZimVY+)iV9i7GM%O4FeGW4XSnN$-8JP~jvF5{qyeUrCkoJvJ3fznvwn`MY~Bq$
zFY`nXo*S!d4+AeUfdCvn)8=*YYe3Mp8GP2j6l<83$*6KTK_a2?Xi0Qo6HE=PT}V(w
z2S}nKrM)lX>%LCQwXPJ-WZ6&@IKCXVo27aV=HUs|>rXt-WilhKJi0E5x%P6Ww?Mz+
zrnR;EY!7VfJ6Upn*YK8fK1W1Me84J%n96Vt=D@DL3o$;=se?h%wpS;|xlyEMPV3#G
zma0pi7|qm49ZLoGVa?&*viI#`4Yi`Yp?0fi=L71()YiCTOlqJ11;v^8?mY_6&Dqyz
z5<;66>r3YNe$9O-JeBo~|3Rne85BJ9OlRGVrT1HM$EIJrEOM(<d8>W-z31q=RXVHk
zR5HKd+T$AlU}j%E|KL~KOrPleYupIbFXJxD-+=-MEC4EAeWLR0$T#&g&)vd!OepEa
z_X_yQ%lH!zUw%fB?u3!htx@0Lbk(;Yp1_!JZ9G@s?HFG3Ag{k)?*)uPkUJ6dagMM{
z(d$E=j3dyZMTu2`Ie?tmGs)|4;6@IG&Y_<X!gx#*aM>yp-gpRH*ArUZd6Oe_T0x;J
zl~yst;vHA~c}JlY=5C*$(mzG|yjL+aMe2t2@4C&ZY;G&UiT<k0Yl>YkcZ2-VSSOxv
z6E)UBd^}<@!b(AJ5i-|jBt*6hKfaJ(?#-^L!zNidqxduwtlHrb8?V21sUUY1Jp~1c
zsX`u{u^7zjG>OSfR7k;KR3fea*nWw03T^Xw=h+NCF5e^_-_1_tv4k+vG?CZr)XB@Q
zTvly;JC&xvMtVlC>MPs#?=C!0>{2Nh0m?`LF;6$?W{#e@H*Ek){=a*JfB%7n{0gLb
z+7GI48`yVAaJ8K*T_o0gAqmE7mA<wch-=U@1e5D4g($7T_xHHx3~$HnQt(zAA5j@}
zc%gy#GiJUVq+b2@8mPOh3RRH^7n<2(r;dAYsjwY`+NmBHpft+I)=|{u26eI!FV3R%
zJFPm*GO8%)HAvJ(93eGF>yRC|U>$x*iH49ucEu=hv1QF8QV<K4$zTn<R~?Q)hYfFU
z$R1u8MyPN_o`tnnhJ25>ZOc{eoiumt-Z)!=_j0MreB*QdL~X6!ZmocsEyLS{%Kjsj
zT$Ux4E@*Vv2IX|%dPiGjD7Ht{Y%+Z;_0p#}li@vZ_q{O&#7d^lw6nR4MHfN$6`_Cd
zchmbH;qj-{Ncy0urHMTNyMy!|LcHxnWoNI6auUMh>7+aU6)z5)Ec&LSXotUEB*6`X
z<4RlQy-!u5`)zk!lIi`x94F`p9GKGu-CTKHMBfK3!sQqYwRAB^4W_EY%Pf<rwGfXU
zw7ytMO>vKtG;H#V;QR|ukJ3JpH|f}9J^@|1(HxDuQ!VBtmKu+o>gBuzwO-)zL*OwR
z+dNEh7;;;46@|D*Ie(Q!gC#DRkenDoHBC8G1zRigVw5<1=+;JgoS4q`IfGuao>G9N
z2`{@!P<Y82a|y*=)fvn{(-s>LiQo+DdbnMy14AX42&&Iuz^1uTMnq!BLJ7)78m;l+
zOYWb!J09Tp3|L;0keM3$KhNo>(d@BMU`Z__*CgQ3k%po?*BsTIzC0m2y()z@2h?xw
zf;la_A_P22?`NA1B+c1umX2c(T9WN#kVjy~SP*R#d-2iFCINK*(wEvRD!@8aNWGr5
z_)$#J_j4LJ1}1rXUw~tc!&DB3-}Y98evK33(-j7zz3twoMm>!;Me#dd6!AdR86AcZ
z29--}b%`>E8gZ|D<(Yz!2u+>v!fmf@2`C83fNgr3!=SA8Zrz<+P~`LL@^QJ57>pR)
z%1WOMm<cYFHrs+eYJtJJq#u+!-ztxzY4eOeVY_L`R$Xn?T*(o|-JkOb!g|9hb8`~?
zL<M=JeU+jdWCFkP45&fva_53lzc9wY!?uU$S12)EwNnzAha;<{WdRx(*{E3XKN0U2
zqv2=#OCW%B48DMdet4Yz+F(#>R7$|usrT!63p1h=72wlWGAzcqSKfM#>Rs_EQshp3
z%d&pXD{ixN^l0u1sfMohV+<vsMCq%8AiGT0T^(o8*>myzO)`#%@Hn;&VNnWkpX5LJ
zmEXBTz*hHd<-1fnVI{}Rq|e4eq;d&)8)hHTb#4{*-lwA}Y&cBD51eV7cMmtS7$<s0
zxtO*)M1FSDVa4J%>9ayqmgIaDsn+{O<#Dv~jg{LDF(uo^XLMsU8+ekvv91~ITBYj`
zTQ6QpOyz~sF3j_yb#uiywOEuw_j|bE@F>Jt<(Vf&<%r{<q~$#Pj{Dmyf~7XR@2<ID
zNrI+-vhOw4YC0?VpLpkgzB!T+@JTc+xqxmZO_uUF`Ng^`x4sB3T{a^<v(D^40clQX
zkgHkvcwQScr`Th@-{=9$h1Y?)@0ld!*w-oVPybFW2oC6AXBHKMEPVg`h-UQsz88e5
z!Mq;3Nx3kkIHwPnW88y~pC`6gAw1CWgwm2k@Pn?9Io_AXvGGCjFj|gzOUET<%1wa2
zOMqh=LBXy(&}4=%k7kGaZC{Ys<j*PR3VNZ(F5UXrMbVtmFXF4jJ*q*tPOMhNun>ce
zA(R!4{z8D$Ak!K!6s#|%+HFZ?KPWdLl0!QR8DTb@lRA}Kj&^`=%t(u%Ny&x!IeW1?
zy$JlWNsu9cL{$MKdYAwZQEs8toBxBW{pZ_K5CPpcKWCnh8G!V@=;fODxo)99my+_Y
z{RD!bxDlL`9cx@elTNN6>(h*)M3R9ARcna;)p2wHPNV6|?o|rzFrE=qbrZ~@%PixJ
zIxlo#&I#hujKTrCg8G_hme4i?B_9<7GwX`ld5#XF;#;u9d<9S2`4?_jSh7$7*il_;
zPbz$2D&&{+W1_b0rq*63p$p-6H^jG!gZ#ZHZCf--<$RYROY922Z!hIS55m+T0isI0
z1E}DL6t>IKT<ahc^s@wzl(g0MTN6d{k1VRZrDBMw+o^W9c8U7Sfwl<}n^^?4{bk*f
zo8Y2aezC9hRsWZN;AViVlj}0TEm_!il>bWw3|<wC4{#C^*vtih-5Sn__awslO!%HD
zd=4t_1O@N~ZZGC^x}JER-o1HK=!`G~on%WAh}SvaFSmQ0J}}z-<!<kRyM4Yo&Z_eu
z$xhT#H;f^4c$q(6Q!7FDOkQWz2oAtm{Xp!@SIF9r*lbtCw8KT>n$=gQ3dVb{r88Mi
zs>-iZ+R%laQYkCT&gT)*sxA^=Q;T;p%MKRzsQk^UXk}OKhO^8pc)6aIK0<OpX+QOy
zRZ}54)6gjQiy}v36kjR=L@#tg1mW>-GY$t-4~`?W-bdcMEStNxZuju=Ib>35xe^>q
z-AF6V(ONCRgMbZ)_26Drw{txGtss5bu>47ieUz+vj}SP+<17U*OY;+eeAg)UL+ao1
z>_37P9Wnz&?h4n36=1|vRw(sxYC!#<kd)hCyY@z01J+OnVxraLS|>7LV<~<?UP~*X
zoxuAjY*AW7RTfkvx&Jo3---?EAQ}wR{#Yd&oJmE)p?h%AiKj0Fa#W;T?^w;9Altj+
zTP=F<lo@A@nK7I1#ETueaP=zIRS}`Gn^@nETZwnm>kf!MiMwTCgD-Osb98jen>~D{
zLq(}_Ztj0=C&nH5cRTS=&z*B}4tZUlDmN(#MjU%|t=<<exJexZJ#Dwy=%e1T@U~Oy
z+=F^K8Z*uvb?yT>2p?F6fNqQmX+*LuHOG6hm#|qgaB#O7+`r1S_N;z_bhb6sSRuY~
zY`H77qBLk@0k|o({8_}WpI)*Vo`obDU;me=^q)Qo`hi3M@sdc=C^|Sh!!>6V{pG3P
zO{3%$et9RnG^9DYp{T}ZDEGk#&WVDz=~Ctf1Bt)d<SjOqv|jep|2p!&hCMU8SCIqa
zs!$OIh%}=qBNhrE*;PqU>ylv_jO**J?eT@3Pt7>i6KKvJe)#9f+79*UehV8dprW79
zmvz>agDRn?6p9dqoyFQ}K|CFO)qrV@tlTLvbcWx;6#gDt+IELS+e2Cwn)*T|cv3-^
z?_8$=H=n+`96n&Z(X^*TsdBB~mH~<r5KB?RWZ-sVV8_-n2C>BJ><~7k%CChRE%fSn
zuCon>A*rSkFD7u$3aqK-w0shqQ5M;8<xRHIXmb7Wj;ZqsP-`;G|C7hX$cx_1|F_`Q
z5JaD0YDD?a=@(7{I8SJ*=ex9Q%b9b~XjTx{oKo?ZyOg+7wV?M^6hkRaQwH{@=w_O7
z3&hBN(~$_&$!5@iMAg+YR58qbGPhWN-N}&_jjUW{b`gxZ3P*p?7Zr&(>L|BRYIxJu
zJ#_v;GOsn!syqwK44xtDvFsk7%q1vEOo(R~x3zh`H8PCy@qaC%v^;isNl1nYF<f_`
zesX#yo4ZXNFooN(Z@%s{)1ZFcc!sa3ahcU9l}0k_W-&)0sKjM^(7x#}3V8^k-cJcR
zip_1HP+<xmyE7?sw2<8U`t+eLsEjhb^nyodB9_n0KY-@P(z9wJ`l+TxxX4Z`>bTi9
zX6Vfm>~3F<oFLL8_4zteJ52!aukpB+4xRrOJ6f@9Td#k<6)bs+j+RJ5e3@{iEQ;vF
z&cvbFQ#`mKln$hGp<B|K{<%m!`_tD`!?3%h!u{!|O~n=LxW~6Pw7xG`fFs0?EN2Ek
zL2yrpzn%7iaH%pbjORBTD#=aG8CS76<PU(9R^3y<?DzuTgh~yz1+F!9)b|8#{nE~=
z9sYu&ZyCc?IY|vMmFQi^S)k1J&j!IFcCNiNwyub<QrGf0RQ)is6fJ_*lv$PBat8@v
z;n5S<A?A(0%+?CqRfy(T$?bWD85^Ob@?Ps`Ycz<BPehlOo=B-n=p7{OQ4E{n_g{Py
zx{l6m9;;3t)7UJH+bLD9ukMk3d7~b%68Zq_{~_LXJ--W``4#JADB6wva?b7#`aMkN
z6F>e+4*3YS1}zLv@Bf#E$`~p|C$9r)ojLI%y$W*z8T9@QDL3V0-)@_@1`qLZiA*P7
zjx>780DSp2E(a-^qWa7BS8)wqq9D`C7c17~9T%A(`iB?qpZhDXdn0UjnPL2-=<;Ei
z1oNKu8$hdan|Ur#U)KByQk{@^f9tlXAl10T%mWHWglRc-Hit;RlRh06UTfsQaYZEt
z5I>TT=v~;`7iyX8|KF(FkusAJfTg;tSQM&WAL<6x{z=^y?5mw?ix+Ov4Pe&{CAXQe
zuQ{L)#h5i~+L%ZI!8UFu>sfHA19tW*$17-hnP>I%albP0GLwA%fK_(p&g2w;{y%)=
zCG}t0D|=n>@teJpJ*QfmbK*^*(MNF4?_Ga7ss5Mf^!2|@94!Jqb_INFy<r-{gHw&x
zq2OkA)hceeO$gK*@NYwwAzXgTN5|nR94iK&8NyPi3tmD!{HQvJQV0=2{rY_RELRTC
zn&LN&OuLT6uBw^AY=Lk|YA;f|Zi;LvcL+VV^2~_9p?!D$dFSHpu0wHsK!V9xW35Zx
zjB=M1z6>Kd_g%nL1wn8@!5d?&{$cm3J=Z_lDgxbq&b{EuOQ>W3J^nL1t*&s{a4E#K
z?663;fgqhw*q(}6bAOpl6(ahMY;ZBX2P{ALO7JH(xHlx;+rhlbz8y?|Qn^X@fouIA
z?^NIgj(LXlj3O>zhTdT5irtG!)fG$g{u`E#7RE_-vVZ69*y*-}R`R&mnhyq|Wd+?P
zu4-eUU}49AJBV>ktW1X5o);Usm`H=w5pnuDV%Z~+F7DRWv?R8qjN4!eN3NT17^p-N
z6}4?xpmxM`WNB&Q9w({S6}141tfNs>2B`}yYO|lrH-|;RQZ7vHsiya&yMBhZj;=J<
zDT-D_i?^F@erJ_vuThT$)qt0j$y+&NIg87Kqd%e?v7GmJzNmkO4QfxJQsXxfmpf*$
z*D@;BB=L3B(?(Y=wtoOh^Ns@mJreodLZ{*9bAiNuN&jyK>EQ#*o+c0zyOJl~AX0Uh
zL+@KDqBzf+o&-JxjAH+RMbg~q1D6v(G=TIb7+F~Jj@hx*vJ|bITxR5QJNrj}Eb)<x
zzO_hXh)0|&PF<9hSvYhDMly=wibPiIB%aH$>EhGG>>U02@&#{m<xa;<k&UUxut)(x
z$G%E=<@^_FA((_VtAsm*-~>YF<ERGE37tPB-AP>F%?YqM^-d+YQMXt&-k`fnPG~um
zyiKK&KS%y)0?$j7eIh$ok{lHIx{herZ3FZq%b%DHzeA#?KX$1SgHz-EC%e@?^`dS9
zsBOID9A=H>nW8i31kcOdaAOJj8Frcx@Be$)!T$<7-W~sNss06d{)RGOY`E8Sd>im$
zJzg9Vfeh|VFgFB`zthGthzv^B-(GP+<wV#MT5pxyJdb%^@i*(THYT+`K)r4i7K<AP
ztjj&BLjmShsdLtD65MVBFhS77a;}pzN=PvEmUy33PqiYKvc6^2kcneXtvb(sVX{#-
z=GucQ6;i?b+)0T-OgK7^JSNniw5Mld5I$%+rpi#oEW4{ns7e8KJj@S<TK4@M@19ZH
z(Hz;9x<hZnyERIB$M(9&sYJb$Pd)%+>9a@lhyTR0^RyCFcmA^>{Kq4XfL#edqATRE
zHbwq=6aU*7_8byi;E-;eX);Qs@wa=g^Z;*hihBp*0~<fwDt#g5sM>D%^9gn!*OxG1
z=E_!i|B_3xT{yaOKPx80_d-Tades!MNcL;1As`X#rN;Q?A)PS+12|iof4GZpJIBK8
z{MD`Ox~vo98i`qFD%6gDZ35)}<Hqib<#|Rz@Glsq*3qlbA%T0MsZ~X=M^+4fnt0oG
z4I|u$u*z$^bu-FR!IEtk1O@G=cT2yxN1BFDp1p_Au6TkJ6#Mb<>gG#my02&5&I~u^
zK=@DSFLP*MRloc<bMX8!hYY8k>Azf-zaPafhv5+Kq2ip-S>hTDplG~PhjC{);RPKB
zoWY|xI&aPbNoGp!GZI{cv2r(0zg%B_>2|4$bJ%~73J0wy_UCdm`eP3UB&Zm7La0Y;
zFFp>B?f1@_BBt;RBf^E#|78`%Ura5evkEgKSpGnkSKMG1++De2kM;2yPdF>w_+k{#
z6&b-<5m4S~CI|K<C<uJ^-foXHpCJHH0(KFc7%2RZaA}XLd(K`tH~9;BZ@rV#n<T{L
za`_WVB@_vt_$vwKxbhSA+!meb|1aJGJN*mw6jS$79sMr`0|_P`AoH(YCtqh4h;rXw
z&;uMmYX35nPVQ?LzD?6flF$+CHLAP4GJd>J;&24!7^bKBT<lJ#p{%m5Fv9WF-hU(h
zO_}yswk}=Gir*;_d$#YnQ<_%7G8HBK4yJa+tqorOi8iSRBjo0zq;*auAk|2x<Wxxg
zDFW(iZ)t;hMkOnU3SX;n#;&v#tH5V-Ki8Hb>ne%yO)4v}z{GowJz`YyUPsc;6|Fb!
zfY0F@@J)wFsQKj25yrAeb7t^=DW037e+3N>l-w+x{qTQtk0m$d&)i`DtT8~Qukp{m
zj~a&pNJN2M`?gjlLzFjlbAnl?&V1^(xn`9W?rM%X7@>0c=$!S`i3<f1Ig4OmK$a&d
zOKp}5lONU2J@2&05LPG2bQ~4L%8)*q3Hv7isfXZ+$T{?|Q0F*UB=h5P6q1^XusO4m
zC4-jJ(|OJxW7DJ}&k!fY5CI$HYnptqc83(yVTN5;xn!5oiF4@Kqh7EOeIrUdp{+I#
zCe5vc)Ull;vJDYcDE!P!!+*vvQwzcEllthY_vfj`lsDkmL)V{>M6RRGi)Y&<d)Q}J
z+W%*t=zo5Fg@j)@w>fCtr|W+{{2wC<Uv7FdwH&CvX4yV-b(|4|RNLFcBx5;U6z=71
z%}j<}FUWGs@w^m*4qTAjBS_XrXj2uH!+6BPX#z2O%Vz*^qJCs0j+`1HVn-4pllBra
z>N|flbX5_1E+538{R`30@HQ-=lRbf5a~^{;Q$%R;n#c+d)X_W@W@v8KDbD2|@dBS7
zL~8iVg!^1)&lTfM>p^vxCFUUdQQG7}{w*EQwE<dF+q_1CLzo--lM{JxkHT9IboKQ*
zRpd=)&zhXaW<FY@Jo(D;>eu4I|3A;Mhp;ScwUSeQcz0xcxQlw__Uh)lwRb+>wzMx7
z)i1ta4SII=f8O5t;{XT0VvQcLi?2QGcBJ;Ofe3?|fp}adWD(Xkc-2;Sy&!Q8g>{@@
z8oRB9A$8(JghTgTBua~mm39(w_0^6-jy=vJJ72qD5!v1|jKPO*#jiak<bYZjS-yf=
zFVoJ?BHBTY0_rF6ip->VVG_%2HNK{*c*Nmco#VPfec9<NFbnm-`#?d}1$}OeFSc|}
z0%Ps-r<21eYKGluHbjp@3;{F$;wzUuF$Qrr^#9@PJ>%gHqita=YIK7TJvvdNchN$K
zHhPcVyU|4_TJ-2dL?3PRUZWF4i{3+YG1|O;Ip^Ga?mhR@`^h&yW1hXAXRo!_S`MR|
zErzkaXWS91;|&#MqLbN~qLW%LB6R<?Tsny;;miL=2Z?z4A1!wX&rtP$;Y|Q9Whr_L
zRuwv!wY=QSHK4xYe#zjFjHpTn0OIrN)N`(Tl?CX2t|RKgo5y#|f81FibkJ0(^D+t-
zM^3RGta0Qv^r~}Nq9lx9RXn6uJ*u1^TZy0g3KP~x%SHPRz$bN%$%1Vr$X$Waz2OQ_
zn+?XpDSG214mCBQbbuQQ0}r*uc<kM%cHk3VG{1Wz_E$B8hxFEc%O5jWn+qAzYWDwf
zx*`XNI(wSRsP=uPTjc4$aT$%a=#^>ssbK0)xX%ow4g<V~dM9)T<{~>Bk&ERkGHH08
z3r1jI1$lg=R9h;uySQdpl%qMfkOdgXb5UxG6M8-U5oAw{&vkAXpJVz6fJJ0rKI+5Z
zDarkxJ*O*??%y!*bRlG9|9>G)Mg+kesJ1&{MJ7gRoGX<zP1e0R+j(3MgSF>{mPHqo
z0;`%GZl@(9j5>iu#u+EPa_RY{|LuR+@z}+LOGS)za6~6h%<MxldR_%#XJcH{GmhuW
z&McKghz3`huL|uolpG1wkev70P9|g!*`KIqGraP}4s?`0_AlxbvA-K)ZIm%Mf1Ue8
z<XoRJvSWk%e%{Mp!&8O36v-fG5l3dMFj~z_nq-LfgL+onx02qBF3RLnmMP<Q*u~}G
z_}!i`?ctvJLzCBz91SHF$DAvbXA|KdfW-l?jZ;1+M}YwEdNV^4yTcQP&ZlhdeV%)R
z^R@bT-+)7x`v3RP+0ePHtt7B_LJp79Kz<TmpFvJKX;{b}F51q<5JIgEi?uaVo=cfK
zH%rwVCT)8~+Yw7aQ!yF;?OM#>Krz7Lum1KR_~>9d#2N)W90|Ix^3|mGdO?YU1h<Rt
ziWJtKPJRVMAn~VB3_;S<D%zexPd|Fs38p`cNBVzkxb2HrT5GebBRt#^G2Jx$7X1UJ
zpnw<N0o!}tn*ckX8i$yPTJkJtp+1XL@=Y{MAx^NE$06~(;ZPBGBzq$kU|%;iIKTHX
z9RCHV>qi2t?-@QVM9ijP0Xb8^cSDHFLfJtYcrrVLN$@=5OBXrklYeT?XBBwI5+ktl
z6R+Ltx(;>hAr-h1iEP50gzW#xDTNYD#{XZBA+y&2@r)B9RcG_P_H1a78hmBwX|n)n
z6d&FwZcW3WnKu;p<*uXGcva>$0!mLC`&&sf(3|0I9E^FlSbeXr*WzIdIRWxdI`5Y{
zIi=7@cRrmrO|);0$8#a>SF?Na#1}mG&(7|OjCJCDmrpNyx`&EG*`4na&WdROYfj&v
zxJJydRR;W*ToPArw}ad9YzOpu@dE<`;RnUmDYt^BDbaA|+oMJk;q@@$;hW3fBb|36
zohgq=+7Fz(I{!(fb6XQU9{6U}y&L^&mrESfY~+~dNW!SvvxyGZ+}_y~5=gN6bsWX8
z?g>{5?ua2qsf2ff{sYXgj`@N?9lyboVq7(8`lkVhQ|g1wgk;@J1AvGkjK_Q=^j*R@
zkwvDmq{Bz0r@Pho_ewZQU)faN5JXunMN_Uyc#<3$K{0}x4(dx1+AUylAykgtHSvMK
zF4klT=MaIZ4>C8t=SKsmV5g(I`DQjb@OCmbxZNXJ+-PcoSp?pZmo!m3c$Zps0%Aou
zv!VXSIq=yTYUYneCgT6<v!+o5Cyjgl%#&gV(Q2%bVz=9tQyQv89_fg{y>7QlZ$z`<
z?S1uZY(e&%k`meu6AG|c=LebaTltZ>{FvdYDoP3LF^c_hKc1H{_v@uxnM|Aq$fDY3
zd24w1O9ju*!^l#I<Hede^u*5KUO(F#Q@Ic)#M{h2jrsrh(=I)~Y%<~E61r)yOlKUS
zyuB3wyM1RYNms7EvuaQ5W{if@5#+1@_Q$C?UKGt*?wfIfY4}0cz2_FL%`yM^Nra>S
z1ICaBY~wRS?eE0Cez(8hyY5QGpcM}9kbF})#e=7VMD<E$NuuGDJXz`kr(@HToo{0U
zYJQH{1|Oq2Wu6|M@B{&)M}!*5QQWVcAR$SAD1b!n6yi}H&I+qt%uiYyt1sRQ2{?(q
zbTQndMhA?no+B~M=0an8;x}Z9I`e-!SJ<tbN-J>S@gD7|sbF$nYod*24+wVFl<l)j
zUhbOux<twOd+d}r271hYq5(Jz;T)V-;ksKTv@_T52Jfa;udik6^->dD%2=K@|C>=|
z18LoF{vVH~r-Z=Vo>PtXxxe+$IQOJ2dBhllMq<6l?I&(0i=^nW=JrFYQyx@}0#6bJ
z2#vDPtGC(mSV`kF_?%<3b5Qn4kw+(C^kSSn;r3!fG;@y&(6o*4<(_^#0}?k<>&8XO
z4FloZ$yj`>dD;oS=8#jKPR*-d?v|T~u26$v@4&Nuw{FWkE(7ZM8J*GMRR8m31{3Y?
zLK9qxZi&C0?jZpcKUUKzkDFSmb=_xu9yeDWV_`*q_yMlH!>ucqRjPUDFS2sHSDL4I
zRF(E=LKv0P_*D<>>TNZHs{0@6a92W)QT&;u1e~5jE*946-uD4tvo0oP7?sW3+~pg9
z!<PIB-&S~6Bx^HbId8b+h7PQyzTQiyJlzuGk3dNJ?IJtts2?Hd=)?(#6!*d=m4g**
zkW-0(kf&HoOcdn)v)Auvj>X|PYLJPNp*nhO(l9hM^%P^0eK}vOw;GBvEFnn9)5H(w
zIa#Y#InB4re)@{4hQH4jysdyZ4f~zOzv8=x3!UP`2&w7)Yi_@+qTrw~ftUm?$~+pB
zYYo3yIc+<JakL|nJP|ai55@s+V~R!M)B?T+9VFjWOOyve{9IpZRX<D_2h>7lN7Rg1
z<MrvzWXtf9P+%gQYfm(u3Dqlb(ePyXCvGtRMb^#6yokDyrlyIJenaIIqh_#O`s4d7
zmXtMq+~Ccm-K&H2BaDY{%;=}3o0CE$CXTPH787+G3=(MKu<YPWz>-Ud^EyeAjm6ZL
zVq%-c>leO1DP{7#k5|lQzNwo55&}Fvt5Hl~FWdP95gL(yAs-*iu!bpw@?Srseqrtz
zdsqV7wN(5s4!te}TMRwFQkW(f1q5cCzWeKZtYd$Cb0xc$D2s;$7}jeZT;p)H5sZtS
zkAl1iRKMt&sg3a-W#s*}#CDb)hYSHpsS06^umz?hn}Gxx8fqZICIWD)u}SB=!pZZW
zNRMSwQ)v}+`stzSV|(QuUX?WJD=QzCaOmn!UUMZ-DvTC&j<}IrpR)dA*Y^wi=T6q@
zofWx|i|N5nAON29kspeY03g#ev%~osrNJv7z)2CIu6`W8NN^4G5FK*BH%$^(vqY71
z_$YC`4WC*}$nz4m<b;!M5(p9XSi0Q<J~}%siZT{|WPjD$*umeYlkP!6C|mtlwBZU!
zWWi$D7#c!o+=Rsm3w>PBt9scrV%mB+8(At?)Y(~hZm8Fs2?wb}6n>I|<NMflAG`x^
zEk={RK~*vI{&)Ttdu~aRi!v*8j2%Xm3a3RaQEH+WB2ls{0}1^DXbNN4YYVE2A#c>4
zW|iO;cH6k@4H>+9_Dwa>e~4Q2z~j@TMUlW<E^#Dv0WJ)dV?Lj9l`^Yp*`J^-=TDhT
z9E(eDU~VR>XIo_<He+i@fg|$z@;X?muE2R6jpK;(bG_lSDAaUAH>Jf$1M9HpdB6}r
zt6!DX+1?)zW|mJrhD)yxN<O%T$M*mpnt5qUFD49-F`ZE#Q-*92i(kH-MOq3AF1$}>
zYd8(1T_JD3IVd-q{-U&RmEP_`#<A8~pwX0h^YFLRq(JQMq_;J00=M^My*D;+a}7vI
z#V*MU^bN0W%lDzEJ~(<9KDe#7{n2BBxcTV<NZ`I#(2wynAs%jfOG|R~&3<UFPb+hA
zuJr8<t9o()Jc#QiYtHneeUMpRz!jD+yR@&y`2NNIj~}0AvWdwQx(Hp+o+JP7AG!kn
zd1eo<zjK8HKD2s*F3HZFY$|QrF)d4w&@1^a0NsUbr=C12-@~aAiCMw(nsq9oJG%jy
zJxfq`MgB*Fy<Xs8WbKd+#!-?nsbow-JO5Bz?ay-E-FELrgxsTtFSuKUOD+e3eH;!}
zTkXL>^2XQ|7R|!Fdq}FSan`w;(B||+Bpe6g+R+|<lqH`)O?5qXz7~q_Me&hw^4<Q@
zrss!mOyP`^cYs+kwweU|{o5lj34?NeGqAVb(@$p7R_HNyyAB*k$A~Ts6kL3TlKxOG
zjoiQIAM;2E7s4WQ_O#TfyhPRrJ;Vr(Y=D)}q+G8vIZ6km(sUMd%L9VNHD%8OGS8?(
z5)9EZI|gwWeaqy)Rf5POD>|#dnOuyx$i|Ydy|qOQ7v1fAMORAO#_nc8)yo3DF>DEb
z7Ll*X^s6W;0+y_(Q~75PL)7h3wO#MH033~$$OPiAYjBAhG`|-%glB*SxpQZc5TOD(
z)?Q&!<;Dn^Z<v<nb*M+tL!ZOC&XM9i1I87BjldaaHa)iE%g2+5$xk3Bu;Q~B!*^E9
z6~L;J$y+Q_g5vDy8fw113cT-nfjDG4ZB|63a{zlK_>&HxnhL_~gpWD9dCI}bI<Mt)
zgYGZcJ&ZuKPLG1@z1JVtD~bFVg7E=~Q~Q;^wzrBKG{ycUf!w19_0~a*n_e4s=%*ge
z#^veKgp6(S6PTR1U?>ogl^4s%q1@x85ki-Y38o#Dk{Yfiv@a4HvC0F|D$O8=43!c-
z2mh@dJ~qdS(SOGa$ThAzsxsS(jAs@1jNO^Q<q`{Y_Hi88nk=%ehTe!!hBq`W2wxn|
zGLQOPkDttIcXB%J4yRHgGf!45?|NI3=MdX~m+LU2BbvLD7!;U@#-5NF5@fg=kk<aN
zvgj$f65(my*>3Seoc?tA6|hM(RSY_v(+~LNvi4FT{77J>$i;i#Q4-f%5@MuotjDLC
zZ#$ORaJ~cl#QplGA1H}MV&uB;A1Oj8S7DpY5*T7geu~WFNY7CZ2oKiKS;D``P>H~!
z7GBKLzS%|%x|Zk4aby=4h!UdSs?qd6p!=1tX>#sg!M6A=a?Rv4x#zB1-fNyf`8{o>
zZAA=lmJ6?kN5G~2%$vRl7RC0XigpqDoG2EGreL~aRis$%!erS@u@7p8uY@!f-CUvl
zNk|F_oR)ENfkkCVYF;y}9k4fr@mKAebZ0mnQIYQwCeyCxSp!E&PxrZwpt2;Io3zEM
zWO*AFv7Hjizqb@lOBH6k524m$-MBeY#OgNgQ(v}+rSjoG_B@Hi?ZN~&I|&e+Gi7_u
z>48K*k6r%B3Q{6e@Zhnff!L>)!Kr{P*qidnZ-!|*TS>fGhJxOImXY9*+gXg3VQGIb
z`4y-<wXXY^CFsaNtj-O;0&kB%A_gq6*1yZQfg`nul==NleD@jP2E{Z^VP9d^I%}Ex
z9qGj2*@Ixq^KUvL|Lem1kLz;;8TnUil?vKt<9qb5Uy^?Md%uDjRHMcNEUkP_{#u6%
zngn(fRvjP6_S}eeFsrm&#+=L%eKknSa{=taoT=R_;=TtXqoVw+Q%*bA_4zgmYriN~
z-=8hl=_z_f)6nOXCiSD*q_azzwvFY-H)w+*CnA=eBIyIjUB|$&;pFCbHAbtxQ;`t=
zbiBmH@v3ASd?zc8zuKl3q2?W^4@y7~8uK5#q^5GXXgh~Ym#F1^HrGN@w=PgfpthAe
z&>xu2v9AO#kM8E1*H5SWrxkxh0HS2&a_c&c3;1px4Rz4nDesT39`LKTLIzDXflb>}
zl98R_{<gjB{ZIE=ednDA)xxI?s0e(oeEf1r5$V5Ny_z;p%fG|oPJVTJnv7VNq0e%J
zSjk;;%;;WAtRqS?k6*yc1RX&k5fg0-b=E&AJ8b<-LcZE!{t5vqjzg}iLW+;EY>m{V
zC_n4%DFusr7fu(taj+2D#>;QGmWMO)M<=n8aH#~E!_$6b8tG+B2apNR4EQ<vL8)fI
zC{s7rL$OZKw=eNL?;>e5%q^;~c?>SJB@}1DUe|I7lerJH6zMnl*E5<WSv>X%tr4%K
zDec<oll<qAZB4&WAnj}s&i()&J%ge%QqC4&*eoUuf={<JEf-A{>;lRhwa6~#zlmQC
zTZ9fVGaE3!hAglZn?&6#tCu||7ARxhekXc(TI`#u1WFboBM2Z2*(u!3tIoayCe{Fg
z$uE`+FG5i9#kdK%TlA5Osb5^zARbM&3<EA_iw(#<cVoZvfWx{I!Ycg%?LIjuw*KSu
zv%j4Pap<(lOTWV!h3g5#-Grf0N`l1A9=H3>XkoUs+Zm?d1xIg`i)Oz8y~_308DM-{
zX?}cOE8(q>4?l7S5<27E5x<gjhd1tl8=K3CR^XS%k9szIK@fOYQBD~BS3m4wXqO%E
ztXI)7;NVd=jn;gem4$#=nQfUDYZzw2vD^Ho<#z<|huoXRa>Bkr*Y)ofCIZ0)=O6x$
z_v2$Z@FxWS(L8{3&G^q){x%C$_Y1FFS%pxpH?HUt{Sn*k?8(-1?xa;$Rzt@#;(EGd
zr^TD&I}v9#)h67)Up^AySFBznA1)n=4&ur<nT)-y{GQBQ;&)gz#4XmeU5pFk9KgWe
zCNzF$CR)f|-Nq|+Iim>*FyGdRLK$*w{6$*TT6J@PaxDqi7*4nIU5SxD06jrk04jbA
zn0Y0iRkU>70XXl?6m&|S67+ZeBFFD!gsZddD@fiOFb07Fm+{whuBf<R3=7Ckrge`c
z5brs}R$3S>x#qEGTf&H+_%w_X8=4HwaqZw`yUM$nDd`!Ybk26sGFaqkJnurq56g1y
z(0lntb9}ZNgC_e=IAp#QLIO3X-Y$%N$=1GGm8y6eg6~i3s8hJM{A1PcxSdTPb=rF;
z9=2##zVW%AZg(FDw#REj>Te56O$a!&Faz?oRoJ2g0CPOm#g2tV7UydADpt`(TFM>e
zD<FxH+vjA>yw$eq6?M}YJ&C*K><)*}aW<c#8sc0Mkvz~X@MBcj#feaYta<M7{jIg5
z|9<qY`plF%gr(aPvYX=_Vc^{UYX3|U!m;6Y#QnG1KPAk^VZ`W&v0Q&b^r7kHK06|-
zMPSN3zzz3gI<<YE)c$o;uP3Y<osz`W1J7zSgV3j7yd&i+Iq<<Dr}Ww5$569gBINMM
zw+DP3#=iT-9^$S>sY>htsqkur!B;&3SE7ed*&CC!y0d8J+;4+&jWGJjQ3D}nF){cR
z^nB+wg1X$pU$ukhZ-YxODp^G9h8>S`+6j5FpI(FY@kwqH4Yq}skkG}^tmG@*w)g5y
zzoYGYT48WgHbI&EkT9fD-1SyE1fGoZ1l7+?;C~t4lI(w$oc9zELmEx5x(-t9w-mZe
zN@Ilo@>=5D2fpw>7l_a!;PHx@SBiNaSMlM)bMIx>{nqHVdqYV;h1Hu#CyTN>@u0Sd
zNCI|}g}7<5lo|5En-jn(`;v&*#Ndpq^b^TG>(fU!Jr_VJuWXcrM<v4V4<7-DLkY<V
zq0Nsmz6L%&TUSygyr<q~32+9)C+}L$Uw;x@*f>uT%S)m^olKy)Tf+d1EGHmF40qJN
ze_K~`hJ0!bpr+EFdv_b3<}EbW7b2FNI|FVc1^^>3(4a8};?e}6{ZZjE_uqB&4i~u&
zh*z>Xmlp#^NBZE2Z7L|j=^M951EPEN)}(CM<3DH|Hl5Zv*64^k66|lGQVC#s3P?ci
z%**F_4Aj8}{UxK)28-WvHWakW8YODfgUC@kCT+#e7-`rPlIhW}_GW1o7O#hYR5>N}
zworaXzs@H<D{i}B!$G@1krtqM;ft%dhC>c7h{Ibr*y<~r$rXU*49~N6yhK?XO4Itl
z>HT(IdB~#ZdK%yl06&>+?XS2Zm&BbhTzTr+*G$#dFtsyTN-GiF5UVy~o8a;Qx#YA<
zBAW~wz&~ChnJJMronN`otTuvX^iJ23Yaa0z@n9nBjK94k#KY5rZyR``FTDRHxhI3i
zqJm9psRQduB$<K8-_8xoNT+=Hoe2&kj1uwa$JGBG>&Wd)?O8!FI|{Dk#?h6=GQSNk
zI$~K78mHONSZ^2iH#zU_#Yb`;l=N_JVN0IsC!vL|N=(d?kN!6M5p{j_TK$(_C=H%=
zeXtkmgs)SqSt<1xIH*W9{qH&YKc{Lbd&0@~z2VpS3+u!tn6iSa9mMY@)UN$}F3wE6
zr=4%Ri<0k&e;ScMzgy-wdSNk0iV2iDOWvk%|D8>DOO6;3yGAVUN~BXSdv1qeT!?wF
z8N`J|T2*A1MTXDbouGUN#s)*JI!m6vU6{L_hjM9J&k%=zo+Tvm6A>n|`;G@ebkoF-
z5tc{u;I;M7)Srh}z7jL3V79&2U|MG9W2Tc)&BPUPF<l9{y|fDRNn~yz{u9nwiad?E
z1X_wrv6P@SK6wXKfdPfGCHp#rmy+6FW^Jt-nyMc~g7Mr%Dwu%Y%-{&zo`E79oGNgb
z{c(fEGJX^ARx+X|Qc~#5Y$ali@w<P7gUViN?4auY((RG`4P>b(^z)avZ6zr%O)D30
zeCOWn$^hOZE2VQ)iCvdN%lfZEF#CJ0l`&cB+8=IyCmR`CTjT^>E!fWPRE>-FB?B=w
zgE`>Dvh6lvN|dEPt;f0EMTMh=ggl?&H0HSz<EL6^y83G(!I%Tji$GO$JJSR=Za}D`
zmQkGk5iN^Zx-xsfb_XS^dq7<w=si!B^v?M35JTMc*Ek@?cRY(ql`_aj*DnsY7rE7+
z$)QypBMB`lvs#?8s$;k9WRK5Vodfg`q~O@1HXpRh&_jVb_eQ|EUx5PuZObb|_hb0_
zte^?@s`!nX(kf=i&TkWhuc<0YNK^N&^h^JE8N<YrW(m&&El_OA@$-OcL%<^Li7){E
z$FoGe@*g-AZ<^DmvQW2xA^;b{%oqoYYTt+I({QO6Q&3G68VcDLN%puvQhl1rAz6{|
z*Gse3yso*xf9AFPR!e>Ditq36$IMQp{=azVwX=PJjZ#tY$MNbtwlJ2AmQIiNUqN7~
z>gjt344x76z1VxH!x=f`2HbL9^zyJmL|fCyoxgXFw{E<Nw`1E1Fj71~2)WrOPb~PE
z0rscD8sUO01hf_*{4dW>FdcGyo~Y+s)b^%JtbOkl>dRYP<S$F+^r}@C_ayQ!YV;f~
zrxMr_-lIez`_M5IA!HXR{(;jgYJgVqj8)w6G-ST*z*eChT60=stLw6iU6a$8)8Kn{
zz=1kxN|G0X!;O=2QQselRCr%@3YB6{$T{Q5+E}b1!jOz%pj=c^pjLl=wl{-+(Z-f9
zRIh0Q%+?As5n^5!2B)A%k+bLXiV0L#QGhh!CzU`n+GkhQeB{qpptt&S%K0JM<1fgZ
z<4#Gzd@m>h-}CJcVAEm>K53(xi%N0a?0ZWf8Bxid)E8C|M$==tBse9wVq8o7ba^k9
zO3jb_Ol3_gS^ql&bhY_eqD$R`&uf>`)XBoDW=W<=pCTo|xaJ=49K%-OPgANqy#ib)
zy2eVw9?oEdf)bzfc@hJ11%N*wyMDnc3*=NzXpuoO9Y;Ap;4ptM{5Zn>cT~d#O+PmH
zkDLw@%9yc<FdxvHUgF>V8C>7~p0IrYy&p`#?}c5b19RqX<TL!$snS3_Ak-;xqOoba
zBLic2c{kQZs-!Fl_u;k^(Yg@_iQ=qi={7jkH<J(D$h`wzEGqq#jVmJ1Pvv2ertlcL
zGK;EUkR<_iQxM{IRY%+RR37cK%rm~_<l0ZzOC|H^icqsxNZ=<!;gMbhA1Y3!u`D*$
zOLK_VjMM%00fq7S=bx#As{owyc@cULe92Wyw7G^X#Ewk&F*33e9{Oe=Min2D?0Tmu
zUudQp48pJAF6l<lxjzx+!9YPTcTsF%(15ay0KNNtE*M7yu;z$$<9jwFYsvmJG#xfv
z^AKG=8#8$xoRS{!;)+f{#NS2L<7F4sbTr5qI`u|i`dlg*<OIQVsumolK`^Y@_kYrF
zO1tQcz(3LPZwTqL+O76*-1WN9{u#5ls5|Pso8004UtR$7Kv?EFSl;L0PaARdN3q`P
zsI2NTB&$)@1AUz!dnLLV>57qXd-A(8`wX}oxaE`XG>*u4@_k-e1P<3kjYfSc_@*{6
z>$|&n2w^!WgpX@(0TF8Dzl%lzrvM<#b<|dBstEk0H)Y)L@WGx`Z-j_SjvgG+3im}2
z;l;pyS2in|iJmqq^l1ORV^Yb2Cw%!mRb;J+Mgf*Jt0DqMB?sdKgy*gJ6ZMxS+u;*v
z$|v<`s>=CFA)_=^tau&v7v>hPP;lK7^+N`XTd#spiHeO%D;FkYe$0NYg<o=6U*y#6
z)xEvB6MY`Qj>+O*W&ecY)~M_F+1OjzH8UrDb0F5`=hB4cqB=}u@su*!9?4IM{bmAN
zp=~RA{()G5EAZ1<K;bD%N69~gSWDKFH6$W3O>>hDWo!xmP(%oYemZ}*xigBHz;x{i
zlnuEpxN)OE<QjFxB0(_$|FD8)`1#CKycp|;czmrxP#*8)^$Q7C5))XNj{hQkOnCcU
zzAQ<e*pcuzhWNX38g=axUM(&vf#-XU{_x%^4vCwVCnaOpIraHC1R4i`z^u=qmQh=D
z!4#-&JIcE6cu--z;184X^sAN@NzRK0Xr)1RB^lS6c{61GZj+?EPfuCdB_MEVg=cjA
zksF^hV3&5xlm}Zs2NX)KTcoCH@HXU6Bd6soN6ZF9(gmcY{=GQJsSp4BP?KzDCBhXf
zZ3{qm4a>p_9IfRrFY3X+@TX*6;e8DA!%1%gC3Xh9bG-fP|C@#h6{3u+5tjofge{%f
z2<pMWNY>Q${r!`im8$Z`20>=*W42hLzYSQ}A(TfhYDd4l-{_0ZfeC0BRi3&P;{DWk
zK-aD$XYu;iAx<uU{LkthPG{HDc0FygttbB*?flO!q096Sm7<%hAfnO#<9DV<$o{sy
z4R^`W`rbN2&=0KPg5NgfbX$Da!k~Zwv-Mq%zU?{O%~fI_ZR6dJhAv3ZyuG|f$H$S4
zaLzoNkn~d1Z8J62eNIy-cA2jHscK5nVb$%hW{`Aj0mXZ{$+bU};*TdCw#^du)8@a8
zuY<GElB-jCWN8&y={;R?k3tpi_RZFfynlfnt_LMa@6{tBohzmbBVFcA*=z8-CV5J_
zVyGRSaV)q`i2ndnvZafwG&na{V3<XL+z*FwM{!*go6=$Un_rw?SoTNdcrCKN;{2WG
ze-<3qc+Xf7h0GJ|%0Q4Vfq+kty{4r`sl87H!H(CvMS6NY2fCEiKVAMbk!<IFA%m<d
zvytQ>_hl{j7D%S5R`~&eI>yGJ^NI~^br&<5a|gn>%%2<95xhS<;1TO<V8<g!gX6$-
zrMr$=SMQyqS44nZ-5=ohGwZ~)MEEL<iE6qH@;@1VV!dX&4eDl7Y<u^@%wR9esYR(j
zMPCC4o0?5FFoZmGt9<Q+<z4R;Pone%Qfm6-ypswFX$8f5#soPkE_K&mQuD|<jx^@G
zO+I@QyC1%kpU2~}3)W26g+;dCaUBF<7NW3ooOBs!z{(;H%8Ii^#|y$RMn7h1%L`d!
zy6YuUUYumP4_SwQIV_NLY^cPOpp$idc_c>UE79BacKVIU*+NVT!eoaSZn$pfB2n%r
ztF6{%2lO9uVgJ7_UE>N7X7&!W+CDaCj_aTO0x5l~7X?2##A6#cO`hsScH>m5D)(g$
zmUOfa6W>??>3SOVpRf?STbIJoK^^JWVl_Sf&K+#?0I2^NtkeAxfyd4;c_P`1&{-JP
zoW<lnK;<fnrxFGC2M1)<AJjody~RZ&V(I7>CU6Jv78K6s@8;G-2dgx!bx!Gwfer*)
zz2d8;FeAOa`->{lalE%Wn#srd^%(UbfulLiaqngG*3qhI<sZrNxM=Ro>Xmen?Vx|R
z;y1#OVzkd{3g@h+YYI$)rFXi+&DeWuIP)S7FQ^15sGg%<q-^(oolYiFW^MaW+vXwA
z;qfKv&Lc(S?j5t}#t(0E+7v*UIma{+ixa}>s1t{AKh%FyfG3IR`c2cnU%P#=R3)n=
zyjzg{;6>mNgGY?4C(1M0+4VMKRQg%0*o7BQsgaastq+k%^Y%^^jBB(8(icCm1&)HV
zU&HCNDSzXj0E(yBL=CX0$F&aPTTz@pVY=^Y0C!)wQg?i>VBl}7!b6EZU=hGPi6rM2
z%fv}ZH_7b=z7zuDqkkl<%YzHeKA^~82=c0sb2m>`Yyne^?fX^UjmnG~i$QSMy43*?
zkXAXV4(LY!28HwD5Nqsg_PG~Ck?HkemWy0yMro>8k4EA$w^G+uD9z#gIhj#q7^wq9
zcH(8yM=mDU>y3KnpaKj^o<&=B*4RPv%;y{E(Uh($Z5vk7A0z`0cXj7tLhiO~Rr3Kl
z&44YfX8b7Hxe-)UtrEB`6V&Vk;Y(<IJI_v?6Z=2#9u5OE53ZJ40vlg^UGYQs_OU05
z<7v;6b>~${x&zTaeBfy3Id1|-k<5s6W^q#L)|(F}xm*Dy?%Nt;g>d)Yw+Nl)4~p8G
zY#;*p`WSTlm_zYG#~xI9LesNI<7Lg?4-FLip|@(_r|V`f@n;Tko_aAR+;xAsF>Q4I
zz~9yWwcTWdCb&jBf_i=ggO#i7<8xu&QXkBC8un4!C}17hgSs%yDE$%ct!d}G56(8j
zBAv4qhXdJ7dH09bu^`K4Z%et4hO}%pmXrtNS&TmO?w=(wQ@CJjQ=}>c;fGfSvr#0%
z-FpR*)YzMY3F#wWe`I5(R<ji%0nm-N&`B4nwn`rqzy+FEWDhsm_6xXvBA!`i6S~Qe
zZ{Zsc)b?CfrN%a^jsxh^hYRx~+LF^Xa#n+}eu=ek{@yYa`k*1T2bWyo`11)#lUcdp
z1JLJWvs*^+YJ<5&f%TKr+TRfnE|^C$bWw=QOsOo#d)DBL5K%r0wyE{1uk=<XYNZDq
zjf@lf-Yz69r6w5%0whPIQZkd7KAjgMF<VA#E!Ta+w2s^wDarM+r9G9R_bB}S=4U6~
zounYcFG8CWT)5>=PWpA1LD}Ae`^YaYvFRMx;^2L~iVHr!@d}sw`^{c)wR=VmNUQ>R
z>NH0Px^v>M*G(gxl)5<)?pex#;%E2$c3hZ%we4F*ID(~*aOY*YQaL7n<qb#0+=TvR
zA|q!vhsl(3Agveg?n@cU&L`K@pXxscn@gS_&`qWU1Wqdqc@U^K`06}R7S7&#qr_D`
zRii>y$k#(lBeKDZ3;fz|VJ!BVcj|y=Is1$;`~3NZi^|o!OBXZLk)@Y0SRvn6W8oba
z&9H?sCA?=2;8tg-MK8|cR&!6KhS;e)Cez_io>XS-tPD>7Xqp0_Ib#bAe^H48o6vZH
zFMt8GAcv3>_@n@Dn)S>>3#lx0+JM&5VhQzwBXR6L8WJ0Qq78$Ioto&ulaw22ahaUO
zNu7oA+*)z<v(R`iOh}jj#Lu57l;Ll9`z*V}!?$EfLQfWA8g+k<83xpinm|M9WABb|
z&7LAhgw1rVItni8u6&l#vouvJw>2OKP5ud_fI?_FTb96VBRV?LKdad8sO3kx<K|=Z
z1&LscFgzTK>+J{K)jOdbT)}pT991%LTSL;|8ZHTEqfJ2Kag5jpXQ$;Z>Er~v2}&H5
zmx{xfoQF#e(e$<Vi$6O^GlV`tMexeEXjK9iEZ;OIy_d@|9NyR#fQ^26{(|KyV!W9B
zsoPSmn`%C5T}_Ms(n}Ys>^qmeHyQOCafIWozjoWo1&nw9FG5TR<&q)aZ;H22=6)qD
zDAqXOHnLsS|2Cw~_hRl-|N6jjKq+ltMeUme=<LI&Wd!&8K7oSO{8zo~nt9mnR_z2U
z8E``%o+|me5o=Y|OL4zTR#@0)#DG1r2|%mic;BBH0K*KT2ETP7=0l`9kuuYIF^C#G
zC(HCq#KWc^+-12Wv=F&%S+1>%^7a)G{{1i0P+T^c84O=Z;Ujqx*m0_<OgyoKdiu#Z
zIsKB$s*R3wQiAUfpx_&B8t~)w3jP9+!OPDw)KhjyKfb=j0sIcsVp{ZhEnj1UrKi%C
zuUVF`i%D0F7N9Ly<=M0jIsI*_uWFW(Q!7d8k6=sSiq9i&1jI1sdawLvbL?Ep_K$ku
zcV;|}ho~!d_&Iu@h}i_j#ED^W19XYWlPtEf$-w~*w$l+Z&yB}u)<61s*j7%(YOuA3
zEX<Xm?I#@H{0w}bq{Sdpy7IN2F&>O`5dWorn6Tnom=x&`Lv2-GR)gljf$jB9aQ={_
zwf_FdfP*&ms4g;E&wKCP&a{jM0-LRppJf3(xeSueuT~y53!JMTl*EI~UIa^=4Z55&
zq=$#ct{DGPa!i2%Ug&@$Du_#BCr=3h1YlDLEW7lQ;PDn>7Q=Uof9<d`Tj-+TBA5-C
zjsBJJ9mio==danC>DIu(nkNja>$73n6HR!GZQ0+44HV0<Sg!b9N?OOL*#${T#e~dg
z9tl3>nsgIa;-D-4SIO_LW`L>ov)tp*z?uEEKNMuMMt%bD*?~wX2}smPUns(S4}_qy
zlV&AXXLRD8n3g^y*MJSP$m@r1venmL0CaM8!KZ#b-`F)(UfWJKXNaeEFhcgUKPjrL
z2VZ0ak<aNERqa!+<L^->%baA)!;d%nGiN#)j2XC>jZP=El_!ErL5b%v<Nj_CS@<<-
z!orKWbaalK;CzQocC{E&=FXS|)uX6KMLcAwQCEdNuU_4MK@&(@@VkuleQkNEAkt8T
z`yn76S0HK;$ipjidJSmZ-BI55lsF>x2A%lOD9}5<Gi7~gw6bHII_XUYsW-7&_v`?K
zob$5?;G9a2hiHXtEpcftbwqDl9@WmJ+4S~dkF>h+v4zy10X{U}biW8g!mtP8=C72@
z9-#aj0m&(Z`elSR^NnO_ZDEZs>K#F7%quL(Td%41M>a~UI4mJsfDavaWuL|0gMNX*
zV60NRk)$*@KoN$jVXC)j&rn6npXyOOH_!N=H+4PrBTdPHMnOPv?ktHb?rX=k<2dRh
z`de~?RJFRTQup)H!{%V5nZcd1&tchl@5jjFJ+01hf!44j8tSZz#=L+xiyNiMQ@k1{
z(@*{$i9MGAQY(VADzxf=&=KvnD3oaQLUeY+8DCXMc^1aGVn>|j12o%j*#8pk7U`q5
z92b!4xy6n^J?(cIlg>R7N%Rko_Wh&j_#6zGEMoxY?KbP|&{d{_fxJLcsrYr|?4hTl
zLD0*D_lNV1I(RY|bV%mN*SqEU-{Ri7(?2QCgcc@!Y1Jn0@{4;lmk60nfnX0Y5{+kq
z<M!iEOb3TzB`p75rXhVL|1x@eLYs<F7Kh2c43!Ck)oxg_x6%_x9HAQT^wRQfn=6@&
z(kwWBudKY6JbVfEZrV0z)tgogTscBuxpSD@ALzt7)ll$q{nREtnH>hh%cy4;^Cudk
z&HcD{(hgFyFM4nSj0o3hH=u>MpQ<IM-k4m*7m@l!NfOY(^DI7oY<N~0DODsyS>5h=
zy&=7-+;3ud9(Qu_u~8@N)7xC2EPLO~_`~RhyJf?;z`+}>a!Uz%Jyh;d6?ow(gzq^*
z<k|HYH`#*p<??MWt<6&Zc$Jezkb5=GzI&h=WmB!#3Hx`Sn%FZ=&bU3$KG@=|22M1z
zAXM{}(!&w1XwO4PX`v}nsFZ$h#8#p(#E_$S#U@GALbUFXvUVe(4JEi<1aF5l)!3%+
ztsR8EX^MO9ohbMTp~ukFflj9w=8QYOX&Ux-kp>-*`5yLJL@jJ7KiH_{yuPQ6FjrOx
z0;vUnZ^b$ew`2nbs>}$3aYeatUrFKS)*jWPE3+Bkk_t|%PtbBsrj%NfhT*74m<ueZ
z4@y&xrA*@YCoFhqd~0+pE%|6#eI+8jcN+UH!TBW`{GE!%fo&wNCElf|l_n*8kd&_D
zCRXi$j&R|+mJ!sAX#mb&U;#w`q9IK5Qv-!cBf#V$1w9?0EKy>0s%X}$f)S0)*I1xB
zf9J;Ps;vqFs2Kft(~dY|XMJwzp4u^C_KwA?w~3q?^@%576k-pez$gP=^yKii{ZwUD
z>hHhLd7DxyhcxAFSQ#dvZ8jY_BqJ7el7d7CXLhE<sKK|4&g^=uv61*p@`O=uRL1ww
zWgNRd7Z+PbN(`Dp2yA8~!8;*s{Z|%{Rm-B3#*~a@SKw>-Z#zrC6dy8258R^q-$^Uu
zIvm=LT<;oX5#6$)!kx!GG+{cqklN%x2(itazI|mEst;IpTblg~4k6ds_zoHcb(AX?
zC(6f$$gnOpIs!qT?}eYC2a!?eh8S(H>1*&MPpyypEiHgqPMG#si}?247?F+ccId1`
zVl7Cl3_fz{HJjWvo;ET>0Prh5A<-B&73ibl!hM-`yPV9EQZlCDE4+T=$mKt4Jw2b5
zhPrOO#ugPgOVdsIsSevMs4y1U`F8k*u>q#(S3IgLH8UK3>Rw%~ZLKv?){@Mu2Cm@`
zf6<G;WnbWC?hEz0OC96#P_Q#Ax24|L1>h!2{A0_`-&L(&9er86$?Wg0PeZf~%9I`w
zmlk(-f93A_%jMZUj9iRTEglVyID7#b%Mr?#Cs^+BmP<+H9r~Y>jcJ)rgY@2V49w5h
zG*0U@T*n=7LeM`Tox6F7Hp<@#$mB1WR*v;x(frZnIjCSqaH>GoPn~2NKS)_rTi-U1
zr2VaMim(AOMDgCyEKT6F+F?&f9Hv>-H=upiui6B9kN&;hkS+z`QiA_>E^kmzhDac#
z4dVrv_?<(i9pj{jT-U7bbqB%e#A4%Pi4kr*(r;3Am>i6gP=J5)8I8m6NAf{EK^jV?
zf3uDa6?<-obk9w+&QzvBX@H@F9CR(CmPw-ibVM_essm(lUx0fGIbosMBMB9gSZA8d
z^(xcUdjKW#Lc&sN7dMqHwEQ;g+w7QWQb3Yc$=~9yE;4;J9H!Kr-poW>Wp7bwD4Z(g
z#$zLOA!~(vkxn*eM|$OgBuoD7rmXF-rXZfYzKFw=gt_d~yOL{`nRu;$UUbRp#`mPd
zQ3EoWLNRJN!xVLknhH<hOmdTSWkt^P-&rIsuZtXxFb2+UPUY9Vx!O)-MqBFHQ9jo@
zJPAR68i7xfOMcV#>z#a|@XcP)4+Fd9Ht5!$IKh=BmHbdzvvi#om`nUMD$Hou(MH<x
zA{r|&i+)m3WSP7pFD1Bv=CTdIY17m>X-F_9c<>I9I5hNq(S8Xm<?sU0I8iD)x>HM@
z)7t@^K>Q$TWUBrq&-zeUUVn_>{Na0@c<DN{&m%8eD@HgwX-{38T8W(EyQ8zrnand3
z>n-v*ce9Tg4F9h(115=ve^43yq{Hb-Xs>+h;S$!jOoWe+n8*Hv$o?&UH2&)CCBC}}
zgq0)X$7xUKD}BG;IXy5MF_F?gcm1Vif9$!I4!)kjVWPi;*_ULZ{n-APG`xM2i4T6&
z3_J+mG<t+YnJV#lQ8Kp%5OWC_0{iSa%LXW8kAVs`=1NWuG-Y}yxLiReZcj9>68ZmX
zR^F)+*w7Ahmwa+Y7*?w7>5Zi}ZdbY-kN9J;W9kqOy{>4v-787}XKcNWi5b>~;**|>
z3}1<*%qm-B@f%v}W64}Ld<8Ijm{G@D4bI&>vcV=RA0d-i=mx=(m0k^bTQV92cfGql
zOl^2coAHp15Dsa4@#!)0#5T(v{*Czv*lPbhGO&NpwkTi;QT7c9MkOp^Uq?<cGK&O-
z)3Vc5Ud(FK98}hAJ=+=dk%sD>Jq@s7M!~&rc#p=N!EdVpHAu|?XgpibSFoMX#3&oT
zf(ic}zUUW`cpB?~d!7O;aa%fH51bd?pKmw1b$<;kxR3*T0#lauw^X(|xpuFR`vW`X
zfYt70xA@jiK#B-YMeCoI?_s~Aeq-5sB92}y{Th}iMSU22<U~T)9!u*vMS01;K@cwl
zBe+W`SQCjxx=>_zZyn9WRDlYxXG!@UOGa2%s`B(==7(y>RH-Vk?s!$BJ73NTY-W4a
z(}Ao*v@ksT5>4Q=+Wrj+g9(tE(vAU#;gQA-vshKjL(8Ihb?u8d-96h#$KDU2R&qIm
z_@O^dmG)T*yAvBj8Jw3)nJ1yC&X<~3C{nbE^jCdDu%fTMWPv0mt|0up8EI3xez1r}
zL%;E<1*|Y(#<n6`#b0Tw_iA=PGst%kU7+KRFi%Y4w10)0av+uC<ps!wp2(2{s47qh
zI*aK!A6;s)O%r@gKD^PIcEaP*OSdjD4d9~^46Y0twvz0Y1gX5{%|eQd*X2LcG=bgD
zhJd_C%6T2C1?SFt6lHM(R))HFChP^_0rL&at(QyMx$V{oK>7!Evi{~&lpxQc+bn_C
zVc;)ebl^64+??jm)R|gm!cn2-#rs9@-KYT+qjg;RKfs+CV43G136+1T5M~6(I!HcZ
z1O<V8_V*Ru{JyDx$!8#L>}ymt1&XaWHEZcBe$9~$wh{C+uZ2kBN7B86)}npE{qj*N
znQen+W!Q@G*&jL#eCp->eze0fam<@xoem8L;$p{apPCR1c+A_}$_O{xCm{n8XqJow
zn7DD6K6sP@6-!@Dvn9Wkdp`MG9d!TOY;STH6fdN6k(J^;2-z&?0DgUu2-93<vZ)*;
zxR#aJn4<;QEsibAR#ex@7~M`c$yL;p*M7gyQ%ZV<wKO(A8+$0LzC(GI9u|YdgUS;t
zOo)j(>M4cBmdq>Gx;W0fx`CNT(ASmiPJ2hkvVZO<M?dLt$(>nq5@c->)^fcO8w;<z
zYXb^AUi1&S?xk+eh=&o@7G9KCqNy(O3uYf}d;9Nf^CwarX^w+#LCrLxY0GJDs`a((
zbM@0YdWZ5JX=1&GT<!(G2utkZemJ2~7`r_e_w}I2?J%kL%x5&KDI_!Sq$7}kKMlS9
zI?>O2mU;h2*O+}0fLFP3+r}lY_%b05w-BS97|>F~1{r&kJ$vuJ+o|Pdw62q;>|UZ~
z6M8jj%f}U0|L+k9y`cg40~_Rjd+rK4ni~Zmo2D9KWJW8T9&EvajVFL-6t2jD(2Rf;
z4lgdmp`V79OY(~yL*546-d8fp*QYgua|1xp^aB>3^k;I$_g_^{l|Uwdhfgts9FzgQ
zFk?8Gtz&WFdJY4Y;@CMk%ju@?E&D@?Dv3)(&e(TJsf!R=+naiZk^&np$KxLQk?Wh9
zT9`cV+rS*|UwA!lJ}6BU_-*rrVhOPf739#Pc8V@W?TzSb2s|nG`ifqv`s$ilg1j`x
z++3D=%BQI=Ldrdf*oNq(<E?_=W#x-Wq=5)Zrwm1Ym0VYbZv6cO|1E}SvByY6FSwO%
zdDm#8>j%2=zQk0!t*FNp^jB{K^-Q*!;S;OI`!QlYW2nhQUMSImrnD5>#K;W<`no$U
zQ&^IiTwX+FX+9K^&YO?PtZZKT%>Y;uBX9IR<DyPk<)?D~`jH$M^yv;rM1E@dQ_MNp
z^_OlpSUFa|eHVbQ{?8@#eAh>al$X#(%A&ERBy2XqaS|(|F^vOH%{a(4<DrWa<QXcs
zECx;eQHQ?*9E5!}kmpEQm$*e|U>g}|8r=D~{KaBmG~bb$=e$0nz3P*U5I=^f(a-3L
zQUeMWs5woCJ13Mr?w3fYpO*0?EAg#V`4q@T*EOYs-YYAi)@S664o*nKx#jUC9lw!F
z_mloN<S^$eY0HlA<c2!P&I?-XL%=EeNkO#l5S@b&-S=pcYwK6YLr=YoLvIgiQ$4p!
zR%WU`_6V%~Gzki5f5~=8=U)@9`;s83E0e6#&1^Oo@KzNog<MNGEjMwS^@J^R%%^Ic
z490@Nhp5Kp?V1Kcz{=s<Vs`0LXD@q74=>SjpoFHe&VwqhIBM9Hlzzsfq*l!bs`MC+
z0YQSr=3}<1zGm^Yhv#Z|nIp@9+Czh9Rkb{|gQPlh=99J6*yq$CKCj&BXRGy&f+%tA
zR=UxV>B0@4L0p0l00I9d@noU+_}QUPv+6c4(et=8vXHN{dEr>W=nwOO%fQlGNNP{H
zlInOXK+(s7_BoNmudeDC|5|slH+R4*zW>MVRBFk{VIDoh&P!Tj@GFRUa?5&V1YkiY
zSx_RR&Q&guL%uqAJ`+NY5<;$6qYuH!N2P+?qGq;vd~v~y3gJ4~(uB+wFjpO#!=TzU
zvuPs!$|*VqsnN)dAy*q>Ll^>E+9k00QW`tiXMU;Mp(IIM+n~H!AyOFUy`3&ey%|Og
zUCvbSQP+&CXBznyp%ejKA4(Vv-r^gzOfNfGDv{UB;k=Mk5G7^*YikQPUcx)`B9XKC
z4Q0pDA`Z&_!EqvEt@wkD7b6%Vi{;fZ)k16*7Ms8#Y)p7nkeEP}AU1$Xq7~)<J64wO
zKUfX+=Z-DAxS&RIcXw%II2@m=DlQ193OF?!W$6|~L=&*9P;4ucX-*vQcrDQ=axEJg
z?8l3p^>YRCJa_3rhAEkzC49$b2WqE@a0`6}yZmsgXN)fS6WFKsL$sz6=P9(GjM&=?
zZoPuHWu-Aqg<e~p1}s8+Upj}OCdV{?e8$I=?J=zcvSYHO@mUMO&LR7sk2v9fBm<$%
z+%G4O`K$b*vJZUDR~6j1@KUqWyJoWn&xlB5`$~glO^Lv<CdDt!m3B*Dtu6eXX6x#!
zU=S}(IfT}zQcAPZWLcBib4Fe6m2<W`S6XZU!JsS~ICR@gVtRHCyAlXM9L6i(e$&Db
zctV>4@}^x{_CFuXG4@Z&^VPV0m*{7EGW5iJL>hxCWu}6(j}6<hCzc@%IyH;UZ+SFT
z<5r!+1??-jyP4`$Rn#P?deKJScZcTAT1ntn#jKi5o?&65^>3m(;QKrhlGvMO#kO=L
z=kQcn<=~*De(6K1xPK$Q&#Dp%Zut$gNA;E2D|$R7&Rwb{f6;XIswVTy(c4pv)caMy
zkxaD3f58?IbgAMuLiWa#1Nf}RRb)q$xPBVCzfEU<FN^o#Mc7uk7J&U@PxjF$=dQbl
z3HRIO9mPf13Zmd{Zi&W2l!w$3#Uua-r)1p(SU*PA$P6Rncoh^dB}WD-XA=iVZe)7-
zmm{%KKZYlI{8cVf93?rPm5yR?)je*6%W3OuiM7h<Pnp^dJaEm)<gahfGw1rydnXUW
z5Jmwd=A)k7mvm;zO0cxoXUBb3x6WAn6%PY!E-(MB67YWjR4!w5HAfddC+MgX_m_tz
zuR+6N$0bkkkvdRNoP$F}@GhR8;A$xt4DW46YQ0X);Dp8X#EZNZpei1|zLtm5sfqTN
zsL&{t?k1Wn6c$O&(=7_T@w)yLcQ#WRcQ*|_g)&t03IP5zathQ1<2%W<Ns?$z;2CN7
zRQ|R>Xe}#@g8%3{D&U<V$8UO2T6a{M9zbnksS)N?%Ut~N9ammVY;My$UH1Fgf|3f~
zjZN88=;k}aZIB7;5%3(}V-)DPDQ%j6x5`Z2yBXILZd7x@<E2+aIQeC}mKLD)GUgvO
zP&d+8KRV}%^$bGvfr6TI1`6si-BtAkdD<CA*7bs2c_k6+(2e-!M?N<2JWy0JlJL_p
z+#Ix8LglqZ0NmC0xgEJF6_DS6eq?{atOUTpvH#W++Y1yaSAr>d%wK;tttoGPH~5z2
zJ*!eYvLG`DMDJ}Ab+hpcVdMzP;;I#^qVZ25L++gVX47%S*)+g6SHDQWzIWJ(c<BG~
z45kv}=KeOw=zkWR=YPpw9+CWWLKA$4U9IXi5Y36W&;CTu2r*4%qwUyLZzMi`TOVC1
z4EIe31z<#p`?tJ8DAo{ATWKFp92zXmg7HG$_ue!2Wc>o3`1Yq?1=-tV23Ux1kt%@u
z>BJsOtb!axo!*3yR8*1l7+5~~&Sd7WoF%|MDhLh7)PR{|`L-UBo}-lja^M5ivjY48
zmdfBl>{b+91C@Xc;ygudQ{M;pE`~A3Fr(yOuWWd)3_xn}0Mmwt8fVZfwM$t??u=8v
zjOY%Zu4R*okODyw0r<65mOZ=S`Qjp_#uIm;(I=BZ-+{}IvT3Js-oBaQ+0&k9eZ*rU
zDLE6{a`|Q)0PCm5&b;KBgn<uZnjY?i533pFcrUtBNvX&*eWm5(zT$nHgkvZ0jG>Qt
zu56>UZQMb80XG-e;!liy%wsvi;1pnjOdbn)wk63?&j0|7da0Ghk(6wl5wy9S=IBnM
zV_ZGIA|90g>y3$=HRaIO*0Tr6j7|H7kBbg@E_=qz;SijarzMC#-?)S5CN^8raOV~|
zvQO6I;q7%*ca7ZJ;e)MC1ojwLlq9O+4u-Y2_mz@(K|u!*PD@&RYq#;o;%F)t?fur@
za@-s8a^|WQ5h~?a{BTm&*^Vp&xSlR<o&Lp@D3+h1f+ZKtnI7gk@dj&;w_1l=>pfnT
zd>19Z%{%>lmDhuIt6W#-vw?`>wVj0f`(P2oY8vY)ZNr`@s<@?u;ET$`PR7Zt&Q_8M
z&42G7JLyLq2j^lZHGb}CEDNd$k}(pn2lVP1qilCaxea-7KWgE_X0U2tpAM~D!ngRg
zi!bG$Ma<`~XUb^~^5WuD13@NNU+NVqs383%oTs+zFQ0hs*{a1*QymHM=Vlyffkh8z
zjlE`GF`_wx`H^tJrW#;lR~7@mGYay1s8<RhAc+gLo}B;Q>5tTcMwx@wz$^!4PLMo9
z$q*~_)i9`zje+gGGCunla+Ep5FeeFF&?sr!7y<ig^mpYQyrTBc|4FJ!c0;NvHQfO~
zjFxiwdMiT!Gr#Zv>=#GRYI(?pi~Cv8xR3I{Y~^yllPhu)t*DwRGbpK2hX|Q7XF@mM
za;C+@+(tMX(QbP?KTf(`uSb#z!Z}38!XiOVnEf%0zGn^#eOZ=ZTKxT>7LMG8$~0@D
z(9@-25BH~K4XF|)R4mMsLsk02@9{U2p&UN{A8~IPRb{*Nebb;w3y3tRNQp={NGwF8
zyA~-8A`Od@ZX}g%q`RcMM7q0S(F+zV`aSi!_P+PCFW+a}-<}T+hYrSYc%1V%X8wP3
zCWND)G=5Ve-NLEF`5y(vOZ;EM#!~6X{<pq?0lJa5QX19rJy2FS&56>JAwicu`CW6R
z+Ty1uWur_I_nGIhE0{v5R1_0RMLU{uJ!rZ^IWAC7l4BLD2jZ>`YA;_|&sMd*rI_dx
zMzRbLMI{Szm%3-?iew!uz?YvL&q}>s+Q7-M8M_kOa|bMEUkY|P`lV~wtDe#$UfH|P
ziHeCUyN*lt;c!UpsTz&KFX7&EwfW`S#h)Awplp3PAI!FB5S8nbVvWL&S&VpWuIVf*
zD^J(&4T7c9N;6?oEw^07hS_dvOrF5te2yWVTo9T`DZ}Oy4a#o9e2wb&+rt_8`+GB$
z%G%oU)bap0*3g~vfqFKv**0F`ha-`$5CTHIh)bXgs#ac5_B2_4YQZ(|^w~#}vt?BF
zKEkT$_T-_sLGHN+F;5zKyY8aVR*CHSbONh3D*J8hs)31T)gq@b95!1`81|Y$bL7Jn
zisRN)OR~N<Y}<N(8nIPPE?xmS7?Uuf#=7mr(m{S)^U+d}2JC^o0mNx1(WVouZvAW0
zCK+<X)($k*{eH<u3vX|eKHPOCeYYS4df!JW9Bu<Q9k&A11efQUtc|el(3LAMgrP>+
zH}_1rYPo6F3FEAN6xC-NUP*VyI(6~=H4c_sJ{X^Zp2n-pk>snU`CstvvTpg2yfnmJ
zBwV5yDuX>(0uy4{L%cN0vkCO{*YAE1wm#xmdYc^$3<RkO8>zDRB{W~8mZgnZ*oqwH
zR4yfKBbK%}LMMuzM7{Dlt4z!PGOBiV_ZEDq<U@LMd*IxneSD!tFpd4&Wjpx%7N*`I
zMtY*7uJJ!bwu+^S|BXqMHfF>1XC2V<1pM`NEA4VCbxlyUs$LBY#3C_|w)c>_iPe%~
znu(44no(E>%daL#35MvNgo%6uF$h4ksbf_m!!I9K0yO!S`zB3f6r9YkjJ9>ko))WK
zWxVIxXlSTvUsVF3Z_o){p};RRA&?wrhMY&c+HuZUaE6tKl2nmb%9F0VqDy0nJACR4
zwu;(jD?zxrk#d3OD#+U8EhT29C^-mNX=z4>zYkPhbTsdAPSn_~KGPm>nmy@BVv$wT
z?5pLja9nq-3}|Y9Iv-1<&ZEKS3KSPQBHe9hPU8uik2-L^#_O=sR{>4Z7>uGI`jtSZ
z=9E~dbp6cAyZg+)rtHfxKWGYCYR7L$1->bU1plD$t*B8;NJ|6S6NvY8;p7BOiY9Au
z8>}QI@DlrfT;T5CY60nV<Tzx@5+Wx==a%>#t>x5%Q4Z%Nd>|{VXqDj{P;%*Q0kl{o
zv^4ah65dF@!eE1k9m%2{Ow{-yPZ0WCczU8MdhtBB`yC|}nDG{GE$;D+s=J?n;b+Ji
zK*REYAb1GXlUcBJtvS(d+b0=X-(#XWriSH*Qgc(@F2sIJM#{EBbH?i2Kkrp}$#nBR
zdcJvDTqs%uUl5RkwF9lqB}KBkRa6vIJbFzb|6*<y++UJ{mYaQB?5l&Ihk_rB<&XsK
z*)!EHI3<PP<e8w+epsC~dErm2V6~LL)@YG~)IErJS`_s_n6PHYAQb=J$zv;II>yjD
zKm?ACyBWO)FX3i>>?3tUXcNV@=QKYK%-G?Z<}{o{Sl`xvinw#9G8A82y`H}KQo)+D
zC&OK3s%oN!TLZfo@)9l%uyVu)Z}Z$dESHD#5oCm&(rI4~TtD9$(|bFfV7)&f1BQwV
z<v$$~rTZjUS+wO0%<(%TRCN5M(B7)WUW}!P;0-kk=2z`j18WxW=3YS@B8LLuQ}dh`
znt;%V_b_0rgJ|rK8LnLzJ;YT*U)3(dSo9@2bRjpKzs#>y;aTysVSHbW_PAGRx{zj3
z2cQG|X2$gMZJy-?OKjTvv1@0fTZCN3&^7gDTJSZndimSM$F3pui4`8SrMLKVE^Wjg
z0WPl-Fjj}3ifAdz95TOhweb?s(0&Qr(L?3pf*iNYF0SZ18*7t0JYmFg8Bhy<ZY*<Q
zwbp>vp3ywS^q}!K%t7=X%6s%g#_~iH<Q2)IWN;?AoDy^yH;Rs~S%VyQgf0#63_g+j
z*w&%at$YiUKZU)cYQuC@>Pi6jcPe>}#E`a#U4CPNJTG+t630=Z7;$0g@-`mVxUTS^
zg4b;17ui2AGs9PVMYwV9MI;eBJ_18IDV53Y*7x;CW`1H*oK~1bZD$mHY`3PkNGR@~
z-p!6ylIk@Y_>9fJrxuRsS*8kv^gE%&>Vh)`DWhh31aF<7zWafO>~galay!qn)UUs=
zet3j`Vzt7#So;Ab{@39Ci(T@f4S#$Q7fmgmy9`}xOZsQU8Z~1fx^Z35xL+!|u{nOG
znAh(O;Dl)Jznjb@5SW#b3piP6#eZ$|yGQw~5C7SOC!&M8fGH<lG{$x*w$x8~<b$~Z
zFxyJZ!|AjUU*TdRj35|#!Wa{{ht1L+Sv|;J=4j#ynXOHLmB&?=^V?(tDX(;)jH@TF
zsEgqa^g1z$oyTDhyX1tS>96VN|CIj>^8@*6qDYXLuHVLCzUiT`L5HGyV7Su)&neXr
zcLvSjOnbepUrz}Y>3d<gzFxI#|2!T*z@o;V(-%ZDdK&+ZXMr>Sq!`Ubx<{s9_Hjqe
z;K>i350MiS^*_}yeCiC<Df&E=YVcYUX@{3XF-?lN+d3vo{4mXmUb~+!I6IrDIqS}d
zizI8HJSbc*3$dzp*W8==8IWKi@u|PIY+39H8X6YP!o77eH3q{U(SV$1`betz>Dv#t
zw6dJcKYT>Bw&m(Pzb1@aqkSo_j{X3I_wgRj<sS5kFMEF&v=2AK&d6rD=-@Wa0!7Mk
z{a{OWgXR;fIpr8oJ)0v|Gd5E<SvBC80)3s11Ah<gsve$rtA^O3g*0bkUuyjnxD&vS
zJ)MzjuxO_sETrDR8{KCTsX`xsV88f@$_Cn)b4ci7nv3s>)~l1_ZE->`cqLLa-O(@P
z>ZQF?$YP_;4<xh50_$W!D2IMlWs{pLM(z~U+!V9rvk%h<oP}Ji>&_Oj+mkQ$2#;$V
zK<Nezxv0*<OnZ%8&Bq2(J@lc$LqZg{E<HmXp`~)}bBaWKSGqn)K#5O4HdU}brU!yv
zoNR1LK0Optq57tX)%7iI10bg+;(JwFK6f$mJ-JmzH!ID5(v}(aYtQ*Ku!sD+W7if@
zVcScv_Ot6}KAqEXM=b^W7r+pQ25W%z>c_smu(L$(<$3@%r4BLO*st%t5Z0G#k=2^;
z3Id%DdU`QYwFD!q*H?edS&6d#?skw^=XLrDX|K>8Wu>xd3(LA3;8A<;Cju179)tbt
zP^huIjL>U$31Jgv>H=koYWn!q4)?NSbNYq|0P=a80|E)<uMXgn8`5)krP>ux0<p8l
z&%1QoH2xz=z5R`(S9A>#N<HA2=hco}TXyqI<qVHn93Cj_^@#<h5DV}^i<?Aq!bJz?
z0Jg&{Ge-W?mSs{!_wba8OQdCf^BSR>GNa(GConM|H$Nw+^WhhU7dOLw*>cA~XJ09o
zdiq<_YM}@$n9VPonFmH}Mo40P1LCP+X7<S~mXC1u{7T!pKC_u*o2rIu`1%XaBo(l6
z^i(ZyDU5|<dZGNkIuHspK$E;gn%Lhi$+YfnTfl7a<%exIJCnq^_5rQkY=S|rwdy12
zY4FO%o5(IzQ%>vCpAX{q-_a<&tWP9*_3T~Z!xxQ5?Wan<YJ3|RBS7VED&HO-9ar$E
zRcZAe-kR%}h!@gg-y0QCt1<~xs@R!0`Tp5yOnqQ7axdv-LK+)C#O1Tmdhh2HD!t=L
z9%jL^jmm}F+#+!JHcb^Uq9RGsU;^MhmJ<@}Blw};<j@{b9f(2OZex|@7=!3Y!4SK;
z!{&Zs3f`s04XUkECbc%zmW#ATSU)rBy04A}t<rG7)jm+nDKYz5-Fji+e^eG9hX<R$
zIT~~|48>P#DnCE_CA6D1P4fsjio&V;Y44NV2G1Z4gawvNS9am!*_~h}7d`4MAc=}?
zB}E>9z(cjLIGqH(0)6a9r0aY&ECc0hmn%<ME{A*%bS#7ps|0`*+wH&BG0O~h@D$#t
zX@3q1TxxuPUK5o(jZqM$FF~P3_THV!dcsc*<)CO<QOW@*DqXh#S^%mYU>2q;*fNON
zs-$l9etn5Fa39<4aL|iy+n6YIhKi8R<n9|M=+?N{@_tAd@<aB(%o&a{EordT`6bcq
zh_2<Ql2oA|KGnzpO&cZQ`JG4d3!d%|x_`HWXVtNe6Fc8A3G(j5tJq9X3u}YIAIHrf
zlOea2%I;lc3olg+)Z{0K+K;|rAYN|(qFLdqi{w?eDrWIG0<-O3dR(Um#r-F`?WQb+
zUF#@|MT?y>N>19H^AY9Z`k!J8hrlx*!cS&8&#AS=>2wYu!m;-XJ1bwO>YB4LJlbrb
zrh*)`NyyB)tc45I8~;Q^zkQKEoyx>xVRA={32R`w6Iw<~q;O;t0B)!{))!*4TuFhN
z53BE$jfr6&P-r59*wT{}>yRnItCZG;>@(>d8172Es5P=ej#XQ|=-QuEdx^mN8yvQ!
zp3<d{vP0xUz=nnip};!MR=s_5kjOZxQdiiFA;q?TsFfAunl5BQ(XFAW-!Z4Y^7~^p
zu5L_9^}IJZM(K|=evr5#=c_nX&`PQ-DY}u<6i?Dc#i0<Q5rq$2c9D8VfBXi9)<UHT
zE)?{PG66UXgPlY)ZJh0FbV4tW$xOPk`*ko)79yM9ko(^8jI@(40(HmvI_QHr7>(0>
z2QBeTC~(pKznT}nS{buAz~tC12LKB`weOhZPR66ssFaab$Ez<0M_0lqe1GJfsw(2r
zAObGV(9%AYg?YjY!K|_x9&r;Y@qsT%x{+T9UZWBf8^h#I0|NsEgZvy&%=2VjG<rI?
zs1Rd3(n3)*P+}~4tGUh&t)usIVNL|n>dExIl!6o+f=w}L3{eU}L-2OEUifyo;Bk@E
zq(Su~y=R8ibU&{EE)X*osCb)$Is1s_i-t~Cky#hlLW(!TO#$3XZVQ<XXUf(YjpuE&
zKKf%UPC|$e_8nNiJ=t0PPQCavM|eE(6Or*JAbvQfWEh$jKHKGZxhDc0GdP{ZM3ZLk
z4bzP2NIMi{kafEqQIJZXF;4lsLf_!^qDa7a&A8<Wp|owv2iLk4pV0Pe|CD<=ecNe!
zc+8(gi`Ly^`L0SzIyP=JzbT8SUC6HAB`gtK-<fHHX2zpoUw_`w3MXDzZ8XFV(Y_py
z4+}{HwwrO$MxE0fAI+e@3A830koJcu$JZ3xL2!@1cJgYN6fAni!Ur9JOzddLSF{FY
zu#xwp3lbL31B%!<_4^rU5lA|iGcho9wu(1Da`~->V@9a}@U%rB?q^CjD|~2C%Bm4_
ziSOPUVh{!9-iCT}SzG)oJ86Py_1TR5ZwWa37qkH|Sa5_(wo~%B(SX4Tw`Y-sESj28
zv4GZy0mh4rEZAGBj5UfN(XN0-tNf0DiYEN^v`5Lj7f-77y9{X&%jr1zd%euU<*|`7
z#vmcL)j}K-L%X1P!b-`RZraFUsZY4zZR5gM$9?n#c~)!6YLFm~5M-bC5{-(FwzPRA
zpUi6fHbHLT;6=wjncxG!1e;^<8s{EldKDZ1q7!Z_K&xQ>qtW^nU|d|KSzHTifb=zK
z6v9O-Sq!T~hf1W`;1L;d&Cu|s%$d^K&1WlF4b;@!dhWZKmUyy_PX#$I;y;YL%fI=K
zZF9X;wtqBYFrMp9${l3|ga<P#{_43T{6j3{Xch~XMGe~!kk^*L(#}sS|B_Ovbj=Kb
z{gmGsTskh=1D^mk4YhPJm&qX}PQfh(`TO$T;+<rttGKOp%lIH~K|1nd^mRNN9|weJ
zSv4IX=pnR%bkPy>i19--z*Y1l!cP#nVS2}^pb$N7rnNNjYrHOc;n$k4A&{s7wva#^
zcGVZ5k?#gyrvqaNPfbKoz;4%;G~?O|BUwbE55ffbfu&N&+G!O}yQGfo@5-6(aeqfL
zVNQ_?x+Vf=-hRUyWiENQ0j@z*z81RuHuZ?MD_z{!c5MvQU-fzE3yxL%kNcdY7xy_^
zM}RWdTZZAPL%!2g;HCRp#QyK!hHZe7@c2E>eWy*WIj~xu{=of)CfvE4dAHr-Vx09w
z3Lmf6Xvv~u7umJZr+_di#eLTEagF2S>KWnRI!LwMTB*lK%z0&6xP~XCLmp3cT+d58
zZisAU9B)l$=vyaMP3Bl8$m<KOH=yw9#=R8L%EqYkx?dJAM+*!v+DnA-Jf*T_31^03
zB_rV;lYQG9NPGqGwB2A#9eCaldV*3>>u)_GpV($IvKD9CdD4r-8_|>;s+5J_ha##f
zZ#RhCVFh`tcEn)2%mQGAq;IpqTXNis*C2r%>Eb&LmP<);3F}z(FqN5J>Q6PV*8_3Q
zMIUqCPAEb-v5;kCgw_e6^+*z&$oLYTj<lx{{JLi2a|lFFYRzbm6-p9+R{{<rrUVIE
zOD8Cg^S!dbDt$=lEdi+fC^D)b_0?^*vh0L%H^b`(T~Fh_Ep@iV8?8yF*AL(0{Payo
z`JEQOPp;5V+WG81bifooP7<Q$nE>gjr*aCC1`N3q0FwuGtt7A2>ZqFG#kQJpzdFw8
zV!zIwtWGUjPiv>pUVct}tXBQ*s5u1}`*8L$y^b=TO2i?|v`_nO`+lS@K?yKt_|i5*
zFUWSOyheEl0Ftd&=^9mfytfqZ<=tli{<W>jEf1i`=%a<N0YFhXXN8F((T}|`TlEql
z?4i7deNVS}mznhO7`%Z`Fl!hZ6%r{*2JmtcsJk-cl2Natdob=MORw>%Qv)`h==b<|
zT`Mw($!kMZBEwgeOA&l`r~(&VfmU623c5v1d)f3PugOGWCrlmsd#sEyk;1DF%%0*O
zb&2)xJ{gU9g(bo7!$iq=yN>Fff(0Na9$YK1iNq3gG32B`q;2Q&BVyfBdcV9gffEba
zP2gQ9K$I>*W&r<Z6nWQ33CCiD>m)-N7=_KVJoBkhGi_XeH|Oww`{n>2W>8Ki^n1gY
zoPfVLE?;&#`P^#&)**uB`A(&E6&*-1exLmJf1f<K^5^o(N=KFm`J}eJ#N!|KmAQT*
zORjP*2XT!1BD~aM+Vns;Q6ucJ=n6b^-;j~TffetATixP~7)rP<I6*E*+xP@;IvJxV
z9W4q~8d2sV1T&owJ*3vl<O<K^n+(%IAkF<=`kJB|nP4{RDs`4!OzPNtb+3t3el#KY
zDJty28UMI^f7KX4r9Y1AEWJhE$G{q&7G4YgUOY<E^dJne5Dld4G0|^R`s=6)G@&#F
zF+z;+$-KR)=m#cnAHkYfN1t9&lP9%Lb9-6FHQl>NzoJubmt}>+WU>lqTCu31yei#N
zcAPK5CEZpPKn=pjj|kM$$qt!^CH_~hIKO7*Unjbgn-l=-{}Qy`X@hi*=PP$isDmk|
zAGiM!!)y7r5`ZWO+|KsCTvCd7LNw}zOB$CFAMJ1YW6JyrilJwgrA;Ki(6a%7!!lqT
z=XUa6lb8!wM`W#cY#&bsc>)1+(A0hh`R~NMjE)|7Lr!+4d-9L<u^DVCmbs3SsiS`#
zb|*58B%)2aN$ryun;e=e#7;v9{sA+aAyUH~dQ^z8kY~F3!U`?VYJp|=3#&ZaXvJ7U
zq=#vEjpu_q^Ga9M(?MnG?%{C{m$01TIWngK+}ULaE*SQ%-liP&Tf^}vhJ!Svq71R?
zE*MoIg+BrZHLhtE>yfU61%tjLAq;^M?gV?fWFfBUiYycIJjV;6&;;3jGD1ZOhcV23
zlX07+uc)+sNKFo_P%S_JXqGzZ%Pym=-l~>i_4Hurs&cc@(R-ZM5W=S2e>~aCSXE>w
z-$@-&-X*2*9+v!9a`n4d)OtVkS0~n>+;^z+mp}t~+xts9hZim{9<aBF*BAG*o{3T&
z>a>hXJoRY>L#Z3&%|mKA-`N%A&d0OT;~=F|imCzhrc2AYw|TKnoeyhY*Mu0kl59Uz
z&%aiP{}xd(SlFB~j$zi@HY#=Dt-xV+ZU2%6x0eQm{(X$JybnJzTiUpG`Pt||e8sF$
z9%7r1wrD#O;k0)e`ih&b?0NNC(8L=R2I`PEUj-TW*LmJ6YckI&l-6Ve8@~~T3XB1i
zfwPKJB2?sE#Ag39Q|Ey4jyz4`@cSSBzt#c3KwFCF*n5Vb4ZA?bJ;ixu^Dh>_KaQ`g
zo$i;-<WIMmuCX&vMJeM8odk*_0Z$(0YqAn=mj^X0<VvSl$;dE@BMj+ck?Wuqno(I5
zVgN)out)rMe}Z~dF(Kg?r&jDH1VHZv?|O${z79#7r=nzk5^EAF=U7EwdfO{^(_Uk)
zSoWT>4=-i>&1m|#opxZ8H!vma^G03|wJUA5+iFyMX?s1CJ4yvfe!n-AH;YJ`BBkmJ
ztLU>xDiLS`By{={Fm}j&&jg~Br!mSX?1*UPaSV|czzw!~2zJ^w0l5S7@-GR^UVKCe
zPq!RWiHQf^_o0)mCa3?dK?8n-5~xCd1K_d8d#-8!m}L5u45&%`mgYrzB$40&Nzhjj
z6jN_;3M4y>Z$y2P@a!YxvZr`pQSK_`CWt~{&|7aR+UnzV>LrU%O1Ig5J;$o2c=BQr
zkNT&x=&L<C*gl2B^IWzGphv7!ey7_G8sLx|r|Dh>;c>6(jQKMBY|RX16l8FJ;To*;
zRX*{IWh*~2B2Mlswp=|02J@kG^~oBqQHD5RnZQuUP#%yX-9$yWQBE^A!$N|?FC{32
z@lPvM$x{3pKkQMSa0~B$+ZuK$qr{KIfnetay-os@xPUQiGK@AH{zK*#Axci0?yd*~
zL?drbO8%Sq_uE*ChJPo6T@e-jM8>$O$CdA5QsG&?s5DgGy^OIdk7;B^6!8ESHREgG
z7F}e?p%Gg3b+9UgeR;QU%tj6$yzC@t!*r@r1S+20Qsm4P35Usd8Mh-C<XtMM&C-E5
zXQTOP+vORuEk)}l+EdnxxMkXFOc<~XWIL2?l|=)b`vSN)<29t)7Hr^=Ml)YraI!J1
zMo5^pbvRYrn=$vAxfvokc7J#xH6}mCm+d-KLmUDNDLp2j&Y*;c<Tpq4lOFToE6qmD
zB5H)-h&&;QBm!VCNFrkp7z_eT<b-Hv;U9fQz(m4<PU9~V+0J%V|Femhp#edqvJpr!
zeZ^H|=%Oo<MlvvpG{HdZM<|CVC7qn#tP!)<s^F6j1ZH$Z9p>tzW$KI~1ft@2g=+cj
zzAC9Iy2vMPiV%6qPq<>G&Cj}&Qn~}>t#;Bbvu_KBpkU57nQp(a%8I4JGmX%M@A+(P
zip&z8(i5~wt!jDbjiOAbfoIzrz$1gFbiFQ&6iVbnr6|(7CeALM-ywHwUMT2@06Cxx
z&xo&;3s-4{Uz%L`h(LqIAW&XT+%9$P@UPtK#B-BtAR|Nc#CLQ)n<z`|jK8HOzXIgr
zH>V|k3XtFb6(CzXtu{9N<1zX5+A_2J1sled`~QOtpRCaW1Vc}s>O4x6G^7mH*W}PL
z$=C!o;F+zwH7q|8g9Hg$&};8?*(<bUF<sCJJFT)m4_M6tC4^p&kic*mkM}KvLZ~Ms
zA+s4%ViYiLPZsWHycTLmLr$Z(DsG0;g4XyrQoiD##ZUdBj-S@W&!!dgV*IGctIN-l
zc~7yd%c}PucH~y&6*)06dloUR&V5PNu)?1q#3t{Tv8P<ikXswo0Hgvvm}rSO9@q@}
z=C*&armqCu4*y^aW(+_f9Ez6c#BaH%{)e=V!iSjbm$YBph#COi5}Ua!4sm!YNf;BQ
zo@W6{r0E@RkT=e2a$fA9)2bM@S^c*%O%m+Sl>>VdpyQ<xv$gxbaJQZ6sSgQ8(3fhw
z#tk}ryp-UIp5>r~DQy9AWyrmmp4@y^#!SS83%|uKqOfgZBs(S_gH7K-X<ge5I}D>F
z*2hf26{%XBEsEvk#45Ig;qyqmi;jfL01>P4v@fi3Os;-TRVxaRu4j@*2J+tx(}8Tc
zb8-c5H2hNB;~?pkCvRDxXys=By#1S0`U|{$E#Y?lr^BHC>oB<a%@<Ywd>D^^Q(UJ<
zT#uLkmlW5gI_y;!i4!^!?s4(&8O4{0TSlrQQsv|EoTPJjkXJ80`xqCNGYPvn1g$c&
zY3D%2UzMCkej1<qIa^3SDt0uSKH-Wik%B~G;K#rB99t9uMeDQzvLsSWZHhkX{j!{*
z@?KjN;^n>w{;nytH{&DC&Vmx2ZKTKeP=Z-PY5_pQMb|9%!YRnmf+Bl-XH^+ubv$Ki
zo^lNcNrEmTD9=#W$}@nr>u(k?;Fy!rqz{|_<e2^ba?DZ;gAM=9;QcFrdrhyemj0Dc
z;C?NV4a(3N!5_h(61<=BP7!TXV2UGx&TyKVRpx-%kPcBH#{xk8iRfxMh2q7HvtvMy
z#AU4Z07;Fq)odST{%pS-e0uDEz@R;k2rQ>E%$G|)aSvCS(NQ~zK6pesf%H5Eq$00f
zZQA-c)=DLiDjoevO$b98(-?+;(yp>~*7t8cQv3PpROAxEd^7yC05O-*Lt=a@LTTem
znW7M5;H*I;+eD#k!I$-V@9j`5PEw(}>yvHW*@wu#)4zA}>CSV1NdLJ1PX9V$wsimC
zuv-Z<Xn%3KWWy;T^i+aZq(eTY*I0~=NN;W0FnlUoeTLO2&~v8S*vbI(UsD~%Xdj64
z4R`kV3}XE_wAir6N|Nj@XjOR=5Yvv!9cM93Xs1no6A42GZ*x*zMB~L}y&*t#KZQnG
zSwl9=)oXhV7I5C7LWb{MDCxPyp`*C@(2(O0HeozDLJzwhmI}0T{Aj3ZAx}OBh$<d}
zNs+CM-!iq4tlmE(giSwhKAj$QK0STo8fWe+chgzAI$)|o+}y|9ydQHt3}}S^6aEMI
zpC$Wkp+EVbg}?pJ{ts8Le_XTl2`m=PcB2_Y)H$n=+d*#@C{NrZ#}@+!TEXmY-QWzs
z-KsloVh-@^w}$S8#<SeVCS}FKCSiSWbA^j#Bo%RQO6Y_2N)~7+)nKT2k;o?TgED0K
zTFnJMs2=*g8UqK*5|;vh^9T6RVzP(`i{7=`_Kba8nGD3DabT0&zi(*hK8I-w!6p2F
za`S~o)a87hdTE*dN_*L*ceBCbr!IqE8F*-E`;NxNdiB(j&Vldas#)(}mQ5uP3seD{
zwr500DFYE`>`5*dI(~nV)~3^Y0#LzP9{z&NZOii3){iIpNCalSrSjkN9GT<an7}26
zQJmuVV2go`#VM*gsM<&b&{E)8jQ~Rp)9|E#0T<rh-QAOG&}PTn>9puH3GFMad?4zp
z2*G&Kder9Abh>^_>uttH7czU#VGk(ap+I0*X9z_K2N+w*WeM-J*#p>+#mf+J0XXrU
z4$U|{&@mRcEJlQgQ@Y|>W%uB^ORQx_6es^5tTiBS_87`IpZ|kJJBj~Q(5U6ux^evP
z5Y$JFo(2WSQ!=wWo9uKth&QWB0A+BR-N^hgRpRA`F9No5({?eG9Z4sP51Q!8H*VkH
z`Bh^UQ&J^y)9%#|>W!SQzI0h$HWn#cvM{*)<hr8HbITMLIb&OU)Em3?I+99=j=<FO
zv{FptVc$y+HC--Qm(7c^gQ$s)by7WCZJU=Yf?PLUWka~y674@Sk6^pDezc#**^Nb8
zj#e#g{qjtIJg@}Kh1$B(T<Cti6;1bPt!Ol1^w}(%j?6#C_X*g#A%xM5M*SO##S(<s
zh{TIg;4329#EUk%Ca#0F^W0tmBmm3C;z!^h6e0cNW2uNvKzUxEcWfT|oa7AVi>nBM
zRc)n5@WM(xC4k!mnRpwBgXu;NzAa-8jcm4QGp~Hjz`Bj%7t8NWfW)3;^=3kjd)g?l
z3CI;HP_2BPQZoJ*6HDnYCYGAJtztJWEt|;pbNXzAf{s#ujo!Yoc*^xGVjH-v&%fUu
zPn!i&I@N9alj;R&COAm~P%PjkPH}AC{Ne3Mzxn$n9?rW){_`eQV>w8smrFb*FpYHr
zrSmy08Fh>e$v^OLKgYgtg9kIWHWHJt>OBMorI-b;`YY|8CZd0yfA!=+XyJI#Y?kSE
zxlH76WlzC~jh)LzxO&IN`7Ixg1EQ?W-~=AfJz?VJ@9E*@-~5@E!|QnUNa;z}14_=9
zvdPCv8muL+IE<@rS4QWk?)Nm?Iy@Qrac}p5p3rAcGY-qMB!fwEd!X=`7f4pAXUXc#
z>|IM729yS{`eI$mSa2zjf~fh_#hwso2Zp0HBfSpM0}4+A(EL2PKcR)%9ceILllgS$
zTf-Xb@TWJg@f-+uF)2OqR{_aoNTFM`@>gz0Y+qZo@i*5|pzVi&P_znZ`igS+`h*n2
z0|CD_?aX`pd&ned<74rShWTM^)x@8l7DPXPX1yM$VqL_B#o3vG^IAr8e|RMgAhyyi
zJb%GRM}CnhOKB}_eXSvD)REBbXQMAT>5PJ-P*wZ87_jd_)pY)c0JMteuVR2e3fE!b
zKYTC?Hkw3qn{zWtI8>SxG<x40*wj~Yd0+9p-Tdgi9kM=-(6#%VGU~xsuizT!x*(5(
zJWx>{wUGN+kVo)Muq$)b;Y?P?d%L1L(<tCL?aewNT7sKh;!lW*sXFfY5JjC(IAj_}
z;shcdScrecM)mZ8zS&4{yHKFgp6w;A0Rq+Bv+0Zk+v|4!|HtH(BC%@E`SzXvA-DXh
zuejfDTKm(HQvP+Mb1D3XW&b$Rsz=D5jX%8A%RiN&{kbhVwQ!&Fw+j*|d0~H-eYH0E
z*1kPcI!N=8dWvS5nZfpwv*;s^kFs?KK9<UYx2M}P9DDT>?(Of0Y(iwcYugKLo3+M^
zxy(fA1|)1`r%g`6+t;^D{l8W&1r?5-;-X%i;o5JqJ;0`KGd0jz!9c-bo)x_7-y8QF
zoS2z@;#aLk{$+15Y)*9KuIt#G`}Xn_N~Q;iTL_p{S9=LeRP(+I#bh3CWnU%}G4JV`
zXruSWM9Rm=Nae@1BOGqyQE<PAO6B*f=w+YI1O@h{@e%>(kF^-}l<+_{Tqv=+xk?WA
zcrXRMyA`mRh9dYI({*~~H6A<(5dw1W0_E{L5bh9*uPJyeFjCt3F9-fK$48O4%^yoI
zUl9cfVLDPoiE|!(d1vTh-a2{@tDn0Wcp#l%g^2C(i0N(Gf~Z=LP+;ib4S83BkjXV|
zm3qrJkcygDwz9x1&Ggo^zVEl<%Skq<lmyBkFT?N?$n06<@7%s7Z&={q54ru^-?{xl
z{t50sayijZ0(>!y+Vwb@QkC_uT0n7A;Jp<LK0F@SG4E>M9+9*Mc>nYF+W?~f$%G*O
zBvqA@i`A@O*%r6E<i{WXHwV=2L|#h%Oz;Gv;oJ)~fMK%*vhqA4Nlm;p+$MU%X;07i
zK5vlwf$dK)K1S!q$U2n4Qkyz~>bvdj{nfF3)z!&&l`#v=cNHdIYV{Q%&u`RRj1I0}
zdTcTq>~Ei}YEwy{?ITRl>4zcwhP+Ry%A%L=iN+GNi0YspqdsTGOKY1Q3*ZlWww_Pw
zO!Hp(>qBx}pdQc4e?6k76BvQrnDwH7h^Gx|%myB3EHNC${WtL;Qy!58a9yqVg2Y0k
z8Qh`crqx4hg<x2Qu+dc6a<*zFvc*xXbsm?7(u_sfsTe<$EE_%X6&zo-u63RMJ@X5U
z*`TV|A`htr`p4&L#+9X1*a$<>wcz7+w7r-f?F@|j=Dz=eJ;3p_{SZ(#;ruDJcOh)r
zN(5nLCg_RbbUyE|MaX?}ov()jN9wyDTV^}7dLtXOeL^=ln6EeZnasBW%<yBEw%oe;
z0d<VuPAl?cqsigF!wY1PzKQ7X@I#aAXKrNMKf;qlF;O%!NL93`{a$lVKwetfqsy}c
z(n~lrrbbKap?c6xlSS##9`jIloEeE9Eoy+DfnD5VwXhw)S=GTu|4(OS`M0ymZN~YR
zYqAzydigKcL_O~~H(V-TihCCt@5g1RyCmGyF;v<tRPA;54Y&OV^ycbW!^9OFhpk1J
z;X!C~lquMgbza2I?a6dweE(k``fuhyAmaYNhyy<&F?EoRQ6=hv3&y)AgR4HVoznH_
z7e0#5qY;2G3&Xi@KWTI)07<kFCXZMn=0rZr-LLBEFBfy1!(Je?HsvxQC;Xh*^T#tq
z%KaS<c=mK=)_m+YqCwt1f$WIY7(q&6ByZGjC~th9b*Uj)EYl^t9KIV`bZu2v&9=VI
zK|{$nUo7Ng%zS>|uq&V`!ExVY|KR(iW8p!`E_8OX!l~R-4cM5jvX}lTU7BrFfR+{w
zu_=<UwLYJO{p5*{#O96bD-PQf*Pn6o^C$CUgQ^}EYrf%+JK0HI7Obi0mE{ZH?;5eN
zK|}>!+r7dzlCU(8rJU%=EuSoMu^Lyf(L$r7Bey4dBLy#yiL{$f^BU1Q+Zo&1LB}Sk
zI~dZ|sxmXUxrA444kl@bKelSFH1QGTa6hw3dTH<}&iT`4;i?B{iL6Uw`Z!o<XecPi
zKHj3DG&K15SbzU<R9w_66a(emj=|$s=>PlyVj`akfHZIqC+OAg!e=>YVo<8)p;Nlz
z0D){AbH>~0xIBxv{@X`+I7v=@pM|R_XDhQ*2fjgJs<7-to}SyN_1B(O7s)H99G^c*
zrStW*@z3ej9}x0M@DRzNw03}^db>3Q@3|l&415{9Q`tR^>2#(d^L?lU?E1syxOY0!
zhNtT#ZbP2R!H?+!)RNKdXwAwM#a-ik)!@XDxT$C-<2t%i=W*KbBgs7WTUf@%>)0f%
z?Vt|lB!9H0#P3oCWfekmOfrK=#6_BUFOMHxqdji!FLvz*BVnOw`RR=IO_9&zef*H$
z3@xDID;Lwx=u+p`>RK(mEZKz@1cwVd**vR%(oc|P*_gViJc!=8%S;X5iHXrElh$gW
zG)rYVL?hrCvL?JeJ{ZpMg&!_0C30ETyyL_Aq<;#uer94Bm23Dwnix>=0nv2Ex3S^}
z|8@F*|1pR@y+sn$=UIfz_T4=Gw}a-5ff__FjBCJpBQcKneeix_Rizq^R!+)ca*n-K
zcRQK6wSde_jwI>8w`c>)sba^aiWg~VZE;-O5SO_EJE6P5<M~F%8hzKU8QO-T@_QGk
z|N8I>iv-V^Q3j)^#EbLibl>k)l^<+pbHyKGvG2~8aJXLV7QKHTkF(47W0B%q3Qc~4
zN!2RN#(nN;sp*jFzHQaU%s|ur^6diT3SO1QVw<wYRA>6_phfysfl9TbjcHP&BRLCm
zW#S+>-eu|S{OQG8>#oH4>^cv3=Ni10Bn#|#yY{G=oh#h3spe#}@ZIqWF3YJIJ)zl|
zxi!xZ0*iH+qX))OaB^;m<1qWFa)aQR=r~J=J8anVY;*dsI57!CC9r2@;k-NFJ-38_
zx-Z3VME<g?u42c4hAPzXI@HyBt@NUgLP=XGCD|xh6AX2E=*EV7)US~*`^5_%Z9K&D
zfnW&-S*%2|xlO1RYM8>fjwNG5n>WMX!Oc|x2{<GtOY$fSBQp^?R?xh<3SZ+pGNDK#
z{cac<O(O7;Bt7^!X-p=Y2PE1X)tg1uvxFRu{=6JIAkXf`ZdRlQ?*oC|y2iSloj<2-
z28-0!-y7Wt(9e>`U9fTpf(~2YLYX5^>9(S<#Y$Zq#xd!;@}CKZjJQqvV4w{4rqTQ4
z+Hd{jHXEk#l8w<e&F=+RqWewfi<NQA15<%dFNa^@o5^f&2R`}7CH`$j3TSAjwPrNK
z50Q(kKlBx-&Q(}U9DgRC4;&07{W%xFLYtk9m0UJ;&q+Vc7m|p_;9(Zoc>dvyvP~@l
z8}EDXrSo1|%_6CCbJ^A6cb;Nv&~x%h8#6DVecNUSIlTzo>t#s+!^Z-R?*Tuymm5P^
zS}Vz{HkWhWQrEHd@Y(-7DBhwY_`gg{v*Ra>2=RWhHa@1I`_gIKa;APJ{`2P*X|6TF
z8TZw_L#s61fVvcuN$ndgc#~K0<7#Vby%M3*1949sp27h7okYsv+CqUE#(~e8u>FRw
zC&ll^-rXM5<?dCx#w%8bc<{N_aFv4RCT-{SaV)1QcCh0g9V)Ndp*GZ%-{aVt>Bb>%
zq6UG&o!5S{Ch}$uY55{5UU)TaY;Mi1^I;bc<RWqkD{ks4$?1nY5Y@Ir^IqwtV1df-
zkw#RVE!P*$o{Ix&9rTRajDcv0Uv0Wrk|~X!tnv-ht)i9rhN8Xje&KMs!XUl<7K9z>
z{dk=)Qp^bu`m1|vD0>Vl8f`%>hw0YE%L;{!E<cb5k#33c1!b@$86J-bKJ0kp6&~kH
zqWA*NMcrMLhHxpG3~ZXt%<RGqLw`xe%G><%oleYCKjC&d39MjL391){Ii|A7hGiMz
z4$8o8Y>nz;#di+JiSx34jfh?;YVb!!5^|%9Z^E8-?V;UrlncfiqM*jjg3E#|cH!eo
zL}9Z<M&VA0nn(|QveS}>^O_99Ks;;Z&RE`D2r;uDK|ogG$B!Rbn3)Hl<M~F_mQ!ch
z<XF61mMDr9>kVGpv?H>KoQyMNsG#5W=d}C|E9c*@$$xzHR=EK0TM`_!#L=c>$6cN(
zT{n1T?x410Fwb|q60qmlh#3XHHfVX#{_v#X>kqwGb2gr5&cf-I12#@@0`)5Z^C@g<
z+u5AoP#<Q8GPoU_wLfxBJ(Fx%KbF?CG{gOEH`RPgbrFj*+=8b<TmGsr&5~wV1%fho
zbh}`kz#Y8Mns#HBJg66o*mR#IHB!L%mn&N~{UoYMVCEvvV{)r9P^8vjQMW+A=f;>u
zsa5cPdfvo-L5|aPlBUpy%dTNu#j%H1%y`8t4~Kkeu3}<fKaLF?fxo$ObP@Q#Ng@e{
z+hmc>Ql~b4>G%bMZBw;{N)POWr_Ly;U^=;r*$R3s>3RO8CLF50UnILbTVbxGv+=Hn
z$K$0z$hqZ0St{2z<*`?k<#|%@2l6$VF_n{(bqhABciZF*`pOGW3oCD2mKIaOz7J4j
z1W;N!9Zq&n*-oZM5U5v5IW7sGEhdck-kH&wj5C^|B@)fQ+Nxuo(iJ?Ps`MgIePsWZ
ztOM1%)Vq~@xxL$8PX}Wr$luFb&(%y)#cpAH`uQ={a9^M>VX3?g8uevozOa!UahW}1
zDP)A5`Whcg4AC2@15>0<GK`(dr<23*TvzviY}5CLMhIute>;qugZZ`t-c-T0%4}mY
zbCtDOi}UytS(-nHPV1$U)v?PSOuo33o%MC{wK*C6beyu|h%%{&UlI$sTX%(W8bv`M
zbpo`Y^wJ_&zM5T%EqH25P?eN=Z(?C$lE5r=H7GO9SEWQ}2H4vuvb)&iKKgO86l@D@
zxvaiN^lUnr&pz5Ji9D(h_>0LxbvDbH3KQB|V69?1FyRyP`jRqZI4~aA(@Z~%$Mr|M
z{2BbWUDn^>kn>dQy5W@tgcB}xhM5A!%KKRR(B5GLr)P1SQslxQObGGQx7n+IF>ks|
z@9N@KM6YSE*?wt!AyLH`1t-mVqDbv%qpSmy!shB+xRs3$->k-I%Q@_3s{(Z}+=ev8
zCaGD~^Nu`i`f4q2?~$nZ#J}CW6$q_AM6@#Jsc5X`VT8l$0$$h;w_4QjS(yV!ZhzeH
z)4hJ>E&}jmgRW75TK*muJGpfyRnzsz8_W3v4nnJoT~`)entVHa*P3S`ZETCCkn!|I
ze`u!2@P5?EPo5a-7GYh;fKF{fEC&h>h<DJ5)YJ_Hg@-T|nFj+9sp2*xyUoz12Jwg{
zznBW=#odktLD02MgRD+-^8CRx<Xo%pa!aa2a3eQ=S3&c&c`#_z^HeoW14;yFRJl((
z$1*`6iiF9v@BZB(9pGbFhZ<dm%Nt4TVMFw1MsU?YAS`Vr=@wk|3?ne?&FQK%yN(6(
zA?bdKwE47ZPH?>w6WNTE8`&#IUb`zP$>LU=AZ)qp^6XMtP{iZHP#Fy0U=}L2h>~|I
z4y4$2^czza38YUhs6T<>3Y{ugE_~<O4K)UK*Pa0uqV445l+A3DXFit84MBWp5q~Fm
z1}t>01XkHFl+4n4c$fj(83oX86*&JR#{KtK*o(#5wjBXNv5Wn~`QillHiv#?FpAIn
z%HeD-ipm~#C+QaPru_q+11Ie{uZ=4&z6}yfrP|=DfITUJ^QA3>?QrfpeB}zucP(c(
zjhiujM!#Y)cFQg(Rrwqi`O1qY_-6fEO_mCUxs)~Tcp~-rBCIu$4DZ;|h!*;bfPaS>
z3Elqh==++5V446eG3{xTzz4+*puCjnYgp1SSr=-0b3{<LZQsVa>?+cTbQ-1k1lEHo
zECylm+V~u<0gH+Wg{)F_dQ2pg!W6M-)r1QosZzSNFq@$Rfslfsc>+U%>eD`dHsN&z
zDhHT2=}?=76NFQJDjY3klbA7dj#liABwy8+@!%M+@M$^feSX77mHOf4j(u{>QQJnO
z(x(KcI{6YrR*6K-1#BrcVVh#iB8F=hjxodL!WC<w3e*NKU<?eOY=MBdP{-m}IZW%q
zaIN1#y#b)*SGN<HA*2R@l~ktqx=*e%FHgth!&3}p#0%6*Z7``^(yMKkgc&IthGW&j
zvpc&UZT}97G0fVy{`9fX&}97ny1ImI+rG{_3rpV}4)?F3Oy0|;ScQ^Y8>5~-CfR8x
zkdy4W*d(&4QNz)`^2R<8S+cbRPk0ENH=o>ELx`LgD_os!dUr!ok4#qq+YpH%*F=sd
zz~Urce&)~kFE^1-kB`NfgS!Ll4fu)0xhKleq04vhiL}k$^Bx4rkA!?4Sxcfbp$^Xj
zfl@WvU;{Mbc;=Bmf8cz^;I@$)8RaWn3}t3BvM_H3bYM#~P0Esd<LW{a>XbYAeK{0W
zh^VCI?o}yX)$D}GkA|UJbW)D7)M?g)gb6B~1|Q8wWo3<EW07nmx)2x&W)np7iJOzt
zN<h!<^A-qKH<%|bp8Yj7xI{%PFfE7&!4%)P{q^m9i5tvB7F@jhY$7^s6F)7`qFioO
zGtVJxq_BBcir2Va7{n{2ITHB+I#rs6LnZj7+O=<Ruo_rJH#1+4e<GvqG-u88O9-!|
zWML;U9zpom#ew4beg9K*slWAin%D<u?-PG6xXq6D5n02skK=2(r=UD$R^qt4KN#=?
zh96R{$sDXzT@87)A}S`LRp0JSMS}Nz-D~%)6R6z29=5;-wa<R^$mhM2fk>$e%P_NC
z*j!!Vke@0V7ziX5K{eHiEx2rT;wt-t@^$hn9Bme7+-i1Awmp-1W)6)j*>|!APcO1f
zgo+<mRYVN(YgM`pTJ1nhxSe-&t~`G}0gHQTTh#iCDhp+O7ZZN{adN}T>4vLt%Awbe
zqSb9cH=$w1JgV?*=!?n0{k=+l=j!kj;YpHG9jm{?3d_vDL*1l7A0I7N@)}rOMd5H!
zSLI$$+OxS;)zI2UwOL^v0##LCu@maAx=Dj#o@QAP*Cw{Nmt*CF7#^N64M$};u3dJs
zD-1%X&fuP38iq+&jHd1~-{x(G7pZ2sO=gNwQdb#IJWi^0$noH@Y;I`T0pB@Vy#GFR
zj~c|V9+DNH75E<KX1PAHxdyMB^mg8%E0Mwr;%B&O%>}4%e?ax(8eS|pWTTxxXKKFA
zy(y>xbjn6!o2<ttAw9WgZM8;~Qn(4-YFSzV?kOx4)SE0HOIxb*OU!$|zQ}Fps4{Pw
zo&09QdVcqz<egmu>bsPd&W|U&w6sm*0|hI$OQq|#ck|0Pm(CW}`$^norFnzZZz@R<
zcUP8brSDN2s>(OvyVIRBC)U?tt|TJ3Mm(nEB5Mq!rKXxYmgv@hzizR3Xed6pkomx9
zV4)*{nVgg5*OeEGRY%XlOm7=E*20m8ZdOo_-uv+wOxLY?j`CHX=S?K>trBd+mVWHb
z<qye<hn|35Hwv@E#^$tPwA&;3K-FZ0e#s?NWLJd(9vqii-+4hj-J2h(bNs#kogiIH
z=ksG_VT-jIy7M<WT7)fa>n4UQ{GRs1E2IgC)v}%&bS8EbE-t<LVWQH<Rdz6JSAs6f
z{V*;}q$yxeHb*2vYkoNN<IC0v{5;naQXa>zfi};$C+u|(2fny2XSqyFL1H<M)l10&
zX=?Mg1%`py&K@}-xJscGS2>HPSkpCZpnK$<$<vMioXS)ocd4DJB257DboOOqgE$%Q
z5m~J`h*}Fs@|=(aV*gcn`L|d4U6(<Q{OhfYZlNhKoh*azp0Il9wslMc&gyZ=jlfwg
zHs0*38N;_<p9#DgC}vwEtWRz{@2dfq-elP|o~&f!?SDR}iYUAfUu#&JhhOSWR!geS
zwVdC2ZY9;7BtNy^n@q5=>-28-3#kK3#XS(PpJuZ@|Jl$pp-BCDC{M)GBC_f3Td_9y
z=rWkXFvXg7IJ?9&nZ7Yw<~)93zkuLg=4iarIoaYlzJSwkdHza`;o$v-lNRUcU!jvV
zpEm7q*dv#0%Eh~**Ix)5S=_(gz;m>%S3>;#p;or&XdxX7cCuGzP~)hVw{U3SS>HKF
zbnZyjJC23}Kh$W4$1GaqvoD(NBynn|55_99QtXFm3YGY7747NJU|nFa%hcy$5-WC0
zw8%=_`Gqge{3{$&LkvR|0(xY+z1Pc68MFqWJHe*@$vljn=i^D-GX#*)K4g!E*{@f4
zr@6rqRv!oK?<$v4ARi?9gL{f}Vl1bG`3vy*^4+9V8>~9<cgLC~=Y^p1U-*V>+>br9
zcd__)jHOE>xDC(sQo~|@M%b*tCeE(%ma1|hjANN1b?rh_i<M?#z%bYePV{Hv&GwwL
z!zqz+krYqq(cLfAw>vwv*MV0EfcKp*ysix@a@`p_jzGjvv2Eb|>x%4Y09WLF&no!w
z!~mJk=EK_DnS<4E3?1aps@om+mfwXTP*z{I;^3!=@<WE%bLFv3X@t8Y!d*&K+SHAS
zFUQpzKjwB@QjHd36hp_N6A}?QAr9lK?cn21cDzA<H7;EqEljr+x-=IL7+pc!BjQd=
zog!{7&GJt?d@;VkmZMdp<qE7n9J89eRhUIChUqF8q|lL$BdC%@)v%>svR9S3=v^bn
z#=kuoZ8XV}j1~7N4%V1ZmxC&(7k?yr3(lMm+whgMTYvlV&3&k{dcv+-hOie-^+&YC
zyse?Mv|+W@G<vD9vKs3-DUKy75R+=YQWrJZME0AQZwCN#%^FbJMs&|rX5q1-`n-3~
z)MK}70uZTc#agRLABvU%0NzWz2C+ZXE4@YS2!@w3eMFUY)<ZU_u{b3=%$HI-_QpNr
zSQG6Zm+oNa%~7Y_i5v5IY?E;tC(w@YE|Lk^RS8|*(%Y$KcqV^%Wpev)cf3fMmYQ*s
z<*PN1kbO*mKR{2Oor?qMY&Z={%bNvZ>cFStYJTj?R6b<{Y8fT_PSs$_kHKtBd_ZC2
zD#hB}X~Dj7%=a2q6KjUTCt$&&gb(_-mP2M>aI>woqfkKVW~_VYaf@V2#Y-6SUTcZ<
zJ{Bn1e2KFhH58gnKfE{FoO5$IZSe7pMW{@A{c=>3)gcvSwBZ;|>ZN{B)7gddj%+tB
zpE`AnC)F`hFK<huk+IIk)tpqiYX^ZD330zaU7Tp;<o@Ystx;iB^Yc=#>lW_8eJpx|
z^Q2u$_Is<N@lOPJhCEM>3jKR|I<Hs8S0+dHyDPa264JeR>3&X)Z9S_G5sk@aX<xRD
z*yH@UujDJ5zB0{s&p4yD=SIKOi>vSRVU6q;Wm&VolPdiyS_L(<#2@?zLXtU5lT`y$
zH6Irr0ym|D95$sjX5)MFg}7v3pr-nuNk727rK+1!J*0|ScsyP{$;48<s0!zL@k5I;
zqG{A=I+ff0)HJ@5rraLf<W_xs5UxsbT$xJ`nJm#AtEjXuwq5$k?FzF3c7^88`G3e_
z)vRdWvYxL?-Pp4IKu)f5w$^lhuyzwKV7Q}ocNGkpct2XI^VJsTF=yGK>EIDO31@F@
zE3U!HznYT2H@9iMH|yYr+&-&VNr1VJ&2&VmP>X)-mBmCAq?TwN<2=U~ULyCj_DXjn
zj457-?bws9QcMV3E47BPXiF(@R}aDe+1KQBA!9pKpuO~}5;}e9Xt}^YsR@5v?csHi
z>8b|Ai?cof`;;rlG~NSU8Xnd7GRs)4j!`14JTk^aQ*ds`@EG9`IWB}4kgid?H8Hl4
zoGW`kx`3Z3h6b&9x4@W4g%PiEaal86s_$91&8`_izI{$4qJ*2R?Pfn~);acUZ04{|
z!YZ;3m6KpMAbDz#+5eW>U{wc@t_3zj%2zsgZ(l2TT6|cGjp4?PCzpDS`@+y{0|bWW
z=QQuSiR^9Xmp<wqU(IZmSaYGg#9}9OSbxm`4$qgT{+wdhC~P#bRf;jOTzYtRfR)F<
z1SOD55pb?q=?H8vd1BlhMt0WETxzC~%PO#x?6Mjh|Me^44CxN35__;eF4)U^pvpz&
z<L}^+vDNB5;`(Gr8xEbI-vxF;mo4b=D2`B$8#q71(vJq#O!YNiLYZ;I$^YGa_zxEt
z684w>qZ3zS>G|dKbSM*@%22N;9`fOB^R{}4My!T1f?Yg_&sW@9aBn=)*8IDNSZLHl
zKVB@=+PagABu?2LL_C5%4i7wgWMTcd`gm=X&@6V{H<-FEaCeP`gMKVx{^q%*R&Bm)
zgs6R1Ir+PS4|`*jEsY)f(9gtQyXCI>yXD9&D^o;zcpV)<v^e|arC!U5sQ`A|r$po|
zz>M-FVW?KU*`@0<N&Ixl{^e|&2VR7g12-tWey?nxI6(m8Eo|_?xd}JOLhHL*zT#S)
zP-JAUl;r~I=eJass-Gikm3~@VlTY-0%>J61+S~q<A-_UUpl3f57BQ$5EBkR(Izwz>
zt{%^lh<eJ&YH>K=JEdh*CcYVO$xd#}eZ9N4Y9@2o)Lxb%_4En+oR<1Mcn7=EPl7N0
zKeoO)D$1?xTL}S?5(%jx6bwS7b3kGU0qF+mZWtOwr5gmKk?tNkM7leNkfA$=j_>9i
z&+|Ut`<_2tuH{m0_ukj_%YC8E&+aK)5~O+dljxmlsHYzuwg}oEt5qBHzXQJxli8dr
z_QkQQpDlF88yo>*BqjrPgd7m_o<(_l;tyT-;4e>#`21clQ=lWq!-4yFV~O*^4keT-
z_ebg$)ohtVpNBIeQKyn!8^KW+p(!`##sUe*Mm;`|C4vBNak=27!xe#2A2L_CG2NSC
zDt_&hZh_V_DnJO83gpGrZ(QT=UqQm+3~5ldyA@}zGC0{J+^K72=4`%N>&is7VX;%d
zj{#_ZhvX6`#P$a%-~QFw&fty&SE@i=6%q+f;O^gjiY(h(a2fHLC{k`|CehjODy0we
zaXW}U&=mM3#roqPUaK7P$vd?+!sT_NMhe5`JJo9IIVRcS&V6jlgJT<}0zGWV_jq&U
z%yB6hj9jCNZy?%kyf2}OnE_7bG&@f0bsNu=$TqyTWk2RqWjSMMF$K#Qw^SSCtWjhq
ze*fK)34^zJj~-s=-09N1aJBm6)RGOJTwXTN5uUm$!9H#Uqf2$%ndu-jz~!@)>+SBY
z(6n~_b*bw(JZyd1hm@_JfRSHm(9ArdnuV{aHO^3u0!VJ|a2vgBl4+6nqa&n6wJBn4
z++GObsXYK$;Ql3BWn^S@UNtQ2PJhm^JJC&i`;)f-&g+b3cy!aqIM`U79~)y9Gj~*6
z0`f2(btBvhR2kG<zvkU;j$7d?XRLn4;trE>VJxdySX|RKSOR;&FSZ>5tZbj{=8wN+
zGC@9cUs{Cs()k$2YEtkGQ8(0l96IcX6lCJR&pf|BoSw5HOih8o?5K52a1MCEJ#4Wv
z=`cRqXf1!C(n0ak8dGJ`up`WL$j{Vj`w&d=*gkkG?V@}b!2$~H9u)H|id4aSS0*`R
zQFKf&o*^d_Iy&B&(?v@iOcBu`ADv|+3>ZG&RpM{keL(+wPhaoY$)ME1tmPo-FsmJc
zR6OE~Ch<5%`BvgMri808M~cD$>c3B!UU#5!z9OAmvyL91ZSnPq2#)B@!9@ZdYfrEZ
zlXDqre^*c)YX4nR)wTX&TpwLHzkY#Ak-imN>;k#9=0)=;%-r*!oumVo8?Jf$!w8^B
zaL$z;W0jCgaPWP}EEd2wh5FS~O_2WYyXgh_->k|voF)Kew5k!bz4?;z;g`Iis(@RT
z+(?EKUOi%8^wxXfW>DPgSSd0(6`)uMgiSOt*(sKufKM>f+mcP@o)Q?$U#v2@uP)_a
z_phzbg;$AOf>x0pt4@<!x8)_2de4+MCkidrKrTLWm#YF*Z%ul2H)rJehP2#|ohHiE
zAz-I4l{&9wr?9RM`|*woC!RZq#pa;c*f1ef`OP|m;&2bDs|D1P?&amd-}%Z##?>h&
zss)M}VW+Qm8^fYV%24GE26uOM_I(5jVVCsK{8F~=I7MT4L8?sYMwo~ElR!fKpm%At
zND_Yt?4bDqT%xAqb-#+D^EkXBJVb&WB$r_A+<YQGqQvbM2oND_3)fm2Q8=9<NK5!^
zQ+&#MKY6W&#^Fe}Iv#sG`#%V1TfgQ`Cl<TC*KslnKbDvQi~@$nV+?Z-@TcIa38(8i
zKJml^H$%VoX~tgNiU&V%+OcV7<bJR%H`t&Z5b)L|4wuSb=`dVw+iNUeer~t+x0?5R
z!n!d^KLlC=Hl-cbV<M+puL%MvJypT8@>KFpzB%V%JDuCn5hCcg2)ZtQ)<Da3-E^s7
z@+ehFJ2UU?rCWLEg}we)VI^wEKa=IhdRpqwkHj_#VswhS*echn#M8{iZED7I;cY5c
z$6u5k_O3Up=iA$@N`0fjM@u1nj0NR^j>(NK+5L%cW+tthG;Io0a;i8sXIcoMu`=;y
z9ON9fiPJU7I<vTM2~%CiFM)D0OGiW)y(8q2;9(az^cPkuMSPYY!(b_&OdNBO)o}XC
z$=Q;PTXQfoZ;!phWG&8vw^t-pongtAXUO&8c|sdoMDXkgFjB7}NYt|iiaf846iv=t
z$89D@UMcsNYC&eTD~v<wcE^sb@^wQmoOUkPZY`3-rUoNTXXo*8tC7}gq{kth^OrTL
zs+V8<XD%opg?XC2TWs3(#@+V4<|9M!&biY}8$7`yd(AfnL9mVa&O>{e%bE;F%G&k0
z$kSLZlzuboUu?6m8M%f}{3r~cq&*(=?W2-1=(u3<d4?`xv{jc7mI$#<Et*g-)8bN|
z)UaHkj^_}u_{6f>ZPr^WrPjE3t+b^!ZMP|?BFnl}y{2lulQ*^5z9iuV_}0)h=T=jl
zHtHaGU5*AjO|K7??4qvUBZxD_qIC(|v&Ar9XlzGFmmZ<u52Fd;uZ&~gf5^)fq$SZH
z$a@tSGw&oYHKn90SgD1tj(?~CO4gV}+KoY!bdNH^%*}w9azYWo4JAcr$FgYdkGjgo
z{fWG1_Am!vO?gG@lkIx6dPDl0$r_-(IL*gi;8aWZGSizRJH^7Y8Sk?_>IQo7rfz9x
zm<gwpMDG@Onc)v|&m+-L`c=hHnO*)xFLB?86y-vb9p6+x?VCOqzG)DiaXtOcwx5B`
zu}{QnYIW+IZO<T_Em-Q_?e$9YbY^X|NAnA?)VnnGQhguo%Vitv7#_Ql=<QE}7%l>Q
zJ+s#x+P)s5Xusa9`m>%4@!=640zxJTFcf%g7FX<rWdp5O^YO8mXa`yHJ8gD68<^}|
zRL&VNtJ#4yy|s8g?G(q6MkD8X^S1GMv>)q-{w&gKDzw_8ZZLpe5U^r_r08))DaAK^
z47@(=l|1PN0#x-zTPu`O`$L08(tIc!27Nol^KNdu_ok=v9Ma2pJL{4a!saVJeXyUg
zk^{mcJ9ZaGDA%ZnHkfPHcr+KotX>3u<Mc5OX+12S6l5IvM%8?%^kw@C|JiEJ$!;|`
zJCx6$d2X8;@?Kc*S}%L<yl1`AOH)C|b?n2|LLSUn&;O}};A5aK3D{!2H#bKcu<_o@
zox8U*4mHHp*<9&*>|skqk5$>s{KR$Uqr|c0D5x{wRPd#=(z$T_T(%?=@n@o(y1@*f
z7BRXo&r;RYLXpXOwfi)a0lv>YEAYNptDUo>t4pVL1EYzYlI9t6%<NuBXxNly;3?DY
z>4<n-0r#&12*n<|e%m~d#M;Z^y*#Npecfb~^Rzn>vTIe+bWsNCKt6r?bRt`ZJdx8u
zT*0$n`oKJxcaT%Wx%$<3{-sk{_J)l|+LOSEe)wcSWbn#|b^_<eX!KmSQ<pG%=+{w6
z>n66jm^Egd52e1YM_715gI7+dLv}67KS6v3^G^s~gJ;^vpHJF-4!2y7fjE`clB5j^
z>s*{gfKyambMfRvj(`nfBN{IeO`vSbuoHqTEAV=Ciwa(O?(fll8_^$63*N}}eM66D
z(Ek+ByVqN#qri!i10G{8BlPvP9&ebbqtPY{#e~Ir^!PMeqq&8mKX5I5jb|W=Gz(P3
z-B_|KAJ(4?b<_q=Xbmf>9(}+a&rt@5BYkC%_hC+l9>lcl2r<TQ)-4&bn&n!GVX5i-
zbu^mj05eCg)lYpeMPTmn<3sP;Q*Ta_0ScUI&H4qeOCA-Y@Z5Q6mPW%0<Nlxycm}Zf
z_hS2G14Pcve8bTIUslzXY%;Gppk)fx_L6$#Bb4F&w(l1%`*TkY-?RT(6#lxoe+~ry
zy)Kly;-I0(9M}o;?9A9tWov8hdLHRq+M6TTzJimlOuUx~@R$a!PkGue5Q>tR>^5H4
z*#dm@Wmz5`93s%C`Z;(4tp&!6NqXKcH>Y+L<AeD&-r%Jw3Fi%O=CbkHf{cSIC=|_y
z-pfPr#?OOqugPIl_O|rCKU^M_@L8B6T7vsP?PR`*(rMYAtji>Q|4VA&S`#{*S5_Bl
z$j@Jx1I&{g7N*<l7UX{$^Lh&|eXy9!6sCFId1b3a071{<qs{y9)A&g{%}^<MAFOk4
zkFCf7&u^gSI<Hb)(fK>i<v@;AzeWo9+1F&pl$uQ?9D#GhfGR}OTIaB}y6XpK@(BHq
zrL}6(Sf1^-iR3j(%DibJdhbz@UotF&HcP@E;q=IzE&;dmgI_BJJLMRUJ9$N0%5ck&
z7L$&UhH=RmbYt6a+=(1d?7edUkh@}Le@N)#x0F2K*vTg;`c2P2ef0Jqa}2N@GPu7Z
zKJ|02eC|Wdo~!9jE|``Gvq?Bp@tg%uQnnmGstX1x&!rKYLwJ8wWpBj+&0)`w>g&cC
zU<>m|p9=4LfV`6b+W+9BY-Q9~eYc!w0=iC}crv2d#N51=3rs9by?yxxTXVxe_giOd
zei-}bV+xPb{&TyA{_p!Spgn-(1Hd5;cfbZ8y5`Q$rV{*Rn?J+-_WIN+299J(sdfvU
zX%Zi7!8mW|CFyh8Yv$AtN0Rv-_gPK<auv&Wx4lIhhS?|goGhHuYXs9f&B?l-bEbJ(
zfNpos%)FAfCWsnJ4*VWpz3>P6sJPc^q#GV+F#pLeI9_+TeD@j*JS;&uL?!nU5$kk8
zip*Dw<ZrUWwK;||3<|j~Bs>b$U}LJd2<6u){uCi)7EM8&ercTp{4S`$&CLDLtaf!;
zT!%m8_|v&V31_U%B-2OmSM(5!PljntBf0O5n9)qZNLTJX>7os=Jj`jQ8hqK@)=VAp
za8(FUcx<7kYlRS4Cd3^E^?erHExDkrh(M+EYRfH|$fHGhR7+r~MV6tM>~dX`!aEhP
zR5Ea4jgi3c8iC7xu#_0ftZfeD$$r%wIEfPvC7bSMNTndU0T%TgpcOpy&q+k}@zYJa
zi@4;lo}JolJk_}b!mBm)F?pferrsB$sk2bqcqs6*hu7FGNqwte$7GM0JKyl9gnk!J
zGapLxNhR0pW727u+WUG8)KOYHg#j@J!N~w#Fi~zKU7%bLTclk-VLXtuG4P$jCL+Ab
z5qtn%{ccbQ2%aX=#jru)w?Jw>zo1PcQf1~M&q5B8wusXgKtRWjm2U;F^1A%F(i`Wl
zPwL>l5~G^V<!u&>X7vBb82`E;N)@U9($QsGS~~*+K4|ic`8kQ8txu5?9`@Pg_0190
zKHk$+mQ<Tb0Vk`n$8leUj;1cdXD|rCFO+O1q4hH*xV3FtYkz}+icxZVRgF6%B7}T7
zr4J?)$ja$3*}v57nwji5SEb$ralGCRf-uX~fAgZefQw=1E?eaRX{0oKhnB&gm`LMd
z`8NXSJ*;WZdnkw7!a#~^OtJZ*R(6=|7$-aC%~OEbsM^J-O8ao{7H(-3$!S}25IAWi
z)#GvNctIm6`iQFTy3#RA1^}c%R&8Z65spExvLw@9KKvniq|ZeP^?g~K8KwSYq0E(z
z&uZcRM3#|39w+=zqh_c5l&&5Vh+2%J`aDu9R+I5h768lM3+johwGqNOdCjL(YyFWw
z(wEC{JHa@63v*#pjgS!wHl#$Jn#B4)l;~3$U^3pgn~bd_vDoWVoEn(EznM0#Hr4OV
z?yrhPef)JODZGhQwx_G7=g?|FL=JWUYPRw`;c4S6)HSK)vCvA~TQ%l1IYdjmS}_H8
z0@<E@l#P`$;59NnXd!ZIa*(o_eFDIjq76&5>#MxIlrxq-Wi}`m7-<Y_62-2i*hM-I
z=`qx4OvhH!^GPl^lw4c&)UclHtaC*j9&WQ}&(&F13vkA4&E}QKB-c)bfh$vsp!Lg*
zhZD{^3z<gb%>yH|T0a2x=_B``J;{4F7u<%YOFZCb1PAwq0?8v>hMz+}wG=0R9112B
zPSw(~6M)VyOO<dn*m+wnD|@z&_AEHj@p<*B^86MZnpE@j=#s&8l;8((_6J||DPKK2
z#=Hrd)G+G~{SA4Pu9cs4k?>k@S;4xaJ@yPGh_d13CoD5sbJpSTwMi7i(Y6D<$V1hT
zPs70yb|{><k64!#nJwF1k`!4MCytS%RR(ArMv6dJgi1&N?@SMqZqA__g5caN*$(W%
zuIoXRaZ%n0_@n*mXKDb|X}&4n(0=$v?HaL%!3x}_8CeJ+Zl?QM`-Z{rWPADq5uAA2
z@lL%@4y0inXe&?bG`g}Z5(?BU{PM%4++3jk_DyuO$<zJvDD~FzO5fNd6^I>~pmz8s
zsv!BSUEo*X(F>ro;=VW4((3B{$kil=fwF#9UGJmBg{gTUSFF~y^TEIZqdE|~hg*)#
zTtm&v%c%W0jJ^J1vw&$Z8BQ8E2FrPDssS@Ms-CqE<|`@J;b+Xl(+iy?+F1zA6#FXt
zf6wg){XaW4X-_2<iVr;#qhyaj-F76qpvBa5%9~b^K>vBqA$Y=sl{r2n!nJpG+CsN+
z^U`9XxXi}HYwJ?;LjEyIzjn!j(NcCe(>FocG(FkL@CN4YwGY+71H;x_l!gEkbpmH7
zku0~_`Us5AVUG1Z6uZzk;gKR`i$G9wj1;i<mB!3@{MuX5D6A?3KtGly$F{&p!P{ab
z=;aGspH)X^wBeS-Flzmr$Su&qRooQS**I2gjC#L;^TL*a<?+`%N$zroUXUDL8?`|u
zA=R01`peA$cHq>;*Bkn3U#F^$$P`3`W#n_0=hs955M)TNLtl1ANl;YyQU$lTG;MmH
z`6<)2!=9Upx%*jp^sKd%@*FOBi^6AhL*f?TxX9|!t<1Q7bh#?}U$$2t`6G9=*5}bt
zWL|0LQT&Bk^ibU=Fw7OzEjl)r2L3(Ow;v0pbA6Z+d0X?x^Q@3?m}4iYk=pKhSqDj@
z#YEu-e77~M3Y{%mEpgYAk%g->--!2J5FcU#0@3fzS6vWv-+2n|#Aaz2nlfG7Drv6T
zovVif{EY{n`B?5VKHcrr>G6?G>$n89d1TnP=)Dv%rX+#!HLLO3xFR+7h4K8T%o3%%
zpmCd#Upn6Fdy=kPsw50lma0w=IULHd?gaMS6)4bcI2cN{hODm{6$YMn<@}t_fP$&N
z+xnbst{s=XA<a?v{e3cKAiHHl<hu{yCm_AWK*NrvLBz6<e|<$x_hvaNS5OPRT^=NW
z42(1lt6Z-QkS01#u;JI?g|*2?Q}hJNIt^JAuP<k)xROd68c^)@Hpy5u{N#K-_2gR}
z_1Ha8FJ|Jln1>CDVs009zhe}I^dR3DSf#GMvCv&$dgTv(C;zqkrq1pF#>UcdHUt^#
zn2aoaGWAV_<LPM5b6HpI$#9@j7ur68fKVXY(-f}Hc%S$eW&h7XwN;sqgivi{r$&Se
zUUXFua?H;&)d#SfjkLuXkwgLo5esmqhZjTT{P?D7+zV~B0F!JrRjT)#cDT}Fav<6S
z*i048f#SM2DcYB~J`YxRzu*b3V+kzJL|p;u8Yr-#kj%USO1xOAfZuSD{xRQ?m<QYm
zbBOxsyNpZdHFDC1gBSyo8Q5x7qHH#2me9z$<S2YqVEH##k`HCbgdXt%Z4Vh-K9U`8
z{)L)`KKyew`*(2B5{i~ChJ0_Q>qd?F?I%}e#PGCgo?7_00Vm4uRG=-4(y=hLEj=c9
zJAuJV@y(CN$H45o8gL&n72p#%xL+lj76?Lq1O96cr)6C)wt+=oH&5))e%osjYvfp5
z94p^8(sXzo>j@CA(6*l`6Ai-2@9eVdLKMp|*;^@mj7NtbjFK>xZBYSn+N&w@AUK3D
zhbRHui9%TpXXdPN$FLaBwH~pix7SHYHXvEJ{A@LK{NcmJSQR@K`@*35$7T>vTpd_x
z5N62hwWr+}dRg{dL7rc@n9mq1J@_l7iI=l|i;43x!ln2nn28g5Ic&`j*`6Q8-JTrx
zw_*XRK+FbrDHg^AcI@==9p=fw?K)>?n$Ak9ha_8-O+){@o72?5yE*;OyBS0AqwV@_
zRVZ(Esgx0XU7~R#Gl*{DgUwP04p#cP8x<Xc^g@&8wImwF;5XTk=_Oa~!^~2tm{;zA
zJkZ%vNh|$3P9RTaFZpTsDZHxChHZ_$xnyf3D;PZPvsRw<{4e2{=A@Q$ksZnM6Vbt|
z+Qoa_YFwXKweB~PA`H%l=&gnBm-w&(<rFT0hj(c{A&O!Y54Jb1l%)?6@F>tN^9<~d
zJ}Mm9YqOlOvY>(bePj9aZq{>Y^cZy=%LyR5M}U&u*CtS`T`EDs^?Rmu6meEyr_~wu
zi{lXW@5jfm4~H)&oEimF$dl~}P$kj#pFnimp3Vtvu4mU;qrLS!ULJ1vrYV<oCrfhE
zdoOF5|2%oqfDWU!EKkr*>+yMR)74OKxC=DRYzLE_4*0Oyn$_0O-#!@TjW=Bs(Ag<@
z`z{)PwH(>x^h4)4CsmpeQA5TrVS<&6ENEo0n1QQ6$~x@&bYY=c{>82=dqn09&|h|@
zJ3}GuIOjAu4Je$Y&>NK=O9F$}og4#P{!%b9DT(3>V`~!I+p~pgA+NJVjCvrymJYp>
zIoWkJe)v1CO~RBuu03w2aJ%oEjx6~4SOPt)NV7Vuqtp{<n}IZs%1jn(7r7xfBSUeQ
zi_iS0^DM#%P5p4lSDo<`P*#y0<Tj+YHaMhjegb;y(@Z?M-KI-xci<qdP|f`BMOH}Y
zBg37vs^fNbs+;=7=n5bbnOMB8eQbIsdx`_aa&Nw10S_|Q++G!Zkw*&$gc(}i+c*zu
z(Q7Y5>#epcOCBndmG|QF0c^KUs#Mg|DOYE{y-!CCRz?F-^;e6%58E?HhwQlOf2UzL
zFvSQdP78c>{rwpugB!Z6XOcqsQf6ziDuY7Z;6pVuwne1yNqnts9-Wql4M;A@dW%b7
zR?1G2j}dtO?g1-)d}5fbM)@namy_;rm~CAP<zgj_F6ZoCZ3SKA_H<F>4!mA*gbW0@
zq1{`HtfUnZ&=LYGNTxuk%)ocG@sEWfSzE8hCU_P4;%_Nxv&^g~pyvTJRhn^}^GzS8
zU)67dS?VvlAUoam8w8hAyV`I!BXvSk-rA*1T=J<_?(Bz;W%A#r)OYFJI5vbPa&nD7
z>G&g7D^1477xuG19+v{5w1DE<c-Wnf#hv-v+JZ@8pCmtDL8|fsC5Bq-j(zMB6{n5k
zUde;05HU8Z?w-#X^kp_vHS>|eM}U$Hjplhlad7Wno9K`8uw#WrajS(ro-^viqa_#3
zrjNz$tF^;#^Cw?quBrbn_dUAOAm=JxOQqp3Q&^xZ`ew03sqW9Qwz_(LR#;ef^OTft
zy0jOkzVqY4tR$}B2J}WApo?>ARtBLY5bljoAZd%aqk#tB<foHBH(K&`<^KKstiePT
z0=<A}ZJtsZw+tVuAEA?qUiPnv!N&=NH|2SMWldS0Wn_d7hBjad9(d&YQ?-a&H$IJ{
zN-lO`9#^0c?2*Ln*JTZ<R)3+Opr?T1qqS*)=qX{f(poM--+B<C%6rb%q05PkK?Pr+
zK&<#>k;6108`y>^X-6m@sYdiBeVaE6N)paYWv9A)(&9=w)}I~w(p}nhY!Uh=OK-Js
z^=cQrLJ-JW4{#cG;d5Po4WZx}A!gFZT!h-3&RDQ)Ei`*8#VvVKD_gP*`^T;X3boFh
zx4U*fkDI!WLGjxgi#SFURT%B?Q75KMs^DC-iAu4Sy18w3ltd6vp-Qy?6qHFGodJQ`
zS67)H>p%Pp0f>m~;v|G^;LTj(T_e|q%d>f*B&P}Wj!3yKATLiJXwlU5)CE(|t|PDa
zTk#;Bz68CIDil_}rA!NKgZ$B=_O(ER<_-`mTU9Hg?GBVn$g4NHpQ@-!Ym*x$oel5+
zYiFzga!fJOd;MQba(pCzL%KbyO&GxGwA+~}PrOVE4zKTHCi5yAx5Yz?@NRHRywz7E
zsupVucoD41R0Wy$oPwX1ae5cYXue@0vnj5igQn#vR)o#n1bd4s)*&K$0c%v0ci#H%
zCJeaUm#|aU=EUpd&X=#s;|FZQn<jWUww8f-aj#$V$@|tca`sW=lFx@Kt!5{z`FH7U
zmh^^ePm`Afe=leB80y85K?{Txu-Mmk1=py0`?@kMoI@^5BEuFA2aK*V$}hJqO0;L-
zdiBWCiQOOa`9gK1d`?xN7y9YKwo?gD{DnsU2(dh*!l-$dKyul_6Ia86r`nEMN*9dI
zmLJ04S(&-N>_oi5TbLGViQ!TGO$l#10i{$UlOI6v;58!S8;|;znJ#k3mX^~H_@-oI
z`DV`Aw(q~~hQL#UpuvIP#MZC2t}-%^^8%Z6a$luLPO5dO^q9yekl8@H7$7{<YrS;l
zf(lAGH>gy(Q)5-#UfV~NNz48R&n*M++*Zc*Kig>Mo>K!IM|j2kWT?ZQ>(S11cq$Uc
zjdI-S3Z9v--!`x~vbXY~qZm|UpX24+S-zU!duQoaeQGw@S?*A=(>|zsZ0Dcq92$Ci
zX0=r@NK@6abEpc_$X+Sb(Q~yniQ|c8m`ZDvt(;A=?A44;&fKe=yy0x8#6SYOr$<UJ
zGv7cm@nHshBRlF+|CH?1chB7V$R~E<hY0p*ANycONW2jtgY5BOIba4{ZDvnRPTpz$
zxJ&r$3kZ0wTXESd=GswBx+wUU)?|`^bTvT8&<+x#&3WTz@hMl#pAzgC2u^?O8EO_Y
z=;SG*;gN@4EySTUxfa>%!Cvhm>p@`u8xjK0DwUl22wLwI*%j}PalbPXHrYrv_FHv%
z`w0aEL*iAn6gutTDwhZ7*5j^hxII76?NGlfZGMHlxT>ifa%m6(<YPRiFCI|V)^r~)
z9V~ynC&C|hPf=zCa4^{yOQyX}Nz6+WVLr@I^t{Lq$JO1WJ1#58UVYaq#=Pl3Q`t7u
zvDLX0lj;({x$_#h3-F&(7fDPJ8J-`mkXfI`CiB!h!+$Fkz5k;j(E|A+OM)!%2u|Q8
z!A_y>I3$cg6ev2Hul!mrB~Q1xY2p0qQqadAvDuk_T>!N}>AltOpFYD%Q`akJOX1c0
z&Q%$ifNoj5sfPb_*5L$bl6FJ%9u0y)T&^fmudsZ1lh|JRu%oJHy@;w4=?7+CE9Y4|
z$diP+6K<c?>gH>!ky}-KNA8i0b`ss$;*vGf@*jl%o8`sC{ATgboNM(EEg^kLqua9*
zdV7v{X0;JlsG;BNb!85k{hRG=D=xM1J_2e3><*kW01-bK{q5e-UUh!Vx!Mo`akiJy
zX@hI^UfePy6@ZkU`)eT-pCC<NK*RM&o-Q|H?M)MAfKIQ^!Zl?Z$H^wmww3eH5;mWX
z@1!{FhR*K&1z&HH467wKr`jbR6q!Av^Z{yq2JR%B*(TOXrnP2(gyd}){^t*<8_$%j
zz0XlE^$!b`oJMN6(Gu#^<Dq-L=xyHtWu#T=^&7UApfAxU8z%mH^Q~Jn$4O8)p|3|(
zHn+7ch|l~3LRwXEm8pt+j+s+4>TH0K#hbS!<=l2Qb{2cODn>{oy|b9RJ-;r4F#_0w
z2kxY4dn9X}b0icNNQ>u~uHNp8bCvaVl~yA^Z|pmM$j<;rncr(A&lj4wxVV+cgS|uK
zzg5lV6}0Iu`)BFjc0m)3(4+t7oz8ebW#kao55tw=(@25)uHFi|w@i|i!6Q!&Nm!<z
zTATbdDu{z8WmOcZ7+YBkr}?_;s<*4G6rMn@4ky>(xn(8iHD3lBW{La0>s!^GmNm>$
zR6D$HTB%>t!8VbD6}lmdl-G2s^oKpX85Om3^Av~*7BYSNN!fY;JIPX0+FXxkOM{Pl
zcHEAvR<7Q*1RHeQaBIBa%VpG=ef3t~<69%Urbl6S<|Od#Xk1&PJ;v<|6Eret+G4*!
zSp9Fb{d5Em*mHhcVy&);g&7r(sr~6szW~A?E*>%Ag0;^g90MS2tP#m&o6*6nw3M|^
zSQ57zgz5dHk!3XN_{Rwas+J>>x~yk!!TneQp8e_$cCT2<6~Thu{do1OPxg9Kx)l0S
zpS{A7r_zuKh$Z<%F9WI<lET1+9``KP31Tq0DmdIzNn*-!{Vj((p4OgC^2}48gQQ~_
z;)pub&QipXf?6K3xfF~eFSuQ|F;!EEI}Y)0o_PPrb36>L?my8s`GA)jrGJTN2Qvo{
zEzgwdway+B9s(1!SiP-8wSXp6Es&53fODIkDTt{Lih(p@At*Q&{-O?xz9cjIi>=iX
z>1NaASqy`=A;W;xc^gn|zHmBxbR_va@=1(P>#)|&42;e<4j@^ldDfCL^v2#rb@#pZ
zc>Qn%=#n(|zFGnm0r)TUa;F;N<v8`jWTw(gvM&7*_ghP#4DH5AHAj}x+mW!7%kGhq
z7WMpH&u!!@ryal^2mn!5&FB3#v<of_TwIf@1IZRZ;(ZGEZ4tj!9!aXnI>tZtzyGni
z!}0E4GXatLMqgY3Y?xA&ib~75&q?rih9qdFbx&aXG6#=m&hMCX^Rl|hF<9m%&!zq&
zeoY1@yZ;He?~J%?dC~dS0OWZa=_il9v3!L(Snb_-!=jnoyStkh=LaL7(UrT@T;89I
zvxb4ZE@;Sdw%%Z@elB!^U)*l~2%eL_@1YbQF~_&3w?<`K?B@$jYq8CuW8Fb?z3&<s
zm)@z%Ddq^vj@q6EN*OGgE7;#lrv>gAexWmbAtAr<mC^&(!6BvIu-Nm>!52qnqx>b*
zX&-I)3_)1sNRwG>?nJZB^^oAXmpHrZBmfimo!u#sl?<jgDe5<W#6?5n%mCNo_1sQW
zx<R(M+;{WSMnU>Kc0ml}mT`%fV2eu467fiPQ8PFHjGjqf=2PgT1t{Ag*pXp-Zc==E
zcr=_xXY;Xh!#Ba2u%xP?msmaz9*-Wl?`>9ykY0S@yD>YQJ%)G*f98Rf%%99c7N!ct
zdefh3dn0^yIpAaA%}@yK^?0NyLSfW1gX-|nlw{9mp(CI#Se#F$qO}jMD_1qyoMdV-
zaj5l5;vWM6*WE>-V<eeI<4pvqoUgI_Qc|J2ckOwbe6aWNu$k`_pM9@eH6M;bd!z|s
zZ=vDs3WNywWgC@!9Y}IQYL~meM_CKkxj_+Fp=I`P0G^nMXETuBZt<xe5ybkrTq37c
zq4{_|AT%|+?Xz6%i?5p4aa7dF{L8IS(+4G~e`KrZl89Av>=bS4>VQk~Nsud<82A29
zsl7V+!YgEUN`nLfJvyKzn{4wni5oM*6nL&PD8|-sn)R21!4^WG@K#w*jRR685#(@&
zG|>uOEXhkI<c1=KaE{vfX!@iW<;B5;b&b`#^r(l?;c^Ou4AsYD0jIE!Oya?<;a(I<
z#6e)*l?X7eWu0Vx-uD_X7p}m&M@<LZ;{nxeY${0|`Fro>uU%xU83hA8@_g|OV0pX~
zUu3q@sSKD!q4xWkjZR|xzw!(^!=S(rT~5Wa*K}Dd^Zku*>&`qC(DC@-992&>%6+%i
zq<%mcINGWxI3M@14*_22^fZ(UH?Nb2!)AWgx^{)wfC+qamQi-}NV7V|_wQF)FNdy6
zAj)p3!!;_r!<^?=FjAuF_Qt(_Grz!c|91<tpJP&D2M$DG&o;M6Dp>0YU2EDja=g4H
zo-N|$8;W3XB1v$pt@Yl1-#WnkZz}I(lD`jY))CqIRF*%?a+Fz0B||)RJhvOdg{5Bi
z?EL=zS36y`eAy;GO>*$N7hVnFQ1Qh*wJ_Y7pQt`en2>icvoYRo1EJ2jbh;2@rW7Ls
zuQTrM0nT$a;(-81u2kC!nECfEyU8u*>6#5+=jN&L9kinY8NQax-KliZ?fj!~Hk)c*
zj!g+8aNqTON1X@0NWz%K<{C$CFZFGFuX4EdMw6MA(7k}q3cOsDw|vzdoI2RdiX`u0
z&v#v3cN^@1jcd0aN<5L-<Sc7WvJdX7;&-oYJQ=r`&vAy&bCxQ$$To6kY03pZVA{cB
zxS{E}AGjK7XXElqj?4dCaQC}UO(xvu!KE*TwjtpD2-lhpyz?7U9O%Uw)h*Slsh+CL
zWn3bgZEj8Iz;3S&7YHcBe)Ieykn}s*Hgs+{^zmeC{MZ?hjkM5g#x07<9&Zq)gnLgi
z>tIMCe8Zb1%UQI_W+;EK^Nl!J&Uf9|Ztj~t>PB!lYmEV1F~wq56D4aYL^Ng5E7keb
zp^BH`oJN@;Zs|19zO=qi%~^D`&sTcxOI7GZiNWiw$-J1d`mkkZxyD-ccR?P!Fk2ou
z$rKXNZ&|zC7*vHcHs05*##nsMfr`i<r;WQ^`Dc+33O=#x)Y#&E3m<N_G)0TkTlvrT
zj&e-qtM-ihPmd6tdbrim{>Q-`M{z~vc=Yef07R~8fQgLHUP<mWVQJ|S<s~*se@0yD
zHZ)QFBa7$TmL_Mbf0l(83z+-$vDkstBLz&mxa|CR*T6E_GcFHVxs<iM-0lI~7SF05
z&!3f{;&^xMBSt{%w2Au@7|GkkTi5$5pCO;{-aSe}7j;8?!}j(dTC3Ah`f{IBwAOI6
z;qe3tc)TTC5ILSBChADz?BO-Fzeo!%TFq@Z%aDe3UL!c#<06`|k!6(un>ob{dQ2U1
zOvR)$zFIJs`>p8&E3tyOA;iV8zNYem+$SP-xo8R$-}}c&0>(b1QuIKW`$Ij>Oa<@$
z8Zko*z?h({>9riN;5aVC0&<j?6&IX`SqM%kI<V2>k=$IZjAqkPEw=eP<|Nq%C3?NP
z6IWlJEiL1{F@#g?#G9m;-I@8bxTPMZx1T)mG*t(r%)gcZsRdi4xM=!fp-2}Sg*V#P
zw`7x^ApTZ<VBk9%62Yt8S$n)7NQ39lhzNSUPT##04z{Kce!dvm?=-1xvU|(XOc;7x
z*$}2$s79b_RfUIzg^&_|kof*Nwhd6=g0*U`6(pTTa}ptfTSh=9E}B{D(&Tcxk&{|=
zx49QTy#6-;?A@OTn%QF?_gxDnjq2C-<%3|aNv{O9imn^5NoS(mStDIh<)Jh`puk{S
z@Px-Mg>yf^Iyc+Us$bk7_wbemGYcHQ{qjVCLQj<(S{?a~*vHQUU#^(S&gpTWxRU}r
z{jurK4ZkJpkTCyO-`3<IZQZi;h^v!z3pxd<eDIQuv4<4}o-bg{NDJRvZsk^{-Z3<O
z20fai0M6iwzqWO9g0n1+93L|tT0$iuJae{0j)ZUP=Unx&LRS@3yD9m6a5<yPt877t
zuT*#PT~l5V<mR-tkz?0i3D#QiNl3Bld-|Enh-3XFEL@>-!@pU;+Iwrf_&2t3o{lCh
z;@ejtkz|$7&1ob9MTBU&WDy7iaA;@v*Qsk%F_o?FXHx}J+l2#WU?K?V7$e~{Jt~hW
z(6c|L*;fxxXuIBM{?R(&Fgz|T`<ZCdwBg_fGV&MVp&Qz#_VBC!Sbe0G8cd>c6(yT=
zpZkH^<)_eULUWq@kE5E8&Us=^1FOp;!)Noq;Bii8bz-#Weh%L>@OS(o>makkr#Ny&
z8m)_b+Ue`Uj0Q57|Duu(%NKN;Rt8aep64rDZ?_W@y@Y;b6e5g#_!%R5<oQR_GOUjT
z;*JiaqmZt^W@3mmBMWYdAhzxC+1m*XAY{O!;qYT;_QmorMPSJ)vkvK*ojl$!ML?FI
zS3dB(dT!=EaO(#WS-O9aq;4)rH*F?b_GDa45%g1nuYOEb7YCq`aoGe%t4txV723k^
z!!{R5L0vdq06BG441zl_1AGcCzUfc&25QrNV?Chmtb;%;l$?tbYJRs~UaJQh1uj%Q
zHU(<KIJahCzfrNe6mnAB%zxzBXR0N29S-pAon<7d8Eh6ac>}=|R(;=DC3&bfgkAy1
z8#(8^a8B29GV35$e==hc?ZLymiJa7}M#p}<@NV&}6kxH$1lxIjy?i(`wn)(f6z}#w
zctrXUXb)y|<R3${2k($HpNW~Pk^akw%7^tg&?L3yx5sIk3kL#6BcnavURY_qv5}3u
z_8thsH=^!i)Tr$$c13V`<!Ng=!gN9L>&@rLM4(o}r0SvYCN6^}jx8>*Qf`Gew#;xz
z&UV6e>*whE$aG)G$OFMmA69x|b~0NS;7tN2p@QyFp|Kee;`IhWDda~<47lx9iCdFJ
z>XQR@oy;(c*#YyKWKD!UAclJB_(TboKTct{@^EW}wu~+4isn`0#YzFE$-#EfZmvjQ
zgtp&Q-er<;;Y;Q_pM?8D!z49`rodih+6YgV%vo~NVR_U{M3!weMwa?_NTT4t4IlIy
z!*S-(OT0QyH)!RN|DJkb9&i97K2|eZZ6ic5Aocxi(dA2hOoPpUyOkxdAVaWzS<7SU
zV{=7Nm|IXtBJZ}}(G^4&PHxaF$6;kRI~gg>Et{vDaj|<A&}NeJu;ye~-ug5<i;Ny-
zhl2AgMsw~e_;t*jiFpnlydQTR{(R?_Z&|b9ExdzUVW>A%R)Y89p=J@;_|7kp({gQi
zwc+<?)3vTArz8s%)<p@Voys(+H*ZFFJ&YWePp61-)RdjI+7GuW>i1LD=&rq)1%`KY
zq8FV}cu$K>=&n5zq5a+cz16z&tZzsDv;Qhl+>Nyzf9=0DXbC{t#ic=**V&IZ1QR4!
zU@07*zf!W8EdA<#6aSR?JuUbv@zq70i$f@0!#z7pm-{`tbp#8-yc&ouX~7jCIVwbV
zJ38n-_<&eRm*dH6TvVuKd3`a-VU(^Lf|PDb4VCk7NP)Z_0E^gFSJIRVueyS%Xj0Cp
zHR4v&E~d)L#GmcP4OzCz5zYAxy%6O=V2}JVh22A0rz<Bca3L-f>xOV1-;<&6^iY=L
z;h=v2o^g?F=XC-EhX+&Rofz`6)>YO47l+Ed4fy>$tWr9Jg5{(3FLAx?yNg+i%tzMH
z+q6W}p7MBmwGhW}E1^B*b<91GMqP}<?c6WaXlJ#BtKMnz!&l{%F|qN5t8#!m06>I(
z|E|0esE9f2!lVy?M3?~SG_>04PQd$wY8Jdyhq!A(fdf^91bIMh5I5ga2wOTUWti3l
zfc!Qx$7rFRZ>0w_2VI@xZ%mt{as;S94&eK1Ih-&D7@&xmyJc~EosQ-NfP)-OB#E!0
zKO|9_o&YOl=9C=xdd%K|ri%U<(uE=Hb?}17xi9Dc%_f-Q!T-yGSm7|cSK31v4glOZ
zlsq|Kgg>coK`rCApc9K{FPznWLgOhqd7Pm2%}RELS!TQI3}GK$72C2aTR6X0sg9i!
zkoTXnD4(>36U?iRR}^}V;yz#=2SRB+uV8QI7=d{d+7JlYEt92%G^h*^Clkow&?(Zd
zQQ1d%3CkUdl+o0%1T^zSMj3SzHL^?vDe~>yb*8&fM4a%I8NrfrrhaR<6@_rSooa;d
z;iQf2mWCPIT!KP)g_P?3_tRN9My+*J!9Y;b)352&|FQf_hyl92<3ImB$(p|206)U%
zo*1BQb3bDnAHY5m6lbMOYAs;(E+;H0sDoz&8ZP(k?6CzgS%exr>40Ev*xA7wR;u9U
zZ3LM_4#hzG<&b{xWR9GUhcetL8h|F0QdvpL+Fr;alY8JxJO{%bYN)kCW&%A~6l8pH
zxespEIY8$fTLp4jWnx&tl%&|y7||b?ky(>!)M+G`hr_hP7Jz%?=L<a6y5)kzrT2m<
z-(8YA&hnac(Av7<R6et`5nVk$K9Kj^I565>kB`0}Bm5$J^sHp_b&jG~?M8u@his3?
zBM3KO7>Oo|6dJ9v(rgaK!WM(JvC3}aqNu0KCVxK4<yp;P3<pUSa-Qc1Ru)PCUE}v?
ze3z2AA8t4|Uur*Ar*rkq;*)wl#Hd|2X560uo2Fgr2(2K|acNmt10IrB-8OS|PB&f5
z(8xuZJ}ic$h~uJIr6??jGID#G?Pcd$s<-trz$g}jNAjfOKHE4Fc#p|L&bc&!Kj8;h
zN3x}1x(=ukvfQ33yQ#?nUoGNIWJ(godumdk^g9d+m~fas@i=;9KbRG#W8QcFyjS-|
zztIhO=;_s$A^>Nu`TZHlTKdl?{ST-AbHMj+&1~{klmGZXboGC!aeVJGLa{wzz}U>}
z)4}i%T!oGDH)8!}LJ^CdzjCy~yXKx2g0QA|?dd+sR|tM}pz0?)H;<aG7#IBS>#a}d
zQ}vgM6y0_8m+AO8v0R5ctu|ldf-bcBfGF=4tz{9)_Qmku!JyB~DG<+%z)*v%rtL9d
z21^9;VA`0*&MbhQ5ubg!CZ&nK7p0PPoI&LiD+}{gR*oF5k61&%iy*@Fd=`PS16cZ`
zR+g&Q_{6Mor|;Rtp~#^1n|#TnRVgnW>M4Tuq0VsEf~L)n8H2vz$7hS-rUFy>^#xz(
zhWO<Pu-LcD?RIO|G{nVGXrVoy8p^!{Y^_0N3r&p-pK@n_^2Hq>w{P6G9tMmM=>(=#
z<U%if0cPdp>N^okK~*g~-^m=fUiOEd^yoHz)O{XT2c41F&*Cva9&ZA3A3q&+r1lez
z66ST@-46dO<&+T?hA#~x@)v5om4i^&i7Gs_KhooFOE@c2;<lq8YWuH)_g`P>PhL^-
z(T%_IzYGcJ02A`XuSsKL`~ga+f40`m-KmVGl&F|}j)F6I`TSTc^T(irH-Q}W&!5;`
z4kQitbVABoKtGvreU6v$NrZVjw59?w<bu<deE3~~(+EZ6$x#AnGTZrbjOiL@ukm<A
zksE}AzM;H5C9d2t;W<s)Ba5xRf;zhA`_if)&obZ4=S1{RF|<Do>7$a~|6Mm9r($>5
z0s>zVZPm@=m7>QwRtDJTXa?I87LaG2zm(<Z&f+m+US6gp&c@*KW%a`JToLXh{qfv?
zbL4FpoQMHr*+X6uE{jB>h^dZhTU{J)u-OAKsKuHGhG+YWDN1Sa-p9ibFD~5B-tv{2
zBp_B{0w7k1V-{Ufn05>w^TgW!m8~E%{D}$6$M4}bFIuFrg?%TbGbviElG#jU-W(uX
z1IDt8in<lBLC2#zw*PuMb_5^|Z_7`1;1+7RcMh8I!tUD{tUK_G`;e`<sQV7koNT^<
zt*jhea{U8K{TDk}%0~N_*womh;Zm>v{~k9X3P6oD1n9FU;)6z6?ru&>d$!qj*Bsk>
zzezLgiH2H80^s|RKS_j4hYOyk+P5R8%JO_wBZ^tH8=8P0;o&p!K>n#^i1w;1&R13x
zf12Is84?nd1*IDdUP*pD-b3oKmh6lC*YyZ?`s}G+2<cY|&s^8(-Umr@`Y?_Jcp#t`
zLBlw$czOqRKp^+k{n(_bz^oDtvS{ft+H&p~lJRHauBJqUgXlhfylvE#Aw>RRS8dc&
zqIL4x8lr+d?WE+al>k7sJ}J&zgr&{o3@hwHOPGOKT+-QyihhE1hmjtw_8WK_z50TI
zSLJnmA;v+>qO%>o82AKU_Lr+a+cZtYQEE=cNILS44l|6b8I)v5HiUpY9cPjax&03T
zT~t~1yxVK4sIfK7@`am|>WO^if@cC>fW4Ir@(viCt$ZzCST`Jq-_8V=_!zVT2bW52
z(;N`YJmCV|I&u?{v@!gS0M<116^qdDWJld3*~p*D%~azv9-n}~_3`KlfMzkY|C{8n
z^oZu~ueVlzs@MK6U+*o7k5F_7M2sTlPokD;alPo<;ayh#J%7T89tWZDE)a<M;3*L2
z$9(?^W!fx;q~1O^oQ1W=*QTL~{KDmG=rxz?;ydbN<#4U_(UpKnrBoHxV)B*8;CD#G
zjYjll=cIoqsbkv^Et$araQjL6QY27Erw@vvr_sOgq>wmoTe?p}wd8m6;NC0RhyZ!!
zc0zw`Av3Be0?Q!1;={OPyDRaRc)SRFz|oy|u$=io)tvci$!(kg#JNH!@6U=P0MLHQ
zZhKW}K)Eprm@^ofQBHx>1*$5*jbUbrk0l;PVYb~SuQF0*0FiqWu;vZ?c&o7O7<3f>
zPSJX2cnPzbsbuYX6H3PM7HXfs>y&R3%=-n9cB;oQYZn6O-wj7^q>Py7&by|?i-S&3
z5#z3>v2iwXcMh&mWw|kl9{B>SdmBRR<lDmXf2XLk+P@Fb>YAH>Yu&#@J*3nAoj8Cq
zNBH}H5(g3+`#6_NVA*pIE?JK><KnC=R9PxQHr1<))%Bx&PFnqE4$UeocY<Q7^oo|W
zmGYNw3Jk6Mcg)xz-o`xG#Ftqd7Agz*N_hxbiY{jc3d1@RrP?aJR`CL@+9(_UN7Li<
zaw%4?RaZ5EhK-Oljc9tApcHARz4^H_c!c7+S9SkMSmXOUS)A$W9vans{(BM}m{3bC
z#Iv+kxq}`W0nV?ic=x*fS%8|FLp_m_3*amr8wF3&`C$VfsnaXa7nx<!^QG3T3xou>
z@;8iRDy<&bM@Z#xmvIOKId9-`&kv_Ao$Xl3Z+;Q}-DU(RqUU90_!|czo!_NDzQY~<
zr9By}KbEefEA%F8QZ<#hMb0&^-&YoJzG2AwdU;q|uZ!^n%6p!((SzPKtol+8s~ttk
z+O<)i<>-IGOIPbJ@*9OYqa`rBdh6ro8%oOhE1Gnj4j^M-syeRi6GfxVPq94Bu9o9k
zfzO&+$=6xE;AOGbeCuVFxsXi(yp4<!PvGo7@Spr&BK7Wy1iSxDre1B^IH`U7uMy+V
zK`SM?OT!yAI76>;xu}jEFZHGCdk=H7a#IVP!C`XW&;l1HsE<yp7;0vO)zAFe16LS1
z&|kzuc{YVKIjyQVwr7UTISWn54-YL{zIIR8ryX5dm-o9lhn&_gM;TE1WS7%*8N$yF
zM&}-&xU0#^rttavNU$*()cK9Y&SB2Z+GGmc_%eV7Hr^8uPzv5m8H6L3<AZJr<J<4#
zr+L;=cWy`ReKePKCNH0whX{-}Dj-3T*;G<Xd>J4m&H1O(^><<rvn7y-7Ut0wyd@-n
zMxK5?6n^=d1Ujya<I$bt`lC~h|G^}(_FK4%<>T?p2cDw5s|9We6$r(xUO66ildCFq
zwBMWz0qEvF8vILVnJ_htbMvyyNL;?)5Y&EBJ~Tp|j{wa#k4H|zN?Y^>Z`@hiTd}m$
zpo~p?U$wy+-|digs=FUsm`5p^n9uI)&FV;l_GIS^a1$nyL)Tv4U-U@{i>?Mi$i5Lq
z0%w@ek)hZv1gC20WloYlkn-*<E!8!WCA|Yc7vZE4BXOBKK!7o&%Woga<#@4R;`1fT
zp3jAfW3_)L7r)Z~w*!1Pvq%N~JyS-}K1JLmg#>;2`6VvtBS(R10j463!y`-=v=ooq
z*W^es1daKpe;}(Ks=HL_%1y3W^D!&^*IpMb`QwEaNX<QcaxUxtvDzY_DZ=F>gO#p?
z;+I}Wb8eb2URaU*B(60>JWiEm&-*)hGrTW5%BpYS<KJf6P1Vk`WD{aWHT`h@LqBc1
zcC0GXVS8u$cLEEtse*%>;{`@XYiqR!v~p=Q5!$8^KByQM(NFJuDV;>;5CAJHMeS#F
zUa67xKzF%!Y(@aHMW3j{X1+&pU-T>aMO&K{ZIJ|v{iCLQg0+`j09+O>paUKZ&7+aW
z={ReV!QT_+42{tRq`?r=r>|4AG%^tC%@YOOD<2DiKxr{fL_8Z*1IXNDjx>GzTr!H6
zM&?`6aeVH*dlW)M5*qAGO(-9*Nov3Pnip^Of<TsI)L~_x<SBCwKg%HHUxcYRwEFWe
zhexZE(rI8DY~~7bhQf@2;!HpLQQCF<>F<d^3Uhd}N669*Y_`TIRnRkKbv-_a(UJ)|
zYuBj`++$I(Ep}|%k5K8#pZ$t8cHb`Z->HIIOPT@tE3-ktOG^O%lYGeB?(<h<xB?nS
z=EFa3uK$JEGW{8Rg5RJU0B3ujO_&<r)Qg_Wo<T#LdT3jei;wXx(C-Ba<~JgrYz3lI
zi7A4Q)cw>kuT;xI$z2|4tL@L#q(Iuy6}oqrE9|Zx6ex`gbU-0&R*tm@`~8-BO=DvY
z_l+#P#srh}ZHbEc_Mz1^26H1rL$yDwoi!w(+ggy{jniT*H_~Zqf{9{rdH6*#NG6ff
z$Uik&t7>|;$0@FhrmoiZaZTg_$!AO19i%Nv&3@|xxqIFn$F)8|Chu`Hw%f8QBq7%>
z4dmiL#(3N&A>>gJH+E%Rsl4u8#JGbZKd;N<K1By7?FCiq@m7-Lj<a3}EcsOeIc$i~
ze~%LzOX^huzpGjUv2?E}0h6DXhu!fK>lqfiM~vptx8{5Jn3WP&9bH^{lFPK=n+3(2
zk%Z)E#2SF-S()($06HZd+OlExY~9c0!_KspknEtQhXf@h9%DIjvi5dg6)mb!`xS{i
zF%ke}_f(zZ1|l|E2iP`^C@R=JFOJti$HytYFu)xJyN~)n4aoG34v8?)aoc#V+2r9G
z<pp{4SIogS=CMfs|N4M`{j${I&u3g~xYZl~mkH=Tiji2_%iM_daT{205fJTzvo$g8
zhjg8&+)CMfk@IqRSjIoe6Bg=vj^)5l$|7*W+qHVdJs->ReanTLyvTk<YCxqsmMLj%
z2ew(OZT3nPY8aod(rC=&%&(-~+plAk44oV8gl*2gekqvYhNIbRUBsmOEkI7`<(ccT
znZMiT!Q8cZ<e;JRG8Pqj#Vd`HBp|mWC=vO{*$jg$WheL(qK-h%*W?y~Zaca11s8r>
z;Y?+#LbP`2owYX-IiE=V`9(Ud7{LemAcZGNW6!EgL*xwt1GBH+AF3MYzh@_N?__>J
zR7#zukFJz(%uE^UYZPb1dV4;AtW9ckuT~`f9yd50@4Ck1km3;6c$&1<M3-{UP1KO=
zV!aKNR>ED`lk*)xgOAtpK;G+sG~G2uw`nRGLsPVDJ;S6g>2UN_^BJiu(Il(?M+mMc
z4`876Y7bfs@Aw&O=cv6N#Pm?ZvNPZdak?5~P}A|S7|>20L-_-+0-fcT@cc{*2Op6Y
zh%p2X^rmuZ%%N*;bAPA9Evv;G+4%eh0^-vK;=~U}5&pR)+#|tBB8+H2QfwqBWH~2k
zstd9vf)2<WGVlyhWPoxg{+9M+X|1<xY;*G-`(Ny_xsb+#DOSNGELBXJRccl6J(hT-
ze5Gz)$MuiJyy35<RbGd81FGq(lMm|Z5jQ8z>)Ak7#`diRfy#e{J^xRI04@>Qu{vOP
z$BlVSHNf_NhrBL4A5XC~GZ!Tmczpa~0NWfRA2zK_EZq=eIZklSWfo5J4Q_Auyq-Fo
z?aC!*t0W2BTZhNHP@c3fQ21Vx+aTYQ&8Q(2<k~wI8=tb85^p>m6+c=iHJ)GPX;H^v
zu`qqfVKw<O&bul&vDPX$)El9*=RIu{cb?2E9!|B>`Tyv8>#!)>uI*b91Vu_jkX8_+
zMY=&sML<MCQb2l!8bWFiLAtw!6p$E(A%+H(?jE{f0O@AP??mtC{oea~{QmaZbi1}W
z&f{2XU;Fy)@iR|5$UqLD-!+Es6%_p7dQ}5!B)$x$5z?hyb2q(N-a9=Iz8)Za{heLp
z54=8al{4J#MvaItV9qZR@R;5Q({CA8|LOkd!w!UiRTnKU^KU-HSy#XR#ODTKk|1vq
z&+c|gF}SQ`2y#T5*MhA1t>KyxXPEqu)^e){QvPM|29^B>UeURgK+8EDPVRj&Y0`(x
zw@K}Vw_{Uzp}R)JbeX}w66#wzLPH%ggE5Q7kD8hlAp6M2aUO{#oHR?V+^;_MkLE2<
zzUS?mLzJ^O>{^5<^Gh9eAoNz=PWA6gPyfoh!Xj7L8oE%|EM<jD?lLD{Y06PXHRijA
z&g-K_@1l%)B~jk_YFU26h41<5y$k_UmwpWhu-Y5~+8#wsl(KQ5ga}$02q+kf{_dy!
zkBb1f<KK|toNxlC3NPax<{y>1?Dapt@~~$@EJANs!s%ifTp9y@K%=RhJ(PrUXu&1f
z3((yxp_@RB|IXgyoS>j>8%qRhSPu)bw}aj{w4lIaAIsN-Q1;<d<fRQ*m6@EXApR|J
zr!<)8{B6a|JWPI1*CF(QWikXC59w8h=y`t$%wpJwny1+Nso8z#h+%_@pDFr)6#bYM
zy7a-o3W0$0Es;0hLdYx>L?<<?eXNN8hm^fno5m}C?f5o9?R8>|a9?m|x`@XdL%J5N
z4w;6YP(ApbbJxR?z)*)}R>3bso@X&>$jaMYkS9f;ekA=mfXW_i3)3!9OB2C9DN=71
zc`!<5h(%_EKj9WV2|#DoYsD?%2P$#sq|dK>qtIog03(NAE~qkm{^s)}liVz|>bull
ziE0<X%KnnJ4zRMjIC6F~S$UiZDBnX?9$2TIHNBBs7mfium|DSJ^Ry=WQ5I0Rx+~JC
zzC=t1uxjmDj}*zfNZ-=s0FbVuvDD;xW`<tp<v`Ab2$UYKQ(gkBXq}!P&3D{&1`OGY
zM_XAA%J0dit|CB6oK1fLNa+9l^Zxi5{8<u60O|he9@?1r52S^>><2?Ki=Yzf7u=rQ
zl^r=@t}PnDFORtW=`?%#m}bOwl@Sl)E1ly##W9u$j&=VtmnAtu*y4e)`;)~0rm$ku
z&bSB7cZlz?ncz~R>~!YB;G44#8Umhe;~6uyI&KqH7AGy^1PXmlBOVEb(Dt!~9p5fn
zA?xmsPt_E^H4lj(m%t8Z4xVII^|KZ2DvpFo-7U`wH#l6(<#D(MX<{bka&tVk|G@E*
zwv)siYZ6Qc$-es~Mzib#jFUAhTFcOtN^xU#tw~#PD2z8#ThvvBT5Ok%o>xxPUY{g{
z`^%Xn*%B+g=k>zLq}<S+HTmomQA1E$@8RoxG9YUjd+vMsuwg~qb2V#OUc7k#ONw8I
zvU9tTp7@n#Q0I1qpf)LZd~4!a-2qznB}bz)Y86<{5de?#Q_d;WW6RHSC2gn04g3Ar
z1xz&@2Y~>Gj^7lw*Zx^p;WPaG#_(s#iOm1eSOM7;z*YLd4%;nw5Nf`GpBm(}J~pEf
zoM>rjttIlj>v9s(*El(8gc{R`k;xy<H}S7B#$D>vD(=V|=FE0e?(7(~0*7zRoqvB#
zet^&xX$Jo+YaBqgo&kqy1LfinYcuKmS}jjAJ1JW;&q_@HYqEY=M0Dx{4(T@ynl})g
z_KPVFHgvHZ8aoS()<fl8bQ5sdL1L8S`7_;0H*XTH0!@L-b(9Cz!Nxy=U9-+DzcY3z
zb&cL=Zy8ZsZY_k}8?WTO-(XOu6BoT#r}3TCofEp--ukJgtPC7hc*DiFb>3_sH+NYZ
zCh8Qj+?JP@XTv3?zm%#U7Zo*s3h3#3kgNXm84&j|18ZS-0lDC_;xTypNa9c8q#oVC
zbqnCQI~FWUM%QUz&+pG;AJ8sSOVw5b^o??R(S$OrVoQqQDkd;|)bCEeln#>1>pv;g
zl+v_Ejb-mN1Apq+PZc5%9*X{5Z7-AS)EwKqL7HBcNhO-t^p)IDa6b|Mo+Hp>`3mF=
zXSnKk%cZ$`%Q49GM!w)TQnCJ}iOnAT+AnHh>Tk<m&wW=EuDCb)Xi-r+cs(%i15J1C
z0ehUzI<SI>>au@luh@A-hW-4g?_Ggrw?YV&QQ2JrX++&&(kThwiI<O=O9iP7V?Vz&
zx(o29nQBZ)LTL#ujU2vZXOC4otxbBhS%Zi3H7=z3r^q_YRD}S3W)8?8tvLT*MV{T$
zN9nzp-qMh#22*w_|NG;YwkI9c!cCeRnVCbq83$fr6W=&kQG>$;=bGe-M`uS@G3<RJ
z%C6<zsaeOThO@|uNpRw-Wf<-PN+U_68{GSD7-g$mVYVUW-jK5)y0qKtY|mE#M((Ii
zuAL()=Wi>isvh;4R@lzi0Ls(h37LfA=zcTb`Z0=C_={3l-Tsb}fz*CehGtC?(?(@K
zn$AWdETr)`d1*%54DD<^oL&wT3dr}J=B}rP=T<SnX>;g`L9-sc`4`-=K;gTAuJ?qf
zhCJZ^?r&;lHUV>yT-!iYnI(8RCnr`lmd~92x=jiO_O#YbKofC^N@-DVX8@z02{4N<
z>y^5k+|tAG_A_sZ`0^y@T9p~FiEbJyEsK|wz1il7UrUg>r!;$mvFuTH-=jzMa^RIl
z%5(rWv@l6v3~Ap_*EiYWy+=diOSNwqYiR!<T4_yQ)$r`*DvX_87RUgBE=8uM&<4o_
z6CqcvIc<E^n-5CVUSl$m+iLX9tvPbfX9t4rzvkph`fB;w<D@@Yo>IT|hYXLd%3X<A
z{mR*XaUdxy6geWFZ5q<o8postEW*zdp>`j0yP@yGv?Rx#<si^bkfkUXnpEH4!9MjE
zk~+YvfH*6nX}yji&=r<&Ne9g}d6_Mzy4w_SC62yb)zEk}{68!JU~c4a>AHK_n!#3W
z2{6(9F$W~(bpMia4)OoqNvAi?BmPm{Jy9Z92k@x>^|njUblX#W$Vcac-$jj;KtsV3
zCGA!twx9+2n$||Xd5oB6GKPJic5lej`F8*ZZD3APfhga^sp&Ux&r+|Tsz}j%S5rTU
zTLVCjrEW^RLc(Zj0%Wac&1(-k3dd_Zbc8HS#%zcXo`?A(hUlXby_&CYi10!6O1dU&
z6dV&JM4g2?vA+dPdp_EFb#hUjI8Rk8A1+2OfEU~9i;iW2SbN{Vasc)f3hJ38vDbxb
z>(VVCJ0<F@6U$DauL0W?^gQv<;?GEOS|9Bow|yG)d!(@4`C<)j-kW-LxltlutliR?
z8O6VVu_zqiX&eFu={Nx#q92@2`0gKm%FZTAom~?q5kK-Lk#46GzTRaoxA6^PWvj|D
zed8j)xI>edKUaL0DSo%nJpj|H2*jO}lE{Rsz;`=|X~q<E*F{vZkqm~b<XDC-GIe}p
zrYi-Fmz*<FZ9Nw>q_7+`$-12ywSvdOO?V?_rCG*b|6L|)tH+Q*`aIJc@^cD8iHi;y
zn#T>Kl#vZRFeLigY_)}&7n)3hQfNPu>P~{<_F(l`RzPi`EM>IF5=7s&Gr{Qtlh@LS
z+r4P*7x`jI3{NO5I;wV!&C@(lR|Zg?RE7vM11Q$iP=KXK99Qh`fthV4$^E-@{{JZ{
z4$1#>U{-N_%l>iSB+6VB6T(LST}x2WI%|s{%(v?%-sX)&DCyO7O49F@rSbeJCVm7y
zNs8go`Wb0rtnuN*NwWKhwL5hY<v76PbVORCht*ril%EFUUH9LB^@uEg^fIum-HBvf
zEY_f)uwa(;O97d*Lf8G(Jez|k&P$g7ajOcP#(qrIZ~fS7{2A&iIfaF0r4LuiwRZqj
z*$#&>Jb;Js$J|YVd#lAVtqGD`Dc&myu9)rR7xzH&ni;B4EQeWU1Nvc|Lm#^omwkJn
ze&!2BZsPT&aE7%$9gEmOr-fy2gUU9Rq_oNahh~MEa{+G6WU4~#9w{nvpRd;aWdrt^
zqP!!aM|j_K#@hYv_j`cKXc#rD$=a;0-}(~R!$BRdeeG3ehLhz~QkMKZSx*>weFq|8
z5wLb@B6qTR)IcpIGOS+nfs9#aWukf`gjn^-{vk38gAat?O1J@h5@h{n*pBJ1J0B-<
z0HLqd$=^!x>XE>_@x=4pBmS49b4q{R^D4T>Gu*T7e*qAI)fg_^%x`y20v!AAI|LK&
z3s|E$(Sdg=&&-lfT1qjW9vg@ru5abHca@cu7wAo1K4`l8f;Cyc`N20Gt_oNb$J;mc
z9(V;QI^A#cN6HFA;ON6q`swfv0kzAZSaneRgfko1v#Y^*W8B928>nF_lH|BC+)5+=
zy;W>}QIHafsm;T7HBtBM^|=mFEq7}qa5|md6}@W?Cqbx<JaYk60LvqlTq$v5zlI5G
zC5BC&WL|s8B2XTrfk^%BU<yns6pz{Ytw(Hjm&@%<IH3S>n3HwcUC}R4LQ)F5Xi}@M
zGcKt0{$|+AJMOE4gOEbt#(^g13=O?Abn-LrVq;^gpMmQNNeQ`ZPL?R7tKlVay1d=K
zMye9#gFLZ%81sYd^BSYO&gZ*hO-|>uf-9Cg%fr6#v%ZHVvX_k<+^K1<OF7M&diZ05
z?BT-1CBQ2oSQgSQ8!We69RHMu#u4cD{7*)eTE`QDQ*(Ds7&a95YVx82f3PRIbSdGI
zHh;(n0${&$v;o}r1@}!mX$3>FcyOEXnYmMp2NZrfG**)O++ffd>(_!h<ZV??hq>-A
z#*>&h*au9`SMS*KlX&7X+!8g;h=#W;=7w9O+Ve}9Ecpv((UMx|RB@%Ga;@nK!|Z<D
zf~tyE81S6QJP@);;nJ&olP_C6eAN?*d^5Z`xA@Os9Z7lVG$6FjEbSM4)IZt=Q&J!*
zR-5|DPC()6cVFNB7Jnn6p%IKMYb-&TAJdBIdo7*4M|<{R<6;wy#7dum=R)!hyJL2s
z&_ym6T{E{Nrk@|Ly8X#RWexBt+R++aUwULcUZl=_38EB&Ox&lrY^s~S4B{v%-rsiw
zV4q1d9hUnm{RUE*%LK{wpZw@%4sjLj+L7^+(Dt>l@jPb%#c}&AxY(o{?F0WEq0R<m
zSK$UaU&dN7{k?H%T<`8-!^!XyN~uP;ZEkl;Ra2w|`zbtOyux-fkk{wtft^^?$N#s+
zC=f60Ty;La&Hq8z>*6-BG*lXn1nZh{d%k+*napP%Mno&zCwX}WxvYI~3GDN`zU>R6
z6BGCTQ)Wa;mrla?DJ)10-aYmG9Ct-sB|gS4TWO~Iyzs+2%t!JO%?{o@5*u&FCh6kP
zAW&bU(vL8ovkrL<n=g(ku^jg$7a#fgj?RldX~0+@<3n4TmKhs*A~h#(VVGB(iuTEt
zvQ$+<qnBE9uAr;y5dfn#n*!@ozhAi`mk|DiofMapRl$(b&y^Tv$DaeTM5<jh9ouve
z3c6kI&=hrGzX|8Q6Q&c6{!FYkZBcMs@>uoVKF&vjWdb%o+JN;eY&pHhJ<18I&2%8B
zt3(fQDV`pp8@#;GG?a%Ur@-#;{DN#<Rx*&v1o_@@_ZRjbCI&z+@c-L&);EfO{6{;~
z!2+0Ci$(vKT6O-r!Vu#dbkM^~$S=2|iT#KwG&Gq&fl9^?XeKIKH+QV5?Dt^x$`A_6
zY1iM9VopsIVRGu-8nDET^JfEs%e&_8-)Xh%&E{9|>kKDQU<vyiKtPF$jWrd|l4ty9
zeQ}J<N~b1XzR(}1R?5>Jivw$$4d=hBV1ux@pX%CV^aTM`dGdNYYwJW?q$k{J_?~J|
zhNyClb2sSxgx?*Lcx!qy$=a{lX}yeHgS|RmU<q(IVEy!4!{FjOH?zJ&yOTi64byIi
z>z1AIJT5!qRj&jhY@M;U`8l+T%BTgbMuGh+T{HndWsUm#-(lcQX*qgt@*fW+Z$JP9
zXRE;{r0WTFBgtnECNhMyWEVNZ!~4*cgz)iSQg5I-J9jM}EacX&wd^fWh9VM12S>!~
z7d4=mj~rqcVQm`JvE7sVq96dl+2Yc8T)i!`m{sp^+#YqtxSC4ZY$PN45XQe#{s!ZC
z7bhsrwe4}&e;Az2!-N@8gR5U#Jou${kgZy7BL^S&a?eUKmOi(AA&oy<8LBQiVv@={
z(_wIp2|gx(V;+#JiY^U_n;1V@d}E*!h*qWtiRg#|f`)0V^dR7EqIvc);4)l=^}4;n
zrHGQt_lmz|#tFy66tDk1J|ra_bs<<O1+os;#kyp3Jj10;U7{Ogx^GI_k00_Y&7{_E
zQL6n4;aaOG%W+C#IWVZQ%;wUs;9s4*c#8sDt?Y3K3$e+H7|Npel?`!ZyPGn&mA9sb
zz@=|Ciyh*Y>NV;G!E5v(Pz(S3_vJyu`ANTJv`a;%&ZoCuJo+Ce$hzv4lauI=Rjv$|
zdj7e?D&(hzpi>uLk<RP&^HrnGEiN}-+B!eI`EmWjorK)FqGE>KnOO3r_xIK{>h<tk
zijc2F&qdQPr_pAr4!WYN_Lc5^`b8c^_CLw?dY=C<a?geIh>WF;(Aat`?wA(j>(`_c
zG<|G~hz75ZhMzO$Beap3R~6co(u4BMpEX(0Y17(>gFnzLb<-d^(w#_`xsku5H+#Rh
zi*9Ul7O%XKPVd8yVLq?xpTlkx;T(4HVK%7hOS0rye=0y$C=^{2$d^A??js*U`HSWw
z(kLU)xo4vTtn`q(fNML~6%+$JWvYu_A95?3BDidW$|Q{+m8jDUllvplA)zNly3f?U
zM7g%s;^H8MwW<8&+;miMB305~DJqFK|1hA$scXUU5$@t3Hj%?x?x^2c&4MOnnHo?I
zIGt--%c~l|WqD)LG?98jn+0d1<Wl|NuD)@u1}QqMT)aE|)n&vN_!e`3@>0ZUGuH%l
zT!y)%Y5;4QTzo8`u_m<gNh8yF#`Nzit(x{X!q@KNAEl@Chna_en&DFOr^_vNbvHxd
z&ai4txEqm@{mkvDxekQwRFs)zve0<amgah*+{0H$QmQe?H<<nF!e_hw5UDa&sZW{O
z7pH^A=I01x5xAN~{}&};L$UvKpl5+_-cLzQdV_Pjn$!7)lfuNY#KS#B#8@i_!Rp~n
zoQ-63#f%~8>G}Hx(`5al?KO9%?uQTdHq-8)a-6KOkcQ=Q63|IztUojH+_%54?fd79
zuR8i5uU9j{mbdtUE2ngbT|@sM2Bi2JJ3YWx@rlYx3l7#`&lVoA%5i^jCM8ziy<=1G
z%@oD!DND<OX}C-)sEvvSC!@fJC)eX=K<YD}&>BI3iGNh1+`p<(N&DrU%-d-if;G1V
zUzF?ExaT-5(dK1;(NKY#u?<`6XtrotyMsu!6d$L2_iFB7Lv@mUtFRqgN!!|-eDC;0
z1tfpEY5st@kJH6GZIyV3+CpSjfN}nMx~v>CS+2BmSGyMc6P}vXNx)fvDyPRR08H$y
zbx2oi)rRgzu4va=wIisGIkfnRd8zmi2OVi5xDtDXH+JtFik7Lg9Ix6<RRR($z(J!O
zjGJZkKJ<M;Ot>sZ2E*mLizh7#BiIvt(ENv(i2116>A|0~&Y1q9>}Jd8Gc=(bl8Px1
zwRkV_Yt{F<-Qce0cN|COvg&nmgUCPll0aywO|ryAd0@f&i$=s!32bmZyrkt09lRu<
z?xW!~_T@=jLa&$tK?Z=uAAyb18Tcyb_u1wO0Q^b*|MGzSUn7wp_0Rv5LphECkcFxR
z-d5c`P4JY^BPSYG+parD-2)HOyg)R1?++QjGHuVsl3=Y^I0TD|rRw+bxBBYaBc1og
zgibFw(&|AA<{>=%Z(^kROndIRTMa~iJ&B?4={(i9nVL9yQJn%5p?SJ>MN>{YQ&2eI
zQQD89$X_3=&47)S`&67dPQN|<)D!jG^AIHiT#pjIS-43J^`iY|oCf7bQD58s^Ig1Q
z#+8fu`N{UKg~C3q(I91S@xnyvECIL3;Xo@7_JC+ju<L%I!3S0br=X3to|1B1G!T4p
z*LpA=UOHh)F=bAK3Ac0&G5u&dx|Y@&0RL0ijr}d`g#SN<9bKN+-@<OxefdB7QQU?D
zXCfdU;cNU`K6<P^g9-_`W77GpgxwT~#*1vUtP&gjb*txk?akCZ*}G&U*I>fe$_VZ}
z#%SESStm@kAJ|Mb%tzuP?RN5FC{<*hz2VH#bgO#DbzK`_{39?Ofjk){O{Q9jun@<T
zaqpID?r`)mjdmI?zfuC>ig}PWzvW&U(Lp|pGv?z)$*HD~_DBgAGjhpssqbg)IN>A>
z{zA?%oANMR&!UVx!v)==@(9S)?wZThE=z(;>3Lp%75x1%Jug3kYUSXOOlz%iCpxc|
z9_ZRdd%x2O>hPd;)u7#qCJ|E2blv*ELF;^rN8Mpg#2`8OMD_7YC8g4M1Gn}qIIF&e
z3sd@4-Bd~w-ADMZPVKsP-Cwm{dDTwkCEV&9Km3V91PN<cX{fiHI->zde}4kvwG5U_
z6dI=Yyi}WEaN2#c=I$spb9p-2Ft$!nN*L0(=>0mKZMptIiVhp9nQ-~p@$pY*o#nw*
z9mc*-RK)oVJ~d8dgeoFEX@g~7Io53vs^eSBlmz_lTXzBQ^>m}4+}|ehvHVpl@0Th(
zqzN`Kh0~NFL0Vri<-G<6N|MXI4-Z=?P1Y52FWLlcl}cliHMPsxhvD|H7R{gFtTF*e
za#}05xmH-6z`634(0p7Cs1*fAxvh$K_H>oeQkGi5#}ytYa&1CdQC!J|&gU(Hn@D%#
zL!p_~34X0e3@mrER9#VT?gjs_-f-gBG!GLP!tI5M?P%{omCw9J0_N)R-KEYtulHwa
zEkNxg8WLCKbDMwrnxpjVxkM@a6nuMpxfB`y7r-cH^7<JwH3zZV7*HJ{nL9|dWS~uQ
zB&^pN0?{vmQ*SiL$si6C&a>5n8AJjz5>{gH>(D_ef>sp%NI6JImS=bT_fVE6YM&WW
z?y3?9Tq8^KA)kMNJDoT0cA_>*p)SoE{zL_I@03kfVV`V025`UoB5+oJ$@jvXcw77=
zGX>q~k%?1~pp2A|)xr7P%A&Q_MO8gY)aaZdl=pj#lvOx<9g90PsgdqlU-&LP19$X?
z_YFAUd9bnMZFjtUFPl16@>uiB>yy4s1?j6k-%N-lgYsWm9uAH><=;MkV2<{2-^XfO
zb8Z9SkpV$)8GKV0M!}&`lGCm8{xqz$C3<mjz3i5WVF_%L#W7-$;Et4KC6TaAdYZ?F
zvm2W<)q@_Fm!Z{zfHbz%K<2UJRb)5QG|W}MXy5m?98;;u`}P;>UhepSlhU3L_X3}J
znq3%df%qyM^SGJ)=WYi8URNo9N3$3p(U!R-INAz{kD2ZIqVx>+#98uJ5*nf_TqSmi
zP(E)#nQpu=yYKj)ORJcK3=pfSRq{NlZf{C#99?*V8+vlF7pHtPN9LWSXb1mTmkF_-
z=0D;j!^8I||65}6`#Y|xwcQk0yldEwS>RQ5*Ef<{Z0Pia$^f=j^Q%x~I<bz;nZiFK
zhMx+7RU4ZD`dw4B*CpHR)aI_)F^0_DUXfLC<fPH5qnah`Tb_MBsIWlh|0&=F-~NtC
z7*k@MLpxx_3$JS8Ma~IY=ohWwiP_T?_v=^$Mq7l3(cWz-k<S%5Td|fX8LTemDu43c
z@&+4m@R0n#OUt=_k#)R#ZY?PQRLv=~DH^fn=E)DlvHrr3x;@(-ULU|${duup9x^4|
zw%<>#xb_xArrXESa0OL+jfqhbT`~`&YBE#7?kq&KDPT?g4V;hFy>k+o`a-xl{wx|L
z7X)PO6$9+E2C(}3_l6!uriFicH1!tl*V_NX=lHL;O*nw8&-^MabxOV{D#~g~Rk9n?
zlVoIFuN%p)oyF1(*F`{gs(7RlK^*ZsVd1i@LO5QuSVe2MPke~lleq#;G1dvsHPjc0
z{lQBrl6#uA2H=YDdajDWhfe{-iqGkzczIyz82U-L=KqW?78O3&l36?v-v)eX@vHD~
z2PXU3unsD3vl!VZk^n%`=JUesx#ll?ZT`+v)~H~N<=l-AN(}X%LED41#!vkNls*Hz
zER4E`aVMqN{&9x^<aoe8m+{dZllA_7g%Jfk+++*!CE4(&iXacN)&A9hmbctkqpXCc
zzLTZK%BJU01Gj_k5;VOvB-HW<waj|t4lCxqW}h7`VX`ieTW|b9(Qx*A=n0#Ys4uUl
z_nqs0l-JfG!d&S_IT}tn0z=VFw7jJICBsxO-+H^k#gqQCW8U73y`LyRR*j40N}-Kx
zRS{XbSMe=j+aCk3m@a7!rc|OB_aN=NQd-*a)6@T{&RzGG2>aVB)B6ir{usn3uE-y@
zI<Au6TORFtkoLG-Y$(>hYQm7j@E`!`+1~=E*@!c@IgD_$=^rbi`Bb;nuy-JKIN!>{
z{Pp%_5JhEqyq3dc#V5i79rbbtje>XHx)4mqqOE{ON0sdseg6u<+R&K=Lbuqh9`SKc
z%Bz6KHuWQQPp%~_2c2U?W363~0Uk$<X@Yf3XNpqRAk2F&hhlxD)vJt6RV5#3VtTqf
zdZo%8@?|%lUT$CtVebR;7EA(OZ%|j1GU=P9f!|MnPNG&(4?TT`;RYj{)yFor@rf`?
zDX;ycO4SnLswCR!p@ml(8A~*A6JNG46Mk@-sKhN7{dlx8Xso(>vw8<_zV<Csc@SNH
zWvYN=t>3V-hVB4g$>f?)qyCH&3QE}fdiRypQjli${N{+&<w6+$3;wnWj*;<m6RD1c
zvMG(Cjt+Eb<mBL_FpE%0r)QRDu<ngt<6+i=Lv#9r&g)%B3Yvisu1p7KdqOe9yTUiq
z`moyFx|)&XwD7_Ljl6ZJS0xx6mzub=(J3-Oqbox<n2*f1oiW<egqJoK2;Ez)@Gp0?
z82=$Exa?w9qik6AEX@4t?xyYmx3d~L`+y=h`y`CHVXX+eyLnWT)A*o$ylN1&72v6E
zxL3B+Nrxex8V~PD*ytM5U>46baI$`PP>Hg#t8Dch8Q*nE-fhZzoA+hnHus^|_~ANi
z^B~#<++W!bwOV?$DWLPyBqc7=Qc=MA!`kGUtE`*C5a<4%e&UCW&yQ)qa|xb61Ktl4
za+cS^7RBrZtZ#wm@m>1+IbxLEm#&{Q*Dpxdh^~tQAK3MPlwaDmLqnM+sTGzh`_7^D
zEoDK=)sn1SsKt5ZB!|kPqv$$N8Oq`{h|KtdfB0^V&DrF3V9+{+Smfo;T3ni?cWLD>
z{OJw{EMY51qps@ADB-Xkit+H&4;d|3swaLb0fav??@|^<{|b|3r9Pls+127Sr$3TW
z<m7&K{#}X}yXY<De@$JbSqs|ikW(ccFGZ<N>95_Sn0vM#%Rq-ZS(F7j(jgrSkpOup
zvG09`af^%Z4f6}VVl$eiMue;v3k**`e7?LNp&K3F9?j~mklm8e&2asX8{&-jHs8Nm
zL`KG9^S_tgSZO7O5`M~NuI;0se8wMZC3(JGclMfu&2vOhf5ZHokVW?%ZkvZ9KzTS2
zTVSC~^~g97J1ILB9>>~k`X4UhtIBD(p7j3~Its6u85gp#&8;dqc~>9U(GkkAQN-&~
zjla}>i=qUsU7vaWZ7UnS5n%In$?D<0NA;|_yW4;|3ZnIs1f}#m{iS%?gRt$WCv-`U
z#mPN^WU;sNHzAtw*PU&SAmmfv&O@%l-ioc#nbcD;DJ?2*ObWUG=xmYC-iM$)Oh~u$
zQ9{k6?1_`>?z*)+JfAJKZ(w-^rJ4O!{&ZxNWFy)(9;2fWG-x<|$t;%V8T@V8$NuDy
z^Yk>~rJ&dFiC%q0%EU+*nBr9G;fUOIpNjeflV0T@uw|S@i`(US>F9y?lf7uK5ER3a
zSij2F0C(>=<FB)Y(Du}byP-GKmQazoiW)nTa_MXIPzgyDbk$y~PoZnwI5Y^dIZHrk
zE^7Q{eyca=!1A!QpL}vl^+I<{o?}-I0-lbu^m1L*KV8c<cNh2=+j*Io@`mz&!{#!P
zhG+L1<=LhO*7MYC)dlM(y!=f?aCea#3f;iKog1@Ec1JHZN+5-5Fsi_3_PePgl5G&K
zyI_95J5C$3(eM%`6!WW+HPP8)VrR9qg}T+0CF^xnp8DjKxUFOFf?lU@oI$h6Qx{zm
zhl70X+I$ro_eN~jmrKF<69a@e>VFXvcEyav6$Jc1Oy1!HeRs0Z)(7wWu2jVbz+>wm
z&E?`xz<<`iDAI6DR(2x$%24m-G-Dv&Y_&mCV1yJZNRh$o76W^e#19tHUTth3MK{UO
zHC?QAPhKsFt>pIH8|cWv7h)Mixmz=dD2#y@Y3iv{ip~|V_Uvg2#xz&;)uLN#(f2m2
zFZ`c2TN9MA%Gq0#GwS?6XX4dLzq(0$u|m=4<atMDp1I{3CpWH}+*-QoouzbBVzH%G
zEl*59_zF&XFuiD+#htci9f;$mH{zIf8R(g?BYj6Wu{(@Uaz9o~RFxK{t}$W@AX$6Y
zd%2>h1wj^9$+ovCgkcG<0cNA*$qEyZtS$l0ADQm!HHw^nO$Y$CY50E>msUour*AHQ
zGI2hR)fF*=n5$H{Lh8BuF5V=r59T5C9XsS7OHMsriyKWdsAAgmYR{7@nN~aq3(tbF
zL2^dbqf<ljclWb=qrY0;*O~AG?~T4dIxO5iJr8aed8#^*59O%YSx@H7e!G6%k=`d@
zJ}1|<th<O^S*8T|nOafe=pwpp!ZcASoNm)|?h5)1_r)6VHmQiO5of^VcU0xH0tZ%T
zqdgX${%kBboj&pJ){`uiuBqc~?;a(es3<Pge@w;hF84T6TJ|8ihKFvwC)aF<)ADkR
zq`s`8Atl+mh#5CYh|Y2j+wS`E4Xz4vpZMud{+jiW#U`%xRu9oZgOhEJYJ2a##_Fh-
zz#N3sZ(JZRo-<~I%dlb0#&fs(+n!eH*p!M>Q}&rJJ9|V@M4fzkm8&+x+I$;(8_<Mp
z*aA_6r|r;5cPy#(o0{r0xASOQVC1-aad{EW;iq%1GaLm}0ERFNzWaz*qLdm266v|H
zNu<-e*s{x(+|u=C67ypbgR~<(O^*_?VTaVJ$YlUl(XMoHk#fEiC6g+>5W9&V3z2kY
zrrw1sPp~`vjs5&_J^1R;NP(WLUbzL|ySbfQ5AOCZ*#=7Fs=hhYFS@hHA&3`1UKAnK
z`GjkCeWg((eJSIV*yY7mb*dq0c`L~r7v~qAFC>uBN547lyAj)y$1vzNGfK(PW83aa
z?a*dIyB3r<=zScXNVz!%W_~(;KArJc%5#@#Ws$>OXJ52URw`+RNQniP9bSCA@~GU-
zSV=X^!jQEvy1(5>QR+53Yd+yOs*=_)5q;*R7}cRvLI8(~Kk+@yK~DGVL8q$8l8gGz
zS$~S2Hi&dAijbzoirguM1R~0_%ZribSG}`Y%|DT*D-3ue2Ngo>gg(D@o*~dK{@@MJ
zBErgVvOW5H!sT%}6aF#cx_UwK{!=3h#C*N^T`xNb=G22{UqdqE)hTE*dPyNTPdFY9
zLlfytJOHxP5VOaEt*r$b9>{G%#Inq$J-p^ba)Pj1Pms@sq<yK*b~-eiX4qyCMP<dn
zkT$;a&f1`&ih(j-I*T!S`85g=*JY=h_zwAS%1_59iHLuRfWLt+zn}T`gyfXVs%PBq
z7feyB!MS7m&4b{trRFrbhc+DF)Wr^WCNyGAafsy7CwkA}DvjuGaO5R)$FL&1M?nl_
zN-0*_nYbe&iY2osVsP}K+d%z{@~GSASV#hmquSf-4M<$VckoO4!(d|Tl;QfiiyQEJ
zrQf(F=2E8M(X6b@1?xW#1>QaM-hU{9)Q8kuq)bzf1EG%-HGCK>+%I4x6H2${(nVJP
zNa-1lvXT(b0G>>Bx$PXsSf3#VHUD7Y#jU_+6B?vraqXoswRa!)J?)g0W^ee=i&j+9
z?@G#Jb9U@hKv&Pr0Vx#khE}IDvTI!FYT#2j{d4w@lz^{EXRZlBdCZl?Ser7krfq$E
zqu&>62v}OR_#(NutWO7<Z_Kl!2;G(z5F0hhy6XvXp<wOso}Ejkr{lV~Nh(~_W3~z@
zm=PI(wmfs{F<`|3I&i!kZd_QaJDK5Wt>_Kn(D(l>No2iMB+d7Gs3;7#i3ykEt~X|%
zdt-w&7q+s2MYd>EyMliUT?o81P0pAK57W9dqIZZgK&W0i#2}X_9L79(DreME6)8C1
zk(_K`%|B&1Sx&5<C>k7TINyYg@){(vQhTrMVU_`)z|Th&6tv+_zx-{*Zjg8bcY(-u
zgs`2K9#E=zT+^Eid+oAan<f=e7SP60yZSuW6;nKTJ)H5dg%d-5>O;RE)TV{pmQf5&
zlXDJ9l-+(92WVax(lItE=lvxn3Dl$NS#o^_$T1r}mtc^5u!QKY;&PUAf6Ba4+jLbk
zZ#LpAaJf+r>DwHm8ITceY*hUui0jA4=)xsA$G3rNkrHm+_wf8&$K>`1<#r2HL%ze&
z3gRF~nUDT(U&$Sj7zP;e$N}$uW+Z$U_~3mtAx#T(rRas~oyy%T(@@&cy<4@|i^*T@
zODJ^}k!CI?7J!irtGhT^Ls`cg;3VAw%MkqWoZs_(oACdmfk_1Z?T1q31Y8S?SYgHh
z6=5e@tfU0=Ym_n+?*3&_em1gz{=oMxYPV`NkETwh)r@W<-pAzpsNS_#!E4{cpw)w)
z$vrHD+!2UgGor~|K`D|FUbv1)Ei@&#)I(bh&V;uPc^j*VO1<^*u^Ky1F&FkgP9v=m
zvzk>7x~!n5a-2S8+g7QU^H{h2v;e2u)^ExTOx<G80D?Q7u1rkc*qu7Ms4n|$EG4xz
zew_E1sZTnl8hpHApR73J-XxiL`O@Qc?%S`goJX2nbVFmP+S4mXM74?^-TBTqm?g_W
zvrcdwq!j%m0?7ZZ{M7;#?2Id%iV@yu=Zmn2;JMFP;okbX$HXTyl1cfj_uOauO@v$p
zXA)-jV&mW?UgC115_8{&?bTY-!ym~ERco=Qm&u39YwlCOwgeXg$u&M4oe~=3Rz#6n
zjJB+H;&IGBtg1TSE@y&U#lA97)Sz#ZSHm`bHE-b<kxs2zMGA3FXoyBK+wheGuj!)u
z{0ZT_?&HU)!RSVyC7bgt9ItG)fu1!au@~2!viiU+r$*O`if{8rRMTHz2@15mTX8TI
zce5V7&6F#^NLv7Z;i%4qNBieb^4VuSOuU-=wK-inA4x)!e=9Tczr|ZFETY>zufu{a
zp3&^JzYbS4o_zYZFE)MP4Tuq)TDcb1945F)YBs}4Q;|!=(;@ahZUoQ|1oHhzrpJ(B
zKAnEMpLrXya<9>;?t{;FjXU#XG<A+@hO5o`qvtu+F5I950BnRnCylzOSD{<A>&pV~
z>f`f@@I-olEP!Z0_g6@lv<y*Ji9E<BF9UA~D{{&4eqzmyM$hAsLU6Qw$uPwxoYw;L
z2xqII^o9nsL+0<q&x%b8A!>TLK|ZWvn3y!-6_M>_m>AEUdAkS_Bl@55XQONlz!Y|E
z-q`<1ZjVo#NZm2;4$f;_7^&C;b2T~ZR^n5uO$(INEq?9>z<JgWY{8r8qCm)aL+Dt^
zWN12Lv+N&rFXvMw-rE0>Oa4o)c_Q-HdiA)q_(^JL5`TqF-AS`@ZtYPVc$_M~x9NJI
zqYm2EvO(g=#o$I2P88m=--~MKqZKN@quL#<<Ql7@5y~Y+O9FoDio)_4@uM|y%bqP)
zZy2;|)j(A~5x-?7tyMvektL@m*($v5(t<vDp8XKt+b`(pk>Q23N8Y8|Y1+#`X;Q=A
znuxWHC^8GE%k);Ed|ezgwS(`{(LU;>KMUP@4#ehAoixbi)BMRtv+MzZ?;2C&^lqZN
zAq2PjKP#ba_9wVJpSf5L1TmG=ffrZn^9dHd5b&4q6iyA|CUs9d+epGUWa9*T?`1#i
zlcqXZ0t)`7hRUs-RF9u>k8Ixka9yw7TaY}e=N+q0Xb_jw^ydmJb3#uu_sLbfPAUhL
zpD$$}mHB>BI(U%Z$EezH`871{0_;(gt^y<kFsVqQ{Zj4~Ov}kX%%5Z~*nJ+GrbsIT
z=#8T$QA+yd4^TDLvoQ-HBb2*g1yTEp?rE*@hgne8Pf>j?$piX$iDUMjF>Q`BUb&;@
zVIi?kB2YJBpJEO+hXhrEg2vxeVc<N=qxdtNPYEpl13i4N2&~R^oKUcPcmjitKjzcV
zy4QqcyzXFSl@0N0uYqRq0%RFCzj#oYurlbhG(Ao=PG==PY?}WZ{u%a(umeH6OX2VB
zis9d?HV9DiSS_b#NnD@>mW8;gkXq1@P=p4r#S(|ue5T^IxkNyZlJy%X2KWymzw5tN
z|C07f2WO-**jP^W1*<{<`44Z&OjnPk-dk(mK4yvRUkhdM1jVZI3bf#hja$6NiaL4F
z(_r}!l7a!o4oD}Np?J6s`m04ZQf(9ROJT2!cDFHAJ82|AQ)G1y$C&F;Ux-1e8PUl(
z3$T@U;C}S={PdOctv??4SAK@u|L}ZVSqbNv{)}kVJ=TMMtHIrr*Idw~6;m(P(fygO
zrC^L!4TIi&fu0Owp&<?MgMHcIIDA%d3q$T8@ZA0~;pKdV_<Tl7>}6G^_OSaa+4RKi
zVO)31-@&f4Tb}Q1YZiwXT+abjdLvBhf+t>iBfoqjmEoMLsAhA=a1>Pt-T|@>PJN`w
zr|Q|3PiaqAWwluJFVYwACxC6gld6xnNnVCK>FzIjU2wWtdXg+4TAM5~RnMOq5TDMW
z)`!m#VQiDV9Q4(-+9P6{398FHQq&e8Sj>~viPA~gi?4z<<9b<!_o~)IY;1yaoTu+S
zYoz0;HQCdOO`)yG1&W(g?5EAB%#>I4g236XJ!qzNw;3oCN1j%L;oMRS%m~9;v#hJm
z)IJ#%`g{CR-!YCswz6_@tG9VMlMY2zU3_G3P%d<fAJlkoIca@!j=D5dy)21jLhbx~
z_aQX(GL8F`)t5q|X6Ltu7eIY))O-tK`&u{|Va8L#1!s1P_~EQi;7b+2)i#BA?Ksg3
z^vcPC#WVVLej{-0G)zU0vGh`Q4ODz|?Y-2e;Do(nC(5RZ1ojS;>LS&88$V1}Advi%
z$A=@I1Xb0uU$*uv`hS}U>GyJ1hkRaxZ93_Xl&gF0;Ib|AS!!rrWIKc*W68%jX7URW
zj=A@ZpIQJwX!Ivfz=<)!wN5@lg*PRzBEytBDNWKIsbMqHEx;m~c@`qF-K+@h*m?(*
zq$a0<`$HJ-i^9T<+5B31m{V`K@9TtvvarLgh|$c?&CVghq(1Ub;!XyvIZ68namQAe
z``3JN>p!c}dN{UeQno6)dU<3*sZDav+{k8`baif_NVSUN`&t#S*{yNqS?0>WtwTse
z;n9_V2UCS2T-gwtW&M`cWAs<xZT{X3TO6WbqPGsXe~)#T(e|BxPwnRa_QC4nlR;Mc
z*{^~QB&@${<lkc=0==C07(GK*qu72;Ucb}Zm=gYWw^XqRMJMKRypckGvC0(Y>zDw4
zscTXpahYrkQvvR0RaLhRmfpk^))^#Mc)2zH*{?M%8$F{2ubrbRkZ8pUXoM(4LuHOu
zI9}o*aGVG0JV|P5XO^H?U6d`-P|{U)KwYHBNvj6aWqqax%wkd|cE1LRKPD^2Tab9%
zJLMSZ#l=Ri`#2Qq7{{ekCzJoE0!$-T()yihOjhN=`!UpP&4EIB6ym;@IVem0-578$
z7tnvIYWtXehSsJAsGM6KR^$x^5v-0QDCLco49{99_N3c+E!wsPtg<@PQY2d7&yZ{S
zalH2eR{KSTzXiCQizEDtM0%~hY3^9NZEu8cG>!#(rZGfloG`e2W9T(3pT5{WJ3iJk
zf&t+I7uZ1~&5E1hI6<rk(ai1w^#)!PhdZW4?V_CFF^hbK<)ac*)Vzm?E$;INi0sT)
zUw4;0<dfzk#99Nzk5!x~V4GfhjhTM8oc{=b9~%B>IpxXpredXh>!&cWjZ^(G^pCoR
zSB7<cf{2ub-ctDKL_*>NrtQtqNE0F&G%_TFv$Lnixs_Y#WuA@G?;#|xs4+HK1ySz3
zD+cUQdgPn}korSH15;jL$E)v+w^_E>-CO(-2K(y_+}H4X&zoJ-mp&_!!pIRn^6Ctn
zn;x+|PkZ&WDcZKo$;vhu2IB8c@f}^sg8c3zp*f)~yn-j`e3Yff8ijudrptu9p>2+#
zH#BDKugC0|7>vs8nW!5-+)j;Hna>Ii{k>|9O!=&$fEBU7ejo3Zn@8Zo&)bw_@=zQ)
zu><{)<<L})!zMNQu~}9+PdV3>tU=K}bs`1V?LnQuUP512Imvyb$y5hIysLlthl1OB
zv7%#Ympy(Cu!tc}L%=T$nGLeh`5iv(?apGf^lt^#E}ikO&1M;)-5eBKxRof&`B>oa
z2Oq@UuTJX1sncGl+h<*2_M0=3{oa5c`p7{w0T|+KZ6E`~-JF2F_Rc&(iQAr)J7ipY
zW~Sbx#<O&tW`<|wsx-M0Q_VONQ&~M?q#KiL7(8{klf8?AdY)=2<G-&kg`J;N=|Z~W
z9gC$1_+|Ap5`Pv|--%k+bK>sbr~rNfWfEH}U_|Bhj7zho0{BI4)jKj%qlz-9U)e96
zsaKDVO^Gh8wK$8Bs2(@f>lHsTq)}dfi9<wkO>t_k@OgYC7m+t+%Y@~z#MTom-j{ym
z@N=R1`X?DoI&6<^wq<kmqupBW%eGCk^yv-d77z(vJW1n-WZv8ERn#tzp?X<TN#PYF
zo9=oNpLNF=_!lV@(_OpCEF#?Xvr}#s0g*#K6FS!L3F9?Hd@5L(7_sVfmTpY8+<)E{
z=L~3<bx^<{<lJk`&;-8yyIA4H8*|aOYJ=6~4vBy*g<JU773%y+7Qs3fj&fs`+gMN6
z9mSMYv$6N2mL))VRwUR?*ZYynjAuFgeQEo^B--Ub&#AaqG8wQ>f-C4H!koc5fbn<u
zYC#6ry$aJC1njHgn*F<zkY0d|u^sa1h%g`6r{s3z(A^{^v-mX4lR*;xS!#RVQeJ#c
zmZ|nL$mIv&XOnc1;~QOTq2BvVe^^trU2}?Q_hs-K9ww#n?j1*`iFfemc5_JClP>)-
zu<v`=p~~PzjA_d{*6~+wc72c(PfSC+-+vb5uU~Su63m6m1ut%UvV!9qC-Rx>TG1NV
za*35@9ocA5U$nMyEjbwP34!F@W))}dpil&N1e#T9i8&Sht(-LahGE;&Mf1Y5eF}U!
zV5f<yL#b0(lt*gpX>Miz*-DHd&l|_QO$%rjZ)=D_jWc}hrpOzzKVvwNH;50~E?1sI
z`E+2}UNhr4-olO69OdLp^^698((mu`qF{~#FE1uKJwD7&5Xc0DNngQmY^JI&8ZmKP
z@|Ox0{d~Bce@gTSN^ih~pbLaoomDjEYr=om>&LBD{8R=UAHW*ISdqTR5aC_Y(Te|g
z8Q)KK#v?w|sWN;NutFbtyKHu$j#}$%-X~5Rfiis<$4#W%Yn#j?m*~ay_S$-HAIvb|
zCY4s-DTdUf0U=J3o^?|v>xkR>=4y9<vK+I%(2Ihogn-r5NxfOBht+|L-X6x{+=_K+
zQr!_M3VPGSz1d31Nt!2U4&omDo={pBZewDeHrh@-oEO<z$~+-V_RXoWaU&&KbgZ9?
zy0xPA<(-SZ$~-|+bf%nB(M~FJ-}-2xi4?YVMQ=yx+2{x^wVkh=PvLZojgaJ_!hq?P
z5)M(U;rDAkc4-R(rv`SzNQX1c51?CqO(>V~y7NdaAVUR+tjM*g>|@uQRr@l=N8V5d
z^tL?AZXmZ(_v3u7#3J$18kn1-uGCJGds+=w=Y1;syiNJX>6inn)`D*4@p>^618;1Y
z75c}h=GsbJXtd3YTy}1})JcaX_)b~_A?~x+SDd?3x+@XI^3QtmdceVET{pLuH;YN*
z?SdGcLCv0o*<S2X!(|}wh3sn}42Ea6^iiNt4X246&&zXeLvOQ-vu=QJ<))hc)0>^Y
zEs!<ULY%<>1xjS_XE6KYPpV-7I?*{|;cIqx?pEDiIc8#jH7ej3Udxb2#C-~)80PCu
z(XkPHyrx9i%|pkPDU4L(B!-kQ@mP-#akUW8`h8b6PIJ9gzs;<zm?^jLRFT$Kyo;@2
z*UA7&y`T=7TLd<WneA7YID=#%I1d=hXPaDMq;T$pHMhCNEIIO+(d(SNiNF%;(m;at
zZng$6LExoY$PzBw4Q%-$Ela=qs4Qg3KkWFJu`AFqtF|lvbT60}a@?n|SCe7Lhjdg~
z$*;<K$=G+T*_*4{()9_Mh29eLrK0B^M#7zG@dxy=?=U_tF)?mY?d~hBocgE+<o=Hv
zB($`w8ZAA>O#x(@4N%VpcKYftAPOr4UcAnodzIn#>lW)Vp4rEbqvWdMY5-0jU#msv
z-6y*lU<=)Fmm+EcAg@jN?-8LKSKlT}xF6DY#R<Iw40l(F5Ry{NV!lTbfFHU7QGa5E
zi2PCQypdEO=QSO>fwn94PA0pHo&+LHi3mYeKc4~XC=&p}YO<*8;)?MbmPx=|ZCfAc
zS^Dm<5GHU>o>7e3pjJ5E((+u%2l#*LUbt*s(s_5?N25S<3;@FHm0R>@em%XfB_VaW
zQ880(YqztMwipF0oq8DplhTDwCauD@)~81@-BT+a9UUbU9GaE3KAxBkHgjf@2suJ~
zg!|EEeg5+(OBx9`$68#7<Hh%^uxHak>NyW^h_TqIL_u(peGky^WnV2!5T3xEfWu_F
z)8JNa2NIpXY3bb&H$`mbk9p&eT<33mGG8pa7vHrI6_6UYp>p(TWduwVMbP8?M{{Zk
zYCwW{McS#Z&*T^Jrx(|O9S5CM{<`qr;P{jiCWI<;d&(y{q0`k>sRPJr;;M5aITD_C
z*N1ingIiPW1uDpw!%?nkoV>51B}MI$@rWeW^)UBMpvlTh#nV?bk67eJN<mXBG(sD?
zfP?U^qV%jYxZl@`nH6Pia~>x0OGiLhd^08%wh%y>wXpCJCY{bJ_S49vQKU{n9OUoR
z++gtfrbf|iHuoAFBEsWV3RyUm%ceP5;$qYMa9V5cTvPp>u8dhFJC_%iaqRuASt!tg
zKxSfP;RfB>7t5@wM-<Wba=(M`P4q7F9O<e}3Gcu2+QOi6X9Od?M+Mgo_f#udoQ%^Z
zpgU<JAX&HRqdL;yn<;iNumNOfwq{tJVOWtVtRV9#&}F$_u3^KTYVFTtScLp1lKhnV
z>Iz`{?Hc0}TkHH7D;0fucamccI(>%SJHMQ2S|t3UNbmY_^cNeB9Z<6se;vH-QY;n!
zf+e56aSwVS&9Y0|$5hhlcSBNU$e&ja=NK<3H<oBGC_@r;{Nz5W|IB9P94U(!ZTd>+
z-KQmKoS}N3DM{T^2U}?)9(PREL0LW`!sfMqme7b#Cs`KI(iN{wpr`ZD?|xh6O6i;H
z-|=2C;Jp=8^O6oYsmy4&_Bn`v+2fwl1(3;@UPcUvT8oW3rU=)Te!^_wm2KG-reo$4
z@@QJxKxA?FZy<)Li;aFuc*pcLR1Vv}O2_8(Ik}Q1pT~DsPulUSV54UTLe2H20ifO?
zfUh32h3?U4#c^-^sQYC4aS2RQF7WCRQ(DjEP)Ui&scBr~5=CXm?ahhAI??@ILF+;E
z_+%{ap`*8oQOOX&wx2G^U2=w#IKkteK+bN%7(iBZP55-o;5loc12pYkK+c<XC^gdd
zHT^5*h|GEQ2z`Lp7m7a+b-LAH2xJC;fwXWSDzScbp1yvWwKmMq^FX7sy?<ikj58Pk
z7$yK(Hx`q7yA^-{CkK1HJ>zVbfC3P1*0dLy`c)3up%3_2ByV^x0^z6O;e=}*$8-LN
zfLW-u`_-D%hacChfkkm~10&ew*|HRhuY{`&*mb48G<$1QPijXGWIgJcWgqYm9m@I>
zm$N7*E$XNm`Ftg$96#P!*h`S{dmwC=&SPKrvWR?yL+XKmRm^aazF31L!vFOi5Fy}p
zSKHgqX7`F|aB+UR{H!s}e?3YLho}Jo09~Q=YqL4i05tL{OXM~g5cC*G`qw>0=HZ%h
zeE;$#Ynlxab@K(u>C9z)#mlXbaxTjZb`$zy=$kTIhi^4I&2a8g>z<RU?02tr&Kttd
zlUcdnb}P!o5a|kyMw6#)@12cCmNwo;=+iPRkBl);SXg~S7rT3%g&#*jv5Q0~Q}<O4
zo|Lhr*J6lM0tT1Eps|}OU)vG>=Pl!cptbJQ<lRE$6an6FLf*hoW+0(?$l9nuZ0iUb
z)?lb5>G-vVshg{N@_8(E{xD-8F0~_x-i~d-TjXv1nk~53J<C%r+uC@q7)V2fM7Wiy
zkIITg_hj6yz~={HtHGM1x`ti6s4ccI>7vTth4l9E9G3@C;LdU7$}rWlzQDT{^*Xg3
zn^1wR@>lzTVb)c)`Io2W$PPt(D?Dlxxv}m;-y^a3%RTucw+>zSN)(j!82qJpf)Y1r
zrQBQZU@dpLq!m%aXkfp>!&7Ho8n>j1F@puZ1!%bN<%80mqH4n3pF)HiWEEW6_Svtp
zgKWB&-7A!fo^{v8u8EVe?Rcb~e@N2rI~6v3$+F}0u3h72Mv$^sg}izRyUJO;hoEgD
z(Cg;F9b@nP&(wwJ56@(@7}cN}w?Tl)%hQ;QpDdYlpSi3$#r4OcT((T;)XcsX=p-*R
z5VXcyeHYC#4*=~onUlP14qEcerrz9oR}g0`>wkE7u^(9K3aJK`gcm#}(Jb7C0Bs?<
zWv1Gx;PRBFuHhRk{wv)-7S&13${Q@~v^Th^;nS}h@n$g*)9X)rY5j$7trSN$7{BPH
z{gpsd)6{rhLR4-iK^AdV)_=AyK-uk2g1^^M0630GAd8eF&#_2#Xt#)lII-wd1X^5j
zQA_eHmH~$8d-+Z44$L1DJXI1P@vzCrUi8-W%;T<GoFL&o-rk?I_=}{l$~CvZmVsMJ
ztc+Cu4^w9y6lK_k`&~dlk(Q37B&4Jp76ECHkPfB0C6^LukPt)~X+#>9l%=~tYU%Fo
zJg?t(zB6aundOfiW?104pF6Jmcgda>t40E=9(^$?ers=W0j8jvJ-`&?I#%<W;bEly
zjF{{c&90cFPro~oELZJ7r;u*eqwpZo17zY=fzZ`sp%4CB`FRglROL+S9l@ve3lfjl
z!w;#|qOgbaGJpGzSkZK!lVWC+r5xa<`c%7?j=neAaF^9S7+Zfv3Pt4O!*-D0M=UK)
zKp=k<T3Oo@<Rr5f)8|*z)6a?6QxrQW8W){@%G(X`Z(0<8+EjSef;!;K4%kgq0VLAv
z>VMG2r2V<6T|vCl3j1fc-ParF@0l0;AMa6*c_W%EsOK`ByT~4-_!3$3Rnm7Zl#RAn
zn~&Iu<Vs?BuSNz#_x_=r9a%Ki(1J-%s6!*<=?;%ZWDY%g%5ZOgV-sNSXC#cp3MwcD
zu$1@%W>NqET%ddbu%|NM%v}8{GW6(A_43z;cbX{JW!eil`2_`l_ax0<>2SWeciw<0
z^D~AFNt(u`&sZyEpThM1u~E6M|5yQ_+A`h_ci1R<S_m%00bp*PYzAfQ2*(D76)aLP
z5{3&b(B5HUzCiu2hSwxOD`5NSylM^OpuQJY4-ceSVu;^<4)LD<nT|tpitxa^1&~6^
ziq<==PDZ=;^9bCYf3lk#^%J^!yBPd@*YJNV0RPKfkNw~ntkR{pz1)B8{7-kAzd}pi
z)KA;DT1V#3AecmB?Yuh2sK!rpZaV{vJu|Z%uw~9kfA!En25wq1(odoeW&P*8Mh&&f
z-inneFpOxH>dZX&%IxuopyR4|jx-yb$xO^&9$t?w{&xRM3g#mVWYl{a>O&0lt313+
z_8gxO@rBg=;>ELhr9HtuKVRH%8QpskTs)^n>9x@nxuQRNx6@kzH}QSQ{rpV?4ll+y
zJ6o2qVOA*(3WRDb;74V;A)h6B;Nrz;i$Ba7rKU8F(cQ@UJNq^5^fuFs<Z6|!xqM})
zN%AqMNlpzpKfce_B)Ms2*VX{S$5BT2^bvLXABn#Pg#)Pdd37fve4D!1mrE~xwyFMo
zmAIFOi#Yj`L=kcZC;V#VkXmc5XMfp3l#H-RzhAz7+B}T=YohwMF%vLm*oV_3E4w2T
zhzd-;-APgWM}Xb;4COWgdKzh*wtJA6=z59HJx5~ty^ixCZ4K@<<equUdhOmBQIoM0
z|6el2DuSl-5A1-7X3E$<Lpig#Wwtszj<r}@^;4odkKy~PXQkwWO4^C}!Nn<!^Yc=L
z%d3?8@bWKi?fhzMA}e8x@~f`6UpQ@D4p*Rw{}dWM%XJe}36dd9Ahi$yp%s>q2ngPO
zyJgtyY7mwG-8PRZ>&K`M?X77W$x^bScgv*Ah}|2ZS?TmWqDZgx_c(W91)qL)<D>&7
zkbZZ`-<X-;XZYJaWiY*}JHpd`6|QQFJq0%5Kmv>~!5y?uCs8?(lf7eoT1<+h+dnoL
zgt+v;?V_1Ro`EbOyWYUQ;n64)<z2$!Ctu2qoaO<Zx8+F)4m|G_x&gvelFug0OeK@y
z*rqruBXn2H;&9CC8lyoex`>S=_7cpu)9Qiw^*{#-*h0VwN7Zo~SI14jS<+xR8KWJ&
z+g9~I(Z?$PgqCRK3Dm4+m}n@<y-p$te^3JR*R&CAcvcF{H%ng3wD0O&%o_oJ_4{^r
zxDpHif3ChX$`L#g|MEykLv);RlBjJIu9Ec2e^iD<w?wbP@#M|dkNs?MXoXQyMsZ+I
z38+kr7z^D9)D5s(Z^HG?uAk-+Zg{DZ0BR?e#VdVA5GFr*$(?r|Ltqv5$HA}__k-oY
zkGS2Nx2!$Tv{!P|L#)Sl3cJSSd$fL%Y%EFuMwh6plE>?0&6AHf%mweAW)Ulw%15*m
zYBBFCinp2`fzpITa7mHdy{gqR<UePEX5eQplrQxI(2l<WTQ4K+iA7Gh0jmF(RFAs_
zr)WI5TyFL^_?w&vO6fiT=^y`J^oxc%n#V1bE6l_8??zAE%BuXT{r&9UW9O|GQP5p<
z`#tz_tyfOWhM$!x?C8s(@z^cF?)<UlBEohco&T}%y4g4lavKTdDc_}`n6%e{6{#x0
z=EQF<a;|i!<e0S6ofGeE=MV;;2o6d%IYYH2<_?;#%AA85yXVYLn^Q?3OuOailb`pd
z>)T6x_cjl}0=4v}WBXy>_WH)lCSI#PA!l@OW5~~;Ony33ZU1fl;?6Ouq=-Rm_OiPF
z#xVcm*fZWy1%A>uA)j9)DqUp{($sFer_i2R6Z-ZodG9@yfQzc#V%yNcB-a$b(j)op
zPv4XMF;9nI@vvI1d7z(#KP4j~%WwYy(YO><w^Mm*B&Vk=gI+>G?cm?w82~w<Ub3l8
zRB5<<>#er_fo8<1R6=}TS|mr&$A30f=0VW=GtdC1#gTHOUI#)+?B;ON^DqRpj`oww
zCBoztIeOe^)~kECfYOy3B?>k&O6=|w-`RwODB?lfLVc_xu3YSX+I_Oz9UC#P$;+<K
z|H&T7_KVwobYk$&Mw!TMf`Mz9Fv@r_KFYYE1T|&ZL^Y3P(xb2E{QFehMlV|q#Owc3
z;W1wQ3nQ-kHSNAHe*h$<*>9>}YmFaVa*JIuHsm>~6~F&>za0Nnf9p2Wd?R%Elwl6C
zHCJE6XZ5ao|D?PX-=XriOeA_UiZYRGltIIb#U1g<y%|MVno7Ff?JdQw0TJH`P<RVj
z!<qx6iO8F;{IV2)63Af732gvSnxLoECeK-Amr<zLEI)hqD^`Hi2@Mb5VQZBzozPmw
zyK8{HPg2}uwC`)hiLIE*di}GeR_*6*eUh0!oCsg2x{|XtyKv{nFeC25gCTVOL&lq|
z5}hA@-(At9v}f}1s7A62cjXbVXY)!DkmLpvd=tka0tKKJqR2BTN3t^wq^TIJ5vXX|
zzBj&@el3f0M~X$d4!C3vut+?+9<{Nb#tXet<Hj<T-fa6J+BclmC2V=kBa73SFEc`$
zVroy<%Peu)VM<w3w$Y{hRcPY}N!>DJtwjbHrQ-s$eC=YO2hdD_jVth-O?O3}_s$td
z9fk&kjjJD2JJi7Ka-t2T8A+gQT&j1(|8PqyN2&S!r$R^=Yp)hKc^z=lI@RE8zFMK4
zNr^%n+6ja>p?+2%!&A#{n^Xrn5f`r}F9AcXb-F-ld&M!5@Ss$z7>NhZiZ{|vU;pDT
z0!U?_%o|x?@Tc{&O~IK#A%LY<%ltp(iHwjcrb&{advcj(%WmXU-78X=GOCK{?5Gl{
z`AtSh!)p}fkJrOOo0LIvK~5Umh{t=!W}wBLHO&~Uz#T0AuOSKGY8LDO<jDo#M!)6~
zXuu12b(aNE0uH*v>nXTF0RZ{wzvT&zjhFrIq@&rNn+`Z)bZ<ks5ptnGVqcr*SYe&1
z9R3Su+s_T_=hlVzuc)@ib33ZDKgD|*vE?r{Pa7&8bId(pse~)~r0k33bNNHJIvUM+
z1UtZpb+tj_>s=0#PKbE*!TAyy^>TdGb%%2IRja|R)XN_o5$u&ee+@1AyOjNvE22#t
z)ITp#_Bf^qnJD|)QdaEKc#!eKYkXXwp$|886@LB6Phti4fHL*g+DW(CJ<i?@sV|?A
zaaL`<Kd0PKXk68Nk5KclnH}RE+Z(iY^iCaO7e<ELt40lv%Dan4@A6iN3?b+pFv2bF
zqK-3;Tyxv+5^_|+fp@h6(eH<Ej7<%ZZh5=G3cQ&EnKu1XoF4kixmTw<Bh^xO-hVC!
zefV#3HqD!j_2gJAGbegg_9rF}ZV%0Fy?Z^5Kix@aN9P0=cg3y4NmFk2)%CZiO?+#u
zBd4ei<(|p%Ig*y;zF8-GHS&T14?^vl_+EM-Aon<re)G2?lAK8aR%4bp{EbZo*zm=+
z(XL(96Br~)vG-PSHG7Vh{p?Fb;csLmJ4xWy5Cpe!<J=@MjK&jI|3PMA?y?bXZzF)S
zzk0j$Y237aQBiGwJ<e9P<7t1>&}O{?bFqMZ4Iuf^T@~hY?VJO)rr;b6Cj$rdE<APT
z^0g(KH=q4Ie5n4;{wmNMTKD<tV^~D~oFUD8*QNW%%{~D#zT(zH$XW6As~)8v4<+o*
zxW*nEutOt{ni62f;r_cZMDs{K#2#TfJ7aFy@k+hohkdfhUf-?p$?nRh%Y@>G98STk
zru*bt`{R^ozYK_G7K^`*kk4u4%*Aoq%Y#QVYF)Ia|Cy9x;G*0OE<>|R&TOtLqjB7B
z;uY261WyMX&}YmCe&GJWdc0JRvq-u{s-~{WMPM4c+b%TKzb|~(x=1pX@~ZCgx!pCd
z&4l&E%`W6-hi9Sn{sRu}_dOz{s<a<N*Cv)Su+@cRMgEo@JNWDHFQ_f4d>4?qlOz#B
zqXxAFue2-gndF2YlbHPZL9GJ9aKCCLY{}sAHTr#Gsf44qajeMCM0K6>;6PfN#95I5
zk$_npYxvzz4#XKd69sA6ev<$I!M`<Z{{e#S*4v5GyQcd?zN4zym0(HODYBt4mslNl
zd<+!U)gzABGTV{`{M;unL;^!fOl;FDNkG&6=sqU$O!{Y-?v#CHO?|RT8#Lu3CahE?
zuz%u>bMJMtui~c$N+$9zg1n@!6IE9g95_;pk9$-hJ|=3470JcWQ&=H%HC8Y{28d_?
zgevJbKwHd^Zl{R)H(&tTnFnO0=F2XVzx4VS-}Ut^+<&ZL-c=|`+gTKqanvE|<g{sz
zhNrx@y`j)o^YyzIlh69c|9j*o7idy=MKmXVUX-ZLF2?g`1kc9>t3ZM171{1)zG<b{
z43^;l*rtWTP-jb1d?a<HO(l;F->UEh)Ve1g^S5VnbXjPAXLO)HaIegVM%h1!$I5oT
z1U&glM5l-0f>T@#rSTWN`gsb81ja$}8)75(SASN)e!9L(Js7rdja9zVK1#d$v0`mb
z#1rgC%h#b$lVHm1M-mzeXf{j9+2P`1;uo>DP<bscdWBiF(iriZo`3F#*oyPtLptzZ
zegyVZ?>n1`^IDskd(m;J_nJTcZA(>W37M*wp87i0p6Pp$jOcY+31=e5P}f6=?y>Gz
zx!<jGUfp6$%)DD;>RJ@aEtJk@4n$uWwk`Yfo%Lc=7GnR$=klV#?6f|r9pIs20$M9Q
zt;Y<a$~$hJ`Eq9-k?pl{AeA53C*%KUxA)V$f)ZDK6^MZnvU=fSb&r6SL~rO)$nTaO
zF8QVzS>5M%+vbQRcDsvi<QV%Vf?l)cXG64^D!x@1#_`&QtHY2?wpvC9ybu_w-0XWH
zA{}uPnT9mJhJA~>mAnIbI*cj`^3|4i=VN72`#)pN(8$k56~v<mgMDb;ztTG10d(yH
zi?6rPZIR3u)LdL|SL%Q?m|*G!RV0+~@5^1OFZJu;kFzOsn%ksKFpS2+;p5&mPb%b^
zsDQQJ*!38J6+thN{5J;XnYUX^kEB#n2TxJ3TZM&BMoTu+iJ53R3GYUdug+0+eU$b-
zRhMD%_F^(>Vm9mqCJ`njSyIlsyc#wWj~M+{fSew-+)VkCd-&AdVv62%`-Fmleq_Q~
zA2#Rz_o%KP*!-MEb@Y1yku6)~xcqRjf)Jy~%*hEQtWWs(-`#o~I<&gl(%aopE$<q-
z;gM|CC@7E964ak_1-d}K4Xqyl<}W;b%kT-RUr_o3(C`huCmN>dG0K%|q>->5&UJ%B
znvxVm|2{k)AU2<<DIMd5g?<QPCb^i-);6j+`J^Gv1P?PAooYzd@Y7olmHL@BHcGeE
z8^~O6Q{BgjuzIk1Zbv3N6<j)<o%K&lVnrL#nZ2b5F=SW*(J(QJ{Ns2rH}CYTh<kHc
z{nok;mo@gj9)BaQ<~KE1qq3L0N@um-Kr=APgubf|n~L6SeQW#9*g2QU|4b%Ju;Zop
zb*r3TU(ubztFHj9jAkSX2aWuLkkh}@ECb{x*y#Jl4<{tr+@;?uWLoWO=i$cGYj5<(
z7EJ3tIWD&EOcw5n=br9v`Yq2<WR>XZ+aE*-@jKiI0zl56Q?mYU4qj2wdwgT$e;v$X
z1@XcJSm$by8J>?%SM?pj$BLymMGExyknO7eM+txgeLV5Ay?xR}<3vqK)AdkVI={y{
zrO(>D?6W&YJ>Y&?`%`!+bHJ#Dyz+8AwLOTazX-VPGUO&0jk$T47Unx#oAN?dRh$fp
z%?qV3?!s!dUG|V~6EYuMmPszId}20lm0QQ~tm-V7JVZM~(s>6qC-_X&$aKlz4*h!i
zbrx}Ae8=*Lp`dk)D7Q<01eM1eKsCg>cgI5}K;O=ke+-I-NQf_LOa{3;|3{-^m4C@Y
z9hC`?s~%cu%uxKauSQyed?q`t0x_t2Jg{LtkH4#iPS8jv2jE<tMQ=KTMgZ!uu$JGo
ziM<G_GcE!6FPAjUhd50{mckRX@0SZcoQpLHA<%Yr{tul}hSXt|7aFQ!e`n$LyXPp8
zyMh)2KdwH3Z70)|#IM)BiRzaz%%*#7Bu^Hq(U<uFz9}#nf>&Hl_cX&aYim-J($7i_
z{n~1t_)xzq$j!al9sGnLVM5cC5usl~uiHGXBEoL89w_E#IXvWh`g*hgSmr(s?Yk4Z
z1|00}$z`KR<97A>V;nV=(gfAo-{>3h7?c6M>MPq&)XgC&S--_>{XOWrpV?StT+|1$
zwVr`GZ~hX&>Zyf<V1YD@cG&H=C~$KYyMEeh23r~8r4*nO{#XIv)yQz~a~_R*Vs<+p
zQN67=A%jWP!7Zq**vY)s@e&1Azpa7^w~OAjYiWD{5fi|BlSMpALyo^+;%K6REsbjL
zjAUxnUSWqzF+7h4OpRvfMx@KI{UxSXm|x*((bUPXQg3^pGy;($yKKI~wEM=<Qsz|l
z_C}0(&(UOpNr9t1W^s7yk@KI!8M3?StH-iqOSN?`BwNNb#1Bd%@n4GgiBzH0jW;&A
z?y$x%Tgox6782fYndXSlW6KPhA<b3_b|JF!z42~5pE17HuwShEGtbR1nfV`)pUpDb
zV#hNknO3$qsOxP#qDJf7G_&{6_tYr024<1*TBZm}&$LirFtYn*cf3M%?9#+8n77kj
z<LqQ2Nz7T!?@0}xhV3j-fw49FZcX(I?fRFmro<j1MfD|}Ggm?+5!LR|NEv0{<_hDl
zHHj2jFZ9$fx`7oXc%(|b;IPjwpSA5?>r-6vHF`7K4Xwt9@Lib13@#kjP<PpRe9IKh
zkLLvFRz5na`u7uh6KwYPT>&y@(d3^-$_}H9Ny!cr!QG3uzjgHIpw#t+s@k?oaN|M)
zu8V8EQDax+c2oSjCvRQVP_W-Sb9~Z&OC(Fcn)aeo0QjWcGM2ijZ$JInH7sk&EH;T*
zA<_zb>yB+Gla`6SXFj+y8R2MH&0(LaoKZ6}-L?f3uxI%cwHDJgxa*9RtX&xn{yZzZ
zyAyO5Gbz(g@KdtuAq^W%o;AsZ%~^ijDA*Tv=bzeR7u-G1N##Yj(7%+-^iNIKac<5E
zC?XWo;6PWt<vncWI7oMek9_vtN~BCXN~zXnmALWxc<yiD2^v<!e}oWDv9v9gPB;v+
z3T5AYyoE=I-exT?IHvO3+qHdpia};#e(2Um6KIhv^%Q0BfTD$qqvfLt1{3J#Fczn~
zE0wZc6vxHqsG1`zhO=aiepF=_JgjsRfRn$Tn9dW8w23}3SGmsdy|(#E*;L;G5l8jz
zd}r)>()sn&k`2-|ZTp+FI?SYKSceWEvnPE@)JdrB0H@JM(&Au+XZ#>8?QF1iRK&up
zC>XB?)8Sw=qkQG!T5G2imL>`2@{LZUpn&1Kjiyz2L{!vGTdC~+56;eHk#Lk<q0wFO
z?z!8QZ3!TmA~%(y+y)Z2^*7!59?I>+DmcNY25#dg?_Q2LU3<pjgy#dN5dNVR#W*}a
zmxx8-#Kme!foW0rZVG6f{y~|S_y_Iq^bw#1p>%ipa=>}M?q`-k9J4ag@1m4{#N7^#
z){70*0Lwq#h|oo^X@2{8D+IZ_?HGlPrsnlJp+H^y?+Qewrf&V#MC7WFNbMfw<6GqK
z$KeI97J0Q?e}v{|v?mO&$TYJHKfJ+}c}=qs_L=PUrc;@JRl$Onhj)mEW=8<{#W{``
z>W4SaJ+re()udgtK1)j;mN-Tu^5M0X9&IQh?}q)ni+75cG{xq}dB)FE3jmMWAN3on
z@{nX_;sJsJDG!l>w2AdxK@l5H<C?j-tF!0>5`X-0Er$bNMYJ;^Td03y!Oc&6nx3Sb
zhh+~)ijPr)h=~WXC{>vi*;m7g#PDi7jJ=T&%u0{h6I|^G9hD@OexW+$d+t0n?K_R^
z>Wr+OLEcw4s<OASo^|~bW9&@Gi^VDkkd%r*n%YcEE)VAEkI}KorLmA^sX7&(hn-}?
zjsG%%i9rFcL&lm(JG$RZG;jsT>)a88J(-`L(F-Y1zX4Lh8T>FXb;C4sIN$~9h#~q)
zK13Ye{{i_NzXi304)1hwu3q)!eB+Ed>e1V@A&E!WCAo2pO(Ge;-49e9%oc(qb49#;
zfqpIO0E&3b*e7Jf3Um^afQJ+}uqMh6OePdHlZ9OKmG7?)JDLHGGdX;2nVdNs^adcg
z*D=IJ-TZSD#yF7_{KoEi=hb{Rrf30R?GI|&&Qqk><Xow_X#=$FirQ<Eqzw+!PM;Do
z);YAO0eIiiVyrAWv8yvF)2^s+H>+aY7ChO~4o>|P0mlQVnpyYOR9R-@1?eVJCTykD
zG2p#{KRF~*Rz;ZQ+%MKTKPtYfV3#Tx7|kN&z87#9Hb^74%R3VS_~#<0BQQ+ZZlXHs
zR`sD^bd;W~DT^^JVU&^Byk}0>OEvNEN|&$xkDEcfprn8Bb6xYWKJCTDHhE~hojAoc
z4K7rE%jK{);pFcy)5XhZd>V?fPl5TMpX_?og$zVf+QcAon2S*z1ZtpYfOK9tFPIX{
zbbewmE;gUEnURVXaLhC1bdLZ!o(CD|<>jAtC(6Eh?LVjd(KoP!e147>Xh)ZIYX=%J
zBdnpJpo{fEvvGWn{Oqbn3VL0<ce$9%UxkB|Eh&5c%WkteXz-yj_z9~rTwkIhSRLNs
z8r$Wb`g;-^DVT*bWBa{C4`8M-8JD>5T}sids29DF;qe$X;IUn1tNPIUIc?9L`6ZMC
zsXu}VO_}9y?;dv7r)<F(Ct>3@V}6?8k7lEZ9CHvjiyZthu<BxZVfS~!0M^l5`Y;$(
zzo9B6TYr;(7Q>E6nOf|!w|-yaBGZ_NbZH#?OTaVlz|`6HCJQg{zt0|h%cUbu4`-(~
z%hB_fqGGHnn-##A`XG6rBb;zN9L<kRCMykW<vnA49GHsx#Xv{zmgHS9|L%RYm(Ft>
zloeE53}4D4A<x$(;%?UMNmL5p_jY`<{Lh4LXnJ$?WbREV-h>EypQxsCsgDiQHpT~c
zd#d9$YdploQNkH=gnh^24EtUD%7Z8`)*)m&v3qSfIM!^xL@YQSozgh111fXm`{9i1
zNj)}WTN+Hz74VDEFWUvfQw7E+&W|=4Z?@z!;`yZ!c72?%J35SOCJb@=^Hm7NNhmhX
z^?SpI_t}uc9BkvY+bO|y60#dsWKPkOk>?+m>EA8Ch2pY*BmRixp*0iJq?Yp*N4lNU
z262C(kCFYc)y<IbLxRt6!|(xDy7x@U^{41-_etr~<IBuC481+yg$i@o_Mm*hfc(p5
zU!`Fcu^d{JA1BFV5?)QWM1PG+E?jUwPY5mjz1_>0J}kC|fd=(65$2cU2juMKFw;vJ
zrrrJ_{;FtV!-cy$<=f)C&~(3kboN1IrhAskM0xYHClz!1fjT2Iwex-(zk`M%Ca%}#
z8;$>%`p;<0P&Rkt<1<_C+!cke@rYl8Qb;6Vqm&=lX;%b3?x0;lW1S-$AP*j=+qjtx
z8zoGSGrT28O+&K8`-5G5211pB?~TcdF?GCS_X`#&*DkY37CUz=lOD?Bg(~z(4XMus
zEFVsz<Vr_a0nj-gplXQBZ_TDf3V_f+4DI({l<3*s-bQ+}Eyd;0QYfIPmwS5NY!@`L
z=a@wXTl=l6PCg^%2X{KnpRBiH9LpKyAq25Fts_Z#yA(I35dM!_=aT-EVR%!t6kuVx
z1I{YTVg3O^DCR0-6{fy$9Co+A<9Q%xjI&6owva`nBjO5B{ysm(c&(uCdj%q|3<X}A
z6XXT-L4(@HzUFmbX#n)kJ}CRBRp}qcKHUH3>MY&2NaEm8%Zlhr4$r9YX-V@ZS*;vU
zmO!nQ7RJrJOaBfPDA~%>cbF#nY{XSE1MgP{UO?ODDoSp-F2d<QeBrmBm9C)Az!f`t
zmk_?0<~rm|<_$Yi1#E5&DEX)#8#dG8OHf`}myWR)nKgBo0od1r4A2i$lr^afY`HQ?
zu+`Ew`4gvQ%fl>Aij}#i&=(kFP9>lKnv65B``CSN(o5<9Otfjb12b|-bUG<)-94HB
z0l+d$?#!v3YhNNza3Wti)3xo-H+z52Ka6p>0mrvq_b6|d`QBL28uiO3lr`gK#reXV
z=a0$T1)$w-KoNGy<8dZbJIA9a+<m>XN$BH0;FoT@;dV=%d%Ff0kXm-dKn^2wOEyum
zXpt0C+z)lKCl9}N<8)9|Pzy;4a_hUzCpM^`39_ji`*8e=>j%cwnf?!6mkVT5*!%zD
zT}uu66Ob~$8dfDO%6}gUVU*2_xo!&vhvkst$uII7KjTaV&SF15{x7CP)1bhnHUjR{
zVTm&+R#(o95&>sbjE-o_uaY#FEkP;b{Bq;_$5mo=<!A|DP7nv^b9eg?yI+~ciK6#1
z`pEdlV*D;}#$Fcg_W%O%T=AV1jY_;<;;^fS?I}TC@kIgA?LNdB%V;P}Xy+t`<6+di
z3D5Uvv|vJXWOW2)SEHlvO)g+B_n=Wi`Dn^uJch=r`{WyZ=R^vbR8Shug=74c{|gTM
zg(ylu+Ft)#n{%YQ&YQPPk~ax>CleL2hTwdhqRg?wDVbmn*kF>d>(b`*1nr{ur{1xG
zhC4+qn8I~S#tk3DuBFTY7w$w(0C%Ee2v+pAsxLyd5)5K9`W7DY0|uCdZ#3+dDh@=c
zeYd>bSuKU@Dyb)v_8hsaFR>E%?@PO<MDXiB^C^?!EB?#-xaaR!oh9rv5o9rtWE%0j
zB<9I9<%viOI&@&>GSl3jy?wv!1$acxy7%Pe#&=7{a9H`^xR97S*>kyYSZ7X>h}wlH
z;^ND0t_27j-a|?_Jn;r%O2NIs5}y1ku{Z^6xd;J=KHRq=22M?<^6|>7{O>#O%x=31
z9GNS96M_FJwE&4n&t;r;(yU0F#_7FdANS-baVm3kWXZ2AEZjdM2bS{-f91#jnCDFp
zbHhN&m;E)7)D0uCvwCYtY9YTbUM6RRJFnDFcI(d1ZJX)7PN}T|y9n-OgafF&6nw9)
zwY5;ntw`-KoF~*|i^$kPZ6CM<!*za`v(jfydxz|Qp&259+})R0FIhGi1|x_Fyv09!
zakrLcd;xC_)ahB%zDn2*3w~!G)Im22xLAB=CXb<d!^U7cs;RHp+6^)OvC^@^5Behb
zG;Qg$R}||0G>lUNML6LF?nWyt%MQ$|()+L~7OKP(KtS_2tI&tuk<ItvMHsNz6^?K$
z!+l_81z|T&Z$!Q*0G9PLyf(<RV;)a|8&~3ZSN+6nbo(#X5yE-trxN!_n)|HvXf|IK
zAGeXyedP47@F~&`ekp+b9#Wj?y;Fpv*~P~V$icdifZ`n+A&if&7&s3KvO;Eib`kel
z>OH$_5fb0Z3ZeuBObH`}!WmXkt$H+3d%_Yb&A#*}!k`+I?=~!-p)5%apuuTN#gD$)
z>2_25*qs?+ZDEhJsEF*pu%8N6X<%c>1CxzB!6ZaXaUHsWNF^9M5~j+yaj$)^cbLFd
zu-l3HB<IpF#WXj+RTyH>0~4gwfO`eJ`7c0F`pw6L`_JESZ|~HDtl|%5rZ+$J`^c$Z
z@ZW$R!p{~YMYiw1-!jjb$20rgE22L(!P`BcG7JgV($Ga2)BwgzX7u_0o%G7D(5ZB|
zz_=Y_Vy~+on?*`V`oWfu*%{Sx{!jYB4zklUgTs5_l#4+)d;T}<px=_UVyQPONzq2J
z*!Y<Ld6@4okJ7?WPZW>eTVQ3e-1WQ|dfrDC5K_Ixv)}rejD#5LDdmKa&cKI15Cagb
zg^jM49yO!IQPIeKB9z0uyf;wHjMiA|Q*R%2GmYCz`p2W?q7pBf;s&#FrpG9%Hzexv
zRQf%1LD(~aOhTT`*t0vbZ%u3dhAmke;PW7#p`W~*88KTy?QS+J99Mm}^iwMJX|ZyV
z)P2G;f=@R@3_EzRuYx0A#G{|;UX!RYy<N>pploRNO=s{VkGY8pKEmZ9?lmQRGkIBX
z-1l8!EjLHK^31t>WsVJGW%uG)+Sv#<$V#0l-uUuiozveLk!Goeji)@;rlOm6!5=;Q
z!}cp;GI^za_*|+?Kw=<0EpGeN$LND_?arPVqAFS6!FwZLS`7NiOfoRT&$7^KJ15Ct
zei(`xG{BxD=2CQRy{%em{D^LRSEpRo6pWfz{$8EH(0L-!B59+j^Q@Tq(Vr~uBctiS
zQ(!eYPW~J0Iwy&UXK+%p;K2XaD#Da!TvYHRP>vBNtF<dJ#!wBdY1Oii_V&51)rz!T
z82;J(-r<0<>f+W;-o41?lIs9u*4^1ep3FsIe7FV?H2AFg@^Dj(g2I(#m?t3NB(EJ;
zVvg5-b|M*E@o=>ovuSoxxB8WbZ9wurCXaTk+s{w1V3m?#(?bl_kp)U%-f7y^7d|Z}
zOsOUEyy);zn$%7bW0)G9C%PA$Z59wsrN4@du_OG(ar$MCDN+0=RK~SZ&`jaDQ-^rr
zzC>h_Bp^hm>Z+LYbJXIn`kJN8Pfi@Ir3O@n)$s<PT(Ad&BEy`!Nv=V#vwfsMpVbK<
z9)FRX>c&#_WaNs>G>nAo-O9{pPZOFB`pUh+jN{#NmZL2~Pc;328XDrZ-Y>cA#jtnL
zggwid@Xsc!?67#5AUk>#cYGcYf!XD8AST;g)l#M-I(dLQ=zykFU<Ex+KAgRm#$S}2
z6V@3aJW-^<RJ*V9%bc=>*ePNq>A>iVsP4BP^kju~Z^142k%D02h(oDCF&7}^WX&&&
zl5396{jGexgBov|5tV%GY6Jsi-nsq((L_8(gx@~B)HxGeOle$U^fl^l%D=w_|MxIQ
z4|3ky+6<=fajd@MzdWi$8+pyKGURTRXaBQc>c#PS>&J(=vA@?}@rLh$*`SmGW>~~*
zas}j9v?ye<z-eAREFq)7{QSR*ofc)62E||O-f6tfLAg=p`q^o3E}46ARp#Dl{$qVf
z<%R7c_YKF<{s5qmqNUHlzgw@CQXcn4O79!IDxR7yX*pzj2>M}Q_ayy)@u4jOv^akb
z>j26NG7#_jWz%;akTQg#h~+}(Bd!zv@6*T@RA|E4aZ5Y_!BJ<@PaLBFb(ns1IsWv<
z$113xjVWQXT>-URf!(|A?|1zi_Au+Fqav(Dnr3q$<k&Ze!gay$riM16L|DkIJQ@eX
z{w}g3Davl_5cx*-RECMO&8ayTY^oS!@xjJ4vq3`F^I?gI@+#gu{)lxpi16Sx&bRkh
zbd$=fm-b=8m6Xn(L(>8TdK5?+kH{hyIsN1FlnwJrh>u?iMYlpiVO*+b$pxO(%l(Lw
zx6XwYyLQWGFR&ni&je7st$+__J4_p?5@4mXW%lvzS8=HxQa6OpAB(ZFV2*olBdX+a
zbwFeuuEIQLQ`H3%Tium5<Q|C)M?aBda0UU6Mvt8^uKh?8e)ROm^8S%G7m05A`0FTH
z*SFiXsrI&ceu<|YTit^_$F>kbE4;xSKAzgM`oDau$H!pasQvg*PPmT`XWTDV+xKPb
zmD3|M;vFn1^(a>Ea}KhuzGW12uBU9!_4><9K2rpltz+(-@D%pBNI3%Oz^cD%wl0qm
z(AP!0L<Z%hBR=V^KEB?}l!YAp)mBf&3dmPX;^=DL6s%&a6up=>4d<u?k1&wy$SM*4
z(aoJ&;fl<hGuF8qu)v4^2JxCvk&44S<7V6s=4Jxle~4dx$x@8wdu^8WaY0q|Segqj
z0vF|@+QoK3W>;bk;278#>`A`O4JiTcU-4ej_FHg50#yO9*WkeV@}`#&<p(X5Iabzj
z6-QzMSaykdr8tT^O|=Drr~fDRue^)hEehy4f#+&@JC18pWzSO1eh#RuJe_a~A>n6#
z{Ka3~zdTq8u&kmg3<(9#aGaTTGi`LiXy!RV&Y(8bme07QQi0+xNY=hp9>z*WmQb%s
zQ}$-XV}l!3`8r5HNkA7D7k)ALm_1vW+ewlBetneEyH;f&eiKCk)$jgFl@s;u3M1_a
zbF6cMbd$>EtS#rI_c2e-KTMT9g;qSrzw=4%^EmsFvaIPOB3aan`)?Er2?)EZ-e{)z
z>1%>yf*JTA#>d2j<7f8<U|s;{<v~`5Hoda2a=UJDD$WIXxy_+H!|ws;3LNAhUZK#}
z%psUx{`33}!o-AX#9MQB8r*iU@<*Ok0*mSzKC}B1wz9xU1Vg_mPJC7gg91-W7e2Vz
zc6TPiIHC-=iuw+(Ie`3ofnQx?ucD873D46BLmV`Ky8EyM)<U8xfqW^O(>%ZLud!^0
zj)a<NR95~JzmB7BUW3d99YUyjxl}NH@L-;=){R|y<R&@eEHlR1B^p+`k0ECtH89{X
zL}j!%L%k;JxiR;v`{t-!QV!M=)z|K1q>{&~IHGH?6DC;%XIWNuPFB~sIhOd%iR5;W
zaXFT}v1>sI7oJl(uW_dHm1~8vY^7w7AEsDzV|>YKlJl0?7?VGI$IO}s$1IogkvzuE
z1nZYg3?nUSbfk>r<G57660zYk?`&P1j3@dy#PFG4=7xAkni_}c+2n_$`Q|NST{BFt
zv$pQN?<z~ksBI5N<0_YPg?~w2)!bDw5NVa=_3^|!51<<M{Wcs@gWH#B!~%!2Kz2l3
zf25|->mdK+xRuiySuiMo6xB0GLzp$!pDin<i9Q0~Dh$F$N{Bff1gUJ^3N5Bm*o9;Y
zu0P9a9a>bmZJcpPogfyk?B_Rt_(}{l9W9b@_jS<!QR+9n5bmO^-fc~ekaamF85Wqr
zuIIPe)DXHqtI^-?!NIbvvKZuMTQf)bHSh$w>S%hp;B%OKyxQIG8&}zcVnv#Z8#0nr
z|4{$P4QV0CQ_}@m2^yQt$juJr@IIbxqGrk09<d#vUCiWf^)hAV?(|>?l*?sS+xIr0
zja=zd;ETX%S0Z1bWr51Wpk(r7CTUBDkCjhG(ts612YtzzC2x5;NErAEO$bDe5*J&9
zGLitdGD&OtKB_TEZeGy)Vjz|m=eU}Pa48-YN3g}5hSuFK@JT>6F#RWpzN>mqpt}Ux
z5IgCiSM%o}g2d)sUtgyGlM*)F9yAW|axIYuh3OM`V&H+vTtVt#q)g6A9x4n;rDmxt
z=JJw^`bz`>Zvuj|$6EB`HImDkGnX+_(`-R9$#`WXrHEJWe+eGDIjdT-8sFQQmf3zg
zUc0&JA%YUdfTvjXyR5M!O*VsCS5APxUQd&Z7VErr9Y44}F1P^j8^nh!KHf|r#y+#J
znmz@p+g!}8Mk~+=gACKr8?^sE{F-RqZg%!Kb2W({8yS5W$8^4z>Dkw^JT8b{Z^&}6
zMqecmvD$qEM8i3*2&Tw}`@UMYm%9dm|L^;2FptDi|C7(6gOsoTaQ{nYjVD!WG4urk
ziSTrSg@1rr=-aSbg6cpTKqE}}C^j(k`KLa16Z@NODTz<`$Ozdj%-?UQ9xItroqeOF
zVAoj*{tA05g!O{qp`18tc`?+hBRb6M<61dhE!qYQtGC@848}#1dcHMPuU5M%)gKH=
zDV`ey<-dptw=sc}|0R-3jQ{rhW$uHft$lz_O}?t6mVE40!f&r@_c$gXpCwP~DiK$Z
z<JX)@8%gw^>A>}S-^brDAycWW&wZ2|D(2gl<gY*l-@$woJ$5hy^Qu<;eUsG|9YC#z
z4jP<wod&u%%o=DcJ2mI+df}Bt_f?O7LfBYFGBa<hu5E6T$jB@dU&6IMUyZo+88Q|8
zE&lV~-qi!)V}Qt%>B1O!PRTdockM4SiVk`az>IjX(CAt4waHfE0HrW-`o0>~61ETx
ziongHt$KFv{7bL?7n$dR)Ytp=IS2um;TmKGx}rp0>qvcv1P0!5mDfxfKGyfM88uH^
zlEBhKW!bW7fEC%EvQa)RxXiNlHd|dDy+5tUID1+(W|*?Q&^I%LD5@2yV<-;QbF89=
zH%%MBQvE9)lMc*UD63Hu8x{!LkJ_90d^QX@oSTjtz}PJI^{ygz0kFQEIs@~HXWkkT
z)PeIpJ<VzrUwR4^9|?u{ZQ<-ZT!<f+6l}$4z(M5P`Q^U4{L=1>#B}#RVqc=PV3g?x
z`~=JZ|G8;%)!5`M1bJqnLZ!6uTKEqUkAD0dk-e2yIAP{<b<Q{pqs_jzCfuO9gy1i(
z#Tet;P_54A2`w$(aU*~|A<gPM4a%(5AbV0mP!YpP)rZkT;{y4U<fCcE@F~KM_87pS
z+bo;77nDoG`ZS^6^JJ8%>;CrAV~Xsdw~){K%<31J8cbhY+%d1;JJIa5@x7Ai@TOUS
z9iF}bc%}adOh*eu-UZIWiX|^|4x3^vLo$r{Sjt}x*JWg$W3|v-z5qGlA*oT&|L*cF
zUVf`P99cM{%R0;kL#wvaZ(pzeHwk<b-ZJ^VmKUj6X5)9i=x8w+6S;rNMV*Q9?EhAP
znE1@Mm`C&J|2s1?-Oaz*fIKmpdjv47a020xkYd>XeUHKnrf-p=>DBnmgg4!xWeDLu
zVj0Dd4fuVwBpDwR^ISTUlj*VN*)ENj!Zpx7@|sCxH;c`ShA7D0ywaG{M3^o0;)rV|
zDSKY!oNdZWR03u2rDLND{S#J!{RCoEixoQbSB(?2=tE$Z!k%|YY`#4s7o2D_-55+N
z{4Dn$W-O$qm!P5QE)(kz<6p>M-*O=04|!4)#*`?v(>4<q_wPbS@LSu7MS+ace<$|F
z`u4%PU!sNV8U!7md+<K*RjS0+nSrkJ{x@4JB3uiy)XBB{X=XUOZ#nmoe8O&N0GIMS
z>q$ja@AO~7BvCSUBaj_<DPDSTx3+vn&%xagD~W@bMBFW~*wG4v4WbEvj?5FARl2RK
zYpY&5E?3G}47JUud3b?FtX>bS2BhQihPsJ%Ow2TVoi1H#nlq{WilU^URjunrF*KKb
zzm;C^6cW$2R6#QhNEDbO12nH+8;?!m0Acfvje+ff$=;{qb4DgM1J(4K)wx)6!02iK
zupB&q11!%VJ6^LH`D|rusMPl-=3v4jdKL~l$F^+F1bfC91|R1I0BvIfkQ*W0T5k!-
z`>NlF5x&*|j)Hk`tN4sq^IfG`*2~DbhwrcKDBi+x;ICHBB`uzskI)c4pCUXv?uN)B
z#S)(c$RH(;XH2HcKjNeXIfmV$BISM0|7xqfr97U&aTxe9u!xo3%KOOcF*EtupR#Z{
zDC;}B{&&oj^(q3SN^gh(_X41F)gSz=2wD5Woh(hvIYjC9l*SRjzJ2lodx##jpz21`
z8sF2$h>r0(%uQdP)o%5<;eni9X_xMGfb}c#4EYmc{PHU<Ybp9M479$3$W7LO+)4P;
zFCTzKYY2Rt%G1}D_{ef_)Ojxg(l&h2aBd&$9Tw0W2?YK+;(r$ce=z3rhQg#GJ)tK7
z^Dj0iHc08-EzrH*n^zq?s>n>L{TO`w-;_=o9}kL-{wZ;#_&JE2B*o>rj1t8ve+Ife
zwP~PPyx%G;@kxQ0SO_gGP&Ktv<TVcEkzk=zJiZb1qJ=mt6dO}FlK~ZC@cGcVnK@V)
z50|2u{p=_h8y^l#Wc<(7+LEGXr#JB)q#o2GlWaGmL?0?7iz<3D&4n^;n670RP21F(
zb6eH9X`dLw_H60SZY6_pRlz1E6U!{1gp_hejmm_c9{;)KeU~p$9MIl~0z$gmD-vr_
z@}R4e-K0uR-MwpRiHof%N^oaO5X0fEWmNTOoS{D7-+f8+l%KSy*vrfYV^aN~EUqq$
z;ijWDEH87bp0$X&o#{uO{nT8)#K-?lWC{70RxQh}SC=GGbR`CrChnzDC4RVir|sS}
zCtpX`n@<mx8hr<)^v5){?ai8KTVBo<lyYfoC_A6WKVS>q`bk}oShW;iCSM|YIgpac
zV5?tt_ULJ@2nyK-Hoj7xcK?5+6VsmS=j{=@O*zxn_$M1c@$_}}T2tlFJ><zQWo%9O
zOV;l{PO0S#F?3livp_vyCrQBFklxy_CIcs61Jz2QkCBk9ipZbq4){;?2mm268;JLd
zPJ^@mT_p6<7m<H17m(F<Gu(P<mU|sIS#Q%|AnXqjwF};?9Pw~=JoRQW)JaQp^3tN!
z7>R{J7Ah#{H3_nnS!$pGG;+1XhK}0v<<xEKXJ|u(`3?^->rj)WG6xK~)?npN#*dcU
z4aLU&JMsW~Sq@ipDEWrucf88rD>6~gSLSi-nee)|C$y!TQaC~#otW=1Do}W}a^7OM
zU&uGWk7x3gDKNrTy|una!eEL5EIz|)z2D~Rfz)6s>A=uVru#duIKF@#i)})t5JtTn
zoa1C)y-c?%AC)9L(|!+7aI0c;NFf*klJ~m%Vdp;QJLYxyKa+VZ8}>dpLtfT^MYRfB
zrU+M|LEG>RMN4|fy^;y;R^&b^ZRy9B)Oxc_NyKk3F9wxc?2L0a71uueU6gTcY5PMz
zF?1EtosjTX?vnTp)IuTR{@WmgTbic$zqy$i5uApS4v+3#7Ct5@1x3vH<1~db8cKjg
zj!@!9I$9J8do<6B7Vvf6)hpD1{5vC*_2*F6i0FwIK3h|}C@}#~m+&%LwW!1c+;>ov
z&NZ|tV4x+6IHxCLKOwiV<v2%Mm*%Y$Jra3NPZO$-3mQt^K8&93_cNYM{Egx8Th<t3
z((N7XZk>*m43&cCGEoD-b^aXZpQ!S(pRS$K>+jmFbz}El(<sY*>uN?<$YlHpRT~=~
z*T1O;q4Cnv#P039S=vNt7}>hKH-&h)b=JQ3_`LPazxSOB$Sklt(%<uy{on0nNW6bE
zuEa^OZ<zXhbp=xf>{li~KP{|>Gf!@{3%8PTkS)MM4m83n9jv?ab3Gp?>$sK2Y)Gy)
zSV}iz{oZ4C@k}f$lczTnQrXT{nXJYA^Iu=F+IpfZ{Of~BHQCv$RJe^m`We%BrKIAN
zfXf{w&`e-5_`?R~wS>UFZjZqut9)*c5UHyESG>+_;-|pKk^dHsoT?CicrbO{jC?$5
zR4hp%3J9_PL5*M_`sNk6ncPoZU#+>AjJa1g|8?WWNI*KfV9u-Z&r{dp!I&GW=|wmD
zMLt{lWA^e&{@9;y#`Bd2mqQ`c@~9t1L+`_^S$k-({s1ON7Aqa$rAA{C662vDP&4d~
z<oJpm4HUfriVd)L)cGCy_A8}z^X-AiCs}iV7o%M3FmE98^f}&BM$KpVP$^N6YB-1(
zJap2#7=rGX|ABb%E#vvWg2X=X3mX|3D8~vbS6~q~Zm!8Pr~(A+7C=bh;b8%m1C`5p
zE9af=qB=3GS~{H=YoDQhsDFHTQTXnpKA|Jz6umB7iKoNT_Zu!ws{n{?Wi!aFu6g=G
zxvUZBP*%vHL(c9<EAI)BGife;EF7u0_tU5o7?>>jC#_cXfLp}mn;S}Pf@#|Mx|6?`
zfzdM#%WDW?Qe#n`cMU+lq|A@8V)0`rRdxx&CA_RlL3a{FE+O(>e#Qr=!1fT24;)pL
zri}f}$u;pkh(imk4W&vF&r}CAH(V@Git;5nCJ*la%bfn(L;v?GClg@l=Y6o5^zRBu
zIq~UC-~nO##H-eHb6$LKjxbiLyaM4sZ67``(0EdQAc`%><o^M3dnwBz`ng1y?7TIJ
z>iVbZ57a$%S@hdxL(iTSNLol2kWi`1hKS?CqBZV&dL-^oAl?2nOpVjAzmFh0RfchC
zo>!l(Ohe&a&s9I^4@R%i0IAgX&jIW>=v}k~nep<_6_&ykH4|0-*s1V<GWh2}svGlA
zeul?{t=^EU=8PNW?WyqU_fwNP%d6U}vDnuI3GDjOZtD=Qz3%g^Q0DE;l?g$Xitw*p
zdCr<uP5JIv1qE%yD*Oablz?#?3#t|DXaHEuHDHVkO*SB~yXV=u3svrX*0dLu{I5hm
zD6&<YAHvjyWVP~3YKwZ;j~A$-wV-EVcZ9M+vNV8uWc5;ffwF#6ZD`oX7-4$=QM*~$
zbw~2@`ity4s)YK-{JAdly04nbfg68JE#8OE48gdszl9lUfykfRh_INr9iVQSy?E4U
z$gp|qD?Bz}$SXkp&A90OpYODav4ucCOnH}|PkEQ`+e}oyxqBpAfWq}yD+a{wAIJRL
zfuAHl5tJs|4MgrA_;uexr<x=BE_mUJpA~55x+X()s!5#16@gB+Z@_i=Ij$~K4|BY-
zMId0i_X7~j)ct;8?0^5M2ssq{6QhMie=3)r-EHjgX16TWvH3dx$3Jj7HdYdzt|%zs
z^^{T}9+-Ea-#9lD5M7dSnMm^*4h7z`5!4iw8{!9qROSOk^$(t5QJ%gR47UaX$)7py
zWgfjwH|qDJ&6tYTQYPVMXRg>{z#Sl;eV~W-p_8A74#EK9%pi-H0?~lg?rL88alF*W
zs*u5(+8kGvNWW-~i`-Xo19jqw%#*}}sQo5?<qjx`zmD+|J)h7(r;(vN`i?fc;thr-
z;OL-+;XRTYiw0pWv6Y%bj#h_L2i{YOxJT*O-ELSq^=fC9z=+v&@MbPqqml>20BwKd
z^zp!lm)nF0ao-zKIN<+Rxw)2;>A1&NG?XqPtDpMPoGIRb(L5hW+4}1v9aGnCX;Q}l
zg)V6Y=u;jlWXm_N`cj(g)ZTR;P(e+DC{M$GO9E!1d_=7I;)t*0e0{Ln{n}=^!ca^z
zO0X}9vIn-M9W&<t+He2O*#7tW!V~T4VCetdKN(vtm|4jdIh|!{!M0gzpefK(3=r;T
z+?Zfuh2EK8<KM*doSvM?#|Sfs&Ag#*0^9GL^<%(1tMdwisS)gkiC0P3XY2j7rZXxb
ze$m=DXOc(TmXraOBU3+}`*0g5W6cV4aKj7SBU0P_pC>-)#R;;=j%EBAeg(!;C6qM!
z7Ki;AMF&ki6KLRF+ADCJGsI2Ka=nu!Mm9jrBKf5=yIwX<6D%er`3LHjgK>4;G_|dD
z@j>WFJYPzib}3`gjo<ZQF_5*mBD<Ike$+t?pv4v~4gN;0m#>e<xm0A%>jn`$1*}$6
zfXY%DqQcD`Tk!7mhJ7HKX~7CK4rTS%wO+WnQ$9T(C>w0ci&RQ4+5KUda%du4vuD<&
zO6^EZc@14V2FBt1_7t_Q`+SI5<4GhNZ>CNxT0k0yeZvpJ#<Umg1AlIMi#LjDGo#%4
z383SJ?)0w6BL8SR#NDXk#(In~f4>$7&l%O&F<uq&qA}I)@;utCFwK-_>TdaRfUjEi
zD`Xq)k|Mh4qfurdhC#6N@e?>2Ca|mTI?f|NyHN<-djiu+Tq4s^TqEjOy!ETWA_JFI
zOvyX?S^|yL)18#{-D(?%MZXUre*^}4j}ChkGhIHxi^)Hlig+F15-ag^gDgyjHZPl6
zcuU8pdlSqpC@h^&AO5Y@r(FW(I;q1Fk8@yEoe%3$BKlc?MufhvujMlms5IgQ_5Agz
z(LfQfqgD)uxUb^ySP*mDBqyD}8ZLF@G)Oyz*LuZl+4^U4uxs<X1hgz6pyfUYM><5N
z<N#B&0but}#OAx6jhg!BI|T#-l%)j=PHGaD5k(2u$?}1ibp$QA>H!Lpnih4J(U=yk
zc-9vm0bQ6NZa^R~xMoUv8GK8&#E?1pBp^fxWP=ab+viQUm1VMg9LbU#y$~DwY1Z24
zxHut3m$4ma9}j5xsKZxL5rC_iyZ(n!@M|;*;CKUErR5#-`{6}DC3sc6JCNLhm?3*@
z^=l1b8n+kw%2TC=^!!s~WfI&koRuqr$+@w~d7a8+B!UJmkdB8&N$L+kBJi9BBRKkh
zEC8q)enU5<YgEchhbM$j&2_+-BY>?hM*LDuue<&6a1@BSN3Ih#*@<$8FH5<6fd4#s
ziim8()3E#q_sZ;Y-Xq3>a4{rgSe7Zn)8tH?d>EEz|H~}kRC0qkuEh|Ggu_kW0vx+c
zi7LE2yPoz`R0;-+_><|%(t!bl2+b4kxm}CljQvX1V?Ah+Pi9!KWdP2AT0)ulCD|VM
z#?&EA<lx+nLV6FR6tU96zu^f0rs{lzO_?^fT+5^o!0#RZ3WdT>MwXaRt@-A<%V_oJ
zP|P3`B*RS$zeO3?Aly_$Ryb2#B~OK&etAkb=R;cdw2}_ox3bV?D|Ki;ZR!9TciVRw
zUTRR0^)73Zl|Kw0D>JCCq~%H6bIn}IXg$X`7H_xxzF$Jq)#WQj0s5u!{P7K%U(Ell
zGa3HZc>Mpr@}K|z4VCLZ>Od26!;FH37!v@9^7%Vq|2{<}m}#S4d5W5a5ou=g(>VFV
z5*DE(l^9N@s>C$-xJas5)OR}3e_KOgK`X~^jkL@zAUvxSzSHx8g5ro~k=mD><bkaC
zi0Fv+|1kCzKvA#J|Nqh`rJ^9+Ah}5Af`ZZ|3gVKo^a3IcONoGpG)Q-ebS;9E(%s$N
zoeK;6KV0wq-us*RerNu}jQflZAj|W4&N=V%I`3yGtElS5TdJqTH1fzLr7J-#STW0F
zc^Ru0rVi(IVS~58ooEM*(`_rvjlqU+qlk!@5X_7qdAHm7bPpYhfg!XsHfYpF*)XFc
zPSm{@fs*iV210Y2j6QPpaHWw_YB~@3_whO4fG;gU`3_44mvuzwGR5Jhs||N~1p@zv
zgfHuuQUkt9bj=R6;q?};Ique;KBrS2G2=qcI=S1+6j5a~e|D&ARYlBBopyL=I0t_h
z1P-fd*oKXs@B+&#P|DQ#Q~_>S@Xetxc2o(AAgNz!oMv}U+A34_<K~E!I<6ZR$Q{hY
zm&z^S!4LrHKa|V5oon!$4Hb$NhMlSq5qxOleg$3$vM^{G#~CN{W{vz6$<l<U)#Qlo
zBH4#b=<@G3LrN#lf{=Bz2GpS7y*vaNedLjcE7_yyOAzF=Ax$RWoY7*FE2OkxjWmv~
z4HO;%eBIIAL!Eb?i^0MOt#C04?bov0f%<OiELeR_IgS{q``emqGItFhr5xb&6{k)D
zYGh1zV`ZLLYUX@nK!Sd)o2VAGH&a#VBaB>7`dz!EXl$p5L`6DjrsNLP;i~u@zX?x@
z-usKEC`Ym~50DD_jmiXHHy*6ylay)K!&#XAF;H(lER_o{!CY```5@)Z)*uc{7CU*I
zH96~$QW%0F!t4~AM18UEq|)LsfPRZFe&N^EqB|cn={P8}U|4O{rf`{ykp+>o^<nLG
zvb7MsP7`3%yfsHf2lG*_JHK+&7<d@LchU4DV*N;z-227C<nVIdP5jpPN=S!$zqf11
z*@}LN`C(q{zIRKE$;IQDMUW?Il$)N4?|Lo%#PMY>nY1r0h@Fob)%0*LBcFd`vG((6
zzBB`7)$>1$$|yc=?LR%}|GoyZU?O%$oDuc+4NO3NQM3H|Js^Qv>&)y4UqS(iJHx-8
zGFV+v8`92clE1^HWv-iuGQ<mekjpK;$D*cpId-pI>NU=~5h)}V+d-C+i3LG?w191}
zczoQf8AuB$VMy3p!mDe!t;WY;5`dKVD8H;ex99M{6QwxQV}~$|VdGy=D}Q35b?N+k
zHgwt0nT`1ju$;@)+Ng(FXas~sdQBwW;j3P2LfM<?mWQVi=N_$wZ|F?2Q5`5Gh_Sep
zTICf6U!VO5dqymY&e0MXIp13jk4C(L+P>xQBMiKbA3MN$ppJdi_g)hVP!cG}YJd@;
z<h90je-RIu^tzX!SLG{>i0kM)?RYb5-20{FXXPl<gal8mqM+`j1@L!z8KJ1vR*0=W
zdJo5-d3gMNU_nf(PX=!MV}rWwjeReyp1N7Mo}*QnTeqcsi<DZZGr0v;2Ln_IL5(QB
zfOnX!oXmTH8P$yWFPt~7a^h`hs^xTC5NXSKU#XCOz)cw1pWzzIX2CZwzOQZ~GXF;w
zg~PAL5E{I>CKD?Fq&CH@ZZAH5k@z+L^;=r%eeVR{^N`a0%_&RU!D1(wlF!lb-KLS3
zDQcHBuMN`pUPr-aYFv*x4>Mhh_1bT)D~CT3Q25$x=9!h^&<*POV*9G`H{&?p*hC{6
zh!p)aS(yY@1A6z%ksNRCObn<5g{=Xjlh^x%_!@HYp8-^ieB23ylgY&QJdDO4Xa5Y5
z)riRwR1R2PJm9oeAj4Z}c<Pn-`X!F~J%cbXCn&hSKw(PwrfCjA5un!j`p$w{K$!l7
z6blKwHQIJNqLqvb5aSj?tk<8cK5(1R(-Go~$<Gn3o4kmy{vg8IZ;_KkR*SpP5X9`7
z(u!iERcKnxJrmsREJ?aE+0aB@9L_)n`$Qjf+xS318XwlSaUbu^*?8<ovlf@Pwo9)H
z9xjITZy}qGck`jOTpKkrZOj?H;&us<Y-&!>A1Fjir;zm>3K18N!*`5<2=9k{pzki+
zkt`p%%R&2@Oob&zjs7e%;>4m$cZB^NP6^=ISSNikMRi_D#$Z)lwd>1Wub|qQa4eB$
zU6F;a#^-AeCs-O)69KAT_=GZ)BozhtpiR3coGWr1V)VVM1@#?aB-<I(_+wbN{Q7Js
zI)NGVzE06i?aSu-A_h(M0e8Yu-9?!7AwOQT^+%^q(yVv;YDGcje1TeFxBiz}b}W{u
z!P)sQDE%*LgyRMpCAH3U*=t##n7c1;btSQeO)4$!*q=2a(KaNbem?w_kOhHX19BuL
zV}In~KGyH6;bY3xx!&Iz&I~pf$oaS@XHXMtT}(t7Qy}(KHHfXvoGWT9u(Po-;1d!y
zpGzXSK^eLvzymvE73~}a6dm~_W)(%O(~B_e)VV>^w+ou1vL$SKP)Ee9ki_^?ys+E9
zB5566j@Gdk<k-5kGUzn}Li1Z#CNYr>F<wwdS*nDi_86A~H9DbUl@xSFD?`K=v=VXM
zQXd<ZiZMYd1<=vIt!MQ0riuk{y}ho@qI#PhK<HS*YWIf(!6GgCQ~h!c%6xDN&~M4m
z=!`faukX{e<~D&cSn`9JHl#8_R1j$o{zEGM%^V$;jr=@SzK(M1ff0DgIGO25`Z%#P
zCMbvb?PM-6J|-^RvEoz`S<-uE_dNHIr`?pt;}xg2pfzPh#UxmeFDM2qL~5qW^*97p
zz_4>{Ib=U+UGU~Ii0W|5uWaOE%s$SxP>gFUJkZzAj!aAkAE)NG(9^0Zr7^`^oibD6
z*B@v~?-q;1gn*QaJ#LSRK?ml9p>H)P-{Cm5A@25itH(=vQ@d|h#3V=s#lqR6Zq3Jx
zVX)FC5C=tqyg|Vr`o5S9WjI5ls9{=4fyRv?M(Gvk?Zm^tvG9_Q1?nK$uz7iV6Q#r(
zSx1FRoe0GEC?Q`#b^g>#L7KkaOJ>BiptzX`s<rRNU{B=DuTQcZ*T#Xl2ed2R*lclq
z#Y_eXeBE|T>693Mk9(~vHD&O_`5h5Pm`+U7WpF|H5A6dn7UO$GuaGPAQ_-^C>_6n}
zrujYjZet8BCgY|0791$1BaW+jF>vRn=IMATSA*T7k<=XRQufbNdgHrj-tNE&nl&ha
z=%u!$L^WBxvnq+%$8ze@gEh*vTU0Z8M;v(#9c^-#Jk{!v0{VX9)vLu1jHtg<7PgXZ
z*(ToYbM6PFj*Il##=iQC*SKLe{`0YN(-t6-<Rt9{1K{OhY@u!Yd)(A@<5j6?T0m!t
zm%=vnmU##=!d)Y6dv+N$hY^-VtS`;CmTMqXp321KL#w5Eeg&hO9H@9@MxxiAN20ee
zcYlvTa;qUIG}sfLW;Xd~PUgr6DuTa1zrJVP9&;2Ht^(&JKF^d9H~Xz#*OYo68=hO}
z(+LDZh=M-p$gPG#mx!F;n)8=2L`&Z|ML0YpQ2}`U*@g62-t(A-)C>JZSgt5-JxF+e
z>{=r(wn3?7BdLZq^>|~`X(;U_bv#L_1di=}<hxRhNAG}^fG$}rf!zi$X21{JMV$-%
z<^^!m-AT?f6dn)Rn@}Gg2mA%Rz`O2(1hDW^%l+O+b#?;-^SY*l3#E@S&4pGDX;#Pp
zb@=9^*y7n=!QXUHY}W|+FVpt#s+oyCj8GCoT2lhz*T8ND^k(~%EAcE2G3<+5dco5b
zBP1k*-tRz|IeoZRpxlpgQm-*wqV70Wu$zQJFyw*v7AXyZ53eie^HC<)lIq}7wWTp#
zBPqLyXi}KLlXmEUmH=YDgIqD=yCf7#&4VKIOfm^}sQb1T9$6!ox4v>DK9?2}U{D1C
z{p{C~PeWu#kB=WXtapBW5b7{z6cnjEJ=)3&dUzyypBEM7fr{Y<?044&R$&`Uh6ERw
zBXcR$Xe-aNfW8a^h#;Bb_-`pZ*+Al~-PzF`x{Eceatqr3d0SO*q#KJ)Y=3SGHG<7{
z0St4^SKcS3g%|xN7YgRXgh330(gI1@s<)pm=m3@w4%-vM<yNCT-6`N0Upx{g9X_EB
z?~e+8e<Md9@!_(6Z`Mz$ma&Te-K77lRc{3P#NS$lQ8&l~p}^o41BhtzGXIc(?#V5Z
zbRI4&i%>s`RFZqFPtV&rl6gr~`>wDunYMg*amND8JEf3_YeSM^7LQ{ct&gG70WyFW
z-`tZuL9N!aq#TqHsZLAIV+%B8S&P{->Rf|Zf@3Shk(IKi-erA4dtrk*Aj7L$N=IQs
zeYNHuWM<B#k{AfVhsRT!RHtx%Eyy+m2oP_Kg^?(p18J7Ss~nPxFjbJmbsqvj)uEMP
zhgxi)y*z7>@}Q0nKr3n6KM$xzTwIRzC$>os{>C&<#)pcG%irf+2h?}W)cKd;{+{~_
z&C8xYgD~szfA`7!#g23^V}i^>0IMCdAaFQWafu)Pq`n<EC&l`?tbr#g%XPD$a){96
zpLgr#RW#&J5hn^!Ff#vloBq4nWWfZQ>H5EGRFeQXSH)I*eEsBXoc7Hr9+PwuRywvE
zh|F6o7Eh5EJ8z4ju0Rao0}iFdP6j<Ce;NRRF$@EY6`ot2<m4Cl936eA19MQ#J7UU8
z6H(}yoT>;eD$MI_Z-Lo;4Ra=~Ny-r|bD*Yo5Vhwx7f`)BD`d|^xU57tLzjsgV!gjV
zKPqUX@;p)|q?+y}>vvLSstCCdND?zDDg@jc=u493n_t96=NSmGUeD#eMsgnPUMw{+
zVkwXrwDoCGtUjhY7XYLCSG<Qpi9iCuz>Mn0tYQTcZebRXicv}l@A;p7sltXSXf+}L
z?`;xN&C&q%Mv<lU(f$2&10A5cKX8F&U60a3mH)nvf3F+PL#07|`PZ>A6|KNADcas+
zTP@ip-RLG|g^mhqV9h^m)RgjQTL;N`6MhV#9S90dhNt;*v0O5uQdQ`fp&NrgUesDG
zFX4%9oM|WByVr*^d3ClX<BxR_MVYb1NCKYO?L<NRV!~=(IaWu7p<7oYH@X+<S!4VZ
zA8V-S`}8Y>Tkl^sm@ja|ePVs!OfPwMJAy9O;p%}b;#&yOc;nX=e@U1CEi_n_!x3N7
zL1f9Gj-3I1#-<0EQ^KWPIyEBc3vQ9|ni8_!dPh&<ovjcGWhIQO|7gYkjGs>G<*+)L
zJdoT27~k!skA6mf<Vg72yM_tbfT{IQ<GcbSAK(%#u;+PstX<}IY{_rGqWl*(#-j8U
z{x<y9;6KbgKwND7>8{;BaV5<7-`WXnQozoO^db~4_xmg2EIw|g<=p;N4Z#xuUEfiQ
z#o)y~k~#7rQor@k8{$%Glot}_^Rff?q?pJ+!ay5gME-HVcp)4NTf=kkEo`AInk<x;
ztIJfPVDq9LgXv>pDQMY81G|qNh!v$Ag1Bsn+ft(KTD>u&(oI-%X$n?*x9@n|E(S3j
zF~~tNnzB{B3#vUGszk;X<WKr!z9Lr`e|~*{qxSx&y&kEUPU3MwcjpWTdgi+p(AO;W
z{=hab+$%|Ui5@j1@o0Up8E9j?EjrlsC}p}nlA#oT-=<wDw+Kot`4fXcsqA~C6#oBi
zv_IkG&!6hje~vx&)JQV_ct>mw?R@yg=(pV?GHwIdS1vGhpEFPnDQ8$yvh=vNsPT72
zxoY0K5V~*Qy6q(Jpz@o!R)M|=Yj<3En!m>PS&nFqc#ddSE-Si$tCIwsnlFOE6)(%)
zJV@~2KaO=peIGVm21^hV-)Af~Vp1R0FPpT5juO4?FMVJ-f%c<<OLJ2W2?cMCC?83A
zTru*y_Jjs{WIc8r7A$oVo`u_=Rww-YL<1hsEOVQTEy%N3$7bzLwson+<&e5-ZrQ6A
zyoTl~UJXc+z8&*pgaiI-AW835<u1Y1G1AYY_nXnIxNJ?)-qpQtdyIEL0)Eat3rqkk
zdJ`i{@NCr$Nga&mxI?RnZFt(e3GVnykKwHMJy#WrDlZ9=9QO{TSt@D%g=etz5gO)&
z3M=F#JQ~dLggSoI+X^b$?^B?&PbCqLI(x2vWi3dV2Ul0rem~|%lGf`&->qaE=5@Yq
z91;eOSy<XLP_<VNehF(#31fYpdn;9%=7m)CfZ6Z(DA*8Qigbn?jSebOrK0~y$Ur5#
zUT4U8csS(pI}=trD;tze*0KFNOgF40(egnODc?rUQW)*Q*o*p!r~`#0f=?y*()<Fj
zR0gh_VJ)Dp@N*|_Z2NO`|5rP^S>NXVL*1I#uY7-5U}`y9U~4(@-56L~a@guLqf6fd
zNaJd=X?uD~S3ZR%5y>)E!7TpKL{5yT`KoWy;LTlr5+@@Wo6aCt7xCS>LzSGbN%y$F
zBnb3K@NQf+)e-&52Qaw9O3kYs-J&xSzbeB%6xG{@6sVfLJ)Ug)O;gpcqp+08lrKlG
zsXfr85DoVww%OQUwqJrK((W-9r!n@~R|>5xT{_$Ec*S01RITi53k|7vvB&|g=VK)b
z>DMHzs)DCABCI7v0;YM-kGqK2vUWxf$-#o;BxvVAj>=QX-|$KmA~yTUGTYc&nx@f@
z=)^{Cv2DaJd4c+Zd-aH}Q7$30E}_m7PrYRZH_<r}!|+!3xkX9U!aO+>i96ELk_Ym3
z_M68he^DGbr#o;W?S;rq0`OZ@d;R$vKvh9}OEeA5Ix*Zf5WyQcV^DmD0r`PL@d+Gq
ze(;H>wEPt#UR}fJ@r9q@9W|l+&t*>(Qg&0soWirzmMR6EP&wT`JYoZpUY+j&$q#0p
z(Mny6qjnh1J+40aSr!r&P_!>Ua#|$jrV~kMTCm1$==-GY>&8M}0eyq^9p(&$-=iPz
zBXFKO_4PC8BvDT;ANB7a4?S1Ca=e?$q@#GZ6r>$St|9mQs#)%9Nu!za_q<?YiD1&7
z2jGOS55}{3uCHQYcuAHg^gs&)Lt5s??U+C3?kTl@_@9*YpZ3o`ulL_zAL$8z%nzjO
zr%6Ox;1!ORcfm;&(VSlsdLgt{roD+@Rhjo{@u$2-uy_*MkLYp|68xvv+};SntqUa<
z6t%r}e9gK)?K4RPFMTITr8pdZ$nSDw-2P!XY_iB*YOrc}CXk8K7T}Md!;ifrmu+qG
z{jgJw8fvevWbZF3K0xw@>P?7S;(=7|fBsNNJ&+r0F`4O;&{*u;N>!S9pE8;a)LkWc
z`dDwGLY~kFKwkY#B4IXNpm<RNFQFJBO4_7Rj|m~+Rl4(lZ!{nQjYqc&?|}qxoI5@f
z{Q$<l+obs$Y3)EnspJKhzBm9<(V#`s1Iot`!K_#NfeD*qYV_fTKOJhv_LW>e&oU9$
zSz^xcJeM8w+da+Yk%WD-=M}=|!Zr>k^vn#9_{jDCj>K%p25@f>QCm?SozxMhP@Na-
zJu6`h4$AiwZ}TwMs#sY%GTe)`(A&rt%XL53SO(*%f%N@#_O-TLn>>xZT$wgwInTNF
zIv2{HX!;D%HpsZ)5}su6g9}~yBbiba8qpT+>jN1&M`Bo$9I>nXVP<GS;+vx%$3bkX
zVEtz)8F8U>-z^^zF#n<PaNhs^&HKL}1wtSGM9L*EK6l#b$(P@ojN=E^eVz4%?M+p{
zdhbZT8g8a<hWwm)X1LXK+MOzlTjiF*<Dp!cRQ>Z@>dHO<_5M62>v?V%h5*o+t;&6q
zYbns$=qx@(-8U~iN(_pB#HFK+CKPxC{OU#+xJ3Lmt+gUYgnr%<c-Qor|NKw^M>yaB
z_)Y%=0te@_#}e80c!L?5R)0P-0l@+?e~KIb6mb_Dq{5I~k#2l*WUG(=oqIEG%;CS@
z%~4eVs29+~cNkk9cs!slKIcQB63h^xkI(Le3gLOIY2<vI^ReB#9L%QkK=Tu$V)D<I
z>BPej3}A5}K<9)&@I21KJ%Src6Cqc_U|K%vJ&)_s0Nn_$Gab_Bhb*}3czPIVnV}7j
zKmerAp2bG76fUD*joetmM}A<TWnrFFum{W&9X|*_*4x>T0<6I}3Dj$t{_~lA&xzA{
z#P`f)qQ=4u6em3EkC<grw$88S+v|DMT@b!D1B)xY{yo=mMp=(Q=Jx}CvmgJJ{;VeY
zh5sMjShs&3!$`@HwdF6xDuXqm{N1G{QVFj(wPF>cITHY%pLisVSlsOgFX~IqXRkf^
zne<&+^50bOd9_mI`Wc^!jT~*op8Je>FzxWCC|`|rAAvn=_-}*}s8Q0fq1WN0tppEe
zT-%$$5?47kfs@J8ZXoC@YCUSLmD}>n6dj;d2lXv^rek`uo{l~EZVIHNll;zGGc;W#
z7h7A2OIWGn*>11w$C9@blXlIxd3QZ7WdgBCpd!%^lZj@4zyzfHzreq#ae*i-id~SH
z3#9(t?Lrd3$VL~fA?S5isPTh2AM(8mG+1;FhhFiFql66#N%1^?`GHgZd(i3ydFZ`}
zq?eo~0g(3jn36@TbRr+4J(C4TBZjcxMX9+!g!o=D)_ed26=QI$q-O0<%F3Dowg?+b
zBYymd_?W#7Gg!W5H;0ENiFb*fbr(xzjzP_^L_w;0Xj#b`uAnVV*7=?}Wvz`k%K-tE
zSZ^Q^yTf9={;LrpcE<yAx(&?2OfjCWLMIkg=QZhI-}rR}yQo@{5*I|2_6YtL1<1a2
z`0)Avk#5QVNw?as^D#nn%KYipSO&GAqXNL0Dz-s=dIdFIIbzokfBE_OT(-V&-17=5
z@s%p9-pBR%sOFbrH6L8(w(z=56Rb^5!HL#|G;S&ySt_XyKmsQA5da+qUwAFA1)yWo
z$pxjDeK7G3P-?r-+f8i<$Q=0FOMN?&MQ&h7hBbt0Kv5Ly<~9xo_r1<rKFVhUkKmZG
z-Xc91)F5V8n8<=y`LBE4V09^$bIVTL(xLEYa>%gb%gO9PIn{%)1rPxSPzCKp!p|VD
zP8|`@-*RX6zgQ2>4omJ{q1R;aXiKBQCXQB>ANIr)r(y6t+K7)*M10!Ci}=N~xOw4y
z|Ll&0Do19dYzka~&i#l}{a|-8oY*+J27rp$6rJYM@H7JY0ym9Guc_^zZ5IesE3HQ<
zTPm?NP{EOedVY(Sq&)dy@_X<wCf`Gzl&|l{*!>|X$#MV-`XFX)RQzvw`QH@|4L~{&
zf#Q$tDStX_S^z1}ZX)3Rm4#XuBpD4|#5^pG^N<kQOAH{KB)>d63|$FT(_J_1OS8bU
z-%pGAasI9Md7RmdyO^Ennl@T;Qcc&e+ty04`;8`GaH5&})Y2*<MH>z88%LCNcN0(U
z7)}U}w>MnaoK*5ec3-EPmD{N`Pj=<>3k53!P8ce{b!=T~O62izQr!ZxU1t-hM}-=R
zk-#+C`{oW}#K(#UdD@vtUc())te1rN>#t7obgINWoS40p#ce7xEUfu2xb&)q5C@HH
zrU2rZ-J|8$WuS&ipgQ_&pcPAin`D5%q{TBJvJ!r8$CB4qjx*ThTvjJY;Hf{SP>^7$
zgJq7)f7xGNGKn=@R27}8nNeN)Tq|Bd$i3y%-cta894@+$<>N2M-r5f$?Xb$Gg`f%5
zf_dW8bTM4QnyX-e*<RHUrDKk0l`L+TDY(}}%?5dyWd&^toBQE1Igg(H((OW>g3q-h
zu1D~#<Ep7>sMdEwF@edRt+C?dxDvx~lv|4~zw>-=A$ZVh%v`@7X;EcOqnYRSfd3+g
z<>dG<GDss^s5E-AdgJR!^{BC<F^_n49v~Sb`46{#ro>lDUDw67H+O1}ZKkvyHwSVD
zefotwnHnlvJ@eAqx)3Fpq9)^pYUrk6_nZh32X72HtWuQFdFk)GJy<Rm9g*t1)gHA@
zrh4Uyc3tKua|`||5V8FgN0<%-uvO5ppHJ-0yK!B+i`Ex?p5OD%eqO<?lCw<(OetDh
z?-l{K;0>;rqQ#DisCqb!=Nzjph*h>k^NePxIAY_Zc!W|AQjJ_X$%quLpo=FgRFNCd
zY^0yAg~4FwwiVSvwZ&|?Pc3dGA3+SaaHQF_`D!Iu<}a%P%dDY`n4;Av(ks+~(sspx
z5`_V$0*G?walzK*`_+}n@rQ+cq-H&3hp*L_^p{AY&usNflv>{+zC?~1Z$3Yst_^e_
zHk0ZSuve@Chfm`#ByI>!0$J{0i3t-ym*C1Fds#Qky*FEmhDc;rq;Zqr_pFJpzg+r7
zxwTD?)Dels#fX_c=X;y<5Owy`n<V$W*Jni<!+(y5Fdu(m*v!Er$Flk05E8EQ26;e7
za>EJ;NIWu;LDcFDkcr^x0JZLTBRK+<b9AYxow7?pLwvrhon<0QbG`?r{)lE`yWS|v
zZzGoQ@O~vFb`~Rbv`m+LJ{Q;&L)0yEsS`t(&V?^ds2Zs*4NHOG5Ks`i9NG7^Vu;9+
zR=$#}<5qbsf%5^mYv8DeFFGFW<ucGHjPT|dx-1#bW*V23yJAtJa=&v<Nq!kNK^f{Y
zAcOZ5LQ4gsU$=zUK!tbhHdobp3=+@p_yLjYj^A&_zYdGvVzx%*|2-o9UHjc&QGXp4
zWa7K-sk!ymOsnvYQ_}7ny4reA>KJ=)(i<nfRT^EsvGh!Kgw-!Dugg>GaQR)HT9wlz
z;!hKv%SlDEk3(yZzx8rxotDi`U2N254?7ifVbBo2i?c<{Q%LpjdeL|$=#4335TPOU
z`gTE|SH9So#Y;Mm4VuY0BGb<2ev))8X5b|Z*Kdrw4(Vk^5O}jfQufLA?&Sy(O~!JD
zC}Tt`cI?=<6-sv7C4~m%B{X-EC%EfZP3k&N9Tye}fV#uosZEG?9b8Ic(`(+1KRv9R
z_YyoL>*ZX@$$tnq$Bf(Nd|hdibmk+nw2pQ%KP2N8^m22{M74ilR!leEA`_9@8K3!3
zf8<j2O@RKB*$joPK5A`~kw!b^owuI)?8tsytk@*lb!_w5`6X^p50Ig^azh&H7iVqb
zLfN`M$tG7XEUhIbzHilB@t?t?YM`~FyhJe{_I)bZ!EmbSwH+O=j;}h1waK=dj)#bU
z2SZUouJiQ<u`v)d-$L{&lfGwc@!zA5sfN1pXGCC`s1cr(izc+%Ix-;$YrH7Jd>L$|
z%jH2Y6G5FCQir!q!QIem^1A$TZ_;9AhrG4vr>;0Ds9Q8(%lgE6mT$moZuqo27F-uP
zvH3lM(>q%)5_ctDACHyT>S`ftWx?lKb&GTmuWxF9w}0iM)K-K`tz9>9`6%3C$v`*;
zA~2P7T4t?`VRzP4bsk!4lS0c6c^k8QIoF^EjpHXCYaG9_Jl6%v&7uS<_tQ_(hT)n}
zj5hn@xfq(Y?J!mAsbfW{(d9U2lH+GV!`Fz3v}~PPYig;%hY$E)U@{S>S`7ZETkkJ%
zbVv!`I6?(=smRDZmuA>!cb(R?!FYFPE{l9y(sxth%1yWWZ8U&>Y7b(f{dHYGm;I``
zSj@P7ESxem2%<l@1)A0a*I4fc4SG1eBmuJT`&5oV4t@Lj2$(nmvaEC<%Nn9Zv@JG5
z=pKF4OuY|jDDpMiTkd~CoO;Xg{E~#VoB<}X2Iz=;#Q_joOF(KW4CU*ItbzA~>koWr
zPXu_*+I~fPxg3PnU1xKow7m#uYOF_FbPa@F#3aJayV{nO<E49WC7!9MN9Jskyot<;
z%4ZFt*k3b?Q2z45CoRQUkcW4?_!XNR9iEums*Ku|_3(Jw>}Xbs(8ro1#}9MH<mq&i
zy7;9GDN&SceC49WGPly0F4~@+jk*l?=D5$+!`N|npME72P;r|&Z`M2{FDf0rj{^#3
z)DX$K8QD{rPb}jd_%9Uk@A~N<s;IkKDiTl<`S9rMJ{aY(ng3>^i*28Mpoed8n(-)w
z_QZC!JF#ZBU6f=ez*Q<1n*39AK&oQX!n7WF@yxoOZzb=s>E2o^LaU^jL?lkYEMmOw
z*GV(1DRzGm$Orj_zv<+|^^2;i(4RP!qZM-v^2tf}9%@T?R_U0R4MfRq-j*&?D%0!o
zlLWsn9K<j`ciJ^Ov8iyLn8&!n;KI(r>I<7*{hm=&8YfJdT_J-Pr0rafh(RGHR#`fh
zCv>zx@d;{>CX-j}*3_{6vjDxR>mgGG`Sii+T^nL#P)F8X5K$@RnXFT6UdZmw>!$6l
zpDC&de%VSt$yTP4FGO@&9L|$uKEt4U0PdsE_9G(X>Tnj^-S@cp*XxK~<RNkPu8%tz
zbV;jTk?)9irIeKpKRkDvD&R{oC?55!$-_`SJmh1`;jS8QZ~dvx)6mC1k>=)w5=iLV
zlwPh|N9vT`E9p!pg6<WTlyUAZLtfofDIQc6qbALvgLtGl@$LiO-V4TKS*|jNt&0(h
z5!~Lmdsy@(4oy|d)G@MGj#v#_r5V-Pwf_I;K8*Gzj#3SV)Y4@gb*fGkaW4k?E7WQh
zN^^h<aehg>-w*7My*{6!5)$vsmlZ7CmUwryuQ7YAbEoHJ0>yW(cD%kj*ECckDpsEH
z-Q|wM#^48=rx*fLiwg}D20j-PDO`8VUOa8#!7vZQ3yUSrm%m=L4dG{kT=`qxy$y<I
zoF6l%0vc*(ImkaMK{st?9;rS?BFIrup5-+L*~KqR@4Ma&GHXDStX?hk!M?>HNqa!3
zW*9YcImL5UYFrG~sKsPR=(yCtni`sn&f~tgfn8)3pyy)))LFe~L4);nTx%2%<u+te
z9uKMoa!E|MhU12ZhM##0sKZ~FrQQbpV)%ADf>k9jAM;xvf$ncvL9JnyP(qJ0=};sl
zlhoJcPY_f9gQQ|G5yh5aimfN5l(oElS}+|dN6mU&J_Ukk6Jo(8c|m6^(|W2Vb+WhG
zsa?74FKKhNwT_a?_?iw%)nLAzGjub+#GYtR_MGy6S`z<{QoQ!3bkV6ad%sCpp`25D
zA}DB??YVpj+>>CcPXQUqzw%p+^m#L6YeYPOntOETXT=ov8+s~Xi^N4+qsq7IW9gB3
zWwm{)nxmMUpsE^(5%)L4$v2Y|-2QK#om3+?E#rqQ7A62kE!fg_(F}4fVBfU!Gc-Qq
zL}Fs7NW(ynOxnJdfl4KD>|}k1e;294dv*!^42eURpkN1Vrq$7AR5@BsoSB8&C(e_v
zqRl`=(y3OVKUD{L?a8_ygG1c=O(o9yw+7(agT|w@8g$VPpm-(XpF>-)%3t?9gqAfM
zMQ<EuaOqliJ%G`+Hx+KtRcrGE{4+C(Lj3G%+sL2m)q^LhQQKnY(`950ztT5+S5xl9
zF>H3WseJ}#GN$6#^GVknF2nducGOI(r&sW)#BH=ksektNXpr4UYKa{drtM(Z!ltyo
zk8P*WA_-;~$(B<*R{7mkxJ#>}387ycY^H~{-eNq2rGLUQ?>i*FOhIgByw$OUT>$n6
z)df-J{4*B@u4;OPz-TV->n~R^IDW+syGNeYo^G1IP7R1vGax`HOzdRVb+_4@)n_D9
zU%&o>80LBIWr86MTd>gbqDpv`9^T1SQV0?N1$}p+Fjw6w%`?*NgOakw-O~L0;r5u=
z`r>XjxwsUjfAJjTB!QjSu<_~jcR?%Vj0wSQ(H%n!Bf&(yE4}@_Akz`GpbR9(X|j8M
zKN+7c*gLc??s2c=ni-U~<!4GaH5NXz-bffg|C13EikMzAf`Tx>^=31dmZeighcE1Y
zYY2fT3$J^u<HtoZze>EZfr`6X0}e_3%(IDmNfZ~y_noguW;2q;f7Ova%Q9KoH&DV{
z+7o+dTM3VcRc=lEo*Av2EeyeuEg{mm2beZp__VA80~3At6tCL^nopj<>Lxr|S$C@M
zdE;J6y-6iIC3tkpVO4Cap0s-FZO8<tqm(ICCc%!EzMhHJIf#<N<FIU3*Pk3&EjD9H
zc);&+S>JkP-1dAik|g!6*ZjZ()3Kk6de7sN;JAg0TeVsZl%<|}2kQG>&mi8o;)$b1
zPU4leTOJh(wQv00v7u8>nrdbxd&f@sM(jrKo%R71C;GdQ+sE&M@XTSH-w22|@7*#u
zUE;Js++})sn-%CqY4YZ9Y)f|n94oVU-}=dG3lblh@>CL53}p-#Q24od<0?@EmgsrI
zt%e*Gnl0}mi$&iE&!dR*CQ{F*)||BMO61UkP-?NL-(2b&sU+8>oK#Vi2}kq$X%Tjo
z`N)lpQf{xd-<nje?QNAN&y0ag{V1$EjyDW!u{UOP5YNSe<tIR~akcdALadG5ysZP-
z!tRPA{Ur4xnv`k$rCJA>(~^PbGD*S_hGCklmW1LuBlsHm2KMF1qA*bz9tdls%iC33
zG|Et%t>2nY5fdtD?$xF|1*-Hgl+s9D93Ch*SkWi^ulVef^n-Wm-<bU073SX!<MHGs
z$&>Nfm$`4IeHg!M<=M1Bm@p;yow_znXA<My`tgzZ1oTiWy=~QkXcN{=9V4x#dUTy!
zzF(bx9PRFN4rnSL5O&`l<iuQC!b~-yV)L40B6HhacUQyyLUBC)=&;M(3gaPV`(t-n
zW8!UxSgX>9_Pcr40x54A<P$Al78v!2uIbuew_%vJe`eX=VcaH)_F365E_}>QRqkMH
zWR+zVpl?im?z$D1u9vWD$s@A#RFD*irdfreClm6feVZh4C9@&H6E8VleS=-Ob<B2+
zyh1vm;k}M`jW^Tw1CZQPC+v2-{<>uk#pk9L(*ZYnREje-K48lQuWWQ;zC*CPI&+%u
zezy5=Myf#o#JvC|WWX$5<|VlAJ-|)dTBf3lY71&6bM^oc&G{&8Gah|l=>#6j;R226
zDUXqUjSIjbRp(CI)VG4q_1s+tukyNqkhw*M9xl2^k4Z3FDS6@ldNaS~D$`eG^Lg$U
zG2oI7C&4)NAlwR?HS?p@S<!b&5j|2|ujT1M(n5*A?bV964gJ@s-9to2?HJu}PugX5
zK8(Ol9iOt{Joa*;0wbS(_%S+VQda+^4Em%%tpjkYpfl!l53wm;z1@K$9f?i<6Zl?R
zRcb!8N?4x(o68p7QOs9kZ_NAo?&0OLAla~D8&PD@&6m<h1Sr_WclKcn{)+MDBjj6S
z0;W$~VE8Gvo#ask=m{THbKn;L6gfRe4;IZ~^%=|j>M^1m)rL7N%jHB=Qyi0vG}*Af
z2OY8__5l9TWn1@c<tQeCjS+ftTc4=LK<!OJ5R->n{}MZ2hHGcTD{6X#y(00nL(X8l
z1i=-VHuaIQ;rQ2I^Dg5bH1TRE1xG%x3#Yw{?#%t;?Uc!WqK->Ba)P>(RNXd$sbx+;
z-mtB~x3{(!R#@=dkaDB#wv@D<)1xe=gx?HpQhZ@(rqKLd?W?RH3C}P;SWtBfK+I`&
z)bBlP`z&{hA%H|4@+v3FYaMmvLDe>J(uxDNis;XOAc;dHi+p(BasTqkJ!%}S_pvYZ
zFrH~U9^WP0SQ{)tD!MS%8=ObgV0B29w%PRs(433hT3G+sCr2Zi1a}ETYo_Dn@`x`i
z&er44DbZb=u{|G%RD*q!i6@}|3_%EqfGO;l50^OsUxAaSIER-|LYDr2`AD7n8%aRs
z6t4EyssJ~O1X1XnAKp9MffmIB%^&hjf-f(z<A<$G3&nTbMnSq5Z2$+K6}pbpbk?rF
z^s7%OGV|Qth(mYcN%K4J>)Ib{UTb%C2VNw?h^J7}t!x=xfDFye%0z#MW>*a{D}ioF
zeakn_Cxq?ozn!zQ?Q9@0<W=`xjqwifV<Z)9h*V_~3XIZu)mpr%jN4>8?2;T&y;or|
z)}m{-6kSLl-JcbzAHl4}Thkf3v%y!uYsfcVjK32}^4Smro$nrinx+_DoQv(mt2HzQ
zbKPz^C{xMAVm$-JYb>(Gx9VRP$6nW`xPG*FEnXpiHuveYdosn#Ko+;5ojd2~i;HvJ
z-iaHYy=+gdVj+mKS9XG9{7cV9|M+q5l|^6`dY=4qTZ@m}DzVE5m%bLYw5BY3+PncL
zjHCLhUAx<njh5Ovs1b8#Hrb1x8^rbt7FW^%KyTA0+MwPmr<VDC-6ACPLAWGqATV;W
zFR^>I`d&<10XZM-Bo(GlEqz$m{mp(?Y4_XDp#i*BLW;{0SBoj_XAbC5_QNf!vVt@3
zD8}Ee0>J0$*z-^c)WlY3R&SdUuMf5Cw|WS7Pl=+p8Tsr$D%Az;h&kHS77^ZFygr1O
zYqopAeLEOcEu1W#>C|#}Y?DJ=Ys$<#rXGJUe(o=eQ_NoV-@s(%fjv(rgSYATU}6v@
zT{k`%h#JJgtlaR%!SFUgLvw63y(Y^U=3PfXrDtfsL(a8lZ)`*wwCnWCYZ*uJcpU7G
zX5~Q;-qz9vv3TRs{?+H1L5U0WG!t!k+*DCv@LZnj@;IsUI-bRjxFrEIEUoZg=D)9+
z1l|#m(G4-M{({j<NsrA-Ya=Gs6apYFVXC*SH!M4a`3`RNp0>AtEDri|{*5V!dwoC-
z%6AZ&NaCU~S(*{aMYY#A`g+;GbLe@?X=n|)lf35piTS^H-v2)|VlvWcZ=NldJMU%5
z<gK$%-xm;2FDM;Dne>|rb_h}jN_-({)p^OZD@^B+f=M+sanWnj$wc<3$E~>jqS@AN
z8>R{Y#fPVa2sF-kWyj86PVAVeZo9=Vs>KO6Pobw$KDP~h*Xo?|oS?b8yCBEIr=-mj
zJW-_uJOeGWbryik+%-;;7%hTsbOO3+wQ?Nd3irHCrLQlxPo&rl#}f={Eb5X=$goW!
z%|#}*PFsstX%RL{8P`~srDC(;3vPwHes7?!Dkk0G)|Og%_gt@Kev+92qGFZP3#WVM
zSF?80Os=Ki8p|@?Jwp+y0jC#hVS0XCoB;K`5n3<W^Q9Qd(F~lLQz9_fTVw)fZNZ1J
znf0nu2<O|CxsZZxi)m+P3BG4x)+|(*%mY&&-o0nwgQA@mhGd6<CPdh)PjG(6N~BJU
zXZ^eJnT^<5agll$hp26X{j(PDsYvZa>iyR>^{Z0G_sTxVKE3sTPP$P@xPJVLsmiAH
zB6F2+X;RmylX$$t(C%DqI`@x(_RPily<g;}{?o}^G|_mV8%0ML-XA4<b=*5*0yp|9
z3-M-MKaS6gZ0bDrq=(*Y`TS=Qp)pq|dCV?7ho(ByA}y69O6b!|K)A9aH5oX;5CB8(
z^lFlIk>=Ql?;4)dl*bsWB_C#VzdlyHdl&c*IeJbc$xNLnJ^Kh>^wx6{Fr)pEW6Wl<
zaDXA;<*^eio2nCR35~z#6^HOmukNbBJisAQGFgU94UR4U4o)uIa7HFff{nEFW>v)^
z4cH)7AM-=x6Dn$u3l2U7@wvzxL1!w&0rmTqfzzXsEO0vLeP0g1ZN<IJ`{o6gX;YU|
zyRsn2la>nDQY52Yd_w$NiZ7z^E9_;CnpmloPLGsRSYsWUNromu&t-@q_?csC*D_tm
zAq$bZlYqteNGAH8jtZs62DZF}H)A~S<q_)qNC~oIV6BKR*{(U@zZRo{@(VRsqQ72h
z$SLqd8_Qf3|6-{B-dV4BwCGVjAB4E3%t15CNX?2R_;V-X%NYS<lYrb}l+yYS-}c{B
z?q>b;^I!a5ceS|HtM6L;$CcKwiQUqlx#|oR+zmzbSC;zjsH(6)R)#IluMvl|Vx#$Q
zYCEh~1@^WAxackT`OW&N?YqPBNCgJV4lw#8Aa2F37kQOI6th|5JS)GG$wjZe0Gt6;
za#h+V<$!FOuI7Mjz8qe>l^0~?3?P9d=(hE_<L;_|y$kBC{kU-qt7?{Ukm?GWcJ*hS
zh_@+#;tIo-+WkIpKjtYKdOs<)5_RbuRO%fi&=bn0BJ7NIceoqg@qjbdhdj}cLu)SF
z{<BsQY8cIX%GUV%gj{o#*(Q=%C;I1h-<Yn`OYKqJ3b$ZUpvLR>U0FG<d|a9A_HEO6
zewT3aH0by)8=Z8d&97u-PMEIG*nwYx0b;}JomrVzz)LS4f<yS-XDwfK@zt^CJ~4OT
zNtdY{e_zQgq~V@RjWbHfig=_)mIhVZUPpe2;8`{L+N~w?r7lsZm3zg0Mv0M5nlxrH
zF>;*MB9FHH9E7>g$Guw`_YuNn1bswTyzhhac-pu!yu<Hc{C94q^(XY%wfhz}&_{XZ
zV?dcllBy81E?cT#Qg@0q4pfzJ-;rLM#wel^>oFN8E9M#DG)`L2etp{@JsRmz<Cv2W
z<kz3Wm_<KuM{xjW_9qz-UR%!-y~n~yNLv`toY_4aTmcJcOP;*AO9A}AJe_xLTT_h+
zk0!f>4XSj|E*e~uOP&=mP7b1oFE8eBNF;1M&qZNGM~h|bLpWhZg+s6YvhmY3?^v$?
z3(NCu!W&Q7{)NMmO6Y>&@y)w600bfYZqPvPfNm6G`htZRzpL-_w}<uzusWSF(B&_(
zGe;cQ{$wP?^Ed#zDjqSMd(a(_kX*0bR(e8*IAWh~Z1hKuea~=#y0t0MJwZ7uR$Vp9
z14f!wrMFqFhnEA8U^P8bW9m9;Yk8lcrCa##OTkpwdLO^2emG)hSjsgHt;SRau&u1w
zqM+M)XYpXim4?VW8)3F7)v$F<y(<<@NEX6S?J}N3XwHfthrx(|?dgjjnSZ5PKUR6a
z{2!@S-{00!JH6CUF&dXKyX?2kiaq1y?2%n;*B)lm;ykOgClim4^X!SrY%4v9c^lK4
z8f}T+Z=DNnP<b=;0n-fKN_5w)!!bo~Y*>|(<~CR^fvgKCY)<3%rS9Y4&voMU@n!?1
zizW;|V1i4mbqp@nSJQmhbuH=mD*Q^9iV9QOKB}9;{zo?t7EAXl`~AvUAcwxW<;rH`
zm@Uxs)mJl6H_+pUgNR{@H;t~TtD?%!fx8pwX;oc3^f_2&pnR8R(=6AOr2S?GkCu#>
z#QP=>F}`6L;#L2ReHA9#Y~B;Q^jn+}I(&K$1$YdNg>|>&+$iP^vbKnm0*sGgMA5<^
zqM(D(BQah~#wei}6Hc$km)9ue$0cG=ZF?xRWLoQ93H`-5Ntt4ovL;a9Cm875!PYS!
z2q&SJXNwFm02){q`ktdTr^6(Hnm;zqcCuy4j^a`?m$1xjJl$RfQg~!$l;-c?jU8eL
z@O$aP3lU4ts@i{GEB`(wK#jYeSPV`jdX3RhOHChdh;`1T0OLTdd;R`g8g{^e?~MRm
zoB?0S6YUyTwddo7n$}3#GH^ap^3d|8s1HDmUbk0DS@4ULENLa;Lf?ghf;aE!zMQh3
zi;BU=r*CxmmiO0a-jBYhPOlp)mvo{5Wq0qE#SfUF4NsC}r0>4N*uyCM#0)0%oBdj|
z^^HVt%bQ`YfwnXZ446KfkQjs&-)d`@Le&3{3pBK${3B0hgcvgX87@^RmfHJ01XcNP
z=$97Dyt}^!e=wRN9}q4nm8P{f;e~j#tLLPoHpm1Bk!mHp2smd4ETE4#<%bx~sO?g3
zc~Fv;9)OcozqCAI7j^Ks5L4I_pgm?Hn5TQo717H2F&`-Lqq(Eo6y&mC>x*-w&Shaf
zEzi>$0MZ$t4;s{94?X`EY#9K-G5(%hxJ)c(?M4dIoNg^77U+~s)EuEBH(TvQPY(86
zLzg6~Z$|eH>BNd&UJ_xXio!;dSAO)gR+V%c@XhKTFI{swCdT2z@j=~pf6^F*Of5bV
zu89B%TxqER7PB|D7Q!hm_5#g$)?{gd^#_d;5rNIQew}1rxypUqoZ+uWL1cGJ^!oGY
zysTNf-^}q*JxXZ1>y8x0bA&H94PcnB!8XG(PA1y7>WoQcS|XV_xXW7C@wNalQ*e26
zGVYLYlDD}`1xSNFPRCn#MxgjG`T%pynH#n0=z3h7gg5td{%*`S`3PqE2;bWuL|457
z%UhmlV>p^P{`%3f>(O0Uy-BI1hr|EqcI@b<9~`*6a(5o!FyH!2uR-|f!t*$9#Vco}
zXFGX4nKs+~+o42d-p0_aSF1C1(;l0%Ir=W*{n}!tKUY+gFU8nXjl|*{(9b%xDUo>!
zjKChYZ5GmA?#JC6<!kJP@N^87oL64Qk7g?i(<xs}aOqz8qS{SyeI_c*-||RQ2jrPy
zGK0c4YL$e`PPLMh6HJ5eJrRXtIJRtuvhnP+VhF%k=i?8b=$_W2lU?T<b8WcxV+uyR
z*%#2C%gwwh$L7E^4*)F7sJGUL)$`=l!kuwe6=<$W$4lVB=&4#&C^Niu3#%gsF-rnm
z4Kh?j9$O!RQbYNEQUE=~6v4KW5^J~dXj&K;#&S<kwL&Vmp8$qoYoi?>WRqUd>K_oL
z>Yy7#cP_&P%HhCVj4h%MePWQV@fE0TpwMs@j=t&#7#OfvM^NAW)haMANX3h)S?A=e
zopz29oj~FO`V;{v#)=B_2G<cf?qNnPOnxs5)FUp&!kqsWta1jxbH8yN6;UyI6K4Z1
z^P1#!PY|7-;oo??4!}|ii;*%ykHUWmIac@u7w3NhQ;cW!R{DjY+aP?Rcl|k)D0z_r
z=D68HC%G4qeyb$<zkI+tthZ9ScoN^uhHfcoSuZ)hV7HJN3Sa(2DzSuJ^F*t9jLGw(
z|36*;e!Z{Jo~TGJ>kSU;iYRcq1>u#*QjlwUuS7F&j|%_~f0prp+Ip?MihR!fChgSM
zuQXUWYe|+@-A)#`&jonOM0&Iz{y#$&|2}alpS(Q*Z?^6U(F7`CIGlLP5w^4y*Gk7*
zU%=y<OzM~~6CHm$b-a37XeF^Ow;<#O-S(dn6akwFl7qTg>g$I<Ha`9#RF)9*hA*G-
zc+sTi%EwDJ2egR1=_t1ozZc}Jx6l@)*4Zt}^a85c9K1E<O3uI3#{f*Y_6@!1wV4-K
zN|yKgA^}@rHx*mmyQCiT6D@A@yxPmTM-J<SzzxcvF()?P<3IwQTy35W3#WBZXFhS;
z8RY#w#=&%mj_TF%>dO_Tt&;6dCmQHaLV<D&RI#9@Sce>L{F8t!Q{pKbrPcJBFOSw+
zG#;RJK>`;x)8m9Y6**MIb8&||9vpX5%hXO)P33oM6LncZ!BSu{Tx2nQzT{6Q2dF;#
zA`TzynBk||jbRyd2)(?+HZ6KwmRCIHKL5##P)BKm^#LX@*rTFWE6HLv;U!)Yy55(b
zyXn3atgsR&%*NIcjoCML#`F5kV~|SSE*`{trL2meYgmX%m1n7UmGvjl=;{a=v?Xxt
z#A>`FWmp-?YSOBRC%qrML?s<*K>)}ZZ>r%MXKE3L{CzJS2y&F)UrET+nm(fknSZ`U
zkUjKXaiQu5HgYpyT>ScQ)e0nqsdI`)0($!D4D;MWbb|WMLD)MI%g$^VJBy5-XH#Pg
z+Sh&Wa?&)DlNw8i{%Afo%B!7$NvB>m*;Nt9cRu&NMxgOL#)f>jXN%w&BnmvkinWT*
zvu3sD#rzMSUEONo!7?7WYWE39RJ&SQJ6(S8(&7Hlk2?4-IRnTek39k>hIOl*azR|o
z!pN0Peqt{k<hs0&c^^qyjvZE9D|oT5JyY!pKHNTFO5G-Ja3x4P-bB_kdw1fvW+k$p
zAf^DgQxbp#Z$ImQ`4?XNc_qrv=l|duJ+P0U0A4)OlRK5ELr~W1!{X&ed9n1cY$N7?
zyl|vv?R%?I%!uB0WARW%#h`nV(Z)#OChkG6nT-I9PRHw#h>^Y{{XtNd$D*7p)}_fY
zcqI?V;MulA^ae2DRID#!Kd2xqCnI)RXWvyy$622`Z6A=FJxiP0+^e?gY57Uh^{8G>
z4<=+=RYgj5eC}(Cj@mbQSsZBn2<Xhu*3?$z&n_2++P~kR$r;0oktpR-f#otliSFa|
zHBh;A@M@YhioNXS?w!_6+pbJ}=TkQ&cJ`>Oy#-y_-^1NF@^~}_5o=Maz{IAl0SYG7
zKm4#$!FI_rF_T-f#eQ8}YMmhLG!d$NHdZXDxY`|Gby7bY52sBMX%{y5o=gJjellrm
zp$c&kHaiVS@qC;!b0W`9H#`jVZ{eKHCoh)orao+Vte(9kh2_2lciS$Hj?8=H-EynQ
zb=<s>sEf##IH)j(B-_i{27Ss|_bKlJ6dy-WCrlnny?N$nX=~_DgsRQeOt_xQ_I{Bw
z$t6@je!4l4UiZ^UxOKNO!(P~R1Yy6O0bX6IR!SgSIanlitcH<_T@-qmmCH=M5mc$I
zpE*NhQ=D%#$)dzncg@N@MhbeBd2Lmaq93hdsrIyEN=NQqJgEZ-B$D<J$*$z8!G#`}
za2*t-H5uj`wUlkjpL`Lg6%NuaW>RqL5cbAZgAM4I@NYEbu7>D!-@@Bj7@Ot3{Ds-2
zXdn*p_Afa4LD!nQ-g4Y!?Hd|0cDb$PWFoblzN4O-c7rZ0NR{B=rZQl4>$<;dTv}l%
zzU|11;Vt)(ystYL8VYG@mcga9-iR{9MGIXXrKLDrJk>IMuyGbs2X8qRB;H^4eR0KJ
z5~Q}Yz)V{w5*7n-`yykU7NrIaUcoT13;(i9_cIKU^#<S~&0e(xn`G5I|G&(L^O&VO
zM$+l?nB_sr0^nI5B>p3o0Kn)g51?0xh6ULimJNjVeGQ^ZgLxdo;e%fZCDx5d!~x}Y
z;~mWj&b7oVbK}Z)ZszmvXYAQ;gD9V+yD0vJEiH(e8D;*9D*boWOp9?z24G7S_N<lT
zp4x*|N|p?pfJ-Rgd?oaLTZ9J_ygA;g?@UxTIrRNiC!&7A-F_a6{D+3K?c{Xg#m__C
z%Vq^|e89}Z$Yc0-Ea^u2U1~(kc&gUM-B~<-2ydhQ!}#N|#oLpPV>5}v7hJ7{Ya{Bg
zP=+AFqmC%;F5e=lxs9vN3al%mcwzX?Yal6mPGtl0PkQgp?0G>@nQ&@WLtgQ!YQBj|
zCGuQFm;aXZqS<fM`9)FpGKm4}a5`7On}(;6xa`2#L}(}=rU@2xNlZ+BA@{0`1Iq%w
zt$npCA7`=`vlB%4<4Vowi=8V~SIQyoqB_W_+{(M5VVVvb1);HYo+~7M?NKDa8P8i{
z&QraJ67JsAH2tPyot1Ly6Bi4NlRdO&?Put?eIBvuR&@Jd_cXZX)pI}V<Q>T2Yb<<R
zpkd+r|FQO#QBk(x-|o-|!hi@!4jmFArPKh@ARwS10@5WQAT2eB5+dD=q$oLbgGhHs
zBi-E$xi5VF^}g#_@4NP1``dg`!MX1%&hvL3$Lof=3O>2*Vy+i_mx(gVNSss`r)y>!
z1a>@^PDPQ<=PU$kDOv#JuC`98@rvN`MFyeWc0?mSDUF;dgumgUdLqf=;9kY*!g){w
z6vQMf`Mm9U(;jO5-8WF^{eU_y%Cx_8D;^;ok+W%5aaFwt^*`m*D&z*$s@uwNSG&P(
z)G{ozPWMeWc;s#3$5Pl;E>C7U4vQ!ABu(6eQ8E7W3}5RArT^|7Oyu_vvJ@7lqnM8&
zxYUq;Xu@+!STAw@FErtKeP<I5uaEW;uboZtDGn_%<Zd--z_J3oIvO)VH~~$VR`f#c
z%~K?YNc;Eg^NP(WT*hc->~0B}+s~YHrEcr`s67k+%B`*A#9MlaFL}Jz6d0+jDE_jX
zf>`iz|JNvLCzQG9|H2Z^wQVjpQm~nUI>|d^5khr#nSKaf<w~vRY@O$DU0<w+L%p(`
zL=V4*!gNc~`TpNN_npZdA;~of!Bhf*`pV~17ex$iY6UMQ3C%{upaImL2S?Tiy;KVs
z$JMXbN-fPD_6Epnc%EK(e4yCt#MKfvY#`|d%w(4%Z&9~=)H^SfHe*uJiMk%}d5Ihe
z$ZcsNlY0a9ntOiblGKG~eMhKZ!+6K0&MxxFrgo*`y;6%iS`*`dcpO$-;Jy!A4npZm
zZ)M^c8*uhNXt*-!tv$J&7@xi<H2+p-^R0^5>9D*2VcORPQGBCyv!<v0^Dwf+AUS;#
zK1h^F^PiHg*Jk}bzPjQMKnzqol7JWS6(Yk-OyO}h@v!gQE5&WAWHdthdve<cUi_~9
zHZoC%WUPVH=e8LWgrY;WYQ6!<pgY&`c3}8tXRQfe`b9$6i?5Q&!QK7za#S{Z8h_GV
z@{dON7_Z*xGCf!^Nv$+Gd@MlH<pHta@o3o-<4R-1YDXX59ewy9E*_sOc5l3_CwqB!
zx`Ml;Ur&|?`o;KUyb1Ml_=H0<p=8_jr<ir0Np+!~y!apPZ&D^h6ug_o>e_m)65iR<
z=~gG)TgkU?evJYjDI4c6=6{kSoNK7P_pW>T%;P6hl8MHjGn0T93k_$}<o!<4tBV}Z
zuA#z2m-;y6vQGAT&#@QIUG(MnoiX<@(wasWoUO9kw#bwj{y@EH`nw$8#G|Kqmbg|;
zepb5N<Gp#ows9y`!Co^~)7h)9{mmrRogiNKrQQ8dJOYVk_b>UIuBxBM_CxZ!rM13z
zHJMGJRFt(q{~vNEcJh1<1n(H)vkBJYsjeMeG5!AIUIPDeCS^Ju2xf2ZURQBTKzgXD
zUSw^~k0{CBu9tn5vlt|VbBdvO8fAdd_}H#-A&pQ0+f)930T-jcqdtv&pm@=hWBsjE
zPX$MRjL1;uB@Rjp#=IY)1{m9?Czo@ubXDnw<?qGqFy7PWA|2wqq%2s-pLn-EP;xSp
zsPDGO$&h?al^kkgnr+rV%&Pgzr4lNSRj_N1hhJBETD20FJEAFSih8KWdR_$a(*{d=
zppKNrV>ypK2C?KDmT}pzVPT7&*I$AJLACgYpT2=f8U+Z?6NSt(|Ff@%`5P|&&)wv|
zKev$mv#)d}SG}FwHYuOFA1!3ig3L18oBw3C^m)nb_jmoLmuFOo$~t8a38=Do9omOD
z9*CO8aB<b7<y|@DA65|USDdJs-6i826JBk&&S^96AU0C^=G-$8vp!XIz&$`FdlS;J
z_1croy|SEn%2!gKNV-=bdaO8P78w;hnbOeX!C$h8K5;Ba(j9Nb+r)&X!XvAh?0vPz
zGF0@ox&4^@AUgq?FNUD!qO@IaEo+N$k2~6A*g-M&@hN$|ywj2WL)WXD)H!5J_mMrw
zp!;EAMscEtcsOOEP@@s~O|SUh7+(Fko@k#Mj8^I?fr)OIl-o4qk2@XcWx($%99aG?
zEvs6GYRSAqShRZb_o8`KPsHs^+RS_lRRkrdD@*^Bx6bQH=uG`ABt1I%^_I(=NyZha
zxf{!6FB0?94~_;GW4q%5Gl;n!?^om*+Rbol8@^8Do#twPJMg_q$hsnbE6Me??h^$_
zg7{{QZ{+1x!?n53WBo+c6%VUrkBVQ`)_iiC)utjPQaK|7Bckm^t9pM%M19@2ERUzF
z9RiJYYSwZ`yWRDB*;|4h?PH~(J+<%3l>-$Am~IY<c;eb}Ys>>0L7O+_mJRXa+0ABF
zTwTq}`3dn2$2+9TZm3Xkk=dbnm{;j!%iijA9J%;a`rXDX<&xIrYVj-XLBqK#0+G;@
z>>m5ccSog@f@Ri7`(ah)TM{Kz(@*e;aHO5%q2R47Y0$CmTezm+t=P1v7-uj`&usBN
z>jaY~-*)@NX$%GR_*gbU;)#UpFk3*x?OVdWt>C0OPNLJT`6Shn*^i&Plq7ttlVvoa
z#0yi!8dL&$RoX*-em0GC$gER?d#*F%Hp%cfc<gsZZ+zB02thG{<`%)oBQcltH;+RZ
z%3ISP&;T<@$LOWRLMJQ2Y=vtg!69@^KU+^|Zm6((%UH>qEauKG<tOYkLkomCP>y}R
zysd3kUVaks`dIWp2rmmm0hA;cD`iuQ>Ed%88u)B<_yy&*>y>lex7aJ-CWTrtLKyj0
z#IqCsbVjYMaxq?K$mVV>j+D8Q&U@hBmf-7ewTKZr^kLh@6@T;iqj}K_e`s2AHALfb
z_%}A}*}mkf4;*2Hm(}+68q!dJWccvS_feR);@llJ%Z&e>&b*Wg?@$bZ6Sn$t3R5Go
zpxS$Ep6BHh!T2=t!r#06H@cUwa!_>Rh7t*3EWw@xQE=raKIxgM`lN>rXt)+nB<RL}
ze|k2#@A!3K=e!NRzxvts2lp75C^umff*>WvTEs=8%hwNgp&bpa{xH-}xOA@9#2ASx
z{I@DqA`yHsRz1^x0Y^K8P^|FE<nZSb^Ys}ng~i>1`G2O=KK~!4)ZG8M&E0ol0rWK)
zPS4%r4YTb{3Q}S{a})4V7m}3@w)UL#xg@>Oyml#@yb7=hOG5$ZvJ$L%_%~l&()7v$
z>}T?V(wbDr$pG*$H_<jm;uAKc-3^fSnm3g}<Nvib=oKq7DHUDp(110c$4T%DHIu>p
zS%Tf(Z6eal(|U{+PZ7Pp@q`c6A3~CrEKD_GzX_sfXMJn#Az(y@Ykoa&>j`%MuHCU2
z{+&I(?q94;A1HMKt!*WFp0Plbbsy!59aaSrHii$c90*zs_M55`6{e4~-25mzw0a_O
z^mQk^*7i|>Bg=y9*ONPRI1>twQkZ5tbHz@0&TUKFgh|_Qz7&C4M8RuX1WN|}B&v0V
zuczOfmF06ZPMYK`x?bfRYRi$r=vUd0F{hNCk*1$gFEl2XgL?!p&NIbU%%`qg*?IbW
zCT%RSX(A?G7o;0AW(>f#1eHO*M=G^^P`yDPoeo1jCh#3r|Kf&MTJaKzgs(DE9aUUq
z_ls|NLq=(d+gMv3LMdTbT^@<uQ@6dZw!q0&&M7gD(+~L)h(Yb?ep0(z4wSwLeRxMs
zZ0P9iJf`dS|IsAZLelKN-VcaH$LOav_vM|W<(zkr42R?ELLNi%QNH1bS$o6_i4?C1
zkM1-MA&^A@!|VFai~ev!!XhOaip_IpQ=IEren1YBixECvC25k2g&#2jIzg8=^|epZ
zrjZo#r}wu$XPI~NbNK1OG!vA3qQ7kf9!2<0m1?}gLoa`P@1DQBhawAFU{2a+xj?aT
ziPx;?^%Hp5&a1IB6F2b|R;u~)_v;y%`mBSRdOF6Sa~UipxR|P5ZY?kv?Y&qG8_j@$
zmmx@FA@diDFmS-2@z-5`)l#M`v%nfv|BQ~14QESH|65XX?bW_#`QK%<|M{s{PND%g
zA+z6^BiC5Cy2<n^1bcMKd8w=$8Iw(nCbL!HUZWS4>fRDhV|2)?Af|%f%6xh>i~2A6
ze6RaHr78UYeM(04BXYP`lj#0(^O7G+%3P){76%bdzwZ9Il5!gvP4c>O-yf6gZdB%%
z_|GP3;p+LX%DU~{VYY;;UJ{2;w<9$q>EMQQV!GUn$1s}zMZM#Z>=9+W#%8$^O@a%L
zlg%<&YA>yqW_*6y8{21-=vuu3I^`3C<W5&9ZmFM!tCq}$CG8vDp2wq|6Y_U=<k-e~
zS-th1bH+HueYH1)cMwZ#e$J4!ZjIu%Yw!`2`go=E^82J|5z1m&&TP}aPe?*_?axn3
zfp0b8Liv5K&CWyE5nsf45qBSrS(EEcmhxD`Fzamij4!(0{#ejBapsyH5!p(3q)-bF
z!@_)c(c;n_H9m9tonpf!=}#YjskU|6Xihhei|DnPtedc0%A`2DIJNgBM&Ld?&14Ek
z11R8jm^+((U02C%npZfy31=%=^iSQ45sB<RRj{)Cn(zhQwpmiwqd1t^{p3RCL|n^k
zE}0AQ*xS=(OLO_6=;d|XG&ion@DVF)d0E~pu}KQXB{0)^w2{Dfwr$h?G&8?a@zxmj
zxF&7lX)(pg+_0<Guo=}i66jP*8J|su%rQNZD3CUOx7cle3wiSJXr;uYAO}PxrW{E$
zY%>N^{cox-xi&8I3wiC2=OGDJs_Ry+I})RlHp63gIKAqMi+V0#$uE&*8p8S@h(LI%
zki+N&-^M{MVOT)sE*FYpe@AP$_0FBg>siZk1aNitmg>a4hzeLCZG*j%z4#)+N!Mu>
z)hT^-tk&&Q^~;hq(Pyoe5o))I3W|+C*zv56!kwH-c3Hd%dDLf?G$-aFJLB0|>rKw3
zZ&^?@8}CRFi<vg}{_)A5obKNk?-S3Q6uq(cQtGb)wUE`LF_hL8!vQyKD3kwcL)~3e
zLPNJ$9+S8<fzDB~2{j5x{6uT?oo&V-mtYstae6<PbGHBiUsLnXWYuXuU``QqI$Oek
zi%tw?5Tb0>zW-{lPJ5tVPh;Jmt-lZr_ayQ-qrgGnEIL^vl@0*YarkYgL;AJc<oV^@
z&tE+NdCran&+<-G7!0z<PkbgoF;2&ZkfA#Qnscw2?8OUPb4m=)_Q4S>2A$Hcm7H)a
zv#)*siE|eE51g~?-q%nm8<R!3gF)S2ZBWlu8<Wrwu|kx|@J~}jtNL9C|7YpFsJ{%1
z7Kh%S^8Y^g{#gwDj0<q;P$H2>n=#5x8^0epZH$x!<(GBaHslM6UldK{T^N2s**JBi
zrtYsie0e_`*BZIULY6et0YRWz%&g<h`8J9?qoz3x@}K%{C6yPA>yK0zWEF0Xs9JxG
z;wlz+p;M!*IXa>vltwsirFn(q)!|dAzmh%Ldg9J1rTa!_cl<*Ze)k3QK~MS;vyS?b
zi7OX}z;tCw&-bBjn~i4l2NL8E;&CF0;1I4yAbV<Sognair~D+|)~2USn}KY?#tQlM
z=C(5FM@^HP<N_ir0i~jE3b*hatfuqA*QSv1lbYu7Sn);5FPuWyC+|V9!QQu=?HR=K
zHZi-Sq*A4%VOvMQX3=P%)(|6+*u)bjWIiFj#;)U4HNJ_+!F*()hPzaWUYn+KVlDRa
z0VsBB&3M>WT<eIOyiAV{1vR}4uao#TS%dI_LN9|9&z%yFns2=Bpzwp$tO21a$Dhnz
zFeChR(2-|*d$#N;Xbzpt?52@08xIjWQwfkwxT<1k-oDE~hqxVnQ@N|9#RCVZMp^q1
zu~S~0Wr!}u#>EBCT|!6-59F+IK8Wcus=b+<<%4kfV-NV{)ZniiLe&1DA&6G0vb*U7
zpM?YRRg*LwN8)!zDRIGB$$GJ~$E(fa#-#P=a4=gWprr^>G+!|~Vm!c!=kx7ylSJo{
zMO8YhB>}wK@)&9g<W4j7ENO58Y6=#1w>37N>VV9g#xoyD9Cu_^BpsmmM$+h~3NQFR
zH-gTTuEi3ZR?E*0JeZ6q<7|_IMI>#WzH4uc%~M2aHP~WYU$kWE_5ENS=Sr&d)A)b+
zNVa&fG@t(S)G9LAb$)e^!ziDgdPp{gHRrHN2LJ0~8|lFfb6We5;a=H+f&CcNgE$c-
zCS1Aw84-jMkz19K5dQHA-2Dw%pc2VG?Q`%!vk0jmPPT#a_9HO*n}Xv6ya{=rKM8Rk
zV?ghK43Hsx5@Ixl<}@OQ+s!&~5tjGi@E*Pg&#lz}ep$s+>eZH8^{pElK{<ECJdC6S
z=Q^Ms!K5iB&pQciKsNo{3G{8w2;cD*$^IWPurH9M-=skE+C~MJLeFvjqKc1uqWziw
z^Rol_qo~6UKaas|mB5tEiSJe%83dPuS<-+5i!aH1eI=SHr!VGST0&tV>wlYH@Qw`d
zBLI+qjK`hkehwNmb%mXArw4p<CAZ94D~3319^7N=#mzInjt7zujYp?IB=DP4|F!3J
z1V=BfpPH5v=9|$18|N8M7pd<fKx7SKM=v6<ioSc|oJR*WR(owbj~U_&<!SS|Ompj$
z=e=vnGumQNPKX+(M2{71e^pz#33{HQN$GquMTSx(bAGwf*>fi{1IhJeaD#v9ue4{g
zJb&Y~X^IT}<!;6fTY;V#TYlSab2fZl`=$5yNWa|lLGC(ZwzoPQu**qLRuxZ=5jY2>
z8-ZyY_Uw`ue63(<YVoE8SBe81ssXoSgFcZ)_OOwQCL+YGv>vnpYz{hV-<iKy&~AVQ
zwZn+27u&>bpzHr)DsvC}EOK#4>5LM3wHj1WOi(VlfP5j>erp5qrS(Z*a;2!KM}UPA
zS;|>|M!}xE%EFWKK_>HP0Va{#>Z}TpgN>K)MN+oY()hU3VF9&Gg;7nKnF<2c^8PtX
z`v31(P#u5;CAq<Z+5;>o1+5+V3JO1?zytG@;CT0x-5^0Pk-oYrjq~TpgWCNV-%mzC
z5d5i}l=t7Mjz5!fbc5ojzO@GKLjD}_<K;Nxr$J;~Q=I1AafT`0S13yOLIPi%6KZM;
zpybVB^_=Zl9hN$y&3*+CW4Rllc_-QZycM{xAo8>Ss}#vOdI=M{_t{YD(_~>M>RGm5
zKbW|`#!&xL;aEJyeEZM&@1eB8Ouk5B{$z!t&h4A)F8^Yx*5w=QacdG$13<6>JjS=u
zh^<A!@!}xM0Sn_2nIC4vaUtVcC(duHl6(Mpf%%PO4)0A%qEot%#}RyyLo>3b1!7}A
z74$nAZ`<Kw&~+nBg~A9PLyj~);7G&>yP}~P_G>g)_v74Bg22)bFFWAnSwYkuj?9H{
z$6Q|xA|gM*{av=pVngyLtre`^yYy#8T+|;=cO6dwnf!fwJBuK{*IIvXyq&tX(KB$0
zem>f03f}Ehrovkrm<C#+9YeCOJ->HhzZ)i{j;>F+X!f<x&<aCqv_s<|+*=*>slE8Y
zo!V~k^w1FF!vpQIxxjB6A!RnxC;vgVJlYyq?*G9O{JQkX&MkaXs_`~tkpHRwnj{X5
zNoxp=m1|l9cf?I5u&3JfxH&0MVkIWIC%WxY0IxMG<ADGlxd&;PRMsv-e}SIJP_3K8
zqat>V>^Q&<WPaca;0?7OAKp_SK|@kGZH^5b^~UqHaq3p`1e?4U#6RcUmHI>|EiWFl
z={}<G?OAepJSif>i|9@g;&W&Gou?M>Fk0vCf+J7IHv*(jKipWxYjy@Q{MSbEbuwKi
zZ7O7rp8UA6pxwU5Ludceib_HwZ`2b|f?mUY;#vMV2Uw!t9s8%`B}3$MeR7V`1A5TW
zlxyoVS61AVs3g?;fkE|){u>v~w{hafCjQ*ZnZC|kd>tZfb(XKsO=Ky&zVd;^w!iOL
z50(B+txyO{!r?eRC!RoXc>iJ|bx6!*#!7J7Q3ELv>0@^DAl%}?vAVDx$}ID8HGq7?
zFepxiPZ$u;MenZSa6m%uq!$QgZ4R(PZXt)8$lCq$g~d9QWQ`AO81M}-gd7)VG&8Ec
zE<`SxiiBMEpY}`2!e5wk&eQwWx>B-wm{{;}r_S3n6i!u(H@*8bgxl`N^6QqtvpqB`
z^>%xU^!oY_+@9me`EWu}Yx{`Y^RqEbh%ek9oh23d;2LQvqlBe*mh?24+S(A7a-k9H
zeIv|CS84Eg!x=?A_TOaASkFVW3CB&hqeal~J@8UeD~4DIju~JK73zzhAZh`r9ecN(
zt76KC<!M}6ZXVy?3EduHht6hc5(c}Dhp`C-`85gb4%u;5uxs}Z(gvjJm|oZCGl{@;
zS4}T=wwhtq^a0C_;x!=7Fn?*1Pr%&-j)=GC%++X;wO3~wmP|t>6;tn&9A<rRgnhL1
zJ%4*#p;p#DSv^zr#U|)hU`<{$D>do3!}n(RTY93NY43FE^@Vu_Gog+P0hM6h>hkLb
zFV7)hV|e7BtC5ostM651umZ8jlJY0aYISL+_rG`gUk^)Ewa1f;B;3^sFhCJS@~xld
z@0jGXM&iodRa&!imHcVk4q-kU0IpmA4ed_)q=e@>{15i>1+kAW7h*Zo3XI*YnSuPw
z7td>lgOOqF7jp%jHqJIBxmBB$Qb{m~4|#|y3u|gdFKiaMx0{H(CV(!RFtX)S{nOd(
z`=_%>?T&i`ecZU1#rQ~qz-c0hF^talC6&-*diwD&JX&mhH3*Qbp!5qTLowdgs4EH*
z<e(Oi;nDCwSMAz?O#A~*)96bYV*=HKW)|cZRpXIUezV}d&yUp`c*xgBPgh6dE~?K)
z<8H8{aCv7o0XoGG`H~#Fc<SnQ{r(TA%Ga%*8L__gH9%Rg2V=Ndx8pE-u6Bu6e;@Mq
zlxKyccw4~h2~2+sRI<^JXTV^mIiXSqVcu(RdRL#t+BU#M5=ba4gpX>?Ix+BLqc08`
zUPAaeji%gIlAZCgEP*hKP`<=sAbpTVpfc-kXwp~pUEHex34$O{fgdg+sSbI{3WFp#
zF+DfU*3Ogoz)R8*uC@tDpw7|bd1JkW(+CKX_nj5Nt7v48r=I)I7xs~Z;a}!k4D@b@
zns={BA&BMqAiuE{dCn#+=;cA{T&ILXElrcIdtQ<M$Ym4mGjbNh9cr|fkaYuEws=U;
zH3$_olb`OdO3vP*bEJ1%l_FEuE1z{pTbV5BX~(`o+n1L^w&Sm)+3UoVeDYrWXdDJb
z`9mM5ApN;#X%sIs+QgZzhnZDx>En2PJHN8wd-U^}X<*a2?{H%=7aV?3&63w)@vdF%
z#J5x0>4S;Xe)~X9Bvb0(-jb00F1=&e(a~b_gMl0_v9hHZXPGO{)TEatbRF~f6chIs
zM3<#fmp_S5P=}orqam_wRBJvB{bZQ>$xegtQ@A=^2wevHN?xEVE_HuKSjh5Eidz}p
zW%!mJU4Y|e6tSdtrJ6SpY+;vA0kzBC9DL{|_XD6O6Bk+#=%{hc-x&ORciK3MCR&Ew
z?1Zx^B_RSwf(&0>(*8RIUPkVtM0?Mw{_Oa?Y=y_QQ|N}d>9C3?TigiYZyRsyF2kX)
zc)4R9oO<WZ*vBI;*YzBz5X~%Zm`^`i2_*2}@TJV}_I;_HE`F%oy0!Yj#@pkNzS+l9
zF<8wx+@E~sWXU7-r;_}i@4|YD@>dH73P)yp-#bjds>?o))G}~XH_`DB=SiqkWJy<O
zp^=Bye^|s}N%qDfhJ81dIxfm8HiS|Hw`pcXD$+aw{1Q~Gts*Yfl|n4^G{Of5<vE7o
zn6cLnt5BvAp>D@Wv((Ddd29+2L2g24fEM3F@My}wtS6z&PlFHtZ$l-)&C4PFMgGy7
zc)ugcge(q@3N5GP38$x91aA_T#JLOmc}7Yql4XymmKI$uUu`4?3r@9PoOXp~c{G{d
z7SEp-dGh;?7v->}LCi(buRl+#lFj6PC8IJLxYYnpdV1vEIA_vbJVZ7Cfpc>-0VPk;
z5c`ijFBO%8ywW*8D$3qDaBf1PCl>%SqZtrw2zJG4ZC*Y5Q#8)7e3J<v6(hG^svMA|
zijU^diZK_qy##gf+-@O*bER^U?-sQjRKLEX_ZM+N%c}rjPVHM%-)(-dW`geaevk;B
zQ@z5%ycl7-m{-cI`2oClsTvBQtdT3K8$GBmY<JHBKCbcCtXBeYg?(@4D*&vu<PE8J
zT4(UPD)YUR6%n$N%_mHkWIfC72K2p*U)cXSEHm6JbDyQz+y9d~xF~Y-#OwrUCf7mI
za;wJ*xVj(~$5VAaxi6HI_N}@c0#jAts6yWU!-^^GV}rd9{^-=AkR#~cM+%5>nt$fu
zL*i)2i&bA%ux8;>$Y)s_Kqq%w4xC!Zoh9!<IcQN&{JbO{t-i_ItKRzTxTsVeWxwC%
zqrA+_6J{cy1zuQag!gaeAL_<-Kga9a70Tb&*J3l!LZHg8e6<j(8VIa=U!g)1cyxkJ
z=+?iFGP+ut$mEdzw8&Bc^6|1fHrI#@)vK}>cRW`Q$T5BNi#~}5c;V{ZuyD#LQKKC)
zgb}Jez7cRekNr^2W5HKLtRnlW;+e&a{vH_arTOfT1Rz9D?0GaZbwjRJ`SH=UT?j~x
z4`zL#3Hsj57XqP&{YVx-2?uA2lA{rwM6i7c*|LUgWGY0mECWqvS7OgP&#=o%y;(#4
ziA%xt@t6Q%X))+Wb>G5CvX~IG+Bp5;tz!e?eo}m!sIPwSlU@nf*z^i*Qnz_s4vlv6
z?T8sC2wJ>3+ZmnN9CkA=X*$cDB-X#E*EvoWvi((MPeM5LZr_JBaeJ#Aa9baR9Q*2(
z8il)^2UYYilEBIz@l9iEth2)S+j&O%R=mvo8bqh4Yfg%@R|xC6{iv)hPA0Afby2&~
zDt2~#^k$*l?)MPvww`-2iT>Djf96&aW0YTgr-+_;pl?#(IY03&ITwA_Jazj#A;Vd`
zli#xc7i5Gu|H3UR`2d~oa*|^CYVRaaFWaQs)8#6e##>dRfv{5A{h)mB%&X<Zto!4@
za`J;2_6&{9+vQDT=Bb`jV&p^^LGeTfW@@pY`qzeX%soP10P{+pp3C%a$F;#0AMtxQ
z_(~9-doWD8#jEojm!nMbLl+>L!nr~L*;9{ks8|eBOpKQHWvSoj6P%L~TCw-{$d+C{
zCnc7e7j)UyJ(#d6jMiS2EjRB=*~S;0CGEvnxy(uR;ss7|K@YMam2?`HsR8SXY3G8C
z-&*<teCEB5spGmz!BjchG>*J}U`A%bG(2e1a-uX>=ycXMxwf@O$Xx9zBNWaHOc)_Q
z;<(Dker|%VI^-HyrxL*T9v$v8P~&3z=)pRp?3Xk)DUh4-xVHDTtviZ|7j}A+r1!lJ
z9G(+toX48>#<Jf{lv-vhCkmQfU!J(wr}^G1a$3TFP#Z?v|B-f&CFK1~dX3%OlY?%)
zzUS<88N84iFIwe><2cfEIgmBQZby5$Z>6dLW4`gEmNnI5kn?2`!i<6;#v{jom|AR_
z@9^^3an{Rm2Ud}&Xte&_<WHz_>+!)JJ!c`8DF?T{`s=InZLzHS&KSgch|u!A^CCnl
zX_msy(cn~sCh%eQfSbkQJs<|lo4e{hk2xne@xQ)U5g%}x`l0mz=c6(0gSta?et#g^
z?1IGj(|TXnyXd*B4|nJ_Ku+j-XMh@WVVm?91IY^kgx^LWs-+=FYq@&V<J6|4jn@5f
zV|w{lqix-a>X{H~p}}dKOsNj{B&!s6TW~K-KN0m@-Pi8%GzOOPB1JVf5MKzoQ7QLO
z$ZQ5v@U@nqMB7<;LN@|8K34IW!GBaOb105`Xf7-O7`?-&SgO>6`qT7-uJb~NmEM8#
z(2eN68VWu$CPGHo2`9Xw634z_@RD+taK(Catk^Bsex3<wuzqu}G$Agf`#1i^Ge*Y7
z9mh+}6Q#Wu`!%m|YGs2df495dUC0DAGb!u@u=?32V!=m_fmP+(#w;2wFQ?raJq2{_
zx9yL&;e<kaY?j94?npG4$f|LWFBazFEIr5dKE0mdz11AmORzfDjFIave2s)JOTe7j
z_JV6KCg1RDj;BQxEEeIzo}}(cB*eDTT^|m$QPb;lw{*<C`2U4t9G&v-9AmS&cLg1H
zd#J|cet+Ozps?^GpjswVb~SJVQsNr_qbtmpw`=!WsfM_dP{<a|QK_y|sdJ-5IgeV-
zd-lD+>-P3-!=>{{(PQpNn?(PaQC){%9r?=rPLAmhR^@_tGFR6$N8)CQ$CH(FN!~N4
zWe9%unz5vSg0yT@!<2hDYO7%iTgjqJQzxM-q;BdVq^4GQiPYhWm|VIs{*HqCP>KE`
z_8-wkbD^}}mP>AXZ5>4?*WjDRN`cqncXe&cx0H@0mVb(>;_K|n<3+6^P}AK0Zj`OM
zyz53oR&B4%3iFS-TN;+)6MNi;G#Yc}%tP#Elhx77_YS_N_N4j|el8&a6v8W`p?sT)
zU+{;4<AD`PX=b@dwL015^{dgxyUMwQg?sav<b3FMq66vQkhm`MfvhpFN>wNyU)z~W
zs{RshgNp7-2uwdILU42bDVZf>CChGSB4gdYB}e1*yvn#8GtF=>6wBu77Pk>G@vN%M
zH{==Vp4tmuV1`V^AhwX?PKIsQT&42|o%qANTw5zkDsx-5EA_~RYn->%$H2l=eQwAm
zp{y&oHf2+@RsMEQbf{8$7ZZ0rlq&3QP+vb-4(1V(_THt}r)}bt1sbcMG~>Yhfb#-_
z1#$ZD2duPPV9mj$BZArN=f_(rDdY$UdZTo=04XAAr#6zp#HA4pCvJPXMkM&rAJ=9s
zYS{bLuLB>x+eXT4oNw3knab~?hWUbhy~rAGTt10A_2(gGuW=jIfEOi+>yK4FQ2IIb
z_SL!_be_&OH97Q&k&9V`HLm5PZZCJGAAvX7=<H}?1>W|1C)RpK$h?<2b&2c<&DZX_
zQ+lic{0ofXPaksZOgXLgH`(=khM=uZ1c$2f+JdJl{?xX7+-%#6Rj;G9>qc70wc3WN
zDo+%!GP~OK7iaqJ5Nt;~NW9~KW&xoCa>R_`I_I^)EW%P^oK?HaKj25{<@DrwRcdzx
z{$s~{b{ww*&SjweiP!a^UC&7o*t6Zhle5v7^b5;QuD-y%SIB;ruTZplAw;CJc1OrN
zsqN`6ml_~diy`g+t)9B62JdTJQC|YMII4SlC?|Ao;u`oXyEzJLBgn&W8nz(XpHGui
z&@aEuJSe%$!^U*_c`J9vV``?#3swJhhojzR9YH4!wwA|}&zDnLJn_Pc#aAjZ3XWD%
zz3UB-d4CtBUwzLl{tXMA2ci&<cyShillQKx-<_0{M+H`U_kW>4ls&s4m>4STU4&zG
z{Zpu@6{_|D|D_F-TeM^ByHyT%>s^@k#Qoxy`GCTW6Jz{4q2)VTp7q}AV=|ic;RlmQ
z`s-4mei^y<`?lHzGZj9!Z4Tpn=hIQ`G1lK~WEf#%i2ogrEfx7GzxwnI2ktH@W`{+@
zt^iQI`-IatI?8LZwmA!VI(mqef%bqN=Bj**`bW-2(nXo!@Nn<FaMR6Qipcdh85pVy
zzX~q8vQ1^mbGsVES;i&(Uoi0YfB6nhVRA6fD7Qr5AV|dLuB3Zq_wMQWPaNXwu?2bu
z<@b$%p`d{8mf-#{M)VL446+2*XNyxW$fSpgjpYt18ZLV-ePdgG1aVil7X@hKfgZ@8
zq_gbtl8uzN_kX=Pdo;xPEhE=*q{w7$G1n!9A3${8hg$T)5oXID3ZMFu(PUuNYtQr`
z-GB7*xW6rn%`GvcaUlc)43i?$4^Mk>t(!-b3RT>Gdt6@X4p|QMnt9`|s$nOZI&<U>
z?Fj3=4F)9#_x20)@I4s{-9M>U_2X^_$u!sgsont);Rjw9Fo$c76OpU6I@L#zm(tS{
z&Mv<{qg%8WWo(MD^%gl?I`-y}4fAzP*jP{STN&R(q9*Y+7Im)K9;qp4CQYi)te+W1
z%$aZE824PX98ec^;9TqO^)DSKJ-;1CEX3B|x#M*r;u31=s%wSNe~HWLfPt@SB4{{3
z0>K<_kmjVtsBiZx>iSTxI<@oi*`d#aslKY~riC^?Bngubk4aEVk%o#8bM>D&jCJ`=
z0|uIgyu5l_E1MCh&|(<d4-Bu4sVCU}K=~Id>^<cTXei@m*aVb!O*0_YO-AhYI4_$o
zNom*!pV&P;pU1FxX)!YYc$E118;&DZJMrgKk<Ahwj`^e#iLTSCa?I{D7!iTCymT~H
zzXkpC<4%qaBD+yN7a1dn7x;@uzmXYtVHg_x0yC86M)W*bdn>xT^(0V=`i;|%-i>tW
zH(t~S4HttxU%~2-Xd!Apo4Dw}r5&(bA4Y&?Fh{wsu!5+1SI6cBVQm-0ke8Uvc?@D>
zjXoGkXg|=zi7^89*63v}Ci1(hHvSeax%ec8QW>2f>L%*@fhWljU6U6~I%H{f>tCo4
zcM!&nepAi+GA1f}GY*{V8-EfA5PmNm2w(25&^9@B3N217)dEKv_ih3-ZB8VS_h~x5
zd~h5a_v%A$2-O<f3?Mi>&TvoVjbGZR4#kkGtgxGYH|;R}?$1$uD27GlaV55l$IKGE
zuWG?#R~zqf(8-;Xv&R5T$79>brwb!m@#T~<Vg68QRw*ZWxP&3^u;0O2!^zqE%~LJu
zckgqvb8Tf^th6~V`Mg6<-%1CuHa=Ihv-v-Z8m8OYh^V92jDEjjcBGPQk#E<o<Yq9S
zqG1m=;>5GR=WSG2D4%)iMepI2|69}c&0EaV{QnxMw=47k`IIo9Wh?hzU@r5uI&?wf
zC1e+}F)js^F@$!QUa59iF<Yk*&18b~>!PcTDUO$e$YjoU5o+R|4A#vj=OEwAJ1lgM
z3X|*uke-VVO3^+c6OwlbqZz*o)3cTU?w!?v!m-n<@Q!$%B<McQ4s-;-%M9WXER5Gr
zb<qwmnBmo9&k+6OL^AL3gx`Ur^T9?RsoQ3Is7XHwP#}vB&skFMc3?A#uUh7ft)PE}
zMG!hI-mAFif_e@Mg%QJq7U_E;;pOkvhYu3iq1?Kf$Y*NZ6c%7_P8WR!-z;jr4eeI&
z+#-Rdg7drP!T0GW=`3NiuaNiDh5`;;dxcgiWgzvsi+JgCGern<SDx1-jeJSwFo}!<
zmGW3C+PVpwmAkQMv-X1xDpB7!JR!lt;CK2_wLNpqlj?n0Wvu7^Q~YV7@<inW4-hlo
ziavfFN$<Cn+)jxRAUj;E=5TO=HcE*Q?Ef}O%irgHTKfZEVuA8_sebPyelJpAkg>PA
za9B{$rQXip?Yyg^XO~a5^5oEc=#M~!sC&?`Vl-oKZJ!O=3pGfOwr8JU&D-K1C3`>R
z7P>JC;m<l#C^yC@Ej{q>pPlVY&KyqUR*;;dMpJwAw}mDl$HX+X(@o32Xj!XR;+?s@
zb{U*VopNR5S(?(v^R!FutYpZjJ0Tmre-X=l)4m_1;4!E01cZJEn{y&ngxOzJ8qY;r
zGeTITIXNH7vJw6a`=WUDqG0ph82X`vw%YeRFPY0lCBiiP>o?@Y`d3Y~ti3crj@G11
z^9RrI#}Jwjx#BoBBgt?q+wwO)r*iP??h`=32ngjWUnGuG3kunD$Z63%oQF`jS}%R_
zY$Xh*0S&8fNWeO?)#mrR;OeT>JT0bTZQb^5o4>T~YAV0~tQ<l)ZbVNL7PcH9W~0%9
zUf6&|+K989CUXUvAW>&m%}F;aSI<L2iSRyM#XkqNC-l9+1yFb&8iI+~*xU=gR=w+X
zB_JC^SAfO__Y%j)Sqb`b=c+6?WE0G59-^Xps*ES^!5(K>;kvJFyLmjgoDX`y5X(6Q
zrQrQl7D$1AWN#P#ap6RF?*KGAm&S}kotBYk{b`M;T5x#M#m>zypSVwX=Y*(ye4&nG
zHwp3_j*!LEo%HhK(D!pt66|!$?Bkw~i?U}<ew~zxnI!2~RkTGjstGS}N1EC1+(Fy^
zf|2QwCPOXJe5{C0h8|-)na8B)GQFCz(S}dQJWFv|z_6?;cJw^gTK#nKk_|xsFIJ<V
z3vO_tJ)YmOca3LD?0W-~SeQKA`R!p#SASNk&}p6R{@FHz;e-<zjF#~6^vfI^+Wz8~
zSd@DLatLC?BBkLJc{A{+?35ZfEB-I#-Lz`PMq7iwp2ZH^H#ED4D+TMc-n+I>|8~Py
zOhUvk)AnGp8Y^fV8<)fEmInxG$ZhDlTRR>%B1qkHU5@q+jOr&tfLx+4{P3H`liin3
z_x|q|0R~;avld`gs^8H+4pZ8Pid+4(Fb$MElhl%6TVv$GL*xr<E3PSEyDf&r7UT;H
zedy_6U?UP{7*|OMvd;KHlDVg#3NHpSFwU_>iP+!d7odv2j>HHRlnx|7Wo2%iD5>%u
zb`<P@#>@u#C!Fs$IRlnp?aC*H53{o&7MoMldpjc~eQ=RiFYlyAwA2ADgU~C){`u$J
z0{<8RVu@zASu}x_Ff6kArzZI6h65)%-q$tfcYYdOw~*^^B2DiO38_$8QG;7$Y{`yT
zmyIEl8*l0DONWjRgR_&@x_FDkIQ>O>#1e5;LCc*V70~<nzYR4`JzVUSU-pKbrm73C
z)8Tg!;g|Z{%$G)WwBYn}>elxsIqc{sbr&^NB>FjNjN`TAI-ugZsP$*#Le0Ib-@5L<
zZwqRW&rXLVH2D(>y1Pua#dl!MGI-ys+@2Y&%KtWRT)lGW&Ilo%lxV5w|J<dO*)yWK
zl~_?2V8I?AYUX9~2eqH9UaL#_>bX<}uZ|yN5z>!_=EW2w?IxXlyBzkRlDtUft^ro+
z%`-pxRSL(&7V<odz*QE`@#BGrRmLH?PKmbT0_@b<mdPmS4lm?a;8xu6IO|agL(E<t
zHc*4$_a;ffIKRrs-4JRXTZf|?p7e_BJjE<1vY5_RrebYdnTva9tXl_G0dtd{poXx-
z(zLCv>>uVBKO6m0ef$K)3*>?+Uo1O3EyhTa{34x1ViQe@Gb(Qesy}=+o5&!mh%Jc0
ztNK7JCx{&pg>;;hjDka?hsR_T2(H4f(Yyma45D45kg2MWVB1t-omRB^*$Y$j_owf#
zE^otEWa8l-Rb^_0w-MNMi^I7ZG<HuV9<)C7d&p~bI?5(NMK<SY2P~zdhbRaG($P+z
zga8JgGR;z-e87R9)`6L1xWNUNieQV~pq(9_ZSrMA_(5xTJx@;#A0t}($YPbu*gZ!M
zwDD&Hti=)Lr392GM!ouj=4i&4L@l0)eNf`^cUXgvLCi6QZ2E2EadjE-A0UDP^E&<N
z1~@B9RfpoXkV?qrzQW1fz5DO`82D%<m@Nh~6|%}PqViAg4NZ?SH3%yHlb#^nnq~aA
zw>b>F&H8`eX7=6N%pjx6o>cP60Mw)ec}q5%nu&NIoXaJDRK=XWX1UH{UBFGGBQib~
z-l0M+9jwKAD%xSqqn!43Ij-l`o**ONWNr4d3SFcJ=>-U4`ZdJ{j(++iL8fPS=L@4s
zJ<ZwwE|AdI9;7RlqA_|#y{-{<D-)Y~XY?bR(@*Q)#E~aHEIi?iMPr#T_kRo$KmiY!
z#%W+>ex}$~frY>pKmEjJsRX4x9`IR}k}Kbp5@!MmiLFpI;Rq>mov2ga_PXinYz&L3
zr(5{Y6*R+DwA54a%frZXxd%|6BcJSgBX)rtFuFY_3P~5*$Sav@cqODtiZxJFhBHfX
zl`8~1XgDc`^j}%RplS3hh?n(OR~x(<RBtLM-aKz8UeG^k*Y6=eP`sL^1sJD?+s=G@
z2Z34ENtZPZaoqVeaW5JQc)hDmtVb*K{I|NX<X*3?ACUJwBLCwp$xhSo@S>YI`xuY`
zH%dkP<3QsgwA}84+tARXc&J(J&Y3N(qf+VC-5XDEznJ+;y3f<H_fsJWr&As22XWK$
z>-o91<5bKQork}i)j&yufi>(_k;Csjt;CQ-XUBIdp^y8*=)WNseM6DO(d^FSV}ZVO
z5Tha<X|o(_HZYaG*9O}uS7}tO_8vaR?}g(sCKmsGuI(+k^VOgB-6YEklLBkw<`Gq*
zAY)FW97d$6MGgMqMGOD+#9Rtiqs-)L`=2yP-z4~GbotKubWO$DaBd`tg3F&Z6Q5`+
z%za5F0RjoWw^NV#JkJ~-sa}7=azT>(YEnX^mIiE@tL-{<P)f>Twd250ywUk9=uBa%
z*AvQ}64K>_5N9Gf5>xA4@~ZKAKWjM!rt_Prf;_nVhZ6z`1F{1&`g~=8LBx0#(k(<Z
zZcYe_F<K{*G&R#Zv$+4s0@$gqK}*73tDnQLh<nsn$CQ=@ecT>J$_1vFSY~ghdP0PG
z)woCNpt6_-@|v_pM6A~nfD1d;OGXqNV6DH?a9vNYqZq-}kqf{*N#Nw?QU7Gh81jV;
zBgQd8QR2Y`_K+AZ#l;F<row)>>TBwxh_^Ltu(9|<J*ktst(pVwWDQRJz5&E5Q7>7;
zQ?>CS<SXUE3Jo%seHZK0s=NQRxBn*r_~!F7J>uDuBrliqtke4-7hT=^4CMu)Ee=UE
zd%Esnc(0+i+Skzq5&kl>xbu&3#H1rcd8zGbCJb&h5`RlocLu4s-~U4<LRfEYtTl$Y
zzVFX<svzi*`dve@O*5Is`WB4dyI%D2i^kO@YQUdR)RB$8Y^^VzM_2o&A9j1%H@;`s
z*@K#l^)UI4(=$xT$Pdsa#&2OC{t89J&Pdr%CtLFV#fq;lOD{>7MRxmrze%nBIw}tI
z4yoi?^sK)y4uy|nu=#TThfyS;0keD?WElL(xK}P?(i38QWDn98=eVH#ZfSX?lj|#~
z#iRP(b?#{#uZuyhR67#<f8+O#Q<x_i5{ex@J7LZgB_fE&h4k@T4*n{7(ZL+6tInb$
zQ=GJrX@3SuaDQDp2?~h6nTpwbZzos4PYTtw<voZxy0Q3rKrlw3l-QiMev&wQCgqa)
zT(2g7uwl^p$Hzu-mCD}IKbd8bFS5yeBp^eO@5Er<#(GKJ;Y=@A2J<YcgP8<Zn!=D5
z?bV{(k`wD|ThP79%H|mVxqeBp6QA?P?LKkX$7iX@=LrkPb2Unz2TfzN8^|g)?@sS7
z5|BX-!xMyJuN@Ar69~D4P4cgohaYvfYss4BlWRc;pz@M*w~k8M+s?myO8R^uB1#T~
z;B-zqbtjaNFnd78c9mAQfwP8)a69#wMWSKYAaRRc;)ZiQX(Ve4s5`~4wx&Rzl>ya`
zmyTXb(OSdN#fLr`B&;g@Fi@p`<#tU;Z-p+dXa&f<BnUP=KN8q*CTv>)ZTJMlT00iQ
zZ9B2#(;lwYBI9n~W1J?FkI-U&Mr=cNft9D$pg8_O8sr-yL9-<a^{4ZRfOccMO2pf6
zy@M{fcn-u(J&`*S+xTWYUQU{fwY?7!g_#y;zGJ6o_4>k{P{$yv3p0`n!9ru_*GJPT
zwaM|MO7{fr%2vOWurrkEPA%a}O_930Lo$kIzzYYvm_h_8!cTDL(LCDMz8FdS-HeUy
zU1STHPNj9)C&J7<G=7i5xf<EQ^Pn$+p6AC-e8-YTcs|?gtD1)?EzT~NXIb?+3yC(&
zL&RyCjrq^2x9$zCX=3UB9V@t>PG$XXK!m#*n3!RP-R}umQ}l0$_9?%?O?TIJpz4MT
z%!8e`r||)bXha7|P6n_~*yBb*ph9b1ey4|$rA=xE(7!T{3~2oYup{UG=sWQz#JSx6
zF^Ehe^h9ryr==i7k<0!>GKlN~JVf>Cto~I$2>7h~vfj~$^Fxo!^>_n3h@9^wv3y0e
z!#xxDfe>qTB`wr9$33AkUwGff^5;A|QK{$kca10Q9R(N7k+fh5d;WDhv`3JpgGCrX
zw<gAX`$sER{f9?fe?`&?6YcMf(P1cP8UL_yqj7(I(BXRnhd{3#Hh~1QbTmZuiR_3r
zH}H85O3j_`01QdYo%*NrBS-y6uqOtH;d|sHJ14CD{-GsHL~xyrBPu0}ewFLJmK8g~
zwtKr6e9tR0F7+tqjz>jZVW~dLpZ&QX4K>8mb^xwtMT6_)l;H8mw+9-ZZl<VjQJyye
z>)4T+Web{z_4He-QSr`BG211qv_RWrCWtaMo5l-B!mEwnymCACrE6sxJ8ViJ;^CGD
z4g8P99ZIy17kE=`R77W`9d?g%zfflh^YY&`HNT`@yu%KhUo%4c!lmJwYB?|s6AKbT
zuttz2GsSX&n>8c2+$|2HiqRf31cC$w`~6nyx}%Z5UAN@5@;!wS{<rwl<v)EOiAWkH
zVd3L3V*>oGXC9M)KDh88C8vbBz9CHK51AnRr3ol0zYVWoL9_Vdx<GI+AlD%wie@pt
z14ZPFho#oBQq}F7IH28W&nC<0!l;ygoh9xfQL(nIKw6f%b*w{)lhvzQ4iR*$8bwBv
zj~~u|taJI})>bz&?}@Tr!Ao;HTg%C~Z3>D3I};Nh3Pm9V2m+)!b9kyx>j`OW54J?f
z=xg^Xo4z-{2US_m8T--cA*(luEBhgeN$?hVjj$p7){s+CIB9bS@#kKn1$teG*A)D8
z_qKC|p;yuu=iDcEJx3V3M%;Jb(IfbEyE@H$39N}nL8E)ch__G}j7(?<a*JaGrz$J>
z=B|p#>_=U#II3$>>5ueHlv)fhNNHBBgWv#w9IT7tH@W=JS)Kp<OfQX?sXXi11XXE}
zDtz)-r^A?y2hsx(=>%+*CfZp{NGGm-V>1Qe9L9yPE4Dr@&7$U{ukhthLQtF5U}%<1
zJq!QjFLvl=z%o<5e#W*S+R;UJxz7PxHB?>r5K1kI@@KtX9b%18HOZBlpVPnMH(P7F
zR8kS-H(PwiqD*Iwk&@2xtz+mIC%;2oQ8TjTi|Zku0<e8H_Cp&#;y(r2=8l0jpAZR-
z)eaEG5;;7tJ{<B3i>+57^4NTGXq<>IDYzJ^z98)<h~vyg?+ExNxYv}0XK2KzD`q8j
z1ToSi&2khJj55f$&s#J#O1U8xaqN@SAlulO#gwy6a$VH$w>PGaLVkJ%dSgyN@mT8?
z(AUs;ez+Ein?7z`($ifw`g)^$BSFw6p-WUF{L8&hcHAWtQed?1BvovXjihtUB}EhO
zHt?h@kJVipcvUF9u|fo<D)YG1+R06PJ)gYCj-fIFNw`1uvjfJnCuC9Mt6WIqE~6u#
zo4U!XCH}q)?I|O=qr>91TosGX%iyBfxP6PWiXnn=n|;X*85llkh322~=cE<S8MF&u
zi=Bk-_a=+PY9}qs3~`1bXOpFY2H;iM8`BWs`=kd?ue*;c_SkLF?px{#M7If2in$*<
z4t3pa$V8Pa;o;BpZ;qFM&RztyC|s+o9oB~(4u0&9Bq}_dZ=S@HMXrseGarfU)DD@d
zfW;ynDRi5Xza29C;VtloYqjc+nfZNE3fQE8_#m^aCGT+xy6pU!uIj*&SZMor8g9r!
zQVdBT%nMT}tgY8YU?OvcOfb^st{g$lQ-lNo10hn7p|LT67)mUS_Z?56uE3mnI1TS}
z$o8yz2!vqWSR`4F)=j#nHulA>w7n*X0)+)HnuVhck0C6I1S$b2`14a4a-qDF^~hu>
zU8ps(aPi&r#w$a@Q23d}#M6ffB=5i<nU#Y0I8Qrixxfa*xPKda^Gx8lBP`}8dGI8p
z#I--ROS33e<cFUPMI&qjIS@~;&-AJtb8BK-+roNPUhH$=r5VzK)R2Re!e{@2uKjKH
zE8T!DLw}m(mkQc89@^IqK9y*+KU4;72SCoIw}i;!-`YKQpWN(9oc{%B#M{pTIWCj5
zcw?yg9)2N21tH9w8S&{YnQ<d}CU^av#$b>+ypdv9H<1HF6l&KVFQU2pVrb`_RR^bB
zdS8-CsOZ*eMh?8pR3C1s;ScUeFra%ljBkJdS~bB0aL<Gc`@@a1Q}1ZImU75YN}GQ9
zg_G%H6e*1H0)k5@`67(pQqhDIbRpYuS-uy4=XNt$ZN_<DN_r1f%qe%+A%1&J?TvN2
z0nRx*?vr3<Rex*f#ZXT9GN<R+F8yQ_k*K`q7iBsMR1VC&)YI+V`-*M1-<keo1rl5`
z8+Ztye#9pjw`f-+$+2H(jW=xeW4_|!<JL*G5i8Zz0z*au__wXwT4D)pR)&$LN;oN&
zBYE+Y!XLZ&1c*hYZr?{wTN^2_saicDuL`>axvu45!(5psEj(I~o^3tnN9Kp{p?I{H
z@0OaiI!)r=jm=IfH&^Gj>>S7@S&?z-g?VBp+{f-e^)++PG(Hw?wVpJr38=F`F&xCY
z_k~QjJ56`pUUN#YYO6^6{PFv@cMGs<OF^4j%aM_p#H*C57PFz;`$qJZAPoPI%C^>i
z(P$)3dwJYEHF&}cRAh>ni5gp;F521@foN96%X_ir;bB}HFCY7Z)FjgrYtvECRSXEX
z{3>+kmB!qGeZ%Fk0~c9H$7V#x1=jRK<JNoE4(#Cf0GX4w6XiA=x+)3$9bC&Ss1qL|
zxWUi9yAm<JH(A;*XDRO6md;c5Z2|hk5M{y1XEJ^^%sNDV4bKwr5v}5^<0e;oZAqcR
zK`SOBh?hLxyG=M8)`GDu7T51*-JewFy@0R&sBPJ{?n@Sz^kLu)Zi(%Cj&s^`8DpiR
zvq9HamTcn}_0;R4{t0G3<PQ@g{qxt$Om$FXpf?L0LGf#(Ngyo!*L!e>YW>{gs9n*{
zDb;#M4S>N+*w8Z2e;U1ID9<wb!H^2tX*<bc$79Vl{;cuweC7!&DefvgUzq9A!5Opi
z-1RHEIlm8Ri(PCLj}dNwQH0oATFdYH=IgpW(f7rhsTq&oEr!cgcRw%cP;51qa*Ft;
zDw*<WR^;F67(@(>4mf}77+{9wlU;d4+cQwbSlDD+nOHD)$E8lk^5nT?fY-<vFf2y}
zkAn~{u41r4QU#s3HP?n(ahNYxnU4bfz4d1V8Qs>5l&%J({O^dNa(X3USEgEmYyyI7
z%^0jrj|DV!oSepE#7*XMgKNReH{JM1Wxtf5`A-Q`1>ZJ8jA0XY3bxFSXD6(3)1#6q
zl2V%G0xBIGkai_(FapG?eM<C>dLKxi{FTojMb8*GXtNx{W4(2pJW!dT!k2%o558XV
zS4*l}6Xks;58O|4LPX0?6P+egv{Y&9?1V;5K92}M2uhX>xg~DXzC1{qWF>QZDX;X(
z?qKPA38+P0Ju!6(F}FEe)?eT(*l+P|c~u^5818Q)%MHq5@2)Ky1ltm9^-0nCr}lHa
z<CO*f{HpqYXnX6hDBmq=cxVu5r5gkl5J}0QK_nHC?oMfmp`^P(>FzEm0qJI>yBnlq
zW(52m^qlvc_vd;4{4V|w6u7+Z=YH1Sd+oK?rvBCM%5|ORA&0{N6e%EFA4>eHUuQoG
z$Uy@3$l(sozpN-J8hu~j*<FYa0y_v^*og4fVk~jDFNvyrS#GM^Pi|Ih7DQlNkbOwA
zO)S?ARZoaBOe)KP)vM3$sl8nqeHUv{LVMUaCw&BPq1<|q6Detu{ff3$0}pC~Fi%zl
ze1rY*H$dqqBJfD7ul)wGUojk~1lG``RR|mXv>v(M?7nvchQN+Fs|uoQO!+YIkFNOi
z>C`|Gc(!yN_ip1bqodj@S=VCvkx#!w!0#IMGgU@ZdhAq!E>2{_6Z!HOnMjYPa%AwB
zlZ^+z#FVGAVdhkWl3P80CgV~GG^|1ZGHnasM3i3CKHWUi36nB;Jnkk>F8M6%`%Db=
z1z#ri5?v6bVLc#;FOD1-9EtzY7^m|JiLmf&X+`*r27Nxrhn&Z9L%zy$>1<R)PopLT
z5rCO_Vx+nB^L1u?u7}E+zp$wQEzS|M7tOI29o448;?K--JdtZ?3w$djZ^&`j;RnJA
zXp5TE-LPh~L2$#?rgIJVd&7=qfPfPd+8_TNn+{aBz&>AVLFB3TQST+08EKRf@C;!Y
zC|yA_C@TO1n0_ki){5)wO(9+{w{az(3Ilgt$YY&+?zzxQ%U#X}1juP-bTC|C(BhvT
zkNq)^1!GW70TC2X7SW}aZE!re(;851vU*hFBb$oOAPKAcV|*um24V$4P9F2;T9pud
z3KXdBz*A15dbQh^Z-+VK7wx-&?Pk0>Sn+CiuH5|DgozwvZYIY26`X_`1@Ks)+jxT~
z_X^UP<Oxrr2Gb%uDUQwMqFfXibNHWBydD!Zmh*eKOvCtX6_t~t-$2?K?SanZBr}!H
zean(~?}OJQz*+_$C15%uEwv{^$TJo7#7TT7ClcrPS_Wmy`^AHiFxVA4GHk3Dd>ayT
zzClE`7riBc4$ONQO*w-i!a_Y)SC;{#wz-@&qp8Y1UHq`j^J38{j!xl27O+93JPbrW
zf`$i=;#&*esXacjYUeuXzId4#_!dYjW@2~Fm&`ute3_XXPP7RgY1zMDb6v2RXL{r(
zxpcc`<2XN~C3j(Fsm-<WKHu}~QgNe<eTh?|H*x7ScgnqPNcE;7(s&65$!_#q_@Ssv
z8$F9{mJ&uIZ_<~wz9sghfh~SbKgy#R*c;w{qxAe29EO%e(2Mi*ax7Pj$Z`M4BS*j?
zH&}P!h_D7wnWhqJn2%7{r^DfDKJ9rSd<Tm*G-rpkABILt1}>8lpS4MuruJM=B(amm
zeY(04;_9?zP$e_;$o=tUd5LYS>Z>Rre16kdlkrLn>EWVRyPkS7*t3aF?o;s3OPBcs
zW1NqA)IRwt1&W7WdIgG9Np*DdKU|-dR8&=zai&-KjPgW=Mz=+bEi^g~t0)PpzKW1q
z^F_gs)?#~<Hm;4tU1v4T3(Saw5@W|z7lOP@!Znr8zeLe{;E2jS#@(aW(rEL#p%+$o
zO6AjofptO+>dT|)9Am2(BKBWxQ*-DS9nY4+b{yj9mF#riPWv$AarBj>?&jXf%1h1K
zi%o|ztJj}DA=wq12lDNjG?U`=CCYOvLsAm%beAS;$J?>bniGtxg+GktoQo0MUhEEb
z?fbO#6`}>xFFL!Nf0t=yOM?zGvAkSRbNe9<*y!7YUvVGF6w*(t00v{B3Iepl*YGo{
zXj4FmNyw(UhbZggd%YpZmPrJRDqN9q#)^jfIx60Wkn=m_YP7a&2rdD9>0Whq?oXQ)
z6sbxTU`L$8?Tk6IXQY0!OceFu_3<q2z(c;ydY_@7Bib(V_Lr2E^fa<2ycv^T+QC%q
zf*By{i^D}h#L!C&&2j$G%k)jZXgJW2>-X$u{#C=>I!V-qg`^&nAiX@^fPsvl8>i9w
zDwdj5uG36TJ(Gx`f9Y9J0(AF5(A+TPmH(Aoma-76d}&Kxh5aEG?J9ziQK3Wi-;qVv
zMQB&V2=H{?;Dx_>j&v&mX-D3voW#r!r9!q6ZF}_`71myEcYiOa01Qt^{So>ci=4r8
zNBrqclhb67NF~k=_8BcjpR!tpQ;Z5_t^|c!orxb|EZgK}M*VZ*|Hr<GkF=t}G_(hH
znJi$N^IQoUp_j}7fF<lE1sU9b`{#zS_>7da=R{0*qSI~;8yQ??1f!+UX7;;mB97~h
zRVm;jYss+wQ%3i86PwyYyH;cRi(NIxV+q!1P_6+%KQp!D>F9^sufrJHSwJ&C-{Yn3
z<Mq-3`${&{U%5A@gY3f()wBtOxyxT=Oxnz2KwDrZ-a}@NB3vTZ*M7TWO=0|Y^T50u
z{&&FZ@>3Gi*Id)L@6biZi!H1i7el6{>Y0X<jj5BL_kbpL8Z4L{_vi|er&4l>zr6cZ
zKX<>;s1k1RFyk!$)fc-pxt*`<i$oDtdBfpnOSRRsz8Irqo5rS|F9ySZ9=n}Qe*8=<
ze^!QMJz*Nhi|aE1th!)6TdY1BYU~^hv3^6>3i1N6h+!wOb1Qv4%S9V#vAX@~{fmc6
z%ZXd?eQVfml>%PPDmq0n_E8&jNKa_ApLicTjE!{x--tM8w&QNF%a#J0*ob!-;@G){
zHy<YYnG?AZ6Vpzt>!a{IEvPB$J@8_=guS<xquT?NZM!p7Imv-++3NQVwOQT8K-+s$
zkG^RFhCMb#YyLk50ZsJYR*s9CiOI!ajo!nX+bNA1y#~9iFow?$KpB-^F*ulV(5j}h
zZ7BV}xn(>z_!ve4dMHXg*BUk|17@jnEpUT{dCAg}_CHEE`5^)RdBF9BFC4Gl9*L6i
zSe}v(ZFy8z@{-CYk+s;t?uUZ{m+kCJj7$S-aCF$H4(DS)(AURy7>g9NAHz!P6CB%z
zVPJQIj80^)P@|Bj1Zax%2S;I&9)R5X8M;rFdc5s)dD`Go=g-0xn<nUJLZry8>o%Sz
zaeD}n5s0xPn;rCZw#F<4phc1#V)hMdGZHZUw6|xL4a3+d&B>`jXongg!Upu2CN3%y
zk3)zcpjfP!cfg{ap6?1F%K4F~9>1O5ODY}l%9EF_z=Qfof_Sg-o$C>jkl~PzkITHx
z3rVV*ZINI0%YG9hX29Mw*JY$Vf)g-pIlbpQsMj3{boDU<XJA>{_6k_<aVA6E4hlQ=
zX>B+X9ZK2v|0&o`SpLu_DqBAD`EQnO$z9G`%|QvjGV`rWvNQlry)@njBUajM^ZQE}
zdj?@=H~q{Re3|i_U+i%R@N6{t-V>F-{|@lytHnO<P&C&L&rqw>ldFSSt|l{PU!6Ek
zXkWbujVqWoysR6jdZ$}VQ%}SUVS}{^G&lAGTQzeSP`me_VDW_&B9Fra`a=Uicc*mD
z{{(DO4#=zhK)1Ud4~}@MDNo!w-h~?*pY;M`1FM3h+Yi?ez_JZkpp~9aiJGokx90~V
zIhg}7*-Z86nC`IOSv#>9cJ_aq&;Iin4=3_9_4@CHIM?85B&FE^D`VwQ>o>qi9{*I}
zk(I~sd(&sEWj83mRu7;FnG*2y_#Qs?4CT3fJ4vFL=gvnFA9lg`8Nz2sqPU3KT|dS+
z%Y@4)8%(mj9bh2d;Ni}8v`&AAkx-G;msqwu#vr!-Ld6Ux<LPR8d31-FByZzFiE8X9
zf!wOX!%vMZD-G4TNCS~fZ1c(Sdh{i-IZA1uklwvCCV3IhR`5-1)K9e(LUE<j_*y!I
ziHg_iIthPpTqzg^_`-Wz_t5}}eM!@GiKNJq3`J|-RfYWkJLma(<ETquS3$??L|58j
zl-^EU!O{r{eMTgU1-nihTcqOBt-0-;tj(tTnHoP|S9nDMT-~@iwcEPE*?F@PbLd-i
zQNM4xf|r~(R}|6aTJoSLfC#7tZYSMusHJL}8KakA8~`7;@tX*Lg=9`I9XL=I^=nc7
zGgn@C*E8PEQ4P<Lr;bsN8O+Ieutejy+xVi;mNBSZ+u8hkeokC|NhBC$ho7vxN9k%S
z!~ldcY^d<+33$O3k&+OB&5H*ZZ(ygat<|DCMA#<AutH4LcWw0G;#XUDG8e7I1hZej
zxCVROb5e{d#$Cq|kO)XQ<SvlTgtBM%{1`X=78kGq+ctpg6X+#zprF%x#Mavnp=Q#J
zzkZW1u)GC`DfWG2!Aj0F{&eXBSIgeZNc|J}Ll{1RQX0haXxwK%diMcx2QW!1%!v&3
zd1b#`+aCye++UzglB2<G%HiIL^WuD1%9*&G#^ZqF%f%-U88DpjGKGwJ-jRgA`XT2N
zdt9doxMy+ikp?ofQ18)nX~rxHcvhxidra*i&_r3gL|PNRKJFwCTm~ig*@@?6>)%5%
zCzK}zx!FVC#cHRLV2pLuP|AXV_Px&&ciAp}^jc()vJbf^u<_zpj1jOE?dnseOG&}o
zkl-fr3H*sg*5<^<MQsf_J+08-9BH|nv%<}c`R3WEbK=;0Bb-jglon$xQUbVl38Wmu
zOAYR~n8iJw_8_hoaN5y#zt~GS0f$g#)Z7wx+WWA6?g6d9GSs>vJ*YG3U%R!7^_x7s
zxN~#nlqx#)RCr-wus6vZVi&sQEj%b}rzRNxpSS8ipH0#rc`@wIhU<k^yjdZ8<qA{*
zH`|H~0CTJ0Em+=oxq^g>cY_*_#147}`iQXvy7on&LBx7RthqY)+LNeOA3k1{*voka
z29{7=aG)=X+>i(L0CQG@5gJW@wtzT2lF_+cU8T`6c(UUc1%3o^-LW?Z*K6jp32)@F
z^CO;0@l~)|ASai8l+|*EXZ;vdneHMDVwa&1$Iwa^ij!8nH1W#}i~cSf4SwV=<b3d2
z&F9Xz^!%su>nv=&v%C>X*Cr*Z(;q5HtiX(Yn{jtRJl;W(-q|;Fh|Maufc2{6%h)21
ztHZ62*w2EgmE}V5c&i^)LOZf;oPnYF9+LYdj`I5-kG?h?ZgPeh3r<J1BKnFBpcX;a
zsy3!X&Cs8_`<Dtg-uVD0KhGe1Jn)jsd1<27tM9>GH@)%J`ioucYin)04YYiN#kc7A
zCr?ONb@Xp<Var3=n_&=_cwfL9gKKEh=vf%zX-z;}y<2E_S<$sQ7c^DiAf9iYvTB~-
zZUReM!(jkZ9ethIW0bi@M0U@mWrR&Ih3=~&7aPSF`FMj!b_h3M)xbIafU8QqNBlk+
zsCFimn}%3L&wV$KlQxi1qaDeb4PWp6RuIZ(V)dPqke^i8$8(7RVB>zktkyR`n0P)D
zW>{<F&VE}jIYVppidU;m^=-xwtFe4vA}p0IM-0fypXQA&K<KBg<-QvAMp}G(X)Hub
zB&nU`IC}Ze2WicJ1Sq<9ISIRen*S7yw&sHNQPYxe+U_i$uc=hKM*M=gki^O&uT(a?
zoWi|bRAS;{!!$#N>PMZ));}u$!e_t85$QDFXpBrB6M%&uq2#p{E3d_85kq!N4ybbN
zmuAre!i7(+D5`yA`{+9oSqDC%sK5M-zjn!4heTeWrEV%nOza+aNXPmL<-X{0!2qjt
z4edzdbYp_fA^WJ%S-6(|WldToZ5k>_f_>hRS?_71?k7S0c;Gkp?QOn)B*R2l?N)0&
z%Xy;NiGK9D-gYfr?phKK{$Tu+{_G`Tw=aV+Y*o|mdgyVZX{PKcmVU5$OkEK@qv*?6
zRx(YH`^GEeG0gTPW9Iq&bPq~eFPlxqlm)YXfsTv8uhHS`_`tk?a+jx{M+>K4$WhV^
zJrN;U(b^qd-F(2Stp|=)b%3Cskrj>_Zh-8NIM+4%G3=yUuWce$o`i)W23c4T8WCxy
zAnq2W5R&QiUtVERS)5m_*7j{{&5C0vZ^7#gO5LMc5G{df4UV2Wv1QXXN;R@|-dR`$
z#E0qkdmn5*iQoR&?<b|rXdX43^z1s$=)e*4)}Qq8|DwZ`>D!Nt{tq4I&@&L31JKIe
zl?<t7Q8Yik1#}}5VL0O=Nx`ZxX`Y9@Jnc}1+4K)aBB0&4q{z4jyFrEj+KhtV6dU6W
zJD!*lidNO(oQd-m_K_iZBTJ>#NAHI->&wjV+KMx`CnIfCHyONQp-sRwe-HiX{|}2%
zXs_IYelJF$G&~8G@&k&_M2(m9ItY<p@KH`|-OW!!R$qRv&eHAFEU&WW`~>T{Ya+7V
z7=-X&9iKs50`?0Q`MX$u*}FXf4XZ|7Pl6s;C>Qu@7~0fqLg&z}R+{DZ%k!nyiESwg
z*tfk6ymVDzc+OUU`&+Hm(nziOL<hIk%+U4e{!@%pxM@E`?cI4-W!~ggip`-ltM19y
z&tF_y0h^~tIuu0(Mo}v3if`Gp&h<X(0AH##F=&EhQZvl!_^Zj(fJTt@ejbWAZD+lh
zwNX30@UK)ey60Mkt5IbV>Y2z5(Zj^lu;Rkjj0n1U4rD_Eq=W;uMtsEX2u2v3O2El3
zEX~>@fHD)0*ieIUNRst12JSO+GK+F-X!B84BePpXlo}j_q<qp@BP34UPJEX3;=OKd
zoxGd-E~4?aA6ZOD9XfiUDR0*H6b30klI-M7Xi4Jp*8VBGOjq3x{3m#@FgLDLFhnH|
zNHLD|>IwQUg-O4BE*icx);{vt=!^dbFMhW6wf6#BG!V_U9ls^$T>p`UHe`BzC7Hn`
zJR{(JJButfT-EI`wkPR*I-VsmDe_Fj726f0d{Vw@;7a|}f*p9pY+PL19ER;8>{@H+
z4&6YN4Ymz>DDmrDi8djzOJ}n)x?0r>1t5Qdp#C`KB&5jr7m=Y}?~HmepwVI&m4wUo
zw1-OCK~`~n>D?cwTHo7eY@G2gehuTR!lAw*=Hx;u^@CzS284+L3Yu!G-$fZRiu~y_
zjw_$VErKoi@TYXF;))50uP5$-5wAt!+>~N6l4#CLf6l*Nq)fXzA1#Kl8tjv6zyRUi
zc51iG8+{NtCv>Q~SLt%aS9t%LUif}NF5ZC;^&U2CZBQA5UGzz7ne+^%wI3Beri_hs
zN0QARK5rqgob+I<C#eTJqv;rDP3xVpgfl$O8%{uzL0K&pa=*T)gw5LDL<-X1$K=_{
z=CmPm8PgvEXmiGVs`A9Rp4U|=EoQI^c^}`yh4gdE!YJnahz@_oc=i`Zv?xA(WD_h<
zO=<rzDAngh-z$>lW3sSicJJ6c1)aRI#M4Lo5#MNnARJ2Pp9@@TR|nh5r0ome@BhI2
zYB^RY#d7Xdsazl98U4g{ZFw1JT$^!^0BxDZ@aF_M_nr#*2pzSY4FPjzT0H{uILCOy
zPQUorbE0JesWpiQ{f>ncziKL|0xfJPvUVSB9QGq5^O3eA$k~tB_YNE+PU5KIsKtE9
zUc7<UI!Hc9+PT)ajXObTW!5>w8xMUAF$~P9_YE};wnu>OiSUEZ!^i=cZ=F$r)<Ijz
z(<aros!=7-9{|GvYkkdQR&4Wg#I#`Xip@v}WIo`}`26md|KA_Q!pO+|&!d5M+vRx7
zIOuMQQ(y;};-q+Bhgk~u!_$AA;p}}3vYuvyAl(3~$Z#>;g1uha%*M0k85a&lxx!+-
z?eJQ?=fj->jM~xt!9oeA?EJ=z`$>IVO1nUobFvP@LsS-_XtnCiO4Gz@7bmzUguiO5
z=y1Nj9d=C=u7fVEd>w~^$DeWC|NUr?rEuqQp}ptq$Ea3_GGc|~KzwX^!NTf3sy6fR
zv>D~Cw)J?7H8hh(Txs1j3iX;3H%%U~G5Riz)uzWc{CrbyJx0M=?Bd{i`8H2RxNbu!
z<E94)+Ax#rxw`D>i8$x*!@zr1Lk0SUi7MIYf^QY~dm+e_=|stLQd@uu2>SLH9FyU{
z>E1pi)LRjcZ6sz@=hk0IOC@9R^dDc+&sc@`5TZ#|_0BX)M^yE|)QIAOnc!qvhL0@i
zi4Ur*2t7zx=tDza5{MD+Qce7VFReDFrlskax|k+OEnS^8dW^X3^N*&nab3+iri!YN
zQwqx&VDkCqKX2J6yUJF~B%sf;qIh9zh!RK{dmvt?f2(cv-C;APQ8Dug)ku0jfgLI(
zl1%%*|9Jcm8Tf~Xu_f$ER1DBcl;I6Kdw(Krq!`tmPAnfG`c@yl`*tW@%+a_sPT{-b
z9c011GgSnm32d;?Qt=E#sM^SZ>#&6TWTvX8=II-vHtU#3O>G!UI^R}SBed9bCx2g*
zU3-IQnwdRY^Yixs*9dIOnz)A#S~@;c%4^5vCqxEV8f2whqUCDB`-6yMgt;`aqFt;A
z`Ye&KK88#^+<rvjGv;8SoBYMPHrS!Cf`_04F(6A7V{D#~qK+WkG-ai63T4i~<SbSy
zVR0hL@~W|O#*yJ$)ZtA!Y7FP|m45kl@l<TJ!s~)%9&l%2;R9bxs9YwsUFUsl&3}M^
zj$v{01lTPO?V~0)x#>aPMfxzTVDbSwycv`!?+yl6B&wVm_@s#1)}4J1HF<AUO?84-
z@W62Y!_nu4mh&N^`td`LyhFuI*{tKpfB)>ie}x&o9kN1N<HO1Hm%!ngV~~%;2V(fN
zqpTQ~wrXNNvO`@F=*s}i@cNjtQf7|Dl;nhjkRcC)aARoPA^6r+8GGw2<$l)|sxp)q
z*9mHk$KR4ToXrWkONPz4T;Q21)Ru)6sXK)-6ct2*t=je?<uvk>M82wDcjO-snXVEt
z6}OxdFpU83IAmv#gaRDCPV(_o#gC#OwU$HO;L|<+hTyCO8qdjsL_}5!%s@uTv-^2I
zaZ(>!R!Zc$z&U{-<4#|IVk@~y2AE!Oy#Z`A8N`=Ej&JUc+pzIf&~%7FK{{2Gf$`Mp
z!-BnxATVHW1JE1_C_Uk1Mw5#IA_{LrWoR@7eDgujR*Z7ROgQy|q~6E>9MN0Rb}a6P
zwDwYLv@5{D`j*~Z-XdzR1S#j8iEft#6Me(mgLRI}ewm~B%o|*V#Wc}4@l~q-T;1On
zIdkOhrv)5YiiKkYUS@uNK-`JTjX8>#CGgw6KDTgdAAPiT-`yT#h0iX8L6jyY3Whv@
z@kLaC0OL493cR!WsOv&C?A-$DZXdxmeWd0_bhMT>#$-|iu{YdY-s{<k$XlfatRBKW
ztFq*<y;TW?2*=4@=Jy2AZWtSO6UG)&x0&q1befo#q;Jo6Q1P*IqHt55vGI*O(bK$s
zD<bIq{L#lKU7yng#TSzBBKGePZj4>k7q$s{ts7xzg78YKh-gD7-D%W5mrPIB<M%t7
zgLYLY!<fsy=zPV@9tTxHY3_SPQ}s6U+`dHSZyvX(_+j3kbBMN?ay-lJz){kD??^mS
z^vyDT?%%I7a3;QUV){W;E|tLKF>`mi>l#)Hkdyp)aQW1V2YEP|r-n`<kdajWGiSH1
z7ksmhJPJZ`gqjSA8Cb6Hx<TjF&&be-%U7_Ot<JlgV7OB6M8^TnUBv%z_s?tbudBU+
zSxStWDhg=V*)6^cNN4hMvf7F_Y*nb|_A%HYH*}e`-xu-WNClw^dPb~blnO<fKB^Gq
znCXuS-uJ;31QM)coVOY8T2M77--;BDA-cNZ332f6NoneP*#+H9dT<M!Aq*PBhIl#q
zvF|<b&gW?)9KnVYY7t%PY3vW!0v_WLjeV>&Kefu9sl`N+3Zrtsa?u6RCxbAvTiofm
zaRt6YD|;{cqeQC4f@~f9DvMen=8d$~v>SO~fx4`euGciLJ%^(Nilw~#-?{W`#3}Na
zn8_>plNZaed~Yf8W1mJx35skju|9kQpGmGpQHWzwum<cUzWCvRXXd`Ve}5;X1$tVZ
zy8@O?<9Tu<f<F&xZ1&5(lq>H6dV>0riq@uo;me22q9^{s75};Rzrp4ADT?wU1DGNR
zHD*w`v2PpxQnf^JB}do-pF86<8QY%9l~rMezNL8+<@A36wjHV^8WJ;f;@%LCW^N>b
z`AndbVS{Th9-h8z!wOn3AF{|tY_VQ6H6(97r0`W#d=?Q+Cqjs^M1@eXpF%3i=~!>j
zwY~0U7Fa0aQ=AVC2>`naAJf&XtJiZ7GpZsfzkD-JV<lSGy!(jdi!z)h#7X0Vs<y%v
zZ=hxAg`yYPT(6ORwYBBL<>-ak!bEe43dg?3D)AprM~GHbwbSCcY#M|Yg4yf`=sYQz
zDcD>V`=AOv$t>=Bmed1Th}waNO%|nl>q@r=d<U3pF1o47%xwH2-&?q&h{dLj+8^Z{
zrgA9dvTGS(avL0ma9d9F0jfh|z<{C-80Pj0sNYHkzGlo#Gm7MAf&@?IYG(nK6f?`W
zmoGd3G3CInm3t;0TbiiK_)8D3e;~!5kVlvKCq{PHmCe`IETSJKF@EN84E%cBCKK06
zWFFc%nM3to;qmwTbO%&*z7E4vKBD9K52HraDx3g^4(UirEML<%4a5O*E$EuhTycV3
zK|A;avRzN2X?VMfA6C#t=0rV?yxyQkZ$&Z!&K5;jpeHmZs>`UmIIq<Kze3ZV+TnRm
zIs7R##YbwvGoNFhwr9g}9QIA(FD|l_gtGY}xaKS&k6u5Kbo<1IEme!D$%rm0z=qmP
zoI~k@tbrP%a9caGo_ad2$QjOqfq-#NVM{iWGR(G4Jot>Jbx?tzuZc#X^6DzA;4OTE
z9De+*CyxVMoa1c4Hl~6>VP7fYj@#mNo@U}W)k0<7v0$Zxxcp%S$VSR9_9jcTZM>d<
zPx}~nt!JKVH7Lt`PjD>y?hCGULRN_AIOzY!8T~miden2>2`p}c$5KuoXq<fNEEk#j
zkB8!Z)w?;GF?%vYA0QP63Eaij{{td?$aVfcMCO)QJLIe?y_noG{&|Y`NM!iF@%v1-
zJ9@pOUqi|y!tfzBC?GzV!$SYcM{wTX{cyjklPWqLO!(7?p<<6E{oQR=%G`-j>p)fn
zm^**7BWPhmKfDWGX`wq|R=%c|DRZpWGWAN%7PajYSN7b@TRlf9nP;1(+Lc|{kNI4s
zKtyAb6NcF|+tbL#<6S2PS*L2@KHtM$+9SVvLD$?W%AnX7uf-8QNs?I`thptFJDVW!
zBE;2cCPArHEMX@^W`m@+5DY%wEO~cf66JVIZ$Vss2%yJ38NL(pk(W9G9U+S*E<m0?
z4^YrHemK0(NAm7kHwvkI-*r7NR9RS9#%rx=^A53?Q==)#R4b^Tm&XHESPSJ1YxgL)
zOov{Xuw9*J<pHNe=E}bP4-^K_q+K)IXDJMbvoI2|3UG@0FSdFG!iZnz3?9GIrpI&t
zADIckU1su?$4#jezz|$)q72e!E}6a<nPEW)q0+67XFlnz!CX7|yFB>;f<6_JMw?>n
z-iYcrxom_T`2!inb34M|81u;5oyOon;`8M9l<aAmYlTm210br*TN;@vI&}k`>hu)r
z4eTCua}A<(gr%4Ry7-pF3SQ5MyAp69%6aaHE=<==#%IKvA<m~#1+Ld`X@p<+Q|YXw
z_);6_!qy~cUQ<*s7-?8Apubk=IkyN=Sptw`?mK$3p~VS9B*IX+$g2gn+rWyt{Lla+
zxp{95m%I6}sU*#5Om?c#ar^mjXf@!P*bH1~YrsUc-t}ZH6!#I7TeIMcP0LN|)Ys*a
z%ZuzDUdPoB_2wE|n-i=HA?|Y9Gt+TBGqKfG^~Q6LwJt9j2;Z^^1^mADt21u?|9x}+
zeUC*i@Sz^B0jS0kk>ash>bP9^Nn4^;(aNURI+X4XOV3SM#Nrn3(E0})WggN4iA$5m
z`_nWHAUV7i8uED6z=uF}xR-5qYlpdl`g}yFVI+L@UU09Eju9N1EwgeTy8ZEmD8)7d
z-bwIFuJe%2o`gZYx(7;O_(5zQ^0o~)$!a@HA2S+oVdl?jH~KGSWN@&I;Ms8ze2C<z
zYi@Q(81X|E-k8(pvWtvtuZW^-VAzk(!!<sgpVuj$KH+sD@=T>jb09#GMLT{VLqo;8
zD7v?kA7Fcr)AJl=sXM@Ci0wX!w<t*(N0eeITY#A96SK80R-#Oj1Q)Z5oS!nJQ<*oR
zArA*PNoE=ZB>|xP1>Kj42C!@@P6PUJ@1CB%{2HiG*4^V*=A41_u3{1am#4eX;Pr@Q
zyFRecIzCyyTs%#tyU6X%js(}TK16?QdGt@X_}8KH6Z}5%4S(kMn~}};YjM+>7;oRV
z-DWGK!!#PznW6E1of!XqK>xxU(KJZ_$DL2>A8;4lWh_GSUJea>9U*47n5jt5>_J^*
z?M$&@2erpY$9Z^qqTsIna>^Nedn_nJt-bIj^B2Pt38m6&6NW^v5E_92>3UQoZ#sn=
z0b~tQ97O$1+V+GtSEhhvBu_K6Q-;4k3(gW~Ul%~%eJ_p>bK{L3k(yV8g;7i!aVdqU
z?(W7YsrT%J8t(NDHf5cC>Ryfn={QLrzP>BcbG+Tk{SLj0Ni_usH?#GID0EQG$cYfq
zQDwi=dlAw(l6SE(Pp_?m0T}U=lbxLl6BJr^VqiG`c-a|<)9I~RR;n;rGa@h;c%STz
z0rCMKU_(i71^7^*QS?4ev+p6}eZSN)flg51b%VGZegSW5)v|TStlRSro_v3wdot;L
z(z8mpoTDnZ5p|(gEHeGY9q`-(4eR_*zz-D{J(2>V?St_8WxC#ctb9*?{vRddz_^-d
zrG}Ek&!z(wt$OPsHjV0;B`UZ>@zp;oBp{6h-=&d=N_N#v;Gq~`kT20X_!HqZyY-B6
zD7a5W;o|zZf`sgPdq~TXvWcl&8jyF)2@vGtPic6C^?Q@R3&@k0i;Rl5r{5RWQepHP
z?NOw_9d)wl>Z14F9NF?V_v%{7N#5fk***TICooJT3=aRS#yYz`Q$OI<(yl6?ww6h$
ziT}b!)<j7?tRqSeJ7OKJX)S@Jp@3tjuUjFXrZ<iV`TmJEvOG_M-UaaYi6}ZJ3-XJU
z4ly)od*f=JPOvvLSy4JXJS3XW9A0N&XFUq!EvM;a5jnxPamS@Z-0GM(!o3Dx+7Osu
zE`HFP2mE&q792+vK17iTx!Q3~X~Rs;c5<TTJg*kbRPyCHdVb8+)ug%1^(%tm##T$I
zDutY<fXHqxfJFDxecF*!fHIl_WII1QX>OH$b8|@RC?_Xe=2c4VgIeDt<m!+|x+Dvq
z29k@+$^I_B7{3h1{U_!8uh6^U`#Wkd;Z+vfg<x^3HmE)q%XYi|Cdjg*v-($*sf)S`
z(==yq)Jly|$z6at>luN$>(Z#B*9n99ce?X#6LoUL+I@YAt`KFR)&b@bi8azzQF8n%
zQA(6khH%vkZF|oKjEy%f)?x`=!CYdgdqxsax*eWr$tv-Jix?uktpa|F+vJ)p5^cg5
zomA{u9+87&m4zfvspJlF!UXa7pm#*sXx0jv?bx~6EBH~m{NgYA#pKctD19E*<Z<I&
z_h*sU&IGG@8cy~~dr7_k=|C)m?Qws+%W?ZoKM<5gKGEBg=UmuoSZ8XuQ{rOH95vO&
z%R+(Ma_%=jxW$UK7(RjSi+*}vu%}Y)zRdG9#}R&Bn>lU@Fr>lx#~jaR%r;;R;|{$a
zX`GyZ%YnH!;O`qj{OuRen^g$rZTq=+1T4fv+{uMs5h|SFe5i8~OecaOmy<_7uh6K_
zHo)!`QA6qHdh+{qvb=Aobnw>+185Z;@aK_We?ktLd<13HY-qnd+xd$2&hf)Pf(I~(
zhy}biDnx}-wj+07##(v@W?oXGH>;S?ZE^Eef_Ah_ylr1%TqL|-d3cNddiXzyf_9Mj
zt1vlcj-@8lo}Ft0MOVl3A|*M`V4=_$_S-dn^Chu_jcl;j8b@K^UXDx>+kgz~29+it
zTmfez&Nr39bV-~65bCUS7tb~f=<ki2=;~(^=2>ecF6}nAhkcI{Gs$Wa8=&*PuH0oR
z$~QFhwxx9UCxxu@O$KYNk+CQmP2%O(dTGZ|gyGTaS*~`f+tf#1=#MuQ_I|>SNu$I!
zZ^F4R!<R6*Mxc0b59M_}FpD<COb|>1w5z?VOTwK1KQJF1j~{j9Th@Rhs8#dz>7dH|
zuiL9O;~?^LLN5bgml~YcVdG`dSiMxn*U}%C1$pbq2SZ*r0$RuYL_BZAf4>saD;K%{
zhz)<<3;sWWd;?VAUksS#&)u;OTwpjfcmtOI_Aln4ok0Buz`L2&Eq}~rRtA#S7OYfC
zAc<Oa6B&hB5X30BE=4K2Si^Qd%J|@k-%mx8bYF`PBZXL0$-VXC;FWAN9&h2CO7JT7
zqw(A;Q3?y*5kGYGnmJyj7b)8+Y=Pwr*7thZE1W#qpb?K#*Oa`dnU1R>K2{B5q0RkR
zLd>r3q!O=%ASGqfX2brTU5))1_p4~AZ&7KhXl@{4LmBK~VY|^ObB48eo<AlN%n&Gd
zp_)QWCj%K*MfnPW(p5!3qHVcp4rUw4;)1ZfF}WAjdVyo-bOc5nHG^=eIvzL-B4JSK
z*O(3u*o4=<Sg`ZE_>uB+=aHAj>+Pyu>M(lF#Y+)N<Kg;S6EshSr>0-8r^#nSq<<eV
z(^2}i%RkE2->hTspTON!7~cx84r4}pWd8^(zHb<SwCD|k=1=$@sqfaYPrJVz8!pM{
zMdJ!mSCZ4({dSJ_NPRzwOycZ&yiNlIO(83Og{ruxF%(~&>`Twro&~QHER1Wy^n;uv
zO1F|1@$$K)>mJCZu$>6?^*2eP$5u=gBX+*i6<4%ECM|J89JBP^<^n9Q9#IVL#~|Xe
zer#H6WOaW6YZwQy&&C7}=3<v3D?4r^-y87Su}Nv&<M+yWEB$yWd;5iq)?F`flls^<
zvZ+5((Ip~aMP57k7tW3+5^q)ZGrRLi0RB#{m5IfA0H~}9hN(oZwE|1#_h_O1O?L^Z
z|H2r+90YC}Zw-BR0ldEx$5nax01=SQ@yfS{L#i=m^~>w{KeGp4??x2iYpaBc?tH%O
z`dojoZ~v(5Pm%jZUHJ3=%t#+%?tCpIGA{-dp)vuv5J*z#pNKy3{EpN#@Uf!v{6qWt
z+kIp6NolX~lx#uo(LP2#_9dak^=-foH3-izd?2HH`UJ!MbVcdeGZ^NY^hyb$I7{80
zw9+U}*olaa9iBH0d2VQgGt(y_XLGG8;xyj)6r;>=Q%3h3^JEC)dlNUtWN%~y4}F!M
zv`R$9Q_Tn{w<P}>nuS5FCJ5=AU^YZm2zx4fgMq@`hba==T3nDyPUJ^pld4iBmNs|t
zTFB+%L@YiorHmpQcxs&Nl8z(c0~J^_vlfjHf<zjPr$4>p**!xxd`1rqf1EOC#mS%&
zsK%Jn_iir{Z~m^10{p0C8YjX?*z50{I8)23#d;&DSPqO;9=Zhz&8J<y|7l1qls}i0
zXJS-#(C_$4Q~mcf&aHIk{0n*SD1oQ{O8JrAV95h4F;Z}P$A2+!chzvA!5uz#f~Lh0
zzmrW?w>J%2L1AXU*7%RZX|e)H@Z+?O#a3AD=(<94GKocCX!fkPfPn^WngeQ5R}it5
zb_Uq(MD+)h?FExaHX^(+_>r`IG0oc$;gOK2txj@)ncaW=Y!k}Ykjrx{QvvULQC<PI
zM++k)rX)p+_LF^_gc!48#fM^}!p&oL>S%r0_2Yv`bxtRTvGN(yKk+FVUBA_1M>gD$
zWBEq1cJG!D0V`}hhjD&52i%$>Lc|K(V%07ZxC+M>&Q(Ry<4*BA0dNp?+}@*yKJUL%
zH{YHYh<s=J*jTUrZ!dt);eZFJ*ICN)#?Z7w97@^nyWY*($wY^d`7d+V9C-1islV%}
ze%cGk{|Ub(qQA4rntPd{+%GIAX3%Nc%?ilePO}K=jj@_toBv8<M0LdOGWt!$x8Bsw
z;i3TXtLM*>B8svrHF~>>Y!{-0--I;^Dn}CfIK!li1l~DWrB8Cv%Ni}G*if-vKzCjX
z%YYd7GSRUkX7XFtj<ZI)7w{YKhorz0uPJXaTVZrNYeIW+O04|R4eWx>mDni_VDKa{
zVUqY75c?u7&UrM5GrJ4H*d+QJD;;|7hz)$vuCU;2KHta;fKmUbMDr~JAl($<dE*PJ
z7&K*K@L3#b=iqVe{!wgH!R&?e6QiZ#3W1p&$5jSlFBpR+GjF8BoZU(u&BAM5Gpu?^
zOGgayd+9(bQh&ANVHTRS!y*~$hbvGhUhj^RqPguCsCCn07xXRz((L4`&@4hpSJ%Q#
z<GL#$#|$lP_1H`J2ejT@OZE9bH@DR0!_xHrUrER;5YU(?VtC$b1Mw^f4V5`S4wXFp
zN^U3Yr9{T|Eq%JEk1t6jH-HZ-q<!e#pk_YQu{a5B)Ll|X1BOU2+ech46t}k-94>1`
zN>r0c6$iyitl63{7CZsTi(R78fCzzs3O7_$LM6}Hyv`tOPi;N;@t`V==Qr^&!vNWy
zOl*8jYY+oS{KZM0YAWQo7tgyp&~^=D23bx(Q?k*qOsX+t5@+oCK|%7ySSmM(g9AZH
z%@@-3AgCI(Q#gU9L*X;F^_g-$xW39#16W{k8vB9HnVmC@Ykwhu$(d`Jv9U9ec`IU%
zWj#qgYpluH3dpY?eO8sueA0c$C3BM-cX5<Xw*nDxj=}M8x6D#exRz_gj`)3R>udus
z`2WFO{l3L-QINipI3p|Yw4dSsg%vPeIv|?y{BoJ%KprMQ`i`}#iT{ibBs4U-^)%Cu
z9e@5fX2(~M&Ao0zrCdFVDn-*3tr0QSeUX<V70=8yu<?yaWRLTMC2cqX-~B2j7d>rE
zNDZ83htZzeX+x~gPvJsU&86EyUjAFhTG&H{SRtU_)BPw#l-NQNT$R>p=+t>Ep$?Jf
z77G$-k0lbUI#-o5a(MDiGBNX&Bu~g|YR&GhMp~$H5UGD+)fzM=gA_I8<%?puus)ea
zQX_`DIwr7xt}l-#%ZjZ0VxQdC6q+ek?`pp)Sd$9$C0u|>x0>=-Slm=W3;GdrE=rJl
zH)E|P49&Y65v69m`4&6p{DamNgVF`-;sS%FO4Ykjsc&As%xnGKc1sZ7K>zE!?y!}R
z;WtJ5grhQj<R74FhdRY~S1$G}R^KBk2MUo$yJX5|)X8DS*^5T5OL)w*Dce5OF(P2?
zHLk~j9NNe*wCeGv>@J(iG<t^d*vk!Gq)aEUEH&HHjsVe}66!|i_5i^&(1_s3O4%Y5
z5x+3>E|a}jzc64xr{bs$?wKiJV|@0mFAIzF2}1rY$nxV@ujl}CB!)j9)BQLKP+_nr
zy>#OPB}x~hJU4xr$|r%119~n!ClPO`ox13N23qxczbsDqfi|qyeUh1mca!LK5XS|<
zq)ZT%0Y$<E2CnC?!|Fyzn@Mi|hs;%!dSI|ZY|}bQ__9*r#UeLV(nrx6{cp(4cZ1NW
z>%CE5l-e{csW9~7_U`Q-e4P|<`R=<|W`MGy^`sl$ypQwdgv)w%di@<!sj1TM=#>m|
z|MWl8gKvM*gL{D^DF59G1`Z!dX2l~AsPv|;DKr5Dpj_qV*FF$M>GcVks69r(cSSu0
zok!CM@;<XCgsv92Zbnm6bdXY`!H=b)a9@M1cT_WY%a!!%8>l(7XH?Z}lpk@-NJid0
zk7D*nN##5@X!kXqVMtc%9zg|S$5?_nMwh1+cNW|Bv*Z(hA$i3xeA<C1g|MC=Y$}&$
z*4oh$wZ~8e#*sWUIc8eY-iyfssbdx0NtpsZ%bcJ>aglT_Ql{+GuEN%9vN$|C-LsBJ
z^HTWB0zO0h>zb->7_UpWrdW4=liXA$j(q5B3_J>%v%vV-kK&T@H;wmfo?vp%$^v#E
zy3IG;yHlnm>(vS+lt+ykg3Sk&=|#(@?_1?UxVs6>Upa@z{RRQ|l$Q0sFcUxl<<37R
zYDsFYboAFZI8)#CYv5Y716aRdz@USugGWUB7?pP57?>$Q(>y_0149C5GDxp!lcBwe
zEC%5#odF6i);oPNLX;=~ozW$oQWn?dV4$WmNgD}CCUL*+GQwPRQruvmPLbp(5~GPx
zyq)1T>o%L7Zi$z0?5UU&k9`wDT-ePO$z0YwHlP+}f;(nl7Q+DzuabDaik44#*YI7Z
zm(G2-l?8+<P`Gep_Xz1Xy^|)fo&GR=<`?X!o5~8B2VJY3pXAPP7a67DCsKtK@K~be
zNI<0!!(N6d4FmdOSH4MJc4o7wkFru?j#zxR((&dsYN!P*Kf8SqS+2@S6U;5~B!RYU
zPww(M-Z}_6x4X+is!)@qi1eyHvuh__X+L$G2ucJp(ff4Mg8vI$<H`S^Yf;Pv<gcWm
zGXFQa{)kjfeg7wm&9!5@mjt_a4B8o|w_Lq<TuJlH2Zlty0E8T(c}Yz=689LRW7UXH
z1+*d#CE&{jaI<r?TsC?5iC>DSkbSj9>2-1cC$RXS`pL!^`NL5}oo@FN44`UVlr&~Y
z=Qk?oCZSr-mW+GMBzx)4{n}y7EE^eQ2U4ZA_+AUUC(5S1K}cNOSj_GXseJ9n*MltW
z{P8J)b#auqeQh!Yc!SPc-qax6oGrWvC<wTxk{?DzkPj$?Wl9(u+0yqHf?ZBIB5$+;
zPY@{VWX>OEg*rG-Js&}Zo|~vxP1H9he*S>%yR{+QG_(69vIdWdt#aLEUz-zoxOz6u
z8o^MoA)GWo`I}&PD$#fG*J0iDT=Jy<@O75P!Wn-dL#f4GHLK8U^l)42F8T+ODdJAA
z*@aMMV<7AZS01#>MECWm3t_n;!3_MEebPBRPw2&8kg1>uyR5}iq*;%Jh-BH@$=*Da
zOWj+SxnIVIy(SJd%n3XNN`XBVYAPYDt)j+2+c7oU$xf2A9@t<|BKB)r%Pa=mWlL<V
z(1!}?Lh{OCc_!Jx#1bO9lsA0nBEEqY4p!->Bp_s8V!9-!Q-%Ah*!&G|^_*3sQfFu@
zp=&HuWrineH&BK1;*bcii!D<l;{d@ZPk8~H8yCsmb2A6*=@1*nklKXvhlq1iFwM=!
z;sz#NDwn=`l%)P>K5DJsFK^CYm%!!k%2&!rk39RKZTAhW9&p!ZJS1Y0uN+qtWW&*^
zxBgt_v6A+#jU(mo0n;$W^q0KV8Ky{E@mI2$aF<==6!)vqdnwAe?>)_TQIjcG8V3Ff
zT>!XdO8?2eCNNvlC;s||DcZZ(hQ3((D*q0AgMus+F_T;}2|fVsD6XiOMkmNC$eBrz
ziE>q0DX|ti>nyh*&&@2=aEH(!Q7|fvWZiKu$=AJNRqN-Ecuu6^!&<i@6fP`O^&s~l
zKt6GtgPxtdH(uPjwuZ3Bg^=~0iK|sUw)p1z+J`AyTRyb){Dct+1ElfkTKvTiY-PjA
zqzuDzL#TS$UR+e`d1yfZBpZ>XWt$E&T4yppY@(aliIaFiL6t2~o9u@eU=Y4oCqZ)W
zj>|yk=kiT<1%b0KDp7DylL_i%Gi6QjJ>lP@yo{4{-7SjA1e^+m0kz03UxwmW_ltvX
z7&Ntj!hz`!7}Lqn@xHm(6rUD3t?~)gXEl3juKWraXIosM(evoz-nPTk9a!`SMDG6;
zx9+;TmdJm=LLqaX|F7IvUqKuw7Och=2ZX2Zgd(J-_k5YA4>|3S@qtd?VPecGay!O~
zc$ZRNIHxwsuK6myvY+-EM_;5OFJdA=52);UWURaBE!SHOO>(Fg(AT=%lyL!JO}_dy
zH(vMi9zCPYOsRM|&q*`ZNGZy}mSVQ2nm<tKqWjIiopf;9D<qPi6Nuu|F`@?xX?MLE
z{lwR;FA=np|2)5<%5ps?d1ozoLEgUw)+bG)=kSD>!U$dn_TpHTTF+E`U+s{|pzEmW
zIr*J5qXB$dT%oy>98_XuL$mt8GBIvzIAy*QPsAFB!ZiQa9h01(rmHbxrs}7_JKI$b
zES8w~G2M1s2YB^p?S6_RG9UQ1it@h_FW`DUzN=XuK&by_Czl!S?&j9=&|NXhNQ;oV
z>zr)gx?i$q-!n?T;tIq7=^^>PXb*_M{@QTAngIJ3=<6vow2-iAx9atpaAq8yCu6yi
zEh-J7s|ajl|FF{WNd@FP2hBA;rWmX*O8fLRO^RXsi^4aG=w2IYo{$}efv`w$c4J}2
z<C3$0j_y#VLLtps2dmfiVHLy*LY|;lT;4$9a4IdzD4pBc3!Z$b;5ZRBBh+Sx;%DxA
zYe9B#CK99SE_-nj7su;wbS_l&*iL47dv+e#dk(8OpL|<l)E2h<iOC#lHpnxrZIoi$
zWRZR6wU7p^S-b@1=8c;-DW@y+mdl&=in-JElmUH56q|ZwW~z5m#<Er!L+ZdIEWbC0
zqU$nzCOP{?GPHi8<I)N??TIKq0EWNnSIbWtttLp3gbD~*@WzgLADCSghF=l^QE&*E
zesPjO{VS&35JT<^Kr*WX;*!|qO~<XFsyi(Mu=wWP(MC(tZ+xB<%>;ZqrixWd7##rf
zi`DlOUlU5DIq~dK_h|139KRRc1si`Q93^)Gky7FrIn7$2xy#pK;n1)Z5N38wKXANC
z*aAYOf}p@KswIR7-uv+BWTiMzLZ3vz?$z!P%<jl3VYi9$S^`FMR@BLQ+7xMqs(QDz
zT?|!#PK-Gb(G@D^iz9qeyYro(H}AsVHlS(^Xs%yZK790m{ar1H?VUd7J3VL8-rMuK
z=UW6nIi!Q>l88sMPgJ}%K~7lUCah2(g)2}3BalA<nlc;f>9A1sb4_;7srD23^B6u6
zPq?9q-o+-1aG7Pal1o_-<&LxexK7C#W`RxpY)>FAl_jKsl}ym3^yW0Ft@eE@QeyEY
zFt1b%Y_X|N9S0T$7?S8g>>FW`61J7nHqA+AB_o;fsL3WZ%(Le*ibW&b1Km3kav~i?
zl&oNNKLXe=(zezw>YKQSLO*Z0T=u!Xlz%HJS}Zzhgo6Yvr*mSn^@eW;M&R4Cb%ID0
zTbOT&e82f&XpN=|J4&BjwFB$2qQ|!ec!+$#o`{({2OVJT%B#LN*d)(tfmM8Sfa$(5
zu;QjXMyd>;t9ufnSaq5vmb`!6IF#S{nY{9UY}Uu|iQh3lf<~k@HE#xX=U<#EJ&<u+
zk{Ck7tf`@)v4`pFhk#?!Tta|i3%y9^jWvX0Mj~<9VUUd{p+)oMQ$Q}PnAh5Jv@>0i
z5#w>V(0B=qDZbrfk=UOqi*s}OF*gbLBUwg}pG*xCX472#1U!>!w7tV<oS<TNBxKhW
z;BNBdgC3y?lHR-CD+Q!+$LYEU0E}KRq)v^I_^{9a6{g+*NDwDp)#Gx<&u%02Jz3Qx
zeU^UV-Mp4P+AZM(X7wlW!Ql*(yj@f7GJ>;2S$D+-r2-PrE80s{z1D$(AVk-I%!S|6
zv>F&WY(<V5fyoAcOQXkFZ^r%kV!Z@1;HP-Ivql+i(ktVLv-lQjp<7f9#RU?CO{ift
zv!WLq+FMS7#u2XZ9IVOe8Ihnw+b=`=<b`{=b+)W>p8gsj5G|p|af1#-&G5C^5(*5<
zxIhT@7>1;7HzbcE_AOcm@`y#c3lrFR%kx*~^qmhy<HCf9w&F1vKMk_CEx<<4v^!Az
zfziRCNWsH~jm{(M3}Dd`FWuHLHnzpDIP6l5D5{#pd(=s>qBZ7gEWBp{gZI)UTztT_
zLl;?|^%YT6zRA*=68pIIvR4YRoJIu@>v2;BwuX&78EsSH4&0}GbCSqU&|F03Pw*+K
ze?&hxV<`Wad$5K%VS{?N(D&yzJTGP2SRv5%D(@O<vIDgA#83EYVVoBEEfYS$lGo24
z14ENjN5F%u2X>@wQF)ImyZ}b%uREosVJUGrGG_#sd_XUlWn0r)XyX(TkoJ4=$OP5t
z5VglaOr+YR8=#c3o+x}>q*Zm&tv_n}mO0(kP#_5+cydy1f4^D#2e6)UDpPFBW~$h$
zRWy+w7}_h-s4=xqzUR5*D0Fi^ehIjDB-Pv;n2FRxQVB9O8Desm?ElPPFodjNZu}wf
z;!r5n{-R6TH3NRvmh(D5dLy*(8NXa=^&?4)+X|I{*a`}5EqxJ`p1!pUd=9TndxwB3
zIK(DlANsuHBM0Vp7GSg+nM%O39sH!3%<hg@d-0Oytp)Cds?vq3jJu`76AU)+Mt+6;
z);DqGQfO9El!IZIBSDS;jN}ne!gCuqSd7Wb8=a0vk%T=<Z)q1}*CP%|_r}`@>~)UU
zYb9T=NO?6#nia*aR-f6a)I|Y#D5rZq^r*fh9YQv)M$xos*{|dnyq%?48s|Fogv#i=
zn5O9oE}M8;uaBJcBEM03D-Cjj4NM<msa;GzTFCcn*bNZF&H}G_40ZoRr9tP0h&iz4
z&3J!Vq(&m(x@I2OEyGE-dMsD6$%LVqQWiPr^2Am974mTEm-lr*^Zl_a?kv)8o@BNU
z=ITsaIiKG=Z{KXzI(_JU0ruF5NWeQjO&c|(ur@V2bbJT%y-d@&>Z-fEG|TgjPCnAM
z9ItpVQ)fFhcMF-&D^;y|cR8aKH7m@%tdk;ltQjf#4b#q_0k7uK-A=8qz~puRA&F4K
zA;6NKt{{L_TybGvteztSLu}Q+`qMlX{cW;auhsXTKc#jod%bUko9?NFDsnL5?vcp(
zJmFo!;)b`~gUx4L;a63LjVyT_rL{~M-V|eU9|(0uCSkhqGbQGHqE&MSR>&Y%nZk}&
z`^$NYU*AI>-n(x7F3{qP`c&diT|y>a{^6fpBj38;(G32f>9<fRSP_$CUb0hS<+I2&
zh<fjAfd&lvb%VhSqEI_v)=_2#Byj|51e@^{4t@Cop*e|rogTsaL<}M)q<uk|G6nB0
zHiM!d>%`tuMSOciES|i-P$>AFV_3>d7P=Tt2g5n_HgkCEHy4@hix^wKZI~GP5H3?X
z!MX|AaXk*$l|z8NxhDew>8_KR5&i5NQV%NXxTYvU+Ar85&8SZ_JDwA4>iba9DV}X<
zGwgBVEy`OLlQe~}8*TYUt*Ppry;gLfyT-Nws^&X!>ByMmx1>2*=mz~Lz?^f!m1ae5
zPomIenTWl1!zo^jz?36@DR28~)-M3s0QA+Z2^A$aGk4f(&<{WlP%piIULWaK@x<&;
zh}(_IFc@s5R!g&p5Bq+XB|pIk!+S@BoJZ?hnqO3LSsl4(*@g!8o%CrZkfAiNnXgxP
z+6HW&MfdI5{29^?xB$Hed@H@1_*^LbSs(GQ_@In2B*Ny8618lrfW07t+D;Ti=CEc5
z#QCo3ma*-S1R;_BoqVmxu^!)#FP1yea9t|Xx1s8(!*^hXtqGuHkR9w5>^r`4xLkPC
z`;h8*)to{0H{K|6aP%uW4%nXT&;Nh4z4cp^eYoyT3nDEd-62SK4hV{XfJ(P?gVX@h
zA}t{)jihvUDcwVNcMLIf%rN`rdDi=`wf5e}KKA>=`wzezF!%TVUe|e^pR?XEd)v(g
z-~knI={dI6+5;GI(da+AqyfR{mJd`~!2b5`^b7~7{ouI`p>lrPl!MpIS{V_Ws||d4
z?*Bi2d1eDF{xSRz<jQIOB49;14ZTLY{S!{W-~B(O8`@9N$9S~q|Nl=oTn7JNCmd#3
z|0Cfbbr<9Qp9zOafq;DT!9*dGF6^8H>5RtR?1)(8=Azq-!6Ez%Qr{D~hQGxMY!rVQ
zcy1;1%SHhP<_5H|?G9XS0UB3VD(65!n+3`C+ysw=aDdBu$&`BE_aZ_4#k<mfT(r*}
zjtuy*MR0O&8^pj<y(5e>`WSR=&gvxcLJxzQ?a?dm?P9srKj;MSz|cRuccJvIaC;8`
zC^M@3!1%lbevOs3D`lL>{oM+39yszoz90FCqBaMU)?j`qWQal0C;Fh{Yvi%dnxG9`
z!!{0qM!w&xY6;o1*Cv(aHV#!za-?T+RWr{`2~dM)NOP&)-W4;9viWJ)dq1J>Dgl5s
z!fxD&($!7vj^jZdy+pq;{+$4sQ5yfNCH`M~>NLf_Gt53P!;Ikzz(eRIP%vmv$V@M$
zUa<u*qRhzL*PP(U3E?bt1rvPHU0)+wEf#SNg^z9oeu_W<R}?4eP1Rztb&=KbmVqFG
zw(o_q5p0I`Z&E7<AElqZm-X3_lb4+LNS}pmh2J?LJ{4tPm+Zf`%>K!Po5b^BD=(VI
ziAyePUDl{=_QaT~oiIL=Vo_u(H}db}PW8UOub-5{cZ;QJCxLYzf%unwj$q!}1~~_u
zC0DAMU(tk&GV)xmr&B*~0kxF8_kz4ws*8LI@FTnYVLk?Eb~M&}A15#<itK<MGa=XA
z_qLP2e{^pqld3($03MPrmZ^4CVd|oKb*h%`$-pJKGEI_YZ1KUuj^6l@YQuGNBpv$w
z?*Z#Y{)n#49sDNHx%P}Ye*c<f{~tYRGvYpFXQs+pso1EzZP=jZwEgdy&g-1u$P%3#
z3(Q{g9fygQ$YCRi6#!zp1qrvKjXa>h$u$lPo^EkE%jO-uwW%sqnys}To8IP3Nt%%?
ze-ig)ZALlunZ4cR*SUm7Z|(~!S+#P**7M!J-ed|-<iB?{OKno79v0|R3C6bSy{aqk
zfmZn+kLPi?=f#z>ESS7SO&BhI9r3JRSFzDc!Mt{s)bZwYtDp1^09Z4PZAE~3yPi6?
zuj|^r%51+;5!=gkeG~&Vb<jp{_Hf-Dr8mGY?*?*rc^f9;-LWDj#KC~gjdAs91fu9j
z5+p8a5tixW4XO0{0I&Y;)qAxGz(DMv-VZh0cir5+ANbIeTiSr8$~mBgEmozG9WByV
zQ|r>Doy&VA7Z(=A_#WrM+K-W<h~ixPzb@JY9~N!H?Ne?s()=&72L4-K#j5uquhR1w
z{eKRuNIvNjes+@;l0-P8#i{E=lFKNc79nEuL-v(J&#s1%knZ{E*iX3ry5Db_3nCz}
z5*MiH@<G!3ccD|+?e!+-Q+7=O9o=V4CwZ*x&%R1xkxGi8c6IFLkf7A)eoj+stGWK|
z3%>uPh(X(mvc-|^5PGU+lzn4`KVIyx$lmVVPlgLVHJ+iM@6M9lq@A&h`X2USc~r2@
zAF$pM)i$Ez1rfa!co>qM%t-Xcc8IYByp9?zo$b^LRMKM&tgp*QfPrQnfKa4D*-89F
zXm<R8clgii_PGN=JDYR@3Wp2I`cMPgR^;J0rqJPi4zzhIPc%{bA01=b(LD0mVi^Q1
zWVYZ_s?^Li-LN)5o&1bj&u$|>sya!^yWj9;ck8js?r2kBdc#iGbWSJMbhh<&ZJXT<
z-<{2f)@5H`$l+8aX>|JTWL;c(^GbP~{wxx48;BLK{@WnBuJnt&Io`hTTd+q8kI&ag
z59@C}BA)Mpnm($BV2roRBXbj-yADIij;q{z+DVGp&!d+QpC8P$8fCYiMB&=)o#xeT
z5ct}GkT6K8!5}z{25nStW4T5zvu&ykp#tLLFZ6Ak*_+NWB}k>kX7p>=As{`?(Jn`P
z0Z+#OEs52~AXC-0Gc%R;OU{J_h`ZRNkgg+Rpa%vl?B=C@)o7wm+>MU=!9@M6ypgv$
zeVf><pyT+WuX|s2zvO_*^>zW{;j)(IJ(+i%q*n_>=x$c^_RypyxM8PnG05d`gBw)m
z2Y1)53WolqeUx^!<j^iB32$7mJMS8suAT@6FCO&<)mdimTwINj(awYJuLx2F{WQVh
z$K4G^?Y?zeiFy=Wcu_<@PUy6H^S)CED(=^8t?dG>Uy&<L5{p~L?2a|hICbOi%S_Xh
zJU%H=p^*pZ>e*Gk<|F9iAwH+NiaoH(eVC5{e2J06a*T|}&pW{XI0y85B;>!*1*Zn8
z2fUAS+hqLI3Y3m9mjTxI79Rk|jB^79nAwU^foSsI4D4t3n0$YxKiBj|MbZLoi~wFR
zdeh_W5w%AED(5Lsi8w#VZRtDhy!ezC&~q|;0Q#7a$H}Jy0G#H$BC?D*L^pXN{5Dm{
zK9&j?n4?+&NsWU&(ea^mc*BbPC~13WMKIInGWlUIxs^V1LR3BswA%9OyuO5qv598K
z8AwzGrAu5dL?6d-88k6jguWrnb@6e810IN4hovTFUqW&dD%04&V`;VLsQFyS(w$c$
zWAe1=M-Isc%g#^60cW7jk)Rqo8s5c?Xb%MITP8YOB4(3e)UIJIxCg=!3c>~zUwaNh
z*bz_}Bj^>Ls6G`iMgDs${@-R<egootKz`ZEFAYdJa50X0!k<A{cEe%8b&5^U&~S|+
zekS9QZ(b6DGrMQ|1VrciygbA#{I81dx3sQboiul+CxMk^fJDAN`r^$+Sj1T`__m5f
z$zZA%$X*$5m^fS$cLo88&zgVJSFlLE?kqK*Qw=>f-Nzqj00@=!*rq%;U?<D6s+8fP
zPbBjXWkLDMrpb+w@#Y{XSh9@!Q)mJWZ9v)WW>-ie$628@3-`rXA>I<k=Y7Va@|zFx
zwSCc}10z4Z#|!hJ-}}Sf%eq-JrwF^9Y|VdK9UXH|7&f?B*{-cbk7TJ4oBs%Xr3P(L
zYP@^m^3UD2@OI=RD={P;fI5CY&C0ephL=cOU-zr}nou!nvIo3$xm`7ydq(N-!CWyE
zy8yY7wB_wPc_j}%^*74h3~}xu4V(e_E;)GY-#Uv&*o``&Q+~P53+n|Q+A2TEJJq}P
z9B$!0^0^K!0p#)J+2LTaQGR}>pYyj`n?z6SNw@T}H&UC$b4GIoY9QrX7?}zE+1tzI
zFUQ|pvz-?4{Tvz)SFOmdqQ?j&e9DwqL6xR^Im(u))%eLDhZ1}-iuXW=_12_tI^Ror
zN=~FjBlTH*&fuKDwQ;>N^lKp3TY#hOa0`j#%i{#>9)7N$#mY)t5OFhka``5e2a9#p
zh>KYc=ftl=r%8{1!t`exjgWTHRqunTMy(RAnUDszn|35bD&y2i6d?l=>$_ZVK*|Zf
zjIpyAxfAPW=97jF*<M2ZGRz5W+PFI)2@i2VrtqTa_y`^pn8}LIl`?G;z@KM30d94h
ziwVJxdcIuWM*G1T`==^Rd-bg!+I}yAviJg_BLa-6NTGSQDRXy^&PRLse5dkter+2-
z$s}toDO|Lv&~~Owfz51Q{~L9vJfWp&0Ml5#<BD82murlM-0v{NE+kbL0syS*>}r9;
z(ID27=P^USzVd$TlE<^Ir|RREh_Re`jzjRrv1yM*9Kgww5J#S`_BlL<If$%6Dqpr>
zv6EoGrt;Jw?{BDTXe!zXRXKc#y|(v&7A*iEh5$VKZJ*^TW+Fqjj6bTQA**-$t|e}H
z7_gNS7?yd&umB)hV|V+|eF<locmuY9eTq^W_chI+);cwyZyc1F_0oa)`nAF%XAYug
z#(|5o6|kaxAz$~~hK3ioj0!Lj7@DreM?a-BdU>fa>;zaN`mgw<=9=5pNwwI~hJi8I
zp3<&D<|8}sSD3?~@YUSg(Yk~;)IZcePu%H3uav>QwBl%j|N5}=Sw=6?{uj}%f`$Ii
z&db{pW`{hjK}(~iQP^C*(vo!6!zhgR`{b_(0qlk_nt);LA^7JQ@SnlIgP;iQo3ZS;
zRlriW^?}-Mohl|tN@_FujGf@Z6XmH3@yP}Q=%zwplq5nK?5$4Qyl;610>pLaJ}o%G
z+}9TuaZ$S1rl&m9X=8=7O;aSKzYSek6zO6jBEp;$atq9fLS4}`#B>2y;C&O(uHduh
zn}5qBz7tIqFsq2>MsJwqbeB<*H}z|~yjC?BmK6mo7LBijBQ<>B{h8Du)W5n%BGk@u
znOS*LlD7X&Kb6D+(hMWyXC9p9n|CT%#tGC|2F8)%W;89O+Tc<_rNJKExwxJ9nv8!`
z_^pc5KZjTC$X5H>Da{@SOn`!BKE{-i$f-A<mt>eCQPX%fA|Ch1EdfI3H)s@f3)1sT
zznJh3GbcG*sz4u{)S0krt)NtHr$L~N{c^90lYf_WBQrxg8}!I1zDMSxr#X~Eh#N}R
zfzQ2QS7UDzDCcrW;-}in3rwXSno1XjMT8zFpI$t@TR-Rz7_ByMA{jXsoj8Q*K@>L}
z#Z5gDa?>ZBUWerTR%wOBtGDi9rOqwu5vcSU$RehP`3UlH0ZEtd^hbpYB!-FX+s`Z3
zwoE7nNu2EcEzmi-ymOp)p(v9T7RJP6F_R24imWf>FVGDUqAqc7uvDfLR<`NENHM9j
zU23UCIF~2L6=|XhAJN8g2W+d#--|}B5nPgiym{#dT=B(84#yp}a<IN0Y6)i=I*=uM
zEI}fDxvr06rB-Rg%}Y29n`VV@ue1nSGpP}FvzWVyOe<GwO7BdJB_(1qBs`r@M`^PA
zz8sWV0*M5rJ`$jc+@;A3oGN;My`grB#e@T<UfU7L4bPDD$`_&;m5u37Y`6qw|AeDA
zoky4x`N~Sz*bG)?BvvM9k3GXeVn!>BWlb%=0a9Q77Txa_pf`Q5VHTh1G0o#04xqlz
z!Ff{=N(;$cQjnus7Zfw?`MxP79V6pRWK~g_6K0sywf5*9O4oT?8==aDhL?*Arexq^
z@mj?lW7I<en~}K&r;TxDs?UE=`)y?()7-kiQjA%j4Ico?*gNoI+)B#gD}jwrm2cGE
zHN^xQvqE2hxVCJYBjH<ppa=TP_zD1$k6qd5r}L(N)JO;?5QkPg75AO-JCV6D6*483
zCw|)Yx%`@D<6bS|xO0Gs<xgJ-(56Daf6_`!0^ZE2oAn^FijPjb1y=v>z*?SbL1Y8e
zdQb1VF@9=8AT)*$D*LY6zl=%$92Sk+-?)|=u}>eH)p|Ti8dx6%j6R)FDf-y62w+TB
zIeML~4P#~*J@F|!Mu&?=C1wiVddh}^tFvkS9M!Mxf+e`3q?H-|F-a9F#7oxy=YUZX
zh7GN<R}w$l9Ri8rHM<8(!a5cOoyesu0l|+A1$Fu0pgN%;!LYQCzh)PZ`cV1=p}LQ7
z1mZek$^Ki4^?djLO|hb+uJ1C2t8tmEQu5>jIhWBK(-M)}w-TBqy89fDU`3=m^p8xT
zDdp>x^t+Y(1lR=E2jddtBn2F<YJk|s7(OAAQ`)2TPlO|^`n_C47SKkUJMpT*mUjS^
zE#@`h+@@-T<j4yUHXHG90B_s(3$zbhJ#Xu{1z%hIeRfye*7tH#I)s*YzZkD59);1n
z!pZu3^uCpWZp8WI-#7YkpD(C9_IAYwx1}it`{t#ytZP&p4;p5sP%%IKevQ+PT>x^*
z=e+x}C0p}RedZsl357Dva+>qNi~i1969oYY5rFbzRpo^)U#jm71F~r&Cg!7wf#0bW
zn6xCF6Iu~-dKCHg)jh++9i6O3zOnmdqHB(ID#dQGPC_zRTicTL6)u;OtUe7M%l11#
z24?w8_q+itAHH!z1%SJ^U%hZ=r93uM3`cY;d)Q8NQ%;!_AN<dne8ikauaW%_&aWvL
z2r5uz=Gl!L$)xC$OrTDX!;c~sW1?#R_Ea%T;cEw3AElO*_v<gv(eg{F&N>4bhT9b{
zWbHzSlWeRBjsT%1-LJQAl%JaepEgfs^#>I!j?sQK5k1cMUDgXha5{Sbet9okurQ7w
z0Lx^G9F_9h-Njuqx6oU?*f)5TbH~15r>0JVUUvg9(wtB4UDl6S)6C#yVF4c>r=baM
zEDLP42@BO~7o?6Z@vr@9a4Lc)GY8WJLFFIw_k$9QiybEWfyTEw^>vFZj@)CKrdL)n
zfI?w!+WTGG1TFHM>O%(+9hFR16gKp`e-{uOQWga=#S@vB<t?Y-G9Satfz=cb%}>Tn
z3L!h~v0T}$kQOx5H`%7D*Nv}A?TjB${a`YSz!4Y^U~G0C%8(TQ00g6~`zfk%1fX`#
zKwW(P>{sz-_z9ifc(o<Ote3B+5nEt4nfu}==t}Kt#zqj`TE!p%vFXt59ItE#z6Idt
z808W5J&lpYSLkU83zoR>nGM<=l_gMiABqU$j=}jxdxeIEQLYy?^-AhylVsy9O~1?|
z>iDBY#u9pE8I?6Jy#88oyjy)H{PW5alxu*7r8=?{<4p8vUHdj>rlA0KRqY)f33;9M
z!9n-LADiS@h}p{=<N2JmXFAM<h0M6@VXqavQz!)*6w!=KFVVgONc@>GsOCl*dM^W5
zSdQQ0W|J}H+5v&AD)VEmXLlz)<&0KJ7d|;n%VPiisvtc~H%bRLm5%@)F!z%|GOd-!
z%<g^5l5H|-*)b>t9W4&MEbosDB)rOolh4gHj%;g7Eg@WMna{5m_nM{Sk=~^}9c1m>
zbSK*gO1)IgWO-pP8qf%fcjYPhP)c?9B0%$~jS#fWoZ~gU{izCVc09l4tZ%oqeHIX`
zoT?q6a=`NX+Na8Fq$P#3;hDMPcz%MQed^85_|3!sG?iJykM`^agN(R-ycqDtF8L5m
z@uJ}o;Iak*W}Iso-12}OxOXTQ4T_kdtHjAj%Qc_)pHDL?JiFatS*l!s6*iq}d_vyD
z&u&x0r+Z=m<f<m_JESG*F)L$adI6(%ZwVHm6&jAX9nye5KF7WfpV77Mk6caioUP&`
z0}Pfk`eQ$o8p+9C!q6V$8xsBbCO}mC6R5baMVS+D^489UlQMofXZV=r9~nR&>^%?^
zDYBKF!09@8fyu_1o#?$8V<0+u@cJu$v-6<vZ(>PUD7<E5mVkDc%&7Z!8`KGZ|Ew(M
z?$58B4kDG7VE!s0LMv_Se^Or{Dfe!m96tHt)X0AAdPmXl-NOdMVrsN%!=I)|Gf58J
zb30avs}h)pnUEKV<`o2&r%3DmzJ`~ydc#eP-pA8^YVJh`fJLl6CBso7Zr2i+US~o^
z43-jwHIWQZkL5BI5J)Z69aQ`9efz|BzN{4M3%m_6V(LL&dcPFB_K1>nqVXkT?+b;~
zO$eE*%<Gli>o>q4t`957eV@xwN&@rd5@-FF0l_&rzKC*{!-eB(-k(I^teehXe($iL
zQk`4^52s2J^$qJmo5K~;(xW)33J~aoHBV(>09&9?b~hfa`4x`9OuB1&t@W5ta!+gd
zqTZ1e=WwZgg%Ks3iT8#baNt{qE`?bH6PTEesfgnyy%dO=V-KRbn%F3HLWjwnL9PKY
zD##$Atq(*O!rGMxkPt<ge{C`}Y+@^^!=J4Eq{Sid@!ZYeq&lkv>{rY-V0B<jrW|Ml
zE)#SR>D?IhH(6`X(;fzI2WTRyCu>p~5c=*R(s)~D@X5LRX_4tn&_*;!3McT|xtH2~
zGl2p4NIJ%V@U#V98HL>1p$#83A7?;yW$3ZR3Dv9IOLpg6rZ6a|Wij4-Vm5w63Pv;5
zqM`|Bgax{R6$0pjDi->I&zO9^!42y?9|+Z{^po=OwG4Zj5Z&%Qs}Xkbl%r+a^N_J5
zxwd)&-;AYoWVY3GApS)ia1UL`PjyJSwrDS-mxiT$c#<1kRaNCGPaCRE6_7*lB9@7x
z&93cc8wbl9S%v#_ll68UCg0XVGpiF-+lyBD-AkUzjM^kw6)aIMY8XA|!X}h4gK0*z
zJ#q_#G8hr6AcO!tYWEj#&U#-z)T3wav>l1yreE*6HQ9*hmdc2@g+c!8!@si!OLs!F
zqA;ggA<NWnTyi7ZGs(|V5P&Tx0jl5id`(1Znz0e^2gSAXKAbfz|K+UlaMi1^vp<}-
zr136cUr3uXT4-r>(cIbw%hR!<`S^WFvnPzAebeS~>~|@1-AkS{6)1h&3a(X-9-<19
zxI62=gko}l3bZBpi<&hueZ|fQ42C{mEkR0}NiLE_(KrZ25TW?s+aJ`iV|uhNYdS}g
zPg*-H`as;mZz=mW+8ai3D2s27E<q-K3sxs({V!Q0u2Ne0?Ye9-#;ywBs@zCFEaNAh
zho{Cu7wH5=`f1pFB6}|Md+XCcqP3$zHxh`2xM-u8nl#J?VA1$<(d^IYIsNsNhCJ_h
zl7`|MdQLHMI{L&YtY;w5@c!JKHA(K`=F<nKO*vH`fPMt~^%|l9L+}4?cow;lr$*Tk
z=e_w}#kDBV(B(BN?f;%|fWiPG!(AW1lKTBo67k$)>*@PhqOJZ1U2!l}>TBjc|2EcH
zu8J_1q4=4}cLE<d`<AHh=6TAZ>T$e2y6f@6u5r8<)jL7lz5t`f`APO|C8+Jxk>+*@
z2)DSG+KZ2(;E!iR45z_&6MTUb_hNS!2X=p1RVUw){$o6ZuTt}T@`o8MN}PM#-*iN|
z*etg+B4E*eH9-9ai#&~V##(!;61~s#foZ$=<~kE8)6b22U%O>%dSd9bUws!6bS!0F
z5AyRG(FXr$+;1o*(R*js$1*AbyCeb6M7V4A!BWvjRHFhrK97JPLZR$)A0HFH<;9?=
z5I;-gXOHokueG!E1HtCanyA#=tY_(8e|!R*HDwMGTlq|+ei&+Z2cAY8Zu8Tx(UcKi
zF6^(9lV={gfVLgwcbE(-Lvit&Mtx@wL;Als_-t!>T_P-FeT9HnE+JF4T>w=Nk!Da^
zUG-VlhWJR)HAUoM!o|jo)^;_!WV0z0SS^R?aXBk#zzE4}_;CnQJz{i|D{P+;>=MD(
zL)2iMrcp6#p=?Jvxk^b~dsK5DUiGeaHGh(Tb&&}9!aaiGZ8xaFbl1}7FKXj~>v)R%
z_vathcGDh?k-zxPm|ag4-aQF-5B`x}X9+375t!Wct%$(LPL~;#bzN8H5g)A($6`Rk
zGs8Um7EBZSrsWJz>>Fd5$o}I(d|o=7q1%p5-)W8CCl$#2R6fwg<H$(F#!NA`051{y
zvfx8|ay9Ke&)Hcp;~0(a^_oaRDL}I5;gV$F?!&&1sn{OsM+Pv6-d-I$>Nzy#ISJ&*
zHLt}~vj_B5AufSs%ge+3Z+>N-I|FKV5_<6a>t2u$HZ?B|ck6j<B!wWmR==|t7-ik7
zRW@YkQS)=1Gk{!_PGzk*BHEkv^V+0h#XQNN1X%U<f3H$-7R^sq(=b58v)$eSmfO+#
z0~%~BqyF~h;#<-zfmA_OmPH?ZS2j~p52yk@(g8CFY=teauUTk&w|-7&y4i8ta-nD%
zIrivyzSneFFCldyLN04%)kt?^+y1cGF=@dI*(Oory{UOO-te08pnhggVeN*WWsVhs
z$8AHrg(H9jETcnro7u5t!K=%G;7a^dyLfdP7s!T?`)N|K@w4s(06=qpt&t=>@k>_k
z?$9w><uPC!UgBzBq-EcU{xX#Tl%yq7yN+^&O`Jj6$?kvc1Qnt0H`K--STFw0XuW%H
zWII`5=;25D)~MIp#^~Kfvc;hSP!FViECW(_><hbX7P{H0SJZ8x0=(AJS;@_pP>+vf
zen?>1n#+Cc+;4(5T#R&&Q}%LyC+c<Se}%Yx`2twZ0Q@++&Ha%EU@;?-Z>-qxEEw1M
zaYsnPMF(bxzcAbPM#7$p&LF(7-qT-Gl$58VJL~iq_G!@)o3e5#zshfCU2cjK*9B?U
zE6k(&@#<yH9xutA3`$V-PtNFD^~W7XM^O;>oW5qIbUGKLy|FgDe#XhJX~(c-9NVOz
z8_z<sa6<Nem%N&1Z_oA@kZs##xfKx>+hWN7(14=qbUtQ0ftMTQpu3w-Hz6NgzMZ$o
zP?z=Kn*PQsT3MLrS`-~gwb_$@+vTLNnR9z8%MDte3&_`_8~v`Wha-TJ2$icg6;WZ5
zkS-*`Y3H$QC64-07?5r$VDxb2P|#QHCx(D6)12O%o-i5_<00)wMt~#=M0O&AtBLJ~
z`&dJ35Mp{V?X~broH|9sPw0@p*`w?D6!(cC*`xL0Hv#^Ei`SEQm#6aFtO$K02hds6
z`7g*1Eovg)U2jWZGdK9TW!$qKA*S^^U2>CZp=u^E?C|RXGE@06?XvmcKFcUj6harr
z$vehOxdK>`yd0$dM9p9P&eaDmd*HkMke&b*wwF6O_p$F>j<OrU`;i4*YFsnXI|QCs
z#E*%Am@I?iR-jYkSI;M9$Zk9=oZs&oxAyd)!>aPnC^Sd>fEz>&XWcBr7-^NuV^r7?
z1+<PRPJDnAxLvHh=$-wX7Rgj+y@NkjWMldfU!I@SAjRuoEJIQ&_U(0YZy8Nq;zvR&
z&5IMU41P28sPb1HIYp))O=xdn^DfkDPY+JW0$8SdYvr=mt_0<)-k!hFHLd-^1JM>=
z>eF0GT#~2rd&a(a=k1PF#P0r%!Q-fLCpHHx_ch>KkBYQd2XLFF`$k6<(ZeM%*MTqS
z&E5SG>L3Xlv)LpKqc8iQLz}(8+e>BIl23GW2zUatlD#l}Dq%6<HL`g9xXcM%;)X;7
zqCeBEP7_QvaR0U$G=sr_zl2o{-7?fltci2wW?hRK4X+pG^!;a>Jm`8g@Y{C-jiu{a
z2$2p{J5gypa}RE>cqCHybVayvziEVs?b*Njd*#~{(SUr9CratYkd;Np3rZoCaVeMX
zUv+k{uOE~8IE*4L4;EAk%)mMNbyk3z(tP{sli^>{v^H*xlyq*X&gsJi4jp==HN<bR
z+!`>!!H>1wa0FVt-0_fvpB+A0jb$^hHO#JT8cj+9{)2e6OC=lus$#bu@)`9#qnXG&
zebtt`p;=gjecy`>Tb~=BVEE)MwHQ;O_(kd1<vb~pZI5c&0!`Tfj@ZCRiEV2cWOhMQ
z78RKemjQAlCn3nq&%*>{5IB(ty8b$fH|6b@w+s(xnA!hIhLPlvj~?y0h|gRM?<SC}
zJWYz!zCC&R`<DoOY^=v+;!9}Q*$aHhuaS#g4C6Tyq&puuyVr8kHb>#4oelG^egJt#
z91Q2e@-U9}o|?)&Biv~`x^mZ3%kS-Wz+T1o95ZTOMCD?P{`(%T88%M~%FhtjS+}+F
z+x=n|sT2|+XS&_3VqP&EpS8Z`y9Ozz*~S_gZ@q8D2M>NULBNm3`rt=f%`$Ax9TRIU
zJ*+Fx-oOt6X%{^%1VYaCmwl8EVX-1-j`{q5tse2fB0e?UZCc0k2XM)*ID3L5f`oL&
zSoTD69QsJQjapykw1E_#!B(#BO*?U~%lW{X#v!De(O2-lQa8?9vX^q?K8-g?#;h@K
zlu3ql>NBj(T?Ph-td=RR5)BZ$8Xvj_j&x*49<<g3Hm;_#SE)Qc9f}*_%fJKtS%R?n
zz)b0kW!QeUFj+Vv)9<1Yaak7eG0D4+8U(FVm<`(c%Crq&d8$(HT};P8UuJUbqld5F
z-Eu!dL~@*4pOU8k^mhhClDT42qhW9%;9do0r1W&DYps3>7RXP#2OWoFNFf|m?@0}#
zyC@&IugHyTGxT05&i7pw`Q?i7<@!&urytAAlW5+)Tlt2Zm$*&I8JL^yl7UwU&p5r%
zD18A~9l<4%fc-I{NtS1HgmA7`uA4y5u?5x>@BQAo>eDK{BnE|<2Ep|Pwff^FbJiCz
zy-C2MZwM5ynTDi$=K__SJs*6r#lKhBs&ehkxiEXOv2=&B0rj`EbNmpAxL`$O52fR5
zs$*O7t_zXX`mOZ|yO?0@9fW-#agplSAN@tJL7HnWiT2k*CKf#PHzKRGLk_QMVu5DP
zE`U~%Yy6ipH}v{#p?4pV(+}vz9F4~+YO0u<4fLaV`>aQlf{8FiF8fz)^SD{SfozYs
zm{A=HB1XqyXK5pZ5=dQt3hev(v|3i}7e>l-sUB+=b?f#uP?D3Wp#cBV20*l88GD_(
zyTnI#YxVb!Q}x{pcCepD{0kn_W*Q{10@y!u`KqZOt;fp;+p0vXEj4Z2WvvOX+lZ~^
zuXY{Ng+{0j;aj^qb|aHcd8j3c)sx+ieg?e);<0cd;GNys;ouY5;Sh%lb@Xcn9ZR#0
zZsKYDjml$6d#c}pg{Ih;F>|zMdQdYgF)fM=|GaqVXF2odd$9_Twkd+W=+OmU(XQ8P
zZ|atXl5;9pThH@{DU1-1H{pM`@18gRbpTZfEj2M^=0y3NywDSCUv~-dSn^&T2<j?5
zxAA$D^1QZbs{CKD4b_7wb?7huy*bDHD3h4s=XKPVM#iR3W>7vv;EU^{y4GO(L|P{h
zq+&B&kM9I({|A1^IO=U0e@_2D{Gs0bS|SqMFYtKrROGK4qQGdIB8Yx&Cu>=bXLfH0
z?^!sr7uAC1oF#F0xU-eaA__}_<eJB?YnxwAEz(*o&b;#7S&U@D$-Y#oD%d<-E;u~i
z4e=mxEgt4JSQ>20Vt+d(0=}!B*|wk4nRJWo11&0S9CiK?apOW?(=|$*bL-S{N<uJ}
zzw_tU@rnij!ma;a5{W@k+WKHMExr%Y22$e@McgY~2A|H}teBqlO1wufVan}4<i?+(
z%s<wO>Id8zl@tkw59<m*Xz_7YM?sF^c|MegaJ0;_Gng>+qyh#9Tbg02+M4i+F7LS?
zepN?&M#K{T4-4P`ta{r;WR$!J?)hx{9&~0JXqqkZk##ws2S-4D=NtYyAqE2z%V<~g
z>Eaczm^h&%*!m9gIoYc-$Q%lZcMFo9XjXE(LqL!*LrJCuBMqXm`Te#2fErqH*u3N$
zhI2|_E3GLUkZ*X)A&{(MXf*3qs2%VPl^pHnbnwKuFOXn6-5JOese-E*=%ngcw9ukZ
zzV>~!l1IDI2mxU0)&=Q_M;U*dWMfR7aR|me>7vwM3-yzxmY3Dx2<X|@YB=lTouvZK
ze89wOAX-3iod3Z9P5GVlIKBWt`OKgOWCjK0WgeGr(?w<g@3!B8q4;#Lx6FY56qXRZ
z47Jr?dN42WeSY0n*mzQpEC0FM87MDEjgkdo(*@4NnpX>spo95ZTdKsC#JLw?Gq!H#
zfb#l}m0*nuiMjTRFie3krjSm9l%c!Jh3aW8!vk}{aMVc?Rwf0-!r(0@d?6%73#A{C
z&XT#F>!vEcqu~4|-MJD57?r%gbP>z1Y@HznCC->fyFjUqz|p>W*EYE#Jk;Aw#3uN!
zH);Jxq<a=QV)^YIK-ukPy?FAP{sf2{KE+g|eq0toz=Ipqiz5)kpxMw4SfhGhu7rLh
zRyi%ziSy0Vg=ivD7D73M-sU%X-?&dZ9!EnL)<MDS9(#cA^m$badK*Ez->G4V__OaQ
znICpt!XGpT;HL!FO{4uob*VT}SR)PyUSX5i*+KV#VVvt>0ihJGx->FNI#&`R)>p9W
z7qp0Ldeq#2A=$3)s&=ezF8WDlELE=A4ePQLO;#lcJM)0xDz+F#>vjs8=5CGDaPp1y
zNJ{2*$&Gn$gbRK1wtUO>(<^hWjIX-?XMR+7q;#|u&G|n7sS~vafK+~JF{M9OTTO<@
zBQx?Tt@|v+^}Yv$6l;{y!FwOqE}LNnl)`@)QW#2RoevRMuYV)3waL=xXcrVhwa+LC
z>b6^bZGcp~QGlZFKM~C}OVx$1(X{o)iCV}tW#1{yFaMGnx^iQd0V&!92pND?h#ye1
z#Ha6&_lUSh<YfAvWt`LA(BGV0P&aHzn<f#PY4q-y?MT>{PuPV-@;A>c-VL-qxSOgC
zVx1;>ln=Kj929N|ErHcAGxO;A%5c=(9!z05Wcj*&32AeWB#*}k;=Z%1j;21{$YhlT
zuN%u}6m7<K2J1ppBuSF?0m8?U!i)}J%i~z0##}5Dy>|qRRKfaWBv!t)SLAhvZx&c;
zGd=Ebnh(0gIo{PC-})S8XCj2Av=jjf3OIeit+JiwY$kVlH!SCno8VrwG<C_Es*bV!
z;@ppA-+VA>uF?<0vcOE;dq4#n$M%Ak_7X4bGmS1fX}&I*z|k55Ud5>$Dx%z)Yn3(=
z_~Ot{F}#FP((7!w-a(nHR0V<Ruh;*KbbbbQb1_fWGtU4@RvM7l!r~=t1P~eafH_$K
zbf?sk`kE_Nw$9J3r&c*DY({;KNO(9^ST$;7oQounY1E8_N~F%JKQ_THP~%ajx9J06
z76>8^%t&5voL{74S-1(!TQO4de3BOBq@|F48cN$#HQ4(>o;u+cabSdGx9t4`v!==k
z-tELDaxSWNQH-_fPV$&3(4ek-)?3K^5o-fbq?zaFU)Ooho2;9TKdXK0<MZHct{WCw
zV45fy@!@`Wb^(}`^Z|eX;**c6*|2Kf53d-p4B?ErpCqyhYU??rA3bU9zhbMho2&Xf
z^ha5tJ$^-4;S~~yiJ$<_F9q^tc>tg7YefL>b177B&>me2B?PzVSt6KV>y7+&e`nO7
z2RO$I?#5a};U6I>bv8ySt!IV8dGaEy640bNi398cKy;l>9Lm(-VxgJDMK+9=9T3Ed
z3fQL3N!qK-a)K}8ql>@W5-t}A+Dx-6Pf4`&E?o8v#=?Y^$QG1Dqm~gYE$urV1&bdc
z6at$$9pR_;e;9;mNPSF|15aogm3Nk>S*Vzom{#jPZ$C{HxA(59n=Ow=u$1Zr-TjqM
zBcHoZBPm@9PKF#xwOl!+UUwYu<sGhzo&ie3_4C^7N!#};(|gswYIVzV1dl6J)_H6f
z<x)i*zH(+r)|*I^rd+mqX&*Qo=nQgEla#+Uf>hn873j_VR6Sf!yf;Se^~MoUUZ|`4
za&hxhxCNP3KiY=l$NiK8k5UM`Io;x3$8>Mpe67{u-qvARVi75n43W1O`bh$$9C8j5
zzT(lET|#eq#uWNS#2qaVcaBzo1YX76#)fNlM$oAePeP)1ne---g2H8pc_3*JNan?z
zsIqaYO$1Yxk%8k!yx(+70LqqXp>le6MACeTPJ$1zZ?MGcN~P{%+NX0RXsHu4K2%Y^
zd^_9o{YJt^wQ9WD!fUmDSP1lhe&A`z)VIia6REUa_1SnT^L1dvvam*-VdzFvJ)ke+
z+<4w6{<v<-2Bb99HA%zmvkC|()21D9ILslTCFHWhRe^~+04a9bcvCkbR!YKZ3~609
z4pHuy;`V;SI|xv}5hu}4)ihZK#)>%(`ZMOI$iTI&;E@u?$=TQ76Y@+;Kq6SobyKok
zDX_f0ne0!gwKY}R9;puEx0suujjqR>XdfY8`R&PXJ1cN@LxkUo1@EF8qvMyl9a?U=
zG*STFSG{bpNaQYC=MSq;vR%My((S#c)C)YBo2+r~H3JLb9B<)@b`)w9YwlU<P3}l)
z|LQ$Co!=B$(cT_OPi^rFcPrknhc5n|zt|`8HFW3U(Gze1e?2_E>Yc9&lE^G$%ezOA
zv=Mds47a>#!)wn;{I;Tj?bc*B1<@^Cd9j$3Cv+82D0ESJb-1u4dhdWEO7Sf-uJPN7
zcmm=m&JXToJ~kv8)?_@vGHpFUJ1a(c^`y2SXl7U>6;%3;UZW0=La@rsZ*)-1Zk+Y6
z>%KVcY)4hH!aKwX!hn0R{kY@9zUa(H^X;O@&Fwo(`m5N+G-tKOik>$dz95auyZCm3
zefehU-nq85D+3RN7BfTNY9OnhWzAAns-XQ!kA-TANwrj}S%k%~0vP{USnzXj6<7y>
zT{4lE8i_P3skKhH5w~<33H4*36m{M-HpY+Z>?U|@_z^DCM_Ye>VUoM`2-A;kVuyfV
z*Z1v-tRWDTGTkbt<w*t!@K;_F1u6^vOZOXNH`-;nur7qq5<lSmN@N7Bh@eC1i6AtM
zxXRnzFvzp5&LmL0b)hO~q+I)EV!b#*?0n<aSX+|*LuJS2hJwxdW?}|~6EK;Y)QG@$
z(a|`;&%)as!o9ab@Bpaz<cIGz**iVI<TojM3*RgfL$sYJZ1D$3Atba8FQr1gw>p1%
z*UX7chejqgZJLw-{=z|3e+P7cBV}%TL!^QllGZV%qAccU5jnFoOHsFHftONldO<0u
z>fLVeY`KC9KA#9lwo|hFMgYo>EP~~vIIHlnUHmeKf2!p(n%Q~dk30claV~K=bhBG)
z103Z~949H~f#_};yceD(>H_V+?fmhYgk@9ZJsPJxkJHn*T)x;KsFS!y@n7)n2%Jlh
zBUR7Ba>?1kPUY^Aou5B-MQ1?&HQ@D|e^@r{J;eJo%1b*{I4zkyGjSJWR!+}{tr6bh
zz&oY!_S)8NY5IU=Lj1Id5h!N%RWFPGnH4Am+-SwC;eMARXv;+=cN<EtUw@Q-WzF<d
z&aRU8{maQKZ>(8Z#?lf(9eTY0djj?N{hUsSjy;@?P0>Bj^)~aE$~RSXCu4!EAw>A#
zY|h<6bEKs8IIPvra<~H43B43{&@<otGR2`G40n9KSm6!L>4l~m8x`#FvPCSC0xd%C
zPy|q{v+nM_%JHZ~N3S|T>Mzke@&qg1Oh4XY@ZX+O&8(FI`B89H&LW0F;N5d?LT8}B
zihH?t6L<WfA<%(s8xc#R=`V8(Or^@HmI|l>!1^oAzD{TV3$36{ybq8<we}UnE?$Yh
zl&ST$sI#SzCfOFyyLjxtg0{T69rgPqs=)O_MljM%7N9rQ2B4qaLp+VikvR2VUql_8
z82dUwtM(pO(CKW~nVkUdY)5xS554k^;R-TC51IY8Eo6wTwy-Y<s4n&EggyjI&iU$R
z6)|4#R&*!-QR`mn2s@lc2?z~u4pOr|oNL30d9EJIndV_IZaJ|2RW4LV_bz+$9`FMx
z*WG8Il9B#VRBWQbMISx+^5@`dnP{Cahe|?DHj34&Po@@bl*4KVFZ+>^cBYj46=Hl@
zGm+R?FMPm^{`u#kU9!~m%J!*&kjh9c2dA{X1jv+{{_Ek)<u(rGZ})9v_pFgxvnWQi
zD^7AWf`7#I+$1^!lysSoX_FOEjIo-IHBi3r6^=!PXdkFpnpfdcLDI!~+C2B+C1QHC
z*Q&NHXU00_8-DxdvQi!*#V;wVO={=sHY$a+r^F(w!|eC6J%vtFh3(am_0<DPof7cm
zW_ahw`t|jC{v$a0&L~#JU>z%B*hauH_V{Kkz`5Nk!x`o0+Ar&j`)%>s+Cx>)v<G2w
zWWyU4{O4vXQYUsoPCrlt#51O*D2}UbO^}_Y+;cy<A2Dsjf+>W{cG@Bgw}z7%L!us0
zyITJ+Sn|1=J9@-i+TshzKpZZ687Wr-j%p=|K8$VK@(Vz`#+pZ5WgbflQUc$+=i*>R
z&G&-^nNl(GTHg^@5MK!HjRLEiGbnVIX^Np%Eq|zn%fO*u>*^1>d!-V$q!34zWuw=%
z4@9zD?34Q*M4^pD`vr=~*=itJlf9PO$`1bp>bSn=)`lmF^7z5h-Th_?ggXRiT&Qx}
zt!Nd#fmR0q?XSgc(Lp;JQn<EW7q=rvEoWN-wSff7Q%ix$OC-rc_I0f{+dJTC9eE~~
zI6mn@TrQ8%H&H|P$?YvA@@E=%Q-hqPM-~$Eyk4gldteB^8Z&#%(&xDFIs|#I4q~Ye
z5=TIePPprLtR`f7%~t8%+?!yjvvjCi0DDXczn7t<gpW7tOwAHoT@{Zw(?z<Xu+RmS
zJ9y1rHtgFynpHrk`kj>9O7^i9GxmD#{tAgb{F>KE&=Y?4GLV*R5lG|kU2!gCyR^5l
zVX&DORI8q>Cd1EBBfP&%8PrLjME%m!JO+gidiBsp=N-|K9JCkcTO6v0&Zkf}Xl0zK
zw6(V)$<bqoiEs62YTWx2Z*xEgmgt$@%;c5_R8xq*Ow-M*`gUF`u0LS+KNXf|)Q$v~
zTQj%hjcaq>TmC=~^LD#qoQ7pgilSwmFYjpuv*#V5=Pq4c4z|!PyLhE}>jcnE_$-Rq
z*4k`e$Y0*h`+m8+kZ{@cICi5skCk72j%boh-qAPP=Ou{Jl&7)MqZ$8;81+r&H0r3F
zOl+hb<8;!9+fhxo2DM+;q0>PwuN+6CQBsE9m8Cn?8@4Zi792lh0Ns4XQ7(@N2mppK
zh>?@20ta%prV#Z+l6n<;G@wFf<?7=#+S~w{(1}$gi9$3l-`h5(iI$J!8ehVrX?)g;
zRdI24zaD?~?HgVD1K?8G+IKq#<3qj}zguI@J$~29$(ehde4wBZ0n>`$2v(rc;4};y
zQ~2Qs>+G^xv(X8CmOw5IK1s;s;l1hBA`bfnSYru{QYi07$Gr80MGMdAp%rWIy}R>+
zPKYvGyZyPTJ>Ii*(q;|`&~7HTk#B^c`=AI!u0K=Z!WC6%vero>NEdC}oZ|DLkuU&C
zRd7Os2{_>c=^<~upIa6uOeqV~`bF65I4aQwhtt)!Iu%F$l*k^fAQdjLicY`qK>O@b
zv%@%jT<>cQo)RjwAV`CKLQC|x4h(@VtQD3r4-iXF)Hv`r*AA9SUB0}sZNXhsZh0rI
zINw$EWLm&~`;z3SqUUHPjF43C_acK>&PMXIaVkE=tq{g2v~B7LJo0yk#rcnt|IaK%
zN-aL`8L%+9>_B;uiq2+P14@eNmBh3Ld=FaCE2NSn=~w7<7~~2!>;=S{JckBn6^P${
zi(HEks(@Gh+`_P|V`P8)<HIM^R}sqY?b4i_K`10Nf7$>qr;x*914HAM@k=@??9>M&
zuQgcWw<MshlB5H+aouPd$#GZ3$63~k4VBb@#JNmnH2T|4mN_lv0Z)L!Mdv#E7>h?%
zXCeh_)J)TV51<*qcqkZl(#73X>W;^B3rA)1Xm(r9KF;{ENXvTH9T+dru$5qCbEd5V
zW6V(<5Z52QXvD<e0m5IP5xA^II4171O%BK;x7E9;D4^|?G&fcQ>%IR)BBmR3c1*X{
zX3GHe<P$D$88;D<ME?qn-3)C4r4A+QJ!al#+?rA0`zm_`#^>*xl~qK%a}>W-(kIJ<
z$52~H46iWfw9547n!txC@FFff%%t(zv-S?|qislLru}UX@&a&@CT?<7f5C5`H05m3
zG|xF{lFQJMEU3A}ENx|LoJo_7QFaP=dHPH9PkaZ8fN?kDDMZM)``1?#fi$(cGPK1<
zFysjk-YiJpIhp6`M=CpQQ@EL8DaNAukVT@T#F}w0v5-jfTf?8%yDhTd*VO-c#?^O!
zn|VYZWh1=OA1MBK5MwpLa4AYbjYA1V+2smRc(Ys>(rGPI_2mBZlqB++4?(Y`gKjn7
z5MGN{4`Nett%iYo)=>na%@WM{U}N(EapyXmYGtr0X53QudQ-ud)pc?NJdQ0Oi0pPL
z(zn9i$F=BG&p3|qaymo^I!sVqG_}wcdFb2SUN!KF<x?6>fgw#*tgkVt6}a(4-F8$8
z)8fx=Z-G(uhfLqN*`&HNSnm}}R?J3p`<&K2TR22`rDaO%hs@Qq9?~?^@7^>;2fN1O
zYN`AQilWfTlQ-V?Kpc58y5be<7A8O-O}A{c;noXM8=Oa;bUISUups<Ii%oEn7wje`
zQ4>a#Mznk#wwoTm1{BGpyJddXXvGgGto19t`L&KKJqwAL{#c*J4BXvRQ<bo@*5G94
zDQMDrDYhA|q+nm=v%gs_C9G*bIrNQXz3RZWicZn3R`Cdt@0J-sB&1^ayWrY#F}0tU
z5bicpzjOFY<x(b;l(l0Y8`-9UmFN!ZjaK(cYU)`gP<9iJhL{{P>S#sK;bwo_R`#e6
zrGGFj?Pu<Gp6v0VztPLFD87%!k-J}0V+UT0a};0E-V9=lewL~SF(R*qjaF{Dg4^!f
zhJ0{Gi}voD3t$3lT>ilWZ_pC@<Y)1Z#}<NHVtrLVo>JdWb*H#ml^bV(lhHdgJ}3Om
znnKO`d%UK)Voj-8CO3l~qU|R%|3{;J9}bu7w^hlsZJL9A@I2n2IbV&`Y*Z<=<0?5`
zQoUAurxV1HtncWh?-Nz*T$5ZJUfq${d<RWNNy+2lR@&yS_~qzhRxKZWb{s?`t@d7X
zCar`D&8n$sDjM9&;C`Z?8JJqQMxVihz6h7ar{6G8;3V*k`5Q+tbMXy7S5xXF#PNDP
zsBZlrlD_38G-I9dkt@2Y%P>^@!#w=?h7pWB<mm`HwF%A>CdpC_*ji`cQzM_fqSuBd
zA7hBcKL@9TdfTmv>DI86W6-O8#e6>2Z~Tt7qd*146#i+j8`Vzpel=%M+xCZ1uVO5#
zq*-`0?KRb`<0d}W6ZmIBf>-#)O5W#BSKQ*R#Ts{I!VBF-878_5u6Ng?-5Ct}i~+~r
zb`;I6#^M+cnw9=D=QP&|J~S&uKL$^K*KSg+>wSI?bcxP7Wr-F;5{%XCh5jWt_*K_N
z1k<NT<BgDc4=snLB94ugS`O6VL>ceDEt_*1!sPhNYo899AUOR3b3KJKS%yL@3CRu%
z4W;Ra4z8#*0W2a$T5X?SXX_=RP^3w@-=E?9+!5BwxGhx8sOhg}^`h%HEWN5yaVT?d
zG=fL|gv;}VRFI%qp;UW|xA-yKcL&hs4=bRmcRk_giZ{b8;9o397q_b}cdxhFWANHL
zBU`%f@|*i`7_In++pZWB8|N{qfV#QoI+T^<6Y-_vrB7#(W{FKm!+><rJfFq<PmJ6%
zaC<%U(A)Qi*8E}+3dJ8T#|y6sNHR#Bn&L<?I7H5kd>bO9K`5jSdPT5GAUbP$AW0He
zar<Pb7AH%Ar8ZGaqHD((q}M6~Z$FjJd^F?F<&m`vc>cAv8-F1E^2Wc1hMY^NlPUwv
zV-E<{oNkMUuu)Og$)BeSL*b8@F-Hmc7uhk95-*X!t{ZMkI|cIiRFbNF;&ri~SD0ws
zoR2E-jwU8>;NE!z05Z@e0PG~DKTrIz&_sd~L@?rC;Q0r)WB^4Y3NLAv31}o{A8<kH
z;yDtQKo+6Y?O^(To<hJXrd?=20A|~-lfbS4ztnE3YtXp_%z*QJSGH7tB<+klk$7^3
zG=6X=;3MzRcy9gxNGRx_TpJQ3S(@wz?022_K*r>Bfm;#(8Vy0j_-6LF9eLz9?S!be
zn;Q>IaUB^$0t1x<wtU@f=9can{H)RchTjM}H*Vhv_*2z;Znlz)%#H#?;gF=s3`I|1
zujG=qWl5gk<dMGSLVIRn`wJi)vD{>uAC|XX>QeBO0d}SPrrxL<^m!9X{xU&3&varK
zVOnWz$714in?U1C3sC0T|CqOas7QZ&a9YTAdMI%I6}g^D;XL6~)~j{Fw0*VrKnS<Q
z^K82fa`4%AVc8S&D&(Ni5|Tl2JM#s#x}gk3*%{%?>2ioo7%YR`5f?aqztX2_XNsk%
zg>`Nq06YkpYlPBqdy%DIw;Lg>vRFQrYAkt%nS5^Rvn8FFkVgBT9|<H9U;Z?0zqpAW
zrZ`xRUh;Z7H6K}`IdNDj(~LkoLj#VH3*q0Z#WUYL?c6ub9R>p?OI!aQgEnb=b3qse
ze13JqU4B?N($yXI#RAs5GeQd*$7$kj@prG!O^T~@7zxS>^*Jt*u_nrCK)v?dwfikq
z!OE0&2_B`pMR^Xj&Z%bJPkgx@&GFeszey3B&d_TP_bf1#1UuaA+8l*&zyE+LV8_^<
zsL8M#(fefAn1AW&;<ZbGp5T>ime`RLCaJ$BnScIKot_Tn)qBOs?b*)nr6P1RiONn3
zj^ECc?xya$XcwbRzd|DVFtdy)b72VIb5fgoD)GMvcJIQ4;KGcgYWwitGkNt3i(zU4
z@8$z(Tc?Da7I>xAzpbvVe>al9cB;in)uR?(>7fQ+ELc6Il-?+tTjCa<*gve;Z}CCy
zMhIOV9Je5=ZqMLnGY!X?ZY|BoyLNYIX?+{pq+n+Mmy0Iu8>lDH0oiOH*|^ji{t5+e
zCR_?x%?vqik3L%vvu`x_xgu}LyOY=k+^#_T^xcA#byljHsw5LmjM2r@r$Q?&i7OTg
z1Hhy3If0nqM3((f;<RspnEBALJ+gMKvzOV_jjwpFVtC57+-JW{JX)|WSnP5Zv2^t_
z<zqcAxA&-zCT+s^!K#VYJG`Uj^&)NmMBqube6+eU>SMUn?W6V9$B=m&2xV>`*-Is|
zdRwEi+O<#R_HT!+;8WMRkn-qV!#4Z*p;KCSOnh#)m(*#NUKNDWjFSGK@|N)htyI)M
zE?q=kEHiiu%G5sR7ax`9PpdM_UX@iVM}%KqG8WEg8Gg}qqo4@Kh0d&)i!WTPPY|b9
zmW@yL85%Vne%{v38@AED=)o)5L-o?Wc%#=aK<Wwn<7?GDw`>_y_K%wk(Eg<ITYN>_
z9<hcwy5-}MJ1A`r&y3cqr-{PnnwJti3$H-Z{_QQd84F;G@8yfN!%iU!Sd?cTwxkK}
zD8ni{8CH>(i*u^aPax&T-ex3i)SGn*x|^s<f_eh5DWlm}FDU)<OP2lOD+#61Rm<bj
zA{5a5Dn)a87kZ|OZ%eCxSH$8;civ4}=#}5L1E+?nd*tL5eK>B`-%v&2_BO<?g`Usf
zy&OwT?VD+Sq*)eio{JnlM&9&4(!OCWxkt18d!9_=I^e~kCu4tcrP7b&J)OeL!YPKK
zgrYu=i%dCcmq$})m^se$bQ2P6$4ihh${M}=#z=Ca@c~8PkXv%e#&Kex70G>tLQ(YS
zgQ&yWkdEeDdVSAb-4(5JUvI~p%VEKep*vlwc_o_ibamx+ax*o!#pirrVyhHw*eVG}
z%8t|J5#*Ad2j$$Y<oI^?WS#Q~s<1M%yTSjU>@B0B?A|`mp(O{U1WA<;B&0h;MWm#g
zp+jcqZlt6{Kw3(=d+6>i$)USDhCa9d=l!3x-gTb$!&&<SALeGU7IW`?UHey8gn+n$
zuvQncGu5M|8+?IW0Y&&w3p<cN-WY4Q_Krw>m$d^r0myA=$rbIpE{Y$^L8Oo^mZn_>
z`xvm+%qMX35#3$_IMdOBAS8m4+|vJTBXJJ8H%TRWc5twVKkitYc@+q6OESuc#htY9
zy|ff3@+F1P^%irA$CogCx5{*d89}G#Qc@Q^6H-`F3i5FIZ`Wh0{SCB(0z97}M1oP{
zc0B~19@52ciVVj#fk^?VBMqGU3dOy|axTlC9gUJ+kdMgq1Mp80(PWdSdvh`opD$nx
zyMC_zlPPKIfTqHbr2gio6}H9;vF3yr@saw>(=2#<Y+fJc)eH)H?bSpQv~N$9&!>+<
z9BpZ3Pu^^k4`aA_|6F^23VWNc#+VS*yhHisH(IsF?Wz>THM|m(D92*`<ucjH8}+NS
z{Q)NEyWUlPy{^;Rx*jpAxxNE*YE*vVkeIz6P_FsX%vf6$kx{2Ga|zpYh}V~NH_oVt
zXk>BM;-YXTpah0d89FzeAIie3@)!GxuMrd2(~J>qpSs)^8_h~Q@4h4=y~)KTpEIlG
z8>{?IFPE0+kW|m7_pd}))n5f2(6Ltx$87<Q1mhx*5(NlfJL)sc?~3f5O7^mZFnCQJ
zs|X(?x4|oru!|~u3M3Cvtox_i;AWJAMwX&4VJE|Cz5%0k=xGK;Ak5s{^rWq$nkIDP
zk@)?Hw`p%RF#I1vp)t`lDOb%u*z*H~A8t?YO^LnQcj&`TGoadhDA1K(Bp(L$0eSr~
zDEV`B<;C)(TKeLE=oHHoyMEyhP|Kgf+MPgmCfu8=yWzP1RJ>adON#48f>#fZBu2*X
zF@Vcg+uGIL!nJu!`#iS>k!p+YzeMe$j5^}~hVusFt|ad4fr0A-Wj9{ocf757bM%p)
zqV*Uk=5G&Gvaf)-%m;VMmltohAFP_c-T<DoxT5Rgu}xp1d3N{lMQ|^HPrQ~KV#eVz
zxsXHfsE=<Rt>e$;t-_(m0!-oO{f{?dH#<pu)n5+u-7}ljj2|c)FJ~gOvl6vk$fD;W
zOfegy?+m~=F_x3njYIN{hpf8%Q=*L}mR{+h9g90Y<AuG&Qf~9VQO%vi?ohCzEH+@V
zta|#Msf`^lA&xf+4RZ6&{To*9+lP?aHxY$}<nD_uKz}e1yTGcgx^@|QI#45ggs8NT
ze0Jw52Da+`$t8Mpze{8@gls~CwX_fC<Q?zYF+cfs;_f0F5GZ&+H~UkfJu?xAx)zrt
z2@xp~=oV{{Oc*AGqI9Z#HjxA3S$|R4eRMx`YVk2}I-e*ISGww)^|~P~9_y>s6E++2
zxIruy4`xuHniHbx)Gh~_QEKp0$(9Bee6raqyQv-CquDr)>}A5ycs?Fzd%A(Y7tE-H
za4)4FF$mI4kYfdxC}o9Q3<*^;l0YyCC=Zx*Zz0r;K{QQ0cgXbMP380TX`}x8S?k57
zs(CN>otsy_)E-Cp>ggBHXq3I#nr~gOGG`{hwWiZiUv~azKDmK23B1lll}qH}<G<N^
zK~<XNXNE?JDy~A@hd%OJqqJw3j4<4MN>>{}<y@$B>Nq80a(8|Ayv(}4->prWs?v<d
z9eItJQU#rXOkR)dJ=~;+U;CJWM#YJmZ)UC+<GjGqZmPT`?YMqr^PeS%tqQodZ(1jZ
zVK*wovpY=cu)TQ6M&`5-1-7OMv=_$Oc}{XdM@_p#KKQrbAv<mU!335VVHlkC*Q7B<
z;t|hX5Mv!8e)wiH;YHXiOiK{dyCtm6Z^9gRb|>Y<iNbx<efcoj<h_QW9clXz-pAox
zOLISP>sdKa0f#B@_i0hqmBf*diKFFit%h1TZMsfaVs6%sOY+f|H8reDB#A=bab{Nx
z50YsY#mb3*kMMky;jf-V4mTDHDYQ>urlIV$36P%i@7raZ$T93+9<t!2k)L%3r6Pve
z*DzE}XF0MtiCZ*sIY9Ub{e8#ztAn6g7Ga-_AVxjMO{X+(_RXpUBM~fl#Bm|p0m`Wx
zyu;TV`9OKOAMsdxl}ktTejq`pSr`Sl4;F?wb_+WfZW1)<_`F{t7kh`xrk^0RB-8D9
z_mmnP@)V;Ai3%Z{?W^_>uzjK+y7Wra&A{Ut`Ch$Dmk;(@yVGpImzB4?J?N3^77^88
z5Dyc(R)j}yP#01ePYanC;n0rUF;a&{wHrRe#6;>pImv25Qm|kj#h`DSM;=wt(nEqH
zX`l`t)fu~k8$@{1le+|^mvu@b*@_<Nri2m7yrZG&B%{&wXfu=Fu?)i(9wch6d$c}Y
z_GIylMC@1et-kUMl}ehc0IK;%^@AuYtq&l%gOv7x1oZ-y!6BhFVaxeax$K^ZxcVfU
z#fRf$kzXbyC!UA{OU6U@5LKJzwEj%r7g&NxK_}4B@s_*i`$N%6eBMnDtM2YgjW=9!
zcgS1G;c8v&Y^!3O^R}2`<{@OE8%H&4Y>3UyCo)`Z!3K{6C9_m@Y`~RE(rN5)&Fisq
zn<yj0YU$}-&ahT$su;eeQ%1eSs9|VqN*n}(BL7g>bAd~5(^*}2ePhZ>{<O@bal`3+
zRGm@PjqoJ>A`~K8ZRRd!sa@MAHbv<fUcJp#6S11*(c3Fg{ENq)fCh)%hL1G$?u>l*
zLEF1PF)L8S!!)0p7lEaFHW0u$I3CO3tP)<kI6pV=K`i{b_rSGUn>P{*s(W0`g3qDK
zz-mW7MC0+`TUk_;Y42Zud>t{b430^4hzV@EUB?X`n|-*MGU{?1y!f0W>hUS@^%Ry0
z))ER!OBl4n@!bR#GtBr2$s+=RwI78bO6q$f$<AOXu@&9>EP{M$;-DI}1U~I%|0!0{
z_|w0+FtaSF{@@N*o5e==%jc}VI3g-xzMR!puR_o!Wfw3iTLO8MVL5=#av=cjY;ffL
zaOv%(V2EPQi9)&(H1XmD(bg;E?47IA7Ath6%csIj4or7rQN8m_*^xYnEMV1ER-AZo
zT4y8zda>ZL8eUlx3(`XI`Qat8DLnTZ&)c9XjnE<Rb#x6Tf6$t6G*g!QoNafOYC`WH
zl!i7ntJ`KXC5!z$yiT=f=v6cg%C&y|oBSl>Clj8zZNDi^`^e1@%<^toRqS#HJV8h2
zhH|LpZ8+SCG7_v)inbtbpNpDosR(M_AKEMj4|2Xkzd20P@VL?re*<94btaB@yXD+o
z!Z@t;*7d&P)5H!>;g_l%3zQG{q61#k@psD#i7O8y`q4X%(A3iud9}Gs-)Kr7QzuR~
z>6}bZ;_r?xrMgmxF3;w`jrzlGh)Bu<`12&M52pBtRnC0e>s_hKjIdYD7cyOz1<JEn
z@fd^ky-oEq1P{Us1p~?a6C%V7ta2i!t%e5Ri19E#2?n2yirvGiM(Y|5SK8jqkS1r5
z6m}4!jp}bUZKAUGkKZ}Q7=5p$JGSq8h2?&PP4YMGR&~AGe)uj4ZE$b@{G`Ak3Omhp
zE1hp1-U(rBk{Npi)QqIXC{**eHji79UPLr>q4{zesT67TqQTAto;4CaipqmT*~Smg
zRBX;HK6vR;^O4M<&7_Fb*IoiGbO2av@6boSKs?|nF@2CFn)vCQy@>AlNW+App4+w?
zMDbg#GTqi*#klu3HNDC9+jlv4jj_7t!*yOY`pY<ogPuJx%Aa1-sN6uBKM8soHVkJc
zj#%7&i>Ufpj0O-EXU;i-BF`B-C8fvsKA;h^7zUEt+^nM~d542KVlEO;mF4-+3)GxM
zo^H1kdrt!c+@Eon;#?=uB|0QO`A(2xX$T$yex>14u2c9j1BqwWbMo`%hlE1$;qRAv
zNl+Ph9rFlBkRMI$3~!!ly8@Oz9w^}lr*Fv&Gk(CeK{G!n@$R5zu7nIUm`M{jT@4o9
z)IIj1rk>ED&x@6EI0I2IkK=i*H$F<kj&6#*S;rG^$5NthGeI?XAxum1*GJ)ZIba_^
zW9_zc_UG){k~qo=V)uA^CefbGh$wZ0mV~A_yp&>S<GcNw-LyB7KHNG-TMCRp66}bi
zx+bjKlF;(I?a4t-ub|=FL>UF?)MBq@C>yrfYJTRwvMHxaZDQz0t$f;Pw){yWp6R%(
zGw6xGCx)|suo{iXx$aaq{?sSRTc+<0r9=^;nfzzV7Su<44nZsWIkh|JbV0_FG$I>C
z$Pm9UQ7plgj3Cj>TUw*xVAEtPnv=0gfzU|T5EApB!E+Xle<3-$&0i#r*(w<pO|H0z
zhK&ui);lGujkyF~+sT*DL`;6VD~|LVI9XQ}OgSM$sxEwfL@IxnW->!NGL=(iRrg9{
z&CFNLx8kC-@n=oQ{i>{&7s--0{~hwOu9kp8qzBYW95ekjv})3p&ss{B+Cp;PNjWRM
zDpClfpkp8Yg|AZJfGnw9r}lUztKnEDk#a4Hb%K4`E0`G!;&O=}+~ow1Uqz`9qGT_%
zih*a!3|MkIL)9Dk$c(kM$oa0}(}uX-917IGgJQ%!?KU^e+J6VQj~%NX*g<t2N+Bx9
zP>yKAhyPNCPbT*Dam!;krq&eAJG=QZD{+*<<o*6>wcw%vHAa*+^NIYJMdr{sWsGk+
zL}VjR!K}E?@1@E%E8^fusGwH(XUYIYi~U-?8Tm(I9h;6{kx(=MDyu|oM{A;6Y>+Q=
zf)k)Zxje5OZh+vd+Y2QO0j45P3wy}OoZb5_fftuPZD-Vq(>Ibe@Tzlfq9M8qBTUY}
zC(-WpjoL<?dbdl3V3K><##->|iHB36Vg-D^yt@f5oz*A3H7vSqU~fA3vFl+s8Ss-X
z64aTJ=zAvK81yI{&h2|)H|BQay<!0k23oOf^cBzI(bA}+b`R&+6<tqyl&pGqmmxk}
zR$wU%Ij2}Y6CZckY=`cmJ#24EsnX=%{KR73N0Tiw$Y72s>f5NACw;{;zb&0i!z>Kl
zho|_E4&INur6dz*h#%(0(fq(ox*h-gY>7ImY(m<W34P>N!0LAa=aG6j23G3hrn+;3
z%g_h0kkxXHIuH1WlWYfWwbJ!k^vx5Fu0cMrO;EF6gLQr_=#wwC=NXmz%|5$_*ewyN
z=*FP-7JRi<kT@<ELR|Wew;Mn${dY5?WF8&STK|~k8nQtItIl+s9@MP9zd0MH@n|l*
zZ@9l^J-CtLIXu{}Zgy}T*5g@jx*3vz+fi^DXBIp<{X9z?64lOwpSD~lwDyDL1d7(x
zp&)!ri`Ii-g6PU?DC}0SSx*g>FE-kAm>G!J7|^v6qPf#%Vk>P1*lG&x0dW=dtCPu~
zTT*)r3KH7auY5Lg<ET0E%_KaFn-He;`~IB4UnqTW-UXnT11U=2hdao1B`}57sm$wj
z$7L=MkRt4`48NtYxVs@EU?N}{DW#tz<6B5oQ)L{qc#Qea(dMzqJREkza0qq{;fQ&c
z#SKbiw{oN1t%BdIKghzLP+m+*S-EGYNDDI;In<mhiDfH|)-&q;>S>ejZ@;XD)ln{M
z77&S7oR4DNus+2;+eG$G-kb=vS!jf|U)Up*sJ2~kvj@+#9kX5$b1)J3V3v{|;)BdT
zoL|po6Wq-|&_*!7^=8#xX2z>+7fDTzGOj?oyZ|NApOQ^0Jh=AftIISFk#WQ@X-SeZ
znSm17tY(bn=|K3y0DOp&c=TH?HTDgSN=+|5n|`oR3&xSnTh3l(_KN}}Qk?y)JsyqR
z=}ei%UrE?1Gw_)F1taNkLePqgx?N{CU#7-ZKUnzV@wU;hZy5hci7UU_*Zv<m!VdgP
zLC5RhM(FV}S(1&GmSL2^SPY?-@yid`Ho7G;EJE995!Bb^j=CjM<pP>7G1@-7T%4JF
zzBs*uZsAp~EPHDqpNK0eP>O*k>CqLU0P$LS6KH;%UQ-z+S-Y3%C+pRYJR*jWF$rJ5
z%>QgpN|eYZ6;##~efrBev(~<^vmi&|u60~k0Z>}Qe_3_{XxD5MDBJ}PuhsH(It8h(
zot>3M9T8?{-xl|B4sVL@TLg#=oYm<AilW~M$0r}=XAeE)=`sC&HIrQnOHzBiN^Liq
z3w+BV2SG23F2p)_YWKruc=KtQq1xG1_-Qj6Bd3L}PFUCUKl%^rAdnHe$@oHUFb7|r
z=DgU5<%n0%nuM(v5rosasnL5QAFFe0Reb}BLBHabBxDdGcVki`_l$E+JzP`ok+>N8
zNKeG<dT1OHrWk_T7wM^pn1VFV_;M+!7K3y^4buiev6Dpzf#A?4&!8ICq1202AjB4d
zQpR_3ml4slfcET3XG}7oI2UyKDPSbI;Zh9|Vf))oj05ci5JuvJr3Jyz8rTqe7Eu&4
zZJ76ZyB%C#n_rc;IK5y$%yO|Dv}o$+ecahy!<<v^J_7)hg_cv=<7fYDO3gZlgaUat
z%~@d{VAwC*sh>6HX8B1q>p&|Oge~%)t7P&>ZT7s4RlZ?&TPI7Q?j}Ui*7q)vHBlo-
zn@xILbBDTc80lFlM|GPdXDi-VOO%t@i%annU0k^pRxo)rV-<{eNO8zBai>DlM;rN|
z^3>;pn`HsabHD&Tp_q61X3$_cOq2I+>M`#`qn-jae+TJ9Z^ia+MT-URZ$Gfpdaa+e
z{2pY;Ql$V5Vm;I`>|Q$~)yXW!(i_1X4N}(6PCi8#e+s{xnSe?DnTV)Bk;|&N`VNCt
z_Ya-5Phh8|e|a(y{Vu_B2hevEbB3XD3SF;PUG7&qi$#7(hXJ_xb@S4lwl;r}O<KZw
zF~!XZegEx+TK`-`TI_2rV;Nl&u9H?$d62o6=cN$>$u2U>>hMX?fX6KD7mB&OO+VHC
z_p1XCT1)$_Y$2*}lZ`*8!@DI6S}o+4V@3!ajR0rW5>EMfC+=#f6mGA=m!BtovThwF
z2OU07C2uAD1rzR$XfU;Ho(#N*ek;~<g!ULKVTGl5t$h4xk|9wjJF7($G9CQp6+!P$
z<0pC8W_?w(TFD$!=N&7OY1p!7A76_`u0}K&T7<FBj4?Ouy3vG{Mbc=^_%|UlWhhtP
zR)(2ifgh&DdrBkOS7?JWUqHgz1^Pa?O|OK;x_l;a{u2SBNDcjUpOF;%SUh?>=Z(Cm
zDeb?l9M1oA>J_H)+Vxb<71@K;C#|M~@urY7FYiJ0p-3hbmzn)aci0Q{JML#FHu0h>
zLC@8*U%m3l&Mx>=XFr9A+996VcybcUmS~o`fS@<dI-lb8$-e7#Mr7kFP0iN;5Fk6v
zJ!JNFaxyIw%wbt=W#C>{q@*H}S$cH}x@qJp?#uYlndoWVJOFcS+n;FYIOjDPDMe)&
zCq_?eU9E(8RRj7Y{*qJ9wK<Pd3#)<=SY!7T<j_b#c(lE^cH<qw`++K&2`b$&`3|an
zc2Mu^a{v3i%v@}X`b$9rM)Z--;Q{%tv$B5s(3f{Iz4BowY>6JU_7aBY8}{dyXWHzO
z>7M;45EbAoxM9;3oZZf$*)hQ-eeE<MSS&k^d$-s8@ZN$G|F2qQSZouaD5YC)WC3$I
zAT%DT{Y$u?dF>a>uJZAn6|VO`lox@?Nkee=s|5~hEZ5<Vy%C@@e2*29B;?)jRo=_n
zD3Cb_1C_wv1>ZfVKOneyKwcv+y8+ESmtt}`HngZKHk>OiG7N?Byngdl5}(DFw*C_3
zF@RqOz|w$qO<tf|M;U4Gyv5L*4+2rZ-x*rIR7^4gSExy4i%Y9~-u3KEev(yHrNvw2
zX(g|kqnQk%_!=l4mtN9r-P{z7+6&d0_qC*M7D<@8G1D`V*<k(H72x=`+4In=jo~4_
zamAx5P<tB18Daw5+%!J)H#Yzq;kz^Wv(4}C4+wkw^V#&#fKi3tX^A#LG*P00lnK0H
ztxtgA-@nwnBKw1`w+vrEwgQjmM-UlUkfvi6D5_^?JJ~H%v@L#Tysyi#k;Vg?Cfy}n
z)NUgWxqSV@01f<1>!Z;cJp*N2?9o4?3V#OYInMjjFenM8i%5_BUXnce$ej6-E}DG?
zXy1dCZbu@Z@dan}W|S#RSH3}1?`YZ>#;H>~Hhazjyn56<Zt0@Md^o2$8ALOYM?WN6
zwsxKC_BYjY?s^}zLkuw}OrASy%~ytj66XZWY_s2>{oATd`A4q8W6$u6&(Z>M*boGU
zJLDC}A7AuaGT0EjPoI?;*NvzgtlrjF{leFW6Q$puIZPusAwsz}sd=yR1T=X4Or;c!
z&3oQFouv5PgTF*0Z0E=tw)`s(4qYd0Ro&&SqKas2-Tes=GsTQ|h%emG)lI>J?uh-k
zl!)v5*?Z+EtoU8^=@vmT=ZMj_P@y*)q4x&{XZqS4us~Mro(k-Wo$wvRfqKgN%@v9{
z;fdubY0!Zg!0+_t*Rjf#``RH*?t?ZLmf&W6?u~yhQ#C!3eMJu`v+%&DektDLA3Sd9
z=O^A`QP+g(x|-m$obQ$x?a$dCR`4y<biyjM1{oAmEP!OoGusG(4NQb>chSj~X_CsL
zw8u=0NqV*}?@?roPszS=kk&zNX`D0!b6+OmH$;dvO~Eb`>*VX45BK49*KA@F-1i;9
z<FJJK;=ZW1Du`Y03tsaP8qIm<{aGMgM-G(uo<KaB;1=i3_^aZYjU1Xe`jFCXDgH3k
z?SMdB+<E3;?P%BJX&x()+OA|!?XIsB|N9(@K?+y>x5@CL!nd8_6%_YJ(=xmp{V19y
zhOy}4FF+ts95ko3Cq5qp@)o_^Ep0zysf!dUo`}*WOy2gsub#fm!?2FjjJizakbjhh
z0zF<-E>}0ef#K|ySb*@C$i=xw02Z_eU_(ukR}#@jxP@%k5eI*~k*+obVogV;i{NVK
zM1Y3#1oV!Yn~5|PJ)p!G?IdjR@teMSXh4B*`=;D|s#<AyG$KE87_EPvR~|Y6pF3Bt
z!Qly|N((Bgdl3_X_mc~RPczn4Yt$r!^l9bt(rT<dLLs8`p+0#b)s+r@c{CE;=<%%t
zoEO%rxCGWk9?@P&VnU8B+L34UxT^;3zB;xOg0a;S3^rkBqk!5^^cS6<F_^u>n^x+B
zM`yd<@v<}n<a<n^_M1uqWBTU|ilo4fDtgb6&Hm6+X^y1S;uq}U2L+ig7w^703!r+L
z@}!kNtVER6KjoHLRCwm`GttE#+d=mBk_som;VUw!Z|-ULcuM~89jF=&Vu^Idu?e4X
z+51_4geM{xzE4$-@B2X#dCh{=mlV)PUgq^A0D%nv_&D^doPRXl_utm9VV|bZ51_$B
zff#%WzKZ18!&N_9uKVVa2Q#4N%rNcN_Pr{ChVtIjuznM)3o^-rnQ07_>M6{`qa2dd
zlmo};j&*%=m5A_psAmfoM8qi;YseUL_V<1h9JU$L+{3m>Vm?dh@ttQbJsQ$B@Wv0;
zA~R*mW>yp?a{}U;vqL=^V2qpF@!{e+9cUia6GF&u{@S_%n3XZSDed7eRpy5HPK%?7
zd(}y>(sZw8^LzChkP+uA04K3u`W`Xwhv@VksuC#{?cTAfk5S)D*FNk&JVHpC%*J)P
z5LKJ>-yG&pGbmN9O&eKPUM|#+*EgL{K1@%jr!M)v1kk{A9t)*mL5JxDz|vA1Cvvlg
zrI`cJY<liR5h7P>`7p6^3SwW)zqd3#x<?rRhElyyH^@8Bt*V<_C@wy;p-!wfoZ^h=
zdX}Tt48$s^RtG3Q6q^2*17Yun2PkW(G7^P74+4vML;~K-4zSn_G;Q7qf%|0L0+=KI
z5#S#QQpAjVnCg$(uuiv(jVL{iik)UmZqpV-_F43CbAaJ%lU8-xl^*AP{cgADeq!xz
zRzN814>wBt$?(-eg@;<-N|*WYd=>b0J-j0O<}ea&LaAMU&VPP_njC{J-o88en0*eY
zdxyPNxQx}FY79lqt(Z=0GM9{dCwjgN3FaetN+ocfX_&jE)6#;M41Mr4+wQIq^#q5d
zHn2TMM(aRnGlp?allLp*cjY*4snF6J0(k#bPUacq$Fs)qP)tL-Q`JBBm<NvKr}pzt
zI9Qe@0=UeybAqfn^SdO(MDOy$fy<(zu3IBOQsp{0{OJe3Br(tJ<FY>CKJ>dh89);c
zCol#oUEz4SR!l#+&jxz+^yq<psz52If$5at#V=IWcvPl6KeI>YWfI^tY|phkK9(ul
zj;C*f&=S^qmX^Wp9GD@8b$TC?dZU{>=+t#OTL`=p$vn;gW<a>Jn>yj+o6#(Au#`vV
z_%D-Ga){eWx<H1y;TY<tJz9=Gs8udq%b8UgTH9aBYJe<@P-BD2Pk4491|9vA<vKku
zITkN!k$w)cc=wJL42$Fn9Eod`C-k^|+jBe^KuZ%MUEul~xB!Y(67rrzu)XsbRJUtN
z4>-<}Vexrf7c)PaJH9Wem%?jSwwuj_`oD0f&V(+!j(=HeZvTj8CI=Fr^G;HhGj7^6
z(1Twy7u!;Yg$XUY;y>|u=cX?Nci$9fz_^+d^V+edTllP4Y)t5UvWL+aXQQs{asEh6
z!%yF<HSc@rS)Y`a?B`S;4t=Rm?dB@%WlL@><OjY2{qdWW)$;4t%=QIn@)ZolpTOMr
zISSAotO#QBS@UkcBFCJH#-TAF0e<|Zjx@J9+kY8tfF{*zP=xR%dt^i?+4C#UtUFl=
zU(-W`mwOiU10v;&&(g~Vi=BC^Ri9VeMYu!gXu%Q|t;N&Omq~+pQzXH8E(sdkQ#_Ek
z;hAs1QDr2!YDN@(MXMI~`64R3iK#yN4bftv0PXL%kR4H0Xf!iRajo$lnJQbQzd)|W
zQPBQ;{DMJay^loqeWI4$VHnpzxnc&xmDfh<2~L{k9<vBA;N=3A!#Q;{Z4{@~AB!(i
zZ@oz8ba$>`mS0$Qyj!vet@@J1Omwz&94>-fdZuXvB$5OGrb@ZSFlHW=0_<0X`Quf)
zMa6BO7nq;T*DvZj?q*YSlVSb$YN+ZH!eM0!+&5`6m<gA%Wd+awNbdO)DQR~MB=Su8
zIu<J^zhd8&PWt)G<3!fpa*P(k#jR-bb^U$o-UDNTQ$YB$`l({7dCU0RQkeN$(BN8X
zpw?nNb)1DSOha>{!t$_8^F{Z@=whWb8sc!V;64rqZG;jKJ5T<8AKEb-MK7~g)=gcp
z-svJ&pqLSu40yqV!Gd=;m(@+4p;&<*?Mv-S`vY>c(A1MLnfolgq!+x^0#62kEu5#-
zT<qAesntx~LUSmjKQI=fI@eYO7rHuxo8)*3TRK`Vzx#1CcYCp@cV8iq6J&LAoM09a
z&0euC<0`D<z|8D}8Ihdo2ueAhsvsfQT6k#6ft@Ji!R9zyxxz3~j$<sI`($r18snSC
z0!5X?+=WhWi$L_zS27Tzcq*w9u9^>@N-0PmotDkT0Iw9=1hwef@Kb@zr_(q&v?5(0
zC0F{A9{G+vSi+rY5elW)CyWVI3suqo%>w8;DTxo`an|Z8?<RBlu@_u&-X|~mltq;H
zhw(I**;?gRn~@Il*vn6=FYVtZGKp?;Dbo9_{ZOHo4$XeAj{T=i`f0~o9`*CA2m($+
z{ceq5Mau3If7xS$p6_b*l2~F$aO>+Zx}tzLw!}0EU0M};k&h*tEa+T*7!TklafsU8
zK4$-e^#Rg6UgW2j%u7~5{2mm}87Wls54ZQLf`9zqifg>D&i|Cx9kJ+VD!KZ}f;5)T
ztm566_)PyCUab0*sCzFtGQAD;tecH>r`}cPc5k!UeYiEjD7I=2g;a(!XHjcix*u>R
zaoM`~UOz%mxhB9COsz}eX!ry6lP{XjNYv9`Vti-96R&9fKvI-}AjP@vkGXw7E=pZ_
zNk0&ii7Qvh6CqXyf}(8v7TsGIZs!=uONxey$&xpixuBcvr$6@M)NzZM{8xrHQ=#~}
z0Z+}(%<iB|(IrSPWYB&xf%i2V_^U7EXHZ^x<8{%4?KjwYbxe%={Zpe<NOOL^pTh~-
z2yMwke!{-}_<N<Q<2%e*pP&?lE|Zzs!=}?h-TIA%^#)hmqE9h$K<N9c1B3HsVCUZL
zr9SNKRnr@W=Ht)7RiTQ&B*C~1p@0+gG?cUP45|Fz1mM?T{-qU<O)4C=x-A+j+)NQ+
z%UXo#YR`+3GC-Cez-(DaqZZI>-JgNaSGdGHUYOr-bbsTye}AJ~@+@)7`{Bmg+lpLS
zck6W6`)5z}M1eXl^PxFQIiQwTNlnMW|5>yWPG)=+4PYN7iHeUluJ)_ToQ4E0vs_vJ
z*}ijjts72r>93CFzLkaUFL6m3=l^t}{HmR6b-Y;<e07b(VnJetNCXHM0hEXEyKAAF
z`%$vfj_K_=qWMWjA;w$aFM&nQeoT}=#71-Mf$pGg*VK-pBn<`hOXyJcj$OujttZmy
zY8^v;CDKY_KU?nEte11<?dRmZc_Lh|WTxI5g{R)Sppnl3Ti+h$cB?s!VeJch7VQhy
zOgh<z0I{zNH=yTjvpLKus71wpGo5Xzj`i&;>0qMK>NSCbHtWG|_qf;9d{f=N!6N%m
z>I%sGqH$lTU;}Q4D&GDuZ#I>qR}E}gesai79YFU;_QGusmL$Km(7C078Bk<VTtJb=
zVatBlOX6U1eiS5-Aw;%Z=Ov40Kvo)U?GhC%bvv?vPvkjc2+%qPk}LzLMFhqjKOk3W
zsva}#XpF6b{7)s%*0qm+w2t%tQG_zeZ+v)-oKDv^9LxN)*Fyf~1gU?l^_rC^pi=*O
zd1Whzck7kKR$ukTD^GfH>9l}0q8IZ&U#{bdg_P8U1SuGUG|WdyBGll21z*{Nj}s>Q
zdOPAiini4T9&^(iWcwelM<CCRgV`5qDV%_t=D|4N`I1#cA7LV@9pA||K_6Kg5(woA
zC;#>@tLOZGL_G(xlZfBx6S=C04|m4}tXeg+5fwOAG8k~Hw+ep_`}KHD#$50pHW709
zQ)OL3G*&Ne8rPXS2Zj<OM20-3^CvC#OrpK6#xNV*71TuDI0J&50=1F^8hOgwguA(n
zqa7SM1dR!Rg-#$o7i9s3+I1w<!w%P`zFF7m9KOJ1K!TYjahz7LXxnVQ0Z&DEEfopD
z`by{@Iz4SsxI3WUC-YW9g+*I+5G!`LHKeBR{+DoDM7tQzv#K)F`iJ%=@AK>*jat{R
z?;-<QRIX{4jW%7cZ;m|3#!+74{3Z?BH-pt@QzO|E_!iw#9Ch7P$*>4#*1sx8eL_d}
zVW#D&)MxWJH|IVSNm*+Pn^{f%z?$=nQMIfIhzsAn@$<Yq@;=!Ds7;C5eQYUu@cmQo
zn<abRR@I?bmVyU0cb*doBPHVaD%t68<vy-<D^x@)G>HHxR6mhVU~m9J422m*N9k#q
ziBbgh!v#V;T_%8px$bpuc`|+-Fg-F2?a?22lC@Vc$5dzRVX36d?&SWjj_*Yk{9<(T
zcKzGHG;NqjC{`7;lgFqlH9e&9Z0sC<wNWJ;u_K}K7-u`heW+@`+B&inD41!X`0ob(
zT0^qs`tPrk_#KmFL4yDkwU-%fC6fdgKY+<&y}>g)Q^FRDUP9dD<Ns5#xpS-2&#x4i
zBN|@9y#JKI_30wM;4m<E0sM6*gQpmO!HTK2wT1hljlpyG!%CKNwc8(W!%f4XiwI?L
z#UXHsvq}YXST5<?3BoOQENTD+YQAeQ#^Tq15Gs-NAv)?8@-eL+|9%|v5yWkr0cWJ#
z*0a1keJ)NWGF&T~q}9d(Y<a^y!Ch3Jz;mecj@TTjhWk|(itshy*o01gd*k)%m01G=
zh#Q)LUW&!>x`U@71<mX2-J$0!gV#gFA=#$FUlBmrMeXrhaN1SBss_ca9r)SI&8{$l
z?^vwXt24atz^Mp!|I>nZe=5=zH)rM7&xu0qArQCjyq#3NB#C)VT3o#(4G=047Jv)F
zrKNJ|;Wyr`!;TLWt(#7hgm{^oyhv@MOT$e^dlM^KceWf51f@cBOW)aNJ1j{g@^M~e
zsCq2F86k-vean76+|118F{pZMLe>TL)Nj{t3X^$fr}ca_<^$lwc=PIdB`Er(z{?*2
z(!-9?O;19KuE(Oo*)YQ=m__fmLfPGC@#oW##YwyhXOXr~xSf*;jn9tLqw&E?lv;xl
zo>z}kk<0HvkP9TEs6bS*;crQrF}GZCG@p%_M`_2dy)$C};Wq@_aaKG24}OE~AATcC
z$Z=2$3PT)>EC?mM@ys7g;A!6&UTxiUQ>(hpo-93%E#C37+K!i_6r08LIMv+#T5L3#
zG-++IuON+Ln9Ihse~EIj`7@HbK`Ez`O|8Cj2(#zI*}ivirRHjoMSK-(X8D~Zyj$$a
z%v@duY_i4xZrn;atX&2T4&S>meV(J7<@5q^npyytL!sod=po}>ul7=Ntq7d#@bzGx
zko9FY)*uGSJ=$os>dH-p9{hO&hhkrj4Q?pVm_-MAt=}xCCWa%ls_Nt<W3dh&;hk=-
zMe^F(q;g*s+DgKPGR4MRND3z4o<r_}d9tm3slk$>6$u$wh~Ks2x?!xa3R8q;Q8waA
z;QYV|ns~SK^bd5jLvYScdL<F79U4<=(*<>2sCE3pTRx?30V1QmJ)xCF=Qjnk(}<2k
zFJ5?tJ|dg>{EN&U0t!k@9DJr7q|JVQl2BYXu;%LryV$}$keIhCsBY(lUwudX)mAfj
zx#aS%R!8<~Bm)(aZP{tbbBf-6MPf;cY|b3lEW23Z>J>~oM;_mOJ?7c?9HsM|9QM<d
za6YTDM4kfrlN6B~MB`;sVi=bY0KtMjjKW3IffOo&kAd^`m-5#RpehiP!Lu+ZF)Hg1
zz*6V}F_1!FqA<8#&Nn=W0aKrv{$uE6LvcGhxNF3x5}YMv*ZZOko9KZr)CE^rjCVX(
z%chEUqp3gf<J@aFYI7t=tI%9I4x;h0M@->4s`m1h0(q&UunJYN$kyhIS;pux!JJ=q
zrD)fbndysPcL6r3>QfAJU7xT})|FHqpoQDqX}ml*Qw<);Q5s%qeW9Zonb7p<aYIS-
z{S4qbRIdYCVRJ$ow&|2FYdyj6em3oTp8X#U3j^_Qy1xrB++BBMHxvfd8Y|U=kGHeS
zR|G*4WAmcS@R+jPR<Gd$>xLf9G+|S*wL}a;dT0p4O}JXUV?X*M@pW?M5#o2x_>cCN
zhZmAT*dTrob-b8K-DprY9>o_fZ<_s^YqaUKQ!Zyl)tc!8)0%>l^G*L(#pNGFEFC+1
zgB4}~P&AzR@^!rEc$ep?UkP5h%?7lfLXPFZ4S!4G>;}sddL(oBCBBLkIJUNO0X6b$
zkv7tj{!Ds3MvGPX@UlXxeyWBY%ir*Un%Jt_QRNaV`k|~kqM24s-R~dI*+);&yVdsL
z{Y|ok*q&;YSb@E5AtEsk-uPu7+EWH#D(|0sD@8szZ#j_pqZ&k(+3bc;7APRmr{uLm
z)Xl)sh6bUcrbR}6%(x0@*GeZ`D$mqNM{l#{HW2+IFZ%d_!-DjE-3nK0h)MC2!Rh1T
zCqrOf_knlX!>T_ERq6v(XMU%s7HZCWX6tN8*snI-y#eG@nZ=#=(tN^(V2D?~>TnIg
zWY^D~$s$WMOgxY{R~MDX7KC)bkIc>nlt*@^8t;?kOF-5@Fn!hzmO1OpSjT02a}*k;
zW_KO+s`%^|)93`HNhlu8^K+AeH^(i!`s@RK9>6imIrBS5xF_(<HAO!b>r2QEi>T3L
zgs1tl4cuhI$giaROp*z`=B>+Z+_rM0T-Su5`fonR)85+X1JCUl`{7Y%Mz_T`b%lg~
zH*CW-vg4$?{r!U6)8Uw2+c+#M1G%*v^u{w>cj~e%4(u{*SsYTrPmw{?B?UPq!PTR8
z=KiM5U;BEEfBQc@5i&SzJmOzEqHp9q>HqDCG6O(~ZNhf~X0e`E+gR`ck&%!}W;8k<
zQxZOR>oImn%?{V4DH)~?bJL!8Z@-#>kyYwCJvFhXM|XbE?$|Uy^A@!_QSl7S%G;#H
zT*EF;%YpEO++UIy$+wR<2Kf|{Q>7aI#un^v2(2%Zb;Ju?kUcwEVg3>9z-QeUSGDg^
zR)(P}fu3syyaw^@DO10U9A~qcSB;aDd6)YB>dO-M>a}^#{k#3TKe!*RbORcNCm9}s
zt4&u6P3~&}g=23~xQFt`OnLUwu&iXMe+?%I4~o{V@j7qDX*ZilK_4ncKipc4zqU_G
zV=3NRfkQv7qg|=0i=<!?qd-(BKyv%DNwWnd3oq5*TRohYSY33}5S8y#17O}X<}0y8
zPLpiMZ95sYW5=mvV_*!jq?#s!y(6l*794607Pe_Z(R%gGbU4K<C)F!w)Km#WJ#yI(
zd?25{W8CKo4CArN2Qo3-wsfERVBT;I@ein`2+)qJavn_V3RUk9_4mdD@HkyIJnWOQ
zmVkbJX+N9$9~Ao!8;=24ek~#1h;qVKxqi2tSi^3tG7CP8-E5{Tvx+Sq-8H}odNcy)
zBPc(n+kBQsQbc#gtYeX)c#Pafah-_p4M>oP9wo7W5}^W*TtQILTI3|UK5-xC4ZZ6_
zWW*StCT~7@U9ia<M;w$5er5n!IMFv$A_Z)Q6ISD67Y*0H)<l@GOUt=-#+xP`9_H`N
z<xtE)Pxl=?ug!jAFW&zrFk<8EB&?k^=j73;0p9jPdtmdFP^8robAM$s60e}lL=v2)
z*#ymFg_I9a$-rh-I^o8n&(%e~l$G=lp>Rvj_Q~aFmn`n5?pHG@fG9q1ug(J`aT}oz
zHsNOs>fsx5KQvGYD81b`46UgYFUkP-9@KO@f|I*a*9%wNqVvm)rpW5<=JO};+#^mc
zE5+#%SW~Y%iU+$9Lfl3pshcu0L^)@AN{r~^@y?D)kXs``Yh=`(AMF(vxnxd)TC(Wn
zK6_P~UrqjQBh_W4aKC=}QMhT)qO~V7;G!oB01=I}9tmxaa|ZtC&sk|>WOE~A)Z|2l
z(KCvQ5`#quI#nUZ6Nag;jPM}DT~op&WK?TV_#D60@;gDhr2kk$?QVp-X1f0c{p-M6
zQ~Dp%ob%+DiDJurr@6s=)so1PjP(QcLNV~3wb2KHVf9UK%~IPKiphhXe(3mh(VS%C
z9ymS@(Ec}jAD4$;9W*}A!6B5(NG2#V-G8upR{M0>Z=qT3b#=sprwTP>7chlYO<%sV
zoV~ZUxMotS+PGw|ug4KsoJo0b`-5Oh06cQ%4u;fX31M=)MV2tZWY|Qt`Lcw_>=?EP
zbJv->lPVACjMc+sLE(W~9+vtF$>auBTB}vYrwQ1Nw~i4H1vGhL5XceRk0{YAvla9D
zrP?GIe-5`cg62p#lEcQ+`2T8)R0GLN(yeC}Q0_>B4pTR8KU)|~Kcks_sjis4Dl+Bf
z$&BzgDp7%acR|*kx-x1H#>#~;gJ-^B1xPM#S-cRvex5Xa3KurFwsh_rZ4v{5Hf~nB
zZYp+oo%B<vhDGTj@jdT6Z_bkyqHHxCQe0L}GDEOcC9N8cXFB#LwfW#u)V0$QD8deV
zF2Rf6pDXrk`GXS4^pHLGr>PN~+$S`}1R(rTAl}4GU2mq-Pm;MVC~5iO#2<Tnj`VHy
zrcmBK+;aKEHbk``Rxd^zQ$_D;HRAZ<dN*}uGxdSVDWN)sMcsim5w}pYg0=kmVDW*h
zdViGbN^Pa}5dw>*VMH}~xvYg*tHC80s}A(@%VJYF88L>q--@0)Y?tNybC!8KK1=3$
z?0)$)FH0t&czdh@(B+ilvhfQJ65I*cZy;(n7tEp=1|JQMU#}a8SP<c0E;df|<*Q4J
zJP2)vyqWYKxfM^Wmce*U0RPq>kxg(v0VHvRKle3TL-T2eGg6zx>_Sb2r@<w627omi
zz!zRkRK~KVT{K+iWb}RfsM2$_vVjFnO=2s@AyJ+hb<crpw}R63rkjD@YW_hl<H!2Y
zer>VV=o+`4gdc15+}Huo({RT`zL1&)OsXke{k+q?jb4+8^l*2Y;d?Pz#}dH5ZzQR=
zzsv$MIgJcgp94tsM=0ED72OsAB@*z;dOeTCiA9t8AFAeJMB?^qp_h!bxlw9qil3RQ
z_Z<$T)f$&Kq}P96DVt3LMu_@88%bERkkalcf;GiCV?fOkaaC*8doG;@^_hhd@t*|?
zMSbC#S(|Z508uP33Q!z0Wx&R>D%@x5_0)hXp=JeaXGBcLB)1iAbMZ^VmqFWZ3bo`v
zo^<Xo`)(cQqcDg#`4Q?8Dy_>BzRP$d<x;Q>+Vr-~Jo+j~!OLGgEdVV20pg)`gc0@l
zMOcK+>Bw!{^pSA^axABl8s~hAMHJrAt)_uvd{mYb00s*Y6m>hEhM5F4?PUcOs=PZI
ziy}?yWKIa1P^aul7xiQfbM<!%qk|r0byk?4twBUT);*$1MxPmfyEWcT<4GSc`ceSK
zH1$4!)uSj*)N<(=nJ3ek0;I0zkGjWph(f_rWVFt1E0)L3_p$)x%bEb9SG~Gpe?4iv
z^Y%~8sJRILW&N-uA8{W2Yg^+|Pj3rU^Bkv8vyJ`)x9yaJ2H(W&taReS4pQx*qiq-_
zL~GtHGV5CSb84UP=VRp@scH<2v^ng5587)}DrV6;hC&Bl`u6dF+yM6bXDn-?^|;i%
zIqs&cE?29%J+b)PwVG>ERuO%R-rE^o=RWf!G5fYJ-*uc0T$u*cWT6q|9k6xo^G!(j
z4q7KYJU4d-&Xc!C?Pgs%`r*En7g0Pw&eyAJH5V0LS#0dH!wz`Qval#nOcOXwViA}Y
zzwb`^w@^{un>2bcW5Aj{ae>*9weArbhG<v>m-ao@xCH~qT_HkE1blGrLOEO{ex+jA
z%RxNwR)RU<B8$ol)qI)_&rlv1I$Sios}8~c1SdRgL%lEDKVRdk^1CcH1?wSNLZuWm
z46#x!q`dMiZt>?F)hb+xuw3#?lK#y7AstTMgxe(=LJ^h7;f9!iu`u;^pZ%>JXnhQt
zmDO5HfaERd=K{I`3R`R&lW_)o%8z-mSkkZK-}GBY^O)_u(>}P5dmbZ-_DsFxi_9i#
zDG31*zpUy{F<HvaO#&_TunyM4Bk}gJEo%)|qJl{qT<1YmM+K*;3RmUqncKYa`OR?w
z>Z2Y@J~sgEgkV)fCdnjnIeWS0i`)O|e@;_+#o2=;bM%PvOaN)N>MB+PLi%XF8zT#R
zPALQNdf84YhFoh&EyYSNz8;Hd{v+nyrzRM3s6%g0s)t6kVlFq<n!1vKn~s~86=ChL
zuHdyK4c6Q8D_aX;*JV7(JkQWJY?HQ-7Ugao2wsH<o<&9zyG(PKKBgGo0c0uCzaw;{
ze}h{$d(abU$A68edK^0#a)Y(D4W3<3Nq^0``uH7S&c4-tq)8>LGY}qBZ~F6_@mbZ_
z`!2;%D!J!AT_$wh(tNBhd>FbFjDL9zi2vInFcDvT4?Geiul4K8|NB6T)_}MdD6xt^
zYl8GT+AL8!8Z$57JV%G8>-t{9h9q%<H*$;LRG-yEKgV9a@`L;SUPi=yVJg1INlX=Y
zG!Gdsk+`JsaU1DU*!M?in)aJHOp$%$K7lyg8S`Z$R7=lYhS*WTpi!L6x478ICp#lp
z?Bahtsnd@<{HO>YNf|A_S$jtHh~~_K1mZ<bBy$>|06}6NQ?VSUH?6#CTKT3ROK_F;
z9&7Q8hm^Oi>`9Zu8j<D}?z7HFZ--Cs_}N%3+K^+J3GrRX92z!KJ$_eutR@G|mmbXZ
zHf`<vC+xc6qf^pZCU~P#aWA5m(ixZV>&iRXJvN^7kto!KZ5KGsS89xnQ?|J>k?wY+
zIEHjTZCa_fio*Ib$s@WoDZ#FgTJV5{W#Yd4+2Q&6XUu<Elda6oy#Irmbor+>vAccO
zwf*1j<)<d5pz}iecLCF8rJMSbzHVrn|8~}e@&e^8JBoQC-|TdqlRT{V%%ZvW;v8(*
z^i|eX=)qQWR}B+r-IE5%xmr!bDU-|$p!<K0n|7mM?QO817Y!Rah}<t1tW_uceL-(b
zMmTQPCw^P^CqyK1n&H0mGrIoLsm^jFTPVJ&tWtarC^_WF;(m=DXzUT#Jkn8FLUYL-
zw)Z{)L8s6eK;%-nj44L$o?6S)r9o@TbtiNQD?Q&eLcj-fVG-eH{zfe{=rk!>opIEH
zWBA%<nmK#lB8rd+NU$;Tk8cS_v4;^@nw{s<UCj5{&uVL;-}^(_h{7NYAC%gn(ZG#A
z+B)gL=P@eKpMPZCuf9e!8jb`mk=?#6pPD7NBzURA$XaPey{t7i679huPGFGZil)fG
zb~@7gF59Je*O8nH>|k61*wQ>!aP!x`e*jzybDp!mROxqs*M5^B=YH!yH?=<&q0#>b
zdcFP6O>OPWA}sp9mpP_(JRPT*W6Y14l@15Bmop{J-&u>2?93jT(bHZz_eKzfkTb2~
zQQBOpdb$68U-3{WErw^SpLl!`S5T;w;YwuLFkFo7V%lZ>0v{B<1&F~*&v{3T)NDhi
zhMtP{HPsy7uaFm^z)*}Ik)Uu}(k>U!y%Rti`Rr@@OK7?cyU?eF`Rs}jRrq;>4+ow;
znm;2Un93z=d;BZ;qwbzagW*W&Eu6x)smSuUVOjS<Nsmr|HPE_!LewbxBtnC7tDjnE
zBfWNnIeV93*Qcc6BV$+u<YSpqhUx_cS;3$Kpe>H+L+Y<2au>P0R>8HO&n$VDACux>
zPS>lPp^B|kMM|&9DvhfD+cCiF`2AZ80dDp|ZBDI^qJO3mOI1CtUVazntUL*4^^El3
ziX{-9Sycd$cjini*QI=G4i=)QVg03-e8Ko&|0)FZ|4XQ*BNlCY|6e7WNsMnOyJ#1I
z5cE>1J`|Lwl*1wTmweC|Bq#LAPU8rD_+^=@>t4bdFUq9|eBK<jpJa%UxK_@@mu!cb
zrihXFo=sv2L~8@01lj}KH1sf#1{UxiWF(yKaBRe*LRL=uy%1M-I-=cfB|89V<^owX
zIJ>l-TAh!(P&}_<*d0bpWGS^iUVKrWQ?&$lZxOy2XB1u;4cvXnP)BJX7IyfK{sn!3
z@#HHOhJoMjWL3PjC1ro1DJhMnv+EjfaS*VhUom1b%8OdG8`m1VMJ}}js=1w!q-yNc
z7f(qN0N<T~>}vvFZ5=-HAlog;>2FKRxG4@i45FSN?YayAMLO8#!yC{DH*BhSUPd`v
z)^`45Z6eBlG=DlGezpt${XhNhpS5xSX#VDeoNc@Q`%wWS+HOg6(?qUf)}JA>!Ng6P
z(%4x=0sB&f6>JZV7&@y+6m*QG^1dBhhX&dT{{6N0gDtVi7iFeYm*vb!Q}X$PJx<$S
z19&g`Q7%qH50UxYE8}X@#zxxSen*NHOsrboohK%6C|&*RPj<j8)Hij-)C=tcFpffP
zEE03tlyl|Bew(*v7OmJeM+)z8A=4-v1e9%$MgYBUgU_umD<PPMu#Qd|OSA_4jQ?AS
zMr9krw@nR;rcydm(RWDFNr_|njQ0>hiq%v<<bd?Cf3p7Tx5ZUobVGs&-tY{O(j-5R
zexhmY_c|o;bx{k=S!ee<TnTCh5>a9LZvmr0L9#)ll-EDmSlvQWw-<wj(=%{C*R5>(
z_lO<8PmI{0ZCVK|x|JdGrrSn+SImPNLWlYCjziSmI4u0OuJ~BGsTBx2yzB!qq{yZE
zM)CeRPqwhb#=rkN^3(c1=gCrUsPg~!5vB}0s7_G`OvzqZyUZhCFqwbm-fsWl`Fp_S
zntBerxdNjm4wXV1MFWK`FK+#89}W9F^#B=hpb*+!Q}}^3s;CB;b$p>eR%xD0qEOr2
zNseP#ry(ob=z;Bi{XM3mo9&D}bs(Z?ZTbnBC@bgoCxu-@(JlRUI!HND1n=2lU~faI
zuW?3`p^Jg^Jz?eRk6!UVNHgEBLWtgHzh_dc>_Lg6-9#lE@%I-$DZn*&(LEjzC>$2y
zeO5-Y=y-?yHXAvXz7Fa^U5-N@{i!RiegP;{6KQOTaUTea*20b%Qw%>1JxwKiLma1;
z3G{Zl-x&zR$3+0)&hI(sKL)bX+fL2@mcIWj-&r&KpXzwL*D3ICkNh;j1UT2%6}3(t
zF<hX*V>Vo)xv~zbB@k0RI-7XQ<D<uvgu(=+I9q<$`gjZ=?d?&Vh^0C`AgbadgTKdL
zpoQJj;fUrsAc*a_)xtj>c-f8sNyB50L1jA!Vw&0&vlX$CXo$y0btJeKz#oYe1oXpb
zx)QmB&vmPfGePtf@|v%(kt_O!l+TYXG|^wCL@`sd-mjr~CO-l!(Bw!7HnA|e&YVtp
z(GKwb%Z&HMMB8;<g|%&r#T#@qAqp&z;-qP2(QD*zr6j@V^Uak%!WPHD;Tu`5Dozbu
zR=7EFtAM$LNI>=<^2OuH>Yx9Ee0lv3`6A%7vf}V>kMs$n0gAAVSGubN{eQQiOxs)M
zM^obcD8(pG2td>?X@fa1#v~cEtJ+BXozCcHmT4f<K79JJG_{5;LERcoXX!StwW~Vf
zz<VStXMW(W9v;Zd3%{ioX4F1rBU9<FoL%xky9|P7WkJR{wjtVIoB5e>qi7h0-qNvt
zCB-44O(FfN6B-GQLe)TzlD)3%kxU&^rYG-8MDuhY-$jjN@A{n0_6EaLg7p8P?Ja}i
z>bfq##vy1RxQ7IS2MO*FT!I9b;O^RJCs=~JySuv++}+(83+~V)bMrjkJ2mgrH&Zn~
z=2zFP+eOjc_nfo$T5GR$#MsfrbPoxp;5sUGEkgw5!l-s8iOBn=TIvWce)wawhObS{
zeg>w&)w3Lv2Mz!2#<pUK3jZIvG2?%`v7u@_!a~yjrCs|3CH@XdVOKgNan}TxJGv&s
zFjo{Ep0B*1g)V1bk=H-ry6})81UV|`PM^PhWtB(t@pcUG-u;5oTZkR_ICx%qUsD(l
z)Td6@NtND3lfnIUWCxq1lwsMEl<Z@n_oJWhU)<w7)&Oa!7ns?EMv|{cleeua&d=X&
zgoP?>!H3wkO+(NuUt7JL!hW`fi?Q53sQPp`BKc$L;~W0B>U7i`9Wg=a-b0NxM5UDh
z(_&W1ltOP3vD4rD+M-SrKIBW-|65AcDSNvd$Z3>s%TYXTzFz7j57$LXIjx2dACbU{
z{YO-TmD$U4^S_H~$VUFrtSUsLH}S0fZ*O6r5CIH2z_JTXJ-~K!xD5>C0rU89C|L+!
zUQEDvy9BxlSvN5Rq`ku}yRotur!{>y<uW^V?{Vpcl8~3SRJ%xQBvNRYy|I3=qjf+@
zmqQB)_>ErZQ0w)fITD<|(kO?$Cy?pq`6Z{H!-etbH40-mJ0W7~M__M9as(kOv+FLz
z-eW}3UtInW$opjAeaN05=ox~Yi6{1vun&7-yp1TA1{AvNK(6>phW+7_eE+p&b*#a{
zjq&F?Q5GHT>nfTK9JP>_72<A#49&UGrNHj<|If2C(2OYWFaYlkQv7;7N{IoN1nMWl
zF|LCD$rfOIIQcWU3uAI0#4SSl*b{k<4nGErtr?1BC>T_mR6gKfYe9)NzL0XV0**Wx
zRagfWEOqxov%yjrFO3g>#$xb4rbJ$IvbxAAyinJcy6k!58dLX9K_D)>Qli^@^k;8j
zW#G0_L&fn!w!Oz?ASM-l3?2U?{P+PPt`yyvYwVku;7H0bxeg?oua$uA@&}txG<||c
zNF)a3y=1}tqT~0x-1}*NR>5DzA~Tb47WzLxB>61dI{yw72>3b9|EF^J@7Ly}e~SxU
z0k7l!G}flg_nlFI|C(l5MVnpPOjF2zKTNt~U6*?T3C|<yW%nb$o!0Of)$e~_?lEEz
zpVXL+tX2O^U{Sb91bm6F_U+?J<@5`o?K6rWPG%<hwQfq{7am*whd4SoqGcZ^x7q6L
zL;6a*iGwVX1ZM)@EUk!60n&TiM9T9|-@W{bftvslk|6Dm_46Jf5WHWMHDI?n^jF2f
z0cfIgi=O5!LNU7QLXUH5lY0UzF>en(F@un|{6McrnbK)Pl$eC7qHUxi3*Rxj(7R_~
zyd0HRiFAy(sUdL4A!)#Vvxgw~C>yansd%v^8g|Gy1p3khgn@Ysx@J}0A2HJMrQor9
zi=5||vI)HO*#pe`-V3=6^43wffRQY48XcUp1|9$1w=VFA3n~AXO$mo1O8kHJt?bx_
z2!#*u7W373B#C(c_1>tSzgriX5ZLZ$4@X2CDaG;h3p!mFnZS;DDy`sgFbc8=59q5Z
zk^cwC>`!qlbr1!!1OT%?h|_}5TCTeZ-US_d@*m1cIivgJpl=}v+Wd86tT)7qx!&W)
z15%O<{UeNkl%!s96rT%DrTc^TX8P2o5#n0e&kkP$8TNCjD7A6uATp+knxSHm;^>$s
z@h(dviK<FAn_Ho=^1^ikbReQ%ym9aQ;{5s&SqoDVT$Y^KU7gMY{ot&>D;ZqW3vG)p
zQftVM1*&&^pMaY(5-07)K8fC;^n!3H{5|;oaq<PGDtUch?_B?Hr<xJ6)4TsU_*|yv
ztBV2UJ)fJ{j?;h7UpnysXJ2d&-Go9r07VvwaLkj#PoB2{==e{m$)ElQ=p$9VH-W?=
z$Y^-fUH}x6gi?9%8#Mj$R)h!vCLV@QX<+DGg>oO$ZW0l{N%Pl**aSxawF7!Y%m~(|
z6q+*iO^Jn8@>=^U_Qmf0)RnGypk7G(*|E#G_Nua5St|-QX(d+oMkiK%J3h~5sGFk0
zyQ$(%G&ED5@=Yin9ObmY!HzhaMRYbG{3?mW_!FfFYCE6w6r(Ir&jklwbx}GQ;ZPC>
z&u8D)%%xOkS1x9g32WowTcq%U#zGy@=@NgC(f+ei$%KA59{u+U{m&I|!oMq(jNkEi
z(uD?MM$VV)NdKR^|Gx5Od<B5b6Ff~UI8HrX$$@6$3jXU=$}o_RA-qNRO9)aHHuNh2
zMz@js_p=rX1Dqpcl&^G<s=zAIAeOui@9zCBmh1hXAK2RnyV2n;Q-tH9S8hyaq4x)9
z16f2N*p&LTLzE2=cZ(lNnfFoKGl@dX-sHX6l>P2KF)t%z<>ZCZ@@OOMFMhndt^dL-
z87dy>o1{iig7=1US_(T^F~(?^c6PtFJGe@eyG}K%sxPDZ7pJ;6o$PinkeGQOGX0i-
zFKo{2QWKlKJk%qMmulz&4pLya`twKKF4w>0>@7H~_CNjnf6LYe^Z%zrpVrc8gr=Zn
zX|*MJ^+x~qWAyiG-WyrHOw-V>xv;9j@x-8FuKMR)BD0Pm#3q4R=QIMJF><fLLOK6~
z@DdT567q6GxsAbHy476i<-PZb%36!7Z5IS?pj9i(8R&uln{8l1M*0h0#$8V=8L=l{
z*UKyfLj*H8TuT0jpU8UJyRZAcgki+64slk-xx{B|XGvw=AI*rtw`}sKQL?-6oGX>`
z4zoU<v+rf=6RCQj9rCN%%c0ZzLOoTcjbL-zA%-zhpF=f_TvT>-VDH(qo3sgXKAIv&
zk$b<TmzVdwh?g)bio|~CNrVOoyKqu2C@3=cpz_=ByfUb^r1lP=(_`C|L!o#ze2#J;
zc_o^0n9AnMGr5nSrM&uc=>HpjxXi~j2Y`J8iWa(x|0j_659WwvP6b#<-0wCQCyIj<
zG`PzF#Az2R<JcL3fQo$)Knya5kclC4i7<=co1E|VxC&+?6vt!3wn#fXvdiy(vY_s4
z&BkgTP@%Z^mqSyQ+;e&*t4-umWwr#)hguMkV=CrLfQ)g`kjf61IV&vv<|e;ac};HF
z9kj10K-kRMu^Qi^=)2*=sKg_yalb32gBEU_Y?^r%x21nqi6>UizwV5Pf1~6%_{u=%
zEp?ACjXNsr8|${{Ahpep$P57Y!!@8I6qIJG$I>=wxqn5e2eWNcp%n(~@}^g(#5+Uy
zNhCqxBVrN%)?g8tgFMgweJ1|rifkCD!3YRNt|BAmY$H!XewPaQ_C3r+UX>5*`vFF%
zp<~i)2*W@b_71Q5|BhN|uKYa)>V1%92|NGY-<HY0pm))p+fd?Gd}Tvv0=aY{p^A4A
znh)u5ck_Rz<A+n+MG*3>8X!dRKs2QJ4p@Cq>lhM6(ZsI03T_ep*)|>^68pywFkx{J
zX!3k)e(n0JH%!Np`TW260UCAVloNFQrBoVf%)7PYT;BLA<-|+7;42Az=tmY@o-DK>
z=a0^%@!CDgMHr%^)1i|9?1W7tqM&Ic`ITtoLzcWI!9pEx7LwH24xQoH2!TQ*Spduq
zlji*&oP$j8tK;&2Z(9Gk;{98SFk?ZW0piw+uI97!Q&w~nnr4aX{8PR}l1RtE*Jt)p
z1h4Yj-*5cS9)?<F&DV67Pp&IfV)r~*n_;8LjjKX1w+w}}3UdjhV=2r3Jf>3|IB=Xm
z<~VqkDHxPe!J=fm#x_CTwuHXl#o_y%Kvx9SP80>cwhPXXG3=Z6br)ek!+~4{HsSUV
zkZ{dcH~14!(S}wSwc0Hc{fXM<fOZj(F41=wga5}$YZE<Vp2pxZyINJoC=1)*D%y_7
zz~@(pf|I6APZx+%UaVu5(7^aDTt^HQIWuktLEbJS+ok{5pIUbc2bH*?4OTR{ao$=8
z1jsAN%96&#ml(%jpa>~kPd>Hxycx|gS9CAT$k2*<_u2>%AOGbi(T_4QwQq*xpgtWF
zpV}v`Z$|-azpr@094hnqCg9Y!<q7PomoL0G=WiTRCcs{iR<Jemc=JX?5{wqViPQw&
ziR<W5=Q|dh*bu9f%2{D)GC1Rwqj9W}PCGKApnL4TlIIz<<;==7i7^DSAgcH&*J6<j
zH~DJaILQ)hhR9A6Mf`OX)kKntT#Aa3QEoV%(LA-mc7<yvmN+y!&>sjF1I*&{e$1BZ
zXf)U?z25MBvfTnEAFT3){A)z6SL{3Sa}p?96vExs7b*=5NVx5oMiN;Q-^wKnR_L{2
zt$iK5{*@ID8Aqsww}^q8uqfn84evy^H=w;rJe@6*zel0?iUFa0@X!2Q9`uS)XPseK
zt48ED^~p2npa1Nd60VEmPFMT$k@}9#(mm>k9bV@K?H`|g?jhaFi*UUleh&EHpQv%~
z)OBk@s{1UsUXHz~vemeEc3r}K8|W@)v7fP4W3A+ga2kS7&yYX*7(bmrzV+ovrLQ(m
zQ@KH7tf1uZ%+0{>GYh^UvPf)*GVIwUj~arE&N(tRft|ILGu!S*vHn1x(Ivq=4l0A=
zfp3&|L^_Yc#uj!t3{)hOdw-y+#m%{tKq#~~<un{ODc7M>;27^8%mXMX9jm7a_OdDB
z{VBh3yD0D9iry3{iun%aAUN%4N09CDaoIhezmb2Zh}0bJPI?T*Tj-yLI|-Xt8gT7e
z{W|S!GL?-G&<3xpriel=6rNgCzHq__JDBo}L*X5x!Lk(W9hz~An!u}=_fKoUe@>%%
zL@Z;c9XTpo)9#!XCwBBbnD)LGbMINb$i~o6499?&IUtJ|OXlFjvS^!uI1e3h#y*&G
zzhx(tIdl|YFaPd`Xf<C(FGcPeH(Q~nGn~L|&Am8Zp~nc!r$+&H^Sr)+j<TG#%W7D{
zFo$AbFemPvvRH%t21}`WdCXF+`3TT3<XACU=?66-lX6;60kKu}tX?*0wOaRp)AMkD
zIBD)M!YhlVn*7;P4J8I(R0N!?1GE+l8-9O$I#R=b3{!gjs$LJZoJCF%;}YbhkeyvB
z)8bH&#Jo%WT$AbhDC2fI>cgg}o^cgmz|?ETJkXtQvUkYu^F5HuW|0a={K7C+F#Dw(
zSJt+Y1OykzTtUYIt)s3Y`$E#v$d=l`wUf6pv9v=OkdNr&H@7D%E6yD*htmnc&H>JW
z7wfHwQe(Z}j}LqCl<IyuLr0_0fJ`LixC3yelyV4riAE(0P%6X%tHWyne>3xMa^1%d
zCbRG1$?if~J2aErvnhS=MLS@Za-<V5a@fo7du?a4#j))Jmd8z<zBh9k?Cz(!W5WT-
zFTZ;s0&dX+Aca<f{Jk5|8yoUa^<Oay>*3@c1CP%J6Q!@`7=shIsw8Ia{xHA%DIV{U
zVY|{$US}~S<{=5{9p;C+Z}`yd3`DZ3WzByRkR%2vT;{Ia7kK3lepM(8R1&80YOEH@
z%>HLhr%O>0pNFkfqHWm4?ho|)3cHzdT>QP0bsiy-seZCMsO@TtqRHrekPA3(zG7C9
zusgh<V|_k}L1Op4#nw-e9>VS0OM)(O+3%09u^W%)R9*U~XR%((SPs^)njQwN*3VeC
zUcV9%oqV*_*$reJV6-0Y0T%S0xaTMQ0%{0Ui~}S$rUb|N>yGi|eqhqG?{+|5Hwp-{
zwU^9VsM&;IoMozwC7kzg@QLXgG#V;J?9_`!eW~f<c1)+hdpWDbU+23w=TuH97&VX7
zc|?sp{rH1sB9i?``FR;D)^xAfVB`Zfh3{_bO{viRw<9`XJ2A@|37R$01vu)wu&+~a
z@0Ca7ikyZKguTy8%YJAJA?u*XU7=FXk<Sv&g_luK*=7z~rXNgmS?Wv<VQY20Cwn!0
zF53GU8=OZi=3>zvkhkU(uSzkjR;eQl*6z&6+KuPxd@R?lCwT<m`V@<x7ssdvIi|=W
zS5K6p1U!HJg0z~Q@?krae+J!af8W_V79b_RM5X*n6Tu?}gkc=n@&zZ7G}Xf8gb{P}
zXQtP<0FfVvAcZ%Nz~4J55dFlP+C-@55*^1%`RF-4(Stq;?69`oJipneZ5j+5x{=n+
zsxncidN#ZLd%EaYl-Ms90MD&b2CG#+3{X^>(=ffV>8co4)6#*uta=AvUOru<O!GO>
z{vJrm7&%=N;17PM^)SNHDh~?p6$5$bdu#>1pEv2J_&79dZLG+5_=au5Bn1P*!0Y6L
zwM-wg&00&n%fTe|_6Kx}Ut`=Gbq24RovQn<5GLO^uMaE-X+D6+Zcf*9s%p0A`oA7?
zi*M0%U5t#6Hzb3Q^c?hg%pG22X>4@(<a=D;Kq=n?8L%ARk~x|UX`+5DmezQTdu6D?
ztvsMXi0^Iz8G?Eh(=$l7$ztlYUn2XVKMy=$&Vvqt)?w}K7N?45t<ZBcjzC%y`sSvg
zm~w-gC`WQj)7hrjOp{-JAct}W<FXYB0om*8FtS<h3^_P9vxCU3pT!o*H7dFLOScYZ
z+JKJh=~r$X#Ut>5=JzC5)y%h7#HWNu)?RKz_~_2FVEs9#GVCMn7I#8CZLUipiyv#O
zetqU!|54rk%Lp|W;F2%bLLO@Ph!YO8sSSJIiLoaPpU*fS70U|lWw+C=)uhF4<@d|x
z+&^GHr>EB{i21eBD%JnzPL{L)9Lq5#*Nxf)BAFGu-?C6*OwhIy%8yLMf}f0I(Gs)k
z+$OVUK6;3z9Epi<hTtpRQ+<SiQnEMXWBQI4Jn9WL>J#I}UyLOagTlF)lBK&)fkP@N
zv7-;#I0owiIrQ8a;}0URZYFf+BI<))zcCQME@t7*bQ<Q|4dj1U0k(TikwPub_{5MG
z%)Xbazg)q3lSi%ps7K)^Q1|YOR!<jucG{|+26pqo*P>wda)aYJr}!C8c1OH)rG&cZ
z18S_PcfQ&M{$*J&_Z&xll!j5fzo+6mgI5^z&-DMMI2`@9!qt*Qk3mklBWC_b*b%iF
zu?8XI$gDqxgtYeFptvwQj_He*DNi)&J!QQ-dhwvi@{KpPsZ;qruPe(Q=Kj_tAwL!V
zsY_tL2VU0R1EHUg`sHrPSs)LTHkHfv@tHKiQq&C5hzPF3?qs#Ob^vYAa4hA6N`X9a
zv1WBnV&sRU#&ypTxk3h~$HPLM1>BNI)}7>qq#a#I?`*N^9uOC<%tcDTVI~DPQS=tK
zBV%(}T&mt`u5C<Dg!8QJDxyrY`n7xp-?;D}Z)^dC%<*EYA!N7?m?_@+SwF4TgNX>y
z5d4>zcqIZ*Y?-$q%4F5UTA>o<v6-}zY=s~MBLMWX>G$RpkQP&KO$UpsVyd%S<MV1)
zD^ZiFWlrKy#!=t|Db(z@FLb?<Afkkj)ybfBy-LWfs)%kTJT3|d{u>`h<_Ejr!n6Kp
z07Rklh-1)0*L<HJ(M~(EB87izLdQ}G{dXx--{Ko*4uU5f;jdA@L_X!>CX`4<kXi2I
zjL9wF9D4v7?y50_YYvmaC>Pr!${anuGYSk4Cc<*|XG`$bcuPN`mxau3>k_Zi(%Oj0
zzMXq=!8w-T&ci`&q-O#S%?)Cx3#gOhdH^(O^0|Tmv!Z2+;<TN2#dIsyU&Zr)1C$bn
zW&U$`ybT<uRyNl=tVDTN2fIFGL_3xZ{IXQ@IZv(l1(dil4y?ptx31tu5?Z!my3&9N
z_3@BnDB&@2on3ENide3eVmMo4#UHg#)oM4;DErRDsILZ%W3nPVi#>tGF3g|na<AP?
zYbUqASs1oqYN-@zt#J{ey%T>`Hu)u-R(wh-%PAe|)F0l%G?S6OvNyKECqeiQF`rQQ
zvHTT1$+8oPqXywp^-ZJL<k5iwlhNu!9HWMUYyy+{Jc69;OJilzg$gqMVB4jd@lo@@
zfe)_N8MOK-Opu_IryBv~2%)yGw!N2g$9n+ZLc5;z<}L84-=MswISm&944og7`<Xd}
zX<3kx&rKN?LHggi5$_$b9-Ycy=jT@=QOI&#BlQV*CHljkgZsBWO1gF*&)dhSQwUvt
z=JxCs6X~rNa*3wQ>nT^=31BeJ$zbV|vhZQ)yp=h2f080*-=!8uwM+rDmwiLHe05(r
zLwGRf;AWAnI{2f8cZNAoE5c-!5xTO(_@oDmSmLE3oXFqVqU_<AGKlNM8;r=bqR8Gv
z1(Ut<d_5%SI9Yso@MdhH5}ktE;8=>%6cz>dPFy<Wk-9unV3;}!>N?s#SQUm&$4?P>
zt3=4`TH843wu@uQLzVJ$*12J~%c|eDsGZaOEcv$(AE)UW+5N1cXKKJ;Ne99!tADpu
z01d!XoXqnE0=wt)V-llH@)XngoaVL6!P+o&jB%kSn+XM+%_P7mf*zw6$nPaQc6)&o
zhfOK8x`Jp@dmbal!p1sIt(xv#9oi=6-CasW-Nn}Asocoj6vGGPRkl|$G*t|QlkSJC
zutH*^H9(jUG+~~1Wv^qqtkTdyPH^wNe?+FHTX6dQ_CCgy2FJv^!@WO1cEJy?QtW0^
zkyT$unv~BF(3(RI0+qmaVn-KMWoj;cM>v|KFB@M%i0tM-JO_)8LNo3adKN$+^7d`!
z&T_k#`&k>3^f&Y(4|);^?b0#rdV|D{`wBdzabW#h@=A*uzug)&2oF)GO1`WX4$b~s
zV<ER+i{5&t&(Q<v9fd}}Vg=7gxoqAQW_XsMu<vzn8}*`Mi=<QxyhF&6siet0%RxaG
z5kH#{Xl@hm_F;C8L=W!^DAw6~lu{qmR~LQ3YgmkZ-hg{b1ZIH>O>i*ZxmW1)lW$rC
zv4c|7(hz`Jk3o_ROnwLt3^Eo+0K3bA9iP<E+<3aq*wd}iQj4q4n*?DK-`I4o{DA~L
zl?9N+>nSq?P`YkQlebnwQ=>TE4{mMW8o=tdfV@#p?W6VwwDQ;_`|rEehR1uUyu4E-
z&pnBzKy%>p0+d#)&b;FNU2Uc;zABtR9;iXNtu@e@vuKlcCMOh-AO%Lv>`o)5%@r9L
zp2lfqOXpAC(rIv$WRLs5x{nNjwZg*bV_@c+rQ1z(P9R?tdMw~l#Zv}dr!Nf4G{PP9
zSK9j=3_bQow?rD~k`n?)9p$oI*87Sz`fh7O=YuhXhwJq*6%yjJ<-6I2fp)9hfDISu
zXvH%^|A-I}A1ypaEj$j*uwTY;8XQmhT7Xxn!=;h3gZnM%Ttw7_O_wb_?xUr6g@^Cq
zu%;qXi{5G^x@lgyP)WDmSf}B9Q@XKw-Yq6`jm7>H*tEhGTbIWdJ@pu$$@xgNTdx-<
zo-%)Hkugk-(0xWp>OmDh*AUMRUNSQ_`P)Jb6i)s={9E(I3I)IC=U)L<i4(|d&vCyF
zu(?7{zvdN7PPmuqLK*~7fgkn~ubRA@FNYc16&lq1E_}v$_oadI(WVy05;Ew*Jt94k
z#;Ypy2j-lkvAD1GA1{D})rJSXSKW}*)Ch9^prkQj0+*^h1jlsvs;`Up<a{ngu4slF
zSg>Xg1sBnur;R_Jq*p`3s)66$gFN2fUPPfow3-~U+W-Oc#rKtPB3HQh?``N1ncho@
znoAZ4RQS(>lCDoMZ*sMgI>qrz(1L>$yh0j}G7t(xM>=s<sZpt)VU5C+0jGJcx=^9F
zH%Jp%x1_tk+Hs$g`F6wQ4e%6>KY24oM#DK!Iwz?J>=d#4&A!K{!Xj_QeJiVJ&6TzQ
zdIsh}%0DK*|AH7p{-l<Er-{>Y{u90ZiGQM0vsKpIDHBcQSs-ToS#hD^u=SuDaq>NZ
zADhAW#-i{-$aGaC*RbFp=GBT~$Nb@7BH2uz+hhsPhHgHZiE!d!kcEmPmI;Wnn5b{p
z6}Ao{;Yzihv$9)ncNX=2xUuR9GDViI9rWTJkNoqA*?g;gWPWXW3}Wu`QmC_IoK;yZ
zNQ23+Y^ImtWd)lS2y<CBo=5()y{yG#;f0TN>xiZ^>GiU9a;=MohiP&gH~im<>y;(p
z&@cnD>k*kLXrMVEKIeneMn^le-8JK(i|JA7%aqG2XpI`k;^0qLZL7?V=ZBL|qdILW
z`ab)$8>X7T+TAc+#2%8J!4Yw;CO!a)wmF!l2=cq09{=rEsQ8qd{oNc*u)!{r$pD_c
zQ{V_drSOfj<Tc&R#s>HCeWH{g&a-`edRfW>EQxXi{cFqC!C)mv3|cGKVxe7n)oH$B
zf1W&WoyPGDS7}?P@1838ve`W@EqPY}KF_yh(Gvm7#;Ij3vAwzR9LcM9g|Vk1u?1uh
zG`gfCpgWQx7B+cRCkFIcc=afOP8tbL(>O4ON|f*3*J5$Gsd%EU7U=<TG$`1XzRPxa
z_3R~-`rjYx+Yj>6fZ=-Iy|-$Rmu;F^#r9m1LBMZf_T~lKziUGuCA!knabGQ43`wi%
z%U8ezoZJ#Ahu)y1?bmv%j#C)gh-R_PPz`isyxnoA=rqTStw8MXMUCYXZK?PbwxqN{
z<E{-G9*wFo%T#7LzsK!hW=)8x%urcO4frYBQsH1)j<vobh3E$rq0H}azI)iI$)H?7
z2q7zrhOT3vIsMMff&q4I)Hg->+13hLhq23_=&XxJRgVErbg1Yo-p$9jn*k%NovSNk
zd874JJsNGm_+u0PmM@Rh>^lwJDiP;(@P6Q);SfS7#iyFf18L;mXt+sTX!CtY-B2%)
zR1VNHr4J1@+HdGDH`rRI5}eUid5yt8zVJlua>4|W=TGHIjR0uD2VzwsU{opON~i#N
zw<jQ!SGO6o3@K1ZHS>G;o^Q%FfQ~cVVC${s<nT@lXQ12C6jtw`4|2P1T?s8f`$S}1
zIX{qdfgIn*m^r#w_}Er>qFJjPst!oaqJdVEt0>hn+Wj4zMb&z*vn<nQY#k8{A_-)7
zF!}J5i}Qkypv&qbamg#DU3w$%oc$TD`}iMrb7q{^!hU-l@PW-Fw(5h<fyM1{Dqk(&
z+=I#8D@5e?W&F5;zvni?-42x2R3}fzH>$gnf5aVAR?=T?6F6LBzo58O3(ozeTdVC>
zhS7nH#i*_ulJX#0?2*r)a<|QB<CF=;*1m-8XzvC-h3l}c@_#Jo$&(n<R^fYcpf_6Y
zY#7V5J~FUdypt0ErEK@rhVAhR{M?_0j*&XyvGq1>#4)FOoxHB`c_LV<l@d-17yZ$r
z(qe>J?CQZ^JbzAf9AvU{gjDmGSX^V+ryI=3<GE<~F-%vkc<$PR-k|(zbTSR&zAtlU
zc^V;;165;^7U?_nZ>FDX2d(GT9QJS}w9Lk`r7v*JlJ52#L&aJh!}DfOLd+FNk5+cm
zWO__)_x<Y!(Hm`bnV)WJ%F?vAwOaJs6QGMhl!=;FOLdm<W%}ekZAMdgKMEC66>B^$
zbg!c2wyB1YF;kNFf2s73SsaiHsp$QHW*R{_Ko;BibA6$M&Yxw6PA5y{b+?NFvkSw%
znvZ*#8fvjt_~I+*D^9mV<g69(d)7N=N_AM9uZyhw6I5ztbc!`;bQ)}7bNcpjelWG$
z(;>fgXi6#A!^^AOJf4BBlO@ftVs5Br?cj&x+mX-F+x?itbv`rNItzwoSS4O2bj1ts
z?jK1_=Gy4Z#_#(<-HYmIg+^~><%(<K+vQiwT$_K<r>?h}Cx-dl+f>Taw2q}!8O!~i
zpusOOmpg%y=gGF`njAmwxt(gY-UoNN6`h(q^`vPuI2~xoyWTdthVB16d@g(%QOrcA
z6XINHF$}Ch3Cd?wYxF7Bto;#J@oDVZ&p%#^&Ai6u(bX_Zo`EAY`NK*OW+Zy5s_|T>
z)!Z+5gsD8E-zA#WC0eyMmSvy)NURf$w&n041!i+0PHP|A+xUPvecPqB5BC`dY!9#V
zWzA$9YboGuN0toNhwfxvIvZGhYE9v^)m+&<iR)ur7<{oq%iwJ)tf4FlvXD(=VNBDJ
zg;X@yjcxoK7c!nk?^)5j-F{qZRPuYW@t(-=DTsU;bDq$@T5;bTc>HLy$!e=a%ars|
z8nAB)EGU#4i&PNkV|3_wt%zzl7}%?T&jitDS)II*K_G6F7iCOQN^z1t79r3J7?#pL
zrrRj@OOz?a!n=X6^==`gW<A#VDwO(`$8tJKA8UYJs8Naqo)><Xq#aR}-_Rgf5?|IO
zJBfJ=(WJk!q=`}9wvXa^mELPvgaz{N$vUF4EJWMN?}@V<5a+Nj%3W6HJGtx;PvMKA
zDM!e?Nn(<VvwSuwBFk>Tq`01>SP!Us&Z9m-w>+A9q@Sr9E)p=E&29l;G%Ds?+rg$A
z)d-;@=Lhgk`Ao^@QbQ(olcMto2#YqdZ3h0jJ~;_<r9o#0>?-q5U4@~8t<%#<BV}E9
z^*A%1T;+~f=nqyU4;eGoOy=MPI|jl`?`Y1Fsl^qWRbv8!f@gMxjOWedF42jKslh>A
zCWTSC*HlCn!58@aUQ0G$sQ;hA2)!l)i~=*)6yl}dDwS4(^ln|5ENO$48*TdPNrY7k
zLOyfftkim@Fe4J=T3}1G1M*;<j+Co-g22E;SIT!%vw}aNYp^2)E*st2YF*c516$aI
zJ4GT>Es^V<rG^YbEe`Veel=U<@aS8PK%uqSeaYfa1vxg$4a&c-e*68p5BU5A7heIq
z5&J0Pbrqw(%48%KMtI^2eKt2S6=IE2&%%LKgE<^FMJn?(iygShANa`8nOj85!-Qey
z{;ttRr!>=mZB6Q!9DGIQx7CELZv}d-ZV8dBW{$Op&GUxe%1q8b*a3sGOqEZ;sc5T6
zTaRdjpFTys#ad7a9<CLATy>sUdb3F`zb!7Bz^IE(moNm|VmSN@r45<vDaT--(ja8f
zw}D6?dPmw!`i&NT&&LKQ1&j-;K+e4NL|>NhUM@zc#aa8EK2ECbzz|)gjb8o~ZH2oF
zUP&OCuH2T<3)wy}<zf^Mhu-lgn{^;)uGG=$J@&X6-d`v?4jGeV(J`xY^rhG0cXsM)
z;KX~g3(wfuuU4U}RRsz}=UIkRrgMICI_~gsj}GKlNBA=z#n}<Za19Ma6`<2XC5znq
zGLJP{;1MAT;}fbej`o*j^HIyIadLP`ED8Z0`44wwr-C?FT>D%Tx=j+sOgFxLykVbf
zsn8V$0k&wmMP*y4h<$d1mTAGnH!jSZ-8eJkYJG#7@ZAXSfwuP5Rl#e_t~6d0jSrv!
zBER-i9=j+C9Y3E#P}Y{W@u-L7HnG@*4y3J@8r01fY(crXCtwS3QCeZl&~*2~szdDW
zq3*~F=`;l2h-f@OV#tEV#lB+W*!_gQ8K1AUoxm!@3+_|jK$BGm-SARxwsW5g?Tcv&
z$ue7${q}21O?U5Mgbg%@u=dO3;ML~Stzb%)I>@BIV9+5(R<XAcckya=Et%UF)j?q{
zi2l3+OqxqE^_9BX*5Xe7W3kWA%S$=Xpw{E|%vYOy63!StLfw=?@~1KH&h0V6{04eP
zBPgB4)(rOGx^1*D`{7b;C%BORZCM5yo@CN~o}(dn3fJ|Ri;I$HmAt>k<&d=>cR#DD
z4TgK8;}(m@+GY?!zDRa=wQr+O_ZOWcQ25hjU@{|>>>A8vu4@l9(yJIVH8uu95bYB7
zg#9v74}1A;Ta`!}1!tgTNrEy$4;vp8E$yyj2F#P<H954Bc^_^_b@mC876Y*r*le+z
zC(szb$7E3)$$i+m%yZg%L@U*6_pGp4uH!QVcYev2of{2ic+UG6dJFZ{(glj4w-1lU
zkuV$O87(uLRu|rpw92(HXER=BzjR9`r+)W{kL8;ph8#N$hcTG^ZAf>TCb}7i`5{B^
zjDOJT%i=?VJgqgDY3~g5^@iiI^w5vFPvZSx_Te^*WS)c}jkYYp`yPoS8C+ELw{{JL
zD0T1_Cs!KwoL}nNjze`b(^-9fLiG5&*zE$2Vg7#4ysx`<e(|s0foNibH>}}!mdcgT
z^%5g;Y&*+i?lRimV^5aP6>Fe%dZ6mrWAX{+NbfYX?*&|)v1i3kK4p$HZItOLQuszP
z-9z=bS3>DlD7!cgzK6m?S>sui3%<Fu?7l3X>n*!sTR@sTO<&m!U9gMBe>v}tAhA8X
zEtsR<u8s`Sq-NnYMSfLnGISl+>@3>{`?#{)aALKd_?UE;uN@Oxt8m(Wr()|Q`%?|h
z!eaCT;ceXV9K({EkFa@k*u905t}P-20SARd;1qqRLRF8CY-ZF;L&dao!8VRiGCYx-
zU)&lgMM)C}lww^0UX19}nZQ$mt+hpdM8UcevSvxefb^3$iM>(+6dJ-D-Xo)tsd?YT
zuRk;w4H?7rwHsO|4>df<kHa=|!%_udc|U3)*_V06{!vc5--bCOAN9bo*Y}?Z^t6SY
z260{&-r(&^5#N;R&xsb<s)8>y*|XA@)$!^b=gM!o<uVGkG$_P;qmJg?s8Q^H27EJ-
zA&sOUDN28vmmTF6)U?98U4h}vXK`L)XIprA4C<;BIM0>4ic9oH74ikB$kLnaSoREG
z)U7!IvG_oPV{YZtN>>%hG$*Wrr#Jb0?}F=m%avx@Zdw2e=ZFqVxJ5SEK=8`+B9$2Z
z|I$umpj7L(dlE>f7OT8%g$sV0;(y`#5SY#qlQD2x=MT?QVD(_-<U4SMfMU7SV(PR)
z3b<?`KxQUD(yyo7uHxqedrr|1W3Krc60eUDMY#CZ1g}}NzI5QI9yz%X5-&bGrIJ10
z=&*YZTQI*srzAg5nk`Wqf$G6_KXTd36os#Tfv`;)8&kv{k2fh+k>%D6df$M~_p;?E
zun?}xxDA4I|7^PsoAp3lQ%1`7R|)#}*{kTgn|y8cuhIG8gx~_;k(zOgc7hc2zbg<S
zr84QaX#$L>Ik#&SC_A;2mzUSAR-L;mY6m72Y4>T{7mZu~6TpokI$-W@Y)a8lA#l|Z
zJG&g2b--iYTV-IXTEOlv{xXWB`x7UrD{jL;&>wN>-_*g|Wu&71wO>ZDa@>}B;KzHQ
zOUjmviI)lnB9U>bc^fQat7l!l>||V8>;ki(@A_ATQ?0`^m;loB_}d}CXc34IT#Pyn
z4Yz{mxLy?VXRt=b5_|P?6N3fadFEdngzP1C=hmbpT7}a~t3SNcX|zknAmw7h)(ItG
z{w3L<p9v<yG66^q4g1#%3VD3nL;J_3AhG4szr)h36FXC1kr>MkvlvMnZNrH<Xi4N&
zO(CpF#&yr4FN*Hg+&3^&v2_!@eBIqDuoh&at<%xEut@81Qh&jyD$=veWy)x_NLc53
zVe5_d29>|1ACpItD)o-m+gmy9*H_JaF8T=7gU9DsC@-iICBk?s5Ul6R4$beTM{_mZ
z;c>PKCH7R=Zi{)&o}Z_#4Srt|dMVKfIvV|A*;5mmuTm{kBxsAH6dq6)H33CfEmYWk
z^uYSG$#g}hz6a~Qf`@c#o(1|oNLxlcKkR2xn&GqJvQoW)c*sKV{=oKyy(Z*R&aip!
zKpXE2k(9a}9>zXa%n4K+N^kPcrH6dl_^1j5;&KTsczZ7`{dzmk#z>|!I^1u=5ynco
zqg$#;eW5&5*0US$bu4oX%?6KoS1~LIwTTI(6A*(oRig=HpqyoiVgdu_t7V8uY0hX4
zpGzhz%X8lcAHK*cp3iMm6Q~gH5O4LlfJZlSBvwVsqW5zlEuMN3(D2dp?o4K_?$3%t
zg+N1D*Q1q$edIpxk|M)dsF!QoF#TAMuPkGC;hxv7T8*N#eNTxS?)CaCMY8Nt*zY#Q
z`|dOrT=Iih<f`|MTI)(67VsNK*C_PHYtgL6?SI7IL{13D%Wdy(x%h^pI!e~(<YLq-
zNKhqLsMww_8-H$I=K<mjCvA+P5M+73c~oFHv4n(_)of@Rq4t3QF(%u*Bq0XBp9Bz1
z%Yra(xJonMx<5CknBZflqsT4%N@xvxn_|n4dq$c?+{IB1>RbP9ZogzSPpikEUlRs9
zY;ez0Yc8>K%izh1ozQQl#yN+=D{?)zNHY)}b4Obo2gS2L{+{G3xy&OG!8=~%@H-+x
z&zo|yNV4%$LCe}&cN|ZrBnjo1s|UaA$rJ)>6?EQJe%~8<6~}6@)|AQrniwCd=h6Of
zzFipLnrHR9M7KF8&22fbAOFsgK%fXy)*Ymf+wx)e(z#^?dRfMI@DPo>8*?Y9{j`Iv
z2O69HY@P30ZskkBiuoOCAijPzO~R`QN&)ISs5f&CfU$b2y)aJ-0??}tf>eWuhYQJ3
z1cYoB*s%_~rQP!C-~&B2IPOAK3-6|jvv=zwGR1aD@F!EXLa<!p-nrPhNM1ap{`PLO
zaQug*1`3~hWj&xDVV{#Ky7AAZ!nCBImJo6k(XQ-AtM`p}K6ZEQl3TVV1UtqugzHSc
zoD3>rA^E6#wI<2VLngvk;7xnf=PD9C*K}DNmVuEn{|O)r9LF@kVqJ9Foj-!=@qCAQ
zP?y~V0c%<*loOR_+M_vy$Oh4!5~-`=4M8+DY|kmB=dfsiN!RYAzw8g0r~^%6i~+1(
z!e6x!aowS^hnK5GeVg4{3xNazs^NMYcs^ZK^r_4pj#E>wy9T}2|JTHbZ^B$>dBSPA
z7<Qv5C`C~!ok`=m=_fzVRT$ciOl?OTZa>W7%+36TU3Au4>%_qm|36z7FT3ZaLNz*f
z_sZAW)Xi9aD?^E#2SM6aYGw50H99iw3^z;;=FM!MOX3(nW0n13)|ke)#MazzN5A2)
za%g5}Q)*T?UKQ(z^7KY~*>vr!8;#iadmjcvOY2Ws8|2K5xAEn;apo;5JZ@QQM!eJ2
zlYWd{ji>Xku)L3~GAwero|0M#cb&EY0^J%$SFW{N-O9~B1}QL$@Pu*f+MQP|4M<kl
zEgHpSU>fFgObE<qV^M|cyji88Z#K1g<;)D%|J8x@Ybx482#UI`1)+ov5cheYb^qmx
z)Tc#xT%*ILtOgGIK@`J=0b#s)KVm5hfoZl-Df7KP*F{Wp<M3$axA&Y1(q2h)4$TOg
zs96#?%^;%2Gz9~$3D7Gx^=K~Le^?Q1Vb|)1o_iJUd9v5ae*s|0<1fF8DCI<Lx^N}`
z0kVD!*G3OFqaPv*gEa{Zjft!gcAeWnx)*m}ol~tts*$_4;pfX#uJ&8FX;>{d@NrO5
zmt+qRy83P+d{1>=q^N=vO2)m)Ii_&B5J9g7k?Q+OA}@GP>So%&p&cC1QGYUY2$;{l
z*;%{nG55mM+#G$rI&RTJAHb;Fp>XRAJP;?pwIO&5-SgOaO6F1QJbVQ%72>ON2podI
zo%J~!)vkc~>CgNq?U=152?T)cFY~;a%wcW=^#P-!-qh_KHrrj9*4=?jWr-I%-sZX5
zXQs7gW9+2gGFx)EGK#zz`h}LRgp|zFFqnQiYy|TVSXqu4%J3b1GgI#OZg<>~<Uji|
z7U(6AphZO};{R4)Jk;Z_yQ+aMWLVRsHB0Zl8Svp9H&~W>x`ACFSa{x@+A`}=4>S*7
zz1XCL>YBiHI<Pp8w-`spQ*@azpfn+8P&t)f`T(k*`Iyf*1`Jn#;0GT>lJOdaSl)n9
zUC_I3rp#ltQEPVqQhy(^7cxAFa-3TES-k6ky!Cb@-<>s<*jdw<i43tzagC)$=n81$
zG^>~<I$luhw{7WTIV9|)W>2E#(Lf5mgCnfHcLdj|mzzBU%l>$`{nEWFeZ{P-Qd6=J
z<0RD(#~z+Kbo&<l3S<!y&&c}rnCpBuG`^H-t|d5w6j0gi9y!vqvxom;C$MlU$8JZl
zq)4#0K(WiCS`{wmCa`xP|6HReSiHO>fI?mcDP#|X6PdY;>l|b&G$>bSP-pZ_zDy{~
z_!I7Guh!$O8M_8`p*I$4LW7^26})_N==sno_>jKBH8hdg&d^!@__4&Cl19p{Ln7oT
zm<QTI`r)8Xz|KeoF3Zwmv^eH7-`)EB(JZ52R##0X9f=&r)&g%uDuKM$z^b?=fg<_m
z^5sd8y3<6X#D_}l4+8@J&m@gJ^Z1MJg|@#Nk9iY%B0I!aIl_)gv6Js|R<TW2owflH
zV#4X#M67Y-iA~1F$bIP8RX@eU{mvG&{V`p@bjXBrxn6_Cpp%9eU;Vaqx^K7q1u`!Q
zN<N8*_}odvySLmo<whB%VwlP8CWRQDw&}mtdm6;O5u0*0ggal<;?$dMxoBPP(ABJ$
z$*7mi%{G}>F>1#A9G1rc^n<G5Seo+%Zn|Y-$(*`wq=YA~qpqunjGs_QS{co2F<J%h
z+SPTA;T1XUmqsT{hDMbTA&MiXe$gs>&TxE`uwz^;>v`7r;m^g4^(pCDGW6Eac!D*S
z+Cv{?9<GxUsW$fTGK0t3G6NyGSp5T(oiBp>g~0em|4vZ_f`BzIs?YryT#eZj9L_o%
z^U!VKI|y%W!Lf?h)dkf62>yN3%6^R9Q^AL?n|TFvX*2_w?uCUf&^ekiR*dF7Ml{Op
zb=}BqnO)(Q>7_vm&yNF1$3oZ2-xrb)U!xN$emKjy(3>bu>hy5u*$Pk{)1D-T`f#!m
z{n+p1buJjOt`tlYseB#kBAk<7SF$q6{%HUY&uPIj6wqHTxF7GK`3aqwKe);4x(X86
zlT1o6XrO8Z!mRqqz-OUn0uQj)!2Sd-eC9MTW7W00r4AXQr>l!$z#p-UMw$fWITp**
zr%dlBx#`cJ+jqq5kuG>0j_|p}%bJRv=+xPThh?z!F^0ZEs}YHXo0K3a7pD_efa8=K
zMZI!LeF{`WMeic_I{P|BH;%dDV@z8vy!gEpJEr$TWK<&ZkqDt_iRzAa!3{&E;f+Uq
zID;H<OFU;qO-cb41!DAmfkemAWK%*hllOII-EdTp!Ayxij*1=$UAe@cXm?Xfjh}h(
zK;94PG|}}WPNVU%gKZgoLtB8$>uY2pArZi`t_+LJ(xnK)bXU~#lQiB!-8?3C4G!2Y
z{ZXwFp~Q4q-=&zzG%sH2Sc=Z`@9iJUJeMW@0=xC)KogZ~=`or6aBlsqO$dgzvJ_hr
z7;><rjf%W7oBlWg$eG$BU#A*2Id6MZqk;F+-Ip;tzgg9zXHD^eYF34fqP{pHdG4I~
zkI79NZlg@baS8cmwqNp2JPxn)zZli&Ckewn4g39^_(xi6XM(iFd~uE4Z_b-J0WXqg
z$~nJo+Pm8^l+A}sA~^ZbrZc_5Ne-xHMR&}u;GgdZD-&xYgylRb-&e{DL6S(v7U-J_
zj;!?Ovqv(C51Mm-iYe$_#b6kc@2?x)4GNh@A0^b<MJHhDEM2;H5;OLUp?ez^03@e2
zx9s)Ta5CNZqBbemWIU6#zc2eq?TKoOZp!+IhM!Yon#@%Q72=G;cU#XoYP0Zl$wfe7
z_arHa9QlAsX6g7F=ifjT9CnaC6<0yky;DE2xksuJWJ>7wy)pWisc<B&SFbGK-=R9-
z<W;2WR$&?x%%H5rUE1TerkZ&F5cayY!DbSFTAFm_uE&jT4Cn9@rNUsQ$rFOSKN1>X
z4n>4Gf)}Hp9tVjqJFS{aQ4o=aup`%;(BlD(R=rhxr*up_HvHs0qan=;(5wCE%ot)X
zm$wWP^?48`F6E_nW;?2I`oG<b;Cfx>cD{}9R1CHTQ1l~HeaGGJ1|oYaWzh9Oy;-hK
zf?*^dhV91bW8t*DPQSepi7mO(KbGcf+x&(uz=#tK!HRa_F#lpJ&V9toz3zF;j83MK
z{Ogx)R48R{j=+zgQ{7j2smHjCj4dxKS&oZX0eW*kP#eQpZxCdH3UB17-<10B<8~>&
z=LowZl?vKetXimDl|GVd_tT}rbwn4PGiBl#30)t;Muk**fA-n;3$Zf-!CV}G8q=of
z;OxMm`ddrU6F;bh`LOS|-%_ifen-f9@rv^_de>-dV0)r@KjXN1M>5H{zqxQWR|?y5
zxvp5D1(YKEe5jOxPf&s0?q*qNz4!43=dvw(0@2Pl$8li{pYIz~pJ^0?_zeak5iF#?
z%5>kQpAvrDi+L)`o8YwLb*!*H63Jl8_wA(UlxLWJWJ!4eQaDrT|M+FyN2xN|dhEr(
zTxW88&-qsMo@)tV;ud!5SV~B?X|#gQ-EcEEbF^tERKGJLhcnFK1ejqMXaf44bzSmV
zU6I{Lp+}2PQ<N;M<Zzr7bBXOMjYB^RDc%pfJ^P{4D^M6;q%H20!1}{;wT3|>9$90}
z0(=X#NeooN3K^~pjOTX)XqZf+*d}Xj*v5d#M8PU%;Wj>lR{M*puWV~@oLXj>+b~hQ
z@Z;FY`VR1k%8pxp=CAc?h1QcmDbkUK_%OyJR_}SYiRLQ=^op;J1M`^dk6$@VB5Mst
z&FnLE20W-zK3^3K-v80xJA4}heCQ;xSe`4WvdUB)VT#bHWoxw^9uh1ueQmz&sa7J5
z%y!Lc)ur<L*f)eTN^-t57IOYu&dnJsFhhU-aLz~PFuAd4WRncS?^*C;DfJCq#APIh
z&b0}S*t)H5fnX)ig)%vLrYJ9?Oa=b@XSZ9YnA~*ex{v+XeS|1LryTTIo{S84>EpL!
zDWu2KL}DA*_fL$3&`)oDYt5H=ipn<*=wv?*LLYrh&TLw5!SG&g5nZtp3Q&b#oxc0c
z8WzaZz}edZPC17e)Gr1>?D<k7H{uE+9t%g)1;k>xt`QU+b~yd4EXL4-dN3UFHvF4K
zG6Of-S!06&{N5Zel-ZPh&UD;)pW4@U0ktJx-Tv<Q=-iz((cWx5$Es=ybJl=v2|qtB
z<ZJsUa8Yg!7kPOFT;#O~zw5l8vTt__Zz`3e;8r)pbigf4!KSn36q<*YX>s;|d|6y&
zopZy8@47G<U;A%5HSdJtGpCs9=0f4qwqvpO^;_M(!u7LaPhd#r11+F_dj%Z!ZLHbY
zvd0(8#ujEBE|SKIOp=i-<57Nv^?Ig6Zt`s@N_~UJ8-XD%(@Hip@Pu>E7(d!dERc0A
z;!j$rg`NQZW@EDMI0sfKkw9glWEB}ueP#XzA?t^87uL&LPQ=M7k-pttiLSLIO2R}T
zNEDQvdQ3r;P;+w&VCpX)zH(2po5;jgdZJE!6zWC)aG6K%A4eDWqLOkAN1!))=22!w
zQf&C4Sby(r<ZS>2$VpF@p0XuQYA3EZ?C6751I2Y$ux~jf#dg%u+en@~+o`58`Q6Bn
z*tfP>M^lQWb|-P9KTFdkGk6g4z-=;?!Sscr@rAk%&o|a7IAV$qXfon)rHbW=-baDh
zPS{uvdG%?JQ`tA!{+aWoj^zg8yU(V1LmE!U3vO2J=by0Z+P?xJazopuK`+k!)k(+x
z9*7E@M4|Mav}>s)#{N!gA<m622vpUe@Ku23nZFD<ibTxfo$(HL@Ew?hzD&>srQR0@
zKWHDtup}=2VhuM<*%yAS9GGs9(P{3u+pzV#%UHYAs9g-`OVoT94@#k12kRE3vg+!J
zVB0objqxM7E7}SFG1Kqmw%=f?XIzMu`VIST%RpdV;rrXZ#&ux6sz8H5X9)iLAcoVQ
zSN&vC+b&n}y8c#}+fk_1<L_BZRQDCabrg}Rx+e2v@KRxb=O5GL*MKMdT07k43OE?#
zrHR9jZZKB$mp-uXZnTu=A+>mrM`|k946NBs5)dRKSeFO%v2r!eT_R;sbO~0hpfJ@|
zT|nW9W);xX5f^Qy74irTyr=9SEvH5IPKjd_Tu#&rd{W2r<zq(rQmJRYfsysZNXJ=t
zI1=oap3gg2-x3|lA|}nbkn1T{z9aaZUQBs&y{RVIOr$%Z>kMTaj*Un8(rqSeTL7e7
zG^buJBXeEx{MHO|pw|L-EJ8h0S--p_=l=uGkb2-<f3_hnZ}#g@D(8~reITYE6NP0e
zFCj1YkTcwSmM#npYzQjVBPWo7Y#sfBhXA9>C2L-lX$DScAqIuiHLVASKv&=QVNYOZ
zr$H5t6GW-vxV{n+dr!*q-XstO-*_ORUZh!b&h>bHOq=9u=9s$(?kTO%!_pEGLY8Cs
z?@x4D1yq|aAaG3BA{DMxCUBhUTVI7&xBt<>=3lkx=;K_ln+`964P1BJPWLnY=JN;{
z79q~dg7${z2o_0l%^iAG5^q3&?_w7p!5z7<qh9G}Ql?4JBq0COzj?!VJCt^(?Ou02
z%vgENbwaHACyiWztCM{pB+R;RJU$dL%MlEWxiCWhP?ecF+S6NXXN0-pq!+@WBFNvU
za)QFq(?SWa2Ws&rG;tPsODj7_klo&FAW5JIvUaKB<elXJWWIpy?J>!rtYkpHkOc(|
zEeh#gEDdRV6!&Q*tEp^Y#@7n4MWB2V$!{`B(UY?bpAqIp>8IP1t2zWZ7lA8yhn@8^
zAxa_Gc@H~~JT1WPR~P1jIH_jio@35NOVK5lXPo#BQZRMXC^v;*pkx1gOXuOHeKpdk
zuHr~=BH_hA-JHoXf`uZ_9~NHNeXaP(hpVlkCq;a&WaIoAsLvmOBDq@ZB=Ia<nw@w)
z(npUFKEtx~6hpoK(U-_=XOsJpOCS!!sgUY+d1`N%9>!7a<sJdoDv=p$HCtp}KE`S5
zDZl7l@VVTeQ=3-l_vEUvFow5KP^Emb6EJ5$m=p<7WIxNySDQr#WH3fdRSN8zsmf4<
z3}$^_Q3;lnD)E$GCSYN(XOXAnl5X7I^I}mJh;!C=iXMMgi7pV-owxDiS=#m}%5opk
z1}plrGcyLpZ=p0!-o@MTWukadJ8x8Rv*qqeQ1<cRQ0f1m=`7=-dcSBtbV&%pAV`UT
zv~+hjf^<nYNHerF0@5Lkl*mYTcXxMp!_W+H&+mWleL3gle9qa=e)d}HyX&HpE@uvx
z?3l8f?Lkf~4FLp9tx?T%S2!n^8Lk&Wvo?f~i2%w(I?NoWJ(d8TuG4)pW{cdZ0;t-~
zPT(V*muZ*=8T9C5L8HjN60ou!a1W1oa#kq~IWAW}72{M02BD;V881{SF}ps5yw)f%
zc{a%`HSIIpr7p;_XepA<YP?D1x@vH-Om$l@uCy4r6x=z;a(X1(OWXTa`%X$Y3m&8h
zr{z{Hz{=5SbBYvvUYr*+IGH)nKA0!@l-xgts?X6KV27Coc)<e6H?F3$LqTbg!?J*v
znA~Q5ODpG<CxY)c$`~~Kb2Q!3cCguUwwxC++NDO!CL#JjSanqWF2Ut&sd;tBziemw
zuv(QP9;w<9hLUSW5vohP%0U_ZBf6M}IBX%S;K6>GtXQABxUHfmwh2M&q$&4=(CTMZ
zh7SM44+@J2Cr}((>u6h{d;`DDMYhF_%ec^3u@oF~BvAID7TK#td_ga~FIjkj*}M<o
zM<9Z*>Wp#bxdcp|%bK?C1BnZyds_6Vvg-ug;*8lc`w5>qqy1Uu6I}lOK38OxjMd@#
zZzi&stp9U>sq_A%N}ma^y6e0LxX|%>jx9B!B&Ud0+E#5FJ1+!G<MmgG_k+|Wp2@=o
z!ZKOhM!fVw{OfGR=EL#gj*318!f6+AL=fYyL+qXBNb+)PeGcOV>Eb{4;@dm6Mz6)u
z&xhx&O?OQ5=tL-${sNf&@*QoMRQF3JB*;GU$oX)3wn*+}^vS*zP(Dx)a030<OUuYf
zo?@dPUko{nLdHdwc?^g)7_r(E{SN5RglT35_nsaYL>kVb<>vZ&KRlx_fRI6Wu@3cS
zmU|BGno2*<51XrlO}g;$A{3Yok;0vLo@xXqX^Wc{@<AjXF4J!J(>1nQxUoS3#U87W
zzAE;0hss9&6|cwJGe<FuYK%JB0ETiKq>78K&^Pu)xK}8d+N%@Tk}I(x6m@(7xbbnf
zj`Mk180W~RJ|x_B#I-pGb&|i~&v;nEKFnkKd6NF2*rXObTU@MasZoNYQS}JcQ-#sx
z6UE(pK{Qm0ucPCW(1MXag0z@ytb-n|G9om2%m;q@GF`Hui!XKE2Y7QZ#eZxd!X>V=
z%=Von6#=Uj!q4|nSs`mau;}v7D&(TCneab01bk|_saEWw1TGF{VwtzPj!d~reoRUr
z!5>L)G7$Vf@X(zK5}VP`&Njk*N@r+l@^|*Aus^*jN8Q%Nbsi(rUt&u^iNjL<@+~);
z^)p_Wk&2-8vcpCw`Xq!$)-dMvo+~E^d#9cp`FMY_{W}$w4Tx*Hk_0O`gU2fx)C{D;
z=;lVofsl}(7CoRr{l60K@D#Us3L&sI4F5pNyoRFeG)(MWL$p0uz!=c$mWiT@HBR<)
zp703Ys1w~1UZU)4z(7Z{rLF*=uec?0+7(V1>ujQ$`!#chk^8#?Npe#9Yf=d)K%2!j
zcuZ~fIzn8cF0$F$FK&O^bf_QZF<sLWf+6a%=6gHvS%z%KEfRKgP?SxS^_U2{0Tw1E
zth#WI73o(2<VqQk(t)&6f%4k2X_b4Yccs&!)~@PKCXWf9Uj(96+VVp{#VVdIgZ{&0
zoB3WDBkW5Gka6X_8(M3D!<ojXi>Lbin9nd$`=L5h!-b}HE^nO+6jHdx4vQKXq6!9k
zcDEuOyDkIqTO2&q|4w83=6oM2jEPp*@%$rJqQ1Q?61JM@6v277yM@@JkXUR5l~TMK
z3|$-8^<@cIR|NvTU6~47c&Up=cAargz2^5R-*08UtU3?1jlk@q1s6R|huJh&T9Yrf
zWKwTW&-eI5U?Je=m1fJVb)g|^zO&C*L~t`VCcT?T3$Ul>*5bS3N|{@o_>xn94f(;}
z&~Xx7rZh}^QBy;*owO=Og(A%Ydv&6+Rc0O$(#O8KdLjO!o8`SXqKpArn^dLhrCL@~
zWs%#NVYu2=7xa9S+R4JDV@ljS{qLi&cy;bhcz<2(k_ltK*DkSHfsv%gXq2$whB8s@
z8FRh`{>pM)g34|a;nfHG8aVGWib_jsdMd`!Zq0#PrdbjuLTc+fpL8a6mYpQE6^bib
z%$x(C0|K>|{nA=s&8mp+1^<?pLp=9&PVml<77xV*SzLUhvom+v-X#U18&K%^pYO<f
z2X6{?x9ZN1GAvsG(6u($MFienCB9LxrN^Db(NtYSHs4@dw|u)Co|@&O4;slosM>jA
zE!uk<?9QClpWPVJzRT!mz&iHo@e{7fUk=@BD@S2yF}TouGT-T57kLwj-jGn~>sbVF
z_VYOTJ=)dxE%V%*_QW6l&wN%l2dxU*T`tGuhzJr!v_{W4uthZvxxXM99zw<?hlj(E
z^xE|}d0t;4RDGlu5mXd*(7o4)Nv4V<cfFJ_+DA>&D>CZ+m)%HtnW%C^mo2DdE3yt~
zIjTjoWdwAUv7y})D6E3}szTfJ6p&J!NnM<n+8Ifq_bN*}=Dnn1VwL&))f~^Ec}9?M
z_0gQzuvganP;2ogYrb3@e5XOT*l>w$7z|12;}myC$+c}3Ik7!j?j_aH;arepR;|p?
zM#vwE-^_zC%IJo5!uDl{lX${O6RHsUEv-p72@o{4c!SCJe-8vtoDZZXaP0t*$Kxuj
z{0+c~o>^>aw!Iz4HivXaa}+S<NWN`wEA=-&98xirA{fB;a;tXwn^f>AdhJ|YFD&3A
zLl8?l_t~!$O;HN5{{$Tu3Sl}UGEXNL1kN$P^C-nEjs;DnM=QE~P@2GKn|hia>^HYw
zxj2C&CeCl@O9D6@Bb}a`uW{y%3^nx)s}i>ntoeE2=;8+i?sEdzc=u7ll-_Zh%?E#8
zN6<99yCg)64cI}onkrNx(l;R(&9;4#ssc0XeyCT(e1EIp-_D{f)g^_p2gWkCQ!DmH
z<vv+x4!b(CZ$<ArL`VLEa>Qoyxua%L^uGOj0}&&TeKh^fc33KoAXc9O+DF|`#w?-0
zJ3$ada$>2<J!)){m0V>Q+QaqWv)Mt`Y!it0={}L>BAu155A4KOf^IvIyeBAYKC(f-
zw{l?oc&pBS8IXheig;beWMXT#j9QI<p(2L%%mYb+^u5k!Cmtn)Xar<6M6({Y$$M?s
z0i%NxNc)?RL}?PuoKzats44_I2*ioD)jao*7a}%W^!vj0Er_n&6%UIzAU^P;fB;xF
zRUQ;=+tFRa@MannT-}$^<-~jpA_0HJ!)Wf-?#9gN@WQ_RsU2=<0ZpK1%M#l*R5^WO
z)PdBMN+NJZWZgQfOl+Rs7nc&}UG(Yh3fqD*-U@nt%du)iv|6?#1f3JmDZjF7^Y(j0
z;2ai;Dzg<srfJ6gij9#XI1vzV+C0(_IrB+$R5b_{8@j``S-K-QiJ}C|et?{oJ5H&c
zD74Y*b?yc$F?72r*8$D>)q|cA-<S^{E+H*wM2Tm`?EBav{W3S=-wX9J!xm_y^az@0
zo~Yx<V_wJsdJZ4d)5#j{?=fJEw)&Zu_7OW4vUf;W>3iV=)1%~50QdO(!3Qen0DF6Q
z`su}C>PvrSEw*9rcXSYG0Jl7_1v~NV7IUoaOr2h5R*@*w!tXME*z;`7iL0l^27waw
ztKtV|W~+66i=E{%p;-wO(JpDgT$a6m&G-x5A;s^W=IZoa{$2-KSXfIHlTN4i0s1cy
ze+xYCud6SQ{$s%nJ`X2#HM@N0_OOX|x*I+|)juGPGQ$*VH}|SH*@90+HuO3=g4yXg
z>Plh&JE`zbMX1YI!Aw5ni75gsI<;TVvE7YzR`+kkR+yjESwWy^r$Wm6**E;lKMKTT
z79Pzqpe8Pgis_?o$fILc673)=TQt8={Nmf_5%eM<O*zw_8rR$k^}Bi!f}LAQ{|1qO
zARyQ#jlSAOnmql?lbGXgU=PlSfb1<}=UTz-@nBqZ$u1VCk48i>_Ik+DdJ5|UlV0r)
z;Cy>e(^zM(=O}FPK(DIBuv-QjZCY#|%052mTO8IhYH{2vCljdkz7OG3>xQPUUeCcs
z`Wk!ecvhdy{`$bC;66=&N2Qnemo);r4Ci0QC-1Oc$Lq%E%QC7C+u?gncRXUCn-2Qa
zkD@IFq>ai&tLEXJSQdFUZTIvC66jRsU(wL1P+IE+yCL(G(>??>{1ZdC$DqpC!U?(7
zLt32NAl}^BGA}vxl-d-Z&Farvv^H{XvEyFoHGX$1KAXA@zmsvvU8%cbsvo}=+z;mQ
zW9y}|m{qs+e~{v2*O^4kQy^99;*9p&bg&$7AuS7%XuX`2yNpbyKwJJNbIn**xG@(b
zFH7<shgC!*4voSC^{Tz3GJl|m*`Z;!sF|TC*BG}%;l3K%L7<xDAwuwJT5xjP7LeAX
zCEgPBg-pc)*}Tw#^Z5V48bY>+FLxUJp5U{3ju=KGVB`kiOXS2?FbH~alx4CbA9FYL
zyZRBp+NNyf9ypGpPkwKmpab<XIVPz5dL3B)8}Oi=lH0%;%W#D9n?NM`z;&~AVD>F?
z4&->UufU=Q&+|Rh_%l9cYeZ-Ktdw?k2PSqmqTnsEezoW`YsL!pCumAEFqyu*e_6)n
z9sJ#ZYD`TDB%l6*-gMF88U2;W_PLC3Loz4W9b0LijBWt}=EFWc#Nr1)qP1_f22B%j
zUcd7RIe>r$Y}Zv&=lgi`N%hs`VOl6VcTuALcmN^}JSWX_N5jJ}7t18Pc7sAL=IAHm
zVz0SaKZkUwK&cgJjMz0D<C(6Is0i#77)J;G#T-^}Kc*gNnw9#KY8wON3NPQD%9TLV
zdt(hhoGzwTO6RU==Yf)dG7vm>8=thRETW@`**XuM+@uBQmTyY1ih|r$(V|1rZ|mBg
zMjBbD&qBb4=q_tpm-k!s&@MN!{B|A@T<cw$T{>W>L^2nRMHE(ENf@DTOYPWtU${cd
z3e>_mso2IntV4e&C=JL)b*`EQy57N-P0v^(6W!0;fCkp>17?ogRkO#@Pa+Ql%4hFy
zoDJ|neUG?ObG|lxpb+X036`1*_&=^(0LkI?ovdySunV_KaW-i3oayy|cxH+JMrX+7
z^W&KwAt+{MQ=fqu?S2WIIO{P1gpE!aSSPs||0FLrr}O+6p4e$#wj*uY4P<auTKj8w
zx^v5%BFFJ+vk(t&tjcRFx1rv3(+B;1R@*haqw=6`&5yMECTu`K2811CR4C_C?B?J<
zh8Xn{0Qz%pzxY8%gsrSbOT_U#C=H*>R^e#~$BsAs_O}r^Jk01{iaYa6zCb|l%Ko*U
z#X0{Qn#qIsu#uGDTwLGFDo&>j2_uCHXl!JT<H%wUlWC^b0nm+(XVWSvfBG8XLu+*6
znfR^mC-ZItci9Yx@WN3Psa~hr>Zi}cf!?fF)TI>G6Ntof4yt1;>;*aVK$y@kcPgqL
zjQFPhV>T|J(%U9E*#Kb-6#>d!l7ql`T>?&dWGSOHmnVuK6`i_)>TQQ7h$nNp)DqXp
zd3eN3M~C0<tv*!%d4!eZu}7~&#kUZDO2^_$jh$w1T2J!{eR3sozp~-%huXf9*MR!3
zQ2+V%X8gCHF;L{5V|*=7(d-twWn9{TQd1X*z*)SH`_E}_=Cy}g9GM<WcRy<S5Xbi{
ziT30{l7R?{sKh~371S&ATlF1^j>XT>Q3r6Cq}ePsY4TgFI4q>MYHa@Z2%KN4&4;96
z?{0q~?8Ur>I+0;c*E~&3KHVkPGV0at^#{F>W;)gZP3G6FH0xxiiETQtXw^;K$mRCY
z*@Mm`Z6?BaJ`iE~Lj^R78=;N!aQ8F{UgR6Nj^a8(u&=)ZOT8^3f#_S26cw0a?G9c3
zJ8^Qg%)t`=YQ>|^e20?{9YXcj)OkLQ``?AbeJ(ktAChp%cibfI4vGdpi&Unm=zWwy
zHAsAYr~=b4bV~?LWKG0;VvGrO&>41mQ}kmQDaw#Ux@aXxg4~CtAv14zXHHB?8f}KP
ztg&Cl@Q8AIJx@h~;QDZ`mAc6OYx{Lh!)0~XNu->v8rg@BVX;&0Fx0&3qmF#VBMyMf
zk>=i>cd6S}E8{i^4qxegurU3w)j5D8v+*?bKUwoBCKoHqcnV#M7DWo61WXIPw)cKm
zDKb0y`R{Z3eL6xH^qKiG^onhvuV1HE1+tYE4cmYFu%l`lCym8lx6hQ4eF;u~CuCk7
z_58pL7(1OrG-jGVN`OT$@U-%k^0=KPvPta)xj{;XVC5i|cOMqGNOZ`uU*HXj>_&}}
zc?P?Y2q=e=Tah)#p9EU^2FW%RW*@M3*AWWuB1Sy)Ig78sg9b-Tu)VPqT#;d0b^@$V
zWCHG2B~_)?f!sd~JjeqKT_xHbg-}CIdl}uPP{Gix!6pZu*q$W3S=q$tf`}%ulflRD
zM5l!{wgc2U5POwsyc9`0TE-1cOf1m~wV`BY6b})+zA3jYg2<m{JC9cfrYi=Y!RtRa
zy2AR@{`Dn51wrI)w%-+~nX0<wf&2ak2sl_=Z#i1ke+g7gUyfy~svYtpNm=;b>_{2Q
z?>6l}c0#q)>jyPP65eh%ssa#+D_ImQ%j<U>BGbAt2b*R}{%Os-5w##&U;0I2DO*$3
zx0+?3ZLtaNO?nUJalscjY&I!j=@}Sin4#ee*VGrl$nIHaFgL%sb(4BLzQMTbzD8L8
zY;YjuvhfBtpwRtfF>6O;*BgaG@~ErP?MRT)qVm!~DB`E@@4Xg1kLvo8?1nTx0YX<G
z7ppBw(fIqjjT1pclkCUI3b(Vp`t?q-FJi#ep3Qk%7K<C_?-W@ZB`A6j1n&tXp6w50
znp_XG-cz3K>l{FFa>9YJbX+6e1>)Iq>uaGU{LgfghHBBXNgvcShl1rVLi>vf!V3nI
zY5F+aVmiOfG*I4sCg2_*8-wC($?Zv=t+sS!sYAG9kn=mqigbT$^J{#w<G(GVTuLql
zkv!;EE9w+4S41Y-&x-#@!diKJGOcjCiK#q1$$dU9cNGVv9dZ>s@63f=y#s28oP<M4
z$Xd@WidWGyuAIcy;M>JYhhNQ4W2?Drn0J8hZ7mgheZgpZ)e(mVq_on={Hnc5z8$?g
zrN(JX-o+nI^INq@*)q-Bh!yrO;}(gL`={k-h6(c|V9*!-{d0F@DfRZZU$2Rsuj7j7
zuU{NsmhR{&*{x9XEDNyJywj4yK@CJ}(hbV2DVYT&LbUDq3#CNh)EEROp_Tq&2_A1Y
zZ`!_j`3J0noGrjFP0AtDn;i;oYU*JsiP``v9RQ1p1rj&`cIz4>H!%d}DOI5dx^{9X
z>mNP?V{x?(uh~w=Uh9OsP`z1Jg5?qV|Hi6rOGK;zsP}1MbqS`*k0oD<JrdsteI3d0
zNr6*J)-nC!9cxocbNxgyE9vDGW$-wdR5jvlbPmb-abKca!{2MN4VqdddcDHyd%x4f
zZrYnt!NZTu2dHrj9;A$%QjwJ!)=Y}6j}61R^@27E93YZqUUPk3DIqx=lmEiz^LM%y
zeoG^`&~^vd79zEBzmoU`;<ngVq&F%#kn>oXG+yJ+MrqE2v3q-TW`9(6s>vLl{2=yW
zyS8M(f5Y%vrZKX!0b_j43Pa4hlkr+Sn@rln>mVO8@Q%}w3~Rnyh+4yC+E2zS#z=)b
z5C^|iJ$ITaZ2yn@uw+X;PJZZMkzjCaa+q2J(igbsFPzHISNI<KCn+UhgI6D^MC?;)
z^@afGP_EMnL8WfnVLticjH(;8g1($vf+qRp7aZS*?wo6a!!b$b&mt<jZrA}9+E7v)
zs)=F$#%Lg^^&hUw-K=wIE70;=$9KK7z<eNTJ7##Fk;qD>yB;~`rBp4f!MQ8{SpAaV
z$|dYPgMuIVq9yDR^0f!`@T+3L^T}0-J$%Kncko;e!n5d2wxsaj);NB7o+1w#sR1B3
zH%c*ejer#U{sr=~W+;S023fwtE&W^}y*%S%s+fo9DxWxCZ=4qoR!N~%Or^9-ZQ6FA
zL}^(4Dtx$9S5|~o9@?m7B6c*RWpura{l^geE(ekUwj*yyXDdY^z8ePy_(j<cJUL&I
z#~p=2n}MRRK;cwFDY5+YH-qyHvIQ?dV9A$`9=~uBdo?f^W)t;nRAD035gmgMlNs+#
zOMaA?b5?zyZVR+jM!jX)72+VgUJ2kwa#-N#l9j!j`*Get)13M^U(pF796220N1p*U
z{M4e<R;hF5K(|m#x)|U!%&uS)Wda62mrtYK_A>e7ZuTKZ3EFM#9{&y{og;d8xs%CV
zO$C5v4~!QY`IEH`&R!lF&^rA@1fyd)qy&$110E6E;-NB7q?GnswI3*=1ZNEeja%3T
z=Y*Omf%-A^rw{cM`Iw?xXw^LeDYT4dKa;R>A}o&ygw9qPse2lXI9Ly0pwp2%FBZaW
zU2&H^d@YZzQ0x%XvNsr_Te?J%7Get5`R`r#CS{X<e^`?F|15w!{jcxcq>;w!?fEbJ
zssAyKIPVNX7W}XR48KRzC*K148~9N%2<PKshRIvZb(J-#70X)LnT*-o3_&*+dp$w(
zUH7-&<|+lDCnI!{bIBt|gVui+vV9f<ENja*H}Vo<n5oBBHT#AaaxE7k)*O;bDD;i>
zD;|iacq1Ms0B+8LEmcmHY_JqN3M$fHA!5o(S$nxKxGU;-{Z^1b(PVU*{C(b%>_Xvg
zn%im%k;rZ8@iQ~K73FKd(aqIh9y=F&AF~X6sTHf_%G#1v&PPv%7L9s2d}pf|YJc_j
zCDO8meWeOxyccb~nc}{ImD8iJiV~=QG_Yj+M^4h=Vg(2L!yD=prf~Lj76^L_F6mf<
zXFuti8<RfnLUw^nQ&SIQ({fc%o?+I;1a<&^5+FwQM8#98tTr3)=xsE;iYfb(r}+<+
z2-)Rs<6pC-ecdTG=`3+wD8<0`07V*hPxBsAv-x0R0=dw`AX{IC)oxdFdqPHreJ>g8
z11v*u5c<~6d8UZd&|=N+-ru%Xp(4OrU68N^aNf}-85aw?GkgRoS(`dp#QrBDkytJL
z^mU$7ur)s@>*#y`r)R-lIVFL`Z=C34sYwrYZJ9fq*~W0J4VRlGLws)sb1Um9;l+TU
z-v|i&9$eIoKO8<kJ#I&nx~zT2ljZtfxjphb-5WDZI9v5)FBiEtFKiT<PGJh@wdWxE
zo>u*sP$REiY9mzk@w1j;R^Z1tn;8<hoxuh-DdTAE>bZ;}EF#4UhxYgEzXHDu8$sDz
zkLQQr$|;zHmx170lJ@E4hP9z!3~KY8L8*zfgj(|!0SY?0Y8#8CI=?L<2)w4^^HLhZ
zI8bdqtHcKB|Ie=8L6p9R1DoQ}&l0?go2?H80moK=o2s8Ukq4oj$?w`%<d+z6dm$u-
zNz}M%!k2DH5$q`{M`0-s8Zn2{UhN#B=RnX11=l%0jykx1id~#$FeyxM^IR>gSL3JP
zA@wwOvEd9-S?|h&lDkQ8yjcY5lqwC$AFl`rPJJ|KG(z&tLEGQSVb$L6|M(rQw-+mm
zdz33qQoEi@gsR?$+dfsx02lfDk964&8ljwS7kPd(N=?^zVbx{b0}^1!V2L%5F|y{f
zuJfDJ6;Wx^MRO!x*gab8XiLGO<EFd&QTmC1l*G<3-^PhAu44RuO=#SN053a>XQ-4%
z>w8u!c~nsssTjtBsn3~q1Z{ffnS$2Cghla#$xMtd4?}G<N9`#XxG{dR7A(K4ORT2m
zwd43;(xfl0yG56m!6}`8g@2#Ec7B9-1n(5!i*7Y~F9L+bQpmH-f~8?6E~*gUIeoEa
ztM0OfZ`-~%WZ36*%BzObxN!nA8su|yO8jtaCGOBvq$E!aGxZj;fq3r7k?b*%mt{l|
zR|5X{bQ17-!1;fCz<Mj^ft7QQ>pO!DN&>U!&EJtx0&Ga=0n193;CJG0b11kqo_tMV
zijQpW$J#KBCAE1P9Z1QCY>`$q%P9)^?hy4hzbDBv!U#KVpprSH%2cJkwkD2`>nlJ3
zy`yO}ry7ZQLo0AIv0cHCcRIQ3vbYA2ayk9ro>i&rb7T0a?mT%2`VH+Pc|ZB(8(j}f
zb^@1)@5?NSSCr0xy*Oy@g9EPrj=Eh8xA;Rwe8P`18lw1=FwQ}mCd(toUx178HwC<T
zm8FfFm}qO!pz+r-@2CL$V$}uZx`w-Z=@I>q+4Fk@t9jQm0JkM+z>qyS?q<y8v?*z<
z{B6LohY2X+hi`d5&9RRG3t`Lwx{t&4eQ!e2940=Q_ic0dUh|w+UdZXZi~vvGLHL(m
zpRQ&k;~6w85*DWx1f0o7Ci8$HQ1D%Z?HdJY)xS#zWeQU)-y&Q>UHWMB_c#0o3uK9s
z^~rfg-A1ml8<J1;$v02mwO@C*AZ$rzK*)2BJ3jcf$^wzc+7u2(BKuddg>gDU3nt`U
zd+9ULXVk|_HCAeBtHfVa<bPp{>_0Z}v9F)*RGRK}K>X^G3fFNXV=)sYSKQ_d=mLHQ
z^}B6c0RYWCK8heSoSo)Q#2lv>G0mC7S^i;TG||Df*4tz|z!#80!vsjHHz&(fSmbRy
zP(ig$w~^}OBulftzdI0Zh3QYFS*V6bJ9}t3)|1;-FhhJ@M%CO10-5v%-y~WX_E1uZ
z73d8(<YFZ2DH`F23f2&o{8jTacJbo6m0opkGnzl(Cl<4N4I`;OW}(CN98NytVF{j}
zx13s0F<u9&qmNtc)#_7*9{sjhnatjKD(ZX;YULjhkgsFR!YJl;aRpEhXCj``e(xv{
zQG(yRIrh(!K8Z3fX%<b?wjCK*Fk6jji%D9!M3_7q-*BypSe|Mw+A~1gXstCH(BIm$
zt{zSY>)Zmbf$cJHHeHRar8ni*I;xEH1Umn(qI?wK{>2l%boarEC7n=}DEG9zG?|i=
z_!BAD24QX?6oedli-12K`f3_K2U(0sE4;`28eXp5*lnE*OFSoZbqdv#`1VF|g_0+3
zZxjEKYI+@Lzt3cWbl2-)3F+AzHc9ph5+Bq4k(yH|7<o%&tJmb=kdRyF_DVh5rd^IW
zz+w&u0-GI2^E6IE{ou87UihV$6j6}%5%SyTW^xUw{&Gy-&Hv8?%6L@c*_VaZi#a$c
zK2l#?CTrZ*XmiI_H~wp)2sNd8C!6_l`}J!ITZ5xHR<_%Uo{CC6tx3RU4X{UTmlcdY
z0;WA2Z`{dH5GaFf3rheO@f8!(mrBXxWyns$DjVQE>Z>%{SqvXh(E=$=752%(cKi5G
zRvlMxcF~5npN5w^hlS{>{aVKYm?heLxtFUfvvndD9siBu;XVQO|BUa?26@()ImQI<
zudX3hW&U+?u=REmS}Iabcsm7Nyb3yZ0Gff{_Qe#M(ar=K-b|lF^lmOv;2g~kRW{h^
z?(TCNFp8T;9^ue2Zj8+>9Ul@bXqd+Vh)#On)?<7Qwcbwd@5wwZ%7`C^eN=x>@mAaq
zudzSqkNT9@wO*n2%NJ`XT+C-tTbw^XsO-MqO6H>nT9VJDYrsCS)1@m2y{4;Fd50!g
z=I9o%J|wSC5U4`9V)!-CsID2Uy{oa)%B~x8548*ro#a$R2Ww&iO#UFL`PKQ^<Ffw+
zuU77cp`_)ug`tqN_U~dXV-hlqCyOiH(e};3&rp6({IALjsZfIjVXNL}A<R)kW|4q>
ze7aO$hWs8zZ3r{uG7?t0GY(8~r{-duO<YW}9l8o>AXRPoDIh@v1kNm&9~aEABVTUI
z#;`<5_9HyVZ?~Obu8wVwcF$$I1t+~)&PKI=j>Jrz8$8(xTlD2=0#L#7tOk#jXs5YG
z)vV$2DIb|nqKYRYM3i1J{1hlN@6rrR%JQ`Cc_5T)inE#j1TS%H^G4(@x!Y{`Qat{9
z5rFL?E6B4nIDnm<owor`T5URCX%oWT0vru_eUiQ?!^Ds{eSv9LjT=w<lX~Za;>+_>
zOA70Y)`9kq@sZOUY$-5Pz$k#R?Qu|kZ8<ei(@ubV{_P*t-FNhXL5@Nu@=6^dNo*3%
za-;1MDNq{<SX3;4Rlhx^q;CaEx90erM&}+F!!C}f)+t9%S%_-cVbb3-E5FGX)WifS
z8*?FE>a77?c|y{yo(6z~&HxFz06TK)xxbYV4efqxXAj_Me0w5t>i2a*hQjxQHhVzq
zM!l1rzB2;nK-O*Ps^nHFA?7ZY{x1;fU<_PS5_E*H7{qHiCJ0j;+etnAVOAhB1Sv(q
z>!~-*#heq>Zp3_lNWuFp$HNW|d&_=>jTb{M0=7~sGZcD!g`Mr6kZIamq0Z--BMUV5
zirw>Up{L8F@yaJ{SpBfNZ12y2Le$)lm1eE-muWm_ER$d&`Z^StG4niqD?<|PVph@2
z;uihi#KQN_?EVYO_!U6NO95EIQsD5s>6(mSVj|9Oxu^)uJ!I4rTJ>*6cCs1~w}=s0
zB3*Iau4xE7vYk?c8Z@t3vD%4kscil!966nN3}b_g5mqJBiu7;o?h%6YDbr^6{DsGT
zI**U-jI%cD54n`nJ$u(1q8=!Bi}GTGbwSv@4{_10k%fThPD;>a^pw_PzAu5DFCQ{~
z+1E+fwB5OsJlM-DSLV{A>V2qF-^VylTH>Vr{gPm|#G&bgo0T-+mco}T=M$n8Z@@Kc
zleGLa)HG)2ft9vDR4&`IZ-#>2q;#a0XT`SaHhr(-9(U!`5<}6_$fe{BBZaQZ(&f}E
z9U*r~^42uVEY=;}>YGb|0(*75u%>-2*t11<*#kbC$1W%z{ut5dMmn^UvPRZhVKZX_
z%bhSVhKNPn?7=0{+yvJcJ5dNqQ%z`IdY`91za%y!z<x#^{ig5#D+MEXly9nXteM%h
z)1jk2JIyM>FktbPKdjk*H^_5a0taylJ>UGEwsX$d$rf>oQY80Nk-edWr*PVGTp!hO
zk9}n#?UMlnb~e$JgIzLUNx4`U=^U-ydmhJ8T6TrC4Nk8s7`wghwk`6hTEU<;B_I*d
zEqT_XZiV&-d;@HZ+0V!MQ#2zLEnX`)#U|Wh1i4g6^8I5pup2%RgJRV2)H%wA=jFl4
zi~Vs}>Ory~n@@}4ucUdWlb3mqz!1iLEtv7D%-E~pG6mxu8ILx7-P`8gp-AM`ZwNGE
z4hzedU+TXNvd_t{`G;o31?{4ib+m@t)?l8mmJ9F?oRW3eiV9DZoBe$Zy7a^KqX8@(
z71wwF>_arNL_2G?uRGdU;%XxO6-5=E-xD&<auem@j%Bk202WJTR1j~Mw=~Yzh_s|1
z_mQ{Z1j@WKx`oC1d>&7VjH|7_k;@)NK69~y&xXcpQ!0tcN~H6LDP@MNV>n|dGxtl0
zrJyvIv6jVpVMmJD<=Zv?uH~5Cu$_AX0DR^ziK)epjllUdg3iJ%!UZc@e@!lUcX4TZ
zx|xhFcT|YE)6F#!l+~nNjLXU(!S_JrN+m<S!@|nrIr-pnM2E|F`9=CjW_0E(nPAS@
ztXS))ND-10!|qp&ZD@8EhlUFYPLtU`<dGXX3T2SUnvH1<j*wRu`y1A46;?g9PiHh#
zO~9`joOn?k!9)jgV&?g*qN#%Hu68AvxBSltEwp3W$8%if*Q`qBxz&=Z47g>w%>I3F
zY@VO&kqr57@_$&u;i^M&c`~qSWoA0EMKa8pjoBEcU?$%ja?g7%PArJyGdYmIMbYjS
z_!1sXzv9C;?+e6fL>@X9I>>Le4_!Jq0-)N+Gg3#|+X)+TsG<@rTA5x+8J^p6h6KWC
z#)|u+znHMJSYr<fOBALSm3BBylWW9kP%noDA<k##-a>{o`IXj7(h&a#G*np*1x=_C
zOViD9{AIzQ-mL;1+4zX(EBKgs?JJaecdwI&IUxhUYaGXFsT6No858Hg+_a#?W#)dm
zKu;4H#*~?lFDlV<l3tr)>us=Cv(eWhq79meQSeJ--z{KolBPZi{p*%DTiKU7+kCZ#
zB)Ho7&WXo#e)+AiPyNM9QxBp#fS)F*%@qWYl*SS|15Y-YTV1{L07k5^XI3}RPeW<9
zkHX)MsC*Z)V?LBzbE@{{L#MCxa4Op@i6Vg7{g+P5jh#yw`P%O0T%#2q#>Qem$>ejc
zQRnaxO=r%ys~L#lBxm24(gQ)Vo=9+=gX(`^w4pn!%AW~Qa5=in54|KAWUTE1b^&bJ
zNc`5T%?%J8+YNi#DUY9#4*Zkk+++)OVmL$mc2~~Jl~q;$pf^~C+;!igr6`i(llu5y
zJ%ADr+t=qWD*NZ|>SGZ;{QeL9d4Jw@p0aDU{gxU^S8*Y;#%D1-)*$|5C+++yN%--Y
z;|Ni13K)x50wTmp`R?5&f3(xK?)ox{e&Jk*KgX}5pGT&P)z+)KZ?RYd9O^A*eA6hd
z3Z6Vx0=~Th=B?H-WC0vxJni2wGXRUpzU*%E9QOy(OTpqjZM<^Uct4M2fIq(_`1{O8
z3hQgyAr9hhL*s&A8Q>sFNy;Wo0=~qKtZar)wzEt}fj=6+<I;PhnJs)nXBIyiqK>2^
zq8&zGq@zg|DE6>?H~FU_toQ+_WThc=yX2ss+UUkEnGH#ha7*Hv#)4~F62#=~W=#`2
zYY`X19DLdD7}eRWz#GwZ_X-G035-tnG>oy$S6e5z&Gf+|LWuSA?_$g=FBcs<CHsUY
zV5|R@C>-ZMy_3Zf4L%al==KR#U>Bl$>(TyB-!a29?aa{Q9hhkW<@GJg=x6kDlD&^E
zAj)^U{_PA3$xLjdjoGLXeVKcRS$L@B5|8z#2)UveB~iZ+zw1X@UR8MCo+gJqzHw**
z{W*tr^Yt{3RQ(uT5}Pfsqd|N2bXX_fj;DY_%~@o1#B8e@JkC~=L)?+XO`e_pok&%J
z^1ZmGJtl&%0ibE?pI6?0S~>p^>Z0mY8OL0;xD`Y7k4n*O7_nr~&y)5@!fm%hNgc|+
z(%K8%TCCDtFYi5%)||@}0iNkZHkPy>TF?*ag(_SQ39iw+x3YxH8oHHbElS_2kpGc$
ze5oD&(OSH(o1nwx6>gVF;LyRW<?R49S;A2}Nkp1!gs^Bov=y1nJuI`fSN)y&G;3MP
zk-p3X4*duud3q}4<=4<!6W>cojNL8q{EKr^xx%Wp`8I6rpHQiB*q={TXM1bsIV-YC
z=*C<*q@9Tpyrr8`{B`x#dyTVm`Zd4skfkqucB`j->d&-Yi8OM5-;N>gmxJfZ(*O+v
zcE&kt+7<_5o83DkQ`_A|;G`pCsn<)Z78yFs7ot8rG&v<@+9H4VS2ec~QfFnuV7kD|
z{o9vO6K;1JNu_FMrxw>xPgm>c|D&NV)0pYc>=p4E!0R#1y`g5!G(uL86Y|pkI{OLd
zcC}!4EZbUGnm94rN_>aQ<1FDjsXIDL=4P(S5K#)oj`(eb=k~f1SHNz(I-~5_&z)wA
z=Brid`)_|wXCMn~GWv{4=o^*jb5ec`S_P%9EpVgD3a68-DoCc*yT(~M$Q)HW7KoUI
zi;ZPdK>u4z3d;Vg^Ir-m2)s^WRO-?=@erf79nH7Iw_^swU<x6ZhGQ<<bics59eOR<
z2Ak_00QiH+$P)gX;r7V8!7Iu*(s&jOekRhi%<p#cb%>8H+1q>TvQy?gJuvCEK9x3X
zIxxg2`#Jc&L_B2K<5GM26wq&)CE#CI>D;?J$&=_bU|<)DfKKWd!O>>wkW&fm?1tM_
zg}m8phY-TS<m4BgPp&F?_9`uG%CmO!Z{~-A80A(c{dxCYCObgtoEm%8gAePPX2J!O
zhIW(yu6415Fe%@!A7UtQ2hY`&)4~MIl44>7X*{c=d}i{lRSQ#*uJFUqo=Od8B%Yd6
zZ6B-sJR|aA*l8}~3KWMn05sd&!-RtV5QYXTf1=g}g|5X9LE(`+mT*olS=;o1srm?7
zX<adsNh*3WWY>#4V|XHGcC4~!T2atnNbtTo+4<3TN~0-$;{9Cx>)6n0Kk@A^VY^Mv
zy3(=@<gx@c*7hYYIe&5fWvjY1RO*Vr_iu+j<GfZI9VkBeAW6r*$YPHs5QjayxfHA2
z#=-%or{=bCb7C`>_m`2fGd!?3>2L`BpU{Bjy`49=7if7`nLWZt^aq!S1`D`9lTV&C
zeT$tg(-m2xl|dZeX0#$+eX@YQ%H|Y|y8fNOnmmQgOBYVuWf{%4Ri*MNJt;|?RT^lT
zR2}_IZLR<>HYyhtlVjb*fbSoWRXO@BKyY!~%J{iCD$C<iz4eSuxJ$Y1sS_LsrH?&p
z6!~iEalYO$>x^lRM{3BT9U#b!>#D9aWhdLPPIj3K?3G!>9#b7$MjvVD9D9175uFD_
zZni7d5pYTISjyGQv`)ZY3sFe=TSaEu26JEhS{%}_#q|x#JC5R;BKCJ5w*nkMYo7{P
zXn@HUsLAW;f^tas`~&q_BOz>VuaGY}pmw_RM>37T&A{Lk4)=aVX?_m&C9iab$7us8
z^7=PsY9}S9mdb|Rtf~xH29Fr&1~zG^W2$<cg1%*xPTx;5`&tUogki|pY=YvuaTHN<
z8jmhrSfF?t3b_bw`dt)h`T*XX5$A2>d2s>t9fX#Y-}U1}_%_-KGMmXkpzwY6#WJ>2
z+q*ph7$=Awg&Y|DBinaBx}oaGT*A5&+z}}K_(sj!@#?!ad<sR1k;hdjZkd^`@6H^b
zYh!Patoju+$;niIn`ZArDE@MtiMaFv{==N#vbT-R@fSPQwsWJ}%vRvf=wF$?s=BeF
zQ6|Df!c$+yfYY+v5?RD3wnsn&aBYviJ;Z)LIu$`cr;C@4Xrq6miJ5~Df4%2~>$wH2
zAlRQV+AS<=zLY=vrSzT6p`q6xfD^KB5p^_+ZJ!v6mKyQ+YF(%x=kaH9LvCv%xgtnu
z0AFgm=4EUZ5}>1z;Rq)<P$QJgM-RumA@S~-H-XN3i{;Tyq}rTSz)gQd{2^9=YL|ON
z^oC*+dP|A&f1LEb5NUIPjE%0g=$)>v?3`Zuj=8v7wB8R6dylR<k9u3OoLQv*%chzd
z%c>jx+42?Y`)iMg;6m_6?x-cAaM7V}#u4TvD{>oP0~df81iP{rr+;d-EIn3>LQcbJ
zI<r5iTN}}4wpi^KoqTdnWX|9p(6nzzt6BVNs`u{}b@q>S)%LV8Y~;n@ot=We^q6p&
zT~)WcLOdj|tyXGHslq(6K8*Y{*8jzTa>M7(X?`$q)#({ajYj7qJ>svuFdjpk4{61-
zPzgHQwl0TSdCHPrwPR$G-bUkkqxK-Zfy#nNd&RqsvF-A~o)}}U#GgXkb)_3<0(M<j
zU3X&yfzfvK0$~)x$B>7dFFm!aMlEa~pk^H0ct#H&J`lxOcA~xZ-mer=5Ss6MZ8AJD
z*J@qDJqyt%(X52@TPTwgUOL(c)jjB#B+z0wF;I^~S1i#U53G3#S)j^nZdd3*A3nSv
z>3eCJ*;UTYSe$k(kczDRKHmJ<(|<&tgRp1sJ1iI@d%iQk$R54qlNtGYM^juOSJOL1
zZl*I%C8XEJF+vYZ+)GzgpYw#VZYiJBJfi;cS7R?Zy%X~N?uKQSHfZ+w?)X}Esqp4A
zmcxXWicP#Itm;H>LGCdb-hUmZWwEdhw$uQ2Y_!}_d_SJehwO%_{wWHhSc`tVB78LN
z1kbYI&G3g(47x3RMSIT0f3J6AqqaohSMjfIh3bIU!k1@Ox)qknjIBo}W}=|wiAjXj
zg?mR{Z&f#~nY2io5gRu5>#Z^2DNW=qjkTpBiM3I)Ut4Hw@`4@-Ge!w_wAa~znMQI!
zBv?1>lr)6-kXy-6qCMv2i7MqJ=|R0X`o4Qx$hmxFw)!HQCUw%nuFJ)a)W1JOMX#hh
zknclIgi*8gS_lQ1a`w4-<%nUOXqGiac<}qzqnYHwb4GEG!mNBY^4bv=BV68L&A%QK
z2zx^Ac&sPE8_!r{CbpV3o7a(i=ZiH%ENH=FaHW&O5(Pg=^{TvoEmm}7%VV=gSq&a9
zGj#+fA~8>_mh5!nyx(c}LMj3irQ~zolCH0Vl{dLmo%gYVJwXq6f#hVYp^24PGu<lB
ziJOGubRvi0@HbfAcc|6HQ)Tb=Cnl1!4WqVdeKg{Y*v_>|d6iY^xUX!RFN<)+R7AD{
z1cTaeqSYbiJECS1Wd4%;H5`jJa2i#m@Xb$Fv_Guy(_~)-xTc}sq(zy4eh5eWRk=yL
z>SL4Oh<NR<FiyPRm0Zh$*Yi8tY$-lLd$R9lkd+;aZQM9A=8z$Xfu<du;(>R%7uEkD
zQDim&&D-uSd5#0le`e^1qBVz4VCBgh+d?R75JR`-vs}OEu@;Z(r<^kY?g+J~NU*%4
zxdpi8W7`O-2BQz<WH?nwU1e-|=yXu>gtGF?iHjx*$EVIgR_Ln)W*E_Em2q~P!&vie
zlrK-oc8VQzz0z{N#It4eXy=N0H1Iscqdy&Y!R0K)Fj}0?&%6E1Zpf|f3kQAC*e{0{
z&r6B{cPW32$7D+aD{c($+!yP6_CfHnORF#Gjhu9?t_P!l<$Y^Y+dJ37k&F~7%Hx(E
z;qM<^^i5oLb2#i~n<shD-7$>req!_N&Hts>OAaN?pq7qCcky3r#pFRC`BA=zXqI{>
zR=(S-<l~2d^ZI>qf{HL7jihEZA~N##1Z94D-}xRSn_8amkR<TUf@OAwkvk-(yR1a#
zZLZ)(2QFjnHom6QdM@&ZApOo1_1#i5u?6m=Cz9sIXFJ=9DP@7A_X=iv7RS(sgh=EL
z@7IhfUtEI%lBlyhHzSVA<(gDqJRhO@81)XTevc<+ZqfQrA$NQ4ea-g*?%KQj$p<YI
za_7N^Wf$jd2yw-NCQjT=QhsdTr5o$FH#nB2zTpfKkROT*M-K4A%X|R>(tx|sMJ;1P
z@8^}H*5->ZG8B&gI_IfcyBWVGffK@(n~nND2$gT<3NRv2*`3kGqqIGQEypQn^Sw7T
zXY{Vg(-+>SY_MxOsVz89Va^__!B25@`V|?_g^3^AbC<n$UCV$>Vj})EO6%|k?EImN
zw{pO{LaEhl)s#}i%#olM$-A~`8=eW-{e{7jp<a6(84~saB??ZIS&TYyndu09q3ZwA
z5KI_B>Dv5w+Qe74Oh0YJR}$&#R_48<G4S$<(GVT&+=BG2p@6MK{pvy>t>4)DIa0^w
zIOu-o9v-?T`WXZotez$6C3H0|q3{~deCSpIo%zDX<FLeG;OIkeORA?hQ$M5KQg4h}
zYPRM-Qw8Bwf4ow#W>g0cts-sa1cm0n)-_!^C-_&s#_lMC89L*MXk|YQyiBZBR1$T&
z3P1bT2VJ-GM0I7kF(sDE)Z;C+?DWGQqE$Bac}`so26ngh?2_?Pk^a>PDKT{(!UKZU
zKl(H&j+)Q@+ul^Ln(jPedbK}s^SG9Rhwi)pWok5`9sjcPPSNi@^x=L0Eh&v#&xQF&
zc>UP<gh)Bl&eXA`RZ*m51G@4u{|y|w6kxv7ciu>mV0DP}Jyhq|8D3*7chWMX{81qx
zQZ#6wlnzb<|NU)@>8Nr9H@j_t*IDiclXQU_nu_p?Htzgw6+0rfy9jQ{EOfQTv6rrI
z4gB)Nn2vYaF<n@`iBK|ePT44zKxD2amimj$)Z~s$k@A~{IGkZ?xt(yCYsu;vf609n
z|EM9n52wJ~Jx{~VbEtR2%7jB}<Vi$lF%Pb(^xCR+?N3Oi%N@yqLbt-%SWP-PqmEKN
z{PmaBDXQhXB^a0|<^4qBUXrZ<x26f-dQ1#*yfg;nQ_4$aIXh>Gq!Rpcgngw(jCbS3
zYpqItZRiJqmL$;vjh(_8^>QTi$28%0i<w@flW;~OuJcj)oo2rX(S#{FHE}%Ohcg+p
z&FPf5+MB;h>9w+&zpdv=UG9X7WrAq#DM!yqxGhJk@eAf`P`)eMR@YR;JV8hv)0W6<
zbX*4gELb%WN1Xbe*r0g&59Bb@m?r`F-Fl6yas$IM&nEY~x|sUDY9^?1GG8*yPJH>!
z+N5Zsex(_(fMpCQ@Zg_}@>;>jY3E|Ay`iScy!G@0Xi?|?c&Q@JCnWftkX#8;)A_H|
zO=8H6T&FL#!mqDj2vT0Y+7TxQ)Tp@HZ?Ht{%qad^i?|QQu(A%%GZY@qkE^_Lm3<c@
zjPua1pnSnNPCY+l<Wlf~J1}#V_>DqCqLFD&mi7-N0mlfs*gBXq8fyOcKlzbf?~Djk
zvr&IyCnUD`Aiw(BTZ_nzib`SkMuASO;^PeXZ^0pABK~JNLFW=9w(+C)Vlr(eL)?Gx
zg`UHYImhO4NTb1e_aECHwura)^i*%phx#5huP`~8g!CuMyncPDz`#B6+^2IwjE+5L
zrycu-1PAt@m7Q;x`-(#4wrFcV*G2zR(0;D}Tz|pM1}8E0nUA*~jG`yRY5RQFBUy*D
zVmER=REA?Opt~j7`HRF)?E6}sdK)&YezR*5t5R1=eRi->QnkYbf9#i5<#3K2y*a}%
z)D0~HXm=Mc?O5u&+Pt*bVG4XD3BuZ5<459J+JODe8$m(z%?@3AEZiB-a_nEBG>4EF
z6T(;hAdT1L!<$&e!A6&rEf~1nB@gFyK(>9L%gk>BcOew*jh#?;(7|6>x2KLY=69a{
z8)Bqq(O<`iUJI(0$a?<i%V=^tO15k0>Ea_76zAU5+7U!AmBqY9v#d4WNVy-T=tRJs
z<h8_OV`y>!o^o8m@Nkdu4C?folH{o737S;>qhtQnd!~*eX_7ySKYXu8DW*&jg|BbW
zqv5te&ZC%1WWKH7=P`6LP{7;H{l{v0U-ZoPm;@Z$${vq-XRKYGM7IfEhrZ~AOEhTy
z9sC)iGL|Kj;L^{aBxqNL;}kH*tVyu(5^F0A)a2vl1$)CI&y}s~$QM$>x<Trr?$-IL
z8N4Msr<OO3Bly9{r6|uacNCj7xXz;X7|0}_OiT5v-pgL5A%91ucf_hpBO#b$nRmgt
z%Qk)$oAOs2_3NvU2V3%c6Qh2&vJCi{L@6Ew)mKXL)jgr^C3(*ynai$M+2E0@6kkN2
z+t79QHRi86hL;z@9_XK#Ih#^FLlMuGk=wqHaJ-;7y<L^eTPs;ppVi_+x4t8xl9Y?Z
zt~~--l^u*S0z1;a7cRy^8&X-rDqdV4<#E2RB@F1IK-x&L-&6xa_SY71#JtF(lKN&8
zE-$<sS?f}oZZ+1%*3^73ry!Iq#@cjT=}Ow+iN737!EPoiP85C3=+Z)r!O83mwI#cm
zAERu^S1G7cp4_Bptt%cAaezt33?I_B4b-aIp1Q<x9WBq0aM>E#92KA>^e)v4pzgXl
zkRLa_^OfHuAT9SAxoZNZrCZ9ljGLTFjji`f=C#nkGa>IYUsq2h`ls{T87&IjGl~YS
zSCO#d)|*A4P6q#!l3$4lw9y#1D6A|6CD01ZGkd$a^R6@t=A~UubFRpZ7$eKAt{>or
z;3VeS6Mw%(DMh=g4{+`Mf~C>g0qU^6Lw2{?7-Q03YWUA8Z4|e~Lf%goy_&2M$Oand
zobxw}4|}!u!j|(k#ytlAR%0<tZuIA$y$=8EuoXaiDxI{A!@?#Kp}bX|7w~hk2AefD
z8z=4S_pXuWv@=wKlm^s-#F?bw9BVudW*mstg_})ZHL`V;@T?w6r-oVhbA!07hpCaO
zZrgOfzId$N)&dcIt`E@NYFLHV5`X$L=iU(&qipokz7#3<U3|vUTec{&n8xMV6|a|!
z=c=xjfxTxplV2LYIpIO0-ph-@|MueMPWi^9!|Zpf%vbDjWSw)yq;v;ri$*jZ1<2i`
zwB?G5{*i>!UA!<b?g+ZO4KNfhJP+8;VQkdc)-0D9;3*eIzjO+F-?NayI$#CMwzk5t
zR@^H`oJ(K(y8c<iW(&$KP@qlzS`TPHU)e-H`Pm<Z*zcv)>9$N0Z=rtZHNG{9hYG(r
zI{H1jw@s^*7B*L&{9U%n=B|`S<Q=TjV{EalOa}}rVde9KYzxm8Opy$H6M;KN37s#O
zvrCA;!hNAfx^GD}i;Zj2><*SnO33P7N+TYc4yumNJcNGPt8JYCxTL~wWi&1^aqE#_
z*2<Dn{+h`>LFJs{3-!-M2$8X(F0P>UrElek*c8;RqKX^X89`v~t7)l>iO9?=8}&=&
zn17P?Jz}ihhlg~@lJ^@7{V}Qg8z0uGCy41MT-H;V_qkT*jrf!9;u45|L?}bXZK3;S
zd1_8p3ez$Ir6p~;<{3G<2+0EGD(GCG*V#cBPG(3HUy8H5UNxIwetdY{Gf>U$Z>|u@
zADDSLwHsW{AEm4>=+P@hN_rSR=t=%7D_*!qRe5om{z83ORcUd1j2v7$zaitt_9s!K
zwomhhsd!unD(Yg?%JURx)PY|nx`mw9xjwBl^cSvHQLcDu@D}mu;kwFYl#j0A*9cO0
z+so5glHZ^IL)ce`McJ+I5+flE0!oLJ2#5mGgP<afq)2zoNTbxih;&JJr*saAgwowH
zv~<^yL!Os!@9($2bAJ1}&R>YE3+7$#TF-Oe_fv(B^G?OE!hKl(;0a@eOSR!i!Hg)0
zN~nAV&B(RjJvEN%ZA107LS#PGZy(5U%`p(MfVk~{#JtL=n~-=OT|490V<yH%HW0{Q
zC!zsN3<}1_G}vQWU>ei+`<Ax~Oy<&LzS*vlvAyVEeA>j~y|v`#y4bdaqG^8+P-1M!
zILmzeV{u-6XU@`QRU2hJ$WplP{-cdb_u)e@9;_G(zXFjJyAo`5<{}hh@^COGngq;g
zJ@+F-(7MTA(0W>OGS7Obdarq0AsTuRTG!-CTMM-;I@aTZY>bYs`j_NVso33bYP6%%
z3f&pCRj_S3Iq#CEe%7M(NWIamwp*9Jh8TRlF>EPoTV+9#EVNry9^6K|A{nYO0M#%q
zDyZ?jvDp!wJd!iy_(HGMYVQEDp@2){3+RgpRdC}vSC73nW03QdfJN%Nc6>xUDc;u7
z9d_+Y!*R=bl}y||ZVg!xKyNK9cqv7^?;aq0VIEfAHZ-+mXQ|W@F`WlunanAh=kNS*
ztoiOBSG}wxsw6x~$j;oUSxEL-vn^hcw&tB^lf>uK&Mw4jxk_gX3pPa`5qGLu?(!Ma
zawD8Se_gJ0WF0i!33r_`_F!Y$e{d4~f@JcupZGlA2e>dcoINhyfQG!$W5r-r4B>>O
z6#DwAh?*I`+Uc|S#eUyEpCrM3b%2&L1!Kw`JHIda=bYXW0>U}Ox__s@=@(Z<=n@&n
zp)qqbqsNyh$#A`sZZ>au5cs@ux|isKZ#!jSkP$~Iwh#>gn?vQ!kacKg60$Q6alW5j
zdhjP875KSdNE9WFoWuPb<tX-Cc!w8qBqAffuQeyXG>BZKF*9j>3Auicfbq>EAAn4=
z^sj@WWI1>=)W3eq<_K@kMY13=w69f_z6kMesPk^bq)VERCQGh#M98qvuEf*3Pc97H
z6_~|>Ta;5+f%L}|4B>g3?@AQoW;z>JlWe2dMsLn1n$G+BVmFpA^yo2Vf05aPv^cR=
zpf5UPWEq5Qbn_@h->9TPLhf+kWI2?M#Uk$(88*T2A3XS8SVuqJeHSV@AyE`o9!5Id
z_983Np7jt=xD>QhqWXIlU->|D-erQmL5>sa5?)DylKSasrX{?0Kj@D`Ob)3YSMgI6
z!4AbAE1G+`3n=6}_q2>`+nsu*BN{4EEUKfRq}bEyvtHmsWyFzjevuk!JT7z{vo5}u
z>Sp%j;DXI{p+p^eBF-T49k`D^tg2Obm%CrEAz`k@=Gjbkp8I|h$K4)+$r)5x(4RX7
zh`q*+yhxyYnp7)Yga@jv;ee&|YW<$KAsM+<IDSDG0j{p_K`$fx*)L{dA7SjH7f%=b
zBhI-Nu#FT_pL<XK$%nONkPIoYdI@KQ9&$-mtb{PSdzzO?dVXtBX9~D1mz8}G5e5UL
z3#VNUvL2C%9cHqdMlhs?3&%Njk<hk1^l#^5oEoEwy7!1so{3b!g-GvUEHCucNm98o
zhC^<&1ax~Wzs|%+R{u{y=v53Oy>xP&ls`txonOK^*@n0$-DP!-CfWQ$vV@=p$k=O+
zOkO&jkAnRXj8}9?B#2`0_*kv|T}};|6e)G0K`nCXI2X&Jgu=&ejZspIBhDbYf4$jF
z5at`qX=rWgozI`@aw&o)$5+C|b_=5UKSc$T2`_~{8+K~hl=^Cg6bjlgN6H8CL_R#4
zbsCVcan~Wn(rnmWiuJk|4E0U(zDAq%@NU#UW&j~szD@{e(X4p;mXI?`F_Wg_kT9ft
z=&H_ook{}t=a<ex53I~WQu}8R1!>~v!-`^ihS5c9+Xh6$m>h!w-_<uM96@yOM@;tX
zAE4WTFerJC+(;bq^Q)QbvZUg<y2u)6TQ+9ZMpgXOotx|a#^lU3bIp(it-tKB1||Hs
z?jyp>b&7>QUC0zMd)jNO>7sA!%8~Xd?Rb6YdEd>9)2aWCy@K7hI9ChJ6shPtZzOD~
znBo4ER_%ViJSj|0`=RNES?fArQAtoBE|cS3=y#kzdFP`RN-5bORqbT8gLMixNO#lI
zM>Skk>C%jkHjF1ch)0@md;cwP?biHN9#?8_L#_kb14x9rzloYg64P$yjtCE#$|O6q
zMa_m^JSyp+C?5SzhusE@SHmQH<nb%%o-19F)i&SYO4+;L;>Ry!A1gmSMy4l_8tt!<
z8g+j=>jwV6`rbNf&1`d-G~{eOCcO@o-4b_+=W-saZkoBwM1>Y1mG|Uax~yngTpFM+
zN^A5}m&&AB1SQ${tn&Wtqi$W*C?-}|v)Uc6^P>u0{kn%ap>Bt(lco_8TDXZ4ugvjM
zfccqRm>Dd(Zv<uh2`ktX&bn9CC_#^Y?oJ}`9UtiKls_5)I$x!I?I|RmhH4RF6?ay5
zFnGks5xe<orU(aczelkRKA9ul!3tWfEFJ&p(G~nuGqGbGKV*Ac*1NpJAkADM-Z8nQ
zjqHgNR+!-(;G>mSve*yym+T}+Olx_ZA5!yHh1gy6w=0&O2iuP+uZI8i=|@>X*ZTpq
z%@LkYAGV<n*QRPF(%!BNiNW)_RVj0gfjj#SZUGT~nXwS^WB}b3kue%?y1DrD@m&`O
z9BDx^2bSfClEt9I@)-f<jXx1a6sIFvFq1a)0)tUc0D%J~u*9=9@s|qvSwo`RTiLRT
zLKXCiFZduRsKC&4{OoB1+ts2T@XroI)8MU5?UZ_Xtg$bg#JM;x@sTvuK9(lP6)aiz
zHarVCLit6r4vfdjAorI`8Onq!895|kD;?S{=YbLHpo|cbK92CrF&lx>Pjs!>>2%Ba
z5i*CWFtklS3kKK685O#|AWAEDzvP0C7gWlaLE>%r0&ZBVtN(m~w^&2UXU7sT=WeSD
z^mHz{9U>nrK#E1hXOL2ycd(jlH}wUdf{t6(C|5Q^yUYLqgi`J=b6!Q1-<cZBlnL>_
z)S$;A;y}DQe7!MFrYSK*k;jWLuiJX7$x&;&k0%Vz`<-r*o&lT{4WD0)(>0aRCs1m{
zTJbN=A_DTwBF4TdS;JIlvBA>fE$#vAEDdI(x1dd{>FE9w*U8~}R~maPCF{pqx<z>S
z0<yS-9e$>+!e>IgVv=;2o9C{I_z@Vq7%p=&TnnU+2?LHYlg_)-$cH%ay`KCEkX%vK
zuNZXtBm3=BX3}#0L-0!mbSIU^0i6l>yi0|P!Pk)!?Cw07_32D{F7qMS1;x6)%<Un6
zB-$w>(a)GPMuA27h)|Lt2S*vQ(M2{PXsq-)x+UD5@Darrz8c}AscgD(lw&ssVy!h(
zcK^4~iF)|=)^x<>sMhsh>}rIQqe;aJGX-9VRZ;IaKjXOoA8Ke-YE*>~Phi%QfWxTE
z5HJ{Ei`|8PJ@{X^@89sxhjq}Nw15|$x4`u0na&Wfix#iaF$ea5mE1IN!KqT4Fxm}%
z>6Pq6YOS<Hi3nMozi*{rm=A@O4b4zp*sZ>jjfJ(;{QIszz1v}^q1_@&@;)<0>bZ<y
zMN1qu16-2x54I&{Bm=munL!urXT#F{!dLQT#b~~Rxx2!446fMbvUVyt*^nRd;B39c
z-}fwrCVSB&g7+sF5#kig$9(Z8=}e8KJ6$4O(T>4e@Q1iwcY+NH%U_N)n8%ASl)W@6
z$Vx59=|~n<08%4VbVJ`^5z*lAUnEm7Wn~beQ7K?|MamZ}wPap)^N5+Wa_EZ;_TbQy
zpOTRbhqcRiyF#W~7k)8dDSe!8=E_hd0z9gqloRz|7~<a!=kG6Y9ADtEX?njE*HQq>
z8YEmK?f<!sh7JoL81wnnz!~ZQz3lj6`a<|q=>R4PlVlEsB}O>r6`q$pwuZDuCP}Kv
z7aJBN(<4gXPpz-{^tZViz7s5d0v!qfBt)j}x3+0tX+Y_y?2l!IM4kiTz^?k2I8i05
zg_r9?%zzVxXxb14IWz*z$PAm12H26rp99!SeiJaAgUaz(kBnt?G04?9m^7kco<D?<
zi*FA0fsn##*7p~&pE<<NV+?iKh621t>;o^QAETI$SkIyu#a2kh1e%n2T>~9$7-E1y
zzUnYYPzTOp9WYGgh?Pdn6<K>%_BqEO=w2Z+@Q658yZ^pN|IcwUA_cCIuSo2%D+{o!
zHoqTt_aDbj%2(zWzQFNC%)fot7wiw1GoZ8`)<wa=RUfbqQ9Z~3uT}pm33Pjz6-=gl
z4_m^aoP!G26Xd|!S#sIUZ0K-*JzwQ2GYQWjh_9hthQq5DF1e^0Me6~4{)f+D!7sQB
zC%-XzTs|Pla<eAv2F7&sk_GKNO|rSv9JcwM#~Vzh5{S<llY{}?3_L%G1yhXYm}N^I
zVel|jXiBze&@ygwE+wFdHmezF(IaJ|LV+fZl`hg6@g?%@fmXeVS{ow9*)Vxxw6V(L
z<07SRDWgAp?uXzB_;RSEg*Gai|MI~c2a<{(t~L7+KnF8kxFj_Vnki>+-FtER@o)6<
z_hGulG6e&-mP<0d!Jq#Gf~C}w5IfKeK-d2)$&h3zcjk%+@&>BiVbzQ>l+r}UKz}Pg
zR&`#T%F(#TIu-!AC0H3&<*oVQr1SB&`774)<ME<tKyb2z>Q=#ffU@OO(MhqrdMCyE
zZAklGeB6QSHm5%iz|#19OzgErzGl4biu8sK_cTm!o0v}8WUm%!-sIo^{jef4hp%5H
z^*eHyBWSu+HYQyNxACk~1?~H2Bf`{3h8Dak_1HNn!i39z>i5GwxP*I{EW!accF?1H
z?}=SUC;UF4bcIH|0Tk|Iz|{FV`ehkDz9(k3@Y(v;5@qFe`OAZ#tI>94v<ulER$X~~
z1gUm;ylTg5G%#_wBHL22uqcc~G%WgicUi)V*!H~A%Y%e_xQqsv@8GKj9D^9zFaRW6
z?mOE4^Y;C{YjA(R+gm!0E{>=IZmE~Jt)#_2*U_;5bxYT3fLmHD^-Pj(5=8Tc&;5ji
z+DFFov#Qc39DAZaWUXF{lt#V`w8SRX$CR7JAQ^FVySXX{0@?NY>PRjyL~>VMABI$2
zGlJmh;kC2SqB==DK)Tzc&JJ{AxgGKM-{@(B-PWmv9AJ<_9#AQM5H2H$(75`9QKAof
z_NAhp51B8v$Hq4b8H@!!mf%Z}sB2+>&0R?skjEk?iD+BAvZ*!$L;$zVQt<xDZxvHE
zNou^SKoj+ja0l)t)SzrCHtj}53&W83hADTx0bI%{g2Kv&9Sl4#?9-2$f3f@jvGV`o
z`YqDH<s;hi9PgOCJ#=Y;j8*?!M??WY+0nhj_qPTeJIl0%8H;G(f^GCwPG-N8H<=34
zuU`c3?h${&v?m<J_4a`|;N@`mf5#ghTFoE<Z^qz>o)NXag-snTslZU2O^~s&%QKd4
zzlOyOnV59ZYr3OI*#p7w4Ef8hGT27>f)wMLw)nQ=ifno<X|<~HSN?S@fOz_lVF1ih
zty@J1X&TVr33w1}2%L?cEerW$H4AyR39|M?sVkdUE7B0~i|mih=~h?B6mS!ZPT{ff
z$HYgu&hUXJ2lKx>z?{3w2=38eaJbAvs`_wL5}&eZY-xEC<R`W0KIZ{uj%qo*kcj%{
z)BgvosA6KB^NLy$pC0Pb!Rus!J#&fpVfjDg<F8ZJdF%chm(&caU%kl?5i)wtPUqBu
ziHor)E2RD6VdQrbTHvsMA#adh<kBk6PnV3{zw0=b8*->18^o!c7aJs#zTHVM0jI-*
zf{0}x(fZWXSksVeTv0hXTzUsm)L{N)cM%NzElg#ud6|HcC`x#g45uwq68?BN5V8o{
zL=EA$5F4ZThd}fv7z&E#2f3wsHWUhtER`TlmhPa4Mb&PO@H=pS@iV5F>Xm@-voofn
zAm{A*m_W$A5FxKK)poSZvqaho0S@Pq<{<#CNkulD-4#Hat}^^*AO2z4iEwXOPKuL`
zp{3*gZ(I%;Mt+ko!0f5KM&*`{oqrg308n6%o*i^BhG7ROY-rxjb8*gCa?4h5V0z+o
zqGvnpUPCFF@NbLMLZ!UaT){-gl4x~&pI1N_b5cB@_s%<$2%>;9LR2PEJO&JY&P5py
zThXA`^}I(gE$nT@evQo3hAcMwNfmH2Rt&BMldWl_s!3)hwUdLT(Iy^!a_A-WPymUj
zCns6WMj3+iyDGCkPG?5JGgZt2OnF9Zu4E)5sY&{vqB2GX3l=_OTND^}M8Fy53;QC8
zexfy5l<w-8bq|*whUWrGhj^Pr`Lc>pom3d0gJjbeg9>DlF9T88*^CR684Aee4Hy6!
zezTT=fdD%J?{n9G817pI;;*A(Wpr!e;i-e}!t}uXYKdNz`RBR}I1AHf!jbG!HTPJS
z0o5!(xxvzWd&vdF19gP~I?p}$iyyGo=Bta{`*?par3zoH1L!*i<h)O`Ty}qWBw&zA
z1V{I1owSX@QX0**bU8Xky~av~d(~a^w1AxP{AGBggb-afU-(PaT!+E7OOmkU%kF?C
zRJJ;!XwXr2rdt7cFA6^Kg*4!|H96MHgN>1#iUlcIFs>KfR~dl=Ri%gFx%dhkOjaFe
z&HR_;>Ut}(9AL-nfC6a=23;%mtb{Yprfq8qlHZ0)S&VOWmm-y6XG};-Q(qz$!?2Q9
zgU5uRBo6?9{)wId!n*G#gE-Qx!f4Emh!mwfwn}PI!Ee>u>#R3)g~#q?F?9UBY5$iC
zd<r_!AFZ`7B;?vF;{gHEfNrO|-aidG9o&<Q^aT!h^HHw$1}F)Xly#|Y21+u;eq-AF
zJFROU#MB*V$W;3I46|A@V1RUbNrv$y#pY3n7-^wgGC3{r`Dg@>=;FeFjKd1?SWd)K
zT@H`7EJL?gZ1AGB-g*T^1hd8nKN-nQuZ)|!ykitL_~p`92``$e%c?_Lz2HoKQ*9{o
z8>Jxjg07Kp+>$P}Zmno@!r`HTho%|UBP74S5Ge9X3sz)DU^7N)-48$o*kC=ms^Gh-
zCfi;qqxh2c9q+1jRa%vBB`1dw46$;un#oK$<CH|;p#=?0hRaHl_FtJPaD6626ibar
zkXr4!U@1vu(KPop1G(ZlKrXb7d=I|B*snqTza8MeRFZ#xx5;tq?b6pun8T(3^S=Q(
z*Zb$Xf_c0^rk2ORu7U^5Bgy_nNLK@7JnwwM808|1wZB8<kIn0IhRHGm(p;oC4XT!Y
ze8kdWR7A(G^1`)M3wFMA86uthj(jB}jD7}mz%ZJRSmfq_ZsM^)(G&_!G2p`q&z1c9
z)v;g=inol{p@}eg$j}h)8-YOS&t@cVu-VvMa2{b`;Hut<Vpj?(!5D3Q#cP~JQ?yyx
z;V<d<M^^V2r-~|@bUG^gD`PH@$>tEq7_DQ)nnVCaae}>UZ2M995p=;yK^Zcm9mtna
zZ(-*_vSJ83$|-%JX-L&YzG5PIp$j*5!P6p#zk31T5SGXB=(5XE^Xgdvo$(5%F-MIM
zN<x#ZNdnd;drdbmqSqzvoU_$Olvah|iMpcyBbWcfq2BZNX$(2PE8w@VJ~wvjB*r@W
zCH}2E{(FCIsBRtV<1}uT_rS)6-@PA7(DB(mkPeFk10Ic$k0JT_rB`)Zv=6lpz1Kcy
za3qh{;AH_j)G;HU5#07VUvi^0T~#b0=LFTJg05%@P)oVvz|6V98nE~XCl}SjNH31x
z{;k5_Mok;C(n6Iwh3xT<!hQ>J%xoc<wjsd0OG%U_OsE>2YnBnnE+dgfcZ|dpgZr15
z*rrrcV_?C_nd6)*l|-8vP3$?FP>&e-H++9d2bDO-pxO-3GJ-?g9A2XvYi8sMS@#d#
zEqT(YjQk-5TsBIbgS`UT{7n}?Vv>gN_F_PZL0q42O#@>SKLG8w<-)0W-rs+YW!aZx
z{>-vpAn^5VYut+Z_xwR0-iyNX68eK9+Wz-7hx>;C`w9HqIDO3|EtSG57`=Ul{O3V^
z_v4mYT!H6bz0MF}XA)(ndk=p-gLwe@L&j9yM&JW<au5Z8xxbBkUS@d78TKe;fSP15
z7X#J-sl;P#-tCH4W%Z9%XzQX<kSxKk^q2Bf&|I;Go`jIx?3d-NxLGNM>W`W(7cU(M
zA=VvB(($#NR&(KzzBpO$2!rd64wwM%o|v#LEm%UJ>Zuu@>gmj1P@)ES1$}q4dVA5&
zPUWT6G-L{B!&!PgTbVEw;v9%lwIM9z4nJa9BPDpj=k2_V%ZldW5JC-kzJudn_5F!W
zlp3ipl_e<Jf4H339tJN3<bS&~FH{#t*W1U>@m)rS@dYT;#r9nOn_DRm0jy&6yGGiH
znO<=<uNSwxYDEz({Ldq3b$07|uG5!T9u9ias`^Pq+qygu#tan3EdIj+C&<7~J;UD&
zC}N^#qr-9}RS?HH2E2mH!Q*lnAdNvIS<>W<M!q3>BfB5l{j$wXmC&=GNTm^JO^3=1
zYnTdt5yTm{Nkam4cx+<W7ja$xO774#l<s~gJDrLm_Jol`W;n)YJZ63=fLv?4Pn~rI
zNIC~9--T}4a6cF74&v?SLfmnvc+2Pl31Txmx9WL~t_Z>Bv%z(7kD|n;zoscdaYsxc
zThSHTuodZ!YdL-Gah?>Uqct$$vR2aT9CRN=t(Eye2$B$XpP$8hjpk%fNddTSPs$nk
z|M&3%#CN@0FM=<yBE3vs6M8FNJTM};6*>UGQ!Y(rHTV5TYTekzxK-@yldjQ-owm0!
z-1GxfskPPm<C8oU9)F@QAgGkC*z@mj05*;i-Wt^hblPSyEUbfcWaal6i4pdTKbq36
zLJFjl*RHA;Fn?)F{UNd+?0_5S;T$gz++Z{;ss<O37j7WlE=puX6S+L>SAl?NH_y1h
zPzTEOHC4UIEK_kLGwJ8@Ncsx1Y*8=N+nEe`kq1aJLv%%EI47^@W?*}kdscBjBRTaW
znv6xO;ouUvHa0|(!1XBR8___Pq$gKLZP!OVl44C$F?g`b{G71CvV`uCw0b0SirH~y
zehCakA%iYoQZ*QhHDtS<^r{|GSiGdkN14cYSJ@X+;0c^o2)@<t0cOS(n=1&5TM$t5
zR=@DY8Ji@Xs<pENUrY`tKEJ4>;Co-qIaz7_`}3V~pn%m(EMHT5?8esG7U4`I>Rhl;
zmG$z_9;+`{6gzF=f4>ZPED5F;wF_2X_s883^LBt9$30tE`pe}TwXv3WKPq}R-?;TI
z`*j#(cLpuwew0HAF2IttZ(ik?#-yc-d5P{RFWM<Nu+yTcP)II48St`G+?Gs3W$K@!
zUj5e>RPHzO-M*I?ci`+*44e;x590;yyiWjfGvE`;90p8OT5v@;L!It`5(tNFX7ftq
zej;yO3p^YtbFlJs_~0B|bXl3-${4g$zzuF(?$Fk|dp@|N62w3cqf1gLh_mh?@j&;4
zH-7%I>=(?Q(REjGs*syx2bosK-Y#q+HiakII=q}H;2K<ZpC$E6Dab4pgjQ!ks&g>7
zU@FqT1)I;5h1@kfO5$@_{dJ-^8YwL4GC7=Ihw9OCmtSE@I~Qa$$(W#0J}pty6?WN~
z8atWy_BOqf^v+@+4a(?!S>XlbHns~XO*V)927K5<or*l5r+IFjWp9Ep3HZ|f%t*2}
zrm>U;09l}<I_a6izmEJr!2Mr{fBk0xh1K1kRPRg2yR{5W`|5oa>26ARblZ1wVzpDG
zs!(Yj=|BGBOOtqz+FAdTHE?=d0q{3WuPzJ|plW4P^zU8EUtG+mg}R|q<W73>z2B}J
z+&FX`LZ(ZnIZom)_dc5)Cp{ATnF;(B4O+l6YS7kYIXr%f)fb7GfWPo3Q*hvXfz@7h
z?N1NWz{SFyjDTL2$Ohqi4!S47*^-_?GK$2DK^~;jg}l0LBk}5<=Gm7oV#9Sji6G@;
zDP}U_D;-q7sou(^B9V{M7Biyh>x6Nj$J|X28Euy3@$Yd(C_X&Z0#1LN1PS}a9)Hm*
zpU=dMQ;P4P^7Nbe_uh<dP`?=^JEEI>NpUW~4v94sO}!uHtLA#ve&7-{Xf3|kLB$Mi
zerKkl2$~j7Z!dcHs3JN-esZ{DNSsbW2yk(+{!%SAPi5&{Vrs00SQI5#72<hov<<z+
zB+<YmhQ5-gaPg|8a4~kXB&2Ke7yOg;><Is4?Uc`Tl(W^&SW2^tyOaO)cK-&z?-I!d
zKfdJ{yyr@JG->6uRdqn+hU>O_Z~^`O8SQo<_z)N1WjR%e^Li&U5S6WwT)V}i_2~%Y
znf1~k!Jl#NJ6KS$Qu7W2d#|Sxd+iYj+%mD+xq*w;9y`R?RCO&7)j;xii~f$|4N8w4
zyV#%<fcwAB2$FlE-Q?=mAm;EX_CTjd(~|bPp+(!xH>Z5$1C`YG=7!G!G7R5j{R2wc
zG}V5b1~_EnJ4yTte-s$&1J9Td46oILbEJa;&&a1$1Nfx4ybAJW@Jw3h2XP5%+B$$r
zD^e5Hd<|Gzt`ct;wS>z7Z4ODHfd6JUKzdmvL`3fSlGB=~YwOMgX2>5`zEwP5e0Rnf
zv1qE!!)%|Kg-M~{Yhh$Vk?t<8<9zVv4zc^>Q@NwZg3b65hn$U&v33KF*eD@sdfjly
zi-6R?=g98(@9Pf~X39J(bCvYX3*x%j#KM>v0v!2OE)(;etB1I~3-W}EsWu@C&U@YE
zeiRDNPxj{LML+Hm;so_33HIC$g-#ZADb&#91<Imrfw^fv1wZs!_jP*gHCO_jr_`(Z
z<^OtxsV{be{@W|u`1=)dX#_|eE=BZgcQKy=Y#Wy`#Rm+Ok<vc1P&DRfZrGy4@)kYQ
z(``CD?@G8iY!ch9889r;UON3fPK2fJZfDi>W8>iAWHb73yBS-kD7m#JVI|2DT&o#B
zAJKR;2!-zUGTQL#UfrGjiYEY!@vGG(`Tk4<vx>S8LBY84zy{{%)>H(^4yWZ%ZoI1m
zLjo$1s{SpV(q9hlDC@)nq8?`LUrddK|1j_OGQu!YaMLbYaE*Sl9}uiZ>{vN5Az2qC
z3Q7_xb7P2HRwBdox(BdP042|BT;wOk{>_*w4@ByPD#H}E6Z0=SBg&2@FG<b=zxlp%
zFlcLMjMUGIMRU`}IvB=ofl9f8RGv6IxS!F=flKdN(Hfl%Mo<@=;f+GP+oU6!997$w
zsTWyL6EoJfgczmKR2R>}Kk;TJ2AT|JF#c9qvM6IhJta<6JdfeipwOKP#im7bH??Oc
zn0QtbB4)MYD1ls8n8&n5GPF77*;Amtl~(*yIi+K^9ncP@=IUfxg*h5k(ZNtvVcHQl
zJmxC%oF`REefo`u=l?}2|7KCl8GBiNo@H9au|FWQd8sbj8Yt6rRQdYPt~g_{&EB}-
zr6zjwy~bmCw><8!IqeJ`Yh{<jAOOk|J}m8o-|9p@!#<j+Vr9nIIG5sy3;F5c2w4WV
znz{ayb*1pCYg&f5(3{AC0AvH>Z0xC?>t<s}YSq;qMbjyJqpfy>tJ@s^%x}PpqHeWw
z^nzySIB?<M5+b~GkOh#olo{n_STm;EOHWLP`+sl(Rha07)~(d@zdR$Y-N+~L4u|~K
z0+uYtH<?&m*wPJi<L08&$>URy<{{3@i@yx6&-jVM{M1Bb1XFR{7g&;o!470A-PQ7!
zWBo<F#BuS5j;1D0?1_@pMA-|2{A}J+e|==mK`a0ADp3nx>V3&8rXtWoJoY4Qy7+e(
z@?Dt@)-#*gFYn2?^x+$>Xe*)uAIf|~o~gxi3PC3`qQM6*zrKaZ(*ycFda|RymT7PX
z?1ownMG)T*-+%J!J>p9XMS&9m^;E`-M7BG&rg%z80u}1YB5g<*{(@5IPF#BgWBSKB
zO)Z_a=0b@RpzQ_QwBSzNR;ipWJP_T<8d0Fbk^2|TQ2ulB@V`n~n14YJr5!`!qS7fO
zZ0_d1u6b-PPu7Hi=Tx3o`i)Ip4S2Y&53p@@Z9fw#MQ$0(>w|DWBWv^c;AytP`s*;@
zCouzrC)q45^%Au)0-(an^mzg6(bipwqZhg^vAwLh(Tw1J3*H1Bn8}_4U@AY80nFt$
zY3V)&@aBf@3J_pD*~cLLgh5xWsg0ir3dhjHom5A$shLHqDxH_VPzeHyV?kB|h$HFU
z4~W-8c48QHQQYAsk0OMLk=&&w#=eZuOHMN9&Fq*#K3eC|3buevl#HyTe3+aAC9dw+
z(&Y<%3FbGM%{~tbzF<D$b@>7(xDO8|a+&nEF_95#B)ykTmm=Jj6Z-pZpGM$sI7#j{
zfNYlDN@!4soexDDN>+EZy?_NxIWTMnuFBKXR}2DGTqt$fQ5CU%6<suQKPN6IDGg5O
zs3J4pY#ps6B!|$wd#JR0bvj==Vd$;jSKlphIx~W7v>q*Z+jXMinIW<JeW@7e6k(m>
zvHPO~c9yvtH!yk&*K_iJasSuX1VHq^=wKY2cwu6rrV(1v7cQ6OM?i(FiXbVy$~k6|
z@nRknupRDociuLZ5Ie4R<(Zz2NsQi^-fN#K;F{OKUlDv?X1wKMFffkKpVyaW!OGzc
zHT4D3*<-c!<wn|-pfwiTIE*m7TC>bP-A)qYzHx(w5Gc|`W(%3;+m|3V0>nk2=_+U9
z8{B&;(VEE*-yIp)O4trn`*}Tc%kh%fuucAH|G|xSJnt2)ZcXF|c+!>4yrFJ{pvS|L
zh{nSLqKo=%ug1hfOLj0U#MhCR=6Ux_T58iLSR!l^0U3WfBc<OK$E#~YKy}y)i~^be
zP~$4sG0Yf0RqR$3C2Bfb++i{P<7x_Diy}{{oaA*xIa|+neO(r)E~4gjkDSBEnzD-M
z{rUFU=R)O)XDy%$$HjOd7mL~G6(mc6$9!MOsm1hU((FXdgLx%H5G%MQB7mrdEd=>0
z!^C>H*sr9v?PT1ehrYPYrUZJ#u{RaJ*J$xfCM7t><$`#;TBNqP&q6slY1!spzsLNJ
zYi*e8*|)w#{=P<-cS0E5fz}z^Y;vnfCnRpXaF)h8xW@g15r=`R`mjOer%3&e-ICVn
zk3y(=YbQsVw5w(u`#v;4d`mteN@*cBU_}~B!-mtkF!a$?%983=={0O?$K#qhPCd2j
zg%j#wo|Nn3Etww3LTk%3BYOnjO7yJNbY^r3<7%Z=#Bs?EjV=54#TJny)J`Yl@ICZ&
zvjzcebhF!<YlyC~8Eei{PBo5ZSE|xBy!vI)|M*7mg|E$c@i#%6oDTY7{{SI{_{pqS
zm#R&kziSoZiC1XNj(bx?I`WiyF4HYta+T63A=Ph5rX72Tift#tPsX&DAZPo0+tYU3
zjVB`*nU!;!ThoFb2Jr=XeHU}e9;oH-VrA!de(IU%y3Rd1_~CXYMJu|AuD_hFwPW1p
zanookygpntH@G>xyWAfBOa3$94>oUtc%`xWkUoV5pWMa=l4<QcN4Xa8?e{U`$tBnr
zQ_$tD%;7$g;$5%pGJ-n-G&cg_MzkT2!9yH4JuW&nL~RDN@wrgrv1g|yie0^VN2=^H
zGwH57swY6`K%{dZAx6cJ&PrjuBOnT!nM(9aip^Sg`hCsgh|gMMlJTiH3SD17jYnrz
zEJqy~S+X2H3HM!`v3%4h!cDv5$Z1yw!Wrwxf4`>AfaH$gzx6~Z2rz`ba9;^iXp84L
zFd4e@rfC(A%bzZAd+9PXQb;nGws8Cr9S}zQo>?a4)XU$I(-gWyw0)b8H9ihZJwC1q
zdp>u6yE$OaZU<;ziU6ibBZ2n&P_gZDnXEOEqLKPbRFCyf1ovP3Wa{<Q(tom3{~@>G
z{bjbpNWV2y&P(FzBqMgBL57}Bn;XxLe5Vw&B3P)!;1<VPw#dngRH**MdJ@H$8~h3C
z=t5=Mi8ZZfa=t<4<3$VHyF9KZh`WO9w3fkh3DaYth1$fWWbEVo_jLN7Jr#17d>o#}
z-X17B*E#03{D*PE2ixeXol0Cj!AR**T>Q28i_52^FNGotHPDl11KWlJmm^d>IvVmS
zOa7SH6qQPe;dk$8gWYSw2wPL$@ro^0+Dq2471E|P{C*t@1Meztdi^}Ana7=plpEog
z^z-F=vpMO_^y_k`wTZ)~-Lsu1f0|#6(bT9-=I9;Mz|7f9*vUdd)Xl8!z;~V+v7J@*
zof+SODuYz_whwYw0644?yE#p_KTOiCui+*&RIa<U(B&gFbi~hzEt~V2-L2o&MtoYh
ze9BHLsr=p9NZOTa<iWiv*jYbLn%Acvpoqggx5=rGn^E5-k7^G~PFJU=SAX7!x5@G6
zt0Wpt+@QWbU5=E9xTx82xzu@Dq$XEf6h2#`H(hcn+Da_3)-CmZfy&cb2jTpUzyG7*
zryS=Flvd^;CV|N^YrW8=L)o!YT{WRD4ui+SF@EXfL;)RDf3odIL$^3DkxR{QHn}rK
zz8_q#(H$HQqfz8iS|SInMOenH&x4@x`a)F-ydqrr7oiWFioI_LoKEC^bTH`p@zdp7
z<gNrThF8yDpZGiO0Nt+br|p_{di2VKd($NN4(J7KOnzK%t~V_#xE`@5PE1d9G+KCj
zAM$N>{}lhem+88l*TDg^3#YZ1XmEfU@H2QmDK@II>B`Q#18?oAqPk7`_&c<hUJW?w
z1tl9r;nl|-_HvxGe%Q(wKwv5}37DF=h0S_3I@WIIApG!%6_uHOhR0}jAS3Gy=}E?d
z(w`411QCPBWCZ&n4CxH{EAmTWo?S(k-4q(%=)LGdiF7@M?-9_$hTpay$ULK`1{%m~
zXou%iiF&_zPmv`|_=aLQC#VcQoG*u`Du2!#KR9rQDGT3+DY%FL-r(t+ImEceA$f8p
z&sd0$JmPACaEcspC4FC(&3~#v0Y<TQmmNgP{s@dvEld|W$Wd259${yfCSJ*nv6@-_
z9hWT&fUb$J_yRjqHC0OyV1xU}&PdkiSW~rEl-xkohJsOxUD`W}CG=W4iQ-A`h8cI^
zZCRzcsng;9$+TUW$#Qd_^Q|)JXSzZ5U+9iKsPI-92jH*GC(VgMmrs9wbTs1T?+!>D
z`0yl@F*4_E##XBJf|B%~&y0mnAD^X=s4YdeZL&9Ao+eg2`nEUo(NfH~mV*2E#(5?m
z-iw-edM#Me$v*h_UHuas^W1uIZNeYo#jl^S?^x6%fa%U;ZRc{_3VSr+8qNn-)bY&o
z8X;1LotnlqX_2|Z8h%n$ZV`Pu6X2e&Pf*MN+}wtVNv0g4mcN~*Rm#ruGQV#H(F961
zv^<ho>8fH)t6;Vx<|_O7`sk|#(|}agH^owkh5VDNRy&16LA%NB>rWj7<L<}dxkr;k
zZ#X_0xaWDDj4Mp)<LkTxC4Dn(lGAG9GOgB&3^`L90VQE-&!-iOs?XieVZ5Fp_1G_4
z`<58T56=@l{3eG*#Enkz-s5iHd=}evv&(2UFcYVPnfr8l^QhK#qI<XLM$LBkA<Q{a
zY`BZVP<?lkhmy=_GYUu__OktX&^|wvWNQ&v`oof(VHe~$GNwIm3JlLMaAOa;?uKdW
z&WF>o9eX!@tde++O1!(ny}3PYEc4Sb!j?RGGQ?s!C}LF>)&=aTK|L~D9Mu`D=AO=4
zu6E1l&%Mt>LZzHEq+&?)%6k?t%L$q4F&LD$5^pM1Wd|L8Y5q!sP1$4Z$I`Qj;tOrc
zlnHmsqzmJ==7tj;EPW4_fqkN(Nk^F~q<%V*8(dOIKd%5HGO^&~8+hs>xeQ{_yMWI*
z*L66OAXABMisqT<m`sg(7HI6S{}VpQ#No*k;-I4#MXX5W40%~O@wQ(E99RugSiZtX
z+{DnAXrLB?aKjXQ{C2pE;z6ODuIVckgD@HXm~^!YX~APn*mh9*jx1PO4oscjE+?nx
zDZm@%>!Lk@uPZz|{k7J9bFSehz4PD;<H83I0GZP38og%2ZXQC#0|AnWfrM`l9=5ht
z9KfP9K&$$Mg6_`WT%R^|;m#rbNy2SQe^?aK2-_R=zx(!rNb9dFAc$8O0Hpo?O%(sb
z9t%kS#WMV!P|p0A)SA{EnDdGf9%)RsDQh@${>@J-?_<I?SUl;f^z%HS7h9ox)wp{w
z!hed|cqMMb35KW3UqaEzkvvT$QTqmg+s!^$uGZ6pGxPO>VCyCuDPLpaWtfZA$k55O
zV^g_=>e`XO+|i}d$?o%(5m%^&2^P`e&?Y5CzDo5#OE<r|g<_5J&UiJC2h@B=+t!XB
z?hg$v1*6ZM*-{A^_`X2HB;O$3OmRYVs&BL|^7)&R&SLd1-r>V~Ewew+`cYcF&r{!O
z()Y%;Tpk-R5Da`-QGQH5&2C;crKSs_aU_TrAy`Gq&chNQBV!2j<np$$*In`wlw?->
zYvN#$%b@a_i2|RM_pWEeKqEP<=DW}JIBII3)XTx#42+Q<(qHxuF*eyUHeJ_;U7$?2
zW^8@SZMs|$UHu*bhoha2K-}zd#1K-H9yjBg`ULgNV=2f3a-E_x{&9LXELP8Xwm1FR
z2(RQP+u@j~1pBOTm1e}-^!n@1f&`#ciExpTsh6Ns+GZP}d6<xN9^&=d2gJ3OSS}Qg
z^Cu%ZB&Wf-%f!R;4<!Ird+jbK(hCyfCe7%y2HenaT{Ur6@M5)d4P6S(^=fB%uDZ3!
z%gjUTnd(F+g?-4%pDJFZe$2Z38?TZI6UE2Rn$n}9Q>tfSvhQ!^V|vpmEeyQQ!p+1A
z1?!4UXD=WY)7!J)M0F^IRcXV?QF)ffs#n-eGDq5d3z-076)o$H*`Gzwx{W%-TAHqI
zPh4M1gjNs7V6}Bs(oWM2@rKJ-?Vc!N*7d=n98hdVFIrtQ>pIQiU#8Ju+TrTDp7}Gk
z&bUPq5yIEV11)-L|HrOGC4JXs$}&ByB0tT&g<bhlGO>rhW7`@+?FU@?R$}2l^>=II
z>n^@tl$b~NZq5FX@wPL(+1I2Io0qR@@7rm-CdvCC3-<hke9wCtEcxy2ZJ`0kUz;BO
zi0PLt4K8{7aFHGYqEi|6kgW~@oz`NmOCG(01!TiKpk3{&^uBGT@XJ=J6<aDGLzY7s
zzP%n@CVy~uJ(|(wB`nyF#1ngJbVHl?Xx*wOLX^CAr6}q$7~bUA<`DQI85X8}eZ;)J
zVM+JZvV0=bVYppIwjfVv^gwnw-}SX_SQDra|1BL(|4TBZ&*VXvUzI5`4Wr)@O2Hq0
zF8Eg`1#?`?S6Tdn?f>bcpl(KWZsM^*o#=Oey}U)%CH6%uxzQIAFFG63Z1JS{RTc@i
z^_DUnVRwH`&0T0G_h9*o8qmABS>qXMABd*{smY^?aZKCuO-&^=<L`wmdK1ia*-IY#
zz6Ij%JLN6J3fuWA=^svKoK()+<8km~0hMWBT>s;&h6@m{qCIh8{+IkN^>bm_sBBlT
zrH{wV_-q9k`r{c*|FS3hzd7(7=JGlBEw*&{``Qnw5N#2l<=N7os^OU*xpsq8F$ICC
z$wE+?51&sefD9To?)%{k5_N&Hy5NAkf%2Irg+&8>#}wb{Xaz}X+)sDu<A(QW7>s%m
zKqKqtO8qt|#q}Z|X1*_9+*|8wBOAP_0777U@2y4)B6mJ^R&1Q$?=_+;!x*>r<fQB0
z;S0pm!M;^okw5hw^8C4&*#qtF<)U$Yf3jI!N`DmX+g;4$bVe+ToJ-zIMZPHf{%QYJ
z?&S|&UP9gI;*knjla^ET{49D};KQvs3R|2%o$@+a?6}c<#`AHrD%y(_@#yoL&vd{q
zg5H#zs%+MK>wUWMgWEq7vECy2lEEEb|I~{VHoN3qF~AXXjhO=Bx^dqBfq<<YQ;d0E
ze+;PTd#jlvgfFnnDCzk9>Y}OkVQa=G@h!hG^Em$>5lJYe!f@>j%y{tZA;*Pre|2?}
z%oefhj>Annf|>FKZy}M$cCGfL%1PhM&GL&m--*;@CtVgiE)IR5JnN^CPv2$fGF|0)
zuX#khZCIS$F>1`CY`SSc;@ha|^GRzyvFj-Kx7|o@0~0$*H^ZsCC?#QIkDQNbUYs@X
zdZ!f%#+;Ywr-q)*IEIHmtHD2qb5H$Gl9L%!MhRNF^{E-kbr5p>LWe(&78Z=B^Ln%W
z%4F+}YcCJJJVi8}ugM^k=X--!NVQAF3%}I+Iw%bUBut23xQUi3kjx@_g|v{%QO~a3
z%Z{gxIOHbQRpx4F2Nq;^%<uGm$6<%5U*8z&lwAst_)WBqY1>5$&pP%{oczM(TOlkz
z>`$Ho3S>2Rsgx&`0e_cyg&xRf+bkrts=QiqU1~3y@+C1~V6Zu)HB)G}U(ckTZuYx8
zYkbLs^#U(y=F?L4$$m>&L1GLkoZ-`-;V-0L7FXhBk(AmqI^FCa#64RntPex;agd`w
z)JI%h9{IHbiA*k}AJbzWV3L{r@~M;G&_p$sVFQtU@`B(+mD_{}qKr@?T*ks5_}FFe
zwac$B0b|k8hUi8;z1Ma3$&oGF{qwe<4CTk+?^MA2Ir-tOCr4!3=oJ;N3st3y3e)=F
z-?-eHHbvn?6-y$m2kRr>I|tsil}SYy@p<>fMj`QB6G>dzDmo)Lo7#iJ|9>U<p}Z_;
zeB}N6o`L!Aju!IY#cUFcskvav2NgVI;2$l!zwQ#5k!@IEPOA~b;Qi7y$r2I})*e#g
z8Dn_mK!~bhuskit>hmIUyOTu!ab^fHWy5VRw+xWrHy?0{lzy}ji-E>3#)BSF2l<a#
z3x8FSM7J8aZ{+4M+)TQyr9wk^<nM1qfUiHjGeAuvbH+FSvKH@sh0A`4>;0LbzVKy=
zQ))ccu4QwgJ-reZ+uL<IOv<Mf!&+f32XZfHN#@<wXNq;I?4ym-0Lhd~=PD%NLk-t<
zPw&ki(XC=rdQs;$9~w~@_LpY|x2ox6=mScMHYNf{-A3MciEiyt2tirN?Vx!Q$up))
z>4KxMsO=y3Hc1klvcrTb0c14&zQ!hY8R-16w++m)s89i435e+rv#;UMLcUh<>2^^+
zo$H|DHLK|ey|+2=PNb0KPlsDzuB!KHKV2aU_;TMDn0r?k!<9~U#NDYzFG5p~aFR6P
zYHd>8UUE>y4PlUQe|Y0jRDyVqdX;k1l=#e&o#f{{5MY7CKZRb~>HH{@qi`r!NUp!R
zzBp;8wTn?hfd^cs8@(=;z0)Ly&=B<TgdvA4*Ag9(J?HWbuWaXMjjtB(yhsVzE=hov
zTvb5j7dD>#T!aK-8NMID2umo&(ZSA?cqvF~#MUQqf@Br%d?(?h^yP8KM~*IFgHn%*
z`Yx6LcM2%!RIW0H2ld72cz?x6ZT2o*^z-AP4YQ?kSg`HGBi7T0dj~b~6|h&luYsIB
zO8t8bmGU@}&j;riHPLHNCe_=gDPIu6nWA_|otZY@EK(Zg6m>mKmO4C{u$$(Q!PVqy
zYf@<voISbhkKZHYZm*9T*;`MIKu8Fot%q^RR#vG$bXx~h0=4sK@<bD?K85spv(rX*
zl|;0G!)PqR^`iH*M9iRW*5GFG_^FU!=2;VUBVjt`_gP4L_ruyWZ+QdbrLp&Cb8Z`%
z;0m{@ZrIKBh`0ay0A7hG<Oq9Mm2LQ*_9GQVVdJ)V?r>zL1~@Ex;3641WOwt@kc<rE
zyGd?Ee6^Dfo%SM8ag?h6vHZ0JvO%7P85>7cGKI4^D8}VlT-dTH8kLkr;!vcJA+om0
z0oif6V3#Q(d>+<>=w3_v$vfNljRi}o5p5eK<#csyIo(css_6eq(5stkVs%D-v-Br#
zHLt$tL)T*Q!!5%b*0u7FljhHUE=NU2HM$*AeTj8)oDb+T;9o@sM3kcyR@XvxG*=pc
zpyR4B8_mOcN+=M;(u|qyC+lSTwRPt&aB|PhpBykYPtPb@=5JFp=%1j4=<3PrjW_<t
zbUo@J5hhu;mPvq9_z*`LK;^$<Bjrc6=#DA@s%|0|lTH$WVr6$nEwR3cAYyY5?+xfz
z4FWZ^(_Sd#lT$B}$?y+xk+_FZrb09x5krH{tvXzewTnV2G`N2`=5OO?;xH{bJz)-q
z{PBG>rXB&;R>XV-&Js@4S}lWEzm+XlhfSrBrm?TLwmDhqRRgIV1)*z^8(bUeKnkEz
zDml1~Xe>wRdfUM`gdVmWK<CTp4T&h(*7bKk-Oo5^q-?+<@(t7W%GB3iz+CF#Yf7ox
zs~%BCti<W&?EvIu)^T>GDD))-g@a?HRplYj2F1@ax3GL{;tsVQY212>6;KHW%O%;;
zekgnlmm1-Jtkm%KG>kl<F<~E&KP9bSNw5AW^7&=pv12Z-$fq*rdDyqVm3jGrA)cCB
zm&9!~!8E)=2>P_iq<eRbpbKcKo%GeM^1|wHa6gbW{Ehc)GW41E^`#kQWr8R9^u5jb
zTDy6(wrW%--B%w5kL?O!bF~10inTP4xS{xbqo2|Y8VYxhZzcy|77gylYTxC1fOHc-
zwh;8=^hoI|z)&umy#lor6@2;S)WcI7^AtT+2GcCmf6F={!g*^y1hq=Nv_?W|qcqax
zpWt2uf4QN*4hZBJiuclw=ZE9aua#L2q&?+RuRLZWLH0yT-`I7B=+aAkd0^jrOw+v9
z160L(kKs>ecp)Fv02Z)|uByvTGg0_?AQDIOONnud|70K;o5H>}TYI;SI_F7q*v%3U
z=a>(S=lQCv<E7qwgInZeK`_te=3Vk_&uQ|&wytk0+@O}J(}IH4R@6`?Vhxlu1#bDd
z`=hDGP~y;Nclxd2WZ*)J4cJ<uTQw9!gU3Yr;;YV~<)p~ROLn`Lc4CJoqifB-sgXI3
zjRZGKIa?(zA>%<%XcYIGuSyx{K6=Y(K$IMDh||_-7^69fU%H8MC2?O^E!QNg<!d35
z-&N&}*KCINeW0(K9H>2a%R+w`|LFSRU1Y8}*K4dk9vQoe+9p_0J#+j}2PmnCv0>`9
z@_AiopAJ||400N)Be)XuF}=KFp6epDJXAgHE<!@|v@~9g?+y1?ZB*nE?|DA7uBS1m
zXS8A-X{bEOk$=H^rBZpN0tRpN)UYTj3cKw!MND2zTxTk#NQ6s{oQQ0syu7*k!tc5h
zwBpTJ$_w|`wB3JYp_poQ({S3J)6eMjT<ymmS5qxj$<tHj-Jj>RLW{W^>eGNdf#2Gb
zg<&}2Al!q^^x~ky-1Wk*shn@lO~iWQIQ`R|q{*taNgYdlM%Z@^*Vt0e4o!7XlDf()
z*i&VaRmr<g@RVIwF&D_XiiR6_9k-N!UZS=xD<z9<LiZV4&(C?>@D`sWx$!8*W)7FS
ziT&L9^;2yulY}8b@{3BWo7M;L{s-IXdg!9dWDoVEth)Rzz9v=h=_D1;8NKW4v&gAv
zl{menVPNvGFxCUm%deTa9yY^mxj^#bX)ut$4J5XF!(T|CCgyiL?qwv#w0l@BLONxC
zXI*9ysJuVMY0Y=oq?;kl14{1$J_WvcHhPd9!5ZJ-*p*KHJEC!0sA56LN)HH?y4Jl7
z3u~8_OM!Se4?PcUeT~TE-f1Bb3&RC(3zVwz=q<wUkt_ZGO7GLz7cj)pZ)JgELMAuo
zVQIlTiK6ZhLEbj1iaQNJPBKe%kX*W>T>j+|nJ*wKf?;&zlR#?sU=OjkB-)t(l=Qnz
zDr4vm0T0@5e?$eG>OW}~tpowsjmk7ob1T<L^o^xZ<8{^~FmZ*a7#aP^EZXXmPpjh^
z`Fyk7n_@k-=}!RQ^j6ji|KooNV4nC(RMY*p1tg#(Sn{qh$DV^D*8JD!43!;orD)n$
zZJ$1DZY7+|ew<g+ovc9JX;NPMeOFiV4bW;0N(;2&P7+iTBJO!~_{Weta^LTZ=JWg5
zBp;9M<M=p8>6S_^?_-#XU+%Z4FJid9A5-9-H0tQ45m(F!1rP~>b_QBIJN)%4{n;xp
z$<cI;ZTzNAS3n7#DOb#ww*}srpakhFZC5O!3B1;_Gw;x?G~8xV<L|I{eFcvIMGqV2
z-QjAYVQvMg_1S=Ubn(NMKFLlIaicXSDsD6r#Gxx#kQ}dD+<tArSv(CWVST)|>E(Al
z&uOO=^VK3;bhF^7=5s=?Uu^#Ca5|k;??!e<p)dCX7~HKYi5@w*t!D;AZL!U}9n=w>
zKob4x_nQfT2$bOptwnBC8U8kU-v3wYO%^DNdEzSG5SDoO?D1O2p7^B{^Tq3PuhUuW
zV5V;gZCaJ6b=PTVJYk6D5OC<nOlVCSXd+6}^}uT1v?;iTe<SotYIcqfWk!as_WU>?
zt4G$>)7%zqKp_jDyF%lMKFTQ%ST2F9nj{_c4e|@2J&2Q9ikr0a6!#GfT`C6GwWtu1
zpTe{48;Pgs6#?@!Q9Z@4oo{8OZ!niFzM^{{L(+7SWhYlIu@F;MMCo~3Uw&Jm+A+}3
zLn`jGfTm^;kcCMWr95@@aQfR`zg(xKthlMnTrt|IeoTd=lJzErS%gQYzO(;g^0j8F
zw9UtHA`;zNA|a)z^rkCECd-thlQv2OsR=hTh`fXG^ie|+pwYYShBr-zoW_!B-?Q9m
zAg*A&wucU<NSc4vciH~1`AA(WUdn0X>qkfpVyCCINbz@^21JSN+XkAnwhoj;gqWMe
z(ewK09~W~Qv+Oc$thAH-)FbJBEJAy8%#nL;TXgVqp8f$WG!iYIM^|OhN5q{FP6Dnw
ze|m|xByn+lk+@mts6zot0{WkJmeyA;p94vGwcB-Dy8|LP2sig@bkpi~k)qql<U|1F
zXW4ju-I}*Z(amyM^(*`c22&?0O=T@bdPj|2uydfGj^Lr!771U9X?C7>bW67;pbe-}
z-~ovgkN@BL2CO$r&HNm6i3E>Y41JzUC0ky5zWtL3W{C<p=h|qA6ZORWcf$v=cnk^Y
z|HIT-Mn(BX?S5#G7Nr?NDM=~mkdzc@hVB|b=^6&<Mp7CHLFtwpy1P52yBp5q|9#I{
z=OcWWwHTh~zW2WNwSO1cmYTOoKZ(#eInm{=8db*E5YBVs=qN65>ZZRErOM4e-UEf9
zl&7=cnsy((nB9vwQgVd1l@>(qQvoNFpE&CzuVbfHDi`srZJQucM}qQYRk>E?ZkQjH
zi(d}cr5el`6S|8`hKxf3T91i>QO&R*cKbpjyQHY!5}a3PP^1UcJN_O2m_{vjE5Ji-
zr-dy${jxfcNXj4ht{8o^$+|whYLgtjUZtCNr*vQOv)=z98O$&rk{edg7I_CuDnxy@
z&pL=yNFjoqZDX5uYAuqA;5Hi}e6T?${x@_yN^MI-8(=CB*^lhryQr6i+urrDlq!P%
zIz5y<A1u<*K=^&qdgY3&q)*%Q<?qf|o`UCD3YldoonsGeaPIv5vGi)qynW#)#ux#;
z2UI5HsRac;AeC*I5!zG}PTC^R^*1^YDkY`vN7<hE@|Iqt^gy^^f+Oe}0o(nRZ*pZu
z5xaZ)&Ss)VS?uB5`GeC^_cc8tfbHvyvP_!|xM-I)6pNF9qWO9a+hv257Nw)~L*Q*U
zDyfN9O#()tf@9R{e?O+|7YD{m(s#t#c~^bKL5=GR(YI5GS89IquhL1@YRr2!y}#1x
z-(LjlLUE{}_wV_X(ec#oT>~7fk8!ZB-i>xIs$7A9vtEu7nkf0(i#<yU`|mfM41$S1
zL(P289uR6J?-k_aBBZ5+9L=he_igh=gmFg^404n!$b=}*Uzd*Ssd%H89<V*_-rW>n
z?XN#c9K#jc-k6`*I!p7HpRNI`eR1axglmiGZc94nkbb~ko?V&ZoB40x!{EFBd#(Ir
zb6)!hmrq%7YdF<*SnS3~V?pyxD)8>7x%J#kc@;19dbn?lCmob%9ro>|oga)%!+eb_
zCUL<fX`W~BFAu}FvpU8uv4e4sSIr*7_TDg~x|%@O4%C#;amG1me~zM0Jt#GzEqM36
z5*ihek-HRrL07{_@6w;E{2238%o5n>igTI+zR4#T&jdO`ls{jewhv{vvBy>L`ucUp
ze%a~3hCq$dZ`Q{^f5X}uDz3k`wHfux;O58dG<K-wkrj!^vCN#-Xre6s!-2Qg)@<-6
zhz8Jg4IJgqB*-4*mT;N7VDU_gUIp@oX>PI-{h{onP+72ze%u?YddDy_(cz7$r;m)=
z9Q^c1K5udKsP6Qv5Lx8&c!I#^;hm34^j8vq6-Ib(uG+MYkY+^Bu@%vdPu`Ve8@H&+
z<Y!S_t%;v$$-x=&wr6`Z&aMTHqr)OZa)A48-xvVgIDB!@_{|b6?!WFIEM`+!5=+)z
znkmrEFqHApx-kp~EV>q2byvlxQe%C_b=ChVPYak#WOWg&oWYyby7SK8046unm}&Q{
zf9((%13x?ESvyj)zvVsafpTcpx=e#R$Pr~b2X`8z8X5)rgS^c|xND>K2JF)zTb4JJ
z8uul@W{uHn*2?a)q8*`$QE)U+G-P+<XKI_3&Nwpk4D;s#_?HcJ{kPVyI^vYqrJ|{!
zT`v_}g$@g=N^%p(HI1@JQwWGGKhB~|<(93FNSXTjvHNyD+x&$uXwpg3{qX;DL#}PT
zPg@h?eh~#_yv@l`hV7Cg?t7zGWD#H9u+w9d_YC`eT$2oqKJ9)yC!+QZ;S<rCAXcVz
zJLTm4o3Cp+21p^F@WCEgSj;kQ9J-lahXTCgv6I+;PO{B=4FTPy1(~D2ehX>$XR3Y5
z`OfN_cMl%$d^Y?)9%&Dbd`jxr|KrJJgF^;mM7%B)HM3Vq4sv9pmF0L?KkFz0tH&mS
z6c@g%CpCC=HFd^l5kxA&Z_~H`1%Ld5dC##|orW9U)A?MdH%Swg*Up$=o3e;vNhyTH
zU_Rx0X09mUprRI+(cXOrb=W=)zFSU%k-eD^W9SiIGA@*8mg;;y>;R<gxgIrL{ak#D
zW%|!W^_$k(^5vU<XT<m2B1)i6QOv4a4WjtXxjx1~*CJkmbmyH%@btdiQiD`e`TZYu
z+F$Bdm7v1?avSPT2~WeMZaoZkpX1r?4Nso@#7q2M&@!Jt$$YlS4nQr2hCJ&mXXsMP
zG?_pRl|0+v7B2jHtV2<utN0>5GKn7ZMVLs3;*#%cJHOPuhUnXVmf1FZu5`Jd-qUMM
zS>FL;9Iuz_f2$R|;Vh%qh`FN}(>BRQz*46oN@@QJ!Q>U4y~#aY8Uhp;p;|H?7nNH3
zfYLF5J%L+)UM>c{?(WF{f~?egDcUZJSDR}1hq+Ymh7@>%OBrD1)1t0Q2@A8aJ1bE9
z4|cPsJ7Kg*TD=adYk_>0t!ZDNrfL7zD9bX<clc*Nq`zp7eDV1t?(mY)D;Wf}8{@?w
z3iHmiUs^z!5AnR!Myz{vmb3!#ELm+@Mt_l>emQ~F%%{-egc!i6RNIfwaf2jV=Rds5
zB%S3UNMQB&Sv67mCsr>yq*odPYK^^_xWG|p-Sz=E0^WYu$ZLH!?<J`K`<q?9^(J45
z+*TpUv0v;JuLyN^mubWQI&u>&3w1}m&JLiZL__kLI5rCb-6-Z8za*w-SS0OP=2hEM
zQ1E8nE<`VP0s)VthlOW<va1}UH{;bxsvQvNdi6PJnzX=rIyb(sqt5kbjl$QrjUL!-
zR|g7qkN1HaX54$@`3<;~*Yj0RB#1xk_XL4kDKGoiIu+VPew}K?hu7-WU+0}2j351e
zdR(rzw!Ir}rN4@aa^H?Ai!|uv)jg4G`nRERDlLvmc3)5!Zvb>J&#l5v{}X7jXd-cg
zF?saVh56ysEfkFAoA3K4Rw*1=p9S-y{74DmCp_&|J}DqlXiFt~#N_)d2HjxPOv`+0
z%;I~2rm=>yg+z@9a2cdVXt#N%q{{VVm2ZstuX!xaCwMI)ISQVuBKS#mr?0oCzpvHb
zCI$DF()>eYf3y*ghZjp=yE6povpeQ$MHnS^WW82)#c5fS%97>5+wJ&>Cw~iwDBc?R
z(bpl$p!(|(?={i|ELB)1K>Zk1!8={={^7wSCb-XQ>0Q9`{{S^;$o-2fCfNBg4VnSL
zxx%@j)HHxt`+2I_$6{7#>ASWLC^j5*t^e`vDAP^}ULt%Nv0tf>$hmBtpZ>|^@>1Da
zmQBNXoGZhLE$_6<gJuLc5dx&ff#H`~e>gTr<49Pjlo}PXQsHK?o%Kv%+D6LR%S4)u
z&9MOw?l@A7lhR_gG#+5Q`!kqCp~+cOINP<bTH(^o@{v7{&}cFU08X>{xVuj4yCvwZ
zqV8QI@j^BK`~gIr0%Fz^dD6O?U#q+s#nx5zs95V72A4}-4iJZ?i`dV{upUj{asXop
znd~?Pu&;~|y)2fAr`$MY`5GHK&60!S3hFN=-5mE9o}ZYoay~x2$KnLe5vizzNO7Vk
z!<Y5A9~#l-eV^`@pJ^Gc`Yk0}W+dMaOY+z4{9M4dSu5_nz7I^d2fw)WZGVzblJkDx
z=BuIYGhohVIGvEj!CXwkFOJeC5dgq&6P=`IgRK{Xzm>eTK*PVE<(7H-ynqO-Pqt2I
z8R%eM7S*HvN@(l>SQK%T8ymkBm-AJq?{-vtQ@)@0z9>c3N<IqPc926QFKGzGCSm9f
zYl%&}W#2_jNfARyZ*Z=Ky3M`9y&o|3hmN+P0UZk&YS=e%r~8ub`4WnQM4aPrEW;T?
z@Drj@Sxd36rF<8{FG5x0=*j=T#|cM!<6~*q*_?B_(!u@%7YmPiuo;ja3MdS9$0OlI
zTPY%OWhj{xR!7mErHWzyxl@)80;CXJ1XeXO4_mJ1H@+&H4Zy%y2tcYQyzYmqMXeS2
z2C}8y+NGSsPKH=q=h0BU=$16xxVqjN9xC6ynQ!~WplSDu-ET8Cc&S43U`np*rcbhV
zPOUZ`@$AD;D7zN%D?bJ>2MaQ^a@e@$VkuZVxO)0S!|#d&lER+tvs}uXrTioBF^%f|
z#RdiKVi;uJX&BnZ*TMGboo1Hb_fIdG1H{jwebZjT)W=vpv{NdJTs+0Tw0hHrjV>Ad
z$i~}hIZ+U5-B6!!=Gxp-W8jtA@sJUHSRpIM;gk6$^ADuBsX85T)*0G=b=F*IDC{9G
zdT74~Xc-=KEqlQR80)}FI<VMIG$Y8v0jL?H#=GlnNb}mm5386Tx5Sz1{Lo0?f@vu7
z{HSH(dH9n7h6M9_U0R#ys$C>P-rMcOs{cWeNV>5HUyIS-(4O|l*35)&WqrobcphGE
zT5H1}A6K)xeKJhk*-m_lM}PAxj#`4za8@mQS^~WL`EjHugP`D}@$m@_ucL{AO4Oec
zEL*M=O4kV_zgQT<Z=zh%itZ@UeWp><SD7;zM8V*1(EWVM&!J+_n_tEEVK1YJD3n?G
zLhQ^XQ{Cfd1ve!#H>FwU`buHmv?I<CRS19JLNy(MmHQcxeuY1NZ5-9bQAlq$dJLeb
z7?eb}yYM4V>4#S`o9=Ib);h_aDm$%OXZI&?KH3m%>~;b%ISu+QlN!BKoOaMJt!O0i
zbPgIEHZX%oO=~l;MLU&wXxyS*q^7O*+0Q}f$yjGsWFa5h-0RjZAF-jKhP#C{KQGr^
z^X@~>qKm_Yd72CwY+{bG;cg#$fXH7>%=+eu;SgP)53{*j2jDXoo&0VrGDm%~uKYP0
zK+Q0d&hfxVJ6XWNj!iqF_xU`bSFc-z!eLtMCi=DS)4|<5IQ4Ntm(Mo&XL5s4iT=sT
zMv#8ZK4S2(X0NM3r<-pziL-x8)OLyWpK18=h{V3!1A8msi-3LYI&>VIOd*4h=QR@K
z`=I>malZ>mOrXVe$uUXFr#swj#&W%{WF_W6E#ko4g+%jX-Nh6{0k_g&xw-4oZ8n=t
z-#s!YB;@#Z@KU-COjjXI1d*mGB5p$w3`pIy6Z1DK{o-ejc(|<05b^q?L=W4>zX^tB
z#ONVfqOr)IwLIf(oO%^_eb#aZaNJdAP=z?-j|q}Agex56Pr%*sLUM|r<Cj*N885B0
zi`OUAR33aUhfoZDuidVj%D!j_fj@i3x36zXPaIJ&+9A~uRZd}g-~HX@g9*<lscGnp
zS9sHpt|gYsFlUK!(+K3m`z@jVqc2N7ZdXk{4fspD0!L2EfEU<&bD_b_?dbkeu^#>U
zRDZ9=>YoqD^mad^7o2Re@2VJT{Wv@D4p_bW{Aa(SU-2kn*Cd>c*cDhViD>0ZDMdVb
z+DxC;_u@?bFzFth8X$w8fj(72nIHs6bL-$U^ZSioAktSd5tyK(T_r%i_wJiRrg@)C
zaMCXlKOo*$W&VIn`r_E@`gm1Ilp7+@0-eivaqK}jiM+;Cp~V*M_38+F0i!N6RyFpD
za1Af(IbG@?AUiB|RHaj7sr>ceSZ~4<qPzBXaKGKj@3i?8oA8_yU9esS#fV(si<wbF
zlYEDEZv^_a6iCIOzYzoU(~g-!k*&N|XA&e*W)3OtuuY#9jvtQKEIgp|0quGt+8M7T
z*M@J_Znwzc?kbE#W*LIW@F3PAIF69pVcL-IV|}`?$Cp3A9P&(3MeNxuFfWj-wO_25
z7JffKMT5JdTQZ&P@s3-QF$m}q;Xjp;crEx25m0Tr6Z_D!Lm{HJ9Q*}y&!*p)&QhR3
z;zBz@*JbBu9x1K@N?5B3?}BC$A9wFtEove}X%xqV2z1#O6}F9>?mK+5Iyz0w9AAi0
zSqw@E+=B3@HBK|n65vsKihDAAdpoCH4kEAVHEAfCidi(-JH_C@zG`-M;I7u^6F|Ei
z`M)fHdsWi1O?bx>AIOS$cTVJf^Aw|kKdtO;b2aA!QH87Rn}(;$s<K;)WrggcCSJjN
z^UPxG$(rkv$5pSp2f7QTNXYnuYxmFO-aTCYa!?A#PZ$^`Qg%M@)6p@=<67rkK!%Uj
zhqNT>21dQ}yWI-xUiGFefBjUy&FBW`a|Xn8{XQ$XTHTjQYCJ3ye15?_1|mYT&^|&j
z(J@P--j-K|4#m!W%)tS?Q`@NnipgATqSMUg8#cZ65G0<vgLj-b<W=Stj~zrD(JkNe
zD_ryX%8Y#@hvPTp=_a=|D(axi2W>E0su-yt#Dn7XxPQ0C^U|5xI~%XSbj@3d*gCiK
z#nl1ECskF^<=*J1+2^_)mZHs)0v@Qs$en});z0(xZ#uPl-NdPmucP)R-qMY<f_VH*
z)UWAUIOaTNTBS8Sw^fmUmrV^LsXUaY{EgWF$-;3^hmZ!4UyFJkf0Hnvf1ta`io|&h
z*&Y8Q-^Bg*FNVPGH9&iNK7uyy>U~bz?&l*Psp|SIIO(YN%hVR7DsgP%ZYbGPuP;n=
z*TLmiZ?*0Z@sDv@M$VtiAl?+DN_zA}qq(Q-xfq5L6@JWHh7ujYyE8E&cGh+?k}~U|
zll!@!f=48SB_B6ZpSJ~x?3^MQavjsm2R8|<|2P@RtW}~AqU=G*((%-^Uw!c~o$h`;
zX!4a<^K(nvb_}g#Meumz+Y|9GYaO9G{-TKQW`4E0$2c`Q@R2qSsNH5INsEKZn4|*6
zNR?&$ynQMM8QiMpYX>rUvXvcrZVkj<r*ghcA!r?gY8jRDQ?3S{c;%wt3HD!xMp+Uf
zP^>8c6jO{4|1J!zy>vAm`3W*(8GozwC>r5yX~w|A|J9{_V4l9(i+zs!e0PpzlF>()
zIl|pbiXp$R1D`P{HM8-<U{T|_Wm4Ys)!!!=CCsh!sO|GxJnUh^KB)vOrBX@~wW_EP
z<^pMu9jxwb)VR*RKXkGP74>|Y;_<mb=0{x|8Bo^P(!-T<1qg}<_Wbyp<N;;I`SAxy
zK%I%atLSTd0n=qvn>Vos+l_2BGjt$(Kkc%*gPQpY+Daz5OA=z&WX3B)w$q9~J;YS3
z-X{v~Y&0;yM9Hj1T0v(Z28545E**moGG{E%D`;dI#8s$U$VJpGRn^4FZij@=26%o-
z;t9O2R0zT${vnlWSMdR~f}-QoRZx6uGg`aOGWR}2fTN;^Zty;ujGJM>bFVnm0g1;h
zvol*De5<2@SG}-#kOqbS0`Mag?t--no6-1@Y}Ow8=uB)sKsd{-1}#yD*$#L|V$*fc
zZ9_#sgH*0Fw7Ah%Pc_213syOi6=)8`rWd#FPY+i_a)U$Zwi(kTW|g8Rb@^7`NCawJ
zSMS4sWzS@s7QOyGnM)fsD2~9ES?BM~h4oDB7$ppG7y37Ib0x9f3O>RGfK98oa$iUK
zbOA+M4kN<2*PcSlx`92FN?|q~w5u(i+hh-fU^jhVYV-&aY<VpW@t%EKoi|hi0ZE)M
zsScsTkt<wv=4?xwJ|&X51H(}Jna|x+C@p1lYZ)?{3<@2<DU<9guqHaN_emVgg25k)
z^9fYaN7!Q9aA4PDGF<IBB7tFk<YlIhY^Kig4J)uHQ&(zLq+;8%U=sf})xO?(?o%y2
z1h8`$(Fl=LO=?2SSD3(xyQ{B{$fTElfRL~X=3+B#ZbOM6M;d}xGwVJPw&TwsxK4vC
zT{Xi~h8`OU=S@?mbCo4W*?3vM9O8ur=LK+PR=bl&8NtnrjUvW+Q<r@anA??)mBXSG
z*AWYZAqRdl40031M*TaI%F)Z~9(E*^>QU~(X=g`HX6!6U`X+V9snqujw$x9h>uobv
zT^0wPT5~{DI;AAOFynxLP&Aw5pY>UxHUzkVVbb=-ZG;aa*d-CdnEc1beXsI}vTPE`
zxT$%MvFXmBo<r|)C2uh3^V%U}9`)GB2^2KKIUINB&Ncu>)Jo<UxO3f~^|bz4n)6*=
zJXqm0%6!Fj@?`}Zo5{m>ZN_oTfLk1!Y5vjEENh3yFbN>E2Iqf%Y2Mh2o>sW;eSKow
z?w7Y^Y}kGDtb4yv`TOZ?zQC`<)Qb#!@RR$vlX1pS-d(!@Vu2-s^wFvx#X7IU@X%8P
zHpS#qE3ke{9avr}4AHC<LE<SaU1GdHa0q5tHuSl0xj#HRt5`3if+CG*Q`zBCT)Ok$
z{4Fe+4Z8^EcvQN7+qd??Vk{Sby|QqfrLXp9WjC@&yI73~ODOEJ)6zJ4&+?u#kM~w{
zvx7|q&xb@`wEXdM#<2a>I=EtyZR$qx_tvN9=)#8QBC-z#o~X@(gey7`8NG9#yc3Xz
zKemm#O*lC13-&Dtk-~%U%}m$c(t}wO9gjzr!D}bzc$uBJY&awDAj0afw@vsNb{#f!
zW;6*Eej;}5B*VL5hPNQqQ)+1<TE?eylzPAX^3A<+mJ3_&s&bB}Dz@3rH2xVa;<0gT
zH^KP$vA&b^g|Ux(fE#vJTzx{^%wJ_Mr3$?iG@1{>=FE<hOce}JN(ar&NF;@$$d&E4
z$^vLgx`uBFXewXMd&OK=oU+8+g#W~pwfl*Mz%cdiUok$Fr`$~UO5??<4Eo247Vr0p
zRm`MXdIAhJv4@zhD9Y2bZqr{fppoP7r7!r^>y#-UuMQW-Ip8-0!`V?x_|B`w%=pzh
zWq)a3hks|aSC<5g27Sb(xo&`=U|_=VepL&fEgbx4qh~NQhk0BG6X)?>T_HJyTm8A5
zi%ZnD3UyoqGt$F@?m>KHFI(Th`8=f@pMhQ$V2W2-BL>?ZAx9jOE#NJD#OcTHmL(o-
zmv5cIDyEO&hwqXe1mkkvC#Ig^35cyXUG;jccE0669n%^X*;+6T4Xh6YoQ2@TFnr=(
zZ_`SIp7%AqPEFE)cwir6rfgQ<?4aKpWZ;hy6AfV>@$fSXgr*vhUw!H;s5C<*mGA%!
zW?u2XH6Q#DjbDb}d}@T;=N$faQTQ?CW6k#6jz=(>-Fvvnkihi<kMEr25=_Q@|DwBq
zFGopp={04U(hiBD*>8b^sqvIvo@f5j-o)8IfMclSLn7LI_j|TdH0ZDHwnJd*l5zgW
z?d%v5{@%4bxl@$*OTtT|{h4w<uVPGi6md34r6L5|82dQ)*1qwi%XxU}{W~3QZEbT}
zLX*`6d+Bx(k`{rPANVcL!izMCIX|-9x`w9;W138AXbo26m&d$=-Cx_a#t1xPvXLOQ
zb{u~?;q<=}<bn~{mQ2(A)>uY1GNU%UOgd)GO3Kl_V1vtu@*7?=su5rp#Ba!fNav$P
zudGC6K~ul0=#|-St2>q~V4pT$h~gi61YeO?KsX&Lo+GJOCP(sU-OUk}>RB67&xjtk
zj6D>7x_s84MC6KwS#aT^W*;$m3Z0_DmbWdYYT-sf$LTqQ=v^(SxwAPf>sb8=thh^T
z%CDY+(F?RJo3Rrj!B>`g>X7~fIDA7RY$-$&M`6;wl9+Y)RR!6semy@2D{&FWL^jLr
zu9n+h{rrTUi(HMGMeEvE)$TH+37a9{|A{GX%Kc}>KSGJJ@e;JL*HnbrBThY=gQ~sG
z{Y=!o>D1u&mD>KC-N%bCk*tp`KVs1Q8Dt`#ShFTA<O@gbh+MGe^8B4-7+71P)0{+2
zj|JiT)eaI~zbXTL7q0fICv)|Xz9O6Dp0sZH=pL!UZO=!lFvZ&~$$g0F#_^`#Wa1X&
zbXziS;u}TQ4YQ89sbY*wyjx=_uCUPlVaUwVp7dELVZH>>xNli^>uE1+OGPLy`2w0q
zRDsBFZ_mVTW$Sk{6t%0Ja$TK(s6e0JAC=as&A`$Rh@!8;#PqzpNv|INC|Z;VY(DG0
z-=4`js-nK%@^_rb%PRsWe@tho23et(h;Z&#Jr2!!E7$Mj9$U2Hn-OSxmSes!bnA9p
zk9(I0;71mQMweVi*#R#mbx;objkI@+?tMBVmeP~u#ZTB<UP{-;A4)aJ28bhc<ELT^
zVeFe4kd>?OwVUyJ`sKiTm-#xzz`Ifb4%@<t?O26CpGQY!{;dL$c1G&NX0P#g)52#A
zS$&k_%fUznpq8?#LpXI1HPy{lgv%f^;+e|gXXw<=F)*A6`&r%L5du4U*;RumQ#x|Y
zI|Iw9W6N^n&vb@LRyIdI?^8K?1>^uGr(Gml>ezciKN4(L)9oGA{CBCv18(4Z`@?am
z^fI9#Y~vE+QLvGFGt%O1(pVqGdGwZRLCWfP>LLo!yT9v2>7^6MSpu8iecD{6MF=X{
zqxw>n4sxQwX*FN6PcompiJPAuehyP%C(w<5H5Prm9>?^2f(ndw9L>3`I_~5lA~T2O
z`<ZfN5mdfFvb%J;cRt$Ph+F4Nw{B{^1lVzWQs*)f#(qqVNB#Y&?^Z)<)sw*S!iY;V
z{qv(~H|m!LPb<|755?B3oWsbgZAXTQQi$7SEy56PCdG$X-kGU56f0e7mZWn`0;CMw
z9kL=epefiAqB=g;AIah0d-bm?$nxm!K(RxbL7lp!<;0wDfn$pYy{jr+Lpo{Iez1AP
z2&RYbo2c-um}2<YeLnWX6mak9LwL(2VQ}?~t7P7q@)Ek1b2R@i(Rcr&3AV>g^rT%;
zg#@#HuQ7>efatm(4@acL<4%6Gz<bS-4(oPb0jR&tPdW<M_bH@#3~*!@xrc4}Of1_^
zv8vSG$dO<dvkeu#Nf`_Qd0VymZ|N7WUisW=$fCz8Dc2~6?8a*J{-KGio3IgNG@DTA
zr4{*Go(HRD-iQ+<AuuI`Z_0z4UCii2YPmpFu4$~KJJUDY$28ta`mWZ<9jg+1WybAo
z019MYw=$U$zuN<DOASBG`W{%5tmJ9614>Dzy4k%)8bg+9zcKmo*9~!}-|)bd^zFgC
z6cKb7?yTbf8q$~fc#+TPIaXjdlD-*6lO#E;av%&3!eO<fIv%#0&!Ye%MD?y_T$0&A
zY2&_MWD?)p&J;$4;v<#@Zr>SLuRE^5pEDti45^Zki1nnhEtQ<&|52toz2ZDtIAegs
z&k62?U$40DCJj=nH&Pv-z<c9BJc+(HC`p0F?ZNa@D@qu_&}nvanKDp{%UuZx9<{rB
zY%8NvC1TAKSUwwl`{U|)Qyk7bd%EKRRx(9uHHOik@fM$tJDr3V40yhil*FZCt>eD{
zkYlo#Xf}D!s5oB(O8C`1xz9?AAJ$(v1o6W!NxSwRY$;Z8g8>mQi33my{EDT{(NG0I
zth!R><qAf@!0byH8V!%Aajk$Oacz{9{bsW`4L5hCje*jDXfav+eYs|4*lgyqhcBr>
zv@{JpMFZ9Y*oH|4p4Rfo{dag~Bmw<?oUtqCzHsZ1d|0T%oy{Z%a*OuegpPW{*^m%+
zZQL&n1g3c0)$kUjDA5Z)xiAo?X67}@`##I87{z~AMzinJqwz+9mU6pMI2rM`cQZyP
zxP>1+ECV*C>HRZ2GEUU0G96*<i0aa}5rjgio$bwu@sq=Xorl02o|^{ACk+E!F?CyC
z4zu|+?}Y(suz^oB@*zoVJJFobcEJ~=n!K)*Aw8odjBtj`o%L_BV66MyrRTGJf0}8A
z7_Vf&1LDAe2ucA7KcC(ExxhhyG?L);(4$<v0kjWPo0*xVTqoEDjm=0Jb7$Qm@dtG@
zUI@jA>tTy5Li`p=_{+7)Laowsn6x1nxKlVshnqSrglV|(O4&lbpS8?(b!AX=Dc?GY
zzo5G_kRztZlP%TSC!;V82m^_d%R=P6pf5X{*!YU^vKVSgo+H859wjNI%`E6Bn9Y!Q
z8nw1PH(Xd^i0VC|J?Ql7)0&Zqh6;=JPJt>Fp5M~PV&S45Alxg;JF<WSu5ZkAn~8{E
zlHSSi{Qp`8Ehz=p=Gt+4LaTQwHWB0}HtxpP>lZG^xD`{XNIc;k2XRjo-G&u+xTXEw
zOChu5jEyNUc=`F7LNH9^E-)c!H>pNW^zVXyYT8Q>SN@TkOjf`jhlP^`s-1?N_h)P`
zY7Bh#%B+&4xzo9W(!m*2i|1$JQ#$$bY#{RHyC-O-S|#-wXYN*)Fo^-_-Fx?~oZ5D}
z&+H(}sr@F4#G8%3sgL9aktc6tZ*YX~|7_-c=8)kWlwI9>N)WwTAgQ&a6DrCp!FI*<
z1$&!x-SS%$H(exO>(h$9(}5;Wwt4zY_1_kFLqI%cR05A(mhbb2@`kmo4|5E#ImBEY
zzF%+l8;6?T1{gd{;q{kReyA_$rIbV`QxLnq$!xow9|8?FiWw`qb0M3}2>u!-rwD>k
zK$AnC+)iazXQ=ridD9PY`CGdK2swg3tjh#hXZp%;`^kgrcE;3epH3m<N~xzFq_8U`
z<Nd<nAP%}1i(fA_ar=v^WV9_dcuf^8@Z**<Up(?#nc8OSZ&vmXt6=Uy8x%H!4}kQA
z^*nLK9~^X?`<s=I-!BSJ3l)yA!=3lYKuAc9WOutx#%n!V?PDw@;9KbR;z$#|`0*GQ
z8sBVR;qyu4th&2HwQ(07k5k|E%CMxhvmYlXcWc2XLQurDk9pv=<a8(3v|V1ul$?jI
z>+Ipl?7gnijM>*&=kZA*husP56QMbOrCUa2Z;lc);cqAVnyQ@Cu<O8UvheB#6VJ`$
z>GjG;##EPBHVm4`5Jet*;bXmf_P6XXhvauKQT;cMG&XhDX+@|Zbx)6$?bMHs<sH}x
z4UuY)8iTg&OO}1oO9NB6ie*g-yewv6k)d<U_()LZ&AHD&;#%R~%5*$;;P(_`$xEaE
zL@ZNKSEoOZdu692fACoF;+U?O!xHgAcJON>VtyLu(I3Un1&+})r!R9*LK!KJ|7MEr
zWeU1}%TqC%A4~9^qukDZC2|v8G~ocHk)#mWgtNJ)MBDWS&8faV^Aq-tl&zRT>~3o)
zacNg)tG1x~$FNOyYC$C6r8IIfaRH@R%-7E%_Oo&%D+C3~S$1+vFD7J^A6*EuUg^ja
zR)Dl(NK|EI9y?_d%iI4$q2mch5;dH1q<%}7{mK61s#xKVR|oDDce;58)Mg;}EZi8G
zj;|=#O?D)7Yd@I7!{CPd?OGRG3b2dr(gvX2qzhtAtLh4xER@y9XQL8Vi?*sM#r;$Y
z1OfXhhX=EEp=2}bjugARpeQPn;6L{Yrswts37w}&w*4Juc=r~9C0;2xz<HK)6i@6s
zBe>Ip?YktE)HP*mM|8PhGTh<IWbu>Cb89wvpx$Q+H`w8+asoZz%g8Lo&tTPLaTb;b
ztvmaftA;y<9b4)Ltv@HB6p&wJ@4@0uONe3ID}=M}PeVL$r_D#pAhdOq58ElSCCm%p
zFpNY?DS=T{66yq~DcZ``g5;1wvg@|k0dUIXFL}#n|3aNXg`=+-FL1EmBGgMSmYX4X
zSWYUwExM7#kj&<422k|kCdx;;@VpztnPtazTr3)sT79Ou+d#Ir?&;!s4UNjE&$E1P
zHj?rUQ|51UK>jskwAi34udY^~<Uj^u&VxibR>8^{$wZtGG{qKyn8CwVj`r#h3zCKM
z4>EAwbEKQ|9W~FMHac<|qL4t4xEnzh2{Dq6FdF7FoiaS^Fd(eGsQr!tyPFm7&A9TX
z?~fda&X@bT7wcOgRUe~O?kmNuNf<XW&suIwC1ySq-?GX$HJ>O<;d@t(6m+WIn7oY-
zv?_U}U{>DKjrSltduDoPGg?Z2c}c#?jafspi6Kr(0HPsYLn8DgPU(kEnizUX!@rw#
z+*h_R+Wot}W^Jig(2YvHDSMwwY~};IGE?XEzq;=jrZKQ53ErX*XyKMeeiOxhZVP_%
z97CbWs}>)qJ~$V=XuNzYnt!3ts3sDD_!iD+u)h?zV&|K++@lO+*_IkWkjTygHB!^V
zox1l@rfc(Tka%vcatxaKP^|XvFpu0Ck2XT@zE)UPNVAH+{N_Ww>Voo}N5#y2%=W6N
za_9}Jdlm1cWY9p5i3#_@-|mq_J~GW~MbJq-|J^=WChoimM`<O{h@v+SOYjk;Ca_>+
zm2>SCRVt(NPk0p_I7HW@y)=DFu`RJiw+R3q8`Dzjzo5u(F27&J{5M^IS?1W*f{k+q
zYld)^xc7sOGoO)xxLO^o^Ky`__^IrOc>Xp>RuqT#rPIMFJU~ig(c)3S`kUm<aJ7ff
zaD+EQnv11NruiwK?sy5Bxc|dp_&FwG=7dB^oG0PF1~p6Ks^=hDktX+x>mw!i(+l(Z
za<Z>s&Y96G8IxYc-|Cxz>xTh5W{r_ie*#rX?FF^FG>Dw59KgyN8?VyKFPr!8z+Q?m
z>K)H#2E5Dg_>(N>1>F9BC_U$R2fag8u~@mTr}Kk&Ye<<!!!`zxM+O9@7gnz+GL&_E
z!_Ye{|K`-NBZWZ`KqaT8ZrT5P9B-D{&o=C~vk(I=Q`46KvU6R$n7PKu6(}&jTC9!A
z6hUYySI0bUBpG%dWy@Ol<YzusUJv}znzH|OdVl5+l=*ZYVPT?Q?~Iq9emQ>aUp{tq
zYJn^uV+-O*-`sz}K6}&0*=^?(z|ovSjBfa}&M;f?g#kM?-}fl<hgZ_r-if6fC3%+$
zSc^$3v=`s{d>yU##E0?eLCfc#26aHV0v<q`{a*>n38d6r_eGEtZt*c<0CLiUVg+g{
zTf+B4TQOQR#BXL6aM!lJ3t$P^??|GoDZ^6TiX`zGt2Q@d)cN~52X-qp6XbBGo&@Ha
z5I8qVz&MML=DPj{J|(E|gqI5RMA?N!v-8nF0Q$-~3*VF&7nA{K+JArLz=$og9=#ux
zhwR33?b&zS`0H&GNu>9g8fX+&li$=tG~*0~wC;^$VXyr&+$U&ne&DIB!-rKq{KXK7
z)L<+)SBoPPRDC^bxp2xMcCNbBl6c)ii_a{2?nkG?$F8S;sf0FCM%wp2^1%f>FYyhD
z7E$1jgW%lI+PwFi**)g`Zm4jzH_egPIm{t{I+DSGdlP(x>TMAN))j<XSV%RvQnT_P
zWv@;nIGWBs@!xLe*uG1tbR39|#_}<JW3@f>2XUQuWtb8dGS8~fBYrmBq%!nL!qD@H
zepJR<b(wr;vRdnUeMO9(7{dK<k|Jid2$y5$-*02}!of#%fSqnSYE?-#RjYG=28kSm
z!ODn(a|U~iB3_#PvZbg{x3y|t<(;zGuRF1s2_~Hw(@jt1TC*Jah{sijHUIjAxJR@r
zJ%U4@)r^5he7<{$8EgM^wYvwEC44cVr{kjNKH6m=9=ktnL7^DOn4nK0+ZC_a9paMj
zuM2rie0w=Sk&6Z$_H`48r-&jlJ0Zt$rFXoayPJ|b-$5J9gpPpjsfx{SLv`AOXiG7C
z#Z^%=x|rxZFd)Ksq1HnMu6<p%oqb)#`IVCt#k7@S))4hJhU4$-VW{-J?s;f%g%oDi
zXv0;6v|B?H`n)0Y843s)B%Bb^S&#c}fwP|n&U#k2`@3+Ak1@e{r%*aG_@4DkDX_V%
z1g_$==OTO?C9hs6iXCZYinNjyfpX1KzZurm%8RU>DDB)}bTT4|Pu~Ll7PT??dpTtW
z(h}~ycNS8i<A_^18Bvq}W7*={H*j9{G&u<OGi~ZuQGX-JEseK1BHq`>s54pA$9Oc@
zYr6e=Kf(CBr0-~iFdY|x1S_?s>L*tIJSG$z*L~Z%3^J>p+|fFxEuM*gya_Drg{aJF
zF(a~&;ahLbKyoT^QzX_KMGNb%Cz)szFZ8c2MZON-FQ~XYn4r*XUCt-f41$b)9H8Ie
zfj~0T{8{3uE^?1ZF(*w(6)W_qO+TNvbXbV5XFLF2rhaG5nwP-G3BzSv5Is24+JprO
z?z<v&GI1XkzdDuaqZUY)0cYcW;%ASR#K>q-OGka=fl9MdhzTdK<$ersN{J2pHA|cJ
zJgFT)N)L2RLqVy*846d*=uc<8e;tR*_5fv?!s^#66e~=(B#(vwNeoKzT7p9esZT}b
z!u^EOs-G5^Y2WhBRGblexYn@}My$u~v9fkNQh0GAIy)c{WsJmQ=Gp5Ylt9E`cbe`s
z(&dj_a}G-(0q6;gGQx!R)G)}k_uSUap#`*}t4u_FSlW~Cj<EIfH08Y69t(42^*JGm
zFL>sC|5%ctFPXiEphb<BuQ&g`mza`%G1^Ex%6px=o7)kT;a-rY<j~x^<@C8byK<Eo
z_dKOf4~Ha=R3+Zmeedf$-_Cxut~E?2*IYD9ZF4%;xc@d0j=xA&B6_nO-<#WW6z%)@
zleSkO8H5LtjK2h?9=e^4PPKb7yNj+rE-7;#9uqkZBZ|TtUUDluP`+@fSrS-5d4pV>
z8?_+DwVgHYvLkAIc%5#TtV!@nZRmrSL&Es?Q9b)e+_nsakchKE*12gRnMfWvJPzxv
z!Z6CT0YL`hsfJ<kfR|dF2hs)2;hQsd+~!$ERvMaoGC}{DQ)BR>m*2{+;piK))8IWe
z+gy<fPQ{s{{c_Z7ux_?4hzH0I6dxdMv|GrC-3Z<6uuKOQO=y9EgjV+!U!J0b$1i=?
zSvq?*J<pP)BbMnQ6suD*tG=C374(#xsN4LMLHVw#Wf;mUQdpFQ4`)x<SsV7Nb~KWO
zjEch-&&W2HZ1*WDd%VRy4^fEn;82X7-*ROS{(c}fq^Ded3h3~?X-aB4I)X0pX-^R>
z3wbbDgcc|J-t-&lCQgeWgOP}k4h<<|03~S$L{3zNr+HumN?9~<@9LBUkx$)8PoM#@
zGxZ2KBgm))O)(t9L0&I@H<fK_M;82?nxc&(L^jo5^t?dr{c+1XBiTpm@o3S<lUGl%
z8i@NG>|T=73KU=xvAgXw5==ci%}>YDZeCz0Y{tI=d8!E3`{sMCK_p{Kk8bT>0{=KB
zYxDxHtyB>|7Kd}%L(8$Kt@H&oHBR~?jk8pRJ`c&Pj25HZdR&ILiHhl%c3H3Aq25Ug
z;4;c*7dp0*jGu`yl%K0H^o;$X;#%zMehLK7fK`@~q9CvIbq=R{$`O(T)Khj--)+!4
zC5@q5v?6fq`!BlT1#9!qsfcggwNo+XwFQWSdP(8?MRX!n&Y-~}L6pfm^OIozy@<|1
z&$z+niYFa0tV|{7qj}|XIQlU*emsrn1oJ~V6(fH0dCOjt$`ZD0+7tIYkgV-JTsD|8
z&$DWbT{Z}PyWZ=RlW9AqHBo>86nFd>_fAhO&1Y<?!_r<sL|3dL0kjWit(Mg4n&wx7
zmzZ!znig*OM=85A=fl=5bBIy8_j>Q==HZ`*0$bm?(|otlu=F~<UkaJZ8GD--rkoa`
zJdyEpOpe{SWm0F5pO{tKBsV_fi5R_HQk$Xsy5}EgPYl;)3ee`UJfVD3>k=>sjze8X
zjONH<)abpTyHoHHXETc))#O_sDQ*1Hh0;YUX>os=!3F_!9f!NhWxZ%%>UCAOQ$Tx5
zO<Y4$Vor>HL+{N7SR4^BZMQ|2I6(iUb0VK_!#p?k!+}fmcp`l>e*Hx7%H_qqDR##{
z7WN(zlRQJp6i{5dVzYa1)(Tl$Q&w-EJvLn}d36VWIb(Py=)iWPF1Dgm;(%lsPP|4B
zQJ?y4$b4xgN*5O%GEKgcdv6J@&~{Y9BG9y#XwU`W!aS6+FvMk#Fv#(<UJUW-@X!3l
zS(BItjHIe(dorRn%ur&=nxKZ2uVh!piTkHFgq+|G6nw6L7pg}~+%1k5s3Y2-fW;bi
zG4#Dm0MAo4xeX#m!uifIEiB$HtbvlH8dTL7nO%csFgh%f7qY+C+Uz?Ez0)l;j2FPs
zzcbKKx}-+ZEBjFdo!FDUy#4FaxmO4_Tjod@{M}<Oi#9Sd{KkyU=kB03JSfxeYdDC<
z8*h6Var2eC<)x@v`GL6zC@2u6uw~+(>aUI&>sD=xDkun-iQDob2WnBbgY+8n+rMW|
z0VmWE_;I54i;25-4h#8^%(8*C{s4T@2hIrhyU3PkUh?FbO`BQzL|klF15S{h$2$uI
zJwv9lv5(Ov!Mwml<jPM0i%jl_6rPHo14Z1H9o*9j(|L)wa9LvQT-)!rH~ECGPvzau
zX1mudX~5iC44^l5pR4ZYma)<|4QV1^Cp!gqR#j@5!{`Z3wJIEBu8I24#Sx#ngto>r
z7c8?953h3y<9(Llnl}sw+`o5=E0naBtNEUVUu-8-04bp-7@xtV!Jj`XFoHgC$JM()
zUL#r8yBV^wHaaF@-apYiiW`~SpG<te7WgtHVq40KK43n`dxyTsJ1p$f4eey4ZyA^=
zFK*cC7t^s~zkM`6@i9F-o+Zf)^s!$MS>%{B{3YzMhX`K6=17%-3GT0OuEy2A2_?CN
z0u~5Df-1wf=5~Cs#@@?1CBddu!eh+bEmTiUBZb{d;0En((*9QiUy=TLdvfg{+yW}=
zX~THC>&b+M-v~dbq@H>$i)Bd9XV!nZljI}sh_*Ic?>>cakB`}p*QFOa|A2>xplTq1
zhDfus#FfrVb2_(v;EWR15jr=<Xv6n8<v{EvWz21>D=9W>j?t7$Qki_D+*OOg%0*8*
znhs!cX*1y^jAMkylXHJTB~Cpqm(_2c=t^{E-VoDFO_Q4a5g%2J21ggW4rH=czz1LQ
zir%8ue>0Sq<EoHO6?JARewtA<?MmE4b$E0?w?YNs`WL@DF-nJ0U(;JbLf$n1R}P)<
z#e~{UWGNfpZeC7sB48iTlA52!TVZ?fOvc#O9r>L!J#^-bw?*k~&vi99Z5Y_3JYPBD
zOb=43PF4hMgC=^3bTnP(ogKB^A2$(+sg%CfSGB26#4^p?T&0QCdq`QW!?d5V3#UeI
z`%9_u=1b05Do;*GED`nR+Z8tRMFKQm11{wrN+W#f(;N>nQ+n{0?5@fVbmtkC8se65
z#SkkY>b0J3B5sy{{x&P5Z1b9!0{=FJWMWbwY<{y_WN*xWS;5v6KPN(-SZ9I`&b0TL
z=~9j1kMa=K89N-iV2Mi#5dK2`8N_P_jn%-5bzkQr&h1<*Gorx0m8(zt8^)x1p~@xM
zQaDYEFgzMHY$a&}L?GpQGGY<;(Km0*{z}bywLC2nC}RmhcuQ`yTn>xAPY=JAQhcj-
z=Zny;3#%M;c)&H^=_qj3u~=1eu}d`{zLeL_N*}A?#mHo$b;qY763ScH&LMOy0^S+B
ziw$mln_R|i0r8{wP5&fyQ38GX<P~nB%yb){#VVeRsq;AF00BvO6eznYamE$e0)Gzw
z)Y3qwzt;Ld_?rYCq<uz)yav|7Bajw@xPR+KHBqI4(@yk&$Xgu$?e?A(@I2Xoc<i`e
z$5?Xdt!9c`z3fPFp1tFpj;uMbrBnYQahaN*39C<*0$gbr)ZUKh&x%l6Y&OdhaIhWM
zZvP5T`rGZ>Su`{rL_NzYZ=p8`-+;4+84|*S>MELEkwv-tY=vGi9hPs1LN+$`Ca3-z
zNwQ?y=g(pCJcIw7^Qdc-ZF#i<yh!e-AuTmNwb*yXF5=I+(1tT8?7h|#iBT%$r#0F?
zL*g^kmbiD9SBvtM9aQt>R%T42m1YAcyZQDI4+jpD=VQQ!Fol4Y{Vqgi08!8xq$F`H
zh*{{5WF?m=A_~ruZ{-#@b3vkLK_THp@<{UscgTUsq8jm_WvDCI6DrIN=lwyuJ=w1y
zJ-Scj$>-nvIp$~3u#r@g#Ozu*DvW)8D>J8S587c-)B0)}IK}!3iQSI)1>@1@@;&_P
zN9AWmKx8jm(0_w$;A`{Z@?E<`sD(tSJE#QeauV-lYjUH(56|MeB*dgLok<J-PO8jj
zFwOb}gQck6GnPgxc{~1v(mLFnMvL5<Ih5?wSDoPZiV5G+YQFR82uv?@FS~>09WFLt
zz*Yy_qG84c%*4V2X5V%@NeppV-@Bsz=Ua0Qs8sd9Ar@}u$hg^XZtAr!KJT@B8{RiE
z_8zh{yoSvR{x?JF81k297rl?QyUQE5zwru3fbN*SsxRFXTqjlWYv){%TPx&j)Uzc~
z%ltY0!J;OUVnOxoZ6y+qPvSmN)Z)b-o)=>>)Gz}qMKzupeZD#R{XdUaYSTHTBIHBl
zt`<b)%1&<2B=H3q!`DAdXObFF-kI=25xxP^4e?_eV6O8O&YQ|!rE?qTK;qJ~-<8kB
zrnK^7;uspRu2e%fkXWN6Z}vHk0fZTxwc%MZQ;*y?My?(v)4+oUU|Rw?+`^;6h+=Qq
zKGhG;i!UxxD%Wsn-mo9?AUx8-E;w(8SG^sVj`C48Lwmqnfv};#mP8?;_f@G@Nz<H7
zVW85xMIIN^IJ9+}0?h~DQhy6|3mgWj1r9x6lX~(Ge)TTOiYkMUAHS{;sDU6FFWI*T
zG@<)j1EIGMm7(kU#IAtm<GNb6;3^4>b&ge|(D9>a`T`Yb<hd@(HOSO~rZBD4=K8Wx
z9Xq1p>MM}X!KwGT^JQQO6#a)Ml9gjFn(J-?@tTg+bapn{?aYZ5auqHa*DKSc!=&pf
zvm=E`Gl_D(-@nf4^po%2AZ7NLb$Owv#|!k?2dd%1yKc80zOq&6($yH6;KZQCBk>ur
zv?hiWZ|!^e{hcTGYIq>EzS_GH8EGzIwNfxG$*a05kk8U>&=~W_7yS<U<AL}mv?cy3
zi7jq4fXthzq3vU9gkc}4H5y86fOvj!KfXF6$xHyD>^ddgE&--^$3fodg4x5o|CF~y
z$pgGTDe)s0?@J8$;fuC!LidK>5k!b&TcL94ahGVrX;+kzo&VlpVPuz3Q07IAu4`)i
zXU{R;Wvt<5kw)o;#fEdtLN--T<*VaA!EO2D^2@Ti>H}DMuCwM!T(_tCRXImyF{xkQ
zi6C=}APcjs!0niu<uOJE&eg_kf~j!C+uMszPH0eYNl}Omn5wf-qCk|OEb<W8Eq}lV
zcyQXexuo|k&D;u8Sum@&d(8eQhT$V~vG49~<>Sa|mLW_-Msw_k$_i=kt9II_HaRIx
z3yh3T->h)F#|?a+o8Wv=ce;}eAov<EIQbV~82<~*dls@kp8w%hJ5n?Z@0K4FY_(RN
z(la*}(5Xga1$p|d_P(dL<i0%)+`NjS{=>eTI5pIai0URu7`iX9ZL4=KX=5wA_5q&?
z^TyjQ#xJ+9e2~JU=@opl!c&xADTDl&8u8H(csx+>7kar`THBSu?r|^0laXd_v$tt4
zi_$9olaVwt0uy~8#{<Bl8(&jxtyAcU)qp!1Z>`n_v&6HuFpHYq!el&hk_SnlY#{Oc
zn=Y4$Z=rpnZXp2CU1SArLYVxNZ%gOIP!f-Sd}B)tt4V%uzs#nvjsS%Ew(4ku`CVfx
zib5N%kuvDZc`XUMhpOQnuYAu@L?)iFaM83|(}V^dYR616N=NgK<?_j}<3iTD+FfL*
zL%@2eDEs8zMd{pt#HrVizGS;UhggBdWZvR4>AtjSfUd+p)A=H`6|F9uy}sasnDC)x
z!(qK`4Qa<iBAXQrar#KF!?c9_tT$h|_a*81G_ddF{80;?Y!?7JsU*G)=Hs!MD6qiJ
z`A4nQEz9-=OuK%pRDj9IERKdn8FsP<;HvOSn=W-LN_~mF)$veiXT7Z}r`2_xhxUHy
z7m7kW%YEpiT%UsiCO;P6@|}IGF0a%PCgq02mY2=`&pUnQ%7SFY+EcXP%Io^Meh!Cc
z!^Bv^c7X^KY17}YYw&L4UZcNO{6&SviB2i6S>>~re%xiPmU80cja@%Hk*(x@EdEl<
zH%pTKLfjwT6E%)Z&c+mOND#}+szL#+Qq{ICuI6$DwAPK1;sH6TH*oYnB$68~aL^i(
z!WX=(Y^MoLP$XK2QN!GwADTurYD^1^z5<kDW1^x-XU@T#wD>c2S>EUJ42}lX7_)|E
z4t90ky*i5X*ae2db8kM~Pl2oqU^g2rmvILlh)LU<%JepwyzF(oe2&RNi)lRcZSQ0^
zV~0UA8E4~es&~nyA`f-YXmL?wysTc?*Ue&7?goW0nkhXe^sPa+B^=-8#{e>%(tqA?
zFj90&?Hgq8nnplls}MjD7CZcNE80B};nn%-7&i>Ge8W!z2`e2#;-R|Ab>jpL9=`o#
zy4DKL7dAs>YEKa(Nyb}6u`0yNXzk09q!tIV7}dh8Bzqi3@?p0t#`r#*=%-YLzoNN@
z;~MQoIS{9Q;7cC%>z>v)puLV3^`X0wN)&tze%w$oO@u8W{+dae%9IS|VpYw;RVPq+
zJ?30TQdslMcy$IsWX0mlOyZy6JQTHk_Un}?Vi36HA0f0t1BQ2#X2GyR1u-b5$Ic#V
zYd5=;SWO*)nWQK)ud<nylTq;V_p+DY0*^~iep3^CyWr4~M~aA{M;RPDS-suB%L@Hw
zW%NHc>O!1VE4SGwsi3n}11_;64w;#hU2!}{zV<Q37r><V2%#q6JE*nhIvMD(Z2R@O
zFwgTjBIvGLMHwHYNn>u=d#CMH=h8+f6kj7+f7E%`26WR&B+T>n*l=`HkQGk}V$oTi
zRP^@^hfv6gM{pYTW=b5y2GjR;n!3Hvp($nelKSj^%g@qsB_vw(foXndH^5c}6R3V?
z!>08dSm2vqz1T6YnjKY?9@)f^B<-kMh?Nq1;BCnG9q)OurvQJWg{nSw`~BB=!>!7!
zT<L8(kyLB(%P^RR1Gn*vv?=yS@l0tQV(Ej+XV+U=cjVnbg-Ll;22NU-{QZAud&{V(
z+wBc_hVJf0N+hKuhe1G)6j8cET9lG*7?EzIQ@Tr$?i7$3LP=?K=%L>q`aI`<&U4OM
z@B8Kb#8TGMHTUfM-g{sB+SldSSl?-)5lZpddYF<&fE~VH5bzB7z<MkjP3>Vm58&xp
z;#eE`Y|vffB~JY&>H^rGWzdR(q|IR9L={AwZ7Z0Vty$_@0M{o{D&R@Czx)Dd)d>;y
zz9^OgicR%i5z8kG(5E{)y-_XYu3qXz>rHj<4c+MvJSSy|dr_EPj!K>40i%n7(4;KC
z>G4n;rmh|eeQp~-6;pfx5I_mh+BLhRb!;Yvn-SE^hQDhMZ1Ea}dn$zvfp7g`F6tZ@
zof4r2HJWw<QYkySXm_~?OX{GwYunh_9p7V^1qBWz0YKgqiCE7fjp4fUeQ}J^x`t*+
z>${P@dE4xZG)o)!{DE%_Bg`S5{w=rElks{>%hK{%2Ms_sG>%5sXSd5crTP2r2D&?;
z)X@`nfG1xpl9Q-yBzE*`QF?z{>I-|6N*BsWMf1U6LcypD=I3LwOWTeL1I=MSV4Y0m
zg+de^btpdV3&T2n!$o;|=)_BAw=N#JcCgS0>KZ6@{r>ByzpCPAj9}ccb=~(@-}PIB
zXAoH45_g#V@oJ~+9!cs08#k*)gg|nX#h{^JhD5^<I7?^%Y<8~$2T?jYy9#~;B3TL*
z7jIC{Cf^6#N(W#)oQeeV2kLFd(mV~8ELwslc&=;h>{zEvH?~8q-d`yofpHB-)dgZo
zS!Z^wU*qxT4j4<rCSefs4_PF-?sax|@N+3x&cHU=@cb`Oa!{b`6LMD7R1#hEC!~}g
zgTrjgN|YI>&k+F(4(N}3?{E?O91zNSCeY9dz&l#eYsW%`DCdx|e&>_Z=>_VH5Q{XD
zK+NLkYHU`Pn<Mk+hD44myJ)0xf2riLQG^_7{A~*SV4f+5n`s&wzDF^a#w)p-i?^`<
zYvYEVxAio4_y@tx9+mG=#s~f0HL%b($}QsSni*q(jPRc={0mo?-*o4-ns_F8T#WO*
zNV5mrcJT1>I3g>sh{P_kb9D8#oS=!{mAPrJtL7QwB-%NorB18d1y|js1?tHGGp9&D
zx(4Gnur+^&^y^=_ESe%TJFmysgHW7<sb)+!7;HbMO%i@TiK!ZYM=QMSbKtR{x3f!)
z>#r(qXPp35ZyIU*P6W+7mG)>m#zBvB%Sq=08El-#`HRk4odCUy{dm=;_A;<nN+HMh
zaVgLuPdy7_gG=*?jg$(vKFMArAQxSR;+u{)siFIFNPt5M6NQBXMM`eSQ2;6?hVLxF
z4?l9FQV77S8*~in%%#$o9+uz>TfMWy^KwImf{S>56~uiHvuMC7uHVX}_<2@{H-%vM
z+v<3p!15BgGxXJyNsiBSOxQJ#pl9fh&6FB&3~_1l1}&3x*)wQ)+%{@ACQFR%cK1YB
zn}32x*J@X9<GM-*j~omLWO$-lb1R}Uo*f$P>^xcpO3hj5$BjBS{T^=$lJ-D<4Cac&
z+wF|`8kJQ$8|6exK=Z$fo&kgl@?BsVBA#Je!hU<f9lmeRW?B5^7c~==1>92wK!NmZ
zi|cr34%{Tf=b9NjMx1=P6H&)>o}gwwY!BH}r65XL{PN``XVGgiFyEfnBVm0}(~0kz
zVLa4nx581M0BRt8WCU>eQL^1z;>shvG&4UYfqsFbhT1A(9qe{L@u9Q^QMHZWC8juB
z_O&Fa*fhD_vsADR$4wK%_GwU?a2?$VRnZ?5XTA8*uv^O{D)55{)oeV|qLE~+j|=1@
zFR!n?F39~_-O?A-Fc>-Yz3*^I{3o^rgXhmr8v;I{{sncUL60lWYd?KpbCh(mx_%T3
zP)+~~iRT%8*~Jp||KQOrF*3zDECyqs75U|ezcEzVil^C{@|8v-c^_zge`j9*-?u)6
z0o+=-C6V?UKu?YG8gK#9Y&KQ#{{B+Xr*`Hg5EH}q08UUe(^E!=W!4wru4|j99H1-_
zSB6l<ED^}vErrS=b;(QpEBNqQK{(Hu0wKN9(OQ-`rVldNa5j4qo=+IfCmEq>9y#?a
z+2Kr<B6F@^88DcQbBM&1vY1z)DoPY`-{ps@CdCCjOvQ7R5Zl(m6hp_?ot+kCs{}Fe
zQLk0AWm+uD_P_3LyBuT`*+K>EWy{<XzRDY2n+xy%h=*(+ISW*?#o?bbpCwDz>$25c
zTN2((#`aNgy;knQtkrhMNTmwHV84aLRr@5ufTpz)diDd*bgAF4KApF1t2(=<Z@~-C
zw3^J<Fb#j+h&9CBs<MKbqGKQMl@ERhqCq2ASZ}Y|A560l`RRS~r7m%z=*^?h3S3wG
zTM9Ftj@`V4qufH&rf4nwc(LqtW&_OZB7Yu67!FO<?hGdUjx*7kPC=3KCV=01;#(*C
zgahCNZMB<rE@1V+tOHu9?k<(?H6ym15xw~Mq`HGjt?4?SU*JPZ15mc%&sv5uxTtLk
zy|MAa0}0rh_arIXd(=;0NQC?<t9!j(VsJu@UF9%EeUVlC;$EdMvgFl}W~0!uuxC)Y
zo=yzenEz=akh|_%I&T32ttM$~6nojZ5%_NVZ|wp@s-q$2cX5XRTIsh0VHrFD&KM1u
zZZpER63Xh7k!dm_eWmP5`IcNfK5lnluKe{e!;Qr|v->H=^Hq`9^DJ-Q)$B|HsoTQz
z{x86^SIq>|S7Wiw?}j~m|4jZd1wPme^wVTHBSg=i1RyUGl$O;fiSYG~u26*6pFn`A
zY4|LBb>`>o0&BWgD;jUx<pHOgf1%u2&r2kb<cFDq2yy*}Gn_;HDugeA>U&PLq0S_Y
zpkar|raHq8e~Ik=S_|1{`yItg7ggtPPDXVm;gNdBc9Bvubt4ho^|GP#>s<{DwDwgp
zNm8)=^-6m8JD(AAb3Ew{K+Yw*0B|PpA(YOCBDmk0Uyyxvn$-p{F6&(f9$m!7`Gjb=
zzt0c*`CVay8+*<!S1g@qDk$iI%M~T~qRip5Jnf}q%$LEZjq|Ddo~TrPcSJWbkk0B;
z%?H`C6}0O-E0>6KaCcV#gL7xa&orRr6#=Z@-=dxxnaMCR@wMalKiifEl#>mc+6EYA
z2SIoDQ=d13<*;2u0S_}YrZ`DUfc>!|Xwe_DE9va$zTZPhI)-jMoOZ$2F5K{w85QUK
z6FCnkXEn-XyBO2D#jEA(o=;5IKa3jdyEE|~zkAM*Q2bg;f9`b4({Zs|=VJpnXET-c
zXLsZb?S_%}Mbm0?%s|CkcYH<I1*F}eJ=<RisS=<H`OGzM5F2UhE^<id*{mH+FCpUd
zX#9<s|FhwEFVY#5MOwM0pYI?1YHZFzc?Cv;5mb5zc!lk5u;MyVSo0Egn}@4>N%K+X
zOzu#CJhQzd$kvc+PM;-y1sWNS`gn2ppO4O6N^oZ3D`*3mG27yqOS`7XxRYCk^x>bi
z(AD9-Zdq#UVEVg;7Ho9jjvO8XkPZlezM|oz_&`gLAK%p?gfW{}BH?SS>M~IkCNnB9
zrt9xaHctE+X*j@96JxJ2^KmQQPw9-30abVZF7p}!JW#luaKI<Evp5G$YM`fVE_wW@
zR7(hl=B@qxyW1=CG9+{;{oSmTrMiY-d2QE$#|o4|bN7BQdQJ8uL)ex`fg+!MRW1{v
z7T8-s`Vg=h-n8fnT7x|`e`4s?-DpDXvaHN_f#i7fHD2AKNH%r!vYG&3eq)a1e-zr4
z{`2mLhfqj5Z)Ci|pvXGPDQ;1~L9Xti!ji70sN0iPE5#t`HKU7PpS%MJU7GD7Q?4wx
zdXiqg{s_bl&%H+h`k75)23LqFOL=f9osdcIL*v*b#>z8UcZW_p)h1y1t)m}DN()Qj
z*JUYpxH@Uq2QU@}zlyR;H-R*v9bcmx$;I|gWvBX=g-ULdA~t>cPg&^F6gG9%Lpr*f
z<0y%9+=yB@=L$nF-$8-)OXB&L3Z0Y@L%g54hl>2mudl2u=CaHASh?lGbVLE(|4`c1
z93yo*-le#H{!D2vdR7l_yRb*e^DMr{-Aq{w$@2G=<->I&($L$Iu$C}EL|?)R^p5xL
z<kzQFQd*wKe4x0xL~;6FIG3(WK?+9d)Xcxr^=-Wd(8c_{hdOkP&q{sEtF5?|c|Td0
zEcY92L?qkw8GTM{aPL$Dc(MTF3qBaS^KPb~CA4~IMX-{(C37hZxVrsnJOv{w9dz*4
z$YDYr4PU#eX321*{rxU8p>8qGMjsYnQQIXc6VQl5dsi)~k7=OrA;r5FpG&r7Vdpz<
z(T1z2_JSQK8yl%Z01c`b8B1)Gr}eVRoF!A#W1Z1s$R?|h@*2@Mtf>brO-?(=FBJfD
zapa0mICA6HxEo!`23QCI)^lh*TURy8iyH-ydZGS$;S<m!a8+2zw>dS>dme<VQhKN9
zk~$hr^+`MBjYdm%6+m|2hxb<Ln8??)<U~-PYe+W(*n}`>tOWCQ3kqm(k;O923r*MN
zV-{|(RqYRKD2qp8b1T`K{f{>>u1cqdxL@T%A!d`cR_ev9{3Mm+70Mdpb_zZXzo&BW
zB6pt`kANy)P;l<&&|j$cUx`2aDH@XD!!I-70v5CjCLrjbnw%wqjg_b<*1f%0pzLZw
zZjW|C0x02m2sTqtI2N}*fqN%M5o1}Pdx8lTB7kR^YLZah=YZ=IJkgXy=+XID$g{a7
zmnoPu#ltbH`<SivMHP^(JB6Z(Fp&e>Aq+NF{gMd^Y;#jrYVEYgYV-C?_XMa^;cB=3
z-up82cd$@`;UH5Os^Pv(9tZvTl8iWKU2!STVt+q}<c?;k9$fyB&tz;*lFL>e5A~S>
zfduedps|l?wQrX-Hc#|B)1al^Ve~jl=5x<j9iM1?^-jb3)t|09oz$j7@G|$9@dl}I
z(qZ!=(kUz?2@6<E_|+A7ANAc^Li6aheRnE=c4~Su`+|^?&dqci*p?f+uYWo394;k$
zGi7!XmGYiVJxle9Tk>GLYx%_LCE+ih1EcAd>w_Iw`7~Vo$gswdU(;UjZ!LfvhCs~^
zs@OA-lgd|p2nm4}r^G4Gat7aqtI^H{=cESHY9~@i3X$MgIu+!yZtH}clF|3^0lv;=
zj59&?z4DTH&W)lBKWb0cy(yK<>MXaN#jbF3WBY0C?#(vao2p+)vQKAZD<u<Fj9E`r
zm#PhW0GsVpo^}2Q6omsNF^fN8dtr5023x?s+uiEDV2~I$Q8iBl8-;H_%~LrGOR%Ko
z%h&1XccI)(h3gIxG{W~5d@uGq+@>p@XW4_0Gj6i)mvxrwhblM0Ics6NkU^T*Uz)T;
zvY*PI+fFs&jvrVl_X0`*>S#3yWEwo@^6h!egcNMLXqFnn>K5B@_tnTD-fosr*De$h
zYMz7N*LlVrG+u?CYDBxDNpf@XtF7(vI{kcQ)3=A^_0-m(c<;g4vGj>Z%TnTo2PJa#
ztuOD*zEowr)-}LL0KaYpZ~O>&A>Yuy)PB(tDRIcYIt0kMW;}y4s4?NzJVu!0zqNlv
zWM36xI4*4EN-Yt-9TC*TW`C=Bsw$8L_uZxtDu4()0g0?;c3d?GgJ@iIBRE!-#zp(%
zOdCsMEA^CqP5@Fu&TikUdeR3zbnO`GYjkWJ-k&$nt??hWZG06+Z-(lR&i|Y^Ru=<`
zc?wxc<!Jai=QPAqOSN$U=6gjjklVR1sG)D}Xnbh!3)E`A-sp6F;JsqfIEY#&Px0yG
zPEA?NY`>A+!Um(y(Ks2`cV!PL86Jlb48+WfqFVEHBM=c6VS1gw)HF41r9?|b{-_<W
z=r@C{J5{&03^-uM3y?A^uyeG_S{&D&(#>YCiJR`Q|CDyq=QrZA5D7$EU#*QpAG&);
zf9l%PfOVM|v39-X46Ir?1?;b@g!`jfztAiC7W=Qs*Y7s~ic7rY^}e-h9w1c`WZN*u
z)K+~4##g>{vin+q$!&KRbNLl6;6M)xtkZ2rtpx3Z;({Oq$WDPGLPG>5!V>lrny53<
z5cx0)q9{<HrFjDnjme411bpnY0(=Q(6S*aO^Li9PGX07K`f(Z=kgfig7M3Y^YgK$V
z{ZC*Plnkzl<{DD>=MM<`>f<o(N~G4=+dXZN(~5tTB@QMb%zn$RBJHD<<vXx(RjP`e
z!{MPcM|4%g*@oB;r|W*K#ppe^-6t)wMkq#b_T2FLd2ZIPJbYt|E4%cAZ~&tt<0b_Q
zY1g^=Ig@3G8=)PI-6Dvq3pxlq!YXx%y}Di5iD3q2ruUYgd1$q^KRrcD@g~b;yfQ$#
z3>H2oG_Mfbz`07&V>{BRQ$YJZOu_CedcT>x(IWL|KmT0=qHWSlIplV!Bb?`>dx~J-
ztM3_?FTc`W{Diwt)HIgA`|xxl*N_6gl(*JaC;KpPIp`xgIVedTa+7b_L&FN%I|_ql
z7f>4CQ@Q@NhS=7x42@>FTSy~*#bFg3vSBQ0hzY}!`uMHSU<5uVn-oGx;R2qBp$<8v
z67WMPBi_q^o*?jxg5_URJkuR}xp#-^J-yDeRmyXJ2b~DB(%@Y5vD~xClF?4;c{T7m
z5No#<{v}H5*FZ>{z10`){uRb5VqBPiB77w<WZ{;(_}a)cg#7C?8xoVh{UOapI{K25
zxs?Ghp33i#{sq%ZdA9r3SXr*@;%F=0cP{`!%{^wdoXXP%@WPJlB3tNbam$8-?B4Fs
zj}aX@?$z5l)PMNmVXl#cZV19Pcw>>gB8*C<Mu?+lUOGLaYUiC0KVY@W<mP{+NBV%G
z#_RNFhEXL56QoiFHRK^yZdh2+dx6WER#3{Ozu+uJNY_n=P_{e%WQ^i%`kAGI(#4M1
z2p3qoR>DR2-~V`;jLI{;f@!!uA{Bjz2@kjs((PqxSOlk|B+ftNcAZ{koZ<cD(Yy%O
zTWVEy8=Vfs23BI{ek=$Q{uJBP2g%um>N@OxS_f6(M09wdNY2dJ!S2V_N2*A|13zB(
zzwErKl?wC1KNY_`giNFYvWWX8uu;5)8|u$3N%ywu>!WAxRg!)$r1q-cUulU7dQFOZ
z^ywzy^Go*($B)aQHognSF(Lh(vUF$Z*;GzEHRy}oVWgzUQo7C2F8~aue_KpcdZBF-
zgMqbKmUl0^{*j>>0NE4_eNSHY%s-13jnR<=7}?ZxR$>r_n=s698<atRN)RQe1Jx09
z4ZV^~CwmE!qOVRhrCS<^Lf|AIn2$9mvq-!H&Jm_p6P2&6y{8{YZVk4ragM{v=v>bf
zC~r1#lw}ybk2sgRWTRyd!=_$1Y%%ls42)}&E;nm^MHA9R2XwU#WakGpXS%6iiFx=r
zK|HFNk6Ab_YGlUb`!`t?vq&Chik}p5wNEl1YtH!^KHKb^@h%p`Y_Q5D(7l*sQ_k8x
z;M3jb{DJDVJ>y*kHr5w4T-FyQj_YOnLr{~3U0s|KD$u=_)0_MRj7PhQ3o}!Fo}?b<
z9TE?}!?JJb72ymgY-RMis(!V;XEprJ^rL$8AK;>hC?HWwkJACQa2QvzpXmaH+C{I{
z-=L%J&D4FAbVjHZPr$+MiM6`a@w36e=Y6K$<|^QG=GCwo*8BIN&n(o-62RC{jQ5Tp
zWBAPQE!lUXy=E~b9)X|o_I3`rPWwuL)&&Q68^=gDx=#MRe?j0!_(KI$M!-Um9)W-k
z!iyOy`L891J37<n9W1Y&Y!E#`GXQzFbKe=jl{KA@OQt{038&{rqk{t$I)f#=y0cxB
ztoA5Q2ZW0a!h{yjlkra07`f4cdGq7M8)EB`gy+kT3FfCU9FAG?s(Wo8A%rR-bRE!9
z>*MUL1<nG@jRhvRIPN0Iu-MB)3J8uh?<oQfSTRMi2(bew>A4WQ#Sdf2463<lc8C7C
zna6Q{bKB~j`;6Djy>~q9hR!G4g&!o+aJc1j>7I*<_vFpc*p)4VQmNiP`VEkblu3tQ
zzC!}(9q(3xRP;xJW)0!vT3-uDpe}L1pPm1afPrIeV?S~N>0gD<tt#J>uexOclN^7A
zrFkl_fV4x)@ia9P^gdG%DHEOmiP*>-7I$?wi487Y%{!^1G1-xtI?%d_eyZ=)c|pKI
z)%Hp_-%D4K6kUnWx~nY<Cjl1iraKQE6nZyX4>Nw{N5~I~<`JV>H~nndDH$|kG329_
zMgipfK&idb$&uoym}5fQ?+;&l{X3?GT$cyx-bg>STagMpA@h}e56O1<sxN#fgdh!^
z{D;`sNHfc#Aj)hqX)Qr}w1y4>tl+!`1XWEWUNHV`qxiXQzhJ+!kk>m139L#zF}z+g
z_U-QDfZ0E}6XdXMi%UzaOKmJr^Ej(MB<ivsBH(ZbAZ5_-Hwal{d=NP)o*(F}4bpzK
zq@47)lcN@wW8pm3XP_jqS?wO1!&~thI*&kDy8nZICCbkT>hQf}>4C2wDg?CBka6j~
zCVkAt!=ygzoD2>~01wxtTMu__c-6Gf`05UKr1I~b;HcLGlt%8$b*`Z|W-aFF(eK6H
zvAnr#<G;SzH~k{Ej>mPR!^}P7@M&1;y;VPSb?vWFIsF3^d)QY1pQ+=C7qH{kYj{8{
zVz1Ljk8HHC09nHYuvT)&a8K;XkN5i+F(H2A>TYD?fT|cBF#LoFw*U#X!nJ`YT)M2k
z!R28BPc<)4V5tnfI0+aKGnp4KTQJghwfxfZ)szKrxhKfpsw(WqT>tBT{v8{CcsIlO
z06O`C9leSE+bz!c@^uNIAR^4LM5J?w$p~5xaP>og>}OcL!B<*QBACIFtP4SNHJrLi
z15c6}_YrJ2OAKW<@L`t_^LC72h-BgcpO^)!o$}iI!Xrd<1=8aZ(+55oz_X*>Fb-qY
zwoSZNepIk$=Stsoe8(%WdZ-=ios<|Nrq+T(p)FDS$+*Uo!}A(+MxV3nx+TiphhJwF
zZ0zvR3^+!%1qmdQ;VP`OLYuI$iH{yA5yrha@sG1Edu-zhMnq#`TQBV-Yr=SnoF$V^
z@Ps%i@UPTw7vd(y3db>T@ZYvuDi|1vhR<hxk}}vgd2O5%2oMc^AA?EKqipAmFF?-K
z!UFhKgx+6h?wiTYeX&U*7|ei$Q0UzSt(edjCiQm+^2ir?NWV4&=JvAD3{suIHDTy)
zby<yk@FEZulR)M2mh{nlww@NC0wqWv=sYYh2aJVKty}Ap{nZ!#Gn?EVJ^=Ircjda^
zcFEo%o0^fZ;S~xOQx%RYLll^S*a%96Q3dWKeG9{roSWqAz=Mli9p#M1a#)~i<wW^y
z4ZScjUL`Vy-2IV7@);?|tG==vj_R?M*C#4Op>!58WzT>hryS6^frqS>PNwEmx;?l2
zuJe3wBBZ*bUD#S_Kc$GB0zg*+o^gr|JbAdySN{S{aaNDaA`uPSKWyD{&n(>ksgtz<
zP2>4pX5hj(Kh(aqB$Y-Pcrqx`G$~)BsxN@ZkCp;ktYtk_C}Z!uY2M$Y21>jZU^$zu
z;>NKbYcr<+)tctu=q$TUhWO)B6>b;MY5AUA&rXj1hF~ebd1+8xe1|)Zar@l+S45_`
z5;N%!-YYGOi{dqG3jVa7%H{C6N93Wg_r81zzv)BwwOEBtjx2A19gM9^DU`+BTk|I1
zjO97L6>k4`c$3}s+aablcVA$lj@?BaNuZK@1tJm!5zLj8qCG)*1S$(!<Ln^CvL`7m
zCZt9tYzDDFwLB3_104z&xub)C9!;mXV&+rd;MY(iH%nY<3!7CKNnGn7N#i6RwR=qv
z;glsQ!}Z-g$t{JXu4#ZnIVJM6YVL#KSF?5%gKns0vUoQD38IrzZ^so7QqP?{jPdb)
zwPF$f6@yDeuoDAQJRvTR_XhbCSXZLELhSR=&mmWcKC)@mHXO^+KU6>?)JL$Eid+}V
zZWlUSO(gmG3m$;|KfGD!+!9#q<b0GH@c4I>w{wice*X8TcSob){_`|rNW6v^{FSF~
z$##Fk4nF}rQ1zRv_yoQi6g2XY3Qb@PT6P>C66)%^ov?%^ik2S0w+o`eBrruQl8pm`
z@}!;OEesz^S2-?#khkMJnNP`-w7V;`K7^sQin+6)ir}CUqO``ACvqYdd=n#W%3&Rj
zsQhHheE`FQOz|E>tCT(tu&;j@kd2|tiNJK-*BF=EY?KKhR0o@xO)ifY%91vSuqlV7
zUvBL6Rmk;kq9l~Li!;|m`^EO=-G9s1){x*cU^Yr}1<x~fMk51O0h*6TsciCCK3CJM
zM`Kc3R~+F91?Ro~b1WY>#uWbdvGDvkmPZoK5TpNbEZ=XBrG-WOaL*NZ6dCTL02yoR
zND)gCh23A)9iyoOa}0wNG_G#wfB^eD%5{<}!hCmSJH^}c76$_#WS7f+L{oA{0!wIu
zH=DW_HDQvj$9dt#$b<vZ()V<?M$yzRq87@YD;jzi)$b#5=@rFGh7`8#hosY|gDhZR
zn}?}FC&Ew8g|N{R;(XiLj-0_*)(@dfvr{4M475#Sn~m4i(UMIoeYWEFj|SQ-?uvnG
zpzt1P$;9f^BLv|<TQ7|9tP>(v>^lc1RfkNef39;>+zJHPzOlWLj22>^@}LoN01$pl
zQ#*O9c_*}fcBt_m3Hz@Y33wE7Yx(T|fp|1CdiKAcup%oo0I3?a{z9V<459_^J~FkJ
z{Lq6+*FNd)k4yBbJ5YJ)wf}P{g9^xjAG~=7R|6*+N=tp*kxYM<qn?7g`&0qrV!C2?
zi7`F2hn1->z-M=yBQToRXB$|O-b%iayhat9?FqFAxp`u|JGj=}MJY!sQ78Puy`;}}
zUXkGTo;@Y1>gi|~m0Qsiz*dw*e=e!CJAiQ@PK=n+;;)hgTy8?wvPh(6i59nuqvL!t
z`gK$tBFQ1X!UVH1eoC|Nrc|`AWBYuKqJ7`S`SPko#*?IN0*_vwcWk2yaLJthf4OAA
zqpQpYsBD-YF%(5X?lJzXon(qpkgXE3iuP>{SU$|hJ_q%?zliJk4^)Wc+}AvU*#?*A
zE5;&Y$$WO0v~^oE?!f#UEz7qT?2h3<Ps8DG<kD@+E=%vD=xLk$(E%Bq;~BHd|B4;Q
zh9n^d-`_nI>nb(Eo`|gdPRu#0`s22GPBF{4obvlm0D&%EqS;*hAC~)^{J1_^M%U;s
zIxr;H#xx8qEi6G)JunDCeQYe_h_W*=qx4bE19eHyK0KjhuYa50p=QZ3HvK1*@uwZI
zuKM#d)9iNppu#@)2RaACDs>=IWEb{>Gt1IhzVt?nQf!yYWp6sW-#C0Ya9mpRQTkUN
z_!EHicL2)L7Exi&2btUI<mYRF)s+qkH4}n%;i@l$igLj5b%LHe^ake|zc*RJC6%Gq
zr%O_ohJ*KuWwkOPyMt{<oNPeRChZiVEt0S)>!3tnQ5A_l7gMj0)f?j1d(Ru)8(<09
zRU`mtNo@-8*$4G&VJ&xfoWtDn_zdXO-~>5U3`TAo@Ah;ndo~0ga96nJv2~Zf>1}>t
z`!?&^6_@qqK$5*OtYI7HE?a)wzNze0O-?*e37uQ2*Tg0wH)Z2$#SfzKTn9=L{Eqc$
z4OkC*&9_nl)^SFmq%7m@`L{HBo2B?o+0*}fmH%c$OQPUY+}cfpRD&`;Ixf9R-h#YA
zo`(3Oj_msF+swLMMB_O|vCMyp|L5^%U)@;(rJjWE4c9CH#l`0ykLXC?ZEOc`f>JtD
zr7=@5k(21@0hnQxB5ZK5g}m($f%kx>wseEs0pSN3PdHf>27PR=B6b#Pi)6=)dwHw-
zz45NXD)P8Q#_(Eyj<$Q2um(k+ItRlmhkNBgBGf4r30Xw;?2$eM?A>50xsJC}aw`+r
z>H*2B=RCR|`)tbk=luiH-fCFXhHHyoGG32AdIX|KbYrNNh&=~*SvH!5a_Vm8!u5&c
zp}t#B@4y(YL$c!tB?!01XT%tkfr$H$iShqy#BF=6Q0z2W06Mg6otL=WWa1+dj>z4%
ztWb67_jK$nS6V5T`><oz6a4CF%huVhY8VeSMY>;9)nuGA?7P&)mlr47{hX$KK?;@*
zmH$eWe}5@+EI>%wL>GNANCTd{TD$bCV0Ui{!gkJ>>B|lxf9&iL@Hhq+l+j{7E+?2U
zfI9mN_@)2XxvnS#F3O0UR;X!qo1#%`e&zmBbgMV*ya}yPg^p@)dKE#xHFpAhV5qAL
z_O`)dxtLIDn<jcqh}Xi7U95BpxX+3itg1?=jXwQcPY^bO_xFqKa^dkW%h+4%@zoo~
z!1)gm`3i>Z)EUsr&HNr)3X5$-b>8cN27ev|umM6k-plTDzy`p9Fns$7j+@QbZvom(
zTo9kw(Ezme8QPcuu&ftSabNlOY|K1Unz8<$L3Mio*4=;Z^tSU!lU)0--i{*)g^@*d
z4aQ<7Kd<EJQJ(2GnQ!psGhP(fUbVZZ{b`Tp>4s(-LFq4qo|wkQf2Ejhl3TEaEuwZ(
z5vb4bqB^K<t6;McMijwzW;}m>)FbS*Twf&wsQ)P@9~1S_Bp>aOOeaY@>~6XAY+r**
zY*2K?+7yh}^%V|E=h!0#h5~w8H)e}6sk0!$ahSt0Q#Y)zYc7s&i<u=2K87ri{X7yJ
zuNNl6i3NXZh??wE&&l)Hh;$mey}~`NFtYjlSu*}w5trz_^32PwAZtzS7;yV5Fh9Zm
z=1f5IWhrk~zqQJdDQX~}ZvApt5m(gZaPPSSKoB0moIP9lk12-`GVEyizdkNdbp>F&
z`;$>m;zdR!Gdh#zub75~XCj6<b*>JE_&U>_4^nI}d<4$&UM*%kKxY_}AXhNSL0`vG
zzqSjRe4L_1g;1)_cK)k*`8y~9-t{{GR^BG-*R}otpvL`BK?uX8Ckj8N-KG6Re!Z`|
zHzyeZbct-v1fMApv5CnAiYCde5GeriEeN}2vV)=@DB`6ClpmSbio}Lg1<;8Dz3%M7
zST(E1?3Yz(A&5&%tojsnxO+bnUuG{hBlTRw(9>36eXVRByWy_EBdCZFvl=L9kWeT7
zVU#Rs{W(t4ktNqVD{(|s&*Kj-kE;jbv5&;a_|Fo2&XUXT&arNO-LTkRGr_$;nO@3j
zy0r846bNLvIw-DwGDvYP#-s!+B>9!9OP`Ey6stJiX5pNUKQ-MF$5+^||5@tZ{ZrG;
zsn3bi+EaM)*4n?=3|WqbjXSm`K?R(L?<?rMP8611B-b7}H}<{W;$Nw!tZmRvRalCY
z(EgHA`trs$MLe?4T<sr8h#}x43^?35jtapETOdqh0$vzMqGd1R;_qB`Dm>X%CU7;S
z{?OCJk6N9on1Z#%!sXQ|sY0pBL~lwMQ+-wUgHeB48jflKKEo;{Dh(SWet{x&#&?{w
zE>GXuTLL*tCOC)pvFH0n>aKDff7zs5e|SYBaZVFoUTW-Ca_{~#!Z>;*{I_u?c4Ih_
zq_P!vMOTI%ktvJZt&VS$sS+hdFh>T7i>v|qnQLo|hErQzkbF|mC_`#=Jvc@F+LDnL
zumyl1q8VW8%r}Ubne_1--o*w+iBwW*>!e6fs<IDm;lIc@|F2K0t@LYDzbdaJoAr?y
zV*);@Qwza7T89SL&AD=gmA0Gj63fe8FyZ*uy%AW=zI}tHBAF5LX1E*;PEH5^%x`tt
zK<zkT6r17Ub_>}xiXQIoBh@AAevT&7Y<HqP31k7`Lg|OWY*c~8`$WJK*4jRphpBoq
zo#0{vaEohVwrkL{WZ`IM3KTa)*Wd$1g_=$54U^+dh2*EA7-unlNW=lF*CxvMoNj5?
zOIbudmxmHk93lJdJbI*oIMkL6!x6JMz+B(k8{WiTHel`4uhA%DnJX?+!|buPIq^77
zeM`qfjm{g3-l9Cy8&2Xg_f=E>+FW~m@qDfO4(ekH<Nh(N_Z#;a&xQCd$JhNQqwzsP
z%4Za6!3EWSLKVI-uJFG@mGiGq{W7=UJWATyNILpR`KZ0k5uecqnVG8lN#$3o>BzB&
z+8$qP3dPIM*9T>zTq7(%PX9M<=wF!$dT|S_?NQFZxBLX$d<-Dh-}XT|zpNs&NJ!wp
z$G+~{g@j`COIR@hW8hj)Saha+P7wER`EFIl|Bv$Bg$R<k=#t0DByl&F-byXcFK5<D
zp4hOL3~=#_bJp;*;zzJo#D!Tu^}zfZRLL~ko^V*nv?A+j-_Ot>B(lq#x8H5+_Ow`b
zGY8neI8enl?aFjH^9LH}n7uBBY}!cJb+f}9d>P6B8u{89q8afp=nwm|Lp`5x@xQwc
z1eVYfwBPi)`i#0+t1On?s-evHu@^s^IaRKaU1jA;(FI;G*9Qxd!iB17G%wqhaa=ug
ziGWl?hSsC|kK4UR0sMWPzO_~A@^3hw1H#Urxzf=D!4w5~z2z7NQK6nQpSNQap%p!3
zl$-@hn||m$OH?m-C|~9jxiK8wR5I6n9hdqX{4Klmw|wqIEZ^4C^?@kVv|(ivrU6?(
zzmJiD^`=BwtDrX~xb%vHV`^L117A)O#PEOexepng>d2w?+i2ak+X~6yK|E0wgi;nZ
zcDyyTy1EEfg{(_Ez#(WXOM(HV%>>tv=!&Qbsh|e%?G7vB!|7PFQZd<lnviRZy1OHS
zE$VJ-S-y(q3Iu0otJMjp)@Ech>Ay#A_`ZO5)jtm4_aCzZ?)-)ci?8w>KCfo;s~4&b
zjMb`3k@QA+=Di=Enx13go_|x&?RWF7v8(I!+qg<Ewcs6u-a@C%$(H&##_j`w2pV+^
zXO4e1)6k1sUh^L5%zK;qKZ%K*!34LRy{dr_dj^_@Y5)>gLJ4Lja+49Si;a&uLKBU}
zN5W+o^y!v}gCpA;FNa_*A*R_`kmEkvJG*Aa-jOi1^AuDccux0&10Y8YJn;OjBw5Vp
zaYRj+?_q;?pf2q*OkCr{rbc@`WJCK*kbJg^)U1=JfE}79p6}~YFd8h(IZ#9h3D9DS
zO_ZK}MoYhTw_J7Wk=t~1#L<>EJ+-2U*sM4+@O`nF2Met~E!|3w(U4|?9mVt|olkv8
zP~SFyM#zIwxfVS(=#gspG~nMwQ&1+9vEhFfP3?b*CR=F_kU6LyYV3O^|M8|Gt*h|n
zj8(9C(@Z&7we_N<><Z!2)p_yjyf&Si2|aPcQSbineDP8L-kZV}`ij=8JQPl=zg9H<
zG+4JS(l+UB0mTtj`&H?80aZ;!8LhtBG1vBrpln|@!F#!SiNb=3AD51(sq=1;9D#Sh
zl)TYZC(T<9S`F5LCtt0uH2ZjKfSnB0IXG-BO9ZoNQ&Ihzh3i;9t>0R57_5&FVw{b`
zug_ClNA*C`L|c^mO^uhPT2$G}=lSDfFzc6uS2hHKjAddX*s+aCpCZVqW@Qv$7OqZa
z+Fup=wu>XvFL&-h6>1XhSb!^-_;RBnF2kd_&0(V{&$Maj-GRBs4`UG)u3A;Y>%9~U
zOu*OY4(bT7g~4p_7r5Rl1%o*KGpcS6V9c6$?DsyY@A`{0hlVOZy(EM|`S?qO58``1
z|Bu7bPH#gBEL@&u>t4UrDauYRKFRp{mZx+95~IIJJx^J&U9f)n9a+N7Mu|<gg`_aJ
zr(ak6^XtnlzXgKyS!P}z0ju4><ifFsg)kd@lw0pt1)}RD?Y8ZtM)Y9P1?-VlfT)rE
zuS}F@wTt4hG5%BEDCBQ_BPv=nioj|`4C$}33=b*9urllOngHD*)Ed|0RhVxNP)&Xq
z310S$p>L)y`=L2w^4_1F<wnH-+=YR8u%g))iW1!FfA_2A*sZ=%?UE)erZ4KUJ>F+u
z$n8w#k#mesO-DbWCk@xhC+H^3B2C;WLjU>)46;qv|2CTLAZGnBqrMmqEm$a7%s+HY
zh<EEZLVHSg-v3|aB=9Ie^W>qo+X<R&f-`oX$UA9v?4M0%TV-<60!yKG$2)Wtk_Fmt
z;rVh|>Q~~DhhMEskMt)<V%E<;LkJwB4jnR5(Z5jt<E?SHty1U68VCv|fA^(J5#cb9
zP}ymC5w4<ahN~Y~9n|iRn?gbO3T%Qm+1JQ&9>Ponh=SbtOB6)?#x2o(iMER6rfy3C
zAP~j+4U`yPkB6+`ddvo34oJHmiEVa8@7mGDCLZlq`tHXp`#W9@F*wOzFKeq(l_yf=
z<2Ru^e9Rc20WuRFc&el>(9Z&Eb}<(oAht(WmQ+$~2w%T5zqyTO;|(we?QK+Oa7Dz%
zroQXf#Re4(5lGGvv5BlD$2IIFXIgB=8}2Y>UhbfT+%~C|t>@d?YN<k;S{XFLD8I)-
zkEDViFaKxcy8jiq8w)&*-fdOesXNbwPTFpK=rcW!uKD-NtI2;f*OCRi3?Cu)bsi%z
zop4<rZE${SuO@nt@oF5Ov1F^@zasU+EzA4^!WHWWkj7@EFf3IhuW1SSK_KY^9rOS>
zdq#mf2oxeV??iY!&ro3&BykDn-S&#^&5Fju8co~8-7yZa%Rtw5OgUPVE)&ZGS5xMg
zkyYuXurhjgcmP5+>vE{_yr57-{5ffSWulPc?h_al8SI?QBH$6L`9-m^0tcSh+X4M-
zY{@ljZjmj8<R<UpNS-a6eiGfVHB$P2$q#Ho6Y2^u2`@IGN7ybbZ3!_J7U3d3n@?`n
zuGd3(MzFtM?r$!GXdaHk|6n7#hnso-6B~&n`-6?_SvmCh+69wq>k0}Ay;$2Z-7O+E
z9ZVQ~xN+7LOs-|b<FYr+vV$onXnMI>wwJhqMfb^Zrs`BJJk(=-+Pse1$?qK~HJlzQ
z$iigw4_0guC(!E`b9&Q1ye+NZoi^T*AG&3kNG+pHTNqWK3>zTEI7tDzPC<ZGr-RaZ
zw>aym+g39ERII#0Lrnd(x!@KHrg1|xE(J<1uOB?n*rMzFS;kZG>eN{h$iwTh=5^K^
zH{YWiRDhwc=x`2D?<*3ka=-$2xAC5VCqX@p0SOUP99UW$A`{q5MKOWFQTC4+rOwIk
zMh5x5g>3O_4U}dzNNE6qI8txLgVl@}(~wL3^AEot06f{FflPn#zxK#;faa9t<+4kn
z<8s2vc-76HQoswxXXzi;^G_+T{8uTkH#s7xUdV6K$JkdMsucaoi0ay%N_b`J#iJQL
zzxXPbN(kV{H8;t>+gpl4ZIk|vrRpY{0{7-YzTLE#?d2q*u$Ci)!x-r}*U7nqc=ulc
zOK|(N(g%u$s{+s-zP@PWZQ0sRRn@DG4FRnTF&W<hPeG3+`j}NvB-ir!XjgV{*&G7Q
zV?qEY0x9o~WIv^ytfuoaX&2E=fN30?=(0&F`skqlReu{l2{0oLu-a$b%nkB8lQq5u
z(3DI?@3S$<Nn_UmrfnIx>8sRWlsgwkHedT+wdZ94M5=b?RQEFWng$`$eKM>$K;tuk
z$f<_w29XbQM1r0d%~xMZds%_4T&r96weux-l-heim4@0aZbqL;QAt04MnxsAu6lC+
zXT5y0+nWdPNLJ9<^3i0PZ_d6fpWXKoZU32R=yv++!WU92c(^`%D1n>q%<!U9=$#Wu
z-uZ!*8YI}g99JUu(=<!Q%3B%4XdecnT(zElQ1%WPme48Sz`LSp?zbtc%rQ0zSJ=dT
zg9aGDwgc{t|Mw%impS`1S1p_K=eX-`U9+pPRbGn4blH0m<oE^Z{YQ+(BO``|FAe8?
zldlANxJIQcNjWa#Mtq)-1U$efH*wHghiK7W9dgg_`a3F%BZvRv-)<)^f}*8EH3Lkv
zG)inAWzR~t^Gnth=X8+z-(~R)_=vND$v;B{k=<6u6M4%XQ5`Np6#7~%nh-y27n#e#
zxW6L4iO(e>w_}=q&PC}q^&H=OpwKm^PhB#Ue|TEXXQle_vv`^a;;pr4RXZ(jU!QVi
zvAP$RgFLLocgHd6J=V8!>vcWPY-`J8#CPol&{M_SzKb2AXh!iz_`<Eb-_~`CKa<8|
zX@^hf><$xOl2>8|XEtNGIUPQHLNJxzr$z@Ew$Xb-)l2oGk20rqT3KzksU%ECuJz~+
z)$$Ow5Isu3I)q~kjn%g5&j~NVHkJR^35WiD!X2)L<~;ho1yg2m*sFdG+F5>{t=Nr=
zuF|ENg)i&;5GSP~w%t`{=hsvC#*mhpsfrN=yBCFq!gCB>reBTeNfz5!mH;z8!2bL9
z4=CH6CA3r}v^T^B6)(_e)Uv6m=Z?m@wf!;MQy4S|wU<uEmQda@GIgpx=}DPn|BCZz
zESxT=(l%cb{VZ1hxVrGzDpc$fYe@&4^hfL!ss0b24ocY%7t%f%j2i5(ePtjYy{<Rz
z;e<^;Zk7{S(G?WidTxAyQ*sbSq0zc&@a#IU?nzmMZZpWl+5cWlosFkAdbS9!6E^LX
zo^X;h1YRUV=M2shmmx<_+s=DJ*D@VS=+U1Qr<=kuhwHpu1Q8jjx9Ty^3w4@c*(93P
zNmCRkiO%pI*1YL8{q7biJnepKD`C3%RFilNmZX+lT&zAev-fWw|L;>6k%j(}2b_!-
zYW00aj>`@YxcXR0IU;JcGrgQzvERTQZlE~J=8S+Vjb~pd{*_Z}tDkGv16eKX8M7Zx
z;1+F;4#T1lb=TMbb}4SJ$URIj$JjH1if{otVV1F7(i6zNI_jJ-I)Hf+5DzK@K?3DO
z)>j@QqOhdx5olwAR@_E%RJ~%tW9A%cpT-B+GmxaIn90M&)!oGq-|n&SS|(mJDG}Sd
zl}=-&tt;al@g^C1c2vF9PNfX3n?ilYkRu_vPsJfoqTWM1tar8Q+z+at4oR6IcjoX`
zU7RB(s4mfHS_jKr2aQ2kXr93aUk4u0bP_s^Ju`NF9-R(HvIa32Z*7u?w{62rCrjBy
zdf^@^pVZ|$nch0(+o|r@NL^~@*R>vKWZf4V=pk3z>%tZ~y}%X%htY7Kk9`F^dy&lN
zU+Mq(JOEeN6l=4kh}8O>*Q+5n-SMXKdKnb~uM>N(^Vo}Bz|LE-D1Xv%Y^-6{rc&i>
zo$IDLt1sfqI^=E`R4H56ys*;Y{FnE$^_72oanA@)fUlI!fYNgM_F>@L^>Mcr4|ucy
z-bXw75c|m;{0cm#te~!GPxZorrWg56x?RU(lRPhdg*k?BoR)j4lhOF7q|~UogUoIv
z{4S+=Ic?hZL}@FM3aO+3xI*L02@8MF_US1&o}fbtsLzB{4Cm%s5~!fm5&baJh$v8}
z#1;kh_pVWf&g%G+lvZDo<xvK$Z$HoSjwsNUlv*R5BJ3jB>1L!Ql(CyZ?dVOR^Mr2_
zP36qc2x&2F6=$_#jd5jOPGYk36thDji1%Qm)=Xn~fN61eLGSgSaIKCWqx|`gPYv8%
zXyo7FnrHXDrrx3HN0e@y>7uu3me1u&kr|Vg$i}LIB%8`&sk6!e5ihs%EIh$@LKNV%
z_LBqt<NCMf%C%z)g_G<J7C*<w7GmjCV;;+Qyj((QN6l8~#$H0B#5Mp4yR2~Z6U;!x
zJw?$bnUdY{YVzV$U4Kxj==zis(^kJdcPZ1EEy8RJaDUB^fXdig!$~2%oK3Zkw5y;y
zdh29Uc0Sxwt!S>R{YhSaD1!;DhRQ+oTM9)*&(om$C|Gxb^Fgt&M`cIDVVZa%E^ird
zoO@GCMIa?%yyXL3a{b}a_7VGR$!68}lcTp+lSzz(2)B2cxiH07>b0h{=|}b-@2pmy
zmxNT9TJK%r6#75Jr}VMjLQAE&X7V{p3~lc%wYR6AbE&g}P}!4yP%Ekv&xHRY!yvi-
zdR?!J9h=YGrb|ZRbaP)z-^}<@okWfx9a4qu7bdlIY=rH;mU+BjJnL#aS^ivU<;*x7
z-aclfJ&;hkhfKVJ#77Sr{9jW1ogxDUWq|Wd4BE*Z)*HbRqLV&1w>DVrPNViY<;%r!
zZvACNKnQ9@L2>jCjSMw1z?gj-;^%MVu_^xyyK8su?molV0PKF;G{=KLLi3epS)V@Q
zz-<P(j9tnwrR8QG4MZCpj8mnVB_!)ibwyXNBl#aCP|};D>5E4V9$aR>s?J5f7cEl0
zF!yY3wN#aKl*&Ip=x#2)d0$emd9t6c&&U?>%9{EpT}MTFNqA-xl3CJXc1$oi<9zLj
z)TMPvY4vs~2V}9Au7$O{(zS#a0^bRlzE5QRlTpHrTRfTOoj;_`1w|F#$G>$~N#3ON
zrT%<C8a<U=#FcAVhWEzfM7`1YwDPtsy@v1Xezky=ZQY|t+PVC&_`Uhc)<*42pP^PH
zGEFI@$Sq^8VMO{tt?=|uTYB;O`X0(k%eIIN6JuY2$xN>Ua$3Rj4fJWpsShCz^%Eba
z>o49ltbKKecpxPmg{<^B=|~n?t4_8*cf8bhy?$zc>BQw)Gt%&FOv|7?yxx9l;e&dX
zX;&Mq<u5>cq(x{gGBBk0)AITbS=~tKWmXH~Xh+0h+wpxd97+D!4SUCscc?MDBcdwn
zMsi;XJK1dY0o>lfu0?K$L~P>$wbVh|Nr!$`692ENz9g;Er0cEcokF8)@uzK7#9hAJ
zIehHCQ^MLcUxwar$2ywES4!V}3H7~i-}*Amq1F`XR9<~`e<vE5dUdFNeJ1LsYhT@i
zhxcIq1*%=Gi=7?3Dq!DBh^2YD@A@<uF?Ta~?elBmQ2hxw$;{93o(Xnl=bymMPXF_q
z4q}^-XjS_9%sX9tD|Z9D_l&AkG6ZIg=nix`>@q3n&St84XPX*Y+!k?WL$%@OL-xC<
z|18}Eh+d(nT4py{J8KD|;A@LU;y5T-d-w6>;xQsSaiL=!5B&X@B)UoWG2Uw8c15iE
zt9f*5XDdYa_v(0hBGGF!SjwXfcgD`Y&TW#Fo*xu)Fg(7ZDuq-}a%eGbn#-z7N_o0;
zkJn}Kr(0EsUXt@k+skA3gp)f!)5ueXg|+C{i>1vSB3$!*hl$Pl3r1>fE)$H!K!$<Q
z41p_K#Fy0Z(bb?A5&ddebH}-NC|XX2Qlt=^*ygMwqdKi{AuXRXCG|?0wRpn|W=v1R
zWtTSNWi(b{?W3GqN8O)-M~T`DC|)l=ChygKQPT~$s$5;9<4p`{A1`F28xTAkx*0Q6
zhiei_Kah6TI2U)_fhRu5GA~c&R{4-3b)o)wHv)39m%K<>Hqp@a)NLQXP~iZ|4C9P1
zoU?%$8w7gCos-kH4QTD?Mz5X<W8njAH}$sqq0wej2GTWHW+9zuw6E{Gi<ei8QWwaf
z_n+WM(l1gA(*};m+>4zU3cJ4VMpCZ5*0t=t7cY2gUyP(#oKXn*gIw1b3X@)a+SqDk
zTgOc$^Tg=s&T^IK)3>SrLGjFqVcx6K&)$cRkw3b1=-tp9_6EHMO8B9tAvw0rq3Ec_
zuHV|vh96OB!>7W9A2quCYH-z?dS6^aG)r)H!mnD2t3urG{0V{pr@x*(O7BB7nM_@q
zsh<rC?+iDpb5f+d8hp-XF;A|>DpAJMqH_(_VePI~4Yhj@9B4+`RBtAlt`2w3J~aVc
zrT-{-iimD&b!L7oo4DJQEa87&T<H1%_u}y(drWT%Mx#F;=@NQ2Z1_%Wn<&|q!Sx^u
zZS@*6N2K=(Lw+mGkoH+o2oV?KltQ$c^B$^k<PTZM6bv_jD7?r@h1Xm28q>|?faOhU
z(5GFbb-8H@ws<+0W|Av3&`e&e?)_y*yfGxA8Wl$z+z~2Vs2!hS#B{%xH-?I3)8<T$
z)LxIYwS+H&0q;<V!<luY5|)2HY#IG!8S|KU`p%7zwx0UV)b~ldj5Uoi@9ObwSc5%a
zH=7P6^)DL8CFB8W!}zS+%9a`;>$&V=!`DlR)A=%_UfWlNyB#x(MP*O@&YJr&rn;uT
zSLug#tr-_m88<f?yHVSv645bydQ&AOviW|v@(zaU&QyKrOR+w^&<P!)?`4;li?2`Y
zM6W(fIPKv}poFat)?UobWcg5ToR34&wVaoa3qx=A(=O~N+cdfqU~Sx#wFDXSFVKQ)
zG<|C)qCUczCh~z7Z++gu0+X>TEJEzer&s0TdY*WnER`+UhQ*cj#=v{qbv9k58ZE_a
zHJq)TjP^#GWvuyX8`Etf@r`9ASS?&^J0Zw^FLM3*=fFpPlo5F!wOY%rUzjLj>uu84
z_R`VRR?6H{b8{h!`f9J@pprv=djneETj7go2h+!*e}&S!zbcZ~DRDNyiZbDhh;TK#
zYO3xP20oxtquJFa&C&BcOfaPS<w8}G)GG%LJF@en5sF5P<Be;H^`KF|S|2qABhj(?
zzZ;XkD|2*apg+Musll0e2E0!<wBGL46S6^+9jBKhND!8kNGm*BLC*WJ57i-in9(`^
z@JHgcPzffIsP#3u%7qAGCw`!W`(dfZQ%)&mzE7J^Si%kKEh#)NVUZHH8I*}8{6uXB
zImKT#Xz@-E{FvM?AF+K`8uEh#{JdgGOh&KDf4Wz=FY<MhuFKX6I)Ud|&Euo=-A+{X
z9is3oZ{a>;!o*CmPMvFw*h^@1*4qcjdr8fky2`fNQfC5IWe_}bDNi|HJ=7MtdN{}a
zuBQ0<=5uL0K{}d{``V)4Hnc^-$|K~ztCiFXJf9y?P*nw!A5+PVf_&1D`QB`B^qgRH
zbIQnQ>o(eI2}AG$lD=K%52_ggJhhY(qH7;py2yoSdxh4dPm!heYp2q^9zmhUdvg+e
z5i^f-{4XC~J*#NllO)v$*@=vrzu9$1CxE2yzWy%EVab{ry?C`y(Sk@4T6h;`EE(hW
z(r;}pq6{%*wVeN6a6MC~i(JNwLGqNWV7}t>jFH!MMwzdY|C!AsmDknT8$tb?j?a~T
zWRFQ|$Z4}kO!}oR=QWfJs~9i)ahIFGSBHaTaAX4Xa4bWetc;T_xMapSC7j+fg9+7U
zYhOloO|Obk#O32_9MyY0-IbpQQX2+ooGU354?mczA;&o-PLKTHnSR?-)<zB^iF?Re
z!dlN`XS8ae(EUy<p}utAxApel7Jjx}Y0#dGATD1?tVhV&e~X-n@T>?HKg#0xvZ@)8
zo3fZ@w(VK=SC`xK=cOwaRguy>Bij6^V(9hr%a3*JYoGNS`@^8tdSmuY=OqsVgXMoz
z+t09Wdki9~QlZDU9UN^t6$4G*r^5j8#j`HLr4jtai!RI*a+-9IechybQ+Cw#(YZ;c
z`~Sn%TewBpb?w7~C?GA;4FV#<kVA(wA|L{yq<~0w#}FdjAR(>9&_j1gcXxLTAUWg!
z-^G3V{J!^neE$Fr4lefIYpt`El)!pEWod5sN>0}00rkjbTbHu%!y~$_;am;K#WKTU
zd1o5rdH7}9tF*z}W*H@3yDZ1pBo`^qnh(FB9%InTw-NH%D$$}Jk4ibj<!NvQk#FVs
z0z?obIc*qYNSAd9<$ANB<AZLfD0bQ9jF7G7eeazgS%(D-^9l(vD!5Ej1S{oMjRYzX
zk=GV^B>uzk0z1vJhj}o>GiLXf<o*-Cl+%OWk8RQXJF{JKUanC2kcTzbG$@y@Yy5t4
zcCgMYxffjaN;jl6{$z3yVq4islIq$2yK(!6h3@WBrI%hz6vh$k8U0yhot4$q$rK}N
zt2}vaPtYjX4Ub=J#wEsnVmN+!E6R{d@8KpI>d+&5V5{YAf~AO)01Jc96X3{vqTQ#g
zTl^psre<}mv47AMu_H|c{hI8#ru{+{wGZ7IVg_NaVZ+hcss;|yz0q5E-Ckk*qMk|r
z*pIuDWxsUVnU$-+ON{dQEkEaWwKwEL*3`M*xcON1k9n8d^{9{SJI7Oy!5$^?S0xOC
z`M0h@sR@~?M#M7dAKqK;I*U0<%1{esb^0-OX=DYZr74#*@g4W>xczy}q(i(HLIiz2
z53l?l-;n;;JyEe2lvF!KQ+8_{lRZ#rOL$uxzxdtES%hohW^I&{<iDG$Xab<AV!U6E
zF`~hZ+V82~n<26Kh`Esi|AQZA^1UvG6|60+2MgDM%CuCXTM69R0o;)TC58-*Lqg9y
zq~GbkhP1O9d)80THJA<Mi;B4B2I)1*$+D5(1T%*_ny{VZk@-)H4q=g8WFopla^6{i
z`E-xD5EtD&PlP+aQ@+p+$&L>XGc(jr2z!v}cF`R`9VNn-gcYe|PS6EM!Tb_*8oR+r
z;yp52&R(bmtQ{5LnMe&(hpcDfI*zmQm4q_(^lOW5J&_e9_@tJm%Z8otrs<{09)MK&
za2meyR7tQuOyhlk^6#^C)RyOz;%Gj}J2OVc2uo+#$*U$eo!I>VhA=@p*|gzfMTR(V
zp5pFNX3(6#ps%KCwlb#I!IA8owyAcL?eJAXlI5=qe+0SaUc341+G0<mCzr45buFTo
zs%dFly^zSDlMPR=p&2v{+IDw4H)ztS93mU_s$y^wV@x99UYdDU6d_UVzfr8C%pO&w
zG%q-D>%&@Vp3IRAt<Qv7*gWd=W7binPbD-NXsY})aAob@<}ZBu7Fg!dIM}REAl>_!
zQ{%@@AwL9I;P;%F+9NgW*2qAONWq&0EXruh?~}cYmV8n}OrB=9tWtTYnJ=X(dWgMS
z2dhhjoFQx|T#j48$T}6LUyjiCn?!;#w+5Iz$lr4}UA)u#;3h3(INNKp@jN4i7xRLR
zGRFODkoo$zyZin1WIC(=zB6x#{_W)RP$97G|9X1(QO7aw;3FthrLh7ieql=K-wB-V
z<K0_U;3BsV8<%|M!*Fl%<;}?frW6>n)ua_&MC9-Q<3OEKP|9<wU8p(kGFP;5MX)PP
z*RjpES<f{Vi^k)-Y5=uL{bjP#?}bqT(Xp<<pm4X7ZPDBy5E~T~;hXD0U))2I5@PVn
z;#G@8Y&>U6i)o;Edq9at$#SbOgzlLbuZvBOeA*YH?C+7jGT04Bpx3Y9!EN0Npb|k4
zwI*xwL}|5fBtacX8QSEAqOejtc%h<(QcfGDkc~V%TNz&3A%{mm?c34dfAAmB^TkX+
z^}N&UG>}ZAAH$@W7ugcRR++F@e>?zoIrx|)V85*QhCRmwM&<UQZeq`J#5!g-I)FXA
zNUNa4Z?AU2YNA|!ls}Er++XBo!0>d<U77fG#dcoO%&GYvzNNFUSaaj$A*+_>cOr}1
zrhVg-_}ys8kIT+ovE6y=#fqfVZ9e5xu;T30vKEyU)$9xNFu1+Bc6!$hXK#lRxw&%v
zRdvG!B8kzU_QiO+o0YxVn9lJz?>py1L8mpah@^*X()Fgr&h^yz>2%;=!Dw{K-nSIx
z%Fkhmvzew%Ghx2g=Oy_^|H%Rf-C@B0+3V__VjVI$LK9}MP){;(F+`Cigg|TQW#8E|
zNP?Y4B*bj`7URnC>@M$FFO+VIxq4dp)T<m9H#YDsbWZ4a>iCL(^`6|q%qU!3@;p`D
zx)qaitHol@p{$tPW~{6@B;J`>X8x@w(J7_(+q&WSbLIT++rU#X<6ANq=+d>X+7-z+
z|M~u`QPUnEZ{%lNOI|GsT`eoxGGZ;P)TZ>h8J=_+7hu#2VNP}aL`znnNl(LEPg#15
zsS=<1yCs)D^2ytp!^YkM(_^OG=c#Y(H&o(@sTMOo819fLkwj58oRKjWRho@wOV@k8
zf{9(+jXoXnLUR8!pef=pLYBGp%u`$6iQ%Kh$2(uUoscNT@iwcO-)#+<zq$0QJ@xs#
z*-%wEkK1)E!eRO9e14&BZQrRjfk}~~lNH#OW@?N>!P|8czU1rM9g{sX1oEH0f1kQc
zuOmD{zx`$*s{#UKUfto_Zyf*Jt>0uWM*sCf1CS&&25`OIgaZuTsoWu7t)Bco50~pa
zZ>s3^-f?CJFpK+oiAU(pq44AK*rJg#91-Y=Un~*J$>O3ds;&7_>!Nz{!1IYk^oEuh
zbV%d6q_Hcy=;*4daq~bFA~#3F$w(eJ6mjx^4vguv7t8t6Ei^aR^~dG;H%+upx%VIO
zo0Xhc$So--2ZFzdJRlMgL}}|F&1lWe#(@f~D@3HhH#hOJpPg?#c&4|VlS_Fr7H=Y=
z>3fowblL;bgJ=FEp>^7evNp=~=@Ki1A@DQm&%wB1geDI!FnO9)?4@Zm@2ZzbtH|k8
zE3V)cEd{5+*A&Nz57IFv=XS{R)rcy#s<{zh<*T94*_?ftmqo0GmDp!-;XSm{o~Y}H
zVs2+4?r6P7>+Z%|vlD0Ab0p2VkVVDQvb~!)23GNKQ>rcwV5&BTM~BIVQ5YHH3Ns{e
zPkqe7+3hNHCw^MmbqJi?${R_m;LIlu=1#h!N{`w|GtUeg_RguoBbckCq|Xmqr}h~R
z*MjfELZQFp#5{Xkb{ww`ZEMwzvkK1W7?ts(Xx5TgX=)@!FHUjb;Nl&CRg@Duwx>Mo
z#Spq$Xkk_=7;|D2Vptl;<@NiziB)9%;IWcoykx{;CwSh?w>k`O@uIJ&X=}nlAz8_O
zG@Z2oK$#ACMTGD_nN1E{6FJU0p3RsKHW_NGjBcielo1+azfL6&MBV&W-C?rrq%cFo
zmzDxy@lw7G*t>x0hB@A~(pq-k!L}Z~y!ZFQNEZ9yk_A2=eQn@jMK29J55gN(cwcW5
z@dn%+OI;Zf-(q)nd&=&9*4nQ(P0l<lI81;HgkBn~Xy`&*g^rKr%$@CMAJ{@vpU0nx
z&CWQzH%q;wX}YS>abIRrNsE5bczL2#m$6TKb<wo?>e^VY<dDT@<40X8#(9%5Vdrh4
zm7&k7!Rjb$GA-{*9-l6C&0(U~s?gVE3d(g09S?U$CYxK1o=}K2AuZ!9j5PcvPgg=F
zg9ndJ7(gYQ8D6Pr?%A+ss5njb@$(Mv3Y3pt-lwiGrgr7bkHH1cc~I>Pn=kG<3sq#T
zo}5(zoiq<L;t#5yM~u=y;)tp^E{YEO8<|#*@Fq0LyCHj0_XQaL1!#ATT}*0~B-RM{
zQFso?S~!mGC+vhs?}qXQ&_q%bg;S|uJ3%;6C%yR{rd0JB=k$x1bWueFSAcp_AZlys
zx!#smvfK7n0|D&J;u5pqneYd3@~lh9COe#%-f;9!NMrvb4O3?z?>+YYNkrGv`lX!C
zG7+iy3~VXkJz}`2^=h~No)ixXzLYCMB8S4k?T5bdP7IBgK^9&s1{gdZk0-+BTrbWJ
zLJa5PwUWw0q!1ON@Eqa`0=Mb!G0KAD-%<nw2+VTW8>Du!aU08x?)z+bG7n+=Oq|}i
zqeEZRZdR=Xv(8A9=zIzTf>RQu?z?UP0gDxdRSSYvOA}hw6~N+xpE>=RXo*p)GUUv4
zwA{A|rmY>=Sj^Q9rI(IyW8MK82hSm|_<piXG(Z=X@2Jf-b0ND#lF?YqN!UpWouR!z
zON^Z@v@qVcRVqJRg<^_LQ|LA6tG0Brt$lF6aO9$Rnn;1Zi6PA5^LZC$XxY7?K+e2A
zmahYP%*=mnw52Q=hRjX<bfs6N*Vm0H)M@2x5!WPNeUl5}4?|~R^Le;32ms`ihB+7O
zIiS7csI=MY?xy!UOofHLBfTFcPjyx?xkyA^^iLGLihnPl4>p{BS{VxhueBQQR()@p
z&d#@r6Twr;5Ln4>UA32AHJ>>zR|qdruY)~_@2`;2wtgyanY>%tL(WR7*T<kNd|q(9
zlL}pNX)n^*EnsywLL&>LjZbrxEN7#};N$yh9?rjvdU4_lDm-(&UQM96rm5$OwjoV(
zN5+B;rXsKn*}H_o5_v+8SqI?eP+gDLG8p`Ofg!6z-1n&)z%!$&rf<Z1YTy(0fQg_b
zh<7r1aZ#y>=WOQ7N0e!QR;%Ek+s;#p*_X1Lfu?uRaf;)Kku4i?${;{-w>jf9@c8!T
zyv(+CAWq2Y@#bz+13e20)AJ3a_rdW_TleW_+H&EW=E#f3X2D^~m>hY7X@pIRNmi>~
zsUIV7$PM<JE`nYkpR{*C(a0G$KPcfmO-z#sFLA#Vd!mdT&DKPM!Zd#DLB|6D=4ZN_
z<?8sc)bNlnWqZ@UMXdUbqzV&qb-cO51e}{QEG|$n3<DL1xl7yYtetGxVht~2FWC?)
z25)jYr;f^f<HvMj^@`~fAHyzyI|nik+c?fX_j&fNN_Rexv8@Wgic1Yd|KM^tc4#NM
zW0*YZH_tp!{tH_HN@V+85e%jgfA9_V87&dg5-0N&%+2F+uPNXm=pMCq<8c6kPkP2L
z^=29&dw!I*o(Sp`G@9ul)(H4V!cFh`&_g<}L=D#*JY~rpRfq>!3|<nNNXgWvPFX}r
zzNhHzeX(`M0FU<DNVWtnvF#i`4rV<2Lb8+&{r93OK1b4<KhaQYOkRXCMWv=H(vD!N
zZEtl$DbIIarcY{rj6|<H;*eEKOXXjW6b*|BqpqX=MA}g^5u9498DN(q+F<9WPn>^w
z9wR1(D<lz-LIRzTS5`6rW*XlsH9>#Y<lo1v$*7Nc-~WRJ6YoiC1EJM>(F*J=!BVa9
znX)v1hX~lvaoD@3WmVIj>h@gr*|X1l^h22fw0Uz#i(3~zeC7ge`>l!zo=64!F78N^
zxAK@gBe#z+o@!&Z2Z6ep34e#Jeepn5Ol!3&ey6A@o%W5EkZ4)9N@;#?&1MAqN>-`v
zs$5P6_^B;s<unWCxdm3;tn$?vp&!Ahbn9zE%8w@H_pFm(HxW5P1$uZ%oH6;d3O#Sx
z;&4C3EvS6ZMD14;xiX$AUWCbd>2Pf)Pa7;pOxzlI%?&s%e@%8OoRNO;H4BTTy7>l4
z<AvN@E7wsf8;{pvP)haK$8<kHpc~~yU1wbnl%Mvp6Hel~K75Q*I+MtX-qJH{+lK(j
z?%VsPc<3u@U0df%&1-?0tgm5>Lg5!By^XV<HF!8Td19c{o&!o)nepnAeFP4)yf%XU
zX<h^E$Kjo@L8Zp6noU!(>E%envr2$%iBvni2v|0Q+Fwm8;%V!LC9M?^*4?bkT4>X{
zXpitYv*CP=xtrQ9k9|kQ;u7skaWsS)j?$`j!#j&YoEEUDo+!zEp?vciDJtT`S=21i
zc=;mIf1oSMpO#~ew8&|toZ2kKZA!ZGRTa_h(CRr~+4NUErWs81G?LC^hGwbNWSU5F
z>Utph;di%&s-s^U34QY#$+A?Ur^W&i?6A9vh(a$=JVwuOim!EV2Fr?53AZO+0xe+-
z9l7YSEwbv6A@SD2<3}<J(;^;~K1p)I6?Z|Tbz{+cRGhCB=KIynn&oEHs`iP4-2u9h
z*Sc8<+V(k~t)tzOoFGD`Y59O;y_J(^7su|mtyK)G+|N$WWCR-i=s0rR{#?P^7akA(
zzJe8ZSJ3Pb76>1Y6zAsJHFq=dplcy9JK=G`33R!)g-;-V(9R@g5v&f<MFF>S$~=+9
z9rv?t)<&q`7ZKbDHVn<QZ_k4^+Tp`AO{wiTI*Sm`sVTlt3+;Fd`!<myc$t*gu(?GH
zR|qc5Qb*%fnS-87d<*a^Q;|wamwkILb5y3-oVy3*S)?@n^qO*nNI#2M%F|YD^N*n*
z`v&oEAp+6+3W`x5inDI=KwMY?L(8(E9dO_049yrc7CiljDpY$4?op|NQ-S#;not|m
zeuyWkqWI5;A$Qzh|4*;LUbZUjr^7r}bJg0+!za7bqfS%$0p|}~Z*Q(G50-url?HjB
z_G^fF8pVrx$Op|)6xByjRrIT<@L6`0yNs*CBbC#A%I)Af(6$)8HH|>=vYX(YSx$Q;
zZ-7Q^y`qZK8P4RtT&)xvn9$L0$oZ0vs=}BiZojuWITf(zWkTUsahhEA4t1d6cFrYb
zRB31TNm1+!>jwX2T!w`m|1y(ak7T+$rZ_O>eHA&8*~Vn=1+u3!s2tR_A3`T7?R$s8
zjxyb#JuMz9eF$C?=Vj)k@;a=c;8?P1OW%=XlIAKa!t;&==sXmQ6QvzI_V<a`JdP9K
zdbim7aLV1<!=l1$Jeawa$0R{$J8<+9zxsxEYF&V;+6^_g$8meHi;+^|@40;C@^y`E
zO7;3>7er0f-o930T0fSKVa#JvTx=NYSM!K!&xg^(&>-;zAKF&lH@)S9IzcDR-klc6
z>L91S?SMAYM&v$GmjFg4aTi;caz@7#nFuIsRp|Z)^e-rOCkw{!Cx5dN6XA6m?}gR2
zd+gVr{y@~N4DfHvuoR_9{6n$295)plKcd31LyL}v#A)TK3b*?^@|w*xlk-^ME1xgH
z(>!Ww&P1ZHqv0W8iyLW4WE$sVg&LjY{yWw1Nf=2deOE3qw8aKK2r{%s6W$I9+js5p
znJ8Kvc?TL>FaP1@GxjT1{{lfrl|tG@RbTYBCTGy}9qY#jra=!`q$WeNJooF#QwL*f
zHe-Naplr{n>m0*dHaS-Cy=<jIchoB>o-%<3>dJ{m&aryN)BxTsXJ8MWG+^<Xp>c%2
zth~uKmnMSAcX4a`RdC3ky-!`N8yR~)MLzeZNqxil?=blMq4MB=>qz)fc}W1EWo*tE
z8heM=-RSEuNUmZr@uMG*pAXSQ-`&FV=A`{=#4kSPijR}>99!){A78U;=5;FJZj&+O
zih+;v8dtX0Y5c!aiB05F+R&4l7jcTNDV;@bv}QZ1q*X*KTylh#1W)R2cZV%0VE@z}
zlU36*mzHG*lxEuxdy>9BFr|ZcxAXVIaJsVnR$H0YVWcP-BvC797LTww*Hqap+E_>$
zEcD+Syb#|eK&&wY?DqKRf*w0e#cDOaBto<?9F2;qig}dldJtt?mO=$`7HP|Im(dch
zuF=Rp=GXkoVDSzXE5x%$NsVU86H<tJRHp#>mtfP8Oh5LGE{2HmyTvq~-)CWZVia7T
zx+Dx=?L<B{p^ZHux2f;H^lf6@d^JDgxEH?qJM__wzH>7IsUxiM78U0-<~RQ^zYkL^
zFrt;zI>Kl8#5^q_sc=HYNPe)M<uSkKX$BqmVBN=54s11A5T@h2g|NMu-EV5o4}FWO
z`0B-lR(#WRkWk7Kh%Az;WuWn_fPbO(3hdOJjoxM}{n%urAqLrNV>I4sebd{BGbzjV
z8h372d^S((w_wp=>$rx7@Z0m5m^$})!;nc$VckUslSN75xJtECcjbDRn&*X*bG=PK
z!}&JW1^n$GtZs+i8~`UynwCdqR~<!UlgpbdPa+&Mevh8D#wsjdGGxlaAlwptBs5-z
z8mTD^kEIRIQ4W9XH^hI|;F4GNl3jdxdKAg0s!q9hJ%$tY;@8eMRsBfw{rVwI&@fl{
zX!XMVfhxv0Gb)63KR0`%fKqL`akBlZSFrnoVH(o55Ztd%>{3ToTS8BMXri4=tv}r%
zFUow*)Yy4=)yA68#r&lsai`EKS^SHQK@aCuGFhj#mt%*Kz!!Yzmpgz*MXZS?d8Tag
zmylK``yLm`sr@NuJjJ9m+w0(<fWTjls<Zoz(oWHs2>zh01jyhE+b?f^04vz>Y|YQ^
zI~OZfYK!5USC?=x`{~-3W{FHX@3~JEO<1NBF$VIV&0ku(pB~1mPJUTE>aZ6a!C6Lt
zP?ggZtjh2fF3UfLU9j{jq@&n(iMut}m<@lI)ro63#Tr-e8ccOvlcwNV8kVp+-l{E}
zugF=X05-|}sdeM`{~Z|-F4T>W{>EVh(av{c{WBX*-~u{2I643go@X_zTZ4u=9x7gD
z{eCWRC_A77O998=(c;DEl5o25ixq4^DYyRZcE3Q}Zf)10-n0^AH=_!q`cZI(s2e|K
zn_Rh!+X>Ig(SwGGqz(mWD!*B@q`+`4u3I5dipLl$o_&x--)PD1I(!s${S^oEGcU;;
zlt3g8SgIFYG>1!M^@+>+lQ&ZY>)ri!V(Y!3oL|!5+?zfJdRK_0k%lt%JDJTG^s4K_
z#>_WK-tXMm%u24U90R#&P3#_r4B?jsZ*o215ilf$zij|e53m16JyaYQX&Xopn$3#W
zHOpcgrt~~tt^_h`I=!S2d$W@_$7#1}TPxD(%QSk>Imc39{j<d{q~Obkt1ltAFKpkw
zensz!OzA}RSzG=ha1^__@7T!J)b~gQavXVcdZ_{1tow9aeuFZ(XFV_ETouzYt|WJh
zf7OvZ1@I-K+}FGPF#D&$g>}|c7(D1Qx4}<`$}Q$b+-U5=UUI$iW6jdNKoYrc`2IR{
z%N@%;6mU;9XRYYh;rkk{YVrtkjSSNr<UsUoH*2rPEpNi$awoG~<@5biJZOi<mDTqB
zBjJk?$n^S*>|P3lrdWTu=X;1XKbz2XmNBi{fsruk^jFXzL6H>s(B=)ll2<0QQ|@+I
zC~X=+GM{G}<k_4t*+HoBM#5~GJ(;>}7DY&kCXn8k3i4rN5v0vGnrXVq-?UQD=p)Yk
zi*}qgQYd`p%0cLMCZ-S&K*&%{s@Sp-q5isuiqo=B{E4cIzrAh0^7gdySah~6ISq0i
zdk3oA5d(#u_i=;DTqN+A5wuAb6{%Dc=1u%J03s^48x-S*oO0V#v}|H<>aDL14~#K+
z#R|-F8a67&K)YBOv3T{fd|g3%S$*RCPA7tJ*83)6FH$u^yqWg{3nn)1kT~d~RPx>4
z@tvvNH(pT@pWO0KX-LO!@#6NU>daHYI(2OhO@kO~-!njAO*9u5qpog{nRckI@uKGi
z#4L5c8sO6}n4En7B`lliW&aVRmUaC&`|@|s*pI(dCQBZDI+9P~>Q9{lU@n8c1;7B;
zaGxtf@`{b8izN^k-6WgimHX^LwR6d!z+$>7UI6a^)~DYyB-eXC487>e03}N$V&;Xm
zdN)KinX%~Z{GhHx3e+rvO?320ABaJs{39kLlv6#t9L6F!;z5GbfrbT$F6Et;Dif6w
z140(OkTJ=T?DB&#|0ZvV!Q?Rc!S@u5X$Gwv@X@)gZy}^|&%L4i9rOcRC~sx)fEF~8
zEQ2{>B5RJmGF=1M_}P-2Jf)@}INm9PYiFnmSuuHhk<*AA>Wy$wN;jp!;PFisXV^0P
z4EslWQ88#eZ4_Stq)_)If-P&CV@%vmw)F2-G$8|2(nKM<7n^CY1HTaQ)?Ds@6OrnV
zH>JtqG12cHF)4i0R%|1Z?vlpSHIXE-G3(vy#4QG(7uiL^eg0oFRc2`fCIAT2h(mF#
zQ=WxJUL=)xqV@Z0S`Youn{rd%Y^R|O^l8CfnAQqfaCQ2DFQL|N&+Xr?7+m<)ab~G?
z^m2_@JDajMAKq&_mZaG%5L9l|=C`lcU#3-AR#d#JBxbWOxR||~Gv0d2GQFX9CTyxK
z{60b0$@R9Z%=}Xz1ejJ@d1d_MwfPoEJogj%f*3FLAXQYhBPdY^%}IDg-+WGJ<-5V1
zH26v(iTqh;EF0PRzk7-Xzaw|Pxj@1VMnTtfq?C=b-N%t;J>-Mqva+cj;xB%U%PgWF
z4cAh@$Tk5~(e>$Aj_0e&i{CpmyFUR3jp>i;?;4cWtguRtlRIslOYDe*@8bZ26*}s9
z<dFKZS?=8aWr(j{Ro!Lmjb^#Qs#dv~736mf|7<@aRL5F!GcuZM9rd9$XT|<<olL}Z
z*=@O`6TITyzN=zZnDt_Ap^n=s87B&Jfz4aIN%~lIpv7J$FTeq73B~=~0Lq5u%31ah
zIW6<y)ze{wX(A|RkU1k3arXIT&HTo{*nKhJuX9%cAU4gphQ$A+SiRGQsEMITMd}-c
zjw`?xFnI_P%`P$$2~5rx%kOAb=p5GbPCaivT3m1b`!9*)!3Xy~K?Cr-L5An3_$mi<
zN>1y|{(%bK)IXGWSj1|GLTKF9JM_R|%PemsZO=0@lqdbE=tTRW{3jS-JHimT2qm`u
zEY09nS%K~!2v&dB8bA8>M<I_f)<)iAX?XT;t41|}3(&=Nx7P+Ky{38I;UY<~C=sT~
z@C?ZWZsIQdk1i<1xZB0~%dkuGd?ry%La-g})_T5xAg1ARR}`ixWTkktoW_H~Z(;yu
z^XS=_&G0`;^-J{W62nfH$BM@r{aDc^xB_eVz><LZ6|vN!y5EBCfX0T^XgG^*$U(ET
zOs9Uoy`-_|%`1|aRB=^e)P5s$Z7V8G=fK=cVb_L+vw3&Zna74YhF_!2L*MNcn~?#M
zRC_<w+O7@+#q1sH<fZ308oJ9M#5`hCF&_GzJ2N=p3)@i-32bUduQ6`=5zgm_(J_G%
ze!E8FY{a)YAT6~L=;A*@vvDP_2x(Ws=J?gFoMo~0oi8h&1|nu}Ac{^|17Y!t`umMH
zDrVuUmaUjvfKI@O`kHrnEn4|evbJ4&zws>Bo_%Ik`OT87ZZY~p;kb3v=~`F*u%~B<
zvbiASz6bKsMTfAKNAy-IEai0Oxi(fggViZO89hWnXJKj_J)&B{Wad;kx{unf;+__j
zGUvNpZ?06j{to`6p&1AHid7rxAU-JaQ>aED7y1H_No~hdo9omSk}BZRfO`gnZt_aM
zRT-?(9q?G7Sx@aAqNi4P;d$hZA%Ylm?cmEK(^F}l!wx?nrmJ*=;y+(LycMc<+E2Ke
z(J_f?`%!?FxaV{ous6S&#(5u&4#0;BUNM{r&-2-@=5ZELvKBr$c2}Jt|GoK#jJo(U
zf6$o2{-^)qUzyW51q#8Z_l$;!k$h)V#jn|#H#z^j=5qh&2D&IcI>pSDXlip-{g|k|
zyIA^`$!X!k`kTEJ4mBWKJH!6%SP-_Mr<;drbV&8X+$N8=NoG;^30_L?^qbndt#ri)
zBV&cqZhdrsm|k*LW_vWR?Q*4)F{6Rnx7}&bO22S*hacfsHg9k(O%9iOe;$z(C@#y6
z%8a`B6uGQF`uzc0(C$z|Ke;1@5Ec872PHeP2X@&*q^;mx<U-AK)tsiiMp7d4lw^GJ
zCMN<v0A^;!cG4paA(v3bhG>d~bt-wT(9Jj;0chMKUhRL-xV%o^-Ju_n!a+=FQuaeH
z(Wj~1cM0|@zXDE91(5p87X{$KfHNrBycE4zHoW6j87EYAQTsV(sSI~u)i3)HLlk(^
zg!`hp^e`sG&II2?P#@Cp^V6Y%Y1e);+4Nzz?b`_>wpsgDkwVM-x%jy)+U~j2v>FX8
zs;X<-%HNN6^h>LnTxYE)h0m}yathjL&Qe|UYDyO`)3kQSLAq=jFJA6gm40?I3yD7X
z?457L6ndqe_jr=(hwyT`y~5P+OWNwp{cSQ#^ip(Y9a=9W=d2r*OSs~9wIhYg)d9G^
zx%Cv=<yL2=ja#4XRZ5oWuJg3B)*a!xlAsC>pSu3W|J!{(Yob7U^l>9i<8`i%*P$h&
z0}H1Q;cNbGG!~}RhWR@Ufv5Vxg!z8Oq@IiH=+vr46}1<zb6P{Z^jV=z*7$(?rsv{l
z11Rr2q$O6j*U6No7jixA>@-aVd8H)?2vgHig?UT>Si6q_6jyum)-t|0-%5n<SnRpB
zJlOI%^mCljE5gM*MXb8tMIi<fp3dQOt7CR4qLN=u9Nlzb`4;ES85?y&9g5`bllv<b
z_P5{Gcn8g!%I!K0CqL{7B2~v%)Qz=WfzaKJv+Q!-xxqi$<fDIIZZ**f-CzHu4z)qc
z<3pwIDh5{6W@+rBC1NO6AXNZa{MCa<8Gj~J9MJQ|%`__V#pBUUU6Y!UYHV&_-DDq*
zP<o}WD~IZ$9c$6zBi)dX$;a}0Z8@SfBbNo3dhNZ$YMe63m@8^D6kl*(!YX9iDXDmg
zd_g;-iNo9zi4~3M!Sr?fB&Ft%sfuie7gxzVMmut91V4pOH1S0~nh&F<hAnGgP;SRE
zRJT)}taT5fycfne37q(qK$-V40!M6|`JJJ~n(WEFp)AEKdHaLHbU3S+z&AW0-W+90
z#j7_Ehja76Hja0DV@lL}L4+oP=&1eTO)V_6D!%{Fu-TWwZKr#)vX2!*h}l-;c>t)k
z<QnMoL)X5W7u9s=x~|8*(%A)sZHg4&3QyU&ZxB;2M25yRbE_CjN%qlWlgiuK)5<}n
zs}Aj^*Ej7-lr|DU^@+Q+3q_SXmv!yi%7mV!qrXMR^Mrh4%ooiw8Z0_1oP$S8n!+n$
z_LJQcW!>hVFpp8X_i~<Y-kwbQa;Yil0e6WCG)Dl-eEwOT<)x3XvIsI+#9@+<@vR-|
zBj)e4+8Pc=KS<h2JP)LW_44@4<~Z_1aiRTlR7IEDkgJPnOisTQfV1qWK4Kv%;#*0~
zP%Elmr_`#Qk5)mz?X73`CKpLMx5wq4(e9mKKgD#zG_OPGH58sw{`{rBd0I9oh7?Q$
zKpt2H{c_j7SGj$7-P<g~MWc43J*2sExZ#3{?Uz29vhz*J?X}XRfl?=``0#=CnifE!
zFVw;!)>2fU!C$gvup$JA0Z0~Y+#afZfB_q!nTqr!dVsoS3hWh_NwBHyH(magboA;2
z1`kxnBebw~<5kHR5V~)vkunN<Z8WMH<(ULf#lKxOb5{XjG>aphv%|k)<{Y(u5!={D
zWz2t>@qEa=y)n36#Fn^ij|=%QNvN`U0pc7Sz4CvQ^Op;0pd`{w-H$cY)F|5b96U_L
z@d5x7T_#>@7pR%Z%MDEw<tK<T@K-iDIV*`9xvgm<N59=08u|!>wU8zkTus?G?b>62
zJ@MiA8_3Dlk@Q92x&oCOoI6|e=Vj=+a)fS}VCd+jzNRRhN=eJgsI=MgT3jmm_Irwn
z&*cJ0QfPXqiU{p`q?qT_Nu-)8XRa6?u80J7qiM2PrpRF*5wr%5uKG@_u-msCJqlED
zpkE}a*XdbosyAk+B>9OM9WC+H(~q!t%;MiJM0c=6r_KpLj7p7qtBa@0jJ5U`8ultC
zb@NaB*aO8!3-q~#y!Npyf<7y>Ij#|kmsbM^7CxD4eeJ$r;b_*n&b`BTF5|wFRP_yf
zsz||8v4gi`=xDyFG`e+lf^#}B<ec}wCtGn8Tsdu)G^@HVW~yzeE(gY$6rieHUmpIZ
z{CMS27E0%Q)ZSj}zF4_AO;h$8a`X4FABAN|!PBEn5J`EwdX4;~qD0^g{2AmV9w1I_
zV>)9+)4h!cRdLJjdQMGRGRoRx%;_YwZ+B#P1A1~=ROQ13WU<KET0SFJitEK;<q^DB
zmP+FnW!2(zltbl!%>3RIH(tNT9W_t~pq|soC{SmcVDz_vw58i%im4x+dxU$)rAtQO
z#2qR7ut**7Z+}Uhu-q#e7E62f#zNS&8&q?UX>I|q4ZL9qfFjXo0vKX*u2xp9cYDgL
z`*tJ0bX*JnKwD<!_W!jIa5GvtyGQf=`7K(U!K#ph6^Xpq2O#;{QI|y5E03*C`|7p(
z$dAq0>Jq|ISEG(9lkvIgmHd~jmnFJ4+d_(oytr+tGhkcfTugMSpWiQX;iTh;KHaJd
z=;jj&n5H{*Y0O^nRh;dmBLDo>hyawLTQ{!p1sb`WK7wK_Wcx}N+`Qnvx?1`zc+$?c
zA?_a&13<89tvKsdBJXtGNDtBfa2e!(xD4MW4afhb{0I0(AGj}YGYYyedB!(uz8NtB
z08T3jIncrYro;VBAx}4{PlS#563jwi<k%9ymI8yp(hJ%VLa27zl0t;ePx8JJx*o=)
ziP*9Mgen!1R8`N=O-hQy4!YCoj~9kK(abnniC|w$z0BfmkB{`i2rD@Wgm3OM8?&`4
zTNYu{9aozjqsrh(*JLUC40nu|5m-KN@?IIj%6yV=F-{UVHk3fwBg<YG=5cN!qMsn5
z)`4N$1~?xpwh8Pq@pO2HdeNz(XS&e0Wp;k0rX>lunw(3fZ!ZBlwI^oP|Cn>tn$T<d
zxO%4;<FySGlfb-1mTo})GI2Nk9b#jOR}b!z38yd(nas1)5pt+td$&l>Q1wylI(2jp
zikY$bLBkc3=jn`7zolhU(|+3e8aZ@R)O8@cKjAY#`}c>>oSOKW_=Zx82^P>fo4qtj
z6X&*tOI@95-S$khck@Ylktg<(nc#dp9zLUaVArcaAcnzIL7Y#-xm(eS*+Hb%6_VCq
z0;t_+!x4t_L=zPNC8Dc(#RJfwNan#=D1&H)41)Xvlu>AQ!zXjJ!QT1G@RY_$Zqb)}
zF;UP2XuO}6h$o0QUb(p-=^-2Y+?F-5g44FfXT`*9NzxR|P@6Bo3sUKGPLutXY1DO6
zLYLfbx@Nc6wie09tHp$;Wr~%3I$`bgx9ZN*S}>1hjvitm3j=iOih%4xwB&$&Pp*MU
zEREN9lkGGM*iXqtU@zL{vFuswPcZ~c;P&$s!1^=Pe7u^kEKWI_&X)R(9CZv1+rM3k
zo$eOW#&Z}b(U>~dW0HA`@4i1{*<NsS!K`g5<{`wDB#G~XcE83q#<B5YMJVZ@7d8Qz
zSKp(VwM<UokhmumMTjd1LpyEnSR_-5U-nWIZ*<wLz}uSzc?BLLZf^KyJ<r(aU0;O&
zg3gHDj^6ztp40X=XU#^*r(3UaHJlnQk+Y}US9%?j)UXQRIq7wpGGxuy5R)Ctn4*H9
zPxneL^uhP<JU4fv5Uhd(y-8mWa#|R6sbB7>`y=BHTedjf1NQ*b1fA1wTa76|7XAE%
zZjh4je%E<@LKEWXY66fq8)<&T<1`(&s+laLteSCE1&p1h$wzLa$E$-}(=bBj_-gXq
zROr=buu^)^ZEzpYF|IBaDYVJi#fFW1GDDupkNi%65M{lwUshm9`@XQ|20p=3fcSlb
z7u(!4wMa33^2e)_4#)pzv{swvKIr)yS`xuuvNwWkqyX5`8GtKw9zQP+{>NQgVC!PS
zEwSnjAs6;8muDWhM|po21Ke1wf~WF;LU0|U_1QTsJl5A^O$MYx&MG2`aq}3dMkAC-
z;h<@L!=Lb~AVT0mJ<av`w&1~!xGwRdQC)Q!5kV;gZyd~Rcb%As#rX?<j^8tMG(1bq
z5EAhzv|<bfH~u~{hPAQ#AC;&LTy4iN6)%QXd$rwF7Sel~V`#wR%s}AGp@(EEmai}m
zoNkK1AQw>11?a!i!y^%s;;q1Qvv4~EG_Y%{P53*0$rGL7CVS`0gI-3~K=b)OjiRhJ
z7xY6fo~`eS8P5{kHqHTp(flt~?f52gE+b0JvxzzrFA}s~74g>|_+W*q+TcfNkfw=k
zc~F2ig#6=lLmc|pWM?xMnWV2L=BrBXgGFe=BO2GAB_XwbOGV`*92@2mD`@Tex<m6U
zkQZHp3=hB4V~_A=UP`Qf_v$Mre0cKw{h7BSUO9lx>O3p>YBfPqcB6-HGBtMT!!dk3
z=F8QBBG|>H?e+9Kl^h01ik&4Us@#>5<z93n)Y#Tf2}*YZpB*Rz{62)LwmmqLb2eEq
zYsI=gpUTFyqj9X|Zji@bmFFu+#CJla6oKom5KF)+nPgw|g1YE)t+2Lzx!UYp_v`m3
zOtP7@4){E7X^Uz4m@ME_$Q_LHu4+NM6kS8k_Lc1O|3?t03+o$wKUIhwB1#lT@}Hfv
zZq{64qAGG8;(wgFZCT4QUMLo__V`F$1wcx*HY+_XUyWT=X>}To;Q@@=cw05@)$exq
z86EtXlC|$U8MY!`<pPCD+gN9~_noF?4gdPZQL+L9bFzDNVa)<O%Qj4O$z}-Gz8k{g
zlg%Tq>$OZ(hnR8SwCxvLH=zy?2nIAZ7K364v(N0OXG{ufe>{#p&;NLmj3Sz`g#KUR
z!GDZJyf3{67f!z0&HMu*jj#C-y_W$j9spv0FtkM6thiJMLBqXg$lG}DR{-92I~~6B
zOU}MSRcsOaqiAJ;p%`>8yW#U_i8%}pbv@oEy(BNs4Lh0KNoD9j7g>{T90rDXQ(cnE
z>`1`yQ&#aDy}@8zR`&$sumF9yvHan9XLy(heXgF8YIxdn&G4ZYQUW75riLdo_fSF!
zri0XlDJHVl7%1GlnZh|6^PtIzguXP8MAed)Dbt54!cJj_FTceE6r%qu@fa^-mogvd
zfAt%aW_sH+xHV0?)LaRc80v~rLQG)tFpP}*%u983>1`SQzn1b3gN0lgb+mw$O7808
z6uQ$s>wOXz%Ry?2U40?g)nIiWAc!`$_7piA^B*aKl#PSSe8ECfW}pF@myCY-Vb!&m
zs6k&xO_dfAq5L9O3uP8<AcmD{QouUOr8Bwe7W-`23)$K^aaEG9uvM*V7LRbQsHiP4
zVk6%yLVLc<Vcb$7d%vZdoBsC6(RAj{qUEj4V>}Jsbu2U{rD`(duhHY=FERUCF&lRy
zv~Jb6Xr*$`E*_GvJ3;&?OtFn;I-mOikTegu6ZBTBhy!jv*iJ@=e$;oT%BY?C)wxTg
zjKlIHnR_{rL8)?CEm`7i{yVlhetg#%P~B^~GRVOMbYe4l_R-86Vv0dtjrda`?z*?k
zY;F5&YrcTRg-yNF7G{qd@-$c?q4)c!c#$A9Et~NY5L)h(y?UC;t-lvBcU3NO(_H4)
z$5^!Ze62YBT%s)lNhxeCr<^<#1Je3k9kl0uZpA3~?m{#f&Eam2?#ax($QEy8Vb{e}
zMeUF5)GY;U2vXS(PD2vKkKtYml>j`fVWTLtS5dK7ZQM=h^;T9+4taARl==#ArI@nQ
z{5iWX{^&?TsNG>(e|rV6P((X3=}bWlZh&V1wi#EH4Pc_DSmqRD<TS63=YiS6qcdWt
z6-%z0?VPqA-yYPVUBLece(0+g*|7qutA04b+3BCt88>14TGvBe2wS0#DF3q0wC!;Q
z|2STriL(7l2BH|<?#HO-Hw6J6JoXPb$YFDlBgGx_*V^BuF?%T!&?0@MpT&2Kn6u*#
zVWC}^#B5d2`G7ut3n1mafG?w|hB7mSjZ%t@PlkAtUW_OltZNw}8AA_MAlrvVds`o3
zI)GEIwDS}OXElCfR6~~US<AFe2MD?NeKs1KxO}oD18^(n?0S7PRJhD1UbtWFRfTz~
z{Lz`nG+s^+`aHDjk#|lt=_dz=XfTCcIEG#e?oCQJ9)SC2rh4|?1(~0F?s8n|@8Ui5
z$*ffv^;ayk?5(W)bj(=jnz3=1`JHpQJej65;<*dF4|!T<FiTV@xr2LW^jsygu^f+B
z^(r7B?Xy<K8aehmhkI#FPb}orRqPdi!@2q_RX=0AMbb+A<eh%yL!6y}nR)u$Oxsoe
zL#!*l)(ZX$Xh0wAV?LP8;#zUCbi9R{|G`FvvuFf!JmA>2^rT}1goE~M!){5){&!n5
z)1Ei`)h<*6viqK*gWa5_DoRK1&3$=Sc-0EyO-#d*cuE23``4X7&r(3N`-{nsHJtcq
zdk5bme(UOsnPqr}QQ`PwwgO##dGwX;AyB}yy>er1b46JZ0CtX86i?6G$uc?|4eH7K
zI1mdvG@bcb#O2NwtPXnykxg;hwH3V9<9*r6N&0~IlSEm8jI-eBa{e3Or?J)xas`z}
zmU-vgg5H50kH)-L&kuyoD{L=EzhnY~!h?0Eq<_(R@%2BBJK<1~vcCZQG3+}4zv90D
ze10E)Jm@hdPo(b2yqNaA#B(%#QrCy|_bwi@R|IKz<GFd5tkE;KB!FtWp;P5}@GZcC
z`KgnM4s<z|U9xhZ|9X_f--hmeh(}HaKa#-XP;w~Cl_xIsy&diLdnfou60j-60|@ph
z!jLCPV+W6OPZ&&lgEKBIw7sKz{hLlgyXm|GNeHBEQK`hjS~>2gaefhVRYOlXQh-+E
z<BHDs9@&$J!ttRM(P<<YPEs{wtDUv#dXB}M*~<(fp63iR%-|wBI3<vobfXR=`v^Ie
z-|Zqt4%+`hm0FJ<&XAoYR2DAjlIQuycDL^3bh}6oJP33Uyqnufk;RC=zLMJge}}E{
zCH?cq{}n8OBAfn-0RN@GG$0;(MI&7kawlAA)MsFjB=qs$zhVrP&eZQayP;1N_x2%a
zRzZ58;C70G&+kvQ(?zCIuF%VMCi{o+cbs4dp5v^kKsKg=prVdydX1%kbwAK-872%Q
zl^DMk#-VWhrniHUN2&~&&T0+t7cGQ&oD=0mH1<i182CwNr(@x}S=?i|*TQi*!m1>C
zz8cU35P+8`mCe!N)GEIv+*Pa$5Z>jYK*UK+Ia8ruGLHdW>%p6Z)c}rn)DC?FZgx5B
zBFAGz{1+cp7s^Ei$0xC#P&o%Xyf+NveHaF89mc-Zd^@~G_Ctf{Dfyk_j9lcwev0@i
z?ElD^vG@bpgq3gp4cfrN@096Rgqr7<d1OAHZ;Mx!f7tSfQr-j7dbJ-o96WeSA^P|n
z6+^@gv8Gz27D1Kd=JLR=V<Zcal_C3$nXh;K7n2yg)X9vtkANRLSNZ0>HipOX1<lq|
zQ)-8`mpQ(Q2z3poc2lL--~IYYST&!zk}|vqW=aw#HkA_1O(`)rX+Dsg@_5DbGC-_3
znoy`DR9(T|j^&+HTzc@&#I|R64F)6H_$E1$*b;-<wp(4w*k2SgDaNy8*D|#GN(dy=
zCh$h9`Rq)Ux5xt~PGNHCxi5~e)9FJJr84da(wNpouS5Z#4!!?FU^xAUdBELnC(bug
zU>hAstS~X^PpZOByLs`Mu;bQ(u;bd|F70UA>%cZ5<nMFHcW3IKPnDaVR0Ysl?PZ>S
zQj9+R95p;|q)`$V9YhhK_wq1V;eO^XV9mZv(xn8epNf?<^axU6_W-~u^3_90*xZ^Q
zmJi1}5xXCtZw!UGoDBvo9KDR5U*TCdGcj=JRjUlPpurYFe@VQSoOVH^B-SUD&<{EH
z#!!KA{+lv69){`bgbwt3If3HzGBr<>DV<QV96!nyv-nr^=s9g{`@W@iXrl`yFgRN6
zmdDZbWJjttzyUq-QBY26@?{`Aqcxo8QK4mZ%eGI`43U!ienS--u!oG`tDe{O_(L4>
zC!Os`?{8&*56y_hADheoC^r2Yu>$`#O|QS1zq0}qVg;M)`Tq@jn9-xUf#7I~!LDuq
z5>tf$Uj_iOx$=)KAId(@<PVA&sWK@)H*wDT`?FHrS&IdCS@Z1f`cA*moFTx16ppfZ
zINFLQzC_S`-&@W1W2X!uGYXYwum<jS30<Fb_%9M*Xu4?o{swYcE^0VE;<olvUqc7+
zlE|<)h>`*LFXVy3rQR614rL+n3X?ALWb5G>F*qqwZsS9upFsA5Evyn<tt;i2C_j-?
z<)$7cTg9Wr!?Xb=tNNsj{aH8)%(Mwm1aG$NfvWDfER^3YBOs@RUq=h@6)PF2>D>UG
z7Cy{TUzTUmEmM_}6XWJ$YRUK6J9Be<hZMjex&YMa&ns;~V4J08j5nNOf{yS{LcgmG
z<61Cz$exJiprVD()j}&w=325}r&(M-Jj?$=<Rbr<DCqq5?z%qzjmc*<GFCZdRRv56
zKqN)O;RyG#TB05?d<XZh$+dV3AL!kGw$_$DSN})Em)0toL{2V@63Dee5|~fxdCxv<
znLG2Q?W3<r3u#^TeSOl+lk=hC?r@Oc5do;^J_6q&aH5qTp~1eJmrs5kt-K=<k>-M}
zPioIXb2#-hPadZu(4jyBQU1Aob22{7&ZPf5RtxQ84dGLtnRFmeU#A?b0MfIJ;~CL6
zj$s+<E0N?_evnP6PkLQE97tg5=a}P3+7a+YB&>aa@|WJBoA_3D$VX;A5Zd}1MYr9y
z<<{hE#{m6M6;BTWxOu}`TTy;up!Avo=nFw(viK++kh^1#+TVOn+IA#228hy<9Lo4V
zlD2Hn%pTtzt7>S)@W!m>4p^c6>+RKI+UbH8Y${=xAq;O>tK7nJ$Ew_KJYP?226_wb
zPlX))2xk5L{qGWEa(D50zN)9D1HB}_&*x7-_5z_E3iUkznG@0WS(Nm}qV@E~-~&$I
z+ax*GH3^U%c4J=D(UUb<tMSg%i8Qi}wdkCO_Q)lsz%$yUjUHGcFwmd$5h=6psT}U=
zU8yJAaomYP=qU~h$b>{1w-QJgHKjP^k4Q58qGLF*i0*mc&q0+>%I?&-XSPZoU#wE7
zZg{ck8%Ey7Ny3pwsHRe!EBdH?*xd3wG62c)-A~|W0|zu5ZaAv#{!SQ3O5+pxf*Tc9
z5%e_Ua;m#i7-3DhVL1s@7xa#BDhhuuDtVaA5uwSA;NZM7{I8xGIAd>Y@8sDA#TFAk
zV=V!Q=R;|gE@YtN=B(3VAYHfNva)}`ecB*oT~yQUqK0{I5_9#{`FY#lych1Blb7n0
zNWJE+E62PdCz=TzlEU>t_imQ=R=r2_lAgl;@afWnrGPNLn62;J#2|X?)nxLm5<=+Q
z)~6UHBxb*_!UpU;Ky!&1Y_)&DC;386!TU`^U`t)^aBE4vSgDW~d-Z*dI-Z_G;wp&7
zjr`}_n1n24<2v!<C#;&nuJ6;q3WE{^mj1T20v)E<o&aO9J|Vacx`&1;BEi1BMuD@8
zE?|-etId;DvcP}8br>72TXLT7Nh;G|W?&ea5+KhW^d|()QBqt}?;>X~A$e`(5%lz5
zm1&|6O7KdiDzlhRHb0uk%wJysI9%yWcUd06u9{RrY}WIW)6bZ+hPJu#VKu3ax}NJ>
zd@BLgdl}Apu!ILL#D6^r&&Yu<_yk8kmidkvzec5Pl^n(7S-}ltz87&%pI!H!_Pvvl
z;n61){$$_%IU85-fCj^@{Tb8+V@>klr5cu($8dg;Tl+)Rf*{aO?a&G<i#%o6<%;1F
zE0xF>8eaVIu>@TNQT&9m{Zg;64oNLnNOYV42j-B(w-9O0I8mLaOZR9gV)RHq8=|}_
z=r~dh)x}QC*MJ;x#_1lOY;Ut4jS3zz^WnV|6v4L$?fwxEhO0v5R`}9zJxX_t(vEJ6
zC?fpwV<!K2n=I!H9`lfRI@uc9f4&*;hXtg|{smFoAr)g!@2()Ha{dmv`ScNMA#qID
zh3U73>zU&RgjJSJ1%(a2J`aP<CtkY)QF@OqYexS1@Wdj(d+^~TLBpZlOfH>_;1t8h
zpYYSu6@ej!6*QYKA!zL0<bsEUp91n}sYEDI%_0?qA5IT<cf3a&N&*_?F-i)7r{snj
z5R1@dFg|F&*ABn+J-B@Z>IfA38^BE>@>84KT%>PW?9jB}G@zmH=g+&0y!WBtWUJR$
zlLnajp~aX45^ue`BpKehZM3mGq!5(C^Aj-aSEryEmx`ZQ)2Cl+mj!p6M=O*}CG^T7
z4izv#iRg}}k+Juw4mI=P`QaPqnTSIT$hgELN4b-y->4J?zs*PQpZwcn-IzS>K&-}J
z#o$h2!TXPUzHe01-4~eiu9P^Z5A+wOI`^ACaM<>~Q<!4XvPc7j2L}Uxr(yy2V9mSl
zUU9A=g}6gMS$DTPhpJnG01fzjQ{eNyi^eno;d|$gFzQjr!Q^hXWS<y_Z&2LUe~_p!
zVwag`w7H8j({#vbrp;l9h6brZHdda0gtEXqhD)fC_&OAfVs_oKh8Dppx(4caVjYRA
zt)wct4xt1V+x*V~Gh<78FmaCXOu3!8s%ZoatSD@FFB<n}AUKpti#$T-<^sVBl?{v$
z6R}6Dv&&<5gRcY}L>fsBYCFMWBoPN0xkVj$u+(!83j34j^2UkW+VhVv63tHW^cpW9
z4RedQhR1BIt8&&#$b86qHZwTpaJ6$bl1|d%)Z(K~@ovM}LS@@y1u19{VTyR!$MU)V
z^IcrR=nDZ#^hSH8aN>9^%v|T)dq)2o9<%C(s||$*qc196yK7mmvT@BW$?jKu=k^-h
z_>Uv){|}7v$&&=a@zet#(&~5FqeV8rqxfy;JX}wT>>L%yi+1qf#Jv@F3S?fv!G{w2
zlXaq`90kI}BdDVQ>gVJD>r;k|CQaSM4@6eQFv~l{y0Wrx^d;vzh(WG$LG_cVAwuN{
zYARcvJ!b69@2t39?Y($fDiVhV(iB6VA28X6sRy{i0m-5vRW6PFS5iCEV+(Szbq!qc
z_@#`1bQd=?uZ}gt7v5wHrK3=%hMYiY*frN@Jng|YQu$-<I1o5YU9WLuH2>knr%BH)
zA6+2RK#xd+|7GelpwcREJ>DpQN~{IYcosv`JZEl@_P=1}Km~Zm1KIgc768Uv>feO|
zcqW%Xp?A&C#<H`f6{FUw^-kxf7-q$Ck^td%9EY!T>ZV`iI}KWZgpYF+za!1TXa76V
z@$S)3`(K^qF4ZZoVELlNYMo~zGMh6y=ou>Lp2#QSAY}Uwxy>UD>ghm6JW5pZi#8EC
z2!O^PsbCTCp@@ykrVUi!dGN;JU8vEpqe;B@naWvnsCYp$HQXIBXNtQ~x|&hpug|nq
zUiA9I1PW#_(fZKmBN8Rg3yu!lXIp4!6chg+U2hpy<=S?Q5>pB3knUy@(jC$uDH2MT
zv`Y5`q#Goqk<LkjNOyOGBHc9!ndH9ldB11B`&(=ObRCB*x$f&aW1Qn0Bi&HN)ziP5
z_=luvWOm^uzDlLF%2i3Lv*(N9sr2DE-ZsiFdpyZv!XZ@C^12bXQ~K6o<vL!PP?s<A
zRj?kpQmV!3pG_`$K%g9?9t&)dArW>i61zF5={0(DhR-xUj7>84uR&7~DgW_i<#$<|
z7?vmNF7hwhr43^uValYwd6eT=I;@5Qy0{N5g9H1VVrJ_Q%G}~V{|F)KL64aqy#ty(
z`8dc#;|0n|O48=g^9Vqc8MQ@$^*l6`vl!d#lsK<1Io;+HicIIrK7DVDO}UWR0Irrj
zFsGA^b;}M(#0s^Kx`hz0wiuvRxm&ELiRJ0tKceg8K=+w-YDiwFD*t<^2E}Gyp}y3b
zsq8bx<mV%uyi5}7&NVxeJ<rI`VHWW_aA$v%%vlWx+$Oq>l2yzpkgK)U`(?<#2u{74
z{*pBtTOs{#dQ%O|0865&0MG=W6Y+RIn83pCBiwe=qX=kjRF`IpKioP!(h9fJ#?g4s
zf_gFkaU%Z_3@!%BY?Wf;cSTdC5$x~v4l6_Wtor+SSEnXc3P?6mtTly-MZ8nq6h@b~
zohCg>X*yI;eQdjIe#~11zy9&PBnck{h*?2xpOyXtVd!;9um3W1NA=#umOvOqeiT^x
z-NbnMtg;X?Y>j#kATLgv37jq7#HG>B)31Hpv90f3jF|SAQKLwS3SX3PN_cOiU0&6w
z^=M@2?gAb9e1EZAMQw#6eoQ~7z|dVn%=NVP9#<GxX%a=*@cC2b-L97*My$JAm4)|6
ziNyY?R8ak$FdfY6gj*>?nbSoPr*g#hi}6YK*|;MH<y>YI`~y^ncWysl6maa1cXbzd
znh^VZt-<mpfL_0gas2Ykn)qFatPhgPQoa59jEcjPRTLQ_RL-8kUb2sNKYwgTb7%6~
zr(zPnFm7eZkmND4TK*>+@bAGVpab<z0(!w~A@-^_dP~2L^%74q_+1>+1?;BhpX(`8
z$&R>q{7=cb#p=EBkK$D;_!05IxJC!-15JCnBo8ZZ#i!3;>S}beU(`~RxJC#RNCJ}H
zNV_L5`_j3@(*2!*iuknFyF<sEuy<Og*>S7SmSVE>Y0KVJd~)j5W`y(fw^gY7nr_If
z6g1T&<q)~$Q=HnntmaUg&G%00e4LtDS!K!{PU!<3opn~8yYof-&jDpz$&%zO?;YF)
zr<(5wyS2~#%P3>1Q>=`v4IL4g#Z>kun=Al)McV8zOm{{{buW}201CRflb8jZ;Mtm@
zaG6*QAJ4iMky3Y0bLyUfQc-#%-#W|R1ES=A5LjV_;<e-w5O3Ln3;ln;tnJ<a8Ev9i
zvv72-4>Mmr+iaV#3}}jo=!arA<NvHwCu$#l00>KT4GTOm*wrhR!=Yvm|F;5|Ad=Q&
zrYm{=5ME>moTRXSuz=4A0**TZ?-prkh&qxyLPfys?kaw$9bWK6Ft(@X+A8kvQbFg?
z+wllDKlTe?F}lYkR-$DubltBF#!4i9#a)Oc_Ral5=y18E(8;?fE(@RHVrR<dHNMe1
zVby&UO}LD4BBwA>&*z1Nz#vvmv%sJ!9RF}^Gq`D2h{v^*I>hGWcS5}Q6R<K54N4TT
zCmcw6HrC|96WAi2d*EQ9IxE_SKofqo{TFhfaoKc4!f+c^pWe`e;i>CkEvmIh8_Dcl
zf{IT~ZW@NGKydZ*eDW7XS9yZGzx$6S@KMKh>womHT5%E=+O2sEw0CSq^lGaaM^cx{
zzP9z*rb4bA@&M_i$iwFCmn0oqRVc@7P7v(>H`V_ST=&1_aQ}*~+zb5`T~W<h^@8-C
zpMb1rN3)6C5TcPWDWbS&sZO6HV1Mv;-jF;|krm%rWyrhGiZEOT?e@oQ(MG@9h>g|L
zN0~{EqV$DZsX|h(s7jcQKu%~Ebvos;7biIdTS0s4nl`wx`B==qi+Uy_ZaZ{6rD|~x
zSc-Ng(5U%)i7EK`>;*?yj684ONJ#j;knND?l!Pl4qI|pk|LCp2@;_&I&)3N-aP8p0
z3oqW%IhiI1V3a#p0k5iGEt>?DQdY50HN&ot!fn_QgM_;~7>gox1niXkcX#0{v)6}z
zv@?%%8G`@pMTIMEM$t^)nxz?-`hWG;<++-;0Yg(PUH?I^;eKqNxI47iGfGMPxBL($
zPsW`$O8e<PApy`^pveZxix7N2W6Lt3TZDcAyi;_7fQ;MFMy#F6!qyYal+v(d$(U$Q
zQeZru)yUiCxHhKiYFZqTY3S1V;;c)~%>D{TT9czkQ9f%6d&<tBn>u_#m+xDZs08yn
zfGo+TKsH%001YyoVgxaZx%M;6dmx5dAyJ3I_JY&}cFv``GH012?z@$!f=@e)HGSaY
zQ`FVe8~NhG7be8(MKMLKv+@)(v=UOBB2F;H!qjRWCMGG&nfzp{pVJ-5^>)<b%MZQ>
z87L`lWM_k8l%9qmeW~%wzTacr3KcRLMdbQXVXHqk<D&AT)OHltg^;Tp*J(5~JRDK>
z%@<~Bh1ZIEMj!cB@~<bwl-0&CFsD5-MYLS@0!xm6?S49*(~6UHYIsrdX`xt?e*6t_
z^A0g?D5?yI775osR8oeT7UzHdI7E&VC#l&|X!jFS0MOZNg6FNMk$;pn^3Cx?!Ksn`
z=U?!^4F#n(kF)VmFaUBV<A?&Y6lcqb*v1tizn_E(;(d8F?$EB)PN|AuFNk<$8lfHz
zI1rOv<mixZ=BTA3uf6UIzB1A8p^*@cijDW9KsgPQ&jN|Ho)=S*Z5A5Ar*5$i2$G4b
z2_K|=D!^6o(fX+NYvZ#i6U9d*g!l|aDQ>;5sBt0!{2dhX3eE9CosqPJmDX6EUJB#x
z%seOV&M|VZq8n5T%Q97<y8i1xte|rNI1r0@@L~0+LzH?PXS#LZRZNh@7)aoM9EiW9
zTmcTm@;-(8zw1Ad5;S~jT6(FG?z~6v*k=*9(EZ;h_V)md-%3&AS(O~y%~WX4xi0&(
znco!XWKo$)sHVO4K@R=;k8{TL=YBjm?$x)9!mog(_ff<@5hC5T*&BXyexWFSSh#UA
z=|~F`Oh$=ogn^K0TsUr)?p3=$wn_$M6S%|mw?%X&dabMjP~t1F4si(aqbu>I8rA1a
zz?InJsn??`@vU>Cjj1$+uMEV-vX=1bRrVlGkI8Rkf(d%&I2?psUfgRwk{$$te%jD$
zb2UX@Vog^Zb$U`{dpqlWBn`Gjg~g<F+M02qMQ~d`TeoBgg?LG+b4efqo%hbVIvLsK
zL^ChZ?tnBEr0(a$K)EE3m8_xv``$>B%ijUm!p!0E0O5D(a_M6hC`?a~LA_h)e_gWw
zb?4YZP-T!*HZ!IZwcib-md{Q?QebzcFUiUl$Ln`~l+Kj&*pt_M<$=^DXL>ku{V4jE
zOPfRqT-loQN@&^e;}z^4u}v45??f9#Y<`+~CXI)SjgTRTkbnxV8;2$uk4NCY%+Mvz
z$%Hf_Pio3bd0mH=-ogqKT1px3=D7@e#-*riG#!nl?M%nzI^EB$^*1cb)&Q~R%H*~!
zB}F^Xk64~>%jYa@z2VWi;VM6j(U{aL&!kYE=J^-pMR*b$;j3pojmE@JZ1Dy~a?rxj
zbTa~-A^yQxrY!Z|R;3)0EVzl0F~oa17#Z}p6BAJPY8O^$A61;ePVVizdUCnarj9Rh
zKtx`J$l?z|t0RL6?QH9*f~do)K1%J};Cmf%yhLWRzUVp{>EL;2n`5z7%TB@Obg^cs
z_}k(C^MwD`>HdA-h#`RE>1bv-`vM&sp=`G5PtOlQU+?UX(uMdPI*HT7Yzx{fC(1lt
z0dy;^Y8b}p3gplu?fhq8ctJSi|NdD(upp$y0XkL$8@>=WE~Jr{AFj=QcA)c~AUI1Z
zuYQ$4!TvU(N#r1hU6`+vhIfQek5@;C(sEE-nx%`k88SsJVH&O*5F=xjCo|@6^lsd?
zIj7B;^^~r#k+51TGuF~y)kBLF%RcPdD#YbNrw9Z84RgCx;d!7skM)|<3mU9`YNRf$
z76r8caYSg!c?jO(D^9&dSf46<!bwVZLKfcjorhy_US0l8@LB&k+ny%IW}Zq0o<xs}
z*2-)+|M^fdp2%%Kr*<eT<SdrT=VE!{SGlCqad<saR9b@;VOT-a0MD?j06U%kQ{Hd?
z5*;6p!-X1f>H}h==pyZS0^#nT6<?=~uS9mRKPyQV&u{+aq38Qa(vPz}83zklgyC9Y
zeVSSyS{Xa?OaHv{Re~Hz5V2MWXYtdMZZp)S>KvJv_D9F!kAB>d^5myPGN}$FfnAcm
z9A2AW+IpqMuP5zIyX?ms^=F>Bt_d6GR;lAL*~dha<1JRp7K+~%*@{+AHCD2}ODLO{
z4DX_L`NGSBDSFUlD4A0@(w$dW5r(JMTCsOwV&@zlgiTKf5XfAd&v!vz<e5ga@MDW}
z`?S|z+u`bsgWF0yb5xYC6J7~#7RC^F7pTEc*JJzT;1(f<u2(fkDkAdPB+ayVRtTll
zioCA^M7u<tr?~N~B<cS3nKO%lJpUZoZ>Kv&F9jEB=VN@hI$3oEwiy@58%zf#@qCD)
zK&fkEl2NIA_Gk94^>R^Z9M3V<hKF}07QLj6>fQpbAhT42|GbRtlkz~=O}ZCib_G4Y
z{rM5gW)WRjqUAkmIwF<>LN`ga8a>g4W+zH}lCU@8!tX*zPc}AgGJ=>C$(E+K=@5~~
z`PHDI&(-UYLTXwJ`-2IMU^6<7&nm=f@Nv3xTs=|L-HBaPGnVArN)2UGt6Nx(feDRb
zRcAH`sp`#KJE4*Npt?B}Z6AcABfK!xYv@w~e(E~?BzkaM4L(6rTl~x%^@Tn{q_pt0
zi!t=J1kA)witp1cIX<Lq?~J$C8TZ4%1aiBxJ?>puejHs7UVE<k;a6Dgho6g0&bwb~
z!SB9G__ns(9Q~N7x7QR@{aG6cO2tXkjrq6CogdJC*s$_}U9Y}u8qw!^x6t*M$2L13
zUQ}y@++&2DeOI?Hh^K;Ist5IN3p-n{r6RX}-{im}{x6?Qz3DM3?`EdKqRYBbdCMeu
zeo@%1g1!b=KH3>u-tAm{V}9Jiss*;NSSKd1br2{RqzvqpdUDl3cy`5I?CMxL7m(`Y
zsNUzj-Op*Bx?VB2l`=YYF%c|)T~i-&PFDk>97{%Z^SYjT?eeOg&yT;?&&gXdS;5<1
z1tnXh_epa7^)pri{ERyRKV!_hCr!76LtmN1cR~R_W6EoMU=8b@97$P-&<GCv_xJuB
zzN!VPSXH5%rkL<zaNCwV1SD~tsA-n-h7IOKop7Zd443x34vUeHNmB*y0mpna6tz=e
z0X$5BLS9w+|JFbMy<@tgIE}zwx_#Y@cFlIC*Ql;o0kH{Rnt|KTnGoGnU=sg$XH7@n
zpGW=kV<e$~X#L+WNJId4(&S01!x4^u@b;bPHj%_<JPIiT{a}v@j!^`kP(fT+pZ00D
z$4T%JHa&#{b~hKbxRf3|jBqQ&OOg#<I#!Ga0g00j0(l)+>#6eWk;>STSGFU`>diZ~
z9VX4<IsFE4;FyM<PC&VWTrM~<G<9D-X8D|HKW0R|fafPjEJT7=xjGm+7hKF?@zIF7
z>A+e~@5>D0!Iz3uu7feqy>ncu2{cI}%2YLCWbOzUSe%O>2I3e?C4n^}4WBT93av{U
zeD8v+S*eyuPA%YC9T{pgd~!T9;*fcalJCC@W;o#e_sssAoZB5k{ZM%Ye!1t0!8hV%
z$0~gv?(6^j^12ax6(N)RH+-J&Mk4-8u(TleSIB<btCmd$c?_10(xk`HZ;!^Ed<BuF
zd(g52<q?Sdgwv6g1VK(eX<AA>Aup|BsF1P{I*2+##O-ir2%m|CG@=d0l$*u^3nkv>
z5n1k7IaGI7ufzMY$pP&ygLeejM*SiQ8-=P6P=>dKT9_e<M^OmepvF`x+cc;nBw)Hd
zj9pBq^FN-%^BUNHJ&D6<ybb_Q;@7J1z{_-CEmnL!7&3^PbTyTfwK#a(^ySt>Rh5lD
z)NUXM(>~Ff;w(q)jF!qCOXjZIPjc#m>`Et^dXme8$`J6o&5oIq`~I^Ss%)lJUjUJQ
z629KawT}_IJc(t;{LkARfqBmKdi+c2y(tR>zrlz%yNdv5vn3fa?e12IggJKO(W@2~
zYS(!NaR!`+MJP^CD_@@Y*Z3*KWSOo8uru7@ZDy-bebe2WYUFN-I7mPd+1t>`z{8yg
znqR%z=?V_GN+8W$GmYW+NwpXx1ne2KI9;?Rgu)89<9FGALm74ahD-}moBbvfHSBh!
zjd>jsm1W|Um}03s{RxobIol-ZL-xAWMNO^ky1V4o9cDK2Udx|!g-F8_)9`5I+$R%}
zTkOVZOsGz&e_Dy27n)5mVxe3Do&tUB>E!4`jsJO->rC8O694ilKj0RjBVif?IxP;b
z>_wRVzQWDR>L~n(e?G&(Fd*?J;1}WXgMmt=1FMAVC}kZhbKj{Q85MiCo5asdQbLMq
z86gozYe%|}xMJ@KnM}SZPR#sH{xhF^BDa_1*p`_aoxu<Vfh(J`-KKpd?}S&a3D#(R
zFO|0#xMS`3bT;K<urweu7NxJvPI`mxHNHsQNqJ$)2%|n)3v>GI;J4F8j)zd#BPu9o
zAn07&T{${b_Qb8NhOpa@$U0=OBC5Fp#bmwGH|&RZM>y~x-3{p-`4a{S*TLfwMM#GJ
z0nX;MDD&=APvPx69`v$Pyhj6ajSH}X4#*mF4d}-O36dQCbFF&S%*i4Da;=tW)wMvR
z2qLH5`1XYvuCRHnZ6)4o4pKDUp+gHYrsw}`0~7FpC&<F2&S0l)3L@3cCRxX7eDVq3
zy*?)N(J|N%f&T=l`;iMQURxC(b1NgLfpOm5k*Q05g4-QM-2ejva(90ngMX9|t>GDY
z{*PmDcMiS4CGcyB3qZnOvKTECQ+eHzV1EM~g9raQ2BRHvr=s6#O`<n4a)jfmhN96W
zq$1ewg<hW~O5o4tndge1Bd}9fX^Ho|A)`8A?n;5V?B!9B!Ka|Y{Gq0f)h<$Rfcm*N
z)N1&v_GNnIqhs)GCDWs0@Eg^?j=`YRjuqbrv1^++e)~oGo7(sjK!c=OO&p%Lvz+#^
z^?Uh`qbBiDHHka(h82j{1Xpq*LX$PHCq6Vt60xqIgsYo-L0YdEqjGpLX4n92rD|bQ
z%U|rHtSWWwH|tcA$<}N5Fk7Mp)^nNBvhVo(gx_gP**B7er}t_Zm*KxR`IS^Q8xrO*
zm)Buka-IFstouh2QTIyUEREq*KHGPF(G=x?JkdR|h-(=p3AfD8`bj150${W%zhn{u
zC`L6Y{O8jR6o)4Mf7<rHA=voIV?ao{!#6Sor3O%ZTVDpYg+*_O#{4aBQ5l;3VwXQ-
zB3b$)cj%BmNu2NCMD?lCX#+^FEWtdd{0Zkl@M3%-y-nAdGX7KC&B76XPH~tfq!4yz
zqHM=Yxh2WwoEl@w61+G{Kj1Is86{tD0+~4l>k?EN_qF>IV@OrKNj9|i_cNsKmy4Hy
zqM>;s<_ccz#Fnc@@5An|0wFeLg||(pCvoigAiYrV{Hr3f+}laBq4^0O#+jL3l#E_Q
z{Y@R^LM2m8o%>J9;Q`osf>lT(+4+juk3$bYT{QgkIRlwprT+z>pw@S_nQU~>fkMT`
znwegxmQ`Uhko5O!F?fZqwI|_&v_DP`a)_*POOT0UP&su?Tn>rWXGK=igJ@k}Nikn0
zvmybNQZ3}}ngpA}Y2akF>kyDE9h@O_8KOqQ$(uFwvbUY9$lwFUgkyeVI_terHP{lj
z<69$XI=eNrmVb_Z4iP5*p$y%2+8$Loh`9Lpv#HJ@K|bXn6ql}AIpV_dXgEid+kTPv
z=6WAmqdaHi>!U{A|9Z<>QZR+vLP`50*L{y^7GEqaX=zz?`1fPy*&JyUJ)qhUr3B=E
zs?qQnug<h6RxD@rom3=(0df}c`d1#y=+m~d`<vq>;DYLPA0%<kd4ottm$Y;YEPnU7
zIVte2_r?4v9lX6HGuj`)H&Yef%>%{uQTZ-^_`aExr}6}EWPmBl&gbUx!V^;F*h8pe
zzVOCXdPSDbzo0rU>h#~zG3H}Bo*kWETW9A&!UT^if09p?MBbGhyZ>(8P9yGL>Y0hC
zkL(UI$#Q=(pWPQ6pxZ4|RD*|cx<QttdT3u2A9cMOk3-}$U)Xe8nKRlkkJE?~Da*eR
zM|Fu>KTosMDOmC<hLbC{vH}7d=d|m*oqwZ?s!xK4B~ETvR9Ml$ji@3A|JsN^_Y~!F
zQj;kdq0>l)0FLH-@WyQh<h_S_*{wn89L|~1TOm4cg%v(FMZDReX2u{+6q-59T(1dF
zkJN)FK3Xt$f1wmPz#WNiT|C-3_)!0=h8K+&qa?v%qd)Gl^BE`cNAG;rzZWRqGkC30
z3DKXlbh>iH^TSpMxAiqWnUINjN7r7pCy~fQ`!l>Yr(?0X*01L-lZKgKGUuP|kNciZ
z+hcVNZSFsQ7gzMp6JlsPGn0Tw`R%MwsJttEiOXK@o*`hf#3^Lsat}ckYOOVwd>`&=
zM%@I#sEdFAwc2wJh6AV-1=4`)lD*D{KfsOoDGt;E-L1M08E(H8gx!wPyY*ggZl!&!
zho8YIGvZ9|VnksIr8>~e&cwG;zZLPH>N?~$Bjs#N$Wx7QbN06B9@IR2-SO0%(6G?G
zWso73lhR`Y7icDq9bJgZ=^hYK$c<G}c?U=$=7s@#lMz4)dp-RQoXSn*|GwQ1u^y;;
zueII?86_LIe!bY4=;CNTk@DKka1&LIO}Oen)s;RNmd);n{ZrWXaCh8`txT;(;n+{3
zg<kWw;YjQx50L*STsVaA2-aYv=P_VZ0i6ZP_tj{p5L8~#<7D*^`Svr&UPA46>yaoC
zcf?yf976=`#+gFM0q$-X4aaBa+DCI~&#eM?Ig~b<VYM{K?ZB8|f>HgR4#bphFsNlM
zTRI_2_@4YcPX3e+s}Od-q0{C(|0x^$3{2_z9hS?fi~6qYML&)+B49z?jtb=C`UF@!
z=H<Kzr;*Nus~>x1Uaky*$eca_nNbFHlp17Eku6HmjD``VHP%FsrM61C2#h7Xh|0i~
zko?7Uo>O9Bys*&(;yeg_WbBqKL2E{Z1DI4&H%eyccG=TQm~0*1%EeIHkBOc|?#}u4
z$d-D4{M*6}_VLT<LvXBsCElN(4x%7ZMvy=G8NMqLti0cyE=A@%Yj;huB5%6y@7jE|
zn0u>toQKdF-nA@V*mXdk9faCqLB|QS-yE=lw4GVeK+(c`%z*Q0G}W^uUa^a5&0b4J
zO4WL+iO$7_j~|VEE)uI)<q|B&5XD~s{UHR+BY4dFT0fsqEQO}jM)F%u85>JdphlVR
zqd5H?5DC_FUIVdK)I$z@^(ptmBcOLSCZPvNu*4_UG-gtH9aOQtMSFkT^5=4Ci7Vof
zI*RYIH(PFtGbzKH7jqbTE+OkBo9GQn-G{dQf>JwoqA^_&LLU{_*Bc-1Zw`U`H~QHY
zki@Lz85N}C^nbc8jV(5fk3{Fs_^fIEzb(Z72Bo#afkN&LOY;m~GjL}@8Wm$4`qYm6
zkP-T*N3<kSjHnN~<dBRIa%7-i+`|d?b_>-Qnm_Bi>~`KxYS4zqTU)G+Gl{^sT^*@w
z0p@a!N)rbMsNVV>MOcv-g~9;`HePr;SPFp`N2zsmZbT4bei59;ecF>F?$0&})X|VV
z*aTV^mI;hz0z`3%@JjjmG`~XM2LDwnZM9$`(Pp>21M?<+?`KJAN^cycMVPd_w>QqU
zOp}J8k_>;iY?`<?bJuOf?zSA;2$})1TWuJKG+Vd2a^ogmP2Bp~a|C;9MW9Tbg@@*{
zsDM}_BsEiOO0PAvp86N;$&>KNfLO5+QQOL%`c>{d!4@kuy~!LTbBKI{fmAj=-soUP
zh2%I4RKu$S2+Z@nwpl)xiv0*?y^&P_LP!wLF(g3WF4=p1dp1^cJ%t(ilgDyYdAjF1
z1Vn7AmMJO}mQe}=a;SOvux^<z5a=get#G6#udJSsX!m&h`RV3_DKzoY@6|`9H<7+o
z^GwL4#uoeK4fTdCo*o}ROunqYS`DG{{-Is|?orT`fnHKuz3J|937o>xKn6x}xW|E=
z`F06O^^b+CE}}=3_W?o=gewiS4+HYqtIIP`qm}sgR}a4|k_`^706G5cEZ<)hfQ>s_
zKP_+-qW}FWFa$i-^uxtPCz474CE>a1`p*tdxD2<67EvRj=y3>s?3Pfrym<_!{9*_{
zcJnGxQPyjRT^)>>svh~OO3bq#1%LXmT1@eY06K>OJTypLC-Oz%UINTvhCb;EcaUh&
zR&|hSG)CNed{sKENX0iOsJ#cBi&yfgY=%F&B&PZ`j>1CVy&ukXV}S&sUpOkgH^Hm>
zs|n6*UaV`}>y0j53n*l9HS9IH^A*iBuJ`19H$95H@bk+YLpU(mSPLtXPP+|-<>pcq
zA4l_lcvg6mwDnw-PA>9AKT8j~Kuq23?ecf0!|&UfVE(+*($NgpMV7EGe$0=}r*e}V
ziAuolKKM0so!J5a7z!F|@PV1cDp5<a7|31n`|LYgxPQ$>e<`%J=|F`xx=|ms@&Z!^
zIgpH^EFsQN%L>|ZSv^~>=S8Ny(70QWR;@tan)}&T3a=N)HHSLFtn_cH?r9K%$HsEy
zqFCu$2=}S*tN^8@rq7OMY)Ba0QU1s%+NHofAQGKV=mHSwv@-7aUyIMCT;>es_<uT2
zb?^H=+!*D5*d*wsl7KWDZ89ouvDz=yU+7`x+?@7N*^g(xH2H)6c1%9oV<#s>bYVA?
zF8YMN=n+$i7C9310zj*%8EdI90OX4L{{y)mKmX|Wq8$a`^E<_+_5JO6$&oWpH9P;y
z7#~-o3K2xxsf=$kRZydn(%T@mo5+<WP)kSKEW|j|)G<nl(Ir0zLzN0);^zQm+{+F~
zBjjj>N^9BA;8dgn@I-9iP5DA`XT04+=`(fT4hCc>1`Srg6g?<(ZK_bwtjj+df^{tj
zaK_gv_821b(E!f4RR~<GP^MB-e^Z`G!V%z%ONo)E*?MXydnXuO)`o@rhB{2d6qTPU
zFiCjh98Ap{2lVThmF_!dko#|C0!6OJZhms45&d)UV=vOCs7zk}wkXm_ced>9)B{Y-
z%q%^!VVWM#O6f4NOmMweD>rEds=4z1mbVLXW56l>D)T|<h@n#BPLBVsaQ78IN7uJr
zR=zUBa!>xVWZAavF-ya6P%5zgzHlH^189@Dd$HKF5F)9Jkn)v(7-`xR-nO7=Vly$C
z6-D1Q17m6<rSREmS69n<0NQH9nPNX)*K?N{{HXW+{i+#;y8hnl%cq_w*cG8IvVT3$
zu!s*o<7}>+FCiifYa3)O)eU%xd8>aVWiTUEp)Xi7h?tc)NLTiv<B~J+yM=}aOuKLd
zOXR>9AU2(`z?Fvs4%Bu1->>-m!PLrBzXL+b`(4&W=Kr0P-k*X)=JfC7X>2#~4YOp<
zEiW>FQxY0Vm?NvtS-*)4oN;@sZTKQMq>)Ct!|)G*A2J$_hK5^>@}*Q_$o+}+2Hp6w
z=YBZ^Tm~fRAzy*u>AreLI}$hG|7wu~JfpXty5;`aJ>%d4H5~BL%vEKBP+kLFmGrVq
zZ({)2?g@x`Fx04h!~#n!<=OtSI{M*h2i7+<m+KyiPg*Ml-N6Ck7iJOB?pG1}lH!5l
zGV3{%i{R&XgQev2cxvl_RJ%Jy#hw`p<^8#0LOl62?k+JDW<3m3+fL3fwHlt+(ghg@
zA%mh*vl-Py$ZHv2vFNEJy_lRBqn8HeLD(IE&y>0JXO<@V{9_crk<xI3nJj{9X`^dv
zy7?g^CR8y8-0QHBplsqnMMXdbWqfPA(O~9=S-c(<&Sz*kIc6~VqRoy)_C;qBq1B=Y
z&|@iNJ1A_#Dr65JD!tJ{Ljq!arms4zhsfy7yW8^m#X*Ua@`H!t`jN#&+@;F~*O5gJ
zSme{@-*Zp+P4+-HzxP)!PnNKM5YZS0KSwi9qiYbG^sGC!kJ=Zw)H6BphlRYAm<LR1
z7Lh+Lx20hJ3i;?1c;PDEzj7qBw8frcnRdB6j5U9NU^|%|HYY9R@$viz|4ER)N0=xf
z!+3gq-)_BU{EScL$=LK)iTO6Fd$*!@WxuK&UWv4<kQhG9Zhu^$lcu_KlCazFvcKcl
zUt$GA*K}SG^LRvJLVPNH`0uC6h(*d9;_YiL>kZv@>5}IYNHnXjXBtnkpOveYH}Si|
zbblNb+vXdVEf25OcQuK;s|g+{gB)IGHu6~Blizq8jX}uM+e)(<ZMY-|2x-O%0t^I0
zw=mO|CYGq|DN@~(E55^@Z7t=I#2=iR#E{Gv<m(<xohHq=W(d1HQX*<+wDm@)D<tJ!
zUagpk#2mNYjQ;XI8%f_t7~x+Ujz_0f?~h@RoTuC<F3cQ!Rb(~sEojB_v#1dSp3g)o
zxolF3t9_#Du{qp_*V`VBYX^^e{>zs{p!-Rbp4s~u;r_DP(%07)``bEd)P3tQ5+8tW
z6|5s%k7Rnw^M3dQJKiI7WZz+~!k~uI{H*;3Xb<@669yQL&c0q8{ak9#a*x$@Pl=}b
zb~Wu08YSi<YPXql0dTbr8q`<QPfrMuCWFa7SnWfP4ZW6A03+d}?Zk+b?h;&__%wjW
zmG0%J_LaLQ(|rxiEWd42F<NMgIEK*4G~z37l?c7N)vq#jC!km)rQiDLwqjsI_Snjk
z3vpYQc3!FjwPZ-!{dZRyQq&t$v==?sM1bqqtVFll<0u)B*r}JeUB%W0JV-Ylq90bE
zt=EHw-lwt#N{R!|o$UBRPOiK6-!R)D0`t#8^-Uw`56!2&Y#F47OAUXZWH`?kAYuCn
za}<U;=bkq!5BCBQ)A(P8Dm_j*P-^NCM0T<RZue1zenD>Ry&mrVBz_2DKW;u#eaIa4
zF!nMY*e%RDS^;NzY?1ufOwv0T1Vd}FD1_%(3e(qCURlTkun-Mt73Dv{GZ~N)3SukS
zzKFA~67<XQlJL0o&HCimfpC#8cy!|cJS&jz>kx8<eu7Y>Nz1y5{%I;H?YPN-qccj=
zBdby^kI*!SNa(5FCaMQ|HC;VyifSusd$IcqGm8DeZ^qv-Spaj-8<=TQ<m9pYnp#K0
zEbmjHw*&$8FaT`3SRG5*E$n_yarjLpfSFOjQ<9XKtC&X}5kb(8HI~;}$?i!uBZngz
z{*10xvKaq)9J0hBlAK5S;4`1W7%4~sDH(q^4MgF6PLvf#?LhABd2zYWP?oqOGe5&;
zEDb6%d__6<E)(V9c&oo{!&}P?m!qLSC%qRsO`mezfKRNt)=OxG$IPl91_l@=O<z83
zyxe|5r2m^rJ>Bc*yEX}ZYOBx9;v)i~S*Th!r7AL#oXE8Rc$N%ZI)_jeY)5!p&4gH#
zmrQ=BbG>Zv8LN8g;=VAzzM}T<i^KDGXODLN_mk|1GR+~?U%M$}(e(ziD?OJxQ=FEu
zCPr8_w!!)t=k_(T{Pl8aN38}+xbFNr^*K;!QXrgog=MgMTeTd*^=cFZk=2^Is2}ny
zMw^9Pr!SBv_sU%R9*-=5^HLc06Fy1dy1V*N<yp6Hu`ppwnA-NsCrKrppKDjdC1%R5
zWpp2Yx#0FN(|h-VWA_>OPdazMK;dYr4b#eHVdc;q4V!;1s(b_q+neD}V5AK9p|IAO
zpR4tCU24@1f1<=zLoUhaR$_%q1BdaV+lUVc+~X<J=Bq@=lQ)vY*=US^UJYZuozt(H
z4>?Vq5yqZrSa?sSmc(p}Wv9~pTozA30AT*coVH73NfsFEg>jknvc2y_ZUYwfM)sMM
zQ+^MiE`4nCkxECSxcZ#(E7kO3M(2+VvH~4EFp*?FTm}^A?HaTvuwah=V2m%(+aYZZ
zn#3{BQm3XCd$>Kz8JtrSJ(YPNMH|@gN1;3cC;M4*49E_fY@%sX^l#@1c3bu!$zTt5
z#D_rNB}&!fNX?5sd9BPG8%Nz8*y+lN;|N2;o_gD13fmZfD<=uHjFKR4=eRcRp`?5t
z8-%FFk~ph;y02SY1%CKW7%#B{)})@~X(pN?$Q1O2Q6vyVd~az8fm`_6<CluQ2HFun
z|BVws(>B>Fh_+Yfu|4rm=zegcp=Y;W?PgWPiKk8u+Ww?S{CwSr?J_cIKj2udd20jT
zaKEuaCc6u(I@7SwgB@K$0_OAVGbWy4L#@XMCX^j^mO9n!;nXxlP|u-P=?6`V$OC3m
zy(QYH4m3e0zuGM)D8)S8#mxK&@Os#&{DW<G0lSn_J=Fd5m;FC>zD3~8Q!5#|uLiZ2
zkF6PtTa9P;Xcr5ek4xJFG31ZAVnt#Jfe+D0-P&8Nt*EMh<q8$kw7c>8Y-7FJ)e0At
z4&wkyx^3m8zXP0Mcei1OkbasbE@UP|RYY$Pz=OyQ!Eg7c@HqjEPZf-g$it@?Q5f#7
zHuvJAZ*$sg882ZBNwD=0?0msz3Bo&cZELJg^LrV6oloeA3w!rUS}W!UL)eAus*<A|
zMYOp29dr7ym(ZUfh7)3D3Mw1M=bLATGD?f3lV6OZi1-JWo7$#V6Coh^<DscV+VO~e
z)$P`RY-)*{PWItbDcR-cCU8DN_;1bf)SNcWgVkaP=*VqT)Yez6)l!U3`_(d1y4N@$
zB5rg?zXlC+I(tUIagy;wspxJy1q|k4%tMq&-hxe6YZ2uDhvgp8zKTB8zJ8(i;awsb
z02bUg833GsCt?bNA-`$#C~ZV48q6leEVc-4cG6-FhQe}h!x;_qTx#k>%Ixh*2>_9`
z=@K&m3b&fM^<f((s5dNJul#+p%ZG)Y`n?vdNg_PK1ZMhiQSk$ZC7-Tg&bzm#@4M%5
zd$mOkw2x_>ejH3Z0QMN>^)roEvA#`OuY~D+Hj5?0{mJQs9JcX&u<GXQ=7w~hF^os1
zznG-P3zIoNoc<{GbYJ*$B&u+{rE<L{O6IpW+9aUVU>CAQLV3eo?V`8m2)>i&ikBC8
zQ2orrGOc*+rwbso)0K}?clc#KmO}T-w+AYh*OpMGq#k4W*i3sb&$UD(sZo!!GXeYA
zF&_JJEL(;^=4+aXz|!MDU{TB%nm%MRV8c}Qn|3t^Z2lG%zig*7oVS@u`n4jy6_&*9
zr#1_`R&W2V5POMKUh6d-4P1Y})*r{MUR-#u6tYbc@eS9{vtv+}Vl~`b%m@ig13{#{
z{KoBmfp%SuZ+-m#2vBmk?M<`nK0~8VwX9-T%3up?{6;RCJy!Q6pr1H01CGW?dE|uE
zZit;W1tL8cJulM;hzTI^S8lgqLfs(s$-&wNy2Sohs!DOO^YsylcJ2Du@5AH|K0V3)
zB=Y=vK*VO6nF^)4aI3S+Tl<?XB}h41;@({%@m%PMEv2Z_-K2ndKaSc_r(o?9A_Y)q
zniFQlJaw6R*B&KmcN)-eUfXtom$PC6X($-BW1T=;Yrt;K5De`w`~n;8MfgzN5MOVd
z^oM)wGew|x^_LTBkG|Cp02<wQFp2E@N&-a2=~e9ADEk2SsP(2Xrk*RJ2r;uoTWHJe
zo<1oR2#lrew+Bf2!LH#8mRT0|3}mf_KLJ=462xr!sRbrnGRLjmv_?9;XH@pO$F_!^
zek~gr{ApWwi!Wpj(h|H)JANVtLc)4@g*t~NgRWhzLDp&J0$Di^dv&fRj)d7|F~fx#
z`y1;NcUh_vue34f_%L-17&WlB<m=Oqqh{#J{SzR?_1}~BJMJb|0O|*TQB!i9PLNvs
zh?M>~jd2Lk&R$&*7a->$dRuUm+RFa8^0$!Pl_cF-ks51VWCp%RaOufOh%pTh$-bSZ
zuC<6^ciGfpF`C<S!XwjQ+ane5370RvT{ANVQ!sos*xH)@w!8ySC^m_<u+4r;>p^P4
z^yu$swu4kjLruAu_a;LEK;#JKrqxa}c^s0bS_ZDLiJt6wrA;PEm)%V@G`I}hfX_C4
zeyUG#eBMQBCs!3C!_GMrrf!2#0**eX-Nt$LlGE3D1BDLG!o&}o@YY2~LK^wK*PH_s
zP*@jANx%}nxn#1G;D-e1Z@bgQ@xHp3f7%|z_LI9eDe#s8E7vfvh24#v;hV`f`3OV%
zSz}mheQj8GZj*PDp+xq-Baq|kfoi|Ddg)tnvX7|igw6B5L-*z?7aXLQ8|g27&*(|v
zydS5R?_K)a@IUuu>l{}VhDeluXmbqL&W?kfeUxYbQ~nSQJ1aC$tH&g&;5fSt8Cx*m
zT-j7nTim2?vml5f1<xrsp1iy${4T+wuMIEMQ-dwZSg-{2QZW6r=ChqXr%#j8&elNH
zPW^r~V0&oLuu0H)3*GU}m#~c7uPnuK`AE>Lte1~tHTrA&WBvOdd_JNI^2rjeZ)f2-
z>qinR>t`W=jPmTl=;beSs^dN?T;HM^3vG_?yI<4DJ$3{14GeWqT?O@hruk;=X5Rl2
zP>FpB_sy8Cn_6ZS^Q?5+GO%nS{M34JN#oxOJQS(Q<gsd9sa9aQLK5~O3+2S^r;E<$
z91`VT%@<T`r8oe&gO?{2ogt*6K<&=xX)b{wKx~<3dy=?lBPIj=xPls+wj3OoF*GaD
zQ&hlXCGqhv-LFP$2uR7IMb)=nmZLJ-p>&3mBr@E-GXe{TV!X7YH0GXFCnjnvkOifx
zQ$;%ghGJ)dXxBirs0=a=IFU`&VwislCg>TH&5omxp{Rc?bWquK8=$*OBUR-YsU7+I
z<#+9%-zFk5PsPl)R}aC^W?G#X4|jw7zD!~cGs2UnZuH#&t_o}j&+6afEin2VN}ZSz
z;<iwp`gV1F43)QQ`23p($ir+fq-$?_jBEN>>&996NMahT*U$H}A`s$1M!Nnurw27-
z?^`JQ3v?g!L8Lv}18#(<%B%bS*N(Wx)ADFXv9XUyT|Hwu;*|6G4TnLKc{y<p`P&xR
z&XO0+95l_0m3stpNS$@eljsk9v}mVqacG}@n~9dH#sqio)TNaFnx9qaym(^5LeOHE
zJ0zR^GOP7|f~NWMELAgOg^%a!t3My#0xMkPE&z_rn98O7uTTRO(sICW$<EiFi2%J&
zTr@dY3k}bzSD2i7eB+(LQ&j|7RmrUa4Y|IcB4kXLD>S^lNsUe#YLu#OGj$2!3k@dd
zfj-{=lxs4C^BZ^BkdDtrz>=SY_tW$6oTlrIh>Ttus>S)za^i$g(_&RU+GyDwvNeVu
zj>$^zK7^15W60rSF%|gRRyMwtM|c?+i$lHk={H!+{&qu}N?3*|h~K3y5n%nw5N&Ok
z({}gB`Ak<aBzf=VvXgE0anLsx9ZnA@YGVLGz-5C_Q5!S-tv*9|;xh`b(Ap?HK_LPd
zD)!iUHVB`Rv1;>8@gv~9NK!oSewEKbp9<K4=!DAD^gP#_e{8#`Ah7J?aBxLGyuFz=
z_s2%%t8$eZ{$CgI4q5YU4}YMJLH^DTQ>Ow<^>!M(_aUc>dWk9&9b4}5wJ#sh1bM?H
zBZCNBD_yb8ND;NM#rI)^R-+OpNXK1!JY`bxNr2P#B4*#OotghLbeUw_v>}p08jxLU
zgq?y9?x4TM1_>(?oH|GPmXsSh(O(~+^ZtAXHQ7G&HT(jmPoY=R8Ia9JyD46A>@0`u
zvnUA4>pQtO{m?&@sx%ua!gssRjTz!|IKPm$u%E~ajLo|)tBU(5Rc1=|B2V|gCl>ww
z?T>Zg`Y<xz9PR<!VM>zgPdOhV7HSM8No~tkFGQ_^Hsbbk&P=0K9nL4@ZwPI6Zgz5G
zZdtXhMNoZlc<Ajk2|jGcGmfA1ylA=hqwJq#Lc%PCN*=_D0(GbmRa13FbUp3bs$GSa
zF+pw$9=A-XcDXldeUJ%51S}MlqdMpu!C&r6j3n|`G^9BCsM7QBEZ`3XWe_C9Fm`=v
z4m#tWF1LP36bk29COQY7WpAe~KH=pO<`(WK$3MGMef;yr$ScEjcku`4>@>!qsOq7|
zl_p)cCE`I>dG<eC_TP&fR0imToMGv#oA6=v>nkX{Ydox9s=jZarj=BPtt7+)J&UuM
zPu-nTlhAT|EhX+}j_eVpm7+WG^h-*=a#&7jUc+7vRM!{yC#P@FOJw;L+b$1|@qrMT
zTU_2%-QTK7M$>W1a5y@N<(<`#Vu7g~qC$|0Iid_I_R+XW>9cDI9$SKIBXi|4Sl6t(
zwTU<TyV?_*){sg(Oguc_NAmzEvV1)OvdAdW9P`7rNv6Px_mue6Z{(O$hdzqr-{g3A
z-jzYV=bkD&RQ`-D=ux)=Bl{BI7}<eG?(?bAzYeK}IFXiX#(g3x6AS@bVZsrgI>4TR
zXz_JW++HV=nUa4(l6jLeat2vELHG@(p8qu7=WnS0^kVhd147H1dU(-V$w0lv{6t7U
z<J91&njMI2?}|mj-hpsuTU~$Qo0;f54bJDvIEpIsTwx!WhJGOo9u`1xnn*>q7rOQ>
zH`<D=#LGhVKDJSJ0)bFIUfWKEm91(f@3at0a`Gj?MBr{A=Xs&g<?gMGChgTg<`;gc
zHd9tkPnc;)AQOt1{Y88h@t`~g)~1#LTx%`sC^yOwZ)*-1Z;@Tw{>TmR!g1kjdsD&j
z*xHLv0^tds!ciU?YsJyDL%U8iAleT;R^k=C5?znvxkaX@tk4G}y$4Y022&zXpLYx2
zMNb{v?5-q(4}_x?KJ{i=dg|4$2%S~%z^*Yxccab0xv?^MA67weYhw*r={U*+Wr_ie
zu7?v}L*(t|oK6FXioq<e{3Xw(Z`2lApqNU%d4oom!b2YwWo~LkRQVL70B7ygSpCj$
z0UZg4#F(`E+%vKY$@u!>4@e_%J}C42w)X2Grw1rw^FN`>8I)f>)_fjIlz+d~b7|bS
z{Hb=X5REs;G>hl0P@=rOTkT3}S^8GkTfuvoxBg03mGL2YqHOFjC+&oePdk%^C=*}^
z{K<jfd<+WV>5r$hKY0DOq6B{F+S#JjvzcsSudP6C4u26&VsB6d&TzQ*r>IHcA-m4&
z)W3P^|GqO24MLdp$qt82HxcBq7RL02Pz(U>EFXs?p8dkHu8sp=P<d&pEanW{>Y^z`
zu3xNPa+X<j3_b^H+tNwnTVQy)D01Aqe%C4C9UPDlP+30vyCb0M2z^ePR+8rB<eVW*
zYxNVVPg2f5U#=lwD-hpu2I?iE`5w@G%zu#YJPVBq)RNUT$iwk}({s`J{mY9?fDbUX
z&S}tEP3&uk-KaXJ_fgCinA3bg;--A4iOgkVnWe&;kR!9-Xn29H6F>^gb%5nrGmHF7
zAG@}?h4*Rj{j4g5`9(a(-(T`{A7IAVyCHMjQP_@?F%tKU^W*iuJZ;;Ai78fXn<`{S
zgF-`B_k9)SP5JQb9}4Vmq_}BI=OinKTnxKXzN7L2Xeq38C-kJ}K}NxtmJ#Yw7L4C>
z>qU0#+YG5BjRmdg{%}1MjicliTN!)13uXt>vhZ_|nl@h3<)UrWTykwUq19w)x9h)U
zSHObu3ajjR)`X>H5U4t@oirve{zDlzYZiI`Js{12cCejj9Fx;|xjV!+8xqQNzK%~>
z@%0n=I5_$WJz)p;0x*BagZN+f<8Boit&d+Nthm6r26W&6hg9M-^q>eVAq-mwb3F0c
zIW3fl!2%exeOg4eL)c#Mq3G6cfg3&HjM)R|%8Mmn?8EWQCdOVtDUn4~@$Siuue`3|
zP+~uh4mqZm_X((%(_!)k2%7TKY9yEY-#Up}wq9ijP3srA$pYA)d`!O@obUN=mj|xi
zT`D%$xRXFSd-_rNs6`#HwE!#w0pkkCQ~kvW-5^9f%(ie&?6CJ!Dgt#pW1>?MWyW5g
zf$^X)JXV{;$$FtN-{98+I#tQ|j649Oja~jUH#!zM7&37ki7h?DV$t8+nw00L!!_f^
zZj?g8>?TI#wno!wg74ix51KoC)zaoC8OIZ3sfAts0LU1}KAbDd=Z}&7GWNQ6i!RcL
zJ=3pDJH3kruhIG{&0db+XW_5u$15GNz&q3eVotd&Trf<EfmPoBqXl3zb|JrBcsvfq
zcKGPoW?3$&iF@F*$pZZVjk>Hpoco#6ycPBn0PYW#y~Y$53CjaF@E~xi4;^o`hm+X#
z7F%!L-y2QoE#%#GYJHe2WdyqC-KoMQu|Lc4u44=D3CjV+_N718r8()rswCp$fR)d-
z-E@%#Tp3)(nIzdIZC-w_p82v84}`5AS3R0WnYyj#A3S5}ek>`4q*4SFuV_#*pN>B-
zU;Ca{W)XgPibbBXSFU@OViv8a01A@vOHZ_wO_nkZ1hc-xT1`K!854(S5v_Rx6LLAq
z!0!PPNOH97?me9nv8eN3Q1PNcW$kTj`W9)W7-3lFH59nui3A;>r)|)tv-C{MVTj5l
z_6;SLCwB;(Dc74T90>7h<&u0ej?Ob*8Lm=MK36i>#&!U|9kt>5;yo8~zg-#df@V_o
zDix6=Kr5b_g2wvuk{=>69>zi{wxdyFwTkBuKb84v*2x9GO1A&iNVqvE-IY|?I?sW&
zo_*rZSF*fzSSwW!Jh?$QZwx4qLyK|D=$ji1oJRO;t{q4C=4xMYeD8+7=oJbs;xp^T
zc#uG%;Z9J<Mw<w|_%9DJfe}gIJ~yMz#&8Of4lliUusyor+l^cC)nY0&aAF|ibFIkm
z*lpN7%AYmt5sMPES(&jrrXbZ@*V3u-bkT8L`KVLfAq}ZLgWh~7jsi1A|7G;fL?NvB
zUfhYZ<?8@F<BZSy!7M1y4j+h&k=>6S{eCOq4Z_+Zrwwm^CW-x3me$LGxwHO&P~PVX
z4+RSm<+M^(N)Mptx^GCGBu2q<<qPY#{LA;MX)<A5L6Rr3Z(cJ!ub;<}b{HSPG9V79
zzpk3lHQ>W_Ce``IRx9OaD$B+D^}IcdlWgxC8C}@hipQjm#RZ@gP-_$#QESMLJDr?=
z#?#2^mC=LePlZhgGFuS!QWvP=-I2yP8b_FCW}5d>dzsSC2g7*{dtQ@lB}46Y*ME;f
z`hbZW^F~}_kS3+^x28c{KN+XhAfjt~-9g%k6Y%uoj56M`abgsze$i1@{dB=HO5Aml
z!xy&NdUGn$=JVRatcD$biH8QRji6Chb9rBS@nL020Oh;ZyG+;8I%*o=>dNZo)3IOJ
z%$@0B<UG8e6cAEGeU&D8_G{<QK?zfZGWO2~wS~JkdTm43t&q{|wuO`dv%k+)tPlg{
zaODJG@tey?VdA*zO4d(EwFyiuXf{6Ij5TQBpW)MgxXGnMpBWY{6Sw%Cg-NT_h@>K0
z`6n#7LqgOp7?ozCp3){+>k=3)Nt|@QSH%A6p>Pu-aoZ7MW{aiW9Sb~7eQ;$zhe0s*
z!ou}aDPwV5#K3#+rRa?_c4(_El8VA77PZk&gh~3~G-kQCAX38z>DK^BY*mS;8yAuz
zJ`B~jNX+^8);0fEGGot)VF`Q2Bz;b-o#yOq*I{LfFO5vKw|mf%gBHCZ*S&G~<%cr#
zfCSn4e!hFtyQ!rgiLx(-60_I~bfccGKApi7SdJZHgI4H1c+a~XF3z|vJM;knvCBy=
zN_~SgwarXvUv%?({f0-@Bp4Sa+^((D9Q{hFI;!7!wM28(h7YFk_PIafGU;Bj5#+SA
z)JltStMB1UKVMKQBkKc9dy8(|u5mRO(vt^JFK(Q^Mo6B`wcXE9kJhMwyIY4i+m-==
zxD7?VvI+u#-4STj?FnSBSUKawcP?d+_*p|S%S+z0KU3$migr5qNPPP<bp-|@=A%95
zU-`|JyKM%^r`YU4Sj+ig84rcp5aKS0y9F#k7v=B4=Ld%d<(f;w+c2|H3QH(P>Ftc}
z5MOM<L|VXH<(A5Yf|<s4F6SsS%eS#_aXs(;=FB2eK$MiGDRWAJC<!R?_^x#1rvHwu
zMu^>R*Rb4iMYogNQ6!mVm2zZcywU87BKoMs>bs{A-4KjG6!xcF1Y{Ia7<&nW(8k(<
z{L^oR9FTx>)x>Uf19*awM;;SXh^e}%wB4Rf1lj7FFhl-uMI+*8RGS*!cC7Ru4sBjz
zQ|oaIW&1TvY^Def^5*uJoO+C5tpy^2t0t5^$>odl`+2V?<%-OP;+{Lthq(uodxu42
zRt4S-GU+W&Yc<(ub1Yb6tO_PXc&v?y<^we>08e$~w3gNcO=UL%o*rx*0QpdmQA5)Q
z^+bK+^D~Z00PWw<4+Ex};=NJM+HCHZz*9rnyjIu#nj1Kc^!>?K_>i*5YJn{Fc5;H{
z!w6R8PyYFlw3gF_eb=cPflU{*Fzz{VkEj+N)*2s$BBNDQ-i|?DxOQyu+uVYm4wD&_
zbHz0UivtZ7{i>@aqGv?I4fk`DarigFM_Vl58R~qQZ1{H0L-l28-Jf;SeU|#{K_<71
zTO&OSi(l8v$UU|<WO_dA_DCa2Z$GZW<uc1HH`51WV^KS7NXtqzC|8pYVs&xKyW_?<
zmtP0k3w7EvvMOaS01VPQSN08z2AV;>r;z*=fF`zb0gN!OP_DlpgimNO`3KK=Q3lhA
zdIw9!zp|ODI1Ix$55mrMTd*K{z)Ae$7mjBoGfcyeJw+3fz+mUbYcZHmt&F}mC4DKC
z&blwg*>WA-`-Rb_tnL09@`|As^*bT0)H_|Rz(jfi>AOdFpDGuCF){O1Hy`b7rLq5t
zX?d(}&-F3D8G?9`I6D41ZhN?wn$)Yh$8m4GkMq`x+$6*4rqUMrQprG6>iU=o$3nzp
z(CTZX8<Vm;S*6K2Z1r)T%SNINfUovbpEY>UKG*wH6&@qgJ!lpqHd`w;8-Ojo(}3|s
zMfY<?zve1BZ`rrbI*#wa%8~tIt+mbd7cB>BZw2%oL&%Wk*BHrMAy`}IMP`H~ozW=M
zT&E&a#)jhm!`NE~MH#+(<ANw44H8ldQUU@ZUArhMB_PsW(z|r0v@}R}Np~#`(%sF1
zba&SRzem64yz{<u&L6)SW|(1g2A;j2>%Ok*Q&PIa1pnT^*C{trti{m5#2Tv(_<-?*
z8dolY17fWFgLU}Fk@;9r9gk&seeqmg`5pf``mJ8yCGTo?zS2obsjxv1Z$BQY$PKkA
z)q+`hk1t+Yja+J@!*#;^Ti&P=AxoG!s6H2TerDN)kT!lWev$N!(&8h&ZguQ(_*q%K
zeXYACT<c+!*I}j6#!tP)QG~+HS={n=HAJU)*h43}zmz2ucXxVXNWkNqzB5FvWv-`m
zbEay$Wp0&Yuq@7>{iOrbr*gu7m~^ZH+LcGit<rzGUFHR>^uH@)fnQy<27oZ1_UI9c
z$D?HQ)Z0_PC<fE(FYiF&CQK;`uC7uy1W1C;=wkgmIp;NT_Vs{E{EcpLhZRnIym#TU
z*(d$D6tn&e7Xu0&Yuze0+z&or{T&+Rc$M;F@o#aahWCA+3};jh1ie_L+lW4EK7IOX
zx%XXw)~n^UExi3FEMQcg{ZhbJPX)Ix0{LF^!+oL<EEMIp{n&fYE`p&>o~Bn%ae!Zd
z+v3&wnjxiN!WnZ~p^L^`$YY9Y)|RfU_>T8iz52JcsG;32H`4&N3Z}H8G94;Z?NMNk
z@o=gdJ-J7=3-AKFo(ur1g?&$%V!UrLB8`*eB}OVZwSt(gD}zikA!+@WOQu-|6tf^l
zaPO_hwlcoJVln+sISc-SPv)vM8HeEgFa;QZ;xucUqLGs(QA#u`nYnEe-is=GRlPO-
zLn*X#gJP&oD!uYZ=*@>A3b$1E^SiCilRMa=ZILk|RU5!V-s@vecV(zhRm5`yVd2$}
z2v+ay-2!o%^KM8s<rZ1@mLQzMS;Y^}S`Ru><f=j&ELX;bk2W4(T$Od(<X~@NT0cd_
z`TF`oASy<|xr6tv_)yTj-?XWx=!^H<%i%qBiKk1XJJ5Fu)XO5_=mipL^td<e#K6dD
z&MK9zW?ozk1=zLpjh+}u9%NfWB=?x0tqLs+4K+o$B9j**eb><g`A^+8XTOe&KE@H>
z_-0^u;??i?@y0%8N^DGpHV~ku$gJM5D!0Y(Ki>s_Iqi^<-z3IRd;ibL{G*nVbgoj$
z-(SYUE%jr_37&6p*zMA}(MK5Z{ap)bj**z3VD{_ZRhtoT##MEy$E%#{-?bAfJ`*{N
z*l#PgwrDz6mJ=PHiB$x)vv{}MoVg>``&HwzQ`ypSMa#J=GMQ1sA(RreHeatG4;@5u
zPAi#w&+f}`hzg@5ff=MQ!n8lHLiwZj9!em3egSbice7!}bIhlizu>FSx>JBX*1Mqn
z8mAO5r_S9+2&-~GlC{FKKB-7iqD=KB`yu(<5|JX;(g)i^l=-3F9%H&cHS%=*mw)B}
z`8xu-OGU*QrXf80*7IK2b-k{8KX>+6?;G!1rSYAneo(P;Q`g0<CDvZM`2DKX0!dM#
z+jZtt!D5dK2U1%E0lO61S%28zVv7R<T~;#Ph^7jA%U7<Q`f=r{Mio9tk=&u@m{C}o
zs+pMPiW49Y4aAQLY5k7UFekYC4eScgCk=q3YrrB5x0O>}z+C|RxIj0fat<>I23SGo
z6UACZ=C{~&$T0-tEDTTi{&M;KQ@hhA`_Nr*p-Y@4(_l?8-fac`r3uA<-hbA=?&4ov
zwCS`|0rl^8>IHC(Q|?8OAL4p!_rwuID&?K}5u9KkAK|*Ic~Q`Sz3k4ive*zxUtu@f
z?P+vuS3yPZQ;<Qq@ur@kVlM0HiXKucKZzQuB*fWq_twKft0StfV#bxCoYP6!D=<%q
zNaagDn@6_HJ&D=YeImfY&G_{3bUJgkL1tonA)`G?{#OS;+&+<J%7!XYSdH;5<~v&J
zhtp@Pgjk}?0^IDO8t%YWI<G`vxeys7Nges^6)}a*4q;~w8-?8o6ZQR%6NQ~4ssp_1
z-TJIzfXUZwvoy&7RCZ2kDM@XY?QE&*b^iYKQ&w?bohN_yFH-W$0CvWLtz}pA0nG^?
zJ<tf4Va}#69^(n?&ObL^e__#h`bG@Y#li_weig~|1@}%rPwP_g+9P8M;FUErJ`jUR
z41_Pb3~Ve~pZ}yP!|J7u`IxRYyYT(~FzxWdAVJumB1Q1%xemvf94&o>l{WG0r>(91
z@Nza+*ml5G8Zb47cezFL%)9f>cHf-Oa*g_-kw-80y{2WB#0`Kt=k3@2$=s(iy5CbK
ztG{-#H;z#O*@yCIR3}VRIZS3;APyV%pELUD*7V;r%y?;tmN^ts4J7O5&fjk)_g>7I
za3IWA2VDC)Oo{df8itM>04dRbZ?9mJu7x)09B&O=-zuxIC<A8-q~U%(P}=hT=SSxi
z|1}Eu7k$?3sw3SW=%vvZcn35T5<ttKw;l#On~{KJd)=kqUR|*?rpvZyXw+y8Zr`G+
zbTamL=tthbt)9ed{z?Z#1b-}dbDe$ZgZfmxJ|RR19ss<VOIatMJ^z)L(bh=D#UIy0
zVv&J}CDAv^PF@4)#U>-!rLM|a`sdkS{(IZS@&jA=)#jbcq!JfjRABd%pqnh4;??4k
zCmM5bQ3QuY`TerFScV2aXO2EHiO=$&hH89%jP<c(TIsi2M!4R4mI^<G@_R!)i0<!4
zKCeR?&!;qa-L9;3IEK@5WRl1WM||5ZWVa9P|B7#TOvgHs0MgmmU-p;|9;WjegXb&r
zqH_P^dZIqIn)#JLimkSo9@W8UC8&+0L@n}cT<-3fO5L9Fb1hD^L~&~84o~30R|F9r
zUj7})J)2}pe-H!1sOQJYJhk!qlF|P9b4{V^kmq5UjcAD*Y#OsU8f-Ui)LmcPy)?8=
zCO6Z{01E>N;(nI*y0<OD-&m)6@H4a<k)lbE+@s5*J~H*Y92oR;l?(DT;i3Pv>+!RL
zEEUaIQq)0`%@k<i`PPaTnMc1a#pwxx7~PQB`4j5kSJY-yQ_F^Z4vWcQ<v*uW8fjJt
zGGnlU5>2%GS*?b9j>Cq-gGIMHOvjBYH~oz}pZ281mAKHM!n48=;Z1nVPOX(jzAJT_
z5vit*`|yirsO;qbQR|qAes_)eKj`!FH^ny}0U>r3Dh`O`Ow=DM#0S2^LL2>?;L2XF
zg|(3iB>aIl5|`pE`l_9d#wfm<#~tFZ)hYDF<0dBQr|ODePwH3aw|LLrg8NEU``#r9
zNB=-78C3<J_`{$*kzs*hZqduay@71vs(7ajB|(9|3jjx-U=%K9%*RTaI~b2QMbTJz
zCk&-QyjAU3?~Z$00~J2^;lG~#>`O*@^`$SY{v=A;9?<TbWihq5nM4JPt&yn6%;{0@
zV1NnjI^HvBRQJkJ$wT(vKw#?u{8^Oq#rtb;t9mAj<+X?&T7aDLI66<A1w%CRx*KvI
zdCscq!3OX$`BeIyXkLjbM$BK2rChX$4;QK#IUmzLRN)Q^IVAxaFGPta%k4A_UxQuv
zDzjr_Fw$}+boiSE?usHXt?CDUX+xh}wG<pO8fJDY-Fh!U)A^JC0%7L~@K6?B)VJS0
zX*kUma!Y}@Pi+}6?VsfSl3|cZue*=mEtbFQ`eSr8-M$~x%`UfBvsLX!Rq1Y2O~Bl*
zuEZpaw@jjO!X~GN;b3nzRZbD!&|7*T1}&i|N&BK&zEzM_0$0bfGz(Z|A-R1cmtkLW
ziOVKPvSS3P+dT$QEshzE{gDzap%$&6Ga|;%W!==Sudl}#-Amrie+q;pJB^?x&_n|i
zy_@3ok@dNq{7f|Rm1F4}e;kUh)qp?ht@UkJ8w81(7T{eWX$c~pBhe_eK$WTeh=cdr
z8J}I!(#IP--<A6ogZnSqQuyN4=HEqiO`p8b>)CogI?`5_DppG^w5^l5ev5&4&I^F3
zTA;Q}XoJm~Y=V`ni53~58yy@^l1lf-0FXPb1+t&-PG3$}V;DnQY=L#SVJNXp`#=I$
zC5BK&D`OH=A0R1fXG)d+{#pdt{JF1dgBf3Tf;%06Mu@)}qnHi3dPg!s@}AZ@1Q{Wx
z3<ysef6;4)^E(RGY){$-odY26(Y(!qF!HE9^*1B5{pX|+jnO-+C`ym3Wz2W1q~!%F
zrP`@mFI^=_URwG2a#*c*qrVI-A0sUF(xs4cI%se_v;eZGQx3brcLkfyDZU+|Jbp^g
z%301fnkS#t7Qw371U$a)E68`RgznD9?{+y#H8dCT@14@{7=ZSWyCAo@+aq98lk*dh
zo|dj=7())<{e#YAJ#(_%e)-1+!6#EbNm*`m2iBlpqr6FVUqSH;HzcMlYa}#DbWrVA
zXV=`eTj}ONgdBcSA@EkB;02f>B#UGs6kb4=xmTxU(6Dwk1$N?GiwjTNGOac@v^WOY
zbdgc;dV_K}sGP44ET(dvgi{W%H@H_UoQNJAl1Lldahm^U384LV3BYO}GgcUaMvf;V
zdC&X~b&wxD`p;Hfo~oZe9?AssGN#pUNcsnkgGZF;xOM=6efAsp^Xs5)pGVt2fF~3L
z9kE@-5W+oSCC#qI91~Dde(lyn{KxGjFKn=1sS>RTx|J^DhI?Wj7QG@n!t^d2j{!Z?
zZQ@1L+(a-TAu%iY>+BxTL^e}1<9kmG78g6zAA$1XOpjjiEWgQtGMCHW+qq4zKcUe2
zUJ(<P9Q01oBd{xrQ=+s{FaN57LGD%kqwQ(RYhNkc4;OkKq3EE_PPms4AThAIi3_q*
zjsZ{t7LeNA%Q_Z#Ssrq<>j&}2M}&ib=4Q9XRv*D=5v!8>k9BUbX&)7x7IHTOzS?+9
zakron_WR4YVCkPu-aUARy#oS;%6f3wqYxCL(V)|><#LrB2rA5*zr4}Cb=ax#*iAQ>
zXJLgURv+JCd1x&Kvb79oRPB6_nAf$(Ro<462IPEuu|#j3qN_#5oo~~O0f>2dfp%E)
z0E0oNF3AsPhr8@y7wHM__P^UpkrxYP^EnBWpmCdNy7NO2R;JC@{<*VJ9n1OpI&k$t
zu-H8JQb;85dPD=yWgf2Os<FFo;qr38=a|kEbucyTtm?9DQf(_B8Rb6dY^DRy7kc%n
z+CN2_)lyt64cGNExSnkr-&$0Z9_k<P3H%&y$XYz#KY-O!+!Yoz2K`kg+SIvf(atQF
zjk^fVwrWuS>@c9n-K4VKWxDJiuAWDRFK(j&(T!Fjck3<2jBubIK~w+4x*SXO*$X|V
zi10D<la(iUjY5tGd!UYTExbZh8sW8oo>nnAG;i!eVo}BAcbs^xh#Jeo+p8#T_osia
zUGBig-iN^SVG)h*qk1e_=zR1sO394OKuN-$$YiED2v$TdOb-Ht^7gw^Y--NF0rSZe
z)Mf#!rU209-w3c!k|3Vs?Z8sbv`DqpzTJ#8*|iJ9%M@4#C{OWf1eF&>|E0;<z)s20
zC1ZA}-}0;x!_qq3#t<V%0(;*d<4Q=7mo}bAP3;8XgJvwmQA>uww?&B_#02*(G>+f-
zHmNgZ0K-FJYhGkp5)jP5#Pa47W?{th)#05t9k)A$r6J|VP2YEei6!D2FwEB|L|m#w
zP6xQ8_WSjb=McPO?>ZlWYucd*&%5(!xgJt_SAgRbHO)F#B)aW^y%ea3pY}UqxV>Y4
zG%KK6ZtaBxHDEN=(&BLZ@o({R{m0?^hT!PB0(4akY$)JU))3Ke%YwSZ@8K^0=mO^Y
zMrRujpz3of!4er|qxVkx-=8y|Ohq#*Ad&95w?riyeqooGHZ^9GEY4I*^|}mYF6TSW
zHCE@KF53xtafWG;($1ogSKfG~FB)^Uc#`S>mNCqOf*AyTa;Jdpwec%TwgL$EX;p2F
z;aAbL_Ve`ARpMV}SEcj%B%!D?Yx!_5S&?QmTqOr_8Lq$Bg2^ORAKjebn+V$N80}ir
zCe%DE6kl8+qwo;JkU^;xUi0R9So2`!Wt7T13-@?#XFxQjsL~fT?Y~}QlV34`-M{pc
z3u0bVLjsewN%aE{Xrh*rSf{di-TmP{v_VGYd~?+&vSsGp^WG>wB`HGG_E=^Sp%wmF
z!^|Ks@E;NP_t>ZJhokkzM6rJhTVl3*4iKIT7=vV3{4$u&(D4oyMrHag=Jr(O9QkOj
z-c$l&U%iYj+8j9&-V@-lLT80@Bqr`@g7xqnj(be_YIHfbzL4?y7;BZ4@nT{!oWnUr
z$2pm$JS*I%TrU_eKtH2hPO$GyvpM5x8+v5;PT#T+>q)?`7}}ur@7%O9iC5&W+!K8t
zW&cXc6*;s0kCEic&U;-PWi6Alfa}0;k>;_60`w&CdK?aexW8FSptOe_3n1f(1j&1Z
zh70$7bo2dsR~5*VVpEXD%SmAKiX;2ODkC18l8OolApvCOA7%|Ty%;INXM!-2t1oqm
z9dG|gqpchj;z?t0ljR2D*P$YJE}~jodX#vPh0AISF_P(~h@xUGs}aB6AO^KS65dqm
z`zjol^HhL@#zZJFr#plMDD(D@D<*(`M%l&^>p^gDv8&b=Q(*XoU6__bhhei2056TS
zv9RNRv%&oK3sA{-|E0b4UVQFeaDooSIMWoAef?6%n7|Nh+bJETI>zg#%e1KA2pOE(
ztuR@CvfXa8!NDbBZ49hm0_#=xqD1?x7&K?oi%c7Db_(w^@}Ytbdt0E3-HgJs?YHyK
z==WItMa7Z<{Cx((jc1Uv>o&WF^3ouW1wj6Bev$P^yIu4!oNf+Z7LJusJ@b|NP$;QL
z;^l8UWCo(udpWH-!e)s^eiVQ`T5D}I$-a?CLG@BM-Dd)=?h~rf-R-5^kYuw`WZS;Q
zk^BAi=6*0o^O0T;)~v21>IS|j%}ZZ^rC!_gm5fhoxcV&~Z|9X5zMfwa1Lmgpq%_Ty
zZ^d6i6Tmg4DzKTley9UPKf=rm+^?npUH%I!wAXL&tvM-m>Qfm<Mp^(Bkdt>P)Sio7
zQ{<@)cHzr+ZCHMd$v&FzDSLDy*qpV*|LRoa#v(<~SpS&6_xcrRc~g!3)<~(@D8S&=
zx(mQH{Z;~bxE@YD>ZtzxP^V13dXDNfZ`aEZ_pM3$uN|x9W@4%>{m3tZ{2N}h#24ea
zv%vDBQgYFKm&l@dpM45i4P=18|9Ci>PtaFtEyksJ?n&}5_qaYR`cGB6mVV&5NaEP{
z(X*uASMB|jZk@za<fZYV;=T=JK9F$jbGIgOsn8u#wz%Lp+-b;ASZQ`~drZPKpA20-
z>tge)dXmqUOd~j$lH$qJ7&Cua&Ioh8;heBXP|$HPhE;4YddTN2vRL>HCfal1bsgt#
zHv7PwwjlX)rKcM+yL?oKyUS@hqyt=V$38aJS(XEFj=D9zfR7f!HfkJW#a|1<c+{H;
zd2AqG^7pN7o#$$JPj5@_=1J6)EQ%f4l@_i?*c2*5WrgMv?JN$X+%A4c@m@?~2XYSx
z8h5?Ey;HHE;9zGhuBhPA$f_)!tM{8UPrY8ewLVi#;&llbOaR+j2fXE$X>cg>-&fc0
zQBHWc*2%dysccDtDuD0CJfz(nu-c;?v2q1NDdwbDjV&mKlg=yecEAY5n28dtes`u_
zLkf$TaYniOQG$4gRhkzUeYkqHS^Z@}%);%2<$>+TdrodM#YF-uu0=HfirB}fv%Ryb
znbD^be{`_dvsgIii>a`e;s~})!|RF_AcpeIr03oaOwZmq>8H9MwZ>Ucs9Zw0`EG4x
z)ZWF9yX@NAc9VN((tUC>Z1)efqu+CT;M(wjj?K7($6=1PhppWLs5Z7~8>8Ynj*ElZ
zYOmL$na3Ay9ZRO;LQDa|>uj!=iy*2^PSd%b&Rk8@$!e{tw`lpO;Q4+_oDn#SP4%du
zxK4V~oey_xqX^h8$*N|)1q^Of-w}*B+T_S-WuLLod3A{?{Pg^*(M^9Gh$roKwCuei
zMhyCL^*h7({-!KA%m?p&W?U{wj{2%E%w{>4K&5{s53gzE6`uPFC4QVBZwKCLW_v8s
z?f2+aG)QA-2Ivw?tqcC+3L7Y`br&5ZjXfLmkO7{KnUNG{_vN)VF(23hTtR{Y2^9x~
zRvGYrbhiHHryo;zi}^v&i-n5e`1xbk6pS-7EbNt$QvNEy;-*86^B3OTgt%mj?61Nb
zcvm7gKPUqk9YlP?0=ejryB!VxGB(_qZ~c{!2ox&4ns;bNHDAh$-U%305efiWT$}HN
zO03+d_<}c<h(-8Tb+%49Jt<iP*--7N-R<DQXYA}Kjf9O5k>NM+1xYOM!_6XOAKg)b
zi*31p^IAFGYu*|Gvb-$Osi*k4x#;O3_<76%ZpL-njk2i`^;Cv1g$19}!efK6MJgC@
z0~A(2r{D7lX}fywb#-#s&VF2Q`KiTNMa0^0CS~F;Te*f|{8O|0l;?v!QI4#{4NXfK
zj`FF00tL^GZmJ5{Bb#%>RrbZ&<bO#QUJ~i>I%x2xR4&%RT~;S6&x$_SE<{a}7v3Qa
zODmB<>|~n9fa7^&{2B)BCWq@?=M=PNtE=f|gCpKD9LIHn+I>?#pM({wmEG=-OL6i3
zQE_;W^I$m-u?AlxB)ac6IHaa%>ftB=@*3_f#!pI}Q;|u`8sgiwgZJ0^2=LW;rFbid
zcjknj=n=uzZl&}X9`15Bik$X!8#4rq<6(f&h%6k2dFzb-d7y8mo+%`$Bmj|AR@?{3
zd&e0C5KO5~nL3fQ)4UJ?DpZCJ;vYgwpl500G_bNZKoPwk2hGhwAA33CTR65<3JajY
z^VPWD2T&)E^NcELP|wREtKS98wAQ|0G>`;+yFX69dN1901N6bLxrjb1=5DM~kC;hW
z>zUu~b8nQkYdt0UAu)D=wFognZ(QD*d7EHQ(V-pgjvl|7l{LDJ5gp(nz==5VW8ia~
z8QTRU3RshpO#q?F;J2_0T*ABI*)%0RigKJJ)>;Bk>Y?Su{_lm9Po0;%(2-Is8L2Mi
zfEiHf@eySBffaGOMyR}{w@Wwlu0*K@K4(@~1OvB|4S-ItOBYPNzP0a2H*Wa1?Y^EC
zEERFTIg?b@)kkm-eSB5q)Ft)qS|V=Ekb$$<j^w7~$2s5*1kioEe3e#IBPCF0amNV1
zK}s;SgKYu+?b`cnGkY9m*tbgK!5*ov7+yIJDfRGiYIOSh(w4JmnUsnjl+9*YUGV)d
zSH?R<8g-H55zk7ya=ocGm_4GuSW!)=`%>g+*&Q|p2yg@NjGJ>Yx%O(n*X%s~N`$+!
zo(JXeY-=gh2&(lShK1AS7BSx!_e&pBN1|{?Jnm9pgH++th5gXsMB%AFjpx6Tc|{fq
zN~w5fc#IeS;b!xfZ6{JxmrZtecMleY@U4h(!)6WI^Ibkrg{40@xw3_9-J(WE9{|MJ
zUUd{c)b{0g9NWj@eT%EcL-Z%w?Z;mtU&nJz|7qS+7XSVtW<ANG>ZV;Ce(?Pc$6K0x
znLL4ikIr?oz^kslSQstwJX~l-5IW0r?D^2;3f(iW(|I7udZru(X-)&EJj%h&!#2NC
zT9E6X8kYUX-9T$Gf2%R45OYb(y<!cyuIpms!+qR*ILIFnqmsJT!&+_lGkUd4#>V%|
zlL4kA=x6~0E@#XGky0;3we<*E%er{HZ`)%y62gGFj<@|`>I><5&$BH|hd$=%I@qht
zUk&rd8PW^Sz9TQHkTw^yHBMi@fW!CsU?$UdBByoc&2&v?EZ%3RX=ZEV=Jn?MQN@da
zItb$|(lqhra>X1lu=^$&BxC(sl&1)mn$jL)0&25=IKVa`{YZ$d$=yoW;(V(RgaN(r
z=?CFQb{H;pGhh3Ll!$aX@k_shujR$pa#gR2_HgI=N-4KSM`J<u^w544hHK)-p6kz6
z@qKvj$V-viVk6_j{Iw|lO}Hn8VQIK3GG`M25LoE&NyDoJtpHn|^6AhA0vT^YOq2*&
zq70oVd2CUy*WP(@7tv<{oQ5h<eeXxfO!TESNPp;$ej0<X(Z`-;G@S$wyvMeiS;?xG
z%9X-+(-r{6tuAlX5nRVRUu&hNXs))iVI9E^Vjbb9w~+u2@*{?Vm#DZXKb|ivkcbv1
z!A}tAG7qNpGO3)MpJb42mX8%lB%pl0mcd<YNVw3|fHfapp$1LP8MGm%TBEpl@XFcW
zec9s)zqR@`b%e^LYA-s$)1QNlPi*%qkJHp@=cJsw+Jjy<Q63w{He?92MH=%P6BEN-
zccu!G5V(a&LXNhuy=YF}9fbS=&X1b(f~bwq#9P~^Ufa8pIxi6R<u>xs40Mn3{n%t?
zq<jKO0ZF!1%T>}+&EBj2q|0~k@YXo=0%744fyy!+gIHouNq1pIqxHws6_spQ9XLOj
z`y2lH!T#%euPTdW?e*EAYA67;58?i9@6khV6w~cj-hT7m2TTOI_5SJ617M7BtU<c|
zl3kJHDgKM!NyU9V!z^OZl+$)R$PAFivt_aKplMW~e>WSJ>w0<M!9#drdq2w_1k?Kz
zLRLOgz_;D{e0TVP|C*K~P_)Z__<XpWUe7SBL~EgRNsAKcobo%3EpVpZMDpujPnZCM
z$FAC^G?JoLYsgcPP}_3)e5JzJ7JnK8Yp_$OaSWaIVH`aarRy9W(|>yQeGA3UB(_S`
zL4Q@@iGVeXiwFon)nhXi?zi;>qskZ#3jimVVl=3e)mL=9O3fCLY3aZ8yMQqqa<#bT
zFEm>-Tb{-Po%mMng?+s!a?r$*B+(o{`s|m<pUKE>syYKnROj{UhrKvM<pi|Ek|dCc
zR?Sj(rgZG6h3o2QDbCQ_A}Fd%DFC@T>>fnKh=pQ;z?&G{)A?#fk|8`)0{O4UI8aDx
zL8gsIX?Oi0N|8g)2uy|=#&W`L7+JN3n;a`rZ}+0J`=`mSTXJD*Vse{%+U^V5@ZN;(
zjXkWG2y5y?&rf+$JJK&lpiVZP<_)9ufp_RN3zmzGS4Lwc1AU#OAQ%O#>MGnY+K1R!
z)$1v<uV}j&8=%_NlRd#McZ!&)Ib>EO64*-L-|L}s)LyCXV&Mghpofd5rJtxdE?8U)
zds0V*sNLB07VwHw*$3Xv3-J^dzC=r`9IBuF<L>xPIl0VwT?aA8u3^}ty!biZSO}Yp
z%7IkVbGMlXTFsas4to5+H+k>g|8m8!wKd3!>_XSkeE!X;(@TqGMYM=fp1(DN9Uq9*
z?fs(ik1BgS61|r87d>1NCEvugz&;*G3!oL=44`jIDz#gAx`l$HLkOtOshju-CXQDz
z&d=JZB2onj=f=e~*W)1Bds;l(fIs)~x(^mSC|{8GjOZ~HVHYqZgHO1Wk6B0lI#$?b
z)AIsqG@tQI)xK;bIqOBoSLoamX?`#76jH%AG&mm^v?kiG62ISYTS4Q0?XA=4+7a~1
z{r3O8BqrQBCi&&H)H$>4mR!m)ElZ%d#oQU|Qx`4ztt}PktYFOaYIu*V^?M;G;JKWA
zQV0oo=iveWXZvkwUn1Hbs^?*<USqMLLbsL*Zd$>!S)gn@vI?hwWAK@Zhg0wGb|RPk
z`CFbwA4wm6MV|liU><BQe)T^cZ~xj=eR28cnw%0<Kq*g)mZ%Jy(?}bHpSLHR_77hS
ztX?vS?amLvv?~65YQ8&<qYH(vT{ArxLk3I0HkBu_W_6Qk2UdtThVYey?r?hT`hE|}
z%lw8c_|M|G#wMQKs*-9gV~rZaQA<GFqU-=6)ICr9GPUhkkKkpp4c=d+ovz)F^1y6>
zrQXW7aj)clKucVlhf6KGU--+Uy%{r-aaen}wk>Wo+ZU<J@hFoy+~rGIg2yFBe9*%U
zMO>&(Skxw-ZwdXb2|PX~Qeki}C_|MY6LXQW;n^P%lCRdV6gS^e{_(L-I$e)~8D91C
z=*W1ltQ$5r6Al|H0u?RqXcc{|!4dIS`j~E7=YUs802l_FKWG)V?j%eix*Cyi8S>*g
z|IPlz1!X~^#fO;%&h^;smjqt^un5M+J1?CKqm5ciSnK??1x3ze=)ICg&1cF~$i99O
zkM-9vWy9QEj;eYYE>?%36afVI1+rG=4#~Q>PzGZ)U+zNa6@opm^1U&+4-yyF&az<V
z=OigLmnAKTuBPLXF%>h+*|my(+0Xhz@yAh!`;(92^l4klV2yrYrxo@3tu!qVy+#~a
z@+tg&s`#7Wp9S>P{hkIGGFA$TNla)`?*Lm>9@hN8p9t~v61iGpwV>$bK$>wk>{XOs
zj_ZQljtpy3mJI-w(QO?dp|=#58ZM$~Cj-z%cc9k|r<8Q|(rC3}qYLXgALLtfCX^xj
zR#-e`-*Mp*xKq~dU`4-)eZ5{>13uOeWk@M$o00(x<77|Dxc3z9@--S%r!T7APxGgZ
z=F&K{&pbJj-uJoQkB{<d3UPJm(RskdhR76txMt5(uuU>qXcYJfh0Km_hE=Kruq7b1
zn0pN({t<M)cw_vaX&=ce{HIQFGLbrw`{%Dr6KapH^Px8Vzufzi(a$D6Ccg`iDCNzr
z#s5QN!iwuMs}LtrPTz;8^3G#Te3<l=8$CH4g9|Ql%v*aE=Pp|z5F`+o5x*6U-qaN&
z%gaI>0RP}#)n0$Ys_2i&@?1AV!R37(%8x6`&p+sJ8Wk8^c4&`+nA5k!pEHg3A!FFs
zb}$_J<dTR`C$*W_lZ93@Lvids81jdI9?k@JevzG<U_MsR1VTGba=ZQ59IhfkHrScS
z3=w$6CXs_{4eDT%-+f=iQYS`>(I4Ak5h6kv&hmo(OJF5bw%i_|qEhdD0$7>9v--ID
ze~8Nem$CnseT*NX?aw5o)zu2(TaUAdl?_$`SfC)4n9Re>bv5REmTvi=axB^rU-|?9
z!kr10gY4yYOvwrzG^e@tG*k@Q7h-_<)=&`khhlCNK4;2u*+0bWL-YML^r{ZGFB!JA
z>vz70TO~wRDS6?K7mW*Ew5v!rw^ZKgP*=p5f2020;&koB>p3L%Ou=Y2Pf5jm3l}ba
zBQhM6ph4*wHEBLK*L{6ZIlZ4HejmkW^dFSi^6c)^Gr^EU+uBMSabkD2<@(EhZ+ti@
zHcqs%_j@6wWKQL!+7g_@!+a_K!ZwUplLFveW4uS7b@dr$9sUwjF}u<j1*@jkatm3m
z!RD3VUXNr<L1?VX?n4JyEnRJ-+0*t*h52YVKRcP%;xt?gz2<`d)c;7?8`XV3xG|KR
zZS?gX;<|}qq-5QEEcY?-Rg<KaSY`UB&afsu%fKgCaFYr8u6U$DFgcIBK``*u)cuFY
zY_G|&eB?vW_va|@XdbQy*F^^YG$9<9wXd5u1%$YCQk;jI8ZJ{$50VX01%8c_-@)1%
zK*se|3MS}@hN#yD<56Mou)@pWl6jL=f-l0)3d+~&cV)2~I5s52B8>fU?`&sqtVf5^
zl3?&yz%-3k?RpO11jAeNBo}g5*tQy~US(ngm}kUjdERBI9Z*;bch=kQ;&<c20As?U
zG77V6t>6G$!+^w;OuVbT>S<=Z>86i|!;AM9o6E=X6vh`>Vf8e~EJNoCsTL2BHoE+<
zVF_;IWNR(@nQuhByPCFx4{B;^(|JudJLq(+uZFYdma%FsDrNU=<Mu6U9fQg>q}|sJ
znH+0qlh!3f(!x^>gs6`{x{pmVU+gWGKp}_>8Ip#nlC+sUfqMI&`Aw^Tv&?>f_4$tR
z_H}%-`t~Fc_w`;O8WI#DT50*n6v{u7NN)2i)Ab(N>PPgO|5Cp{botp<IHyDCx>zu^
z7l&xtV_6hfVz_Vz-OIrdCR_?}TF=R^!ffv;;YiN9>xf^@eiqM>9rnM)Z8I3;njwN_
z5@h|PDbSA5X}7gp=Vc8``SKgauhfkBQNKQ#uH@r*qWGn9VnwAklSqL({VgVP>qePL
zIU^Jw$yuL$OdiGj6(*~`WkfSBE=N5mU1ysXVe2y44buAzBxXwsPIiC7wJ}g|U*V@e
zk$1n&hyxw7wT;r@n^<VfqmeVJOVKClGY)^xW>>$#<xXE-`5!@UVWP*hR$VWQXX<xL
z8fF+TfLs{_V3jnNcYdZdfyqs*SAyeZH0iag`Y<^aB$@L$)nr)Y+G@{|--%+k!X|vi
z^Ejb<F9~xMnky18chbKpK?TgXA)!=UFX!SeKy2!c6@%eTj6TcT!FWMJ9p8cgkU}aw
zUo!u|+|t~9@2qf3(Dp>t^*%^PnH>HLlN;ZgSMaP2>S4Kfr;>CeIoQLZ_J!`>f9X|p
zv3%hmaQ*RsQ<JMX)#TxNL2bQF-2rMWq-o=l<9d}Rnx3Ex@D2nN$9kDYke3r!&`QW%
z*W4v*Fv0vc*X?;$7KSCopK#?JAP}eZpy_h&^+p@kVOapE#L45Zb~VI`&2c$Q$aZ5c
zOz56Axc5-gVfJ|WI821=oFQ)B@N)6tJ#R@-R148iMnEbNHHTXKH=E5AW8dA3x@+K{
zHaQH73>%rIxt(N&ke!x#ia=}r+@uY}I>?Hg>l);~SOezT%Eh~biV4k<MSQfxnnMa*
z)~cnSjkDJei6$xw0{NPubGWiX@Isu0Y#4SDzrbEm<-^fjPG0pxnaG^lCT_n*%}_t$
z_dQ#Y29t%s+FI*_Y~^AKm$I1Uzo>3i76PER>>O8NZBl0%`Ct-sA|6R#qB|6hiH>Mk
z*Zv}KkJ^axH@E|%AY|>O5E}@5l#Qn&`hjfan+h)UM=jbG!CY19Nt8zURR)vteFcFu
zDAoJNn~-C1@uVTPOD|^9s0{@XeFEqj3Eq1bJHau{f^2>_z+)}3MW1Y1?bTNuy|1sO
z>L#W;Vss4V6xT|h?4m`e2wrn_>F}P%Jcit$io?irZ?^Ee&YrwYqjW}jgSTC_`&$RF
z(vflFW`eE(*&Qkdc7!)=k*v(7?ki=sCL&iQ_+baZw*;~oMvU2_namC+*t$EJ*+eE4
zaj}m7OPJg%7PLe@OKOnue3fY!awI;=ARr0gN^l7m=vx@X_BIC#dgZ(vsHPn``oSoY
zxSfQlg9siLwn%Aks0rmo@G{<;ong4Ht{dxn254QDZ?M^f?>TLrlfu^GLOt87R(68q
z+Sx@AX?8li<(6R2_CIx(jYl)tCqgRUaCx@4pL-cyljy1Bl_9X{(~dspsXF^zEEEmv
z-e*ieha4B@JRdx)_DFBafpgyUbRSx1aT(0d<8ZF$n5G}$|8j&IR_$HAGdHU4U_vS@
zcYv(gnPV32GYzO6ILwu;5%+MGs$*EK>1GS2&gQvK3E!)3(J?-4)~Pa=uKO<al-bMZ
zCcd;i?Bhe_R8##&ON%lQ&kYE4tUFDGaN*6)`4ojNU^}e%nb-3k6=Bh^ip2;%>CxKX
zge<bmnN>a>1oqYN1qY#W-61!IhQ6`}?6wjfd`o(lc|wC5;1NciZ;y*4>bM=>)wbjK
z94rXU;qU?8Hb@%ux(FoY-uuecFp5urd-hOS(<PYa0_)BM19k8guf%LH(Z>1DUX^qg
zK(6%E$WsipZxQ+xMO9~MQIl%QwSh=)f)rarwWuKO(CY67*ISN|1laTf*DSU36o^@s
z`UOY;DA!{^=*8Qjl>fI}x0CvFF2_^hf=7<@&I9~lCYfQRiB6BK9v|spPd}0m#uf7H
z9o^;-bPqL+;jlCCP<PA6s}bdS&)3zf1FplHShrmXj(7~wSA^JQrSk9OzVuul+Vs=~
z5_;|PN|OuxTI>>;jypJwwDZ@(6*Q{diHoPopP{dhc=MxtX7OxOf9^r4ZbHs6LR_lZ
zDiZuooL%<q9d~LnzV44reBFY4WC8H=YRavk&2Kxtk-^B9Q(w}Wcg)=<#DBDxBqm}N
z+sLOq&S3E8El%q)V1gWLR<mjlhFd+e*CNLU(mA<l;#2$nN8UDY=F>l6Y&GNo8NO{w
z6sR%}lhbge>7IS>H|L1n3#3B$%*FDkj7$=5<@M<%!T@jH0`i((>=+}LQN|`NRNH--
znS#&3u5(2?-)OZ=`dPT_Pb(k&jkQwf$qz?ZU*6?j($P%LlT$NUmO?zQoa><d;)kjB
zqW*%Ur(gZq5vE4#h#l(wD&Pd0@7WdE>$o=Xo0hrVO)2zq=wgQM03M2?CDp~{4;EAI
z?e-zvuay{VTarxisN^FbRRFdCAkdR+Rc%YIF4;0mFr~$!+Z*8na;^vT14~7AW-rD?
zbB~#IzZ966bYJhV!UEY>>@~7qEEWA1h*C?>tM>cRQUGBKhfo^=1S|QF!h`Nv<A*lh
zd8_K3d6t=$n;i<FtEI{;kDdseJzt`W!X<^2OqLV;a-4VR&Z`Ia@UMmQhd#lU@g{ML
z>Mc|zLr57FgmL#gF86Blbt^Aqsd6V)&C-<?73VTQ$Hg^=%5zBNVxQZ>y%y*gzSEVR
zy`=HR#u)j3>#P6Mh^schCV+$G;Pezd4zIkbj#RybgZ;6N3s7knUafq1MXj?=w5gY2
zPQSe+jSbFEDc^}=EgnujS{HkBT;XTY6;9(Xc3w`qg@2Z4uK0RFUV-!KCy2^BwRxlH
zT2Jg7>E?TfLA-{I=nKPeNwZs@4>+`aVuhHasBlKMB{eb+zu%GC0^S+}@1#?al;1l8
zMZ-;~&w4Y+t7eWA-1UJ5!Q%tdBP|yb5mVd}m%QeaD6#7@1)rM^YZI!1a9q#g1hSig
zl)2t4p7O_oEMB6v<3F8z74r7SL@~B9vkyhbMZdd{|E~#?W&kK955!LU766}$cHvoN
z8txoDV}gje5q+nLIJrZr9y)oi1q8y6yoAI#zZTxvrcHW#O_30L6HBEHT=S^rb8jzI
z<J?4R^*FbK`OF)+WVA53_g!$R-iZyf)F3|IYmOb`EqWTuX+_X6{&G2w<MuLY_ZhC_
zT|n>H4sqV)ygeH2=c#6xn~D(~x2|r6Z!>pKA&~>ghIrKX-OtPoyA#=l!L5nK69wKl
zPfBYr4?Q&3hq#ppOnQz71JY(sjqiLcvU)L@_RPin*BtZ`-slAZNJbjg6gjtvlF(){
zT~;yhr{$))jNhNcB|Y01P|^0hlbp&DpUKaUeQPxEfr59BgL7xP<&V&wQ>7^zmv8W)
z(q6Mzg+I-Ru&b@k?p$R5g;5RotT^R9+`3$68Wfi{)efI^>Lk10{&5J4%W5t%FHr9;
zzKPm!_+7HUXDn3Js}BK*bUO&I1Vi_z{yq2DPk$Nb`)89<rBj&P1c{kQ`EO|C&GaFd
z=)=z@-GTg`Gh2=_O|0<m9D!R)lp}4l@kF@WVtgl9WxZ)6UUJ{@gYkW2cqUZX&nQ<I
zU+DmO(Rp~0;+!W8vcdt}xI*Wqod#cUCdI?qb)v&ndxWYca^>97QuNp5g0<1@w{p+h
z%Pih!-aLzvW6BSV(r_{R<<6!WJi2m}mFC^|-rhS}?EXp*Nbr&A{B{2~!H4rI$GRXA
z-od4u+*JVjtaew$$V3AA&^*x=P(J5IwcDCwFCWf{{!&PQX5>uuXSkLAd-ij2QBt-P
zQ3cP}B0fOI_`etK|NbUAi2t9%)=>z2BJ8Y}WjN@T3behy0KpHI93*j@c1ScmQuux;
z<>_*lApz>%#aDRi!IXU{Tai+C>Jv#q;kVlf(zuv>Jhk7H>n8j}EB9<<ph=61rd2)O
z*Qe0E2D!P8<<hx5QzF=(WHtE~{mr);j%koik8&atUjD7pDR}peb&0_rk9n}$TE&dc
z7-Z*tx*ELK2EJ>*Nx@b@BRpq3QCT`OZP{p9*_;49HSz?V7rQ!ChCXmys%C*$Xh)WM
z4tU%h=lWbM>Qjqnn#^0LDZ;<IoW(=@y1A6xn%0R-JmPr}V9&FJqpLhxYigv^<xRt0
zaCcgN7tVsfM!X1YScEUwlu0g?gnHagy_L|OVPxKUbKGG)2JiVmLTTecJyEgo@R0oA
z5D$RynKw(fa-OCNa5)}?i8#n148rGKj0O`8x+!h-TI1c;2NEh_Cu@D6cN0K2dds2H
zNs~$=*YdVKOc3_@p4*J){&KM&T8zS`k1g~Y`*t-?y_3)P)EeEfBI|Nk<WrEFPk{eI
z9OL#M4Kc+dN}xV2VMLRlzU+AeB+*At)r~`g0TXe!i<_=4Iyv~WO}Gw-<s<L12SjjI
zjPGW-A0_+|$d`;C>&ICKGwJ;m3%gxb-qXj>A2z)?&g(_NNJVtW&4fxC36PyMzl{VI
z4(~kF2j!iQ#p5jH?#2>Vq<bK~U8?FI&QGVKDyLR*NmPO(o|ULKe%BlQVI!S}n&nq$
zNUN@hE$i_LH%3&-E5E2&^q2q5+83~msl{5m_$lVXRzRsQn3J-gbnb*s++|GZag*B!
zZGNbmo6xjd+2*FvXKGv|?a0}gIRBI9Qkr(7W0>3w9xvT*{{5~0?+J~@N4;U+0bJ#n
z{uT^g7y?0pSf|d&{2U>a>a<m2gO)G_YYQVq95Rx+E-muSc|WQ5jm>Y0qAmM8Baq!v
z+<3FGOz;8mbPP@I?Qn1RL*3k`ri*m{Cy07{lltPiYMl9XK2FeD!sdgsBK^=BsjBl*
zEPSJ+C3HuhX`~3eX0ls8glk=*@TtkNbivK$-X^xEmK`X{NtU|1)I7hkPtyO}3xIow
z^kjgjJMF~}E(Ml>md&Mi@IRyIpWKzu5^cQtUmNV-Uce71O3j;U=6Yf^z%}C-;KKzc
zIr^^XQ#_w&Q7+31N>^kp1d+VW1&tyKhYy)=INU97Afj3xq}_xSJ>0}X9Oq2wgzm50
z^$QML(#(bS&jP$n0XOKfyB!tRj~XzC^t#os-2H+=Gay#*bf6u!8an-h2S7-S0SLvS
zwN5d6-U0FMq8qX?blhkaR<n5Ts8lySGX<Q|anH%gi^aNS>bkS3rpHA}{qr#I;F2_x
zhsI)$aNhz>vcT1fWsCLzyi&7%wzviu3NwaHMDV|QBF%~_o6da4r2~{s@I(kU`e(g^
zzj0L(f8(l_uwFW#QF#8EyvM(p8DSRR`qYXpUr4z|$LF#=rn+qllCq%;SR#J{ne4{h
zewwn8xQLfyb87I}W70TTN)XR24~VQnRw)A|6j$E;e7%swHRa>Son#s`vzL7+tzyAb
zYftcbQC_$Gki|8)!4R-VC5sAB)uwdwkq4h$y`55z=F59~v=$(9ex64U?KqW5CM9&{
z&kN8M|B(&$C;-)~xu<9%@h2w)P0HmT%e&vKnUml)MU3Cf5DZd5kh2C=0YoOK&luS2
zO}5bz2_cbS_y6U>mcl@BC$&eF>5Nha`o<{6e#DR~)fe{OTgw(B9qfm)<dO$7^Vx#O
z(}DGMwwp6_ms3zje;dizu~wUYzRFhdyYa-*Xauv~%F*;*v1ghR139zT1~ArF)p6x2
zZHHSgm2|7scxilWIR>@p9?sipx5lF&s4beNha+>$#M1T9lbD7VYjlSaS<s=jdFQwd
zXEZuZr@rn-N7+jL7n+Vgis9`XS;Kgh#xn|}HdLOy(o*Tpw~0c~4h0Tnq5N|X=kMq9
z(7Ub6Nmk45ZdT%F>Zj=<#lsP7nfYC4iS(m3gNbtvq{&dWISzX1#_Q%LxBJOJ@scD$
zlbkS&4xgxd5*(RCzDg<%NPlA05~0P6D<r~<TI5DfPn=GRFl^`P^=OMHUIX%kve!4s
z!}Y*9NG46&+PF|bPGkl?vyxkGWI$j7Q1<oFscYSfIPG^F;nUi=6E2}z4e**IOE6C`
z!nzwmc3NEe3rOfFaiyPd%A8`J*2>CWaCmn}RzstC4k~LGuO7<yPBIu<?Fj0LEE<7X
zFBxB)f3OgG?cuzA;W3o~J8<ZlRSBt~I4G8o(m+<81pO;FuV0`gi&lQKvg|4N2qYh(
zCq^zWg$vK49qiLtJ=)s+4*8DPh~}>Q-0>Y}$gu)8o59IXIf0QVbWa=W-{QaVsG18(
z$%BXkP=t7a^83uWm!oOJhk(NT_a|y~v(?g@jHE#&0TW_Y$VsO74`c9;sceBfumiM+
zK=RHeQ4py(D(GuO+ZV|$ft59hijfZBK1C~PWh4xs*^$TG)@R2`BI^_JWJr_o<RF}#
zfw1tE^1^Vu%;aQf3O?{U#5&U^+FB@j$XxZv_Gj=Tdl?$KstP32Nbzk6yf)x~0wT+i
zSsv{_71jmSeeQp&ur&W&VSyQdJk!~iDBR{f3te~FEq22QO*=q^1-m>9&Rl|c#L^Ep
z#P*qzHXP>S1s!UeZ0<0Xo?4tEl8Q|Gr$#r|cV){=b?{2ZK&$EvX}Ve}i?45#KFKa0
z%W6Cvz{|uW!zkscfcV9nK^Wpl_0^iEUUx?KL%sOk^tm_;lbh9H2bM%>EvbBPht2^n
zKNLv`jZ-xCS;eikpeUm1_jCb*#@i8~A_o>95Bsun451Dhx`*3}Ci_VkvA$E|IQw+6
zd4z5RP}hHG>>CTseHS4;m_NOK<+?F}^(8K<4}+V(`miMk24vnRJg9KA7?x_;xbs@%
z4W0D!m5YP!G%l|4Zx20N+3VIyo^6Ai*M!aC_)%RiN3bbtgS6a^ODpG?(%fR2PNeoF
z8Rdkn^=1ed9P91^nY1fyhl?SzM{QW))zwowM4J~|S*3H=n<{Sm--t>xPmk$>9n5EM
zr|I^Z<cw98wdzba*)$M-o_i!zM#E~l=KD92>qDW?>q8m$VqpKo7cbzI6WMxk27Fdu
z&M^4Ba<=5;tBGE}MI9V@vpl+d#ZEc@Hy_OviNOYP`D%j&#z=mG>#OSaqOcRs;~ja@
zTqjS2tiXGTfp@`d#d2wlSNc)E3<kB%U)bVHhpZ?<@-jsRz3>cHR3F!kn2PjeQ<9^{
z6+OM*;sIUfoKG^9OlA63<9<f%X{C>rcHib9!Pt7@?N%jmrXKN|^i7{w2=4XH8k#}~
z6JzK&8Dw%g_&eG4nADM9t#^wr7+k<1(mX&VV*6FcK!3zEbxp~A9lPK)pZlcT<}cxn
zBMc_slIq0d{;gD>85|4T8w<8uT#jh}W8suO@n2o3ux&NNh~2^50`D9Y@p`n7L)dV&
zD{QE%@JwrRO4GO<ioEi1jCE~PFjAEg`nwd>_HiiZ&}kOhd(Kr&^DK$-7*C^Iq11%p
za@qFcz<rbSEXroJogE%iUql$7)AFF>dgvAWt)PkxVTHHba!xxsWWgq<<MxHV>Gd<!
zuBkt#ms5t5GF(dbXylYNkYjBPBQ<uc#3uJk0p8`PIQ<rOh?#VHMwdT&8Yy)^3}RpZ
z9?y1flH$Q+yxUVg!>u8;5Wzz6Fz~1L?(7-dU%9AxcxcM=a$kRLHmaps+>_YMpGvXd
zXS0x{;p2b}hsA{3h0m^Qc?x}pJA~BjRrjR<YL_^+Ld6k<_+B}N-40X60fYPF=&4l#
z<If-^MoFjlCamYG0<y*f(w1|(549O>69q3vmV}SKixeHoY8Y9U-rosQP&f>knh5My
zi9ss|r-Dp3hjs9&z3lv&&Zl0sshV}?mt%0RHUC$SYX|EJQFEA(C!M)Ej#}k^=#HSJ
zm^)~&G!v&*yKi#aD|elvc7>8G)I~a<^r|2VOGy}4ipslK96AUZW+qi20JQk#Wy=R%
z@UpLas0)VBS(yS}ojGfb&J1Eot;rDEb?<unVz;un02Z|6dHa!e*sM-iu35M}4D9|n
z<%Ih~1d$0j>c`<_U{ZmQ0Hs;s#<-#f@Xir!5AbsX2-2hrw*gqcTird~`7_qPi$~0L
zuwv}w#ewR20!NIQ$WEAfVLpO4D@>r-a><@haCLquzuT7W8$MqSk%5ok33<wUjOIEH
z@jCk)=ZV~PnpjGSxwsU;89y0yP?{iCAjSK!-rOje^D`bblJw3zFczuq+7dqh)^?QG
zBLa?>K`Us29y<XRAu`iJ5NAbkaEGKgNS^9uiyQB3q*5t>zt{`eC~9-I|B>?@@hO+(
zqzV=EOrVEB#N&!ior(8;(lACey>1qhTU6oy#|to;odQrwOt}!c<He6#y`(t>OsAWp
z>M!p0)OgKFriE?a8r*QZ-d2-xhwQ&y?z+QmI4~bf<O^SYxU=*My+SJ7I`q+-FP2xH
zYiH|hK(>>PFWSb#jSLRstGB0yc`0KX0*4OXF1lSL%A_3`(wExpG&I}q3H8v-N@zDN
zKkOk&!Bb_r2+QEduoxMS*jb~`l%|Ve*E1VKi6r(|8-vvxI`yoT<EE44xv-mHX)da@
zcx#ZbIT6@>R(Wfr-39%pT)%_BSHEPl_vw<o!W}Zy$6<r<{CyjPgZtW?wqs-zm!qdr
z3Dti$6ET8uh6aKWjvxFJ!1vO(H+Q-Hvn^DokunsQ&H`GeF@ke|J7?7X*S5ioQpi=Q
z%!})0cZk8)yknx>vc9>gX6QbDF4F^CuI{RftZEm5fM1;+tC>;Tasvxp*u`4@a-%Kt
z#9se{uuj-g#n;Oc_~4@KfIdxkh{rO{?alcmw9_JG)gVdTemj14`}xa}s@?H^I6v3r
zOct}AW5Tgzlcj=qSoMRS_-lJLW1QXk1M;Z~F^ig4Ho~qqoxIoM9NV!`I~d$3R)~W(
zh0nG(Iy5!H7rW!oi|BdtGR+DXY(T_g2sqms)tw62tT|EeSyTXG!FEAA6D@XE140v;
zVNKV4Q2*fi_2yzF0Evt-eva3NTuavQ=h{#28_OiIgCt)gpr;6%)78uE$9Tv|b@;#?
zxDznzq65G`Fr0v{>qWS;zSeHh`0yNDi?b`$<EHvkO5eAUbQFgn0dd0(waVsj-z3SX
zoLO(b(i#nm>u{OmE7*oYUZu8E$apd7Sy)c}0p<9w-nSS3A7yVH73JHt4I`j{NK401
z0)muu4J9Q4A|gl%h%`v&fOIoRh*Cp$4bm+g(%s!X^ibagzx#Q<=YH?!TkHM)Vy*du
zS#w?c+~?lMKK60^k<O35S+R7USm#INSNnIflgtha9GCCu+EsFK)Oe;{A5|!XW)_j!
z-lgsoGJ>zDO5Dk@Q@y#sQN0YDgyI6yN7Z=M)z1b2hrDY~WEp+3#5Zr!CJc0!2M2_o
zmlfHZt_U`}=x%%*!h}i)<n;C32LIT|Zv*(H=t*um5)rF=(&d{&a?|Y#6#mFo`%mc+
z2n!m?*u$5f<8VQ^$7Yb#mg2XN<8qRyF^!*}AJ?FywmsrPrFBT>EL>Xg756-6<9A30
zeX=h_c}ORlHse2*Hq*L=Zsr-Hri3t4j>i@f3R;b3JR#2qLslg_$WosLZVkOu-VKSy
z<6l+UPFI7}OAJb??cX_6&GH*G`wcigSO2Gw%5-}Knfu=gDH@=VswpEf)hZh3u9$Wu
z7)-0-5wci&;=YuMXvH?vD%1Jn@`lJSvHLSX5V%K_>s)n09SBgLd&|~zPuYUNluoef
zl$Z8dCLM8w(8jiS!0#3jNlxNaC5brKck0oYrVzRAt|_j!dI-rQqm*ZxePm6n&PO*;
zVvj4b@t!HtOs@{9LHu`n?}&MO!leE&GViIHte0u=Y+2A}`uU=KoC@#Sn=u!3b#>HS
z6I*=KpDkI;)K*E4)|}d(^JYu6#yWb!VJCBT=~8>YtdH%BKlb;IL6?EB{`mG;5;tdq
zL9xD@k)%aAV0JZHDy=<zvovh^%OA>!w7wEJZ9O#c&u`U1S_uO7uu8ySu9Mk+rHypH
zQi-3OOetI;CADd<7Ox^5I>+GtL@o6~zT=V$CgE!}s_s7Nx_(_s6)el`KLvbQtB!3O
z)>x&@nbe}H^N@mudd;vB#W8~Vx&V?ebGeZ-+&(c}{B*93C0yYWV}Gzu8Jws1KAc=&
zyn^P`)m(4qx|ZYlZ@$aQAjjM5jrSrKOU0y@RCOu$-D;c9HzL!J)i)y$8lc=4qAmFR
zS@27MFP_=gCy46D8w=PO{2q+R0r^lp_KO&{QKOD55<SV(1_v|9rYXT{SGb*PNf)%{
zZxRKj5jw8)R`h%aN7uTSyh_5F!5bPs;feS9j>Z~^?X4qPnjR3DXhuAUpMq^CQ=|;V
zcNO#OVzMNzD|`|dq0+!2Udc+U@(al5&`+I^Oc_p9CBvA&58P%Pn7ku&k5Y{b1RYkG
zAN^q(3f~%{N2qeNPXwhZZ3h}8@>G{GITSVz_5zoP>eN3~)b7#7@BgieiUF#qyGsP^
z(D+i<Q*T+V1D97t%uPk|c8y(YD!{iA4j?LVzkf6I`q>)op2Q`?FOI5LmZRtg3e>f=
zEzXFVm3gnNCYhAf)W%=}CO;sjJM8>hCm1}sj@0tJV&l_;A11$YT2Pp0toYor10?%f
zN9g0{KR$l>xLv#I{h?wpdG*Tn^LTuJaxLFZ<WY^4+ln!_)81dRMO77y51rA)rh{3(
z3maNE3s1Ts47qYphh`|XkzRw-p!JK8>rFkcvOYBau@YG;V{{6igbGV5Kq*XNtl~27
zp?&W!?l!R34L{}4>GH$f@YEf|@Mw645WF@_o+<C>IC$MZ$)I6~tD@&ct>y84t>Oe9
zEA++7+d$CHvX{8|a&yGX>0~dvV-u5vK<M9-*c{h{Sjz!8hIXxIy;b#U^^BJ4J!XhJ
zV>oRy>i@vYF>>|!ve&ZEz`bK|Wsw1}^W|XOm}ZM~9lKtA@M||+1XQZB2cf4Fcn>)H
zS_7nLj1Fq8c-}?3^2xi|oe2!mk?g*wTYJ!KlpY~RwhNbYeUoOe2<_nznS1nPgIrEv
z?)>NM*A}DPYFUU=y+KD@X=%~J>uO0M&#R~Ih%SL0<TmzxXSt=|cy^-QTMtq7o9Yik
zX2V~mdN{p4Y3N-@c<@Fa$Cgi9i*yFyO#cD++Zu}}_A|C|tx0yQp|PH~Nz+rCjopQz
zNk$WrS^QCiUn9UqVPY5Wb}<FZbJdCF8(}lE0hSf-9Fq8r8mv4QO!zJMD_L~zYjaH`
zK9J%!VMp`~13RSn)I<^cEp$;>w+eLDs9s->TydAUIzga_E3$SExSu|)2>T%g8%D$N
z;WOFlgS)5=>rlB_S?YvT0goD{vw^)TZ;_{x|8pSRwd<MZs~Qr!C8*e{Y{qtr>=R)R
zGlFpV_(ypieY!xV&WsHURr6dr$YaLn4(4zSOZLZNKDciQk{B(!GhLQSuvtb`j#vV!
zyY-#gH-q2NIJD9o6Br_vyA&XRtHOD<+bA=Lf0alzko$ySR4&`biOtWm@&Ge?`kFvl
z;{E?^4Mw(ZH7PM;I^K|8u9aJ^bnR5CgpEPEqMIl82^d7Tq{q1V$?-elt^|DHx9A=}
z)pY@i;j$MsOj?j+q+5wSj$bly3q#BrM0}VDa#7Kn%nxrc3#F%t!Y8ZLtJY_H(VHYR
z`sEFo8BcZ?d&tG;XTqLJO9Ro6T*u-)egR-oz2pv?C{R!{C)DaQX^Rz^4w<c;cuYM#
zX$Q!Lh_wDC6Pe2jCnwdj&b&<C9m%y~?7upY-m^m%)mj@iUMN{-v(xxGoG-5bXi0@X
ziu{B;+c&T%9PNk~z&rO8n3fHv0D)k<?x%?qal3Q%>DR~E2J3OBZ2r`BYaLV|2=5S+
zO0wQE70%{)bK!M?+T{$pkME65Og#ekR#UGY*xaWSS$AfW>)@2_httV<s6yTX?u2kp
zw%V<~$sc1ME*guQb&Do&+c(TQ+&-;6NzDUU#6EQExEsNGgC5+YhtZm!(c`bt(u|t_
zU_2CjG_#yN8u-Qq{Q!k_ly(Ac8x;KwSG<ia@OT2h-Q$1()UYu~*Byvczcrk7vf1-v
zj3EH6b}J7jkIYwTh3sOvB^9|OkFNd$L$~{u_6ONkKFwhsRGA;JMpT#Hj;tSJk2b}m
z!;2oKM-M_64W}J0N(3gMQDUs%vx_>UXH=Jc+C)cRE^9!RVS0AwBR8JPSsi;IGGTq}
zPtsYKx$`)^+E=85#_Q~08KmKSpsV-#4I2mj9RV{zhI|62A6{Ui$0sC75FI|B1l{9?
zQDXjJ+y4zW#Qg_1>`D@bmE7M><x<~fKRv?f7US5?j02absnlnEINZ~r@x~d~`(rIy
zsqsDn=~iUr-nqU?%}QSHX5wWrE$obKu+Gp}vdCbeOrOVS?3-g#pHf*^DcjaDJ9{?5
zd)eZeDP3TJ&m|fRe8-E?J4E4F#S37t;9>?vJn3*=TtGv4MSh6gs{T#w8`lRASb9t4
z34ajNF+5ZRzsB}WC>{p?<e`ilc&W`Xg~kVG&|@R}rrTfFQ5{VmVWzinNCc&GFydHc
zph<Yemn#YS9&wSEyjF02g+!>~_L4UWhoThN`!K%69$6yKjvg~KTg`8ak`>;0@$z9j
zw-Y8)T~iDLP^$6c8otsppaDSH!;DWVMsOb41C2Umd^ZIBL{~4iq|p0l5)s>TuC;+W
zX@PbY@dnHjK7BjGz*jM2ah!#ivgf5}O8r1@1%L|=f(c1<%N$UZuwboqxi;`N5>;S}
zy?^DS$HxCM|FWe`*-rzIh=`-b-#|nH<!lk=PVN5KX%}<)E<i2HkMxlf(FH1!Nrq5m
z@CB>;YAlrXygJPZld8hmeS0o5xA#|VQ(v{YRJf0qNI`Vrh`G?E<;s~%<?o(_6RpVx
zk&1AUZ3oloRxN)0G3zRuQeDsO42r_ur2EWz&P}_39zP@=`Vaazc*naqBwlzhkS#OP
zO#JO5HYrL33VkyU#PD5$Jljwxm2a_mhUmkK@7gEu?348XZa&7mKV;%nSW9A^S#U%?
z6M%!W8?{9%TD%wC9(N<orV3LBXc(QcqW0~|p`60)Cxl9AU<CwoMsQR#n1~qV8(I#k
zLTFZ-28#J2eO$@gUUkUDvTry=D>n&O9?^T%l3eiPwn202^T@!f=fC{9)5^KBBDRc%
zH7!1f@|&{=u>0VC53depOh-<1bqJo=5Vy|)HFM!K^P8N>z~hDecYb%{ZlLHoSQ)W>
zt(=m}@RermKi)$q&iz^FHT0dq26!po7zS7{0rN{(f&IGEA%Oz0h+GXhoz$rK^amAi
zoS&=s^C;hVg`5WFu#vjX+f5~!dw+iJAhWHz03Yd3`3OY#DRxi<R>#>Qz7RSM3hKZ7
zsVu|VqQ8-T1M$0{ZjcX?a?(32|D@P}6J^hAe2!MW!V33QTr1XGdJhseG;RBcPwGQL
z$rqy`WU8)zxUrzXrP9miV5DEIj;HTHvCIk5!hKbp>mMQLvHgNEr`Qy+V&J3of}KdF
zj10NCw2{qyj>YQ{t9=yHh=Pw<hFP5vZNuvaT3L7t8C4gvGxTke3N3ah3u)6okrvoe
zC!&PxB)+jzh+#;w&{8=vqd*-(cFO@{2}V-r*DV47I~#RQ!IF<DxqqUs&l%U0X0do(
zznu^K2cIS6U&&=Ki2@e+ujC?u9-rs)r58B|7#`A7icQ@T{ANv@`Pa(;SQz(#cm@O_
zIR!p7Iqt3m!`?R+)B{ZX_N9kS4<$IM*#ixw1sH}0Iy9~xAOWrf+3Hg~Zx2jxkoh8%
z6VS!VpZFBQcPJAR+OQU`bYiBi@A1SVGhOB&{ZU%{)^J|K(h$-N*MmI!NJ_&(bw~??
z=|Oyu9rttOP<Hg4XO95_d$xd6B3NDDGh{GS(x^O71&kbGOXi=QmGO*czFApgl+Pye
z2?q-~ln&D6N#_gY6pIdpCQ5Ul*DnpKg2J9=$DgyvZUgiPOx{W%ZzfvR=6?$0#ps^`
zj?4oq50<3=BN4L*qu(<!Wt`1Dr0)B=k;mTBxRZkcPbQnD%C(<QTghUr{rzRZ*?5(j
z1UXf*)u7m{cl>YA(i?FXaiTPtf~-=gy*b|)(Bq|Qdz!E$TCqN1kYeeeP;a!VL{b+4
zX7FDzb0j$q$v(?5Q$h}397w;{w;v&+6k^;m(tubVFY@Da_W9c3OY8HuEoHm11aRK~
zRTO)g&Xxr@tPL)2R1V%LCm}9${6yIFxop9~Aij!85fg%r{|Zy#0X-fDe29lhnR{Y@
zPWXA)rbWTjc@9$zzz`s!;{=xucEss~w6$x3^Pu{#T?Penen%HjZ3K<$sOvlOwlf3W
z=2kvQ&5y;6J%A|15>xe0fL*;}8g@oiLJe@fCqmw*$EJTmErsPDlFHN7%|nIz6Z@7K
z9Y8az?uIrON2C)6i$<1nq8yOg|97A(;du9?g#AM1^?>Uhf@-Mm?<*&0JFkCsBlF(i
z@3m+90m^YI&KwE`Kko6;%TC!BS7^uG(rjt3H@-l_3HKi)d2BG4LF^dL2bh&l$V2nL
zB}u!khDKa01a81p5e#u2BOO^}tIs~19HbX>gO3STW#mAAew?P4a)`6&=ntaD%7uQg
zW@oiJ98C+(#=AF}@PQjCF2j4o8ynioFl`ap#$lf{lt*P$bg%H0OD#9Zdz2w|IW(SB
zdHD@n`Om=f?PT?gg~hIU9N>ur|8x!7ff-=~ADenCmQ2?B%l@61FY)>=FtGkv)&8G{
z68oPU-!_R97XDuu>%Z9;FFfx?QRmz?8zI0MTqIC7YUXC3npq}Qebj(bhk{P+fp(5j
zDlJN2^G%RCHNlo@;5qdCaoGmJ<c6XOTQd5E&=7Kh53iTAFkIPHUH_)6l_3`d8D;)~
zYz<NL#t}O#+ZZJ1NImhh_J=P4m+Mu@s3MD={vc|ZrgxVnWYi!LIXvF{Q*Nt+j4nV&
zOEk1y<sw!(Bl=Sa?KUGzGRYqp_+kV56EbM0ELWG4)o{?F{`eyr)Fcg>ak1G$nt;sX
z)Za8y_w^{X5L=NbmJ^)af__Df8X~^Gj5m{8u6~JlGl2*|B6kV?zkfCm9};l7e=gw)
zp5>!^OzB@ycK^q_EgWdtNd%T~^o#HCr2ygxoQ{I8UyDItR163%fy~0gl)%6A-x~#=
zZff3iGf=`;yrr8Y!M1qs^noFn$B0y`Qj^NbV75^q8t{QMcr*~(8$UEBQ^ndt?FPfi
z0m0xL?+f2KH!Vs{z!L6dQyZ(0_Cbc&nQ~tc0UB)^Mi@^WP<<X+yvdh#((--<s+CBC
zrmS}|$A215JWz*hIvA2=3z0LJc_uNAn!Ip#z!lGda*<wSR}Z<x7msCqIAVkvu7-xE
zy9&AiHg8kPDOeIQrT?6;ix)p&wM3@f%Zm*EnV7);B&IfJYY*A~<B0*RITrwR>s@NA
z<<27}_YR;ol?^a*5?qk^3e!ggp`;Qk_>&MEK08U}r13pIk5a>!dGP(~05=Ipx_Xdq
zB}FA4+1>L~E1uI0xg;PDdjApkL|x@XUCzzijs=4qypmN6uOg6)-CRvioEXw+&51xP
zt1;8rLS%@T;+T3{d6~qo9pD>7UD#B|2;=V?lau~L$z!UX7v^D&%B<{0e&7bG3m;r@
z0-FQ%7p{XmaU(@;2;(>URvSO^pNaR<B?X$pm6twdJA@79dv1IYEWk&KHgA%6`4N@G
z6#esL&ZyS<XRvrLg3c%Y^BnzCEhxd0$YIcbi=?}_dSQKcj;aW3&fS1>G$~&2^|Rob
z9~uqn+0`OOlj}l*xCaW&IHTu-(-D9z<s(i%=iA=>Twm*AK!bWSClh%<GlVqw*%s0v
zf$QhuU=G|y=bZvphbBoO>0tQK`jL3acQuUap_*<bQToOimNPZ5xtezPC8ITqf(n=G
zFS3*jW(;c2+>FqcWF+=`Lv(Y`I=fV6Jw8)41hy+do5gMz9|p>}g5vToxtbqbVQExP
zN*G$r=9HviHGid<i4wN_t&ffWiXN!e0g1m`fg{6M&<$8JAP9}RNxdJMZ4PT%BY=_>
zBM|SthWxj@jQOv;yx4X4{5~)L&x6)vDAR<+8%n$<`}_mI*KkT9C&Hy!MAwk`OpN@B
zN(&9*q{T&>K{>?AXu+U1LY94fG{o&>*=oOn-9Vo|klnBp%OkFL6bbwmb=Z%(kycLq
zjC2x=n-Jy*S5uC%Jo1`J;Q?&Yi;R=5(3}s}Wh{EjTICmA=EQXazo_2mL}$fD_?sJ~
zCY(IX(7NafB>65TpoJdkWI@nr{s7A-cr1498R@qnJ5#;*O3Sh!yPRS^0w7KmsK0+U
zjZM%*60UxE0dfdohGD)0&;y`pc-tr=Sc8GuySv;0?KitNQJ_z`e?VQs8efNi2ZR~Q
z_MaD5&_AQar-KR~yR+VX_&}l|YBt1}065dWt8mjV`nPbB;HCqLB12(H#{@;7o-9M_
zJu~1tK*4uGp$Z$-z)nSvK+(at7;a|9pJa4M#{nIewYD%cY1nSyU!-<m{P}^*1vdbK
zlCTest+q}2uCdr44|26Y64kn?M1GSvJmJ;U0AzP?1R!oE3}5c~emac&^Im{J(=oJ9
zhTDLI!yv6&5oEAL6(rFX*x&(QY!QUev54~t+IA}>_z(yDiuCm*)4CYcfSBikY6kH{
z+_u;PxXT4!S!2ywCXS^-@fiT}v;$)FC-RtJl05qk?QSI>{U)JCLbK@bTfF)C|K9@X
zM?6++Y;0S(u=kf-q1A@B|1)QWkO1WBr@Ohihpi!z5<J3hlBrGc_fCFe<D6Y=4mT4m
z;M+c`uR{AuCD=*;5FO+^9<2Im(^3oJd#HWKb92EmgN&vT3rGsvWmE^HlL#@ckc>;7
z7^k=CgDIk6(pPw+M9w~NKcrUvTAJJWTeW4WlS5!~*-3%%^y|oHGntzdPBA(y%+sgu
zk2lcqvD2Zr3ApTny}-G?dM_^S#U=Acj#J{|bv9Aj)kvHvINtX-FQKBFQy-D(ljJc`
zE-TLsytM3-UkpZuF#-;S>VgyAYy9)xj|5+9-18?g-Eogh_R!-!UU}^NNBhzi@a=vD
zVxsTw5D%=hw6sc-dEkZE`iut9j{Ns+Cjq}Ju?@v|_+T1${r3|d5P8HtNs+_~f#3tG
zC7w?Kpe8b96rZGq_5(E9dHGhhw@;i93@h}w)Tbk)4G)Z{m!1uzF(_XT^llNwt!XG<
zZT2X?a0E&?AJ(Im_CGTR&jO^Y2M7R6KVJOFZ&w0e7Foz>P<zc)im7xEf{Tt_K9s4E
zDMTq0Kc#GfF7DI%L)tW~Z?xU{q*B8v!`L|PTCl5gA~wi|YzaKK!NXLDwv=`y6D(Yw
zO1Yc^&1qPZqaLLfmtqq!LMUX%m+~+%OW}-W#+uHJ{E16<25O-#bPCD$@r4{(z&Z$T
zzkcdR5$`N~)*}1=DwgdqU;O_2a;!+VRuC9mfh>{7o6jC90XFXdQ2{(wW~S!8m7D$A
zd@qx0lK6w>pSzWl>~5uNA^WwM<}wT34#1B>JDN!;EYwad>rJj|n1~Q+c<?`uo?G>F
zMxE|0aGraCCiC&L%70IOFL3WJ1;Mx0GZc510%w8Kd%?8;G#V7rs)dJ5Xaizif`pjD
zxK)qOjZB^}f784eOp9J%pmbYRrM6t6ii`t;S5%Hz@lF=aOg;0UoDhUTbP6*`!p=EB
z=Nau^_gcuVTZt?_2bDflTVtj!$8TRTC=g0l6L;8VVtre=p%WuPONv3X94W8TLyy0S
zp65`uqK13rNIzilf~(V4*OiPRPen1&P9E0YB^#>C{L*1r(;lcod!>cm!#-3?352Rc
zuDW2LK<d=CzEHKJjAzW;@o{T#sezNn(62RZpv9+=ECz!ziU3FSK;dsf4Us7V<uH{2
zA79h9jo9kW4u?j}jZ6Taz;i#GMr>-%b)7mvB(C2koLRTV0Y0jsKMN?18e5F!sTZ69
z8^05z`W^-#&vS!v|82_oVtw)L>B?fqy_v~)K8r8Ezx0;cF1Js1fV?b?+QV{7Z|r*5
zO}x1*>fKIC0n2%^6<>0mFB2O25M#h&Jy|J9xcrl-#cEFQH}UFsoAZrcqBgljrC<S{
z#w$dR-J0XLu0UUTVrkSLK!-H!==*EPl50KpZ9Jxag!uV_ubsf=Hf3W5TxGl-P?l*M
z$18T%Q=Ip@g?Q}O1Gb<1PCFALe5cc8W<*_d#81>&Xq%pZz;%zI2&N$B#`8ALw5V|e
z;F$7u&R{VIyr1BMwVmoW00*Q2KUP}L(nq{@+{}#2+qOcZD**(Vr}OfQD1J+p)NE!Y
zEbae&^t$oY@IlS`N&kLQ4@a$KaG$Xm%UrG+^ETkXtFr~5`d@~#l=AH+*|o~J^t`A=
zoWdB39%7Tc@&Prn!MtuRT;}rL;D_PwJbFU*$-{c#2-oX3Ap1{!bWn-#C)F!)Jvvs=
z_Wyn8>R<gACNAzNyL2+739Rudr<lF)6SHx|jERkvAq7?e0%NBC{U)QJyjx}AZw`kw
zVq)>8x8#`wO4mKo!gaw3azWX^Cs0^;_$eC1-<C&QjFPal!4SfLQ61!M)ASPvR|#kY
z9QbHc6K<YcOx2qIAwG~Q3~td7(h-o-(I;B<$)^#wH=5g4SW4h1RYf#SFx-Ah_=-{W
zE{E8XgqRYP$v^()poQtgE-QrcuoRO%;mQo2Ewqa9KqgIyt33{`N88;-)^(Pd)%jxL
zo!2PoFMq0y0|r@0xnwNCCqF!X-0(wH>Ow_}@5#awK^7ztrC+6YN#_5Le*L7KcHrw#
zBIRE|9o<Tg4<wA*21OB30$fqStJcEVzt`G_fI!uWXo{TLo?e%|vxQQB{tJ!18*(<Q
zVfG8zZa+*-FJX*8^l(ys4v<1!GyrtID~4XP5x=AqI0M!c=j7YFoF@SHM5|c!4_z7>
z!VQRSw(x|}C}}*g5v>Z#T!UD4^u+w4n!Ln*zy?IyT4yw6KqyB=bmf;-<!nV>)T-qS
z8SrJ!Ji3tOINsA2e&c-&mwOg~GhXjltWy=s!Hj+X<R!q>&D#Z5dULHxyO>rTMsdFM
zC7J>1z`Bmszv`LL<B@eXrBXN}9|2WOE5OB$*fa3p1|UoJQ08<)nZxD13NV|Sf(RJQ
z>jgbcq;I5XA(Ex9y~Ia_4cJj>btF<G5M7St@@fO4SiBDn%$`0Ixu$ZOFsQgfR2Po`
zJCng;7c0@RV?JfgCN7wq{^)57fEc>ovqk5Fg+C-We=^^?WG$CJu`P*|D4jLSPU^ni
ztv~7$v07>k_Mf4Am3<$I_<~{z)c7oHY@w<^YDIIQ`S)$tft0(pUUE@~z6;p(wU<CK
za1eCyBT7Vb!I=RxH|S|lgl$nw2{;CfvfDs_<i>GV+xIQFy`;38?u`63{JX{pu?pMm
zK=00ZE*L(wAuVJkbnr(Rw_QFSj}n7_i$~*5a^sv$&m!}gfRBDRxvG_AUU79AL!2E?
zQF#V-j^jZy3R6aKPTMQ--r|RyS;5OKQme&e57J4$FQFGGOL;=PWfSe5R~nM#b4r*I
zl>dcNm9hmY6CQ4qu)#chS+|1aR^r-oz~JN6{0}F_D)tBTW&{z*i>R}O4QNUdX?`;I
z8x+mV+%{V{N-2w3vt7(EOMS2F@+5~Lh1fm*?*ceLbjo<6G%kDI9KlRh$gKsp&yt+f
z4AN$3{a;B>N0zozShP|ac6Z@uTE99B!grdAETQ~6>pWj*Fg7Lt>{Zd7M#HWcdpi1f
z3P3vV?79KRJ9EHZ-fc}TFu^O*WCG{mq9WC;m<q1--c}av&pE*Go7joy;}By6691E?
z4P<Fs@msFVRa8686=01~Tt$QmapGR!>x+1VKQ%<zS~=^aeVW`_$$?H`T?ee%ocN8)
z=P?Y7tV7h&u|}HB0R8Yvz`6VjLEU^6LI0lWr>%Dc51d3O_$@VgC9djMe`vD`k4i*G
z5zum$5aBdpP-@=^=2y96I^JCGdudC1j@zLs8NuphE&x8d^Y&c5dsa?}M2j+?_~m4O
z>3sNA;p!(+a!-AQiRZf!B6|Z|78gl1z_Mg)e#bh_l^NY1#4of)DRc^XlSr^DuTF#D
zJx*{taUXPwcBsA3-M>h;|Jm%Yh~1IH+}2-swcj^+e{J3a6<YwdmN0GB*M1mUADSDI
zzAr^OWBez{9!djYcq~T!UAe>&ywzg%mP%-_MS)f}O=u`>ot8s?z@ovd&cQr5J*$cs
zp`H|D;;rmB%8F+cY#BD(u;XYxjOzHhgp0*XGUp@jg^(t`9QXj}jW>xH7Gk5T-p4SH
zc9AA69jZZ&6Y+1ZUx<-!5Ulxg0{Bz;rm)=)8Gb|j(a&#SKLl5CW(Bi6Z|V-(pn_YW
zl9|X{@dm`YezSXk?sMxCYEx9gXD@)RWieW#YlJdhqNVo?$aNxMr+|*xM3<%DQEN&4
zt`VJG%ok4RxPlAc!g5fB0(4ocenAglV>3A;y;t8E<fiVD*C%^mZ`yb_Cq3e&mY*C~
ziFT-;9J*E6pe)4{K)NQj2~kv?ccD2c1-AWIlIqWwLhrWIO_l-UK_2&3Lj8f8lVYz}
z27F);{^Jm?hW=?ii&}sb-+F`EVL<YbfGBLTE%5yPJHn22n3vwJR|!w$@tQ*oTCXB!
zeN0x-K;TGAZ`b=*-xk*lz(Oo1>HRocR~=ZR%;0BDC1|oYGl*?pJb{g8C*k@-6ViQ4
zS9FPLbIyqje&e_zQk$ygJ0*v}wB{A1IBi9y-}k?f<K96n>96r=q#p(6XT2l#&T80C
zi0?~f^1KoYe8ueDiR@&6#<;_}*LLaADMBQq+j00OX!mZU8($(&D*~PZw<Ke{B9jJE
zM2F`tCg;sP&uFj2Tz23cPf0)+In=$Z(~hjq+zC3B1undtY1>MBaG;8BZpTWO2{I(V
z8~$#_7Ew;=l56b{=ST1E?p}&YlsGaH`Q1SCKbxP|cn=!#H!roOUKjt^4-g*Mq|n|h
zJns-#z8jcR;?q##a&${6OFHy}a32ULklE9QRez_Wkrw`e$L@ySfC9l4d{|}%Myyze
zt!ynr2Mm{4R*JPkvSor=eu#gu`bx=LJP_^!tHM18+U%9CGr4S`0PExLoJNyD>BSfL
zfv`2=niEpud>f`!gD7DItr(`{se_J)6qCQr8QI1DQ7Z6M$`dndhi;iCjKoc9R1g9a
z2h`m3Xp$Q8&f~#oxubX11lj`t6eA1BA6Vu+OWHC5>IHZ%*p`a$BB$YMpa<&Wl(I>1
z539=##D3WZW>%s$>b*6oSiB=Xuk6P@nzSXc(2|!BDY^Zw%)6-SCD43_c>o%Z6vc0N
zQ!@PUa{v%9SgrAFMOE~dX;H}0Oc>#3+I@iZR({x{6u%9;#J|>BtNSr;=g%E5BBh`#
zuWerPFw<ln-PxcIE3W^z4Wdg#I6pSPNOF)1i)lRsoOS?^x#jl^-zKmZ0K{Gy6WyxE
zt8d;_4^GQtk*(PRDtGmzyjzaaHL{)4Y-e~}md*~l_GO6(TGVY#@pEOsXya>z$U7^2
zePDBO>#`wt^D6(yccCNVX;e*J#M9cc^3DB<nudmVe_g~w3hvy<c!j;Yuy~&%GskV_
z>Z+;jI_W0+oV6s?(J6MH_;G%IFQ1!B<+9*C`Eamyucl{ce?IaX-F9i{6I40^Lqnx!
z*Y1*k<v@_9!C^ZgzCey^UJ&<ey{qxsMq^O{g6}lPScX~8fv&MPK$Aqsd3<2yD6|Ji
zn3)A$ykVfjTqUv-R~hDUK~#0xflOy{i$U|ub6u~MK${PXn@s-|j-R^dfOT4)l;RI=
z<@EgnY4)f}et9gOPszQ%n4%cJ0j_4Xy@#>OoK2MY7!rxzDItuME-0nL%@|E_U{V^|
zv<2zJoo;WOKM^T%ro(RCD%NdFfqK=4aY_)E$MnsgT0qoR!U72T^Nfr4cw#=`Gu7Cv
zLVysULq_#`McZ>0W+7g$Hm;)IZ!;R2Gtg20FJ%kR$@EbEN(=7@A80^M)QLd7Sd>Ee
z$B0p~B)^8QpP?INVA~0yG8zvJOgU#SKQ)QeR$$Qk%o7#+O(QoZZ_IO`HW3+=cumG_
z_=<55uxdzedh#RDyxKVK_ZUF7ubhWadTt(W=ewCrW#sbNOwDBVYn5F}pQwh4YVli`
zDgCf`w|*}3%Y58ou*|<#o2%dRXGfn~s9$+XLSxDoFW7X!K>L@!$*M-TGO;0zmDi`!
z+Vc&N<yX~{txDPN5jJm)+yO64K<{==VRma~{SgJ!T<WnHeekRaZC6GVSSuBLc*QIv
zuu@jrcW1opecaALD;lEk(s@q@ATH!D;SW*Wevc^e?TOXUBfmuc39kW^??f4HQ~Ku(
znCqWwX#1dD0PQO~m!@tIks`%qj6_S(^RpRy6`fGel9ne>CaK4hs~R6`fd*+3sDp`n
z)v}etq5AoMD<d0TqAEe=1^mrN)#Dm&C+ULaE@JWys=YfbZ4FFp<)w*Cw|GqS)OI@2
zH#JEc^LjMipi|rg^d<`Zu6eFuUTou$8BCh)-9hFfsifbyIsaklSSW=RRq3^+k~oIx
zud90hpS~+ZmP5KxTT*1p`wwA<7m4T}e{{T&np@@}Ttbh>Y%#or|9#n`bs3S=%n_jD
z#+|?o$?Hv4zdT!dT==1kxEaUDZ~P!5_-3OKQx@SnF8=N@di<_RjrgVI?Zjj)Q~vp(
zGoIjup|(Y6(vsA?*NtuMVSmTXIq5E&<y-VVcSD!cn~Up;Y72B6t)}Q$UR0~8@`@7y
z-RVa~wRz<_ss0e;BIzkIbblgfF<sG37wih06rg+nR)5!O0%&!o5R-ZsZUA)EU(N9n
z6-Q(MSpU7vZ0V)nJGAFfI71HF5-EwId{-mercc&#m{@$Ak}k6W%mO?n9DE<^BR&48
zhYy6?BI2NHKh|1F=`cK7a{x%&f`N(ng3TFxD}BW92sHX0RxKZhcQa&`ym4sox9(jR
zD7(V^auYS*v8Yv2sEk$_CAu3{LgEh!B#Cdwc4%YM_}Xtc@I5s8jd(aSGdkIl5c*GZ
z*2XR1b9RiHC81x!LM!x*L3|_G#YipC3BgiHe%6{)zRt+HglRB96wE%H=u#1VmGJWa
zI4l6wKW1yT)|qkeAaw^22QN`m7HT6A2wwYp-5=r?SIp#P?B#W^B(FDVkzQ}P%~#3Y
z`6BAqe193w{?+frIL=7fQ|daH)qpdac7wv+_*&iFif;_uX_6ssay+&04eiVhe37PN
zw5iZYjNmD4fkg8!E-RmU#AeUwO0CKdMcbW-#!|f`w@GL30=sr<lbGLTC3dy3yOp&x
zpg@(zvlubtSUTKm?Zza4ud)O~->Yfv{0c|J?lRpnNqsDs^xK~MiC*n?X6=%W^3>($
z?b%#s#(|leW~<ZlX(*dxwbSwg%IP^^<3hM-gu6C~7jo+0e(Gzo8$oK#xvt&qdn{gq
zeJ$todSF0!GIecC+a|nf2awo3wB?xqcu>YF;@4#}r9X~tk6^c=bF?y8ssXoi)`MyO
zfNg-muX(<49X0E5RO?Q-NQ9oaod?kSE#nj*+q2OhIW?wwvflNIR=`?CF6*gd`U^-R
zI@H=75X|kq*aPh14g<9FK4So&z!B`1xk9wy)^Fb@p*boZ5a-DI${qYH`%s1kfgxc0
ztgg}po#?J{oldYByK0o6GXk93dG{)s+E6<UB6%KT?H8Yn>x+X?{m-64x#$AMSs5w(
z8VzPjWPw{OSb<_eP8e^khoJ=FoROnIPScC33Y`}yh_jaj%iA6k91I)nnn(T40w-A4
zm@#rm(G2bj$-B*@6zp)n-+4V!VWpkGW1@T|rjtWbYj|*p;T_qu&8+`}(L10#U|+7H
zPk8%Pv+r0R)gNl9hh!PhjV_p%G`=Y<6qkaRB}#zyaAW3(=96+5C!jTVH8fcTND_X3
z)wLZUBrQ?cyIzlMHzE4?gt`HHx5i>mme)_l$#D7Xi5G(gV0iPS*lakfZJZM3#TE@b
z>tR55__yhr*d_3uCISB(!Vch`9-0rn`22pY$=X$(PRinSQJ2@#&rkB!F!rBg>Un&K
z6a>5+UfhuQ*nNaO81p(QM^g|wa}N3FaXgp6f3<vOE}A+9ED@Wdjl=tNYfZR!wCrw^
zsvDhu4T(&ye*}P*TSHAFU{Gk*!++X%phC4jb6GiH7i@PZ4<|HzggZCKKfOJG+3emA
zHte6@-iNmxh2%hDYY0uXoOcW{60goQ4-$%HCz950;iqlVt&!4RCiw-`o?(!UJbTL#
zxlo!yooqjvkx!UZh<u3EGW$5&Pn)UA;oTAaBvSc;P%tyGgG1*12m;3sWq#DLp2;dx
zqo<sjdLt1Kj9^ZAFa+G=cf-CwX><yXM|Ln69%2b=g#&-hp$A>8W9DlR4}Oj60;9Z~
zsAspF__ZC10){?1Y@|=~<d>or3^1#6ne<^@mcs`)aufJ~hHft(E6EmO{qGg8Nw13i
z*cD>AtvXWoh(XW(^<^oWN}(1ZpC1ceA=4}DiTJW_r1JhvbTsvLe%Ak=su)?(`9$A1
z#LAL+F`Oal`WL>e?cec4_;OUJ0>@I38$qM-!KuYlxk;j0M7Qr6Usp+ZsMYv83{=OE
zp#r@Roalq^CO?yR0AeQcvPb{M5OB<c_lw`o0Mx4$w`*rRoFDJ|R<o9^BLsM|dK1mF
z=iKs=;PbFTtNe;p58iJ%h{^F)5G7B_G|J_&9<%IH_|?!4yrejBw_o*C@?00qMVzFd
zQ)lf|p+;1;IMw;E(r6oSZemMpV;yU|<OsP_*Pm5Oih?mCM-j7qotYz$m<^yqF2g%}
zHLY!k?fs>9&f;TnMEz646_K4G>D^{&&eBu1ivHt*<xa1g<}baA1aU$i`L6`F4{NXc
z#yGUK^B(VRD5UQmPb>efJ~~f)KV|oqyW_KfHJww2KrjX#jimu;d6k6q*47tcfx7d3
zYrh1&na&?i=@)lOdpO4OHTFF4d%~?7njE&rdTCQ>{b}f8=3Xdn9|e^K(zxZ!9co)w
zXBIXfx8PtASuwyG3f_-ron_Y&p1@nD{Ii(mIwipdgX*h<2e1?byS?fFUmvU0s1B89
z+IMu~e6*u~9_*ViGb6epYpL|?nV_UO{SV4*K4OATRnI$dS!S-@=%bCU=BjwS{<e7d
z@fK+HEy~ek?!m54341Bc7Cz&&#rswjlkEV$iNCmhlCBHQ69G+BABtN}lvv7PSUVhT
zz@?h^Ue7qJo#@2^bvm-aCNEEol5$1ZdVzs{bd-rEk@hrG<c&H>ztQ$u@ACGm7R=hu
z5-5v2QpU}I42c{lyf|<~uJ-GJ$R>~wRg^x8&r)L>E(7MdWL9^6T8tuG3uokw+4$y?
z@|~pS2k|5O`5x}N9hbn;n#Yxr`r*i%F1DSwQG^t=fFk|15^^oU=5bUut?ix9J#qJe
zn+fiyu4oTT+B5Cg28AEIuP%c$+c>}S2p3Ym&4M$GMXP|#E_W;S|IAY_{7}IhnGuAF
zYFSSj$AuTOU3Kabq2l;4JM~7dfbx<5r3Fy3{WRklrg2>y9hlCaCOfBJxNc01FyC%8
z6Ic`1eVecUCi-A*ci6^pyLKyn(xjBvND0tq5pk*jb{hMqoIAyQ*m@Vy<DZ7hTm!Qt
zL14~L|Auwj>deb|vUCxq5Sh&}(eH_QTT)t;p^3#?OXc{Ec7Te|U;!=>M_swU<kPx6
zLg)=7rJ9qEy%Cq|*YK&t$=^FMy7uGv<{tsWjFXmjql`$oqKky<vxt22gyw6?2i^=S
zf&*jIRa^GR+2dc{uk4E;0!D-7<+49RvK{W`H?;Ik&dRuqcln49hk7Hg_7Of2658e?
zaYwYn3#B27AYWYYZ~K>QN*N%Vg2I6!%t1J0N9TimI2342!seyrR~N*=MfhK$DXoL7
z`Ws(_a{6&t+d4_T4ylF&cSf>_p^|*>L1Hx?s$fI9{egEd@CkuKNUS)!8}&#kYz(iZ
z1}k`r?F~37gGt$XGxz&x`6qVS&=#oYV^-`@tarE<%YjPF3Vg)TNc*J$hSr6I_FUO1
z!hTI#ihNHl<CEd|L`@I;i+9+dcJm|00*r()8L@V%pq-V4JzKnc20{e@XO;oet4RTV
z?|}F#6K+NdjvX(+meQ6verkq{lXpLU3oyanyPR33{b2s9wyV=90Zex7eX=>#QB$!?
zF$^e<0H$r0>v*-a4C?qtniH$WV>2Hs+9<TARBEmm+a+ILw5`g$r~O`X)Uz=9NDkzb
zcwzci4!)MLIj<Q=EHa@YwL6N26+mr?aQ<Zm=zaO+5{2=-O~C7#5?hZ|(TC$)+K`=0
zOgzI7THkAIzi5*@KTOb#_$>)}WM<0gapeQy+JYFZCnSwcPr9I!Aet=jCXzVkzF))|
z22$n#n(&;<BS$u@hnYSJzB8ZO9~l?}03q8(0yxB*ld*E%vsNitR0vrFZ>PvQqvV&T
z5d}X`usWXylxJ1@p~r`O=TLH<_VZ##zl2QZ8C$vf<qI3PPHRkP#?s*XsF6n(DS$}#
zVSU*3ouVmluRkNSJg&lAYpKKM64@zfDV0;00bs&I@v~}!pc%lsu${p%GNzS)co9QT
z(#N|@qA>3!TKw4nOY)J7sK`SAm9mG&vNsYxU>YUDOASuQh~%XG`kv_|SOtsA%yU=b
z0bTHkuN>Wd5tYn$c;Q3YNN)oF;1psY@YL@R09mO)l!U~5=iiI+^W?-=$C>!bT{&NZ
zJTXjZ0BJ0U&B!W5&G*^+?PevHPB}tMS@nA5)wqg_6^td<ehDVupDpv&b3J#B5-mQa
zb{{t*Ddvljo#AX1*(ZOThLr?j-OF?!n7q$7S=!R_Zh%ReF55dj_3DyLD2dVEeJQ={
zorYG|RmaJ4i}4`I*A5Fju{VzhG>E$9GyTXs@$e}5UrZC=P_HN2_wUp6iM2BIiJfON
z(s-zK-Ck>Na0lgjc{Vm2UR+SvClZ=gdH3yO5bU2{qg+HCWz8G=%ftV2Lgo)EV;i|Q
zt>3BdN=SR$^jSyRCGc60^s}8xttU1P;~NrOb4YjKzAXrKIj+OY?YBr}0vfaUl#vo}
z?e*2d!Q5ZS57wA!U>4DU_GoI$g7^l{%?_C0;+KASCo*t6z4b|(!=wL_ZpRdPaTMw`
z{uZc{JaB+Db$X_@yvg;m^(ll>!sOor`8xMoTet9RKr4enTHP*!(z>CD`ZVFX)K!H_
zplh#T(|}UkDGg`Ro~?!BVrf{_sW13;U_kpGxS+rR05p4AGeBeUyNT%_@XrU86$*(+
zfoezG0ay#b<(=7IZ<7fPvTS}R<xpzWqJkmnGb7Wg6V_1mS|!nrz?YNVDcGE=Jpf;t
zW0IHCGU#6ECgeMfK7nqK!g^r1;Al#lT(v2nx_CX3jLhV-7Z@P92rPM`hRlOC(pZ)g
zGN}O@CnRmH4)0>LzjbB5?U5?Wxw_~C)k_dYZj|&e7joj~7B^h+F_*8j1aep7ul9!9
z86oU*8a$p#<JOz5<RcN1(1C7Qu)ILL%Ep%yVmU?6|0#3>;MGgaB~8bU(y-Z=6lztQ
zxWNG(Ux8tzXq0JH?!W{+s*=|F4-=a{8fRXI->LkcXQ|VLU=sFylIvP$r?ct1p@;et
z<draM>}IF6DDj=cz^WE(BPaLjWq$-?R1XxHT{uE7HzBm$$Fmy^hq09D)V?_<rU38#
z80JIw4)F6}`Qn_g+R`@6b1K4+|G;*wH9@%ym(7uD%)7)FM`V(AbB{mFwQe-qNIa7=
zQ<0WBNqpyOqBu;VY@nPm9+>O4-Zq<*UexMj2vobkJgyEq$R{6p3@3!lyjuQJ;gmx+
zUOuizY4ZvJ+N!}yY7|kbaaVF14iycgcG9dPkYl%|wLgkU;>4dt$+r=;AKrKF<cWNM
zFVE-p{-MB!Nh=dTTB*-zaIyXhK=xjVI{fCU_wL^(?UW3n*^L2%eW<A?4j(CW;i{@G
zU}OXUv*(%ia7v9o@m^?vTg%(8YNV)7ExO{Q(0`6#T+x-5OiR!Kg^Mxz&oW$rvdLfT
z^=GW$sMLl-MGE)A=p^2|Daj6mUhGwt`+j}phm~K)aHCHhMqQn1^0Wj8WFT~q({lr6
zTC9C(n!|t%F}cQueM-9#4VRUnn(Z}?F}vmDd4lrqyvc%k`jNfw6cjjc6jBVv*{(Uz
zoH3m?-4k?qC#Yp=hl99ssz(DQT9#oh4rI;y#=8@LF6-`0=(r02>xka+w+UdRa-$;l
zMQ{^6<<XZL!tVlPM_QbWk$6X5&TwBWkzce033896MigbB{H@71g^G}k-|77Ul-56U
zUBf_)M;5Fn;l*Z20SAg$yxkb!E8TGpagVXlc@N8-$X9HOs<u>^es+4=@>ZC<=h?+{
z?SxMIOE$U<87pjaDs|&sbnh%0yi$RC210gE)d<g@hQOptQF$5@tU(Qz0Vm`o0p%9+
z&$lc9Q7)@q$3p#yKaad$@E}4LPka4}t*(_Sx*cvXvoU3=LB__$wmOeR24WJ<3CVyf
zlzs!476x@em6p-tLDS^AWy!;w$Rx+@*voYhnn8zV1Cxmvp-`|p=q#JT@}&H-0|k6W
zU3;srx)b`+wf~Ov=N%yZZC(Ib^s8bhAE=yK4CV%Y0V75p-U6G?f>mGpl(=v?zGMVC
z^l*&W!_`SeOW28U%db*f0|5OCfydw=tyNl3^`K&0`)ep(l-^F8$_ZS$Ml!p4?aViK
z=87y@|Aag{st*lhuIl?CtvnCxHmMGxhFs*yXqc#e7iPxv2I%P$2Low*t`Gc_P(r!V
zqFa7~rIeFU+E?g)kYZdmv4Pw?$Sv5rDc^G|yOscQ!3fjjRskFg&L>xZ%+P`_z9Ix*
zA36L+3Rj=g#V2eH8xDu6oE$#W|LxDestVZ&T8jYP%*1X0#ojlm;85!lb2)!P{RkCt
z+L9_v-fz8181lSz5$FA5z}TjIA^i~rqIHGHwCJLL^z<)A|L13{3<Cu-MF~mH2d;y7
zrCr<_G*&x?(T|}%$PR)y+0hm9=v{&FD`CFrdAvqO_c!mXoXyNwP;o4bdwb(-tjDP?
zi!D9R>O3lEkGjQd1%4gj-e`~LUyhWY9=ofw-M~(TdXa@QoTq~ZJ#n|wM^OHf@ur-H
z4e!vMUe4}$m!7lYuAg$sRx@7d@l`*L0Hh~vmU2u@Lje@{cCz=y4N-!w5x!GCfq@_8
z(Q~?V0FylY64m)~Cc`lB;qJh@jrK8IksW+zZCZg^UP(g9p#-BEcyF(o^#?lo%Mk%E
z!~+-ldhrX}k@!*&!^@ZW=#M(Z0Ilqg6=)GoC-PkkaeDFYDj5R!)k3(+Gc!scxs#<(
z&MCl{f_j_~%s}=E-}>{B{=v-CEKaI)wR6d$yH;sN`RN3p#yTxOz<^s2yh^p&A^olD
zu$}TZ=g;n*J?5EL8T@`&Z9PT&V&X%CW6U&5oj|*wm+G0C5Pt#fOmn*LeF`2d+DO-6
zD=8;1JOpHN1oF-4cjo@!GPpsv%C%YeXBUI9vFrF<!ZY5H-%qQ`WOXm`r`*o=s9WgH
z4mHc;AD>e>j;Kz$uK)ywFvh~Gb3kV1;L`t$^4(;5_QXSf`{&ib5Plh#J#1S#MHQ^o
zsTP@o?{u+r*0U6O{M{3%J)p()H6yD8npe0IE>G~GN4d%fXO1PfExlI;obTBHe#^dx
z6ArzKt`m8Ijfe1l`wl7nz=myr*{nlcI$%%=-ZS^7mYNtym0rG#m>1d+%udn+giIFq
zX}koEGD~MI&UO2(c`yONZ~Z_dWtWg|Mu;*@<3LfNJPbRS$n<gQP?azy?{Jnf96IOT
z!W2eH8HzLKmc66;2Ke%0oQwp{0I$z2;-2C&E0=Rq72Bo>HnsMD*F^@Nf&+mxG58iR
zvovnrOX=a5q2u*BJ<4~T*#njtrxCs9jg?;sUWyJm&*|K3@?M4SObiR3OgH_|wRTNF
zj&_{xZr~MuEix_u1XK;fJT0A$V<#CB^{Tcs6XMP=ne~k9n{kW7fu?J5q|TV_4l=fk
z<r}Xo7QgMB^IEh`czKiWvzqW?eniW;lfbHa@p#v7=CUBg{%zLHd_S`9i*iq)h5wjU
zcukZQuHe|%v~|BAmB5l(?PA#sZcgcN>2lW@PmdRg((ZAb%~&(?#(`n`cV9dAZQOqS
z7g)nhV()py7g%i9A0)C+KoHE7<48LX5pc9?T^<%luEa_QcVtQ$S0Bt4D0NT)TMGx5
zUyv~~5t)3?Q3E~&ze(Wzvv&Q~Q)A$%$=ILt=tyth%F0r2U&}WBU8|u9)zicL+wwtJ
z9i`QUxM1L4cqEHHo%U}M_Cobe&dqmY^8rbR{eqv9*FhD&Z$65gT}~4Bsc%Q<HlOvc
z^1xbd-3lEy#zZ3Y>u#CcZ}zrdCJ3uFl9SFsXZUk;K3GowLE=sQ?H<zvMjt%WVE0R7
z%A2)Nxc<lYZbh1CKN+k<oPRr3PWB2raQ;>WH5h~H5e{GSUyMU#=1bLGL@#l?`oIFS
z=lx#ON4Ktg+BC3ij+(_M+NtV2Z_b)_q0_$3H_Vjl&gV24UQ3a^#pT#ocjDS9S%AfZ
zuQ;UqKEL~Av3T8$sTxs+I=?otFGxQ+J0%drJa%ltZo<&}CWu~#VvIK%wogmVZoBdf
zW|zW@cet{&AWh)80nqSA-xdfynSgiE)TnkUdTs_Ag?%!@ndp>OwAuld3MX}9eLuhv
z44iI$E^EG~y79hSz;ZvHT`tJhk&w7h?Vy=<Rh~4%f6)9v)v(-W<`0K2G()4HmKzV^
z1(D}@$#&*%o_Lm-DYrdx{L&$JI5FCRp{lH0p&W!nsQ1+2_X-KHHZpm9&R%^>b+Zzs
zYos-r(T^QeeGpE7+G}-nR;jpozg{Kg9NX(*OKAENZF=e!n6mRZT`hb8F~-Muyqy%?
z&QrnSEynd~y8(RFx-+K?lBg}mPn<(*?DvrQ;mN$hr+8Bc+G*?ET_acrNs@iP(CQLz
z)uMnvyBv8h4(T(z>khhi9Zt8s0aFp!&XtmfgFo?h=Q(GMVGyypMEz7r0H!^Uyy{_#
z=nfJWxIO)$`B)0yKGUkv@KXNx4=pMV^h?dnxWH#pjEKu|8_S{N5Ypw-ITxF0h_9Go
z-!!bTm&Pr9$xB4eh~1N_VV7X>P*Zz^o_<}JO=n}3Tg#EyTEd=zV(}57Ky8(GD~_G@
zOJ8H;d8AQ<5K~K_oQ1@e{IhzjA6t`-!MD83`mCEuTm-y#6LkiPgXO^|J2yTh3DHiV
zg?cpZ%!b2F96JKjipejOlw6-A3c8l5CSuVh&GQSF(&aXe1Och8#(6)C?IDhbW_`zN
zJKh*HKdp$n8Q5GLWQ?2rSuk<83w|qChbk6OiXv?P_M;to>k>?aGtOp89`&8|gX|f;
zCxq959ij<f@i!Wtkmbn-LDW)!W7SWkaK3Elqn;-Zg-?R@XC{u^KrcVbGZMg`&Z*Hi
zaebMsdt_0XFH>Xj-dG^2D>%cisN3m5ZkR-LL;E(Op{qB=Q+!zWt4pHR4KBU9F&^|o
z`5#{lInquS(X^ZacUD~!>0ODI?a+BjOunv!TX$ZH7CfiaMIN-MA~@g*01zl&%JUN5
zfWO|HCuweoc5*frVtWT1^<Bsv3vh~aE&~n`kK1m^952x0yFEK39{Ifum53U|`F?zT
zd<T->F);qbcsk|0;OvWJn=DkUWl8P1P3m&C4hr0~2%gP5Pl$SKdr@nhDWXOIYxGs7
zP?^8mvEaR_8VxDjttB7sCeCB`&iZ&nvgpXeITN%NZ=jqg!NACBb5ZB>l)?K;U^FPl
zVF{X4Z;{FWM0C$AHvBXMY&7pa#pHo3Ev#R^eYo@_?u-PO^p9JM4rZyG+kJm|9PjV-
zViF-B5HqYUuyEG1?9~OqEAFeTIQ2m7EnU2v*`fp_XK`A8hC=6NzdRgz&6?zX0w*eR
z<yA}aI*(*kcqT4#!&z|?IU_D8kt_t+pW!fSyy=)7Gir+em>lmg=Yaqk*_nA#2Y-1)
z=#g~^%{BHq6=jg5#+7xB<WlY%t)h{3T*1}-F_Yc!Zb+lT^)0NP$Gg%_>-OBJiskuH
zOn@$(pZjy;D$>+QotJ(uf9Ob$e^nyqu~+i9AW7|P<5PV4GCKE9WNyr_cR69Vx5$R+
zeX;+CtuK#;vJd~hhfvv5cE*;JkZtTk5ve38`%;X3WZz=U$de^IMTSt6eczV`VGu(0
zec$(WjBVb#=lT8KbKY~_bDzUG^_Q9NeSNR%vt17LOA|}$ZijLswWTTDHXG%Gr75rS
zmr#T%{}V&y-zSZ)6$q(}Laj?IeTSCEYhub{?F#wkMb)ZebuEJ<?i~-#ziDih)QTxc
zYYav0>3U7rIw=y<xcpZh7}vNkUtOd|6L+XQT`p%W%(hPLwYZMnSTFK?GPSi{L{eGv
zY*0q&l{;?M?w!1j<M%pOW#WJNoURo6TPu^>eyi^Ec}_pO<LDSB>~%YYaP(FT?w2_s
zIq#l&t#Zht;TE9C&>cbI>>E4x8Tj#Ww%zS$hU45(n?%4Hc<DH<kJcs~xZii5#|65U
zbnLVBbX1JEP}6FFlv8KP-Qd*JracA3;=)OxHm6HhsxsNTi~oSnRd$G3*v)A3utIsj
z_HxAX&jev{KnuzOfwdmpss1F4^mYy`Zl%`=q~2~6X;W#lYa%yAe0vnZc|-=rH)~K`
z(S}X8D_i2wqr>;(#bc%t3Qqen)50S6=ckIlx=c|@?DvA@{B7nzTRbvrOU(k>)->cN
zuy&%)@q;~%X3(Hlw}CP)TBN*1-~2A7kmrGV*gu67H;*oducxK+8Jcp+TwO_@Tg#40
zK3x-3(J1NXLlCXQ>Ox==yP!R?lMC+l%oyc9ZI?3q5k2CbHw<$m5|#uMmbddMfUXcA
zjn`!15ky57HzYdM$pXU!`D|<3sYoF&Yo{oV{N*P6`h>A!Kv(ys2DYhy?ImD(`Agp+
zd}36l`z=*86*5*GqosCYJ~{LwYxUdY8AomVdyr~K7=5;RB#^rJJJms3+kccBb-<YZ
zS=|GYrb!lbA<buJf@LE|qeO*o(Yxi#NChC3D^ia&eY0(~Q+C<#L^m$gJ@NH>(oBYa
zg4y7PnBnv@#Q-vP^Dg4ACr}?FNbCxqe>{1>vr{e2NU2%#;is;1`k0sut4KIm1JCRu
zYUL2;&WEw`UYp3z@*eMpfeW&40@i8BP~+f0tX{#=ri@kBsJNQD<+Km+<PE&L1x7=Z
zmsI2NHIdRgl4f<FSQ_?vyh<pHg3Xu*xQ`@Tm8FN#jwoIQW39T`rR}eJ$d$Ax-_nq<
z^N>hswguy_?7n{V2E@VTHEF|QLwr*nxf$!M$VuTQnz?gvxZSR%S@AKv2QtQa{k|af
zUbey6aikI(wr!9n`pV8i3a{kVEuC4n32)J()>8pXVAwuPq(s?_nXK%}5w!xQ1jxO9
z6N>phyrlI<+ws_+Ik{|I?ZwFcQn#<TMo&ge{fxOjfGvA$WQUva0I3vEuk_jW)4BoX
zW11^fPZ6bALM%yfK{{c`x8dG9MBL6)?&U6(IdffQCRVrIn}F@9En*Cg`F?Ag5nxf7
zV^Co>t30K7ZMVh$j?DJW{_?BD$ATqYYkV)C0~k!#ks|-BFUSo~<rkELV&FgOTDZ9i
zKOR06Lrf-72(aC~Z#}c?a5+sg$rR|xxPLt%b7}G`4G1WAtmrJVq0WCrU`}?rA%!Vb
z2<S=IiJ9pyfL&tZcEu->uGQxIHcHo(FADm<)ZLD=mh!VZeqGS=e;rN>O{O?sw72nH
znk1>yh<eX_QCOe+VxvY5FXGn{i;#KM`2)5(8+VHq;=STENn0`h?2$hE9$Vg02}Ef<
zsry#_>$oq=9~iY;M?Cy~AWqs93QIL!*9f$W=AdWNomvgeey^35qfV?2PaHkeyQ|hH
z_~S0SpZK&6Cuu7sr_Z(~B1O2omamydao!!8@s}+gtI@3Z$d6gX>mGl5n_MwQa9;D@
zF5dCVdKh;0d1s(AYJFDKGu&df;|O5iykUEjs-{!p1;l4}<pT$)XpT@BX^d!Ga<lIC
zXqEZQCH87STCc*bP(J%v9ZM&vm-4Wk!nfsNK@4Tuj9U_EEL03S%1ie5v16P*;i1@d
z*m*$(Xc_H44Ai#(Kf9hz_v@F^Ap2kG`(HwYelk^Am+|5w+9<Kgf+i4&?U*kwsgoEs
zQcW9E+|Ndf`3>g<(p~Us!|%4DvPF~-wz_2_mH#+Mvb0ZYylGQy<D+Wa+sG1u*9>D$
zEATr_RTgt68s7*1nW^DH@17%2a7V3p3S05r?w{B0AfHCFq{3iVdG;yPVp85lCXW`d
zQKiEEQS_XrN?8xftnQ$fFbZ5JweQSRDRo^t8RrRCEz#nTTNEMfHnBQhmYC&Fn7Er8
zLT~BU<S++FRZ;;FVQOr;K+lJrmJZ1SWipkf=SM@z@6bO8s~@#2lWpGLFQ!OK+-s`z
zoIfC<_aDkbj!KSqNB9|IEs4h(CSFxtpA)D<>_)S)xt7m+D@R5@3!}c^>00?|(XQUt
zF;7eZTW-Gv2*<&CmUj(V&$z^<weM`#4L*N%%y-oeI?rA1K#L#u*q!|LgG^ngTHb}k
zXFR@g72FWY`d9WVrC&lu147y<mGl)e559t{k3_lmI8OJHY3&)4z9Ldz=_+e-HmWeQ
zg`_JEOHn8SkSmbee7lVjqnyHeQ&ZvOS5>VqL>^&cHFS@XHpS|;-@2@K@CI{QL~z*j
zU1L%x8QP}z!DU^KYMyCh4R8$6_@aSwjyUE2mNUAopLZ(ng9wF5p%QaO&wE+?cIKf(
z=`R)sXLEJ3Z^-ZHX=Sm3Mde2-1tmbumgbF9;k?x3rjsPAe12qkVfmr4f&}s?6Vm->
z9^wiXF){X8vo5NfUsP!~4}Icbnb_S^t<8Iq{O!T?xSdipJCkzyiY#G3E0~Up&-p6+
z59Y$I7A<R`Wosm380{m8J9op#FhRcY-LDWQyON^!som*0H&i_tEIvfplc(vd7a0yP
z+JHBT+$tQS-pZ;MUm~O$4oJN(p1tW~=H>TiE!q402sFfe4QIij=TDcEwrVl9vUHE6
z{7=2-E<U&jFRc8U6G^w%7&7o<iV}$6J8m6>sFXU-rnV#aG`?L-Gc0X_;dLH=rS+7*
zr<JQo$$qoT)zHAE4vyzq*$>P=SmHuTZ+)7`)}FA2!U^V~>WTKUtL>Q)UJ8@4dY7^L
zJ=YG`wp#CLF5V!V`Nb<-$$__ZtMH#7xP6XaQ^k9ZB*8pdg&d2Bp6n8lDm8ek*E@D6
za18@OU}S^=r=_GVDD0l8qAgk|3Ui7$2_&qqM>=D#9Y}I`mJJwN)%P_Hd#`=Y-$<=h
zv*d&x^k=Q5SCH;35O3V#a=Ng+kbGhub(f}2xemSwNs{$<cTd&{7V#|ltyF^cZ(wd!
zHHE!onL_q&{;`CP&3$sSfVaXUyfnzWj*&t6{dFJ9k@dwMt2hn!=h`U1_6a8puV)v7
zqEmA2@;U7MQQ7R)J_MY)_gZc>Pf<`FEKf(Wo``FVQ}r-#{7CgbZXAQTt<EpMiH@<_
z(5i4Jtj=Zw*!oTXmvqbOL);oKtkMkaBA7)Le(UY{$u8QCq3FVn5u?>4<!d|fJ>gzA
zMmh<Cxf}zWVuMwh6c^(=VI605)7agg2EK%aOt<-Vp2J_Z5?wozeiZp5o=OUbqf{1@
zgw3Pc;F12Jl(3Rs_emcouT%ZQZpw>0$==H~)RsQ%;2|O38{b<r^!kBw?oL14e2^8n
z3D-&9vpR@*?Py=pW9I#8v(DZkIQ++{p02+8A0G{)+Jh3pW-HX+Vu=&&gJj?Cw0E#>
z^xS&vKQ-!Q%Mx*RxaQ<JY<2_h+O7`t2=HGy{#!xILxC;d<H_Zr*<3mRJF4b2v;2V-
z69lD!d)^1wNr9A(&KK4Rec!*=e-f@+c}soNehmj~TL$B$A{7PTuz&6|M8V&Q@7wCC
zZXXp76SPyo6zguqdGiwX1oXPf?z_76hQRq$*zr~THvb7W*z*4I(NL9l+tcZjb1V94
zDblvXV^)#)PZ>~mihr=;Wa&VawZJ10rFNbu&fPyAT@NB(3LDJosFr-LU>;G|XL+y}
zm*Qvf$nB3wk>*{3c7e0bGCx&sJeT)AuRXF&Od5qYc@z8Uu_ubOJ#Q`}r06+dRD)KT
z(rHt^`5BpZB|-(Pq|5JaS{Nb6Rfy+4+_Zx`O}`bkkik%Y@BMd4T1Z&$T9`d`nw(9&
z+wD@LBwHhGZJ*H;)LP=L%%ivym-)c%sczvuM517YZ^DG51EFbA0-ba8iix76hf`+V
zz<67iqUb9F(DmI>rrgp?NpzPOo;2kVWy(=b@!{qLl{n9Tbk|-Ok5!|7haqE7i$8ru
zLq=aH`N^;8CP@~$T7S%O-;EGqu0NZP%jzUZlAd#mgf}T=(>}(%T&DUW(DPH746>~;
zdKSh7%ao)1mFd?IEHJDtk3h|Y2wI^7ww&T|r1KLP*rP0ex(LnF)kTqH>CqeQ>*B))
zGG;qnoPNMRg{0%PLXHtyHG~n*+}mZ)0{5&m?RX1Ksx7ze<%>OA%`FV4ZNV}CoWI$F
zL7CPyU(hmlS&wk{s*p>Dz7gz;A7$9Ulx=#%*L8~FC4ZppDbp>zdYpx}CpXORKEFm}
zxC;5I?ujAW@=};mqRsIBJHNF1%@2GG+`6PKfg3M>uN4;7g$7{jSJnfS-lHZn9%tik
z`*SWD$b_V$+thS~h}{c05(!61=N=Skyb3Id5j<9Bzht&R7uF!C^{=(NA0#Sat%60r
zz`&F(6VO*bZ*`zeu}Z!b5v<}rp=Lsk82Z1!aoH2A<ycEI-CTKGCt5peRsI)ud8$w7
zNy>?`W*rro_xxG7+&uZG>j%sSIQo}2nN@V-3?@K#;^9JqITnpfQYpiIo?oJO?|E1C
zU0y!o96TeoQHr}4yB=N$(N~$x84R@mdY)gNg8sY+VBawvJ{Dw>yLBJXk|3MVu2w#W
z>*@ksH#W7o<G4Aomacu1Hp90zN-%?4^(n8q&Abr5y-WHwGHM-nrX-6af4G~U_%zaM
z#=Kd(0g>W#l$zfjpZn@5w~iQl)9v5O>7A<JR9`=uTv#8nr1qU9azN)0t01a4X~yA;
zRxH3Pi{eBmTm0aTIUUx0s;VDXQ%JHc=M}AujOfu1`V+mf*HQz7FZt4_h+)im04CL_
z;R%B{ZHN4c#&cq-@0^4G+K9$Th7I@L{qbUT{=6>rr2BPIg4#v7Q4=yYCZ88K2BRiV
zAZh*AH+;r@27NMr@W*Y*s<dfk9u^MP(LdddYTdnt0%-j(!j;KBYTxD};$Qa<SB&^#
zlwf`#g(;)wgiqNnw{yvQ1U`mNhz}vQ;-E|2rG1`V)*_m}UJQ|jXz9mR^Gn%Sq18JU
zz7iD3kJfV;Gup$y94g)i;Yf!ZwaaN-$VU2VmGzB7g`b}8Z1y~xnWcxTvnuMP>2P{Y
zpv$YYJ~A?SH{g-NzS6H6E^qgYroE-+*CW+<_4ml_q>d4rR8t3G58AYLNc-bAQ=&?e
zC#H34zl}$6XI@c@#F0+1^^`I0igZe@E8QnOX05Fo2!6memhsXVLjXg-s2XPI<>mSR
zw@Q95G(M2hKboea1j6rL$pz@^Fv>yULcXfj;*{Gf9YN+{+7V&{)0h8@&JxdC(t{8;
z#yUn{J$5~^**wB5tA^Fib_k4xhxJh0XjF`yAA3ckz~Ijshb9hG?e}YB;+to~u4`RU
zDB~5p-}~w9V;<?5`6_0$eCkCkW3e~XmW)Y`EKCbutYM7nZ;f})UYL^R=h*L!;j&*j
zssMVXsp=V|Q7es2aCrvH<u+!2)?fjC<NY(ahZ?TS7Gdn@r%d`-uxq{=R^kty-x8c(
zcAWVzzz$VxQg)S-NQY-v0N4N}N~0x@8a3U$D8-zx>er(c=^H;5CI|h#Ef&kE$jgLh
z57(V_V@L+2RtZrOeVnrU7=tGfR^#>5!H2a!<4vtcLS7^Q>;PPb`-O|?F)7oWu1do#
zE9y9{fNJcJ!^zyT4-*|(b-2yGM!f6~qul9(C7ecl^-Fqq{vGy`%F3j1^TSQua3npv
zc&0`Dn~DOv?`-IezruSRnVfRF#~K<=e@E62?QG^HLG;kq;8yG1(ya!5-HUU{6J5Ow
z@avL<J9tWQikw@%+5*wL4aLtm&nQv5?Kh<1(D82n^x<z+npPs9aYVg>p0U*^hW6Xj
zTY6L7iqEeN^H2G`A1Evwjx$tiMJ<qjDi*}(%dZCIX&{sz#*)UpK|R?<dKCQO`}zGd
z%YSwQVgcB+F<ZPJpjF(Rl-%B5G8g#h53Da&VvXDqum$dDUP)zaW9H8h251f6R+%x3
z1j!zYK#3!9eos>XmpZ*~0SZ8e%f^QoAt(ml{p7`f(kCZ>k1nd=a?p>*PRI=Dd`yIk
z5dn59`4}_YC{=ju_3+06n!8bJ9V~MhPgIv#$kS#HBKg;L88P=vy*BSpE~#;@w4kQz
ze_w~-VjcUwN9_1;JcQT#ot1UVmdrq9J|0%zckMr4;)9@0cWRJpwOOQ*4AG5*(tIGz
zP*z_ou3asX*$U_)RVzOY|KYBjuwd!`<f*>9;X|1*(rE}`?{o{fZiRBi$9;ZA)&tl>
z#%GxtK2dWgUV+}+pH=l5RZ`U^-0!4n^*fwV@Em!i<c3dh))-D9^&k=k5TPOHrTaIA
zbG644_~*R9JC{iK<Ymdzz5FU+WLV*kvdn>!qwR+U%2a6G&R0|4pXay}A<0W@57Tl0
zDY=Ip!#e?NZZ;^ZrTt01ipjGEOJtYlS~dSq|8nDQ`CDY3ca8}BPb%|0yU)2L4hELi
zgW~Ald-Ojbvpx4(Fg!dZuXL!=^N=I6^($fw!)1`39JcHebcfK#pSunFKI%o_xvP#%
zd>7G8mU1Nd#>Wb<u)GH>k^TzjXBOKjNedWf3(In(v)>w<O|p)#BTkqAP7sq13O>gC
z^T$etzG4AeT>c6+LzLO5dGxweJtCpF<F#DJ*<c(+621H>nL~+*rNYLnrNssly;7?+
zTJpx`Nd%C#lg{|>U|Sx60VJ0D^h9v0Kw9WM-B|kTlxYxCrKXP*1*gZp4gQ|e!@%o1
z&KI_c+mATR4)M9s!P=UuJyef?NR_#gFE~HGM;wOk2x@a2dI%hsZulT|N}xTZjw7wi
zr&1g^?egqzbJ!SD<R$1#Aw^P~<Zs1=)QVHxbaHw_`v)kzgBGi3e9@On5;sSUKG^$(
zY*9T6+duF~e-O;hefQl}0e8fs<6%pafLJu17oX8&a=p;y`21Zgm-q7BGG6!C<Kbqi
z>vq&WsbZvI%g2Ynue3){QuJw)C8tGjWwios`icKI{c{{+PgFeO`o%I^wU@8J?_s<1
zj4+i-V2!K^M>;~J{5@G3*fgrq!)NgiqKwxSeVqoz-gy2xCntqD)fb^BlDh;0CwR|-
zMt^DPN1Cjoeas1>zdD>LK9^2ai>f{mwC=x-p-jh-IH2{7DcIBJ>$z}_GcVcx3KzpS
z_7C^`lWoWg;AP+K<#yo>7znH%qPZ-@<_)|UcK;sK_K%ACh163do1e~178i1lKYAHd
zrk2ygk5neI4x`lH#>BsBnc8lJC5>RsK^Uv*0?Cet0<GGmeB9XG6Cm15oM<PU!coV=
zmxc2fBI-c{QSZ)Eoy-O*bYEo+|Mgfw$Rt3<xG?73?_DwpvK---pVD$9%mfbl#lY$#
z|1?SK+~QzO_z({KbNd~l=K()dEv6!I%|>s!I)*aq!RUfdXN?+gbvW&?oj^TB)zc%k
z=o9E)Tz0&gt6`X3)P%ak@a}-+-uaF;m_UV9BAxEZ+1FWvNSH`)f|#*p<kJ8yQp!Ve
zY70Rk_E7~5?CgvG*>Pe?x$}>>wZoxcKR6J9-pYWGdd~Pm()ZsS<_%{(GUngAM`^gc
z(+aHw<$N;s`Ueg4hrgoLDeswSqjslSr&*4ZcO7r%Y*+wN11-%_3e$!uESkHeR14(F
z5N`69l-CQn46WSNGDz^0{kcHF+}c7ij3C<&<?m2Hiu%T8;^V`Uq5r|f4##Q9g(=*#
z@m+aq#g+H{RA5o2(Oor9HMhOgvjN-1uCu}4<LSDqQYcI(l@ksz#62aWA=>vPoy&#D
z#TYfzC+X`^LY&e^2Dz<w7ic6iN9cQW#)S`3s+9gRVpG^z6qk5~QEe+CBcgF>6MmoR
zT!L6if|7p_7K^zf$f-!J^2o$9q{d>*t4uDvn~xqv-sAE*o%<A$nOZ*owRfvoML^k?
zZp5T*mF#)|H#UVnEs=C)qF_nTl77Bc!wHc>eB`aA+0Q)@nM3Q-MUE;8Y<>JKnjSMN
zPpocrJetnqE`ehNeLl0Yd?rjc9~ppKe#VC2SgE_JVJ4`BkUeCg`Xi0CH~!<oo_jCS
z*UfFtnns<5MQ0<<r{Z)D6JO{VxBsrUep+|#rp;>6r~HYHa7LNhD(~?N<v6R&H*j{T
zx$?h4!b?{7M3<+Y+1vx1dp3VOvjDJXCK=XM_bA!#P_FOjt_7<X0n<PURK0XAT<0H6
z6)HKfZ#fFTyWgnulHTxWn`gw51_y4IIJa}?Bm~pFb0NOiFo>bEJYu1{cODamj*!zs
zQgOeD#`iXYafxE2Z7E7^0UXkni@ngLOJMbFQPX_cMGCb;Hr=A=FAbK-KUpc9Lzp+Q
zjKqqvmpfN49^=sE*}U@z{qq@3`l)S5PS4@eVhv-vp!}|w&+%|z&rls}aQ~2Ba~!uk
z!*2U954onko5Q>}Tjh?sudAd;EFHZSQvi^5GwIQbZ8a8V44$fcTblp^Q3ca-%=`w%
z0B!5V2`T4KA_u;sD?@O<F40=jpZ;tqvzlVZibHETp12p-0*kTN!voyd?KH^d1AYu2
zpq^H^=x^0J`FuzF<KA(KQ@)<vwFp!9t#`2Y&v(Thk|*{{W4^b3S3*wJ?<BVW;H`!}
z(1yBTQKtv3*n3ZY_=nN12UARq?H7Ju+IHxB)s?U3XlnW5wEnm;S9);o+HlrEX+s7?
z>J&X6hq{1?(5_Q=<&5x8+m#WdSKMwmN?pR34Ij;36w+B6%%?s1I<LbytT-R7I!ikT
z2CcBFNP(NrL1k@VKIl4)sV<A~F6sW))ijEaf;utXc=Ew_ZzzFsd-@`C!xg?r-0o1f
z^43WB1OCC^lw2C7p|KpnFIM7Sk!{np3f*m|5k>xtvujR`I(=Fy*Hzx@p=DivQxE!-
zkt2~zm58%D(O~M;w(h>ESD!OH;I&@8R7Kz!)E?wlp@GLVW}Dahy9maBVOqoe*}ABh
zWXt&2AsLN3y*qa~<*#V*oil39UWZ7vz?~44pGLa{Bt6#+$dmOh5)(^8*Z{tXK+M76
zuw~S;&&=3E_|~cT=C#Q+xs=c$VDaaUIMKFtqe3#$;H@k^G+Ym0L5=#KY~-U(4jF}g
z`uiYMoG<G{??x#degd;Se07q7!NeN_!MM;9sa=RIE0a(FG~ts5Zio!A_8yRU{OL~P
zZor>7wtDbC{TQ(5<`*ya_8}g51Iq8C30Gz_hd^mkLXL_ZV%S0nbF;mZvM2Pm<-2Fa
z$M24_sjHRQRhpIntL9%+Y7?M>IU=1tZT>)OQlDOFKN{Q-ED}4QTI;!iY~}I_ukqua
z7Sc51-_`j7LftnbnFM^p>G>8@=1|y!X?pv?Jj4CO3k?}0zkaXY=z#CL17P$0OFWxC
za?<#3ucg2B!=iA~<B9!|r{Y0CNcYK4%cUjm&=1drt^ryvT5l0G_)%vo?zPJe8*v>)
zrJFu2>@ueCS!EPK>#)az1ut>#BIg7@3z_hLefS{BfTuT$7W2I?(sy){RKxQs`&cv)
zbkd2Rus!?(zjLoHmANyjWG}kddiJOnw)ey2Y5x3b&Y@(g^FLP;od+io)Ev&e+rup%
zSn#7FHmCfnCp@&kT_7-ap4=Ahzob>`I-Q~!!%pa6RB>yomKlEYxiWjls2)p<IXjuY
z+*5~6bw($Tx}`r)3f%G^iFb@YbL))Q9RBKcLox1FrO+L*>J=J?ihGB?tx1J-R$%&T
zGMAhBVP&0M>C))V+3(2sM8nK)pIv8^=dMG2dlC=N#@|o*%?9*bWc>SNrC+?r6DT*D
z)1uD>Na@%<-2KuFNTE!lzgO@65T3qp^vXtDzpDEAKlEvZFmFn8ZZ7;$OcZRcpYNDK
z;woP1H>=|M7ka!3&Qy{;{mHI?m~nVVp<i0CQWrJcb$cF~a+==#=>a_E8*OMXn%I~8
zEp=Z^ZlotUCRf@HQVyHGX&+#uh(iA{x?PSaH|Pv$l?_9@>c31g2RyE7L^MhB%i)EV
zFP6PcIjyxu1iP3gxEwR={u!cp8)X{ef^_3=3-WvBkkZzZ`7)UOp2lI!yX``v{i$~3
z(n-Q?X-ljmo2`-jzhtCKjF9D@sCc6Py4c(|>EUcK8X$r`WyT;PC(=G^jammRa(l~l
zPi#1b(`P%jn(drMnVjISm@>7?tU#*J3S(z!%JpDT%<<f!{`5GE%WvWC&c2LYX($(g
zfW8kkGbe3(P!;${kfr%45gd%izlBYvEl_&yi?*_Ah~J$Pj3|AjJ$`kKF!iOrqq)Sf
z-+X(ItoAH>%!D$TR60?+{u<-PBQYoC_~Gn|`4Bklsi_{*mw${mzPWqQC#F0?GOHUh
zOFsVEb^JQM+@SiMu`1P=Pu%CP>EAcsY>N6$FI>K(k?}l;9xP@KWyFqtVk~4hj8-!e
z7(v8oxt{ZW=<Fc_f-Teb3(x1(1(Q^fM|@PpjNR5{xao$yQnDgLMf?7&2a){H8qW=T
zFqlc@_SZ7YEAM-kyg60wj7lw>9fK)7VX(4x*G6%zJcvcy=XII$ahY*v)v-5g`ROj)
zA@mwgR8DK(ce&;(iPX}mP_7qe<JD!)iElNSX)FpOT-UD$&;br@4dmNzX=N><;CuH|
zBh<L&-oiC(i*Mez^3>vReMCo)5|F6tewEPW8(E}dVxTZ&1-|OfDlHBOphQtflHNa&
z1m;IEn-V~nh)5aDW6{SD9SS|MI3TqR$K4D#(c4+TEur4Bsvz6qUa7XGFrZ;>aY=h7
zHyW6qUYWllzNaV<z>E|iKW<3blq>lo)TjB!6C5-Z`*Ii+E~!B-&P_py;Y1YxF`SVi
zWfz+dx;b3bnbwQ^9f$QlR);R#d@07*9Ro&&<QQi!9$;>>$<IbQ&@TrqiKX2^5(&6e
z6s6^P`5ew01pB^v#I>mIS@oGWRD;8oVsO<E&!$MpKagk5bcMW#)O+ahDwmkYriy=M
zjWLN}T$`E;mG{q}+l9WsC*aO9uJD#*WNIQ!AVPGeoP`T73kz}Ra;RvdDrh9%Zg!yG
zQD^4+&M*{ul6DKAeJQ-j(VOIiv+Gve3$Ga%Y+2P`OQWMWUDCo?a=I)PX7{9*Uv5`v
zLO~kt{}sKky)=br1Kryu{SNl*ti#=utsa7Xv|F{y+y1n7o*oz0?LOQek5!&z88UJZ
z@kk%#>f=Pir{S|do;Rbi(uGiI`ID_gWa;_wNGDm{;P&x56yVQ5m;}K^W~k)%t*;Q3
z^o#oaAXCE_c8uH^p)vXZkxG5cX#=8ovFv!)3q2WJRv9fW$eu$~RCo<hB4J;4Utr97
z>pZ5D-p!grV?aeq-=yUU^uVuw5wcColbn+U1F#cRbQL0=8@eAy`irFi+PA(?^q0Sk
zMGjr(UnGpGKyJv)UrI?BVb(%Mb$VMC6lbXWE+2;>>+dFvnAhkbCipwL*=oJY)U~i9
zBcvn-!-C6gGvu~rP(oTn-E<Y1O`Lfi*m&Nc<FL#vYs0B8m>Vv{6k5vyUA!v*Hb+7h
z1P&yIquZ@@zBQ^wRxXO@N5vCn`d2F6?7hg}DLK_2?(CK*)l8t>*k+;+KT_lzT|27c
zV*ih3Jz#|3#B~0rkL=gjU&C96LTry@4q6MMQ<9+XnB6Fn0Z01>(gp!gQwVkRk4wUU
zOysqdBTKUUm(*~S{R8-Ru!wzoZvwijwU{rDjI<v^>%DuIBMJGlEd}47fFfpSbHt)o
zEUA?q9Opq#sY?FD(H`$^VHxdcP>Y?i(k0u=EcdZ$;_9(Xf7FUKlR|F+7lPAnJus4z
zF1h_lTqGHD_x5wQ7sk77cY^5mmxb@_&KWR@3zF_+XyjM(26C;RyktI(9=r!t-tGQ$
zY*f0>))b5cISZZ$ahU+be5%WOFJU36%HNGJH~Kka>3^ove-$8i<o1p!dyb~rU>OCV
z^YVz?)K&DLSBz4n$?tG#5m-A<*c^DWVUwUPN2{Wn|CI?ycJ0*C7#sI-v)oxoFr5@t
znBSEpf1<eUI|rIiixpDX>{_=x?=@#T0JQ4*#{SX@h22E)wk5Bj?18UjX}r{zII>SU
z%reAyMjyXTtzcZUaeK+{JzVr^Cnz4dBj%vo5e|!%`mJ2h{&Fr*sjV%I*1Ek|yq(f2
z_WgiO`tX!&6FkX*442g{_{PXOw7^st_A`2bMSAv@MVx<g2;;}|u1y=V7^KnX3bs$<
zDo^?XTm9P1W^HmXTTre0%FyCsf6$r1&M^tU1|n;dC!TSRz$J`VS#kc(zqP6~NMM_c
zo1iq*ffjA(P5|d%?*h8204_!#1)L#Y{3`qEBB3?$S5$(h6S|ZZpufIH$<e|9Fb<|I
zt#CNiz3PUaAnyvEbuwyF%y#4bl3<P&DztVjkk)T8B=$C0f}ID((2NPPWVdPOIa7SI
zyPR+d!=!@ot1%JA2>f#Xa9AVaTNkvyIv`vqGB3*>nvPt3e<AH=^W<`GI{)-bxS_Iw
z5te31;u2;@@<_nr$R(m1*&KNytTi>XQ_9o0<olD$YkVf``G}}q{lhsA#Ul-J+o~xa
z95pL`w+ddTykppiuBWKG=gEsl&3Z(IyONzs;P*au%i;WuU~=%GJ?vAVl}QK<B}Tp3
zK{2ghzI;iEjv<;pg?yN>->0pSb1cdZ61#}|U{i9fIICL^>`6O8pp}RKb1GH(HZbT>
zx)b{Ff|a_h#VUTRyj~3|-fq14H*Tw()xf)@K?#XhU~-H~?=BXtU1w_VKA&Y$sNaHp
zbdL)k1#CSPcHPHZE6MqE|7y+(WOMZYAnnnO^iw0_hUVWQ%TM+e2R9nfMqwT@CD%!1
z&vR%JJ=>~uhx&_+#nw<i@5dO<K2Yp-R>@f4<O!86C^Fi!T~D4g9+u}K8EL{C;UMvt
zmMm3`-TnpkbU(Gy9S7na)2(7dZTjoAZ`-zy-xwF;Qzq|%Ql!nCnY}dX$`zEZZxK)?
z^{Xl0JSi-laM9u#y*_gI=V)jh&&zLl4pVsj3Pp)XO$Ra7TrM0zbH~8;uYT;Gi~SlG
zyz~3kKK~~nTjf9Yuq*}px6>oJzcANDPBJ=xEDrb%D87AQxj!Rz=zBL(9Cl&4FS9F(
z=}%@9y*<8Hi-U=tgu|`jcmK37J9aT~f#F8zQIKjBt1Y>kU#$t{mc6uYUr@kSZpES8
z#!sz{n!m296@W0_yGK7_#6<c2x)!&><ELTYtiMJxtB#p-I+|0-QP3KW3@TnP1bD>Q
z=8LbVAZAqCbmq<d7wD71AuqqE>nYl=@QINM*mf4;#7D#3#EY-T(&BlE3Cah=?+Ygh
zN~eA5+{O(h##LyiNE$X<6M<i!_S3EYN;vL6e*6Otk%D^%&0Q;~S<N|8HML+r807Nb
z&#!2neKed5#g+3HcAwYgegA!4(KtBhwT8p%Xlc%R^GkFp?yfe$4*Ifp?j*kQoT^Lk
zK!W?YY-6<?IOyL*R9ZhSPBK|c6vO3%16}PK1&*>M9^*yrk=xQ)JB@^uo7&Y#>889h
zqqUKnzA0A>_aukM{ntZ6yb3#5-L|L+YL48^^;Y9uI)Yr@h)-;zq{~<sa{Ug6j0>`V
zP&Lj9JFEjz0aOW&f)TQBE^|rjb&@VQyAqi=9$*ai-z1pyg5(%0m-fuw8G6qxVpw)x
z^21RH!j9KCOvuNNEa~SFL8euJ!xUW^dLcvrdM$UV!f{M!FMyskRtz>QYxG_#(68Cy
zGMn+A5L(9u$#zSl=mSz>%J+`Z_B)L+H#8rT(_Ob3HW3&^_-J0yc!(??(39f`)?{0J
zG6+5AP0g@4fG`yCCNjGbA6iRYNv$zBQ<gY1YPC*sWVX8eiz;Y31tPm>Ll-MrPM?;&
zvZ&PU(ZGbf6;?F`hBLBFNOp#5nF-OVkk=0{(Zb&IN~lC%(o|EwWQAf7%eO8~?lf)>
zzXD9O9g%JSa9jHAawsCPzpeO?A68d<J?C-}ADg|zeMWRjtLAJCd9vo)Nr7N(iQXdu
z?NFqT1Uf}uQ8}$_==I1KO4;hA6yHV6WvM5E91Uvi=}l5*N$k_6V0|d<%ov%L2ZDZ8
z`kM!*C%4VV_;K-ao3z$$;l@~^CM`o0G^&UfY~S8~ieThbW$30x3P5CWk*O4y{3SXP
z#F7RJLg9orSDs_7CENST$+z`%XIs@VcFm1cNWG5%c&ZQAqY#}-Q)sR^V#l6rgm;1S
zl(54yERA8EH(kaL8(w;E%I$9@($^j5_sqd*;t*9tQ-8@>`#6zw6b{J>>VE!9C@2N0
zjDk@E?0j*cGBS=M>UedhhL!@XYk)28^oYE+YAQ0jve~Tlvs2PWnZ8tf=g%2on!4TB
z4kP|^AqShna~q&Ith!UEu!iR52(swiTP;u+Gva<SflQ4Pjr1d}o|n1pJ{id~JwJVK
z7pIvC4y}f*o12o2*nA7!`qR@m+EJgtDs%maZyeGUZqT-K#tSMv)hDm5mc~hM5a)+`
zk;N$On&x<COpka|@Th(M=m>snKLn4DtQ%FI^7C1h@&mg5>Tc<$qA8X$F0-`V)&RQh
z+L$|VAaSN;Kf%S<L!ul-j{<VLt_f2Bv}_3Qrdl5l9Y||kx`QJPT{}>=R$e=;)N<_<
z`9w<R`axQeLu2K=r{V5qo!tPyJEEy}_jyH2g(1&m9duD+PVV6sCO!u6dD_Z2t#Ipq
zWolehIJ&NnteEdn#-mdD9|D4Ypo+*xl5UTAPGpIh00uyNgK9jV=}>s1R9Vb5e@w;t
z(?=SxcdF571kZ7-MtsF-PJ98!dp`?~SdmezjAe<22<fLKk1)6NKb&wL;`1b<^KVNh
z1V^4nQ>iR<>>%n1ie-D$5Mi3V%u}fXjl7uKk6jDcDz80tABazOcbCj6G&%=aM86eX
za(E1Ss59}#Znz7hl$R+3-quo;g@ao_xgQ%iTEY)MKT85|`^8ac3jrmfS`aMbK+44t
zkn*|!(ufHqC=kyFJEn?+qCd@R)GBj<Hb<?8*Yr&o(b2h5p%Sn1#{5hjs$Cx$7xcOw
z5-U?V3}0INOZEZ1&+lt-$doS%!=fZ=q78!{OQtF`bapyf&m%s!+mNSn*uY?s=r?^_
z{L)*kK18*(h^agmNvdHxCx^VEAo5?gogGryxHK1Ry0T@K(9y5XMY_~EILeT-{`<We
zI+Q?3$8DO~PZuJeO&f$)yDh0}a1`I{q}1ho9uDUy$~P*1uL1#g0TO>v7DNtY|5l!c
z0u)HIQsQNXfOXpTP7zIowh|XS<@Ft#xDQ$qr%#e)i$}{#vqQk>2>I6!i|1^!RznQj
zq$oXmZk${^K0WztnVFMru@8}8kc?ch*g(E6*=eVw<)}nl$$I~OfRt|8JzC`?D<8Vn
z%?KY~L(Ve!M&7)+KUsL|2@zkl6spwir6Pd{CaVKg6^A~$Oy+ha=f$^AAKb$S<urKW
zXlJ|5s2C`wA2}))f`P{Jzl551G!Qv68rXZGNY&=lAyJF&HLmXZdm&5Qdm1T2IeuvY
z_mDf|bLG@8FYUb)v16;eT>K)0hM?3s51F1R4~CvpZ~1`HPJ6K8X?bU5i(v1J0A^{K
zLCy6ly}Irf6wx1^l;Bw!nAf+BOfOEiR_ZXFVG)}<sOX3g0u%JpOqBi*4kRe~&Q|A{
z7iCABy$j8(KEC)$pc!mcYI4i$S-R7Wg0BHz$%QD}t`@PW<kSfId93_tIpRDx-Rsua
zDjp|oSCLYEH>;0Qs6skyc+8qkPc*0QlVd<^?9zYO*pt646eXb51xU>Wtve~7MJF{D
z!waI2he{p~XzF@gRM_{{_(f}*Bsi>PYNve9d}=2Z-n1kbSN2Wik%%()^UNm*Ki9P7
zd+6%j)%usRGITvxUT){|Lse9RqJ^n84L`rnFH@&hAP%||kKe3CB9uu6Y&}rwtG&aZ
z{Gy`c)s0r>-;=&>C$A5OT<QzhIXjwn9{2Fem?0Gi%8h4w`$?bsQA!HOt0coc&$Y2k
zD9#c=GITj8=c|51zws(nJNKyRS@Y@Hm*)^ElN$y4j?3mUipbo`-*5e)wF|Dk7R>_#
z*qtbWlk+jB^}HW7&vWAh833t|>3=+yL>#BiYnNzDTsE~}E9I4L-YiqzH6ryEhm$FC
zXOaqQ!;jp+UC>iLa=U$*J9Dipr)OP5)Ej?A3qc$Z2qFY<9Qaq;B_ws7`1CD+T@iii
zH@vSrUcy%VNqF=_S{Ln7`iqs^D(i6VG};|<-npDchG}x+ZG^64M0??UvhD3$xrYhj
zej)ww1;<Cg7zTHTz9fLgF+!6?uH~-$evH03EQ@#Kneq7wwdKs3M4?e!^57^&7a?I(
zRDVayB(8<ZF@)zL!z|TBg{$r@nNcb3tbtT3Uwbn1aen;#soQ?3CXX-Hw%LZmN`_-a
zE%&eUliG+FmIE#YTw4i*9@X{Yd_e<+@FTT#)cZB^J2n~3T)e^ZPU6Oytzi;$zr$}O
zY<+)?{NI8@SpREsq<0BJ@-+Rtcj14Z^M;FnKSWKH`4(f+G^ep%D!@Rz>=DxNGekuk
zU{^^7Bhm^7q#iM<=;prrP#ud40}tB*H5>R1SU!U$gNRKcEr4BxNSQ|c=!$nm|HPP8
z5xtY<tQ;af9UM)Q+d50`x0!(|!VJzRFlFr@)9tpa#^WgexA-ZyQYG&VqMU?(@a>y`
zd>X|bE^2J+$0A0C?y2s@GsmbvS_WFFwY@~o#`7`euhuLP{x?#V{t+jPW`$|rdQb?v
zN8;WqAfK^Dv*T8me>ff+7Y^Ycf0QRx&3ii3x-31gJ}0=PKZgIhTGlmrmf!m2#(j(c
zP+<aX(7dr#W@eW3?LwNN^~l0Hr`J5o1)=5U*K3QPy<U*}hnumXDknTv(sG*9#T#B?
zeyTM+QRtuaU%BZ)T>xJ2O{v~*4%ZF0lTmliX&M>pJTy-Y7qB`kH7DXH<45jz7SF}r
zk<O>PeTZtj=ZJ_+%LtDXZug1rdHW^yg6HHs;6nKIn05B~D1|^OdOex`|A2%6brpc^
z0{Ci=I)BVi0WpL(wLD$!@6hz6TZKQ+kQd9PQ`)KYz0=otoXM}^g|^>W^|02|#r#a*
zvWL7dk%Yr8%i($25x&*u#k9Pt97@DYu{^T5t0Nz8D=x;J-Fxbi@nOE`I$JqgT;#gE
zUU<6--N|kgjpGB)ncg5NmU4b;M{Z}6+jehD((iXvFfR<X+4nB@398IOuJ*oVv*{eN
zG1=85t?H^SYo=h~%~i->V7dw)l}n5?WhxH86Gcl@jpVDh@2I!mj{|3tFh3GD^OUdY
z=ubhJD}aqT4O4nbR98_p0#6UotZBY;ja{X$+(BOf<T%pBBIfWBr5$4RiswMWKG(nE
z%l|$UUNR3Bi2wKocnH7oFMsG-0tJbA3O0?;LR7$~;a()T@!;=yYygMI1u*X52B^!h
z20z0$Ko|2_kU`(TnA*;<`=2=2PC&=nTx`bZV=`OWlu*lLKq};PE@?Wo*jK}NyU7du
zUI@oah7~^DXSa6R`BY(v_p&BZWh$FQane_;;@0Zm=u$azlv)1jYmJoG#+#lETW{(|
ziu&kEz0CY?y$}QG)sAJ?ItBc)MG|abfRn930pS9Pj!|FpTCt}}bu(3TU*!IboE)#|
zeo*1K1<@Hy9FB2V4iS{-i~fSoEVTAVkl<BG-M{P-u3&;fi98v;LCLSh;3j+BQ|D*x
z(yVnhOUWlPHvqGxf0ES4azw_|IZMQ7<@_oy<}HLaYekBUSq$NxEN+QMse>L3SHb6e
zpXVr{)@zPso>Cks^`Pvvy6b4U&)aj!%p*3xV)2#^DZ$*oyopf2zRV0rxh&t~SL0Dj
z3%LmPD~yf8j00E5dz}C2qtQQl+R0syk?;$lWC0q<oq%(dImB1S*MQQsp>T1!ZG?t5
zh5b=9-Zo5IB>BSOO&l5z?%nDp&}w@wnD6#4tKv8vi5e`V<;#y6S^f62XPQ)_z8+Lr
zf5YuZC*bB;h2J}y-L``m<-OH3cO-t<>gXH%`6*aT1A33j7~H$1{^Za~HH}H;Dup4-
z-Q#|S-7>HngM6p8OCE&|X}uHm%TItgOnVECMmwceAlP(`&#w!^%l%$z&l%Cfre$2-
z@h7V@W1q0MsC;9AoY{g^>zkLOy+V2oWsmdzPjK%5D_5lpi0V>qKXw#*+(Z2@dGarT
zVs$|_)+#r@q68nINoIIgA6)Wz|3=w=Mf2BfaKHIz4@BybNO_sZq`iVM(&a3}vSRjO
zE?(cQ@ziH?xpiyq+{<lg7&N<a5C&{(e^U<DSvh#*>!e%d>&#iro##2>!pEcp>(Ad`
z_mLx`O^a(se0>pr1!KK)J)tB%Y2D!F@rKtjbZ^q<6knw^S80x}U@KH&c4{l^J5X^b
z^*I!Z(H4v>)ug9ukI}bj)rE#&k%Nk!{+r)x5zh-tyUqu#JAU++Z{d=yOKs{LmAg8)
z7xu?M&LOxvlU&Z?n-MG)VDIb%5ftCQ2AQE$bjg9tl<05F&*3pKxLwl|sEVYR$QY7B
zVXt1SW#l@;g<+sA<LPnVpNMsngm!g9C)x9rbj?klg~h{Mg`bSr({WZ`ub+}~g6$XG
z4pD`8Lnqc`*n+e#;bZ1TY1DW1Rz+#^F)mbPJ9EYOWnqn)$`p?btv8;JBR}7*RAS=_
z-S5y=R)>GDKMO*Evh%@`GkirrSUYxOFs!tjq42M{FWp@K^Mbx<-xIDK_sD8D!ifWo
zpuA=OUd8x(LmskwVoGlkCQu{Ua=ZSStN`s00@UCiEq?VJLXjBv@DnbM<KOR|f)fv(
z3*Fwm_A!*dQufWG{dQm7^{^=AqSSE*$qOMX)hAAU&h{ID1`IrFy=Co%qsdOVBGQR;
zZN0={q{I4{m@vTEmOuJ6f@@1f3b7Rc1Z%yq4)6QSFk$0RVcI=tlo^;`+8$q#;VU*%
z?PpQx!Dpy+gv+gIrJ6b`*{Q<>(ZhdE)JAnQ^BGn;OpQK~CuMm%f)iZWkDcACNqZY!
z6)vp@jtc{N;jL!uB<zGN0x_=b0KSw)&&kC8zM<b4K~fZ}p0_{$h(@I}Dy-*5!S<JT
zKm6hDPi=LNwwr=!oFUtg(hz|t8gH_CF`toM)|%l)K?>iW+(#w$kMs`rU(uMhx7Ay=
zxLL!EsK9@s^xcgDS8P0xc0}?E`<9SvyKeh&>`O<6FRo(68~f@kthG2;h2jyaVVX}k
zPh!7x$=(eJrb993!OP#{v#$w^h6_I^<30Ky(DRRY*Cj(Di#Ch;URpe74Gv{0;Ej#K
zgX|r>I7_=F)4f4LCh=Fi{d`8cQAxN?yvsLGdr>9yL2gB(1Jo((QLkv#7eJ&wnZ0^<
zEnJG2T*xK_{>lIQOh<*P<m%<;fl?jHs&dUZK~FHNpCrz7$xQX%W@eS=f8NT71V@Vr
z&ul>OIm@VzXfFO!@GbIdurUi`<I?+UtNn933E<q6`a7AMz+u4;UKkys*MY8Q9k9}Z
z)^WYL4)Jp=dbrMTq>FG2^xR&Id@#CM#aJ=_r~0LtRv80BB*MYy1i+f3i6ij_u_T7?
zeIU1?F=7XX1qhoq%yD~71Xidg9f~iFkS6R2=BOv*Vj0D5^>dD(Iif@Oc$JAT@|orH
zQ#;P+jxQ_|vE*N~1?I(b$a=gjvb8Wus!`)gWJ4L3`yqR8gKua6bdO_hzN1n3I(fqc
z!Bu<kZ0I$o`-AIGbrhG`TL4>Zeo4(|*e=_Y$=!_(QnAD2QJE1*%(K6zc6apV)nxb1
z0fOOPxlEhG(sx~YtuD2)g(*U%DZR2ZrTv2RG$k44d(QH6?y;Ja#X&;R?ct(b+xWdn
zV^PaLcBFP0&OJFjcToT3l8XQ564UxyYSFftcTfJkcvW8fJ%u@6b-67;_1((cS6J6a
zVgV-XWvY2b1l&I0c@{6k+>dQ^nj)*ktBaK*z_Ey&Y$i)gF$l>|{wfewe%c|5^0UKj
z+fh&P;4E8U{qf4g1nAC=W(=)8qDada^@^WUc=3@sjBU^H3%_zn)Z_e%O)as4;UZet
z(CjGG=()uVF;3rk`ceG-pG(GUEaL?Jlh_wfAJgad#|(~$o0@NflQA-F4_Rur6T_56
z(XrpOjmsStk7AP-r@K{g1~-oLbQO=cL1FNm?6!<xq%%8Xq%+^`g-6G|5`gV50=uA%
z85F=~MD@`B-#g(yYW=^TLW2Ku*bt#BlWtpYRW&KGU;>D-Ca!OBtcpe|@!_doI166+
zwYeWma>_Yy+D2l;Tf`o~dZNdMM$bH^mf3%MJC<qr?63B(8OG5_u>xvfrpE8V?;YF~
z&*QGm=COvKrFKm23{%wh<+<%bSmRQBI{lMLfsQpzn{wwa$1A~GtQvP>bTYs^mBRCi
z4$OM-ZKT2Ha(x?ejIQpebM9+!HVe$l?kFi~PL2PSI21-dVMm}*QW}__6-_}s`7sVV
zU5k{H$KBcUS!;1Q>^Wf3llQmFR6V!i9P2|6Kl*)4hK)Q^H+3yREn4WNy04ehvyMFF
zDcPif`XScoih3G^OVgBR{hkpHT`DpR+8mAmY^zi4AN=^6cQQsErEoyX+a&*m!W`VB
z+)BaNKGACYU^a8_HtMWh>aS+KKfh!Wgt+p?0yM@h=jJC`l7IWF^|HEyjn0`uE)3+6
zD4LK5gViHe0*_5>Q1Cu8=m{vASCpB%+KTxrW{X+&D#ewQ`a1EJC)a$jO6#4i9Ii+|
zV4F7Ui?^oEcpI|Pe939#1<SZeW8PN}Je$p@u^a6%vYS<NNO0ShTwTwl|Fjaf$m$NC
z{$=o_(XM&{LwSa!Ih{ep?^px=_CRJUpp4y_hAtq9+7I|db$HfDe!6}cieybca-Eg)
z6wnqKCqIrxKGcJOn;`!*C7cHOZl9&|%Pn4dMgNvwyZrPK9b5ac^lqN+n<EYVQ-Sk9
zab(WR4ubI5vTAI{VM$c1&PLR=alEr>3_75=qxs@{w97Q_Qe3?(9_2%Ejf+A$Tg-U7
zH-X+BdivJ%!LgA7sw)7q_LX?t*<>EJrY^wY&Klg)SC`=Gcq&0}`WQdoEzNIu+)2*1
z(ow(2pEUgTs6(?OIzaIKeM9Btf9vd8$0tc5kk)=lWBEV2;s5>l;U&0iv+N7G?ZfRq
zQlD;GNhI3lgQJq&oklU&FOP0lXXH<Pb8vU>Yu$QWR%yn{!4;;<REoI9TYK?B!+c58
zn?bc7PptF}g)KME$6K%kW!SUT$fje9;}h6f{*Yq<x#Dw;4yo5yxCmvgdxLbJddL5o
z7`(3b*B%xt{IUBzWd1yp8{Z*BhAEIO$=<+!=8DdA`LGe}YX#ahxQcH3W;|yT$;|j(
z^YdX>USB-A;(LdfO(9OU7mhu4iCC2JJK7df&HC{D$!GgOO4?`tJ6=)T2gj>_>~fjS
z%Nr#7w?TR3<rkWYfBjrEQj#V!s$>R`fETubmjFe+I?NrKaqpi@A;C0?$7R`#G)E{r
zo2s(!UQLij+vU7_9Hc)9T^YDVUYy_44rOW2oluk5<|Y;jpOy4hMJX10qI;J&*Y-7*
z_M`J+>URAK{V#?75vv)+>^DiG;k1GUnz~`F_sZ`Kg^yj%uy5B4V*le^q0h);<)9P&
zf=K`nG)iz{F~0Bt`_;#8k{z^0s5<6fG9@Nxon91B6&YLp!`u!-wKv4U4=+T=pzXmv
z{30pB#IOTmyLk%2My2E12JwIO!yw)mpCIP(2SlXV;~QxHKV?kl%f;ndTabO}xo`JU
zw;dtT`yQN~0!+Ot$E}j`U)r1#zRB)&Y%FnEI>8T|o2b<cT>h5-UdF#3_HXGPRCJuS
zss30cy+T{6#=irZN?<6c&c6Mr!v0^^@l|ZBpO82ZA<=W&xsi+taIw@`&dq06xx@-7
zqKnO>)aEG)bx^l#YsgfGazaY}1X5|z!6g_hamlCr!g+!7*$W!1RP$E)714L&;r>~?
zE4ja2OXp$(ZYUf*HHOX1M2(u?m}`4>TCp`nI}mv_Z`97F{L>w37IB@CZ?xhrmow7%
zRE<l*c~z(C84*yZiR?(mVc<fI_^ZITbmjB59m=?VPtZo^a&(kHhw{_8@Y<!TYU|_+
z?HB&H!6M$^bwVoRjp!qs^(CM&lDiO$;M_B&N$yBz*Z)LE;0KIn5f?uI(ljp!xc}FL
z{4c8V-|vT_;4J|%p4YwX*gOA`NPJv22!JnAf})X-p<)~S|B&|HK}~kw7H|U6q$&bZ
zLr10e9zdFO6crGVDm`?N5}F98l+dJC>C%g#NfT+(I|1pvNiTr_-^1&@-|xP6?mypr
z!;B{5kTK^u=j^rD-g~W)FPBPBuk&#ZcSh4u9`YZ4u1cQ@MwUH_caTDxT%3<=v?mh)
z$?zLUV7j!=lpG%Yy)yHI<!+DFB&&|g|LXL5AcTe0#R#+s2P>6SFRPNt9#(`!4Ubk#
zXi?Ifplx)5$TI_EGDCQ`LHF@)OWlS&5m=Y6#6!wT*~@mFvr?bU-;K(iH*P8j)*u_G
zaIwz{0`wxwxr>7^cHuN$-318W+gIO|Y%%iTALs2#s+rK)5n9J17#1DAhfz43Irz4f
zqg+o~sUQ^G@111|Zok9&`NMka2cPE9JKvO;TqUKe*pnG;g|2nnxMI(<IAU9)`}c$)
zeP9OGW<TzWn0AC#J(zle2HZZ69rJzH_)t#SBO11No7(yhKD9NzGCIbunh2D9&=}I4
z1Z4OE%vq3sss=Zega7<!+X6<fLjZvM#yR4&dXq@C<AVCW`1jEY+gSQ)(VD3>EJeO5
zuFB$WQfSx8FjsyJ%XVyn50U=Z<@|kNOr@7kf6C?Es7v&_&9Zi>3Q+M7I>JwdA9Rlw
z>f}ElviF2I|GaZE`6hBy{{A@hrasGz1w8z)%+HCc08w-Db|Go?*J*R8%KRNZ@Vy1<
z`?wHUh&t$}80@$e57cyrK$nreqk`0wP}(#6q7sguU&d^RB`kFW?7m4E?<QZALOQZz
zsJ#r+gD`J*34fYMTe+@nLCD~}`1In$NHm>FBFb$v9${TBLMm257(>sVgJUx^IiWy3
z!hBELwga5N*H6rtNdOMC&=vjhEHa-q!g)^SBwAS)J`w(|FA<{_l8hCF^MmJVp%B>b
zq`<s^o!?nB@3#CXO+G?;e_3bgsDfZY-mg$GS=28ox7q_BY=B-Lvd>l}5N_Z`kSa5t
zD#i|A=YAAO^*`4Azn_4%>VK_&Zqac0Y7DUcICwYfKcj2t%T4Px9s8N`{EC)$(w&1&
z`y<frbt1M(w>4~@@g?CtNJuf1@4lV_bNQrM7%E2XrF}LAQ`OvD`+-Y<>xPjq$x>hh
zSyQSeZjGtjEVX-B+Mi!qr2*JyLljP6tjGXFrtIDW6N)H2@|z{EkI4?<c&uc;_eR<8
zKKUxBp*p&?$x+v1Z-BNCp&}dk&Uc%i{T+hGoJLvciaOz%^$J@Y)gEFId_9;*`)LKZ
z#T6iuN5WQ~whfj$Pe^R@)5e~Wl`EQ>-HSdO<-)qFB}ajv@u>{_&T|WXOhL5G$0K>f
zgw9ve68U+0ETq&|%adcL<l9F9^%EvVMfuyJT>A8MucABso*|q+Bs)83_QleqtOKsT
zot2Rjzvvf)nCTXlnV)v#{IkyCyk=&bz;%uPWIY7fW&isp&JYO9LLh6^KR{e2_$Mv{
zWA5w5E)dza)lp!`4+(^0yHg@@-yvhTSumaHm<5`G{CD!(=4+4m%M)6ek9fELM-(Us
zyCHjw6!sC{o&wxA!boAYf!PpSSP`=L9y<W9HIzd=aQS|0sqMn;`>_nLC*GC8B%avk
z)zk^3XM5{v2$AK}+#;Im%D^T#+M`@9!?{oQJX~&%+KM5-DoS@qY9l*;F}Z>WcEq$%
zn6Waxct_q{uCnc?{(~GvZFxV?&ymOXC(JB@GC`DqLhN3)9l(X5!%lskj19Bc*ustG
zKG%LmnSoWsx7yRw+`=^(5K1RuTjWuEAC!H@r}Pv;jUKA6rrh6eyYH>m5aUd3t@NHq
zi0AGb^}h)IjF~-;0VMP>;Jw4&l;z)lb}P#MQ%70m(FjV?25Kndz%@<08WS4)@0E|U
zzXfDT7Q7m*<ToNd#-jTOBY<M(!<(m>T*G5!rUOZtf(n=fzVVf+^X&zH%#7gvG+;%y
z9dK67WSq{w{j2$?kZm`4U(BUfS0rv--q-fFz+?xmZ&yuc;%cz}>LWM~9N%lhu4HsF
zB<hf#<p4OYcBOmeY|}?}>Q`8{#h3;xpsnk<paS4Jies%#t{f7q`dZ?4xC8UX!}5B3
zmFwFy-j!_A!&zlWELGw{OTWp%TKje8EK-CsDz?~M5q`wfjh47!db_{I`$t@e)T+?b
z4G+J`q^1@atbG#4|2Owu3QJ7%21ehp?$-Y|^ZGykW}^c8QUN@FOoI@y<zqnN<&_<1
zP$*H2fs*E57lL->O=1|+D1@U`3^c4<)P2-?8%!iL4|)#DD__$BAA&NF0fKvs`(ydI
zk^#j}d{KeCUQ;xk@lUQ=g`SY6V2x+84AT`Z=nSgeh4)QNP-zK!pWQ<~t~^*;;+pR+
zvqfd`4&V1}F4=!I))e%L`poEI9pfYy&BZ-R1u)SUlUhrK#Sv5WzyysB--DI*1TNnV
z7LMz^Iz|h(Xn+5m-Y}>DR(Zk6#6k$maH4$|hws$rdUq*sWZzVq8rF|?Wrg}P6)d+I
zhaTOx+g4Re2(+~5l3n`$bwEc<c)awi|JUwQ0xmVxli_<(6+n)$y~KX?k@L_HoQh>L
z03iY=g3j-Kfa-x4?qixP3Krx|aIJM(h_x?H%2_b)WCbHB%<tdVicR3%u!EkNbD=s#
zCUN_EgjpVI>fu=c@N=7WWnn>H!MJiyD1-slr|gnURjC-8N@4fJBQb|3hBEi#g?SgJ
z_TEMn-@c7q)?_U3{o!*WQmDsxQMX%xlMc3W1Mg>tttLB1*|c>=uCh&IAzKpsC&E8K
zMW|zW_m?yH?M+hn(aH$sm~bG!X<8oSz}6td%d7*Yg74KI6d(W7(zAA~mIClQfU3{l
zzfsYDVWWqCu~Dwu3z~7m8*DTWU?aT=>0*)pm-4xxelqW|0Oh`1&lk2L`M_cwIag&Y
z+*TH=SR)Oj&cw%N>;;)))mf*~NP#M=TM3`wN=NkInn#u2+GDbv%h)zIsPAf3);*w+
zPY@Bq0h_L~kRqlsXg!!L;64X<5Hq6bA95c^wc2agjECiwL@Kh-YN3851RU*mV}Xl5
z7s<|Z0nF>OdU5Mfff^Q7(xC@5ItC(9*(LJ@C&$YW289gdJ0egBZGhK@&f41t2W3b&
zbGA5t2r+(>#aJlEml83am8mBGqx<>R(p@}Tzy4F71GwlneoT!UfQ$`Q_~8Eu&%6H%
zo-eQ1kj4H2;JE}4jO7gfly}{fF#q!t6M3WEaJ2H!+6ACvn6UeZbj5xU@ZtGj=tgoH
z{X9)T@3G0mf}wiYPlTm9T5q>xhgcVBar=Z5$iCx(mE#;NBuA$&vt`$zMWTbt1L-7y
z?$Z6K!F`p3%Cw*U)_&`ZEE%?fRQpkKuci9Pp#n!q30mDvGB05l*I*Fn`r4@7{mp+H
zK5VDq)!q}H=*<4`y!G*&r%n`vBHO=;^gx7@V8TzI%yw5Oz^(5ZMYVsvesEH%x2IUa
z{DFduj7)oDxWUiZPyfob%Jr?|G|s7N|6#9&i;Jp@rN?gnvJA&o;<ioN3-$-sNM#%!
z%-q-W`l3xF-{^*N<Pxt>%i-}*Hn%UTviJ!wamBeKH#5;`spykAz;xZ!Rl3>rE51Z&
zDVF4X3ngzJf*_GX=kr|eMYYQ)r1@<#vNXie9Sq!ORCRkGd{x`Do!#6defOs0{u;30
zLD^;=&;)9Tpnekj0<G*T?zc^USK!kD?vIgz7F|(#TGs!L7ylizmz8h1@zy&n{J`AT
zzTf_eDo}xWVO7d*oU6*D_1!mzE7KqU^Lo8`jV2GgzG%J{q>D2CRCci2d@Uj|KUS<S
z=+9mekFc11r<k{jJ9f4IV6?<w0?-xOz9$*geWGVZ$_F+yEI!ns(q!~%W%K&M%T;sv
zgJ|+Aiz#ysqImmhXP$L%3BFH2_?(u(RmIAMm3o(|v2Xl}TMJia8`Cv%6QALhvAKKr
z3n{oNo_<F@e_MVjza2b&<;mp_^#bFiY-96-txe-Md2To%E^?Lfh2W=m?TKFnTA6j_
z5=Nd?nzpABt8V{<k#T9>f<IZxj3h>9=8%e`m37_E%SktGwH%W6b=H9Ol&;=DXe*Uv
zpnC|0&{Za2`6p6wKrxQZ5XofnIgo)qsILkvYtHgCQib#ebJAt}oE!u&CuP+Cc{aKl
z@Yi1hSm@x-Xt6N1D(hW`QUD8$rylSP{pVcVy!w7KFH96wue$%@wv;>pFUZ=k;9yZJ
z9c-<MluI%roxL~z?yg5bJs&u39=`=k8_!jiAjBj|`e)6HP~N?jp`dcYEW#{#NM+P!
zB~Avm^jtbjNo0>SN?9UC@wFIU#B^mUH!wv^7X<H;fqOc~NV|b|wHGrat^iob<E45g
zu_r%s>X`C|h3JhM(JUkfHxjnLzc3+es-YAsA6XB1&i<90l{Q8>5}L+}j*H|S_!vZ*
z6;xct0rC3?g>4jaUn4mWb}yEHGGnVy8g~u8q+a8PoD1<{hE@Yc5yT)N10kE?oa>{F
zkqyG<nk<>2H+jFNmPxWHkT4}@-g^1($n*EzYem^ppqoEdH~&oujXHC*(;*Pszin2B
zcK>@fFTc5UE6UVq^u?z#fE6{gJ~+5pxT$GIjS4(!YrWmIFJIo@5;1Lrq@QodYvPGv
zSug_oN_H`u!(g3JO{Mt>)OrN3JN{g#)WR_^6~m}ZfBjYGF;>0FGz8+?<z$la%skF$
zd7=HnS9~si?`}-ZSIvaGG1q7Cn8w_*qVCU+bE68_%UafRx{zmnCPL1i)fHktNk%ci
zak@#>*{^RiL!WnsY@4c?w=oFNAWD=tK1;l3kf1x@or@zA!Nl@F_SmpV4=&2_)|HmB
zWE=p8bg?Kr*%3zj1&eI${=XY7HRFln8CZ4+hvoVImBJqgVAbCspYKxAcI}yWKfX_~
z0u2^Z<V&gx{3}c`ma#Af5Cs9*>UZc1YP<w1P_oXpEC<TGP+D(d4TLujStJkybp-f*
zfR=+F0mGblS|avicLbtTQ7sI}g|M})PrTdj_!wSF9Rw3y1~?3k=`FcWQ*3l4xIJ82
zMqsD(s+ll$+m;@XLKs-DzpA~=?U;YBcA>1PTC~<<$G8aI8_W^*#Q6Zl(F?e94g@HW
znGQ$GOCy<jLppnBR-bhyH3QdZqn=OiIK<DFjSbj_La)wG$j)#K&VtrM2is*_4jhZ3
z{nHGW8PTIs4>m^HIj+0R%7(u}QPKYAOcC_r%SF@XVW7;4rw`V0{@G>yfaEv^n9P57
z=j;Ez%x>O$QG;i0iu142)4!Q%e^mI<CkV10|1?6+PW>;;7&UQ&8NWlN*=P)b-PI*4
z*v()<*uq|pBqf-TO~Ez<3Cv@?XCm{Elc<i4jXb&~7EOFMDaT+yH-G1SL`w~(C)6$2
zG!yW~nf}VMZDtIQ2-Q4K#P*%c`YCz<e5rm>?@GCcM%{%FuOn*G94{N911%Zf-a7{?
zF{uKW&m#f_N{=cRv_25X$0dVcn^EUOpP}UAIjq^xvs<qM6<%He-9Bzn*+($diOOt0
ziyW=y%`Vx>?(T7dr$&{Js9AFOISdM2A1htP(GANp7;NgMJcAaxg83=I*0|~^BE)`~
zE;L~28y>^O8*79}5@bDmi>>_sf<y<8{%fyEaNCWU-0U?+V6VN%msbBbBx2;ffka+j
zuT-T9R6jpTqOMv^Or|Xl?2%CeietiUA;gmHk!F?Blv6?9{1@o-mO>x2;+)M>S}^hv
z(vj8wcu=4~0uL+;`+goQOr|AL{2?|gXW&$ezM<<^O-WvL1!=bojKpmyIvryWzjX>@
z&fe$4vu_=NTd0YQBAKH}mly1$VvaA1B3~D(;Hs6ZKQsxF>x$o8EGl7a(IsQeR{1gU
z{vI4h$zyPcNk5HldgW+n|9DhM^UD2BCX2r6ol1~y6xCB818-N3uV}MU|5_A5A{9a2
zaKM`qPl3wgxxPxYz@u*hcgc<b_cgvQn(d$EJMbY{Vj%@3dn!DZ{P)@W`)-ao(+8k|
zx@!chPQp2&pVW2(G|-f5f1CQh5B7mDz~01`S>;(Q{gMZoS*`P?SV(R(5x@DDgXn&c
zPkwEq4iE+@&z}<L1`xq5#L9EGIY+y11xat5=f07=(M0q+ztKcIE##55C~)Ny!RNYV
z9!=zz6Dswd%Y{s8E^>1*JB+xitOz1=#3|C`Z}Ch0em@R6=oD}3u+XktT{bq_pS?7|
zGzi;kztCJB{McCb-p0fAu0n^Rr+bC+UFOPu^W9CJGz}QWZ1FBA$^?)^jG(wMWIO^S
z5$VM{07*o7h&zeJPh}FXCx<zRl2l|RD0vM~*&B=laO_PnwRw#XdH^V7D6>}o``iEB
zcn92nw;Ehh&~Cu<W~<o)TTOaRs#oaWOKW_y)zIFSsv@OQw+2(~m7ypGtikeG-Y-0i
z#O~vlI8tB`Hjhc2bPG-Zzc%Cyd>+h12BH1I{6RS-2zpeY^LVdC)I|BpW&{Df+Gn|r
zdC`o?ChVp<sKh3X@!wo`9Snt_Cd9aUHkOPp7n$0?bEmvMUAc#(&7<#!>_5t#%jwEK
z*kIwi9Z({nuezav>$Mz}P+B4({;tI2qcTCaM}1wFNWFv{O~%nOxdBaV$2$!Yw{NGE
zuw};0N^FFP!bf6sZqClBnwG#(cj+!Lxw?fFtUJa)oGfG`U;@;EkTeC~m1O_H9N*Jh
ze=Gq~Z$J16QvZ!2{=UO}3itywU$Qs%_e=>T1Bfn<79?ahqEG5d{6BaV{5xF(H_VD(
zdO}R(BT}#s{q`gf2co50Kp@NqbBbGc_b;OE1X2Y&SSb3LZjXkaZ|D$|G8!34%{6<*
zMQyVhs7M~EkTjdn?}3LxTKLWc_Up^Zx;pkDdkRhTo|A!l@)?+(vh&=05s6%O_6%;D
zsL3o6{s$z-54*4;o^<Ty^U$a9Ei<z3Vr$GZX_-{0$N&ZjYA|~hU0KQ=7uJmRE|Od~
zxB<v|FNj1z#lU;796_RxAuD>IMxdgEv>pj1$=#h3nfYCwE8be<fH()s%0Kuex{-A=
zr?Gpuj)2~Xz`pnXzkmf)=`XNASXu4d*8zY9c^?Nz!)#4jBSK@d-Z=t-uG1IV9r(6n
z@m_zG6$?-%FSQx0{iy$T=C)<`547b}PK9^{ZTJm}j%&s*HctrHbbK~fY}ZT`U#e$o
z*_-n8iak1?rSxS<r3$NvYjz6%l2+Rtpi6-2CWaU)>}OO(hk|I3PRVA7+!L}c1(dno
zE*?2qr#K)6had{wBO1;kjt9X5&mX=?@Iz_S-zdT8&o#p&e3$2YL_As_%Z{#)8tjdr
z_I$r;KBPB@0CZrO{F}b7vR+?d*5&aSB;ul`juMjiQn^j#u`RSmsSB_Qz@GpW7=b3J
z+@8DrNE+zf?T1g+yW)?AvW)|W#nwaDV@b?Ee?Pam;869L&%%2eI{0zRoE$>uY|wQ9
zM|U##R&r#Lz%cR$qzJ!)Wa-Um?o`|m?o>mmUptV$al_{}A+Z#oB^?ZSl>WbsoBw%r
z!1ebqkCL}xow0x0s>N~grtHnIi|1O+ZT(`C=048<ZOg$@WU4m4ut|<sCl{#B+Hc&g
zGMP0LJRLw;PVMS^5-F!VVZTctx&_A?bc5^_7V2{nWP1Hs?dg3hKl_?+Kt0h#pJmW9
zBd9k?RC;risy_|pAm%q$nS9&(CfKh?VNH)tcesg{7NxWo<5cyI4S4XZ3I57hc)K&z
zT5VsKPdi9ujIpSHtL3-5I)LBy<d@wIv-Ul`Zn^f5Oty_u^Y<+mL4u9|VL#P9Fqs&Z
z6DYuz@qA;6j2+D2GMqrXzY%0jNEgHf_l(*m5M;WJYx3XRJiIgw3-Uw1u+i5Hg$A16
zJYozXi>ZDzy(jcaeyQWH@Wv<}AiM#$X4vF)?{0CGn>2Mwevid`ab$C2u8^G(1SM%<
zO0F%DIDpywSbRuU?R$e#yShs;ZLthZ3)s)I{vi59=>H0DOx*}?)UJn)j(h$U-q`;~
zc*DHN%!`09(~Ady$?*TL&u#f*p?20g6``qF`<kWFJhS%W>?+~Si#wLv>$}+=-=Agu
z@V4w^_esY6sASoAeAKUF_zNoh4fQ~Y<f6AXS-iw)?bR<8AO%_lpg#s@u)+$WQ_`Wd
zy4(CRxk)#};p$c4T5tOn?nhy&Kzopoq^M*s-QbD3=;W@9Uni08@y0XK@0L3n8liT7
z-g*n>YYOrH9?KmSdH49{ggY=t0V)a3mmPU;V&#9!HZjQb!o^Q|eiLmPo3-N&!mrSX
zJY__us#KL`TBMLhvi;8T#gW%cIE`@O&w9D?b^PC+OZs`1eYTb>znaz8`birqts^j7
z$3U~U)r+ss1$yGSMMdW@;VFo7K%LPs8~b%%vhbm!fZf7}12Wl>*7OjS6FqBECoIq#
zy+bO^z-MQmW|H*Q1Z6yN2T;L?*26EvOl@)*=Nmzx(nqcLyP8Ytp})km4k{I!F~9J<
zCr4`MZ?dxzCG#H)=SLrx$swam`>a|S9bA{S{t;>-LYB)waPFqCd6z5&^B+v+%C!*I
z?_gxOR39c4f}MjCodUZ<fN*mzHh9N6*!kx3K_k0c!&<bDU!P0J?NtyCAz(bWP*GZ{
z=R@)#BB33yBF(>w1^d8MX*%-kicRfdz@18qOdAuh&?Qzthte0iScaQMSaImbiYy>a
z<6_MEHzfXBS&4dkXRvX7iX>CN%uI2_dYZmZB;qZ9fc4^Yx_~^@G27MhY|RfvZ|q4g
z*Np@v1AvVO#Boj_J}s@>b^m?x$6Cj<ahc`k*7S~fO?)!>aj@!@0wXP`@9CO*6)0Ko
zw*~v9(2t|0?VM*?#eUiJy>CdH{~X10>)Y2uW^(@h{Kp(hG#c`De#<Ip(%E=7UYI!G
z9`jbj5C^PQbe1BgWyV~iJ&%S&`&t5nuGyZV*Yfvgecl@Ic^%ZaO(SRee%G1c{-Ppa
zY%4H^SLEY#V>0xc;KAa%-y3?rdl5#Q<D_Q+@E9ZB>4kqoUs?EYRfO){o(BQw;SoBk
zxtGBN+=4HQ^;=uHbC=v&5guqB^_dis&nRx+BXikwy@HG&jfM{@MUvJ!k#6c3oUFu(
zxPOR29#!Out7f3>nt+MvDIgPw91Vq<=Vjd0_pGjBCT6B(*$3NXPUiH>Qh)&8GzwOY
z<h%Mf+T>KV??2iWe^q0kKHm#ufZ-_larNBsJs_3$j-#O!m~3q#wag;6;i(4D%u!*9
zxLO7j-X+3zi}V@9ocxK>)kY@+%($r@HsLs&)7`Q$l0SXJA^pt&4!m&*5kLlTyOY)F
zKlk^4UL9}(H@VR?Sg14Asor!tPyIO!h}b|{<G##PZ<^pV?DDzu<#FGt0juFlT9k2}
zOR(2&A64(C6{!rvhKB&+1p&BYe?ks4cn^{dkDAPxrOX>k(yh-@r#X&5AB|b5N)-dx
z(7E89$p_iG4=qY#A_Aei<G&)@e;g647CEyGFp~o%xr{|Xl?x^0*wjD{$UqGAbww|I
zdb#I~9ne9j6j=md=cNE#*W5^11}uOJP*JbweCG47nN;8E;8+k<0G;pe>L2+UMz#0`
z?l5>uK13Q7k`LW~pP@bdN#S5vQR^yNKJ-~4jw#cFjkeJ3U%0s#wVScjj>3wmuSGpb
z3=^Ri446s<R4GN8=CeZWlL)X;F}MhN+=A|B0@_bvv;JsB)|UmPz6&X~w@Z!D1>L))
zif^!}GVVSpGdUmSkNzxUKa3X;d{=+}L2lT0%=Fi#9uD8pjWr81HCEayJzswsQm=Qu
z*Oj6Eo0C5&4a8ukf`dn-)YkT_zxe;+ou%0OwCBJPEPP#8U-&YKRllS6HQ#h`^}{^u
zzh)0NP-Xei4U&mOc`iky)lqQ1Ohg~9t^GQpsn98VEj?23l%=Z4=sCTV=N`>|orpM-
z`+;}S?BMm`fQ-?r75_F&&xQN7qZr<GhxqzyHyGX^ozEim(8imT+>9n#2O3y&EXp%h
zBGMQ*^e&&j=Gj!mTOJ`;`l{#h6_6O!#&hCc=ta#3y|(7FN%b@-Cfl--&o|H%D1NmY
zgMo+u5C7Gm06@1>!TTK$<|v>|2c+@GjJoC?NNE4Vaw#y1T^IJI{5cYTnJHlPo@A`b
zi8d~cDc*Z>kTPh9k_W^Eg6}^N&#^1IlNcgV`GZtDfnIf^E6)jkGAJr=nb6B(p{GoB
zV~S^uT>lBO!0z`^w@faE5D{4H8As-F7O<@2u8K}b-<_)o<bwGd?x|`Bys}l)lEaSX
z##J<e`$ydh1qtwXqimn+$_~9Sav2_K&qS&Mu*iKsS8K!SZFFkKM=7K@!Q#gIL1lVc
z3Dh1VI&1X)^91^Roc>nITgK;mds^W+92s3tln<G(0jdW0(*KJ`)Wr+v@Eh+dYreA8
z!~gZRez{AnZ+3cnC&a7mP&CoG8PRvj6uSR82;aWE71uJs$b;X&zCy9OjB73>mP1Qq
z&hMf`BfrjjZhppXM58ZK@{cv($1bST=<UAdxZ0bv3bfMm%K(1T6?c9E&ai#JKbmij
zm)M>0H+$}z9Rdc8i9^}SDGzn>?iCKa#De1q*-s9Retpr?#MBt6Uv3hZB;o!QeRZ)5
zQyr}<QuM#kcKX@KDrD79CusWP{k<XSlgYLS;ozjK)>GR3qldJ<9ernNk8@sK0*Z4x
z=^Yl`>d%nph8%QkOr=Ks*;=R$sDW=puw99Mm8E45ca6}CqL=P#v5GJI$AA%_zAuVl
ztTdmAX@vOPH(9CZSS<NJ1K+UEm*LZ$uzGNMs%yV^X`j`?Mt(-*`Sb1*R~E^-N1VE)
z_9W6r+B(gCmvK9C-p8AtIjLIvzv8EQOxffqMbMPCUw;H+<6=0?bYixyjQNM8l79k{
zBruUFYZkm8=YJY@dyV~_8`6`$J})$TRc&o*J5dTd>y7QF4S>u!!>tak^TZY}Qg#jf
zc%K)&>_g3qwte?HOijDPqIA#d%k#^BNI1dX?{%)d1-qds8~&J8v&rK2JC?Dp*surV
zgw{jphQ^;4rDOcg-)uiPZ0jmh{d!*ZSq*?i4rr20RC&LpB_h5vc}gT@@GROmF#`&*
zkLOBFIQc$0S&J;xE3Diy5ZcwMa&sP<raC)t&AYU0=DhNtjux4%_WN{Ml<p}r^<$$o
zY?@-~$H(17aX|KaJ;%<z{3&<U%xk>FpPD|&6g}~Y&K^*6W%%mm#`4HMa>m?W$3KN$
zBaIvBGUIB6G0nS_=*JwP@GMt8{hmvBQHl&Kd}%JfX1D(iO>LgxP&{VqVE<?uHa^@Q
z+N<+PaBd&?r9bMU#f^oi#&-TZhc+e4_Tk*sQ-bWiiQPu~mpr_+#(Tamil4U^H~f)+
z_KZ~|so(|L3X4{nFYP<Tyz;2++V--hoj5aJ99Z4h>etnM&XX)N!DUp<XscTZYTBfA
zxcg01!`@0_cCXydmFLf<o9Af$uY+8*-gh!qg-I357>61w!78*zL!!99dc{O{zU8pi
zT&<l@?A?ip^Uq#uBZ*#BR<-D|1_zuG&$*z!$ZH!LN+37r@I6err1RC!NmD<mS^)$O
zdj0^il6Lv&-nPiqE9Xm^7q*ECGbQR@G=DrVcfML0km=<e=}6f)%%hS-x^ExIUuuYm
z+6z9fu{08Vb7CR*2jOhl@twMTbM;xKOZzX&={}^eyX`oCioJFZmtp;&dg4h~l9>4y
z{rY!QCpv}C+Ew#5&tK5Hb|0|Wym=;k`_&817s8*%i!82?b5)0wR97T(1?NhZTlEp@
zJ2c;VL&ij^E6NApc?eM*OAUViOQWM*S+7F9EVHRKMhuT3`t2$eQBkpRI~n2AFr!z1
z&fK8hh2ghdJVXKKBPw2F`}mS4*NyiGK7XkkQlg!lEB`(XGyae&JqI(;d3LNJIy2dq
zQsOXp{BiB}k-Y!qaZM6G3)&vgMI+YeiT9bZGIHNJOr$tLPdQo+W}4bun#r3V!gP!)
z%yRYX1P;-y2qeYhtCQ)0s?(Kkg~@=otnF)Wn-4`7{=ksFL5_w6=ZjZ!+f5fDCzIn)
z-a`9~EsR+RF)b`(h6=8HA)>2015y5F`=XarMxC!M6j3n%`{eiX!|SoHpMRIf^B6hA
zbE)`q1D6B}&@&vmgdEpc91Ik#o{Z|ue9<#@8)^(jIxTttO@FAnh+p2vG1r}#t27VQ
z#2}KXJ|kUY_<k&UHKjtai7vJXy(by&yw|YMipN_8jWrNEUf)kl8^999EnRBfdHdtn
zXTTLtdanC`Su4{YiuF4Tk-^tJ`kN8$aw;Rv{AVMSm=-r+Ro4e#m9Gi*a@!HgzFtq)
z0>RXV3f1l{*(6Xo1tTd{zFTK4@lHgWq4`ks=hPh+i7wmlqoI)J=k6O{N&QWLOmkIE
zBr2;Ey1|?;_9Rr~z_tgMjn&|5j|^-T(Nwn8NtpC3IyGMPA$zZhkg2~JO)YL1jr#|c
zzk`@`>z}k}r8frgNQYIjG>l+qB_-$r8Y-Sd<`MlA;%-sFW^`_?rgRm32<Ejl@NkQH
zgUxVGQv@_`O?91JMJM>9-*xh~_W>EL$nBNXqrsbga`eZekPJIe-)o<pH>Gu}G>3p`
zTZ+R>Jw=1wAL12AVLZ?<^*Z9jMc<`%2(!4xdlfB)lh1|#C9!Kt?f;BZv@-VTj$vB`
zL=u4c!?R1&IIXCIy-u5m)&<$EX^=kfvERZS^n~Wdv8)A#sEpyJ0^A?KSYj)2I$zr7
zYl#_td<W!Dsax?->lt3FNhCsdPLoQB@pByKnjRWa8M!@GZ75Kxzi*j6fVB)*j693$
ziGxy#=+jZWDVnQyx2qZhR5cANR#*^HW{~rHzggbUUf3T~^EGQtAz@pUIXNjUzdm#J
zah82IgZTA5*?<Bq)^tx;gZ0(TF=zWHa;QwRoZIzb*mo^lXQ({8jbOWtry>~MHE^sJ
z+z;Zt&Xhs1DT0M1bBCzK3(#ofdSN&S{LdOuPHoGN4eECjo>6n<@-Yf9OvySsy?&^X
z`+@51Og?_7()b-loqKbEUqoj;+X6ktA`=4a>+nAK_p{+eS<bfvQiGcfYtA@t2XtK%
zTp2IN00mhjw)Iy>?GY}+sQM_R1V~7ORBUv#B;l4~RN-n*7G(9nEndw4C{;vA8iRo}
zcxH^wOpcg#Oe0-XrYpM%(<OykGR#=n1cXfL3+JrybgyBU2d;DI&U-O;vVl%Jk1a_V
z)NbT*cH7$S_bDR(Yl^k9X;*pHmtSIn)B8g$o7M_y#!gmKr!!Q+24^A7aGy*sbQ1%#
zNZYrpZ8|Btf5-M~G1j|wocnF>cSDn)*E?jpzrN1YkNEesa<G|SUcBdDopnV#s%T6v
zBdJ-^;ITcf+kAf5Q-fnr;Pjeet$BOg*67uaiRR9J0ICkP5Zp9yjWkK5+TB>~N9@Aa
zEJKq<N;OD7;BvXN?O&WPK0H=qfTQdyE_oh)U!-9HBtVJ2Uw(bE9i_u75}NLzGRmW}
zGO*E{WFSEo_w*i_Rzr_tWdQdqwnnbHbJV^4(Y0Vt;i+$hsN;DS!uDAH-~Og4k7fB#
z`bH}Tj*TlFKLp=<>`ayLh!)K_ES*R6;e;BSe0{0fEQ=qIcRhHu+Sx6WL*L^!y=x`G
z;1+`8$JXovXG)KqIX!74sa`mAHJbCkxYXE^j&zz&Q%73$XV^IS?}!|({a~HPs*_Kl
zv(@~)%bG8+TEoBEKe5}D^ZJAT>Bgy;2@$<$)98qLGJkh>^;~nK?;R%yHseEbU(pT8
z!@<m1#djuRyw}os^M@PL$=^)k>&{XAP|jRGRCGd08tr!K56;s4*_z!XDyV;xSjF8_
znS-tuB09DQ*E2wR+n0@K(q}emVR3BoN{L;$mhFj8$X^GiZ%pjI>cn1mnKac+wkzAQ
zCK1b2$=byYAb2|?Jox?ks{9_Uw0Ek5zuj*94;^eL#nV5Culfb>wKPe($y{yaJK+v9
z=gDPFG@oC-L(jTT;GE=^;M!btA~I}wn!hLbI+yZ+xJn*VC;fi=+o>%nWT$xej%=P9
zPn`_62@bkHO$)hZ=9qtbFW+wB=#(08>?o7e+IQW`m~guq*<Tc474+QmEmge^rkXYX
z(|r9*r{YVzTI{o<8Q)Xc!#@qiD%`ofWAJ!xgFmtYhD9I?kTsZSNw3~5NwQ2Wo~wp8
zjvL7<%ESNm_YhE!dDj0^X1W%*P&8@)aRl;!RR!AU!jpNzKzIL=OKkQ)>%;OW5i!u-
zZL(T0$+_6P>mI_ucYQd7S2AvLSmulZ83~1*Gj_8Y6#g@+Ob?7Ia~ES>(p7TmdxqnQ
z<6n0@Au)(~ng-qIDt#k<Jt_Ckq_R6Osr)olh;od+ip$?AmD?tuGFznw$-a`%WjnmC
z!y&u4zf(XkqLopm)x5lPHJ0~&Fjk5U_Q4MoXMA?1NOGV>0-xIKlo44brKLM4s>c*5
z$jmC!Hg%nrIsf7nIKn1{Sp)n`z7{XTi^yl0nDBsqI2yn7jLPs&p=*OtO!9_FJCIqw
zMj+T3jT2VF+;*nM8iSKdX2x=jfo#i{*K-sKC|)2FHqTl%ia77JI*>58yj>>X0E!7#
zdQRh~xs?_3mW958WMsr|cx>|1JSs&u@D$ngzhm*W7%aR>&5gAe_})jE|FYy2HMEU+
zme~r^8y3#+rL5`v7yR;g!nQt!`w3{^0U?oS*!RFlVoEE{m+H_#R(2q<srM0_5sfPG
zLo$ZjgZO$`XfN%-e5lQYw;*D|x*J=&7D0<OrX{8cZt6sZ?=LR)+PXyz+c^mXEVm~s
zPQa)Ofv5KR_o5i5x0>|Ua_@=lwP5%4OHxS<E*?q%j#df2pLQc;gIQXH$1BX0Ct9al
zFVBxTxt<e+hJM8Cl0~hoW%!Fg;CpiXuj)1|`9q4|3<A^s)-LX4FGt-oQljxVKY_wj
z;CM0on$PbA;12cxnG1v)wAY**v=wm3&g?UBI6(D!d0rGp*7W)<_<AdP?ILlG7S4ke
zpcHK=xUvZvr1Kah$otgy9vpGCKzn#tfVKB0IqAif*A4TjjlP}vR3uS78fZ(Sa<$@W
zQ;rRa<fAWrsE1#FGC3n?nI)dNLmdM1VGJNJGT2?8wLS@?Pl9)%e{))34OUzGH;HeD
z_hj!D)|O7)FraVKy=sl{hqHB#?<WMV<rM95nOGD=AHVlnjuDx&<Z#`PFO!J1=-cqH
z=$7T!dSN&Rj?-p^0-tPhRmtcS4Y2|VL7)>obmRQ)3%ZqY#*G*6yf)H-wi;9k`swO&
z>1od1s2)8a2>KFKqELrBnCiYo&a)N1ZS|;`;V1@dLdgO2^sZ%V2EFrHsI_;NO|OOX
zOkD%zOnvTt0gajqt%`;F!YdrErzvl#VmJ%0!i>M0;6~6re<GtB{%6O#wpY9z#bT#c
z_Lf0NrWgx*Y`P%5uJE!?GlEEnO4`HJncYx^+WS2Hw4mtF*QuYyszK4^J|cBuqPV6s
z-aPfml#E459eO3@x5x3uEl>;Sy^3-Xwrm@EL2Bg_&-XpCk-WaZ{@?3E%~Mtar}14v
z`WAq|=)46Omg@B5+e2^YAeQ!zs(~h}ez9!#PvLW(YiYdn+CSEgw^0vSw3iPD{Dmy=
zx)t7l2i-OfKKLq$)<<;%PUIcGU3$b%MEy(yvUt^%5~}*^XP}&^J5{bP@qQj2jm&J1
zk0;3)i(yB(&A20IWtE&CJ)HlES93Q1;QD(9%*yr<bqIT+3AJ{OIHL|?&8{?VChaBH
z`d(uBeAN~TmL(6aHhc+i%C4V?m+$#-mE+lAFk~erq<;26gaPw`q5)-B6iA-mAa4nK
z;U8ixvGfZR4;JJ%vw5Iw*L$O(q;oQtG@%`97a<(LFpd=4c@(d*(bZ79O#P2%gLn{M
z)w8|9r9rs;H*Ev&E;VD9qc@6T1Mxy&jhC%CCH0uWla)d{v+%ZrY~rjSZjE#m7LJR`
zh3j8(N*r?MabyRh70O27Jd(nU;oS*EV6eR}LBe&unccmsM$4%3sF3G6h9ct`v^?Lh
z507?FztMy+|H_n*<MA^;VBG)LZe5R%3paA@wrmiDa^A6igS{pSYBS85@G(SG|1cUD
zc7hD3qUm*R4B9ep&nI$Zzy+or2R{G6^?VcN2&I)G>!GL!@R;>lQ*?Y&frE{DtzgeT
zT5Y2v+&AAEH1nnQzWo?hqU;f16q9r#lWGG|ul|TcFR&i~$5;ytqKB7jxWeYFL)Wc_
zBHD32;D{q1nug(j5M3WUeQ4kdkv&AOWnAPDj_I#s-<yvG@o}BCx5_oqxl<;7EYm`*
zUx4ll=>6<my~xtUy>~4seDf<oJD&0Jwf?1q{S^<UC#{MG;!Oe2+)PWNPcX@>vSiMb
zu+6}=`54p*XT&%73~NHKmX;OBd{kh5#dUSIgn%V4962iIRXzRZPJ^a3S00>6gYdN!
z*z4ZTdU33{mNscq0(C<WjXp~@UXlW%fN%7^ojYJBSo{aS%<sZ^kCPz`Yw)b4)l-T4
z`dTHgSO-|OO5d}R>J$GuSm)xjAXeTR+McJ`!_5Q+;{`X=-GFQ?j~{TFpI0`cUeS+0
zU4-5CrY%p3sh|45O7?s}zjQ$n{XJS4{?<zHy|0C4RTT7|82zDf#iQoqF4*c)!ZZKS
zxr6aj58C`knwS3A5dK{$zXbK%I9-{`tio^l@|_o6XWEB)6M{y*jmMlxjW1y<EWt}W
zb+faltPuL9%k>>g)S}|1%gT5C5n_Gk#cI@eMBj?<?1xE^A=GPwKych4yKP+Nq8i$2
zUuDp0Cgp0X>0H0?c_Hs>n#}uM(u-pO8Pe;~nJ`$>j;7^BBZ-it5X{tH@G8v}L=1I=
z2z3-UY@V*U!%hXlc3vvA-!Z_okz_$~ZF)U!ZKSP-284(-zpbk=L|RlNowZG@*6hn9
z(a=ute@r_S`nKtq1o%jeIdG31b;Xu2P=?!-UzR#FFmJYlt;-v1+F)evqsuMhmRm-r
z?(WDXngjUU3uzpjiA9XKAWH}PF_O=|kj+f<*Esfr_2XI5ubhwaF_@qvi9ekMC8~7c
zL6D9Bcw-4t5HG`1GtLM%Z8jmzqQ;Ge$zAsxMB|RJt~oj1A34=vYcP5dro&UtUsA#|
zvNcTJTb6zMPjq(V*z<=E18_kYSPjJ$U7VX<Cxef@YRFK`CI!da`AEE^5fUlxyV$Y^
z{i3_Iu#~IHjK-4fmzo!-UY?ZclB8j<^<-&bkFycWr8wbQqG*%f42J)0GOF?<KCcf7
zBQju>gEN$RZn_BicW(mX4WPbvY=pgQGX4&JY3L#}tlu%uEcjSG+dHC{--q6Aavc9v
zlS^G)<}TVJn$>Gy8~ZhAkSui3x}gs#n1;}JH_&B2vQme|n7n=IaOp4XmzZQ*W6vB$
zN^Rqzw(an0`!};U&rc6ki*^;|@VC24BH5i^d2jW1UPYvc4=n<d4z{S0sGXOuzDU>1
zjzme*CQI2>dif}rwzMS9DK&7H<;23e<Haqm0=m*&MxPp~T2va;=<JB(`G}za<+oPL
z1eME-`*^R`8{)ZxCLOGQ*A|lCSN8B2)RlY&RsuQWL0EM<8<D+VZTk&@Q{=cf1w7*}
z!>U%?7c4U)nvz^t>6vOL6d52VBATNYM??y~QY@?^k;oV`h9EoyUk3ajtxvOZ7%GoK
z(K##VociBN`$&H6#7MKTR=GXAaxcYQ+{@P_p;#2K@F?v2RQYqeYJh~NDm?u&k6!sC
zosWIZ<3Ck^?k@8)Tw3bpqv3Z})xH-^re)c0fU{Ao?>h3%3l1m^gZy9nd3|UMh1dyR
z+M8y`eG_`|^&VXGo6X1!w9O#SOj7r7L6IK`Tb$k5u0ps3FjNkJFt2KzKK4A3SF#is
z=}i(G2&b29H{vuU%#Rz{M+gJ5qynyO5u$54xXH2he4g`-lMEPgY($G(8qlV^#~A}b
z|Ccxesp%zh`Hiygx*1*y-K5&A?7g$_t+T6roYvS>Oj~F^ay77cGp02k!vSIBa39km
zFeTKtC})z~+rw!+gbluw5uhn5QM#qI^eHj7#DvixhFF6g>a?vEB5`>Ohe4t`seCF<
zVpOY;@oB!X+!vD8wGY@D>m!L#pZ)jU%!Ei(yR){YCsf|$?-xMy@vq6uqoqWcEP#SY
znSG&xro<yQH3%N3r2*ik8Sz-0L%RC8Qm_j&jSo%>rtHl$#Ii0=c*1JPo(#Af*X|Ha
z_S>{#wG^QhfzpuiTDBA-#Ew1+|7XNT02uqX2g6k=ob0x!xbOMVL$}zio`k$V*cPB&
zfY%@&qE)SQ{ZqH5lw`7)bQcpm?Pl=dEbQkbwk#-0c6mC|k^g<KL&J{cTnkETF*tw-
z1*q(Yzi#wa^izFx2ZZZHT*1lxCVQ7xPZRJk2&7b9lDUZ-@|F|=#c?5(5x&UttF=e;
zwYgr0tCpek9!@%+jBD7EIP~cy+&7mQ9v`40gE~I}AtHbsfW2aKY&&@K%1XDN>X*uI
zl)`eaXYOCW=2qjHd>q!xVDaoW-~^`Nv@>i<SPFPs)eq(X;UU$P#4(Fquy4;7w_q$^
zM@xEaJ#bTvw{cm(yD(#0*h1~y8U5i8LSBWW6V@sTwSW2{7apw9$X0bW;U#j@IPB1I
zS2c6a`5dAjH**pq7Uu>Q7^r?(fixsQy%fH40>^i!>neDW)BZ*imRh<K8RTX4Gv&l2
zaU`|iX3LWLyo&&>LJ2d;ltObs>`#t9Z>wjXaiF{Q&pj^UJhx~eIoO#?Cy}9vxna*T
zSgOU)Sgy5w+SuR#)9Eb5!z(w~)$^jd=(?+1)2A1Cu_;Mjg87QD7x(9QJT9`MuYJ^t
zQ$JrxP10CTHt<r~U7T?`@cY!LUNO*Kj*%TIoL1?$ErMtyP>UNC4rxTPEEzPru2DUA
zZw(ecc>Jd~B}Sj;6ZYWG9Y77QSb?nN9_Udn8<NzVmLH!{`!qWxu1nz!p+nu(N?Ep@
z{qL>;jk*4Os^P}<Kq`&4js^*6W=YBvRlr-54zSR^D-Qm4o@kTsmaZliOA|FFgxYEP
zb1<fuM@a=NBt^K6s<KK7tpNBOkij3BLogv<;d)2=i9ARgl<;#SfSyF?Sai0R=*&%4
zb%L1V#BH7?5hUSFw4VOOxXM<;b?ae`B4ct{hp)}K-=Ju_QYno_4Q#VTTn42szER-4
zvbp*v@>C(olmR_|N6`MauIACyQkgI@_dMR_u>-H5%c5xM>rqIHobtAF4msMs*J|}6
zf&uZ<VAt<GHNWZmB!@tS4eL&B4<TK;7fxxf56;>xhbk2}6m&eHCWT@kjZa@-)l((k
zY$mGH%n_+#!~KV?wOzJ0b`1>SfQ%t(R-U4%X)!Xxr!>^p#+1CXX{!_*jGX+4ZB2$b
znRX80;te2!PrWrxdD#1oqv^^!e1PXU#}aXpV<vFXJFj#yG)zIw%ho3Cb=o~%ux0h|
z%CypXE&HcF7~lugv>fmZ_uawgQh>?roe!%awCQdjT4VNU36vE;DpFoN({VvLo$c|j
zZWdkSB*Mn+sa!9Qt(3lB?D`Kik?rY^=EZQ&v$juu?eN*1p?`z0nMO7wNxn6esCX;|
zoJ4z+H{d{^5^Y%WN|-w!*C!6W)<zndV&Le@vzZ@Lgr3j9_TFz#CsD@^Sp9zZr&3>u
zsvdd{o?Oo?xscAke}yr>!`QI3J7T#^elw^ayWJ{}<#eYztmh?HAYQ<O4QxqS0)n%}
z9__Utt%pq|N|X5ZLG*)EMN^aSd8bF2?6vdb{870Pyb)AT_XXOYHN{;an+F`%2P{yi
zxJA%iYI_}BNi>@kb&=;D8%xR&8aWsZS*~IxvQnerLkd~r=82pXUc?5kk-lyX<ke><
z$S@90b(%wLtG$FRyFYtqBR?9w@_}4xxt6e<q;fcyIHTh1@_<8sqp*7MVH0aU|JhO$
z@9IbB;XQ;199=W^fMm=*jXU*x<q@}72X+Y>fBmMZt{6R8#)Bv#lx+hB?_JdJzAPxC
zc$+b1r=C0MZyA$^v8>099es7YE`M2GEmk&;{dlRqg%W+Pn%yjcbb8h4x8tbiE8jLP
z$ay`y8~3HG@{@<w{v(Q_Vx*||r<3&SM&Ra4WJ<wNKnhc)!9Ve8kDLOfy(!v#N8bv8
zi2<=CqM8@vxWljtz>0{W7_E>;5&*L=`}r<;_RJHQG=C<Ln%U+-JoWnh$-zpG##~c(
zsE$S2BC+R#nRsH0YcZhm&AdnA;un%?e_WBkjn6_0BD$&?8lVKlmRvOx$dV7F{;U&U
zO*A)TzqPq>zCfj@Oz$3NGg+QR{*VyFK_0+PHR1boiAN*j`m)v09eMB&@$>EdBwh@W
zMv)H#fTk@3XCJ-zT5GSC3#MAu!KGnf;C}TS({|~`zeubGwX2(@;ZG|YYv<fK?xA$L
zVLGmAus~x-RYkzfW<K2y@62`a`kO#UJ6Pn(F%X!mnT48M|I{_wai0l`U!^hL3Hn8j
zS9fc0v}mqnyUIVz+A;%vhD;LlQ5t=n!U=G}8ZrPEoP=bVGP^M9Wg~5(OREnpwRRMY
zCqhgK+d10SpF(K9?MhzXPw8V74tn0e=^%1Zt%|j-idFr9Fd%QJRHLu%EtEWBskG`<
z(nfP6V#+VKvt>XX=!g^>rzM@S2x&TZ8#SsiWjUPdoQ;?g&2Y&~l3Y41Xk>o`eXwz%
zg_~sM_+89pI_u(6WJ&9dpzE(&hcD-*S*~6nNIxOTyjKFZdZ`>w5a$Iy0ZB5-e!&Tb
z1j8teJ6wv{-?6MQTX*LB++inAP=8dJ3Z6CQ)h&EBJ%J`DPjs}0)qVtU5VgI76o1`h
zv7ajPyV(|}^&+CJPd=gM?XmEFehw)4##`2Ms1~U%zu1&MNB$~+J2qNKwNoF|6<q{k
zoNU4r_WVA~7y4qOA7K7x!_#r}v_+CMahILK&%>p6wlg{VRQ=sYBis<Em7;`Qz01?0
zFc@y)OzL#24BYyhRKqU>91r3Pgr4uSjO@m)8K%i_h)SZ0n_sgD!u~)W8fd%2M*4)N
zfYyJXv&PWAJ>#ZGH`Qn0@3#u@>bH(1`>azgbHyGZK*@fWrQec_L_BmF_B-<XqL^5W
z8vQO`tzr%luE$yO&4Rm)x}dK|a^%X8nf6(KE)4<=I@#EVEUKTL@kjV$0TP*E4E`9{
zNb@zGjWQ~`bK_)XYT2Dod;J%uPyT04&co^v64Cq&zgnHO21X+hN(rEnLX$8QDq*1F
zzV5^__Q3^5>?6_CCx7lZ3fWkji}LHMQ`U=AH{&%DS+$~<y694%GmDMJ&M}DU#IiRz
zRtuCIyMs!!ARXoplsD;vz(HZk#TP+%dZwg)dPAKHJs+)5n&~lq_`Xj>*f?o7V<8(0
zYFDk&XIDefbW88cfdTFOXIE^Y;TD&7y;dJeW3oA+=2;MW`yu5*dH6<;2<(YV_sfGa
zi|6OQY^CO6y2}|R+X8&-I~jDIUFc=4NI&D(a*E1w5nDZ!@S~NWm{^4LXCw`J;`h+V
zef(O@<gx)VOx<K23o27i=6!lcpPltVS*%J12K6;Iw@)l8x(Q<^RoQKr`Srw_3>YQW
zi`$a%zERj)ITy+!@^i<scKVr80bFQd)8ipR`xAz)8c9^j&v0M=>d2#Yu+_Bph?%ZR
zq;;lz7+<2dI{O?vh&8i^A+}7_aHPp6E`nKW+K$g|SM@=0$p~((zjrHXoZEmsNlE?)
z`b}4}>zr4ok#*Zs2^8FXb=4DS{iN!AWxzO*wE2QBmO$qvNFzVqDrrY_ZWgt3&^v~k
zfw%3MX;bA$t5c*@(ls|sp{8-Ib)JG~g3#?hFEq+VieC&?%}fi7Jqil0ZhmHO`=;Zv
zqkN)o%^aFgz?tGO0gxe6<73L2_ic7g)HV>JG>)^Asi?|Wb5eTYQB-(OTmtf1=@j6I
zdl52Mi2`nr`79FT*2Y|O^FqUze-H)`w^T>HRynX&#=ajA{MbRt7%-2n(eC|Xldst=
z#(V)dedTg7S7xev&>8txp3c0TqpoZ}Lz(Kq73T6gB50@Y)o{b(iw;-Cr!UW@<<!%q
zP%!!h!W(4@9?s7LOMS<H%KT(S<yhwj$Xrz5D!t5T`*@FJz-j<C317o@WmpQw1TTn>
z;g&pC!69mpQWHqyihT+5=d<d_A1xwCHO)*jU?_n*8N8LDnQc}v?J$N#mlDot>;PfP
z;w(eM)1;1PG)>oNtM9%1>Cs<UPbO8vxHI)y@Z$38u)hX<ket)6V~5lGoh{%-F_S{X
zNZ~R9p~n7~V{kxdI+*g7n5eycn|tfa7eMLn^_1xy+Etv29r`&(=X1c**-#hg`4o(x
z5q6nKX<AB>p|-EJMDM&9MY^G|tW|wxI+v7FO8h{k0+0?O&^IA{LiBGi*$O-lyFBj@
zt+ij9UR%m2TMcN4x=wfP`=nm)lg&|c^}8BT;@}>(1{!V|PS|(iQ{ZIo*O(l%7O%sQ
zC`*SPrA=p9u8gv(0cZ-(Y0@K@2v2_E3gj<BmtDOrt25)^HS%Sw;-n!?87k7yJ|L3>
zY2wy%d$jGfv&af`N-w~12E(aOD!eesRm16Vw8jvdt|@RMYK`)+yQi?(MnVSNRM_a{
z;%8rSWPxzi**-fDx{$HJmDVpWPh+nA`jirIyM5EegMIm?hSu}Ab;~GC<{XlL4pmOI
z=z5J+vC=;>!EtPT(^}zr;e&$$u_Ms^Qd2A8kCW`9jEt(=n)>>vMWKtm*W#C=u4<t{
zhK&m(4M|~t{9DJ+{xjdc^}RM(yFGK>uVvZzNh(>GeFmxJ4C>n=ow0`c9#eA|kISr3
z$jXn?r1`fzg`D_2xE)wZ@wr6ea^iX6>$c<5=HuIf2ic-5{|{I18PDebzH!HHDcUNE
z5-mlky;o>el^RvG8hei@YLlQ<Rl8Pg(W+gW*s=Gl9kKV`JIH<Y`}^My?)#aCk?WP~
zbAHa_INpaoK!RLo8SibxYSByFITB{+yXdphEYXkps4OE~dwsDvoV0Yx+gcHAn`~&b
z1bUI|_=5DF&`){eNc`lHmz>wG@sRUQ@j2B>cLsj5Rrk`e?1@bDMJl^AOkrg6&u0>R
z^oJ<+tI}BShYrI@$w>gU=GU6siPwZWrMtUY>A?3u(n<F+hrZDccjx2Wvh1YIw1%-!
zOijpB>D#MkDN6nBr*F!^Qn-1pGIdRtJd=i9JIt!PoBdb`^p&>a=--=o$)>%hGzVq#
z5--2XE#Hw}bnG~4A~VJo^hOV`@>l;(MYy9HlGnE|hcME%O#{SXpFI}0Z*L$w{O|ix
z+np(zz)fffy(qBQ5~WPvU+ua9_MCfMSwZcF5YJPmOBySPk&l$a8<<q4?LO1(Uz@a-
zP9maKozc=rlbKF&ZZEKPsI>R^P9n>^c`?Da*~wO<9@V1@>gyAiGe=6oV}x`V-8Ntr
z!ZDmh(v!SUqqGBqN5JBtHh>7=;27NR9tEfUM4Ytgo)f1_wvjr8$ex)J==hU)Zx-fh
z9c0kLL1b6A=~=5luVgOf|Nejl>LvSL3~Cs)q89GdoW-QIe~N&$AB$4$F9*DJ=PC)+
zSXy9bxt9DM5AhCDF#j2=0S#XvFm{d#+XdHthd+jrT-e3ug)#BO9dMh%1e_?7-pX*~
z1^=_>v1+TR3&noFh$;9C(l!{)X3b}R^=Hz~2yF<v^G1a#{Pm&^_%S<Ep1t~7cLQvJ
ztbyB-$#CNlCX1KBbEt*Io%K9>WL|0ud>OO((FND&#7Zqn6gT_788~*<<v#d)gLH5%
zGVqSP2$Ms36qJjVMooWKAR@AXeS0|BLnKH<^0(e!S5VUSrlVs``SwO>`)}dK$dRC#
zZ3lEQYl4I`o1bwuF5I7U-?suIeV(*hlWU90cRbyksrDKnjcryFJX&AVU_%=d^d8_+
zW)VNCk4Lb*bQLJ4x-xGMp?DUrL`=2mffYn?P;7?f`Uiu6t2{YQ4vX(d@tN?7+Kuhk
z4(Io~9+Ht+dq~#*NxRm{!DTn`RPh)6as6Pgp@_I$2hVt_XDZ;oQ}zGgbFOo^xb$vN
z*#LsX(i?f2y0VcAd?69ILy;mzk!)XDT-4@D=c(`id9u=R*I#6$x^Mw1?;Jz%>Pw(k
zCs|&xNxPX$><5ZRekHx9a=<db1?oTsh8JU;JC<4sf@wIDDrbGMm<8(4;kxktX7)uh
zO#>5Rx>%)17Khne>$iC?5qnK!D}XK+adcI3j=r-tU0Yd}SO7r*BVRf@+FVbb@5vJ&
zuzHK`Tz~6yz|`5j^zZ32Mi}zA0e>~Xo3z1EB{2(ZT!1)D7|^Y$nv2dT`?pfDP+0>C
zEsxF@A3%We+?}$<Rh7`8brc5=HwxC%4w-PM`hQsf6;B%91Aal6%IwjswS~1A&l3p0
zVia|HSDTx0Y*F+HL7{o!cX_wRql#39C|Aw)5LKWd5{x+dt2%AD0H_#QRPXfOO`Y?3
zo-fC1`QNaRdbml|Dv1CVMne~;!l4AM45D_iF<6%YS>{21(%B$F-n~=;)`#kF$AkL!
zLj$e`a)!yCNZ*U~HX|3104Odvp_cj^`_~Jkh|5ZE`P`Gz8`nT9k<gtu%jz9A%c2E8
zIuoJ`1`(U@oKe-?t1)5R7Y$$7YbH9C4)C+lcS3iG3x)B|Q0;vl7+}+533fjW<B0Pd
zz8qAbc2o7|0~r%6Y`lL;F0X)+froTTk=O*!uET#2EXeO~-@n@aRYkw#(VfKczZ0f&
zl~7&_^Fec{Zq)fi`e(F@faA`C!+kQxm!V#P^@WHOz{1OsS_A;t1^vy3q8c&>;~l9*
z_-Du26vGMvCZ6B8=s^qotvFUc!P7<J!!UO7^?m0ar!H>S@y8NYx|P!r<`!pKzzGAr
zNwCJn+m|Bscgp$YfwX4LHGE!h$Ez$W(O4;51n?@_J-g?a2v`?i$*ewRO~aCgyU*_W
zVQ0y7!D;Z4<u5#rqCU(sd<SGEL~~yo6vs;2LRCGO7A4yL-HEb7HfcxsfKQ29Gs`a=
zx~q@q++jN)5b<vY1+@il_jPg+pPS@V8R%oDf@YmNm@TzE2JVCqPW`|*{RpW<dI#lJ
zJma;toU<nCC7VUO;STNdu=}7cnI15lSZ{(_zueMhI0nn^3VIBDZS3Qpwct=ie%BOB
zj(Cy!MbZ|3qG^@z3wDmZUts?%7ct;7f&l-y^vRU09FSD6*hV{0Vg-YD{BV%NtD@Ot
z$~>6Kj`X5$2Z#NUrG^oARL+zK7NLekU=zdhIm?P(Fxzr#T<t0gyB-aS$maezxREY7
z=g+ZHvl)Hrrj^}aeySv1t9-mU-eQ%d{9%|_0V`eMI6DNHl^W{0+-o6r!B|pP)!;d)
z(k0h_Z}#;jt|N)gycI{sMf|h0^AWv%cq}^SVdl2Ebe;lr{UZls{Y2QEifMPsl#6)d
zh*JKs6E*uh<(QW%X+C@(5tjp`P77LvAWn3jfkpa!(&FT6C11MqVfETB2u}iS@Uk7G
zHG~M*NOILJXMH>#)dkFCML^W9uS?7`W~*d`rHknkVHlwUpBR45l!W@W9&wb+V^6{h
zu7{qTgITTZQYZevk}}YMl?=@A4PvZr-bi<CsU-FvZ*SQc;O#|<dB@(CIo1qWi@zna
z(Yc`Dw%*hN&TMC|Ee-gyf%fa+V=`rjnl;{f*9x+XmuhXKy-z~Run}FM&8dKGxDC+u
zQUKcCQGjLg9U$e(tkR_!@7JDD^Djs<zi2E~zTXxFKd|KaF>-SCCP+kgW3QM*qttE@
zDfQbIz@<zc82=qx4j2XM0%YKPKwAI*i-i~`_9u-8dJ(IM{=(|^+y6NjK1gSa?*HhO
z3`(;vD|~Rf&wKqK)9SK(-)ihKF-yI#CQGyarI_KyMWGzv<*zk0bkXgo%}@-NqrbK#
zMI09D*TG99ZL$OU#*#m5A9cQT=~uAM-fB}Nn~SlM7I6`>@M93Qj3xRumrQmO3Z1Qs
zQXpl5hpJEPzHm93QOei<br5xWabKwF3}}BvBac!smOa`as@l~f51NJwjO&<qEk`u<
z3a~8|YB6wupLRZ&dvIcCWKvm3_@!)I%tr~~Z{uBmXfi_TRIU|}YQSWox$EJri0}<m
zm9HOAgQO=lSfH!Z2hM#xs`JX6Jg(2X_RIY9dbw3U^Y-dJuy788fOzUesm@EEnv|yJ
z2b$3JcM^ZP)!KYNRQGmcGH06SqZ(zxlEQ(Ha7}xl!Vcx=^I+}h`tndi*s?p83rZH6
z38I8!VNcqi`;QtTx9!iKmUWU+f+17ic{^rz2z(|26I`;@Q-1{>j~n^)otrh$nEg#^
z+XOZ~%K-v53at@}K0;T;eZ-a=cR}sDO32+OLhs<0`yHA(G{!HFVMhp1*p_A~SNDRP
zKf75$vYGK$Z`1|Qr-s^B+|rwB6Q@{9{GZ5MbvXcX=w}Y?w@~#zlQ~bceNgsI&Y+fA
z+C2=V$YZz3>C05Iw#7eJR@GyDhh>();?rk1NR6)$_bBVlFFx?A&xF|0Cq5d=x6=JK
za<gAj?5sKUZIqE#z4WmMw(c#}%Q(eZPPkE(da!i*_ZN@u9Je8GhkO)3H#Tz|yLu;x
z#q2@#BR_jVND+LOaE9PgY0AsIT0K>2CyD4C7vabs{IdmG-S)CrR^zcvNNUlUhrDfR
z=Uzk1A1(+eaMuS#->n3^|7bW=oyK$9`zw0fdaG%u6aOjBmfQxoR@GTa!k#nzg4WlX
zM~wTuPM3v^g%FMP{%T}F$m~Dp3%Zu-WY*8GW!J}F9Dc$eJog^$;iC4R`Mo)+571Bl
zTGiY5o|XK|fAyc;t@W4l9L%@Mtmh!2ec_MieS;>nS%5OohD{~B278af&}s)P4*$1Y
z$4wXdVrw&01GO1+jjL+&&%Fm<M+~4;U`51HealUu6d=}!#1##FkVq=BSl<cts$C+y
zE9_6A)>HqL-glsXE_R^N)4T26dpEU?xbs=b<<(tXd#faSBz_83?Ch|5k>(O9<fnSy
zyl@>Kj45`nw~+wO7psoJzgr%?xhoT)g>G<pNIjqI!)Q$)Iw)}6AM_!=P$(X_g}C6-
z1+5Hm(zWGB=S){`g_zDkNq}q7>T5mD>|qSV-)iR1`byRbjy}5UYZR~lSKMu1zg4DV
z$<sSpC4{eu+NW@1^(Xoh)R~NCxGN1YUJu)KSB|~}ix2Qz>z7w#(y}(f5Kg?GEac4j
z@O%Ji8Ov+m0qCse5hx|);O(plcNJGa1;AK;cQYunZ{Yi9HD75z5l`$Bn^b)S1fz_T
z(BLCcAMDonD$B>h#g6UoS6NW3eXovNWyn!TzKraW?I8fm45bkUE*M%R3*K}xe(}jz
z!0xR7{VeW8+?J$cc-K}6c)*IuKlg=QR#WK{rhzD|ht=d*C)wbhArslhdrX}og!YW`
zi`H|>!YN^YMhVirW+1zT+WEjOcMOV5_SsdlG7Y6Ri&3*npf}<=O9A`9#C`Z{CGPgn
z&an&G@KggPsgIqs3{5NOE2a;gLy-7OzK_WKbEoi+^8frVl~eBy?#T;dNZ)`Ve0(2n
zjF*tPsnBstG=kqWXD<aY+$W>*d@%B`E1kNFWrA?OnBd`NjKy4$J@t~r9=~zyS-b}3
zjcsiT6}tFgvzWu#K@mjdELxMv_`%uf)q2nq#G{LnXSpSvV4<^j(y4wHDykCQKScCY
zIDXHCMk3U{_>!uDWBjQ#2tF$@rxD8M-oycmE$IVrrQ;~GsS<s<MOzD}8>XY-)#Bg#
zt?|YUv4}3S0Zv2zBm0QUgElgDuUWapU*Qp3lX<xxsD%b^{SOfeupyQR9Mk=jh|AX?
z0Fr4G@^B{PF%9MT9R@1zqqV3LwN(R3;`EMDhng2FkP1)?diPY+ugE(lBI0#K-N;PJ
z&bGw@C<&`xJN^AF?S<te|H{`-<JoKOkU|$?6|cih@!mruWY^+&_2(grm}nmR>_)?S
zAX8t$Y<mt*<`U_|^;Ed0MUO2G`8g<hvhx0?z}NIIKl)}4q(fG)!E6W}6;ejGTVT1V
zBj3t+?AKsbaar}MF*jg-Iz=A)V=Mxeu#C?Bx5#TJ5vfHoE8RAqvJZz-GYbBEV<-tY
zyW?j%OEKlH9Wj3;d7M7nYg)^^BEb(oshdWxQx;Xiwe3gqEY+pf*~M+>kosTf&K<`N
zCCxRJO_&xG=;-c*q8+fP@P!K?f>hOe6LNzz)srI(2>0<%-fcEMGBYjF%2xjnMJbCa
zGTo0Qk$Dw*?t8-2DC}>;ig$3KYwrBvvJ|D11VjmHe0kFa!Msc76npRn5X?VTu!D^9
zd(@{eam<Ff92!KyuYO5N&=~tclb;#XKF;Y2(uZ6EpaSMmKnfuC4+dDrp?MZT%9UIx
z1-??C@ZDso&^Q=pvnWG0%p6&OZf~LEQ%mNc_h`+q*dijMQuUcZNH*+r5bvFa^3(Yy
z@Ok-*!4`=Krn~^?|2ESrt<mjLpW-&A-vbYhf$h{p*Vk|K&H*5!J!VbD1)s=Wxo=N-
zEYi(hGa_SCd8o|xExX^IHX%UMpx`s41KrUTjFIhSm=p3L+>#<)66l~|?j(T9Bxtu&
zp*e>vDV|ZuTi7-#uiaC;Mty=8sU?X#DUet+H+a;@5N6Ci$l-vt9=g-``^>{wzX<1g
z(zRB$i^jb)Z2Khls7af)RIcRscB}Pwyh7~xE8vh@T&w|^<P|VXtlWG<-uhV&zFD;x
zDF>;!zJ$qU5)7JP<Fk1e-Y+DDXdSNf^-)Q0%}u~4UWQ#AuJjBmt#I~r?e=69{v(?_
zdC&&rw`sTBQDiJ1f9IBPC%1&q_Vmqp7>mA+U#qq?^|R;s$34ki<@H~gPwpGkO;hpl
z=3755#hBF^8q5x~1SNIUr0Kuf8DXJLGNh82+Rw&I%}lxJ@KF58pF`2z)td*T+};6W
zzqhZ%`vLW0C9i`lQHm1fo(0;F@l#r@!jx~HyI5sSr{#VoIjMl;mtnl{3VmPQ#6ovU
z;B0{SAU<9>jlGWOy24YUoXmJQ*RHHt1n@rSqUz-W@RUTS1tIH8jMYLZSO&WHIi!7)
z$fon|NT&9IwEmUllO7)b_69VBaj+xoY;N?fn%@aHCWE)^O(z+MvSIYhAt|CN=qGZv
z%7P_~2JcAFbb$YhN6^Rf6mB=9Xr;iEaR>l5oge#-zWPLL_;rsVl{0rke~N`SdO|p(
zhC1DAt3|1g-n^lc3;HACWwMx%A=WkYT^IRb(<wON=85F|Rv;b7K`!#m<}|c+8=Kql
ziJzGo(jsC%6~=cs#W{eXpMwatemj(QJGqD3a;C_D2|Hu(S7%YfiyFy@$pp3&2_d_(
z$0I@RPyH^>gA&b+FSJ&Qc_Pq4gsYdy1R}%>mdS25EtD~*i`gkd!`DXBLspWKeQh=;
zTjwW-nk`&d)x526ciedMCewdqvs`nQfBx8l`V{FaN*a!-oiYsTJQJ(;ntfD64irOw
zOwhkwiGPLq{roY>AgJ`2j_c81!>#(kx1wsm%^~U;bf19F0qb7Lf0pUDva|UHEOUnb
zMq(4X$<BO!;&TG;MkmC>$3kF*D=qctol`)bnAY!fWYI@-Xe7jHFxaS9+)1{eM%-`I
z_+ua0pv!Eq)lBkcXpD%{A}2pQ4gWpZ_m$L<Ik=h5dvER3vWclvE^`+t=Dqiu4Zn7N
za^<_Rp+w(spn<e_tOO=ASD<62^6YHPxEDE8>6olO#-B-~ab2Qo)ojc6%Kd&A$8TkG
zF#1B|CP_#%r1A0EM}iwR&*fRyH=gUw9=QJeyNlB`8Kea>kQiQ<1mAkBbhkORJk-yV
z+hLUC!9Soe{6C;^>_&F@KcMkd`F}v;CyI;!(0F-b2y$x+KxC6^o<BYQFORc&{#txP
z^k#aKU%mHnZ$ahE2Hh0sF6h@jece!a$)CEJ)gv)c-R*M<&#cJ4p}X0tprk&rLdP8m
z*~FLeTY~hEkIx?Nb-x@H7otv^?nz|N+{%nJj5eDg`w5~VV1G-u;nS3H;*ga0RaKlB
zEK%Read}IGt7P}+%8~A}bAUMAv?qrGYP@v$v=U?2zk|>zSB%-cTwvo9!%dWz+@52V
ze+~?xS*|5yB1dO!8dY0zmxH?_o4wB&;1>BJkM-RrJZj2CrqNf28}UW4I=s=Rl?r?G
zwU?C@UOWIv?=sK@Ww<9t<a`@sA`H4KqxWfIh7%pvJ|6`B062D*F7)e2*Ffi?<{*Ag
z9%P#q-7M=TL?=8W$Q^i%74|In%gt<zkFfx@^Squ9PK^X)FBo8*z0K}Q5YOMbMN8J7
z+wA=%C+3@24lSI=X4O^nWlKk``GOvN{N_WH)@FhdGkwPNCUW8HEn@#qs_ZpQsvpCl
zHmiL;GmX~$L%Izra_9xk(l^FB2%rh?X2>QFjsGquT9FQc*(lTN5p7RNloR~T63M)J
zXHQ2)(GyiNpkusmYFY`nc<f$2W_W4jIUjg8fc|no*yIKamo*`zwaLK%YG4j=mdzv@
zTw16(y^77GV*LOlUtRMqHLReh=xFL<g`oS{(5J?L=^6|Qa6}=MJtCXNykI}coUgd0
z6N-#~+!E=Z%ailZ2F1u*oG%EMa&(fhl~&w?s>TIQR@kqi6$VMTbfRHDG5^GnPW-Lw
z$bEdlQ%6k~=9uj@;DGIg?%i^UB(edeIro7~4waEi*N%{bPBzVWu#dd$rQ;}G81Lt+
zEdL6AdN2*<^z~>Z(CR!pIj`GqAZwOeT*XE69-iR|_7fMypDPzW$$s-$?tgQC6E~0l
z*I7oKxTB@m%<l-{ZZDk8puJTT2`w@AgMVPj1i=0NFWuj}NliNyb1uAg(5qw>;|-P2
zk)a4$vU5xmNJYYp)m#v@^?ik8%nX(<?@X}W<@&Wi^q|nOGE!f}QT@(PS7C%KE`d22
z<rFT*aShv%%r=Rt(pxqo1{Uc#u-esYE5};IPy}AG)oSvI=a(*&ENnK$elq(wIBR-Y
zT(XP?4Yv9U{Z0mz*snj|i{nhD*kPEv_#k_1;?h)4G%cnsFcA^rkh=iH&Pnc6@3ir&
z9aF~eU&1dJ%vTD~UG$#h>?%4v01ZQvjbyM}mVuSQx6JZU+QXwQjIfqR3A|A}(SM|k
z+xqsNaG{JNGO!+Rc)*}Yy8nDDV4F^zS;t{s#CraL%cwA(m!0ivL*TF)|FjK9KiB1X
z6imOHg1TXF3$1;SNhH)^Ae;`J45L{EyMMyCY|5sFO8Ao^w&dA-OB%0BkjbI%4Y$zL
z-e}K&>YXo}hO@L9h93&0vdVHRHD$-q8;SW(1+qX(O=Ie?ec}>c8mw?8oAJ&pReJR{
z$^{7ciMpGv4FiOY%|fi+_fsmW8wXvSvr?>GrItP0Jtl^L#$fSdAAkfTwNPvLsa*jC
zsh|g?@#}!Akv_)YlmC#R_2c>D9Yi3;7e}h@I#K*L%YOW`50f$S(EW6}Yv2Z+b1~5m
z@ELl#jA-H{<BA$s99DHFBB+~{ty@l&rpc-R`PrqNp>mR*G`mBs7S#}_1<%hm=nL}t
zcL`W*ep7h8cgwx~k0hWc*g6uO<;(uS22j{1%5qN-J@6m9Y~qvOlW62-)sWkE+xk?v
z#5EuMT@wt)M1e2PenwDky;UN`p$#!jauSr`0Kp+MLiXat-rr1(tIkfjs{?R{A1i0=
z#P>gY;EgiFs1(n$&o<^ksP<bLZctvPu4kC?nD^v_D2e;`gI>W^I0%BX(?gVkWij$b
zoP_8}a#HuTc;?CPms-C#nHF%~8yUZiIfrx%q2_^+#^DNLOUSn<nCMWP-%<&9JOlje
zH!@Uf@pmhS6k~ilK`i<w;@dQTJoVt8K6!fy!vDl*dx^dLztc&ig95ZHRYboCEPI|S
z#q$|{geb^Skb@EqSs>$2^^H6u4}mxQFcqZlLCvP=2Lrk5i-79wIP6fw{`zaAAng{^
z3#8T$-wV5!7(*@5Rz8#A*2)CrFAjmIW+H$O2={!PyF@52?D*HfWj}*H+hWjGul8qk
zjf2ohk>7ERm*Wzvc#1I8z0|Q!#KX;VfXYj3Tflz&kHe0v@Xb#BI#>y|UFGx5Wc8Ew
z7)Mt%-%jN0YTZ3hH1*KA_mqGO9(wv(H2>g_LO%*?&-~7gX@5U{3`F7a^o7FH4VmK+
zEvi-m0g~Hff~;i${$%g(E2NPa((%9-2<V`X+nx+V=dT|nfLH)4OR*wRE0K6<hy27S
z))wTU2VZm$a}Z<$0U>1GZMB^f_0@4>KR10*8xcpNJyI#`vwA?@J8}3yF&!Ygk{c$4
zRs$ckxbsgA1v*4c&~ZZ%;+-OXAih0!I3T`e6qMW^t{TqyU3{NYA0*KLNw8QO{JB!Q
zOa8Vfi_0q<c1*za{o$8gl{<nfYOT?i;Qp7$^U^o9xSd)hjP`w^@mu3}7r8<J+otXD
z)l=25&_Dy;g(0gc&_oDU>h@k7q;jpqjT{?g@d$i7D@@fH13a=(x9N^6`q7gzjS^-i
ztXo{{<?fSD>qF~yY1O^*W!ot1WIcx3ZRaR8xnEocT-@m)d;p|q!tu;~GMMJzr`i@c
zZpt_k`ItU8<D`-+Cd)-o(z5_khSJW~^}54;mswNiIeI*lq>#NO6iM3)kc~;wpCMr$
zAUnk~O%6#<kVWyX-*^Vm?HNu`0BC~PyP1^XlLdC5W_dHN;{QhWrc#j2nQ?<-nS$!H
z_i^g8nfKs>>tw5$sfABefl9^hPG<<pWgh>RSb|JxCh_`fhw{|F_VwOL14bHtk*h@H
z5T9)Z3MpGF(3re2T6VAwZXD^lLa`4NcB^rU3m;mk^RUxcKXAXW%(4txTEH|{m-SJ0
z>@v}JDc61+-N@Q3!wYNM!`8OcF4mcbaiugaW?1pJncix{hoei5`E3*{iL>o^3lwHs
z&Z@sWR}8w<#8)=p?T!->8h+r~tbgKbZkWvc*u^RlPC6LtYR@kR8QBz{Y%)06+l;6;
z!`vO-?lfF@WQ)E2VIhYQoRD$b_Oe|LF~1nc$yEeuX_`MB042SStm@qw67Yg3O~ShB
zXLwVzb91W23uh-TPZRP}I=dTmlMXLZ?CDv_X9Ea12%bKwSu3F!hTydP^t+{j)G5pW
zVV-@l7KG@FH*Zk2Z=a2R0<3WjN=nxeCkpDj1B!k%tDj+`B3@$_E-e(}$wB?T6Yui$
zx<F!5==J@bQ<edSfnBJRLy>(08;dNyr1LZ|(#Qlr$}v`G&sQ4Fx`4yeO#cnqrwzW2
zol@mlYZK<liJ|--Hm49*$M<oU**_6qHQPQc9>w+997Of4OpG2M?_7l-PDa{CIaSZ5
z(E>l<wnC78;5(8zUy^p`IXYcb;2SB_b)@0Z-NV<oZT7oMn~wb)mpNW@u8~|zpH5lK
z)v}&kjTFyjZdK;2C|+IS8dn-i$%v?tQjBmJ&*~$HI6WMQ=_8?^iW$-%Vx{%x{)S7C
zxY6O}=?6S}WllrK-P~30`ow%+(j=_ouHz^D7V?|pMaDhox^dYBVLJQ_x9(w6*470m
zDU`2f93q~wda<_JM#)@$Y#_#Hl5@#+r&9|=Ngb#6f~N3Ov_dF-GeKj(Tnp4%`-vph
z)q5!WUZFo^T{Gx6C$uv_JY6l-_uQX45_OcL;0vwc-sfjp>xp+HV=7-C_srA)6C-`w
zONLlFHwPtbGqPKB*cdj}6o#~<yLelSZ!Lee2>K+f`6?#;%O&VrcJN?PS10K+f~SPJ
z8V9jQz#51Qr1)EK@Z}E_-)+n<$~MdWVBUR%*ExLx3De$pMJ2bmAG8D}_rMntBQHr2
z`%AN{i)3CF+!!{~S*M-&o$WVR`ka&(0L1ZisgtgHR{0q9Ari<-xY8L*c;+L?wyU#F
z`X-avPE-jQCs1o>@!<x|A2;QPufDKY;V!UBErefUB0vM*)axq)uFRJ51(l~vX5HGS
z`FcV;V-Mqd#liTq5-*Qd>B1eL@&ZuqhBN%Ag`F?H{QDtZZV4htWnmEJE8B+*f@$?z
z)oWQY5PY)qlo@DUUcvVg>9rN`nF%2$tsZY(h<ak5-EE;`nT)_DDCjy+XfrAZul9+^
zUeHfX#J;4*0ONTA5RmB0y&1c}uTuK>E|l%rQ=Q`UTh7Ce8iURMT(l5Tqbf`Vja!@L
zuo7$Sui)K?6HLbp*Uqxu)3k&1@GZf~qyX_o{kgB8m~xxH&7&2T6d0|I5oNU3tqGFi
z&0x;7wf62`lYoVt(c$%i^`AFmM7&3)_vCLUdXyIs;rU5#y+Ru*`vKd2)i`eLoEl!9
zi46`Y!uroQxu<JxTN8o@$xn6?Rf`qFg?d@FCP|uLz}uSf%$nzk(BIhD($Uk(?k3K0
zdXvKFH>eW+V6>l_Fo%OJkc$ptDDC=o=GS7KlT*oaMY0eLDj=q6B5QoqXy@DwR54GF
zBj69#Cw)(HT`}Q9>)%!*<<Q^x8C2T1;z5jZG(Q>~Qg?OTz`dscD2K^k9e9sZQrDd^
zw^_Gx7P_De2yZ`r6EF)d!r}8(l0Lm$y$8*#-&wpKN1*`0nYf0HV>QjE5C`s8=nrC1
z;X|D@X1eN|sgS1=7$gHxhP89Q8wqhG#gCa9tY1u35=Go;YtTAJG@eIVuMyUDI^Hhq
z8@g_YD^kt;M*z)o`qE99j>%ttmgP%iNa2_KGhkNK#cF^HCgb$$##&hnAc;28194Cb
zIUku{Jm~-(X7Uy%NkhkO!<>Y$+f*b8p;3W;a>UBoVUhC#5G5_oW53FlHLvGdWpe}i
zX|5}I)D!DI$Lm&8;z(;cqtTcNOoWTJ4xOHcs~k4$+O9KiW{S>%zxMBzPpfkOTJIB@
zK4xo<Pu11OW?|r<<=62E$2_`%pctb~M20kHO7dfhRfa~f@2Gwi@d!B4AVB}S<uc+m
zJ@tl!Tt$56muFnQgUJW@HWO#F8}@u~JLGQU#iYUHu!wAQ{O@fAPaJ-&M$9KEGnD@2
z)#HLoC`iQOKzbu*so6z&l}~cV!OQ5oy_EMrMxUS$m?`mUS6DYmZktZwaVDd-eS(iP
zoxcE!@2;uvWqc7uZl(QXfB7Wu`2u4@a{rEt&cj3)g9p+#662)pzNw1U6wMiB?S2%{
z0L<iRNLs7?Qbn<0SJA@_-J*obmvxq9!v+>*Q(GTaCgW$9>@UHKy#+-xA$b`gwA?ig
z$Pox`zU-}0G!6b$ZQHQ3dp_siB%r5cdPi>M4_(&;#XWX>el5`NmsJ1N(ftF+=v~44
zu2BriruTkY<n#w5iFWCLGaZ(`Bd-RnYji#V3H1@(9z39w2OI|Ui5zN-)g@@$j_yif
zE%`MvJGGH}1-CnE?ia}!f0w`$backK7OO>7G9Vtqe>&e_Sj!^jvibSB!ap(quG!&k
z&I#7BY)g17`K<*!eI!Nl(7|fkUhBdZxZrecRnGQD3?N=PgObE&>tt(;3TrkV0-Mgq
zKJ%kwgJEE~(V5z5ZJVbSphq;f%i-tb!+F|wZ*w*MZ>+z%Ftj#q)*C;sUC+vPa32pY
zjP(6v&1DNzO8`-Q=*mO?tbp)N3!lA=5GHcE*0V<h?9DBl`(G#rpHGBnU&VH|&~tp2
z@_qn8`W6!{Xa&K&F7^O3d3J1F!ON@8F1eab43R$O9W%p42?6!N_TP?Y;A;jR6oAO*
z<VW<y-NoK6U6U_;Y<y5u@+g28;3-}cciVj-uw~^gQJk(Ux|yRvr1Vw!zuFqUf3-Ef
zCWB3BuyyuMy37lUj`8gNGuF6?7IB9QOCmJiUC*N)lm&0k{@);WPHFT5p}oiXZ?8vp
zv5j&bQ<PhT>iF-U{(lHXJPinNVL@xfn+fg~ueE=+DqKIvC=&32rK*-(T<8{IdinXY
z+@fyE6YW_JQ;MtQ#{_#dv8^VbhpuM3j_15>NtwBHZApIIE3OVk$W6#xHDnJ1o-~XB
z$fn=3*u0#-Y|Dxp^CM(CH$BZAk10{K==IwA)j{bvBp)y5oJjEPkzJjaxl7;vv|$W*
zbrk}ScsTrensPI#cHkBINh73BS2hmfkhc{ZBt#Wr8ucTP<Hj5XwDB8|nuXhwcD*Ux
zzR^ggsfi6MCLz+dVJ)57YtU)@<T=I&hGnz~`Uz6LQ3aUZ8N8_zLxBZ`FCY_NVPyZ-
z*Cxn3?HKAu3DSvoKm^V+i_pol&>i@R{v65E*l={a&bxK~1UB7~&wf(UFqao%jGe}V
zs+`FY@dfAxf%hX7-}}YuU`N&HtL?hZ2V#J9qFPA44_JIOFZ{KDfYiNDHx|<AuRFHO
zht>ZkEL>>@73?tug+0B^0MonAM-C&peQWmixVMzR14JT#L0+io@Y5XmsX$h_OfK}e
z?+?4rHQl>)AzNj2lTZUf8~7VtDb`PLEke?Fbe!m-vzddU++Pq4h>1UM^qFtiFfc6A
zESj*uPk|9eZrkyezX+U@$hGrDCdr_M&$KmT$7J&<fyiXL!1NSJ`d^PwyzFP0ONll=
zYn89N!VW}j70q~}D17xQUW}EIZvF-As*~>fZHB)SkDjyNE|~OP54iIhhaan?Dcby3
z8?g{o(!d0LU@DdyN30Ne0R(lVrMzayUvo&RoxtqwLh0+C?qR+^vsyWl6AC7~&bt8^
zK6L@p9%TS-p@3i|4p(F-YKAKwK<_z^(gdg$TtKB};1tm{`@gS#U3-$8Lual9EkoHX
ztW>YtrZDS!Ri5pr2FsEETfir3j_p=$>_5v=4=7-!FzG_9f;WXwXF_?q(;J?L$P8({
zMDx~<meXIuRS!>AlRc1C*xuBA9~C4c#EH;}niV<vjDALZ5LRv8<iQEqkvS9)Ht!}T
z0}<R<7zTm>;zcq}sRJP!=cMVj<@Q=GJE;Rz`}=aMD_T@snpr<>p67FW)Zxpnn9B<7
zqvpS<Dz)8dEMzifGA+`Yp{U@IHAk0W%~H*D_>6U4(P9)}R%{TWfulOE%-II2ZWq#|
zSmkA9qT&vsRF*I!u0_m1LD7^Mo&Mab6d-mwG?G^m+=E^?HMFPiWEz_pDgpg)1v{y!
zxF^0BwM$H%oQ3U?AjDRL4_lQYYrXmKt|%z?m7LEi0iR-2#e5m*uMh(EfDB?qO~KlQ
zB%%bvu>0f1S#D}Pq`h9_U(EU$3~5&UZM9?Bxj&PW-qV!7MNDw#3h!-ymXw5cSu`;D
z3SITO&>0QZ&h}a9clC40JG=x>Eaw+C|JZ~zjkum?DNn!wyAo{UN0-jR_l5P-tDcFw
zz8RuoD02@f9oQ_@1F^xhk4mq^nGr70!#=ZB<1_cKyt%~uWvov(?0cSOTDx0?63UkW
zfJ%&HeJS3@Q_h5k+gGM4=M^hM8%3o}%|eJdEF+u_zVd7piQih~{^j@NI0ZEU63I?Q
zH4qtF@RX_{uS0Qn>}Km`Nj}oelX9jMSI(lzA*)MUWsYSJ6}o+qis*i+eB1_q<@CwB
zYOiNS9Pb)@Wqmesdma?C;qE*4ChQlQ4H7y%fcP9ujHpZV`tW}hWPRB9{2_q<|GOWl
zq4!Sj{A{r(+Bt}-V0$H5<29PTY^56{@^1a-d+Ha*zl8g%B=>bSIf;Qxu|sY#{XD>c
z8BiHZp>BK!-FyfkYrW8YZ|g;!&LFs;g;eTd^M7fbT{ZcLc<{lRP%A~KG1*7cbx=Zc
z058=Cfu?N!xv&pX!9{|+69le`j6Kxw_2Kvj#*f-oi88kKZX(6$VK0){c;OLSWNNk+
z(6b~*8~(KS<-x(!=U=iV_C$3(hctQQ!W5$gsc_NbIg~^rolWqJqt7NWtd}mutP!pY
z%slhM)HM7^m3tpw`IcsE8LtcdqW+nE7AE-+elZ+OG2UNyKO^R9pe&toe7-g9Hyb#l
zD*?jmey~~Nt9p4<H>gN0y|6i)VD{x=O$C{4w>{O8L1s?E7$1C>W4b&HlynyooSVyU
z2U5F{=Cu&ac}dhg5^HDKdGZX#Mm)Wgms4k`P8(O|ooZ<~S-5@7=VWrnq3GUZgWv_F
zk9}{lQFfi2%XHBd1bL+IWd<#5`iL60^1<kA!csx%zkB1mwRU7ME4`^~I_kAK1S0V)
z@I?GjANz$YqJR<y)9Z&fe?k?1?x53&O`u$t3o=aHn!Y?B<HyPt%{X*yxrVvzmS*ba
z|7liR%g?)#JNJ{-9_`<#E@B!L?#y8RK+{ZrPqtG<PGVu6hWaOKBilL1qHg@)_G`$!
zi0z28OwXho7NX+EYBD0`p&aiu&x+s0Nj)!CmpFAjz7Jpf$%1f@y>rbCDF;h_{Jro0
z<{rf6>v)5oA3yhq(KU}f!Kb}tVn#7oLao<#Twoml&2D*(TKe@_Lp%2k&TZUqJg0`h
z83I_b?2NZW8=h~UPMA~MV^IYj9Xf*4a+k#hy1BgUr^H+Xkg1xR#XW%1k~AeV*!q0$
zjbinps8R*%TK(xvSyKR6kKzmpAkZn3h;j?umIm(&B-uY8&!^|j+l0)fv{F=>a_?R#
z)|gVvQG#w0OAvw7APdLs)d6-i@(Q%P+|<wJ>0HvHE@(b~0r?VuC{~T{_;dWJ(hdVL
z^2Qy5_MqnBqIU%nEew)Rr(7Zj8az#zt~GL0NRplx;RM_bx5nxGv!tpij5Fp@4upTd
zqH`D(EV;EClZ7lpWPugF(PP8y{9k$6(z|tKZfk>0SZ(5a<mS4WMZ3S;nB@V=D*{`N
zb3e(>teTF_jMMG<Y<<5kQMQMm%DO+k3NJgHjp_X9_;@iZA634CT_Swwdr`8G_Xr5`
z-%+@ZKg-EaRpnTWH|hm3a-5x$DZq~Ap6Lt_kcna4{GiUGI$ckE&g2lVv)8n`$r20E
z0AN%NaP6$gc&iyR9T<$t;kRA7Gf5j2kA|0mPx>Yzx8iLv<o6n`PV!~l7TfM}Zs$4Z
zRt`V-_Hs5LM@49(FCKQO)UvI#nlZ{~ub-^Gsh3TODz==m|Fp}Jy*u%wBzxGlWygJF
z*~5!v)UrLobC?4z)s6Ui-8VM<krQfO-}lEkChl&sbU&mU!f@d&RM^*lQXg5CN+m7n
ze4g)vIGL8kvE5wpTNloVuK4A8*g%CEnDrC?OD!EpX!E`homJ3q%=hzkgRHJ@Jk7Td
z1xJXlMg)AL(il@r&(b$<3ldmf_{8Rrdp^3CLwvZLV|p!fwF8M)0M}nd6Qt*TdtOZI
zP16yM>Y^$Z9bYC-67hx2@OGd!+`eoBw%|G{zns0pne<O5qaAG9;$**VvDNt%#`XyN
z+05jg{kLKc8x~}!wY0V2U{F$u>&OM`AC=E8JM{G{c{GtOaoX2TMcci33}g>5owLGb
zJ%{;eXl`q&PF#&0?JtHh>j{coImcNI{LJK^sdjJJpfJI}o{`g#A-)$QW#Ar&lJ>x2
z9P-QOe0!E`V`TMKLzR(OM&7aFG^VYEN@McHC89xC#mSg^r%n*No$zXJx!;V}FEEj3
z_4#h)QXSafXf068xBKh`QXMHfP^?~1y6g4l{ICm&O*X%_7EURuz`B<>j5{N#-^$JO
zkfrNabmgID>?nMT19w-{-cN{a!W71d;z+4{hE9*IH;!`K-3`uf+}6Tpe0b$A4XYJB
z2#vZn_#5;RA=-_0IyAgL(tz*s-Q+kYeL(bz)C|Xpx^p+GGQ^)2cvLz5r7KC>wEdOS
zwH_7j+ex@<kJ&4eqWT}nN{v*z4RU-PF9X8GjfNdwyM}$;5uSNK{Sg4!CgGjwKa;U|
zVzN}vpZxvV;s$cc)XX5gi40*^2=av)`<|zgMYwK^j5%PZy8`&$NZ~J~r>X6^NliKX
z^ZrU6k6aa45A@v)4W?;01u>xF-uqoeD39L1m!(F;?&|(4pK8-<r`oR;%(dScjl1nv
zRj@MaeoDllAJ3Ox?NX!JUtx^#YFE0UZYE<p6j>;=N_^VV`gAa|TkxElHW0+P_3d~e
zQ~^Q4^sxrF17$yaf$CtfuE8}_pI+MA%bu<JN67i|d(1i^sMAe%mJ*LIkA__<3(g%s
zP?f<3)8~!kbkg^qK)3un`Na3utqZyi!s&4a6JNc6RX46SSk;vb+<YYPi6V?Y&E$r!
z2ifA5VmvdysPPIT|G1u1`NXrcp92J0tlN?IClp$hz;EG8rto#1AMNG0-8+ZO;Q<bV
zbq3=!-px|!8P$VdH}|>^y^`2uTz~iI2`vnc!+&Rc{e%iU(fb*(6r{YtZmXH($)#W%
z{COZDFj&Kwq74Z%96wZ;-CLqyn%Q?NF;N1u2&Kv|to^S35N&a>_jj~zi3xfGz2MV+
zDF}&rLMOF%56l4B3s>F$JAYeQX+M+9*n2|URS8O#?!`rD6I(L`a~l={LRg2F%gM*T
zhzGySC4M3zyKpc)=*t!JoELj>igqumKe`UJocb_ntA*>}!|><dPyqGGxt|9)Og_QQ
zUr%eyR$LDl8n>gf!YHRHKO-t9#@`nWNjVd|T1CYl$ieHEC!ht+0qtm&zMofA$}sND
zFIaZ%Y{I^M*+}sZuJVy}SKnksT}@`*P;Q78V<Q_{daCXKOe3{T>#@q=linY>7EFH@
z9VPoOz}x6|{PkfCULRYJ{ao;2XY_BST_;dM*pp2MV+U*hgoxVn2|H{c>AVitg5Q1{
z{SL*93*&`55^*93iG2>skL8O`@%1F(l7|YK#yH|0z;d>t(V9XwgRHuxCp5ix$q59!
za}+BWwR@u;88>>aF+zTm=dY%F0D(OpuSgcz%K#{5D2r#e5o-~ncKJ;14_Yfn_B!vm
z=`mA5J-7SXpQ?qB_5ja2*!KiCa=2;jz&ccUH~whOz{xA9^F808EY;DH_o96+<E+7u
zOMYrlv@b#iAhGR`KHurf1RAJkw%ZUnNU3p%NkT~8Kziq9!np7|7J;A;Dj?*o_tx|Y
z@!-?E%^8j26uJ49OE~p#kKm!a2!VO0uGgr?^F0*@LK}e8We0Vrh(pc*NktoK$eD?k
zRt2yPoYW<F{%L#Z>{bt*uz}WX?b@8!I1X+%kiZbm&I^>3r&FekGuB(=Gpsq|y{gJm
zTvxGJ82(!H(7&$^&GQR5M?>J_B>HDb`D^&^*Sed>Z9c(;=LZL|T^7-&j3}3-E7FAu
zPWkl9W(^|v{ImQbbWLs3^zf;|p4tUoPZ5-Kn^iAvvzsix8H}K(luLfs1Pj)(9r2XJ
zI**eye(9!@*o<d584{*o@vpQQ9TUTrdL!el2QJ!-;8GM>c~57epRT%*(gI33Z_2wU
zwpp*Uk0VaCfT^uvuRoip$SsJ^eMpT0O?17}h_ya3z{)z!)0vLiBwB7_0yS<8U<kR(
zm#%mz;Zg8VS*jE4tD6ZAebHh_<Zbzw0gk0`W`O4G-fsFZUk9fKt*C^Y-co^QEe3)j
zx={k$_vB+p57}e#(pyB`UPkgQ+a1X)h+wVo4e>l+!8K(jC&d$Ac`p@^f%=;6HI8B~
z|26fbRFT(iSz*a>B}TO9Y2Q;t5asOrS;;NAsdR3GCqo3&)aBdb1($y65D?|&cD<z|
z7PaXf$9MG!KkZoLdt{gCEvh-E^gbR8S5yy7EC<hozC_4&k2y3H(k~MYzF9x-R-p6$
z@-@t7`~$+P3DHP#QrkHBXJ`zo#rtU1{hb=4=sx5OrpH3iJI|t<A6Gip1qUUGBxK7N
zKSbhuJ7#J?op-v|cB^J0P_uM0K7AVeHMr8qTZK}$g&X28r&_2sP07G1Bt*ad7R~xG
zdoDQ?sTLY}ujeiKFtY~F)bsc5b~;m#4E}DCStcbX2EC7vISBEO!Dg1e>WFaonAo)T
zr|d%?)X=J+ViMbcN5H7QaR~4X8|r}=^9HB9RvEhJ)GP@Htb3DBYl~uq;hsAP)pfAn
z<qVa<Aw6YJ`MXL2Rigc*KR37n`vLBdIT4xB#*X8Nx!%|A!#lvq<^BbEuofYm=61?s
z)X!5Gek`Cw^^IH3W7;iimnc0^cvY9WDoNI-kZhngK@e+5P5lBCKJdo*5@>6POdUO)
zG9w&3a!*X0qw)>^NhW4ODslg?7s*e2!AaanoVHZnIhaXpBri3g%m?mdh0E_foD@jA
zT!f<5n#NNG<5dwxDXrg`FZOI&s%>Q_g03ydz|X%r!XEC=uirpC`LQ(^r2)6OE;ThU
zfAs2S>+e`%cy3W}PeXh8LuF%%v73!Wo<r7QXp4;=GVi0f2Nge47g_@D59>w~G&&Io
zQ15@=lsDcV)m3#B;j}pOpL?xq?)m7Vn+<Q+d^YiLb<lf|<oP(qhxw(_Hw(X$e*k{u
z1x}%aYXHW+tu%^>U=t|%C{8>eb9FSV{N$WlIw#9aZ`opxGhn-Y)v<Gb&@A(Zd)q_Z
ziJ`(56<oxF#aA>~`DgHbnfV`!hdaDau}S4Px7h)Atfz;g)cI<c`-UOL)7JPZqOM;@
z9|C=Q@ZFxe@Ic0nJRM`%Vz}ks@L(;wiQh*Zz4-em_R@2=Z;C#?55Iw%y*?{Q_0_m$
zG9PF8I}o?nr{iWaVDCUpl5^8Z%V8)=`hg|1sLZXxu(V?2uQ-dgL+v0dF<0a2Cn?rY
zr_Ea|ONRpSbl!l+KP<@LVIb{8&`;1?$+NkDX`?<&(9g3)Q~pu==j4?-pOHl>ug>OT
z`1Rfi{=IzePk1u9@df>joa>#?r{<u3(?H5~$i-U5&Nx$y$dSYDO#L)JRObMJ3N-WK
z%6a3n`#V|=Hcxot#k-jRg|1=VQRa!SH}ksa)B`7jw28R$5~9*7+kwAnf8^B$YW=J(
z>Vy#6R2i%6+27=q%%*&H@0?}jD{E=qO8mH>*PW8(;Vn#==#hio73s?x>H7D^5&aei
zw*VaeJi)zhRfW0i8Oi)!BQAQANjJ0npZsk8S2d6-W$DL!vw@?rXwZ0K^)Ls8HABN<
z;`?PU=G{ft0_DLD&*G@JloaVf+TKO&x?TWPQf!H^d$osxt_4}FqTw~#pD+eclvXOY
zGmP)B5=y>Xb3-1#^}vtalh=w13)~eyhZxVX!KV-OKjW8e$$6AAjm018RsG^kX~r`E
zY-OMTcEa_~M#QmUc_EPvqnI5`dK)ztzZbV0MJ`e#gZKP3_Q&fr)@(C1##viR*jlyn
z*?_DL?daq8oH1Qz$Fu<AgV+RU0Pk(2d--txeU|qv?BcsPK9)1|sK+?tQP-gORzz9k
zpFG=NWZ$fOD&6{#9=i^dj78sN>R$X5Dt;A{)wkW*vr|r&&wKVXO5)@Mzgp?ec;2mb
zvLIUxeFhSd+-goP@T#S<ME#1U&MT<fUo$*em>&ODF*w<i#^kU>=)E@-<FI5uz+Q49
zGjBj7Kzwz1<>B7-$xqp0UHwM7Y5p%^4}H#pxAPK)C7I8c&zpf=Nsik#IwPOhC99zL
zqcC)`Kx@w2VizLZ$g-(>?$#k%JKIp`g!OPM_NHW%0M}%0f$V8Iu(Uzj{<U@G?H9?I
zhpPsT;W~PPMJe49G;&Pa{imrXO`8KNYbn0tuhHbxy+_qi3ooU2wpZIFz&Dh<OL*mo
zV7o0RiaetIGZe4veUGm^nf6ZnKUPxDnjBMZuQ5lc=k<L#@mDPLTG<ead;dIja!pJ7
zqH{a4v~auThqHK(!)v!`m+|UufmDZGwbPT2+>*1GLu!_;jxrlrzu)(X4j7n&B?rT8
z#>3sq>cucU)D2hbQ0dj5=)241uwmD!{tjpvZPM}{!yD$Hg3?ZpAUX`5q>{R(Z-xP<
z9<hUzb~IIjdAy)1Dt%@NOw!|a^*?Y;bG|PFlP`Kk2E61v?as8soYrFE0&}D45I8!P
z^iP8LLw8JOBvI6W8k~Up4M~Eo5rb*Gk+<=q*B;M7l&oJ^WDjNm-NzPwJjz+MJ>i0<
zeyw8!W$-7hWp<yKppu4JpDLuqxg3OS{f;Rt)677pL%)J<A8-47%U3NmVvv5Kb-bMF
z0r_~kfmF#WdZm-Ry`BxWk>06tp7DB<7?k%+?()qEPFaEs<!#oJ{as=e=_}Rb0Cfw`
zFOI%ypaObrDIuT+5c-#0@5dg|-bAPuggE_~@6lg;$!3vUm-b%oHFCTTMC6=Wzu%5C
zTXvu@q;{F}w>d8vH1ct<g?^|m#O+*jdLWl{Jzjv+W|rs)%5D&dq_c8X1tcdo)CN#n
zH1s(dE_xX_H51%OX}<r5;5Qx1$6~@gn}-*S!`)yZv*&9=s{|otRfUx?XxF2kf=9u8
zU#O3YHR0pO@U7yT$`=RqVKjG<Uf(zo?IF2nxX{-<&a{=AVCnW=p5I3jvD%ZmgDSK<
zZ@yFWehU=NJ^E6?btF+fIB29CUk{f&ZY5<wH?s$Q1PM&IaG8cY!H4i0PuTmGK8ugZ
zr6EO1wq$6)FOnLzf~ie|3<Q_se-IhBuKL@Q{A_!<ui&$E){%y1O{@xx>3JR;^WptV
zkXkUZ)8qnJg3^4pP6Nc}kComxvb3E%D#Ja>u4=6P&eOX&;U!<s+S+M@uCb2!_ejYJ
zL@t|`BK)Qjw|FmS=yx%rMD&+$*~`rv7S*Iam=*aQc{i_wZiyLIR#<<PbMO~FxFb&u
z0>4p|j|-)$n{ZHy+xH%q^GT&-kZwKY!D(rM)B-G-Ce%8nU_Je_8kb*JFR;?c6<>_e
z&v|n9?Wn8h%XcD0u9@JzWIK^zE4ATbok_p|0QK>3$*MUDtrdF-hy1FslDv?4NBrd%
z)7Nu((s>}Ao+s$KIo21)H{Fa|4yBG0@=VYO3{aqwWcNy7f_xcg*MT~Swlr|n(~=#U
z^~3?82=BO?G#pjN?{CG{;DpctZaY($$V1sz*%!CuSqdVJ8xd#oLBX=r$s`9|Wb%3d
zqd%N1=n<-r{F!{<WZ5wlHSm9)6V<hdsl2&(29cc4dM<$+)WDr=BR*7|xRU&MzD+>8
ze_P8WOF-;O4u3vS_-BXornF&2`Wsxx$Bv}j(q?|J;(AaaEk^f;51h_y08gLySpH#7
zkMIg!+0mKw+jH^zzdKNNup-js{9BLfU6~;4V?S($ZdSge;lG;XJ*tVHeLi{HfH;TJ
z399{#Tx9YtuVFn0oIj@MAn)i^e?|K_Y`ielZ`Pa-2G(NPPwT|pjrEla2$GvA`%R>G
z=gp^A@`QzfxBXji#G}F!ANhVCfJD~5ph2=fPL1wf@cgDeBbNtjD*_EUG2_}*h9-)Q
zX}1TX`ZbL@BJ_bX=;XB}O<sBlpON3Y5BYxx->n6lf{Ywt(yhnP-iAS{!&F~>>ha$y
zf(FQkdpsk`r}>2G^DZEeAQy4z*Z+sIua1g(joKYTK#4(-k{A#H1?jFC1SBO?N@_^y
zl7;~W5k|U1Iz>@ZkOl{&ySqE2hHkh&j_0g<&Ue3m?)t4YGpxm0z{{@p+0TA9y8ftv
z|JdqNm|!umZl^PM?-kapVb#5fvI)C&pOdks^>^DIH7r<j*ghxc0F5Rxx3Y)pywDJO
z{O;kSZu4kh9nVTl>i%B|Eqbx3`+kokOU#9eCO|1eC|FeJk!i)6P+E_B3ec>V^-v#8
z8y=m`ml+IM#D265vvrXfp|3{aFogCwM3pI3LTbL74cuF|{f#6KvpMXv6g<i8u{mG8
z-RvqK`#dRpU3)#;dV(!~T;wcJaA}=uW1<IER@fZsoQD5nTO{`6{ce9NNLMi*z18fK
zR2Uc*C&V^_I9eKYF9=!PAJYAjTIzIUqf;<i|J9-s+}13H>D+%ckjt;+6FO96#tQB}
ze*GKwtkd>W!5Kp1jT)h1r;$s!K0--rYDo<L4K<%p=*-qv3O%ifr@67Z)wbU?_v*5G
zrd(gxgnizO&e?o?Z!n0c#B!h$peUK<<`us^>zmp3Ib+HxQ^(W!*0qR9G3*kUs669J
zkubb5+vBwzb|4v%(`-3j5|mq2^-;jOEOR3Kw=VuA_2$S+jWekODNvZC=pwKMNyz!g
z$5TA?A1wgJPFpirJM_%PIJV(#F+G~=_oe1&aN)=8SE92`9LYHdPl}xRMC$@e>A<ju
zWw-d@)lUIIv^^gv_uf+}VJvr2akr*3oW@LUoPu?Qc!IG{?7sUAT&zgvcM_z{l5d)W
zBMP~pPW5pc%ubK(nL2S@gCmIE;{O`alQ4m*t~an&b*eh`p9`3G^?Y-W){7@>6M!uy
zz8sB1pp$+J2<H$-Lv)nie>Ao3SFDNtz$G%;5C<MPfLiRt96l))zKvn*@`pn2`cqJQ
z21n$dwqmX~+Y^HGS@*HxpYAL{AiJn!s>YUZ>2)=P`>J)<Xs_0WS~w&>C@nv4%Wi)<
z?oyy?L5?khX!VO3ZtsU)g<R7dLYluKsPCo6g4WklNv*cQTX*)>NhKK&+^Dbj_Zr0_
zCUV#X%jrgDc)p2~Y0BKh;vdcOgFwQs>hQd|TcPK_rLFa9i*QQwwiMg!tfw@$U+?*(
zncKuX7zgTwjSmtk2i9gYGb2;b9=Pho_$Z(Iqa*8_(usWAtH~k8!6{{&)uHEw<0<}I
zksd15EnmFGp}x~Cd#zJlT$85fi8p>vR{dbij_$sJ(|sg<@eL5iJ)@`D|30lt_dv^s
zCOZ!1KIW6K7vf?&GZen*n2c58$8yQtcRK0inVD4Rv@l~|On%P{;(Y(gc|O~<kBM^m
zC5PoSum@@ja#>8!O+NI|ouQ558Hhn}+I_*)d+ioiO)apR;Id6XN#miUaHhbpcEd$`
zCMr<SZc@0s#cpfZRF2*w%NP0n4p;-?b8H0P!8Ysw{M{NDaZ}!acq~RGqXxtuFB_o^
z_|9v60NS-TGhL24X%fC3sU$dO<Xz#CgA%#3_DuCQT4^LUannLwT1^0j7;?N-L@D_e
z)hLc+TIozXd4c7R-!><x4QR_I`aWD?UAZJ!tt*2r(jtkjRxllL2`<C0SlbA#Sqp7w
znAiqWQ3puF_Z{c8XIo;v)4U0x5ya_|jndL(!?&E^UnRf|J1scM1?+EHD6QYLxUhq;
zo#g#xZgqKQ92zZAEw_|537TVbI|EIHi{BPrSM0QlCw2N5w=pwHrkUQ+A`|_fl{iFg
zF)V|w$|e?P)qh>q;}dT%(t57Jbf2(ttEiCp7JDoP{R?LYbIQhrH_^uJdwXP43+}xH
z-#)D&&<=x3VyP`8&htrnG?wCNKT{tYepcn01@qW7Er*{a$#!_rN#5`~eNQaUf}8YC
z65BvaoeR6}SK_`w)A@~>TmnflJ1rcHe~%c_6U2rQ#omyz!#t;HJZN(1@|Zuk)rxf4
zqi^;S9+6myos@<zj;W~a5wOSnGMRu)S9`#o;u*eGgx0Kt-1@n-B)18_I@PisEB+jK
z_zNF$?3BRGwnkuczmy-Ter{|*DTI6t%%&fXpx!(or-s^N?)iawFgk<ge*9GNBeL2G
z5hHiWrz#FBra>bPKSSz!?lBswT`3*g-2YCz9b0n+M6qW3v_Npd4!|hp`u5?|(MN*x
z=Yb~GU-{B|ZZ(mlKjP<7w}0iew^@@!4fC<5+$x{Fa>_t|^6r%U_w&e{0fKz(n2*|4
zs`4c(o6lW3OC@o?E%~FOTiZ_0=Yycx?}#Qqi>J5C;jP%~3hADU%2$%`c#w_*t|8W{
zzexHL-eBPt>5Pj6nqjboq#gRbW1P1ckGd-IboWLYPIYe@&FOJTW7;~&m^`KTa5_~9
z>6>L0c3OYqhfsvOC2-%w+U~uR=CcYf>(73wh3~kE+L`+P`rRj~pf*&@VB-!4l&2m|
zk=JD5xY`GfX_x9lu|oP$C?Z%%@S`NPG%|chw4FsZl~IU<$(Uu{@oqpGHw!BToqScl
zQ7ETvVvA>_rGBGHCze7uhw9{bGe@$#lI5Su&N8jV;gFE*Ixe~Q&eZx|ElTSPqXLGq
zv58|X+CO30LrrtbwHve|p0E<9`?+C;QKKqTu9>=w7t4ouJ^1P0dM(hAykA+Tc6#3U
zy5OiSWlmF8!z&)US9~~30yn-OK;m6xZ}{Zces6x8L%9|oip7F8ITt5Q{#@G@|ECh<
z=itQ)bLXcl%#RirdB87D1b)iH4fnZkLBTJuM&7$KG?fvYd(yKN-jY0?CH}eZN`t_{
zen<B9>DSYA)Ynl-anofivqbk^CFVqj0+8%JH??=O_~Q;OYU2r8N^t^dE%;z-2!SC^
ztcyE$%9q6PYzQd(!m1Mik_Fwl`DikcZ87^yoC?Bih@A0<egM!}C<1)(<qUKSwCzA5
z#Wo1dW4V2*d^7-l5k;AJCbgL$eL5#5-c3HD@4orwvH`znHK@Lwc*w)OCvJ)<T)4g^
zy72pF-UeXUGu1}u172^s?!Y=U^rbumNnov3FztKg?X{j?&gP=v^G%8jJ@ZaB8K>+G
zQ@N+Tl2+=igNDuZS*MgApVgn<)og#L8nH*>aKk}IzGkMVOWgxEJ(T7dvLrni#IM)l
zwf)mAx&Z;bUO{-3nxd+yp9<pFC}N7xJ0iy-ssC+pf{1STeoN6-@5m_m`>i_&wE1@q
zZH@QVgcu8XCb@9u+EXtvQ*ve4ISeh#){7BcIp8#;3Yt6L^&}rvZo`&ovZnBmN2a*z
zqpl{*AU-|><?h&>cu3i-VD&Ypo-1L45k?2+!LQE?fa(2d^_i#aRjK4mWjr{9OgIV8
znydArvG_NuoqsG8dJCUGG4&~06YOA(^DB3wMNh{QQ<{utBR624U)#jLMh|RP{IoK@
zXKRdSjA!)-Cwt3BCn0BG$ffI0G2-W$wo6y0Pp4#?`Rqqs6(^W{hh|~HnlW&YeAT<{
zIo{d1mZ42%lvw;B?0ZTElh=HO*<$fubn5AI9xx#s3r0U>_i3=|8`IUb*)Ia3q=KFg
z8KADa0mM}$b6s{#4mMn$V!7J*xlBo+3{6iWpgTj^0l3b;Sdt;U&cYbgT#<IJQ%wJ^
z8dvC}ns@st-^xD++=cy2%`3U*TUuci|Ka)2d@NPU-4hDfvrvkHj{z3X*FeSog%1LP
z9&w?g<%te<7=4fXCo!n-<%Oy?h0nl+ix@S}F39mN0+Yp#hghrgm!iG|0s3sM18C|6
z6plBDe{-_1kA6`rI!aLpr^=EVi~qA1$pGIBu$NUL1fDj`5->|mzY7O|O*6Z?k=Hf5
zh|TIIblDk;*W&v}V6Oo}q4BSUvST+Tu8fjgx<6(DPWlVOWo>VUUupH&X6OS;87i~B
z(}^jyYpgJVBm8h2VLxFJ78ZJ^aOpNfw3yyfLObZ8di>fCH^_Rw>n3jjX#0BvCPq57
zGb%tAhPCyiu&P9P#CD_CmgI-o$7Nes+~{y{ep1eS+;2Yh3_qXvxx><*7ma%6O1<w<
za!Bocp9BLPv?8d+poEyXcKOSTw?4$FWw6zB|2s$$(ei1}+`>Cd5;+E>FoZA*HTai1
z0l8*b)H9A)yW;@QW7225^o_e6m%C+(QV~aY*~%7~>Tr<e?hcytLyb^~!^6!ql$Sb7
zM!Qnt=!;CRqb(6I&VXoV3cP=h@Pwb-O4S9@?Tb!k*YrqdTf9c_u<ml%RN4g+Q5!d)
zz~C`7++vHc)clS(>FYDfpMmYq`guThMI1yLqPx!t(!a4GWqjQ8aZ+bP3g0IKI&ngm
zi6%bc4+3|wEM`X*tAI_&t$vf<%q%A~fQD3|Os1m;$F>kCUOh$8bCEidVcQ%?O!mb$
zw4dtA?9?T*8D#SJs+~J4mEM|`-ut=YIQ=c_sY@v01F#m7WZMZ~{Oyg1jl_cZk=tKY
z)akL3H->VxuqkdGfQWWQa_^t1)fPifzU-1)Qwl#yBPLG}j?*}qB>C>I$gp<ASYX!8
zYWOwMYN0Bw7V!8Q=UDEw*pa{q51ttZrGj2OQvAAazwcaGd%)9Q=rSI5a^UJ|;-=sJ
zp2a4~o6m}Xm{jv#+AlIqyM4Pa)AV`weejDK^X0QN6fwg+N&A}G&p|G4e!irB^dSEx
z{Sj`(m-=?^yjVp!n<<$9=I7ea<<s)-BW1^eHZ!|{d({qO96zW_^NiLqMR2lS$5eFU
z7Q2vAzo`9Iq3dwLde!#M<Zv#H^Rjp0SUbn~-ICFnDYq0p&peID2n$+1Qa&|pJUg-F
z!&|j8T1{fQGWv(dpEaI`iSpnmspH`hSAFR`s{hT2SubR*hTJmH)c+CP3#h?H3oA48
zv;iSt-!+J4o8<7;3Lywz7>OaZ|8|1l#%pCXHN+X*<-SENe&V37x+-b4gyVZ0$Yo%`
z4i$~$rr+;-iC7dzBC{*MoT+>mW$?!o9;e<ue<J6(N*Y2{az9^VXHKc$7_+LRm5*R5
z6nrFKN{oY}Ec1BlnEMf(A`u5Kvo`W}3T^W~BO<e~sn58h=@*5ZqA<dl+L~i`U3=QV
zCz9zU;@l#9bi@f&$F=&qJ9Zh#u><_r5#@O@Ii6W-yI|E2us$L<g7eZ{DxFXyG=jE&
zvQOGq#H}O@bT2O84(n7&UVGT_9k^9W0u;Q-VCOtSaUBjO_t!`4i;(6e^mUKd3(5)v
zEC(B#t0eqdymc>exW^W)$2h;~54e^g8kTJPdt%OqpU#|P!mu*Gy$3~<xO$NZxlPFB
zkGemFTfUCY9o_8$leOoIzD$K7AJgf&mW1bvj?=nH@8xc{OpFGZOeIxJUv~U>96FPy
zz^9w5Oj&C?Kp6uo=VJAJwN};ezU4^#m*93yr8!m=HLt7uLxcNgo@y=9Mj$4D3GjL5
zl4VlL)QB^AQ`sHUAzc9UU(Pl?Mhyb-)njji)h5dA?U9!5b<GVgbILH1`D~5;Ha1xN
z7&8z9))3_v<s?mVrz?gUmP7Hoi@#{Da`z?S*4c?NW-<k%r?{}umlC*MJ{R5otL<M&
zy?okw6309Ptx>3X@rp$(VzAveT(#TeRy`i~-PHRTps@t*l5HYam<BNWPrJYB^F=%m
z&Ps%JngB-UlNH$7usSk8q2{%)Tg^098>#AqyoP{XMqyG;^CQ(lx9w>kcAiCXP$7{G
z<*6mGSTnzD-0CuMskC|h3(GrjS%lFUQ<)z-SU$c-sPNTvdBw-$<Xa%rh=W;U<wBKF
zt|*)Jj)7K6h2>R^tH-wWc*(a~JZ&X(u)bw=VmOwILzmOVz{*eTI@0Y40W4|^wAq7!
z>zEO;<VSD4P-Q-KjMdoW#jq{3#%2_|6nMl{MlwJ7<3iozq(EgU)wvV6@l4=+lrogl
z$<E|C6cB-spFCD?9@eXQohV?ve6_+2GZ0&tG7rePT)hGDoLTY}XV93&+4UYa%9ZLS
zD+cIUXCtiJUYH6Wth;MsERuTA2GB8aA-*>yN0}alCugSjnkfT-^|x_af)dHjV6Sb(
z3X`1fUpmZC+X-93&Q4>BHQ1f*#(jU1=7slaKy#B&M+3L_?gupB-)ijC??H(=3MS4k
z<oHOj_{r0bflw<9d#-apbe9nsc#D5w3abocgWYqO0ju;!lVEUGkxIS(c=Rd?TUD1v
z<eu|=yv<yfi#Sl9?=6)0?zl!wy&8z7$b4?1)+pdxQZXUEI|7h*t=c%e=e>7myqnyn
zObd3Ou<NePQ9|EI?D>^&c0JjNVloL+`x>`2I9vF3xQ?GED#GE34plw{63biujg48)
z+tP<?@_1=7CUOw{rWn<*B0wW;|B-Hs%HnD(edXz@<ZfHr@XhD&Dysy_ED>|LN^2>(
zH##I^84+A<0$((qo$5(;0&Q+ojwc_<f_zgP)>O%0SJ!-!QSx&aKDCKII@>DH6Vog1
zcbg=@lF-DaTdCjdpqW0K=IOCG;ES_fU$2${0Ne_%Jvbf*-;ro0YW(ZD`vHqh0hF1M
z6S@khJswV^#^TFI8rYcNDKg@Kc)zl#lVO8!D!!7Q5uAQiSa!1A*XLj672<L%S>=sv
zGevx7CYIcbScjClS{JO0$POj6!M$IrPCG<}K_ra(n9JqwbLiG)2-tLTbS9EXE>gpk
z^DF%=sILRF1K#K@2{`l8Z{gIVGx4N78d!bAT|TvP-tXtTVUyqq_eL$JL@~G2Gsovm
zFx=eAqbBP|$lF)j<ujLyO`_DxbyxxU$5tm%ktFWt#}E4FdnAG8t!H%}VlzBpYj&*G
zfkb=(W%29AyIBfn2F}Kfrub5F*(V&HvxHK*`^=QXlH$#+45h90$NM`W1;*8O-P&cr
zJ@GiYHm2Ff;Y&*ys?Pk7VI-HkP6VP2)vUrC<Oi&{l&dmA92DgWwi#WAYy|hslNWGA
zSS;?b;#=nu`D2~P=)%TUu%KvGZp?zrG<{O9ipl<6I3<=~|7Y@y4qt)&r~7@+5j%Zm
zTbP8H(|aa3s(X!v^BvhH%;+_CA7R#$UjPN+tK^;}Zf|5kKl5J%(|!rmTEd7RG7$a*
z2zyI{z(&35-#NL_VEeTSM?E3l-q*2$&6+PdIB`<@^Jai;G@}(6E+#vUu{=1}5A#PX
zHtJ14@6)U-{nH!rA$dF^Dm7#F;DIpe_te_ayy@GS!2R?=%22<K5|D4n4EjO*pIs>C
z0UNk3r2p^JiUjfg&LxNE*YTCymX!V&>>o@jDSac+eliNL7v$OZ!}mLgww|n_poJPK
z0FonZf5$F~4;7_7esMs4s{Zy@noYxS(N9j#{wUf6f?AAp!fCNI3X1RKUnj*)xkf7I
zx+=YziT{i6;wu-l>=;>(UTl0)F!^q$?;Foth2X<vgz#9TDm=yvODA;xhXXSagmyJV
z6#`KtJ;cszRZ);r^&U}<R)t7GV8MVoNYlV1IX+$flS>cJZ^EON1;;DcZO|#b*@+^=
zv*rBbVE<4kKDtp3>@B}8qd7yDD+cPi0RAc~#W?4`NJ~ewU(5|AK=*)g@)X`J#KRDE
zz%sWf&UE}eiEWlQ5{BDsJxVo`N^RCY7hjP~UmkN*uLUIi)|8sJ`g3rK+&I8Of#R*H
zYTZw=VQ*evUx||)Y5%z0<94FeY72mH)1KF&ZG;CPKM<ff5+6Gn0sK?`{b<utlS&4T
z?hgeQ3Ic-=R>8YHCDgUrF-zZIMeg4KRSQ7?{=u1gjCMtV_~#lVPO<xYS1N0ftRTa>
z8=Wrr;I|c2%s4Ql9u@Y^4r8*vd#FgHkz5+ZTYNMYEVzeC<gk%7d|pZGO;ZcSi^uy*
z`1D!F`$a2~va}~7;<OzTWbijTy>m!Na~y1cw+RKet+&i*e9(}^&REhaA9`oD${<eD
zwf_l3V&81N${@m&{ndP*)NYDB$8K1@O5`%DSW{!%a#rPR432Icck<ol)hE|K!{j{$
zBv`3K_(O+^XzQX-C|GsJ#jk#0%S)`h{6ad0Rn>J${&OM_<p6&53kiR-m7m-oezQ+9
zD*A18rE>c}zOeIi{FnsF1fvxR=hTZG-}8|7cY&aC{ip%T$G`&M>l`ZUQn~a|w|lE<
zz2HT>YqAM|)YRQc#PX&L10^!=<Hpq$0|%dW4XW0)1-}ga@4?seF(8RLfM1|reC#P+
zx=5>;V~v=PBR&;~8_zNiTO5O(@fa}+?AFQ*D5UV+CEtx=dP!Ko0EgFbquom)&rRiC
z!CJHA^p;hK-t*uFG6GGAfB;FMuq!0%t&6$_unetfaA8Gshl+_!j`Cx)TK|I`X^8O)
z+UQ4Z*kSUl9|q#WRH!^jL%@D4m|oVRW0`Oc@1}k#crkfoQ<3;17QtO<_M~252i?3O
zP9N__I`MN897eKLSw~C8wN^?K)6w3*ovGeJVz>^OxFD$=9TtC9W{H9j2NUv3Nso!}
z8A1BZAGgl}UeDHC^puBlh<OS*t#p-s8^|7C*oMB|_Bpl7DEOc7R;MIgHxeBJm?|Bb
zfMTzo4>-OjfE6#3-EL_4JEQzDaNw0A`6JRgy%of7Gspjn&S{b{GAO47Qv-P>d2a;-
zi3TD=Jzr}ek2W^uM)W{C*P9bHh`!At;S@KT#?xzmwnND0mz3l_IT()FJ0UR1QDvtY
zQAXT^WVmObhqI37P9u$Qz%)AYCW`_rw3Ld+y|@W?y;*W_t=`U~f<CBNsNRvH5!z=(
zW=SdD5!_|=+K6O-o{0a3Ai*X+cc0muHud3&ABs)}QqN)=p=#`q<(&c9C5QO7E~-k>
zH8-N}Lg?STzo(u*MWKQ-PoV;ZFrUf&b$F(#MhQW3AmxM)GP3`M{{HzKWhOcE6gcEo
zm1lH`DnLZd1*}cfxqsose*<bTKTZ+0a#U)()wA4x4oN*7%cPFaM+bvNDva#_LRv8_
zwhF&fK%!DNmZTA;q$YkQy#ZiwG|Y(QZkk}ID!iDR9xWWnUZB&mE7*z~9t9j~Fjwe1
z6K0>;fMal1(s3~A=X^s9QK~$msaNztp=(E@wLhj|v4ntG<}%8%Z`Gkry4x=l$KgE%
zQ0z#-b-ueR;x2HbPfJKWcXN+1rgApNAdOX&lasXSZcx}fAZT&Pupz*;7}=Z{;a=Q;
zCk0EPab_5^vLmMBS^)7M$pKs_)wat%<qm@MM`JIwU(W*^A{OST+kboKA0PeOQ%$LR
z-THv<D<*r1`V!M-dTs@r8%fH3#bCVOzn}YEKt8;?7WA#TB$!CjDKhEQ-L2CZUl<F?
zE51jm;-`6x{0>fK9BjdpK(g#d4Fpq3uR1Wq&{B(^8*$~~>@b){F3Z3M<G2=sbE#J6
zRrlUTq&g54tb4aX4qIh{6^KqF-8;2*_|3<%7nD??{O0>s&lC;A=x_sd(kXg`lS?R+
z*!4LmgytE|ql_}CneYwkZ?HfL)(;{b4Vijx?WqCXcGepn79q(z@6a1AZ)X=MrNZqw
zaZl8q=`4<BG{j5qx$4^k%2ra1BZ!?KTGHoK@TwAQstlzYiLNAnW&4U}AGp{4>hE9_
zo+(RUTfDd#)bQ^!_~&zG#~%mmOGy;v%K*6lBlocQU6wzz1pHId{ax7E;@wAnu~1cx
zWz{qV-DB$o>Y}kNQUfenkhhLJhqfPl3X1|13VKjS>}?~JAjv*YNb6r<{7(4B+MMoA
z6l!Osiibxg_!pJL_nY++@?VP1*;!0ZNjXf$7L=?%NUV=|nz2Gc$pNHVAyC3eRwaZ}
zy!l2H%k%d!`|tRlBl^uV6+vx`-xQ-MltT*cd<`JftCtecI`|wx!fR~Uq8;>{wq^I-
zvmK+>a7`y^eAqGg*W*ye`~V)uyO4xtogW)fx24gRjP8-9oyx#)lw3HhIpgw;-K6Kw
zgJBwOpaC%wrk+FH`<Iyw@oBqG17;@38b$WMnVH_-W>&EGBJTCInY{tbj5Fn+V)&m!
z{qsKGd2MDZ9^bn2uFVXYbcXBJ>4>UhoCPs(-!!~c$MpVz$>U_)3ebL}9~rVI9_?_u
zV7;Xg89^&{63Ck1UaxAgp?=4E%sBga)kvDX^X+%ilGgW(wYyJpkBd?GDsMfk3JTWG
zcJtj=>j$X3*LvSb?kYcaAI;D;seh7@)wY1TX;oD6>~6pDDk|g?WIR1PD}#L|F?&B2
z#Oy~=ALrhGfPI7yp#}0rFiO=Fq;1hh9-&#b%8p+5L>u4#6p<Vg#1(~j=R~|<2|Y-D
zLIuBkuL?;!HYUt2E5PpZFTxJ%kh%E>uD_YqI5^0JWQmOw{%_*@Q++(F`ct6Xo%;5T
zkF4F4*C!u1VRID^J&S(=-Jkcq*9_Pe!H0wD9>zeHT-#)UPSfQ`GHoCouxAMvn(J>M
zZy>qoIZ!tGLRbU6a$3h&r$FMOIk`)M<x46WX`B(kiD=Sb*Ja)1=?hp4K`#S*a!Yc!
zJCSqKr+{82ysY@&18nGx`r{0+Gc{`Ot(I-{fE5iMUgCQM&!f|MRLldB2f*FB$0Y`R
zCpiDKKPHS5FKmbndbt|H<u(4jDrBEub$_QzfmD?hm)aY#YTDp`7p3W>2KQ;C6vl^>
zBE}k+pd!&J#+ghs`KVjKd?$S{fROxWx(kT*Y>bXn|C?<&|7}}UFJGFo0DR?h9l*A_
zlXok6|7%;Be}%BkZ{KzsfvR~gHc1>{3k;dn;Y!xys{MM3OCgCbK1cQdKbL`?g9_Vt
zR!JDIZB9WJE=;)3&@@6TAR7pdI&B3s?7Y7dj}q=NKt=(sXxXYIeX&s}x_S2rVO!j3
zBG|;}v6OB;*rQb8Nv<sw6*o19uwFb%L7TA@<o-?Uckgc2gYnDe`I=P(TW6n$J_=p4
z&Kq5Q$WjCJc*yFxM5FuUDB;CVo3%xQBMb=K`$mgmVje><-$(Sw;g0;Wtf3L;d3K5#
z*)0pQa)&ps9y<2_<$pV5)X!kp7H>@(?fz>Uf4+MD`cFc9sH&P0uMQ-{b)cvcb*x&r
z|34C<C=Os^gW1{rZ9|3dnJ>>S`Vt?{RJ@o9$nUN3aH0`)UiIPZ7TPi-WKBI=AFD;O
zg?dc)P|<?iZxBp^@Oep&>v`n^IgW9T0Ru}9UT2w*y?F#u72VI^hN2mrS51JSzW@!%
zZpPaFfEGlre~ImfeGD_*K~$TpPmzZB{;G8-iMPDZTpubjz`G}}4T4Rnn3#TU(YKrb
zG=QT@hyQS2A<`Z6w8U_+Rmy9zU$tc<o}ymmv?P>qaravVCMblyQQ;#I)TAva0^B4W
zA?-M%g(w}(HVd5#sXtE4p3n5JL5qz87Y%=)!VRSSCc~Hh7sWP{mAj_|a7EP@_t{@F
z{6A0n*FSf(m+IM{Rg_t-X^}tymLj_8SoG(H_s{Fo1V12|whz2mQXmJw4~}G(3X?!F
z_hzniB6lnYEK+d{-Yob5zWe=2kgKSk&@W(5HDr$5Voo}`k1|bD_~yCOyPYU&`b0Pt
zP3k!tdf&1LB_(kh9Ko?uME((*8nIzH*U$ja6db2`O~x{~yw(`m-xC--w8YZMi*D2Q
zZZe|%Jup%qFZZ>DsvZxnBOW5FdbuQ>PR6>cFsu+M!SN32ecq?_c1=2ZAn=X8`e^-#
z=h_dvGV@Guz~!0}Wx{DNmp<V8IKI)qFb%ShBbh!_S2!yA-`8Nk6GfREuYukF`Ka0a
z?TKZ8;@k3dls5)EaVvSRqW{01X#AHat|xpOuK_&qM{UxHj9cd${GQWaOb)#(q-wZ&
zAVl5c`)h>Kx5Gbk!J>^qAl$gL$7F{LgBtLvUAe~ux@*_X)_c};t2tbB=Z~P;1O&SB
z9Nvfwnt51cLkvAy8WPGZC+Flc^={VE;O37iI@-I6yBSSctl_?Ey>coqW*^;ttZkx0
z7d^nHuph;)u+Ml*YngtNx9g^U6QwF^pyO%`&)w4zg%^nCXfLxP)G?`3l+P-AX!ePf
zt(Ac4d8=kstMI)FlhT6`IN*4C_Z1WVTv|^aFLwjA!<)U^a{m~`zuy7i=&!rayqJ)S
znJmy<;}XEO_}xw&vHsr_@Nk+4pd;?2CgpVI0enRDbn0!Ob^uZSa`L(D4tlSdepg4n
zE=~=Fb|9Mn6rxIJh>u(E*P`uCbP|X^=MT-F@8B{qwnSsr2xl13Cd|-5E%7ET+MS>_
zMo7u(Y!bS;URGGMJ}w+_pU~UrlkjeYN*F;)Kr~AzmvUl|WvK7zobg;hjMkf3rMRc{
z*~Nsc;-O4e{X<mVDe5}Sq~6XV{kUKWEc@PA4xc%`IJE-zezRxr{PpF==+tG>Zs{Uz
z*2BD);>{SKI*Aeg9z>%nEJ0fCaG&F|(*Ny(0WiO_?7Z;*<82JD(?|dIHyiUWTb<W5
z<R%J$hCJZHq-zwr>|>A?Mr%|1?CjfCa%4sSTRu!SjKgemSf}Tsj+JGSgj1FBJy@Ax
zTHllF=x6zQR&{GfZCyiW;>DJ?QMbL5O?%0s&y|t)lU){?i66&UE}09OwW~JYe)KwB
zkQab3OQV{aRz3lzj$dUH>ef=Y1wz&F`C_mP>D{-$^vKP`cU+S6zj>CCy{LCyz1q;v
zO6+u}!A$f;cOjF~R5Qe6x^q+*EF-XL{=5>HZY_}GH1DNrV~gfF|0NDJ)Mb`ElznDY
zy1yg5*!S}%!4Ddh5ews)7~F-ppr{X+#X4!OEC<I{X>Vq78fF1oE;y!0i)ftm{4HGm
zP)E$9%kSNKR={B=lU@joXEYe-glzC{dlsnbEoa08J=@V6S?7ICzuIa5K&B^+l!L8w
zo5nZprG-Rae)=oW^=I>#1N6+8m~i2LV;ui{epvsf=2%RnrAHR_+h|;YXxl>9U=tg!
zY8Tm-s0C~WN5=%p$+fJkh-n0hPl@l1Pl`C%G>2LWUaUYiC+yL@cI|RHk$yl`j1RCZ
z*3No$hqEjxB%Q99S;z1TVbzB9MU;GM3V6fg&emrWP30c>39u_e%8ehdm?@rLN0skc
zuhUIB!d;sU%ukn;Q+S3>C*4k?s|KH<3;24(TgzsAm>@?fZnC;F=&_@zr_d-WV4*Wm
zHGVjJ#32PzwvX$(d7YoWV`Jy`;*){gW!$lhsE+F&iu<{*)|os@qv(_rmJKI5L%>D*
z$h8BYJ7!#ImR?=bh^BN6gwFXLvY-4xa*s*&%_8Xbdi5;G2V7!Y>a+~9$eLTYvb4wg
zu)V@^hrXm}bhx8=q45<g%Ix%nL-l$>YwS^0pIPa2<aakLp1f7hf2yFV6b*{E(9eJ)
zqSSOqx|H@lhEha>4@y~V$l=f+rbwcEgozUT!AvM1rF(cxCMmmP=~7X^Iu8u+(cq_3
zb6fUjJr=s;X#1OWwz$?IwnmWNmh`dpzp>))axwGIzs^*(y<dsTHi5)iXQgzc+$?VC
zNbS|Vy*UD!vmMmEW!v)gqH>N}shApGo|Vp2Gf1d=3p(kzRNDpJWy%Z12lD^PXNS=M
zA>9RX&0-k9NGm@S8T3hOxmcEO`sgPG;@2;qY`feYsXbx#K5lk8TqZkG_wrCqEdB1^
zy|uE5zFJoDT5RXb*<`wReGM8^0r<ncc|)EY6Fxub&Se6<1qY|;&g?dyS>4>ZgKdEa
zzHIFlSvh1Y+<4FAT6!?7+ps=^*psB;dyR;-{bbxkU9p<Ul&=ueENDNy?RvT<bW&y<
zG9MENx>xWcka&Hfu(rEYiL-K~P;M7?dEDS!XWgGQzm3&Y-p}kL`xKxM-AEMSTmbG5
z4j|glA*~L41UI%JU6v|gs-z*=$N{hO?0_*i90W*W`e)%o=m=hY1+EUa))~BYnr=5^
zd7s2`Iro}9qt%f=wE8ibrzeV8L{a_jU=^S53tX28=ad@-FNi(wl5fiah-JYye>mJG
zi-n((lC2;=s5`GJQBsSBHV{6!4EhxmqdLz0$}XN1D*QbomfpLG_2cRF{k4G=Z9_#C
zB0WnnqTFmmdrB&JA5h(7)ej3&g+wPiQl%Bm<0L;P0`XIq2Mk;|60-jP6~<L0>91ao
zndM7om*y|%tlwiu<kz?%Ip9=?lW*|z)N-I^!EJ`tGzOg#siip*f1Qv-ur1hNdvUO<
zB95QW4$Br#ehRwiDqUz-J|EOxq<)Ai`D22|Tm;lL=hM9)7xA4&WxRHL&weT+B*l3^
z(*7tHXY*jn`=`~}d%Ww!R~mC|GZIw?B?d`HyJXZSOY-Yi=Le;H-<TVf7y;pa`|X7?
zU#m4f#|c~22_#nGW5!pn&O6+9nsRLt%!<=KU!9Ji5BGZbD6g_|L~yW7xq;oqT+R#$
zS3fG9e1DjPSM~#q#I;<7OePGGEpD$mUmM}`pYm7$1C{xuAKv^cn1+RWw`v+iG6)~C
zZ#g%xYmB{6#~2FomKPnB-o-DI6h}y|20oxWp1sq%z<@UHy91%y(GzN+6l|<V5zN$!
zK17?AtSZTwoI8=iLgT#o+S+>qBCYfH8Jj+-J`3AxlFM$-lF=Mvh{$A7>UjGBvBT3R
z6^T*^0IS|KHrk(K9LTtV0&+x1>U{QvSQM+XGnS&3xN$o^$IjElh-C)!d|>{$Qk*d3
z8n#S<0Cq4V-h&Hn^s;~0d1iI87|@7N%afEA{Wm5~kHx<h+UWSd75YWj$k)0k@^yqa
zSfI?7Eg}cdIrBKWs1cdqS5DyX^EojKyb$#&8_I>xXxjFTSLIb_c%|2ELfrP&t}ee*
z;z2ijy4u0F*9FaZsnd`^C+}PoXz#T<sT1AHEwQndxOB})Yd_}M9@ABw)LpWC3Q~@J
zNlp&*s_x^k3mCMyZVfl|9E7yNM<X3I3h$F$_XFoS%2ur=?5h2=JeL!G3BFmZ2qwn1
z#O=jx`?_QYEbe_H^{Lv*JpDS~RzD4OL-m?sM<8#`lBWK0k$@t3;z0x=-eD*8w~l2W
z==s@1E<muYk#ypV8Jx#B&7$v=(gRQ_rv*ukEu8yYX{&&AiS9fbdR|yF6BX)TnGr&a
zLi}n_5g+fZrD55t$BPa-4W0`SIFarhXE-sE(ut%9^Hsi^K#}pT|31Fn+hjsXNwPcG
zZH1h;R)BW7G<|am>~u7Ey9zqidKcJxL^~!%{i?PRl|Ah8lWvExY2^I<C+%aD1+#f5
zZ0GLFH_bWVXGBe($t|ZFpTwC60AtwPz!-MszsIm$aC;X!6c?NRCsqr5efK5aA5Qz>
z<Thrp<eb;3D(up*!bRJOPS+z=L!y=q(52(qTMjtlBa1`-IKRa{?kaJp7~g<b`<je3
zRJGZq^t$Tq2deH}b^sG#>v{go8Ym;brTr4ViuW78PLqQMoCB%%kt``u<1|1izq!Qg
znez${D8rqQx@}rp@#`cGpxP=NJtUHeu>zMKU;D=XFI;XsG9*V8DTnxN(db`skftXr
zomf6=pZu?S3(rZ@xXKt=up~&%7r+Sw(6ZJCa&kLS>^}Fd{8G>7rK{?F<(FlGQr#Qh
zQZ1m0mWhBKQ9kt}$*#N~nu^$H;rMGMVI2xUNjNK5E(CTW2?vyfn-%eVB0In)@;jDA
zG2k6(u{MjSb(yLHh;;!bDA%JEH9ycEx6V=)909!?aBXGOS=38TWaWSs_0{l4G>4B&
z+s^GvPS&@XWxo3Zx4%<0w)zc}J&N+Q|HbJ3M(Xr|gXhP#ASt(mU0IkwY3*WalU92M
zK2B^J=kn)^BEPV)x^>ukjk9$ndJXkJ>1-iCVI^#@l}fJB<#l#fP*-qnSut_>%s^sk
z*QhzuOi`adyl9Lsdr3lwJ#Vt}WByEChfl&`Ojp8ZNb<5qx!RSS*3b91o~p~}jA?M*
zT7$cxKiJGUU%)(L0cG1w9G~QUWsB(Y+H*za{JmyihoC6Un0;+`nWLSAiCoR*nFsMM
zx0vndl#ZL;gr;qERotccl33FIkTA`~VTa(h&{oF?r+`IGjesiyzr}Fq+R`!<KG>|R
zS%w#O?`)I2|K#T0`Pp{uM9)j~0cRq=wGoX?IZD8O=9f<rUkmzT-h}3G-bB!zvq!wW
zzxaS={DqR^x?mjURVr~}SW@2NvaH|EV_V<en`M3BUT44D;67GI$!>Wv3TGYrX}00H
z4$L_--=jFn=h^P*ggi4B+jyIF9`EG1bX$y-*JZgo!IsY_F~_KqW2JmgR#zvfdwo1&
zM%iJcO;^xo-|O-oN6-89Tm%0=1MyANBcBT;j?$9+(lNoZ!?g19>vC~n+H2xx_;h}+
zs>dba5)4=X(%euAYzk~6fgaCFO>wxMk>4vYOjyp<eLWP)vAKH=h!~8m#6W+v4X|j5
zmcuGri1W29O>FoEYZRQ4FbyMuTh)$&X5$|eZLMEs{3<5Sdzq_JW)-7h8@rS=-YLSj
z83O2l#Jxn9h}6dB4{G|3oeqCBV1fmVIhn5vJ+k)+`z=%q4AGUtCeJoXUYX4_XzAE7
z>wfaUw=R=<sav!Y%t<*ju@VFOaq}hX=(aEB_}d#D*EO>#FZZz{oHCDuBa}}Cd|0)?
zlQ>N8eVMMC7>*21*oAA})afqA9p=rv${!meGrP51%!)&+c_sb3!|6{Bg{EQW+*9_t
zFE#ckI5ZE}U+2=;NZ*IFX3Y3rH2~@zZAVE0x_#I6vh@tQGPnIG$@}s|#%nGh!((Ay
z+G(%(SK5qE(u!o~QW|95LYJ|#pkkiNda5en2(uVt4=9YHx6bC$fSpH`QfaHhh43E1
z(ezJZqOU&55^pqsVdWXpFxPD^!s$=aX@t3u?oW;jfr3_pl!Az3*5-R`_c!hdwA;Z%
z&AauIB*d;t2P+(L%9YoOFRv^rm|waZkgr^8R=x4Azd7<@tiH`#y+|R(C(^UO%qHOi
zM$ta<iq%BdeyR(uwOzwEKJ8_nHzW9nN72Q~S^`00LXRk@x;%}E=12xgN7wRHY9A84
z7T=1acOH9}4l7<&Bewln3Rbd?6~er|d9V1+HbQoWO+Dz-xD2eOYbw1|sXUQMO(R?J
zn<R7hEJXI3nQ1}d1W@tIgkHrSm!BycmI3UNEMljT8;gYA7`)ImCM}|43{{89RcC#X
z&Cne<^t)DGZ-j{#-C|<lM${8b6d(BVm2si;o;t0K2*X-$3;i9{g~#Q1zK-fN4`9y{
z1Eh4`QQTpX6z9P8=8%QT-!i-{&$~8oAMC9N1_jGI8HRG2=rim#Q7Nw9d1**tnt`4t
z?0gG$Up4xK)@e}$akqncNy7S`)8;E#-%b;5{k~AE+pLl9!JwJ$nDEf(4+c0F6QAgb
z+naUg4(i{C!#|&&ULz0r>!8G?kM_oV=8I5yT`FKL##OPufc;oA9Df0JH8>4{R;(5T
zJ(albYDiPCBbTFcyG!V3zZ!jBt~8`zkyxJ6+AMIoG+;4fo#OA+aa)}~0fu9N8SrX3
zYC4{e2Cu2BH!AzK=F|>{7aOkDOd8;xlPzNFmm{AwKYzrR0Q8Bz=}mpiEwq$_JGzmK
z-dFcI8pApqEnRt+$9oaxwq;g*aq-N$V)B-mDR+DWC4{6jNmcA&Va=D$RJr8W6B}t=
z2jy^t@yNM%Ia_rVCTPwMJ%-W|x>`3pJHNQ0`_;i9c4bEY#y37s=OX`}%I+$=${)Ul
zlz1l%enyvG2d^&A-<=oeZkeKzCt#+`-gWn{FP^+AZYnjGJl56ARhF&+3%C7KOR%)g
z+55)kW_?G)QOWV4x&0@WyE4`L>G>TRHyQ`l7enX_mC8R;e*GASZyBlj)1Xxgbs$s@
zs6muIgC<DVE*x><;Krl{3_<rg`FuEkOi_3Ni%N8~jVP*SN_7g&S~d~M0y$|{SU>@c
zKZEFno_eZJSK1UUQ@D-}qr_&l4u7($V@En}{upuZm2EBD^f?l!*6ks7+nG(-Mfk!l
zysGEhIySQtUe(?36e}qIv{~MCIlUMYls4*%cUdjI$9tYj@^c)=g!FGgBKJZk$s-(_
znDC@~$gJH}{~gF^7}um4BSGNan{j$VgS3poZ+u^L(#}-CuS70lTqVY;%GMJba&0j4
zXI$e<{*Z!a`R%*9b|-NztC1JJLWXthz>JuxRvQJTbwCxq<#Gc1esRdUGK!VzG3)0f
z0Z#8w5lgWGG<)MOmblhqJ?ug84ok5(8=vC>myd;4lBFf$@1sO~K^2+yGZVW(;@g!g
zmz%#hm)I6>C%X*i19Fz};&=886W9yIyG7mX;<PTu%x6a1SdYqI{b<Lo+r2j!c@w$x
zmhkemjd^SYh&RRExV8pKzOM?qkaXh|-1!3X?H1qrh_C}xf2_|v6w@X!HOm5>4bRRZ
z+t~~C)~j11V3$_wBim_Z8^xad$wN}ELmD_6Co8Dd$JzP$2j{2jdf@0eof`}|l7xm-
zEy*Iar+cDahkE7u>yMLoOv*0KW&qU*<?aOS8h@zw>F6u(9u5`|F(kOw75}{_H~#Fs
z;}t@A8DWrnJ*Zczwh~8#0}$}>mxI7*86hI0SUYRd8BkutL-dB_=k2N>T8XBXkCZG`
zyWgOqtvguYTNjKAlCp?pHpC4Cf%lkoe#voQ?*NTI;?c)>>w?t<B{a2GQvD6}33oIW
zS-;HS{Y=Q|2=;@4#y+E{{<yF^(!A38%!1E|1Xo+*RNf-`6ql<+yqP~xya<6AD{(?F
zE>U&ToHP9{k9%6^qf>{ee`kC;$z-Q#iWA>|f|xzu)Vq;*EwR_HS@lPQqH2X(VvitQ
z!Tdq^e_IBB$VtH*IH#OCBUt<=-AcAN79aJ?Qa4M^&|6xSWBC0L^ihE~&(;DWWBo4I
zHFgdJ7TF=gc3}7$eK~m<lY=Rm9<CmJV$S7G>b#Xo^FcRYnafvVa<mK|JKo`SPw53$
zqO_r$N6`R+{&6gR`q&oI#5TJ@m2)qhcXQ#)9$j*EZfkzT@Lo5%QszeL#o0M)T!OjG
z5ZUnaq`@x>qK7>+X&BCLiB7`>8aOGB7Yy7$9~gE4y=TtDGx1a$(?^|rz}|qPqw*Az
z4sQ&*{y1X~%s9fp+qJdWZer^-eyWEHiK>49Ryvm9jb24{fa}M^RXw1U5_PpZZp0;S
z1?ud-i*4cVoi}b5R-(vTK-3!ZaM;*f0a)i(ITcmru>#XdjO8JdozPH)YsePn9A0^^
z-1ahBBsANw4(Tf&5kbFEF}wUa-hWlOZaawJR_ln>8E?YZoA3H6=~TbjqP1M#@vYZs
zo#$2Ljct6nnKvJN<-;)_P3%g|KKAof?oK@~U5;+yEQ7p0kJP#eB9e?plmKL%WJvc*
z+rFKaZ8KoNB-ZU{rv*+;E|Vjr9woBZ>0iT^9_D={A=L61R|lk6v5ltO#58y<ufFpK
zoOT3<S!z0flKwfTnki;BBJ?-I(sN}Hzj?A_2r-T3p_5Lz9o=TXQp6$otM(B5v|vtP
zv@gK>NdMCM^ZkjORuIROQ_rii-S#DaGzNzO*u}8LEh%f6!#4W>xBT7gS4_<p?>M35
zQp(fFRAcVtdpFtf6y!E>z%@2*(<ga>3c9-MfVkU(&&*?wq(EJMoKf&f|6BaCAZLu<
z2lGo>L;BePN(Vgp2k7P!gqoaIR?&G~9MrpQhxrBK`>o;zR)7>Z`kr4-=a_c%^G)<J
z&3H&Qg_!ZYG<5Ef%KJ1VD_!YsgZ2aJh3{M*VdLDAVYHn>id+POH|t^eg!A&O(?xfn
z7u-9$LHhE$WV=~}5zu$3^L)$i9I~ZYIrYT$jIWziwiV%drhd<$cVl?Gbv719TrT2|
z*E}*p=v^B}vIUyJ2Ys<VOe6LyY`x-C?7jcr+=8)}FOxut<dcFNG<JuS*2TuIXV`T&
zdn8UGF9gdE%c~w$N6#;t@>0Wo8Q<uA==BL7Tzs1gW<;!Cnwftn!aG()aKcxvXJ;?&
ziBk9WTH}LM*(M)PC5>6**cGj8RrDVX_?}MkwI-HXZI7jq_zu{?_UzHM&wiek?VRT$
zOn;07t%!l19Ud<HDAzZt?LK{ae?}!qbWE$M&8X^0uE20{jJ*$=5st&j%qO>T?eZ`Z
z-7A}MC;0lP?XL#+;rhgW=U)pA`BAzsnT;pl{c=@Q?a~twThV+IYrC3QmobDG82!x_
zM9fp@K1Zf&wYHJpzjs+x)b&P}T0_=`I%_j@w0O1NTd0Qz7@H+<|K7rN>YF0bfu<Gc
zWbqLkw=F8oF{`pORn{&1NGUncYRgf2Qh+a7owp8<V7JTJjnx4Kidv3wx6XT2wd=M+
z1H;-d-J-zFF9yEH!!ln;M&|b+wIjWnmSe-CFO(V+)${Sydh2L7qaRwb0%D11>Z9kp
znykdlj3iEywM)NKY?Cek^^Oihrv`<yYK|1y*2!{4r0m;>$zcP?&x7b8w+Ri~ku%|1
z+XT~#tI9%ef!A4qgg^7ykihbv0@srzGdR9R%519F)>;k)hA7X~GPkt6Ou9_;f#WUY
zV&zfl-h|-A)@6C#R@s~JdJR|0g|gS}_~}N~U-%AnK!{Vv<%q+<?|z3<Jux<=uXD$y
zleTPaqHuRSg_pKsY)>~PH{GLWZ@A*7C-8U$5??xh_ddAL65MGCD%<Gq&_;cvvxx{h
z;x7|;yWE{*-7@8w=k>MTBwDYI5Oi->joQU-XiQ~>j}ZE)xQqJ8{4Ar^UjLjijkwf)
z@}si)PR4!Z@=m-m{gKURD+!xy8I5E_BSL6xmd&%%S9%uV47XM))S*Ux0Y(z+XmXc>
zCc=hCTLrhi7eRa_4y?UgWnR_Qv68_Kbq$zhZ@7-~qS_&(LoZz?HbdDw#MkruZL2p0
z_p9t{CE7LO+Ah-K@pM*ye&p~UwX>M4uoovaopRAS(q#u!v93i21;<SDO`#H=HowaP
z+U?F>QX>Oxksslya+a?|7b8>d%-$?hhu^#jY9Ztz+9~vlRBd8m+q7VRG`!Q!&a~ls
zbylarnqYn{h^X&6HhBg4wbj8@yT_2&D`$JqBmAj+^SkunX&tMEeJWy?LyO$3%(lvs
z68<Xi%Dgv}`iD)AM9=O8Ml}OTR_5~J2F=OE3e1anz4l-@d(ve;R|cITdazPsm`W<?
zf4?rRh2CpQHk4|e?V_X<hh%j6o}eCPy5wlu+xB#Svgb)@DSa9Id?Dg$4^;gre7p<C
z^D7hwW4+uthQ8k=j^F&5w<`js@}tOv_1K!(ZtjCq4&}sbR*Xt9A!k)qJX`O^b~H~P
z-H%Bu;T!#YzTCKm*i`0Mzsy<?vvPQh*J6bjMuXLTk>m7dG^Y)!hlDbMm*=W^Hv_Re
zpx#Z<KbQR_iudD7{r;m^IM137R&{dLFJ#hhjy95m@BJc?=<0AGWW6!OAFlV076AKT
z%sID0$^#BXGO*by6Ka<4wbA^*h|~_N?#RJ;uaNjjJznIkwir5Z_f}`@pieQ&RSMB-
z4RTt0(e_xnv)5D^<4{%8VCK|$rUra;)FY@X?ftcf=>1tGuov`6nF~O(2j-8yv3F_y
zMfc<HfummmSQR=?#^8SgJ$e47`)#~>1c1|R6cgIgN9pnS9AWpkRup?jl(0dm9xJ~7
zHti3#KFI<+8tBa?qVzU=q{z~+d7jyqA8BWg%P&SeyzD<X@gALxkT(kot3hY+&RLX`
zbZ1c0nd?EDHfZW3-)i!^oW7SP<)2B)@w&RVOD$u|(Knxnf<z01ZSa%!I*7LAe9Ocz
z!V_Z}U;Qd%$AD#6mM{gIFgk*bb2${tLI`-W|BE;55p{0E`LN3bBn+`hU1XI{Jo0N;
zQabI*{xkYWWa-63RtwZ%dx5cVBE>tRy+{Gag0#44VAGt=2oxrzV?B`X9m{}JA;0bE
zTP>`+l*tQH9tRJ-|Eh?K^$aWhg}(6_ciC-!5cV|&&sJ$MJN={E*GBtISQw4SsGY<`
zFK-#E-VxlocK9?EY_UF%LxFTN`~m>VPB#cP=Ch~EJ&B9!-X9rww9{9e5Ar%vP<+Y(
zB@zow_Hx!>>hGzV7=bK6W&39nE+mHapYYwno>6c6(IS=l_v1KqU#-XLj!D}~d|;|F
ztR<AFz+*v%JJJ7M|MlpFRsYu@{o%7+Tp2#o$!gd24l_T!ey*&knt^5dWWI5Y_BOc=
z0qyPr(T(3(yqz3eGse;_#5*}FTTTU~Zb-=2tGx@`#klPgpQpYc{LSLI0)wSZ)#)-Z
z_)n(i+m*jK1?NL<3yxt0?e>k^j+DK0)pbZob#H-f#KC>rvJ)E!v6N!5EZz`JrOTD4
z4qNXTq)O~|MA@Z`WNT2gYfl`^)|mGUgjvzX_FWvMnQ-V9%nudR>}K%lZe(HcKSW<$
z3V&)RWOx?|Xq9N(a+(b?YJlDxGdoqU$=D?Zat(_u;%w6!XeF<W*}QCW)zHX@QJo%f
zojT|-Ajfp_wCFAQv1f2drqS4oGh;>RIsDS6HzfOgcRc&FGe*_GN{OqBN$SU31FQ(@
zZP)Esz4UQ?{AA^dWA0%62oC6mm&fp=NMW&xbT5Mq5IZ?po}KWk@e!X(kJr#r-Wp&6
zz;4Xb3Q!dfG8p)xOpE+%|Kh|P5Ic1pGdp5^BMJnXc_+!Mi|qjVXE?~^c#{HP)c&<6
z^>w?9NrVcZi>i(pUky|^nw+`zfA~u}PXUn5GawoxWDkLZgcyjB+&JM&PXuQ{53-k<
z!a2P$*foxs{wa&Q8A2M1q=@%|p{!TOA2~cROElhXgDV{bJf<k9__%`|(43a`E%pa9
zHVBnP9%G!}q3<fWuC}s5`F+Q@3wUm4sTscjK;XfV#wEDPcw|=Ljdk2vE%7xzSvc>(
z5{|^FV)%Hk%Ff)A&^7ip#H}T4L;88f5kK^NZejeYg-R<1>;G`}mO*W{Ti9qq&?2Fw
z#frAY-93a-pg6@{N^pnb77B$@v^W%ZcXvvQySux)Lr&iP?mg$5bN+pkUkO7p^Cb6r
z*1BwU#FuuRe7J~}Iw}r0tq>o|xJ;j)A-6%wg;}v_6#k<%pz(R*zl$s%k)bpH_$~gw
z5BmT50AfT$z5&cVj-1n*I?^6vt#{nFLU*sGtcK^wu9bw8dH=COyZ(3uRatWRU^A^d
zjf4y;Hn^AE`hcaMdrx2s*;?59_bb18Z|01z?I-2B-WdzFC^Y=xJ*+6(RzoxYL;#Tg
z2P0@hY%je#ij}%5NBVT)LOX{9Kufi+)U_t7QX5Y{d!7e-zHm6dB0s2}PgCrQzgX(?
zNj<aS)PI?RA9Jya&@aDdy_0%zmEQWO`!@~!>M?-rAW^66;|-#5{TmEBqq|IKc**^)
z<AWs9;h!ktl$6q>aph+n{+S0o=UF;dnG~FF`>@4m$VEd&taCGI1}y^Y3=XQQ%kL#}
zCqTyQGS28$8(bMxZB$#k|DDnpiKoP^d#wm*zb5*dR)}JbTr06rXr8Q-#W3hR`8S+@
z?fVAMIxKmEW3C?vbx8pOR*epOUDIHyb_I;U@P@l`s~4C1sZB!=VuTN;x><z$RrMm@
z+#=W=>NPPaB~#rqPvG&;sRrNOT5sjFj1;*sz2d<sM}=cu>`9f)fe=m0*SAh9aYw!<
zoR>(i<MO7pnDf?>nN|=XdGnGy+Y+o(lrnBcH3#H|r3C)6*^Zd$F1?dAQT>60RVetU
z)m(bj2by*=Jr6FYrS42>Z5JoPMO@gHea{EM2Y-o(;EduUe~y~cRtwVydZpz_p(^?k
zF=SaX=pi6vhC;|u{koRtNk{zo&{?t*9#7#C6!1+!<T9|?gU5uzv1iC-Cqu~7X$kvG
z%~;IsxW(-L2jpT_W-r~JHS?E}4N&1lP$_{vHV@iJOr7_PMc$zuvTxby1e))Tt{WKb
zAesDL$wl&se|2^h2>|Z<rYepO@~TB(f<IU(CV#xj$ITEB0K(`0#-R!N$RfOfjPMzK
zf5DR*D+)u2OqZp!_WN(@7<b(WT_ZkZq*L-#k5B70FEWU-wL=d7f0<(tr{`dihu}Q-
z-%>#w=g$~K@JwQ_5<g_TqtwR7HdEfkvih{+V4jiD62<Cy>$n$;E9~`68gCO|SiY_F
z`=!d$D}&bZ=?9oUE1e1*2E-{)D(&Wi$9LAuXhlnC;|D`qHQCJ@2*w9lW^L=2qSaF2
zz6ZcL*#GqrbeXd;uo@{hI)e*b;JKmW;~FM?Jjzw4T_0B}1|%s4wi>x6d&8DgoJZS#
zN>1YK?EYe_L5R3$gfH#7w!jAGVAFj6g(*|tf)5ScPb588OVPl!PA|8?mMG~Ng*cNY
z>C1~QAVpo@*5h+29ijZTUDo^8ld@E`3k};3#pLTbaoTgGZl?r|sjWz3V$)%0XTNIY
zPcNftJ>K~#ytW;}<GZ|Gg2OTY+N;May~bIrM~YK#ig&5A<7#a7R^M1XZL3+h+E7$m
zZPLa7Nq&6#Fgp>4vbvJpG-b3Ar5ed?O8bd)3|l$nbSnv)h_?z=f&C>bn_zvHA7NgX
zyC3*aM?#4&vT^I*Tm$KT82!yEAkcgDO5;@lZ-HKuOX=C~TX#LA8e-wRK|NP4F-l`Q
zfIz{1K762$Nhev3F<l7@jr3}kz3DdRS0PUdt4-rfK~vh6jnDm}no2#*wTSf4Okh6Z
zCpN{6hI7cf+3D=ZXwT-#HAg1x5Ly;%G>nVqj}7NrwF*t8J|8bh3P+$R+Rl{_&(ziX
ze~iW{*C!3F!&7Rivwr;D|9}x?_#HQ0S++8a`>NHsIg9st<FAsm&S{GpS+Yru_AHh~
zkFA(^^+&HeE&0#?y8lu}rclpwv^<=#*Tv3)d~E1aXZo*ZvYL*O-d*l_*X47sHu+zs
zF2f>b_df3PkQ^X4-@-|~$kPp?SzaE&jNI~2udwB{=p1oZ;=Qxq^2xh!<l-O^%>$YA
zjk7<wPxb+Md6BTAO^sUZi<xNy#oA7A8*|yQeoL)OdY(v2=wYSHVvS<Beg%p-Yr}@V
z=jCLjT8_G&GmYm?#({J9`&y*Aku3eQO-T@P`zt*%F5H5?Ky<&jD!k=!C=z*hRPx)h
zavgBNNC2?1rk;-*So-{GLB>>^pp(TQUYNE!i(nB@dtPirKp+zTq+3DhVz|`3yuO_#
z5;RuE4^%YcCSX|ezeI}mJN#FyN>e$~q~vDP`yaT?kY@@RZ&e{I%{uUHBZ<;+kl{cU
zvMhf|N;Zk!3ofDkAX~{}L6iC|Bc4rQtv^WR{Qo}Q>Os^~14yTK`8XVLbVNb=6J1Q~
z16nq!gz9_z?AN<TaYw8Y(DyPlxcC+ycJ&*Z#1V-5md@+Ll-b#2y=0T+Td$=B&tVaV
zV|P0jv!(kXft~R>0kF^l0p|!z4UklJ_(Fm4#m@w{@iFv~4{OmowO?LC*9q}G&86DI
z;p9Le6c^M3%$euqZw>!>VORpbEMWtXZ&z8zg?CjG^67g?r||kWYnLa;SVYW=?9q;j
zB6{wSg+9?-m6;4LC_K0f#T@Vh$lpbfNb{}sv0N!Ge0TFr;FuUaHR|2=tP0wJo0>?Z
z!;;595`xD9;@FznDW0T1`E1WoDCv2RtR{e)o(2^vWxl)Dpu*WK=1#=p)RxfRXlKy$
zE)V>~AmxfEg}yA>+%*<{{hb))yV*{X_?_5h+n&fo&U@R#ooGDqY|U?^aVh{_zyiB0
zOvK0Dz@w13DOw0lTRx_G<RKRJmI;nd)2LL)R2@(Z={XYT1abqu0m-%$=<NbS-%}Cb
zY_bwfIV?51o7|kN8mYx=*I4$Z2(@B3lBv7aXMaq1e{8iHhgSJic9}7^E>za~)>+0k
z*SU1Lg*BxO57RcYV|UPWhckc}tSzXdpbzOSu(u&EAXBbVWntoyQ|cPzlpIw>&$bxM
z-EYGfe)6W3&*L5~q>R1!s1zQ|eHFxUcD>t1W3&*!ncJZ`C^?ntQZ=vq$5M8KnVK>-
zpSt%2e^r?#0fnHGPa?}Nu+0_wdf$(KY<M}?AJZVX7m`CMO@*8(IAf2u=bi-v+(Vr5
zhULvFCE6xApWd2T6noz*$TYdES2y3TA@_W0Rh8+sc=W%-az;pi%o)|0S;>V0Av%P8
zpJLgTArtr5GC-P4$IZTMiCP;WdJzw~b9()Bs0#^CVQgwA!Wy6;_l&JQ*n&%#b7Uo{
zvt~PZfEjwZhZ}RgU8ePUwTAM~GmIYe5zW;w?i6IY`N!+j7H=%5iVoUwv0x-;`0(TD
zH-Pu{w=HYGw*W$bup!<Eiuj2p`9@J`7m;E3nYmay0X!Z1E8f<QzxNypCB}uEr=|HU
z+!x@UFRs;N48HIq3~;}Zk|hIqkBHu^G!Pt$5v`C{pG34;q&3D!VQW#G__yvF;C4!~
zZ?G7hZoO%+pP{9M3_$n>2JM|9#g}myd7rtTnERit7~#2$jL|vExhM0Vh{f<+?sQsb
zL~va4=laflB_F3#V1_|sa&WpLCH99peb~FbJ1D#Op~*q4NbMK$NMXuu&@NS11G#_R
z3nvxT0l7`N4_g0!r|F*;T8V)^L^VKe;IM)l(A=DaCZ$iUfSM!CtdoD;Rrktzd0>}*
zUXTp0#nk|9q&}XLQ&DwZdRaD*tjuz%LKG2cks`{M=j^+qh(UAbv@+rPwyAfBN0yt?
zBvwfuE5Mp{UFRMI<sR?+=Om?=6)^LYq;Omraw<(tRy~8Be`A8vs&y5Qnf~SsE`b`Q
zVuF7SQQQi>=NiAzzX*CK7pL4rIeFpT;C5DiapMyGPC9W)du()3XM%K3w%5{m(_g)X
zhmtA{%SN;UHp2nr*7V&z6A*Z{#aoGD9#|s4Jeqla<<wJ$=<d5^O$?I*46c1zH0f3h
z0TFWnx$DaG;{0l4&>~L*S6Sd&(Tb2yH0DR1RRNfzDPvy>|AZc)eh$|c(Y{KXC1Mn<
zk(rFEyzRFnZ)5V|G?Yd_FT453*iJtSE$?&j4aJCtTVtv6786YVP>U|y!(_-<c-g1$
zGjM{0u|PGBSgGz-Z?o@^i8Fl)Djm;FE&;o*u;H|M<aTF~a<I@IN_6O6#H(EXg2)eo
z$s@hp*ri<8(+!ooD|Q{h(lFGKKcOVzQ^h11*H;I#qGwB<<SzmIja!L{>RsV_%uE`#
zKg(JzslQ<hAzaYar+QFFR_scIBHzwHJ@DN|0i;}@h@ef>z8{!?!AyzAcM{Lca&sq(
z5Rs!<DOxiNL_dYy-`pMvA+B#RD8CTJ48QqCv=VSN5{LoM)or*^-=ZI{ZF=;1H0{ro
z_tkoXJVm-${7_za*E759-5V-6dQVf;9-xnG_};-G1MRu>*X_p@2MVXibs!lK1|0#K
z0zW`5%&`R{B|_X^cpHWa;M+Ka?>i3N^^bx#MuL2=M!#Zy@w0LAPVmRUn?a8DW2_q;
zPHB9ba%m6kGVVEr&ZA#<mc2!XW|f*9oidcLsQL!E*o7IY<y%RJZIfAef6F%rrBmGC
zM3E%4KX83N>3~>|P*^+Ilw**TN;#KA|A1h6jC6=0F2V>=UvEk0flnAf6g|oQB5>P!
zih-v2E@BsZC`H<5u34I7)t7GsPIGf`(}OqY#xz@T(>cDLzHt4?c@?%mV{+ax=(sj^
z8t;~OHTCaJ>16FqB)9tu=}7ukVb41`*3_nlHM05CfB%S&_O|^vlNV0A(hIf&;0nX~
zqV%2-`Kkzc=5j_A>)**kyFZ1l+w)(9dlN#r+#{{^dArxOEVzscU<0uR!{oCyTBcv_
z-@tVj_19^9cb_Hs-d;G6jOJ&u1_g}0M`aq=SyBo=Sa=DYPI*oUkk3oBdZ(6aJ@*nW
z7!v5}!ng0>wcVu_v6-0J{hd0N)H%*8r%yIYbyv4fePpE5(>428s;nT$pjl*q<X9)h
z{yhWTNTd7uM!dv0>}1c|gS-cZ0>!BstnV*HczD6l5&)zytUc}1SO8~+$p>{nn7j95
zv*C{eeT4V-TA#-?k|G%4OD`bQKj`6fU8sZd=$B@_eMYN3g@{Ksr<K<cq=YL##Op@s
z1aayQPR>z8!Tbj2u$p4t2U=tP5r6j?#XLca=EINSZ<YmeY$oUN<5}YxkJ*T|C%-}@
zyFv0gw|^>22Na9j|KW;zHb>|3D2}o6cQL}VCAs;_`_knmR>jHt=c)zvH12fEO^wHR
zUIhpjkULkzx$hp|uhvi3TfA7G1I~Sq?zDO%g|pb_Lz@oR?tQ0tPnBEToC|cbfAC03
z=i^$k5ZUR+yuP+AJpbO!{^qdArMz8$jSG(MsaIHuM<X2t&?`{MYimA}A&lM3PNgtt
z^O0&1eSw9%PJMWhvhk%>0_T5`yVhcihk|ktnLzFmZMAr?D04~t-AaeC#jK<1_c(9F
zlepNx@96i)jkG=JmDz=AO^o&{Z}DiF0d#R<lUQH?E>x#pLHMTyGJ`<iY~Ie3rQ&1`
znKfDQo|FMegyzPq(>;1IfaJ+veH3+G84U8J=ZAyX!DpEw+sL-a9e)dsYZ&4inE;C5
zRn;JXL9H!7SB9^>&gAsYs6m{|@W?FR0nir^#q{2ih6?L`C-K!TrE&CtGZeiQ6Frmc
zS_ZM=kCEwn#n4Z9ZbU1HRRzrH`8I+MpjhB08U`4J18EHd=rD{g#L9aK;2Z}~A|j0R
zscH-ejAKjRT%)3R<rALM1z2ch9qz;Iyz(1PPIiLvTNtzP1T1i;q1P{}$hrZ&==Dzj
zOU4Gd3t~R_>jRMzx`=ScpT-mDSyz#hd=1UY#P1E}V?nlh??iugoxMeYW-aYI-d$KU
zxS!{)_8vWHJpUNYT@sov`o5_fW1lvfw7-+?d3@<z+gu*ot$ZqHK}?yp<QZXK&e%%c
z1Yg!rN-O=n3uO#Xzwgf=@IxD_<e$c>ZI_<;SF$pt8dcJqmvF>tr_`G3L-E1fS^Pt#
zscbo`0h@}dObqKAEhm$>8?3e7!(XmQe{l5K{9K8n;qgzZOXGY4fZEN94TBPM8O$*|
zerGhRce0;vW5qzq<gsvhW9*jcc_yN3{_i`9LYUax*2XOtk;(N`o4|xkY(&47MVcP1
zFwWDQGgKjsf17hX3{n~&b=<u5neC(l(c4Ao6MujG;2XSN;#1l@J5duQVjtf%@Yb27
zMCbNb$MaL(tEIgr=B^Mi&C+|4eo2|)nTm<98{cfP8W(*xjT!|f(>RYo(KuBLm5&Q<
z4r&FVQlv5G-E2*;&OLek+1l8{|Mb--oFD$%Q&*8Z2P6dqNA)zl60J@y(Y~wc_cyb8
z^UN99M=zBi`)D78Pik=jcl$cYuHdnmbSiIFs?+dqv2QUXWMl4Ef_OQ0Vbj@$iHz!|
zFTaW~`%wwfwrALdtvMpD!l<%SR4(roe>Ru36=4u5lK=c9Ux->npzMbzc)=(T5*IbC
zdYj#B7o(EsVn077O+W8{CbBC3m&u!vr<E*!F!jMR=s=kY9Ap36^{kiNQa0oh_QgVE
zqn&KXD7G^R9lKKaGn0aaUi6V0sqEo3n<%Q_D&c;+5VM{=i=h{?PcNwVZ;@++@?%T&
zTeLyUfT)5`Z%+mkC4Rj8`Sf^?VZzuW0Jlqp{51|ZdCRrlm&p<WTQsl|H+is8`@LJW
zO_OZ9&_tcG<&2}c%)H8N-wP0MrdheCQ`*Frt1cc<erCp!N%K=6xBKf61(yQ1Z`rq(
zgX*gN4IuM$uu%FUoeNXsFy&a`XRF8Sbi|s^*miWQN9kiSTaHT2P?Y*&H^@0@S{i8l
zAVxud81O=oD#ql!JFOzDujK%clp+30rsA{v7vfwwEowCEt;iQN`lHZD`CnI-RK4rn
zirF%;rIogeA0Jaa3Vi@78DF=p_DldHF-#iB@Yj+o;}`l=REpDDAKeJ(YU39gxGYA=
zh47^7Y*j=GSDVAgcD*825ZRtvX)cG5xa(!M0%zAF;p{7-k#MZvr?Ae&3M~8Eovc*H
z{V5UG=Mn2+ue&uL&~XEM61h@+<xJx29oa5l>fd)j9N-5y+*208+$qQ;+x}Q9J}+cr
zEsHj+NPeS)Oms>;^oNT<(=hCTkw8{BF^cZ@WDUJA0XUhvIS1`Z0B?H``A#Px-pcdu
zVQ{;<Jn-HcNMwT}LSU$<ly>Hu_gMpl+-E$`cCv%?Z!A6IZF`pUBU!lB6FFqt)L`#o
zj6dEExX1!BU=%^6GSrLJ$&?Z(HmRI}>m)wIj>*CY=eVO95|6)|Vds+|8S;yOaz#ew
zh|X;g3r31t&1IK<);W*NfNaeM;e6+HsC0inlgF%6{QM7(kW{JXSDfw9wRDONqmXm5
ztQ^G)2cQuPGDD;O%kGOCUWqZxlmB!J*>oeFm7TAlD#e-{-!6`+iCA?bky$cD;Iukn
zw;C-rPL$a(ShP>goery~_q?ovgp#~@zx2za?BN(m^w9k`fGD@jCf6hB?Jkfm06%r(
zcOR2pYXG;kiq7)O6WIF5ayRT_okMCacbP=|9jP!uLc6Pzdf(pPeG!H3&{ryf^1BQS
z%AV+zROAOMrN)~x9nq_$qCQWC(#e0^-}YuX_+tT7#v}Bs@`vxKSIUHOU&WW^5qo5T
z0n%0Bm%3i4JPD#U1;u)&MKo(AK27>zIrdoFY*8#(VqcObcJkZFBUrHPjE_rNYw+0w
zjarV(d_-mJ7d^4lJ8rw(aY9{eXPMS9hH<Rtw&wpcs#D4Quc(ZZBa6xggEH1Zl#gyJ
zG*viU6QAGv)^Z=&_l$YHf_6GWQ`h+H|LPYdkDg&Yn)LrD6A5mBZOl}9rJ|%XGXSRG
z|C`!B4X0QjO+ZsVDz*l=Px>pKX?4++X#$j~i%rgYciy_c7p?0<*Z;^;Ew(;MtT99q
z_Z+uVB+eX-+gI3-2LGa}xIp;Cn~KzX>I_keQVBe3q2^;vA%L4k&e8=kMh6aXn?NkG
zTiYa5(ilL)%<KJtq{z4{&UJsEyMwZiO}$Lq_q*#sV=Xd0t<ecIy?EA(^iIeW7C!RK
z3fy#uK!!H}ZX2e$=!e9V?ek%kLgraTYD8<8+NN}xl%TqkkD2e)F_#5zPJsvXYd3W%
zDmzyDn{0J1?ZA6*S8E~P`zRXwrS*hXqa<sg#;_w@?W%YGU<3*S{G1i1?^t}#pX3Ex
zR`XdWziiEs>|W~`Bu=<h{s|}10r1}4Tt-|*c%}+_z8TEBHJBfrSpL2{Xr>9#QLv+q
zNJW2WBV*C#PMrGE=&ZPd9IKxok9Ac0BO#e!h?|Ob-P6aW`G)iIjoxUj7w^6o_3_(i
zkjVSQ%~mz-HUtyI7sxSBIwLlEvB&C9?IUC_OaOz(8KjSI`39sXVhiO!(nzHn4+sN|
z!U-+Fn>)6Ajk4h!xy1K#mkmxDYP=<46!D*`wwm^mo6EK%dA1*~o9cU37u3r1wa>%J
z1m3b}hdL&91E7q6LTCos$VBn1MyWpTEEf&|l*@Xi?pZ>O_&vA5Se_#F`POB%CII#M
z4vMMUAsstFzDPam)%SziE38Cc#N)IeyF&7emU_Q-`r@tPn-zbgX`K<@Yy!wllKO~r
zSeZRgRkx!7Zp3s!FevdkCC?Y?5DuLlpn>6D()kGRr_}3}QPC|q#7-wGj6;gEdm_g-
z)B#E)IFaT1ckDbDx5p4i<N#RPzV2fl!ppDzCJx-=KYXb)bukRNcrVXQY$+iQ<SoaX
z`<&(#PJ((ug4$;vz2YQ}0|AK(jjW1krIPXFTxJ;WIp7G~ZA$>i`!Z?72o?9J6yH;F
zUMNCf6)q<gv{wRwcy79M2V<f(jBzHfz3=#@KM0Kkm*K(E6YY@XZfTEfvxNo7{LQhz
z7Xx=Uka?v^KdnDHuCjsp3wqQml5fn%v~_mNF0NJ)2KJ~PK4Y?udBY`p3divovO&#;
z$Dg|OBx4<NmJ<{jw>VvJ+k3Ry&i5#f5bso`E~aop?+e52wrN4AhDj&cu8gr!cPx)~
zPY|Dr4+<1QB~>HY3h`r(Xn=mW0?^lFRyzmAre2v&<GMYiSM!7&#UU#w4TG!1d7o!7
z8=OSb1mRWM<D|{0L={*wQh8A~Ii4mhD9>7v&2sIzpFEE?66xw+_3q}V?$PnK6C++i
z5(rp!7cFS*Zbg)nK<*{o>s269xSpxFr%|(nO&5`>-Ujfdopmz#)E3?)<w#d@XPd&D
z+Vn%bspdb>AT%Zf&=xWWNP_1<kSFo>Su5?V<L2bh)9p5;Rt-%5|DCm7A$P+|)X9_;
z4zJcrY%7LqgcRb?4O!Ezgxj2EBn#Cp8?^7VXtlJe+AN7X0nERX2$k~8Bci7_j|Q!#
z`18ZDW)i11odjyPBhvp|CcjP7-aap8HXPWS)x@(ws11L+?w2}5{}|lIQ!`s+rr3MJ
z))!QVZ8IF&87&2-QBST%F&sM!23-hrG`PDRBhwB+pLdx>>^)C;0P?#Lq&@e8$tE_K
z&m46e%RBi`3Hsohp4DWf^Fy0XXhk5uJ)8UfeYKOf7S7_@kKTJKE&0M8_;<%sY}jyG
ztxDLQQPp`z6`l;e{?SqkQ-2%__BY9%4@HC}jFbBJ?Jth_%lsEqX#(HN1h3IVjb1*A
z5l2re#OcwwpevP~df*ml=0|O<10B{G!ccnL`iXCE>7`$Y2J;eV3KYsQaB6A6sCMwu
zfx`pO&R0&ogNcM%z1qAJi<eroP4X;qDi>bH>0H}swT2t0iu&9wA)fRGb~v18>1I%P
zQWrixIp2GNeO{@p?TV}I1z#Y)q_S-fanhFkP!Tb)3(8TOwLhnmJO`*I0JrCC`qqap
zOhWJ|A#aN`z|ly)xnIwjlGW(9n}_Xe<uYvT&#?&W;52|cOR^>k1+8+w=ILVGg@ofb
z0)g~v2%DAa@X443bm0TQ0Ffu`LY2H^uTu$=e&;aK7S*o1rm2J!BpQkR87%1glr&Mq
z7pc<7<EPALyYMqY^yrW?LJaYQn*;0hN{aA|I?}RUc%@!KYO2SUA}Ajv;u+o;;zouX
zN!PoFrN?vMqV!OM@Hv5gslx7zub2dZelKy#9u3p@q37GzpRTjAEPO#2L$M$Xg;@Y8
z`ht`WB@&(KoD^`b2blvrR;zryM%%@N{q`)Afvg`^Fb4$Io>74yF#C2w>#rkHw8S%6
zrL&B+ao@;H_#*^f9DKDKs!$Z^Nsj<ofmXYirrwj>WKe8~MQ-A1pU4RvM1%Ea0Eym1
zo!~8!*W;qF-*W6^6fo8$-Qr%Uw2;5(^(1%QT_o%??Lhal(I8xO%sY+=BlINio{F+Y
z^^;MI4)DPU*sWTTVp>5oa+SiIa@KGM!$X%z5r0dpIt;Q=ZhMzUIgk+n;@l9&JEQ8t
zaJt!uQUjW|Rv@K!@*6b&Vvg5jjb*~kK<;ud{$RTwXp$&8U~PLlk85<H!Kr((Mx^%b
z$(Y1S(QeB(=&P6+i$^W6doD$Qe`P_?YKv5D9hBSC^&tM!V{V`h&Plp=7bk+PC+Mo+
z7mHrE*3}~Am?CA#o~_-C?cjEAns_6wg>e|sz=OVz_Ei<n5l3@M`UDu$U`TsvkM{a#
zsc!3abt!|L2EE1LwI!GCQ1Tx$+fSS3Yd!h1k6#4nLxE&pTcwxwkYDGqX&JXX#tvxW
z{9Y2#A`S--!^72?H3^{Iuw?!{x}NvOQG9o_fUV!NvS;^!@HTx8@57r%#RkV-#Ok-Y
zJ6bq;?PGWf&m!6mqTV`7w$JjPe#r-p7Wj4b(3z3~kYph%`H{?79L()7V?Ep2dyBt{
z)R?7lAHoN?B`(SO(tvJe(f>lUnU<(GdK2g)sLd$vLFo>#eikeOx=&R5;(Jxe+mwRY
z{RtBKp6fRKqko<9OzsI0!{4SpyxiwUg0#j+kk-&aM}sZW#Mo2Ej8?mPXUt-XRQe|*
zWNWd(M*@A~i;rntKq%e@2BLg9=g;%pvSa~j1D;~TkLZg$vXcM9oXo#%SEZPi{!~p&
zUKRi_lZ-!rQY!vNpIN;UjpHly$%Pd*$mI<s>n#;b>iQ({DBJ2atZ;VwVI>3VF{skM
zr^gu+%EUEG=*sybmxDTP9!TQ?fcSs1ONv@!P1f%cn2V0B07(aE2JyKDgX8<1*PUX{
zGC}m?ku@3HDXaV8`Nuu=&dh?H&pFA_qyR3snyor|c|$t0J#elDadP*o4F`Z)Pn=3(
z*a=0#bX#nHvpYF6#iX9gL-mXag@5K|QGZ@apvBXnRlJE|i82uO=Ew7~7YffemtQvl
zJVUMc^*{gpo-?T1?EWc``8D(PEBikwOM@Ef%I{=KtI86bS|&kf0}NH>gm4-Wq6_<8
z*625%o;gF%vAq0h$(G(!`J@)yuilPn#j5t@_(Y<g#pu!KAGnbUN53(8)?Y#gz?I>B
z<@mA^HnRKROxNIZ-N^W;DwotYy<lSdcj#56GC8fhE$81KyUoj6y>=KrJb7r)R0<vs
znNV{t;kReV!l<Qe@>8ED8R2<y4amtq683Vq_QQ%YVjp-R-x<ggohgyB9dT|mA0GR7
zujsHdc1H@&HSSJ&=CwM|<FY;E9s4aO4=ur7tA0LRt!S^&>(vZD9g-`FKb!|QN(%&)
zwI->V`6fcVP0}4s^NCSRtuKw6Vwr+8MY)CDMifnO&M>+=KK%XhH^*2?xX$$j^gRGO
zQ8ix)4~@doq9W30b0lMYir@3<g40AsYCo0fii`}7uH_L~f)Q{aq1U8Paey|7#Nto<
zD^*A6I<}Gp7E44tQP{st`>ar6VBRYxac7{994vW%@Gs%FGfQ8cJuR8SKt^e(1q|gQ
z+cw{@?->z6WH%#$5baPuFaUs#a7)d9+bC%N$uazV2<-XfH3Lt!z%V9D@Ta&l7h3@?
z8{TyjpP!&1G%+4k;?9xK6V=^U;X^nQ!@5A$DyI|id)=VV7T!18!tRkBz}DTgE2`~Y
z(VGlf;>)=5k6b%AnA!u>>^Sf}ORegOESwE$4EMdl(m3&Q66Uj!ujcdP`sS}2g~fC!
zFLwxP>uF2Tc#*AP8kvJKdmtI6_Z$#xjITYDZct79>m2#-TjtXjV2a0c6g2S3k{sHz
zmzwssXC{+143(1b|3F$W5r5z`axNlubKoZvB#_Vi5WF!_phEh=-=KB|CbeP`1})@f
zfc+VOczE7?9c%e(DB!yl)$iiTh%Ym=P4zOy%i_k!lZfhTh&??9EfjqvMcuh^zmhYT
z<Xr6Wuaic$>^N}hOWE^78Yzk(3d)E5e4InbQj$2G){l3brloBUU6j223)0J4oa6D~
zF*6RyeBBGOIAZyqFI}_MJHagOuyzhv0rw)iuk3qjwl=p|7D0f7dB%MOt5;1U|EYqe
z*XprHZYvi`4cF?+s3USumZjE@&Kmg-p07RaF2!p!`Hbw!Jnewo$ifvFyo@(m^!vW{
zHm}s;2M34iCBHWT=Ik$0Rh#`<(0_?uzkC8D4;8itNYbqMEPJYa+=s|KjSIU=juVb`
zrWW?Nfl?4B5+5x!u=pO`EGGHV_&)DVq-&#CZt<42U8rZgK3@97H%}?+6^r}Exe+Id
z-a%QGBE?{;G;+Sqw*Dk@4Y2GW){2wfai{Q5qlfoUA!Y&(WuhsZh<R5q=2v1ds-Xv)
zZZskDuzrn~MBQNE*>7UoyOUb^*e$`@oQ_UeN68GRkF?c?ux*DQ#oyPZkkF?>yqTHk
za5t(abNdq1ki!Np{0|k3X12~f<>qjihA$@U%k0_5z~nNSq2Dw#fgWrto*GDYipBej
zBOwD#SkTavFf3l6(X%R9q*5`p07v9@*-NqV6+QK7n3b1lM3<vmcbPV%*`ROAg8Fz8
zF~)hX*&^zx&XV4(iCCGuFK+oNkfWj}P~rNE#?GY~xqx>tN#|MI=`adlo_;u1@h@sX
zsQ9n=!Ab8{-6nf&mr||Y8M4@heeJJ&WvX}TO^R*Ni{#kzt_~-*chmCRe5fmZ`vrk;
zMtaT;^++M#1Gug_yo8QXJzdsEN+gcVvq)0BV63_*t^=F%;b6A>y#nQ(N@`)78Hv+o
z^nm*_nJtZH1;kPWiYr8~p~m0X-V2(lzwi!MmCiJ{LcHl#HGKtBaL9I~U-|e+`WJTz
znXo5RvXpeU5^vyIb(Zm?E9FuwkEd@`k!z_;AdDpY^QsJ?8@ey{+@LMfHN|S8!Hx2_
z_pem7auk`f;_VEk3{#@x6)<ITH5hfZ;teWMRl5n`TP4jV*-JW?%pAovhsC@eIPLv0
zxhi^36gZk=1`nk-T4y_7Zqti4<hCDv1&O59d%fqSTzKaiThdHo@Y7XaEe)KCc+AeE
z1IQQtDaeKrZZE&-%TY*{T=KcJK{6K`8O1XV$tsO4F55}zWbt$;-!0y+2NT+3nAAN9
z-{wIycKokWmgwjvJ>Gt$mAg@nx6QPH8v~+%8D98(m_PCB(&4X`P|PENG_FSjU-IUa
zbIzo5<qDh;Yk&98`q@eyP-l#emyB5j$zC$W>BQ1lc4gd!UVrr0E!AtXSyW6G^=;8@
za2$^k!<YubnNBCbb02#?RvLFxJEuKmqK9t`Bz}5EG*lo)6uCR^8fU%q!1uSa<ZvV-
zSG8~h71*(Jd#=F)MK{TLMmS((64tSBI1j+Ogie1ePowlDW35odvcW%!DAul#P)KWi
zSJoUQkusjiI_+b>O~NZ@|Dr3NbultaYWNvkh6)yb8dHE|8KHQsQ>BX;Tr*U;YDIC?
zQ^5_Ela0NP_YqW<pWF^Ht{mXnwGT=%@vQmNVKr}uW`^Z2-~N`0NG;?Mj>LWKV&UEt
z5bezPG>(_rU!r;G|4xAzA*zRwoTre2oNZ=K6Zh^4Kz1lz#NCwkuEi*BHz#I8Vu2gY
zR4K%u6OLTmDjG@_uL`u%c9k^YaZz`3vFdd)fkVS^$CHSfP7F+Ae0H4NP*2M9cRMI_
ze$MsL78p+Dps$nzqoR+FsUBqFbQ1xCp0c{{4${HDGwtoe#U!a346og7F<BH7_AGyI
z$){r_M2Z_45uBF?LA)JsbS5NZ5rsuIhB)MNVzT7pgiq1!;D@gxpXEeO3@HVv2A+Qr
zZ!;H*$zTcPz#y;A_2rqc3S>FLYDV4r_|NqIqeuSV>HXTBTZ8fwfb^y(iY5eobVYnN
zn1C+ZY&ezCB-ajZ9_e#A<S#@qo|1HmRY*yvPWIRjp21@1U~9cVoU<VmgXFjx?`Wv@
zs72ilnu9;EoScSGcr}EXe=YlS`lA03qfWfVrx^D#iJ)7NtgyP}c}w0&^>U>Zh~df9
z$mZ<NbJ5-DrOPca=g0-gWXIply7Bnlq|LValiz~G_m@!VB<2eBYt_$KG@k*L9J~E@
zEt_|fzXb39PW&~{GT;^BJ=l&~MHK{fyM8js=hWPbpn$~D@(Av6^g%Lk@OFalP!{qV
z_aVlGZ1vGZcr|_eVnYLw>m`r_ckJIM$TbUyr5#Uf_^CPq8RSfrCEI-Cu-R3EjQpa&
z;zSPp?4&Q@=vNL4%Ddk{SpmT>;ii5BP~+A-dP-3-;lyV11PYI&yyV5Xe^Expe^ACO
zK{&1c1FCZC4^ejNdMCunlb%-H90FL6m5!xAbly~F6IGR`81UgDCmwmb^@wPFmM(@x
zXS#9ng;U{ho+f(k9`SoV8pmz7s_o4oZCQCP!u?7vHn6XVqJD+fu!ml9KywUnug6N0
zE+;R1;DFI;-wLZ~Yz%$rN3!8*DKd*}byB~`Eg@A2FpudwS|;6lhB1>kRwq79-o?j6
zK=_Ibu{(CU1ID>#$g@m;t3s>48Yr>(6XjXWdQ}$b=S_Uf@!vWH(F7x~0J_>?mk6W4
zCiZp>e^I9Op{aGjMJd^ix+1q^Co@T8jNs|oUmCSS)kfw!WRT55-Ajh9|H=~3o75En
zr1X>SRpHIi$BTxS+je)7K1&Fvaz~@*Q0b!#=b+$3qpn4~C&ELaD#bH17Zg<ypm`MS
zN5;2%x65>MKGwhFbVLU^Cy&Q{qNv>^MevHrULEyzNLHJB$r%^ukOQ0|A8GWI>D<N$
z;9cYc#tayv?wJ53B;p7XSdi!^qGwBsE8ZQd?XtQqjXi$C6u=)>#u3Z&dHkVVS4x0`
zd0Jsi#B|a_SQ4yRqQf`VxQrJ0Aw}>JRI|tO&-)SNzcogI$4Q1IFW=+VPD+^mBolZz
z?$#n07FujlJWtT0RZJb=^m20|!VhaI6$qyX;8u<dBu!2-j>7J^?}sPAxIY=prb|0%
z=lANYwe<X~WOR(L@hay5a-iRyob>50VV}D~shQMq2#bda&`dj#zrOkB2dI)<W=~Yh
zK21yOU)&Bbto{IhWN=0grydSm;Q|J;q#}4$_EtOTNQIrQkHnot29Q!3is3Hb&V#X>
zkT1ux^YH-gn@oFka;Vqo>ZHnKw<x);`3)xk;()#)zy|eo?^&^*?jY7gS;6Lyh;!jk
zOgj)K6s$(m8gC=D^zQLfoG%aI`8tv(I)kztjXO?vZ(fkq7{{{qc4s(dJ9TGyfpRMy
zMDb<hD}R?)0#;xP&CCi)1^q>Z5_%-|yJehrCddW!^nC%f-w1_KU;UvH3#!crdnTBr
z*W(TNvrCZ|$DSp3&j^3222or<ANFy{UA<=lz+^|YRCZ^(lvnIho`60mk?{@2sHj^4
z2Jn5neyqK=dA$AD>Gx{3e{0nt15WwLchr$~Eu?%DY4GfbdMEutz~`a<j%q>N+uZ3*
z1BjWHjHE%$V={1))FMRoNt1a#!Zto8SXpMVZhtqX0P3+0qP08^*XB^plXG@WW7YDw
ztt>pB{`?E?fwst{Gzx|lC7n{Y_wo59QzG$bitlHmZKPCY%*e&l6)E}lskq)O(daiv
z05af!Mghmw?{WrwECBkgkV;${#Xw=loddDN8izP9?LCzOrTol4s=L7vdi;!!gqP#I
zNowY-NR&wrq*6;U%NXsGlz;2Ct5lMeEuGXwTIHno_!+iSnL&m=za^6<3-K%}g~|OK
zonwok5b=IDeXJXQ&NtM*khX2E6h>#P=@grJTy&`sdE=05ZutkB!IH@ONj;57(c|Y&
zGgDVFQ$m<IhsE!QjwpKR_t80A6JM+#A30$POeyHK)mw1`dYj7n!2}j#c~PbQ+P#R(
zBHsqaaF9^g&!YIUBJPJLEoQ3uOjC{cGnrl$+DV!i`^8_?E`Q6uT637X5G!i2FEQ)r
zlKS2z78KsM%;OQYt()cCkJLs98MqA2Beso5mc|j<sOSAl@;OSU8kfQbk$!ndCfH-A
z1QI#}Nb(SjGwL-wuLPJY=PIyX>UcAjZ44JsPzWHV)-<H~N*80W#7%n(jLp7;-mD!u
zgN2ZL<tQf#^OsfV_0Zpw)Zy+O0|8E{=2t&BraTtKoSlD7Mmiq1<xGZ?3(|lD!_Q#9
zw`Ttj?&qJXY}<Tuwsim0763llcnccrN_#+HQ0vHtFO<pjaqmuqxhz#;nWLHfx;k8>
z1|J0>%5Fg(A!KAhof$-XvH(&7I4~Wh9f#fs_M+97+H+uxqeado^3jj!O2*o`1i3it
zM&9(w-lq;vx!2|eRZ?Z)m!a2c`yZ2Uf{TOV@v-rkqxdB(EwUkPIlXI1dV#Fvd9UM#
zdedHO^vyMt=1>8+3zmI<WDmvYd3QPbnB`IEz;teHdabpD%(%6lPvPC3!l?+;?J#^9
ze4i@k&Mw7@mXJ$%cBUpnIP+aBtcF^QQOuPJ^L0$7k2Q|Yj*PEES&q(AyE=g8>Ct7X
z?y*?8zuw8jDQ6aWCoK!qmFUyUcBA_)t|mz&W~=Tc|NTKAHM_!M8hg6ioQ6|mFf@L~
z&8E}wLb+_(EjXS&Sc(MJCE)u27hJB|C^PW2|G6vdnKJYnlzZQhvEr-sJJ{;`0ZZR~
zy4R<%8)*^S%?mYAt$LFTGPUa_;ZLi@64}xXwCc4aD(HqkM~kG;dL};i1xSgCC8Lhz
zTO0SrGZ+Azz^Yn>4x-^IiAIjnmur)k4@2uhCVh>)Ko}iQs<lZGNsCAIfkyi47e^`H
zH*-NAZmU~pyEW-_dAiy$7M7nfa$380I5W_D!u^%CzPny=4<XRFc;(_tr0N>KxBe{3
z=Ju+~z-Fr#vj>pwY$I7P2*YgwRj2#`aRss`FJ7ooRzJ|BNdCn>Xf~8A3F1x<MSx?O
zT%W`EsDY~OIP9<aj?}LU+O|~TQ>Y_=0Huo0c>qo^4#_Y}8nkEiPn38v6i&#CO{G$*
z%TZv^_O<WD&jFg!1gmM=<b!$F7mEN?z^CEVOI6b-6yODYXfc~k-BEmjiiY;A20u}%
z&Gt}Dj@RvHifO+A;U3a2j~*+c(BIlhdssASa=!&c9XE>8i^l_ThY<TKXc*~!xNrkN
z>-(t$RIIheVixw-s3V50($QtGhZ~m{+wPLKrPzj4x@K*UweidmC3W{5r1tQUE~PY2
z$is=e`}yF*3z0j-M#kM~7H;|}u}7NW9Thyc_St}z^MVEjJGxsWe9HZ&Kn36?5?u)w
z^E{ywD1h(Qj-!;u+tW>kX}m&j2(J7`lp`|Wr-(3@N9;HaU0{%tCFB{@{>X7=$3WEk
z$sunx>Aph@aV8kTJAV9N@|!%-opDjOWKhHhh>bmEGT1ppA^@WUkFZ;wovJ!Qz&gOI
z$1jKj51qi3wBBKI@&kFTVdXH?(E84dk_FST8he?)J&WSOKgiB+=a-d=xO_Dh`CXoH
zS2@O4$n-aN{BB*a`-DFp=Gjq&AI|qiT^-%OPV|65K3=$FH{Dq4zr=WT9(()mx1>-|
z5DAy*1XWVDbaY>ZarYP6r0?jXF>E)f6Do!zBbm!YecxUa<;chKz6XS)x;=fhW*2GQ
zW!)}K<C+y@u{>71H(#oKJHMe3YgxOFk)jKKr`jGg(P#`=SE|eRU&!&$NEw?n`ArP?
zel~gCx!H63meBP}C&yjL$!<~Jrd?ax#^YzT5`ixsL&q7)lj<>1?lKK0-KC^&Z=&#$
zCOvA^h-P1UTx|U0PMR5kq0N=)W>-!yh7HS$sidK`YzD~a?JL}v473aSqqbFIg-#)A
z^g;G0zar$k;<P%F+HO{4A4ih+%f>g}x+4wFA|D8`U^#%r7AHmRDaR*4!;wF>|L|jj
z-Ip8sD9;Q$x`!#>^=Ja;>b4`jCat25{tIMh-Mgw{y@|2owCLHOh=N2DD4E<XHyO8u
z?(wO$nybk6HD3p+&sKS-AyZZdmAkd+ScnU4@47||x&R;B%SfT)QodP!5Ci$+lCj6U
ziROa0ZE-B3$0Od&08J1ar{^Nf;qiKYFQ#f~uQLuS9Y~L*B?>@)#=zk}(9(2mqO{H7
zc)P|E??jwTZg(+=!x{7hp{?p-i)88(Z9~P_p4agvfSxZV$L>8G#{8PLL))H8y8cIl
zWK0b(ulcllodwd}^26`9%Vqs}S;VVVwWNQ|4nr~5p4cm_lFR8Q#&5>dN}i`)?8?}G
zU()DJT^7p-JN416{X{sV%gSFSFEla9XJ7s_B#+*+iLJpR&}94{4}f1j;eVl_6XD}i
z)m~bF<675WkiR)2`4d-zD>yAfwM#D6`48<hCK3W4JfQ)E?LSM&s>ga!R7@62BXP&<
zs2V$r<m<qfiLdKO#3g&jO<%@LQioaS%LDQ89B<ywo9|)J)6d3~KpoJEzte$_7vuW#
zB0`q^7QZ@v?D&M#I?LRVxzo5e%E^e9>Iyh1vnKQZqy8^|Se;*)+N&Sj_TU3!V)~NW
z5?Ie)X*}A9ZzYA~h0krz1u0Ne(wWI5YK_rKzwO!ax}7pPRar@lG<EK3$sx0Fvb67N
zurk}c$ytA{T*;6hX@sk#Vy0$-%v6^^a}>z;xw(47`QB5Tz~^FHY2_Ut+FlVar$Upp
zH@bPS>TM*oxD10Tecn68)Q_<|4^r`>@2BEcpiHYI0&r%h?V)!PKQ_kTuK>PR2LgqD
z6x5f|ZzMi5s53`o%O!sBi(sWc83wkbMecbrnW30tj)_?>yXSqs6J|k@%8<*qYv<H1
z>w`Y_W#6yy2IIVHdo2Hggqs3XzAGvQJlh&d(WyONIFuzZKe6veYJAYpY@a~%{$@|@
zz>^gW^U&FDB3u;;L1ZqI%mqh8+#fdD$PPb&A!9t@8GnfF#aP{*He}H~CXZ!coXs0b
zc~#GPLXAQ3=r4sB=_U(?P=`Ri3nS}SY2w`x(q!>kmuus<+yXQKm-xp56<<U}gSjms
zi%El40yX%2<{fWfoL`49fVRHfT->>aY|9=Rl>TaG6sZytO5k(mh~U6Ho7l!ep>z>T
z2(r<|tQJc%Z&2tl3<bH2d`%a>k3{Pbo$hysf0skVNXWvU!%nMW<(@&gmeEJ@KYsYU
zCNGOz-yW=(IX`BA3P)$<g7u_}arZ{is9f?zSmF=*V{|N^IO^Db52A&eUI#BS8p}PG
zy)M4Xe3V-xbIEqmY*vX^lrFRL)HeR2q>%tfLOzutRP#-2%Y9G&c|yqNt)g@=0ay4_
z`5s<6iLz_{&;p9ijeVo^Bn`R}t$fskvgG6|g_ms4b1!s%k(88-NurJ{G`o(Yq%HV<
zS?*h@+`nOENNUNS`demmB1=4S*Rl^gVsmnCGs%f-T!^B`kJsSbnp})TF}LS3mh4Yd
zopuhnzI|i~8m63TPoCHdlmbctd-KQoUG5vL04LA;c;wAFN9W$IJ+2>7zKYy!-sn#r
zMtc}{HKG*oIj!Gh;&f5U!~~CtnVP-|1u~OBO5$5xPSBiTX}x0-CO?7HTxjjQRInoA
z>r1NOW$SX{|78I@K4@THA44LioIN$E!WBT;A6tLBS2M%;O7<_x_sH9HZsl*A|I&FJ
zE$oJndSzkk<ygm#DVN=Ld$^r0yb=pca6FEmS|6tyk}l)z%&zM5Ik(JMND7KlHqFIx
z!qalrXNXi8GyPa%qQR=oB)e5IMPsUDiH_=7j{W-{ZROVuR*I8&Q$#%SgSDwJ04S^}
zZEH0UuS6=GP%(a@>zCiRekaOu+MT%yu17f0t}p@Fw{V7#0;!;DN{zdaNw#vzq;Xbn
zy9;?hsKrZomQJDiDdn?~%FnB$`?3`b{;mITfPuBAeAxFbKAVBe2IZ12kEcX=DAh|{
zOFG^#dlW#JZyGT9&gM|P<_SZ2Z*lD_a#;$zEy;II3)H~QTXtBkdtGjXL<n(zw4Ug<
zxm`v1lmWaXNtdiL50;%0#243&yL%G7X{0%~5vc5Cx{b62W7*{!0Bh|Qw-%mSp?{rH
z-+w$`as7n+#l9MDwL6$ywfFJ8a0Z0cjlRE%`$v5_xX|LDcIUf?I_GIMJyrLwuj+r+
zkPlawZYU!U8lG>U=ue|&vJjoGW+PwCK;~capU&g|4`rVY+GePrJt)}?Z3T5i3j1@&
zuat;MAT#`Y?Hpr)K5mL*8&lhl<%B~LI}^M@@LeSxxgIsbeH=l+lZ!i91LxQg$D2W&
z%kmAkNP}=lQU(wj?{yMhKqtDNu-G#xYz*ZPBR~hdJ58>w(f5SHp)GG%f4_cvguV9O
zh`^gU1_-ssS49v0C5f$t9{GBE(ZM>#$|UV9vV+VE3N}U$^m!>uu(d;pvzM_sM+Fip
zxBIHaj_$xl+zEa(4mUnZyhFNX=OnRFz<&ga!>Ev)xbT6Ib>I4~?Hvl*Gv;Qm+seJU
zms)Szf5joy`<1C3@O>X7T<4~c25CXUb%KIkH{}<5ld{wUAYUqS`x@$Pq&VJ^dU5B(
zq>~#HRoFH0$iQ`+CWYP(#k|L}&=L+t_#}`Cx_+z*oO+#W`O<8J5+V<fuOKUvo@X2?
zemw&Gh)QDek=|zFJ{m&!9pf?;gN>v(4=X+6AUSQjXV}*+X4Yn_9yXHL=<)+850f{o
zNh+ZvI~J}L?<%E`<%xH3*%xR;RYfKl=cu~8j`@zFSMMMuIDDC0`LZVc)zso&B6Me&
z>7PF&7%d=`2i3nur#9ooHlM{r4=)##Z9gZ10e%d`Fj%3J@VAC6p+Ld%mS3uCxpYm;
zq|b$guw{fEYi)}Lw2#u1FPvgYf@4JL#67}p#!yhmgxr+TzmOiHeC;4gVU@?QT*11u
z<atnv5AC=afy8cR7IF1UAQ3*3-*w_&x`=n2Ehh~084^@l&Glr9mg+rJ<|dCzIxq%&
zjN>AKh8XSS_yCcf^3e<fu3m$USA+9+`>cr%le&pHG9&Cv@>Pn0I?!nJ63afMYTHr^
zjufOPRNLpj`-rJq(vtujHyN&NYncjSQKZ5-kr#CMqbPZqQpkl!@VXEI8p@IUIgT)J
zdB%C*_Qvi41Zq$4NzK3L3^6UU?@j;hZg+`9!_0rNaE{55@k)hL@QcMwdi=v>Ne%C^
z(W{`NP!(BKi4oT0G`D66$xJ0Dj>gLG*Y$>SoI5WM+`Ww3r;x-<A4N}VBSW%np$9?|
zsfAyL>!+j!kIYO`P`;NS8M)m0qxF_&__!UWx$p+wXuIow_1E~5Z~k?O6sEW@&z5zr
zj)b@;0=W|n7~ZjTGgr1o6ese?@UkWx2+>i0)%eQ!KHTeX^VwHs0b?DnomgC**?(-3
z$8Sb5ET#6lO^s>-&SYfkSa6FPOU$IG#v~2*Cc<ABEM}bqULA?BzLjJ?=?WO$n*)%h
ziSYciHG*KKcYyq>_S~}ZzKX*^0AMsg4tRq3Zpa7+!Vcu83KCdBZSN-b7XFY4={oTv
zJRdpdmZEt0b!Tbbt3p(}L+LRhhAna@MTxPslICIoVwsloK7mKK3pS`vPI}MfiuQc%
z=Gb8HC&c((+rYXb#yfxtT6#eQ7Sg2R3n9ctbKu(f^-Fwt!jH@!U2ASjkYN}AP34RX
zVmYJ7hG07K@1*(cf_XVG0NC0+Q9*Hyk$Jw8lU{d!$nFxldjnWr*ceiLKwNZl{J2Sa
zvH(Ag1_@iQirPxX`fce5fx>=#c=ul$+|9d2mPZK`^A$mxxoS>W55DxzH{ZwvzkKvJ
zrx5v#*f^nawa20GxmM2!jFj%mmX@c}(NZ=oPAl!>ezcTt0+}NR!_DZ(j=-11aMR4b
z)+qkgD)ehto!01BA$6s{meKj{Fbitu)Ojs4b0i?FajYB5W>!1?COWC0rpwus9_pbU
zqk!_9tm=9~p9Hy_YtYFzLk@Xf|5g%3*B%Q!q5vkmWVnF%wAmkgBPmN|R5M=23Q>3B
zaxTIllsby|26ha-aVTe!rPJazTW5z%XOFce8B<yBPshIHn8c%dxSe;|Sm@N{9cdIl
zg##8)8q+I3t#m-lYf+Lv`0Aq<NDP*KtrW{<r2K!#dh4(#+pc?f7+PACPU({FZt3ps
zZbWKm2|+-*L!`T;B&53=r8{KkhVSC`{=MJxyzf5@91O!T=bU@*wPUTF91l#?Z7C;<
zgGGWCI4xt0XxqycOh@vjn)Sr#G)r=}9aerwQ=?~=-_d4<yp+wkDD{i4EM%Mz0fSzM
zz}=CZq175ikqW3ikNnc&b9zm%Wa1;%&<X~><N!)S%J}xnV7X{o7WhHp)YUKEiFyaT
z2~h+B#Vv;1Y)v3vSby=tbIhbh(Nuzj(i2Tm^z`>-H2o4U`4#Qa#KMXBliu?8H10_j
z;|7Gb*^l?#S|Hp!^m!y%m)v6npz@fCKHczsMO{2fSN9F$-LN}toj`Pcfgj?m(5=Op
zywMxU&@a1QOf->zb;c3fcYajv_Yq?`31CzdToS3R7t4#esH0ANh#_XXC9}h1D^J;U
zIBcExu-Q-f(8~kkcg?@DT}$GsXMx6soT@K1e3+->S-B>NczW3!hV}t8vY@t;FI}3O
zQnTA?>m-*+J#%q=l1DRnorgY4#7~<TdL{QseV~7m>5{|l7Z~k_TWS^JzY#g;--rya
z)D~k8MC9GXANHWM1|dgE-pGUmKrJbbR!5d$aV$$9>5s-~>DWl)5iV!Z(p>wy8V>v)
zi;RQW@5@vA7zn@DJjdx}Nn`I?{M)QhZcXJkw*7277{7m1n;k4-h1+AzSDCm6+==G?
z?n-4(&KC_P&((;0bqWMA)r^97)$f4WvuTyUGc`oNlW0WFZuI-?{ordTg4sGa03MVr
zP{&hL+KiKe0O@}z93XU*laKE3G7RKH=0}&y+%Jyfdut(Qz-e&=4_SOeYBrziUw5t~
zfM7l#juJ%V8?YM>0e%*|28qtr!I1QBH+P_LPFNScMuUGF^1V^}bVTwE4^QAc-j?wv
z2*Ru8L$_$YA9y^La!){W?PgjU1&aJoT9LHDM;^nQ0e9*t#u6rE4>bo$9;9{puy=Lh
zLig#oFxS&zkZb9vBI)c{QPS=QGFUVa2Z}xDbF|zy$l5<R-A2hI;A6Wr2n{EeT7dxV
zt4u0e{^3fyQM1Q;p<&c29%j5hpfnvq%fYcbKXl`2jr>{FH(JOePcn&kY3(*`R2%3y
zsWp~0_kOz{>_~_M<X!W}sFbpVzId=_;#(=VSsS)g))SHV&5~Yu<*dg3;q)##l^wMw
zspT4-Otl0Ern`xG7FQ7li?b=AW>k`ItYk0(tUzC3My&G#XiK8hJ;ByrOLfm`b;rK5
zErSGVUHfEpu67oU1374MXIewq^W>ESGGXcwb@4s<M*ol{I~q*hoowf;H1(DUXr?Ns
zdqS@RIZ~&2roGi^wHYhxj+f*yz7*#<v>L{-sDLj5{BjT^SNV45tFqw*Qd+z-E-dQS
z!j9e-yjT=la(G8L5HB1yF!|8Re>qg-rv?j!5drlhNCZf)y}W@>l@y`9nXb{~{w<AM
zP!XI-hMUlA^Js(1%I+CO$RVD5U^Ds1Zt#d8>wnBgOQF6;r`+|ueyjXcF?KSrRoYxT
z2DuZMdmJtbE+d449VTcO@iRCJn?;Nb%(qd-v3cek9oTh|7%JnGO109fHvc-%v|$W)
z`Awp(uT9&o(W3I6?poLIz*p4xu`a&`D15)I$wFA(Z`b8%@YQWkGaudd$0lX{GvOMZ
zqpsz=`B|ugo%TYbNlHg1?H4jah_Lw^Tb-74xlmlC11Q%e`Ha&-ec&tq+A71Km5AEO
zmJw&Aq3R0O1kKy8Y1l-&K!sbHI$(P(Sb9<y#xs_wpEAI#fC)cWXWJ))jZT#&9%iGN
z7OE~Fhd_+o(d}w8U9I<5ia??V3EENw@IyaYt#Dr+0xSkGLoSd)I<g#<S;hTZ(Z{Q7
ze{<6orLEz$0O1g4EZ+eHgKfvyS9@$e^$dWtfyTAH1wVN0B@BSj-<s5YmY6^Ss`EKY
zKhScJ`GhX*`zcoQKxtF9h@f$@U{qvn&?;y=C>&V?0rzDUNT5TU^cwu~b~6_~6eMJf
z;#M~xw6mM+DmxvwMNk~cJ5OD$WDvauJCCr74Xj0p70q^={z$YTLL5at#_Es*KY??1
z9>aQx0a3#}c)>nllY&Mo$rwt=#ohbStr{9Agp^K3){TmJxEk>G22VVOF9jB(J3ff@
zmS~FGd>2~)pq*1Ad60MUo|B{KP4^J>Cgl3}rv~6gzh2@S0F3^;NfQ_kjV3p{4HI~n
z!u9lg6Z`Y++PD9v@;#w|(0<KT80e8IWeI%QpUG?==1$srKL<B{$XhKfRZg@wQJ`aM
z9SQD}*CfM_L(buSf0J;2`<<pR$X;a<In_qw1Ih9i9JMq=`lkiG_xWOLFT6=C(m-Fy
z1GYzD^Z93tl7USjS57iRHcU+E!<#R&Km>S|dIecOv~~rDF!(&LeXj3KZcOHE(dJ!J
zeT9nj>po3;*Cuhk*0#hCaDPXU6r+6kxXCi?aPe{`|Ho3Axar~kQ7O4ld)8qp{#y`>
zwP9@}&^Z^a*&6)XGspNOsT>L)61EP?2oMmIM<h*s*Xz&TPyZf{q{f!47Hxl~c}hB4
zBgwWtV{W2NFpQ($^0d=La{<phLi9VH^4=-cHx)h5pk#%9b3veBe^wlokJN~GXgapE
zQYqqX=7sJT!Gcuq0_H1F4kk#PEFmwA_fiiR69y!jsJ@*Uv#&Y92J&`yT6-!jk3b2v
zcE+L(^B`rW{a_Y^+4;4S82%*4{;k>eWy3uJ&Q*tejs$K^1`EaTd-H0Q{7Nxdz`nG~
zqXvynzigrBe~XQv{<c}}w*$ijZ%i~vtC3}WJPJ#Mu;5L2w|YLpCa;Y&T290ElNRJ9
zI=XZ~;^%qPX{g*m9W<OXB?1`HNEn9+z&J{v%q97Hh&>3%9v?r+WPdq&PyJ>PBK#zM
z^SG4FQ}_SusOeujaxGl8;(E3t-Df*GIFX}N`vdlN>gRI<*v>ituk4z5{)oNWUj-!s
zU(X^qDrN-~nh9-!0@L^=+{Sj#lK`vl;h+??coR3*Z|t|T9?lr&Nn<~b)RbFDcQjDM
zvX}@&ydG+~8jGhH+pK67zGZs}BX#X>^!+gTX%D9wfrzY@Dx2vK)ftG{Wxe}puNi0!
zAOXDgZMFIuw|x<Ou)enMrBcp|UeDziR+2;LHZ_UnY6lTI)CJ_!!MkgU3PegOWy$yD
zH7c=PMB!|sUrPPw5kwInyA#5_-SoyJh)1e80%c(s#UP|0)GgykwnMXMzPeIv9BNqC
zDy1hNl1a3z=3=bt&d&{3{phG}=!Ty}=T=zym9QX|lpZQ{i-hTQ?bL~8M&ec=v`u&X
zC)=!!Y*&SZsh4bciuxf4qdgXOjnWO>H;NbIsY=v8woA1Aa7tKjw`B}K@dM{$?<bkf
zSxEJWQ(p&cFAXI#2V1F8eg-+|0&IzQ-e5t-yLQmHdWpJB8n;sg=_s>7v$`eF0XMN&
zt_A|yb7{ULvs8e9gMC~Cp@tG$r1q$vD;2uosR+M-Gp3`jtL2zyq5P+KX)e%e-m6j>
z9$vk;)B+<*kTy6OwFbmdSecbXvckjOBoY83_>dzn0b{*-_Fe<M*kg|`jzd%x)c{E(
zONBvNC)Jtf=4L5dt|@4X7gc5Gke~eJr6~Ca<>7B@Ze?+bA6`N8HHe?itb-iD3_C$g
zU>2BF*gb$^+w6<pXBm0QufWe!Ns%zeTMS|u9$d?+?}b|;684q7{eVYx<jwu1v**kv
zK7H09-dFPM-K6;_vl@HvC@%CJ^{(;24~-hv2|iXBP<(qDgTmt6j4`$^on7T16^ygk
zn^n*T*L$D-a>5TkW($vUbj?-p`JDo&*pvf5MGGOm))^+_qjghnsm4CM8WVn!4K8sA
zjz#C$D)(#1rDsqPveNvTQ}Hn%E0f6S+y$a^<JhP%day(4pUjJm3%^YdT3MP$%y&vM
zd9liaDb+IOGw|82;YZsaldL&*5G$zH^NsmbYV*8~A;vMXrA<(o&Pz16R&|6-cA7C8
zb~-hhIg3owzKHz3tYszdPQU8EWSCbGf5|YkuALC)bo4Os%6$vi@oVFvBdWhm+W!#O
zfy0LRMbinxrgysVQXm*d00>O-#zy0H7+`B?<WlW^V=216HrybH8Ke(F5?DnG5MC7s
zpy@^vi3!`+#Y2Qzhk3%+tp@V|FwQ!Vd6mdF<&kyly)&f130frglpu`)iKPn@dJxBb
z%#~w=-|h$#(uMvECPDur{T?=|7?`wGLw7U8%J7l<1>znuHXRmvFt#7e8~bcRTtL)I
zw%dWj9it!~eQ5(LN+TLd69`QF#!8c3j#~vxAi9v0^NvNnXkK+|-!;Qv%|6rEFwK7a
ziKr6u*o1RB0{O|c(?mKCA{AZ{K`wN9)<u!|PmaPpK!%ft4qvVKGqYQbx*MrBMv1@3
z!PmBmHJ3@0yvsN{vtNMDAq@Fz!*DE2Sthq}-r4Za9UCTpXU{Mu7^(ss&gYnX-4hIs
zotQ5F{%_gTp(G;MFjN^)RQryi(xNxY79R$ZsMGHf80UY8SbbaTmag>wX2II#yb(04
zsq;RFVmit17B##6N8!_+KYwm*UYlcx;?Lmo!`Ei++j#P8LrCm~Wa-`2!8A;tOR|%+
zEIuyJnFUCL-nbM-MW3@d`{E8E>~KF;>1-?BMvIre-|YX!{!Ke-huFLph*G7Af(VDp
z^re!QztEYgH?D*&I%9QCb*2bZ+xGR+tSTPLR^T%=#FYg6Q`_9XX`mxFSFyUwV#p-3
zDu5nHlN}O1vM1~952#YAq#7e=Z#*Y+K#?=)r!gNWusf1eORikIQ8US}l@AZ?$!1qH
zd%D=gzsYxvKJ4fB<^8-2)!&w+cEUWB{nOY4r<(0UIht_|#&3_<id1C+%LmfuyDW`w
zy9%$8I7+5*kc_L1<{W1pIt*hk?4Po_ycdD^A|(dSVI3+Uk5|jvCobC>QT69GEs9bj
z(~oI<s~~py#xfY}-xI`HbDmBrkJaMzw^>B{JVy-Z*{EzM-2IXwM&Gfr22ht<VXtBA
z|0{C-Rg)qclnemj+Y>>)Z`K0O=i?B%alQKkk;!0r^l$qPjAIW)6Z6}ae9X{7oRnS*
z0+YZX-tq)6b)5OFf;$jXK1z6*(hjq3*@U@`e<YsohVkhpNvSzhOS)V|fioC<mj7`X
zhKM9V20KU|_?;&C-t)%b!0E+)M71AB_k_ZO$La!EcgVZyODZH+T?}y8>xl87N!G#X
zE=D|uY6y^F3ffgsL^JyovpNA_bVyW;^bGaoEg96M5mtevk5e@1abP22jc;5Py}1CA
zMG0*r85PHYdm=0Qcv^)VRJIp$kk9lzHV<8I&=g&o2#EJE9P}@~Wc=0HI>ojDa0~eP
z@_%~GV19eP*DKrEK==}*@Jo9?(DSUgQ&p_Iuput}>Ym*q<h2MkgA`*b&?Q~Q6d_0J
zox^Z<b111xoOAtAu!AgNf?SAcmE#2iB5)*=xu+z6o+-9{C~I8n@|+q_*!*c`Wjh+z
zBqw|JiK*_BPZQl@MA7$0ezb4)fi<~Yr256>v3L>#`Q0+uP2qnQE*qloSG_Z@h<~fp
zY!8=h#lpa+D$9SuVh?@0@=6${g!B_ad-tcKVUfU--AA>pv7@_*dL#=+OGZL`kTbi(
zH23LU?@<yFUu<6M$J(T!7&&fPph8j}#rvw)OEUG1`94|df?J0fb<FH<LfPs#c)9|;
zgbZ5;^JSQ`uEp0|U3`h>_jzG4`7)oQn*p{n*UKQ>AY!s+<L#@{shVu@oUK<f@j{8H
z*|S4&P8EZ?zuR$7-RpqMPN`X+HX3JZc~}MyKXszn-cG-lNTnxrv;}S3U~4>>Hx_7C
z^uhMgej(q!gqa8ouCuda3<l;hYQGy;I#|}%QpUH|^{~lVF^d%j>Xx_lq{YeB;K86l
z+v%@z5IU9S@L68Sdd8qCyZqG=?!Sph{l5Up)@u2~<ujey6i9@RGq8QA{+1p7l}sZJ
zEFypbn^*atFT{<_B8=~jUvEanz*z4Wk=+z;<njeXIvc8&<RN*2Fm;Dv_ycexRwL<x
zg`i;T0WAX*LYSovbj*-0R{J2OrQKj-*ykj_1+NQt4n^p;n>)5rI|=P_FXnMC%=fxm
zl+c<Sc4^m5-af=s$9Hg?dRty^JePDaOg;q(m|c5|_r1}YG0JRSdgT|R=k_{25tJ@c
zAt+|}Ezep3>0%*cU+j*IWF#vRq<`I+21+Kqie_Gn(z0sCv@P(Uf@5~RP*M@eJm`Y9
z&9{qk7b0{iL(;nnXaVTDfI`rzGPakfGBA_=lOcN(2_8>bP1PvX902V3ck;`MjaZO%
zTHBobvhS9{(be;mnO~CK@5w5#wJ|1$0AwSQrx}$XNOQH@b^~5TA7t){3KY(K6O4nA
zAijfsV~#-J_IsBmJYJ-ilf>!BWxgi2=J;HQ3Bh=>4rhvqMY`06X)B5T>ikU+kg%(6
z&#>isz(bYdWj>3SThXWFf9a4;inV-U9+(8@JX}XbQ5J>W<)ctnW1`#(2bXiNrl;X(
zxfvxC$CzImHOsC+UjPYSZa&9)Kc1GK$RdTGl2vX%T4yzsD0v&@YT4eAOl!{Sbvb>*
zI|D~6;(xJzaSAl&cXdNhX#wdTvEm|hqw|@E^VG@y*a1QJY-U=tk1tiI(b5<HX_D!r
z&pO59n{7Y4^n#d<+tga}cxhZazB;N5<u+uC6cN!tVn#$qVy6p4rTw?~Y=twLU8<b}
zdJuLsS!+2SFT%T=bGc+@^3RLwcbW9#r6M{bLVINge0EYuhCiAB^l}>rV-1SK7c6YQ
zxWc3w{r=c2SIPLH?QjxORQpO>S?43YroZ%av*3X20IP;{!TjH5|G#=<Dkq0`y;;QQ
zVTR$r#LVY2lgiHj)&s;$q!0-}H4P8~a0^h*CML)Rwkd)X99~ljzZcL%bVh0{3c%>-
zf)%(zBmrUY1t0^ET`gLmXZF;Z6(dL}oQqD5Fe8t2|0nJP?3M9t82VPYS%4ts^=fs#
z|Db4SjU|fLVZ1jK!{GGy3292O0!fv{ZN9^9FzpTzuMu6>YzB%8X=qpEFg9w7HeQfv
z77>)pDz1rlIQ<lr-W^Z<!?Qr#?RnrVTzkjc@Y`Sfz51}=(Rj;!GvaN-X=Q8&V?e0q
zywOo$@1s#!Fb%fbd113Ij#(Kj+jOCSMHrU}N=WvERSo<2uC~NJ?q6QzEk`rsqa^6r
zZDR=ElL2eVJ#@q+P=jE{pL5${AjjSE?mn$FKQ3;a#UpkCabzdmsxLEZYAlK47e>4B
z!O^OyAB^a#Hd()B3LfU}4}d-Wh{~ZoS3Fd>xw-)Bt&n9H+lnYc5X=9rsIO1(dAiaY
zUB{Yl_zzi3me8cx8P(#`R~b5Z(?u8i76I><&dUjCGQw_y!*RSCNwMM0{Xj!n$s>FI
zs1xO=Ll7m3CRX&WZRDGA^^e8#;5&?qF2AXmn;aq|*Jci4T96#2LY9Qo$i7`-c(cb=
zD0t$WRioZuo`!q~zQ!O{S=F|)A(+bK%?>Yr)m1p^0c+sYWSN<u*T#{lH)=pX?#!Z&
zQCo+6DqmAn%0eF)>?>$dU1n52Fp2n%?8YrNh9*Ed^@USEZn>S#DXqcBmztDL7b^P*
zIpj(q*SYfQzdYZC@4)G@+@bXX93|zP0zB8VW~&@6nD?WLS2ngszXf3<8*V|b*D}(K
z8OkEn&mu9tO-diT;*t48H{6Q$pwB6^vu0AtT}CB6T^gN#u}gN+-pDx*juedY%!|1g
z(HB|-=fl*&Lz8Ga>z_1l<}5DaE@tZ*<{-)-f*+)}CG%8@amsfskhAEz$Fi_3{$~2$
zA(KwuR3L@@%b74w+u&<>Oe||aEXe$@17H1r%1_1rdVSTx--TSyUf&z=`bwENKCJ)C
z>(k;tdws9}>-94=5LGK+8APgH+;+f$1jK%TJjwJuJKzKQgHb*4@?vmOVBDg-hDBGr
zPD-LhW2v;^A=iaM4$TK{#G6xyy0_9OEl_0M4{tEqoJF#3$Gi9%aq7;Cn`ebOtn_Jt
z(vJZqgjf3UmME;`JvR>h)=p<t;77L!>wzw*Cf1qyTU1KDK@|k1@Uzh`{sFPNAU9Mt
zJvd|#_Mf3T6fd|}Oz0Jn2irutm=0BAf}+WfHrRehaiVfd<C57DOL%Gs0{dMg=c4B^
zNnu&a%urG-Vtjxju$C{E+~2VD4iDz9p+Gm`Yc|6Xo}E9i=}(4QA<Z0;gJLeT*2x9K
zA<aybnEkX&+s_tHl_)N|NuT?Z)g5jX*qZxi6D<H?!J>dGo7%;cmvkPMjD4>#BB=hG
ze1fVF<4JD9u-c&Oz#+S_1Z4mMX@-?!ipfoDC0pg8!1peK+AdDe{pm(l_N&<KoQ!rq
z#b-T^h;`~X-F!Q}1s~(uwGifWqKkN7;@*rPx_(f=pw&%7)842efQ^51<>^j~XkXyf
zh@rglix)ucckGA}*D<5e`<nwlhCN*+&S3dmOk>m`yX50CM0vgYGgG24J;RKxM%CqZ
z)*4kFMJp=NN)Om(IP*Ny{@$dP%IN$vc(Op6!tMuTiX@fC{yV3X)#+E0&PgR&=IPoQ
z<tD$20)+1zwp!AsIKvhLS2kCP8GIhS1T9vBJbv>^$z5Kj*AeDpN3pYl1hSLn5%mH!
zvt&MKm+$EH^6RGU9$CZqf=AYyO3xcZ=&@$LK@3npn{@oq+_%w0sT}>V_&!_?^VC40
zxTt33dhep`#6zX{)Ws`;xGxp!1xgyHU*LW7Hilh!HfK5cR%sg*?wCPRw*Rj;VVA`5
zwS}Vxs(Sg$5QXJp^+7)7e$aI3g)*;y@6%ZnV3AiKI9OwJwlgALXMIQgR;rJm+Ubuq
zz10kv4u2F>83*_-LY9}PfUz{#?m*@#{)WTEqx7r(>fplj#%?{yZ?2zf2n2eJpQ+oa
zEao_2>p%}^rccb-Z@=MwT6bz7uh61vf5e+P;uR15hc+TzYM(!hr<LF^PUTgWxQ01F
z7!FV79Lo9P4J!^j2BS3r)%y{#S<aZ7Hlt`~uQ3y}jW!ZWnK-J+AsA-|ppb2Qlzz@*
zc;7f#vV=Rk@`^>!qI|ORGV))Zx5!_fw?tt4QJ$VVKu)m$WEG{AVp|#iTlZGv)hEy&
z&XeV`zYy;FkK~#eh`=)EXJtkpENE?87(Hsi3Pl8&1>C`>Ou__(gG(=}$O=Ziw$8->
zL>P+^Ny1%M#Suby)uJECQJ(6=)6d1DU^i`&zmvfVM&*X#z>(~X$FV5kL&w7yeXEoN
zp{p27a}q6eusW9tPoIdng0P&BQF_H>w8@UXVsv)m;~~6e+f~ziUys_m{ZZr&8i*~O
zZardLoGV!s2+L>vobQ_n9s>QGE`e*P5^>ZT3gUe{57$Eu#=7AIBF6WQJjOm~j8OO)
zUiu?zfj`J<7ZyYiM!$Cc*`QNK^36-^zr`uvm%o!pz5tWp0<|7Tx~VKCK^M8B4)|+A
z955WB_|l<?fUJqFa#%|SH;7<<Fi|Mky?l9ueqz%xdqaa?YXB%+x#H!52!;mE&}%w5
z(#FR#9m>f&LJ)gPG8txWap6<wk~TPP)S5oS3m}mhvl!xl(AZ>ARq{P1zbRQ9MH?B%
z(gni>^o0vOrlu$_+T3#oGeU<Rg)xSpS=e-r^+t8YR2IuVT?9(RH}&>63#6b#J_A{N
zj@(ipN5Q}m)x<XMV@1Z;o2$5<G2eSUc)>DJHnsD12h|wAEG9&i^MOD51qiM~Kz1+Z
z8#=Pj)78pT{O-hqur1kvd~yNo=2~z7epqKoHm8;49)B~%Z-hlt!T8WItbiE#lBKdx
zdWDOmfzx^Eh#W(QP~^egl+AJ>5X&B00jq-dA#5G&6Jv`cxzhzq&d+*6qKDHdoK(S|
zPfo8lTR*op-0Hgxy+!-h2)2cn&XQ&Fa{G-ff}LQb|Dm6t)@o@5raKm5)zLWm_L#Tq
zq}Y1}=G@}(_xwMjsJPWz7_f}vmj7xphy9&i=nYepeWmvam|o!j4kX`)-AhY<by@yL
z!};16&bb{KvEP2c$`01r7rH5E6bF=;MvL;vVbPFqacjm`a}jQ>1F<^-!{Dud(s+RI
z5Q$dTL`Ws@g@|$mjZyl8(J;*wzM@`xc7S+^ph`y85ej5V)%y0Lh;8?zs@G=x9l&rv
zJ(BNrA$o6B4D4f*bls2YXx^9C<D!wUaC)nOtMO(^kM+v$jI24non~z|YS+zU1}h1<
zH)0K^i^7u%ARrNn17AT*-1dv#7<K!{+gl)<`N(WPKy-2>zLN;L*EBAI`mi_WcHiri
zbgo$=aLKltqy++lF)+6jp7b%?t#NCLa05{}Pyj77B<cSwH3@tLaZr0&KHOfgIsW1~
z2+p`8cw-9i=C=W+-|6op$<u>JBiQ%H?=m$Irb%*k8i*IDvHgfhi|mcD`)4Pcym1q6
zsO3Od35*0S-oSWC>zGCKf{~&=A2|f){QGw71eN61SIyH2sjOB~rKvAzEHgm9aw_j>
z%}&StbJ)g#j^TVJQ;!=?vMc<MTy!0886(>tG3}?g@<@x3EY3e_vpjlf_LDzMQDow~
z*?B-d`gc17qP|~Jw=Ht4C%z7V5@o2(lH5reYJCn?_DycxlbbwR3FLsTA?NYqZ^X&{
z+CM>IyII!Pb%r=1i1-X?e!pa5sX{|k<$|d_rgWV8%_5l<&#0L_2JE|dzxT-EOzSWY
zBVJ{|I_Fi=qrKKHrtBxjMq&%{e3L-8zi1yj<o$cA^XoA(&!!_TE7%%Y$M9G2ue<zw
zkBu1p0HHr@{hL!QN6t+?cUgA{;1$opmG4#*kVnCuj8QPGh6yljSL=)Okjhk%Zn1Jt
z#(-17xzK$4?B}_0L)f%t4%A&?VqHjlcs!ur%5;AiKAtL5)0O=L%x*Jy9Ro)lyXzYp
z`Z+>S6^2Q+gfRH~Q$otr8~059mpq5W48v;sO2hNSP8AHt(?e60);}WU|9_n+d>^ah
z;mtn{hH@Z_`OVPN2jY0I@$G*4XZi|U%&H|N`Z~V4)&?w#hhe~|*EuhO5NMU~rIWQP
z;#n02coyWnYO<UF{}}K%@iI98$7~mnX-V?mO}(_iKHYUAX@<%@rQ|Vgv4p|e1v9!k
zkK=g8Qel+a7@?KF7e$mHzT6X63f+QN><YF9meqwe+RX2Wu>!;Y%0LF>jJ)jvgJ~TU
z5MZ5*3-&`8ML|0&@ShJ8a0k+UVtarjYcamQsx)AN`}otWv{@J9QS^N5;wJ{=rw|g;
z4r{33>2CZP-dW)hRJ7e%MxXm$1V<VNKIaCgFtA8`sonag8P7YGU8o@q7tjkgW<Osg
z+8a%52MIr`3k_dQ1*HLlGyMg@=~OO1B*5c+AJcI+_t?*#K1h^x%vhG;Ra3Tr^UP&a
zRUB8^nZibi<Mc`d;wshZo+RU==-6ZdhBl?-dcsTb!~!w8vZ02zLwgK`Gr}nMa!6T|
zRC1M<*#@uyj=x&p?l|ek)@qF*v910nOAxS7^=g$@8PKAPAwwC?Ub;FMrU67^{K@sC
zY%*TvN4^{Vp|dsEV^?z!)4r*c__I^cC!(+i1eah`-lAPB7AK@e!0WPE_cqM!yaQQj
z9mdmTqerC~o|BeRG|1D{_!#SEW6l7EtpQHicND2qr~}XGhk^Vv1&nqinE4av=<ps=
z`Bv#witwr1_6Rjd^_yb4SRADiBgV0C97A%&UL0ScQbwM3PC55&KPZ^ZfEFg{jRu=c
zEKA$3CWuyITyu^rtV6<5iGYSO;OjdGsaEI;*apqNY!=7K+9&oP<>cleD)rxLqzAP+
z{&@ECmi8}jh8WNj6Op#km6L{)GFz~v@4@i-Ds$-vB+Wcd1FF<m`t3XSZwF_F=4|*R
zm>vx`Rp-bCo5<~^X5{9!&IjCT9HtzbJZ_}I?dQe*rWl^7e|3$wErf>JTDhO;TGoI^
zzY|y}t^Hr)q#uzm$%0$Ijsf%I04pS{m&V8!^;s5juZ|BYXP~bP%#7-ToI$?Z7*TLY
z0z@73A|z*cu-G7j!Cd$lI6$gViT^y9*A(6O>S=)=rHU@{;<GS#lAMZui=KMB`eOcf
zxpdE;Kiy)2Z#+H_F;3@c&l~i0i@?+6OR(&sKRw;F4n457toT7+?;H-E>CfGb49Bxy
zF+z|2jG#h%`$=~+NcPVpv$qRxuP13pxsbh0s|TWE_RX{&d>;}02IbIiK0T4^vE=(r
zs-T$2lk}#s^IeEH*nS0TM2<=!!?~WYu#!Za*D->>DCWpY%L#3D7!d{<ngxoI91eEj
zsX%ns6U1+M6uEQ{7yHK*dea!Z_iafAb8qVgUpL#7?QbYO-t!zHF?kq1Z=~mqc-}}a
zB|68dzjmHtwDu~N)?^|qNim0RO<_3QTLIhp`+_AQKd>b@-i~J3@6MrRuONNWX!TT<
zdD28aXU&_kLdMkvjtEA&1Izh;9s(yNM2;mhMby&nuC!9#Q^(6Bt-+<WcRJn@{O2=i
z9pEHlO<n{4Y^0S@3_KDZuz3y*pA#8^t%SkC01>}=6Tt(2+Et)kzy7-0<s7T$OSXP@
z2e+|2X^U-+bf31%`4`VJ?fPjg^mPq}QE~L%c}}AqGx2dVE0S0%=v<x`8?~a+Zh_Ew
zg!HRI7frp7*_0*7DkRjSVpvQ;I1w<hiYPxTdT0@IL~umS<As?nc?$2{<50~;0)^V5
zF2E{pd*8J&b4ptN#JMYN>u1~)2_t3QFGVX!08#i>omfZ3>{}b?MxIaql+tg7ns*{|
z#M~frz(l;nfmrXsrM+D4{!u)~5oo!WpZwjfqN(@x>ez|<m$Ufoke=a!q>wv!<W1N-
z3J*9sfMONte;%GV;m!@GkvX0?`&*Ssz^U@o6Fh;`7<i<7|MkTCIV@-Jh(GZu0kgI_
zT-sZcK#I<;*NV>QtDI0lVM?qM(0Bx_NTjnWibp0~uwe$$?dE#K#rzujJ0;SHhw3j{
z7q68Vp2ii9u33$?#zjd9rcPJI9bH+Z)_vzWnD^PWP8h~1t{DvN@~OCkjE3=59PCRf
zo4!&cyRui*b)yuyLw|s%zeN=FNC=Ns!uJ<MY;=ijRpP<q;eJ_%E8`+>Zxf+VY)a5$
zjheyDsSDTLmPTJl6i=5B;5!_oS;>(-@1{77G*c%fy*<?^y{+lv_58wtq!b3crwu=_
zg2>kpNm5<Dz=9BNR;PVVg&cCo+4cw^))vuw67a;}0v>sK;Z=F?|DW1FXE-eY_GaFP
zOH1pUW!FRvl&6m80bat?vnlyM4>-Ext9S=8M<n{vljimtig6;MSwR7T3=voi5sdCv
zZD%`yX8~Dl;q1GZdz=MZC}O@;1PmD7H&e-(`Y&g~rgC^(KWSt8BNy{de%&E|Ycy`x
z=JLbcHX^T}@%5=jW{Opy;`<qT46_LPsExKm4ciT{hAay+t-olI4#$m`HYSESW*}B`
z#Je$2dVrSVVRiT8<WO?;`iUCk+#ZRhxybK_$94}MDVFwh?*Y=bgt;2}jJbSv63-Ue
zZ}T#2hbo379O3mMW`soB?gM3_@1bjZn2I@Gzm#&2-z8M-ip$ZACqSvU6(A~@)~B-f
zN}-B5H`yJ3B#Hl*g-QkmEJyMMT7Ky`kYNZA0ZjBqhKL$W)vuoauwxi$<1cSk;nE~u
z9Q#vweZjJR#KL)S^DOv?==$R8@%<WR2eL81`3_|uBM}!!!eH`-8wEi~JYnlxgigcT
z$tLBM^NA1+Ki+G|oupx0RBgo!pf!k|z4$Ew#h!y99r>6faPzoZL~g=p<X(dXY;*jQ
zz5UUtUb3eo{he3JH!hjT$!;^kNAL?lGhnqlgek%I6WVVH(jBQn{caIe7=fM%To)<A
zcf=05y9H!Z(n=&Fa!r!<qLsI~=}2a-rfN)k>ZF9c{ht*sA~wGCOJ8h-8Txg*ffepV
za`v|12t>8g%7yCO!3s8ijB<6JG>|z;8;<RL>>H(PHk0q#>>olQYSuRuZvW%f{yoCL
zmm+_?8#w&}H`5e2I?ZjsyX_*})%=%tBZmV{R4iR%R6R0E#O+<mc0bhlBlgM-g)@R@
zHw>l~DE-~rnyF4iU^lQY%nc?@i>tACH<d`tc<)6*_ODMp7=qQO$)B}vp|2!DOrnJO
z7Cjv|C7}^##U$QvgiSrcu_T#UJju3n=%ibeVcV5>y&5>5)U{F)M97kF(;83h<Ow71
z>x~^iUitCDX7?6qM!e8GX6NZ%my#HC6L}*t@qjh>4uNl76^Q;bT1K2F6B$)sU`3gY
zK2ON(qVzTAlFref^T*5vJLN&^&KNSlFR0$$j!z<UP_=!TIDaaEOS6txe*b?DNBZyK
ztP|>VPK}wf4IBb-pfi0}?ZbaOb(J|N0NYyDp8`0UE@TdhF2Hrg@?J-0+ohcw2=noP
z)?vakI`JxCzTyyjG=g`nv2F>b=t;S--a+#*8bmn7xFh1*eK{I<Xxfl?p%hXV;@k#=
ztkwNSO=mZxX{4;zMFqFRl6=0<WW_Uy7!0%R*K&_M-<j%2nVoY-w|Hp7aJms#itmBF
zV%I0uxh_9Mw0^k4UvVH1H_e;hx(M)+cVLcDbf~M{Xc#pG*`e=#vSP32-F(v?&N*FJ
zd8<o$fwf*$TCgovsz1FWY8{dL`JnG)en3e;B0UO-p~|!_e`Dy$=cs~XxHKfx<NtQv
zfU_4-d6RSX6$pKUM>>qui38?w?*VJ9o_93bd?S{l(5Lua-ZGK$E+rdbaehr4M}<o1
zPD|1_bxZ3z!sm1XBO;8uwqv9Bscmon3A{L;faAn@p)$e$3O_Iu+gd6_8jhgtc*n=@
zgu;4){Jh5^<@N14DsaCdlm*%brL#SeDT!S$x>;j0Ss|VXo1YHyD3aGLP)aDJGLY8+
zs|i9Et!$$H<i%GWlX1-^JR);8m%-V-8W(=JuHRn)46<dcw;i5Ms#lP!D1@Jm+ltjK
zg3iH)WUs-9!smln%`!wxkz->p+V*hX5@$I|=rcJpMC#ev7u7ay^)S2Z(g(Ro>+ZUr
zw_lG1`_7-_3vrQ@i(kAh<-`^@syE~Js-srS(ILmQCOHA_-AgLQzxVD(gqPX}umTC0
zko$k!rGJ1&;8*c~u3dZ!ryQfoknVTGMcbM64cyv+c<T9c8#T8~dRE&BR~x(eA{PQy
z?`f91GbIRIVhQe$B4EF>M{sF9u6K|L&JdC03ht#vy`wbqyUXCY!!-uQ;@sgZOBd1A
z#W}=<mvT^zkcFnD%DP6zrMbQ5_#=Y<Y~~1YacaQPZ?A$@`=RpaF>O~npUEIDVCyxk
z4)`35oj@G)pTp31q|M0fWS2DP1CJD0pDpB3xqPuM5>W&(TwK~%<oBmkN#DenZbr6y
zqNjaOrfN#z_giNz3f<3>PG%mIX51v!-JdEH`g(i5r4(t1@qV}OqN~bQ?CbuE_!yev
zh<H73+inPxcI3o5G><fRP^?~bklg>i$|hc2jkVf;*z*m-3XdY^dXi`FrZ|Tt#C?7M
zd(PbKM0AsZwD)>)`#>9mt31*>$_sNL;-kpaa-}mkY4R_WnOyGkyj0AIn1HtZS0sG~
zF0Uy52HnFy(-j6wY?+$%8hn*za&HH6+jaI<Dl5?l#+ABc>oKjque@J@_E3@%TWlrn
zcrOg6vUBK{EdfEgYz2tMD?`S2^$iUgOBG3s_X|osOGUy`Onzk3<DViEMQXRV`4@5S
zND|9xR(zP35XUada$ozTrAg)R!KG1DG%gWvIym}|W}01P`8SSac4@IB#%IRxEoo}o
zbx{_Lwa(2$AB{BL2Kw+nV<=^DB%Z*6(13f=0y;MEh$zgbFKcW6m<zw3u7o*u+p#na
zKiB<v2So=Wxs(%QK8<2i_|IgV770!~Gw}pDt`P>i%;Q47>xsg)+q&@Zx0}&BVQ8cz
zr&G4kgOS3`zCMSzbFYlV9?3Z(@+H-U{RMUGsQheQ5|QXON_tT696q_qk;3MEEVUrl
zRriDG$;Exg%Ca?s8$PX^O*AO|cI30~wY5-gx+6U#gvZoV-KgFM%e6bbf?#(m7(K#a
zGuS=v#&~ROBt<~rT0CcORf`Aym8|}GZa&~qeJS$x_yeiIT`m8ODxcqW0@7~l?z&=5
zaqVW@tj*Tw_eonje9AQ6V)Q`EGlviLN##kSG5~PBaW-d5H3!1}O$y`N*0Za&4oyRk
zE%T-)?;Y|x;||2gp-DU6%pTdCg@=$xaNu^od8<B(dmsK&(Mi2eXPW5qhhk@lZ(M%>
ztq)z;x^pmBqlqp(|2EooOtnEkx4Cs!S^FXDN5dqqz+c-<69MoTXW;r1mo$Je(2=cC
zQ81o-f{rh|R$;nh+}BY%SUaMPg9&qRzP{jc2fRzfYH*PT`so+KR(F@O^Yypk8(6=l
zbgzxW+{Meb(^fDFIHc-d$)z)&2=os$OCZEm&X(NnPLP*Ww)xK3qDQz{_!qsi*f8~1
zkl+(?82`%Y1rwqGiew2w6Yd4aW-JDbD;##8uCIQg*rGr;>eY=rMbLPWbzfE7Rw3Id
zCGgbxZWGs41><KlrX1v6UD$o<c2WNxb7q>)x_Yk%NbBM%mj%*Ki-bHR-1;#5N1q&k
z7!V*95%~|;h;TCJfY6Lf7<&Q;{(rf#PcLCwaSeVUbNG-rdyXQr-lS`20B|LmVbPn=
z=}Y!#h6p=p9AGi1&)>xN<nI7jcgcoke3~dJ(yjn@?8roJw%kgsEAivlvWHi`u`^g9
zf7R6r2obcd&t7ViP(5((^*<3J^dq7p%rCgMCFyQE<6&545UD53I%arF4qopyos`R)
z3|d6*(^aLnB-cpmQp$Q48sgRKEXIA*KD*RnhNp6sH^$w)1XW+ck`}K^{dTGABh=7!
zdj^#}?jx*(!e{e&fp>3Th86KplSY59K0zN+Kt71Y91c%WrwcN>4L|Ir&w{s8IsJkr
zD1!3NS3`-HlNv#Ik*Cl!n6G{#&+~A9Bbb&Zy&O%&sLeK&;w!I}bdz1Dw1+N9&zjQP
z1<nEuh1(wiDUnDVt-VqB!OQs)SmXa>trKXBEDki|uRuK-|JMN2|22SxgAR%*NGhL2
z9OP*Pch+WxvrKa6aDY{Ve0&<sNa9apU*JFl*QWh+qm3?rPUv1G0pU3i+j`rqrtus&
zMmK&SUZizRvTKmQ(2T2-u*U!&WZ=3aaEp6dxL;@L{Bw)XNU*Z#*4A45Pl&FgNpnBq
zV#e!1MOPW2WZj?ga&UyN;awOXYp{dg6%y@2MXP!)_Tq7kX3&3URGVsKvtm?t(<)vB
z(^vQLo=o^^!#0iE2oB~G5g;blAw2GKlHT-Q%+OR!eho=Pj{^7qq)d>5>~Z2C?ZOB7
z*1bE|jqSi?85EBqxv_N%qKlxRj-J93o|%gbz`L20g??-dY7Y?-Dq$9E8zvTXQ#CR(
z9k<Ea)PQWEmT+)GW@2`}33LNcm1bOE8;)R=-rz5q`7Jn{-Gps@igi-`f1?25-zY%g
zupZ(v+rEO!b(x$U@9?S8u;fLcWw65cn(xGHG8Guh-V0nS9?9-^nIiFGDr|6R-g2|G
zTYAh+sa*JaB_?i!{4P;wPYxjuJRZCEJ&ZYXH8OH~HVcVmX8k99o5J?}DL=X1bU)qr
z`<`E|d})5aubb!Xn_$i5HY%Ua<)%AnR3WLS*ZO;Zw%LXuo#p*kw7Ux$Lg-mQx8`yT
zl^t`c@q|rkVwvb~1TObsOv3vi<8(eZTQBn2I#IP!gUA@ETGzqR!o^cL=4{lgheMq^
z>yEH9#<TQ@DVoJ>|83K5!|CFlz7oqRD<x(#E%z>Ow63r^x%pqhJBv{Z8EHnERm?Ni
z#oa7dPuq*%Y~tgD+zfnf>Kzughp4n4?)%0rFO-ZHGK4R(b!(f_IPER#lluMa`Th2<
zB?j(icdJ4JM|NF=mpUe3kLDuUEn#sc#>C*#ZqW#z5s58BS$*zjlF5~}2Ph{Yx25_|
z{Arz<3QoHj>gvF&>*|GeL=-+mi$h_v#RO*yC6A-eX`I~qvfmF}Kb5z42@OD#vk`R1
zh7=R%9C%O6d#4BX8kMaEywU9Q-*Z<h`Y&+|JejqR*{|*#FKwQ*<GL~znL!vT+r3;B
zlxszzGuYN*;9>%ee2vY*FzE2q3@pyL9LhDz4-YnCjHEPqi08<eX1o`={TDZ7xFCbw
z8S{8nv#f63mt&`C>BrWooZMB^a`km9{v?jDLXbWwK%qvl4B;@h!XQqu{M3f1K`vy@
zC(>fEm!ALX=iHQjLuRv4Dg%H$`JU?M$}ZS63(=pXc^LZK{l|6d4TSK0&+jQNf3jnY
z)DAmQ-(N<KWX9iFIk#~x*-idPEU)>AlcZw?Nf&gJEf8u0fkZ$u5~AwOXpj45gwt1f
z-?EQOU81fs7HaL+kIk^L?;4gpd~{1a@ChFul4qad8`RQGhwRchY)s!DxJHO-0?HG1
z*wpgl<9g4o)b!bvcBiZTEdJ|Cui+W9MY@EoZ~tT)d3bfxApH)p0?mO)MD4LFL8Rx`
z&{(;rcauAmc&Bt+j=_8<JxW3~X7RDO5PLNvm+SEz6Gx<WC_C<rvb4)J#r6JYm}Z`s
z+evIWbjm+Nio}PV)D<yGi66T;$xk*n270}3(-=|4P1UnMqOiC^SD6JNu1I9-UVX1%
z%Z2>*!yI1-K|=U8RfydOnKQR5_^z$d2FIMdOT;s_OVB2rf(8}P?^7D~OVPP*1+L%4
zkUj#d>v}a58Q{Ppd5SqOX_K%2DPB$f?z&F}fCMQP=|7n1KMGXQBGu$T>JrHxSl|+G
z$z|dC2Nw`F8LxA!Epzxjqd4FJhyb+$sBg?;IdV*~b{&p@+(c3-$7Z52KeO#%Z??js
z<t9GbdkSo_na^xgqt))sIXQalVLkJ6^!J&*9ZyUn+saF!5L6?2z)on6E}RTwW+Kbl
zF9?>jlbM<+ZxA2vgMVMIfA&}t!p)~lYJ_g?#g{LsC8xh5uMzF!^(u^q6*X5Vc3phT
zyIpc%Ua_s+9v+-vZ+uAMe_5k|-vxg4JNw8{9L!3S<cR2jayKEQ^80b{=yFEMsCWBY
zyYAA=VMYM|>Cm;&;kuzgyx$`R5(_tpi`rMqQJQ6+AmW-Eas*8|#-Gx)XYi-PnU%{n
zE2wvT8q5Ccu*EAbA4u(R4<Whb<CY(%D@GAvg{rcS?-_id!+l+x%r|gpy7hK?I|>a9
zYDlKTBQ!@AQ(zmaBran91>URnxRMHC%N_VQ*-POA+a>3n2W<a-_T=LRlS$#D?^b0G
zC3g8<^pksc+isT<F^7g*cjCRge-^W5{SrHQ{n5K~TE>LB$xaez(9+*p7`$!k;VZ9|
zY1X=FV=q*wm#*MrBH!9vDq;WgczT7r5Ke^B#Zgf^@CR{8sPMg<dV36sqaga&?;*UM
z582HtZGJnF{3oG(<%V61qHjkp4#IOieorj;$Cy66ciMbx`sA?V*sdxtyeCfIv^(*^
z%^igh`r)@1e_6(c$NOwQ1lAI^Q;|8dqp-1MCcK)%7p03Xf{zxTVZDlCeNQK`FNa5x
zU$HIF2WrCyBCd_~5?c{j7ye#uLCq@L$iJ(ozj}oD{}iY`oz-)y|L(8^I0azW`LOWa
zSYS8ebLtF$)33jhWED#KkvI^;24aw6kT`r-m~$~ELXPkDrr5TV#~0QDaXXl2px|pN
zRj!2->2EY)jYyNM+drj<Ai8kG_o0nKW{2Hs*<M9$z7|@LQ?ine?WZ~7>e)gk0`o<A
z_i56~R43EP27JDr<9!#280}&g6I#0ObPY`6Sa-o8ID{QY08aF^c}+ulf6`@Cotu6Q
zY-)*%#!>krW~bt4S}&TGy{;awpBBF@rbuE+-c>~xbt1mYfWg;`#KfzfoaAD!mEl`O
z=cG@XxqkrvslvpQudMZQv9&jPge|XYJ|FHG(mcw!BRd-Ms(FAFcsm*w|1V-h_!lvL
z<oX6wWOUa;J`;Zs?%1EHbgJ!oGWWmuNK>wO#0S(F;9SQo*iySyqqKM&nr@Rkc9X&q
z@A6xcy7#DQRzAdq+O*u*rZ27O5m%32A4~g@_A_#SbBQd8SIS9c^|5pvl=Zr(@V))y
zOfXV4J#53e^T#V|#;C@xgkd^R-F*)`If;BlJwjyaZr@pVEH;DfHTig1?LD8z{@X$-
zmqhY$fs+Yr5AFH0({D;ELYIeyt!i%aIf1)VbQx^wSwBu^hAz+hl#;C~l7y(o?~Qn7
z!qvnE<8{~zmeBhCsBTNd%4OZha5_xzA>aMUn=9YhqDfBftnhw3NYID6WE;-jCCOp0
zq|B>0I%lm#&iO3G&t+y8)`vh{&xMa``PTS;z#|_toirMhk0UeRLwqO9j`Ir-7-`4&
zm42Bq3p+mq>C-y|C&2L{dmR~ZFQmu(*2yK3c5h*z61nhs((u_FRC8N5&EPG!csRO}
zxuWP}MfDWSXy~H9v1O11k38TK8^mtb9f{H`MoVWkjknwR6JEp{e3({5re-B$4Gf*b
zR&DUR+025+@=9pH4BXy~SK@k78W-!=lg{x33i%2re-tFGH2*2&FL!maa|$?vR`H+^
zVTdOPiSHU*jYb-AZ#_{;73SLGeK@FuQYq5h$ZVi*#HTCaRg-5=Vt<<V){Ta_6K^ef
zVxFd*^r=RcQ2=@S3$qJ}{ibM9u)>9gmPe0`ynO!09xW{nnbUFUlvgqKV4ZJRH{Bk$
zpC&@r!m6%Qtb@?~1E^>`O!rEYdN)-WnY<%xU*-waeTr}s&B;Y)MPf4R$)ylxMdJG6
zsGf}TCB~JtR|?fZUl=u>H(t(8Y>r33AyWkQRX)={6`TClm=mD<hn=?u`LF)ab0L*h
z>TjRJ;zJ>DaXDufI{3<Kq~f4wF)Q`i()Z>U`!hhTM<*Ox0VvQcno^82{<gO{@ayN?
zE4yLa`t0X4kFmCybfVV#u{4Db;dFrSyQAgrP466EFd9w$FOT7)Jq_^;L-2{|Zc6TK
ziH+UTg;PuDL12wmo$_YI+@BieP`|8My;qqjB3<6@orAk4c@82xyq1R{M&%=_a!V_k
z+GA<lwvNDuXZOjGcR%TzGLw;{GR*FCn_|VgeWD7+MM&%Ksf=JLj2e=$k-nG<cgGdp
z43VxU_~F4e!ehJDs2?j&kl=T(P_&p`98@2RnIb=lh&!yYS@ga-;J#1aOslX&OAlqJ
z;G?0rU<?-#7r47$-(e0ZZS_Odertwk?;UWWE5frDIydmO@jC<cIAu>Kh9ZgA@gDFi
z9s|2p=g+D;3f~h48o_%N&aQHdOT?~mN1olO5ufbr-~P&Xvsb*!*mS*rk5RZWbhSj=
zT0>L2efwR$?}qYgeZ#LFO?|74?qmkvWyt9=hhpaO(^-K6!Bu*w49dyY;7+@&4C>*K
zmQ%ZNtW3%hLAs+8;gPrTt@nm^5n;>{6AfbU=MN8IM?SrY#aVXQ%NSa)pYJTb)!Q#k
z@Rc=xcc|KE>n)~_3=-+0W6PipMt!)&Zn^s<KGnonG<9)uHN}wOZQvFSxtTrV{lgk4
zRjETHg+XkbhX6@FM5L)HJ8u{%(>&6JN9IUBhIpdyN%<&c!2|ay*CyC&&UG#F5VE*s
z!dznnG+vA_HYBpO8(&5o#F(ymSBHL1FDN;}5S&RHrf`Nlo#=5vYp+xZGn?FY=65QE
z)I#rTetjxkNd3?rdwY88`Lu`VFrG|GRXAZ^Hw3eGTdAPN?@xXEEB@|-ntaZejXLPg
zcGvP#P4(K?M9G?L(M8vRguhkNhmsgmn}6?hP95gzu?7jHW4_i<Vwm?hr`>E>9nLKv
z_6THwJfZI9o{dWCl-P>z*GLr&t)0QAUg+yA{iLK>_jtX|$g8;hilsm16U8CE5h`ze
zH*QT*x;MfMn+8yD!MT;*n_@<^()wx;-JTG~G`V}EnaGEF>Y13iu+Mnz-Q#Zv%qnYQ
z8=J+@6=G#3tiFPku@i(4RE1mw$huigQtUxFZ@>|FTdD#s^Y97Vfy6%x@V4pw7x4QC
z^96Rp#dj&XH~Zq_2?TDMpfMLEN<#L7G+)up_9=k=0RJSdfCEsJSK{&E{h4R(|F^Mp
zh?A7sh;2P6vi{=#+B4g~_RL*C50_RjlD5Rff0Rf`DE#Mx&+%v6_%`70))}6Sxw%Iw
zr|B?Y=<`8x>|SlbQsPrvXRjio&+XX@2hF-xu3x3Y$ZWtR&pB!wjg}Gk_DRmiAP(E8
zRFkR2b0rcM%j%}vs&g9I!(*&A?(yp_7Phte#t8;VLqi)xt*h?guEf8gvP(k}QzjO8
zttAg5|Eq6f$&$FjE-qxhqGWn+VaRs}{q+<527DjR<`x375&QKE1pSXu?6)jx=bG;i
zGQ*<mqUqm6#)*hOjD`6+UbT8TiJ^Ylsa54MuyoYEi!lo;*uRD&aOQoQICJ=wAgt<B
zN?l&ompfNr_D(W3(pL8rnM2>f{Vc*~J7MAI8pVF-AzIPhcj9U+n43Es?PWNYeksKp
z3G#J>HX8k_ce0=G+3l5iMl7}RX{Y}9bxMBkX_KbDHCeer)7rSti&7$o|AyPoj6_rP
z%XM<OH_g<hVe<du>@CBh{`#(O1tdhILFpcmlI|Ih5@|`PLAtwPKtxIel$I8d4(T40
z?(USXp&1zZ{^9?;p5r>N^SO@WexBFr3vloK+iR`QcWnyy8S19d$=O!z6k`B2&_xiH
z@O$@Z>9F}Ujj1fc-g%@6-#%}eDznQSkKvN1&lt;=pg{u;mjKRLkHCjNnop4dTVVal
zSzQ4i^HU9y2b~xct}YvW)&2(VZ@%qBZ1Y=M>QOtUG))|v2gT1CUS4F7^gbZ8e8^n8
z9(P}>LHTqiI#E@#t43^o`mD*cmp%RZ^3Pvi@8Or?hwW*L$4WL;`kifjW-YW6E1|n~
zfZP-=nWgcK-X&4alk^mKoAqCj;GsyrSS7Y(iMpLH*(;Ch|HjXYoQC(ZR<|zYf=q6%
zR-y87A_F@>{QXyk{0MxH&C{0%*WDWbBqkL;fWwEQA#<1oTmfHL$pvG%vr?hMS91H|
zwdG=Hp7*|36CkEM+vFv%79|dQ=*0y-a*I7YS_-G3&h(ucDg@i<r^9WX#Em_F7M9Fp
zyf*$BM#bpw>#5P~zG)&w62-ahZ`3IZ8)q~`4tMA1OU9Mx@mD^!)g)q3BYsK4QY`Z#
zr}S#7((*)1R|q`iNf%<d^ga1iO1O};$o=0!C8c1F59vn*V3Bq;O7Z61;iwI=;IL;P
z?gB1<z<m#$Y%FgUAE31Xqw?Xt!%OK=cy)yB*7hmPi8vTZa)3ATN4*^KkE833+p7pT
zf@XA1LI3HxJpIRYDW_B?MeDB?-8WH6GQU0B(km6SiD@Oz3H@(-PC8?$AXDMAK|{S#
z)Uc@)epTimQt}D}FTpm(#`x+Rkb$lyN0g3YFi@%nWQq!wzx5g1NhRp@oJY$w<`&l0
zx&5G{ai7nFmB$NLMzikyyF`@Iu?^7X!VKCTW2PaE_%0G+%Y~agJ0HZgTyN1?Phd&;
zS`t?R(AIP*d3g|m-;2`F{;HA@FossgOcu&L54{01i};Rg;P|Y^D;OtGIrnivZj1zJ
z(wUXIaiZIRCq!^_lT0CK{+XS16KDM!rktZ%aT3JDuMA-avOa~{OOt8|;SSu&b&?NL
z4_dmJQdB<xpV1g0_Dot`Txf}5ZSQsfBKq3TvY5lY8d*I^#0lN^Qg}^ZU2~M=?oEeD
zKbospsc>f4+}8)}*TIE}{+r{M)02kZem!{h`m02YKg9^L6JdL2chvSMq@lW!s{7JB
zku9m((?9^Bcn$ocsGH+Qm+SKhZ$zn;W$M?*@3VGq9$gFc(J!qEg#d=(U#m=21Ur<j
z_48I?oMADsC6VQ)qdS+o9^(sO!D@*#OGw5piTWGrd;cM!`#c^pNN`CU7y)J4G|2_V
zuMU=evxo|~t}>=Uw>3pwi7D;i$~Qeh25y>xLPu7Hn}@_v^sb}AU!zcn<cI9XHg5uj
zq>sNq1p40|o+J6W)`n6J3(LP`=Sbs@EkEsynJ~6mD{SY%+WJ}1^~ADcYUg7Jd7|@2
z`{j^`kSD$b>)$UZOrlK6cx4Z?8wIe%y$BJj>p>Y*1@hskLcULqEf>Bgw>^Wvv61{x
zXX6lQlF0<FVr*HIGn!I}M21)2|Ab!v{{Wxhe{d^ziP`X-e*j352L3qap`u!63q6;4
zggh4HF^$nR{EA6w8zYJ26Hh8&bZ>qNlR&?Z!GO(Ub-^Th{GB)2&&*HEaU?MRvc0K(
zhHJ%NzSC)AozbC;1uu!IKt`rSwaFK)|0#Z|eN65~+8EtptsUSy7<f#%<!`4=?myNh
zJ#;ch{fSWlZ*&tanGBy6&&=BIn3ay4+l@H5OA3AcA;v0;*5BkXXOsRo?bf2Eu}*=O
z*s>lnJ0E<-qmxr9ZkI6H>ISs|t&qb#f2Z9R`L@VYXml$pX1pDe2OVy4OEsU5?SXR@
zlz;S5?99pwB44}uN_tL%DVdla3p?!&(tr$79+A8D$6OV@%&?m0G}#b8sekiil|WfX
z(lsdZ!cN<;Rbb})eOpI9y*AZP`H{(}R<7pOi7DF<)2?ntd_!N~Z|b_M4S6#tksPdg
zn&b({{_n|Rw@bvL|6=1=xwyCHsVlMa>!mi5Yywtnm}15^u-O~B>7tN&-8Y{wlAf@I
z%HO2}pCoT=%(5Ia2|;-Di?EYmWTRL(_X=<u-hnrcn%$fmH=pR1i4KZ%`CXjiw&3=d
zC)=4*i@NufA@}BI&HMgf;*;Oq;?3_6uE>+|*QFD{dCVtses>->=071!+gc)o-lw?-
z)0O7XzaI$+#Qg$0><7(ZXc7_6O6$^-dv{cZTDOTkIRC1LEbByC1eF0R@%88LQM7Xy
z%VFxUh&U+!(@{Cgr)+8Zx>?I_o3Uj-sXy*0psf$<k`;65P!8u3=@=(c$`p}50%eCq
z$AF1R@4ZLQLz55BY1h7Iy~L34Q?*|ccD@oALD9xd_M(7TBg{x0M)E0ZeC~iA|J;NJ
z*F@Upog(#kXHMBfk*DpLsy*NgFi^S>%gZ9mVxpW7i$b4gO8`->@|jxnwyQtGg_RrQ
zpJUl}h3tg^Xo&qDm7wsyfmwgEeY$@waB4vL@>x!Gxlh?OAFSDPBoo6oVaRh(jMm#!
z2}Y)#Y5?z2BO?x2KPZm;$)UB#@J;(x?Rqzuyo5{i@PXx>MSN#HZ95?hI;YZ?dM{r(
zZ`~y1ObRt!t@WPvfmFzy3keMX4o|m0eGgLi(rdE6$R-!zYaQfk0xBsHx!GGW)>LkO
zg2VUa2c43phB2Cw*T-ze8t}*sd}dv~M;q=vopw{ZekQ~+#J;Kf17)4PD2HD|6XKw4
zo<E;<K1U&7@O-%W>;!#cBhBVbI;uV;@#L$>s$0R!jA^6XYsCACh7B`a)WKk#%kEXG
zR=?BdFhZWbByMrxR4nJ$&1cXmwZP^WtFM)ZtirTsTV_oTb$sVFnVmK%=F!&$0fiy8
zTG+zp>ma-H0e^)H*xv5D{p-t{Y}(sUfC}+Lb=Zr#HBagP@M9&ijI7@V>)B-$KW|!C
zl)W~cr1{3IMM%}G><M$7hZTPi=JLPbwIM07mfmYVct(k8$9izBf0xkzhybn)MF5@m
zqb@;4HbeMXh7ga~i)S*A1F+uzlK-QG=8FNaTEne4e}3arHj8CgC_)=U0dR6mA}UpA
zO%{~wIu^Ck`U0uV?o^;c#a+3~KD7N?7H>`SvF^CGn20A)pSla)e0J$vK?s*f#9=~|
zHI}A`p!Ijq<HhH=I0bA;VqQ9pWD32~gdD+sjY1brkNnW^plcIiYh3p*&^U-3KKoY0
z+j1V|!M6)&gv%t2zPD?O=(aE4VNASowz_P|@|<DLZ7vZzR$w<N34Q2!o=SSk>hi<f
zQvj;$yauvGX`FwuMH~m?g)!jvfE9}WKX|YEav7ThLq2K%9X6lTLtrT|YYnoQH=kyb
zUsPGB`Pv$?&IEu^kEw(Mfm8Q$c$kC>;I6*SH4|Yz_TInr*j+lK8TfYGf>~H)&_6JJ
zJl7U+o9c_0?~J8%p5e?58kL`rN5Dc(t3;s<S^A6H6vk-%fTG@wHYC@6wnnqF|IWw0
zRvTE9xo;SU4w$F8J*cU<-hIDLlyEsol_Ncg!r^l`^=b8nC5kWo$Z#Qz;||DaR9AS_
zw6bI|<!(^kmVczoIgOk;wkt$0FP+XO?xfTnr&H3atOk4`m5~jsRmxXFxX}d!mZxB(
z^Q>!>6j^L_MlnGEW+(bUU5CJw)8Ys2XBNuo=3H}baj?^#=3Zoe<h%J|kBUI99-?i(
zbE}Mb&pG=EuhCsZsD0h!8z)U-Z2^NrTd2)Yt<{^Of<VEVu;vI>b-<qTMV^MAdY)DE
zpft49fD;u(lHOBvR`WGL!bRg9TK@-S$4@wXDMqdG0-Qhwc`$iL8~$j{1=ikDLejgX
zVI1I%0(=h%vHKU;CWmN$(7Ldw1!&rX?O*47k19o>pTvb|ardy&d;l`gJn#*w!%Y)f
zhEm6ZO4n`LEKF&nYay3Z^Kx7(vAwP73i83l+iaXAv>U$?oW!_X<H(qcU5!XG!k&@G
zWOBj8<bM(VJYtNyK#SEunxQ|sXfv7A;SiCx)fTh!t;dC<4viS`>!vZbD(R;-S2`PG
zA+9d#0W`nU;YY8yoFCT_oFqX+wgbN|Yp5dwS%K#xEa`?$oYC-p#+Lk}2={2#2Z7t^
zc_?b?KfOsjz?*bKQ<_fAe-vO|iXNp%{Q0Rk&u*c5gZr+;&$r4_mn$`80zRkg$-*wd
zFywSdzMm7~jy;%em(B=Nk@af6^_e)HPm7yL(wr%=-q`grB&q-i(W@O8nD(e{{@Qyl
zVLR@5Bf&<Z1h3@@-Ol0MRg4T7Nl%=jk3*%(a!Z_wHW=3aVNQLvZ<^Px=QHb;P&4t$
zRjCI}IX<TEm;E=6bGyInfkfi6&Ha+CW>y;ld)wiVD)Cx`|4R%($>1_Y@k^p~GCrFD
zcG1HH;)iu{g>3w2{f|RZ&1Y$o#FZ>(R>PJ)<sf^Div3zU+}mBCIg`xu-rI12qY%I9
z{Vna8j)rT-Lt4nGMMHhBmYMM}sV4mp!tIE;dfG6AIp174*}5hz8n(N(F+e+*C^F3w
zamfKq{R}*O9k%O;vI3m_s-l7DtLE|Y7J1Q>TJ_O#(<aP2gI#`Rnm2b=rlHuZUiXu@
z<xs7>GZ9{H|6M9Jas2}kCUnHXmkxZ)aroP`w1L~K%N}~^%m}rXB7k^^sTd|-`*^(5
zIS^SwGK8sQIH5(T9!?PI_})2@Tl+9VBFkVb4f-Jx5d%!+d*#xee`x#6U#L9Uk!1F3
z5p!wL{1Qn9zR%yqOcRLMG|f^mI1jSdg}SuKvS$kHv1tix|F)wuOu<_F6#}>Y)UI-@
z`pD#pU+b~1blsRXtfMF$B^2+a`BtN{2j?YS%<zYmAgkphcK%I9_7ClJ+!Cux^jh8>
zZLFA-BGSn%iSUksh$!pKaGUaP*j|TUBfxOLmlWtYQiEL?w}f{{rrMV~XD}MNXu`&a
zpQP7<Zwg9vHz|U1Q~$q$cn#v3%tNvL8gNXaeuXAlw#9ecKkpV}9IzOLZF^g7dk6HG
zk1v)sXc6@rrNv{)ZP|~|*!+S<6_y_6T0GY*<Qd=&FgMbftRmH))TN#*AN3GtWYr{%
zmiKZgdLFR8e9c+bH!?lhPhGn34tg_HqZr1ay*ToCcKblOxt7q<kD7a|MASQ%CoyB{
zwy4mqHT6uz_B_|I?SAgUX{DQv+paYK!SUJdO?c#h@Y5ferZK>AH!*9^b=i47$y~J&
z%y!5*<%RUL9^Z3`9>5gcb=cXjjjGZuF-dYzMqQ#>3`BBQk`Qf^#s=_3S<J)0!uJ<V
zx1TsFu6l5AFwnXhR2FPP4C2Luwm0anch`x}*o7hzC64#h&t?_QIMV(BtwbEk?LK@}
zK4Zn6dKn8ade{>7?iggecZui)r6J&7qoZeL-GS)7`=$FgG8*st&aOW%-T$R6@dxj~
zXho8XGtjoJ1tL)@R>@HC(carG))RkoRf;xjCx-HO2-vA$P)Gj(Lac`k>e`0|!FWjk
zK4MJp9qY__vIcPXF#)vangB8$dX$+JWy!e>>M6L&aCDO))}Z0gt*gqbZPyNGNdw=L
z$5NBV?TF-v?Bu+?0?1K@w(j(J?hgjSn&jechf-6f(?h)RsTZrN;#Zy;+OTnL1z&gs
zx3|1L^jO2d_va+8N++Oad=XP+oBFRw?*D`H&;LRj1!9={s`_jHgF6>OlL`7uQ|BP6
zuLQCAMq9TYmNua@rkBINvcz;V6>_kk^-tK?+8I;}@qHr)<)z#hoifGr*edw)bepLd
znrGcSdHdojKFhwU11W4Yg9xah`FZ>3DlY@7^sMl8hSfq|oq<0JZ;M2^cEdAST<~VF
zEI<FvopsIsRzE?LFsyXS8&KW5yZZ~A#MQSfSX0V7=JHJ8lz@5tzV>n6mlRhMuHK?=
zRX_j^ao_g}A$P9O6qss>th)$k6Yv<IYB;-U-EE^RH-(hEuD7h=e{|i##4;p7P`5jF
zO-5$SkznUQE+*!$B3u?UszVlJTemn|J$p!=onI<sM4T*kbLOhAp1M-9YsFUFU=_W0
z=L8u<b{ihLFFuf)d1zy21Xw?Bez{b`&ex9v=3?3{CEF5>S%zXgcE?(Nie17Jh<FJ7
zO0RcVe2&XR$;qN!n5ScL{B>%s1r03~=|f~BGu}EhA#JlF1YY+Z2Kkm6Hrcj(#K|8J
zz2n+a?r=V#vmZPa&jFl|{@q?9;h#-j_zJliCj4IGJq&7JbQD(kac{LJdfOJxeC*15
zWv@2v44J4dS*W+}Ayd~N@nmO-m!+SRooi3w#ZfLZP1yBjA-HS|q^1ARb~7wjCMr^h
zrWoYf`<XPPd3i0QSoh`O_9wE8_J@icK$N~Rl@2HC5Hp5!U~{jIEWhRR>nWvTNX1Y`
zD;JPFti--K>6U}u8<jK7jo9^8BMdj(`7uaQn-&Hp+Q{XC+HM!ZeShR?;vIFDAitnD
zl{k9O0qkY;|8)|7r=)oSp%EJ~Fg{kOz39HMlc90m<I)tpll-RLc&;0Ou(yFLT`N*T
zpO`8CVcA-BmP17_l7g8POnntLf7#LWI-#1a^6hZ_-Yh>g^l;x^;HqY6t2RdzI1yh&
z`6$;;Zjbf&AuhJHOJ|xPCE&xIHfoh}q&CpOVR%!%oV!oee0^BG^m>{rXYqWG+UMk2
zu1CU6DS-ySaCZJ4ZaOC%pnVg-;7<-zDAuhizU*DpD|Y-rb#%jvKOOD?O<f_zFY&!$
zTM+@7c2u%j`5z<%coJjtlYZqm6(6A+rd#RwT)r1#Z}9cS&fzxxmRPfAQ;oRS{5k2T
z!cU`BrTB^^stl*6qxe6UuA2*9Cq3wB6H-j)Vbz}<<D;iD_<3wSQKXsd*gh2^FSeQs
zAk@s)ddf4ivUoQoMhif=q>sFs)GB><x-V)-*(m#*4WxgYS6u6UI*0^P13Q&@$EBC*
z5}1J`EM5o8CU(Da_pNo7+<_5`0g)Z{S=)+!=XJ+h;GG*b;0e>L<A6IXmKF{8e}RkV
zBR+ou829T0NQi~5C#r-NjD`=RA(Z*^AS0xY;cK%H$)V0o4<lvS>RaOlDX<g1F%-F0
z=sWf@G$9HAe{;(w0!lBA=xMzPKV(PgU97ewKvlcHi*4owW%S;lC69B*Q`7eIyQ)j(
zdj06>aet+1p!S%9#eaD<0Uw5V{~_{82fL1mu6X(AL#B(V2ShLoaTV||M)LZ{(rty@
zh+#3yH96Z9X!1wp<FC|ErprhkQS$Q`;@^TLx#!OlbgxLVwBS>o+aVU`ZWC`i0fQ&>
z%|A%+n}zndI>6u|$EW>I{d@Rd29F@{I|%R290I<hOm^IamWisqJvftO2FJE%A6Dbz
z67;6suV+i#)zg!m|4iGoHrbsaJycx%Z_aZK&QC}Mmd^<D2cxSJDtjN-&c)loZ8I-T
zj%eK%7^l@)-{ey4X@p2r%z8IWFHr8r^o1f%JmDdD=Vp$-pWe|Vr&C5MK6@1#J>p&=
zzhE!n22zHmbFohD3#}W$_-^N$F29L7`%}b6m0^EKk}pBDycT9zaRSvV)evPVJu1M#
zA8g}SzCH8z?MsP6U9g;_yx}!@e*3ikC&EQnaMtuJu=k0uOz^1bbB5H3>9)w`y`{D|
zE~Z<rE;50`8g5*?`JGh-+lr>g$>e9b;+}_xr>z$IWS@JB%KU_e*HRoKhTQ}ZmvcLf
zbo-y53Utw#l0=USE_6gT9;J=BPQxjuD&_h8Ut&+n8XTE&<}<-F^Y!+*ir&SrI|X+;
z_6(kj7AFm`HLKfVP$)lIUZperLryz&O->UBSqq9>nFRU#($qfOscFQ}52<nuAtmO;
z$WB(7W23hrJ=(Y^S328#vu@1}fRZnJ-6DT2hrhp@#@PQmry}91<Czl@0@Yc8x+B9W
zI0S0ZFMepXB}|0?V=B5^Qi9#`hBUS43`>@Z$|pTawDedoKnNRljE(pXkCHb_USzy>
z$t(3PC>0q|kC79$Jfs(dMdWjRH3avBi4Vwp-X>7zpmzd3+AIky{cU_KkZaGf&E;-f
zs*@wLubaZtGVIK2nrzUB!^xZx9@)*6-WaaU1L8wWXnQodxJ%e>0SMZ0G9GNN%yDk@
z_)ttE+>Ho7ZG>C)PA|hlGu&N*Mos*Ji2v^;${ld)wlh%?aRV>rP_wy_<LA)jOtqi<
zyftd8D-<Bf<253Exm|(JuWz{n=x2O2AjA3$WLS4ZlY!4>)spEagv|#obui6I+4aHk
zJxYjKqnqXYZ;_fE+q;P(Uh~19G5eyd7GDWW?M2t!vt;N}8!wQBI5-yhvI^6WZT5{g
zk}DNW?ETjSF1K?_M-DSL?^d=nD)hL6(V;vCFDXa;(tB!h!od?l$$LX?LpXuvWIo?l
zztzn(3Gq+G!}g2=or4Y(lfd0@%0oN4(a$xI4MNBQ&Lpo=k^n2HeWX9WdM`vjW}8JZ
z)d+TRDu4Q}IMQZ1I8{K=&Xr6K4R41A&?uL+DqP8;KJmry?&_^Ankh`*qeyYjZr^Ca
zdS9!tqB?`qeU(uN`V_r@1W$e*``gk015)Mk;e0^?DZnIjp(I}y@Hj;<^?T4ZrG4RC
zPPv=JDDf<D@ZQF&ez)jvg%0r_1eCVw%L&Vd5sK=OMS&`oem&;z_#UFu0Mv*SiZDQR
ziB1@Qa~H6f=*#XnxDTD({ElAKS^zLp@4KB-wG>p+@0S)f*v1h<V*(2=Pk0Z$IL3{N
zs+g<fh#*F0?fUyYV!Qol#pk@K;07WC`@pZUm8x2l>1?0v$iZa@$Ld>+&wT3X*84@h
zs72Bnnw6cfa-taM&kt*Pl8e)0VJpNN(*c{OfX5$)_nD77>;Zo+4c=u*snV56(e>gS
zVp7xrS>MZYB_i^PN3Z_+p}R8OaNljih05Kqh!ldnd&jnpRJ+2Rt@It+tsQS6@lEl@
z{?W0!jxb10-$%0aus8jt59JbY(s;!LfGmQ}M{Vek02bNS%SZ_;dVp#gnJIl)!NG%u
zXS@1t>DyD98_5+>{{^F8UVmt{Q1`ChF!|j-or>bd&-4$ixP37|%a-eFwI&61%g&15
zZuJS>33*3RDA>6w5h|Cfb^MhUd=#N~^)`1oPo?PO{0iT8bxf+Hw#1n@s*6js=O#94
zW=d-b9A@)+8IOgF7(Le0WoU_9p-~7?)-1s+o3Y->M((|=85rLWN}m~0+OgSfvwD25
zL@9jbn_Of33q@(+Cq{Ciqm4Vk@NFC3vV6l$7Wn6HSm00Euzoq=x40uO#Hg<@k}$%f
zZ@%Mm0ApYyvx&T3prO@YOX&ZZAHM|hV^=585TEVX6yTTMKHMLwbhK{WfjvyORDBjk
zZ29ne^f-VpW#657EbaV~at=PO-fn8{9cKPy3`0zZ0bNe|wf3)|i1>&;>x;*`HXd+y
zIj?n!QEgxqFvZ&pPXK`792HUj2bP)_Xn187`+n{(Uq0tjIdeQx`5t4xO#I<tyB~3v
zxD8jcva#+QHs4($I^$N}R{U~G!upSE65u<L@@#Al)Njb>xd)wtzWbsOt*2M1+UqEG
z+n>F|8&eavLU8>WxC7Wi<>Z;q1rbw%iO&^Vh_t$Ww2=-$fVrg86y52Z^U(rMo%uVs
zW>vc1M#Ti@7eA(ydu?H?uN40nHU6|IERf2un(%(wfI7r~rp{Wo^-;X1J$E45ZTFK!
za~C~!dpGV^E#rm@i@EcZDueurV6zz@2+%uV%IrTC8#@&$vw1!CocIar!+z$D<{WIk
z1ViuqpxYBF&Bxjw=lvR*jt4}AMQHDg>chJ=w>%ruGmr<^_8}k7qaNXF3;iY~cpC&x
zx!1+GjGH0#2QyTKjvBoAWHdL3mmsRlgfR^SJs<`C29mTi<@_)y<fK5d@S<|{2aDf1
z19g?>fwq`apOnjY&Hf>!$pFLJE>MYzp>$Pw*7+fYc0s?2R{kHafmNgBB)(!-b`4zI
z`AROj@g5a$Whi2L*!R7wZn%UN?kfVt$`T>2x_hzla>pZ$W?3?^QZV2d=W%W2ZB^}J
zXJQ3t=G^DCKxL;@Vo6z~74oA!!&pf50w;lG+7n)LMXeo(0?|LpEqV_AAqG%xk~)9=
z;D4VBkogEGH_knbBuqTY>Bt6J)umLxRjo1HGI)2(+?o*Hu-9sl`_^l%&QQQ*{ShAd
zjsPI&cDQ8G@Q7_jGQv{JM_DHTKK<byac+9!!TIs2;qBSOnOXKK#g`&h3(tJ4B@u1;
zeE7|iqu?h8l0}B5falpN&vibjteNdt?8wt4=@04Z(xl-I-N^%P<N1KGuOI*;%N^&^
z5#kTt`94!+X;Ed}PWwH{&1d^4rLZ(M-#)e8ecj&TskUW1&&Hte=`uOjOx~j0OnyU3
zeB$%h8LUy*q#J|#i!LoqtiQK6MWcEIK`Z}{=YQMp=b%%H??ZZ07<u%|#b@e5)U<T!
z0yw{H*H7}R7fyW=rn82DBv$`-TNndy|2~6sxQp&x0hb$+hTUd0?}Aq{%WD<uiylAc
zBb$e6d-B{-moNNw|0~H|O&Xmj(42j|<*hE1YL=UlIEC|4^;Tt}8hmD`zJBrJZZ4~~
zI`{C+v5L<pldW4Zl#kER1ui1teSc<b;wwI{IS2RlfaLYq27ojOK*e6a{_|ZSQe`4`
zg#Zt5l_{g<%u6A@Hl_xS9>D!O4?E1fffE$M7X7&FA|h!WL9Wf63UGP2X;Wf5-m|-A
zqEvqM{0<fRU0*NHey3VclPJHq$TUhbFP@p=o~Nxeiyyoel^kgF>x_7625mxqxFa6&
zW8!UoDc7@JZj=u#bk*^IhE5REqZESnMMDP;H;SUGU!<am9pC)$0LFbYp52*dYW500
zSTs#BC{iU_lOX`JCw7kpP=-OOEnFCm)T=7JkWUQBAMK4>vBWtJ53mn3sYImxHV=WR
z55Iz_(%Q#Ixti2YR0rCIYb{vNj<N4DB&7m{{WHMw_R9MXch1bBv1?RDppM$EYyDnm
zDq(1iXI7{wCI%@a<{gd5B?{p)s2+X?q$KJ^Bow<x1y4X(*vjN*%L`KH0|F^F$VGt)
z2QVUg*czF3@#&o+M;gMR{5!GqO6_CV$0s-<&fjB7JH~ROx(0+w26d4119_MjNs30i
zkZoYJjOn$`KZ%zo@o^5YPsAjAH~-WB2>I78%A8GXw|ohtT5s*jh%75f3FP8!%2LeY
z2bd;#l#;;HvNzl#9m;uA{+2eXH34qAIWcaw^TRiZJOh}wJY9b2Ck-PBroH^yfExXM
zmwpjw`=!1R_-)$r`Ft{!V>`$xb@X+>oJNBd^`#R+l@i&RCu2}<dL-4M16#3+E=-oa
z8XCOtztVE%P&OlT?BP&(+c`d|^pY-d&hu(74yROYablNe!YVbQ*s50ma6Z^6MSVl@
z_HW~q9-77m$zKOY#O$Qq4Hyvc&ddN7#K*!6%$ZQfAERxHot*4*yUZW&WwVvo>N@R4
z6xm+~ce6qowSr-bf&{&&!P{^{x&eudu(iWZ+8T>AFBUDYMZgsQL;?&6k!*En80*Ts
z%6LkcQ|kW_Rg$i5iAqZcbBNuiL19&Hwo|`f`NbG{4<-$6W9-4Ikq~jFLv=9Fo*WPz
ztbB)C4}VIgGf-VyWeC<?{qjXmA8R}zg3SNiQqhlQn-?@r?D3%tqhX@JEuRKFHdd<&
z^L@^aR^D>|0TusS-LH^M1(i^;58{}l;v)CkS~jy<bL4O%U3wD8Amf9Bw2EPWvdA-h
ze~bn{gCLocqSon}wmXWCdJ}xhHXno<*nA|B!jrynf~|KY0xq0EU<VNNH(ddPLC^kU
zFzB#_K&N6Qsd|}*O}8+o*baEJ>{Ji+0(lnJJpC&ji^Ah~1Mu1Tx4sMhh<~iJoUce<
zQW-knbS-HIW)wHQ=LQK;g>*xCZ(SM?i|IL_FtLk4UPQ77^rDb`aB>Nood?5x-c4I{
zJ1=YGir2jFm2$c~=>pJ25|AkiBrY;d;-pFXcavCnF*IZ}+<h`MK+P8;SZb5y>RyK<
z8Uf;jKbZF3xh#WWfbq{5TEfZu)TdMA@cSm>%gjTWS!sZrm;iAOl%}A7>Ia?e%v{aX
zA*Jh|hc5(uLzmPz+vVjbH}G3s4cq8GsX_UPOf>yeD_Nwy62FjT%fvv5+jC^FRdo?x
z6n-9~5e#EU5Uh>kezzam3JMl4QSrM9ivzpHMEuUVIQqU8(DmU<!o}uTusx9+Od81#
z+0G+7C)AyO8lxgI9-GQ7OlY0QSuA~=^B6AHiv=`fC}$8#|CrwXTlTPiLz;xlq#ODO
z*%9zXIZ{IWe7{GRu&H6pRM}c2P&wxF7cA!edFb7ATiY}?3CVtrtsKTGIBgA^^$bVa
z25E}uW_94%Eu-WGJby)<F4)k{o<Z^TvkUVpOsQwBN{j7w@kI1O6!i)X8se`7+*6RZ
z=3kl)mwg1*H=yJ?&1O66)vYvoZ#Ueu%LIC>l_RNX_^-$>s~!A(kaZrm7nKj7l0#PG
zpY58C&8vGT*mT3=6{b(Ncx|V?d_8TKTbvRt^{f0f^Q4eyFhy|IyftWjK;$aD2{{rX
z;P&Z;_x)S9745A?V;zUwhvVsl2cFkKXU@KY^WNw-Rj%Pw23~;xD{Ksd>?N3#*k9RS
zIzZR4uT#*5|0Hw4YnG(sek5%hYtN%x8O9MWqP83N3g3c%-+oy+tpdcitpY!OXGRDc
zUdJ^Ud|q+9DXud6(Mh?ah_i!cA2{Nc78~NHZ8%aSR!W0!N$;9iSb@2}b5X}rnhl+m
zi{sLpM2CFOD7K`u^EMUHJR#$~r;NW?X;Q$We&ba{$fBpC#XEb`euy*eU5E0^-yKNJ
z@wiJap%FlzG}c>*!P^LGmp`!;0gvWx714Tc((jDss)(FM-kC>&cjnRFb$%BDFppeR
z6M35-^*zKFySF#|k(_S%l_IBPC%X21f1&%I6R`2iPz;ruOsKA4LoW?1N_}+aF9~4q
zYSi&_eB_c<TMEXNXBWVB5}mXe$57Lbca~fLO5iFsf$E6#R;t?Q^>ua@SxchWXMPJ;
zV0>s?`auC%Y6KDX>he2dF`_N2w-^km5^<67zC8iW0%%PgXn|r=aUtB97nN<SAtz2L
zwv|&=U%w}R=GQ(M8yAa}LR!VKxsK|Lccx#RJ53SlA{vGGRdk(+vb?Nld}n9=*PbDP
zDI@fqnX1AF8u1y<`b~!97w=vg%FxC$pegX{_LM-5FbkVauU#*$q9Zqz9YI6cL$R(K
z*~d|@x_H+(mh=6aj9@p{vR>=fTM0!C*~r>UW1I1u7f((0<{-P<xb|hgul?C|a_vIQ
z5IuT>_#R2eJHX=~VpgA0JrHrit1(kH;nV`vSt_DJFr`j`3-k-T>OJy>q*CQh&4>^Q
z=N93RA6{e=UWf$?MAgQk(tH7>e+P5t%%VUEDx|M-*CVn&alj|9Xzp}se8M)Lg@W^E
z&@b1kn~imphTzRof_-myO@`zRjTU*sb_bVb32Bw)nHzLiWbM3ar?$wzyLDQtS+eN^
zXN5(7b>Tq6v>%07X2yKn-=k84<ewZ>hR~%sN%ro9WK9L;f2S{%Ynxl7*U#1Jn&?e}
zb<P&FH=e^*vDxnh0RHA%fQ_s6UaHSJWh=F+vff#-uBf4D9yNq*<~)$Tty08FnV%P!
zvTC;G+MZB9tNOZh!Jg2L8-PXYJCww46VINsSo#=pb<mD`4_eHqx_oD`w+X&bY{mxC
zYfnLOv1R_DqhB@>ZRipqp|2Q}1Fn9uY887vaH4`qJFSAqxexH3i~>)cZw_;7QNfsK
z*t9s1ae<!WpN_u-a=Z6yA5IaROnt*zeR_qoO#2)*N?f0h4_hW|w@Mk;S^|Yxzl;qD
zt5B5=&yX>`mp%IOrww<{{bv`|vI6I$#!)W8@1oJsK1sV6edaQ0l86oGj^0;L`ug@~
z`%M8C53caglHBJD>%iyCzp!a-#VA&?M067-7{uLv5_W>Nzd1f{H~D|77<%i}PP&_Z
zdzI@HwHHdrh-ks(sth9rt!pCR<<mRPmeuGTcl5_tMd6*bFP^kSbf@krd~9Bxt+ur(
ztkxTcKRu1OCM$7vW;MW2ZrZe*HL7qMd7zl|Vp{ewrQL1<bRbC<BWxiGZaawdW=0l<
zDbIVm?augcR+B9;;e8YgXudgWP{Ph#G!>~5L|`)|(HJ;X(88&sKO?hXLd7|={TsTe
zBGXXK14?1U#=%2_7LlSM0p@naRI&a6(Y@D!ip0GF4OjD#Lg&9U_+2{KFP#n!R!Ncs
zdaIO?Upz|0Y7kiE<6)(oI9c_F${E7!^NT@l)=7<(KfQ4)w9nj{e_r8MNO?Di6Xa3w
z^+mJgDy7;>VP2W;^7g(xcKi@yI)D&%A2YPD-You$p=!STr{;wjC6<Rt&}GUaPX-$5
z!y3imJ1#p|@W&4z>#eN7rS-~H{7nz_w>?YIJ6zoDWqKyr4!6*^qTIiTvc3Zc#)E${
z5sy6tg-oQ$NhFJUlw6<Z!Y7`0uc5N31KJ-0H>cQ}c=D$KISLZ4(w?ZH6n}#MKLuCs
zi6x;XNWh5{9-?;w+a{4CzZIfLHwmvuORGkrDSvaVf6fyS7@yQ1{;E*-R9koXfrf~u
z;XUFJ=`rI>+Kovx)6urWci)e+KffOif8~UxmHY(?%^=Lx?<s!Lv5l(=FmA_;A&8#u
zN(?mA?rSj)s?6TYgy-$t1yT60^8!uS^a@quD~gY~QbGaiPJrV>r`D4B*Ai(-P838Y
zu?&5oA?nxEHZW64;wyXC8q5qDR!mx*6kL^r^0X=@lOlQoOBL5et!9`+R{}=uCa9E$
zT+Z3(f6nN#O$W`ZC?t&S1X%3GEjZ+^luV|*(?<p#VD7F4a#pg&<mQ}iyGM#4X{(XS
z$;?(={@qrSGoiF{%qH-Z=ELvy-EME5(KV|=;&=8M#-v@BytC9BT>0UZa$5Rk=Z-Hs
z-m-?R?+n5#C>!Quf+rtmCtvz3ChC(1Isd9z%KnDCp3c`wDv3o^Cb#Xss1!AW<=~eT
zSx8AsY$O{+WkL%-f1A{xx>14OTn_ELN=qsL{=~9N-1eZ0&*S?*Z_{GWxe*6gay>u9
z@EJ+;Hk~vKHQ|eRp&@7p8j|q#hEEn+6~9dV9tiwKaJsb&5CxQ_UK&-!13T~n{uQAT
zJyFyYjA$GMVVU|Sryma&h_`B}G=Oc$u;xdoTVlc>e2>rm=`4x+*Meo8nybSkyj!pW
zcMCR-`1Q`%ghlHyGH$E?n`ozbpWy+2VO;kH+#iNvjE1*Q4Fd3VKvKkuyBst7U$<`*
ze_<=4;2LL8(K}y?#aU_DKlF`Q0uIQ()gsUD4oVI8f-5hwyEd2?qf2+yBA)lWQSVdt
zue?xZ_r+m^7ALqLPbfxPsDEi^mK|jzKw#6|%xW+pyibfEZy}tvSA251Ct_`DxZeGO
zkqOEiPGclBFX08m$LvG-CAw?x*uO?N%_TIww$smejkEq>jSHJUoCSi9yWEWh&U+&c
zGsb9iCL#|%Y7;rnR2KsjW*Xj-bb=@E5Mv)M)OVB7dd1C@<Jhi^uAn;+%;D5SS>LS#
znlaL5=kRM&gYHLbLJUKefAT;2U+UO^u^y=~0251*9<8)ezud}ACnD-!?T$Gvxco^%
zFy*rXvt3j=Rmz#UT(KxR12rkUn*n#eW^n9AT1>w#G>gu=S$tG(!`ahBmMc^E1F@c_
zo%43ldkcNXYeRjjwm$VTt?u{s_+T1OjDHVSoyUR&rRe5n(<o00T2^67ykSn9!R;@z
zun55@@>JKVYcXTRZH0|xiF>+UH1{p{QtI4K7A$^1PdYK4b_wQEIe%Ru?rl<mrW>jq
z_vS%&uU0lp)aFr%ABWFY{M>sr&)R>G&y?ExDrVH})Tf;uHbB`c%@G8|MvrHaV^bR6
z{oGb$GQK@gRuth!7?@?(PiYFmd0hI>iuZQF5U6GcciG-j!;PK8sTTy)Kb%WztGD-4
zKC^JEnb-N@`CYvU8SX!GG2!j*<ZBFGQQ^P>c~;X2`tK0IFXN!U0ZhYP9oAR(zfocN
zEgOG=X-mkO<2_G;4#x*=CZU~2-^W7`j+mRrzbG>zj=zTZhQ8uG32W_8?}Ap5sDu-Q
zaQG|Q*Wx8<Z?b%#J$`vGtZ&&JvmX^nLDX?11EO>~0eT~>3{hHo8b$SCu_5x-F_q7h
z#HyFLGKP4~-;In;zU<DO8d5rOvUlx~dBjBeOPvZ>t(5^|!#P3r4|!lGtHX`4dTnGU
zcM$Fi7LdSrv@){nYv3F}r3ul5`D#G@HrM3DK)^d*7)geJN^$*nmBKKjYIDAppz?}u
zE?3lbQ?&uL9A5n76<<|+UYx!hpVN%=ihqNqY`k@4tWp|ydVS*8ZC^gL20mV_J}ET*
zgP}!}ub+ay0!JOB(bm-A8#%T9YSw;tH`-Sa>>mh#z6s+42BKG7`M_Y^G>Ubl6(V~@
zTyFWpsx{|<lf8bk3?XsGe{}(rDSi9u6Q|<E`#S-a4#p)qg(_?d=2@IwUDls`jCYCL
zi+5?kDQw!VK5o@~wL20tm<S$?oZU}x)lYpqDeD9y;jvj~7d20>a2at@>N8N$k1<^N
z$PH#*U&W2Aj1aihBjR=a^J2$+0#EobMZ#q|qE48$xwNJUc$6b-kt0gqtY6P`|C-ZP
zoHTb(B06lXkCC$q;ztQ(JRE0!ry;n2!xzGQiE5_fYA?;NU91P1Z6Tajl>GZ7Kc;g0
z+@NPY>Zt-OIpT+ZTuzU89oDi%Z_{&iHr%x2U4sz91K_{SGDh@3;Yl7)42Bk)O#bvv
zz;puwMh3GC<o<8PGkJ7?+{mX>l|kH1ph}@>0DWu;%oi<~1NWkUDz--$ny;O|rjmj0
z!B3WO57sF4?=PEbKZbFGIPZIZ6e!8PTGe9r+l-_1pB@XzmB4FXii#9Sqjdj>5=vhl
zVw~zyVrLK+5qIJoe?tZCk$>C|<^EjAPLf~pf;B*b6UpziHkl|uc5d@~=HAh3kbeiN
zHwW(gM-KWYFJeW5mKs37B!cg?;=cEK6=Hx7;9YR*<t*WMj94EG;S^ahL*$WH+K07Z
zrzx*W8ehtw{{+99pW*PuZVi18G=L$;tH<6bOn;%5Dzcd@5pc8aH@GC?FbkLmXowlL
zSsJ^-N`rQ~-|S`5r=|RU!&ksiz=VSi#Ak2R0ykRSebLBc);XfM$Zd22^AJ4%cns}C
zqG2EDV>hGts?N-;j?8YiPkcb}(Z2s}S`+F#d?M2cSJbpu%C?DgQYEV1{Bg7L)%bwl
zW)Y7(-4s>TuG&OhE5%mh;VtZF|Frv;3L+mr{gMB>W9|LS81>dLBFf@Mr#F9&K%5oy
z&5!a`%h>SYJ#;t`7bdauN6Y=)-ooAHvsKl+r&2s_9U2Ws`D{dxt>x4G!s~f=eGib$
zbmb5Jg2`v}WsYrvwShrL#1%3d91+(^gs(Ug#QehS8hbk2r?f)!UIHTm-JIU5Ev8X5
zdOkj)p5cMSXrn{iQoA|i_UyH1`p|K+UQWQ~R^D1r_xaUz-@TKqCPc(PBCBI%)Ica;
ztqpxJs^*u#qCKPNX&<Ms2xdZ>Uojpf&xmr<f#pu6H}FC%m0&j~<oM#mH9=^zwxS+b
zs@h5#i?H)RCsHEkVgB6N0vq?9?dFT$B~7Tv>{40_L+{4ohG7Z{77kDwRo%u@c?g&~
zSlvlziy?lNwqYiTbDT+w88oKvxeO_mujp{^T$xv&mA|c}+psQ(FHQzc%5CjG5{92%
zHFU@ek4J!@Za6Q-BRtUh-*s2tl;ZG-A_#A8L>%R8C=6q~n<O<Zud+%<A#C2)!&)0|
zv>R&v{msSk`kE63O5u((Z>|sQ|J>TT(;f6Z%C3gou5tuUPs<skNo@<Ce7v^3oUc)|
z)!zz+T~s0#!}Xgw(u+kD%omMQd=R7p?k5gmrMG`KWXAJ<He|tvw9YXMU_+js0W$<+
z7)fcZ|3=Sdh+rSo3E>mfj1;c%Qj2e*1_wCX10?PSN(0PIJyDCT|E(<1&{52{4pZ<R
z<q85T<XF#j>abNdd}uOJ>-=Y!#0M@IJXAemNNIVMi?*L*AdKtf<Ev1$Yap4CRb)TX
zwg(SogO9%`-+b0ohpjVE`>n?Odt>4tQvn;$i(KZ&5(jz5@eo48!52xs`ANIvPg+L?
zqxK_b+mL%wVo~?hmDaRjYYc|R>W|>IIRaSXF0yjz7bpDw?dr7jj~lF?b8+C;<qZZ7
z5lJb0-u_?P5}WgYy=uJ(hrP_ST4=1pqY)Dl4qI=Q&wLB2C`qlJwZeJJpzHl@Z<lK^
z{>vNF_p44fVDhO?y+zM7zmf5@yA2xJp!?J4bW|TXv(5g%GK}A(Z)m)n7j+*n+{kJB
zCiEU^h~HGFP*e^K{^(6!a|tMeSyeg|pO|QiyO~KI$$4>;4)515luW?gLfYJbjGP&R
z1oK+%w8lxjV=X(8s!KYm=~gE)@;9S}SduS}7<Wn{`nMmmlG;q5+gPVZpX^oKD}HBT
zttHZlNVZrl)OtmN{UvT<_Qi$OXe+MEsx|3Eo!%%$Er=sBuboR>yae<i{N`6rM4dKb
zoJV}8ZqXYqs~TKf8%mm!sVmPVh3h@<1#AEZK+W)qU*tIRG-SIn(6*3TP-*>UEq8II
zUwbaOByiMf58<YC#4J=gTIX_GDdJY%{8-}s`Ja9jJ7kr%fO67M!p-%?Y0VB-6I;Ke
zMY6$-y<Cc~Q6{5`Z~utfnSr%kyiMxVUfmMm<>u~SG#ElRA@Ng^j`#&_H(^71TH}4h
z^u|QxshUsfx8n!5^qsplqI-3kL`0n`X?*R<;LVqMcExdsmc3l&Q~9NbbR`a-eq0(D
z6a!vMPn-Ysw1>M%jZXo!%5vg8-i$>fpMz?{(;cqnTnE3EbCPXsK>(>2Lza@G-K5W=
zAkY0qG<_csXJ_LCNIxJf%zQKNX|RM5u#&zpZ)<|?jw@2foTlwoR%kEglCN19Z{U3W
ztwUaZt$W|3b6N%jO}%>hMa9kVtVdGey4L~R84oRk6>&~&xIR-3i{`T*>Dt>I7r#F4
zghPFM_yw$%Ng6I9aYrGOjU^3G&%&0*zkQ||um818G19Td@BRI5E@LE7+#6N@ck@~U
z_dXc9Cb_KTUBxgslnFG;RvDEJ{{cFM0Z2XcAXK6?JbeGWJu0Ig{-<2T=_x8hH6>t3
zNje3)eWlR%q?<V)JLbBaeusEAObzDTj$ESoQ|LMU;dvOL#RsAB&fpBGA)Asg<Fv@6
zPrnn+tp+x?YzC5`s>}=xLgeK(epCFpwOp5td~r6fG>9|BDw7B}jBG~G(t?(>v8lG=
z7;Bk@f$Lv7`p+rslhAd^bgBZ$a6HI3NDF4WDp&cpg)_`W!dwlu1`0WPmx~Fs<PM7f
zAKV*VUhWf3(drmE8{o$#1seVcn_F1_U9sL`1_Vd%&QvM9;^WY*{3#iT$;^~wHTwMK
zfz3iCo0vXh%AE6WIC;*^7aBPA!XO+6G{BcP`%9#H&vM5@7a~l^oGvkXJ!C^fg!}u-
z`P}dJj5xE8N1+ObDWX{Y4Nmpw9)!Ox@)-w-2zS7GZW#oq<gysydKz*q(&zUGlx|x;
zW3J`jeIb7Xq^aE;9_fxMudClJ?E9VB3)z`v1MNHQADQ0d3i$IUw_`+0tD!^eh1a!g
zrnAk0c>0rgdn4<2yLjLXLyj^;%^5BaAGK5|Ek>X?g~!c+sxFp#Uo@prOr_n9Ec@5W
zKjo=fqRNOnW`MEp-~}Stxw~?(#ARf-L~EL28uxUU6zuC4z@X6_)xud%VaOJilor|e
zey)Ax7~StAN|ms>(U~E!7htOH(CuA)8SKO4HH?32SL+7jOK=Q;kF7QqDp&Kbr#@<}
zlGS%$3bRW=puIJpcDExs##Movt3c*JTl@Z~z3R*RqR-g7zq3UjG_UWKZ0$g9V-p+F
z5bq)=EL525S1V}*d^hu$@IO*@iAE%u+D+=aCa3zWb!#;UG}|Ypdd$Uj^S{Oy^D*Ix
zRQVx!pLJu#hf98Y00%TN=QwSEGmJ%vf!if8yH{UF?Kg5i&vxtpN@T3UgAJ*kwZj!f
zv|Yz6ab@{c?YC?R&F;fxc+4aFs_GA!!sVaHA9+FOt`wg@x^IZFIR+tz))(eNr{AG~
z$dbsKyLy@=;8uTqxFh6KGFf=q?hpT(uSiL3WNONlgTd3jNoaytYDjyneD#&1x!ArD
zo6pJcl>Frw{vsX@@;0@Ixw;T~C;A7)>J5E@i$}a<l)B;9&hKHf9`CDmYm4m4t~<8U
zlh2^HZF|jYWo=P5{RPMMD&awo{bd0wz_no6tusA{HNdp`q`Wv9vd_@DMr?`w>Gsi!
z^+5`z<lu<f-{ZF1t5)u(!wCruSYa{$`cR|&a{@fS7vI26gu(MoF?tdxDa9~U+5Wfe
zQxyjTg^0uo)7u&cn#PGTZW0Xc7&=Ftn}4;yhA>J%E?@RmA7G;#X{b}sb>VqQPiX*%
zZi(CoXq-n&zei6-kC|+pSE|416&ijV^e!pzg20P}5qj0`G9>oldAAJgs?9_dZ(J^l
z6B-+((<NFtma77!L|eV-WKA^NLs0<pGyYL}oRo!D<qa{Y=41@ze>#9}*t2ZCBDWQq
z$%iXb)`?l>Z*40-reh*ca=!ull+&Hi2a#b^n#X-gVZs-d)V4rM`RT($0gSIXi4Hdv
zVq6vb20cwa`)=BUhk_pF%Bfxp5BB!J1KB2aI?2Q92PP>WdSsFBW2J6T??$EGD79zb
zhzXrwbE9&c3cjAU@oEEQW3-BIuPa&b$!MkFV=W<d7mbnEMx*e&&XnLT<;U232G<Wk
zM)~%Cn+5atHWEwVW!QXUh`Bl!rr`q-@15cr1W=Mwov%(`De#%kW~PsG$HU9rU?`Uu
zTooWlL#_hq*4C#_Av<=W)0WVXoOt`rK-6}&C7*l|c55JHwL11HUX`j&<$NW70(Q8k
zhFnh+L==e*L<^s?2FPMMr+S>#2{<FV@<c46T#6MgIr*{qu4^ca^bI~bHjT<ND)&tU
zmzc~(y1S>l@O<bF^`E!JZ$`by@g`KZ-y-Xh3GD)Xt}EZXq&;cCqX^Gkd~ahYogt6h
z67g}EoAs-im>5n04W<0-4Xb`VshPQZ`E7Cb!C4w@s7mRN{7=fY-RTmHo!YX^L@L-K
z6@zcTvNK;-?1OC*z>Z-CKhJ`yJ?W&C9!`o<3;9a$xf~6xVLj^YQb^Ht+3?@j$U8)Y
zARv6LKLjRq&E%ikI+_cyX72DPv};PH5grDy8ALwyH5xl+Mg=rs&Zq@m%u?8pruF;Z
zo;TuMj^a(*19$Q~1PR_kwo}}w9Idadls$M<C?CxWdG08pL|2~eyxX+&sq}5`%*pLa
z*);jC4{c+z-Dv|}7MnIW)#J#_{oHQDw2qU;Hx5RKO=IzhwebeE+`PysXOTaM3z}u;
z-Z7<oC3<@?>wjl&e{!Fwu@L%JFbNl>(AJ@6gyQC!K4MRa{+)0uRp-Z2gu7^RDmt&V
zRhPNz@@h#VvlN_e6teI3diroIl^6#-8RKP3dTGNXP17w-emQ&&^#&t}{k-TtXhh$A
z4|zI3nRUd%CiA{qG{W1b(!g_uw4)p1#VQCV31BJyeSA3xN#j)(>`|zjilb~k#S}*j
z5uf$OyDeF@0wY%oUF!;5lkJ#LT~C<H#prv3sEFJlSG{V5x1~O$sFj7;+3JQXtx?oP
zy%h>tE*uzM3Ynh?D(w6627}^c_GYQ(@B-A4e^4)cnBnrNL`Gp3{8v_I2>f$)SO+Fu
zPJqzn8^{+fg#&WHYDoQmPrltG-w!rezM4@e1=PNqWQ~*YbHM%ySulB-OhK`DcD$S^
zCoQma>E~K*(yD?-b03V&?MOzj6{gN<eDI@_Qw?u@ieN2TY3I3)#by3uzt8!l+~N$G
zxOJn*cda3^e%{4uUWalCHMxFPN{7s=v@cEA#WOF?uStrkQ2G7i4B5ej0$fHXs<C<(
zBv0)0#VLn#jvkPN7}EXt)t3*K$PVsURRud+*{!z9?clT`zVpb2G6v^d6evY~VT5fK
z1Q>!hzl_O(UkU2spZeObuYO+|tTCUzdYJqEW!yjGsBw@k)AI=VjPHnO1x_oAbXc|9
z=Fo51NUEpue4{x^3R)Wet>EO7s^W>c=Pauv(TpX+b2$Tl7H_?J&8|5ss+S(pub%oZ
z+MkL0I7HBuJfWbtV(MIQv>kQ>d!gY?K9v0U^tk;=kPa&v-p9Bl^pY;$$Yv`ewa<?z
zF+cRbm#CCncYXNTw|6UQ>gJW6O<k4o!->Hy;bTTGwEpoZLsFGtHAD(F-<NO7scuM8
zxR*X@>D3SAjNc)GIOxG=QJ^6ziZ0b1%r_)xT3C;EEWUiJH4fjva4J|_yOg^;UHbS9
zVqa@R<+9zcHc9=ob1RC&+uq8@wpva@{HbzI9fQZU`nF#3A-H*ZGD&dBv1X<leGo#R
z45H9$$Y{G9Ngk+_nI%=mrf{C%Kb>oaMo!i{@+cNgaAbidOC9kP7H=9ps_O`Z)z_eZ
z`@rVHl=6w&we9WOM<3(y7ip7IOMZvZg`A4BroGC3auIv4tWzLFns|~W)$JSy;GbKp
z7`}LPb;=7UxN+#`{~zYwDk{piZ~sO_LWBX4kQzc65h>{!Kw1Q(96&-CrMsj<x?8%W
zyOBn^rCX&N1cs0q>VJXW&-06W<Bhf6^={&ZOE1pz{O0jFY-hQCO~vZGDD?`R73h3%
zV*H7U1VJCJN7F@<8hqd2=4tBvuE|^F@=@w<lW)GKFbubyn9en*6gpTAvY+Og>|noc
zcp=u$FUu2s=XCpuYbulN6+P4F%fHfg?Y@2`h>X>#br!tkXU9C2t@@=T#xqNPrbX)M
z*%<}>eQ_I<_0nh5xqkJ+>Xya<$v~kUFHY{eJ`$@!Whs>)<#)VCYX|!}&xubjq;_?+
zT8$W)pHb%AsxllF)JDdH`PtgG|H#p|*o?{ib~4J!H$ieI=dP&JC$5}}LW4r`!$Cp9
zl-*jNH73`qJD#ddc--o)QzrYhMnksG>G*R$LTl+PLDoD#y!41$yP|MXXi|&IY=SdI
zFJsO2a>u=*`J1KtE0`COcF456<8r#mZ(lh7USCa^aD_7HGD<FagzfSxDeNE~7<!$J
z5nd~(&&E<KcL)DguwONLSA77T`3d_g)<!1)?`-%OA>~78=!6vxI1u?cxRfX?`NVK<
zwBPrDzjW9iSf!i4DTM~$tUG%NeHHga$C$8?-9`K-nPnqE^&ZXQReLS?8<@!x31BSc
zVN@K`M=0wCe|Jw)t|;rUwko)BJ3lP6%w;^Ipdwc~naEp&S)7l?!EjWg0j-t6LxQZu
z7i@DwT3*P5E!bYq(Q1!54ngS0YO^EZ==Hdh5oB+guRvWCb+Vei-})s?QV0X=t+u(8
z1Hh-oC)+we!!s(W)QQ?I`C+U`js9{*z??{`(*A7Ef=B}f8N#@i{~I=*e<sS%XJLwt
zmldm~7a;v<W2!8p@dCBQZ8`ttt;7+}i;_2WJ3X)QB3sV7Kj1zw<e_#`wp}3|Xk__b
zv7E=nn+eI%Je{gKJ}C%$_Yi@Ru;5OmuMpz$xPpV%wBFKAYAwdJ^F7D*p2=;<k3+R=
zKdOqQvf<sR5-1KvBs@@Wq^HVH(_kPG0c)K0gxz5)ewc5O2SOj<*cCJrqo!1<<m{H4
zEvXk9OSb8nJ{~3ockI)8ij@5b1iTt-c7+ULzX>O%MKHvXdNMq$KAc@dn{^lufuFbz
z{vefBbPt6W|5{8ai55A?Y5W=n6JM|#6}G69(sp`)$`-DHZU+Cj^{n$XxcQyIHz+UH
zq@&9LA^qmPME9L>?jDH+?||b9`M8?H{(>btq4UsfIK{bBW87mQE??FjWI3hd(#q$Z
zZ$y5bWp0*FJLhG?28xp-QiNs0Sv%bkwJPfFZ8u{j-CcNzF>CkgL?_af>v@~(yNiwo
zn=c@tgUzcdTb1H|ysO43tmyGe-J_^3z8o#e>aL#ZJvy^(;AQa_zi&<5xSJMyn681b
zUB47jy_?>+h{M(|QL1Kh&K0bzP9Xn+FL!d^s{cs!=HO!jmnOVPBD|?1Uazwj#s|_`
zi|~0$x5Dbx-s&xF0kWKM>8Zg0tYoT^sR(m<SHAu5$2G;qiCS*(peojIoV+#}ajYFT
z7reVFZg>PG_S5iL%Ui>jxAf!|b|1MsZ`4PW_4kFWp^Cr%n(c8at~VYwv1*TbKSTfP
zqC%yIPK$3U!q9Dz_JtK)B=(aoDobBietVZ;{<v*dY2E;%0=$fig?j4GuS;rYor)`r
z)k{uy^<4~rF-!c>&(F&JxU}0I!jw_2^LJw^-1>t=qEZ&cv1@5PWKJrcvTd8`(G1o8
z9?jwZW+hp_JXHJF^7#oP%@+f2&IdbLFo4zG>pDX$nYKY34j6oYL$ASc`5g8|^i3h=
zA4vM%vuL3^98F%fhwTxuz*<a-V0%<hr(fWrr2kFNRO3glC^211d#HUCyA#9Up>N=z
z_He(HnJHD_I~7%rc^dUrx}1jS+bgKWPe;i2qNpgG#g&>rWaA5~R^q3t`&(yHjC8J7
z-9RU!FDEK+;Lellk>Wl?lcR)RW6~j*!YIxwOnSu?CY}5WlMYJMJTZKtb5w{soPb-g
zD6~K?AS7pX5+!Rtu%Q~o&Pp>uhdxlW)EOeStCWR(t=m`GxcQ`P8|)?0_SAmP_1Bbs
zf1Iwsj>-voNjsufDQh{B$ebTaZ`jYo2zzYx^iHJqj`Cv`p%qNT9dS&V=V{0{XIS<K
zW=Id`q?_N7n}z$-JZM-7hBv@C|4ry?#n&AHsRPINsRkv!OhKEkdmj#2j^IL?ED?QY
z-Q!qVQCf0#ivy5*TFEh;gFqP_^FmPg!NZ|Lu)T<24CeWlWXu_3*^;Y#T8F1$_+-KT
zjN{dl@WR8aUu#j&upA0v=P@&T$4gzwpEJE&?x37bZ<D;u#~V1LlX$fQl8pztk{^)J
zq>B+@0*RVu*q(*0vA?ikbXIhd+BDjSJ30<{ec4eQoC>EVKbzQ2Td9=#CKe9kl26Sk
zv^w2R@{%p*)#Fd!eNLgmdHU!ppRP2`kjpy&Ce}@L(_Y}~NI_;zXrM_RLCSp$a&H4%
zb%QnN>=b_EZHQ{)@uGCS<q$FVcC}5>J0{*QBnndtnN)9=m?u7;<8M)QrG}i+-L>fr
z3F}$ihxbu&<B9tvbFdcEFFr^>##k?fQMtT);GL^|kwCj4dqlv<(zI~69gp)%sVU2L
zrBiI3+;*RR+6by+eOtc9$;2&7b9<F!uFBa!z-HYPopCDq?zxuNPKSYZMUX=C)5z{#
z$L~>0=QfuoVhYnovBds9Q%dm^o~y8C^}%D|PYSCS-;`Nr!z)V4yfA;7ZJiqrya0)=
z4K+bJl{24a0S3On9m#$8`^spbul@Xc>I(d{7bWH~7;&hs^gDlzIbCPE&R)Sj+<cr(
zr=)|TI-x@c9TLyYTd)?y<XlT-o;3B3FJU#)gvjkIk^|Ix+6Sv?T^}b5UTC|N$&A!a
z6r=i$+V5uYj)U?I6>R3)l6a3rSZrs)tQq}QTOJm83Of}^H5s{v1Qb*~(k;3fg=Srf
zy)vTaG;j;u$&&_bcDIrH!QQQ41VILaj!@E@!2R}2s{|mbIS~@j>p0;w5y19h**c|;
zvbticG7RL%P2}(J7ezJXmn4bCtmqe_HWH%!$cin2o);lN!Y<zYO~T%E^+m>9l3^qi
zef}|nd54UkxxWbzQ0&xg75A|gY&(uJ$nBuSEV>bgwYhs)@3Z6j-T>2{3bAU&qz^52
zvYP3fkZDBhYxOy~Q^|uw_>anppS&D4ayXm{&F7VPuDK@|AN$#P<~10K4W3auEzs7l
z1Q7CtO)Dp-eb5)*bLKsn{2VQtCR3u8kXn_MQ@ZWgLv15?ZeWf<Uff6biVj`QY2;2d
zP!&}+Ev`+h*?b-NtYqewIx;w+X>nB30J%7>ENVJtH7&C@TRPj6l_cYrL1mp%@%*s(
zb&j-n`c4Sers>#)l$cyk0&2TsRFK6x=!S<$Dv7%7LN57n4n^Tyge(1mCy$eqFOxfs
zkhD@co=&$y(w-b4<0|5yc(Ay7p*PpdIW=vFZlvYVh{N<UIV=!HN@c0CFvtcYY6N$S
zp%yP!40OW0$sbDhJl$<P@=SIc^Lf|B^s522G{`sRR))Gg9AaX{JCMVzBrk{&y6`zQ
zR%?KZr(7bB%j(0xL!PhNyZ0AnO5W>7L_SBZ*{yWiNR0nHCG}6>Ft~W*Y~ilf)ULU<
z@<wWrucM@Xn004<=~ukTPJ`9hY=+rT1nC%!*Z8rW!jHqv>ci+9mjOOtj`DE5{PovR
zm?&YCu$Ezomwna1XO^?quX<BTmTa!W1CY*L8>DGV*M9$3uIILS#dL$-FNoD{Br>Gq
zBt~YmVsiqn9Nd+Ho|q#+O{adiB1EY)cjyN_NNEH=j4^26fUfVSM+7BdFYP{~Pt_ZR
z=4P<EAtI;}y&Kk*6FURj{MbXPXf0{gf?RSlc+Uu3_Qi&>963Lxs9Do&sM&A6A>}r5
zb=iFKpco`Wn##gG&M~1k?ZKUaGS&2m@bYaw4DO-z@FyjdutKL24ryb%nach_7+45R
zM#$z=72rc>ybw0zfcP?}Dl(N6+g`l({!&++s*YVLd-ejoklP9U421iF=jXedo~!zh
zAvC$8z-zmbBWT-MFGZqARHVVrrc?9!)dUlSZ2o7*=ick@kr(^X5lES)v&g<#qk6o+
zVjZ`GEGm0D&lk#%KSt^CvST6#UtgB>ft#M#6fk+}Plni6>`Y9vtU7KEK+I0)+bWWf
zE%V<8OMeEv58I>#X&VnV9K86Ewe1MMYdBMB3U9;GB%JK;;c^B|7c8S*b6XfCsd!~#
zB%iz0H=EVXWJlO?=m18GFy8N$Qy~*?TUv#6HNjkt<jovk)frq)t3nd@Hj=1}jnru_
zDm3cT_AV*LSZLP1uX$*h2JP?!X#CLJXb)n~5Du8UmGu+$-BHy3;owk3NtHpW6C+pG
z{=NZLc^CPua=+lDw{1tyH3On!ZG2M=W$m=+@<&mi??-Xp?FWS7o$K*~d`Dkk8XO3<
z+Ua!L{nDl^G0{uv1VLBnoPQ1cQk0r@n`CaSdgdeiTF0G>F~xt_)yI6_`z|tfjXdE-
zJzl^34^j7CfRun>BE4uRvpCi<Z`d3tP*6D1HA@;(reNf`sNn!rt2!6olMSWj>Xhd@
zY?z4-!$_LItqNrS_2bz8yW{B15E2C~Y>`O#v2f$Es$TipsdZVR^4aqO76VAidxs}o
zTgQ<c#b0BK*)Q4Nus&1gD{;P?<Ni@WVDU_vx<HZbQSnJ%>Qls+n*deJdAxLq`G@cc
zi`a|`kZCmJA!vNXu0Zg|wG{x`bx<I87yA1!1o-L8Ak_`1>?c~I>k>%iCsP)1DeAE@
zp||C(?3H5Ke5JksIFJnqK~J=stR72SNF^WAEYm%#(xotI1^lxS!G5DDfB1{AG_Sev
z34_q&q_LdP%|4j@I0W=b;{-sbiyz%5QfQN;TAAek@VHZ1$9`ikJzZgo3G9}&nNd;|
zUa6@Y!3faVsjuklY{5;3fl;7~lZBEgwqQ38=F-dzsp$GK?dU``A(|out9%)jA{i4}
zNQhcDo8Z0G6WZVyEGirC6v}+rLBT2}dFWI$Pf?is(Gj}DB!8v3kkes}-Py^$?z&RZ
zpEPQVsIc~D=Yyfl<#^}a#>wKk&HM!1e5rwPEWZMegsR~CewTJ}LR^SEWbcm#Bbz01
z@LTf-_}hNw&T&Qct#x;HCLk77SgEP!S(aKjoLeQ8O;inbo@9GJA-hv)!k|E;?D>3e
zs)PpfT{$VO_n9G=br+YRW-rj?o(C_)o;8vFVyrTwXgHl0q151>*-R9%LEgAAI&td-
zt+;5g@e>&tovJ*?mAT?l$Pb54+_Vx+TV@UV_QpYX<_A@IbX@nVR3soZX@V|;Q9;}5
zZluLJi$QkYl>XR!OTx%+G_1=j;Y8P)^6Hr>Wb$Er5BP*ImqP0=zuIp%+8r)ECu=H|
z^8MzLU8PmH*Z+y`%wc~6wzJsM;og;GKRjH$9VjbBDlf9|;9fCy9jdrBfI2X%8PS@!
zlh+l9V|2o0F)Ud3-F`>tcqln6hAMR;X*Q0j2vC0Ztw;WJHD}K&x+?gECO)bco4i0y
znCwog-(-~;9kfYGp|suNdOzFtUI;oEH<q4s@{s?1`9ZNDpDYiJrp)oqGUK$1w%671
zcS)I2p0$fPs52M@td<LUS$NHA8WG<pxYDLh=f9g)%$db)C<7r>bp{TC8u4hlIj`p2
z$Wh$iK79Yh3M)xBHMHo`Jx68G83+j{M)cQIKctbc!Cdn766{}-qJD}Sgm3r83`$3`
zfRL~nvJ$qUs0$o>|NB%kpVM1zc)-Kg9;bP+hR^Ip+-th?$nwq*2Q0KQHYG)4bAdRT
zTsC$hHk4X1rL`~|EAcFMa3;@_O=_Mk6N+TF$0^%;peCg~A~@CU^%5&_v!Hl8a5O1S
z12>TsRw<2OlT^DW$L;xK#NS0p0N%0g39#Ie(f`A8zhCfi6<{U_{bsoz2Fm<lxeM9+
zh2>6t#d7bGyyL1N{9-AgCrc>Zgh;~x2m}&#zngh>!d<pUArfgaJsiL0+?%zmBo?A0
zY?+UQpw@mFrUUOLgN>9UJI=;DMQ%m_W$O>z?T%kMg@m1q50gHO6f(N6#3v(TIZT(Q
zBI0#`vmV4+S98_sW(PHIQ?nK7^o-hk+^7sr9zN~ww~&7uWh!b%$;sg9`!E~wC7SNm
z(R5~sS=H*RX>$g-__tWv>@;7umN3Y*eu=k|N;bYB{SI>=QE$Pd9E%aL*?~{aER{fC
zP4p^#Ko6ik>C3P`J^DqwvORkV>NFz9o}{9x8Z@#h1Z&m5ujcTjjCECa`yM`PIx}X(
zJwMgBtfwG|0(MCE{Y}57*?fMuy{!u&*{GooFo{&+`xO{v+QSkgrJ4X~)WD&KXdQwc
z;5f-#XyhF}&pXt3d?xJVUYKm#U3)v^+?t6QPJl^pp=phNCufsprL*h7!HLzx3Pde7
zD>hi$uC_Jr+lfgVqp(~e!X#7Oti)2osg{NNWMoP2iLkM=k8<t4<{ZGFkk%@;d`wzj
z+o5B>H4T*bX9=eEr*^-lZ@w2K?>ach_%?Z-TFl=;qLUAQa_jybCAG(wpL;YU-0jP#
z_Wa<co6$zY%puLEAN_WI5@wYUya?bo0K4afxEy`23=X^I1jbqX-cdZjix>EpQ{@+h
zdp9E9PEo^;MhXBTzjrpBy9$}7FH=&dcPExRUHslcT@4PvH5bIPB3ljvF@xKZKqCtP
z*Ic1twbzpsehoGsMd^|+?MDgM8v20BaN#QuQj`+kNCNpN6T=<2wsinF;-^p23EUt#
zKlLYwf_;52ONz5g+kn=p_8C5NxCHw|_L#C5>Yg-7m;yNnd*+@K1{DA|$4RtiOyS)|
ze||J77^qu>h)s4xrA1>=Sz*Jy<6H#!@)Y{kT7}08pvY22@Ekc&UlPiHGlR*cz_j`$
z*?W8-Z&-!#-cbBos^fK)>QKAviBfQzs=3|Ad#Ym|uwweMWVGTNkevV-yQSd?X+ZPH
zDoee2rAdvX(djzyJCf3IR}l<h2-!R;(5@Z;z%t|BJw)5#1^Ds<uVCnSGGDiCJ~Dr|
z+KAh2wz!;RaW*tm{CSI1Z3y!hl>Pwuo>$92jrUy+ZDtt`XY5gZ{|rl|;kVoE53yfF
zhQPj0(bQx<HS6S@-7R;`$cJKc%2sWeuI=!h*2Qjto4;OM^v;b&PCEoh#Fl2wFsnws
ziQya9`*LynGIN6NQB4!yD&DEy2x%2=uj4OaC(xBL;S)(@3sitIr4jjT+G_vU{(>t<
ze>pDE_p$>Rv;;iC#UPe%SIfoo`x@VfI!m;_$n%+en?Uh8uOOAB?!wk&&WgSNRUl3I
z#w&^`sNbHmizxn%^Cv4jqDWzh?0YR&EveUocCSJ%l$(-`&o3QJDGzQ6+4rQ;vz97a
zgiUb%bX;dPcqi`1qwO~PRmkc%wa0pi_Ll4ZG)%%fwa|fVyLxcfY%AG4IS&D;iKdt{
zn`VjNj3$=GOrm_cz6dvsU#Zsz+DCljO=Op<+^Mh6^ma(Oir0<D>nzFD@<wfX19WQ4
z5=FL+jES0(T$PlnOWD?2Duj+aw(Syyg>p2vdqWgbnZc$aEYC}*WPi0X$|d!;Z^%g3
zmYjMTQ0WyMs<E#w)>2NHypg=^#b!T}8d}OAcev$hTaodL>TPNW)0vjdNzdtQ!d3|(
zUIVp**78I5j%n>#Dt#BrrR!C25v|{}&9PfTVR;IG7SBvi;d{BB3ykmG4Cr`Z8N9s%
z%ATI@>zan74lr0w8psX)bs_jlA(;u37+#@eP8;I@X6Ko5gz(jLh*%!}40!i=F<rV8
zN?1ovM8ZI~k9@HNxXInH`Oof^P>yVgiLUNF>h@!Gvh_#kij|-aja)m+vyS}JbzxdZ
zqQvmupmW1~)==@6?;MA+&Omwr=}nu2C8+hcVvq=Ju1e6f$@dC>6ak`?*X)_cBVlec
zaxgjnHm50PH|>lAnWT)3QQHU(_9{?|ZxEhE!;Xd-Lub}$^V#GlNwO8AqsYc%g>4YD
z?Iv1NgacL78~3E{d~m0xa3Sb0HY@s2>`XKCZ5t!OUt#>>+O#ikMoJC7fG2kqT0p{Q
zrCM&tm1$O6E|O8Sz`5g-qMcf#wz_^X@8aW}&#f}-a|35z>neBkOqf4c7fqOPE!Cy0
z{cvVMS=aRw-z?@VzW0o#`;9w9GjGf)BD8u5B{1XLJoRYiBtqB~Q;QD}Fb6Ef43|uq
zEd=I<HJ|ZnYmHe1xlPo%8uU*6;%NSSqwREd?7s3+Lz6ovx2JyB(fjh`ON%vAiX`;5
z3m!Gco3CpRdKCxrIEn!JXtrEJ=lQPPMv9^i7=!%o(Oh}6S!HjDp1hkNVuXJGan7%<
zkaGDT`@li_0xow_oEHam`b@G)3tskABwW%@>Wo(!KTFQ?LGF~@A2palFungQCjV`c
z#AK5D=Z7d5ucw5?*O%1p$LYsGT(A6(Eg0nRCs~u#4keJcCR0dcczOHJ+_I?}=V(q5
z2ROs2;O1$2_$>SjxSS6%f41U#y5w?h*`d-*f2t9s4oQpVr(K=mz%L(nQ+IGJEyk6n
zGB0V`({K2spP1|p7dV{BfAs#sHd)sl%0qu?**GYJl}a>;m1of0<~*A(1nXT?5IR#h
zZm9>qv%y?x7i07EzftQIW%t8vN$7O*pkya@t4P232MSg_li=g3i1n#^!8t|A+qHIo
z(r5IY1IIZq<f=AEabYLmh58>Hwc$LN-9{vC*D794=V%9FJnHnvz8i&JwiM7TOckKq
zVDC_5S~@*by#uZ;C1Fk^O==r7@%v_cdA8+V?pV9;ia*s+J>?eVz&x}BI-iwnIxY@D
z&M!HrPD(awPjb&WWo&buyE=SNcIoCj-j`Y`N3H0=%mv+l@cLvAv|0|d&8_K%xVz8e
zoKDhTmhGpy>Gt_Bb-C4EY*uf4q#%Pq&v%;Kr-A*th@$4C$@AU<RtrdyjoL@tXXI5d
z&3Edr;XA6IfR_E-izL&aCv}!dK(+TNcs{BbWp<d$Zm(N&`+0k{9S<rCXoSp3AJRH#
z*0k<S-G$Y3v|n5=dwL>&yNGju-_*)_(Ul%54eShWeJoM*+GVcZ;OYHwYUqTQ$aLL^
zyIvl>nlVv&qQpfo?ptqiXl#rCQ{$rmr}Nc(Hgvhv()!_aIefenH1YdQ2cTW+at1dv
z{*VQZ%!~FnUWTn?0*&5$A3%7C+fzdkVGg{Q`C#8+$<)anom#;Ez?eV^T7)YExD9*j
zRx=<7!{k0CjJ*k#hPaz*R4g7GRk5rLCF~begl9I1Q;e;~=LAjYZ>$sylw$%U`JQ`*
zt5dXbd+5rcK$SIR*zRB1%C91PhyKgdsM>b_mBtFoJEZ~J-umF#OZ@T7E_q65zmdj!
zvQDy<F4FX-30e8=aQ_h$B8#*ITc?S5odlbcK1qZqJXi1ccF3Y%+-oPXyM{`Le)=#$
zJuyGuQ62rVbelfIp$%>mGk7DnuZY0=`=}uTXVgv(M@|fX$=jRn`Jwfn7&(?Pe+j>D
zK1b%wTF)S`IUk&A(HI{7$W#bv$kST|aqL85u~d^vHOA#{*?pVBAn%rKLxI9D;*}`k
zpSlJnyY$J}x3WMqPJL#jVlUe8OM6BcUb_++(yS&moxa(cNPnI8@ZN^oV+?Zcju__i
zuxSo@$~3PhZ5vPTcC+SyU3CdPm){GRnKvGPXRq5cIP(T(Ep&`;3@MsCn%8|>rny91
zdfcX{A;jm&rE|KCW<aYLmS=dn;CD%!{6X}a71o`v#Ssx4%Lx^2TvD>OKOd-me4R;i
zXQPKV9!_Qn7-ooJ0Tb1V9%{`S9gN_-;2MOy^YRb(?0jAp2=(1gC9T-p2#yku6Dvge
zntqOVx3;+;26;qObcM=gnBS$@Bp&lcz4^4r9`DYl4B1-J4F@evw;xBx{X`w^uo#ND
z&zCF<8f9{U#xV(&t+%H_Ko3GydOM20IZU*U8|3(GwIE8WY?H~?a5z_*emdF<dMUMX
zEIF1mih^Fli_p_C7w<1~_w+1813QEZ48*0YbtQ+pc-B@<c(CR_Ga1<Lyt4zFg+M;r
zbgnFlnas^kvMK)1?nEfy(#Y~_`O8EJ^y#X$B&xf8?Iq23_#h1AV3vlV5B4*(vKq78
za)n!w14RI`q**wWl54=8ch428=O67l!<QH*g+9KSjfb1bBBN<x>4nR{cxM5(GVIyX
z<p{|Yq#o`hE~G=U=Z*(>KQv$)5Satrg-q^<SKU4-d_ted85Km`mS4GU8=c?REvD3s
zBZQMJby2*%qSOxnl=^rM$@O@EQa_WiIQ^Sa53E1=D!!#p;ezLMky0I$iw8Y|$he7W
zJmR}w@$v>5ugBx?N+TziTWQ?I7O_*g6ZvIl`v6mXF%Fk(94G(LlnU5qalMM$rCaPG
zy1$ijSUXNx3h+)Z7Gk2N;EGP97WM1yoDq*yblqNtr=ULL9DKcMPMH0d=7cZZRdYfL
z>^jEMcsR06TANT{=rFKaU||lVuZhfK<$LbPH75B4{vMW^H5UqEc`})I5wC)#$^@tk
z`fN2ARJ+(=SfbjFJ8hH3Q!j0GT+}HjrOB|{DY5yhh^;9rme@P_5v@l226K+DLJFL@
zH67a=Bi#_Sam|MpvfNhB2S8n*=~aT)79Hq$3z>wFne{=**xM!P^$VpGcu)J=Et;#n
ziUo6hUCjd}fb0fxzW;V9FjIv$K!4p!Y|%)dYlm~_m5$SqVbfj{E?s~$fV*26&g0~@
z+##Bz7j4=Anvm+fo&sR70=Rd}N{#01bC<sJ)T+rD|4(|{1TGrif{LrI`l*Z3ek{Uq
z&N#tFuD_*gS6TeiFp73GiKLik5rYf|wtk1o>ecGt2ANBb&hr}iwcO?W2lB6LQHz8U
z#m-cfCW2g<(tIYZTvAud5(*uR?{q9%EUTu&mkNQ+tC**=xR<WuEPPA(k=)Um%66;o
zy1JB25b_Nh4OUY++htg({fXs{T1y}Sqh<$U`R9i{kzdIhE=FG4|B#7Y8Mo3lcX4uk
zm5084UYrQ7bhdEL^sT*T^=P-9QBrfQ14jVZXu9}3Yi3%%+iXMF^H|3#Dpa0tyTKvy
z`HxpD_A8wxLP6E9ADm5-y#W@QB=SF{5V>6(46f}o?cHH;S3b)!1e$%=yj}RvM*>36
zB_J+b!z@Ben+`G4UtOzv<Sm_$(~~=`qOfk}NH=_b^OEhyIsM^xMl=Iv{H{A3W8}XD
z=r`&Z<Py|=t-z{dUhNnSBmjwP19B7h_-i+8EpXRZXaWavr<fZCzAx2up~OqI`%bQr
z)$<FjtH2DcE5Bsl*oLrL*v{PJg_Xzu7{I1o5D<v^1_zH{uR;+f1fBt{lLYM>ne6Fu
zxN^$`08l3hgZI`R;e&E^DG<cN7tHN|z8=r9Dv|H7I+oTYugM5d`W2SiI1gY?Y5^^p
z-g<|68mvvy6`=Gx70K<d&&!m`7jfNLQma)3>DVSLv4Y3(^G(gho+(f^*^%6hX3`LJ
zoNC~f9ehS;=4(~Q*J<>J($9ZQ=@*KGk|9Z+_-HOGx{j*~S?hUyc@|b;dl=ZEbJSL9
z7&27}qh_dm&>K<ARg`nI$0(-(Q2Gbst||S+08i>QfI;&RC;r_@Ar@}`HbU{?s?NN)
z_|-i7Fjh12{BA7M+`6(qo0foW2qgAJ%CXsl)P`Rh_4|jBwanR$qpQ_tkxXizL$z+w
zzWo$(2PWc5<xd;(&`hJcET;8HLK6%Wp}Q`KYSDSdhdf}#7t)Vbl%K!<Pb;MNFk{0(
zyTD#+ya^|#uYtUqBP5`f`+K2LMXB%jVvIVj@760Qa+8SSbVbeU%b_xHQ@9SiNN0+F
z;r7R`-#=pOd~zo%Id)tNQ4A?xc~$h<cSdL_7GXA1Sq+qM0+CpV96!JlQnUWhJ+auV
zz)#3BuwQntfm#9DLj+pAH(2#-%c9|9QO#!gv3iZ<8uGG1kM-6f8pbDNL|wbs+xbjT
zar)*^o>s$%!AQGBtDSi26wut$C%WmWXRc&^_Uy(Q9*+(rNwPe+YPT}33LCO7BIvj_
zpm@A~uF!9@OQ6d=WlFVGei+$l_ohL{Z|@M8ZFUu;uU6Zeu5^~vY+7R?P4*uOOnus!
zof)&pnKoEz@`v4Z<u%^c?28T3s@>mMbl5r8d|%~2H$29ud{cb~?z~$B+Ad&SJ>QiB
zE>AJRYo4=slbOQ!tL|Z$eqFI+2p=+lDe<FL*=v_`H;#>2hug&N29`UqLD!wZBTry@
zbO8w5&$JrP0t;qtH>=STNFxoi)c{g<uoS$XL@P5u=rp#~F4g6I*~lvkZb|i$g}?0<
z?{P>1sUFRnLT#rb`Luu`mF2M8^h1aEG$HtMkfFWr&~KitK_=X~KD7=vEN?7(;Zw+!
z>BdDEku5P@putztVZqjcerc!^n*1=~CL^tJ*prK~7E(o)P7NOJ@f?`zKKouwD0Pd1
zMW-rJBch;baRg<AuNTJ@>X<Apvvp69Ng7v?g``6bWgFo^|7JysJml7+GV(r5vfzNO
z;#w!$$gyGvr)~(aB;>|vy4=z!7A_CMpn#tpmJZYAD=1I`(q?Ie^Fs*_?fvB(C>f(X
z*S>DBp>UE2FkyNHY)EP5ANyU8n8u;4<hUgk=mV>X7UoivR^oyoi&qGxAeOpEr?-kF
ztzewk=)w=t<-E|AW&ZAKk&h3&Oy__px%~<Kq#2@2)+mqJQ~w(Tj=piYG}80oxNoov
zi*6TYQ(2$x@kJo)Ua`R2YLq>_8R`xc-v(d>U5&Z_C%Y!c1X{S=0OQKHr6#@N6^-X=
zTd4sy=f<VP<CL31bn<AB8fk}v4Bt@pGjtsI#`*%mOE1ku_U=%5?#}!Ym;4egV`@z}
zo|Cr=w*9!lS2FXP`-_5)u-Jp&DL--3@Y9Q1Dyg*|qVLTrn`qOd4B~us+NN!yIIMxi
zaN~(2g*;ETk4c#~n<Sd3clbRT#E^eX3ZGSAi!c7LnC@3KoAFL5uHh^$$GB?@KKnHW
zfAtE3uhbf}Dw-_xc11+0VIm)upTP$;ay?VksT!)nFABQC;OAQZ&ZH4o@lTqYa5$3z
ziIB}#zOElG2XgiEA8^Zcendt>ZU6B(fRB^*3fN^`A50cGIMw>~)$-IH@xXa0W^Kvt
z#m>z8K-A1#Lv3f|wB^h^g}OX=4A1lPW!-ZcZeqRK)_{T}&A)$LN>ATs0GA2R$8>3>
zhz)(9Sa%IBwls6I>DT1@OS*ivH7bye1KJ)j!H<)n(&3??J{pR$J<t#nsL}tiZk!gy
z!yh;Kk!+G9D5o|b5PVT3wYKK=l+hI_z7R`T9QM2?OcOComZ%pUt2W*iR<?S=CK*ji
z1;=>JAKF_7VB%{(-Fh6xUG!~6WoD^Bj%bj~Bh%~ACp>u9$4+>xdsEZuA+{AF#~}0#
zOnzKKLF<fwy4-X<zE9rOI+xmwb9`LwXq+hZGrJV4%$7-Veh}!GB%PTGoAYxp_;R!n
z@nPU%OaiRI52#fRGPe>%ZaxiSu8xyS;vMFt9_sktGxncv;%WVRnQ)s*!Q1g*e9i1E
z$0rKzcC=+y=!ykUFHXssAV7h)lD1a%*C_)pdX)t55vwE$5ocHX7FYeCQwGi}*tst&
za{d<0`aH!K;c%+O004HL=cC(cC6Xh!$TXB8^%hwFZFIQ1oRf7_W<sT*qd}33$ZtB^
z2#`_|?#x$sauh%ui%#LRxG|hTADVscts7Fi2-tv~mBu{02TU1mn7w-QbW+VOF6Sb+
zihUvi#V+Mf8_PYIy|3)HLa^~5j;`AYJ1|FT&)@~L6FrWdPQcZ8GAK~(aE29r{N?CH
za0I6?9H}>|cC^AI)My+I_|yAQuG{&HG6SsMaFvqZYe~Bt{Ur^w*qlgm===Y28YzKt
z;c!hp&8Wiib>i7TJKZAyDvHq+_S6b}APl{p^jf|3=R(gLTtSWVFNa}dp8B6X9$or%
zMh{){w?o+F#0ed4tkYIVFSFuH@W(bIx+R}D&JzrELw<^G7L>WfglYKK;GE<^QMSdd
z^S~c;=b$WU3G@h$f_p+NdN03(%DXMg+pxt_fq|rgLo~<kUGEIO^vmK$e%w(yyvUU{
zGJ&)-A_7x28z-*mjazr&+vox8A5uO-65fm^?5VnL9KMOm;ftp?aua$mZouMfi!YoT
zR!V}*sU30BC^^p7-g*$Gw-OXG@wD=t)A9BOMgp2R*YE6sOSogP3h-Vh1Euo)1Em%!
zvT_C976!WXc@Sk5&(+Z~utpF`k?Bk&Bez}e=d9;7=G52?#Ko2xmJ<;3*E>L$0n7o=
z$N>+B|D4O;58`Gh@a;B3wX(!t0j+)_LrFhwE=~tRmqAufkjGF^=Cd&WlSmXBNtmdU
zOHW>BJtV?%h@BKHR%_7M7HD@r=O~6$$JnTG{eI9KjqA$}@<7?GJZQDbU|crk;YoO6
z+3KPshH7?fn6;j)4PnOlV_~S>T_6L}V5_Y(B+2GO-p@&th-J~9Pm5(u6d&wi5!|Z`
zMfAaJ&W>Jg#M1}wXRX|#a-;h(U!aEgWWb!Kb+p`<REA=ElK)V+h*rzbxJ+SjI>)wu
zxWJ7}pZCeVPR#|*R)DGh@qc%EGAYdc{=))5#sD4GFn?fr{nu!4_56+e>EgOPT_G?;
zOas{(*c6c|bBtmd^kMqcxjFz%y@_7(t-id<xTg!Q579y7og>q6Z1!xc{xKkY3B5vk
zf!n|WV!&FkJ;)&#Kje}KjT(Ip*#`1B`10#w??z2=0FTg=VP2}ODAoB!hcgYeqCE1k
zpJ{#uuW=r<!LJg^jvxy2bisDA-Bi_aJL%@x**8cH%z&1StRcZu-FcG5mDh0J5;mr@
z=rGh6_Z<B0WNaoVuHng@mm_{&_{}OK1G)<5w;r>jbAbWLQ2-1N6@B(hTYP^;z^3TN
zm!qK(hlz-v4_{<F!Xnznw$gZiz?RLjHPqIg?W9ZQ8ef6$a<2rX5oR5UYec2N1AuJw
z6vuxXRpz7ENtl5bOTzM`{a?@BV}jom#DJf2Dn<gQ0Ay6~@{owzh9Q-xt{=}+s#yI;
z-TMVNR|PABi$&tjez9C?i`2U-`=t55<U}Ip^Z>3F)m2%>@%vZ;6DVSN%Uj~Fbnies
zczhOd{oB4!Q19bpwgGee&O=Mg5k7m<_E@J;?~?AvfIh-urN7IBYNP<OadR{(HsNGN
zg5P-f3RhbwW|9sYVCz4=#?=-n0Jz%Q71R3(llg;?0@b}I2G*sP%n923-1#9sy4)n;
zn=25ollXC~DYpoJkpxWv)&p#Y8CXKBCfaj_(Yi|W>2<=DAvI^ZS4S~X{B%7-Ogv@5
z4|@kRq3ib)poDK#Qni`yWV~!vHPiysmDN^Doy3(F6_2kB-vJOqZx_sZ7&t6AK}`QX
zkstnZA|Kg39O_Tv8*RGS#7#u(yfqog3}F6d&;HsWqFUnC5BK+(ji|*@3I5bE@izmi
zuV;%$pTKeZQMRU=hv03W$($)HxPds`fR_8|%#YLV61)6>B$IzU0w1m(fe#w*I)Hhf
zBxL9X?xU8QBJ($f;!FyupW#s6xZ;-jDbSS5kV#TN*Ea&zw1}Dl#r(QK=Y{WIsk?3i
zQju%6DGXqnE;FTgxvZBd`ccc6oyeB<VNt<d^3vHaM#{KNH=wHEOWULgP<CCwm;BeZ
zw9hdJR^Fud_L1o>aEgO)!JUW>U(ms!PuiJByO**&>Rut+fHx(l-ug<H+EHGX6`7>)
z_Ih3k+3MJ*fNytwrv;1J$Y>=$gleuY7w{e3r!VJF*y4b>`g7#p@XU6uY2DDj7PNN!
zMlf}N*gx9>RBE*SyuTf{tHY;4`FjO2AuPKZBLP48W2OE;!gP&7CmFaQr(CT*wECxS
zqI$u^w|1>?_p=>nBJYB%TqE!($l4m}L-5tUackGh*(XP74w6H^7B`<V$u<5nu=QSL
zMekZ*ge}C<X~;)tZ8)6R7Wepke0P!eN85<f0(iZ&zun-)=d|VQ^ml0|eF?3(H4<^N
zg@+iK<`)}20b_QuBFdzfN(kaLju3lhZbhY{!|imolEH^GT6E7%sUhlrI5w$WMKa#z
zmyi&fA2#R`<dL-NaW_=sNn<xSI13_9(US5ZkUAMn4<0<Mt32vEnuN&eQaCz5dNJOH
zI^F1n@e2{G<TQzI4U=cWzE{HBkdds!+skdmLeBz$R+MnjAbC7dMuV=Q88Dnmczq7A
zL36c#r<^|w;&g!g9N~ZR?ccA$AA_&~i1KT6xq&+!3BW>A*_?TW4X>O}m|eeQnA5Db
z+l4ToF)#4^qIhas-e_-X!JM$QR6fmOfQdeUC!ZFN=O6a>WBZry2`q6_hyeQ+%j)y~
zg6I$r2Ix)wAA4+mL>_Mp3i&J^(9TJ>GrhcHkDy<s!>2kP1L&VNT#dt@q@<EM0WAcC
zWdNXm0!~3{oB2(f7^<s&A)PaO4}G&W*ZreHp#$LT={kSWdl_ZM;&3g1{s|R#R14?M
zl#9~A?Z}qHe(Cgg`lqLC_|dbrlnIFNMf-EGFQKVvJ}cOXT%5$aVN5CqdIwH<L^wW)
z%FWPNmRf&jAP?P8QMO<82};IgB^%F*HT*u-5?lQO7@jR+kc+A$H(nP{cF<;}0swB!
zsG_;{m#6ag!x?i0w+4i%tIBy6=y=3Vg-7lauE-d88IF#d`6cS~CKql!gIL-eSYT@S
z)V_ChH!R-V03U2I*n~1`Xl3L$`Aczh{gHFAuAH7gG#A~Bz%+7{>C3*ZA;&W>tS{#k
zd+6enGCdF14K0cH{D{6#%A)~X%ELu^wk}|Oy*Gn@s-2GpSO(}$T~(#tY#Wv4!#NrL
z2ba?E0?Vye79SaSN&A}Zo!f{auC6@wss;ytNL#7-Cv>|Y7I7R5>tQuAIXWSsM3GFQ
z+>zlZ1-`vWFOibkFEk+g72VKWsE}1Jtt&Z%LdagIn=RO;h*-$33V-FLtJBKuwkNNS
z_95<2e1T_S$)q_GG79o~l`KVG74EUXGC;^4Qxf`J)hDQ-5gbm{GXfh~Ad$i+Nd4DF
zgh<D4f0I0<A^8W9io<fI8P>^ZHj>ky_4kkGwdjpW?B#sFG+Ss5^H(p?vsHu%pMK@u
zIyKk7c82-u+T6>zD_&R=q1l<s_2o}v<Ko;WUpLw#C)h_DqNR)USEdm*4OC^-7A5Z@
zQ=~-pNZfI`xf)znfazr0PuxW&vlghT;JQMiT{R-O{3cXuOI45?L&$#Oh9jmsS5Xw7
zm9cIdl8u2{F=_Rd8I@aP?$sIPrI4t^ae8n|gx?IKo;!omL&+gWaF)T{Y1`;4TL>Rb
znsg=R>dDhdT=CjCdPkiJ`lq<nV>(9?`m|X5Xa2)#t}owVkS%zHWb;AuGhB^WN0hNB
zteBr|QOl>@UjPI`5AuH_qw`N_S3d&fG$jr6ubKJvq4EqA`wb9080N(A%<dcMo0-|N
zxEpl7>n=Ri_<8PSMXAYCj2|w0q!X2lVE1?`=O116gK#2_#NWH=|5LOBBe5&o?@@^1
z(ptgQB+TQBRJxT9QPNEZJeNU`jJP10?IPL`ebU&6WNP=rMR7I;Qs!~^+CbEy50BC<
zFmf~sc0{beddb$m4P^XfmUMerdH8UbxF|2a$kjFm_sxn|`%FXhCrsMTSrDENI=%OY
z`AhSfCIUk>dj}dOBF)n#S}M@h!bv_+6D@HzbJHY}gn}qyn)NV(wZ#L8N}aHgDD)8C
z6(#cywn;UQ5&BJaFdx8A5_+&WtP6u8F;&N|{1`$MU0ulfV2zu7DXa4mBabW1|7+h6
zaQD2~p7W2y2r$92`Xf4pCiv0}6_%qksuv%dwCl~iIGdq9-THog(dT!Xi}i5#WYoe#
zoSRLSn&uDSS029q`ze8pSC;_5nYYc)#Hj=2Wrg$9sb9JkI-VdJmp@~}XnKpI$e>#L
zeOv#-gweJud|mAoz7GCB__`c_u6F54NvA{3MZ^c~ExDyEWd%yEj-v3A%}haD+x!A;
z$2i%y@RZPZCCaHJxym2ZMUi;v`lHJ^WwhjP2|NKUT&BSXxi3tRV-+R$jg>ylec*TE
zM2!up%XO=JKhZ%6CR7{CImKRmRL<yU{M6uuLK4DPH)%_pVmmOaD~xwDNHmLl1A5P|
z2ykElJYPSbT$xfo{_R=x*v-E-3rO{`{1yJ4SS|i1vEn?EOEVg&%rmPxFP<n=KTz9l
z1O2FqO>xcF*PVQeor#e^J=^?FIcICBp}zy`AHNBa#w$k!CohDQtP0S={*F!n_TzrY
z+0E$Rbg(vnmxQ*;>CLBkqj6tm4UsMt_T~PY7hXIfsjKv1C%AM27^<DRU$lDxm|Mxw
zOVDdKc>w070V9)aQP9M4sU&eiET0c6inVGst-qxYEt{YhF2bEFm+0t~%M^T~1d>S{
zX;r4UK))?y!_6QBH;Z-&#B0k{u*m=f!!&tlDT`!5yct;1R6><({UZ9oDS{fY)<IJr
zw2#`@ZrfPvDKl@VfGPO8L*$MEGGvxBPzW(r;YC2XYyyOwO(5jJ{%?jHJWu_AthyP!
z9f29F-NtvL*@$VwGETFJC)(e=Q<s|+1dW%3sS5IbJ-j+{IRCWO8+vo~G`6IfRkmM=
zv8cGA44i;CL%<#5D<FnKv%)M5Sf&#YazS&s-$oh^vMOi7;uhCLKg4#`P(bz+shtfG
zF}Y9szq_L#Do(a~l98b>DXEQ-%9*;EE>#-wS`d3<I4`SKMb_%%5rnVOt%I$k;$^0$
zt|GdpCb`K%Zwa64W5lHI5n4A>@>%>6Yhq`wCm8OT0S!@hwZ*pSM8A)IhZSvtmdc(B
zXtlw|+oeoeCGjyDUAUwmx=$#Pa}gt`cHJB!;S-j78l*tFnf7w!ce=R(daB+87{Rms
z7mhx0z@N79K;maWu!^c;E}h>FGn%Q|jTYjNW>?PMi#HH$isvj_Ka6_^JMk$hTSX0@
ztdOQox8vx{e2#+nAknKWXC;cZM*7sF7+ivt!xcJ1s|_yCLDyO7u3-6deRbFR2yRzQ
zaM<r5>|ZJ@JTD;QC&!%Z?AHKRSAo}@h#bZ%x;ii5R%|r#{(Rjbik{k|o3Vd^oMv-a
zcTJ9oztQnsAzvfHN4HQWjJ3)^IiFmYYzaV36)_L|2Q>v$Rgy0rBVa!T0lE<eczi6b
zO4<*6F=CXwzOVbWPN`>5c~-D%19NRd5DRN%$kzew$>ZiXX19eCxDz^YZ9AA?mV<gj
z5kY?ayde+>2HBz!`U|cYoue408WPW92kLBU{Mv3-J-nm)1x?3NuHL-c`FT}COr%Ff
z*h1`wM(3Q=0>}VA!_n_0mbV|N)c$TXA-xV)uD99$^1lbJ@=J+7{mxORbnfboYWKKZ
z2+zu_#kK|LTLdx(a7EzrYsm~C+79TVuH0>2vQduZqb+st%RXNAC+0W^eBBfu;`;Z$
zavqx#11SBCuvI8{41m%f0KjGpiKhX&{-!lj;SbbQt!kRS_jR9qMdSVh23iyp@7nX?
zNnE7%hl{v**KS5J6@h+vXa>eI=RS4AVCVdU1}aiym(4kkAkvx0;W9ecsF)zt{M60z
z)n{7+Y+uAk)dyvg+u^@E?)M1PNFolJ(x~(xmCpJ?OWkPgt=liEkm7>>Wl=OusuCX%
z`r|p7UIjmLIKA*EEDsj58HEI~oN+@fKy>VvOg`2nISN(LDVyP74^|bZijr~$L{e8u
z5r)6lQNyu6|CNJ=ip2b<DOE6mJN<F?+^Q*+8Zsn!yr&iKei|C|I;`qqki@87ExQk-
z|24^|r0Jy6_2fdpF=|kj#&URH%QR7rG~gqnvU_PtvwMMeY{B32eINn2a#%j{TM3KF
zyA~W!c6uMn8KQoGL($+?=<5rFUv9m*v*t;S-*PyV32N(Rl|By;n<t`ar)z)HMmnrk
zJ2dJqd*EaA@h|9=I<Lp8pP;}-Os+3r!xu}sLQjIg7dpDH3|4gQMC3@7NK$ngowF4s
zhUaSP?r+dlSz5~~gxD71nQ+Wdx&<r0oX({Vx9$cDn>tD7=f#bsV);?iV$N~ObB&}p
ztOkw(gkHvi)QRPmMX&ByjT1Il>SP(%bcXaOZj@lU2PjhubQUn4j5A*+qvPk4u1Eif
zG}qNB5m23?YygDZB!|gNWqg&EExXBdepQv>bg%x4RC53*&xOsSd2{8d0me+F_K<mf
zX+85jl)YJ^TERz`d$4d)?Nq(SQ1YU@U;p^qMZ9Q$IELw2r5FXYTC!nNXK_Qkp1!-M
z$k0@3!fEf_)8U>_JYg;ej1!vn^^nNJc1BY6lM#4qE~bsCzjaTx9dTjBxeAtW=h$Q%
zHQ}(&w+&S3f*8GU<3w)5Nk-qSa&u+&$z22}o&6)V;`y1|?Fo&uu~lP2r8d<>QZ}bY
zz09t+gSm@SLJXCW=z(Ia;Yx2-Rdi^S#NgJCz_pd!h=q(JZ{6dU_vzaS#gtvgv!%Kt
zptj>8tYBMdV|s8VMG=-Wlm}p^?D>g|4*Xxe`T#65e?9$NJhtvX>sV#=8ypt~iMz;R
z*9+p*-MYVK?^hQc&+qEV#Gmc#)db@G58b&8u|yK`xvkOp8<}$aR+#gEMOJAG-b;|;
zEeie>?6yZIG@CJAF3XavTI5WPHMolo7{3Zhpi_>@l7-i5f07K1z^p;pI(+ML)(Fq#
zg2sF(u*tb;0tkFH#B7UqK*V`wnwup)(pW8Gi4%*r2)J)NX~pI~i^CPM-n`+hTS?Rx
z<i!*k9meYtZ^GJ3p>p2=(Uy&>dyGS|KOjbg^wAs>E2VE&0t=6W+6tSfS={`MjlqXg
zhl|RtE<KZaALSfXzyhDDmi*ikhwAt}8U7NuMq;mg5&n5pi2qbWpZt!Av^E#15@uXu
zSq<g}&r;=*1V@|V_8QL*;(P4RwRC1cD3za(y7zM*|JG;`d3SZ#dL8gr3P1gp(eZQE
z+_EG(F$WWtc=F+Cqe1u3+i3~ji?a9cD+e!%^D5#WF1Ms=a-Lu>O1^x3q$*an$o8g-
z!I?~wmKB*b>BOc72TgU0DVUuBs%yB}Vzifb86t|xeNi3s$<+uiKkUD#uJzim8@azE
z0h5ceK!diKXqUHQpds|TI^)Xn0|6~z(1E<Ew6EJIkErWAPiT63+9qxet2J$^(|_V3
z)$ed_m9lqASloleZYPWuFu=7%3MGI}NLL7j-;b>4e;RuuSnGZN?6d=KS=#O2qac-q
zwu6WN3Q8i4bXOUr&w-T8tE&@s!#nI*A|?6UdC=OI_h;Bb4Ia_oK7h~-Jbek6LO(d2
z&68%9o%FF3d)oFAYJLI<RaZjch{b{#T{TXK@Tb2?Y!@?*O0lfdN8O@j`8Yu=>Ud|v
zCPE0n=k0}NkpzwjRUAgRm&Bvk&Jhz9f+1WCq+!&06|E=h8$c$@h1^`gP!5JLgp5)O
zk52}9kReB8_k3zTGy@Rad{&{RLMkDX5j7jVA*1AFRo9g}`A>M@;|i0Rg6)<*M3A9i
zOLRO-)&H6L9LD_X{|y9Vv)?sM{!S0l>E8)^)0Od(EB(v~@@g>}hZ8Q=2iVg$uGhQQ
zpmFZww6Y=Y%#(kimo{4@p7){#lh@j!Zo}n2(Ek5ads%F)!A)RjZjX*P^uN=v|JUE8
z!dYbgPr88vCp02EXl(X}5WJbc)sGEaE6SVG^Po4Rej-lR{y{WUHssukWTW)!^E}~F
zNAYk-3TjNz71>*iYNyNhX-1@psid;+1NMB1ZbkIDdV)vqV)4P`sLCe90xTbj(!iDJ
zBPNOez#_6<b|V-KyKP;zgcms<2EG8wkaesnbXk5U@mkp(iQ0hin<J=uISI=n36&()
zBmU#xU-$2ekpcR?Rcs+XmDm(VV_1?)DWio3IC*pHR$wtu%Q+pcTWy`}FSQM2Y5ciP
z6Oi?-|8|{FzbiSFxrzhge;q54^Q-HG>Nx1?zPe7(8(zcdt`y{yX#QZw;t$evd3azP
zmmA*HjZO$gSbkTioKV9DCHTe&YGjzcS2>Ot@g@pvx4{4Mrf%pLU1?VY+=_N@;6Qjt
zW<+rP|G7A;(Eo`ju>@CuBDGUTW-ZOoOFuI2l-k9ih*UC+L}{LU*|2V@+Bi79FSH1p
zZDp}Mk}_0<Q&d4@=<%uUdg`2Amzq)X<8P(>?>39^AMCxH!ocL;*>%^^%kwIFp^tYC
zr~?Uqu_ZatOy!XH)8P$vmpKrGX3$0?tQ!nYIJpz3iyw>T=DB^?8>Gd@*%?c=(#iw;
zQ`o760Sj2|8Uk$iIQ?HFJr$aZQn}ijB|>>_b;73YY4pC-!<5d+KJ6>F`6@+7I^uk8
zi!i?g^gPJSeUI=?ZzC5pk3j4PdeO4`?Yg8dHL&b467|#y&fXcMKp(A$N>R3w(E@xs
zdE+5O?ViF>KHz853;Bd^vq_&HhA&Y5(l8mAgnvd7%M*s>pa1rntO^?<{cqXm_@8K1
zC$)zAuhZk{blpq3vmO>+3`8PeY{<YVpXLb%UfRRGn*<IRduTM{G*e~6?J^>Msf#gU
zPNvv45%4{UFnmR*6@yADv}jPRx{|uqa7q;h3_i(B>AJ2y;ARj#p^7@SE#sxwR`$QO
z7JL6+YAx1r|2cuUib0!yF3tChgIoW;G(uN{Hz&+PDQVdCrTOqb%jWyI?WzRA9<`l&
z_m$lBN<}$!gSEbS0)1B}7sIKy!1627pLEjG37cDdOE(F{L~1{-is#akW0&5gAn3~$
zTpJzcV0w2v6B91ZKT+Uxy=}gdL#e^xW=C>EZ4E;vjXQUMLxHqtV5AS0_%OtFYWnSM
zPMd9l7Y?3`Qo!8Dlv7V!LzfEJx7|;FqW~}CkOGz#s0o}9bW=S8xT1V2QPcufCu})J
z-*w60(O?~Klm;tb*;YP-9wBlh=#4ikWXKlxn)rYtty1>8INTRI%=*_i0OHZRKk;b0
z1!?{7cm&)&bh$>cnRT42#S9!y0E7WFbBX6~uw_e)wGqcrlWU^w-xROG$tR1N5!aw4
zbpj5KYDjbrjj(kA^tkdOm<<vu&K0+W#Ut8)HO&cg8-M$iEZv~k!P+P+(BN{W1W03Q
zH~n<)P5T=ROl|B-bTvE~rt6i7)CnhfF8_a+d+Vqu+qQiaky5&wArw?%K<Q2qDUlR`
zA(RFsMHot2Vn9%o4y9pe>F&-U6zLwMk!IfmKF|C3==<$&ul3t|?LYAk*W%{7&g(pn
z^N1D>4Ky@@q^s9#Vi+KoGD3s(8t#^80WwrVeoLZOy}AsQ2Xa8%*bOpyc*(R3!&dqe
ziiN<vhB)CYn6ks#qs`Hb<gYt8!m|7b);A$5CTyVXp<3nlFTVGuw%W|L!U$!eE#)Lp
z1eY}3#3Xp-Lt>rh-k7<7MdI~nfs@MRz1`1{=;HE$f0dwp08RdpqLOqVBx*C6`b*b`
z8eqEcfrOVX{P>l8P1KYjX`^I5C`~Ga!GMO&c1h(D6eGyGt2xq(((5PTlLE|)HycN@
zR6v^kb?Hf@%JG$MAH@Z@&Wr{~0>$|xqI`uSNNWo%X+r))`gEm9`8p&m{~yk$PSStp
zd`fdI&(Y#fV+)yQ!7H-lRji40mNwsp85HaTw1T_UmcX_wK0tvh$o`|1@~VYcw~XZf
z%=(mtxXLka#4^;DRP#?}&y9;8AIperE|A_I*At@k-lL~1;w+sU(mI*HU%LN>K2(6%
zMB;eJz-?OjXjSYiKT~~wqW4!KQ4-D1#&LFVoA0*+E9rjz(#dYOwrid@j(+{g%NR6v
z*iEEf@?PAOU^Ff?*X1P(;B^jBYNUc}VgoO5wQ-cXD8?2Oz#8<Es9Dm~t~U0$iD>$r
zHVC5u#s($$A;9cN7utB5kcX?EE2n_!8O@nGnp3D06S+v9qlJgIgYk+A(5`El9NfGM
z#_C3CF{u>cE`Sj{<$*=~eq?L_V>|;6f6H87<K2J^f@(w(KpO<x6O<F}yq1@D=YALm
zxcyVD1puub3WfY9i}w6`=Ffva7uLk?ah8_?Y&2o6fqt#EQp>JgMC-FJWmgj4bsecc
zbpBrIi(j)NXf+<<`OdOKAd>Y<!4YBUGd9&s3&+r34y9!Cg~&T|Uw$6<H$t2@7V3Gx
z$9vB55iW_|9boi@$ozgh*JU;|Vm2>Qy=%`e4fO>jGK%P7{z?uUdJCa4-7UFm-Nd{y
zSYZ4#l>x#`4DRu<pK8?MZ=Okay5TQ$8pDn)mZC?KE$(2)F`$FcG*jyrX2bFRn;8fQ
za?oDkFx`}#OD1L)bC@CX!KY6NXoxFV$O?s7rYKsi&`NF1u+W-7+dt`8cg?3B?tk*7
z?Di3MYwPe1bIH}*)s%OD4jk@h*H|V446uKJk^|8YJDg&h354+H^*6cMn%nQ3oA%^e
zUfNvYjfHPoeOVkiY8n+e-KMNu{NPqtCDAN3@;%<BuenucFwM_%&$Q>bz9r;T@c446
zm(%0YBF=ot>BRbHwI7UI_h_xdE&fFY%)-4OggA)_0QqA7M$CpJ0K_cQQ;Al@1Vxs?
zKLXLxwC)4FE8r`n<|hOx=3We@+P{iZb%}<b#?_Ac2@A#PCD-|6O10~FMOLXBjs9)v
z-0XR(wiuw6rcB_2lIIUbY1b_2s7@<>8XLL$H&sv|<F*Qx$DB5>dSU4Z4&zBFF{gX$
zbRactFNdqaV53`Jl3CoU9m|@)gQkG3CQI8A#u?hmbq00sUM8NIQ|+az&x^|nPN>*t
z5`8k(cPBdjaPEQFv3XW@!Adrp_%o9*gExo~VNFU9l2iJzFwovpOL&M1N8kEcN$-P_
zz?GIn=3_aZ^xZf3?5cr%_L0fapl15$jG|MBN*uSty+qQrqxGw`Bb=WHZaV`eA*R=2
zq7&UGxab_4zmirA|59(0el9UIzNT(s{>~0#LA~O4v8pb00M+(fzzx7;&{l8?9`usS
zB^=>>tPR2^i`Qs2u>#d`8$)KjR9-e|C&|OwT6A1pH?Os+Dz2IUMi@24{KE5^?gxXU
zu#X-gnu$6qr0iUn{5ws9sq*r4%12ikX&joT;VuV8?20!G^2{%f=hQvPD_G%HJ)yQT
z$zj<w2_ur>vPn>$O+W5n8c^Vt3*RLoQ-w|SzK4cja+}CH9}eYFk{2wMKw6OAs~@NU
znjF9YNh<)XKwMpi855D?ghG9*SJNLKUu9#={F`X7#11C_*3)-9dou?~=#j@(Dxp19
z7J}1tMaWrL3pr+@<G1P3nXFE^iLQ0C@fr~Z3yI?gd+XQa_@F=In|_|dzWj@ao`%A#
z_xBd^Eez<HZ=F$hTloQti80QcAG3XFzAZ0Kpx)9ZN~CHS257Er93%(9G7&xVaM40y
zMFv3(-mM_F@Iy-u$b2dVvn@K1$s`_c1=_dd0BvQ8>*4t7$cmZR&ju3;MPbJu_k9w2
z-x&r&(D;8y8mzee)CT+74ecnI*miK-@s?2t{ewQ@#yET{6*#L%wWM)~J|oZFzQHY7
zQfpn(nk;ZULkL0&EMYu&?0Rphm|<*F?wnc}Nm?%eXcKNlyB?FJ8q|!MbILqUGwV%I
zNxCAbQFPJxY^&#dFZqn8P0ROe{3~Fzz&lB6F<2PdeQcDb9ACd1S`sZssXLVBF3_FU
zJ+)KOLJU0KpdM`euU`;H!!i9YfBG$vVcMg2XHqg~eyp6&szkzn9H-t}B>iR6=-qMm
zV*NYT+phgPaRDFojcFKW{qr7NEF$P>ASw01Q*IttplH4Z*T`LEo1%vTc%YYDmm9ff
z?}pQU_qrz-6AVlKy3aUg2)s)jopHLlVAqxzfqNj0M+BjoszCX3wH$~7paA<j2)(a?
zNgX>|YAV(iT*eym2b-CZ;bnULui8J!6>)dB_X<vMR5)&;V^>4RuDPl>PkX03ML}v}
zX5j(%d+Cfqz?|Tv4bn|4Nb?&?F)M|&EH=nO$Z&v>8s~up{<5$}uscbw_$T1{1G}|G
zqh#)dtLel5&o%!)lYP$4TkvpR5q-f4vE2vNs!ed^S@}_}eylUXE%I9_!rtofAS>oK
zP%Wbe|K=T$l?oO`+svbxPTfT1!_h58u4l>=bMUO^FHIQhL+I`daJjiG>9upyQ!E=+
z0tVEs&g<0e!nZaieoT(}K4Tk2nPy`*RNM`tzZuB~cxhwp3QxrFS(w7z_p>JUVYGK1
z3y-=CD_d7Ol?G?AGf>;EyB``@_Oz%ZwI{t_o$3(j`3$HxuJ<ARJD+`SWq=m8c`gMO
zSx3lGat+u?cW++q;TSmhmGzfY__H_JbJ!aGy%aT~N|AqSSu#7oEi?U*KM4#Jg!unp
zLqgkIoRpuV=l4I9SbimZI2imxQZyxQOOs$sL*8!ap?eLL(kdy7#3of+K$3jA!R~wj
z3_1u%r85PAYUG}9<{cVmId^~cmcVA<@ztUR)q@OGn{6>xJ9mXculCLcwCnDIOB;&U
zC+PNuAW0-F54JZu{1RM-lT_Qmb3pob^_>wKG(AGjP&4{vg9hH$_T2ShMCncfgeEJ6
z1Y%0oS(N_$(L$?r@|#)k-5X}pv9?8s`p>b6lBEA2+dr!BgxF=0uSMyrhO$EeZR%Pg
z{H!tT^n{1nj#Xc{_S#-`nQGSSSsTPRgzHI`HhwjkdYWp%psh$ZAjUBP{H`Ad*g69l
zQ`X*?1GTsBIPjOPj~yZRJkNw;9Xh%<E@?L%WZ0!5haRi8TQ3{Jy7%${4`OOf%83^d
z)!ZuwLvf*2iooi$O&a&}Eh0ITXdWf$TcA1Hb#ePK_@bza2D&xO<~&*S!&0~81fNhZ
zMl9Ri!_(LOjJ~#!?Bk2tcpJc{Ff`IaJ>5(;7LcEp`nb|dbasmTdY3IPl+5b*s+}Xh
zXM^Jhq3$xtL`%E=f0l~%<V27Bc;e7%T!<l(bXaAVW=QV^0QgVH@-*H@2jpmA`aC8i
zLX%1zvf1$gHX{5{8{c&}oej*l*%|COlmfgiGdckXX?Tp$`?br}&m&ZI6RQF($xVx9
z`QVy~s#@7ok9y_|r*(O&9j!HJaT~2U7>7Gm#diXceq+{Nj=I5;T7>tAl#1cJ;BvI6
zgfu6Mi9!J2NiNuv%~@e*bCP|NtZi#dtr!(s4(+>{0A+5=5U<vyx|{5g-B*Eu-faWe
zX|kK2m}xR6oyR!7g~4x8#FLndQ=NG*zH;ovAxV{>wOPlu@4%`Z?PGi&1!PS$DAsZw
zK6P_njC_I26i6(l=++%|UbTG|T&#QT?_uD!1U}cdA0&XsgD-om!Rgdfx0BQnWaWp+
z$E`__Ppes_4R6>-#d^P){1A75H6UAA`@RC+XJZN?V<`)_-}5@X0X*aD^h9q>4&}VO
zQ@QXRcUIcWLt=36+w_KYe=vBszo&9*Zh!nhe3VPj={^06h5CMicC5nC3)#|N(7xpE
z#n_gaV79>iJ9o!l3e^VtSecjtLXJ?NX`$fu!uG1WoWwJtBj!sYh>ps<Crp%l63HFH
z5A7QyC4`|tnv-Sb0t~7yTW2P&%(CbC94~9i(_tUK)h^$tj}~%TB5uwGo13kR#m~xw
zLRB1xQdNH_C_^ol1#>}4PNd$NpO>V`a81=oD6*<Th$&QLFm8#5%m?Fx7M5|P4l?Wi
z-{6H7G(;x=DfSw9D#GjE9g5f^d&Xw<W~1JUdZ#%BBb*N|H5Lw9e)l-!s@$%9aZ9<%
z)S~TFhGh(M-r{q-uA?8xh>X7H5`MoYN@?z8bh6+wJcvFG|9sY5`~B*&r!{)F+>wEG
zZp^l}jUk7>!HAak#Ll{q6I@3epREt?a68HP>D%rK&M(b9RQ7s$d~97iU*?yru^087
zUUL%P<HgYyIaz&nR${eYexv8*1hRZB*$yf3>vA;f@<pG$xvgsmjO*^LDy367R#vqi
zc1o1vw`>XqMNM;c!A}4+PoXotu<n=U@h>k1)4xQ8&u(ufj+)Ji7l0bvSpLlt0<1yl
zP((nX-jqD{Okfg{Hv-2=`fTV+r1x!zNtn2PqZ*KnlLCK7qr;97L=Pc8AcEZheFa2W
z{-l!{b3P0P(z=k6rDo;uFggpanbXATBDBw0_b2z|U);<7n3OUaMsM;0Ho+s@GP?cQ
zgv=31<=}{{qq$5kl8-A(tR$&hA%$Ugy84-L-88<j`uS{K_2b(0mk50;_i{vKZ)j1k
z+ejLP(6u>=5vkp$sjU$4n8-xd(}0yAk$OF8_ccu`H8E&L=1Ya$&6`CyZMU(brZSob
zf{*KUT&0VNkvtho{Z|<ITd3&y`D_GxZbDp-HEOv%xv#&18F+2IRiTtU$OQMh7Go`I
z3VeSHshJDL6gkMZf`uFRF>S$Ci|#Ho-FQZ&I}0pK(>^CkUZxu(pD1=jS9XRQIYC}B
zd1Enm%E9!^Q&~grM;p+1o5G(rba_9fi`w5@N-0_wVGr&1YGvD`x;{7Y{C0P(LAiR`
zoJOqY*=~ZP$4kcd&tEA&`j~YEpb36{Z0087d}Km8tPT>j=)KD}-{jFu10Q^;{{_0+
z-4SnlymJMsJu{`S9YH&KbCDs^S2h0jgo?yW%9E;9k1u0DkW*|(YZ#5$V!Vso?V9!S
z*c`33ukf5mLh)YHZjMQ8Tm_P|UpwOm*DpF_`;Cz-&CUwPRom<zI=w4H(?MT_4lE5l
zkG`*Z$8Z${K#S^GAL|CF=qv8Vidp$r?@h{`MLoz$rTz7E0zVm+qX<*;Si85`HM((l
z<$Xg52yGAz_`#w-`MUd;x~4H0N2#p{B+#}(F+@_RkaO~Fa&*GT0x)O+a2N%9Gtk@7
z1Q_WimI>C{&-Cgz8)s%8W|3-Sg7|5y8l$OV?2}nkgbxKh->0oIW2uF-WJOT1Z~3J(
z6U`e*jfe9qdvKD)@-s;;7sznc_uxv(lA=jPX@fDoKUwng7t2%iI1YL%jz^kbkXZv=
zvLJdywDk}af5@#glL5=9LCDh&GfPN=MI5#i(qjtmj7(5lW^h%@uBH*x6!&;bZUZZ2
z3^jhY^Ao@ihfrwg-%Bdro)<w@!zVi*A1=Ek7-hq&D!OAOdV9(icZQtT`bWlqSorr{
zpp~Pf>f2Lqhti}L`Qvj(jD)0{1x#uR3{-cbZZ4nMHEAM}H3Ob>fs&sQdWaA`NGMRa
z!#WRu;2ymgF%s)8RW?Nke!p-IC-KEaA?Jp*R`4*KH(-~4#mMaBq^U{NX^U6nq%AmC
zVX$SdH^B9}EOp-GWLPD~@gj1rY0;bV^n*A7vdq1=@L+qUcKUwyjO(w@{)!ihtOQ6t
zN>+yoY_t7ZJMs#`q*8816_S{*Aac!OpA{AH5wi7gC@gdzjTt}}`mdLlb^kK3qIMMT
z?f}F7qaH7?kNq%^opWz=Ca`b84~uYWvCpOP3ZZ1;L1TnuU;ostDOkKv9GdaGJXl<s
zidDQYS({<Fp0(%FOX^vfQ$esWG_yBAR2O($I;lWF+`Vo(az8`dc^EwBIh9^D7Y^SC
z6=JHdu+f(gEzB9^@udz5;<2;ddQh;W7bVF^8S2-6ZD6qIimb5rEqbwosLP)EI6E=8
zcw2_r3=4rPgQV$n1Fd$2{U#%Rc#Ti;N>bAE5Xp2|s&GA{_bplCr*q(t#;`t<#pX<>
zfyhXzT*Xy$GROD^mN<q@c|pu;w>c^-02vn^q{*^o=e^{1<30#EBM-i{{d!$&S3LW%
zNbCA6VH@PE6;=<;q(1E|yo>Ll^g%g}r@yCaF+?-~ztYy+4EfS@g=Kbhqb+qr+;jLF
z&Ca`%La)i7gDp4KKIG?RkJ7<Ik;Tgo{l}V?Njsg~an(yLMmq8;7YAg;+!p9WmZu!n
z255Uq^Ri#4ID#dP-if)s`0h)aCj>MbBmNQ%H@zEb|GC}|r$*`ReyAjk9o3~axV_zX
zRyn=S0T{V<+Po;xvavsdO8WDpdgK?R`hSUPCyi{C0-_bZ3ct_?hKpxL;2$qVYXf_K
zyYWf)Q9$|I21wM*aJb~1+t$eRz9y@r^m<g9d>{6+3(_-`JlmiO-TJjj$c?b$Ic9%>
z>Mm*%ZTy`jK%mS+(o3G5aM&SLW!e0)t)8wDk?mrQ<BH7jA-uY&Jt+KvG@Zgf6VQkP
zdGZN^T2t&nubDa;8^&&=DEG1Opbc2CjFwQ(qmi{S2U>5Xw=yh7fiTE|5Os8r1y}DI
zlnx*JqGOng41-TJAx|jmlNw=Ep~Kjk_q~Y4N6lk^H2Lx9z4J7AZE!KyG)9d4xwzk7
zPJk02=S~OfbVoCgnu{Oqu~hChKklfS3Fw7>?5Yub=Zt<eeTWW70YsGOk2G&dC6u3a
zALXnk)x0d7yyMME7&tF@^IG#)kM5IYmUvU8SPM@v<SR!$p0S0+@@?e4M11@>^jD<p
z!3i*A^-qd;n<n~%DS#<p+BC;<5~~PzPOJVx<rFZkcbZ+KX=$VF(WjSK?x&gf2hI<K
zVb~>>!=;$lk1tChXH9;&(kT6#P7T1jG(DK9=E%~#gWf%D1Y?nzC02P(FvYhaByo)x
z;H3v$mV`Zji0d4W;bRbo(=wOR?23Gj{n0F>>2$O$`|x#ov?o=X^RW)xzAnfxr@RGF
z{Lw9i3xfD_gAa2wkFW0FoW340SZ)uUqI9^cK^wbbmAlmT>1$Uf=O$JMeHFJCrUlaq
z4^P;()#XBq>HU1Q1NOUhOL89tFJ%Wm(D^HOeSrq|0btv*dY-!;EGsIE>|mQry1&!J
zvK4Hi0Rlurn()KIRw0)NUYuMLfOe;b%3Y5z>Q`1|>wB`#thN0~+h20KZkbRwH>Mn<
zC?Me(*&UPJ%J;5nIkPBztSnRFoi*P09_N=Yr?DWzRd}ICjCKzsY=0on<M5VpZVpBa
zsq<m5&4kJ7weKzB0=j3(Cj)ewNgnZsd+W8|A~WtP(HT*SR|Vpc9KGfJW*fh@Ho68=
zR#J7}c;bRFTy(vhTvMCngj{Q{_)O8W<gHj*-*(NS?R|z3-vj}CVf~E7vkCJ_Pd*N(
z$<?HI>ob-gN(+N&UHOAyW!xp_(bE69T1kf&u(eQ2nrf^u1RrTO>q!@>SC(5kIiXv6
z3pgN+%B6(_@s-yLOzxf<CVGs1KCDH9`u)}KU&uv(pTbc)1G3j$1X)0Qc^}$bAkf5M
zF?i85k%@1o|36@VU&X9O(6%b3CkR@;krC?81Y9`C<Do<Bgbu()XSM)IzA}qWem|c(
z<GD=6rNCoee%V>|QK~X-<=v2iD=zb74@QaS+q>-{^PNiP3~;V<26z>U0iO7ktvUG6
z1ymmw2#t~f%y-h*R9-f6e^9S|uD?GcPq*8g^e}W31UV4En+-!<&lA+v;GfsifEvAf
z&9l5QhTl2{xv>{*TSQoMkaVLmSG_W9E&qF1X>ZGsPU@j(1Q(mYPWd-++QUb;>lbU(
z*Hv3S5gSwh5y4y%b6Zh!1zkA2R?o~@zdzS2d}k)%(UNdDmHDJ9r-rV2g6C2qeMh`U
zi|QTZoQUeJ7kYHhI~%oiipcFp^bUK)Y0L$7s~!)2JUa^YEYj3kjS8uy6&-SocTi85
zVEgs2GE9r|H~>6l1;tDeUCS>ivqdptbW1G95{^J0T7I~%y^Q65`cU|X_ir$|h9{~?
zKY{lTXz2jG6?VYx7Kp|{+9&SS(j8AhjOXY8*KpxEW<JxVp7_oZ@jiNp#q+C6v(PzW
zPpOX|lI{!PjVf+wb2eLv4K->BvI%^}%4NqR{hN$Vc9@##?^_P-Z@)`tk}L0_OIh=l
zf;`)rdEO|Z;@)@^1;dvO$_Gmyig=c(ty1UIt*KZK1qFoXnX%qnow^xT5O}A;Yp7oi
zw~_2g%Tjx))6LI;oJ#;-i_}O(#X9SH{E7(+c6mXjxE=35lEocy01gx?S$quXGzjQ)
z#;QCX={P;b;oV&3ItkeOOndb8!4T(2j@lrXL~;6NAluw7*gzF1@k$MpV$zlkJ6D_$
z)n@nOcj6v63?@F~S9uq~!H(YTuYYD}_Wf=486x`Gm>B%*@%3nhvqgH?XB)(L+ShVL
zH(#xnBqz%o6E^EI&I&RMy|K^O7##EZ6)i_HZC;2u#SiDjACWtCH-8>W>aB?wLm3AD
zM_1vEhZ?7A0XdOEbQbW}Yf%QUFM*Sr3fqHmV|yLjsvmF0^4gfzo*rg*C-B`F{@3kB
z{a0Ba)Jnu-<YN_kmMKO|W5c1mUyBL-eiz95jwC6zWZWfd=JGZbLJ%6H1z2W)Pzl}E
zfn^v=#D`@GTak|o_LJGPP78@UsS?on$O0sU4_$0ViH&EVuVr9{EHA#xq{T_NI$Hi6
z(wz(8>0$>$x^wbxcuR7HmOP&yi4zAxx*IFd=H9BDc5Oy0o6Oh23hLBAt(oG&zCtG6
zc$fSs(N@C=Drta_?gC(K$9ECZ9g`Z#O%dkQD43^8=Nz-=1D5A6{sHtvWVgA_d!i6X
zn`PB;2a&m!WkI37!w}!{_wSzDHNJCiZly?7F05I9qF^60YqV!;GaS!f^>4qz|BTS}
zi^mB_f`Kx+XixYH|5AmN@sdae{N$)ajxBzP_#v)`z9HMh%M>XhDaLNes$>I>&exKS
z3Ibv>Voz3bGM&dK()~dz$1Vo^1DQ^2tHt3O8VOg68d*vig*ET44CR`fc^n?@Xe0_i
zpuOK~*EXx7fR<VI@v~HhAln{fBC_dnZ+p?vzYx!#HTco7?puI!TCw)Jw3%NB{cs;7
z|Aq2IqL^Xzw0=BqRGg@%yqU}v$M1)9k!e@{rNo+N>a40dRZ0I+-vY(`pPkY<r65%r
zc!`1N=a9s+hX}KirTuUPQM3s4_p6P(yG!|v<dX6*ZbY2HMewsp8D<u-zd{_K1TAS6
z+LUYa2JrwiiREvBL2{=Z`fX=@hk^C-e0-*y1JgdWj=F5F(Dy}oI#(A9Y%*tEd=#Cy
znx%w_nYhq`Z|UXp`+<~cLuWB7W&qlKI?qyDvz#eiYh+|4Y11Iad&^t3D*#Qu`?CI~
zo{~PDeFUV5X625S`x^Mgi))kt`g|o6uJbA4_{r%M?s^hottvp=&$uJ1FaGTej3R<W
z63rjym1|ZsYGYNs5$&)TElWnEy`*J1<Dx}chKtvLj^g$A_Mc|LRUUpg>x8;(cM9Z8
zhI{Z^4%~<(4`dKcS%!(7nv3S$vtPS2GD_K4+c#t@`kmEkqS)y+NaSgb*y*=fZ{DJm
zwaR4=t<U@tSoglw<o@La(CuQaHi%U&Pw@7w<mD?15}*YM|K-Sa356@d3Nx?3qAIWX
zSSG^`5Z^s^xo8D!^a^Wo(Zvm)?+$8z>9do$%3LHMmOgT`6md8a>9^f7eW<uz?0r&O
zc!<9;_T6)wY27!5G6CA1P*$tD$G^<}PXf}^i?DOGrs*c7^h+Gx_=+4$z%}XVn<vPO
zX+(-<?8b{bsuNs~YN&bZYN(H2w%;B3U6}Y5MWnsuU7aD9O?~d$;LnPVpZH^2sCJ)t
zXNjm~1K)gGn$(ODkCd2AdUO9K$YkbvxMPzTQLCi$LdZ0AIC~x}>|AK@hDwmIsSmJm
zB$WzC8qp<fmX6<(SAjlCM10(}>ehukSCDnK<9af3m~W-Lsk7&2(J>N{L6su9qf1l`
z@Cq)0!|$VG1LQE^1k#Dcg1av1=m9Cji2C<H3Nf5710^*WY#zpxq!d^S(0x4{sWGg5
znsk4rEr1@fLMjia4M6li<pw|wF-mTr2`J^6Jbl$AnHy7L0hxUJhFs2qBAmlJ%{h0x
z-2zPi*hz0iMUkbC3~Q^^Je216{o`vatYP`V3^m-xXhSfWA}3+E)glrYxmlkPKpQ;{
zb}7GDRL4ikKE2+@kyT;tXtAz{e(-dOS+3qZn6I=r(pe}|GFD=3S621NRA~?NRD)eK
zW*PJuVz{Pd8)+ME<=-@7V^|AERw%CDbN^Ro9!`JZY#0nF+9EQqfS1%btrvVVR%i)1
ztT$CjG<8}X2E4o~KBf0hjs7TEv>A_52yQNG_{W=Z;YE>!1rTI3@V__b=mIZzfPZ!9
z)zHO*6$eiMRoi8v5)(x)LkpRv>uY4%7vcq0VbhdNr%g<UNufHvRMz{<H0+eFF#zKT
z2Dg6zVul&dDaN@F*wHx~FE?Lg%H};d&y?L6GE;KLh&P^Bg&Bkjhs+)t>GCkCJ@spD
z4IXM}=M>|cDd5lPHh0+P36px7S|QoN<awOG%$%8KZ0yNJ=juxZofRAobes9WuHrCE
zm{#ne7l|88$PBTZZq##6{x$iP{+L*$?v2ew8-66LwK&`g1061Xfgur;&u;#(xAFz)
z%iFqrwE5h#r<I-NagFnSW9r*)XNO{L1!uwhb}daqeL|xC=J3!T1hVg)-n4Rk@;9%W
z{qT+YI81Elhg9qj+}g<Prz7|;ihMqc3c|TO2!FWWh_l=_E#x9Lzp9!abjQxPyE^?F
zW%gDJrKn>~XdLftRc%4<K8KbBvybVMO?y0%t-hq@d4)HN(wbJf!mJQE7GlqYMuZ&=
zGMmJ9&YmYtd|mxcVw3kPK_uh!0vw~L)tYDO%PFc)EA7a#`l#TdH|X}P#yXNMRMjcm
zymZWt&}uY8AthGasjzxCD{<yM`rPGT+6AEUzedf-<Pb0BC^+!i%QU>Z>(_FH7y7N&
z?06@zjTQP0rs51JcIsK`u$i!W(3t_zvm9Q{&gk{5dOF$^y+R9pp;cyy;d&U)E?=6w
ztRvHlDsk)mzBW}j-!1Uz($%#Bh@Lt~r9;A(wO}W3h4MR_8E31uO&%RnaJl6+`dv!7
z@=q}D{+8h)Vw<ddubDpFk5ummR&Zr4&6Nn|S)vhl#i7Ls+)ChECcc8k06}Ve2S;co
zIFDv&-iIK_X3obkb=MHw)g33wd(715xbthNpKRZH>8l1Kuo~;&)Gm#g#-L`%=5^gN
zr<Jhj@17n=GXGMQTVnOCXPno{By>ff5Wuy@_LoVmn|!CQlbFqj@>vZPGtj%~M3?GY
z&oxEt%?E<<XOuM+I$1ew)>MGiseiY%{V4Y8wA#a3Ig2q@%rDll5~Jg^c}9Q;^YbMT
zVIgetpv$y7?&R^;y!C(jg?dloUl=|#gI@wgI8MCrChr@jNrFPDM9(aX@z44N)i$NO
zVavyUXQI}%zVZ{l2|~gohf%xDtXizM`cix>eRLFicSMWJlQ+x7@fjq)ek3*XBCO75
z8J)t)SnOuu2Uz<4K~p{%+5}q>wAGGwIXg34O`wYD^utl6iKIbY+}8T2&*BRJMP;F&
zX|*mum-%wLs+*=eP|b%&0R2IMwvNWfhAYxZ6e)B>qQw@R253g4b<814Gu&n&;p&uL
zLN5d7o9kJ^q6o!mx=JYL@w3Q84~*2^#ZZC}Cz)A4DF&KdA|y`15a9UG$ejxh4hE&f
zjAM)OSMO_pAX~$a&D9x(Tf@xP_NONWiwOX}z>Oz*j;P+7x;DH9`md9V>Q!%F*bbO(
z25T{(D=%tl747nmGb?Yc;Hr+rjl~UAmm%G!)TSdczV2Ot*<D3Bi?H>V{S$dQhdKWB
zA)ljnZ}d%3)G1FDxyx2zE%1J0oVg5^q(ir(Bx|ITXuhH1OFA#9%5dB+=L2K3^%W%A
z&-6ELHmbvpMH&I7X}GiRSw=@M^UNiu^E(6DmnZ8(>b9=U9jea9L1tosd!P8G*hKjE
z0-HR9lrd|v%rT`0iRJ?Zn*51fZM=j4R*ErsFKfxogEoJLB+P3yS?O^xq&^_`IQ0Ul
z`0-6OQRmQ2si??cWiBZ1JGd(%(7e0+(P_d)M0`S;;Bou)%7vY*_V<)TR>a<EKg#T8
z`zoK1T{=gF?}C%SUw*6nbl!N(=hVqZ$1|vY()JTU>9Ur{M}-8p8IRcoH-*AZ1MEj9
zSooyswS-6GOA!)La|^=1{WWxYP;idW<r=&W6r5vZ6wD$MOHPxBl8>03%SYY=@{tDG
z`_-A(wCFH*_v^udCY#avdY6}@Sjw)6?~KkIhO`I6Edz|c#T^H(+({=^44Ki8$RKfx
zBL(FKh=t?E@1^Q>B)@#FU^N9ZSdDn52QiuH;JmN%z3zq=9fx!|?>hqrZ!ZJJHu+dt
zI6lCxPZe#-XxFA>d;iA8ct)^$Q{BB9ColqBjR6T{-&drXO+6f#?HunfVh!bE12Rw*
z42HFPlI}a)(K>Na?vTNQp(5Jk0UtrVzZWaX@Za4<=)7|-sNRo`JAe;ZBpr;{Yo-5j
zFH-bq-+j64dH!|+wyux=*B+kLv&~9=%l&X^%EaGUl;r0v6<T%M^`9+OfY!Yv@12z1
z(`od+CN`W#Uo*53GzKYaQaBoD2Pm;}D!x2y_2H@ocA%l4zazd({!7GH^UI5fug!~y
zFFSYWEskqk{!PI!O4?Ows{(7uST%QkuDUj<Mu=<69Fb&GaA1m{cUU4|Q1Ysmsy<OP
z%SYwl)8M!!HfT-iKnVY?Q{bR(doM1qDft?*X)=)o7apY=+=yV1I3`{rrG?q+UMDhV
zK*>y=XrJHWv1<Ps``^yZbBsd&pY15SHgJn4Tz73gyO#;XL`ENp+B$f~q*@q&BpR0E
zM<Dq(BiR<l+<)B^p>D7yN=}@G^}|>z7xjoWe!F|ojw(_G-)nr|dm?w917&DnzJWeP
z)rRGXRw;-v5GkXFvHX58pSblsgA{hHw`f)`oyg|jK~NNGoX0>)Q_5jTP`XiS*p?N&
z)Z3<?K~OMM5Y+Wyd5A5O#Zy1ImLyyvS8FsDv<B|YPk^9gqm|IJ+23g9Zz;-nusvxu
z?#ErgnzH>9p9SaKN8)U=XFRQhMmeHO?^$-{9_YI+6zJP+)u&b2xjSaTDQ^h(9Iusq
zuUOq<knn&~RXicTxWZcSB>11MF!HYxI1ZdZnH_`((w0-VRDr}@Xo{5faAK<Y4L>CL
zlTFc}LFQ?*8AHj-Z4Z@2@&5n^KMl#evrr0G%>eg;TiWNCETmB;mi}C{;-`lhA!|%4
zPO^`VBj&n0r=F(+qmOlU!Mq;&DR@;;9(F_3_FbosGLX2pDv9qagdV`Ocats{*ZUdM
z+2#0aJw9QgI8K0_#Y#t>V0X~X&-n)(uPGE5T5y6>+F8TpXd+at&2p&pxsQqXL8Quk
z(}ekRO)4BF!vk@8ugN2>XOO5V8%G5g@Yr$T@D4Fy0d#di$2m5vde(u^l>wG5HLiux
zz+~8JfJLEzy@l!$O+aZOpju)6a?!)@IqeMpcI5yR^7>!hbIBHv1ouL(`80nF6e(MK
ze0$Qsb91>$-Q030!{V`RWpIi01pnrNMUS=H`~e#U7)kTnNzNcap<vhZDND?O=lHqO
zaz@{RKkJ(y71;E)hu|eYQS6<-<IUut-XyAp=U1kH;2L<<=v$eK+!~-0j%k_=Feq=w
zr9?N%ah;QfJPa#KYuR~yrI9y!pE!&1$VxQ}w2sQG2~0+1))+C_J@up0Wehg%lEzG;
zxZ#fX5jx`{8EDeLTmIifE)0;rL@sIE32{S7<O>ZFi1XB%#6aP$Ojw4vFcOEb+dxRP
zVVMV*fnLd*)-I)eA~nD5cEx@ASGe%}rzhvWr~qbozK63euO7~5=Zub9;-xkf4YTN>
zQ<>;#@1h>GsbgEWh}q$@LKIK@5cz63JX5M#=3Q*)?Y{fe6V;#cnU9WYt<}8avniD}
zp6$ebR%=Px5~}dWBZ~5+)C~6I$pVfjKI}OSISpH19nTa#o7gcL2E!#$w0nFw&eptk
zBz@EAF7x3Gv<-T5RM7rZnJCq`E5qrq)jRJv^W?c`0EGxhLjjLJ!URYYQr>Njz{%nz
ztayKx?4%Ca4b*mbEKH8P2d5biK+b=E<ig`C%ApcV#Ml8-j%^9>ClJA>#kUXSXYdbd
zDlax##=zJvt7+N6@#x4tTq2XV*&))TYPG^0l>IL$B~1MLjIW2sX)yGoC6iu`Ug5nW
z{^8`}tRk=P`2Fdu3jJJ?S&U{BtQEIjdRpV*VA@-(4RNvL@x-|;+L7sOb22$-%oSuj
z6qt-SOMP*lzXI{wd!L1Yn)I$9<v7nLy?}0f!qBr8FE%u_yT|Bu<V}Qzc|1`6(L^e1
zB4sKQ$`_1ykwRkPyw-)Y!C}y*MxF<2+)CCve!>SA8k?E6zce;VBeLQ%1qvFGG$F*x
zb1-EqEvS)uuA)$}=`rxs+<a&5GcLOA(m&2o4J~<uV5E2Er@q`xCxSt0V2J8Xfh9T*
zb8bEX&@o{cBJDVw7f{<9OlegE43YAeOg9272Ta%qKnuLNB6yMiNp$#X5wzMFZGgc7
z1sp_782dzlH~Swb{F34yL`aM#zyP(hjxy;tV)^p2_i`t67uef`MkjFfC+fxe4_e3M
z_Sy4@qz~Zj?}*Wg8IKzCYyY8bMC~xLnro?s{{lhw)Xitu>n<BA!7>8aHyq5LUz);U
zWd6#{%32y!%v{~Zv!8XhOFbM^t_gC}^90QHWm0ImHhp3lzxi0lu-5`UGa*1&n`n8l
zd~=cKnzmg>y&lji6aPnt^DsqDgH>VrRq+G++os9rXwT6!87&5P_OD9X=*^MvXO&oT
zctZ2ld;}NJ?>fD@MQoh~leXK@r4^rJ@>DW43Nr_HWu_jE@{S~s!$N5~lwnCSPfTo>
z+kyS{77y%%HsGLRHLvo!$%POnhdJpS0a+p6ar)DV6#IE1A5mzB$qVz@d~2ZLvo)^8
zn(5lHP!El?`WF`;g*3XNzS+u4{f?q6%ezTJ*K*u7%pGL@Z<IKZLVSHDY%1mb?X2<p
zq%qjhLy!W1Ztv057hJ^1g(1doc>`|W2RKXj*GC%0>snE<n#^m7LHjMjY5^iepuIXS
zz1_Z+&b<eZ`e)i3(Muk=QoJ_56EfS)6K1kOYQBH#GvcP_B}SqpVtWO)$tI$jaG$ip
ztZdi0CM<712FUUfd2lVDhf2{_whk$9;Ay|E2FM)j#B$SZaRPx>V3#^FQctuYQ|9;G
z-{HV-`mH+RHmZx!FGODulK0y(g^V7!pJem{^9lJe#O4IJXn2Hx4R@djV*;8ey6?FM
zcb}rXDzN29dqUH)|5~C>=j9T4YTOJ{IAJ|D(6wyJ!7EnCX_Wv{8}Ynxj+2SHyX!C|
z=rZto5FclFX8?~%7QC#?BknR}VY8y|?c=E8uik226e)TfuM2iInJ;0ZjlDl^;HJlY
zV9fU}(RjzWPp^6<5>~MDIvu9^A|eqHBvD?DGqC*1&_szq-x@Ft=(ED7VTbK7FNWhv
z!Cf4OuvwZjS;b%XdjLy4bmeC#tFFI!39^ui-+pI7bm|M2uE+x1#m?V(fGMa+H$|r6
zE2)+YdUqfiI!4SJm1u8LuRzkeD_kUHXqHP&s^Ro`uN-%t<77y2ZBjjvXQMUQ<k*nW
zfLIOD(F$FoG0}Fn4w>dBLdju>ewV{oiFGv>1e%mcAm=H_IbRr()fpF0yO`*fQ7Gb=
zU=YdT1YKe#4Aof%cK%e>OGNr)3n6^kHc$;xeZ9!wpm0ji7}cyISF^DR7ihQ@%vxp6
zFtjYEMc~+El?!ynF|(v2t&m+!T>Zwr@gtTv^sCgA=QCLg_#=4ow;B!1WETxxd!O9)
zr{Cb$y4xsTWG4n<KBgFTMy|$!=xKmy#lh2k51ew<L44hefDo+)9sQY?f2hBK9@bc(
zpQo3>%OyFDD~&q^YBOR<k%T7K{-9eS^ay02y)I$jE^lJ}TFt;%lTe~1X10+^&A>5)
z*m_e?O=}L<kDXmNY0G@DBUiautY(~dGvv}e;XvkbZf9#dQRf3n#7G<ilgjB%7csIU
zvtlVT7}Oy54qwb+EK`*LFw9P?pvkUrox!!~>Q|gO)J<QR@u@a2j|WiK{>|CeREPE(
z=oi}>14&5fpH7PGIiWwYK6p{_pRo4%9?UxhRbNsX9$jw-7R&zff}vM!OwFM!ze-O6
zT$sP1p&aY%>H2QM_OAjv$uTNLRd52vfrXtrQa1oWIdV6>DhurCIukJpD_9{_bT@AP
zROg@I0?5MyQ!ONn%0~6<0H-#O(OcKa;5q-1p1>}oCwu>vo?K^KWxkQXP@l+iK!t1A
z5{TR>&<Ec#?zgH98M_QKGY)AwpzfrMZOH;dmX|espoLZvbCPg*L9rFsbM!aJ*S3Hl
zlC^TAPy2VBNSGwzLR0%>t>aJr1*0f`tI<F}O@yL7J(o08GIx-kE%MmM;cek1mYc#%
z3g}{vs1+pJ)I059p)Hv6mXTyhEW<8<1Iz|-OQsptsr198+44Pa;2)S0R~r2!1LcP9
zunN@hoLv?XK8O(Msn>Ic79+ToFyj$#Z*__=N<EbmD*n*m7YPv$DY$%9_)@(;U%GZ=
zya6Lm3s*Q5%aVoP-zpL=ylp`?hig6X;E#u0PgUa@DOXr><WBAK$`TOS)tMv0?WsBk
z#;L=Ex!bO*xr-==>US3unEC7_t3SQ&k1ycSLZ=C6$!|UD11PFXhbl$CXaOD|U~NC0
zd_qQ3g(JQgU572XB=;~`UKSeUP8HnBC{{D4;UmIGLLj^_V{$`4*q1J3SrhW|gSz6X
zT(8bC*0Rs7?&N|MV5YgJsvKkAH!EY?5)({ItdVIT-VwyvBNwuALqeRKghX4b*wEgh
zpF_!E>rR+JFqc79ofH_tACwTOUp8e&KOYTW*k4T<xFZjg5Oy-h{U8?~xN&=MLP?YJ
za3t8FWu=iD{_eQg--SM_Sv<%raezl2erWW5rte0`OW#43QyezK_Pp~xz0eKceg3!K
zc?##rPX|;)UL1Vnw{r736qui1hc0dPYNNn^EDZmEJqPm*B4Fw3ffk1?{v&{}ZZ@U~
zT*q+4lg_lyG9zZ52ROGz6PgT1gz07xt~2F<(kyaZ@a7}~|I$dz{;iP!=094Y;WqLS
zP1_mbgL%_(_?8;zONS_{UBR`Qy1A!DUL+7Y$NgrKX9R9Y@Pi=Y48NZm3D`e15|o@_
zX7rnYMuK?W@D)9i%4!~M%g|H$iua=$U_4UDLe6DSb9N0PHEvFt#vjqInxv55wsTQr
z)4-|VKY11?{VcLWPq&-@P-G=hMHX#nKCTcbvH-ZHVJ_NmCC>|Rc%H_PWcXcA&~&aR
zAir9}oXwq9GxN5H+h8@^fNx1NF`;Ux3I5<?Dh0l0Gij?Z;IKE0*XRgZ@(YH}9L~vU
zunRmUgTE|4Y<Mz%)nG}9fFe(BP_gxHNddP~EBeFg<2e~vTS6=+`n~iLoF}_p81=G8
zueaaeN-kUIMk6?B#Zu^KtW=w_YV04%U!!)NXMI;7TmGRFJAS5}?N!LiV78wT^UyOQ
z-6LSsu>M+RCzR{ieeI&kDjA4FivLh$Km4q+?VXFkzgJmiRF!3Z^++xgsIq`#a~J;e
zy8q(X%=$Yozu>>}@_Q{P*TfP4UVcN|QieQeJu+s0V8%{Q;{7dF^lan)@*5OgY1cS3
zUQ)Ag#QEcB06vzKwBlW}l&`kbeQ?v`c6fA*Vy+6AH}!P_MO$5QmRspKKQ+C^_`P~C
ztGyys+IAU%i7(X^Vf*Ar_`;A`NL!C+Ua#IRQ%?pKMhw?9u?ivTZ6=|nIDeb{!71R0
z-!2QCcgW9UGwQcJ|J$*V>_=@>`kC~<U&&_jHhqHTO=?agzPd<8Bfe;+M5yY;I)mqG
zhdm|O*F<a>(k2hRH1G46kzR-hY7#-{>T<=-eK>_uKuXxYsXQ?C|Bu}A{}BRhAAh^O
z`};R3Kz)+}V*S$t6x}P(t8Syulv7rEj#%&#?<O^OBcUMy8(KY(J+gl({Moe`@*7wp
z2gG*teMT_%Y1*IE>s?-g#$s_hJCFIM!~$f~LrSsg6<?#3bE{+~z$*DQ;FR10N?ZQv
zlzgq_H8k0r=YMuePPXk^{s;9s64yt0cV0%-r|ngE^x@J_fDwtjyX<a>_(}%F0kP}Y
zhe%c7;iz~aA>#judYuR#|HGl-xB%JPXNLbIq31z;AfEo57`=09vZRqLus`>~21ZDf
z-Eq+W!tNNfcy4zb_y@aVzhlDVsA#$^@-vxI!}}*d8ES|V?J6;K1jy{ovY3FLD3}Ci
zE~S7wM%LAp-}@fhgHNe!iS|=)TyFqSmtsJ6G}5WLyUs+(915!NyCN&Xjox>aXzdIQ
z3rET9F27~R8`K$YO+5Cq8zpu{Rtc7<xZ0DXI$6+%zRFW~Ie(zi|5qb9Kw-b_JPj@X
z?E!n>h<dag??t;cSl&ybrk32l`QPk|>HcV69Q3z+@yvg<FFw4mFBY=Fta*cC7;wae
zGLK`3kz8hs=YORit41K{&GD)Y<z8&3a<F`jdO`QH3G~A%W{O@{gfztRb=ND-J#F`h
zM{V8&62`n*AdQ=s?(Q3=T7<z?%pv2!J45w%^{V?SfQ2u^CaL-VxhezPs13Bbwmui%
z3oqiHU%ZI*+3J%h*<SIKn-?&Me7&?x%~Bx24+OHzEYE}dGwA{Nk4|_=#=(_Fe2x_x
zXu|@OS5w5T$ZOHXUe(T&XuI08>^H{|54WTp>aq*l-xS!kSFG~3#|GmWO6dl=Duv7*
zk9%3`3W{56oW3UJb|t>4Qn$iPhM+WWx*2WH?qA-59>t*6d)UTQX}`ow3?r!AY8S{L
z5d+kcGgQN^0R_<E>>8vUfx$I==~XI~q?bdgNmuSY(Ye_B)dbc!{)V#xd%uf~0jmjR
zPxR_<MsafTz`HFM;8@O<gW3jMMq@!G8M~la5~=TZC1NdJ5;Gw->th<QQXzbS1_@f|
z@G&?Ic?wvf>`wFE2#aGz5V7>rEcXuvvXkxTKBEK2yUt|XwDp5(vMoGHQrF+@Xw+k-
z-FD${^E)h)^dsTGyY-dG5)gOL(56UX5mOnx58P482e_q842no_HS_~JgT%0WFXpr?
z<uL3=PsRIL7WALAQtOW77yun5O|Jqjlv4`6bMwILPhP1D=yfB4yZ8^h(*KTV{jaVM
z0J;dGakCZ+B}=JD?R=1+Ll?Z%Wr5>RhCaj7<Zu84_)Ot(r`6LY!L}phk1-=mU)`GL
zHUzi-<rO>~_w_k#aUQo4hJ+pcM!)V-yZRC=jD}=kOOB1!E~JFvz(RjtOIIVOFeoZO
zZiU%GUDw6B5h9sv*3<IQEGP7hDi9vFH2ZZaDwyitEw6OJ7UrHM^oR14oOtTt)TwMR
z#HX*!rAgW++)BPeLIs)c9J64dbaTM@YwEuSq);6sM}s!wBnO#@%w0vi@Af?{8ad}#
zZyz3H{RwMfzQ9^I48$Nszx6p}q$sR~v^%R^6V>=)k^$@ombgE19XBxNWa5w=-uT4P
zXkyZ!5p@rHb_$7|rVa?dF?FLl5i#Jbv9vT>5<ywd)|2kKL~K1rK`8zI5!3Nm>i>CE
z;jf1GHg@9X_Ip-$1dM>d%LR=&ojbtFD?>hjJ`YXU)wf__rz1cag9I=~=6nzeXqC=A
zd3*l$<W>4{SBV4g<dr@RK*-~XXJztOMG%0FKMe}^&o;(6H9r)D*(lPt(&}Y7+$88o
z#<#nqqcQ!e<mR#YxH%oTj~uR7FBU2#&&73nRxlxs$S^Sd3iq&v$FO$ZR19{-O!@l?
z!8zvIK)xZfoWpU@F7Zr$b{Zhp)f-A@+G*-*-JNwo&apA1C{v;sT}-nV2>ZU@A5D3~
znJ%_L)D`A68bH~(LGCSb`mJ8eTE8`|N5`-=P9jA^f*OI;n#>b*`1IruxjKkEHA`#L
zn0L>8zWAdTySPwd?d#VVfy}4)v_r+?C007?lxgEnKcDK%uWn=h4ItOb0k2u)Nj3_9
z2W%v6^AelyS0(lyG{chQ1Gp)ojSQ%Q?|9*#Gv{voPs}+x)<AW4YaC&WYDCNpij`ss
z`<c$eV%A)U72@!fB6DG|15+g;K;P<hYav)ky(UZ+P@*M|yjpU))Q9WhtwAEVpR64Q
z{LPlephkd4uk_y@y}44mLBmwBTzOn*6=fC@Ll5IrvimD$oWco3YJT<T{qLl?b8D-*
zXEff2PAgw}XpzRXrxkSu)Elq6kNj$l^xt8lhq-U6+3M96@MZNTMwmvLsgwfIxap2#
zpX6pKw9=jFk9Iw4$59F+hNXmyZ4;Wea_P4^t;a>#C6<E?3*E71q^+u@S0q#l!Qv+!
zX(|ctyy>H!1-XI&1D~zwg>RjN-W8!hPRPb*?AvY~RFApO7U#9UvU8Gq^V?`~)7-Wm
z)rq}h_WhyBySI%@m2ku8_)_CHr$zM+$f+WKaEV5B1)p`@oe5Ioj#rpxMWKZI2-6<K
z+Vm%XuzJ7`H@Io;<L9e_kOX&=ig<_a_s~!O5FsVqK*26IxV0?apnz^(FP>Y)ACi|H
zJWofTaHybN5bDa+6njChstzIm*`J3LQk4!p5v^a@z&L+8*gQvX$)c(Gpn<nr%3T~o
zEd;tr*(<e5kadl6casz*#is!htB_U`H)dMzQ%~dp5<Sc~JY9Dzhm7b(2nSzZ3A)A3
zA#U)ILdAADlNeTFWOFyR5oU*Xi*^|EA;H0HfHI{5A35of!*m}@(-duM=H0KPy?S;r
z*&f`Vc4uSsbf82B4{<>YE4WTb?ASdyu?1dPXZwwId2CN#()=fnFE;u$VKpeN-u@c%
z^Y0#uW8?7%75@E&IsxQ2f$t}4wxuy^T?OD%Od%WCpv|nNfC>o&;M4A%jIMWR&h3$|
z=cnPv&<K!+l`U`BH=iQn1->j0mM$Y`$8Tl7D7+g=JHC^72YCb!82bW;n)^tcIe)ON
zb&ek|?G*nqUO~H?AEvKWsyjXTjx8+F7~;MeU%hwyh2OTz_c{V@4edm*kE*!Yn1~@D
z6964r_@#RY!)B`c*cm&JB={Mb{t23d$aSOq;yl7>y<1YD8~m<&Ej@8ze1j=9XYvR#
z`WRdbSsDK!XegikDDtzFVl?;=zm!-;K0PewW$>E)ByMKmQ2iM)^sL6L$gEpIOl!TL
zFX!`PVaR}??Pzbd`?ns4OzY!?FSb$PqFZm?-KM^?CBJ$$T_<YwbT*shsKcOAl#xP3
z>rrlG{26R8$H1MyRbS8GxR-Wsw7Qqe_C^XIsk&P_Vf&bsI>18&`r+(rQaCDPh4oAt
zk)bxITWU+{Q8Q=T+!e>ec}y1@zfRUo!*5CF;an2L`;<0}UW}Q^Ji7bn$8yR1mCw4f
z2>Qrw&plai?8XUOttw;Ia-Q3d8s6)DdhWZadn?(R4(@`v^F@-r77rwT2-JtQ;uR2^
zbndAVQYL=w2^$rWA8=ir{je9Ucvn+sT>krD1D9~7enr^w9m>&>#)a`H3*4OE*uk}K
z*gjs%Kg`4ag)CuU?9aXd7A`=w+g~P*P~NWs#Un>I)P3Z=2&F@8Un!`@*%SMK#himI
zxm*2jL|f}>_f)j0I&GLJxrZ(j6YICp!wG4{dAA0wLOX&bDx}?zlp|yi=~E`&pilA|
zwa+Dltb}ST9p-T!b)oE?D^T{%ct7o({V>S_3>cg1qO^XqcXn0#hrKf~<fpx}^-p_e
zBin_!<Y_|hv{XR4m=tbz!6-eJewTl|abm%8Hw5rZOMhqR^cLtZo?v>j6Vwg}mwb=n
zcyV03>AdG|_VnrU2Z))bajn|t&^M8~9t3C0$J3udrVNTUK&L95Qg+h5$PgcHcE9tm
z>BJrB#|B~V*1PE*U*UvY5djx!**Jhs$ygufPia?fRmH9y!Trf&-n@J|tNnen$Mtku
z&&OB&4R8uNdxUDt7!I!OeQ`}!UMm|UD52roalTm^<1V)0cGP-s)}_rAC$>NN`2JAz
zM~<$`-g8a0^pD<H1y#s@KZI3p=}itfw=}nY!HUg`?1|<VG&-vt`<l4Fm>j3UfBH^7
zuh_IZ_QnK>8;+8v(A3!$^cF4T?Wp$N`1~beR_MCkHavLSpz*7q+)jla0|reEOxVdL
zaVVbG=ykkP*Qa)uM;`rdYC!tcgFMabcTUTQ39a>lgyGDhK&9Q`y3@Uhk;{tJWD`D~
zKS~hbVS||I`9+4Ja)<Khm4`IwJl<(%v(aCgm`?6H=(8TaiytXap<R1B$M9p<jo<mf
z<kN?2*$O7;-J05B#_;Mri?`O-S?i%Y&DJ7Yal}sz<R^ru$zseZ14t<xJOhwhi^=@f
zLn)IZPgOKzTRru^)t63O`yt}dwSQ2$7BNZ5Q<-l6cTS=A04g=YqV0)1<h(_k^ABj@
z35DjV&YH@gNS0_}Qs4cXn7Po!j|;8>-rFXC_qObY)DS!NL0hiyepA6fa3y0{z;T8j
zq4Pa~Cd*vES;LOUAcy7w%KKFW_74@GFdkUKvtSGW)rh83TWs%Y@M{f0DV!ncsLQVc
z_;Z_a8LiQEe2K01c__W+hA?p&0a^^0fWht~v}w9i&rR)H(*62K5dZK<>YIGx-U;Fe
zNSmA(hKMxB?xs4>M6a&S%miel?McCu!Q#WM(a>Fr@hBkUokdr~V_$$E*9RCy|FevG
zQokiIZq8D<(SU1%94&CJ$D|xBwy3RUnZzwcs>@Fa1lLu>kCfcm6TNe6e3m$|H<OF-
zG(Vd}8f1TI$s#$bRH^wwgRpupI<vIo$)Gurl)2#wOzuWY_whw{U(H&lav~Jh?xb}W
zt2MUj2K2B$8VsD;*l7fh7ga;<pS<SDcnr7QpE_f<B34nCDjRE@HBIvyd<N+M4MzbZ
z3V`Ym?7lr8{v;3lHgugxrLr8Ec7_WRZ>x4n?#Y-oX`uZ`gRp!1R(-1;`N07hN9s!R
zBt}QjW_C%P;J&6rZ@gJ!4JaZ`deHe3B9mcDwfC%&C*JD8+iPJRB!<9oeuS=cUYm_c
z&A+=GugneIFwl;pbDEuhF_yo6_Zl3clJ>~WhYvaU1AFv+#O(h?+k1wy{rB(xEm~BS
zQbmnut+tA)+9Q<OqeZD*wDumc5^7U5YHw=R-kWIcy+?=@dxY4r^LzJmeXr|#{l|^}
zo&R0Ok%Q!j*ZcW8pXd2Fk48B>hKSXI7+VaJrifoZd(ijgU84%ac8w01u1lCpE3rX9
zt#N<cEuS@S)cfEe@NrqXL}d(z-UJXb;J=72uF>Dc{3c$2S*MG+cc^JzUcAzx#v7o9
zwQlkr*QHw4tcSFrk)U4e)BNor-RYCr8f}UNY_v9{wSzg@;RDS5IL&sl9=2;&7j&Ui
z-e;!^Y4I+wj4=3*zW*!y?+7An5PMLZ|1SmG&D@@3k!^2U{D}d+(<AThrMBev4n%+_
zjxmsU{RVjRfvmy7GeVC8a#gNJPyz>Lph0xvU6J5jM0pQjbcu|=9JLaoDK96q{_AbM
z{u}bqiTz7FXXf_scboJxH`MXlGU@7M{DI)k%m1xKRV-otD3g9au!;w;w%!1?UhU7*
zNUhWlevpbuP<GM$2YEgSAkS5H{)0U4n7s85@|-%8Z{76l{~dWgzub97_v~v=fuIjt
zp_XmW_U=q|p}s>0)nuK637wE*cB=RF89(g&h?TJZf8cc)Gk-0XQQQH*F%F97o7CoW
zM~7B5W86CpJkFJscUr9miP&kKmDgs2g;paEZ6^CiLHJR(;~eb!^9hkPjn+TbT28C1
ztXIlLdYUBL&LM5E_?&j2xMe&dr(%C<9u^X^(YhR_B6j)a;)~brmF3ig2$FRwJvPc*
z9QQeL&VIUO^q?AVb8SI&AyU-#&gzt5V6^F=uJh>K2qNLVjalz2la`Clpq;rXuxY|K
zIeK!twCyX$0&a_!M~$c+3;v1@mha_RoztGB+-K#K=}iK^*}f-d+lLarL<D++pI67e
zcH7?+XeC^23hR^qvTTCaPl-K#KLZ@uA27{7BJ}?~c+g~@dL~4B3<_>7<%m1J2<Prh
zkX6%n9a<Q}+;RlKt2EtlVqNouj8tBlo0Ee}4nWxDwZ+V=POZMf{%^d2M^<)`Y1X_R
z?Ju{nBgk8f;+)bMZ2%B}5%pjfLF$g-Nyd#Gz?{P<&F_@TU0)7MxmK0vYbg*l)y|zL
z3Bp{@3_zNdnDBv}t(M}L7eG(st(qR6g*EPqyOke=$1CG;sTZC$e|&M%GoCP$mdjR~
zrGg)|j5-sVwrx9xBrPB|gud#F^NTD?&6N-TzViDl5CXZyo@<v9K=WzC%LKCIEj_1b
zLs7yx`poq(b0yn9gs6!3SCR@Z;QbTJge^?rVLMBY<nrhrv-C(taFz@uG@Tkpw>uT<
zA196lv&uLWfHXqH27Sn!mJs;$!xrG)9D2ZcQg!=!1JF1+kNMb2W;6X)?|sMX{_`Y8
z(PWd4nrP#eFSJVjXeRPE^~+wV?@3S>2j6EQ>}=uxU)lAq0d{@FUv~W-5j&uTwG+So
z9jou~m*_G#s(CLYSF6-uGFzUKP8{myNHOr8mN)L<8suLBnUCdHV2P&yCjvf&uGOXO
z!8_~^o#kWxj>Y;elGn@uI#rgo0%F6wq#nyHm>K+9@Q*<bk-7`;ly38RJ?<SBw4J=(
zb<R#iWTg%^_9|tG0^FPbYTst2`VMGIFKK0sZR{4iVLgJFf%+ESh$RWJG>Mn?r>-4O
zO@${R#IGXlu00U90##r9lTo$%eKW-r9G>=R{rb!OxDDy01V)uBA-?u_Sz`(r92w!v
zK=Uq_svD!vTDTRcjO^k>w`8B^=34ihDyNoB3djEp79F1$Tn>C{u6A`<3yi(JJ6+;l
z!`aA3X5O-Mp|%N<SRZRbk>Z35(QK&z`i1m5Gu>qyt|>@bsGMo|$cK&8pfy9Yc&4T+
z{hnfoFb2MFm@I11O7Ub?gW>NJ-KFl1qJb1tr%!4~xjd~_3!Q#<!P}b8su{1=YQVP6
z+jnldaZ8Sa$7k)ENY^y}kdpQ*BHRCUrubz3Jxelwf6S5pyV8ifaT~Y#Uw-Y=S;G`U
zk4FMM#>sBCBvUt5WlLtgD_0L3Dej&<pu<Y0o>DM_rpx8K!^CSWiUgrFW6n}!Tm)7>
z$XtNTtS`4$@JWVf?=!J-9?eI6D)hoefPEyd_05uT@WU76Rfvy6A49+3&wK|Ebia)=
zvf1P4@j~71zkk3!{K3$}c)2-qN+$=vI#14gVE1PMkacO^gzuD>sx%J{@p`TrC2SXZ
zgFxGwFkpe*Gei&C-0y4l)Ac9g-(@MKAbL#(G?xMR0^n;G$ZbFyH=m;zzrvD~tDHPT
z2EL4pG&dd8yT-+S2+WB?<~Uq+YVuy@t$k1LmA$i6esACrU!_GQ2ks+>BcI*utZ5)l
z`tpm<wamu@&8_hg6Vk2MPq7X0Db;9AMq+i5S&4L>JZN2Pb8D1XvVf(n)A~?#{b8?b
zkxNRf6&MrmE+2aQg;qU%yLwT}3#O@*#Aj_<xi<LRHN2sE`;k?;u$AShpTFUHpX#^c
zhV|;PGTXN2o>z`nZadeH7)9POqFjc0NSaOx6nZ^Lfhpu5w&1Q$uGidoC)Vi0)g_0!
zEY;K-^yyUeR4#iFV7Kl&P<i^;sZ@86G~$(I`oqMFu^~w7wYTDWEbSiRDI+ML|H3`(
z%DtuXaC9PHw`<o8U2w#sLbI%H@SM)O*S!nDrfU;9PJg~*--s^SKHPG`8^{q@Dcu}t
zOs@e@svP@`XFm;U-h|FBQz&>eQRNLuRD`VdXK|PJQ0C<p6tFn7vv&ik+gVinH~EYZ
z&>zSYDH%6JP4(QEqb&odgPk<((Co2tz82lp;?%!(ZfC5){t;mh|I2Dv<4j^RVIO|M
z&}i%-k&!5Fqu{2Y4yMB`C$JoAA~Cj{x>OMoV-~R!)>mOQa*jAr*OKNVJ&ds}N+7yW
zW<W=iw<_xMA8x2&?6rVs`Elz)L3b&t8Pz0IPx)z^&*0BYQB4S+WqFh`-C?mZEzi?x
z*1?hZnI^45B-4cRtT~yVw4V~K5aPDDr@rObr|Y3g8%frA-Dj$^hlklV$k}CD003%r
zyfsli=ep3JBw%OF-EvtDI~?e4Z&uHP)F5q>*&@`?NY}5hv(+*|<@tmzPTPu}q{(_G
z5nYQMP1HC|<sU!qvYYtuN{8iMnPDc`RN;oViShGmfn$rO!xr+RCHkR9m|T!|N=hBS
zrN%<d90Xt2EVTrc))R5E-(@&mW@Hi{!&W^}qThDArA($fQQgLK1<P*=o%XuUm4A4z
zM9_p)<kQsbwpn*~x=rnJ(8GoowLqb49NoA<_rLdf-GA-z?)IM@)&B8Gke{{b6C|Am
z3Y&-zFCBVy9`&E~#evPwvOEpUn}n7ktfxPBfqtaXpLV$a6Etu`84iYolbJ7__m@XU
zpBG619rcY{(RBa+Mh!@414CQs!T#k387ASw(NZp(iCvJ69>U~<&f}4MxIxAS;FoZi
zbD7!^^SV5J>D=@D?~`;ZmgAy9#C10%(maXJ%2X=oUUeu}LwAKj<}}N!+pN`cvrPw;
zrjj@T)g9<l@s+N}C%amhnG3@fc9)q$pUrY4$VyDO<O}qMA@eM1iELD!mwj4x!RxbX
z6V~nYaZd!iW3`Uq+IkM@s(N=_%+2;q5~86CWYeOIt<+T3&AC~G8gXT;79fQG*m`X~
zS5?i>GPXaN?_y@9k&fTgYKODOhg=@*Pp(I=qy6l}Bdta}PvDr`iy4TOon0gmLIlsp
zpX`Tr8}KX5OTG6!40iMd5EC`JW*5?~1H4*DKvq0NCmo4a<xP+YxXUWh7}dhvLdPx0
zWP)sA3B^-inwzXTuk!}eEWs*QBL=5O?iU_MQx-~B!pA$Kez-Jpm^g_4H>3Od50;bP
zocHcn&2SCyibZDFdTv54EXNmL*JS@$n-i$L5&-|MyEg-FFRUeOwuuPJLXhOx9M+uF
z!mhdBJK3b2ZEUW%IIYykP6z4>ROz7B1WVIVl!3Y!SI>AIP;0xUSdEGS2Zul1db_;o
zP^t|j-icHvX*J<|li^mjVdL10OmS4Ut6Lb1A6*%x6uSBRya8<4m}OJE*X*lzx_68C
zy~W>*F%VEiAPDh%el0|*?#xeNI0X_}<=^$j?{c`-E<-MzjA2`T>^sXCag`F}eI%pB
zwyQSyYMw#JB71Ic_L$$@bhp!0esglOsf3|wRni^I&14&PrySetkp3g<-Sg0^xSM&+
z0@LjB7mK=Nh<Ijr6_<^cyw|m}1(J2-a0cgbqy@oRxw3yQXyr3E0Pa5z!*mgUbQz2b
zOR5SaANiFLIoJ#iIay!(&JyBd;a;lq*pf+=g%WoW5%WU7V#Q=GdC>(=@DOf#Ya%}$
zYPak(y&OM{HEA)aOR(oWAqATu`K>6uZ246T%*O3BO5dLg3ohlh9WMPPJ*3u8C5G&#
z{;@+IqRdDx1kn*e0hl>(w#(Ud>Pwh!Dr`a9eP27dJ+IN;w^yDpHMusR7=maI2lIlM
z>A8fi9?^+^j@fC#*3=-c>(dWgxK8=pq3<t)nOKpg@dGvsZ4kv=-Zps->QoPxSMbGx
zvEqxFxv2e&n+pg&>bcy=Z17=J3*tO^#VfA+H!*h?8Yc6q*ji~;hW_6y04P-WZr+x?
zhwO`5amO6MN6n(i%08arJxVp1t!>h{Gvb|H1HkgQF(aCyqk&zjo34TMhVCV$4x{0~
zh$sXFz%z}<QBPgHufZz3jLZIdp8Go|t7f6sjL#J-*p%t$K3AydBX!$kNJpn$fY*V>
zwl52$jFu5|hegWtEvJ}1Dr9YS;cnB*t3;1Y+NqgUf%NDY7d&O{VeT~5Wj3D_F<#l4
zcay-ixCZ$JWHs{w>pCmocN-z#2iAIqVJ4^%-oIxx?zk#fX?{9Q*=*g4O@E>G;Cm8)
zo3rsast$qdewRPl_gkMf?J}@)X<K+-zIP?G5#T+OKrt_1>$aCODcrR1G~zDqRQrPM
zQrSRB+ayc{vRz!?lFz1^X2hxUTHYJ}ZapVnyrw5Q4~YuG#Au!5=bg_mxQZs{U9^E3
zkk9haRt&;bJ$?5@2FM}|n+|$|(5bHQ;EB+%bho8fR@1LQszUnz(mHbtL6-*JSDntJ
zjpze`#*1C2D3@nJ=VV}dYih$wOb&%3WF|9z&O`o!#oEhlMJ-~k56i1}yitp+3Yl})
z4wCLzPM$21E)leQ3y_QRi7XkH|Gk9PV7z&)lKt3$yV>Jf{v9bp`bC`v838aM1}6Kq
zNiA#dLZ2QfngdxXXQ=so<lc`BDPE7Gt)4tRU^&l!ymv?8ks|d}d^qF#wAmN&gvpn`
z`eyQdUdk@MY!?qOK~or`<jca}gdBciCVRtR4YWm606BQ$UQTOxMdsV6AfDDt^9th6
zU&nn(c4h!P!bcjm*0r0F23DP9pL#;lpx-|zIypXz`wNT4Jb4auFf8ejoBxv1|5pW&
z;da~qGfZXre+g3o*$QjJmWmR%Nmn>3a6A?TxPy_AkEPJ9B-~Zgm1oU~q&`IUTf2GF
z=Y>Tr?YEpNy7FCdd_#Fsh4&z+o7AlA-Yc}3J#`k#MN^bwilVV=AX&XHD-zdGqyyxf
z(ToO!j$^^?VBfs?6>JkJa9wd2;=U4s7^I;XV&Gmd*rimybNFD8j=;E2xoa*Z3UX&B
zDI!7e$zug`74OM;Gvw{UmFr5JZ_pt359_ZBZD!(c;>l@Ma9E3;kA}iETjpByUSB+-
z)y`YtL+mhg2o|I|5&7Cy=z8+=waqE_K%-%E9~ZGebTWVUm6r2Xo&6q0LK@(`*!>hf
z`<)Jxn180s^!~CLo8z80?QA<X?ZE(fOEGFE2W|uaYv`n`5@jN}0C%P*skmWxen{^U
zF30W9HT6sq!{m#*J;{{b;{&p5n<uAjjKmI)RqO2!fWva8Fug@B>Y1um;uLI1%JbH!
zY&^>XaV^NM(XP%@;Z6?+1zWm5+5PLY)`5VnJ43EIcIy(1y{Mb@7U&qzPL-Gn$NmK&
zn&}j1*%dBjY|mk?5o_)KkhHgU04U5ATDMnBw6)RL1*kXDEE*PF;TkZt6C<%F<l1tm
zaLKs1KB0|GO^n<m`X~n7sMwi7<|xiv9}kSeTD0(Z$YqJ#Ses8qS*YkG@PwHn2jkdk
zTby-62qfe_ij*`ikG3kZ4DTIYNBME_!O7xT^xEFw4Ons%$F);*4NbuP<`uf_Q&KdZ
zvUX6PYhcgH^B9bg=qC4y{vq+}QQL;g66c;26(>(jVG-n%+vcQ!iTv|vsm;nR9jM0D
z5QhMwB<2EO1h3d*)3xu~keU0`t74r`ufL`u2r+J>2Hq%!K!}YSS`ir#0Xv&ICY!#n
z<2oO{V@ygw>#uHiM8mQsi*fgG%*w0SC-Wx*y|ovDr1S2?@1c@c$|$cN5WPXNjMa&I
ziULXxWh-9$t<N${xAsqSKVCa@1=KipK|-E~<1V8mA#Go93i@v|68W5J^c@D3Q7ggn
z?%Q$<iZtzc2~a>;Z^XvU^FMBPYVLiqPyAYSc7c5nm#>^ETKwW?i0YT+W!aaK3QI>6
z!Fpd}dposQPhy)PgrLR<myHTFybraz`bB_})Z?~7W@3FlPU}uSc$XzXq}21JYF^Jn
zq>Xh~s+akBGbhXl<h{^YbNV0`Cwlbeblz)f*G*J5(g{!byT-)#Q2D<_()BhiR=XHV
z$f09;SKH89g?Ec4Dc%Qlmu>akia;L%y{IGaOUGoIdAb-j9v@RLretOpJmR|7XQw_+
zHkL=K1g9VTB@U&SF1#-~uX1AZBgNT<V+8)#Hl~_@gQ7QC^3uJ&?o>}b1hA`svWDHs
zO-_3;v7VpjF>{Awr8_x5MN9uUxcOuzFJP<5T`7Z>$2i5oSH<F>mm?j)Hzlf*Bs8+%
z=H-cmCC_?}gJgy(QzPgqq;agY+mreCgFjehb9ttV_QM&41yZ?({;;c-*bXGGUE2ag
z2Ts+M=Km>j{EcC_xl6c_CNZm6YOq4<;S8L=;-x8no!s|T0BTW$!Vjg>Qn|XDvru2k
zcim3I%FoW0r8vGD4TcGx-l`bm;D)RY(-<1ZkSx@BC9v%S3vk<mwQIqF2{kdtpWM%6
z_0&x~I{b|)Y#l>DKRwD@PO{JUugmWHh0}OQJE5r_0$%R!pSLQWcD{%S$R}&l0O3DN
z7i1)_dInS)NxUVX;9pe}bwJ#T+3IqC(5AD>EpmAJ<{6pr5;+V{Ldi%}C|{MMKft?4
z3wB_nE*|`<{Z*cL=J`e$bJ!c*{|;ROqGn}$$t($G_@~sHOl_k8D)a@WMO7uU=T6Al
z7ISHNY!;5de~*VLrFDOK+@VHIFwC@A#Y%e$G8Z*QZw%xjx9bkuy>s&J**oG%hmJ(c
zPCvJ4oZ+{!iJlQ0Scz#=(Y`yCAi-;YO5K_Nkcj;;biZv~lO2lS@Rb2vt6?WrEGrDG
ztA4xK7WW0yz-U7zly%is=POpwNu0B<;?_H#o9(H^le5o5{H^yUewA7ecT4hHbV^%I
zWXHPbmnR=@3^PM-fZ5KNM7ZAH#Uou|NF8nK0i_c*-445am13~3DEJ}!15^A_*#|AX
ziL|Q~ZBMSt^=uZlVf$jC`+@2f<Z^^5NtNdcg?XS9$ILFnUBSl@1b&#m6k9qMWU$eb
zNb=hOl#-fvy7DlLa?geD=SKdf1kiP~gs<*I7`*9JVvKI`nj~9!BEBIj*t1mRIi%Ei
z-fBTDOpHG_Y(Lt+XpcS*4ryz%pdToP6`s_n<hc}CwB<t3oLx^}Y`TZ-><7)M>tDPx
zT=`i+ZvOnDA%ghzr^=~LPgW}w&lh~zyGkmQn=>U=4e;%>X8!t`{7eeJ1c@<V7qZe0
z%Aw=4&^Y+@`iJK*?&URM<LOA~O4Lz`TNN^}F-I6~N9Z5@z0M%H1a9H@<oovkg2-ND
z5T*J2`9blD*>~|poN2#6PgdU~>6BK3Kns*NWywjkf?nk##iY){=(%^ZqnYEu<mhFu
zNd#7c9=shG1~LWTex<%`T3oDqs@-RG_gOpH9I?n%N6kHUZu!%B&?{TH1E)OY%9=^e
z%h?gE3JM^+s2X@4xlFR92s5KR(@xW+(x`Nc5-!oV!M5ZtLV$DNqtm+md05$6`!6HZ
zh5aF4jht6n%!AS-?aiT0mv6<@Uf9p|qH)1pl=8-O?3G3JyOy_UZ*Lx+a-59&akDH>
zcsf^;l!%LU5I^Yx58}*>3#o_%aCakwmi$RRf5QER8STn-LQ#GYIl=CZJbfGHz;Kn$
zPxsvO-bbhG6p+}K<F?(uO%%j>S~(07i`mR35XXL7=TX3lI!UPki>%wXveT?~8w97v
zxKZuN+uY4<gUw5&A>1r<-TX)lJoc}07-D-A4BGLHE>-G;P8&rO?{r5y5>$X>RbQ%1
zO*;(DS|y29XobkpCylA&s|^bH1+w^O2~~Q38`u<NGs^CL`L&FfGxDi8k$=KEqo+`_
z`@@Tu;lwX1_8W5NSBTZo)AQWAEoAUWi*k4cpkB)K)IY4m3o%7^I~ZY9S=g_;H4$3;
z@aN^F=;}r>(U~F#%6S!TeK%52<KB2K!gHB(s<tJ~+qz(h$5N_=!N~B4E+s{}Eekn$
zmmN7?tR)a}ntQ_l2L&-f!3VG&EKtJe|84@?!7^Obh%%kb(cQ&%tjJTM=LD{GkLikX
zIn1Zs8LVjLIpiAbyo)AuPF+<2u*|n~y6*9|dGA)!Agm8(+rn4<*5!nw+McSB1%xbF
z3iDL9-Ha_2@UjJknSd7EnE=XrKWI^Q$wnCPGxZr&M)BdbAldPU>UV%g8~6$$=wKSi
z06Qvik9g<%`Cus&H2o!>Az5@jgiqqD4-xZypTKmpbl8dhK|&q|PWQ!Wt-pFgbNf>e
zeT84OW%Wi&uBz~+N1H*Eb@ha>dp5Po?-AzIUd4>ZV1i|F*u3H-LE>6j8$?Myq{wwj
zAIJUpDz56GGxz-5M1xCFe1aj=EHKsK?XKO=(aZQ;(4Muc)i@ItJNm&jtkx69>u&M*
zZSCR5=fR?U_4xEFr=hYl$n+H=BcU+w{}OJ$1l+)ETF3zp5P&G%w{=HD-0m+u=MkhG
zG+8O-XY$+7?WmqwJDlV<C_U1W9E~tPeY~-F_o;zUc30rbAX!QAf3oCcbU;tU|76MU
zk~VG?kQK1praSKNdk4GJFs#W?ILu(a_al{NtB`TOAVP)TnPM|m>LRjvRxI>sU}R8-
z1tp&JSvm0wsQE_OSlI|ukomGvf?SFsU7U)<%Lt$vHv)i3121$Kmmzx};Lud_QdwFk
z1n@udvV-0<O$IQG=xzT{acE~cr8f2>%lueeEII5#Opp`=uz?q&r?~+M2c1(m+={HV
zOOG_ysWQKBs{o6SG+*maRR5{h#}O<`%y81xHs&aBQa^G|wi7?dw^&6Ld~e}x1ip|%
zeDrtO0;WiME{~Pgm3`)KsJ2CKdan=XMtjV5U~6vJNmRc)aAbX6)QCbFH1G)t`KmK>
ziSI?VYXH#kvmsnnoU#rZzlzjTmhK{IMtLWQ;N!Of?}vy5-aW2-plC}b8yV{Mhgbyq
z)iBE5SQf=x^jt9m)gE-A>dEws(4Eki1e6`m5^!vSr;G+g%QDqV9FaqCXM5nDthg02
zp>^LY?%{C{aff<`q_AxLIv$w*X$GSYO{+Wy`K`zViRh=T)F<}a-gf1xZ6=}Q1|kCV
z(IR*$0^G9m&eDv5c{>gbHcgRC$Gu<UcYeajSRDck?_dqg&EV@Yo`JIW8B!9A?<yR;
z)ta7yDP_DTH4qVfMNILC0)wd@G6ec<mcM;_%;O_-R>w6Z3UfI>=R68gm0P-RlVXrP
z)uCT{-S?Lerp9_@<nyU=`;9MYsk}k5-a=G%6M{NQwYw*4)I)5V<7$4*jADuibyV0g
z=L&hX)*c5AptNRrV#Blk14u8Xaxc(!E8;7>`XJXe^v-Kd9!|891u#b=p%HSfr=W;3
zeSQRM;zJ5^rrVB7;VaF1AH`Ib&W^p$)p>E%>T(%E#>vb!>&H&)g!?d;1v=~_maLCQ
zPNWx6z>(ndsmo;JD50(}ylG~p*4psNO<5l8&C<FZ==QJ>_AzLMR_=B83|-NEA`W);
z<!Rn^L{CtPnVBIZv$ztK6tdd}ze0ywQNxCIJvT}=nUm}nUq}Ci4Gt1#$5-=r!xry6
z_8+sTlV`5S)+djWC8~(-rTEwR>?E`>S|fLpE>sUC(*tBh^r6We{yxbb^ET<Q*Tc^#
zhc;=NZH2J7X%VuQ`QM62)A=$!;Su{X3I0l;;xU3u@Rhm`(7Ytn<XNXD$A50xQAIJ%
zXFO<p&q}&OiToBp3=>lzuCz7@qc%3i+pk3SGHhLRD78Sie5+AR%5@>4K#+=vJqY6v
zmHS1kZBUTY4E?=rM1>oC7&v!rUuwSY;`9kIVVN!k;S2x$M*KoBGEk-ON1!0ydV_rx
zW#jRp=fO{E#B&Cbk5jgnc?^~NK6Or)lJ(w)wA4==GU6kbkEs`UAR9gLy>c}9pzL5z
zj9Yse!^dk3jd%djON$pUNOt{og~&3G_*sqdGRgf<2UgIlvv-YKP03-Y-Um5|)A|rt
zIW~OCHdJ$fnPqcVsYMybDc-MKZmgBQ+hcW)E<1tl+%Rc}@p$Z%7}EM@Jq{_RAavS{
zJLyBM?n+MIvDs-dVDIqeaCd_mPdV9KBGOYOn1{FIb+rhR&eoELAP5_rgz<{i<4V^B
zB3$C9)7s<DvB%zz=s;enoHI=}Ddx(=uhm;GnzZbMXRhz2^Rd`W%@1lO?u$oyGpE5n
zn<d_<*}+HUwml@2K_=K&ZF*u2^@dAJVxRd0&V5FFxQ%+6I|o6&IC-34U8%ead(j?U
zqVI&extJ3_UrWNBawcw4T&)QVe4Ldx@zVs1@ASY>l{#a&p|X@L$<eCtpOLz^^l@d;
z=AjRjGvT=ekM_VD8H(sej934`o6P#FlSIsyA>nP!Wx>4Ol?g9~qPJtSTBWXHujdiP
zIW+FY-h@{0&vUDV1A1J$@Tw~-WT*B<k_c&ee#;;f7f;*tACX27&n5f@Mzt=oD%PMF
zK$p;;3oCvcB6=;VLHCFY9)rSvES=aA5K5iM)jVQSByKmO4iwJr{|OX_=!DF#EVl(H
zL##KtWku=nC+^J^nZi}Bdjs@cKKQvESQu>VO95e7jr6}|v+eQXPZ*9rs5)svy<lh0
zg4Vtp7QZF=<>Jfoj%0j^HZOY8*kZM?BT7aX{UR!0<PHrQ`VDm}m;NW+Rhh&6=!@*=
zQ1z>faQCvjas(~&`1>Z?|E`<WJZul*p>{XFZ~RZ&Y*HyYGI;d-AcyJ?(Zbu-383Tq
z954$qy>A6X8w$94b&ALKWHEC{peOiqccgs{cADSC5`|;X@1t;eShI*nGGo(q)C0ML
z>W<Bu_bkvkb3cWSy%#Wp(^t0Ldw5py-Yo$({+*l0{atzsot!w3>+j&hFC84!0IpuA
zu_iS`R}roUjF79Jw?Lz=$e+o@3qPIu3FU8QD(cHCEcPsX`J&19jdi%pq~E6Nq^DQU
z9&@(Zt(^E%B99y}cV&2;oZ@<McGC|U0r_{R#N(*<1)c0~`xKIcR@ous4oVUV-^(!U
za>!{fdPDF?jsP7Tb*D=mVE1tqgu~B%$;Cuob(dWXb@D=R&zz;>E(^vc{oCJDmtH{e
z1~$@Vkv*<y8sP+g?!8{eI7z06i;$F)DC^6K3k8w|t{;Kf(XS7c>yHHJ?{^y+VHsJ(
zT0<9vys(kn_=)#`3z7UXVbkaWBS}{&RaDX^RVeW89UnGQhNX+b_0Ai^8%3`5{UQn6
z3nOP8kBavZFCFQfcYFf$w)9-(;}yfQrpO_MTl>Frs4O+!X4GHrZQF!M7{6f}nmsyF
zrdm=z_@I#CgC-P`X<g91Kbf6XOzd;p_(6NmksS`rH0(ULMQ-&HY!_|C*0g&N!Bi+8
zXoYhs=QDVP^&Gy45<eNV0@08uOiR9zZ7oZ_+t}+Y<QP#F+H1;lwVn=Th@exjN`<+*
z>KetKyzjuwWtc5g4b%)?aq);B2Zt_kQCTP{d=g~3)D5&pkFW})+z0hnQj)AN5OImw
z3XjDZ<bFA#9bPx`T>7)#_e0h45!e>33IXgeUC>CvaFP{90=2s<G%FYPv(Qz{+~HRu
zc+)`)E|FPSan2~{<XoKzeamsLCj=HDVw`T0C+Ky@9kSSWlAe5RwIfm$`WfH(>~ktf
ziBQKHiO9A@6UrBnkyWu5-^BNHyzYgTlyLa25jFW*b$2Fdo0ye33IE5J$b4C^DtZ+D
zefC%z*Rw_ke^tMLhgC90>yZJ}22W4kGR1H=ww5(^M`ZQkxKE6iBSbfESdEp$az5Cq
zKn}kt5~HV$GX#yk$4cxjy2L1y?>kXm7-RImNu4)#@qD`U8dL}uK9fLxI&{gTE0yo&
zrDxV;K-smsTygb4e#`f`!ts?|$WxreUEXF`jFmJOj|pM1xCrodSr@1&kDk_Xu=v+&
z4l1b(cY#Qe&Om)~^)uwcT4k}S_tOjAvso%nB!-u0D?k1G*hhlbapj}P;@8ln82$xl
z@!r)RpK{!C1Q<E*nNEEHc--oi5)j_lxyO*Le5wU}8536U$-Jkc2)fK$X1mUT(t1d2
zsJOkdseuM>AVU%qnCi6qH2vlG+jj+rk_3jD)0<c0WOwF04Sy{%zilw-r-F4NII&@|
z`I<Xpu(;X^{(60tZDZNwEv2Ol{rIY!Bu?|<c%5NQ!}6ciwmgSoJHM&f5g}qSn(Sd0
z|4A-{OE56~UN?n7RCG5d0He^o-)YLvwg7dT!8|Z1G{}4J<|4_ZP?l=wuK<octGY$J
zOT#+y`?edD=|B<nk2D^LG~X{-E}b);+=z%Z%7!0>zj3wzBcnhP<PArti%-AS1iGk|
zK<{B4(V>FU`$dM^A5t7&YQl`k0R+w<!UCwGu6sJd@=ZPp{8%|A<tXvjjT`h5RB}46
zIpHs8k{{mtL_{*7HxUmONBokcH@TT5EJ<}Mc`NH3eeqf#nS1w8-ot@(pnUaZZupkT
z*hijkT{5Jrxo?LbSMUfJkDeoXWShsmM_$@CuO(UCqY?9P-e`#VyA6FdA)+<gq+4zI
zCEmcp0`a^)5q2_d_CuQ2W~OQ>YB128h@AX!<eQba0tH8<-k!#_>pXf{_Eg8Ip4f(k
z`JHgI+z;pzX@v|_LY{3cghQ6)tAqofl6TBTG#v+*BZ*q^Cpa}1o}3iU`1q$PdAJPR
zcP#KoV5FSblf8b=pB{eNzUzI}^5f&nLX8dz^X_!xF1pKju=_Nl*Y;2Q7{v+07i)li
zK+@$>ejyLHt{V-696{>2E!~M=GjsKGM1r>i=j@KiF{k2PR7HtV3~Cf_#eLR7aAHPR
zw~q(7L9R38H`m)mR9*(NMIBX#hBHK7Uz}f^bVdraf(tGybzu3#Az54oSl~gwAhKsd
z9jh_w**?ff9mJry_2z*1HN9O-3qG++_#9kUG(6-w8g+e?>fl|nHNzot9VBEib^lV$
zi0~04Q?lU6jR^uz$DGn<o#gMBGFM32CBh{OwlFg(Y{S_r9A1X?nB~@T{4+J+E2Xc`
zl*dG_72dQ@lq}r*M5yu=-hj&Xj<u(4mD%^wEjx%Z(Q5%Qmn;Wo-RJZ>xAz^+?dqo>
zmW4LBv|TEkxCT3QyEu_ob*zDFAg*{$6x&i#+I(8$rZ<FQN88ca)q5$OjWnXVbL^q_
z9lISaO#Ks7#ZPCsL=GE%O_2qYHDl@pMWW<AW{{rP+W0gNu@4>B`VsncxWNMtBQ5Hc
z0-&qXlr!%8)$84F;wbOnTI+|$t_yB0Vps$d2b-5QlV6Jc{0|;bV-daU@9F(t&%#rD
zBU@8cgjX)Vzs}Slps&~_hIF@Jc_8;CrENFI-NWh%NrWLvLZ<0so1M$0DvjB?=t{co
z_nxcQ#UbDDz=Cfq7o=P(!?%l*u0;HG7jzlR@<8tS64LHejn`<G;lWAUdj3S8;}rM)
zoK6Sz55%}#huQIap4MNKlhzooQ9~-yU1-nq7oa3+Ix+{V2z*GDGOk>bX>x_}hFZ(8
z)jb&e0UcPN+>Eoj-nIJiaCVCs@PYUqV-DYm$Haz5h#xa;H*L<0h%X_3K-S8|UKs-I
zChx{WRrK$h^Ivp<i+iH#TUy%SXRY(z>(w6RaJ58fpwszsuEg3kky90IMlvYKcycV>
zg=~Mnpm9eQ#xxAQI+hc;+@sg_N_Ek!=;P|Ux)3pLPpI=Fi8k~)<A)p$uy?pXPo`gr
zoFHZg?E)Tls_^b0xos4==Uy;}RUGz(i`df4v8$$8ieOSM1nS;F3pfHLoH}W15xRD;
zq}<S}`R0h)51QKiCm>HWfWqE}-w{`XirRnC{Uy0uIO<e-_2(RX9j#Pt34+6$iz_Z$
zc}`+o*^!UU?v;K>OCo?>k3UCUDVzn91u0dFFw&c;HtAjSesbpq{cqE#gg*0cPdP3Y
zn;b2`l-3{v-#wTB*oopsUzl4v{J=k?7H=}yQ!GCCO&HnTNh5}#sNZY5t$IVu<h<!4
z)lkBtDntm!KMnI?ex5+XyyHJyD>bd-O(&x)_`!3+;<a2i=RKB$ytPLS<)yO)LEOS&
z`|)I))S0tM)DEYUI&$xd%nw$Ke05%MQogOCIp18`$m2N$B)(YP!<d#EzYWDtqZNhE
zGG+Ic)nF4J#2k(Ne!mGVb*}o>VR$Shu=RnG4u+mpa+Ht37s)&?$m3aX&OMHeI6vBF
z*J`upq=aN_u8RN!CxwrIhpJVev^&4ndiqtAIbh?l^&=#!83XP~65o_`^H3s1m03HL
z#JIs^nAnz;hyYspz5cZOl|I^#?wX^a<!TiP*)f^Yc;qpCQ>@r!Y;t7&@Mqe|d%+s%
zs3?iST~|mz>dTe*g*3WgFh@V)X3ntG>^KhGwGb`#d3)3fx7VfVU7%i?gpJp8mTz2I
z<luFTe5I5kJYv9j>Bz!3%+2B=nW61jU%hkc4H=ru6&9smN2F5|wtU<*r<qoFynVP0
z=_|kGijjB4omm{H`pi08E0t?3=b(H2O4S;C%V{3Z$Qj&ms^6JLF1=(O);MOu`KPK>
zsr0#g<JshS(>s0myU%8m)v2pcuFK&S#_K9vvZi2nkdp7Q!+6KGIPeOYH}X9X`L5<3
zJPeqLk%(?7e7BX(*mk7DBsWVFOwr9SxBD7W^5$OiE_w0^T{_wG-d-Ec8eW`>=4%P4
z?~M;~mcC;9&wO&9FU=0f&KZZkXp)0oY1K6L|2-cOS}v*_%)@)NNiEYvU9<XPWqW?n
zy+p4!7j(j5j~)=Y24pu7B-zP&TbW_|=9b^MZ>e>qtdZcdnsP_k$=m0NapDu22p^N_
z)%XyU>k;E8LvNmbuBMZiQ-Sz7!|=P(WZ^g7FeN+dxlSO~<+8-5jzX`FWE(ucA7Pr0
z%k^P@I3LAXU5Gk=nB}>}njN##w#(h<+S7cS$s@(#h&2&+jH*lQcxd=K+%shaDSov_
z0vv<rjs4rt2z}@s$LI5A>a|kM`-<zV#~o6rM68%1-!Kg`L8l3(&9GOlWvRf0WJ}x&
z!`rxZnY`0^Gd?kVOx_i5+If2rvf*&?c7Z3WQ^r_@zJEy;d18aUEL0t2yccf59@muR
zR-jNmFqn+=3c{q-m*JC&jA&?$GS>yyZ#CJkAVtzO1e#{=8}Nmhv8NVd8kic_Tbmm?
z0ABzSF|{&35hxMO=nX;jG^6`eTk;=P6+-kzRGI~`1|g{-*RbE({1&Y%FD3y9^@HO~
zb!wp3JoJUBE~ZYlCXG7XeVpx5;|G1RkXs%eUN!6R<-_m==yflQV@0Zby1Z7qja5e)
zne3Wo4QB7^Un3kR#8XxXmXCm(Zx#N4wBHvpRxZ?3-bwQu3?IXtAVKJ1DW`19VnR>s
zuBT3(3d3_Ke@w(ozL8lmgKJSz(enjkqSu}QL)#`@D(?!4nopP7ik#MGJ#&H89Ma>Y
z=rm1!5F=ie$q6^#<z1a{4J)TR|7YN<bbVpB6K6|q0)iR&raDh|>#XL>zk7^FI<#Zp
zjiVO%+2wjNMgTW}IQ(zyMabtX(W~*yQcaI)m5h+#@jlQoPz@5VE!D=yr`MFf+^47`
z1|6jn>s-3^`&J~Gu(jSd(z)+SQXs>9!VM0xV&fQfQG$ss!uPPtedX-}xbc48J$SI=
z$W#;1v{Sql8sP(+Ktx_1AG4D-VhxX};wM#|04$rscPmx}ox(+0$)sRMo@eKydC|b%
z!~q`t#Vrx?!I1xVhAIHSp2&>OhI9Z`<g&;69T6mS?qv>?*<+-CWI*7#zdX7zilASm
zqND0y*!>JG(X)IqnQ!XK&<$oCvW2fw8Jie_q+rFKYrqBEGv6SbcnZJRwbC2QQzbJ?
z#Avb$ZNDOJ3@z&;I_M`<Mb2^0>+c-V_qFdU$$b$CemZT@m~LBGt<QOHy2qD+lVwwC
z&d(F{nVhMBxtXkSkja;+QWzeb6)t!{wCQ!Rf3oZkGHK!RvVYMFRBxJZ7ngbc;pUjW
zS}dW)6iB?vC&42<4EpZ47pEV_?0pcBWw8rI-Sf2AQ(RSg=CgLV7jO`O)qgiwBH4gH
zuvgwjN<q@vyplF4J*qFKhNt{N`s=Pp>ZeG5^6(H^VN-MTHX+%#>%u2jliKLpBr9;D
z*swU}z><4i<4Or?`=QA3QZ|<G2DiC9f=A)h#%9@2i>i0bev3ZitnVMQI?P^d-Jm-i
z{Czp<kg_Y`M+PU)6nsRC!v7Lqh>f^<15V<a92Z==n{P>-0NjF0VkFe5i2avWnI4y`
z_xViGOPJqE93x1=$9ZL3Wi3xSFvc>p_?N%JQ$D}g>((U;nk(5YEn|BSd*9rjSp35e
zszSvK_rpVhod$M~Xxp^d<sqS2;o=q#y@d?3`KpO2Z6WllGHLn+h_ay+yL=BnuGup~
z%mc&wD%Og-dB5N7ZgkAhJ^BaXJkO!rrQ4n%obztw8y<hNIw*GMkk9wMaDr;LTZ36x
z`z{;%5>F0KLo}5x{nOcEnLB2O4+qh5c_Coc3KOh|RL=6Cz!T)9FeiO}-9^a$Mi#RQ
zngS*`vym(zuHzl8;&Ax~>Bsz2hFGm)KuZc<O`?(_KOCd_g|mxg$MfUv)da}<(t;4<
ztoNnl)^Q&8w6?hE{LTo!<w}Cv19izRJcVE>fw?JOk-_Z%c7tk>(w5q<W=0>%_t9US
z2Ii<ud${j(QNrGbEr&dKXMH)_p2nwvou7!2z^@NpU=IlgkJS~N94WSKTZGMS6NA09
z4g)T@<aZ+Ga7A<8S}5=6Rl{V>^tZqiB)^icoaWAx3lNeEUG%2n40*7$W?M0a=uMwk
zOg0a<@keT_0!|-O9fZSS&b&T}%462yCfHCSzZatG*44X7LcD>+KMM$luGM(~?iT2F
z*_e{8+0+miWGf2-A{p&JWa(l4cR}Ja_r%@Scrxg0syqar=Ue?5%H(0fbXD!y9p4j4
ziOS;?eGw>8xf<6+t&~JzZUEgEYuw1}ZSiwEOQS0K-~_ki+r}bPAqJQN*KZsjU%rQn
zXZZ11wN6!_5@s&Cw1<!~?32?-efRnR8(eB!mA8$E6JcX%AkGpAIypvjqNo3z`M4>|
zB=vRZ(>Z-y&1fNDSG%p<ygd%^A<J%15F{;7mM!A#q0%4TXJ>-#bx_}K*!xM=BJsvh
zSVxdxoQax$Rl|^fBG%A;0LmxZVa!}B4ePK}BAu6}1RK^3a|E4NJbMU6Wzck<S{2b(
z1v};seQZ4sQ8h`_Dt`OcDwT=g6)mTrega!bkD8x&>ywXnqwu5|xr}qnlv0e#eo8s3
z(~M1htlodG>})YG%buxc0R5a5!Cn6#G^`w|0cwfa%XD~WCZH`ee80550KR4ZT~!C<
zJ)y<v9M(F~8DW3tX-wUQVxPnErFUlRX>rm1QSE}4r^wc?tR-q~GSR)Sa}He$e1klV
z_@jgk80c}hK4tgqiP&cTd8FHEn8xu-8czk9jjPk@6QJzmXSkX!-T<pUS9Q(6OecMR
z@|hN!=+TtzQ+-bmJ!nH~O;=ZpY}<9GsnDioW{p-`wjQy%>W^$hR&DaSZp*o92_2^K
zS|cLbR?@1`M}NXWM6%8FuE-XXmSM7R*J)-nQI+OJbwzbAW_d4X{|nj4@JosaCZ@+9
z0BqWBfQeybFVRc#2ZAzYnyo6lPb$W#d}R3QYjumy$dDytkV)%(Q?;(RGyMnQk9?KX
z6UYB0+#O+m!YRzTW-v=~gLu4hABSM4dj8yV1H{I6P9H{9<je#uT##fvn`?f$P|Jp(
z)a@lmI&x(^AYn;(GinoLTjLcpIO5zEJ%!wsGKhff5oq}{_#kM1MW&Q@(bn|7DwG*J
z!=|M-pA;Ut?%APi(8VZk8!5pRML%eihKKXt8|lm1B!%+EupMu}X@9Je+)=UklW>7K
z&<hlCA;0dljjv@Y3rKvMJ0~ZFn7lQ_0vDR8a_DiR=CC0xgzK#A_aN(kG&I$1@c!Z<
zf5uS%)KJvGqo6rC#}+%mZ6ELQeOFOO7txZC6pO1%1|i6d)wX@nyWOiB?F>YsUpT75
z8r#T|d+o-&NgAh9;&x{Jc2YQ0%T~3XhuSXQ_L_*534Q&Y<>6U66mg6H`~t+6@}<u~
zNk6=QC%0=Bv6;u#YRilk_Pu-M;h%blN#i4qXpDV8N#;X%H&XVts14cWdikRXZZ<Dl
z6T>-AEp<H78}nDEN{&pjVGb)FU0Bj3)Kh8BL=QLmAVUMViLyZ**hAnS?X6rW>+qgk
z>%GPuGm?HpqTOB)2B9T&TkFM2(sO`h6&5gOX{DM0m^i#6#30%=fB|(>V4lX>?>5YQ
z0VdGNrqX}^SWCaUUj+=qxp6B1euHk(9nvM@gVy9|bhndiyO)hO^mIdE(*Eb{RNcBE
zxc6LK7+#Za$>4hF!-?L2nR%JceQx2lKKhs@@Lz=<ztJ*ZG5r4)_D!*=zhKuMG8xkx
zMJtW=`@=!h52;gbG}oK-Qzv~#-P{OuhSrri{9IoOE3j^VB~vO|wbI-CGu7Stfr)i6
z)G_Pe7^n`FN*A0k>!-IUcY0O|9pM38k`qauc1P$Pyay#4#|Zo~W;j@s^#V{v;+ahw
z8_Y?e=kK+XK@YezUrVlzWX9@-D(FJ1&6hUuBb~|LR((PvM%~Z{L#3rWoNrPtEohk$
zvFa`(>eOJCW}yw*J{6ebFGis{4K3J&mLe1$#^WKcV|b{c4=v+<Py+Fh7_&Rt5W<3=
zJl$2RCE8E~8b-%COOC~K71G={eg+k(ri&jt_97uO{JW6fD@^Y6u(9_OX=dwGB6N)!
zOp%ms?{pK*W%1GUtF1q{qGl+}!>6xlke@nccZIOckaRJiOjbd=M8ED&D+I|SbXhn<
zoTc~fhhKHPZWmMf2IBc>Gkv&XJs*AY6M5HD!CG8Bim15sbFWVLZ`4!z&zb~}Zb>%y
zC{^@%G`2HaXSZyPaD*{?MsDo_GxiVXi^E#F#!V(sJu@v(&(Eo#EGUIz<WX$3i*kni
zb|f8o>LX-8NsRxJY-0IB1I{6-EJyFM-%tunYk{P#_?u;Zp>k&QbGL|Z%e#6Bua>Bz
zFZP*vQ^p26ja&<u+qgb-NB)pPXI^kA4QRIjCxS*i!6Sx4jR`KkY3c2Z=-s>O(<S;G
z)doF-Gl%FR9fb_}h+>o4mk`pv_+ybGyE#aLe3c^c>&<=LO)+IBdie4En$3b_87jj^
z<3KD|Z0;g=tO$aOGqb%X`+>Be#v8tTo($=Q=>H(q6J#?7)@?b@Yw<~Xm&rsMQyAu6
zeJS3_oX<05q3?c#cJ{es4EGS?!u+B>ni~{spPc)}Jg3CwKWEs$+Su|5;j*w}J#|$R
zAwJexeiztLs-dar7B134Z9>>WyY$ZY=aWIx6Vz*vuBcsL$ESU9Z)Yiz;T?iFpnC3P
zF?QNx)wAyd7Ew6Vl4E4OZ&mirmb(0AE!bLBH1eW0H$|P*FRobk<B+cD^&mY}_{8}a
z#}57G=fn}R#(=Gze<a0NE72Q%?Q5SsRnf;uIUyx}-ODAklLU=_-qq2WCqf2jK#^i+
zizFW=`ve|Rf@{my1uM5^8)wMPZN2N_Im<rnGK<K@I(*i4@o%9mESmnQHg!vCFsxP&
zJTqOqc^q#V)w1BWslUvpF_k%#30U@mt;-3LG>efl=X7?7tX@l~Yae>S=ojg<P3o+F
z<<@YC%Jno|f0H!GHyWulA#_<9zcu35odisI*BT0pMN=H-NM{I@>T)1myVxA&!I!7W
zHZ{0-s!2`#4hq!MU#_h`-bD>GLM;{|<Z+Z`FDh$OxuH^*J<A+~qm+hJx-7RO?>YY0
zWoJ49xa?^B-!40qhBqvv>)d#|NED=LPx0ynJcZ^!Mt5-%ka>S&cDCl$m&<|~T8v95
zis4Mz&sp4JoQ9^|GNhK7d2k))r1UMq597}?kH_38Bgi?x=Nqf4u=6FgPi^s{RXT(Y
zdZ!8pUEju<Xqiv*K~&Xel2OKQDJPA!zdtO6Z@pBx+O#t2XFdi>spEf}jGVq#J?s8)
zaht>0cQGqD!%au}xJMTBSG=h-QQYx(pPxG!N_Kn}m@Mua;aK4H+8^{yapx#Ub?UA7
z>ZnJzpVx@JgTFc$+)6B`pmmf3^f@SC87H0Bp1H66kYy~_wQH`<eYTe1t&p8zeqILA
zjQ&`#uLfEDqxDu`UXRgX8FecnQ@)4dbQK&~49t%~)vjeQ{yBk3oWoMYWb@Get-L1P
z^*QTW`+l-kuBUmku}F%>3?~K-?2KD!nWE}0iL#tjWDrY`4Xn<lX3NyU?MzB+r|l=G
z_W?5I>DSB#f=8?<w=9y@aH_?KGCv}b5T;F7sWssvIf*&fEyk;QyCA)(>e)dtEs&#f
z4YJQ>XkG`E@QdbSPs9=5`~LeXvDpz@&Utqq=zHvduK%p2&wf**q>$Bb+T9ibXp=VK
zRmz&vEO>mTtqFCWQSTJx_324Fo4Hy3t^y%Nh%pTK3(kEL95B1YrHS71(~!9%k3Y5$
zY77vygP(;by}4J{5PEgy{k8?8lBe?%S6U`DKDfQks0TfnGlMuFXrINhKH@aGG?4U-
z&K4P<r`_%A<lQA{9NohT#{xc>=`V!-I3Ie*=csIgB<JQc5g!zp!)?5<?BRv&Bdn#$
zu7tWSpbK1%HCWE%65tyPnXbY`K%DV`3`z)6d{kW~Wn&uk#rYpUx9+E*FM4lB8_AZI
zA^Y`FztTBuMFfzjoOgQLHJoA0>1rr9=5Qf})r#aHfEp6_tw3@JZc(eB%1lp1m(Ex$
z?I;J9z3qFa*uQ%2V1c9c-rf2s{h`Xz+CLc5khP<a91q%B&ISj!Ac=2G8fU-t{S|Cz
zF7_9a%JBO!E&D=nkJf!<dRG7OHKII4Tfw%ODl4jGD-VB4$=V=&kE)X-MwEs%v~hI?
z5iphZXnio^pC4S#cETI5CH%m9*UEm@qEyX{Et-GNK0WX&8?d9%VFs@5eP`emXY9}n
zu}aN9)*dH*P4$>FPNH)Eq0xHV8mGw;PnOKgHaA$kJlM)KL<06?`0{k3@ubC*J7G0t
zyVYnl7CG%QwF*EBEAf>h7-bO++a>gQM}G(nXF}NHyuE+br#Z+_BOUvym&;;+BIVM_
z?sLO>>lAD{Dr;5&=~ciwQ8x$FCP8|L6}#&4Tdpmvc4ItS)Rv`P$Rin9UG|l;&?P~g
zQ<9phsDI}?1o0~0e8n*QXVM5*nXQ}x>PQSNBkoe;qC=@48ge;}to!QzKJ;?CxiQAw
z1ayoT7UY<maYR`E;aq#nPJOobR?YA@KXEeep|6TK002qt)FFg#^#vsrlyYZ=yd589
zr-q+KO1TgBkm)D+m7`D1OMWI33lDIpPgY6!8Uuvwgv$_suzmEGux(6J`6<l8rALJ$
za>ImL?&$uJ-Wh<{PNIYtaHraSUz6kj8;@N!$Zxy`P4B<`74ee#ESmC<KylvL^LU-h
z2U(hML>(f_SsOt0KTy#7*BNmsqv>@1%-wYHw-4&_Ai0oM2s=AUIM;G@tt+^@)>=l_
zJ2jCs+;L%OypvQL3b%t^1MSA`E~`sS4tjM9ZxhL6!}lZvV}U6Haci}c*{+w7Up-kU
zyTYxlV`odMwEQ;7>)@?7WMS&Kq@!f+`Ppmu`NI=LBR&a5P<Qp2Wis+7PVhzU0Q(ub
ze#8LfmQ7#J##+0W(h^G00BrSV`igBr1$1WrioZEz*JI2kY87oofQy3#48!uivYvJi
zBpk`lCSZf?2{vN1`Jw2&an^08lz#3pHnT2`iCaFoeQSj641%L`-z$87Mwi@&0h^5O
z59n>UI1qHwwxYw=?V%zwWQ@R!3bMYlUr=pWmR_z-z0=4hStWV8*kj8M#vM3qMIBqc
zF?jNe3z!^WzxrDK4{>i9RaM*f0g?hDtu!1$KuMAAR8*v-ln&``fph3?MM1i|ySqWU
z8xGxFhnju#j^~~C`7kqUX3aWl`N<FLy|3$^zrdP?9`lnGmZwkv-rO;SBT2<(<??Bl
zwa|lijL&nPbv)b+uemagBX=vp?dFN5m>T7M$NJ;DsNH$u##B&&-TjW^*kBs8A(9PN
zcIV(&GcR84bmh}^dN=qMNybu@e1UTEy)ea5U<tA^yY1>*G(h-4n<vvFdDvlNrHt*9
zJhygkj^^;j+@retIr}Cd0c-i1Z(Aj-<fx+-O&p<Y`V+HXSOm^b3u6v;*c$bDG;N%?
zGe>s@F!N1L=I-lRB@d^-vseXYRghBRFgYkO-COpftGe#TGy(8!7Qhl!)~^AlBD(B;
zJCUm%X}kkx`{9-Cz%=!OSrT36-Hux}9H(PA7NwXg+fzN`OFeEm3lA5=xq969@kq|E
z1e!$|z0xD_-AbV;WdOj_{Cd9Xm(rC$b-BeN{WrIl{^@PQEF0R8QkJ@#moeu)g&Q+d
z!yF)w{((z4i%rU&%lx~*MC#B!pYnK;GCZ(Mxkk@_AIib<^wY24O_V;KmeN)W=WvO>
zo{uh|2e3V+UK6_)EWkLzy6tu4o~)}uBzR|hW6%Gq?;@I!?85Va<~jNy<_6fJMJUO=
zi4JtrYWHmR;eA$!S0tPGCjdCr$0jnv@34()bPU_^-rFs+%6q{S=1j_4EcN=5nGOjn
z_-Z#WrmkZ5=)h_(Z7@saqrSuA$)vK7Tm7_-)E$3`%$cUJ>q>szvW|_8`+S`O_0=~g
zvdJ+lxUL`m7B)?$LZa!`N&ladR(SDYT>ZuSAsfv_>ca%8LquFPb89qXJUs0WAlQj-
zunKIbzZ!3Ky-z_8FSI96tL6-qk#BP}xAFL8X4HCcV&f&&2%2(~1Y}$hGX6C&R-;!e
z7^h~->^&s#^jJZ5*MwD1$^U1nPkToY{d-oK*g%u_Zm=!A;fzQk-p#c>n&Fc`PqM@b
z1-C2HkS0q;7JVP#MvW^kY!N)GLuF>?VUFmrLYHHM_u3YlQF(MC{nQEmrX{k_Pg(XM
z!+CC-hP=%9F!JTGNx;4!YUByS(D(g@ysck~$G-rGgC)<x$HZqJM!|NyNoZBkpf5En
zC2v+fe&Y!$0>gqJ=kr<T^VMG~Kely1*~Nj8sTTN+L+f76{d5155$Vdx#O0I%A5EnP
z04C?PUp0IZF#ia9FpM2`8@{-g9T!%fGur3AtgMcN#Z|Rap0HOnea8q2w{badGeZ8f
zLZsd>zwJ2YDAl#%XRRZfwV=s!@BI^;FymB8;3&-AIPG>HchQE;%=Q`y1ILUSyI{sq
zS-`SuC6qQjf=Jk{#nQb{ZucZje_G9PgtbPWoB00X?0SW~N<&{-o#hwB@^Uq0zy^Ua
z)w<}A=jmx{*C3fdO$f07O=qz;YTYb(c}EL7j>w3n<p<lI_U8n9Ej*xr3iv`xy(~ZN
zC|SFe{EPe`^0m?(;T?%jd^&|7FXc`O52&1{>C0%R9H~_wr#ih(ajXSsAFrt3^k+dW
z)sV4NP>R0ajN*Zi$zsSIH#rtJ<3O3rD7WLU^R7<57aKR4N+S+Ka)B5ioa?EDhh`CA
ze9Go~-V{Nx^r=b6yE)z)zqW_?#g`5BYnyZXtc!jX3G)ZcGCA3fJ7Kd^i0&d(>z6d-
z4G;Jpf}<=H49VqC#l(+WszAgjv0y&^aE?bG9=9Hx73dPhUNqimKh>}(J*Ka`puhf%
zQ=atbBJ-f{Vl}I~ldGc<h>^Pw+6rHwyI)v7Ty1n}bYTdi{;p%h6McdfPvp>Xw{-DM
zKTUEo?tHfV(zR)-HF)}5SBhm%kYl$?mt^a(g@+QoDG8@@<c7tAv)kL%NB@G-Wkt*k
z&k=dHc4cCEfP=iM<AC=YzVY$>I+%<S|H#QC)X{9JcKJY{b&<f)R4+{DgEl&+jji2Q
z7chpA!Vk8-H|-v}q=LYTMlG&>^ybgT_<xL~iw-E$-}o-kip{9QFl|}IVh+GULnThL
zD4a`f59f_z_J&q2%4F_``N$ThPGntI*6s~f3LJrLJ1^q}|EnHs-it-97c&C>D|*->
zDbkg}GAR^VL7^gRsru#>6)HhzKkt8REPMvt9w@3;FN;^U?W<1@T$__w+&n86z^St^
z-k=Cm;oq5}<lgT}qIn)rA)yoz?|QR#z8BrbRel>j)mFJ+QWh|YDOYcp1i8rwy>$F6
z$Z85O3d&e27kRHCOT_tQb!Ww2+oTMRaNkquiRSAXB@T*Q^>*H0mK9XonO3fy$4!|g
z*zV9Ed>c4IOH|xnn8$OBUY0I%(z<S5a9I<&J|4ANsXj3@n*|^QPkIy?;Zv?zhv<bu
zUGv~THuV&Xr<?B|NjYM<?8{tl&daq8NJy-hS+q43=?mPMnW-Bulk@%5tGd{?ViP)Z
zQE3A3t)T$El@*!B-c%x)I#GX^mr~~Ca#rB^{tjqn>B8Sf-^H!W(NkyK#G5OhIU;+%
z_NdYJl8L?v7&r;j*g0uD@-$o<pD2Bzx<^cO=z3Xoa)fprb$!k5q^hDezQOHj7^u=W
z`#`fODEa7ME_jk$XVg9B#jIF96H-6G82;?KSMe(TV9YvXrLZp1q=`<T=ICxMh$*Zx
zj6$`0Ldg^*7EzVOEJ3Az4|*}iM6kC=1c@%lpijU7ed-}){WV!)$Z$b!5-uomePCyB
z0x<6kx~u;&mH63!e&`mC`Sv#~%J6)_U#}Ei4tCK#!kR2v_VRBI4ra4l7Cd}`DvU78
zdqV%1Om&+<Q(nqlZ8w0j^H~=72Qmnjo5pLJBV6xa?nZ2Q)uB&?CN9Wdk;^6mUX81~
zER*d}NCNDKq<in1U(0H)zT08s0YtQI)8^!^vSlpB_amzdpEk6t8t5_e1CyjfX*x~v
z^D*sYd|LUFl^>WwN4a8s{R~MnL;c%kd+jFJ2!(M`ogCrzEV{(6_=t6+0!HhvCO(`w
zaFpa|ZNAp-{Po*TV?8ZCA9Z<DSAEfwUQb~}1V8|cR29uc%Jp!_316kpTeo3N#}%70
z!~};4{@imOTI&pBQO3YAoR*>a0mhd$RRkS(aD{79P?k0^C{GXqAO3A^-@7QzqBd_s
zN-xz)b--n|!jEQ1sf)z6=LizU(angD{w$husEx|?F+OEs(i2CA8fY?8Y7MYQa$~kS
zJxf;QrtD5~7>rp4QQd9jR0{;(9IIOk-0qK>w|;NcX0?qU?zq<c_SNTjW%Ov5W+<JD
zap&U8(g!j@u(iRuEfBI8!YHBWso>}94%$F0xH5a!wL3Sw;z^0>#G_Xgd2-ITc<dzo
zc4f<*1f8Gg{sQ`@w5Q4>9^W`!Q62rd_UTY@@DqX06%VCA%JdF0_19go_r=InyxFC;
zo{!GILxgPuC@u!)?%md{)|JDxVy^7wp8Qm%$0eSv(ior;qyaEgM}JM?M+&GSQdnJ!
z{ijqcSN_%laF{(>JX%D$ivlQsXrm82U)ZenphaEJKK$gj68zL11*r>_K~p&lq4fPH
zv%2hyoKG4l^CkgJHxqW|IT?0!I3<1k&;2!V!@WN2ON3)SjVmzvW?s$X?q~OCJ;{B&
zi{3GT?<@e*V*9(v!@e0k=mw4yhbCzlpXqe%tEh0;8<-UqN;uO%!5eZzO(JclSgw!8
z^wZuHx>NJ?KN_n9b>u{jKjfCOO6}u?b<X6+7Y=#Ms}9^<7y!fP<IhrWh^5&^OXc72
zQKXYWPr}2z5ZYiybZo*`vGX?b9uF-$+N%zR1_pMxB0sBBUp+#-fclBnx|qpdU8r_+
z4qEI6H#j}l0`}+;wC>TVoK`b{8rt<<P1l2rTrUV9)a_#s_Kb}4WB0`37wPL+Fh>*g
zQ|Z>PCbQ8@*^=jq^hS0uoHkwnAU<!Jx2f#5@S2qGt5xI8FJYI1elQV;NK|KQoV)6Q
zE?U3RjMZsTavz?8l-sG7%tMq#rN;Zu@>JzF<jSGQEj4zRzk=Rl-kU4~TQYr-`!2(-
z1w*Xp?)k;4HlyStPb%%0Uqo-()Dt+H*W*MktYAsYw?jWgGn0--pQVL|U`ZY&Nx5l#
z3R2tJ2@)+Ed1tW)+mY*DN8np8s1mLK5QK}py=v3g0_MgtaZz3M0IqGn&dM;9j4f&{
zwXBad?qk~Pnn~EKmAj!fgsU_jveZ0j{_AJb%Xz!Ov)%=ed5yvTt+`?RaLpe806EEh
z2hm~~pw1$&m@)Sy!8$~-i77g?c80Q{d|54cDnV%G$&a+T^dRzj>&b@`nFjj}@<jq&
zQ=zwAgfYWy|8CQg`Oyza)D}(B3r<ZFL=`DGH^StEXorfC`va_HFo-k|zbbE@WKe*f
zwG)Z@A7+r&BS`XthuAaa`iMSk5eslbYmQO1f&ZhZebgHRDsl$QGs|=3q_s?;(QG3M
z!I*IG8$nJYwed&u<`i)6n~^`>H?br|HlZ^&pSd0;F2Bd7FMK=tu1mb4c&HzS^q~Oo
zz6tJ5B<n{Bki(loDOD4NDi%!qSnmLdn$&Qz^{aEMN%D<N#s(-u*3z*Z#u<4^PU(94
zKMDgToQ-o?$1!I)c}(_c@=dzcGbl*?9tvrOcXGY))9z9D@IhlnopN1G=4uRppx}=;
zaML`0o$pAJN7TVeOcAA(;xZ6OB$ZQKCfR6BK2)MRiM_}C)yf~`#BwSj<zl7bM8>8n
zCelf$h6)jFq&EXc<HHs^>GPPSr3JJO3S7p=KYgJGx;uNoLOg=y#Y*tt*lmN=0;XMK
zrG^$df?0(&lS7`${oo<}^^Ze3R>{ZRuj?#};|j{=_@^y#e7^z80|btV&m|XfcBcpf
z&>ATBo;dv+l4ZK8%PtP?gCil$nCo6w_Q|?I_NS9yEZ*1KSBXKtrV9+ciLTrgKLWrD
z{uBn3F9QUXF;?^Roi+{geCG`xVn~e@_hED?nOso6kr2YJX09!>8yH7+ee$rvlKKEf
zVT%(gS9d*DOgrKfv&gFJ6oI8O_hNSW8OKwK7fmO=0wJ0Px!)`VEX=O0CNh#SrBvDP
z#amgWE1c((>LzO{TSF4G7-s=|d=~)o4>WKV67><~Rx?R6&-#{~y^iDf)(iayFz|Zz
z_}e6;Xr9bb$t5R(Z?vQjzmCeHw3mK5diCXTtt-`T{PE^&_#0&tb;aFLO?^w}^{7Lb
zTaC^>Sy(kYrA?<<OcT=&41r(qwX->>_xpI)-Lp8eZB|0$ZNVK{c1u5*wXB|B>GT#U
zwz8W*LiE3mEXp~Sdx4{S6Up{S+%FU<F7`UJECY(p^s5dCod{YJ<&t%cVj>;+`0|~8
zaL!Gwp?{-2D-^kbRhvE5CS4^k{?e~qec;RPC=1NEEKBW7iuYertk_6zK}MCRwBm`v
z81&G7D!E)Vv`76sqGG)Nk~$Vn8&X(v-5L^cva)I8s5g1?fk<HUX9F~hY3)1519HXo
z0q)%(#hAwgB8%W<-cC^zHN)u)%*^{mGSD=j8>2Sl_f5Z$wejw>+DxXApopUzZBXRy
zUE}izBkl?a&TIp@!n4J|dGi6j!YrBC7bh)M;4pz5`5gdAYTIO1nq$E8HPJ}3NVkSt
z=q@*2x6`9W<jdg7dACOvW=tPNlX7=T?xlEp+70=f)f?&E;z+iC0x;zko+(k1t;1B{
z=FXU4-yK{;`+lzMCQQTx7ll>p{w8ej9UxoJ<YSoBFiaCXGa)cA*KelsBlG5EPonc~
z6Y9KLxw!?IWu=3m>s^n(`AB<GS&0*)6hdk*l5TWIZ1)D+oXMge#YM!lSTglN>CS;U
zlZ6BNyw^+4dLr`Uxjj^i3gw9>o_;VQP5Yk~#bRL*AEv<z&Kn%+PV+y~fJ&gBK<>2M
z7r)iapytEv?M0I@&phXe5S%R`e-P>_@Nu-Wrbsbd(8;Iaa6R3gv}|{AzAV8(%kJx@
z)4ig|Y)nIAlEJTC`^$q0gG#_AXwS^?K2ld$b#VUb*Fc4f<v)$NFm>~W_20-&nEhn$
zU`8UIaXB&;jywSBtZ%L+&MP7m$FMTJ|EY^3bLuTeRT^l2nVdyONr}ie%CAejcbNi5
zK%FTkX^g2-t|?JmK0?3CEz-zs`N^y1qft8iuDB140s_uSX@m|bgsa$n&;q9Jx8-i~
zld^9<<iF6RXVmrTl?LfTkzOZ5CA1q;$RNa7Y$$744|rtxOcaY-Ul7<@<4FMfh1@~#
z7|#5rKZVfQleV8~8Z~nIDFl)$<=7BOvyAXCjm_0O_OmV8QV!W24i!iMM{Wy4PN}ue
zsE`|AjF9Tnfx$@!KFMW2`!0C0TQ)dTF{u^TaFzS|ivlZuYgXplFY<j7*S#p>?Z*Ww
z#VXR3rKqiK24O2#cUd5F<1ft*ZPFR39XlqNd}&{)wbObwUf$WoaW|MEy;i<5H8pFL
zJh7e|WBe+A+`n10BB4vUz9B8+7~T^u1Y1w*-FxayajdW-YOE&})wx_vLunZPa5eOx
zGc<cCE-9Ry9rsn{{aqe}A3EywIZoVWM4`%HC2E4^My~G_q1IDj6c9z&8VR?dF&{MK
zW)$sII~+Ds=$cw3#wgwhx`(vT`aV|wBz^PFLM5oq`90PJ_%l`ABalG7+D;L(k;5r2
z(hRp1+2jI%LO9>C8l`m;xjBV|+T_bNhzr{tJFtl>r#LHR5a2zd8xpl9c^qb@PE+r8
zC3GvS^S~sX9_j0Yn`oTb#gk>*H&a5slkzzjt94IW7aN`+J-3-T)wY-lm)Hybs{Q>s
zPL=umIeo+iuVL8YbdeN<!0kZfZGYLW#FDpQ*TFoI2$;m_I>taA+PoOd)i)6$m54>j
z&3n^eTk41;m!TQ()GWWDw4m-tZKBpf`Wkkkdt{u%W_l^wID%C6g44X*I(we|n>e9n
zxdK?{9wvRDGUc36%GuG$W3kZAJq4yv7t(2FeR<Ht#k&uF={&;r;|Fur#>bR_xIzmG
z6$>kUH)68zHs|}jmM4yG&xsyCpit2qpnAbFo*TnuS?eQnYY-bs!o|IR$-cXDa6I1^
zIxS|HfHfJ?CC_D6lFm{)MZ;~=nr{7g?BaaYdvMmrTJnQ3(y}ff_>9%{n#D{;oih-J
zka`b%v^9CY+bmWV?^<$%tIK_JT|1Gf$+W`D-uTP=#o0g}s68zsH&y`PNN&)dcP!L>
z(p6NvewW8)Rqq#9VQd=7Ad6kpVzcaI9Z+(u2qA<JCJ5Hh^yCW<))j6{6c5u>_KRHP
zJE}P*JKyz?o6w3FCtz_`aa;7I)-pET@PgtjhHpitEjh+JXz7NSDFf=sC|w*LCEyDu
zVb?APc$t`%B8TF0VjbkE%;n~BJrp1q00rzm*FDcueK4i!4eP0p1#rD50Ig=KhhY)-
zq|#(`ZohWt*Az(bQe9xb+Dxy>X64nMCmL%I#L<%v8Zaj-`0R_j?Z{i#*$MR`b<2e*
zUg%+p{{s?%hB@oD!0c8_>CxS!dqI}-Q4lhkR@5GM<aXXRFh5FkUe67=R8tigl6POE
zh?{brx2=1PeQQ~30qIs%mdT4R_!Nn{TX9E$#JwyH6}cPap1Q`ocGl}SK<`EEUvJ?u
zsw~;#?%#XFsXGFy{1nY*(K~jUcQGx9D0lGp7USj>9<G?*+Ndg1gBrt-r1ADzxC>yf
zMd_i3M(RWKM7!`LK1rCxO0Z@LG?bd<OV&&#%4)CSBJm6a#rxZ}PVu*E-KC9!c7<_=
zOayCo?;4Gt+G<3Ex(3>KbSR<wmGOHw)@`%U@-eC6w?k$Hr+DLOax$Tq<6T)ZG*o#M
z^34j783tlV+AiYdxVz7@k;D*6UchSEr)tWVIJ3YeM+&z$H!Y7*K@A9NOcY!{yj6Jo
zMEj-eME$cZXBeBnIgvqH9#a*s->{E~t|m=Pf&&z^z(iO?0z;=46viHD_OG<n!pL|E
z?TMB-*aAcWpu~R7#%`=xVb8*sCy%oA4FHlt)BEd=u<D~j{zMs*jFpzNoI2&3kz)S5
zXOhV|vKo-%w-}sB{(IeQ^#@)sPN5{}C+Lr*p>HraFB<uqA2f0l2bo&F!+S<}%^7|1
zcRB3|0BD7S@5V7W?U2lhmeGG%phSJt!&5ga!0s%OC18*(QPsZf_QL@QRb?p~ooVk)
zwy1HgnW>f`%TgFxxycJ59rP11>$_`<IB|~8<AhfQTgG3vAN5bFk6G3fTlmTF#*B7`
z-OtnC9{b;Jd_9^tD2eIPGmu^QkF|Cx{PRizWiIJJ&9`JAb=9Ka`uR!U>%W3K?`c3e
zd@NVdUicAs=wubK-*}w5SL#VU_D;A#*Fc62>B>QtK`$ogw2)Z%p+IWM7q&5hd+#*@
z-g~NN$_&{rMa(sRVQK4Ce+TTfM}{-6+FryFu?OglsfHTo_u~Xj_&&oS81X7MJ~m68
zDo00r7n~r+H|2~NV2nf+aJ=n1M74P;-j+DQAad9K4pfV0Zl22qQezEZMt9B$m$lgJ
z<lUytExJ;Ws^|_M>K*tA9?8kv4e1G#mvAtli{)`RTvLOST=_g-<Nh$rV`BkYme`A;
z(oa?;9fT7Nt{0ag^;%z+QFq4kRL?W>N{J^0kj_v6zo)Ti0rrgz!1<LgyhhdjZ^ZLD
zArg!ku#Ks4K&ywFa*sVhN)TYxbJCt*%_KX&wZHCa(e8iga6(7)B7kRCn8RTvX~JZZ
zS{JSJlV3lczndzn*-pO7(VH^=q$HiJQ*u73^`%#U<M|(3;wDmfw{a_=1<;JIQ6&9+
zgE_{R5pPIYzaY|t9S$fS2oKw|C^v&-5%hTx+k3+a6Hq|YLrme@Is_XY^`}SF?y;)Q
z**J2vYfqQDUb!9XMkX^5#_e7LzU||K8}5Kq#A-y*C8!itah-vp0=IKjH=i#}QN-Si
zfl<D0Q{%fR6%i4^%C|RieKfdRD7p6KtBTk+4zxpN%rCmVHJyoRfj%wX_Dxb}&+A}K
zo|pD3AMIYrY4%I-DJ7}Yi1P$1LvDJKK>idd9RtCNC+Tzy?6J}7Ez9+Lt$4DKTXewQ
z4<op-+#X2=W3e4c(P}u=_iS^xP5Aayu?`S`N)S{;pj92$hMV}gAN21g#t;}A^0X%Y
zV|<-0g1%V>*q3b&3LV7&Y(vpKJl@aYnLw|KeC+^BnplbYbWm8fihrVB#mb`BVx%2s
zT-2o9plN0*(iZ#5vUDJ|^5R?uI>1lPHEz6o1*~hkrm=22Cz!vw{_9&WS^?L6uF%Py
zls$_O<q$;R=F)(24cWGsiMAt)`UkwBk@dVOey&pR1=WNcB5AYAK*WBCMLgP7NV+-9
zu0jrZO2MSUa`waf%lNyK2yb@Be$}HX+|#KTv{jND2b08*z|kkQF*pz?UWZBi=meA@
za9jMle_)xsWN`AU4+!a_;>Bj{y@<~uUg1u!$^{GcKNmrNhTreW-WYO*q`=zwrjd>7
zm0?R1EU&Sfc~#UO#J-(}l^cRU3of=X2?0z5HPwEb2+oNwGhzbK?Ti925u{^=jf}u+
zHv*;@mq>aU^Hv{>&X}L9VWQ8e0fFidwdxK6s&J54Vq*W_Dz@2!f4%J!&EuMXiA)}U
z?9rOz;l8(aNNdyTX+X$0$H}9hWDT>1Qd8GINIa3i$HPHL;6*4$7pTyglzp+;5tuB;
zTBsCGvF~wu`gIIM=B%uJRDO&z|0@i1qKs{En!?UP1osxyf7<UabCN1-_qVs8iW=Z8
znDNJ3F!r~%;0md9tp_REAi9otY=GMP)6p9>5A#@FVdnqv7NlRD{M%cw1cvmTP@(VU
zx3?hu-`;|$iC-k$OhUF-fO6SXS^Jz5jqRJ?vMJfWG^Rh>?Mr&t7=@?vBhS~ET&3;@
zUo<qv8QRcho1GpE=rf(JDfu-s_T`%j6jUj4iX1gD?53rFRMxlQc1PBKV#YHYlv6lb
zfzFK<0{h>Vm7MS!Pug0O2oJpHSRRisft1vYJcP>bYjme!O*u45Wb=T6rlGUk8tw3h
zalRmc5Yl$EeX9hL^@eso*TZ7R@Bd0DL}YX2rF9~+lA$O~&ec&6QM8>dh-sgjr{YX3
z{aHW$+@MG~jxbx<i$v@SQ@lU5XRyPon#vnJWGIwI2-zG-i?1Z>8%&x_yECXu?rhjj
zVrux+*=Pj^Tjym!b1Ju{)C;MvGJHyr><~5RcOKy03(V-KGPAg~D`-Dr9Dg&J!fQ_M
zGt0dzIy&eC7&Cq^{*5{OekgOA^Y6|2>*RRQ%Ip0(S?_i`uk8QMgnz|AM5*KfxXWpR
z1b-L71i+A6K=k;!9E;D|{X8J?s*T!%p!p}J?ju!EK|gzDklHDFJR(fv@PY9tcoolG
zEIlT<7x1CLyvmD$a)GvW`#_Oomfq`6Y84I%CJ{mXzEOf#cbnPNy3E@4rFc@>Cts7+
zeStWn*Fpb*l0GLc{gnwc2g@DNH<2APGO>~Hop6Vr<ycnD@N**e2qgqLo6}eE!&vEI
zmTH@ZRHIUIl>O2*ugt}YU@2t2?Elk-(B|!G8thntO$0jaX&564r<0J1sm2!kj(;W#
zB1skh%V&@mDI)-=1^~wN<AcP(>Az3VUkAw=Wg8!!m*`^c7*v6Q+b)qjF&_NN#fTso
zc}SBZ*gPl>T%{3U0V9C7+K3`>Y)YBXxFt1^PSGvY<gR)OKHdxwFu6|$WZ=k&Ndk!V
zv^1HG*IFn^oJrd$QpI1@1<|*Lx8B+1L_rS~WNnS_f^;>^Qr?ubrlTO$j7_%x(AJ+(
zS}j7|c@Uw>f5p+M&3*O*Sp<e8WkCvPS>Se_qP|2S)<XCt!M|I3s0wtm1fYHlGCthX
z<8k5ypGIqQT$l+2T5M+`bjgZe$PI5OQ`&*53wYdit8}@4qWnLPOYVs;E}HZTNZ8^$
z{aM?8m!tpC%*)_u`WNBH4dW32y{n5Bi?<9#XXqvuT-o?ZFSH|th9)YA@KuqxjZ1Va
zwGCSpfr3R`TVHLhjUEZ3@pPFm`!!HJT=hxWN!vk#BalV(ltk>yD)1tMZhi<P-KSjP
zd54?k4r-*gp}B8y%AH=mFx1Lf4Og6HzTQGI+R_O0wlx-;L!kd5qRaA7gq4LelxQ;u
zFdR(mNTwy?iE$&9lSO~gZugH7p&GV*haHE5UH2<DJhN;$ZMb1DZM&|jOuw&6P4=Mg
z5WYm*gxpF=WYU)aZiD|1UeEF}zNg>Dt>6Zch!-J+xVDDhRtnny0=Hy5ddEs07%C{)
zb%^{etQrkU90AhJhXKo@HX*8Lr~zhf2uSQv^tQgl5cMY4WcJULQCN2%+sSA=r+fdW
z6HHcx&qN#Ij{P`w>)+1}lD=jdV*O5PDg>#h<}zx{nCaq6xZzrS;ymlgbaHNhhv4gE
zDNY>NsyvpO`DKo9$+kYNNGh2JYeRV=r@v#~aQ%og;33GB9UdyZBM%TR`rC(8_r_!?
z3j!&OX8`_*`_2ETB_xw-|I1yX^ZRp`+AZVd{xQ4x=WuP4!|xJbx}Bgd{4S9p5aSiT
zT^_M>=S4d0z)1F$LLl-Gjz<GnAWeSKu%XZ!udyUv6k$YhX7NYdV7xP6S4L!G()i++
ztvrY(TyHhF=?iwOlYEE9SeKmOgK>Gt>E`jPs{r46326vyOiC%K@y29$Clwo*+7M63
zzER45ab^E4mbWG&>-Ef}R|PQ(wg~KbHnS4X8@|<W5cmAkai#a$gP{`PxZYNzS)?Y?
zg8CgM`LLGn<8o%U`bT1`(kx)&y#C1&N^>=$Gh#dH(nw^a{+Dim?cVpwKhgIbaytFr
zqVK!EqVHDWmB#;yzB-O@$BJps?fCvr@G8`2JRBX!NQd=FMmOaF-SmP7QVaszH});G
z&1lpQ29RikY<gckYAm0k%WWYhOVubf(c;^AFEZW&Fu}*OUP;<hJMoK{Us``nQn6Fg
zx*1-ZRctEHyu?4oNnwv71e&(9)qJkYoQ`p-bgw)@HXVD#@9MC}#4H*tn8T^Zn9r~q
zMBjPl<Tr|@`vK=g*w(~ywN2^-V}3VtL@I#RPx=hVYt&LMqsqt);34XqP;i7LCusMo
zh9uG-(;c1D0$z2B2mYx#0~g;UHU77g;`8UE_*llu{bd65Px`h-F~kH?bPP-Jz;HUi
zZ7kCU!l<<jCeh$o%@SgIv_14uGkbHGjyG8{Vd_I}`(8XC*NBZV0={0MTD6=iE;dB%
zN9g;XSir>+wbUwh{Re1@glTF)8U4}rl&haLX^?wV&6I)Df;+AXR3cpdn;N4a#t);_
zW7-5?vAz5W3wr*w_96tVy|B>djTxed(4@$pgKREePN)Iof|rM@{xW9{O8DGRG9T$x
zB3lKVmM~mtKw=JvXt3IvOX}PlhatVuV7ap+x<zpg^WHSr6kb1t$PE$J<V)FPjNddO
zfF<z#GW2xoznvNJzs}5NX1~573lmc?6RCh>zD#WUz&h*ysi<k56N6WVXcx8lYQPEF
z73JX6`0XcU4#F$)WGAXe0572iNKoy2=@GAF5qAVyr;=~>c#G6jFVO~&iCK)RVs!+y
zc~Hdd0!g)gHCpSWB_Naw(6VN!I8wbG3atoK#8&I*m2<oaG!hP&$lwFI8}od;x3{*j
zYBx&5YCeY4y<b@aP-)B65cFZc&7@Gx6n>jYVZH$?%kgp1KM<MW%+mAbFI!3bvb(+F
z{fvP}oktRKm1__dupUqLLz_Ujbkz1aw!3XlA$x)8$OdY=hGg=Vi0Rm~Q&zelkKZ{w
za6vSw^1q#?zj8Q=)q>#5wbIhkYJ4C~(ejK%{|7EKOca9_xMzUk5=Z9mZqL^Q3ukiG
z`Uvhtlz2p2eu(tJ7E>5K9+lPSqs`+nC}2et+M(@egO#orwuBEcF3C9CiX_qoq@`^2
zk7;uUe+GZldz=Lmd$MOD3+4n(03#T+aNFnpoh@{Eo+2*&QjI2%v%tKPXS$)tkqEXg
zFxEb3yM&@5xp-26SnwMNpB|!y9OI@*H`emCJYETWyTaKicXmcY^}4YfA52}i#Fi2s
zzAYWjW+ycmKQ{T3*glp4DB=0M8T(|k&qn#4<FJ!2w_|fc5=4<TGMIjM8yT)RkoEu5
z;egkG-%0=U&>!mTP6ohzy}eL9U{P=|r>GtHpRY3`2<}vLHqamM^k=>3wA{-uB<eC;
z=3b4U0VM#Y5+u!B9#RoA?M%&RLc$uWBl}FJ(-?l>Aeq_Zu*S<CDMxes<5W3|Z6<6G
z3>ZRqVzy|kN>Fqxhc5+#yjc6w8c*B5A-}yYRYa>Re!H`xXhv<L`G)_>zBDJXk-iV9
zoOs-587YKZ<5F5Ki5w~5k9W=y&@gixZqZ&wN;wQYZ?ccA+g?iJdL~lWl8tQ>%XH=J
zsU!70PT1|2`k68s8H5#_-i6~XxzpaMAiIa`xlYM2gdnpO^!o^HEl4EQ{O2?O^|mtp
zN?l|7bM+jH@FPSF93g>=F;(b)R6(LSH1OW7!H#1?6CSGlVz6+c0c7hz^bpZkKjFdK
zSAw1h$ZR?;)k~<&>=9UZ2#z-B=?wnW=p*dwt*^-;YA6&p500~i^>^D9llXJlw5OGm
zK1x;`s#3zq(F}>0WDei8PtbFT<sj1;ZT^@LHSMcICM6Ig)Hl83VdQ%;BC7H!gx3f@
zmZurY@!&(?Ie~Kd6xrO9j)En1&UYepu6_G~i5tMGwl$HB#pJ6A7@t-TIg$XF-~65E
zEPP7QQ8uQk@!;a;R%)Px?H$yhLWSy?!kYEiTPeWmu$C1#1HWHX&VpD{)qi_Y{(mC(
z>Hd8E%N6*G0?dtmF*%J*{*Sx11%FW^4NkGn@LUgC=i=08^1;MLXb%S$dcC2^!2)4`
z5q$_7)Q+F4=5~e2u+)2rL$*d>Bf~Vkkz{MfI*}lUWqx^N-H>M8H<06NsQ6Oc;Jqlr
zTR^?@Ux|HvJyT}p#*f2Kdn7=*A2a}o{jaZABWJdlUh_N>@#)BtB(T3kX67p)kk+o?
zq>t8n^%*1gS?=4A;~_@+epF9C{&uSJkSG3OH|<nYlC=t&YLdde9mHhp<tDc;Ld<BR
zCwwf2tt%Xnu;l-*XZU_~J9xClv_<N#cK~n#|9Kt1)4+Srjye^$0XS(NPnNcoi$#Mb
z#`uCb@uCf*dxI}o3c#aBUp@-beH4x0=DAzm8?4DU))h^>*2+@=@?)nIIqj9vs>Vf!
zjMEaq#Q8>Iw?`T2?aMTvQrTMDMO-;<wZi7@$+Uii!^ExyzU>_k`8dVsUnK=T6qS*O
zJ$`eQe&i9+>7!eZ=n;heRPVd=QM)dYd@PzHN}S%$yc+|f_=RU{Vp8PdAZmJ%#Q3qL
z-a%jY+&`+=cn-vhfCQBN)K@^pj-Py)vpL*4=*#zA9EiylVBgo^D{S*`6aG)x_YNEJ
zH#f0ozSh~W)xf~Oh6PCOMwU5j?f=nkd478Zq%b<yozg%!K1+%>6kY>0voDc|u-qV^
zP$CK<yx~H3G}nMWO>UDr>ZKOq_9^LRzf6_#f0o2ckSiBAk;SI27OtT&rL1Zp-$86n
z)71?Ev5pf-(PWnjZhpA)UzY*inbz5GkDn|}gQ+n^>14G5)PZ_c*H+C3LBY*jjZ;`-
zv)E2|#fv=E*WdCxXEY&mf`?B0glZCDi=4AEC;^%sN}2lb*4`2h&H63*N)JfBt^<;<
zA5Z>BzHaElC13jsbccSR0@O4>pyBfW>H9G$AS^vZ!kpDL1DTxMS_qQ~uIszr#TgmJ
z7!{}_n2$QfNj15mU6d(AO7*|&q#*Zn#zVZnjRDjPih6(-3ye*H*R0+>sW}fPT8iYG
z8&}i&V<sX<k|bUQXNfYoiO%>BM!*QXL6QWpDHFO7Jb-iA6NW``6+krU>4MHJX_uM=
zV<K`KSH=SOWI3k*Rkq(yXwCO<xP)T*1BfL5dY(&c8T4eQPmYh%=*E-<sdxh8n<$HX
zuu;6MT5!gXB0_1>t>Uuo$Htvg)ISfA6vKu>-KMGo@JxK2v)wj3<Gh4II|hhbw2x}V
zHhY3~7<hmUs=SQaZu@fEPw5lvdE&I2OPaAsa8ID76c=7_ceDx$EqK~9(}HT~6hI#W
z>(M{mtdCWy-28u>roX4V_!57*Sv?+0%2>-!pFU}X5*V+i!~uUC`BX{XUOAMN{rGs2
zwSuR~RX^&J8rwo=;rzbe=Ecz_&R%`=K%zm2058%B4bt5;2it<Ob|erGf3X15QjV&K
zqPkl69meKtg9~p!nVON4|CLt@08zEd$anl4qaS%#GZakjU1XXrh<f!^{j4PGEX$w0
z<|$CSi&SGkg}O6Kj`QgCVdyOS9g)&zN`gC+E>E(^LzG4A)4E>^+I7kXv+;Mn3Zz$f
z|Lq3DBy}}?P!bhk2_aW>B-P~TiHBNsHyyK*owD}brB2!dn-J8HZ4~9l=DFeGHs!Ho
z!mHHo?+kTAj9Ef*fC7|rBKApmC~2=uaI(Z_eM_1ippZLem%SyzTA45dl7Po>nv{Q`
zbPi4O@2P#R)6mA=f26;^#z~@E)X8wR;B)tbJ3_}6<C%KjejboEL5xi7WtmJuYvJs8
zq#=vir?PeF>C|AJN?$VWa)4h{*+alEwFzo0!w?BC()!jEpbGyWIRwUcG(7dqFXOmO
z4SDWQ&W%a9uFwS5k7yr$d7_d~I=ofs7KfF3G)FWTQaAbd+<tFPG}&%L*?c8*vfObm
zs&u`dY<4WCX$DZ<+XJ9k#a)!H^L|(-n%+STY<Jg+%wCkg*J`59@F6#4_y_XeB2b8J
zc+0VbtiMH|z*9bo2p};UGX0~cOQ=i!ScwQOou?ha=<~dp4_tQJpY!lfr>D7*g8X8d
z^a2$j(14UDMxH7}+@{ZpY=?3ip~EeaI*GAno1Kled6bQtoSoXjxfDv%bJ@O$ykIky
zr_WS4osWyZutY@#(x^SUwYNDPCB6Zin0pJHd$keJQPiF`upjq-vaRV(gXQMAthPV!
z^d;2ZGJX==wC!(M!>2c@Rui-O?U@w+WO?HHC>2WjZyBhbx4<k)jsE}U2+)?t^6qM*
zs&q{ij@UKEWbIbv;JZ3ayPqaXf~4za?CE*F7&W<-$?y5r&v<sFyJ~&CG(WjFX>!9L
z84cunbT6Pr+^9gngYK5$gb+}&>xF2#Ip3$!Cw<dlg{+CYUR*onCKRou7VN?(EW_^f
z-AF8?gd9Is=+E1nQx6qtN(&s+dly*DX5=y>rL1qJ6_qZxG2Mr!INZAA*^1kBe_8hR
zpTU=A)@!=I^G~qt4J|3KFuA|YVX>v$fFDLtUHF$Ij{YzLA6B{{)}<_uU~>x25+Tyr
zbF@Ajyz9X9z@!4%PgOrAt0jr?6K;g!45?D8bv(=Zg1#zoD5eB*2and~7Ja<Q@n%ca
zIOVc3&y~xALG_oIC|JBeT)5-Fa^;Wk%~Bt40#Pk#@V5!x7Z^j5--F){M++xBT|)nO
z_4OSD@m)Tw6!8HBT4I(@nDDfMDUt9vaC;>fM8tipvqJwSi_9xj*sXRN=rptFuobkO
z)3e-QG@?acjuk;6kF#Pghos6W9g_j__#y@;Ui7!yl;78`v}%Z9|G!+j9{9C$cR=b#
z!elk>bNV>0^3fT~TQZVuoO+14e5kc@9J?->b|GTpV$r<}m2fF9Ip(EJtK2zsG@Mv_
z=zcsSKc86~UY0r2F*kw9>aNx!p+3~2yVWZ>6^Y<TKbhUig#DnS$V|CXZ%QbMME|N6
z3u}pTToIu+m?$<@mq(ypy;5UjzwtW%u<CnJsS!&T$h`oN5#ss5Ke4t`FhXNDb$B|e
z>gP^mdq1(;c>U@8WH8c<3mN!$Ps}M-e}B9maO>lDU*5HT#KGWfL$by-ez?bq%z%K$
z;6b7B6#Q|_H%#~j2?Mfa>})57Wvi<wduAXZr+MXTxt_<zNJv&gx6P!YZ`y1Dpu5o4
zn*AHigF56+<R9tZaH(LcN!E<1LOS!4E;*x|$JLa|CosgRB8R<3eNCXT%v_ta%;Jmh
z(h<J;2Y=imA;=;&-^87N+<D5SVgXhn&{T_z3ydHFO>eG9vv%m|GlRyv*y!qw$;YN;
z04>aRv%=gWe$ti5i7xFkWlYU9>xv`Vc%+APtbZ0g|8>~_v|hMC>jeWgJ-hWAo}DE!
zKi)c^bMDwL`~patNg0=$4Lp0ZMShas9;L_sP*Lww`Dk+X3jk*w%tT8F4|KYZ%ff?X
zE&R$ok&pKAgQy<p)fF8VRbQfsTo;|)ZtjyJm)Ok86kI>x6Cb>9bf08(Uh^W|IP|EM
zSXyu$tTOD63czqi2Xv;BW)hr)O%}l?4Hd@2QxhNcooCI<!Yc1zO#V%E?i|2uE>U!)
zNyOoz_U!%dq7@yO*P#cFSnQB^8Wwr1I>BeXiPt;RO#zG{&I^OYln7u{lm!GLZq`%i
z%kJ0AAiZHgUn+nxC60p37sM?oT%&LWTYY_)A{W2SL8yu-5PjTBL~+uB<B97Ne$!0{
zIxBhp+S!|N<@5h5It@N?%w#T|rHV=9x&EN&?c5@mY2aWL`)jthJ7$Tw!8qr>t>dh~
zUOJRg^GYR}w{FY|<@dsA`vP)2{lB(t9<(pm{=RRlt`za}6Wa80oZT#aMdh(9)8((m
zRQ*NER`#!r1p2{eR}9>j-c@Ds+ve5N6BDThJG|uAE6-fdd>IU2eU(7X%O}_wR`E8H
z{?@?eS%&ZA6VC_7?elf#Rj)CR?~(h%oNuhjxSW+jQ$AWZq81!bUQFM(_d(<q&%Uy9
z>(2O)ZX9_)@r>tdKg~g2#{hZS&GmG6l9PFowegb1aZEwLe?Ek=z!oj4)}`SGlW;Ve
zLkV)|{oSv%xXm?S+^CZXudOrO&43T-xA#QKCm4rcaD_<WK_fyv0ty8fdEr+t_K{>g
zBGlg3veQO1OAyNR7jwSg3;ycHyB+;WLC&UfZvHi8$kix*c)Hsu(6#~|x2&k#r|qMu
zQc~p669lzSI)mD@kB-|^zRTBt97Zbt=-KlTGOimB8=Y+ZDYe~2Ozo$kCK%i)clARe
zJyIu#YKQ(=B4JCC@YN=!4lp(<&(UOK)EhQyIQ@j|dORXEa&x*C3>59rMW=unq40DC
zrET&gp6u-nIs?^n&Ee=MpahwB-Q;lk8!GVkRv*}Cjo}U%DKHF?n-jQES|z!i7KYyG
zey$IVGq3UHvQQC1lJS0kIelKYA2)voIKcjLBfZz>R>{(wepor-Le{lx)g)8Z1=?jA
z)vkm$b7n<F0)W$Y>Bl)1{U6Fv71R=b)0k}a(L%dcQxW%*gpD?Lw>Z~(JdI9;&jeT3
z-;k{W%ERUC`ffqD@bW}Oy+D6&hQ*-i4i*Oiq%~c)bE+(0_Ki$z%&Te+;|2waSx*XN
z;?~bQh-d+GJin=8T`t;$;!KB0O}<@}kp;lw*Ycq0KBM7!e?ymY6qC>}A(ZUo$!FP_
zD(xB8>ZJ%?oq046IOqK*>4hY>+*sU-)^8N~T_S}6>?@fe6>Vsa{ZO+RKD$x70>!)v
z{VACP4XZ)76MGrO$9sc?Ogn{j-&pF*^Qw_CcR*aG;2#&+27Q^jL{S=BfPfZXV^nTz
zz|H6+{#`J0X|dQFF~ItdwA)xw&mB(iCRES2z#?VFJvafBJx$5$lnItAYA4hvpl0Fv
z{(jgLPi!H&+K?aD>M`s5o5_3m($N;FL43QVeIz+GYPZuzF6!nbuP16wa*r-DmGd&W
zbz9k%r?QJ^>03=zBBeJ_{rDX*LU^WhxYS*?GPx}^sC3+p3$RE}b)#5s$*9&6oD#oM
zEZgsvG?Z$T+SM**E39xbPfcVkCeh2pPJvIhN?!A(>1`{;y`;M!W#)BO<wpWyn|wHY
z;VIO}Hy*DBpHEO^_Q%oR>8^ef0i&?OG-!ejah}6YC46r>Qr>hOt5Qyj2mSPS`l+!x
zDl(E&1RHAys6D>Pk>T5EQduPvo<*=@>ZI*9U(ta7AIRZ^*psCc0*(O010nVMpLXc`
zLF(XtYllYQ%e{L5OY@3Iy_H^$u}S|Gpw)ZQ&(pMbT7+C0cMntwlH#(NKZi}`D}PPQ
z>zwe$3g4=_LQ~wXSlw%E;Hk(CLOUe-xcYo%#k}LmtV%V5gx{koIARK0$iB<?dbdos
zQ{qq@$vM+%(JmDuwV!)7-gqP4F2bUAEB5Fzmh-`f_9v>3ij5L;s$CTTImH=(ZXr+Y
z_XQn+A>LY3kcOc@deQ&V_G(Qknzk)nI9%ww_H5fhJ2&VS8GC91ThMy;AuRWCnpAid
z<f0*&G1}UGG_mqv`&ueV)xSxvVWGPI-F<FIU17cdUeoJ+r^8UZqUk3^+A2R>6K1Fi
z<00pso}Wv_TdpMlVu~A9dYS-o%zBw^MUu<*Lv`zxxyf$GwXj*$;@uZMSN_eh9Ep1O
zkv*V5A|YA-u6T}BRB)EMwO1vAi4umB?|RW-es^{D!<O}-*93q%H1<YwPviVi8E0PV
zm*6;aS9#cq^$;dXShJPsPn+`|d#U(p-K;|wfVDYXxy+gsv6x=~MzonSM3ax1%5f9I
z1olbE_-xHl)pOr$bl50$k&pIS*6lqu0|qDCgx7NiZ8!%eA~!Es@i$L-lD+il9#Sq?
zRw!S*%ja`|z;0^}8jF#Ycyr}nTFh|HT8Z4XH3a{j-(~p2r@9A;YlGDQ;NcR@#<)7u
z4^s<e@QTiTU~@=s1*E7Z*&aRSFKo4MoPjeRLMCQejZCv_{9YEAHG!znAZp{PTAgDf
z;NelCCNWrYuWfdb3s4J<C`L;286Q#eKte}(N=N#n_`F?fG}^PV*<S^EFwzIoGbdMA
zhxW*bXfoH@A0wePfImHjp|*8k-MgO`B5CD9Et=f7zVEA%B4WJ4?O+v^6gSz5QmPd|
zNtdYo`uHopyr|M+$#>uPSxrz*uh(PN_<rrryB;jmsm_f%yEoYxt@p=V#QO?u2wsPg
zpj8f~v33n@ynPkSjt<#2^E*NVZ{mGH)|_c&yZjma+(>X+FGN^<eEew0E4zy?nU0-s
z0>d>wQZS^&2t4j^?2$yMgPAq$h{=#nXCH{e!s0=1KbVqG)9>+Lexlh19v7+i4hW$|
zCxlNBuy&=Ri|CJ_D^9XBB34%_VU#mj*Xv1L!BG*rHOD3szIIs#qmy~>kK9WSY-{GO
z&{J9B<xTWpr>uC5_GrWpsL14%5I8xE3RRw{_ZkamyQ}0+F-!Ol*hyv+ayXi+19d}9
z+sSxKUm_y1VNb1t$Q3@*i@v-MMOrb|h?JcCHqiR9oKfwi@XttS-NE^1XriMR8HnH5
z9NhLyp>it*<DO5x)?le7vW@GHT&#Jjx$O;b<&>>1*Vjvpht}A+ZU%8I%kDSo?9Wv^
zv_A6nR|}_f;QIOSmvnStufT!*MMRvpZreh%xUa1#;eg`a%q@8dMzYwxO+)f=ARCf=
zo<>dBvI@CZf3v5IJr{-cIla~W9l=O?U|kUIB^Fs!WUjKdqY^34UAIvNsZ@&;1AP$>
zI8347`Ac!x2UxKb`x|KI>-7l9Vn67VG9QKuU+RRIFE-v=ArGO9Zgi1}+Fk5xI#4X3
zT|}8bfd9nVkO@TzTsD5Wc8Wbn7K<67-4?|VUBW%2Bhw!7aBg;WpOc?cMms`{h;t!6
z3FbQ$i}2>nl-o|tz=nlLQ0lP@TjLmlx$Moa4Df|Ti_yYaRupmg$GFrt^%Ke^z*FB2
zLhn5u2|G7OlSDV_;-n_JuHH#gRA&A<5GL<>9K%5D#?Jn<Ly&JtZ1Aa_O+*~*J)`+g
zB`a4L66id{a{{HQnJwU4L!dJr8Gj(sa7LeUU2ROB0HTAyhwd290~9*J&WE4HJHO=S
zIk0yMCQG<>Zh$ul8bvY2zjml357(e?iGYnw5ILzKq<gCtzaRO}ga7-lI3sdBQ3XP5
zaTZH10x5z%LImP`os;T>`&CV@sol6u{7YrE=V!;rOYNCOs=dB;M&wuS6)Uzirz74u
zNLaZI^;d>(H@+i0E&kl|T5}Kih9R+grF-azNEBJK{ySnZqI&c0jKiBHfmcE4E&_B)
zrH2oOLP$ncj@SE<6lGk`$?=qe_#C#qN_$dr1?w4)pfyZBIszG03)_zk$+F}5IC_0B
z_BO?>cq(*ST&}jV8fr<g7-y4fF-%&#ij)n$W{^Lutam={sUP6Am|6W$FvH~Xh-!Fq
zIGBP&!1aQ1RQ<ipZeGGCo5ZIf-coTi6qDd**KEG130I{j9l=BbjlXg@7x9#}PYZ8s
z4k|hB>iDZEts5fbbq$wWv2u^QQ7ba!Ok2){vmbfClWaubL~ue_6*HZ(E*^X*j$l^s
z@#w3f!h;`P))W!%5wM&QobxqmJ}Ez|%_!W+$`JM3ecS`>AT#*&vv>pRA(vU>I^uEb
zgjQd)dr>nL+9{zTDyE_8toV~3<tB=oJCi6w-=7`*Jl&b>4zxSl#@ZS+sx_Zozpn}H
z*$G7;Rr0wypG|<pwoe$@taJ{~eo)RmjX$|+skrGR1G#u~#rtN5VI(ha5ZjtJ_*%q;
zfkYT$II@QCg?0L#AsOQak9%$wDd+WKH*ZDC2~R!=dM4qz+<I^fFl&D1zqr+b-F!rl
z`8ldj!i~(0OdXGinT)X<>Eyl*Tnoh9>uUaDUxH#Jl)Pk|_Xy3j?U=Z22<8Y!D63*<
zf+53Fn7=&u{K#KUtsK&C;^%t_Lq`*Z8}|5cj!BSNR?>X6+kFywzWRUUO_p<=(xr^Y
zRO?jIG@@AIU>jp!X=4j0nM*@ECzr}Ob0nl&x$Zsm473|D?r9URc`@z)d;e5@TLFzF
z!1iKO%<E%;?_0y*r-(YO4l!=am>92uy-%R^&m}NZan5V#r(Oy}Xt2dz8gG$RZbojB
z>2G^Txn6n;1sn|@4<VhQ9T8;HMj`>LIJ-_^dtE*8k~Ilw;5j?Wj5f&Mk-Uke$`Slg
zSDh{srGnof{WQ8z`#Yv8TX&Ci*5~`@_n=Lmk=Q+rKbPn(l;lm~%b2GsYsUvUM6a^s
z;%|9ahYNcQn}Od_%4Z;?1963WUIBFo;qlSx=%94xFV?Q<juWG`^Mbf<&)$A`8zq?0
zws^54J6$&Flz}U?B{E_I#Pf%Q2v{Bu`=W6ibNiM$-0K|mg$7<{$#RXKGs^MTXCoNv
z_~bsO5Re@bCF}~^MgV)fBUZ1x$(%+gctkqZu0SH-7|5(v-a?WV`9c?bbvl|@+wtWV
zmc*rl2pI_>(H!UI$T>Z{SS(_q64kD;X(P};qLOWJF{r@SB3I5GQ!dn~5hJ3s7>JK~
zOtm4=kBp11j`7N$Q2A<(-3;~$na^n@o+C__A@lZ%1FeklQROSsanp1;j@3uusW{}k
zB(H^yWSy{HdoImdIpwf4()$nRC~Y~LOyqm5Un9AEK{n>LTA<ae1w)sOPB*zphmh$-
zLCSgOrxE%Q&lJF@g<BgM_khN&>JvjhMi2juGbBwVuHHnUhSNO(WC?#l=FI{zU2F*l
z1i%;zW%4{%Sp>aM!u@%~4wiG9isGflC5`zWr<0`|+D}f?F`6qt$OW9^f?%BO5%gg+
zvtV_|q{Lxtrb6Ap+~34rrHFESl{l#)d)3=g59Wf?o`gDdm_IcE|9*=R@Z*15M*JBe
zcAGIxYRr>Ou^G|B1G(9g{n{My8RBb%k7(OzsEHmr1moi`H$B1$&ztc?qaBw#v(bd@
zMI&NdLU1(oD6+$&YyxaNgl|WP?307pp~<Mji3M>xQHp4SFtJDHTLgmb5@v-2wwurw
z*M?xaAWHU6!)1=@x~(61F$glWPsM)nV&g_Wq&UsNB(Z<N&4vP#dKAwKlQg53#-i}T
zAukq_7CI#Xl}xE>7NGH62FQ_x;9;V*u;lK9^z&|d>qsEsG&TdJOhCABfNe<sB{7Gv
z2S*4ZLa4OGGH0LVa{5cW)^`6W;G~i>D0q&nZVso}ULCJ3{ftO>74kW`B@b)W7l{pa
z>aejuk;9W)C>LYkr}E(cpz1BdqU^e`;TckLP`W`86a=J1Y5+lz1}UW*B&53rMkEEK
zdqBF8?(Q7AyOGYJ-|K$v=lQ<(J+9;67yoAOwa<0NTA#8NI7%AN{nj}nrAD(Ai+n_!
zSGT{;oR86cIiH!b@9%~;$7>r5HE_87Ipc}U5hIjN*F9RQbJ7Z<dCQ^Iyuheea{Xph
zNwN7TGurh4r#F?QfJr7<saU&-^~OhpiCW!<v)O9trh2wJj96>Y_2&Ic)mP2Nb|&MH
z$HN6DOCHX9QGESxdJ>|V=*_o!5fVlGW|<D~HtU#hPSK_1bgh$4!$?+nT!^kKJgQrv
zUvcU1j0hNcR^T_fbB_!wYVH|<vk0rA|LH!z7?4$TL*e-x`ILRTVy?BL6@6=~l2(h?
z0gZJRP^}teOAglC*W2|UBFCHhUXUc#*y2_z_tK4N@|pep-uo->yGdK8mmi~m(N1}1
zWy0z&s+pt{Kd<Jo6OUtSaOl-NWtlK@X$|FEN3E1{QZcjBvD!_P6g{XmlLU=8b^UJ^
zK#^7x-@^L>f-?v5qO{4L|5Ol$kK8RD%D@@)c=sDOEj_?W;N;^9i>y6Gc4ds2u*#}^
zE$_3>AYM{7DCb~Oy)Uk-@26V(>ermr%Eu;GC*lg2K|rM_)~SDK)4VXlgR>GUw~WU+
zTw%^NpR?&OO!~wM69Wwio;!^QCh2`H@q>3u$=4Vwe~Y%s=R3cTH)o~wM24ozK>Tyu
zk1JP=DGF#{USxnvXMAEveov7{^@}idAuYAEpc2**7+R6}+98J03^u8^*kiEy(~~WN
zFt@vNNLy~wRGZ5ZFpz$M*=;)>M5SF#%Ic^bV@5acB6gKvhAVTpcwVXF+L8l-=CYdf
zMI!}|&?5%@@ifk+mqcrzAAVY76*PP|gIlnx<P=8`P*(-ytR5Myx;rFaZHH2f$f#}E
zJqWGi!9Ea|qY=0^wK$FcEF>TiBep`&<D~`dg=+l+x5ZwVw4j;4+p)SzwIo6_56eq9
z387p(&%hmg825#0E@Rk&d|+grhS)H>7V%KfHN$QToup3-Sp+sc!*-%1?a{Fkt@~Rd
z#2@67B(YROH<TZNVFF}xh1%$dQqPGN4I!v+i2v}Gh~B|axC7fj&X4AexgCrG;?=P}
zt>L+il7B?<!miBx4+As86Seh@fj<ft>MBP>_5=6}(&U&vlh&y~8hF|~$YkOOHh}VE
z=@cK`J%a|FuF&+DX_sgD?6x7K8$gU(RN!E7e4_I~RQ$89(saFvn2udB%*5I%G=kB`
zb^p1-r^ASL!_MG5y&Xs8jR-~Qt)Y|&c-hj!mGlfHMzSyKuzHMrXP+CY_Hs)kh!*dG
zDdaQgr<BDz=LjxY!5yA5IOx%8NU=^M$3FGNG3YgFG=$@@c1q^sUF<{(_RgZW_d=~6
zmgC7?h~)O0ze-YYvxfb*siON1SBAitG=H(ZwcN5%c$Z6zYCe87XB(%%jSCIq%_R-)
zvu_JU=_KsOt`y#`p&w#IRmG^RZg#Vk%WwKKdLwNq4DdjzYc6!rh=7yOjf|%iT>ec%
z;tUs{2i~hA5gy6!cQc4rkrT30LV~`V97G<ViY_m~o`jI*l*_)VqrB@?>fw(?*@thN
zGehyG(MkteaCCUhyK>rZ<b-Qb#?!B^E(+*H#+f-OnQOj13t2N*HzMlU>Ix9*FQY64
zzDKqAN4n!^=548U+aPoM^&w8o5J~*vyXFTK(hp5qoH+_K1k`7uJ$AL)?h~zOX<w&`
z6KA`!YEs97h_|ww_r0OReVNj<9UC$wCdy!>{v3@JVhvKH$tSrw-&{RYV;5kxaGhlI
zBUp-|sz|GOtq0U17W0HCzb~y0|53xQXXLxkdt|Ja)hT)@+KUF67ZIHpX^dZPjk#-Y
z@oUCZ-Mp`CtB2ARjQ68Ycs52*-#|(w=L^PLGqk@FFkT%_6r>8@6yE4QTGFa_On4!B
zTgoEieQX4aVH34)l7dk2N;Tb{w7M**=FzFC{csn6Qu9(U1wataUBHb@EZ>zPnkcxu
zh%nN>Zq?Rv%aW%^v0H;i%-8Z*LRp_1Q2*X0Vg*)nsB2jh#CGQ$toAEPPEno%cM#60
zeMrfjsjB9{{c(4^cUxB@sxOKqgiZ=0_!tYRnm78aZ}}<Kzqb*AOUnAu4+E7_k^~ea
z=KfgrHDuL47!yF5G5<WXd1D(6WdO>|%VKtuq<z&ab@b>p_-&w>lFm;^3q?GLXnorO
z-xJir&(tGc;Y)KbUh@S;>l%?%*Fvp7(C%flQU!)MFOU2<HZZ&9-`G|m)ao$t?8%iC
z>9N)ShV&;OJW4~S(>y^Y-cKm~O=P!jG4e!(ykRt#Bf8V))+Qn+cvx2HdjDG4oUxsn
zb9xnx0f~VF9*HGKdxP5W$Eu}71o^UGAq_?;>7nUxGHlo@2Go1cqv`x&g}1A43dE@d
zO_!e!k!By)5785fDErVA;_nopA+a59&j=muj8#C10%T(CB>txT|2(w7Ep4U|u+P0q
zypdk*2-=wbV>Ftl!lG97)}p!UYD>ewOy4xkcrkLc@C>duKd3FL^5(Wo_uEn}JI>nk
z=y^(e*;Q1K$yiB}9M2r&55BQFsfs&+&6cnu#_}5KSJbZP=7V(X9%V{EJO6me(JcS&
zGtgKEQ={tUm%Oot;I)(RIhS?heMTLw{49PREfn;^Yu0Nobmr?wZ4-${_Ui!q^Gi#M
z`y3{RD*VWCP~5B3%6HJD`q2?179tPTV+!H)f=s0eRk0dnVV7&a+v5DC8hg`8HE4^J
z0=5(7^X7d)Sh|ES?Co*~n)z=C6qo681dLps&Al^~-I~vUvkKFnS+FLQ)#))PQ&5&-
z#+dz(J4l~cxpYyp+Z;bGKk42VWtuXYX$M)(PO3n1C;CC3<Ur<VSM#=XRh9o<*z^Bg
z*xp<;^OH@^S8jMDb~LmCH5vd_!dY#z6#YFY@?&Z@T^#QYx1c@DT!Bt>FZ7{+Hm*M_
zj_wsGaiv{LrqyreJo%}6!eWV@MCHaO?;>)5d0}<eze(<Sq$R!b+`}<ft>AG!Rq)O1
z?o`RIMJm>O@26;xe=g_bvp>f7LbAzUCv-na6ff0ohGvW<Xt<>VKc;uZ>_Pt<`|Xp~
z?V6xSV5h5_$#M(!oK-8Ts*W6^1rtO9LR{<%(d7+;(W&}X6gS4)OEEwG7|$N8w@7;B
zqE-+>UeZ&4Sl29}5;lo}lvFJOWSW&({H10uusIp~E?aNp%4OnHht0y6-l%5e-S&N}
z(=9M{Mmq0PxNNa@0_{?Ev)k!pZ%5DyoE@!kF_>(yD4$;~^3|8oL<6sj<J#7(Zt?^W
zm6fzwww#Bah4(QUCh}GbD(d~bpLGU*8u*0Q@s}Sy(lC@iN9^nQ+&vd=@i<fF54>bl
zoceM4%a;Kisj8wPoF68%+v4x97lTF)q5&;kq1S`ZAlRk2$N(B6eKdVpk&5MCX%dvE
zv-Y<Uj~zhG+Y;FK^37*(RLfUeZvYvetw6Y+pi2Ei?YCb;*F*o|Na*g`u$_V)(&U0@
zL{5nCzH{B`4+y}!n|42hQ!uzseq&<>H`(eNzWunK(fL$tohxrW-Cy(?235pB(eO|n
zp$pvU;~RQblx9Fq<c0c>?}mu;>f8rv{laUKvAteHHLpMEbjKd2Lmn2LBjCt)VIfhc
zl6Z*Hm4>(BlEHCm03M$EfAR2RhQubX8<|KZ>Ghsp&+48G0tr*Nk-amS@{kx}*Q+yT
zGHXA(Fy)or{xxeKv{3xWY&j#yVhMF4Mr!qI?6)a-tqU-(N78=1mw6$e%1MlO&S##z
z-CaKL&wB&5Wa*Fqf#$*+r?`CHv19QpN~_0b-%_fRU5Q0gpDz{OwNh?*2*){69f6`>
zyPQ1vQj2PJnl-VtY*iOwn2+EqnUpTptAA{C;iRQXstQSe_{NUi|3`x5j6&-KKWSFh
zh6NJ~CKYHCiL;8b8ymJaey~%a$!<107W9Hk`{JY}h@Lx?c~ybVtu!wgT$BtkDPgg(
zLpmE{VFU1+ut)jGc#^=zFTTK)<SF~Q57!G$<eHZ99!jZgNykX67u!?L%Ttf)8`5t3
zyR0!h+_ZO3`Ua{cU*Kq~+OOfI!#eruuV)pF@2(_Pn)Hde;f`okd7<8Uis$`~MJ?bm
zQ-plfh^Cqt0e7ZfKgoan3pj)r*R^ZHR+c>az(=%8x!jnGiTo}S1-z&oA>?|CFZABu
zA-27lDjeoaF|+UaLdSl0%1Xyt3Gh7(rHC4Dn4OMa3JIHDA4c|XG-=?Gfd&h|EHHUp
zM_+$W!+n(Ri~1c#YuN*g@fdsR1Lj3>9O6|h_dghgo2bk!S6n;~jy6Lx&zArD3gAnE
z#=Eq@T6+Kox|CGZ?TbdW?RZ}=%QX2gA1<j?XwI5boRaxMbN@Fhq>1f~rn(}>qUTYg
zkS5_BFLwdi6?dz%^z;+UjlRLAjzqIfk}2uVV4aO!wM&tu(Jb`w@P;SmB32%}mXmP?
zx4;VwC}L`N6a$xc-;6<2#M2`sOZZp21=a?Jm~9^uxbupGF_Z|{7*+zPO!}+h>MswM
z8t;;7acus+w4n}DPRP>+#`$x&J+6H08xD|9Vz!|6_i+Ql8u{aV18?l^QFqmz-nU;0
zCjR`VaJt4*wmh`I6*HBEYWbNV{mj6;kkeQ1Y1kftdAr9$ZSv6hy<Mlm$*^q+7*;{9
zMvtJb$oH}$6HZn;@EWup^8jd|^BE@fe5v68DijQLr5GdhiV(<V*XqcDO!LI<U_)R3
zNIuOHF=jYo>=r)ygncbcgz%a^R?-Owhwh1wZ%hyV?9zK1jPsjbjs_3Y?y<SJDO%6k
zJBBU8ZFFhyXADWo^O1Mot~2RCFPMhFM%#u1c;A~|W4Zl(%G~-YBH=nz@im?&3|nNn
zbXffl4G8}fosuvtz5T5=`P4h;su)(i>#l7cE^oOtCRI!j5i_)+0+hHc5X2EDtMOYp
zkvH#p18G3;TO8+;Y<isUSN0rW#bT+6O3JTF2^+9;*DuqS`pEZNrgt0be|YGebx0pQ
zzw{!))!myaK~AD~tfbR)`r<@@TbA#rQuth398Y`O7@vrO@8`$_yjGJP&F8xxyHJm+
z+iy7R9H-LW80&8-hvBD?<VE%#?Lf|^*dIP*ho>BFV_1mVjVJ|?FWt$?AF}@-%5?0E
z%hbHXRS;H1a#h=Ni_@$@6FHVjmUg$V$GE_}E7lN_U-W4D#ku(+?5YPDGQQJEphQH|
z(#vPp3Zb7+Vh-fFjiOQ3CkECpuU4ZpCp>;`*0yh1%*HEjh*bVmk+UlL4kOT<irCU+
zPdB0$u+^@srYgKa5x;;jBZq(TJf8dabUZr#-?Q2SbdF_p_R_y{QYRD+78{?j#SYlf
z)g+~^b^K&AVH>yoOg-tz7{{67()G|uc+e;gpyC9<$|#jQLx5+>Qbg=P6=eemCpIdl
z=I5NQ@X}hzWTtg{>HCuSL(p_8oK&g#>7KeYfV#tw+o12Z*JtkojEb>OX@sZw8G5*C
zmc+B7sQJHnl%KyP5`*s`=k4=uG*UL=PV43EqOYY_nSXsHH5GcJMy49=;Zy6QwDS6)
zOX1hkT9W153Vk+)Z}l6r-)>P^W%{kauop97A8#9Q3r2<OF&06cI3JVhd$g4en*3B$
z9pOD8Xb=W2H{PGkU6|Vmx<1ARQ4J)F=TCf8<J%z$ptE)?sW$99$=enCNvp?ibI(|y
ze#ao>LPw<f`CS%+Qu~jM^{cIg3H>`vq^kywy&q~O2nRISymPu{Hd~nZAEBp3q89|_
ze)I#Xbuyy~`l%qVbdSRgl9!LkG}^5xCe$Z{S+;n1dn!LC1PL30=E+m)(eT$%os4I#
zEX-(Ls5XNWMr>!8=b3mm)cB|$RUswoHQ}_5H1eEABL?eS8`BF$v}3z6jL{pvWsvY3
zh(X~Vi6e7Frh>_u#F0E)&q4_(*BpwMTgknQXiT`tiA>gRu;PhW+Q^=s$v<d=Op@O^
zmMZOz{RmolFDb+v%hD<YjlvST{%td>KL8~TChH(Og+uyw;4C;7%i;<?<7%%414IJd
zvkBHDu)TK6OgsTaujb>?Dg0GF+8~XV?+25o-~W}nY7`8stY&xvv9?!$<&iQ|#X8R^
z#CWH_Z9hKm?<%M+{_xUZx^HttGQ97Q|0yRqZGsU>CiZQDcD<84NbWeGROd<4=f$SK
zH;l=m(BBz61wU^XXRivjvIf_;@BD^^zTMUwKmWE@V}EWsKIB_X@6&~|%8Z2EA1t=J
zWMO|7fL(6sp*XqhWnz20nr<h;=Z%icc@IBp1jn)FvpCuwKHiy_EFTwOMV?V?UZuGY
zb##{PmX{bAKYnI<5wres-bB#L`p@@W9?(quc(#H|t?PXm)`Z;WqhF_g>mh$HPjZ#e
zlA1CEbFFi^1vaZEI4P*r?v=v9eK;L-a~}+-sgbm5KkCHa4-11IaeZ49GoxnNcGH-~
zA?Yyid;JQ=Yyrtd9~2ofll;D)pyd0gnXnTNu{j~wSCj;~C+Mz~jeg<ld3mlbzQF|Q
zLBp(yKm9sUnJ#>Mww74wF=xYFGe;psqfFz;VSDS{d)YkV>-{ZRdOGaiBiGFEUp0q?
zhZU4~>z^8}?S^N;633Am_0uvgz{wS%Qe{YR?zcdod;j(HgMjy$eofs2rf>hoXg@x{
zJ?qx2bqa0A=@H8I$2L_vTy1BV`V|?7`C!BT*&^|a&+^CD2h>>UioSs3b~MjLHLy?*
z{pFW3m^$PvOVsECIa17{Dn-UAiiM}905vcxAQ5|87xDW=wW6`>C&9Sg;(d;JVa}g%
zwTE`McHa-{da9*e1%7Oik1)dY7hN-I&}2#DLMc@QI6&Xr*9b1XlG}L&D=iv}#oy|u
zU(5O9>U1;=Iq$t?u$~Kh^NP;V2v@9JU))CeR~x}<kd<W?_DIH*R_)2Npn?N_abjjq
zj?*oljWK1%5`G7vD%stpG|fug>(nlxo>GHD+4Oe;d~<pr0%!<I?n9g*ni%mb1Y&!$
zW%gevjA4oOD5CWiX$Cx)cxfYfIDMl3Xd#BETtOvYEOHV^gKE2w4~=UpI`rBV2SW~|
zcO5azrRgc{&k<yF4Mto8t*9PTS9hHLY%2>Y5JLTAE0D~SCRHzWA#vHEDO<$ND*b2t
zkvL+Rix8E}<~nX5D@qadqJ4T#g|284pNJj%@@E4%#+xS?4{u8uXcmJsUIm0VCAeN2
z|7O4*3I9h()Commi-#X`b>Xz~JEXpK*pDagLW<LOOlk*corry)4-MkFHaC!J&M)W=
zJQU&Feg|<5RKy@%OS(p8dB2+&{>_RU_(N*0%KR=Kg&ajQ_d8@V?TvhYeR)`N@l}Bt
zn~)}gL#HlR&!Dz{8#*vUHmsw)=$_w@shFyZ&QC$DAB~;^>EZWi`eoy@AXznrwyu1*
z;T|a}3kJi`AU}?tAGTHEmAM_7CN%G;oI{@62jR$`>wRT#1NK(>h4BcQr*%18A>x9c
zv&!UWLg>&82p!%G{VH)T##>uIw(`&HD_0ZV^tu_c7Vut)m($@_3(LDWWGeKL3CR=0
zFmy`YsXdo-{lk;l^TK-3BBPVqXNIA_>Qxj;kjlrOTb4b<0;%Jq)(nyy!#V;i^m5jJ
z$4FP>G`5O97w1%XB~RP&Zf{JcXc0yP-mj!7o<pDLflXtrue7V^rdfaL**tw;P7Vx_
zFQ<>~X*(ptg?&t6bNOZUwd||j>6^ir3DJ@Z%4VQZ|5Up!6e;-C0y_=7Jn>_JdVNg(
z)N}Ux)3L$>zrI(wNW)R{^GMOJWlYSn8o4{;6^Nn!mSY{JrrG>Q6_$TXxW9H-L3Hqs
z#44wboFjQ_=Z04k`c2y7mhh_5DS6Cb8B2v0QsMhCav;BqOu>S<Vscdvu7Y}3n{f2I
zPbDOAKddbhv+KfTXmWcV=X!1V7BSg~U$(a`h_b=nO<{vDF)YR!|B$>q@LnF0cC{HK
zei7J?W<C%4wXJ#x3^o!5me}0-aCy4hNCRRP*-avSaY|!7L=ZxfM)A0`QzQAv6*T~+
z>Dv9MA~hIQqT%&7_a^w%xpMTH$G50kY`V+3==u@_6No~)n~!g@TKbo}9_RTlcg>uE
zmgd%A4lw-v1a&o@JAPK5%U>0?-z4+1O{i*-td2xnA6%V6>}{7)Zz3N||Iw7&;Fw}#
z4Q}r%y@njz6*P+HmF_s8iUT=_i%8YHd4}s|bTN1ubBq3R%M7j(Fm7M_so`%{To`5A
zl20?oGzYU1H<-ZZ1rap=w`73K7AqS;a2?xsE1|onv$<(D!EY7vgWZ<0{+=<84+ZU(
zCIjKm?k@Pg&A*k7>T*abkO&}@u6N$Qy?C6u3i!%j$R|d-bi}Z^MD@p&2pE;4zwQkt
zg{OG9sG5Llo~ggOd}anS1N}aJ&xM&rDN^@yy+2MZV7l?9R9~pHIV6^y`UTFRj}1cD
zt#%-xyvr<iTh~T5_ADw>n)oYX`W>E{m*+@>o5*t~i%XGVQE*zGW@V1PY;NwSQcv&K
zhxQwNc}1(-Lb5F`!IV6vZ;k#3)Uxo27~ldK!?B)KWMUaA#L+%}_+lz*&v!BfzvZtj
z-hE$bYxD5de+0v6nfCfq+l@NGrhH@{mIo*EB)UxtM~_FV$QDJ}`6M_3%gh3g885Sj
zKUZvd>SHHdQdpu_`JR5#kw-q81PcNxE$Vm@JKRB4r$|C4^d+43IvKK;*6o(}ZMj3=
z*B!v|kt{(Ws~z%`XN=gs;ap@Npx4?jZb#Ie_?FHH=)7976rV;2&!$7@jD>vz+oQ7U
z6e35k>Fbioz1qnzplg<0cU+$Dx;`Vp;m3?{2<k`ZaXK6>N!yh}hZ={Aup*7sL5Z~h
zc{cuE@{Bln3&QUU8evjQ)xJ0cG|Um<S40FL3iiIL=%+IwlJUv?Q=GCPL5CjO#tJcC
z>Kew>l>RUa3!dIk?g%x959jluTM&?-KztM`dYX+hnz*J;5BZ~WOD9EoJ@Yo3f>x@z
zdFO`gZbhXx;wjHw3!=&dEZYZcTj_sTMAzlf%d^L%78hEtb_BC`I>aIGl6>Z)tplq3
zk1>~rn^bOuQMEklu?K4I=5(+Re_edo>6;dYz(etUpQVac_~c^`L}LY-D@u@yIY0C8
zA2KvTM_;M;gJD?98HIV}g88{;Dn5=Pl<J2@(g{pi?NYa0Yg9OsQt$kKR{<ND+KStZ
z{l9`=><Ux|%-3#lXwiIN{7x~}OtB+uTzKhBeK(psRx*+s*`RxmDI&Q+rkV=Usdv{=
z8?!~vFBN&Ln;YfnYYW|5P^I%(<tjp(i><8NwN5+{NtG2OFweWvm>_#M-PD^2UaR}q
z^=gN=KeYL-dx5)f3L|f-EV1FpU7CIAqMd_s@rJmY#(`GIC9MO{=9WqgY*5pG7W8nK
zy@h3)zT8N(T+qw!aeU>;oaAD`PkFiIeV#!~zx)VfCAK-wRjg7Ra~dK?+`;8DyVciC
zR_i(PfDL1YVWlyv5`g1-^j68_rb-qfy!Qk8XU52di+y`;F$`_Kt(^TGy5lw(*A~cG
zU31uhU;-4x{<B1*f?@*I^SF1Ko%PqUlFm&q_QyK`ES7T_e@MLh2o|{uIFy${$K=$y
z#AEXC6BZDByQQ7Al*pY{*=R9QmorPPBrx##`bc28a$10S4Tent<J64(`mZ*{y5_OF
zt1U#C=dsnf4o<kA@JnyM?<ZYaYBZZJ)J>`r*vG=>&9Xpe$q5BixW&09kQj);S2}?^
zD|Tm$9%OmX0U|JDAlu%Xk^R)@ZsS1ZeLlzz(?@3Yvy{1=(luMPtH?yZX!5+j5@YYW
zT-4CDdq6P>-yNYSqf8@zWp!H|57zT%O&T`n1=10Cngp)P>=Xl0{G0IgIin@izB#}g
z=kLnkLhj<xkS#m2mGnZF1?-ym<S^d0jC6@4*_F213RHIf?MGN|r6vEyeznLUpFBNK
zroPUUl|^$t6s}!2US9Y0Mv*7m8K*EwKgq|9Io=pZkbRtwHeKdwr`LS)Wa00A`S3N~
zCOR$zrk*C|Ov%;eB?@T_z)bs6WfdV<ePBksKl?MrdhUYJGy276sAzme$*|HyzS_ot
z&TZgg<bU9(n7IjP$r`0SY}IRK3Fd9}(1ioVO2TBfMzFX54hgl(2t+q(IAt*vbK>j4
z67Cy8gMf6-j_pBWB(us2^bx)c*uIc=mR%V>sBHGLmK>XHl`g0)ZlJi(HauW5nR=aJ
zI_cgV8~TVF^@Bao^T;*6i6X=qnXZUhzWoZftH2RQdoC9&4dc8Ju)E%(D3TuE@^eF5
zjuYNB!uRT%&Vg*F53iT<?NiVT6LoF#^nzNJ(OQIi;K|r$lnAe`?}S?zl%BAb9FFeg
zUq~}L<^HKqh)|E6tq2k`6dJYACT+sl@o&fDs%V@omjHA;5Vec_xg2p`ew*tDKmIy4
z>a#`t-gxc|Bk!|Y<*P5gVhVcDXOyaZiD+N&K8868bW)4K_r1FK6P988pyJQX9*xO0
zuG(cegN&RcuC2dxl(hu;9x`i*;k3kJ<)=M_1lurtSO{%EwJZ(h0i#$uwV29XE!C&Z
zAa2L!r4Lug;W~GPAmQg7F>mUhQP<mtdS1@C4827v{7LT2RI8ozt_L4spYr!&AZef;
zAJshU1R%~~$wE&9hG?QAuTp=Z>{kDBjxU0!U}l19yX(`-A$%aVfML4WOq@uW*zqsP
ztWdxOW#%Kw_BASNLe1yMgeJyP4Eg@Plz^BJR49f{P7txs?fe&duYcEr{xvFJq2gwn
zqc`FyJlg9-KIr_>J>KK1TPZc>t2xCyvPkwSinb8P2p1C)kOY>`s5}DON|dLf`|S>L
zn1ENp1Q%xg(d&bAM4<rp4&#u<6d8OSWpLRa8rL5iIXk%MWK5fM7B$8)wEO3!klk5@
z<F`o!$3_f4BI+`+DWe;3<q)g7x>e3QxO<*0<Lpe@M^qfnY2%Xg7Pw7+>MM*k=u=YL
z`tz>#xmR|D=H;oPMt>Wn9)^|Z!I*gDi$6E%PdDLizDiyg%PEqc%hPVr6@SO(ug;Bb
zK6tGU^g3uxVc41fxQ5d#HrUB>q}DSb26dimHP4AF#{=xWQlNvIj<2}_t<W=`uE8l%
z_>0qTI54iOx6L5lyQ&*lnO@qKVc(!P!v!;}>kOx*`!KGg&jnYm!f}yxt7~ptTogHv
zX>lk*ABONK`p#?ae<P%(rz%ooquUs@l*U9n{i(X);em|j%`7fo-$(!5EIz!cPY;C8
zj1M`n<w?V3__v02qa7*7%mtu-bHWiL2vXSa$LV_<F@}h%nc$(VFsSvveCv@-59shc
z;8ES13B`pme+gDSX)D;EVZpw<e_CJlupK({QhAqDI*ySipv-DT?n(u`@zB2xV6_tf
ztHFB*5KCI61dvB9{&h~+Z`CA}o*Q?Cawv4-Zr|zvQUe<tmnl=gVX|+%H}$eOI!iiA
zn~L9C*5;Jg<ASQpNyy29r8lzEi{AH~_baz5n1;iB+g7R05eIYJRsD+UUF`5AhQH_S
zvFFul$o9fmxQD~<OzHJht*y#^lf(CymwX8Eb_3?0ajMx(BC@g%x6PuYL^+9omCyO@
ztieo^?)?p3+Sl1LYXST9ihQy&iO=1^r)OhVSvE^Q!tm#~4Lq`sdZ>}&jn8G@^Z}kh
z0sOHp5*V{?Lk%tsU(3}>7vM)2Kj-eKy&r7J{T2=NlA(OvlH$>m>Xw~$cA%r@yf4Tk
z@nax{m}Mx5xsG1SO9~-eo6&ut4l-Er@Np$c`3t~>i%1a<to_j`Gu|&{9qx4I(hxi`
z9!qup6rqlVvJL*7X*=N@qO3O*KnX1^8IHh2O75s?n*8%NG4z4%MOPoB|KV%_rSx~y
zCSS}y5{%lIP;r>nE+`)T5Oug{qn6=@dLl^!`bL-z=PJcws9F{$ECjili*GxG#@;hL
zgJINduXPd`<rY_EK%D|5UVFVG$Dl{T3DydVH`@u{Fe@XPj!VgoxgaOG;ao;#iU<n$
za=7pivFEfws9VCi$QzSi7nHo8Iv<g2!|~eMxLzLOI4C3ATm2I+<OKJ3Wb=gHEVsrb
z7^{Tw|19lp{m#XF#{>0hO+C#ZaJ1ugYisFFZk3=v9@)D6dgwj2Qw2}v+978rM7{H*
zVW<f7CwmiWGb=&(P}-fO68C=>)fz?MEK%|WVr~5e7Gp<!Td3<T(n9$h9;T1Jmn~q2
zTL=z`-rpKs?JTRC4Je>hZa`Q&<JD9X8%9)92QLfNTgfDJ%v_FhmHb%Q(Jcs7=P+xl
zYvUUYVhiW1A~~Ea(ac}8RJ}7&51Z7&-sp{&s%xCn2?40|s$i-K+X{5LtMz`Ivg5d)
zj_Gq$xcVyB&=xE$!E`@d5F?j3sHDQPJj>Z%=GFAcqwk#6{P&M;(kv#g6VPBm(#)_Q
zxAm6?eap|`>r=$d7LzTPSkl?i2^D(8V#Hc%jbPWHTFjcfsT`4wwh27P=U3btV-vnD
zN0@0~`qO>xf^sa@^G_a5Q4u)`7Hq6m8UDEZX*dPlf2|vk-Pb}I-beiv(A|~8#V8NP
z(P#y5$&Yb6QT+F6Ngs@;e>jpRzMA2YUQ(t#n{6mi#aEwD8(k<V_RHs<w*T5^MsKOJ
zUVU-*7Xx3_gHoUzp^}c3=1aym0Y9#InfWk1G;}U(J&<!U5sW<*@}+TeN+-YXy_8E#
z$oHTpJ?52_#H?Km>h)NS8-Lnrvvfl&1n;v><FcGm4n}R{SbCEO@01|Rlnn#eq^}Yc
z!Rm=S_XOt=)POv+J<*ZpcY1eS<j3xWR*5KctU0x<=&j$UO_N@@Ea(f#LW~@L@iN|P
z@Qr*MRF83S1SK|^+f30s!0SlI?4noAq~tMA+&e`R7waU4b-kWgfO~}Zl%^6pZZOy+
zaDqbf;I}@P=Vapkw0b0gUalZ2o*`Pb$@{T9S(8ETVIS;tV)FyEO8%{0@7;=*m<ZbF
zV5tN5N?O){3)-x=qi>WzKXDetUxPY96sOExTvz{?%+f`=`xk^<&MO72EI7A#2#B--
zy1Xjf(-71=+52<<kxRe31K{ym4;$I>lUgxY^rr0g@G*4pvGlJ|f}sw9Ca=HQ^9@1t
z1a&*J6Kp4^;gBX`Kds#??nrK@*&C{-<U=X3?bw%n`$_yv2CJEoLN?-+cV^sD-g`&h
zGn*UjkM9En0gzepCLY9F(s&W=xUIAxfCt*XznJ4{b$%Z%5-a~C*8@AsvoSeGSf`GJ
z+oS5HiPBrw=MFvi)&iUcj`;CihL`${)zqnU@>}2ap_DIlKEeU$5nr0`%>Z|?;Di(2
z(LnS|+7u5e&+ePeq!g-kkdJ`9@wj)Y>(gow1BRQinu#0;<hQrv^Dc~5s37&%ZMqR1
z)Ke3Bvq!xUz8zjmAg@}j`+LdWtxq<k?fm?;Ak?$*_L5wT$HSRlI*uc2UU^9r{Jo{p
zcDY5mk>FErBqo#DP^iM<UC~qHZz3IhhHP9`FUjaSNnlv_Y#QF;ApEB*b$8c)tu1bH
z;kZYaaqGhfd70>*lIqv(ZrCk9BtZn-APXR)q;PQ{lAUjVhp3z6HrG;D_h0P#DU_*E
zlE-^I^I^*!J#BBnMbG|Lry;khTpf|3Q%|@%Jdk7)O9FF~bN>uVgoqZhf(m{q{V`i=
zS|hB%og%JE2@dsJIfV;cU*mrP$s3yu_x)q`_2WZF(S1iZu39e#T=M!N@TUL`*vd%~
zp0^e&c!Y*ePQG}W@z8TO>D5~6R>Rn7Cq?^b9IW$%>v2T@-Cep(lRo-K><fnNc>7RF
z`lbO?Jh9_f#RS8z)ShmB1cY50I`YfDEnC`rYfuQ+S9G0$a0e`Uuc3rfl~?l7&zb~$
zh9LGsy?ZJL7f(8yZApkFMa=~mD?Ywq8SLPZZi+<rINFhLs)SS=t?i@twgrLcJKWMB
z^r60F+}m7GtC;}&dvRg#asu|0(KRUE|9FQUx(%`K5c8Z~Sr!Cf@3zBWsW{?~yuhuk
zylY5|4%pmumOc6ZX9ORZvK0}!cXg&Pcclm!5sE7{xcRrGi-$P*`(h$eDP}&zo+@su
zx>P|`(Mdmz)|;O;f~_O_yyY1*&9?KD<)@)&)4Xb8)PUn@oLVjJ>4H1Dc@@p{(`T^+
zE{<*JNM6p<CgGH=uSL~*)<4H(_X!R|y57*SNE6rjOz<nJ`ChdZ;D&N0<(`02uQF#O
zt$)aHFKjxV_WSb#Z6hfxVnFx@c3R$vmD)><fxEwBM=j#_Aaf-y^U=hhy1v28OBSJ1
z-P^mTUmD#06f^VGTu3E~nk<mCs;e?siTGc^Ti^ppf~tGlR^6m@ulwhi%ebnNS=}BQ
zq_QphhFM`#PbyQlKOJuc%bXG(SYDoFu>pI2iZI$@<v`Uig**$ATq^X$e9*H$2R86=
zfT(M&5@pgTel>c!21&tA&h3i)bQc+w_G@3uTbUv3Sar%eMBzMZy4>ICA;6}pH{Msp
ziC&A|0FMcoXhLngiSmyWaZ%r=7xwc4%}U3%-3d@k_>@^)IoAuH^gQ$HdyR8&h#g*J
zR*ETK#2llBnY3k4TwteP|66Qd{;0>pM9X|>+CDgqQPm#oihWA^wkf3i{ZK<MqBpqn
z!1CZ{wO+KDuDi@?S4f5~;qz$J@!AeFe>(nqK6kD*+=7vr5AfUdM)Ngxs5s}k!u3Rc
z+Xq&Qh)G%RUf6!q-O0ENyf_I&1N?X2pK<IRB8+y^?hG;nMd9;X-Rszf2%ZvI4VE^(
zn2(JuA0uA-rm;S0VOiw$?!hNm=eh!%drBS-9D5MNq}HcQh2j7<a-Zi8EK>K&u~p_c
zR>#GUOm@Fdaeiump^@_04>h_tQH+Jz^+<H)avauIKVdjLKjD=5`GNHO2=Fgu8a%(m
zD-aXqKePG2TljyfFq2&LXI+m|+GfR_LF%dXeoiv??d}kIJzaLKFJ6$=XDNX@*Odh)
z6TljcZ>J=GHkuW80UZ0=4@V4+!)K!_&Z_R{b9pf&sGGbuZPzkmv2}=n)W@T;;)>$4
z6g>QjqPamsiXHD)%SxA<l6rb8o;!a(e-CH^5$5B$lZ)9y(|V;JZ>Nt<K6KVT`Rns%
z{Q%*T*|o~5d~Y|H%e}@V5g97z{8#O;|Cp^i*$p*ze10OD4@&F-<o40R86|?^Ah~Mg
zjOt98m^d0fP?Qqp%csH8AcEC@EO@%E=^rH-D_;Tnp)nyx-oD3@7yq6}YPF7IG5@?G
zhxWA804~p>w_OoBO&q75Iy+F}s6Z~U9Q!4QUX5Gd|A->Qw=vtduutQmUUf_g$rL&P
zi(Z#Hm^Ig9lym=TM`F3gpa-lqPy+jv70?d24x2}S&0&4r{k4`gnd83Ln=Oyh4x5nk
z?Du0G=q5)n0Wmy$Fmh+Tpx8*(>)xy%`?wUd6^pw|AJNp(4Hl|6%c&EDYIa~p1iTps
z#yJsY)2;`G$G*kt()T=SJ!QUzv$qJDKc@HU!1k5W0^IZn?A0`zE{*o>$26ES@eKmf
z3N%f<QuyV=m=0XpG-pWus9^sSj1lbfbZkn+Cq|=P<K02%^cd9JiO!)0Ri)kW&NMtt
zPXvw!<KO;4f|betq`|qgZ0IJ@DX08hDb_&Bg!CY!ah29vM)q|1|D>YQ?dZm`WPxTq
zPbEj)BmGc4OI@f7)gccG;Ix&M1mnGAkf^_jkyg+Ve7>Jkji=!SC=Ww##0HspV+#07
z1nU6qYEHN^e)`@0wdifsdNx4h$ppw8RC3-K<Q3k##gO3_T>i;)8^5z0^v1guB-JwS
zCL=Xh3c{+;vBu$B=i#ULeiCmEh*&({6E&~ZEK22{h)KE}06Tc2v;@ZZe8rpc55>5+
zK?F^K{IpAw#Y@f)eNO60Gqa*FVUS?DJU|qNn!o;Zij&qlDL1C6;GzpP(ly<nE-JiN
z<@oYtk>da>u7kui4>0ndPwI!R*o5qNDq*tJsn&e<{yJA4^hPQsO8w9&ib<N7O<!7%
zAX=P5v~Z~(Y%TX-m>Gq*CY{}tAvLukb~QRF$=!#6hFS-INrQaSaB%`bKYu@`n~Hhg
zQty0T$6GRqqtl3~(tZCeqK}|bM0t#rcHIb5{i7Rw;p?&`rp5{7j4Hm3nYa+kq!g{o
z9-_)~l3eeC-IH6(Z??0XbC-Q-92w|UX3z*0^>y)yFx9W0w&PVk491)n%x_Ew$Q>|#
z9nBi`v<j)X2>Dre_!mN_ZdjjR7NhrT{biH@Yi>4tAJ%ZX{bK$QB&>SX$iGpmflzOu
zBAIs}o*J}sWv@ESUrhT=;?3IZ(3CE86ZO8#edqdUfW|xR0yB6shSjN0XwI*d%YHuV
z3e>8u_gML#48UTcu%ozj^MpD$_O}d1l_aScJYSan;@fU=37~|q0g4D7zwnxvFKLF-
zem9e9?6q<F4f&r<KDzC#p-_XHe=V-*so=$s`zbIl&J*Uyss6^lmphV}iB4?oel6hC
zUahpi^?|6R9L@&0`SQOkY76@s=L;>T9$AX;2hiAt{f-(IH4H0?RqNub+hHUBnX>{a
z%>R%$mxRk<(?TZ*i}aNk5dO(e#gjvN9g3P_QmMwg&=I`7aj?;HxE&JWl!0T}Tw?@F
zKql;*4}^1B2MdugujI5YRI=Pvu?Vf6a%i}h32PSH?9UbfL0=h2LJ3$*BzGK+%YQuK
z;=CT{c(U#mBXk)~(Suh~q*9X_-G5p|ra1dXT>U_gF4cm<`+giV^HTPX?*Z^<WC1>n
z_HD^73Q_Nd*+{m^fW&4{;`RAbG`ES2<fBngkMYs6fV&5KIKv8lB(pqI?mG!8ZsXM1
zrxrW`(7DN?rC;_NsUv0bNzO9ibSj>^MOysYr*_59(4|Q^l4ad53fVw*SHGiJID2aB
zWY=#Fee61LCcLzZG)sC<hsk3r@)IpxzvsWfBQMs_dchukwip5Z?a*kzR<+nj{c*A3
z%XZgS8#b610y?%H#B#z~I`5F!>~-UPwG8J93!7<e#;mk#PXCgdJ61UJT7_l<D#i;4
z&m1<5=b>l_rX$oX%E6oB2yuVQBp0uAm-IGjY-HaT9ecc)fL7Sm`5)sNU|nOSY-w)&
z*JB0r>x(>8K!C`CRi%5vPL{R?+|lVzCRvKpBng)iFh;I{X~50q?O5@cL|3I5oYP|D
z?CVTjEfm~DUdml=wONoe3T*f7XV4pSag4f_Y5~!nG&uVGh<(@@p4U;sMs_M?lq=3I
zn~HS^SqEzIubphhQCvufpEQLUM{3ZDbh^#SV2ACB)gK~`J|WP4i&^AA$9Hmsv?Tt;
z3jBUK!~rh4Z41r+o2+^!UJ3_*sF_s-1M0P+pb<=w_DC!n{~lDg;>mD%@ssOz83qOM
ze%T{aV8vrp__uO8JT1^D;dpy6Uz_o*f(<B|{m`pyRW|<zt@nhf0xv{1oiBwxT&>#l
z3q4;NxfVTmQ2yY}^;COK{E#3v^d*nQ70?=+Z2h7QD+c1ffL>vH9FScVpIt>8oS{Y?
zTFhB}xOnl^)bHcay|+3ves(fPmti0t^bcwIS*>_ycZ&BDlLF&2>afweX}Y|jo>}Pr
zK;_MB+fIO2lF7JzHT_!#Sl2k8(POcwH(xX<H7{EwXh7DUk6~{z&+D7jW4<xlQS4V4
z1_hsJ{+i0L^11JGYX)Y|34mxp&On_r*Q|t}QC#@0_e9B`4&aixrs1_bdc-e;5o-S9
zILB*g^HbVK-2ikQRRsZ$-G+9foKJ&gpPCW@mrAj9b6=@0vGII^_)ryB?vwJ%`>OYw
zle}9)+Q=3<Rngv9o^ogG(iZVbH3h`4{quGnRiCfF#ol0Gf?i;>5IvexmZ#&A0y<Uk
zM<9|I<rojv33rul5HiDo=d#`Wb=LrGTC4OnWKUS8R3}VyyeYgd2Q4~6sS|s`^OjN7
zffCwSP0auMbhDqn3bo7Eka8+J7m{n3Q8jcDhgRv+c#D^hCi9$yrYiE?b1EP?H_!M%
zb#Kf}h7HbK^H3y_D#UxhW~PvO>XIKuSCR)<!H(Sap%q1^NQ${kgZhfVK)csdfiP)|
zpjM5AlOoJh+%pH_k%q#7|0=&gYH8#_5XPK6tX25=kE%%%({AT!#;%9KyGr>@C(Od8
z-&%Od#{*S>+ay#U0~yaV7|#_0dM!pJT$AU1oGJw&e}&TQ3fGevS(*hVgGl}pPXdAl
zqZ4dODHvMKG$QU@hc&sb2`q}HjeI09&Z&~6(TjQ)dgm(rlesVWFMbDiRaBcvCx5ov
z*jz~ZbXlHgf5<639G?86;f=O-Z@Fb>EL&1<Zvo3w{%dMp5KSxDXG)&z$(CPB6Oh-X
zKS@~=wQFpj%RDa7@KM{H;w-TKUB0go?RS+P%HCfp|M_B%660?NUjVM$){ssZlBy0=
z4ARF`Qkt*%GvVZ{uC^k;C>54$rz`Veo9V}Q)GgV|NG*7`t!~qmsnmBZ!%n5?fYnW)
zghX@bX!_nzxfWKc6{y><c8H#SIg1aZ{l|bkjW78O-F4{y>o?7rE*TRrC}CL545GgY
z8<^TXyl84#2H?VmSVy0_y&gkbo{C{pfI(^u5(ucO2C<`<)eidoa*|z|(QG7yHsb12
z(4RDPfu>A&cdh$nOb@UDY>cdkKj9RnScu1kjI2@b!baG4VM72z7bfRy2g8VS8ot2e
z>y{(|j>IT4KL#T|91352cM2Zgqh8{)I!!_?Z5me|S)WokN0&C<9cye=27M1Bkf~S^
zHlWv_t$Cdm0Jot?NB3i2l?Xppn+h6^!V@9d>Oz?_u61^!kp9iCmnLWJC?H4gvMo+u
z`Ituw^t10>AThnBCP>~qxrf`NFZV8Op*d2o!u*$9iXdMX%VMFU!^?!JJIgB0>XKMW
zz2PGvtSXbX3%N8>(V_?eYO?^#xj&y+UPffWVu#h;*<zr8zA;X#CIuM6M139;!zjnJ
zYPW(pqbh$);ZL;`rahTp6<QHIf6{>g>%u5<t%dMNa{N+3c3P5WO=QLzIJN}hi<MjT
zZ=LL9V{?WBhUtq5)i3o%MHf4GVn$t;l+)O!hxzww&&HS9POv&dX@v)8MtAbv&9A(F
zVmFKNw!YbP=2#>~Ps8M#mUcCtDKjMx-%owfKKcHqXl_=(bmDddS_=uMnre9ULyZ=?
zMXsfvHQ}xEXLq01a+)qSOz%U;$62YH=U-tW23UFlzkPSqhXJyqkDIa&{yee0TL^E;
z(!1@MyXSzfM6jbKJX<!Ss*-ma1EHr%4_^K**@D)^Zt3)ENv+n}=h2oquvu#;R`nkC
zOs+XspU3b6eZYi<Z{)=S_H%)OM}pIvx$f=3n&??$f14&)(e_J$eo~0Q>g5&g=4crB
zBbcIJ50}?P`A_$S$+RgxNlH;t4YMxGJ(~Ha9Ql`U=+5SCVxm^+h@NYasGU-;d7EE&
z1)la8#vtZ(EuM`+tj)hKWAJ}X-K#S<E(0?FwDHFff^`YaA-pD+@&>qUm#w~(n}$>e
z(@HU-XF0^gV7;fd40$Q47`<^5``$u_L0)|4+)%Wr`T$8qS-;U5uJn|T(9r*V2!N`s
z-+Y^AY;^Cu<;h{tZA#JP<vvpM_IAJ6dx?oxhxn?$JAKe(IaNIJ*`M#fX4kNPJQOo&
z|F4GvuGdSijICR|pn|=)4~L11A+T-1NH7PC0!<qhT~G{nVSupjEtB7GqkE!!MLbtP
zbYS;o@wj73cvZ)1F+$3X`~n>B6$C(<<NM)T!tlM3Pv5MEmedVt@1V0==V3Q5!}Y_j
zU^N^=zQ>FK_?8$6?%x!6@43x+<!FhoUz2zEr*EL?!+@yn^e7$P=I>k}LGh7y4;jF*
zHp0?f>Px3N#$ru6_e&z+BBC5ZOXRg`eEPjqQKu@NJkY^8TzCqDwr^xLjrMFM#6E0<
zoB6s1_L>~cd^1&G?SWH%x#g3s3G6?^NNR>cN@n7h6Q0Y)=GvD>dF5da6x<nFyHy@p
z&JM?-YZ-TNyCmrN&}g_-<wvurD-?X2J^dM~2h8X$hfdrCvlYqqOR>|x6y3XN9r;NJ
z!|LME8n(}WG){M{TXPdvKWZ8W{M)u`955+IOS18>BP?&I9Shl@w2axR{vquy=R&$f
z(c?<HkH-yIHj$O|H30|7sMJLdP_SZ#n2CVLN_vUvu0l^0L0MxC8%8k`&w1+9Qda79
zBwUz2_|QFgB}5+G;EvHZAr*Vav)a#{<(0bS@x{>7p!?9+I}gGg=zW9Q245w6#7#M%
zc8W19l4!*!)A1D`m}K_8WD5!}!`U>*bXGd%hIQ0Z8^2D3DV$UEC>uO8)4kl<@0J(s
z-v+B>ny&So%xVo=IyfA3hbND!@yRt4lM^^q@%XWO0yb`2Px9Uf0iAdh8(B$|Ta>6n
zTh7NAIW<2!AKg~Z5-U8ZO~o9E=F8_kE5e%RJ*{h_jT`zt12nWC+b7I2Y2C70MX~RB
znr)4LIq742e5(J^=mqTCn6D=or_=f}VX<0_`{OTKjQ8(%zMDhz9w0MN8W{Xq{bB#j
zkc40E0BPMlNNbx%eeW~3=@4jZ{{HMvfpWy<Y3Jjr`wq84rEe=$Uvt;T(^j!(&C(D_
zk629o|LsQ@V}X7IsI!47H9jMbIvkgmYx?L&eM?d+6UXbb&et>EmkV*3c5BtV^q_rQ
zQo;V8)J<$9Ke(#!5tJ<=^93_{aZ>k}fys?_AH1QBgE^_a0})Ia!l_jerq+TNKed}3
z21x>po4+k^e=5(u`9egdRPQXR*1P8UvA9#=dc|ZJaeF(E$S3=R$#$t=E#brI=HSNO
zOgY5n_|1PE1Ndez5M|AK#D7SL4>`;)9)Q;``-jXZmy@;zwyq}87R=?Ap`rydewl0=
zW(z*qz_CgX;nLa$>*>!X+E2rJG9X73^xQnq>mxj!o{yXHRiWWrgwt(5oI^`KYks?i
z5q6cfQ!V2O7Y8(L!;P|5J77>$GQ?>Rwb#zkCmEpph8KSsJt~d!qIWA^8ibHeBP7Rx
zl6S20aGjx3(~WQRDnGS`P>+Qf0R$^2+O!fmI}QK@#1P-1juZ&bZNr0QBZWN&eCngO
ziYOD$Yt>iX{MVs2LqXp9Q6Fh@#dzk>%p;1C&;Jc5Km(XvO?O7Kxb1%Xa9;f`&*u+r
zV&p@VWXYE-*0BzD@t@Dh19JZQ9rr=p++*Lj%eGcgc9Fc|pZ2M>UKp@F|Ktef&7ca4
zT@Q+~<OfKKIF&es(JZN(!u-fFNwt3Po%0F*Kjx<ZVP-4{wU|-y4$6OR^s8z2>;8{4
zqJQvhUcX!I4k|I>!b$TTmOfN;Hhe>oQH88KwWV91bjvGl__R&RQq`Q2tGm>6i*|Hk
zZhz?{cU3hgWa3IcJe>Q*0G(7$skb9ZkU(kT9o=x&3`j^Hk8$*RVyQR!mxI7K`+5`^
z#7Rtu#GBhH7Kj_62h{oN&bz)aWZ|~yGb=M35qx>Ct(;>RMwTmD^SJ$uL>$MhLTvnm
zLyMJ8eNrs8#nNvFua_5c1}ubv8mb$qWFI!@j#yPW*-0&;#WlVyT}x1@Im6j`+rvd0
zC#qCu?2RWAffypgolw|O`>ro;>yzbylm<zSk8TV<YvMSJs3;3rlOq!It^a(}ZtgY6
zv`)L2ep28$G3Au(^VH>jy=g9@Dv}K$JA8a|u7x=f_*+(3<6brG>v&bR@QuA4Evs;Z
zrt9eui^3_P)43Twl<+|0Y2z&6!~tM|{LatPU;S@NvGd<J#k=FZJE6@z``TtW1~trd
ze>VKw<+j|6v$t3$V12n&#ezEVdoB-t+y7<(oUBK=J=Z9fQ#c{#Tx2lndS%yDWyC`?
zfnQ;xr!$%<U0f~H<}A%*<uqP%OhgC>_(YDBfy69f14;bD2~$HULTa6vx<k{>{~d0`
zg^tmqf3OXN`HC=++v6A;2tYkfLmR|#0!DFw+*j&-^ue~binJZVX`RP8f^w8yNOWIC
z5tN@I(Y4D2SY$S_$k0+}TRFhK=i?DU(@P_SGhi>q-o!#9Lvu(cTtyOIGXSRI4n8Er
zD-xoG0ws9$8DI-I7<C0GRN#ae`3>|YMe$v2JCuC}k+-8hza=(h4)&aG!zRY?KvOpF
z&XfrEnwYQ~tb9lBO%Whi`V8Vlw3Y^;+X6!u3%LmQ-fEz_G28fUDwJRC)1R7pcKPCa
z(Hy;po~1!n1sU6Q)SNNX5=_9nALX~7{0BZBrAiPBms`znC%iUjTN3;NQUXab{?rlp
z&~&peMEJ~Vj9HdV;xFNtAnM(etM;?nE2-(YPSeUK+io`1Bsw7Z|HImQ05#n%Z=j(D
zq>F;ofHVt5KsqEKBBC@!nsksNMWnaTm4H-5=}i&oO`3F&2ucYcy>|$ZKxm=E{h{A?
z&hb0{J9F;L9i5Ds$;8S2-rarn*=P5S_o!JBcy%Lir*^30qgWaTsd|HM#pD|<BWgLr
z<E?9n)qcE6!h=wIP^7*#c<kW-L|fw+(R_->(Hx#Wnf3|hwrGz)G5Zf!>E99Xh)<#}
zC!|WDE{5HD$fK5Z%(W{CrU};c0$=p}aPNyaOQgdkZBZIu7dvh;5FDrh&)cZv@Sn{U
zp~QuQYa?GzjQM5iRb^do;2v1f?eT^4mJFz8J+{n3x@bsxa~-O^DV)MS2PU}<uA9Lz
zJ><)y=Cn1`3qZx{P3fZB7~uGbv)W_mI5kclbj`oiG_o@tFNLU4BB6gILSUAkL$k8e
z?|Vd!gj&S8ikKQQwlPW%KR#ywXqaqn>8hk(EwpklEYs(uN_Tk0h1GbV9b}$w%axJy
zZ04<)GVj^G0D7)G14*+(Ui%i}S11Pcz|mdEn7)XmTJfCnMQ*}OdFQ3#dh%PV*#VY5
znI%eb#t9pUAq#*Hd7B&CQIcEFiqqe+O`RXr9>0H}{Q8^1z{lw99rq)m3&hW9!+>Q#
zJ1)vHW267>vH}vl_kO&J9w$IYR0FC<aFfmwkDKZHnbWHr#CQc*=k*~u5SV*^^QDzW
zVtT3l9m~Uf9f-lml^T@_cNzgAP;boIMD0?YJ#Y74)93Hw2xP0i5s3U6&QZgqjR3y!
zNH<aX6E&etCkuFcBM7v9*sFYhn%vkP#Q0p9%`+z|{Ef&om;=zw!uJBwmcbURG0oJz
z<s7&G#~X7Nlv41^(AgVMx2x9hJpy5d?Pahm?OY~}4{Ho3^|1^4tA%NiD||H4l1&nG
zeUNSVH!!T7I6Lq>?^E7Xas>*e^DV?VMT1KBWme=N-aXlcE4IKg6g~<Sg65FhQw@W+
zrb-sbXOC>!5^!x&PJ*~FqRPEHp=Uc?4z`KdTRF;c&0lPU#TUvmsh@1ywSuEJyaIy|
z$2^}eTzh0wBoLYNuVpVOe7(fo{3Z>Ct&uFdn!L7zs%w+A*L@kcUk6)LA6%?8={k8M
z=B874cs#jo+4%}=%6j7gL$yX^Llmm2@!1=veA%IkfU~SoR%iLVW0w}Mq&s-N0(L&h
zv;HvyPuwIA5a)m|Bk^jDQ=Xa*iWKJ4-2nVUMzZOCDo<GHgl~?jZ0-~l?_8Qrw6aX<
z=<`Z*pGRY)YwnW@u;+LQ6J%UHyse*$XKjejb!57BI$`DU&Vn&&xpX(ZO=KyarK-52
zcT2fm=>hwkcSjPh!CNftX?8G7X#{l>S421mjA~R`(_b~@ZxpZfnv1*XRaL)yOQcrh
z9w3NFiO2S}!t(;zHNW0J0Z3*W-vGkVYq<7>aGE0By_f#y^9(aaNyr!8Dt9jnsk|{?
zQhIGtx&BVUh#k(G4Btz83olKGBba_r47Xb%rRwZ|s?{FzLoTJdi?vaHkbf*^XM<1E
zCx8-U1==U{ti#<=$sHmz^^!1Ky>1_<)c!^;mXylLzEaPpcm~UDfa;{Kb*N`|RyckW
zf8n+7=Gz~{W2!uN(M7}aM$J4wGQ%$3;Hzxg&oC)}`TUvpO?T%dqhns+>t6|`)XozO
zip=R2?pB^Ho@v=EN~7-`CF)9r;ZqXGe?I+G$zJP+B~^Oxj9fJg&|7|*?RZ#mGB$d;
za9{VMw!rQw`3Y&n73!j;w3Qm4oBHRmYwZj9KR<lp)8q$c0vor=y?=wn<}>md0E;E*
zx&G@%hJx|Nh9v|x2Rn9G_P1&Q)u^nukNg+^Z|-Wqy1}@Y6hkoD;j}LKo1!J=-%P_i
z8XqnL;#14B^?kLwHx;IS<GbH05Sv=MfXTSfSN}a3$F<VbqGJjQ5XQA$3jo&zTn(oF
z;mcVgOge3$3ITLbkCEmShJnRUFDv+FPY}9I&iODf!aFp&nv0aog7yaGrzj7I^LhKg
z*e2P!AKOGsM**~8kydNN_NQ&a_8(OB;2B7juQUf7s^`F#sv-(p{lW|Az7Rf2Zw~%R
z;-mnuFE;~}6CrRA@l6=s%OJ4F-TH#+I3p`^nF7R&45T6xXe}qAIWZ#&ipXGTRigMn
z?Q2HKWZ#ptF`(Ja)bNEDhvOB@B<bO}_*aXm*@ImT*V+E=x+P|~6vdd`=u-*uhd(;x
z^1AszgzcOOs$pa+^2pmf8c=;ce8YYB!}WUEcdJ*Aoxfn;ilg3&0K+Z8b~Ns-GEtZq
zuc^*=vF+5_51H!+zvl)yuXp?WbcNGbyDi0KMymrrumo#ZT?R@a2s#bRJKru&@+OpC
zChBdw?!COYyv4d6<7$Vj&dD41S6jE!5mXt?M?TuCS=UzQ`|meP9h$F-RC{mb)jcRL
z|ET)r;TC}&XA^lw4!cXOogc7+0o32j@F3#lJ<0pxLUCPrKfAK*;<;V&I?twku4XO<
z+u5B|Z7pm8yR!G*AN|dc))UYB+C*=B-pZCcZ}P55&|vxERTC_gwb$f3Ud>d}i<0Mg
z++nO!6<FQ<oH0FRnEtDe=Wh3n<z_f9A6Y)tszS6ilFUKb=7%kOGK$t|-iF_!5!z%a
zFXG&%SaLeru{nOtI_clZ<fGdzi5sJ10u-9#`*TtAovUvcj8^V%UWm5!eN?Eh!(8zC
zLeBAZM=0OAm@i_Dcwhg9Zz6lVrh%%Mt-*csiDLG-{2J6FZ6i{NNP~t6K^VMD4Zki`
z(8iPIf2&=`Pl+f1ee_pZD0u;B@H3mbXgjb}IAN_us@<aV{9JAqSIviOv?s6E0joRP
z+_tkLlCH1n`YuJ-i5Kw$yRzz1piVjjX$|j8Jh1U|2#tIAas7cod5s_UEz{gi3ahi5
zG$tNXzKv}|Ars`9<UqZkgy42rAAc#1c+{z&(ksT8_>Q-42$d#=xCsee2Ws3sqFos8
zu1$+-`s{hmzgB)S`k4;BQa#{TmbG@cwG@^OiQi`G^UnETOJ5wnuO)Z=D%+_4BRpm}
zduM9>kw%(#W$4<JrVzNZ`vs!qFc{XqWOjQhEGiVl3}?_0Jh?{=k^I#Bh$I|X>Yw#W
z(j}nII+@e#2z*fO#A`0|S0(vZa0pHRPMg*M5V~bF;vg!mSL^qd`?58auSpd*EV0QQ
zOxJR|{WaMCyGL^Nid+8<O5@t7&L4TxJqV1Q?ZUxl4Cg($qNuSIj_)r9EN}sk1dwA2
z5#&e<A}txfcf;WZX&h*eu)X&o*!jQ|Qd9C?2z};sV~RBJgAfx_PI<H_J3Mf`1HlyQ
zTVDRasEu)hF2Z`nMJ>Xyh(=^M0L)U(5ycrT-t3e?3`ybOeF^1lCw2~+PY(MS^)#A0
zLEJV+1N7#c`5RM3lX3*xyy}G|0x2?)QPZOSY*ve{+r2cNB}DIOf+DNr;Fn3M9Zh;9
z_9JNgV|pQh(wDe^Nt)(h*0*=rm<AZsw}4KGG=%dI@tbW?e@~vKU+-NJ!7ewtzqwsv
zl_cqB&Sbd6rIN()^aZDoNn=6&mxOy+7v(>+Zi%VsBtWmdjo35)LRe)_!9_tVzh(1P
zIV+p+%}4bl=?Ih9($jwEa#cw>=}5v+!pbQ<D9QClUq_7e%(@mePBmS_rvs3^tXfWf
z@l|vzd#M2U?lgGJb0@#oWVS6b+U&))Be0U?<uM3oK-5G$P1)Jx1;3NL3$meHp3!&6
z?F)!|v%X>YnB<*=W-j4?s45}H2esW+YP*h4SP|QRL)M@~aYfr@iJDAg{b`2+uH@x~
zyt#y%zk90Qu~~;2*Gp7l-7DBizzLuZ(758)6mNe?eo9)<raYFVJdd-xUMJ}NcB7mB
z@s9ub>kN${p+(Yt@0}pkZuC>T5L$UwVpZAc5Ug+gra_{#cNzjDAD;&CM>lVYHC409
zcqSc{^lnY%JyyGxUs#y7YdPDjxR3;_zxXktl{Qd9SOE-_C@#9Pm%^z)qYCh6W5E5h
z&R|I3XbU*N0u&H<IiET6T?8!CYS$yL?h@vCZO&kyPGGluVXefQ58o(4@>d0##7u*H
z$TBttMcYaMPUVgLSY6)}nPWh362<lR5Z26J<=q~4$X>q@kQfz8fHh;X_3j71o#WrH
z4}g=-GxN8m;+%@__q;Rtm<~bPK;XHXo-+ZA7i}gO6Iql4z&OxA00)8+Y@!JyJ_77=
zF#NmxB_d@4{~i)xJmFY@6grnFHkYZ9$V4M9$zg3+`6|nSgG(~xpr^Tc4|(@W+#Fbz
z{*=IAm;*Z3oVL-NNhy1NKcd-9lYN09Hk3RFq8b%$dJ7E^5o881XfP}Ii6rh*7zH3e
z`y?coAj?Z=A$rzq?8!6%z@vL86?F2^dq<QSl~^yI{iXDSODzyGd<x?f+^KV4K6>LS
z5zWD5b)X}ygH#pqtEGpaBz<5WZHxZ1|L;-Kr3*>ZiDGtx-+}EZsY0qWK5;7ayzh29
z>ee5*ZbZ@0b`AnWWB=xZ$;(C#pNw9*LkT3$rV5A{cH)7hn=eX<GBtJ@+9$It-INFD
z?71#p{3`x{%Glvb7W?%Df^z{honF1W;a?v?32>k8x_c-E<sc94`{o`m#z(F|G)@?2
zDoF#LJly)>4JuJTV;dJ3w7m}=t!KJPtWWS~>b~f5xE)mzsoi!D=qpLTE?Lyh8$}sv
zlasyhqMLr`gY=Q+BktkXxoE+Ve-!gQ{V)38D)Vb+C@Ghle|yc<%>k6#qalGN-Sv0}
zPLf9zOeQZJDoly|CkGJlWdtEjBQ-X&(yTp2Ah>(^jGl`=$DrxGn3er;16)xKeHI`s
zvAX`?J<|12?MUsW^T<PLJH76W&%PZ(s7nR{J1l)Fy7f*8yD~UAi*Z8zsv{Q1*P?;|
z4Wgh;O^;LL3}BQS7}U%G(X*4-nq+%jG%EcT?+=1Vi!W#A&w6%r31rk+Ub4W3TMNWz
zZRsHB4hh-4uD?G|Bd?LOIjdG2=cAq{Q7})iy)mf3ld)Dv+EH<R*mf|Mu)=YmA`sZt
z_NCjU^=_^Ii5jT~Tg!W879iJL0dmb=Hs~fGGw9?J`RX<=6hK-7Y<PR~gKKGJ<u=_u
zHP8!u?RdTHHv{tWFBh4yKbBcSJ6Equ2?Q)odHG@U{=Itm>|eL85_x|5SG#LkE+24@
z+*}d2+iadUsQ5v~pJ!5Gf+Flq7XofrooWn7?3fWT1@NL3qvBo&lSxZ*WU`9jrfKj<
z=`TXARjQ?_`x%ZaeT>^o@xmDt{wa)mJzP-=ZmCk7Pz)!R0wdTwgXFcc?{`V-QJ=P@
zI&`p0h;cc&y7{Xw@`CIZE<X+VAFnu<CJ=eWSiH9SARNL9qBk_%eqZyLMufG5(}rZr
z6h1IO9x`73lGTEC`)M~lG4v>i1L0^$w%wdy!--&Xz0un2m0wi(-X9%Ao4)l@<2%7+
zkvT0!0fIX{N+e1AfYt}#|NgD7oZ|-Eq{42fJ(P|sMd<$NJS9Qs_e5y?M~&CU=CJek
z9ot-1HlIp4<4X>}M5j+)Xq0rt3KxG=iM!)CSH9kFT<zOd_UiWDV4w=<<BvDe;YmK~
z^@SadeEzWj5xDRA_@>W;rv@iP>7TzR0J5b410JPMyCw6b7T@CPt}Tu9EyVP#$YLey
zj7m~}w(thL*|ciEzN~GTVz82uUgvo$%1DyoQT40k6w|Sf*(RP3&p}^n=mzfZBPME`
z$@4m7DZf_U=1KFp<-F7_xTJ0p127UX>HgzvpAJHct3kxuUy?k8D=O50k;3`u$xLs9
zkG_qp+ga=(j5)qRFE+F>Uzsnw(F10IEDw$MpB)e0cxO;YMD${#^ZvK~L6s41%cS}9
z8rri2&n>6Yf<If0wB8+}=1}{=7_RcEV41{Vb=vAe2HT^9=0VNZ57v&Ph~-1e^UD1`
zzm0A$WT{#hPo($^=!KlOr^CK?4phHtkjq<_)&Cqbd5d2=>B1XHmCM!M3rVB*R^E<2
zwn;U7Fs7w+`%I~L7`n)CPQ-BQyU(lsAH+lJVO-e@M<d}6k^t#<@)eS@_TSN6;V&vZ
zCN<%-$Mvi_0!X$agnErp-mlrW-<1xqgEGbnV6;_fe18dunNyQ8j_84PKaVn`J^?cv
z9L5BapzYqEVZ!S;gmeJOGzE<C*@gRK2_iu{(-iO^i;Rv-yy}78xURjJ<c$qNiiW!p
zfx_DXY5{VXD{~1aO!OAp3O2%@<ls(`<!uMm#-qL9R-oyWbR}4Bky7e0Itbw;k5nO^
zMf%)IxdQ(X9VS9>m-<aRw^Ogp`wn8~8RB@;_bJy8`w`SnLJ4Fd5-+tr;8%+ghF&UH
z?5I`%`%O;~mzh$|KE82>wG|OzQIv>};Ux9VBnbq6pmgE78v?YlU=eD^NZXWB&G)yq
zP-$}cHF3AZh<?dLPLG)2%USUs)sn?)lA3f)e0hMCN2Oaz%jxM8jsB!0WejhrFUs7p
zd*4i&5jJ?a;PoGtJF{SuV&|{vq(x5NiIC0j0RYgnpvc&`8~#_f{CCg%?DdF<i9??B
zJrHFek_5uEBHuwb2m0Sc<r6hp@ktU(f@p7$H(dZl%YUG>nB@+j2w;EGL>>?=e1KHS
zDPrd$-HU07YG}@kke%g|)VT&sN0|Z5>R_tJ<_w`P<tdoVdxU&5Ea9WU;1uG;AJut9
zYkd-P3XHN*?={%UCFQpx^`@JtYkGMl$=*tO?KKlu>y;;52@O}6GGsmYM75o86a7<S
zNyV2pKt-M+NN6)BpJaswK5YtbwUpg^K?!0g@8O{laCqOnmzTdELC$m#Gxy{xV;Hov
zI%z`^xDeZ{H5-OhhCpX13?waeF4`;(pSyXDo>TQNE32r+@<vj!N^0UUBV523yITYI
z@s?<r=-G<P|Nav?@;f%0ix&?jpE5nKHhernXmiw^I8gqNPw)ANm=fId*D!{B8UpCC
zeoZ?-grJkb=^h|SFij?ZiJlR}589i5|4qSQz^&;fS}}f)0HD1D9MG|n#Ct>vOQ$0~
z^d&O+oAy2goG)k>AB4!ieLzc_eUC2d!^Jk~c!g5)<Kyzd(#Jdbj}a@n4yljrv{|nO
zD~3z<lV@GK@~ZOMwR52%#5RPlz3zUUZbU6DjO(wf$JA*JYkXOo)6?IrS#pzmx-xla
z?uax-(uYqptS##34#nyrAEQx`-6t4)e6&J=xcdu9Zw~w;>#9aChto>Z(v-1kK+VA~
zJ%i+JRw($Ui5?x}Bmje|6P&c!(1iDjYMd7L;+xBAk8=@46L36!%qYS|eKxmCv1gH}
z8KF)E5f?{7F{q4ZY}hw0IWJ8mAS}nt-(`KyTSV>7u-I_)q%aezDSc?0O^)O2iD*?E
z_g*dRs2DOHJ09<`QGI(gNS>CAP=SMtIFN&u_}81h=zwA+#f(Hhdo3p{tzL<#0^=pZ
zuF;uyIe&lR*T)kTIf#=ITilJb*%b)8cy`ukZ3efoBF_nFr3m1`;e?D$FV*~l{JxwU
z9iE}>34XkJY1^U?A=n)xi9A7}nIK{p+>Y>V2<oapxD?GG2iPdR8YzX4N(tMwr*cO=
zUO~0!M=2$*9^*9h?9Z$dXjb>!M;Pp(_ppafK8E%u;cYgC>QSj!Yig){D9uaJ8fzbV
z3k`b*4R+pJM&{e^>F*1;Y&eOxo**k;Fm?~fdVU{ly3<fLIt5J&dwNLk=<j2eY%o-`
zPoDkiR`)jv0mY2v{ou~MKTPrK4Lj$wT`#ST_jZqn?LTcL8YEA4Q<QL3T0q*IILUgj
z!{jj|YzDb>;)fS{@KQ8=*ChlDG9d_*X!_drRWM!wQjMc2F+do7xTg(<vGA-DBnzYO
zj-`*138Uvn(#s4<j`&xZ!y~;y6^}-)%;i`{NJCu;?iU@0D(<RMN}C(mqsba{)+1u~
zgIl#Og3F`gD=Xt|J<RKG?xR5;sXl}<wni6swK*O=_wUA#d9{|g+`$?2H4HD`XZ@-;
zh)6B-JJ4$JLzZIlpZR-iB79RyuRHiJl?w^)N#xq?-pi1(1loVYm*te2e%WK0sB`%L
zu*c=!_SnOTIRDcgeXjs}^qpIpcxaC#yLt4YZ?>kWIJ|q0r@e_WfHA-{*b#Ip4@1&>
z5zSZUNl#0fmPNry+%ch0S1e|eJEz<BaL$8z$<D{3tmrt>nh)}Ht_Q0jc>Lqt{XiO-
zeSo%l-L?tUHp1GoZ!EGP9Pi!Q%6-L%`yBK`e*&5^=IJWb%Id&s{esoj%S%u#Wv7sH
z`D2ZAj4#<@85c5Bi^eFRm+lx-IMw)Q-h8(KJsy~DeM@=t0NpyPRn=HJI)YV@JDx-^
z4<D0fYi-i2%l|alQ0oA$Ry(8=_4?;eRmgB8+k1SIIH^Idx|8L%>-@`DO|D$Dgk7#&
zHEXMpmjV|xt`o)&xM>-|2WcRIFHvu<5DI~0o-lm#l)8W&@mxi7#Y;aY)9a7*>1oYl
zv6J)Wbfyn2gW>TQxlnD15N|w!A}}WWOQu%xaNKDiB-yF|ncee2jfgTGH;&;HJJXb$
zTY<T+#OgY6)BO^pY_JSddNM&w;zU-uQB2PU#d+zVR58_?%}c7tQIlkv5Ep}3(Vv66
zoWOxQ5&UwZ;QvBD!qKkcIAy4|T87W9Q;Y#(r*7oK3P|HQdmeEd@HaeVB2-IHH@Bks
zGg1IQcbkI1Bs@gEuXri$Y4ATnNrE5oEg+1j4!M1!<Lbdt-K1EpsTTC4gQJR@!1BX9
zxg}iZO-3)ddQpysWqqPp3*ct8Pg$_gV_~^P<?4QD5HDJv!YEU7M2a1_ne+Y`>~Kt3
zMoxdwlrLlyowji2QD4)u2(gWditnY-J7I}$H9hlYoBEw?NVimdxAC8cM`uXDwWWsV
zWu`D+i5w60ru4dzhngc&TxjIy5E2%r^Nyin@ioJVSvzChdMNaoHrr5#q_xdb3F9tz
zx{vK##Keg<+jNQk01zT#x9HWI{{K1+39p*I*68$;&;)iL&48P{rd|B^Ku3nLM}$Yf
z;bpibdwIRNOf1q;f^-5pinjFkq;WX)W}R=h$RNE1756w~QI06Zs>|-)j6RvSg)I6{
zXMPHQ>|kOS8+}Q_o%yiu((Yi@p!Cb^`=el0BT}Nb_!(2KqIY=by)jP(9%W6$R^iK3
zxKBydJmbpcMIR3o3N#HZ7f+WOt?jqq8=MY+aCTQ)>!vnz&a@z{8}r;tQxbzOZDH4C
z2R|~1R2A*-lO|Dz8=uacWt8m$VXM7A^1`n-#k9Bz8?u}A-u6u5|B+gL`3I9ckT{q?
zZfx<a!h}>)@?~EH{l1bH1w6UvDoGHdJWui6EP#b0U7kgFSTeJ$g~_ISZ{u}0GlMx#
zen#IE+>Uxl2I1~gGM1iNv(5cXue>*0scwH>7*SR-R+bm+W6H>AF?!Okgb{Op{BGUr
z*5UI~a-JzVRVq>s8+Ql8{%~!#42qAnPOT+e2W{&4pv|$z#3O7m90{Jqm4X$Zx2X$;
zsKRKr(vTXD%R?uFAEUXM-0fn+i}aGqgoit`(}<Z4%WCsR$&V!p?S}4U?9ad9x!+o_
zUlvcc_sJ6Y$#6K6^gn+xN6&p?xB!rdg&-y&)c<fIQ)W7{Kr80vy5xI6tY~%2&^Mk5
z00%S?nL6osAT$Y;K}EqE7#ulw@}r6^y5Q7QxF<0(2|*=HU0X*4cSt!7d$(#TTkAV7
zN$Y52N2kbdeNe)+SR1XcJ5eu3uNXa(G>mSWoQzFFetuk!H8F~%Z%#7A>oE|yTz#of
z7#?*)l7jA$c~9gwWy&SIP|(hDb8s224~32#^$TWx=ix}jDp-Ag{^+-vHg$g&$-Z!&
z>a4tAmoYFKr*iWh0sB_(PcZ2Z=`LgGWyhSTVIsn=paXh!z6HLNYv<({rphS;Kef>a
zXM+EQFa8da-XXtm#_~w80ZrS4RYzp|NH%7oe&4NcbF5=WV%%78uWNI2aaZVSyh*cv
z4iovGb`uRhOTFRiH962A;O4iTNjMSJSL@@tFEcKFuI`?M{9&TQ3)D?cra$Q9l_OpY
znaX`38O;f#9eX7_Ji}9djQu7kp}-<G`gp!KocUbEg7mNnL}sJa^Aq&`3F3HO{kaZ$
z-6-{&AF`)!E9ro~Z18f7U1~{)Ji;>mD6lB?f>2N+$we7)f?0F)Fs{1pxj~<v`z!`#
z*jOXY+1eP;YP%JvmwxhUOETE~u(Eb1GRye-UM^xv=GLp`kV{i|>pQ4<y{bG*V<R|G
zSMEM3^EA5n*8w{A4M~*!?*k<9`v6U)jUPy?ugP=^TEktt2L$o82T{XwY5q?810OEt
z@0*kTX{5fevwR%3IH6?+<m2wVEBpnkAu>HNvV?pAJm~}yM<5gc*pAR?GvM-5%fLE9
z7n~O4^=bf>KMTcFK-qm`LK*RcTwS-G^t#Fhu`uN!4ME@1va#&EU`aT?6;}vrTt#PJ
z>*Qc@TeIOj$70S{ri=O_O_Dl74-r@w6aT3pcy{YNVS5zBncVM$@a|0lT4c?MQi74g
ziBiiHJ{!6*Z(D4;Yl<70y5!Dqo8d*xcH5%yfME1V=y7T^6#oJGwJ2S5r+h>UW^H?U
zyF{pVc6^&Nu0oloC*?e0B<+sN?yvm3e<b4e-}yQEcYfZ`a15pr=O264s5tVxrL@Yv
zsBtaHc}q<qil!!(zFG!bXE)-!>d*n~me-oPI+|g)Gj_dQyQAoeu1Q6~xO@J()TBzF
zPmb4CmpH#8e51ZSZcF@H?S!4Xeo4OL>vx;s;qmc&^|+y3%_q*-KSH6~OThZ8=av&!
zMFDh}xvD0VYiDVM1ML+PdcMBu^pPjt)S_9NgIyhDElDtcKD?wud0I6BiWw}Wfl7h<
zmH0vq9E?_bAZd3xR5+rPd{<_)N@Bw`V8Zw|$5XQtu+v~xNa-eQTYHE@ou!@@)jxyj
z9PZDTmMU$Y|J?SOIPft)sm0aNk!s=(YMK?LTHVaZLyi(32FjJ`Dr#%w31(hj(I+`O
znyr!MbKb;=erey3RDqelyPdXMjGg~%E-VEBF>C8KQD1tJ%`1gkU`^H@TifAFQAcS=
zYIA%4#s$$3^E>|sE};B{3q<kL;-fa1pG~`cJD`UUL-9$^i-B`TTPL-Kd*NGao6FA{
z7IYS9sC}M{*iSOpMbt0Ey;~SFIW@07sj(YEM>}046!IIyFMMlrm|}Wn?BpYb4r?@A
zh+rW8qi(SR1=8ABf%@GHor%eBhF>F=7~&}oOeHBMcnuDqdVTkDc9SH0v8Lm?%~lWH
zR*r^%^EAuzJ8h0Mm|@9eL{z!8%fzxH)h2de@I9&Z5y}n<C6@7943Jf<>yVO)UCLYG
zAWelMWxFi~RgP5NFVn8xkYrS<BbNC|n9&tJy(=;Zy72vSD0FUQqe?7tEBT3}43Ttc
z$}xqx0P6d5emS19X}dYBP~uDG;ZQE+iTWf`;!7LNZJVkQzIE{)z3LG7yu-txpaZI#
zd`m1g|4Z0hUnWk{XK^&dkNcc{9`XY-+`ZZIS)4e@E7|cu6da?4(9)HeavyV8kU8C1
zI4HcSTeKK%c9GD)q4a^&uIQmFsbA~U^tP17b|!&8Lro`$kh7-~l(YI}K9tTTDu2l2
z5m3+&FM*7Sfa?X$1Ljnt2FZMtsF>bOHZAE0lh0<zub4x~WU#S4o!fXYbh22+Vlohz
zF1c6{VzNDxn5slsk=xAlbalo>@IW;G=p2S@W6?*~-IfbkkwWJ)SJqf&o*7pmrJ*-U
zp(?o_MNH~U>tTp?@D<AFZVsFI(3;G3<Ag^<!_=3o<(MQ=)=qzn6}19HlrN$Xv2BAE
z#BHef(V%E$lC;czsFOp~ahQU{;Ka$n81oU&NO{rz4vUTY4~d>f|HEmW>?6Oa;oZ8i
z6CVe^ZnJ~<0@$<V1fBDF0x#qCqw#P8y8vE-=kr(FKbTS5E36tRlASN%>fm)*boKM2
zUdcDv+g^z@XF@c+G^t;ohBxvZeJIdd9<w2GdHY9H;JD#H_Cj9NL37gK<(&w57gdtk
zH#K{t%*LHxI6mPLVH7ALcUK^J(PGm^W~gcNJu9i*=X!Uv?w7)Zg1Rz4A(@>oSU+h)
zJ?;y)#-PR)?Ock@j5KWZ=#za*WUh-_j2E`(_qJ6J>}c3JvZ&@Yv8l8$ex;P}w|3%W
zcoM_##hZ1wcZ0l6jN}~5JhilMIK9<bBwg~*Vo8W#3T=LhU+esqc!`{U4J(&dOYb{a
z!e{O3zFA-}u!fNsotB7CmE1%N1-G+f<vLp=05tgj(Zc?loiG;JY8J;YMMp<`Fn*sk
zANdihF*fN#BRZSPaqlr_WM;Z#juWb?`^x>(-Ji`&ma%G!Z`)*-6Nx^cB)a!*RWIgQ
z&Y3%3if$q?sdoHk*V(EexuK>0`X5HtIy17?_oCVwt~(j~ecoFgD>b=-->-(I?{CfC
z**quyLe!AJWUt)XVz`v6?(r?_W){;nM}DA}a5pLU69IzYG-(#Gm5T7&3+e6;8JjlL
z#=elOrbbFq$hd{fl}>D&E3vf@Zgn11X0O?bQBHBuc3Uwo%hmZzt#}Yq_cpi+I=jI&
zEaf7MMH|2MUeu{5tEwo~4`;o5Ex?++2FV2B@P*rGvjs#%rAnhL9l`{Lu_goRl3MiX
z_2nZFh*%E58O@hou)sL7y3c(F;(~iVagvr|23?Kc+OL8PmTm&*!~RB7f5*dZx?h18
zxZG{juISoQ`fkfT_{?$FLw5Z4ws#Iy1~P<QYZUlvyZ$&cqsjF{_e3I^{b1+ydi?6F
z;C`3Vg7Q+@#PYsH7ht_ID?@Xt|NIXUn9d41Cc`1i0id8`HCS&rgYIG7M|6@z{8Aui
zQk;r&Z#Ok0K+koLur(%oig}N=ns}HFUzxs>>mcYf{6owbQjNx4K%22|3=fZu;JD_?
zcRE|5rZX#L$*{wQ4gx-j$`N9@Ge?Wh)5?3E<bBl)k{56TS2bAUbs~1ZLP0CGqG_TV
z2mOT#tToeLs=V-RA<yqHJ|VktL}E(quxErEGnq(3O437ZIff&5BGeQZ+eWk<4`UJO
z3%;Qf1w}?cA$Rd*TDaq5plIC!CvN<+XeA?*yVW%GVb_(gtFEhxU&7EI`-M+}j<D+l
zELVc5SU;{g;)xd$oRV2xowM~CwyctC52lv;AT#OQ(Y~3Uomz1HkeL?1^Hdqg3;&s<
zZdjb1M~j}Kvai5-1eiI7p>G<J3X9XCetw^RbRU#96U2Z)5+I267t&A}@uTM?C8b>G
zw2?xD1|fMV`Z)#W!GK`5r$tI(p$h*ks@d<Q{@P#<uI!6*g>s#z&!NttrP<PDHe4Z|
z4ydryOQ_C)c3kz&MAoD+PB3_#yNTI26x^iQLgaIGR?oP+i_vwfP`d>zHJtNJ@s8tG
zE;&I6JT73Q?*Lz=L&XNJ*_Ty?P*>y#m4=+4@w<N8%HLZYHLT657>@PHWTY?-r{WOU
z&JTo583%0U(F1h#FX$@gAHxLuGoKI#a`bQk(A8Umucu)7wb~2$tqeuT1NRXJ1zoF|
zZ;Wptr?^_XYa_?p{WHqYXVWsrS`aq$10Ch*cmQaK*Uhn2waKo}Zmxd2u0!*R=g%)!
zr3aFgDt(NHMdELQA$;YHe9O`?93=z`Vq*zH<wFMBYvw5*RnSY(Ez(=dxo%#(EA31Z
z`t&#jt5H}98z(aF8q!rWib^5h#6;6HhnRW{qMjqV4j;toExK4O0e|y3Q_=A*@2B>0
zGFthojhUTD@pEbbyqyRfCK!89VjMF_Ouau`_+X%j6o?CUAO;#jnORnJe+gIyI*ZXL
zl7_BuK|%J|O{fFacYb^oRIC7z&c4kTy&#8)j`UntrLL(rfVtJtPszKi1LfpxzpuHM
zdo6zPPTohfqTc_FYps9d+5yP;ZL*uYwWo^hzW4@5-TBRL6dl!5$HfB_IvwgV>pCaX
zX3ErXlLO3E%Z2usTciP$S7H0(4VHaUbv!!lQ*}%J9oY@8JDRtn{tP0f2WJQPpro$$
z+8KG3@T%pE#p~!ko*~O9qjmQruEQxHT$-p60gU0J`J)>Yqn?|zd&FebWWc?DO8o%4
zz$4Lof_xl&LhIEj94G5V?ZEEmJhC7>+;TEpN)^8I{bgN?y1(lI*Kns;8e45R6b%p@
zDJhyZd0%nZ7;1F~?6!@Y5XQ2O)d?+-W7rRVLNyoGvZVMqTgpL}hFazpb*V#|+-NEs
zwdhsp??!b{CFfOw<Y@gFehIbKBEA;f?b91AO3|x9MRv0mC`1FYrhd`i+lNwC2Y;rC
zGtN8xH(8Mns|cgDVZc;7N}&m8u&)d~umcO7#VHNm&9);Rr@<W!;Bo)y!`671!C3{-
zEY<Ew+~7T%X4zZ>-*>*+QPfbux`7FAfAy|TfZ-p-YG3qO9TvS@QzZGT5;$fY`X<qX
z^MVOk4b%PLg-=GS!JxB-II>WM?}23jHhkTW=V?2lOiIDl@X(5Fsi6c9?G)co#q36D
znajN|N3rm|P_qZ=6HJBoWh>46S%&a$h0Ne;SmzPCLz*e;PMJVh{W$SM9g3Yz!a2p#
zN<70Qp*56JUH@~UtrCid6rHI%jJLgWv|7CToyAw^=47?mWFNV{kagC>S4qg+ip0Mk
zBdQf1ju}*zyDNbT_WvSoy;2g2kQashZ|=@a*tJ6AaG5v>>;6So3KQB!`AR+`@NQpw
z2+K1)_QQFL_5ywQGA<$I*u+GzqeyR7_>h(mfq_+}9MJf@rCRfpz%E{_ia$LWQCpCR
zFT#Z0%=X15ZRRWOlcsYHJ5&CVmAKEM!1*t9_gI7pyGk!6y>t@*IOiRG9B4F44FzH)
zIwFG_Q&lTIYDRQ116lkMa)P-(Q{3Ibp~$_O!dcdKFN0Bdk8>hswV=h@jmjBgDR@{?
z?xGp%qe3b&SUU=l5l5b^wARQdGRpSC>_vL(#3vQX6>zwp)I#!m#Cl)JptF(`9Ads8
zZS2wfW_+**OMx<bX-a*#TmDl}n$sR;U4v(aEB&-UX_KM?jhI&co&|Hl7q4VA(!@(5
z(<X!2(*5@_8vHxr)h*U+hjYp)sgkox0~H2jH~*Jh#o1GE(!+ZzV<baFmi>4yJ6>&R
zb4qI9v<~-)g<?K|#c?{hW;Iy~C>PidN|XT*FjXpZ*tUy0@YZ+UH;*0}3TRRvb1*dC
z^>^n-z@AS|FQrf(*3$4_g&bnE*~|XH++5`ZnC-XIEMfD1le$3$Ac??kJe?O+iaw4o
zG?0-4A-@&`baF)7WXbsko7CB6bH4>eb4UbD;;RtV&X;g=eE`(Sx$lcb1=B-3nv~hx
zz3ClK^W5uyl#$j}w=VOg!50<k;zfrcFz8xnxsyel%F)w4m5<a93wn)1DY!7k0ipos
zWVlsb5-)NG;vT?o*BooD);&+#HaU2+Dhcdo-q}{HPHC>jUP*I%LeInMc8V9~=7~BI
zZEc;XEL*YdV^zf`?H`z+MA(&kKW@HJk1(O()h|#m#FtS0-=N@F5dZ~ehQ<Jf=JY|N
zx?e>=k9z0MO?bArO?a(YZLL)0><#wj#W<#Iymao;?$BZSP#Ds5J*`&J3l&=>klguf
z>KY4$j~!1BT$VU#;6^y-e<m|i=QDU9HGb#n8Bx=wW6v<)rf5``@p`Gr`iM+M=z@VW
zg)mtP3B#=-RuYfdX~yzu<mxWTTXDPo7{L%WlQBqY4RW+LZw}S3jOk8P5|mmTX-ty#
zH;BEoG(+A0b<wxLlB!$mNl&mR24!JCW|CS*9!|oSGR&<R%vq3CG*QN3%1CTc$L%h}
zy+>%l=4{w{@H{WSgD<;lOEu;A_`ZBpDdt=$t!}7pPgrTAyLlp;jA*V8FF4udA;6Lr
zd{S_5J**PdN3fps@iqi5&ePfi<Tw6~y1PtPoBthkr~Zz*)nVsta!L;CcLl*P+<QCj
zj_K1MG~HJZ*`J?&{*fZZ+7S=)u+=Y3aomb3AP-!ES=4T})4N~#;#cpf<h2Tpi*xC&
zEj=X94&=J`XRb^jJ%g*HwCjuKe?u<8Lc0WcqUCShCo_a4^Yo7@=9rqq0+f3W<}cC)
zr(R;HIKp3aZ6FWvUaa&H^C(3!WkS+#8DR=FCS=Wf^gBbEUYaskpe@)p$V!o%!=utG
zqd&4pf36%X2b<WBo*-2pUy)_|Nj*h9HrmS;N8!nI=;B0>NgD2nMfcNkZ1Q|a;6oz;
zG7SWkz9s913GO-=O__iqJ;AG64ma?e-TJxRCbUPP6}@2=&ko1H5b5wsSR`Gvrw07!
zZD0NMz{QV%Q9u2UM*Tr+lk>HzYutMM7+|&>4aBFM^8tu@aiaUCk>B!%&9Uq8Z~5fr
z_6V96w>5lScUns+Y4=dMKJnNpCLbBh=j+d|D$A~Z2o(CKQJ<X|_1Wzq-}<u}K#NZA
zQ9Lm-bX~WH=V!)Nbt96V>_#2!V|~UtkR<8BG|hCpVn!;nGhMlp)eIL+DqAyYW1uvC
zDIy(y(ARd-x>5zJ4y_NZq6R`NdcF+T+jf9@pk@R;-m&8fd0CHjl$#ywhaA3m`9bSq
zh`j<MtvejV1;knNkuK2?&)HCksyriBQ;IlocjTyDO?btR9RjIzFZ{<iQ7H;B{rQ72
zgOo7Ep2FZr(XZ-Mho4=Z9WOEkch@-h+v9hIu2fdLb-~aX!Rkgeu1~eOeE;fuuF`jo
z{|SlC`r{|R`@lN>?y)>Q1;$U`k=*Lyf^5yBN<N>rYuFyq(vivpJbwSQ+5`|AVjRkK
zkB==EwjB*?AFERS`E{l>X9%k%rtWYh0zg<_ULn8MJSzxi6sLuSRG)iu`!PiL9Gu_<
zTdmAa!NBOooC`v?gxAPm%!EbetYp_?gSYlF77@wOqR3c7#I5w=DHJd_PKlC#4mRIm
zdjZ|UIGP+UdhuF+{ko8Sjar4x^~HO=17iK3p~8ib30BA@C$f0QFy7%%u%7KKALE2^
zTCQc$?Ks;NWZFD{Y*Q*3h?x&!f=gV)X;^lCDBx6TN`++1BPt~4xDY@GA$npWXCbwu
zE8i|2IFKLyM{JeI(KCM9jCeP}h#3$waNeKYtS@V5@Sf8nB{RgmI~;UeWs+g{!zfw)
z5IG_Ge)k`){72=9qgM}Ko%QhHN!-H<!zXF@OcRy@*o4zA#lcBkBwKi<FUI=0ug+G5
zNSDs*g%4K(w?gkD&8o!RW^Qqy@tF*Gu>l*Oys~7i0R@45I=tOneeChr;S3j;1%N8n
zN3jg&G_-SO0w`aiG6>NDqXrz2Pqzr(cjdON5rD2;zB{ki0pG+?ymgsY#wE5Cw5~3V
zL$9owa?vO#e`i8v^UCS9RD?QW@!9OqRh9l3Re$8kFpevEpyEEzSJqhMOOclKK1BRV
zHe<>O&HtTj>i<qQ=ugS=Wj|8XIxN!mBAUA;d{-lc;_{1YoPW8i4f##5+(!Ju;>EhI
ze*#X?U;uE62Cvo>0Kx=jzpqc7fIb5iD-m?g8UYEw5rWwRPP0IMaDFNB;ZV^j=273*
zNPTC@(&19h!B|l<e|#mZrlAs2p|^)!HC8jChW-$4rEmG%$QowL+}}L72}?;n!E;TW
z3~!rhm5;Ufpwn@v=|0YWGf5#%j={F2tS~dPmhP`@L9Dai@<&w+Iw<Vu$xsgdZh-z}
zbrh29wcSenfTTXPkX@#fR|+qYCb~nrnMwhh9q}qpRHnXlbVD4;u_!(EX`nLWyp8&~
zz)vmNRlw%;vksf@i`~8Es)Sml5m)h3?m_@R8t6Q$P4)NV`qRre$Rf@YHZV{HTJ2<u
zJRs~M-N}DgIq|IRGp3y<|6+30KEncz3u0o`J^69V=x?z9*pPDWtRE-KE=BU6Evruc
zGh&p#VfDkoFJjcgSZ|2Rt(iXH3Rk0iQ4r8bS%GH^w#Ek2d?1(ht0vOMqA<u3+%g{n
z%g*=sRe2aOWL6gKP*;H)8)8R3?5-}M=Zk)yXC$!tI$EQYPp*f~q0<g#E$;uN(P{9x
z)hw?7q2w~5+6Tt%&6KhC_58J^#&IH}IF}y`HmE($1RP^00A*)!rQctde`Uo3O%zRB
zRrQ_}7czOB=&QOBF|1-g+?2}a^BiCFB^3Wsk!`$%uQ}a;jF8xd;$_=ImJY$@P`!pg
zD?gx>!uoH8(DokZWQ+1CApLu^_)Nqj@=D!I|4@8*ykXrPlZbp;Pc#?1-pMo${MKi^
z%m?7?$DOeQL+`@<HdZ$ai_^6HPUa|dNbXSlCq5@5?0W%p`T)wBIaGi~3iH}Bn97UA
zTD5Y-9X#;)Xfe56O=2T$DZ$WxqT-Bx!j_IL*-KaXwRDPhyjtk!Scn%qqhXAIpsF+C
ze#q8=mzlJ~b%_$EZ(&8-f1cL68v0S^b3LEYU6;8rxlZ1A<+{$Mjz0H)!YdB4PRg_A
zM_IJ`c?EcWd!$K9ZWD&_a1u`t8WSpG7<kdGnFRMT`bDzsiu2?2j~8vt;#n07<Q}I5
zIrUe<PAX5^y+6g7)1Z*eY8>gZzG;Py8YncGjP+LT0$bP8$>fl%-SbL+ZJAzWsOX$E
z@YI*ChAFCMeaFdD-&1oXy$EU|c%-oT6N5aL!3e(bB3wGZW&=5$3HC}MX@w5-lS2<J
zZbVOvgubWVF%Y4!M%F(66n(tmg-0#UWg;p@=<GwGR<Ya`{zv_D<E`EKOGebv=nM_u
zN~H3<*A17afA!*b$gS3BPQQg&qFP-B$&k0nZ_<ozDOi?CCBM5^Rd8j=@HvXvuE?>K
zvEe|9&1(_0wzC}V+W_AXwH{BD!T$--Ze0C~wdtI&H}=||OK%vwH0e6GvtVa(wCl8S
z-lmlaL48JwzmwhCasKwEBScn;nyX~ay{uiIZ(Z)j`f=M5&GwZ)J<^o-EbsAZ8dZrY
z1I)K;{qbx~+z^ixF*}Hmk3gLOPOJp6_Bsy~5iMX1kc_pf7aib7+f7mFvUV1I*=i>*
zmY79CCE{fd>6NK#Q%I_5F?(n;BfFBZDY!k3JE2cMe{^}U5>h%WOYcZ2Ls2STU-DWR
z^YJC2)<r}%HIcWzJE>9~rMHjNwm3pkfopIS^|TH|MksSjz@gPW>HsVT)W5~YIbk$}
z6f&iw!(BX+C)!vH9*KxbN2myVW5yOZz?Kq?=toZmGhxzVx!=~aFglHai*MmX{}S2#
zcA=>)W1qN@vet$u)+<pyNGHn^_br)6z`Bi%G5crr*ul+|UG~OFuEW`(G)?wx1ej*c
zYeH-&{u_mmP%XC9WyCzc3ykkmKX@p0+%H6fdhj>D3lM4mXVLVC4p)JeOO-EM9`9<0
zz-E~2sQ9Mjw<3ko#RdHaTUp`}$GsLEaRv^eC=|hLr`wRBk>6gu<3(pKkI1H4<1#z1
zajr@<p<~TH-WivEv{@8;o(`Zw{v+#(*j`xtdl$f`03e%afgEo?JbP^RFD=cC{D>mQ
z3A_k#WVEPF!f8U&p;nYgRUVNDm2~MQxTmn?S(MlUrzQ<2X({6d(!8p;wt8S-hnu04
zp~hvd3|4!*U_qN7$;KJQ9x695f^{?=Yb;dYUvd|;r)LOPmn>CzPHQK8p$6}Mw~0Vr
z(8^F7Todpn|5=LEi6sv6%zx#*okTj@X4rAQ^fUF@jSbqA32+ciDl^pHKGb(ORbfJU
zlxInkqWKwA{2=(X7oNM7b-hKRM?OPJj5z7l?dacnzYWDs^c?M8M_c1ZS<hmyM<uOI
ztAglS$yA`vyP-O)5V%rH<H&x0^Qkp8s`&6QKF$~4sNu2bU)P965A_!ZH9Ad4pa0Wp
zpMrjex};oF$rKr%SgCin%iKm>*Xq_HQ}c@->!^qhJvvY2xNv)oGSJIKf<OJJjl{C7
z#<6VS&KWO9CC+<#T$bG4Tdao`_W6&e0X&)nx-;ZWr#JMpmyWx~@F78feAy?*_(1xA
z@8r`gh8K+Gfl0I$4b7g4(iYAPaK*eZy6qE9YD25GThA2J#_RAeRW;QIXz*WxOVWP|
zF7Zp9$3Glh4C+D?V}@OCsVFv6&b=H9m-^E3CM4=PIXS0*e81W2{8Uk|#p#xiJx$3>
zpGpZ~JaP~}Z+5w#=MF@=v|oe~Vr_xZFud=%Vb~Rnu&1)0m&rfI&Lt*P!Zz?#i~!)q
zCZ#{myRvNv?-L<TN`rlmfSu`q2re}0?k9UR)L|;7f&>?j(hc<{Peykm!>6oj3)^d9
zCyRktmQydC{k=n&oTTzU=rt{cU)8xP$+vlRFk$VwY;DR$ubl4n3iE~ab_kpQgX+y|
zQyoSE+#&x6asp>nx4@0bLM5OR2W;1C&2rnISBI~yTc0+3ndw|QPMINS(v_d4D#4YO
zbXfK%3my?+2F>Dg9u72j-}&LwWJM|c6Q3=9i&EMPzr?Rg0PCn}HMd}&rS%tYm$YTd
zjOu1xePaik?UADoj?!afkq#;zkePdOlCkmV8$sb*Rv|WPG*wSX4Jr+K8hmB;<;I-d
z)&>h~d`eX2R@w$8h3;suT*&?4JOUT9W42U~e$rn8apFJB8=Fc!GJ{xS?u|`JVeeD=
z`3YJ6U;53IreOp78l~!mN`v;U>zpY?srD5kItvHM#X?mRD3}y|>`0kayqC-3yL`tl
zH77)UC-kDF()5wkfr~|S$z{vD3ns9wsL#!$M_VTC*EROWfJkpScYbQKK{tMXchb$&
zz;El-=23RN^Yf_x%M$<@@wrz&QmKjE$Dir!A04i10kO|xVU*?B^)tQ{zI7<^HaZnQ
z=6qP=b5({-S0(a~2b6Z1mQYOtVkn9~%h}}F-`tE$7>jMW0X6BRLsOg1R)jx}W?NDA
zwS;_HejZU2k@)I&UDLKGwBis-P$TGo-`hO9QAYePLFria59LuwS4?Xk#7K|?YWonu
zCL_M7hrHHey%j2vt~ZNQQ@J9d?PMy{VOdvK<11OUfi{ki8kIzaR@BMH^G5j%W`=x>
z&2w+sqwTa-#%Rd75>!_C;Rn}EuK9SIn4`P9Mc;EF^`=fzGf&KnSCxk*C&Rr5H=+H_
zYZ{WNZ_=L=zB&0#GEG^d=c6U3=@y)5;Bb$l??W2=jbFAIEsIlk1>59~hMm{h)<rq)
z1MSelh4FBd!20PXELr*)%ymk!1Gwzid&Dms3ziz1fE(=(b~k84-SB*E-zpJQD=;dW
zu~XdUTbSp)5%0-C%S@{MY)jQ=hJF^+v!huHy7vVIA+Rlrq}pS@+NomM!_YaI<L!0d
zO88Rz6w}f!Zk$Ww$|1JK#N#MGdP3?o7cnzk?bdNl_R#~WgBN@r1QL_awf14%@SQKy
z2)6y0%_Sk3XDN{_t0tSfmNUf+#a7XO&{`a1suz9>K_1MXhx{2TgPIOPfV6V#|KV4I
z_7&RFZQ>DQ`c8|o>HD&227nx|17T=&FU&1ymRrviW=0#XXvQkZRf8Q0YDxz)8E+K<
zI+&=gvRw|HiAyUZQ#4n9D2?~#D)9^Vmr#q(l~jmHuJX0&=S$Ptqr`5gKYHKv=o_=K
zmFM#nmf$FeX36W8L<%1(k7a8{Zr7ur7g46xRP<^%Df&|U&|<I?&~yLpuzDC%A{Ffi
z;AXB)b6GJu^D#e^oq)$|r<270i$->K7}Lh;XYYj}VjpAUXtUQPztO$dVqLId9v&|s
zSiFxkiQaPf(j2m|py90~<=NakhryclepXtD%x-vJQ0Hc8x3)b5#kwst)J}Ma`PfY-
z+ei6#ml<eD&H5?7rH_IeTgzZmPOoJ9u9rLc(X4x-WIF0A8+1Kn*NkGUhxq9^doX1k
z(27$C)j2uWbOd!M%=&Z|6Iibe^<P_RU(hw0O!Cf#?FvN;M(@oj)>fY`9l`{Fn^3@D
zkLj_elm07X9R-or7f+NA9jYdbt-6>G{rMo?Ln;de#s2=ZEdT7lG5mhI`oyt}|Mhf(
zem&hcXWW}&`AfHr*bBJJdnqP?Gyx^|@0lrcH<SybaU{bPx^7L88RFvvDw1h{`ZxK~
z3uZJ1UnOYmEhXgcx=%Hl|5Q_P!Vg~+?e6`#pjn-=qF1gXGUrp^#3j2~JG<8wkX3ro
ztM74-z6Itk^iVD)9-y(7r4=m{)b1A5-0lq{FTs=e(uUHsu-*5cUGDURSyEAm&vH-F
z+6lUqir!zXN-s_6oyeCiig!Zwpv;8URG_>4txUB5wrDad%L5w5d`o9Nq<;}_(q8v7
z_Lq5J0`M}5ucx?!{n_+*9$lfU%IvOBtPSV`7kM;1pL+GA(X1RD5d_X#SD;X1m~r2-
z$f+w^`z|mO=y9qB)vPA_^K@`ryxfK*(b%`0WGlZ0^wPqc>x^*%OV703C!P&CPp^UF
z1Qwq?NIPtRu^``%rveiOM>C=w^pGQ@)7JRFj7j4`p!49q^@R5D-9wX|IL3EGYmJT)
zbGquM#m?;ro!4=XrzB3c3Y@2c@q)XfZf%12t@H!i{aydV*}GE$Eq@fa+eE(}Wnc8t
zyQ}}CBod^~pof(GkoLka(r>-KqzEXO6sJY~_XwV|?~OyCAlMLhBgmhnE%7C7^miYe
zy_&^Hq(W7+xc3NTQgN)29bCP6DJL){x@+R3mCnMxmcCoAD-%{dir9zj<t%^D85<g%
z=W2~iZ9Tc!TEA0Rp+2+x22Y?B-wev~l^hPW%c^HR5jWQSLDxS6w=?6LV#f&VzpM$}
zpG++i1%06@?6Qu+3+@buiAe(e6_>RU?Q9{Jy&D@N#d${j){7ji@I#qsvD}xPg?4@o
z0Gse(=r1<GFJb~G5ZF3q=r!?fK_K1FWgolKh*eGZTvnOr7NG*}YN;JNS*XzPDSW!n
zmF}liJ^h)2bSZU3qOq>bkBIWChP>SAu|IUpz-H9ZcYh_i;URRROM}w+|KaXEqng~h
zu3?&hQUY?MLr^@Rf*>Nj1VlxT0(N>ALJvhk3xuK+r6~$hgQ6lx@4ZWhfFQkhA@oiN
z`F8N!_jBYt-_Q5Q`~GC$U<|Ll_gZVNHRoKq&oOR$`rFvn3k$=y65YC$!!BqjY*_jk
z&$NyA{;Z$!pO*R(ItW{N3js6m8P!P}Pr0&9jNt?1py{lgO`~|YML3=($8h(nRAp8d
zqU_-FT1^tMq-+9Vc$X1X)h9(4zupFoM7uRK$e|BtDSiW5|EPzk5AnkP6!uw)d?2S1
zF%!Q~@HbMuV}I=I*TmZ6-tuP*l^7|9$abj)0;}se;(6MYJnF;3P(4{pLfXgT4`y_o
zq(^1)2K|N)aNVB-9cn*>7zj_a35LnUJs(bZ#4|)$TEyP9l`LYCncbC&!m<nmyi_~4
z8){^AsYoT)dh&xoS}LVJ-*AYyKTjtNsdFrRwC7CYk~dX#okzZN%CSV8@T{YqCoWK;
zyPTzXt5Cr6Y*UtQtlraa192hM=8BtV8~ZTBkAA4nRa)woKBj0Bj1tzqY~F?`zxGS8
zGCj#9wQ<R3W}YiU1ihK9897}j46PgH@R`}fqEz}lX3I;9#Ig7ryNyzcmmBZ7Z9X_~
zUrCo--rFy2MjP&{=uaO*@B2FTc|v>QMT^&7ua!hRe&s-|?DB#1_{Lu;?$kiu7RR<O
z-n@!+TVAbQvV_658(ErhAv=Q3CTriIUfFtkgSQ1EyRTJlvmh?&+U8z|yN!;<Yk952
zL!%g^hO+isk`D~~pNdp=Xf_WTtdFd{To<{3b+7JfPD)r=X-8ILMMV&vKT72k&{V%4
zT?*f!?&h<viR01tsRaeN`ZH0(0nCoPC>fwkHIzzY<XO^ROI6PK@RRjcyCQd}UaO@J
z>;_i1t1HS=n6%aUvj^ypkXJ~yAJlyxmF>w3<EEKNx4vl7IC+Ox&Q}du>O0DAGgtRr
zWmw*3!_-A!K_@_Kvj8ojp$=*!V<YNE>>VX@w0?BKi6{N+Y#x=ew=Wt}Ep++Oz}2=`
z)7-gfCA6lpxB~3ZGxsvB*aDz(gtqZJqO62dSX=1Ry3!A(Nvx8l3SCg1PHgcaw*LUt
zYz?8kLN^a8r8fW}+}zMQL<qdFI=LH&<5%{>(xcE*X3r6cmea$6Wy!50uE^rC?}m}v
zs8xmdml*TCK0~iEyg4VmsKV->X8q1B19|Fb9m3~Gm|J56*t!dk*Wq(G7mnZ?Kc}hZ
zaea4t+3R!MdW+LMjBlUCan?~opd)T0Ct{iV-RPvx7*(fb&RRxTd5TRbp+|quV{~tP
zt#<j~qdalC;cYGNZg@L#1cDV^M%7k6-8(KS^I3ks6SBC0A@2CD5e>FA;uB!Mp=p5F
zf6codD2o;U_Ey0UX%GNIx`@frzJSPn>QRLc^|+sG#dCk`ZfF1a@vO7$Mp1HP4L;99
zo`<sY8&<j7)8*ZxJY8_gG^_Tu1%B|QzMZe6RbGB$5}U*yRxQq1m5RuTkkvb$k^2yQ
zQ~X$3$7reSOG4oL<XMSLLG$$Me7&;93_gr4BD<9Oa_0@G&~z7&HWYQ=I<m)|+{Oz&
zXXEPn(M4X)&R2uc<RL$1D6<s_r)C|f7q@I(7PxMgKVEqNsdRbvhaZDj#UeI1t|{B@
z5nZKIjZ{Zm3?h*{^dCj;MV}d`QCK<Z^GU6q8^|H6hBXL!E6lHB(9UqZa*@gQ4dMV-
z*&Kk~$!f48;wMM%AxF${aG$3`jtdv31K5a(s3kn6v<q>kSdhhHgy@St&t+EyOJ|8-
zsWhmRnEBvz_||?ZWM1&V73;!x=kSW!A{8sIfo`B8mH*zbdrSKCF)YeiyV%5R7QM>_
zf~Qx<LHt2_X8Z$n6!yH@r!T*s1{lO1s=AuSqtbrEBH)J}C`f}piMc?x(w2MS1CTR~
zGd}q<tm?Q?6TFH6<}DsaMg=Q@E7tj<k{X3B(ZDvDiy8Y?3bRp-js1U$`tWcO06Tk5
z!$5a5rY>ngFsh=je-R8JxD{D*nN=8DbY?6~5evPOMk0<(PTh;0GJ@+gYRM{TwMqt!
zw;zluueu#4H*0_tQ^C;WXJg1{P4tOpS~DbxAI1^*2N+k!e9e51OJMjzFV`}<k|;vR
zF4oL%NrIW6XOqIpzl3Ei%-&J3-Rj0h$IvR^S~8@iJOj-Z?;XFA{7i}e8ezzjgvh-V
zE0Ob&O6v!rM5o;)(aOP+<%ck%1M=xi%wCM#W@gjs<mi4;+4_b{u#+;C+q?=!C)B;l
zv<0Vq5Rr3X>O+gD@bW=TX-mG8erdB-zQ<)Bb8e;uz?JBQRdTO|<DGdb3Hi%!@e1~p
zIh&HY$XQ6`?3AyXWs@ZF0AY9zL+tK-YIcNA9&zIryyvVr`>%ACaG1`Zhiv7HK!f{i
z#7|e`y+Lt87cTw%_3f<eu?tKPLDJDU23_x)=C>CBe1U~N8cZDJ{09~lLP3V}97H2&
z`Ycn!2gS{m^L<U<k}>S~$wJ~KqM!7_{aUSE=w7NdQ*wv_EqvY{X>K$BC_b*w$g$s4
z8*md@{j&d9#OYCV3Va#pOMx>?crx`V&`~?D=mv<JpaF08eR*3~L+B`tHsH^u_5EDn
z?#Y6c=BLUw=EGLIq3yL{S(CG($xb;BBDiNAL)%hqw@K<5P;mZ(2EH2FDR?`v-!wp>
zSvS<vTJt#q<Az6<Es9hZ?WeYA^4GkpllsB!){)eFFM#_|!Jvs;o8KN{w^jGV6V7R7
z8u<_M%lFVDJR0~B%jrC)1C`#Qcnl_I#9>($8JDTBKRs!9pSP^nPp*qg-P5z!wdq?r
zetRB@8V_h5Qk#j?MU>{t&z3h3NBVv`+I0|E<g%??HgmOH9xWCTmo29|HBhc>^~6h~
zMwAE9qGqc#EiDTMR}Oa3bFhQ=HkaYSOq&`A{7so{9Mm~`m;d*+{GDUJ0K}<3BL1bJ
z+#_`z3FsI6HxsoV-YYK_abv&<yD|jIliAQBZ5&So*TOSO%P3GY@A+o{v37?#u5)#e
zQckjcCdAW7;=m?Dj*zZgQ9dC1W9ZwMi^2Np>AqEQnkkMo{dP;Jqjn(B-0}~|hiMD0
z3O@e~URj6UAFC$~J|0twWjj66qD6U^?<L<}FuY;3=L#dmiwXL;plO^TleNPQf774m
zqmLPKA<#|2-BmZZW#*TDT;Zz<geXSBO>mRtWiOF?WBiDS64q`+qKwR_(t}rX*}oW)
zUV$#birN%yaUrvBNUeHGvV?7{;G1coC`kK*dwjF<lMu{3#-YhX<!H0si$bJa=C}SH
z&#PnL+WoR*oy}`ycP%9sCb5ztzeFA;qzd&4Zn*q_Dh5q}L4{-6`qPxo${r|Sc{gIu
zPk)SxmrwZ)-5;ooL=g7}BbxvGc{%`k-(X`N#ubGnDxv$G3=A`LkZn~Jwd#X!T@rTu
zt9L&&5Zk1QRm?|$9@DRLQXV_Wdlcxo2OOT$M*#F>GR~BIA<6mL$;+q}X9Lcjfsn;=
zlOdlWtg4r#s*R#peAF*Kwr+s9D9Oowub)iHAhf}yDpDz74z1P<m2<V?axdAf9{2%$
zYVEAv2K@2vsXpe3w(SgD_l-bn8UC$$I2!)##zkwVgseQaclK*M$LXi|mWnzC_Y9OU
zek`@uL!?xV_HRE)sM=%l$J24T_2d-D#Qiw-hQ|S}Xf;>p4+%oq?4-kOB%G0J%##D$
zcosjqv{J%mp~TMZ983N61(3vEBN9`7t3NQpKK>g(dasD84``M`Z?(y#s$&bb+dZOs
zVe1TjCRq1ghfQcgHrk)D7<(FEd(`z=x%FgtWEhH2r_9~#Rl1~Uu)RT)YnKFPwJpC~
zZZ_Cfxah8cAI4rNB(@VZ#WDzaJOTh$g39!MyZkcOgP6%W?15>&Yk_>z{feDv@#bXM
z)~<sDX45x0N>yPTl#0{eM<X@mw?ig@-N^5yV$&~)w!llHwco1{B_=JyU!|fWC>8hF
zenxeZc!<VvbS5V-ySZcXMygTLehi&U(eaShz(?}BG^ML^dnup#pz3~Z-F1wDKHT3p
z`m=KzxmbTm4+O&!9EF1OHCl43%ne-@>>qD9*f|p-1_r^ZAbZvCE6*=ES<dGcF<G<k
zSaJ4sHZi+ezwUkEtP`8v*DBb4(C5a7y5FxzkE3;BzD+MXF*0J>n%k_(Xaa_7K%G>W
zIvjma3x37zwh^UlMwCp@nw)*?=|!N9d9cQC<x~*wFCzD3fNR6tgVP;(qboIqnfq-`
z#6n@60*jEuA(uizOGpdiT$G+$Qx!2yRlXFp*kE|^cBZy$R(^BOdV$C?ejUyEPKTn+
z6Ir6MJD4=uz1xWXrfR4dIB5AbzIiZVb1P>x?a?(zocQ%bw{<bZc{-$C$0m9dT~UZ4
z)+|HJqmNIo5FnXmyKrX{$AbwBTSXK6!kVw9Vkx(KonOhx!eOL`+fsyq&JOfv^Ol_5
zpeH^Uw!9GpL4A%}eU4aYKk}kJlg6c>b9yrP)=_@Hkpih%egLZjFnr(g`bJKPT`Ha9
zi3#|DJ?$8k{g%q<cKc&1rx*L#kvLkY4d|o&kyW_)MMdG-chI5U(XxI~7_|$W0sePQ
zLq4>}L9!r~66#hb6Xwu<9rxxF&yx$^(rUvM&x%furH8rt)U-#xv{c>x&@En=8eg6s
z(xx-Da$a;SIv|t*0|{4D%u8idy3wGX=A^A_LWjg_=CibS*TF??w;EI#tia&MP52u-
zo-1HjdJU+EhcEcdbTe5z%fOT#vLUduz3K|5NP~J}*(jlGo4&%+Y&u=21zzc*vpe#9
zR7g`F{ixrzbh!Co9da&AVd_iHw7KqE4fJf+^!luqUB?LGzP9VPnzgsYUB}(c@1qYd
zx4m9U4o40nd)GcR99-A*dnS}o{-JE2UzBUgE^qwC^rYx+Vf8LPe^z@Iqt9qyZ|^qc
z6DG_gy#oysveUn}Q+)o3EdU>9AKsn^4OAhzsR+3UL4_%e<}Qe*`|Rggnu;~iX`0GC
zWtS0nl;;Vr8N#4O#cKOs8}$`!NZq$Co;Wq03VOm<LlI#FdJ*5;d`hS#0WWe+)NA;-
z*@A5WZpmrkVqqjp!v2~ARqTk~6~EsqoZz#+2n=VT8vZAK$N0)dJfJ(?CQIevuS23~
z_qn$nf_c<TSQUp+6M49}x5n$2UdSD?x(~H%m;B`kJyijXe^!NN6%x`kY@LE?)iYN2
z8m|W=^D+C2AIdSlykzR?$%P`Rp!w6v77a62*8MH=CTX`AW~BnwKRy=#6s(g}Rw;5t
zn+mywCS`0}<aae=SL1?ulkFHLI*v7zx8=fstfqud#d}<o%*mp5`OL_{JQ%)3uiT@o
zwo8NCoG6iQ&u|~a$nOg;tKiK4H4KtHcu0F?$g3e8DU49qT5gW(H$D4yG@vn_*}}^(
zw4#vMFxZS`CA69rLh&n;&G+8M<qC|Wr7MUnM7@Nyj^s!WThB}p(vj6}?)1d<#%~N5
zuaD6y&$cyGn03I%EZ($ud1$+BR+gt`Z)keX)k5zYtY|hsZeAIp%4tmtA)uj~#D!6J
zUAHbFV)p*bhwFrL3xVItQFfU_C2m`zLJa+H3J=m)R`v#Xp3d|;^m%}b{9kSRWg5)t
zy11HGPZ$Fu*p}CHK5m?oZ>evk_Ec4UXd<AZ@+uxb3r^``mg=KY`N<H6f&zhTT8TjZ
z2H-$26cYD1xNz$dv39rdbbZ2n>xU5biF_GBq=~e+rcmu{k4u!hM|CNEMFg3(>uFvx
z6$S|IQ=q9^r=(#=8xV6qr;L(VAsk>qf&dl!nl^>G#=_vJj>J>$xtaA#p3>bP6eaOV
zgHW#Q4b_g-!7lAS?d}gSI%7e=^ctPf#h?Nu>3|i+KX}O64YRQicKrXKs%dgT?+&YD
z8lmemtwQM4BDN^X`NFoeTN!0b&5ko}qvs5XO}#B`Lc7FL(>1~TWtDB2JY3GANX+BG
zU4eID@w=UCUAMhTVj#6Gn4K4njh>7gv!#3fUt3BmSC$-CUmL!4k)MsHG-+Ap?<iaO
z!uD2WTj8l54}?v$xQN6{@z9%WmufBBJD(}C|L{lFA!c-%O)yX=9{jA$7)gMFxq4Kk
zYnxNUD~mGvCoxhL`_SeOhK8x<kB(MZf1PPwRrHzP;UI9hU<cY#_x+V6c2AHpfmr)P
z$ypL}AG6*PX4K%H*kY!o?JPsN!$gvpSS!dt^o<CVuS16@WvIJVr3W!Ow&)_7?#qUT
z63#OAyp6{Fm8tPYDp_*Y@S@ro!k1gZ>oO7Ra=Gh2!qqogiaVB3JMk$$Hb?K7n{XXg
zrAtaU@5K%Z(VNy30VrJ07b{J_c-dMBvae>hWoT9pbtY?zzw1mY)9Z`-m>0&AX$Y&?
zWdVe?&GwVtxZ%B&$#_6~@z4Z~Gt!a%Pxh3iOhU-GwB#&j;E~VX#%hX)MiI8Y=UDWA
za2!_aC5+hR!rMgF3o(YsKF5HZVb(zsp9JT1^(Q*B0yU&@eps7%4GIuM%S^1Dfk@f@
zhfrxJc(%ALCciT+r?-6Tg2!T+rgnwn_2Q}K=hHPUB5A}TJa+nyM%L0iqG;TIx{3y;
z$@{dA&?2(#a)Hgz{?_h+Ht~VT{_eL0yTGTvMZO}{Up&mEdtnMuzsb`;6V!+SbW(Du
z@4B0F0*o$ps^V~B2(@bBQQB4!0n!z)MrT|;tM;_ky{N5J<i4+Dg8Z|Re<)Y38|Y^8
z(n!ofUNvx=21U(|4gKi;@d4(r#*Q;a<NEx+xldgkF;0fb5}5HxT-eaz>(C9rFH~1Q
zcjb}s4)P{VaskpkuY_&uvO%cy2J1tSdCDCB)w=QZ)BRm>k_j{cJf{QtB(c1U>Wc7n
z`N7H(YQkWGyp{=*>Cfx*@NHwR?dsX`InYF(OQxoZ83xBx^&RocgoeN^?)f_-0Q`p&
zHGZ`#6czGYIgb|RcK|kSyOUUAXB`R>#GE@GBnXBps1YoD9~bz9MRB>|GZg9|0_-IC
zrmD1+@*?0D9Rtl8GW-0*73*~N;;dUsFSIMxz3(!N=Xo`EbrGqEs>}L<T?+2`lXozy
zGQ`@WIqFiyXJj!hkPxseJ`-PsF|eaJ|Mj`Mo6qWdEnN?~U^bW8GX_>gK5l)-QjX2E
z@6nx6=uR<Ku#(~040K3HW#qjedzDv47(W=x^Zo|DV-XB>Y;G<j@qXl*t%m8viaTr7
zfl3H-gfeh+lBmQq$9$M5KdVA6CjFK{L1lc2pETv?&&uc`O&Vni47`l>TNWmqJXz(v
zk(LqSWDR66Ux=Nz+_h7vzO;Kk)B}}=(v=mgOnZA~tfET!8j}1W?#zDB?+nH5vT2&V
zd<_OOIgs0OG!n&8PxwsBzY?-dRdAUAMu<PHW+BuHu(VlE@?7Jhf`~4zP(0J{{xf)t
zKQx!@eizN%AMF|<XqlZL{)Jbn!tiC2VJl8=IJY7m|KQe6BSt@Jip{DMQXLk48$E$D
z{Gp77iZ}!+lmjgpajhStHE-(}fR){92$Pm`sKb7>E@ZN}a6*CY&}3s4a(VKIR0FiE
z+<(=QKRZ=gjx@ReC61OviK9(h$ea5$40!HbebGC%3XTsfqiz>=C9kWZ3s>QDspf94
zi#APo#$s)h%(&WZQmq=!rHsH2*6PX7@2vjJv=Z$8RDqnW4hoYSOJbBO4=O}IDjk{9
z_8Zol++fjAT<05U;lPDKp)S`3io{yZ9_0Nzd!OcfkA-O6b7rJy6$ekJM#ipQ-))X0
zh@2AL+3c;{&v3^;Q-pPlZW|>`Y-UnZkUCocGD!2mE=^ASOuIai6D?@6k(=AfpP;j+
zkUPi{U#Zl31{9NEjg(Q?#vkN|UnCiF$@G8uB7whbN5*Sg8jr^MJ3`=#+!2a%xt&d#
zbLlE;Jp?4r)&5F(8pTP~zVTyuQ>-RT(saVl^khyf)m4jHI0f~J_A13nSlwi+4b5B(
zQ`a|H)c@fCpXUd3nln+{>o`NT_{#KK!Rz)kR>td!Jvsh>!ZxQYkDL`NSj<!(dG^hh
zMcwA)mujY-`@KurOj&fkVg@Hm*-oN-vKeJWOg*K4{FsjD0&MQ~gM&Z$8nlqlwE~6e
zgy!n{4CS(0L*t{n6(aoFWKyir){naqB})i{(P=7-z=R9{S#Amj`%TTmKKg-T?VTey
z13<$$u3ygvp&_#VuV*t%dN#v>8F+Wn2$##^{~7rNp<Ma*n(puMSp|u1Vu;b3`4{Z^
zov;7yMcuC=pwd_56?Ht8@MnUOw+~xZj)*I{J^e;Cv`REUNls$wjKySYjViCKoG*hH
zr85r80o&b34kLv8K%gC&J(v+ZHi3lBCir+#*k*iEObKdlq4x$ooyXv)dtG`8Z0k7P
z_2dim{k|f6xE-<^WT^w+a$@21t98r>yGNzcRjk6cxWN40r)<^{3zUwE>$qDN;f}0U
zIgOizh0-hmiMljg@MrOyTV}IEdu`&>80W!*KHRe5T^px`7<7j+O?=7IpWjn&?2#<n
zJUtzw<$mwXfxy{;KJ0gkw=x{<BD~`yJrst@dyTvx&pMonc<wx`#l}q*2(8sScI!-f
z82hmbvYOE0c=sNw#humSS|P27jP%9K5T5MzKFc&l#mPCkULRgsI2EG`mv~ysb3Wud
zqK}&8f>9FVRVWIOFum600S|qA4G=d!GRoH6754NMwyb;C6J{3*36_4=Qh1HUrjAlr
zVm0EWohY{ZvgVoF63!+Xj%l@a>bi6k44#e3eKN5DA$wDNJ7<UcqIksWb>dcE)sAds
zHZ0k7uc`OV075?P6{SwZE2RjG{rt95>&DS|K;@(OtJv7v=)P=yH2I4AfG;QRGhdP}
z^B<0V%fjm?6NL?+JdE=;PM}2AxteILakP%pN0E8?{J)?{*jVZ`8#_7|`OE=I<j))(
zrSFIS!_xwFhCTgGFowsAj3SJTm+T`a6-+TQj>@Jk8o{u*!_@xTH1Jy}Lv}*kaeQte
zp>s2z@a0<#!f+eVuUBq`lC&Iz>6;I`5*Z63iQ_{Xh0xtCz-Em7Fw8rmLVV^zjbVJx
zSPSMM*@}vyYV>qmUaX3&Q%s@;Khy_w476gE?CtPhf$5)i??GQ{k`yxV!rY<_i5<|C
zUAY>U5)i(ux5JEWPim8QWSR)bK20sw+b@}3aO|y;(|y`2Jc{Ln+CvM(MgG>Yf8UCc
zzbzrR=PuhHy%k--;8t|e&kf(TBPsbF(X#dcCEp4aUvr~q_*XudcdjhTUowFk#iZ7;
z+QRe2%EesJ|G5S%yYGu?g&xh`=^dt7WyECq^U;bezDW1>=kd~R^p6&;w8K!S7YaXp
zLM==U@G`l(wS<sde{mKmOXjLuY!)ZW6QgE_7#Gd@LpF2Xboyg%7brQ^L#m%u8Ue__
z77%tl^y8_SERdK3d@I|&VL48ZprB01`7=8yZH83VeGxn`w>o=s8J9<;+zTu(J4W47
zf<`e`XV}Cd^?jMM0qofa_K-41$g}_cG6{cwnXR$QcK`Y^p`<Uv_)}ICh@d%v_TZjA
z@aZum=OvSsn_$#5x~^|eK1Fl0DCM(A^5!j__pHb!vAThz>GSNWw)jZ|J~It&b$)B?
z4)+k18)Q#jcQhfclNs4Q^Poj821_W<bn6+j0V2{5VJP!@th%sd`-77K@BVr<?!)~i
zW{&;>nK$JZ->FaiL8p{NMT-B=G4PMR4qqkWuhB2T0~sxSG|2wE4mt)uzpXEQkt8X@
zg{WUH=omyzF}ae_#!*QhTVejgGj94;6&)WVRGu6c*ExOtVZK}!rJUDVT8J7(a+f8+
zx{+^0Zw;rxErWApow}L@^a9T5)pqFh?CJ<3b$Gf}vhJW)oyq#0GBKQR-M~i84c=@Z
zak}$}J3*Fy&a9u~kJZa_i_;y0+!ItGTGn#Os}y{+INJO?jzO7uXtz;6xQIytinPMG
zIxADDC|A(b0~Hrk){9pl{}1cD+yX)z9I0l3*9|+E>;A|J8_(1IcYpR>=TXiAFxvPf
zbK1aFn++ZPXrFmVA%*9=vw=P76+I?n?U|P9XygeOBnu-O7ZxQGAw$~Ygq^<1@^vM$
z2%*V_WpOiF4|pjynM{~Gm^Flv_>x#%eSHlRX6yHIQi}N8K|@7qK<Q_mnDgm-5I=k{
zlTR4rtaG%Rt&ud&o_od;OGT5-s`B0%5kSRFHe~`>(}{HFis$R2Ck?5wOF+z3f?V$(
zM7W5x%LFA$LX=}6R8Uz7R9_zxK1<H8#5wPF`+HX3VU*1V4L``m!7?KOh(sSHr}@2W
ziTx%oFE(!tA`EBR<8>wbpG4T%maW!L_vj)n3O!Dn>uHl3)ypVbYj$jt=zU1LCwl1b
zLd2Na{cXwnk9Rgl_p4$P)ev_2SH<=Z&zqZx#UDp_DcWKx^r_X!&Io!V?C*_H6lFDO
zbU-wfx#YMh-0$Az_4BgX>zLMN3g{c<P?${e{}4KQu+uRv87?|o%{i60ARZ+>dnrJC
zH6gt^RO|Mzn{-RIg;P27;P^{5l6Lq~Fq@s|%oyYC6S5SvC$H*wf2lpADF<I`tqx#V
zeQ$qb+Xi8R3rb&Xs=fB*cCg}8lN2)%#rFC>Oi|>@L~#raq@S~|(d99Iut8kR7Co_u
z5Cz4_K>jkgL(`GEk>y(hUt2M-O8$1G7;RncVn;LW0vqk(jqkTxuCwVs-1|!jAmfVm
z$^xvA!#_UJ-_;!7CD~utxYJD_`iL5c%BPeDT(BkHq2$~Jkh2oBmywqG`_EV~czl25
zwNvc?By;eRp^j0FkkLWuO7*Bcr)pqTc09<S3y*mAZQhj8*wiQT+oFMwRbm7j-R^j5
zB{$7lSaRkx<0?EUI4M$Wey25Cq<D*;aadezF3>sqJ^zkQxj1mg<J5S>-63l*Fkp~6
z9JKj;Dv2^&GVqK6JFG$yTDe+FP{`$<;=s0ZR3p^zCdr}>v=D75vVHHT1_PeCAt$qh
zFx&*)a;VS9BgA)A)QtsL=6C=NSI*O)_if6VEAGvBQ!<{z<8n3njRSZ-+uG5dV>56y
z-u<b8)`=9I9Zm8IJnXgqZ!|wziUrDF{bo<sm|~Tq@%w=?2vsSGQk=ykJRlgy+UxA8
zkE$2FM_oZyMn=VjqTB)LE1{8#5Bul`c>{T@^OR7Qsp`giYj~8PvghMLXr+_1;UHJ6
z^CCiCZucgM9OU>@BSY^5s;>#&5E>#Mxt%g^MIvL5N~0c}jL~>h<{fqFPW$}U_)vY4
zhuDn8q%{B3-If+7%ZW4u!!5xdKYVa1moPl1dE?#ZD1t-h;0VIOr!#D&+t+6q_vKk+
zzxSw0v0_R<$=z-Tng6-1*j+`UVt<dq*vBY%AXA=U9EteR%GPM{0FTzdm9{Ih@h%3t
zonc-NcZ!qm9Rh3fZp_1(BZ4+?+0tOYF4yL4^o+pK%XfznTrPot!E<J$&b)?@<o{jn
zzjV~iyfjdhy(kzF-sxgi<#tG}iypKqP1keyc51p=C==K7L(@?IahldWzS^vv9!{^3
z<0spJf8r0qP!}o8A%`3qE?Y;2YhL3l1`oNpAC4LJ<X1E0mH9!csyK>OBYciR3UK)@
z{<?hSo^{1qM=zf>$;s0CcsFeT_@w^8Bvp%Tbc0OP7rT3Ehm5xL>?!26iQ8O$B4NsM
z+1*d8dBUIhh2)~>txZ+F__W)h7aTuFt(&MujV1#3J&Eo`I2vz}EZD+{tGmlJoI4Z<
zhLa)L0u=?UGF#b-Vi6~S0+{=q4r7~9LQ2?(0jsWD*_e@3<ZYALSbf(qTj&DULbNON
zf*hPGUaKd|h+lnzA$#UNn_O<57oMYqW4JtZ(w$?Levx}?B4Nr&!!|B^urigBTk@+c
zZ2alkIpVXf8(g*z8Rc+UOR;eFinoJ^%5q_YvLbx{r<bA?Mf}Zq2l<_+vBFweiMaCZ
z5wkFbt#cb(k#r(OiE(Z-(Li~A?odb|{z5}g{5KjefNg-Nr17hty|;^U`g%0A_V9ws
z*~9zwUI>ZDBz7wKxpVp#I$jFQZY{)ny=wLVmDonpyGFNC=n$t(yBb(VgYCHzIn4eP
zZ;=#57+1(6EI5P<h52yO?)D8WBDlrs3*=UZy5#pFzIp->7ZxL8mG7CZrf4$MXNl7=
z9nu}uS@<JC#1eYRnS=iBK!9IJ(c^)Zam3`Yx7^A+?S-t!wZ-a@*M?tSzFQuak)!m+
zcMPCrESB$w#cTOddOee$P0)AVyBq*v!4Lojb~I-+X)UZ=*NV_-94eU!Ta)Urso0&F
z{*n}WL2fbcfm9Y+DNcsCzup4zTJduvZ0>IiZZr-RJGGn+;J(i&zf$$1;_Xc8*Nh8>
zd*7o@;J*YRbkwDM%z;PfxqaCF5#jmYxy1O__q=pj=4eC7UW4R=e*jwr4I^N24Rq;(
zabqWLK2{E3A#=M{;mu90*!@-`F@)VF4v_ii^WghCjfPz`wRL|zR!$}ou_Lic);e$D
z16)imw2j41{Nd-#M9oi4agp5D`^K?9l%pSB{T`Y((NUkK{`FpW&pELI3zNHlyg%`T
ziTUlbXOC~O(9<%UIm3MJ1oLTO>Y(5W;_`xmo+utaA2$*FXjW@Gw<m*ugAd`oV2SO!
zc3ajJ5&`SOz&`oXASU`h_Q}e?J~<n*U1Nhy-I}cu5vk83q`d62ezN(nV&QmO?Kgop
z`%f-O^cBUL_tYyZXg%`K=ZgmvUXD&!j{6Bdgf;UMxRS5YMDajLH^p6Zf4t-9-aBUN
zFCGGpiwL#7o3D)vB#Y-><ZE;JwV~#3g=fQZlTzb5&>^ytnN%^kUaKz+gSFtD7+t$!
zGdJ)<S)EX<UHN9uTKqB!t`j6)+hAZCB2l7?AVn2&HmMGK76z&Btpcl2kzKbEWyZWI
zo0VyJ9*z(Xwxa@|Gsvv}{RoqpR^Y(KKcVKV&e7N+BtU9}5@r=!Xs?VYt>3)P%1cH8
z@vhbx=D2#Co4kmzUc6Q?+~)p|c2#<$`Pw)$lQpkooiuKZaV6hjOYkx9XKvkkHK|nQ
zoPM1?JMCv}@VjMs6VG|vVhp=GeQ(_j*vwYk#@Wcud0XOqOdM^`RE}y+-=%>2eP5hA
znI+qmsNS+aHC<({Eg|q}VR+@GtuGfE{V)^lHVUt`;VvQk`~*C48DYr1*xDEa`lgGI
z7JC7Q_8IpvuKX;Hg_wi6K(8KKbkA0Rv?-v_>h@|%$BPXbjOWcJctk2}^(EOYpC5<G
zM;Us#?N)<F@y5YFKOQ-df-4;Lx?R@uGXS=4tU3{Wu`qY%cFWH*9r6p>*bB)C?pCS>
z6Q42L<;$MauT*!oo&Hrrks?bKEyaN_aJE)tKAHs6=1Bd+Z5VlOIPm!X(9Cd!c68H;
zj$w2?NxYWD=w@8ChznULrAjE7tVuA6e4+Yc#|?&OeUF2y8AacNQqa8k_`sLWo-<(v
z-FLsVaRQExn5k5~#3siY3@7N~n|~E;b*YpM&#a=+n*?aBLsyehUc{P3lFXU+c>lv`
zt?tM^PL@k!XU*AiCzuehps6WXw?lsAaV(^6Z$on^%%j6@6j@M_-EDmgwz=YQ<EB_2
z)8ya*?v;d0Pr8+MDWhGYwsfprV#>3GoaG99PNe~_``r7rMFhGO+(w>qTWvr{Ei!id
zdD~DYVYd~|m{&HR`c?KS*4shm;{K=Ur_2`8)L5XvEpuCM4)fYbS`vr}w}>33oN?M7
znpG@3EN?FlyUoTAx2YBmIkw~sUX%TIK#%nGh5?)X+&{u6ejA2?@MQFyL<uP-zJmjr
zd;Y1;>(2YL5siQG`mj`mW4*b&$gHKwX}zBIu2Axgp4(ym&EPp-#ehPbWbmB7{7+J9
zG$5raGD7{z0V%bXL`t<==ryYpJwi(Lmk$s4T*38i_B7QS4%pPmeH7U*QYwu0>!Voc
zm3>?EJc~vK7_&jBF@z9fbEPNBUv8sq+8-^h&4u-i5;_|Ux>et8=6+6)^q7aMsXgP}
zGJ8BkBBk0qI(~)DoX*EQLQV2~0MyG>+s*9sIUY;uHTttLL5%~}nfgVyp7)9=Q~2yx
ze+^tJU9LFZt7Du0qokL^Hn&ZZQg^3bs^XsOr|vaYw53cu6=!>1i%99yx@b_hl<R?g
zfAI%%Nn!1C4zTrT>>mcBpN2?gxzB(Uq92>-)@I$D-M~AT>6YFr2-|J)N?#k<t|%;9
zc`o|2-;*&^dLqzk_yWScE4I1CU@NCR@B3f;L}}lj3x>o!$+TY4qA}k;?F8rJHIm}c
z2fNdGUFy1L=Bp?n!m2#Z5Bn{WLLyg;$HY-f#j}yQSVfPAu#d4Cs0kLz3e!-T&--gp
zmE2kRFQ3z~u^zbH>tn;eJWT1ol}nNSzplfTHxoxIQ={v@uEQJBb@=jP&emD}aRX74
z#;~L2Z$_*bV8mkC7SncfBJ@m{XI_aam71|7K}vxmkw!bYb7#<dFI_f32M9>^88xUA
z`x3^0@g4S@7py<nS%kF``PxgmFFig8sF<Ih%nKs7-u{`8of08;KD+wkYs?+M4-xBQ
zSLtkTJ<Vk6n;H`B-orOS<1?IZbeE>mS&M%QE`a467?k&H-SG^6wAizBO3rgQKmv!g
zNl)D>-V#jMt99F9Hs41{4-1M*G&vY9nWkF}=8Rl%oUw6z&T^I^ka8g{SADoAaL9Es
zWcu}Ok@J(Dk4_fW;Yt=Vc*so&o4tjtC)d&Y4d^%5U$UMu_;%%$;l?7Vh5|IKg#~2+
zA5_*A$E#3zzZaExads38x+CE6wa_T8_$;x$S;#2O*q=5$FpIu*vW>WpA?EHMEEB!L
zdRR2%?=dQDbbz}cg<hAMmchD(wm1?A+YlzJr*U3;KR*$KhrIBM(`^W^+$EoAc)!=d
z)2n0S>z9Su^q+QSSq@p>G)|6l{0j;(rin9L@nzd6_;jAQg&;mw+^L3$Ft5o~uFFkN
zLWw)xJQQt|5cx|+)@)2Z39Ix6`IpYX6Wq+Vi;?j%N83kympDQEm3@Wzs``~h_h@2M
zHp(FFA<;woerml+(Khj7zSg<X+Q_`hT4{>hNagJ7#`BjLF~#j`CkEu7*2&L5AhBrE
zez9nK0Iha$Ko7pB>xYt856StD7XVP|e5vNF#7jV-zC18I$t1hhGe-b>?}y@{Np>&)
ze@wEo92j4?(wdV9wKUbs#%bi|yoA!QuIBr;Z6!D>*9dy+ZEtm9YZzaTNeF$z_Xn5h
zDkMh^aADF`M*N}N&a+nY$1hFzf&B&ScvlhZhj_)E##qgUo=xf)ad<7mmGH9zv8vz^
zC9M6(^vd7g>qYfMi40xN?KSW?)kwLxC~+H`3ibZ0-TT@)VHu6q`^a>YG1K*%P&2Ep
zO=naEeyOxYW;U+t$9zbO7N{k}l@rI$jh)b@eBDJ_kMzod*Mt7U_(ltKzZ&`mhR=m@
zo!gHrEc*n?bl0k_{7ldEw31ZlGIQ@&gU!@G6Zi|M++ykP4W+Gp6ZLXPWPO=lFpz4G
zZ?kahH;Z~3=7}@CUGzBuxQ`u+6FgJ)9cPR17_5p0vu#@6pntI#F3|4eg7q2yn^t6B
zk~GvGLchojR;sm@-f?`RDP8@icW>49KFUGlkZo)JELLa5gM}U`!#vk*l84;4r^!p=
zx0P3%9~ecMPktT12bos(c(eE{>BIxFEhQI;Y+EshHJH{kQU%yM_woa-(;3sjeOwsA
zu#lvccI@V}PYPAfA~1K1RP5?F*xYqTDd(If>+J7NwlTBAuFV`zgBTZ8Ccc7Sw2rOS
z(N~IQhHt4d5PDWlzY1fh+zd|t3{*!Zda<TxuxG-#pz@%A8v0sd{+%Jfaam;uKy43G
zXio50(k>J~6E5B^yO6f@nAz5}jmsuWS!QR<Pqkd;Nm`t-Tw#=wf6?}cM}YqAw9^aR
zo*SKJ>G?%f@N0bI;H|Es^+eYD4RzZ8@bRZyESe1E3G!bzbZV2BR)Ox_jXr*oLF&Cy
zsO2ruTB#4Dyo=h(#pqMaYM}SJrlG`Ds0ZHS$?MKW>9a}$K4s!DD|?~_7bS&<d~=&G
z985}1Ol#*A_N}l97^p}H!{Z!o6i>Vml3hQ*FB^{Rw!1I}Jk8|v%i74r5YliU<PVEm
zIGi5$7{!lNg@`A`%Sd$yliMD#0R!r2%ERiZ?TUBw$)ojCt0hTv*%g*KxGxNb(fU%~
z*n)?YC?QqkV$YZNIqZ3}l5WRU`&+oQ(YttSeam;w;tI>a=)53|UwNmh79=rsZs`?=
zoL`x7^^(^h%C>!~^MT#->@7<-);`>V4NuPt#h&yG^nIs6qx;7a?r<tHnKP<+Qq!2M
zfZd_MF?UgT1(<}d7Jjd{mWs^Zt?P%%L7oYT;zkqVJ+lSs5#8~FP7hrK_tK0tdgfXW
zxw<h2wfHlUfBS>m%4gSSYH#j>=X$JVojL~eILpe`-ez+1-9fF#jtWbSqMU|DzMCb5
zNL~owL27A(^c_iy+Fxyuo1mCtr)645FEr=}LB6ziHyC-F;Ym21Xj#_&{w&P!Lr<1*
zmzvbYA>{X-;+=_?EoxH3SGY^=^c;M56gYl``w94ab2qP8XB3+k@6CUeJINreQ5c6D
zur#Y>kixYXfc!rVs#-*);!@G%jX(30aO0=$Hm^27M?Dqw)`NG;f(1b7Z0AxNs(A3B
z>-8hPzWq5>*ms4Q{m)Y6DP{Z9qRRx+SBO3D)Hu(re#&RUsIn4zUv;5yQ^oyp2xO?N
zR#7GbdzvbSHGA$~r+W6+iQoAUH}J3D9V4AMordL4k2o``82$yj<u8-(W%-i%yk;Q*
zWa?zr)HR$L-tzuPTY7OMRRx(Wr8>EjZkX&E+u62<53t5nFrBjURu|YttyI}($sy`(
zpp^`aSWpV~F#g<bFE8nSE-mES=P>!F+oo~&GstG(SI(Nl7`t)W?VQxeULe~!T$;=~
z<WtCB3Qk`K`UeO4(ylX}Jyj(R(VqzZ#qQnWOEOIk#pe{%vEqR1lwev)5Y}J-7Z%3Z
z4XF8R>@UQ&q!>#brkp6-Sl=q;+Pj^!2$f%Jq+q_`^KMm?xF#yn5LdkGr-wF%^JSWq
z?60-ZeWbf}Z0}w1_!P*g>ivjHLUWm0*|+)gZad=va+hmG(Yr0l!KM~i^uaofov_<V
zHBC!Y<^J?q?C4H57rz5_tI4G<D$arm!q%Wm9f#dm@nHepU?HtB{Z25!)r5o)_f8sl
zbl+3mv^L1E6N>VGZaA<OeOFy^cgn6Z1HBf3dE-*Ppf7@t9@HPuys|1YmT!Q%@kS`K
zEXTw7?LzhpaHzcD;rvJDF~5EB81APX%EQ*NI-|D)<GV-ASt!(A-q|?XWHCQUjwU&e
z3EWfveJ2R`hcgr;xcNLeZ$qK|GU&iK7Z&zb3&riULw&@n!YTRSWSFyTgw_@E!0Q2`
zM)Yob={}HGx5wB<T5Vwq4+iztvLdX@6dgOq&)<AztR^oxgG6*+542q9hsgPc4Xp9y
znL6d=<IH%)dOrH&r2;k&nY)sI(sti>ne1_C8+x(I+ef3MZ(E-$V1Ox+2vlL?o|VKN
z^fC3!`O=g7L&)$Q{-eregddByf<0<f8K_FLGqTUmO*A?S+oIQ2#N{E`Wquax)4L~<
z(+fvRJP7xPRRzobnvZj8=U_YjTvKs%KKb!sVulb`AZGH{UW0^Ms&r$FI_~k8>V>3w
zm1i<j;VRUD+))PZma0RR3m55z4Ru0ihq|&civ@!j{GHz8m4v~{w_&I${^{Pz8Gb2+
z98j7eB?hgA?w`Fk>q{px7kD?_FuvFNmiI(%olx={QO}>ZpU0Z}Pm^+ezvuR_hxk-<
zrq|}z=?KjJ6GW1fhpTM#IiDGT+&pAfswioiD>l;+Je&C%)bXHvqm=3)HQVA`M%?1%
zs5tL6b5s3Rar;)?;UB|(0Oxk)d&*JzS(S|{fMERx&l0@Qq_iADet4jKPAfdH%wwyi
zUN-h*Q{*exFDmxfGcdtzcE(Q_ncnEo7uSVgs7{)6j<fb9rRpcF))y825Ak;qdqZ8Q
zUf1Itas1$r_=~&FC01wcgy{=1ZmZDYFet1@{xL|YXuHBr-m{`4PbMi*W(>K`UF%n{
z>uXn$Z{>>d3(=8(vLT6yVKQoj)4o$kN1&~LbdYzn%^L67f4S}%B(b4dSUH;nyUOe3
zQNm_!-HGwJ6j3|0g;-yGKw|#tiefdT8AcT;#Y|QwzB%jOX7jq$5OQ`==#3rd>mH(D
zd&)M~=b9U3bJs;Lgfr@Jh%z5`(wHRw2;>Uqs#mEHgUgzuEsN7=ui&C}QR=ckKjwf0
z|Fke4wUGpke|)m7xAI`K=Az1p@{FR17a0h=Rg!gPdBMYz?~JFrA81@eg}S!=arWKg
zP??!`YXC*%eO4z9^j1i&=$xgMfPvb%d3P2ATXD}KwflICErKXQj6YCu+Et8Gb3d;%
z;sIOiL<Hr-z{KLM#ZanvB)8YjIAw-7Rz6o)ugJbC^7_AvX7*oimpwY|=+IKDlVo7F
z1Y6Yxe~+58;!2cK?KT001%C4!T)rYv<JqY#9DrsYbP1!i(Bw8pu>cJR<|2<neGDsd
z3t;&J2lkLJW)&4y^SjJbm;Sb#&x{|x6+4?Gkd4DC6r;Cs2Ua%)_onscT+lniz;hm*
z(^@BR=sDldAVygb%j&Ak$-upqtqbl9gZVstFPShe6|&AV@j)$b>kil(SUkV;C2WT*
zE5CL<Pl7MEZ*;Fai`UD&10(N0n+21cP!N}y+p;5cuihC_%2fk?#Aa?~HI}4=3OfDQ
zhHC<cNbQJ4@r;I;)8%C|5}6#Xi(M)Zg@^6Zlfi3^{9(awB|m%qc%-@5D2;XR$=ze0
zchGHewR_~&-(t}7O7_k{PhrI)ue~lziJm}iWu9In6(?gso8;gd63pdkdbTS!y4#)?
zzwLy<>ybvId+x-YFrt<s;ipEFRM|Mg^H}s7*uf-B#BhItjSfRM1zf|=ExTboN_s`h
z@x#lKY5Vh<mGg>H<>R-W>jndGzP)Lluq$M+`ofMd4nezm$F0{w53=@mF+?}0Tnx$l
zar9tulyS)S%6>^|r@L^QB&UzL<!aLI)kF@4w|&1qbtNYD8&w#fz~Q(duy8c6KfXfB
z(RdA1B(p_P_4t-7Z(8j#$r-=pxRJjq85<c-JGnTwuj)dF!H-bGj{1mzU{tm2>y{X9
zRNa~Ro|^XXSR=H_&#(^YnJ}69*MivhZM7OAx$@K=eNedq<80pIaF=&K!_?7KvBmY`
zrY;>cz;QJyg5`wTqdrvWEvaAT_z;}2{GmBM_y3<cel)6QrWlyxZ~lW{Y&~lXXtlG&
za}6;*hy3D;k-b83TVOhKzex_45`jQXa*>A8&N4Idcgjw$e0V8xVMs|_`@KM+{mq(0
z#zN!}YQ0TE*O64S4Sh`7$5#?PTv$RE%*61wTg=oFWw!cE@wqBjNmW<!Vo?JWjgrA-
z)$X~2PNRn#sQUDDOhP`si+n~A^a6g|&AgJZwXoi=d;3jib(>5nF6U(n$i@2~%NF#P
z=X(G^L+maS36<N5jS^+`L6uY?f3p19VQ<>Gw~#p+wH{Y)?E4q(6&5Pt@uP**9nN1b
z_&bw;*ipnhdh+l9A&%exLsJ7s3}}D=&J$*+ewqKZ+D-*D)(cf9Gsev17fuN$?^I97
zZ#*5zQE39{*<rH&>14=1l01P^;g%)*h-MBF>9*Sr2xF(-j2=#M4r-9ZAtnbB*8U96
zOwkt+HzAI%2fGTcBnQMQHJ^OXYhZ*r*V8jXF<Bii6Bxj0XT*zCw({ZlTFLiaC7!b<
zDbgc}S8ORcm7sE6wUIx-80{g>-)#XcQOM%*W3cq>_Io<%wF2oiSL2~xWwMwN4bzlX
ze(1xO-J@gk{;5>M%6$B4*`CER@X;<^EYOW=fP#jH?zl7T!CTNx$ie|U<jG-oc{dro
zm+V!3Wf~w@TWv)7W(m_xRy1)NZODs`Lyts-fF7{ftwhTv@d`;xSq<PVvcJ}k^(qiO
zo)E4%ped*C{NvFyC3Htg2Gq{7--D}%{#t((hWe-!d>=%KKcn;}a$<M6*Y-Zc<^9l4
zZ#+vFI`eF-a;1ijCh(CU2IV+@%Fwh4I8&mxUC^46gEqMp+?nLohIBo0^(mlTte$xC
z$Zjv+syEDso%_%9W+#re!Y(zCyjnwHRV4%vk!3#HS>EF*^LoC#$pOs&-~Y$_zlgjY
ztR#oTN`SSEVu865UsP>z0|R=Cj%-t|-76J;oBtaGY|hU+0rUU$tUP?8!0c=KCu#6}
z7!r^)e}p(AA^I!Vh*?e7)(2drJ}~$O1R&S{%k9qpFSi@YXyEWfVGZc&wbOLf6kO}a
zb1O6ds!=_#a=z*Ho|V9~Pg}eJgpZFF@p$_n72HpmcdNVOxv=c+*zJ#p)b1jei9o~B
z(2J2YKIS{D(F>WCUMU}+m0%6{*le<@sR?_A#7)DNx0(k1nSx)H<fTc~?FN3iveI6z
z(RFGY93`E~?p9%?eo>IQb`XfZkEvoJ<(&pBli#kl%OP%rx*zP$x?~3A>=419`dSA1
z`_9^mTiIc6L5DQWCZ|c@*vYUQqiWwZ5M$lF*=ubj!xZXmjWwG719cV6zHnyVb)Ps+
zWET45na}1U-|GSvwNT0|ntY&&2}AEsKF2217-lgl?f}7Wf+~N9eM|dyulEEne8@`6
zX?{Q~X%vbRp!YcWd)%!$5d_+rB8#&jr%t}({?`)CQ>3B#%mF{f3}knp2&WmMC_0WR
z3<C|VHg7g1dc%#~W649uoU~Zrp>Exu5+t%71h`s>jqx{z@bw&QUD9Nv^g``@9XRfs
zk~vHmt83s`xfc*UQU@lp1cUTyn+%8UZ=vB>FDC3XFNv)V2W)lbzuD?|Ky;#6Giv4a
zrh0k}Ry^Zm1Th1P`pDFHWl3yxYrs~wBeB)XG3@KBUs?RLA_5jE?@9GUd!Al$dm`Z~
zV6q`De#lnW7YA&02$RPvrQ8*^%AI3dW&%TewQ84}kFn|9Wfe8RIi{?0jqcQ6M(}bc
zGLKU;8+eF6@_ExWvcI}g3vrQZ3QiWOydppI&UCK>0Nl^~$PIsv$6(Ts!`G2F6-`1*
z)1VNK&hGLAu~ZoAx#Wc3a~bt?8Goxyi)NPD=upAyzU*t@v`*{X9gwC6-4uk|@Pa|_
z?f~pyFOyhe1Hwl7PfO=LM;1mYh7bV|a+SvV(L@5xUxZ$8Wm@80Mr1&5k_4^5K1BwP
zE|5R;GfJ0A>=ZbC{IN42+H^EPg!b2aM`~Co+AdwyJ~}f!>{)^pAr^CW|9b2%o`E^P
z6VVcTnS#Mk9)JvVPqhnije92)YV6O}@me=fc}(~clVgzC;vH_F-znT|k?1fFiTVD+
zsJ?egHCZHk=M8dkWDy&q{NEi2e(XI{4%C2tFAwPV3!LA#`FKn5N9gwgq$LRXhf5IT
z&mtkemLPEdcL{=%8d!qB6mBY<dsVpZ)WoT$Cb-Oat*y{_g_KgT1OfjsgOIMR=1E$D
z;E}^9Qw9s-vS`=qp^AwroBGoXzgP$li{D=aMaaoQzd38TU6bpA?&v_wiOedujZRe>
zn+WZdk_4&B-8O|+-nuu{Wv1UaygkD<QJ4C)M(x6MsNB-KY+daegW;fOZAqbaJ7Kq3
zv*JFVUF%0uIXk=%ePf{-zNJCj(D3LDP~3RG2HGT*hK0?bGAgf86HQlQ_d$Kjjs1c;
zSjQ|IEMCzW0U?;_8Q+Qg8tW#1*zBGQh!0CMJiV<LFvM#9#%4GaeKpbYRnSq%K-HZy
zzhKbW<`kZz5J~TF^sw?KvZCRzK90fs5WAD;voYa9v?pg@JDxhQN<YR~#c$!cb|MfM
zx%uggaC*0^T>@g__^-t;|7>R`?EEorg7BQ5VfJe`LeT2bZkf<~w(|3yj+Ca-)m7D(
zEACd((T91iyi^;+yt_sBJXvL+tuWShan3H1(v=<WWDgcD_#IiekWT~UIh`Q8Rc-ER
zyF^;JK)p~~lm&5Hdb@{Gy1~w&@gw7Nh9JJ6(sSj#_?J&$_X;yMD9l=~AcZPMJW>Ko
z)&5SEeHt#})KbyGs(c+~(-#pOd<wx)DDi~7#YM@orGSCSQ;ob>udUqDm7tl>!Uipe
zU+t)iZZpB+?dRa}&OKU#(UNkR8XtNs96PVGG00;G+lFh-cV;}tD52*x^_}0g*eL>J
z+{`ty;!i#Z?4mBse+>mSRhW;o-R%h{>6kljlvF~TunL7~6?^lP(^&xY8ggd&&^(DA
z@BNF%ZR!=F0+g-Z!Y_}Ot>#R>Dk5rS^`A$7H$z40M+7zGb*qv35uMJtFBif|xaLI3
zHO{9M+fl9*<kSXLrp{G#-j#`v12Qies)3Z#SY<A2YQ?pVXl9!gt}mwOo}H#-m?sXA
z=7o=V+4s_Q!AJ!crmjc{^-Nn<OigJ?uJp34y9Ct7>8e8(glrA1bFz<&BQDNS)C5C}
zs1kuqTdrF@lLq%H*p>H^(x&=*eXy2CUanTG9yA;;j!o!x)L42RL+^OHcIB?tFMWgc
zvuL1iFdzPhzJcbC!5HUO^vHTb;R*;!sq;CnK&~t<)vt|(kswJV!gE1)(yslhs}4o=
z)AE|2m_=)^rAp#~<KEBYFb_jg)tRETwMv2>9`{ucai_l>c9cK=dC11O6#3S^rA^1$
zLHm1RoPEu4v(69K%U=zm+hN3ga>rBZ2Q?Jh{>7*4*ajS1E_$$FW1T_;hVE?SM3twx
z+*=6b4?PzqzjhPxAe!FCU-RDTz-MDH!0>23rKxw%4zO5qKn{KbdWy-Fr<?k?5M|kA
z0PY~Qf$Pxwz$5;uzE$*pcO))#oc~uxB8}9M&?W{ZApJ-kiKxhz^E;1t?8(O1>77nx
zRkK-m)}8Z`X$~F+13HpIL#9@UU9Gogs~3E9_!EiyG#G~sv*isXEq;6Ufp^9{m6uzI
z9qmIs#%llFFHua|FEO5c!S1!}9Br)09Ltki{PlTWPn}Rkb*H&2{){o==QT<R!stEq
zDE=H-d{R3#g~j8wDUZ3+fDV(zoC;%2>wvsXw;)Zm;LrMs)<7{Z1`6YzDM0HZ;YpG3
zngS$G>_uyeHjt^rlvsDr#AV_sgKTjF(m<dEpp@YNL$BTG+e0GX<pdK~F2sXu`pvVR
z)Fxaq4<A{%(oAQt;=5MER*@Cv`-FnrL^}`!&*P-vIo=bUZTBoX#VK^lO2BT11W_U_
zcAtx)-rE3LF6EZoa+%ZEx_`u=K|7cbk>|91iZak4bqjs;*kWx9w6{|)^|qQ0*NAga
zFl-iSn}1C&xXM$gG&cT7<082TRg6PzRMpW(_BrryWGjB1FXCS@HI8&xGDO-I(C1${
zkvzCCMx2RyG68m5jQ-DV3u=_FWPpSo#m=$4=#XjzgR@CfHfy#mu>`jkRBLBDU-Mk4
zV^Q@YE{U|;LLBV2sEFLB{r{+Yuc#=qEDV$+L9(diBr3K@k}OaKq97umB1n=bpcEik
zLP1d?6i8B0kPM;%k|k#lBuJK=qvRZlP!!zr!}LrK&J7Ru;og^isa~sl9nOFD-rxQL
zE^jrqVSjOF<&Ad`n|Y1cKJ;W>?zHu`IvEpmGM=?_nCR)9C|YuBwMWjh1b6A>&(-Np
zg<=fj2gy81;-nV3T<~cFWM=GEliV}pQq?=1pU6xL&#tBq?RQpN$ogMPn+@sZWc7TB
zkQGnOw7rJVDgXX-`%CyT>Hy~>Y=Hhmq;qrZ>{|fbxBdP!J;wUjrg-*?{pt_KkdomR
z>1BQF2<iZ~U%KAH_k7S!sUlVxd~1M<1->;oN#^C${4eWiWE?I=uDF2_*xk1e`OBtd
zmzja3Q{HU|?ju_m*_)DRoQ7=`5e^^>)+4_oYII%eq=)bA<3|q4#HeQs@pBIwBN6sb
zJN)EycS7Mi4B>bg9U^LpnY}JUyk;L*`r~v}G`20UH4N(d<6(((Pr`h4k$;SnyhoV5
z<Htzd<G(fCgvQALb!*#J=tJX+w#oEneKN`fpg)m40ul~F8G>74<eo^Xzk@bbxzjGx
z(FL%xn#j5t7r-3`I3F1fgs_o%;;uc^2=`XE4_vJgS`bwF7`c0`S9hPRXtHg@b8>()
z3TE}n+1-T(&k);J<4Eq*8;zl(o1a*318?e>#i*$q&FZM;;-~cA=sXIN`TUA&BG;px
z7DJk4+)OdaFNT-W1^$(^AqJ8*h8R6-@^J0u98%<~)xC7yALfU<lk?L-#!w&_{veLR
z&o@dOeb#<(0Wg+BwDgVnBn|OF+iLsHd#{&bPmzl*&Ld;j0@WwNJr|z7*F=(h2H?TS
zrGjC%`R$XP$%_Re<)HHsoShIXp7{vg!h((YhzAfM=F}72HLAz)9ct3n7ej>17hTt<
z&g>Sf?X;piAN=YB6QKr#0~I6yoT%qz-r?*CE$bv?cV`lRYi7D>R&c291XOGXd;<?B
zO(DR19=<a?=tl{L6b8fhx6uQ3g%_qktIS!i*FO$O9!myNdDwn^y1Y*-j6yb#cEKj6
z_*0H9*C=vMBaUT--JwbNH=2xt32-g|z0zp3>n^lx*D16$W<4c)96aF4B00JWr`l?c
zbMq@tLUzUu2WZf`f*3l8h-1NyL{r~W^d!_L@ZIy$!n6WBNP*4p$;XLs>APJv&3CAX
z<2+5W?}Z?XXf_m9xR=Pvf(p)h;`+U{=ZXXuY8p&|+`rLnazbN4nbz?JD=sh2{a*<o
ztu?B;B(}d!MBbpC8Ph3<!e%wqx@rG>Qr*g}7q2scG5St+ln^2eC4|6$n0P5+^qIqI
zjw0CblD&*6Cik>lm(TK*jaUs;qC)uM{!R!fF|-`V+~~wggy{g`4}gCvPJvDqKzIh(
z)zn-+pM%T?`^CJTVzwS+4ZYGkuzJf4@t8@#6hEAk)B2Ds7m1dc5ZtfFSlNPt%D8VT
zVSp}d8}|I(%nLSt>n<Hhr>BRL0_3Xs<zL_Fe)N234O*qISbnPicGD~BF~srvwlftK
z?$oW5v~pXRT&EP|4p)qEuK&sEGuk=oMX!uZzg{~2U0#8-i#q$QMqi~r91V>6b=6Ll
zWUn>6mKc42OAbdmv4EGMjS3_;s`$6u=zfRi+UY=3Yt)tk4W2XATlbZ$wg!XEi_MLq
z{PIC^(Afm#?R1`_6^`4F4R@(wSqeqo<lI?^h-|`I$X>8x(D>_WDYpbj{16u;V57Nv
zqCr7~fzOLW-=2cZDRZut!8?RwGW#qO?tupH_CiRNjQN%#l~eM_TODnlSRXNug^<=f
z3vo_P*|g?BBZZA@3FWKUI(6A@Yng}oR@hST{9Q3n%1L_}C06)Z0a|nwvx@15Sa6gt
z`mohs9@*_KxOl^j8|=`qd|)8svlL_~JA=7lc~?aWC;}S4eeVMAD<j%h#44{#%csXI
z25^Zv{&M&O0f*wQI1U@eFPQVk0>H?L_%1>HBAyHm=#r(uzp>W}Wv|<l%Mg&L5k(Vi
zlY_nT+WFBFbQs!L0_X0TwR2zl(`V5@U)EjwMLlYdqDSmg$iE^ya-ay0ro#vip`!?o
z-X!<G1xpU9rNT{A{}e0@$~g7LW6aDG{V?*+XfeZ6bi7Y$bO$_G1PNYL2HsVP$_B>k
z^3=ZgMFiY(+`IDooiuTC2S|=CT_U@hFfna>UnN_yq$fWPr;~m$7c`<MN6Y&G4<)Vd
zK}3Wr;2%#vzyC?W8-4e~8^FY=gZM?W8~%HCVCfhRc3o~<K6=*#%D2a_#u^UGPePkT
zQiI<L!dav5<<(^f_a?$2$;txYmUs{q<O7hpG|XlSs%gAv^Qq@CG{-L~fU|_cA#`hX
zj|vvv8c}w9Ql=w`g;0-JnuNY(fO|xmUN#AsTOnmBB3>ieI~SE#N>Glh%Ls}vw$XI|
zcl3tUvFMGQe@AcNz>d}&*I37;#VT^0O_mNc3Iijh`6xYa?e4|$ELH6**T!ltRt0GX
zr~)CB1cf9)eN|wpW#+$xP?)N`9DE8UV+d(p&kOP^D`Eb1bqco2FgoN+=ZY~fL8xBf
z=uA(1WKWYvw=iXz2xp40P&a8>l>xj3wE5V#^KG56L)4;l+OHus0TMrN2yC$SvimJl
zoibnO0x**FmrGCw=(hUHmdtY>4z2vWXp;rt1rM+iUMID8J=6K!hm!Xu(zQLn-~rg@
zCuMMWsDoLVV#HS@D}v(6-e-^x#X!KH@V^fF0?D3^C`sgBRUamgH)o!dM{AuL9Uz-Z
z|9BE@o;zSRO2TDNR9JNrO*vY55iSqcKVjWuOFkpd?@G0sJ*$wEX$L^O5+&s1xt4UB
z4A^d!HJ4$Fx0s)NkA;BUQZ-w5O-VkfhzY+FE+@>ied=4LG7wEYiof`cnRbw)DX2up
zEuP1Vi~G|FziU+Xc{vwb*^s?-oF@^EJ=lx2vAzAVGIw4U*UTvJhUrd?n_36r{Sg#`
z8L366`tC}_a`(In4}FFXEbs6>_ItiLdz9e(L&>h#-+&9{1O#TPcN=B51~txJ(Qcb%
zz!R@ELH!X?gt>O1yl+1Q2NeM}>&a{|V%YT=c~k%V$ye|?`0**gSDX%q+TX&?!tTW&
zNm>BmNiF7f)V`Nsm*4*d{6S~sM8y6ZwmbBeHj+7_EKSJs2_HZFJ#mPHHi#+)4bpn#
ztC`lh(KBku<>luGr!=+KJ>Uozf|WCLF2qzwss(%IjW><qzkAcGuiX3m6^}^ttHv;i
zd}CQqw|>dwxVF%$n?dC1>lE~(*3%CV@99n1@&fPGSp++!rc6JkQ}O_Ieups|;|875
z|BBh5)~!>}8&Jp-$>xYeIuiUMy;nm#QP(pyA>mnAfiQn5GZCS~vz`9baAD&D%^3H5
zb&D<>pJEIjXToY0REX?U!@7Il=U3oz1%<bZJk1(!$LrYkJbG@dXTJ6;ygb(mpzV(;
z0QIxy_j=p`(l|Cy&JRM_8g&urVi6kx(}1jCwgWa|U2m)}>P-R|o)4_DZNh0NrXifp
zDurTtN2L+Sg{U4K!$Xe__U+NZhLQ+9{o`P}kGNp({_Z>{s;+GZ8-Aoh61otQi*J+u
zI~*hNtL%{;OXx_C<!cxp!eNctTdKzAm1RdDP!kb{*%-J`6_ow&`=ixrFcD{~Nnsgv
z&2a4~rP7_+v8cCg$gkssGHFKNv<hhF#W{w&vqQv##jexj$;e<Xw(NEjIQlSgzSwCZ
zU!CbxU^D_isl)u1!c76^)_@jf7uo+(V_89JEE%B2;)_0e1>9WFQ5cwQ4L!HujL~;&
zzd6JguCM-hN<89V&2V3(4@41Md@D}Zi1_5P$RJU7N3q)wg|}bGuEVCl_g6z2!P?<h
zriCtG@%`@wJ#>;sy@KDZspB8N)`P=^kh&+d5K8kKzE`K<wDJg!Bq6<~@R1y3P`uVT
z%%DIVWl&Ud8{Ji3l4nh|N!y(9saNF!E`|en(?=l`oB|}*1;zu_tWEUx$vmxU@0Fxi
z>B5DV^1hZCIT)7NeLQ*M%jNDBp=#Eoa^ab6!V^HKZq(C6gKq)>6o#+c=obQwKT^Q9
zZlvM!c8s(O#B_M9JviUub>^2}%yVui^#-h3r-SmfY1Oh6dToj&ASBz3T@SysyuWmn
zx6LZIUGaB4&(5^x4IAL9v<BgUAWq~N*wvfGd0=M^T1A&W2@@3&P6G|sm4}8NPGe4L
z4D|KV?#pG*<JN+nArlAP=F{<RU(TSOG6U-%o|xU^cMg;+uEQ2M$nWtVHW@b|F>}$u
zB;X$Vf`t37Ra<tXUGvu7NhHBbX3jyviovk$U$ya#rfmkQvJx-)ZOQ6y>T58R<h9dV
z#)DE+oHqO0Gdr>I8Tj3U+6mV;0_&e`$RZUH?o@ovFYz(btwgxHrl<Q{9ju$z{QmW^
zlvk`@!lKMgGk{8q<E&nZYgLua6--S_vw+M>e#mB!lWHmL`v|cYYFA9Pv*>wOeYG7@
z0%FPzRaz`VMPG@_0v%`<VGBQ`8m6QBwM{{*;LheUO%G;S2c3%V{;H)*PMLhrlKXCB
zH7#6UO?I>MJ4(NnCpDwk5)!NE=l7XMefB*7yOPVLv9k#nj>`ZI(A5E~!$v@3?3gvZ
zFbuHsq4{Wt5R-j9EQ`~**Vt^mSz>u11h)I+d6@rO+ptU52)!$EsnF&`@fSq@m^tW3
zzPvmF4=yl;{=rLmMtInbVI)B}ru6UzW7p~v_7%GAQ^0v4nyh>aLt8l?IZ85Wnrqkp
zwJ5+MY)cMiqH3C+&<dMsZ`@zu=-;1Y;tfH&0ZZK$tVA%A=|F@_%To?~Ll>;+pi)Vd
zEU&-3`y!{X&112(B;l6yXM@*#l2rQ1?mQAFy|h?erKr!hIJQ!-`r>wQJk7KorSuNg
z$fiNC^Jdf5LfX)gnhV<=`x%U%q?CK)u1oHDUpvA*{*GoqVC4gMu+QP$pU!?WXI~Gm
zqZSqA^bG0%z>CIG>AIS}%6ki$3FhBz3bgzHd+9r62efe7MXR-}loI`N7ny@<zWut*
z_dZBsxX{+U8Hf+zbBkhhZV(=qQF}_&mx|5-fT<Fv4KDcH!HUNud}GkvJ$!CS^j;ii
zcaQ@$#mRqBZwZ7`j$i#ft)u5Q5PEK@MGrYXhf=R;m8bPQYbPd!Dyd%|O1;wBBY@Nk
z0i<3QRFv6oHQQ#N{eM+>6+RZX_i^!q>9z&x21M$CfaxMOCtn$ddTz`{K^apGwa5Y3
z3+Y18&D!?F9?ldA`Up_;_}mw=m2?1?lI(4=D<ILohoI9+SSi=@R@tqEW=4Hmd7V-h
zy9$G|fN2lzwoZHu$MGWtuL;57YoZFHJ^tH&^h1Gi6V#$14<3&Za;*?BrFF${zpJG8
znm3XWK@v4w)LcZ?7+GB8<RHwRGFAwz7FZ@QWVJw<{JzH2YQ-r<Q=`|TTSM0v32XA(
zY2oQkY8lS(PpGq11*rx$t;*^q%;~@rCZoaMNL9f7d;qn>C7&Q&Mt0)~svCFx<w4Gm
zaM}Rvi_O_8@0^Pclv<Cl*QrURnragU;tb}X1WwbW3aPZ&AFl`RcX<wBK?Ll*tb3GE
z*%`ATK#n|wwXjC{iC{rgo0Er62Xo`Z8MVCU5KV&2k58q<$Nasf)?d)ht|ZE|Ub7bm
zZGf)6H|&-F)Cb<=MyR1^bQa0<dU2x~5u+FisaMzlREFnEC@h+yG~B+v)BLPIWc^7*
z2Vxy6es=^jQjV|GpyFo=LB$$7uE>SCRdD9VpbOx41nOptg;=+m1vlE<Zir8!jSJb+
z%6|&k8qz|g?!|3KS~5sjo@QQ8ztV^>out&JwLyzpa@FLUP&vC$Sfkv98&%sBgYwV1
zZhZ8;Ze2u!?WqHP6B#M3uMdpjyF6)UcHOnMOyDLnvu?TjHGz60_Jb6OB*i-BH0QVb
zl~=j8_3mbyGH}uO0w39G;sUWCGvIesy_uTIV6sQR0<sWIJa+k(>SUi3Jd@TN(VokD
z5$Coo%kzoJ%5T*gg{_Zdo7c)qdjz-~SM)g~+Op8MhyMciQP7eXLKc;@?S@)V8%n6o
zp|c(Ro_iCV;VOb<j-3z1^6VzMp7$DK);=<pCChc+;{bFq`)(bJGxfsNM6>2#7dqbz
z7BiYffT@bx{C59v!e~6Xx^XOl4!XRD1M|f=pX0}?<#^a&#Cd_Gn5*?o$9wDtJLkW>
zRT?D`niLuJ<*Z~;BY43%B(~>;JtMGovZ+el<3<Lt_@4C@!}uKMdKIHuMfm1dEWQCT
zuNUg#h@boggvK6U`|HL`vm{LFjkOzPtedRy#S2?EX-tBPWv@+xe7RXiKd0W;WNU4m
z&l9(0UV2@f!05k-qxHJMKp>wsPiCBFx$f;-v8e7b-q2;xwNy2lQvs%JzboWUvmwnz
zUmtM*gToT|Zg0RYd5&jyXNS2u9&mlzs>_>R;l9vj2c};}n^&!!YfA&R6`<z`xdOQQ
zX!k3S*MLs$lY144&=B%76(F#L11<(TqOXNoeyB+=cD{MizXCW7<veq_`4BC%bbm8*
zxj}qT%K^Ye>N98VTbHHG98MZ3@gM%M1p$-Cha<sZ^NqVIR>y&hil9TuVcZ-<nAfB)
zOAoG_^CDXT1(E1{aPH(OE2_eiHPsYG)Vy9m`eeRCp61_<83LO@-f6GmnVP#5=?M1S
z>W(K;6hQZ+E{i<rY`;4DKZQ^2lwcFmy~G^lrpe1I{y`smD~#)IKvgZ%D`#^W^b>{$
z)=ix`q^mIUg^Uy4*S6EES+e0GuH9|JQ?sVE%id|01<siu4FHyAj50@P=C%Hv1|T#s
zT|EnnNKp>P*>fT^2f{Ybeh>)<EG^4}wFL`!3AFA_jsYTO;@=B7tpXu3-}#0VCqZ#!
zcX476reqYj0>~Qh)DA(&0V1X5W4X{lyQw-f@L5c}SI4l~*H^|u0ERuF`)x`!-al-a
zuc&l(@cq$tWSJc8m(qIUyrW^qf8PCJg%@~(&dqV2hEVt)=*5Uur1@yn1@c3s(Mjse
zTW@+Qo4k6w>WM3TZ=HXmVZ;V=i1~dc*LB9vwKM6g)C#qB;zR+NvsGHNf>M88QOiaF
z=?>O%{;s!}=N1A6Ba!yR61w`jU~rFBV-vr6P+i;cbInjn3*8RA-of2%Ji&H)m9FU)
zrI$GEi;?Po+x6E1yZ)A_$J!-ijwU)d!tAq%$C6>OZ*^AkR<Yqt*Qu>)-Xt#RtI858
zB3Fd<8B94_fv}^4jMH^}uDAHq%@%zFiMG!D7&eztv{@bdb4MK;fTpc&W}?b)wDl{C
zBI+`$H<Zkwt=;bStd@&U2f`57|81#w9{ZWQPVTKdXhA9<vNE>(bD7&$+DLh5e`pl&
z?s&5dVQgGNm%+;$cv2<skK!jse(UvE{Y*=L%!O7VQ%dv~Z%eH`VbJ5%Y27^w(ZxsE
zDiNdZp101e5o9$GGVQbl4H9^dL>Xl@`pYGqP0X9XNXN6YKb=^;*Rww*!fMzxtj>nZ
z^JVlXG&iSHu(r%n?04H9WW@KS?wwp9U!8K-G6!ZRdC~Ol6FC*`kjH)w@Yp8<9{Zsb
zR`o%W6W5>3yAycWL^IJ?mP!##VCrSio7_}ejG}94X4Vecsg##Q#+&w&6Gm8I=hw2w
z3SzUnI{O-w0uc@i>Qu$qE!dus?oowritOw2)S{GO)6}9)v}QqK-Ip@mhvosmW9z0k
z36!1uBlxA~jo_dY%hVd8XQKgi5V!U=y!4YGh21nn2nMDJ3Vq;4yGtL~>17$o>%<mR
zfcPAE@TKF22o5+a5F&^%6URCq|FP*B$@%&$VMmL_7Vb93wx*wObD)@<Ir+=l$reN?
zT_USFUA}Hn=~@{~V+<U0#BY=!%nONYE@z~B4}nLx^Zo*DQk9J5?KEyByO5?{=1FQw
z=dFP$gu}&uTH;&GXK8I@LzeiFkR|@g<6kRiA%`b(B9e68WP(hZi#Mbxkwni__tJA$
z`-9A^VRqQM7UD5QMqGh~g<+b%yi)XlL!e9Snoe~r=qej>fc0OVME|Hnc*K#;W`9wt
zsk#7k&WDG(1rC+d449Y*07e+xQV`%2LzSw6k`Qy(9#8|Z!Xhbbpl>q+I>}`{Tgqmz
z0ek4;6KcQN^)MW@6OOVg(l@6NZ49l+(Wi<1a{SDo106il(T5H2+?qIE>C7pJPCf2b
zBNxq?Ot@pq3i@P+$3y$<PBIV3CQsKa?~T!ctej>!?dmhH>)JAcjBA5e%p|T64--Hl
zmQIE9&GDg_5MfBeVxd_7zcnoRN#&KC{=)giZos9!h;z-)!B8`%IF~izw%B}@GUr>2
z{RHr--y?VKX1gO@O!hgp2n0A=TU&LXBYS%i_|yy7)so%f<kZ!4f8Cy~QRne*pZf0q
z_Nk}(+o!$=Z5{w1A4yhnQ7cc-*<Jz1M>d!IJvF8rr~rfJq9`T|+!r;SM?AKKRSq_i
z;QS-|*aF-ET`;p`HO=81O+_5D_CJoY?dW~oczxq|GhuY%XpkO-M7$qf%`7^toa8NE
zR~n@fnk1-auH<dF2^T>U0ObHBlR~T~VpL)Bl#$Yh0nhHF{Gu9T-X0f^$&b|<ugR+;
zU96fGhmRY(Kl$6(ed`&`yfUTgm&;bS>5@M$)ut~;7M+=p&7$=x?hmw3ErmHD=r973
zE!MI4Z0@x76U;<ZoCMee>I!ccT<Sn5)7Hxn7IJ6PN2VHWKkV|p7T_DITUTUiGBoR^
za>mZ6IdPU9+Lu6WR8Kmz_+_*~>u7zO{Sp<%6D(=c>HOMNpmgOsILxpDKs*KO*R9Z7
z2gk3~i^^BthXZiVxXgO=9|Lkb^ocv_PrY@5IN9ha>@}biHdI^(;)|zi{}Nx^QZuIR
zA+W3U<;)-uHzyeq%h_DnIU$)nV@tVkuS=lS1^ATcg=)WKsd;01@D*#plXPYxUAF1K
z7Bxv{s_uk6G+i%JU#Q74H%zC>)EnuTTO^DVcL@Zh>s%ZJQ7pq*m~+8n@&ibD3E-|?
z+FGT^&vDf)N?ny4G<hUVvE2m~d6qK-f19qC*Rbp+4veg~b8^#G$PQ8F<ju6oeP}SY
z!4>Rmr>MhjM5S9a@<q=Xxb0NK_o^XY$OZx;aArcb0|FLE1GD#<KOv&vTA;-WQ|w!?
z?nynAo3GaGY;~?N9$vgP)Y(4y<4uyh(M|b7db`fOSeF)NlK)0?E485(>g-jAI>-$J
z2m}ggC`6GXty_C*JRh>TXY83dQj4Prk$%zS>;8GM(HMcO{i^U0d)i5+yPU?r5GxNq
z#&Ne1>Cx4>&QovP3sl0?RZ9la)uH08N~$J(aqjj9OMUpi-Ol${N|LmC4jxDNtDcUe
z$)?F+H6gW<;9q$m$|5;wTA-*|!qU{qoI$89L%!gLlS}fI{jf@JfwJ#fy~nvqXoSE_
zKSjuyXIE?jQ|Y~pc(^N7t1-%|#@*b$Z4oj&k0?Z&N8OI!`Dx|qqcNl|2+;}pzzO2S
zege&@R`~&bIUJIkfEMPvEZtS>fSvW}aG)M4NCVp*VlnV!4%<UQeI*Xp$Cc$WjE9}+
z@14`X{&Y*tGT}a?#-(Yu$|n6G=~af-k*7k7%p<@WCy~#ypl8*H-E+A$d}79)=))L&
z<q0{ALnRevwEbG{!@LQ`+|^0$6$(0gdixdnfMfuxDA>Hf`{$A3xz|3JUU6Mx>zCP^
zyxM4*g#qr<tY_!nP`Y9w_GdPfvLX;NBF<zUZ3FvGQc?+R4}1VY&T*o%TP;LutCFL(
znjSO!;-|caT(?lHBocPsJ+h;AeKd<_l5SwCFEOUU!a5d#d-!mqVzI&d0;zDAK~0u3
zN4^~gGB&_7joHuBQV!VJ<pbiB>N&pj-1~jt^nNbmhnItxBH`Sxkjw5Pe5Hy{i+@Ay
zpz9c=p?3a|^T37Mg1WiM`t}VMeve_eam^e@P;IQm+Za$&j*X+914o2!&w)6{mOo-;
zJGj9zp@Ul#hlk3`gPVbZ-*@guF5`BB<Cx!Aec!_OZcOmB5Qca$y=0T5^aBxGH)*n)
zw$Hd{)nc?&;&3dhurn5%<88B<ErGWj6P!!n$Qq&K)&-7q6`6cv6l)LmouDyNb<sY8
z9!DEzQ6r6p%MOy<4<p}PA(~Lg&c~i*qFri|m+7=6UeBVuPuZl6Kk<fR4oGbB1|d^(
zxaK0~p{Y3)Fg0gBW@^6WjVcsJ%nA8n<U~;7<=<15QlY(KF{CLp8!mM%Q$6v$o))!T
zZ9fqt;at%1ByI-GSA2}S3xFhJ23QpO=kWf#v6sRhIqMXGo-=Hvne+%b6qgBFI2OGO
zRcjNjqdUpeDxE|{^xs9u$O_y^X4a^pap+E(UbYFC8-0HXnc3<^)<$D(Pddsps<cu?
zZR?x-;vd}k>6ed1|Mmj-|00jx+jyL;XfcG-Zn=iO!kJyK$(cxX$W#ZBo>#1A46s+S
zFqQg=5V!&0TI&OVv1JW_ETx0slUK*R;`MAV0JX{4yGN|uRJ3TbTd!omh679j?Snbr
zrv!m@S;UzTo+yA59(M2lxwmO_N3bDZ%|r|Laad9NBxv%loQRehkNjWMb+{Z=DTx$`
zJ!6xxYP8gpvSc=?^BkQzKPi%=h<+o#LbmCBwHIgH{#WPgkLD21;gYL%q{vR=FA}Rj
zfZG!#gAj^BM0<`mSit?J?P-BIe2B<7>p?%WQ>cE&Qz;&=#QKTtj9&H;XDKRegd$*M
z*t@ke)iiBH%4ztt*+{nW4+D`?S`N#+waC4>8m{h1W1L)Pkdt|cpF4*Rb*w)VzKLqx
zFDcBkrwMywnz+W`eA|^h;BkQV=%##m_*6n`2Ni%B)yjai*HCI02jHQu0p?9umzG{S
zL{uHphT!n8sYKPi_MlC2Iy95BUWnHjKY9k<lD)P5^BK@0I9k}A88x2vJAU9-&VaRe
z*vZyt+d~QMncN?TY@02?R`fFP4{;&nFY;j`X(1x!ei-~h#BPPMUw{0FDbDiihtLE9
zm#7i0?f%foNY$feKaF|vt!XSiHD$d(%-;?kjn?gz3CT+vEB4)x^{ZR*47A3jhDsP`
zk3ue?mRS^t6vT6W^4=3wOtk((#1!R~+88!WHY|E>Zmr?zx;}#i3QjV~)R;y`Y3Ji|
zB8CZv;0-G?wNRhSXsDWk9XhpZ7_tuSFUA6=_J<ijLK)%UBUhcErTg^rH=vVq%+wp7
zV(RYrYtYC6H0t3|0DFD<EIXp5TqEW8;VnG*s^NHrfo`P#(bs;5B(LVUH|WZM-yu6G
z1)LYPMB83kurU$!C&mgKhO+Xt`rWQz3xXa34*B`ehjLWTCQO}c*fWkcFA`#l7WJDS
zU9qd3sY|-c=svF@5yIq1>aH+)J&VzF#*|}~JiCUwxBuE)f5T_eUvs;&-iB8F`!#oT
zFUhd~vMC?3LMT1YVXLeL7y>1e7*h>GDan`MjUfFG8F^l{vahn+GvuqC*|g&G2!~}X
zcddwa9OCYRLah5|7$;q(a<;RWlWboBe$ksYMxKAMhCaka4O#m0DX8dDrUJlP=_NFH
zhe2jm&>GeZxQ2Dej79W_3<1U&yPa>8#Z=^BTRGJq+UR-wyCf%aU_8X^%$H!*3o^$M
zj#<d=a<<CORqLo^pZz~)MVtPI70vkn4_5TxS%cQxhKo_B4c2q3NLk;Bb!#W0%osAC
zxd8(P%1j4)Iq@RUZI)Ec$4-4aI>I%Lrs4l_gp@}|Xd_#&ApXY@Ld4eDI!B#{ke`9+
z{0`VorCu8)s1#<Jx@_bv*tY9;Z<La&sW&FYBE379Gft%RMZ2T5HOjil&q+~>$7zLG
z$q=hX`%;VulMuw*{rftdG+0V~$#u%QN0vYf!wo#FaLs6l$g<rA-d~VU{1>KO5}jXu
z!s(ntY@AGX<d|pv7kmB4v@OSZ8HWrqQp_@Lgjvqj#7=5s`l9pClCbQ)_7=(!r)Ax2
zkY7<Q&K49g3Cz=;;K39D)#XrM^6!YO^hb5(75qTLn)~r+k#Q4GGC6qsK_Wq$g&fT&
z3)$**$6Fe6MhIcgm==y0Y69P1rfqRqu+rS?;~ZaKB&maT^#w1(__)5}?eqk-D^lI(
z^1=VfBbZ%py6U4sI`U$CvX4hDi?zCwxx?4~p0}LF72?ApeHF5tF#D`G)q#bz@TUnY
z4}4f`UaKbL+#_`K?$_fAdb&=S(45k^L`ZP?+^tFNM&#NG<vUzu$fyXWj3qP92^XK{
z9VA1-&TSzX7r9|X)pM>qZvY_dX}tF^VgLy?tqw`ktTW#{qh`_E^-$6&sh;zQs%!rF
zz1HPDGWc<F`Vn{G|Dlljj2hUb9U8_5$Mp%?fM%H3>w))ZUES+YuOP|w^aOU!@4EI>
z*}d7^-5?YIHI)x9F3;!8W7=Mv*H?W+18WejWvI@oVDq<}n>Rx2a4tpdV|hmQdpC?V
zatEp%X5v<wYLgU_tET`EuFV;Tx#)|gh7)rV+xw9L4-0F4+B8)z?E*;BIi7|<SxvRG
zf-IN~=VwlRM~3lfkYRlJ3QxvRi1>QFydP(+FH?>FR=#4M^&<*V3xGb2iC7%0{7*wd
zPnQ;x(4aqwT`ww|j$Z2~s`8XSqI<yo0-N;pwA%hz2+P*nDoPGMNF$CVxWT-0!v4%8
zS=JGo)SLDr#Wl|&Vqm4!R3o%9%X^k}vp#%Y4o<>3?%6irf*32vl=b@@i~Ft?E#v#u
zIWJ8K2Ky1aA*Ym^ZHOzfSjXU4EvUEd41T&v0LV$o8(ti-P9;xi8?DkXp997B*uI^J
z6!Rp%h^z#`I0C>Ej!l_wp&G~+IZp12Gr|!sUDiG1dQ$>Mlf*e88+m46BcEknxct>w
zs+;$0LIv*gMNh-78v-U*a!7bg0G8qjyo#BLRBxKl1OPg|00>f!-AuEeOGyfYmRo>e
zO#Rzq+1=1}lHP_~&+}WiF5*4Ab8mtb#9;unP;hI|LA?l=MEn%%(a+fv7i*M;EZnXD
z2RQdbmu<jD?KIv5fOzxTBS8;$J?lJ8*A|CZ4RJs|T>~udcYxiLqwl_$H#BWL8?k!4
zObQJhHF6z6R&&Q~_=I{OmRN>t_Ju9!zh`3`sn?kVv;;BCcD{UI&+vb3U=P>o0kC0J
zq+}t*8V5uumDNe)X-aEO3bPQ6TX#2DPpj$&M|%ShN}1sz*}dB8%7ZlzUQs=(54z6)
z`1%B+1YfyapO~ARX!BMP7>i%h9C9&PY4?`tCNhn)oFj?D3xB9v81_q`+0?sAiKxlR
z`VxX)Ey6ZU0Z;fmpwsb#VJ|8JAGegc;vmD43<-B2lX(u2a5cN?^>0=ll0!+^=B7XL
zv@}IYWJ0btb2cA!IL=3o?8J=x1(d@m-F!gZ`_A{-7x;4QI~9T9q)`dJ+YY(0ge=;2
zj_#^HbJI990$@(3aLz>G=gEkbbYk3>icZ{lS8;%=5CkitgvQS;kTS867q;q;{_r2)
zJ5@8n*;&Y~38LnMj&UvXg*+sXlRnoBTq?&d7_>T#`}2k|md)JsrkK#bw*?Ws#dbyF
zcJ3%4Pa$4F783TqlzD?#;8yyszuz}PranV{xl@q{nvxQ+)@6;j-Odj$FRvK_0Ufr8
zR%S)+P_v%*QzUjZFj!^^uF=aqSGs8Zj4yyuQ2fe&QDZBwT=%LV77U5gG^ZzNwC39{
z19=F)vE;x0F!cw?&=3YHf5-R`1AQR-bJ5AR8vQR<|MJs5A@MV&{OT_Rwfw8e)Cr^!
zj)lZiIbX;`Jl7Q0pC+QB48uK2i#or-T_=Sab&Q-x6DxqF|H5!n9jW>YkK&%M&c%6)
zN>{{tYFJH|E$6lr1s5jirJp8LKKRkBfvI-aHWz>8!nA?4ow25|d!d+?H*{#~-*gLS
z6v)0B*!eN|h$1_ohcGwE6OK|T#iw4&QbNYQ$Mks;+XOjhTEn+;bej=>oBC(k_J16f
z%i)q}dvJa#3-klC=i3PCfHvS4y~VcA#%7P<$SmE3u=jHHUY);A3D(Xv2Z<gmR^SWP
zM!ji+&-@x-6wucOYt<Ofbo=iVypS~?AQ|lM_EZ>X>G+^J6(>~;ckiPgh;X2(HdaGW
zNUqaF)Q^K_>1?KVQY&^-ubyrAv{+^Fn|;|usR&3`-NDed6Bo`{zT$&-J9qMYD6pq+
z^nd5n^QZvWxEN7L*CahG9Y3Sei=iI;xDLF}i?^9Mu>}aA@`7YO`nFXk(u+DEzH}IT
zHxW=qkX2V%#>ieUNU-PG*D_nY%W1imDt7Z9!H<>6<dFQ@46-cnfGqu@FE}rNSLe`D
zs^$C0o6O?5ulxErH^-L!AfG=S@WeE#d#;79WjrrK<Xc($^kGoyv}ki5*GZ`-$1yom
zh*`o^*~GmLz4jUjdJizbm0+lx<lR^7%QNM;a7F^gN+zrANeUyS^BwV~p>Qu7c5|H0
zzXA_%4C+<`V9j$%ofFDCFtrG3-EYE%hul*wqvpIGZ8+S|P_nuBEFEI{vUN{o$G+2c
zyx*XHv4|d#{g>n7;zdFwqoBmVVz0NU8MH!e4Ry=?%u&B~n=O2iCMXksQxNL~<9H#?
zNe*)yKTJFj3B@GqDM{l8XbGLyQ`AvHS6gFi;(1~*tWUVwu27-BRGYQzj|uW8MadC!
zL44AkN-|hmS<5VNY2<_U3_4*a{HK5%j;SX|Xi>rYbd_MUYhH);cRsH|dNa|Yg>Q{N
z>wqY|UBApa|J0Lh`-uP`$SbT@<RJaleoqNxpT3FUYJ;c{iKjGY>eJ;9whs13cuNGO
z>{zKLqgLr&^e=!7e{M8#tsy-n5F=OJd;_9ckgY+8-*bswDyE!&O!1XW<a=YB`(p(D
z6a3Fy8>7p9(3}#^biN4@VwM!kg@elmn+D@uDhbIp@W8s{xpSt#B6ZFUbTX~5{@8TN
zSkzyevO_Y!DuIY1PLGA&#(8fVwLhJt5+z&#*|v(9(k1<o6XQSv$=U0@b$^Nj(uJ>}
z3|Is@mrz9$+)yKsD7<7#$ksc_JA*;Q2QkwS$|he3k%x<W%xs8deDy$W`Gx4$6m>Os
zDZvY=CmUU}NQGy1n=Z6?d;oxdPq&d`+GiV7cXZsIc({kEuGO_!BC$8kE~9{$HQ4-%
zn0<`yD64_%p)}%OSq)i=&#nJQRzo+)YLJ`*KR)VG;~jy*8Zxm3AqW#N;th-#^tv$q
zoC3J&P2*swm{Sl@x7j-UBm<^A!8aTLqtp3(WF5zZrn3moUR+5Mn9@!?rnBKM0tm~=
z8yp7ra>14qU<4E2KPQn;g<&s$fm{~(P%C7pl?T@ogPi!38Sa|{Nz&hrP6O#-kWadM
z+39u2WS#~r0{2zuh)33Ax84C0-UK~+?<Sz+NluO9246Ao=44dl8zdlUx702NmTe)f
z?o=ML@UXofhI=>rfQB~Q)&W6NbJn2mhlNo9f4=s)DIU(9TM*I19oIqs1k$E`XB0?>
z@b=$;8fl7n^-tm6XDEF4{#b<bG%%m;WN_cU2~3hXbb#TMp=wcU30Pt-O;ex5uVCN{
zHboIu_Y7EnmiwB0D3o2`v<AL~O3z8QvA!+8J=TK{Z%98w0RQpn=?ul8;Xy|%n4Nka
z+(2oOW!k<~AmEkzcv8H4*@Bvupq%JB6=-9niObP2dmYmPnw%T&zhX0k&e3q1G}T*X
z!Sm=%+8T1;h@z9*%-v#VS#jJzPH-d8uJfuQdmL~${*nQA@=GkogFRzf05!Xv;9QK|
zRaBeH*KyMHiG=jUg#HL^T2<IJf~Ve>#?h-2i1k%g_~3|`$o1IOrZoD!?p#6V-BXxc
zS~6(_ej~{jpIns%5-MUqLPa)|P%)I`;a>&M4CdzpY!T3B`0!s<q6d94h|(quV}CxF
z&C{NiUCZ)25|J@$^ZUbJrw+^feRw#rx_9eL++IT*Rj-RIUFH_rw&P^MBvjJVQ+iy7
zPGx?&zfGx%qkKXx4!VyJM?v9MEh1GMiacjFyPh*C&eIKIrr{j$qh#<IgXS()#e2T3
zd+#%mpCQY>#2Yk`g=fgly(1!grDt~909S*Eb|J0)ez4dS;rshjsM_4V;6V32(Aa-t
zL+z~+x2{5VL_Z;u$Sq?Po;I0jsn)+hCy_+D#tH18zk?m@n}3XafsEh}R~yi|@x#vH
zH>5(f2RhQz`!>?-zxIb%uI@$w#)R^F_IVUleD4jAv4CfPu{{;_th@#Wb_7Iknf`kt
z1C3IRCm^_sa-pN&5$w1C>0r@GMw`s=lkz*XAb}u373m1UKHAfdca5~(;X&TCq)|>V
z@`U+<AIx#F3E7bJ|325-WiD!n{<R1Bln75vq6Y0FuXcKTQ~C()nWA1<!D|+yn|Eez
z>2VHeBj>OZTGBbP$w?#YcZo|fZFOHilq75zd~YTnPZ$Gr_B`z?w_{Q>%Zqv1W%ga%
zd3raDT&hkmQql2q8OXgJoqA(DcG=OhYUr&|`FhN9pxcFAKqC@edG!VOLKl`aJcbH#
z<MmRsmq-{@sB-znXTdYKq$cQX)^@h*;4r$MoAaQ{tMszI8nE69M~#ZdLe$+(R^!-%
zz3FgoT3F#TJK!{VmRwjRbW9{9Tu$VU?%2WbkYk4`8M!*J`bB976ivRp(68X>+LdY`
zW;AWNTd_Z1!I4!oij5k<m5n&`y|JeOhIX;hBBhMaSam_+r)<r-peet=y>9`>=@<(B
zQ$bbhUHJ>_)uJVtiLRm4$KJye-vswDz$sU@l=oX?3=*aQYgmJeJ>&wv|Kn>0W1{Wr
zP-8K~p@d^LN|Oz#Yk_f%J9v+4|BCgvOcHs3uQ+(#2RPd$;?<AW{OO*!r2pm4q58jF
zt-v*cq94L!r#go0pjBxLHu%~$ylf|f6GlWF!+np$fB=hSRH$uCc+yj=r)0?A-i21E
zjOWO{#hR+7KAxr|ND?KCtqX2GPIu=w@lWt|%3{W(|KHHvS*6mXvJmDNPjmU}b%M8M
z$enA2+DYQv&vozjBZZc{`y}lLc46nOmXzSx<am0liHsP%$$;llk1mKHov^wBBki%Z
z+AOKSm2kH`;F|h&g^!4uY4vhEt;dW4u!8Nr{93e--~5`g=;pdMWHpDbvjfaj>cPiX
zN7E?D1))ylS^56%yi%E_*|wNcDd`Mgs9Ekz^1}P3?a$hXGOPGDoX_zAsZC%YRTc01
zG$M|=C;j`K#-Sy~z{OOmIM-5=$T7o>IM7)PO()^Mje;*%;28=x5Z>Dl^`qYaHF?Xq
z+;%iSGW+EL>vQ1YQ&6^OJ)|l#83;_{4zx%M@@cb?`(RW#+4)aV<HeViLg*=|!`<5D
zu(>q`BDcWPfT>WR8F8y0$>z%Af|xOspN1iS0G{2o-8%*+KFP7c=8_XnX7)!*JTp+r
zfK)xjk$W+S+={qXpqho;efS|y3iof<P!^xC)9`~G$dFj4JWB7BvP%5emr7_3^NIS4
zzGcH#qBJ43dL(h6C*}H!1I~j9lp~7l<qD1ePA%)Rg<XD-Pty48V01c?K}|9NvWdZ`
zDy7i+`&5J)xXpgHc7fz@#V$dRU%UGG@iV!lOAKS;jo@!p!3UYepjde<rk9bNxZb*5
zHk-sjusC!>)?Wy=K$q@zWpYSRe8pe6>phJt?kUSit>#=D@pqqKI$1F9C4n1VZWx_w
z4(4v#J6I?q`(So6W*?>XP>$pT#}B#F+Gg*w&N0yi`Z1ptLcbv-zIeBiiiv<YmE`8l
z6U55&PWR-g{QNQq&FrYjFINX9d!+*X;{L`#X6sWJra0+YtK*jA*5G{k$aG4^*Jo*N
zt?Y$cv0L2{M7ha-qMek11abH$w}w&+4sas}!@X^*3aj-`xP(BSom#w(e!t=Ei2#3p
z#)r(O(X?J|rry#*tG|Dw*M#?Dgpq`9n`XUsJoVJi<xT}V*ngCeFn!4uIGg^=V5-x;
zY=*h<*<Hg2k7(+;bIP4MHA2m4!09yi77hA+oZHy_z@Qfsz%>7XGfs!X|KS6Dp(1@z
zN(VL1+D-HJt^h{1jooj`8wGqplm<P*73D3fTOWUkC^1c_%u`<5J3l;%xZ>KL6V`7d
z>k%Q6FIbJ%-7WHUWA8SJtI1J!YHHW3pps@=x?3yrefFKqcM9#=hiBi)5#L58Q2YCK
zNS6Z-iu^kN)=9?9#_Mk2?~bUT%kbbTZ~9@k0NKT`cs6|fP=IoyG|bP*H2%dK8h8@z
z8zmCr)|q&bKO%}3T2?wVTnNvVATQhz)|g#(y#^ed+Mvq%=&xZCfZ%8F>-_fS+#B_Z
zMGRLoOsEpEXBJIGDrZt!cg$h{%ujCt(0x{N%!a~0&G3!H$vZz$jJwu+UGADkmOdW_
zq%blyuauEc$l;eg!dG&N&YX_J>Eu`e44o}DxQ5Z#aG9iAu6PI49W>ijO)5|PmP<9c
z=xcG88`FS$HuU97?Z7ZLe+pD+b7{=vI^b3ir%<4yiO0SOMw9G749p9iPrPMFikLs{
zKlnbOR{`pc(^L{1KWohaG;ByiiXt}<K!8w<9(5QP7|!R3nNe@Wc@7`&RC8PyV6oY$
zrtqKCja`YD*yvDy)}Xo_>(C<A3~hX|EKJfGM9pA??Ku@MDV7Y&79nAwS3U0RAxEcO
zM83k@*E;T$Ic%^-C0k7QL~F?@H8q3@ZQlAV|HtRmn<O$M<uhnbf@w5ve}&NH4tX?z
zRc^ofqk9GW8z2CW-eC?wW8EzrtlN#*y4%C62v`%73T=rw1?8UGGkROwIqDP1>S?>M
z17cPw>&dV4kx?!;D*XFu3_TVQ1G#nQWBF19g<N%*;yW41*gf+)qpA&Xl$V8H^oK|=
zx{asH8-^Oz2QV!UN0g8fY<=unO-meBiO4shxd{H_`dgRH?mX%q`%z<FcGb!wB8LVU
zSWrGM%v<!jOJzWaw=GvYaFes*-8AK{&s#`bjvtQ|ES743IK~T}MMdWjqmBE#GGi3P
ze|^E-+?2cfY|V>ICLS`By=Lw4QTZUi-tSDN$09;{#I5b<Dt1_KrP-dxe|s5qCYr5=
zardEt>+f&+(t4;?d=y`dshA{S(?;#KYwvw`0nG$L7GP59RD(=oiWCxrma{#jz+pN{
zqI@+duQ9OJgQ)pIl}OPWVa?!~|2z)|CFC0vqUWmI)zzu))g}7RF(_LS1;I9u5m<0F
zQh;gU97ht>by9iBHzY4ammB6g$Z^XSfJs_oKIGJ6ZCd{vxM}9q*gL+pgoInzm+!k%
z_LxDnu#0>>`+lq90N)H=Zr7y6zof*_iq%>8E%kG0^YnF3#nPSvTOnc9fD=jTYo2E#
z1kkl4uMvebOU$w^w`P1h*8zf8e1UwHxE5yb1gvV<<?EuWcX!KH;z0Kk_Tap~@<BxD
z@}PN?b-&UK3oB93txx~JJPtv8F7iMcB4qg|ZgAgVB$4e!=J@x4AcBkGNmy7d&tcY)
zF%>VtnjezTkz7!2hh%OA{6~ak4X<=LEcE)txzLBpU)a8_WqikDqPZ{RIs06?{4)=?
zwfhVe<;jU>sZkUDrtBF$!;xmai^OGxGa5a-f+p$yed(($7M)w6&6l>%TG(gVGOA_<
z2Nh>uU8p5`ni&V%@WX<Cg}{#Lgt&Y@*?IDdOQ|gZaIaq-`r-54iXk7Wvq4X>JM}5}
zJhPhvL!-8^go8&LvpS68YIjn>dt<)N>88a^O7zqQMAl>I-(y!H(IQ2Y#zWzZQy@yp
zQkivQIu5mea)Qk6H#e>8t?;Ca#g0F9sJONvDEiT3yh<}hcEPwAn*REd8!{hycy}=W
zj8|V^IzvQ_nDqS##zqO-pK4{;{X$fA;<M+&BkPXjqIibKh1^6>$;Oya!L<*!-){sM
zY@AE7HXQMjUoC3&$dGkLRX!yRojvu$o~W5hF6cj@!$K5j7}|F$^tIQ)e8qmMtbd3S
zXV=SVNN~<$0d5~~n%`m}=Zlshn)TjF7VC>1%tZd?i)s|CD1B51<&;bbhq&I6uEl=h
zl}%wEnG!E}Dp_>%I{ET*8R0Jn9R3WCCB;ZnDEMDHK9RYqywjNk>sm1=4BMv=O^c{=
zx;{81%ES?Kc_+zW0$8UnqwBtQFL&)&-3=X((s{ELT7QRJ)NuPdW51HR-FNDcB1p_#
z{JCOIGt)9TrPF<S_1-A#Q4uI4!?2IHOKHLA9=4J*5iH%ZOA_h1NcUvd@9{sU>!V>x
zdxv~QD%`iT%nb(~x&6FI%M&1ZaFryIwb2b|fl4c%G_emTTtf2@^vI*!wn^Tq{>;i7
z^Nx*maUt1e!O*nRmX#7)<;88TelB-ydde<EvmMo=zfD&cE$K!QBkp#h1U3U|Bz0YF
zLLx^Q;%cVrij#hi;9MVtvepA_Axcf{cvohgEBZzxXE<wj!;I&vOFg+i#Ifj0I{fM}
zdjFJnGM-)`lhbrQhojD{ch^`+-NW>QMu4Kn57$Q+;c)}zL>|U#9x)nm!XIN?AH@}I
za<^y`6E({=(vR{^)2p5E7Y=)WGd`t)YuYg!R6jDlK$8O%<f(3@?A9*?h6d}~Uv?i>
zM0eW*ht`7GtT<qaF{FCp9Wuoe^t7s%d%){<3hnzN1I!1_z&3T#9PoZ^p8dMhF9-ex
z6ui0a%PC}E@Uns-ayh_JqoC&NHHr|n(RSin+>3chj8CUkc_iT5KH0xA8>r8FTFdA$
zV~bhB0`f*1{iXgn%{H9|7jFvSZzcd}DNRyYuUCS7yHJ*#tUM!zCjHeE>%J!25HI%<
zT@x&)hJKy6(i<Q7uJm9ZCjpwxR<Y-@%l<oim8KuK#2oq)?}L}|l)tmP-+Ee^;N~>3
zRW!1?{9QTzw;;+@rO6fSBYJ?Wm!mk3H{W(Hm}r&tn<P;cMyB48LbS9l^zIF|kh8z(
zbx+>-3|V`tk<}}RfXB5=V%TLu{GlqzNkEgA@czdne#9!NgrA3a$!w<Xn^^9)+_u2U
zIQsK#wmCneEJ~C2by>D{m^-Q&;+3Q21))SLYW5)EkF=|PJELcldB|Qo<KK<Pgx~Ec
zI9oPI;xw7;K(EAae`(S@czCyerhR@QdQWtRvpW&?T|WUPHS;K7b0y)+y7A^}ME>0;
zH-G&2mBU^2P~B7FMH%TC*=vL*rh)Ag1ZAul4s$9YA)C0JfRr;xB;j0Y3#EzY9A8GA
zSaN~QP?VwJTGxvhWbY7=I63<5J9%{1M>DO58RsZIjh1uJRf{JKUVZ;w9$cP14lH?W
z-^T{VX(ZZ!vO?&0_?MjbN-+Bs#S^Yv4z_G1&<;(?%H6ZU9pIN#Gaokm*%KAxQy7~Z
zL?CW`Se89_Itpf&=?&{00?zV(t-ct9b2s2X`x@%NSJ1R<^%H^-n%JIS@EspR(aijQ
z%VLQ=2}?*%JSD<@=Vts^JH@6cVLc8kXcES@_y0N4n7@*XlUtvXs3_W5dR{T`S;u3u
zD+&U|he|K@c9KNyzI)$NM76v8#!?TZUr{lBnq_#v1~iz!^J_)ew(w549PvGj1_|yy
zO0lvQ*R)&5BC}8N?bW#|-_WO$KBQ<S67JHl-FLUkZC?^pGX{5DXj>WPyk0=WOm;um
z9)N5T(3&T@B~orZo=i5`wVHLfZOS+XnXxJi=10dB^ZyXI{-z&oV{XX3po%iJIxKc)
zBj4<afZpEb0lz@AA(Ju))shQsz3a;l_u8qUeD=MiS89)-UgA|Lk!e;a=bL#i8uvxY
zk;>hTwNo<@8m)HB&YS+J>ro7-+J67f0&Z?ryK|VJ+Wr-8Uvq(p&i0<2##}u{VL##E
z^mZ1oiv8%oJ!LF!Q?D92;$z~#Ky)*N-JZhPfZ&uW0f}&?;7iR{EsQMKV%W|qua*aM
zJ|lV5=4L{Y{A2lg6%R+OS#Rw<&g}B}#NfR8kT!&c8qBG?vn4v3Tr5o`v(RR4RAr!$
z?ui;TTG6jI$M6Y}D*ned6kz}bN;;4Zg1z2=ZIdg%%r3^TQtd+YU4>{!3!2B<A~~(%
zR-P8;zoO_{ibNZ##NE02lO)n@C~#iM=Ba7iW~N<Nj|Zx?k*BRw0yQVo$O2`qG!MHk
z4;9&pn1KdoaS96i$N`H(r}ce;Z`3vHJ`(ZJ$jpi=k^!C&Am}g2wNY>Q7i0-<jq<U+
zf)58k4)z{dp1?mf23KgoOGJA0M98!IE=UcJbCjQ`>sfE%IV`_+Y)NXq2{k`G+}QQK
z;nm#AZ`m{7(mEc`ZG%F}#Rdm(-K=Fq;(oLj4JUcyPmsOYdFXrFpAoRNTgT>bCkJ5C
z`Mbu;IN(`3I+Z}{@r04A%6DrP+^7xpjj>f#&h1LFVx$9jtw*ZCV-wU2rN(5C^6tA_
zrM}JK|HA+RnOZE0y=A^XExz?b@Hm#sm1Rr5;yTzmdDKPulw1@^7X>%Vlfu#LsYG}!
zqnyNhQb7U^udThYTU;|wkWXL2ou_Px{JKI6VAq-RVf}4Ub=P_)*<ObW!zMG=KURnD
zu88V`(q<M&D}JJEzB4#vGh45s^`Nq~bh7Q?J|^E}Zm;tmrLh{vR0ea*#MS5+Z6oJ)
zhI=pM1Xw5&|C*caBl)Oyx2rw0m2KOdyG@#rYRyQ3)&a!g)kG+GuqOp^lrVmy8Yn{C
zRDaZCOf%mRl}c5jJwKto@5$!;l0#Zhtw&j1%>?vYg$nJ~4>-mgZ{t%dKL!Lc6HzLI
zcj+|7#TIPRn*P$Q<|wFPY>XyfgL>He{5D)!zPqb;WNyDwy3IXP<SI8D@iVIq5OB^X
z%EgcE_9W%3T>1oB1DhFVMt<GSQ{I+55*)t9sp{a0oogEeF^_SZiD53*uJ@@cEIr2=
z;$}|u$FiP?b{4$@X|JZh`f$%{_NltN?v{44F$v51I^|;y6v&&jO`L+_+~ivB`~zUk
zx<axGhWEfR{KxMULKqIW<c_c`N;}2~8uS;>clYov@d9J7tB5&Pyq3y@&FiH~QByTP
zKp*2toNq?|1QJDK?GnPaK_Wr7t^FPTO^Xv7rM$p;#X*{pe6EyJ6Is!PTIQ8kgs%;P
z@@bVs_bF~Q&?0x@9#7ky@Z`zz84q>c`7j9@w7Dz!kN?pExd4^YU9e~NsY3}|4(39e
zEBEyn-VW`NQ}>3Xb#j<n=f}QW631sTBLp)EqU-|cC+g(|Oh#FSECPpEX4-`%f`#w5
z2IVD{OZgRO#bDT)vP%pm0FggCho)X+dqAJpVcg2BcSq<Vhj7czOXssFWl=^Gj<YR_
z^b+zRodn4aJiSAKQUoWyk$+T0xh*)U;kn{3WuT?Xn>^4T0;GOzeg5+JtW{pc=7pTs
z#O$6mlKFyPg9&(Gy-!+Jvw@0Ha4rQTzG|wJSV*C#-bKr4yk;`hxddJxpY3nAPV=4o
zEj_#%JOw=bTIC}7G73LJ+yU`;?-g4R{~`hC8(m)e+Y=$rT41;ZtpXr}5EXQ(q9k2-
zeNHCglen!zLf8eBFi|=Y%h^z6MYLSC-4;HIaoMe>BoY!!7`RC<!43Q+>>ON)sCyD$
zuaVAmhw!%%WIqL70(SjaV9KgR{FI`o@zK&R+v0otX`5o1uBz)GhOJPmpeL@5r}O;R
z-P)5a9z@g?<WqLWxAFa2Wb{un%dnPUdA<8UI(_y(Qy`ZRGt{DM+{X?*ADIGhg0*_^
zPA{;kNCRNTBd(C`9dK=WT9OpvAv@(Lh_$%My9HF--<Rl>WjN*_>z?P=sh3j$HF>a8
zm}Cj~NKrp#nuHh?#DuyLujHyJ1Ukn><nXbSSKGi;=usWD?|ymMK^7dIbwmFGD-kSd
z<x=Ux>G0>Jf|K~@H7JZu3M4iR{I%>9a&0a-c%dmpfQ;SoE<%r-NcF%Y5y94-c?p;{
z^SrJsvO%^mVIzEsx!b4|;KhG7Q9mI~())k|{Rf;NH#p~KY}h+!{)1$QF!*_TSAv;I
zf;OJP&hJagLNmW_2wh}M?=}S@#8~0`I8&a?&7h<n05eH@ruDGdo@0`J9!Oi%fTl@V
zH+Fc`X_3Ch*T%EWx@<7p5HzYpWg>vk8Oa-05O5HQ%kE_TV7J0gc%|N_*VIIR`Z;mG
zz9h`*KFXA&mhp`pksh%E)lQR7kfcL9g0C8#?e&WXJIRA_c8K4)`GTgTXp{e|mb%Qx
zI9wWj#jb3d#nfZbiDqu3JeRt$c}c^8h&n4M^9l+2%l_49L0>6LkMhSLCCVzqc{z^!
z`(J-C8Y1m*seOQzn+EAdqVhmDWyFlE!h-=R&veWFhK$k=KHvmv;M)MLsHZ5@U9MY<
zH`5f=r+}f1O_{dc{IYQwZ2aVXy2*ImLc&q%K;7jJ+V)q}cjrwOA-VKA=zWXuL>#5t
z7oNraAI!aHRFm7+?oBV!MWuI@?N&+zflva7fQo>K3P>lLCY=C6=qOD}R1}b|sDN}8
z>CI3?dJQ5-4<(|ML_#Q@b>se@XK#Jpao+cw@s4pm`b}?F=9+8H>-t@;L!mwcWy-tb
zo#KYW*7zir^-IBW&w8wXMDVL&Xr`Wt&4Z<R>Km)?zVnYv6d=Yzpk>v^clz0TF3%g=
z=DD2mp@`Qq=%wYp^)`EXjhWWq@qd~du<@Qc0+qX=y3lYYuy=D|t#h2Nm!jje2NSu^
zhU=*pC%0D-4>nKMSx<oUcz$7**vnv3y6IZdxJTK4URdp|r<lLkg47pL+yVhD&b#Nu
z1Yd~vtyYP7ukHyvNIV<-xlvP9XStV^a;YNpyf(yRHLv_Mj#psIBMr^*BXZY2XKt2P
z))$L-rH2~_;^Q<1L$g2Z`A-Ic_ViSeMSbfL^GlQ<adwWNDOv$5kpnm`SO4Bb&lM%U
z_b%p1G4bW4lPJT@qvFVgn$6dXSO3bcE=l<f*+~~}q#f4lFaPZ8@F(!p%0<(sJG3y#
z2OfD=aw>t_$=~*)=nK!@Y=#ADHDs*ZA}6V5Y5wdR_Th2WMUrEYdi5a!_L^)WWpRrp
znz!}LKyDrvj)z!MnP}o@D0rsB*1%^(^V^O7C8UQi{;mR^WV&5Msc9-YIDT+wlJh2&
zxOxIxKc$K1TArf?t%pJQyTnvSJB2k9YlnEQoub|hmUpqzR#oV4MR9OMNUfvwx7<|o
zeg}4!(~g%6GXgrxGL0J>rGwTa5-F$ii;T8s3x6!z*V%&lIrOuBuIcGDux4`<G%~4^
zhKvQqf;ypY>kDY%<o?70oa>*!7p6E?wU^TO2mDdwpjlK0SP|jC_Y~okC3Tz)KHU+v
zaj(NZ#c~8nWOJ1QBT%ig3|9wRx1Y-dJ8yuR<tOT=%-vAMQ(CaI9r!iLIG!+hSu+%X
z63zha%YA9u;N6;mrri7;kbj8T#VKAPnqP19SsGwCeN}2u42-+FflFm{;*IPc$T>8>
zh19a7{G*_-YULQ_gpZ0BJpaBBOn_or80_wz7NTKmzyx$5k7sr{v2Rm$$3RB+7zm8X
z=2e30sE>%*(oU>1$&**}hbVJxX-)m1p^R&HlDsHavb%EG4(@_iql!-^Wz}f}nx-0<
z(_GPS!gCpA3HyVGK;@i323(Wz?EQhzNz6BshV{eUD+O@QrR^)(p9d?YJQ{!$;cXt*
z5-!w-``j(2xdU`hfkX)|v<T6(C6fN}cmMlEd0tdhGpbegt%-*SjXq2F`8_@u4O?$%
z!%O*)3OYFbT9&#XjS_X2q?Ic-s)}Zm^Ggrui7uvz$L7*)V-4#C3Glpp#@E(jo6ELQ
znY>11@*1U}ys>VcmiD6uLcE1my`+e9OWV1*QSU6Qea2#tU(iI9CAp-`)K#1NakMu2
z{`f<dQa3WoAvE6gk{VL+A_Cc2HjJz8e;mhq{?9N+L2#<AG?xAQKa|lvhN0K1q)~dV
znjX8ST_fS}ax=Y5Qu+Pc<D*4}g8KdK6v`7EP}b*s%mMF%ji?Id0Sph#2GqpSEe!E7
z9b-8r$lX}i&Qg6OWS0Uuowwm-Pl=)T?`ByiSOElc&820~RvEGfbxi)4sWCkM_+d_-
z=^r#~_SMY$^T*1FY2(lN5NMTF@jz6df_`W(iS4m+R4xf3f*Q0Ya2eD;zWCKAPvs0y
zPJIY_%#AWlc^51DEo8IyleKH1`E~VS8DDU1ipe~adT(Cg$sHBSxN}kUpPFNPu|CN|
z8!-~Xk_T*Qr`32dt}{kDV1O4=xWOUGan?^W`PKp9ITj{ikRIDZMIB|V;qPPo!zk;X
zv8dxvo}%{y@xSb2Mlb4vk*o2Xu4;G`4C6T?M=iPh2VekLpnwI?Ksh|B{mI#Ut6LMk
z<|f~Jw+T=(HZ3OPbZ<R?L?ySNaJ8Q+BnCJ)+ST{UgC{>A6K*awPL?vutIj-ZR~G{<
z#FmS=LAHh#Yn3X92^104ShHFK%fmKR<a8PkG|DW`fYQu5wR~nqc|B&(mgfGVU*)XJ
z?H2{im#{$#<_nJf85(kv=OsgbRgdK+Qe|5^Lt81JFY_s#ivjADKhH}^CFm;Y*OeV7
zv#c^ew&6csCeA(06h=@r#80!FzawT)3ojG%X6}<9`l@XteKOtY<1TDeu;be|4hx3*
z5J_Y2<ED%kUJIk?1ghsX^K~5yQD&V=L0uQvaUQH2W-QuZO!iRpUS~OrVk3%jgLn&T
zMYaBNFdM!%CYVmE-WGwhZZ{JVdY4fyx}kqWhHB&qdo5ciLFMgnL<zG1oRdgV*3FYW
zqAUETHBORUrF(ZOFnVC5S0dxU0i<+@GaM(`H#EnW$h{D^7nuwpg?HzVS$i8E1KLb?
ztIZl*g>^g*0i_IKyJ_ejur=~qCWH7W4<}(_4kKHY22D6?#8BgRLC-Y1{ahi49BzKH
zy?0}N=hB^YIzdNuOGsz>35^X|(qsRev!*<qg6%{TuUeKiE)+1y2RAMSJ+_`anu9OX
zb@h^P29hEVVKAq=rFdnT1jFkv5M~_S$xgQoLqf>M<TGU9z%TvXgl0%qdjTU?HTOU-
z4N5nwFmL;ZT_9C~rcQA-cnL7y%e&fd8K_K4C8|=KfVt0gQo<5!Qag|Bc=PuhPj?-#
z0AV{)C+uI4k{7H)Lz+w;F#RPU%+tyrY+YYxQ3Oe>+(XD%5x=|-r(_%uUo!fz+x?XC
zk`1HwjCn@sm2Dp;gIUoKD2Gt6<oXCN-9O$vVD!-xBo2xgfBeeur)Z7AvvLnYbpY6k
zSzb?XmdoHCysG^N&YUt(p={-}PIWX-Hrgd@G#ATmLM5d3ZWLa>RdzwNIR!RO<MPl}
zAg>Xz&bgdH7N2Xv@DfsN3sVf6E&3<NgZH8dSP>ByWre(oRg(qf*kg%s!prCCrEZ-<
z=04Slou0DM0y8va`Fjx)heG5PZ?v=2tl|IsYRRhX?RB3Iw>Ec&N=sG0smM8mkX~5D
z9_FsfGsvPNy{=dE1c4;fHpXT@&~LG=eBts$k%#2Wq*k#=p!ll#t#fnq<Ff+p7;D;g
zwaQ2>JkT%>-Rds2H3XV~M|TfO-iMx%9txv2x1)4dETC}(q*VTntJr$wxBh&(T-V&S
zTsu%}-u+gs`3C6go<cFBpF&=3`a4wxP{&54%Lur=G15z$=IQzYSmP5Kp0WREvP7AR
zEa;YdIg8HGbbqkC_Oe%!Cge!B<EyQ6@alg&7xt%5yo?ZrVg%@grtj)o&4Qn<5_sb|
zA9Sjk><tdt;vz0`wt4d@DVy{F1ckw!HF6pTwszpCP&FfY$PYgvX<{kzmAawL<>X5R
zogO-`k@lz@t-jik>t1m^yZ5=RM;6j7vt9gxTc00Z@s@s5i}+GfAnek3th-D3V0UC{
z^k@2h+P85~q#OhL+G4j+lp8yGort13!b77rqs*Ix<tL7=zDATMmfsgyqsb{u-!4D_
zzjeb_hb`jEP!7UNe$zJChg|Du$7^2uHmP0Bi>CW{q{haY>bn9_SldiWGMJf6k9rz-
zRL=^18_ilY67fK^$=U;R)xSq1T@sK(Mxsh0Z48Q8j9;Yx@ao^k6>L6bArrN1{z7pd
zRN_tvSl$>TJr9@^Nu)x;sfnvhQ#7(sC=HJ2sTTu7DYl(5`Ma@NQ{190;w?5<9xqpE
zqI@;;NIOn&DEd7zzM-W9l|Ng?ttW$N<{#0;l*r&Ej;yn*k@$>7G#!ifh>LDRwqPAF
z_>nml)VZMg^E?Lbq@nzF_tEYu_zo{F=arGnJvgzQBb}!st!-$}LW#%(Bae8E<Oj|?
z7FZakv<?lIx_H~JX%e3wL3FKbD}TS;9c+Ywv-KOWGsz3gaTzq7{`~V3QL`Cr`M=i^
z9sn2MiCH#->%Z=On<7M;V&%YmcKN2-{0YOQHohIC_>^Nmr_&y+)Wh%aufJ^~1Vx~j
zCcIn#pDN4D^TrK+$v)4<?7$sj%wODJH**IQ+0+G8X@WTIrcQ}2$I5{etO2A+QL~_c
zowQqt%R6*ow(lzPQgAyl<{XbVj)0BgNZ28k^w*Jd37=dVYhDB~@9H@T=wVD78?%`S
zta~-pe0MH+`)<(8YQrii{Je_py*lGp8^gR7=?fSfUQ!~?v$fC3id0(&Y^{2!*^gRY
zZDaCk=_~SZL*Ef3X;(6!gZsoP5`ylzTX1|2amUX+nFiMx796A;Sv>TWF68i`&cE9E
ztr-JMC`Y=TQy@@Cl{S<3>%odw1qUmB=h%&}G`jH`uT9KWX9nC)C2UjV{i%G7B^{{}
zsr~2##;F2h{v7RloXcw7%vi3-xb;54%2KG%>T84`X6EzoJ4Tgf0(O=zp%#}c<16By
zf77dH_kJbQp+TuJA>`^tn@*NDd@p(8o3$ho*Ck)_{g&9WsDG6+kBr7Bo#{I!g`HFu
zQFuL-{peWid~35rVwbL9L)S!HK-^1OLv`1YO`{8=G1kxJJ7ngb3P%WuB~!>|jjXj4
zD~laYml_VVYXi9mBeQkAzSTjPfYh~62_csnjWLMz9Xixzu>)awVU739LVoQFG;QKJ
zSD_$rq)-2=p<lln9?}%z=||}GmH(En@I#4O<DYcr&dBC9votYUg$dGBv$V;!_wvpj
z%}AwH3s{xpVM9cWGu<?w<Zt_6+mxsu$oKZF)q3GAHQw@aM%TF~c?x?YGT65YUiciv
z>*_0H?p`$K<~)B9Mo1v=+|5Sj?o)d=e2zS^jDat+mD}F=%m(IEPLd=33SV9j`=f+&
z=nq;G9uLA{<bE7HL29`t5wkWbG<!QWDQ<<{tEg1X<($Pvi>s$0*(ooNFIv6&jY?kS
zaio<-kv2`YbK~xcN55{{A?khL;3XNG<X<z2K(?dS&$Gg7Rd)*h?;1oG4u@tO@JQ72
zqXJ;YVMTxE*z?^2`LDa3s3~Ffvm7!x$1A69H<{&#wR|utYv%~j6pCx~t7cg4GP1ZK
zf_C){ZkubxjoB;7N~+rh<K^vQVLcS>;W0u6xk~uEhP2Pub_=|xl|>CdIro+tWrQTe
zt%!t8$I}*4d6zMv;Mc!daQ3l8Alwhu3mgmzhjG9Zg6ox9huXM}+u`lKlm@1l{uHZb
zo|yxC2arKUrNY)DiCJgheg>?{xC>T<^w;?{<MGATr5&m*t4xvxIq+k2{8RkaYY<tb
zZIj<&{?{J|UAUK4(|*1E^r#d4>)n*<0)gvX&ha27_wOaV$#W93{5`9>O%9&0<Jmc^
zPybqG{tpYlJd4_b)}O9I--;Tsxc20J=b#BM>LBY@eDU^E_u#cbT@{7Mx<!|C<m{^L
zC8Qh3b{cug;Tw+?Y35O<v&?KSN85FT4<T2Enxz$dgM%ETj%KkG@L0>d9g*`ZcWvr6
z%}!w9g*%Z89WSv{i(84`^@>Q$o<04g$uWM)P?(S-*Qkth7YZ7`y#;4$Q*NogzP>@z
zoS2#p3n}_U8EI}yjl!UJR&~cE6@=&CToCKa5Cn4d_);>244(~Td%gba?<1nAN^HNt
zwyk`yumAPSb)A9kt20kovOV_a3UtX2kB3KX?;b$L%EwoKlUbvqVXX^G*SpO_)vh71
zL;b~@X(&lcIzkyQ@7e-;q1)H&A&MbAMoMN;hRPcn#)=J3*-7nm-)1jIAZMIM^N}Gq
zqpNPJq>4gpbW}r$2Vy4*sQr;*OLq^%6>0KtqFt#ZKs})h2;_0>k*7Ue`QC<vNtrG$
za-ldr#Z||WXsQvjMHi%Tb2r{!kQj0FfVFejx(fSraiP6UQ?dfw6b(dN@x_GP*9|h4
zeF{_FPJ>9D;4xsrvy8g$rQHPC%=vI9Q+<DTF1B${?`zgWaMCA#rJC8eL-&sRng!oS
z&`fR=MKsS0>{;*&QpJb_mqZ-BKNf|dv8H;W>tTpM4Wqw|g5-5pfx<#UVXkhM94!6)
zZGH9ScfL6c?*zlKm$!-S-eRjO8|r;0e1(1HH1ltFQ$~i}Gf;L8r(8p{TeTlV!`?l0
z8iO3ah>FZ|n3e1fr-C)Ne%~}l*<1^N{<^>=Q_2sT-F<o3<secRja;_ibjsIaJ=;63
z<TEG9(0qKUt%ewbz}qDz@KS0*9={EgNC`Y!spCwD)Sx{yD-{A^ko~ROi+<zZiOL!<
zks+SESItec|59oEy;0l<2L`Z>@a1=SK`<A<fZKO0;rWHrYee3!n^@+>4Z%JM%B)sW
zuMI1zif)zV&0soDEnIL`FTQ%Un2_QufDu8Xl6a))-i@N?&Z?i%wH$=i!|X@rf!()$
zY*Njv@w146T_2YgMyVa)BDQw-0Ng?}&aUe$eINbH((s|YM8-SHFD`vzp~{aD<w8o&
z#$u%}m(Z$2bn}k9Z1lsEB2RND&v3Re!FK(<uiPH#jSMZWL~NBkHhjvCT9z2mjTWlb
zawa^|Rm#lLRjTlhUlG}_sID@$M}FT?@T&Z*sH2uOr+%}-!F8aD%HVVC8K|iHUjp<Y
zO)yYS)&46U$In$B&b+uP_gQecT&(L@`&68;kR1ZfJ(5>SkQ=#<li*>i^<9a3nb3S3
zt`i#MG?F4=bk$PW=gbeRukj?h#atv)!RId8rHW%k?yDKuf+@f>Bb}3T1Bn_W_x0K4
z+2(4pd(GO9Ud-%dE97Rv`+}~9DFk1EP@c_C5Po`K7VTy~b{8I7-murz5zYCnrc9#8
z5~DmDqq+JGd7Hzo$fY*8f8sTmj)1LDtE20W{i=ZsWPZK+d({~IyJ~zLuy6jYYQUhX
z0n0flHUYI|*F;S%#<DNomZSn9G2s^UThv`bSVz#cHZ%0!f%#R_+K=`utv#_FOq5Yx
z=aRtzO0z->Nw#FSJ946Cq$wq+ghef6o%cgow>@H}{KUvJ2IM&*LPeH~LW8xnI;zTH
z$3coli9mNPN5E&Ji=YiM2ZAR~IS<)7CY|Z)D%X;orgum@?Ydjm9|J;Sl#8sRR<xU~
z+U*<@r#$L-nA8HezRckxZP;A~kdo{oczK*<lw17Pns)v-bVuxj?isGQMLn5jCTlP{
zDgz&fIsC=Xo*y(^Yua4{zuqT+LmL&gAFTYEC1w6M>+J$Wo?5sk?f;GoM26CIKN#4%
zk`_eoPZht!6<9?B7Nk5HAz^pYpWZ~?On-LR`$>x6?b0(hvm!#~-Xgx&>~-54NVoEL
zt=&!+{id0Y+AvRoBkyEYug!U5+Fpllx@5ppfzZ{wUq{uY%>W~#j6Zo)+D`ick!S8~
zM@MSpPkU{(GdhUAFF{Y*Hn(KBe2;9C%iz^T-=?fRd_aMaE1USaVp>ah9rxD}d`zIN
zfe-<S`kIx{uET~Jlvw0+E7E@uPbyEs1gtdbJT9#S0GaF%2(4%Pmj&1A{w+*8;4ty)
zsh|sA`}-E2tmwS^>sL!Phi+kWRTG`J;Jc{4wkdn_MnEq~kcNXQJc%lg3K8)%p<0#M
zpPFoYTQV#Qjtt-e$EF}$Sj7vklc%4#!@SV?d_yr=i7>0nS`Q_z5J&TEtx$dDlm0%d
zM@{owq_BRj?u6;M9{3)mZ;%sZ%p&AP?zwTk?CH!*2bN~ylp2fnIO{AL6;I&Zp<}Z~
z)f(*5Yj7@dc^3>;o_VM~mt!_rGa@0Wz;ud8ewnLyBorC}R32Ao8?8>AyV7M;T2Zoq
zzJ7w*H+aJ+8wT%amB>)|L$WU;joazKLFB3ZD$8r#a_aZIC0U1`RTft3Y3BZGe~U_l
zkS6R)5t5+Ff^IFW^?3kI!Z>vR%_<Woew}8DCdWkAQ0^_O%4s%a&^(8%0!KXz|1equ
zK_}93gFGAex#oM@5o2?1>lTZ6OGO33pt&`Ip#j{XJNe<JpyZMg3JEy)s=ZYy%S3$Z
zGdcDm8jF}}(?duSNt~LI>SJn%{b&OHL(_H&-)vlSbv>!0-L!s<NLrGnR2%GcbZcK;
zpa-3Ypp`42@h)24MsDP%WN1m#)vqn)bLh00af=|wIVAfua^O;QX%Q{Fan?70Td<7>
zruW~dF!{g`0Zekbq%D#Ketqp_KnWWeS~XM5y7x3t_SDog-DQJ66HJt#1QQ!ANp(#X
z3wmFoCsV$rbO7sEzccbwFO7N&rKBIO;QdEpxw(h{vWl&xGeDLR7GJ>^mran3g75Wo
zYhqG6%L%s9?N1YZLq{D?9m({vIoxD7?pY7x`D({(8&4H+=N`Z4dSA@7-V7yV*%Ph0
zKf3r-DishT=S*;OcH(obx&p|c4*OFIc#5e=W7_IWU4CtxE4xoi?RA2AI~?H~yi@EU
z1@#~y!qZnT^P)h%3Y*kP{`CnA$x)FI#Rlo$|LV647yKdUIdiDz(F0e|kQvH|?!1&2
zo^A;a{kE%j)wX9h1iwl(by9Sd!D-ZKasITE#F0H}^hTfF)f3pS<wonmeM?y00z#5N
zb2Q9r#8KLWTBW;=@)W5?7DFgpRQFz*<xPj-kBy+oFuz0&WJ}#5uf*uGS7AoC$br)Q
zDvgaQXH;!LO*>AXyO%d?Nb`^vcC>tWZItv4cgq07nWwIXqN{}$$7R)!#Vr1ETYT)2
zhkzGFi*(l{7JC>5O|}WvU%}nV9BF0Te)-((DwBMk6%b1;iX-nr|6ug=t1rLbH0o5~
zHrb)IXmS0G{B`&&xONXmcpQxYk6x@qMsl)Wu=DtA${)1WR8~|G><!dAv@4v3b8MWB
zC$WjMVLhtrhZp-L#&`?!QY1A)Zdi)c5<G)_-F)yB4SU2Ei3*@{<kkzeo4;bCWvHr#
z_G#%KH{>%&6NqkHqu82@Jt5z{E8zJ$qGP<=Hx;)RKfz1L-%XWhx!2kK+IgqTgZ+YJ
z>rliOZP_g0=4{v&FN-YloJz3UrndGe^k!$O+N%HYw)2_U<qe~P=GN9E;)rOrABf;0
zBHk?x0Y}35Hj|@$U|#Zn+^F~d!DMl5jP1EsDeycc!1SSj4<ZQo*qy&Tgxc-SO)y<=
z63e*{0zOs<rq&xqEcesHvPu0o)Mj`oGWfk?FhH=B$EYE_XU#X%%W?-X<G$?pu{Ez^
z`%z|P-0%rUWzpulcj4ogLZ|0&(J(&^O-W8yFfG+rsRvi1^!a!$;u}OiUAoIV9A&vq
zwu_L_bzixWr-RZ}QQ0%kLE;*D8k$)Ms^vs_J>e*(oMM+-e0{9#$9In(M`-k^?J~eL
z6Ex-vg{c+)O=HgTvoW{ANfrKzhqCz!HRfMomW%N~Lpx2E*7|(+xx7Eu_=3q4ttj!q
z)kUsYnF<bM0PFJFu{^e~VMhghy5^Co0zzlm+lV8IGN_x?R>b&u#eA8U($PqQ*`)Gv
z!38-+HN>357Yp}uM!x9auGr!p?ReiVOmEShmXm)<r&>s-MZzW3XVI@0ms?rb(ZL^&
zq7D3}BXGW5rKPBy`g~#+R~vbN62Yz>t4QY8vmzEfe`{`NjgMdH^vp>PDS2WQT`?-T
z_QhBTMFe97_Gp!dU@@b74C6_YymolMCk-mKz5d_yq>4X#(kjd9BE?@{`PEQQTCL+V
zy^#Q!CL)@^MCea8VH%*lVr&YSYETcN62={EqzYVV&xo4BybttrHwa!L*}4k(4!uIu
zMT3WeU->%k`*2Q7&u0o{I@e=jwil6*izr{--r;badvm|A#KnDvYxlSVcPOd6)NN2W
z$-m@TL(`nSZP*oSK_Bw)6%Qa4l*w;lx~cxzVllRoAy_{|OvbJ*(D5rj)p*?X_ScDM
z8K@^swb6}1O%y(a`I<%ZaHo?#l#%RvlviP&DJAV0b7!y1xM4Fi2JK)Yq5E^7@`iY`
zpkMDvO`D>>=Rm$6Uyc4+NAT28;iMj-H-P_~1Kr@qct)HQHH(C0Q2is<f?dR?3+QC|
z@)t(SxeN&`4G*sI8c+J8zRy}lqh0#Uv$N`kT1sV>^R=;){xhc0E>9{Q`24mtWTYuW
zq>eaK7dt<fx(z<k?q6~T8d@I<o*LI4EJWtI7qQlsYm5={IMkj7+xeZ;$QluU|2>SZ
zxg~fwpo2b#D36=3spyx$dw$7^sc3)~NL+~eJl=BpdRMoFh)-LRVMIJ>$qlScNgmc?
z6ky$>K*~a9x4pz;H2aN(sa3<N-@@cYWs`q47Ugw8W<I}uFw{OPjR}A8;u-Ks40Ec>
zo2Wsz#i)V&Ix2jYS&+Jloid1(yyto)s+diLNNmAMG~twdVnP<@g=`9Ui!h}YjJr$4
z67AHX8)+;eL6422g<+pV&IS@b6X1xU#f5XaY7}(#P)o{n41v-rufkRIQQPobxsAf}
zvdbH%bxf7mxUL5A65`lI@FbSzouR3BkCPf03NNXMF(&Y$cX`dTZzOh>k+6wbls4JC
zBl_NYe101>?8rO3^g`8rON0zcJl9GWfznmX;nl{^>GahM=<~e6?Fz(KKc_)tafjb#
z@O0s@zx&XSk)7VZ^`R)J4@I5lQJrQw=#+ejQz@Kb+DbjI&oet4rlv}r*1|{hW}H&$
zn`4sIA>Nm<K~&K%zYm`>k3%d@9~&YSYs)BV3c;5xT8!cZ4^+=9n_3H~?NO8@)%EwH
zNpl9WCi&m2{wbekntMYVrBq{z?huI|=AO|h!{t2HmDF&Jhbg-%C|WEGr3kANHaB>T
zPm;+rLnfI#EjvRd78sYy9uqZ+jSX{!^eS6rwYukFi=4XzO3{5d3@?RsNElEG$G^$Y
zpcR}wxqm*$R@kE7;BTU`?tee;YrV}pemfrtP~(}f_-+9j6TW7liPPI2lsbR@r#?80
zFN6xEL=C*HJ=2UD4MuZ^d2eU##8@!;jJjBp8M@qnI=J8HcXcq~q3y!|7kTi-|Ajo5
z;eTHqyt)5Y*`U}D|Nd6-I`?y8I6K^#_S+fJgU*Payoo`{{#zx-6H>P_aDau(6#aR@
zf{tU0PM(KXqq&;|4XDb=;>2~X)Y)9N`hcR_`aRSEe6w16%B2DM9EQ30qf@#P>RV+e
z7Gqmq2#FAol_A+b!m|T9o|NuzxU~*#733aym!aBaDW}|SucT7m^yR7KQ@U^q`6D{U
zLxZoG&t>Mk%$3ZBuWp{rxh2gz*X%Bgk0C9^%0*L@4FuHuWd7oNi^AQ_swn|UeR+x|
ze17F4?b$oKN?(}N{H!uclCm;ds-$sV(N90+|FlK^z0dkTe%k_m<_XcA=kft;M0tO<
z7*#5tm~#i=^b}jU5i2WAp>|$a;?A5nLjyIcZL~OCSO_)CD?u7J4O~vY?e8)i@V;5F
zmT*Q`AVOG$5T<&&vD^2wWY6`eOb+A>*KCHNljB;-IY>u7?Rv<In|PirOj$8vqC1uL
zvS^&>7%{KiYGJ51>95pUhBor)O?o?N{Q1e0v<GHo!+Rqj9)aD5%r!gOkgrS=51zj8
z_IBm3o%Q*7><Tm2rB<@$>C%k7Y4kPi;(xXwvvH}<|C=_1^=BL6z$4xHTN}~?wIMxX
zmaBI_8xnAx+U9~Y=<j-{>lnUnXdS%I7-qmurS|TE)TkOWLwyEQ+uTzyCiG++lGC{v
z>N;1^utub$*gCdjNNL#Hh>XF(RWU_)s)R4XwIq&GbS;D)O&2;laZ;_CYv*Q9Ohq|R
z5k8ejseasRy3E$4L{6qUGI#b13|6vHOYJ5tD5CJP5k@l`_g*QSBu*r-@O$A;SHil4
zSpHD6>!rMZG9hSnbf(`{F}p!~csW*=ScojbZ~*;aacO-|bKKnb=WeE8$bLq$@6gdU
z)1P~oh8Q#SujNA1rt9zf*KhjOZ<!`|LFoP!9H2K;vZG5=W1%uhZex~TrE=vou39j>
zy%CmG-oob-gI*wdGOk|BNN23yi?2x11H4g&+4m>Pqkh+?&0k-1$xL$$@2UBYw=Ho(
zT45@BUzW0T{n;#~xy$y%VQ?<5T#WxO&gMU`*N5bqkFVletn^k&t*r5FvyQ<;3r+){
z@{sylBdvlW%%{GS5O=E&yWz`@quPKpyb?&lCC6+OlZDl1@?~y+;3pyCv2W4N0?FJ?
z2Hni^z|?ePzb}iIu}l7M`ZBqneOany)v4e5GWdQDdbCe7cOq{%o{v7P@M7%~`S&Mc
zKw}0a2N-CjLdgM_d-t_ed_YU32x+M>JM<Uiw1>TT%Qzr0RY^4Z{~9qBOSsNr+iOrw
ztB?2X465%MV}Gnl{9aN)OGf#5-5^ds{`Cj78p;L^NZXIQki)k$VsT)=|4x#k%QS1>
zPf<_uU$h(=Dr!MgL3~)AdrVpxekdD|2auz?NI>HyE6d4GB(I0MwVaO16XgOg`o%>;
z#eJUUydFn2B9nr5=k3dwp6d&*lD){ONrEMly!h8g?iS^jfiR5}B#(Q@0<mK8TU0&O
z+_gM))J_FlfiWbWzP%ywHA49g9H|*ag=Goz&BwZ9Cn<@=u^5+kI_z}y4L$uQ;#<8Q
zoh~78M`z+gR_nnu8FIWu!gv5ENROr)iUNX|*4-d6Zw%0RGT4v&8iKGd5b!*?KlQ+0
zzjXQl&A&g~)Q{uPX7`6LkR!86_MMVSNPh#VR(g?XbvO(=dzdO!$BC|Ke$nrXW!S?S
zx)@nM0}q7j!Sk#3td&jtz%H{?ig77K&BWcKRks7P&xy}51=^vg%`K4<3~x};Qffs$
zmNC?~gj)^KWo9C^8{lUCwd(VfcV^!Ij%p=N>wl_N)R!gxsab&$eRq|GRkmWIolw31
zweHr=fuPlC5Za>qW?}Kj4xm&xL!c2nz%dA-9{%^6aJ`?uFN>#X7XSWSz{d+oEaGRc
z30Q3Guk(JFnYc?M;k2WiaIUsl7X+N67`a7zLHBF8Anm$DSFvG94Pu<uo3FGuUpa&h
zCQm?W^@%N7x)fGvFYLXKl_8+?OD50MHp~p%^@H<_S&I1Ku3{=CA2>J&#p2I)X?=@W
zr2eGMQYXqr8|_cT|AAFtR12WMsp43|g}sF0J8ap|u4J=p)ArH#7=P-%;qKibxN32m
z?=V4B#P8;15rOVt(@E3KORPv2oZosCSZ{uO#_5Cr3yQ=0)z`a(NnXRq;P}m0ISBEx
zFXC5=J$^I|7IMGn*>i{}&<$Bv9dZ*CZjOM=j;J3ar_5eG+X8sFu^m$Bjymy2;`oxm
zt(y?y(BU?lf)mhny~5AL$+){$h3lH={9BLG<o{ED(e~f9NnJwFpkFBY<jFzst-qgA
zIwiG28L0clp&M47L523G^Q7j*2INF*c^W$}cb3ib4DGc*ynj0`yRzvxB?TTdzn5>r
z=@swGBH|brWk#mA*_Y>9dK`t^fh>z6_2&}ewqoYnB=*%RU}d<w)c+|TTUBEH;x+fd
z;`}l)`o46=_;P^}W}+<J3q^*NXxismiJ}%UA__7O&LkA<R(mdT*{v4O78z}}$SL+z
zli~^rt#NbjW^OO)Nd={T+VuUxn7;i~7{&iIQ;^e1fvk(ZGYCam9y-gpossNW^Ff^^
z;QhX~iZ;yLz1m(cWBO>C$nK)<3i(Ak`P<6Q2Uxq{pzEuS<K61JNk8a5ux$RAeI8i2
zYI>q%LMCJ_4v!Vt^OXC-Vjd%)3tra6np!&dB0@kYv|@@C+Ia(tTvGFLj4_{0i~1O~
zw70j}s0dOn2(rd`pTO@S4dijwW!B;#@|*QW#)4`d1jJWvw7tED4q3e|@&=dptrnnO
z6x)RZT~W24nhVqKLBV4mQHFbsDq?tVHYb<5AKyVNhv^1?)l>van#kU2bmcjN2{!<n
z=!U$m4gkV0ynJp?PrCXUeEJUejkDdC^PcWN)@Y5HD+qEkhXOE=Bo8`(m<ac*^@n*r
zsFLYdCGhPKyj=FjEo(rl>IPiH0r$O+y&-R_SKM}c?enrmrUtY3d{BT=$wprHSOCB|
zz1JW_l+A)Rle|fHKMJRrQ2qTE8H>$BuGQCzWxdGUy01mQfx<*(aZIa1jguR=dUW7d
zk&k_Av!DK^lDInn87Ma-CQG=2KKF`Q@k+mQdsH@?aJZs*Z20T2e90b*mMJ}qWjO=N
zHKzJabqO<}IAo+510x9yN*uI&Z&}JAfRJAwtkv0=!IXHE?k=Z&rW$Iz^R}EAL)zYc
zT-GyHnx3xt{-}wfjI7lmRAFhj*X(!K7JU*6Z?x6Dn>H_A%ESzLUSi+4-&t{=Lj-3W
zHKA=E&#B7_cDXM=TB5a&OAd}v=*p+}j7x1Kn{Nva2EzP%;K1!b2M!Q#8XbTH_tdAf
zEDhzV)psm;3##v|AZ;ie|Jyp?6_Ae>@-y95l8oW@Ko6#;Z5{82bnZ@In_cfL!Oo1|
zH}pV3ilUIo$Z#X^Rvl8KvaSQ7mg3{(dQ#^U2G87|iZu+Hd8S%111vKr0Py%U^wKW~
z|Jcz$+>V$@A%wH~fe3hxD$UR2eM1j`+Gl1dA|We{9`BWpz0X5l?twfu_e6C7SM8D#
zn=i)%e|h+QtO+f;3S9Ws1;Z0$2qDJ6Bqg%-UiZe81a_H#4!yK?wXa?VpZ0*K!>NH(
z?7_B}p$a_H9w1EURvrK?TTh$<u^H*WLXrKgO-?8SVub_(x2|-&@vYB*L&!cg$rnhw
zw<7G>MBV)@^v8DAvB7_jcVugR_7(m0I%a0S_N6Sh!|3>4!hncfCLl`Kl1X0bs>GTj
zBxg8$`ACW%(s>-0xAyL3x-y*Bll8b&P^CB5Dlpp{(F-6^4{cA9MZe=kMXHN$8$v=U
zn!URftIh0744Ez~m^f#f;I4%K$fR!eo%}jXS*$P@5dzI~fA@`3rwMe&`XW<lBcjv&
z)@bhNhaHQz!D6bZ9t^Tvjw;SHwGtF;wL#3R+JCj`GQaAU5m1JFjR>%iTq}Io#zL#k
zPvQi+wE|2wsk5x`TWc$am)l5QJ3g+LLVkcj@&}3GNaYeGLh3uXKiqDI9tFVrm2bYK
zfsfqu!6ml*lHt|6DzIwlVGL(*6@9AlJ?vg{f!IAK_lCzwm3Uo;i;FggGakIVaI|jv
zvI9_^A34|xU_T*)Z+2Zr-l)>B<z?<{sBib~j(FeuLn+e1Y{0r#mZlrv7aG2MY5HaX
z>tYEU7yw<a4fdX~$*`lDo(EwFS${>oe}iRLxilbS2y?Y;>59^LU1S_|<gNxPrFQ%|
ze659>(ZJ}2Fhd6G)$GLAsy{~m{kCrUptJw%F8Xrv{%@hoMXr#Dl)dfy@`_fD^Ao-f
z_5aLxo2A0ir`RqlQN_EhGOmWFh(?F;<a|H<h0MOZ7%L&O)fJwl+L9E<Afo9`*Dz8U
zRqoAOh8(ntD!!MpvmA>|v_W!=zSlBz`vRt~?#26>uJxz7F^+06+um#=?THC$;uVv3
z9FBxaDaZ9mw+ND=#(C=nM8g2_z;yOwS^9vk06Oe=BCKIVE)Qk5np=wEHy_Fr5}7Sm
zX=8**@_b^MF|`gHiLAoMew@l$?Kd!#V-NLqzGdK{-&Pl98b_BlxW^@oY-!8d9$)B*
zDcAyf>WX6*$?B)Q<8#T+=YzPMn&zy3kgUxQcCUcJ-}n1+s>u!w=jDUXGU8&Bu5vCz
z9(Gp|wxQ6Wo=*Vwfd`pMI)wIYTe`GwK?^Dq%*wYKw<kX75{59+;)yscB;~Ge*2GzJ
z=dFTJpvGqTs}Pdqg5<lQMBpwB-hl8_*V*F5&=5qLwgx?5Z3sN>ApaRX;CyM9`$`wC
zH*&9ba@XC(3Yl;wEpU8RxEn;lKM%w;f?v2>1CXfQ+5iA+_eJD0|IF2xBH(E(ss>cj
zNvm1}{+<<ia~m=cbhg_l(DBbCpE4Wx0&M!MjDH0Hn<{pxv4p%?0dU5C0sg_Oi+K$Y
z;Z#T%_*R{0g)2^3*1QScIiz;35oGST9W2QZZ>fm?moOO!zBGQ0Z?MB3`2T&xfR54-
za+_ke6PK3(>Se3$z{%u5s2n_+ICWyh@VVDF>nz$$G8fvP`b;&w2#D8U6T6-mp%${r
zvTV9j(Rt>wW8%yJr4Lz1kUK8ZFd~f&hi`Ea<np-m;=M*>UXEBTw6v4T7`>mM)bXdI
zH)aNgn?&jf{gox(Qmgplm>mU8>F?dv2Kz?zLPVGqC($cuapK+-OME~~`4XY^#0!P2
zROKFc$jGL)MtlSDKm(`AstNBbGMGeu<wq>)C`Z}#RU53|n9R@~o_BEFMeF0mqI8vj
zB>6<{rJQM<ZQ#kQAJ<LSC>jN*sE@ZKO)R{$v9rN90g!epCEQpYiucBY^I*Ah+L<*V
zOopzyL;Wt0o&qn7?IZv+2IxJVzOSLWci<q~Tyk|X;z_O{`;*Tv(%r`pl)Mr)c+{B_
zz)A(q1&->%ZEhPY(n}M4@$}WvmvPcH@2@5?$zw-`;D*Si+~=m*@6z+quAV;pQTvCz
zEfKW#)gK+RgMHk#vv<EZt{@m6R$fYer1t|bGkpgdOG6Bpjs`zv!|X0|M>c`@+8<=9
z6Oqag@?~r=q3D9TISpI6>fJb9@8MUfH2Nz9vbGGmziGCL?C}A35}pJ&O}0l2mRJEm
zi@|y%@7o?_f)53{FSIV4&(vc0_d~(v&d*_mgpbjyhyT7!IO(n*pn34uA3dX1^}*n}
zkeAqku8Rzfjlu!oU2(r2_)SEbN*|%g_g0nGDr}YMW_le>P~WVUM2cb)A8IxwsBI+J
zsIq~)lq-~e){0osUG7DeeNnDJCr#(lAFbRrjhTuQi74rrHB~F0(^xL|TejgzuR$PQ
zA34|?ITTmQV@}|!HC32$DC%vaj~A5R8J0U9O~nQBM3gk!p^560oX)Bw#<%aWb*4)t
zJK}5eP6h)~XvX5?RzSFoZdW(2`iNUF!3KE=tE(15miWpofbkv!&j<tIYcy<kB6QW3
zjikJ2Ptz)n23+lx{pggN!0*0}gP7Hq#QRitmkd3=0&lUOyXswsm)s)x)~n8tAw+SU
z3b0}uWFH*m2hUp%8nzrSZ-%3fRK};-rJCmeqdXgRu-k`8sEGa`!`z?q5iHV9qA&gc
zM!CgLk~cGMeGcfZ{Y;YXMqSLA_HCU_UrsVeRQo=PhoC4X-vA!X<@-3Q*Zs4|rO5v4
z=5E8D`ZrJaK;0_C%=@c!Wd+_Omvz%Z0O4*l#$f~@^v;cVWh9?X==ayzYHy_X<_dAl
zd~xlb!vl@dQ2UW)7`$wp*{Nr>{s8ZV3A9iwWvt_i6m0EK{BV}b2?73<iiJh*1}Vqy
z{0Pzfu-pkXVfYJ%=qq_q#CC%6KYdr=e6&&j9QU!CoOJmuVhX`^PgINTmy7B6^n(Gw
zUum?qWyOST@5V13%hCu7=kY{2uhRI#Zr)Z86AC@22A>npM;c&t1d(VuA;m9XnNiLS
z{^F+A)+-~D9RELxC6AnD<%xucFd;W`kcoXs*cxJ&ORX2hM$W@hB$E^z{?xqw4I7Au
zBMg>+KJc`l)N~s~LU=940Z@b&kN9<;!O2?#n{2x6IWA#>ma(X)=j(bI$do8>p5X%~
zgqX|l>YVjg|8^Ax=nn$!#FMIT?GIg$2Td(24-S!{mPgijI=zNaU0#3)*%u$Ax^*vc
zDvPc{-T>M>=6X``ZOoxG=5U59ZO~vqsyfvBgU`qiy^LcMVfk>WzxDzNBJJ--D<ogV
zE}c4n5uGtQhy805EeeejMEINNu6zIv$K-&B!7>Zgi}^ATF)ck>bxtg^=6D^e#H&@_
z0#IbyGebk0Vo#k3pxY)`|DfO>=t(bGnB39OIw<_SwLmND;hW2J{3Y0luYQ&8B6YA3
z5dQWlXf4kAix1zgw~p@5TW9-G7v<Mm$J7YiI`^W>_c2d5oOITE_qA~vH-I)y?=~$}
zdrt2)4Myvnx6`?o>Nq8p7#1I==YAa?>@>=WHpwXw{zX3e+HYOFi%W^M!Z0-6QY=}Z
zuqao?SJ?G`TRuu${C_JRbvgdGcr@Y9aJ(%?psDdln=<1aV|o9p92m~^Pz@zd3Ts~9
z9}pKQ{dF(mlQT~1<*0LBX3Z9e!v2)uH|9K%(G21*Cif8};L%F`F(+O;woOKg=dK2=
zxawIziwt(z<`ph!2e4_l<Iyz46WWN_yk?DyVJ*$G&#qfO(t3FO4xt{<msH%g0sKS(
zBx>LRKS3l0Vp60-z3Ctc*Ajb9HN^2xvl^eJhjS@V{6cH*C`451293+tx0(i5dendH
zKpZr0<-ScEcJ`r?SwYC1gFg_W<(GWlJz(Chet}j!>_UBO9{9Ua*jNK5TqrQG4Grym
z8V9ywhP?#8HgkqgZj}?y+7D;?3|#1b?3nejy62$=HdN&1!Gp1)a~fTmQ1c5+-*569
zE3^yMIRliqqR#|xf0}d;+3K{uKEnF%gVN;xx3H2>V_)&_D@&BAOAQd7-hbKE^Ys;>
z=^jU_oACWl;S19F!6l_(dUvYL?P)q<0J}FYu|0YOW6PdG6%y`;-mp`QMh3;gcp7|n
zPrJ>oX?eJ>zl+r1HOE*iM$hNFD8EpA!CGsvE>UE%T`NhzZDVy<mF*U}7DgSVj#?Rs
zMYH^66mOPKRii;VbB{8#7IwM)p>rTL)nTrqc3t+WUZeCMd~I~g_^>xfxSqO(%Xm#)
z!FRl3FGLItZuU{CS4&AFk|GKl%@omOIEjmpn{?Pa`m|N4>eEG{QQ)1yM3t|jt4>Mw
z%r1KaDGfs!B^vv92{5<_=2-{Fp)k1Ra$c3a=Yq;)z*N7|4(6_Ihv#gjvA!s7=v+pK
z>V}a-KWLQC9&~E5P109zW)@a`$>2?Y2UC3y@dRlG4;XBAhJpTZ@hD9%@K<vhd>=((
z@b@u6b5wlM=!~W-+=dRXj?aks3wgg6V`Zefas;zkAYw_k#z@0?CyA$SiazrPRPOKv
zhQT+%O)_Z4aa6=0z$}Z1Vw31w^_Wrn7SP*N6AHlSXFyXDH%8oTS6QQ{K!F?f$x&3-
zlI%SZ3mK@fq#LXmC&Hu7tnnV*6S%_SvOr9U6z|g@Mm)Tx-d13RL6*QUx0H8@gP(qs
z<Qx7X4*OqSLPvkQgt!cV&ykK_iD7&;FWcWs-;;WZTH~c=88Wxyf(D~wlRqlIJxUgm
zn5xpc;z8_CFr@~8bn{j7?k_XPHRO-+x;{XHgX9nlKM2}QO_L3E3q?|~ZXK!LSt~EU
zGHSaIF`E_H8+k9fwVIh4rmmovz`-Wu$FIOY$EgTjSVPt0Wy;|;R=8W2FjJ0Jgj~7A
zu|=_$vL(3dFN#x>cnO5wj3B195*C%2aZ<iqA}`$!gaTQ#twv$=_~t3Pv?SeFjn}*1
zKPiG~rCaUP{BAhWdpeTmtzuR-cq_42LASpUnue(*lBbc}N}G3em7E2I=1i{{kP42|
zG6yCZiSM5bfJ69l?<n(o@q0S>vy?lD_VaUW?T{ZbIc&9}G}x`lmp8mkQA%m#XtMG>
z-N+S?iCXdUaYq_c#$An#|8%Z3<BAhL#;>=()>fjd0_%<te1$-6V)yeDtFbMW)E@LG
zp%Apg-&?q6qe#0JL0jFkQTIU6t5v%n8=69@DGLTZR(Ax5@{x5loCQH4zGJXZrSRxb
z$4>6O%n`{pC{e)?1(-0mY+sA@T}~McH?3W-Y}_qACaJc!hWWPI);3ExB6Z{0r#|5a
zecajPmL5MazI)qJXaH-q0m(rghA@7=2g7BY=g&j%%jm~XS^m9Hkcj=MP44JS(*8C2
zrz{M_u|IjX`E+O)^t~+pMh>PoLF3!|XQ%al3Am{(sM2=A(5%@{zDqKw#~hC@?-cq2
z>t}%BU`zK4>c5ZiS7HEfq7j`*+>(d%myrqvuQ`<E;+mFYu+NC3aJrp-5TC?yzK0sl
zHK?&61n=2R?!GbJRK$HX+`X?`u9RoRVTt7X`k7I5rv57!I=q>^`3_Gi94LaYIubeq
z2fz~xtHAMH`G^V%EJ5ov7CYi>Bo$@oP^L+reP^&4I>P8Xyc@G|mnTU#qt9K%;Zm$3
zxEeAvUfTjU%xORqXH;9%?1oTBuZlFMxikTG^jo^F>I@gOeo$~a^%@$V9%~wkg80wV
zI-9c7<#HK_r>t=}zpeQcMTnFVNDD-ZCMFZ#bI(2nI+Mtwz`y+%YB|`>#Z*5)Jh@%>
zmF-|Vze^E}_!oz8bxUr_LqKS9GbZUJpK@kbUd)MV=N>-os$WsBo^=;wx)(&^M!<u;
zO<M2@K<~^|{79~>d;VG_^(&ez9teR+{IwJwT~ag=>RnRcyw>%px4E&v5s@@F|Kdo@
zv4ft(nTNavaYkT>6TbQiG;9@;dc)7MA&ho6J&#2n1k8AmZ9>DQEvl`VHWsju$ynl2
z8TL8shdZFEnyI<{5`b<a52modPL~@H{OaYeFQX&u)E7=)Wes<j_RQ|Q_;daO++)w>
zw+@VdL4MI!|69UNl}IH%9R-6`=nvdQcfiOua`7h?b=?WbQo^JKe&{lAV5&p;P;cOh
zjb|QrVsaeaRjbEy*S<{d@;kEQ!T<5~H(b4G<%6?viCYI^<4o@#-KDR-?0wqd2@RD}
z*q`U!(}b<5eP$o%jpn=f7O5jf^ECxg;pGNK6%pN(KjL$5xsmTh@0@uY37@e_HEySj
zX4E>)mK$NmiA>7p1YDAiB7?uoA+5Gcjqn6Fkf@l4I)MFfBt{NEI%&egqBLy6&q27f
z?-LAo5HD}J0j`R_G)xSYoE-{_-@d$-)OOE9sh^I9?H$i;)vtM>XS@Hn=d?3$<w3`@
z?ymBu$9pqCJCG!def@Nv@>jeq=1>g;U)tGj{1IrJjeaL^@I6=P?#Uksu4#`CIstNd
zHb6nLnumt!gCZuAcB^hy2y0<3RH}LYXvJ!btnjdLlCtRr)T{Qm0Ra4R8>@T67M}WD
zblV=fFNqK(;O-=%{X%3{v2m*#4;aEl*Jx*Ls~H5*+*TVhfgd}S*I6CPs5~MoRF^Io
zdMm$KfVl;G_x)!$ytw12mm{Y6<Bf>VPHo8r{|$hVBHgSW%=|j!`M^+2>TMo{SbC;m
z(9AGX?w_PO(EXO3{&`qVZv6So@7Kv|ZWz_3_C80vmsvj7vm&b07L=2ni^0y$8Art6
zbi!0LlGFy&L3G*zsDOj3G^<z!Ln<{?Tb*fwqX8TGOAFiUg4_@x>lc}g<#VqPz((6?
zmQ!h)+Yr#L`E4v-m(ei7w`SzDYdij}aKmE8Y$>K9?xndvMIU$cx1vZopv}xub^Y8u
z@cME~w`_@GdsSkyVc+VRO2Uve6<xS+flPb@cXF)7TzP|Uv95tDr|II_HC>NWkhL@X
zj2)`f;*$3Xd_L|54y9K3Fs!CwE5A22V{V4aK;U6|bzaFAuyMHG0GiF@dDe^iSp`zi
z>|~=XTFv#~W9@9H{C0zB$BGXClkEWvmh%HE`Z=cea`(7ynn84p)7U=?sn&{Vo`e}o
z?1;=1Cr9#afewb#>E`;S>@Tt=wJY4pQ2=%1TUk3%JWWi$kqFpU_??Yjk85E3qAL>Y
z6KE+`E*P}h-0kMksh-X}?RD|qvnb2<rIHQ{EZQLAN%vKDHDVtgjCJ=kOheUp;o%EU
z<D_pI03vgEux;RA`sX1i$r`D2A!>eEWvxnnPsIx&m`m0YVPgQ^CApi9Zarz$zv|)q
z=QPZW<Sa`DOmW*4;dnDH8aBAU+UjIn!C%UE9(_c-bAFk&tO&5ee!-+r!zSN=7Kz!x
zUv~VD685hivyUd?q<=TP!1Vr+X?3?@(_ndj<Z^QRqP}J5{!H$Xv~)N4&XpXuG_t6W
zcdF~Cs;DSASe@Vf4mQC-oCLs7v&2!_W(<XbxKJ>^3BZ2w2v#~Ya#a_Tvl&%-rayg>
z(l$&Y2)4*7CT<nAJbQhh;n@4g)HrR}b!(Y}o^;pM2b7Lnwi9xpWo>Ig-g3tn6s)uj
zY9Y!I4dni^`-}oSnwdOw0>~ZHl&wxr8zJDL;7ix7pA+;3TVi$P<=gkYM&5+Ld`-P*
z*uYr#x<dGijEKR5cBUEYAcW!N7BNp9IuXvo5I8<90We)3teAs{t|onQ0~R167oMDB
zU^)bzp}2G?+DT-fJNkYw6!?ZmWp7(7yzZmjpS|eoXF<sypCNKkYD)2UW+7uPr$5-g
zl47OzU**ti#}Wqy_Fw}%oJoFn;!g-V#PHn{Y<m<kUJj-_E-DZq2->k6EqZ+5*&~SG
zL2J89+5yvL7e4^A9B~9ZduI^p!oy157<2fB6?PfvZOu(w9`pqh8FhJ3XFM3zNs{+8
zp&1S}33}N<OC6Cu`1G01*_V4;*;~ECcQgj3hf;4`2M;1=IY|&E(NrMkySF9US5^td
zJDjBD;_x1nHGAV^wDB-c)xN1k0#Ctk3V6Oh`fo9FJkw8C$T{ir$y{{5ET}JP0hTEX
z&|5vXj5_?MY6M-{$<8RzK2{Y=1_LT_$uMq<Lu4iepLOmx*lU+{IkLm}=N`1@2xTg?
z0dN7ZpO<h6l*a_CDO?vIRhgZy%V0!JVNKJ^$<^%0)Y)(2g2b3=LG$ct!8qk)U$E3s
z21}iLtyR-#r1W^}mD?~HCRc_6YH^MPQ&Lx~9D{&zOB^7<pj>8=`aZt;-*z7Gh`7bC
z9ZM-6O(h_Mciv+&$~QpCYDuRRS}Pf*jCds)T8e}c%vmOupBJki0HS27a?oEYZdSFc
zeRu$dQ`JFW&+05Xa7s_~f-3O|hnrGsxbEf-I7n`WW2Pf)PRnb&kEGs?K83>AJ`o{m
zOaq9Tkltt`tpKtI!@&-aJM>YC`7Dj%;&$G!4?Bx2FllW;O$*=m&fFNh9_WSv2L>A>
z4($re4<2$c2VC5CFHM@|B&DqHZ}KWI1}`@iC(aDRNU80lFmtIo2QbnR8Gh+k2PW`r
z_adUW^pY!u&!5}X&D4*aUH8_%F?Hxb*`3kroe)tT4-Qpe@y$(!3(Vr?QXt@vBOM1|
zLKQE~jA+<8{4G9l=?cQ`;W~nVf)|VhDlt5-8Fs;_umlgJGlkby$WSiT28dQG8{C+z
z2RiWP`<nl}YMNps_8rVRJ7?_!etR;jKoSL&mJKT#P)`hH1n|e<nEwP!DSzFRi)N&<
zrf!aoN{F(OzO7(4Es~?iQGf2y=`X?pks=D`vSUBm1ba)6Wsmnzy5|41$$M;K?Nh|4
zPIs0QLAkFd(momw^Dy$wVBLA|5gf!Tf!A*5^^!_7pLgB6?62KQpVqFQ{z-pWt-(Fb
zzm4h{`_mYKw#4GZ_L9)X1{C^^((ck@iP3InOX5%~$F#VMR<DH$sl!%n$7#gZH8^l)
zTLHiLsHTDNMjA0{BITBQHV6JF{ryREQQlSm^t3Kd@Kc3wQWy2&%|<AqZ#-$*pd6GZ
zlCGByv)tcTcU-p)CfR2xBX@ufVfDL19f|_>t9eiWn9FbkW_Q55fl2;M0Dj2W(9LP%
zMDW>mFkzj{oN7K(OR)Sl1_e%cC!4#&HRPwenezfPlr)odbDABY2mxWxj>vs^hG^BX
z57md*x3psRe@-(sXtBee$)UtA{O~y(ql}F)QT2||<ru!geH#o}1|{29JtB-FmO~9&
zybgY-UdZY!OPZfw7EF<wI+st&u4MCo=SN~FP&UXn0>PAkf#*z(hSkeR7e<}kA4j_B
z!=se3C2|x{Lhe;Ix_`d~o`MQ>Z$$}iZ$CP$5U9NkrEnG~eQa*A_`eT=jo{C5|Ey=l
ztF~X8T0A$TP2`@4>`(*R#QF~<DgTc?qWh9w*8gDv{NL{>dbG<VdE|~O*z|XU964E7
z^}LLv535SGZId(g@r*?52{@<HLufO%HN(@m=<}QehO;7cX=$q10=Zq|t9axfI1fH9
ztBb|79I9ogY<9r?jO<aGji-Lf?BSgSQc)C0DNR4r108AXosF`7ob?Q&j+(zmO&>*V
zWMmOR*DNH<nZW;0|Byv{6&<&QK|Fs-epTD7QEn7(!oiQ0qS8nI)YP}&<f5TDd+u}>
z)dz9WKU8*pkhaq%GnZ8+E&&<SYTd$Bli^z2#+mbz+oqFO(P+>sPY~!&$NdlL-ZLo5
z?0Xj_Ns!PWDj=aj6pRFs3=N2ajsha1fJl~PlLSGa$w6==DUvfd22zuA2FXc*Cg)5y
znI<&p>=%Av2Irjrxpk{<)%`M6UGu?WzdNk;tS1z(Cq4r{zdhkEQ$LW1<G>w}JRp3^
z)OrvMn3`CJm*42nP&eG6zX_Q-#+4tK?>&!+n;<(oD3SyI;0Kkqxy=x9N80yCfA`x<
zP_fnwxiQB<mys&;mu+keYT&`Zv$(*Dsr1Il)yG$J?j$%aAa%6>q+^o%Cr>W9jOD`L
z4Uxz^^kpk6?hI7KzDZnKKkh^YCXZ$iaop_o3}VEc!IUnY@|kA+b-QJ48D}zb<eS?J
z4#cxuIhP0^nQD^|t;m`+Irh$)e0!23w>$$c49}xCz-J+xUM)&ffZts4__15S^rtPY
z(T5)~DFBn$>&O%DMb_8rV6%B=aj9QoaGqqHXKWoJ_UjtaFeJ|1Gmv?MtyelnUNVO>
z3&^z67}ot=lc+>ckW9;2V52Gxo@L<*KP9pABar`=i?_}?FR!PSR=aEKlz%+%_%JXu
ze^~#Ma3$G;t3y_K>`CB0e~}6pCT1Ia|D%6#m$JY?_Nyrs{U94jq_?_QYr$6H)B{G-
zr>is<81K^Aa_xBszU`7)YY}QzRb&@#vO7<9jzkAom!H+4=>E;E{M-5k4`UEg#s{VC
z^si4HW>$e$PWrbVONXh4C`qb=u{;3zGFj%!5q4OD)>l1X&Izc;n$ack&u{#AW)Zl8
z*%jIaL8PhNH32i7^WsbgY%7E>S)UGg!sqBsw`RyrDWR^mn*;NyW+=rFLKw3G^P=e3
z92PKhU*gN7uK)4(t3lu^<yHY`g}n&gKh^*G<Uk@#TOUXs;j@nGW)T3~i;ipP35>+(
zon`5Y8>>cRls&SKP70l0`4g{CtLv`mqP<bbF!@GO_S(1(;!V+tTtih<A7R=^<@(N@
z_NT61gp~OyYYS<tmZG#U%d}pd?_k<cbHLY>I32jH0n(PikbP}J=pxt3tM6|bjXG~m
z$L*#K)FdIJp-d#3yX%9vYEmm~gs$%9*3QGoKJK&?qxJu0B6Z$J7Q~g~G$nhczCI=N
z$q)GEiSe$)sIjBgG3aIvI@WGsw6#RWLEdGt)^PKz?`Nk#vNSvXqA0v><;oX`51`ZU
z1PaaGvKwEo(-C{BXVG{<-}V&vwk_e{o?Zo^9+qu^1OSev0J7mE0=oh~4@z4@%_18k
z$P`}-;{Bvgw<B#3AjdH9Cmgg9QvlamxWw*#lJeG|LUQmyf*0NVUYpV<Uc=an9sYMZ
z#_aBF+O097sgYKlT)b$z@wB^cG7}7@ke}INsMdmoNG%*>OiqEBjYKom%zO;*mRFC#
zO1t2JR1oj(nml~)hH#v09FmHE52SCH?=;HoIqv8R34_)}6u9M85|{lRJiRp(;kTTU
zC0aJ8G=v<lTb<eTce8XH>T&)+RzH9Jm_KkP58193FvAe$I)ggC8<0?)L36;!0cvjQ
zt4Yn+(dL&9l+H6S8KO4Qk)0ZfQ9C7Tl1#y;GIo!LwuKB8{S;fip~Z7G>&0GZ!iVLx
z;+gj*1%qL7L2Z49Vmrx+cJU*z-8Q_pVUAN>t7ov5n-G9gaGjqR?B^-vS$S}8$G!2s
zq5Dq+5ZmSb<e)=g55;zFr9~Z#R<zS$ySH>|9opN5Ifw>*F?F4asIcONS~l}c?%F=l
zCF?{!mo0%TO*EgR?#8-D&<dsiZu@~+Q=|TaT(QSzY|8%VeUrOa)4{bBG{6G6H4pu<
z=1#HIdp{Db^Hi!o5B{|6w1+WxT(Eb~0L!i`P#y9bsi?(|<rGjQ%Ll0okM21cf$_+7
zh&Ty_XiNU=1Rb&TyznE&R?#$9`UnpESH(~WhVWs13ynj<fDdm$v@ia_oaEm-fWu?B
ztxfrNFjH=H9}>sc9Iy)Lfx&!&rX~mvJrwl`>pqp|c+b!2OKj=xOk%f{VAvkT(k5F1
zxI7LD6{A-*eT4A=q9$&oE?*1CbMTRk_my@Dci8iow^&u!TzaW<gt6JL>DI@SRJ7tt
z{qhv-Q^$u?UM#AJd>ci@Hp)$iw5A1mIb{~((og)<lyQk*Sl}*sOysEbj#HBK6d7ZC
z_w+?3_ZvdE=JGs2AnIF){I{sDZ?HBSi28K?E$WM8x9g+Ai>4V4{FWfTQf^tz{4#zZ
zzLb1=yWP#zqx5If?^Z5c@Pis*O$an7L2Qld7J;0Tk^$?*!U?*^Ylt0i5W$Vs@$=97
z%^`8?pBB^BTq|$>n)yIULh9@82Icfx2l3Dzhkga>AzO-i_x4*w#M~VYq^%bCuggUA
z<v8Qa*9$a>WPwCL>lZo3cvs|GbK4u3lkxZ%!zhwX;<n8C7H98M8SKPWkWOQaZFpHZ
zQaW8tp}Wzgui*ch<40gGUe1K$MWG0<#Jpg^Ol=^4vUUMJ)$GN24YhY1Z<F-tGCICG
zYI5IflLh%R7=l5KuglImJ=Yh@qb-eto{tNpxhTAtjrTNw7%&5yy>=)D%=~gz-sZmc
z#)btuhEnjPgiT^+k2GGByT7jh-=_s1_ZQ~uz&*6`6|;Y+Ge)NG0~-Z_9;vGxzKGnC
zyrhlZ5P*nm`luHUx&Yp+uEP|l2yj2AMnyttMF%sNJ%+&mK`jGh4%vVR8oW-ckZaf@
zs3`{AS1F^9)Ee5(E8l&#p^)d-Lnpe{uDY6vhX5jxv}(#R!~=RwDigRzN0PwZKnVHG
z?e9e?80d*W-HAwbPP`it)y=E(-->|Ru*B6eaSY{(X>rzX3l{T=lO7}zqoWZgNn>!C
zq>c#UFYSI$T2*kyrK9G)DOT&9rlqHZO>U82@>%iIHqCaK<s!zr_?IfKZ+2l=Rk%L-
z;Yd#ZVO+~j*8GH!K0^Z)h5wjoP7B-JWuP~d%Pct58?yNeOPJHa2B*8iS%BVf2K`xi
zd4=hrt{sAYO-(lojski^dl8^FL_&H)l)|CjaDcGh5XZcRG3qO%vrG`zFSkta7$nhr
zQ-xG2oIA1F>|g^Dze4$4=I&2KO74LY^da=-W>=B3`eW;TQKfV*zW;G%f}l5w7us2>
zI4DHWJ94JWekPWm21&>IzZDGmYdt3+m&pZqGl|~!i+{fo0QyKzW=uwW0Qp5Ij6V4A
z_qUs#><u$0>)&jR4Gj=LwWag>z*C+2r9}&sRge_-#+kfjnY6ctN;Wgc9^BEZ0q4<`
z%(zL#Nr{A-VM4X2-coRQrGKCk*mLUfg(4N=D<*)kIiF;budKbXn<+R;3pZe%zM;%+
z8(m?Ea$)ddp6EXp`|^s+B?(3Yj&rNB98tR>f3(U|UJ|m8VCvmI5o}7r>F2n2N8_R0
zev-ZwHU#cw(A{ISsh6p#3pXaeeb&3QOL68|xHcOYj}E}?((br=Yf7S4zvX<k-<Z7p
zRbSiqd-*pnenYd19c(h;WasV*si68hC8}EpShSTDW0Z>ILaxo+pW%Vn=5*nmg#z6%
zp6Is?5F-PMRtmy4rN%>H(%FDb?2qlV8tqmde6<Gh2ot!w)(1=smwdy(RwOTOWH|<~
z92!z^9qwU&ALUoaxA0&*J-}YJ;y0i*e#lT(8YnsMPGOB=cP8e0`RQVKf??(3O1Lr+
zN{qO6{GC9vqFzvt$og~^orpNaq)`BDd!TeW5g`>27UB-~+MTTVoWOe)43g4<h6sDR
zu-%sXbS_hFrm3If;w4^i;Tmg(<9-xnG=F=0;`3SQQx_>G4LX?RRoh#*=<PgH0wbNL
zD@Q0ihRgJVDM}oBfg>1C8{7PVm_<vkOR!zv!onG8y9FSHG!aqP-Y`fHlG)yw$mDM6
z-JZ%-F}68nsJoG++HIk$JNAlA^h--9)HzL6lEPws7vEQ;c+}L$8!xrEz&EI8+aX60
z3Ih3jmdl#~X-QgJ*(Qh=%3bXP+`A^v6RRP0Yi#KUfLW;t<?8-u((K<(1Qu*LWY#~A
zI|4y>#EYXkGgXf9&hf%xQwa17Ec+A)cVKmKrdtae+*8@&{_j+b_i71n5yRN2Gr2F*
zKqiI?pTQPMoHP=Lkqrtgdztp$-YQK9-zry{*Dp1x7c}G=0Gw0nmEplA%ddNRQR8fO
zeQWwkQU)<Q3=5}SKbZ2=O5JReV}Mop=MsHg<Ls6GPbP*j&O&*w^xm5?>CC$DvI@>=
zgP`@DM`|~y^D!fY51I`gbTS6dB-I}WL7^DTiy#zZX7w5riXoQa)|WUeYdiRCSXKzs
zUIm4_psSkKc{iE*-V&e-{fk?xkgfrjSwol1#z`LQw!b|Ev_IFPEsM?i41=VLi|mSl
z!l4;>t!pe6zjOOsch)j#?CiO_5-_I@uVTlwR#E=h`Vb%<ljr%X|N7zm0I)(0R%nA-
zIQ$_le|&9)AOkZYkMZ4UW>B_`oe4N&w|tUUjAHrRR$M%P?i6uC2x+CiwHmd(m8Tp(
zucvSkb2^8lu2bqJ^)d#|>X>dNm9uZxmouSQF}<7w+ian6Zcw7zxEl?Nc2SMjnNm+s
zToxV9xdHrE&{O^ztkL*|Gwekzt*$?Ra_If6+9dA?4iGam7@FQB>NjVD(;ezG`9azP
zR(=-EZ(1AKJXn|c`g%f(pU~cW`4VyGyS2$RmI#;BlupK=*`7KLICwE7v~*F?Ub-J>
zu^Nx862x80tr9$HN%R$dwt&}6U{OgZZ>HNDw&2kH=qkOHviQ_To0-_%9sS#ob|4!d
zjj!d|*Lun~@>bmrHr>)+1R!-}P9#E8uKBydFrYDMCWT+p{q~XG6ho=c_IK<Rx*Fv4
z{2;NbA6)0@aXTM91?c}MjKzdMCj~NC{S#xW5``Y*9PXqBCr7r0Mx_!=uD>K_Pa2jM
z8YFf;vpFq?mO(B|K1zdqsS>mKEsaYdOVN0pW}I-MY%hOhKIU!m=fRUM>7A?BIy8Iw
zq;O{}a07W8ms>47E?+DZ{dyq#Rs7~Q3!5J?%WZ-AY(~6!CRM3;R>4m~YEO9C%9bWv
z8b<cU8$XcK$%JygkSSI$^L1J|H?k%Qrzyl}GF*(?=vrE$*4+hJ@ibSEx%*r56`2{~
zU96^o@k`)80O%%pYn-zBRZ%(x$RH49)|6tUi*p+Tj_iY`W0(jK{IhMeGX7n<2GD>s
z0sAy<*!9XgnY~V#>)>iu;|HAs$7#)fZ_B@wvXxIECkEu0eCzw;>l<+j6bKKf@;y$T
z13NVFOw3hm;h%IsEIP7X!{yFc8RwDddzCXNaR^)E&*8ye8AbfXy{J`@W0mWJu?C}W
zfM8ffp1+BLUk#>$S9FkkC#6|=IRxt-R3;GeWdzPHe0qbB)TzLR-1*U7z8sscVP5>U
zMLq;^0cJFjPV44zt3A%^o5_M)qlvk;JwZfN!j^&b-ic*@!|N~WDDJYupK5Q&8VAHl
zZ5Ua;y*`_+8y=g<%PWHXmV*$n2?pE!t>GnthOW#yfuWuD=D=f;R;n5xXag5!Ghp{R
zw2!>k=lt%^3q&c5&G5Y+aC{4-i4tGP?R47tb`3x!Cttk%C&8pXWMHcAVspGpAoGZX
zOiv;eUM;TCg7i*xbXuC1y4Xsr42g+57lkZvm%hi>A2~)k+I+6u9Z_p0N+EKakJ!n(
zDBZ->hV>oX$pGJ@a1H5j@|1q>L{MPSLO>}c!UBO!RLp#@#o_JrDo`Xm&%XwEl4QCW
z_Nr)Jx}lwT8E8n6tfCdD=_l)(m54)Yc<KHAXTTaB<2f-{2U)|%<#Ecpk1@F;y7XU<
zTDkOKYU|@N-;>Ir(WqA^kuRS;?6Ghe&lHk^m9utdEpV2CcrDQmXpi}5nSO*guv6fN
zeMvu!Bl)QSo9}1TKSHsWmS1lA$_icVJ$3pYo@{x@Qc0dL`tz5utPs>Q3DM2P@Uk5O
z7l&&kH$5ups?$07dT(}=sDU&%rkW#zc-)^(@)_}~7E0u1%XAHo2bnI$nmSE2Tsvl;
z(5n^AsO!zP#bCHsjEfD3c(N|~TIQf;-><4ZDr4>rJ7Oy}M`e90^9%Y3VKdz#0K4Ii
ze8o{;#m3=BA*;4Wi@LPS%#j<w^lGB@@eoRIw&A<H@lag|BMHq44ROxr?R2Y*d5M#+
zCLALWY&P$84QTP2S|e+!a4xzM;RT<*XI&<oEwus#_yGspBriu&D;=`tYRna?uTl$U
zDyj7>@28r>!y&StrFt}!MfDNNF&#oC^HuDw+JGVFM`T8kv2ew=D`&51Jal;6rGXBP
zdeOG?$C`Ze2v9_9PMy^{{#qPXJ}mM8;B(uQ+=spVYkcI<sjl2(f=bSVJ{Z`1tfo)8
zxIEr0?QF;Cx{?V=c;`KPBRT){KMv<AZSpf3Zn8h$quf=HCRQ#sdH6AoQDkX;%KQqq
zpS}j1)ss%n3LQJH6FbArTfXi~mnDPBuVBDlDT$$8*ibB$&O<ItM|M3(ilko?7@!x@
z8_2Ds{;4raA2bZ!^(j3b6*s!dak&IVoR_e8T9rSo43-glj&+*>PE>5~5GUeW(A}#(
zK(U>v=0)gbj;kTmIT;&hXqDuJ)C;{NZ4C^SUGv#?_34C21yG!NsN~XQ{I|wt@Nj5D
z3SCtXkdJeq$yd5uttue`=1zP-xQHwQBEzOob)BLp7v?jrrodTL08zHi_R9`sRz3n#
z#++T}elf0zl!uOSjZOItaEzyY&iIDf4L<V)Fbad-bzXNC4nRV}SaU>-(GTkQ5{hwK
zC(>nJQaY1(Yj`u8Z=KI*^@)>v7LB$<iGPK?uV;2y37^p$dE7mfgaAI0Y5FefvxM)?
zwYP5SeMTUG5vWPJjF1zBokjs3YyaX_(%i|`0;PVjqHeWhZIm8tVr<!P$g@e#MRvLa
zE@O_kfIQbpSR3u&t{oky(@c(iYho3H!m6vGUA00e-ocXi^Op6O2GJ8zy;4m0ULpF0
zNq5<tA#kR@A#n$L6*eGu=_9BT^JQoCjURabo&y*=p%C<JG*<2NX?`-JWbw+u3j9Fz
zNyf9jX5t^@k@u?~s#}M-Z5<YnG_QXCLz5{BJ8XkKQ`|fIOMR{aJ;Gb?-J|+kTa!Yx
zW7$P!WJQSk1DH9(W9`ki#77xMz353zh*ec$iu|g=SxO9OH+t!9&2rFSyUAcj>!S!6
zkqmG7CM|b>L%P>)%Z91W?hpKcRz*JKkd8EEYWCNtIFRDT+}mBZ4x!X+?XIErhkYNM
z*V2<J6rxypxOc)|p8xK3fr)OW+b;@ve775ntR`#fUFWkmA8K;fGSp<2=S4Eu<ibFl
z8(z*gS7cSLI17&3E9wjJ^^hTpy6Clh@=JexMD;=M-4qW>TM*h9Fy{+Onb&-v*BN=S
zfm$s3f_mHusCZAIr?(r2^s0sd7couNIODco;SgrJe{DYpow*yw8_Pau`}Nh>XWRe>
zc{<Z+amsG+3B?FjXp}ftJ~oC}jJ%h@+?q6oD@9M<B(N9<j28RYCb>nO>@3opil}X*
zQc&o#h+eKCdL()e`(pmPMzhAWtJw}G@5~39PiT`ofa3e%wXs-pSnS$!H;w1nFViam
zzIu(0C0gjXXHsQ)Gnj8>G0EBxh~~3%7@PSi&|){>FSoKcuT7_?a5xb3bIQ@bRV(az
zN?fQ2e&<AK@2xREVFE7V?#;1rUyV*%0mHR$NGE8uhq9Q?LfF?1uhh6rNaV9V2f`UN
z?=GwzBnb^Lyl`qN|4vN7_Anm|auF7nO=4auY&L!Frg@xY`XebD2zEmOMWHYa%usv=
z%s*cpbhQPm&4jpJx>Cp541&vV4t?BYIcyyVc^?S=Q)@$W{;+jy;rx2<7yJH)=jlmW
zA0qNqm5wgVupZ(9B1@8y1@6^zy+WfWMoIZ)={yEc81_z>_`jUI7%$!tLg8j3ILh4+
zP}xJk2t1)9wKwm_W;3pS&@J23tF2?neD53Czok{dhboKm>eJY3c*nsN=Stsh1cP*Z
zLg&**{ESb^T$k|a(D4)RCv_AshRG5kZ%by?+3Lam1f4075*yMv-9&k_cC;SELFcES
z?4AAV7tg_KW2DUCen^+tIqIyL_d~NnW680R`fU4?KNsWrqSCfcBr#Fl<ZZ!U4S?dc
zv%8;YozeMLTk5bfzFJA~6XVhhO=edC#lz$oZv+1hE=T1xYk=~^dy}n%Ai@GN-am3~
z8%$c~bXbM8$e3huW9G_3eofefy2U7{p;7uZNU^g22o3e7@RzmKxbx}`dqt|((*BhA
zGP_JQ4S&MLLtA$}PGIY9T?cI4S^sV8t}OvKHn(uWK(_97;xANOYY8}(+Ea!x_|%k~
z0`ZkBcpI>FkArO8b+w&@mldsK2>EzzMo}}tu}`GYL^d7XMbA=S^Ha{Akb}eC4wF)q
zS4Q|U`<5eOUd)fbHJccmFmZ9C0=Dj&N4D<BLtA$zMHQ8dAx5t@L?hD0_Q_*@Q6#oN
zI?|U}XRMa?W__!$$D8S=5KH#t1$w`mA71pzj1pU5L!InrVy7T=dC`t#Afg;lH-j!g
z?3SiK&RjiIo~Q-}n}_jfKNSJNOR9vT-erjtPz^9yevQ0G<zdY8<3Yx8s0ftZ5j0Sx
zO`g7Wd=qDiK)ZBVrW*jG)}*ax?A94gw?6docZ`t;Ofq&E)eE^JE5ql~k1XN!gPZv?
zG;JAyCA`H<%A)hXE#bAIk-!qZb@`oJA^rMaEa9W;^B_z3T;!o8yoHPN=W7?@sLZ0R
zqAl=g7Gb0g%vf7%Vr~jjH+)(nmu&6*+k(m+E4JF<(#x9MTOsulvn}YhVIY+;5s>N0
z(C`&QU6iTd)M++YOqqZ2vjB)_0))Mu&9BtJS7Le{LOa2$Zi{nfG&N1r1)l6*!ukg;
z;lUJMYagN}ezd$1O)JuyfXnSvHUjTd%!6Kk0(hU&r+(h&zg{p@cNJ@_aG|3GLn|uO
zX_Wm>txUi7A;_E~BHbPH3t;z+6RhO#4_VV9z*?4OCgmj2QQdSNL1%{Dt%+LN`<FmU
zk$stDl-U#rq_pWgoNOBIJgkrN2oS@xP)cmkPG%ucxLUP%2~3ZA3I6S!G@lB`4lay|
zk`yVcU5gd`arbvM-4Y3B=Gg(lbd6OSVV3v$h1iUa=hVgS0*0i_#9y1+!u!Nrmg`||
zh5%0{HLrS9?;n9m%ZZ5AfCulfm?+uwg_H${$X$ER%yAO6OOKhBtLM<?@wJQWE3ltf
zuQDFaM@V3IsQ<9kXTQ&}(_0^B4cNdvQ=^mx9S|GOC)(_^r4{6)@YxwfE!qwlz^wfi
zw{k7ml&^ei{BrzvsWzxC3h;Y=+*cklgXvQM1*UvLq{XKYnd9%09Aq6N=XMLZXjMsh
z)3+xuyk})cVyl`hP)<gjnBCa}b7(xTudwt0VVehx=VQjs_~$+8!Ki~j(d4=42a0Id
zu0LPD;KTvtP`b*&Q$DhC0J6RMEt-Pek$zq;r`#@&btW$29<^_LM(+hj4ep!YJ9rvC
zD6E?;qy?;ajQLt`g_(NNxw&8DwKvezYL+hY4{dn#I94Nsc};+|bJ|l@p51`nc%v?n
zTSkyl<en(TNMT)&31tpd>ezfq)*>m!;s9)~aR<;~XhZBo2#)M9<lPNwgZTS5Vxw&e
z9gPt>1CuNS6H2+k+d_=vhrlFRRO4~x7lb}A${np1<<phpjiR0f)C-Qe6$9*H?2PhN
z>}V*@hX<w7pdtqXnIm)hnqXi~k2*A`7lO>`pB<XhTVQ)EnEqo<-w&D7KMBnLj5Ic!
z&kn!u>WzQu*8Y5N!@1W_PLW1voMY#sPi^NJn_f<(tk0^%xpMgH!Gzou_l~+Qi%p`x
zDr69u+`ayZ>U()PR`8FG9Spvi3Vw4zyu8B9dqvAuM+%vHUHU36wS?JBZqN!UxhVa7
z+sXC$RiyE9J!9OmHw4%OH9542&wPZGt;w%K`j!)ifqF(PGFW`2442*9^dMMgx5I|V
zrnRw3?HG^l@W8&|ID*V);YM`F>!Qk=P_Y<AL?^oi%0&mC&L#-32_y#9Vhv1ol%bN;
z{dX_D`nuo{cRHDg)<h20Y8@Mz^4T9;)5&L@utEiRO(YV_P*Z8CJ(DtTka;ig2ORze
zKXv!_$NaG3umibkf?Wk1D;UKuw6RIU=i=mdKAA*CP;tRp9VsF42rK|b%5G0CLOw|k
z(+Q8wEs}!7BO{pCt^R)zk7Qb+QU{b<?)l^KCMpPThn?!c!UDKm9fQ~CNDwW;FKZ)T
zzkA^!)y;aklODzZk(mmXr@IVua8D1j0UC*LMir@-PtlV7K*QBk?NgfdB6AWvyq7^t
z=~!;`syv7h`Alnc>Yr{mGKOU<5Jtp9sWEsQsnI|l)!7SR;potQeCUbSc9o3om$)lr
zGdy>RHZsP9a`$WXyiBXll?-|Stf^&7Ak(a#iGAB|hT0j~V`O5u_+Q0D4*xpv!1>y&
zqcX|udd5TVvC<2A-(KJSUR>YSK4HU%4pnE>bn8RpDcI~-Ux5-z!`Maon)KJ2HZHA%
zK0{x<YW@iEc`z-j;{=ik7mW%&N%yRRtlmB-eU|i=iVm4%(e@nyU*X6sMqe1}jZ2Ra
zlcB3K!~6ihc-Bq|j{$>A<VEJOX4;$gfyl2=j<X{T^mbFwOtR0W`8raI+Xm<oK8C0u
zilAJvl)6|Z&_;uaWMDTA8U;gB2lY<aALBqvc^OD%I&=G14%hXBq33bxF95>yP95Rj
zAw<_Rj`uHVq0nS<H^>g$c!RPzNr3<XM-#!^g@wjx)JHcYMoAiA!PaVzwI3iua&Pgi
ze=uCmQ)pJ}2pYuH4`V>6yUk^cUJ!hc%n{Z<7(D{8KJFG=0B^X-&A2s`&O}SqFu|_x
zt;qNsi=<HbY_H0Fa|4Nc1lDX-n{05HiP&V%T!6LuqluiX8!xhT>1bnvU?MN*>vi<f
z)E>v`<seQ@L@dz4SA#1q=42{LQKRNH;L)gdb19J*5z#DizNX2d_ikY7l@S~12X%H8
zt%mTOUPd%f)E39+Lg@C%<j|ZpiBu?_KK(@=o}AVE;RX=!FP=G60yMep^|*x&Sj~(u
zaA!Yn`~X#zMeMd92eqs@4z<|V=XU^ta0Nc=rpEmCfOY8{PCZRy%>K6=IOMQ?aqAy}
zAvK7i1RfX?2dZDL%yh_5d(15cf@|gOyfR`h=>9uTAN60JJ_@sXem;9Zja1rxyd^+6
zT4I}VG4+1?h{kwY5L~du26zw(7$(1QY^^ePnfOr^lAwZ0Z??2JwBQGhr~*R&vfx+N
zyepuc<+$3B;A`|*wYDbH5>{UvaKzMindc|+tn?4QA6p)qxX(#ZL%`&5h-b;^>DQ^V
z=VU>3zGRtjR#shG&4qA-!2F*4xV>v%i7g-peqrFie>EmQRTPZNMnKP-H~?JOo)-Ej
zKn_Yx_jqG}Uo2;3t_x#-J(vYSZOJ<AuW(uQ_Gp(I<B-~;&y<^QfJGN_pa}!rQM#=d
z(3Vo~2si$2P}j5#NcH8<Uj4XZ7|D|cFe7j$&}g##)7FkU+S;9DH^z={Z5nA%a?<Q^
zUjxf6o_4;tl+G&Qy(FhfC4Wa>GqpRlyAHchO>Z&oy_v?uRX6uM$A|#IFH&TyTw9!&
zYPjw?#!e(ZsyJ%v{Ir3^NN-xZxyv^5R*Ip*a>#Q%rS<88@|PBSA{2tS07IFJf^#1|
zmuRyYY=4|a;w?hlN!{5VR6z8Fj~YF|*Pq`DWr7U}%5$2C<cbH^g(Ci~j0lHk*^X>x
z7LPBCmEC~d7-en2f&XCJ;FBQA!H%zL5JQd|`p=A!pQ8kAkEn++;Dla<W?&X3p@UG(
z_)$&yGQAnW2wb`R(5UjsyQ(GN5yOGM@&y<^<fRpG1pD)=an(GyXq-j|zDWj&nTf>V
zYyW`ad+qTu9BO=?(;r>GUQeBmFSQ2i;q@CoPjt$Z5B(L4%&_(>V#IziM3&a7qa<P-
zjVc{<>T>KtY$Pv&?{j$Bzv|ae1MEIrcY&fgV9_n~YglwmymKC(l8PXOWBUESw<CaV
zrvhawz&&ka;j&?KtzUx>m96oriRuMstHbDqo`_{mmtFuA3~-inRR9D7#2p3$IA6kP
zmK+8H^k6=Q%IP@gX4Qp1t1~b}2jN!$8)-4Mcr&=wBhUerS8g{r(TS-C^#5B0nSmA*
zpiC)!GY65WG)o;`T9}3U-$?@UWiv-YXFOst{RSieQor~$6%5f`x?Iy*e{T>_d&`n4
zQXCI#^ClvBK5ZWEL^Ac5<F6BR$&w$fo-t9&C-C!c$%J}eq1F2e*X@B|qBz~?;v_oI
zgl+D-Zt|=8NRMph9MYscL@O)nWckD8ShJR49nrp;Nxx}t=&)W~B929*z0XG!F*s{5
znt-)BlPT1a$}d5vg01AKLNw_fK$GUr!qf!ip`0}!lJvPMOorst@2(FWiVeJ`S30IQ
z9xnK!eR<-9iN1MV8u!RDL{4O3M|4M|g>J>{x8N9X)46<3^W(nD!Fh=$=vcyJCWk8x
zwm83BXk|}iV>UFu+~p|~9Ea`6asdN!H&Bu%N2`0>hNRaYuNMs3{0{$M>I!7n_62Rr
zzkS~M0LbRlvAzGWPJopGwSp})fsMyJAUrFVp#+L$nHtc(05}Q)B`HQ$?tk>)C*miP
zqZlFD_ptZlT_yLk?J^2FiW`UkaRVBuCSf3MKtT&$;?k?D#Tx6Fj+PNxxr?puC)7TQ
z*C;yO+rilG&%yqZrd=&Obs+#)i|=JB-M@s;ULoRmBG6ZoPq%}K&!b2V!URd*qA8M+
z@~vy%oeqBWNQu1xK{gS|u&W6UFHw#{u26x!+c&>EOOBeh<T~xO=gih}Fig=%EH*W+
z@7!ePqGN-{u(}w#%m~^#iR(Fl9NFJOjt<PKgFB3;Xwee=&qI_H=K7#leF|ue_}zBv
z-3I$WASG?mrGH*5F=(;GKHWP`cdZPD`sQFn;pYXgTx?IZO0omB=0{6hw<u{aG;hx@
zlXS=%tFDL3|5(I}28kFn0)CPlqKG0LjdW>Mynfq~#1*4B@(bBDz^eJ6GEiB@rr>`d
zyDD>vWulCI?TzIO0<?oQxMx}RZZn#1c%$*%+j_kP;(gqJ>-quThg^!m!sfZxliZ?H
z(@iSkTKf=<hH2%!%&c8m>#&1m7{wb|wVk=UHz_zA^!l9FTKfX8f|o#Lb3Pe*xep7Y
z$7rX=qF@%o_dG-dL@!JD98?AUL;VQSWS|bt^>+60QR(PDG_I*Rm;}&G4eiZlY+)Xq
z1H%GS{xUJW@x3c8ezoBk-#jUD{MLs~;@7gL=p;5)1zQ5>xG0du*ywQSJRFlWa+`KO
zH6^?z51(emj_-ON;l%>C$+71bO-gY(j4fzAiY@r)4Z0Opo^Ykk2-IqhCG1^>g^qOx
zwwddPZ4Nc8G~LkEWoTka-pDuzjQW4@&n|~Jc%ic<@2K#n4>=LCO%|SoC%>38fmy&Y
z<1w|i%67OP5Z)$c{(Dbp^<1HxgAZfCWhgmTd^W+W3HS!3yy$X<>TW9+tk%$i(7D6D
z58z^_Ko9)DYFZ&4=)2LSYq3k$28m7|uSpK?sSdD}l-?;|{(ft5QVzx+-TM3B*3!{n
z(AKG6u&b0Gm2}%8w&vIxh9ij3$Pc>iWq8AO*=5B^=|a&j<t>EF{6L8#(j}no{5#^{
z{}a~brf2cz|2^xn1h6jC_5o3J@OF;rkMoa*EH2H&p)AD#Wx7ON0#9R?)uV$7t(Te>
z<m$*xc<{0e!e`roJje*xcwPdL^{S5blEbPHtKzcpeT{#ZU!?_rVme0z(H(R7IJWRN
zYzE+5O%mWcJ|k|tk~T&r`~O2KB_wI{v#)f$6*tBM>Ff%3ftkd0;Eh(${8(LE*f>1M
zp_aR;d}ooBq%~spfsUpX&%B=y5P!QC0sqC4Xj6>wsa8_>R^=+tyOlzoW9C36e8@}*
z*L8ISte*Q#5CHB-#F(W(22Bw;fMn~;e&19)7F3fZkgb}g;j>>D6U;OapT{30fE~Y1
zBitZ$z=4Ut5FNR{>;&!2L2Du?kD*d;sh{Ovc??2_c?{vPAdjKtzj+KRa+!Iqn0oaL
zul*Lx`w907$5tO$IqqAHKl74Q5_=Zbt4*rQ+CuqNm!W0*TXtQA5=*RlXV>jah(mR!
zhV5CR>RPsfX^aQu6+{~T`y1HK1{0LWP!$<cbRyLfb&HN%u!WG7kH~x0V>IK-E{zY+
zSGoRXGOQT4Fd&h}U~u}?<gG7g)I4~Aa;&g91&CS}q-;W8;;D;)?78Y<-Q`Zz7MCC&
zpb{;U#ZpF_p<W30`3THSrO2YP#v(tQ9dz_z6#ency6VIbVPl}^vbp^fXq7bjUe>cW
zLw>A761VdpL#L?`5JlaivGplL`3g0YP0^#hI~Ah)kcU(e=$q>Ohm4~wnt*g*esLqo
zt_KhQ^G61~S)llIgbha1BtYb*B+Byl<S<Y_z23ZXjq-e{D-Q6-od5#5p^Hi$EE$;q
z3fIIhyNxQYkA{JQesg2*otlbtUhW>GNxBB{Mp2e}K$RkmF7OSmli_C!6^=#Az)S{!
zUUsK@2r{|61iG`=o2c$3YoHtty_!*&RJvecg}wV=Zv%I{Y-MJHc0-KGd*xqtgD$ii
zbazdX2%w~W(q;RSl(zcyp0jG(=5AW#7?r>%Tp&4IvNAjvP(xiWe+zOZUaPwPP?3&o
z<jJ}mjGtJB9H0LEny8%i<AA0+G<gk)!O_xASfrr>ItaBY^i@R#vqr7EanjxfbYj@7
z&!3Ozp@|)@z8go12al%mR!~$8d;Y-@OY~9zGfU>)M_M3`H>#^A>}{uhV|3m<0m3L?
zP#6WSo}){vAhs3Fzx{NDiN3_d`!sMuILJpxXY<yT+G&bpc(Ix)GXoxtsZA1z<~a0N
ziW6p&6@75v_$6-`T5!IFObTM(s^8Ai4FT;yX-qKpXKp~NWi5CuHLn_U#clJoThDuT
z$ggJ=5_q(Tyq%8UpNl_P8}=llbhT-5JHL1zuLj&{S4G{{MEB;i&HJ)%oV`bLsveR8
z1)AnO?O^_+Q$?oC83-5IYaTUeNC(Y1pid<=02{&;n>rQ~|GB!WxrJ1U9+m6h<n}GR
z5b}<Bf>?bY<ljVVHWk_KY~1cM;9WRmmUN?U?foY3y50D?PM?JUa>npK?}rEe49fGk
zR=+d^{1rsQ#E&Tl=*h^50P40kwB#yEH1x^mm-$2GE*B`4cJ*b{LO}(vmgJN%Ww%99
zFv%cBOtVd6*SGRtu4+QrpCpn7H1@09HtZexRxV?#nXen}>nSKo_t^ENsyvGxiP_4*
zC||StA*A(%59Qj+y#59zNMp<LC9tSfx1g-tPtO0PML?TaDExFRl^~4PkrSrm%KCkp
zi|iwd<m4&bF3H`S#PqeL`{%EHm2mr!S%7a0lF6FKW}^gp)6g&>((}}}bmo(7Gn)AV
z34Zv6APLNQnd-Dy>Fq8}HA$4s`L0a99D|Xrs_?AbQ>KBstduv|R^3VIF6jcXkH*FY
zhz6ymBL$WgXDQ=BM4U)`sueWhf>X}OlWak0D?p1@_{?n&Kk>%)eiMix4DQA?hovhg
zKwXfI4p2s;r%QOg2f6J|tOEOD6XAjtC!N+Tm8w?3U?YN&#f1(w6fVn$^-rEFyY<tM
zGjS=#EXn=*u~}NnQAp+JWhSz}AHB*-sBW!9L{W)~=#fP+b=C2sz4)Jx5ikEi??8M?
z@u>}!1o^WQyJKXTzKpWh$|-Z4$u_o3<*hiddLg#5I+(5CidH1Yd`+jRIMtS5g9^g@
zq?19GSv@eJNFQlGsBABt>`>qL&3`d}81E4$g<D{W#TLgzY3RN9!L8~k%SRp{cpBfr
zMxx65bv~legl^Y`es+kO+%YrSSK7eEc)TUCJfO4Ae~3aed}vtbhguJ!Mc>8Pw%dpw
zL)Z$bkfD>T_K($7-g(Z$E4X0<XMH@hfj^^MA7Ce`m`58cwenWw8c@k?T(YSLY=fs5
ztsw3o{soI0!0inQ6gyK9Z6g5mXo9r6Z6cu&Ss!c|sS1W4C<0hn#4;fZ4Y{R&wgC)?
za0H**H?%H9tu$N$rC&(=?x*VkQH~ajXsdCaBv;LR^P3*VKa%cDLEfe6J7uFXmJZsT
zDsF_%LoPlv88NCNaWqTK=CJEh9lHAciP@71z?}KLP?)0cRi~2Vc!NfnJ`hKbJS`mh
z5-JC>-Kl>-q@MgKRV)yd{8>7uWHmnrk;-i*-z#oxT-!2Pm<&<j3x8&F&#}Xc<M;hZ
zmHkgrIDHuFlx@G+7d!OQxj$QYj@LdO5F}<x+A1p*c=Py_p;8Lk4pN>n@_$kU1N@O=
z0V4sT!-YnLIai4~xK7=hJU#2%s$t@I!h#K1E4CW)j(Lk5r)db1-d_RaB_J1N82inN
zZalqvnu`L3+dxW*kV)GUpiq+<L?7eK*XB4F5`>xz2AO_Y4!JGFHJ$j9?hgk_<4-4K
zV=1Z@^;?K9-QdlgYxgf})hy7aqrCKobY}hkBMNRI9@~3J!R@yJv>wSt4NEY-S1tHx
zjcQV0tvYi~_vww6EuBAiXBB0xKvM<D@Nc#Lp*x!u2QEdkN$=`zC~3_2hUMH#NMxbB
zxa7K7Lj61x5)qfYJ(1M#pxFU%(OG=iY#Jh_fY)O9M=uz{;lV(*Q8Lv{E7ZV1696j3
zo5Eqvy6Z_yq5EsHr#-#9q0;Gh$f+|wr5gYZLKajPf(s!}uEkJQEV^CBDb;(V50nSa
zp-CC!v9g_R<*KbgS&pE8ACA>!tQCw?ml-boJP)4fY|E+XXe&N`2jU9@caN96UT=;h
zFQzIyd&m1RU?&cM<sQFw=Q60B|33ByRPn};I?y_|!|vVYAd(O#YN6B}G}z=}Ubi`~
z5%TWGd{zzn!}fD#ds**+$%GJAH(XlMbsb43#*j8t0yKM*vc9?7nSD(7xqoG}NM98K
z87)tbGFo(gyl)=F@m^mb=3#fXBBJf{6XL;GxWW=z>sH1R2&~)!`=?lVGZEhV1W$#S
zq81ApR1Zm$zgQ<+JU6<=_qLP-Zo0D|MM8ST&(*>ozoMUx&cWvuh;Lo&%M84TCy$!w
z31c;#cYG3Z@qXCHbRc;Vc1d-AxB`44${z<5LDurR2w)&!mk)j3(-2lH7-m*20jLHR
zB($_I<taa%x_fVMV-0vbzZO1K>&*(Ue+itR7Y&Cnp8M3gTeqO@S2KgBIV^yuV>t@F
zVa&3hJPo-wu8B*H62oRKQ|H|99lE>Ub^TYDmK>!#8#e~5bP>S0DjcfCI^Wxa!c09J
zDv5e_`V|tk!H#<hoKe0EZ`X0xSxa_cbvBFR;AE`#kkUrKFqvq`NcOLvBl#asDd?aN
zLJt1lQjK}UTi&)w%r2aXMYK)pqfHG*GV)Z+EU6z{t}2t;qqs#YJ85FgNj@S`P}yyv
z(ta+wGFmMmlY5$_FJQMTJobog?9Hm3HWC+G-a9cNI?=Wuo>IgJWz3*3Eg)ltucS;A
z%9vsKSH{eOILMg!`<G-a4+fJIczKlT!_%eZZ&;NMt0E?{bA(B}La?dGu)_pY*M@F_
z$Oj+RJI22e8ZTK4L*}|ukmG*pmHXj**MUnIufkk^Yqn7;<xP<FaDRK0LZ&yW*`@5<
zNyWSJkXNe0cV}?P&ie^m*+Fpx$$A|Cdd2!Gp{HWF!*X#+--5eiw_A7bai3Mio<i*Q
zdZ)^gK<Ny}L8wGijMDKlM;dr^=0;&x7JiYLVWBgJrEhogIK+ft&S1pQH5Q)fI@8XC
zv?c*`xee<RyFb&9_{>6p&un$bXWpj%V4RXtuV0CP2+alXd423i2&<)+OScj4R-;{I
zy-q}9=TcDMxsxcDA7<Qvd3&=ax?9Rx*F=VHPbc1kfAGy-!Mv%EO*Cn*>dM(jm=wzA
zlXQt@jV<!?_(bARmlY;9LHa2Wmd=Ob%u!I`*tt<qdlpj1D))vkM(1^D!db-|0$K^R
zf~Q^5&~T?czZr7af@n1nku1@@+j750Er2<hcV46{>7`&EhlP#=D)fb7$j5&u49FzU
zc)WQ7Mnt|JlLIAEpsa`~P<6h1eZS$sdsFbmn#1{*?%wE{l92`V%uK*}TrT=~wJXFE
z1hyU|o62F=wSe?|(KcR7XXOud(|O~zm%zp)@F(61(Du=xc9<1-{LeD`RF^wyZta_Q
z62>-IEvw8R_-@IvM^$tdIHI6Svr8iDU3X?euc`+YCz=!t*<W1(9>0X4;u%NaYsTcK
zDzn|F0vR$4%ssL7KmW@U3GrS*CTF2y$eX{{OMp6K2(>DPl_rUU(C{-=@foDU9G7$>
zYa^1UbB`f5|L`nVsD`34T-ce|#jn$n&fHJ;O_E~7z+a<#vfn__J<|KL5d*^w??yYP
zb2yyHSc|6QzADa`+G;k7Ik()S+Yx=U_tEEk=b5NeWHv?lFV{*;Ebq9%aHO1QiM_rF
zbZegYvf*{HHj5bi4=$W0^$jv+SK?WDc6*9_%xeoJXB-u)IQ=@ISn*5d+4>W9Ll|B<
z$(ps%5>@trj0LoHipe$SuLZ!;DVyz5(O-w@O5~gZdjJR+ETen74cUi2udc^0@FqLs
z4B=N5IUJw2ALPXEnFDo&niC2GbLKqg4Gj#gy;_n#Xww|$(wF_1EUKq;T6jvc-RVht
z-8eLm4vt*BuPip^9eC{%jO>GY+5BV6`-k6;YxbS)u0fx*yqGyR$(Z->M^O0m&N!@z
z&%~$=uLQncZwU?(>x|O*LQaJRq60@o0ndkw$8j?~>!VpU`6kPsImi2rX>!Q%f_$bg
z?Z#h(vn^GxcS}DvE)jlI5eHLniu*`EBduj&w=X1_jbOkmMz;DYkx-SVjh)p<<|`4~
z!)FzU@Rn1GFLT#{5vnAsVEU}pqU*lz9)nkquy*G=3Fl>Xlla*ceHGpG^(C!=b3F^h
zJ+o4QUgA>pF!#>{){Tc-f>Nfe{Ihk@U?yVF@&27}OK|`%??^3UJG}rhYJ@PIl94Pc
z8s|G#!^O0H5%VxU4=+7J$*6bXFC_3T150edN^hfz)a__?%mzRkgzU3Bl_f)4MMnk~
zO4|oNK&rU5yKvA?_K+P0s<1g-I@sr9*ChE_`{1q&i%$=G8H7GYTjXno@6V^A`I^J7
zV=+)0?>NCa(tC9WKu2ZV1OAE$V|W!6r=KBC`}87(p~gz9Pyu2N24B*maU<_^){}<I
zD`tclOo9E;L6Ov~9Blr(E|Bd`{mJiQW>t|7bh^wJz9a`BoB}SKlN)`K0Pce(i|ONE
zhn(h!p;5{4p#PW8`mboY_!Uox*Fqk3#_n#y*p6%YOgOyCpHBaJ_*h_Zn2<Z|dq#b!
z^68MZ4si@T_?4gHve^O-q#NafbR*}0o+po`LAnvmQM%DWp4wpQn)76B_<jf1hDrN|
zsDZXoxNRapGYY=grw>VmMBTdyRE@^02IS_^XC36uQL~Nzhz6U0tz~kqomfMPa^24*
zlgC#$B5TaV#X#GTTV|Y8wui30bK1Sdz-29@#<};KoXa>wGM`=EY^lx6w@2~@nT(T1
zv0wF=rbg~ujhPp3217%2dmy=0EpSbUhZTWb`R4CTx>EDPw>#%*VAp3p$aa1(;Y7ta
z&423@um0NWdgCNn6fd~9AdMbwy9{T<e-$947TdMFh##mj#zGE2$C?7ED7%GV2>h~^
z;joS8hrI(nGYNw)DGpFK%Gum@SByZ5BJB&6CR6PPyX~l*{kWdn33H$m>0$m6oh@?8
zRB8bDXg%H-aXTRSmhN&rY{MP_2%OC$dqX36DQUleI`l`}Vq!;5Gw$OTsUyTKcJy+s
zI|*@%eeilf(NjcMvFXcQ`Ba%mNQ$J8C~b^QpoXikj1CVPf9j`h<Mdl(lv$L5y~}r=
zMN>=5TYBPGgemab<K4_IZ$oA}>hyWTG&#J4(}Fm(D<Xn48YEk~Oq5*6VzUTB$*&*!
z+G$l-egP3t86D08p8`U(q=ozQC^iH*=wvY0zp`~vJg3`RshO6~{w$a-1ET{2rgDoI
z;r>X6jk=-&S7H2ofv{ud7AN5mHGHFOPomAM)mAk~n)HV<9p%k|Avh@CtKLJb!JVJw
zE_Q8xMvJbuR!65JA;Z?XKM09a&ln(|h#8PevkKvKb#1qj4!<18uS$t=7=J5W{T;Y(
z440kqQ@?tzdJCL2ZHJ*w7Y0+#G1+o6p8*S3(D;Ks0i(Oq-{WK_uo{(Ev5&1-+{SM@
z>yn*oc(&)Ayw)a(-dz~kx8NU1U^G3n5`YXj%UpMuAjj6CZR34W7w0jBC?MiWmRfun
zbQMA)At{6ONyGEou(lbg8Sx3;x4%#p9gp6<$O6US>f_S~8cJw>-{2LYz<NlSkZywL
zQjp>!FrMYTdG+SrY_4HVJ8=*Z+?!d9+Rtgc<ql1GOYa$EX8oIp`_V1|8tU7c6f&4D
zz;C+yitF_NT*R4q(j;FIN+B8<CgcfE-JjK!FVQfrCG5?xWGRn-@r}OS@FIglN6-1v
z^pg>iiK<v}Ki_$NJvJw~7alC=a?VFJ!s0E8d?;=>VnZ#w%(v+rS0M8ne0r;YaI(uR
zMivc|_yi(~eg^F6Ew#7CNzv?bbzCc|0P=*mJjMoUP%y@Z_lU3b#{<f2dvB{yTq8Jn
zp3upIP}FLrD?-=zo}Z??2^m!?42h5u&jGS1)-osyftDSAd>51-eRe(C&8Xou4*6o1
z8`R#RE8ZhaK+8D=Vu&t5#imore%#VK*&Hz22rDRCuqe|w3(QYNn`A(A(8eoJcz$UH
zvgFa>IISe8-dW#RFaS66++D@=#ifJ&rRL4@&9b<ZgK73<Nibk?+7Q0ZV_Pmfb*{a<
zkth09L_RvddJ_sbf|MwA)b9|#oWNU01eh-r!P&ojQfTx`(~=xv!+l7~;CK$ieGQeH
z1R4jTg@?Q*M1jQiWUPZkCVp3XJ0g?$B&#T1icoAf1n6k9f_NZLwxIdpR3DiAUD4Wp
z<Uw(ilOX0sSdnf&qM@<qYgL-V6KryirOT@{yj?y8F68XH2NHno&+wRDN-DRNnqGuB
zOu&;?W`pFeGF(>M-rit)YV?faG&h~@M%?#OKy8hrkQBl0q+GC}uSlS4<llM(s|uvQ
z@3Nbw8(v00Jlmy5<sgJa+_uw#__r|?J;oeZLq!}&2olXOEZJf8_#9Wnh?t$Id*?&V
zloFK((akT@a~i)J1+0slP+dK6Oqt4;eQZE#ms!-0qWCl}4=DAE&&qj%6_p23G#0y`
zU3%)-2oC46j{xcW-?(6tryE2K8!L93D?WaI#}sq}IJ=>-mZu)|jjTD4MS;nCwrrqg
zrn?($+9@51@5sc{maiQrx#4s9gD#sxL&zYfKa8u8sSt2*GwM;G)M`WZb-^Y;P}Tv5
zJuE=PGe`71eOMjAh5cQcx79&_^mOBKEfK&@k0b^oLU*Trc}qz_r9(a;{@}|y&~bpd
z)~dwl*-E9+5|`ULq$u)IBInT)4P3z`fCjqjFQ{Do_Ae5HgnWFuei|D&t~a;Gh<Ea0
zh0{)zUx1JvH%usRrNC9&KJTp%MNSsMtUlg>ASV?rtpS%5Y4gi<<&L_%n>%~@7K&@y
z6jtlOU<|8_6Na*a#4~*lvx3gv{U0O-4F3m-fh?l10?IXL0;7&6Pz#t1<k3OrpFB9j
zNXY_HbWmgRwX9a9S<4d9eWzt|+1>>UsM%?)wm^!HGG(*RHk&hZyvcVHs+J&zPATDw
z?$Zz7qbP3zyDijx7rVZ%r}gS-MRqBauR>X4O=|Ikt|yN<Z$r-L$*=aLgTM)Q-C1a+
zKrDMyD_u=rmLw@p2c<3ga=Q~I=jr2Zz({*I3<f1ZKliHOyK)fJUoxUi8#CtEjvD?N
z9XK5=yJmH;VP!OAr@ogu4*~EmM2UYKfG1TAfc2sdA1xuGhE5wW^H(i4|E1e-J9=HW
z-OkSajl=@|*oKs$+uz-IQjU~6pU>FsX3&-IgwIsI-lY^Obq^c#gjI*fx{lquR*s8Z
znYdpfl7E!xbMEJqUL6K7$#tBwE-wrZr?<xex@w(fMT!I}niRfK`;y!yaZL(D`s~dA
zO!bg)+WAqCz7$n>?;RB{doNkA7f;L7(=wEtEI%c|;qE%e-Q^Q?Qnh01__5#Bm{cCH
z&VU^6o%<ltr{yTpN8>Qk=clBTlg&aL^Q(<}I@su(6sICht%`+4sak7xGElU<cH`+F
z>dJfvn#i*rpD(sU<%*AB^iLH9Gpd`V1V44oyy}H(5Hlf2Ae)Dz6$(x%%QlO(+p5<K
zm31Nj#-<J+3&&(bHnmU=+X>Id6^Bg=GznGwwu72Ja}V;gp2-0NxmRez7a&m1OS}`5
zK?k`hfnDFP%j`+(xNc8P0b0YnrBxpoZ!|+hWYtNQ54+HH{Nln#R}$I-E|Bjs6OhtZ
z;enf=7~7jE0sr913yCX}0R6kX1Y$s(ruWq!>5smNf$Tuorw5m7Xyw}->Vy<xBN2}I
zUGIK@=QIFu?yH+9&6C%2zW}zJ0<R$;;k8EllRuA+^`R_-vutZf<{2>e@GQnTkXzG^
zGUSRlp4dvL_m$(bIiFTU^~Sj_-}dqA|3veox8}Xv$~5j^C1LW^Sx*dnxy8LT?wf#I
zD}17IO+45>JyxRk`{(qacEIo!;<8w?$VrdOYqcQ)@yK+1dXsvi=s00<GCzm6m+Gh_
zT!B{z1p-OtwyHr9yWQPaVz<0PGd**K7E5n}o}6K-O%-X;eypuT$S=3c%rXwX4`uVj
zF|wj<`m?~Cgw})H^o=@%v4hMZ$LU%|m1~0tp@E+&7*nd7hGpY35FH1iP@pW*-^zM%
zfAS9p6>nL@TfVwt%{RUKalBDW(IdOY+rDmFzJf#0uq8}pCr5_0QzY(G5*0f9X_mqF
zOX030t(PXmqK%rvLQ~P@#W~7T&X6z(6$$#39zm;^IdT7-Ey%J9nrsCb2K=JU+`fJE
ze71iH?*HZaB!QmKYPAR*7*rAIOVjFZ+=JQ^pS;`WFwRT-D`X@zr9}JjbN`lj{3ipa
zy%NaKjmg+|$yYVv0j2bt3OgVz4H0V3W2#4_X6dmxJU|5^F<lUpGC%$et<8?0SqD7N
zG0`V3>mg@9D65-hpU{?#$RhS9>N1*XHjhDUP54%1+4XG`=j(klRA`aLXtpS&!5a4c
zb9jnyjXDjDoFD@S<|FM&i3AxqK>{EHrv;~Vl!2oKGH~|(<hBPAkNH${<>mXU#X&S&
zB%p=tB#R1@TKmrLn#E?@B53QWETfrxv7>|N0*6k?pR#ctx04@u%X;yn^+5`R&dvEc
zMq7X3wR2pSkH=&@34d11xS|f?*dT<i`P-MceyekNGNbOKP0hfySG9OW1hUcGZU(QZ
z4fq=A(>F%Q&YkNA)9Pj<D1KGZHc!)kC{$gS8Ql1_-xv5-zrVKqDfRd(I86uj`*hg8
zl$&6rRf83Wgd?c-=GVJ>&raMs-_jp(rlRAy!V=Aw1iX5?e`{ovVS0BSz8iQ~0X%cE
z>N6ZacJ(1?mX!nTk!(b;wmKvs$kl8OJchT}%`pWT_7j7QLS|#jooAviz>wabnfu8!
zfBGTPWAg+N0=x8dBd3%)#$cI~waV9!$RghGtd^t18Dqo#XY{6D<!P#0hQ(e@@{(8a
z%+(dQ_#vjV%cXayli^M_f{b3SK)Xog@i|Fg5|k=J_Y%w+g&?2fW~i$Wm!AV~)bE<v
z&+nmiWIU!X@;lxW-x~Fj+yT|cPX>LbHalWLpPrkS!4$Fh-ksPr;P&)LXDzDdv}Hbv
zN(Rt2+N?J6hC^8p6Gb2Q<967$VP61|H3xO^Csb=f<q%#csB3AU!mUXy?k9ucSw8Qz
z{Nmu9>}I~JR*ubDCmsARsui_PgI)1*4A&6VCi<jxgZ$irRQMq<t1g1Ci2E%b_-*7l
zuZF~fWBu#b`@ctPbjEVC0|(~aUst}~8Rn?j+*_uc6c^y0T}E!sj*{mJ;#$AOYQk~T
z)=D}CEk0<Vk?PQlXOSl5BO4zk%+@#kz`Po&%PksJ4(@SEmvM5+c10l!I~Z#G3+oX^
ziIH<B#hGu`>M14PoPHKA?Lz$tVIjroa0%Gvb6h!pxA$LJF9Zt?!`a7f3%CIaBaT<K
zT-T{S%f!WWnXJzwDOBjV&2!=!+x;t?u*rt4r2t#&QWuMjP#1fA;f)5k;64K;|00RL
z0W=u|CCD7*WqF!*@24{@&`h@(WcsNCH`nOJvxkF%l0M_@)hWg%!2xBcqiY82Cd_*;
z&d%jhGzhsooZR1!iCl>L90gycnE^6D4mB_`_C3AN{tCQEq11eZ34?aw2PttcHfn!?
zqnVD#*|r|`FMfG{8lV@ZLB+(N?9fU=EIkd-7%pT~3IpzRCAT{T(Km8-<SWX3Yhrry
z+ANd8I(4^o3)=O)6+pZe{uZRyV@zwKK~aJNLKwWViz7XzzI{KB!B9$KoPMDf<X?<8
z*2n!X!rnTn&2?!X4o-^%Do`weLZK~MplI-7MN$gIAyC{Y9tche6t`l<p=gofuEpJ@
zxLa`x_N9BD_pEce-|zP>{tC%jS<mFYubH`K=9;wMV=&bJH3oB9&=;mqJjgYuZ=z>l
z+w%zz%|TCVx6y53`?zb(zvS{sfqx6@=QGbherc)M{bnurg6YNhM2gS_$<l5Q<*!|I
zI!Wwy_?c?CAWMWxV2P)|0Et$X<H1h&)JF0<jCkOU`?M-z_4lGPe)PbG|D*V_e=e}c
z$1MH$H;b=@&t&r_!{6%jyI1irvwhDVJ(Pi*P(-6it-!f1I30ifpy!UBzp_`S@|$cC
zp7F^L<enbL_!M85(18$}O27rLEvg4i7EZ-oI-(K@Y8}6ceQ{qxf<zp~rA^$IN}i3k
z#lB1rMU9u5EMIPKoBTEAvV`&VOX{6tFHo}kOc~N>e4;f{ipGh!t2qt_SJ3Za+?zG@
zOdpWS_Ys0*sbCM}MirDK<kNT{OnbZ{SsaxJ<3O(lnG&8stw@BCpnDr<JVq(@e0h>7
ze27jC&;;5O^hBgKKKH{wIx=)6Oct|`q%~svPf7<L-9JtIcmMPnU?&>=^1nh)z0}`*
zh^gQKzXYnkH8pb3VxSB~O$1(vO~HI<GADzW|BfwofY@MHcDP__$oY6^<IBP2`DwIV
zUH!=-%^R4axQ^4wf!KbA^pTQh!PQodJ`0!Ia@Q(4nRh=2pQE>8oc=CHl#xOQCgTqz
zJ3KVs-ye}3o(~U+jCvVz7q9@xqlKFqBX9<N<ro)JPq#EZ5yeWre5aTONqJk?i6=^r
zIxKB|FU25H-ANHz(wO|c72~HCI8mI)o+}DT-9Kt`S?Tfa46#E#)9z?_gkAxB+J+7h
zC{sGaA20kK!Xw<olArnS2mfsay1xSgiWX$-=RfThr}Votpv1TSWa@W|<{L3}jSeeE
z8O4ZmOeO#uMjZgV5MaTNf8LBEI#}I-<g@H0+*$GUN(TvhR}YyfBOB2=jQ6eD?{!QI
z2R8qHn|M2Mg1UDcJ%IqZb7>OhLUxtwE5dVWIFC}V%@c&Z-wEPFWV7(RT*ZDlxYnhA
z#3Pk~kF=cyg*sCCrlg`fBr1Q;l$@VUsAaRmNv?JXz}uQ(U)s(opCHCwPz8xtd@O%i
zQq8EgZ9%+)?>63E6<cQDj+j=z+Dua8vr<C~ED7@ts{DmxoY)C;9sy=TXcPRmzvVFe
z8&)Jv7ly6>ygdJ<zS5`v&}-Gv1&e5YS57py2h@xM@>&5N?-M-Qh$nTyJg1E`(-d^w
zIqxwsH5lv(q9uM&314}fDm_7L5NP4{GS>BUg$~JeO@h_foUj!~dORvxu#MK0ZI7U8
zu0)R*xX+N+$Og&&PERHSsx4LBXBZan3p_IISX0|-#&I7N1a|zIn?j9ocUwcaRvw@)
zT*^vwyTqGytmbx6M3u12HrEVR=}lWs>=y}^>#`RG*Esqi^(UptRkm2@?FRS7Q*&hq
zOa2n3khmflasBgI#q>J=zBB$i%~`YmZ6vgIA2m8CF{U2ohw^!0C-gAmlb9leNaH%~
z^`sU^_Gd-f9`O^|k;au|sHw%yCe*v0leZ=2i?k<<Wd)6e)i<=!|I&zpeXuHULlrk%
z;i6UhHHX7W&ZnYY>>N1T5F58?7k(Q65#t<KSq6~{N8Jlje4aB9Zw#?@<Djb4Xi=x9
z78jQxAb~$>%QJy~c`-RiC;Z^FU{;F?Y>{qZamYb>xRep{vSH;hyTn3BZk}$+#y*{Y
zb27yXr{9CLG}T>Q)ArSsFob$yYw5dp*)a8WtE0P23y-Y-nsoEO<o|m4=M(!har{kH
z5RNFz75#aWdU1bu(&OgYC;o=jfuiijzlJc-d28vbhgI0N0ESq-C<|KWw7i-}XXhJC
z^Ir8oaqUT&%<x2TV0Lt)k3%2v-JK*kUSlBhJVi}rygZQ%WA_Ls3K<U5Mpd;N2g3u)
zK^_IO%dM0z*agP%E1YJZWgq&spqs(D!;n0cNOKc}5cfd=hD;K}1IyqWq_58H4eD1s
z>DJajFR0lXIxYzuJf}O~Fahn{M4C!V6XObi$=r_04^Rm`lqh>;B5@k4HU0L6)X*$6
zre2{pd+WqT=u0;Jmf@fnXNqh16CLm|TmPj=iWp#{`BT?~qx822E_LabuK3-caX0&~
zT%Hf}o}_v!o+xcmDw1F!Sh1Pho64)~avUs-aXDV}O^n6qYJbRIv7YOseAz<L;WRY#
z<pl#;6Q|6hU}zh7>+^D5vN#^8K^<VAUdyI7LuE3*L%JQd^YuK6VdNojL7zVu>1;)F
z`c1?FC<Xooc5BmyU=t5Cg}JWLBQs-bb`S{sb}oAKGCK>5iVCN*w#6PjE6^#v$0&02
zJt6;$N88stx5ft>k5Kovb9&~c(Y)<c4vHlgEwC(HWQ$q2`0y_|OP3UVfj?&eXz>ZD
z{_b!n7OFm`!v1gKqe=ev?1?<Rz}W@5JJ{Y-a^bIj6>UJnp0hDgB|sOTjWJGmym9RN
z<*^8WM%a#Ek{(s_*#h06w(_cAZ(4iJAdDo89V-bD;AGQRF|jrMBOF-YMtQQ^JB?al
zO5vYL9x6vqLu7-|V|vM{rA5hh-wE!?tM!Uy$apSq91jsc-3Vw-!KyWJGBqJ%7@x!m
zeCd<C$bL`2>a;Dn9Ejry4tD3*MkWb~3)wNZYf89lHTHd^fo)TC%n9_jX-rTgo(z41
zbr91Hsyy)5z8d@upPu1n{}^(o<#_ck_&gpQHvbb+N`26=M=9p^7e<=D9PJg1rmp1<
zr!2EAF^MrhLBS$s8TH*$LbN(d90k5>n@}AVdJis??G<=@V|E!C_VZ)S&dJD{$+s|9
z#Eka1LA$h8zR9<GHaC+fW-;peu8_eF)HGU-p*p~<WSjacsufc?gsER5B1mGY06n=>
ziInW1d$`o;yYbhwL=el9_tRIXJaU!Oo5y2DmF=-FJ}<BzeB?9uw#N=EnLakCm{Ttm
zS~3Clp4Xs#tK-ECOuQd@{Tbpr--7No8u@_kdbTAj`U^o@F9}EBe?G35-sHa!bhj&O
z{!ctHru>bdlta}6yy&!!TowDK2(7-k;O2p?3;2Qu$8#Ppuk)?hxhRoZct&CXaq|=U
z>7zQyb*vwf0`afeS^X|tU*=?RQtt20Hv_DHhN|GYu><XAk(F#+qD80o?Zbo09sBw~
zLpFxIhsxmfah}DOfhMo9CtEqNJ^{U3Q(KE-PHiW(JjI?!y}4XQt1h`%a7YNyrEm$@
zJx)%FS2uPwT*xSLLueaVTo%sg>RO&+UVMDC#YJ!BZlRT;eyu?zP1uTRvr_Lkt^S|n
z4jpD(_{v}Y36AJ6<8RuYA|3fz^-my)d-yxdc&NV4y@?Jps=SRQf1ir<0K<na1pAni
zJuhvEfB+9K1cospF3^BAtOe2g4ZN*78KveqJ|3H*XN{Oqs?6A0W>-tw^?l^JP40NU
zWm9(@em}9gPfQV8$l{V`4$=Et2b%sxA(PM<Q=)?pr6@7wnO^S}ME;g;YP+=nrsD*U
zBfUmTMFV`|d=$dJVB8tL!ym!Jesk<!#Uo-bm(ZNOJAl|Y$ijV3?Z{ADc2e%#e<B?e
zQqNK8aV#ceQR5W0=Oe(AV^I5B3K~7X;HV?z{}AIZyiiZkL;X`Nf)M;SUhE6iI{bqb
z?|sangxQB~jnvt%N0(Y^;3EpWNmEjhR^9l7=v9YQrq)GxK8;OhSBQ=QFSUi~dOs3N
z6jv*~Ei`nzKfya}yJp9Zi%ObkI|DtW{vLeZ?O0q`J)78CU$5m^Ab@5JEGpd0yT*zO
z1-80UM6+ihD^6Bn?_N66bvb<WH&Y>{J}n@nB!489#j{<3hnVaU$m^NwWBzlfV9W@p
zVy*%6G(H|>!qxvuR!#eh_LNqmT>!lcs1?1QQ%`ENgeg?Rb37P5l7MQ{0DIzd3$8p=
zeE5+Lt&xKk2JQqKPX8}RO8U2DNQBDDDE~y#0$Ko_lsB!fOx}lf>7`(MeC~9f+oQkY
zQ&J5jv_4kwm3T}GbK-euiVzn9z>jf-DX?&N!M9I+y#2e>B(p8mPYb4T&GU=yr;eko
zYD<K!4x`fQ84~K17aH409cw&v-m5lv>v`IbHCLE^zML<L>5Y9)2J;z1+8JZ2mqWAH
zFpGoOt5>9>BEyLeq$F8+_|H=vUXhTHW<K?apmYT?(0LEKiSCQc+aJ}&JI__0&7SJc
zJ>g3EVwspUm1I9xZQ|@H$P7A^p^uU~2X5Y%@-36GVpK^S45P+RdQHAL5NCc36}MPr
zUy|^rR6siTMnh>F{p=_eD3WMXAQS<Rr4^;Gax|x$KbZNW;TM@gaGj-h$;UFncC@Qe
zUkP=>i<qwce(KGOxVQ_BvTtT<193Q?1VJJkl-hBWJaucO{O~AyhvqV6djNxFXYRq{
zNSqc-eeYsXby^k@GJFERzg{|h_i9haNHxo@C9@*>*Z=<GBQ>p<7^$1Y`KJZFy11!R
zQ?4S-g_XPSzrg@N{l|Vx#p%J(KwK}*00=&!`O@i*y{ItfJEYw{1qq~unpWBs|D!}|
z@-SK7mdw{<IpWn$@^d7@g>EduqdU_#+-`sXY;|;+8tTxT@7hg&e8?I<c9ymk&UC$D
z8uw=UZX`8j2Jw(+N)e|cCjFqkcsC%VX@H2f&@RY$HTnbB;wpXy<aO8g@0=9=;c|Wu
zLzkpat@L|roey(9+L1a&U^L5VsVGvAXNGegGk?3ux7p&N^OJWOf@-gl_2Fv_&rdAv
z2lD{>BtQ5m8>duUL!&;U?J%SDhzR&kJG5Y0nP7xSVu_A8RkBf1|J!{3HdWaqCgx!0
zCE-$F&j_K}(racyAh-Hg=Qr$FgO)a`@NqU<V8I@i7lt71!d<xL86K4=HKDiH^XFLX
z3+!QnUU+a4KFp;Sc9OSjEWXE4OugP5cb9P7Xs@Hw<Z4(o+x1D1;XiP|vBO(TjvuQ#
zmqiw0h>wHR&Vex+O7T!QoGSd({q2_;qKF6FDsZVM6c`%X+{tp82=QQyXFy*P<b%gh
z&V}qKl8e{(U*oPu(o^KwNNNam<){KdTbojE)llL2PPvWy+v1H&4SS!Hx-tkUbV2Zo
zG9#laKa%e%YP)X*Q5sV@=njH~7PFDB%~TKmX@b8G)W0qA|ML>4<wf*d0!gf}+#Z4@
zdh|tSZ|Vs(Miqc<INTP{!Vdr)WO6^ka#@Ha+{Yq;N5&deaT|pnzu_HVuk&#*e7GZ$
zLe}nGquq&7L>sAnju7g8r*@7zrlxb43l-cbGL#7&9#+7Amcn;JxB5({+iJlRyWEEX
zDb=HQ5$b}avi-%-wiF{obSp<dQKmZfned=exWm&2>~wHlvJZ!cb;Csn>tvA3^ecWk
zc%&HU01_Dc2K5B$_)fa6fqg$9g@heq@OzOp0Ss{<ITpn6A2;d$dhq|ccAzpZtic^a
z^OVvc`Y%n~_)vh%t2ms)k7C89WZBfzUQG<q`Gu0Dd}ccPkCEXJeG-<0A6XKdGIE}1
ziS1$b_+coGjM>M?)e>S5Q<_r?klpU1Zz`b=>|{)MGn-m?OVpg}koF|;4ZJhb(q)Tr
z-n)i*AV<`9l;q<B(zC-cjF{ZT{Xx2e{7^#r*8F6`?GW0jNJ$E~cDP}N@&_0gbnx6S
zmJjtqP7}qN(DfMV8ki3oRi;>du1p>ONvo_kU!YUW|2QZX;q}sfrhH@4V6V}1nhMe*
z_78md!z!m`O^@S|N2iu<Hs&YM+<Ck&nWpdkd6eR;SoVjXz#{PyXbMIKra$6Q>f<%x
z;y-Gw5sAanC&AYCR)drExjcx33I$Z1htnT^D2jF6S62KYR}+u{miHN;hh6!N;&sqI
zPKsUL&ex_p312QDS&;=^?2E6+R)}}evA(4+1FOL$($ogSy(nT1X&zHy(WL|tf9TpL
zV~slON)07skB@dcyjo#q6&{OaIf<ZpyvIJOEPT?*L<F?*!`4pIuJap~*1j0dg&af_
zaT;PFA|MBvsv06zQtU?NNR+;Ykw%#_sFsYz?I#wwf?JY7kI(-CzyD>PT{CYqRIfGQ
zczu28MEL9<wsd7(0C3M}%yv1|TTI|Nm7Es6^yI*5YbM-y%!<H(zYZGs&Z#^6cnQ8h
z;7Hfq^fa7|56K6Q9>UEDNP!F_zUV)PqQ<~0URvRhNfnqn-e$JE)!|j1uYAPueevfR
zpbu=DN+~S)FBy#bUPn<=etKbAN4OxE!+HJ;`IIT7IR>iD>m?9rF+|M!<qAPT`*0JP
z!Uyk^_<T%)%qgmfBm-WA9}^f#xR(nV#6^7F9rcFR{64lYqfu!#N7_v$R`+3v_`@G~
zK#-{vrPtaLU@-2<|6!ErakT$2>-$CkP_Wo}^!6VXJke~zFm)=cxxc%F&!m4J5J^Ig
z3n28h#b~38$tRw~I=B-cPVr&uEv|>?$wwhCyvHzg{H0{*#ZpG$aei<(q$r5nu2Y#}
zH?5(Va5M_i6=})b-Q6jT`-<XKGMk}T^qov!c^KPunKX)3^B2=W%^k76$Y6R6Zj}(V
zjzKXKL+;_n2l>Rfg9>WNF0wr&pX3<L0_;}1Hj;S_zTX~Rji#~bkspr`yO{F6(uQ=N
z2xQ&|32~71Sqs3m&3P}n>7#WRpa5Ej&WdQU%tfXz6NRKHB5OB6x~@@cxF7#FfBhc^
zW_si=+qnNfZL=@RHN*xP#20Le;MCm4<869!58zj}<VP|Iz)%SwfQf3?wK-~V_b~Y_
zV4QiO#fI@l&%VcyfxJ1F8r(OSpy>PmW3lU~MMBu_iD1}mxQ9WP3#2ht*1fY*c9#KG
zw{aW|z5LF${C4_@!9(0G+RTWX_MUK+*n@@Tx6<zJ6gvJfEL-`+@XjSEQe>Sx;OPbG
zc`Ru?OH8i2ITvT0M~s}vE_r8rx<+EBgh(c;Ez1|OVw)@cq5IA|>!IGyo#}ngnG`{F
z?mLuRr;`Whi{!g5^z{Eg-N9rseDrzwr3En6H0TKb2fpM?U=oY^;w7x_-1EXE1Z-cm
zNTZL9=)!}Z=f2tt>pp&g`;Gzm)bS)Pn4(xM&~v-)94z+LJcA*-L=`Z#57U@f^0fRH
zIlD!s!Uhr~el9czrQ6s&X6#b$5#!sH+Rz;+5uL9fTzD9qJ9kSmiVkGNUj~E|=MA+z
zi;egcEWv*fsXz2RB0MUVt}Hq@&*=@&qkBl2-$>g$teb^7D&J`nSLU<zJ*!CSS?VSh
zSCS`Sc-x{h8A=6uN~=Oq*UhBZAY&AXq-Hx__BJjl=@2@<G%JEun!foj7xNz{<zLq(
zRfv@YkTCwU5{)wLp>$9FG~=5h9BMCPynMl-Lf&0Wt|m>9IE*E2k~CX_%+(iIEPy$4
zKVYi7<DKS*0F^ZD%(8BTuxt1;is$qX*1Y*p5qH=>7%Hh*RhlyTA{L)NU5@nFx`*sU
zLQ!{)GH;rgul1f}f<3xb)UNziw%@u9vL}&Q1(Qs(=rhdAyriD$mPUpt<GnQ*<v$GA
zHh!Zh?dZ!OakbR5of#S{@g?yi_0ruem>Ki)?Z*S#+43j`U&zu%tm4v7^{rMYzAV{Q
za;%-s{)KBKfd~V9ACOAHKOSnEKqk$npP2ejOZ;a9%u_CkJ}36en4%+;g8tF}_;5X$
zEg)g{+pkx(E8e(D1(y<-iTDPo2wOn-@De?g4-ZD%D9Y4ALV|K#ie(!HfxX!_GYZe~
ze@Iiux9~oN)j%qmC%aL(>#MR2TlpZ=qH?wP`9fN7)&9Mu$^yHYz*d4>kt`Lg+l*oj
zaeo2H)(rr|l*8L3W#kbZjZ$`wRv<$M6|U}KmNeQOqb2V_U{nRY-0mseRpyCaPW_V7
z%$=b<IIG65#-$4qxR>=PTX#g7oY`@H5x|rzg#HVM!vAqB1hr@Z+~2C*&^Q}HBk}sr
zGnA@Jzy~0b=2Ix%7OHI`_QL3dEnxW((LP9#EJcXxKL&I+(Odc)cF_})B2^fBeb~u?
zX>5zs*mb19njATP6y4&31tqPvBFLti&gKC2BszeB7(SLmb@_Js#@cn$7DXXsT-~42
zJP8%Jjg1t@chpl`j=9t6E47(^z)51{5YCAq^;B6C_Z8y_gT(4j>QIG9$RVBhiKN{-
zM*ipw7!ZsIKbB)dEX5FtpPSX5hd)Wz=9RU3hnq4jM}eYSkr7^X-%TgvAGR6dl=TGn
zq|G6mLI?p0Q^KTo{<k>)-(J^Lnc$ca(DxYYGBfRH;9WvhsRke85@RDosQ&8nnw48o
z1UegV+fyiG`OkaQv}5h<;l3`z%rec+TD=c3>*p7a5I*0#9j;=Umxn6;P?Ech!Wvxg
zHullcqr&zmG4;m)kf#!816Er&)VZlVa<t9PqdWGoS-d&$BE=eP@9e*2LwHMarGpUB
z_wx$RGdn(gN9{|jv=-H5MzG8Kq5`PPqevirxJvu48x4naw?08#(I7&9y)Vm~HvQ~+
zc<~~9h(y**BF01|l5n1mlK6r++#l~PzvDYFeV+1C8lmo?BmKiz;jVqNE;Su-HW!AG
z^i^*7Fti~6AtT`a1tNoz&5&@!m)#aCF=gzCy&p^6Z3I4ex@$xtT?iS}hU^k0=l>sf
ztXR}dQ%@DkbZw8<MAJ+~q;^fg7mvepCjM(3%EQ%kNpRD9eNM5cG2>#ZA5qH51mG?m
zMx_b<r(s3aQ#&z_`h}#YkL-gocgHEdtet0mZZ@$~<Sa(90_#5)q6T_4<H<iv4=1kx
zxLs1=<As$b$FKPC!~;mE0ZqUkFPi{(O*r~2uvo+Zlo~X4a1ubUl5Tf!-$zMQ#K@c=
z%_X|1#*c(ES1xY7q#)QKs-ER8V!G#z)?k?Ym|8+j1C2S;#nQV=Ltr=M<2#l)hrzGS
z{S0%JyVW$MXkk($VFI!Z0!+(6c#dt?I7MuRqM3oMcoB?j*U_>fAG-fCpVaFhq<oho
z=tJi=6YNlhIP91{3L(njs`o_hq~Pf?l3bV_SykvkwvL}oF!NB9%dWX0kbEs8YQCfp
z`~Mvd^cIrE#Q+lCb@awI?B{x$^;1M$fIW^Ij}tHqb-BX0<R@7<_NbluG}vwzT5q%R
z+v~r8xH%MTG5~|e6@vG6|LZPrxM}Gfy~E4e^VE0IEcU#Ud9zjv{u(=XMO%AglE0Wg
zS?Q9(ufbM%6e@iFl*WCkbkqM5W+j)w4<G`WGET?f4Fc4MrbW~NY@`xEwDmEJT0r<}
zdKwzlTS{zu%$@J3DWY>hW^CQflA@Ve5@uWB(iDzks^C}_=Kk(yzJ86g#6ZU<DclU#
zTjUpQF{(1cj#={^&y<C_$D}u737O4<Ne1s8hDd*<+&&5%p$H((;nMKGmhNzD4NvQD
z&PEt^imqUBG!OcPyDT1E<k1_vZk$H4ZN^%ct6g+vCmWL9#izesib1SqOl&^}>u$b<
z<w{fQ9>|3m8KEL@z|#Dz!lzYuw;a1Cn`Bn()22UghCc`f%l68Baqc15oZwZIl^6cM
zeh&|_xQ75rr$+0B>89h{AAHY*veXDUxPArlj?~|uQyj56oEy<zdk@;<wqTnQ5u6MB
z?kRLCRrBG>qHCY0-XHCaEUYED8UFJMG+A1CLnkTsWno$@R>fDC@id?p$h#u-u5W8>
zTC`|(hKmN6Ml|JsGBDJ!_~wWlac4JM)$T_J6f7AEKKTmVCH}^y$Oq^nZa*l{!5lm%
z#lj-8CDg<4BL^%4?&zf(B;`nh&N0{yKhnLSF>I1<ILr*=K7q)7?GVZV(OsB>PZ+Q+
z%wq?nLF6Y4NIRKp`a-8BW}!ACJeN*9X*$8DKoHwJTbC3r!}W%euIigTPjl1y6NpGi
z7HMP_E@@sX_vE@3C-V`cZSjbgLU`ppOOm@J$WHQ;T;pLqqzJZ}CyigUjzkKOtPVr*
zBbeNm=)Xb4SMEHOkeQB^YqZf4CKf?V`rTxtK+e}nEJh~{el@yGCqkHaC1LY=i)ClV
zt=%DOmbLWk?mTrv(X96({^893!@HG+;kZ(Ywqnn)Q4z`-`1dAJToh){z4oj=8}mDh
zH-A-s@;Pyy408?2a(Ch5+M%Fso@$nK5Bhh>SR*$2x0m>vM|;RpyljHZbgz6AG3T+@
zdORWmMK4ReTs>}lo-&Ql^6UwPqt@Dk4Ckj}=U%WToo5F=%slJ2#7mv((mW5=vD4_H
z^aSnGh2y6_(C`lr(nqe7dx*n@{a~vt4|5vUm48lhr7P$W5ANN8>k=D{e#C$G_05G}
zxAMkiIv=78c#YW;z8;85PT^ZQ8F4C?Tg}UMP({a{rrM<pShPsKR~c0*{-9xpS491>
z`4WDvdMpVCBX9tNpE(!ek?IKptY#r*C@c02gU5L|9e<>pPQ<aYG%6wsF9^Z5m80#r
zL2&WB61}=fb5}fjDRSGWS?>1EqGU&%6A>?<@5wG9dhRi6)6Cn$J`=;KpZorU#(KE9
z!1~wR&BoFa-UO@)%6KWFUlx_TS+|$7%A41>b~7gIs9%%%X4RzE!+zx1*-?ebd~S|6
z20v#qD#1x)Uu^n!>ds4VP*)vQ*<-c1pIi-&T5zsW?YT*V<6{<{SFfFC_0Ta};EhIy
zVvYIR1deaz>xsV_S38d{-r*WM2`~6$Q%zHR%0FFpdX!XtFkeA+a-{{^u%%zBpKYd&
z_#g7~-^YX7&^sA-u%LvmgNLYl+I3c_=<5e)gN9mdthQltS*r=2$y^Nk^@QB!qHF?H
zk(_)_Jlgk9Wwzp;$g<ZTh!w>dv{Id+BZaS(mUNpZ&aO_%D-^Zt5v)=Q9;??5SKyRw
z%N$%gRPRL>w_Y;;clt+^Idf$1{Ch>zTz7YYyBGR3B000H9Jj%a_&2z|zr*#Z4$PxI
zA!$le`?@HYd*d!7p=?`cN$aAnKDH~=Sl~F_WEi%We=Wl}4pp2qE3s5UOpJgPTnf#|
z?a<L`K_82)T`(coy<ithML#L!BTSgZk83dO9(+GA1)-0HpfL<U3S{0{TolK3yz>x=
zPHz(ql%=b%&lw1&_T@w%kqRKQ0~uU8hU?6TXZ(KX+$R_a+9z3ha(tB;ffNp?a#2A-
zwn)TZE6;HZ?Hk{r(B=-1Yu~3S-mz;Sl&f|@xCf^yK1+dYiE-XDTG4+%;qMw9rTVj{
zd%Y(h51N45!V~yS2waaU!46Lr#flm?^R|z&!cZh;-c}20!tjm9VSM@`wf<p3PFGnY
zmVilX1|atreY~yxaCX*&@yc1j!TZXu=S)Fv4VSP0ktujd+2r7`Vn!Vi#&Mmc$-Y7f
zSjCA>$8(+;{@Ge~{x~D%9{6w&B~l|IVeAx?uKV0!b?7L|zOhtgyhaP;%DU@(Gg4oc
zHB{irx<8^r&VHNZxm?gOd~UK@U8pO&+vhpW`&q^!QBZv2f!nRl`2hJ$t%O3w4R9XD
z$SM?^QWkysa&W;atZpu|Eq*K(%wOc5$*$ASx7Z!Hc_^42Gg2-^rjuiXTE6Kq*<r6c
z^+2_Rfp^|GI(gbSb4W4#gVrF@Ol890zP2J=txaT&Tua`L+Dy)#f2}q@2Jc7G6knsF
zzhcZM7n*g2CTUsZ-cabJe4OMeN{O6vhmC~OT)1+M(Eupd4zPJpQ!k^wXXLwnhn{Sq
zGSgH}EBZ8G6+|r8sQw6Vsqp4sCOlSO45jfP=h_2(vc<BNA~QcrdiOo19!H{DiFab>
zQB4qVeWYTewVec&Fhl<5{lb)j&n5bJYt&R(FUj_7<`VtC8hsP7<nso^7L$f1t&*2t
zK=97CwGvq(KWn=rmkoMV-;=Hv+naRr>PzLFL9A9+U~uQSvlQ=Y`i>aq6TJ?a5n7ES
z+30f?=^dx*zqpLv*=RW3RLIRTP@{-;ns-@bX|9p{Hn0*g6Z*P0R?O{Y>yBVeaal##
zW;fOdXze6TspjZ__q!u#tf=DGV-Q%;?b^3&f**Qb@O4?Rh?hIjSLgj=^{&oKwF+;y
zyuNLr_g)hsH<wSej&gx~GY-C*w0dJNj*KkK0%|tB)kfHD;@(z{&e_^*Svz1oTr11q
zZ8Y<whh6k8SYq7<tHBvJv1XcXy}l`Y`Eb25<rI>R5jFlUYv4<YOylj`_bC+fJU<c!
zNh?HgfGwyr;S&da6^Y-N-W_QxSMFOE{3h@c69Sz|CRrEoeeV_nNU-ocqFqR%q_utT
zoXde}vpDs!JC)JFNEAEGN+1%EMKLMUc-R%?v9#~1@ws>UD`Gl;u>74nU5G~)3Gn*@
z-8atRZl*BT@a|_l6Vf6AhbUHN`c*mq>8!&V3Cf|n;=~~tVw~xOUI=uyZnUBl7R1R$
zqx>qK>EOQjC|xL>;j7IvUT-^eJjx0;$1*EQaozCB?5UJBsv*tHB0x)mP-i`nuwv?y
zOpw!cQh~bSACXJ*g;4w^HPME452_%C1N!LDtvfG0O3@MHb>HRPS~5V<)!Lr&NVSt=
zHgAF_P30#dZAY!V84@T5BKoQ`G-0cs>&*D<<K#B6#*30t-^+%fh_h^G*X9-F6Q&F|
zIV90eRCT@tfppS|YB5gX`R8J>rw;uJ%EL#vke-n`9x?a`Df1IyU09je&{~H`xJf+g
zFuF?Rl306Rw6sQH>-tbj?(@Uso7k?+IxPX$uxI3|f!JPOKOH=w<NO~jfDh^aV37tN
z6C5*mt;wb6a(v@9pF7@~x$W~}m$ZlOsLa{)ENbk9*1v!M<04mQ9306?ywHpb<ta_5
z!4kE<5$-We_Su|yhMQWw53YBn$G3kSduv-w$fvxiTMTGT*Pvf+ea#tks8JwjANJ(F
zS5#pDJTeLDR#q1``_Pkq^&4Q&(wf#Uqk={U_sPTZ<r+6#S!r#~w49uY^^!r?g>m})
z(=bI2>9Y_naN_PdtJ^hEIWDMvcB9Q+E6@duUL;kzNr~lIe&5~ZXwfY81Zk|$(Ava&
z`Q_43p|A0xGe+4{%hS8TVOWCVX)qsX(RrpAyOU0bUFhbj)~rJr{7fz1WcK6hJ@<Vj
z)Q<jH%r@_RePR@1<_)j8?LoU)*Uc=e@(6{e89V!?*CpQ^Zq4N;FBeHPO4)*sc~9Y+
zmF4HK`Pq@)?P=`_Vx^Gt%iAAB111&`W9R7w9s~;<-b7<XQ<bOVOv2IdxqP+mK^#;j
z^QjgaR$Xamu!O%8EjkVwnifP?7I&6@FsBAca><Jemoag+-u2?rThzRQbVf%o5)+HV
zUu&sSScrkh{a6+m(UqUfFg`^1;UG|iff@@H1I345Hxzb<_ekDGFlmtW1qgJ~`k^VL
z=-kjPdg!&XTOHw&_EwbSHq*15M&or#bL~0PZZ(rTG1>i(72weZv;u2HNNw6(+BAmN
zbR%D77_WELur3>FrCCxSoY~_roQQNiU7HR-UNpZ=0=we`%aeBNM1}`>7`_1>?)FV&
z%kRFG(J`bYh}%y$2y@#{H!E7c&aKbE7nJ~#ZHcG}tn>tZJ^9QE&wxXYq`2q~Fve+A
z_Bt@F05R8&RJ1}izZ7eIjZFZ3q1)5&02p;d{7K|Y_KyPv<3PvH@ghkzGfVLb`~GD=
z?$8Lq96Gor1A0ZkWj^RyeAK(^8=I8Zl_Huul71Q16*_LV?Yy<FusZ;S4yFP@3~p0Y
z_4T#wt5GiYhFB+|<&sZPb2M6GFCGFCLQex!{v#&-E38cwWEoTjP(pgBt4;zeIYDZN
z&Jf)?ta?=dk>HH8u8oWn^}s-3{0YH_QZ6rl1$ZyC*yXwF>bKhj=gmzg%T{eO{lv<B
z!9fk~{4opPl`+aC96Va!vMkglZX!s{%9?b56>BZN6zh#|pPO6C<*T)aT@z;S>NEgE
z3z5DR6dKmVg6AnO!-T<8ET<cE_Rjl-_D8-&S=YOdR-GP0GVOONn0QV<Tus!ui^Svh
z^Qs}&@xr*Zu2<3s1J<`+)t#v?tmu%*QL(@Tq>@xW3?#cQ*6-}uoToO>{9@`zSd7Wk
z$r%GA)C-HSxEo}&V{p6JUz|=sqx;nMYI>-gGH0cvL)a&5xLZ)e;Ilpa^c9_xbSv?W
z?Vq%lf@^}J$I!&q8}B}&N`k(-0EPTt3ec%XVNW_NMQU2zai8{?-M=jlxs+Jj8@Z!?
zRMat7yuQ$7eTF@u%(Fq{3Yb}{FP>6nJf1;pHh#^zUz9LORWoixMeT(P<Q|y%fyrxe
zS?8{Px&Bs@4qU_{uuX)+CI+AkGImsv(lqLP4CZg4ce&_P;ulbn5=Pgx_ksI%5n=KH
z+$RhvkaDeyGCYbrInAq;W@z=wy-%@k&VftX;SS+az!S;m62z{f#4cU*vk!7f`|#8B
z8^p|?at4mPq44SN3h2(bKZ;btRq1#UC-9nIrHx%Ek3A`mU1#GAG8&~+@39Fe#tcL{
z<;gn^(FwklN+5a6=7V<~y~&tA>t~+0)jD8ahQb8GGXn=qCiY7@etK?}@t!!p=eH7%
z0dSiO=h$i|a_+BoD9oAe%a6Q11lv)?ZNK$Aeb({xgB17|K}Ui!b+z*=5jHq)1+eB9
z-=&ii?%>sO+`C~2*H2Sx*IumIn(ePK14_=_s45K8M{U}klb_<d&IL8k_Q!WvHu~KL
z&8N^Fz;x&r?x4fyxcYMdcgPg0`9CO=trAfvhUw&0$vPu}NzhX!*h~JI6W!5%O$=`N
zarZrfR>IU5Ca8<@0TV&1pp_GqxgEk({vS>PMZC)aE?-H)>KuIFI0<kF`Ps(L`p;~>
znl<*iJ_c_a&U0~kD%}s6M@E`GT^NNdrI7)EsjH?snc_70SGBxC#)wLYbRWQUpYaNs
zSV`{(a=S??8_5mJo1!3DL<f;PiE{7OK8wJMXLx8|OXrI>Jy4@j7pS3#XDPF*Ba-$A
z1i)x+VwB8+#!TiWJVg$qP`~)0pPsbn8hXEz8i?(`=zqIPmgKoju*og>Tp=2Rl~}z2
zvJO-?ujfg~wtVR&<@D_C^_S~O1F#JH)V#Esiq&jZeobF=X~eJ63c82S4|&>4I2*5j
z4X-Y*EMBg$gP>;$O=7S1veBH~p~d_9dCW1b&V)Ib)vj3MI|$a*E>_&h94<MPKqZ_N
z)(v>a6>EE7oj$^uXAwB8$znSMd_9kCqI-)#?e>L$-KtI!2(nsxH^nnRvF2tL`EYcV
zK`=^ah)tLg&{14bgyYVE!Qv7%J}q*Nb0`x96u~YOt-7<M9sNv^X63+6RhTmeD?*wA
zGR)56v+@`V?zi8HC-Mq}h$KpxhEg9oE+C&~(C9W!`+02o;T@tDCGd=fq0>j<>^jl8
zAuCEyt5VbJOsO%Ly)n_tWa^i?M6FN5Df1D-oMlB44c3$=ZO|OPtI?0KJyC{Ap=rmb
zQNk`p99^mCeif!sg;G<EunB1OR!fGfU!EMbP-dXSsQ0Mx<~y1$>@o?Rt!H83Wz*6+
z{jVwbe;25rTn`YnN@@T~6xU>b9F9&^Vck$1IVEm`OP1Ngu_L-z>4MviH)Hc3a<){<
z&75!sZBMu0Ynr56eaWhtjwu!~M<1{T2W|Gg8OM%*cRAoH`8w0_>;P_>JzapQz%>2a
zKbQw}rZ|Qvgp+#F8+mP&G1|0ax>$N6kd<C-fip;}H&agD9m-c@_p@j7=8lb8^wfQv
zQf4pzjKo4Ammi0rM?HIAJ}Xw)y=RIlb#d)cVvY@SxV<Qk8rieSzs;Ff^wGg3u#df(
zw_B|M@LVL&45*mFXr&cwN2BVNf0}ia?C30iU{SP+9Wq7}uEzSwWUiO>z!86^9oGgC
zfDd3w&me#(!sOa<+|8()a^#1n$j|q$rkg{Vko(wWr)g)Z1<j;Tm`?tRPBrt~C-L;u
z8CKgGn)&bJZ?QOqV#svKE0;IeFB*+{tPZ3pW4PSR7y#Tqc-Ja2)Y!m_9kof$2+~>G
z^H1@-l-+y{+nsVLtnarS;TWm89&&&_jAY;Qkd}PHIipVj^9A#dZtn)!O(IzvC~wI<
zu0&7W)!X)t06O9>(O8Iyr{%8|Gf=gqb0N|N(eTUZ?JrDdy8ctW(0Dp*+rKC=_W2je
zGAw>Hb2dJA$EhTTRrKy8&LVbzL?%4kkDj(a3nAXUuud<xKpCT@Diy5^1a&FXh!^8T
z5&+?kW1>#&7LkMOQ(FWV-O4ocs+&=4`y{%JTd#?)NI0u#KSoM|P7Im5BY@~X)q0kU
zh+BrTh?=|%f0@tdQ#vmb+^Udjup|+>w8icCkclDW!$OAeW*y-ndf@BG?2v9VmW3jY
z^E@jGv9y#S|Djyk@y?OJqFS{i*VH~fwJ!@ks62|#Jp(-;$ex6bg{IX7S~e#SViDRH
z+~e!Pli!+Y{Y&}90+TI#*>Uy0?Mo>Gz5Ls4jlm{offZ%T4#|}Uj-LVo$%GC0m8mAY
z%4pQQ)#gv^-Z_U$0w@8}R4gK%lsq|5R1@w8WnDLs>tAc+_t<cQ@RRJS1A0dDZ<G=K
z%pfPe`3I<;`XUBxa`K_?t(RH#!9aGxqthh%gWeF&)7{^MS$Ew&wSFx-?nw%l!tH|R
zX4AZg4nukm#1q6Cpc0-t7JxB`X6h7LE!Q7ewJ<JQMQ)8t>*jCbQRsC~^#0%!9H>&W
zX*`RV?b)2-pQ{vlYpFKvdnuCqgST(>*CIJoCO<TueFpv=_5ONy`6_C2r%^!`O~!qb
z7&*v1-c>+my$g13rpPhY#-G34(b^>E(dY>COm-?0yG-E8?XV6o>b)TWh4CrtsC3mc
zNT=@B%?%dVxE#a#R_qC<DLmt~58yHKj9)}g3XQF341PIh&m5f|6YLqEQI%Cln;TRb
zI6q2yw;J$8*#5JwTbQCpcHRB7jf)w#QvE;^O7m=t*W(L6JwqVKa!k_-ZG&dZ)J8v|
zwW%k(W61|d#)c>tVP=|$AD3o{6u<n7xGPI<U5-Pz$Lmb3s#-7u!}`K388{ABH>c(=
z#~#iRGz6u<jP$0hcj?S>gvjluuqO=#Ykqyj)9qm-7dr-*&*lUtt>+JCKC;ejwAmBK
zr9Z}U9)mt2IUKcMG8!;&#xgyfvUJh>shx*@I~yynE@hvm;#xprxW%bri~98G$V6j^
z=S!XA0rFDkdGHzg?kl(EU#;XGKM$Ukr=L|+W-Q64dpPgx@7P$}cx0XRGt7K+LFaEZ
zeUg<9(vSXZ_u1Gb+js`(Bg+TAPSLb%u>68p6fp^%kEI%7w<>Q>{T?r@Zvw%`=a_9@
zo|2yp9=5-|5l7;DR#4TY{Z@MH<xtBye6@Qs(amWrnaPW38kh0D@Gj=4&pP=}6qjeo
z<RF|gY|sUDp45l;Vty3=149+0+E%^?S30yZ<71reu6*wuGmkygrLNh9OYZ>K&Y{zP
z?F3a1j2r$Y-?7KXc;rKb=Qu8q`KSq_)4zn-K%EDCT23z;cLr|{&C*9+kC3URqw!Bb
z9Aj#%u29=l*Yq=Gd&5Aqb=iv(u{FnRnvGr?wg};RL&xHeDDPOiI_80{74P~-`wM_@
zspu5Jw)-t}JViDBeMv;1K_)jc#)e~Ep0iD%d$}IMmWCdkJ$g641bNu2^XkvyVplHS
zgOGh`6EleYd>iNF*xm^n!%!{GxUlJ7(IDoK1olLGeNZ$!Z3g5abZ|DI<B$c9rKr|3
zi%2{pPOW*2n)s0adHw`a1gYE=#YlbDAh)cj{*6$opu(y33y<`*=o#c%6*XQ2VkevI
zvBMHozVP3-ZaCR^*S#<%XPxp{vBvfN>Rr{An!XSb&FZV(zv>|lS`wR|j@=T@+Q;A3
zBjEWfE32pe$4MPU32;nvvbavopMBDWUmWl3vu)eeC*1{bBfc!&k#GSlQjb#TW3I6J
zmI+foJWNZg>Xu~&TVjZ8Jdcuh3CIuTbG-<NK=U3EQpv+`jpztG(lhc$xLr?RnF1Tq
zg%d>;5x|s*BG<d$_?GEj;a+H`*LjrdtU4mMOlKeJ;5e%DzNUHC>aJP$NX(05sZ7FX
ztLjimlHxFTIV1}b!)#?~B;1YAdU_vRQ)Slq5%@O1<Hs{K8W%&}71t&LvlBVQ>-^xF
z&bLCFbVRaD=#oXO$AI#o=+O(ImM+v!Z}m};Q73+mrjO_Pm=Ak9oVZljz5D~C8fSVE
zjyUQ1C07xn5Ua8!%uT-}GUnU;wH^$prIodO&|t0yof9%eB+}#L+mjK1Rt{UbTMC?c
z_n@aA?7R~PaKQF1zMiv5X30w?N_LvB!Zy@vvYV8=FL1D!;J~ca$^IA@d6|5s(jv)L
zfss|)E_U{y#3|mnb&)wnXRU;JofJ#|0qK35Do+EX=#!dxZMI)#bKa0ii8`-%wn>vs
z?oqNXW>P?!&VWl<t6`kW?e+3Is}I-rDZI8PnUm#2P+LzQ8tHo=??=5Ne0x2W*MP}$
zc@m;LJmmyOB~?d0KuxV*o^RnfmA$N}o^;|Xk%0x;zPn5EH6RY59}d0!gul@0?0Nm_
zVh+GvwYKo^lX?$aasV6^xxQB_-+uHdt&G1ge>Nw6y_gw5Bw7voRic6$mv2bUNO3;X
z6E%<XqqOR00K8Nu94<UuCj#%kW(Ygqm^X4#z07I;F>tR?6L@vawfU2x&^Ac)NvGxv
zV>ooyDzIfe#pKqxQohgSGe{LPaQCt{o6`%edk|n@coWJ<vA-VIlfrscdwg|KHFq>N
z2PK~tG<t^hfQ)ot1A`o*>~yI$s&+f4JI~lZ84}ARG8M;8PjnCb%Hto66M9bNzw-wk
z?KJFUy+zcpXT1u~*{7gWIoM6F_^LxWexkmvi;n%zXYSTnI6IvI7@Mk8`LkDgs|x&M
zh81}QXml!^v{^fk>=Q*JOU(`)?Ys?-_lwi~ShiW`H3U9Y!zAI0{7yH~RGPAEX3Q1a
zSvuzEu%z>0S)d7(8KQoyPr1)*UXNIDq@$ti0p5e=2yw{;I@Zw_#zs4uK=hY-PaUZD
zl5>E&ee6lE+e16=spozE(es$1^zj~k;{1$V`;h!V-MKDm3ktTbqVV`->nupQ7Ryta
zU%gT{Xg=V^btExd8ex?qpugU2KiZoFvTYtoT6^Dpk{Guv)qbNDDnIj0`EZ;*Uf3`c
z4BYGGcYTjVW<PBCyfsM1qcnZ}+XIt+x3WHq7ffzTDU43TX|6Xe>FBf8%Tx3ccPXka
z`7Rn$EX{6jR;O-it=QNe-xe<AG}5!hziAuaKEy=h&FA8@b3E9i<QISk+Uv!acRdM?
z?!}c}r+d4mdCpwz+)iCNsAbNN7<;m7KFQ8M?f-PBieU<nnl+x=rG*SWu#!r7c%A=s
z1jDq!9C3d)!J6FShJ>yPyHiz$lMrAg{0kDsf%!QJL+(870q{hHT$t^$yhrA{42L>h
zitE0F-!L@6jy(|)09m98FXBFtm1qze-n$qj+eDX0en8`kGMv(}uJ=jTBdHJ0xj)Dt
zKAFUaTxH|(yj61?70ogNbg42K0fwVL<kGi2Qv;b|5PDh0M39BQqBGJ?j#gFwmh6y<
zcTEC!`J^PYGN_pKFmHED1X07HX)SnljCZj`ju@4|g%6A2i=gOG&xE^_`15ZwW=`;#
za9!<Zg7z9Ke6#UiiLVC*c=tBo66(L{3Ztz(J!l)w;iY6PNY5WX0tpGkAK8l7tb4e=
zr#tuAV;#YONL(JK`mO6GB#Xk<y4c%%zGCY?dmd^3F9zc2XBNwcUbyT}Cui;_I)BGY
zx(%)Z9*1B!l;dpOb3V9r`JOsYayU_d*~0HOkTq;xVbF5l^$75gte#iorkR*F0lr(k
zNZ5Oo8YA5Ma+%Pi>&Fa%NzEFbEhTow$c<ywz$gxhZ9DAsoPC20*b-I><b_UO`PD;K
znN$GZIrv{>zd$t%sVaH!4r{FaB8PG$Kf>Scg|3o&Kj48mz7d#q_~bIKTT1hEvUeuv
z)W){WeQKT+-^%G{G4sGoSj`p_2ohT5;?wrI^>)%4aFVYhjOY%jl;IlCe+f|Nnmv$L
z8{SyaFvzB??Y^?Nk5%~OF6bQm1#hr80|<N->3tS|<(4xHDEVv>*hrL&k%O<|<NUO)
zVkM>V$}M;vAnccRRT_AeXi4ss93vIlKjH=wB*VD65)`uVyN<oBJ=j}HZ$O<nk$U;-
zgzxwtM^A`pfV|c(=s|x5Ip<Ycso4PTe0E`2#Ty^##nM}za!b|r2A9DHFOsJneTO61
z8XHKVozG?CC*Cz|O71WIIB`ANaa&*J66(mSV``lvPCYPqw?=%U#-Vz>2!|T!WxDP-
zA$J|df3|(DFx!P6J^vINe-aC$Wwin+#RavMGS9^x_|NEIZ!B(MnOO6@|Ji?Dudx)z
z)V@UtmeU=YJj~7A`?~9K8lc{tMz%$XF&O>PfmA=XYK`*%^V0d~{S5K3#!T78(JUD|
zZ+ZN3_SH|ixqR-T_Bw1EB9ThpmSs!dxtL{RpNmclCl7oI%U&#og_9xwD=uss(v5;C
ze(Sw=cW8Rc8*ciMQ@F21M1IV#lG`VHKvC20<9Us?#1g@<nUd#EeofzY9P!#Gl2lwA
zH=jkXb6do6TJZhGi8HR9jlzLmzM7zU@1jSKLQMwlOJ9At-p%Uvr1GkYv$w=UGjOZ5
z4s&yICZNg1Cag;yCMR9DeC^tXd)LXhUSGe@Vv2Hchfwia<CPBDJ9!JNNk(_bs;lA}
z)1~0t`v`!li>;J4!#?+}<n$gXgK739+RE<+LhwCs1)pbp;T#?w@Bt!7R-?r_W9C>|
zZ+xLw!%zx@d7?vj7|SLShFGERsPyGzJL%YF0>2e1_vL<7y`&*Ue=93XkDeHyNi8Kw
zT6p}e%7rCgeva=E?pwDUrpRVP6-Q0f0zH&od-K)T6_@Hy$|(e`d4-ZaR&S`MIwL*0
zj;*9{Q^`;6C%@(#v)&4VeHb=RSwg?Dq}|!Hx0$|sH?^dKSKcLqf2Svdvj_*jS^3VK
z=_BfKIZgE}-ERO&T(AqC_u=|apU~S<_{v~YKv)?)dt&%*H*MH*4)LU`;l*+=ecWNk
z{`p6BhvG>>+U304x@d3y^Cm3W?fNKc*C8&NP?$N67o!Nrdd4yX*4BcOAI?FD6<!O5
zR*1Y8>=hyRC$gM(y3bxsuv_xts$jo<E9BXGAFlTE;;q2mc<xjtzOI_BeouVwW9^A#
zw~^kiV#tO!4Iz0Vvk0y^L~V55=sMnu{3q`38Y=@E!ApEq#<TO@Mn%O+;bYg+7i<eM
zH+9?ORwk69*)t$E1?mC|>rW*qkVU=PBt<s{`?Tj%el9u!j+H-33!d>*SAL|SQhjx4
z#sAo%vD@Q^(p%g@%!r_EZOCe858l9oxsW<7*xviI=Jh$A${1)g9y#@_V@VQT0gWR0
z{LOoTi!Ir=F&{mJO+W5*KD{?zIcpdb`snQw&wk5%#wdo`+E(L~R&xNQ2>z0WK<`b3
zO<5PCQ|Jr$_r_V1q*8?o6FV$7U}`h*PYb6vS5^pF+`SxF;>07T1yP9eHno$@DO(Tc
zj)(|=DU2qXdZ0~^OteSUf+hO2gg*WXTOOp+BDjQ-P-B4#V_p3&)(*dP@}u#h!2J$*
zeK;3+2%rp=xhy9#mAs;PVfGfdDFi$Am>%zESx;hl|MtkiZT|aAK%l)-iicZNC@AyL
zJ*&{uW*vk3is<%gJ1CYZXCHKIFW7LG<jMV)KkaYlPzNysMteNv&szT7`tYwYou-}l
z-pQDQ98z-!rM|n5w!+ZbOw~(YS$J%AD~-&Uj!M+zcZ$V2=I@hb0H=*#ZCL<9?5$nt
zj*>Tf+yv%kFt{gui3CN8Y36!GL|s_;mlNea^Qv)ko{qdHGgxK6IrnyG|0U-{X>Z)U
zBf{fnX%Up?qGt8-1IySm9^41+?V;}v^_W*D?sI}|ORXRVQTL-mO8WANL}}Y6T~Mn5
zPBdcJz-1Rv+GN9wUzmeC2i20p$vo{JO_x#9wq+4t58dTlVboOf9y`yCNXS@ME*1hZ
z=I2k}>kPd1_(d|P&ckGP#vCsM*ydoupS-&iKYMZ<I#-pL1QtJic*zR;1)Lj%R%FP>
z8MJs08COL7(sF9#X76hMHjq$>tHiukbv-0=VW70{L)0X7e8L50ZUlB7b+|_ruF3ix
zN_Dvb)*n?Defe3no+dRjXeuP)>D;kwlj+{?{+&??w4%B_^n6YZNTp~2cQzLel^U!0
z^;Q|LMsI1VzI!dVTw{+Yk*aYgG?vq=7a*wUu5)>55F#b|J^kXAO{I7E{jGHsy4*IB
z%!7on_vN`M+X!y2Hja<49xLXr)Yt3A$DE(IQ34WPlf8Zh;HL00)+!|-Fk{g@5;JT(
zUk&G*#4HB1JP&6~fgD73;H5%ai^?Rq;?Z4j;!9ozZ5BfX#3v4c<QE%@J~<K;euQM+
z?lfI0?&XrZU#Z2Dm(Xo>O#GR-P<~XPy%O66S>NHP=h!=bvrRv>S0M63N=9sMxkxdI
zG(>BcZ%1s8DZ2E))FzSB<D6U-Y&i6NC?LPkrBhj_>oKr<J4*iC6mPZd4&4%o5myA}
zlZ(-J{E&@fNaPcc_?ArAv>bks_|kCE|3}zcM>YNb{lfz!6k#9=Dlw2$N*ZL8h^R<d
zpppX8h=|090RjRdsfgr|kP-$`QXAbcQcAjWV<Wb)_|5P4`@8Pzx<2=P?!P!Y2X+px
z_xqWTcqQDx-Vft5)!NtgI^8vJ;9v{*8BwQg?iEevTpoH}_5HHg$y2mTXj6%%=MN6n
z^0NY$KJO=@Y&|u`y1VFE+ZsvVNbh!f8l=7`muE_IfE@j_dtjbB8bK@Dk5IDhNt!!3
zr!xn`UM^m4ey7QTjhZ$EIFY81e%6yaznE_vG7j~?*q;iiBcf}#4<kusQ328BMuj8#
zlB~ykFi~QqbUTQKw%+@qi++>bviQC0BdWJCg;)MsT_R&fc!Sokn0`F^HFr7a6od8@
z#SzG}cON5oT8!0?`>P=K7OYGmArxSyr~hk=KD!5ohHTMxt&B|U*!KCwab_7<rqX?i
zw=&&=MjIaYB8Ewc#tfVvbQQ>2+db(CPv36yvU<72IOM02s-Nz<Rx+UHrEXG*5(+v%
zbzW6o^#0~0Ie=sSb!O?QtWW3aX}C;CT6UBB>%amw+J;fPZGKtPCoo+$GJbt^E!R{g
zAeT;L4fS;${SkUIIWBtnYdzNhW4Q$IT90w~`Osp&lqJsIFbpjLt(>%=t}?aKknO{|
z#oqyl4RSbzk*%Hn3HA(AIeSQ&#Jm}n9H0)&-nXf^0@oQkZut4Z$Et<v9}2<$v_Jqm
zD5m2@YS%=c`-9z<X!QBC8dJc4NjfiKaGhuKT*9NmJYM+#^^o_FpVIv^?Im?;N!4z-
z9h+-v09z<}lzHB~(t8r%xy&hw>I|{_nqB;suRdIUqB<7xeh@WY*f=~CFGOebBLIKj
z<YumX?K7`$Z#PnSq|F?^WiaBv6@+ME55b-C;Y9v;=aF~Tz5pjxmrUV_)XMM=0C0ZN
z+zIvXesX^L``tB3Y`fRV4>~DT_O<w$Z?aU9KUB-N+v3zh0;xi~-?4K4PF5Js>)pU0
zkWeSSHc%@MzYiD%@B8ZVj#(=g_NqUs^!mf6(mDKuBiNO9ElUr(|GOa>lCDo(b$ArK
zGySFi!VJ@MZFW*Dba2hl+;FDMJ-zsBx5)I-=SX!o<;PGDwwcq~ZDNxyHaD_3*L2MF
zg4Rg4lqbG>r(LO&61H?mHx733iq+GfyD+}V1}SA4Iyh+BG~)GteUW!W?P_+z9b^G*
z$5V41{pz>LzOt#L3H<59UZ^<TzRr)sinWQE9{$CurfTnZmH?+_J?*2Tp4W^&2)f~|
z_c;nO6hXr6LfE=yR*5Xxls}CE`9<P6GR_egmIgq4ZyqcG>~}r|8nH5}S<O!gylf@I
z0v$j6DGIB<(kq0Ops6uM!hTIK8$Zf>uPl6vNtZj*#4h=^vS-;nk!2Hh*p0iIA?M>?
zCt2*@Eko5_OXaK9Z(PyMdkO9s2fp=sz$0+5y;$K?cRw&3Q^YQm7r0`8TpB5=Iw;H%
zc)S{LQkdzNed&-uV}IiFRRYkgVG(!#tV!41amL(c*7H+R?Y4kmIp<lVwcgCN%c(rS
zv!<Hv|I+!z1+rdt=)GNR7wU&2!wY`hj%R^4Rd}Q?8e~}mP{KRpn{N3XrYiuey1j&?
zn4U)QjK7(o6>GH)PQ2KCfm7fz`<%n@k3y%=`a&a3^h~-;5~9x2Ir5(9+v^Qi@Y>;+
z;?@c+y>@MK;qGnTA-lMPw0(XJ2K!|#8MQg*QM+pa70uq_Swh+Kb6}-RNYLgTu@NZ8
z$78`pmJ5l&H6OH1ireH2#dj3N4A#)Xsns;JFs&{7bn#j~6IyTm>oKPUvZS@)h9-?u
zHuXUMl3gryoHhG}3U|X#L8d#7>0KE<*$yyW;)b{!7LfMbZJ~vmzz-JCel(E&=~#7T
zra3PH@b0Y0)KxQQj!ny)H678&srI?w$}P;ZN8Q^~!}s-q+8Ef@$6`JJ6hALUJJ7&P
zb;F}Vt7?<r7(KK&U6;fRzVJkye@d(ly8Z|O$Ka1^n1}GkiZ_bu=zlll&qdo-%l)IR
z&~EtG*fZ(er^J*$?v^1p4A#v$IgaBsvG1aRuIi=gs_Dn|gqDeIO`kjU&o3SM(6lDR
z*Hy-xkoXIS{iBPXDW2w8s!3sNkNanTI0@v}%E3F_+_>bB?VUw8j+cx4U;IKjwC!%d
zM>>Dp<5fB%Y1#u0hP{-RRVe?19ddO>;$n7F=53OhPdT-~#{rj>J1y+gsf<j{?O`U#
zd&f?kf@#a{S+7>!<;NN%e_qx&EqhN7AQg))GT$w92zEQqYd_Wa6!;_^L@a=e4!>{8
z5zfhrWP50B;oc&3*L)w$uB6^I@Ej0wEeZ+GGzt>Ykc&8Mn?7xP1<+c*?8L8pCd=8X
zcqL+mMvym*w7}orjooI<*gch9)M4QY^0$7HU??r$t`KNqjXSF%PuKtB7Qi1GfMY#k
zyI#HY$1%UYQhpxpS=Gv5C>`Xy+Ek=xoePEr*~MFHX1Skre{5=Yc<{IQ+Zt5t!wf4p
zw*3M<lyr%fvxgrCL&h9de9SCq+?v6+_sd=DEq~x<WNqK`0K5eMFl?AEvPz{ba<b@+
zF&Jp`M=6Rx@*m$-#tQ8xe9yw1`Fxj~umH-NpNP<3nYm9RAg7`V1PDvI@vqr1!`wt8
z-UupY71LH=@Ma-J;O60jz%Nh4#veSDDHMOVV4C;}!1hplqw1qB%iRLtg@Q8tX5*Id
zOxL*crqE~TxNtjO(@+DShx)5B!F&K&y}f3rK@%jBEHIp{1{Q}b&q52dTq5-pH1R?=
zSBebBbRzLjKvM|mv2DkRzs#D2Bxg9%V|Y#{Ly08H@OXh6D9FIjn7<Qt>9J<i+*O~)
z?4Y5)e7%0x=ti5@%6no=BKT#bde-stG0C_@y4Mwz-^TWfTeKr`J@G@ODOKh7n*5p8
zh@p9NY0SxkxAUc!p%&Qtx|)DAAc=0uelVL{%a@uNYlzKA|DI<(cbyBoz+>AmUAi~}
zJZBw7D_%~g&TY92PoS|^pJ$uGN+RZ5`=6{uuUy34NfHO%uvuuI$~#Pe!{)~mQy#^d
zY5gAGS<+$YJ`yZxl*w}3%ax$&JZBAguaI`>t?vE73{1hgNSUr`WbB#h^cWrb`dc9v
zNuD>lTO|9vj}aDJPT#Bm1DtD~QL`c({+OGi>?^R?+gY@@f9Z$0_kard@_uk$Tw2ja
z2KQOs@v<V?V(KIwsUBt54SgjLdFyXJ2-?2da5<!B@LeQH-Dx+U*?w0T<on~-%hFqZ
zF1GsTKQkx$g|MR?<G4z0y;;8l5d|3L*H^D>YorEiK`=)7vnEN@HtE|Ub0sDS_zK}~
z7QlSO{m=#2VOSnV8_l>!W6R*BP^<~l#o~5e!awlp_*_Q+bJ_}%olgT|Ek)A5Htc1Y
zjnc$XJN6@L?pcw`S%Psu$NPx|#?s2x)3%2?o4cE!{L!)?O+z$e*Z6@CU|>HuuxyDm
z&lU;)%rszj8U0&9W{E|3O;CgE{KsH(Qt}2W9$z6vigmk!^~dgexK+KL-o~~4bVFZK
zLzx**$Ez8WKE+m`3&EXboh5{RQyOMe`-|9o;b$!LpeU*KlRt5W*KdlW%|Q$N6tKcE
z)LI)`7Y^s<WBgvH>n90e&FM#p`c|iY&mw#yHQH-<%@KSOU%ic+K^<c#fz8_bX4CVl
zhRfE{9=G+N#$4&1smvWl9K1Dt^Hsb-f9Mq5==l|`ox@!71KQSfLe6EW)=-jyQltas
zzMlm?^b5>#EHifS`r+5XvG_uu{6&Ka!0Foj)!?zO$AwSoZXcfeZGKd(ws>XaJGA$y
ze$QwFI@qbXMZa>#w5Ml#W57A?^GY}?bY?Q!{D95U_+)jl>B4cfKvOLC#=pfm5e){~
z!WIj3&1@1?G_Qe=DO?xnZu47wXfw$0$FS%Nsg31Z$f}GVJNtD2W#z{OdPE*u#yH5I
z&bK~HXk%7CY>`PYP1RGl8F%nWM=fy4EXqx0UfIAQ{A_ovf#<lH1mZ{QN9yDG^>M!A
zSa{vTxCgPxbf=TW!+I7zuP$Rg0EaPAf$NR;U){Hg`*=%2qgzzkszbWu&uu>q5IEt0
z^G)=^xU9;X{21lUZw~~#qhdtN54(Bby(~?dn`Q#uYJ$na8Mmi7jZzN33ui!W_5I!4
zy1oBAdvlAc3yTq|Y(s&RCY}SP?p3|pC8|znzQaEy)X)jxh3K6|lU<btZ&8NL^z$Da
z37x8NWh~duebrD^{^7@SX0Ey15YV4;g~eWmFZz!yKOf64GuK1VijbC#(Kwd4Wx9k@
zO0;VGE{gGW;TN6ip(X<3)_rX@DnwgnZaJp-gb_0Mn#tlZ5CS}x{Ps>3l_TiQR)f#|
zQQ>RX0<!mMajPoa8cAF|Np5sG4@vI9ovgo(g!%qSe83vAc06w`CT*u~za?rxa43r{
zy@DFnTiY^gc+@Cm!A?DiRvCf!8{E&Iy>dwJbTsGolt^S^b#$3L{ayw3pOOzdz5-xt
z;Dew_-2}oEJ)ia{j{_9Pebfk+_XM<;<FEH2)3NT&8aUCTer<?JkGnoRgP6)A5rxa$
z_j??T$?FKXuyq#@3sVf*A^7%&rRDg@zj%LQ`?s?v1<WuwAu2b^t4=!e*ZC07i#F;C
zt6yRML9Lpm(NIojx7@b2bp2LE>V{6OP|&ZSa-Luxjdrm}je$op`g#@~UFbQ2pCgBn
z2KIT!3~g)BKwa8@OAW6*)3DtYc}VNGEco9wOmn;m9mrj9KIK2w{6HOc<izdK5&Rd?
zzVRwjE>!~TQNCaH44Vb1zib!ap~QzCkekoh0}SCI=ZJl_KKUmlK2_!{LoeNN_ATFs
zQIk1>!ML29<p^+y81jM9;dvtAle<x1VOv1qFhL0UqVNmkZnfTvlC@t1b5)-cxrhbf
zH7wO3{Y9`luR*oYV`77Xz3%ilbE(OZAgXcWi#Yh}oPYew%%dD1`l0e?UkIx%Z*%EN
z{34@B0mcx*btHGo?QtYO!geW2ktLHxSKmLti2N>dd<aF9Wr8^f-Z7?r+0H9-{Z;np
z4+O3Y*W5?S3Oe`{YSBFUV~_Itw_yXOJy%&h!lL3TfA~DVsWCfLdwY&ISVP`FX@brP
zqmBD$c~&0(ZIR)f3O|pmUWfEFoz2Zu{OJ}>T12PP6mB{-_8%g_J>Ty8ol1TLe5k&_
zxDAj^vBy5V@U$~$_2`7MvRn-J$z>oX93(%?r}pHs*PoAsfkF4;v7!b+!qNgU%0?{C
zUpCINT`2OT@tqVc>g}blZ<iNB+r`H|VQI@xV8(;X0==TJz!h!z?3LCqqv;>N{O)=s
z9s%DGBP~qvEP7SU4+Rc|dLEl6B%*{%k07&FT7)q@_LLCyl`z3em75QIVaL7dMIMzb
znxDqcwdS@&Gv&~=3@`(2tQv*OyVJ#`7e2%T*F9ilzr|;{L2Udae%y`9<$7I()B(TD
z{as9R1g(FRB39yokk_ZPv*)MO>JCSiQOI;3MU^jCkQ{w-x&nS80uj(=bc_A~z0>Qa
zd+LHdaj(IqqDunQldh7NVM5dwCecU<m?KllH;(mL$^q)rjS9=7w$)y%fXmM;n1<-f
z8UL;aH<xCtck*;EQJxkzHdwN~E1`2DRSqRPvc9Fvx|LSe;r19^>4xx#B4Bkm#0<fj
z?%G7I;Y<8&<xgS%L}$faW`Q(Y(95*~u=(Q?-o26+{1bZNk4nI7lhE9b*BwwX4Zu2`
zMHmq6dllcX<yAtbNSRG!j$fVmiD;rp=(60h!G-N;`M=EUm+BhW!{np6cv=N*14PlC
z9ZOab5;KL&-y|TC*5_wvWB#-0j3P_f>OPz7IVfZBKe)YV{}R1j!4Gb;^~5nb?k#Uo
z8<QY+3V+{DE|vG6Q}h`Rof130#tij8TzR(jKxg+#20xwR+$UybKr@|5!8t<5Xj!vz
zvi;-U^=<<0C`(O?#yk?Z_9xS-2>JlAmLsw4@^LrbymM4VAovz-B=w8$?mxX3+%h`F
z`Do<W^X#*1E!TL_b6t5c%373A_PKO#q)V#!c$vTbuJ<6<Y50^04TGdyI`EoEUARoa
z;nI}KQ=VbH{+$2eU{#NB>`EK{ennPua>H@L=pGLx7YlZOrt^t&Z&ry}*80fho$p>L
zTUC3a-R>O$?a^^D)p@O8uvsZu96iP0G2W=skaPRDW#fII433~VSf9jv)!%rCR2EG6
zy1zP*ixAj%-`ODO?y~q7PJ@FxhhiwMN+I8WD^*z8zx!agk2|t-|MEkk{Q}FWrJpWw
zl!+`YFkHJd=}{%ts@V^t93%F7k`g%4w@JG#*1cXYGSGSWKvexb?sufMu@)Fz^nKO4
zHY7!s{N}Q}eK#}B{%EG>?Y_`andMPuZxoM&D=F`Ei_~k#!iRDOjLG-<_9rQ-x1K|6
zct4wF)E6749+)dG>!nNpWg)F%9FdKDBfZR{g5+)?HbPC(U~LT8FR@LO#O(R@8|Y!0
zDxo_xN_WKAg19I&cmD1Jg_{ZAZjTDho?jK7@eLomDX%F6v5^1LTUZcyDHJR-5<a3E
z6-u)ac)~8bDxbZz{q5lA5HPWereYpRm^RcDAtxDMet754L?XM!c`Gf*73^X%6r0$i
z9L<?7pte^FN2ohe<MGowqANOzil;pkKb`T}zdmGh9q@z!wGct)arRBj?;jJX_ZL4>
zFB1|@XfmjjnI#@e&vinr@V#uV_TbXaS;-9QF{uQdlD^7g3v6(Dw`+S1l6+O+S9k1p
zWBocm4<#sP<FuR%MVD7&gNso4rX=X17s#`SWnBLH=RlDEbv*?Ck~$5d{HepUdD~>M
z>Ot7d_8vr(-?Qgb^xk>ma>Q%d`kB=DWQZb5+0hG2bcHF6`y$Qo{+o!_Mj4}an2I>X
zD0@Z#qCuVpcZ}UM{_4xGx%MF{H&D~DtGKl87WGtl=c7b2Z2`xk9r@>8_EmUgZF_nU
z`=-7UAeQ-&+|aTcZXbonG^}Z|Uw?sg(m&d*Bw(smw^$v-oRvZxCYv><jQGMU4eVyy
z90hR6SKm@Mb-(@q7ltbsscoNc9hVj8xi(y?UurRTY2*{&G7zwJh7Fm^KQEG!NkELq
zQs<5Umj|2XyTm>SLe~-20L{KLUex{cp`EF6wZz1;|J%&qz@PO%F@!0s{DtepVPxKm
zuzSlqYXHjOWO}sRP0mHp^9yIKK5JAE)_}E!G!v@E2Nt9=Gh-qiBeRR4Oi$*_TSRM(
z4a6-qhO4vfJ{wxNt-W)Z!jDz>9G>SL*sF?l2sbg)mr4FrOhxlFHe9yz%1FvuCk<mt
zPYvOP^PI_^J{H_vxPJEbrlwUIf5_@SHeFxIH%hu5o42AO2=P!F=m)ZWb&NYW7B(Av
z#pqQH)5K@7XKB>e>zAf2ANgbantFN}DupXdc@?a;Jlqf7hPeN*8$>ol1^5)X500(f
zTl?elldY6*eza_5J>vsd=)xf@N~*st+J(`${|q2@`l)gEZLPJF?C08g6p!R(AAse7
z&tJe$z1{hudFg{Rg|3G4bH@!a2LZ(X81zehf>Lq8lQ);g=PeJbth_h$P<hC-luifd
zzq%&78qln={$A6YDYKDxtTOmDOW)b*$itv_mW>uqqqv~jUs*;J5ErGDxuk9iyyR%)
zxJU<t3V~A3TxAB)af(}iOUVb%3v6XS`gvX)q&Z@x>5L`o)g5pYd;|qx6C_ht?#55w
zNwvRb?47{y0B%Z(J}7x5mQ7Renn>Px<Mi>yU4|gU4FgTjOU9ns!WbNYW$c~K!g7wl
zd$Bli?z8!CHY<MoG}<%^7}B+}Lay{Ru)bk*(V@kyi)60L^gF3L-<B|sQUa?Di8Pm*
zyhv5zqMnv#vbmygb^0md;-%^yPTwr@^nqrWKUc_mrln}cD*w{AshP7UT<UETT%g|p
zvHV|z6baq#$rG30v_cl^+xCGkE^`roX*l>Si;!ZF0E<vo*-a!z`ux=-xa5K!^m^4_
zeIK29eE(!gU5>`;c1!_H`x5kcEmV?kpp{d4NXZwOp=w<o1trjUP+Kv_s1KJJhw!<N
zRj+L{tb?cSj{nvBS(6@bGDI!gynbd5<Zo!(Q1nUGd6wK&@O1*Dey!*nK{)X>2r>l9
zx>~u_R)0B!$$=`hqS5p@;+Y7V+xw6Kz28y9O$*LkKn=e0nF5^_f4I`urz2uaYFUPY
zH95c`e>z3S)RbhU6e$gr0*g@vb;HX$&`6g7%}AD#Q`w6a);<E!HI=z-S@H>wvY?d{
zBQG&8X!Usm`$^>_)4DO>qow);sQ%`Z*F%BFZ-{#adDZ<*a&IZ6vU#Ir=@}h3>i(;B
z4KZ48&4!c%Fbato6+@;+_QNr2C-<^~Jpur+TErqr?Q?Jb>4H^R$>_is!W4iSUH_y!
ze_cY_u}e)(j#-1+d$P$m(~Q}g0Zs2OU#*w|n6x%3#1+4sP|v#Z)obciW=rbXxDcX-
zU>fi%eTKK9x_@@dG|vq9_c?{NiuyEOwN~XVXS*}U{(Ug;<OuItJSY4KLdz$z6F=Yv
z{dCVvz<_Kaj(bL1o>ZAkM46rI5&ZL#{4LrLa;1~>kvx-(u{mJI-C)h$n?2V0;on|K
z+|YSJh<%Z!9)8?qrkPUqtMcO0Didnm$YEsiTIgJb;r;D_$@Z}i1R*WiO%=;xvSkDU
zn(Osv{4Ky%-DPI@C9uEIc)xkna%CN!I!f+*7GFT)yUto}`QO@eOi9&RWW2XR_q4Y!
z(tPqgSoxYR1T77*g1zDa#8Ub3G)i$^n{3uX!|>I;mgrr2dYZ9I!OYXaZfe|;9>{}e
znFw=*Gx|k<|3wF=^rn09CWD*xPx|C(UXQN~p$^U%nmV;B@aNHU_tMg58l}a+5Qm{`
z9r)jC&jP;oukX+whuqcP2;<~Hgy@7Zv0wb4`il;)Ar?}<B%F8lp(gfX=W9+Lpr>EB
zfp*yeSF65Sp1%#S5o+Jsrf2apNE5=bo5a^s@KIBzTa-P1TBOcJa&?ma*)<$7`q6u6
zl;Vv$pNuWQE-5d^jUSo>52PyDyV=vYLrW%_&Prdvmy%TThIkD#zw_lL9v#|?oRv)7
zh#X_zx?9uRZlGbW9sapZR&%rR#~T{w1=W`*HEyN>gWp)$6bdy4Z}13325xoB{AB2U
zpUBJq7FXnvPteCKPNYXw-b%G-=IyMy_KJfJSRTdR%9mFc6)_OwwqT$(b`P#FqO-xA
zY$;T>k7?;TvhV?4czo)L3GynEG)RN}B*o9|-U6I6&7iwpKMw#x6aUDzBNWJO?Q%xw
z3-=K_i2YWdALHfsm9=j2@+%>JIBvcmp9}LYyi7kEP~JmRljz5v54uGZHW?b12<b$V
zir%`NJz}oArqOWe&Wg^=r7z-ceGrt_<ifFKB`9W!19d5zElMi|o?)?bGsQ7ZII$48
z>U-qG@#g~Vukl51cQ~=ea`s@W^z19L7isM;u6&L&GjSAf0U)KF;odmq34DBD)uJ8k
zcu_Ot!4O6#Xmq6Qp3T+fyLt_>jgCn<Cp&bC)c{+t_BOz4Kc=Ag-xKxI)3mXfy)8(J
z>%88I@3=Q}NGIrYrx#-OYP?7mt1(pj9#r5lHa3>qC?E)N0A1o&{GHWv#Y6wEz@gRd
zOCe3Xy>Y+=XoN#N)}w`YDYiC3+PeR>E_<CJbnM!>EfTc8BmQ$ar|^?FC61?J-Z08;
zfVSwJyP3%TKWjM+IJ2R#-0aN|T`JFCP}&=BD*3x?1*}Qq#}x+}!*7nwbTm!^+EWj(
zCrmkziq`@KY5k^O2FhsmBbk*ae71vjT<Y}<*>|!o!YqR)@A-T*a6a=T*3&J)Rg_?N
zE1~bfYKT<va0E?;B&}rkYVGt4zB+6Mzp5d+Jo{PhNT1QG#Y(n=wypMvtoX%EN~0Tc
zbFsQL*)2+>$wIlhdTZ645%JIrzw!3Bo5&j|U|~F=f+Clqm0m_Pc8+5ckS)0!vK5}$
zX+H-mT3**Rm(yC&C{zeO6GdZ<N?Oh~8tbnAQ5DT}=Na4KQ{i=uoO~z#7OmiM>0$+r
zeoFvmF0xzKbLEn8HKhFGw`eNKL3K?L8L-!%`!kJ0y|3h`az4X+fGbHE=i~kO3~Cp2
zY>wFZR_p{Vd00Rqr7|BsSrSKPd8fF%*}95LkARrHazQdAO8{#oR}b_@+dk1wTr{BC
z070XO47lxh0;g5z`szBW{zO5io+sT?d$PpIpzo_k)L;V`IvRM6ozs_@CE$6T(O$5f
z{vn&esreeNYyDxDkdGWE{OEPViqp!DwpOhT8JO8KoB(O))bhJNhP5%wiF*1sPn3Ew
zblmoyi;NNzKrv4gv#j?Mn8`1C{AZ8hk62L4=P(7qv1GaF;BFycX0bjW))^kDdWa1i
zOXRzWKYwOjkX&5iBNR{h1eb(^AXoMiZj0rC^9}0QNb%XG2`fXdZ=B(r(0i^Dk`U;9
z(|hwQfkXcK8R1J<i9TFO21hDKWB#!57pXpBYZWp0{KO$~`0BF}AbeVg{P`-3m@lgv
zvvP;{64bNr{e-cpTaR15ZXwc)=cTX;c%hSPF4b;mM3428>D+RAA1AR468B<wf2QUs
zi%aD>=b?@`{>D;_Tu`NS9<aRrSMd%zTmd0n9oQX8oSz{wV6;Jbfo?o>s*T&X4qgvl
zCu2`(Go&QaFTG|G1pT57>ohtVBvxt(W~X)0#h6pO4Sik$du1`+@TM^>r1xS<XmYm3
zY*tjC4`Nf7YyNKFYXy#0(ZYf&HW9M$^N#OAqUj6MQr@QzkP9sWTbWH(q*X56Os6^s
zfc=gHgWI{CsTJ155P!k+$KTH8Sko%DE#awDh(U(*?%lA3GJv9hwl?{Z+s~EOKlVK}
zuIUBYrMJIy;M(A!#qt^#KQ!J8J0Jgboo`y&cwsrgtm}U30RP^eAn->oj6fWRY5v&^
z+#}sO|LJ-bSa3~K-IbL7Wbkr97Lw=w89B8vnXw%&S~>}f1c|8%f^%NOHKN|FpVS6O
zCuj0Xo6|DOd&^|A$&lx02};@2T>>VrmhIw{inKytmi*USyWPNgT-eZ+1I;=3#Yz8b
zw1rcpZKU^m`z|y7H^brVA-dMJ4=vDyoa0fC0rRxxkN0Jk5GqtD;$88#EC1;+*j~PQ
zb~j^wi=WlvQ~=tc>JuGwc;%e@KU)rv8Ed}_Aa>`5J*DoPx+UGyvsESWb=fVvZ0T0t
z5l1?a5SH;5qdkA^a@<(b-2@C}etG{6b#gERupPGsxV+GTZgZ)lVpp0Rc%;90t~55X
z$=$Zk%D1W?SsSa8OQ<ifQdhHlJ;R#W{yML>s6pCF(}(XpDX<tOn+?{Q*ArOA#85rz
zuuSZYt~>s>^@qZ_Nwk*mhYPD2@~7}`Iw2iP$e@g8br*Hcl_RD!(o*r;>q;K4Z&FJ=
zGn{sK6u$<djTF1aK90-D-Dnc$z>`RE$obeopt}c7qM!VM2CI#|T|49tHJS@k3oH=k
zKl#620M?}au%kFhe(Q-HzFOFcttO;$8MJeXhy@b}9?B<~_-Z-l@CZJ(PPu2X3CF*6
z$a|XCiW8NUA9bPBgqQ_+Ldw8$>p`1{zT~EuM_F3_Qr2AsWbmu}I%kxz1<td*Vfxou
zW62FKNj4fk;8wr>-pi~#T!?Pto<i5O$9priwIQv5$tG`pZ!vxiexf6BuVDT@ZEqM|
zpMG6(&u})@t-N*G0NAYzFTP>pXmmv)&2hDg9z1fGT6vJ$Njg>**UT$^qWkX_Cv|YT
z%}C(7jhGSVUe&1BbG<+2jZG@9IMeEaG@8BWr1Bwpo?3CT*MPQJUhd*~%>9*q@iZWs
zML`=8WcWE+b=8LMd=fKkE8+GR8NX=Ub*_h2{^bUq`-!r3twJk=SkciK88}FDOHemu
z>mKr^-Br7*3l~71J;_qNg4HR%i1(uIZ|2J6DGG@+Er}Y}2+na3n3P1CuMG<--4gm4
z6+j@!dw--U)${~J+6tFHd-c37&JFx1Wz~B<ua{;Pi@T9*EYWmE6sXposCS9zZI`Sc
z<+fSt^c|-7lEbt5Szs~zP5$X|<Ae2Wx~v!J#MbS-peZ5HfwfyHaAA)z55%q{s<rS4
zag)yJGt+EO^98`bi}w$~i^eF?q1IVFid`rV{CH&)j^bl#|0*n(aM<3w!J`6(L2<`L
znhkstz;OAmHj8$#P$`eS5UuK`RC$ajTJ8M}sF0~V2K;6|ytb0jo~a~8M)COmC7-P2
zH71dsfTEWCOP@43+#U4U93cv)s|H7KDZ^_ciOfR@sBBDHn3+WI{SOn72kwMKoP&Dl
zP-UkMf4m^JaeN8-I=b(nMatS$<ien>;zCJ)n>fQK-oCTw6hqS@tGUc{>%8}o@pfBY
zi=i*4IB{RJzDj5<C7s@+8AJ#@7ptzMsVFYIZu_8a%K1sfcPdi);A%s>F3$q$*DZJ>
zO?>1|^N0=+>m>U<5mm5ZR9C?~Z)X?d)J|umVd-LW7k9L53keM{sAK^9@0x(J*(YN?
z0W+9jR<MEZ*MK&25LCnK$v|XJy+Rx@M<y#DaKjJl*q;RK4mq8<-ES%O>Zt|+=i^$D
z3!mX#qK!<@ceI>{Wq6o{qA6MwVgs-PRT&!Tu2P!Tgl>_k7US}Sb(A1{jj<1O;{Dgn
zL)JrL_j=Nwaqh~y6&?epHm}Y{-X{gQH+_3%%2{ZS9(bP*_b7i|cgCv0&xejNsCR{-
zlas~dJ_NvqzCQOd$ED$W`zPR`8&4C=uglUEJ9YI8T~4YEk};(DYu>ZrImv6E{``xa
zO)DX%ZlH%5`-Ae|g2}B7n_52Bsm%*2q0vUe{4}(*AYvo1C-xs!-a|9eL*_Ic`Ly#d
z_uKe1NY0D1n8pD{x||m{W8+6Ojoepc;mfi(Jq`CW`(3j;uXtbW)ykQ166$`?z|Vxx
z#P2*6Px|eo0&3{Mw>+j*4?r~Cm~ncS%}zB1i}U?j&;yz&^-ntyvRp}g7C7EI9JW5F
zJmj3kWVPcn$^f6v7z~(tue5gB?6Q(|WL@DsR^2)C(?2iT+39mLn^na6<VN;eW^viN
z^>tng6<UZ?^q64@VKQN6#eB5W^D`{Ez2Q;q3v(f+UA41-M$;vbz>IT#pqkk5eqQ!+
zHCyF{A2+@SN1hz2s9)K(@jXy?$Drb;+vi&?jhmAB_dEa4oE;xpRXNS<kT~8n`MQk&
zo_+ONAUw6(r_kITQXU7#0L%vZJ=axCzny@Ne7nVlUMSE1jNngaj?#I%l8^eL@$ys3
z1UdN=Xm5r8nYUyFB;j13rADP0FF+JY_cMt!mw;-Vdcs_0GIvTp-A@JV+>DHL6Xr`_
z_M7WG`$FMQKXTS3EAazy71lL(Hv8Ld%2em7+>z$_eZ=x903|MUc^U1eX5NQEuiXRa
z_iPw`V;;I5snw$mvgE`eYQc)5Ck<=&iY6W{d@q}x?0plbyd0&#>DBt<5kQqDQjY(r
zbmMm=-jb^Y$6${BHH*09B1FzIB@UG7F@b3t-4X<Crrcoe^VuAzNcAb7s~@-v^v0A=
zd~Dn+JXwSJ-0?zb9Pci7#cQ>!RAT6FJPD%@n3lIWBm43Nom5&PKoZj`{?IZ3(Efx+
zU?1{1BtC|gGCA)7oj0!-A+B>SiGv_Z5!RE;n>9b)Py;QzS%e}+WPgPB=2^d4T*neS
zB%j?IEej01ZE6>4HuN?8q4bSS^;>1H&fJh#?&WR!`7==H{PlM<i*m&1Qo-3IxBVnj
z*wFfDw}WWI9V_Q2C6B+?a1_sSOZSP&2y9|^6O9hd+;wG%<1#CSn#AZf;i)qnvUQzF
z`{h_o`vExh4N6m3v+3~1#e~rWa3&3i_$kHoDucIS0zM#`BYn>Fib+=2-t`;29P{pB
zQCF4AiHB#+5|g*8<mZE>#;&BqCk3Gbin#;@X`NMQlI)|)&t}pJfMQR;gosI2>b~u?
z=WJ?Mx}WA!2=GH=gh{7%rd|1p-n&L{#l=|hOMm@QG#g&VDNOwMRf%h7kJ2J7N4)*A
zYt?Ki0?O879YYvZ|C-<Q?2bT*_&A>D>}p+@&A&Vx2eWh)u<nOKri1$urwA(qQeYY`
zTCG6U`0}X<dXi0TJDP3m3S15H>>fkH^i+r)s_8MQWgE5ln7?%#xWi&fKS`IM7iNvx
zUe;3&e+>*+_#NYO5sbg_VYwG;jtp$%vSH<!@Z;>=mchoYgnFH|+6ZWP@Gn^jTSM@X
z3j7vCy;l=F16Txn)R|?>)=@@rh429nuE6fXU?gW-CV9?Z`7Dp=(eNV+`Sd&yt6SZ+
zKaJIEY22(vUc<SR`}Dtc{ZVHATB|LNdOu|m<c{_|>B2%dr74qn$b?yN7RGLYU}%mf
zf#uX{OZiii=P!L)k9qsQ@tlQar%)f}q1VQ@rJMxQ4&U2*+1|&%v{<zjXW6og4ySN)
zQvPSOb!VWMh(+fD??_WVvl6miGjlJ-rT?OFxTEbF=JL0f?zt0nYpdG&^uD4D+CVQ`
z03Y_$>s6Z7Tn(EI|N1&@0@2vXS%N4@k15e-7rhh^ndesSs<>Nb8CCa1VfRih>2}(^
zT^fbid5Uxg@dw3EnNd!za7CQE0GWJWJ4Fz(2b9^A(TIuNl2_KAt0I^@#vv{aap`Ko
zYm|nZ95$P|QKiEvr&IU>0j$?R^>2pHOx^HIrvcHEbe<B?;xals1@O*(uV{mI2`b1#
z9KhE6^^-d^12Rw+b-c>SW8D2)uYPG)dX+|QaZQK5v_g~9(rLj@^*l8+1c&W3KAraK
zV2tv)OETRJ8su_rq$_n1{dc$2@sxLN*(-Sh{ly~Bx@MRcvw>rm7<fL-Vji=?pM1=L
zuFi0cW814RTy9F8Bj5%V(Z=y$Y`3h`3Z9cYl#uu`69xw_(ltvbti8i8oCiV(ir^Ji
zlbe?X=!~P*S5G4a$zzG^E7}lpuP}_*9r0Kec3WAMc{7sdVbIdXLv!pK3A2Wd+{Q*I
zb~x;8Poh9IYu5MEi=~1+-x2{@_AK)+E|<yUaR5&5`<>Y`x6*zw*L6#Lek2z50Z8r^
zH?(NJWS=m&onnN|kFQTyxr00?;uF|1NzuHMLKG)sw&{|fH}BktHUN>9gkkl0`LKiL
zCy2oQBZe>Ywm>G$rs$Q}rO&P(bk4ezG@s2#?n|xHFr?UhyQ3l)eZ0eb{<uNv@v>nq
zx2;WJ>!4l#XdeK#@NLGTIJ!T*KWOSS(;$E|s{olck;LfeC;jYX=wxT3P)8XL8g|BX
zUXj@d9I8x?3Z0Lpxs^&gb(s(tAxspl$^Zp>ZP;Hj0jTmG;Bt>cHX?8LEa2iHsbqA3
zckupkwJ#si<tWk0IrO=FaReaZh`bc_aa9=?NN-Yft`zx)v>9psac4y@GJY<_O^#xd
z?p9cpemWJP`d6|ZIV&&{L1Vr2tsI!CqZ*^e%mBAs`QLbO8bpEzb2VHVhEth&kOs_R
zSrId!AYp!JPk5_Wov4#0?p+m{TX;ll?fnRF2?w~0-{eKbGy?B32_5gOl!$ZJXXo>k
zj}^^7y>epGqfzth1nLnYLai|5H})$g9DR@Z+`{qs0u2_1R9Q9&H<ISKJr2~sJrpq&
zi*E;X)^GcHxg@)GuYc#ld^qu)>_FM0TFls-7*^>N=WC;HLLz!nASd3dp`ExTIToN{
zBP&cPAqDC&d9^*9sl7~IOdl7{oX@pu`D^YC>sxw9Pxh^EJcf66&J4-jo3F5=V$oH&
z<7Z+FnEm57pPO(lLBJAee-sY-wDMh~l#eLiQL>);&aFJn1?5JiMYZC~PbH{xGB?aq
zX3h9=O%<nl!lwweh=f5U5A7<2f4HJuSAV{+49H$Y#8&YS(ERjN53B2hI`4%z&83z(
zqxO)MO^n)=(fqEt!dx;n6}ieruYc?=qj&;joPRUK{^%QCFPM6&fPRwprqjZVx}N`b
z=K?CaNw@nc3}P7NiKp4O4Sj2zJH>mAT~2leR^9Q3WD#S(cr`NiTK|Bz9^hGxwc~4X
zkKz6v&$ENWK7j_JCoV1UZebs3|Ay(IFDW`mWiXZ!=mR&x^=G`~y|w?)aue!YmOVlW
zXC4Y9E(DDFcCq)ZxnObcjFA|t8yi4l@zyIY4$wQ;ueq9UW*#^Tr$_c~JIc_<uR?$~
ziVY_F`BV;5CBc8z%$fQX93$(Tv&^{frq0OFcLJa`M(N5b_1YQ@0H<;V!J<#T&!lip
zPd4j1Mh_2bYCLKG+HZ~HF{zO6mjthnUBi8p;Zy3o+1?4IF)Qxxq%p637SNJZvQkc^
z_?99~01z-aY;kor^TP3W@tyEE>4~S;gB(X+Ohup7yP-A<K$`;&J{ajr6w?@=6Ogh=
zw|<A!+eLo?DlN$^GOUzt&)zju)vB2;Y(t8noSwx#j#RZRCfOj}c3d_qY6#OL{}G1;
zVr{nptg$;8j=y0Yw%ieQpB)keKLQAtgn}5w#=I6F3ybb$D3&GhoBWnGMmyXJL`twX
zpz*5Xhf+cfw3%r(;8`A#qLpuMZs{g%Zzsx5g>wYDaCtXDmk@A7hX1u(#7YtD%M-1h
zsKbejXJNOC9<5v}i;Mmgcqt6L$*twQR8JS)Tj|7Yc3G`GzpZa5>?Z%Eo(W;l72v}L
z5!C!nU_gg1qIWEK^QHhK;ih$b;7I)jgJ%z2h%U$}@~sK`RW{i`vVO3`3>yz`z((Sm
z=rfNzvfU>HGT5)no*{(<lKZ>P_>OwrA6?b5P(F#88t}suV?2eq<9TZJMjx!PtqJp9
zS~yZbYgYf(^vXJtUt?So4rs;jJwg%sNMGTv*4YOg*ZnmeKAo@r)!Y$VpndU|_m8~z
z_#s|)S^kRFuJ!3iAGT{(w-mHL&BQ$um9*WzgfTH|FU!u4M15kK7NmRZl27L-s&-c9
zi@Z&9He~%CJXQQ`<F~Oe1YD$+#x0>cP6Puv!jJ`SAr|n0(|f1Gx@z6d<*<Z^&Q694
zmzHy8WeUjEX8wUaKiw`{7G8(u1DP7DdzuY+K7d^x`+c6le3g!lBHYw%u3zfDBOOQ^
zw33OT{8}$%yid|%p6t<av1;f5?ACV8iFfTrN=J+uX#Rw0h*|>X$6Z2YF>bfrPzoh1
z^C2~4B?u*$Z@L75&NNFMg4?Qlj{-?Si^(y^J8efQy5{DFe<tz=xLIYdo7u`I9z&{;
ztESsP$Upd-b1tFrtRYM%Z=?XMz8@p3$~`*>wT(7*m%pEFfYIy_N6Dpo0&H&fHf^Vb
zZ0O=V2slewGk$Y=pGo(&5r<0GJ*b-bf`{JrMb<LM1-=m4K7_9I*J$IX*NXlc%XZZf
zkk<TuUc>piuK^G(hJnYIktIgqhtEQg%icSLOG>{yYKmAq<oW1E#{baRtiq*!tk%v4
zjZ<s1@geTlZvx4*A1J2+cGv(<{So2K^V{2kSfyny-`2w21e}Fc3HPg}AtZ0E)6+up
z5$=U$bpd}bVPNX*s?L;pe4V25k&%2h3(CAyf3!+hYujBoYt&51MpW>$w%x1})!L)s
z9nw*%nF#Ds{i)o>G~noF_<mh5SpF8OO04SjkBqpP_U#jz7k!rd8R|#@Pwz*Zp*lHx
zdJXSWz7VOhll9#w(?8r&#XP_h)|sZo<9Z^c%A&3<5F_$jLP(kqz`$Hix1xjb!<>1^
zL5tAeHf?>CJ0B%<X}4O{_`OInN*?C!$+Pe-5+<*l_^hu!D62V54YgQ(AOqRJzJ!on
zK%~#nk5Sv-rC{8_hTqUOh8NC!717OHWg03B40~c!`0H%w(^bxgFxF??p$n^G3Cuv1
zGj=hLVc_uscF*3&GMQFidOKWqQs=0S=sCM%^U;Pf>Jguhd4Y7*1&-*9AE#{Dh9m@X
zoX$F_GCl^w+)pjj3Ev8{6>$PDf^sg}<=R~TAlMRh<?@FI4>(1mN~7+$@QZ}Jl*sq~
zJ~|cPP_umq#!)mUJa*A&5_xy3GQd)O+t>Qon`m)d{kt9=pn;l3nf)-jgSX!yM8aWn
zG15^miw8uPB<W!ANGm*EYZ-@%q?F8&CMSZ()`!M{eQ?4k47xsm8b$y&A_EWg<F#h?
zm-i^|vsIXa5BFmP=A`1?NaULa!7)R!oTMe|WBx9N?h|_sQ6j^yL)YtMf&}or7TSzL
z2|(V@3N{;q2@<#Cky`klid0mH^?@srjNKNM<#i>`!7gB0+K@-bl)99a#*9TO*xWz(
zi4Em`S;{VHmY_f=N7s@c&A&cY7&|6EE3ZTy)zX`XmNuK(NS0riJmbb#^RJJi*DyN%
zp`EHhn|svq!-((HzN1>Q>@js@chm(dUmp~@_OjRqmm2ItKx-k|2|?83lM~dbKl0Xl
z<cFc+P1-nUj|cAm`8u9TdShZ`0(EeIdx!elgSu?pKW265q^(PZ1f%jJF{K_SH6Aoi
z>N)>y>J576yeEEXMx9Y&t^t5YsG(QSD>vEesvOR#4e|5^FcXdkTZQVo2{Fc5TIQdr
z?T*Yy-^&g%fgJsqj<kx+Pm3|hvC2Vs+hi_Kpjun2aSjZB5bNFv3nX4zuY-gNy!((V
zE#xj9(wNJ&t;Ye*iGc&`3u~4(BoV*+u354$2ehQ)ZKF$GqB-;p-s@wfWU&%3(#E9E
zemD%s-;3a}+2|>btl2kK%L3mkP%6Xlj&`w6O_0Z|4`3uL3OXK7yb6Q#AiH=cb*pfz
zmx9l9!LDFkFU<{72aqSmn#493zJJx&6OVjNx)!6d+jM%@cx=Z&5$MSzAoD&YgEA9*
zBtAzzeT{1NPDLviRU3>Qh7reR;-RGd;5yO{?e04P!s^%Vd1uP*JP`|GAx~MKtmV#Q
zjRV_O#Wl#6`{(!J)RvnbCv-*4NNNm}v`SUozd4IKNv587kuQ%h+izIQx8V(f|CnhJ
z$L9|YeF@MB9%#fyFy#o{)XzNc*C@JYY;w51NopN3KDjVokIZ>vR`K%RZ^r+4QBsPU
znRgj4E8&N`sJkSdV6vGeaezAN+cXCM*m$4+m@zG-XnJ5>i_%3U^-<4nn7yDRt>Z^$
zRZy!n^b==xSQP_A|L-ejy0CcxQ$U0c?vrz@#ZH8~s9&k(P?WDF^wP$r6*_n~g@3*T
zx>ZZjpyFzw+qD!g{TeFTv(cMx&JM#LjAmKjzcUlSN_0y+ro&@<zE~cuG)XS<)91zG
zJ86_%V=*JNKsX#|EqR;NVrAd-=-Y+9hBMFG^+Efw6Y=7*OlqLVeB{m*%hUKJltuhr
z{`^z?+o&)gvYtPOM?BOjdGM~OYQ_GG{v3>u$92ChDf;ve?+YZ4TIM>W+0z5hGlMjF
z^~Bp;9}rjU5AN`0`C<Ct7m15b%WcLe0Zlclm=7@7OPX5#HB;iidB%fktwl#l66~;z
zl7qy7PF4!E0#qLc?Qf(9FT#8YYKq8PSXLwfXbI&Wn3p_!*Plx1_6=Ga*77IyMro}c
zc<@>O{z*6vUiZ}`KA{po8tJd7F^}njXC~*P!IX2TM?HNU)LsvQa6BcKy3b`?9U!o(
zb)rJu(AUE8B9Cl|Ku~b-NX!!h(8n|VJPObmDz-p?tbp8K27L?-0_YFV^9B7LYJ7Uk
z8b=jI!M4q(|0^~8w-*5ZFci4NstA7=pK>_M<78Fy#tcc_!{e5$JxM6rlae~PX&0z_
z?u32-xKfF<oBz*ufC?}8E#t2oN+Nf3<25O0a-;R(eta;gmd_hM+BMuKi9qXDH%Tnb
znEhOtW6(Yfs123Mia!?TIKjn;tt=kxmifJKc9fFyw`QP<sB0p23Q!aqG-m&;(^k)A
zv;EBgm7M+c^rjkBKcB#*>cZ!Xo5P<iMP>0GE!IQ3a$a(LOww{;IvqwvrZ1AlJ%)%6
zD8wE~2gclVypaWuo`)MIv+?4hz>ZxVY*m4|#`5B7da%zytd_a*^~#pqDOvs9vC5vf
zcpit17fvq%cLw5v2I9$*!9BtIC0c|dedJo{5U_(MjO0FPeWXI_)~6l><M;Jpc&p=`
zTzv{+8%4fq?MqPqL_zUdS5e?yC>W)4{xG@$c}Qf_!rH}CLSd9P5P5{UK!qOkps81Y
zOG$^6M~tM{?%+c-#S26(2az0d8h*<flX{lHRN6193k1!8C}WwVCl|HcN%N!+Fh9cR
zb!f+iBRK|1ebQQrJ}IQ9PRWwr>uZtlyt9<YGuDTF$iviN1!B-=BCz{$WA~8(wH`#6
zgx`M>RU1#`LaQTiqw$oM;FB5j+Wy;_SO2LD|LyUchtlF7p^cM&5>Shx{oG4yj9NJ+
z8O-|;Ks^Y8IR&2^eBvAEOuZD!bLn_m(}EFtjKqABMRp^Qwg<13X7Bu0<?%zBHT$Mp
z6B26?CrEM%bW7_EiSQk*iaiYGKbDk1P<;<<yLOg?*5SLX*n`N8y-cQx?f?&MDN<GD
zK5(2de&E?4bK-J<3LR7qcyx0UZcUtjV_4=Lu0ZbaaB~~{EXxaBbTqq9bkTi}|Kf|U
z>_93qVn^uM_2nGSS#1rW6B_9&a<>_IWW&lgS2j>Bu;mSPHAIJ_xJ<BTOmKNh%Kg^g
zFdth?eq?0p@%q<|c1bfIiGaGw<wwxPwjQ(%I*LDl*eQ{v%7S*Xuq|JW-1xu>v&63u
zt#Mq`&R~iMRfiVwRE_y#vImI0-M~CFvb1?WrA8*$f1P%XEWGg)#ge*ytVzU+Bd9NQ
z5FER9U2x8zj{QfDh`eJ*#23wr7ci2KD2kH>@z&CWF@(gisMo`JQg@^PIS+)Z(4WVw
z2ks2R@L9DWGB6kuAP{t1aD*TwsfDI*{EufwtMv;8dGxfPY|AL574q>dJXbubX%0-`
zBU6v4fm?~zmX_0J8l4!SL;J4VAdRwbWDA_0WbM&^JuUpmk7nMkiKF1o6eM9IfU@90
zuAV1?$Uo=beq1sZh?F%6N?e)+(B8U4Ux2^aw_|2f-S;^mjHs4cfDfk@_U+EdXiN4s
znehf0Lwj1Jb{TeSv~@UWP^dn@gWh;h);j3oQp*k_$g`LKHrhI&BGa?`pyZR^Ch#aJ
zms?3jlMoF;T$E6+PZ86xPW~XrEaIuioDGv!Lxc@#JDipGuVE^2!b9j|f}_xZIHA7U
zE74kqhdr(9Jc+{ij;Qq*mmcv5T^Zncfh;R6>Kns?d|{O}%yQc(bu!(WqNKG3$9GVB
zkfc;#(0Dxlz`TV&E&m@a(*HWtlg57^X*TUhUu^w)@Q-SVdPEmY!S!h#Q^^(7Ju~tj
zYwYg=<OvX@2@UEzCL+qwqMwLC^Tz8MAy$FWX!1K<@BIl>im_aixS%pW?`O-PnoV3O
z`~rg}Kgr6bM+}0fCLHbEMk@^EV)orOtE1SOe3n$QY%d7Uw2_t`soOcI?=MdwuV27K
z@JH%K^OS^&!#xBoMJ@=XGd+6{4#!XRs~+@JuEtC*Za#o=jT(r{L}4~=gfa(w!&C}M
zl}&btqqu>_;F`j~R`Ez{>B*th0$F8^h9$n82wCP`zL^Q~Byz=r&==f~!oNh4PBd{x
zydFVcT%7+$C;7jQiS`jgf&a}K*Vmn_vmsw^y!!l)qpiG#83rP~NW^~qc>+FoUzYL#
zjh{s>27`CBJo)ByFd9fSHUQMza0=0gCWnVLHCkUs4#hjpEN0SSm-u(=pa-^B(7brj
zi}IAp3vDf5_(xECS5zrIj<yaH>a0Zev5WglAFR*b*mngU$+P>oeTebRUU{Tn2IEIS
z@o%|cl>8!ls~gQce4>%p-t=Uxin;=(toV>aC@U+!BN>(>y#2vSQ`F7)pD))>4d)se
z;R(@?*gn9dq!1ml9ygwncQ_zwi~Nh9V))~@q01Af@2bS>9O_55EBg8Zh$G3kWmn|B
z2RS)-#w>_zd@_TiR(gPmJL_Mi-2dOQ?0-9-x%mr|JWc{@BwFhPeAE^D^S|}%|2k>E
z=Zu=zMW6-|8;{uE2%rwtl8E!iyp(<G;7Ll+GJeGqZ-i_+IfXdZZ<1YL4McMu1O=n{
z%Y}{+Av`at0=xZTf?8PHn1cC}(@nnGw|OWnR)t@8HrpJ<5yzaZT9XYHQ-%=yQiRsa
zz!>BCi<liTC?u&3&h9pvT)-{r7}l=7Y<*sqh^n)Z-9lL<u{}qMRKHDv1DFE}#M6}>
zvjloPxl5L;K6hz#sKx6wzOG!<9js}r{ATdQLmqGSFqOv%g3+n=tG9Iu*w5!{b5!?Y
z)q?oANZc0B<j)M#%Aro?0)z39<ox*qS7=MR7D=3{G*3-MX%eMS|NlYy>)%0|@&6U1
z#>LI6tEut-hp;z~YU=*JhTGao>p*I$L}W-61?Q*~5P_tYDq4l;S1V2s88pfuGRPoF
zZd(cw0;Codl_5o{P!+-;lR#2|Fvt)QA<Que5FtPa0dkYv-22{Wf8Xa_@B2LK`R6VT
zSu1c)&N=(+z0c>?s@~ZFl0oiRG~z6p?_d$s)kJz9eR5AN_7&5TD<rB^-aq?%rTot_
zqDU7cRc#-CleFD5bKWQ}Y2N`Xof_7^?z+OP#X8&c_Ti8@Kf#?kLLe1t0X?WL=rq#$
ztYD9D#NL=2<0puAd5Py~HN2x+tGzZRwLiFF`?u52$!A{&ALY&~&)B|PzV4ufGVY6q
zYX+7LesQ#*cy2;s!tlxTD4E|{8YRIEFXv(}HMZO8w33%4eG6?Ruy44#9Xn<_J7OE?
z_m5lun31|IP3j*?=X=4kp->In5vnc4X|d3JB>`#5s~j=C@_*0Qryuh5!v85>X{*6^
z$_@x^23_uKO{Yjr){l_Y7@F!V8y=0eilWCh=|1D$QfJeqN9#7(iMF2aR_l%8Vllo0
z-Eoe&(I3xhzJ5Z;R)22c_bxA{5V|h-d1PHwg(mr)i0b$;&ZlIC`ElNw7zc5Ko1X_+
zTMUk8Ono_=iXWc$Wt4^<@tRl;Z=HSe$C{0sHdB*?iMeFo{vUI;ArZNMY<*<%OYNzz
zJWER3h(ocj+n6N0G=RjI@4mSa%fKxot|)Z#G_F`s1{{1kcPNL3K1f<-ZM;5Su*20`
zLunQr{+4_V`l>^yA7vu~1mbAEAWB2&@-j9cwF~8`-l-I~Uiy|i`~N24{~CNh<RtsV
zdBd$UDZ#&TIb!Mf^8Z&8Prgl+Zbk5bs)02Ey~>N`$}qIHP_T|S!G>5Ya*oD?(UH%R
z>R1ix1+GBhtdFmwW+-$}B<(c2U55#vDS%?NHqQN@b&Dyt71Z2~nGN&1sCTFM6P8Qx
zl5cjk6+-<tSL27mJA80-#l9Qf+~BoOOjl>nz&huBSgRb?8t`fmrYm7y^BdX6Yx*#>
zd}>|6Ut52r$FJfwrrUN$uc^($?(a)?=Re-q?DcsCwWaZe@X?dL;JJqpqcSg_Ewe8O
zcb3#_qD!R5VBg<8oM}JUc*d5odEsutu;S?IBloVG*`5X^wXqgHe`)N1H)P0BpFl>u
zka$2eh|}N4DHkb3bqlMjwhaH;;^FE4?YQw@{1L<Za6|3>PdD_t&uaIm#@&f?@2qiq
z2L!+*i>_0EJ-l%#(j5xbG0;;b8RQ)0m7ntzQ?yXlA39^va+_9KhT=SH(pEL|-6FMb
z;n$SA{H{OpsI9M6{un@8vVSbDa|wOXuY@D0h&F#@>Ev9RM4F)`&Jp{Zx3JDEgW4Ub
zVG<@$$=oM9kG%f+pTkCL()<p2mLBB!*<?&@v(P`w_xgI#lHb$$i9B!FBQ|m57?gyk
zb0^9u?ySb|`vdFVN<20>jG;XU4qh;TPQqFLmZ<T=ci{8*l4s)XhUj0il@rOE3AjS(
z6A?--L9cL#5Y8j&8MFog=oCuSmU3d{F*nD{j9$zB-2Q)SCft?}&E(VneKVO|Qu7H8
zer3)=8;YY<o#rOmRo=;XiW;5@11%!+2#7~|5bZQckNQ+jG%@Yy*P3cRR*iir;>y)}
z_{B&a{yW+5$J=S5H@Xp|P&(^P++vgCtuX&x&SYH#xT1gH6cUDGP2HZY#?2asQC1Lw
z>+{^S;WNVtI%+GZwpzm7b7qvb6b$amOTrmD(Vma;F=XU+!3Lj=2jg!Bd~xR#zu=?f
zprfOo6V7fw{aNRM7s+Rf_<jP<_N$X+VXtzjjmzU*sIZ-nQ>^d9Q^94zQi|;(!ms2`
zXf}O5-y@bpbueM`P6faH*3^GF+qk1oOHWE19uw@P&HEEU23=3m#{sV~8Z8gP>*jEC
z{=jQxMmFI(?dEv46#IyPyvueZK<`v+=kII7JmgKw%m1&X!1ette^g|E-*3Y5{!4y*
zpx>y!Z=LPT!&r(KDEf)kyhhw`d$pk!%`#TTMf8IjnmHX*FjY{ezem(4kp&YGEJBJx
zk!)l};p~&8X*_mQ;cASJ+fy6yDjL+%qS5b=Vye8pOf^<xdO;Sa=jxiVTjMGM7V=Z0
zKeaeGm4vrB`|UN!s~6a43DaKLU%Zi$kE3hK+*1py0w6HIJ5gBk^%ziKWqvqDTqb3^
zT1A0lcU*W4qhkku_S(4X;;*a!dU1n<>$R}pu6Mkv`Q%XPL>u>$-{NlUII&H8dT5rJ
z&n|SRTTEf6h0;;#wXLdj^GVQe%-xi15fa0ndYJ8Y&Czr3EgG%VFOcu&#;Fj?nev4D
zcw!m{eCI`*7Q#1(NWQ*>1DDZ<Sa32&cZ@jN!2w?Y;|!kU30IqnsnguXX-bJ`kFyBv
zs#V@}<jtjGXrpV=e>bu;Y0eYi@B`=NL7X}%$&EI%)9FtrsQ_bng-!H)(KG9JEn}^j
zc?H&dVgg6}^;#paq|?$ng;U2DX%!2VYSAw9bNRQ&lY(kg0(74Xp3>InSi`_H9HAEJ
zpb`OD2TgQU)8<NcfS10DX7gBkr(_uNGW1B!SMEULeyZl&yKF&CqNR}&U<6Q4Ub|fz
zK^+~ldmtU4OF?KNyNz<#TZhGXl6N`<&k#bj^+u!Qci4<(`^D9YK;$74v9;3Q@c$tW
z!P#Du@v;$)s+kT`3**h^uMn~##>X!1TJ1Bh(;#%j_p{n_>v{*4dk^ZNYeZmJKxXN0
zmtW%{(05cs*Qxwc**VD8j5;^ht2F|yzEx1S(5lc)<r6iop@1kF5UK$IPS>f><na(h
zlLx$wrbE++!UR6jO-XCOcg7t0jCCZVPW7|e=U48b<MMThD=xDzGc(%OXE$<OC0l)z
z<=w)=JGB#HG5=MLa$%y@ZfD4>rv7V_)s8;ww3QMpJw|iMJepv_TQe5Z#K9ZC-g|M|
zuE(n_ao_sOfS%e$-e<Hggw7dDk3apxgOL91cL^cqDeTD6rF)mxyFc~3r1yJMn%m9p
zi^1Rg4ecoVQ}g5RDH^|VHz5<xO^<4FHbZ_MxW&|a=eSePw3_0-N!YYI79KvtYV4)o
zds8JDmnCC~Ee*@awE+>32`({@s(EXBEy7ei<sUrevwbbz=~vbysr||LefPQWS0fBT
zeiq-Dy@IH0wV~6?8QLAa-Lw2U0Y&1)<nwX_ffWnUx4k(XMY(6@z&g07{3zzHX;^XQ
zKh`HY7U7x>z!{l>cydt@QiJsA8P`hfBh=)K?G4_p1C(s_SFO=VSfOx+7ll;o6oWF7
zYebc^M6K~>6Q7p#rqSR58Rx$jy!-_L@KztvL9ypnNUy(hI?fz8Y$J3Z7zPw@4JD;0
zy=bNI@Lnhu>0g+CrX#gWE^=7Uig@&m8=ar`R?ViDBVBA2PNa?GzizU{nqRJg+fu38
zTjtg0`s_yneA3pL2-wFbFPL#H{nk6A?W8I?4x9_WWz04vZ?DHw9H^bU!1hxM4Z@R%
zU&+460KX?&?}L$=;(ghfqh1sCM$|bcPbt3WTwaq2flT);mQ%f0T5<6{{*Q#PPEdT#
zl`J8D>~nj?Uq5UZ{O3pNqVizailv)BM<Qdmt=$u<Tx&PJn8_hw=>+W*Xu4eSSIHOT
zwv*egyDQr?UwV&D&0XLq*7rQ1a_Y<pXQK+vC3{;1OC}y+h$Rh~PFTlGzjw?A(l&hi
ztJN7Iv;*&^&$%>rR_^JM?yMVika}41nzrr$jN**WI>dZUCwNV4kB*vZWAK(FIme=_
z(TYdgqrl*LtTo$D8DcRnW=7x$X3i(JQNanHr9c{>tc2tAu5^JHYo??QxuQ~Z5-S&)
zskJJ_!bt4Rg-{{e8;M3D1~C;QcS%p37NYgXZVn?eu)i9IJ|+teQj>cVdJ~6%8MvG7
zs}>0m%^(4oQZ*Yc?(AlD?bvC5Hg8c=CeL;4B>>YZAsTF1LIw?vB-x8bDv<|n=;m=<
z$b$NlWtlS`g#NdyvRwqCkorHD^K|#ABdUe|T(hlE<op!>d&#Sx^8GHizUzCdX<_WB
z=@&RS(`Q>PkCn)~J*eSaer~fczXn6zF?MFVc|)ThafN_oi3_ui$#xHSUjrT8HQ&&J
zw;*UP2yAk$Xf1}9zlay4anF?-pBRb^7&GJ0HV1x(AA>&2|7zQT_#IL4Pe0P$ATRm*
zqtT@Utrw(k&sr2>om#+zh+^zxvhy^v?RV~j#C@K<%=IR~_{Ov<zvA46Xp5=+McZ?z
z{Ok(;3SL(2(C%x+lR^A-TZyH`tw;5jMc~BC%&FMfC;qYR$_6)IPXe!rjXPU%UfO!I
zLfL)$Y}m7KQ#?8BD5#A&!JTpr9i>pOf8E)hTC~Rmka%aYfMj#{8ByA8pGXm7^Y7>-
zqRF}i-pTVEp1kh4TC{Neu8NOF@LJPCiAqsKtbu)muZDr4@wUYY)xagI!ePB|h8T_1
z=@g5D!hY#VwAhb~cjpMB(L^Q5Z3pyfL*URDb+}f<zOXORDVmAq>Ze^1?$iH<b!c&L
z-1rThsul^aw`=^=q8cR1^-sz)ufL<bT3!%UMC5Oqf#BQ28V#Pd2I|X|sP$XTyelST
zOkBXFrVy(Z(h6ypAT1@^&E*FhQE<z<&al4y{L%P}<}<<Fs`%9Nzn95%`oH5$VR01W
z?2yEa1x7$qWzH-jb@8BQTCCN=Dsq_3i;cq9V_10I^LPAj{Fws!3}UR@@0E3C5vmt6
zCxg14WiSX2SiH}D#@XNbG&!=rD(YJ*dBqag;I^BYC+sXfv^xIjsg)ytsyfn+bgeo~
zqrw!c#YU{j?I1l=Q&69Qv2^yIeeKR2jX2UH`BK!AkaNz(MU*BnWsU61DI%Efdg~ti
z*vKri;2g+cPt(A$<^x0dsl*iy{pkq;K$1~&?$t9UhP~2f#R-e3Nzs~jmOVxS8(s{o
z8DRhg_)pVAK<&HB;MlnxzMbc08h8h|V~>$-R0kQ#1?gpg9H_xAYU9*;h0pgLtHC-X
zhk*Ru)k$P=h|b6i6eoiH(aZz}md|Kw<O+?Z;Y@NAT5{uoXrx?+j~GkO;dxuvAVYc?
z1W^nz8quq-;dZf|Y<N0>AzM|?qj!28LE~Mx_WciMc*H6J&gW@Nmqq}+yMHOig$PWD
zDv{k8Swv*7lzIAGhzjFUi=s_xj8PmVozM>T&E8~KQ+FG^JV``meQyezvC{MS#b6tH
zc*&QzzD?xPw9B@pgQ)$sjk|{N&X}y{-z%Ag0+kw5`Ip0dzT=%Fk`=)xC5KnPf;_ZS
zS}s|qtbtpOZ6`ny`<%(P#HTS-CXN_!fz|R-zCPw2@M+9XTMoouIj@~~cAT@RbxR8D
zQZm7GZ?32+yTky$kL>^Q#xKdg<YUO}qEqG2+~%d;_=NVM*SN{USG*5~uZky_Pd8?2
zJY8y+jI~wtjcOj*x^*Y+XlKrW`PUg<b@S!`nEFeov^g)*_no!!=ymF;vGt}_9z&?y
z#ebEEOQWAMiYs|2w-7fJT(FIju{&tsR0C&as3b#%KBj@}DgZUEU*J5Yz$$9rL29!^
z6quV<5OG7K6#@G9V5^fTtoB2X2FdCl()_g`$V!gV_b)`NaW%+*UKX<B+vo@5M<GLc
z#)?$=K>=!V4H9k{d5V<j9EDsvdzMvtZf14F<!Ao}FpBkUbQSTw+pM7jH_s5SsDIPt
z(!8dNc$M03qo-i0L*;$<9<)wx341sU&@RvLhD054<pMk8v|CP;9^2N_W$_i@156TI
zo6GkbVOTbFp_MtCc)85EZ}|o5L@c3Gda)7zBS+v^8~oDABHZ2SqC5F%3|PQDwTT_1
z7!Wv832C-sM|ufZx?giM!TluFf!~M<jou{_3(g3?*MUwwv5lrp;*g8PWf=8&e0AAG
z+5X-7;`xTdkMK+Hm(;UMpY$At%>OxDHB$ZUrjji`w92iY+nrqUs9CUQ<sUrq$$O!P
zl8!@?UUuOcvDB8n3r+IQA}qcy5pNZX6}<47YG^%)!>3hnO)oI;6qhX-NgUIi8grzP
ztyHE4y|g9JqSU?&j7iD0HOxJ!z_>d>YBlW4zQ<VO`-e5{L<-o61|4@Lv}){lHZvO<
z@`SF6bAy#r^%PqpaFkS!IeYf$>_z0LsOKYGNm7mxFpl+fI#L@!*jc)qR0CE92J6{0
zG`i{;U9B!q!IN8b*M2J(?|9-_>3l`T&IUrg8ajE-Naw;BS1)|yDEOAsu1O4D^uEwP
zTvU|}M8VbGD-Br6PL`1kjOZa}d6`NP;^D+`l)**B$}__khrEGkeVzBp8hA$QEX*+c
z&lP|P9JF)~^}bQjr9&cpbK%L#)fB@wsS14~5~GG2qKhV9tJ$PXAd?j&lZ7hBAKsXv
z>QAf6_#~*3MF)-ITA#!WTrDr?)ADOK&7hBc)nA9j>E=Xe&bg0C1GSotJ2^mgv70t6
z+}huk4r}gsNlD~3Y{CnxY@OG{f+&*z3d2~j*cb}Qw9F>sq?@d$#fh>8>+9~HdO@>2
z-+Q(S6hJ`q-E)474C-Q8HQwm?JqEnI%jw;6Cw5BBKR^0B*_55-Ib(L5jsMR_?%3n|
zd!ldLoJQ%`Pc{hHL#-HgR?2XDF2T%v5t{90dzYX}3birI|8uv&Hay?j?$NBOd+$5n
z;ukWbnlqR3%xu{^pOG2_UQ>Y|)r1L7PN_f@@dS1YXO5_oXP7gf2Aezu^}?VNmY5UL
z=(#Oy_Nqjtksc;0(`NXU=Rm?2Y%(bMLFt>wS;=#NsO80q1{tv#+5+-VLvoFm`jVU{
z6paAF8sE<M@OSXJ?{xoyY)ptuR>>7byqUa1|E6M5bR(LN0=6Nsy>uu#dOuLrAoP9#
zF4KL{fP>Ksfi^8eDXwY(TL)=gP~qTV4(;}W$bRmro<TXuus<-U&Y7+IG^|O;R<Xde
zxx&?<5~d};o9UHy2CSpzlCMlzbj^1V&%*QJHw0v7rBV|_fvFSx@A*}WMaZ15(N<)q
z=H_MoW&{lEyr>mo85o2r$30kH(%~~CnSGIA&hsmF6O(wh*$7pWH2t0UN#>q`oo)Lw
zk#=jR)BaE4a>k_ba#r%S>AJVPjqtBmSobo5KYL^I<nGFO?!Jqep5{NL^kch|_VGD=
z<*8d~`<KNl8avXmJYemy-4$Zy+4k7i=5CrBFYc^-i6L7})MN0OTB<F*Dq!rSC#8L-
z4Po#e#X;;rrJE3WS0_nSmzM{a$(#<>j<~t9LT<yUa!t?kk0<6f;xppwq?(H+^s4fy
zevf4lk}|-G${m~A00q~-VgH4ZUj{qL{+UvIMeofH3&4S){a%|sj|B`73xS<PNE|Xe
zekU8SVymKI-iTujJT!CD8{otx&#Vl2rekdh3o$fKv@<H;I77R<vb|5nlHY$@-6kXI
zplgF?XAGdIoZ~<~LMI|MG~>OBbQ)Ym2m;Zm_)2+!)xS+N8qq&|jyM3V`k#7%xG731
zuJfE%zxwHJB}VBVHJ1#!Q|lv<4scpm7rf4~@eV{d!llhzFU~ksW)<MeXW&kPQ2pN>
z?nDb%naW&oA;aTTciOfi{*8?2E0~^P&>X9?V+TtFu6~*xPiXG)k?_ZGYadq8o^iNh
zq2GOHz0DIM`UH3Efl{sJHk0lk;ePwY+~YT`+dNw*q&FKICIjTrlGm3l>#ju>XPAG#
z&SLwy)j#06)^RD+-C*{byWuuOX$A#b?;Fm*Ux`V5vS*gRoVYE@7wb%=QRg7RTx<!r
z|K1_7U{^QR&dGWA2BB8K6u55#&-|6W@H_D46!>ELPg2uTHkmLmx3{lGTecFn*qEPr
zqFq9Q{21wl75+W|l^>SH;Y|{iV?VVgv8N7!jKRZV)miFKRj+#i4sWKs4k;4;>h$#f
z6clk5*Yn`{UDw$(>^(#?4Lw=rHK0*Pc}znQg$AG-g!a>HpavJEuO7ypWT7FoN1Nfb
zw5uL2a!qE(4DW>a#~S2$$7b}>`(UAIb*X{hh;%iyN3SYJ17><LLySUv)4oIBF@vAp
zIHQBzQ=^c-(GNnKvWMnBeL&R6Gkhs*mZ>0i(S$ttTduOQ;-5g|g=vC_edI?Q&zN9T
zGTC$NZvXF&8R7;O!;El`K;8a28ON>YlQacMnf+k$exQl?1NH%PEwP^|!Mbnt+R)@=
zJ9oOWxMl2A=b5lR2jr*t$+c-~_+JpGBpvq~%l7wou1&yLLsvUNzu$123yw;;Kv@>&
zyXD<-XJeyXKMs86txUHo{O&7GUradROgLd6U^H#%x%Pm8fHR-vfbQz=r+%->ezr4E
zd<SPH!Jc&5KuIS#1V~7Ob5ciI-=8~@&9+jteq{niipTRaB{L1gM(o+^d%~AlpWgMZ
z^~-zl+u3$-`@J+W7LXRXo`{xWczpwN_jeO~Z;+n`Kf-u}U~rIX0&+C<H_1OW5O~%n
zSb#Q-;`oM8A7J#L{7+N{Li=S}l1LJ&N2z;=Aku{zIwD8?gM6gq#FSQ*u=ZcR$y;Wm
zYepxLoI?YLFL#&(O3|loin5;T6g0Tcz!U^_?!-;05IS69sFm*vrW5gr>z%}!x={i!
zqmmge`5Ip~PpvW%tnB?%u*&uDB@u>AY>~zikxgpdBF``K8Ep9=+8gO$rUH`6oWS%N
zU~UyIiAbh@QR#rI@VlwDAa4s%+dS18#jPWZj(uOis&CF05Vq;uzE%2WDa|Q!)E3K#
zZCc6AVMo}ni>=NNm=|A4wZ_pL3bH0z=5{$WazJe>bHBhynNMnZbpK|T(;wXlBzjYR
zCfIK!sc`{sFzT`LhhMa~9Sz%XT=G@c+_J83wtT;2^5II`@7A@H7gQy))Q30pKXm>9
zdmk+R(p))jOW(r)XCAH!8>^>CRLhYL{4Ag06sGb33KUJ>T-5xQ7kb36>SOK0+o)-z
zq*ef4wIRgqAvNILxBt4a$pSIsF>&xyq;b`fn;x3<NsMR}SIIbA_>O;nfkC=6b6yd6
zq^a`UyjZS!K{3IYB#__@P!Vs|)|i|12udMCGCuYRI+84C2eqOijdU(%O6!$b2cH-_
zlTn9Qu@Mx&bSj(?9|en22>eSR%IivXoME6t#;O+7D_}&MWiNmKO10H>ctkIoA=0>|
zI>j(>7O&PwH`VVWX`!wmlqnhpYP|mKj?zWFfZF+w!ccan)i}<go>kWb!K%|bj5<qt
zNX?FkRB7-zWRa!<agyR}qII~64uDp)Q`y>>Hg9?3+*I)E1f(~&q0D#d6|N>B)Fm?2
zsb=zO!Hyp~v17-n+jW5<5IAoly1O7QdqPJ&0TgjV4k$+BOsu;*f6!AqO<Wtjy++cu
zk_GOWM{PzVBa>@z>uW5QaDRd1ZpFz4wI(?BN>oYzm>c<XLrMBKcijBaHdWJGZJgDb
zB(D|P;H?)?6XyEO!BIwWGM3Dp@p7wZ!u^0^G{6YAbPHZTw+vY9A7{zDR?MM?w3Ana
zprA~=(9Pgv$IO=jk@yqFO$|%4m70g$1Pp8@h)Irj_r)r<VXb+QVn@%gwb;z2LRWU>
zWh1m7V&W~A@%97+gTo9Gjv#`-Jr{}4z-)g4u;_e3Q7;o!y(6zCr|`WJf(m7^haCin
zq12*<R-LT#m$Bvg*-{m6?F?FA(fLvf)GZxwyyh{bjT=N~MRbOE&+?UjGHoDy#^Z+p
z0vy{$O=~{uDC?5p88K>6^3_*{$;C;LS(IexqJz|asM^p_@7uor0s3!zw!Si==lY22
zkwl-1h^$(AT1KAgO&eGgX(?3vZ?`h<tjv0`*W=0moCQFY$>J$c!_*ZEJI_7*w<xar
z>LNY3^<Ac{<*l_#tY!l{tZ%io#dhch`_zr?)NhsMGSyphU1;nG10TA<Cw2n8kp}7@
zZ_bZw`EIhAWI+Yz#i~}4Ieqa=bK_=w>=8}M@on5y_m<e_1T1g{GQi4z@M7)V!(m%K
zy0HJ!&g_puN|!z;tl}U2%jaTlW`(=@KR;ZL@>?G@lJ>p0+4<uD6C8PHwm-VG9xu8~
zrVs4+DvH|JE<2B>mb|nd@~danJDP@BO!bcoxXLn&9po07JNJ$3c#$ApV6<AALe7a;
z(F16eY|t*p<;&AXgOi87LLPs7`dWUb{TIAb<>CCYo(F+HV<`iZZSld9j8JeOi5PUU
zwqbPU{&gRy3HcKAYma!y%5i}CK$VKOAd-io6Nt-ml}J_io+i*8)uAoXFlADN?3~+C
z2v2~bnMwgq2cZ!jS}qd+Q3ft(7$COG(8^kZQsf^lN=QVUHUysY9ND9|Odf;F5(To)
zg!xY1aWI@dt+@na<Bi36;XS58aKZ8z!H3Yg<1{x|c{OnC28O+Z7hjiR9aoXcBJC=r
z2&=}rqStCH9xzGt0lzzpH3d}A6KcZVS&W^P*&lZSd$t(v&20ZI^vsFPO^vcXuzZY<
z(mmZ?+qARBi}gZGCg!XZZ))jheiG#WC2-_g%$rx<u}J!9iurawviTl%!Ff<1x%J9o
zo9^D!JrZ$n*_(fu?juPaCtX7JOEb7tvigU%6joqEmjInALSXT0M|y3%@xyMSN-T(x
z6g#tS4<)QF*_;zK{+Q_hx`z#|<2Dvt=-$J;M+PZbb3N<nX`hq4vY>@yAp{ek=?Fn3
zI~yF%*u!_}JIveDpDr!qa1yK!#HN>IgRRLKA<CA#;CX?IE;xw1lvl+>W>n;m3Y2Nh
z<|NdiHS_XF;jl){7xHGF7~<ok;5~PU(ho1FsoT+(C=ZoiLC5$SnEOr@t$Jy|CnY-8
z;Qeq>P7vL+Lar!0Sp!#{@tIKx(00mpQ)tWN0(4k(>Y=_Kx{Bk;1@GVReUv^nTfm#t
zJ=XCiM0KcgGE}}o@iY?#gUcZqsJwB7dun%B->aZ}=-tQy(`{)wYW^meSt#43-FOJ<
z|1*iyj7d5Tg<A`7xGURHb(?f{SEEhl3E~QRnjM`mbyUfz%fz@2Tkv2DB5wDmyi+lB
zZR+N1I`m>h8Bk!FwkG0f<lJQT4Zqc##hulSn@{Qkzny;PbKeG)j-4!4hMWHQblI}_
z&Rxu~NX5Z@ze`Y3JuBV@Z%lYkDRn0en|o+(n^UPEXvT0o{X_H&$&%|*W>Q1Gm1`Sa
zoQ}=$sQ!&aP8pb`AUkWAhv-dStR^-&<Zf`XhfR|inQfeqUPy0F;a4cr^NN-HHpWvW
zd)geEIop4S5~jJm{4Cf+cG>+Vrmysi%Dd`*N(1f!lZU}ImCb6nE9eWa9`Jd+zBx!C
z;seAXt~5A8I0dOBD#Rd_O>2Q+_j_(pNbWKP9e$r-`L00KMMD@XDQ&&CD6St$r}x(J
z-@SpKe1#%CB)yKW!Y!)U`u~!0={%(P;>-uWoZ&29JfhqOz`MCzfOb<S%30<*tx1Qu
zdM|D_j;K_ni~6|~3&MO+e`oi6o-p5Ja1X^LtPMGj$1~=){d9!B>1@cFQAtleb8b1n
zjrq^E3aDZ9_|qivx#?$k&o$^&)Iaf8e5iga$1ho^R-<}ljmx{Tl^kmS&D&-4MwUg%
zsyIoy!EM)gfd3bJ!$}AAn8`(u->R?(ZKC@Uim_3(&AC5hr!3oLbSii6p|a1xrKUuk
zWD{XMhH0^yS>*ARnf}(bB}#$pyh0+*XVbA>px?>LB+}Bh!@HYWFyhvWR>9r4sn600
zPK2zv-*5*?XPFOiCmqDEle(%~Wna2t331>UACGcx4$UMSJp>%H_2izJYa->7XS9*6
zEtdS3J67ngmN$Ke?R??0iJCpzpI7`C>!d3#>q@j&D!tRyVy0>f2$>@iB0cp<$G>eZ
zqPvr*$U&mvVYC*C<j~b^8TeYHA0^IsQOHDRzaDwdZ$l|4h+Nb}ph!{^XSfWGoF@W(
z0^C!hoXWV1eFE~LDnk!lAOa<XZ&aVq7duvwv~UO!c;4|aaPfhFO!XOqrJ{3m936k<
zRXL2C$`)iH&KbdlCI_cRwUIa4b8&RRTK)AeXkg^*LaOi=P};z>#i&!l3TBGC(dr@j
z4ddtx7KGhonC0wA(`u~H7)`Wv_k^ZMx|5C%cfW}E#z8EXs-ED{vj|K7ChIi`J>X-a
z<vl7$VoNL~Oe#8Focp1f5WJgqB4_8Eb%S7Nj@ZaK@G<Yqttaa0Z#Mc&Ejvg!M7A<^
zu=!}eS8;%m?(WK`kyBf?)a<yM#13&PW!ezX`GvUqH@s(cmj_jgT3Ev+Ek`Q`0tU3<
z4UQjsqI7&$6N4kTBI(Y(+V?|1YkB149`etJ?u0CkPRumqyPYAl^!-8pw3!&0T_ees
zvvWv`JEfYR+#FQn_drQNoAGKla~#~*;*cpZjTnz<k}*qf%m1P>^&6;!{%x?dydW{~
z*#gg7R+N5;!#nH!IH@8iKQ2hY(+SDtj}JH_E<Vhnb@l@qSrmI3k|tNZ?&MX<AKbrG
z6Mg?|6pYqZu8b)BQP@&MWC<U{9ZW^%!wx&NXs$1MCKpUUFR&Sy$sWJ+p*PBg*}Ugo
z&)uW+v|7`Sb%D?1ijbV)1>x$jhu%P_aKC+5l;Yb;LtREmo*FoZ2iSRi>yA#4$}`_G
z9(TD-6ECLn_mdtIkL-<XQs_E$y~O9?GC_1F;;w!mT(k@gd)*o#$#iZhT@H-zcNiP%
zY`*4|5?|NYdXl$i;^kS~*kDx)nBm{Y-@vK|bYHeCc_^SarLs$;6K~e>M#aq3WYM0-
zfQ;x$bU+oOdoiiRxZSrrH0K^xKL1*gBEY}AINn?R$jyYYW4#c=bI>0y|Al)?Ws;Q@
z)0%B^Wp=<0xHE=7`PsKIw&^K#CaZyBtXZAO&o;0V_msvfz~O3;GIi{{BHX5p_|%MO
zMy*)I4!;j;JYCVDsCCIh4AA@iitD@a|NN<&H78f_w-{aDTNqEkIq(DW=&ANdIN?Ne
z1+)rxeY*})ZldXnpYavYIGHBU&SCAS4tBj}z@+8~lo!_;+FL7U7%TBSh@F53=<iO0
zd;vv?<MAGo)G4aU1wgwXDt~6+*qo7}UdDK+ni2b{Jugx)1=XlHLF*>9eG9VZ<Nq}&
z$S(`Xg%8J3folqGxG&z@rx(@`flEM_YuN{my!&O?!x`2vAR0%3=zHv1smawRR6kfS
zVLh9OM!PCT`0K(?%rj(^U@zRS7L@O;)J|!2FHQlvB2{b=pt_)LPVeN&|I*wplsChl
zvYz?cBd=}S@^zuyfZfW8g)tFKnQX`eDXvG1fUpOClQMzhdOJF|UqV6uVzVa@pWT?h
z_igI_OyF+tBjgRc8E67-DjC2(-UV5aqhH<OR;7{&S6gR47lEZ?xSJmA;+CV=?K4Yu
zyv)Vh(z5{vO;B4Bbo*d2<B{KwuRq$dtU7OfXS$nD?v{VnJ%8nt!!>z$tt9H0<<$PA
zl`+1TS}kVBZqJ;@)570;`j6?DEl=|8Qv8$1b(9QDTuI-!qx@zXE_N46q?e0HXTen7
zo>cO3N8Y?PW>3rsACMwe-QJ3;@IaY_S^guoG@lJd7VHu?G+Xa&#3*8VHN+!3Vk6Tu
zx5(A`{)fGGj5dAQwXRfh1nMN3S7=^3ra<*wkDU8t^`Q_Ii<~JdA};Jdi`2oWV_;-C
z8q@JU6t6@`QK&-eMP&+Ck}%<(^pK8i2p}gmQjHR(-%_Sv-69k^GbCdfrhiOy{d7b=
zAv7%Xnc&rXy_q)NdrkbjA0=muf5MdS`<Hm*s2d2tIeD=G%qr9*;k6fQFBcB!D}Suj
z!ZmxjE%1^nbeRJ8Za{W<5Gf;dHm&tqp)b(0=qlbCZ-kH9EGwJ87X32Wu*#@Q%>S}%
z4gbxeMMQBlU91w^!4s%qzWb+RTW5ohxoFB}Q_XCk%CvYv`Yiu~{JWkV(3AN@Km_*R
zXfRrxr?kxva!DofX4@EeZmsf0*3&(j)#yxC3g2cDyy<6RmDY;SWnb3q{X+Y^z+}|{
z>}R#ha-P2E60h_LDh{>`XvHk<syTg*P)t$wVV9FwEpLlgI&KRgsQxN-qc}T6hJV4M
zwr0P(8ble9<f7;FbtPpCH)8B?SX26dqv@Qjn56T2;WUVQAY03;Z47HyG7mXKyvC3-
zt)l!r=r?v~rzjRwDTeJz4Ld6JU7nu@mS?kPDYbLk1x_}~)_i><o<$Km7V&vL=4Cl&
zksuFLh@@JD38wUwA6!E13w}b;AdKn_&FjNn&G4?oiK^LwNR3V)M>7u%K#lGA_1%h$
zne;`;dku1ih~uFtR<7eLW|tdC@r>BgJQ^iLK56K_=*!K*<LY(|VO@k^3bh`wixM=o
z?+Jr)#bert-YcS`Hs=p}S^HhmeW35b<tHS9G~qaXDii6?n^N}Q6!$j=by2$>#RMO7
z*m*QWy~6eMV(_c}v=o-oTKe5o3)6&PwpS{1Y+g;@h0e!@NVzINJ_k-Ea|6b%6=!6X
zEl;-~HjDb|Db`kxZGzVd8}U~bmI-2J$^;pPrZB|$!w9T=i}`ZWdtGu!@#-m;S?$t{
zj+##_KZ&}L>gJ1j8L_ALc;Cuy9Gz-7y!aRQ?xqyfH?ApdEgAw7S+=OXvTGv+=cx=V
zlgi(H|3Gox`T%yUk?n#w!Tnh4_WnB=dq%|+b_h61IkNj!VTDUhBwTiZQLljV*(L9O
z56j&tK@H$yMuS5Q^vv%M1}U+nenK0!mM}<thT=Gh^A^tkSh`S^G4{_dN~_;MRVqdK
zmKyk+{MGZP(LT{~g*I+@@R^|M<ji8|oFV7_9qdu@uzg9_42pY1R;Y*Lz41$UI*iRL
zx^~IuRYT>;_9z1=YSX`tcocNuKXWvANJdgM82}SQ{!QR^G#BNAKt&olG3Lef!U88}
z#gZXG1RTnxFsCp_Rm7A|#L3jVdhT8vGnk3>bU2>lIruRyfqiZVfA+!WvC>1HiGs!R
z_^5#<lE8SiIVo+eUEHk@7gy@7=|%xObVu0Oe&Fe>IiL!4NKN9ZaD>c2bk}30nrne6
zb6)0d0rwE&{a$-z+R^=6Wcvobv)a(Nxy9l{!d~9_+?^{^cH}6w?X98e`ycG6TJ|Jl
z+tj0T7GdT5Ke6tIy?D#5A-8K?UFwL~P%D=jJ7b)YlO%rLmLQsaW;y5UDE(UcrD%+x
zY+dCnb({M^bF#S!gY$5%;?qp&v5B^(PbV@`AQ$+`PH^5B<h%AAek!3H*Y}&bU0>{3
zNtDiA$jC7!#70Gmy)rxf`_4#rI-ypT6I7R)2}k7w_r&a&YqQda^}&WST=YvFGBT)%
zmazxRWrQJlD2JyGN6OS_N34NGMZaVND1Ozb>*C;%1(}0-3Jz7HfMn{!_^0TRi3}QH
zjR2?&IxiIA`s5tVEVCNse$G|6)kJF+^p}J?hatE}T~xT$C{FDiz6+GKbw(FN&^8g`
zq6Wfey{}v+P#45Omg4UHV;=Y%u2$ubzoO}&$F4q0r0sfXXRejJURN<8Ks(3Kv#v&&
ztsn`_CJl*7h@j5*5Nqa~Ph_~1zdMIqb~hsXNNx;vY4Eb9Tt!O;S!(NE4$i7F!FkM&
z$@6IC+iCQr&yrqJ@1Y;_w<kWR`n+ZQi}pnC&xn0Y%hS8cgZcYn8f~eie!pd=iBf){
zF#GmQrtkL+ly+a%jA99~;Dikw>+^h|sR@^~j<l#2vsLu32(fpSou9erBeL8g-RH+V
zHwl{@bTJk41Dgi@zQ)j{7vtwT810~{M_{|6LKE%TU4p-}#v4$YutI3mT{Z->Outez
zr*y#$8<-f@%Ev5VE&K)#mp2856Oex!S?9(MCxc)+SpMpInVi+*FBBCSm7^ZGEiPQ-
zgo*w4Jrsj*9|)t~iy9e<O;Fr1<76wk-mO0lbaK?L07-99groEZRHKMBem^=^k00R%
zJ=EmE=g25jrNKo}fxpdO5{kTm8k8SEi2)bJj8-R+bE?ZV3+sezA8O2q02sT9gWEY8
z;$SSS=~MwtB(Ka+CG#{FT@mg(R?0o@>D5LGzFrDmH8|f@e`oJ%t*;kQUoKeW(eFZ%
z2{1Q&MWrdMN&zI;<0-SueWo~iVeKPD7Y!YAFT|HNsNB)5D~}TNZjdsNMm+aU?Oh{X
z@z-F(fw}`h->v>$^hLtopI^C={OX%lbLwXu7n_{gXQ-|HbNjnVXu<P<!d)tZpM$TH
z;`8Nu`VK=$Y;SsS{Z!)I_b;i;$6>0Z#(-vQN0qk!k9>b~;zG<CF)61GwrJ?8lcOKs
zh@ZJd5`nt89F)Z4KTfch9)Tnm4>zA=Uzv+Eo@=rIL<?T+4Ujj;|AKrL+Ns)_G4Hz;
zfXZh3U&cz_T3&j|HX?g6c?;3QRYi;1nqjw@)R}eaZ37=<h){LQlOi;&{za39Vu*%q
z$Y6jPT0Hr_2BMBc{fba@3RE%Pk9-3H(W0tS4^3CN4+%s?<$VfXkuIJ>Lz-XOcVAOI
zX7dKv_0$n9?h&amqfMg`P)Q>&i;kY*0J+hyL~?0N?A+#&#;=8()_~(7&!|Vim|gW8
zHr}#*@+(435{blk`I&%-{ylFnDA(`C$3$qm<TC71jRgSqPhc6AKPF;rf3zaRhV?zP
zbu^6{r9e&Iad*#2zJ(>T+pkQ&QFgPV7RoW6Kl|8j{^)VR%I`nkI?jI!GIr=re-t&<
z9W>Dzx9XwEO7|5E^PjB}fN5{35>;d+H%LZIlw9<?%*XS>{TiZQ_M5VXQ}R%havon1
zaOtJWL{5ON_SwUJBxufNdA@eixfH{oOS}-L&KR!JC@%U~^h{$e1)IPj`vgplW^#Ag
zuuZ)8!s=H3cVfq=QmT?dT{K~fXkh9@iXcOvl9YA$w0rR#E6`mwyJj9(c|8Qxhz50v
zo^f6_a7EQ>U|3v&kP^9KnvQy842ck}r`?uu3}&|Oe|wiu5IR-%0KLyB1X1z-g%XB@
zlWXg~i#w!xpx8LBUJ*dh5doMjpu@n|vOnL?3I=_mZThN4c5RB-;r4EwjNLg{HcrGB
zR0+Dw$Ugk)H7m^Zi|7JEIr0tuq$o7VoU$tuNLTF}Co^K8#vsofGwYroJ=LL@w^Pif
z+GY7oI?|PGPYhfNG1Kjud*oD~v~A+NVlo`pX+}J^3w{0?j=(Dz_=h3_nSAa7p655)
zfv(gxVa!X2tV%G|jvAE>(8WQm4K95(L3!ZVZA_-o#zS1sz<;toI#qT944&=ZNWr`B
zpA5M9)7_ZGsV#doDPHI14yIg=zzI$svHghaa`aS8i2xj1wS&B?B@ZfHhtEf~AbROQ
z;4vN=TjB<K^=022WzYIRm$4TFnpBWzA>tFH4s^$jKVpF*q9z4<a&7`^qnm5b$j%Z7
zai<%J)ZH#y4pci9&1*a$JA9|xTgRfg*E?XT>{1RV=GuUkr(G}z>^_4!k$1TP8#*Hu
zoqgs*%{3$g=`8yRMfC4G67FKR+~o;c#$bC&(f&zv1~+&UU0_XK-1<PPL%piuA;I*A
z0bA(&NSCKO;4I8)RrMKoYr~L!PlS~B-pAc{|HGb#iV}sXnn*Q{xKoXrNfL&^!_i&o
zGsLRGX_=f8S5{%qqoxP7-bd=T{W@M}1^*O5do<>?f;(G&Y4YXG9otGrTd9g*_!ssO
z=IB=((W%m{C&7s)sdugn@UhZ$vkj}ej}1SLy*mKTj^!uP5PEU+8_?O7dTb5<{#;~v
zq%Xe>le8S@L`J(TjPC8Z)RZGb9@G8=%;%RR#CffWKmL32(0Q}Yk6%*;lU}cn^2;KB
z;>#)4l=#Qb{eAV!uH<_jnjr?!fA!fEYO*)cyyTcuv{Fd%a(#I?Me_R?YA)Rp#r;i;
zwZ7N4sdfjQ8*DpfFm~Ls6|t<EnQ2H?-u>j&FSd-2Hc*;JluuQH3S|k_i+83MLpHxC
zBbu=zo+hz}HCks~?IwD8%q429ivYwT*FA#uxJB)Ig&-R!Qorf$MWxtixZL)A`%(GK
z{j~TYRtAz|{Fh<CVn6u3Wv2i!!1e!eavOvlU4`cvEGpY+S9C?M?@f~FA;16*gA5mW
z9UZ<O*UzYhuX|qah1B0dewTD`dCWA=<;mW@Q@}eJ)MmW2F3&!0QI}x4yMq5Fd4Ewh
zHLNVbS;@R@>3Uk0fxAb5!k@bSfQ{2PeTpNU?3CO%8@&B>X(kxldX&&@!*2R3+drqa
zDq<UTp?uTRa9i6NL4L{C{*pG9g##*W0ynIj5w#c_zuhX~rYE)3$o3a9_NQA*SLUz3
zwtGdg_mVfH=;G+`-TU{)H0J+G0@kedbY707XYQ-MAn+s(zwn71`*gVngzh~w6pq=t
z)6k8bhVI<rXPb~KKD%9LCC~cD$h-=$I3;h_MK(Q+y)Bl`c()4`e8^Ad)<1CwIJgbv
zIs*4KUqM$Fn`fPoR5t6B-yO=`042({Izw}97q$8RXq6O8`7Y87;#3kQ4;<}5k71S}
zduK5P%eTV@!cv}9t`>C?osp4Vfj1hih9aJDZ{JRrH?ZuvE=CInq0$}P-W-Qa>t%bt
zb4Jdw?T1Y`b5QK&aixHOOshl$WVrY4Frt7(S$%uUQ1zhBx1(UkhjxdFZd`z8pH7b4
z6%ps5KtkBFif0+FKhz+51B=mjS`6C1cBw%{1c#Z7$x^wXt&ljWem;(q$zsQe4qa4u
z15cq1of$n2tDtUyCc{H`Nb}qAt)_k%D}&wPg$7K#1sA&yZEhu?^&;P9tAp7CS&-#Q
zYn@wUi7Y+M^*%iYwUvn(*hg#lTnukb|Ba0m^u4q&Tez#voapV}S`?F->jLI#C(IS|
z7&a}ubP-2wD^y<0s0GXYx?J~sOwT?+U2<$_=`s^h?w3moj%GK;*X1oczA8Drj-D|6
z*PCU#r;}e0sJj>IPJNF}_=!3fFlCNSkp!40o}|*za}4y60Vc%3h`77M{8f2XCd6vf
z%QaOC>;yJ$P&eCpY)|m4Bnlq=DXXa^ZRK9!_~`{fC3UTv^ea3xYjZ%Blxm$FQ{1|j
z9hv^Fp5%Fv${%w(N$@kJ43vbS@-YB2S}jX})hLBs4SoQPgBsc1d<R4U>9oB(Lp>YT
zH-}&<Kk$h&+8_lg@5ohZLkC-UP5{t(%Alp9k}1J$SdYH^-mr}(pQ_nNqiO4VCGSJ7
z`(%oO>WhmO^&4rWwql%(;yOJ2aK+vYkNk^<Tg7Gh@gLhk18$LB@;oz+c2uZxp9^lF
zFx<f;GJVVqT}}e|-y9|#Aj)GS72zw(sPA$T$#oPx#V0l2;yf0^#%1m7Z?eTOj757S
z=Avi**2bs702IeWx{tNltj}Edik0F~or9iC(eR&-Fy;$uLyU`lH9MD(89%lzBl)0k
z7gPL+!uDq4KF>XwQ2#gjNbS_Z*iR)jrydbq|2P~Q)DPMn&~La&nEWm2G`hYPbsR^C
zqb^>xLB9fRx3f#^UyM37|IUk<pZG)0+~>aN0*-NLIZ6TQJ+R}R3YfcG+dALoakgk4
zU7;`7HR%O}>m&JJuy#<DHyw~9cA+U<$R>PcqG&<TV^hjZkjr2<lB;A>ASU1ppp?e;
zH!rRo)0GNQVW(2l2c7iDWE*$+tC~@rM}PY8b48#Uj;mfk=df(Rz)zs;A(>qng(zoa
zJR00DW9Nwe8xnVccexHK7m)EPjpDy%l=qGl8G6JzfcL0PTWF3pZLy#3k0lV(<=wfq
zVzswJCe)EWH3)Q1bXOwv&PV;TV6TqtjwSbj^8P!{aiue)Ma%xcY-c%fWF4oefQ2Rg
z=mm$bUUJi(?y^N7@zkg!OaY5lzWE9m88FviU5G9_ThWzk0s9U3rg>r`OI+qGB#KVs
ztHxLuFJ>0fbYf)cclw@%w(Y>2(ZfGtUj1o(S8|a1@uoEN@#CD222Q^XSk_dDF2l!c
zj|<<mHK&i-qPe!)c=3C!_Ya?%0F(75%X<VlDYO`I2FZU1Ky%PsI!2!uJqZW+oZ5Vc
zn|CgmpJQ>GL=cn9E0^!oxL_m+A}2q_`O^Jjffj2^bvSsW`iHTs+BI{{Dna5?B_sC6
zjKo8iBw)LqbJ=2~?<DqD=S+<5^UAiN`=iyKzZFTsE#`7ZO;9DTHWHIcHYTB(&U-Hr
z6^<>?0SaQ7$l$~r8SnE(tV93`3~C&r5LK}bt(gY=@!ncpRk-WDE7RbECs)S3NBP%V
z1RwIt;Dkr{rb_!L1NTTB){)}Q2>a!yfUyyEcFdXunVPL+XuU<iA6=?Yq|jACl;a-P
zD5A>j^8cJ|ZD@3)Zy%(8<c2HRAKd-Ijf><861RCvWsJs55qpBcMjF4_n%KLFyGJ{>
zI$zn0HURRj{^Vv)oT3HCde_*SjQS?QISi173M3_S3BTB4590(57A_BhQp8x!P~WX?
z2u>y5UyPyKIqEMjv4!Gkr#^x9ww^!AuXn6kI{dRd`jdIfsq)WB?CoZcztbf5w;mzy
z@GBNnRJc69WCPDX#IvvZep9km?j-bkMS^DQ)vpiwU&*q|NCNH>4%ppMQkE;Oy=6LY
zrfMAE`Lc^+cJf!Jm@hc@kC5#<7gI#le<>i!*_2M8MD;gOsfpZSym;jZy4g~33N=!=
z3F<NOYj4{^mH2#dxN%RO%2jYL0$oS(kv;CQcj8zsy!L7+(hZz$i`91v)@aWngVfg+
zWgqzcDahXnZCpN_gz6{*fuS&{+!rwY+%hy2m6~vLy+q*uG_QzNo%jx|djz15(&6-e
zljuFtKrd-Y%0Fpv?R5T|yahjlp|-)QRF*bSozZ6~nckR|1tmWQo=u7OG2&Ji%ejkO
zIeX?1IBe}#-rEP<%nJ~Xgup%c9o&Vr=w8==6Utq!t1y0^L^?FG=4EvZ>Ywy-dgBTC
z9B5^csA2WO2f_r5<F5~qr%DPw|6a|mPGy;i;Mnv#a|8tqb_c-J%C&OhWyCVEmbpJM
z(3X2j1yLfxVhVr_bI%XXt=yZkqc~Dr)(~B~<;$7?O=QVLUNB|b^U<=a!&&?-O^s|W
z==lRdyGe9)vioKSwNx3N0`+`6-cRup*L>4=!5mHrBX3(w)#uLg<G!N%@5tDR$yk2V
z5e^SJqA6HuCUT<dgJl}jWE7Y_A+?4y5DBLgfB|#o8CWR++KE}<C(=MWM!vWV$OZ*#
z|5qlppqa$W7ogaE3MxZEJaPP7#4j{qCl6DJ3>g48>T@kDnK4qWNEZDM9!L9^p;@7p
zEdxi-Vrc(gAn*yLJ;#YGR8hjjjl|If3TtRhU_<WVQ22-7P7uq-Mov9hsTl$$&NBJu
z<C8fdX*QJj*Cc3!c+e}8)fbTFH!kw|d8JO<eW&(Rt4g?*hdumAG5-a!Gj=B=2~sq$
zPl+>h!Aie&rJQ75@&{ku?1gl0B%dhcKiWF}^f=g41_$$(5o5tfjOTYgjz4W%^1^!m
zU6<4Pr9Q{HH($Z0{=RNkD_9(z6=FlM++xbxaVbhGj^2&w+E{6pxpIBPvGMek-nOM-
z_Vfxrm7Dn1Sxqv(cb|iQdTtY&OMgz4Gmed*qTH&HTT;#K2<dvfF>&r*9znv4{*C<9
zEz%2#v@^A8-OiT$K|-Ai;u2Kw{HCN{4hb>r4l~{<zdvXD&@~a%xfh0`SJ!z$J57;c
z8E+t1&k@SyRqzv~p$XYz9gwIHlyO(H7s6$1h6spcO&j>4PG0kdh*w2Kq3p?cqG2c=
z{+|t!jPj38n-AFCH-CWDLr?4<ost+NOOE_O(Ep-&-y2DWfpgJk@XwJ_d=R?{tGJX(
zyP?mB5tJc8uE1AZ;)FFP+#f%qfh$Q_B3_QNKSN-FG%Bw9nU6GLBMc{U+#G9Y`sMA7
z^qf@m_!4(A5ppthajKC7rem#fC0}aHcm(febkyIUrE5ZWL>y8%+HUI@TG|Z`r{huB
zP4Tg7^^A05Q?f%3vnY#+Zc;}@PSwAmqz_l4d)R3|&WhKMtUqM&)0^Z+thJG!{mEMU
zEWIJEXis@SWWUu^Ld(|*0u^7FrL;}_h}wTSzlZ>6b+f+CN2ljaNVrx`N%3}kAvv%W
z?_JfpFtHYhSujVQp>FZTx*_YthEo*um%_o-+4NcS{3`Smk<o!9l2ButkCkz?ILYB{
zcgI!{bV<1J^jXcr<wPV2Z#-B)2kk`Aghix@TS^nMS1I219-|w4Jk^~E<1x?Bg$=}x
zv+~+y{q>#&Qg-wQzm+#IZsG6KhqAe3L97l^fkVX~s@yeiToi)7(niqkCjzCNGaoGH
zbZY7~@_YH?UmD7V;D{PSbcm+Gevp2Yxi?gxg;o2B$PAw`TXYE}4=PU*3iSECyOh%k
zfm$@5Pn1>~5oGD?UW$hpH<iE!{a&eE?cy4vo9==0P(6pUU(;GCyTg=#Mkm4fO>xRc
zX(1Zk3T~AK8z<>bXE)=Gs&8fh+b!?=+iu<AO;07GOt%Yl!yWL9=C!hr?6Me{X2&Hs
z-Cy)-T>_i8EJ>_geqna$>+|FP4^{60)ztaNjn`63TamaC1)@bo1VNy(NmQ!TiAoib
z5oH)HB87;ogsVk}42c5;WJIKjmL*$eRQ8ODvLTQF0RovoZb)+P|H1b6zVH7$=S0Qh
zIiB3-`97cZ{oXB_dqY&bZS-Bur`E3*>}w!qB^!m+Cmm0H8bf0`hA-ItDzM%vyMDT^
zfT?F|`wvs&U5c$Hxv4#UXd9*QLR7Y6r^QdkTIXg?f&UIN=YLwilLBkJ<<J8=83CfZ
zTpFE2q`gQ#ceSp^!Mm?xBVbP-bB`+OopZ#VbS#c+4x8apvqzJ%ZitEGfY#ACFu$n;
z?y-iYHMVHxHDylyS5oDMdbBg#gTm1>F*WD{G9YJq_<`u7o6uUy=&-gy&&K>Kum^L2
z+_a>+_M8WNh&TK;trs_yiG!~8oN`nCcjo?8q=2FUW?j9;L4`eNtwX6pNIz1y_hDW&
zXPhQzh>fh#y)nDLA9{@^K#MyS3h^uEM0*?+Olu@&ADrrdLJFr^fyCyOEZF?ox7IM>
zae+w^3yfQAA|9o^V784oqGF?KUPe8yr|O!RqKCXJs|Zt$!&NQu`mT@3oKgk-27;N7
zs`qrZn_mqDoD{KnzS;^P|9#6iu?x|<dCMTA=)|uS*$B`1W$5YN$*w~C^}ZRFVW9a}
z!?TFZ3oeQ^xD-}-?Lj1E=I`}!0MQlg{@#YHu|3sDO>Fgo{711}yV37t6Wt<V%cS+(
zDUR_qcn0>JDtUSK+SM%Kyd)haQX)K^xmQa(w_)%V-OC7Zm-6PA=8K*;#`G2_J*;-$
z#Ss83*ta1NyXEh6uqFJirg$jq`z3;mD~*G4@~p3cs`udh(<O@-kR)>lpo#NnrE5#0
z*`VhMLbDEmjwTO-KR)hts*)EB3{K;|iV^M4KmJFM2DHA0O#5w~F06vT_BGZru}htg
z!^(2r)2zL}Rxz>VrSm%hfY1V+0p$jxNBFCoBG5p&(OcWUO8$uaS`n$e(46+d{6CfM
zB(9tpBrW05@+ppfJ+pJqNwYV!iJ(UPdSU}Y35)gw$?rc-?M@|m^u7O)mxc*7UGTiO
zp0_ldkvLTENH8FJwY^x3yVWe}+m>xVKQ1m2b!7!MN!Bw8?@tkPn2q;WBDwfqH7Ic3
zx}3)>O{j5$9>MmL0|+f`Ux@6|8i@oisWoU`HMBDM>IQ1w&pS;kG!BFv&3l%<f|Bg^
zQ--b0Zfg!NJF6*}^Ko=EP8Cb{xHdWxw|AThpU`o{qRSWZf5cPZv9)+Rt3e@C+}A{(
zZT>l6Q}&yy+QrCSbQR&xK8=U?f7DCoW=@v%40oKxE2eW;a9f*9(C~DoRJ|}qHN0r?
zeBx|ru+k#k!%`Mw#YBtq!5b$*zbc4p!v8ka%+?Od1!YL`(I><w&P#Q$@QY8k*V4JE
zbaY#B-*;Q!5P;(Y7GE=@U&F(pqR<QmH_SIbokE)FWb|x$d9Z}%_~^|ru?iXHr~bdk
z3gj{hh!AQfO|3$!XLZ}Lm%nxyuRPYzrMd21kT5h*h12{K=ZwARrSKd$DB%+~oS>v|
zNyF%5UGD4?byh~o)dd>=;ETJHHqG^t_DA%Mo>sXGf2;#D8}xAYoKWZ+tD`Tg=jytD
zzQJY&3{mMBMl+g8I_t}EeI;G=N}+@&dr0Qv-xsXUe$y{r;{bV1I(n|vshVyxQjD=X
zC2Mi6P4zZX%sfx`f?&CR!p~35UPPEiz1n#Gpl;XM&p(CUF-oL}XG%}hnkzh`_4dK%
z%fvI@6WQsYF~j{=+It%gk1o=?0n3V`a*P++pPK*+_n96hlI1|F3;CS67`J>y6rKdF
z!KDUPSkjRc6QTHr^2f(A9`-(_!Q-wvi&r;6g_w_CiPc)Ack!%0%(QcEnUfcYWKr9}
zw4De}(WtbA*}H*t^{)aEPVO9#5tp7%e)z3!H{VMj<ll2X+2YFqFz{m_;(ZP0{HfH#
zvuHC2!sPTU#hexf*%<*#$ehM|x>tHZ0FNz+N4)b(y`}*Ag$_8*0_yayS=`+SrZ>p=
z0%$>k0zD89g+LJU!R*3pR|YznQG%LiJW;t%>da?q{~@!f{(UV-?%ly0878iiEUu>d
z^Zcy<&XU;4sV&hbQW<3`xMQlRAbXmmMvJ{=yu+@#(S4ZQ3CmbiY=%-AuMtrTf{dP7
zjTno8>D9Hp7sdJ;tmujoBDrklXG%aFUhb^A*tiafzvbw0%n#|qg}AaYI`FzxCmOms
z_rJ)gDR>*z=6q3@Zb)wUv>^`XYF^wdx|$7a7t7BY|1i7bG)Lp;&~Dei3Dtq0JKZKF
zZ4;~P4mId{Z;l9P4m<bd5}u>OZVJ4j7){DHheVvxMdo{F8n8wJ?LH@2HUR1hJ1oA`
zdT(*4S{I7AcEhi*%W(j{T4LrX`nl}kTwgSB6(bgbx<VIq?M6=hZTz8|al~!l-xB!m
zBjR1=<t)_B<0-z^d~*@-q&AhIugpOy%&Pz&U%oA0YGV4&zJJpZ)!nxACtoRDfGGT=
zta9do+4O#|udHcysW-+P(;gZ~XV_iwI850w_4VgqLW2eX7L?xV=ahaVX}CuIx6;x0
ziZ?0Q;|^<ESeZm2mHEJX&b!K*B<YHeHDkYSz{w{F3)-MJCQQ|^lFfD2GAUkZws|+<
z?-~r=&h>Y~08CVPcD1bZaqTGH+LWX*aJ|%fM(Y&Ce7c}x=@dL{k{D~)hAt|Kt$x?|
ziL@Z%f?SN=F=fw@o5Kdv^__8Qc+y*&7uBBnhb&G+?cU)YFv)&cd~`$Yo3-3Yj}vND
zvTjYA04+(wqbzxE!#~)lGe&C(S|H1`;0*`b_OAFITNsZWJryvunqVF90aR9XysI@8
z301J)Cd1zIQ}DAGN23f~im7rc_&O`fu`!s<vDtxh{Z(%v9xuw-t+YkK02e^!gyT~;
zHOg*yK%j?el=n+5_Z^X-9tY)BJ$X6wZ<_Q5(kBaEawhzH-tzG?tMYw0kE&+T2Ln#P
zd#Wt|{h{70GzHEG=q&@<onPm^|LnxCol9RoLc<q&nRuy$1sE)Ub>7O4z?};3{vC|P
z7-nKY#B{(;v|pxZs8Rkbfg~IX$Ab7w!gU-2jl6G`vRUy@NOPi>Jengs7Lq*n18q@3
zO*6JBoI&6ByeB@KxghWyjbpueU*L!ZE^J61EFEoGd7MlJU4CVE@2<oR2<Me8|9H}y
z@J+5ty@-H>Rlyt$T`jBiRf=9PA|-HLoS!JiqQYwl>_-()W<Njmu=$|z&8}#p+Mjox
z)XOj&GW<4$+)|W+yKW>jioFW#Wq4<Z-f`PsZ#3fRFJBWqWKs4G-7R{wO{x@)51RDj
zg}qx%E2g@lpx41<+$dAjkxb6GjXhDA5yi|nKCc06Yf-~L;~Yi1B7yI&RT!TsDl#?>
zARZI{af-aVsuNnYzaHGtuK4uu-8!duxv&omuvs_^w_qM}-y3(c2~030_&<NXV{Zr4
zq1oY1S^^Z-0zLo~j=n8j-6$WVIfIo!>DN2n=={fnW{@nqFTgxxPJj6oq?ZP`m@hs+
zeJH&yCD0(mN9WYq@c1FfqbF&}_y`9l%ZjulH2Bi-Qdw6-k=^*$2#i&xT^4T?SRa2$
ze}-py6H>-3)DrthTYESl1ATCXsAC6#g27*3`2GnC17sw358%6LJj^(8qRT<Yy8+)@
z#atD4CgQ60_-9yVYm8`F(Dg<&Bx#a*(0=TxHKRRNjF1$g-5E^|39$!_b$-fP)UZ#D
zCv0N{?%>jCxFMh&P^afteULsF*(-GU&T6CW^zAdh-RL{3@JO86)*diY?C236V~miq
z)M$g>w6w+GSwiKSl{SmlN2B#E$bli=<76Fra`ONI78?XdtYuhhvsXlKI8Mgh54j%L
z80U(Fc)g&*ToG3~agnIFyIm{b#nslSdcCzxr_z~l3lnZ^Us$3~Rit;({-qzN!HvYO
z<V#9D96^-@GCAp%Aq&@?5JPhEZ%SZAd8E<*P1Mf<y+QErkdnU^@Jr?G?!t+@wM&i@
zmk+YnK<;Ue?FkK_;|rvbq5F4hNo2rlqWz^+LdVZ=jpec~zldHY{hEB5f>d^U(iu9a
z0^%17zsj*+KS&-xoxpTUsNCBcGR4Q?ck_R;BuT!U!%V_;N`4O#IjcZqlO8*(`t$K7
zl}Ph+(u#1C(#9{Oi+-XDmq5X9(f-Zg#<~n#MZmonR<dw1k8^K@Rm!=GxBbym$}a6^
zHa4b~jpseQUz%vp2Tv*M^lWI1YNniCH+SIm`_I-NwCX|?e@@m}G){kl<JvYnr=J_^
zyTV%g$4$ZVb9(A$8^G(O)W{qBHXOLs7yNB(W{@WjJ2Q09srvfp7p|Vp&t4b@-UWJI
z^?SNF1%qQn)(92vLmm(|k~iUT)%qGamLauwRcK>3OV;9LeYD*!3yUERCPZajHpf{R
zxB7&r*9y)o#=6wrr3*q8^W2bzpRh~;PGPUuh|Fb$uff@DD~NF%20}!r(4bKZ?*jVZ
z3P(vgP%Vf}uy_KI>=I}ws{UyEOsB)9^lR#6QxT8$Xd%q*<zeaZB5<IVu-d>T&)UoU
z-v8rT$g2IXCG&RHQi!vBygzA@&zLCzxT4nopO*i@_Dx0Lxb=haGXI^dDzuhzEqK^r
zR?_4*OCDkpW?jBiGf&aaLNeP~nL{;b;2fK*M(%e=2#kQbT^^xP;9Jo(pPOX}6F}l5
z7}cH`B6{7W=LU3(4WdKF7I!kdM~T^YN2Yyiyicq7ao%#}+dwnKfiBt>RwI+4m~3h2
zbmZ;po06<>9e^<lM&)ElR-1*QaX<Ba0n+ecL-xi$S!=(AJw8@{v+2UYdv%8A%8HY(
zHn+*Z8#x@yw>4c1Iy}c5;wtIV8IN+;iCrt+rGVZgEj8cJKfe{-U-iSyzyia>M$7HW
zRi<#zy6+klJ68;^$EL8&8fgdb*+9pN_Zs)+yO1Y_-V+mig{nLABMU3AvdLh=jL$-(
zUExS7oUe_Wz9$ikGL>drP4ave@@QGYHuKq_yg3spW}ESNEK_HzO#cf5j;{-jE;tr+
zaPw)Mc{8&2M+UnN{Q+vt!bu%BKY=e;HYureq%dZN))h$cu!gAgG?V>kwsB!Qs#Ulq
zeV>fwt~&V)c2cC*ev6(*P9e9BCqg4z3pfX54l=5=L|mXGkLb1HgWWeC-QC7$V_0F~
zpRRs8jeJz9Y`$3u1))z0pUIu2Og?LfA^LHMVZ2qmi!hBSi=_gYag;3yjpCj{eAUNl
z&ZAAm(U)q;(f;kXdq`f#+(lE)!(%=x*-obBqVkETN&6k+?W4&m6&gMUT|sX-w$sfn
zp-W$YK~t%&K#h(Id3P(#A>D-g=B#KcDa}mE%-8~xFI{NI6m2Ak9_g8rK7WkG2*#3_
zxCv8LPYU70w~*C`@1K7A?%N*ff%Lmo8;(B8dtm;!Jakt#Zd|{O`qWT1S&kFUrf%n4
zh@%<eY&T**ovoQWt~36Lt0k*XqDM^g2@wGso%|DS72NN4c%^m(*Pexa;F#XZ?A636
zNoaHAh}{4~yt7AX=Fjdrlt7N(@2lV0Y9kfc2JD_9W|)qrCKq=iBh)k%bqW!d&R{B9
zow~b&?%S9UwIf1z*Eynn1g<ek(l&@+(YQ^q9jkcdg9PI(8Xp|r>tAV=qXES-`eS{-
zqxj!ys?K=~%?i%8;#;;!t6QoEiWF6f)HfH6UZo$yoAQS6>h7<K#e)C4G!2%X8blRf
zvyVTKx(>IZg+s+|9~8_r87XLcXcscBkrbMHl-$3@Hb<**l1eCAX&j_1I5*c--g%!c
zR)csEt^<`Dx($uC*JHfjX6EIWPp5jEC5P^mSvuF69c!!~`bXouh`CUpv|{Z&57<>w
z1otGhUL#mAB;7}c#k<1!N8OJ4kV7ks7IT(mo9@`4T-6$Id@gg_`kFJW-y*C>?kAZ`
z&iow|vgaxkQgnuO$7FFUMm6ZJH(S@ycKY6I&AdUaue~|?EM7dJh`|{BA{w`=qdKcK
zDqkLLx*X|+o~7u%PUa%7Y-q~Eb5(@tRen83QbX@qw#w9$f(ID2{`{t(d>EX!?!7JZ
zbzDR<cnF}?x8>{{4Jm4fD|!!Rf-JK94CEYdt+CMJCiCr|Wdrl~P%F=^Wuj&@t5o5S
z^z?ct^pO#ZSVl-=WPJHT?~B{9j4GX2MgyVqfRtgEkO*lD2sbc=XoLS^XSSjkED~O0
zoHfpZ8^r=Z$a=`(eo2sE(-cz3%al7vX?*Muqjnm(Qf;Es7cmA^aE^HnSEH6t2byTC
zWBPL7Mq0#gDA@1<`dXGO0-9C&w%T%n{hGgTGMBjFCAo1Jbw$bc{&JxvN5zV29<qu;
zzTBp|gHqx@>TmBa9;t~cNPC2;VTTX1t90f@5k2sxnB(cMoTPiXZ;Tg{jg9@9Iwwq!
zSKE5Cf2z?{TmH}AvQ_7(n_ti?F}Dj}c~38#;$|66$aGgf`YD5g+c#lDY#pd%#jkb|
zzpHcWU+7XjPM*GS#jxa+ye?%ui*oK-u7Bf}1o=Lr+wCpJwZ1AVj!A=8&`bAqs$v4~
zmJ6qXkGBK8>MGmp1j|n8U*{k8Ji&0{Sl?%E*brVx=8qPmF^;LS&H!BUJ-FCTG{v9_
zBo6A5jCayLznM{g>wQgASl`~+t#X@A2ppP+=#bZnh+DM6_~KIvD>0qz4-QEWM^Ubj
zShEb#>TU-aQI;+J-%;=v=nUbP>eZmp@a7<tue28s!hrcotwpEg1q`K9SH%CKpg}HC
z@<=Nc30JPiYmT;c5En=t`$_TP)3(q^hX8UWkDsd4m~Os#@A!b+SUC)2-icPd+Q4L9
zGTeMKS=B_n0yQI>jAeE>xOnEmjoX4$60BWCQ^Z=Hi_{T~Ey!Ju?Q1@bwqB|wxiLbV
z78(~`_|6;SByLsmY&&VQ!g%{F#yamG9IjmJzc!HTv+Em=NBRY}r1;7sQI4F<&gAbf
zetRZr({rxsm>=qleC?m((Y#Pkb=2H*TnlTh9LX51((#FNyn>Z%oiJc6)>G-qE<GK}
zD#=W?AF8XUu;?NAqU<VW1-hQKcPG8#b6<J*Ym7&;og-O3;x5vBc<*?s^28Il1_YG#
zZ9SwFSR9yuYa`u&Q|8+#l(KBJ4)umIo+{Hq5++>c)ZX>UY!riLh~v;JM-HZT07t($
zt6*hrzMiT6(P<$`d2ww6I;3v7_FhDI2gBB8@v(wAuT)X~HoX%~MbpbhG~b_y#W&E|
zj-ql!mO=%;<q@PEus{9eCY!t2<MeTQ4K$<l5-<{M9wVo*eM-rPm6dy<pJww34r#kl
z;h3UEz?l0n@q^g{p0zCOMxs43&lupgqd&83&qiETYmafH3_~w%Xp8m1`Qo>G5&ti3
zwnrwK?X7stj}uk#zH8ekpUP-fVkatb2+vtX+A)#q*eB#Yw1wKEnvlH3Lu><oJ{>1Y
z*@Af<<n!r!2l6!A|4U9p{{vrV9k}(iqsKsVdXUds2j#$>gJg?aQxkc`(S1}g<(pl%
z0~U9ldU|B|AB|^poZ6qA+3Lq-;;`SimPIKlMjR4OJQYdQF&)!?2;FLs2@l72;X@nB
zG3_x-k4e&?^5ggPfSAyDM@2<M<0CxRiK<mjDTCxAr!<mU(2iir%tKoEEn+LaE}J?3
z3g7LB9eZP)x=82XP<`J7U9Jw!t6Vr#EOCkZl{JW(!TjZ#?ZgxC6ka~Sd>@9`Kn|Xv
z6Q~&PpkSgNO)O2qOJDrwOsfpE1^qKZ%;=F3McJ@dXFghO2?m1`Mdh%(Rq5pVK01~G
z8az-bqJlq(xPvvM%mSCcvErLw-6oMfWtWY;lsUA)i0YL_1J4MtWI*ZCqtpGEpvwu2
zC(wJQg2;ky24k@FU<34`BT3n8T}RwaLWNr&R*)xX*UKiNwCHg2tsN5$Kb)fcZx%or
zv$vj0{*(bH=xRKXyei<7jLzGTnjTG^-$<{wTo5d#V=$y0xJ6=G#GMDSM0T;p;+Cd#
zj%2_=%3#}IJ*L;d%`QGB*Rb13a4s*KDQoSoH2M`!A$7=`HIU_VdOr+SoU&k^Kl{Au
zP$H*Jvfs;4>$}CRA(RV3GxY+=Mmp)0{en}^GAHUrj^qad4l4%3$G-$DzUp$&K8*`n
z)T^)<e`g~d*?&R3nomEiCUvypRzmXf(Hv#frN#T%S|(o@zR~DMgel}^+#A;^n?7if
zXn-Tt2fMTD3Toe`a@aYWLmILpR$xP0FQVXY2o28YsNhD7dKX%uBtq#71(WY3VSs}o
z2tu3yN;`vuj-6nRq2sKOyUM|rabiJT^yig6vf!`F!ldlK3kE+SCz6)4+>;=_Ah5;Y
zZfcrlf8tomMB0MCdks2t{prd~%ypGVkj{<Y6}yo@BQZnDK)ahFO3^P4r>3iP8t~oM
znW%w7Ul%G$M;8^N1X07E=;pRFig&bq?ZtdKEymHE4A0|*i}J`AV?0Or7v>fxgJRy9
zMYmR7$omy5;hryr7PlPXR(gzO%o2ty{n^Fp&84TDb%LbExT3KjBL)9skkeoGQ@v+v
zN~WlZrR57I(`<uCoX~xRU*m|}{9B>k58L-&dmJQ8!R+@r`Ook075jyc8HWy}Q#s_q
z7h4CiZ9`;t7-x+A#`6{N1+Zr<6Hdg7Mxzpqag_W>SwbDN{~q`JNaW6jztw5JDe_a@
z=48$65?ZK}j^x^C+tCW~M+*@*%zqYY9eBZ*rAPEwNFDLNP9gOH3G!?LLKHqC>f<V(
z`nRUSVx>OZzlwLMBVw@pxqNX(NoD9$vgV-PVOz&OR(?_^ONT1P(>Ft6hK%)JS*sgl
zD0puiA489!K%G{A##$a#LD$hOJILK~2A?p9Tl(43!ff6nC^JNNffa^Be|doeo?nib
z<}8w(Sja#nHUoRnZv44vy-wj&WMV^F1=Q#kDN7GbSHXMnEXf}LCVfb7VLNR?M}1bd
zX8k>zCLJ+@TBTH+t8{?U7!a~Gz?~{Tjf(4dVO;Bmv;|e;U$`ffW>m~CqA4ecJrJD~
zAe#*yiicKI2>qA2?)yD9wy5T{`;%SY>%f$a`Sdk9jV9K2D!tvKm=8>JzIkrh?qnV$
z8?B(yMvo~fjChaP>80;(Z;U)kUKwo?l4v4&W#67YAJb+@y01!;YXtMc9v)AxAYYq;
z*mZg?4!9txR|VUK_NnF+(a29Um9-ngaYEzcta$A68Fmo79e!#?Uc`<%D%)(@3L1$+
z_vG$vLQK`*-XNtki-}hX7Tsn$Z-8qi;E*k`CtENH6@VAD1tA}9BA0ec@G`oBIZBiO
z1$B?ndMKg>%G4K_PI5mgj}E&`KoanP6a^6r{c@eK7IXrN*(_0lrdLCKWF6FozaeOG
zxhZ^A%y&ugkvYpW%ug%E>lH_)mus&Fm>fES+!~4~N4eBZAL6ZwahwG1w1*-9(>kYc
z--pPvj{SL->{AzO7&1jl5GUIdg1UAITP;Q{6lVeKv<Ys#gw#9S^!+<A*`SZ6^_%>m
zLrQ!q`a$(wakn1MtC@7yb^bu>i~Z>sTXBJ@wg#)TvfOWa;QWYB?YJgRvF7WeZa(c$
z!nphU`|@S#TcFpYjaRaMyAveacS;A_w41VT;6v*E)6V&l#K>cfA#dYFOtiUf6GD_9
zQ=Ca4ji&~t6c}Z@^~}CO$L_%cC2jqhD1N|!>}>$$)XvgN;{=OZ_&TDRRde=rjO+Xa
za=%VB;x}z8sTx5hEa0udrtpwQ9pv$!IU8`Dr*J*0cu9&%Ou<!={!rv%wl{w_+2^Eb
za6>lMx-Od@AaY>^v#S<_H10zLnuiEJORNWvCZWdQt8S+O49%Jc6-~5PN)s3Sze}mS
z05BX;#?8=>*&PI>&BA;{y32KbR<0pfl?Zn_*P?%3`$~<Pm|SccPtXp9OfZM>vSE1!
zy1#ojsuI0Wdd8US=}2yhN1I7Tih6rMg)|rqZ*19Z<p~m{QMHqiAAFA|WxyvF@2liv
z$FI3-oURSu&qs%F(}!)7A318{xHna>vu5PRl(sj4<<n91R4QiH3}q&Z%clpBSz-n!
z_qei{5V}*-3C#^^2p;?-x;y_fRXh+Ag>1Ve*mP3-&$`C5<DIJS`_mbnK58jDwKT~i
zw_e(_oiauZx~Pq|D~C*c`VZw_j}~N%Wbzd)JaXte(5J~`Td^Kg2rIIK(AL{DtCGxc
zm@iYK%gAn+qHR8>59F`5`Vh-#{WH%A<$?EWrR9+Sbg|h^0c?pCeS(heCaYJ5w3gh>
zjS++~egfA|9oo+YS^&(kJ;_8qz8`gBd;(yuDqq5As^BSR&EHiRAnQ>vgOG%7fMW5N
zpTe$+y@J<<$YUrGdh~UzUQ2fVgS;j97pVSFx7J@KA)>9-r1YTBbpLQKo^tJx-k0Ft
z>_#>~ap0l4ih^nD<eDyClf-(0H_jJg7w+ix-He^U7`HEX%dss>Z&U0=-mNg%(WhEa
zwDBP|uz|RnZPjG1=-f{Hi2jV#xnsS}e11JtCKxr-2)oUEQct3krGQhPsz_|cwK3du
zB@Nm6iVA4FJwPbiv!j-zuPsUOch}Gg7|&#derDW#@hxNJ^>1EnNraBx62CO;xr%3}
z+3*r<nAClY@AWZ$KJXSYyeT>`nM1|l+7EvJcUsVJJ=Z~7>pjp?_7UM`Z}nfsi@)!`
z7bLsvL`*9^Mfbf(9>Uua;nu!rR`-b^J108)*)NjTZbt4o^dnlIou3AF%8uHgkgel+
zuoJk<C4;DguqTMAWN5G`9YGbuZa+!_*eOaX?J-8qAVPL16A$$f7J6iDXghqVz0N3?
z88F?z;3;wx983FxF{lB1qA#;bOt0fUO#(>3SHfTnyH=sI5s}iB4fHzd21uQ8*!I@7
z8b}{>T%;ghlI!lwTb-x@T&NsKJMT1>Dq1U}mC?Ss6;G?4A~5kS*i2?S<G%DN5{kiE
z^&0*?yNI+PDd=fpnu(}<`i$Zs^jP=?nr{AJRG;}f?zyu23Nw@lZ*GffGC9J%gAb^`
zFgz1H+g_?pD;gO$li-oPnu{aLdV;#5>$7WHGl5`(xWfGVp=Rhk)@kyQ5O>a1{~>2(
zXC~?z)1cgToV>n1>;?OUCpYp1UDX8pOHc@WZ_6I_cQ%D-7V@`5of3AP;7!p^<yJLL
zyNgJZnpRxC5R^x0rBZ{o&~V;AHw7Figo#PrluGoK2KQX2Qg?-?-JJbgUxC(yDZg@w
zJyzc1XnF6Op~^fz1V>{~JH;!=`)wQIk`24gYk*zTxK+r^V8FsXe2+;+6oy6QWW@=)
zpoL(V(a9>Me?-Tj{CReY2~ypJa-leU1B+cy`?sP1e61Yo&d1P<Ns)rrZ#0)OP8ir8
zktJqIr1-SW%~HqS6}aV;M^Mt&wfX43*L#5^7pOA+p59U#k94tp5-HSdI>a2rkH}{f
zsoZbu-czYIZ+DX$nQT0(B7cDvKQF7RBQv1K)QBtWE)3>*_$oT~*=y`?DZ+{ki|LVS
zcLGAMIFhG_p=4FM6uUUILQDj?%IfB5{|MCZG2HB>g=?uOX`m9UJ`XX(dN@hq=$o*B
zj<t~cTCaQ}+e~9{Jw0M9?y!8IWjSOcgl>+=PrmqT;|;$=Cztr+p15rc%#~LAkIgrm
zMDv*+jOnEE{5GwLKMWUQvbFl1ki=}ai6i-`Yh(`zWFOCM$hjz4XL?YNJ@%LJ7<`Wd
z-Ezb}zrlJtsash9x89;x{%tFo&Gv2&HK<hz(?^*#yTFNK8>sZHWd(368e1IGtyY<S
z1Q1g*GLiEn5kFp|imf=gL|J%>acIH&1U)1mVSug>=tyGmRR`BW7`x<wL}<p!@X1Hq
z+~(=a6TaP}#$U115`Nl+vVDH`&ObP0=}8wgY!nYmG0jo`QO#^{CiD?4lYoNt;9P{E
z2lHK!ityWFfjwZbi}6yZ8P0_{L(LCtKDngVx=OK)PrsltcX47<e@o_8?$%)O$!!1p
z2Asq{!cfI3sS^IYbpk$xK8~-#$qgIUVAbxdsbMN;5iK&Y0X8(j#gv%uC*I?Tlk4B-
zAr!>5XS0a|&ia-7)bMV(P2F`RpZcIyrdB?&IY!hiN?)%bOHU@A%e6mvQv7!3R%|q8
znaAP|`Vp$CCwZ59UqYJ~T2EEAzwhj#xVb!KExpWi#4CZbmL4$xKm-4lSO?eMKK@M$
zZ!iYKax=3nu_Ln!Os`g9xI0%=u;cV_%pY-!4FmsZtZ{U=KJlY|WFVg&WT}d~%+4Xe
zC2PHf0Ue*6$pi8G6Zx7ce6I%Q4T}zq09gtt-?2(>LrO&#>Lq6))(}8f{vbm9ezT_8
z2Wmxq7EDkKagNEFE}C^cZbe(6Hna-8j@s}$hZ##~cM>SZS#niPFCcso!t)1}k~c?#
z&|!`WQRUfOx{B5Fc|MEU@lx+<wmeuW;IkM^<!1W)M>*D%{Gwce<w!djrrmNX>o`-&
zxR8rz)J{RKY=}7?ufdq-#cN#CYjn=saRNWAJ~(a76FD9IXC;R`2MJ?K*$@5qG*Ycf
z9{HrMIUPM4!$hvww^l|8Y~fC!OO`F*ly35DWQwFtZa!m98hO5(byjKRX~Iy+R9pZ2
zv`bR!0nyI&50XsQ@1n<^i5~Z1L!|pzCh)3s#{rXxUJUNgL08?U35NY*eHxCJ7;m53
zWR-hHr|HTPlxp!gK4Kql4<x!^RzAn{qTJq<{Q7fGtl-HMwJFqf-j5RBF<+N$E86`C
z54fsX60WB`Ee0>e2sCSXt9t^y7$532Xt;nKxV1CH<3xBeFx?T1bk0niq%S5pqya=l
z>->Clap`eXX^a|90!j~92XnPve0Bp%jPbj5dg&w>qPW1T%sZjxegZ<8ru<*7A>DcU
zD|-iHzbl~|=nNo~hbNqd@Kw+sD-urwJ)b^=r`fz9)`#zy<<qMK^~9G+_)(;zSQ1*h
z3GK~cH4TR$2+x=(EwI8~6@A3|KA2-9RCzQylS$+wdz{G=$>`}|Tv5Y%s>vO5>1F8;
zxN01CYI7NXMGVuH*SeNqE7i?cqA`^t$Ku!1&3z+P1KsCBQ3rE49*iBj8s}fJ()~`E
z?FD@Yt$tF1l5{-Y2BV2TVkiuLdgj#5D-V9kN_?4f4ycm2E%KdH8el8HS0tnRg`?D6
z!mYs%U|%34a8<zVtcMdf1jmbFn2bP<A}>rc%eF3?K=wYVLAMTJJK{pN6uNiSDC>+_
zxsDZUfjXCkd8W(-y;jB#%kg|xY1QKxOtf5szNTo_7M{%NP;IY(xPC)G^Pc)!uwGGt
zE?pT*clX%WD?+@4VVN9&XDJmos+B<P<^DLie;WCsES5_}9?R)1DMICH(XaR;eR66Y
z^{L+VPs$vip%m~5LyRSTavB+zLKDYbWB=SLUe5mR`L8t!NwdL6O>2F0kkkgyLbtl5
zF#x%?j?zv#Q8=T>Bn4z6R#iM@(;zef86fo!Xv%%@9h6f9QHn~7LwYFrEV(-y4H)hY
z+B){hqQ`z{{{45~Q~0b;CK2Ccslg-ENqZdK6&ALYpE3SLq-it2*qBJ0!;f3}J6O*x
zr~SfPpoF(HLaLtHxGnMDjtloWvzPh)knfJ`$9*^PF$2><y+h`n(9OcQuyx~Tf%**&
z-HH3taa^FbS^9C~XlnAes+eb(h{hrOc8Ob)X6?pSxA8uF4-re6qRx=e`27n`Vijye
zOp{jshx+VJawD7zw5*vc(@?%YA8W-&)d*^EBI688c)yK&WInGw2G3=^cL1mtJF2U(
zcvwLnhG08%2IVRoxdoCQ8Bd}79eIehfM)_u1f3HBl(c*~5wQTOV+Y`R5=Je3_g9{`
zP<>M>A;=I009#9oS0yoK7@&vINPo(Dx*$Ndhbj0&pDQhJl^mE%Ec^m>-8e>`E1bX!
z_mhxV$J7APD;td3<<8(Cwi%QG!J`3bWs*^^cBR=1TOle~xaB~mi_#b?35|~!?6x+`
zqAl{X<i&T?L9|{YGT_e-#`U&M$xJO#;*a>CJD+5H)N~JWlfe&GgP<=B5MI3ZXI}D8
zhwiWX$tflOiwXQnzGs}X?2}`v^>*x?GU^?i&mBQBCIfr*EwviRFXZOGce7#{E0uo<
zYLmG-=i*>?3w0(O25RQ~NiGKg>}8M<NN)(f`$MMs)}yw-BkU6%BTozO2rAPYtfVxp
zMYn88dA<c1CcGo^HESztsm2)7b1SZ|z{VE?E7DAOe`OSeD86oa`4^OJ21La8y)PyQ
z=<}$IVE77Lt-Fy7Al+jBU1J*Xw?_o=<AiZSEeGsimM9StgiRs=aP+*AZeCRJi7G&s
zpy_`L-}d#k2%ANw!aHP29&v)8Q28$7=$=H%W3*L|>4G%bjtSE!SXDfhYxe!e+9;Zu
z;8OD~1>!;;J4u}gYY4{K3g6r7BWYgrB2ALt1S6lIGg18)liX8u##m0goA3JtjTIp7
z7}FiKi!bP<eH*&wtDO6#!W3qI{VGt1qO?-U@@9-8=oJ1N<%+)#P1;j0;x6>t{;*E3
z0Mq!XLJ03EpZ3WJpC4WPCgeBVmLmi0Yl)r9>&o6djz2{tv;8~Dt=Ls(+k))c3;o&P
zC1m7o_bRw#YtOF?xcLQv+<4i*D`$9+L25XIB#tAGj{X>9T=3i%+~gF%P)pvegK|7B
zx2~Zt>v3==2ib7<Ea;GXOnfNo=%u(SRQR#GbayuDj-4F0TVPht2=1($R6+lofUKd*
z*^(p_-+2^l^S02NvG{PqgBr{Ux^M%8fu8Y!UFH7gi*_5sI6Spx9pHBj*y>APW%o$8
zpLT)R#k(vieexerZ1hUf#Z0kdCt%#*CUSo~#83xqi9A6ibdf_>Dyimwxsg^fpzXu`
z*^C-0bFs4%t0doBmV)Ysrx&w}8?7`VeyjL=|403}%J3{clG~LbOWHUW>9{%YZaD8e
zDlGcGx%3Xx*8AqN4rP~PI9qqc8dh0gx3X(P*8`aW^7#?z#Y}~I)1zCqizC^$$FlPh
zj8yRRO;O5qh^CP*JCaRqE_s#pc7EBOL-_5#JUW(+`Sn%6t~RT9r=H1|aZKC(PSLxM
z(ST2y2Q|)Q@6Ll;|HKuUJO%8THGj_3;TcYp7l(;XCUp)n<loR_`9XX_hJG8)<cfXb
zNI_VT8d9uuBzxb%OZVX=TX{f8*Cq2?tP)Imf`;GI!-A0<nD?X(-+5w@6?9*zjZjGw
zGJ!(7)(Dm0P&&HZQyYk>z$<iag8&W!8(X-1$;zl#k<5w($2IM-lP`d08(wqtvPSRs
zzfQ?5n4he(!B4=Ft43u0_dAEB8=+o>)W{Mc&+lGqBUi1cd4ifsQxr+iyYEh8*(VR=
zHy|mHB)xFsaNo{6n-^(Q&?Ad^<HwS0Ls&B#^K2VRK$kzZQtfLe8Nmw`tYT$V!d<1G
zakn1Qhm4YH<qRt1k9t<BN-qJe0It7o<ojeX7@b4S>pda81BY{ej5=}f)o+g`pNx{!
z6=nL{xt53Uhu+rR{$&%q1(^4ETu7__=40y1$)9Dpwre5JPHXV#TdS=#Ev&pGYVdB~
z%ZO##VoKnA=|{ww{MuzRFY|gjFcF6rCOsxq=-`wj#V|MeUC$fLTYTZ)V=6UfnZZKa
zNWt*i==$*Q>GO9A`FMrBg){?Ngcv{wr``=hZqVZ}u1bjOkM|;q^w5W>Nx{g>Iy4x6
z8Yv$5YvJbI%X(us%GyxNy-n`XVIm^)Xvtl+zeoc*g#=VXqgRuQdCns3#Q`ABs=9vA
zi7)T{4V(h|@I8&wM_<#x^^KW1FtGDKB4LOcp*uG0tN)%9&Btmyazm2+=|hZ-kspR~
zWEv#86@+`}EK+u+%!2{-x%%^HyuCzLWxP1V2pUN2kCA87tm`ZD7=IPy#@(Bxnf560
zk<4JtRp`KuvC0ewc`%{9CL5Bu75n-OpU?JhpgMTATQ#34<KWN!@e97}=H|D%(jV*<
zKRx**c5Jk(DH%fsc3+v-PLGHqSTzoA9q)&yOza=IKnTm&*#2<RD=#4r#*t0ljnqYK
z5xoMIUbZc8Nwh>)oPuleq5N50xOJJsGe@%Y4ZF^O9)3<ri?1-3`h^N)2J}F271X2|
zew%6i?sUSq9a#$8<=(Xld=zvPWtF1BaHJ0cqFs4Y<nDEDYyu(;2Vd&HOlql^)VnoT
zO$864|38ZLpX`8_o;!)u(3gwt@4$a9D0P(+m|?xwTXAI8w!Rm))0~meNleXC>~)F7
z0QQE+#typP5%!cgk1;Oe;S8tjqXWyPqb?V)7=MLdLg&)$piT|biQY7UY~Y*`oPiJ6
ztr<LQnBL^^_U1<FAS+xL{3DiyN194A9lIwg4P~P=u&G^pQgIsru>{yZ-eyOm&joqp
ziEib<joNDb`N|fod;Pch;+aDMxm_cf-|Ep@ebmA8TP2w-?<7y-dh^1rI9bywF@5$y
z(X@E<R;cd8dtE%m)AUlhh$u{{J3qwmO_i%+vZx!my11Y}x9}3%xn@$vURJk*cAzo4
z3L3AAjCX)9#QIk7u`UL;ETqP@a_7eS;119C#oZ|aw*}K5J?@AcU}2sdya&pg3ZcMh
zohgt6&q5MBqmdznQ9!nUZlS120xm-0*F|nn76a4t-^B?*TS9M=QHPqZ0$kvl1?gZ7
zI;Etsn8IOR4GX&9`0s@b8kcUN=NPCmSK*8J?twQq3ka_qQTq((PSh)nnSy2!2+fPH
zbe)Ft!nTU^Q_N<u5tsRNyIdKS(}m{4&8z9w-!b44O#rVixXE1Sk9_YXe3Kp1k}S_o
zle3=q*K5Tr>XJptBn%-Xk!=7aY_$*U$Wek8%k8PK_Tlk=Y>(i&E^CpLC0j$a=tWvh
zJExp?VD6V;NS*inH?=e_HXhYpL#SI%H|g<qClAnsqaf+q)49<btyk3u#C+D&AZRA?
zSv6IcxUn>~J7r>H?E&oAG4jNL)*=M-Gd-pW-_=GH#II+W7mXJF&DT0AdUx|Okl$44
zR?Vw=OciqlLoSyfs()X?F|Gj5x@GiK!Q>%$)D;x%$53*x%VQvQru_R$X!&;5d!0%&
zGbX(i--a(-#{40OtoJwOd1s0X3*8vJdo(z~2a$f*9oGI8|EJwf|9S$Lt94@;<Jhqi
z_pH{<FA)MTALVWheH1En_UjfKqCK@KoJ|wT$cW!GKbJz&?U}PRk!${9Sv{US)+e;2
zT5rRNbIqKeVLS(+8XvaVi(?|cItOj?kvnT#?u8`g->)3gxy*V`b?9U3)>TM__j#Lu
z__=#j1%0fnm@p$Z-4hM0TA_45{9Ll{q-Cu1mUz#ZB68w_V{M+?G)Z(=a_h3z+%1m%
z3p(Mk%)n~=eC788K-V9vH1d58ts!mc$s7H6gK2<_X@obp4zy=t5ZAfOXjkYPk0axq
z;Lc7p89Oj1syMioP;ucClQB~~O^@(7oiRGVSjiBbPWp_W4Tp(L)P)Rh>8UJVV2on>
zL6l{r?yRLvKeQl4BrMqg^BltMfgb`q{?9E?i-{8~Kv@BKQhNN3xPUjM$(OuLi%m<q
zfy(CO&r05hLtis9_yj5ZKc;lhZ3dBXS@@Rg!>Q<M4SEIi*R7hH_SX(u3Fxkp*h@%E
zfmIx1u3&+>D&XxZ|Ajav`8V7&yK_zGS}bCQ>g8`I-wRnVNHxEXp$EWkgOM88fMKf7
z6i(p>z}??-S^L=d9mmulebr%BZ10%h*v>%8%U%6J@+~vct;fcFwdUSg9JYUo$-Qz6
zXFDMr-5;C9KdZ2!*EMO4zj=|zyL*rGB-sET#07XFrs7ONd_u%T#UU<36*xLX@5-|E
z(B{g1m1>d_@X+ZG;qwe;Z4Jneba(Xc$OGc3EI-wXPY>i1To!Ur!F#aV1kCM9LD>_4
zI=V=T^<0LeJp8S~5up~}1K%Ti#nRp@maCSTOl8S}MZrA)-z}8`^#8rij8I%{@kk9$
zc1lVMl`qcAlbjhqo%y0%vGO{qFcZ*iZHC|N8&hM2A|maATJ#oyUVMf%-Gz))B%$px
zgIav|#ah<pt($Szbvw`2Trg;h2t7+4d%uGIt6Y;DlAL|jL_ZJzo(Rwc>?&D_wTiE6
zss#x-n%$k^GsY$CI1_G;!;aTUhk3|-2kiE9&4o2*Amj5E5}Ukhl#{uBlyk(^OBp{&
zFTQW0s$Ox|X>DmDH;ZaYQ6Hn~Z7*6yTYa}Bwnx2igSi>N<xK$vcVcH-8C>$n@yGn<
zMSjfE{CNj><jwaJKX)k&SfA?ja0-5CURxG2<|<{xgy{C*MCpuaR_QM32E>hO`|YQU
zzuTFO4p6=XfD|Rvh%BY@SbTpQh}zVKYF9pejG}uH4Je`7w*$K5)^KC4@N<XFPCeiY
zJ2gXzD9DMY0H&FU5?FYk5o0^+9r$PiOV>glwe-9uYsy^l$N$Js4T>XQ{OV{CWl}~6
zIzr(R<@jOh5#|`H77w66r(V%HgY8%r<Jy;00U^aACNv|6$ES;Dlm(W;B_i$4@C(^C
zA-rGjUC`Z-&BUTRSER*UNv{x}T-2H(h|2R<m`rdt)XIt9R<9-WwI;)JeYI5DAS?FV
zoZwh4CQ7{JWD%y5x^J_moA}~~7wTJ8T*$v{;3Yn=MLxxr!55M(Pmn|SqXv;{)?h0>
z&k~P$h^E$J+krk)!ZuTV1}Zwao<5%^KOhcSHMI`n(<8b`FQCSIDtYk2Rwr_(CUD5p
zfMIL!aE<4&lo2z6i|c%nvaPK*){*5Py03|dj-8bsMbGt}=wn^eBO?Bbbm#xlTpj2p
zVl1ADw`{|IbV7oAR-dX+_%k_i!0TXi^6@-nmVzU+NXu3TGHD9I{iF+<(;oe2ut!#}
zSkk$EmKp-B%Z&lHd+nQMLM^(L`j@5I5I%vtIsH0nIkzN0QfvPz-k;6{)uvfFDG#2R
z>w^oJDm@?|<Ud0yg(e33@zq2TRcbf#Tgh{92aUba5v7~WJ8GYo5WL?riH&%llhz;w
zcFmqK(xumuoiM3N5BiyR6?K(%D64izR|xu$DQdm5@m!3fvRrq)a$xt2_x@c;)b|c`
z&j?xXo`%2=gMB?3^|@Np?Oy7QnT4l5=_Q<VdO_*z>oraWN}<M4&MGX>sDjnKgt7kU
z;0g-JG|buaqXJf*BQMrF@1S&Ej4y*1+H!!?6fzZ@0MA)Dl0zFvogi~UZ(u#mc&|zg
zssR*tXXxYd8Cno)040v-;uXHGL%)mtfNj1MDCxu&zI}5K?As^B>rkoUVR7LEG>*qI
z!6n!Fzr|?9ORI~&@{#cJTOh88?-pQR|H$`PD!=#5E~xh2#yvC-o=<Tyosb$&XeKVH
zhd0n_oWVe0h|N*PQH9TOMw{G1*_<PCAj~(TlTACBa}b8UH(iF8yO|Zas1|k3U~7TN
z@3QuJ(H-Vo#kIxzD=M5D-7J)B5UDlOINe<d0Yy=sX201DzYr&Qvh)5tV}+o*dc?`P
z$+`$gDI;q^ZGPN4CpfX|Vd)>Mx>lVmk-z5KCo4DoCH`gI5m+WtxxH-hq`qe4y{LdL
zgg&c6FWShdOWyAFdF!Hf7PdVG;54DhKInIQD{d?bl_BdPUM0v<sEnAHiyD^IftSC~
z<-xNf_C$F$78g{+2jK;f4+(DwPBR&=M8QP|+~`NZ<Y-r3CC|>JC_&!3m@%Y5?gp%I
zHo_8>SR!NprVWg1i(pa&y3YbW+!`*!gK>^~eE_PHOd`(z$29$_9~OYIpMW#6{!J{-
zAVYF0Pe2c4{^kFxF<puHtPRa%#xn-zJ8PlZB($n2oQu^S96-^bMx_UtUHr~f>e%=+
z2n`nu3;I9c0|9u(o3I2d?QCX+L2Y5enkVpFM}!Zo_+qSVw+TnTHTD9K_d9_Lvi*BS
zdkv6$NL#KiyEVFLYVE^9R>I(jfBEZ&%dcGBsqy@eEs4dmD$cjbC6se>$F1T&Ho5(o
zn7RL|yFc}hPA3pW?2)xbVxFzclT6y@D>^l>fJ~cF3uBC<O_QZ(;AZ#vt@+zw;XWsn
zG!*~wk~+1kf)DYmhw!8qS|ISQ&qe6(s4SV_U2wh+OjSOIa!EZ+C9d)kXFol;%~dMt
zHtsBBHvgOZkjo83UHb-Nc_kg+*<x!ky$vPVp#N8e<;f@Fb%E?epBQ@lHHEwUrMxL$
zb<RY&qh|3$XB<(L@~R5R9RFv@D9m`>##J0uA6d_8`!>9JWbY3)DL1_63(u5^kXD%!
zGH)f*D4Sm`PrXsqnwd9y+LJ+3uo?rma^DxIDGk+ll5D{S+#67In<KkJg%PT?yz?UW
zwY)!3q4<`@d1--KYlX3tl2W3)Pcs*9kzXCK5II{V#V`i2=3>43vBZv@bnif%H})G2
z?wcdOd9?Ors{;o!etDL#Neg#*qLulEJZCR7QPuHrIiWHbZCt`{3epc|w}nSz@Y`@f
zKIx-f<*k5-NUx&p0R`m?Oi26N7+)(X`U83uZ6<$DFSTc%&xeICRi;TO5f-05P`-ek
zfiF}>e|0?vz%+MT>0_)N+B~C}wce)zJZWOuFM8)B7^aR7*Pm(fVg2IG7x>UbC|3b0
z2!ZQQvzO81Y-Hm|5^5)Y3#u`7<G*mDAM_7!1L(Wa+Jr}g#vp`dc&Y#BzwoUR8d0r;
zMztX&k|*y8I;M{zBtBNLLKnXpEnLlMHGtFhPJraoBdrM!1SnUjVJ)|I_9(bIwWj)9
z<yxy;RnKG4JH&qq>v6du+fi$}wdnV3@D^s$TCC0(J_nM1v%&J-o5WRsZPP~`J*F9Z
zD@3fY`{?cyKCG}G=IZQ!Y+;l~Xg_p}JgxC~$M?7pA5)C($368a@TLbv6K*^bE-3Vo
z6i9Bdw$nMsq{kTZ4sv3F*pwN`TWoNV`lIIKvo6Q1ijh{JY0S^3z`3%=2rIHceIkQM
z8G@D^%K^&92=9zXoh*`wxX4m4XJJQ0GwQ+p>1)z88@5+lVj79i>ebSXE=ypyT}~GZ
zbhGDXmNNX~ENcE6+P)>pbZP%CZC<mpOX0D{;LS@&sY2eY-=f!Ikc~~5FKA`FdxTTL
zx)i|pLVf>Hvm5>>uX;Vy`dA~hm$j{Frs_G=#1un~0HBF832LG%j!%+{aZ+Pcv(~Z;
zeQTWmoU#_uu8I<9HMukWT`GH7fdRoA_V(lzo;;8s`sStN;<4Id)yVz(HmwX&|Aurn
zVsVW1kAL74bw}-qG!t)0T)sZDRNudW)ms2x(LGuLa@+T-_yn>ndBw*KbSoVN5S=_i
z)0VR4*G3sdP`9~EFC#3P$R0q|YQ=--KcIfLN8wh`gV1fb@E_6F1i$IW%1q<Q;tYJz
zCup$;x&+DlrWK=_>YrG>Ua4yqoj6!uCOVFap|sn&*&XQYqObtD=Ep^E+#N%M?G#s}
z__OYcr69|KWI&lM$C%V8h<FUL8^rm)vp$6#hpM6y(b=$_XcdCQ4F)QaACCuIi=Ybd
z=Z?w*qXVr_D`c5kF2-`SpO?MYTy#?6<skwYkvYxzh`dX#ul|A_PtTYlsEr*P)|VP0
zH%U8a5n}*0cs&RyR$fLg{4BqZatSq|<4DVSH=6J77CdD9EAp(f-Rz|(-)<ovFcNxS
z%yo%$!EW-4KV|>79X8FYz31dc=V8!(BH2Pow$__CFjt2;E-TTj`%SNwl2|T$cR{NN
zH!ug%jW*QNhN7udRU+GfsRRuYbN1y60?X~TA7YssVTFJC##FfTmMW!Im0rOd;aS$J
z%2y+9n2pnM5wqhpa61NadVgqAI0FQRxG;VOV@5y$G$IKg0Y0+*D+flNe3clUE@OX(
zD^g(nX1L!$7x{;;7>>8+N?G7);eJ(*>1<|v`Ddcd*MD=3%4U#GePtTfSKqo7f$oPY
zdDFa<HpEE4s>4dG$cg?@DUtB${hPKg9xPuNO~pTh=o*0;9nc8nd&sM*(ti&PGltU{
zEq0zrlVKM+dI6AP1fd6CfDJOH;i%JemW(=|EEfW~<cM%&Tm|$_4OZl0pUvu|a~Jtb
zeL|9-g~2GN5t0Vw^Ejg~uVdeYkM2JueP@*vf&GV)h}*-xaC4(}^p?i6HTPy?F!ba-
z$GOYsV`25htO`Czvh@<-&b2%>sMDs7m=7X8@%zQcB1D=TQZ#e$4BC1fZN=3QaocF{
z^zWKN+c=Ezn41sdFF{7)t!g2S)JS^p`Adp2Q$EKS@LZ5SLDLacR~|?szW&im#eCT~
zq+Zp#w+P*PV;q2i-UCbbeycJY4j<eqd^wJFNYh7IUWQN`UcJh)Brq)<;DDtCq=Ti!
zWk5TDi~EB(7SsvmE)-|l0V2Of>UmnkCFxk6E%u)gWi4bM$8%6_@Vyz$8IWHxHEM|@
zJU4o5I#P^L3)vQutXKA&^vDk}XxvR6+vTJ=7bDn6x#t%k9Ztv2tVf3h*d2IY8r=Qv
z;xXWlhuym}WW}JgJVSpykKGjh;hPYl?Vb5sRwpXD%k#@|-Zm_3Q!WSWTkT?+$_Uw~
z#hw969*n?lX7lT@$HsK)fHri1Eh(rOeD92;{eSI!X;f3!+ixwcXcc3%$|%uN6+}Tm
zL8e5-iZdv-hzwCtqXml0Q{ZS(qk=>#h$s*d0Vx7v5JVtR83iOD2m(Tw6T=Xa5Hg>A
zchJ7~eZSnb?z(H;Pk+x^lC?rea?U<`KhJO4POBaUf07a<GsQ*jl5y0kbCD7CZ;CL_
zAxMveE1PCR_+};#+r{8gm9MSX)oN|*7ea;_*27RSLsd?)9|+5+R}{=bb)^BiD*XU>
zr?ra}r9e>eL;4Ui=PESlXd=!k(0!50ku&j)Il~Ybf;THkNPt<wN$j{Bb82CYo-odu
zYO|#tc4_}Q4q#7ljiSQITp`#XZl;Fuo{m2fGvwmxer$#m6=uO6nIYY$6sEC;VUjZ`
z80^H!oDrf_)251uGzXxmD3p0i^P#5*jhZe;o;l%&l7aFV@v{(5SqF0ysXo;|z5G@*
zW!97~g!}>rP=3|~j+-JkuJj)K^{V{A$e=@#Rs+cy_sc&05=^`$37Qbh=9?A1zP=Cz
z2{GYNzO$HpeNG&0uCL+j(FART%L&#)E;r2hA<LT6<z*&x!JmWernsQhvKq#7<iatE
zPwzjuuvUeFIofvpnT<FS^$JgjQPus-bkIUT*8}-BRL8}zOTyGPM^}PY=Q$uZVjVE~
zqTD0!6cV-Zk5_OtclWuW0YFK4c_d8Sk9;X(tJx#S5>FM|UANSG-yDOKuV~`&VKLg_
z%CUtCGd-88RALf>ecASB3>PI=$vloVISR02j8h=Zfb%2H94jS=0~R4eY28$x^5+P{
zia1pzWZGmpvZ)q>7Dx#)`r0Z&ioSmSsv5$I-^GS(W0ogDf;+_D0a;R<h)*SL7Opwm
zb+U)4&sIC7_sq#!xb)nKB$C#B!CeDJ-K1W8EdMPf(`?CM^A@|}T)R<otQQ<*+RF*4
z#yF)h+9|rqi>!Hmv`{i@5mx9*zs^jHk~`KFK-PBS$Tj?^{Tm_)u9|3WC9?*S*|`}X
z#$9=M)lgi8Rtm*ckYW46F3(il8O2kCNXyO0XY?^BjPFz6zjDl?iaWC8DQX#u!3k<h
z6e)OxaZ>t7zdQw&Nh;jjZ0gubNDy7mSD^!v%c)My;h9uJ(8KyZ3C_QPE5{<FHx%wt
zRtWm)B%w8*ji3(dHOu!zG~5$(M)%#o8)MCHeU`eQogqS{uD3CI54^DCr&&m$?0P*Q
zq0>7HndObN)1M@9PmGvb5BK<|TX!0*3I9cem+Q&1_F0MHD0JHxHBjxqem&i~Mtl|@
zty6bgusU@=cw`hCqv23}J8a1Namm)dAuiqb#+439fp9H#m86&Tyd`j`o$+_18{EaW
zlQ>ZHv4lC@haF}P894U4K`EYpwgqWye`90`6h|;h&6_6t#jY6^%;aokJ_zve3`M8p
z0K!)FH1j6-L!FgJ{ZMqghYDb{T$Tb`m1(@7@14vpOg0RQ%p$lrwK2W)>LuO?Jn#Dz
zhUi4-z>j7*fw-wEVBTMN<_pM$>JUZ7cV;!`q^EN;r>WxNQ8c0wD*MrbB_BR-B-~_7
zFVD|>Zql;Rn#mkP8c4yYYjgR^fJCV8br)mm)-=%+4%$iatakjiA*Ns@S{JjXE)%w=
z$+g>FUgE@hq=ZCl>g&BCb&?OS10^9$k!<zEa2LVuC%oNJ4YHZW0o={lS(2n#IC#XS
z3y9kET+zBvk(09-ltadhIGd*kkW!{L&YM(|!dpoXOb=UxwVuAnbS!rlol>~aC5jtP
zbn7WJVkKAfBYpfQ0DOb6li){m-OyT+q&AE&Q;ZR_aUZ*g!}u&PjgdP6F%w%V?}yux
zAq?<Yp;9S|g_I!zAET?n{5Aq(VBsW++1)a(yd5d!>6^yi=j>HJ(CtSym-aJ0j9Ydn
z!`d&+X6t{Q6CLq+$SVSukO!To_`1c7L%aYS!~-A_pyK5fo{+dG6P10(N(Rk5CE*=o
z0GUM>na;{MXj!obrwuK4uzQAlmr8gO1)YC8n6SBb=XZ(Hq5)3K+-7d*$@;zO31t}^
z7w!zkM<O_hY?a}CyBQn9(1A-k7JvJ2%cbnem{qxc*sITUzV~i_@wjOIh$3XFE;4ly
zWA_3~@FK$SZAX$>E!9W4m~{F;s>)WlX-ce*GY>e*oFb$SJnaerGmIM;Kpe~mpSSBz
z&fT?>Y-u8h3Px=aDMJB5zI9iuF_vr%%fc}JRV$Ay8O4Uz_KKN;A_prX{PFeZ2YS*_
zF~vTou=sYr%CY%HEY%IwJVY_CA3ssPj5zOSl^)vho`YEn*=!%rgJx1B^z)zaXB8O>
ze*^N3xIpY6^Eo~eo$9jxm|^}{Xz-I*o+@el2?}F-reF9|6O|>IZjDY1pO7><{@$ln
zY}IwJlIKNwT$0y2wj+?AO)yBRCm;A+9g<<!+j_>tuOUWa_jtygI@}Pw605z`=S;fG
zk`UajQKzANV{!A#Cq?79UWP0uIK!OUjjyCufudKAnnby)BcW<et^}(SZ~(MJolFRn
zjunkirPO*Uk0vGjmqJD?p3kfVhEOA;UxjQyabh&@zqqC>MJy!ro+At6O&Zor-y}$t
z(7p$8W+f_VN*yC1BB?>4fV4(=jXwG#U2$jHp=^pF=g`-OOSh$%jSsjWmw7LQ=Nwqq
zkuk&#6Ln0qNB5p`>Qc;jP@1e;8Ve}vlKGDQr`!~zV?t-?_LJ)`nWcuj#AH}j3Y@BU
zXONoN1lPf?6AiJPSTb{hYF?TNkw>jC%!!Cwj}2VmM!rwfUXpn+hRj)LhF7<b#fzPk
zzF^2eXw3Kj4D2OLE&^#~W)%7vh~S-b2BD53wh$9@9Y)~VcpFH!0#opbcc^Ft%oPem
zhH22s>3pIXOi{y=l~2CVZ?FsI5t<;qe=W0>#-lbt<V<z&q32fx+bElr%d3!1sY6RX
zAFLn=nu$ED?znC`ycL9)dU#2Mj2nWgi$S;4`+D(q6R$~b9<p3zdy$@OO~AS3R$Vdt
zL8bA6wMxH{j4?I#4&jA0*q<?Bf@T|1V`f^NdHrK!$Cy#4h~-r4TSCJ`ytJT+Y$o$w
zC94V4i;+p3$w`Af%vlV}fOPtX(?VD)Ppm{I$h>Z-Y%r;X=$7EY_2K>;bQSO=bv+_A
zPE%+>XAu`iVn_y?1|h<%f(eE&?Bfo2K7ve9s?<X=S^u|y;fMyA!a<=IL%wI3q@2`z
zC!qSsA=-*F5LO;@^i~;a#JnNqltcXUk{N^9fY8-URc+BOrz-VF${);p3U^Fn*V>Tg
zPAj`y%u7od!W=h5xe#yp0y{>6pqlUh7)fq-%=j=G<%C|Mzw;wN$`_7v)OCtF_2F+R
zF=RT^b(XU6Tw@f}Exwjc)#FY-O-QbEY`~bzEI`<@YVh(io}|mzNR57{zO9#txf_7d
zU3+!s!q-ekgQIumkJ?3~#jYu|D;adT8sw^w5}aO9o&af4T9D`ZVPm@AnR0#En#Suq
zloOOdY93X$OH!$9YkFUrJgarN1DznJ4cRLcQ4sdiRur`Y;xP3yuoYv`qFE|R36duY
zAx+YRKEso^@py=nP$L{rBGZa>8yQ6?K{FL&)LI6dp^GF0G^nz|5ZDg=E_IS(c+-u*
z3jxQ_C`qD##bzOzuwzh}kszr9i7|!&SdF>M6oipLO;`n$kETR#V0XjlAAYg~Q^W$X
zj+H+NH-giP|DuR-Ob(K+*6%TFPwM<;iFq32T_kn0Q9SBEI;_3Vfso7Z(ckaw{;a=s
z$>N3H1DTL%k&!s=!UBTBBaC)S`#_q%uKsw^du?2F5Z%W+K{+$HarKhFB|0xjryxI&
zy@G&RV>hxt;`ytH6N{<*>rh8hHl1$4fBl#KV61Y5go{^nO)!aXX>doVGz%&sbBu-R
zfg+)JpQ1hGLZ|>A)jYO1QxnJ8h2P&>Tp;UoBA}W#zoMkPDP86iUgCn2{k~77CwqN1
zzPM)(KTMoAiY5CESd_%gbwxb;8(%<?JIFF!m~aq{v_pJ<E{URl6X<};d3Yn7!frV-
z#OUO4k6sJDy8h=>F+Rg{%q|MVrfiiqC!c(~EqD>tY)qIMw2m%6@ktr0P9e=2R|k5R
zH)ksHuiXUiM$qtES{E_*>|MmMYo@2IkLlllzF?jC*(Cy>V=Y~Gp(8SNKUGpP$gVeQ
zf!Iim#G^v3Fr}C{J?h4PO($deZ&2c0rxT0fU13N9*hqhw$^^Fjw!SPj*aNWLbAVCB
zEJz3AkPI!yLjR59#JY^_oRMUxsJ0Kj3*LgIx8EL=s?N^GSH@z-oDv{vL`3hMRxV^;
z$32!aGuJD2e~pN6h5S;ON@M*6jGsW2D|pySACg#3;9bI&gW}L@e{VbA3uW#8-dtm`
zQ>XkNlwEblW;}?+{DgFqea;lh1N!0}iYBlgA-^s>ST$*nyd_vkQheFLy0`=JJF}iK
z$?tZ_&Ux>$yTCO*IiUDU!wYwd)JVkDNP>0Z+s~$%ZQe2quc;M3chFp20gY4*%Lv*f
zo-n838io=y8U~d|S9tnK^JeoUL10|r2*+fXbc(DryIM({flB0RMk!+kn8~}mXbc|^
z)dYg!A|=t=gLUc^I^g~41pwYaAV$BrZ`*R-J9xt2ykmNJ>>4Z!U<g2c1|!I%k+Fp3
z43Ra}v6t?NR@FoOj%x9q3TPv}RU|?V=x@75L8ll$t&*l^Dt*VGMCJ1XF@p>wx<aR*
zYY;UXBl7r+v~pJ_ma%rAe0w{Be-p-NZm(mLMy;<_ilnsPam}$~B)&Ltm{-NZZzkxS
zZJiH_)+LKw{R4dynigSJcn_WQ*^;EAwf5@wCpYWdei?WGx36wQO-;Y2!$`WlM7jPE
z$DOaO=2DGZ#$e6yFx5r!qEtNK-)=gNa?n)dDpU3#4}^eH{6pXlDRqLK$lq2W?nPwy
zGL=At3Yvf>2*q#5ov7S|PVs+_RxM7I;yH`(73}L=dL(ED6JkE>kgb56E+))Py--Oq
z_Z#Kx!y~t$4>;-RVICvYeg@4KBmHHdPE0)}b(JB}|BOb_!K_^1D#COY@Bp?C`s-R9
zTcBL*v_%dj`Vk@-2m__*P;lCJ+8M0~E9WJsYe+9db9CbU7KaO*U7fU)TeNy^Wd{wm
zm0K4s9x)=n4ARQI*>+@SkYCAYSOLVNpPh4vhYBgXt}X1Jh1#T9rq=^kejjk;v(dJ-
zmwh|4{&Y=0s6^JG9Vb=Ts;xqs_@34YV+R;Esr2C$z*fmhN`yCwO6ogy8_M4)#W}DT
zsl{&ZscBx-h|gdEn3xheWBUo@YNUWWUyptScqmxRhWJ&Z_)+{vHS6!N4a<6`O_bU!
zINp{-KO`zc=7jZ-jaa3m;7tZUJ)>(ooJWXgfz#Pd9jv$_Cxim#=|*BuoNySWM8{&v
z)v2+5-goU>!ng;$RNq=#ll^A83_Y0zmhmrIf2XB^=~KFQ9ous8yhlsh=7E5W(N0mP
z)tO#q)8{A&6mOk~!(dKF6~@MNox(>iRpKvUOiG5@twMGwJ5lW*M$r_2K@5Zb`#$dR
zd<6Pho-S&!w#;>EKtpXmEBtmBC(d3h2#?{E{fYOxCa6L+-G;J}k_e82z01ze2*roH
zR&G1-P#89HEaP^>*&CpOFsKsFc_zVou0&E%-zlKjXPm>-bqgp4Ee{uoj-UaYY*_VG
z>p^9-*x#qQ)skZhdx~G`4WTTq4tv-C4#JZr$u=`RqVtra)Czj51Dt(wt?O8Kkg4C%
zavtEX!QNyvlly~Eh*EptTEmp+Nk_RlsuPS}DxmwdJxW^k{*|9@_#~Ow(x&w}x8?5D
zKkpyG#dnr!ZG$l|+~%+YBzewpSL<)SS^s+Pt{q1QPsUr$bB6uWv(S-i%zNwg?V490
zr;f8z{(`!}BH!uN{ES*n?<t^9S0uGa8_?w3jk6aptmvy$^m*z8#f8@HN(C*fLZ}p1
z)ixuR(8gk_16xYsP`-dS#EWTB*-@ditxJrpa8=d6ag4O<0!p7|xIT$>gStBG>A_Xz
zcbs|3VaUj}={};<7!w$%Bee7XxCf?^WZ91()<43VjF(Y%Y7w#%-9Zwk2HaVzR1#~E
z(*|+N?6^x~hF6d}M{QWkk-OkW($m^7e2EBYfV8@_^A1*jp%>dQ#q5kZnQvVuAvozY
z5EyM|VkdgZ2?5UaC)Ubx0ydx3br^jA_>x7KPJ-XQJ*eAz=C){K+%wW)i+Wgo@4c~@
z4;6F1E<0Pis~WO3*2rOa67o1`5oCde?Bk%%ZODJfq~=~*=J>p7QHv5;fjU8OBLkUb
zNtwX&&FgoP6ab)j1LA34kzNdx;e0VLe0xXuO)|tMVru0@|2t&J9=hSa9EB|S031Ny
z1zn6e4xS8J2>k<1I0jNHbICxFAr#t2doylHN(ncSW}z(vAR}|^p03lk13wi@k<`sg
zF>M{yTfXCR-eL)_Fy~KqY&97#HA;6{c>Y5BwnY2AUNR4sHa+OVJ_YCdQ-!+^rBm+G
z=x}HHi%u`erYw8Xh+V9yes*9U0;LHNon6Y<wbAN0=JG17kF`09CI8w47;xI`4T@8M
z%u7S8%s9qYV1#OdI8Y0Q5I(5l;hUFFRUsF1#TdEmmAp7O80J(obJ$pY^!<Ag-$SpG
zJj|2G!sq1*iYN%UTCkU!tTLh}=HqP;*#%$q`<VyRnG6!xp-Cu4&ipxK${LIrk%wjQ
z9=S3ZOkM4cR7$f*>hI}aBa^;DFa&uy5ecgD{|4;*{(E%%GQZj3DsGF(O-F|g!@+=f
zn@2yswWv?Ds$bt<QwA1O&C-7kFG;msxHw{J@b>hJ9O$%kphMM5)eUBk|6rVtbcN)J
zm7*GLGn36`X&lZ0!@ho8$*3aJ07nf$OQM=)5RN6p1Jl$=WgT<Y6%6*e$+cCt%r4sy
z$5Itc>pQhnhcb>C?1na>7nX%?nEUdM@rHA})nxH&y}y6|QfLexs*4yfd94kVcB#b^
zXh==xLHj<2<*H!tYFVf%CM-){KWG55<zyl-rsA#RSd)VRklw>+KM~W0@q!NZa<$V-
zcBJ#&rJBOkl63FS$44`R<E_CjnHSG>UA#&@w8cFBvP10acYOEjg-RVwzXhrJg`+3+
za2R$?E|pNq2YE0+!_d`j)_+3NuICmh$_F&*Eu<T<ln@miy^;0lIa2!i7Ukv44`{Gg
zLm;&=J$&Y!Ww_vQiq~u8H!z6Ml?UY$QmkcFkG!X@p9VbDGol+?K;})N0%7xP3d3_c
zg{7r!&nn4vH0TUHsgcg8mlEOhVIlk@n#=X?3PsB#Sp|Cfs$pXPBoqFhF0JjB7g_hq
z$p$%3_@RE92*z`jz6$oUhR?ewQu)~u8p5tCpO)w5!O7xBm(K8t(l`dWa0vxA(#zp^
zk)s__wk?Y}i+5qu&^AdCzGRnW%tgyx>C8tk*9N7(mN?*J#88rv6K^D^<`wNz5-nq%
z9bWu>Y_(x)-0tkde;n26f%Npv2Gp7`OgsJNpP-)RE3bwe?!S*Hksfg9zXv@`Yz64v
zrr<Bkj_M&!A+KRZQ!>zo!F1L2QX`)6v=!fJm_}vx17isuLI+R;3<iCyD_R{DSRwcc
zbU}$j7-|c~?nz7xVJwV7Q#lIS-ozBP)>a~lLsA5GIj(bLoXM1I(uvQIP&BqfO_}#0
zDQx?|>!Ld#8Lr1X`ZxB$KpG9Bn9wd|k(?#&1Ka$O8=&`2WM)w#qMSmt<@fo*HZf=z
zR!c?N)=_hV>2U3w^~8sqs{-$&4X5p=Uy!fULO}0G?Ppf!8_|^6(!IkTz3tbw{4U!P
zBlk5RGrR2~qE}L;6RR&hfR1MJ<s+#HaXh;of%74F?@SMALH;yn!blT!ms3gDT1FX|
z7PDuQRe~8<o%?`>h$hH8sw7^~mv91_un0~JuD$mnLgwIH)m#~>+v>Oy!Hbnnuox#3
zp?%wiZTJ;Z_uY<lgEg1g@GoXN-yB-kc2(Rj_8%qGci)TrH4Lserk4B<XjtqES&UvK
z^BKcE@TuQq-^Ww(A92Gi5hI9Y=N^p*B^(bm?(XWEh6RUovi#0^KmNOJ&wZQAi%0oQ
zcD)^wtbbB${31BZv5VDmeNNc7ZX(GN*Xvg{=0I5jAgvf~B@QvG+;Wp13#A)K7d{~p
z7EsI?d2AK@Fga@uYweO+V4Fc2n>9)$+y*KXaIRn^QfyVLB9GDVLnv2C;NhbH+^|t)
zS)&ih*FQY)h1!R(ZawmOWHc&3u#gH7UxYDEWdS&n(?hBvfxLx<eTrMq2Qf@X!VGo(
z(IY=de^cX})zR>ibGn{RWYxUdi>~8!l5P*{$sQY!t+gJL4|XS!R*zm^%S;n4pjw}Z
zIOj^LOT5$^;e5V&TxUT3gQ7fzS_A1DGN8?-pzR3Umzpn$XsHg9fq%aa^NTW<6$ZXc
zV(@$L{#5%{k_VF~D!y93RQ9iYF^>TSI;35uBd0sAH<cLi@0ZaLGMVqrLX5||C5tkG
z=*zI(t&T#>>Bko1+{@9;Rj=yrew=n(uQQOjOT6`gS;UEf4D}xkTqvDJsaEo>&Nq)i
zG~x74bmyHI*}&}-vunupI>LDi%YesCcsW#(xxAqqSWrH6{Kh;Sv`|s}xo5{;#z+4I
z5G(5^wW3Sn1=Nu3g2TA?3_%yvFe6;)`dKVG$`>oFw!-iREJs18|6ZY2ddcfC8_aG?
z0W4-5ywAju=wU^lUgPdj!knyjt|3%8KZ#x)|F+3h2uHb}M~wuX^7?+fKzWKs0Da=3
zv2QC)C4DTsFh*$0K-7JdXgclNHs%Xn&zOA^-BIk%YCDkHu)C1?tDY=txyED3?aY@e
zH8>MyCO0*e{8Yciw<*71{BC~0^+I(}WyHChG-&ud<D4;Q8>asYKW)77V2u%dTf98$
z4pEGu%jaQa3(!xDaWC->caQyJ(QDCvFaetwI@GFz(bs&m(JmZ-yh{k05dMf&th$zT
zbM$L^i)qY7N<jTHIG-A!zUtmt3_j|qF#Wog3#bByfFO)%5`Ztq6rL?+^qD7onup}^
z(pccVQ<Rb}^sj>ZH&CXK<pCm$^=gc{tEm4J7RJ!nsek@WY3sDH2LeKM18VRVvTJ)d
zFKMNH(WMVs0qe9fBRjrHe7sz+v|;~w)&^!w#MZMK(WAsPhXwfWsEm>Ok`DH~pNPq9
zz0sfJpu+7!ZgYXuU~UpcI#H->0DFN7dhEuwF(hEQ)D<o#UnnIN63;^bB-6jl9I9dz
z=5O_}82{pqu)#A0A2$)H@(dmWUkzldY}eLAtV<b?ht!vHdolixElgNMfPN!wTdw2G
zcY0c9zUAWJ9~0<;?_+ZgE$mjwU+$9{1a8MRFdZHqkTIu|UvCQuPkR0v;?`?GT2+0+
ziJ|UZKJdbeKPde#b0rp2d>i*y=kcyE>)q1BU%}&Tdu<DdQ~ARxaHmTuoKflsVa6z7
z1B#`;v2a0YtdrLbgij#evJus@tlevj5_;Q8@)XRI#!{-iiHZrYxi!Rm&f2I#m@6O|
zYXomh!YH*XrbH3Mq&-yyTe-q6Bb47QwK~F1y)U9Z#m}nuK_yB3$9u?oC#=ua0a#vq
z(p3I!SJL%T?wS*{0ekMm2FX01>bUtGt9F;^Y<Z%yg*4red#T#M3>&y4<V*%XPJ<d~
zw5CpC2y-vdI5T(A&AX2HN9_+|77^^iSNATbLJ)3T#!#bHNg0`^qTh#or4eFY(;T~{
za(zhgeQ7v-aH-%OfFDr%t!qE>wRI?y*!m-V;H#sN(YjsAMyu3bN+x*Nf0Le?7<|ZO
zAC#_-;#Z7DvGCXmK$okJugNKxtv}8oLk``YPcakkNw;s>-)nZJ?W!Xkw*Sh8xT$m7
z&#+FO=PcBC>=>WabnE)`A1@MK^D-}yOXnZJhsf5{CGEd&CSQl~78FPZ%vA<u{n5XQ
zhb+dtv<&vb?TC(>&o-lj<YPj|>DLwkvg^t@xgu4lLMl~co48+5-L$Y=?Q&WwV&4dR
zYS<vDS|uS2T%7|EzBp*3KpEqH^r+HDOzp>u(P(jcABX~Na_Gbrq5hQXHCKvaKwcr7
z`#)5&{ua-JjM2rs7_VBQJ~R>toXzX?x+*7s#L2Gp=greUG-IuE7yok`XHxBw+M~lx
zHMH2j4p+zNjqNREYlVc9?sVF3Ygxf6%~mvyooNTO=xx_#-}=b)={^uhib<AwN($Mz
z(;pBCPFkfXgc&zb7$ajT)991_BBUFcb^2riSLdAh0w6_B5%~TL#}xpy$O#a3RUHH=
zuNh4UBg6V14<kKt6>NVJAfT>a53#+JivGT(7yO03&F)^0?tb|k5e=qE)wF^F&wSiH
z^~B)5<!R-Q^cV6cD1E=$2Bqii`@#RrTk=g^xxSW0Mqq)#xmh!o+C{x<0kTeM{Eil6
z1<PTI6!7ZT&YCc%X^BHBblS*YT(pM;p#|pQE?l;7PJ$k~+EPH$va`p_Ik-avHg6is
z`GN3l9)2H=OMxl|A-;m8%B_O=R_GcKHHi<K)Qg6THHA-`Lw7<}K}el)?BvE@9Y$GI
zfTLu_EQzOnaZbEUpw=yBu_m${4}69tx&!EjYXXcU^??tDudS$Cwneo_)JIKW^N}?~
zh`n69(zhdDxkgP6r8&_VQ2FPn8HgH?q{jFiYPd?LrODTpYlMHZC34>~WZ&X+n_a!J
z=C^V6hT2_&wv?uC87qmnU7s&ysNaggGY1UzADOMuT&@BJpDem3nXYi`dMDXL5<gPg
zuKyESNQPl*$ZQ!kZceG7YC@;rMHy~T-B0)?MiFAz`(*)BPCf9SPUurcG1o+b3nNQ=
z=AYqVBf&saWj|vEz2fnY*r^KPDqU17+^(x7_1`g&ak$<QD!o&!MCc$h2D}mqwFCPZ
zTzOdZ0<D1^LUhzqxk6zSl=6{<@p(lSp4_qUMc%yIi<Vq0nT^<N7RbHRoT-72E-m94
zVJn3X4#sfbM@^^Nj=l>Ce2};o8}cpLC-X7~Dl(#HgaNEOjnO5D^wti0#SCTBfv69i
zrBh{CQg;em%ivu_7*5Cz$d&}*fwPk;tHBrI&2tA$5hYgpWL@dw%tPZ=E=doDi2xs9
zL_mtzWIu0(>iV5rZb7|^?zGJ;g4e7_71Y50zMTN|7Ga4~*!uy#j}D$Nw#b0lD6-9k
zXq;4YHWP%yPcmaY?VfviAkhw)eO>KXK|4C-IGwm6I&nVf1>qN5v4fiaU+(I)e6!I4
z?wX5asZES_(%Ws`x0rXzXB_mKpN~ACs}tg{Gm-Rf4>r4{gx%rQvJ#iD)ecE2e$Qzk
zMsq!F_W&LZO!#+QwEQsFR0<jAPSmBjb-6aH*67!}8ys62V#1%^kCOVM#I6CmAR3P8
zDP{fX^mUH$NpthQrG#Tx73JzGWlyQk+#XAk)T?rHwru9<(FzNiRe8s{x@OECr&fMf
zO!(I+R};Tld;c_s_ZI&&mnpzE{N;hAFX~<z(q+@cgge|5zL5)`>rAgfHtl=Q-14gE
zk+rj){<YFNW)OdSb#;Bf{`#`a{2GVymRQW<)8;v`iM;%2tH(_*VM2&5Q2ex3Qi_GC
zXcicHIU+6#A<T06jjDl2n(yBTOOx}Mbi7@|l(EQ^bCp83m}{ATLPQuzoPaX?=A2g^
zYa_6ksMb1FnXSt9d#?{2!8xOZG8i$2RDiIr$tdvdv-kgnpu$<2c+VhE&3-5s7n1w&
zdT22OBH%Pz76k;nO5|s!Rw$fR_~BuC?KbmHg|78`NNspY8$4H!9)W^6WIk2eW_AA6
zqv87lf4ObBh(^!XdlX?b<ZQm%8q;-P!ASS9nAH8)VL|)KxEM|@-|fCn<XPaN$!(OD
zcBjZ4*1gO+stKb>6d#mi<l{rxA#kh%ZafPi__@Rv6?9x&zvoo8)D2#uBB=rHecBP-
zlfGl0=4fx*lxP3FoQ99%^yY1VdkL!YLj6z=v`^_><)BKru;zV&*OP<yWlDOK6>@Rd
zC&s(UcjITTd!8cSD&=nTlV@B<8l3fow}urKa-s)}W37{~kTia`Ubap{GR;_Ma#vSp
z`c?t?7JGPy=&7vvEW`U+8LmFh;6CB$;rcv2f%aCoevtNi!*d!T9jokQZxgwSk^M`a
zJuo78$qJZ!mFgKX;pHI-Hb^)GJ!Xn}euyl_)z2jZ`lI4x1yn;rP1%_^yaoN7F{f6h
zr%J5*V(`<q%7_5LO_Y_$b@N)7p!$5p%&F*vFY1%Iif#W!LlCY|vI@4{Kp$umMzdKm
zz1$2~E#QWcVl1aq{E;}jo>vu7)I&9QwMEmX+lVu<I;~nU7Z&coXh`e~rOjBfbN#}B
z6Ixln{5Y_*4P%`{J0?$@XYFJs3)bqW2EPH$87>wIr;LOSzsxL@ZXCF;#HB>Z$o1^4
zBFcf0BIca<!IMgBVNE#*@Li#az<#KE<-SA12<%zi(bu0}v>=eehSO>VjOgK_`$=0;
zpxZ!oIYjpPFo?&Wbee}PWL`1L5;Hy(!fAF^HG@Wi_wtlJ{EV`TPoK^0$Mb)=<|@k;
zq5Byz2P(F<v{`C?`o}*kCkmD@3`z$46n&<Ll=+Uqy4jbuTzEJy{aVXHV#x0J>iy?E
z9k;MIIB5%h4jUyqqDIx_w4+P1MZaZe%~}w=>L;=;!)$J28AqCRg|OGQ5lyfdwRTv4
zi`X0^>$vYddw3@-TCkRz{Wzzzny{1C-oFgBWdGrGC<SV4Cl^;eHsx}g=nDLCX>y6d
zzHu1OnS0*0=eEvKSNJkkq}S6QC(rcUDz(@TUn$$*q)Vep1vY1~YvCfKHuK#mR1-fz
zp2HXRba$lBj*uc~3<c{C(~s9?&HeWDre=rkIv<^!d$XHKihvTxWk28nS>wcdXB6qG
zmURW~I^P#$8N!YTq*tax&B_1qb(H^y*B#c`qlWFcm=J2?wW~#KcOTz6XE@ZibTF%c
zwC`N<=nM08EZlIH_4Ll=6q&DWp~A>@W4K<KzrbvX+1Lxe#_O8sb6f{SmL8e@GWZky
zEqSpa#07iEv)j9+4vWZs2clac>$a8op(A^E_8{)`_H>3rJvTOxdtdd}VsgDcY_&`1
zwUTO)RcdWqiF)IC|8AD*PqKY7^7HoBPr^>)q79OcpLB?N0K;moR9oiYIJuM3d+a~B
z?*ruPu;ve1Sef3U9oC|Q(WR?g87@U!9evg7;vGKYl|I-Wwb7EY3EpumxBh>AkQVA}
zroD40+WO|Ct_yaieU|htXluyl7tyyd+RkiVYq9J>nk)y`n~=}Sa}Cj;sJ=j5<_u&4
zFq4-8d)#Nk3NWJK|M5WxY$tiEzM(C0Zt&^NJdEso6n(7x-q$n3p)pw|&l%@eYmn~E
zw|kYel+42Q>WA(R4_-F?(G;W=dDM@<yozK~g~NJ|qdoKQmww^zK_Mpil2?gQPtS^w
zy45pl-x{-Q2y@SkJOU76{Wbleu4rGnnW2&wIZ*WSh*LgVLUTJ(b~Ap0_jJ?D!$WO1
zBePqypxWX;PygSXmV-(2-@mf*+0NdQuCxExDoww)v9{@{m%g<Y8WahV4Df#3L0->q
zc&-okJo_V6Lw4#wj5KE|&*IO#=7RA}Puo*a8s7K2_ZcNCCL3cS2AGf%%v(QWj_2Gg
zl^vxB+H;m*MqVgx`?Ehr3=rXK*TMLg#$5@DnZsoDBN>(Oi3Qlc2d2SU^p539_?|Kk
z>G?TNf7z<bH|?+)n#^RSAXrrivW6?e51pOJY$-3EIS_Dl1cpY~-C`f;e_Ix$XRdvl
zK%b5S?em{fCj9a1GmKgi*;jDfbbLvmWO>L|QAyAUV&}rSYV#;#tXufW?Vt9yPK3EQ
zstX$ugd=WzkBF9$9RzsS61WSinlX`C1-K7cijur)HER<v(rX>6FHyb6^-YD@=7})X
zpkwe-(s~|z;*FyfXHnl{Q_(J<XvoL8#`Hk;bzI@6e$s$RX4eLtVc5Zmjahp_b{n2)
zUgo^z)DP!%KLz)ch6qZDKbK5#fS8~jG!V9=Z@#*`+R?ZxGV^9kP67-OFz`FiogaO$
z5e@ZIodKB}(-9gzFGs>qKKftx(BwebM<|l55C1FuK<@+lJV}L*Lv%zA=N*`O`4;F~
zqkm|Yz^(_E1B=QwY9Q_g-m&m1Zrutb%<r6OEvwq55g+OaEnC*}by7ojKvcn|?g3W0
zP^N-t&w9LG_h^V$xRBY@(LQ4#(|SRKih{JJ7XmFF6vLHhOq@Kz_V2pNKEpi^6{}jG
zXw*{gmNOm^@nI-^R*^YusPX6$ewUh!^PRY$%NH`*&mW<BPjCNjqipoUXBY?(XyYj#
zU5Gt6u_ffmA?>};NnCOhG-nITjDCiN;Rn~k;wwQ%U{7-g@nCz`VOn-eQ+i&sn9Uf)
z-$6Xbci&6~DUiTVMn1!BJ7(g$_@#>B<Jdn`+Pi#3d;0}QkGN=Wf}0!M)G>xktm^21
z_y||KNLV0X`b0m@RtXoWE%@iPjazQbM}OAf96PYj_x{s+B~Q<A6RQ1RWhkeuU7?LP
z=?uQorQg@M+B)GNI*qCNM|0-UR7a2uKl8|>b@`+Ij%2@(Jy8kPhX}J_oZqS*xP;{P
z8%iJVmk)+yM3*KH>P4&~<CWOR-%L4s`|@01Q~Ew8Je^4OJG0HvtA5Z4ki9-EiaZj=
zP5PVivIOhJ?k2ixw<LB2NZLWBtLU$cj&z&skt~x*a#zG`_wQGOMnd#ZNK3)DS|5^o
z$ml<B_5b?>-gv)VN8-43^Qyd%{b`pExrjL1&RUN*Kc9plstJc%Y`kF(#Ir|mal>+U
z_q7CeelV^)B4cgM{L_$_emHm!_VixWP*LkU;7+rbcNlvgnJx3#+2`D1lhw^SN#xAu
zDn=0jj2fOk@Z=0sd+FhhMo(19wIAZ5r=D2c0rnFcK^)BWoa^X#gPNj+K5%Q4s~@jo
zxF~J>WZfq>;?cgtrs#t!Lm10MVBHdHS$s#P%e>_$fdL+c<j+B8(+`I@+Jo4Y0{q8a
zD0>^Gr{*}<-_D_QFjj!C(9dC}hV1^ZN~FLXZvZEj!&o@GbIX1582rWES23~!fVO|a
z9~M*R2QJNDJTgB@?{>>|oL;-HyW^9fCr)mi?2kHzdbFS1<5g`#m-IgRN;ae2DY8!L
zWWywj<EGp?b5KFu>i1+0j)rg({wXdxC&c>@pKa6fm@>qB?p31Oq~eTpTi3xJ0_(jb
zrsL)8Nq4Ld-tTQO#MZ)kzMnmL{|L*%ZvzNQvkut;t1eh~g@hcafLhGnj!YX>1^oCB
ztfm;By`T7COZld<G`AGhUMQPTJ>un{dG9P-6jJdJ^Uv><2h~)C%Cq?kOe`LM%-J%%
zG-=K9XvwOqVA%QM!Z~aZyMI++f}$~VX4s=K8QR0`^ISxGb&rterg_z+A`>3vFd%SS
z{7_OMU+7Kz+LT*~Qj*%8Hs46l%hiau3KbzLtIZ3&IKmVS3iOKZ#{60}f`nDHF~*=8
zU6^)al?vWaR!?nOLOl*+5#3-Tc(D0mC#lh{>8^S2bfH4U#q``L-QlJf3|@8lP^$}H
zJU-0J6#a32;8%?cd$P7nFW1?83k6meV)Mo&NP+L#kl2Nlq}5{$a8D*~X@Lb*_Gmh+
zDhE`DmQc<cchDI2sjARe0iz-XidD^4VBHjblizwi#*D)cSTJ$C$9qF0OlN3Gc=`Iz
zU%0BbuRLH3TXUF#X73H%zb@~rV5~b?W)|ao=;xe&!;>p3K4Wu=TH*ApnxIv3hbgc3
ziW@6&N3i?);G(yYd;1GS#V`RG#Dd7Vu|)mw&80Em(>3+`hsCEh$M=I@HD6*s`3pav
zq{wLBI+))+_W4x4!Li`I-L+XtuE@5xnZ}@v&H8lNpVg#JieK2DEf;>czsdiDsijL*
ze@jmkdfLl!)c1z($`2Oqg=LYn(E?fR*?+ov-J;O9X<?SXTE=lxH+8vJbqju1p0fBb
zKKaBRoNlqb|MUG{Ma_4{3;KN{@?(lqxuVd{48)wBelYRf!^pEuqTl^XD}Q5rJRpj+
z^<m;ug)KP^XAj%cMkDjB-249__~t}br9Wp=n<I!Wr)iHfx&y2J`^OSGjaAZ&p1B=m
z`j+=|-`y$WYTnQD@+VG8*9Y}?|J1$lA7ZJ8zvyg}yD0A6(V{Zitm3vc?>}GTzg-Q~
zU%&&a7buFqt$Og?yl)w%t?EY@CR5KlifBztW!G$%x|?W78aKkljm*B_Vj;4w<U&VR
zdr&lv#YTN<goy6x;sN~iE>@qA*TkZBQN`4*@i+1!CYH*W!ZMi<E39yd>9@j*`^;^8
z?5maITRs66i|`5GH+Mu4f<Y)eVgM-^RN9WRqlH1NO-;&nWv`ebrym}bcST505h@*$
zcLAYtT#CC;v|z=&Z@*ixboKU&-~8vne@?>xVI-jEFHwEVm?ry6mrj2J|Ly(t(5}Lr
HC;s|hiGq|V

literal 0
HcmV?d00001

diff --git a/clang/docs/coro-generator-suspended.png b/clang/docs/coro-generator-suspended.png
new file mode 100644
index 0000000000000000000000000000000000000000..278929f3e145de98d555fb74f583bb58f8247e95
GIT binary patch
literal 670202
zcmeFYXIK+m*EUR1ij;tebV5^9r1zG9AmsuC1?dnF5Ron*ASECQDE*QuO+`SZ_fBZi
zi_)b9=|x&-Nl1AoUf128?>N5Wy?=c_z6XwwVP@}{*?X;Zo@<?Z=9SSMZH5b+7s$xS
z7<6?s?~#$w1dx%PFQuUZ_OPO<Dagp^^;|SGjC3_L_>7)-K6G(&A|umzm10J1Zrpb{
z$Lb#cIT}s1KQ;`Qk}j(KK_eP=BoO_F4#kU$4!4Xd@?tKO+Gv)4DB^p~!&G5!cKPeB
z<z>1XBHItzOPguQeSes>$|L3d31=f2@L@&&-E3vDZ?l+tE{tDj)k=j7FSGm_;Y>>&
z6!4=EP$TCFAlGYE6I<21{gTq0Y6g+!FHio#txWM+4dDk77L#fu#6}jTsh)K;neGWc
zgobQED6t<!#`g!mCq&lpOIH#9WRf~Hf9bs?dRC8nOERpjqs}aBoiEX}WR89iSSop#
zmi&qZ<~jW*ji~Pw!mo^OhEIbY<j&3Y-HKkn<wohgnB|)fQQo1>jzzc>Fg!+KI3mnh
z^<eze(W|4}gd?If53y-w?HbiudE|Xn<I5NGVma1!bWc_!u9!xlBZNc+E<SK4GqS$<
ztIF)b!vX;zTc?B>nicDG>9!XEu`0r;QzJbz7;<|iu3K4ReXnTt>aOdvQD1krBYczc
z<`rjrpc-AIiE5h;&19+UlcYRc&`<ML#N&Kgi$|#Q3G%m}s%YmYxH!KKz+=-L)E`<4
zbeTj`qQ25HGr6j9^N++cY}e^t?_JTNpYFXR*>8T6FZGpBJK;u1v5d{-OJkl0X)_3u
z@>2oh!I+I6vqP!nTP6)1IW*`HLZ^ATN74%x=sUqb6X?Lvx368GG5H$*=JuaxbJm1y
z-9wy%iG#PY%N-Mh>aKV{ttsjuQ%~Qa19kV>di&SUqdhSn1Mhm@fF7W3hCWK}2M3GO
zP+K#RnX8eza;DQ)hSk>;wbCtoe)gOU+^a^#L>?9T&`$FKEa9B3gE~2yMSwXG6o0t4
z%^Duu3+L&$p?JVo*r*hCM1bV~As2ALTa1QMao_fPp%BLrAGu0+EE>#vV9(md{>1M7
z2{9-%3_t9}$FeYTjq0J}cj}w3!n?>G&`3r<is4#>)U)w2mW7rpExXM*@Wt<)yeNDj
z*CE)!a7+2;u#oqAx9Btnb>(r;)`;Ou0;av>U4h7MLXd{vr&wuk%Ehpzwm^k?@vI-L
zm2;tp6bpNX?*yxJP?h(g%CA|@=hNq*@47|CkC%^m@tE9Q!}j1l1siY;%KMi)Z{X&@
zmEtelr;YU7n4gj<M`d@3*%_Ft+S&J1P!*q7{?Zj8va5@H&6O@%Fp}%`N!oWP7+%@0
z-=F<BODVhc6Bea8hDsI}^sn@7)Xz%F=vivx(db&zJr^%%K741ml<e9mcw&9Wz!%y2
zT*9i==MYZab{632xtFwOz2QxE($Hzx@pCVWfy+bmF<BP$rcp2IK6hy|(_&f#BN7r~
zM?V<O5Zl91-ySjyI``a&iFQftVmeLB$ct@js)FYkcD&apuGlG>Q}(xCEoU_epSOd4
zqBd9*QlJwH7g?mLr!zC84j^xASKjAPXS#SxgFPDZ;JLZLNwjo_R&b)45cezfKNC%a
zD3kauI9`<kUspF3Sna+1k@H6sx7Hm2wnVuVMikSgrmbME5&eB_KO@2M3p}qX?=o^m
zcE4x&M(=yUGE7V@?Ss>!D~)s|8ipUzzHvQL#!~FGlNCftN*WK6J2+V1yr|bbD#;<o
zCDQ#z#UI6&U0;k~I49%%?7`jw!)+Gzl-BM10V-l`g^N`OR{|7mUrzljJa9#8G;<NP
z-Z9hDo}4qJNRWt6R8Rlonn3c!S{6p;;Wq!z5&Q8vLQ=0LHT}_ISDCA#rK9ztxuS!j
zg`?fOdb)TozWQibDIEAlKsWdv@8{_1`05znEZ>xIb*!4HU}j==pZbbp<qK;au`7E*
zU_saws35%H(Y>43bCPvlCrV{k##Y9S>NMzYz6&mRe7EHD!nnfY%YM8yv6gzO8ZFu(
zc?AUz3)bFWF1c!&S!7yJYqrxoqfwKvHM8|}%li0|-%IShW1cPH@x4!o_un6x%0Mlp
zL=$S0O%0T92dSOCYyajozUV?DyY|TB(YTBCM1XGLH_=UQtmZTFmgLZ|(2RwA-S5?-
zE00mK&aS49cqg0lf8>4r#+lFhIi;vZ)3tE&e98KZkh{o^n)~|`ik*^~!fLRzM5JQG
z0F%U;bgksDgpzNx(xj3~_RZ{z+4D*zO7o2~jbe?ojbpwdz6o2k(vk(NZ+Zsp#T<5>
z7aT)5AIii^>&v(ul+W0kMYT+Em97^GkBUsr=J{*vy@BXKji6TWm!U0$5X!l>VZIr=
zFYQ_p(o{B`evv^8Z1m#1t#Gtw<fr%9_OU<dJ4+%(BUQD!wc54r<@sq9#x}-gbcc3Z
zbQi@XaQMHK&=1z{x>{rqpNVT9b@Db>ES8#dF|GUj-X;2Thi{Ba(WbMsYgItitt!=r
zr$e9D+zx#YC)S46<}Y8p>>&Eka@-vEP`AR->Y-(6jfyj7p?~Ja_s8$=w@moBs=DU*
zh5N-Ft8%@KD;v6$`A6n<t)^A2d+iR@tjN<(*+n)5HZSL$g2oqyY~2v;t?uXSU-Z!Q
zMfWq;agXyTFk%>G89_W5@jv5V#vjLvYg@+Z^ENzKX-Gn5S!X}Vs#Tol$ydGMYx46`
z_uaDWovic2tiw~<V_MX4Ins^xM<Q%NY)J!JX|tvDQx&CjHkmev#)hW8xySQL^P+Rc
z^HWX82ixU&(|XhEsx-U=ULTdWDv)Q5fop-gr?`QEc9B`v!l^^aNe4}b+PJ`9?rTG*
z3rq96ohaSoAHT+SUAG2KOO8TToOgwfW_M3x9R~WAel0a0`J>V>U=)1hcx8Agho+6<
zIpu3gOUfF`Wy%3c@VSihPUj}h`B67fjnF`tZ_&`wSuj6|aqeQ-{}MR=e)oOVEyJkK
zQC}ldqEs)`(WSkVkH6g$8E+$dV*A4Bh0q7cMNO1|+3kgd%J{bz{e<4K^RU`5<7FP2
z3=XFIRK!l}Rwz3lzhZhsW16@;S<0e4@4OTKC}bksY)G$ToiPpZSgE&I7Q4bB^y&@g
zpSpdteY0<Zx~bz~(yfADW%IACuASSa9lL6s{zyX7dFi|EkuFY``X24)SD%%Q5&MFe
z6kLvs?F_csgWRm#bpO_du(jx~OMx+_uXFN)3d{82IX|;2ZsbXi$ydo#=?$``#lW0i
zI=c5i%jL;v5p0qdmUK20Q<RrIE!VXAQ&ds)M*yn2&;4GIOHy^n>-A(=<3=a@%AF^g
z5cP$`LQ43N)g`&txtBnPAt(_PRNo-|ncdq-nZD}O&8K5*hHz6Rll=ku;7B{IL;bOg
z!-tTpuNgH?a#G6fPY<~d73tP|2N`m#WD}IcgBHqO-gzlPha5J{Le{rEYM(ow0KY)^
zT%ePuW9sYreyYPDX8-om?UER6aoakyRYf`a!7F23m%g+s&#$n-d5?X5jI`*DrPYf0
zSk-zfdN(*?hFDugzL-8PxoM6Gn3%rrS7cEXV}`Jib{TqkFKyJo^|LFh>+C?s0OzDd
zu~qA?Uf@Ga+y&xm)oq3;aa&?&gjeT8Tqu{TJcs*@A4e`#0f#%H?N(biN^a=YU94O4
z3~3a#pKX9mBTPKyr)?|o+l+njCtuRPj7{uLOeiz>ixsK6UXsmjy7$!P2Tys^`^ooD
z2YA<V{%o-;d$0#SyLUQ22d+h6$j;Pu)CVHAL6VHDQ~Z8o+l|}O+p}DMa?UVD@p^b1
zJy!AUI_QnRI#%#HuTeXVCy&?4tN1ja{UC>Bl*L9^`VPb+XFT|Hyxw@qP&?N{mF4Lg
z?#H{T!eQ@W9_U_3>*n+Imz``s*tW$RY`(a9yDJ_QZI?w^&RcGaHNiaUm%Q^U-1TQG
zidxMKZRzTto+#~M^SPMiXFR%*%^@PIrf2Rych>!%>^5UXiFG6X>kEQ&BCTbCh@<eI
z5&cMaIC5-vGW5v4tus}-B4HPDStay9ldz)K^SrRSFn>fp=RVXwbYYxhJa9GP0PfqI
z*ovOJe^fY`yA7LZRzasBVWRWPjmoxtw#TiuC@tbpe4UtQX3&Rcyv+k4r<jlfYCPyZ
z_y>3-<PY4@ufuWikK#@+cZ6Ih-PzMJ`MNr@{<MR$(DM@SRAxR>?mwd@s}>?--=EXr
zDN_QUuxk1h>}V`P?I$=Y%j(IE=!lznO^e7OQL+;c%C-wzW5#6BAIO>3$-jNiquy?_
zDBYX;d{Z+(Zkih}D=N=`%lx49hOl`0?eewzHf`TdVEEvOo1U&CP@uYa?PRX|(7=G~
z2Cz*-c8;8r>^!hV4tyZwT>sqGBELpP`FlT5ibT1Po%_4Z9pIbvlL&lBeg68U{4<h_
z8u;%b@CkTF@sHLt0q-dPv3<T2I7X&!tf8w5d>cDFadPtTdF1H}Q&~R-cF=n1Son~U
zUA{v4kn7$P+6T@*?P6l?Yi^*g<lyNpX7A|vz)38?-HUV{GL--&VAI{n*PbuH-Oa;C
zDL_@=cMBz8o3vY8fbVw`UsqKDa|0tj4bLY|d~#wEViE$73w(ThDo-3AD&5o6`nx;u
zpQ^wkUtcdJadCfte=&b)G0!K?;*yGrisBMd;!;wgKnqcyKo4L008tMg!M{%OkMn3c
z`8Ygr@$z-?^xz|%*ZzU0pRcNb0O>;i{Q2uXodR6`btMm<zlQ}3P@Hr`TvALz{GaCr
zx~h=&DjB&1IJsG9y0`;#23!LoDI+VR^1H+T<Iuk@`EOm#|J7AWQcmIDyZ+mu|Jn7v
zkJA$kPj}#=zL0+z?C;M1e(>*(D&nNE|Jz*rHPOHK0y7P{pd$WHt3fVYVAK!;49V@H
zX=nm`18zq8p_l^xT>I-A*gl`;KqeNoPDZ9irmLxL5<vc|C9wLb+5N&MmQneSf@(@S
z8fr>OWcd`d7n8J@qDE<w!`^=Q$i!6fuKSbZZi%<JEShrQ#-;PuH}BDqGuS3+sR>+*
zdwFylcAlSsKQZyeQOJ|2s?*cvJ(aT{LnM4%eq`F&&u6IS#Ix*SY0FyE)2Wxw)tG1~
z)cI)6h4V3;``>^5cMkqP_5-`vHe1mLdryBZIGG99(;_yzGN_=AmJBRqDmH<u!uo+9
zxzbSTPOmFM4<uu)4!U8T1u>pqYx<?;>-(eVqh-bzmM2m72-YfH+QA~Fta@W{MZvvc
z&*3w<JDn$CnFQ(!W3_h=_;SkeMJGkC^EbQiLmbsM;l{CFDeIQv5RnFK$k#>YoequO
zDG9#U(A4nRs2%dU+PZ2)LqK%Ge|Fl0*Mep#J+R<w`m<VJBLX^3%4W)Lq!w_QVG_yn
zi3XCFL36L=37m2^z^y!BNFZHo^!*|j=MJu=o|{=1C9Ey+L;kZ<KJf>{4OG7ge<$J2
z#R^)YokcuGd@+CPVZEwBl}wQTYOdlXvVeA-_B|-6e=Vy*x|mMXbNGH;$TY8)YmnYq
zW=s@%`ae3=N#Z@#l!*~)Oy-x!se-9R(+~9S&@(>l8W2G$K<%kDY?flRBM;C$m=sq|
zkACWx;m?WHLt(x3`03#<F-VEPQ391p8-=x-@ryB{nmu26xD<h2VINuhK$)Wsd}iTy
zz@?>yg<-)v<Q+8z%(XT@(}eLL8LEpS-ki(Y(}aAztOFfMUZMixi9sg=;hluher_!X
z6VP{-)c3043^I(`JS@ijwHGj#^ylP^x^u~?3A3hd!pkQP&?2$W&R!j&9x+c<t-khp
zN394B^^q9ft@hx`9S0oEEW-DdH1GB9W$?OLsY`u3_!89x!k8ST0MJ*N)OSVqjBB5a
z&a1+J`3{ltWq2D^KU#XR4L<vN$8j$7m-EC|Pj-$?_$IuXnr2Gw_Vm-2u$&|4H+G%k
zE|*xa{aKorcemiz0NgUJUzR~09nqmS%M9P<m8bdTO*0<OfLLiWrB<evQu1zD`FeaS
z9@q8L-%E(T&k7TnyxI20TO4Yl^59Krk^V}rAvYlzbi-KX>z5@NFMY`?3=o~x#oQCv
zEZL4A-g86v<ns(HdI0_AK>iTa@<1tf+APbAMi1RCayW8H-200|*>Y)V`tv;C3Q^P-
zodZRXZIb(KhQJk=VdA5-BA$bW3aixc85+m4@|Nk+*feTFHR8JMlF3;P{*IY`yHV)U
zekbLts^%2WMfQM2VS`XibQ#;`ixl}a;Ve`H)css#$h57SS@!@Xd<IFnV#r8np-6$<
zHK;fGXLy&zLchEhy~_TZtBByPsTkjmey&57JDB!dISRmO#cM^DSHqz@p0CpsCR&{l
zW30v!$+-%+eYK^Vo~v96RJ>^2H!#T3_H8Bm)i?}oA#Tx-E8GoS%Ql;GfNIM<k^icf
z^%z-&+<bkVz2A^KG=R@LwGPF=O$ow%>=Oqc91CQZqq=b~j6?Se<J5rAi`nT92wXHg
z%Urs0r)KZRp6}O7^sB;*P?bc50V@nud^mOJ=t^1w`dQK)^FW8tj@cTV>9G;hG?r$1
z^H=)QNzM_QM96OJJ`-r7n*Sjdo@--^8Pf0-?UCv}cwrzdj!D*j)}ITZ+y8V4M=b*k
zkpICLfy%9|3nJJ3vfSlC8X@hVhoE`Zi_Sclst7+Zx_&qk7KzVaO2#<1QR&ySYrlI<
zXFt{z?|UGmx@_1Do~7D(e-H3&hBLjdpHHOE*ym((%3ap3*aUtoBm$8y_Xw0SN~Dtc
zCC*wnGxE#9Ybv0lAS2CpH7SRXMsS{er)u&Ck?UJI!X)c4v?`(9W|=ngq3vjj_A?;*
za@`|<I{+fRlW@=b3w{tn<sT>P_BK%0jCYgd-L(ue8u>?v|JRE@eMT%Y?*gBjWF=Sr
z8PFz|({Z5O#`17KGB%AZ=k26)4czz9Qa0I&unp8B`%-M0@M=Vt8moikl`H5%{6uX<
zQNu@~ET0s?5$9LTJtBJaW)PYI&w~$6^khLdxV|Ga9ZKVWrr&yeRMs#ZZGJ@D0Dnh(
zfGe765uZ3_DI>Lr=`m5)ZWE&ktoX473A3Ij`d4JJ>(*|XH?ht~BOAHjp*$jaa3!hX
ziMkK);jbFWAK+R<YZ3gI9IHG{F&%}R36&_d?^0~xw=eX}kyGk-pa+SV=qP%T2rmQa
z?D}Nr*ZbCOM!&ik()4b|MNI{FbC^a3uE_Eb7Jnbn{nd5pe~<kCU=Bbx-t(3JaqjVY
zmGSjaMzi@X;WGO}K1wNuI;^fB$U9NY`#StJ^>nnmUlGjuXUcP8*o)~!Q|M<q9qx~M
z)&Mqw%0IggdBw4qjee{~kkeMQmxSIMBNXh=yS+O?#6mgXh4>DL<Wp(F^Il(i)&%~i
z8ye{CbSwNvCtP-IRV<=9niLt*#D|3J_?UQFhbc_AVwXg)(Ik|T(Qg<y*;hvR8rY~8
zr;*u<mfiu?zN**`Z~#lugOd-O+up}3P8fcf{yBoCyubl}#KHkDz_-guKIQTJr4jiv
zg2c|_l7(1ZR%ILqT#bJj<Swv0QVpHH+FiOF7t$*c{4Ga?K<}SURJS8Pmu}QeN&D-L
z%pEOXZsml)$L5GYhE6M|%``&`*##LJ$3PD4Y*!AffqbmBlNkW~tAs-Ka5@u7mb2F`
z`-W#lJL(Ac9<Lpr32_(})}0%R7&}M}B?rd2)MmbPMJ@v_MI2YuLhN*UW&3=AwUWjy
zMbuyC!k!0U@^yrdA&;>X(UI^OI8R8=f40fAAJ2~51Rq@yKQrh?#v=S;Cp^z2tb69R
z(!`fNNqc@@>$^Cgb<e^~p%mdEu4G3Y?k<gWwHk$ujS>s+4Ay+!e-H=IRmlhbk7P06
zb5Hr(-dXmAzX)j6*a+KE69kr!X7=fq<1u|^7|ZhYyS(-~9M_=$oaGapa2fG2Xl`+7
z!kzWkN@dY{Yq{^|r)|AN?tFZnwE5!wAcFGF;-47dLr@46XM%9N^vP$=NdLiH5-rp|
zm~9(E>+QY0m>y=_jr_~nQkT`@Gvd?yT1%F{95@gL5gBk-<qCrzIasaQqU`oG;8ccd
zxr5Ny^WOp>-Z|d-pX~#BaHRQ~Hg)5w&wy{(7*nu#_2V)z@`ro<>qwV`nfDN?r0hTL
z$oR{UWw~`N-BBI-x@Six&ZfzW@k$9tmWQI2%c^Hk9JmzS>@1;I(kf!P<$m558(LP}
z7jlyI$Sn2+z)&`mJtQjH5Yvu4Nj#B1qCT?*ObMR7)s1{F>PW~Ko*&FLT0;UknRJ@w
zZ*-Q%$~%y=B_H?r<`^NV(uxt4gi2LMe9EzzjgKjO5^Mo4zdVE$*%n9g<U!S9SuXnM
zPQonUPCyH`-yNrXvrLdwY~6P{(M7}e@gy2>7skh&D8Yz@Hg6bztnq*!XMz<XiF3+q
zQbWog_0L%C55;AO#UFzQX7X&v7F;wn?C*BA%t}0RjLfV%<(f=kW9ztLAQd`Su4_gW
zI_jKa*nAv@RB8Np*MES+pMI%frQOh0hXcJX0?tW>+MU1rpnwZk=pOSl`9$9T0m(<+
z9QZ3~ITHGC19P;BQA-H~Nh=~{-UjSD;t{oN{QLUv&_W-*2tllKXIO>Fhd3`My!@ga
zk{mD?<yE^Rl1YkB4ZlM_z;6??wrqzv8owh78qXin1ZWb;UWJo(#}ZU|1V6w*KD2oe
zJ~V_4<adPsb3%z&L)OViIzJ*rD@_I)ui_WXpKyR0_jNLX;DS`Vmz7_Bk?L2S<vW^{
zXgdeZX5wYI^|&#P&6W#PdLSp(kr-TmPZSJ+4YoUAx_mT8-1B?~?s|wO)N^{Y>tfLP
z$>J9~edq{dPu~EhCH@6Z(m=#z7v*Xa)3;0<LR`7E28;|dA9MR08}7#vnDH7ds#vol
z@kxMi^8kj8C2-(<N<%N<eZCmS9}ju6XG>$(!^q8b0*Tiz1`;!lr{)f1w-bt?m4sfv
z=p)6@DF^(k-TTMlxIxAGjW5H{7gJA&Ew3_}+=CK?+vaE;1%KN1oLaL%ReZTJ;^OOP
zHr%3npD#wlmSHxz^^;S}^wIg=nB=IN)`!e%w^CPSJPnsou7qST4ggvFHM}P~lv~+L
znxMaau6tN1^dT;LSTO@$V6Lz}3O`a7pE`L1Si^G-Ifxh*BpF1)(Bw6cQ^d%Tk!)ad
zti8_A9ZQ=1LxT7o)W-RQMe8i>cs=Yz{rG|O_FgffiZG0|QoTXi%n-iegm00ZL>?Ke
zd06MX3Q1Au(6b?q<4%I+uNI`e6lq8F{3(9<q>JmL`lnw9Ozsj`cTI;nMzZB7PxhP&
zZZ76hsWi*bRjcCQ{(O4t!=DAX&r)2tbs&M=Dt=05S}TYiC2-O-(Wu;b<$y)bounKH
zFME<;pjE#Qr|ci>yU>eMUZmiyP5LWcx+7ou0d6R}yMZ*mxv!Yuh?n#a9l{F!WBwi3
zsmDu-Y`@MW0ny3Gnn$wAC%h2$)MOoSzpoUqqdoEn($06{3Ld3B*nw8lpm)pPZ>mNr
zY%kAwm|Ik#`m6?iIN;{ya&#8V+?l#v&Kt5L4i>}U+@784sg#*dXf<SnUdcFJOaOCq
zi{=N%uLyx0F})oE92nbF&k)OdxeqP4?h_s&;r;7<plXgCadb>57B++QEgnHPNgk3|
z0xFOQ4Abxk`CgtBaaGLQBY?3W--#qn90CB2?c4`A32Q0{d&E2-bS{RI0`UW!p@kpY
zvvmaO{xkm<)N>qXe2V;>d4@bp0fDNs@TWIidt2g&%1my{R_Vrm^n4QWi>>v+ZH;o1
zspC*zE?Q8^fXaT-!`8K(vB%AogAsZWnPtHPqn|V_Rz9yTmql{F_<6h4iQiRqpp57G
zDmAeh@sEJfKYl)l@bSAk(`~s4{SGbIxe0_w*<UwxoA{a_o6SbL&s*}9hMItrWgqv<
zG{c9nu1X`naq%zslbxukB&Or1EVjj$CLa)+w!qc0XE}|nKBy;0#H$Z0olB9bfh#5!
zANQEg*#%Jqg5{{F-*N6yR@?V$sA%RrSFw)>SWedC{K!vRsV``YsALlXW0EPMQtz#1
zUm{Vu#$v0>$U2)Gl)Yul+kIy3`O7z87=X^kY)L*AH%cVs=0iX!+dI9%4X_B7+MOR5
z#>fV;le4ped0rRlK>}|lylxpg_B{4n0GgnV7uen-&n<?!T(^Kf#AhG-Pa-Yg%Fqvb
zU(6ojRqc|FohNfiM;hMZwR@&Wo@Ezmvv<tqb9!$Q5$n1evmC!pygV^AagcCgg_K?t
z9(k~ryTT7$v4tM{#w`fDt-~p*viHKN9hMJ%qV|QBnhn7P-jP3L%i><w&jdsZ4|bNV
zCY43HCr*Y3AEtP=c2`<qR$%U!UL8z3bSHsV?nu~r{}X;v;f&7X?-ZoM=gMI&{`xXV
zP`?|DWl07qYQr_;W@9_#X8;FY@b{2^gWDHnZ6a@X#A-9#2g2I9k=ugh!3Rxo3hluP
zmqkLl2p93v|0q2QmmNs<K8o7N9XmDMpts(-yT!Bx+G6R_s5;u?cS;mQVA7-se1)m7
z)nT*N<IjPc>!}G%C4F4TQu+7!ai{i_KSR4gy0+F}Zp`3zrzut3<)W({1@}WoUFhq3
zUo3VuUjDKw^5*DezkI?Iwi{pz)C8)g>t3o@v#SLm_%s*A^)0T;M*eqhQZkQD0|E%X
zhL}H2ASDucK(yT38;=8^`u82zJR>sUm9hRudmG5*Q@}uu_@ra)NpPvMZxw*jf5Qd)
z8r52FW3E2(lf#u~tc{nvKdPs|^QK_q4ubvbwl-$PM})6;*?v9p*W-ZS5I_Eeep6c#
zO<l|5GRfE8xg7e?k80OD%M_zuR!Yal73uQyCU3Jh%ACrj>wd6-w4o?CG_a3av@qCd
zyR;y9I|&u#yiv895jIr3F238?4qx2?%$J%<$OT3MasU##1|@^FTzm31ZaPVr!rm4M
z2?pxli`grGR`Nd{vt5}O!oB&OBq0GPgl<_g<3+G(;-M2F<H~-25tZV^=N(}`?h=4o
zjf(+{t69PS?VcIO>rbuVQXVBKgaPaNw`b10N#jGb3HYib(9s)j%)YzG5>>6>#IxD-
zb4VF5ll|z5am7ljfkQi$;^(D8`=mk?i(wWna`lP|4Aa{!(D$sJ?tY-2Xf8@<kB-gq
z4-Dn{K6KSE*Lfd2qbk}dr&@$B#2W+|VUg7(VSjYzzIRs`R_+-;fX_UWA~*w(mj_7!
zNUE&y_E7?63jCl;<s#shQ6Vo+nDBI$3buc3z&hEP@jh6B(<A;)BB}PxgENNk0~Yo4
z)AM`XOf0JUYJ!xLBkef5;4XkY{U(hf_&lN`nYx`j>3FJqI6dF+=6xcS_WG=Saz^B<
zsE2WgWS3TN%*x$$??I_}l`6C@^0E;7?E6YN3NXcD0{DhK+WJ#(C#4X~;P8D1B_l*9
z(AB&F_45XonZ-(bbY7|_cj@XIq{_f`gj1#i-vgdxCyN@KQhBixCbeZ6Ux?d`$zOyX
z#nn6_y1>UIui<l(yxF;WfMgwjd`{DcxFPgFKqAKZ80$=8=i2ZMa0~W(+0Y00pbb$A
zKv7n_>+V@0$+}t685%C2=B9|~9$4po`P1;|C}abhnZ-vZVDdV%^fKSMJN{-0e|Gsp
zg76+31^fXB!q894_eK6j?)@HXJ-l&9?N$@oHC@RRUw1yOm?#j{TtC-&?+og6Yr$0%
zu_Wu|{+(#ztDJaz$(+~Z_;7#a`v_&_-6shUJ6y_&8E^kL_noEii|Y{{3yMzth8+E2
zJx|s|5dCk!7X7nz7*|a1n-z-<LeUfGHi1n99!$IrJ1QKxh93F)p%&nd%_Swg=ZybW
zy+T}ee0ID$2b^U4%q?&RyfE*jgC=PXYD6`O>@$?AS4bIGcTY+wU}qAau;K?!VI-r!
zR-0(}0wo=tp}h1r903SCxBwp{%t{!dVe{%C0^1`yEd<A%`nLzUemzenb89p6x4?d?
z8eG-TKf%z|qRwkgw~rmCPe=CMT`nez+Vt6^9Fi|y@GU`jtMKt&0*yvaHn$)cuO9RN
z=@Dv;IxBZ)>Uz`jvMjFr_Hm$rbJ*Map()~JmC5$_K|}1letGCIz?<bpkX0)<&Zq?W
zEE%>62zr2s$D>CgnehY3<vj_rl2x49!9QX)H*OpuO(4P0ly6Dafjp^#dk=NQryav4
zK$V06EXam`8~L6`0Wy3_Xek&uB{ZFs6%(rn34ddn;Tpv{Xlh+qo!D42f236lm$;0~
zA4cq2oCRNgOZ;i^h!~_o?9`hJ9?6xH5X)6K3=r>fw?M7cK0bf`4VL^`7)wuQc=ZVi
z;&r|m!grX#meyUuy=*n!rA7hO<TM%3R#dyXI(BzoW8V^jOSRXo<hCAr>p!f@Tt+|U
z)dEJ{gY4cLfDPkE%m`=#ph6m+Sy>VdiCo<x-SonftMPE44hNDO;voF5(tZfnlzgmx
zl&}FFWFw_6T0~x~`Xsmt317vJ{5tg{k>+RKl)nho`5(+G-b&nT2EZz(ir`kpV%kEC
z<vssHY=5MVU^o9**;0oHJm%tMvz#&V#LK_|mR*in%&sBY9>aI2*oiGsYxB>#;@t03
zJW|V#f9{d4!~z2|+LvPa^gX}7iMx&KjR^j6ci__#cN2i5hL!~5Ra63SHLWt|1}1Dg
zF!vX3t&RnxS+$~?%{rFPkkP0T?S2r!1>Tu%pQ{fHv_?*5h=b@Z9M4h20a#go{l2%s
z8JC1EAkI4khdqn~lnAQqE6aP|AX@<y=>*v9o9)<G#CLdl5Gk#5(*Lxq%OlP$Ose-u
zg(ZOQ=0-g+8(=yFykyzssO&vap^TLDG2<O}11x*!&UF8gVR;k~;&SRTR%Z!TvZ*ET
zFJ)o{@QhMrR~%0o2(&Ey0zx@auaqM)g}qroY^%G{$F=Y{g=>ngjX&&a350zqMB(Eb
zDhyMf6&|(qXE$X>P1ahr?*5<|ghQt=LLL3y8<XmQsgFtVZ0Q}S`Gyxa4Hac;7!U;$
zlA()}?>2Px1LfM`ZRc*j^FCuM+7c@Su$`n2^r1Ty_!Lt~Db*W{{|0LYoSK^iWibk-
zrt;XSu%zvq>#R1*o5qWm0dS|@QL7Xushhy?Hdwzyserbc#7`mZ4Ih9~@dNz3*0}V9
zh;Lj})KeB9&V_f>p4@yN0-%G@X9thtnEQUxfWVO-r!YKI_PbHln;w{i#Ss1tScPpH
z{y+@6%MK4rzKspQRe6eNFKl%<u(wKD4A%!^21RlYGI#G)l<17pNstfH6$x>smTrnZ
zzSFaEM1Q#0u}&!@&JczU@o)I(&8!7+#Gq4d+r~Tm32OKgMce7jYu!;Wg(_LE5ALtU
zRb+W%K5F#$K*iv#XUH$BKA=pX3?wNgL%eYL2RM>5kbaF3#?-gU`q0v5L~9WKZ@q>C
zE=|~NjUNG~tFV>Whl{B#uieW{{HO9!iubAlkU$<lJ0R8SgybIzVn;1^iTvW&sgntW
z8{2U93wN;I54i2lA<P5wtHBt0c3h0og*_saQ3>w{?$s|qpR6~*(bFJpMgnFAy4l-y
z49Tn;iD+7l=KAr63{@PKy#C!iGjO`g0_DNQKt3)iG4GTpU({PL??73lD+cXOlTvqk
zFEy<m)2|)xG;$-i85KJ~9pi2DvkrppqZBoPY)+~#RM2}f5iQvQOAz*s`N2jsq#&6-
z&v-<OrEw(c5HnQ?ukQk+U*d?%z5#piGZ+{z_?{Fo&3ocN7Dz?G$E$+~Qf12E^Aboa
z0cm^7a4==dr268F4k=@+ru5t0sw<ferJsH3^%oz=9#LZN9D%f6HWNRHs*(If5kQpJ
z1jWhsAK===QXoT`HRXoVOP47iifafZ&5sw$$5A}U$M{~?&@pfR-SFyXBj|<^On+G*
z*R01jXy9V5#p;_iSut=%vY}$Z^{ORL)kUO9<&#XySN8(W(<Xk-RhyKsc&y^G@{)`V
zZqQ?x1&X{)EgabEz<JR7ym~SzN&#n`c(5;?AI#Rb0`S2eIc6Dk-JuoL;*8oXJMK(y
zQ|`XCI^s`sy-#=m<gbCRw@+~q#16dp4)6>3#K^ND1j!||h;K{ad3k?Te!nYs`y5S}
zo&HM4$oYwYDO}ff*(TbCxJ>dJ=gG6OzvfQz_1~qUs(rCGQ{>}jCF%p9-Xd&(JJTsH
zlV<2P@ohYREJHlkcPJ-&+@vev_Q|t&=y>LO!!&xtMISYIigWJcPgrT3uGLTp4M^T;
z<ggfz3&6!3BQPuGp(&mm2EQD$J(RC{jP>?=Sf1B9Zw0m{Kl(|Y&JZrM<nrn1i?wWa
z2{f5*)ED}WwW8HN4Qzcdx@Xl`tf&8*(aoN2cLyon9}9+uclx6Y-vSeqfJ!R7xuj^|
z*oty(4s`SF<wu+%n^|J_r_Rb&&2Ya-MUPmtOz%M9aLz1l1T7te8$r)<_KR8e<4940
zaL^?Y%6I76aMfQ)|7(C9gLdmb4JV$0BPz2`p@GDgrHRLdMI+6`L2MB4ATx6yQ8fhi
zRiQmt(v>K<UDohS>mOp=R8A}5LA+zJ)fT#}wBZ|epn~PcGKNiMJrgc*$v*UXGC!gI
zTk$dY%vf&%?wpdKKXrHc35{`@LE*EAObd>!yI?-Q3XghL{ZXOObFfgZB$QVmQ*W5Q
ze69-$zzuZLKxd^SOKtI5wlU<OzYKLPuX`$ZJTLWgZonh!R_~ApXX_#3N=b4gEGb(}
z!1r2sV2&Xv!rhZ}Q_lvBw~N57$<R9dD_Sw3cnEP3y7++W!#sdAPy(>N-h-69T>X)x
zwr}A)GuvKD5f~e_0F71^(6*Kq1KyG}ph@spX!uBAC*<Doef<94;EN@Eg@n`nNcaG!
z2ei5LrNwDZ0G<J+c0shQ*Smo4u9FG>qcM<^pTXj3>l2rCj>^arHeklwSyq5p3C71%
zgw`BFF)5Nr0@w2G)z6`*Z%^M<5Xf^K7&hafX#P?DRc|O=4b<^|b|E&&0n@8CnSK+g
zG=b`A=Hjir$7Q?!Np0(=T^L<rJyV!BCUH^rW-jPFZ?x}HR`Ots-do3vn&5z~vXPXp
zE<C-}!P}qRwqLj%0s`nt{D3xZ<-?Rc)xjHx1-ig(e5dEaGh$Ugn;t5555~cD_qX;k
za%gx4?v}^mA1v=#CO}DRfTZ6oF(6G*vOX#Ubo;SUP2w<4+D1>pztogC@>^H_Et_>V
zbCG6b0d4?G%!8@Lg0{F?2W0d)SJ*$?;g7FhO9JJT>=VWsEl&YO5k4#GF0hl!n@KRV
z@@n?EDNSm(l>R`uAE;50at_Mx;~WXIIK1K6mk|G|HTYJ$LWRX@5V6|0M|26u+&*mw
z@8w#KY7PTy?6%G|@CA**MxfDc-u^1_l4{%yx{SjX;v-$W>Y^oe1;2-Jbiaq(U21j8
z^b4;KRIc$dX)0;H1(`#5$u_MWEJVK~2q#43<5r^FF4ZR<$X!};`<N^#hLqAZewyOM
z>&PUYu*xJ?5-40hcq3Ou7V5;6b031bd;^IaxK7I7;WLCT3bP9Yjoga3SIq8;JDwYe
z2O62oX&5XVi~NS{>N`n3@Kj=R#CabL{SHZxPoz)mvJ(%~U;u!psiy$D2i#T5C8P7b
zU;c`uIx0?tE(LG`j9v(cE&khlP1pXG6c9EJ3R;txF_}pEWm5U-n8)}U5RSA-(fJVH
z;=Kp{Hk+nQz<M~g9)mUyhln55{Rr#EH@^koqO>8a4Z~hM3(j+2<#P$TwxwT>K#od;
zHrRmt!V|w&$V(rdWZtHTpfd_1tCx#i%T8?D{L=`N#J_(`ev$g;jkI9vE(_|8aq><O
zM(2%Anf=nMqT`a)<+_SCps3*kn+!zxUti5k!_|=7O*mTt2v?Vt<+t}C7$LOl{3}>-
zuwy}R;sp}#)jt3O%hFg)pvD7)2q~bBo3<fx$wd8x10<fi#aJ7&fgC$#n||l;{MB|H
zNg(RS`ml?VzGA$;D+-YWV-A)_AnWglJop&Z*4f_WEF&OrD~ydu%=VNh3}~B(N&<jp
z$_Th!Ncmd~vgYfpJy^POdvwuO?n*{JybTx=Qo`F0C)<*0$Sq?_#Ck;giXFpN+hLzm
zVcSAbsb9oi0XN<ia|nUq%8g2kE(1C$Kc+`&l-#4;nv_T8?AybXUp;|`4mt|>N20>@
z-a4Tr0N(U^73Gx_cZE4{(3wLPu9ssBu}A+bS>?Jqk2@xaf=TnUEi0CbW7;gpK%#U6
z<0YQZ`h;VGSKqYSuF8MEj<9C}G;3fb%E5={Z}Clf0=fZ8^a;*Jyb^Ysd2opMy~0CM
zK|2H5%@Z;H?Al-XNjbt4F$4acvp`AfC}9)JWBmbcFspSmf<Y27ycH+8e{tY#qRBr*
zpLO4n8`yP|o-bGplAu>jebVner9mu+Q*U+1DZ{r*!!Wz6oesc5!RQ_=NM#uV0+fLJ
zxpE8^T`X((0w?BD9BK%NiPVH#IsuU8^qGtsrpxn!MPHoQT$B!m?8P=^8><W%s~h-*
z3+k+mb9;rnLjwXceM7}uaQEk}s(#@iY7I=HU)Nsem(^wA1u91#QKU-0T<G=HH+88n
zCoA43-K#9MS(dG*hI947V_W!6`6ApiQtmi*$1S0nt-}RQ$!JF&9u*Q>e}vea?g$W&
z<A6uj-d%^zTv$Qp0?RWCq-g?jfEh5!%}CD)NR#xBZgR0}+X<VTl=>v68t}KWfZsXO
zBI%UUa%=cb31D4EB5Dm!cTNA)_1kq6AgKrgoVbx5NR+6KDn?LoR}wa&2Hjf-oV#`|
zhu~$tkzmua9<?EKBcTJjKujtdM)XF`5!-l1pjG{KkW~9jlM{1dVeHEz@V4!M_GNo(
z8tAI|WpA!OOrBKrMhWmm27aOtimu1%ruoxn1?sNzyS^g_dnZ<kPHBA%Mzv{7*Kd`r
z%BSWOM+GJsVD|s$4NGJ|_)XN@bQ+g(U9fCjeFnd8DvCU^3~hn$u}z_V*4)v%PO3)S
zmD>S?8Y8U5Ll=_>@<%C34;_IrBopJj!8TY3$O?zo10*1Wg~UYLo<B*tpbN-|lrZ=)
zumD<_^U>aB@Y`de$rLGTctZe|L*fZL(>g&UL9nbC8s*K*^OkOO2pgniyN1_TfUe*>
zeS>?kL2VQ~50snX@8So(!+*ja;lR70?BEu7kB;BXj$(ut{{Z-w&#}0-5JO;9zjGWY
zDH2$@nK*{7TA11pYc_InWulvhiT(V%4Mr{S(Vo>#4P0arEAkTsIhjO*w|b!)eon>j
zq8pf^G%y$)>DR*M#37EZKdW`MhF}IYEJMQj!?Y2gUJ0=mbYM2ASkDF89|41O;*J=(
z3roi_uL8SQJ6#Dk1_DN7>khBETDMLE*dBCik{v$4F(4IM(qA`x9RbgJb8>AMsN%l1
z@`<^?jjc_`h+{xVI^jkD7?(d3r%?bai~x_I3|HCliva?Ey(V!w1W&89RZLfc_~pp~
zk2(?JJ*kR?j)kTIf@K+A;BcfGMsmny?(HnS?Qfo(A<nI*Wf|}&LNdmG#={L-BkF|X
z+3&+iAF+A$0)E@0_;51a-V#{cLBwo=hs7u6qcKq|67OEF!ADD`0&v36!1<}G^Q!K5
zP7^oidcV~)1H$A~>qqMWGdY!bM8VHnlisF8t~5h-PNYW4OQck?0q+iY-;>7TGsE>_
zR*Gv(@64dyk>|UqHPU6mCVJlm20f38q5_4A1@MJm@jQN0a1-@oaZzyaJXB?}4|QG7
zVqg8NCMKZ&t7`d+xHoAM1j`Z2P`#}&@9W6mRKw7LC~d`lAp>4aZ%ppM*#W_7JnKw2
z_GW<Dz1HIjm6d$lqGyoKS)R6j4;WVk@Z<0G|AAa0slZPlNKeS8!vNGeh$g7VL=%#Z
zM|`un@jmQ=JY1E8AuJT|H~`iAv91k!uqpuErME{mpT*Q@P39V_j7ySK>IiR&e$py(
z_xYa4l5&D(g2gMscKuo?c&90zxRB|d^Tfl24TqN8Uu<l!k*^teCgX`T?Di4aJQ2i2
zdMDA|ZMByy$ibGx99PE!g`rN->Ev@&{xliXxYSw`eBgz@M)YfUp#zC#MRjIAvgBw=
zA@ondxZt&(E5pr@!OIgp%qI)IQ9oaTq9t4aA~c6=W*vpoud?*qZs>Y*hd?>w=heih
zL_ja^J?-XjR{7L$s*Kz3On`2@i`Z=?63)OCBak!hCuvP{tBEbtO7)84`GqYfGeu2@
z!^&qxUk#Ub>zCu>WvruRIvcx^{Pnxq1@DKhPF4d8oi3CvK(ZD~7thw%MG_iZgT9eO
z26-S&6<>?`gCq<}YwP&6_F^5(r|!Fv=23(c!laeU%)oZcy|jALGG?vn#y{6=q@gDL
zSmkZcHJZ=EY!+~5ys8Hm7oPCGW?+`n2HD@Td@}2a*qyZUY}sAQ^dua_$uFQgi^^KD
zOJLk0HBw=(`LOk%WiMn;JMOgGW{P3xE7Oa}9Z2ihahD?VBIl$Zk~j1+M`r=n<g$0a
z`+_Yb=%`nUL6B*6Lm(m8y5DU<ah0VMHzUsEnSD^kb+=*Y!-e7U#>~mCOg5F#^V$-!
zwR~)LhenISW$#QC(G7|DMZu1`{d{gpfii<_3HCkqJ={KnsyPR-&bW?{U7y?@*@*Zk
znE$-rnQhQcjg3#!q+L^{N8@K#hI&_q2Knws3_%a7zoVii$-^!F*i*X1lJtN7pt~F8
zxi92ha-Vi<iv%J+?|h3`NNnH<{<c$x47E|Fd9S<}M%kzVQXSu(tW6*^9F9PVdFwer
zyHN-teh)V3aY+!wpgN5#J3C^xtA`p|&J)X;XDG+<aMUE8csa|VTs`yz*N{Bl%%V(z
zLoI68qK>B;JF4t8+qu-kI)`qdYeJc$u}q8aZP%E&17x=NN}taFo@>bK^yVG?swKn)
zqF@uSqCpi(bNcQn&l9)2btn-t0{QuZCC6__c8Ys~)6T!|N}4a5ZGgQ$W;Lyo>%xEc
zlz(|1vjTP_uPF$5wlMGA!!uRmezIHY-?GihX5;(42}wAqBi^2MAlg%|zkPZ@%LZ9p
zS_nQoIU(~S!ZImF`9}FpFp^t*WpWfVl<C?HyTs+kp6|FjML!F7BtE5EyE0s<U#H^4
zj1XIiD~{P0NPg1oG?CdUF0pU_k(?$h(kb3l4#ixizq-T;!&v%`#01kWcw;yt+j?$e
z_R$Tg8kxlCb(M9+grF5cFy=KFzoFd&Kg>k_YTbV+?z<D6k$UOB95B<lhx~1QyoK3M
zE&YjSNQCm%r@YqFO+z>qw164*&zTCq2ghi!?w)ZyMCPc5p6v7EmQ~?_x;?NR&>T^i
zI89Isb-KaEM&PWa59|}U+w(ZW4)?b4QZw_%b}Fa1fns?O-<%BDvOL&+*-JzMon+sP
z@d{Lc2w4XtS-c9KeC3OlIpKr4MC~^1we}FAi2GpMfNA?E{A60|QGSc1!@#;Jr^NgJ
z@+}R$+hHS()f%jgQIMne{<eXuDW3QUC^`b#ay)4T|AvAg#qOZKQIVyEz)yb76HmQp
zPKoo|o;TkFcegJSpqL5<zF6R?HQi*|B-??WGW>I~;MA8u?!{!eOkD1mG!GUZ=33YK
zl3V$Tt7(GFVRGnfQKu7x5CduC#lDTtrL)CPzC;a)8^&yHpC`8Oq^Gmk9=tZgA*Wz)
zym*fPLg2hp+JWvnqyP3Ll=6$`c0*5_{1bxamK!5L5cHny;oB)3bjsRi=%vAM+)iV@
z&98OW*q=F&GwkkR0)B=^CN-1@Jbrw|{ruzlnG4s(>MbFG24?Y2Or@(L@S*u<(yn67
zG%MXVi=Kxe1EyCyx2T{mfa>Y_L8Kjr$K4R6(s$R5`I6qN^mlnt@@|hdtF}9+<UUmV
zls)G?2gfB44~Xu$j+n2YkPVG__qw|^|NWRa&z<~m&fTRo47l}sV3b4*l(<eDzk{l;
z?rSQw`L*iGWyd_tptigH#`Y9<;3`c94i&p|{<|$18^rPc8D+;bvs!9Iu9Ofy)v$Th
z)!Ar39{IL`24;9KwB>I5V!<nFp{4rh&MWaU;^Y@&-;O8*^hTYh^6vLs$mtH$iMpBV
z>tsJy0dI;AT@`_0D#1i_kNDzBJiEV)=B@vG==}XE(IgFdtje&;;o$CmCkSqQW<B7V
zISsDeUVB;rPw=J^>&k(=FbF>*PC(SGD6QjEA9DxNxQe_BAv@^Gh>~!b0n6-lrIJB6
znF153e+sv*@rN4d@=cm-h}qADp&FPyV$0=qvj}TDB@SMJWSlmO7Z?M{llMb+o*O3W
zEDJEORR%AI7^Tvdt+wqI+9U?qH8BRwjOCn+2Q3`t5O)B~=(Vkz>$v`32b8*<DD$)@
z0<X}wJUk$dz~%1&ZU~)rgiVOaZ_aPm>6~C?yX}ZzsLC&*tMTF{H+%G=u<aS;&V_P@
zwh?kBbdBh@H8sdQCn6oM!BThPcz)6rWg_5Y&L~gIA<syue?G;Rm*%5nBo$|W8>`BI
zIwPAVBv2H@(RIL-lAy;MtIenc;IbiPySQpNa4!fMgkzAO`}X0#wa@dmXMA4Y)T#Xp
z1N)7|6wiKD<p%z6N(-q=Y1eAeV*hnqmAhvPKRh|Z*U!+F6BED?n;o)f>6XACWs05i
zvEYHbRDlhAODki9Hib^r<!896#p6Vq@LLdjkKTZ#=DXHlnl`&+@@VsZrpq+wUQm*4
z93(m-K_!uFyFbR2yRtaKDUp~X{_J~<Eh=9;Wb<~K?K2nu9-gZ;$^U)e6wv%?IbE2q
z=;57J-B1T%$K+cN;^1>HV3T5U+dq)&qY2vt=ox;ODDR0NLSxv@DDI&6fJY>4b<ZfR
zmwyU^Oq?#-_7cK<SBe!aDrDK_&IvU2_rStO%i5!w8!4_yjNj$pbYVKVmD3zXmr%KK
z(0*~!6H{*;RXwJ>dpKSQIVPTU2}8VBhU5=ja{v4L&!PE+AQI;D02(V=IBh+}KLO&!
z-_l~LRxdB6al}xzTJc)~LtA=}gj6+c8z`xpmVXA!9u?y@>Yb!#>m<7;<emyQRb8}_
zy&GP0)q@@t*`n*&_*}({G3+Os3GTT$CdG+kSppNiTp1j2J+u6>;zy0OsV%C7Ce>|=
zk!OUd){`3I_Ka)U&TjLY;{TRo)a_K6r(MfhNcm88&>|%m{RPMqY@F*VOCL{_?2LM-
zS&_NXm0y=whzsfH(qQ+Db*IxwW8=lmfY`4lJTVX~&Q-BaDLXIhn9-juR6OZ$-c+P;
zjrt~oY`6SF&edneD_4FE(NYBRcc5iE($8msqGccz4=_WsXRibqAcvWSw+-l386Odr
zbMU{M5d=CIx?^O)cg*19|9H8v?OZtiWVQ@u#~rX3&NI0J_`G4y2f<6{{`0lRwn@z}
z@WK4kS$Cd7S!BR+91rwY(`u?!?L#NyqW_@Z-7reVjXTOa)rPq~!-|Oc#rkHOsk)f7
z@@WRU>hPT~D&7}0&~K3Uw+()Qs&BX53G>|nRhxYaZ9D4M$a2a^P{7q!{Q~!TW(>`y
zy=uDpV)2-7<=Jxosn)BsW=U&2#X0uwc39x!w`zYfS}=x!Uq9&;e)o|J1a_fbSmr$q
zSnLzd#kGS8`%!1}p~~(8|6?GHyNeN^-FOvL?*aqw|3q&$&vT^+0(Q4wr0kcM#(x~O
zq9YGA@jq=s;!{a!t*$$guwC+0zBXnk$em*PutzLRBGjGfad1oyyFHrY^0HU{W7s=H
z|A`l^qbuA_apuM~x~WF<y0^8D5oE=dM$GS+U&_Ng)%dPW`EJ`~*^egOKYa9((g<=6
zGn5cc<Mf2q>+nwTqv!~;0bZPS7RtLf!#M{tUNB8OxNqh?s(0<*Urn_u6f8eo2=X7I
zsaO70-g(t%e*ocH+E4&mAFrdmdGMQg{$)+k7%~LK>N&^;DR2FlV+ITHo04N_RNh^b
z_5IJ&7V&v?F>Cv3^Z1R&j41ClgKCv*=EdUe9AGdjGTaSsG$xkr<a|oCY}=~9yfJUD
zu{-0w*%`%H4U`YMIRlm1<F=w25CFf>vtAVrz5j9lKJE34Z3Y*+yPhvl)x`%N87sLG
zU}5_jAH8YWoyf$z_b*fjCyOAQ_TTB(K?;}?^h*UvWJ}z=l4gzibGx*u^!$GsMVW)#
zmJA&9tKtn?JG<XLg~I<Z_VczAEiy_E233hg0^t;agm*XvA@4s1|C?{CQip3WDt=T3
zN=^crM1Z-~O^uZ{@^yMv<krzGq3+b>v~A+{X<<8L2ldt|)lk0D{x+kJ{ZXKf3AK*}
zu0%V|<{6R2C^L>1X18f<^(vp#o6bl*&Tmlc&3qX}XVe#?m+l)72h=1w4JjZ*+mab$
zDDUgltqwg-DM@7O842BpzF%3*RF)&TJRY_G2TZozxlOhl&vHlM{j!3Z;jdm79_1R@
zl~&-D#ecn>e4=?yVS-W+<V&3OXewz!FU2^lR0u;h=8XT@s9cm;0jRwd+Ly~9m_9>7
z;ZsWJW41`P88$oWcXzc8@3>KSd+)0^uJp*MSv`N;X~&IAF;u1;tr1V8W*d!E58S^o
z60ExXn8@!|LStc)mH9H6!F+)!blIE1Cl3Xf`OgF$dTAQEYu-$OqCF_tB?%kFlqr>Z
z>e}rbP++fyt^abIe9o{Q?mq`NtQj^^Id`^jx(hq{z-IgO$N%B#z2m9=|MziwWJk7R
zRVQU<J642JPEw+9?2$d<9C6H0c4TB#iV7ino?{*v*&O?D%&cQO=I^Q3`}6+3-oHP%
zxw+jq<8dF?{kpFE<Jc;*Mozr=@6-Ofb&MsYDch)pdY{or)JbMLjgt02Dmnjqe$D|D
zl1(ZEG`ZU5yhI?;_o<773kMZwb;apfP&OTREw|O9EGd3RETe_a0|w2=q6*5-zsBpl
zj>~zsal=Y)sc*G?r=4-S&uR2PQ9^yQO_!Xc9a<7kC1H8Yyeq{2nRk1Ad{%O{m>1FB
zd>f*rsvFAKe=R2&T4M|9KgMMEj5%&NU|i2A*vJ33J7m5_OJR5iJZ!w8m3<brd(5bV
z|5YMaArvxOxWwob^Nv%E|IN6YEI6C_z1Te-pjvZejr`fH_n90NX9__N<p#wB>>ZcO
zqoGPZlrIKgwUld2G}bag**=tF-+SWxX^zNkhW*2wh++yQFaZW3{@ozl5b|T3@v_>D
z^8RDlX)G>8=DNs(Fp(moDM`mIKWkZ2jR6UC)dR7>NG==hILPr^+eH2|<XX*sGia~a
ztElGBcUD)FS%GaF&wt-<pp+D|GAkV5LXV)O?nNmuBqF8f=W1A9)=PS_f7MIuG7X_R
zIVVoFCL$PGZW?&9tIYRyBJf{FD|d-h-$Zq*U0CAu7?{#`j1r>89y35BKlQeU_&t9V
z40}ld)Dg+(sLB0Tud@E{E&viL(jhkhbFpxbw^V5&C!Y6MaLvt!;J7-MSdF%!Gx-UY
z=C%fi8?n=p@2D@Ek~BZ1Qthvonq-$HH3ZEj<?EU23T)me>Ij*RuptgpA`1|b1ceeV
zxIYdk`H-zxWoEAb8IZA#G#Tzqw0zniV1&AlDcZmFuNV0@2UqDS6k0ccQtIS35uBj&
zzsI@Fj7H<X=ET-)d1W6>us#z5QV@Xpn4%aGIg&6zXHtcjPz8n?Wz_C?{v(P50h>pP
z0XDC}Og{zYXXJsf<s00>>@h8r@|w*Rq!tXmp<M0IofmW)Oy|NdnHpM0Jyn&zj0$3=
zzQ$>h8mp(Y2}-%8q{YkBy0zKtl(n!%i-gYzXF8qvWbP0m)Q+2<n0mB+lp;7dg#Ev6
zAW+=Qe2&~}5}~TqsFk|jNlPZR%KtfWW(saHPJhkC>_DV%r&#6A=`r2^oHd+6l2A21
zx$d)OuXZ#FF6)<Y9VuV43WV{H?H!argUXnIp#7{8R6({i0^d&1hU`&thZr+LOgZD@
z867z+O&<S7OO0rg5|b?t-<XCKP?`wZAXuv=Xyh?5ziaLw+;>MJdOscUTw%<iFEoFS
zNFwEm;61KN)-&$}Ml{{Th)95O>=y=YMsAN8OdL%kz<2(C=@_P7O0yAX`+J#BdHI`L
z_shw1TqYc=@JV^6SF`qS{Re7DE$~|(&!1hu<WMYoBkL*U|4UknbYXX(_t%dHo}C=^
z$X4)^xo+3q+5J}`E)HP9ofsd5O%-;d@rz?-rdV-Uq+?_-oh3DTp`n77`x$e@Bp}7M
z@Z{02GDSO0z2?QcR}0|dzwm`@eioVb<D^#m%+SpIbS5yYAh|%mlc4#i!U)^({E=WB
z!XqE1KChXq%}rp@PWBJEl_9erb?>lBwH;Drwtx8K><=tp3pijw$p5}>zHsy=hJV^E
z7lXErP&p9uu002Lfp5A_HUiJ4s;nEnm)H^eQyK={AYGWmgMXDMir{%+cQh&Wj`sA3
znXlPCKySJc$>gq4@6MGayqsJyKWTLgs|3*C++mx1LtnsV6m0bm6W4}E+)MAgY^0y>
z`3$v`Y3iKu;h8yn=GEkF4zt15uePBei#N3A!6O=kcFfBhR+*uNE@74^vBNU2iL;vg
zhY!?z3%=oOY48}a9>i}irG@;M&z<761urqmQ+<;F?Bp?7{VcjL3sxiknuMOtLG@?l
zn!L8V<$VJ;IRg7uNO=sV?sk4qIqpOqcP@*a`HsH`sKt&fUXFSG>_Z`m_$HRwf(iX9
zTY+BDUGSi)O`99I&|sCWD-_iMHs|_`;DU}GdKBtx?-AXfXwjI9(Z;?5JxS(CY}4kN
zl0i^Vf{x>Ng?|qy1E7VJ!jhOot=rx!%V)l2A^NaWIExh`NjP{!58=M}?}J!Xr_fiN
zPja0O*k6<HcB@&=<%?8zMciaj*uz@a>Zood@JXx)n)kv09tzl<GV$IPI{iP8`b#fM
zPrUlG6Uu4{8$|Q3h!mhH0Fg34Tmi;vU3r@L`w!)h5{hW)tb-P?&pFuwR+{m@4e--y
zOY$pYuZJeB`H!&cmP{|PpAg8M9R3#@h@O`f$96sb`x$=^9_^!p%L_crtHr3DZbrq9
ztaxN|BI*Fzb{ZgU^(9YsVQ0Hj?Vc*@E>dY-X>Nf><MI>jns~-&Ke^Dqn7Pm`!0GJN
z@n#3?ZdxlE|EVo5n7Re2)`CRIkz*+>Zm4s6n1)r~O9XgO=my#-NWnIZA32u*<Elyq
zu5PM>v=`Ic;1;Fbw)bqAE3Ix3o-%uW<$Slfuz6$pZk$_-|J=;?Jnu3K=z;+|FzLx{
zIT73Nx5D3POhyXFc^t`~1-uW&sI{SU1X=!7!O@IbIDm<Op@JF!VC0LNTuhc{4L+J!
zROOTKFvfa?(4(sAa3OR`a_ntk37BtrS!0vHq`7wR<!ZJQ@d$Nxbjm7jJr(w^L}#W*
zjw1nE7ysHlf5pVK5h?2Mh8}p^&E(xqyLEDF3%TFPcR9JwCZ|s(bDard<*_qBxwNns
zbRVgmDhG%;(8yZ7{J7sXE1qM;TCQ+W7>NxfrOdnCC4{utoM8&AIv=a-t`s><+yQ3}
zs_a+ym>F;do{R*lEDq+*UzpjpIVUIlxB~a=11o00;iqkG$J&8Hmo{#x0%W&D<zfCT
z#$jtN5xNe{uSL7svH2G(y1I^gBPzO?x>a|-LB($Fo9>-6K9j&uidNm?KilL#;%Drh
zPvw~{^=3j6LTmGX)O<frb9s3Wx>r72aLD(^%b$FPes_aV)LM_4FFc2GYjd<%vpxWh
zQXfsH3Kzi7f$txYZ#O&firb>?&5zB8X%XB(z=m3~Biuxsvfz>l!j($M)UBff-=U{u
z8L8l!oAtvI@<AX)X3G6~v$U?E-Tiz4Rw$yqI5ru2@WCfD9$F^DcI#jL7W<LrxCMHw
z6}Tuxz<y+~NOVp2*#{b1oy5ub^r;W<Mg_R`-B##|;Puq?1%#&^uwP5Nbt1_84a0x?
z)t)ZUtb!Vd-EOXFNBFtezbrDN@r_rc)L?J-EuOX}-|9}Y8dfNz<m~wFeEzAPM17h3
z^{Wy*ei@e|lykj$R4R`SM^LJ<`U>YGSn)?1f^fvS%rFTwz{)G{I`3%NX9w)v6K0~<
z%Gxcke{tMyC3v{*iyE)r&fHjaMP<6Q#0XC@LAKL+ObvpI{D$(4Y;mN&flxzDr<^Rv
zqpI;Oy)mDsPuHf`890f&Yv(_?+Vo9IoX$$@ZP+SQM?uS-FK>!~kc3ipZT543T?0Fv
zhpp$Iy&tK|mFG5DGMA)60KDEXD%5W`A6C!2lAF6GO;E^rJlo{sG4~S{-h?DMB8ib8
zSo5f3THR{)NIoNeMMW`?N|*CHOqo;t@XuxOvFJr=5VsyuA=?!?g0jmVCpJA1j^w!&
z`%cJtp1|5e$@Gre#;wV}itsj^G#tG#e*D;#ZqL%RQzLo)!H=IajWF8LWDWm!9`1Df
z{Nzj)wErqz7WP8S<Z+qA3Hj|G#4A@&@iVs}5>gF^Lq+2GN^?iV>@#94%*+&!uTJ!s
z%?Bew{yp#*#iEU52=EqRP@l^39q9WBwmeV&B~AW6lFXp9jtuCLM?dCVj-yor?X;Y2
zs-5ci&b;FI{Tdvkidaaq-^*)BYriw%=8xxLX!{!&&I)816`ScOVmcp~l|UO~l4m-6
z*Q4V~dLvK)=f&JcT|$qZ1nxemr3PrU$xBa6`?AALw*M^JFh$>y+&Tg}jb4s?OH43E
z<<M7HiePNwHQvks0S!D*M48E#=^c@+lEG<nV}kd{#GYF@krYMfSO}{dKO<XXc{<qQ
z5bDh!7pbFa>2iT4ar$R^C6m*cIz-z0*OTo^3pYB_vp28FjP%-Zb^>ngZwwuO{j9Up
z_<@KIWubz^TZlx!``=Nb=JAK>gwF9g6_Mter>a|#&1hjTeCNSD?7w)sn2iFP+Iszc
zj*|)Jy7%n-%+M3q<jDa>ZK~oy3)*^IwZ`R{NA%5D#f8*ZLu&<9uPZH`dFMTrZ#wF1
zG4OOzj97T^ZsN%n5c2<b8I(mSGxiM?NaE#ar;@4x$tIB`_TeI1B{8xGdOB8gGL~Tz
zt#NSs@YW8%1ePCt%Rc-@OLsg+_csjW$b-+}*UMje=KUNe;qrBseLdllNNZ-Y6X5U>
znAf-8nG-vn>^Ut0)wq5#1$8|CUS?AbN6NI6b-I)1*71!>iVzu$`7HPYT^hjy&*&4V
z{%ST-(W!!TCM~A5$mSQmutp`*i++V00Das_P4TuFcBj){z0{?b@$)`74rwE-gD6Kn
zrb6gGlQLO)Ls=F*Rq&Id4c=VEY3Pn8i=+F6XGpIMu>iRlTQlO{y@DJMM-+eFq(KP6
z>yEWL!NvEc?s@|{)QgiLwgd9?-CaP{?u{L<Ej0m6P2AY@&q?oQ`_V11f*~VcA)Jgy
z*@Bk4pA{e@CU=X&r-KWTYgLWIv&J+fG-J-ka`?0_UH`nT#wuj~+`)LkIHf;dPuq1G
z?pOEjt2}|t4kx<<ieqIUZMy_U^>j1N?U`^KC)|0>ry_4C7ACNn`8d`;py%aO>9~YM
ziWdr;>HTp3tB%U`Jh&)z8DCdtUu_(s%i2x|4Gs~nUHzHnDEk%>4`VD!K6u3^b2`8I
zeE#a$o+hqcIBpfWV2{i#Iv_4)qrRv{6`%A@)X1vt4jL+pJAA`4=zOzosp{31+foQX
zL$g?t6-sdD$ifwclT~qLPJM+ox=FB@3LtjERKNYnQ#)B+KPNaYLpqx!u!`#q7f;qh
z`HwAP#=3jp$Fm$L_{BJut-A~f;9IkQVPOGtFehA;ugd?E?Xq!<;`Zz0@fw%C*X+&=
zWgKYR>1<Z1VkUut%wS}H8FjBd>4?HA8!os_Ivm@WOp5yX3oly{_!WVx1RaWLHrUqE
zJQZ`ir!il}(lo{VlU6f#hFS)l>reWG(SzVu+C9Gg7Be7MAP;Ft7}=7YQm68lbx!LL
znylgNKi+rEb|4-YNfCgHdYX?~Md%)xqt|Mu8z!e-u1&55q}12euTSLHp2Uuim-zXX
zaadOF@!YKo!;gK1b;qt$!t$rh3z5E#yS1f58?g;Y-#v*xSVGO}mJy2I)%#?<PZCmt
ziQ|YsM!aRGLi(27J}q<+{$4aqg&!Gv_MZ{5LWB(O7P#>0$w(l+dfDKF)i#mw-ht1Y
zPY!VKmEel{mCmbdF6(E<*=GX*3pcY(hPsQ1yEjl<H~8`py=Mw6m)R&5G=pRp&PUI*
zUm16IcZ&i8D$CC2JhhJ0ZXx|*E0@+>k!-5juQtdNpT651Yrq~`W6f$poWG6lga_t@
zv%q6=CjI75Yn~UoT`GyLwm-tC)eag1R3g6=1_EkucBdpXGADI^I`GVV9d<GV>u!Fx
z)U6e?@!_a`$t_^z+2qmm*=cuyeup`j;x97wU*WMzB=Srjv~|7ZbOIgrM)+X(1N{T7
zNUa%le!`H%`I9?j*#Kef+Zkax*Yl|ob+V=>ZJz#4-b0x+xX=!#0Xrz1J}msztgt`f
zW&eAGHGIMHhYFzjJb$#`DuFCH7v|-`u12^^Mey52Pu{lKQ^+5FJvMo9*Ays*!2bZk
z>!cc(ht9*x8doAl4?+G=siYeN#)L9bv|JFKGU+TlAxMk_soELB7vy*_(<)995+T#H
z#Z2v}{DF^@7$0?m2|<sK=%QwFjCXARz8Nrk7huN9=kDLYT)3Xj)9cxq9x=j$c{RfG
z4h9;C9=YV+8O)EeBqbN^0B17+mz_o8XYDmP;l~3;k_2_q#aU$-_tmn)ZoN8V>zl7O
z-UMvC)vMX4bW&NavVL|F3;T*xU&(P0^T}E9ohV^I;h%&}Utv)G@n15{gvMfGvu;tP
zNEwj8j^;Rh?-?j<wy-M4K;WcIa!&wxJIRvgzzSEw0O|)lx-BO?dTnuhd1*8M%YeRe
zA^zHE5pp;GB<f(OWBd*gm<*5h!O|W*&i>A=tZAu%*wrw}uv9n0GJE4Wm@J`Jw^`H~
z7d_Gv7qDurJT_t#_ObuGQ6NSom*9J)*^_4%1I=@j(M3Bwii;U(YUc2BAi{I$ZJ?@k
zlF84&jeTTMGxLSRW`K_{5q&*wLjgRoUrZLEZ(whNF(P>x{X9n&gHrqffQYvuIb4Hh
zG~Z_k6F;Ow5$LFn4DI$uBc6nqagfIW#I`8Qc}N74vU+{kNv^E>lEW{mmGSeSz+T0P
zvGeS7`mFXPfVpMQ!{L>LzvWr8W`D!c<i=ir13PT?FQ!<DQKyF_#{$=PCC3h>-lPJJ
zccAgPd&kBU4&Vo7{9+!WSF0osdyB1{!Rp*Uzjp(2%NB4mzceH)*N2+>U6o{TkJ7R{
z)4TxFi08d$$x*nWcw!;_&pMMEcX<8VR;aa(CxT8FCS#JsmPoWz=k!|EoIX^CMO>O-
zyY}|oD}Dr-?iBg>3#;*@1MGpVQPlK<*`r<4x?@kN&X5<gZfzGydW*cOa~7mxQoNi$
zn*xLMNjd0Jo-4RwCGsiC=dEu3Z5Z#1paRxnub<R|BOBk1P;|qU$bHq#mi6{M7r7Pn
zYSS?AZqBVB?DyJ{seu~U5TVwBZ!-t`pWtzZQXH^xwedMeYe=nQPZYDT3%E7sITrG`
z$!tffVQSLY_lxNby@2Yme){BBzS!$-164QoMoJFRs$UIbo_j6F{+@JrzsUP>g${&n
zCX$KIjRH%nr9I}GIbW$j83LufMV?aa=W{ODwj`^GBphnCm8MS*HT7+55doFzw_6}_
zY%v{CQXjw%Ja5f49O<X*76hMdNt}6B+5^wxR}K~r0nk1kwA{t>v<q$Re<32zhmbaD
z9jj;;8_e0}>h^Mc+%8%3a(GhNjcZgO6?$SNJw?U)U~26w)Q%oXraDv}MeMHaPDpGq
z1JTPs)lPO4`#23wQ_i+v3oYFj2#y^3{lUFa*ChUG2Uz;^c=R3wN`rSnTL$pZa#Wzj
zN_$_u+F~(Ixg6?rl2x(~$Ps^nezfRZb0G%;%iCs7f(W}a(m<-ViRDRU%6hMl+@6tF
zLvAsq<GV|vbW^@hS6!)PE9U-AENnk$!_9nOo?`PL(9ww3wWQ^B47BIu&ZNZ4%dHBS
zflT??JO!J2HP)0YOQ8guP4rQQ-(BB>8J!)J_$;evy&^_Dhc<@otE2%Itln^Wg@JP{
zsu<X?eY~IP7HDs9JFvZe=d2E^ysMh$6Ty#wpH-($8GjneNrTega@VSjT#HeA3fZm2
z$10B}tUd7oh=a;!C&@k;t}Nvr_Oap9-}W63Wbus1%op}Iq7EGfEf=bXc<CQRj@K4V
z^G(JT9eV7qH6T+`uekn$Emz$stR_l!sL)r<Ww#E_353&;1-+DF#Q9qXS@nJTD_X?a
z$Lj)rI@U>}E^INSlha>f&t<p#nj7>jaKNZcq5_7ekTbNm;?JU?g2&tYgXeQ6D;Mv{
zZ;7xQ^-H%U-t+GM4j0qD!YTY%3BVz>=hbGdG`m^)q-N#E%G5!dyixM$NN3m02l|91
zORV*2E802^kwiDuehUb}ip{qvbvB-3jB4hyH`(4{3ygX$BR_fwP=PL0AEZS6#VeUx
zUG}P8o;;kNV;9Hn%!>EWnG-^$B%kyCg}#$tQp}ZNvsjS;sF~vPq=Ct79QFvEPjeZG
zS$_u;iyUSEi}okP0^drwz1q;$&p6?biob0|vliPwA5tt9H0J6a_>>x18C;?x0ODw3
zO5>;;6OET+Y}Vu&yXMeIBC#dPUrXLu>JQi&kU!o}g>-`5Y>0{x7rMT;S6OX|K4O>u
zN=fg}ZOPmNg)2<8c#{Z$ILu9hddqoDN#CX@6agQAf|MD4S$=yKBfmnyge+<4LvR}G
zgmZKHuKLYclfN>1k*!c4tb=czm$`62WS4r=fQo!CS=3wl@+90)in;Rh;qLtU5c?mV
zM<}}q51pgtDw}V+P6S!{bjx-nT;(lnqS!yGV+9ErH3)7I=ox{s==jAKrIz%3u2EYc
zb@N7Np+0c#f1nOM-}vztz%0dm-LxPljpLuh^C6-Za83H%nKpt&N$<09NfA<AaE-48
zGkD3m#~ccK&J+R0&xX~}{|$AX;pu|g=|b4aeK$YDX?-wDt09y_@csR3`g94NArZQ(
z2v>T@UY)(tN-VoH;ow*H@vmQCBRej*NKuw(*heftZh#UOs(?oz*CFfFvdHIxe^|6E
z77REX$X-`H?2F@`=;q5lFyJ~U>wGqSIE<K1k#<8Jv`S>z5Sus}dg}q+D?1GT7m_+(
zCR?$^Z3Z*>e>fT&X^`7Cg3JFMY2=*)f#F&u(Y{^?Llw{|0;U3WYB-6z(L*ZU1yg4`
zG`gzukbth9ns88YV0j%Jg71?cXj$W6M<EMqc#m?v;^^m9Qc-aVuOacf2CCAV!cj6V
zg71BaHk2vZUhCZn(--_ojr8LSB>l@xkIe~95mMAh)26viZ4a>=#7iOIKF9e+!+F3p
zDhlOD%fgj70uNag%E5iKfeJJ%mR(0jej0M69ppijJ!?@XRl__%E{;#HH^?5&4_?&j
zwy6e2<dHwV9S=};kVgKp)XlJ_@WahwYT={?<XTcRz-_Hd)Xu?v0EkNVSf54B9iJfK
zI$^0e94DC<_yFZ5f6QJQE+07Y&)|Z+$zs8}ch120o{u4@MowSR6UDd~h{CVXo(s;l
zKyNofD!nwC>?LLcPSqXx)<w}?l_z=`NWy^H$-pF~OX*5t_VL{F*4^w*g|)JQ!yX-w
z-_plPz*;)RlK}JIY@IwGNgOg2M}v<uR@IS9=f;e7fbj}!P^-5wx)P^$da$m+<o6(Z
zXQ&8S-b|8R-hU1!_%Hp=od%#t=`$l$j;%-?$fvVtc9cRZoT@qKFGd|fo$gK!i}^EB
zgmTiXew7FlB)3&Bpss4uGVie3j_{PSX)ay>j9+&#A{<dHsgl}O5DhcM7q5<v&&ir`
z1;n{8GLVX}S$!`u5%ye>A3(M2H$3(IWIq`%v5N8btQYxm9KM4zAzU-=l3Vuf$NJXg
z>~F^{E|zS`Pr<GC0Xhl!Rt}huk}`c28RpKLZJ0|w4?Jju90YJxr~Z2p$ZcS3`k-c(
zSOGXV>Z0CX%==vRx}9pwHpNWtJ}+?zioCiW1v|Eyw!7YW_sH~JV=f3!^J3BPVVRR0
znHoSf?d)$Y>8RDXvF;QBChH=*Onjco1DJWcp~NAZ!Z}&4clJ74v8$F$PnKY;f5w&4
zjO{!b-FJxFsRt4_aDBx%n>syjsOoFSSi97R<XT=-&l|b@&A32+yWw-j_~eLbJ@JUo
zkPm#W8#eGg%F2mV`W_h9x}CdrkdYSg994(_QWbtqf2zz0GA)mujhr3ACa2F%rbh}`
z)$ki!_8EN<Txz3Vvu*=<`|9p&@|z&*n_k;S0#naUj{?s&z<HJ*m8Eq);UCQ=Rh_fu
zC&r7fjXyW{a9;Zu7XTPumtWK#x~4RAIX(sE-?s%Pca-p+S6@fL(Fq>xleSm9w|;^5
z{TA<TU4;U@>m?PLlM605_An_AVtWTy{QFkmq%IJpDm8%ZN{<ey?X8lf1Elh+V2t%x
zDXPk>uDJ24`VlFE9CkZmed5IoI99T5s}yk$8KaC7yl;LWIFHr?Z8zqhknK^1-FKQ&
zw8_-f$+EpRK*+eKM|%w}Qui(*q+W`yQo5?mz|vLUe7pQBZCqV$8b9cl-*sx-hwIs%
zG>JyU1h5H5Q`-t?VpeXABCeTHx!@%slz#%HrC~XO&bf}AYPltj0QT{(UXb8Ih?H`V
z)gw6{(2sb|J*}JpjT9k$9Rx}rt)!aC_l*?E@<TODZX73v8o>Wyuq{>u&}~$0e8*2O
zyz#(nWi#-kf^xd4mcXi;uh+LaC14F77&?Hq%s{5px<gaE88$=CV_bJ&F4i}K)m^lu
z31`U{lzT9$iy<J``?86xGCZi#NSI}*K?G7I%2@9hK=qoi5s$;3)773m9@du9<-4f4
zFIg?&L?_tM)6n|PPELBk1!1`ysn6dq38kc48>f?~bO}BF9y_XVhIj1E;V}>8cp4q<
zIhPS8pi0Z1J^C+q(P_2arOkQ$>R{&R{Qr-p%!v=7Llqx{l0}DO=U}u_ah^@o6ykw9
zI)P;ur2?{cr3vVu$chM$3)?3Q^6r0DwwCgcfAVTDs(U5ddpu);;S#%A`86Nw<4YI<
zHhly4eZfC5;Pisr9z+T!g~hRC)T$<DZpku~|Fr0A7xmj}?awcXQcl(58KxfI9QK<8
z<2z^H=B)*8AEn8PD_%9AX85LH<~Yn**=Ma@A2EM$uHE}k^LOR{`0=utXR7#Fy4^rI
za((7S(aT9!p5om!i!uXf$A`jRy9MFs`2)5AxS_8gIn&|C^gnf--vhoR8$u6pJ7Xr=
zfdmXX=M(#xCrh!I>m_pF)3LzO-}SY4x39N7=5)94?qFhT3u|Nx4+cu^whs?^U%tW#
zF?R2lTQr;fJfP4_s}}VQR~2yQ{_y{__Et}P`Jkl}4*6PFdhA-Usn}e&!)rGr>j5zA
z1^nm9pmD$Nh*-CLYo{bG!1;HLv*1+uy{KRy35*7^B$KtN!_Upct9#_3CC>B{Cc+<~
z&O45>_!?2Rg-2_9KjymHybNlM3$~kHap+?y-Dfc2Bx*LMvJ>$b?*l;>CYcF==xh$F
zDlGd0&=2W*@<p3ryYg-l<L&{CRIo1|Rk__e!roVGY^%6*(S&EjKY9JhS2)8q@$bXw
zB-GQWInt6oS^~P9N4Qhk=LBjY5bA?4zhc<yeNvpxkxEHD+et(@yn4(=EDElHrn_S<
zJypHpO>Zt)5C)I<BLb>R`YTj=XsthY&_8)v|8=}$MTc^n+yNX@06hsGfEpR1rh0Qo
z5zP!E^2ZI;&yS5l1GcFCvYqG!fgle|(b-|qeztxNmDIX1(d(-=s7`(Ea$Mo_^I_?o
z+Txt4DjA@zH6O~7q}%1Icvu4>z%s|YbDfn!WBAD3*D#w>(g!ihy@mEJZvcgbe|3bX
zyx_3&Iet_o==;5<(rBvh!PMy;6Ny?X@J#|H@ED`L%ai2V#7WDP`bbD3J%}6O{+$H;
z$Q@Qr-zKus8LoRWuPnh4h?hrN{|-RI0tNNbG9EpE78is3bHiY5cQZg}J8B;3OE7(f
zl8Nl2>g*M4*BtUX6IS;U6Q12O<vC7cw9GK_$fo((G5cG!8*AqVn$M@eMBrNkP~oVw
z)(VFSKTdcryTTiDf5lzfYMs7NR`}yBuA#l#<9_4gQ^OwIWG#6QdX!cQrl7T%JqKU1
zLZnqoEXo67n-Khs+E1_ixo~y1>7;jf8MBjTLIpbhGhY43r1TIkihuq5p5zKw!RIsM
z2$O)6mpG8q*%xR{wPJp3>0H4+Cr(9lq%fR~^P+UtV=KeZlH0Q!-|^NR`O(i)Lj_q;
z$akC=ti=LG;}Vo!#$n^#_bwAJ-TBO(o`mI$Itq_11|G>$U>GcYaA)En%Y#bn^>E4p
zC=4K!#mf&_Rig`g1DEnR$p`kG)G{T-aWjUJrTRn@fzHdzI|I;-7=DC&J+E*a7*H2Z
z93AUM+N$nNItwXe0Z7M(o@~VuM+#rMxQgi(@^0!@8daeJ0sU&|Y6^EHKtvy#HV1dF
z^NhdZBQ4v<?d*|o2UyTq9Kv(VKx@>+ti7%zDe!^!m6gKiSGqcKp99F}S#>}zj5PQn
zt&Ghoz&=$PXZ&03l_)r_jy;<GdTi8881B(1*{vFRUF7WR=1ZyEBcH%%p&m%fN+oa8
z<A%+eIl)ZNIa$X2ARiZC>eXS+_U^xPqE~x*Me}7}?^No8BHKg>3wO8LwP8EUfW|Hu
zMpTCoI<_nX&V*aRP|7|-F@8V776vzl$7?n!IL2Tnf1Vf#J%4MDjyq&PUH;^n{|)vs
zbcp@|x@f3~kGgIQrmlaT`@Dk!b;FsCLZOBo&ltNA4orcE7$2(vVaTu07-nQ<L$B0M
zzJ!PE#fhZZ_Spx_#%>e`lw=a42+I?|%j-MU!!G&-bZ)0SOu=PLVA^)>zOY2D1q_?m
z-=_yD#Dv6X$Wiy=6|bhloUJLyHl6y^h>2=xj}p37wwmcmSh{nA0^V33E0&;-2NMUx
zCswaQWeEcGOx9F+JOnyJo0oydV`1I5Uf;LWKd~@m38nd?SHrVf_w}e$?WlB8jyCJ;
zcyT{ZL}<}$!|%zNo}uj@Y56)Fr9>*P+i%XczJ+6T^iS->aOA6){=?;@nuH#OcDq+q
z@fiG|NL-9J^8QA^MNCG^FB)brLsr1vXno+7vt%Hncx+V*T8i8Oxk6&UCv>;}p`-tI
z^6VH~7@K~^x1K;L1Q0z2n;(u#NL#I52=S4##5LG6{3_jNV~+MyAU5?(MnfHm@@EUv
zJ3RRIna^n^YQ1n^W>Hl%ArlryzGf)AosCyKTM0ZV0-q$y6@KJ&N#yq@h%l*@YgLzZ
zmjq)Dw5ShNdAyvjZ&_tQg%C<q0XQ)?L}v!{gQyLz<o&7U878%JvAL>sdq&ZKX+7Ix
zttB`A5jPLqd5`nKN(GzRzGh&s9oOR_sAMBK=v;M<@x2v|;QXMm8y0simL_^!vX+VN
z|NK<kgA+cOs9@xTYg`QFEWM)7o(j#24vx6iYDP=;>=)X4*xr!67Q7(ft+23qcgbSu
z241RYEBd7U)*=_rn;7B(-OaWvK#)5*u3!U_4KhP}^7`QtZM2@$dMeXs3w3a2Fxrm`
z?)-Si3I~+fp|Wn32{gWhAw6pBRd56q+bXek1GdH*9m=E@22&>8UK^UN5|FERQk|v6
zzm0>9win&oa<}vi^6%dSQmhkfuDV_JHS6BZy2~l|ZV2A6{n!CqI`2Vl%DfqZi(z5c
zg&!j>F#)R;RdcW6^>}Aa#whuTg8c=d;*PRe{=>9tDO)r6oZfv`z-c@g%?iO1a6HY%
z54JzG<PSL#%SJ3R&{0rxql3)xdAj!tsfMUMvKs2*A+IZ1^@r30=4^#JC&MiyUxwm)
z$_gHz7=;-AM@j>l3%c{AoFX<X4C{ZCeYPF=!^t)^Kc%#~-29y-dCUFL-D{yKk-RV_
zDa=;qNq288drc)kVu$JX50eMq?17Z3i9~jbm^Y0uoP0^fEbyQ_KQnpV`d!dH^QGfE
z5AYcd1TLCrq3crmo;xV8jr&FN^a$o%IGP;Pdo((DvT=5T+Se8cX!#MCkxXzqTSH|!
zS8R1+nf!6&tUyGQf}DR^J(7%y$LDVQV|Ge^&u$CEx^bc{XdXI>P=R(w){h^abh{zF
z+?dVhGl0^=m;EUC>&r<Jd#~mmSoDSo0P8*4Y))qeBwx5<=>ZHQyJADVLdOW<iI%Yo
z=(we7Iz|U>CWQFh<9t(08qLD=6C0Er2lPU(Uk~L2WVSTc3!r1}(ox|$7CpKB4`0I`
zpLG-+-<@ugeDg^q?#NSC=p(9BL`($$XB(jHRLqVBQse;840}FLwYE3)lnSJ<K9l)j
z5~T`k0DGV8j`+Iy4WwV)Dx^WVwlLr3#jK?-cIKi^oQEBmKlhEaRA)WXRHWmA7v!D|
z4nQ3wICU&rZss1|5$bQF0;Q`^MX%HvbJqQ3u>-G0<BI}MR=_3>rKoLX)IbCxnIOF|
zG7tpMEmMy4>Gmgi+$=Fhr|o5J1;X8Islx-!T$BIX@0*<}jiGzK=cP_n<>ty~j?TuI
zV4d5-r}IL^kLe*_squ+^Df^A!_}<{9#0-d*k{`MDhpulNW&DFNFBG<Ct=xTg`gHcG
zenfXJ*%Pcq@0l-LS@>}`di=3{zGD=TfA_1-Za6Y3b1qn8rsue;X9s-2azZK#%5ej{
z3>58E1PvhhF8Ay_2FwTF+wLO?o99NCfZzh(#b*%`_1c~I7LqhXjI7}Hg!bEPO8jJ7
z!GOX@sW`Wsbe#30dhRhm-Tv*2HSHBQ=iC_|>~3`>5xzjfLvn0-IMWO)U6C^!f5yK}
zi4t>iX2dnRlBjuvT{)?0Po@zj7cx)l0Q0HY)Z+wWWyCukYHy3jw3j^;D&lz8@AfLG
zlfi`XQWDchdhipF+ncHmw>lCLjwgx-BB+bRxl7kNgcyPKl8jdGBD;1%NjKI3cYDf!
zcW9h_sNuuYZ)7WUm(0lo@g2E?XNJUZui75l0j@QL`qL|2+#_*s0PkbmSkw|!v{qlX
zdD%W=YO?U`(#+WG2+8UX2MHwH0Q4W<P-ZX*6%FrET|!IeS`V;L%VRRv^%J?Af^_s0
zCEg2{jh!1A=c%`Nw#p7L#hkb9z?>+sGG8`I99!Gg&kc~a_3<;d5Q!$@3uTGz(NuGW
zAw!y_Z0D*TA!MIh;7pma8f|ms%s18+Jp__PH46I(r$y#a_sy6AQk7R36TANe6fSmL
z!)rC!9K~$Y{pmc#Ujp`qF|Y^suj<7t<p>ub@%%NXpz>Y59n`m^F1CE7__RJj{zxsS
z(M|-MQgvtgrs$SiDG*8Q7>?~bR56Zrk1yrWo41JVR13;w!##XDMDp(GE2UkD+Vh@5
zEOsA1M#>UW-}#Do0Dshu(H<@zF|WmiV;|1-VDR?O+g!3#@nEYqK<lEqfZwZS4_F7b
zto~q_J7#UKhUb-j9S&7}!jP))IFTc<_ZGP8k!DJX!7xC+<}a%aN#bv26T9kMk`_s_
z`mt#1QVhvTId2Lp9Daw<5;tQfk!?rHL8ECWXZTUc&V%e?xE`}Mcg7#Gp@B#6z*gh7
z<-2-chB)Dmgn17yv|Zt3{jCF_EvB><3CiF?3taG2BepqwJavi?E?=->FF_Pd<{bRl
z$CS45<kk{AT4kFH;oiXmejfmA?k%g$SM>YhZ+v@pUV4jPu$*(Y+4uh|{Zzg<*TFEw
z&S{cP?|m*Vq!r%cC|e2lapdT&t2ZM#IvG;6aQ>{Jv`1ED-aXDLN7iwZpLR@YNvB~R
z2hOfwXk(S*7x%9jQ)I?mVSy4mg~EP>eU*7P^qBF|C!zRvrJu+a8PJ-*=PI>_CTU<<
z<sw6UyI{5zbE+E+aBVH5t%;iL^44p1ddR8eO?rsF8x<%t_!jvmvG|Ygu9pm<$rYWf
z)uk$tByTpG-+bkIz4O63sGq`gGv~%wLg!hTkbxbU8of=`etAn#=9hj#?aw1EdPtbk
zU!L9~PA3R|s`A?}$JHl~8(z%SZZA@CvD$LYp0(oP<iivd4pMq1&X*s~%}z&vU1P|S
zq@2^kR6cyxe~&bjRIY_6yL*?G+yl8LwyC#Hq`x}%ZC&Av|D=z<+X;;>s)6nOOu7FP
z1=!8jpnHO!xZ%-(GitwjA)I^;@t8Xo&%HKITfAphbx~aK;@=it=1u6(0bx6qaApkX
zrE~TpwJ(j^R;5~G&-7p8`qkUYOxEqV;l}*y!2^7w2SeCa88hT==b}BkP?nlY7K~%F
znsV%Ss_{rkqiS0&H=IB}Q4Ct-k2rx`*xcrVM{gHgjV>YK@ytjH666r)8SfPpj)fhS
z!_qURfXf;6P9S=+8Bg<7^Jacw9jyJ&GNwWWFUTwurtfh^N`ktD4ljcLs5^I!z`lz8
zg-nJ=@@;0Ezq)jWB6GOUaguuAw)Y3(AAJZ)p#a=7)Uz;X_}JNShaZ`_>{Uo5IV8D|
z;q^OhfJ1S#d4R%!?%X$x5~Qn$;`%XE@iT)J!nwiKDcI4%=~%MwD@9nE=|Ei)`rtvH
z=jy_a_|sajCEtk-WsN`GfdYFW8--HWniUYvbB*?ORWMcZX6^*!KZPpjT)CWLJXfp!
z5vtg!K*nE7Q}L}F5;hOm%N`4K4d1k!-PAm=-+T^}TmUlxBkN}Y{vY!W_Uc##Q(J?p
zmNY2KP5}@i>{m|of|)8Jw6lAeA))8)L&4)JX5_);f$M6H9nv6<Ta1pEIBl81jm=k3
zIIbd7#A)E(Z;IbPKHWRAZ3kVX(HgzQZuCxrSz|(5P?VIC@{&9Gr6iUYs+E0E3=4)!
zCux(%wAUcjL*tRQ#v^t5C|pR;g~$ui1$=LHX>QR7W~%t_s!jIX-~Cm+pY`&{XZmGr
zwmj<Y<NDnOx5p}rt^t1(9}kLH=SeFn!`-zy%Xd%JqkY6UT;D#9br_{6{IHRlY^OBa
z7J5;>?rES_rv-*1IB+c0Ia*UnmA|jQE5A)u8%@JWDjY9_9@z2gPZwJ>x%aIiKIKsM
znVn!|!tFu3qi>6t**kqg&s0!9*PKbad?qjYh*@fUNNk~q)34#qu@+r@i@ER0UUwxv
zdmY%R6fH?tH^<%%9=TpCP>gO%95~+&5P7JpYV;K9+j8w`kiJV)N6cZKGE*`7D;RAU
zf;k5Y?!Xod`wNb&{kp|hfam<OQ|HF>cwdW;nbx-PpQ^vmp>NlE1u0@0Wv&zP<fn4$
z(e2Ynge6O;weq>k7k%rk?#eeaFf*iovv{Q|L(z*&S-vY%S^nrqe+{r}=vYWq7Z6UG
z6Bs{xr!cz%aC*ud)VrLRgzIZJxE`Of!zcfUf+<pBRi<&^wo-={UVr=3h=r^tx%Kd-
zC<0|vw^(I~{PQi(u9@7Po!*HSQ!FX}<9<zO1Pyt)f7F-&+<`j$IcIy(=zd(=y$rvW
z@^g>mR*T<Vg2TSOx?kZd9fPdt-D|tnI#<^+u7*(Q<f;F(kiYC~7#KS@YM1Wva~d5_
zcsNtfG!ym`iU@XZCvz#sC%+ej6CSCBNzz84{Gr3FtdKu~b9+x|yvZ6v_dn|95KD}N
z;RN&WYmIKSK10`q&VD*w$*u-&n|L`iIre=G*QlWIllGx!X_K3vDxCmAeR!5rToup7
zlx=v`S0<|9BpThWWpRxL6mL(=d=OLigZk|oRm}))+tcAidB>YpAv!m|?h;dReU~WW
z5S!0;TarM1Li$0vyvM<xJVos$NWFgw=kf8;x&|MRF)O#+iEO>c2CbMCdhq04xd@|N
z<j!2A=a@+I<M=|W+m96uiam$Z3m>W}*)snzbs^q)NX<FgYIWON-cVJh-DvNP9?!{_
z7sNc5H?PEQnXPjBzjLr2iMu1!9jzug?y03WeZNF;lgec}<|49P?#NnB*tg{uL~M%Y
zDatZn3Go43?6PHHgFBFGEor6u6?i|g9Xz*!H|eCqvzj~C%X3<mIP7I8TAJ21PDYqb
zbEdyei*?g%0q&1y*FL6_Z?J2#<QKM}hz??LkRXFxX^)Wz<Cz7_GW+34bA3~zi~$?*
z%2-GBC@in0!P-*zNB;Y}pnHU`u`$n{#*rvraQ8|FADK+pS6*N*_7={bE{B+P-@n4%
zg=Go4t<64h(0y0A_6h;nPqu8KO*=XV%$$1cpSo_jH53hIOO;B(^b}xEYOnjRJe&T`
zi!n&XHL`vXwGcoE-ENrn&Ld}s{00;n(yO~PpydfFh2d|-?v1YW-`u+>p+k#3<tYqN
z2ckYp4jjwMF}RN&30{1#v%?X4x|Tsf@hxw+o1Va?f(@pzegAv@4{pM~Ro+Die9IWc
zBeVA$k@ZzjS*#{+ZF&9X@gzOy?aqtQ)tBNb$(Prj1k7zeuMKcN*&Thi|LyVFIl)(K
z)%vFEGpLa~$8Jgum|V0gqAr}m0Nszw{@D{&Rr1D5cW6N(!$ba2JB5ez795-l2G9fX
znQfVO=exb5`}EUb-T^Yw*E!Fa*J#L182fprZ>%<G{TA2rA5r(AimGFd=)oy&n$!Cr
z1gW$nd_?YkF$kf?$P(%W?)VBN{CMgbq9bclAQ7q#<T-FR1(WT@^ruZA2R<|1D8A7X
zLP<weRqDlFdr8oLUDHqWkk2FvK-jv({k<UlyN?94Kmj{_qrN047r|F%z&_pRB9zuQ
z9VmU0Z<rlq8_6G@8*Cmbw*BBvQu1-Hx&yijUyTm{N2Z#t9hN<H3(<MP3}K8L39!7k
ziu*|UeE9TsS5Q@Oul7YK37_0y&Z~@yK~<w0dydWuR9K+{ksnqDV8&ze)~MEd4tk0g
zwlwk-0+R0^7g@yAeD|UXy2kjTo&BB=pNta8gV_2pYpLx#Grx{EF-*2-jcfU8LnRs!
z{5A}`uL`~2Xz)EFwpbZuF>hA@NzFCf<2PLTGo2BtB5O=W$r`O}6ODMf_n8%H<8Ejv
zDQ)2&I%Rn|_VVeb+ST>h#u)r?%tjguN@SBMxN)`;L7ik2<0gq6A2~1j78GNlB=X+{
z5lfC56?F>>KJ_kn<2T2CuggN$Bkix+MfGPYJZFU{ZGMmLk}CPC6nJ_ZT|kNmrMxj|
zYM-zJXwv#$EAkmBC{+`&SAuiyJiW4fOM=d8d4Onp`1OUqW<p?14p~&J((@eAM)HeF
zoI*83qd_JmA3Xr`OsC3nBryG9f%I3i%&oxA{MhrS0wKi5YvBU&?4z$<nei|D&U~i0
z9+bMuO?lPt{)ZKt5Ndqr9I2y@=X&)b)xvZ%FHqUJ<PLIc86RwDh3MREvWVur;k7^M
zYk!&VH23o6p&gjEU3(~!P1L(2ik7S-3E$M8DFvS_k!`pfV3v&NF?!kK0iiC5->LtK
zOwzooM4U@3ecpib7UUDUQ0x7U#b>=QR@RmV^k%gEC?fMJJWdeN6hA8xF(@GY{4PPa
zq^*K4o0Qu+M=`3N*_wGOB_7WRA)uXEF8$;gk-ie(elv6A+J@{LV8!V4;pn(dc~+7*
zxH?)|+9RzxfRz$?cbuRZ!weC)uaWiEeTOLuZF?oaELW@P=V=Ac<awFLcirqM@MHPT
z0yz+0P>CU^ygjR%mA^V&deO(=*-x9--`~}-H{t<ls)$(qfc9Y?+MWe9kF5?(Wk0WG
z>MUf1SK;T2n4{B1$C;#x6}ke=Ye0uKd09`*@#dI1=VqgI#Oj8~FhhhPIuli0J<>$Z
z(JbQ6vBgZIwX;LhY1f7ha7K~~1EzhS$xQPXj}(E6D}W^l*@5}u1H$;#DX6NDk5Aye
z35TGF_>0QM+j+5QQf9Rv>54&-W|jwhS!6FZFZFh6pO$^2hX$*D&WVoY{KFsE+RXHk
zPXlV==PZKPn`WT>-AxnPds$YX)yjSO0~>$+)hm38p%`;jt7=KXWh;-M`(xVotQ_tI
zsVjdn_2{oTv@u;G(s_?mg<^WfJhbTU+4_9MMQK9uDnroEk@)LkqlT;=cJbf?;Y^;)
z3DZ7SsW0U7^mG^w{D6*d)4|Wc1b^u{yc4%tJGhV!Cd@fH{7)_@QIj@Q9vF1pmkh&e
zVQZ!@$8C*#O>gJx>YxCA8cyyOMw3a<865{-7$x!4VKE@Db*Y?`Q}#-g)*}7#@sPKF
zfY`NuZr^0D)g4TK@h4j~EqlL{-!~qV1{8HEuZ1r6{ySjXkNIM$`P1G{Xk6bD*aEKu
zWgVOAm0wPi)h70A;V2wMtr#wFlrFCHv$yg0<+n3e;l<XZVdP<ypRxg3miz7y#EcjE
zi)MSFJAZW@UFXdUEmvQ53c3w$czGpLm!Qkho6pCQw2pqTw%c(+Vje{^l9hAQ8$zcg
zS<%v?cUa@Rr-C_(+-GfyRX;a%Y)#ti0)DZpwSD(rp%x7<Vn0TJxZ&A?6Kp5Ix=rdk
z9%2zohz@t<)KNqZ3HFd5F_)mzM9E<qU-P5`;@zD1@@*0>SiH67JQ1GcAnvNq%C;Uw
zy}Gy_U<Nqy-noxygpYteU*>z#E+oDCvx)px9T=N=VFvr{#*eB`v*tjv`*fj5@z?GM
zEPKuSi(lFVMV$1t(Ohn_tMIyVPn*Y}93vLUmWP=<8GuvS1bd%{MEK~;w#(Dl-6mV+
z{`5kbz5YI7Nmzv4EM=xQ4JXBWO@2y8t<l<i$CQm#8Ln&m`8Vs$-UQ>b6bte*i-3Cs
zTIR8mM2p0VMqzX~WGzD4XR9^bZwyvo^k*v~>Vqm9kFr+D;qxn}VD+1yf^$!~>B(&b
z?rGT`e&AD4q_v}4BfH@e_lNX!tB(?`)5YoRIFeeWP7}ZayQQq;`H(y6a4uB<ZaTmi
zqO+>J%9s5WtCAJio+vmW`s*UYl{;)TEV}yrqmvND!3(3nMkSsYUArGwbVidD>A>sw
zjSzr;c|BHaU0nJye*A{AnsjSiaIiI(<-OF{*K>&<Ibms1*tY)!D-&r0Yb;>vKqt=9
zVBQq>w#-I<ks}kS4&AW!S+|(UPizI?d4rO$wX=V2@#OJDwO48S?eqH#nlHcks1;%+
zj$I?0*^XU1ikdsO1JA#9&sra59fC|2<>+87DYot=@B(Me^)a~Oc%5f?_Vm9X{Z%`%
zRP$QUaR^~h5o$S4*#!lzvv96t8^f9iK#g>1L4c{&cBTUFjPG3Je&}Qx?`eav`q@s;
zEQ93=>d~ZOz5RAw#(kx*(hmB&Q~U~i=__Tq?E5@rsX5>F2L;CoY`g3T-!aELfVHU8
zSRa*A7)ujWX%I$XpLytB5E&+p7=SNA`tjjq+uG8Z1?`(0=POM{KeP!S3`&Y;%G+J7
z+RwQPw9yg5u@yiw>yZbyH@+lY4rVCAC*cyz1KWNl#x66;9Oz=_gZB%4gx@x>H(E<>
zWbf?i!-zSqy;4Z?Jb!cDEs8h*55@M7#N_v7@X29S!mB-+xsO>QA<TZ4{oFJlVZDvw
z59jLk4QEO<YQ5dh3LNr4S-fGz;l(TWYC_eyLNMoZNOnXGXvq9*x#~^FRXE#6D>D;{
zdj!eGoK>e*A0VXZ#ehgp!t4VZgfP9Lh)cKU<)^4s3Bd*5#u%Zuk%j$VY9Ewex6ARk
zM{r;(nJL)-!+E)0uDrfO!33vyN@CXyYr6k(lf2*O6x*zGnU0tf;f3!0QYH#p#_{7c
zo054AOg*(Wzp)Ct<mq=D<riTe=?|JHu80+;(kB}6GO+7Wkq0enaZ0{e9n?;{EI{G+
z$o8--Q|`|jDe+{(-q%nRHlEJ;*uhXOX(&8)Z&S)cOYfNvAu_-ox>(|J^_ldxPC+d9
z$3oDZk3FjT(uW>0bbDj$M)GX0Z*tCP&!ab0F3GhbrHif2^{(Srz)<?0gTp)5JS=Pp
z*2gRjsfRvlj<U#df=4=w<avU7%aQrVc_gN}ti)$!Wf3MfkRzAlxKqTS>lOyji*?$D
zmVVgYzX~s-zf1i+ge1y*br8K_s7n>aA@Rd%6!hYhcJC87cN_XRkM3+0?%l$&`hFA)
z5zCr>`fi_#g5?gf^W-N5)A2wK#8}jhMe1yy2|9OE;!dA9^)e&X5Dk;5e5A<RPq*ED
zu3GACd*BZYWo>FX-9u><Pj&{#*L3^HZcfXddup4?90#P+A$t>82w=J_l{@9>VkBD*
z@hvCzQ%5zu?*w-H-?JdGcZ4KFDMAHn9|^1q(EfhwO?}DysEdJ2FQIsV%!Z8ivT(ch
zMX_6kMcQo%F7NX&iJz6*+QV}M1eA)KI{`FX0q6Vm;~Er0A^5uW>=Bj^AI7TxH9p}v
z3q&Aj6Qpiz%8@%1TL;8=><DO%(uY~^vE17K-35?9Pz{NsU<t*`D};bA{jk1E_^_6_
z;ct&yVx!<4rPRHgeUIxbbdGG4^4G@Qhfz2w>^^i%B0%%)>^|>hHg*q0V6f(OM2JqU
z8hL+->{Ddtv#E;$cAvKZ_t*>cy20`9SN;iivn*~u^GjkbV)fDLnD)AJhnHyP(qHju
zgGM(zzSUdhfPuC#hf50h8eA2oPzuI1vOM+2u+ySY0&f@Q0T){=NWEttjA8xqDX;$v
zzVySACFZG4SuyLy?2mr|4BB15B18v~bf0i7?y;;ME9AT2fO&vgBc7n}=9QEN0T)-9
zl+B#~PKlRftjxVo+LWY<rXOQvq^$$FILGYi{5ZDt-W;_~u+MCNihrMQ*|&S?9oXyI
zqISr+S)B*sVid00GR9Q56f+^VRCnI0vX|0+&`i21($OODq|UgY<u(&%**BH)Je6ti
z-W#3RfH96W?#fv&@N*+&Uqa-Gy**?N<oheNVQr@Y{B-s{D^&-7=X;3HGhLZCgR!ej
zxV+hzZVU(pTCd%2Xu8Q(NpMkbG`u<CJu1foyZqMK>XB(`L*3JBCzkrLJ=UH4QP9{4
z9?X8qbMgDjrJJpJ1{E>#_EjwJVkE2*QKP*PwjmC@BLh~KJiBu9<X+9k?c(=SuH27l
z*8hjVk=$a&DMwTCx<(346ok(itSt{FM5q(F=6~_YPkI{;<qdV}@PU9gF`Rm_Z;TF=
zW1_BYTzq}Z<r8c19bXCyh$v3V2x+)cG447l^btADv9?7@8=UTbF>7j=>;$M}RHD)g
zuSpJMZV@b`((?SVT;!Hgh~@q-Tpy$J;sC6!Y+-rG6nLO%F_KiLez(uVv`oVeXDsfc
zc{z*1>WwP6x8#V%N3^AsoFoix*<7M5`d(LI-D0W^@!c*vY63DlVb4A+DC@w&S71_Q
z$@27;X>4S?`bBiM6x-UWK<%eG(Z<lpd<7=srm9p4It|k&ATr>O#bw-_6|*#3*oj#l
zZCsark}0w8oA3$^x5e<3Dws;Jm==pD>HmL(y>~p-?;AgEoQ$k<kbSHpn^0sNdmPG)
ztZWfN$j+X}I)@OEkv+=lgzP=bmJ!)y<`6>0_od#w$LITde1HGtkw=g2*L`2t{d_*J
z=XKp&S9LMb^9jsZ3eC#p5BJ77nz4hTD{tGM4aCE|XA2L<*uAl75!?xvDOj-SxoM;a
z(ukRzrS1nH801Fu@!~4^m-b=C2AW+E#pfSjVoWJ{a{{ZE0`<aMG70|1rW8|8qlm=g
z5@jsi=Fdo(N!#=t1FFw&8h+R$a(T~B3=9T2ED7cY0^NL1Gd2W1<(LMsK9*m<wE}pK
zrmP?-Fh90jwL{eW+t#-c9bI1|P$#XN=RVp1b$y70Cht8*1=6!G11@Xh!}J*t1sHQ9
z=7Rp39gn2gcGTub=m8SYJ>9UeoLs%hqg3cn&k-4}B}VmvGB|fdZac?ZBvuP?FmU6*
zfeR?9yCf~2n0Y+jzFbBkC%iE4V+ZIRp5;peCDOh{f+8qmn3z(-rRk-0S6eiSMQp4R
zrnQBPvKaOHUR*A%+jXwAmyNoX<6Xj6=GHmkU3`<{@=UyLv+Mfp=kmhf)4I{H{B18r
z9I&Jxy?C}o40rXej8j(J=9R2E9M%XqMXta--oom&NpGh7*9c6<+4Tk6w#})>?21pm
zQ*li5;3Mbtl+u*cU3uR}`*Luy&fNFme@aCoxJh3nF!W~U!*ghLdXwo2=atj-U7JqB
zPk#8N+lDl8k9;`1_V)OhrgxM{Eo&c=Zbq<IjO@s(Lxc-Nnvmv$;6;S&uY3{&S=9s?
zR#;f7X@$J8*x7m%SdWcx^BQo$=0?o4j!rld4s7FR-WPmq9e|at;S8a1E`??drI|6=
z++c5EKWy;%UNu(;hxmVTs`20_<;OOyX*^=60N=Y|n8{#+d!t!emgL>xLy~iLb9a8B
zi^&dC3V{xzM6LMEMsI9wvOXB=X<Fyo$iy@O4(dmhxj16rD~22sa}>K5vwh!~vg>ha
zp&8Zo?djDgHSUk|ZZR>O+r`bX!IKB#qKxr)f@bBFqSc&6p||h&52L8<pEQ{~n-6B0
z^WpFL*iL(oNQ@tIKn)-7SD+->l*A+YY20QdT<^6^8y^ue-#Ullzpv))^xfUfXh90S
z6gL7D+HC?`v-$>E3xp5`Gz5-JsAUo*prt2&yjZLhAIxE0QV-wx?9xuQ@a&#5@wV?o
zeF-HTYiW7jL8Udb*qGFnYl0b769_@@eU|N*e@us`aEo>RM_(4uf$I2xo`?zjES^1t
z@YpkeU?}kVQ(3oSy9eZp1^3xp05R)v)!gGF848K}-!=J<sxfBnNg4tn-GE1T@iIgt
zPPg2|2?4uMJHP&=b`HLtX#D0e)$VyzX$F{`!x4R!-QR}T8^4?uDUL1upqkS-`PM_y
zfoP)rE_IIm`6Yes<O|O3!_N~Qzj$yd<l^+_{P*;;dBf=tfRd_gK9YL&LfHWX&0!jl
zbAU++IPg74$lzFv+{}r-dw7e{V-vhKDsnnfR*gZKnvDePI*D#DC~4&K#^L0klC9Gy
zMTbblj~N)L=Ccc9g(OA}Q}4JD1t6`cXw47mI?Yb=<ciUg@D{D5Y_0v_ng`!(r4Cv%
zroKVpbmDEA)Ev&>9S)5%73-|=FXu<?>4nx7X4j0W@>Zn09KRKp->BUYa6TG<-a|e5
ziky3o>bxdvHW3$e5!BgF!1J7pBzo+46&6Iju;!L2>FW-&s^6E%o!7tD=cOx6jDyh5
zbq*P_`aSrYAuXfB{C;n=`?h~*($L8II3$@07&d6yd`J_Ws^kMIvx9f`gG6K^nSJf5
zH*`K{U3!9~x|nXVpkqJ?b=UrQ6E_f;V-j{T#24R+_HN5zMLT(aX2an57ARpE=pxcM
z1i+fXvhi?beESaFTH;sbm~0D0^Uc*HK}ss*yV-!BTT`NC)$LT>x^h;**!asN{F}D8
z??R#S(P0x{tuJn-hA$FGueOHDwEQ?+LSKV-g?O3`!iP!NLQRbvh^2^K?#G}4dOs;}
z2C26E%u~G+f{<QRTpt|mGr884Hs^yU)w#lO2H3cHIVSUVc5TeOm<Tbzcx7^LQ=ka(
z5Pei3g=bxaz|Ozz6s99uIs)b@ng6&B!2Wk9CWo)vMY+E8CG!z(a^zY+DOphECt`0n
zk3n6r_)3ei;gzf=lP$x)qg19KoBD&|jIxJ6e(Lb_S(7bWp`Ff;oESmg6X@swKopvv
zn@o_;<zz?<6(Trxd_ctIX+c!wx<o2#Uj@s_gjNodFTO9y5BKi86CHyOl<*R$<^7Ou
z8tlDc9K2)xT40JB1J<FDCrg6qd#9kKPcUiUqpO}Nj^T2f^EKe+&fU4u6EVPp*pvt*
z{v6%J)PBJQwP|pv;C<{@LWr4HDobqbFGSMmQh+DCN9V3aMKN+tZ^}PBbGba(t9XLz
zHm!=q?a)AWUVenjxI_4{vud;BVT=B$1(%t^dZDK)o!t^A!$DOpnG}c_f5+b7)~DRM
z7vLhJ)|7!OiumlAd!(7c`H9+65;D8p2x01b`$?%Uy_a`e%{)3`TEVg7?>U{gnkc;{
zzNmzB-#vkRB5otQlKF~V1NDhSYn974c8Zt$sMtHHryy=hgV~vB_deaHmZy`<kHy1w
zwkq1Mw~c0<#cr-O>@2ZEu72YXu~l!NhOB7ps8CBicy+I)$!Uoy04~j)^FH-35+arD
zI8$5vdWjMBa9IY>?k_GnJFvw<Xu6}LC`!(1p5V}q(<c_>MmRA7qR_6uP~ti#eaDDM
z1U{&!=nZ*vleqck)Kha%SPqwT>4!*VS-+@YY|UkoKz<zZ@CsH9bAIuV>)ir*^>#^X
ziFEPT=%f&0dQ>0cj)Yly4#tu6%EE>f3)@#{v?qnxrBV4kGUZF6oph`uda%T0f4xF;
z)S|q2K&0}=sy0QUc#BmQKUT>qySbtuVG45WDJ~6@qJo3FVBEpI{!HQEuNHH;4V<+Q
zG6_7#i1mH58?=prj6Q>|0%}CtDGuezs>+-5!DFyu3;fJ@ICgZ0E41@jttxDr<+HU3
zRoDb=H~w8JRQIs3=tV(Q<DTVg82~#T+eKz>yx)=X>!O&DE*ANIqcxZGR&{ZNTp<{K
zL#F2?pdm5JvFkB53$x|KGc=Zec)+|2IU}EzW`6QvJG()SW1;qH(**woz6@Sa37Rv$
zn2S^8%r>vUUw(Bnvv(k}$LDKCO=p?n#jy7`b5GzEEy9nL+rF|%ecDO?0r((zMd(GK
zS>5rdbO;qZP2MRRO$;2X)g)s8#P^72W~Pew)3gz6`~-8=_I6Fn-c0L2AUmUxVK38N
z?a6C`3|I9y1wA_Tq)Vz~#wZRtXbgfqJG)ZCEEGv+iC_kfoyd|dVEp;_bw;G*16Br7
zSG|uGqjflH@;Ro}eNi96j+Bdq`^XuUs*~(toH<pSLy%C<W5l&Kq+^j6Q!5O`L)Fw$
zMSX6VN&To!9eESM%KggdjaYqf&j(`Xrv+LWJznB&agU}kPKz2Z)hw+VqS}=$5*~VN
zuU9NaHC4X#_xq$ylE}5cKq&ZdDKWCq)lUoK<JayQHxv1r&SIE@_tbc+t5?wPRcNH|
z$>toW%m72O-Y|yp&s01njB%gk=Ja-7y1OwW{HCZUv5((x=bm?K$R#J!hHoZ$@TvtB
z3n37Cfw}Z2qc3PNRB&p5Y_CJZ5lN=Zf!?#DqyazN`X;oOD>$ck=1qxm;GLEan{}O(
z$FuyJbYad}k*c&oumcndVfL(yUat@W$3b<H(FpXG5_9m;@Uvwq^su^arw)2~UJHp;
z9ykRwqOUaH0tzM%D{y>)u~D~IW-b!U5Df^htOG!uZ2$Co);T<Ww+;0K%w1b<aJ#m-
z)77xOF)<N<l+8;Hrseu?pU)5Ta@=Dq%$j#KjIHD5MA-!^#y{m@wP$_x^GzaQ1CA?Y
z#a(F{zGZM9|AT%_kExMoPp@|f$3wS>{7WeMV#Db$rEw+)%b0$Z<09T`;x}QWVNqD(
zqFxAs)vo0@EQy(sV>?olDnGWlOT~|f4zpAkMWw4m=C)bcsj`!wqUG4PIkZ$B(4&m4
zt<x!Zy!ZB3q2pJ6$OYq`(JiCZt4&!xER&o14q(38ALVS-+KjHfBWBJB2(NqK2cP-?
zx%47<Kw&M}z-(&h8~N;YT{MOf0tj2+-O10UZ6QN;pA&A0#}>ROd<%3Vc^NpnJw9bC
zcQc{SGT)qTtCXCc!gDNv?hF^#R$7ylKM70UXL^9s8#*Vj>Dg9^c(2<gEDy{wZ!8(8
z2jP0gsg=VJaXZqn7v;Y`&&I8%#onpVAUQze3JRY~{Z;E}R=k)vz$PEPjf?83Un@vI
z`RSnw_S6QzO`3c*<Z1U^1Jh*xh`Sqw%)Pm$%j8`TLn46<F{`oJ`?*zP9<&_$0<ht_
z=Hk?X$87-XeJ`qax#<8HL0r5Z;pDsqC`ypXN^Z_-YAOjlGn}@xW^aNK9WI@e##+Kj
z#f~5?)~2A7p5amQKpdUtk|4_Igd1WW)A_k5u=(lJhy3dXOC8l1ZVUTJJVk=rU!#?c
z6d3WT<rr>kVc4p4jHu+%(ouAhKUE}G?3vux3|CVm!%*%flf-^2Ocpab01enN{GGgi
z`4B8X5fEI|>_P!gj4=&Nhz?|^4WL)u`56<3FGMk|dymXv+9bh%j#m>J>;5&v>&ti<
z+6BZa&J=LCFy&?Oy4Y>zJurnv$3tqvxM|CADPkTN6qN*x(tE_j={2TU^!U>|`45$L
z%0PJ=!uxSt0Vjko%V)fLyL?5;#0eOdikS48^T#JiW{MQ0vn)Fy>quO02H^UJAQ4%?
zwVw<ZHM2D3pxH=qv3Jkp2Dsm(!@GT}Hf$3rLUmp*NyhjY=UbaR3izImUq2+uyN@a_
zrbYj>jL%9DG-VL1oe!}*7qmb~qq^)<u%<mb`9QJ$X9aoiw(FVx!WjgSO|Ig7>O#tk
zSDf)fQQ*0V%`KI%=AYl{-^><2oT`2!z~;>#yR^E+LZWn82HdlNV^`&Z!<{%<Ao8wn
zkrKHqF9?;0nyf;1zItWi)HhrhP%JBaN2)8_?gf^b?tx+sE186{^$A^Ui?s`pq*CHg
z_E2uFCaOr##V%J}P;L!Td6;*FsJPs!`%7$(Jjw#UGE_Kv{^g6>v`{N2sK@oCk9nV4
zVR)9p7fRjlplY#G3VN)6gnyoZV5UEiy^nx>hWQV?vHj5J`D5)PEJo#cpZ>%5Qe~{g
zPC#u98}o;WoWw6O{>OWp_T@S0y)$(8Kb->;?D#(&G$(eMdh+`<U%105{3afMm%QTq
zrOfy(@23x61_RF?PcOUCvbB8JF=*ON4u^@g=7MSXN?CD?LOLNdhm$3Zqv4$cQ*Ry+
z_N;Tz$n_IxZB$uP&Hd!x+&ge+%!{pY`U@Wvf#<nj>uO5m^E~ag=<YzFfO5^v>4OX`
z(Ag>^weDH*-L5Mu5<1yh0&Ux#1cuA9Rn#xK^h^Jkp#prf+cu!Yg>Utafu|sfNm&wW
zUlGo;>`_gIX;Vc%yd83-v=)6o0>2Kv7Usr7;>uK4k_rR$=}wro-%*ds48b(SmHr0#
zqFSM=MWOc|x`~a=?s`ZTL0%Psg6Vwxbvq?r4JK)s$)SksrSaRDUCM>MQF$*XmpPPn
za6+rxI!Bc6kWOVsej#mLGG-a3OsM!Luc=s2vkL+BC7e&QAheG6j<n51QQN|`emb$q
zaiPzu#>O310SqL(<ebBH`x!s67JR1zi5Ir*ixNwP`?z^xnC5;yO9+<pP12Tx1y<+V
z)tXS?5>bv8X!K`(X5LK1>>o2kd&+z=WKyT%s7dBCuzkFEJwv4Ub7CXlBkI2K>p%n_
z6JhDer`WXA%eE!NV;CPbRBkn&0`pbA>Z1&pW*qyTT)S7zKLZTOUIAt=x(&RLpx1d!
zq+mey&Uv;jc6s0%TLynGIdjaCxWPU;S^n|&5D<l27|9-oJJAdLbr;-as+cR9xOo(y
zvS8;3ZW>`MEdXJy;&T%l_trzj+;_V*WhztGhI^S!6?1w;m#O1+Zc7ymv<%rlWpLEH
zOCTV7bcc%=grr@|vwl00ve+L<H%X!@C%2q+AfLYXT<3%WDy-HMmg<QHp|jHF6BZWE
z|CN@6*1_||HklunX6(;u-~8*#|B3{mI&jc@Ya4@>uq-fTB=_NN=Ih@8;MXrok_HEa
zbT7CX##m7TeFHczaC_*8;5d|$#7uaUet4U9?dY(cO!#;!TYRIR(lPiNZlE|pm9EKV
z@(s?#{D8_dVwp*+iTL@kSUZcu5?yyvr7&VQ0=`B9t+Ig8UJEULIU1tqLK9j!VPq(U
z#Mz{!qNx)Dg-rA{sK?$IS2v~<Z5PG;?;I<13qmIw;5P}BFRP{=6TI#GE1G{n=nJqn
z9F9|e%tisn%FXxc|5A|t1iai&@ahG#fEm(_rlkT7P;oqVb0JlCo;AFtphF^zPJ~ua
z4@()2>~RN6wuUgbbxL*q31KH!L@n_;g*hm2dM#x)I~<jEg)@g3f(U(@S<ArhG^tGp
zc2lC#_{3^wx0nRkS2U^|qYz6Pm-Xo5IO?|gNb6yB`<iUm^{WM@?5#@-+w+kB9{wcu
z3V=C9@aPuYzjvq1u+l0iXWpLiB31vrgTcd*L7v;ITU%tDe(0u+ZTf%yIwMj}0qf0*
zD!a+KLo!d%LTJ&*G7?#fUzxgt+cPSbAU1z#KAlpO8`eRiC4X9kq96`*h4d81oXgdp
z^o)9A*cL_ds{IA7o^#n<e@f02NPs2+lf95+cj38isPU}Is`r6OGnofvHLvwxp7MKS
z=c_m-2rC9(%)<}j9*r<PwF<Q(q@aQyW3lId=hYwxT{4l}x1i-<U=}+p`1&V-fBn_3
zN{M4si>}sMHTId)Y{7GJS1V5w*iys{Zg0EQKZ^#Sn;Gbsm>JQf1E(8J6t-_<`Ha&>
zamO2X%$|xZ)Wie$!_}iIZ%_xTS=w@`;UxPSh2@c@B0Y;6s2(H8=*AH@T3GzOnQ0^S
zirCWaNPE;vcXgDTBzF%ro+!<AbGI;YjIeyC9_jjO)FryHOPOx&9H0!p;@j7Di7LU4
z%Y?1g1zuPjntz`lm791#xe$2eqVes+i3pqiXK~I#@F)M-Do9+5D!V=D7llfoJ2?)>
zRZ8BgFCz)4`kMap5d;*n0gS0c`olsU9z~cX7j`*;2P1{JRRTuHI}N{XyAE!}FT;_U
zWW^FJ7cxnUEmCXiGp;0^v(^_eTL5~kmV1&Yjo&R|)Cc4V9UQ7Valh!{ol5Z_J9)-7
zxwnras`+)&Pc8*<B1}6--PaSv4~qgj<4WGEbb!~BbY=S(ND!F!C=mLROwLvI|J;e(
zDi-fkg)I*@r!3pThyez=I1yav{Li)oCqxISLGjC9^*}d1z9x~DmTo;H?dTT$*GIsR
zT%?SX$v`c2$4X9e+;@F0RP#*8M=c2gi&V&(pw~Uri{d?zVjyXoMI<6y(&-jYmb^cu
zltAfu(SQ|?m<Q<IemYOz8DicRlmiMKYoUt=K|M5rf-SSYR2tLY@Z{)h*W`%9#HF%7
z96~|;i?a-wv=QZ%J!WUSy6SsWR%us&I&S*^6=*P=ccHn)T_|XJ0^IxQVE5bDUE1)%
zNB>%zpi;~I@O9caYoh{<^k<Ozn~sxUnrmh+P7lhzE&Qdz{yM+_rBq_Q05qp%C}LDl
z4>h};b_(*bT!!Z%&gz{^xmUK5ojOtnSR(L=Dv_is)sPM+^TG1i)uc|{Y|{{O^S8r=
zi~R+9Ww_5o2I%d<MWS@c#Uvbk>j;~-xeu`H?#o+Nx>u`J8KPev9^z1Um#bFjsJq8r
zkuMiHOIqoN2!P=Yp$Jghf5HV!-;%Ig35c-|B)sKzfkYDi@?T)*B0WOUK~d1R*B}TH
zg>Q1=X$jUwJ^u~Oj8M5>+%_jO^{|ql1~`>&$(?M-l0IkV)o4oLdE$8PM6fqu6@oAe
zN4`+<REHKJZ?m#$6QlX`nl<7i3B>*>O#=0SD1agFM3g&7+h`}*nlJl)Av{ja$akfK
z8d;SeCqmrHskN#7>OJcjf;7m0-|a*^$Htb!+hhp}XQ%w{M<&HZJzSt5cE-8Q>dQKV
zb5O>t(E09Pms6}97psL8V&|?!4LRse_u<YGpUO$~*MJJwQn|&DfBm8)LGTPYbLlZ5
zry!ov<vuJfHw=q2Qn$BAl@|t{o@Z1otXs@2<PQkm(@EZEleH>haU4nAeHlpUvEmR@
zvYQi?^kdSoSl$k2SzTe@eSWpbfQsN+TexGCveg6fR6g*{3K+_5O6Be)?tPfll&Y3=
z7Isduz!cWH#7hMa`fpl`XBV>ROZtUq-2R<hfOKz1i}#wH9mA_x??UI;T&n;2a(~J-
zipEB!73cqK<*E|M4TKhTjl~iu#j+Y$9lDU{(Hh}H@eFXx_pp@Qxw`dk-XP?$99r`o
zlBud)KZkCN=j+ksL&}u1?@KWyZg<99=?BNO5=U{0M>WZL6j_Cd-6z?i0SvxXRmd+!
zDj|l)PQ!P`YaNO{>0yhUZx`=@EEZ|49BM{KLE-9Hq};!+WD8I<WZu5HShp}jEs@g(
zQ4<T55`%BqAKTyCwEj1kxLD#PywIyQH`eEh{Q16!>j{fFPR4Jw)cEeNV}Zf85gr>K
z$?h7U9{vtW)1aX3757%<U2vS7(nZ4b<`+Z({{h1dS_x}^{uo)-Oq_RNA~X1#3;0Mq
zwm|3g5`IBk8Q#${Nt?ZBMbxz|o(VSn>w*gFF>&jQaP7kPsSK1%=@3bv#Xt0=hO4gJ
z8rQzAL`JM*Pd=9ce6J&{Hke4Mi%o=Bq7t$f#HZF@r8AJ5x;gLq)(Z<Z8ThST|A7;K
z5XWD4HC+@@EXQt_YJ*H>;_stJDvQVIPb&N6+c43XBI{JEwGPH+Lsa#O3n36mBW^&F
za6?i#P)*=`!;x7=uJe3l-48{`nKI#T7U!mv_u0S4nC~sZI;`>^^&w9#Y&{eAZ>5q8
zwW0@Z&~pmNYdw*4A$T49aAfRKd{;0^gjS`pH%g5KEH$Dw647`Z3s{;7rENCzMla=P
zK7Y|c&in?}Y8_0>3?crXWA{6y{S7%_mxFCpp&t%oxrgBNzw`R4FDQ-8`Nq=dzMKyr
z-s3|690l<1YVLw@;Z~Sj$F(2|))7MAx4B$u_e06ez46!|W#?*=KVp5|y&fdKc+-Ll
zavoea%~N{q`1bx`iR>l=rA7l=VSu%VIw=`Bdw9$6vTz~vrD<qktl7IBn&nmSw2h*y
zy}BcSn~fMHaghQ~2NGtbmvd!?*r=>R<`-bbBH|}m|BCRx!O}m=zzz20i1Xco;o=Xr
z{sj`{Ab9&orv2#{E8_=%$8ZDO1A%|+=yAcFr>OW+?h%+EIe$qt5=zWQTNC8DzDU_&
zb(LB2XoxVxTt>z>PW}XR%o+75J0Op0P~J7_rC4A7r9xW{D0iqfQ?e*luUw2^Fzmh}
zeXw1y^n(>`{U`jk3h5_Z-ENoR!s97!U8(C)2&lBp)8p0$4(ecd*&g}-?_QY3$%!ti
zi2567eGL8OA_GAA5FV%mB3ZI_Zzx|8_)!}_cp3RD)2>>K^Y!-_x!{`*PhAK*SXc{9
z8*mN@7jNd7gcl9;FfNkW!ASg~ikrA|+YJ;LPlpOp$JPk%P4mDQ3?w<HuiU)O`;$fU
z!{w%13~&z(J=f>)n{+o6p#JNT<dC7HOVmyyB&?kp8%b-jS5%p*u4s>a4@k^)e&&T$
zeffCme=zvpdvJkyf}It!8G>`u0*b!eoH->Mhco@L@1k2=sQ63v4K(Hg!r*u?ArT0$
z9llHa%Fis?UY+V{9pO$pDM}{r_NYzXo+shX?14HYLgAOJ7D|~uE0mzeUumMUyeY<Z
zAcMgoY9VH)ss=Z;LMKEuN$<FU5_Oo!v<nzGB$P-Tck)qMX?BX#p^`F0k|6V=p@F>Q
zTm#vp?>V!QSosRJOiBbMbsi%BZ&Lby2l-G25p@D@wEMX21k&j(-N!#5eDywvfz6(J
zie^pHr-?ZovTFMawsuSK#{xd5LtKuJN6JOnf|~ODwl>pw-87VuhnGeWce@x{XrD|a
z<hE<V@yC1BCWi#?xx0Wn?y@pVDpl{H$(suRaoFP{>&_@EJ;;WPApJ%Y0d)MiSz<!m
zVv(TG4Yd;_E|@()wQwiTe;#4#YNIG^uF_4;%rsopg1{(o{8GOEvl0LP!JQ0Gxg(a{
zG(f?|aaX47&sylzh?3YGsi&ydR44^AV<4-b-w?cV?^gpwnf$FB)&50(3qsl;lX`dU
zLg_fRvmp0LFwi<7s5FkFBU@FNP_<a3i4t(yyV^V^Sh7Kao0c$!))ck0DkhV*p|l|&
z7FG<^*ixFzjx$6eq0yn5m3E1;zUxGiSL1H)Af*L4I}=R-8<(1J7X2?2hs*IFOz`hT
z{357zxWRBWnEBw~OS31RKmV$kUL@w+OToT*aehylWQ<*Q;c+c4Y(rXDao&R(p@O@@
zl^@X+2jx%+rOoU(jsaX7#}1ny7CFWP<YuqE;+;T8$aJ38Mu+AR#U$(Uq9j<cQk1;2
zv)WRpLIpbhj-Xhafj>JhRR6h&I&B}3OvmW5p1$K$!YGC3Bx^C5)7Yh)lM32OMW;hP
zGhoY6c01(E+5g7a|8s8u**BXH+><D}yYEi<cKS}e$cla!?`YOvs!t1WcVs8uk1?*?
zbvp8Gbgi2OLTZ<Y1A&=gX`BRDg8Z#bi`gtm_9mPY9MC;i2zRNVl%IV{W8c`Qm^cA@
z^pFaZ`?3ppPn4PhQjuO-f&XL(wr<|!KmGK)H1X+|E7u8v%Q+EchW7MQ#P~Ge)jLl*
z+f{H9tYJ7-l_EZzF%<qf`i`5I?(^%bGDU|14KBpyd%z(p1#CqBA?E)gtbc9NsyAqt
zBgW5^s;I!=N!m@T^y7Dqf1&McMzCE{tj-6S^1+FEToUv!kvl%#DF_W)G^Fhu`y^}o
z@rN3_ehYoP=5fI2dxFoc$Eh8r;2pesN+$%aWIe-bKt%<kHdJ^L=?C8&YE_PRlBxEq
z5V*RAb4y~6G0G(2_@NMKd~o~-alXu?2(cp%--4piS#-W9L=u_&!@y4vtU!7KIT59C
zJF+gVBVs1;Q@X9x`5!#}zw43fW{>BxTHYXSP)*6wrcZA`Z~UQjDwpwWqUe=9eN3BV
z?g;I|W96hEs-D=@sHJ%%jZzqSp9=v8PZU7t6r@aW{kUVez%kG5Zi0EI^A(#`vBA1Q
zvT7y~hVPXm_jU{Ub?Of(fcWq2ULXB<Bui%2jj4FvE2Q12M#vTt9%NO4AyM+<f#E$p
zNE&-7wx_Na2JX~5AEAm25G4=RkaFT!j4##jN1-KAcJHP^Xk^&`3MBuDpkR~|t5#pn
z1igELV1|05=hU;eiH(o{khzQqIip2o>H3_520`=f8~ku)*pL&^9u@pcx9hc7;t;}z
zKY3)?^xsHI046r2A#M6H946cZek)<B9U3IDgi=GWN2U6!BriE5IKJzZ90#0pT$UXc
zWPX*EC3+|VTyB~0I2U!8aDzuOMU%=p8?e8L;kpw^9N=kqOjFZBw{SEm=hq`W-Iw=Z
zl!P+zH%C18!^vz`82CIqv8e#b`}{wO?*B(*U5BWu(b^m33XG5yyw-<zdi>$C*;J4s
zmD-|$2jp?p`3LszsCKZav2rzR@xa*@NBL{0$BTgh9AkrgBUk3>2;RBU1DReRL0Ypm
zg;@1y`tFcc7(8cE^CE@ZWZQ37aZP>*uQCXnd`9VE)+hcT>G+PVHffkVED5q^lWP*H
zx#7?3f1Zm-(gh6NV`WWt6Uqrr>j<OjBfndY=^lVWAZS?aNbmXoPIa*@6)<hIcEY2<
z33`{3+x<sk<KM+rOVwc2<5s{FmfsL4dz|RTt+E!GZQat$N7g66k<84pZaipc+)%N&
z0s95A!{#f6tSZGgi6HE1Es9`y_N*8?HtA|*VnIALCj6=Fc3Mqbr&XR(X#wjPX10_^
zC4&r!=l8RU`mPX9zp@)TEW{5od?_j6nbtr=x7xk`X}F*c$JWIDh2e+wphy-^2U?*$
zShL|P<fh$&2NV!(WT@S%Q7fMdj%O2sh=@7Aj`t^U{%_Q#Vv!0dLS**TeBB_uIX=!;
zxRvIRpy~8C^mf0Frwk%-_whF_*QX)3m#XtK_3Ybt+duFQLbpZ40Yc}(XrzVryEqmj
zKAtS#R3t_m;sY<+%oDU+i*LD_ox#D3-(A_l*rM4$FkO+*ax<Rd-KJfk1_-=EdH=TQ
zQr@Ez<m;qrgogy1_M4Dv8GI20O7Jy$I6e$fAwPyHf9hzT$nAOSy*Av<F)9kWuQxP|
zDz~!AO<g|Z)j+Pi14cBSYhBb!VxBoGTL1;D!HBcfT(su~HQf16Q~`^}km8<-r5#3-
z7}Jaf|NRkyKeAPPiPo*MXhIK5sv~^!9F}ABvYv$_fDh2uKiTu~5BCjrR>>yq&FSGB
z&tIXAaX239vH||_b(oN48)$3ZRS)wm%HBhr-Ae97kU6U+j*Tmxl`x-muIF@xe9r<I
zm`|XcF<ygT-;d|EmbzbYB32%$l_N_TBCy4fGTj>`|D-)<K2B03o|=>_LY((uMajeX
z!+aYJ9GoFtgr1hXPIHI0pW_vL|IcVo$0&h#WjY<lbxv<3uu6y>r2VFfO56Az(?5%9
zWzU{d;q)e@QTzEf%wPa)<a7)s2)8i@4$`&cLOSX-274_hQ?b8JhR{=R5u%IRAw|=N
z8SQY~ka3Y}$5a5xg$W`++lk@iRE4}cg1O-Tsmv-!IZZxozsefLL>Xfw9j91dOR#zc
z|6cOA;3o`4Ezw^28J(7BNTefjv|@fHR*03eO-@v3(aw!QXp6|#oq@x5s>|vU|I~HC
zDeFR0@#VrZ<uTT7#HmGhZC;_dA9!=9U*A7!zOqHi8%Os?G>x!2cJ=ea)r3C_62g0}
zDAyb}IWqLm72?i}2fQrrA;vw|z1r|_GzwOtG8S*<BYD7{F<mq5P%7lQ*dgS(IVoft
z>zq|0`uV;8t(wO-H%vtS`UXRB0NxZ7bDF09SZ>bpvCNcF%=x}irD3#RzHa6ZV5Z%t
zcR((a^8ir-Vw>Gt?t*0D%IZP1UZcIycUoGx`xNU{tuj|OE2*}FJl%<ehZV<I4M+n3
zQJYd}Ig~6!ddog=vPe_2jagclr(c$)KML>J8^P*3U!#-BL&ONUagf=bYs+Y6=m9T~
zmL=TTAFCFgVbwei3!KJwH1;w5T1Op4-ys5!FlSl;tHUnU8gn)D+}}h1;Q7Z+_W^_5
zkUQz0gZ~=Q0z!YZmV64Mfe79)H71DkGo6fOyeX)jE+(L!2{+>X`X@1{h~hOiHd=N@
zU$JZ_cuheO2Gg2U-LN+@Q<i)y0T7{VCOnCd?zYwOPus*0d=xHl0Rj3qjy7kZhK*Zt
zCE|_<BCB9K(Dm~D$Y)hgkAtk`NR_HFIy8t78NgV5Ir1ep!B#%#t1U=2_!)V$NDi@s
z)#|X$1jxElgbkpc1BG09q{K2lPa~`fg=741^qU+e8dh%003!a6Pmh`}QOKUS@Eg}%
z!4<FqdW>Hi3@!aRxfUSfsGtBwz({aQ58D5Sc5Yl)SUn6;d#d9-3?qEB<VSUT4POeh
zvdSC`-`Hv#?1^Xz-}@6Xv7DgE#>*qHdLy-G*UXtqW)*uA#q<oy=$4i1FtfiSJsu24
zMSqkQejOd<5#Tu)q>Dol^n#@dORThbfs&?r5T29KkAk2cJJfn98jgL_x{OEBk60Pw
zBI%+NQCFn8<nEBV=Z9y1P?cm&76q@9KCR=FV;RBE;p7G1FOSMLF!5uvwYyE_2f>xk
z5Xq6bV*U<j39Tk#S*MWB`V4Sau~r!SaA%G(DHXi#kllCn&&I{d^@E2V<vQHtGX9u=
zPmlX&?={!}D~E1H+^s_PHkFc6ea%*@36xXy6sPWMpiX+PMWhm@+;g!u;49F@5xLx_
zZ|=psNmgUd-MKq2;l++e+^UGc$N@zi#)<*05JDWZ+zjsY5E~Y+Ln9u&hs9_0M_ywM
z2oWdlkVCLnulMDav}vgR?D<5+61YcnwocU?5svdgwaI#&G`3Yp{B5q{4ZAf_yY-QU
zn}VF2A=17aS%s1iv5LZ2q81E#tZcAkkXe>egsouq;t>QsAf>1AoS1vDXa8F9_hX;2
ze^&euGV}<rd>Y4;=Kfi*-4&oS26MSLfraa;i(T~y&)~*guB;pOcT$C|RSJAhcDmcw
z$4f_TQ~pRwvA&=J_2*BamIPold{?!)|1NXV$pBLcpVn54yM2BCfASwk2q7Ft9274M
z_9j^5z8`(a@&b2<pr=!<#ej!~uY+W5o;;6(VsBULr|Q=7ZI)XMRPakDofl2+*jAeQ
z2vDJ|5`McuMvgTr1JgUs7_(3=gXXl@#a@(&+D=6<4qjpe-N>k{&C`?0{IY7p_kKlh
zo*>ps4VlD&5~qO=AsVSf8}lm%F2=0B{3izrT;xEY)rUTh{Qe&_l1~H#$Q^Nndj&iy
zwD(pae0-&K5-?ujYjMj0PZ<QyO8rArTb>cAXGn^E`d~d<=kapd*YehkwvmsFm8k#c
zfeT_+eWAGnb0XSO0pqs;j^!+iCQuiP*IXv`#wYRv9pX&taw5hw8HJ;i70T%<Q>Uy$
z%}#+u?eLURxI%Iqf|`$2<}gD#Z=O<e-ykNqD(xz2q>(Ba6c0<|mf^sa(}ao;um?^h
zQsLLLw-Fp5Pk5FTO=VVB@$Xq_MA_2<b!(kfw_)}OVrJ6-Hfg^<M_VQPD$#090x`2d
z@TmNsAjU8O#ztfEH6&>&rT7i`wWP26C5hjac7X~qt>LR(={MB$fA2j|4g;)%d%$I2
zxV>0;^~OiKU`u6I1W?!W{LGkmh=-*#{Q97gC4}G|5j7SU?D*CP#B>Q{hT~avRfT~>
z%Bq;fDk>%WtS8xy>O5F{7sj=ag(7PTIZkrYOQwC11|cYsU(%w703I`sFi!#$5$hB$
zjSSGjt6?U2G=lOso?z-(;fpUSg=Q>vTG1d*btZsqHn3i{yuaoL!XOu9^`*{@eZu}e
z@GV2I-&eZy$8UctYu7I}n$#h8kV|V32X`&0u#TE-+Nfv|`Rodxe%NQTZ!v!-IJ#g5
zBJ9uD4EW?u^xJ|QQ}H_)eSM>~f3jN%49IR@5T%3{7KPV8+PUNmw<DF<0)mOIEE7q*
zz;$zk^hGO5tlU8M2MMUX0EpAcolHDQC7F)A7st!iJrPX9sTXQ#iZ$`5fI2q6{YJN{
zBqu@wn$$AdF?z;x-Ex{*SGZ>d)`A-`Ig(fyFoN(X;QETB4ZdY6b108ZeRqEL)4b7r
z?anm6X>`!9gFZsCYWc5&uKfF;^SmOr{d3T5F4ix?jd0IS>UbG-tB}dxSaZvE;01kG
zEIrjTfqOJ!&dUcS|B$hwh?ZK{rM6i5>utyT8|}bb;=pYwn*Hh~o?<Le(-jx-+_z9?
z8g4@yL3lCW2FV!~89J=G*s{rnQA|ml*R=&%))%2Y*H*-f_@9hdY8+9FRm$ICbl(ln
zVwLLKn~g6Djd+n$c|w<}N=d5ZL6%yJDb=L6%vPrves(ol(Q#8$cs*WXS&>naQmjLV
z1!KvVP`-}I4@kO_0yDPlh8YK}L*XC;uP??Oz||#Ho`rDwYo!2sr2){ZNNeufTFJD(
zB-mj6i~L?1&ot#)0=c<+>}OiE33QIrAGK3d6GQ&K)=OY5WsJa!%+`|_dtJi@Z&%Y0
z!UT+Nq?_{KM84MrobkVaGdja{T%6T8RFOd@Hb9NuP5X->jW-SlU`gpDkSA<r-ItGS
zfr1|eiv+lk_EaWZlaO?T92LJhD}Kl{c^i9sV`Sn3@yA%C%gbQ<wvG`TQF?*`HNI}m
zP);skd%sHBBSojxJgvu@mGU=e+<+=%mkRtyCxykPONoJHbQ9>zzI&#~_5ZltE^<9V
z@!yx59nVFsryhXNzd~YwnVlL9MFVY5iJ298{>JNoyJVKQHFSGiIfllDTv;hjudr$t
zlfrH~{jCc4f}KRvVEANfsAymUkJ3=KU{i=VAt)azEwjjjyd9LWI3(Dlz;7FpWwjU@
zS4$X;@av2N#C?)`(^osVGQ~;PQf+j3CqTF8zsBAm8e@p<8+%SFp>|sI%5#s@EDY6P
z>ezP@mG6zG<>5EvdUDxjzc^LrE?aP(&B^mnKd#Oypv?VJ5CCz&0bZdUY5$`$15hU9
zH<Y0ihj#pdGW{3*2rKcZ+90110;(iUpOV|S*%``Lso;0?O#YxCZARQCDGh8<T*5s-
zdSkta=k7CRq`j%S3yI+zqGz<KmVi9~d6V_txbkKYU$!?UN=lAi+7FN8^(c#nE47|J
zN@kn!?sEzW_Irzk9PzIzKL@7Wm#+7d3RpLE8iR16^0w?{@uHa+SM1gMMiOB`m+Tc?
zS7MwP%;gXsG>Vl)x&;}A(R@F9b!r~Yv@SVp1J&SFtKYfX4<haUPwuw)D|d&E{Y}I$
z)EBw?@>eA3v!}mucdk}{aR#uV?SF5m8xZX}<XrdYgI`{-k~{>a1;Wb4Go_$J?Cyfc
zYQ(zClM>_^Mu&+?lrwEQI$WexWzl$(8m&yaA4%rX`IO9>b+`;3M?1d4k)bYu{-Dng
zMj%%fY?WWv$F8S6ND@jrj3=v+=DqnIm({*LD8b3GEC@}`7J-wao}<A-Lzq;9+BX8|
z<y=T4OF*jF?UcWO$sQhR#1ajZ-!3jv08&d}!$oaLS&S}(1r>a}7c1{Px2A_aePYxv
zGocrA9NHcl76Rr-hy$1nIMosOPCqRFBs5omquDXN8UCUZSBpz-f6rL+6F}OW%IOCG
z((egk<RGn~sjIqb8@znwwkhiu^etNq?-GatWw$`!dn-2k6{tqeCgEm>`8-){xi?{w
zQLL`i@t#JR%vc$d6vx}G(Y?-F{p=AIx6KbJX{BxUSz=c5i<U$_7iV#H<A72L+EI+8
znxTamVoJg{*xWJwnN=q}SFUffq*obEvIgCz5A0zZqbHP5VD#sOP(Xq9%g}DT|86y6
zYRKn<W|oY9(ue6$QQ-f_fOk<-JtHrTK<$5E{dJGs2+j2Wb8nMM1Hnux-8K7JF|)SS
zFHbjf{f7D5%AsYLaai=<!q`4!zW#ck5{a45u8NLyIp1!nEZ#jpj>Z9C>Jrk>lzOxz
z)$kLoM5svzNiFWJ%2Z;HT0(YR?i5)++~m%yEeM<GyM?jLX2FM~ha-Y#cs5FKP~TxZ
z5Ypp3RHbZe+Se5xkj5vm#1c_sgv9}Fny;R|SKNG2rsPsX61(ERR1fT+iVBqk%9LYQ
z?Ty6x3YB1<*xEiRZ=$kq@82R9IxD|51mxO}ZH`m|Z$X8tCC;aGK8imalZh%k@}Me6
z+({J)M}IhUs*B#%y(kBMU~ggZ_hYU|q#!nG^!2zftyB$$+Ad8G5)u8%_KHaHKZ#+;
z=;E$3n#iR>=(?kEOwG(+56m$EX7I)2{nXzdGFun;LZd(vf1@Pxgq^}~+!hMxGN59-
zZ#Zz=JI1JSbp*Mqf)f^Zo-7{A1_%ohgg&wf68E)Ccd)t=anH9?>Xbak*GCLKj9T1{
zyUc*ieGK8G^W%+<$rPc2vEku1;yjfW8!1&6bYb(%OZ;~4(%R?(BFD6eH)c=1UJxvX
zmm$LU>M>xtDg?v{jygjO(lP`=Yu=$FvImUNxeD`=TQ#h4Hl3_ZKzDotEdF(0jeYWF
zLtFIkIbG>Ovt&VJSbg|`Zl)&$+X<fhzv*nZ89T-M->FbV0yrsvg+wlD?Q0ZcEsziJ
z#M9}5<=L@Sx-yWnFGCX#bTWM%bPson2bu17d~=!e?4E7gVL9?lg**LpY`Gb5KG*(Q
zR-##oU|WYFn1)S=E`(6AMK;+_+UcUA7QB8TXTfsELxo?X0IfCWCW9YNf($SrLz=kN
z(?3w~3@65IQ6M|?W;TG)r+bDIIDlfGAIN<S^7hBL82b9Sb?MoC1IdYw+fp16>#20=
z(M;^&1YG`lcR62s`l%h-e<e#0=ou#FUO$9Tu!M=ZuhT-E4vPb)7P)lHF$?HyMb3^J
z-3ZKmL-+l_UofEb+7gllUapI>37)C6{g5V*S+_=j185-;rOU#eFuar2n}DbhyS%Vu
zT(@)C^K<I{rydY`pk9nQ;V+zk3lQP<@=)K4^hv>11MoWCnmLKsEVuw8Kv8dob)UIC
z;v#?ZC6(NNqdDB4;)(7Y6R?9ZT!XLeWhoihyc)jb(uVVldeI4GnEQWC<n$qx>qt&O
z$r*(ZisvnpB`1><`kDG(0FE)0?4bEdPT!VGK`@+1xf8tK#oY{V{ADcuvZ5e)u-_fo
zwVs{ZOQT+Cb_TMOqXZQ4{ya3EyTH_=`{lmHOog^#^7h%JO0L`wU6zZ@JRKg~VYeS5
z7{+@td3|$CUP^Kr3@}j4X!FTv6LnU#UU_9N!v87N69;vm(!SJ}e><fsC;uK~=z=lK
z`d&hu)k=g|0@l$fB{nbjPXo|81x{}w;;$;%`$QFqgd7xkg@s&FO~w0-;){0jzYr#!
zfWgf7zZ|O0ihz3JTyNDb`&k5pb`XnwrhJuU_>SK!x#YL)_E^)bWYIF;lLc(dBmbO9
zmn4T8hMR11W2utXDHZ#B(Dph}t%$q*ECVBnXn{Bi{x#3jXE@OPZ;#u}u8E61H2l82
zw9w$SS(C35`A~(iHJ9O?Vv0RSB7Cv?eyW(Y!=&@qHOsE|YnJsH1(D-gWyBOe;FQny
zPekEo*yuxFTFTi3^d|>(@9lna((46#Pzo99T(gtIK5V}EERC24qxjX9`T95SAp&8y
zdKr273=9AP<U-ReiUY82DvQ7qD`Q5gtR9X4WDBiaj&y4BpIHEg8-?m2EOMGs4LJ8r
z)sw~r1E=qkT*GB2tQ^tuU(ANl{$PbF>OJs{iv*;t+&E5T%(dgB((GCtQ!^I_ee7MU
zMP4ec#GF@ABLmrxH4wYxGz9lGU36*h(|#>i<n8Aitt`0WQ!inej=IggI<kDs!*0od
z<y6(Z63{Y1Xu&2V%x`sbrgg);J%Y?~%eX0~P8T@v0~=p-yeRNayu-m=Kj@`=3}b<Z
z=zo#G5A1fbf6#?uOb{`%V$8NGGL!33QP~TL0-Fy;k~GQcSTHaC_a)hrt%{Ag1`b`B
zroA(lR#_&=q3q@ZjPSki2TiNB70&yK3r+WEsHbD8S8Ahl+cF0OmAyaSHfQvl`gyeD
zT<83W;>AKqcS`t^z$Z4fUkm!%6}`@#NveSM3H+V)J!5IN$gVjJ4)m7Sb7tG^>$-{*
zUbF6xR{cXVPH*-nIdMkPGI_~)pH&W$Q59D7KB;_luTf^^Nk9I=gSk33`0jjerwTL=
zN;OyW0Q$4a<<RU+q@SL_jF0)@SH4Gdt{fB>2#;<z1uZ3n=K<kn_1;tA+5ur7Y-QDX
z<F2ymvMydiQpjnxWwMB4+UeyGP}x&#??h!M$|RubD)ZBicR5oU{d%u1N*9^&8es2q
zMI~^Snz+)%r*f~c=08|GRK@B7_kMLyt0pvBk+Hg1+TXH!`4U(LkRz2VuF_N^kM-4?
z8Xt5*;bptz%=9v*KeK*Owdvd9i-b*pF+c2=8R?Dt#y6-52Nhi9<n}m~<8{30PV4>k
zam(JPSDl#S>bGq{dlyN{xha~t{&$j!y?>!j_ZrlQKBm%99M&^H$q0PinpVRGVFp44
zi}wx=BJP&z=a&e1tdI2xMmTtHR(41DO^gCjDV=$~i_M{urLLZJbr!?On*Nw|EJTxS
z5C~E~oS98SLuGjyuwj8jfsa==@Hx;b%frfE;TzWuy$yH2rt>;;dh2Dc9x52QMRJY^
zaG>!~GdCO7{o&YIF7u<GkHRCr`HPo2e%~uz=q#7!W^x&O+*Z3|Jo40L|Ha_6?8)Zf
zLLJE?qW+Y!BIn63&(I$Nb;O<Lq$8cEWzR+%7pP<#Ehmo|Q7E9jel@*XZ=|lT&nBh(
z_BPNb6+Hc`bdtN$ma2peGGP52OvE_0fTzmcuUQ>0K7V;Ag5}-kv5{7&_ItzkeT(!K
zig(P}?dEgE7I_K)3wT7`ap6o7w|DLU14jS@M`-q|R_nyBiR|1u5!~b*9i|m_=RAKN
zm0Ke%W@3F;QtB3q1>PDwe*uNtc0=Jb-h>qM8)m<-`Ro%i#Bq-o(4#J7`#!aWEG~yp
zXjUW;xSLmQ`Mj8o#tA&s06?wdG=}G*7j0IO3|)`^8@o!*TX_@xkti--=&d-^rMyJc
zvp3^4CA#d521a&jl<PgQ^EIm{N9nYN^q-(!GZ5$IQ%|z%{pQ#ZTF|$Yyj|c;TRReV
z)#=aCZomi}VwUC`o0zY-_;vTrhE6S8Q_i&v4_a6VSL@$aeg8}t-mZ1-IO!QF5t`OE
zd-F`(<opBr66(HitF^W^dHQ3>yU+}WF0JW0kqEzx2xnFwVBEp`=?5v;F0F>TFDPho
zuK|zaOa+h0QS!s~N9A;fM*5w8ljE{S^%D`$f&<9gv!%MZh_{S}&JSNZvY-PbPL=%;
z*Q7tzb29C$oZCAQI-$C}9t@K+lk#bOR0S~j#Eo5#wG2R}&}Iw{n|HmB*^J_m&bbtT
z%bugs^umqSuHCHOy@fHJ1&+H1wA|Ey@6xj=O6*j$n)&N~jC5>z(G_L;+)lmYWh}S8
z`Rk5z_p#W#0QVANsu+<RT&ox|k4XWbM5_#VH#Az7mn>cd*yK*`7M6*1-dd`tk8J7C
z-x+!go_Y@u=y9Mj`!52G!oJ+D>}H-K{jOE&{s{xjysGB=vI77H->lgu+l{m0e&~0K
ze{JJ(h&SDzZM<Mv3HTl3xJ1aj1m1jI&3ncJ6GnO>tuSSm$M1kEA0p;719omWe6#SZ
zzq#$wBon6p7GcBUc3X`3kA+%`;!msoRFk#9l+AZr(-q5sfmE4(d)F3vyPri=QgMdc
zU$>2L@=U#Qfb6bSpR=zKu>HBZcutm*x!;8yNt9(i@NKO$k_tnA|2$j+ZU|Q-<og?y
zh_w}FPE$8|?*g_`rzJ%`cCWp%!-t1!DW9Qcr)b){cxPWm1|yx8-hSNIQ!Gq`M-wsY
zRzIgU`%!5S?n-4yInCcsb0#|D^{K4_s}<88>cvl-&K_*N9{ai-vsFVb_1@pt`%L&J
z0&h$#9cmR-bBdO&VV&{4`xVl7C#3s!Gd82bVfGl;`jN8Dfaj;M+nUmXBsL+D&lqkd
z%0?4GmFx6PNV<jaVDuLULGQ)aw$w=iP1%NdJh}HlT&z_DP8>TRt4f7+xk1KfVox2d
zU2y{CeI38soa_}8#s#dgNQaQK(FWo&9$`ac#HPBfmU1lOd`lja-El2aT(ropvJ6s}
zqtapAqCB~zx*lof*^#Zuv`-t8T<fru{@idkm)|Oqh!ex}j{#<Q<#(;%L;*FwkUlRQ
za;>b;7GxYNS5M5`4LlSq4va4(d9a^<&7bGub@*^6eNUR-ko$otE&8vyVX;9#Tq~-I
zUsGV!nrQ)=@`^QuN5rV$VSUezEPzRe)rSL&?RU+6%sJ%X9Cs&{2Lq;}7O*wvKZAD4
zw*1*={CXu8$`}LUw>pHgg$*pfqVdfFd?&Yj;Cl5&%z?FBuTV<C83&R=x@q&3^y!*B
zcVS*9^K+V)AL>gpe?G{+$E(=5ws=k(@N?-}_|f)>pFsi~7_?=6Huzydxm-fXQB;4!
ziS3oyJ5H0+w^@<*Xiq9RwI04tEewCqe7_u9!v$sXoY|tk;&}M#w6*!1<7A;u*1$bO
zTd36<YY@Dxxi!BtA4)+!`4*-%b`j57h&y?=KbbNkTZ#)a87`i15AI_t5iqG0dI3DI
zVY$+mtT-0hE*1kosCneGpVn4{6tx6Am7yMi>qtN~#s|B(l3-yv-H~ED#6vJlzRap!
zsI?r+q%|O(LM;^M7KMtm;_w?6yR2+p`0*4!+RInUND|SErE4Hy$DzJ$LB#Z~xQsa8
z*G~--#o89OTX~7W$5`Jnj@aCN+1_y->Ap@SjlgU&paG`|0)wOTcVSd~1mPUgt=im{
znFX<f;%#wy`_1FC5M9g<SO}xxHv~5U!~1Beji!u#W2OubQ#ivP(4^1;qF`HSAPP<@
zdg-~A1ZXcuk|ei2w<@lZy=T1YEw>o-O33=-8v>AVy@PV?L9;B=x6=<=1glc-L>`>`
zETsO(VCiz~+&Yw5J394Pc;L#cC1}ojxA<i0nzV4kW>O&e66XPwIKgWj7+#>i#2ME6
z={*tOc^!xBrnW61g{RDvZRFI#PCkX>!d!8*@iOXyl0+af=XcUs0@&bA@AXZUcYCPd
zyAw+T=Ce>i*&k^Y4$6<3iXHqnYhSk!vq*@+zlT53a};y(<u$>g=O`#<&McHCz5yc+
zLYwjYywc`{y_wSET!1NYmzd#<nhVHHbiiINz+|{quO8nbVn!%^R~BAVWMsxs>$sf2
zbq9E|OB{Ey<&uRI?C_GR1BJPqt6rC89fqZEk-G!X>;3T8F|ZDVR*w34N4r5C;3JYL
z0R+&Y*r9g>oPlH!=3%5^2ks%o6gI*SxnYM(g~AJ3{QHSrg``Bs`)Z~;Q?Kz`DJsBk
zsaT|*p4_GV?Gerpr1>QXcG9|rQ2>p3kbq!mY1ohgC?aOw^#bq!Epoyqz{BHEq2tGc
z(l_5fzs_Y6c)GvvCH>LBG`ZAfh0A;}AwlZa#qbq^N!=;nm)e0&#5dueqYv1A$u5{}
zhS~>zcKz+ejZGuz+)s5=PU#}Bk@-$r9NyUD&p=D7=qrA6Kr99r{Lk;{Ho%Njib3zL
zFNKTt@Ag05YoC4#&y#Q;;ci_@2=|KEKK)+W7NdEh?fp&XQH)o{yR8PJZzo4T+6FIR
zR<r&?iu6yC#Ww;*J}wA+odI(?H#*rE2A+*yU05#<2%<Gqo=~;?`TR*Zl}vy=tEZV{
z@;vUlrO6-X29wSy((PAWzeZo7xW**3=L@3ffkrl-UbjT!qse7Y-uGIr`PYzPx5xT)
zwq?9pG?|$C+XkTVQedKM${>pAXjSHh&({tk728TyD6B_>fw^U^I{b5o*dL2JxA|eB
zoYvJ0RH!HCI+o2ogJt4^XLFJNhp4lRit>%tJ{=N6NJuwIryva7A&5vfNJxXk&|M<k
zNGRQ%(k0#9T~b3yGw(D1bIy7{vG~A(Sv=2u?|of+U%&3VvnRLwwf}u`09lHv$AZ2U
zcy(~8?*`<$0Rypzq#YVEWda07k*vX<Z!G`8;|p(lsGmP9-3-vjV8a*bdjR^^Ff?u4
zyPz%`CyWYyA-EiuGXuNr61NERF~oC^jBg^OT9NCwrF1d^XQ(N241`_}i>;OikzVwN
zQU~=iFb`()NLFx?Z^d0algZh#FuyPy^HgHRAjbaWlxQ|+tp4^zP`*KDF;3>y4xM?g
zX@OSD7QWnu_}+%(W)P(vhWW7!-mlEJ1fI_I#pCB{{rKz#Z9)Ar#p*p0=#(!Dr*szq
zH=v=a2i>~xe{97oR({X^dBwo~XfIKb<En@6(0-d|0bz1yLABZL%My)yQOdEk)f9YE
zN{j;^x3jhz+{NJae+`Wx0R5N)YWO@3dmR5hB4KIPAB{s|<n5H~_qa_BYFwFW6gk3|
zqrhLg%)0q3kOapN0$Qm1c6=e(8n6r>_H?r@=KXYJ1WM6&9*m7z3?TnsM-y8>@BiE%
zQVw``z>(Kdn++ULujJuI&)EF30(2r(nBO6E<9s!yrX8;_@cBndFg@&4tUK{g7(11!
zKY{J`!mGRI=E>K;J%I(i<GyWkL#v~bhr#W%=`o|Rug~7TMSzoH)_KHhJkou?dAD6h
zC~kw+xxdWEC><wpRp){sMi&JXC%n*xqBGx8V<S(s&7_Q5`M4p_%`pxZ+!>@C$tUg#
z`j@x^Jh1(+=j5uLiXUJ8zAl!9LbfKEd$eHBflcb~FQwTGw;V)|db!*?bduf~LH0o1
zA{}iV9~~^aP-41r9Rm9|xNYACo{9jhal%a;2Z+=m7a_CC$f@}j6u`i8=zksM&-a1T
z<U&(`h3Ca5)zctdA#~um;uedxHc@7iKbH!kW<LbsK7|J$i|gcJLNMPH2l8MVIiC}o
za&J@o4AVk@(L`~ILD1@^(R~JG89*+H_)DK;Rk0%;!{@68n$_eU<&pfy@~3-?l&B+-
z5lpwj#S=J%fGvnLu760;LOJ?w3qNyJ2gnc3CFIiDtA3SH=q(l32KAT8Ll2fH!Az=I
z6ox>ycPOwVI?@{b)jAFO=?sV=ORwJ3C3e9;`T}q5X%XgW^#a9NF{qZ$e%nDN6~?hz
zz+U~f$}t?CVEvo=+pDMUeqEkFJQ8(K+d#QjbpWm`T}u3{_7@!_d?hX#x8NTB`QeK^
zA#k$jV$>EKI4iM((BKw|5$JtvMvVo}vxm%<8WOEt6$qgEIw~n&;1QgRUoXqgS3S{}
zhy^Mvk71~;f9<F`sqhLuPbdWHQs%I08_f`YRw^vNwwsvEI4}E{UfJid8bDqK@&$+k
zf<U5z6po#UI?Ql30B{=ka@SD`-4{s%@{pRxqLQi*P#dYikI;bW|6dzP|7g0ta$NY5
zjoU))GULC2$7tWalL2iRZl|auY5XZHw@@FnDqJ^>)=}GW{@j66nhmMDqH$la>wu1Q
z_`Lu1bc#L((o;QaF=7l-VSw1SY_B+Wr=<idOgGGPNoDP*MQy|}yT9(lA;~1NZ+>Jg
zadW{P+y)T9<VJ-swZ&#B<O(@xCvH4?<&zlfVk?@#o$HS^N=%j*#|w~Hl?VEC2Zt})
z<6g}lccVp*amwz)Y_aBU%llywCE&pw4{=#_1KKKR3h)`b%oc1>fIt=*`_K1`aH~Ow
z0@&u2-t$MNp$iqn+Odx5d$I0c<C~bjgY7cKispKYQWM#Sn1QyK+mEJ)GKR>#!SGXE
zGHhpiSTt(hQfJ@KRlC2e>fLqz$WK!FA?YqP&&2`2DkBwaW~h&g)@O)LBn>odufyVV
z`9?1(XU$5zOMxfID(d>%_;Bja7%78QO9;uW4S>+(GW+wLmQqn577Uk~`(xW*Qh4KG
z)CF!hO`MNojtjeO5Eo|LK$eO8UNq|eD>PXgmBJbaj`giRV=ej*=!k9OK-gJayzpF^
zPC^{-?KY7IaM9LewnU5GAJ`u4@Qw-Q@_e-A_<fMmxghx4d~YO8BHISO5^oz;RG#$y
zqNsCjF3^|U6}c<uvcvsh*{u7RgyNM${GsK2GW6VKCEL&0MD}W-X7;~Y3)p_ST^#o%
z1|||3**0dij40QDe5^)}CZK8X1<p9A(l279H)>LDQ*Y$5lP_I?GJ6FmvoWc@HI36&
zVsVPxZ3oC)ep;B_D1mu9Xx|+eeRS0u_TxQ^G6DRXlJA>))ZQE`k3nBqu6Y1K>-N@h
z=T$bVS^6Rutg>T2B>yCA*b%sB!?a-A?xpeY4rz9PQo!I5*6rY{{roy$Gu@2$DUlY!
zN9f+<185c(+f8S4S&m(DoMWX*vqleoF_;fj7bD+J=0zcD8@DNLN{#s7tfynkFklE%
z%+aqzZBsaU^|M1uxl_DKd3*S<eZ6;w=<P<(IaNq9`F6M2M<!WaEgg>~xi}6x_ht~_
z`8_Ji!&gv<|GXVL3(^wbioyg*phS3ch$4C(PrWQn%Z6aj$jpf||56sK%WvqN8C4o0
zQ)07ln=ubEA|oWvlb@gexbGJzz>2L)$;wRJV);R<TZPcy3K2oxT28pXJeK|z<7Hnm
zr(Ch4M3GWeSJoXKiNX^wvC@w5L|gNNc$jyaq)TH}7HUkp3*?B6x7*gd!a4WLRJr(1
zLza4!bi6IVpO8E-4#x*-3LL8@x66s{OXIy?+`{moI8xvd(tQt=AEflIti7gLCUw3V
z0z!e%utWRGfE<cTWS8vQUgmD#_G|qXox1|3l?I=A?+Y)gm|AAKYA%?BwcCAzKSwhS
z@aIUV%WO%dwnwzT^__BB_17<D$&b%ko2NGI)Nd=OmueT4Km(z6QsR-h<O24`x0LA1
zXz{}KF9N(yo0;i3x4ie#EJLw4ntm1wx{(>82wI=QX>zv!S{rUW+7<ojcf8#U3+sy{
z32Qn1UFx)t2EKZ=_dtr{|Mr{mWiL(vN9M&RW~-^U#EA<LAcMx$n;Gv=76+oPpz~__
z-bZ&|(W(|178|eZ@Ro{av!;iF;g7D%Lt$+v$5rFB%-7e{MuhvB)qcidQL=gXf&EJw
ztljyRZM1Ahck1*8*gx#p<NFag#YB$17Jk%JR}l_37amjLdOcVMMw?I4$PqLLy^DGo
z*#F(25v$PW-?hm_ox<-=A@DZyc<bEVNU)jUh8gBz*L2rk&PettP~gJ>Q((V}@JZqc
zgBLu-|E7?P(c{1>O4auwVYA?ey){DqQUSdBS0m5;WzFAZRup?O)&$sv2ZBLBl$23q
zy{{mm?qwK#rwSDmdjk|ZEgJ};afS!J*7bmBQaCA)j*D{TW<U+!0M!}^{hjDvWZ^z0
z5&;R(g^!xuhL_=e&XR?|{uUWFHy_R6B%(i)GUbm?d|zB@G?bnDW_GI8)0!~2<-bsh
zNMauk;@eiq%n10zE~tCkGe92emY*jU0V$^E@s2`dhIz%&x-XD*xmrv3G(SOsO1Bpp
zh2KV(;ZBAm^&V>6-c$V}j5tg0+++I9{K1422&WR#$6B-L4FD~1z2|h`>kktTWl1P8
z@BzQa8$X!4?7>FM#nYLe--J6;BrS6m#5DOeJrJ2l(gP=7bapxf3$&j{rA0Hd(fF?b
z6+h7rAoPdXdyhA}cxL^k{RWkIfb9hCi3p`_bUCq&dt9&g+4)}jam@U&-Tqy_*|;tw
z3~shnu#;+xwnDibY2NzATJug)aM~V=-g7EzrHZC=bPt~DF*OgQKs{F3F4icnhKx!3
zuX9oG_WQ&Sb9R8~t0bSTKfzlUXi}v&W0`(4kMM)nYdA}T&(l>l#Yb^wVkBQ3t_C0d
zUuzN|94ncjTW1wlkg(fw*6wA_EY82;c~ow4vh<7bq9J~xt7$RrbYs4ilLB%qyiK@@
z?6>@Ur<D?%?YC`aVgM${&%6j#ec~HQy)LuGhykj8=n05=5$ZTZ02azz`4yo>?*VSu
z$>OqEPx+T7!!wfE@%S9h?9R0q2SnL&{HGHQ7pj&iT##aSnXTMbZ{~`5tD2*u4^YN}
zm)KXOp$Mh_#ZOLlP(%z!$(t)sok9AAySe)*2<@9TLmmGuT83HpUTU;x0^+lj#liLe
zxZg8UfTjR$%}UkmXP`s#JLuvx9;N2^Z<4$^C)dDk2gB`An~+<|{DQCGuUh6|l9DQg
z69`y6qOATLS-CX0$<FuOZ|(9Cs%<MziqTG1geHYIbmwNvFF#a9ijaCg-&>`|#uS`s
z`;tUY`kg01`$EC4_9%+p5-OcjL?!8r95xVc`9qVfDa(224N)Na`JZ0$(*!0vixxMQ
zL8(Ym-Lh}<4W+wg)DPMlvZnlpKsE*z2?sVj?Wl(_MgQY%7$??$d5=ACHy2QO|3Ot@
zSE{q!8du=V^4fxT&z8W9VN!#OGKnF?<Te|2m#wH*kEH5WyNs>80%v5-jwl{_uspV2
zjt%ei;CwfW5ot&?DkQ~0?b?pfS||FK)~MAQ_cNxxeGzxpo=O*D-?u*shXiujk&(Z#
zd*eyqes{6Exnxs`0~CdojeOQ|dv(()Se%e9tQsrW1t+Yg52ujWb(>4}Cm*1L*2iy1
z>_46lQ+KfiU!-L)8;xX%2)ACBJB+sLRkU6VvA65Nt6E&(0)zVwwrcz^beG>f=&GvT
z7YW&HB9`P0RDe(a_fug1LPSCj*81}~m&uVGS^A^;`SruYSvEOU8*D7}(H4Ax2RZXv
zCzl`iM#S3f`+(zZ?Y1+zz)Qw~q9SrVV>^#~zkT+r?cb73({DGiAaMkqBlnJn)lSXz
zff%5TLRjR3dF5;Vms>9gd%UdeFqsF&(*-L|(`_n>cX^6{k`oPeJ=}eaP$0F+Wk@{}
zL17v2PBt2??v2T|%@AWA)Qh2wkgU8rKx;2zyb+xCz1Y<x1i$p#Z_B3M>!(*P7+a7j
zVe@l2Uyb|V)Qhm{sQD*4n{>_Tner8Wi0@g`bB^PQ-_v?QXe@KpOH_|<de`S`1_JW<
zgSl%>a!X^B>1XnE5U#@ZX}-FVJe>j$?|j@X^ZT{R4iV+`>a(-O2YUn37_U2DvR3w8
zdwK{00f?fSNh5j3>He@V3?9+63DouG8*9$=9oItuhygtGlaKE$6XX%jMq8`MQM_S{
z4KOA}Vu%-SBKaJShrw$So0_a9jj#9N!-SQ)m^mJmiZScA2JkPL-xB#UooR(R*yZAQ
z_%a{<5aXd_eKy75MZr)Y^}engcCWEt^%)bn6xm;^_wDep*5&u{$`<+LQ(wa0<YrSG
zy3vI-lLM(rNeHKOi6jSSAO~aeuzAqXebG*75&3-(7sHn?=Ma%APVsw-U?U}~3FC!2
zu{i;VB;Y<@ip<kFxA9Y$%U@5VxMnIfDXStOxp&riLQ+CG5dM!SSbw$1C*nl!h*a+m
z5>j0BrzTQ#h^C0baM6ufvcQQEKjm7Nq*M9u!4-equ&ByuAXSQ(g#W2ILRmY%6uxgt
zLT?-W9vZ7g-!Atmxi|ufq~%m6A?x=t;|NqbG-seI*OaL;?sIc+a2V?GXhZ=mGofLp
zGI$^S8e+B7c^6V;L}@WihK~m2PGU7l66ADVr2~G8zT=2$&#mh`hC>=JZ&}Nv(9=(=
zB!19)ut3npIW_E@`stMYNhlVziMZoD${{c1iLH2ZBjcS4)QVJ^^Jg)}f!~~MAUbt%
zVK$)bIuEl07uw}?uND2?uw9|$cvfavAYT91LtrIGrDE2}&kKK|MGb&T6`c4<r}CJ^
zNPaN%zCVX9_`nFWyj;{sd3{f+{)PSsy%yMRJ8<1RHS!ezd-q3$+<~1x({@Uq0;-pK
zp##8hLwL8?hi`?3<gI7chq$R@+aZyz)}AscgmR1a{Cv}4R72Jo7zA+Dxa0KDORwRD
zkl<PCW%t2kF7gg;yE|=q<HrnOWx%DS!&YhLgTqVa6v)wdj|vpe8F>|8h!lOfYC1RR
zh<TPfNaJq#bp9=WJjCH?dM@9wMMRra$7bqJzV%}IZXcs+ayR{|I@X|BQ@n5!S}0)T
z&(2@t7g7<tU9NVZcuJGY#<8TjKGD>AXLYwK5B;fr=6lo9qb5hm|9VhL4+J}P?@-#o
zCFbJS1O6Oa=~B*dAdAn8pMVdi%WQpNUFO^vDY4mS?|BoCW(B1e)t}lQZ;$5v{p62e
z!E<tG58(%$rqVw_&b{oh+0M`e`{4p?>AQ)9BJZmREj?TC7ygLyp9SA#)5eO`n1uS`
zWRYjprt(X@^L|OSK2H7+SC>-=OGGUCmcZ)hG)t|Y6sBZ2OWvjs^Bb?(EZ0xlc5;d(
zuwRD6;AJY!flGWC6L%;uXYp^@0i=B#3(uT5K^8C{8k$AOyCnF0t*Ze2$*wAL{mFE`
z#6ZTL7n4ft=}`3{4w2M$wq!Ktw=jE!21?8>XPG=PxIxUZwCX89uW=<kj4A7f0@PUi
zZ?y2~>r9>zR<p$*7(X#`+2gUL7<rXl@cYVsG0SKm;km2)$Axl+VD(|ate#o{*DS1T
z5d}1qE#zwLuipm)OgUhMkOl0}emCmCG)$G~3LyAG;eSFD@zEW*wD_4UK)lhm%pL8<
zmCL~MDw@Ad%JjK#a1RmY6ya=br7(R~4A|lJosu<~EblOQ^)9&Gbxwyl?aTp9;oG2e
zv_%H1sQ(*Q=I1#iB27<tvh;-2dKJn*8v&MoP*VMx=h1Ad3|Sto!G5eb_o!aE@g^>7
z$Y?jk2t>fX4<Yw{rtC83;;+-uJ5`x$f06lq<Ff?q&*GL<M}<YW<Z|!yr;}Ulr@xA6
z7A?4wZ<6m2X5UIz@X&yPR3H}|Y<VkZ6qo~A=8tMu3EoqZzlP<ctVXQ*?esJr10|q!
zP)3h8!OC14zyQrFKkf`Z3Qqdf@pR%)uBh|~ys4OFJp~gvHkwY(hh^9MCd`n;4hrDE
zwP2fq+;jZS2-ECOd7ZY<3;-^YhkU~;3f<w(7^K|TKmi%x%r;X$FFkb2PBSs@GE;f8
zJnqYlRdSeN)&B-`T80rZ6NSnQMiASW<llYD6C0xk-Qo5&;7RVra*55!+~CF#r)PUd
z4`B;moo6V<AZv0&=;b$##TToX+R<tiM5eRZAYks|j7IjrTn^$JBCeG9!p$;kb2%I%
zD(Qj2aq<g0U&g^TN3vwvuvkWv-;k1lNbC#W+}m%EEh~l!mzR|nw#<`BROfVGUdM>5
z3zjfYUeL^Nehk0ZqMK6la-0wEmp$)+mB>;1yv(eKV{cf<$u92EBmg6XbWN);d008R
z3le~}-}Q>-HybNvr@XRv!FWK4{2<XZivs%F^tYrGNmFbnVrjYv6@<3m4oA_KhDu;B
zw2hLk9BVm7B9${hiUyDD>)h)0s`XaX>s{x-eCOZ=|JJ2XgGs>8oN)W;GiZ|u?xmVd
zeI`20xB<ch8NrXx9+|(ceeb$*ITn615$-guy044ay>T97OySF4&3@crMur_?akf@5
zE~6j%&AkAYZ(7fM&Xmw$Z9rA@R!4^g9h=bc@$4eNtQ2;r>L<j-9cX-a1SIIz44zWc
zp<LO|e8)Vta5uRZ79(JKzic92%_({*-L$zf9@f#JyL>b1>5xG+w_?04|H6Ztt@SV+
zarZ5KAPi|<h_%(r%S-9ZNSO?Fk<y_&&Quk;;R5YKFJu`9lGvqvSxT}8d*G^V)mdu;
zD?k-ffX7fzy|@4})CFV~KT_=3G=pg2qIgLHj1nd_pod`>03Iav@_Sfb|ES+gy~a4X
z{G~9H0Q38x`}1Xt*fXSQH(4)!H{0WKFKL)co3@gjt*K*zvz?sAol7wE5#VWib-4e*
z5m@2s1DYF)!%U4EvIgBWF+bn#dpKptUx<4V*CL!r_WY~ZwC`wLy%nq28u=@{9pM*&
zxOTbO8_z&eJl&@DSS#mA(1-51AiR<;GSg+?upv!dNN(L_cfd@liZ|r9Z`<!P46=1?
z1Y*F6HPKCikK|4)mpKethCZBm(7|>F6MBHeTXi+~ve3Po2;iRf04e1p(x3KQC()SJ
z?>K2~Nd~VaS2srA1u~#a=1@e0*$|u3?ko6+UQI|5zM`y!<<TpzTt0QbHBTC#g2&~t
zM1os$YwUD5fKadERc(V6;~<<VznEt>-MZIpz{#hRFLZIi5^|^N^H)7vu_*vka#34-
z#|x`<Xb{dUy6On^zKV8lrO$L2G9@R>^qN|@zXYf+^01Dan)E0qHOkTN+xNiB45(zE
z*p?%mL7bl1T?7*Vj|)#*zN?XuRchJ%&BvSSO`hn<k*^2l*FFz>f(K;DO27X*Bz7<=
zup0w6xIK@_Oe%9#aHjtqIA|OLEhYbo_u(R<v%iuh{fCGh?v+8xj91K%88@FSm-Fhl
z0e&HK&y9bmn=Q<D#O(F7f*&5>EE!LdVFfBVDv!^%{eF-Ao$eQDmRavM76q#E?#(hb
z)WqDP_FMwxaxnmbJ#?em=y>rF(uxN{OIvt>8jTPX)&xb`Al!&o0n<_bKo`Mv*}#ZJ
z?~<YJ4!igjMopU;1fHpE&C3ld!q)%G@ZPtcUzXA5;zQ_?qPK=**EPnS;0(<ppJe7I
zv)w;=ADOKfa-(*3e{Q}NGYOy(pFmf>f|gQU<n{#Msruq3y{EVnd|Am;CQ1wj%A*rt
zrC<6b+ILzs8r#W0RFDF50+=s?TOUpK*zNT$231B1fNNk(sJ&h6Pn1ws5XPXI=|kxO
z-sp1chvrc_h&GsXWj6%Z1dGfo>R)MV57G$@WMWrC)5Bol)J`vq64yEkUTrl-Zb)RO
z(%i1>i;`&DMLIgv%zJ+06VrbXK5P9-W;-wb5GAmds?1JaAMZYv5kL$sA~eM>70>~I
z)sdKmbSB1{gnI649+oB|Weq?-Ec&$!&_s+DiF#IsQ;p4X{1rH!_0l&?B_jhMOdRR_
z9-n5iH(&O<LgQk*^JW;COvo8MxZa?gW0-IeDM`+Gek`s*1K(%!Cv0@}V$UA}1f!|?
zB1{ky*TMx036t-w|K3_(F!TILNZdqxW5nR+GJmU!?_NCf>@bpNVGgM<|FtNgsEw|y
zRWF~%?^R0+&&28Cf3u99XuwzdN9g_y4{{w8vNBI%zWXy($BO~;vvp6PnB}(PSIO)i
zEc$+93dk1%WQb~rrieFt&PWOy{>Zo(Vt0e8Q(v(}28!Wafd})=v@UDsRk?e{bOgdG
zWqIO=)pR&KQEB;JLi;rV6YHX3pu#@gX2%#zlOchQk|`eUSfOGd$e1$sc&u43zby6Y
z#g_{{syq{J?5A|s-!SHB&*0h|LAitXU%o626~35EoxB7&A;AwbfF#mZwQS$=rewlg
zNs4JIvdj?*-dv&EK)jP-KqZnD9Y7h4l^q+tI-Ks=O))aGmYu9e`G*}X`B5B_Md`L+
z@>Isj@0|5IVb;C1gVx(d>SCgnStgx&;5fE$A}m!$1rkc41Kn<wXp(^VZRu{Ovoq_{
z+jf0)si3vh2l%7;Wq5(klJ6dZvjIXQQKiWMNv-8LRczxYM<S}Vhuf?rtK29S>r2~$
z_jg}Y+%2c1>$W_`8-H|r{ED^h%nz4eRsMeIsYkTuJ1p`!jPf2MMC^VtW=bG96MMfS
z3ipgVwwK2Jz;L$2ZP7V%!(`IYFWSFNGpRtZxmVARib^pMeSH1#Dfq0wn+;Kt7=@Mf
z`w7u%Zn;YCvjrje?Skz?)i_LuwT$1%x}!kkW~ayGR6vZ1>i#JEzR&CDSNzv>|0U~N
z_5#JNQHo84fav4d8SLwuo|CU<FH$l*%-?`NW!X1u(2n2=9sK<K{yhU|50S<JK+|@p
zV12vT&${3rb`6xPjV)`V08X0dMaMA{4627ezC1;!UC03ro!|}#LUN)6St~-YOE<RY
z8`Y}<$QF$?&Fv;}tg5+;L&sy@uiP4Wx$r_;rV7V=DeyJx_sG|V_deD9j~~TF;<c1h
z$9AF^aI1g*48kk)b_ow+e2qlosV>Bk(=fz{n}c?rh>C%+yO9#v-fgY=UC}9+NPy{H
zAPV`t7Mo#f>QzOYn;sO8D^u}eg5i%mJp|fSHmIoV)B&Ub&lX!3qk@=F>d9gQ#5HYd
zRGm10Zn^lwn50w|Gl&;qv_ui@^{cwiRjFK@dS!d1d>9!i{E3@x_~4m-OmbNpD(duC
zb7yHki+0-Y|E~9(bhNUrc-|Ro&Zab*&oQM##?Z~JS@Ts}$grqbNFI;<`4Wzm3wFCd
z_X0u;zwt(T7cJUR+Qzu~SVj`ypi5)N9(n5+WD&x(Ky>{hY?QN_VGf1sM0FK~(l>UZ
zk7W#d$fzbYXMlpL==&hGEQ3^izA|4m*Yi#P85D%CZ@U(Eiq6RtdCZ_s&j8YFkc^FW
z<orBUwU`6bQ7I@Y%<2J&f!{ZnUKd+QhD8_*IBWg`4nt^ljE`++FH!KD=f-ApM3=PN
z!H|7|1S^3RM=3EZ_2#TFy`cX&GtTd?nZpdk9-9#}En@eUc%!4n3j)pvy(nSz($L!;
z0iY$gh*M(4q*9ib;+QS*m>vBZ`_{%e8WR3qn{wj`d(&W$fR)D8_=IlV_ZYZi6o@6H
z6BX08$AM(;m#MKMy8yBZ%N9KjD`7%8L)@hf=03!kuD#zDx1TkeuL^d5E%o%1YV8t^
zW<|j4ej$y*hVn9k$>KG2Je>*Jp7Y07^RIXJ6T}k-Ki9IwidS{rBO!%aQ|~SvWq75t
zi8zWvp7mM}=u^epXP({?k@?YRNY1D&6M?E{2+)XPhTGz9KCKac8(`OLB=h4P!_wrI
zrVG@LYmCG^0TiIp8HzOt8g2!$cd9JgkC)@BDCi-(&QXDtPiL{AsF+@v&MF?!`!JHV
z>5H49X;rZ{X#_11=iym&4`4mG85$jNEg1ZE=gb1mbVg>yDtE*YXWJC2g)vuU^V?_|
zFo1N#2c5Q-vFIe9#ll=r6c%3G(3EMv#I)OchZBL4!FZMCjrXmbkI+odzVzv_pSI`)
zIkeLhUy|a>fj{edTI|fy8=;OKbBf%|g{a`)5X0RZ!R9>ReAvrEQNYf$C%+kaSUYyC
z#h|LxGoP)x6po4N=SJ*Eda!S0)A#BcG`X7!FeA|%f65F)V2BxY_wlk`;EiF&xOucc
zVCStv&R3g!F&d5jT=`KW^4Zx30?iUj?mpFt=)BU=3w8JV+xfBA+;SAB(a=K<eqL;f
zLlK1AMRcx)gir5}DznS_c&L)ZE_DNV$5n_Q)kri4PafK?)f;h!Ip-V7Uyl{Ebbt_<
zA-74!_y8xN&FnD;628%;#6y7qBrGTBQzXmL)Sz-&@-Rq<wi(-)Dmyvf2{Lg8Qj2sa
zMr8}SlxJP!<RTm3qG7OqVL(fWl{0TA*2t79@RY`(=fsf;&!zPYVk{eD$>KWv+tkf%
zE$*vVF-7Z-fiU!e@>YY1rr0M@0xa2vq~;VnOO=dk+GSRu2Mw2(t|kU+SL@aJ35?ul
z7{m;2i9K$Dd_u6hDuHf$xHG(7TWv599XU(){eU=sDLo_h{g<ieE(jVcI$?tzAj7`5
zPMvG82(kOIa2eg@p;<?ldS+oNoMM3ozLJz%gb!ozWJkU-l~)-6sBBXuoi7hIsPWOr
zXl0LGpl?mW8dW*3F9OYc$&x#&a$b&!XkVr}b!wXQ2Ni^AXil?NsU8i=j0f9{cW{HN
zjW-kzS}Dlqq~+<KK6upQfd`PgBp!b8I(1ze@4^+`gkl%+l!tMkWbrLkSJ)?8a-BCw
zV1dy;3-5jgKoy%xM)w~(cB}!2yzpmW(0`-L-&yp|l-0yJFhWceBV)rP?$rv&0<a7A
z$(cN=Oi3?dgEcOH_p?;!R6JXb(m>!~f?tJ&g)@9iA+K+;yiOGe(`?aAVB^X^Ysygw
zvs;njvl3sn(lA-d?alQ_T7IWMG{5tv4Fu%lJ!EwXBklh*kfct1xwqJV`K1aGfoakL
zu22!KcwdepGr3*5m3lK}rhR5>%$LflTs3JUUyL|&Yy4HzjwzxSdb0j-j;MeqK;tO?
zr-Gl>e!Ir(h5hgS<?XMwpHls{lQ#3sm3nxw9@rF=*BZv@dX~uTi08C#a5j^OhI_VJ
z9xz<SMedCu(1xQ=`l}qvqRQ!f#fj8B$i(<p(Ia0mBgs7d;^~4>8>p+1-i%cAh8=fF
z=ROBL#?VxMYwAQ{gE1?;9njiQ7Y$Nr?UCY=XeyvZI1-6!%CPOuqa*xV5(&eEsh`@E
zfLKMfCu1GYI?2L`PNSM-P=}0Lknwqog4($vzUQ;9ZquAzoU$l6_DYe3F61C`4c#AG
zY8jH0kYf>TggnCKs3)y-SN6#Oh`eux*b^s>zyG_SP=W!_oe0(ia&jBP6*?As@33{?
z{hLhB6^l%ghUl)3--9(AR@U(_YN?+N<dPEC4$Yqe2P`rRA3Kn_f3O=%dzubsz}@s2
zTn`$wnxtsQkfK4a_+SU>d*hH<Z_jy6uwG~!Njz$|AE+jIjWJ@|Gbq>!b)^%uDP}rI
z5o_8c%Fvp9zP_y!<cQ<Xh@3!xfg-c6pvbvj)7a$WD|H<Bq4-&xDL~pxf!p>e&(7&}
zisx4z@Z3j;rG*RwV85OiHp|IZQT4)l6?wTEb{l%e1$b{5@N3R5GhZmAJmB6Kw>x~@
z`XIcthZ5uVbEVDoWm^x63P)S*GkD~329v69UYr+$O8cnO8huW<tW5i#WYcN4AI^>b
zy5C6mUzbR~?j3<pO!c=|oaGR1@Y}O7=vF*)Oa9@SiX07X)lf-kFuP?tw=J)YZcSD7
zamX$XeeF__Mo<6PlW}U5#QN<fD`y8myXS&{B=3-Yol{T91|lxrnMa4Y{<rG#N00f}
zn!6X5$3TDd&PA783|MvcR;uX}G>i5g#(g^;A*!jUGZQYlzMO>OvnN)dl9~RjzRE>y
z4^&8f?d;+GH4rNe(#yxnNFb+t@<$Uwa0E#Jk|+|P*}=A!sQ%}A2v?UZ)vgqX_B*v`
z06Qm|5(DTjJ*C0yuL!|De5Q`S-`DO#B#(#?1C-ubCx*Wap4D+p<;p~P-8Yy|fyiJi
z*H*wxIZK$UzIdR<<bmJK@&kO!xWlMS`Hh*{GqGA0TyRUwjvq4LO8|bU6*cUyF8n@i
z&u)t0kFh{I*r+g+JuQczD@Z~CH(0`5xN{AtHIYM<xwv#H5Fr@U%KF-0(7Lwu*6%fS
zag>^`h)Qv=D{8a^x|OKB&Rl<6P3R;DUe`5h28R7j<uWhlV%5D&U1+JuiPwt(+NvSG
zDG+~-CY-Iy8bOM!SR42;@RdDIC+}f5f+#DLbYgcmcF-_52JzV_9ff`@St)f;mnk}x
zYwLc`Gsr4$HtMd|Hx~uOG^`Ee*`@!HL}F#u(ZN_$^vcWK28L!e+J-$o6U?fV&6y6;
zH&%4tnMgKj0Cf-`eJYnYkMAv2O3QCNkL8gkOyBCnedod)dLlL9NgVv?-0hbS?4G?(
zwryN>`;`eyD-Q0YXT2|5fmOopE1loJCkzBj<@xmyX)y#UHDhZ_nixym>tksz97<2N
z#4*(f=SL;1)tRL}M}t6CV+o)~#+7G3SnW%(U+%gK&sE!sCP#OE9c>J3ey4|3v&&*u
zxes^L9kB7uaj^%BjwW5?R(I$~tT-Rr?|;pEXHDDDzQmZRc@4|K#&0$N8R?_ip%6>@
z${zBn?#b<lrRemTZ!I7#UeTvi+YJG#hrov4sGRUPH|BhEIc(+YIF9y_1dWUM-oW~{
zd{D1HBMl}4`YK7d8J%`QLGa~20_M%MKeg1qREFxu%?@>jL9tp3zv)TVkX<U-mR_Jg
zdGlmB5GjRfreTwPphBj?I7_#kHtCz|f5M+CX!ua;8Fl}|)zGTHe%?@zj!I&0+aNT^
zD>c{#M=~nOB9pl#6Zecmza<&0k{PSQ)*O12cJz^f-sRR$Vj+PjlP(8;Ib7mXEHv6`
zosek!;pI3(=26bG*`m~$K}Jla_b%(uT4mcK2-2VD(5!vzk-e>>9Vxyu(TIEZO`e@4
z=EeXGF^+<K&6VvP(G(7`DO)>BSRYvpBze7K;tSKClG;p;&!0}$mr5eeP%Dtzzd+XU
z5lb@dayXbFb>!!#C6!imN`UMd2Pi6QI++`4-fO@}wes?IY}LG4Vd&)KRzQ;6X$QOg
zptC;YUv<da-$#$Ov*#e{XX}I23I{euSDNU;6{pdg9$?Okn5Mc(5Ggadlid9n=S<K(
zyVH$j09aVW=YYjD;|O@g_^rG<e)^TP{`zLc(XZ)XCTR{zR`D|cDU`4)7m<`~XsC7?
zlL|*d+kG!i8M^Th!;Ky6K-8|L9j>dQJ<{oxfU(^=;_!*i13p)vvc7Y0d5Yno7>OQ!
zBt3{hb!keT@+at>pCBT|$R+PO=ny$Y5JeQhbb8}84PZfdPtoFb3=x<FLAG8d<KAfq
z2+hz^Zq177=Px#JL#3NF1<*=KUR21<II+m>uw*$OGMEIrkVRu?y9TQiAs-a0A1KOs
zBpJ@wZzYoW{AFe{*n;lc$)Lxf#+0@MRL{)sw6D9_PMRK(kDLJ`X?P{<ZDfJygs94F
zHt`W%ayBygfMiQWs$4XU7Tk4u1NhSUL~*_Ricn)Q$a`mKXLUifPTJ5<iai&dq=iGC
zL=;e!2jv|2pO^Z=dFFDk2Cog30Bmdg%W$$a*Uaak4|rhGSaVm(?!T%HN^xAjhOaxZ
zUVNr6C({%noJ$`tCBU`ekIw+a5%i!3O(3tj4}Fty*#6zjYhWI9$OW7*8LLz#{;_KY
z%*h{42xb$SeQ4IId6A)@s=&_>0|1|Yub0?i2-@0ADNe%5(f~fpikowz{==P-9pIdQ
z$w{%99s*R-WaZgQ>LF&7@~p>4RT`suK#kcsF$tqAV9EnXeQ|f|#0u?g*u*LmmZgz&
zo9{lqzvtCa$Sm7B<vRpy>E2PxU_0Eq6hudn`i(Wm<rqzZx77pOuKRxSp5)5euQYXB
z>#Z`k_pI{GbRfrTu!!**HIvJ>^@J&I#@4?F+~=>L)(pN^J_us(&Q2Z2D}A|sX6e!$
z=lO~{rKbS&y=TrU9`o|TMFlf?95}=$@^whHRaYDJpw?72Fhp=DWi+Ji$8w`)zYGHg
zq`9g_F}kH=<i7_;GBroYr3yCa5sQ0P6NBKh0#kD=K_)X=TN@p%=#iZxmutGjGy;S<
zc*G3Nt^|E%8YY0_rz8=_G`xz{)UoJ=&cKg(^SLT<#wJeUZn$|A%sCU65CSm@fHHNn
z1XHc89wjO6W75U0UWDu;qWso}beIeiVU}Kfo|JKFddvpYn6Q^2s4Ok)q(PN{#jZHL
zxf>2<0hQv=;YdiPiFh@{6DXk21qI*8>Ff`%FYUlkcf0CQa0Xufu&s%BPc;*>i>H&w
zy5!kMjDWAbF5e2fSfi`HA;5Su_ae$=M^};@M(EQLyQchIWBo&uHp#YOkxhiJWX+z4
zc@dz1NQHR2&-@@1u&UqMSrWV+D!}3I3JQ_XY+_#e7x(9@!nW>IxwL$WO0InyN(}e*
z9Cq!#H=}LCaKKYK^hV1FQ^3g6?RJ`T?8R-1j^WL3jczZMJDJqF!|&bpU<goe?i+bf
zlYBP0uln|V8-qS4`%ZzJ>B;cx>U@}42QTw?vL4T^c6E;~X3iH_r=DDAPP3bs3G=ux
z=YQzksk09QeUCWpU$<hqb6Th?et*|w5$eiya`9jS+zKa(qc<jb1mIg06a7CNx}ATG
z?c4NId1Cy{6&{{@-Bp_V!bhsBHS&Ep&78{WRf=nb2l-Jzn#}Cq`!CNYygLNZTfalM
z7~2t_H4G!AQn)ztnmnDVJ<fAuVk`KX^9CzxCbp~W)JP5f_Asu7npM4&Ba>`Tmwsh4
zbd_F1W^0fnm9b@)s~LITu&oIaqeNjq%pYUrU`j%mNkP2BbNTj|_a=+ENu0sZL*8=7
zrN?US>aWT|WV{CBcK;2x;ukAM6Vzh<{c-C=3*{yreD(L_w+PezZ%%-+D!*34E1eNi
zbuou4`<32eVtp-^yFM}>#0rSDAE-;W`49$v(y^{GMiGEh*-UJ(g3)n7RD&t)SNZV@
zrc7)Xm>!3&-&nts-DS_&boH24b?p)MNMw84(0RXpo*O>J7BKp_DjJs#vCpF}Y1G7`
zQr6@_3^Fc9-+R-o^kVrR4ff}h+5<WQ45p6q#V<-wWB!UEMN~i>L|pO0XFAc|jN}6j
zGS}BPpPrB4yIU;0Fq2tULM-5Dads&IU2YAd*s7|kzENk4M`t=dDbX9rT7LLY9LwZ&
zB}V=tFpPBj?kU42QR5o?KufqpG`jm+>?I0_ZA;rmvrJMHFsFw2tEtuS^K<r#M@PKk
z86gsQdsMNYPs?Db-80}M(N%woDeSY6fNVV2XjT0JuElU!A2-#7V0ktxFT<c*=6|wG
z7UK)MulKJSWn+&t$;a4@#gB`xD!N!aPMHaqbgrppmY08&y?Kp8N0e9U+#S|<s(#1N
zY7jThi*-b~3zUYp;}@+GDgbI~ih*mMW4^v2;|&Xz4N489Q<wx99m7dmf8wVk;}NCI
zE=#l*6*1wQj%B6`G+L_ywqx=GByw~aVU?6-7M&z%Gb!!q{B@W(nJ>OjB5?X2?T~}P
zl?+S$c{6o<&8l^t&VJuq&$U^()*HJj8#&Lp@?G#^J9NydTz-M4ygnJ&nR=a;SATe6
z#;(AC*`QFPH{xh7bHD4kN8SvqGt5_E;Cn1XLXA&e{9>!1k9az(j#tz>d$al*;9nhC
zihMgmco4C4{VJxHSzRBSNAXVfa96+muzHPgQ+IkU>s#&V4`4JMp@G0FQ-IJcN$e#@
zFRtBi&b^+@!pFOyW)&zY_OG~1!z<rn!DgDzPadh3S08ks8wPxSR2TN~9jYcA`ZthF
zC;7lp(yt#pWb8#yV%A|BSSDwF?|8o(xjT2zrGTG}n^<&DAUo}P_kHd*#-%!8Oh=SS
zORyscKe`p4RaKT^7rUk3<n~l<UgL$`<ZDd^+IAy<HzPuBf|q?;ghojJFALz>XwyNU
z(p%KsZl8tk(Y3UUtif+tEQH;noxY5)anIwqSPtoPmY8%T`*4=uZ^^MSm2}rz`%QMa
zV@>NFK-s|*d+z4!-a@D1?9(_}bd1F5erR`e)HW*@<q^HJf8K=!!$cqEXNf1BMVKLc
zxZL9^z?Wz<5ytPI0@3Fw)_=Kq8UK+vmPCtWo?}mAHB*u9-imNGqZ*-?B%MT3zu+qD
zf18oX2KYs)w)@=>wn%h+<h_SY8mqNz4WE%4LfJ&wjHz(u%GsV64xuJT^w&~Cip)4_
z3v=`r;iHip9ur!bEYp8CncJ9t{q&T+n^EHr=gREd6}L*gCS<NUpSE%ld@X0rDk__#
z&5tL|a8N8dE1@lPJBg3a;F*vu_*V`jcD>Obz4ph6aX3lJt}6(UR}*89!q!&n{S;fE
zTz%5i+PklNi8Dam7nt*gN(ox@iH7>(czi_q9Wh^79k7t}^CclZ&krC!=FV4|ki4wD
zj?lCrGwG3w#N)%H>NB;Q41YpJs-j$p!F$8v3e7@Sq^?}|u&C}I>m6UUeMYOFelN45
zfc{BYjynb`EO2^nVhxTnU?Y!XloIM(YcueC{Zm4l6hw8ygO-F>q0_R(C)56~r{o!M
ztn%?V)s~V=fH_^j*;INmj{lxgO}FV+Fv!LXc}e?8kAfj)ZEfvi0bM+L-n(+B^m$2X
z#S1w2S<yrHaPw6pA5e?@brr=<o!K^~E0@<6DUYcAI`)pa?f8<`zmLhzu2Yl!cgHeM
zlTlD9?&-8QxNbTSz0AE~+a!snG>g`4)n)p9L)z1$vOo8mQ8#S7{(~R&niHz;&}g)C
zs0X{gkmox9mT}0k?#^8(-AXb1BAmzdqFR-kt8D^!?SL+fRoUL{uHp9zu*c6Rmn1`O
zKxO+H@!rmfn1)PHd_B#4o^Ml-apAv3xrz$Obpc6~73Xv&yJ7Xx2oX}6uOV8~bLKK0
ztbG!(y(Ji?jpCb&LoK-Wi~$UPP=s&LvjW^g6xBqPl+}cg@t2azGx%R>?&`5h2feT2
zO@nhy^)>Rm!W(h|Zan~8OI{saBwfC8r3Wd)q?b{aMUVFL=I=PZqt+tu3V<18cFv00
z7Rj+{d09Gr;IZW~=tbT=N3vUXa;|R*eK}Y@n>R4;yO1N3Ji;~@{`s^A8--4Lc>|`Y
zP_~O-w!R=k_?EpfqwA)E60h`=T-Kl7U1J!|ZndUo6+!*o-VfGdCX!*QB}q2snwLbu
z9!B`$xBqI+6h?cSicZ~nC&keCJ^BMb0V_l|;C6FvrwrBj##{qnC<J|IRZnh`A2?^O
z#_lhxfT_D|m@Xz&WosR{jDUT>WVL&!&98|;-)nQwq%V?<g_5e~Mhxba+F-f6f7m9V
zPgZgNr@AM%r-0<gX_dpm*+FejNv`aV1<|t;tqc-hSWmmt0AhpzRX}~Od*;<Pmq%nK
z`qk&dTGim}%=?(lv_25F{81=InY;JS$W4es+fhpT!^O9h9-@6ia{D_CzoRzB5Tx;B
z-gVNor$gkPqRI5oZ+gO_ciw^z*<&Cebm8>?H=i2tK`7ker-n(}f2RiI{A7#*c^%!G
zxpts_N`QeumpF*0Vo*R}y(Wp6=)-fE=+!Si*oYwe{T*@1bylP1E-z4`yd90{pMr25
z<lXt+ZK86ez8*s$%s+Td0l>8>9T_sAC6)PVXh`n<g25L=e>f>Gw)Q(?wC$07XR`^m
zLBA7ef7!h)yOl7|5G*n+5HFnJausJmQZ%3DLZnaO!_D8+^5`}B5~PyDDYE5Z4G6@p
zanT#jcww)<i{9D#9u~9x-L$jY7L*rjoMOo?n*M0Q6DT2t(>t%8MhKpJU&Z~7mm-+F
z?B8>!_vb-!U^9#jC$u*GJ+iw;$FdCc=Nf!^<UM55(lH<m|6~ru7`PJN3#`_yyCIcz
zkL5Qu3doqWC(LFfQF(f-xrOmhtPijxda^^l>zKJLY(q1<En5CzAN<>Tpayl^jp3LK
zn$6<KT5UDQ`4-{V>q7l|X?aiPKB6ZZE=cARn-I|xt^rCi+<)2Px-($F#<uBW-aCB7
zAWKFIS+KLL`qs(P?Hs7%D#`=6`nbQAwyF_&d}K`(sP6dtzG>pm=dCMh!D=lCz!(2&
zbhB+TG2gdzlklJ?Ed8)+&>0G(a$4dD1b|1nwS&qtralELy;{PURDbv30{_3P5%kui
ztQKX98oG|9eT@4RDK=b>G%3nCFGfP0Or{p^gB>nlehWF>b)8=f3(MjD+W(ecU_nqf
zOP3Mc2vMU&{2b|n8%YfQhcTSx!?NOdx{tO~$-D*Er6Xn%Rgp-N&@rLIM$gWQNSEb}
zYrpd>hFN4Z53BXsc33agFOnOBb8hIrSr`W^=mZ}vRebNG3tRgYFNq_DO{4F=>iPy<
zTF>sht1)pPe)pT1KhhTJc#SoxDI)_eYpkKCr{~9)dCE2-DxPjR?dgMd;R7|NNqVT&
zbMy<c<?$Lcf5WtSNu5wdFa+=%7)FISSwJZ^p-EHPCXxr_XUd-FigAmoykmr})|*Jn
zTS>Lie~8~$llLmxG)s<dE%y^#(?l<{h0yL+C*gIsq+xUpWw+od0M2pB%hArX)%39Y
z>Yo8s_X;Sc-QWcC!BEHg?yp73upf-~pT3^Gna6D~pH4Qdqda0r3U=2Y;FW*Q2IP&T
zrHK;0F@brsGoGuc%OMyDi>Eb`o{jA%Y_?s1ue2O^`-tOvYBO?<)BusA`G052-dZz4
zb{NaG@Ac{2#9c`3yQN{F4!U2?e-`<H*1f)FTvaL~y&0MxbSkZEwuiv9DlyHBI#d=d
z3_b7TOuDlD^xn4V5n8lA^j1v^59_si?>sMekGDhQTi3Z}r+hB%UH^oDGN0F~P7|N^
z|7DS%#t;iWQ|zr*bG2SvbhcObT_^JjU<p%I*&VF9SwBYWEZ&f0>5y?ej;MB<U83uz
z>KD)YW)p(R`d2WiqRg}%FzM4a5G~k#?vc00_kD=8E5G+1zsOu;|5}6!RLKy>+EA@d
znlO3F;ItdL-gmLtkS*sdBoCq(C66WycQ28b1C>?4_o9dKBx2^jN-tgq2{tRz>(>_H
z^ILv@HG^?c!p#yEicD}$QPO`u%Y7(YA5Y~Q?5)_i8z-wBh?|F}7xTW5BjPb{TWf7L
z*H4}u!gbsKmkh0f44)Z-Z4`(el&Q@cTQwz*oTAKDZR9-pS(vr>Rgokq#Y}JA&>QD8
zeT%6VCgyqOPdZbWtO_TcEca!erbrYfNMy3c^eCW)9#Y#J8jXKTs?9{ZnwGDhq-3BA
zkUxo4^KmV~PBu<0dT%%@A&e_7=dfU5Kc6#y#9v9e$Uf;1j^BG|)>_fUNgL;P<9?#H
z!1_Y*giQsIw%{@;GdNyfFq@)ZyUWDu5nxi29>Kx1$yFJp9f(ehHKc=3oh+HEA0Kfq
zi$V3cUN?)H;PN>B4W;(F=5!M!hA#MTq#2JY?7gD@PS?B1-k2?p=AHO^mLlGi|9l?f
z_%u<TS3{G_%gZt0UDlSGSb70Ph$hHep+rpGW1<hoQ+{$^Y;}62Yl9^$*n-8;HvPpv
zj+usEG%b6h01GRVyk9$@e{{|$CJs|l*m3+G$BLawlS~Oa&HhvpCW~ohy!}~)Mz1|s
zaYM#b%CDwdp<N~;ZYT~{BBa4d73`7jJQ&EcFMxkOI?b%@`yA)Ay!l>NQ3cy_e`tq=
zQh{}dWgMHHxNidLi!x0pn2mYG;j4{(Yyb&|e(D;zhnqUboVyEXEP|M2S*7l4B@5k#
zgkcX0dbi@!igfEx{BQ$~0|0$i=W1Lr#w1@k8#)OptvM*=Pb#@xF<<74GncoS^H39s
zy`*`goW>)gHi(MF^(NjTqj1F&TQ5zU?$_T;Q6Eo#F8VIT==!-rNST~OyyUmCL+$th
zAoPWtqve>#oZW?82QkvN)kyKQc|WVOp%~-rbF2;iTF<SK(iLuzBV$TevolIcj<*{t
z>GQ*Pq8a*Rj1VFoo3@G~Fk7Efj6@T&HmlfwMTF)bi4Ehj;LC5b6zyg*q;%0|Myp?G
zVB!d4%)@$%qJYpzu)3+{<}1R&mSU*uN97hMcX#v?KM;Kft>LrBX++qN)<oYMhr5lG
zo-04W98=Kq#d2gf|GrO2j!Gk={pB1eSVt~Xw|Wt@A|&v5lSW-!G#wQ|efP(1a>-$R
z!L#`v>~`m=XB+F3BW30M96mil;kx9777MnztszMcd$9MUiQ|{86czgZN`U3Nftf92
z8&||6s$R>!KSxCrc7?Ust@^|OZZEndP8aHuvMlzU$E3P8o3|Q^0}im49h=GPc;J6g
zK0io6Ba9O1#=g{D^}bH1(vRZkzDVevb1@Z7NQnyR-IG5C=u6v(Rmp^)Sf7aRzKS_f
zOhv6R95~5CbLBkh>X~TGL?lHu${wu~g)FA+M37|7e}aC#*v`oXJ5M4?W9}x9#NqlL
z##T5x;$MBr-ui^5dJvTZ2%KR<f#?qw!^bQ^V%WzT^mvkG#a7Ia?-W22gud5QpZym3
zlI&`vk`nE`$D05Qo`qf%Pwen59`V>SRuR%JWHg$f%=FYe`y{m(VCM9w{sas8<nAY`
zk2#bAOAv#Bbbu?9C0*J+m{cti6{w?S!&cW?q!vtX-lK^J&PM;|?LHiQMV>;7oVc?C
z6pCEpKy<H(>W<%<9z3@!4R54>0$og=X*m(vEN`lkVO&H7Fo_cH@fW;~E_T~B#Ul0`
zl<rsy*d#OGn1)g_CWyJqism|bT*XyP2`m<C%+0k>&uYg-887!#pabK|W}CoVe)an0
zO!6Ne+o}k<(sj0u|6co-^_V<p35ghlW%n>`gDg}%yLt_}FetVO7VQ9;%*A<(Uuzvt
zDp8MeRua7~(e6#t9L>ZaO3b89*{grI3mvG+i2F5LA&i^R+1cb-9#Ds;__s?=ZVR^m
zTDmDo{8+)~jV<^2^>m%WIK)G-&ddHqhNqjrt*!ONG}SVAm8w@4b;Gas!B50gYX{Vz
zfkF8DUUOs7BBK2<`#Rbcc#V?7L+cwb0T1uAhv8kqu1vScsPFcK8^%ymvpAhFv=dpK
zZwC~|ktYD`3;8|_NfP+CrbX6=N(e9dmc&E#i8t3&O3YpbdZnWveZ%1RW|O=OjD-Ia
zSNSYltZxgm)vX;<XFTF=XorKu#RtTb*>pbE7+SePmro=*!!H=Yg^-&BW3@d%+-?g*
z)-O{(_lwmI#M_Z=TaFv6@8=JP8f}R&7E8y+skBu*ok>2|qYUW|)kzltQRju5@OkpZ
z<5r`uplR6R8gF`p(;`~kYeeU_^Pu%e+cebXZB@T}^^T<j-Wv`b<j-ETY)=|Clch!`
zt|&kuhi*mBrX-4+Q>DLA1$d&uB4%<2bzUokr8S-@Q(v~%5)-gJ8mwu<u4sC!ONIOR
zm&<`*hKdA^Z^>*H$t4!7T|a!CG@I_GYUvBoRbMlL$W_O`FK+%M<Sbgq-n1Gt{TKGk
zZY%Nt{q|&dahv0~dG7UT=CGmH$)=oI)O(CQ=<UN_hfKWF=I1SmYatzzs|B@9>gXlj
zW9JFCFOMx*4QqbGt<ZH#m=O5#p!WK+gGbh|n6rFp>)esL$c@;u3xt0S_Jcil`^Mqy
zx6S_}?7icu{@?gv<HWJgLCD^FhtRPJ$ts)7RA!FMj2w<V4p~JuC7bNX-pMFJl6A5o
zv#fHzPM=SGzW4q3-S^!;=ka)>$9cWp*S?<D^D45>@muJr`gZBz_*cP*T10l1E7!ow
zi)XiahGPI&q~U3Hyzj$@;Mikqm?Tz72o=%Tm%cC&i^dmgy}aW@53j*_x-W;^QbAOl
zFItL;%JWQ#c)iZyKH;YoDXE<t%QlBvH#Box;9M2LSb4e7Nq!J=Ml*)!5Uz#DJ2G8s
zQTK-mv9+_KcL)iv;xj@X1(#AQAMn!zsUMWEPM~z2dXWfW4)$s@8I^b|>_Z*+(dFi~
zB{BHSuVP+Y4`8@eBQ*AOUZMud>T^PEEi<JTGV!EIT$zUeLMQVz*Rf(~{aLb}ynx0<
ziRZ>S<4ke$C|2HheWjoK_+kSs<k(UfWA~&XTV^3B++^ICF|j))jHvgqux&^qO;HvE
z^Wq&}WNX3<PZ?YnX7S>KbEJqG%?OyiR9;><Udl_b_3yH#A%7#e7Oq?oa?3ClczY8f
zPZuIVG99KGkNf*$n&~D_YAO*Bay0(pMrE6|`Bl-ga!^R1eM`bxZ$*@a!-iZ}L<Aa*
z^76t!c*D(*aZEU^!aHp&PUGM+uC=r2aiHXKt|P1lH*J(sAMOwv9#~T_sc5g5Na99>
zj1>-jZ;`!mi5R8F>o?kUsdbakZdaz~S&n~$`SCKB-2fLE$U^(j(x0o446i29kT*g?
zX_1z}hu@H|(!z)FORfp{ydtO{%#?AnP#U@|!}P;%u7}%9<>h2<nRgT?8D#L%V;9q9
z2cGylhH`eh>La>y_)cNy`kjXwRb}j|72~DI>X>jx8WQyZPZ~{_47)qK8P>M5vHv>Z
zh}_&sZL78+TZj3#(z4$4bMQK~in<KWA@-|7H(&Rag@=!T@!EwxrC-SFVY32cRH^Gj
zLuGukaeMg>L<}R&Xe{MprXt_627DbGQDsLC%j>ti+|f%gnrZIKF7Pa*8LXnaR8^IG
z4>B&SjTY|~%+|z0HiXR7)E~><65?^IJDB_Nx<TD%u)J^CM4Me8{Uh&7t4|ZXXY<5L
zS}VjqJGSTp3D?d#y(?ZXtMk`iU6L2$^NHp3V$X|yKYcB3XyS8>U(|$|?SdjSnlbjF
zYqc473lkFaz7BPc{k1e2Q0w0AS-Wh~eQ-N<153^ZX^gZqG!u226ua3YF(73Gp{gso
z@d-eC3VJKx^`B1qt3Khgk+=>32h($QuSny$-lpZnnYuHH74)!=Tvb)^<;o+K#_yrY
zWbc~gJb!d4()h?$?RH8Co7oQ7%MKJvEOoX${gE!4HY9b`zr&M+?pj3HwNK7hAOHOI
zNo_>tXT!}G?;Gy0T2faWf4|rD>BDDJZMXh+K2MJ$JSIENV7&>9jgRxE<1LTJhZYWP
ztZ3ZL8ylxJsGR%}m%+3VKW8Ah6Y=FnPd-1d?STuS&&~rn`22I6kwGfGm4uWifQ>YJ
zP2eBBLUeQ;;kW%lcu^!WBoKYRvT7YRqI^`~ym)n<b5-I=f!BJzaOa{bEYeyx|H@>F
zfZ)f#dc#Ax_YCcMc~aN2c!s|;*zM-)EAG1mrwXtO-#v`h<X1b0j|ooY^3b|rZ3VEU
zC#7cFjU}@ox31lLA=^<Px9v>r&`2I<CaLjSBNpa>myGX;*51tj*>|CTqC|{H@B(bu
zR3|Z~1IhbD8ueyS;`>m&{m1V0=Yg?wYm&-$f6(x~HVGw8K1ei!%^XB3opKz-2Cp`l
z>{gFkKT9bZ!jm~^aDaDLusUJ|Y4)yNt0@`Ya;yJ!yz#vCNi}JwviOv=An1C;SqB+S
zZC0^#rUiVryYq22YJ$=-fj1iz96}5_f~Ocy3q?S?vip){>D`b_;}6$D%@+`{+yo@h
zQi?}%_f0Ie*(>ha=LFQ5SY4cQ=bL!UJbH#Po^w63cidebh|bNUKL*`X>)lCLnKMJ)
zSx3%n?TZKTzoEx&**V57vt$>@v5dAGs66VQBfA?d7dERWWz4Xy<>XS=6@OR=Z?K=T
zRW>fhT#4Ch(N)Hcb~)=hzB#PxdPDV4v`nU6=|Xizh1kPi(;j=iXZo{)*cZ<4H^v7<
zE-ljx3GM@?rmw!3I}<;6GhOqSm%n_puEMIq*U~ViywB@eiNA^Z2ti#IP%8}TbDRs)
zB-2+rO)erXbDt`4l(7q$lxbN$p}omxc6omK#R|P8#JMyc2N++7yZDwrl_M4{z<Giy
z?0>~|J=!&i#?cQm+@OYn98EU7mQL+C;VGQwtoDB7E+TFAGdnxG{kx=DzM7{h0rvwp
zW5akJn59qouiH=Q*Sn9~H33BX`aw-a=!6oHAbji0>{XS$gv&C*JXSijeybXtHoG40
zmi~zq{f%=l-Ruz*X|*zAp)K!upOfk{gIQIQGGJt(pIbQ$uChOSctpo*f;o$b+6`3e
z7h|apK(TAEXgvBX=GWJ<KdwVuY~LE;XWp>%QCFQu2hGX#gU3VQIC$wHWuEc3j7y&b
z?+-XL`)5W=VC&Qle_?0G`Q|=W*P9O1khv{d5_-m$&4q|8r<o)p_Tw*F^-9bjgEcK!
zIGu|>lMeCB9p6{TUI3Gof<JNgVGw_e(BEmAEQayBXxeul#lyy`u8+TX%H*A;=q+l^
z#Ffm&npS@Fe9QerImqOz$j<EH>Yva5^VP;BCDJyzvx}eRCh#|vPHRiMKnFvS-5PFI
z)->PvsdG<XA-~vep+W&UEbj2Fyo>kZvuEX&)@^#9A7;I6tzE=!yK;M1c@HT(Y*Z3%
zURmfoapNNxqkPo&$Wf@Xn*L0F_Nad;3#y{#wm5CBzCG$&Ei&6XAZGf&-)v`#QoS8;
z{7PIHkcgN^T^P~Wnb(MG3gX>Z?!>o!YlApvCFDU>h{*|TTqNrk6Ge#--&`~%77f!(
zX>dmL>o7lvS~r>Qo<1!sz2^0vZoh<1a+1sF*;~U1H}WjQr}Bkw`Psa$YG$=-c<lL1
zi#r~~H<y>0A_<HSVEO^vllL~bvYyM(A-*)9qv|lu8d|1F&Zj1N5o=N|)H+e|4@#8h
zMhcgovMO#sTqkTR2avLyl+&HN&kJRkwfAJQ3yHbkBUl0UfN(ugy<J32^X9>{?QW)V
z=@I0$+YO#j;sYy>j&rE*+Rk9z&(iJA(C^~>gMw36JIT_8Sx0@!8Mvt+47%2ms~v*~
zU0ZR9e3T{cxOtv{ork`V<%R9CT51D-qWhM*+f&=R_n<Eo8(sW<HVY|TxqIVcna|W~
zr$xCo#WKCRV7IZ(COeEDiFXyB%!<jGTak*QV8&=CLHt_|tjd=K`8({H8zBSix^v<k
z9q4#&KEK2q<Jg%i;XzI1-jTCciLo;-5SkCQ$s4|}HtTh~UAD92?}rj2%d!<$bHldj
zNKJGO+0;PwQIkWxOYl0VGSVO#KU_pbRu$oo(DUv%!?%VL)2e)u$woM!+B5FO{WV6}
zeO+La+z)+)9~P!Je)8yN)B{wN-W=bi%R_JMr-z#(U7<?#KIblcyUJ-fS^{R>L|aX5
zy)h_aUU&arW&<Pyg%Wu~C+Tx_eu~V&hYLT;R<?h2)pMS<al2OX30=7Mc8X;1s%5xA
zeSZW@)^k@wag1Uhb)}H!mE;=Pn671$xb`4g63a2E3ZdBs6CEZmi5pYOeWj_V)3`qW
zB6WenY|3a4;NxT?CTMM!D8X8h^ry?A=jpP4d=DxB2r_txoD%ii`OP%pG};|N_n~yU
z(e|s<D053@sl`l)R%+CJcn6Tr@C6=}+URb6xfyv6FW_yacXrtUgO*Al{c`84<e*-x
z$-P3VopEZ>YI~=V2A*hss&BTix0`fKiQMc*S}{;z*r20%@<33MZ))G8*r>3$0I;k;
zP_9Ytms#QT!?V=VKz|}N(j{wy1aZhwSnyh~ic)*zNfNr8{L9-c%bDxp0Wo!P4&_7w
zRNdZl(wlNCzrri#lA1)>b@PNkkx}qvs^|xaZ`j#ajW#M<Rc9wxz4t1Hec6AoLWERd
z4jm1~ix7+n$K}95>_aCFyQV_(Hm~!2tpG`(TGaNccZR0RxuEZvR=u_V)<EQmkRLw>
z-k3ucg$#wO?(Vp!Z~>J~M`+J0y<_BKNw*K3dZGcpo@{rGe5JAWJ09!YcTbqTTl$Hx
z@cALZ^{IGE(X%U!3unJ_I%^kb2RfUOOQuvh=`9Q<!VXB^AA@M5L)ZFFP_glDVBpAu
z?>(r`cI73NS=e11HuEyv#yp({VPe1VI<ZVbFGDI?F5@RBE7?*<ER^jIqD&a6`imiJ
z5y@fGPRvdbWL+W$<pRY4GK_NYu7KXH1P0^5dqXdS*NTHu)SjSu(B+`9ZL-=BeqJyo
z+DJ{~F${Lyw22_D_dYuTAq##c^&*wL3~QUbp7{gn7X@jnC{g14=Fu+w%f>QT{<s{!
zgo&jw8m0M4@XgfoiIAb16t)w2&I7byW{ozM#A(xM(DoCt;x-DuPU3Didpb)CYwo@U
z#L<9_xA=aw8)F=R#>0}Q7$28~&v&%*)6hVZIUb*9yT;&Qdd3jvXw1@yCukziO<v-x
zA=FJzv8WnoN3GFmqU{_Yx4}PNKIwM8V`(DYhT<aCC&d=0$=ie(<HDsZ^|pXmAbH+y
zNNnYfN9LYYv{rBKf$i-tO&{*-POCiR4Sbw(g(2!g$(Q_k10@K?%B<X}VR-NzM~+iX
zUDP~E@q=I~+EH86QCVi;fn!lEj)`{L=RoOYvQ7N=$C8_q!ezUTlGCIoBM!O|(JWC$
ziK%7EFgtsnPBe*A4SHjoT0M%7GL-Iv+vOis4VzvM*$A3fyd*oq(kj}H;;IflVqz?-
z_bIBVe12K`JNY-R#IQy5JZy3jpNM$(<XXh)i-8emQsFz*ctWI{oF&X!QKz4$^iV$-
zS_2ZZ^;tiPI1qglgt*+Ek&iop(quQ836T}<SQUySwt?b*=$6_b7%N>#N0Z!`iT7x$
zN~<n(xH)5oL^nni;XylK22<|CSDO^=vBR%ls?5&cq~MJE)Erjhr(aK36Sb6;(mdUn
z=zB5Ne=JRW@&V-A^n@Pm4O4~jet$dr?12Yh-sXe_S|#6L^^xsls_$6(<=9^L{g5BN
z1AsGutRd;CLVyn^NR8^!En5wPc|XX(qGmt3zKnk~esC)%IkEJRNX1Vd3-I%9EjztD
z`P5F=Chv>xZOXfTuAXg|VKL!90em9@3!01&f(p2>hB!)4mDcH6#kfAy4*Ja_(dO$J
zJ0C4i&tcd>hC_nfm5yc4PrGcJ;NJ%#YO$8>&UW7f^)r!9I{IThB`305k~hf)eF1|-
zTD<~yLcI))!^Atm^c7Zh*)6_Vrm9<y+BV2dE)~AU;mMs&SiPnP-%@uG581dh(;%ER
zT$k5ki|qpRtXOSC-(2h<VWm^}ahV_$&d@M<43H4=T&^vfWx+9KVok@EeUX}5Hb>d-
z^z9fR&o+;?sy>|PlN9tFh)@^mn1RJu?z1t6eKwSpEwi7{k#-(2Eo5%0A$OkBAZ8#w
zCh;@~N#!ekqh{Mve5%#rtYW=tUjuds<_V+r;jKkpi{;xeE~*_5!g&FY_Ddlb+w*Ey
zb)6c!r)x&l_|>rHZwZ6zC=`y|5dOw&^3u_yWdiS9220E+`od-NdIea$y)D+3vFvl9
z&m^O7u*z#H4sx+@#LHSO!pHmh@Xm|TvrL4G9>EtAC3aSKL@cQ3q<@MtNZon+^g1ii
zuApv98m;B>7<aEn^?Zl&BNILoFjnT*!@7_G>$M2K`AML$>4JU4<B7+N`^Jo-VfeUt
zP)6v7!E2cbpX;5(Ly3=x*#bS*7+(PWE<~GD+HWc4*cN`ZYNUHXW#>ZqgnUKbQ8HU4
zu#C0pM-Aj%EBPGADkN~DMX4$E7i?eQN3qTKt`A=+IuF4#t(+=8x?E4T^dTUq?ThqB
zP&cq5>;MxRAx(oXz2{dlZox95d5PvFEUmX@YW+F<lxycR8dS~6w<Sp&2Uu1;en!pj
z;D{B=?>b(l`&l`fr(i{FHY~uW$&6ID<y7OXNd0!H?MRnXH#+co0<Z(#-@kw7C6g>O
za5TtLScLy#vZ_ops!Exc41|xKb?TK!%~&#w;3!Er?|3Mv&Ab!z=DqkBl4Q)azW6XQ
z;e0<!h7@o1fJ}wX)+=af$~0ZbM#>p8jS=T8&$%-vA%X{&*1bZBbK`EhsIE?Xe-`)8
z{M0E}8vRiLR<XkNkbl;Hz<(i}Z{`jogFTHUt(nfXdPlQ@#h<FlUNe+k>-{<RT5}I)
zv8{}c1V@xfzFh?T+NPvmOz;c2($6aAiptIr7cGsmmD$}bYSw)jyhVS_r{-qH`7Qm7
z;@cl3`{AkG)JB7A3V<e{$S3`!d)3eHAD_Ef?*5n#C!>!5*Dy>N^X$|6o7$^f-<pmt
zvPz<a<8!-SS(P@p6#r;hzQx_IW_G9lbC$4O=nsyvnknDrD<jEuwkn^R4j8hCp=z97
zrZXnt<w;eA5wifyUGyWUvt?*V9vXz&f9soAq2uMb5Zr(ASd0GZ3e(zpX{Cwy0!SdP
zK9)t=AG=qj1*P1+*?;czvxqkO{?Ly!H#QUFTM}=-^x3GB4tY0<avbCe<*{bB9%HtH
z2lsN^o)3~7?!=9p^?_c3)u-WvmCGUb$f1&fOA}mXp3ATME(vq<&pBD}Jn$h;N5+{f
zNIe?DKFlUV`Uf_N{~+M=TZTTEf-^AVm-$^=&tjd*UcteX&iPu=i!6&WAso_O;w)WS
z#l1Hay5y5DQH*Hn@KBu{JlkFl`{CnX*>aSi>;~P&d1trB(mEeWScg?T{ZeiyZ_lfc
z(}&RCl7%q8IoSxKunt)q?ZQMU<supvVf>ik=d!acf$R0eYbh8}VyA7GQTm{&{!#v=
z=>78*gFH7BR14l4Fy?!1mm(WwtRGx`ej}wILAh3$?K^ovQz4C}yecCSy7s>Ev|jWS
zS`lyhF(tBU@VP7h3ntCbzVl_S7f3s3E6bqbqiAf1Gg<}l%HpXpkB>)1h9u*d-L8|~
zQ`%Hnz`O2MPsRaRBnbOO&MVPBU@Kn=mw2IM6&^jzDWrhlHgkF#(`aRe>Tee%tQ}4?
zfgxgMCP;ygWd-Bpd%+GHm6IbewiWk@%IJm75_$7WhlLb&_f*gJ-{}+cCX{5o@2Hg#
zGQ3diXe-O+dgI8mNv4(umQ1g4u>Ev&@LJiZEl-EzA!qLIafaM*?H^#RmB`I|*G!a2
z(TUxpzPct79GcAEG#*Pggr0+$eAUr(Dx<wEnRb3G5PC!AKq`#+x%_N73(B3-uh{o-
z2<s)g>t+tE6yp-^o!#4mF<2a6unRZ3AnbqZwb94*UA+)#Xg}-h(sXc?wmFm<;ZPe-
z8b1S8m(skbwIcL?_+|d6Q@sp*=Um!*c9WZce<en=>6+C=l`!&p<9@(NVN=81ZgM`6
zdx%8nJ4eE(rqC&-nhwt?_N?K2S^Rd(Bio~^x72&iH<``lK5XvOjy$30*4WZ8qgE9>
zNb;OIRvXdVk1e|g!ze4`N48i;r35~-bL)41IF>HNZ66gS(n1y$SiW!4T$fD`9n0YV
z!b&k+w0c6vDL|KY%PI7vx!%q2AsF0*k=#<K9G_Hxv9WKkCqg(d^dxDnei+Rvt>SWe
z63a)>pzyMxvltPX{keG7&J*?#!CwIyg_p8z5(ASwCmpWapDwf!G$n01cn)2(y5Z{7
zx%V=P-E&-<uV^FY4$lt?m-s7KOZt3n$DLOD@H4(299rb>?}VShx<%0^F;#~+CT8{S
zLU*CPl@WbzqOw=Z<6!c;#`40KqYwOcp#ch?FWY_sp!52%@AWnD*55}t4faK}e>>8L
za{!whK-ub}*$(%BFr&7BAG{+C!JM{Q*fp(tvzoMkK{AifKILmyJmWb0zA_;nvth71
zum-$2yg+<3#_E)rymrB&0zZJF3vM9q80Bm0eRNnLJkGNUj}+QMxe-3-PS!JjAI8Sg
z2xh6TMtWgQn^uiPZk3q!mxlEY>iav>279M?$%palu~P#bdJap?^PDiZ!rqW0NU1`f
zLff@5vV&Z2?Dp)){G6?tWE$@7UV2t%1q$`dQPR;1gj~ZN?{u6L2^g*1&<xe@IzFOV
z<`i42kma>Ac0%u$!PsIIwn?c_VXgtN^+3+i=r&D2hz`un-zZ-U^*?qN{>bfkwssY=
zujC@rK`1V6jB(_IvXDbI)M)aVXiUiQUJ_*1g1w=&wT|>|fD|_Nks?iKnR0KZvb#uR
zv_^%UTjWG4pY3BQi*HvnlAm-H#<Gl2)bpa0L&TfNNM13bW;@VQXmu!UqGZJNLoqed
z_VB0S=b}tVAn-B~j~qLKwz=lJLUiZZMU=a5QMWy^EkbGg&)Kni*~;a=E6Q5o>B~1?
z5fE!I2TP$SCNGs0(RnN}b~y@FgSF?<Kb10#8wOEd@%^(;gg<ynf9t!ps#b-PGAeg%
z!ltdw6e%UPvfHH6ypo20?gf;KYTBtMdRDWKv_=z8%nR|W%bt1>^kQdZtQG-nkJZ~-
z8^=jBmx`ks5wEQt`v!p~@7Af`Ktc%DJXDsTiwZwG>wp>|)qC4eAm-)TmQ6ICWjfo#
z)19@Gm~e&(($Ct$kCfR@9VTim<i1vQjhjKq2+Xt=$K{|N*5=!AdNZ)vU8-IWfzLQN
zt7NxB5JjVA(}K5y?;$rDDW=?fNPyPZikeGt?^=CCZ+g@pX25FSwYzlH#vcjR7x7!A
zmUfn2m$|xLW|gWI)~#M0Z`6WkWVl>Fj+Rny4nY-Brm8VEo8&Z)7VHs3A$B$IX^$2O
zd%-8lCHuM?Umz15+{Z|trxf%kf%Y$Ddq%fMsOJ6c{p%SRWq@lcub;6S8|}U?O*kc!
zb27wL1!Sd&#Men&rN5Adn1&dTwG?Ul5iRX0Os9@~S_ljIb*CHzI@ENx+3oILN4cS=
z<fA5fuR+$rA}|8wvv-Q>OfFq~qVVw7>8&l6`)9bOCP9u1z2x=R8GeUPpKPs1&&7id
z+lhZy&Gg|;P=Nl+162K5zs{mVGRgIycPg8{WCYV~c?MBkCP+};gV~mHn%CoKLelF6
zovhbBS&jqx#YbAq`ynDm)A>c#$hZxSisQfs^n1v-bqOY?3^`=jFCo2(cpAm4_d_FL
z$PYN>#s@lHObMo;jUsOd)tX%+vb*`wt3E#=Uy`Hs#y242Mk!yBYUmK4=4_c&?5v!A
z?Lx3`%4mvC4}9*!BEFkiOdjX7xY$IW;;SP)h>du64tp*tN2+;8=c=$48D<ZrusW^6
zOnx42P@lJUxh?%D(3v~Fy|TB!VaT+=!RrO#Zn7q}|D3S=-Hq!Py1B+v6RgI!?=Q=v
zi};j8=@2prbchO1GFEsrBO3J;hoSWZGlV?+vrAA~&QbEvz7(f>zX|JFf(aTbw1lj7
zt~P>2Atb)Pj_1bN_3f~_aqqd)0B1GSS=jme*XaR^Q0WgjAMh3kZ%jX?^t~{LqpcGR
zC0?z5#8QJJ#2LL;dTu?fDB)hvhNaU-!hIYCYEhUQ1CF>pWlJvb@HJr5(nmgi+qU=i
zvZ3!99<C{Rrriiw8`S{co?uA87Ctd^qLyyGI@Hg#h*M#FN^xf{APD0en|#88GGT8&
z!+g^pu(jmUzQ^6@^NKZl$kpuzMz;4L*Cc@TH>HKv5J!H*X2at0quW1azCx&CpPsPb
zUK7p~cfgAy=T%tME6R(0&#^0k6(9bxO$AE7vS`5;tRVYWU;4V-#|YZ+;D%g3EVl@p
zR)++Mt?;o+BRVaIgo`5G#p?ZuI{u5S#qPV;DIcA<$ptY6Ve>TQfO8gJBVPv`81{RH
zFXXS8AHq=)qVj0F27bi{I$$fe69KOnvEzggOKSNkI`vtNt6U8<G@EjeCwtiSQXF_q
zfUHpYdMoH$JSekz2ggH!$#ecY6NaS`Yxh2JUYqv3-uO$i0ey!pC}Fienm0RZek`zm
zS~9F$0<2%B7LQFhau*hCXZ^KdbH*s1yeTOO8$H7@*<RI>ynq(C^J~Q_tRIkivh*gf
zF#S4RQHo5~SM*o*QEazp)&CiCRUw~~KySMks^8P?+p@&sbXmBx_SV27b=y$dIi>-&
zBz9I|fp5D*V!mWLbmAaM6j8^^UodSuo-k4qvE-_&(q=J~qSLKv=6E9QGKuGzF-~-!
zWvK(^tOpX!jIbpSQ64_`QPZA*IwT$0_!FaD5es3#E#H3>$O9tC)&BD0=lr6mLc68q
zbQc$XoH1mJ!gGi-YjKYh-&)&a-c)Y>E)S?;+a3p~AgrSzp>PlUTS;oSD8{>WV4BK>
zl#2_*i5Ru!v5%OxTtBb-N=H$eS_RgAq}^QI9F2=&xdn*YXsI>Ldz<;4&ciPTC$v)4
z%sxKTiX<Ax2cvU|y&N2Ec-2Qy>8y^hXUr4ytwo-xMCM^NDO7DyES!%MpY7&fI-1V!
z5%CMT5)TS#jQIBt;25wxk5!GLkT?5h&z<5C^arJ<Aa8OlhO**_aHaq+F&Rg1Gx3Re
zvzH>c-n(C!vphOUqN@ytZR73@k26eWe}osGN8te9Et_ulbf&8^!VZ|z6UFhFAmc@^
zmDW3YB`S#PX7+$s_`JDdL8-H7@cBHBW3k~uOCJbrqZ&7WANlQ0$P@5zxPS75WEWY2
z@+<AQP8=5HXWKzof#UHI|EA9Xx4{d>3l(z5KazzeM@sFGfE#-0AH_wHfbX4)p$s0n
ztQ^_gV`q@%OuQp_>4DBBx-z#py7H=#w(;PV=-yHe0%b=Ma#VpUX{nk_b_;VMJ`qcv
zn*THb#fM|#`WbcWMfF94X*7J|&eHUXZ9yZ!H|Nj!Ja$G07#c)L#Nc@r6gq^+>{S*+
zUi6Clw@90`1Z6(1+_Px|IO3Up4lCF>ZVrXhOyeldNZ%o9Uc!JsSy~|!V!4wO<TC|G
zy$FP~JQv!w0r|Vktlv_+|D#Bt*}36R+zuDq*6tTXWh*$6ACa>1fS5VJ@z$B7$@+wm
zd<P49O2w~s5|E!pCsM{pSfx9pIa7z=m+-jsg-9=VIH+68%sKsQBZvTT-#jYed^nb_
zXj&GCdP4g=6l^#xh0*t@olJHS&cmuzmr0n|l4<yqTf)-q8G`Zc4~{ODG9QiIcv?=}
zCQCY&ISBt8BEAO2(u|<k5(uzKrpU2Ob$?pc_eCKO(1R1=SW^dvokEVnkCZcr!=ADZ
zOkz%;GtE&?AjjJ@3>!?U`KlS#p_~wIc9^Pnw2I_!>*Vya!U75{Jqn!cwR;+CvU37r
z!8UT>f^sb&5rCfC?efjammsjzygxF6qgk^K*gTKh-@XNIM&0Ydfw7{3)o+J|vxLXE
z%MEIbueHMYHJbFk6}tuWIjUuFVTLKM{*6z-hXE0Oy?vIH)Ajj)vkqdmYag?)C2~n&
z9;}+#`Hd~Yzn!TB0l2Vpn4=xrSUPlvES&4{Ub-1|q#yBBKB{0OscY~e%zUlA@^sRI
z*pXjJeo=VdlV2$qVoDg9(wDSidfEQEcYbH-J66C!6dF;&+;n%`Z0Jf}bs?E-5&XHI
z)gsT;E@S@`E)Vwm9{K%Ylul9m8ZeG5DN6iGDKBj{7OMKYNZ{h4EI=i@E*IOJpbQ1`
zh1&DykZa>bghckuBy75fSLZP9@-B0e?y4b}Hsn%o5!C=Ot6y$r#+KWrI1Zv>$Jd>?
z^hcL&L1jC*2)o0=p|tQzfYUT?vNAL(Q_}IOnfHWo*Ahl4yZ+wwjQ_bNaTsc>K$E!l
zMw*xns~N4e`gj#`cc_ksu4mc{Cql~p5I_vAU{2pn7)pHqBrOLPCJ-mYiqfLlrWSrd
zphWCAAJYJ9jlR!5hj&VNN{ywlQL#F=W<VLd-iLvu<@gOI^0Q$lga@r$uIwVAc!H)+
z<*%6SjD4hHPm+aX?B++Z83xP8y$<1u>rc(rVt!BJALh548wp~!4l(+hshD$^7fJIA
zbO=sZ9hi1=RvGeUK_wCyo7e;$)b9RBHTAmpP4O1EhM<)<ma{jZ;)n7+U#GSgW5puC
zAXo7##&6}D{J3i}eBQ<M_nOF+aqT?zohSmxApDZS@Vl7Kjq(O-&O|Zu8ty~Yc!Cgx
z!9P)wI5dt;NpU;jJD771$5oW@I=J}U_=P{vO0!q$-fsf|vC+Rqs{=QIv`h+N4T)j^
z!Xn7RvIE`jB2yNY3zO|BCGT8Lw%$+kqpU|Pj&V?;?kiuob_#zUCd(~mb6nD<Um9a$
zY!<$bcRrI|DL$Vgqr4l#Z!JIgT>OEQ)$ww*Jer3SX+V5De#3hFT!t=vUYKWL<|~)(
z+7}mZziD*dS?pP@yGbdxoG7)U)3f(wnCvTuRlI9;51rlTAP&)k0pHhqS@priavf^@
zuQs4Yk?ryuWyw=*Vr{3M_IRf0rF*sDmi$+`Eu~su^H}e?2@^7X#iLutiOklmdvmok
zAXcoUajP8qb1%g#;0a*j(t4AWEYq%x^bFTw<z?MH1VDW@v%9-f4jAo;oBx<5Tols}
zYd`+p9Zf?c5q5{y1mQUl%}hE}cdUVw`ScT~uB@oI`D%9O-AJ@4J2epk0_XNL4J23`
zd#OqOEO1c*MrR%-OA>qtEi}J5Nc#a#30w?foTIFERYcp+N}&3YCM5c*Ja+tUtzcTL
z=;TG{hO#~jZVyD%1XtFki3fZaN}F2V;#$;6!Xd0e9CWPK6gB+ZYrH?!fQ_?Ny5M1#
z*7oBK2nMY)0g{4ZMnO|A2sOdAsqg#y+-Eh8L*w&TFR3npt)&Q;xi9W2%9EGxegj0u
zpU|ESPa7%SK)G0czuWKSUo-Uqn5jq`(g{uj^;v_bR)aOcmNLo4<c}OW2z6LR|1(oS
z{{XGQWm^<eX|p~UepC);aL{G~8h9aQ(X|;D0Om>pD^>mQ{Zlqf(N1DD@$gX>y!{a|
zg7A>Au@Ko#43pz}K$9Ngb%@_MNEV5w(AmvZLVPndHiuA;;6dZEA<OCuS1Tk<3h~WT
z`B77){08~F@dx28&g728DlvQs2`{?PW7YY3G<-Kxqe~?x{P&liTcIUoT_5|QB}e--
zMpftyKfUje;AL%1nnxDVAULUU6Yb7;L&hwq4=~#r3LVzU*T>E?B}KFII(&qHKt)gS
z@?5?9l@KZ=lMmGvcr=M-+a5>=qYBWu_}mr$zd`LJ*E|F>>430u=4ZSNF6ls;ibm<d
z{;Igscwyl#puggKxi{|nH(l|7sG0$GAHt<Mgys_=P=FGZb5gTwGHJs_#Gip!3UYy6
z5hohMM>~EJtep1Baj*dbQM_Oz0SJInE2OL-3WxZdxBD@rh;c*KjjL2FNUCk2GTB`%
z5HhT-c32ianN{HsWPT{3s@7i`QBs|PQqtEnjqj#fBWO2jlaELk%vW30*2Lz=_VZ+`
zEmLf0QmLeSq4*y~4YGfQV4$iBz{Nc<*h3^JYgtNmKq8i_?R#P>UM?>He;31o`oa~0
zr6jt^;ahceJ};8hIhe<KF-i^G3fERj04^fH#rn<UK&`x{Ntimn&?U*-O<2U1MEgHJ
z#94kfhaCq`Ru@){XDiyi$Uh<I6;cA>di$14q}=jOADcee#l2rEXo_$w<+GgU0aJgj
zB|vjnW5<k+3Vykt9HV?q_{-1gWmG!}`*5=dRGrc=W8C5TZ&vg7YEKS}G!>b6Z$2e%
zguWMJ0i6hNJ>FW8HO>IY$mI;BFka!;j5oTYrAb?PCB|0er!%4aIA7bM#y-2)T4Q0J
z*h@jq&|>m^yc;{qdDqY_-KDx{ZJLfH!G?MGSg{I7n+VM+)}aiL$L*g$X!NM*QobGG
zedR-bMzn<aCtOt61;tmyYhtSkq9G<|S^|z;`v)KgF#a|O;{XD~K`>Y9{pau{I9!G?
z0oIuWmA^yKGjQGrpf#@9+Zh{uW~*%3z3R7Clzw`mhS&6Ov?*>6v%Tj2zSt0@VG12l
z6JAf_Qu?A!8IHA>|D$iV9>C){d@gR~XC3JY`<HtA%~32FdkwK2<#-`mt9BGkn@S*l
zYPe7~s_|SJZD@{{rHmu{pk@$tDn@a~IX0fJoVWV?`;;;McXHB$>ggecsCE9eabv~t
zQ4Z^0tGb0`pvrX{foF{LJSSRl99FvPF;M!T;L@5QJ*ThhK2@xU1z?-}#`8UB{U%c|
z`|n+j{1aYaZBRJCRKBKlZ_rF;a36pDsXrj-fU{kCs68h3cU^{k{$tId_oypi07hSM
zr~l9Tt|D+g(0`>c+0rPXHqWnVU5(WQv30n!^e#_ZB=bw~pI^1*=4kI6>ZlhRB4HvO
z8r~74&)M+DNzouzBG}WTK3r7k6q}18BUZK|lqEPsrt-Ee_FRCO11=DfU_WA$99fn^
z!+Jl87>wZy>r9*AefUghaaKxRZHmD#hV;di59^8g@utMuefo^?Mem$xce>%dsR@h5
z*_+(W<6PF<6j+WddD?`zLN!*jRFY~6J$+KbOxUX{2$>nMHmULfK$wm>y<+z4=WrS#
z63o7I&${t+cc|2vkj-J)@m^IP9mCo0RdN|AAGjlE5D~}D+s7^{A;uR^(<M>|3CX@x
z;chC998xvYRT2Ug#y?Z2iv9kb`GS_8m}NL%AOzPWh;Fs|b04`9xQk;5=92J*JE<W(
zMMO1_SNY;ohkOn^d4jO_|5~Tj+Yok}?G^{U*lJl}Un9<2v_%4X?g^SrhgKt9za1RD
z+V+%~GZ-`^+p*2gDK~hm6~>Gr6T?%)7hU7j)$04Af+1wtp~!FOzMR-YIX?Cd&Ej}J
zG4ePy-D@8<KEb+P8(eo@p$;Ya-ej?q8ZfE!_i-d{mvTrM#iueAJW{k{BpK@rwcUFM
zM};T`@8i=bBuN{|h!Tg29-S*5<YbFnG>(|)3nhjCLY4S=_{S{Xh|8#DBsSGF;$-U;
zhL_^WUFt4S0rnwnvHx@YcK9kg5T<1ahEiT&j`r_kb`Zi7Mg)|q<}%}}?RdJ`X&~zg
zEpOr?#(yh>B&d!k2933t<jf^QQN<5?Re$27S`9J?mHIN-m*WLIv+G1o-5iB95jweu
zvCPHjuwv9a{Wldy_`SUw&!EgHJRI5L2#D1hpydrm!h3Yu#Ofn#NQ*UhV@TfZv?O(N
z(9wFq7E>^k$heqDyr?<q!^wLe$qfmE1_>Z&Pxa+?4I8^UzAHttoiR)_KQrGxmRO1b
z>yMJ>d|rGAjbxO6pN5!93*R)4u#k!dV@Y|6G}gFy?%%}JivaWi0hoyIDaF08Kaa&>
zz&?|AcRLVi<}%yi&aMq&n47<x4+_6k{Q0F(H>i<LJOB8Nr?V~Ju#5K<aOuYHY^-n2
zFu)K2R({{ZBgT^?;;cYn;Zd&m`pOa9{d^S!{;-~b@XN~95Tyrb0VXBns)YhpLo;%Q
zTa9kjk^?hXyAyUiV1y~QTQ**aF5n13!%Xg+*MIEgZMDFVx?iL@l~(EnWw#$Th*)p0
zl*HHGr0JI^QQu`}xrz3wyJNNQ(Ba^I$znVAB9t4G<umg_%wXvo2+9Z}mw%mn)8}0}
zU$LzCbN}4}p!lyW9IrQZ!KVG;0+7<R2g$r!Vt~ef(aUHDboEm|2`fwBqZX%sA9FE+
z3yNEPXXCpvlqBxja!{I|FI!~nxiM_|;+p>bKbCt2rQC{+%@34$d+eq(*kj4^JKONa
zC9IaU!Z^;d{&k|gQxAe}mZ&+YANK~B({w@<XCc@wjd{wlsK~=}+vi8sQrHyncSw`3
zFucV-NYcw#htaDb7bS|{7F)C#+-+4=y9*J;j*@hG;iXr}mr~L^OU5gbZA#(lBt8<L
zQ6*j*H&l=)F`{yOM9+~k$V(MW;!koTa&aJ%)v*kY0hA6JI<5g-fFAV~gmM5!0KEO?
z;t2ngG{D>eqagkK4b%D`S&;>WC4fQ&r4o*Q>)O9^fQKud;I+WB>zLO%doA0-ORZWa
zkbh-NoEpxrIa;+dc~3%+{+{>jH}Kz0;0*1To~JrX9Z!{>;_bj6e~a8l=`eGKM(BnW
zghaKmvn+0RT~M|`$FJ%lNp&JmOPE+PBmL^AOr08?GaCVHg4a}x)(TU+<VBlQ>Ni-F
zjE3Wf+Qge`uA=2bvgN}oygcAV^mUoCSdE-x5}kC}Q-8}bm0KN4>wtlG>_h5tFyJd-
zbI<-8>f`cmy(_@eW(E4Rxczzk%22)S?b2L0NTH;`YhLj-zB=Ts@b%A)kl%p_CS7#R
zIIu+geZ6kr_4d7%08|YKHmcztvG?0&l|b2_VmnqSq(YXgagk)I<#0?>%QV9Z><2Jk
zMNZZ8$5m*t7k>00e}?Ezib$;FHSEohY<*TnWNC_pJ@@;_-%tCS4JHf!FB=T4KROU9
zuMs>>i_w6gv|4r(DLWjxE?m5A)#|rnKRzQG$qYtAKvG=dJJ;gB0E&b6T$sQMHEC>>
z{A=NW%bN)e0VoiHm!^w<&h$PWJXw!;d+`c{i0kf@*M=Cuy{i3u1nbJ5-|)If5SAqA
z+I&EN;Q#k0sW`sTp!_eLepMP|e;6v*gqZ%n7%B+wpgi5Iy#~TwuNyr9Y&UDtB<Ci$
z2A?OCsG!#%qQt6QWjV5_cpH{07*C`6#WA>eO}z^9gA#pjSQ=%pmpUYRZkaFPc&tlb
z;cz<Naj8*bDAgV8h==zy++FMHc=#s@$3>mMcY)&%-x>M1_0KuPsX_YErzJa7zPV)E
zO`hrgbE3Ay<frqX7t5auvgL;rr-s-al(!Xch859ccVxj&K%4%zlN-SRB{n07INc^-
zcZ#~$o+h%52b_8e6${B4dV$)2(vjVRxLb_f+OAa1dcQ{#Snd^-;0b*Om2^q&+Tn=v
zO2s-I2T@S^Wm96KgpljgQrFu1jTMXFM9fx;RkbR7W2VeIFbP`y$IV)r9&%A?v{!Nh
zW!m<Q=nmo%`g+Gg`>a5;nG^-$aYg3Szdjl#$L=^e{uKDVW%G}{!pGSw^vl^}ManDP
z{8z<dT!M63iT<1ukXQFdGx|3cV<%Q=v3viV5=8)}ToI4=z+P28P{HkdbHcUA+^>&7
z%ITw8V)}{@RuEDA1NLQ82Ks?L?R2SAu=pD>$gU@fO}x^0+mOroEyKlP__DEs5hVpN
z_oAE;=AhJJJl>f{;S_q5;y%O`Uj<>&;;3iM)VoQ2q}^B9QmWZzxjp4bzW`h@HTBlt
z=pR=_3&7RReq>+1*YuCwY469CH@KD%4F%~FFqs&yn8lef>&h-5(!cZ5Ra!`l7NZ8z
zKcIAMV9yU1-IZv9vg`NwrY}7X8VL3);%%6Di?&7r&fr|G*989zKyq60UsD`R+#%V`
z1J4|ja{M;<5<uBHQ~h8@{6UJQf;7p64fyUDEHg(uKVjOdl+V_P=-9t+%#gKd6)zvX
zI}ocy+~%caD%S`FVu}=Wu)6k<-(tIQM#K*<)pC1P?f)T^?l__Rv@&9evx?wliJ#({
z&)Z6Un_uljZpB}1P92|_X8W%oaaV>QRqI0Pxa;MTiI@JrZD}1YBlL--lf`3(p5mW+
z;Hjs;zTrJOx6^2nk|4wxF=)tmToNg}LuQojz0-BsK!LPW*a}_mvFxv1s?A?$ZOG-S
zc9)&SpvyH(qc~R)o5ZF77t!jXf~FAKsm))x7p<-4&%DZ2VtP<aX;rFY=3jPpAQ8L{
z7^VXz0LdF;>t?zdr5raH;^i?$+?#9hIKrkk)WKh=Z?G^^f6mv?`mYKOj;Q<tR7Npd
z3chom|41Jy2?gxUm0o{$UojV+yYEXrZ~gKM`fGnnfd-ZtmIjy3Y+~gJxbSgOX<4hh
z9HPeFQAzmr?+{Dm!|m&cZn3C2+Iw=4Wv)2Iip~->1mQ=ziSrZ5icg7)ZqPfz>F7gc
zGx>;!>7_kAhJP{2<haoWQ%d(fy7=CF+%=eqw^CW-^B2=61f8J~IiNDIot)!lX+$Zi
znz6fhT@;DijDRVf1xQoQ7#hT}$F|KM{rR2KUclwFd((UM|Hg-L5b$m#sNe36?pGU*
zUnw|>eD=Y*^pDpBwV*2l7vjH&eU{V*f{x<EC1l?ozvgEZ>jnu~6r<+?p7@GbHbKSt
zM&j{4eMPICbIbJU?P+16J0*nlzX-4$V(enbRgMx%N18i($rX}4H*(`L&~XRQ^QHGw
z?3lO+&LP)f!RklbD@X5<vn@Aeu9@RS7F+j_7)_`lHj~WS$}#)Wi=Z^z<la$01*jbN
z7v|gk+&eD71!i7vd-co96}Nv|K@23I^6J3-6q~nMKqDPN_ptKQKkAly0{5`j{E?&f
zO~scJNTcgmjX0~$5`GiGRtS$?KU={KglIR(Azop0I-YNcc&H6=^EMPJl(VS*B}!XB
zL2M*Vgr!dZ>A6no^{PjaZWei;=}d|Zc&}5Ow^LQtx`{0blgcK9Gps^$Q(zBAt9K43
zkb|9^#Q6yp`NYl4QR}KyN}ER0DFx!ei*i--V>BeJrj+MFH|pnivdnw0{avqsgmzoi
z53G1eZo8?3tAdU~+I7TQX<wgw`W8{7bClziUQC6n1o}$PTK;X?;KY>-Hu`F8Z$#;H
z=y<EvfA9M)Ej}E<DAKowVTL&KxV|r;dUUXSLGDg_z`xPb=rHuX+GH`0zDV<@xPpdH
z@VmqWad!Y-T3{BC0-bap)AcU(_$fCkcaw!sv5yn)T&2$e%qt@1{anBB5a%cli9Cp8
zQ|MX+DAHvu=^jKi5*PlzAuW~XRi7Fx(PEnA%e{6zVfR#SJn0so%Hm?fCM6_ZVHVwi
zIxW+L#H7b2tgEJ<LXrKC{r1VRbRq>;;z@~FybN$rg7c`sPvxyfFeTH4qqaBFy<2Sk
zI~L|(P<Z)AX6K>gH2)+@cAl!OO#jMC7p}uj{_4g|8s&sed+{Y3OL=VSzuAQ>0Sj7L
zK|TJ#u9rc2t3ZNsABTbRwiz802|Zo{9y0;um!BHzA@sSj``cWLRQyyGvXDWCJpsx~
zVmZy5D)QKZAr(dhWTL2M^v3H|UCyBJQcsW>jPgsTL=-chIx|<qr)Q6^rD!o0s{F>K
zrO9jYB}k#A7GdKrY>wP+9>T4dD(eL>>)l0PcXp=09;6Dh7C+=ZWjwj`K*2|_(?uG0
zN{71@c9j!<T!#llo(#X032*_vd>h-)``=?`RGw-s{E;S?e`HCUb>$yNq@D|}(ctpe
z<eiyw(aiORw%i7#6|=5WKVb~$CNOUk&6BaeIapc=VGD6-A`-GUJ0%)VReEr;Hb@O!
zqaK9otIQI9B-)QM4Jam7l9!ZHX&SG#I+x0nTuLuFA+$V!ueEyvM6@<6JXvqqV`%!j
zlM+S+)E$ehyy;Ajj3}{-D~XA0__AwoP5<$_qu!pw7s%MOCbc>Zoie2XgCs-*VG|)9
zKn_7LuQvw%CnmDJ#g*&J1>%CQ8|?xA`k&u<p)#C6Z;D^FXX)Qc?-kH}UT28D^XX)k
z0(uLADXJb`BpD6-@{1&vUioR6e-u5tBAzJm@%A~b$cxI%c1f4>eU%w*?t7$S5Kt_K
zjIdJ0ek#2w=UiB)ag$(+pkqPd+d*SFDXnX?AGLbkA$PU<pNl(1dEpc4un0y<d6g@A
z=V`8-gZVFG<{sthn6GjIBUhs_^!{##!<K=?wwB0VP6TWeSNmWL9lQ89A5Ul(SNWZv
zF!1udf3@%@T*av|6U<{#lS%oz;J-(TfZ+3al9PP3e!*pMiJ%gW_<nZwHH)N$MI-f0
z=!6ud)k#O)$tXkOf;au4N<bc(Pe@=#`@SuzV5vAxGUj4Jeu3<Iag=bVusnYzx@v%v
z9YW0xLvi22?_`8dU;1^h`6D=eZ~dV4o9NF+v+4I0b6(eF$r~SRy#C>1`t#UGO8nw~
zXp?b(HtEKpO_)8-@fWai0JMp}2l1`sz{SN2xm|hn>Z7>{gH?Q);|1g_<IXvLH#wK;
zOicJ*Jl7j1SzJfRPrjCCd8tEa@KnC009M72>jpT~H4BAU!c*WYTHabZ`bbrTUtAO`
zqPNnCinW?el!851gB;LF;pLnDEiy8rM8iV1aFroPy%$v7|0DGO!ZZHo!;BAhm&cF!
z0`Z9|;wzoVHW>^qJ>G~FHHdiK;?;iMK*5h-IaY~%PW8<o^a5!$-xg}-nW#^ql>27w
z-P$QNs>_^y{b^qE<Prv$sRvRe_ztnMoTH-}ky2mjuzh#6)u(c=wHNvrn7kTKa+}~Q
zmz=KgS%yOtSf+T|Y!DF|RID_!VZ@12m3QHwlrs##d@KLCEdyqPW?Vf3`M8<qKXd(W
zQ22lUSPg^#Dli8Jor_BV_0Fl@elAgyGD(`BMofhnKS{nnl1*8Ga;^6+)34CoG2Ny2
zTx@x9J_`bgn8Zj}Oj}68`Y7LEAzMLRY}|y!jrh{!8Z`?XWPwr=j{>b?(94nc8n61G
zKb=)Z%RzIq=k7qoAiImL&Oub{6h|nDV-0<Q`BM=KaFV-HO}|f~8_d~y@;@hm{&Nxz
zk7oG)&q>U5p^rr@p7rtW%;SG(r@n0VgD*Zx3{MG<gSH{$0dwICQ1AvpQOdOZvLR7l
zh|H*FWFwX(SM4JR*>w-X0;l80t}87F6}#=0QcGV?tuf5sTaUs^6hU!#80k7u^nbaL
z9wrG_XE1QK%!SW1x|Zs{=h5e;oaT)$R>&No6S3?TvET*Rr3+VdPx5|?)~3LI!hd?H
zjDoJtffCBO(Di5kD&YUyNA5c~PPX7o87vr$a*ED>8~T|TK`T744-1&oQ-bbw9nDkx
z{jp@XkCaFxt-aJsuGjcZgYd6IfLJU2=I%gpal$xD1VgdrYNPzK<8^Wfg}iDLOK!QE
z)f>h|=)sHUiiS3C=y%J47DJTfR4u;mOri`vQ_J)BDRM-v3yFmG`SJOwdu5o_MggPA
zs~A1!{of1VC&&MU^9|x!pA!FbjDP*zfA4P|T1EtkBgW9%-v^OnJ0VOWRQh2w4>HO(
z*K-{f$&@(BN0pRLxma?q)!RV-U<`f-m8f<Q&4qriQq`9<$ydpnw38ut!4XR5W#mJf
zuX;y>Dg5nq(RrgQ*}EhfgHdC`HrSd-78K!=Fd7Y(eSWZ<Lo5`EO+hB=7faSnU;a|9
z5gV`_r7ZRGA3X`AYyI}$w=<yqN4gB7n*XM|zhPMaqv!vsix~^FmKu=|<2D>e456C*
zE~3Q2`%;R78E;TMBP88c`+T=tl?wlyYTWIoggZp@Y_Rm=>_gH;K`r9T-A*~OsusxI
zY7Rq%uM@&0RxTRrph3=PnoV@Y;z(kK=y!vBJztaAFo+1(Adlxy0S)q6&>#<7&((Zd
zKgv-7yE)w#*0-&-?Lh@_1h?r#a3#7YBl$o64L)(-I!0Cb2dGQ&lNvVs?SD%pv>*Q@
zj&%5;$kp<2=%=IuS^68Q285@%U*L=l<E1US|APpwa^e)}OE-r|^nX2WSP3(WAO1Ez
zEIjS0O7^Z}nAAZbeF0B)Se>S#{4$3JWfeXj3c<3>@giug^e0`ppyGt>`I1c|*PDx8
zF$re8q^T)lEE6{}<h`C6*n6jF@>56hfFe(G3YpaIElo6KX{kIZad+oJfhLAk$#=_Y
z-ztL^|Mw}i^>ckc{%49wHU2pzt;wY?f1KO<aAlB9b^W@`!j>?#nKJs^w9QB6S?lSo
z#)XbEipn>c6BTbR{vS?i_5lX!HBXRVL6}~t%uVhG<<u(716HEo4(9Qj_2@oji(3@C
z+}#UI2DiPd1rqR)UPk)fg=!X#5*CkITCUp_ML}VyUW2J)PSRztn=N+|CsEl%cw+>3
zuu@IAkL57V_i+B2^5);oXc5qi?hU)yJ$XaU@x8`fs-(GN2}NOQgu8F3W`xeW;D5|t
z0BP&e{|G4iKLU!%Z5H^WeYup7`suGf7tlc?luY-kG)w}H1|1i+2HL1zS_jB5E8Vsd
zcsNm#y6{>6EULiRu%wBuoRj{a4$)uxKdFl|{hEDaZ7>YjK3mSF7czOe0G%Ho%*H>(
zR~?iE!%t7)1?CEa3&gMzQD%aDtz=9wi<e`%Eb(T6rWQ*bD%+JNCNo!lmHGqfbuYYN
zf+T)Y=)Lissys@890bub{NyWrF+{rAUXIBIGSYtgw0)wX6qK2~Y#l_$XsG-ur*o!^
z>~XilEpESA&`yq0r-`M*6<oXd#$Nqfpu)vs;T*XEAVi}jdcoxRr>g9%YC9SdFG<i>
z!SzFIB2#j2Vf%Sx@P#iTe$qojdtW6Uj%6<NMv9RnKmTl6vhsiO9g{r2J>Sw}o2b4;
zzo;mD2~}>gM!fDG5@1y-JX7JEs~uD@2{okZL{h@~l);15r~qXvC#<1nvxJ49Ce@tk
z@G@mwzwC|$I>Eq{#rRIO3PMEFG>e-Obdle`CW=UKm0;g5<a)KrNjk8SE7~l>n*1bJ
zr%9PsTU|!$atBvf$&u1;<N8G@U6J6=>urbMWd_EzoD71o4myd9ub<{2XKiv=eK|*z
z=i7#|!^Sw;tzEts?*B5ItP-(z*R0+ncqzB}-0#1qhJdB>Iir;}ZB?GwK@YGfnztz1
ziU#VnF)7k=B6>sJMUUN$xp9bOVS9`(a!%poHj97IFP#Mp)i24@pts$#3FV(3ddT{G
zyV$v9$z~<^@K+IEjhm%)kc*E%r8!qu1O=HxUjhY~IG}2n_0%zI^sswL=~%w`qu8*Z
z6YLSoIJmdi9erWa#qCb3!+&D@Pe4tKGb?`VojrL`Ebq}wcq@2zpV;(Sp-=G!UHb|1
z`#A(}_zMdnY5UdEeX6vr9=w0)G}z-G603YNo)`Ck)S>o$gjbdEO9QZRTXVITD3Znd
zdRQK_;cIz;&~uxmRI6G!$Ak1^Va+Kt*CSrfmPA20nO5<i?Y;=pMK>>(>K6*obU3sC
zXK+`l`X5~jq;B2#?@Jmm09{*O2b7vVGnZW-b`Rv4w&6Qg2V81y$qD{Eybw;@*QcSx
zO8gH(?JhIw;De3Q(}%x~c+_w^=dZ^E+!|)boBjf}Y7F+vB=hN8b7f5hPi-xzodgP!
zU#Rt;{8e*ar!y<Lg6yIHZbX&m`5Pvwj?284!>Rf<tz=kQ9G_`ron#;1n61ZVgp}W=
z)w=Zg=&i5nNcOd%@ZUHmwzI=N7pwuL?0ks0$9#L)JwQZFHlldq8u@b;ECuW&;d=^;
zpk&FI2;V*ceVxb`H=ak}1_sJ3jF0A)=EJDoM%lB-r0~xM{rF@z@aRWHG*~?=zP~k0
z`GLa<*>is3#q;c|&VJ+n8Qb4B^DgUeJJIuh@%5HbQN8;gsDy+x2&ll&ptMLcgdkml
zlyrBCI76p^bT>!|NO#B3p@0ZFgmg#@4BgD#bI$Ml?>XzPb>F&tp|kh1pYQjRpo!Qa
z@k2A`yv6_#Y&_w)Rm{KUQ3ANuQ*l33hbg*)M8FucOb;id7cia3hlt$|DXp?dH6>11
z>NHG;^-P^4tpy{UKY#fopMg>KhOHf&g>G8<Sz6VO=0~7_W5^Ew>6N0WMnK?K80W(l
zOrfvIJ(}&)I_(@Kgqp+U54+@4aFpH!{QDC319hJN*K;F8Kew;gaf?{LNW^JxBl)c6
zjmvTNDn)<xCTICmP<5Yr&%8IP-DQ{rE|<x2>VpUUzd2HQC%`<OIX^BHe_;a%k3L1I
zp35;%k3s-w?#!A#`=8F}g|p{E3OvFvRfK6p1bp4{0v?FR!Aq=8BOjlKXMyhCLmbVW
zQR08v-;-SLyE%zC#A$^3cN*eFyDHEK%NxR-gn}-R#k_uH(+EmV-Oy@3gk!O10@*N&
z-xZQaFFPm(;ik<@<I-X-vX%X=7l+2_wO06oJJ1ZAJOxca2RrQnaeh#_w;+@-x@A}X
z>}kmfx_T=JRcf#S#Ie1E0EDjc3?Q=zF8ImhcijY8hp9|X6|0C>(h2_`IrFcXj5L@L
z@+}&|<uLFHn@C3F-s+PGJ?T>Zfgg-K<AK9ZLf4fd`owFl*0(<s&M$P78_%Tq&7^Uk
zkd2bb>{!SGmtD@V;`_M+drgfTJ=|xk>W_08bC~kLVoFcxHpX)4(1gAz3(*Xjow7oT
zYPc94BI;p5TWEymG`&o2ax)z01{<0WXG%8CB>4B0-a4Lj-sFxAk+lV^sw8NO3R=F=
ziw06}d?rPzTwv%|Fvvy$xay4)mVbPhT;|)1uGTx5)7-z2@Y|(aDIN*n*4w)@eiQ_V
z#6|ajFvM9r=$WPYH=rsW*lHmH!6sYN)(JP$Z#0O2FbqriBXq}eFYuRVMPnY9qwh_T
z^HBOQ6A6P3_L|~RdionV$bRu)M+D)X@Kf6Qt!?uO1aTCF0rwk&bIH@8XaS2%*H3?i
zyAER#ZKDBkpcq=kn=^HQXkL<xrV-|;k718NSM}00_35fWmj5p1lntI2F$WJx-SkO?
z0>!)?dikn^{5?SQ+D|^mUlm3~fZ?&XPS?i&z-Ni`bT{B`D_}JDXJVql|IR5p@}E<d
zp9?*KFoO0Dhh@nO^WD~nMT1|zB9IxC%Z!h$#5xE>iwS`FKS(l^WQO9y%Z*j3$*!G(
zD3FY({b}FejNZvKu2X6Cx*jXHMwm`nlV$-ZTui};F28W6qBlKdJLyXjDR=4E%OPjK
zDNGE%)xTJ5-b{|eLX3c*;8fHq3%uk0)KsWKa@r9OM3D`!Ep`?^nB?{J;az}kcdlj2
z%6t>(lzisn1yR}lnRn-zb%amB78=Fd)uIN9MEA2~IBKtU%myRn$AG4jUn_SNEwBGL
zlvmb?LFrX<J5GNYnqQr4&$2wZ-r+H9s>tD>;Gs75)Xcik@Li>#Yntdgjv!-Jz8&lV
z%qkCg0E4+u(^Tl(Pl%4oxwY<+?L;o!e#h~++l0iqVx?NT1riV>hM`k8ojo~N{ZZFT
zX^U)Mi%RPBO#K=KM2o>;=**Np#&BBl6P(B+e9@A3o8Lx^<lm|PM|%8A0^8Gdni3z>
zo^#TQ?5g}wby+kHSx)6cEM_@WZBe%2V5TlV2SEOw;LIrINjx<3i`_Y1PB5}Yk2<99
zQx``hLw7C_t^qWF@nH%|BtK?L_x>et7l3A|xB~o7KDnWiv%b#77SE~2JZkg1_=~ZH
z6Tt!=x?BGeKwNL?Q#H5a7zUcJmo8fFvYIMV%K62vg8ZDIY{>qj1dxY?0eQt@%~C0N
zAR|+Bg}BNo1gHCOkfxrR2fi8xeX?oF0y#2MVHYRK=T|{47soi{tyFj4UDdcT9xrhB
zE=0KgU=k2xZ@`X2TfA2e0#H6%0_(c}%@ev|SK!9Yk`e#8=oL20$qSKFR~h|WVZBmc
zoGoC4ITQJ#0t6)$BwbyV7^b>}>-jkqFdIJ4#J~L8Bg>FXd^xR`Nvhd(V(8&`(GOIv
z3aMtoigH*;(3Pw4B|xAhmQ*VygS^V}d<){mE25wWcz9kBoUcsnPuXp(p+za-OM{ib
z8gRSS`ff{Wf2s)2j;xTA6wJVn_P05Y_iss_w<3E9cpFCE1UHjp^w%7ryR=%bEB$v3
zmqt4z^o>^oF8b(u(Jz5OyV@0X_k|oBDE^yY*HPwsc9=2Y`-RRuq`7QQ_PG!Pq(;FR
z$bWlC-Co8Q*_$Pl!cyI-e3kqMa7G}m6#YM8xPo*t0iG*G>ULX0u+`+gEE{{J^xr^+
zBok&`1$VXto|fg{nlER0x>8`ze^00Sx4@=fT22NQnzcWuImVs%cHeVgYG5VB7DVLQ
zEx_z~@Po6Wdjwf?80+&9)Oi(}mJ8tn`;Be1o`T=~K~s9WgHV&4LeJo#omcHmO$<<E
zbSvOSvSlG@8eX2Qd|?$5B-{I2qpQKn`)Z-2_j^El10SaR62SbLxBCn1b^kofw=EBk
zA1b+JHS*J#Ph(T2(;s4<AqO|XEuxTR8h&5D6FhQig}o5KCMp6N*|=>QatL^%eb3vM
zoP|qZ1dkQy{XDm$7u<*F&;k+OV#->eeByTh!wMig)YDYU2V|ka<viBnX@KCrjp0Q3
z|GkX?jdYB3Ic^-YMcC{iDGaL%<6otl9ruWjSeOiZD5T@~cOuIJB@4h|Zn&jUqzZ8Y
zr;$nFRWomlJv~2<VdTLQ<t_Lrc7|;nb>Cy8?2x^WZu2B$jElWTE@Kd)BWXV>=vHj0
z+kW+{Y7||$=~POzT8FQ=Ll3}xc6tmPaSc~Lr3Md|-P?glPZ2%op<f-z3mAzF1<{%3
z4G?cUZsa`&9sp)3DV#-c0^hTs@=37L`{LuxELR&7aPm088xPrB^~`3-;p#on0srnJ
zJZd^JusRT|=(ZZk{$;aXAc+xO>hAJ51Byt|5xNgFyBP*LS+Wk8sgZ&qSZWkXO|BcO
z9gNui#w+cbF44LF;L4LAV6cDB8mv(5kJa0i2mEmR-6riAyt(HF>7KV5eZ9hSiA~p4
zQX)=W^BCq=e!^iN^tLq%jZbG1f0StCeWRF7WL0_M66YheH!J*MS8!D3sq2l$l5X{I
zp?;xd-`bO?O`~S6be%;1+be4VcT)fI1bKYWi8D~CL~zh?tiDeJpE-Fc))@Wfy#s#K
zo+CHH!1!viI{+n9^#cP7$J4H)^z1^?O+xSZ&)2kN&Gqx&7P-{6`I)urP@>_VylHv_
zJR43=yGxe{W;v<r{tN;Ehlk&yFuU3HjNIiQK*Yyr+3y@w*QpRb%6qDMZaP3%JxMLs
z-?S%dDC?kO>PyAkEA*P~FHTEAyo9jK56Ejm`DlD=kurk*4$yJ@>d;v*u2yZ*S%)eu
z*nfFJq~Bntsq@>t5j!r<Z_m=%#2<Qhxz$&kW|H~c)TwFK=CKXL){IOl#kpzr%lB-e
zT=D&G)SWiBO2FlqDY5KF!jV;o$Uu|=hhrk}pF!R4cBZBKLx4mPQPJH_l!!btUUz-^
z8+=YSv#25<gIw-%*6E-Qd$s<58jydDPLt0uZK6^$G_yto{wO|&;j&9F1U>rRa<6EF
zIw&}jwFToZg`R{OX<f~z5cVqdYmFq$b+T{`GsJ*Z>L3q7Gq!3DP%1%-%BqDd8~kB*
zolFdPwT0@>?};fGgv<mgy-*C)%9Yu}W7x3-WJ()Tn&Qtn>tjGdWl`~|&O4-OuhRsa
zpXPc0HMf)o3!QTtFg;M?G3YO~CA&_yj2*$lOaUuSW0H^z(_c-7_X77qX1kn{LQ}fg
zP$7;Ni_?rd@K8A3J~$pGKwFkDbH%h~UAu2FT{DuCfRaw;4#3p7@o64fZC2n0>}?s6
z6e1mT14SP=J}mit$dVX8zFHuItpTzb60xDO+gL>^C}L&Q^X}#{<Jm&(e%JN>tVM8R
zj)M2I%;li)Smk}aj6kbulMylnTC#D8b0gOJ_<=`Fr$EfXPc>m81-kdzaF=$(++R2D
zyFFVe$D}R%e=og3RWX5g!chiBZ)ql<G+n<mpVmY)SNfd)(OH3+-udXC*f>YoVTC(U
zeT<!xpqZ;s*nhy8%EjTE&LuzxIFZc@V}u@X2{CcmxlI7(7d>khIIsU<lzAVK@HeWe
zmfw)W0qS?xwHv91Hs|SR-wjQiS-cOu-^q`AqA8Hvnns#a(nYHwpA{fNafQ$K0y%X{
z6GgnRRQp;d=DZpxYvy!w<+q#sJmtq3HB8FTg5p%@`2%QFRr)QiV4cL+w)?X^K|W?w
zN9dcNqWjA1W%zwjH?!Kf`?jIaj|qEdCeSZrQEu4ZVd7IAh0~&MLf%kI8y}fmN~DGD
zt_LxY^OEuBU?Njc#bz$_G`2n_;R$&*ZthF@m?twIZ@w2$5~PuKwstojCB*r%>;amC
zw^UpEY27#0P2cloG#5-{#aubw42#l=dpUW5-VV#ORJ9y@M1Fsv-s)pIPVuN7d5WG(
zW<?;^IoF2cbiD_&M8XU4&bUgM2M;YbO-vB(F7V(jz&k|<jN|kU2o~okuhyoP@Xc0v
z86T%5{e6R~u;m}a=X$qgZ0lmn!r{BphJvr-4urM>Yr&W6Nv&T#^|Bt!(TjQ7$v2q=
zt1q!u#r!r6kZ2cbl^2(7k%#XKj9*$2w8Vpf#!>^jH>t~q$fcohV}*TsWLr>$Q6m!=
zdGdWFR_XW00TpWPKl|5(F_vdw?he;mHI?_dxu?y1R+F2vn!K>W;dgyhES7`OBUk6h
zJ`&rd6!fArud4&%<9fN8cekBahoa>u>TZSJUiIKgl3wJk%>8h^S9H(c=aODU341H&
zgUKT>#O<uztb6Dz;4qRs7w_AB1}2Cb-O=^%S@ke=7cy*;3>M<3+?{hXsGDhPo<+5r
zLJ@rLcW26s8nSc?C3CAbA$|@*#}#1}QGdq`8i{`Vf?q2IIb``D{ax${O;3P?;gp+H
zzpFN@{kpKgt+VB>>AvZO>+b+uptCj6ajfg}t2{^$hqwQD8@nI*&C+X<{QUsbd4R3^
zwfp2t+%SnVICIv-)O#hmP5O-XHUE);|5?Zx=mF_fI+<X4gXNEh>i2MWZ&VvzUNNwc
zkiG0nZCq$*L}He*(`}DYd2=VJai68g)6<PkdqvwIl*4^b<KoYr)%#g`V2K0b!E$QC
z-^`xbA4N&SMu&Jp8bV6eA_8ZmABExs(h)r{-Y%^gJ-$-Q-6_zxE-F?59X3?|WjI=X
z&e$2``s{4N=B0GhqegNtgw%RTn++fG)HQh>7J9NJg}U4`Pw=xwA(rkczc#CCW&d<G
zNO#-FPsAtU<iex%{+lnvnLM8x7KzPf%nEN{NmMqGDSboK8Wg^d{#A?dAMF&cCxd(C
zfVe`}b~wTk^5_>(eFQDF$)XxeVxMU`Ps+PtiKZ2;i@|5><ldYyn;eu*RBX0u-K!g`
z8>xI%>6Eyulw|ib=>BiFD;&JvJ%HrZw60HvAnEmg3^}{RXwX0zJI+|+M)gr0!smK+
z*J(WGs2MpOlLmY-KZxB$s|M^`EZz6inzb70>>a0nl4dQ1JU;GQZJy(4DY|O>@;2Mz
zur*QoOJH`{{Sc3;Sw_pf_JghzRqJyubQ!nTK?;WlNq$%wunl2fef$vS!EZg9d=Dn1
z(L!<^fI2`~6C6M{M%O^`*D~DxPxHcb_ist}>w#3@+8O}VDHS1j6O0=*!xcY!e^-do
zfP%}YUW7TnI{v;G`Tmb7Ip5vGLM^`?O{o;`!Hl_5AvHq?N6q<}z43kuI!HjM3dQ&o
zNyu_Ub{@TSch>y#>#QFu3oSlDQ5^UvB5}6!65TqP@IX03l&tE$;t;R|+3PthecGH`
zdt8|<y>4C}f?j7St1X*kq<Jale@z739q+06a*KRvD|#Ea-aa5xli~pQeEb8Pvr<;M
zMCt<#w#7b6u2V<wmpu$q@kT5G5)W$$)vqM+?vuNOD_5J>gpx)f;E>u=c})v^5cTbn
zZJ&75W>}wm9<5}}S}g`IE#7*pIJ;?Df`o3xw2Cdlr)FbNJap|0L@iI~=SuKvwpxOw
z+4oY-Y&UHJblw@L-U$>;One7Wt0(C86DLyMeuNd{duf{ni)ImX$2Td*bX~w#U-_$j
z_r<;p%TxAjg-z897v;O#C6vVS<AOwKmf+!!eBE_@#Nt-nN~t%xM6i%0HRz1h9$lyE
zp(*OrwD`b!QHZ-dX<73C73Kz<{O+81;iFL01sXZKCJ{XvW<J)g1K%BA-ZFkmk`ItY
zxXuG94djC{G<st6P((54Ed%>YnXfFQu>f@7nylS|`fMULq^)Ha(*f8SW{nJ6YfG+j
zz5WAh1Ixdeu?(>CyKB=PS3r8we|vX>#@{{+rIya*UtN9hl*l&7mz!a(t^bF@Q@sqC
z(T77gO+c5TpXL)|PrPT%+v!{znjc>S;@N>rjz*61KA~i~qhwd{0g6hxmKo;pdl*X6
z_Q=sFIo5m6WJ|+8CsCKEUUmdClaA=ojb_ZyQf0s?AOs`DP_juT`ATe(#jpMI_gyiB
zl#k4qVi1byML?bBGOPwPGXEtEpKvj@rWaetRZJNo&gAMtbv0=C%xg7KNqP7w`C^5o
zMN77}ipw%WvVDRpX(U<!JK6!CdPl(_F4AXl{pUKVH!wcNVMS3wwVZuA-UeN3<KJ3q
z4yv!~Mg=()nFD3aR7#K+@>;PiFsm07Z^-c&RG+l|Lorr}qc<#0GBEK~FA3&+L($QN
zAa%H~4LHXp{2bLcYyQ`(u<vKH5t)hiYBge19H=mpxkqtbwhSfsu)INKc8mvt63{H<
zAyGUQ5kLq?l+XOhcN-OCJP+f!c!!`$9#>FmorllZyqwqx>71MvO+1(fh=Hk4rT1dg
zfC`0njLC#Y;GD#SwLg4urDme6UCTnU?+4^{x!P<(-~C!7AYkS*gQYbCF~6_>+Hz!f
zciis>Cn{8~sm^~DBD!Rq_({Kp)vgU3Fw?NDT%W3BqE;NOdKP2I0M4Ta;%>nY%?G-^
zSNT@quUJ<TAn(RMbiFzH!tU384@k3{!WUvVEA`u6EAErT-4;_n{aY^4Jl*VkC^2O?
zlbq7W?Ml}CBv^rL$@FR_VOfnVQH{el?IO46cjaltXEC{zM3d|6ch|Sv;m>(#J%7#b
z8EHV$dgcj!v+<I`4}$ng*;RpRuy0u@8MAK7lYCxr0fg<&@7%Jp9Wrt=o0n%=oX0lI
zry8zTRTq7J+CVl%Y^pUr<|uS1toy9xw609V8G+b_5c(`~<lvgw;nrG$+nqd_JrB2>
zZEK?Ir39cO`*-vnDgO`*C39s+@@%br>DEOl1!yuw)>uj3{wf@HJq654^<p==X2K~z
zif}{u(o?YAie`Nxvb$nwtz&6&Mh<Crf(m|~Dt=&wpU-?-xdL`KVfpLkTy)M0Jtvrc
zwkkY**EEZw_9tkOY-#e8v`?yaTgrC~M=3&)lh9XjwRT;22aeyUl^A*rRtZpo-}Lxh
zu9w0j-zEHdSMCJ)`yW~nc-eXYuQn$P>kA*rTa{H95qS3&tZM%!5C0jcMUk0<S;Kn$
z&F>vD1!_8(7A$yt?C3Mu<R(PV+n%b*D4zF|dRcY$rx9;VlFUj$ni53XbWGnF@I*<)
zY}VHComQXldBYQ&Hwu}}3{0R5TN3x4y`gDfrdx#<7|9s<$;F8vNNqntUiqeXe3yW{
zVnH+Unw+o@#upe9hWBIQ!Ra+uKx20j84W~#1GW!ruT?afY%EjG29Ts9PSZ=}5jNLh
zBvo#T$9#Q}q0jP@jAjN#KRCYL;Op`4o{j4`Y~xcSaz`ifGL4}GF}pzgn@hdlC<%a^
zeDkaGE3}<!utzlUXx=)^t3<LGscdJ3%vgMG`dOnHj2oJL@|}9|-l$I@pY#0Bf0qAU
zD)g1g0XS#f-S=y%Yn(GcG>hVS=cp4$^O2j>O7<3Z8p+Lw3b&s*=lD*2N2Ak`=67Q+
zUXaioo1{N!5?$|m3+*Nf{|IAlnpl08=g(4M)WlK|O@W+P)P&}5cBUR%cFc@PWEtCS
zc9Bw_acS2^^xWdRCYKP<K7)lg>mcE5=0W!kaJ;Xxul2wgwIXb_u1`66Ss`wKC9|n5
zi59XP<h+<}gCW5h0rEw{W>Z<e_UXPLjg?CsN|Btu+rE<J^`j7SWsJMeu&?qsjY?%2
zXK~c+C+`Y3>iVEt%8-GHMDG~D>ecNrBJZaGa!Avi;z{rIVVbOm3bC{qvu`r*Gk|rM
z=kny}_j3-h@0HE*;JL?f(3TqRip-dBGyaC7<7YO-FzLYplde|}g8ICd%wVHk)~Y@W
zxNCKwmAPf-cbz}{0SXVd>Fdjyk%Ba%{UlsbO#axKz(B%FgL%e+B=36j>mHx|*6@Nv
z_<a%nw|gPkjLo-@lKQpR1KoG0)w6<Mc5&)%(bdtbMy{0hhEHXww%USUV^%qp_cYq(
zrRbO15CA(Oc3eSMw?nm&BE2YYYH2T~*dqd|t}Jwah)S++&HW#P-Zfkg(mEXT92Mq0
zkP;ST>ASs>^YIfhB3qwdgrabBRc3jDmbAug$qf5|Kx#X}tWMW*w>^`RaD<TSEOF$G
zXtflzi}%G8a@4TCFQGk${Oqxykbk(rms%=rKlX*g3&EA@iG@t(LniT1ISO#i+k@5-
z!@pbCun*5BR5XCACxY}w!yXiLKyyr)$`vS#b4$-p6`BSCZr0z45qS-UQQES|nk7nb
z0@AOX#cL1x>f_8y<kKA#(?N$R5W`rE@L`np-BvMmcoB{zbmPkBn(rLc8n9xb6pHWP
zclgSoxOC~YPgl)^;Qf3VzZDdL1z<D({3f==(m92>H&}47@0F7!PaGouB0z2!HC-Y}
z3Fp!8b#3vlPT9);Z_UfD7#bm<W}=E(HOGJS>0Q!&CE08YBQ}0$sb{p!>14ud^~`X-
zy@rL-MmJmo&^Lpl74jmnX}P=*5sOx${uzJWM^cl;bohSt_8T7_eT5NIQrySMkF~$2
zMfSQ*ZAic_EZMy@x@9jRyo3lRvg>3xJR;wBszFXvX80wjK;UY+{<N6gu;HgTh?$Av
z+DrT~^|4VVA%+^$Z&{>SDMT)C9mBo=qybX(l@VDEiiuQY<EBaR%%Wl|N6f$P7~ykS
zXc793p2wARuaUzhQ+p(%=EiFL_~~8NWjJ41g>QgbiD5P!Z4y;Du7MW!3Z~y#So6Mm
zZI~XBKOB1fT1FDir<tZ~;uD>qWXmR-v1|$4Fne0`J}tifJ7pzv>E%=*LMf_DtdIpX
zrn(SBb6Vjiz70ftAvMD8fb7^oj{UmXkX-OT8ik<u)M%KDGFagNV>cX23f1Yw?zwRU
zNF1WZV=lQ-K7Up*-x81&<v2hruYRs<8&(tI`c`%5l~bxvhx_<LOs+=-*K6c3T_aWW
z-E6Oev|I8-0rX3mU@f@>o20#Ed=>ZHv*lQQ0AL=kAyf#KXE0`0O+sVEHVCG)zSD;P
z@B@NNrs5h>7-p?AuJI`1*W>Lb`t4QT33JG^pO~G_aUxL+M+<`z#IBbmy)p^l_pb;2
zI$0mZ>v6-d;(QlEl#!!p&3J?u_j()pU4XW+%C+PigWlxiq(pn5zGL&|thxI;9w&!w
z5YbflFd>(>AHY#eH1P)3Fq_w=gmG|keHjg5_>CE51C7$Q&3UgYKlQ~WSD3KfPIrG!
znVPCq&-MowWxMnk6|s@2u9M;Vnjq6b+v+90R%oarB~u(gI)&ew2p!$(_-x4FaUU^7
z5*GywKP*H35}AlH@XQ87J<wd3vctbd0Rdz;>be_Ez!Wz<6g`7mV&y($1=9;26@>03
z6Fj~KHsth#L5LT+I{=H9K!6E=Lalt}zk9`)h^dqG3C+pBH6IwY&bKA`QoOon5xsOi
z9kWu!?oXe}WvD+@n=sA*nf7s+I<U-TXP*C{;_-JlMB&OfiOzrbvT{oB+l`#q`P3{n
zNr!|HY2^DvkEk0C_Vo|1j-THMAJSV70NZ7(21>88YSnk*CjwT(dBA!jG<O+YaK}4!
zJnp3t|DY0SYSFA<sT!R?9aG^(e}Ya$Eauuw+#CBKcl&W}NoaHCugW4Sxl?b|B<iwG
zyQ0!#%OB81jQiV`j=?YgsBPIY1TAmK=Gp^5(e3H)pPOLBf8YONG14-~o)BEeJLH39
zY`$$dMT02wRoeavElDU6qm+{v<9&+8Q}O4S+n=M%(!{>1m>6iZC6f(Ny`Pk=d6_Gd
zQZ@kuA8J?WP2=yBzo?TN!ie-GZnRu`lI~e(0E8UMNTeA0BqyYP#(LJ@WI1N9QXsO?
zuaib3_9rC#wZrP`s7FBh-Xk%R1u$v7lxR{cnzEVi_@PY<0{vp-L-q`Y+O)+ew|diU
zY&rk*Kv!3>eb(~8aeZayF*CQRRs}p<6sKD7>%k&+iWz&F4(@?02dGx$LJu|p#k=L6
z&`*?$iz_grM1_`RXM7c_cpg`OK1Oa>JDw|op@X#-$jJzMuy~`&THUEV^d-YBs3@tK
z^bJ0iW1$h46_Wh(jei?AiO37}J2(eEg-nOU0JQnKQ19!S^*=*_F~L7)xw`AN@8wo!
zbIsTK(|tglRCVB2CE&hTX?PCwi$8mNxSSyV(LZF7dh@%t5SRCHO3iX(T3`8(tRa)U
z8)e<aLcP4HR9+S84!W8~i|s_@B<oV*Wt$9@zkqpObBj|9$gKy%IiltntdHkvo!e@J
z>KqSZ9A89m>-D1`@%)o{-^a$;-U)KKsX9+ad1s3Knm*zTc9M-l-L9$u&qHBX0-~96
z)`oQo@XE^b0cD`e-VOU5v~XGS!2#XIIuN5e(NX(){FWM`B<7f1rxEdideKdOlw009
z>@cKj#Tdu@W!AL;r#PbF4b8!O(j;URgAB={34HX6$;#WH0pHN(5(MTmsMilJ<C|hy
zB_+WQqTH=#J#8lkrQB50PV?N{o#*xv!^jR?sFGDtEFMVCFJNg`&Ho!u_7*p2-T?C5
z?2ycyEN6)}av-0~Xv#5fz?3Rz3_mLGX!P~=d706z!SQ9TY=6l5(hrgHT4UftWau<g
zx2uaATI-Yhq%_>kSk3pY%5A=X>0>jsuE{cYCV=OmXLDb6k+_t8lalQZ;7*%S?rh6E
zQB3d@1Jsb{y$~z}VZmste1sRq@y0GdA^lx#=qt}X*RUPWhcfBG*_OrRu)<OlfzRS`
z#OIy3&V#@Pn1pOD9N{vBa*0x5K9@dgb_P)8StBhIQ%FX{k|N7NcZjcJ`NI1LBWX*I
z@L6Q{agG0%?RJp;*Xmm#$P5}=#jp=P^$VRG9H1mU1SoFG$$EFs0m>dWYvmz3dC>N3
zrbdP$U4O0It$;cx+|yEk;zWd{|3^r*w5cOG$1mpTH+^5L)qjw!HZ3$Un^|G`5nx{a
zEsd4&yr&uLx5G5W$U`@39ZvdE1My5l=xV5ry8t&1U)7~WfEUxC-!60+_SaqNTUNwO
zA{cI%6E4kW?SGV%7dr3S-N{FK++-gF?k$i?bg9;I@u6DqIv&jKCx4k<vw`4YYh_VQ
zR|SObBU<T+G*8VIA}aH&(@EG)wzF8`+%#jCVWDk*`cj&S8o7zEvjHiNo=yFTu?mak
zOA{ZCF#>l(2_O~aTT<kiQHseY42nOh+B%#u<X}78USbMSx&-ia2|DlS4~-1R@#MdO
z!Oy!)fIyD>>xZxe&=oI=qV0Ur8Rf&=hAJ_QF>Lte!&@kLb{8NbPWZ>g6LOWx*{h;)
z(`bCjCcaPUbvN0<g$l~c%X40tQZI`ojG$^04(1z`fr=_gXaJ>OsME9Wx*eG>Wq8<A
z^?SXT7KD}dWC%dra9vi7sb&`_mY(YZ2!PU8;$?|PzP@irPn*k^m;;*JO%%N99?<zp
z5B3ny1MCx7lw01nUv#=(86idc%VXS&!#svV`W^yNUUMb|sqQkqq?a;H$1=l&UYz&F
z<2qbC8(xb8lL<LlJ1u?>xvtYSSjx?>HeD}nnK86^h(ziJS>e5iSz;`(){2gOL}b>n
zFz3qdou%kPzokECUG-6Lejhpyp!B~t-;O3KbC7{Q-Q3O9ZVVupo|?FD*)wZ&8|6yi
z#w=S@`^~Mg;&l+EQZ%W}zMGJ!UsWa;+}Qb@Ns<w|CExY#B<fC(sgV>st&S$Inl(Dv
zPIhM5=VPsJEDxYs3F;>L0<gk_cb3#w8>j5m6Qku}%NTE8z2cbBv+KkF0#7#;v=?WA
zG!xm6p}YVQ;K0MoqhC!?j7eeM6>@Mtcq?L>v*eaI%Ym{GI$x{91bKKGU_Rlq==EYp
z;G8$JzVuaU!#U3{8pVX}{Ih1FYB>n+D8=<sP&oZ&wpMM^=pSaj37l>1UKHiIG*EGv
zMj?UyDg1x<*r0poq3+J^PTGEqpr}<?u+><B6~R!TNK_V+a>R$8_4}9GEFVYl<@y<7
z=~4#l%CvsH68KLR0Nqi#T_Um23>zbE2UcTYgA*Ck^jLWo<*%cl1aG;V8oK-tLg|n?
zI;#BOGi?R;69b!NgV6{qOt17-%9iJuvwGt2=||I5bDA`O$SAF02Y%6$E(Mmfi#hlk
z#I@5}F@WawfsImj!DEd3#fL;y`144U@g&@&q7J&jz9;*22Y@E4mrS|Yeuapc&BQ(D
zdls>7I(99Ohz`!<z>(MZN&c!wDv+l$A;QoOgdM$naatlxvMCRC8aR#(8X5%-9s;C^
zbI#Q7MxT{nFZePiq|bb8k@ittIQ|lr(s5Q3<fx9#L==T-5TR*YGsxO<GOY1jp8}iY
zlw^V}J_>H~6w&Vaur7d$g8lyk9p8@MdHwBPiT7f6U!6hGZGOM;X}=MCi3~LBk%=i<
zSnnRiM-)V{LW)3?PC2ow3w?9W`&_V)%BS$ef^7Hz0iey0j22$pO)zw6Eq<k16V<(7
zn*-<xbuvODzBBQeoVR=kfiub|ID9q<#yy8Sy<uJvWpE{sPO0Ar`!o2_=K~dOg2ABu
z^E5<9ye-{sS@E2AS*1ry2>W^T(Okaw4CHN4KWz2UZ|(`I_{7?@4|#KB+ra}9;PcDW
zWz9%ht3PWWrge)4BnR^nQ!4xcS~_9LG$-66zmpibj0<ZA)q3rT96Ras4wzg|heh<&
z8;I6!5#$))ooqOzjy10nE8>HfUP2sR?Jz<n7sE=7>90<9Xk2_wM%}j7cfan){J@Tr
zACEEqtAToVg!6_QR=9AG<TQsW0DdCH>f7?_E-{~%uWrW(5Wh@ayDn|oTWM90HRl-6
zCXNl@7{Cucu+rGS^_Ki9ae)5!@(l*WcI3VV{*~SNUVFa^^~H4OZGg_>J%el~@p$+J
zVx^d=spzYXILL76zZBGf0YL?i1v`$_eUDUhuaDK8&JFJ&b$1{b+vMK+k??>TFbIij
z73Dj|_!(rs!C5KvbnW%+qwk6gp?KbIAZ>L+I1@JyHuUcd%Z`;=N<m$vt+-saYz_vk
zm@>Pd{-~2*))$_uY1yS;X=)fjM8rF!iE;6|K2GyY;+Wz5GpzkQP{8+q8+W)5IP!n=
z7sy33;emSiEczq9CE>cBt|x%uQJIL>>7^Fs57RGl7`c|$So<F!>h#zNZpe32WTBxQ
zl%IwO#A}uF5MEEW=YzpL0qe~4wKjDSWf!}Wc&nO}Pi!sDnxGY?H~{0sPU_b^!6`eM
z>Uz=IYhFBe2A8D^ju`mV?1EtN{`3;SRN44AoVT$?Lf;uGrGs(V7XtTz)QgNP7jc{&
zkMUW_p?|n0vp+A<+B+b;`OgYHQQx^(#UCYKL@Rsm`Vy+b*dGORPiQqo)j$?^&1Va!
zf=hMgAmI|d>O*-6))E&{f1xV~VJCOvL(i3Pi5civQ{u8jEb3i9&)y!u(9e2#r|hJ1
zsx<G`V3Vjc4VSx8%W%Q4Ryq5py!+eMnxne6do^>NBWM?VO6KovZgY?TmMt(cF!8N^
z3@%9&v@sW(%LJkZ^Ld~(CQ=zw;gLZC9Hq>axiSTg816qfVO#6;cR-bbVv{l6krBNS
zY6j2b2#vlsXf{5oP!Gm^G-q2V1k}jE2Yv(2SW&~|!cuZrpNR@oh;yTOX-T<l`+ZLo
zE^pU`MKOo<*q*2zHnk5T&5)%r$82ico3k$0t-_?xU1fh+G>V$>3G0W+5*t>CfVayY
z=2_b-<Z|}>!Glnxh@=#3CgyHx#l>IFXTpPvxY~6Ydb+X833ux2Gu&xucc$7ti!`i|
zhnF-C6MC&E>OoT$7?uy@Vh%_JRi5ukumz-C`(%t*X~+7OrxO1qhX+8(j{qpS79p}f
zd#L{iFrzR!JPpO_$&{kSpnGmW4p^nx)g@$h+@A9A1UZowsC3OSSWRL1APGuUK;C%o
zNoUzyb`7)smh%49;nuEDuu~1``-mV6(agmuf|Y`tmBYc?2^wOEG&M_=dN~I@C_G;k
z5w8)zA{0wp;3gft23zwwUrtz=YM?jhfBI<|gS0Q4WMZ*YRSATaqt)XIf_fM}XnSMT
zX-Wfwk~!0-TQGNP{=yj-TEL8@P}y7b#3#u>X@p3X*2MpEBV!aT7;w{cW{q=t2?fB&
zMM=w>Mm!#w3GneZA)T+zamuPDYaicY=3}phvkZLJ3?8yhGfs6*V7`_Mb>i(8fz!a1
zMM-#dhyHW!jdA}iv>E#RUEyP=`nB@DQYKnpvpdo3gptT`^42w^<|L=tQYa2Thu14A
zHI~XB>D@D5MBZK#(oDFhomvy9tEAsym7h8AXd%upSe6W%WIyQbO}yuHUX0M()0&7+
zHEJ<U3&1CPh7{};7N_FN39;_!!(z0;0JoUyY4O6aNZijBTc?CD8U_r9I{dg+B>jLL
z`)ysSd!3w$;5uGY(sHF~q;bP)mmsUGN?g2SjE^H#L#wS#)*yJKuYF30c$<n1z8+&W
zj=!b26^?K{X>SlB1B)HiVY$zdccEnoQ}49?_0eAHIJP?8d++w={qOr7=QGT|)R|l0
zj(Pr`>j})Pl<(Jd3UX`Tdf>jp@zoP&I@6w4uO|mTkr1BYIUyLTT7`ui>Q5M^G)eq~
zb0;K;VsBzpcmkt;)82V|0xHU3!lw`**#KS(uE!3;T3{^OZ(j1B^?iKK4@I5cu-yXS
zkee&$-sb<PW?uF{km;25?te%C^ZPwO`$ZEs!WH-0ig=rkarosE%}i2b2==Ew`q#O`
zw9m`6^MrJk28#<N&2c5B$ftCt8o0+|3@uw9So6<llCfcjPFw2*{(us{GW*PIRv&y3
z6#JdMD#)^24@^olO)fFHAw5my-M?MpWT{}ky8dn%W;rH;;hu#KFjV8p<l;(P#-KHH
zs+BuK#$H>q)n>ewSP7-z_mUtn^CH<B59a;zB}~cENZasPGb8|zA%D!54L6wiy2|yS
z=sHH1NqBiMV?_PBW|Zb-M{RstcEv|Gm?1G3@|&5Ie{bDO5K%_Q#@NsdhL&wbDL)#P
zFODC;*;VcUiaD!{4@a4RwtE$sMZpQf<ZYAwzpGUX|M!8F{`UtKmO=Ak|E6Yf(j1|_
z+Fq{vv3tqiIsN#e(|J^A!D7Bgmz!6;jTam+v2#gpRW9!A4hsfHt!-cjbDgV@qeqB`
z)z0KiTYZf_5?a_WtO*bLiyqv%&DQSgx!+y78hQ>#PCo!Yh|qax$F?`17`N$<ZE%nB
z=`?JBb4%Au2nmC+l9N2mGn-2`$a;7@^kl7`{gyBGZGSrKl>j_yAJ$XiF=Alcb_0TU
zn|_8U_H{xRX$B!Q3&3n$FBTt#v_R9^UP62Ga60EKEf<XO+`NJ&J3zbcgBB#sRdtUB
zo=ixz^t<T2iA0Oa^qkWO5by7`VkIg_SWv&DfdTWyNyyfw@-a8qwEo^FZL#VM89MsZ
z1(@s8b>iy|{qPYXoy-jM!dm8)Z)yez=!VcTjhb1TENj`dIh-MnMDK|}qagw@-MtU+
zMO^(Hht(V0ZPpCVs+NN9R9#Im@^`$z6ag$zpKgd6yPY@gOnaK`NI=jn!bDtD_yTJ7
zvR=OF_o(C0@qe$Qe;=SNf9XZ(lyL)xfo?$&)c5G6i%5=icqB2Pc!qVa0skfB0WFef
znb572OTM08=BR@Eg!JNmovkQI|BUd-!{*Cr9iP9`Ed2U!igm38D`F{tkb<q`xHg<h
zj~$PA*Oy6ZN7~^qO%k7E=HvtgMM;^*r`Vs5C<_XkCW|FyKvV1>!z;j1N~Lz2FBjc0
zurH50_=?HZj_-LWpF8x29QFOC1AV&0X0<5iX5jM*xiAZ!dEh!FaXS4*h%@Tg&A$26
zY$`WWh?6dXM)G=g;v=B2pVP%}xo1m0tt<$9Xx+>$CY8nTWge6CJE>W04AQs~w#cSw
z{{*$cEnxRZ9wkDLH?4h2IR22~OmR{RDI*nkfmfj~XdqgfSmI<72^2ZistFN@$4T8m
zvcMYuu(O`VXaV&M9w=+SP6NK(@^dV=h4pxe@OW$d0cRQy@vbvJ6CvF0=aj_wi-0?4
z7VibqCKt{7WhGG_QGy{jL=QU3T-YxvUD}?8Qb&6qNJTL~iWa$%MzBunN(<@$aVAML
zm$L$F24k6}MFbdBB%X=h5)e0@mV>kfphd>~=HgJfiC|Vt)9}F;f}f4i^*JjXFZQ50
zKo#_|NuvpD?ZKk>iE0n#GC-5)D%YQ?UOBxqOx4HIHA@AWlJgozLKGBOxWJSiig1IE
zcg%p&TFyUU#;cXIE~P`f3lPn%TrZ*gMQ5DE6{wzzPB@iZ?$uAEdLo$Y&|hBqH78Te
zkzLkTLR9a_I=ZG+P9_Lh(C>3S-NYBUL~*;S3_v%Lo#jA%^<KG+9JU5X`RwbhU5Zh%
zn~HlXEH^GC^EC!XVVxwTUNax5P4sh@HK$Q>ja!|X|C)Z!nkyB9ae1}Xa>N?ve*Tmv
zsWv!~L$%&e9c#s+GY|iH3TXDy2N-=xK>aJ2J!FBhJ+QWJMLHyAayZC6RK)wM>{H^P
z9k}->vnt|elLX-VZKF+w$-XwM3(MUZuw$FjYx^3ne2SN&rLY~bN2<4l=iGIoJ^s0-
zFzDa?N4a88+2cGy`Hx6P?M+Xe4g1MtvZ}kvi0x;Ggg9H7SFJYkW)0#0mJ-;Kf~CWn
zUqJM2$m5*fR*dbA%X7PCQO(=5BHoSz`NUo<F=!>%2PS_Yvh%V1z?`+kr1#V9hy`gW
zsNpvrE$NT2#+IC$YV^l9?U#D%^C{ECp5RbsC8ftu#O*q}6h2zusv9OOi>Bh#77Pv^
zJv`ccVg*wAgh4>mqR-846lcVnnEV+=W2IV=N-@gyHmqgwp%xw6F_hqMhHdUhM<X9p
z07#UcdWGQ)VKT=EzH9sEhpeFaN6y{$eiw8GJ{=1}7&xSSSUdAh@<-Jp2OG|;ej7OJ
zOxbdN0z}%?C#K&}NF6<6O1Cw2+LcA?g%)>b@VkUOo;~>e=TxB#^vkyl8sj2s(Sg53
zv6+B^Vz8E0J%Ko>8$!ZFsZ7v<TE<QzDxXWH1~07LJ(OForDT+1zYMv@V651{1!nd-
zY$mtI`h(_afxPE)*33T|brGrT^Z-p$FKnTcx&;flUY0mtc=2ZBm?6K-D~f#I8u+6z
z?WyB$epx^X#hA#O`I#?LfS54U?`*cD$fY^rnRSzVtk_QJ6qPuHb;a476Uu_I|BM7U
z;fZr-6%Z0g#7>!dh2$zcz5zs%jMCETVPRYmWNqCOeM=@EHmH`x`a*F1Hm)Oa`?ML<
zq<a%_CA$bx?`cOa2UP+*;|}#2`evPjYMDpng%FD=I;;S6HSzhPkL$BaybIaT#Bjzc
ztd42LJyC{g6QjMxIJzzcBX+1^K~X<o?eXCP$_fRw9rd+5aF^mb4m=8pocYWOk*9ID
zY%h-M4etcMrOz*hsF~1E=X&}5{aNnXg80GNmy;xZ);87Y6Nd~Vxqa#+1{p)bk3WPj
z-EM?KZ8$e6p)OyI|LTSBod=;9?xI+lOzCcU`MP_~=oGKoRZQBPA9g#J>1m#fMxKrz
z0yY+j8e>fkQh~~$HzoxzOL>m?hZo}`<v7P!USsUr)x^OHnked-+;c#2ZDv`WA-^BA
z=V}Oy^>gJjAqtC~I9|k(7t2nL7OUH-&~>G~3pnN_pr5Y&^dHl}KepV6f7{G|P_Nbs
zfK<P__s|3a%+N25>_Q($yH*^oPmxQ~tvf?@x(aQiay4?T!12Q?EB9~=@g23#^NM&w
zOpO7n^5oKc>Gh)6L8?@{`n4Wzjt;Glr#{wi>1Vcl5l7hkKvF&7g9M`488dtx4KM7c
z_K|bABHIb!5>vEICa81}h@Q>0dYxZjr=?;8jn4oL5{}*%+d|w;QX5US=1j&(H-GDY
zp8==FzPu^&#YlN1?t}Di%lSr$i8KsCNNI-+v<*n)?qbbZSE$vFzVsN6I`5*NU75lk
zm)1>MD8V6yFT>P@*S^W>5<VnpGlR8GmC$rF2apPEFigU=X4l5zN6BRdqw1aIMc8qP
z78!U+!_gYOuXJ;Vh*Q!C-V`W{N=*Ed$Mje1pVKKKdn-&!ekQrLn{S)fC74(9o4M6;
z>OJbsyG{ennm-F8?0E}co}5#0xe?Qf)HF=SU|xEMykLdwb)GM-^iybX&6;16D7Ks-
zOsO0XpVWO9nomAqNbdqvTE(8g{2%Ski~n4|SC^K{A+kkAGM8aobSpbiPk`!|K3<YK
zHvx3}>=7V(^`lvRfGCcdU*CWtduZ;n;NrWE&Rv5yO15GTNYf21h_>Rx0=Sdk2>T^9
zr@DVd7{~(VmvUOFskaPMel>f=CQr2{SiG+3cHoG>y~e)xL#ye=T#ccRsx)@yJg)Fo
z@U<GPoR5}Y<{`T<-$vdS!oR(pO6$^Yiyg7$q~0%Q1?ag?S9fOzOH#;hxSAi|eLKJg
z>9CM~713ati_bGzir0qdT5>9k3{acWZhp%sw34^sV6`YhA39#2_}$sTUUC#;6`=LY
zbNzV`+eDbq87l*9x)~2V6gmx!-cM6)l+e5T!|gB9Yb-YE+|DUb)cd)<e)^!blp?C&
zA)%~JsqC7AS@Sibrid+8{|S_*31s{G<CJe(ss9`}g*^)|TXL}0(Le}^<Rh1s6`c2l
zeKk*lapf!Yn5Le=Vh)%W?S+6M@=q^Dm`5~g#DLrP7QHM3W5B~x8i%dIC#;ZVXR1}=
zUNAqZ^-L<xce9Y;i4sIJoO`pPrt$GBeI@33$B~c!gMxYrVv`5s@>mT}n3Y!CCQ<PQ
zf`=bQKrmLZNIgE2m<D<r-dB7RN2>JmuU6XY)1oMe{x6j5$z40-%KbjC+rGXYMAuL~
zsgRk?mnG#Oombi*=%I>94YI(>Crscmb1UHzPtJEdwI`6k;SDS;ghZ!e8$0DkeP;Fv
z^;>-o?h^Fy5yPiCHqOMu;I{D8307!zHqgQhQh)6F@XJ2k&q|e|^7!T-a0R1Y7BGUp
zHoR#4BRCkBq)oMB7XjWe4_CU^d7Gio0stmp8+aHNDD@}8qK4BPGrW1MLN%Q)|Kv>8
z#79?{cid1`HML}4=ndh~&RoaIjes(;M~LEN535QT-`NN19wleex+VJW>|Rf`e|&D)
zwHOEUEPLLoqWP%hm~zRW7xcNLx!qeI1KHzd7@_?k4aQWhaE`e9Ml7t5_@UJc!0D~r
zz>=*;MaA$$y0qr}KGcvCE_VgYV3Q)}-dwRo=iVKZ6Bw=f^H09kZ#U6-1=SV`H`ba{
zcPYLJZ*F4LL)N(Bk0vuiu}SdwxQ5@Zn8f&p;IG?^09(Vnjb^MbpF-L}>Ui48+m>3B
z-^11Kv+Hym(?8@q=gD;y)i9z@jgWb~ZXbb(7Zn$+7hEEPOsaX06O~N9AIDg)cr+IP
z@VHpG6D(XDU+S93z|&GIgMv=Y&PB9G4?uL4p0AM7%@GIHr&b}~!Myh%$Ymq_w_Uza
zukS~6N)uRHLp<JEylP{*mB1)xD)jJ3q5x$?bJkYWu^=S<)zH~)gf4!E;e+QH7#9EX
z(B~u{6$=tIh)_MI>UiREH($IG`Isab;RUqmz5F3Qd}uis2I$;P>)ve@?e;oTRC5(O
zMn3%a&4bdVhJ~CV6B$=o?EMc~HV+##?dQ2Y%UL1FoM4D`qz(k0@!V4WEHC=Q6UDdW
zqy8s_^v`D58c^>}bV7%Pltm-y?~viME!zFA$#)ctreEfdH*ZaLuZ_Q>?j!zvUM5Y*
z(5yivfUs|YJ12ew<lfO-`l&R3+qet578-WBIWU;0$aXbZ-~M{Om#y$WDG@s_(OjHo
zg*ZB;iLY!3_C$(;rs6v3BLM>=kHogjY9Ocd<5Rq{sHYQiEc6nGgp(_;4dDgZVjO+B
zPReooo{HsutM4PDe;d5bDasX7Vvi_4{;Pkz<N1w;NZNg8I98^|EU<}tmVrK@%JZR?
zd^Yn8Be%o=eufeq8NJ6)JMKSu5Mn;7Hfk^X0h2>0T<O7;mdh?VTeZ86^MH3_&E=;x
z`NZbw3hAavLx+}w4Q^#65G<q)FiBePHxw%jzj~fTg_pFu-Tx7Q;Q<+3?kj%g5!~yb
zc_o$IOQ!11BJWhIg2%k<PS9W)T{5i-mrp_5V^NkryWVHTa#zQGU*oO^x-h@nSi9fB
z8P*Bu#2WqxGFmV416IHQ1F?Y7^M4pPvp@gZ=Am^h3I~$rQGZB?`_3h~PCzyOvlQ*V
zhr!5`%5I`+3GuQDbz90kV6DcOAcIB?#9OpZnAaMI1_5WC;1yTTZno|Y=Q%&uMZ{;$
zKLvOoxTX2(i@ZzKZ6e*q1nSLBQD^E$<&2PJ7@@24wqK2NT-Xc6?m3t0?D><9u*Z8v
zp`=u~hg^JC!=&@7F0|Do3-5|uk-x;ojn;H*Jb2Z7=<p%$0;;c#_^fvyu6+D(ziLQ(
zFBoK<X6g^^@ge9fV9>AqtfK@16#7$M{Ek1VxQ#edhS#Gb@4t3aX?|4paaptfJnin#
zkZFY3_I0YaMS}m6Bk`bc#ax{VxZI??n^Lu36{T~T`Fy>5mjXmnDWJVVT@V0D1CxRD
zwGQ3zivJFmaM-_0%f$=u{Y$-)0<Bkue$r-wocvvU@n7*2hKiW3-vUJEuB_9=t~E1C
zMY)a^;7ehi?Rg!smFmIae-g*eSmUB?$0GGqkkZ0&B8vVTTstr56Yn$>C&<CcdTQz3
z9os;g_;+s<?{ppjW@!RomOmiUHUv;DIxje54XX*qGk3CyCP)cF@xs*R4&NgFe|IQp
z!+)h83@%Wwnm5}8nFjVe8;$~c*S!A@s{p-gGa>BKT^Cq?6T;1~7}ur4`5P5z#TB14
zdl?8AIDHPMn@#23_IzaV8B}{}r;Rm8<sPb@KNLi459c0@`;iz?^D-k)3GunYtC)HI
ztA>Gfoi$>Z8P8j+Mn$q(k1*=n75+X9OEFVVVC9bvI_Ar!V=E`}UeZy^S^x3qFGJUy
z&ax3ubw}A7m!r}b*K1Axn0SAI(4Y1ce2`2Qs;>)>Rp1cMa(SP@1qk7#*7?xw;O0KV
zcVEJUzev$_di~8%Y1DO9q-V5YdRDu@Gi>B{I%7Un<jXV)_&=!!qaXj=ul(RtXBPXr
z3eZ93S&0KZ(&kIPJk6w-jQ7Yn3Fyuc)`H)HIp$HtJR{FEs2@#8p?q~XX=M1(3F9x~
zSal<tP3ZkER-!)#(UDPB0njC%`|7Lm%z;A)=G#y?(fNbq<@`@*&uLRluJ@DAwEX=5
zElv&JXVKVsY@3_oa15;B+;&{sZ{I_iZ-ANze8@@tc|`O9z&R%vG(AE?t6BzRinK-q
z9x8b}R{29s!T-`?xbm+b|Kjgw9p*<HOGZW)weEg3ccl!RYJSJ06@ruF1mWE?XV<7O
zL^%wz_zG)?ZX|ob-Thwn7NFot#ZQ96cLa`Zt=k_pVF%+4wuUb71jp=gq7eox=VI6N
zH^iFvc^}7*9kEE@o~|Z-K*+up-AuPsUc5go`J4GwO_ocW(p#c1UtETn+RwzTYe4;r
z<FJWDO7oo6mRqlX9&wgSi?wbJw+;xhnd)|JsTa;VMwP0X$_#Oma*9Ld``tZTro=X%
zPitpGNxIw~g>#wkzg8$3e)k3;)7=)SfJFv-GFuRa0(Gt7b?aqsX<ztlHOK6ZlXIcJ
z5!`Qhn~+8^?9s~n9_zykOC&QZ1L=-ctzYQ!l{Z2=I$#G*m6*>%m-Hh$^M;R?89Tl2
zc(t47j$I_oHxeYQNwo46tDK5MTyh1DXfp4bxQ#9r&nKG(vL6_}WxO%*I7nvy){PXm
z>@457{k@Dq(82K8gWfJ|scW|0Hhk%3Q<$=v$ngIF`G4`z6^`iLW^mAbA{tlN{hz2g
zwwvMKjJeX^I1<F3u6>N=|L?51pK5g04FnoNzM<gI*Ejb)wQ(0Q(P<-da)S<)WrH(H
zvy9)9@aR&A>8$QOi5PvF_S!TM?!B%;LQ_;*cxKUHrDq*4SX1oXFSqyEl_RM$J>|4o
zgYeJ)f7qM>H6AGJp6+KdDp;dF%)+VoT_DeOs|q9}2Op)}+sj?m2`#78ApBcTTFVwH
zCje?`91gO**I_SkMdjaGI|*7R?3jP<wRlziE5*-&VsBOpXHRQ_u#CssY#eHB9^oba
zHEB-Y*j|wbpl5x1f+-BowA*l^_NlQc=4_uw2`7$m7g7Fk;XWJXK0C~@2#OmJsagYS
zF&h~Br>Epa|HyZBn~lBK`z|=qI0`l6nl@+5UMf`9ddLH1!E79R)RTU|e{ocxCb*_~
z8!J=QUTZn@5q+>?;>D-C_y4qo-+F<CXdAEDWtJAP>h|tOEfE8O1V_l>`Vam1l^<l4
zk#IoukAx}i!J++<{1mwvr;3{x>)$_^JlhZ?f19-RCBa`<EA0StUGgq1otNXtyZ&~y
zJ2J$%_M*z&PYN&tvc@UtesGSg-bO|JKfJwlP*8i=?@LKacb9aBfOMyHgEUBYcXxLq
zoze}`ozjhTgLH>yq5Iu?zj5w8GxyB(AI2HSVXgIheoueCZZ;;{>~&c;)UzABelIqb
z0(I38!*vAnZ#srP;UYyUAWD|B7yW1KfFRo%nlqDbc+$bS?q+zfhSaeSDnAY&3&;y1
zNNELn%^|KRmH{l_UcgY*Iu7O`;DIk?6L9v@or2&mVFU|UmJ)7{SVbf~!07n4ybr}G
zDN#@={2SwEsQJIa_(e}Fkw;R>TE_jx_yw=P9Hcua*_aYQ{J}GQF#e(V51t87SaGnf
z<HU60vSuAKas{vxZ<~UXKLIM$i-OTpY}~Ijm%&(4DRmv70#8kklz_(b+8l8tm5Umb
zYwV&|sHXfx9Shw8t)z@-pxYs^vCuXS?QY?2k`bd%O&tI%Ibpustuu5NMYS_t9n7+?
zFdZy4@qZ)1t2q)Asw8M*!H3584qL9~<~g!fZz)FPV;3PlaQb)Q`gbWIyB68xdfiso
z!q|x$qrL$O(MPBkHkQJIx0Kr_2OLw^m3tkNNJWP;AS{n!xdDH3ld{+3b1>&NDQq=^
zd#@>>`wZC^?$ATZRlN{jw}Adxrdd^EB9jT^;S&%`{7wBiB(Y*yp6k#D{q`8+6I3Wb
z8(^v?x)CAZ!6&d7?{VL@xP6MmB?4fZ2^WQmgku4WjLrCct-q4?SGr3#(nMP<DmScu
z9k~#uwu_!A4cL!k!{x+RYLRi4E4jOUCf(dZg%3L)+0Li2vK1lV6Hr3>=7zlOyZU@m
z4IFtH5)*G3vA6i>yx-Oq^)7EMGtoD@z<sZL#b7Mgfb{gjf+nezLWeMEpj<r+19(rQ
zhd2MjdwS(~u6Jz(x|n?7aRH%2RH8F~VM_fCDFmSa$pO}$O+D|xLcTd5sd^rj`@8L<
zkZjT-*%kBJ2FSH6Y>6MmtHuwyNGC=xL%?xTbHU2z)Fwp=8dG7~01G1b0VGDfCP|fc
zi#<`DzN1~Cxd?6v#J!W40s(fz9D@`}sY~0}(11SR{{h!;a_Jb)VFEZX2GQj$D7RHE
z54uwDuQB`nCx|c-W(O+<)sE|28(S2CmuCOo>hHyQJqtW9OjoD1+FS)C_T-<lcz&e(
zvTLo$Wqw*g3(Q~sH(dTV_t1Cz6*pt6WMTZ78z5seSl`1ir53<!fXI<gL41RdDkU2C
zg%>=>&}ZGk3t<QO<<lg5-K&bED!{owEc4ilJQWrn4Ig@_SNU0fM}tNoS_ovK>4KGU
z3W#;sKtg+oCzLRGHbV|sbg$mvnF{Az32$ag58`esLh)?Y1tQo6?ho@4a0d+pf4W^q
z_BHuDNdnMq)9z{A*8aIAvMB-c1K9YvNad^JQCvT<c!4dlWNz1lT1KD`^~zmOrn&KW
zJu6tRXXTBvd1N{bEG7lA5^rJ*SJdKByk0Wh?FbqseJY`q=XTB6%xEyRDsA@6cWF?f
zmcKUA|HXe)qx(ZcgiP)Qw|fJI$S?fX76V#23gH{b9Y~0v)HspseT_hx+f?`@%J;4%
zQbPO-Bo3$rpLqT(go~J&_N!U1a250~W>v7WPSg$bwwxkjw5WlS!^u&HRMPmIGL+Sv
z!=z}TJt#ths#?Qua9nFh1%qdm@dLs20Z1D_top*`Oz%c79p>{Jjh9a`(fY3e<-pg;
zKLCvfYd+TRc5{r)d*+K6K-aK90^};JZ}B)7X?5DUUpEc)fH-ckfv#QIaIDJhjaW7@
zgbdIon#hBienb2T*qUE-t|9i<%D0nmVuW$Nnsk|j-vH1?2<$61LYM-@_rV)`h4<fc
zqW>!$Q%&^`L2>l|DnYSx6b6JyCEBh*lUTTzRL)S-aZxf5b&44dl~jI0;LEu#X7#m_
z47MZkl2Cr5R|H}>MD%jze@r9*5((2;f~QUYbVATVenWlbiHfcM7JK}w-1=XCZ~_X>
zz}7M1w^t-$Of6LTs|H{EZw<bHgg+X5qAc{FJ7?g~%9seCZuvuEGYBm;QW>urd;v4^
zxcftFIZZI7{`9u^`{sB6ISW8bWLIzsnIFbn3>oO+Fq%4iHJR9!P-TNLQuG&-wLK<T
zSg$g@K(@%14axSW&uuXE&y(dA74;jil=+Kb0swxc{|8nb0Ql95X(9f{GyT_x`F+=^
z!T*W63bp_Lz#JmE_ViB*O%!}03VTO_o1eLkEU*uo`%1%g2&_FY_~$qD6-EAi{o{yC
z9sR?|{Ua&$@9D+YF4m!Nr^O^QS3~PN0urYQcozs81f;p=^H*8B?9rg#vUaJ%2J(Qc
z9YIT-zx3u*UY4{X{W$n1h^1r%qZvi4yaKj-|MIlJ;p-4iL9zE7$HnnDZ+Aj2W+jkm
zO>{i@#U5r<+d}~9+l(0Is~r^)VWdTVM8-r}d<&8pG96D6=~QTftyO<9;k^GBS2hFx
zS8eT|9^8L^q)s#bKNqZ-6Zel$DD%g!&ppKaYL}Rp{Dz2AcnK1GqAbL+Pmt$t48!HP
z!ENDasREkrU~8m9iV03648OF;RJJd844a`W^Gl;gGW#nQeRP5m=PdSBpx2rO(AXE~
z(UTDlIlc;6ae_+ZHZboA1Q$v3E0!WBKSnno1$r8+G@1+%nf^rljaF^`o_G7>AGko+
z>&zDbkcE_O{7>T?|5(!Bk5vZs3L(JO{4}#v3XKzB1Je86hm=1<&M1sVCj&J0J@imK
zQn0h`_0Y6HY6y{;>b2>}oGPAYfL%aL%Nd#yM&4{_gi_IS7RL3AD}0Q?X2PWUGS$UU
zUVOlmeEadDKrRi>g~*f~{;;Xg0oIs9u|FV$5ORn@G9~W~4b^K9t_6&`KG9Tu^NW&-
zC;0>NK7hHh`sXC^=KyU1fVLdiec1oseExsF#j@jHY0D9jV}Q{Io&&h6a=CSA{wSgV
zOpq@=F?O-!`36VX1HMsrkW3nz_=5&eJNtUzU2IT}COVE(Vu(4Po^zyeRdps639EA2
z<8%^rn+S3Ug84}*q9)`WRrE%OuOy}=LNG8OG{5*lLtKUCA*SqyekEQGz=Lj?!OX4^
z;X&9L0R-Fp0l_wy|64YXg-B3qV2{L&6w9CnY*Fb-U?;=|%gNx-`yK+}h%A9Hd_a>6
zVvR_9d9*Y#M->_I7glZv!GjoC&jJ|a9jOko#vs|ZGo+?H`PeQ?5*J8tu$<7YrNGp%
zTb~d}uL+C?Zlh;Vu`h1}2x!-<<EANp>HSI@U3c-&dp8Got(VJ{W_v1UQ-wb<G&zs&
z56gdUDoQk9KnvhU%c-rV{12z=-xq)bGgvtUfRkdW-6Hmb@&M-miD5#rg{O_?H-vKm
zvwJ7NV~bwwPbo{h0(wZ&J4qU8xw#Eg;*~J*@ekwi)U&MvIcrQWxgr!x2;>Kf%evyl
z#6Vz(%e$z*u<qbAO)=3-Cli>O0}@mGy=h%ROGNbPKKU;&+bPN<u5wZIg3bTnq(IVk
z(SMtb>-U`7m4BIy&>yowP;A`(SM&Wp8~&faWk&k|5>1~iEG4+N9sM1>I6yZMdQBQ`
z{uVu(B^6#rpbh^^^h^m5J-e4w6vY__J&~75{f!a=M9;<m(KFpT?Hp1EKP0ZVeyrd`
z@5;N(I`gsL@l(kXI|<)q^Rf{;$kF51qt;5E=*G7|UmK=#_L9wzN#hO|5x6a2Aw`dQ
zh67df_&>beZHlSZe`y~C68#A(z>0AHTZH(p$OyR1eP;uF0JLO?G<_JbAq7adiRrcx
zPC*g7hCt&qyo~}QdGT`?Bt<X=@kD0RXB|=0;G9ui-e2I{F7pncCRJMNHL7Z<ZgT6g
z;mP%7krU3js2(AbAO#C(;;i&#goX#<dBu<*kx+j>Vs|YK=RJu}qId0`ZqM}}^dEq@
zU7rLKuk|wf4ZwGUY+L*1*vP#Dc98-97+c&z_s^;O$NT(wiWXils1pa91sDIw9)oE@
zxbS3pSQ?db=n+*~xDgdv#OFr@=-{6oCRo4jEXjykZugN9@ErM@jE8w_1#VW)=DHhe
zYO38MhP%bSe8G2fIz>LJgg$M5v;sQNs%q_VE}uFh30GSj;hx=Y5olD)VMme}5Z$f$
zNJdj0Kf74I_yLUG)*P|uqzlY7lj_HFe`mxk!gL(B+kNaL5iZ!<B`24;AC{;^Ne{cR
zOGqw{_$NUMnr$TYQ044Izggr?#(f3?+;u+~#zD=0oq_eT6VX?Iv;etD7=`W;iWp(e
zCnH!Y#ZUjihlu5;1o#l6hcRN<f^BKS=;oy}rC$-2<(vi;+fUF80q#JVSVk8}oX^yH
zB4)TGNEo*u4@BlS4x`}#(h&Z4I*ydV>r56``;(}<<C#*a|DP`2{R`*_E{9KS{X?}`
z(Ba9nu&tK$nrcmkdi&_1*wND7+!ov>TW43tR2!Z6h_GcU<!^BjRv4!V7p&W=92Vow
zj#|Ak3!vclF<S;^7f}0;QCbEDADG$<7sFg=;oy?#;nF*-{b`^3x!}5M?X}^T;c;;w
z2l}3yt;Z2rv|ULf2$});*q^21^y;aJvT5fEpKl|2ZpRNR=Nn~J=E{ttX`iOo3Jp)j
zG^;qKFQb=`ebzy{^rfYMsoTMYX!Tba`6kG(w><(AEqvN+YNr&D#Sz?zv9elaVt4?D
zV*^1~i1U?PyVL+~4BFa2-!dbhV>w(nODf|k?U&;we9bE77{cmLw9UVt5*bfX6>8qO
zuP4@!7#-yk+X#O#q3CYrU*`~#C!_^qdtpKdWJf|7G!DWmxjSq?X7n4t`I;Hsy>pxY
zS7yZcCo}3MauoW<6#hLA^m++>MTO3ATu|S*Z=MEfK+l35+MeB|`$h(DvY6d&U>U~U
zncOzjFdz}J@Pb4c2wisff0VoLPqrD)XuiU=X;$v`zesXaQ$7y$+`xK1ULnws$ZthD
zV;B*9Q2DBc&ZgnbPxbuxND*uwwaAAesuDPmI}m*hc?I7Ci3lsDS~1G)F4z9NE7X%)
zFw&DE-;-CatQX&P6f6>+9Q5-1V7P(q%%`CtPVd-B`kuHd2^pvtj&~p-HkedHyzSd~
z!#{;&>_ey7w4D$IFeo`uh9!CkI}zBBn!q<PV=C)rbLby7NqbE~FSDDLC?d1N;s08z
z)^8%vlY1NVfa7xvO}SWT04ADqSQ1h9*HSgF(Y7sruN<%Gk^)za#9vF*REm5+DDKQH
zDvU_b_;)wz|BO!l-y#m(Ofe9Wf#D`2xGB;&E)`Kk0@y{f0(bU{V>dTwsMBwb`>SQf
zDU1h;#mjVYB5LiC;VtrOg`t(7`cSOH9in$EHZHWIBsIE8otJr6KHM+bEL1ET<&-b0
zOe@ZkR3)KspRIR^fLHp|d(krrPPK6Neh5)7@VHRO0{gR`s<BCdG(f)swnQd8<wl2T
zCxvYbw5JLD;4jchKYEH5naNX>R_LP5!qm)QqfguNlY9Xtew!f4SBq7qNTcLrcjgF;
z^2zr3VXIFT?3UsvWj@%>o?veM>=L4UZ(ICZ?c!YGs^VYqm)>9Tm%w4re|sB$$6tVm
z3WrINKaewU4g7rrljs?^UM*Ey^j%DJoch@8j|T^xN3W@L=ekD|Y;?*)1-KWkpkjus
z!CTXLlqoe#W`k~cf<qZ*v18C{;iy!2uwSktH{<G6S_TJW+poQ?9MeI1xc-TcF;K2a
z<Zu<L*+r|_&TVxt#cs7x!|VpBjt=0>1%?X6I1zp8BVNHf2LLKL94a4X?iN`P7gLqL
z7tx_tmH&JHB)Td{dIA-G9=5CSI0YqYi9bg}@t6Lgtr)9_&wd2Kz);;8`ch4HzC3Zh
zK93;J`>zj&X!yidBbHoXqHGh5L^07(hm^ulNMs8|^d$J3Q#qfUppma;vMvnsylIz7
z+|E+g3AlYqUH{a}e%SBx{HtEJ`KMkM)-yx*zfUr%=!qDAPS`FoQhawKq1CRTJZ_k^
z(%Xm&%u#)(&CT|FUUi-w*)v^R6aenbO=(q9!)(D})AP|=x8u5d%1Ela^7O$miT5&u
zX=4ae8)3E8Ia)ZjU~;^wiX`s+!__f%;b;afTcqaKQJ0;_s^_Yljl~a@K|;9{QDh@3
zy_T970bNJ%A<LataFy+CMBKlo%=Na4-A~%00g6rKBvXYEuz)WT?gG6|It7Ubf=t%S
z<zpzon}eQ+8Wuv#MJ=Rr5<}qy>bg)zZ0iDLCT6-wB3zuT%1-lj2b}aej8l(G5DMD|
z(uj*z@Xbj{)aHJui2$}Oe<wl!yf1EEs%O2w^c8Y_nbQm;vFj9BIVT%0<1XUbD}K7;
zVwi@h<)`@RU;r+wiQEF9M_yyo=>w{#ZrD$!{f&cGqmJ^mOtO*b{KFK7|99QD@PpA0
z`G2B^ec?aXs|>`V^grkPzeafzZ;&~Ts%pD0?TZ52+a3eG+8%@vaZdX!l~n?M@VEzJ
z?zLg@xYCAihj;0jTwP4ESWQp0Y2G%?N`5bHWFFN#Q)xP%<TEOcz-rN3aH=Jq<Pxi#
zsFbaoU`OFfs`!Mh9i1OZ><<}a%FsqI7p~0-Svk-5^zt&a@s;#8bUR8!FCKnAx;Iu?
zJ(GQp;jJ~j8XiG98^yP)-DNGLlX`P;bAXjXr$J=bNgRVT4;svGdk#kn)XKMe_|ymx
z&*6X|tq~9GFjn-7oM}CAmhVaUQrKZCARU-EjnN<;4kwURjv9|2&be6qSrcCKyHglP
zFsH!~HV&;(L%X!1YId&m(MX|BRW>pOpz&aPSZf)6j6{-9LQm{8WDt%4C@|?@a=W~;
zJM~`4-#~}VbbmfNM->B_pw{?10#Hy^Qd>U%UG}5?qR8+sVJ1iL-x-6!D#seH<Ub#!
zn84Rw78eBpyH;Ce<Jfu}!CuEWs<GzNV_xi}vP<LH@o2$@PW!FC(a!8Aw>z(+!0(>=
z;tMW$SvlSvKW#S(G*j5j)j571o0c_G{K)tx75NGW&A+XN-Dn};CTuq_yx%3H*VuF_
zd26*jR3fM;+%jO-pnEWx=l;&-TRO7XSIBF32kS|7>=7*$J}`4^U^geE^Vo$p0!Z*b
zsljf5RPBBb%q10-0=wmZ=q(YWZ1>sRL>Yu626PGF1@GV%{B`)XE+&ME{Ri1>HP@EV
zWI=J?!=N7_bCz`J^*D(_z+SKt0d-Sk*(1~S0MiZ&98EyQP-_Q9sa}ZpId-ARGWr7a
zywiW+L~GU`iZ4jhvR)Y|67zJEr(2MGG(+ez2KKNw?)Sb_Hvc><{+%v=MW}0o|K~lb
zE&nOU%-yi;JOk3rzMloM#ZKGxI)l=UR<&x@$Erq;sa9zX)RrHcyCA`PZk7?F8S<~F
z9Ivz7OYOV3iQ2E(?F>imU9V?dee>dg`Rt0+bQALAUl+*|iArr2bOJv2<@iomv6+p5
zaG1f^EZP-i#_858G%Jqx4a8}1;TV;$d9o=QiJ9jeKgc8S=D4PxXFA4UZ=Wnr7QkON
zFM|WCp}wwJy=7=`;l{H&v_HAv`CI1nueiQApi@Ki4Co!HpR1uE4x-yY(b@{0r)2H`
zQ};fI{MZj?kR0NqlBl_G(ZKu(m8@Y@igKy-$zqek4BxSCd|g0YBCF+y-PW91g<$GO
zN2LH5P-4f$=U>NKP%g?M(*!8crtjV*$O#f%$rap^078gzU14E6ik*IFT@@j@xtn_w
zY<K7}^y1c%r*sZrFz>F?_^Dx@E`J)yjJGu(1mdo2d{|svTwGke{Slz}gN4*b@0^1O
z4AKHZZMKU~*I+J>9ygdlc-v=CbW9j=Vbf>uQ<V_@5!%`rspSkXGr)*}p0Zy3<n_))
zsYD?#?Y(*7RMi=;y|7Fsr^wM<d0ugGG2VHyqRcjN5V^}XTJBP`C_ia7@&vRA1S{A*
zHs}ZTDTwHmXEYw}@R(?87=t*PWCl5lx7ySbxxhbKv4&PUJ$tKcGNoxrqkl@xS+$-1
zXc>NM!RHU@m`{<lFZc#-_@r>VfcthunA7#bqJ|`KX@AncL?}3@)OjSy|LSI0QL(zN
zSheycj_^#=ilW|{k6T)E^#BW+)l$Qs_hH|)KILfMTA{<s+SpXjz5^kjbCy=#k}uQE
z*DKAPOvnpo#;zzeo|<ymJXT`HuygWl<cKt>@bSW#yRt!;W~ejeof@rHOPY9>*w%jV
zVsa#}*5vPJ!vT%-`Rh>)AwZ!>T^u&wf7vBXLPVk`>V^Y$HpB@hq&m)Evi*rZ5%xoS
z1Im(k>$pBFK0*&or4Mp~#y4-cLTnHxjCZ-S>COE!XU5zj67O~6sgQ$HaUAB=$T5iI
zHeo+uP`IUx6c!K~*z#LLN_z-W7*u_T@aMY{nt;NW+$Y}e;Q*DpQs@@l^`q+-%JyHc
zkAb~42umWDPD4U)w>w#S?=?-FKtf82E)QO!rf*;{)YIet?c29v@mP{DG%&xA)>d~|
z%+S!#ux;DBvkf%QTS;O>Cf2SkZBH;a?Z2PU1OgA{uTS}?1Z(59D1&%S!R`$4rfzVY
z?w4HeLHyKAfhds<v0UzF<<gb<NDcemIUcljTjO>OZ!3nmwDsyZ7=pFkn*^1YaOGm5
z9o%cBmdjxFsYP&a&Fj5+$U*$?+|xX>JBauX*kUNseE31B>(#E8pnzD^fRbmf5Gwan
zzyL9qz2Ev|j06tL&p1}A{*}T9PiBlZArN)WdK6`ZD&dsWyRA15wle*L#LT=!Bkh@C
zQvvgIvF34cKjr+r#FUldGP1-Tr~mWkySpV?&xVNRc!B7A10n_pW8%5|WXMN20~3bg
znj!8cjA5b-6e2Ph-4YI-n&aDLBis-6UfAR(bb)40zVpkB*eXS-PId~Cd^;SbD_<i=
zEBusAA1-7%7oK@#peQ)SPZ)wx1?3JI%Q*o&8_~W$f}htY!`7I64uXK`+{lt)#Aql=
zd#p_@H!62hCZBg!*}YCiMn<vc;|*Q|^Pm}vgoK2{W-kn<%{qvPu&~;7!JJR!a3YQ3
z$x?lZ#bOQBD6l;spxIy{k<)<*>l%^+LvRb`$y4+%D~t;A$-p)B6Z`4OJZm;nkiB$W
zPCfyFNW<Yk`h{4o-WaO%x;T*E#>$w{GBv>oBEX}e&4S@p8<m@y?MO^{nWyB&_I!bD
ziDxg*;&ccZk5=d913I;#E7aa_BBe&X8I)Fw!>ZcSa3U8aIppod6o&MfIPRCn9Ciw5
zg57=m5UcZhSxlBU(!rla_FJ7tN|>UM`nK9srKa;%J!jlxf(8_mjb#nZ_?FcEzBgPG
zY9{`J1@M=h>4y9G1B-~458Zbmd*I)1Y<<{4+Cd<`KgbV&dVxr?%oF)8cU$v!MC_F&
zmJ5RkbU;5r%L+6>?Y{3sa8pOz6~Jdp3jYKLDq=DE>CO2_AvWgQ;ZohNcpwsoXB6pz
zT5Mrn7LX*IpxB&OLRC~DcSs#I>Iuv;Pn`^h9KwtQVC%gIoOh_2mmC2_(CcV)G^fF6
zboBJ}Gi~n9)T$MD)->ueUwFsZB~;`c9OimN9`Env=PR_71ELMT3qv6i<i9tcOHsbf
zr~hqSS)G*>v;Q2!&Mu@Fds6AM!AwNR5_L|e(=Xl4<SzL>!Za7iU}@{IH^Doa?QHh|
zt2|;HvJ#*X6PqFqZv1$RUEer45xCsb454=Zxw7$yZ7uJ?{j<YJTcfU6`eaF6+}ZoN
zKK17KLslUweix~<3tmmF!-4+QaV=s=Jw7+q@LJ||s)n&VgYgnuD^AhwMtZbIQI+kl
zJTzC-MjRH3u5kb2s_-4B-F5aJlSi+7@D6rnTSN83)gPMD?T@-|c`h^-+SEg`l>h#4
zIKGbvppBdD+e(9}ahvzg9=xu0PL9<x)ruJ(&n1)v*i~CH6T}-A3mvB29G<sL!`d7Y
z9VNE6kH0=xXMK<*lS<0hr0+XW5oBF$cMN|-y;~~%AaedbaipBa(2IAuKnnHZ@-j!O
zHR_yYj6<ep!3TqW1&}djI%hM+x5)_zcxPv8yK5dS+|u{L&9l0{JEo@maOYBDD<Sb_
zXnsD~F)P3V;^G*IVPGq3u|M*xq>lg$0hm>mXgq_cvfzFc!z~CAzVtz$IPQkx4+X!O
z$Ro@E-Jig)gHlzj(}UwmU>>()(mfwAdk<M~hncxBqLx6??H&VIe=3JiKRM~**Mv<d
zI27_Q*Vx>FekM_ft^7#dkOlqAkQrjH2`J54o?SnDHbkAOKCBY^S)_$r!Um*NO%IX^
zeGsuTwbV6I7ow$pKm>Ty`|fE*^c<XByuoT!GNdjRRRPevI-Ds17;TUm%|d9=8EHMk
zw&XY$NJ1~-#`?u<|2j?lq&92BM0=vZ!QmpbOpoFj9BAYqmi&>))mWk{@F@!E^(t5|
zZ`*}`JzMM?#*RtANt!H{N0LrqR1!eJ#+LScj2ffnP9-RUAxI6PT|;!aO`N(~5K1z^
zq>vd<oZpi%G?i8CPtJ>?LqbY1`M#ls!@<oe<z=TI1Kiq<I;=cQ7fLcsv%IWe|MHS@
znMIE@aiV$*vXT)}*_ze12}t--KVG~_@!fYgwRqk9Jjztw-Rg@R?B0hvoEUN>R$vM@
zqMkCwlFDD-nofn?oGgp!$-h|?V@2gklM}=XA+Y^;#v$NHjaC524^5wT{Hl;UU2cpF
z+uu7r?#++HGMg=>N99a0?2dM}MeOe@h#|4>Y!VL5MZ<QDAHwOHsWGxt#$fna-rwB%
zI0>xZrXX#~6i6oKo-8-g{Ca-8Ro#wydikhkNuKP*fz_2qV~tADEO-V|$?pOsSj><6
z2?TkjPsT!E6GRon6Dt@c12km66CdfzL>@gi_`-Tjb3nE)(-f5VrsDAg6mFf7*G%oZ
zP3~HMqgX78@Lm$R<i4K%V;zh!GXrpi0jzUUvd(VEG5b)&ePVqYHicY7N4y3$lkm(L
zc@jS{*QoXC?olKIt@ikb>!VA;s-*swt<6o7p*S*!-BE0zWW_`P+b&=GdF<<o9(}$~
zvnt8|qZiNkr3406CimQ*borBqi%njx+->_XXap9xx)CigWu2+ji#YF9s4K0WGS=*4
ztonLO^~OEoA?+v*8r+t}-YeX1d@4q|S3=0v_aKF_eyhbpAgX%TuK>38Bq8-F+)W;o
z;u9oFpLV2@k-&WSwrcR&dI^TOJo4Vb#RgJ4lcU77(Kgggfa9G?3BLpkjKlq)L<@#p
zSlk_5ZiKjACCRoDQgW%V?Y#9w=5k~6$wwocSu#NDZ2QwfpjAdQ|DF?m_FXZqlNN>p
z&BwknRdU3R&29Pa<V|Z~nvWHk+#;dXA^K)Iu+6RV0$y-Z`3ie%5OGl37aOQ`Tgy7i
zt4~`PMZ@9|QmYge<xI^Ys!@ScvoVZb%QObD*rP|?#%cLhyXRxAY05*5cCOUNS^>EX
zp9-a*qA!h|#=m|0IDd{Jv^Sn<dcN6r_y)!t>doLiz3Yg^cjwPmEY0E3b@^3Ah#fOd
zx3z7yBB`9MO$z5>*Fyt*Ay(SF(yFrf*(v5Xw{lfgT<8wAEQQb}S4h6wpv`U%L?=3T
z&65;~n&*=ds&<_50CKVntsD><LXUUXaC2$0Q<6h8fDdUo-wbb4eq5@@y&9&`!S*bE
zM^3I@V9w->vCa;uCB3F&zed-d8c}T;%Ht<iZ?<y>-xWk{G&)xfWx6+>cksaU9T{m|
z3?5gd=x`Q%|LMN0&i$5YspA)fw~j)Gw`z>{nymqjpQBE@#r?8UX+ot&c!xJkR0dA}
zfh{7I=k5o~BayHs#p%NLFk49HV7;m9>!{SZLy@?qzas0d1KH*<(Uh_XT(V&k8BWD8
z8MKRHVxS!_SM2gue-tOQgrSd65-OKH*bisA0O34sM~nmGDi8W@hp&wDFRS!6^7CKl
zsi}kp?Bm6fWZ@exw3GAG43mu=#!uF5I1aaB(=7z7p8Bq{lH0+U?^_7jocBz%P`NMA
ztawuY4tQRap?%inCuSpua2hFthVWd5xI>Q!$ilEsNbQpq{F4b(_geRy-^9b|thq6P
zzN12r=Zu4X_Vx8usy9>S#MtSYZ}gz+-0PRX>mq%y<7jSjSM&!_jhd^L(rN@*pDRC7
zaJdsPTj2=d=3bd;W)2~U;kZffMHT{NXZC(@eDA-vS6lc{Dwk77ZEF>2`*^FfZXPNX
zw(eUUvI*Mt9wiIv3#X%G{dIH#^y{sHTFDrJXdi}g21nGiU#rS)kEsiVi^Ls3NVGK(
zq<sfoRlCAkgT&lqiX@`z&zp+q*t}m?p8i~wilUDCH25a>uEjjgi#S|rZEC5T8zFu;
zNH3667z0{!Av>~+ViIsN#K4G$PxUppL8_`nZ)1d*=0k7tBH~Xp*pw=C?};s|P(Hx*
zBz{@c(AL&wbve^*KWq3@Oky|i#rs7<d@N3LszQsaRJkl^gO{8<s8IIHxVIXM)ATr2
ziV*tmX!8R^u~Xh(`5tiF^S6ugk4Lh0ks9-fCU;lNG!KB-?aJk`lT=Qp?Eol{@4kT3
zK^T2=H<rF@jX>7XB0xpUaLglY+;E!r@SU&pVcg#RvS_WS${)-yxR5kx;oAQtyi3e<
zB)_Bt1?FKiWfXCRF4-RrDmr`a6hSorG~&I~LJM7;`yp+6*vDs$<ys{evk$vQcE*B%
zJRg|7Kj5d;&S)nmRt(5~L4k&kLGM}HyH?7EWO%Fa4zZTF;yj8@LmJAMj%{XVnXSOA
zjT2=qE=^eHo8Zw}`e;DX5!FYHZ`WY=JaVLgymWo%Wad<Nje$<m{Kp{VBH$y7Cw#S*
z3@3M=HCiADE?fM(b&3b+EbnT&(+wKHE(ZBdJxt4F8Y8Q|0sG~fQ0KF=&YQp>xqOc2
z7`Q(tkL*sWlAbKdO+_xaRD1Qj(LQJN{q$XX`CUHZd)G}xYK?}{Ne1-rO6z9RXHWC9
zK`y0S1#qSjIqU{YVtn;yfXVxOyLJgbiE`M#tK<DFB9%-p9dDugcC=u4FgBVgr_i?W
z8!#Lz=26SO<9}znm6hLMd9-+4Z!{v{_IOg_^eAjf?(!DMxI8b%Y_8d|ukw0NS8vyT
zN!ahbSgI5i0|9I-Y3w)MWE(KT+Rx=Vgx*gyeH@fH?@KHCa|zJVrF()rr4<Xm^|R-~
z2^m2@0XzEoX*}-bIV`G>2o_?`5A{1X)7qcwmGR%D>GhCbhVv(vU)|sksd(IN)GBp&
z4WH5;S7&8%MNe>@KE-|3z|U85-HyHKKlMBLMC9giF=QQfrnelN{s?SY*OhD313-zP
zN6%%eFFa(w*oA|W=GGn6+*)m~qL89CW6?p^w$Ew|b=q`zC4vRxLY39eQ)iig!=}7_
z#qqC1c1nmS+Y;5ziRxoc85>f?!6~$Fnr!i?rLyrwxi~1f-})z}@*~sgF_Tae0*qv5
z+ICkW2cl;|eo4qKP04-O^KfENP|(BX17OBE!nQ9QQ+3{#Qv^<~h5<qJY`?pVyHAXj
z&~7pU*nfX}1`+VpmltmzHOeQ3GkYKr<iRF{PPP0FE;qQ`5`QZ@I2-X&-)o<ELO>4I
z8<dS1(p>P%g8lnTl5u7(1@dH(Fv7Yn+D^efNVr}|L((aVY`bJo@~4ym30$@_U$&<~
zk5BKPBnby`QWT%^=z9@}Q?g~BY?Wti`!Y^RU9Jvi!`JdDqj{)?!V{>PN+J^&2Q$fP
zoKLkiI=tFgFY6M5v4FQzRJAuUv^$nQ!2a`7M13?ad(_$L>}zS}wSEJH^M9rN8P7`T
zbtYm3&vbjMM=JEWgXXG$jpK(Pr{+Bu<xRaKCaM7+#EqGOCg4Q}FP>elwCP1S3eIYn
zUJHxLnI+|8B5_E09D|6a5tFr>i3;1bD&rhPgZfNgGbNEH*t)B9y69WNTZPtYP9Zbd
zVykmDOG9sb5)MR@JC`LhSJ%dh*X}&58FNwhqc&IT!IkldVnPzKR+o|VNILw5OyQ~G
zdUHj}d<`z}_$lYcu&Ad#%riI7^AX>f0=u+CR_etySPm7VMD1AKqcT?>O&pUcl}O>X
zZX<B;_a?vb4=y?!S=rd#Ev(R6t~VDG2~(S*5$^!p=a?0PVzXN&5dK6gv;qYTBBOlz
z&wc~-AwAS)VrFv`jiY%6=0)m`#)0d|EsnhST9;)TV$cM+{Nh_Y?l;9}1I_n>A*05c
zp8+`Eg^2hdwx;UU%cBr9GXF8~&Y?_Jng~xEu{l|~gsdQOGuaaR!r3QPb;hbEHPVO}
ze0qttnOa4^ay!vx<RyNToU|l&>9yTA7dMNQm|*`ridbivetpV(HC-5~EYBq;D`kWF
zG%f30^xoX5zzk9WY7n2uHJ?J3`#eXhJ0w~bnY=V&owfT$!sc^F$8$rR*S%4Rdconz
zGW+^7Z<ZjU$Y&S={4~s*(;rkt9%N*zIoVc=;Vsp!D$3M94P1u~_gqy?x)CvtD_9y6
z`Ml8h%W*XZtvN-ctn$jA&0o*TYdvgzEz+l`jH0>Kjgg7hH;%(jROu_(MMh=Oj>q>#
zai*}y=o~EGlpq1EL&pMi_oOmbhVrt;+&9aZlpq3CKEfg*R8wy%rmbutMk}6|ODIF7
zcE2jS?bof~LXf1o{Im@F=3=cP;6b+`mZ4EZ^W8fRv8uM~JtHdD*$Sa$$l04tM4#;5
zF|2(i1gIq}f8GQnpg_hF?Tg^aaWH^a6%2p)`3I;4BNRf&DAkdV_ERpQ<$_Nok-hzU
z1Cn^dQO-r!@Z>;fkWH!1&Rhtzh=c|UXwHy2BT^7>+^!%C&pbM>QOC)lG)4?gV%rew
zI%1O%VyBRTPf?0>UxzEHG)k391hrZ>A1!06su(ru245n@tA|QZVufr-M`vf1n}s>v
z37>Zo?ta7g?KrW8yne1Ud(@KO@7~RnC|(}B$5*58W})XU`((en80R`IM$(;{EBHIE
z`3VKFy&{gvZTEAUR=t$=z4f%j2X1t?j2nx!;zLe9u_df|mnK~0({N0BT05pY=uEY0
zfr9(uGhIdM`P>|bk|}~NaBC2rtDXU+!Jar+C~NRQ+UImqE{IF;mv=IcD82R$Yy?##
zZG#WD6q@%UJs;2baeISFFOzK_ViZ5}Q%TfAT{A<&5@pi5ea@Fm1=B<p9@)Rz;V>=r
z^a%jzh9*%<u%1qMN}p$cKpa&Q9>Az%TWUaO=>j)=^X)mnF>}q+r-E6aH*}d7Z#GY<
zEe#DDLoaz9w5Im}e)+)71VSY63mJu%Vu{);KkVV!YzTti^V9uRx<_AxlwLSm`MNAY
z;Xa{JIye7|^ij$9prY~qV0WUvSwb=b-j))Fi*3IK=SMsd?VVw)VN{yAG9C`wgdrcc
z$P$Y!sEGD6X|4~?BMy0%noOwNXX+#Vmq0vUGQ%P+u%*>bd(U1rpy!u44N&`gWgJ&E
zn(d5wl3is2PCNmAgo1Fx8Rf2Q`;Iww;m)BmF+)dKX{Nd#6y)lzTsH;VXzr15bX>5t
zdg<JYsU5vxGr3d?x@;LA?jT|G3OLCHbzk#WPf6U%WLPtS0zZvXr<uTb2RlW{?ykhD
z=ZyJ>Ag0}&WSv4NLXU}`?SKqjDT&+q3up+TE3x=AkkDl_p(}Wc3nS()-~DV26#b&4
zQu#xo+DQRkxm-gg&=RAyLMz?I>O{XZ_4QZUXl>B#awC_;E2LI_I;bAt)DW4(V%-Oq
zX|)q6R4iz1r2#gos@>d?U}GqD#~#z|BZ3(hbKjwc5xm<5!A#|LMbL4!jw_@dgFrms
zr%rUeeAy`eipkL}hU9oKwQ%<~$Nk|zws(dyaNYNcBAx9-H2@_>GA8dE@N46p<kkx;
zfaDZNC9}_D!9zLIK)1EIoI}vWN!Xl8ELG_d;`6wry_<#2HD7mTdG9ySxE3W>qSgB=
z5Kh(f{^Eq%D6-NTxm2+T#qoGSd@LTP3;e*t?aIXJ7=H~j|BJ67pniH2sBmRSIMNdw
zu8LB!V=!Q7s!r|w_!aDZg~2_a>EjC)&eLbLq$fk?i6v9z!49+^)}$Iu<Z+@hA0>i@
zf8x1)``ega?<*q4>^_hsPdmi-UspmZGf_-$3KwBhendVNGjEzf^!4}2rE6GHWlf6`
zY#e0?Z9tWX^$?VLKky})cfLG7mR+_2H3CHk2i7miPC2;jR&ue-Z$8YLl2~fjs~(V&
zkj!vAlb#~mF#3s!i4tW*r?iT%7eXR@>fKE3h@lvDQz_%!!1tkp@B;0%h(6yGJFG&@
zbS0RX3`POt!y>h$5@9g!=@Gx{QJb_?4vmO)Y9Yfj6k%+p{n$UE&p**AIAQ%G+x6|C
zZi6mMu@ED{mZ3Bjq7HNPBHtkh=rdAXU&oGFKeJ5MiX<Qfj1U}=%vRCQf2J0z14AIo
zd;iYWUDFC{J*XE@Xf1ugx@f-;KAf+VrPI`~V%F=%=W?>KTA*`fZgfTYeN}x{Qm9;x
z!#isOJ=j_!YH**2i1zL&s<<uw9VfhaLJu~_0^Ofc<J03$F`Tz}oruG*Wl%XIm}9w`
z&WfmJeqx7M{$D39i$WnyNjxs;6LWjGb8pspl?opHgN6yZdOo6n!sySY6HbrWV@5I+
zqfu>%;qww2a6_-0Z8Sm%`lydWAl|7nJ@JG!dyFv(v?S8&bePu(pn$6P(w^{oO^XYl
z*dLUJVZ>n><L~lS@sN0M=LYFzbz~7rH|pz;Sw*y@u7mE3T_!6@CN>?-<-6W)@@|jh
zM}~~GMplE;;o827z;)33g)C%T(d6PL*Z9lJB;S4UWvo(VYpkJI^<>f4jAu!)<on56
z6?r}j%j0@vm^V~P!VS_df#Vjdq|(ZB9{ItN6#kGEn#t{VY<*(nsK}%JpZ%b&h>9$p
zwz))dnU;w;KL$djGwGSMsh1ySWo5D2tn-^q7sBu9+=x#S5)yt#yl9*pO=UUQ6gfIo
zfZb{hJH*e*ft7YEQvqDFvlYc8pJG-~Q?XQ#JNs$e%q|}v&dJFy_OyyDUXwaY8uaw*
zywZ~_rNi)nhRL5F2p7{9EJ<0<0rlcu;jnRPt?+W>L3}6#JUMwDnT)EJ_T{BrhgPlD
z2Y^0=I!tG0jEHw9lT?Jo<HqQ?uS*<Zl(g=9x_wkBE8qK8oqsYnD`LpSf8WR9opsxF
zn?sr1OtC;%z7Np1VQ6C=qF#a)6#n&eX+o_zcacOAf(zhpsxS0_?P#1t0HpI}dX2oJ
z`tEKk^Pr@Nmf=RX9G^3y>cUsF!^?6XXd*m3S|6?PFbN+frLwHOAEaFKV2K{0!p&c%
zTyxn%_6jX7F*B(6&YLblqhi7SP@qtnz#w+-5+xe_LRbXSLT{$*mjIkvIU%SxgsG|p
z$sEBq9}>q=U+Wq&EZ-&@xI|h_r*W<(#lF*c0&h|$^18^XylqA;|Fb|?(($0-&HQ_)
zXCouJkQHBO0+KxP%z}HGaz40&>3@9hYbbn7G7%h#XIzn)6Xumj>6_;=y2d8kY=J(L
zwu!P;oMLQiX8_ik+={8y>Y6E|^FgGYpndy$ud8v9-_=OVmqgFopQ}_r3OzKvryN?B
zO5}E@Of7jnX^ZA|m*Lj$%0Pnf88h$E=o`@Gupvh45B_Ef_5zaiD0DWe)BACgzrzN0
zBATh{4SS~(w*%4{W<LE^4(d2q|By=s&I=PF0(LYf1X(}-I%4pN%RV#NCby#h$Nq)4
z7}Q?*0WcVFet{r?eYrYIQ&5@v@`4N`o|JurPtY7h_Ur!LSPR*RBsXAe50k+{KFPd5
zRr;JfHkV9C^o&xgr4E*9#9huWok=Q*HuiH?LI@ZWu;`yS0E@bbc+SxA6t{U&w4xos
z-8%sh9YJg<We5D6>6!lgR^GDUKe7nW0l~j&0=&tb*Xas<)B?kcKQhy)j(g>Ms>xlj
z<noIX%ok&D+MWWYwgfhGzDG$4IN0n4f9EOkv_fJzM&^Eax=9SP02OO38qck&Pgfj$
zBeGibpja?ouB-!r`nG{(vXd5w%!d+jO->o?M^{UKY}7Js@90R*IzL-OW<I!=>G`uV
zA$nZ}O=>e8^MtjVJd1WXIAvV1#mh*R?nY5fQ+aI39MrseKHUV2^DZ-8NZc`P?<h(3
zujO9_0v2w~5A%P1Md2O|cpIag)bU^yHtTyleKaotxwuU!2KC`?K!@r$oAFV|{c--|
z$Hlmh7QXcFO;?(#TSO7wzGZD!)Q9tkCgiIKIcaq=dU@jGf{32xN}k9;{}o?=Wi*nb
zI9TiQVHV-#$egp*x_;7FbBMaD=NO7g1yBr?H=D{2RTdVjZuWZS<QvuLilxOUW*o}>
zUOpucO+ZYH%<8beN7ZnDG2&Tly~ZXPeHu+yt$#2TDykGgr-{I<u4e{ISZ}VneMMi1
z^T7TirQ6gl3fEH8Zvcw$qm;lDzfxo(4F&evLA&RGf?@v38tbP5O&}P+nk_{yKIVt$
z`qZE=qg87#pojCGdgpe|DZ<kI;i?f{K?*I{`RBCp*K&!ZrA1<w6jG~WUM!*LZx~4h
z?+3G;Ulh|Iug~PiGD4t9yr!wSSZM^W&07?!bH|n5Pb8YDGzd@krPwE1$)|S>t(Db1
z?eqRCPiF;F7XcB57sl^xwteKpS6W@|^H+hY%w2#Ok%AAo*j4IGoo-r>rRNLTDGrzT
z=prlo&W^MOBZJd1jDL9E&Kl{}^BRk8pb<8Kbk};a@cE{fMk=^K2&Q!=BRFnAXGeka
z%E4OumWA1a@VJ0C-tlV*_rvH2G{NeF^A#Hq+~?-eIqq)=3HNvD6pBC^I}!D7Ab~J8
z?;9GJglCnhfPi9!_>ddq2;q}#RqHLl*_a6Bqj4?p2Y(Rw;X`1MErf611?fno7w%QJ
zdJjEViOjuJZ$5yzSMLIck;>=IyBc040(2Vo7|W6vQ`LsL1K~a->>sq24W|nd=ySk!
zfj8%zEq+z6B_9Cc7uN&)13DR5tIYc@@QxRZNn~ya7%j6uSy$(Cz@Pr^4WII9Zu^z)
zp7!u$WU;tLtwnpHqV!pnUr(uC)RIl)bmUie_c-MaR*h&v(LR6bl0rbSA(p1a<1#Cy
zH2Askv@o7BoBr~r(+6~-Q=b~8wrH|Qgb++3-ITtMXLoz6l?uWEK9-m4Uy2m2>E_jf
z_vngx?2~cmshuqWz101(C5a5gg^bU=PISH}X^KNjvk8{_X=8PJ1L{G9_P~Du5(;V<
zL<_SfY=3!9{(XvUsJ?iI&hZY$WH#jKP-9Rc<i(egphQ9ihCp21f95?>3>*KCDhlZ!
zJ|S^%@DHs9e3s9fV(y<+AP|L_K@Z4y&TnsvWci+F#0kwBIy~-~feKL=Da2DM-ISmc
zG{JIVo5(x~LV9w+Av|P69*Qzfi5qm0HH<nRX<#I!o!>ffyBwRvLQEd-)t19M<+qzY
zS6e|R$BVVI#4Rh7_<`%^_j|P_i#3MK$;aqaf*7>oQ3+GEdz4-!WHK%HZog>qqW1B{
zqJwZP?TyYiWg{IJ_@1glUhWLaUla8O(%6uVI5N~o6Ox`_Z)x%LxU}_MmsI9HCC)??
zvM`*_AvOx-8a`&EX8q~bUdC5B3M~#y4=0VbK~|kU=hW08IfJ@Ji#6Ys%&<~qQ)_&7
zzyG4}C!J;+LFh8*N(h4_*59f~zZy+BUafU|K8RzzLY6X28pM^DDfVVQXF1U_Y`guk
zRw{RZ;<87lE|a-)rCCOL(2AYB-*DexTN7Gg`X3ql>#)usGK9yIxy`$2la{gh?#6hO
z!AcGLFiy<w`47Pfzc!7)sbUpt*O4vtoMpqJn}-<u);8+WT<}JyZs;==3r?cjqn-C=
ziY4j{$2COon)_brS2Q8;BGxBkCqvf3b?9Sgh&V3MNgxl`L9!FPDyZ@AK#-^=qZ!^E
z=b*OT@#A{x4=B^f_%8?{;w}iTqw;o2_{bpa6E(c_{W$)vpE@p^%81X>y1H9kY#^@&
zjR7x!PvTY`YdaN$k3oNfWzZm?2_dU*gpi<5VKB*%o%)#5Sf9wG;8rCKvbU|}Yr}sp
zJ$j%X!*>e+a9SxeDS^4mU?k7c6!*uQlR8K9;dIg<0eU51eu)&IMRupKn#RF;1E9?p
z)t&d_Ka*&bOM7#~`4re`*W6EeytW3s7G@*H;{jIx(?En=2hrW#!$WC3aUF(LHc8wl
zVB*tSGLbm%m3WV9j7CtpDg6^+oIcywRFMqiU%Pf|TZt!8DT+o0d#b6WbQ|wTKC&lS
z-daVk`{L@ixm_!~%e>ebDPt9PNr&$x`wH(`YI`Xi{&&)XhyG^<=h#z_gJOT8oWa#a
zgCONsn;DZ<huh+C3Au|}m)@)ir#^I|=_l2LDH6;X!Urs6lVzpFMHFr%PcV5(W|~IS
zc$1B-In%eJ0G?iGz{V{;ooXd*-p*pujFZtmR&RPeP(Z0&8Z|F-Qc`0ZgQd&hd$hOQ
zewsA4W7)9>b)ss1ynaSI8IJ_Ri6mgEJl-!6ur25+)7Gi1iHy-aOhrqf+>SwW5f3w2
zspwrjH;=l`AT&p_WdoC_KN=2yG}-1!cEu^qKLGyX>+^-JWJqY81|d#8fm`C!k(1m*
zyyos?b^Wx$LI?@aecr^R`ebu5U27~`=D;=>8fpH01x<|Dg~$ym*JT0KUO0SS`*BWu
zOQBSWNoFJNW6=J>dA$;8Y8t5quz@UHz!TdHJY`c8#NiLmd<XT!%RTSj`GTfE5j^EB
zcdzrU5M@4`>m$_yRqX>YP*5+b2Mcs4gn+3kz4-DOd@eTqHUEU47<+uDQ)D0XV-xjo
z;8!Tr>JQ<1HV=yUHjeh;M7g*UkCC_wr72~{4%N?=o!=!95MZQb7ED0(NKSWJB>{q;
z=sX6izm#)o$jV^|7KGJ=!P;&PusLmL*)-j!{NMk(L7pGTUxGt-ph8=uP%;;QvdMoH
zV&FFb)8b**h%;w>2leqRr0A{*3l{!|h;%DrK1V|bUe}(Mhqcj|wPM0*46%@s8Akp{
z05_LD64`n4-mbE{a*&VvIZCTW$crA)x(`IO;x~=}2hd6oF;Lz!9|Xq`OPFyG7m$gb
zaSxm62{={$L7Xa70sPBsM91ntS$nOyZZo6chK0dz`z+zE1hu@YoEgT-RVJxwl6TZD
zstn<QnHC_za3CN|%Q=Lq<dno%q}d_X*x^dlcl;9=zVOG@gm-~JwVLmN5rafkP(PTf
z{^cVu<^WI#XJ}o|dS+)6v%Fti4aiqK?g7M4q?5L46~<Q>6as<@!Is?}oNZ!JjR2o-
z8i-aVuafaI4VIcBV1x8Sq>PL)yud|Et&kr=f(7Vc2l2BFMlHnQF{kJDEib17a^J(&
z#Kh21o}iOk0NYLKny$I=hg&!>t7PQOXpV}CItwr2ZiulR?FDGGN6zryWDprm0!;p)
z=5W-nEr2~ropmmIqmqBj&J~GYbEvWFghwh}f9wv-XqGoR(rNtrtHo&jDaq9Y8zvss
zmwa9g8kTTbfLW@y^n?utitAt~LX$t^W0mUsI?M<)zXc(T-c9ebsG1@ElvZwORHR;p
zdJuSSk(2vn_4)Gc>@=y%sP~m}s~2!)V~o6~Uk)zeTN?<!$+Rbq;GRUNR0L(@b_^^=
zYwMG{eD(fi-r~XWF3$=(xdLsb(md$o`w1_<ChsG$^5bim+M8ORM=^C>1>I_|v|dbp
z{l#7Q+V>N9TqIzjCe_bxj~9iEqk*dQU4!K^hekIcb%(u~0ZZw3z`xv!@tnvQkh^3o
zwWlCdrkTwn46hKm_^g2HixtUaLNi#U?5qi&x$NgOQnJl?L7UjKhFQ0#4j@ahQ>JLt
zmkI&PXc47F=&s<9Kq8l6HdCB!aeD%{@%Uz~l+!B(hQPFd1t`MxNB;_Cv#cRkfG2!m
zyxyPeK1<i$naE)z_kSlw<oi6raJ!ShT0T2brkcmGI(c`slD}W6QjP~8nvR0nJs&qj
zd0Jm>kJr15_S#~vJPAZlxQH10m!`S(>ROv(6elY~Zc9{9zkD>N%o588x)wIMtApu|
zh&8uj+-azq)j9b}i%{ePe!+rEb6*;bqIuE3Mpd4|k)bW-k~82~^lok}B{!i7CX;)H
z`c~6PZobMDzs36yyJ;mVZ*c{$@xhY$8uy%7M4MCAcBQ}CzF=nwXP&^;*0#>8ZRbU@
zgMu*EB`OZn?GA6X$(HncE%;M7-;`40PhAMal`H5i(jY|cqXC#I?n~!QP7>GkcUVuL
z3Hv>uydX_5-B6tzS9y^N{eD6qTy;6YgWOf<9mQKne3VS0yFQtBqEsso*x;HoJ3ej4
z{$<#xAJde()YRX44EhNp;)BG%=fUIF4i@I1YMzHGcnt6AQ8?ZOc0wW8<GMk6bP|w3
zuvG;PuB3oNX|uo@`VCw>iJn12LudbJbHBWx^J|QEI{iThkO~Q1w~+;Tb)c>CpHIDi
zjZD8E6nZO0<f6OfvTWZQf;>~9)oKyTO|4X%J%|r<kUxn2?0m40Ku9Ha0sc$?gzjQ!
zinGL|3?BC+ZI_KeG;8K(K&ySD(YGc4z0&oD=pulDGew8bH=Qn|`Y|pwh764*6!f<G
zuwk_oDB0K{DBz>hhVT)a!U+ampDe42U5Mk7ua^ClS>8hbheHIh<hst!<?Ty|H&>-U
zccmv0)Cku(sPXt|+`Jtc8Cp1aq?*{3g^znbmY8remv?%ql71^1Bb>xht<Q7v=vc7|
zKkxPxQ~6|4`x<bSF?I$0KEX<a;-E+UURLwL<;(yB62*!{#JnaY6j~j~bZ-3}e?FT!
z)m*ZgsX@&d$O0ytNXna?zQ?f0B9#MPp%x8Y1&Y0Sc+Ln^tds-CM094e@Jkf-=7woy
zp$QCl^Nw;hYG})tt32&XZ@(gOTQPPSd<8;tpw1QJ$B!T>yRx#f4@ivXCedl)Fc72+
z_;|FtU7MkYGL$uEoKpCSZS_aa*1w(d*<6~XZGz--*hi91XN%-d`z~#p08PN-7O~j~
zee$b~4VAn7$&L*`Q+|FU5sp!R@O(=*p9XxL<5l+zZSU2K8<PLU*IPza)pl*5G#d~$
z9n#$(A>Ccl-7QF`AYIbk-BOYwNOwtx(w&l<ZaB-Q-|w7pemVTH9geluea|_snnc@h
zU4<%zWWKT{rKc)=y}eizA&0A^-PYc$r>L&O^5Aj7*`S4bPm%d~N{`kXQ-}T4j>`QG
z_alVa0RkZo-7@|8+!I7<>8HI36NwlDj<=y}r+kUzAJOXL(yH{8iY#GcEnMZ@k9HK$
zikg2ctFP_|PF~EEs<|IItLYThF*0w>a8M-rw#z)<_wapByZEL0ubdFj6WCKu6kKoC
zTb#a7Ts8HyjJ~H*92-kY3fny?i`0)LbE7I~d3*VR`I5j@$?`Yaibz8B@if5EO{}*w
z^~D2>{zxE-f3qXZw|9FJtUh$)H9c<x$Sq}vi|0$~bt*4ZBS?I=OkcWLN20a&;+hqc
z7IToM7-3(Si}iZae56xjf-|$oufumOuwMl0=}_%^m?C$Y`_ALr_Tq8D@z-K>)P%?&
z38F=XR5p#hp{|ka@CS83ScN9W8sTF1e^gX7+IoQH;xH*29oh%=ufE$aH)GhNg?ry0
zHPVyp%NyAzouT&tGyELoli5oyTYR^@337QVaGZzG?G{!t37oZbe+;pFe0)4FOfK^f
zIR{+quj!JCB&MB^-u`~fK_YFA6&Q|TAg|>XLGz4IK&ImCVp=Kl4Ym-d4gL%?)s_Z7
z$)4k}fg~OhMz{SvnPVaB!s+4035UK2D2D3aUinrg!f#QY_b@NEWg?LPl47+<Eptx$
z8D^t){YQIpX}3h0H=phO71P;6Twb$cx%@qI`b%}1{MM|VKL?<F`Q}}Z&e28rW!Ug`
zRzG6BQT=Ij3ewkwJ<(hnxVBVfyl{4_-r~sVyG?)QU}*IFe98T&kR6pq)9{f!MY`Bk
z8v>&p@yUy)8w5%}?vfTGgSpf=^o6_we|8A-M$fbVv;gWI{*#cT(coMUuge`_oS0_Y
zDaNvFcX9PsPJ*Ww$#$lmd6{}};B%4&#k=^i<0Xf-uoh-?H<7sjb52Ql$O+1?8zY;O
z4};p!XWjnLupo2KCLKCTCtp%b%x)EYTuZ)6D_>d<vP$ZzXh#DxuTeN}bZX}x<r<t0
zE3N8<uaIMSHTaObHj7hfCMOHm-5V_wCj)`WSV}}L`D`1Tc~gT}m_4Ha7})GCZg)AN
zop?ndH~SS@W$s52-)Siv1n%~{h%f+;OE<9I2mGVbrpdu9cFp8T;8nW|u(is=tm~IL
zgdcH4BFEDbX0=q9b<{Q)=#OWre<^-xFU{N86^6y0xkPu8yI~qWhW{}%z3l9DI&8!`
z<@eU|(BXXwt%Ai`L@t9`UPt7<K|NQ(%9=W}?q8JbpKgpT1elzvoUP;iu32QHq+S()
z0#oMka&JNAaTo7M>`g0EHte1O;-}eSLoMCfw@nnD^;IhoxVA=lDb(?>g#`sOF642S
zr|fpSLY4Ge5PCj$NYRvd$@U7_nlW5#OcV^u&(6UC$(W@ZC~l}MSdik#HcF81MY?>}
zEefy1T~Tfe_JGJm4Gs(RzG<kh3!w}0F0?+}2rC!hU!ZS#H$u(@Nw)4<`y9puhmt(n
z70aZ`733SXXAv(})A>+`Fc&7+A4&3L5?i|`9WFE{uZ?pNiN?hP^=Xd{L5RF2$na+{
z*ByikA45QMs}V%vcJOnXn{@#K)G=?3R2GXvJ63|kM2kpV=2B6>#3o;K(kcGj3Q!yC
zGQcJ;Vs36jpG%GQ`Dr#aDlj^}$gxO9+{A>k<Z-8;2QPI4RcI7obR}5aF<EJJ$komL
zd{mO_OCc;QY+Zf^q<d4{+WOufaaUZ>%8u+2v73X|EBR!A3D62KKb5lxi-6)!t^u6X
zv^6=jRz}C=?!ww4B8d&uW!oNq4@iAWxi5QDk5&NKv8Wr!Ad5+J_O&=@TmM+fdo~Ej
z{H=iirH^e@e~2-D6YQNTe@VqxXs14zfzM9Aww`9{C=EYn^*OxRUA}d+oK3~M`tjj?
zK?W?Tsv3ED@0r%hSAM$zS+g=3g5ldj=B}R6+016e+fMxJU94&sU&AJQ?iqq|^VGEf
z8GN_g{py{Frrvwc%jFlNnJMw<MDYZH(z6b$l#QlB-!K2dNn55}Z){(9-lnSh0HtKk
z7tK9;RpU39>gStc6%)G4QrL5zGXL7)>dr;I*~IS0d&_&ng{$px_oEREfjTh=dWnn^
zFF=1J^I>tL+QD<bcz-yTUq#BEusa_QB1si<pQ;jGX<Sx~;F>DE+W#Tn=4vco6v_r-
z?ds~<&p;@;ZN=rtUpw2Dd1_r5I>VHTB^kN5E$w$$@%HbJzGr)5WVzA5HRN)W;iQJ_
zMVd>&`QH9)HP?Pw>WnZX#R~rw*F8R$Ejuv}dPX-QHjvv)-_)dIC~@<2FKj)Q<@V7)
z5nTWmh{J7d<<aLFPNGNQVb~V_=;vbb^Yi;MKtL>K5xb^RR|Fo-N`D}D@$y2bgu70n
znEhV1y504c1Pz9c45C^I%O{;LKa04j#2<l^Y#i7lA2c{m&<Goj&_N)zcs)@1!P4&=
zj(dSmWI18af{<_4nBmC6v;4<ICe4vD(}v^7Pqb1w-wS9375=9hE`VNGvH82_E{UTu
za&M%zU|C<$$V&|WCM^Rnv?%GgSUxVyypumga&p}|RN=$+jr+iPn15ID%wdrHThcu4
z5QoXm$EPdkAo?zanV%~zwGnSQqN(tLph-&JtFM{6;ty3K#|dQ?xK%VH@s{7)UmU87
z!>|6dmGl7$w7@e#d!?@dlMNwM9^g3*QT0HW5MRS`yAdURv|NQi$2-+vVLaD_8vZo>
zS;(qpnnR`r`=);`j_G@mCf|igK%L4(WMz~%scl{=pB8FfjnqX@611|#xoS5UuJ{nK
z_(nXumYr2AnRq8q=GXU2@ALwA@PcSLx~MvQtCLGLTs*2{nE$J|#9+A7OkH{~Td+i2
z#&X+4kgb0bRW|?oG4!FH)m5W-es6DY^{p@f@zL8R86NrykeBRo5PV@@f%Rw}42kj1
zMJFVrA?~Km&S2p$0Ouwha?^*}-{i8J(K0hCp_c%+gM^ICekigJl}<rvO1KW|kpCt}
zU>^Tn%X<J20m3LWJG98+$Sj5f;Ze>m?roVgwf?zo<0StY3PHv+AKiZO#_2CF{Rqp}
zd&&=+rsZI}d*b@~hO%y_5((S$?(>gN%aQlWsSYa~u2*63ZO<q2&J0#f!Q#zmMMr}X
zz@b#B4;21-Z(hW9DSYA!NR=&5&c_(K3K!p###O{N>JIOOKNp^Rp}jmvh<JZnH{;u_
zH&t{2B7v@asnP>TU9YSq=Ux%5RC4;AdopNrq`d1hh%72*jsEgl#lh|}gDV`FM~`JY
zcEm_AbcgAU38JJThQP=e0N0cZl3b}KbfQTdU#y_F0E935SJEOrLFrOENdq3)Og|#~
zm4fBF_p3gsf!<(o30aAvZpld8;ueRN7#Epeba1f~6E1+!B=^(Yj~WxO+IUXNJ<An^
zE|U&>ypd21I)VEUpB$z16L$!hBLFrVSS-feA%ku@mwpcvn=+-`TB+P7cE2ZSqUC9^
z4SeAz+J$=hxjn;g5HrR6cej34-NCE8n8Rp=N(_K#ck5t7tGJ9Jo{S>A^0WRed?FcB
zM~OD=&rF@$%UL!ew=YF6Kf1CBexQ0L{{ETui;p`J38P<?0?v0`q3Y-Ra|R!X!B2`v
zfr~tDAGY$(y1KRxcIAubVV>!py16`cOqcc!w;dwCWyT-=D>HTjU6@b|q}#kLEij^%
zTcaa6ZyZ&_uSb6O#wpxMedG&I6Apud;jy9X(T*FR&o=h$hnoaHHa}-~2APh}_d4_M
zPgb(0Df?f=-?TEsS3*g|QcxQVj`l>y9HXg?_iPof_z_Q_)tDesd1RStn;wh7jzO_#
zD+j_tu>?3mD<f)&+!e8WsKu&7klT|`4t4k<=;}P!+HHd_*l)&i?onpOjD8`m+%4=a
zL#o9=vq5F{b4U{~9;Bic;JICx)b2=%f{Ag%Rc$3$pGGo+E9XR6sZa?h^JFvMB%$_;
zyy}P(GiQ<un;ZO~ys`g{NV-ScX}ueMZi}!?qXPG<&Fq*qtJuiOu&_s%vWYL*Gay9O
z6TINY&5#FF!RujgU|RyJ*i^$0;mz(x88kzMfKx4EDWI_eozDZ<Ov9^q-1z~R5(%h1
z$SxH_ka5<;hJeGj2Y@w+pHu>Sq1i(Re^%=K0_r2xmdgFvFso#xoCFWA=UGuQ3nl7O
zmfd1Q<=u*>r)Sx#9?<J3PF723Hh`?|;8Jh!;>HpHnp;VSHjC#SHGO@eY+jd-z&|D}
z6P~n!YHTL<uYa>sP~=Zu^12K7!0XwL0X5SZlZF@p;9Mwr?cgJZ#LYarO=eerzr{@1
z4HpIQka0g@e+iPwh}S&@RjW~J`p+mA`f<~MnPf$2hFH2_&l^1*-~ztEK&P7=rH#`a
zH$TdFS-kw2ACav)*&ka%=jA0_CkL^5eT>7PQW)WE4gMl_Y#*_-ZrDAW#w8dSs?-Pd
z0aD1FBuK%}<g;!U5!)@#ZXKyqeD+)PNGRpyn<bfV!$oMPzAtzWf^TrBa)q>Q-2%FZ
zu!g#LDSLE*UTz9c0YHa4)9&MSc5#92)=uW(ky@oNnsX~wNq}FK`2a^PA^p;;Myt!s
zGaNHzRXA=IBOe_DCw_4OqZgP&Iya6QOQuqo5mB|#M@<*%*|s)RWqv(9d4YW(U7Ura
z$rzNtjSr^ws}2P}e=PK%GFqgF=)3r3R_5U}40|&;XtlpMo6#AM1s7YUCau~RVKK9S
z=AwP@{4j8r(PE{dboZ-7g-Dj;mfwLj5Rs_$!P&^r%w`)!*!RW+L%K=FroQ+ODrB#4
zcW;!tW78`8ryp!Q3xc(o1h5!Zp-}oc?>Hn3jk?BXH@n1ov;Q&c&b6Xx*j}&&E_N3Y
zyM+6rb{K7H6^&}7bnJXKt>E@&Rz~H=m$+i1`^?;X#qWEj2@`A{0c_4@;@!Z1AfGu=
zD2ZtP_x2U%%~x)GrOhpoh?QuDlC?;+@8EJmcySzeI9WX0$G~CP%HuaKhkzo!{2=+}
z#^bf&^PtxsY)1$NXcLwj9U0;j3G8u)6pMLgf+0xC1%Q*n)Qkj{pQr|-9a`&$_S~jC
z<EC*aA6&bW@r#|*+PzR>xJRpDz7aPa#zS*d2)Rv&ka*xIAof8EWkPe)0pT)4CMRfo
zC$VF~lt78<3kb;^C}bw<BypRARhUE45%(_ag$`iu-$Y-XFd;&lj0ArX%n=D%t7YiG
z5Ma<KmoRP)|7_5b;y^_dXPJ}wcy~z|heZ~)@6R1_rV7v=g_r{V@K0u5>mhhEyB`Ui
z!#HbQf2qQL`faKTM#2JliF)Wu97DOVardTd#Q{4VDEiXG?~mudNXIMN&W2$N&cy=3
z!_s@v+$(tiS}_G|K9;Eb4nPn^-zLp0jHf#va((a0lKlv=Dl9E2JdE?~u)&OP-xJ<E
zpwGj`OV<C}+hSn+X@h)Jk_P;~{ogpH-L3jDNF)+?+PeDI3X=x8Z65X!;v-95^Ujj!
z1Sl1i-CvWc0)C;5k7Gy3JEb_Gd;Rk$!E|I{RVo=sI3(1O^qHunzVfMPy27o)NP(eV
zPDc9=m;C!nBv=8Cn9Y`02D~J4wOUmiI>nqgs28-)pDxSDO3XVwHv<Pg5*##CnkTQR
ze3kYpztm>!Sq9`odo#j|djP0Ilc%0nF37JcswFQ6Vv~zq`|XFiyzwl+VBU5&#mrtF
zGI3$mr<ozgyCPO~2~(n9^m(8;Y02WG_nuai{OQD?9S5V!RfPGv;-!!FZq<n&_d-MR
z5|u~VZnr!ZphF=Hixs+~3xFcu0F0lWxA1am$$)lHRpoiVoW$*kDjU_%=7%5i1G;1m
z3Y%;8z=O57Qi|Wq^h74(-$Y8e+9KZcx`x{5vZh-}Aj`~pv<?v}d?VM*j<hO;fj~pD
zX;rqC+a62qoaF2{k)#Gv)o^xJ7Pcrc)#YLq<6=Bj1zapH^E<W6{q4$zDZna+41%aN
zyX;oIV)KjD5T>vN0Hmdf^yvzb@Kp_ob{^L~>|}biD7D3U&Z(%pYQJYEqSu$7*hp~D
zd2pHUf{|CEU%|EDFg8rbQ<i8jV8;nl?IRoliTyNK1P%G`WPpzQBihkW{Od&mA<=I)
zySZ&yzPoTGWR*oSbRoz(?cAnGfFSUixrX-r`z4kygF}gLk*poW#3)L=Z&lSRG|67u
zY8&`;<a%s~`(kX=rM=w7GkOT!|5<5UFKVyTXwO~!i)vMkuDeW!QTcr5EB43V66os6
z#FJ=abPyw0NP87OHUcm@y<2Xsnj+U2YxO*+yc^GHOM)@_BH9%QZ}pt}<MzC}QmjBx
zT#Sy1qRTeHp4bfhfYS?RX8NRa8h=cc{_1Ayl6|H+k@{$CrKgtD*u#oO%0}&fBn<@8
zodX>RfTi=(V%GkdyDAJ|>Qz+nesp2ph;!apKN7DZG0akZot~yLx16|PW}lGM=&+hE
z7P6f5rZOd)e#7Hc2=Sc7dyF>H0|jygLm^-c=hK>HU0MjC*Gd8m2yxiVXEoS(#~x#`
zuy405RT1uvbIK}w>$|Aq*z^G<waAQrUJQ!G7uLOXqy9oL*G6Q8TwW-u2Z*i@;RB!%
zlndMITPYarRpJ?pm{W}6;xcQI*(0-5HiivQ>^BkTf$#-Gb!xzRyDYa_>L(+YLP5g_
zi*1P@d07vC&@OhhibQ+g${+~ogdIjlqZ3B5UjV=49MnDoKw=(=U7Et|*<@gL&JYjO
zwf{z;@gfY$lODjo;*m%K2PX2xc_qq1X2_^wRg2~5oe`%piTK?W;%S^Cmjc{?b9g$f
zB&-9p>AFT<N3Z8IRGwC!A4$pkqChLyg#7MIrzaT^TjU>-Zb+o1K0WTF&uv;_h2ezU
z?0l(QvwUg)9d|2p&~k{W3)ZZ~?ckHM28#92;kbV8Ynzl0uw$R6-Trw^JTNfM_C-h}
zP0!X=fzo&QS_ePk-w93iRFRLZkJ&B`2ZwzqUuEeE@)F4c>bS5b6}ScMgux|p4f^_N
z^gyi#kCZzM-MSL?ei$_nQHIrkrbB!OMBH%Dxdhw=pH#s!$_BqM1dP?22uq+nTVpF3
z3w^wYbdjIz@+VW|;@N#?u7YMk@#0enY4Tduqoqo`)pgrPEM5(sl8_qi#=Vy>h?J-q
zxRWeDey=D2PhC9|+Rg27cEQFla{iM1!T*A7r=~L4PEST!^Vzo4=`r{upzt?MyXwmy
zNXeTrKGR@3T?241WGMch<1-X88?sKEm>L&RDFzv<FE|DT+@rPAb(<5j$UN3ua`>-d
zEB^SeUqkg`buqsVMC|Xf+~~!tlp$$&D$0J@)iXRSUL1@u6+O4ec!VK<06vyb5?&t7
z@hs=_vUatN?(nN7`fBs8>HSwLJF2$RUj%xIoQq0Vr^U6bNvd4yyO^bqYC}a;oQONB
zyNp5d6O6H`$}39QZ}MW4Ru?a~rntW9MK>=$l>D6q@OR(_5Jv$Op!!;Os&dYX#+@@i
zm<L^OBW=wokR^AiQgha9j}CdIKB9ca0f=J_L2gmCg0LWeUKUCIgEl3}MaOzm%S}li
zV00J8<L)1Gq+i_+CSaE+woT`C`ScF))%xu4Vq`V(Nhw}FpyMTi3h^7j{6o*60^St|
zk$mffM}+D{3v)yr5r84r^QVR4zc9(tCo+2>;i?+Q?_Y~PV-m{MUx-F*JL=Ak%jG<n
z0ja?}?^$8OrxY5!b3MRM!if(t`D{zV#{@?vI*~FOL1K3?++RG(V8QBcmH8+qoU?@t
z1tE9I19rY9d{z>%OwPcwyXe6UZ1{}GhJ(>%X5kyT77(2#ivy~S2)()%i;IA@=5tho
zcD{j__?}y6<giGRm~J5kBKkuc{?-+6FG5IhRhP(ISttymNQDg_rBgDYER)yRtf#2T
zH7he}IpaJLz$CWy#eqqpmIZx4&I>>$A5L=Z_YIKxjX1#SggRcGpHs#eLbAJ^AZwz@
zXC=UrR!++i{B)^g8N@HP^Xv($Cc)%a*Vo*(?1=?^_^yo`$H#ya!inYVBWfA28;O3Z
zL%-cSJ?Fzd{h~>8<=^7K0I!(8;wk*Fvq|=qj!9BNCHWg4#4$>gcAkoE9Y3;1AE=H7
zfa6PykQmkuiztjFKIRq!%A<ItK4<kd2fVKp(+7>yW6w{P??d5vFh#Rz@5<|GosUdx
z@Lg^hq+{nK2cGo%xN|o?H+wy=rXtXrJrxf9f*p~%3>A{XSpar@|HZbBE=#r5q|&}7
zU<;1Wsy5#n!4Xi12E+`Ck8R0{pS2y1>&i1JJ(#>aiO8ehNWo7k0fI%I+huCUw1H*7
z<}{l7C<4f;7?&pTYTdwgxet-^Blg6TTq0)>I-RU61B0jo0esS&#w=lu1MZ_fB#Mt@
zx{*aO7;Q-=QGGdOU-lIFRHy(VsjXuqKar6;EK-2+;F3_%_ebX2cS3rB(j_YA(N=`K
z7+vDHYM;f3%YUvDIA7Dh84J;`6v~Z5{X%E{_=3&tW(sh{`#muvKCg~XWY`soo0}`^
z0<02t`}0#Yb}#}u0*zvpsLjtqIhTm<$&b8z1M&}3k<@mf^Z0%!zIWD0qghQ0pSkvd
zxmsNUL|0`Py@4&Z87<@js1{?RtjIP8o_kqu;4)iyF-!5<o}a!D%~N&C#Y*Yx!<3}n
zCcoq?wJ$Of<xHG4%1!9j#14~V2L#45<r>5lCutY3nW%(UY(gBjx(0^6i7Q)}@sb~3
z{IZwel|5e1438S^3NpEvf6GBfv60Xe>EsbdnMWf{U$?JN9FGwwt<xb^@2-opg0*U;
z&?<g;N0fg9x0EuHW_w%6fWsJ}7bU^khNoD;U6R|+V3za6^2gV&cyCB-nx4BHRe{Wc
zi6O{Vqve+!jo7Jwqc=n-<1hm{Eq#WXUP-TQ9W+tJtYP@@X6Y}r(VLDVTP$t=!f+*L
z3Wf>XFF(lJE48V2UNGOk*m|XL^(X5)?lYE@))_C@qZB8tLb`A@OZv_mAjxl9U&s8c
zZqyfkf3z60B(Su%cl@f^X%it^$cJycpq@tSw>lBg9yfafmfK^i8+QEmRD>@cOe;h-
zk&h~<2vG@)_4mEVXW3L1b2N>O^Q}pCstFqOfFO-LO@wkim<nC=Xzs*ahN0Qb4QJ_(
zHg{TUO8Sc+3D?otLPd(}aYgdkPW00NRB}-;DY>_?z@g}#NXOiuz-QHVW5is16j42I
zv#4!36hT}+D>Jl7Gor&a*i--EodIm-R{M904a}j7pM%PqUrA-4KI9MElo@rj%e9f8
zZv^m#A^)z~lpW{-{-a;Uh2-13uDOh1*bug2=H_NrEjgmT=m5%9*dmIeO+Yw&(k9LC
ze)vj#(E4#JQR$_fdYr07&qq|bxAIR=yuSe73)dlrn8(%OJlsYcD2y5ilv1;{$>93B
z<S4zG&hZw6p(63h9caI{Mre^J(!<v#%QYA)YV{iJHE3}*hPixDrZCcl#cThg)INUw
zXZgOqOPn}g6z?2uq{AYpA~cA8I)_WPF$qs`H14P1Yw-9wnW_tKt>ew0MW%H^!QcZ@
zP4ZWccT_Xnm$5S++ZW{%y+B@Nz!{t`j*Czrw~>|G%zvv`1><qYjw|{7ZZet!Z*L=@
z&{}G6aH#3%)_DW5M=1F_qk@0e53_s#*t2c!Me}N}SEP(f1tqZ<MDe*=4bBED);pQi
z$jK(8-^Rzlm&v0;!v-f4^T#+oL0Y46feas~a;<6xc!U(1I9-u$kq=3SpaMq$h=_$^
z<woB%;|6cBE!@$(i-6jsiMMwIq`BPj-6|%s(GAoP72nHi>vPR^Q!xo{8YvM=<|e7l
z4x!Y1_=|0IIxHS)<gMOc9+IC1Cpz^x88+4>7I)Rn6r2ubC{nAbr=s^3>+h9cMCZ6_
z&NR>b&)GNt@!&D_?uIq;28IeyD&^FPK+{Sd%g80Yy%_Sb{t>5&H6FLYyAhhwD%uH+
zM3UcnjxXdmg`x<BCa^nj?(3beEQT1*n%e{>f(MZhnPvg<*ybC|ZiI^oSv}iVhC3fo
zchCv<EPUvS$6L7|*lV-4!=q91XF7iU1D&c|F+=3P`Po|8dh)%2ZgQB<ws4tosGW*j
z@OriJwNnlb%M(M|b7sEveYkuxcC*NYVG<B+YKLdNncK~`w`bcL*dU#wHks)_?p?^M
z)!|`hr>L)F(bD#x@Zujc<UPaUb3r89!XPY}*+6w#K($hZliu0dkyW*bDNtqDmO%sS
zJf0(%wd$U8m-Tj}#+UL%(|BA0q6!asxcORVfTqLETml)5LRu%UmyM2{H6SO1>sVP?
zVX#1Xn#S$A0rrEHW=gZRT_nimQuugm?gAl?`9QYid2~VDgXR>XdY!rKFHH+e5Q)pq
z2swA-R?QTT6TnO0cg&mJ-aiA>k}^QZqCT&$uU~9Lvc9zY7r62-B>N-aWt}cJ%urev
z<eGg3-E{kbF>w^@S$g-HNMTt2`X7ZS*)Zrg(U5&!p^_g6uZ|sy;1jEA+q%@Cuzb`o
z|6_1q(RuhhKc99_JIzeM{8foHTlLdOpWemV=D;X}!R-Hh4;F9{-#p{7gu#awBDA_+
z{9fPU(vR&bv(1!C+5F}W&%8oO`wU|nT1PO@(L3QxjG8s+{MDL*m2GF>ed)Wi6EGoJ
zPgRif?2m3XrJteBY+32(!V0Hx%(^e7YNsqnD1Xz5W37+V-b-8JIoHEO{5g+7cx2wN
zGa-yndh1Mi*g5{5O#3yVc}90d61g*dc!!Q@vlMo78-dW&ccXUpFZnJ+U$LlBz;e9a
ztEy$1(~ltClW;J^9>m^>&OA~cK!8qwqRzx4eop{}><PC?R#U4box6vDAt4fM*q=uV
z2YyB*@h}EPJ<oehc-#a7(WJoLT)!M-oNinUcMipP#)QL~%{qRqcbl-VusEWkcjDsW
zzk@)gM!u+$m=5~Ecyc)jWe=vT#jkLQh<F}-olQz_0}+=+BLZhV29RhF(2xNP{AR4C
z1ajfDB784(91odBxw=Z3q3!4^rFSOH9O979696w4tX%5hCW#5SF;V1)r7X&XK9&5B
z5)}0h;R{&$-ux;1plq&|I(revmitlWmg`Mv7fKNJt47|fXQT1xf?l2h538$_hx&`E
zKV)m2kFg@5JCdc6K1yFF%f7Hq>(Gc7(Ng9x$QtI1xe74FSU$Kh3!V!cu^RP<7?S7*
zes@BCSrkXV@k=69Y{P1BZrZaM?LRO$C(Jb~jQ9d-KH=%))`osu!Da#d!=LUs6(!Q&
z1QdzgC4Nc!=G!D2KUK1N`{fKYVm=AOA0&Y22<Z%|RCac*%4&W;YnwX0RbFF9ow*jf
z={-p@gGK}d^Hq`G^Hao9JZbKHIqDLk@b4x(oV-t#S?oE$#((#5jcm}xuQfY*_Yn}^
zgZX|1k(iCTm-gUt(RC5~gKOh`FGP#%auBoNZ(hfSF!y&F?qKIy^3P1>(gzfN4vR{R
z{vHUFN#zZHW0M^|R~WASI+MAs*c)6yTR8mi2590#bD?xtyILSSObl+lyj(o=)+Ce<
zh!UgM{q7QxvVF@Ih*+HsAl&679}3Z+D^Mq2oQcJu!TQ;r4rsys;y#vSVfw>AcP3k8
zU>UjwJs!3IQy~3;-Sf;GrmI`?;o)Jg<N1-b>D#wCcb%_2-Q7H_kOR&2#`fDCLNsnd
zD~q@Gy}iAT$6H%V0|3!#O!Q#p^GjAn1{6)in{bXwDVd7<@7#*}XW@&vY<kesntt1F
zF4-TAEG&<AMM|9%BCu3811qXt=k5lZ67&Uk`nQ*i|0P26KTiLD(+vN@<3w^OAtOQK
zrM%Tkt#aYA;Yqz^+DphdpC;t<w+`0jw<A&uZMdta3$pzfT0Wu@0%&0>-OsO_KwL^9
zqx1tv=b2yQ<bsKp7pDluyLb7U1WW2Q<9$ykxZ?Mo)ZpE(VO3Rs$p=`X#PCT$kES%n
zGQ_P=ah$;%P&-^6P*XeQFA)hbE(%dupdUIB;g+D+@kChdzM%GZ8{UBQU-N)fQ*S3f
zcH#<2)Nr}zUHnRM%)-IwhVLIn=#7P<Zm)d>EhR@l>th8=lr%y90$5t@H7N?F__#OO
zufG?X*9%Q7u~0L>bM8@#Wt4`flsfDJXt-{ZQ8kNXFp{UlNRn3Dee@);dET>_@k)3;
z>!bkSrYeA*MxUm#8fjW{y^ti){3G?!{DmvJlWn0&(Q8Zv`8a}+v%SJtRp7@*QX63r
z5d}apN3v#>zBP%dhIB+epw&t2=;%l$LETal|DGzFNK?EQWcqM(y5I741$GJs&^Nx1
zaK*;Qw~ErMGY_Xo4C_u=Vt+S72Lu561HQY#IP%|_b1iV#65rEb+u)97^UuDEvi}!E
zjsH_1;8>)zz1o6H?okxdx!pr+)f2ipLEN4*`Tz66pG2WBer6-{s9{R)`jeeQP1R@d
zICa0ng3rpjZO$b{{UW%p!Kg(c5hqRMCIj4gWgjm0`6MH8-gz!MDg&^v$J$2>5BN2>
zwaAZ{cEhQ(=`NiN;<3JZIWYYZh7b9{{_<qppVqV%=*Sy3dvSeAQ9?x@x)5Cf>i5ok
zRDbl_AO*V+#k?vmbTpIHhgwH}I6<`Z0sYkIaOO+`cfjMScsB}kzYQQ;5oES;v*Q7M
z3RP+0efFQx2e*8Ih+NuVEZ@9aZ5J$;gXDPuOU8p`5Splgdqfr7Ir_9ak}mEk%EUHK
z@Ak)u43t?(zEOH<j`}cwVoCkfGo~Oxq@<)&uKSfCnNCH_=WTofb>XT(oVX*Auy2<0
z=9l+wZpD0V`^xim7C+iWTxAvNv+}lxyFbwUa{$dr{uCpQn3YIx3BddwE{g%Q8enOY
zU6wk-V_QWLEgxh{xDRXgr&0g;)ju!p@6=QxENIf_m#&QD8TPBql7E>o6#Fp605eM&
zJRYN3K!QE_<36$CkaK8OIt03G{xhs35%^OLY>WEhRcL4<xaot(GO^7s=EGyzQ#Ni{
zxf@3Hxi|AO@8;*f=f=jyMl{Fl7R3Rs4qluW-=N=r?xz%orlL8Eph#WX@Ug1>L(wzV
zTCOaFE=Sr65?Bq*<D3m)B<lswfsCNtcld09znG&~n-O`-K#-z>P;cN%d+_=ff9Cg)
zH|T3noM$Z71a&9~+xB*T2KAee@0?nRo15+Ue#wTS6f$-|fc`Kyh)-_<i!QlFYDR7v
z)I67G^pF?asknoceIr5=>P*ySrr{gC25fu-zOQ1EIz3&9w_{oP0K2SW|DbYqd07k%
zi=ZvZJ=ftU{0Q)l>^?cEz(m4La<+SGc@g{{PR9zaF8ohyfh%C9#|1k8#v8WpYbhSk
zDyezTyoqS{pq75LdHX-V_=m>-ongBr1XKL3zDTcjQsK}_49SV?LoZv`CY*s#lir(^
z-Q@T5SC&+_$<<;`w(o^5=&+bK(#=(ZDPoB8%=l*+Tgb&fmg4!IPVp5DOsHVDTH|Px
ztp<iuhAV727Ei1hU~9JoFl4%umOfDTN-#bn5uvUeo&XFer9|5EW~^{hhi{Lt@_GM2
zpLTQy5yp3=SQO>nm>+QLG7=KOb+f`@Kru8?oP<GxLJN!I>V!rw2!^1E)cKPX>lBd|
zqqFtKUUV7m1SsCXPjr<uNf5Wfrb{nupiskY9!6_qr{6-;>fq=Edy;FDagk-RvucBI
z(v<pTh~35NMMwb)xv$6^x?@xM!>|AZ$t!Gx@qCbo4$u<UTSPS-58e4UyB$;wD)d5=
zY&r(gfq-<Q0>HpVXw&lHl7U&Tfk~Q@Q2n!ZJaAylC67f&n2Ra~QQ5<5(_I;W#6p1E
zftGv!PiBY+C8YXK9(L2{)al^OU$(Y?vA+Mjl0gnQn9NJ+-ENYHv_~##C%D+X;q|SJ
z<2M*ynx(H@KRDxKF=7susOv<uy5{KXFVmfB!ek3Nr3pmbO*C{c!MzZ0h>DCXygZn#
za5Bgt5yk5kqa!P2;pN3E`fIok6i#}S@-BVg<>hC5#_LhizOB%2s-uVk1k9SZggnj^
zeale=b9;oEo15RFDX&IU{vPYcYg{hEiIUa=UNMn5`-_tWHWRI(%NX3iTeidWi-Dh@
zyTAGileOn@yMSJy*-gsp?V#rbM7>p`%m|z5@{IExfj6>3!Gqz``Ayf?4MyqE>ZK2)
z<Av~{=|rVxk!MQgdbT!2*TOIyj&AmqK$#$*y6ure0{HqV$6{|pkwkN&B#}BM$R$xI
zJU$PeU<hR0#?Ax$xdM=>xy4M4G9^n^hoCBNE*&Ra0cfpfN>wNsxsAI=a|HQ&=wE+E
z*fka>F$Dl`w7_zMe?|uY9dY=JlWQc+H~jXG6X`#%<nN>)?`V+#7)6B{$4^DQQ6(Zy
zEZSkypO$t&Bp$A|%;}f2e~EZ|Qjcx`@^lK&y3yrO2K;s3#I%+>3dOk~j$et+#9<Oa
z37q5Qz{q`BL!r7VH6xN+N0Bl324^W;6Jod{g7gIQ04&6=o#;ZNaEWx-8`+#>6$&YV
zB2CA9F%n^HI#1-ChDQm9yk{<6DsE1+->_-ENoO}}Vc&!2A2irnfTe6fvR6xj;j%11
zdY5ka33y&3!1HQ${UvrIo|NJ5^O2JwKR=%#SN)y))y4L3Y6;NZr~>zXWwYq9_BTgz
zgAU(iIX~9l;hR_j$MAO_6#sLZ6aCwnUwrs~Kc;`%tStvP0RzFI<sU(m_q58G(x+h&
z(dvX}mlt1?BqLO8y<hY}oI;=7%4U-_4|S>rN&XJXY1!zDtiT*6JXvZIx3{-v(DQGJ
zC8b`5`L%(Dh}0xL76P#sRvEP<rUS^LY(d=5hbI=6&|mZJA)$b?Jg#7yla$TdUmyC|
zKb?o%O_Nbj4xP3WQ6v|F*-0!y2z<0P@Uc*K1hjnQE{>`DjhAn)n7QhNfXRzFkaxs%
zk?VTCGBbPrwTvV^3Ql)06#34E7FSr`rauV32E$(+@!3CGC$`y^sGX_fD=y@lwd93A
zGZ41}FQPoa*T?-JL+G~WL>4FwuQ*vJAPnr1;dZD3;kV+61Ig4sNX5jR8Ur&Qe#hk|
z(kiK%n^RFchEaKKY;1^$ii-BmDNhzis-*>ohE`m#g0i<je`H+tU<jeMA^hhJ{0{>3
zpIhedQ1XAS`)?iS_!Fj3M?y`ZFE#CZ;i4om+N>KEj8FP~)Crtk2byl0mcFTMz|KXN
zmL4N1{!aqPw8nggpPv{4bZZ_7G4UIZbYglnOgRLV!%>+tMx&4o)52fgdn$@Um`Wkb
zk;=Wbu<C35@D`*P82Jx-q`aYjBpIKMDV}2tBZBbI*WvxaWJTb;;?)$M00<jC<pVSX
z+mWmWjt7>fD7K;;OzmoER1@THMD{e2g*anjSCiPR6I;tpY76-MU#SHhzZ~wZb4(&r
z9NdrqyUf>Lm@%R4jpl_b?U}#e3Uz{n`hxEA#E}HyeDG<ZA`Io`fQa`#50GnIx3~J>
zIK3{AlPdXJ`kfh13dESGQ^j!EEht~^|B$q6l$axbwaH@8EFp8gvArz`pc`Aq$M&;A
z;o=m)FCqc&P|7bEGI!#gIAHqyeS1KcFaM{biL3tW_LwCtH2mN2mH)i3-=`#0JP(Y;
zJEqSsj`PT%0waBz)?t^UYgUo>6FU`Ox=$c7Q7^xHYx`Jo*50sWT1zp$I+XPm@0|Ak
zrYnr)3RlYsOv6p~w5h_<xKhO_mDOQ;{t{^V)~rG??sC;@az?i=&Mgk6qX5<6Sq<sH
z{l2oTVAo*$DWTAy(FCsXR1rO(3xbIE(bvI)_(|R5acnzI1>)8Rq%N7@iymQHr#!F#
zT)Mn>mtsGV%rYm>FHVhMAz4GGLx)S@3#7W{a~||5-qq5RTjzl#fF%opK{CX9p=dtn
z7ov!w!aJ#_J59<!Pjz5n$1b%afwU@lLpKf=YjLv~zY15T_TgerR8g_crvL~5pA>$6
zFteow>}~)bBZav$xHl-a-5ZYeJig-ADXqbIO5PTV`+;91LVYn*8y)2h6J3~r`e6Z?
zZ$#i+P2rqmty$L!6b^ceuc>JH1?C6TB0S88osLw`Lj#sA7R0TD5r`MC8X$e%=lNP}
zrR{N*DL^1_gfyR~E-W7|R=I_pk5p0wLm3Vm8u+6W1h+L0TKzCLT*mI?UmAI6Cu5|e
zt!=e1K?zS2vrjR1|2nFJ7)JEvq~9$_9dd*jS19RU5<v^o3Bp9a)}$MiRF2pZ4C(P;
zHSH?Rn|N<2S$oZ50~wev9i)?P3%%x88TP{Vl85Wy`Ev5ZJ8|RZ;(ljCAw|ImcMUqu
ztPzBDK~Ew?GT^yN;#^)dzHSGs;ExLTyo+<jJ%pA7#}3xl(>IHs+OZqxeID?Ncvwx<
zHAPh2SPt+JHdgS=^|#qRdA3+_dWE<a4W#!{6c*kxH2Vo7J2~8cbLj4-q$|KTEAn1R
zJV`g;WO#|Bw_JyJP15m#rCI{wgM-^}^86t9fGV+OF^*M>a&}r=_b9IoH_pH0<KS3l
zo%8<o>Mq7Is#WqD^;$||F~XlB4@3v^*B`~?6M%_-s`=q(Q0XbFA6nm+)3R(Oj>~VQ
z9(R`OOy8Lug$Xy$jkTj4eV;d;Y;niNf85yaRnM%AYkI`0_2#130@WI|_|vBZA=t%I
zIHQ5DBZDLPTSYf(LGT3mzK?eofOYbE{gutF<H_vi4tOo7WJ$sU+Mf^2J4_<ncm3sB
z5Sh@3tgxZ~cSNm4Ty8qkc~%({V0S=oiZ4IH#hx@%R1n`C^(5Rz1nTbqkPYV_egSHp
z@a@jVb*j<=>Dc?u1$zTSA`-M@Vxhia-l{41m|BXs#W%7<(AahuH{wuPb4bcon>CMd
zp9%;Z#M!^p*-a)SB#6NxqEXovQv}*+0imJxPaHVeNn}ApPQ4g2MY34C@G%!h)xCZ(
z!zs)h1fp@<l<JbWY)NYv<ytbk#s5CNP)uk~t}_}BP*V?$$6z3lP_G7J*SSlyX`rhL
zIqRKg8x;f*mgz!LN++_Byv{CP*hN^K3!^m*sjI+2a#b&%6R(Su_3MUbhiN=aJNP`p
z$Uhv7dJuV1WpW1|UxuimlhldqMsRFw{HpUM73>_S3x+B|z%Wf{4rVIwjegu|-FY4C
zhj?0F*;?7L%%G^O7b^Emg8e(sQ_jfFa@5ICEqh^B-&selZ4AR}!=X*A7dID%P!!WW
zY}JVpMYgiRen(;9rfDYum`%qkr!^&CS)rlPJF1^#s80NtJG4w${fvt2QUjhy(NonU
zSM|Ob;ccHc-ink$QU*)$6~_<91nyW*hO)D3hkK~_SaP0oSEK_6hsPL_x1b9^?HFmf
zsd^iIY^G!>GSDUIEObx8q^ud6m<ieUF0551Kc7xi;v`Ksu&9W(%|CGj^a(E!(^iq)
zm-|GIMrx0Zl^6W`od5V(Z-PXoKfQTmWBl>S-VE>UT&K%j$3ttD$StqVL(_%P+efGC
zt5G4J%|x5ib4N@Z9U~!hKY9$>1f<29Umjwl3pJ+XcDn5o-Yb2(XvsmL{<r!V8E{~4
zZQL9vs7`r>`lHnJGgR<ia4uOZlq0M$Sny6VwjqxH+q=6XF5C4;)rTcg&E->VJ8xw<
z@5%7Cn*wZ%m7w}Z1`FekC}v7t9eQds8Y&0sL)i#J5Dq4?Zy2;Fsf4Z5nTHyK6A-Sx
zNMhE7MjNYL6B0Yqv=6eKJn=o)Zix4wJGJg0cne9>NoV(<qN18u>k2|D2JQR?*pqIf
z30o%+hG3sPXFuyCZftIj2Jqa(VhkKFw^%PW*uhN=11ZdpzC)BQWnrqnKTs8ny@RxW
zPdNHJ(BKyg5Xp_>$3CNTJ9k6x?(-ZV!>h0y8MnjK%+sWbClhXLdUNsM<`Yr}4|*4c
z9trD;r#v<*#6(502!T~V?h1ad6;JZq*#+Ol+q|qj5Z)M6gMq0cnU5joQp+atVIge~
zUmE_i^O@&qS~&h-oc{~-(@Fb4sSk>L0c0lfRBxeWWEGv%J&;n;Q%weu<4xk>0t3!7
z0xBC07dAMCL$TJNG$3BHHJakFRy>WBzw`uM+h211lDtdY^nh;D_77sdc}7W_pUSg@
z-7TiLAp5#?G{hkR_U?I66frlbz55(UOu+wQ$Nhzo8~Phe(`z3DMDm`oZ}G{wl3Yl%
z<TUZq7?jqfdU1V}jXXW#`mR~h2)NqDG1lqUeLuVKt~NS|nc3JTHzO(bU}LPUpJGuH
z!mb;tWD2vikms2vIymSb>Ufx@9y92D1~k8)i)M7nuWu#Y?aYXLo@+OW_R08USgpet
zI)T_nLgTn~{yZMy&D?!^%?PuFN+TK~`OS#nXr&?CNa5sOP`n{Jsh>HkgB+r?Sl88?
zLdtd;E)a__RWwphoU|#)<;175z8ZTcz`A>aX%_p9#!K#WbyL8X7mZ@*srNoa;3>E7
zo)G8XlJ`gD>m-GH$#wBCvJ)F?$Klw^1H_zh$nu5$WO|5<Ir0<Q-nN`Z6G%PhI5}+0
z<=znwMSHDRtpMbF)Q`7bq3PZXGw~e`Zoc`{F7u@D!cVKDP+$Wa;^Jwq$@g|>xCC_=
zPbGS!(()h$l~5pe;E2~+&;t%K&)m!Iak2SDZW|Drtl@x8L_j!JCtgjvRgu|qf%O4k
z-<lv&IS4XY0jz_FsaR|?<b`pNc)u9aMUV6yx{_%G@ttu{LsaaCCW%1AeWt|VTm{TT
z5d#`9A%D9<#*Z;18Db$!X*vG@tonfj{0Z@1P1_;~(tVy*y9r$Wvcle3!3-CKSb}8g
zAVknhV6;D*bB5RZ<PGUUOv==A=TjD1WuX6~MPTssye#^oJNV;Kk%<5;!eHiFfTX6;
zJU_b71pGIs5_R$&GMAak*};?Acy`x;p9!n%WjYmTtT4?T^v6my%PsC{4uSV(dKz}-
zr+7NhBi$D8IaYL|eIKC-sPr%OKlYj&`247CWD5hWOnq}VE6>!P)iWq)Be#P+8d=g2
z^Q*zG^dQ{{PV+GbvfLUc8rVT37_Hk<p)|xWIa5<_<cC&ZsQK7G>b|y;C2P8&tZ1kz
z_}u!L{f%gv)$BK7$&eFDuX)9lS<qD_Vf`nbu^E*NV?AQ>$l_hgi3K|wISy29R=sKu
z{=lrsye=Uzm^;z2&QyU19B-W>%}QO8og1bc6*+`(5n02^780BG+E8e&Y8|<j#^eV-
z-X?q4cQfT}>9Fm*-!@L3Jx36u=Ypx;{j5SMadl6Wl*9L+=Q`tt<w(neW}7_;J61T3
z<T`45xHh>p>R_iwudWm*wq8_BQl5VF3Oh3W$pTQ8^TltcOK0n;e^}2fafJPlTdnOo
zWVSC8D8E_kH&Q>FSbx^Bov--+xYz*^&;xL>?>sD$a!4F59V%qp?o4w+QV{8U?s0w5
z#yAIZIfU8Tf3x`#5FqDyvi{!C3B}H!B_ZSMd*KpFEYb%rFu^ef;`|x%txtEuWd&Wk
zCG&UPFJX`vo8@M=;xSqNK<Q3&fjW=KlwKd57eTSQ?Vvur`U}gGuikTd_5MasU7;LC
zZka7m{BtszXpZo`U;*g85Me0(Ju<OwVA}@QUgD{8DmZk-jUO$OAfA^zrZ+{qkH!Lz
zy{V2_kl9xF1q1C1#u;*U(YbB}JMRQsK`mm8ZiG8H{1cv6DNcP-rjf!6Q?#{*JRTL4
z#@(75qOnUwZ*GE4KYx$wf@QTD<2zugyY7c6+XP?{P95$W{qo8|ijSBA<wwkUv{nBW
zLtL0M+oa?W?DXSDWtELiiny86u1W?>ez*wG7{z3P1)+6{PGD4x{b&THJL=2V#laFQ
zG~G(`^(MsI5c%kRrlFzZ&nP#oF#3$DV%99}j#la|7{n`Ej&ECV77<9z8%rDtsR>i>
zmb&2^E)x&bN^rC3vl`7$am|x%TrP9ffBvLd@$G7GQq0b9V>fX`TV+Xm6Vm6!l8VdE
zKoOyM?BQ&^T)gk6d!`o7L^8L4U>@~}el!aYY6aRk9lOT!+EVw6g<HA{mJ!q4rtS~<
za_RW2AJT>_d47}4lJ#tr<_IBUqx`9ZRxn=X>Y`E!4bh7QvU=|w3O#L}xh`Q(d&V@k
zh{0P>nNW4!x6a+GD1Pv)3op6t4^3hmrjn&Fp_^Wz^L`p9i;#DHCap6cd;g&N-VNy*
zff=o^LHhdDo%fji`C|RG+?+^&V9VnB16jAs`n3ZKqgtOGcyHDsJ6)v;ypi=IW5`O}
z2sbj}SKGJyWOgIZPx3IQ-KoY=$8Qa{Y>|RIp6@q^jXuRxvAjjFp1&2wCMJAa7d7Xy
zSpR{(GVMOn?7+Z!vH;%gcxhfdy3)<5^q+?Nuj2efw)ST3YG9y&*SE9yqND7DZxMAP
z9lt1wqPkMjj<k$2bEQMU{ranhiM7*0y|s8*Sy|FJfuK4X3(T+d`Fsh;-tgPZ?})0*
z8<#L}`cOlI<mRXs>*zU9TEwb|7dNlMgosRsHKtR*5e3qVEoj1vXq$m0P=at*=t<p1
z{?RvUA*gq~ITZnsa40is2&uAiMD?N@of3TrSiMMt;WD|8{Q!;?c7)hAM~<)%M!D(q
zBb?EP%!VxGCLb+(<joxw$@}!%^$PrCnxFw72SARP@9<s<%P_%MFJ&~7d(*r8>!085
zrClCv^}pY4-|v0|_b|9O0CPMRh-oGF*RV(k%_>X@#SKVG!sZ}rH&I(?Kn;B-mCu>_
znC-&(7P&MtNi00;7Dk77s%}c*C#n}=+~e(lQ%X9*5bDTJCdhJ5`?BqQt#EG8kN3JI
z)1|V@pJ*O5Y1qAApHu|BR!?`@BpAmYa}KmqiFjfzP@pj?wkRQ1wRIG9va2aZs~Y`e
zkU65sXQ3zrf}PKx3qwa7d98jmr1&{oY%nKagv2Wnnci%dYoW=ScwH|vSr*bq>J^EL
z&Q1F!L%kR&Gx^uuo<7d+3H0wl^aY5D>Tvj*cFt`-?JR|DrgrZlX@tZ8H6$fehl;uZ
zBRVSPH5akco;V?o*~)^XUwD(WPKiq7hPKDsOV$uPhx>*6d+UCO>9dd>Pi}P~Jr9NU
zh}U9KYXQu<n&$z8sQR#(bI^)V3+U@8{$MCk`X!i~brC|LZi2P)#1L4YhS-aMXXCfG
zA_kaxWVjoqw(GA3)7Zg#)GS)ro`Ka+tNs!Qz#EQcQ%MhlA_MjZLra^=>nRTY?7>64
z3nmN1I*W;J=&d~{NaXFp1A?SwZ0pI{b1J2)W<Iq_4u4i&E!a*oi^&H%&`hi@aBJ}M
zenqtv;N6d@;?pw65Jw{UamzB{qV?xg{`;ATisVFZ5zc$K*Mrdol4i)QtUX_$$1#)D
zWSdJmP!q_x@ZxKS>W%=D%F7kM;Q0+07q92pYmbBYulZb~48o+;9_q{T0)Ub;B0Stl
z5dyfVJ{kGa3vPy*mHyvmC=Wki&&vC`?ZJ8H>-&jrBg^!W_hd@aQUeniA%5`{(<+|?
z8lG>HQZwJc^Ufz_CM&{cd7AZ{N!CA%laA&A%~5ey7EuPfMTB*(%XC|RmG5~^)N0D>
zVW4Js(?n9e?0A{3H~KSSkf_wz5d*{)D9tLgc<H@`o%`AYiYHi`d<#hrtmX)93>(u)
zBGk$1KX+vcfMs(%il&R3c`QWmRT(_p+#-aJ=)I{PrDopYD1xpO8<a=v@^Eyq#vANx
zr>R5!c>&>ht3slSh;Fa+9NA07*TBN*SPvn$FxkYUZvOthpao3Qc|J0Tw0GDArO=^c
zSVw)x8XIL25(!{BD+WY8kc(@is|o<QToICdT|M+i1-XsREBVhUL;$8%%s|8>t}bz7
z)r3&8{OtfKk#>WeloU(DZlb-LfL@g1COGz!APYT?b{YfsskpqJaYLR=+-jpO3`D&&
zecv;jnFbTR!g0pIVlUy=*&ZpvP=o+r*P=K_9BHBY1=ky2h1#jKwak=Q!K*Y5BXgOT
zyv{|_KO_DNJpec18W;uxqi$wD)=f6EU2929Z?>w)@~8#!lL;d!yEf9Y{g(5RgncOq
z*1^+%s?mXMu>t$6p>pwOb-_)>SF5r5>o+w(AWWuhuJ%E2C)oY#OHM9d6}DRSZzcX*
zmAa-6bw8OX%s~c?_Sv<wq=?`EbYO#3(15N`jU)=30MzGu@tPf0(?p)&_hgR{QQ{>i
z2fsbCgko5?)5b4uok|ZTlv(geJiVlDAh{b6bZ}^+uq$*p9C&buz28BQhQSGlE~T;g
zqVh`T0>$x<<Wgk%pb9D0M@+C298nyvWBwW}|AnY>XdAl3M63|eL}8K;0hWV6jimgb
zC8V>x0T$r}nb)&EI{%iiU`_`zC0QiC9Asiq^zXX_I6eVB<5FB*u-`!W;&Od*7(Ch_
zAoF1<xc|KA+Q*}&pNZL?ZnU@_unGB`H;?{m_~L-_VX(~aO)<f_rLuzg3sj*e?}+HA
zV)r+TD(2BXtjNN5M`m}vr|(5$i=knpMUY4W%rprEVuYVOnt=?f>@vk1f%M<<j*`m%
zd$@o}qr2PO`Uk1aD%Uzj{6mg4tAiii!QTDhI|x^@$@;pwdf2x9b1`SfZ+!OMJBDo+
z8ua9#NRcoA*MB5S22OuXgTp!2H|qsDC-Fih8LV&gDtIwa`#k8d#Sq+Y?|2Zq2|H0E
zo5ODYjJXWr0l1Z0NfEvM3CfRQH{^z$6k`E=-QZ9t|1L>d+K&x;P5aPsaXfIGry&3w
zdOCpAW9TnVNKrh86b@^)!HjtU2_W9bN8u`!N74(AeL*sswuX^FAn<Os>J1=vBbhHt
zsG&w_u$)Xi^WJ7s2@Hk*sQK?rp0_|rHZFzJR%3X?D)Aa258!uq4*wsz-a4wPc55G|
zBqTN^NbOBXN=SF7NQfYUARvu&cY|~Z(%mU3-Q6wH-4dG+HnHJb=y~7gob!9XKOE2S
z51%piTI-(ooY%Yp@BvU+1F*;BR}72zBdW<$yY{^(!e7VkVd1T}H?QPF6|+wgM0BW^
zAtiU2?~L2DCCWo3{eAF-aIi<8u!q7ehdL#ib=6e<sg8dx4&YJ;#P0=HXFIW0F?Ke#
zIDxxVGSGG+7BX1>hNQ@?4;rNu7O>+0+zAB$u2~1B&+I1Ft`r;lIBJI2LH$0IJd@ac
z_aia|N-A<U0zIMtf2O}nAF;b&`a)HB28SMru}G7p?le22Lbq{tyUMvfjo0L75MAV(
z?pizwc`Do4Zrt4-VmK111HLREs3?QUk&LB{0K_2y0{{T4z$6I)mXaO*XsU1CeBmqr
z?)NGB3=auuky)Ea74C;7)fifs&+C(3`_J)qp#+|uJQV69O%{ZUm@s4{vj9jw0||tH
zmNN*7f8bnLzsuL@#MB`tavY>xUl;wwAa*Zgh(7T(jJk6syzIO$R1_Tr_iLzR`(3tN
z&q$|mmDkX>>~f#Tf2(7EdMT73F@To~$bQwi40^*?k&2xAX>k2eqSR2ai?rIGxBDiL
ztWu$icQz4<Af8uLgRDqJ*Vr>z46y-Oav$GrwI!Y^A8O%{cKxS=aUwF<9Q#M)lRW#&
zxD$eoq34M;Sj@EgRPLzrcI`?^f869M567bp)n7}j(qzh6ob;$O)l-^=wWD6}LAnEH
zy(XhXU=lCDQU~qnrkyBJNkFI8UdN7#LoNp_b$MfZq5y{F6kw*Wcadus_udvaS{jSi
zg)iDv`kymj%nmg8xhPSeJP6dFMNQhQoXCdIc9DiBj6hEm+AZGqO$VK!BggH7wXoJd
zeaD~nV^xoOyuse2T6hp}cf{F#K`MxX`*2T*el}!*@O#Krp=q9>$s!~PwR0kPBhX&k
z0GpjntZ$^Oh}Rt;F~-(a+;$wxT`CS73tHBFzJPXLd=U=RgJ(SQYjnXN3zY=q3p=p(
zZ{-)J007HD(jAQv0v14^qz~t@u!xEMtYkq;Cm+H{4-zb*VMSrLnFan=!HDj0xU=m<
z)CLAv<P#qCiK{1$g}6dc7N%T*PjgkY$|Ecd4Go_cW75-EnOIu>J`Wef9L0ZrSz$cT
zG{yjP(AE|xB-W`x+ZZelMGqOuiv3s1_22&)j1U8Y3QR{+`Qu_@RF0nRYw-Vh3c50u
zERY5dbUfs4bSP%UHi#8rzY}A%I7hPWJS(6b=N93Q^Ad$Bpf7M^l#b#H28n-|&z5Aa
zd_tDQ=7NbBD~K!Qmje#gqvog=*6bErXoA#oyqp$qM9|}CO?_&y*=;LPK|mIQsTCZ@
z(HhM=9^c5$TpOq)M))PTTTE~h9Za9*fDc3!NRRqfWH8klVLU+!<*dXl`O_;)9CYqF
z`~z2HO5jGD&;Z>J`mF#1&8l4k77zvbpO?eGx1HMPt6{rmJ7WddWISr$LVj8J%TRWs
zgIiz$GiNb<LAZi8fONpe9-*UAl*3glAQ#BCz(war@!E|AiVFFoUA-_{Tmr{R|BNM>
zHXCrebURGnUj+7Y;H?Q(rFf~&qtPwABy>6^`XD}zm;5B>l2UB3gno~D6ZqR@4%XP*
zsNC9#;S3XK?k`LHT(bSKaeNrvV3c$yPUAzV<1!hzLdF3|zJ8n`ucN@iF7K^6?s}#Q
zlnZ{8ybF^bdh_?K$00=*|6BciPxYsQ6sji(=PJ!?*(F^ubkO9ohAGMXb5#C$T4Hz%
z1ipZbUbRHM%oH%CF`6$H{~_`41U#QE{r1p1)-63WIE0WRq0ig~Vp1A{YHTx=4;nt9
z2(0mD;-vd@OE3x{uMy4W;!aOk_sL)d!#)b(g@D|5jaX~Jo`@lU6hAVaT}g?~sEC4+
zU7Y{HCj^!S;u(LHR)VSk{M9{HB3HRxnsLYf&3aZqdHB#<gP=;vH@!B(&uUFOyyF02
zp}@y5q8Nx5tmDOIZkf>@`k%Kh-zVq(&(Fxb{#!Q-MScoRk4^)>nvt=CX8K$BsO<m!
zt7k=#0AG>(+gp$KwH9-C*x&00ss2_!{(`5bRHm6IXkzuK8rrfHdFL-boOhDo{xxJk
zJ8rKHQo=dMz?15I;vH^u^NQl)k0ArG(4)PLVsC2k+v;VLN1{p1sg!OivJg6R<;QNj
zFJ8v!)3}A0WIVIz!EvK93kF@Yf39ixWU-1Yo)In4O-w6gVJO0|ab-HFl|4M$==j`%
zNs5%qRBR{RomE<@UoB*NH0>3zHna~MF5tjndgylw!`~_Il>Rd!qLBjK=^MX3Um5j{
zt1zFTEslKP9wBD`uKHiM=D(hr1hOxnA)83$w`ZWCq2Z#-t>!aC4*;U;*fL=6MXIja
z1w?Bl5M>cI(c@Mh`6&C~!zGe&$rjL@`1EKzkiQ8uBM;2_dI4iDN~LW(+C~`L^+z=E
zwqZ1bdB-F`@QxEqWgocACcb)#ertm+S2!f~LM5p*u9V}#)ZmLA+brem9UmSd;zeN>
zSg#YuaTh3-<hZ+QY(DbLur=J4?ux^h4rG)l%_q`OIVGk1hg;Obf?2tT<+*xkW_jzi
zeDeEiUms?DhK?>!#kABvZ@F0L?=`^WT_kyd@_{uWDFt6h;sSrK`=8I{f2YoWl|ZQ2
z!xvvAjfoD%rP9?agODSn#QBQsQtrx<X(OGi6?btXs321}BMTCp*NR>P1zv-CJO>+$
zBwy(ZzMGW6z-yUp`i^)j<#QT**>1Yg-9pdUjY&p0E0#$hnu;4KCxU|TO<;Aw(S4Ni
zQ#-g40SU!6px+n&Vlw`!Vh~88{lN88%kgmjcL~6<%Dj)pj8YVKJGbdwsxc$*1e_Jr
z)B;r1PKbe6jMsiaZNWup|1=D0MgTf1hXH<j#T0bW&;Z<e;<PRS7mky2jQ_K%{AaK6
zuLUOMpZ82WW-bf?f=^cv$PsV_DLc6<Tu~%y5tPO7U$!gPKZg1SE6LlS8QKDd1X&TY
zg_6K3M#<wNP0;X5e9iV{BAA?lmaZ{{5>VfsdTecGbx9%Mqge85PUJ**@!OhJ56*^B
z%sXlCp&?2=<qM`V$Yf<l{4)F?dXzjB6Sbh)O}Of|r63V9R#!s>DHH-zqyyl)rRwG;
z;0ap`ic3g9`!0|0_pBo!C@TJ2g&U;)Tj8n__%@Y?fwRuf)Iqb!_EqWiKVQLrzia=8
zb{ZHUm>?Q_38iEjJ81M`^~l=X063O}3}2=3AwzJV?g7!FEczme=!GGBJen2nhIk<D
z=0%9q;1O?RK|DD=+u%tB@f;W_e2dq`t!btf%akHdB2+?%mDExqas-kYV^Lmb$Ku-t
zKoPxX{l|U105@w5BmE8>_BBA6rQe0-hzHA#V3z`k&Qi$;K_Z^pg%N>Q<Nz<eVI6fK
zN`ew!gYn^Hqhs-YSO4Eb>suV+BSgm7k@HDsWza^QHDO3W6$AWr?b9CM8~x91|DTsR
z<zsQbKGIfJeh<A7NGbxk#!jD`s2qeL%O735>j+sEk^1&gr7w<S2yFoNZdPpVY`bpn
zXYAew(T*1&+DT8bA+gZn`K^mvFGNir8S5v<=+@oyG8-FD>@~68mXr;?9a2SXtY6t|
zW`JB<xvM07YegT-G>pZe(0%L)5AMJ!dkHnCOMOAMsK2<_pA9fp=zhsR#p>;&KgCM+
zZA13gV<Bzekk;S}F|B-`B6<FQeYI-VD9I3eA|nf(mklIZu7SUGLqp4723|EZufO-d
zAR>AlW`H(6zao-X0cEh-QH49vyUgww<*NYjI797KrIqJ44zj6+OTY>5P%SwxGxg^S
zq9-<~as;xyzBn7yIGg>y*s^ZVvF6#`arWH4`EF*yAG$w?$b_SZ;R)3*cqf1Ryr>Hz
z0787IN2+)-iX^{=G2051E<dlmSbWbn2^tk$AIGHUE1uw^16xnt2#~|Be>)o`UY}2D
zXoOLx6QgY=L5nbBG~)q59Uxa3KzN2IucYI!xP`#5i3bzc+bD??nV_Kw_Carf0dkcn
zx(`iGX+WSO(Rq>D8<aoz(ARUM{hEK?_^Qv}Q^1#5@u>m<;B;#DzQ+I~`sC;TKga#g
zsTV`196yDfB_>veCWGe^9%I@u3S`l39W9S?-Xw%3j>l9wV&|15$tp$)rRfv=dhIkX
z1G?ED-5$TaIy2L9QOzlzJGZe{pEw!l73C>P57FudC=*qRkaa2g8IBF99@?b4KwTlT
z>wQPzMO>jsyYE>K>H)#yW`pzVjdmJbeUzwWkI%ngz;w1ss@DP`N71~LMa3??%)yQk
z+06yH!RG%*;vt6hFNsGK@+i{!d}~D@2sm^hW(!%7cRnb-gEZ^(^h^X2Wc0rHXeb2L
zzs50U3x_beaBQE|?tY|cm#f9z)uZ1ScxoDky<dSxjs8v%$fW@ZPP60e&t~WGTTh&R
z=V*NzBG3G1+a>RT=eQp##x0wvo|Zgat?XR9QV8y$?VkLW)?{|GomOKtBJ&>lQ`9JX
zs!GxRdJIGMwY<LoAs8$pSa+OVmQp)a)l<UY_~SDNyy-XJmwEE#(EXVb#Yc}G)p-6=
zny+<58@b<ra3)kCp=?0#NW=@$y^y%gCW1fxP|6<;XXd;hO23D+4d-+Uy8v-Me!oFP
zH<Kp$!l7b4|7zsw&0jCc0_{60642CM3W|^E{$FJ-vx@3<`C%pDlLUnUY|sWX=>lVA
zde{&JAxW&9`zc4#Vx^D%mp2Rzwq!~*l?k;h85<wloV$*v^A*Xi-7Om@49tk7V&c?g
z$?k1N7k%rwKS{mDGx;S~IMv-95**6CA$MP#vCKP;(UD`ald}utOJ!q|4SKD7n0o>G
zdiD9CZP=)a2U$$$6THa9(Lo>r1#tSXXtg6p^Sq15XleI(Fsl$F@AmN#q`U>Hs^vmE
z@kOWXYH^oqbfdeJbISjC0fa~KpwtFa`8CX0P9$hj+qg=h6GIaSqWO^Gx_i4(o-+0*
zUyq~mF=R>RW2ZhlkVX>@ewr;yW{?H(!$)(#HR}R<)wa#PWDY0jk+mOPWmn~Nd4c~`
z<L?oIQSVO=1&aN0VijmuIhn3CcwRh$;=8yg^uEbX7=AV6aeeelckl@(Zo!doRK9^b
zxirfHPXR1|Pk!aA`1APEe2DRF6w#Pg?aFJ7xMRUp!+kpiv`^cgEaWg2>hdQ<&l09e
zi$urcBNlLqz0=~}Pv4ZLPp<{ns<nN5EGl;QitG*Vlmwls2Yk*sE(Q5bVF8z^_E<E7
z(Dxf_Uxt&m<NL+(iQuO4ESe_AQ!`Z!4XJnO!ks3NIQGhk^VKuwH}Y@+5z84)al_#X
zS&s{G4E_Ez(2BwYJ1kb?P3x@-Ps`=J39Vo#cgOXm;3Xlc$JN~S^$n@vIW{Qi*Ln%4
zQ4j9l(OSd*H4n6XnO{0<^gCgvmF;={MG)QJo@u;U<j2Q{D<(A`m>au(%-E~_?>+Qy
zcTM3JbZ9&?cW39`3iai+OLfjz=5PD9Z0}2b*92mtiFex0&xH;nBE4&b);!){?x=m5
zJ|V0Vjg>g4zw3<wvJHe#Bmj0^4{7PuMlUkGm>h+v!4iZz6}d<fl*X%!*qSXH;=(|+
zimykGew!D8zkWzxyDRyD4+|f}o(bHq<3UdC)UoKZF%*XQ(60k><O<}J;!%>lmA~xt
z^@ofj0Z<F}vW)`d`G7Cv;<}SJK@39VDBy)**33V-{ev>qLS8*|QnIwI!2-X1@OCF|
zG)Udfq=)Uk-lGajGHX`&fm_c=l(!HqmR{`JNggq<Y@%6vh);>6ZdD6%Fns}gp0E~c
zwbhQ@WoNgXk5ed5g?g*s9DrupY^zbZj*?VRzJ#!eqlPS$gJtJ5DL)?(L}Rd$yN!wj
zLcTAVkvN>?8OVQczK(yj+db9jdcJWR6Np?T>L;tA@ifAF>j-}nC;IJX>nx-L876u$
zsR1Hi#l*XZ;|clwmYs<rll}0qHx_cyAXx|elZv|zJ+?FK8m#S0y79{9!y)$-_r3|I
zEiqtw0MJ_azrA~#ng`4R1$ih@#V(G&Q~$d$`qM!ab08fGI5CQ}cqg@=uu1nk8(+>A
zu5q@eqMfmRX=YBGc$b^&GTQD-`_r3|gX2qf4h^56L)L`&O0%oyd;@1%-P@v93;Y`n
zo_AlUmqqEHb-yNU;{`M+c|Bo26E$2|D7thdx^JM_+0?Zg)~B)oDH{0n&!7KnuEPOE
zW!Km9&Bw%yO2v>0@unjS22{2b(NH5n=U=)=p=5OF%n+;SZ=lB238q~@69ltLI*xx%
zWMsq)*vn_;Ua%CsV9T(By^3`yP;Zhp8C?1vXg9S402WJ)-;+u%L5AJEH&LnNmu?dB
z@{fCZdnW;bL$jh}V~gFE{Nsl!9u<pM4iQ2rvuf<0S@Ww(jt<bnK(Y03X6GDd9|Oq=
zM3dXBlHiz40;qok^a(EJ-S~pz?nB&fM1Q>#nN#qNvEoaBr%gUY1MQNs@UEe<(|2mK
zq;HqQnsv|Mv*?zUpPenS4hHE4&t|gn>?ZI5R@Jd5QuOTW_2rBJdnu1Z2;uG4lRuy1
zR?BCBvvyp*z`E-C+uk#Zg4<@K_bt>v3{MJI7<wlBDcO&BE=3$8PZ89tNQL7f`CCf}
z7!921%5<ATl23`qUUCyq!0D~MUuGy?4VvZPKcVQ!^=m5RF&Xz+^~KThJ=i{ge49~J
z6g_uA@NgKeu5nnC=c2%vX=60OY&b4j81AC-(@*_v&U*?8Bb_caG6<TDPD_%k)8!ns
zAg1<D2wL?mQ~#{Xqb}hR*D@v$!rI^zopZAeCprH?ze1}2QvVH-qd>7REiHXwICW&q
zTaw?aFZRd3mrW3!_#cNUE98rMr;Xh_B|1@OW1QaF?P9d-v&yMnOx^Fkgm5Rl%M`o#
zN`IUoRgzFZiWQxdGZk~_8Wz`Pw$aEE8TA397h>oir1SJ}HDZ{hTm=4YjuaIoy{oBa
zjm6;oWWll5mVH<}P~Uo%dplO#UbOMvMXDr!cyZ}z#{L#E{}%}pDp!pl{isHt2M#>p
zRiI2FY2r7CbuWLX{A!Fy&YJOeWrW%Xc{zpbSs2RT*ihc^`O@vBIlbjv&GMAE)+T@?
zJqD1Zlz(3{{CiQ<Mklu)MBrS5_M3Y7wk4c$I-d&e2**WFcJJ1)+;_zR2qu?dxUhRD
zmM@Lp0KWUhK^C3jKIeo%6qR{yUY`B-2@;)<l*%iw`__bzC(#jP&WX<FDXx~Qi3amz
zTC;{mTkI;&CTsA1Mo!h3dCm_nBoj!JNl5Fb5H&y6_NPvKT2)sOeNp8}6$R-!o|#yV
z?J|r=YhsA6&XDcteeI>rvvuMQ4M0tuNC|mP>6+R0At%4aa)D?5ZK85uDs9WhGJ(&~
zN>63#tz7H8@R5=iMobZ_m7apu2C<2|78Fxil4z1g{lmjtUSGFl(eZ^2Sl+iIwj*{Z
z1_<yWNW|eTBgzqt=hR4Ft7I$JBL~xXRwUSFR$Jn#_7ajoaSX|K$Ky2^sGw(C<^sr9
zzLZ-}k^+SB6mSXRW2@+y10ILdi@XU+yBR5N0s-qfK1vJpdTmDpaUpU$9FA|{GzOV9
zmJ>vfru6n};}4AvNtKSj)V~$0ssF1t{{48r8Cje%RkN_%gQH^rT46|KvR`iiFbz0I
zfAwhTq;3UxTn*Ii6<BZc9qK(l6oeSpo=5Yk>c9bAO0MvBCEj$c<nt*M*NtAx<J{6=
z&oGmBiw=lsi<d41-UlO56OBK}6a)bmUb<$l3`qeoI&9bie(60d;h1L=ty}7IWC|<E
z1}^&h@0Q}uqU@7Y6vH>3c#8GMv?efPmXz@#xU$WB-rHm@YpI~`&Pi4lZ_joUcRZ5k
z?XOHvxsW=#bUXAn$W$|U>>rI@MtF2abKd^0E2pW=Az?Qx9kVo(4<jzU|H`W?CwxP0
z(pj$E7=P6EJ7n_nX%T2f^yz`8QXy7e1hE1#yoHg*BWda43e{~TfkYcy;(=%T#xLmR
z6i`=ihE!18d+raeFwE7_{t(5y%iemmxFT@kNY&LvQz)5`<NCaE=5`Ws0NL}ElR7FK
ze#2H{&}1!<YF%Dj=Gd_4jrQQ0S6-c-M*aE;?yiGAWS<=>Z_kA31^eQ4eLhlb^WF{T
zzlIO720Xs~+&el0SqBeAI&m8rY&z$F6TvURg{C?d*g0iGOTVz`Q^)6UUjPwJyw<di
z2fG|eLU%@Q<Wwfw;?mQNf+<vP14RsBfWA}#fiX8Xmyb>jnqZX(!-GH+zki}tadHAD
zgcy^#HTQ|95Lmc$cf4V%^0HV;?t#w{F8S4YMRoJA-@g(6CfL;1b{jZa2;(bgF*ter
z={#e!m(Kjm>G4j36g_QnI5?61+8j_!xFpol3U=WNz2`EOO5w50o|U11rEZvR_|2gX
zO`?PM_V)52QlN*ST>&*3IMOLjMpt;9fHCR>5;Ynh3U%QS;;8R*pswE8lT}P%7@9i_
zAxKPLe9So?FAgfn=w>$bbhnG@PMcc<xCxMV#IM<j^s`vB(RcNTqxHs5gAYxVA3KtB
zkffwi(T9mFtXOru6ANrPWMhAX-rbG}i5r(3m!Y_GIb*VYEiq2t%78~KU@_r`1?J@P
zN25^*`9rr>31Q1tTCzs5Ktdz>6!bx8-q4}~S>bxuHZ;V@iDhUTd%<-2`^Ldh=Dq<0
zvKLJp?YI^*G3M4?@}Z7S`1f|}iLvs1$g;;8a^Ln=R7R$$@wCKp(mDGG?ABd10kW*g
z5$)T(Tul4)-VJ+ya-)3N;4=~<(t@{(NkM%W29sqd22+0k`*~^%PKn00hWgbs*`*+f
zJYA%B?A;|iL*#?b{koatv^Y`B^HI{HseCRYpdXi%5x)o8W`)|ANej=tpv4!715EVE
ztjFe?L(5vg$Dh;8t<XjHTS4R4{6ILq-07k3IW`wD+Vcax`R8|c*PtJsZEgIfD_OEY
z=Tj`$LF1<juCbc(y|;4QzPCPs9Wxr?3F&XF7mX$P3#VO7UGFWFn-+W#!jb^m-8gyZ
z>=<^=2jJ_)#YazdT{5L}(f+ty<#eKZejAJ_p|2x=@n29&W;QM^HFB$AfBA&wFo@1H
zz*Z|>CuF|PT4^Z0Y~yn0=&ow}`GViZQZvi=M)U!A*|M@HehVs{l$xp_tw4yeaVgxS
zUT*J=SH&lYp7<=-7mEVdVgm_3ICxL8%N#t(mUnsen01nkHke!^Bkn9TDvCD%Zz)?E
z_i=NnZkq`F?&i2<)?OF_LuH@&qpl++HovQ{u(SDrX9Ma6JHoO8KsZH1*evX?JBi##
zkZEC<!9m2~&_Em~T$tE1p8eW33IZMu;@K+}8|Emz7ySI)74-hGcxkE8so}7X3YES}
z2br=9?mA9nl?dbj*iEsXsod+k?B#C$^CNN$NBXb+Kf~2iZKgxepY|oMB>+qS+6W5R
zgc;6%{XQfKk@LUzNO{^+{hU~GGQ-~Ex(tm^+M|m7_xDGStDJpM^oLjyRz@(Dw)#}{
zeYs92gEmIG@Eq<b+#r?<Rg#U0SMAj~E+w&ak*sXI8Q)0Gb*Fl3S!eT+-bCw)y-Ra1
zDH;M<jIf`y$fjDp&#dE&L;2;sLGkW&9YGXVSK0g88N-(7N)>AVK4Ai(o%5rijJQVW
z(8K4yW8It!vbxwfMyi5|V+)!F*k{*KrP9Q5wW4VDdR5*%O=#O;UO#+$VMKaDAKCJu
zKNrJ#3P|`FIt2=TwY8|+!0_;A?U{Qz@BD+01f||DW)}!vZnK1ljaODr&%z7QZrnF0
z#+uSP-QR5+?j(nEEqj_mLT{FT))$v(09b8dBZ1H;SSaA$cp%4nz#JdF16iw~RHq@p
zso;=Ht4T^mS|v80HTH=VHSsC$dt(K-&job3<i`g9B=ZG$2S4CS<7e+sBp`?Y%zMW@
zNy79WfSDRI5`B%J@BXuyJjnf(Z)hSH-Y7~~Vxh)-vw3V1prs!vbUK``aro^vg@YNj
zjYZ<Z+xe=NyOS~A^ODd|RYe@!R$$AvAOKgWD#Qc7jA**Z|GLs<bgShc6na|4X*QI+
zm467|SbTNE`7tLr-y2p&^;S>oX)v28?ju8Nmv(7S)2%ORpQb9mDtjBAGORiasN9aC
z8HJXlCCUEk{cSq%h4yo|uPty+tkKDVg$8HxTlq<TGIu-b__62ep+J^ewh;xea($38
zBDf&(9SHD7;N(T~ccV5Vk^^oa5%{(BRC$pJ!Px{HXeH#{;f!}qA1n-&f{_Jlq{nM}
znk+A`N*6jjNaM2OLK?l#E)c1<@!uhIZQ6>VV9lae<okQK_s-<7(no51!PHcmrgLj2
zLX+zCqu5U@=$0Wj%fS)GF@&0SphT5#C8R7ceZ>Vp_rg#h7Lb0X(m*y@*!=>r319}S
zgML7~>n~plzV_Y7-Z0g>6rb+-5%|e&|66bcIukQf?xLUe;DSGvywucvs>}-8_44!~
zBqsIZHaur6w^jB$5BBsD0;-^JK3%p_^>RztCM>wE@>DYQMoL>6X(3+RJYLQXEW#Xk
z+eZ7rb^oh&WB49e$5gXw4dUiHGbD~Db71?tDm^h(Y^?33f-{hewZ6;XV}p`BOLY&%
z)52QHSEyA+=lnN@0-obV`Q~hF@&@L_)_$l0?ia$5qYW?zgM@)i|0XN9>FL7*(tfZV
z{3<70$1Bz4J#Qm0m*(!pIfJ~zXMmia90-Zhp}=>ZYiVz`yuKs1<x26b_@?qannuU-
zajLN1CnixZ09@g4n6976Z}>^A(B-IPTsP)LEoE0<+D}o_68W;9Wx6T7%8e-%5jaF}
z>y}fn{K8s_VZfoOSiJASY_bf^91m{C_hwkKeFSFhYKOXz?;luWmIsBy>j1OH=ID#q
z;s}p$2W(@}7tw1hDAm?p%%F?!>Aha1X!9$w#?5*0>3U7%paZr3qDMc!&5J27a#o^D
zP|vQAS=&HwJg+Lw_WG^ThT%+Y$L$=V<vuf|8nO?=y!Mp8=ZgE7@<Ns`0I|Py_UrRG
zE4N`quT{|$x>x?R^o{1nw~bLsHKGn0I-bbJ3>Q)s&2!#|c_75-9Vw5M%#xDLy=CCt
z*gaf~Y^JPMvDj#7%T6>50PElDcnO<HXTG7zM2V7jY;dUh&&KG9PrF%~at_y>gjGtf
zCI65N)e|D3$<UGKrfZFX#yr6CC)4}xD(Dk<dEvb8*>t|2(6R(AkFQgyrXGjs!N5rV
zwdH-+Z*<9TV&3{o`D*>YWZf~(6jOV2TlEH6g_R3Zhq2zDB&3M3JM6w592m%ARLxJk
zE3s-w425|flvi&jmAwZ-Qdg&}XDdE(IUi|X|Ng0~=v^Hj!NABL5_#wvr+i;h-kQw8
z!Qq0L^h}j?DcNj0rp=Rr+jJa(gjOLH$Xfwo$KTpI2LsfwbK~}(Coz9CubQ_=5HUH(
z_^UQ3My9_H92v3(@+t74@dMeQ+5nFAiV*qT1C0uQ^h*R!o*hKTB(j5Ct6yu+H#+Dj
zV|)<#pO6i;BcTl1h#1;s2Q{SDoCEC2a~@NC0)dl`fez5gI7l4gz>pP`3Gzb9+|3J!
z2!Esnlg#wjyZ0k%ln(32esdo!9bPO0MZnkyp-;~QxUtE6w$z?iTM6R21pu{w!XG3e
z$Ii~~u;O_+vEqGC;h=!BJCjd_EeNMGsjRNaq;yfZ8i31cSbW*|vgaEH$$gLUUFvOi
zDz`UL%he$n_*%RVI~#Fnl)dpKC%p=3thh{?d(ap^C0T#}Pirb%5vFQAaSF^cTwP5F
zIGwMTSs(|J#A%ok@7l(ejd9r`b%J4ge!gj|^f?>=e;cnRn}ljz%w}^%giqI>qV)|B
zh~`@7!-Jz8wnm8GjEVFmT`%%u?iJn8Ckc4zl@jL*NdOcwpI*!nN<OI<6NpHsh;rx~
zx~(buyz=ag3F$6jURN`4F_cHa5K0@dgN7W>tE4VPw^}{;O6o<pz0NzF`oIbXHoFsU
z<RpZtDXzm4--!s~Wk%z?cnw2uw^L-b>RQ3)fe|TuzPL6MKdj!!0In306=FO<%+9-z
zB-PQ|b+qLBKK%mE#eu%@lLB&s`B^8)ip8(-N@O#QbCZ$m@95Zj_Y}tQ8de7pr>B4P
zb5-_W2IE9Kn)AH5{)Pp#k4ep<>tRvK@%fDYzah4l+|56;Z<KBNXy*BUb`m!j&Sif+
z@44X=?gjOVY8)6*6Jt?`6=b*<bG?eIj|WA%%||omasnqi#J{SjggwG-wHIIN*#heC
zp0oIg9D{VueSg;#_o$M?@dy5F+^X%Pd(lLhduCW{t<|zA>v%}PMS*}LoW=#_F75ss
z((etVc;1)4zXP${3RaZL-9@S<9Xza~z5!`gI@|}EsoDx0O#J{B@If$<oK!pr;{KRo
z8Ll?pV#&#J>xydmehYHNksKn2!Tl@I_mpo+!O1PTh}ZmQ;6|E+BPmZ#c+Fuz89mV*
zSI%Ytv!bB`Gli3s^C!<uTS9^au@d52rJ+06#P9@%eep#D;SpozIfYi5i0;=!m{h28
zu~{R>q|<o1M{`b*h~Js@k36yl*n;Ux$Ni~`-z8>ClUFqdV;LTPKELp~w4}u!pV~2q
z^7a)_tjnBw20HZG!97U;DmBN?C!U&3&H^U_X|qAvX^~k{tx--E3VF2lt(<44PjaY_
zd6SgOt|JuXjcc5(bgZY_GtI~DwuNd-KPbKOzW3!R*I{A>=S3h3C^Z>ArC3^khqt<p
zaJZeN{}Mi!=pUW6Y;Aa#-SuYmOz@3#*ybsC)di#MzojZ_PAHL|90@Jwz6UA^HKRR&
znzT`mrk`9vGWZJ|&nIN3E)EwMYc1Kl=pr-yQWbO{44J|WnuICZQIF3!7=;AVp(ONv
zLyJob3f|gwo`XY+O6(zySSJQ}ZEZp4<R08T-Sp!>%SFxR0i{cRbF<L(iR`67Pdk;#
zYcD5onPycI2PfytT8Wbj^SP?Lfm(LYYX<@}-R`i6DDD6P-8+~aSNht<`ntm0cEFu6
zxTww3@;0Ui`7{HFH!ytkQc;onu=t!t3Ihy7<*!FrMfZ{PL58#%`l2->OmqN&t5CFV
zv^*R1?!F5G3i)H$(*=?7C|E_0uLyJSfCfIM(|X@q8htj)HhI3UvM(5u80e^NUndM?
z)%@0{hx0LZ#}ITdfetFyLlqtr<?z3X@?K(Zcjjp-2FsRVMUU}#Sa3wS8Lx?7-@CUl
zMm;;~mQqCQ#ejD@TaZj7`AeWJ;lkYR_yVj_iG|O{CF2qkP48ACz{1HDyc!jD+D#j~
zpS^emK0^CE4{N{Ic=ZX4ZmNX6w^NVE<-5!d;_X0Qs1$M@Ia3nl@bb9&nNY)MN#pow
zIV5t2nS%PV)frT_*!(8h5J(-=f>HcFn4Qvmq0?x~xMkZ|&dSor*Jw@b^{U+StawA0
z<axn`Xw2uu7qbeUlKaM_o?VioW~ojxpvHsiGX?jv&z1pLBrT%{ALolwHT4cgj+n_U
z1#7Y=qHvsTG^_L;340XWbYLXWr?0W-;m6{KulTjyW%V7W?}pn7vN0Qc1fTvkS;4Di
z!>hcBCPCK35hL>TIJv{R25^*Q*j>(NLnMlAe^+s_NCf_8BJ86z5E3}^63>y$W03Rx
zrPmGj2Kqu8%0?8+CmpzhUx>BIBVQz`kUqKy?lXqXaH{^!7yBI_dx@UmMvD)A!80h&
zr-?eW7#5pxfnF5I*Hdy1e&zk3A)%YRKBR0s<z9Z|{`&41{!?`)2ZwE>Ilv3p%a{ts
z@=EEf9$MqyDwLy=e`6j^wadIU(jpA0YqJw3Q14x=XVb#A@IL_&xrcjSx2s}2BgQjy
zzFN8TCbc8*x(|5$9oQR=F58tS#=x6Y(pShZ0uJ@k_3>kFb$q10vpW)u7~tc|y;RiV
z%I<2arJcn9ULF_#oOHJHjdmY)hCk9)W1<s>lfF#~vyOLAx;Pw90QZB1MXz{%)QZ=2
zufgy>AOfZbK4;qUJ9baBiB+v7?+qvG6$MexTY)s&!?e-rBo{sQ`zySighn|Qrxw<h
z5I0??YBPQDo1_%fi83I~WvQ2?yPNSW3E_0p(L^9@eG-wQk2|X6rQ$+khLIH8R`1vQ
z@vu4_1y_E}Y!0Je-<I~~Phu4*P*&JYoaWffHrmQPq<^Q>@mx@*D73D?o44?49`n3O
zuHW64)uG2F@W2Qbxlg&qQB2wN(jiVcvv!}R0{c`z_Y+HUhjKhDw(aciS4kI_n<8X0
zmg=wOqKeD}FS+mT{NI@_JFD(p(=X@k>f?u4Hd!t)k%|(|*$C%76uOX9wt46DWnY<A
zu821-L(JA!J9XB35l!VgW0R-v2P{8stRAYqzbUsC$+-l#l+{?}+^_nBVEowS8H;|z
z<HWiim0aKddnd*ujHEHvv4eA4O$#hXDinwIxy==waP~^@MqgDaNF|t@_I^;_VB_l^
z7r3LYz|$cv#Ooa!4<7g>e-~=VC2~;Fh5jt;*25V!+J9>WNJD<#>e0aO9IjGl^#R~q
zrwy#H9VdS6Vg19&tNZS;inNj6<qLALs|JWlj^r8&5+Po#C*hWe39x87`1%+VNDEgQ
zcc`@z4^1F0m<8^HwQD5u6wUG6zytmNaRWfWDxyhOc<cx;@iq)8vk7psBt=o^AI1Ft
zxdBvsnGLTqO)Ao=`MXukDaI4Bc?k{LkJqxwQdQ#bxjkWE8^{4z-=7zKr{Cytf*I^&
zB^3OqOr=&q+(W%-WM+$Km#XT^tqE<j@>`9qN7mx51VXGww(LNFhr>pgrM65n$6{CA
zy96Ds&A1^&@BIrk(z6Rw)jm+lD%BvM)puL%Lc<bA5pX|X0hFQMx@~V2o*TKYjoYI$
ztM-2hR2W`mQKz&W7VwO$vl8#c64G08`T=hqU+ASG$YpQ%*uCXUrgh7=lkGK1#BAyf
z?IkvT6||6~HgO`y>x)bx5xs=Gf<ovDy%HRu=9s5+)~h6)-0Qzk=&$dXuI9Ag{fv2>
zsdd~mN=2iG=dljs-TOY4VOQm*%?ooH9B}eBKMVaxJU#%A<h={qKAmAP$e>RNhfdcg
zm>pST`kk&+vP6JgoZ?83!`6OcU6&Va^ckdT=8D@I<5?`7b`9T8m}fN1b+lKPvPd12
znc?2_h8=ARKf2^Q$+J7R{ji%HlaOqzp@B^w`?$A#hIILKU}1_Pd;%@C@lt3$QMylx
zA5)yN^q{gTMvgq%wAN~#Rnh+LH)(!iZ8RR>5-d(s@kk50sK3Oyi|o|_Gbq8ubNA5O
zW}yTNyWYP7@9W+A_keQ?OqRXIsor^9{$M-5wm;@S3k!{;wm6~LF|u|iI5>{LLb^@)
zZlRxCznznWQ0>~KzUL%w_`JiT;w>hy1ob?|D*PHP^%sWg`q{((6eUW;1ZTTc?#OzM
zO_MJn9?rS)bNq+=j$Om>6%cy1X;g;+c0r9MR;6XR>M~b}(VW)RNsXULh`97~7xw~l
zTd({`D4J+-S;(vTv}~;;Dn}{_@kok}Fz`mg?+h@%PM{1D;&Vil?&&7|$)@X>+Uh=b
zc3jsjwH;)eF=jvNOidsYO_f_bR7AWa=E(|KOLaEe(b9-kcD^{?oj7FcFt{y!(hItb
zZ9|FD74Z&>JaYV|Y=`Ez_UPcSWx);5_lCBvyg*A?t>kvhlO&bMsJrDqBXIL>zTxya
z{~ENq_B^sXb`lUsd$zOHuOjWuy!kJJMj;nEWo*ygGwR4{y$W#&n$h-3!L-L$aI2~k
zjKlX;gIz2_9HE>7vPt2Ps!|F(ydJG)La2Z^q0tkQ%m7sh38XdQc);tDKzN2;417ZS
z{M|Q)>2-Qqf%m+{=GR@>ou*gwmT`$bF86P`)-C6MS}Yk|hrz1r9^*l=fLxz9G>?@L
z5+0l1n_{vd2t+edc4D^s1woNs2MJqgori%f0jSWW?jxpd+&X=&-PEjQQ4XQ|i8Ld1
zGn(1WTX~y*b7@c@bpMh~j6h3xi;L_h5<H89Ww;;Bx9eYK8C9tl6ie*0O5^xV07<Lc
zk436CKGDVhM>wa;@H7?}HADVJ5@=YcJCKPVv3Ul8AW8*kPVXD!z_5)=XctJ;&W@uD
z2r~j0$dY?|=_|%8$31N)=VV3h3mf4#k>L~$LnSn@1e`Xfn{GNy7Xl%jwmt07@b>Z-
zZG|`I?C&dTH~spb^R~*ftFZSP_fI7>Kv3(q6ro}lQ8DF9o$eiZBwAM|+t2~RQReZ~
zTk^*3z!GT+wYlSduW6w?*LL<hoipAMO%ZxMQQP{x(JY5yj3eR;d#~hIV`sdGRI~Fa
zTB%~jig4d$w-cIAbNT#BH@PX@tBR&ROS30+;~#_Sq<$>E6CJ&dP}*NwGK*@wK#b+?
zVhHsXI-BtyC3-d{a?S)l84`;9{4|N%unA;{+_8R}_Y->^i<+K9dBF6eiC%r_y<$En
zYy#BsVt#R_KOWHzeLKujpOo8D&my`9X^z?NWTnA?c!_WSF#$*`8O!AKF-ialt_fwz
zjCGdaH}~;rT@#MS9LZu<^{Y5{s4lfd?)UBS%*w4^Z3Z_F9EHii8!&UrVP4t;YQ)Ki
zRu%a2X`QZTo5R)HboN4G?ACQ=gI|g~P<kIvc#LKDPa^i4FoIIieo{(FV8lM*j?%jw
z@^5Jg1l3kI?0sRU&F6-GxpTM(^o;Q8$Vi2M=Ijsyz<BZSoJQyTHRp!=dnDt2JO7pH
zst05hH0Y^2Q$MeJ@^=#}uM>u>2o2s&z=n^uj+5*b%uSi?L|Q5}HL3&C7r!H|OI$Ou
zg<sJ5wsGE6ohqOmrWh)OSJF?}ntWKtMXCrKg8MIxnd)07T=+F9J{5P0p7y@mwiKI5
zJU|Via24vYRVPbX1+Llt4Uzi=D%1A+W9zoeN1|RMT60fc_iebT)ZOg}+IL($A^67M
z1x{zBxWRHfni$)4vXIY{4NBdfV2hYyMV(k9nv!z*(lokH{Q)ia7O?9Us-j21Cwz7d
zxc59-Ur(^?ia+FQ>Q?KUi8xk!&wfcvH&~iOL=m}O=BmWAl||<=d-=E)dA`CUW^VbB
zeQyhXZ_@c7waY9!&JQ^13rjz=c}fW?<iY(wZ_=v1BH|BYW7|h#iPFu|JG*yjFxqd;
z?ljB2Tc)4JZ(r1t?%7T5O&9;F%ula1LiW7B%Uw_R>{-`sW7ncMoQf+2P^V{f#s8_1
zDojO>lZKkw7|ISV*T!b^(Kv(Ko+}z=qvb|(f{6(UU#qFbF_M92l4?(9;zmHCesv)$
zT?=62liJ*rLgN;h{KZP!vXicO`V?O4e1OoGYzH{y%0!jvMyDY!K(6trT;jGN1W!JS
zjLj51ku}?Em|m+ujcY3ijxV48bbfM6LUK0J-w%D#I=?3oz3Q8QUE~4a-OWgA6swZp
z8TdpkF|7)*VCAC_<S^k~!2JRyA%%A2(;WbITq9{m4GIs`*bRz0r%LZ86g=-hwIqT3
zwgZHO)0%V`qDjx_d>a+OJWd`-TqS7)(E=XNBI6K{=}n_(*)xNJ%Vy<5M5b+B7BX3!
z-pG>l{UL)%{tp=}rx;(z#!T$A{%Sj==v}!EVvJBSkL9O+3U2e6T;R`ByFTAr10w=1
z@bxa$wF!J3-}#LhN>o>hQ|5&KA_;#*+ThzU5X%~7Pkk7e@||bGo+K)c;kpk{nZQ@D
zkMC=&V)m-N(ZzFpG_&4`xCbu1J-UvS%*c*==1dnL<~OXx83n$#@StP`*ZzcGatIu4
z8iZSoMkzpD&K;{+&}-3r4UG*SkQuZ6S?aCGpv7m84Grq=JjSfQNlVF$SEt5MiXh%U
zaT_!rQ_4DSyZfO!P)5H}pRu1i&2eo4_6WSYbrdnltq)M*`<^l}7d9^$q`QeLW7rhT
z>k1NH9C9IH)?^2E*^nP)?-pPS8zOi6!fNHdDIW!0*MrVSY9NeH?<txsd;8tXIX4dG
z8&Uw;$Bc!vKZ}{zW_#+L;JHZW1t~5G=I^YX@bjb?WKXAys0@>_td|7KpBohzK?ChL
z#VApy1X6?kfa!b2ba9#mYsb#<NM_UuQv*86mnZ$|xb9&|dfwK!$Mafjc^k@Cpki=k
zgubX~Be=_K2-FFLeo=jYF7;Q-6W2_Mmb_{^T`61$#32}GdMNdY8tmVI=gQ?rGu=$J
zSud*rlE1u;wY48wt+4CvdOFINAi1Xdb1qHpT=?~@ImUt_3?Jwp;VaE>p%IreMl%7U
zv5t2Wi`F8}M8#;4Pi6ZqoK2pJ^?oj}<_oHz2xC#i`6b6IdLQEY*&n=6tmpnDMy70j
ziiORKO0n@KHAi?<THq-0mgR@p>@?T?ofA{b4ODJVfRYxS^k(RDRhu{tQAd}^j__5(
z`6BcT!|<`SnfrOj^yOd-s&~_|9w1}obLqJ573LkETxqKdqBYdo<A;%ET)viGu}&;h
zgtmJM-VL_acy^KQkXK;i)*|E2tsH|aMw_j;em~v4ORWc1lg-?3uBxmUIwWy_{%6OL
zSw-`js3XUC$e6H4v&a4_EN-n?+CibraQRan>XUq{HH9g6K6BYz#$S!|suxFG#XmCH
zV+E@susP)Y^ej!{OMxuVyxiPz0F+S-IeblN3FJkhga@OSRwocXoOn<asYgIyph_6s
zcLrA0c|)j1vq$wr!E}iU`ZKhB%n4c(jRr0%Iq2>P`nN5b4A2^NW`K|CYKfISu^t-P
zz$D@~AK2QAj}8pE!XJnF8CD!$fIH<z2aYWzU=k9!&`E*p6iib|8$TPTIk-8<b<R-%
ze^HqFV)%8C-kXXpM6#L_Eqb?a0@MN6=NK=RZS|NpLL<6~yle85_2+D%-?0%1YVkpC
zn=zK1fV{vA6(p05AI7Y&X5@#)3ewTQA>#?ZKHu9Q9>@V~x`8o8)Cv$kWj7r)h5;xM
zX!ff}sR~)+&aJzGki~%AQn$HhFKfi=#QiSkZYKeogGXqo^;>><z;Wwy|3-$e(itBn
zYB!zBFzm+M?bl6R_i&EK-t|jD2!V1x)b@%~Qs{bcq26FMZ8}>g%xB5_!!@Sjot0g?
z*Sekwx(1&!#sf<v8Md4=T8ryM_Zi$0l~?jv*Fw9_a_(b4CgA5I=>B<kdsI_Sc}e$F
z%N#J4-&-h>HopwJ`kCPwrOu+9XFIlNb);yxQXl#9*`upRbc94-E9<d3;^_gF6l=@%
z9!lsYnh3faL4o~6xNkyZ+oMJ*N6v`WHL9r~$e6P0EbfcfvbBER+iajg@4gu>Br_C0
zOT%w%dW1`yYwDkV)CNv`Y$CA`MD92y1>`X);rfI#pcF$yd~6QS_GQS!S&uZ2DkBlH
zs?B1o)&U<J9tK`GM;Lv?&?3ifnpM=pytqWAH?%(EaKBa#C=0m7i_XP4_dFh}|D^hF
z-M~wz>1;iTqu`pFNi?J_Ny$u*+Z_$T7N9*7147TbAk`D#8M|z}yxvYDPW399-SokJ
zPV`mdmRRuk(d83+@4S~ex9;1X7TX+NN4DM`ch}!BxZDS&Fn+1%*!`ReU`E4j4v-|m
z)jGO?fkIyuo2m6no&GRao?Rl$&<;C-?>fs~!rsEswam%o9^xH_>$(RC&V13pnScR)
zeBXC9NOMKA)otq-Jw(`*c;xC7{^rork=(+suV8Fhw#{n|!mqha{P;2j=s9bX>UtYq
z9&*UW87#wNKIL<974NZ$%CC4sLzfO!-uVTgk<b69T;nxLA+w(>Hk|@A%x32UUq8kD
z=?r}<N)nCuaIcB7V0Aj%!lXmqh%7od?`rNk`DJB|Ak9y8yyXnvMHP);N3XBA&s-=!
zI59!w%2|1z)+5yGY48zNp<dgdEBln-6d}9WAdd%0S~To;=;8`nwETklgZ)Ip!%nh-
z#L&b62T`7Ne`2;>0{3sItoY2Yl}WVe7;VQ<Ly!rckSXBOQ_@p=?h^a9A?V+TvpEI(
z@~ro}XgjXUL)0<xUfsSX)(^r}7^gp}|Fl1${RtJ6sWJetP>KaQkP|S%f_N8gkO2wr
z*zHMo-AL>0=@9<A3Lq)q^P)<VlV!E_%xt9rh@k-)`Ql(U_S?SWcM|uJ8E&qNr6&1d
z<WPWcBdG+IXleBG*=cL$x#7Nc>af_*BM_n+NtF_1z3VW^Brf+3!Pmzho?)F`&*Q*s
z?TE5Im5esi=O$ooOmoRK8@ut0q4G#Xy{{Oce4g+v>&7~b@&kCQi=YoT7))KAf7h?o
z?7^TIW!Js*b3A5AfMh9((>q5=w$8bVwE6U05G5_Rr4T-`S@f`#*aIRgvUAtrhYw_y
zYH#hqqMEMZ7NOHnchjAV59>$b+x1tI>@|WcGz|CWACkbi5yW8kuDQleo$x-C1L+fh
z(Ans=S~l98i!^hb8`Z0-<`Abcw@F$#r>hD6;!B~70S@Bv4{8yP_N0(6@!xg$(QtG_
z2W&gSeJ!J(Eks&Gsm;&UE~k_d9(Y>!&A6F9Xzk^p46bQ7gdH@#Q!eTA);`sIt)J%f
z-}70oAUO@hlqB>53Mm40KiPp_Rs=#~q~xA5^qm*v%agype4KU<7UZ11t2H~(vGHE*
zBwzr{+Kz2#=NT2X9d3ScQV0{>#7rPk2S^w3T1~|?{wA}y_88e(T)R}g<Xr5wwD@td
zDlHsdcDw8Sn_(2)_N$rk;quGNH0M%>w{ulv(RdE?w7kh@lJW7CCJ;Pm16aXN)=C}K
zM2nwkl|Qf(ZVb$E6!wlGSkL&1K8|_FjVv_P)!tgpyPuy-8N3qyi7XA}+ex)&;2~O}
zvBPZDW_R3IGy7EZkBsQgQdNxvMM>n!P_p77ty;L&utS}3HvN&Vz#Tx532D<iK1*$b
zUuad+`B*@hvI4~ftlws&KGj`|Ee2>bC-5Ip^i|2AW-i2^?Q~9hOS*~tox}*D2!3zs
zA3?i(c|6AH0TCG0ZnbBoB!mz3De^^y_|i5-JF3Bk&%gX(u8q=HpZCv>-_UXd>-NUu
z(XkQWAv}xsp;`@2so4s!5caUm1nI3x*+?u`KH`dV`k+6Qx{8?=&G(FF9+zZnrmwP<
z(Rza8Rh=pO)3@`Gs$dlgAdFDXWzB8L@u$lD9o+MaP=)dwO_Y%SY!IM+e32wr#Sj1C
z8}GY>Hp$<YZr(Ync}FHJ2-nPzJQu8jeDIj0&9}bJm@*wyR#jNKSc$6*(J?ud(X6r?
zlAl+-U2n+70CNv}KQ|hgt<K;tOAkfR`qq^^;i<f&-f6Df0UnjJ4T=>;Mg^FRU*6Z*
zlPs)w-=(Cx*FS58^ZEA%N{%t&9f!E<=3aR-vu3qDY~OE6W5dCi_@}2U%aIbPI(IOP
z&`8@@$P1orXn&YNp^7SRQuoFEY3$tja!E$W^B>=|V$<F1Q>R)>+itX6JFeUAu2hW$
ziBM|$AkGW1HR~BaMqNzOR#H$uIOUu9YascB)M?vnAqaS`7Rj`DjmVI49^hHW!fOFS
zqL&}4PujW475rf~H3HS*56-52U-{K!?@pTcZC=mfVI++<S$2#5G^50E0&VhJA@SZ4
zTfV%DEOL4h^;cQ9`JE3JB`PsdLm+R3|8^3|!#3MNiTApzw5X9g>`~p&mKW-S&4s+}
z4?35JC7qzMvx(4I?SYF28>wBnU^N;#<pY#tsXu?=Cs>O};wfX2;j}F84qo=WF+XNu
z%2x4Qa7}zxz?Ipw3?G<qjaBGvy~L#bQDk!Medoa8dDq#xBh*(E`j)lrUV49g30@j(
zX`SA9ci=My)x#H>VM$XSTjpZ;pEJ{Z%*~(myiYOvqA%Sd*?aUhU>PDR-zXo47P_iJ
zUBUEU6n|f)(Dq<F8ua7T3X$lT81a`cU*>nu6P|~Jgw)uqNibQOe+4EU+?i4y3!664
zvg{Xc2d5sm@ocEJ!Vvx*J$+y7Al&Ee%D8E}A0M;&Kylisk$}5kAq-o-2(ajRB?JW8
z{qg7lWH^5<dV<b;U7&$r0lbqKrzBdWIN2BV3JSMI9F(Sd)Vp=c;}_x-!$hfAfJ~!H
zc^k#M`-|-XtPmOHToPDw_0J&sLe?d7AtxV$ByaOrP@+iWp?VHXmHpH*j#MuFc@KUo
zd&8V#=ajBorsM6=f<PMZtJwmqBtab124s7_rUe9n<NC8<2cmq|Tv<`ZhV~D<A$4f=
zw-}-5&iDKCC{Y^=6v{s@n>t_xCdc_?^bKum#)MwCg>rs<1n(3wuepESu)k7{6;;0z
zS622MR3#C^Tu5m?9U#D|l+dPEbV>0_7nV)V4@fgia5?7I!M$^-mGxGAXW+Ivr=r9h
zEoFEU*V#!;>P2F`7kktYyj0c~qcnqwftuY7(A^Zx1f9YLX^>IlXg-H?!+9gX@_ppT
z1M`YIWz1Gg%Yza~srD+O0z2zdik$yqy{Up`?&s<txts5gr0qz|0T_9=$d>KZRNSS=
zl`eoqH59SwHiv@iTQ<3?F_@vUsbaZ8&yWRMjiG!sC{YO_<jR${?A!Rw@JU&&voOfP
zQ00nSMH?@=2q09MZhlL^Ir=5=`R<zFq89q0^XmF2vg^tXXZm~q>R(OJm!C8mezJy!
zglUt`2=^JQcRZ3-N<1TX*Q!j@)@d$2hjtALS{Pt(MdD}0(*A*2#Y-%q{9HV9(mIh;
z8HL-$y{59CAC;EA&0-$4R+-G!I>A#@j`B7}8)wz8-d;*WeKr!Ey)u)3lO{cF^WaGm
zara6sk_-}wxj}QLH$*Jjw_7ODRc$dY8#ZITHlk~BwbsjUTy=SY0rvJ5{JQMvV%VNQ
zMR>%UvZ^wF9H<CQyj<dP+sV?B7k&qTNRK1_Ki1wdEUR|w8YTs#K}8y*TR^(IyQGnn
z?vO4Cf$O3{>5`J}2Bo{Zn~Uy-Z=ri{@BKXQ`~7-<c^`+@?QyKN&U4N&#~gD^m;dP?
z$B!u(e+`UT5BtL-Q&Uj|12*)sTn4RBfU}O|+qqOgiGO{j-{RfW!`Wi9z@2tq@PiZX
z$wmx?$nv_IQT+evf3(abse6}!JqUBxo>mwX;G4MoW*Nf=XyC5E0xI^!hhcbfEM5|-
zzSmR%q0$62AN+g#4M+t)ly4_gp9xX9#}0={5=(~$bHbo?6Vxgm`JvvB7%~}T5qojM
zx$Oin+#F9hSdC=j^1cq~1qK(D7JwXKs-f@2mg03W6SU53()$83QHTSSZHau@z|uRL
zQ;Bl}-8myIVv%1%1Rs7L_RflNH1YNuQAcv8HlENF#3}ZT%)x=D^WJ}R`qhWlX7C!D
z*M)WDmwrBQ5q@(O*)M!@A=3kYxQfOxMum>W8@PvEb=&kO;TV}}VKu{cD$>C|sbK4C
z_d!WLx1T;)HlZQmB=k4E@#v?V+FeOQEehZk2XM>^jCSk1`$p61YHW!&cN?j{AlLpc
z&9gRqea`zl9ruM#<GcMt;$~fK>Tjp9I$v241~3>~eLBl&$oqiJ;kVhP*)l?>&}38}
za9&eWBqSKARm|fPOJm|`5U_`ViFqI(2afPceKl}CA4Ax0Z0zI<wM1$*Pk-Q@0DvO_
zAy^V(2nrF|VH2pHXsk^~-`GQFGC`0$5x`YOW^rwNuDin?=3VMb#F33#&glmphqbe4
z)_=z8DN0n{!h!9)tGV*0aARZlysKw^&ES~&oYQP<^x_~%gEVy6p62E10s3JPng|Zu
z?^M4vpCyA=_MI0FJYX3=5@<Piggd0owcJ_ebm!X_&Yc!2J%qwDHP>#&!^9KAGFqLm
zz3VkI5ZK`p(i9Qa<Rvp3k9u~LLzQsEhw(**scAeLhEV}98#f&RCd%D*`^NHx!q`o@
z!+h?87HhX|d~;oZ%OK!+^&IGxUgQ2uLjpqFm-?mH<&~j0W&HmkTV7r!wQI+eP;_u8
zp9$(9>d;=QvDI&w%Lj~KzA3Ay&<M1wyX1hP)M??KOux=PU?^WhwZU56vJ3bneFs{m
zaKhi9nA#}S(m}AyiqCER3WFS>_+ACJ3JB_o-UXJB=)ji=5sWkQLoh0H(D)2PLWo!4
z=~oHz$N&=vsMk794u&7YQVWTo>NI*UcG_`({ZH-0R45+MXx1UIh6woEZz;R+hkqe(
zk3*AKDJEc50|pGZyx*AHo|rp~I89^BEN+0gTPUuF(891P8v}AuI0p$Eo8mk)>6vtO
zk-@>^T5RC|i0pyIB6=n7iqaS}%VB<amc?=o&Odt?I^?Qn-dfn0CJE~*Dp`{L<Z%Mj
zl7SP5S=eX-QQrE@!6(n>?_WwCZO>;SPlT7yjGZ>j*E*6Fty-J<W74x2m1(XwM=1;!
zm{{hRBeD)P-C|HsQoLSR{nWQT!aT%vpErMfG%(&=ajTvF6<0G^$S7Qj?lBD;l>Wh0
zZ)f`9W`X!@buA`^o%RpwXMD*a7xv|1G!76^k|Qun=j#U(`J594*Pc#>5sw0+q|zLG
z*PZvOR2@xkt4-F}>+iqW%M;V`-4aP?EUqv9BDA&w`&%oE`X65o`kw432#n#&T9tL^
zbr1Cn=IlOOvT;qSmwQ*n8Xo(09xM#F281oIH1Uo8dq!;QrOjkH65ARImy3D|z{Lsj
zx_=FoSm|ibLK*aQoRR~(-Pv3vqtKy`d+BR=PW*dHG<qtx5e{7X*83G#4)FxzqG@JG
zP5#xz?hrMP<<e4Kbe5Wa1Tb<)RRRZ&zVwwlnqGRvkRrb=m#}c!h*T=T1BSnxkTL&R
zbzr?I4cg1+f>c5~wNY~%*M`&xHfAlYq&|bLHeAKpei+*19`XgFaYa0_@*V;=DXU>p
zO8PAi<OAN9pP|TZ&G@Zn-<erG@?L1lC38kC`;eG)Hw~$Dt$8FxCSBF@?rb@iZ5hNE
z#or&3HT|49YNV*M@v67%9TG>y+TL$k$i}keskHUJFtV5}uGqPG5)a(;2?0g~+ukbv
z!6k)--2eOb3<DcNz6tm=+vGbOk)VC;1`NlxJ6ZuFmoWbmRMR7<zz|Zv{BCqe?H4i>
zG1QwJxj|p~Gb~u^BcygvA=(hMds#3M`>;?emmmtMhaec1JH*wqtY)o_q3?<YHvYqA
z#oR?)&xbec*zll4`rnRY;>7=U9K&EXMhQg%9Mts%D{eCq5)F5CJ1gWWigg<X1!+m+
zdcy!e6jcp>gyQ*^8af}UE!(Q;FBd(SHa6nOr3k|5tjx-(fprUM4;M%odp>8lXFeB$
zu(G`n$XYO(%%NmVRPl?K(D3AWv0~wwSbI(FcmoM}p%kx%+Y=LCW6Qn66T{y7lBvAs
z`&{}^j9d8?O~YO}eZ$eh65-I%;2u5CoD;&}r&QO&rma-RgQmyJg@vzmk6bnC=Wp8#
zEN|3T@?Nd)<F2j8nU{!u7j^-+=$$801*XdaP|JR7$w3r^TW<TvkHm)5gT$!Pov*79
zM;lkzy+5?XxMVZ8eD4F)j>gae${Rfi#319P?)f+h*QGOrYewq^DtR5RT``CLGD0%e
zhRaC_<B5~1sEc7o`8eca?zx(4j4O^QK5G0aThQ6slFx8x)J2Ssr+o93lWF!sx)Ptl
zC`lN6xnYmR^{*7L%wQ^k`4kN~$C7j0xUJ&UWfWWB9%*<M%Tf4)!_K+D*!Ll*6Ugb|
z)kWw5Vxr~aVRSd8OXD6n#d(X@S;Xi@t-Lqy=J}S#Bh9HPV>E=Vpr_Okp;r?pTYP|l
zod+r9UVe9f^HW*P+@@X21fi0ODgd+{FnMcRTD+ItcP7_bH>b{qOnY);S|UGJBuMwM
z4=gxND7o>Ia(Ok_#5^vHZe?b-aJ2iyo+ky{fUNYQ{5A{C9pSL1CK=|kHBTuWbPgFu
z^DRAAK4u62JxVmDoa1>3e@zM+jNvJ9@uO~^hA$#V%RY;tROr#)B`S}gFB_()(`4Nx
zTn;4ZW8@z+32weAQm^9x&5pZEZpowc%LCSizEn<4gCLy;8C<s!RDQv?Wa3ust602v
zL?hDPteaVqwLC-HlpdF{;Uq-;TE`RZQ=jQETOM2!F?x238@y+m+Kogh2~=oo;lM&T
zOXLAe4IM+#w|!oW1<B68_(l)+WSZvcb53>Q;~Y;vU!vK8|Lp~^JzdeFb3e?ghu*mX
zlzQKWv^9Zhi18~Oc!~E7@n9%5Cw~>-Ux%skfB?p2r5vsPLTZ5c8Yam1-+x5@K&-T!
z0!$dJ>9DEfWhW`78?i;x34nj3Ha`XeuLq5TuVTfi1UR{J!X7~n8{stSux4ru`bM*%
zSs8xzdL7JvW+BsWpu4p?z6h$G-L5@@r{I)V)c=^W-0_mK2-Z@HzT9kEl%{M9Ua%4h
ziu|L5&r4UOH&@iuc-mft_53!7)-MTz&oOY2Zp;O|uHJ+P=jjq^*zO<_#e^@f*h|e4
z*N573V)J_cNSewQqpI@&q#kGr;^3XW=Ga06-8JjQZx7!aFRm?>F8JmyqxDg4#f-;T
z3Ru5?fCIlJkxSx;M|ga|&)13&0Dt&Ukzfl!*l1B`V#Mz9!cwYo(1aHk@Roqvo!-N<
zxauBj!R{379<{$^Lr0V-%1b_rA#zx-P7qs75AGL=hmLEU=l!%i1s0EzsXT;e6NDNc
zL1Q5iiyzL_=+ml=`oONTV$UC+DwXrl325e%KNt&M-DA5-SG<yQ;>Lve60N?SczK?g
zhk6oHRtP7>f<I{!y`=YOP|A~0kI5SiUIxate?2y81MO_ZZ9Wi%+p^;56274mUvmZ3
zH@Gylvofrty4-iEx6{l&bel2%by(AY!@B)x6v}*rdp$ttAn|17wtT;V;W{9R(%%=8
z_l@FuVLKLY_eUYqu=jPR9SYgrV{h~G(g(aezsMZ5Uala?WE<uj&Fz&?HIXt2I3Pd_
z`=kV#6bG+6?iYP7i&M(oZm1mY%+0P{>?_`rbALr<tWAFp+zRfq2#&?b(Fc#2%|s%H
zHGTarvOae$4<u>L7qZzYk~xQ(3Ef9Y-dl34^JSdq$x~Ema~qceG+k%7+vgi!m)qcE
zc0bWs`%64JE+kMCH;fs*k2VTS^}aDTA11kRUcKH*WAeB>*<isIX~lR8SPB3850X!q
z+ml5}0`ew1qd7edh)JJ>5o1~XyA(NgCJt7uj4y6h;I>l*lB61E{LNYs@MF*H%ap#a
zqiCFapZ!8lSJh{)EX|4_8nG{(s2#6+P0S<h>4dWPjy=Z#dmKR<;{Ksz*KVRcP$-P<
z3Su0RfvwEP{pvgN_Xs&fr;+V(bt7d9SzTTiEh<J~lwt#XK4FwS8ceHFF(<3e<$}%P
zNHRCsxBkH*Z$ia5n8TNk*XG;Tz}$M?_MHsMGbu79<x@GIyzj%NjPIt_OxzAvSTYqj
zjeg0fbh@X^n$J{gYS(#hqmKU2V^tH2Te7+lryNWr(Pz9LHM>YZrN_B6Tk7(%7*#o$
z`KgfQS$W((lorcP;~dDPdwjJ~Ab%504~_n?R>OZPx@UN0=h`1r(i?mz9}025YeAP<
zaIb1&bb;mk*nP|hj=+6MY9)AZH4%3SFPCk$WOQ%Oc+CEd)MI`N1R{l%5OTPV*y_p8
zy$qX^V~1f0i+q_)LK7-rej&k?DMOdwOq$w%XPC;%ezW7ItA|7G_Cvoy>7a2X1I8K|
zlBY>0*4^&kP6qp8J@<yx3^g*W7}Y1tada-NH|+5XrF_1wkjYVEDuV4}A-yv$@s~Fg
z=Z{l`V9AJBNUDzG3~mc`@9<(Z`2$5;>ehYdlFmxnquUVw&|Y%MtKOtXF9njvc4;Xe
z+YpIYQNX;0AcP~_NA^XF&tdaJM&pC9Q^9pK|7DqFnaj1fpgw-wMUct0p0i;E9U!B+
zMWT|*v*tX-CX>B2_9x@Bk+mxQHBI~C`SpbMGw>a{5PcgNl}8P)=Ztyu0sE0!J^E`(
zoTYOO>t0q@T{0O{Pb_l;Ito%Zl$K+n9iBU7VziUd3fITRR(oMx`N*-7hbo@QHB8N>
zBSDv%WgXgjBRrqS#h-}gfr&5YmBP9U!HYDki#}{0y@5dp4^#hl<_nW0F-CT&^=l5A
zT(;uP&ct4;`7B4P_@^e@A6@sPPEI8{MfZB>g@TacHm!O3iWHh(w$(Ob^Gg4@JQ!wg
zJ&!UU%2^p6cR4$n(t&0GJgJfUzwgTLo8xCJVKAI7!~*EKr4(w6EMo5UUa>zqh#yHh
zpX%;2jz-X8!Wsrf(=Wp)YRLN*-qc@M5>xmS5-U?Pi|ocive+Tm^b=_K8$^mz)sQ14
zZH<UCBLv%w?sYW(k9H6-OiKR}c~vSI6iC_qG=x0A;uP6{yO|LMKcd6GR0TgGj3JZ4
zozb%lby9Z>x%!MQq@AL<Gu;{v9pPsNNqr~~rHK!1)Qfb~KsL^ml{TK)KbGp{H;Yqu
z!4@CHjSpWBoA{>9V;??>@avz{m2};XGzuY`yNdGoWcO8v9~%Fvwl&e6)#_PH5(pFF
zef*__W3i+qzqcg9P2d=7HB+nB{NP#KpIX*xHcoBHJN@=xGOOZj&o%kh)RFbtbO-M%
zze&6R(VM8UndQTo!}nYd?KaANm_#Hgo)0}!<=1nt_yh=HD$0*R5xad-u=0_`HmJse
zU-`WJ?Djf~!}|kJExm$#hf-lRv()nHjyy`e2Wl9#2e>tBEJQN4W6v(d5tdy+YwtlH
zXU1bPM%lfn`fy5|9J#JKEBG$IENAFKEVF*405jtjY&#2w@A4Iumh?j2eFUteRjPT|
zN$D&#)<1MhF6}%7%STk=vvCRE07H{Ku;pHJldut-Rl1Cw{X1zLJ6w&b8H(LZ(}+aZ
zhaU88u*jgJtJRdpp3XR#Kr#*x&v|Z?toaV3c7NXK?I!wJ(|2;fY8nT<kxiu_cl=Ac
zev@51{XIlwrq<%C-4F4w3*GGdW3OzNd+|ln{(FwzyZo3p7iBNY)gznUP~V=MtaFHf
zUE1aEb;zeZwkPpxP<pI~<iqD>x?%g8Q{VKN`H;?W*hWsK)FZ9?zuE^!%_sAn%ze<!
zY+ZifednF3>f+OfVRvVE_h@|j?WM!|NLSq+wV8M~;aP&Kzl1rTZI0Ygo^nx8I1vYr
zuq@1$+G$I)^#Rd_SDKemTcuR__(DTKC?+pj5a=BDA4tXkksKT5R}gAUo#R&T&G~MO
z`7HaCvS4~LCNHoKxyTU*KTvwlAkK3bVGHWRlKvRwNd~M}8y1>hv`D4Gt*%C3mM@z2
z29}}*&z-+@r|KZA4qT@O{>{<Ys2Z739%|VadQKb(r#G^yI1&UY*^EKnqUYRF6z|_Q
z4?^Qtb?KkCcM<8|(Cqxu8!H$t;kK%UQaQiEz-~ip8VqIkl0HiOh|5cWCq@6~%wJHd
z+qG!j-j|rQtwm3)<>RNmLL})PP*@_LbG2rgZgc;e@zUE7w|}3KDCLTH6n?T~F?vkJ
z3Kmx5(W|L9-v_?cL}M!uy*8%;5=<No_R=Q`%rTK!N+714<F=7KTBBj{`r|U5XxDQr
z*43Sq%rV&~4d{36Uerfp@FEV^tGL!}u54vW62YTwaz7_?bK2pNWNbJL&BIvgPhot#
zJMU1c|CQ6mOlPL&qY=ofI9$Jm$y>^|8|0C_G?VaZof25YePN7_(#z8pz!l$lsy+*c
zzUh3U))KEW_{cR;%W@h}vz`-d8#b5QyOrj%7&U2WK9)cx0dkViJHbo{{WlK%7b=b9
z{VzD0Dm5o71Mde!tJkZqss?p&4-9S7ntZ$hI#OF4pqCLyD(kjYJT%Vz<*!l4SCBr^
zOK;#(*Z3xmhl->;7+4vSK!Q`gbaoLn4N~OgmPld|q)URJjb3HPJkM(a@l)m@=jdBc
zfcdS4Ef>_t1eqmnNKK#$81-aUNn-1Xq$4CkOF+8?i=Z+LhQfw=!jV-{sRICyJ_W4{
zezaX?*`RY?5xuWB<(}osF`fZad0HagpEe(_`-z-{_k5Q>82*LYCSktK?A-G}TS?iX
zH!=+*``U`d&GYI3AM>_T4jkDGo5|`X%8i`2*DINTzUa~9yZB3j@cHLE!-(am$5aF)
zBA!w9UJ0W|8nY!&TUE+ujV!J<30=1Q8qIUHj6RXRl#)m9Yn=3&jrd`{!>e;?ZDck)
zywO#Vk1$ROVKe0qYrd206gWrdkzT@_Bb?YwjPF(Z31`cV=F%=3yGA;jH<y>Ma~iyr
zy%V*+T5JaZzmuK9q2A!fFv`xn<8J+XJvSSFn6F0nqt_=_zzmX4@(X^125LBP4-+qA
zOy?cLYnBZ6Pi~7usiow?iFMYBx3#Cu4&y4L*X=5vOUdA_J|E&66P<?ia?6%Bht44%
zI%m9h@zMv^s6<_t8&?VI=UoR#-y|jgAMo|hIC>$YMeoam4{Fyy)G2I#ufo3)nqazi
zjZL}P5Rqp6yE{EH+)XP-dP2Bf(l(<AZOzHc!T?q%4qq%ZUkB7>M8({3{-Q!XE?DWw
zbkSVllxZOkC-<FNI}|6KbF&}mEt6l7g$0B|nV2hw)UdHpqZ7}e>F%6P>IL2N$GVII
zVuZmUDdmC)2%DZ=)N}!%JB4{3*#En=*{b!C&})4uQix1wUc_K<`S%7b=fg#>)y;vV
zGG-}dI4A61h(rQ=sJw-jcGsF|M>U3y4vxktJl{+<HshkdHuVutj8-{o<ST#8=K1hx
zdE?e(rlzmbJm>Fyh{c4tnccRda*M@me8mn8C!7&k!{RUDU22F#gsum4yg%2clIY%`
z#S<R95Ba1*)jL=zcb(z`h1&I{lIKD+f2=gWk0g2x3`;5_dQ;sF8oq`8o>(a&m~FlR
zGT|!FX3Ad$>Gn)Ft4%VdlHse<(>lpTigGsR+GLrQn@=b!RFnRaDeucTX+$ODRxNJ6
zf#l8E_c7GGm)@lo{qf=N$p&i4NP0vRymykox<f|5{iBD)B*~11RB;fI$aB}Cs)1#p
zgI_=*qZQMaKnRHmiXWQdUizD2QaZYe&NTH>t6CG&MPr+~z+N6%`{vK!<B0_CJ&YUk
zU0#3Z6gn4f1jB8e5h`-}@0|X`3G$$N`M^+!ag`ITtwNHNaAj*kSSZ51IUVA(N*B@F
zvMi9ik%|N9_VL!}^!Cx3a#JVbY)l)hf0r?t8z*`6`+t2Exjlf+)k$#o8s|t_9U!sQ
zPvzx47-DffpLbjM&ibEf!%z7MLE^S$#rjp*A(3G`|J+!F$$KKAj?qTel4A~>OA<|t
zTA-7VZ8=pOwZo{_`2L{z_CRuu_HTXtpK6;N7G@)SIYXjS@JMhOzjUaC*O!DQfZq)o
z@xF2eD-T@g&c_{shv>Y)_mTcg$j8tyt880h7#p7mCTBLiOMfml1Tw+@gW;zF%nJ!a
zgzp1}hE$<wNdEq){_^+lF>v$`HK26ZV7yC-l37Jj_d?+j5$$8Ug+iOEo%!WAoGaBd
z1}5gd^K(AlSEg{CdCa^uO;j(E#roGdy+bY3tExxYV^_R!IiNzdV2g~kph-Q7YU0p#
zF+PX{Z~4&2Y{_D=cqabk4qDLzfRM)G--RAf6A?}92XG>B0DS_E0A7)iA)W}y45r(0
z2TqQ71kmUsh;v+v;P~PdMLly^FtGGrB^}`Bb%#N122dXdXg?eiuy~j8TcLf!{ivr%
zy2%vF;d!C=wtp$qJ-;F;7{fY%YDRS4`?#h+NLVezbKW7&l!EOUL}K2~*0h|(+@e9D
zzN|_x61q|tCebAdg_Kbgrp9M|<GeEKLTg*%QW;=hMGO-HBZxv6P~Qnho|5Ljr-~Vk
za6kR#Y}Zp+r)eTm$&t!>VAxCum?dhB)yYhH4It&>Thj`*zsI6n=+CjRI;zV1{QFo$
zJw3AtKnNu!c4A1H!WF3IzhP=RQ#M9A-5iJk_LK_zZ|3FcWb^=C9x^vO(LOv^lLnaz
zo6;e4D#1F~V#wu#d6otYyeC{O^t6hGfHd3z)RJtM#C@sAN4hl9*I@qHb`##KXVZ|^
zy<&=l)_cyf8Vt=Cm9?3Iz(Q@R4wys&Ye=ll^ss3ZR1P;oXcp88m@g2N-U3=1=YQqW
z-j0(|RCb(I?9XJ~smSL&jhO|+uWZma9<BjIBD62-FBjV1`b(TFiX_$jBOnn00jV1h
zSOw>c(c-}Xh?D1a8<~L3(?<_AM*#{AsOVs6pwkf9@vHrRd|{x3_cIoG;+537P%7p-
zQU#Ks7#Hozme6+sun05ZA`FNL;6<;IEdoIGoytF-xjW6O<KH=3S-lpq9^Sp9u43Zk
z*kFOPPBC8<US17hOco>6DoCxRk7m#GRn#zWCZ!j~@l9cTu3eQOS}CyDIFiqF5w4(Z
zJ)#6op<@p*Jxa8~<^AbPzN_&2$;uanl4}0(WR(AUve1GK+UHMCmV^&HnT14}TY(NU
zM8DanE8cdaEzCQ5sr%jk;cf$$q7@t{cpop(x0ozDQ{!5)*Vh;Io$yxjrt@XF*ZmuT
zESG()-h%~lHH?`GDh*(%G0E4jGtn_IYsI{-o*%5uhtvJW3)Ey@G3n9QyG{X%7FJjb
zfkpA|`nO6CzwZVqT1M5u>;~h)p~fJz$Kw%9tgWDK9n3VXnV<3}7$JSgG})PDTKt*0
zuF;3Cm&sw(WolT6_ifdd?!-=Kq*pJc#>*t-m+*H319Ccra&MNiNmSr)h$^!N4WXp`
zMmq`zoyDM3PY}Hk*^eP-`KcvJ)FImw!2WC^1{H~+q~!SGuey67uLPypN90iMsiLfa
zZLN6!0N;w=0FX*X<`zp?@T!m+F;JM))&>ronQmJBe|ueE!HTTgp2dZ)py?59hjLPP
zV;=u5j&kP>dVxBDhhz7dO7o<>=`tBSj$*2@T&8H<hVpEMwBq7<z~0=rJ5wx-oPwew
zO5j^hI4*1B5}@ATtuq3axC%VHv3_Qc`4mptY@#g};FA!Gp>OZ)Fr1dsQ1JFH$HXhq
zQ!X;PI6g`cYFkmSpkatuSElic&b8Su(_?UKhcWO^?<V50ew0Xgc!T8kg$$C5Jb^S~
ziW#eeh>bisLNh(?+%}CVVN^(j4=-L}o&HO{YmE70A*!=gEaGZGGYE8CmNm#Z&Xu{*
zO`5eSJN?ZbsteU9h^tD9g*5$lH2L0O>iExSBK#+s{J^NL{Y?6lWPsx4xk&E~>VHVD
ze^Ow6C!YnkvmLXQ*=j!5+orA8p%8u5o|2k~tuOb!5EU5lIBXKGwpk#pvYc|Fyfq&9
zCZ}9simEYmipv|%Fvn<aZl1dA2dl{&30T|r0B9xU9&f3dcmw?jWJ@|D7P;U^RKg*>
z)(Xafk`m#9HFoqxNVf*qv+2BH1$;(m`w&0D7mD7{?$!v_M;&_Ch{bp-ydQPDaw806
zP+o4nvMOvIj&AM1q|jL|K_2FO9@}YZ_&huwlIv9o9gIczmHj>WMS%!9rpS3EZkTsq
zG%{hP_1-b^O<_u>VRFF%H_Le&e9WlJf&E13>o$=n$yDVhA}xydvftF=FXk`_{+)Ux
zW<FR&p@RgeUz|J>KjP$+pS1k@n|~fWaypneS~Vv)jB3NI9O*b(z~s_0U#G5|Ub|ZM
z;?T@eJzFVDd^BH$iYJo-uh?ONHWv<^#qB7)jm{T4JsWLyC3qKVt2Muo@P-6<DR5T>
zdTUB=rng}uTB>QH-V0Rafa@JzzawHEHkTr}`G}+GgeB(UgvO2J>CA!`94hina42UD
zZnyqzm9r@(4+PPBmN;C3*!rav$!EAOqzl&gFOxgD#m#Iw_ub<_{>e5WAUOn{{)5c_
zo2QR;g)895qk2iJ8fn^8-v^4br0}?v=W=M3DnBWbN^KM(CI=PJRAsvla7Wj;g8Meh
z^l8<9kW3`oC`>Zy+vruiDF;lDlWwetv__+5vtkZy$0{>^D9kbTwJgwTdR#~^8O?UD
z&&VviwsGaPZ5hu~7rWTpkdBj%qcA(v(^z6+E7XYV-y_hUl*>Q3pR`^$SYyjeS=~Hj
zFG&$N{h?eJZF@Li(!fc-UC3ql+}`ox;MCs1!LjMy#dS}Xx6ZEhn8kiFT2jX@PpPV4
zZ?Gryq%cPL7g(cbrHvD>L}}rha-AW?2ZG6o8du51Teqlpw#rkx@`zO@zr5f1<Xq0}
zPHIgm187ua-d&@Pu1gn#>?}y>*>3Mk#ns?l(9z4%fWlTWLhX&`)HA|!#eFdn@2=ta
zr>XUY?gPs5uw!JbbtjO$bZ9MP8e!Yv_8GFr#a<c4m*5K!zUOgPt}iw4u_J&NH!>iG
zgXUw0WKppD<_UXxVpe4ov!7WKNW<|EDmdejb(~j86Y}{uXmJmTzXyPzB)LNV?{JXn
zVaoc?a3Jz09PGWV?naOxdGY)cESVbt0oCr`-~3lx=;eXw(qmI*XL?UrmBbOXPd6nM
zhFyO{bC}Al0f_Y%j_KLFKFybgnLJeXoWx*iNL?Jr&U~#_QXfd8$tdRYtUD`IXj+ov
zSRe<SuYhtvU@;-h=b>(D%DJnN6D=@&drtRN(Hwmp=&aOZh-hb~xoU4<;FLUO{{A-G
zB1by9j;gU_u9Xlwri@3sS+81NetWn#;oKs}t>sf8$4#vU8iWCPftke%geK)8U%Ypn
zN?pcEr+ra}LT>$y?d--%qMw8j4r4WC@$_Sf&q-50|DZ3UHN{YbfXNE&QnR4}lkHOG
z2R_?N-A+c$YVWY*(I%1DCFk7BINgRz$+h~b%rlMT$-Wo^hXsQ7Qz!f8Cj;M#SRAI0
z%jsCP7gH;T*G96%EWw8WA#`7Ba-9XRLH-hzt<cND%jk;1u<4>m8-#-8f(p1V_H$L=
z1a*$^?>Q4V^)d~Y`BON%A#j7L*<Mln)TuS}+M$ca$ga(dL6i?fDu+sWZ}(F=&w%%Q
zgxf{BG9AoKf`RZe$bYwhzZoPVyN5}r(KW(MJ(AC0eU3C~$>p^>KH^CAJ!khGhZ*P(
zQ1poY4r|67gWvxVvP}H{gtf1&mO|KqPv4%7%#9o>{_9`V{h!9&6MWow?Z*y#D<z%v
zto%M{Lb*!bI)CZ=XVp9%v2uYV3QbK-U>*Hx-(YSkS&~LcWEN<Dznd^AJCn>%*8PDQ
z*K>xUd7OgCd0c4}034c)Ze>h><&W<ENmAX`#V%tB*Gad62jN<Uv*2pC100gEf?n$S
zzGEO_g4bHeGiTlJkzVC|tTL0f3p5P)6Xzx(31tEF#J~#ZH81(y#l;vp3$+r;B@<(*
z04=uc@}xN%iOuE2uhK8++`V$7UwY0LsL@_?O4L|D&TYM!hq}zQTH=Q$%S^f$mxxw2
zdwB6C@^O#N7B$fMnR^en^1T}QPE9={T(8afa@?y5ysdxiMhtEB_{Pji#XPIhc?R}6
zfizU`5lIj`h8r6jch6Q?dL@_NDaj)d5sds4=O=)932|>D9j-q?_V<MM9Qh?cg8KXp
zI^r{bH)$)l8d6ncL_H*UpJGm)6fy~9Gyx;buxba=;hj4jHs%p2N>i^^+HIWV7tV|+
zvXRLdVL43O#8zvC>4wB-Q;E+-G2TJ8F%1>F(u}m<p-~8@-6&(>5K2^5N_*M0wO6LB
z2%r=LoVTDP3jm*hPdXD(vGW%`34SF>FNp|WR-!!88V1&CeHLEqMI=J<HKF`x4*Pos
z1i8f1{rf(W9(2dugVKFf?XpAe8Fxi_^VdU&ZwWM^9VGN@b_i*(_!rx)5gW+lr(cK<
zZ`6Gvjax|W4$R2W@P33Pl<2$~SU&_yaSfQvT-hD491e}pw*S<JCpPSYR0+KAyy>c*
zU{_ZVj|4tbAA)1N*rc;@*iE^q$kj*O^WEW$06=afeo<>hX~F=Mwanc`t|?E2{vy99
zDnp&--nFClxx++AE1#kV<VuNFUvSrbjO`gk^|mvL^c8Y@fW9vi_a25PMuZ&SHgR}p
z#2|b_Abi&<V!+O72xc-$m{JO(Bu<uhcs~2~I2zyn^e6bab{J04E;T~??J@gdR^m7V
zLVVG_Z^iP@z*6yTzt*BF1%4OzQXa_GtAAgc;+KEn(l>dp3M7ByQlQ8Frp>hT?{EH7
zq62SOuuddBsBcke#H6ZwZVeG1WT2GqK4F*gwZ58%M~rqV?;wP*u6$^Qz<$-f+-O5<
zCZ(D|_#)kBDoAd$(p;q{e^l{e)3}%4RkgRdk-&Vm44iGNLptG5(*q>XH@k;i>pJ;U
zm7nR;*yPGtS=X<|2BU|%ny7i^N0^Su*{W*7^z}^#bv8lodgA?HJ-W4R<xHaa$Hb+j
z)p~k*VhYsgxXML~_ng<J9A<a{m%{x%Ag;N!G&c5*rVGL57H(98NAkfMB3%U615Hr`
ziOL_&8u1Mz8m)wgSX7k<<9Z7D#%MRC!5pn(_#^BT)klbxXg-3@(wlY#AwF;QN{iXK
z1y(U)s-N$tw^s*v+3&U)*rPsIgowty%@EA1>r`t^<i*J}Fvx1XLA!)aN>8n4Onz^?
zF6fjYl!;KS69W27=)9R<7Mf)Z*{<te9x?6CtHZ7~wM+YHC{am`)uRHT3<7%HTd6a$
zf6;CF-XG-!HBUIhNgL+V@oz;W0-V0w|NiE`5im%whEDt{oL~Yn9zDNDFw?-J?ZDzH
zvk))m<#FlCW4t?Rn=KTaMV|A4Z>GvT%u2sb9>iH(vDde77(!6j>|@-bVlw#yQ2R@W
z;gF4C^0>>$vx|bQB#y4cL6&ohz=l)CA%Vxn;3tZ^4^)J|z)TxQPDiF&pTaBZ;CRtp
zOC^5$P*_~wON`3uviE+8e=9jnQfn5?XRo(dSJUC7XMr#SRynBeY^D5*YLQk)4A)zm
z`P$UI`8t{J@+nOBH_qoKWUlSV-ZV6FCL@_*58Gc&2flGNq=2rXXm+PcF4qWp1$uGY
zj7opG7d5yv$Z(v*;Xn}!N)Z?1?P%~rbCcge1%7&F2fIvT*T&(?+L|*DqjByPyX{|A
zDiYF0u#N0Ml(##t!={W5Ii5H!rHO_M!08#m-EHNzmkz<^=9<9Pz$v=Fa>zjU|IR#;
zj~@05>~=<WmXRe29u7`}$#Z8Y8!0R8AQlQ%b<{{2N7!m0({gC^Cl)^n5sZ5GhaDK5
zXGzNYM-5H-r-n8`EdR_;{ZvC=;<v4(ztznB`<uTZ%$HR00MmC_sm>i$Vse^J6V$I%
zmdVrHK4Hmj^V*#0(>~vh{V$3G>R7LG^G(lut=#Hyd$`C(t8%Axq<aZM=UAtMdCgbW
zg_Lmxbt12ltyBA!qw1VfRORyNn!p7Vtfu{u&byPJWb4lr;uv<DqvZJzB+tK9Wy`Ob
zmYZpa2QD+a9U1ft4#p4h-}cX49xg3h+eGO#amx&RW8)2l{rN0<e=}-}B|Ne%%X#;^
zcdB;HLF_~!hjbFV#1+;kg7@jBwTo!fJB6dxg!AYQqy5>ccj)Fcxo}`B1a2KDP8d7H
z+UZO5XnxgTLTAxyScphI0gUtnHnR4FQfH^x-tx<{wp4yML_8Iv54qmO5V8HL6va>#
z`#`Ck)x?m_jN+Iq6v%}9t>z1RCsGWC(qKN3?`St8Bf!@|%Sbz(5a4{}9LSP4RhN%$
z9igx{`=@R?%-!d;vjMm$0$M4`-)#!=277khzwKaZY`C*3;O~3k^pL+nJ$)l>phpB<
z+W&u`Yo0(Rbr)hK(XckHS`lR-r9#y6INII^oodUJ+AK|fYyF`yYu%~+5KIE=Rj@sh
zN9>iESi%78PsvQ>(CZtTJ@latuFSq$@6<Rt4Tr9p_0ahCBITSfWPX({^}Xd*Gc4vy
z%}qcS=_K5BFks>uIgS}92GCg$=bd^3^772u)++QEf^U;IMCS;WIzR`uoQJx_&->hU
zf?9h3PPpZ{eQa*}e84h6sYuD`2Lj0S$4sQv0=dY-bbDRbReu$Gc>N|QsonN5F{P=`
zqeF%m6Exl1+Y;|zWHP@iW{g+9e}`tD@0h;~hDx5zAWNNncCVb=T*Z@V5)JfuA|(y~
zB<f?i%6$C)@En5qq<<2%MWj~xn>SCjdN3yM7sTYRe}D7e01d^@JLN{|*KEXkg;<(&
zYG1ul4<<)}MiD7~jnA&xbKFe8CbGPpu+Lxhhw4Do+i7iA?aG(5Qm9)E98@YN4&{VY
z<c$;rJFa_S&*KUtfgMhLUljC?9<1xnBKP`HW-2Yd+IBr~Zbg46_t8&nyoE@lR+hOk
zeS#BRy=JqXRb{tb+`DCZxJJ~hxBJ|mQbyfd;(dR^SRn!8`I%F|gdfa(eLPq7@l6;G
z)2js!@>|5TI=K|?i7uWOKCvqP2<Go&si5}u?#I;iptuJ%Ga8{bU=<xcpzHg3bHZ+a
zq^tX-c3M@|0a6Bb9ESwM&1vPTdYtE|DlG<7W7D#^33o!1>Hdhv3r3>Brr?yCsIc=?
zJsM;83Ymp@_Lch`n%DHReOzRoS0zv@S<V^}h}G@SAFB}OKJrR~KAP{dxXG5I5TS*V
zzglP?7ATi$e34ZD&rm|~FLJ(JQOnox_qWIez6DZx-G4WQ{}XdI;Qag+p7mDi&gF;z
z1~wmKbP|r=XvEv<FrH2gUb5~jwY1P{RZ2n#`KB*5N+tvy%2{)?BDBh#CwLQjvi$cJ
zQfTCOE*#E#Q<H^!_Qi|HIm~K!ZYYv*jaEDquG|(7k{LB|cgj6Y6D92@)HI~j)#Jkn
zc_uIQ@B6~YZEcG^`9Fb-O--L#(`z>iPGfQ`mXq|?cc(*MkC7D$QC}E$yL&(5XWkXp
zZQ@BOC?s$@L#`ZpHc<o~QY))Qe`vD$SfN0RhreFwAp~64Pma+#h@X(x_=m<$UFjoV
zjr2xX3ajBD<aA-({qa{=2s+R=M_GXyzIkYizMY~9x}Tm++wHuU9&VMm4asGm0P$OW
z+`SqPqrhv-8|0R{bBdPrua_rEGVn~PRFQa<?(F#v6z|&s6JTVHauzaM5D|Z-wDqMb
zqsT=ompq^=DnDyeUbaus5F7&B7e%T6XTVlsj;P%HmXhmw`ysl5qs!#DJH}qW>E+jB
zB&uW*fW1=AS|iEdMkL_U#63WaC6ItjQ?NnUqvrQe@>PqztLCeV&^9<+f*6_wa%@z3
z1|?%4Rrv0!UH->Dr_2c9jp7AUKPa#tPH~oqc-=~k&CL4D(!4h3w)9+r_Imo`)xtQW
z#l?eu=r*tyFDXe7QW!6|T`stKJ@&wQ$N=u@#@F-bO5Q-YKZT11oKy8(wmJI2Et`Ev
zi(q;pNnk|4lN|$O$)%d}q&nb9)@Yz}6GgrQ{PXb2&PiPT`1jakfDdyXK;^4<4cn`2
zN2}019PKu3P-3J)g@&b#-if3gxb{JUfWqw~eHBe5xu8Ayc2|16fuWMbi+5IK8K@0n
z=f!xKz=zIlZSQP*-=&5hQS?w*rC<cV_E8*T@9YiZ4c>vpS8n&k^~y858zS!TW$5QU
z6u+~<dc7^X_CK-#{=c%pGyO0v&Zlhf)a$bQ(f{**llM>AK=K(Jt$N-zn0<zLW(#eo
z9^lb_GBeO=6kW`4IWfGXtz$5B&F*`+w_5tb6J(OETA-sl>CMM)wOVejnzvo;vB1ng
zP#2x2YUeF^d$A8+-jT|;f<p578fb!QUexy!6LLFAc9$4+fu52>i>V=p#keGJ35+Ew
zUp#Z7IdiNW1gKtR-8{B~UZ(`CH*SgV$bzHUwpH}}9pGZe{ZbL02{2YUPf$PjL&_qG
z)^NPHe|m)@D?s!4^9kqQ!CQ?7bB*L)6l*z+7>J#Zw>d}S1+$Z*Q5LYvM#66kS1JfO
z>tenQ#XcKjXKzoJBOn}7BeSnUFMnoa;R%u6Q}jpj!W&ZaN+Ht%5wKVcwin^bGSiD6
zDUwI$j{h#|hWIMJ{-day`I9R!BkbWXg+zaK%FWkE{XDE<8FlSGpkE{-(y_=Mck-6Y
z(YqH{{^gaFrFD+SIPtOiwz^ARYN<@FS+|Aq#Po4F3N3B%Np5Tjc&lus&Po3tF)<oC
z9Av75iHnO%xynLKpJ;(e4Z>zk+Gi#g1EdMAhsuieQ+p=q$-%?BEeelIqO0ip6+gw3
z;__sZqveqJxC8&TU*g`8({0TH)}q%iN3a;u8?TnT#PEr4VinTYp>F*7i+8-1U%!2J
zF|CMxf_7wP#VFb}{waN`_H38;+CaHWJS}pY1PK=*Jh4$;6ngmm^C15$o7qWt`131=
zkz6>;S1<6jg|bj4*{TPT^qpecrYkWgb`rO_GIQbp%_O@&0tQ9N_V420l4!R7??a}a
z{V5I*jkNNWDUhmm$qB(+>Og-D588dC@oTx&aib)U-bQ7{mgGLS^P01m(@aB!@j*Y6
z@mf$7=E8~|(q~~J7=S@LP!K*T@!5`RiO#A&KS9QQ949s&$eihEq1G090i&xx^yYr3
zX>@nf>n2t9)9H1dx8;8_$G=NO!FMoII*t+9xw$ea+|HhLq`5(R)AdSMcN9;xLEY4J
z9hZl3PjHE}sp<Ep4Fuf2RlN*PU0>#IXh6K8xdi(R-B%{`rF1GHg!wbFFRBjIwAc}H
zWO@^@)t-<if{-AX+bb5){%E8+z91LOy@rjhKT%hJb3WpdK>_`R45b)b#U}*?4wX{a
z$IefP7}EPA1GEfI8$|LkX--G(%tb>yBdcWNpW()U539exCgx#0pQ@&Edur@4Gl)p>
zyNHJUqS^UFJvr43Qd<3G%T0X$rB8{|{W9!A2#UH$X}CtmHO!C?T(-ZN!Kt^+j=U<d
zyLRYBDBsaubo~ajc+mYZdG|CWUEsXW&l*O5$RV_ZUq0N2c;20;^fZ$v`JliEG#$uG
z9M%4`)L=@cd8|FZdyQ=Q_lo=rDhXov0j2CyzRa$r&y9_kb&mPpkUi+f)tOwv{X3D#
zVqJWv8h3yK0YYr|a@nlzH*XvwUo5ieQw^kO&REy<`zys@e2iT%+6X=2_=G@W5v;&@
z*E2L2W}3S3G;ZNgbYb4&_dZTySg3mBxxUPB$dm>4mF<B&&t|A)<GtbKZ-?_!v6{YI
zC|U5XQgcRf)hb;Xh!AI9NMmHA@iW5f-InI0l^ha2Nz@~jSN?yYA{t<ru1Us(4KksT
z1e~C<Q-SG^t=I7kN$j<@sm9%@3k^9IRTiV!n#f%B+UWT@T>TtdvNK3)dM;)yra14`
zKV+eQ7WOUzdK*?Y*Q5=Kz&S3jVK0A6<|dk6`ah-oiWZ2fPBXl-Rn~NtlRxa!V(fkf
zNwJVT=`chHOfBc(+X9;INd(&1ypp2PAGcLT;Ti);^8LASn5pAxmJ3oHeTf#K?qDIb
zkX%pr4cfejXx`Vr`%0S}!4?s_5797I+REf!?aG0ATMKdeP^orUpacbtl9`^*#0?3#
zsW8}k$vGPmZxM{j-5J>EM?~y2j*E;B(K3_~ux4?qlqck~HPqt1q^HO4L>ViHlk)eq
zF$DU@QZ|WlV}BD)?+a)zYo}Ul-k>yBF}SUM-JNo~gQ?MC6&&;7($Fa?;UnM1s3ec&
zCSmgB%eYx*NRPS55Gb$Z=FVuQ)W$j6t5zKl%IE`%dbBC3<zdYl;6GA4G>4@DdJTi+
zJI8;z_<u<BpNJ^<?6Sb>XiJGAf@kQWf4cGRuE%n;-7gV&p(qTgfd=bsnIBMoLyfRV
z4hrVFM+o5yR5?yi%LmM6bv@ID8-!+FCQlzjCd9(m-w!zc8UOii92%K9Y>x0OXD~Za
zGBVUZ3o;>oUm1apovVs8n+H;sA`mh}bU)A07ZS;}j~&LWR3m1vtd}Qq^Hq!v`~6ya
zY_sM5A(N1X2VRRY9GK1+@PI4-xHwh*Y3&(!zb6+pCMW`n%8lGXM4tcL$$YvtpF_9I
zDT6+dl?=1yuMY5d&a+)9C0qA$aWqo|XC@|&2490Qs$)zbK1G|O3st$v)H?s<{#Zy8
zmxr8*<4Us=I^ml$zGTOjmlUptdNWOyTYezH2w4tP9dSHzCME?JtLw?B-TKP<t~8hU
zcYNh3djo^<wi=A_C6vtvL&_J=m<22*i<l-<EI#cQ6E4+RajY0QG}NCai0FDq@4KU#
z>;|!U>sluCM;wD}CX1IOxi9vZG8n147x@}YuJ@RXXZp>XiL5`R!&9;m?mS<aH`BG<
zm0#GXRT+xyRV(QIQm5aG6<j#D@9SFI`HhVZ{ac~W<BxY1Bb~1D_?P~&pY8>kiSEkC
ztd0Dx5cZ?%2JiE<nu7Fy4jxutnWY<cNIwbF))V<*Ki@WaPQKG*zLg<OQhKADqlIej
zGcPrIaK-;n2WYZ`09`CAAd+G$_Bi6|@Lh_SZ{o7IR+_Y({V8xe?mRdiFhsNO2aN1c
zq3{P6Sm{)$rnX$V>!Uw(%Px`5Fb0gZWdj9Q)N$$DemP%M@$COBw1MEs8J$zq*U4<Z
z+HS1>WuOJ)kB}{ig~q@ws|Ow(ipU?Zu4RZ*@Fi+k-uSkNp5g!>`@>KEGxypsP9??n
z8>Okb+Jc&_BK9kIBg269vCkgQuceMwpT|u4Rgu4%Vg0!KIfW4_^RxHpi^{Atoz7T9
z{^wsh)%8KjD?!8RZO=R=MRdrwbMMbq!U7$sh)WCCoSu6AX|eI%%OcQy%PVEl1%E`3
z+4a`!n*YF$WB`5~S&=`=Q)?*R>kU~PW<>VylDbEb8Y9!XX{ftN%3;~s9qQX2q*Ry&
zAPf}%VSXOY>}cF;Jxg|#^XozqI(vP4WSx418Sb(bmUe||<y6A&p4;#CIAKsz;=0fA
zp0VkYxOvd@Y8&}xqe7@nobx!6H=%43&>zRk99$<k@o`NI(A|&)dV!#Ry6G#p|C&Sa
zxm!J+%2CU;(L3*<dyMBIZXug`y!KIM_<X1?+81B7)J>+5BTlD_oY_}}E<ja9fJ>h0
zd3_*ra-)A3Iz_RyaA9Yh<=qpmV6qsC#Iia$B5bjozP`B^m3vUDqt0-}op*5_+&P3D
zf*=-Uc2F-K-+z@&%_wDtf7@8ab3x8^5zBhQNBks!?y}*P092ZEb`g=j#rJEcqXkA+
znxSm-rG`;=ry4j?E|bN1#nN$feR(yeVh2OOeAl`yM+&}4R>Z@W9;xxBBGO{kvSY#h
z=QvgYHyJ&LmSBV!4ZTldwD74_RuW(8w==`gHVpy&`9G`TAVFo4;UaAgpZ?y_9GMuy
zX7-*{vfn9SK^ovyHhDenJq0s79_-+{*P(}eIqrc0nKlGPhoC}tftwXH#R4V*n7xl#
z5>5nGY>du~on0U-VnSG+2)X=uxbwF7dcEmgZlEI)`rt&#OkDL-gDOCtD%;(OUKU3d
zKPi1Xb0oj0GkZL--fgc^UMKLiP8+*>;Ol9XH`4}T+tcA~Y6H5YuA+T#JeeJ&QSE=!
zRjWzc-}N|hid9Rr7MKkCKnw>7?|mX<Y<1P>G~#ZmAh*7_4Xosr3Rn1Q6NRb?^=!MH
zgw%X}UbjqUuJimVcrfs!!_8hET&p2u*#MAWtnw)aAlj|vIb<UGYfqH@pDs!);lJdR
z?IN|6z%qCpabHq1P`Kxs&*BdzlZ}&(;~@SYPCyY<G6hUgEV;0}nZLWT#Eceii+@9W
zcZu}%YF<B6){tW!&_8!eL+wD*Ja1bIz<Fd8VSF?hcWdiByVQ@GB5I<`J2_wO<eFiW
zf9)i%<ABg<{HfN<t}E|#uC84ZO5JM0wIr;PcyUy<cO0*SiAA$4rjS2^Xn?!iH=NhV
znY{~Ugb7S!lJ<y1kIkpFX`kM>qy{VVTr~;}nvd37`}M)z1~U?2)DRnXtD*BV#I+3Z
zV^4L{IF`HiWlf>Q9I_XOv6y@8g*q&dF^M|e=^fo-#!?o6qV)w%n;R}yz*(E#H>r)B
zH}N4-M_duq$oYaF-LIo~&@5ic98K?*ZzcmfRS?e0<p19C-@{iVxEQbBeT<Y%<6rP;
zPC+@X;(A5=Bp6xNCM{g(c$R|2Od?L*<1Z%)X&VroXhRz2mk#7#q^5luA!$pTb+IU!
zQa*22I)HYzE(E=|2rIcz`Ke9KQWrpim+Y<-7`j}DJ<)>^;BF;Q%`F9qWQ3CKn)>`9
ztL#07i$#`wnOLRh&}XLVwt7qVHUOb1Nxi`fsK(^wRqc5c`Q1mIzjPS?oJBGIZXL&d
zAA3r)8z-l(z&qu($2or`w&Bp5Pp?xRclXzl%f<d^liX#kNE6?h^(T|rKKol)bHw&H
z!$9u_m*2BiH9u1-E0X~r<0;a7Y?k&w;uN2>G29GFYb++DA(N;v?N|DhfRyVis~Kp0
zouC^-U8~TzA4HhN(f<5Plh32KW~qUn*>s_t)IDGa%BT~HxSPgds~2l7b+5@|DJfcQ
zJx2+66Nj_xk7P;sC3z_Y6_=R~;7Q5IgefPC3(&sDW&cgko&@N8g9CFPaDE}8x=5%$
z$Gh^IF7-YvDl4x+@xDAQ$@a7q&N#B<CUbk?nMUGj=M?fhX_dgI+~Gk(xhqOQc)?u?
z?Dy-w?6m-STfp_vV?aM4fl+zyfZ4AvpOtfkuwDCmksfNJMG=Yc0^F9oE8b5sX)jlB
zM`mOJgCDx;5nq(xz)u55Opn*>n|hq?YZX(eaV7`T=;fcz_`R}u2D88}eYqukWNVM-
z!IDYRk;@r`z+4RTirUOR$ZCggkJo2C>_X1VF+AMEFqYkG#;f(MRK6NjCcab8fO3P7
z(URU(M{xPL$ulHrJ-uR2wv}^F<Ct_gdx&kzy^AV)NM^g+Ne`9D-PN&}J<T|~<dNbU
z{1|1Qe|`)>Xl{J6%LwWLF4CIl<B}~d4KRVAuA=Xd-Ir%yK_=036yE=^ZT<Cf2F_<(
zI)Ojnj&Jk_GeN@1c9ofEN4h<rJtq^uwU3*^UMs^7YA|z(vK4_`EzqS&u(S$VxgY&}
zP`V7>s-212Z{icEe_VI+<Mg+htJLHFbWH=~_4ilT2+_?9H*7+sX#mJH#4M}+ukhQN
z;V;-1FE_sxMlLhWL_;&NVCG13AQL|GP%)I8%*%DXZUfL8+&eQBF}L&%IRqY79<Fy=
z{N}Tje)Rgyi*e`kyBlCW0)QY%xH-3MAVWPKdO15$s9}E&YY(uwn@ch3C3@O;?>>^G
z@VH*2F`sY|YMx&8M_s~96sTc`OnBBsyF}ozc{a#Aj^(8GsrE2O6I6ANWTk;Ul<J+V
z=m4fSK0F-77F;6r?oVktV3X2NzL3+7Mv1}!H3<E)xn)t3!W>S96HXef4a~pqI)Nww
zO;{Z+A`7ty)-RQ7aUfcL^R{^wXcWa4<Kc=}ho~91)p2HGf5@!oT-_#jW<Db;n3AGx
z^FMY*o6h5VDA346Xg*UJ4Q~3brt~JHvlZ>F8W^1!({p4u`QsSSGYRcUZVPbKLqH>-
zK5V*N{BT=j_oUJc_9%#X?=9=!WjlCZ<Yc};!RQY3pOC|7)<_g{<q6Y*xDvm8#fcB`
zR<wo#bGB0Cf8oH{N^#ZmOJYiHcmSM;#iQt?jq))F3<juZRldtc0fRx}{W1`ro1NS~
zfZKZ+s_}k9|M4cl$0)}Im`*?GrCY%7d4cLW9L?dO!1uCA%5hANgK(T`y3YIF{nKty
zjXy0-zK?1ti`8$f<2pheIUHEeOCbu+hOEM{<4ZI!phI?QA8q`e#MSe72BF+!?3o(j
zqzNET+vV_=xa&$c8hUUz7a0=eHuK_??|fL9imFzpZ7|ydhBa3CV{e4{eQw3nZdqzq
z^oB7RI(n8f0z6r_WJp=zMZ?ma4)_GSz01yaX@sgux=h8FYZJB(nn;(9DJCqkyM3dk
zdfT@i@j~spPKhP6HYIK@0Nma4D*^q4xLpiI=l`SYtD~a)*L{)h?(Pr}siC`5O6ibp
zrMr=mP64GAknWIh=tjC_2<aHQhx_8$XYb!V=iWbDtR=Hp>z(%-&-1D0Pd&!FM4gN^
zgf*`tSnih{MTVt-Y?>Ox1fT~<vN`Txo_ximSiX?dIPngqK-R<e-z|Mr!0~FrK)(y<
z(ZWdiE1HyysRS`U1V;)Q6|hD7CN|JdJ-m&3zq@IL_o1w$G}mvRRmeq8un_6$xQ1dZ
z9+!8z+UoAb>BpcpJpH>*D9*_@+NZf^RBc>8ybjODrE`X}ympUaNAbku>)j`xVX#SU
zD?n<?@4uW}Xe2m^q~HGj+x^Duy44CGM>g_Sv-=NV0hB%OQE8#Wrt$7YG407;zA&6^
zSGnenzE1hKQuI*#apVLs;c&@I=v1EbYQc&tcP8uW_@|nAJnruU<@&?R+xFXn<_hr)
zPSfdX*^Mn+BQxiR<>dz@f&q7)9+$&kyr}f+ZIg^#2br9gZzjK~QDjIp)Ix<s73fcI
z*cWPT#y;;3PfzQ-`aPehE#v99vLn$n>T3Dud?W_1;4wxmArB~hnMJ&g&M*BhZSdos
z#NN;?V88J!14@Z(&iAOem-lHHd6qbfM(3sNWxi6y_sD-R{$*pLRYcEM4|)mW#=Wo*
zC5lTB^ZlZt@yaxbQBX385}}VnorOq)*U<*^zq0@of-OT^HEw6#JnVEMx6c~Sq`Yv?
zKbIMT)0wd=g=p>hvhq_ibqH6M`HJ*O*Kcy50)?1A@Gk_X0u)QO6*qK;1B-e%u#rgl
z$kj3b&Rx>k`=F*jJnnP1+FxnKV8HfM#Kal$=vR|b=mck;w|PIopXthCUrE5Ev^+k8
zBXkFhL|d$h>+Z)q>B3)M0pz6*DAQ_Ld`YtvhQk7@{tT8xFO{4TuRD2NHwRE#ELJ-L
z3LeAA`Rxf8QXz53j5tG+$;tS9%6q0-q4G)(z@ME@DSDE|B+*m>el>N8JLysbqImE6
z16LYi)raeVLC*zj)vhO9hba&fanC}pXg#33N_UmB#M`kZRd1yPHS50jvdMj)II-c?
zi!b`y{~qSwlj@yAxpeYmu5E?6UuZ0}S<4ksl`o%7@&T@yXx;XQ9^1^*t+S*fS}+1U
zxPAK^e-=O0;ubS?_&8{?Y(T^HDifu)1o5z*ni2m#5itRv5N03Rg(T^xZWy)Vsl^P^
zIrNjG?7BeU?KoB9!m<67@vE*?@#p<tZndjS8z2ro>lq}n3->OQT36(3_by}MHZ*BC
z{Sn79sYwn-{#O>_Wyro!@{etY$$A8(nq`&uM}O>+3Qf{SKX36i82QgVo5U?*8_@pn
zS1CO3Ka|4QUWkBmivHEkz{oP^xR?MqVE0g`RKG4_xFczs_Z`4(=mR-ef1*g(OmOt6
zCgnd8#|6maMA5`i%%uUdRhk`JM3yu9vxa9YL?}-RzrWM{u0!`68)o_ViPxrZyi%pl
zQGB1?q1JjAQm5Bpt3-^gNDD~Kp479imKCe?R2s|g<b^LNX=heQ-1qpLnf*Mi25J*b
zubPs%juLyryqPCZQ)NQ?$pvH{eqxQmy?f5K2FE%F7Z2y5d^pv*m&TiCiKXoRd^(z{
zUZFT6zN`&-W@B7WDyV{&Bpgr-U-$eH*Hu&HQ#7ANDThuC?!)hXRi{b8Qb!jPW>zW%
zeV_9N%bAyX?xYpvi=$^Xeu-`ZqrCNJ`cT5fQ~(mD8Ol1c=eL|*jyB|NPI#+C9I6nB
zO%?vGw{K{=R9g&~ey5E?0+7EeMj}SD0VNe@{fyo{=47Hny{ObRJ9;A2@SFzqiFM=R
zOI}I*ke2P7v(okTg{&7hc)>>cRVHGcxC0cAH{(G@i(se&Py8+!x4CTiW{vN~o|ak)
zHKVO0@pDH68vt0*+_p%cmQegfdzsE@i0vz5GoEv(P5oU{=^+g0ljB9!<BU1PvqHiI
zf=mN0($v|V(WD%@BX%4ynJyAUG{{&lo|yd8Fhl&%e!x$dy7ywG*jWR)Q06Fj^VveT
zEF+*EB#njOJ=$GA{J@~6n5tp?a7}&F0LHqScSN^0UdE)RH0(NIl1rlnTU=GTE8y=Y
zX`t4+0<raU7~R<ktK0kcsMa-kzn83nZ6}UwSI6<_##c!$W$+li-*1k}kl;W)!89He
z{AoY>)i0ha(l+57ksgfcKGpy(Wad)__R#UA5uD%28Ppw|tFfVEZyKHT4(qB*G&j=e
zS{@`)>UWP$9*#^R>vsBnZxB<pDjw%Ev^(`(R-FDENTCk|E8lJ04FDZFjb@V2D+`x{
zVT?lO_;adqEdxZH-x$!hb?7vtoo8bri}jQLK@b001r-etBXA@Z+8e)YB^WDN;@!3y
z)ftxI+$tY6k1a1lw@YfA8p@1Xk|kP^;89GoHqTy*`-xplYd_R*5CHkYoAE|hFmBD#
z9rK2Fb7NNI`pkz*O;-xM$AEJ~^Qyb=Nu;1f1ua`i+kJ6)<3oU^5R4LqZvy8JuY8-v
zmk*MnIcTod><B6IH9he8G>rr3y=z*aZ~ZU6(ujlyE(#|AVJR_R#zIJib<fezKC;C_
z`oE?WRs!Xdus-LPp`|J7WL!?@AM<sopha}}%jp^v)q|44sd>PF;CtV~2NJ$-)~VtS
zz4RY#eGRCrQl=(x?(q~oLn94SY`!NYe|Gb;cnqElDvAD!Beoou1A?nN7}ZhT<KU}6
zdgC9S7cUcE98JBF5L>1bvYdN`{_`n8tt#QM2B6+XGt&Nib{k6qPHsO6;IKT)edK6(
z?3W7d8%kpj`1YwF*jVe4R?1?sNC~E<soBR>8k7h*x$1muXO;0YWU`fnL6#=u(vojU
zJlfV5_n#~wn+nxHnPgT^&c^?B7mxGnaE+L;@2}?_A6Ksik2|b00arB8bz7JtCP1_a
zDs{{Cl&EvOX@7Jv=wiF9au{T;R^zI@?`{Wr_ChXEea~-B&n=N}egJ!<L1k-bLm*+`
z9if8TpPk|lg{Cpj?e=vCX{tymT`+(I8;%@?pvT^E`T+0{LyFSHzPRf9$NqOTl;VL*
z^EBmv!zWfA>XQx4?Hq`F^rJ^jsX{@Mt3k}pDP^$StP0E>FMf&CU%s5QSSne^`>xqg
zUHq=LB#<XIR=hL!N2kB6PID>{ArDRZUwo+|HX<+xVk^PFkwG<iBQR}?V;InKMHwy$
zdrAF&&3~=XCM;|yHgrnQz8H3XvZZA)k&xGq{MY_x<L|Ujp*>U9&dZjCn<t1b^3sWz
zKt#zMl_}+o#7rqofRR7>cQ`Yd&Y0N-SsYCnDI~;j!Rdw`lO<uA;#1+p?`2ziPTDk|
z9|IT1Hbn})=3!EE=vb|FeK;+aco?i(((z(V|1XBP;|B_Zw@#&wzSj3|inm9}0qmep
znHwur{loKY78{|&^&&>R-am6H44W!nz5bSowjN5PFo-Oe9YblTsa5)>B1@-KtG8c;
zk@yUV-15}v3zfZBw&w*#tY(;3R^Xh0QD`=Y$u&M-zf1)t$^V%Ha;Psz*?(KV2V_?P
z_CI8269K{Uv7a5aQu08vMA2v>os8&kHn)J&B4btirNK1R5(+_W$V)VK82c<F^}1%C
ziODu~y0Zu|F8LOCpEU{!rWCz*U(`9e@*Q3S2Zdb8cad7FUt|o+kKdu1=njAiT7Rn8
z_Uf%+gfv~JC$Kn!5Dh$co;TfZ#3}YWg)J)V$(14sEUTnIl2CW^^L&uesOuzr6<}9)
zkL&|{%0YK^DMc=2P+o!=O+^N$FAUyBWVw(U%f8~%5o6u{I5WhA=4whE{mJdagY4rJ
zaT^o&U)<{+#PZOe+c1HS^wdQUh9fy9H9&O*|K;`nFg8C+kQB+-4IOV~QU7ckMQQyv
z*M=75QP_(O%%FGx8LNKQOSFl{x&WUO%$)o)0b@dqeZ~&W3(hoL*t0%b?ca__#})`L
zq7p;pE)q@LP5bj2_~P{_bxc#{h&&7W;P79wz29>{53TQ=L|%(Oni(BN!WF>b%Jk<|
zaz9hrR=_+*-FP1Ly|Cg;$;BhyHq)Jza^iGCfd(zeT}hXaP>b;4IOs7281pufD@Sx7
zqO<pR;e@Sfj=hiY5dvh6S_|Gbe4xoeqnCmA*I_t0lJ5Y;{p#4m=6&F{%7ej0K={ak
zYUa(T>dn<=vDK<nM^c&=r`(vBUAQ}rM}{x|zS5tVG$w)Z2R4@=w(q)hJ_%6()Pt!h
z@m;s-MQLg2WQit2?jB|>Vi(H853l3&I0}KGdb_FhWEMbKvn|lR0MV6*fBx2D8)zj{
zLP7W0e0J-(Dt-FE$hc}e?3&F~ZS@l}TtOV1x7hw0IfXCHk>F9!tSzXdQ8)HK&z0*l
z+y@jHr*{bj0|}_-GKper6K8jVdto=n%Prc!y*IFO4V-@lKZ6|$`4YvacmrKwI0M5v
zTF~7q;a$c@k-=l({SRhgxs?GUpSZ=-R$Th2So*2`%8xgK5Hb~@y$ODolPX*{e#>RL
zT)Pv6k-1axR3iRwO3u&k8QEt80MJ7v3>l05idsv<z#!|DN<hii)*nEguh^UA$XDVw
zJzp4FX!p_)(NGt`n$PdKwjO6PZE{NOZ%&7~tWMT@fqqgD_NTmV3_mC6h-N&({J#om
z73E*7r9rr+!;t(}sxYl5Q!|4E2*}wm3!imb96V`%bs6@jl>w!%6J`zz!QoW{T2^*Y
zr~-^){Di(qT<FCl7>_R6nmkIdMOjH;Ijm++z-etGe`7v&6%&gl1ixOXh?>#c|8@_U
zgRYIkvd-Kj#w(Ro@4W9ccz<$A^X}C^f-2TRCK=w68FGRpi}zDUl{80@gv6|uDeB|e
zRV7D&slUGMbRBo4xVn~cyLM@5HMHt1%#xqq(z(1Hy4kEfmWXkDucC7`<ioo;Xv(C{
z4AlFC{<HtghI$eS7Xi@#T{f*M887(xuz3*y5iP?ExU<iA;b^j);^3^#<?He@-3ueQ
z#hnGzg7Ac&TlkmLGYR-1hLV>kjx#tSvJYznh#M0M+0C&E02nFn&AT5Q54sfoRz_?}
zTJq$t-80W51X{M_Ry||j7)I0hK9wvO{1w(?Dey6{VZ!C}bMr{C4%7{MNB@#Ch)OoM
zP6CZ*Pswg2acSW)9_N#2WYAZ3nBbM@n!w{cM;o*Y!}z<cv8%W7{%l2x1W-DPS0q<n
zXs~%aB?us`YJ%F1e;kW<>JLY37AA-rm?xvGUIszIC}JsRosSPy65sM5WQ4y7RG_4@
z7N0)P_<0-kBRP-^(3m9Tt~Yr_{&t<(6;J?WZ6Lo_254i3?r+DX#W{NdXN7(RhLmDm
zc#O{zr2$zPoueWNqvK`gBkazl4vorZGwF0H7dHJP`800y6dZ<f7k4=C?dblvLGu`;
zCEzeS@-7@@5LXkqGAT2Yt*R5|YZ)fm*x7t29&qc;g9#f!2u19&Cbmq?lT2LezSL_u
z>7#h48^YKE7>r(<K)5My`~<<S<I900hOfY|gA=0ycS-{IU7P}4VCnTd8)Ij@Q`Bfi
zS!#NBRTP&;U8{N^v>tGGNrcLPgoc;`w36D5jtg3eN|U=6Iou~Kx&D(W*6w(?#;sn*
zN7;#1=a~?g#Mub1Ne~!`#Jd0mD`$6}!KAr3k@OtXYe0J0FO60zY&A1M0yZzPq`ay5
zBXlVF76pym)c@qe_?zxyy#g(u&H*D#Vp@=VkS$UumlO^YL}>ZWsvo1gm&>Zo)G+lo
zEHJ@RA6L|Vu9T%1OH71kUZkoM0vg0=>x^=zlqZ6X(>~=1PO<Bhum*j0hse}Q7>ntE
zs`Qh9nr8Iy(M6xjAi^c^kjrcM{K{l(+hy^CfZ`re=lLx_$`1b*=cP%EnDENvrm5}D
z9jK6oX|9VY-6rh<ohfuV(-Z&z_DB>%n_3wW2lc#-nKd4ITf~QY4X#m9X5Hg{-~2hK
zpF!MGNl_LoG~+O76nJ(Ysl~QchvVpd>5WW)2N+R8lNbJPnG2=*e@8i(?By2IsFO19
z8w>-<8fo3hqr|^yyU{en;qM+2#aSollz*XK?xMIu;UOp`af^lB)FO_!?u&gnpa9{<
zGWnalxf@OUHpudPJ&XFUS$=|y^#Ezy96<y(I)mr5JyuEJUS1d3keLtFsK#(E?wa%8
z<8JMP+I$MNO%KR&8y5=-sYP@haYrHCHyQ(wXW)$)UDFE;i|+HrCFc~2c%i<g3zTdL
z|CAiH4w+qqpud3O@7-&rZ2LLWp!UbS>LNy*ct=t(-}>?SpM8IT%YgkOF=vd#`_M&N
zaF~VW*Z~nNT_jUrt0I%dLwiqo$=Zowf;!-CoM%5<KM9m3{^fQ6OXgkd&gtc!!9C70
ztCiV?7byw>Lf>~awOGE!9|MRUB6BWhC?@H1$DtIqP^<tR^Tdg#xt`)vvK-)z+Tj;x
z`V7!N?=ZzFK#qyP<#gBF3RYXi;A%@emV28BkxECc_1cMCaBQpP2K<#2uSDtO6CvPb
zAi4KDDp|Ns(AlU!*D5?JNxlZ_WIr+#DA_AiIMPn{%Bs>s2S{rWszXL14{Hi6pN-b%
zd)M5Ih{5><t|odT;wT14xndQ;*UCZ0HHDT@dU>hAVJ>7TT1q@*WMtCJ#PLyR$%ONY
zIMx~=^UuICA6zmx4YxKDiDc)bx6o=MDbeAX@2TX>kJ-jwTp_gWT3CGGpsZ<xxb_n1
z%0jnHLdudc2cY>2&DVFg2X*7OtDH0g52*NjItqhgo7CX)$J_m?LjO<zQ^mpO<9pGW
zeII{v@DkJ+iKO(@+~QCn9FXSPK!G0m?CWK3I<SuRP9jq;4x3idgEqX6Fp`1xfv~Cm
z0*F7+@NL&MAWcI87rrfImNrVe`c2L*<ag}~USq$<k$-A$$-HgvMN5IWsAK=XNF|p9
z(JGX7(QNkPe@D~gdL%+Ma?q${Ui^5CY(g1`)#=dh<@z|TKGKvfO9;M5mqSj79)FM%
zzdk02R{h!oiX;aYuHjW|3RsB8KwP)URxFQmR+2n5*>S%f-LBebHEC)eXVnChab&`x
z&sC|`mfAo*2N7$vZ{^mGHG${j84wM06O`B7Mzm{x3f~lfq0l_93SszI`#&W+W~h{t
zf6F{vluHAzL)VO>EB5W+_8{+PLjrk~M$a7Wy7lvPawLCc2w$Q;EEY2b7|iL-yC7{B
zI60CfB*V;>_!)$6aqnXo_w*QxkK;VLz*-x5+5zvi64^pAa27RfLV}AVch_HUx6;E+
zMnWRZ2&&X`_Xbf5XbW^{Zt1-1RRLKJ;7Mi7t!8vX#$ocV#s__xXQWRPrSOn?i?Dgp
zPeTH_!9n_~h9?XZSm>F<vN{TK;qj?n5VfCX-36RrtGIV7$AR0OVB_l@lqOVs(P`Jg
z2&U7WW(k#9YE)nPPQeZ0-I<Pzs#k`huX8@XH|uVDb763L<?u+C#{NObh@xYOw$Q{!
z($ng!NrK~HIf-lTYlMJJ959?P+)*Nt(cLW<YaPgr5_gUu0eu<l`5D0nA01n$g9}m=
zk`A{p(uoRy-E1qMqEarL0A-Lv6BqCf4rz%qnch?w{CTZRAI;>ADA2X=`Onh5umGhH
zPxt|G)`acWNTzgO$Ye5rA#l}#dK2a?2hA#_-kLzj*>CSi0nZDXVHTDTV>$#}^PDoj
zfal$B(ax3%C*|IFUm#+Q3O*_PktV~$x0K~Wc*A<Mu;prS?9XGd?B4)$@H;qkzDxcG
zb-v7O0^~7ue*3xSk+6Ji8m_bVw<KEX`Y=n1YRC=|Q&^Aq58T8tv8!d&n}}74+7Zdv
zwRv_#npy@ae{^Lj#JD^xOPm~!oM=6OcMuJgKOl9qeJ^hVJq}q+otB^m%UC$H*iA9N
zsz!r}{p#>W#5ycQ^it%Z@rO!jP?Z9NB7R9M``^rtn-D;op)hCRr?eE`e+|B-xg2w_
zIK}b*WcQ?&s=*UoU%g(+^;i%v_`tp19P?vb81#gn&cRN6?1GGF-f;&F$&#J@RJ$n;
zx*8(U`Vq)b#WjwYpfN|V8KKZGzx8;qGB;q}q;IOj253M$f_ya>J&zBIPiMmCHq#^?
z8%w61r;Cwwrg9jj!rjv_$h@|DeBc7+j?F@$;Y?Z+Nmfb!3LuTFBH8X|y1G|uy`b4M
z7FyqVrHchPc;3iUF{ONI7*6>B_^&A4qn;EJiK2;4*|WpPTxgx0LM7Ut)dx1)9#PS&
zD13~q|B<})-}(s2|B}LmQCT~z^hOXGAza=qcuxcb)jH~2cO1qj(g+{QB)t0-FYHpC
zA1)%P+AtA#i|b?&mbX31=@QmaxqDYV$IWjIeBeP2AD&2I`W4A{f=M|IS2|S=J%wU3
z?Kvb11<JSzUq?dH(Z&#L!1mprR{@%?0XVO+TVY{9N+G<VvCTfAfKA44(={85AV~cj
z5vK{n<tV@6I?Op)Wh(JnP8k#gyPfM~4%qYmn^dwnOI^=nIe_~%3>_agJ5i3#N0QhV
z5`t2sU91kvUQZNJ-luHT3+>;c%Kp>19msjNjN<v+659M2isJ8fa@%_fDJyJHaTa}$
z7b>;5mN}}}HT;t?zPtxB+sQ4c*7srbc6XREzI7L6tBsds&-QXo66o}2&^&`b6vRfV
z#qNqsNW@)tis{36KgvIMjY)Gp`b9+dn`&nAcq|7#E{cv|1f#c+0tYw@=j+0bb=Vs@
zCG>OPy`rMv;^>(89X7Di%(xjL_gm9Cw)PGe<To<0EtE?DN)3NhaUB){NqT%0DCC$2
z5hpDamD)17`KWcD9x;TLO*VjyNM!E>mD)`C5*3vP0N5MHr$agQPU!e8iQ*sDyX|ul
zf*$rw-ThQASd-^RsDJ-_*jf3}bbhcfC9pwtT;M6kNrzeuD;NSa<D!{gO9G_;N_7b%
zNG1u=8n&QHt0@hVruE)2dIZ{8g$HM;v5=y|TFXKFS6dUY?3C@bz}T0=_|!xh+7QV~
zx{p}d61euMjs)E9MDa*54-^0hW|1H^JD$Q#M&nB0@uew#oEK5*1Lg}Uwo-{s0?~w6
zjESfku%M@ZZ8Wq(Kj$P*@;0Vd{aEvt^Udj>oFe>bO#nFqE`Ixchr-F<5caYgSfa~E
zX((#LA!Eb-Y)aZN$#7-t7nkCNw$~K9Gux?(t9=&^mbdi@^oQLI_&_k+Byc`HqD3+B
zJt@?DwkjQ#SIZHq{7%Y^8nTRTe{>Fq+ND_b-BrmW-7P$dF}6p*M)qB=wg$v(s8~{;
z12biH##+~?os@l;y<ZNR$tSP;@EkI&qB#;+V!%Mv(eujX<?;lrsp>Dj1}ae6G5uYU
z%?`ZwvlSMqM~xRMVG!>7KEI-4DD)0K=fdU~*20l_qGUAx?;KS7KTkQRKOHP$!}{BQ
z1>Oy!SPfB(Yo@ivWe>jV_oozUL|P46CTHIE4HJ}Dr1DF?zfa=sh@8X1F_|{?{6^)?
zh2PIq3D}@r-g5sV6zyNdDkI(#cP+Je17!YEK>W?Rs3pG6-atr_BV3Wq<}3<<z>>zN
zz>;h%<n^7%eq1E?sb}qtVW4aCaXBh*{0Z)!aEER)%{;yg(vq3s#fNlZv-}8dWDaGP
zZx|W`NXizx+(_~&x6Snlt$i{I->`_I+ZXdAqw;+hW91~I2fzNbqwu>Ah1nO^kQRUc
z5w8lkyo>6;_!;qY0JVe6TrV<<EY8DE(~F`Yo2lq`cKyzwC2Brg9%?|u%Dp-krQk0K
z9AGC`C&_Qck9`rjiyQrM++jEm$Igj{FA5}IY|h#^C!Lq?#DM|%)9}XJ{R&Qt|AuSJ
z<JUX-?#Jz|?IL9>SZb8$`Jzm%tuZWa;#3jZ?E8zKCvO-|;<(_RwrsHBm60}KMsuSS
z{Lh1hI;+KcJC{TAG*OX60$iA<B(<50Kn`@2f53Vm!!&R=;l0KW;0yuy4=Vsy16;*!
zV?zCR1djl;a3ZY8mZ*&jdYS8YBxIU!#n3z$Z>;a=osJB)00!NPV<^AJ!i5{y3vL7M
z(WF>Zsde;B#%oET->Xo_L!1H>6`yNNa03$<1{_E8ks(9;c89@~MO$g3FRl=kI%@_Y
ztg7R7D0grt4nab27G)B(YjqnKcjwP5VD@Iy{w*0HIy&jy0g<KGOR?)<f`W0XAPO)-
zGTardH=$<YPlOiMOE1f7=-Onh)v&)zrAhmuP2Rwq3oa6wr?*M7?(>oDuNfw%*JYJ?
zwT!TPF^G^D?0VxXP!=T7(SAG;p*UWMB?S<T2fCKe-s{n@N9~u{W33+hmdnj9;QaMj
zArlmOj;oB85=n!wwl|*uq_O(CcXw`1-s<G4Qt<4i#{YZk=*jN~^%Qg>oaH%WtZdnD
zDeWS|c+1$9w?m4nY;`uOapyEE6_M^c%cD)ao=5NdAB&W%<y(g#b^TVL@0@DcO{u~j
z`w9o8{-Q3Q50+~<mAwQ(Pj@n)Z$j_51CQY9Ssg9Ea1$rvDkC0o*B2lkO8SzJVsZO<
zE5UV54>ukKO}P$4wW<56lR{G}=F@54S#7`A)R$MHa`W}r%04XKe0CnG_qUm7NuAxe
za9gi38OuogIhR215&?ma$}qNZQTIuJ*vW@SPJqC{H7ZC2YV8d)mf@MsAP^6>KHZxi
ze)VTRMr$P!5-4TGDesC7=1-k^wlvC9k%2bqHMO_HH=S2PP=mJ@UHY{^mWb8HcdIV$
z7o3Y;NtgXs(Uv)6ZjW(1L3>50MOax4sUkUdw8?0U)FBM%aHqd{gOU?W(Nj_xLg0zw
zRmjDhJsG#euW~GA?yK(IGOhwH;~s5}MIq%Jl~+NMS;4`|b#Nv%W(VzZ<$N_)zkew4
zHGU$sIMvpMUCP!ldSaO?M5CtLI!2MpF@kV$HQk+pg9$Gl18o9KtdD=Mw0j+|{OQU$
zyBQf9$#feV5j|cRw%Ts7q1A7{+jC3Oz4%5T`rTrkT;V6)dqm1-|M{WUipvVPA<0Cq
zaRgB`-owl8<&C#UNE-&hrRE4{7YA3JPxlncL#yr+0<DZLHkx}<ave{;a$<+A*MQu=
zM|Xdk{mgep!bu;L(3@_6NX{u=$njY%zt}dXxgu^5#I15@%W!Qc8sASEfCl1B!5=3`
z;LVVA>a!cpP)YE^YV%&+Agc4pNvkc#7oDeC_1+#LK8lFLc}cpF-JN0VnHd0pLGBsP
zC+H#wMA7{mfteE@A5JqmF{?hb^7y<DR@#csz29V6(;#r|?};YB$galT{{)e;#^!Oi
z#F(pIesBMWW}7+aHpKXP93yOSv8>X9gvwvj){SYHZ7%ooA^GheMd<vy?1%ElZwRiH
zO(3h@kr>Oz&WcgSs+v#tl3TGxy3aS!rvcHa2=wRByEtpp*GWz^HfWiAcFA!-xy>Jq
zMr|bLGH2}d)uMs__VUny!b!(x2NZC7E~!;ED9$VtOD(qe8wjq-Qr)lM0t#-hgyj!^
zZNU_|Jcqd<nW2My&2_aNzi~MztshDzYvLye-o6`9vd*bFMPA)DMPSl3ExW3;)%!Nx
zc&WNzpuzjF>dMDToA;2}(Q-jYumvG-4D878LeW0Bv!p3`sR7KGu<I~Yd?TSQzjDO#
zpPe8htL(D)L5bKDaF{CfjdBg_WJYO3$h-$NtU9@ha06#>>yt%%&7X{I29wheKB4>J
z4}NM6AyMBjspb=r80Th`vHqB9FW>H5ER>WIh%PodynDDRxo>(-L%BVc=~Sq&oSW5$
zn4qJtveCf(SYxNnGGjjB9>J5TI$us8Y99#fMEBaMs@DG2%LC)Oi3ElgRBFs#eJ)p|
z%-hbnin&)HESW#*P_v|^<A9<=k3FPaDZffFr8zwyYgfc=pd0PPEI`e;g85EaEtB;-
z8Q0esEnLxg-*It@rRboR3iQE3kfM1g3S!dg@3S@!y;7C;nYoGQu=fy9<9k?}JzaeJ
zwfx;6=m(-$Raz&^G3aim{bc5#X#U~a&2d}C^^pcYdoIU+{^(}7g=bv<ER&A8)x2jr
z#xeArx{a<de0(qi1Xb`KKQxe0Q^~+#VReH_bjEBwsAE@nd(E7caFGw|h(BNDnUOTr
z5a~)<VqB!;OH$KChx=EDnC9tS+cZsRT4wWwvC1#^$9sQ_MkB8%?5hzErNM}P*kxGM
z6Qy24Bklc3Ejd%%ColfJ=9w(o$%&k$j3FqN9h*P&%T|gy_ad`}7_tcuEILO2qxQm!
zVMH7_sq6H^1)E$a%JRN08ft)<BgW<xtTr}~jU-ivvff7@0#{eaCYa7bjS&Z%I_U9k
zbqSfm5f}RhAgok0wFg$O<u_UxxyE{@m04%u!(%D2o2fxIa0n);WtXIRfQGEZP!a@>
zvuYxnGQw4ZtL1WW=zw)^?#-F+yj}5^P$lEYJpa1DkDQIDjg;q!nv(bgfbh3E(b$uU
z23|(P4}kpZ)FwMn(OK<VhZy-QZ3=;_`2OZhhB)58IDyx8+C2@W#-733k|yvY1kbcr
z8gq?gA7iut?+)M1F3oWgIh@XTAz95ldXTo^Fc;IX^$6JrolbZr+`@lFJz$+^14rIy
zRrU>55eQzZ@`7YPHy}Wpu7`iNn~0}%_6W$PY@(Bi*fg+?Yc+pxz%+sci&OA>g!Z30
zEiD9#;S5oT@py~!pMs}z9BCY`7Jjz6oAS3{qdNOIuNkDDyo+sUaU-@t7<5{2ug_Fv
zcAQ0{_K-NE!Vv`{Q+d5{{VIDgNBa2Y)x&Dg;qT!fCn6fPO5*a1Q>VjohogqqJo|6R
z7!bfJ46sZ}+fJfdgO{JUz!sXx`CD^g0wwrdt7BZS%ixP0ZKa<9>tme@J0h^GDhK<y
z8n>oI?l4TI7DUyK&nHRkuxSR+-ep<YrvtCP$Av~eN^*BgqvnUJw1utjq)cCib4o^}
z70VAeXcrpyQiuv3Cv4_pR^F&7e!iWO{<D)<hpomV-u8UTs#g4Xc4F4*VrX?g2zHWj
zdE2R=v$Re8a5cPtG23?I>UoW%yAUCQS)Np{BA@*j=d&okd~UO0Pigtc%hl+FtZUNL
zRrdo{-klsP(Bfhf|M7^z0=E=*{Q(wSA^qbU4F#;e^U&qA3L(U@CB5Q%xzk9L-$97O
zi}Jyl=Al#$=NjmTDAR{M-I}4L|Fx+gsHyz$R)7Ms$~FW+Qc89)7=gAR+#4m4<Ck`S
zfs1`#4;AgVzUn#A9dinEo^M&8Aj)5&d+fygq3!-U7KYUjBhv}`dN_OH0p$8pZ}5q!
z*k<?gDZxajxRWF;b~si@HlK|D=u>vpXqgWm3cv8eWX7vy=iD7X>^95gk{Eqon6+96
zB<Vn8Sw9FEkT$;-NkhH3!{|=|i(#*PS?Qoh{&z=}tWK>b!T<`Y)`&3y(0!q}q7>0$
z1NQ6e)%nfAlzjnBDwAGZ(aTlOpe2F9@Q}9q-=ud-ov?JbMyYJ9+HNmV`q+p489dN=
zOTN&dIsE{G1x<~gDQWstrOuG`uAvSLF9`Kr#?8{4B^k|hN?dR`;tlGY5D?;(I+({S
z5hc4{WqMQ5k0WvaiyB}trt3Qp6O6r9_>KA{^RBa!_}))L+AFn-ha`@qE!TgxOi#`u
zCZxO)UvMYt4Y*pd$pvPRjBzF%N~0Vu05^WO_ve*7+)xGo3sgjc0jvhJ$PpqUqJ~UD
zK4i2!jkld&6AuoU3{*f#W&m1?qx<b8i@w&tP!i|;T`!S*O#aEqM*k83be1`T`4^l9
zT*H8f0al}Y9mPcR2Vn5jyL~_h{waAI+ov9vZI&h!?J<y!-<VEk%J({8?MlAu?dV7*
z?@UGAi`mXGv{SxcHih5Rp#ub!Z6k}!y0wy|bTXKcKUM>pSO@Jv!(0BmOC2hK<3``<
z69+9ry8x4eOKC+@SIn0bhu4iYV`UXdmesazr&!dhqu<5Wyh|D3@j$bT@@P7sh*Biz
zk5f{lQ|W))uwNoUIX4=d^xLSYc%9wYN5q$_{>9d4P-+V~!wTV<B17AeNcl3S50^1+
z4tLe6{$QG(N(vDFJfJSP7XDmiE5-=*;Kl!A^>T2crm}&<WY}TX#E&7ljp&r{4ZU1Z
zH|W;|kwMij*`O$d812{;B`1aHCIO>hQaP9s@3jWQJlN-#pc~%Dj%M7QYIem>_2T$<
zy&ntICd|!UBS6D?2Q*hFL?!e6uoyPlC@F9xxtMe=2<CsN<Whpyx2Xb*X&z?z+W%8!
zmHaigTi|4Q>NjcVuRtLqrfd|H>xI+njfBW|m9EvMV+AspEZ)<PY$sT!PwmgZ&d+Dp
zX}Tt4Q4%D^?wZ%uLXn0fM(?Gh>@G2k)wmH8+M9m&V=q_oAl+S;ZX9yqz*-j1zlgXu
z4kxs9888ODR2cdIaT$5hzvxVH{Kl!>gUxz0+p2iwy}hxuM|^+ej1ec8h~Nfw6J8M+
zbs?y-6Lw3!vhE0n+i&8%_F=lQ1CTk@c`m#3<8BXgb=C*L&T{QW4&+r`Nrl}@_<Q{1
zEC@u`#l&uR3PU!N+uscsS{fv!CPVPXA9Ek?`+xhNsi{ptJXDpRTk@EAeWiWTx1h}@
zF_$fNId8ESk;NX{JcN1!Jp3UTg~SF)vj2{=#$60UUtD(aZQHLhHE5D!`^9YIpLD|r
zXJcK~L6*vo7rL_~=Cd9jv_z{Gdv9}HhuGMB8h&kKlVNIo+5|KKKa$?En6k}EtorQL
zPT4R3-{ptY1OY;L;8qCt3{{Y!J8+?TrjYj;mv+PIULdV$HE?Obe|P==Djfcn5kDux
zB{T6vZ<*%RRE9nmy>A2*E&P;lWA|&QC5)pw5bA*(P~h}BMhWMO5OWQSk0x#!Zq<sS
z*GSg|-Pm9hO~=KWAW8y?3Zx%xNd32{DRyo>Q6voTNWxB0AbN?#g}_BYkIU9|@tWta
zEJ?=MzevuA?&v2lI`^M(|9-({2_;=2<n>0SweJvW+s^TecvqghC`zpL!_BE%8^CtU
zhQC+H#6}3#bC$({pyrB7K}Rjy-xw1B8h`OywljGpM)0y!dU!;tIV!bbWy9AmHQSkQ
zeM|$+(jM-QCCX}b6sdHnyc#Oz1ABE$Um)UCrmw|hN`GK?Y##ciCj<#bDEGTMqEpSE
z3-8&)tyEn5p6a-UOTh!j-$f@Fxt4s3FM8cBasF$FaM{wj8F)XAzuWe%-c>a&+9QJz
z2gmgC`BMv=9YYm9r)@sb!|m7WAD=eA@V*`}C%bINxpkL$qdEu*0jCdIeqYm6M8;X*
z0R#N0xn-3tM=5ao0=k9Ey|R`mZ<5-}PxrwzWIVLbdalZBI*$eoWG=JxE*ETQ3<EA^
z!eTI4n^rutpQz%-E_8N8s*|U{{=^4{%i-eD;|Q2j`8W}0?`{7v)(4Y`E?CUTG2lpR
zexoR8W%DMf<6Bk`MvFC4>EF-H)mKVi_`-m4j8LeFiwOQ+Buo%os1xN&33OpVmM;Jc
zRwk}R^PdJ`*RIT@zODLvwdUaVe>G`9%NFu2?rGsP7krTTp3+C)inJW^Nig5D0NdAo
zfCT!Cv>Wg@e-JMa=WnDmDvir^r*~ugJ}nVX(qvV5ffxDdoax+R@TlKHXvwAeGcrGG
zg9aWmV)zJ=Zg7n^I{nLPyy~~KHz8`}#T9V`Wma0aPJGP1W-3yAB%by9#pVPG0gF4p
z>d~|K*}Fmg=non%)ygtJ#yQN+>V}()N*<*%?2D&z&^7i*8~w!1;Ab0s<q@@pn6OUM
z>$&7)^JHAO9kl@lG^aD&nblrrepg_PfJ_ARzD_=%%qW%`gh(;tShg^>pWWO&F3uVT
zw7Y+nZ^0Gy_1Lv)Z>`j{a=LNf^1q8~2kkcat2#8Skt0>gBqqvx1L(*aPRitrPXAod
zdo>J{X-d!j+9l^7`nlo(q0i@DzqnxX#)Xy?#@Av~35U-E5y}|**k5RZBuzGc8i5w(
zx_cUX8!56Ocqs_9PRsshqio>da=h|LhYa%&j+cPZ9MEOAdLG)NP`PW6!~SHLV(Y#w
ziYb`@A|(qq>EYz8eFW%oW)uL(*c{NYz78-dyO#V-D-&M%dQi?R3hiY}l=i{fyTc|N
z7a_#vyQ3cnnb^rpRStLs19=|6f9qxj0jwkDI5%@@4{}?=yT%3U(&r*!ay2kf6>LX0
zv`>jG;d7C)RU#J@lQj3uzUp)MRyJH~_E>>j%{;(>*ub8;sk+fIajo(o<`lF!xg@se
z?fCNgN#(w_rxC!uGkE*khCkbA=C0l==Ei-7B<(BxsN}|lT*Lzlko(uA5C8<2DbY(J
zH~Jm5uo%N<9>SoJ5=8;;{@L&RtNW?0r`51V#OVOvzSX+6uENgg!GI)`?M%-#Un34`
z;8t3K<?o+)qZ3{uzei>zSIK%COKwbwArdY_()&>er@n#sU^@Qd+Fsld9UqBfU{}FH
zt^9+w%;#V0Mg2zEvw|Ij=iIh2(dy!k*b?j9@h>Cd$;G-z1K~A%XXb5RIG!l{qB9!n
z==-VC&!St;fr=i;XB8SL30W$}6%!1P{?M)8!8G3V_U4U4OB6QuXJA+$OYJCVDSY2R
zkgyd{p~Quq^(6v_R{8PwjjF~MRITA(1Fs&uE`NPOiSvaL`{~_*4#Qw)`neRk4YW6a
zl>QdVNpW_h0*<7?AmOYh`Rzk@$b}JC(YQFoijOTZSPQO;tOommvm7ez{CRWs1({Vp
z;i?LSi=XrmXiMWB<5}K?D1zyDP)5c4E_LSZ7eq*b6`!QW1ED<!WYrib1-QDNzkRm(
zDBzFv+k$HPA`<(Uy?Zj1=5fXy0b^L>(DO1x)d`ni)i)`CuWl0Sc*SywRSp?vJ?SQT
zo5knx;VRC3y3JXzerv%^5exjS(!rh3!mjjc(V^<R0c*@CrF%a|0xDDa>w@)QyJ60x
zF2ls&p1P}3Ka$c_QJEfMz@bQ={JY(c;;LEYr^j@O=HBAlNGjsMPFk8^fpJ25MPNbc
zp7wOQwI#^l;$;TY_THu#hk2(ktMSYtpQNmE)%I0;;eG$)u-+5h=kczFaHDF~hbO1n
z>!nR#6xB%Q%~rN<%huNFBh2eIu$P>++PUqdFJuQ{%WS{~WnEf!Lzy8-g!UH@P{7gV
z*#I-Yzq{DC0mel-qxK#Hva{a?e;#mD=>|`ziEPBy#hd`l(m?umgE)l{nBm>Pa&-?X
zG$$Nb#63TK1)ok)6~uLi%j-w&`-L(Nl7o+~|MT>Gra+4VZ>0={XrR1+qkM*4%=Y>2
z<McBwN#O%qhIO2QNCQgvo1VZ)O`Cq3qvn_D{*G+1V3}V=F(1C%CtKq#l;!HiX$OSM
z+2wP{y4X6nOxKurlG|waOvuw3nBn+Es8YoiT2Q{lr0Nnp8<xoV2$H7SZvfr=()Bk_
zL1>?Qs?hYg-A-2a)S-$RO5q4n*rb9zlW!INL@fZ&kPU!S`v}e_IV;Ty_q-47PxeRe
z9XI9x7mnKp95kUA>upF6?3|cOj|lqwWCJsshAMbUDl#aTy6H6fmLA&#?RS2|61zL7
z3sKBF;h)uGpp<9d2tHrnkF05_^nD%y0aF5JqLbkCXb#uW8Om&NZlEwFYX7a))6oZB
z988sLj5q4ap~IFO!9znxer6e_g(vF0WvLQ{bM97wv{EY%rNuy$jaL66{MB%l=5!;7
zw`CFiG!k(nUhGoO0XS7PW>Q}xyG{bq2rdD?^=KC92o%B*jF4|Hp1Gl-FJy|xTqIPz
zDs{qY;mI~Ljg_eKGP&|V8#h1SnV1nTK0jQV11;JXwk|sXs@=8hW?q}Ia&yDjM<5g{
zQQRvO*d|{>!oNBA{z_U18_^dztS4BZy=XB<Q3wFJhAJ?{!|G1(1}u5rY5{n#_@#K^
z{{U?M0pDCm<&B_HBP5HdDxpVSuM_{I2<8fYKl>^hULsg{i#%~f>K?f-n(nuhW0@*#
ztV%V5ctbPF33;{5xM0nvNDxSwPs(Va#|WucE#i#i^B(*4_iDX{=wh{rS$Q%W-YEC$
z33*b5dBOtHn6go=;a4t*CFX;<<Pb{*)%A(uo<RSvEjF%l!?v9TOi~5NYr(tAZsTe=
zs8)SV9)Jw41B#_HInJf@7wb&l)MgDl`iR{zFl*xR1$U>2j%9KA#;EM|>&1FUi^uDQ
zQK=hP|M3D%#6x}z*xJF_x^sR5)1l_`ZwYo@XHn?n1898)uhBU7@St9*7$tWRDA=m%
z8Xo+iT;F@%*i3kRJowZ}U2(+o`dv*g@l3}vns<r&*z)m|`kZALt2wKHRka+D%-8tX
zo?(HasQjlSGI}yuo8uusV?1W{rYs&|a>Z*^tUa9$w3U6@!_hlK>9Ty_>2Ea@h9hoE
z6Ljy=s5B;i<tBqV;1Q>w9;t{n2#0UtFFEqkFAy+ZNi(Man_PI58BD~)ARnuUM%|Wx
zL&n1xSK=TixZDBg>9ijJJv9&rW4X_5c8GwC`Lb5Mrw3VR17R@4p@Wu^L!~QluqQt#
ztoBNtm@-F(qfMP|g?1>W*KL4Ffzm}*nHIbSRIKP^9s@^Gu}0qP>S{fBeIG9*fSzJ%
zL@<l1l1}%%)i_}3A}^cr<c78N2m9P}F7QDUuuR<A#s<fdI1@z%w9*3tG#wQ{slX-`
zu<VS!sQ-AfE#?X&;9=2S{E+QS{50~6F7gIY+FhUDB)5lgP=AOq$@q7r_tYH#E3_2%
zLh%35Gydz6<>UB!FjZ@aK7)fC<cc@QE{OJE;L-$Vg^Kw<>+2=S<8oD)#N2S9G4wC}
z6A1+TFdmhuar)h8sr2}1NK_es&v@cv>-A@1x<lQQ$+@(ApOt7_;~~7oOyV?4ER}H~
z?FKd4yf~O|gX>Z_EXqW0?CBz&SdLi`p14ru@s7R3q<)r59#n5TLGN#IJ;0ElT4kQd
zUAk7)b~;3k<7zpV*zk0tP9(B&iPac3*jh>Ko)GLWe9Z_rzu}`0urg}?Zne7rPBvvL
z!l>lXd;5W?i;i#AJktM}OCD(LSK2-uw3k>K41{6YyRq$($`aJRevdkG_pB$BNllV{
z7uf-|&qENGV*neR(^6Yd^C5^u4`J>`QWWkHh%e=sj%!cv%Sl)zs6<(7z{$5e-5gxA
zPmXBCj(U*NfO}U-Zyv}K`ko30rPMM6f}XNir1uEGGLif810~>l(V!?qe`d(<n}c$?
z1qZg-+*OxKk+s#Jz+=4M4$Ytlt&I0+69x_yM8Dag+X3)Fl|tW3N=k;sjDVQ&pt3g=
z(tUJW6uj1|%HHckq~<5f^`5`OTDDLMNM$p#6eSxtb)JeUO$uf4*+f)Jo=n`|Xw+EI
zwf&y|`}X-4>yqdG7d!SFDQ1>K#PLR2r}{|*S<V$Vf<tp!7$lxVHhR_vE$kyn!afQ%
zFDT~Bc?KdJWeP<WCuhhpAByJ&yt#pk7x+kA)A}EKUVXI0t8egjT)#GOFHjfA1Bu5J
zl2^~0iDsDRDOj>o%*0N7vt0U>@nvz^61u|=U3V#bZvAoIVoSH4v1FQ-(&S435N<b>
zogxkcI*_^p_bvYL=cNXJfn&CE>XH8tMd@0I=L-X9)GE6OM#$-31}(c@e1_CuimI-L
z{{PNeJ~YIkSjdFLvjAnSb#?;?2`#<7Ws}7byu~~_JCG81116B<f(3hHL_f#n(II%X
zj4&F<<&t>dNm)Euz(eiXg9nzWvDCIv3%*r6=r3A)S!2rpk7ek%)9sujZugD83)upm
zSp>T;OBWQw8FWjoT`m~U6KYm0ZFD}&Vs#Oc(RL%-TmE4uR1hqqxLJ8)eF*`F9Rl)r
zrA~f!m5vAW!7#sL3T^kNE~QU)W&1xBLx`)K<lpyN`O0>|G=o4i{lVry#Qp#CPD89P
zLU<UwF;#zk$s*Jm{)qdGM3PKMcGgU_4bcTrc25#V0!|{vF~yn)7J5E!-E<<8@UpKB
zO8ARm2iN|Dq&f-5!CduHsqVdIwX_(23vsF7xrbAXqaH2AFZp>y3qmv)JmaMdmx%c<
zmiR4|EIXq(sae|ed)trmbfIV#F&TU^+3W_sSk2dQnz`fF(P`_5ICPqRqhK&s;0eY)
zpfKj|?xzV_h5?+J&Xho^^wZn5Wepdr{LhQyP($6RO9GzrTXCaf1%Jw5IF=7`g4KS@
zt@A&L!hh0+W(#sp7pc+2sg%|L1S}P(dhoQ2={w+n>}G>y_<Pp#leen=@JR@y9|Mw|
zp&=GODd#O@M7F3P<I9AyZuV3=L>Vp>1RbGU^0^td$!)O=48&C*P~m#<N#PZo_Z(zu
zs7=u==%cFh59Z<Hgq_(NXT!OKlLQH~#S3oC#dPb>6;Y{Q34$5SbgF2U-gUNr@!86p
z1w8Q6L0#c!mvy-h9ESq#GJl`ewn(hwA5V+*ucwVDZg8yR2A(!f2yC(7-_gPTA5!$6
z&-xQj?-G}lH&me$am`LC^NwK>3dc(SCQ1}<|8LG6<+Dm?uxW75Wo`Z&3X$IU?R?I7
z=}Al%s8m7;bI1CGb>RBiN9r%K;(K%Y1bO0Jn+Bim8%R;fIl@o8!)kDxxE%n7bfcJm
zjcW^|RS>|d%_#G$t}XW^p$TYIu>`qU^g$O0U<*j^hWzUh;=%qAUT;Q4B3eM7P&cJ6
zG75x9ruqU^@rN#of8H^C!tlV0b{`1n;y?tbxL?3mg+Y4!|Ddz}RUK)3`+G8BQfnu7
z5pe?HV0a9IXBj<=*eBS0@i<P+&lX@ixK_J)oR5NrD1oQ_@}H6HWA5!3rmtyx3N7yn
z*%b-fXp09(Ly<6EQ_g&<p8pirWll!&o?M1StU-}XiFb#;(z^!_NSyQ5xHju2m8S5+
zlO?r@4b3sI$f|o?u%Cu2nxjXKMh&>H8Ud(!4qFr|X=wK~&2b=+&J!5jz+`Aosc)6`
z`rrwC=WLuWSPbuu#AyO9@Ybx3d_v(KC?+-5hw6The{a-~uV~K#K&P10DpT5qjk8(6
zu@x~C0;zGN+yCM6?@#@M3jh1o^WHwa?EYZO4DA$na9G{Lr`$w$nOt&gI175UpDIyG
zMD=VvdBpGN?2yNv`FbNkacosKSvo?S(QE0jowkbOe(6$(aWkGWn1JcGrKwH|QQ?>~
z)F}$N0w&Ah6BQ}p)jH-U<TuA7YDnIn<IU6(M288{z02!I5m_AhfaNqZ|6H6*)EB#L
zb5xV4&f7Yzw%!u@%h#~c(GIZA)aut+FYu0wGI)f|0OJacXwxX&M!1m&8Hr)RcXOSv
znH->z*IR)&QJDkH5jg)6?LRx%j7{?IxmH6cT5jbY4b&zFBWTpaI|!FfLjRyQ|4}W+
z|Ky4MFU;vrVUd3};os0WJnkO-yjejIpo~YL{chokEWAq;CLUO;lEpSo7avJY;g0$l
z!ThI^WN|3=U_6)KsHt6A|0}gmk#_tk^BJIz=5*WyNG!Wn3=$V7Z`Y=i=jDa%qsX$t
z3_MS*jPiOncIuq|6t)_h*G&_MCX4Z9o$pTdj|puH&E2K{cNPHU{>l%pG@+pDH59+i
zB!vT@&Z}*4^B~e9N<lu{!R-caVe~o_uYbSdkgtfqLU0Z3*E;=~4z{x9$*g8)@#YA_
znQPko-w5r0@x8;0@K%b@Pg!tc5^PX(*{<M7+E7_fEE_m#INyWyWOwcMW_WrBGnRa;
z{$TFKS|;n|KKa1YB&>iqrcoW&-AV8wx#}C!w-Z@<{51nPQl>~EPH+WCaI`g48%9mi
zaS#R#Lc<QDuU!cFTtG_MC6H2fIVEU8pZAi6{s*>_A8>)i(in3)NRdJmgsh;nA__p7
zfD@ontOxqL?sTAC%;vF*d5W_(@jsC(D=AT&TS>&)5?c!==t_Gzn)NyWO)W=sp|W`$
zJ^STbX_*)Kv-3}_r9#Eao4o%VhJ1Mcb+M~$hGeXPfLYBEIK3JF-|r+|5@0z1v!_Pw
zG@BA#U2wA4&InCBT&5_5>?cj0bu2uM<YsIUSu<iRFpNI^DQjz|P=ODpF-9f3>zR<}
z^tCPa0b#bmGUAPyiXoEUA%oo(%j;L$XJ<)79R`@foQl5gj;a*j64W>r`GgmqGdGyq
zpUs!50ZCh_%?Z?W$d}Wopn3YNY$4WVu4JW3Tc0+{D%%C1J3K5(QLjB(`N1+={^D#K
zz=66W7JlW+94@^ZRM+;kYdJM&JOcR{j$_>}lAIYVWVi)Y`(Uu=8;jpu*fyGd3piOr
z%r-rVAI;?+BM#)QIsZ7oQ1DdqCZXs#{{zDx7KOic+(b;Uf@x+PBJ*ad>SfzNS(g+x
zG5!}?`|ndF5B-VOB>lwUtgN-gBqxg*g-5Ud-b~H#b8r#aBKaICe<;;G93JoVxAX+Z
zO@g`FZuw<|#XXhC1fXURj;Ct-5IRVOsQG^J11bhs7h}deT1v!^(vUM5RD=OHb-zdo
zis_B0PKjGL|5zgb3eShZ*&@<xxLJ}=NtSyjg9#jlNk(Ggoi7C)P!PY_B)9|2`_5nK
z1^65$`W7oh=7r07e`Zxw8;GkYIuQpx<d1$iFB<i^GQI3Bh@ksYlp=>kWuA&2##%1`
z&>R1j4(aXT{(2%oI6M-ZGxgsCP?4WYXg|-HVgSQG@1TF8h}E$S0DPDbh_*bH+!az6
zMJ4*o8}U6h?%2cNX2?%8GmgqN+$hQd?l=}1wx>j%5gFCR+HS5R-WN}^s<$$*EUrt{
zvvw(K>E3`t-2!AW-sUe-y3{K=MYzcNa%v%BY|W8ZCyBu(KkkHZZA{0$t*U=cAIggC
z8O$ao-XjZ@tKy%qQAJG9RCykz0C;5_C+OReQ;3*OhfQIVfCaH?Nc}t|mI|0?SqH!z
za*IREb48zB`p7_jmj~{c1&Y8g4u2d6^Geut!eEVE5h#XdZw^o0O2B(}NN$do*KaRJ
z!%^(3Su6gle&(#C9{orCtp2b1dHc+e_{~%O3<BzB4pFt$u>b7Ee|}pKR6eAq`k9*+
z>~EkYc)#VCTNgy->$b)9A{cvdSDQ2?U=lM_{T}%QiSVO=D1r+{6x$KJ(+Lq_HhB`w
znG#`cL*AT$xtIl#lBkqa{Qu$YE#s=*y8Tg*ZjkO2loaVM=~e`!OL{3t$D$FW1p#RU
zL6GhS>5$F^NFyR4-F4@}ea_y`x&M9c{oEJ#ty^F0-<osGF}^k4aZ`?FF}PqdVU5d?
zCl+0jfe4iuMIMwezDMb*m;~r|biOXLIq-5dC+n<4(0`d_+FlJy*y|<cz-mK~$X(zT
zga}xegY=?mFfhQ2aVqac7igztj1R7N+wqRqv!iF=?DE)Wb%ArX;@Yh3(so=|ApOvv
zt%kg?<!V07GIVG~`G{Ax0@NrKAjWQ*768q*OtIs?x>0q?oM$$)>?U&5k2N%}P<kgg
zjH8>HTT5Qv;viw32EVudF4Eu7|GI*!{+z+HaL{Ms$KzdC>8zY<D{f0g9oEn3#Divq
z;>^o7ql;5wFWY^><GGV_iK{2*)m+nm{~~`si?@f`|1O=z8|3Kx-s-jMT0R2ayd{KC
zI|O+@{;IC=1fx{McYJvxiq`ESq1z+*59RFlC_brcJ&>2W4ht4*)59-{Bcs4VI=j?}
z@wPXoysUKRh;f{WDAVOHlKm(zpP|P;%4XE4X#XO_hzaNH;ULup%K)!A8l>wn<~uGv
zX@kd`ADLBUqY+7?DUZ(V)6H{uQCpE-qmc{rl(!PI!PVh_Q)vhE6E2*YYL}*Sx;(CF
z!Rf#$;wWwa<b?0V>_P-Z>ov?g8s~A$!qsm#f8+pQ8p$Y$UmbBBNP~2452@kbee52u
z5$ggyY9r=93G_{TXg#5E`yxCuX#Dy22UaD#eJbV0ysv+hZmKX7CTOZ5M>jXQmh)5#
z6;O!M4yC_r0r`ik5=ED!<DLKhMRVW6TL)BTbyHCIn1W^Oi>3V;5rhILj1YYLKUx}m
z!kF$5hiR4)EYR8HHX?=I*>8{Hm$t_g@8>e^Dw84mT;kFJDIyjr@+Z?{J8ujc7?<D8
zwsx$n!YBFC`9)0&tQZLovi9c+4(`4_M7eWZ36GL^JgH%Ciy6RiP1GqDLM!Xp8M@uO
zIAXqdZzVnN!vrvomD@_L>VmE^_7@w9S=-gg0&V$gg_EyHcrbvG7rY$w1WKmTZ=4P6
z)PG=1yu-Vpw7{Oz97!#r{iX6{&EW4zo0PFJ9~u5;8mF`YkDW5qq(4IyzPni7-?y3l
z#h)?bi+C*v+?(w)tY@Ag+cyqkYFQ~Pfy^6RN9#&FxA{>#S#5YRezp>w3p-gjFut1z
zzPRLlmVj<sYV8_`_QIra<!|fF;_uy)MjmO0QdiqQ3bvCTsDJ)HQZ#?HZo}j74mPCN
zdW3dok<ww1i$+qV&ji~Dp-m84i@dLxEESELg({sQvAXnGSFs{Y!W?#=WCh*R>_A@5
zb~@9_=O~}x*9+r(jzt#44aRB0F)N<w@W&w?nI?gmakP|Kl`3Hf3dyFa6eD`ivIFz{
z#mZQ39_f>Z@bBf7=@@S+YDvi(*H0=ooB*;Wy70qvtxjc3&PpF!NhPOYeQ9xO`GU{I
zw2m<}-ick?$oJ}K^0WHyf@gGY4~M{_Q?tZjqhBn6_GcVE+NujNVI}D;jZ%+1{&aO`
zhoaU0_{1E4J~3+s3^+`zlB&*O^_#EEfU{kRcI9T?dtlz{VxZDk0m)3hrS*bEare*|
ztKuCVUe(z_3kOsGBfAt(i0Av>F22J#9;y)^bJa6X%=Gcxk8_=awOFROFZ}!AW)A#A
z;(t|F4nwEe#aTqSZx>jL${h`Kf9`i++;1{!!A9rN{53r9Q-;d2>}UyyGpx&Xzf(@u
z?g<0dmU*i@w>dNolDfgDN7g(eDok=m0UZj__B@ZUTiIawV!|bI?r0DxgFs;ShAcU-
z+U!p3Bufn8Go^hKD*YhGqR)`gwXlSdNT@6p)-S-~&>0t&_pv$`zy<F%K=tNJY^a(B
z7kZ%L(HKB#`CoVQU*T3qYYVYL`E;CK-j|gYJliVcfh>^{DhmhR5@t_y#XkJw+vb38
ztNE10z0!JmbW+a>s_)eli5)9nO(e4mGo5O!e*Lqr;4bkL290zZJD8!>ga}2e^&oZ^
z6Y5;(o8Frhd#)obY_s8ROB{c<4gWb+J0K_H!h55|u!MZYrCF`M`c@&ZIN1>`)>7~y
zx|b@VUEYt<PYiOV$>@OAbM420(tO7?^zzJ{!SjebTisyIJe7NZo-@Gtuxu(-vEo{K
z_&m%nlR1i{6IGWA3Q6GSKb&TgRQ#MQ$l$#Q7DVHAy~y^s`z$n=ntM-nosD0-^4{sF
z74EOS_9(LL*z%dL;c;wwmK7aJNj~x}<w0Fah6mjz;5BOl=|<1)9Rf**9bmt{WEJF(
zf%UVtNs_jVZunN4W3=VbmO#j7rdmcR-IBRXT|nmHswlGwA4Bd1PXzuwTYY^J_Lux{
z9*x@YUc21!N+DNkwtUHa^%+03q8)%P=<mS&_xz!<t84YcWm~<$kYrK_Iw^k~{_eA^
z$Zqc6zt4X*3xao40Sd+BuA-Ly_4_A>H9Qu-HX(_hz{#99LeF5`Va|XM+CoP;&6a>p
zxjvE~Ee5^ha~my`-j^&n(&PaPiNV6{PzYh%W$@P{0!XSV-Ip8|&XiBl4+q-_5DOwp
zF(3#c5KR5)9-WoC$w;+k7>@*WIx9P^N!uR(#jd-VqQ!2WLBKTsti8GK`B=s{hoTtR
z9Ck5L;uFuKeQ|VB6O}@n=u}wtp2xVMvpH#3vNv}OF6dT5%}4tO@Rmac-X7Zx<Y*|a
zI6pMZsX6HdVONu_-O=<{ZN0ij2Xat!nj*cbYD6z;sz(9%6^0kMzo^my^kJ3{vhvr$
znzqZYb%JF@TFv(=4if(DWAUo~>8w03X_<edd|GcbPDL8LUP1|dC8-Zgsh!U=N!00%
zb0WpsYP+;}a;wcu<=;+aiB~iDKcqQ{d$}vsO%`A*#n2s)>ccx3u%hQUtVqj+`AD{L
z4;ICrg2;rJGoYM^or4f28bBMMTFh@g8b|P$E-Mjl1z%K~*1b1HU910zC22)h|6*&v
zp!kLTEO1*{C#cW~Q#pJ^oLJ@qt$97ILkQn1+5OmiEDKy8%z1vR7l%IAN*3w6RaM+O
zQGr_ieXZ`({^>5eXZlN8DsP(3zCh1=4|n~R3!ZI&+xmJo&Auu<34gQhGff4x>zo*L
z7m$!20nKQ3zwyW4FaQ7O+`m5YzpZgBna~eJxN%Dq4)b_uQa`UPdKVir#Iv}aDS)BS
zUY=6lJ`cWUnxDc7q?n|m4c>7yH@4qvHF9FLw3koW3kZ@tA~>_j*29PLOQ}lf>VB7F
zDYda{2V17ga=PWd-k(Jk-r~(z%jBY*frX;S^2$(BADGLhcv!c;Uguj}7659KV>7Bm
z-qsah5$7w`eV`@mylw)SZF|mL@ODu%Y>Qfc8#chd^;{9(<0j6-bu2D}sD9kUxR0IY
zw!LYh2^RUs9eniW&mEM%AQqkxsIb!Zs0Vt@dT0I4>V>zP5(CUlBCX)vP^GLK`e@?~
z)=OvzwwiMj&HsH&|1k&M7BrK5NjMgINdP9i+N^ubCV2y<^~KU3Rr%4}5ti|p1{Omi
zgB;HtFhaADJVNg*lV<8jRJW3PVM35a)?}>ht}_*Uz<OznYDeR9VLMgb&!!3f=%lAl
zD^2^6-<wrDM9kLnp~x<Y%wilv<u0?rrbc0Sb9k7fd~iS4?<uX<rI~M6rP^9%rIJ#K
ze_}IVz1iC3r{@oplNY(Ihv_bkXWMx6Q(b3lT!F9hjq3R0-SzsZqsj88>X52T&zxh9
z?XI|~YLzMe`~@o>(wX1BR2j!uHtd(@G8bF#o?<!Xk7&a|)I0(P+P}-UKXcGuU$I!;
z2(ZxyYv15<>T(db0&5G#+S-$U^E~{A9>5$)d){(_=iwdLIQZRiDBUn{BKFzI2JBlg
z)oD#>Xwt%9<;ok{It$X(3Xy-op!^t1kqO`qm5{a6vv3vqCuq<;;tnAMv0wQW9+;;}
z>NswG6#mLzMbRB6KFgI~S$rddIFb=XlPxoMnCtQQc!ZO@tC6wDlg%8>;_Yuk#1NA`
z=zwq-Rlj|1D^E$F%V2t+#(bTr!;k16-W0d9PjhwvaiuBw(Db2kY#15uda-%Cde#;)
z35=|>*4dGxJ`d1ooYwlwC|}mnN1g$$5_XZnSy~)eqUaLLdkph_aj|_Z7AIkb^;>BN
z2u~iLZW&v_+PIiW%HO9}-=0i0cY|MlUq4msKaKYhnz3;}H)@K#ja^ZL6#8kO@(}D0
zk41lT5(2m;>gCK{i$@<htpemPw70j9p*Kmt7faD_%g^rBOFzQd!1=o@{QYzNIk5?M
z|I|Hy$m=I0OemwsKQGW-zE3Vyh0?-8jr|P2cWE0S0&TU=DE742d+z}5cf!_$ST|ez
z!f^Tmf9w9;8Ca{1u8PXAH8o$<5y2&I*=5yz5@`{d=$`}+!~CJxcRcM^vnePg-{aF7
zKjyD|(O$a@B(65%zlZ`aR6Ma8EiMTGqzTbdk|7l3pSB=u*!WVwYgkoJgdDw<2?WC>
zb7vVtg)3{_FYqR&cUtzsj=S3*<<Dj9_$a)k0r{3nE~??Tu#7nKKNL!mgmQoQnk!ii
zT99`b#Wur<bb`^^e7^7Gb^KOdW)_2HVlSRKF8}gI2DB-<n{yWnb`U)+%llN<gMqHy
zwX>BdCI5ss{`1ROKY@c@jLliHqR8nX3X~2AkKg0(AjG10Zs_u=aWK4Dw#N1v2vobg
z<J=KpH?xb&8+{%o8@ZVHO#qu@KalQ-n1jIx9m@h~Myt_;fc1mAxUD%$7K(W#qQ4+M
zKk~~_>^(j_?lO7alC=@4@pTJjV{yj@Xx5I<ZY-Rz`*oJWQGgt6dZRy*uu}ys|IxM@
zv)wQw<W=-if=S&H7M?MkxWHaMhFVAwK%~6;LEs4GVt-nKq&k!8R!ky~>tOJYDzA$B
z*D;efrVZ#W4m_?P7IwyNO{X}E8Av~Ju@-5~7H%E|WacnCSnr}{w`C@=aIV5JJAbn*
z=4IZssm*;-v3esR0pod^UcKWx?CbcH=N~`D|1`^#_S0+R0v)LJ5*;nBe^521Z>^^O
z<((^TG$VxO^(9(%m9SG3Kn|GWBKjPbebzR1%FOPt8!>n*3~T>e8f}&Vl8+wRK$(ui
zNo7;AIy+XeAqizLiQg-18fDQ}#7`yh0|W5*JcGm`mvz;$|1FL-7#THR%y||0<%5uf
zjdowT(yqJ7x!i5><JqAI=<dDJF3VVWG%|ng5Jg?1d=g6_3J1ZU$36KTB{|G8w<tPu
zkipJBt|^>T0G$5OXQTVA*(buAvu{#f65oODUe6w{`F4MjN?Uv<ty%OAXw-|pW6-d_
zKEf5VU({>Rzo|l<c|CAw+~3DR{rU#6dbgo-nue|FdhFk)j2C!tYVlARKRw~Skm8a(
zZPpuGL4?$^b^;B><ovH<F=CL*yiQULeS<rlnO!7JYng9aBt@?s08<8EKirv1ebLQb
zHlkPhP2Zs~`e72ke1to<y__&aA>|#dQy*&D*6`Snk<ND(lCSj1I>O5jb3KN;edEJt
zVRT<+?}5EG5ik3(X}~_XimFLHHufM|pn|A@8;Wdh76b9B;q?6W&)L&~S3bZkyQe1r
zhosYd&VS`^f=8g};MxsfWcGCrZDzmJaVQkOSh6Mg^d32ShqS*;{kKqFDaw!1Nq86d
z+HZyR@3Lk6`me!R;#?Y?=35QK>gq)BD~K9?2~fMr?hrhzsw`o^>1ve#09GRboYxlz
z39dbFLS`4MPD&HLReh-@U~$(9>>&Oh4b6Wa&C&=4xRSBh`t0EYr?@+HcYaH%q-uz0
zqKQ@YP(_#yE%E=+(e9#;zmcFz>!EtfdKA6V37NKrcF;3JF|KsWFZOf?U8&eVOW`PS
zXa!*)(mQn$o3N(`pkv!3+mnI!oN&{|Zl!~P5iKjS>oK1FgV43MDE>_?`{g`!O0i&U
z8awP-;C##&S`&JvBxAKZEc&Hp_N7R};i&RNl`>~tN4n=mer1MwHsE`X!ME1?td<sV
z2FKWB-eBi1*U!9TyjzK^6aO(lI0qz_?tD8GIk|6rZP0c5uy8sX{1ef^`Rl~87^AnE
zU!<YxCB7^OqAQ6!i&jk$&R3EIU&}vRi3L2<K6QkmYV*syn+qu|q*{HYHE(8obFXK`
zuhzy^B{I*s^5mNSUEA;%CwxWtNkXNs@zXk;G1}bah}@^>Pf1Z$H{2m?&vyxY5z_CV
zG$W=WhIPeSl4Z-0cDLaYocM7s56fHIK@KOPj$r>(h_ch=(W_ZIj+z&%+EE!}o|Ex*
zK4zii524*7lP8mrkEAihmO#AUWNXh$T7Am69y6)<9NhO6FvpR6o`V3LIKUqvKqEYv
zCHteC270L30AkSF?;Ko}U7v5x0>mdJ892Y|fFisk7fJ!ToI4?I|3n6LEeQZ+G7w#j
zAJ8q1Mw^0~yth|bS=0sKhD3Zx?PpWXd8gwfoCYQbKYi^=qM_mw0-bUI4WEU-&lRgJ
z@h7yRqEJz{h=z$bEe9GiCvX03C;<~3hX12`?%pY)NVNzK$cy+If+x|6v_K+AN#H9f
zKE8~vp{&kspOZZpHT2!_Ewt1Mn<l`FmBe*lIa$i0#K(wF?TCORXP^wf^J7t`X1C|G
zO{yw^Jf;7LpV$`pBbSF!QI0{}&gd)sTsQ8VGw3dbQ!}eF#(Qs@q@Q@=G#(bcBj)+Z
zGN5E*AAnY2<TSunJt9XLjMASWu>h<u$QJ<PWhK?>9s=2^v(I!4<IitmEIMIMLt>CY
zs$wQR_5!NcubPUjpHqD-f7XXh<0GrgUqAmh1qZwl?#Q5j;duiT+2d4Y=jcq)zGCif
zNB+#8Ktp3%17_a!nc@_~mhOeI&##A}7c*`E4^t!kyD<Fcs|?Tsnp<WiM%4d-;u5~`
z_dD)L=PNi?SCtQTk@C!o5vQp;UYVqlEmL<?_aV2{B8%)1b%=#d`B8fog?Lz-BatG~
z7u%Z^2P~szd;2bG8Mo>LM;54xLc*%~`)d7S)#%K-1>17*J2ea-33@(ZG6pk1jDh~O
z7ngd^g7TtD^-v6)GdZqaHofZaufI8PC>Lml?|AzFtvg5=zes|fP|p?{MhS1+7Xhx6
z0<tEcLUtXIC4IKH^QQqHVs$<G1xzhvyPJ6174m9DLyx|k0-USFI{!{M{1|4YqdNm<
zcTYqm=#%t7!Uv=^07Y9<(-goE<rsv_<2<(4FE<M-qKcZ1XMpHQz;3R-3M80J!HFI(
zATNi2V1z2v<=NE~`|`J7VazvSx8JGX`zQKACr+&p_^SCV7kYIxRdrrc+J@=>5SA^E
zw+SmVFa6PU)Q%@nDKPKOXrig_cYY53=9+0yRp%rOq$so>Xk-Whzq6r>T@k%h^U<V!
zZ_Oqtp-A}APZ#N4d3TOIDl;IpYN`nFUWJ=`@Wf9u!)&-=gn48)W@XU=Z9B?<%WU6F
z7}7Es*t`NN!Ybe+>&_LL?L4i(3#~u8YU-`aWQYe;n@*(`S~)j5&B5AWnP&97K6L-@
z)1vFtIq&-Xrl3u@<-<i?+TpsR>N>`m_P_UWd0~=CcSYBj3y9SHP@Im{jT;f1`_|}<
z{7w<ZiME!o9^zUdP($B|#9CH~SHm>F;n6-uJ9RXM*iL^-)MJ<rJolKuOiy&Q_%!fE
zn~H>qS*cOs*R-Jc-OF*S{|BWVAX3EQuxYXrwNRrCR-;NsSJ9Xv0m$5MIhgw3S$wig
zLp!cWf0W2?LB??TYSF91F@M1}Q<C!0sFt!HUXTokwnGbeI6a;kmvUuXnYY^YxDLY~
z&hmH9A3<>wh5WAIscHRpytU;s(UB+hUy04WNHnx%qKq;81>~BXW^ZN;FHUw|fiS;V
z5bD$mkFH5=j}gI@xOCVAJn7)xPRq?ajpzDDNs`_drK3u0(_o&l@94b;r^6b4*B4+4
zF;ug4(LOP6j=jRB6Fl6U;xcvvmGh@P5N78?%dUZXwV|kPE!n2Ao&ua3^Vh*hz28`K
zbbZCq9Om@J@BCT<B>fjNJ?vv-)~*)attx_ai89-}J<l2CqY7;V8+|r5jr%v7yHMgy
z{xtK{{owVVXezMPt9fZ2cr3*GEXv#6x;FjZGu|b5iES{}L#G+Df?BkYdFT=Pm^Hin
zTcxnx_(NLni{}29=q|KWs%4)})aDYFkY-Qs=&<fj?KAFxivLT{T1VxuvXkfQo2<If
zPQm~8vb%@>N7-HKn?nlsy;>mTklo_8D`e9HkRj71W}5({q~W))*V&K6F*Y5&N^1gK
zZZ+{Ui&=+uoEOOzYY&lPL3uph+<8SUAi??BkiPH9T~W(s*KjD$lD~BBkWB&H&9@r#
zSu?(|n5%a2&IR1-CeU)Gnl^oh!o^WMiBSXX4yApm|90h=T0U{eZU*!?34oS^aIw%v
zZ|>e#q|O$%97M2m8CV-VPkyI;>HXe%L9yewhbVr-41}L-`P4Yq5WnvqRd5@3Wbm2f
z5;+YEUntkc^s>ASTIKdWbx0A1daiGxIZ-(eOFgJV5^4I(N{C(!pnV9Pher34!kbto
z3?k)6_Y*QT3w;5Q93y2h3#m~xj)HqodraiyqCONNb7KZ*RTHfkZ*7WuapY`O@OOY&
zP17Yn-bIB%I<D73S{F#Vk?U#xmnM1imRG*+#r@25uQ?0F2l38lIvtTUPLq(xa_6ta
zF|?wHhs$`OH8FX2=w<l!N?VN%MwoG9=b~1$^t8I#qXZe;?<d%A6k=%5zP<XOMafJ8
zd7Nn)jst~pjIs=9J%*X6e6{Fq)jjBwEBm2tY#v73z4Tbw#H_%5JyE@H1BuJy0B*-`
zQJf`1hTm=wH2&sQ@Xf-_m9b!+@?-)gnhPO|8n*AJpR+$eL00S@uLZ_CZ0R1G&(E&b
z$k99i6H--W;K8ved`git)xdeLb)~{L%@nKUgYHgMk7fv&aK!uh%`SPr_Fo{&oY%mv
zH6FL+`l!UNUwZ7c;DDUkKbe4zaP-0F)L6^M0SHpnSc=E=yKpyOvC4>h*^zBQktMJm
zy`p%-t_F`Xoq=n+N9`m}1(w!QUg26`k#B*Peo2q~JOYoBzh6*GrnWd>v+?j1T9V*y
z^p8-+C<R6K7=f3-GI^NHh1-WGxah}Q(EI+uQ~Qokg0cqUO|Ag6Ps#TCO7U-FanVk>
zrS36X@9xk|rA{zkdI<v))i@C4e#n%vSCFd07>c)I0zOJ+_eS8|!%V4ovl|zw&JE*H
z<i74S8@gmkbAI9;IYz^0`@RW;weq%ax&RxVGLM&oAr`rHc@B6&j8A}+_l0prWU_E;
z_OpV?>LYcxD$9W*V?U;Xxjw_Ec`a!I0`zUrxMQoKWE*%_wmmfg#9@4l?*ZRq0uX?6
zXQLs`xg##qwDi{OHLG9$Dq0xkz14z!MLx<`Oo}7AlA`8B=)iUd!k>q71nPjd{DgaW
zX&S@*f75sNN7G1!XoR=;MS3rCJY>H3^t|@I?#J|GgK#aX^B6(hygvd@Ugl-lNACj^
zQ(@{*`%tM2AWOH&b>u^R9w<bdQ5{On@Zn*uyCP4eK8s5UB54cv#=$gu*%ON7#SyR)
zj#Aj!S`c9>HAch-5WsKVZ&Qt?fdyy3OoQ!h3xuq1TV4MVe>581$W<vh6kKTiZ<nU3
zt?`O#4VS3+u)q5br)F>LTOF+Tu&q0#0!^a9YsDU>uTl?_M3>vg-x&K3Eus@eNvbUR
zEjj?vS&%)cBhw?62ZX3P_;X++*>lj`21M$DHc;o`GhSfceSbOY*w5V^<meMYD>0uA
zGy|2iB^=;CMDWi!Kb`ft=$vwX=O3iPgsxLBi6BKsv-^tZ@mpQ9fL^x+sFlEX2N|U&
z_a`$v9^duzLpTmrP`P^wMj&7spkeC}!$)xLREA!f2Xw8EWN*WRPpz-buFM0vK{u~I
z{GE(65WGg6;tjy7Hy;IL`K4AquWt6DeS=&tR}RP2@htGlN3ns?fTrB+QhBewr2NBO
zx5Iw~n+QI9<@{f<CN)hUzvK-o7>rt9GrQY^fI$(j2IF9t=9^A0*Xy{S<BjyR|1Kmn
zA(#I@Yh=Hk#Ls;t$FD(<z=V325@blOB6JOZW3fS{6-4d#7c;LkpGUmzSCv{Vammy%
z=&DjX#Z0Dz8f7W&1D(HaBMAcnk+MmtgMUE5`3%1BV<_Y?G6Da4>Xhc?4#=-cX73Tr
z?5FnUq0VLLKp70S6R=RkEw-8g4{d=1!l0C?%5M{Fu%YX7n%lg`ubMKpstMoMUlm-V
zZZ3aHJVP~)?FPSTB8~O?<>Um$3)c!B)Dn`li`4>@>Yo`4Umm$Nr#&$oYV*DwG@I82
z8V$i?oFpoUkOHEv={)+=#Tu4aI8EN*ese1gUf`(oz*_sUJ<+V{bpg^iF!wcge<X8R
zQO~+oCOKKXAsZEt8!)mON=a)}+~=>;g$)TTa9*$NFTdL;(&o^6o}vx0pZwmLg!e_C
zqbV8CB>V=QZMd%u{o9D~V<gS$u8wDmh)V3PIq1SAD=aSwU$}NtyJ<1XQhV>VE`1>$
zXfbKM)9@=dMVj;p@l*qy#n)P3ERoTdsLfptB2-i8eXzcQeGhm`O2Od~`7i`mS+ull
zEHQ&<1h;z$fcBpUndp(3tdkjZoiW&9_b`YyjoGELb#HQ2cInt~N+3*Ab}cTOldiz>
zG%08l;rqbo#MBh$Q0BHcnV(f}+kZ_*`w{twToeFoCiBODn=Ws?piZBO_IA%3$S^*_
zEO0J;%Xi8i`kt*d+oNLbxPkh{gwl0Zu+Zu7Q}=e4PC@L~Hou$yIs5!Qq-MT_+cJT!
zYHKX^(TV)2&|+4oF8%)F!tVEhB=(Ew>?)X-cbKXZajIru4_rm@AdW<_mZ5=oo{#zV
zh@<%rz!vVqb*6pImce9%BLe2Bw$+kO$zc=vh;GUVsCD>r8Ol~#i6RVNZsXz3JsYGu
zE9lV9SLD3$2i<^BeXFkeuI1c+k^3)i!D<|p=Nz{nH^kk4wG6y+6guS&_unZ`8v8;w
z7g|~zIk3>|KTWe7U4SuS``70T+~-1qtKaTXb-DZ+y?4*eWUNq@a?|H4^<?9-1_RE>
z+zX#psuaf?O`t{_y+ky)RdZL$f0%k~S6C)T=i$5s8(~&WCMq9)zx+_Nq?a%|jUU#F
z`l?L+Vat!+l<n>4`RXbi!|o{9o`Zgc{?6e;DDXYmZ(W!}dU`frzZrDa=h+i#o78*3
z5%|g9yJ1(~%WG!Vb*{&5dlrSSC2GnLGS_jUusKo}Ql+c*$~gYoxc|H@*H>unrSEyQ
z<oWO_<?+mL)XdrXFx0(Y=D1CS%Q{a9`u%11tY<+9sq@cy?X$?S`8luI2S{u@D+b(`
zn08eL$|4?%4<-wgOC(wAE|<R9Uyko}r4LSD*l)s$;|5X>sthk+Rfrj^giL8_y{Ggd
zRJQshO;uI)nrGTrMuivk-NhBrah}g(PW3OhLf8>3kYWnK_(tCzj7~NhO(`(glTZdu
z`K)CMgk;<TW6?Iz;JKZ<`p+g>`ll#Ud327hkp|DvPX$cC2!~U>7Y%ndU8a1U;566^
zaNrwdj269{G7ZFC9C5u`BkzJ&fTs_y{xmGiD&3=N-m`$!k&7AsK03MhHBb^JbY#$}
z?dtNFJkP+U+dHtaB^O259)GINr=X&sBAT^O^U4^`l~})t;Ik<1O*lU%z3_@_ih6SL
zCrU~wvY`7LE#g(fQy?)?`OLTNcZ63SRV1QUqDF2v8+(JR-DjMBlO35RE@$|^0ranL
zrR%S%)D$swD(QC71yAJvzi8J#5Gs*;5&Xh!T6P(1C|Rc4H8Djd{41aBhBbrt?g&}c
zGJ*V}Lt9T^8yaO7{&YB9o1i%nVhOiB`K$Q%Mi{LtZ|_O%Dy$hXJPBi`>EvvDKP{qr
zOIK)a#XV?}n6|z??UsN8dG@!KSt+Nr^`_dxgH8gxO|RPR=Cj{YC?45PA)Kzm$8{~8
zBHITx$D1NU(Q`7C0X-ivT0b%;iTD;@U8gVnWD>@n_b#QLIVnvxczPu(<7*WMJW3ZL
zuWBo(TV^pn-YhEP!2-=+`^KG?n{(DY(=KcrT?s9~#XC+Zs!K|X#}|nCM%(<|r>vVR
zqMtIhH^T=%l-yT*gX<YOi*f&T+u&(8mj%g&c)?NG!O4vaaRbb+K=kWTj+=4Vgz^i@
z*T_s5OjE@4A=IuPih$YNl<En!g-8%vXe~oHJEB2G)}}r{iL7eoJ?-MzjI<3LWkCzc
zU{KP=$A+%sBy}y><9<-$d&O3HCZ+O-$iWY14PU|e!xcC;)4<6AfxGuksoSsLvH&;n
z0yNPw#{Jda40$7nNsv}CH1)?FMO($|#TD7YM`u4Y_dRw8*)(VsW>**tAOtKCjn;eE
za!}>b+~ZAj!iinR!LO}H{5RJpEl91Xjr;G+u2!(AxS9@Qn?Rx?O4AVWCM}^T-P|sX
zi0}2KcQuWDSZg{$=P7Bp*a03$vYgbj<4BeQA|)5ErC`#w36X~FYW>r+l`02tdxEFp
zx3s3FT8K)#iayfY-qHdxnVZ}PQgm^>$Q7>{+z+>_;?l|no=+s)^c|KR|NP07pX(O<
zzhi*FOQcYmkxYb3EHoQ^hM_2MnBU1b?2mv(#ho9(A51nL5pIm|6G5yIu?nzqXKzdB
zitikwJ6aP0&#^J&up_1?+8rf470aa~ngQX1fiNVZ`B1R}QxYK%o%bv&JF+JO$DC^s
zx!iWzG%)Iq^MD$%*7l~_)}H+HkWTeKtOujI<f6O_v3U*HO3l*?IYY4@W=OOe&50S=
z@G_OF5N_EKt<VI1wP+f#JH3HOzfTr?B`o%>fldg#k9H_Y-1xZX%ql4+Jk80b4Uk6t
z;UD0e82ZYK<liWorqb>}FVn?tT7DL~e4DKPsqIBRl}`7REq4aR?TWebD6l}~+VOl~
zL$9+ClTzY`s+ZNdh`+-~(Nnis!HTjY-8y-?1n<L6`Fuz|I!%VpRldvSIEMwL&0Lm?
z;OeR;`LWa*UU`hd&?>{*;jW#1^J0Zwba+yU*!99Yez(@|Ogw#qa=G|H_B1RwLL(&X
zeQApK$v99X_+chUvmo27P053n9Q4@r8#n^rhv8v0!$0*3LDpvuIU*}7>nJ-_L2<n;
z-P_Qwh$C=UEUwQ$ihw||AN^BwX_8!dS}1SjS=p44S8zd{wbM4nb9AOXvY;3c^gnlM
zC~XANBH+$KIzNXAWNB^u$>7q2m1v21-Y(D3ZclvN#@;V7s7OqM8&VltA~Euz?PL}S
zpxI6NI8IUyMkRXPeHkXiqx>YowgF)8vd1^G?b9WWLpJj*H{UZtT0$wbW$-2fXPcK-
z!bz$e!FY{1TvHXqM5yczQ<-#MTIelN2y7&OBHMcVJi@mO=y^(7v!jGro6In%k?D<E
z?uH70l|;#42iN&&;)aXy3;KJ%Z#`m4*o7hADE)y-$^P5h#a0(~`B&bB1BSd)&8Zi`
zP@x@Vua_GY{AZhZ`}2vE|L!&_Ere4rplcbVN`Lo~`ZdZOJjut{Gs~#fRFuxUi1-uS
zLP%+cVN$db)At5gQ&Fe0yHWLn+DZQQm0SoK<-DMl;4`HL*+u5YGwfl(rPiPJJFLr$
z`1haEijZ)6d7D$7-wg%wx&N@1Bzq|(Gqq;_=(Br1HCF=H#bVrzr0?~gvWiZ#fe+oI
z)3?J=Y@6Pw36)Y@Zy-B5kUksHc{n)C#(;P`UcATQ%D<anJ-YtiJ=P4Dc7;<GN2*=)
zDPQj&*&K(3#IjT1F-`L_8R+cS5SCOOLuSw>c6RD|n0?AmU9QHCe3Fgg_xBQMXFX~%
z*rr%rDpnyIA-mdO;|gm48i5taQL?&T!Jc(XfkK8jPLx>uOC67ktbz-yT&Jz4G9{Mk
zSH?Cidv?pt`wGy+re&{VEFPR)hjiY|r`GIvRF6=^yG=@*_c8Kso%I#SjbbE}cOB2Z
zqAgZ5od)`BJe|T}=BH@L7`RSCrSJ`NIBj>y1@Liq048ueyoNSavW`h|F02)$NYVJa
z_cuWt4zAeYFfXB|cPBEr!>(+Py;N5Oh*@d}qBvsbOHRiedGF2+Re`zpcj@VhB#oq;
zU9hGlAD_}aMfVb1jgbk!LB_ni=V*yv{E}Wjw1=t?Gzwt1FK3^z(+>s=>xkqtI$P1R
z)wnX@N;?LOjyEx4?lb^xwk+g)>&qD+cg#dp9!<f=YCsq(rd<gBME;nH$Fd(CIqh_V
zewrvyj5}<P>v-A*@3L5Wf7njnbK}&Fb+UU~`bznzK6*-HqZ*zVkv-ny+2{?A2xzTL
z1q=L&=eGZDp#R&{@K=v*9ftlMtOT*$r&7=vKqSt2{$!D5{VQw38y*Ba1Tj}f#rk|B
z1Ap1KYTV@Va4i^Q`YzAQZeab1TvIjpf`0Ca5jUZuTgj}gT1X<XQKh?8i5N<P;Md_A
z;WUTI7Z5F}X@3@rm4PO09DsAY6;i9P_1UrOv81W^V^{Ahpa1gx?9~jVNTcFOS~si>
zRxhg$eVn%5pEyXI{f>8SxPSKNfXe8P`WX--_6Gj!Bu0gsJRApPWEA;Zk)E{I)ua(V
z(y<;)MyK(*M=p#zbG>LH$Bf;BKRxOjhv+&!D35aFq-YsUDL)sJY+pB-;M$dL_2#-w
z7HfN)QZ4_{E87p*no0rP@$Dt0H$J<@;TS9`9!nBjyv25JPdsKKKhG4_RoG9&YggR3
z9o}HEF|Ks~5=ggP@V-!uk1q!4Pm&H&Nt>RDtX(M1s1W`h5ajdBPd?|@W}I<N;(}M5
ziG)yx4sXnhH-YX!cKYsf4ERVrLBoGSR*?cZQ}F#?gM<^%xjerYA*19+v=^6g$+Qhz
zfOE%3Q^q09J~4CIh9ZI?mH<A+o~G$Zb_5ex?o_Mj;joOq_h`+l1>QjPB#zoaCUKJ%
z;O>y2>e0fNe-8!weNIE9UWb;XEKy+L`l4aejXjV3$=>UVc!oUfJf@1Xj3s@HR(vgK
z_6UQUzL98!tF2WEjhg6DxWgPy$TnDxsZf%IL3ne@=p#Prwlgng@tmGVajQh-UU~Ll
za~rB**)`x=6l-yV_;@6yHil6zbbG$E9%S&@X%|+TU4e`f(jKRNtX3)5iy^$Ep8)dD
zljby|?=&$k1XUPI0W2=?3MST$VI9B3ef?qx-nX!xZ>fE}wkv6JRWv%8-n<V-z5tP?
zq$Q?J4>z5g{6>zJ)^O7QH~8BDIf3eyD0@%GeeZ#{zCkaI^BSJKp9;+`_9C7JkDNaq
z-D|l~Y=b*pOK2M3NO2Ke4@Vf2(eyiWRG09sVVW!1LZwgbZ5;_$134hXTUCJ1&Cn8q
z^bE;spW1S-SJM3&DdUEG5a%zm9$U94_%I%pKVbfu-;uyfV_DoYhz$0W5oQu@{Srvj
z)RdU(vVQF4y2>h-@6c^ePR|KLoa*aw^nAtEgi{XY&2b)u+-42X?I!TvM_Yi4N!x0!
zg-+kBengs&ReO&djzL4UpXTz7tNq!7Jy9K6-^eQ4>Av}^b?OGo4!&Q*FS-q078L4w
zDER*5af9VevEf}l4g6FlH^sLV+s=i<T5Lqm6KYWh^lBUuEfaYJ#WEfiTa5g|Z)F<z
z9}u@PG8@^?dpTYFFp6i2KUGRXOu?;WbQ6BWE)Vw3u*K-a)WjA*t}hoeWJE5yx2?Vt
z9CshFcVBJ8ww64PN`$X%H%C7zw>H00@oDGuJuMHoTK$SBl!S~=fc_{c0*6LaaGj3f
zofr~5<$(Cb7U4{zE4@&MbTuA2W!)nThhb6Ewjh{jWi-eUKQz%pj-GR!9XLPCZ&YGy
z9tDJst0D%Y0Vxl_-bDaKgp}krCG?yxM$l4p=bb!Fl96F+6${mQ95M0|-6>cjgk@)i
z=eTzF-CbVNW3V&`3PQzK@P2({UJ8rMxIX+`-9=%iMXxM+^4*Ca?6ph&{zQyiqk7I&
ze{=N3o*iZc;C5)f{agq)y97BIO+#RswA`N6u{<|5ZM1afD2oC?3%&}EX6-8?L5_!~
zze85tVX|gRZoTeS_z@TK1&G^1jt%qq83a*r7;%xSJJzT5HHyPIsvoNA0{?gLFo7ef
zA*hl+!c7zd$jd-UrfL6oA<UawBn>H!T3r!yfrykK5DDcS`zfQy_G$F?C|`J2sT5ft
zEghh_#vN6T2wjE?64C`b<gE+1sLMbOLRaa^aOs*1L7^Q!N{MN$93R%m>YkZnW`<Z)
z5s4z@D2ktr4+NVMl8R%>LfP3#qFHpd*O|}^3{vRT>&RIO3}I3&y@aBYtQPyQ>n&u(
z)~M>BSSmBPlLjZYPNra-UQQg%b$`#Zm;S3jHNA(!`Z?g>rhx%xfez&S(zF}<z<zt}
zH8=kBPaXM!(i<2n@L`pY%2+*%ZXktE9WSl!{70gIj^)<y={phX7fAd?OiBv!>^*ch
zo-RRBN<H^-7~BnRPLGb;W`{x^n1PMB@1~~1cN)nK3!_~a7~C(9GqH+>Y~wrb99r!4
z++!$t=(_q$p6OLbBQYt<&z+l1$<znDec&IMxLsaENzXmG06*CboU|VP$r?w%C>OLv
z?p;7juZ%kV2p_lKZO&zCI9<revbOr3a~7@RF?K$Sdb+*A+gRnE%wfPHJwzslYjLYU
z(2HPWGo9om*fLQg8-B_%-uCfP$T#%o80moG5&7}{sq`eOa|~xsbL*0L+r=_q8f8*w
zLA%<>(Q>)}@djAOLhSDvK>R~XUw=hJ3o0a8hr?|SohT68etLJh#svB}2RN#RuHB7S
z+^&&;p;A%OmES(c4+zFtkXH4Cr`VG|PQDrRwcfILPW$J{k6tJ>kA>KI(upcy?uC2;
z-stX6Lk2^tWgB0@JDYWHyi21?@|I457N+2gr#p4rz~e_ish({CAZ5xGmyD)C>5E&7
zr}R6gizoCTX;k_?i~D;_&FGcCC1C$US0vf~r<d)0`T1V(D^(|r&LoL8f7c)Wiuc?=
zl|^5P=;TAwfwkccf9N87)~6{M!f^onudb(^(5_Aozwob@<v_+Vp~#aa!{5QG>rfnR
zP0y~C!Wc!8KkzV0p+q*!r1eq{iNCzt*mK!D`qtbyWJ8(B%}LGwUN;(#n_#PSDtVoU
zIiNc)H@TIul}_PX{8YJ`ewuAjJ-hh$gf90{K$zRsX4%v%Tz^WKt7d#Vx~b*$b@^$X
z6BXCOK=(&sVaN^YAKxm^v|&r52yMl<<?qWn*U=clv1W?d;s!_$Q(HXq^dqOiDc2v2
zIy0Zh<Gx@68o_a7I6M{>-{QsH8px(&IhaPi)-StTdkJGJ^!oh!*`$_^{Ax`7R-a4i
z7PZBmWRDhNYXBP=KJK!RMq-CW3^|V_46p|!A6Z?zi7}Dtmvh2KqNyJR!xe~ul_Lc@
zC5L?ppVK{A4GtJuK49;I{gTt*F|^!6JOTtj<QA=gdYh@Z&TGlgm<|j4V)G;sx0l4D
zsMqzcgUj?^*2xw>>!C6@dA$-Tpv?#++Q>jH$!_Iiwm6U!Y@~f;p+HUZEM#TVwU*&$
zal`K~wi@y!WWJ%?doCjQv*{pFLfFzk)Syse>L%u$>^|Hj2C^ixUZ%N}!ALiaDUbK!
zWgF*1K<0KiO6dayGJ~f~S=z-5A?N2-acLA_Z2H-r-}#5m#!?U;&qQBe-~)W-{eGJ7
z|6c22>EV+RVsBUOy<jBV!DFF|5E(xQVElwfYlsJPPqPfrt#`c2#8H<l%j@L9ukrg(
z`Bo-nPmrG*|ICy{(|c>gj-xESEPyFJldikx1CEX6JK=RQbN`w9E9LRD=CDj15u04e
zGPy4L7YJYYOOgfAtBz%g+{aIC=Rc8eeZbP3W3~YqdlBkD&4Z76%6%2p_x>_J3xjW_
zq@|^(bg31N&Stq@T?Gj7dtE-fuxvw;?d+X;4o7EuNXzco)d03}7OY{Uz<ncRy&6+T
z|A`E^(P&K#r$jxgpa<ImRh)YTL518a1<80S0=@QT7cVO^j(*QK=%s!f!oeXI<1HOS
z6`97kV(n$Xi}Dzz@qTc!LooHt`S-9;Ew$ssQu}uXod3!K_&9GrFtAnF6c-`-c2#~i
z&qwxKXNaMF(o0~R8d7tgOut!h`e-;7jS^0h`#3h)9&<!Md13P4gq7b5eVT_(%>rv0
zp*BEvnujd9LSH3SaD>M9a?iddM#>xrXi9Q@4@cx9H?_-yx9(c7N(u(8?UMBlHv9ix
zjZ;^{D@;n^>#&H0-GH971~lM^HFO5V()?(>fKH%Hz7&uBh4}WW;w?hTt7ebdedGoq
zr?pQl4V~wNC7err@8Z;iAMiDR5iAm$^{h{h5@JEM#wa=j*OMDhndlVWfTaP{#TE3r
zva+&t3`FIAW>@qdDqqeCtDFHZ9SlfoQcn&$%|Pb}fpwQOahrFM>6CRyqY}Jmm&#~5
zF}pNhIKQ0l*t8G*GX&8OAALBZgNG+45sL0heS`e_45YgcpMe6A;_D<B7z0p2KCL5;
zpvFhlBJQdjQY%Rd99!qRK<I4d<;)sEkP=8Vz~;Lab~VvOV@mb0E0GMOKc2k}ShZ?`
zVM=fqQChs`H)rQJBJy-*kNr<CEIRKIVVp@IUfU%@a8Dw|yU0lyobZp`e&OAb=OAOW
z>`C&YRwU%)Dr@uYNosqD*N!EJ$k8Q1{N<NmV_$107BL&A`K!dM($EN@NXA=6YGu)|
zpPjpI<zom<Ir!{#9~`RANo01(z-&JG5fXymGQ34w#qs57gIe{;<OPhA^0!(2a!_Dv
zT!}<)&z%7KDJu~Xh{g{-vgqdhP#iBG-Kv;C0b2WzV=?xL#Ms5XgI0-}qN!=-4w{4C
zWDUn<Ejb6g&MRT&C6zDJ>WM`M*a(d=9;XIS`#Sr5m$bwpU?ck3%aTzEs5bdIjc#PD
z9>SRWW=mJ=F*3M{=2FbVd;|tHuYoRNy3xacyhgW&B&(2obick9Fk2krsgWhYU!!y*
z@cxK2IHokLtQI8eCbac~rS-`Lt>fnW+40H%*M$ZYX3?FjU_%8Gh)?PJ*&fel5bO`g
zsu!^o!fC=T<%~e)K>cGT`Rw55=c{@O$2H<dXmeoRajDZH=bEIAMVxv<4|i|R-(bQa
z(teIQqnfIs9C$NVf^qSoR7+nw@%rjwR(P|Ju$nB+F6`)4#1z0Ma*Da3{9y|+8~b~X
z@|o}9^9cSR22IZ63G=@_H~%fX{V#v2!>u64q<riFoK+t@tHVTzLRdw&bw}LA$Rj7U
zk>)SMb0xCx5Ku@4KAHa?oWVbkqmjdH+a@@Q8eZ8K376P$2p^aJm?^d&(67dEUEoT!
zTNt=%BN<4w#1i+NS{ovpTGT1i(qD|m{7WI;Cqo-2@&=FgCDkzhCt#5j%)jZKJh`cG
zqjAt9ojR8om8$pi(~=_4+~NpB2{b9a*F{2%Wm~A&Sa;(Umoj_Te0iYhEjK-@Yb1rf
z8JAzv<49r+qW8F%S)&chzReZl5|#i62TD1Vd7YuXgIQ2WQh9iWE`FbvA8aKdqJa&d
z{6~BZq6RrnGDgdvD_^uv`&KeXCU;cc9MpWXU+#aaxzMDZ*XF1tOB}_*B5orf>I2Qm
zzi``pB_Z;Afo*yNtOPguS#I&;ZiB-iO7(9npT?Kob>6ExMdq7fGP}~2UQXvJ$G#_5
zujuKYd5;ySld0eZ{J^e5j77SG7|4nr!_R~-D|iT*jJRpiGT7Pg=NWoz7QkxZfPy`h
zyQ<`gFhr(P;((&e`pf06O*+_B*3C+j&!yAt5Uj!qp49RxI~_;EVUe9ibAy|JPiX6h
z^f|j%Neu$5*4DImqPu=zgF4np){1#7MYDq#mJXDTQn<f_6Gz8o{f5}3E-MFiS~(;0
zQKz!>v)M9zf+YUCRc*_6P4(5_07sVKUO=j+yK0+JrZt`=z-KBgO%~{OcqG^HMXqq;
z=C|UL213ydEX|>(_wO`$okY>!=J%caQ@r}0Ad$%dod(!f`w=mCddYv$-+`=MsKp)O
z!Gl0<cSQ7+!==0Tne#@XaI#PX2A4=3`-sPt_9B+D4I&s}y*POO+TySzW2oL^*E1bm
ztW%|cdWGK|htn#*8OrD%+&`s*5p}<DrwEZpK<0}YW6&$mv&ZM#Wg(@bTb0rYZ#maf
ze<(`{oz8nUZw1RCq%|ysl~^NYz9zmsh|b2X@4vV55ogOs&##AbfQaR)EB@n@F`u}i
zy<C=8mEGME=|@j<oMs0ntDrhw(3#C7kJEH{*x}h2WOS>BN5A;>kw-R87TSKpg>Ooy
zQ{=ZH^>f+EA!FVJAMX_j*)*>Hqs^yW6<v|1BXvgL(6kaim@;;jkrOvHF67Fnw4P}@
zo`3nu>%8dm0S>7>CCrXAyHD}sqm#HgS!2R6uoYYcY{x6lhldlVNcj3pey;BZYq)Fx
za2{PfU13G}I7cf=XpIaki`M7W%vpIu`7-ozOfwJzmzu_?vRQ!eKQOEcx|oZArGcAM
zCC20Q4-H%<o^fldY3&FejX$mEqXVvlLe&osrP_h#?elXG;%6oy){)TehlsQVims-B
z5uAroc!?6{jf`B1$QaZ|{y?e#=Kc^eoGI0Kd`~UrAjfJbeXmM0IQ}~(S&TMa<ENna
zyqyBgdpNTIf8u%i5+&Shr4BU+so%RTnI6RvCk8nHZ!_M$-<`EWa%(A>h^fX?5j2RD
z#@NS=R)Z2Z-mTVnVuvyCVC+q0#exhKklF*p=p%w!_TFVXUZEy23AL7rJGAp$T9ivx
zUjZTenL`&)D%cEDV5Lz3G{b7pnDurlF|cTC$;St)0NqmiFlbUR9W%QBy^&nTzt0V3
z#nP1NC4yF@=2ns={|Nt3J3m#1auiX-OEN|^0~9m->G3vdIufbwXz{&u*skgoQFt^>
zkzz3Wu6i)p50!1c4de;g%7<oE9`@|WG5F^*g<FT@Q(Y5yx$9gK=Qc)>PGP-RlA_UT
z8X4}Aqvy!4AHHD1>)MkioASIb+e*w~eicTN`M~%#%0uaZPo<X{tgdhnt(_-g1IM%V
ze306+n7VW&(>~vPp3L0W_hg43GOx8cUL6NnaPgw7zF3;9Fjx2534CyTAmW>T_T_Hp
z@{jPXcT==DkE0d&PvV5C*DClX>peEzoBA@joStHst>*EpkYUu;|5&4)kNx54(WiX;
z{nJm#)O^FtM<l#Rw9AgdYx^obI|I!JS>hNpDvd+O7ckj&-<TrPwurrVi;qm#kY~$(
zB}{s4LRR>A8XyfR(Bm^J*DhN1A@6U`g@EF^e1j4kta4DV#u7LUDc%wWqTyA8{2{B^
z1`uJq1vZ)Yo!i7C7-axHa}M|eK19+2aqVP_?@gfu7&n1WMrrbAJ-Pw~1GffrrTOA_
zcsvHsW>b}zj_|SKljqU2iN*qX&NQy@I-QF-N(>O~J~*zs^XBXsl+oBF7S7brFc3-_
zh3aapTLT3yll&HFIpbgd)Fyqv>;n#RHBc8nZlp_;TwMGq^eL3u71j#B4%no}gPZ$6
zcwxDDHh<Dg$`*$&{~{LrA`hEi!Yf}HRg|*e*4jo2QG?k^ezBBz4-!@1!HlEikd#=*
z-5#O4xCgXeTih<@`YNJin$S0F9$+ar`N4sa={4X47@OZD?}EO+yk@~jU6hUF!fYkH
zQ@n{8m<WJWSRufqB}R`<JEz1~&jU5XY-RJj*$M*EO<a$gCx?)V>6eu#k;^eZ#aF&o
z=d5ED);^HE1v+sgU;ex2<WG2u68d|x*1(qJH1hOkOPUingJ(;~p(6O>?#CIiLS!M)
z2m%Pc&YdF+4rqDTeq4AV9H;bQA?kjH(605k#4|E06{pZ}I!fS)x6;_;WGX5^N7vi8
z*0lHQ9GS8l+50%ejHtw4wTj5zxq|mG(DBSV-K6hwmxaXC=m#^Zp)@;mEhbIyvZw5Y
zdo{VMVr|xo2g-#pbv4+D8;~TrOt@6{{XAAetjKA>FSe4#^3m3fE(6Ch<Y+k@%4x1<
zx3@>@!`WRgw88Mg{ZLT!v=7Pk=K<I*BzBhhbZb|Vrk90^$@pv?@PHT8c7`)9Z#j=t
zcj<oRqeYi~Y`__)2~w)A$|$<(!xsf|e{+^}ce5NM$6_DrafKVKe3)=Z&q{anki()C
zz9WOxs`gZjxu^R#(?fW~c77}p+P>Dl2F)@ZZ|H}Q;yU)9R`31@2`XQS68S>eGUeW)
zkuNDd5e34dIkaELM3T_r&brh_s4|5*i-C;hh*i9<rw^FL#lEUJt!>y%6<3M;u5Bmb
zGBU6Ub$L3M$?Q*bM~JCrRK9HQ*U<w?J(=dd`=_LY8Ag<qAL~3g=m|bOqu=HFb4=1V
zShUs^7P1?gMPuTgG@uhcQ0wfPI)b(U710d1ASoQdJ65cBq|ReX(OAzub?|iAdUZy@
z$sL(c&&_|uX8yFEl+u~V7&K$;o;pDxJida6%Zv`|h`$0!C>a9egqiOkG$NcL?!S}Z
zzm$c2jZ%eP{tEx*7xjs|zvO7dHbUEOCFU(`T)^3mi>Wx=dPT2RZ@Tvz^DoAT3Wvwx
zQ*y8%e!mR<;1X}5f<5oeu3X*Aq>N_ovX4KDiD@4bi*|XSL}N<!9TnV$Z0MDkDE%hm
zv5bfWq?X7-PzA+)Q+7{}Tu>2y4#1Z>ij)rCRFb25)P@?;N3J#56Hi>A1H;-8eHGn!
z9Lk|vPDh!0rz^$~uVTC2K2ZR-`<b2gv>4YsB)ZAytmG_yH6!Z0Y<u47s_HI`lvCfj
z#dr5)PdN5Q?CRzVzNqW(x{9k!H=nytFSjOrE0VA*<)G2xUa(I2tEb~!WYA^~BTYw1
zapxLS{~kx%$_$BfQT=kWt11FAJSvBnw7Wgb2A#2N86_9r=RX|F!7keQ?`b_e`+`0}
zLT9(G6z*(H6)SPj&(kFZWi^-^&5}itJ1c(qj!Wl}2DfAoPVukB`Y5gQ!(W62xS!C*
zD1LrTEHHOY3*55t0s~C<7vg+X;RyvgH%Yx-mj2%PBObmfD{!EXK_KyJa)poK3K}kz
zH5!$J<UIzZghS=vs;7Lx1+QNJ04QP&q-L&q)YPXKFP<ZBK1Mw(QoW?RH!<U601V0Y
zU<apd^ODlk=Bg7+!(05xlb5)m+*$#dZHmQ2;n1=@AUpN}Z03uF%Qb<D4Usx8WQi{c
zyG`NgY~Td`fDR2M0f^K>`Qr@kPW1c!))S$i+@<yq{hj%yY%U3;*99jQim!>Fv?8sH
zX#ZQl=I!t!X@sm4vDj)%H-tG<zIz)z;WL88CFzai?2cG@I{G?KzQ7psbJ_YSV}zXw
zhxuqcmPEQU$HvMx@5+eJSHlDL5%e9zOqlHV*Ojd}1R?QgDO<m?=$?L}RKFUYrlZ5S
z4>78K_#QED?AbED_x`VlUNu>K|4>8j2q%gu2#kv*LcjEJykUQ10NTU&#M@AvoP+wD
z)hNf6uG>tmMl`#eie;RVPmbAL{FxVO)dSkIx`7Vp*R7vvVW|#3`YvS;1jqoNgF4wH
z<F+}T8|Pv@HbQ|C+an(?5$4DvkxcFr@L7VhAlG&@@9lQe-2CJhj{?aou<=&pizmce
zxdn3Gsl;4pKA+{WRG(DmcoQSq{H1Mz)By5-QTEnhRi<70KPey$(zrKikRmOcPDMH;
z6;xUp>F#cjE~P;_M7q0M=>{ojgzvR+-kJG5?=$lrzdv=3nPc`H*S*%c&h<G#eoCDB
z8?P<y9E3*1t#)FZV1QmT3g^#x&IY_W>JA?HlE$>*O8!1S2g{9=64?zx0pYOb*-Fr+
zRfX{wgU)W#7;(7pC%M#jTFlKAoD~tZ?OwKE(+GPQNiEpoE-GDo0O5V|iEdM=oUF!_
z8VSEM>Jgz^3*nc2+;hQd@t}UQ_%TtOs-WO?mV4gZ^rU#o8-vi;dF~Dc?%UphYh2qH
z+{yj=mE5PmA~f69N^&u6WUkz}F<oKO>YlWnI+2=3zs~uzWZ$QL_Y5*;JE4uOI7wk%
zhkp(2+CrZS*sTBx5p`u<9GJ#8*kj|1g*%Tq9qh=!KXMVccdP!nU;h2%uprS(?C~%a
zn5)|J#LvI<J#@txv-vTgTWa-P4mw2274qsn*O6!k2~?vRc{x0+olJC8ZVlgoE~G7v
z^pZ2%>XYkwFbN%9UeE(siZRdiG7Jrfx6j8XTpl6)8T$*=VOBkm=VbX3V_csXvd=is
zT+Y&#XlbP6Gebkp_C*aKHbsUTygwtkp%?ZWj+Dvs`0@|ge^+%&KGI!x0Rg>Q+p>-5
zjm(b+*vc+{?%oRR;NOq}M<MJd|AbKfjVGw@hkwQO)$N(zdiybIC?ffSS0bFFl`Jg*
z8HRA=MZUBb@kkcRg6PocA#?|?0pZJeE^f(bP#E55@FA7<;fUlOy`Oi*N0%uR)2CV_
zuar!E?ahQ7P@=8i2n|(BfY5NB%qefWw{L(&fh&eJwZR^@svv9Xe|}0B*UtXQ_A#1V
z)ZR}@yA4f!5+<Fyqqi+R{lXul-rz4LFN@)x$eqO?)=)KixM9H#Ebs8u=pQyP)g%2L
zy?PXr{bth@PhsxP{Dq<YPhJSTX4&)?3Lt>x%r|`ytV8NQ7vD_7y^nyuCiKM4v<8N%
z_a>Vj%8V4}?tr&AmMw0zwA@4bq-IY=z{)={Jk+S~RrPU(_^5#1*KA@B@u1-cm}qp{
zu!Q?#74~Phs(NwnQp+wG3AC?E?|Iyb?2ifsd8<#B)&tETKT}#*$gU?nKW=sL$?oBq
zu8eHLDMg9boJ$%(@O?C3rhVP;YsWl!w&YBgK-c59Q>}3V?`DIzV`8uEg!!L29q{MW
zRw%6%9U|I7{K?Ni#-2*XqCRZ!Q!PNHy3W(qBe~{=7FQ!JzkdEJP2#{mQ0~z^MYJb1
z{sUtr`<>i53VTZtX-jCjulU)7Gdasq#>M?dS_3CC)X^QEr1aCij7Gk`gR*`4K!RDo
z{Q<RLiNEvUN=x;@AkC?fz*KamIO&$ab{t^`vwE{rrbyB}HC=c1*bQZ9i4znj$;>)+
zljDU;nv;Kn)^TzEzGX5YbpvrhLKj@vJ7+UhVb}6nCC16j+LitH<aFP}Y4HvSocecv
z_;0~)e}b4kx(fy1!-g3(T164=;XvD-;wgyKAxjZkAR8(T@f7-#c4x9(a^iCBV{)iX
zXSDFa;|occn>FmM^vM{dq!iVLXoh`03vs3Se8eP9_dL5pdsh&jnN$=F<v?f>uUa_d
zj1!l4|2HL~d~#V@mo4}E<4Gs1i<MJ0%-HoKZfj3Tp^v@4b3e;S`s4mq6;Yu6zJU@R
z6H&zOLIXuIrn4a~`C`uaS7Y3lNx8>Y1xIRcR4uLk5l0DrX5&54G%*l%JJc*JK#7D=
z3*kW-Zex`?yWOhi=hmo85ul4VYADBx(mK?Us_N25og!RI+($H<DVT&7!cJ1N9)#(^
z9D?yJK|H8ns;pF!JdWjdvSiTHF0yGReIw`T`>JV+sqKu}?oz^{9iidK$@WEN3J?u%
zwzlC&5(>3)8z}tEBxcgN4idj_kD}cD{Kf_I*lEGJ=C6s|XNBKnQf1%wLCoZns(1Nb
zc-<np4py&y9Z{%NQdg=ZYu*!ESP0NjyyKVO+JXd1K1vg>K`#?DG|i?OuP!oJ#y`Cg
z-TMA9;ONIVo#<k6ZvOWAVfGsrwSSh&El{|i@Y9);AUysq#MPl=N<a*|!UBQkkVMXd
zi>MmyK=OX5sx#uoB0-kJURx2b;R0)Tm@WfB5VoM0H6wQ=M~daj9XSt5R16PW{Cnf8
zoe?a)eVw1Mjo1VAWzP%=%hAn#)D8>OAm9wum0~FOd3{~vt{nOrnbO~}9=nk4*`Z!M
zuYjmp<9hSFcOMb`wEgQf6>_0T8;}6Y?LrlRi#z^h7iZkOv1q({-_f%v3!``{`u#mq
zyz^wcsh;y6VDRF?aoVqGKDk^qr#vwOL)Wq~mi)3F9cuY{Wz)8@M`3q$-zw^#9ju{^
zpCsTXfY#7sMi^r=G90e(EGZn)J0y!~i6nrazewMBH!Un}tBFJw(WR~H-eXD9z@?hA
z;n{<sa{!iReHE%~b+MtY!!Hv%kxrgT%y8iC9{%OP*o!Fz4WINBJ&gyiem-h?sKU}p
zAp&_wLMa2S9tP+NxXHE*+zz9X9sQ!HVLGC$;GFA_c1UJuWwb{RI54{5W*s>6Ie+}y
zv$7AKmCRy5kWg4^AAf&=SgC#OajT0~4SbM;I@@=v%)Sj?S-x-J!JjDqNx@~bB)7@_
z=j;C0f94NWdmeoh9P0Qm98Am<qLi40A>n$UXCsQ8qZ@Ov2q1EKGrsV@;UC~SBl94`
z$T5v6GuvXp|2W_ultdr87ga0(QaWM&+1^8`=+9|stjU}u?rAIBA2pJ;t4&78@5iD!
z-A=I5Q^EqHVdr4l;8&S#p^qez5Tdo21rjiI7x7J{M)rP^+ikR3i1#EI&Ib{~9zW=o
zxcNwTuhJj-{AI^%ln?`B$$H=%0%XFz>e!D@9!Afz?Rrfe57pI-eJqnB?yUl^Mxos-
z8UtSZ<F_4HfyrUc9je7cg2?P;jQ{1g|9M>!Nj(VY?F(){YDAqUk+MLyKlRalb{>u`
zXd`xs*ftm_D#XZFvlkP>#Vu%^El2JpsK197tyHt8q$Qu(sryT#0_aWZ(OexO(P-L1
znKC;EBRq)&PrcauiBDTn(iPE`CP_s}TYMk&Uk1vpiAWkMeHs5m+V@oYhw8!x@AhyN
zMEkS7tvn;xBNfST*0@ncopsoCbN_l|O<@3+w!S61$Fu6q{y032Ra4JA%#Bm0YS=Yj
zb_h^i^cNGaKj%ix^;^{{a--Z6tkkFauH0%Rj;U5r{paueFW12IrnqP0NMoagQ7&3n
z4z#dF%wtkr+F>N4xgwZ#bEegLoni+w3gpB%t6+FrVmNflG5!#)y><E7gumA-H0-ua
zxSGI2HDQMqi=CAo-H&4eRQq(QxKvL&#1lm(ZW4w}k;o{P&=W<XMI9<$XtX^HL-@!_
z4MF5NdqoR%N#Q6GUq3+yv)dy^4J3j$OgAGjvIhTpo4?#)Io!W#aR#k{<WcwPj1L71
za_ZRD2Nx<;&_Jx^OkyEU0PvTg4Z>RgSz8PVhSx8u-ey7f5s;F8jgYV+B<4BzlTZmD
zFymhBp^}p5y4QE!6%JW4g6irS5lLc+2+-|FXU8<?8PPBD=ji6EcA2as@fejE$M~TJ
z1VpkZ+9BpBw?@EIYgk{(=ZZRzws0RAuv~)F8X5^ldG@9_ty7u=O<o}OOXke8jWgPd
zId>ob-mj=+O=tbvM12z<eC^I_31J$Q#`-<6EJfZRhn%6Q<i!)&y-B9e7g_)1-u-c3
zP4PSkXzeK>#C+^98v8~@yj{?_o?9T&tsg|OW?@$&1>;L%8X!bq|8Td`lO!2HfI{H$
zENtYoATl(J%^AeA{CpA+NkU#APLM5Lif+RUN#$rDzS}!>D~m7_W{enw{mJ?vnw?nv
zfdZ)M<ce@z&Yos}^pqTU1$ggeFLV9*-vV^TzkbKzzwE$6lc{N`g_ph1Xk@k0c<i=j
zjh;jO!&y~iq+MS|4f4I>5~`N|1mlsn#SJWF&6~Raj70z+FV5es#kQM-J*Z!9qFR<W
zAHMfsyHtrmE^=Qy?Ym>QkJhS*{z<A+URuBsmxH@xS=j1`L(WH1T;Q^k*3-|bE4w`w
z#WsQ?^l8N_t8@_6qU#KgYT?iM;QzTP-o2~Ot86tU%!)CkzCD_;FEfzpbc34pcy(ce
z3)F1wK-wVIymSBGk7@uMS8l5&^^|;9$y%>&&=bp4Qq|r-P~`q0if{XuF&?SgHO_e5
zi#EZjtEE8PoqkDT)uoNUaT5R65}k*P5B$XT^>G=%!gYH^FM^_lpt6kj!i#O6pcNl4
z7n$!8VbOk>V)WT1>ePPUSU^kYO?Jy?+DSZ(eUd!|ERfxDCCsc9T`(vmQzXuS>mo(T
zq%6P`j%`RwPMGuR0df?~DCzt(+)KcLLs!16j@+xDDXBY*Mb8iYUhqR*`OJQRhVaul
zSd9;hD{cLI$MGO_qE~J6FJX%U_cz5yC3p+W>fUCTcI312o20t+H$b(yyc^@UE;M%E
z$z=3&U<#IrO4J$TPpqeYpa}fQ_Sc*Lzkh$o&F=?_rG`D|MTA|SffN_r25&-L1XlzR
z#&?L8u%7{c#JEW7i*UC$vc1H)+!vN}Kx<{M;=xK#+G;GS6Yk@|-6P`QKGq1hmm6cf
z*t%GOhnS{?Bs%}#xr^C@3c{Y2cyt=m2i#RcnKHI3;|cz4_Xzx~MkG0Ojn@KWe|*_4
z?WqQXY03N7iYe$fn#9W4`B~myk8GOj?<P{Yt)d7e42}{cb~!I8RIl$P=Q1@coGvd@
zEu2wKcscE~JLx^4oA>%qRt3clI@o-5Fr!n6di@x))avMo(@r4~mrH+w)X&qIN%7+=
zriRLoNO!OQc8&kzTxk3xeACP{7a8~&w8ee#Lu;hf^XVkay{qQB<c<4B?n~CyL3&JR
zs#W%3T2Ln5XuKckw!g*!BlO8(RG9kqBwe6F1N_Z^uU6jqW3@7y;<^4{TY<2o|9x2#
z8l7GeQzKWKo7k62<t-t9IS~62l^QF#zZo>o){X%|Onyv!^C1h?71OQ#>q9n5@w<7+
z3nzZIw?3E~#QAP^a@FP04B)LFG<=<|*fn(8b@S&mFL}c?q@)l1+=eOGbzZbAnP%<R
zeq44~S&4jN`ayewW<Xb)f8bs1iIVNte`{MOQEqhh+Yt8VmYax}Usj>)Zcm>rf>Uy-
zHLV4^Go-^e=NVe<;)wCx)`t+NC9$Eyov_eB0<Z9}``i#s;f!TeRQi5ACiR?yoHn1f
zPo{T>d#c?>NV%qr#Jkar%T2V-I%zGatoKCly`cK&Pp)YEAN7;s+G&Is$!;Q@{Jaf8
z^m$Bjb1iyxrF8$6>NAS@$F;Bk*CHn&z4T*SEVEhXp1vq&V4z!UZ~T*~@6JcNG0Amm
zs2<SNsV>>;W7Y9;jBAd3rnp<49etoa#tJ<7=Nf;q^G2R9ocz(dC$tAjn}>qbHu|Zt
zy4`H|hEeLgqm@1SMR7Fti9N}A(Iyv&8U=;o9>_YQS`sab0>ibW<hpc+EKa~;r`qQS
z)q_Hd_9zHZJ)c5`;`no75C{T9(;gCo5^3L*)D2M*s=z21!4pq-aLZ0_Pt<_Ld2nno
z|5ABx9UmQ{(?pA)@H@^s`Y?ap<gY2U$GOOFuNl$M`91qbEi6vk2463J;N90e?Lu-|
z-S^9r-HyfbunygQc?~O78c*eNs?c^ka=z&BUfB<fHEksNx1&PF7k*TNqe9kR)d%g5
zg&lB%IYYzjQP%g6!lRQsWb9Vb<dQBu6ZR>GohSF3d>eVG(2`J09Y0gooUIg}G1eLH
zs}PPiR|HCv2lZ#DOLdD;OWB=0kaf{h=P9H+iU?7q3ymiA)Gi6e<S^}~6_MP`C#@J&
z{80j?YM~#v%^|KVZvx$Kl=M%mX{<AU6)2lTe;kkM+_#!C*D7ab-Z||+?`wXs3*o=4
zn)&_HNv=x2+M41D&YAv@G}#GPn(G0Bm_Jiu{<lsu2}6&jYSMozq-SeCPvq~7h%1Z|
zFstf}w0-(fZoQpnpL9K_z(d9pXU@H3G%jh8G;Jx9r>)bkzTGq7semR|HMKa3KlYh2
zcY6kBiH7sp4y6nKsSz?BGXlcXU!-fbL>k_t9vQQND-1+Bc;^Jpdj^6;4R^H~xGp*R
zNlVZVsk`T%tbapl(g*bm4a=Jgak$0q!A#sB$&bH23`hO|#R_&@7;-4Q=7$Q`Dl6CO
z(b~G|PxX?&xlQz`+_#hVZOrn!(9Dz|x@P(nH`iNnk)oayQuDW9lm9sQ0CLsgW_Y}S
z=tH*f0v|%;H26V~5|P27k=>m{sD~8S86kq*lf338=tYHz66tbX9|XPBcL(*iu$Jv*
z(o0zT$EESM%E)8YcMAtK9OI8pKW#G~HeO5I5-%Wbi6^Gx<03A`9_hGGWJaFo922F=
zTj6k-6-7dgCN~Z5{@pZCpw9SxeUvbnmJvQY7zD0vWCZ`d0!t)Aaeu!8aHKE%3S@yR
zP_oqTG<nR26CA~?UDugnzx_Rr@C&`@%L9GBX=2?g%$>%lxiSan;jZPspv5RX`T}k$
z15~EB-Ue^%#ai7-sUuOf5F!{n$;G)`53|^SFmZ}V^HwTb#ySt$J5;1<ci`{jUQ}<o
z9LZWjp6yd5|ER``?pdgg|7SeC3VCK<%JrJ1Q{#yBa2AB;hbH=`4@fAQ<pjEeSTl(E
z!a6B(L!q9mOGpZI_$z6<C}9ZAYQi`AMsaighSOhPZV$abK850?^&sYmg0u9MWb(@p
z5M(!Ky882EJH!p9YpYlF8a)doOgq$ZLbYhTp^Tm6-^B~I-{M83ARjEhnN|<YzmeA6
z+TG$uaRVj6Q(7~4FpCJKJ~^&&yF#Nc>tQ&dwrlIh|1mEgyb9*25bj$I$E~?d?72mw
zfsQ%0+L<YC%pOczY7#-C{-|NxJ?t_dPFoCDk1YEQmRpPpb*ky<9>S;X)^Nfm+%Mx|
zsciVLc6-z?cL}oJ6@5Ev2J5%Kz1@Ex3{|8ssqJlkOn;j;8you6a9vcW@szW(dpO5(
z$(|F_HIgLtuEC*TQf5(=10DePLF8xuIm@W=?Tt_&icDG7QpN#G+Oa;y^+Q<b)dhE%
zfBo>$z3ViIrhBelr%2u6j42+~s1F~aB=2)@^WwF!rHY;HcNA4**|pz}wDI&rA~fL2
z7N=juk|NFtdCa2Q#^#G_w;>#og=rRTDMjYakf`Ayo&Rf)0m@vGz@0M`R)bg3=lMge
zTuiF_;`a~!UYuF{{ck@Q?T;2Ffy2o7g3=<#Y)I((^&^u8m29lnQWT7;Og-n)aZE+Z
zF8VV&=QEApPC}Maoi8Jpbxs4Abt~z9R()1Ao7Ol<aVdBu`fH?l+l!Zd8_EicS+S-U
zL{e(sj>q11UgoQG#J)DQ{?{Wg>Gu)XDC%Yl8Xe|_=L8e(x|iuA#?NcF7qR>cgcGks
zZr}Ybycs5gtG~QT;*p@X-y72|-2nw&Q2JO(pmXeqJvCnY`aOsT?ghP*%toBZavQw<
zPZ^<8F|6O{>JM62B7OBx63(0G&+)t-HAqnheiUEMJQ{pQmU2@a#)Il`X-V<Z@8a;y
zInsIQ--<&W@J8IIIN*!F^O-g@pR<xNmbIb;_gmCiHcD8pdPwQ(eGxzc_S-3|$h-X_
zWNBiy&el)jc-Cyj;DM>2jpC;-|9Tdl-JFG=-CKkAm~e5|E-#0%;dVt0d}c$qSy8|U
zs}33B+{%q^-0RW6VE2gShi7}6+e_Cr0xxKD^>|BCZc)WzHfOk~Nt+HMwWYQ<55&W}
z$mg>>v#Qx6akq0Of(+CWB<c_YS|mg_n@dDO!3fUi#8|YZID5oAt=9d{BFbDrA{do@
z@c(rPx_rZc^l6$JiH5=I$Ey_ql0fo0_RM_D(q!gM*slWHM7s~nYG;>+17^)ztW&*4
zN>+Y#sc0TOZqt3f(Ag)U1?n~ZcKoZ~U^?&qnRniZKRF7og+$fLh4zHR>gNlfI6}`r
zzzD@nzDOX3ak>78u0eXxNt)Jy5{!3Wfu$XP`?OIsEe6+aRV|e`Li>tn9H~AsjU|!~
z*hizjNSR9AzAb>x)mB2C(!PL?Al%m|cVyFt$BDuKY;X7xUc_!k6)7(Rxz0;Du^m&j
z-+wzAayVRN`|*E167@o6qd6rNMl<~hoaV1&5;?0l^VMcH#~86i&?oAumBWO3h_Sar
z=Z-3wMQ7?|L`}XwzrMo`?-cBY!wi(6+y97$f~$UsSa7uf3xgu{E(9MoPtl0z4tH3E
zAE1j>y5CO|A!rTA7s8@R)Np+z&g2;ib+O9S2dah=lYnFSJ)+V^K_b0q_KJxdLS>Sn
zX8*C(k`g>2k#&=@(h&nK3N^v_jE2kgYzIypM6m)gD?=Qc6%@KcIVv<I36KJ!3r_(N
zAI!axLJ(xm0&l9K`r`EBzbhesG>yMrZGPDF{ZscxDb_jk!4}xz*WAm!lF?~v!~S0d
zz@MTf_?0)B2+$OJ_X=)@<-N1$$GtB!>yg%8b{*XtVTG~hmzc(*f%2m<Vb5O|K^IX_
zY<~~qbN&FS3lCav2akS&s20jptNvh8Q^gOF;o94nukg#4UlIPCLSP1oClsIlu-D)(
zc-dQttiRL}NVp$g>d^U=NhHaN(2IpSR=~PbmeO^3@-b}Yl!*EN>Hr))MxyR`G>+dT
zpF&nUE*TKGZYE~Em=(}D{lNe0;PICY^VG>FS1POj_R7l>+?;xj2)+k30Jn^R2lY5~
zm4cQ};^A`)v+>;aKgB#~^6fXz-XnQMn=|yO-r0X*_B4xavO+*3WXRZT@o`o?7}AL>
z*7|c5>9*v8w!<`1fJSJr!rshV#t`|`Na?iOk9f$w-d10U>8e*ww|;WadLj@v<2Gce
z&QlVVYUNzzrr4WMvp6hp3ffyrSDKNvJQy=ko5@%S;u;WfR{dk@SStDwo>_DZJhir>
z8R~b7wja<B!Nb|2hw~Rr{_SkHKzT-T^8jN@_5Eo__??8|&QgR>zsNKgct5;7V=3%W
z5nDy@Q#F=L`{9b5uwp4HxECRW4M?&6Aoo*UelV_I!(y5NmuqT7T=<g|cglHs8JQ4)
z{B)fu&_Lvu@&Ch&AiB3+2`}jrK}oOwXGu@a`tmfT^aH!WG{8qo>kE$+WV~2<Yxm7(
zqGM<ZI#8K<b#{NuzIF=#0hY&3)!cWm<j6PUxK#)=Fp$#GcKZTYL3V&`t^_zqNn#6x
zc73ra$mhKHosP@FIwGmxpmZEd1+Y#}g1NrwPzE;P2aO8jH^^u><-oe9naE|fxP57w
z%xMz)^=vh<@-4PXPhOnf65snDrf@`5k@gpjo>=B3XG~(I&$&w3(_oCB{9bQ$8`x04
zmuM}K=sLeL|9YhQvE@{6=>?;l8Oo*oUB!o7DFB2z-I<;0YCQ5uCV#oM1LD+r@|0em
z^R8><p<|Zf^UKwCfp3!O=ODV%2+rxlT5?(fyMwG8Z4yTl@Ed8W9M50#SKI$zB<Q`E
z8>PYz^`(Iin4Qo6PgaNW)bFegQ}PyB(@nonR)KO5r9%x3VfW2R?MF87s54@!Z=EmZ
zjbT3$HK?PaR?rr+oX#N@OO+WspkiVgE5R_eF(VZ`cu=){icYFB)99PJIzst##VZuI
zguuqMHw?%eY1ZP>1K@bj2dycULVE_>@Md~NVhI2erK!KR4UXa22o1kx8piD7>v+|)
za8SYk<7e6(f=u&#w?EnRkl3}Fm-?{A#G?`H^u5J{<wb#OIRuB4Zu$n5ivHjM<Mu*8
z*>X56&;<Eg0D~dL)wL$X=}9Qd)6^Lyr&^?C1c%MgTZwI|nL1xz{W9B|ZyJllrplcK
z0GW+w1;6=6uRFsTjyJIe_)n5Tpn?JvEL>NbbLX*U0O<0fJ|TK2Q*vgyqW1A%!Q?5(
zR~^wPcQ4Z@6M6)YrX~!$I`#`bG3OMU%TCyV0dupH=7z<bAv<LQ0`1p_ctAAwjt%>3
z-u9pC4mU@sP%bUpGX4s&m~3GoFgDtq`mPQF9k%7I(~o?Axf@bN@eaa~J^0eR@V7U!
z1zHzb1w`c8`@L-X<e*J)lmz3k$8U;D=s^h~Q$n^_fw+P%-`&@Y60Ff?6kLDsDs%W&
zVrwd!DGZP8b@qT(f+TMV_2!bC7WqK@co*oTnn&0#KOTZvA%6G+eM?WtjXHam^PeSf
zs^rxQzuW%xNb1!n+`sn(^SP=bxKB;UxE1WU{Om>NDGhUGCttOWEcm7(^ApKD03Uev
z6cR39gS<#4m^qAm3WBGmbR1TQYR05zauhP9e#q0KF{tc)?I3?*w`~|gao8}mC)h4@
zP<?%MVfJITCT8_b2v`MmmiW;qm&L1sME!CsPbFemIPqD@KHpAx6dd99;u$&~)a7S{
zS<Oo9c%|7)uPl8uoP7BL`cXYn2spP0I08f7dv;XDe8&YhC(?v#S#u;FF#AM0JN_yV
z|G(b-KmL@Kg!e!JWQyQ}<DGH7KsO8L+yV*D5+_T);}+Ci-4fkDCOOu*99`g)rZrzK
zXt}*Bj?KnuA>Vnq@sq0lPfTLwjQK{qG;zHAwg<R>`Y+i2!+)_*&X9cnt^rGcP5Thh
z3~XJeN%TRC<rYfVu;V_@{M8Dp8(A?{Je$5m!E(v#uE=4H*Xki473l~4!!1;^vOcck
z9dkFnnHEgex|im4`x4{V{qO6ew3iUi76Fwj>~%nIy3`6VwFjsrhWlQE9iwcQ%l!^E
zcuW->%{gB1qHHpo0PQf_s{3K~Yq8s%6adYe>2l@JwuV=faQ5SUL!!#~{KYv~mU`Aw
z7=L;Tn|!inEybi9eVLFUM5-DIm_d$}CQ|Gd1MI(rU_BJ}VwC#sa-xmBVm9Y??IP`V
zkBB0bfTI(e?Tu*jh3ABnqN($<z0-@i>!NZ8O?1JU(;Or~22JpnREG7?qY)7{)RYWh
zDl&%vv5j8-$Le1q#(%9p-gw;f;I{@p`T035@fOTGL(^q!Em6fT8~t<}59i}HPXif+
zWZOq^jp@R2PEqtv{fmWI7Gu<&Fo!vNk&7RGa=Gm6(7+@>+M;PK3Fk-=$O!qkl$+|V
zfODyY_jxIkwBi&Y!~3$8=0F&vGj)pw&1avzR>Il*@E-TZxQUy_pbLLZzS_af(J`=U
z20h?kj94zQ*%*%0$N_h1bLGKB8>Q2B$v7~tRfjoGsF#~g4a*At`e~X9sITP!ns#rC
zB;Wb^Vs0SMW?4sA<Oh5K<rrK(Q}Vo*^4Q1=7k7}*k@n|V-Ohr9qowHSbA&Fd&o}eg
zBSlK~A9^5LbpiSi9%D$>VfKY)wM9Dk*UhFZo)N@znI-Tabka_#n?>s_DS5U7aJN}^
z^uvk|T!#<21nBL9fy1;%Q&SuTyA%3SH|Gxg;<MTm=Qa?+6gjm1k`4d2>;=>Y4=YKy
z^u0mEY=K~pcZ$3Y4?Ss=fJXXEr@hUv)WBzpmel7lftW6m7#5X_`@I)p5p!9|5Ld^P
zd&i`Yit%)lK|`z-kGS(z`DdFZ8s$f+dRa^G(35p{AqYAJ5_b+;!z%UtEC1CoSb`UE
zqVophy)n&OPGQ=1Q$Qhz*i@`X2LaX`x?gm@I*Lb1f{(N5=4kHm?)BBK>NC2qT0EEr
zmx&SFi>g}m;>DwNYzmHYF6QaUMWP`6&}q6!b+vLsX3RKdEJ2&I63w#mZ>5FI+BN0l
zCdIr<Y8_r%a_|(>g61~ns#(Qc5fnanaPp$c!|5XsG_jpvQlwCo6Pw4Yh3h5n<6-hZ
z(P;8>da~{K3o=CSlJaNmV3QsG;RvF}gY1GAhGRBo-V*SkR7&};FAt9aRaZi@WZxjk
zQlc)<=k-}pak?WKUAXc)KvCabds;&00gUnVhIc4{&SSQunaQ%8@RL6#UsOxutD~*6
zRmb{-*dA5N_1ecVjf0gbbuJe!jY`w0_Jroczb5$q^&``$h7KRTTZm(dh5memsf|mC
zw6RE%<&3IV(jur%0XaqP_TgW_!;{ptJL2@SzZHmE^nw$A!vez5ueF@P%7VgG$eL3@
z$Xsgoee3IRwT;3$Oyl{p{=v6o(n})(B}hcq96G$HVz<GO?oGMgYHbz-%vZq=V`J+k
zt22CVL-axE`dg$+bv}4Xw!4^L^Ap1FI9cp^ly858tVVJP$oOR|J#ps4eO!|n9xm2#
z)OLIQXfvwJRi$rb5~ijHCZ-bx#?`l03zszBy1`~#&(~6ywbgbDt16Ump0u6YZJ6NE
zhO3yUP4Sh@l?d`b=~gXVxMn{4ILGx%WIGq`Ni`V_hnf?KT;7)!D5**9uSiU-dl)Ie
z5nBmjzF5w_vo7hZIX~_wvfn$gywX468^^hODq}JfjU+)->O5xkWw}U~)V0QHSx0hO
znD4IXUCsBs$Mvt<2of)wUVpLM=B(5^3VcSL4S_Y^IwmJId;irU?(4yKL57euyU^*2
z2NdN?9z)Jcgxu3vQ|4>6(c*%Ip~9$vpx7-Mu7xw${UDBp^c2Yes?MmhhVSoN2}-_>
zeNf%BZ9N@k6uZOovJqhhtN`i98Dna-03sunS}RrA+rCmXh&|Qj`C~x;0O{vdwIv^K
zcYquQBHPY~T1%dAX8bXLPKaykDk6E9Dxbm^NcF4YPvLP5oe{J#TzY7}k*b9nZJRWj
zrcKxN6FsN<OCitbrAkmj{&*Tw>jiY0GCyxV+?rEZ@Y1sngMWf=@s@}O5tF^b^t{#j
zu;{x3hznRcA+<b1hwRK$RW*5`Nh;ns^Y>Y9I$6#DR>Tdi+9J5uXmVZt0_DFR>i_eX
zjcQ20o9DEC99+M^q#e;wc}z5SQRMwQhDshzRBkqlh5-u^EG)taXu8`8Ua`mm2b@^A
zC6jxi9935O(o1A&vd-nim&VM%Cm8=ckyf&nl#4czE35B8DDO(9%pgs9&DOJv<>9WJ
z)fkyHW5{-aWaQ?CbFeuXguy=e=r+VNAEtRTO5)IM01bd-q$CK_-3HZVPz;QVs4LU@
z3(Ij(l&eC(U^wo_O>{`h39MO`+(xlJ+|X-&Z|?nNl?$FfdCg<zum!tekGURBwT<fu
z-K&vL{fP-vayYLeRr@sCT&oIeUlhxJ%@A-Ka(3EnIe9Gj5bAvH!M_)FG1fONc)dPu
zFH~cHMspE1bKT+!u`rw{)#gfJ5NtSHJN}Ngg)2erehEDLdM-~+o_=H42v@#3`93$f
zToc94CzddRmI}GR6Yht)=y1j63)R(JHB%H=X14F+*?=u5+Kqb?#gylMlve)2^+MH=
z>i|;MA1h;vn`a;|?$x4$PPyJmC<lABTXz`BX*N9y|7;aBw|B=d?zB^w7ud*P%n8n2
zd_O4_H7I(&kU+8=tt1;j$M6&G6<_Y!(k1kV;tAp9`TBPXU(30jD`}0Ol4ymm`20jX
z6MYP_=e*oKv-S)2;iby4-r~s7$`=}Merv-NzUyEz_H9sf1Rh7HH68KD@gFG{q$$pH
zW3;trqlCbVe^6z7^Yk}`{~1?``_lR2ftZ=NT7gi(+05M4+aMA@2GK63v`?_M)=n<N
z<^N#;n7`Yu;1-ai1iN(+lhm5^cE{UOcGtg%1kJ56AmJ&eFT$lIXmEGM8J0}D2NG{K
z>YR(SX8w8TccZ&g-89Ld!713=i)#;OoX5#(#n@<~+$pDg*NFBK%gr7c_7nfeXn*Ci
z#1qTXfL{FPFPsxt$5t|rBK(qm(*IQI+R4^;IO2Gr<7XVKVc3>C>PWR*REHGYc1VSU
z3`L|sYbJ44<g4Y9Wnr+Qnl^m)voQ{BNb5AcOm^j7*!lXTL(cpVBc&F9<wLgx$f`}Q
z0m{v2!0}ckWCWZX%#kpjoMf<5M(1D=|Ap<%R;A|5o9rt&Iy^&2C*BCs2PDNNE)N)@
z2U3^h9*;$%uwDPFwFjf0P>@;=qf*K~g(OZ#q#t<ax%Fy28gaGTYIx)NKpCUvhvydi
zj3+&&>0&U(@sHW}g-<i@Roc$<S4!_{7hI@*(_aZMcWmzytfSA5ji8@&FMa)nN>NKY
zMvB_Aqg17bMj_2MDA1T*tLI?6ph{0XlEnkTYncEn9&f%W*+kc;d@jJZF(tLUPWg!6
zK3nv84e3RbEeRPfz7)kolAGDA;4eY`$K5(fmo&yiWaUd#7gvMUnaR(fYA_pdqEst3
zO50Hdi3)OW6Y1)DvB*&2&Uq)q(pv{031b>GG<S|bzb--Ji{nSl0B-^<3xvIHwR7H6
z{vrfIH;h0g$;;6*B#$hRuT&zZ5{8ONES9VVU}C-PogI#yezQPUD5-69SW~HU8_t6S
zTYn{3#y@y;E8SDnLaaVq1$2Rc5UDqYVs*fN2}(Ci<LE#3_0n=!wW<esw4gknGeoeQ
zlu@Ii>Gs|jn&#9ZOd&R7s;fz!J^<yJp`q)l=WF4kTdS)Hd7D|2r_uB2y!I~f(BUS~
zR#Iq&tW?q%X4x&Ck^KVgGwExZPX4i{Z2O56PRC*uIOwY9_3qwnP7%B8-$k9<=P!sG
zzySqyJx#aWP~1O^POAIya<bA)vsMAvI!<Ok5y07`9iK#I*Zy4(>ZZ7{KM@Zm!5xeq
z79Rd22vDIc+--7_R0N9V7^YT#K?FGuG<bp&Bm|W+bVyIydcn`4I>UK6m&Y{Me`I<u
zDm2W-nG-628W4Hs&d-l!_wJ!3X87==w{=0DPkrK)Wx}<5XxJSfBUHk>*a;GLNV)og
zLf95jsIL4qZ2Ejy_tCksjO-oHh?wz%O0RCzgy`d*PTfswZOmd4w}mU{^*&UB$UTI<
zfp^EP(Bz_sjeqPO%hwCVL;*JqF72{*4{ytbqmoqVy)@6Zn7IaqFRClw<F4Mix$GCe
zU_yxyOyTba#Uq*7wM&QF)CIIYH3T|%jllO_y)B(LE7Gmz?D~y<ZmIaG<&pStX0hJ#
zK}18<stf}TaFH_iP7jPvZw};Lsa*L{3!;Q*;WyD%%pLa@9<OhONsM%%rI`Bd?ic2O
z!NXTDgm$%W#m!9SouZ&4#&EhGSi0_UH2=nzP_4S6c5?ZwC03Q+<J5Afg5<4!Ze+cH
zV9ko|p$>l3d0eUM@YDNkC!8kUvxr^D9zPM&k<y<a5wV8yETV4#d8=I5j-1cVbo8wG
zh_TkwM_`N!E|<iMq7dMG%l#1%NkpZd!_dh&p%rzv-TE5}%55>Vs;t*|3?UYHE`XFL
z0ma!5h|7u*uCDblrlqldBuFF#QdQ-ZFRw0k&9yAOdXxFMb>zHcKSo5*O1_Pey8>)@
ziZS<`aQDwY09J!{b1VX7aXt<tT%ZkrB%-WU$AAIJ!OmoF(PKD=M9f_a^dQhC+V?m`
z86Y%V$Bj1C83j9J=s-g>(gF0SO4gr92z9qj)R^o{;VAacp?CS(e+hM0gUv%M5rrZl
zdDGDxg}OYSl5$T!=YSgu%WZb4q5<9O^NG4$k9Dm9-QN%~tb%Ooe;MIhpwwreM?Yxt
z;`_FM)QImc+SpRlAaz%hj_Z<CvK1<i#a!>N7mcYqWoUiK{{H+?5$PqFmtR^<G^Su1
zDrey3Sh0$<ytcQvQ96^D!(au^^eBs9NZG>(4pJ{e55a9R1`w<j!sg9p3WBwYPi<Ii
zytyh&%mRm!%9mGhyi+{p_aC4^SyqC6n-<QCsofSXw{)+cKyF*BV|CHadFYq$O;yd+
z+bvYF1RC)q)$Ju<n(ha-#SFjmpSM)b8MM5-x-aHdByqMMa;^L@w*S=^oC<uuVE^Ma
zkvCyO`bGWD$D3c`e(iNa+XUzW4vPdPs6O0b@Ir$a8$cZ2VtrZLeLv&Es-|p{Ba@=#
z*<rg>RFOUB_yn^sbbdeE`s%e=WdR5g`Jz+JxO>EE)^eINfB!5l@tXe2r&}W36(m|D
zj4_W~+q@d8)V{lk7nS(fBR3(RSuFS+U%xwF%OSZ~=`{~6zlW`IntMD2aXd=TK)<Iv
z0ncM!@}~TuSm=>Q0$=g9qwfZV6$J(ssA6BL`gm@lYXwAfgG7f!6{PvEJttj&gh;Br
zJ_njx33_+sVdP_lOo=vXoJ{JJobz_DOGT28pQV;k2QswU7j{AWrH?hnLKQ|8&O)V2
zrR*`kiAzTx6;3!T&bA*ee}23}VQuY0NQdri@`lQXI{a~tNC}9knz8J-OJeqr?4$Ks
zf5|W!5oz_0TUO|3`+1#s$n}?zPCnSHdP^;9=w3OENxcIVu=SUzxBaf`T%}`~p7yDl
zUWQYPE^>19O4F09g47Z|!K`=ynEFV->+eGem_hjYclhvqhkZU3%NeuYbG6oQ0C}<G
zbok!8ta-CS9C;zjk=%Jff@?|ErK!j?W@Go<N(6375WR?>_W$C%L6n@|x)DNns^jq?
zR|g-;Tp;$yd#HG>h++N6g6>=^<npl^m$%x9prg;~@x(jZs=Z;8h5=*t3CLf@TVU(K
z`GY79D^XTN(cD-QI_P=%c=>4ISy5rACQASbWqK<@hDu+b@|BXm)q`R#>3p6Q{Is9+
z3DEgvzYyo24#U;N>2tKe&Ocskh#?hS^4p)cPh*va*5%a@@48i3wS8ak?CDq5%Lds~
z)|sNMW47IRZb+G+;IB0XYg7OE{f6x8)Q$5<K|z<G5QoggL0*~Rd?nV4m997mLbc+{
zgCgnDS5i3KS~mKsF346L$k+Cb?Y=M{JbvJR+dqho+YZCMX48arfBxn-rQoEdRb@HF
z-%$AiIuNK<Fnf@ByMU(v_6s+AYHjOuUsL63=Zj$3-7ji*xbN&2Z#h{t6Mh&e0h`lv
z-%t2TuJ<Fub|fd%lu;sDQ0Hm~kGHCHw?%?IsFa{qTWwWi4`qIwW6D_lCP!G@V!k>O
zKnXK<Jxr_w30-2Ejvu?h78rZ8gyT0+8!TNOb-za$jB2m5T{d&Gx4}q4Rz{&=-V<~2
zl|V6>3TL>DjL|HdgH^24<d@IQ^D{dSy;*n<z;f@by~Qr@Nm3rbdnjzZ;5MeBChjT8
zwm*-^i-hly2DqYJ&Zm2ZnccWCa(G2uJgf2g88~cMo?X7)dG`U9N>?=*QTJHX$9MYj
zIwRh`lSqrL478D&J+VYFuJO7tTIwCXS(ElsEZ{GTk9Ow-31m6c>Q9o|hhw)KaCqhs
zw#M?+=rcM&dY=JU4wutD+-z$%XV@OuRr*f2K_u+J9ZL!-X057LTUbrAQ|t+g2>l%?
ziA0f`BR3JrEcKsRjc_^Xfg8%R==l;FD#jE9s>2@J^9|N1Y9*>gRDWEd9xp*>#P&9l
zG&xG<$eE5fb&a;JIA^KpI9W!ww&7>wx5W2L$0ivZLPB|QrVBJyUJ2`EWCcZQBQV^x
zToxh~UG8~H?l#roJD+&YPU32+gMd#9JjV0V5a1RnOp@^`lsxU3DgmR2MWsqz8-ImF
z<S(V)meoubl=0^miuW20lW)@qxx8a5)UoW3F_;M)ySyxBqM)%C>NgzDd5SepW%rHb
z+_FMI(u<k=HCifWEYs(p;_IRFrnF?oJ#>m%3Y3;35tdFW(6i#3%eq<r65`Zw!Gq;c
z9sJ7gFX2oTR-OH9Gn1F7&9&&3F%lUgy?vR_SoEQb#I{>FTOtx&Kb(*1sZ@3Q%JgAz
zPf_(*!^IWr_DKlOY|bm%FQ9#+IrHI6<}`k-;~bM8e%x(c97q=<qPO?;YJY1d=HyDj
zP<>Sn)k`+{Nt?jy+frUy7md{XDilRuk2Nl~%K`^T!l`*UB;k8}av)G_bcEID2zx3U
zz7v}pDc8?`GOJ$U!6rqbJU4Fv?cy(o@LFYYn(J7i;*H&=dKXN}Msps>vHxw?m0Pu<
zvD21ai_hyb6di)T`=*l)kfm4{1o-LJ$jOTE(+@Dl9^f$~$#0-|6j-dM`$@R$FSbS4
z6Yz05V~EL8a2U$3s;pBxmF#PcWHQ>Rqd1KcL8dae((VxAK3xMrJOet`U*A#07Y*a7
z?d6D|A#aTF#8BpM+@WYFDUmByOW?88qvKa8i(y#(`L$<^PpctPF6T21K~&MR?lrYJ
zpoxJDUKiS}vwv;gJc{q#SleM~Fu0%JLcXJgS5OPF|JO#465eP|k6b-ve<M+27>|Y?
z{?u^N<UZkzqhFk*Jz$9RELzDmctKS6FmvK*blu=f=13mqNbxk2{nM<oiuITMTvW-V
zD+-vTrjL@Pc|X%0#Dr<I4?0-6toO)0VJ3n(`nGtn{cR)`-4P_k>s;h8|2ZC+O&dc&
zH{X1y+(+B`s)FtA*nQbAdSc&4!E<;=0|N;OLo8A&{aM=y4s2?=!ZeQWbx<5@=^Xza
zU*Z#U;}|xwO|SVT7>PHdu`%t1X+xMc$0gd3?QuuhPPSW7eQo#PH(ykOp4_4L!`n}f
z^PN}<Ioi6=lJ)#{`>);l1T-!qevM$0Nb-kJT&$9QS;^JR_?pxpZ<Ybh9ZSq`F302T
z586`)t%P!|CaUEP(O9MFTjay}%Kl@V^xaXFK+^tHVc2v%+}j>{mm=TKf*-^0OslIY
zy^fCFYwv57UR&U^EbK+wmD~Ct%E?QFF@krzNZSM;iHbzdA8@32nd3C#ciy|awudXI
zE>d{m)N~sjPg;8cBScb^vvMPbRm;*Ed{)Luqn_L$92ltmSqotx;Rgzr(K$9asH|P^
zh+fcL&iaJQYcAX-%X6RtxS%pPS)($2sCe=;B;eNzgX9>Mo}r^>72mYZU7rWY*X$di
z;udKw8h}90-@TB~%-UbBy8otucSC>oLU_seDL(^=Pf^j6Je?i8wHtAvDjV*MX*~8w
zlWkBubLw~+YEkC1y(Znm^eD8hu@M*%gy?LK7WWKqC2xIfkr88)#)vY?xMf*H5{10}
zbkOe`jgve({wpsw#Hd6xgFZ&+iYm+{<e^$Zwk0V^{3!#}?#jDHlgFNPjR^MV7SBe5
z|HGOkfCTI-(#aW!$yUo|dV!xOA1uIXehwd}uF$$cJFL2}xUctOw;f{`A(vm7H7l6R
zamYw+5wz9$j6vgnOgl~3zj>P4?{X)y!p-3aYe|OWTd8OnnZ5OykHx=E0tCYtPhW4`
zKW^D#=!A4dK0c~8r!<h<RkGA>pPT#jMf=RKZH&swa6Icnh`??FR>$?(%o{GtRk1IN
z1%i{Rg_VKtNP-3Ch>O(=!X1sOI$e7w1iq2@TsM>}TIGq#8zFcoH+jG!HTjWeP8zO%
z6+CGJiB4|;h*e<qnFwkqpnx_+hV%oftfAIPB;!Ley5#E+?(d)BYxCA?d3breQW7EL
zL@MZN%8o+Ar&=_P5T|YYVpx;;*})1KZ+^TzL#mKt7ZF2>1!@pO>Z24@Vnpt}{6Z{{
z<ADR)Yh0J@&o8d8{r!*tM9&i)1JP2@_)mh){l(yATf68ZBLJL8I*d`%DFNfmn(;&H
zZ#`g45Orq#2A4?L81Nv>pKnvw3%LuQ>MS^Ys#MhUlyqb!sP?-4|M^s5VftH9mBL^&
zVvVTo@l77So#ep*aa5u1Qg;{8@j7Vd(o`2&JNp-8i)6Lh(&gGn4<8HKp}2WhE~b6d
zGYfnr6Wk@3;}aA~M4sDKA+?f@8{pN~>4#d1XeOqQ#EB(Ng(5{RsdE<bx3v>tmbUbu
zNor3F{ce)c9P8l)BA8HS?{7ct4#{Ks_p@%-zxJmCLrNBZCPcuwA}`#?U%W~1#U`J=
zPTtq_TJ`|qP0{6*%6P5#7SAn9L?lrcsTt-O3B}g=WHf_TG@Q{I5&AjK$EV3>eSt@F
zCkJ2c2Ifq3ON1sy?<h8TnHW{4+*&Y>W52R@96fY7Yr+>S|5`P}hod5?_-bbx@$3=a
z%OclphKsYO4gS9#LZ6rD&Ym6SOiaXyFcuqrH)#*8G?;T${pP93Ae*!4<LRIc&k2ym
zRU6rQz6THaVhGV+TAKo?G4T;%RJ0t%t=HYn*M=>&tgQuFHII*vZb%^lF6TKEGqal%
zq$l8AI!((*LUVNNl!1QuW1L2m%-R5XKWkYD`h^b2#3u`v+ltR&tps_2{&xroeOck0
z17NEEvf0i>-*Ly9bYM(=Ld#m_<1DA`aLZ+FtMwbXj!+;tuJ$LJ;LQS<;?hVrvya{c
z5M$s#<kA(ZmBG!i@J}Z9Zyu(gCQfZNTOt^kI>Ax)z2E1QN7EJi;E&TYw_9d`4lX$V
z+8TgbOAbyHv9uSm)UZE&!=dFBJ=s(v=pyEfc#4Q#SdH4cKX~7lO#_9mS)JS~gbn9y
zJhGefpwf3lf!;nZL)k*)F9(l{Y@5{vXD3bQ6S7qM0|<jAL{QwVFt9u_A01Vsj{?O;
z-@Qp<QvTgL4~fLh?m|xdpb)M;8EAxzwO)3L%gLVF#1hv85p(qD?o1Wy7Nh1uV?^Xq
zwAq~2Z!buZTH30Ek`<Z?diC@xeT-#n1`Da`C-iZ!h)BY{9(wyE>&cTH0Dx}4Q!L}s
z$AYVtJDOc?IzD`5aSX*4LCx4CuCd-8GLZXnniH5Wlag>6t$lrJTf;JTyd|<nMTUY~
zxb^<zdR!;<*fEB;@w67RsH#0=e)xm7;|c3dGViYPzJ97&vXiY{b&bVT^LA~5?zcn6
zM2?Fc2DUcR&1ZYn3jq|{T^MqA*lp<^`uhuKfY6q{YEOD11}XUvgoW_9mhZ|`d_uAF
z@}vuak?@f&q$s~xqP`k<t|(MsiVNqFvzyBIQz?rujVD-YKdOmmp^agQFf8DurtT7~
zW(RtW@59-_U>v__0a!>CoOHcv3rKnxD1;~nSVTO-M{OS1&b%90Q8W&>;%c>qz!+U)
zC2NpTL*1HfA5WU<v6K~KS*MP6&k>VWndj(AV65H}_q}5y74q+D%J*}b5oulK)PS@L
z#=%Q5GB4x8Y<yhw1K^%`eSPX}hkvy~coY-=c7G2h-%TOX&A`HS_w&NsM}8f3SFOF3
zPo2W%6xF|z7}(WRpSX4xItQn-O9X{W+MU<fweNWZ2?^-E!8E?pmZPgL?OBT+WzrNl
zp+{x-8Lxc=`xD#xBh5*yCn7LQ4Z?lB`j^s&nKuD)E5$sov~X3Q$huzI-zly~hv;*m
z0e+X^$7M$_=v6m=(F5|~LcVNqlXQvRxZwJ#6rTOCKV9(K$;&UAUdQ~ppON_Zps9L)
zX?yj)dG{*JcsFzB^cSNpMuaqo5qG7y=rd~Q(tGE4AmQYEx%5I%<w@0!srJnKLNRaB
z`Zatmm#d%H1f)d*=H<ffLV2``KfgO_epz;k6N7f*3j(^vo@kk!;K|mrJ)Q-3!tKtl
zp@G&`GChKGKbO?|Xvym@t%lR0g869*wlYjuFRikSt*;hHCb{iSB|du4Gq#NJs;xCV
z$&S?bNWQIzbM5rtIaPvl-SSeir&XOF7Mb3!^P?bDSK<SKwMKQ3Bb*{*`Hz7>B^kf>
zJU(q<5(22@fPYw(qx2hq$V-KULm|Ogdiw*+p)`boTlU*z3V^wd=c(l9tkFKgB7M||
z!@C^Lv<ikc!4m~9-OD3XhdL#NcmQg#K&@DZEoBoMPl&FqJu2=p667?`3}3T)P5lpc
zk)_OFfGZE-DOI7x1Kqm|OB8dbzB>T8WmRlN1i)Q)pV)j36YRgfdbNY6R2?W0soC~Q
zzuo#5dMl*psjyXkn7fhJE$q@0&+(kf8*Jjx1y|L%R0&Y3FSHg3=&*_g7H(%GWZm<W
zd?L$IZ*cZKMrhfSr^Qy(@!I_m9!w)B2b3Wvs|qkECKW3Jh#u7ofq|q3<(pnJCSpo2
z=vJjlhwOsOfZ)^KSe6&B8P!HF9kX8~R80;~n~!x(zh5^rn>Kr6(lXFWMBU+e2Az~V
z6hJSAUN2kxA8I!yyuy=j(wDIR*~4d#^2Fw;8)6mLB=3bJ=~}oHSZQNVp(rLj#T+II
zlSx8dyaOF&5<J{{0J)n^e~TI__YPL+Db|H~`&rTy7N!cMqWbe>bV#!v8HojX%RN4`
zHgk0^0x|vaZb^s-@&_-HKvHMWzOsHMGYFP!G{Mqs=iQkPjO|lw(g$9hTU|@}G||p$
ztmE8%Hxf%C@-OUx50QLSh}sS-F)=t_xhb_zhRS@C-k|N;NX5EuaewMdee1XF?0>X9
z$v(|HxI0)YItsdW(t$saiah;sZ05lMY8B5DxyPPV?7W_N|Lwc)j;np|iB9#q<+Uzb
z+PJVwm%1yTEd_|fuOXqI@vp4~Kj@nXXzCXxR#isfT&7^3Kjx-$kI3u~)}MME5E-d*
zZy?!pBJ^wfDsuwKVU!o=<-puk$ESWL!GOp4j7?sfl$irx+7mc)b7&AfyiAO%6@3`Z
z>c0Y%!u1u}Yikv?ucA0w((Ic_g1Hnke5nmB5a*SqXO&jFaN7*ki*>4`O|p}3?{y1&
zhia@1mdJ^4X7K&dmD8j!s6e-nxYKZbWl0GXkhyo5BrL%$^_2e$t2pPdmb{bZh0}e0
z33J`}7^j!Y_}X9XD8!lQap;rH9DD!g2!~2Wez)&}Hz{=4|LgH|zT%JBP4If80)NMg
zl}MF*)i%*%VCRt-hKoIgPA5O6E<q+j9GEIfzDV7q$3vxp%70h}4mb`y@DI&^TTQ8w
z6$#EwOu7S@hd12BP&hYH^k5+kqKB~z0Uc5p|8ab63G+S3B07;g)xFQ}b=X(ezP4W~
z=gf0P!vMPU7`zPmCh(VOkM}|S1l6%S^))N~E<ERMf5a)_gn8LdPViUx_eE~*W7F4(
zW|eH6D%1CgOzoN1dOr{bh&rn-ef5T_5$=EW$U?z+^nt@ju{C1%ebIBzaOwrl^t@1%
z=DsO|)a1V0U4x?LAp=o{Q+QfVtm;)4z1l<Fdes}+q6d!Pwd2XRrgQ8`PG>n~ycE(8
z91EO@2eNNUwFO`$!Z=edsmKLR0|H|;lLv34O=nx&;FO4e@4^+uZ}w=BW!YPho4m??
zMW^D`mdG(eJfsNs2nO5mh1RIi3hgn4Lz0lU_ioKY)1>d*ulKCJeL#&1?4`6nKkDE2
z6SG`qRKKHTNM#8_a_@RG(!SmU8GVL;h($AaCkXql0*s<vw{b{vv>O*{+Kb_U%*Qr4
z_yH4YQ*ruOHuW}yzqzUXZSH*2D`K^BXO(3T10-n|U!8lBb>mhwUgn1B{t4#G8&2Q!
z%?zUONwPSq$t+Y5b>&bd@+WDi-Xb}kVRt1#JH&_i{}Rth8Ojt5%fql@8pDTK%^ui6
zomUXQ9G>wVhQ9KCZu$N5VcuPX`MCGmdb|BDRgrSq7!bORDjC6xuJ^79D#E$ZJR3s8
z?ga`_TF#?_QxKP9|97c=4W<GYat12WhPe;b9)?+`$<pKAy;G)&(<WXZpUU`H(Gy9q
zMHaT-%EY2;i>o)%qKl3Kx4wSdqPv9)F$MfZ+9Cz2yL<BK!Xj@x{|mQK5O5oNajN$t
z?oh~aGCl&4c$DSY6W<?A{GkO-l$X5uw(-jGV^dZOH4MdP_8Alm;U}``MpP6IRV12n
z<4=EXCNL1*yQL#7g<_<F?yGblw5TJl{l%{E8_?G)?e`a->4Vx9jN-`v|2g^Dxz2?8
z^rta>MNi4isC>~J&*-e(`|TihMs5boW!cqSA@VV|hW)%x9>%yU9uVqReAScu%_foX
zm>>Ov$`a#BjJY;V0E4eX&GFdhRA3YFanXTj@k+@bUtL65{vT=I9S&!^t*;ED8xk$X
z%;-r(i|C`*kb+2v=p~5W2}W<xdxR(<QbdUsok5i761|Py8NKs+W$%5yoU`}&&L6+O
zTwE9CectC;&sz7o?|Vu9J(n<n?H@Ug^qVVyL#UswUjsgbN-m8+Cvuh*hAUQs!^UoI
z7_TR~bcM4XD825q!NmdMC$ugG{{fzm#Bj{`>Gs2&$L@nzK)s4daubPEFjv3DX7&zp
zz1r*Q2TA^0X;0#g+Z4H|l~l;k1==kH*M#P&vRL;XN9dd<R4HALmI$u@GX5)$8hJe>
zJt=kSm(`~ZuhTLypIDwtlQAqUNW&3qwtRsNk<t_!AH90rX2R9#1ZWrj91SF)rluIy
zf3qDI33DnFB=QOKY*G+z$?^hG5|$HPM2eUOlEdgf!@Zt%>s4B4ebG1Kt=XxO*dApq
zH*EK}w#ZmN_{v-jdp_ne!ea-G${YMr?5STf2J@`lHrO%0OKoOYsJEu1bl&#nj?RTv
z>$UWg_Hi}z?Clfhk>mQ$=4*h7F7;w?rz2XBuB~>EJ5lj`Ui{)kt=QgY>ZZ{`_oV{>
z0BF8xazqy32D3ItIhjIV+V(q9U3j4k2pD}Tdt7cFQM?i@nm?w}Yt1}!JPRfAS;j4H
z8E|~x#9*El%cSARO+4hC<M`VHk`(ZOe9R4aK&G{Eeqo3Uq6l{K&b^=dJQ}&JTp+qi
z$N2C)>B5i;%hQ=mZ0!kPGEUDHraOJ10Bm#{{myw+vv8olBt4;GbLluLl1S|{Vzka%
ze_C7%5Gs#?P3GzJ3gdf><hG7N`71>c=4!<LE6~c@_84h#j4%GpSYJnP_bF*L%xEUi
zz63fr85`iZR^SeZ>=8Z++?)cYC`r^MMF8wPObJt>^Y&xm+1dx}!o&lv?mjuD6sQN%
zpBW^*+`l0Fd9B*=&$FSG`E}9tzfE>N6vs&(FTT?}A%TIRw>!n0SK`=Ti+fgi^LYk)
zAGiCvPpGa6+2m_wpayeoJU!R^R_>-g#rs>;Ce)8w?$Xv&0AD_@3BS-+Nf+Ol?Hf%7
zj<{FQZfw0f*{;H11BZ+`tVP4XafL2&Nk>&J8}5AsNnU6Mx0~nDf12iMU%h!t=Sm(s
zoe#@H14`5=jy&mGw)d!rM*?ZI%!--UzZr}6%@JhrqQO4<a{6qMKSfNMSvo`7qTeiE
z%97U>INa47J$IM-931AigXGvL%Gdy3IqJ#WxhR0!Jzh{T^XmSCjov(*Fq5(FF`WzB
zGkjd*EMWdrE`UwlYbvv46n<fiU$tpEW?gn45o~RANsa0v)+*>UxZ@Kb`nujTWMlBt
zea_Y1+-yM-w%BM|pR2RaQFjDR`TC=?$#dHYzr83+TpxEbx0-kt=}lem+RyN->3p2E
zm&x@Qm1DwpD$!9BHe1`@_q)^Ftkn^bS6O_Jo(W}EBRqAKJoMh@iUv;1R%7&a5T@U#
z=TE*%r%6%2JUO8t=@?%NyyYxFTe9^ztUaDn>r<WDgbspHEiw?_t_^WHEunyRdL>~L
zNOpB88_RC%RXK;zs8sL-tlktEFL>|~(U@NOBj|{^Nm?26FOx}4y71cJKYS*Ump+q6
zfF4Dt!gYaHDE)#{Kq}<8ii8LcGQ9PR&2R#kpv+3S<;BB{0)_@F61;ko^ZKwW(tBg_
zBjHbA6qEZp<))84#%U(_bKn+g)`ia1@Cj3_%FdPmix;7wvfd?Y{Pc+E9AbN@Kf5BX
zGpTynKGLO>k10KlcAU3*i&6Lc!Oxi%i*DMx0k|Ly02W~JWmBIYl@8_lD|I}vSxcQd
zevq`7FKqDjfK!V-b-I^N^Wyl_>Fg&bsly*edjBB-xbAU@Ovohq{g#kZfH9z@7@R1a
zL?nix0iJ0V0haV_V~~k-h;TwE<!Ot64*Oc8EMQ)VbZ<kBwk{>)m=EKnGaT0m4V)zo
zt>7~GZ}}|sg0ul8wNA|1@bGJ)d+-d+QcLUY#}9Poe=TgqKhO_^A}!(KvJ|J5R9;wx
zJHk%bJLC^Hu^oG@X_uM@HNe*hVQ6sW0N^c-eWx>kUc{YWPUiqzV!cXY#p^g?l23&w
z$6PGF_&3TLq@BJyTaHy{uYYck`CEN6ALR@92<giZ!NTdhU;N8E$yngGcT&jc73BV~
zd5p{EfG|gzxK&l{_RP=u^L^~z-z!5N|3h3_)W9&ERazQK4`G!r_9gY@c_6KpUEmPe
z$8w0Kgz(FW<r~jszb;LLOU}SD^LQEgx5-}!MZs%={M(t>)E#$a1A`!WK`*|}N%t_d
zYK6;(IO7_=F<@*`svL%(w$=5$sLF;t<R_I~P9a_lbfI(X^^U!6{z^Dm1v8-NDwczR
zs9D}k-s-j6sdp^n&i236yEwLLoy-?FxCOXEO{d^5ZFf(+CKdr`7^P;v7k!P>0sK$>
z+09F$HXik7yIaqyDySucAjnHo7A2IMJnhA91=yzV1Kj-Y&GVpsg(UyDpY7i=!BoP-
zhjD*li7-$^+Q}n;{Bos1d$f@mA}S9k(B&AEa2e}ns53jFD@uFPW{?dEhNvUJ1r4C3
zJGF)A5|E-@sJH`zUO3J5v6L~iP>qx+Q{8%PVAY^>-rND;4Pj<!Nr0`<lOTr?>@@DZ
zkuYXXf4FOYn4}4dd&CytFA|COoft-dw=d{uH&mckLt;E3RyVI(_kig1aLK3qQii6O
zCGQQm*gget^y2kLl>sEMfB8Z(!f^qMqG$(ZP;^qhPRQ!hsL1vw1=_YIuo^Da@y{-d
zvnAa?(kzhrw+?Di(v>X#7%$N}GCiGa_b+8Sj>dmirsIntap&LB^hl5UCtUUKc7Kx!
z4V`)Y@ZIU5$&ikQxCy~sF&P=ZFoD#c@2H&!*+_pWG9(wy&^LjrcQY0F%G7_)f3ki-
zz>~i(m*-X5{9-awQEEZ<mk?={K_4~qjZ?rtj-<7d6?i>jc;EEBo=tK~0%?WoNy$;`
zHep5IKh)}(*FnIklJElFyO{07_kF3?FJfp5+*3?u4Lth@*NQ*bOuSDOd;!`-o(9Z@
zyOdkEQ+v)P(yje`DEUh0;=e?gr#|}y_{bmch@%sv@d773Y2>e}$;&Sj8Gnx=x|!)8
zN|4BM-Blh}U0$Bx)^#B@(EA%2J|NJtGUuJ;THy=t1NZthJ)Sw9WJBMS#UopDbMxm1
zo|`<C<8>*IL054Vq^Z>Y+ZT}`h;jfZS|ngP>!(OurXMX7$B`VM2Sp{B{IcXU_cbLi
zEmF@6aKjuowUC;$-PgxoZ!ukTnf=rR_b-#Kcrb@~%|rxDc$3|7@rJa1-y!6uU^3CZ
zW&0-FpGj!9uRp|f!O4!gZa3g98+nT%)Tzi$b2WgTs70B?=#~zVcLl8UY3JkJ?G9de
zz02y8!?ef1H51v~+&nA_%zd~C6oF!SJ4#|H)JPTuwOJa!9ZElDreGD#;Om|yAyKfn
z`{KesKEz8VR;)WLPIav?-N1Y0%MuezrRJ|R^YDQPD0+syLnSe<psZcUPMfoGdoI4S
zudZhk{ru2#%V7H;?g$<HE0^f{<!@#F=Px;ypniHZ{)J0fw=TFV$QuO>!i&Obqq0Dg
z;$+}(x7>Y`8O?|<rHf^rYWN7-#0jKqK~su>Djk>!cLZo>u>poq?}Da*$|r{o<ZbA(
zMpzu?c^qqK)A#Q(i?T&(JW2WdPkna3Dnqy-;iRN)vwfUI*boXIUt+1HB7R{VUC1z1
zba<ta9F%7?rD;!zfD(e_xQT8;P~~1nv)*g?rT51R^(zVVrAAVN#i0I5N2+<sN3RmE
zu#J@(<*!XuE4{mImSoJ{;XZ)Gh<MCcjFea@`O^#j-J@T}vt;nQqlm>&aU<NN8-yDq
zr95zWZs40>2&DO<Br8A>{GZF32;UKc#5^cF391>R)38Q75psq~wpd7SKiIXoB$`%M
z{LFih4;G6Mr552ou1IN6Xe4xy@uLI@wM+-wXO;8g@_h!^<yBh|s^g!x5kL@aax7q-
z!;+po`uMV}htD{&lM8e!w}-i(l^dRZdG!RVELu?+_7z|*C8&N*`Ey5$C{ba|vjG)r
zTDDuv%MOYVjKkA`y!V<TB$LMW(2m`^v!;tthbuk^k6)JV@y({~&(HQZCU4X~JZukF
zSNLNY|HCPvgD9JTqJs%b$QM}9_+I$-IMOY?F;<vuBIR<NhrU5<5Tyb<V?^m!0ymtm
zkR_}fyfUka&|oO~D|Y(3;MW*}Q#~0nr>cE67ZAHr1z+>_f>Dz!4mHSo^49v2Nt$qX
z9k2_Ivo;szMTiilT?p`^Z+99_*eYvV%UmpykHUbcr4sf_OOYU&SO<;Y8_21HW<}Qg
zdiKYs_A3dxeECYi%LjjU%X|ek8`%C{?0yb)trhY?vuI?~o-gdSzou%MYbHJyz{wxI
z`8T^Bc=dFbufBnb+hIY>m<QDt?2WR=Yh&X3JaOXI4&_P4ImdADQE-FC{Y(~E>T$dX
z4rS;mC2`l*Ovt?LDfr+m{4Jtw3Yn2e!fs35cwXf<x@g%VqSCg2RB5=dFD>xkcJ$uk
zY`*Uzxwm7me(rOiEPP;M5VB>iD<Ca22oY6Dh5y!HdzT-)9vT35H09Sal6dz$Glopv
zU+K2=PspE<tQ!wc1*;65vnPVFt0cz%7<lorog?B%pMq(8b5C!6uo@l{&a-_Mc6&q<
zJ8W{(<Ky4&`b82v%NWepK|{AT-QgWDr>_^tix}Jw))_{ySyY%{u1mKWWKvr9q8WUp
zh#3{jg!Y#M+`Gx)!i25`l-#Yojy7|*E9#z$ILpk;5!pe7NsLs*w`Vx|LLiVOU8EgI
z7|rMlw<D?Bl_8yWFLG&W(LOXm?Jkot?lzP)f*VO-SIYcDC}Zqli5Myhot3;l0#n0?
zoHQ{k%!G*ooh_a9&T-F$yv8)U?o9LiEZIlL{dwM<x_MYTG*XIJ{ya_eDWJKxa`FD}
z5&wUF*G+I4;Id=U*0q)Pgn|br?V=5I&KQCiJDj(=xQB5izDWiPm0x?ODbXUW!^RO}
zPH4*HYTEs2RcfM^iMr)c`e8mo8C~thh9dt5;!bGW4rEc^1`mlT$w7;r)KUe5uY9Ka
zPH74=rF5JU1nl>akIAGon}tXw@GU=mt8kIRO|1f?10X1;7tia@z0VjHw&hLiwY#Lw
zaL1ze>07D(_5=*o4gYI}e=LLa3L+B}-RwXff{7K-(*HDp<=`EOo+8`Zm44sX<)-RP
zMgn@(`Bw10(3fX4OYgk@+$?{FjNfa4=wg8Y%c=2RC63piE@soLHHxX&5bYmxy0h5B
z3v>nQCPfN`>7p!fhj32LFbMl8_+z1>sc6YSlOhK)*XAlz?=&gA6hnakdy7Qz6P;`z
zF}DzfjHCw54H-_iKdge8W{rrc*_G&98Rz|bTkmInCqOsqG70&UzZE8vnlrK;ot87y
zw!s%Vbdc=!ll0DP3Xz9o>o5s*Qc5ko28-4Caoa{&Fnpjvl2IGIqDD4??oH7A5u`*^
ziGaJnA+w3uUEb`+t^2lC3+#*sKZPCV_&&>WTs;U)`cxkkt0(@A(^(_@f4k>?ABh-6
zDxwffU#1NH4WNi?b1MdhEHX%;yp+#B5W^xPcf_2gJ4BLwWteJkV(zptG{kaaaDf}0
z5ez43=>sR@c*Nl2ZQvXc6qRfqBWTa(Sm_65k!p*lOLCZS=+!S>>p#;?!<*Zw#;@8E
zZq?u}FspS&22s(MN6gY6cY+yj+F&@Qqxpvcy1k6}@$2v%K2B@M{&cj6Fw8`kbD@F8
ztY9YBzZH~^F^VC21Z)cq!<B+Rwnf8J*~`;`51bDCr!NF6)E(#d&x9M_)j#i8p;4t~
z>xmWk_N(%ow&%Qq(xCmTiBfId&Hw%82UbK{M+p&0jZ{P9d!A!x3=1=ID0b1rzA;YH
zz#t(!50cRLOx)X9LB6WK9*9tLI5!(#R2_XllrGjxat<MkW}1D|Wxv#@5D25%Y*%P#
zRB*UT3RW(fok13KfB!jD;eqO94xOz?r8HT3gu&Mc6^|B=s=WS-pW^M;KE)vtnEZ{-
zC&Vx#yxfgHKV=#_C;<~I(mPF!e07zJU2E{8&6RW!2VK;eS&jisbU2R+=g^?dt;(H+
zZ;~ODObJ`r_4mDS#oqm~Q2(Co<jDLZ3iw&%S4O3f<`A!K%Jo)>RpPOs8o;L@U96MK
zVH}rl8okNX$0TIq%oTdpPXUz>_{XN;@H)jZJSp9)OIUmZTq|-_$ZMz!l=C@~E+|Oa
zt#!hKh|H-0?pqv9D;Z%#3I|-xQO&YT_-`fY&0(bwV9SrbW^IX}5u`zKyda>Irb##L
zdh;0Og_XIR+zrkXCB1uoI2(<P%ZU*d=6Br~PmyI1jQn#GFY@4>T03JJ{G1LrGz{m}
z)l+hg%l78qIdA!h6<q`O&sHSLh0O4e*4q`H3VK#ebsm24oLxUqJ>IR8Vt$a0yrD|h
zl~`Ntrs90>$&F{{8ev`OKf{lv5{Qd!IyedoU;X*N{(KF;4~#tzBBq51wn4=`$Bd&G
zCxi`xhf&Wl`I_R2>u8BYCCQU5Ow*r=Qd-Z^H3X;Gk}@&QrG<De$dbzF7HNyqcjL#g
zX;l!pF>51ieTC<FUI&tNw$iU)S^MxzdU5f#eT!u5BVX(|!R7*`0(Tr?-F~z}$M1j<
zf;}(RzViUHqqm-YvlUz!NffdfRlY|i!oi{x4NC+BK5r)oavH3Mksq}R)34srRs%K+
zp6?5iKjWYV4lcX<p>#xh+~b1!<$@S031|EId-OZrcs>vjdrj3~vg?S~71p88xScLu
zzo`g_`uj`M@~Yn*mCJkq;pi(T?bl5T#ao%9tMDRdZlpLzJ2?4CHRu*qL-Crk9*Uo&
z1t9TT7_|!D5V_dHM4~EVNWC4I4QG!duD!93&GKmVtaR9TmpMCQtm{Z<M{8nRGp2M)
zCa9jVCk@m{KrXfhne&Zu_gejdv|y$;xur7~M6Oht>EG&p85lyPZ$AFa224{<0GyE&
zV%U$e#+!RdVC}7HtbYYf%&ClTMkEj5quIy*GzYjcLQb3C`M27cC=u={73AF)j9C+P
z;-pu499_R~FFDe|iaO$x%pb|tAJO-p_o>JqDWaGK&)RyxM>J4OtsTXG@ExSFr(B|B
za<FKp%Y4A&mkO>ca-t~3@FL&<r67nL1?lb@FRI+4H-E5wHHP(;jG-gw%FaSelO7rI
z)uOH7OgVb61YeYJl#soiExZy38o1N1s3uCercL>MVB&}`{ktw;1TDkm9%|a1_7qSp
za#lLd#|;l_dnza?kqQw4><mtzKSzTpeYy8S)P<zz1Jg7|Fg3Ej+vy#b`hS&!eiszD
zD50ueV%VoGGE7_2Ftn0Qb<o=J2Sj{-V}$$TDIjV1yFHNBL6pPq500{UYF8k)fLDn(
zj+-nZGx0i~i97TP2H!BE24oMe_)7kk&I`>do#9X#+i4WM=58n{EbLt&=}6oA!H_}=
z?_@(;ZQ?%dFJ>xMI+aF{U#9rZHsM!yU5Gy}9M2ToiSMN=?mQ{t_?6<e-7of+4RmyY
zKtdum*goFYB<VctkmavL42-#TV+OpT7rM9z@l`k4V*IezOsQk#LMYm2zK5?TJ`p0X
z2+IOR%ctKF<od5HxPc%DM~d-%V$Tb@tA{U3{_U*)Ss*ruhDu*!b=S2+=-L%DE#OV#
z$?D@A;;?x)bUSPl8wOyMX`5rDaMJO3tV06?e8Wu?%S<ecoPbX%yMf!H+l}vBK#uyQ
zCvGeX_iCi(Amdsp72xE6SjePdW-LO)?cU6WekZ6Vz2?-|LO%|jiJ_c@M4*FdU5Q#C
zN;sD0r0{M6gJwCy0MpxAVWfMZw9ro)-)!}O=4A~5>o{0*3$W4%J_Uv`PY35#ND1=;
z$A6~xh3sF)Ujz))N~{3mDBxi_&(pu%*oTx5<!n$iD<O$JZUYrg2oBx;+1(W!USFX@
znUEN%2Kv%D2wzKNHo1)<s&ieRUI}$L(@5`5lonozX%)%V$&xx@E+bNjGE~>SPTPp=
ztrRIOc#DFz-jB`k<A-o#DEG;3x)?%*B4qdo89}qJGqyH#dzmft#D`kwyTfQJOH%SG
znB5~Lip|2+dkRV|e<~Uq8?WX3E6|k@%yp(b0@`6Lr`uBYGbXK3BdkxOprR~*ps}pA
zEaxF5{S}fZ>l6`(cZh|0>d6pl<awUC^x|J=h+k<S@ZN8rS-B;+s%ZaOqyL#G1BoMI
zo#iszN0Mo*cDnr19Uf~Xn8Z26gRBAbh6qqNI>}bPkyh5WPH9^$bBi4+2>K5QN)M7b
zScR1@d`*{f!H)t?`$k&-xI&j;L9i){)IW|Zbe@3kBP&o2h}m78tzaOs0Z|D>$hAQY
zDHi6t-kCvn5WLR1SAkI~&@)3h$>S;qL#?X<t?~CNVxlZ=Xyk;Bd+l`jlfVkrrs@r7
zxpmk8YbP3L6ZQ4s>mM~g#pNGT(a_8b*O>Rf82F6=RM@?@2URXM-mK=|IwGsP_{Zrl
zjriv(NU2r{{U57<h*2bl4VQ8}#4$A5gnMt}8JY-EZsWCqrNc?jF{FmjG~D4{&dR2b
z#K*vmsMEqs=<fyj_K=DDj3GQpb`l@h5%*wmC9}<`W*tavNXqpM7R%O1)?j~;eSX3$
z-UggiUk^T@e1-!DM);vGL~i0^dIMc0hID$oGmKT|RSHle@g$JIKDP9Ic`6VV6;<ww
znVxHpzwNT3`LxFU>*)zEMy$klT)pJ^ebs$?31OR2HmOr=0A?wvg!>;4uc*KH=&#UM
z8v4(NmlUt!_&*+=`Em;g(+AFUD)@tiGTX%wM)=-tPK;t_OQh7`3Osr_sE$1$qSQ~k
zA+1P`vj<1ALWGX^TIe+WHb3VXA0d2KRBG3hI6j22Q-LKayqP7_X$$MH<J8x(Hi6^P
z%t!gmH;B=B(U@YGdG|f#vj~}1nxz2Avya&d1kso2?;$A=<EosO=K!f#j6VVCo3Z_N
zU^46+z|@Vk87uF*>i8^6mf+K~Zw01p(x~XR@k%GNmIwysMdcexw#GoE@E$UN{42WI
z^RPDbC;|kp<P4yB{rB0<{%;s$wd-x6K(ePi=^+7xS`g1Ax!RZiutWuT^a%WiIIi2^
zwr6}Cs%?hhUUM*e+h!I@HZ`}ML!{RcCIrdm&PV4RXAQtXGc86MlYEXcA&@(VjbQJ!
z^Pjew50!@-=>3R0@y)wc;S0}^$50a<VM;FlhG@F&#;Du03ik-2!{}rQqOS?QPzGk5
z*>66ffJ@$L3ZYC^t~=z_t12CLpN+g{IR^JRRUa%g*fdvNr}!QIMwP8lf4x`WvId|c
zWy!_2qz9j@H2-TK{QI>I1OdMU8AK!%@>**3D}f;yr&tIs4-S`O%ojiYZR$2E#3y`L
zEIuRz$b$D~d3!0Qq?sIw<oJR;(2snX@EAedN2FLIvcW+9V@5$QqhfnjQBk^f<WfMh
zUr;}ow3!^MD59v=6$9qju5egu0*m%4!Me>gd3jr<2LbUN<=bWo>U(@GUW#R6Q5Kbm
zo9?Eauhn(F*>a2M6ZmIq+o%0#iDa~qN(%|+sZzc-aOElBj3W`%S5?dOFE4=iuIr=a
zJ<v8$$2n!!9+i&!WI))bsc@nGYdMr>iLfQHKtgbM<$cFrr{X`YMn4hWGz4jHEDv}p
z$>|>f{UfuTg)Zi!#hwg<=RN6>tU80QNokM`g7#rd!v{)m**ajPB&2kX7C}LG9Ezqt
z0s)J&(9#ghZ%@VtKApy&m}hDLJH<COh*nYzrwo0Z6hmua#*0GO(%9I{TZzLLWr=W{
zsL1@A`1U>I3YflD@DcLavsPp^jDyJY^8|#40=0*gXAEt3nbUg8W<!(S1%Kh&@{)S@
z9;oN{MQFkUJzfjjrQgetd^=Gz3i#lWM41COL!`tcL0~ixFO{vWza*Ov7a0{>4e=Ca
zjFS9syXEq-B<)EmQ2+bOa`Uo)j(w9SaUW}8u`p~{AopPrj}nJ2jE3!Dg=>XBsq`e;
zAv?eT$Kh;Q?i}ZIx^;X5+1swp_qj{14)>To1R=E}16K0$Fbaumq~DLfJSqM`PDT-n
z#?rkFvV0qpnT+^vJFhK4npvZ)WRWSnMU_Wc_@D^6UUQV@sxM-a-PsU&)Hh>{JYjQ>
zkeUK~P*a8JP<_!WOhEo~f3WY?-pW8cU<~j}+VaIr^Y?aOioh+-2Wc^YZbYM?a^ad)
zEB<HCd+ErL)eq&3o{#*2S%djUe`L<O(m>{%fN|hKKC4pvYY+X0+Y(a`RL@jE^=$IC
z(q5hvgv6m{En0l&a2GnqWDn)<MUxXngpfDhWa?$ue6lJHot>8J#?#S_2%(S(rzhH;
zCW#2)zGbvs<PSB6+$vIAFe0V9HE5P?_?4d19vTqoTFPcdKQJoV?bRk{FTK!kC5Zme
zhzyVEoGsK4YEogS)WP2!wxeP%-{GN$1~}!={$ev#r##80WiJx*YtX2VHu*YbIgax`
z2AM=#qT#%p5059|{QUg=pC5ApwdaAep@30a40Bd))Nphvy$UvzUgiEDIp88LP<I06
z(mR;;U(f!-)Bn+F0q{x72uxqwxMNzSG`3-~uHzw}-k4H&mt{ok9ltKCWKCQqI>$_Q
zJM_mSCuty~0QcUA!0*AS!FL2gKeCs<-mYnU=mUkgqZ)mPml0_|6S+K;2Vj3$&PHZE
ztX4n6VF?S!<fjGTgosx1ri_LxN+VUNI1xV1HpU`WLR2q5omXZ_5bsVlAg+|diWG9W
zU*&po&$fD92w0EH)acf;@(8^#MI|LwV0V}S;t!Z*lMCQp7p$50r0WgjWVNuK<fdPR
z<7VxFN}bmc@+UAw+<##ks-gC~lF)!?-hcmBaj<ynA4}u3f8$rsUkUcVqYl^c%xV4!
zGc$I%H0vU?q>DuFvk*Ev;)_tgha14YMK0|OMSb1SCDtgE;+{K&t>o}DJ|id%e-4Ke
zS4h7|25;wQ#?DENtovLba|MUJgGrW&x^`%XybNT0*9dwNif-)F(1nlo#x+k<%Nd&A
zY9YR6-6<y<vf4pk{<PDO|52jxX%hD{jN6m<_fv)Zf6&DM0<9mJG65Q9C?KmY)R*8r
z2&G~*1w1SDr2vya$5tjcWj{Ou(}^mV{b8CX3%-xg{si*{)f>g<)oz<fz+^FYgqtB2
z;2<XegFK%}A-%S~)8So|F#jX(kbbr(LPu4x5x1J7JvnpRb%VDUm>uG@MeE%4&pX)&
z@%ug|!oTD4SCaalVlj^)s2_o5?M5;wbrI;}-lfMd#~`SN;~wIOpN@Ap+dw_Ymg2wQ
zG7^X#64E<)$HqhPm#{`q6Cj&tjaa$Mmn!Ve5d;bqIARkDR&X*d$13cR?Ruc2g>l{y
z;P_Uc=vy7QESN)?uXlE7L*M&zg1kw*FwrM6`k{=i4iU_>{zr)2aKV>Ua@g1Ql7QLZ
z;FlX(x(-3L4oEU4G0pwWsm;ACU8gQV$N5Apoo9S;;Ln~I|6BSs+Lx+KmxJl5)q;4S
zR(H<@B<^>0b<>6(-VeGwXv+5!<a#2ry|CjM-2M-jxT^Ui3Jb@Tbp~xL?NF=f`P-en
zQFI&-^Q{7_UzHjK1uj7eRTUMc?bld6-8SPBI|kblwgk-DPv<{X#gQ=xD9?_q{FooO
zsq!^0h~4|xx!8M^=V_H;&^@SxLb>&*Ys-<-V%LqKDGjBYPb#&y={l{5y%}fEV-=i|
ze^kJIlAA)LlkV%XMDjkW%lTECm8<JzY1y7A9s-s0k{`?;Mfh}UBkH45sho{FpO#EM
zmT9HFc9A%RO6LJnLbroiBI_%RElVbE>(>d~Q0hFpcWC)4Lp5h>=vO|6$M*boM~b%j
z&to6cjm{uclkmhquefoS9YWb0ouSHRyQf$mt2{elI!~4)xcWH0m_{XK$N*Nk^e)`=
zq7J{<yW_z~DJtCSXiHhtb@w$R_17p-U!#eZA!U65^LzJ7G2YJ2HG|*IOD1aeliNd$
zq?C|7Db(`Eb;L=jmoc1*jH&E*$Mlb|_E%!~`-jqq%h2Tz@O0`D`}Lyr#H34!;C-CK
zO~*aNx8M&Z^}>l0h<H&6q6bUiaGRi+ec{JNaw3#oDRg2y_SBdVTK+VARA*$;IH;w8
zPK@_(M-l*e4EyIuj4SEqF#Z-H<iahCig3vSY_3r@uM;p<V-k7#k(C!DYG3I+%-n|S
z1Av4Lbd8n#;DUP#H!X#-ThE3-lqFGD#k&EgNcP(v+on0Cj_YMZDu8e1NFuXbuyPUa
z37{aUkdHNFi*E2pz)Z`aWNwK3_yB-q#U4A)J<g97&3e)$=zg7j_?}u@y&&P(@%Wp&
zYW2n2g>*nx%Zk-r1anuie(jt>9mhu;*%})+QH=;FnkDGNU6YYV0e1TtNDr0CrR{<*
zx>X!Y$e?hV7_F&QYvFGJqK4VEKbs1VV_9|Um1nK=e9B5uGm_PxHW%L4Sx>7hkL*df
z*3zc#*ksr&@5O7>omvD&zy(K~!tb1aqJt%U)-qU_S5@(t>o%WnGm44wrndCD;GF`{
zsAB#JhG!6GSnaB)R7Q`_#^#XJBhFzbh$V8->b#Aqe`!!`jnF0og;f}-l$ie<FmbW;
zwjaBo6IQoS5lX%))X~9ok|R<HIszmz_K|-1)JQz4zPi6n_cMvoh%ckFh4o-nw{A$6
z0t;a%6^72R2d;~=GzX#|C^1Siv0%mm-V4oSD;5)JYm6mjCGVL$+9Z6tC2iUi44NIG
z-0q}C@!~Oo?O7qL+C;{cYCMFZnc%3V<nTa>egd4?(;EImJ~~?>)UJ{Kq3m#DRQMdZ
zRtOwaN6#6`7RCoYwtR&#8jNQt2Cys7PLlY?pEd8U4CZ$fcwL+wxb92{h^T0=MFX>k
zJLZ!OY+3Q{!tTGh6MNh2hx^EGC@tkHQWZ46L}C4T)g`P9lmxs0K{pX_vD8<i&nZid
zjEqVYezqTH-8W*|nPD=Qx@_>Fj6Gd=bs6hr_324Tv76M%{u-S%86CH}vhiE;!`LQN
zEIcH7?^HuJ?8jAhJMtBt%6o#OZ2%viYwUc4+4J6BOq|&(k``u!$L`9Y!MT3vkNqRH
zUz#WGl~Yb}%=jpm<5Qt8W2Pu(1GeufH2Xpglp9iPZ($w-PaYfI8daWM6|8&$v-XKj
z-d>6|DSmF2mOm3)ucqHa;rJu3r*?KPOJ1cn^p2#4zhchTq8J*V=1x^jpUh3&Cxf2)
zGQMHYO4Os?2-*1|H8Sn>Pp;jw^l#<bbd954IK!1Oi}4Y%jkkH`sw^pBPPJ2~Kmb)K
z8iKXPjVT`2iT<nsioh#kVt>zvxQ^#{3G*q*mDG_h$Ub$T7B4cz5;chWVn}s0w|!%B
zrpqyq20Bz|6Mbvsio;dI0Nw>ZUgu-x?sb#2G_dtYL)5V$nS*uYN|y|Lj^r664LS|*
zASR*P_OUe#Q{8GcZ^PNpcsowyq3@+;ECDo$m3G#GpU@}iF<Qh1izeW+R3dYf#el_!
zUP-nnW9cspG_ehe?z1hCT?@F(d|WEHBc+z{fP%lVL$k)+p%Tz9)}A+qT`?OeHq)Pa
z{O;Ym9J7w3T%<>Y-~}MR_h)^k`^libPV<YlIuIA&GKo<ptzPf@0Xs`rdxli~qL^WW
z@6|k&8TR|~K&}JpizAXYABh;+w<@gV0RCi2AzX8N_?z9-Q~jU^ABfWQKt-YxWs6rP
zbDF>x2ot3T+s6}=FIDWXf<bfP`(0H({La9&Tgr(cV&hg(uRmJ;L|l+QtYKg6A{1p4
z<0#o>Ixd#rA@tUqX*eyYIok@CWg&>-UAFfT7IhduVEDBAoqg`RB$Gs3>*+}N`^}Dk
zar-bf$y(+_uiM3i-%{$VV?X`sW3STyN8wbSpbu}}f!7P%%RLk8NS$K0u%0YZ$h{C7
zV<gS(Og~SaXkOIL+2WDGuN=xBZ}=rMW}tY^@!`QVYuH1UNH=2G6vbJXWk5ON^xg$G
zOkCtE9#~m0J2;y3n$qkF%_`_IQa9|5WRYX5l6MHZHY<^eyZ7vCW;$^<ruM!uH<YsD
zd^@v&_jbS+{W>0*SNnRfxRs&DWv(@For!&CybNWs*c!XIH^~{1{*P@`Z$4@LtH$^v
zb^fbci)obmU52Na;~1203e)ZHe?6e2zy%o;#gk&j+OT^@P4s7xF9?LT88ZVo8+0wc
zK8;4s(W)uImgwD_Siyk~*la$jQ=>&4UGbJiYB&EPJSOvTDBb*P3B*_F$?Gl49U>8E
zu<MFqS>~1^KZORN^!rF|v_i*;Oox{RRfk99i!(mDK%3{;7v@T(hRi`EG}KP=WWTP)
z-%GjOPAPSYneT{Y4Pm$GEdngq8H9PJ2_FQLG2{S*CXFIPnN=~%el}opNuzy6^#N~J
z*R{F3Q&v)9Vmg5L`h4tT3K;Tw0GN`3?0%W#?;7+4B;+lS)Wns?DFHR=7rp91*if;L
z6pAqKI^H5D9`EkPbu+GnQsHmTXcCVdj|&stqOkGJ60O?Qi}gjz?Gf<Q%QNM~Xr<lQ
zdnw+J#qgpLG8>Gxg`8bF!h5+gG-R*={}%+c;DaI)L<sU?n6cRMjJAhP#Z6A_hTEG=
z$L;33mJ_Af-}+#dHL4h#%DJtD9?}K-`Rq+k)ddUe?e9;XE}dE+lXzrIfYfDs!wO!V
zb$2DUr5Y6JXXnEqz<H4}msTJL>@Rfod?eHOFXm-7i#0(Sb20k0rbOkLpYK)8+a$be
zO#HCv4W6vr?dB$vJ8k!S%26ZH(5vlYz%|7tDEu3jy^-Ur`?poM8pv1r8k>7_iW!sG
z*_z=`oN=5|3{H#{K(ci%g=Rkx7sY3@$_$FcW2)5i8FqKjw<i<fo#jtveE2pMN*|J`
zaLduY^J5lqeBc7pW{?O^x^Sra5`B~LgaVp^kv1$zvQ!|$GamaMzQ$(B7o7RkPWKC?
zG2(Sb4y5H>P;bbpOzL-Cg|1lE*>AQJBpExhpEW-jN4MCwQ*i5)#-xdQo1JWB6av*Z
zO$^`lk1UQmUBaqgjlX}n!S4t@!yg0Wi$BVS^jA~xqwTf`HSPjY#<S<soy1&ne89w3
zMLSLjaBn0~0=sNZ>h<Hbv)+_Tz=+uc7BFt{y);T0P$>^x3b=ThJzprhNpJ-9TYjB=
zCW`tu#0#T7?)67Tui%keae_{Rp_{CdV<}~?@A1bmdlQbhT|eDp72xZECW?-1nd?vZ
zh?C5qGdCwCXF0G|XX2^Z4_>*Gey6$`J%03MC|Vx<l}`jl=x@R?I({BuP!*xhV1^m8
zd)ZoQ*qa|W{p5x;O+d2v{#LjWK{U+!%n7x(rOa?JyE(|)f9O{4e&2E7hpiKVcpc$@
zxHO)_cL_jgdU=1vQr}^Tlu9Osxs_JjwXLO-w}!Ztt1K!RIAtt^?$#VRbz}vE5aeXd
zkxYssGJVHM(4M55UvogsiCzRdZQbkDz{qdSCDrS(b#~Oxcfb7xu}&TO(*KXN2J{=3
zxR(03K!8rDbc=wy&VoP*7-Un5Au#+&_CGj%LbT@yFFp#Owr-0g<$s;=l^)1WW|y3p
z3cAZ0qphSV3TXCYnh(fblnYP~{{MJ=>2@H>7!lfA+J?40oC~l^<ZkUpny8cMLT5^O
z^!-pq+X@-`FNAEmj^G)<EdnL;9`ZxWz?)s%b<I$C;Ms3*cOZ+x6E7xs(tVcv<X9Vk
zI-FMryP|ckx;8@a(`%OR-iZ6~aZRFBOaEm5G2;Mco6zxOz)jiu-j{W_C`w+j>sVc}
zhk@yi)mzoown(i>)7IcqG3_K9T@EwS#P|~{;be^?RZm6~=Y#Z^5iau%unCy0EnZlt
z5b1T+q=B>}ZjWO~yS>ARX?7T?a?3lt<C$=}xp{H)J%L9}XH>}6+{RmRFZSau!+c+h
z&3!KiQID0J-SaJ*r&qrzt1R%c_e>r~nD)fU?Zd`dH!rs9^x$|h&F)+2usBI3)ag+N
zw4~e4Jn)GiUSLG7mT=<B#FeqMaZIzA^mc>9=~u7Z(<+SC#oaW0ijq3dLMx7ejwf^w
zT||S`&w~56G0C{aBc+Q06$AW5DdQXI-KcLadI2lj*q;X{BfTD5?cYo%K9d8PbXEUc
z_p$5gul7g#?r}EHsL#pre;!`59IXl48?*b&?Y!CBvC>n2l#@`X*|HLTar|s@W#dA}
z>-f9UdzC%abrHB4^K90$jAFfSVv0p1B%3}*!OaV8Eox)$8kL4z3zeW5+@4Puu#X?|
z9+(`sZUHR^x5>19cZRz2B@xswQ<?YF$~mA&<ylsC3;An4JXer}sGl9;V%M=tOp-%c
z<?fNl!$S#;vX|^9FXycYA6wd=c8(^lnsvHeG3%=_%gQaCyAq>@03W0t?AM-e+6b$c
z4}ZbySKR7XVB=6NJrKv+*&Oqfesb$6w*lq3db!<%WfP0{RHYO6=h28VF$~?Sp2~uq
zVvVUaWN3d=DBl|d9WH;ce_nEP>Gie${gzhd{s&W*p#rV@MH{CMcmJpzA14~e3wPMT
zj}8YdatxJ&`_l@YE5V$%VXr%0pU%Wyk?ag|AX<Bk&*)&Wj4!@m1m_NPJwEwjgk+I%
zWV1KkbrA3i4<w)Qm&F*GKy8WWTu+I7)g`cc6M6!n*Ss`v@IW2_vL$t*8_oZe@wn9N
z_Df4A`5ypV^i;L4=}4DG6MJ2&APwk|2~Tp%D=6Fr9CFxA6sHKeh!6lIm;hj-W8|TL
z-M=z$oi}p08RBVfNE1+!IHG{+Vl6!}=@O3+;j#~rxZC$mfx8|5vFYWI4K9xg8q>xH
z^s)_{>)^2$RES4NF)Y>cOC>GUZX#MStMBms3-gQFI~X^E`~{%aoq8_T#^^!zbavy7
z8(5W@(bYPpz`%Knu{!ruaN8ogTqcax`PAeHe%pQ=oK%DRY(V4j`)p=g>6aJ7&(bTF
ziJu)cUh8Zc&9}P6Khu>|Yb{7V>b5a;;~{E0aGkbIUHYZeuMjyDcN=r9V%@Si>0nsY
zt0YwZ)SmJUw3lJ{7>pC}L6Yjv@|xc6|Bi|J3`c`f(;HlJnZwdn`&YuAYkcSv+!tEt
z(qTK(KWeLH*E~2hJWSmuA{pPgD~D-|K_9G!GmbtjZfUE|s9$+JJY2u&i|l(>J;l#X
z5I&bIEVZ^^bInH*D}LnCsVUww2Cu(<L^@w~oGEG07IC+v_DJRw2N&f3nHjkNIs@CG
z`e7wAFqyuU@(=UP4qi!HcXyOF_&^C_<+Cj&k?W~0W&6rAT)I!5RPFE9x*jm}1DL(T
z`^5%ZjS6i)&`wWAt@d#pvTan)zoj^c%S`!Vx_CW<39m*Dz)FCheakpE#b&TpyKNAg
zy;!{hq%T;U<>x&G-bRn}F;raOeR#r<_+ETIq4<Gd%&$m!>Ykx)VmXl)E^q7;etDr=
z!jo>@dTU^n;Tw8&soM11ad}|<Oxmeea?;7p3tJ<K>OT1DaEO7MuC?8j@wOVQR?(62
z$T*m!STT**I7vF7n3}8Eq)rWRMG0y6gQ>R4==2gEI4*W^E<cDz9BhuUhEQ(V$@>`W
z8W)=;>Q?TsLzNqgJxtP$j$TPU)x7G5A=0eO4xbawpb2PCGW%(E-uBI({L_6P0qV!W
zQ(q|YjJ>9H=;KMVP&Jd=85hs2NkA%)U-chQRtVK^)VX)OlhA3aL)>eck^)DVuYt5-
z{*txaz(ID(7c62++W?nkeC9>m$-9%;MCsiu7eZcF05_R#BCV?^;$MhUyf;K*{CcPK
zMk%38I~5PvS8aNb(5>cgkLFi*o?9xxXKNhxBX7vmq$bZAvO#d>8q(Tu--_Z=4~0O1
zP7kk44!L&$XUh^mP+`v3!d};}shI1a8%^%GOrH+}OmT||*keNd$oHq4LOQbKLe_$f
zrW=FIfNIW}^3!)J=(lml@Qlr+zHGDg(K1x|uMPVvasWXk(s_G_?uJHl91_6q*}0y4
zDPp6n+1F47j6IK;RtXaM5v>oP@dMAEq>|9HvpXqX-P*bvJuMO8O;cU&LJUJFYTndu
zAN^`sW7<SLQtTO?d6S4O2BV&(Ot9>#<t82+(p4?g5m9g6hI({_=kYnQ_6H+=`|O#4
z7wAed=BrE3llBbO`pkaiT(Xq4ldA&)PptKs+crmI*UsJEjHP{4+jfs{tvwy-+~QHV
zFL*`yBZw06sGxdIj4?no*;e~z_z-Ur(@D8%y0u46=~3a&R|;ICBv4+To#H(wc+tir
zWeeP@=%m4xLD=pKRLoU*MOSCYQ&O#KHkJ}r-V@H~Q?UHDvt|4J;~!&Yv%}O-16hYw
z)QQI%T}Ka5bPNC72fLPzgN*2}$KPpGJ9cW*cZr~go4aer=}fA2M9iBhCim2saf<CX
zG?ve1r1NIc`Tp3(E46I4Wt--rH|c$K<JcKd6Ja$+UVu5)!2V>g%LgU>G9_iQ^Q=N9
zx4f4PIQ?zU;I)QBf#iph5`-#?L*9kl5S*eLk*v2^!U}z6UA7xc+{TWCTh3T}r1pac
zwt6wSI{n2HdgT{AEjIouY~*$YW+xR&<6O<`7TQhAV2H;4+9c<NKw!AKL9}WDmI^qv
zy_nNOQNLLkSKBdmDGE5J&pB`1XIk%J#cP|?$k$hT=5Zh0e_rybbo;2mj(|ps`U>@(
zaR+2QV*Iq$J&~OqqfZ?_*J9q4q#((s1eS44$-bBV602}>&2Df(=uvF$7enX(4{eto
z$4}H5YoBME?dOi&lT*P>7jo)>+={N-KHb!+FK+6*i_z6MGWIh7UZEyt>0l%e#{A-J
z^nU-D!;}ubL&eJI&29IgC+Q4I6<_pWoTTTv##|m4i(FN**c&$=B`9G-cfDutyAlRn
z;;qy8#I00Gw;t*96ljk?(zB6sNjzy>TuI!!$9^gvvj*q=H~kjaPG%Loxs{^W&g<^p
zMtiJ464|SDM{@#dtMM0hMqW)xNj==WvpiO{mh|Dh+WRUQrVB3r)}#H?G`oW<yWiin
zT!8!<wH_y}JZ!z#c-YO-;^KanVtqI<O)c^-vd4dQ6A%NxJNhjPinE<S|K&@SpZ9rq
zr^1G>IyW7T_we0J1VHrfWeB0gvqkeeF$X(f12QR@yzqoGmS3__c7`|{I#M`NS;)b+
z(tdtyr(?TcV92Zu(wUonXOCZ$-IF>`o*kVEQY-zc0K*IXi{o$V)I=#tMJki=-%0+M
zn!v`zMyku+cI%fwRzTGr%WvYv7!uk<2UvEvB*WEBHz#XvyRK=E05(!0pYnNcqJHH+
z8#?`VGmFOaR%gPEM~&SymN=L7Gu4f_PmGk*R8dywoxOu8pAKMl5F&)~L-zuh7`DOW
zr@S?N=(isV(fqjd^K)VIwQYRR97POL*MS&TY9|Z=Ly!DlF~snuUwbZGk53HSEcbZm
zXx?Gtt&0~T_&VeVdtRGBt_C<7vb6h@Q#Hw+7>1p2bI(BTwa4#&Oi0a}Xtc*Qnn}rG
z>T=*eP_6A)dFGjln?K*if119Nj3-KZtysa%iYyVcB{Sv+6RMc17d~6h*md4y*)cgW
zIAI;|p{G%NHOFHdiTG7>x~J;6VsKiqfA>S~+qq)ux(v(g#Z$Jad(nj=@)=H_yuGEh
z&>Xjae||*b=O=uDuUfxE%`(BP1{KmhD)V(+zq1btIc;a$#;*5#i3-U*n7}Zh{Utxk
zV3fli1SdFH4s7#{+;`JFkY?xj5u<)q^8?prbHg-@fZlUy%k%vPxnTa-jK{)K5*^_%
zxr=q0+;Q#k*EK#+;Lx#Kxi^FbR=M*uZ+4pd5kpSH@(+YamD!KLl-qNyZfhb$WR$N-
z1cqy(*y^6H=K8bRB06XpG*s!}p9mP&HdC{osjY*YB!(7fO70pw&~dSN?tPa?nv>r@
z*adm!d;h>LI4UkI`$N_9Jf&;!h)gOG;+rqYmi9hL<g(h|>OV9DMK_2(;Wgm&Hn_(2
zv4;9On&?^qCH7)!(+M<S<bT(3mg`zS0kPcb6T29r0L9@KJYQZ@i3}yfBLP;l*Cw|c
zS)o)71v8=?IaT-Jw8SBA)!0STAGaM(sVwcT^v4zPIrjFHF}Gc4l-|qS#F0Q^Sx1>V
z%HV_D&zozLWm;mAzW}J8VzKcA<}@n`ob;wN^A|@WOR}8qrhEgdKN9i)h(`Ch6e-v^
z=^7~7r)#siZ;J9K3Czzh6_unLG+*V4zuvahbTa>~PSWuOWN2s7IrbAvJQ|v(bi--A
zR^#+@s?9+?i+h?~PtTF7lvlpR$rLM~e7HCl?WOrQE+(Lgh#@3~S#`agssH|UG5S80
z?!_C@lQV>O=KOPF;%gi9A|wzj9z+Q~oGCzeioy4b^gcF>aB2cO^@4v(SH{bX1eiSZ
z41SX9#fXSDGPce^%1c>!ZTo9IZb4-YJy<J<Cvmp1xfoS=iz9Nl%E;u_DZkY5ZV>rP
zjLtMZTKENU&+`NKyfVPR7bfxthr$v79;)TwmlV?<fj=$19JlB0rieP_vYSgkRC$-3
zz5ewU!o9-Z#8xqa-Y5#LEEd9;OfB{s|Dv|YwT%MkO)5aAhW%kWve<g$w(7QsBOCJ#
z=KjYZYUE{6TIh4|#$fafO+lh-<k-_OG%I*rou@z!fyU2ywJ7O5_@q12fX`nkGTch}
z<R{G?HU7D2T$>tJZTgV%rbwoomz8<>#2^yL4Lo9)x6)p$ht)!R+4FpxhlMn_yk2|$
z5(V=~dA9tmE#oFmNTH8!*92%}@z_N{l*yUfKi4wm#u&1$@R6i+rGA6Ccktj3NR3jy
zyhj1un$qk}5&W?0bLhP~3*)^~v2TBpf2}m><8EE{$Hp+GokjYQ3MUg0W>np#_g2+h
z+dM1jo{vdUXA1vR!o@>#E-U#~)AG8HEq<S&+@q70pVfYqm(Qy{8Dr%iH|4E4D@~})
zNf0zL@m^_`%Guu&tU&>gNj!IIc_s6VZPB+og<M2JB@Q(pW%-XSIF7_bP?3fAYJGTv
zi>qj<AKgsV@4Xj8tl_CYA70xg0_Bj#@FG%$tz&s?489$!$#I|wMdrSGRx5wMNvCv`
zl3JzAifhcPB)2Ob*Ny^UAtj=>ZpQ#;vW*Uo;(h;vCOLU>i%L!Rwo1x2yA2KG3zzDk
zMOai=d7^?p6@ld1StUEyaPtQ^OQfCGGd(mFlX}W+m53W7Z$+Ir^5Pr5S4vAt4_wuJ
zZ_nO;Sl&~f!1A3`g+qY%bD8PwoB*JE>khizO`y^FGuX>_C=uYOjNR=|F;SNOj3(&E
zi>7isdwyTA=OORZ&g^JBWAsVg=Jc4&&6-?w{@~ZsWfh*Suy!rJ==?eKkuDyMwbrkS
zd+BTEMNBQ;St{g{=Iu0{(Fc*9x72|2=4<ODsAFWK5}c#>mTMiNACGXud`!*Bd71y2
zs@6_6B><p{JNvihyGa@9Yn_&`Obf(cMYBG$rf5&CH*9RcQ4LagT>AsTI6K&u;d74|
z+sgy#B|m630Gkz9ahSb0QjWNksC}?k+})!EL|u#&cqvZJD(%O?@?Cc6X_#W+!FaL-
z0fxDTr1fH>mW*zTnkvUmHc^K;iy9fh|3I?=iUEL##Zx&)-3|;7ZOdzcZ2;V6x$PdH
zXVIB1Q61}`Z&!f7FoTj31s}#SGWO3gO1CEaT1<!2GBQ{k1ywMTzBRaJoMvLpulTkS
z6VjnadV>A#Wx>C(e}AaNh|IsDDA3P{;H~R||3s39VfKXH)wZh}k=3YPhX9-~>8%Az
z0Xdj&9TWQ$vIwB9Aqt`-*N2x`LyLY=&LZ5eegZU`@_-M0F<Uam&qNy_UU4WzxA-{t
z&OWoPTFwPsd?$iyKGv@Ds$83_9pmKytmR$h>Hy1JjnA4lsvVYR%6u+Ro@`{J7qY*p
z+820d*u`6FnZE2E7_b!TwcBz(`zhO9`3e9rgQY`H1EEs5w+(9K%|hb<fv^s~5aF}k
zYAIB&1rHj3mMH|u{Sh;*Qa`w<l5k_M&TDpRd*ij+Iz*Xp@8&?s@nL;SJ6(Ley&O`)
zZog*xK_05yX9@6Z)(lg{+J{kUCDh5L*S*i)i~L<lOW5rxKE9?JF2?;OJq|>9Fd6Ih
zNi=Su`of!+t80tz?%j?m?)l>%hqYYA!%&-@g*M@UV6XkT#kk4SgPY2H`NvXMe{t&0
z7JaBE6K+1HJhBPBr*rtM*qCa!`$M7fMvbV=sb`^&arCTv&g_|^&DqLeIrgDb&MVPL
zQ2%lJ39?bKXqKgjf68ZgHTa{if%^IFoD2ID<c80=;?EQ7PES`8XyGHx0pH73(p-CT
z$BM1)CR&2czz;cie$=pO8p6!z?DZ*sJ^M#<Jhwqnt0_jA9>Ni`Gcw5QnFHz$@*TV{
zuA<;HcaLo=Kf~fVmpuYb40>ExcTa5q&;JGZvOX7!`nJ3s9NpmmnPh!q*?K)pB6Fo(
zf^zcacU8JAU3;mCdqH3*I6gt1n(`e^RD9)DH=#Z|QD=w>&-&)-ik2Vyuoz1u`0G-4
zlJMR9U_r7i)^m|phJ}@eI|0@X2B~B3w;$6;^b?$3w#Prc;}_Kq&EQTrOx<{K-A#(#
zAccrnTCvg7Y}x-XL(omiIegsNTrL&fGZi2#wH&Jn^Ue)oA2PIWs_NRWEQ@M)$5}w+
zVfXw$^XiS&PS@@Symy{52O4v6(*Zt23rP)6Ds{exP71e1A6-bQ^7-*&OZtKyU@O+s
z52tv+lxy+qq~{5@=Xoz{^{2yeEbgEtq244(Z^Fvus0CQ8>8PC)<93=96))d{?D}f3
z=S)uxAj56tsAN5UsmkMcjgA;^@1<)Tl-(;sLil_uxf3;XaPXW>RPf2a_cQ<vuQVW9
zzr>@K``^pyk5;*?nw}nRF&|}c0Aho*p;Xcfc8UVr{S&kdrtX3`+oT@uP1I^QOTQNC
za+cXdCuxCZk_IMtLQNq16CMpurS*kr{~uv*9Tnxiz7G>aHwcKt&?w#AAV^AzG!kOa
zNOuW{4BahV5(0vBIe@fu4h)?`H~b#l=bXLY?>TF|f3aY}u;!Uh-1l`~(GnIQwQi|C
zNg(>OlbBH%0w_uP&~l>E0IqgmNQ0;ih*H%4p>I#;Tuo<an?J2=0C*ivwecH6)SFNx
z0H1O#6#Stdm0DmcA`Qr79`!_vN`82)Xvh3*4uQrP1@P@J(9A8GmExFVCA?gv8fRs2
zS=s#O0^r8ParO1WS07a)S#iS$DBi5!OHNQ?lp1dU*VyL{OD?WgyDfJx_1^JV*ejtz
zDIbNpJ*Agl!AHQH@fc&lc2n<dCzHtM)RSB~VQ-A@jQLOri9DMer-}VeM^vlJ>do+<
z^+(^`*L{n%N<MwySNtxGo;pQQXki=(1wiSZyW3OSL!pw<9v~pm;4YrgeWfo|1;EJ^
zr2`$&6o0Dfi$DE2c+*V-hC?*>ea~>TS{yuFZ@5oBD<C%n1$q&<j^RM%Y#nP<fAC!w
zx|(~S-@PlbD@)Iqa)@=c!A^RtJ$8w=dska#Lc~(hGTCeQ*`o7DIUT*(w}&XaAVD`e
z;X_MAqpAM9k3*^bu%!J2{pMMp;?8Alkl)68wij)fqfbe+ec%u>A$>3*MW){l-jJWa
zh&sKY&5K#Z+8`ip59=A2Fq&XyepcV}*hfLx)Tm>|VGg&1S-KJ#KM(`7&k?O8O%~30
z3%Rn7H%t?#6CockWcD$oVMJ}+V@k>OX+1L8AA1hi(rayBmgWz{4V1Uo{=!!gugYMg
zc{FdJm@4HIx*h^o(CB5*3FlvsFvoB{_1Tn5k+N1Q`h7(`LSpCrv*`0EFi~t;PaiTY
zii1CnNxG7-mc)QrB=xB4AbTLXH9B`S8N<8vO7-<c#TRF|^!CXy`Z^j^ZfrK1PDXIF
zvMcT7TJ*l3p~;hmd5MqOOFbu{Vv0A~Je|;Xa_k-zyxt&^+jHM>xw;kBmOOLM)64Qw
ze8Fk#%<N{*K6)0Jw1JC#KLLc(O*dL6u^XQk+T2*jJ>B_nDWtq7{zMUglS<NcieKNx
z&Gu*>O?&JVExnk;IBHz*hV(ql7zm>ONk!?Vi?g_&7q|AZ;Vt-*wb%n+LF8>%mT=2z
zzf+9mFFhkCvS#^c=fw$uYOfn&ktQKWb2Vmd3EXibAX$KpC5k4|d$mNxrhT;9XWVF-
zH1v+<LXY?LApONJQwE*}Y>HX>>4w)W7e1j<3~;6(k+mqrQYtA=4F_=L>JwvKr{p22
ztNxJ>ViUJaQ2f0}0&JOxU|0a$#EGn!R^N#{af9k&*TxI)iQ7=Ys?(TGwu<WL=uzC%
zqajP^A~f=mK<s@S^AOV$N2xYmZL&ooWB0wGVDQA+WtOn(OZA7Nf=sN;jm6QcS0&3E
z?GlF-#WqRPR7+u#Ca>j=7i>-38Yj3C<Dl?6ilD0S+Rc71R>hp^*iP|9cj`SJT_iXA
zmx(K0A1nM;B%CgbdXF_zn4TAxoYp#<K|QY|g=?;|x(ZHrdVQLgZW1IdUZ-Kyud%<-
zJWOnmJ-zv6P*<L8wRqK&F-bV<Oxmh*`sm8_=z^-AnQ3JDXgUj{3QajJ3)OI7C@6nz
zy~BXyV@25D*I+rsWBuy98}-+>*B91qF`{YzV^kOZM^iZ^{L-59x+;?EP5t6?YQN)v
zJefhHTQ+>j9m?^a?V*8=$5k>R;@6!-bI5iE?Sn1bK`+nYf%pMg`@fvC1}}EAJ^Z<x
zG+^7OZ=YAM1}ll83&RR3pLM~P@hHbpKQ&ZwqjV-ZQpsgqnEiUg3Fs?-nstWQ%fC?s
z3k`ZC$F)@v_;mr2ydbditIWsFq(Xo^H4n&2e4X)xQ&UrOwDOf4R;ckRCrX+;kVzQ}
zgsHpVDQ!ct)H41|4AP2N{5Yc{09ul@P=sUI5D?T8m&G3Q4A2tS!gFp3&`+=n14C6x
zFjM<^mN$r~WKcX-(~_*ZDN0Pd56{hdy)PpM3q!@3o;s7CVA4o@Jc|af%xQ6;m{H8!
zkyx4Kf5s7r!F<nN!q}3a)k`S1%VKLJTt_D5lDxY41_I#|%3-e{sF9cJvozH{r#t=3
z=I1{ji->wQzLl`^sX+Z)j@4LS#%b-4XhYKDOeG+4Iaa3S&b#LNcmRZdQ7eJrdISk?
zYwH@(9^heWS5o+EgoUup#`C`7^T@NnX{3wF`Yp_hl6LqY3HMMw!mirw%vWmkCx*Sx
z7BzL1{gAIm)5Q_)34^m4>h+RXrjqm}rVx!{S~fN+NHM95XC?Wb403bvjoh#!bT=6z
z3AcJ{zxJ??&qpNQa`t3^B~LcLdV&BN04s`jV2pJqw2-_$<#7J+*<RaXl<W8EZ{!@X
z*Mqm-gL5J)^F#3cGXc3&eaBRO5^VH6O;(oYnX#dE9=!3~Lo8M9geeHoLfD00!MF-=
z02jdmlxYabTLTu44@>3|?u@N%gkr)#`rME4G9ie;=FFZ;>zP_5KkX3gMt<qw*-ln5
z9-~UgRf~wW9pvCoZP1g-(I_Ga^~;sW$D%oTw8&gF?bCX8XL6l%-7GJ(buP;8`jn(0
z&(Y9T?%kPvL>vX!?b+CO)$*&n<JvROjZI+W_Oz)uSG!<w#Xj$-U0qPL<6)O(`<fUU
zD5m>k+soW>F<%dv!(@rFM#s*qXvt#B5=_#TvX<SlGy!GapO$eQg>2tPFgOGk){|cn
zzL1SsAryCdz{#>oSr@Yt!g*@=T@1YEmMK!;u>K?D>-5&!%?qW<dcKDaI&T26M!xsO
z;b{xN<TerUy{^yey78ygZoGjm>}aElvxkz<ed;VuFV52|Su>shgL0qje?KsPFLfL4
z?~ZhFw4Ia}=<&6Bg>yMlTi-ft#qUlucs|?q@^eaFAGI+@%;V(F!CIW1qx6S>s`e<i
zX|m@$Q!U@~m&wOy9Zs2&$Rn1%<*Y@a)ALF340@kt;~SV^A}GY0mexv2Eld%-oq4)s
zq4iBoN=oZtkBK%$omzZbY{ibQlZUpPyudlAuQ-my@-|G!$`Hi3*Os1h3|Jm~*cg@p
z5|J6s3)7!y7xV}%dNYt;?(b@FzX1PN+w@EVH2N5z%!vas1YR`EM^i_NS$=J?&St8a
z)Pe|PLKS>u^S6q>#1&4?v4&&Fc|M15aN}D%*ysVI1v1g7f^}5z)FqN?TH%I{r?i<7
zuQ=JMXxkwXNA;ripId$o90lRaP(kMbcjGJ+aJ1~4$DCvT(;SH7#^*@EOA*cXAfebt
zwmGsPT{=8f@=+f)F?E;umL4Yo7Q1~wW~?OubcBm#b`#yt@!{-&nf=8^PEdeYL<{hO
z)L$3yg@(u?9;dX%H+e3W3KB&<bNu#$RJ?=Ey=ml2;e%h={RN(GAEZ*gy@RApJYVNw
z;$BXURf#Zu#N5u{0(s*|o+9A<;;pBPcCpV}>Zs2hjylI}BLa&3YshHHD26@pOxl*0
zqi?COXK45ssfhIhMT6fq^<32wdYa{p%kB9ct2f*x&bZ1(vsQZGV7Xd$K^ZRH6<;?k
zI)~kJ$xd9rcHE@GskL@nsbvdL9p9IPd54{8VAnX=r#^X)+-~o=Wx%v~<X0TqbA0WA
z_OsN`7|KyO#glK)s<ySC7%v#>S2MQ<s=z{>YNtChEVN<L^x<6kb;;#~uSi4)gag0K
zHybzgu{`vwdamePBdfHHHs^0z6bRQcs5!B1qZMM!2xf@+>E?UmSp4$BTx|8nIOR$=
z%L2BwXE;9%C@z`(e2Rg0Yx{tAIf937C`!0XJJai&boz&fSff&1csHC6T9~_dvb=s%
zrE%H(sI_<v?_oaB{L>s@k{WAFqa7}gSExXYs$M|fQ&(I#<oE8LtS(G^pXMffDZPh;
z_4vWX_4;ilC4{&mMm9&8aX(FC4N%z5KUNNVp-e1Z-MtolDb#h>MDxd&!{OQw!IFN{
zQJvTIQH)5ADSB&-@5Mw@%h1ajjJk6P!TmHn%d4#hUpYqia|*eQ{-<Ac{X8>))1R`Z
zY#1gY+&6O5XnCo)yLuiI1QDoDr3wtCBoTk#<T75fRQ+xX;T|*s)DiISm9!Zf8*dtb
z(B(WJK0KBqEow~pJ5USLT9><mXIMFhh59L3RDSv*2^5WaUE)NsKM1Q*K|I^44V-*v
z^1hwXtDRSC_jb}YKiLdkLoMYb{n~r1y>wVc%mA2zdBJ%NYdfjkP6YTAdp;>;H87Vd
zUU~w?+gPz#?UjzV!N2oJ{0e{Bc^F{pmTcSOjv87(Qj89+)@2m5pz(l9kzQbyDKL+Z
zvv0j2vsU|ER{lR~0c*TJ3}j0?@A3x3A|I`qnIAZCV%&$4_W&@mLiI)AJZ}*7uA%_6
zG3K9ra%Z+^*d+sa&izf>jKT=UUUfb-%{2%Bb7FrurvryEzrsR^V+ezHeO~|KZn0M{
zMiB4{{~T5LbfhDc$7TtY@JUb9OH7~a-x%=zATkuGK}AaEuH>pOfatOdrz#Mm4d@)I
zV6EBrGsmi`s?&}UZXn4z$>-{H8b@)O-L%C>JstDPWnDdC`7aD}dAQ(>Y|<RhyRpX?
z=WrOM#}l5nk3j@fwt?6Wqc7gCtL%dpPt!yjKvAb^Gb)i06;%x_83*O}tQk`iSKhSh
zo|h~u$rt<TgyJQsyuIRIm};<Tik~B3o+Bojm&y%|Yx7!D@koXKT?YrjQBw){LaxzU
zt6%GGq!RDw_V0*Wk+($hV|jNrx#AKoa<PIew_=NXKRu0Zyy#~<4<u96-<ctL9_$=t
z<-e#+&;%30Hynu#S*^Xzkch?X!7O@!B;vT7Cq7->xU;z38vfaW(_t=z%}ebh5?}oU
z@3TPjKKeZ=#zih6P;t+DV(TWfnRD9tQn0tYT^uxP<F}b%ov+idJb^|^tAT|=tR%j-
z8yDl#bXAKr&<T)6dyf}iSG_kpe`+Orz!6gsnmlk#)T%ELk3psHyxrV#*66jL)78N;
zdal<WBf+FhUi<r|`AtI82vg=1Vc#QwqRRFi7knM`nOfvY{M&K@h(bHjlX%y)o}3^I
z*~K<V?WU`Abgu!&K{DI@_>V%9b@PkR0Ya(}kjF{$pjOidqEary8ZA%P18O=;JUVg9
zfyC|>-`~~<AAV3fXd6@a<MlbjvS;J!;iufs4tbhI#JyL{@gnE<3>mBuWu-vqs@@pD
z_3oQha^0H(rifP}OTpxoXi&|S$IG1<+?J35hE>X%x-gj3DLDnbO*UXq(VjtxI^fjZ
zEU`0!N7d%HY5_m%X`zm7Hq&pYzm;1Uc~WF>V7Pb%O51t>z-9x-^-|CtrbwDv#in5l
z%0(3@K0m4KULp$V*V?s#2$o+Rng`>aMu8Y`*A)0GjgX~05#=w@#cP`9)t*$bcz%&d
zOs%+x%kU?AYZqkB@{89o_DpMA(-h8q-jHkS%@5NS6Rm!;28MWarF4`@#pC@I<{(za
z`l*8ab!cNTQulQ^C#?f8Q)6zoMyt15Y%vC6hIpKh2cLLdD=9ZEe0pKaPHOMo{}oh`
zA8x-$UCC-n-WBnveA$$;_2liE;u<hYQ?0xm%Cd?Oc}EnSKRP-07J|jvNtbeU@(U|W
zN%%#P3G~(4Iql+s=eak9^=;i$oq3&v9j0PgJ*h3iApPd49v2H~5)7GgfwHvFhr>&d
zD~Y`Ud-jZ=)C|^qu|0dnD4h9tbTSPfcwD><xR@;sri;nN#z12vg&`RsZiBOVhgQyL
zzC_rf_9}(C+9v9*`JUPvT1T;>Fi#&QAk1HTUk6hk;T9JGZOOe;ia3xP63dgj|E;Bw
zohm@gc831~NKxK#<Jn8z`aimudo5cY*&l+%)ZF^LHs*~3l7kFjbc2tN)Ot;~4_kTr
z#{!1TVg;X;016xOFmv|dCST7ZaK}i;5Ij!7;Q%?do9~(l6FiGGTB4r|F$;LfZQp0i
zz4PEFFk|&?vH$Jkvw;M(*=i!~0woxLAW~NRFB6!Flc!lcUm^BNpHj9+;W9=VB5p*m
zKaL)I&Kt2}4(g}(`Zbp-B_eTi3v;VB2C%#7gVvi)!)mj6lvV+zB$y+dR7jamTav2U
zXVKNe6QS+vG2L*pHM@HH$Zcxx^-;cM;ZUZ^5s2XH4b4&h=D@gJ)qHt@%?Y0zyp*mU
z+MC@0g$K?2D)v9xPGQNP1p<*iUJJ@Q0GQ#jT{go|@;-APK*L!{A|_?L-}yVj*ftL=
zehaC*KoH;^^yqA#_fxHoT*c9tghZM-1}GZSl0lPT4@vYq0X+pkDf(VeOmY|f{PF^m
zXK#e%A*eX7e7Z|frF};7vgNy_a5K^E1P-d@K~w9lL*Bt&mB6!l-vNcqg&gw2I@`?%
zm1j}FO-h|cS=8pZEG*S{(SW|&bkKgJmFF%amYB|9KpzP<FUcbmI}3}+j=(ej%t!;m
zB&2)1L^I;F*!dj?q94Gx9+t@0{eAy1+Kusf<z(IW*K(e$FG|98@wbjb`jh!*x9oH&
znm9Ax-o)-q{fKy_RF`35%a!!v#8#5)cH7>g)N3@Y(oXApNk2ef&EK8Rm|No80$8ep
z$#b}e_fl(O`UEY2QIOMcaC$Wm^Rh+i?O%RvXLk*04yQP?fsrfQGk7($u#J$#OF;rG
zLMlARlb~h^Q$kMZCI_^%-?g)T^qTtKiWOZ06db>pyW+wD;A?+hl2(ymr*0*5d&U=*
zpCVY9K{Z#~yH<2IMm~Wr3koBOmkv~{?86aODCl=Nn~C;&o>7ywszg2BG2jdeQAQxO
zBT+61zd$_#k$ivDEmcr569~S>8nEm4yST33?nxCWodf*wwHjS*+okA0*sRLSMZ@>Z
zojTiDl+b2hr+o}v{YS2s>YU2!UiKI-9G5@|_WLVD0j~Bh>Ay*yvAnpS+d(KFZ?8kF
z>fdt3J|bw|rPg&ayYkdNSaq~E%Cp%85Lsk~O!#IH&G!n;Ni1lK(bWdh{(h>6sq1qG
z#wZnpbmkLO8lIJ+^LIwV%IMqq{){NqVQeui3jq0N)DirM-M4D9V*k21?-z^#@x=*~
zHB5R->L`J!4t92RE@W3LC=P2jX@w`5sG3)7v+eoH943~(hNCP(O{#S?fO6X55(uH{
zv;98L%%qZdoT&)OiHwLKp|TaV_=!7N_TzsPDNi8MkC@!4c!WL;AUR+@9`*=umd?0F
zNviOFIn^%JR)BGVrcmpeB<C-DycCt_Gl)#d>Bf=%D_P<e70N^Il=GYuWMlVvlbIu#
zAFxGehzNv4L^$?RH0ghCIIsg(fk<!d!~tjPrBJH+qN^6ia16AA^GS>~8JqyH#Rxr@
zO_44HfMx;HK$&GKvOg?fTgh<F2KaiGUt$0&Ps{s%djX*OnC=v=y&IB8^w<p=?K|$P
z#O@!bOa*|@zOn&DNIwPNueLg{@=x5ByRJuwUst@&7peudwl*Kij^7`BmCv~4{dPCB
z-&;odZ4}hbU~gkm{QmapLw{B-n&KoVPBdvEMi38NCl|m-Mis+b-$}T?uvVVJ0!&u1
z5*5sOyL)ukT2GlVY(_yY`0X!C%{ky9;G?Bpt)u*F{3CE%%(Gc*!#)8QA5(e@lBV(U
z=hJZYWIp@bxlu0%f_;@)F3D?kW=Lk=t6A6Gi(sAlva)f<F}cvS`5nJJ6Bk=jkEZu*
zLuY=iLoozietRc5O-Gw-v10b>?{O5&itY)#<2fnhL2zA%er(3dPYT>9ve2X%<<)nj
z<$f>)*Sv$Ci0gQO#2z#LYqIJ+kJnwP5ThZxdkVeE8ds9Z{KDB!8H4F6+axeuHzj{x
zbGcm{v)kC2Y5*xaFv~bV%8NG(gnOFoou$VE;4tOhR7ofpi%tOUyf#QD+`$59dorXd
zJ(cQq^zjZDgLW1gpoUqcMv(OaXzs$cVCChryn_H;+XWA+=xc%ZRgEA=(>r(jTKkEo
zh)2Qx?#EShG*u{2N$rLLuTY`p$vV?7wA9gwyo%P<3^$7p*_~ha6D%jxV?p;|=Tv<F
zEpl~W{pzauOoVI-&~{<<lTWSR`~86E8Dh0SQS{v^y$O1gbtLKH9$LOMBK^1h>VQ>G
zn6#77Tm_FV@yku51hF-rt4G?GwGApp%;Tw7K*r@9Tij(W;-?a#ZS53j>m|J4c9OiW
zVa!gsU!|t-Zriqy13*!|H-^)!^d_>o{tuA?k)rJNMSY{QojOw(ML<z}tMHdGeR!){
z+QxWJ4(rTvFWy3PK)b-%V#cFq?2;hyG2gN`IBm||7a-X_S@gSWzK>sTp9!Lf^^)pY
zjTLh3=7^o{{Bfa1bWVlA5Ks?_B3a`*sc81e0}E-Ikw&*ol9+~u$uK?C3C3b3{pu(G
zNw2XNAB_RC0YuP5c94quAX>h9?ZC%GpMOnax&}5Woy2%$?b!^h{X0BggHv05+oJ|X
z#-x`{YTLfVhS~8a?OutD1=TKlNuvc{XhHH-N;d|c78CB5baFa1OySW*K_!eVstFlm
zi2wvs&rJ1ntmduuy_{^NCno511zN+~d+L5ksPEZQ`<MT=lFR#}t{oYkn?ZM`=GPN-
z4)|jl=7iox*|yrJ^gBoDLst%ysXx#m^aqgrZ`Uw8^>YUf1dcvnyVq{nB(mZ_9+Z|7
zDb$trifS#7z~(rJ`lijL!Dj`<fpAcT^3d$_Q5)n$!-!gvu<qQARtJ%sO)AdLxxRxc
z#B;0ysX2S_V+B%W0qw&8dlQcS)XQ^|4a8`{y`4!HY4L@6DrIp7=>Q<`Bcq8n22-=9
zqxB4{Zx{9h(n>&g>S0&p8N^@V)0#vKtx+}N*2GWcn_?raVBA%mwkDLJP3uqE&5+u+
zJQ<VxV)gyPew>J}^}F*Da)KPdpIkc0almU^s}S&`IPBghlOm)YP2nABL-eL)5>xS@
zrqsNN#l3K;9KX<LHga*+`9-qRC#y}ps&~Y6&`}=}%cM>A1i(hd_!whZ^GkZLwCOJa
zoYk{sy(AU@98{{?K1y1SFK+KB>{sPN^HeWlM{^9{_on<hGwm)C55RuVmh9BIdy=RB
ze(@v8j`2(j=bg>x3eNGh;x(<(rST6*0rozue1%>14*d1Si8VHENkR~8`b8hXnbV4|
zy1F!*QysG)f){kTzuy<c$|M|4fRD~L4|3Sk8^`Q?7u^+68*23}TIT}Bm4M>oT#c*3
z2@7(!aeJOt?R;gYV$j}f!H_WuFaB<|+jp{KfP(t$5iK~IxA%~#)N=AvLM#6=U#qp(
z#UunB<lloBL*=@BMSvpDcp#_df*=18L&R*N#-rM4(_pss$0?~3JOdB3Lesai>DPND
z5nB0v_Sv#ZBFAeiMp{ADn-c>4^L4GOna=mSsXz4k)ZC1@Qh2X!*~sGXHl>W1qpq@q
z?rx$M?gjC)EfdT)ymAixy&r4Sb~ME&Ct8)a1R5q7++bnZ<hUN@zb`qSkVVNiqc@u&
zhOC`4`JKI|)}2!XTrxArE9UElBw*#Q#wOubUE~%`7die9##bZpGYMFSU=2t_m|a;Y
zA&1kGvKPu$Xo@%|dqmH9uK^6jgi{BBko@HJ!p|zTeadI?)dMKa(m6xl!u-|B&>aWi
zikRzh{R1ITc8LHdAJqLp!VtRecBo(8KII@q+3Niv$0*N|k4aNYxnSe_b-jwDt6ZPX
zZy+Vcfw%UYE_x2{?BkFR+van>y|%Su^&<~@6Yo~Hi>vR4yBW=NZraH}doIOuP=3Cc
zG$##>AthO8nvby34QPw>0Ltb!XHvW;L5j@}^%|Bl$(ig^*vg;8TNr2s0wf2i>J?VT
zqMv3jnO*nT*Ir7zAdwj-*5Oi8y)9iYd=^7LTB^s@n?8LuJ!-rNp%$|Qax4Pj)KOrl
z)O0XsZ!@@`@V#WI$?t-!4BV^;U%nOGBr1)b=nSsL+Fa`AI}gbL$mP|7us-i8>Nfa%
z7&cwe3=f&JFyN&eb;-a#$g0xxSsi=godgQM`vI!BvAIz3%nOzCs_W6w&glfYD4y+a
zgMsicl<so(0J!=KwHwyq?h5eU_5hQ+Dg~mFN>CSxu8qQ7?zQ(^J5@fh^~n~8m-65j
zNhh8j16o^|lKHce6;6+aam^FU)iJK}Juk$a2hy@iL75~<O+-0K^zu7K5P=&{_b1YC
zP!ip81Az@9&ImG&*d!CrUOdfz+LZr`eHsu(1ej_Q*eIJZz@i%v7(BNds9Z=%rD<Pl
zbI>kOOg{}{q;b~|hgL?BwZMop6sc?(ogc=LM~P7EZyhhCMq~JHXb-zFRqjL3#^)_%
zsEK);au`(!F@NxJA$Kl2%3z{<W0p~K6CfOShdtmu2_r^?3q>l&{#=}x>w(}8WpNEL
zp*|ax@K1tz_l|*uAcCMS5%znR4#gQyVq+)h`2l<t$l^Vl)lnM&hSZJ)BI>DkFmGr1
z=lWzTIO<(bWf9WeP{YialKfIxH64fG-e0Dzj79V)bPXt;0NdR1BvDtRTjDhDCL3zG
zPq#>(9$CVlNakYjG7E}c8|M>x3XF#zM}NyccI``Tv;auqbr0(W-SECjB>VQKU)+wt
zSb_#;zo@Ic)ydba<d&K}Ab}3y1Y+2ge4HM36Uu#snu4l}{&=d*cXhhO7Zmb^3Fp`Q
zHB4+fb+ppGfb;KCzM6;Reebn%Vr2?LO{)}}x!uxO@UT`c-H9;;8Pzi`C#k^xrXG?q
z<B-9q7FNJ3HmA3)$ba}ZmBsz!HkVqVORk=CgO;$Ka-~v+J`(EFeCtaN$ljkTv?)lG
zP}Lu<Dl%AJ7xjMaB^Jjf<$wXo6J(yH5Ef4+;Tbeg)U#!oVKcOFv!l_|t0@ZJ<C}`<
zlNsPBN?_yUYCbAUV&g#@TE+PI0JI#;c29GI;FavA25uEyBk$DNLAciOq0itksEWkR
zH@y+D0+jCGCEKso=$)@au4eFFq<p*usIJl0rf=8J3|5n8(O>uhc*9py^Gg6#>b;jF
zKg2I1j%!=0Svm?>?5(gU@azN{uqJt#2lv7ZMOg=6(-Oror5XAC&M-M#WVvPp+(V8p
zO8{}82<6`SzE?DNECZjUZ-)(8#kU+rcM!4Q8-lq!d&HbjAU0BAzjX#y^K#aEU{^Zt
zqNE&+<H#hvXx9;C^Bo^>wOOCJ__^Pm?aR-c1d9<dv;*LspwlA47p!K3ZGy!Xj`v4`
zasumUvzJ@zs(O~>JWV^XQvSfoTvTJ2XaT+P+LZW2jrV9_wd>k5`>AqP5+Uo)BKR8&
z_MbngAYKX{ScIqTGAuZyb3m{<c-RQ3*n;69v=_Lqery@vCdlnb6BFE7?~AtOvZo@3
zmEPn8t8X+jP|J(bt7)cQ;+R7N&5&g@q~qyn=~-RT<cWx)A;%{13-4w98OIo<H*T-?
ztXg&w7?b9Z-FT10O<mnwU%Q^&aQ3==-yaZ-O)#?qnz%@9%kc9v;@3sWwx+oqx@qr8
zAxiCzPaw;^+XIa5$-#Kqt%u*NpK#8&?x+vg$vuvu50X1SXB<;A8gbRbFHcnneQ<E+
z8u1;IW=;HF%jmgN1;w;{Om^)DP`t&2@*jXe$THG|g#Y-`#)f$gIF;uc=p_GG9&=&v
z4d};dTSNOWx8^%rpwLvsim)Y`l8%ZbMm%tM^&34M*=Vk?<Oc!Mhu7>=GK4{JI{X>J
zF%Ql+&hulPiO?XmPr;}KJx>;rG1kI!L5_heP|xI}FAd<9`fMQ?c=B)+0U3r=mz)&H
zXE-OACQ$O6$G>cySWukIW{DsGKUT@OdnJr4GQ{)_riM%Bem-c+-%EWyF<+SF`_7hZ
zF<hVdDd0=6z8HE}@Pv(p#e-e3@G_%FE+h2hgOn+E%*3u?(sHp-@m%l^p-q|1as35I
zV6KV-KMp27d(YlTg@xZdm9Q03NGATc%X3T`VJE9}Wyc-mx?;T0DC@|&?T!PUsIl#5
z0xA=E=?9DM-}u7pVK3wCb=UBj3+t=j>Kq_@YFY8NYqCR70`Yi*`|rFmwaJJS4gKCt
zzSB?x7n%c1rrW6+UsGzSW%qPwa4}d5l5I27?}(b}qq!Uwdmx4P4U?geVQ7?6Vo^w~
z2>Q$T!vva`l{e}%ZZri*|EZCZL2ZttDv=ssLcLLvc5Bia^wSSf`%Ch%#ED`-#1hG3
z6{MdVqhp^#5Slb-Xj|gaeAYOm75PIqc5Y5lFhp}-{lj5Mhqm95JCz=)*&bhEKZ9kK
z15@UCBmGZdNWc}I7CvvfuyTZJ<i^1|O)Z0#@C<ESA8aA_G2nr0kwhQrzBYs`vv>ZC
z$oBlq|HOz{p$z65ue(<&J#@UMAe{I!0d{7;u&=Vq>z@}7t9o~Ue)+oTLTxEHX2$L9
z(_PV7YgMHUCKM&~R;hDpq3+*{lzfT$K$k@4o?JAwwLt5xQ}QB0IiWlc^`~)*e8g?~
zo_O?|bi!u?AHC|Do))(|V@<oJ2Que4JViJ#iohNxe4yan+`$#@2nhMc%J>u^oKu~=
zn-0IED0QEQq`C*V_fkiqq>TZi+TwKuDkb<<3Mvov>SaPi&4+$C<Xx%H_q`}Kz(4oP
zRJ|j^&E<AQyg{tuy9tYc9M<k_$uE0LzYWkvk$W%*Ex*2*5L)sJfLF23JiFlz$224@
z7I#|o+^c4Lf49%jm(RN*qgsv0TM_>*{`{sI$DBWw!JVCIVWCpJC>7vKYDz2f0QTc)
zN-J%S0<?xQ{U;2axV!_>{p`Z;nVVI~ho~k!nXNVzuzhJ6uz2I<><$Fma{*tgV?-x`
zHXvNz@tq8TIX3R+b&>yLci!*SJvESbJS@PzG!O^S-V1~3EBA-;@DV64F~r8``oU)4
z_{xyVs7<J7Jmi3<&LanW@G1hw2S$f@NrmIOQ&?g!l{ssd;Gi67rX{Ya8Vt`KjFWr2
zH4V9{dfc@XVg@BSotp2``!GU4eMz?wgdgbpdFXcSWs#gls%IwURrbIE5oP1sr=;x`
zGVi4LwQlJp{gLd=G21&a>^`OjD(#&EHfC=CI-Ohm9P)!?81kX~;{KHeq>bwGSv+P!
z^?64=jvG}GjH+Fy#Q{wslp!W`^UaDTN_+bxlb4w*YkVo;Do&rGjDFyU0tjY;E#vi$
zW0JlYvElP+S*s_E&oQmE!C`P3*vnMJw<X^I8T0qb%29Z;tY-;@26&~~Inmc2-2I9q
zf1pC;=CH6Z`^$cQEl?uR*L4NiJgzXT>V9%1Ywp}jogTL(@Kq^c{898%i|wDrqDP(G
zQ!3#NQ=1Ye&?*KO%6UaA*2L^0ZR=x4azmkAr@p3F_@7iU1-A9Zq5UhpX)4x1Vc`&6
zXk>aG8efuDjc5nslY=p^{^Ljd<@pf7CklWwe4Z*R304>p#wk?+liycmr@iE)!dqI;
zF?+2DY;q21VNT3EVnR>B3eCkwde8($7T`ZJM#mkJ_Q=UJIBU%1vppbnRIpCNl%UE?
zZ~UQ`d2xg!%v#vuxvQ(8c-Uq9>33c{$M$!UYPWi}^NdbM_AoT3Uz=gyJ~-?XVG~HK
z+F?fR^yzXwb;n!pO-tkaX`~~ZsoNTKg*ojUOZ?o=p336rTMVH%RSexBu$%^t1*x;L
zk2eoagWpV^4s5;2Gaf_@|A`eaW>asChoAFt$nSM`htP%nyeci!fEqu+Z`^qzVd`6A
zqEJY)5<<AaS!6evZ}3rP*#kYx>1s8vH{(_=j0+`PCj5;VTThtW^K@61*0wet#S_|W
zWobZ{o>EU0@zEmIL?C-@3L-16gyKz(ekdp?MS$sWaO6%G<ZZjxCxG%p7zW-`erIC;
z+J?~2pNHhlg!c*d(lf$n&hsfsDwaTtr$*rt?H0lG?)A(OX@I0irp9`Nk@~A;(4fwb
zgSa1%&y?G(UY+CI>Y`LkL6Z$nUZY0=$3xSB=h36~+(lgFG|60(g%+kojg?!xH7$~I
z_&)M2C$JsS+jHNHe7STdi_YS;th}k<S@!{S6{(UwqU9#dGde29|DS8~uS+zF2m!eE
zu+?T;Y;}Yp_j4fG`!CMDssOtf!7#G<8_>@S*o0B$q8|2(aofr~kYPXm)`l+{-!H+v
zjwH4*67}*ShRf8HWLXuF#uWO^&X8k5pHG1HJG2(S)6_^&JoCX;lg6<H58(t)*l-Rw
z@TrV2I&`)Bi=`B#)Bno=iXW!Gh1domfvr&ixJ8TR3t$q?u2}P!@%p7Zi5w=4+Z{9D
zU>|>ZyqPP7X<TPR0b~;ILE^;x1C<c&p0ZJm1fTo=Qw!7w-Mf_%il@t*#ltkSH898D
zPW5Ap`%UmaWv-Sw?c0j~M2%8B?eIK~xm?u5JdxCTe4T4}_2>zU%{+dfVyt3KjgDf?
z*t^2@jCg7;_Iyy3tZph1BqG3nuF8Pu+4&nBNO*O2AG2P*54ijXHHg4|9-QgCJ5yT>
zgbt}sV-FJ?E(1!{H-H0b%`3UXw98_T<^%D1{%_A5ekL+E`{k(~=<m7SUSMTRRec`V
zRgQxp(|1&<IqzIHymFI|aEw=RC{864SzkLTt+CW$aaD5MeA3=YSs`RkB(7?7#-*#c
zZu3l7l%>0E4n1@X;LM_;p75{}h$E2D(9lE%a95a{_6pTAln)z1NyNYWvq1Xm5pmJM
zazO-n{+5|YFFrN<)ufOkQClCGnIIh@Yb*y!oTF@5w;;`^?rG@G+t<CqF^%8B0Anu8
z^b1aER!dxcv3#|F`hA{Nq~7r7M<T976-W>k4@q}A<2HpfKAKdQe>Oo!9TKYPbC~c(
zQ2E=aDzngTK54|#j}ikE!>&M%VCY(MN<62$72+j3g*d;Y?^Tcx&q@%a(GLgU9;q-W
ze9o5eIvoZK%J;AvP#<Hq!}nUrEYSn~X=yPrQQQcSeFcdFLysg7@vxD~x9|H%(oHcS
zpM+($UvRq8$n$d&aSP4Gyh4HU*E0+(UM7rM;|>=9ETnjK@XRtKEn{Gp);$^OI)zv8
zsFnt-+z_GndkbzzKC*Z5H44i@>gZV#-xg*g$L|F=XOh+GX@z!9pPGwl*MdNe?b_oc
z*Hr^?mF{=5kd<dx?Qr!)FtpeDS7Ib!wk!**NM_4(hxqNuir;GkX(vRSOG`_T`5zfG
zlW7#}MGhh*v-2N0AluUj3#pR3&m8Y@!USlI852o680mxU4P_hG8YJKK?48{FA`~r$
z#0FaqCZ;`VJ#7}q=w<*j^wS9}|7Q`rbETB^P}Z5`2L7(hL9GW7_sFJcHx5vfo4|Kx
zzZXVF6S5h87AdT(<W2>&RF>P^GBH2?JidQD#Xn!>1VPe>H{;bQeMY|w@4^Xm>umI~
z@$uE?+H%%+ruk~z_9ZdOot+1^1md|&B>~G%?f2u9C7jIe{(5aqsLVFmc$`Q$TQ<--
zT3dfMf64P4kTp_=r~`&D^KlX15RzcMJ3I56w$LlWD2r<9OS2%ss3^~p;Gn{>6c#M0
zOh)Da(dRURue~#Cv<z`zqTa&zNk}+2{Ltlx!*WxJw)OJht$Hfk;n!kcbLG*azOV*C
z>3d;0(FlvBKh^Y4wf+=779hXice!Elt=Q^;Y%pwzCz$wOr$AfOy?;HOyYQksP#mYA
zqHMOG8kc2LPO|4~vvicz>BK6~8vpDclcn}RZZA0h5=EmlL)Y?s;Wt<t5@_{-t(bu)
z7TZ+}ljfoQYS;Q>b!t;<<<y$ZqI^*W6ef_O9c&{qkI{Z}Mkn{Eab~4jbkJ_RYdwy)
zu!Nvh35`{7R50Au3H~zjwY@3>l>mwo1m-BTAC+lFFXf<$)<cu8oZ!@eyiogI4`r!S
zQ>&2%Dy(LjrAqN_Wq9j-(wFkf;`!1yS7lMu1Lbu}h<cA*Yz~U~=`FbOV8?2)`dQ3e
z9w~|EXhJgIi!{^k8noJIyf5+;#F^aP;U{Yxfr`Zn^J7FJ^ZdB9)9lp0BcqqP^ca~y
zQ2@gS@H-z!FYsOMBQ^N;8V~R@jpQ_{hmCPz|DR6k{wdm#qtLn^`__`@4@HZ&qpsjv
zGJMcmUS598%F3#LWZ!a|h(*JZaKPh|qFCc^iuCHsM~nFt&TN9y^aNBR<Ws=u+jEV8
z+g_%apDswHD+q7L!)A;KWBH{Je2U%Xz${d_$+53$ig}tCS&N|njJ}G-K(<AQj#pPH
zLs@||QCh;JIwcK8g+IjjDb6|TU~{UNb|suh5y}_X2I1-op)o~cE#q}(kHMw!@A3-~
z;sHMX*0bSRKJviO??egi|61v{_tyPb?VT52JpgW;X|ZTR`I}#T4)C8YsY1?M#=l;y
zHSyjJ*LbcIM4VTioDn@}us5zI8Om(UOAtW*sYLA-!=pl&U^AS;nTgM2IcT;NdARLZ
zRr6toq^y9<{;mLvQ0Qy-h-ofKkJ!NCb;zhS_80|8v`7FN^b4#WJJI=7<J;dW$R#O=
zmbv~^WO|(C1y$xFS!qZUNEQjlJQHWOTdlC#eO|H1=38Mc&3l8>UTypo#!-!0h`r|R
zAs)x#3MLJAli?XO)smRh@^E9UIBtP(2pF?kr2wIW=*9X`q{1!cG+7p><riC|C!#of
zx~?v9umA|^FW)?dZNKj}(nuwD{Veeu_L}9+Pfa@BH++DzxJa3(xErSvEWYpNW}7}r
zpp6FOe@$1e(uDSNy7Z6OL{NQmsOQLy$Ow3+^X}>q>DL*fQaPa;C9iQu9`*(IW{(J1
zjG&Lgy9u|Fr$(|0S>=3^V~V5)-VMRFJwD}6xUFSoz9Dd4EKcwO%MLIg(zPyb4|V{J
z0AqtUfYZDkwaf=Ya=Lc3|J%UBGM^%WqP{PBhVxps33`skvgpxZ>zICY(p&!8HM04w
zFizmkCh#f3U4tUDWIOi9c@Vh+vUEQXu-E%&qqAEaS-18X^>7f_U((2?w8ZrqZK89c
z3SVl2ZlkmQA=nf117Uo>sXyM8|95}MI=%6D91N9W7J|XJ72!94-+Fj89&;EPLbjgZ
zh8<xottz!Kjn6ZzKt=Ohx4eEXRFA^E3cRJdV-|>YDV$ldCEC)*NFnb3F$?EU3o&G$
zM_za*TT1?<P?S<kkn<xma~$z(M5prOfA6g}mU~lquM8L<T4yf#U714vfUzl|Cqb;j
zCF*OJalx-7E`!~v<To=v-bRkQmWdztz=M;c88FU9A%23wsn5hHK$STe5`I{}wsuBr
z-|>mYOIP|Ei8`mN@9Nv0IOkvH-aIjFxSSi4WO3sxOz!DS4GS39o$D?khK%_K+r6M1
ziy#p!vL|mxwdj_o(S#4=r@g~NV-qN7vf7d^N4so}XkC6Cn;(uR=#~3?r?7q_c!#!e
z-Op|*UO+hbkz|$wKUxoEp=)_w&&~D|l}vh_vKM05{kp?;;%)#)Lk06L+>4r*pMRrM
zq7yE?^H{q8*JZnaS9q>V5q~gt<|H{r`dEfJp8qZJErj+LdbKSh69hyswOkv?;KGY;
zS!nJes*yMsQcjrWN0WzGu&cFA;2%3a$to+O8wLvQJc#4;7hk=OC(aPFj`vdb)1nU^
zT8fFjI~sG_+^M2=b^Ooyhe7tDNLjmNZqMa?q`NKLje~hPn%>H|(B|S3Np&tLsiYXo
zDrxZJ?=3Db{nPK?Vgax(L)7IB08C%j=S)pa&58(wa{#r*KQBw*JLzgne;AKLo66P^
zmrVr=M?;VfdQ|@t2KV%aXm9kWDvK64E=~D3E~E&DLLBjPk<ry`5p0Wjgw{sjpKWP}
zeC?2f2R_aTMucK*@cGme(3;^+b)vh(xgq&|9soZ?B#RSlh<q(EI0BglKtC>UEXmCA
z3V!DH$izf#t9pow9y(t?fWv!&=nj$e4+t@nfmBiO$G^_7I7)}%P%HBZ&Ja!<BoD)N
z_2j8Xs^x(+I{90+rf@nC!CiAGuk*hbp)?_a<~|Rrf<zqUn`M5Ytylxz&Z<gPjr$&M
z(y7N0`Ik!qTE0BBM9%Xda|=cnN%)doxD6U!W?FPnLn%8tw|W+j|K9d#%R-IX*z`s{
z2`I{tl?{Uz#K<iC&T3w$;&)`JFsq;kvfzm)gn_DjS^0(cCBJdF#CsRrcSj}4iqjqM
z8cWir8a`v|+LbiU(Yeo+U%}t)Eez+oN<b_oeVv3?Sq-MOz%O}HAZ0fLz%SoVT<AI!
zSL>_M4lJ&~xyCL;J*2f+5A75W9o{~yXaXwm;M3!iHMKyDHSLyeXPp{XF`YP4QzmzR
zeh_a`*nd?%m?$lQNSHKHE@fXKZ`u>RN408gbo;UzrAHMM?8h1&&-~a2%fWyAFuRpx
zc8AGD!MBzPULer1$w}Cf#b0gdeqRG|?;~0NI5b!FLZAubPribRI#Z2}gc1h|_X|nS
zYDP&;`A7J>Bdjt?-n|0bu^7@zzXJsw1y5(O1U=8@COS^T;rG$7evP+9c-|Sc4)S~5
zB5nOsk(w4!X#B?6+j85V`(=@{)N&_2`IU1T77d^Y${9ZRSLJh{7GI3>_u`!V*P%j0
zm>gnIyS*d$wFg%9p<OlURA5`6po6a4#Z=3cGvn#oP-$rm$&z!JN_?)_+YBF6McqpI
zEK&LG*6m+vg~W?<71`b-v+Nit^oMQewHJ~>mBqh0D8#>-h(G@Yb+Z-_@Y~i?qj41f
zk^eTUH=c((w6D)GB=NJlJ05WVQ_Tt+l8ApAb06kxL=DUmxhA-g@Ja5hdNNTN{(`p{
z_k*rzlZ*&<{ODBjM_~i>BTGOI$dO=oD8*{>J@@xHZ5EZ$315a}eafR2Zp}uc#s}pP
zrwec+i@`$VRosSU)9_NSr@K4VuOKgi_K(-2JoB+kd&!_EpiIx@Rr@h(Rk8FkbS#E7
zbW(k?isUN5%eZCLS!bmsG<{Fts_UuZ2vARL^!%VucZN?&*=Re<<6MjTG^a48Fy)F8
zJ>!Zth$6_J>$X)iO{s9?dq30{KBPGp5^N(lqj}S|uXF-M*I3)}cxk1#@WUf$u-Jwf
zDmYNUr&u?0GXGTpp5yaa@9pf}Cj_41+u!H=_LCaKdjF%pu3$h4RJ{42|D-$yPpC>J
zb<9qcDI)$^*{l-SlQ%xG%ycWyO=oA$BLa)22j-}1Ec^uxg$C*LhHmP}nn#Ih14GgO
zYAHyx9vN~wEF-Lfv9)9WdfOj-)%g*g+tqywU|by-f$sHGH#sUX4u}UL6hCQSnth=P
zRY^9hW~M}odn$zr^%<_BvFc#)6qGn~7Vmm>ABqqeNU$@EAwzwtN+=_B=^zsod*!a`
zJ&&ze^9-E;Em-QW%PnjFOaDJWgQoi+tod!A^Idx4=uHrSi#*LQ!$_-WqFh-0#z@pH
zma_zIY-Ee#<$MYOMfv#olt%gY=++RUk+PRU!h}`VJ6!eCCKss0s$L4qG8Sp*`>=OM
zvLziW2nt^97XSx>bK`^&P+7-+|6ogfYqX(0!*69)Etdj{Qj1&EtcaLTtsyZwl<e3(
zHJVcYB<b;tNCDv(auqA$RrzDl3e%F7@F8dEq_66Vrr2TB)rgkeW<-UM<!JlJbN%rH
z`D8F;xZ%TTuw>tprEjhA6}sa8HA#>rte+(OgNKCp75(@V$<X^;Lv3CG3m@q+nIG9C
zP^m<kn%q3wav`wf-5~N2dYhDVM#xQaMVk>?oeb3&kK`6%O#{A!d1a+(pCo43*=uY3
zH9IURBU~hOYicfBTcE(DmR4Oh8=fL}Es~o8<9w7r4HXPlsk0!)_zChr^*nFGclnG)
zIo1%&0aZkgB7BO7#-+i=^dnp7A`A?bT6~()`1ir)j`LSr{GBjCIM5#tSr0_ezSbIv
zgo!$^ko?+eP+Ab*ty-p@-u?P}Ozb$;->1jlFL5r0BKarJcWLX#7tUxlS{O1nQIH7E
z;|8!`KZ7)o=ozLo=npxeLbW}w$|x}q$Kazmt_;9O8ALYsk3*t19hRYjf0Kx^|K<c|
zA=vm>92dSBA<7&Qo~-&nxGK|jDry$d-lbWcELbMj%ZMNM5Cy;zvSflFaPFMW8Y?j8
zx1z!5cjF}-pTffYGs3dy+tMD&A`rhRAO9x%8%3_52eJ=<AG4Mbs}c@FKfvnoF(Gec
z&<SGhI8lTX^AwU~g3BnQxAlM(s88eezFijKfWyo~{P!#T<07O3c}K&DN?`32Suj#=
zEL3q2^E2VZeinZk*R}lAnJ5q3H+=f2qGV(3Leg1pBa-`hq^SjWpprI0f@j4gOkY2|
zu+}1FmS(hydkJP9Ky}74v>cL7<C;1^k*+~PBK(biBW#Dp1d_zXhQ2lh0um?#1#xW5
zqE2WzyN=<$$L-{S2edXx#^X4EL6bS1`x~R{Q;a{gq`iHPs?pzPNYj1z6wt&(WZxH)
zbhH%Ww^v+2#}ueVvfD|I<qFv28|7V;(;!lIv@?w>@3H5P3k_)jRGTBY>)GG0^7kTd
z3;naU-=huixEQ}6KfP?nf=T!}fTDX*#ZeI$8ea&=&yiFIHFS4FeG#Y{)m#gRPJG)9
zI&pj(5zei8Zpj^uj!>vSHB_N@4s;s8vE|*KIhj+khlQ?I-j>7?^iZ5`LM}AqAaLYk
z<ddd18&ioEUsjTv+@~C=9qGaXe%Z(o$ly-p2DhQt5z<gv!20sOE`NemX<*V|V@dtB
zzQK#~DgS^h-0}We-;=Zn(t-X8(wH=NH{`MfWg^8&```>R&v!%bW^SQxSbWT}>D(DV
ze%|=6*5baS^(I9IGq{TvIfxc~6?WkvLhZt2*7wKp7j;I_(Vc9_y3&v}!|+8S;j4$l
z`b>TF>5ttt>zqR!YZwqcuPiSyxT!?7r|Vw92Q9&h3gx+p&JX21s>1?)SV-A;#1;j8
zKyGvQHhE|xOhyJ0t6QqC3~|7c;AFN>cA1hfi||E{QKAPn51spC!dsk<`^A0EHK(Q@
z^!FBY|Mb`Atj*YMN?56wZFH2WQ#ROX;9&!2cai_)c8WfF=mk;@gaT&rDvs;ViF&9p
z^wiZxW*&(P1yrwcl=uj$`GyXL?5fAKKKw_C@b`ZJ0JKN<&!9$QTk7jqwk>mW3|izU
zq;yow)0>!HR1|fj%YI@i6y{I|@L4HmL@pAw4G+HgmqAwq-|U9<AZk74!z|lj3@J|C
z@kc?nglyd?>tmSDK_si)bokF>$%A|`w>@H2E8EV+F{UaduJwoH3C_QOnJ&$S-s&b}
zcJP>CaO)F*yMo)uJu^vWHY@`FfI7qf26g^NX#-$5?s!^QjQ?Ga{{;yCDblYC{{pYk
zP90skOK`ldrmiqTkpd<B)=;>oQEiPjEWC8bQK*k7leyZeG<b*n8_<}afLzwt9BkqV
zPdkCx7uY!<bKQJvB#E8}<5^7ZtIypBdAPM`jLbbaPo-w7!Ix&J5|g^Rah&cs8*Ypt
z4&b1O*vX(+cup`^RB6c!zJ*MEf7zCke8k7DRzC#uM?&%IE6Ki{zn5W~*k7P@Q&6Ei
z(f>M){)5xrZeyMUsVzFMd_`#7nBQcBF#H?PVVK#F*sJ<M&ox|beN?6o>_tItW;1e7
z@=M^c!mFiz^4>JydtqJzBz2rj6yS97>X4cHrd}Bcex6AlW&+fvy{U15G@`1+OtY1l
z@RW#Pj#-%@M!ZuQsh_aAl6REn(!<a^!qcZ%uP8GK`q;NvwSf4vr!g@xl|%>Hpg$nI
zuP3V2%->BBOcw-bT`&QPzHZ?EyixwsLEY(tyg$PZvQaLRe{dG#TwM!eqfsG8GiU^o
zBS|7FAM?Epv{eO9Ed}Bo;pvPoQ#<^`Kkmfmp$&_~k$jltGeRJSdFdhf@ORmf(JQF4
zG;Su_u^kplp#m&4LPj=&B{2NZvO$N{HcF5g>q`h$;AT_^H)f?G^(hjGtLg-7>anAY
z8IB)%yhS?!`L@W%w^6t>wC?OgfuXV;QGWqPP$A6%&c9cfi5!q%1>i{n&zPeBxp)44
z@AptN@Esycp$&>j<1LSaNkhnNYE}WJLW+wIkldi%P^2_MIn~rH55<?q!`$lkg1sD0
zyAA6F;dhw_(AXjfHLCHB5$l4=nA_6}1#0cjLC+$E@jnZa?FlPw&@w-+43LHb63MT)
zTjoeQ<0y+|Y0l`HsHb6QgtTFu^7!$U>w-gy#Wnc06INO$u|TkiAt4b_fG?5MQ~y6u
zSfP+c!9RLflc#^ZBi)xw$^Sp^=;{4C;&?C&9q$GaAn@9dktk3*j8Nkg+bnSqAmJF$
zb#f^Z7<0QG<c8EjIQA-3&^#BP+D1+6Gz8C7oJHLd*-IPN$)n~|?IN77;BJms?UoA-
zM`FaBX?i)48|iBz{5?T=_8axgYok33<Y0Ziuy%{=K^|n{p(vRbtPv{#zEiTpLq}LT
zE|z~L1W1v0*nciXzx|y4e_o2Ae;t-y$;ym>-|7F^MStIRu-4B&v)3(KtzodB``o-L
z<cPe0O5cg`)vHBPFOkF!^^j>c<C?)troJRDu<c9@mSRnC$@Zro!rSPl7wuSPd;TR`
z&3<wHubi;V2<%X<a)SNE9Mp{axsXheT?tPyvXDIOatUm+gARtHBx!;)EvRQI*LidW
zP6D%wNmW5{(&575PuP(>YdoroI?-*}iKVL`DqXKSFs@$_$YNYhe}rnXQscOWVVPpi
zP*F-2eQP7P{oL+(-j2^xT~r!PHS$FR;rtiqeGwVI`Hyp54`}`F&qp+MEc(A!@1IqA
z&X2PCE9q?WBLB4$5bdiLnRfo4?HK5f0U#MM43u?4`(Gb|qH^CcCyj>y1ptb}t1R#z
zJGfx{4iMfq0xC4jBl0P9(J>J+62a_H$gB;j9}T78!yw!2^^2SpNoTu$D{k9_;;X>6
zVj>e^GPI@Gldy=VU+`W@Yy3aPzA~t;ZCew9C%6P><1T^VuE8OA2oPKY!QI`G4FnJF
z?(XjH?ykX|O|P7D?|JvV?&^9!fhvl{UUQByJ{zWbu0YZw3zSnc?R+JQ4B~CmFD{4U
zH}2mfIG03&8@IIeVK$ISWGVfFQP91g2mWgI=J;qY|D)Nn{#Ua{U5F|BKl+$6d060p
zCf!PyQ2qPK?+E`iux2@DZ&c<a9M8Rk>`}{a(EwHp<)djSlx1R-&rjD^xh#j}!Rcv_
z0KG8B^Y+*#(SC!|G7;9#tE%MJA_DJ~$a*}vejheZ58-9N$j>d|J~w-1Qk8@^Tqhs<
z44^P!dcsXp4lALIE)btc%Wc&FM|u=SfJi{9=Qx_5<i?-&H8dndIPHJczW(=j^-TKD
z6GhiaQ2*Bx5&!i>1yIN;xB#9r2le>8PI}fV#Ac?~1|UnfBEUe`HCHW03}EI+r%(pv
zD7RT?omTclHgD1At^%1n^u`SB<fc#^ukxhOX*oK*nm?wZFMfiR(MFdWE*9(4%Nl(X
zM7(KZIolQFyJ1N<-U060VjVXj1bngL=xQ^L`+r)!KJI*qJ~<_z0TKOA0_J~TtM3?J
z0Ui5-76lg8zYE{f>%Tj(cD`I@q6q`cclD@Wpri<f@!O-A)}d{*)M`MMpT;9N>tRFu
z@z8A$5_F*i>f{OUI#cfv3oUF(>PNS-P<)vT%JsgPBwY{0Atn>(cfvinLK6l1Zq6s*
z!E{kQ+JY9L5U3pbt9>OpQys_sLpt25S>FN`(pCgvgY$U9^r4Eluag^pMosl^E)pBl
zi}5eOOsF-K&GEnPlrsVVFaxx!Wg_&x|8=eWO*ExR0KKT7`&T1_e~N?dU-YWcS$nFG
z3*0*Fy8;-i<43Us`*z3u4Et@m_V{aB3^XcOZD1LyuTY(mDADu$ThILW!M<B$@DAGK
zs1M@`6=#&n$N5Aqm}ff!Hbinnq#_aFeXK{#?`qXd;)nsz$hHk1&C{%^9X=b~o~R0B
zSYbo_oL^<_Qz#3M_F*7G_OBW7|Jy_V_bM<(P6~^b*22NY`tQeXFZ+A`g<Dl})KS3=
z!I)nQPG>(F#?jkEuuLd};;TZ{+I{_Q!EfnI3|T=CT4?^AE_B5@T8m~dD(+HY;aie9
z%P!ROcM4&!-~x<0eeztPMf5KT3)YLQ?}%V-z;A+49&L~aefs_G&H~F(4lVt?!2X!8
z_yAOnh~ncljuJk99DqOw@3j5ZDxa;n)4u-?HPLUfzvn=bTukZzc{Qm0IY>p$A9nxi
zAmRQwNbe%dpb?;OVV|I<JD~M4VJ}gID}6|E=$0dXhT;(gKkAJE=l$sw$q=Iw!f&6x
z4bDDMqz-Kxo`*~p2|v;ToYVjw3_0{S>!FEQCZce?hM>|eaOm58F=A3uR765#0J@bU
z=<b4ZM~oQSI>V!t*Q#FVD>hYohy)gRFo%8ruBwLN{jc&|=KOK@Kg#o}_}}GOUX?2D
z-yh7nH=@V67Y03gvhQf$a(3Pt(sdhL|97CHY-fUn9KlAs&S9Gh_Iz!@X!7jn>vgx4
ztX02^l;vQ7ySbp?x<8>Vf*r#|3<sZ`b9FE^Ufr-~-fx<B+Kp|5{Le@H74Z%Ty8zg`
zerQzaH-{d+z&pB&o8SR=CyEB?aCBsC6IQEP-9~RmvzpJ^=8S+QX0~VMk5m5YZbMQJ
zG9-JuI2`CRdZje74d<WxK~BhCIe%+&AYmK%3*&>FcNK~*JkCw7P#*xtz~Fj(6h#vu
zbiCV|e+zuheu6g=q4nb%A#Z^q53nfzo1+KmLWJENRy=qQKiT%OnmM0u_(jvaG%_p(
zV4@mrnZV2ezb|Fvyx;t!mM5UsZKVa)HO4XPgk@z(8vAd3Tj>%Fy!hA^q^6DPeX7mK
zq*=LOoJRW$^8IFIHyJtidOz@~>XFUy#C`Y$mx|%f)f|ubG>1fz$d;Q?;vF;99h$`T
zlCzj{qjF3I*<Yigbe3yy`i%d2G>{3!J+X%U*D8r)=NCGrfBw0Be~lz>;O1-{b(Dc+
zL?YAcm}swt@u3u<;_22&$^{P3#0ET|OD|#I`h=0R_Fa)<?o1<G)^UL?c=IAImSB{W
z1M6O+)K>=CQtJr>9h9!NA^HkINZ8}&ejkI&L5E<kW}6uBCEHR+kD@d`^AVmZ34+^j
za8Gg4R)$b`V`PV6h0&Q_{ZrNUUxfMIbF}7v^bUX__)qUJzx9h+{_`MlvW6A+(kgj(
zuXcL<kW0jV>h^$^&jsvWR`G&At-UamPOC!{bFa9{c7aV;>pI^Wxq~%_K@Vaz!Mh~k
z4?ci)Fm<MY8OClq2jR>8hsU*gw?xQ8{ntXn`zphgMjW%dv%PCffNGYC5PT>VVhJW-
z&zEbkGoJez62dK+#P!?JT>u(7k-a0+>sdxMTj~q1%{q69PE$1Jw`DP|Z<LTkg^A4j
z0^2Q%ZxJwJ-OF_>!=rgBdDK^%;l#%d%k>`juWP=}r6nmtg3;T#249qtGN2Q!K!G(b
z59ooZ-4iq9em*Z0AB&|0WRiS^=oRkWGM$dI!ZGqxPWOW(7aa}?%ua5wRr49)b{VB!
zRvV=?496xL%2%)}oKbotBpWcOcBo$*{SeUcrx}Ev%fy&bMlXI;Ty?H*(IE^Dkthlz
zhp&3lwc&8^-cX+i*aeR2)JxmPNlaCuRH&z^{!=r&h5z){0I+Lq!&3BS`?ZIQAtp_I
z>GbCaOzX;XYDCX+1ZybVMf7i30Zf9F+q_So`ixL>DxXGNOj4{|yh8jF%&O9@!VH&=
z?KQdl8`!192e6DgVPQIUiQ=KVwrxf5kmui(H;O&`{3wu2!3BTn?}y$9yqMyH>Ds#Z
zG@b=^wX^Kr;5;1P5`~X2_}?KeD3cBclS=2xJNcG97BOUEAKvT<@Ii86+?_w2bki?j
zXJmayd89Kf<qu69FTm2Twi@DXbfaQcB0Tu3MYDB39a^Dwfki-=NEcbT-UNdpGAFX~
za$GV<Ar7a@0VLkdaK<BH5onfn4_9!9i`Smts5LpQzeGRWdUujzB=(XzU!4l9hDLU9
zmbI-)N3jstOqE=W3C&H?q&=CKF4bCj9!}*|rGN=V&-5ty8Cn2soF)9|>h6dApp*Fh
zf;zF3AL3heh|?~ugvNuZvENpjrIQ4oeIYHO5q!Tfl)RJpApEOYJ*Ri()Wo05JG1ag
zlkIUz<Zd1D<qp`4>}KTS$5OJ@bes1p$5eJ&gwOgDM2yc9)WB@{TZ*J;g6;`mqP9ai
z2a<b5g3kfJrU!BjlS7&n<WqU~mH2J{>19tj|L)|@5LIvmEnzlZbv8qtK_^^63*3rX
zzXlumqFo}itF*(lLX%|L+NKpA3dW2lR^0g}H6ZN2=RwQATvnYaOme%sa4DCLO6XA8
z&j0SJ2mgvH1dE?%mV%k(*W*U`8#gR=FcR^jp6!_!|CQJ{0<jE0eEWl7`rVlIy9_(t
zLeTd~T^5<CC9BOCpC0~38q2J+<W|wp`$-~GEaD5q?)@&5;^Z{<mwT!lsCSW5K7y2q
z_+8u4)JawT(%Y}E{#ovP>gXr=fQ+B7ni!_q=z;TUpi2qh@r(vz5`WdXPJ{9MxVgS*
zYtUsThJOnu!tzXwWBFAAeps*L08_{;;bNfs^_?^@I47+u6{_n0Vn2{?sjsguny$4H
zs>A)GQ9N92sTt&*3}G{tAq}*J0v63#Ely{(dN!3O4PE3e#&~Z17&F@&rBHZYC3hjU
z8BJbLy&+$cq{5-c&bnRuUC@W+7mRgWpzLztCOhNvjoT_MtJ2x1(ldMZw1=YlWgmyD
zxw~5zegA9<PlCK$Ao%HCPN>lfKh1TGVlxXi0w6Q;2fnTyQ7jJg*&gkph^nK7p!3Ta
z{Ve;v(lj?<CifybT`G-QqFZr5E+9lZJg}=k|JGY8EDWbsfI%+N9vr0by*|2Y=+i$P
z;T*=9k{`nAFaR~8Ew_`k2XgVr9kwUw%yytUB3>fQBT(5R&i7pz63g`6;Vm_fN8FVI
zB(KL^zo9MTd_@|-A-aESQtPDS@RL3lXT<>f^D3g|6qxdz_GO|6<ND=k&HD@ZJ5%ik
zqducwi68)wj;Xgj<1`tqyujEG&+*;65EM5nC5f|7=on;Zy071~;r!JfuC0nD)`LjT
zEW=F;Y;&lYP2Em<ISzR?%;P;XuUKI15caa;P6$<c4vpz_ef3(T@k!jmoFFa1Z%03u
zg4ya@K*;g*8p*$^zX=;N#Aa(Z_|r0<z9!~#NUkkjQ5eE4m47?z_T#Ja%7<H}&`Jq9
z*;(N*L68{2ElEaf`0UiF)uVNhw_7avT}N=z$C=MKblEOsHHpb_3X-W=0!wJ)KZP`u
zyOHJ!T!z#c3ud+G7aBU&!#WMmlVhD~ual-T`CYBdElzl~*%bm(GTY{sOc~6U3IZ3;
zHYDkA6@n}t<5sZUxR=M#T{k1<wES~xPBy7l@8aV|wZ!6?$tlw0`FloHyi<c&;{ImC
zc^ytmTRGD1j+Ti1Bu5(2K5k@E6)!b!rJ*=njSD{CVq{0z#TKd@*u|DWR+j3<J3RSH
zYE?ZzzHIVMDvek82X5Mo#GjwT`>^6DK3k%{rHxEV^{g%VcD(8tV=}6}!&*y+v_25@
zI3|Ra>v50yvUZ5tK_MGA%<g_8slE$l&?ud#TN};c`09jN>2`H|AE~38#A(gPGzO$G
zE%o`q5hhy_;V`IMW?3!$gxuKV$n=;$-K{RFA(DU*b4yPKj+h)Ns86n?!bu74O+>65
z1Rtnp4w9ejZ6+~t=H}4r^N>gmO=8T~k|&L;F|>!>RqDT>4646NQ(tLz`++*FW8tKA
zjm7o)Y4YD$0EBfa!{^cJON6lojwHEuBkqSENgq#>--F3-OR|y<FMlE@R(&jZ1m(3a
z!yWwA4RSH$&cnMEg-2s-6pxatK1=%P_RM7sk&RvYt<wE<RmR3|r<Ng3ky$?QV)08j
z$eU0?S6;||sV@fl-v!clDbyzdga30u4T%b_veO1gkogS09oHvY3HPx644PCX`m4`-
z`VNeBuNw_|KCMw$13^_%)Xgk$%jprG{M^|~U=7=0IW#V`U;SC&>NdTNc8-(oddz2E
z7F{U+qmaIE(G=V;FFArvUU&@NE3tA~#J=_V=1(!leYWVhj@pc8!BE__{;22kvsZ6a
zAJBn`*MKKef^4Jj?-67xiB{#-9JZ1DSmwl{rYdO*<(etx>U>QUfD(z>y_RUPf2IX_
zE=CBe+E7)n;vSsr1ua;WH&x1(lqwB1UJZYZ&`Kf(+uodVOMzN03$hk1ZV%_gqWKsD
z#f%@&?9EQCWr?bh<Q1RZ=cD0DdfhYcNE^LJTpN8Ze{tch-;?U}Ij@;R5Rl|JRJWa^
z$#sELaN$f)kfEIX<;@}0l9})J_=M&+`X*&{Gnd)>7gtAVifY%4z*#qFI|g^ngT8?5
z<RRDVnvdyMOXs9(SV|sbR@|e!bL_({Z^zaKv!a&s-eeEntNTpV2AMG7IE|wfGM8%4
zD`^ImzKrK4^SN4ib-B`M?{uo25nU;+v1D*M$Ix4w_?TPLKLh@cq*LJa9!~6{lLRxm
zk==nHbGlj%P2X0+KJTG+Yz5*j7H#SLvT(P&YI(Z&wAl7i52q0Rw&rSrsThy{PU~~i
zY$}lnCV#Uuz;7aCowiIy&n4ZOM4=BneYoFV*fX2oyU59fb$}4o{E<-2>lDxDFvWV}
zB{t763(GN%xa;c~<bIjkGK*VR=5SnAY0{Xy(KKup{Gbf|8^;gf-KR%cpj$V8TsAmH
zQ@YxzZjvl~heXoV_B|c+!W}<t^;7Vl)_w1<`?~eKj~`ku<o_v<MlglqtxjCdC{(bT
z??k0~>;R~D^r{iZ7|XF%TFr#pcB?uW)YG^M{qFFcpIdsO%~(bXKd(P=dwg2&v$_(Q
zf2-$nF=#uEpfyEe*nt{u>a*VEs8$188v>_=7%{?XD=r%{xL)yz&Be^)KRLnWV2pg-
zQo(DB=Sq&Q0S4-+!4-!6!5Rka1iHvUY(_E7_og`<?sK0wmCuD>!}7mli3z&OS{W=t
zj1Asz-Bn;6f=mfs(<$|n>u86y$XN>seHXqt%Vqb>C%blO2*<f7e26=W^3UK6mu&Ux
zh$Ddw)wljpwW#gt*#2I%NQC2?@^IiMj+bcF>`x+t-y%UK3W<tUxjfd*ZkuUJIp07W
z2Ref}l}f_pp;(TG$UrEbSXey0=zcKNR1^%&cxy_qLz1_P3>&(i<lz8os=?UpIIip;
zHFw_>wx@{k<U&OtxnjzA-66!piq@Ir7Lf%1y-8*;iR*G>4@3NL&gGthmQ^V>)0ba`
zbKmll=E`)lT;J!}%?aq3AP?*3P_{Yo{EE!vryGdj$b0aPj{COKC^E+^L1Phyz0dO|
zJ}+I=_`1%3CE$0?8=9M`kt3w-I`I@WNkG=Cx=ZdAZe|^7z0k-YRhRlcn|z1)b6{yt
z&mUt2wlmq!Qy|J;F_q|WpL9v<{Jm-B!NWCSKruck>h@S#I0WJ@RHECKB#bPjFq!Xg
zd*-|TqNY0Ss)L4#I=Mg{_I5;4qwGswSFqxEXJFF)V$D(G+BpLF{f43Nuwb{`aPWyu
zfYk1A?MtHRSg)D8Q9_|c8Tm|^ZYVG?<<C_+9*M;Fk;1_mWu~VW6M<!7Xr;u01+$jg
zcMtM}Dj(|oiV60{`s}9YMb3ob!LFCHQ}Mc^v-bGx`FtGTv~`y1#m?SIlLJGRnh{u9
z-9z=m54X_Mrn2Ikr6#VJFRnch{61j8X1y-33NnvgO$Obj+Y?w@+~$^24<YVs=@9S}
z|ByO&Tch7KSZ%F~$MmIH)7Yx1P<KrJ*YwBw7$u{Gcb_Yb-t-ijj%`y?Qi5iRHNk_i
zZ!)53#`axFzsf218Q)JHjc9|s%D&FfO9Co8(&_Vu?|;vIS>qpXr3enD@-kNJhl<+r
zYWsoCS&D{S=VkPl;nGa3{ph9yt?rE7p|#29u=&Q|Q4rD;wYh3sWA%Ao>WbX9wJ$!y
zJ4NO}ozeiXGo1|adBg8_62S$-NYX^6aZ1LUTgS*6Gml^`dViGNs3uD!j=T#8rkl(#
z`)<xW%FTpJC{ms<?2oK6%Ck?d`<)dw0`Wt6D@)%;(D8&1wq8EysNnCooOIv}eJoX$
zpi==`I?!Dpr>uHL8gwcypp2mC#W%?z#p;u@z`^g$9$p5E9xP`VI_!udC-KBvoI>+d
zEgfiivCZ>l?v&LDDoa++zmz_?AFX8@7MgKyQkfQ@3(yq*UhXh-xM>Gcc*_YHqu@_5
zPI0z_9{Ae(x{X-0eJ<3Qi)IsK$2Oie-USZ%if7iBF>5Fg^I@Z>Lsl*2r|EKFBi=H7
zrnOu7eBCxl(k6}|m6`QN2%7fr1ZyYk`OVJg#9I23d)p9B2Stlp9FuYZCl0f&KXZzZ
zm?T6~0+$?o)G=k;{$k1RyrMkzvGm31lJPt_vd)Ev{!=Z1F&eY5DHxQC<^ED(P?2Jo
z$f5D&*%HC?SaWHu{5!O-YV*T$Oq;3%uQ&boPG{>3dzuKI#>3hoiqGS$PuG6a1&)!m
zJlIaLByOkWrE5PxqG$Dl`9lf+>Wu5K{^Fw5zoMw&lFYWFK$ZEpBA9LIpe?SyDo-DE
zvJ6MdJ`+KISPy1{5J%_0IQtkD7M23uf*T@SDua%<g3G}!MA3Tzg7fJkTUn{}8|!qA
zyV9KzG@;UfF);WCei}{5^z5s8R1mm}s=fWyNCh|i6?&~PA@f8qFS8_J^#b^a_MAC8
zAGm9!*j)Z`8+mzP86OIUx0m+`rElJc9o>80&ec_0?y;q%6v;7OBYuZ`&LNMWQC>&<
zu8~G0#Wih$m~FIR!0sZad)<Kyam2J*st)f(tys$*CoXu3QizgiP2^Drnfsx;I~U3k
zKVz{ut|LZ{3M^_(w2^QHs=w4sSj3kp|60A@=O_|5SrRhq7!H^_Xjdx30&n{4TaP=r
z2Q&A6sXP|A_I{Wto!;(%SAP;I`;l2rvC&A3o{-q<B_s~Qg@FrUWhpwBXO)IenzhU^
zM2JXQQMkT;AAg9pQQt-BBKP8jTBH+4no_L!%ZK<oHw>_@yDE>#;h=A@3jvH#>5{M+
z+LqX{wQU=Fz0R^XjJW%$ora!#V@WgBV6UK!hfBBUvTCA5;S7j34u0^gqj{YjunvtW
z{HUO)S?7G8x$x_q0o^7;plFy%iQ5myqs5vc_Lqn>V)@enr)N{ajN^OR{hDB_L`nCz
zRFF1`uc_qIXMv%MZ@r6fBlZE<rE{qZ7P=x!y^buVMYsSqM=5;1N!?TH#*jv+OI!m)
zlyJ8;7}$jcS>t|4Ji)vsp&~ljq}5<a&GXJh%W@OwJ1iyCV42bdIXkjn$}CZ$BZDAV
zEoY(#Eo8@dL<=;64`yLmi=`-bebWX0HK>xl)fU(AC?w1N1RRAH&o8D3ksN}MJKP#N
z>lZG<S|I+IJ!v79)Rg~WzsZTW55Cv*C(<TxkN^6`S+c?ImCXw&)+np^b*A6za#(;+
z;C_oBuU6^U?e=t@q2NUNnuymyG{-~H1tm9NT`B(~$-_Zj!1XKdWrVlICtDF%h|}X+
zj%&@XAkK*#7EFHOWPE|fc;dv~<g4X|BeUiDgIt+8K;=RF3Ihg{#Ov2<Tm{mzS9_+Z
z_!<4|)}JZS=Y6jxS{`j)ZS=)fTWTh)x@3h*eB`vkbWf@L!P2t&F7J4u3jK?ri`ChC
zuCdsmq4+ZT$w{=Z<BE7S=Ak&0$|+O-H%F~Uhx2}6RzGW<kukM1*g0JCaC3gpc-ad{
zy)6pz5Jh=t6UVPy-(t^nWkJm@ni!>|yTZv=3bkmP^`GpCPRTG7DgLq5nxpb^Y!Iun
zU}@w%h@t%Pi1Yh>(1j|k1Qp1m9f^>rqT=HD{B1=$bI1hk%e<IssUZW6cS3hcc<xM>
zU`{y3D69enEXV#W`A(6#NRg95ibNh{RRXQ~g%`kUgc%LpBPrEe<Avv6L1z!D5aT3f
zCJgbk|L%<~s)lTp>0pwvAsFWcx8C`5HK}b{4}v0Yoveyh=Vn}L-{5?;prGI;fDTs^
z>!p!Jr<>UHDx9r~b`omYBQ1H}w|CQ%0u5@Cce7@b|Ip4h{{4O0h0l>Xqd%Fa>fz-)
z|KV|7`#IGpkC@MCJd#Wd^UtzWL%_YjI|~HbnFChDFsVD|%!=qETBR3O`l;@lWjxOd
zsFD}R$hLO#;%}+5{LB9Lsuj`*d90;hTBX7TL2@T+-@Vn9#!7`+%$p(<dS#T_WLfAn
z+F~E7S8iN=%<Zr37@6)I0BYh#>XxaNm(Y!842k;hOZ#mmqoDEYa-AfzgSoDh5Z6!)
z{$H~rA9PXaGCghiWdpGxDfYj75CH9y3sh?T>%ZlFca(o)PQrlNsp!yv`IQXfYI#%P
znz9>zy$wjl(m@!Z6zVYpMc@)auMdqs1X?8Y4X26R^gRqrUn9O`qWK078EW6|eNPgl
z?%DRwy6U=j&+lki@YM6gT>Q!dQ5FwSS_(gR2B6Hpdk41F8{Vo!=`-=JZip~Uw2fTx
z?*Md*{({0ffokv(R6je!{`(lE3G3x@BI$|!25fY2Ztk3xlK1Cg-Bw%Ed%Rn^b%63f
z=T4ssb|4>y5c>V5|0$m6^0Qms1Mx*lm+z;@p(G6(5)o;9^F=q*^$$xg!Ac2;MQ>jW
z4@4_1uQMxu?M+uP8G`3>yi7bx>5_XbC>;Za`jcn=_8^vc+EapFo5QNb!}(>v4+O>M
zHi##(ao7uI+7<>mUYlcI?(|8b<i{C5WXfG7ChCFp@Cl*G2OK0q;p1pkb4>nJi%SdT
zT;|7wbc~Y9)aJr+#R8X2YfS!sr5iYg1>IuvDL$*PFYs9o|20^3;T+SExwN-A27fzr
z=C4AnalHKeTEMQI&N7z3u|WLb;<GpPA$EyoD!b+&UAwocQnnjI;a#K^*5kjN|DmPT
z21j31vxBVc1$U>(KHuu-@-11qhAXX*WX;t5c)xH!ObFoB6N}bvSZqO*)Tx;K^mX1T
zfV}?KCzosa%dZxO>{+L6DnOEsNuJ6YvZ9=}aqv4UVkDAi8Wv@I77{l3<HO56>vUE_
zuZk%Ee*mH2oeA*~b1BQ5A-)N(?uR1kVvL$t52XqEMSnrv(Zy`zK>_%B5J>WKk=Sf{
zs%HGrTFs**%3WXjKwWfNoJ;Tx04N!OoSl#$8#g!y2<?Sd{oHOmXeUrCQn?X{1l&B#
z>Xb||`R7Md>{c3e!tTy~L?47;upG+aDiEWF!M)+6XB%NhU@{lQ!U{XRIYL=WPvblh
zh=*nB3sp?I!S%cx(Uz>yabM;OERf$i{_M>DL{Q_l8>!>1{39}`CVzLR)H`AW!5~Qb
zsr?+68@@});^qMqxU4&*bTYjv<c@R#*-@kcoZnEI<j(j@Rjtt(%A|z-lU@|sjt9q>
zKUR<A`)Q+M)^8nF@};pGtQ0d%s5=bDu3UfFY=$lSLcJ8W4S4h4H{ujMWt9{>8J3hd
zDaW?7v81=We><P(0{D45<*P8BRo_PIHL(&A*3W$veEL3mEO0z;Tm$^uxp1VZ+Q@Gx
zsfQ4}ERFr7hl8$T88mW%aRpd?W_%3UT}Jf_mM#nn56@C7QlDUHd&-26G*QQ&6A=%!
zC*Kz+m6b7GueBj(Oa-eb;#ItA&}0Ric&VUlHfKr*U&wdj{&nbvD8vp5QZ4ivcY9>x
zk;JR763-QRvC(~#8%zu-CKj0dkvx>N-WW85>O{-+G3tuMWAZxG9{_Y4fD@Dx!#?p|
z@rt^m(WF&;KWWkY_E5qIXIy8c(vQKkDz0wuFSXY)(;r!vFIt%BjH9&9{86ya8z=*K
z2=tmN?v)3tqHqDDGQz`T{u|Hwd37~MLQnMBtdmc{Y39MUtb$kQ^I{(tNsuk=4M84y
z{{Ig*R!{l=&y9sp3I9LbSc1ec$RxsDV}C2U_P8!k<w1R+*=h%EhN|=@F^J&8{wOL0
zoq7i{s&C9z7!0&u(R+G(Fhj6T5y4_2hr<kWzmJn<={$1}4^qH9@}8~ocaeiBJTxZJ
zJccHN`J93V%z4Koly6z+JcWCq8Z(qzblYrKMv?7(PIJ5J{7=4r|M2nc(zD&%s?GKN
z;m~thF7NH!_$7by3m(b=U+=fk{`K?Kd9GszA#Z_b07CtiCu?Q+w2ek<T1vFIGJ`?i
zzY2{RLg(p*oLiMQ_I0u^d+mc3=%M9)7`+J@?d)DpRLF5yPJie*?16LtGGd`s9&R#(
z2EzRozOPBRv>;}C%&EGM&R<6BsfUH4*zmL7UP4B<%1>V)85~Mo28E|r<@mZ?@Bvs;
z2e9KF{=ngevW0;~?MDF?3-)aC&@)JNzLYJyUQ9fXU~+cOG3pANW|Ai(v?6dEOuszr
zTECb)|M+ri1quu0KE@2)wT6fu%v7No?JX-Fbl<ccAOA~(YH=XSHI_3K%vogkitS?t
zy?jZ`{yVAD<ww6*f}-h{#F9^>@#U299TdU5ze)kSZ559Wue%>=2_&#Ie=ob)*h1yJ
zk+?)q6_~jF5p-%SnntmGw`h2szsdAdc3f-(N1Gog&k@Q~cfj1tA9n}gve5#bi2zg*
z<UH+$%A+mr*Ep}K0rk5Pso7j<t79ABsgrR$QlIY1i&^im|I*|{10*2$zk+#BCn_mB
zn6gxB{sW?+`y6m{%+s6u;6gw&)$x->SD}VyUYje5Y_n?2#4cWK6E6}50jzbBO%L4U
z038;S>fPFc8k}YRxz20o^-7a7jm284?>FluCxGf_u)5|c9iVhsPe|P6ys<8zzz5J@
z^B&n^<pO0}0-@EL5ig>RsZyPnZaF5;0_8xn$?O0}J;Ao;T2WAv|CYvLb%7*@MNGx9
z_oG#V;A#*K@Th&I--&qCj1|pA0FJ{D%X50@@wa!LBk8fQ-s4%<4Xmmp8ycI()5DRn
zXYm`h@4%lpJP7ZP*Ww<Lr_%(^v<OY?@Q-yqx}N=;E_<&iX1Wg;?5R?eB_TMf_%~-w
z(}ikQE!2RI@3M_qI!DyfsQ$jlVE=k!L;X0Lp7dMw<F^>c<GzujIrFhH`-5W$MNF0%
zS>crnLMJOpF5oLjNP?70aE5rcv8alF$BL=Z6QsYDVqoAycn}#Km#j_4V4?2tca)4T
zRndI!RA{KorW818b&UAhyG6h+a->U1P8t3ol68B&@*3OuhJS|<-=L1#XguLj+gski
z_Pc*Ae)!7=(TFTbo9Y3zSTb~OoKg)P5X^4aI<#EZob^mXFD?JK{H(G{+yO;WA0w6d
zaz<A}?w-`|+qu*;*!h}07Fo)Vv8MqPx}@HU51OV>6F=@i8d-+gKi|^}-M@?dHZ{%m
zW3eF(%=krqZmV?PPz?Y(zC9DS*$MvIy(KVO;8h=Xw=%%QBFV6AaG?*YeA_Pz_f;(i
z>e`Ei$4^pEFa2R{Ch-@$xu|B9i6fDMK=8_VceoJbuB#vdhi+t@6<A43|CxQ`-fv4!
zAR{B=@b|KHwSE`8I*r!q+_k`KX4kaZuJ`Sgwu;p5;yRTgE9`oHlD=$e`PU4fPhT{c
zd&Wnp+NK&*GE}SWi0S8qpFKG$Z7Bh@+ih<jwl@35Z@1|Q&uZ@#@-QAOk$YM7>_R3!
z46u*yvg%mB%7ddl?_m$6OZ#cu!;|saFF0&hmpXFyfPFmf3x(T)v0g8aE!Nmj6wuKU
zC95BTTiqzVq~<HjHW-T5u<YMnLWI8^_g)i)!evS?-syzz!3RzZ%H#fChBf*}ce&0<
zx(+1um1jr%@jkY5w3`szJR~~%a0cAJdLs3`SaT7uw9WVtjBvF;LT>3cGwqp+=5W^G
z|GL*l;7KVMY}gKO*e8VY$uJZ2N}z9tbMuTK-CoNDy^eSjh4pU(qzf4`(2R1*)JlNr
zGQ^1WpkNvMSvmcKCX4MYQ?WcU#E<|x0%0ewN`xYMcs|AFIG|P$P_#XF{1Fz`SJj<m
z1d~jf#(t2bfuIb;3?qWboq@eDo_hScDfXUBRHXxpvvexKi*u*KaCUFUyI(m$X>LY7
z#O)sn6q%o_aunCfL;`s?N76WA^cb9Xtdh+WLRjuWocZ>%d1d!%&l??SB&M#p_f@K;
z<tJjh5TWZSala2Zpmgw|{|o;DmY`)Hk=K{&_e@I_({xg?25JSeUCd6Gt#&7`iG{E}
ztg$?&cB;Da!5zJv&iuYc$NSp8?YvA-s#hvHo#zR%YO=VMd9c4si(&Z6?6&Dkv)}rY
z!|?;U!>ghEeOkAz$5!SogD2F()){&otw&R-G?8ew&C(uCaa?c&R2K>8Ln2Pb*4T)>
z*-!@46<|>=_<*Ek@mrJdB_03B1U-ZehJ;Y6XPV1b<m}FPg#B~1W7N?3QPHJ6sAw(J
zuL?m6Wao=lZ8s<L;5fp<?D?c*Zqg@g^2VtC-A)w{3wD!knJZ&@=dzcxK@g9ji<&)r
zNQ>D&%H)2Kbk3<9dEjGBje5V$G+4@+O!}T?EGssAPNkMe(|mIbq?y~&cvHtGNabv!
z;MpB|UqX+^@~g!!lBsJL?0XDiaXc{X5#538I_@_JDlQ(3<1M>m0}SJ`;R`f{n@YoA
zj>D6V*6l>TGtJ_U8Pl^jsx9E>y7|bI?E_RKBRWNCc+s<R&v_<Tw5IsfMc9!N9N_Cd
zxRMx5{<GUWlNN;hk5QZI<yCnU2QtgXgFl)0;C8C@e~bd^hVfKlZ!ITfCBl!2sCn8F
z3}0y#jz4|idwJ?cdK=B|YwAx$0jgO-9?;BwHREa#uv2HXE7fVsm`h5<tDGf<Zg_up
z>M$a&{H+MOs^sNz0E7SV22VmH&~nT<ek?6>M)#(Y50kpgZMS&BJe{M4Px|a`BN#4V
zn2^!s1aj5|bwv9;KmRi9>|H?9iHxKl0<VbR{+c(YKE+y0qay=4Tae{y^O!2%j6Rpg
z-R0*r0w5O5T(exy9|!j+k<XdZ7(Dh1uJW}ML#Q-RVsOYd<X2EKWtM7h#t=jAsd5zo
zGZCe&b~9ey>ehT}hDiPWd7tmYHM};TlSlmS@O{7AGY`p*0gytU6ra_qSruT4+$Z=g
zFKFGD!ZhJsz~lY23TQk>_NM!iCRI2ng3}G}iDdIr>b{xBL|8wi%hTPKsAIs0Xo+sM
z*>}h8!SDDC`{67m=FiQpSywF|h%Wbk311I~BGVphMWEg9rNLayYNdSzUG@bam1%89
z0r)!3!InVP60<)7@9FN5)$zPWzQxSl-hhrIHrb9!Q-d8$nYqVt;FkZ-pBzH*8O;sn
ztb`L0-pLdmS<A$9w^Fha$!10JQN9KPNnIykCXu(H>S@Tee0DwOfZ#0H?D1Oitu&>_
zicqzACz?XnocEC{!xT&iGAVn)rqo$GD-%uKvD7;yZ?we%+q?V(vCH@=@W4$H&}-$K
zm&0CM$Aw1IS&r-m;1vel3*H^LA12@|xyN+90T?HccU{#+kn|bQ6p}laDM|kHfijfg
zl{jOb%_brKQaz}=)&DbnQ(}KU?3%YJct&6KGnmjQN*%Ln<Xs-B(2oim$Yr9}0Er5W
zZgJfT7exspV>?WJ0;YBa^2*C8L!8Cd-S30$pc3_^V!#zWYyF}hw6dtFTWzbuTVqV4
z(`5FT)b19;VKF`44OY&P(ey@Z0C~eGBy)>YnF1RGhvJW3&X@NJ<>)$KSnYO)3;L|b
z)1i$Ae)iBE&lG7~o9YU^JiSj5@?I@S!*NM1xNJi!Vf->Kd8(_ZWlGt+X1uhd#MY6u
z)gFo+Qx-z<Y8$+^>K#|AD}U#%Q~cS}VzH`?15N|TOWG+^Df)smhD>PcCDx=dT;cI?
zAd)At5%pOi-VF3?%E9!a&t4b@Ji_u&Vwby;gKFp6JEN<8Zqf~bD_vbrVeZmEDmuTu
zMRQ|XZjHg0SzAl%n65|cYy~uU&5Xj@HA!0ySD`*-S&4sZKhGHc%pwz_zRub@@mnI@
zCPd8C>hD}Tr@OGVp>~PaliJpCh2`99MFt$_Z7u(8);`<dj4zf19Ue>v{$FP8?s>qh
zoku4J`@H)~9yEPEfd1B>7Z!51#nS<*a3OjxcnH;Tvu$#@afcs6@=B;XxSixVBm(pQ
zHf%39B+VD7&^VIVIY0jhf`)<5M?2-Ga9xL4n)~AZ<8!qc5R$l>J9&mGbMvE#uLw?L
zeFn%?yT+V}W)~_OmG!<p3P5Ak(-u#G6Dpr-3iu8RmF^Bid=O5lI;Ah_lYAk}Fl@hF
z2p#+QyS7Ya=MDw6&*idqILEvwPt(Ha?oK!QMQO;QgfYbs&KjIooSaO{(=oBS;$#EX
zbE{6Z@~8BFpq5BsbF4WszCMZe20v|K!*|EP9C~P~4CbS8!uNTHye1zG(-}G#30|uV
zA<ku*J#SM?=2~aKE|9Vox~P=}O@YpZl7vKxxR($SNl5#!QJ5_f$yEm5r(I`P2=pnP
zS|TOc;=Kg2q|y{ez~O&+aB}f4#3}2g&%+Fezg45(`AUz{)HW1xm^67u9B}1PkGArT
z(=OEoqGU0XQE@7^`$mwON<Agx$&PEu1P%;>p786zk=`E3U{Q*Fq&LI5YrSc61`@D#
ziehq|1>NM245u;3xKn##B3kjJ0!1&r#_~XF<(tfoBc-P=Yraj_t9h=xjP<5m4UCsD
zIuBo8np4nv23<Y;M&itJdB1*St73MoODJ#ydcM{h%xl#|N@O=-E76HN)5i`0k@^S^
zH)O43=~y3y5Js=FJ_qyA_nPo}-@F=L#snt+$&o;NR@ln$rcTLXZPhhiBFGI5c0ntX
zMJxQz@U74`SMi}$VuXVxq$ygN9}#P^&fO}<{MXK=VCSKrzeh6!U^H7t?2(DfvEZ=s
zw*F)q_p_l(oFXH&Ok1E3l7I5jBQK$>!`5goYU)uU6Wkh5Ob}TmSvRqphH%9D@~#bq
zFfSy7LC8I>shCZE(P3KD_|1m3FNx+OA@L{Hg+s-{O(2OK|3jpCz)sqlV0u0blJ~9V
zOFP5bM<AA_(KQc|L}TW|mKa*gh)Vh3;f!q2kE8e!PR<?0WNyQn2H=9UiRJLMT453(
zb0MV=+Z(+pgZUm#p=X1QY@~rnxUt`s1RJ5>m74Q#&7e<UR0_8Z?hLdVLy@C>hlIN3
zEtHlkr&;`b)};XmSIV+d1tjxteM_Zbrt;8~D7%Q<P_Sqq`o`D*qT%`I^<k+FG|374
zsD`bo`SEnF+UxmSNgA7R<mjN^haz?BPm;^c=N~k#Ki9PgLE9oM2t;P|DA~ZGhLApe
zf5X>YsyU2IvMaFZ72|w(9W+eaANfl9)Uo8p&wG8ZqUSB^to@>c(YDJ{(dtC@Wpdtt
zl0|W?)`U$X^_>E+YN4Z{ECAv{iNK0uAaY(zpjC!sWowKboV1dd8>c5sT{1OhoYI6#
z1EgzYk7tl=yxK2&o(5M0E4;@aGr=-bXls+%ucxCL44?M{Zjr_-5(40fxcrMLOS_}i
zD-NbWs+_2Mw|%7^D-*uh%3Jr}7TIr%V#r%OBD1wD5J{-XV8)j0MUa&e`G|4`&bW}W
zvHJrJH>!_Ql)%)DR=xh4ca3f4mjNK1MP7Q~ZdcH0a?1JegFs2x$I}FFzPGpcFiSa7
z1jf;siUStyy%RvYaxB0Yg*qJEQS8EopPkt}0Lj(Y%b3;TKrBOMqhdq_WH-T+?}bQ_
zCb@My`h!5#{N`F?Xq*0N%M0rk?=0({oMz<&7D&FPaTda+WV%w0eI8rd)tc8%R68c2
zjloYs$R1jWfIB|hzs%~KeEA-pSH80Unjzl&Vo7I9vK$y=x8`eWs*jtZoSw}b@Gpps
z2h1Nar*{{G#z%ABGbLpsl7zQ1N%Xuiovx)DP&a~8l*s9$XrR?4i4|bhg)h|R^m)Cr
z!-SM66fZeNNTHBTUReK+rKvUWk8mimV+$}C_GATc;XbV!wPLUXu?eOM(97#ES~f61
zFoM5NTf5^;FlsXF-MWFdTFs^*k@j%}FD#V@X-M$U1gH@k2|+O`8ZdwUJ2=6G*Es#1
z=9PWYW4fV$%w=fe<43zc!3k&E0fca%c_8ZU2AT6=A~@W4PjEgHr0C0?DFG0Pq)(cE
z#}=M4$1!Qe1)&lbwypS7$EHT|pJgmDlr?7x)>lR0w>}&)T+4GlTEk~M8B?SYNCes~
z2{MX(AD_yoR*S+`NQp@Iqfe5|$&Af6f+%kTt;spDZY%cyIj-KNcTXt}w$}}JY`?}&
zf+G2lt8;To01g(@h0t&K+ou*up8<8QiJB-KEx_l*;*#8-Jnfxtd(`^5OBT(3#v{q*
zN7$DV$=btX3df<@oXvZgPo8Nk2~tNlB~^>5>K3tu5-m$Dr%rh2D(}2i0L$@2KLx$k
z2iqfmu%S}leu&K2mo^WW!DO<k)qY1x7oX*G@FMain$h_N79x%$MTGK=&oMS8k%6&@
zt;AVKGrBaOEIlZu<DP|26e2B$^Wrvv!$lO1#Ax#tZ|Zb9;PujbZud3*F$hG|I?&(@
zt3@tOTgo0xQ)3#9=JK$6o|IYc##4dO?{6Y1+)lxXRKnVL*E6J^ptFwqgxl71uP)t*
zT2d$<<W9EZDKFJ(BmI@`XSds#D2J!~ldc~oR;PKVExcNu&$>hkd7TuFOUG}}1SDYG
zJp0KCT$esvP0OY@JgT~21NvqlPS}C(%7z<|Oj5QycMPc(xbc*9MX_@82K5|^ssDVf
zHuTZA^WNb$Sr>QWd)lts@k>t`#UFJeG1g^AGpWmopY~4)D`w#j$O*#MDDIk`QfBU!
zu8;Tn*tX;{w~3T6B_D-+Hg;rf9XJn(Mt;>s))p^~>L}z$pRFq_4~2(q+p;nGc|903
zj@Ea2f~m0B<pfSy5V;<mh?OC22Mr0&CM#Ip(d+uXuw=WP_jsT4`kAY|H4|tzy8cuW
zuD#>0j(<larbwf7WElY(-3B+SJ-wjaHE*>K{*Da$MlSgou@(dpzbq%LgCv^gvnzd5
z-c(twv+<IeWDiA3bG<+u^LgEj-#8ysPRQyir$2y@q0M2oQ8biG421^EQdvmp?t6=y
zfO}=CpGks9hd(o`$o!*ptY=B};Ai*-71#)XMHqMg?YU<sc)1smDR-!~6bXc)+5#MO
zGPjD=aG4$cm`)B2Yy_1o=Tr!WN=gvhd+}s$+t0>Pc&A2PVNKF!@7}YCizln}4I!&7
z@DfZ6u`GRgAB>jvQL_H~tcp&#9=oyXm?1h**(+sO;0|uDnjef^svk0TRk6YZ6Wv!0
z392McB9rtFLLZYQ^yQ_EmUo_C@?z)LS!pRoe;x0QqVY%Y4lmS!M~fz9b37z<yb7_3
zp5V{iO|pJaI^V1v=_%D&%bUux+SV{Kt}w176;&sQM(?uUxL7b<Dl{Z{b#(_W^Ne1%
zt<$v@3%be<LMhsJ-Nd(9Z!a_7>J?~A`32&8QB7*bKd-=@a8FCkG{3|g<JRl-HX5XP
zyJ2Ad5`|6d)_FX8U3~A4lt}%F2#Ev*p}9EXUf5gij&AXC$%9i3w79U_ZHljY0-+^i
zKr}k(XQ8vx>ii`StO<0bf1HZZR3|FMwpLFo6#BX(C+JNn2YTu7q+RkB`V89h;{}t3
zDoYlWbC2dI{LX6#x2X<eP!0pa1ntOzcH6E>ccLd{dW$uuC#X8G6KHUAloC<NxCgVJ
zvd|72`OZ~<eRqpntn>?mH(rOwzFE~5Ew*m_eIT;ixy)ZJkh~ev!(q=;Vo9tOCrX!L
zcYP=wH6zG&w)I+T7;L;vHty1vH9+q00ZkbK8i|o(n2aJ42|r#xafw#ltOl^zThl6b
zZpM}m<Z<RpZhQi4`-J_twJNla39p2}b--v4SX488bJz0axhFheSeH@aYfa7tQ`(%o
z5tmrpdqp_l#k+(Di`{QrOKJy|T%S#;HWZYM;dEp@&actwJnhkvnk3KfJ-r0$HeK_z
z8T~jHOOsT6%V`NS-%aVl2(hCL{9WPQ>#w%dnnh<_QMstUwisA_IyYuDdT0tPKhngK
z{TQg6SaWj7gnN+as~@*^Qh2QUN#wb#Dx%d>m&jA-l5s>A-*AoWzuZ2Gf80I@oG4;5
zaDE7oH~tKI5Uvk|e!&*giR``zn9-`fyv%o*;%z4)%e_24BuRpEGseB9ZCdBgcrad;
z{@nRueP|+P_QY=8dip}$vK8g_GQ=Cf;n3~t#an?HBW`LnF#WZJap{J-^BJi6-i&xR
z$V7|i%nPWT<N9)uT<O7R4*4*`^Xa2hmJ<=2PCBC49b*4kh;EnxFVyF@42jo6(?9vo
zHnhsVv|J^bG#{j}oBxz)u~d93byE0w&a4Cs>pE!%;xZ+A_ME>TG4gSH54I#Wc^ifc
z@(3k}v(ERl4d`!DL0B|)5snLwNF~Yu!&I~+CO6u>X1X)=<#<aE;Jkb`mi^suX$M&%
zgKz0&{WseZ%`o|Gu!jTo1gAieoxh8EqRnPb)rdsn6=`2d9l5+h1Y3&0zea!COh&5F
zJbS}I{5p}*CL0O%c5hPc2~-nLXORU5iwKk<gd<y`!F-_)ldESfpE))QqExw^^=bSN
zLf(_ky{jc%L%zTE<i)ZN5>vYG+;Mcmp#>U~^f?e%0Y5#IUF^HVcKZQwO+$d8T_hPS
zoB2fiLjuWk1q3<*MOba<-E~RlGtG^C^aX>#c%Ojatd$X*E97$T36Midg?3(b1bDd#
z%&U3>vfGjxu<`=kMW1P&usSMH!Nm&WQm{{Zo%_=fK&Zr>A8c=^2Sso@ral%aqGqpq
zH0*bcA$%&^15ra{hIVP1zYHIP+?`3XBObg9`_P6lh0_YMv?6U`3|p>`KNfy0LB95=
zNjepBe&BCeHGGGXc87{XXR>q!(*Q$>CbdJbgOiaNU#hFTHhVl%BGkgVEx{1p|A3=~
zDCy>M__A`WSX@lX;^ehNF3SHbqhT?*SUXUE1afp3N)NrdXQ80|TFdCBkcRyD3$joy
zHm4JX>=1kogD*f0e)5HktvdQhs&Pz2t-7-yWpEuJPKq3mlcryReZpdMeXdqTU~{@6
zUcY|r=O7+QEfr1Pa!WuOaOyp_GCZ_0LcK6TtuJQM`g^A~vEeqgPI|S~d3AWgpXU1)
ztr`4|E~SI+s44bu?OXx-z#fS2>H3FIFC}fir!i!Py-M8;Km;BBo>?B*t1`OW?sO&3
zGmzh;y?<2t?!_Hifn@-1g>ix0AbZxYn~%4{qP=YA@X(FHYkz?!@X?x%TFF{^m*G<Y
zo6(@f5C3i3jb*wbCLCo9em9j5u{X$m)AUjmoj*AxG3ZweLcR&AP|kBuh#YrmH0rsw
z^obs244i_33-j^${1AQ+Na2ub2%pmW9n9i`2(vOYbf{d6TpQ?dkYq$|KkB5BqI`eP
zniVIG+hN$@SHu2w_O<CyM&KI+<wgg(Ydn%53YmZ`ZcP4RghOy+X)Q1&>9Nd5XH6I1
zzrH?R&~|=r+o=@1G#0&nz7ua%V7Fu7MwdKlG=Qn~8RJP8u2qWg6~G}ROF%u}EEykq
zSdBm5>0|Pr7JV&pyLQtuOjTW`av_~T9nEX9gFJEITu*%HWTkLPNMos5qx6zXdBEQp
zNzzeVXt>0SjoWT6h3Si-vzk*mEjC@KWMa29c*iqL>B9eeOzU9s9w(z9e8q``-$gu6
zL1(hyClIkMDH)MkMTk~Z@zqvPCJMQ(IfQz+MMl>f_ep@Bz3DC#OQ}UOt9T=)L?|oo
z5ZVb*X>D!o;ald+2_SdjaP^^P`t+Jv9rdO9Co0Pqb8}!46nO8Iu5wiO*-(2x`@-0*
z#+_#6O2|EG_LiVS>{tG)P8o4nuk|q{pu;ql_^?nYMRzK-$xc^bbNLWcQY=G8ApU?l
zBFLx=w<Hees9K#h20vB%IN9z-5MSL^!yS*7iu2-&3;X&UznTnx4-AbFzY#E%d+N!j
za^<rC->X*PT&d2K66<V<R$kkqCJaWt_}h-IWP-Vw-_4h(0J}c&@Fn~lkQwKz8ZZpV
zVVO2T?!=GO++MU8GUvLQxXiaSh>NSgEricyi#o2(O&vd#riJO%jwQ2+I->InFG9J(
zT+qJrfy@ovJTFY>r?QnP^}Gcgd;<_$^6+Znm#VB#*%KhEcOo`Am`Mlt$i>>69M)cf
zIC^6bf8S&s2g)ApRuCVVU;3@LvFc;+&yJ4Dw`yXVH5n5=6+HGwX0xYKRLYpbqFr03
z)Se=xD@rEkX+fgwb(t$?8NB}SO@xrH8^cB%TA<yr-3b!|FR#rMqON)ZRy$RM7=q_I
zjBtH7_a(x3-}71D1Ffc_8dEAvL>AgBVj%7SOVEbQM^JDa!S-tGfxk_DC@_sb<4qx%
zs+`tz1ubJh*_V&OHbO_`#B~|LfmMQ*a>Uh|XC*Ov!UJxW;kNK2M3QY;Ks;|~(<?z>
z#?<n>zc5zA2`xYbT;XIc;xG8(VAbZDNJh;{#3+6sCOdQW!cELOG>zAh{`p3$pKugo
z0f(G(*u^^+#~ivqPW|M|&QFGHqr};3LfPBP`<8>{YkT!1h8-N}>>_eMVf5In=WQqc
zY*|UazR@i{^UNq`L3Y2sM@PYQjmwroPa>59Mfk#`i_z_)Q58A~%q2oap?q3`A>vFk
z?kd|lv~95UT2ko<dT3T!<N10Ra7u*>p`?4eS%F`|hx1FhW%BxpJtIm(8^AU;2fx=$
zImSfdgNCc)T8x$qK395%f_qsDN6c1)zCJUG7UCl+59tAxEB#?5x0abDnK~>2+x`ja
zW=`e7m^Zw?Hx7PnX<$PinAW6)$Rx^8?LU919Xl;af?P$(q`VRt#EE1f#U0b37GLoh
zRu2fO7vULO`?{DEIqToU5`l(-x3<!F5NP(+{C#$<6beu;R4HK|2{Tp%e+%a@%#-*W
z&o?LKeorV_A*(@BP8ObpJRppu-C*qu9~C`kMGVlA#%9YHJ_;o8TT-7aSH^GV*R7*#
zQdA61pyEUBDMcS5)O84o5J_Vl*Dxevgn2O)Pvq^yjv800t+QkFmEVx-`)K=JgA-<h
zD~-mo9la9n-#piL-y{~zq|?Wi-Mprf!b|&^Z$=u5Nj2^0^{7gi4+<Id?FY@3&aYt~
zO}20tUycCcXkzl9btbFq9NTIHy4`4}XR&2=V)lwWx2i8u4cG<nlCS0wQVdfGcfj(8
zPv5IkKvip4kIVPZ4i2~;xjzfJEU{>$<wEGYq7;T*bAKe}g;JZO<;}ZVSeN!J0hc2s
zY(%7KFJRc}3<%9ve|Db!9er}t1^BZLwky_?W{+FM6T`NnyKcCSCbEL7FXBPdzke;Q
zP0g>o(a0VB5;<61E8t>JX^U<B#AA&7@Zx+hZnAh=b>dxx<R{%e&wFXuXn&v0V>vSt
zOcinZI2CkWQe06^pZlUFwuY=WLm3IX%&nM|w-6ElhDV%&SUlP`@--roZ1pl_<1p0n
zFiX^pNFf573NkwU_kzaXBax7jHbWk0$Q+hW&H~mu4rGb;*nw|KHDkqKjw_*^>`9*i
z_5f%=wlBf*2JP_W`4$<6Ax^SxZ&XL(au?2<MYHZ(^R<xscsh@h5^r<hI;-jE{~_%y
z!=n1$aA8V@5QYwkAp{9Y>28pek}m1)?nYFQl9JA$JBAi1rBiz7?(VaH^*`r&zrLS_
z4-6M;uh`GJpF4;Xg?&-m0z3o03wkEaHBb~PDbn=mIbNtTirCM5-Q22`#w1+XuQ?Fd
zDy<aBL3-v_f8cxcrGQq}Zn;lU;<KQw4(wc|rJoicc9DrV3j1*7>DK=WgVE*ZiMu5y
zF*in*bR!4evw_W(Ax69%dI6tM72AJG=6sXu3$7!_xyQP0Za(O$d=nZG4ey#s1?(PC
z&-&EKU(Lh~->3LT2Hq;+3WWr+0q6T=^_%TgRx*ycI^dl4WHVG{RI*#v%1)s1CGzpt
za8zI1fH9-Qt}*kDKYCXLf)tF{!J6B5!QpydpS;PPvH}iBE^lcij*CSaz;q+=sw&{t
z;FdSyVR5{L;Km4Y_3h&z;CAEs$XUK;a_7HwGWu8B?ELF33frBR{ArY3h%C^a=W5RL
zyE&y+8({B3-kJZ}i&f*V><RN<DXyS3&?~rVS5?(@`lMTc!CB$zqGcn~%skgmO!Ya0
zZx<Q_SIyDk{dH&Kez3bVR9a^xCR^lCOUNM34Z|Ka1%~}R;AZEGN(j1<GDX-pDN&c{
zbmVYlH8Adh#dPbn?{M3$Dl|&Hz<z}twY~}F)RQ%=7u;*|45iWche*wMs=s+B-X-Wl
zUvKq|h~x`oC~Mn8|A%gI5F)a@^!l!<EusN(s9W)ROY!t<O49_F`&wo}eJYa7+e&go
zNMm9bu(?U#&h85l*z6+NEyK*R$e786)l<%ub)S70Bygpy6<rYcE~`;xm6@V&{bP!N
zjezdl3%M!!83K+vc<s9PBOVeNYPf$b%XSkVbp!JUP1PS~#-a_S&G35h*gg(ay{H2+
zhblvisbpYMr_s73*YHgP|32u2veZ+UwOEx-<uZl%GerD#!xMY31fQIUCz@R?FNk!7
z{gJJIB{J68XdYu_IDP3>Wk*LH?k6@$CDsJb`JzaBG<luTTzsFW)ws}Ew9vB_S-LA^
zUThBR8|Ox+lv^9O_u=)i5ZXUcI#OtxBrE?~Aoy*3SGeIJ)>br=6J^65QsUC$`V;fF
z^SfubLO27+HChFJq-P4p$#5#d*iZd5UW$6$+)|0)d~w8qjj=V^^OHe1Srol<kIp%s
z9g`3cnJ4d&F&%y=TqVtXz5EK3i-;;MC@PfG{<My(dhO-n2#}Xyn0mINzt>APZ7zd7
z_G?#m&E*SJoRsW_x~89-qSSL78t8?srH~JnAc%{SuO1KIo<KCH<Ik1yn}k8=RN@qI
zArV$IpsM{PlWb|Glor4EaFxi2tnoG3OUKL9pE#~ck4%f8ep<OyQ;dG}s8s|LBym7C
z>t=>}PQCi`Zb(E{vq*bKcu*!6?cI(ntbO8B?})1R{6x|8T0+F|C{%Ijp^i?|Yse$i
z@2NtPOQzsku-kPpzh;FAtqR40Z<x&G%`a2mUSvLJsRZZRsq}Hu@oj!frN>O3in)?K
zQ;SP2$D5)F_k3ZUM!$p#!Mot}O*vWh#*@870sCVmS}9uZj+-rNC%1D%7lpNH$9Dz0
zl}xK3);zl(cbgtobmM1SN7qDzz6D?1+jCmr^F4JwJH4ZWWdSM&n92PrM|I_n1rNcd
z#tT~yD`6TUCLjMg#Uaykxvro8uI%yL0rF9MfOqjVGhtmq-|o|tqiA4}=%*ONwO4Xt
z?qqY1uT1x2>woV|581}pRKx6LR?w8YPw}R<+y2Nee-I(y=;A~t(X$N5MWY;c%$&y+
zLL%rgm0+GwDt4a;)JBP2AWT;~lyBI^1*&NR)$=J2@v-*FryY0Gv<QL5SI8DGLr>O-
zp&-+SC*By5A<(wm2_jx_9?+u9hZpOdy^$qsoUcxWpG@l|X$la{*XN>>kAJ4}d8Xhe
z-LGH&>rzMDXlpo6Gd~(rXrCL%X>sXY>9(|*EIumZj?ip&?#@C&qO>BMhy*~dsnR}O
zsmY^uJz2TR)oA@zFN30F69Y$DFJptAPg|;CDRYTzS>To06~#g&J$Zmc)KW`b>bNt*
zj}C@VlQk?Cus)3FnC584TMo%h^_^EoBy-)3mI%K80no7Wk#VS451+psbUKgpfS1pE
zSgQ)rdzt-Ktc-2&j|_U^&ce>IjP|0)yf12hg~~ME@&>p(kY776Qare~y{X6Xr)q8G
z_x&vG=Uw>Nn|Wi}LQ{S(c~!ZX{VKR&IWAE|2dd9De^mn*3Ea1sB^h8)xUQg2U3I~h
zpVz{^)$fU`P{K1|ZGbiN9-wwB6vsaxl(0^Et)o`pZzC=uBI2>kogKr)BIjG<v?3wB
zFMt;vIod)S3J*Si?Y=uX7H9Fsq7L`NAZc8t`Q7CKlgspLSgG2d$@>MO1Ej31sqTZP
zOG{S3c7O-%F$;E-T<x|4hn%0LL)52}OLz8uO?B_4AJXcBy7KJulr^ecgb&u*pOkKM
zjP!~gfv1yyHKcGMg88^y=Qi1QE@c3Ud^M6Ol+(h?IuA|76{1OpS3Bwi$bTQbWH5mz
z{;o5gRcdL@%R6ySe|~Q0I`8Q>s8!vg3Ud*@@8BM*=1G<OulpRufMh6hTwH$XtV?k<
z6>RwcN68L@OCW$>9d{zOA|#S@BZ?qAGQL?3`bcag8Eb#igXv?H2l;U(mi&+#6Z()H
zhBeaFigF!9>5Bgc6}FE|ckT@-*-bMZkHWC?<~Z)cTn#ukIcYVi%Q3}peBodATST=+
zWl89~3A+R5y3?d#3>*yJ&I=O*!EiFkT;X!X*rI^kpAH2aF7=!!Ac`6#cFPmbb#w5o
zWF$77)>}%z?>_hTVsDSFHsO`Z`<#z6QoE~r-+V&Tp0!~)O9-QEKRqO~-c=eEB62g%
zNbNlpxD+YtQr;>>$Cb;fPS@$w9~NiHjy+D?m3af842=;LQUo(+%RG6~dz7cZD1j23
zOfcoV!!_Sr+RI4h#>!F4T2Xh6lO_sMlvs5wWoSRd-#}L(&CJ?Lt!O}(ct>SU*=B8W
zc%h}1fJHFIp=f~FE&v{cPFYyNPmC|bS@=uNbpA{2CJOG))F8AUH7A`%Oltkl9b}sP
zO850NB*U%UwJL6mTuF^A+r{}1P7shz#byfIil3TB7_{2SVPHlTpbiKDEmLucDN2MM
zvIm@#pNbH&W-jjrv(l*OH<O&2Smu3mhX&N@>Rx3nMp`d)I$G_5Y0EX(qH2Pz$iB*R
z|5EFVwFAGaG;FW<_$rDrj@HvdSj6`f?6$g1`a%d=WXPOr>8AP;C;vJqs2@oJ6J^K3
zLis*Ms6OlFjE}+@NkrTWnn#nx1!#`a(xQ%ge7ksMiu^?N6D-+zRzD56SuLw-bSGto
z5@tppcZM)9L`AGdQ@!X7YYr{*?iO$yzNZc!=|mSUi+aI}^GvFs*I84jzEY<Hqu#4A
zj+;WQhblg@bQRqrrCZ%sYlf82=bzem8zy<opixJ#G4NF5;d0iiIalW(Ef*;+?7nQ1
zL)lL^THev<nCoh37uX`f9c7M;CW(!Nx}GMl1B#dRM&z{q&kG=;)x1~=e;&D~FFLU~
zz|OU(IK{E|@Wz_NfWKxDYM9By!l-rAc%iM4Z#AG_MMG%+NAfO^05$lfdhn^nez~D8
z8;r&aE`Bwb`a5IwL!lKqOVTGGpi9DXX13CZtVFG#>CpA1%^${+-qK1UH54yjB-Tfe
z=Ps9g(W3FgcmZbdBzZ<Nl!ok*M#{$c<Qp1gD9AJ4Wdmc+oZ4QlFNW6pBWrRTu;N?2
z(12_m#gP`PG1R10*VNy(?1XJ`aFYr)`#A!ET-l*Ca|YXKcw}Vk)P;Sy?xNBZ&IRHk
zr}`PM+TNqW45j+Uw?2t&J`XB7$<U`Sjj&|TU8PupoH^Ci-dGhX=PEv6mvg@@uzH&%
z18$KFF%H!Ckw5l&D}8R;WQ^33$DQ@O&^K8!fVi4KzZf_K1Bvb>`o1|jbw&3#x{xx>
z_FPh+uEzjp@cW-JU%|QlOy3ICj_Sun@}l7;hbx1H9ot9HCs!D#C5n}`(5l&Eo5Y@y
z{GMU~Y8p9Az^KMBuBTq20ac6ICVAK6zMmEN2$fS39S0(xD9SQ;0idavU}>^gm6l1n
z${1}G-9j=kj!xlQv7ADnb^@)u>;~BG3YikLRB*8d<j<<TOZwn=k*_Gu7$lA~d+u&2
zjY6S-9$%PWRS~ENshAW$?$@fIPjb9%e;=4Telxu1(8B6c$EFQsO|rMUds+kHXF5a=
z=<h$I-gB->!n{xFraX%9G*My`nXsmmlVYL^<V#K6PIgTlWb{a7g{}FTa)WN$Kg(k0
zqq|d0fBe72Eh!=!trxtj%i0~K1lbHjXg;tc#&Wk))Yu8}ouzV6iLO7%7Dpn^b{EJm
z)%`qGq`4gCE}Fam)|-D8Gz4yJ2DfVyHweYxfs?C1jo}$Bwo&1oL<whI?V}##eI#sn
z?q69ENal}wvypwzO(jQ7DK0S$(2|8j?Q%Zx0G7U(Zex8^KiKYT!FBj0K-3&YSPJVs
z4)DXsmEvZ9TxbLqnap)O<@=Yh*c1YQff&{W^6E$YSGzdh<JwNIY&ujnLR{J^XIs)X
zA4Enr7{7Wv+tH%xcz7W5Z?qOKZfGO(R}S~L;}s-kXiJjM0%*8(=?mW|MbV#r(?=n@
ziq@19@&@ZESq&UYBwC<gc;+RBIT6XrngNb8YSIi7+KZidDH09~ZQ_uj>(Li)m{)Lx
z0v4gC$xqFroF*ZC-?3_~7u(%o%K*IkeVG#*n1M0%AE`X!5fpF9jAGMF3+$Tnavl6e
zS21l_6P};G3nx38+<s#_SQg}jxQc?ggz{Kr4ry^?6^fXAnIURhkS-_eqK3=$OXGFU
zyzi4hHaeutHeNjD$u}C+pC?A-@zj7xwfFMaeIo7P+y#o1sJqb^F__X6G}`weo<AV%
zikU^Psn_(blBl@D8ydgISQwM3a1<~npFn+lcL$Y^M3Mmdb*ML0nh==(88$dvXO&z@
zkL#B_f9*=6sP%9ne8e-Q*9HeyjDDER4vM9kpPDD0Pg?rsGsbud*U|a6X|EE8;|AZe
zJoY+CY;T*VW>}6kd_!pNWCK%+TZdu&^|0^XknvV&bf#fDU@jml*SbjUMZl=~&1I7@
zJ7WRnZ6LjQ&7I$!oyq&6Uc)hI9xd%@e>MlrGE!pd%NOLg6Bc>V2JO>&ecfNUY01=~
zg~$x$MyiXR9y1sMb+=?iqZ<31N%5$-HnEPknK7l4Zo?>MxL;+cv>q4beTib73W;(@
zrqC!EIX7}$D0*#u=k;LM^HG(#mmv(0s7u6;tRa03_8_R^^3EkDx=p!0e;OXc4~W^H
zsvCgbvMf&9)$>XD*sHid(|s4kC>`Z!;UY~C1&o6uCdK{?%Ka7NCjnr1)z1j{`Shp0
z!Bom}=w6G9?tS0V@TJMDVyKL^M5^>XxBJY&{M97#X!~ZB!o^?rpMQbjk!JTj#N8Rk
z2!%62CTY4ZT22wiO`miwsJsDevJv)z_fWRcA^Z`c+HB?J`NJP_-KW%B{G^U=wokEx
z+hp^-%0&uHKf?0z*%FhPN;GI+0~g=ZtPWYyPwO+?MS;sQEgkNUS7`^5%))eDok!`l
zGU}~An+{~_+$VY6ykl)qtnK0Jk#A$1A_4O!TZL2TdGw`Mz#ZkS?W*0YrH|2d0iRrE
zDsuY(jJs@mA(C<l{ozd)K9T)=ipx}4etM(w_nhTB)C6~Jka<i=rP@c&^qq=Grrg!&
z;jU$LCE+(AvQlW2g_QvHacthPL&?h<Z3Z6Wa<VF&G46S>J7psC#tq2Zrj|q%b_6^Q
zGU62}Mf~LS8tig_gy~F&C7^B=za@-;2Xj|aPKzz>{V3ly4pvTbsAVwWBs<wn^+}~u
z)?fHWFVKAPN?^D5p~Se7+Lp>I|7<|X(F0;hrm_5%*l#SJ5;g4JPQ!ja^j=<L(gz4R
za%&*H!|%bOVRcl4CdVAzFK$i@WdRdtrT>Ny>yGP_p%T2?QsaCma8X}p9hwgP5TV?n
z=o*N9!~d~$_GiL>mXJ6XXkMgUd9b+?es?&FmC)K2CW}71h$;wW<l$z8te3;$Taz=g
z!S)GB>|f^XK_c59D7SNmeX8EflSHbVhkgpW(E*uOt*L32^J|0J7l=q?rM#%M862?#
zyy99(KT9@c5f7kX4vyFcVefTk-mO4xt9tn4KZ`>=3lU!QHiHHD1_S<dzC!UC-Qv#q
z!r?c8++|uXb^K25r}0W8z_a&yd^I*Rqa}MPm&BB$9xvPCad1wC>vhc^aPKu&Wg?mQ
z8m2VGqtT{~b>O>olEHN{18?;Fk$#+kp#(MD@zFg{NuqV1+qKR~d+s0-cF%dca{{=W
zjK$;$zFq`8xQvCK)}C~Dx#kq@$|pW*D}RYL>iy2ubYr~es<2^uz$1Julb*Aw$_+!O
z)Wp!D&SwfHZ}Cb*WhLKsd)PQjE~==v_dZfWLdWL^@ZA&?(ip%5SrYrj_=n<5{4qh9
z*OZ5D0k!zne)mUr=?1tJ9cB+?Zi_vEw?_GUx5HePY#G@T)yB^|()}duonlYi_DA3T
zULW$bDXxPwEMtAe7gKXe^kVvczW#(&kUwC@ALT7n2RaMz_HzqEn_k0%8o_up9kEfR
zm&KlvO+^b9ZtOc810R|r+h2%$()Dmq<X$Q(8yc+767*LsIx`MrC}N|uEnp62@$<3~
z{h$l^6)&tFbrYyplNpdgdOxk6<_6Vxi8X)3oTxW*P-w6JFlM_A(kRvzDG#{)$$xt)
zdF4VAN)`Xo`h$JPdE```SNmCGYeqT9qsnq;%=+tu)oY<cZ6BLh4eoLn*O<devsgaC
z-)zKsPs{Q+=<gXIX9MkHi6BZWdGo1iwc17<e%N!PYO`RU;-E^V%XkK7ypoGe!S`F!
zqSK>OzlniRMLRT&2PbSg#Y`FAOEwBwwwb&BUkb5}I!DW}b%K(3$E&<^Z~49INKeP{
zqlCGL(eCbq7i|GG_Zjb`Moa@=qvdRsb!&#mE}(oY_KB7L_Nh%lW_+VtuZ%~l#E&l>
z531qxb(_X3?DBnJD(5po=$e7yi+Ts)Fu*6pp}zT)22Tj^{*?OtZB|aCNyh8pLdB~E
zmr03J#tT=7y{pyRUt>M|+=ulmFOECn9{a{tRwR>)pW5Qa=X`viY~kBI*sr&>2P0G$
z0*$tLX>5*}sN3IcTlwhtB6YC4^PBetubjScb;<yBCOZe>Nu78o+xDaX!k()NaB;+)
z&EoA_EYss%g2^0B+apm3xXr2dRQO*v!g+s@1$sqaLb@;=O+LLXY4W~*Zof}_y3+C1
zC5TrRkool1L&fsud}yRfuj@_Od~EbbBHKdXs;(^aeZ%i|6fZ$w)G5wnzf{A6rMHcT
zhOH8|qGUEj>hArPi2})$hb^v$H066cFLp+TKaR0q9b$Loxi#`6w&}k;VB{Ld9B-OT
z=ZIgWuaib{4TlF4Fv%4lA*v#WeL4PlRPUnEYO1clQYXhk|JbOKk%wjUxxsEw-VD&@
zI@XvCaINIJYye<z9^fV*TZHDZztzx!5}OsEHiL~FhIelu<a=30v&%BQj(tg?t8<WW
zlSuVeUf9p(4o{#aKp9OF$AH-_^8@e9L?2l5yv73MhpZXE5OyY<ZK|iSJkG>+eEibz
zz{e7HH*Fe_?}hH9r*BXy<Wo%J|EJxt`a@p3I-_|+8O-}=Q&X2Kn-<)xDFZ}z|4N>P
z`rNGhu-rXj<<mnlTx<>Ep!aw8LW$gxeg|r3lyM)+uKx{u@iP4T#4P-=@?D@)+y5=a
zwb~Hx=J6oJe)Cmv31Hi9S{56CcAK&sB(VR7Sbmd~y%B-fy0{y>OTz2l4GKHR?)&kW
z6B}UoX*QOl@#ptW7D-7`vQ=r)MWV0TRE`8|WxqX0La}d~D@usoY18(K|3WBk^=2j2
zM|}^~h99?9ZFqZ=o8qwILt#?KqhEjSL;fx;mz*i%WC`Ph$ln6ihIKjgN@?4TEMkY(
z03|ceuB%>tGWT1Op7oO_($T2z1PmsmHS1@-6DrUUS!ojY9DZzjA!8l=lM!gt$cdK_
ze2c;pqW8RwRs*%oj=4Ihz`V|oTI!gYiy3GQH0p5sJBS|(s6V03f031;1y07z%wYpj
z*{qQxiVtK1cyxK_6tC2Budnfy2~ttO*WXXN6m5rN)HR~X38%RP^6l$A&V9ya7Bhb>
zjx^Syi}lN;&Rz0vo-EE^O@wt#G(&$QuC=HSx3<G#&~TzqRv5CtT4O}f>tl;vB(Vh4
zugSC=3v?3Nw+dM~L`(;?_fQGI0fL8t=UcZF3zKx`cqohK_a@V9=ku!<8ii`;%&Tb3
zYZ2t*Q46gB=`H=AMM8#S=g%kBGUq(+a*>l*jRw;|gn@)N`RTYfDE?UKGldZ>yV>$!
zjqM9hvPcSDJ?0mpsL;AM{D8Nt@YAE~_lS#3tIrWe+<B3HqGy)6&1RC<GU6{%Yb>%=
zmoID2TT1G95tSu8^f0gqRSx1#3j@^ke~Uu)g?g3{W2|gX>wy?)2?=0J(Q2l-X%zSf
z{~q{A3);zQ3)Hc%NG!hwJSsliQAjUWiVkV<uhVW>2#=i5BwAn)o*hhqJS!%82M#0`
zKm2;_t;ul4F@4>&TsYb+Tw2b<WPw3)5BbC1PDiZ)V_s$ILT;Dmh2-bP{CPYyT}+?i
zrl%aOGGkFw!qpPl1RyaX_7xRT`+TT+VD)B95hY28*;kIdFB+~Jzm#k$rjHa2r9wcL
z)*=KD(EwZ)qxPxUOM_k4WH{itD!n^h9=AaTmHMg{wvnK+^qx`24u$jF{ZJ?^2GZDZ
z$UE?yiX$irSLcI_2hd$2#)3R{up6YKd}(1SH%ewT-8tPLQ?|F6VJ(d!CY;O+33ODt
zYh2oue0)Tl^XUC-xW1LFGox2WVs~@DTdGnlhB)uO^-q6k*g7L@SJ!E_W`*fP{6WJo
zfQN2RsC3Xs(Kr)qH`r)O;@hKQf4Ao)pG5)l#cpqv+s>fwZbT{6xfDFqv$6f**ih+?
z@ZTP$APXp4(|%-U1tl>gN_ke!l|{`;4hxoqS<#-N(l}M^r=Q_xXt#Y#Puxh#)woQ+
zeosaSw0_i@S~>iiq&1pauW;<O#>B)fod~vXH7msiI17r}YKdmC(osTzt)q5hUS!kl
zqaAQHq9@U(<xn?gL1*%7J+Z??2?=+7(q}916^;CH|5?Os$7pVF=HXZy!fm;N(S_J@
zr(_(a;Y}5bIkVn^ZamPtvg!6!D<Pk)8=4&ww4RwcW4Rk;y1gGH&!Ktqr#;VL&OfN(
z%j2}Nuq&hV_nWeWDPmnNVg{*aUEOKgn$ZM|x#~^pL+LZO?M;DC0WbKRvY{4QR{-gS
zghVc#$37Pla4r52vsisb_I_=HSG2H<O{-SJ<B2b<g7V?FjcZTqdvQpGxswvU4pImx
zU-!;1F;Cmu0L0-vG>PbT(l4^;K`uK_+C{5=_(Nl>+vM9&d%ft`a>sBjFkqeCa}Aid
zAk2G(>p;)+sf8SPA^Izvsjo^opL6<rhKVW6+@&i-Ps4D8atR$$75@P~sb1h5ON8{1
z)*sx7>-B|mu_C>DEW+M=5hNNUAWyj_a3u-xSDY66C_*&3GCwH`fe5j+e?D!s19ub(
z&)ocgtl=b9g@Y;8HD|&ynhGh(o(UQXIpX9Mn?C=<O&ks&o^cF1k$1CM&2P7BC~s5q
zyLg_zXiSPG96)v-1d|2*Xkef|#$36`{#nX7s~OWXrB$Kdl_K|CItcOsNWbz`Hon}S
z<sP5~kY$1RzNI$bDW!JQw1vO4xGmocRI6=pUWz3e-}r10%#}#Ex6qKPo99;_ge+U_
z30)Y#p1wW>qWH4s3B$!hOa+pFm2%6qqaX)hC0@p~L)x}aUo*7Wt7fTw6-b0jzCY5D
zKI0wU%8_HvONX1t(238bV(*&7lI9@cvdYv^zSRl+QD>U>54K6aC+z#83t5@oEk$le
zEq~%MNO)M>4UHz5D&%8J$jD}k5okHcb%yfRVi&(od~iFLay8FP+$_d1@P+B3B4o-D
zgW}Hv8+fngQg5nQCo%#Sv-~mh2)j_GT*F~mo{pvMt)%5>MR^x*GP`l1viRNpz}aNu
zyqpHhKoT410&<n=(9N66C#1^XPfhQQc(5T_Q6TO5Eyo6(MvMx0Z$gblY(>RpPP*~$
z%0yDFt|vvlv1xo2KxH&jB)w5G`t_Y9?{E%?*c?i!>LkZ*6j6}YSpktA7FwMQC4E6v
z*)<<pafl|$w-iq!l#^1c1B%!EJvN5)%7ZQyt2pDbA%3V=2*rZ8&5_yLvGJZ6nQFkz
zQLHUhyYPoCQwee)Az<^#6Y7pliL#{S7_eMB=XrEJk=gNBH&+Y<&PI+GHa<M&`akT_
z3AnGK2{%E+-o_VXx84{Pt~08e$o-yG@3U$Q1yg=Hpp!_@da!apBl#iU^5fUWTx=de
zs;g@PE5&_i1;takp&IpYZShl<wJSk&FYCUM%G15Tn2jr)ax2Puo?K^4e20&0>E3ev
zn=%2WkxY?7*A^|kmFa|ns~n6YR3p=n_25Q#$YMw7^f9M7W;9iwaxQZSXr$5iN_DQu
z8C;;4`gwPzQfy~@c8Q_)I$PXzlKa~5ef?keVyX?<sS>(<Ka1brGMO})O76bad^lNE
z@>nBgGpzN^rXU0_JY97JB=WoJ+Af*)4VO07RBDegDQCc>I__&`U+^+7`%Pr1k)dek
z8+;@qZlsyS!ggnIIomW)&zP$xlR-r6`e`gRaDC9IN?>FlSv~al@M;_0HK9?a?ZBkX
z9%&BImBIm}N*h(<E>&-EDD~g!jko!#C#4m<PRE^gI$Y3X{?(r=-2s>sbj|pPh>%)O
zmn}_8zBjJ&J`1?r%1S^veOIBc<G!01Pj|MPC`Q7YR|wFmkTE?Jbxy?!uu<MVQ)K}z
z)`~Z0cML9a*i0G>k%mVXdynVCf)C|wxvNT9jn6P|RSOlZx!;;^4F{rc;g~^i^nWEW
zD+2A}dMEIwVsnS8MEQx+#Xsb*-x=r6>glnvU+zq3SKN>2kH%5QTmDwGKD*U<&<7~x
z*~?l5w2aX4_^O~XC%-9riI-89u@H6nWxF}8!qxD}<zv3}*S9hNqQ)MWsxqfOtrdNu
z#ubcg7BycqQhLtvks-<PJwQ3ks&B+Eea!ZECDyyOpvy&xCBSHzkqvZcMxwNQT9798
zJS<w*d9z#hnyNWv6$22^a&}HX?MuU)*YoA4KD1OIPRo9);9lJI>1telXZG#I`={~*
z%biIW%ejXC6~JcT-6$qf6lw{4pfBI$=^S@2li~(wAqt5MxZDm4+7C+J1`k-ma(S|X
zu?$H5_{90;;<ss4W@EY1x0!^$0C`V%!HZpz3bv8}uE^EWWL0g4?{l)r*!H<CTFR>N
zlc@tm8Wi#wZ8+dPJhK@kb%8PDq)Z;pFm%d~@df#tl3P&074eyl-J_Fo1<ReoFU(Bg
z3as++oYzk@o9;5lVxoX@&!>tiz7N`_w|$$Mmq_KznSo7jS`m`SWa?4wVsmA2>mT#b
z{1GEWwt4lf^)Q-|aFJl(vKJ)Iz#%0)vc|k7uCRBcSzf)PgK}0qO7uTBRVrQFV@{yK
zzWV(=h~%)T30Nn793U_Pofje;Kl>+N0=;%$T`(o~b`UKB8Jhg|nhEl}IM^Mn*SQn*
ztd^xW2V??a;CR*BG;FJBpbOzS?y{BcP(bR##r?Aaf0h#?eN69NtzSZIi?99i(590J
zMC34+H$=rwH3~`@mCV=*7It$I8wLB4@00mgb$31NcTe{1EmL`w5dr~VG^m)Of5mBf
zX8g39t+9>=I3f*XQfT%f7K0@2v`(&duP0xM2?d;{MBMYGDC>)?j4w#)Vbd<3x9fV&
z$JujjG{VVlwm+Y#k-u2@5LHmSah%2FKBXbJj7#UWuIUI3Aaz5=vzCC3JbtcNY_`}U
zL&3ojr)u(!K7ZYa>@dG0^>;~*KKUi?FWQ4bR<i@@!{f*i3^1wlbzdq1IBhmx@=ixm
z1FhEMabawnXq4^>(*+huX|{o#9m^r@ZkM+@iyrr??-sMsy_{l)^##}X);-9wc$c5^
zFEn)D2WDN*+Y3%~azsbGLFy_(hP|sEZy-~<LPmoAo)=akPF!ST#!6lI4Z9Jt{3Q_r
z>2I1vW;BS%?m*r>Xj36<UY-5|A7Q@#A^N*hrEmNrZBaASKj31KU7nI4<Ql$2%T=Rt
zd}Nk<SPs>n8rOxOgk4^UHc%>Ozwz&_<Fc>_w6#AgdXTdXTr|76nhdxnzu@t_ot(>h
zgCvopyHV`lXff?4XCF$y`&xv*)>;j-aR@KR9i+T1&hKK`2otqm&Z;N{F}^mLEmBU8
z{whQ7jHiDZ_Bkj{Ul`#tF<a*1jd2IVr`3h?BRi8V65*ffzF!S1jM}Va={dI?=h_jc
ztk2V?Hc0$mmbLjt8+cs~#J?)g#VRe)5cMrT1CjX=WeIsp2=W{>OdB*w1^n7fNS!YK
zV~3ki*2PeAT(4^P%IoFZw9huJV?WIkk4nimY`;C36=l5>)+jb?tiCm<UTmr-bF*4#
z@r~puyX~=lK@6JGo-g~eVJng1qR#W#u>cM4(LyUqzz(N}*6_zm>fXT^v8U2H_g>la
zYsB`ud4-UCM(AAM*@2*em3`V=_+e?H-Rm18M-~2GTyTr1(}6rO!b_p*!&p|JoDE{f
zW&atX0g8eE{#gfha#!B+$5yj75j1?0=u;iFh-J+dN@U*nYpN%m6fdI(5=1?C+72=I
zOm23kOuK(hj#=l)`0rc-prj^ccjyhHx(L<;n1-XyUT`qls;hwk@TJ<+wz^PAc6j&u
zTc{*i!!3R>gUu%*to-$Xj3?!xtd6wJK^^OzypZCTM{P5sG_XHd0$jQA@L=fer9DN6
z#(EV>DX9TUY*fHf3fzQ+FA<}{6?f<-$G8H+Zk@e}p?o?<<BurVxt29g!+b*g_gOmm
zq!1iZq=Gky!3e6fd`VGnLO}Zf0<3^f{Qbsh|C4-(Z0k>Ss9$A8B9D=BpLIq<UI;w+
zqkQZ9XBUn7C(6C1pxX3D8i;gEaFfJw6h1HO)Pn~}kIIL;z_o2~N2n<8Q-8Id>``&<
z_)Px!0N#heeXkNUnsks7P5vIbQtl@>?%e(G?IN8Q{FXx7n<<Ltec=>4(k$)kF8wFy
z-E(IGal(*uxAgX2Pm__EPT+Dpq<XGdGAELf7%8GYyuYm6E^YU+V-$@NJ>g{@3=*5K
zks8wRDNTfYZ%q0)h4)wTFCwLqq9^Am``tGp*Dr>gmBz*%t>Ynt1D1Ef_s#{nBVk)D
zncIJA2dV#!6Z}96@OZ&t?jl9l?E}c|h^*h8f{=~lIoTsb(0a#ElY{IvG=Ye(zY+!E
zL@Di~j&r7;o6(PZLIUd6P?O3+1OB*@(S_aVEKl$Cs_+5XD$1XEcn+K7&czkqWh0A2
z=1nB|oQe_=-)Um{)jCv=YITlAvIQXoLx7ZFkw$Of5{0XrA%ApDVXf*9J?{<M{Yckn
zI0z8`A`pO&1cP;I%*Jk1)dAE&?stv(?{8LmFZ<E+=06uEzE$7hKauu1%NcUJzNZ!l
zTtlK$`K~Yf%ZqM=`F3Tas8*zU`Wv0n`xjFdiBn6SK9Gm18(&!VK92e7;)^3))t8yY
zjjS-`J&E95i*`of@a*g({?bA96-0F`Y9Jl|r<`)NPYayPL{u!N#LQDm*;KL67Eg44
zB3e#1E1H`Q5554~@w}6!A1H4Oac%IMsdU?Vq$ttM<tf1WWeJvehVLXcw*zn*bVY$9
zlq#O;+4GTbIU3PYUf0G_IcnacrTqP!8g*(2VTDFHD63prk7%r><J<0y>1k}H<WkX(
zo-@6s*aOiomrwptc+m7*cd|C+i}b0QhAk3m<T2r|n{?Kk^T;8A9+1MZhY$!&fqa5Q
zoDixGnAgFuMLMf)&(V|Ce)mV*H{tQX^9N&;#dkhb%5L{f&j^{mjihxv_2unMR@c54
zRDx-s#6NQw{;T1ZCgaDEN(pTYcip<mbMO~<x69SQn}U0RJSIAr?v**$@hC%Qv&emo
z$a#F@d|2tjVy=mi>M3ep;6gpms`Fj?Rcd*zD}c~__HdY05ac;3G>uf=@tFNZ2X$DR
z;TD$`R2O}`bp`u#`w{UDu*JsG2Q8>toA8v+E8lrijNkPI{MKZVcI1MJqvJnYHv7(f
z<2%=6Q)<3k<7!%RVqp$*dyJ-8x6A2%BF}RzV%^pdA~f_3FPVZGuSA2KA7EdyF)c$Y
zQ-$h5Hz+T`II=ji>-}A8Zt88~`mKs4KIO{NNE{*0^0Q;-{j5C?g}Sf<Xl=3pu=<DP
zH&Iu0*^vT!GS9K?9V4~g*E<*SSBFA!Yp9B}Z+t$#aUnQjt5Im@i9SGy{rNUtZOF&5
z<vAOTSD8`s%P^6CiW^MgqJ<l~r*sNJu@A!MBg<~5+a?#JJz98iqd3R@1(byER7<rF
z4TUB33O&~>Qq2Fto7d+b7``0+R=yZM(vt;g`4R5&N4*XK(T^Cm(i4_VNlQnrROUC(
zJu9~D0Z?hR8|4HyP!uy&l~nj^om{oDf;wS1iI5ko`~G!=K^?~6$0yIIcPCR?)S@$+
zX#hn=&0w-(u^YH99g?V0>mwJy<&IMX8^P4OAhYdJ?>ZAnc2#B15(u|#yM(asTvXFA
z<y>v1&h4#5u3=jmKFzzAYESq#j=XLF)Gbr42j-qChXF_|@AAGqaaSmX$)od{$h#fp
z(B=JJlHsAAPcu;4nQG;O>`($WD$OR^sCVNPi%!XkVi>olzZ*1Ky(s0&DsB|*EC#ie
zmjgRk!yr=Oq#U`QlHT{IBHqFLZVF?HP4J`&RgT_9og;~tkmi36R!pDHI9;!fYhMbu
zQ-C>l%Y;0$kp48+j$h{}8Z!JHz5Q`CVpq2MgLiS@Q$r$6_`y}sT{8;x>h;N4bpL5f
zj9vEwL)*^TbWS&!Mv?L?P8P0fTa?Hfn@LZ%)BHU!c6vu#8mimxvc>NrPf<2*d=9YZ
z_1LE}&?{ZF2nlb-0JZrG;amxQL8n9yr3{set>W`&@iDsq>zj}MFx+n2-R~kAmlN2D
zeC1!z?4R#2jW_5EdL_m@T?!h^LdaQALp2EzsohzP${WppYQSoasw&b&Ig>S@Qn2~Q
zEI!(gDX=IewX%nKB90z%$%m5%F<9LmgyICL{rgAKF4LoZ$j*A&(D3B<%eH))+aRy6
zMVZk|MCTddgyQ*p_iOU7x;sdTNH3B7kEDM5D$oF}<(KG0tsA9cIsU?<r^&6St#3K_
z3mCk<6FWX|VqE1%a)I!#YV@xGDm0%gpVo-pSDPaEy!!>Geh){@^mCV2<Mn24UX0-n
zcSb_;zJZiVC;0RQE)=hvcqz25(r|_b)q!swC(+v~m&{dw=v30jXcAGD+LVee?Q$lV
z@IXbz-o<Fcm?Dn2yXWDh+}Uaux_TnygyJ~OeydamhL1qt&G6%q;yk*j`uA+x0v$;E
zj#bJ0)bfGsEN1fbMFJB>zYHx#|4R2|#~nae&SP1oHIMb-J3bfnsoznB`GaE7GqV);
z6TaAJ$>$Km1Fsl09o#=3f__~EZ+Xg?S^Qi51F}fY5Lx&7EZ1E)rus3~F%JBr0M&KT
zr|wH*1YsU*#9OCsR3DO13T(8eyiF0-E($m%=44JJ&%UZG#7#-KAsibr+Jh*9ycyZf
zSY?uB5!mETA}!@*Qq}W=YP(zFx`Jq~Rp(HaY^~MVp|k?$ve1Y!wvy>Ng5NQDaji<w
z&%IFaxg<MVXurSJna{BjA;4Rf`+rz*Bz|~#RyzY4QG=@Xvg*-48`$)6lw9|gfZ)7t
z^2%~bcw_Ox_&$cJY*^&S->(=>rTfW7GjW8}=-<9YCK#jk$Cy_-#K8x-ldJQhMLcq8
zB#22&oa8@w;|Os=j|)uaV$j`ZsXU(>@<mzrv8(T)6!W^p$$;=kfnq{DEi;EDLY3x%
zic!d06?iLAJ=cP=bD9qs+o!(8+594bgVuZA*x`J2a}0)x#R<(FE-fPLPS=2iHF@on
zx{&Ww+V}0;4>VN4^x(z}Nw?LU>(NRdbtHGm)Mtzza14GXK3irjoKoOTGk`so&`7%N
zjP)KC()W{7WmOp~g~DQMX(NwQiY_TcZN!so{wxKBs2E@E&jxxpHqqF~Rt|?UvTp~Y
zj$p+j;)^wn=-k}>V&S?gh3rSTY+e)qp#S^@zpb(GUlbE!O(Y+nh4>z7%Ov=wde0Z<
z`s=MfUwlQe(`})b4UmiaZ9i2pS$U5t$v8mycei||F$jWNA9$9LHraGR*7{t)|NeUn
z6Y@9+Sgr>eT&Yk|hST^Hk8?QX8sW#Sm!;A=1An7LxU&d-nqal~mkJe2UMf&Dr?=Yy
zox(_rC<QozG-L;(4=d#}iK%DrO6+c?p3-IA?(R(pksi)n`9N%{(TV}lPHJHQ`zfci
z&!dF%o_AexIn@_XJ>&sIj5_;y@rUdd-Fm#V?O+XX)B+-2PXC!K0tyi7^r?i4e%5Lv
z)3naCk6Ova`;qz6AB?Xd^hWLbX<>+~gKfUSX<^RrPDbWu%0Go#9FyJ|h!QczD-?0Q
zP2-A_Cw2V<05`=GE5{dQcZPY(G#{gzA_#bOY72`#lTH*zv3pL5u<lI0q2iL3dc0?4
zGncb%p>$Z>dv3q{`%2p2jL4{fqborJhl<L1(lY|kR_xuXOBWi>@!ao?Hl3`gGj-T8
zQM{$kAfh|ow1*YR5j<|OuH@?6MOQ%338*Q4#=YDbf2V&TytcGq>vr|S9HZ$4KO^Ul
zc~Gu>{=h-$#H5fKRJVfZ1Lk<P5U=VMPq+YElv7CkqELWt(YiQ{C@zXNXfKf<)kW>}
zd4NFDLR)Fkz|*+1&+WCz&kUIHj@&W3Qhb533YDjj{|CXw8|KME7e~)2rbgDxoGjTe
zl?J8nLWJhSdSPn$zsmvw@#3F0T+>FK^^#QS0Pct*s(Dd~dHmka=7UQ@gd*14$4@2o
zVVjM=m<P7!yF{j|U|3=^jv_fNh$_~P<LT#%r+U*V{|nR$P6-N1#uc(*92_o2c0@at
zW^!-|!tY!_)SS{ss2huABV{-VLRvuB&Kve%puWyMm!lXp7Yr!wx`gAWJZenPcpHm+
zmQ3C<W<|Qn!Sc;i*Q8~2dZB7U_3j6l4TG$6b#!y+G-9lm9+BY68C?|N)4>Y9iVOR&
zfKcpi&3QIh4{g(YxvJXVwI|B{i<K$%VI;*l9J$ilEQx>xBb?+P8X3pEl+4(v`pdWC
zkjb0Q=%^JwEV&mgF-Q3Tvq2Dh$BIJu#if%xPP5A$29TfqxuSGoZ`jP~jQZD-3b-Di
zihZT;r4WCS60?gn6a9>NPPi@tT;_e!&_hil8$_ZeWa;c#I0hS;{8B#z;)ROFT%VD=
zrsDfcP<M!sY6kxi+!bmviL(Lx#V%dbySRh%DM#vRtnTk$3kyB-M$j4cw5zr>nRExP
zb`OZqrh8C6^Wf{*DyG1zrKA_MbB|3{9{I^ECIl`#*idi<=soJ%3<Lq~+gw$kaOoc2
zdNz31Bku>3ZsZ&BIr2Z9S*GVimbV-^VBTF&DHrpsD<62~?yBK|%emjJr92T(jk!OT
zB^`ESowF^GSrD&uX*irM&7bQoM!zS{C8PR<h^YpBo^je7^WulU{*S9xsOY#)(kcxX
zqd_}~|FQQSP6yj4h{HHY{vibIMb5{fOZ(+OS=m;nO^CIek0)tKUNm1!I-?<Z-)Qer
z^1b6TmIo8}C=<({B(ii;oQRAdRZj8(pEmi)R&`oIn&>=i8mQmR7z1q25#mAVlQ|bS
z*+`qjp!ai!f6blz&l%g7UBU2lkAhc!x>ThOPIM>)ZG-B+^_(T|otety@|V)&P~l-k
zDN*r>i`|N$q;7lt`SllPqDnbI10|Rk8_^Oame?Voh8lPm9-{@<^u&+1#<kD&rGAPn
z_G$S#6MCrl^&;tm#%99h-1=jEdfjd};$h0C3V`h+MU4cALCK5CXWbFZS>8!_W<6}N
z>GJwV<wa}NDFNYoTqkJx+)_tqe#f8Xx$d<VL8sB?<DRCj@1Q>#qGUIt19Xm5gk=rb
zTk`)GYwsnDWGHvcS0L6>iGBR+csc6^&1)$((t+Wo1P(2c9OQ5!pn^YxBa-Z8gWb=?
zAwHwQKy6+Y%8^MXP>6Nw?a$!-oURf}=-D;5yXJ&@@LCj_BYA>k(|W*l5B<=P=?V_e
zyg^BaV|9WZF<@og4MK-@i`%Ap+ol0^I>i3Pf)#`UI5F^Q9v<1_<C7CrN7#A;^=j8h
zA5+=-<-t6<x!d1RoU9uyq;|6#`EFs?eq_*^Q(cIV25*OlwZyNesPem7;LGcguruK<
z9;4~|L~2dcjKYoW9@=v_u3V@p8<o#>zE($3s_R~4$dlo9GV2R(_z+=YzfOoq`BUob
zhRZ{nBWzcLft5_E`h(ELv_e3*_poB~cD#Vfo$>))zQRWbL~HsEA4$I2=+}6~!#Ck%
zTRZy~wT9uEW~L!^43+O0rP-2Si$&#DTlOJR+#rte=SqI{$hRwqPi%ar4D;C$IGBUQ
zUf#P?Nu8rw+9JL!h|=}k!RTQuRiPJfUjMA@>v9f9x7?+^6{=S$TWWo6kds2s@>1Oq
zjS^7kG?YwI#Kf_YzmoAuKb?{<on+u=u#cDw<5d+e<MY`@DSvQRZ}J6=<pEe*;t=l}
z>W$}e)Kp1SY8YxAVXwR~H)wG5IW3fBq2q=5ok1YCi&J;qMnC)verFLOSN94UhuCep
zf?tGFj3MLZa*}_6VS<mInk6Zp>RbkVPPH5>Bj_?e5Ubwh7GSEDqiXKJGimESoCtj3
zAC|0*!P;5MK+xs0g+}i?smK;sj~8Y}Ys9M-9g5e*ZsYt-v~<g6aYaKK0U=U*2|*Uj
z--}ZfQjg|1>;C%2h$!rL{#nLvPv`;dh_OxicYF%#;X>tM{%oLZOfmztCS*Pt4^$u`
zbi3yxaBMKBqB-E3tcDY{g9ebOX)1YhZ7pNQ3rwHF{t&4(a&1FtPJ$S7%no=%w->q$
zsLxw-fOM1mx1`Ci`WnEFrey=$ei2{z>Xm*^&IhR3WSz1N1vfg>EdFJ*q-$WaxKW_z
zq9$?ftN()Iy`KH8-hhhnmtwbi_mAAAx(2bmhCgFpMsVN65F@sCB6@e4Og2H}zAec<
zY^wj^^WHwD58q(D6B#T8OxHWqU;}WJLpz6AOe7mWha`G;SquaKV(ZOZu|jrUhuxeQ
zETosH(Yz!6HvLFArZ|7Zv)9UEI1Xi{a8Pj{>_VM!glhgu_B+`4JtCS<Mnt^0y0@p`
z*m@G-<C>gvVfT!fRX-bmTciaCLjUe;)}FnZ=70nzYbxIwI$p$#%Cvrg<K`$u#r;bZ
zlnMGvQbi3x6NmTwl~2xg*qoGsjx>T9{9|ZOeaWX1cf3-6&h@K9mXHFN@Px;^VKA}Q
z=NZxfa>$91;AO5Qwl!(i49wgRIJu}e(J39lDdGd;X4$gAh-vH2Yxi$&RSN8Wf2D<(
zvuVgLayKsW$oeGFbuV4%UMq$~&akQx(f{{sTaEuF;6#L6l?fa-a5tfPErGF-M6YBM
zvGo6BW36zUpOD;|*zV?F8*FEzcp`{sM*>2_l?Hw=Wm$KnHKxHIv8*k#tU}6N5>Ev<
z1I-VQR#iisxj!iK<T$Sf%kpBoWHNZ43tyWoFMugfBl7IC!9GHekzk&^Y;1~z@*sg*
zcG2^9eL}^w(w{2j$7+~(WR9ckTq1v{{XV1e{#(0vtn*dvKOqke%7-^de@iF++%tY{
z>)1v^>spf*?XD3Or@F-jJejrMgicobFFc2pp-};$!c=f!G(K9duzFY@_;8}8dU4?C
z*G<)A=3WYpP$u&|XY)TZBKBIk^EVo0{iHQtId;Ak2Rey9hz;Q-*5-4n9uzNxh4-5F
zAPS<-d03AK_njb=1;kC_?5z1hY`dHhKNQV+KCu4Q%~*sOv50mRPwmjd$~Px++^)hZ
z(m!XYf56B+?bYn45ef}!9CfphTO4cT^e~a$(MV*e=fK6t^+aeN-yg?865>%P-0PK*
zf-H;VeqDk_Y5PGBCAOHxlR>Cniua#|6*>}>_`lFnDx7p&xGfM7{c}?#c>h6K7T$8y
zo)6Tl+xC|Sf*B3=36bS&xBSTu=t!zx5Y*}j9Nx#O-CE@FF@4EojGA*^+h)AA9Vhp3
z8);FaP1#F?QQ_#}0*QNcyA9c`=B4WK1kkBPm-a6XlA8m0k#2{bfQ#vpHpNtT`Uzf0
zBySdlPWnF&uk%xMP%@qnP8agXeHU|QZ2-M{HkL9%7(Yl~R`m9`2eo04I*8>eFH95=
z35XFR`8u3EOUk3zlYqEihp^s-f;Z_?Oj|9u_hH90`i*GQ>(beg#65Z1d}0!!*qME0
zXRA6|P#vFQN#nk?LebBu%d9!bYEk^L{4veiO)<`svpm%$eZSBsmC|_%o|D>nV*)Nm
zO3Q&y!lr~Pc=>Sx?z`HTrSHpg^JHUn4~onFF+QOCwW|03EGmqD7FCK%uUW?bp0Fo1
zjwoOW_ML~jM=2%f@Owx8M2*)BhB-Iz8^$!ofP={6OU4PcQ@2!t)HOx}QQyy*!W&yy
zh2K1<!{I9#Thj0U4Y)u48#j(k!#Ggi^~%TA{|UblK`GDy<iPl%+1-B6wo~1=IkgZ(
zIC||hr`B<?MS?lD*^Lm}jbj69V?@ZyN=tOT3~RB3ue)^@@knH(j3dKSF0CZJOgEdL
zeVtX{Z9xZe0lRKGHv}5Y`);Bi0>!o1y|Z9>v~w?!LsQ7xNo@~+muGMH2@Ys0ffx(v
zv#k-(d&-YlSrEWgcH|U5k1;X+ubdee9!@#1v1oNyz7726VS)G$Vi3F)EoKF%)nE+Z
zSHvNakqFWMZXw`1h$yX};U<Mvh#&+I%01#_7b1|9bA~NeBPJ@Z?nO?A&+!KwE1tC_
z4mrd0Vy%^;XhqJ91f1UJP;9$atUhrxqA1fawW7IFYg6%bV!k0B48s`g<I#_##AjiT
z5D*K3;P8;>Yu2Z{IgUiW#mP+GTH$|}2D7UDqN7rH>mN<T0=orT)Fm%`0gid~mi}+$
zXaD}wh~iY2)Kn<3e>{iH`~Ek*f3MzZV&HA~bIFxWkQj)R|9gxy_=s!(2|yY%iX4su
z#GHZnK>C&e!2?=BEVo^vUS-&u#Kk3ty!<Ftl4Q3G%|ov2Bc1w~BL$Q7B79(>m}WSp
z{ia0rNS*4?aX2w)dtqCIwIwTtJ9#Tf!uPK9o)l}x29w{}63ca#S<l1JU+iv2c7k@R
zWao%3L-9g58l=%Xwk4y9q>i;&aD_IR8HDgfML)t<NJBob{C8*r+8nWUDqukUU8=4B
z6YBgsFb>4OMif^2()RB}Xfa@ww?h7s@Mvg7WvI&r2^`x&q_9}K5If%l0)h*!XA8*F
zvJ@T+scfC@n|LV!*`ES3k?F26k3j*5c4Hu6w2`6qG0QpA@BW-Bh$0Jrtv_}SM{JS7
z9^*F)ePpXcRTP_M6iX=bX!3pTsrJ`MPQcXMk!WOs9<;%qba=PAz0};PLi%2OgmYw_
zL_6(l0TNrZ&{q-v??gz^012g0QtUlR|Gpys`+#o%3$gpR%GcK_|83VO>c3uD$c(zf
zC2Q>hVigFUoiCN04BYnu5~-Afz(E|FbU}haJSc$Y2qq=soL!&4;^sR^pM|lkopg6E
z|3MwG^k=RqtABX<zE3MzDiccVtcL?qonT}4%im-_SDnR3o^P{@`G`FP#mvSEIFh&H
z#3BD6<Ninq09`B1*^vU%1MMr3^Tl}(5RKRRJCs;WiQ|;Y>|@3Idmy>v|NlHqzKa4<
zFS7;2nu6Hb5d0A6{j;<BQdNRP(9~)=BW{r4jBtcygzt#tLWLzh&*^RzjYdP+uMkY>
zw-DV8$<CV`F1x>Ng~sUpz+U9UP}}@SiUvHF<JBg}ZmTtABn$R=dUmzSc{LUfNbtIw
z55;s_@?!v6uW=uOi)ivFFhERSv|B3lsW?fsx`o`Evn^XOp_>(8(!eUm_<H}G0{?Cl
zOkkx+5yh{{{x>x^|LmdT02JU!JZ#CmXZUZ){zO4(O@yC_nSfXk_>kZ@@J`o2diNmW
zPXAF!iXZogfblaqQUk|96SyERaL}HTBr<S&wUfjPO^Em|gmNBQLNvYApg{MThDIB(
zMZh!>Ek0X)m-bNBxAuqN{#Zt=E=c6ho2bcP=W`SC5RP+k_L!U5rSF*TX!5mLGrc@k
zwSm{l71&0A1kw912b?bf0Qkc=1yzs4dnLn;JF?#Ug{;nIipl4efIXb$pEn7oRe@e3
zK=0$x<z)JA7=P`&K?$H_<Jj|Siz>Z1JDZq?1w05Ww)tL3FXZmO{r^0Pe;)|&5loEn
z*Cw3QfM>q%lXM4o%&mw4I5CzF#cK$n2n7S&Uiou_j-7md>_?rG8*)_thqbSas_O5$
zML3jnNOR~?LK@-FAl;~xbT`sSgB-fMy9A^Kq#Fh4?vRc{H{8uXp67k<828)#A`V~-
z_St){-<oT#wdN!;^8=saCx6zjc;>Oab*>Md$m#N<UrBP7uYkXO*HwWQPW`g-4+7~H
z|J`w+@>Yaly5E`*60rvnR9_AIqQl$0hH*CXzMXBCPeUK;nSB{l=*sBhIBUr6r3#bQ
z6eJ3th0>MFav~U5#VJU{dGl#^f=R2&5K-`RZ>7NHD-hOJgY>t5gOC#V_-yb%gu3DU
zHXK|E$jZuE^2LT2Ubx3kqWx`V_J5y3{v!LL?Q*E#%SmGWOgKfP%=^5dX1Fs4F04(&
z*A#GcnA-ttz7F<CBT$|=0d;N((xwsd0hda@c8YFmM{>Ho8WvfCO+owlg4m27`PPib
zrj5qJ8rYmFgOZ<wc`!){W*kV3u%{4-f+sq>Un6}~qi__(N;C7-|I4Pg35NI)IJSgh
za(kqJCyuv>1IhJY$Z;+92E47}-+$9pc)W*}3lLJd@%2f1{5MO8V7uX9WBd1o*%KID
zfbZ_^&X9oTUo)Inru+YTu7+5DZ<a?3=V8*j`F?0`MxeLOwh_XX$~e~`q{XyHe2pX!
zzk)8(fVJ@KBE)+l#SD&|3_^&<rQc-)>l4z%G8z$0+maF#>-@dQy(!rlNf;uT{W}aA
zxsyd$-c*iZPtLOsmGk|Ak?L3lP~{p)&JEo}yL@EMBc_5Cc@iKjJs`C^I{_0y{E5Jv
zx<-rub~WGY*UBtCoHCh^#)REJ?S2S=l}V5?`ST7Mb#vpVgz#@t_almWdX9HY;m4#f
z;9dR<SpS9#Csh9@Z~o6;e)?uG-9P2ReJ%eWJ`14Po?N`D!P|&N6d)wE1NXuApdC*{
z_!K$$dv+5SRe;=;(c!aU4DBmFz9|J^9*i{M?GCIv(V3+3Kb=qg?_`(ccwTh9VsRp_
zZ82n<O6>-{cPs{-NWDZvQo_oc?wU}q8o@b8Z>zj884k$9vz_e*?<-AOD8N$i)3_NA
zrM@dN?VzTlH1v&LEdj>oVnPYoDi%Q?(8yp4r?@i{C`vge=6`({L&#I2IWBtLYxfkz
z5BNWk{=FIh>-T_fQYKP8sdi~Sf8g9jQAWn2Zbw`RCX=6PCpyJVrfJW|JPV+J6L>;r
zE_aBB&<Z1Rmu;v%`l)}BFR-#e$(YgxM_%8N`nva~bhovV)0M^jlP>M8XxeyTB_=^I
z2JCGmYWS+YpCE$aTu#>c3WCkm>b3gh$uA>Ht9HWqoZsD;^vU|+_a&tjG?H?)QZT@E
zdnHN-*yG~0T?p3cTT}R74>A}|j}&k^VneFXO}wL~T-n+x`c8!d(G9F&`uyJq{=Yw?
z-^^e9Y;^9Qe0~7F>mbC<_-#Fyizl<edBcI|9MSZl2sShN;nyJGgALY-QuMa7x08)P
zF?Z>Y;P{hCkps6joU^m;4b!JlTj7TzypsYM9AsvdJJ;N)(BW<kUlWn>tn=9Q?*myU
zoLv+|`sr*aJBMp;KW!SMea0RFs0m`MUJrT$OX(G=wPg{%@ma)RBU?18e~Z>gyewP$
zXOj8pJv3ze8>AC?!0Y*%++JVrd<2R=M81@2#Q&L<{^#Qie(~4q+5~@}&VPvPhXsbi
z{L6J<ANsNZZseKNN)>-Qn{V<>SwQUGh|%&Uq?l7m^j#ew98J+lcUL1k!7hB)L2$oq
zWI$>Q)$SCP0L7hl&SVlI_#+YcJ2(auV(bo<iIT&Ao;gm{A08w;OL<U1Mj?V`4?W<N
zBfx)-8}W0vk_rhpXdD7$MV?^7$}s>F?uvB*tKYaM^SIGGsd5+f_g7Rwz~naWADGvN
z5>?4pHVX~xo+hLJeQth?;(u`lI40O(>-N~~t_Z-oUFWl(VbE7M?+bBLa*k0S|J4GJ
z+kNqSK``S`uxYR7YeK<4W^lj>jtSqxnUpQt`tB@<0DKk6y8L6GJ54T)mVEM;cGgMZ
zFR2o|f<j}|*zqm~tdjL=9^IO^n!*3g0TN_S2@F7T7R~@T20`E;m(Z0I;V_vs{iSD7
zT}aM=hWEiJ@+u>FaZL+xIf9Ed7WpP~c6Y}`vAA+461)!|@Z+t-O2_Zv=qA|AmC6@g
zbn2BFaS{GKK=N8=KXVY4r8fbf^_Ig0vM@SkKN+7&r60JLvRFI|m71uJ2=gp&gFTbe
zZx?{a0GRY?_3z|7KE?BgnxXW1S^=Q<&XG~zpL^!U{?|e705&>dg)PG2#-V^q;tC4o
znK)I_`RczVKf^-UKARl(pr|^K3Jqj3e|z7Ee_7Dr#J%#|`Qs8(z37T@8=J}ry3ko{
z=f*(Zt<z)TCB)jXdS+|v1%ocU?7EutF!{Dm1iWo}RK0WanJU={ld}5#LvKO*XV>kM
zoo)A7+BqIZJPVMjNxxXh<g-Sn6Fm=mJ=IZ**Bj7j>S#ubPe1pQl)fEJO@=vXlUGPj
z;m?|@9(=3Y75;EB>Kk#d!yfi2!+!^Dbj7kueDL(Ft?29`H;C${;6JJ+KuCc9X2!4~
zd6?J`nDz1oLvf*PUUvjpIJ*UWGw4ay7V}DEpc(PqWIDoGzz4X8XS>YSeV&M$FDo(c
zvLmlmfOpmTtP2@!z9VbCv-gms6B8N1cc3wfH;Cu*Ybm)198za?70Zn>v{H}<-3tlR
z1|_Ksnf%N^F`9P-i}WlfvN1b=KWU_gZ!UOcH4g=h+X1ZR@8(giF#!iH3}3YBzjf6U
z@yOSQg9SHUIGjxZC$@f=F^}(-r3Y_s+tpGz6i-K5AfJ&40Ft~`gJua{JDmc+Lw>A0
zZF*-1&$AED00cyi)8srV_;{^5&-Eo&LFt#$Qnk=;5gjMn12nrgHc2ay8tR`P6R=8B
zm-jsSXdcOAeLl|Mgt;7fel4jX%;zYV+U@`*{LcD5kY~K=U0Wji)+}@Vb>6<w#gp*9
z#a_8KM<rYSg{81@Q|8gq0V!9d%&d9Z{5-vEG9y(2qgrxki)t{3sf?1DVtSFF4|%rF
z!$VT0(4X7zrFzThsd2Lo3Q#Pv@#?m2t<@0jG`gO0x!dvj=&KiDYM$@J6v2YxTZ)Zu
zDSf?5xtMkmp7(73iA7|T8S8f1SV5g@G-1o&_aNw^j`0<#Su!f_wmPpX5})TC+4s))
zcrjOVx1~-$|5dB)@yKhtebPs#KH{Zhw4#WRpYKG~`F>U^ipSO}xt6z+rn<r_@Bd;N
zN|L|EdsS=GKlC9u!y<TLaLK_P>4@==iqcd<I3c(|xo}P&IIo3ap^@GQswuU10*VNG
zc@+Gfc&I|F2zvD+eaJVIkq?^7XJ+H*TMSA>k*VVb0N2X=9I&kjE(Ugzoo6EFN4h#S
zDR>Go=a}W!fD~aSSl`1JU>eDpgW#<(T_s=I>lq3t?%kaH+5UV>Z#7l+B!f{V_X`kj
z*aq;)Lb$zVm7(NI-R6dREEJUN6i&0C-LDqoh4D|T;D<*?N`T6eQE^iR=GE1eas&xC
zv$M>op(u{kOCXtZ7isbQ1JM7j79T)85u{0XpSFrTzl4@RDN)wU;>2$s3-h3tj+_%5
z?myMl>pQLDxip^_@kdbz=PK+azUmBa8$X_z*?CHv2f*AC)}E(?q$9qwN2uYejxJBD
ztF=Y6iG%MIipL=~8NlA{WB3-MTGmP1lISY{R(Uq(isf1ViJf@pDx^ZON=Dm=jM0<N
zQS1Btwr65=Z5*&%dR>it-*T8_o-#d8FOyD-(=y&Flu)kY4~VT*8=Znv*}b3fb}`R8
zTgm<@P5e~4?z4w^zQ-dQqwhfRu^SHr=+t%C>{<Lys=$-tr28eWw(Ff&bQxr>6}(%w
zeUQb%C!~FklhNw+)AGNbJJ=vybWn5jio;u~TV?W<7b|c|a1ao^G+*K%gNX3ipg4CO
z`AsrsyD`RDRd%HNYt!9M1^{<hsk|$P^6a($K0_rBt`VA-<`Z58@U!G;w@xQv@tXPR
zGr=XeRzP|txCH8uuPut|hY`@(I!oYc;2`GTRyVWhFLYs)SBm@%!vp5$5Z*!He@(eh
z;c8w37zU-zyoJFH<a^)W;Io0niyT5~Da?MS=K#h>QuFgvci8Jd9TroM?64sOoF4WR
zoK^oxfw}O{T6vmW*<7HkQG~)me|~ERpWrhZUcv>kyM9@TVRnD+(W3bJO_~k49E5NF
z<W}~@<a+A5a+&oLQ1JT9*+C=z6P>I13kx*@!H>s5Z|D6yU+Cre90lb01Tdf!=AAq$
zK!21hLaJmJEcpY=>nre($w|RD{>U`Po<Bn;e<|-(D0HUZ&5oOK6+Wu-yKJi`do1x2
z0ml)ODqRsl28rr%HJbq4af2*ud!gk;!pp7nh5oGvperMJLL_r}2neKoYKuRlT0!kV
z;KV5934dO|R>t79qynj?9hQyTlb^J5hiyEo)3~dz_=WXfMB`S4{WPOv^X1UOc_kNr
z_@)DO#q41PjxrY64QT~2yPF6T_AEI)r*sZ;eFZt&SQ|nM1<!3mVUQ7%w&L~nyAg3z
z(#!yNR0#6_SMjPB^C)1;W&C%vQQP#c`fPvg&5EpFhXBMn#RW&rzqmo4y8W-=5S7OR
zp514`>(ep%`M`F+dhz?Tv4Y|Jh=<!F;)8_?<d3cPiNODbWi#l>6G~wGbn@}iQ8bIc
zcJ&Am0mncqZ*Omp%O!ajV^$j=tTZM1?es3<bi!vfsp=|xwmy?lmpb)w>N4=V6AmEt
z8f$u{EZB`hB^wW6xo){h3|hPRlJ}%V1|Nn14=6jw@Ho1ZE<SL!<c>pK#<PTn9W@ri
z79}L0A*aaNSoah{ZW6+5R}mNceA&5>_<W=xtJw&zk-IP=)PAd>Ce_XK>$Ard6E*1A
zqV}}Wh7fhrwjE(q{qH<ZiV7eNyKlo(IDg&k7VOo!Jv?`|aZIpmd#B)|>jvwN;pAF7
z9axCxfJeb+)yoKdSMrtR`!@~gzDrxjUvh|9vS$Nd7wd+W30+4khXNL=qvTE6)ZbXh
z524N#@Jir^47P-@{rA;CdHZ+Q;-}(Q8f9|%Za8-MJ9x+`G9l0x0~MD}LLI>3sD5Bu
zd=B=2Jw}DBh4+h1S#pO^iYsXLZ=1sawkZ>GT{br~+c<%hz>K`ZNPQ|uW245E$h-qt
zQ?vAhXcL@1MA#0UzuCbW#iOjtI|pqwb_>elG`iVTYU#&%LnJ{TQi)Y-#NC0wD9JOh
z?j9{riS^esa`y2Zz`_n=eNNH*w@i5rv@7yv6k+BtMFF%#UjQbpw0?{i@G*GWCp*N7
zT8ZIs)+*InSNPReuVA(Fv*7eSX+j_>o=W-6k%mNUgq2UhJg?C-0)v2vKMC+UG;IQ)
zn*Lj>LS5*(?z7QEpYJ1O7@W?6Jded0MXb)uqnhHC>vudPbtZ1sLE;yO^iNNIbyHQ3
z@4VM$DLeK2>$aDlFJIXfEx*@Vdfo7cn5G`LVqueR>$%%$Q7fI7><E54a=jX8UckVV
zw{~-;?pJL;)x!FMXkbsDUH{H3ev-H5A^@LxQ2;$(3hQ_+PeJIR8V{HlR^o+e!V~wC
znF%uGA^Tl9td0BkhH)L^a@On3h2Bv>vf_Q|JH%BRB!{n)H2<wv{I>j_hGj}$p#v=w
zyUf>B2$jC1aKE2DX@~eH%8;0%Vm^x#*N3u$KfL=#JLF0t<6PgYp8a<^vOUgg;XQqH
z7oE~lZsfi8q<MIt|IY?O`!m#m#TLsO;DB;@N}ZzbckPZbV&FC7D10YQQBNcl!oKaO
z@gm%j!pR;^{t$}cDV7cO*2SA6GV<@+ogwNFKhTp)WY%N^HaCGBeSLLoGzV;e)fSkr
z6aK_v4_q<G#l3e1NI>CDj=X;(egz#ksR3Odfs;0`ILw3Pf?#4oF7+3BU*#;u8{fiU
z=fA{z^h&Z1IzxY}<V$G&Oeyl65^&rlI=J3bGAp8ce|`;|*!LcI9W)l5`SFzCYM;4$
znMQhbdL_-p84qJLmf6-osMGbzF~W0o3$4!B+Lxu`nS<{Ayk3}14wp#eZJy!3bp_;x
z)oTeq&`Pf1<MU#+{=~grzMOV>0*<(E<dhS+G|xTQ7MrAR6_4@+08cuHotWPv({EKD
zoa%>98f@|`e$5WCUWa8dmfTF)+}_;t!79{9CB{RL1?iM1g!9dmz_U9&O6loqNEumD
z?Q1)D3h82&@lK7hBYC&3+f$9=8$QR$)$8-<F05i)swP8n17o+UW3`&+GXVnAOe?*J
zO%5KyfO_zE^3Fs)OTh;B6~^Grfb+(eGur%ZhVdM7x+iT;=@nTsl2H^ObH}Aq&N^@(
z(1wZ*Ui&)CvJoOnenz`}n#|mSWzw%5wALM=^nIA3P}>Y^Obr=Qr@|<V`}Itsom;X{
z2Y=f~a+V@j;DaI{GjHGHJ+{dxR#Lq=?lK+u^}hHNya<ZALd8={=c5|NRy7rL55ym;
zlIk5MmL5=@5qgDuaVLj|JX|?{$8f(~0AK(uucFAz0)rL?O&ha>i4wI}p4auM9ww}>
zqYsy*j5oI98eW#$H(z!5d>=}o5Om9?c5k@&K2;mG>v40oQ{r~KR%<$xDxLKlknNP$
zyy&vL@%lq>;Bywv#`KP48U-v9OIL{Mh^(n1vZ-(%!}D#?p!~ETNjLY{5m|N2Sf)&9
z@O&?~?)`j~|1gTstHOmwN2q-i%zYqVCXNvR5*|0MOYTs3cxxZKMNK6rCV4SHe9al}
zPL$Ld_B>A-JZ@tZRW|083Sfw$C~2Flcs$a-6-u~K3mK$ycAWdoRC}>u2@ahCG%7})
zf0zGHlN`eP)Y(!|d*+}>-Z1LR=5sanQ*ZOw1kT3OVl|Ap)<Hk~91tq|Tc!fAXn5By
z<qTP6QnZ&wA>mtCWIB8witm~P&vIBH`5feG&ZIcHR@?C7v;FrK(TB(Ti{zwBR<=)#
zVQ>QXE%S@BDYJ`56&r^Km!@wBWEav!$sTaJox4TJ?{8*;pBV_>N2x_(ecpCw^$@(-
z$GwMhd9xc{#g`ecSpF;o^+arRxPBihG$S(Xtdfv@EH;XziX>t}o`lXStBWNY3B_%N
zR20h!gd?uvHpaKj(rD4*>J6aT88Z4y&9vr(uHe*PeI0!P|NPk(UojD?b`+Rvo?XTY
z4(EHaDjY-4pQB)Zsm*<NVC<y~cj~jN?byRdZDK?56fSq!pW4NN6D@?iN7LVQ%(ov2
zND0CuSp^_C5Tr@4!~iZW*2(-cf2rG)1Nj3%9l>JlN!yJR_tTGdfee{Rhabh#lt8VG
zyd^!xp<ucur=z+`22I?;T+@P79BHtJ=paTP&q(O~uXD?ysj8~)8$}6oYMm_8Tm=kv
zU}TH%Pc0v;NP|eB*QPC_0%pwfv8o&8)bFOTDheF6O?Q8k*#8Q_e`O$BB2e$i*K^uW
zeZbDv((GoRf%!11MkBp_+kkl@zWMxs(NccBkmeF$_;%}Z>^LD$<4+ODkIL>o$NNmV
z+5N%SMc&e2uIPg|L%%?y)p;D~MGN>G`XRFajd96@sEZ%1gYkfVfM!!{1k<|?5>?mR
z4VNY#1BySzOXAo9;aOPN!kDbvbo~S$Fdph70xeEeE-V)r4UHjthWOa;Bfk;JV6l>O
zxom4`C{<{A4Y1TX=c!CfMt#f3mihp6?o|~E=Bx!hK1=OshQ;T-y=O=Xa=+eZ$G91g
zUE<)r`WcUk_hYPbK(OxJP-bQD#Svd_9jCE^_(!C=(YfN4N5A*nO~g+15e0IRQ@+8T
zxAG7U29y`)d(+>!(MDa`dOkB=Y?W!(Bd`!J24LFp;GSCBh}C?ox7I8LMqvlz!;bVQ
z3HY?fjB7{!I8LJ*98kI4DgR)8heIfL+th+S`)2ohsbbGB!Zlk>p{B<`k2-SOksVT?
zf-8ujk=lLd^l<0CJ6;qeN|+h}6nD5_(|{E(l;4-DPnvz#CsWLNPyVL~uL3SKtwvno
zP1xR#IyIVI+7&h4Yo%Qv=9eW_(R24tNvH1rXoU9epFi4t5OE+oPy=B};iy0S^R{MV
zUvXfgKml!A_&27h0i1BQaI*Gr$yUCk$i;aZLKQWcZx8^v15|Rf-Qe=?&`36Lk~>Bj
zRJeES){)%NFPp+2_ZTPOR)X8SQ`Fz|zzR_~6T*hwQk;hsH>vd8WBavCymgmHWRV`N
z^(>~xLfwjYJUU4q_Dc%{3Y1<bmWY8~>uCoMgd*OpnaUlIHsD=s^U+fmm$GM;jUbsy
z<05hZG;^J9&MctFY^wRep0aR$?TUR^(RPx6866I!^MV|(X07bei>Sa>Mqvcu15x<a
zw~Z8WD~PA>N=xoD3fLX(MN$3$3-0r8uL!rVR$B@ySNfB33tm}xeN7yk++DWpF|1m<
zpu>ZBO1urM=L}EFbrTLVW)_PG;~ho8cxdS}v(3zLu<E;+W;yrphi(bu?QA_Kzl->M
zpF@bD%fr|17U@(NnHA*yic>Mj?N-!!qoVl5hP^12n{mlPOsokTixJXWn48NUq(>pK
zWBFJQ0?QTT6iB`JQ7|ChXm!bZF#byTrn!J6?_Q5a8ohB3edLGvc{VmAIt3yBMbg!j
z!rLLu52EyV^xDZd)MtQdo_2I&S0b!)Rb*s_ANb3Ij0;%@P(3D@Dcps#@6AoFZ$93+
z`ek`HX1>qdz#aCe4WK~C5kbO@7NIoViy>F^7|^VKKDlRd-#iy|HwE0F=O#r{A4z42
zL>A1ozZNBGPBPOVte;!0KX8zOzT3!Y`y+F30BNglidka*KIt$|<5)+bPXkg)N<Q24
zvzDzk9<WQu1^Ot;e(;ElpNwhA>Sd*IAbhQ@wa^v6VN@Zu=q;s|o@3@#A2ktt68;is
zmXDD=zC85iA&z?xaOV7`TVT5gPmjGby|VA2feRtQN*B|7p2YrfEX8ZyT-T><`IUTx
z^Uv^#6m#>E7}e9xEky2X0jH4nkc6_sSquGRs_K$Ra|3ntl7HU;zi(K6_HSW~Y}gmL
zgsRmBS&uCZ?Kp6MFKA<BO9-D~*N@(|V-i}!x?TtYiXF(S0r&b)e$aixO2F|f;eBO*
zJ$gJ5o|n#6<MV`CD3&2iCkHE*(%v5A-QXrxY=qk^oynY(E2j;jN6kg_bTC-+@TnyR
zA%QO7@LY;d`Cw2CR+KYXUr$v|00rxmBYKM$*C>{MAAuwcTiUauGmYpeZP?`k!e1;8
zmQH^qj%?uSU&G%m0{4yxXL$_B8dK@lOwLX6fKaId8-K&4-m*xORxa%TusJc7x|!AX
zNS$T3?*?acob?LG8&jPC`cuf|!siXLdkc-#lvTdTC*FX36Kr@(8fC-9yq?j&;;`Gx
z6o4t!>{g$gZ!&WQ+d;VZ(x>~_==w#+=!Z&yP6A_<ErRE6VTse#PbzVAFL5Q<)!9xQ
z1oIiYQYejPMJNp?b?C%p$%2|nJ^t)YZ{8l2EQ~R(l&o%GURUwxF3)ko45$Uw5y}M8
zRyqO}A@w0(9f(MbL}FZR&n{^{?q^dM%4o24fm>fAz5-bWLWc?)Vf}EDRC4e~xllS3
zTvX;ah@$|2rSW_oy@JD3ibnTaU1H%~yA5J&-`Cfobc`UZLEiJrRwbBlt4n$Xg{~B3
zBJD4gAY3z6`t_4AVk$8unEnrjX{FI2qqLqFmT$&nmuHBLVXP0A$R8JL<gFZgXj~<K
zs<&eB?wrR7IJr+7`eh|ST9HmS_L*M8yCZpd$b4v~DtLJpS187V(=5Ck@))~SN*NhG
zP;{PSSPu^H3&2R4aVXLM<8gNpEfqu-UnPP{%*}-Y<}@9QP9gEH2{-DCR9SBI3Rt(?
zRXglkQ?F4mnZ10tN4ESh&zlAnqUpLGPht{J<kAX{X@TtgL~MJ+suFdhbThthWo|5j
zWjL1H9MsWhcZLGSyYhd2Cj2=jKRi4{PkQff><C*CC4snA7NQM=3WB1*U2}NVz9P0d
zEehgI4;2a+8<>~0w$<g2Wu7-zAzF_UA8TsL8U3L+F~$}(%0Zrz+$KvW!YX$X1I7Xk
zz@S`A?x-Gjq9ywtD*UE^Xg9h|No?y&?txJq%+I?Is3cc<Z6lffBuZ3IZ{}ZTtDmw6
zWjYSzDwqHpLb*b!pG86FUA8Afyd8iEtm}5r=~JjAcx;PI_-13;`Uw%7dH9^M^FFK6
z$d)37fa0wY1n(Lna63Kfzgr%-N&2n?AGDhrt=7i{V&)<v@?dYzV4-y)h6*51mXciE
z0Lxr4Li02PBEFQEyE>hj^AL)hSpHf~Xmfsv<3@kgDj<HdjrBr>7?O^~`2@zGr&V<f
zKt#<1Bv!xsCl`DX>8EO6j==BeV(Y!YhA>-jC|@j04XGV%DOudUJ!Sa9Y5bdtXXuk<
zfXzINw*}^1Sl8AQ+G7~=3sVV%HGi3-QuS7y+0mh7$>Y;2mbv?+@{lZ-a<RG+9n1wi
zY*!o*PTpg#x6@;z9Zx@sfAr}0w|cUxiK=su&*$MygGbDJrfeSm$Tt+hdBSfW1pPD7
z4E=T2`s76{aEo#Rk}$U?70agw)zfxO>x5o3KTheAw3r!3uR}o{2z|8+ZIybw(4%7Y
zq%ZhywB$=p`72X@+hvsR5h?|tsJ`8)2;*M7IK~**O2CnGqW&cTFX!Un^rzUARD(|U
zL_WSP=0#jP3OG2*-QFeghndS*$U7r+abw2CaMjI6mGKlDYHL*RcO~0AH7f>lp<+sH
z)apbLU7;qKA4S?S{Hj#Cuc4|LgsLD>MgIlCA7P4jf?8AOY<#l4EH=6&9_LpC2dPtv
zd%Ye%owRd%8k{}RGIWnh&hv7INBrw&J+iTw29n$OxJmidg0L|B`+%y0$)Gc^L+khP
zdS8Zz=a<ax<C{s+-HFPq^dulAkVWsd7n~>+u8e{cb2=^<3FmtPk^P|u={L*^q7Hak
zHEV%99bH{r0VKS(^6n=aInpuqR%47xSwaOvX}sgW=Jq!5wf8r+^{hDTbu^#i$HFRv
z!hw{8GL7Wj`eOP$SG&cc7d~Z0S)aMbO&6;@K9t*PtAMbyCBhGy1pAaLRNobFGytMq
z--=t_qAj?5MDu{{`b}X*$cFc`_t&g(h2pV9lG*-g{8J0t@E;-4ipcplD4u;0VWh$S
zjRKaAa8c8dhXI2|WN_F5l!M$OrzpwR&obfmN05a#U$!H6@&n6&zHA`pnH&tCiJXA)
zrARM>F2Z3{VUlED>eAG9sFCH*Z1oj<vel4|Btk55diZEc^HW33D4rYl)dhl=7Q-&)
zd+VQsQ7F;j3}HmYj`F_Ur$*6VWNF;9DeAG$PCa!b$cZ65U(vz7o4;Nxbpw-Dm0nmc
zI%uqqKjgjo3myTv#usPh=q+Gv6MU5j1ceSBec_Q@{B*h~3~p4MZD!<~MtSDUe#bDu
z*Zl`q1VIn`sI+Bciz<zcrZygbXl8WP-U4gt<&lo?(KqN<WQ<~N<#8-cq2y-kYAX98
zA#mo+A>{GHiTA<KVvbgQ|77aR{Bqj91y2(i1z`wX|2mKw{Lt|Pit3L!y@NPd7)@F!
z^1;Nkkt`Gjl_6jTsu#EJiG}k*tud5td{Ao>tPUEP#I^YH(uA3SOXu4LkFy7s<)~^!
ztiUji#UzWg1JghhK_FhTtVy+_gszBqBiBD0=Xhd3e`OynH)q?dm(h|VQP~_L#W&fr
zlv5;ACYDpm$(FGCSV)ywlYi7E{pBJ!->vex{xtf*f1*>WKe@cmMc%AlJPjbM+Tl8T
zUF?k)o1ANbI((%e_2zEjE0f7^n&!n$)Z&QU=5SVZyo;Tn^mt-KZZJ;yci4NDIqAJ5
z8}10y0&2Hl(vUT~yC~rYY2iB<LStj&b;t9*DKa6?WCCUl5nv_(HcmnE8~~~7o0$0;
z4rR2BU7wXk&e(mFos!JCY?mmH1?j0*m%r-TNGT6$uuiz1vCW<R_)aFKL~%>^t3w0_
z!j&xl>Nm9;Gd?jSH$)yBovGb4R=_~0(C3)Q;e7^*r%_Q1C1H9=7m^S!t0il--sb(7
zD#GEub2_HkRl@h5B0w1>kOtHV=<qiCZZ<VUx9;DYC07n(DZ)7-X9r>0Y<CEVgiy4u
zAOUMM7viYTT-&8|DounRss<mfj`O1!Qb`67)^jN8AxMrm8c?1)cX<_zR!VdGFMc8+
zd=MsPxH$1N!e3}_Ny9+OW$U;Oa0ItB^oQZOSvGEDIb-|!Sb8YEe`a8!8HlasM(9&r
zST4ah;yN;uI4tmA*mV9u`Vs1XBmD;xs%UP>_p%v`b@E$>r>o?GXWkM~nZo&K-`S!s
z(<*0YuB)-tYo1a7yckdmF!@9E;`af^!n|thKUv+I$lg5IE$;WBB1>-PCts|kLdjB|
z6EKORE`&49T^`NJULG#Q^DOU%0a-xqaSjDsGe0!CG}an+GqPCZT36r7(YgZ)eIb~r
zKC~zVb;(*TKbqS009sAIkS%UZ_r@nuMH%1oZ$2i37BEvu={Op^7!p4GG1XqUY7OiW
z_`RbRfG!`$Z81xwysA$cclW;JI7l<1Q-R&J-m=0c?6riZafEdrxB0~N<(mK0e-B)E
z{;Sbo)fc5qQXbx0)CXx+4`-iSgJ>9Aq0UB2;n+D0ZYz8wWO7EYRCLyq2lO7JEwOod
zA}WQx47<4RVZ_YJj(A_gkUq{eM*olq(WYNJKV|14{#Bn_%)TzO7}FE5y_!H%ZL1N5
zdN?NCjBkEI+H>!@A2aJQ!%F*egx)Q%o5Y{AW^biFUP#Vz=i#aYZ+F&|FQN2IaMHqd
z2-|Ud4-?}MqIjXC$q`E}RlfwDueT*+$*h@MHu(DP>sw%0WBNv~*-g}Hx?*z9Km~OO
z9h~s;T`u5WXcb7L%O;HtxEbaF&8+cAasdr;(*i*d*6^1uMlB~=PWrhY_t--jwSi6O
zmg2<Kg)vPLK98?zTpJo0vRBnU%aH2^xsWPNe_I%vyOAx^s4A9Cc<J%tV~a<^#nH0e
z$QKEtv3!}0HGi|q@U`9D(iS@`3L#oDKhsw=rbD^F^r0&7r<v3W<?9#m1zfBTPNR;-
zzv&Xoc-YT;x1$(Jw-_SZU}!h{QiB3^EPco^8788QeEWteQW{mJB(sWsa(Jr$b@$K~
zulamkISAKWgAB(gIo$H^BJ?jrk8Ow&DDL0KT{Sbk)`_LzS=IH$%X|Ma;LI<!4aP1i
zzKk3R4{C<Hs={z8c4Vfv7QrI_K8y4K;jw;=G%qrIDn8(K(+CVNt?k-Ka&G&#ci`yq
zAwlg2Ji2?RM}x8{@Sv|No3WYw3E>2E@vUcryc(CSmqx1C_SuEztkKAK>m9#f1ai7r
zqy9n%UyV8*=dKU#HM^gx_J$Mx>5Rt2wV=wN8qR#P6~bUP*waST9Fl_qexSnVzlo-F
z3#N(R2HaEvor+}f;q?&U0X8bnyY5CU+<a<CY@VBC?wYV%xO(aRy+S=rNZRlpA7-j7
z>9b1O7rZ;vBVD&j+xsD5wI&CcO|G}1!E3Sf0bgX*8JX>p`;@Rh<ZF4$=_c8YtqL-1
zFGqZaC!be&^zS)U_RDQ#dUu15Pg97t8Zs!iRWTN+aJxNtjP@*&LT7)>Op#?OjlO+m
z;rjhbmHQB;?aylEkmill*R7&kdg~VvKb(x0WyHVdJtX+Nq{*t~0-YB*tig!8B6dNx
z$Hn#oo(~%HrH@kcQ%3}8cULwBFQo3@MApbY*yf{vy>aTGE6<JsLt<5P3{5cnbX#h1
zk97>Y@k#UUYI>$O+E}c1X5*w<L+vU}flOH1M(;00Ruv;uF>o~ZWA>9J+Rd5pL&?_?
zlY&R3`#K}7hc`D^<uh#y=CH;B1~=xJDqolM>){Dju6PB<`s|-OWwY${VHoKH$P7sC
z8fJKq-*=oTw&3{<v%A2Z6P1dr8`=9XWL<yQ$Y_*8j_EpMJ>Tyj#I!2DPaDT|cxM|o
zGNM|1vROhaROGkaK&AMbq@`al;M|SjF;b#q`^HFN5CsP~le*-f+x=ryg>kIQW)Txc
z<!M15xMwKyP&v|OUM0D*=KlIYAoHDU^0RVo_!kqw>~rJ$RYKX+y8K#B?Wh*V%Dx9}
zdbircx#$!8pRB7M$9Ht+7Q$oEF*%=q<*eC_InD}u&D!CMdKLvS>s3bbl;0>#wZ$im
z(7HJtE~EglSRlH-zAg?-F2tlL4D#G^cPUrQh{K3De+c)45NiL1b?4Na9R&%V$vq}k
zmk&P}%4~_YNZmZ#-dM#wI~$v4>!6BH+BM^Arbv(JRWhRtedD+}0*pkM1MhEOUTpx)
zQyctn@dxtuF0+5%TIw&RPZE17+KW7ovLCW^QSEi41iqS-iDFc9U*Prb&2p!DIU!|u
zx%Db#ss4{3g*j6`fww=PQr_+_69&!8!@P!-msq-Rrx?n8QRMNJlBmDoWj9f`y;QzG
z@Ks+pQY3A!^Y@97>D)BhlhzYq(cPTej7~Uh2f)(H$|;+xjj-kKlVO49bfltv9{2eo
zg(GY_c2Ir-lxQMGUC5q95)$PNB`l!5w=7a<iiQRy07dXNn*tPNY>43g3pkZV2w(bB
zU~*edpd?hKv-dYTZkg{*m2<qy<6HxrkN(1R(IULt>=_C$IExYh0XuE|Cf{BB!Rv);
zoR^(fz}zMYTI&s#GfIstFpnuOdFS`cwp;vaKt>W4ZLhq+7-KO}+q$s}UM{V4zrQ-h
z!#H~5$C}Y%Ili85I$Dib?B3@wcWX#`zAzxim7DZPsuy|leQ#7ER=~RN&&8O#mHA>d
z%K}$?r2eH`TVniI5gzeoUy^c5mxAwhOBVE*Xi&BEhN@Z@kp$jv%c^@>O~mqK=X#Y(
z<JfH3K?k2vKFOq2M8JPyKVwO+^5-dkP#S-;`wj7v>X*MWwa5XmM<N!gzQZV$>0fA!
zd6vc39K~UbV^Nl;`RKJ=>4DSoIcz^>xk7`Ppz-Qk$Vv>m#oJx2$(tjBOeEVk<zL$D
z?E6B=nLhs{t9}gb_kV3NZmgy)*s=coUEX$QhA*mTp=f%38GYuI4eZ)tvC9>6H29;q
zKiTXoD9Y#;bo?I8JL_Qz5AeaA;V#d}dgDK+Tsf;DW{Sqy;rdN#O|Xi$1`3qkVncmu
z`w}G^u<46$`DpW-DGFaW;%(nDood%G|3%Y9%`<!-FR5EsSO-bK80Q@eW=8sKh;KKl
zDarMg>NNOwWH_g>$_pIR%u<o${-GT9fv_uyi;G|8=*W9NPEjnMkc&IuYUi~*_lI=N
zE;eDK)^xyB>lJGv@vFC1%XSVB<j9??SXyROa62kiCLs1FDWrUUQ^z*S%ghdNZ|wG-
zA7po9q!llBd-mcPEyO;B7z~Z&4v)or6?tzt-%?dymN+?7gKdFipzZ)HAMEVNkn-9(
z1+O@qZL0u6pL)JuT>;NWNt;(OSOIlf^Woq&Omb?6iC`5b$%Xidm#*5?-8By%yU6(O
z#uE>l8zM?^|3hRkf?7GA040Kw&LTFUZMC13@4dHmaW=>dj+m=*If^InHhW!Yd-dyg
zaq{{~QPe{?!%7T52CUQOo{xya;k*?P(cO2Kk1PRxe(!<`k0>=0Mfej>j5JIn1kFEW
zXK#dkrufv~0Ln*}c3x?qf96UJthyZ&nb%<KR{F7_?T#3D&g4WoDEF<8Gj-#zkalqd
zK$FEAl71zz{h|;uwZQ1=+Hn7yLSD4z>39;<<+7f69ZJku2pO&iILWiS#`A%^%8w-^
zAE~e*Ac(eNu3Of)?y`?dmbV^fq;1Gh$WgbC3?6L(`T)lW>NmVIWwOZ=7P<IhmF@34
zUQDh1#c&;O2LpaYZKw3>8_2|^k6=9;FjA*b;`iCDwiehcFzz`jk-44I;#**B1J<mm
z!l0jVc@?le4Mf|vDt&T<<~@*m@D??HDNpkc-nrjLq^ff{iI-ulkAzg64EvZ*=3+p~
zn8|w-aUdkbF!W*|c|Xg+YrXXshqZ@Bm(w^m4dV0a!aI0->3AHSDuTN{ZB1c#+`+)_
z$|p5we9N>rcs&EKpV!oAFn@}F3M5@J!O@n|bD70<RB#5Uqw7ZEKIa$Id~UO(uy_v@
z+(K}Xk31~SBapB|EA%R?h*^gG<dItYL#A%wVVVH4vF$b*z?sPJ|3JYRpbo|hslhKw
zwgt}x)q9T}=?WD*((}An=Dd0cZ=JSV1X7k??^-eLaSP_B`w@Q8;N(@_?2&x7$v|er
z(qBTHVu5NXV8KMSK<TL?#jWpy9CW{v6%G%=`T}Y6C=pY`aiqTG$DQvXBX$^t34^c}
zN4WrM`<d7_T`ejY*eWjA7>Gl*;x1CkDnvlW1pS&GEK&bMtzM-0PWYn|STGM<9%SiZ
z?3VTtppA(aOvo0SU7%@t62z&%DLfE6K%><yC@!o?4Z8*P)t2cr#y$Wd_OE+>16Q5Y
zvBpObXYGaT0xr;8e;k`qY4)TlUH*AvHx9+3D6uIAYw5X_b_e5hoh@GimP)Zde*8$c
zJlz}`2!{<e6|3Zh8im2EvsJX}EDKERHEYdr$^mOyb4o4BYmiIU1mGp7p<+1NY>8HP
zY{m9fkwVCTjrxq%zh8s5{DfI{W$h*M;wNW31p#A4U5pmZl9;E4$3H=bg8Fv_r;(S`
z&|c9S<+^mV#qaj)%CI}#?=s7#fhBcJ>G2OL`v{T`oC7w3KSlc>n{Vd42kBKOm~a<+
z#sGq{JK{*nKLZGe-kjM?A#0uZ`jN~L>8n84b>Y%pE0dAp8R<7jP_Q0~XRM1t?Z!1|
z_kpGkHoHa?Pp_CTh>jI0biNXrwUl+hN6_2xF%q9TpwBd+L!1&PcqKU;=$ire=;jd%
z-r;$Htt1SEE*pWk7Z6HEM~>|q$q`50YVvS*ACLzxEiL&CN;tm)byC0+T&i<Pr2AnW
zdmkvY4IWAebx31@AKq`?5py3XLuKRHm=V@DE!ra6MkBs{Hx_7;<09d8jfPZ%LJ61H
zO1u_ztA~l%aA+j+HDC1&tm|VREilED<ZVf?a7DtEOl}~T3!YvITTH)SCOwP1F&j)w
z^oW2dK%31kmTL|ExY#rcwcU&Hy+wq#uDnPB+~4mb4T~7u$QHc9n^_kp4Ab5{Xf_Vn
ztEpxA8SN}^o3f^224*bIdsc}9!JFQJke@yo#$#!e@~%SXjN15Ef7EQH!SNJ)u;dZ_
zLtbv>bS3jB&N1>P4Rg&mwvF_7$O>7^%ekV1&J9?{K%pr3`5}&8$p7kic)2x=_Sv=9
zkk?ax_{fUd4fGhCc)k{#_@dCx)s5PX7AWr_ue3rKDIPClEZNFgy@S`c99MFOajP7X
zkCtUx%nr5g`=5wA4jNQ^+cz<Wbn;BM<O3@?6ysWtiqR}%YO#Yo5;uwt@zZA22sa|?
z?K=L{05^0dl>ljQi;iSHT{NDKN@&sIl}#@RRtLyKfGrxK81<K@?q@Xow!P(kv|!8M
zdcr%7JzMHhNUIt8OQ_~Nng5Lp@f;`$fH10^fv5r8x9fdTzvL4-KJhsG%;d$pU-+7m
zgS?D~e9xa~yWCK;Q?oHBl5tZvGABH}M^mkEZE85q-|0Zdg$hO&&zFuVwwx?gFOYrl
z$?=r#v!n1~4EdW+fds5N(!i*Vfpc*JaKT!4s!=C|Lng_j)yK}3GsCb9a2wO}gnpU>
z-8K)JTMWh3W_HI-dD&RnkaUxSxjF?@updhf;2^2q3c0`wHXY7n8G!g*C!Y%hti%Y&
zfOkLUx1m37pNm_z)Hlv5Q&qgmDx|I(TznhpxVdVv+t%V5994q!Z|f0=M9MQ9lp+w8
zF|0jP*bQ+7^tuGWrq#iZzX1+=DleRo^@;-u*IFhO;>Ji->Bj<cXe^HkbKCE$uaqLM
z(*Jdgjlj>^7`4UR<j)ET$^Lw$bwZoHI$+O6A1Nss71lR8?QUTGYLH7{f(o|o?FBi@
z8G<K`7CT$tzGWvsn;wngzoo`9NH&A<xu4MbK~{^6f2XGa)8TYo&7UtLdF(C}x`-EM
zuBfzg#1MzbcaU9=mdfk8<i7B-t0MtSp_CY)M*2dNnIL6=#$|uzgo&7J`edyqtny<S
zkOlP_;3vDsQ@EdNMQv#{+98&s_YOG=K<^BP%qoapMZSSUj_c=<gLisSnv~x(R8CrA
zP7@YX6KOe$$EVxksUSRkq(IT5X62Dm<%upC27Xt1xa1)ssD(A!M1Fq6TgX{3WFB+;
zJF`KW<}+y&S;!GdQt$93x!wHDM=qbAsy3cdRpWRNz<W*U_CcVm2UAv3X~bN;L|j=1
zohR~!vtzxn-(yU)+4UkgGur{E1XEv6-0c`Y8uITR=`F16UjK-pcDs^%O^X74bLyqU
z#BA|LAE0{j?|;<akYACGuiTPu0Fk9HSTpx~Dw6<QY3$qI?TlwB%#kxI5lYNbI5E7$
zlXjsyv)+dk3X_1fdlF1($c|D=iYM^fm@9p69-3yraAGrY)}(&O7)g#M?|T3I`x!nc
z5?__P352zrA`woO5L%leJ8Gz%C4w33xw>ak4v^LD>~wq;pp&=8CbR4Vu4;=ULpNIa
zL-?UFlquE#k_6(cU(FS}vG6BTAHcG6X6Opb2!ElR94GjyRMPg1#ir%xV?j$zd5W%e
z1{x2klml*2jAhOF%$z{YJqsShuqRSxeG`ML9W}S~L4cZH+u0)9Bm_<NJrXv_nWii(
z!?432F~~ve2Mz%U>z+gq0x6b>qmLJGwe7s;(C(<Jp8sxF1Q;%Y!qoLi*%)mAu9EH*
zL17b%r;ilX15m*2OpH1P<-8r<m2z0`i)qrHK9a4D?r1OcvnKR_h&avGTEs=X+vWzC
zW2pe^h!U;0do6(n)AeH;0j*e+f#6aJ;*Xr8BdRoiqWLlr`)V0g{_WiOimmL4KXsda
zq6U5o`6Z>(AQN@(g(B;f(FiYp=ZkTL+Mr+btt+t+J~%|kHmsC3YUP*EZ9Dzi7LQ>^
z*7_%MI@bCZ@0iHMmxIXH3@Thu$pUC#T@q)&u9lm@jdu$<LKnOKH{@p?GnIhL;^?EH
znCF$)43cxbsBj-JpG_Xs#~-;$eG?mo4%Y7E2tf9}qmnGjQ_Yvg`I$2AvAzkY0K^&`
zn#<=iS<g>hq6ut<TnF*4Dxd?2ZQhk0pcFLT0q)>*12l^1d?zKztzNe;W1)u!k>r9r
zEss~3OIOXK76=u+5jaV@&F-QU!bQ56GjlVfD)}wFYGuQE!dQuI?5SSmGpto6Na)&Z
z*164|w`1>Z<|3&x7s3hXQZ2~TkQo@$;;m&1YLXITddtCbQs2PNjFxANqdgRV9!Iq)
zqCd2OJ4R{wazSgSzoA;{M}`_YPxasKTFe|f%XaNhwSj$Q4f|?T4*t>UkbDh|a&!v<
zEoRHzTz(s0W=B)ZpI~8Wn(?IT%&?`pYfAATl<+qbHOTJy)wZCV2!=zg$^FpMW`iR}
zm(Z0_X~Pm=lO^ceMGsI(6_4hsT~k<wDERCtMUZPxjgB)cU#Mtb!GNmmn)n7_RWpYK
z2k2fZ-q7;fTL=%$aEJrVC7`dYmPO*lHMwFEuzl0Dr=_ijY6rrCP=zymp&G|h${G3q
z|2wGLifPnBNb=qjeQ{M1VS9AaXo2pRw8}@IQ6@?9hQ161oE)B62HU@ooD&|tah-T=
z5ElfHie$t7Mp_ydgZt8KU^fS~#8JZ^D#w^L+!@4FodE3OlH5~GrFQ<qmnGz#ZDX`I
z)^~YfVq-t79Q|o(yI|-A`%>Ha5M@wg|EOXAWpc#eRF@Z(N)S&+9^hRXNCWYw`FycY
zMHa9pPt4QEKqtTdQry3Jw8BLx<v>BW_vjQ^ZBU~D<E_)$h-NC6VPxyP5_#GN&<f+-
z-kT^9KU!|%-4?6CnF3+yS_Puu6ax-bXX{bImQsH96ta+L_%bo^CkA_ouaH4ZV$U^E
zfcI$JS1}J`yxYupp;>gUZQ4<gaD2%Jf!>QvuT`f;>h(gL^+f+o%N#Z&o<xtd!^n7t
zJ)TFDKV+%FE6-v$;$u!=z5|imf;v~>f?J6?2#bcNwBX;4mfttjr|}69S#CCm)nEd9
zID1i{5Hx3CE{O3N`5dLDJyU6_J+OFYXis(+z#fG_4ro&_dxh|~YS0h_&XWP+6pkGx
zdtpL9K%)MyAgK2euSJ=C849y1&j|yZxh_P}oM+fa;cX_K!LRvNcC3V!h7Ye(((hA2
zSfjd(vI8S+MolTa$fMNVgHO5s9S8zk&}6wSHBzRDQ~(vRzK>7;j1c390b<?Ps}?D%
zT~M`!28TZwRYG1j{iMjhlRvvfsp#bMSHu@pr*Ed0*X%5mJI3?K@LN)57Ix8hx-M%z
zhTKxx&Vieqq6@8)t()@2h@Yzri3!J~HRKEmE!j0&^zD$>8ebZ!QM@lyO4SXYj*k67
zUnGr_@_~UTdAY5iZz*c$B5t0`=N2M!qn}mARIQqa8)4!&`Os!D?0#DGAuYjHkvPsI
z-epl}*_iF+nE8hRZAomRCFQoZ8x_AZ#A21>URqSJ;qR8mZOLI>LA?AB4)t&5g76mW
zKU-d%-KILI4m+;k1}5^aZMr<vUNe2l?^nfWetk6XC|@X$i!QT6Sxvm5^g#rZ-ToM^
z$&(kB))df<moy{Mba@7j_$U}8QdXi;NuMyJnV2hCFZA(N1oEP3XXdTLmqk3)-`(Yx
z{fhBaRwTqHXH}<X+Is6|*V7N>OlA+vK~ymYc@~p;8)hCGwL-wC#yb2P=PR<I)XEY4
zFeF=ROp@gY!@e-O+xHJunJ(5#QdMi+`2#)?Bm47{I~IHow0_zUcL@4pTUSup`$q=O
z2)^BK!}9+7R5n(ZD^$js`A*)J*H#Dtm=O;d-ZbBS@ebhZ^A%IOam6Y#`Xl>LxBec}
z^+$flQd-$ni~=v6B1jKvRF#<T{KTgV<^CFVd0JHKmAA|}chBdI@I0)C`onb{;2}Hs
zA)<7ngnIg7XPkkg+Vl0Io0IrwDH^G8c?3(0{O)61xy5zDsFGWX#ai!qSvMLteChYm
zO)u(gH}u>*Gd=7MYz0ilU2-<pnOt(!tB-buIGyzU*ofg|?{O&1q4*s%9_fMcVxB(D
zFRKf#gO5T?bBAd^T@EVpS6POqq)=o4rwkvVjX;FX8bgf3Qt0qAY48zZBMGjkXA<j#
zA$$-1@>flOnEg%Zwj!MhG}m#flSE!)Mz%kj?Ru}j7xM$Yv0F~ej8}2e*+;722TP%}
zzdky<Kp~yfIXIrt$Y7|?T2RXjMO_TA=7X2BXUw-T4f=CBjnkS}&1J3@5SqZ!uR!v5
zH7jM6_6`_&K!4H3Plh<w;9&GpIomg}2Yy!3o=e(17`V{EqSN-NKqD$DzMp2&)5{oH
zUvlC+q5%S8mLguB?KN;BpeF5v<w@0cN0Ov%K3o-rN34&NRFugb2?Y*`g8_5r8-iB_
zz`X0<!y&EwpiZU^1U+9E4FMTE!NfH@)OQc4FERaxIIqN_B3NEebx^EmjA--Dl+oP*
zd-&ho(a%;5eN+<Ck>jyv{7&2NiO{M>Oj$url{Ip1u`X{=jamRxzT)+@0XQ;w{J4#t
zT&1ILc3p%Quz;4OEF%u=U_Ue=#vEW5bKPJ%a{V>mwvk?ZFdF%xB4t1y{ZrLxK>acR
zipnVahYg-e0oL9wje`~o%sw4&j}^!T%7fkbH>{F8B=^*M7GC_}5M8o=0gBN2VwJ*r
zyP20Zu@{vY)QYlJviAGk4`oa_%mCfx_RVgQ<L-iCWaA$kvIPqX#TAq)AbJFX^CVid
z`d2Y^Aa1rrF&+f|L}RBo_PMvx0n2-Gw*)3yxWO#_uQ@FLFz$Y=;e{srSzZgq!={*f
zZ*DenB=jS!&?eP%#6&vP&m*)5^Sm3s<(aMZvuQ8Fi=9c#X+P~xyja8x+M(95>HfX5
zX=XbEzZin9*mOUScV@ih=)6-zqb)&5k(pDll0(=<&z}fUCB184H#&8iW8z_Ibs`q5
zX>$>GxS0_)AHE2fixs(p3wh>0_`*~E`J6(jQ|#8;^=$%dI*sU5lD_2af~Phgiz|JJ
z$Lf77KhOTFD1(lO-?}vXvMt~F))4w?woSSxgyJV?V$^r~guP*05u{unml&P9+cf){
z)SGpvh#$B8)VRE1=r(R;&OdWw@R>f*y3<^&_bII*akT!#9=f|%nyWj9dlO1-xPuCo
z+bYMoqR1_Z8shEOw|jL%^Bx$2eLI}#hS&d66ll*}2PXnm_h{we`p(0m)Y)pwG6r+a
z^>&oT8`l%up<^8htU%!hFO*QeU5a@2{VIc#z3<)<Xu*WnZETXulI==xU%Qo}qra{#
z$4;N!oXeHpciz4s7iiG7%EM)83hN(HNn2|1l5@3nM_o($zR)hSvZ-i$7=qY{F3eI^
zZMQ{m-1wts$gZgtV94%)B#HpS943fEp##xIMfCChL?@?=OB~Wo#O;=)yoGUC1Tc{Q
zFda0_an5b)F&2BzD*f$?AZqteCBCOwy50fXr;vT>qESz0Bae%z)Pk4U?gjnYDIAG5
z1u2k5hah`06qIKJ5=b>OSy2c20nyZM-3qCZ&x4N=u#LCe7=wqGF29XEc3%zvL(nnW
zbIl-JlwW?=g4-pOiu0jil-soU^&So(S?aMA*|)Gm?vG2?Nv&zR!atO-SivJsL~I|*
zTeV0vIl+1Ytkq`CFQf`La{EWiCDeFc(@5|ItoU8Y)FkAWz4Knpk+_IRaso)XlUCOh
zAul(jss$E&rReg-u??I@-M9H4qU=U1EDpFns{=i0nH|~xL)%+;Rn@J3-*k5)u-QnL
zq~xY!BO#%nAPoW{NOzZj#3rR%QlurMyHh%36Vk}08=l2|&V8Trd!FYV?-=hG?;jw-
zT60}<&g=SqzaPN~XZ5nIT%g_k?3zheMavsDq({$TvCBs#GNzQyf%==xu%D=%J0+xw
zf}PdnV7WQRfOC`Pihil#d8p76;jNYe46LDeO8lf{^2#UPehuy$90m>c855Sz)QW!=
zj(xsf9~amew=SAKD<_!-k`t%WG=}P1?lVm@L%s@qDS9%^rMyY_CtLhg<o*4hH+5l|
z5?IOgtIM8rQ$a*}W%{BehRr{~zKXHX29ZZUr=<)7MJByoLibJZPkMnNlz<mc%X!PR
z#^qJS-&p`Y=e|b>YW#zDkP$|;j|r1z7o(|!Eu8dH?jP;DOwJ%FeN#)q?*{`ipzo&L
z*IHdt^U89uKsvltWKZP!e_H-zkV7+&CET^U=mn6l;rG)tS3IC4xmVE#!doX%9NS;D
z;%SHTIk;e!?PywE&t%pHyapN|p0_c2<=;nCI3^?hsj0x+6(3~<Fi|KXdA-n|MeD*B
zS={<fLW{JB=p262BCkzqPdE#5xyT77d{QQm<PRL3xKAJ$14^s1O<}8nJkAwCfaa)D
zbLM*2tmES*N+TE-xC2o|Qif7p^&9h*uAv{A&Jvnc;XB)4O3c?DPt+0Pw#m5bPcsl)
zxzld((Np-)2@yRx{7UpMNf8O96wfF43MNTD7E-_%*xlIuO9y&C>b5D5VH4XkP`MGN
z4=SHObMV3wVBC^WpYmylifFN3oG~*FZ9D*W&&%lUOd{-jTyWU@toa&f7jdsL5v!9w
z7Qo_{lb)v4ms2_CigodA*JP=lAKhR9R2@o%P0uGT7dGEc<DJ)svo$h{Gponwlc;<M
zr|IAVc2VOfo?k6sS~=8Q)GSMwgMRnrcAxnQcOP-z9&*F^9>^tQA*4|HkHgYC)%ePJ
zwTi-3)r%;o@YwsTnLueRSw>ZG8uTkWw;J{nJzjB6Qax2cn<KP!0#h}C(E<!o;E(9(
zq)A|JNKEiA<=jMj#$*t8LU!OJ<_w?o))FWjS>{dAuvRtT|76-#^MT`ZSL9QfQKfID
zeJ2@OC{@-^W!T>Sv!#zUg<)Agaoy~m2LD;|1zU=mn-VI=ilvmICbiHhh8IqJ7<4UG
zRw#04)YCH>73$pj^X=OA%6{7Uz_l|U#FNiSnXhQ;7q{P72O_G9NGR5NS<43<(xw*|
zxmyaYE#wjv=u4(~b*thxtk1sdK7Jlh5y$Q&rkp-xa8|Bc75<s$HR}YocyvZ4pvOh+
zoGJe`wuY$=%nqWQ-4C$;9KG1VezX{l$H~(jwIMG2XFRZ2WxJN`mK1uM0?&cL(x$<C
z=~6zrk*%s_3d^G|5;8$9Y%&`jXOn4XI8}o?-88q%-oV+8{wKAI0-_>!i-eIS@te)9
zpD<f}i|=WmP`SVdB>r|r@csSm0qOoigEs9~^V>QW@Pb|bsA*+izOOABAs2^Hs`q4x
z>ihl15Wj=%SWc+bKtcbVi;53$L^IYptrGh3D5dm%`NqQ$`POIT;y!pTP#TAA$))l$
zbH_9WTEwOskEXuNkC0%D_D)xV+I>Ln7`8`eW;<1t0eR&yU?cZf=;<W841D$|?Qogq
zjaKl8{+&bvPt^kRz(w=RG@!|%fM`RaM)wd?kbHkeeG3`du>e|lbnTS_wdQoqQoLi9
zIHFbN6)g+zPA5(mG#gl2=gAN9z*Q%u#<x#wlh!EB(--69ZSLGAl-ev__+73(JtbX^
zGV*NY-xb;dG-B#}W<7#iK=)FCUTW3QqHhr4!piPAx#fyqV|Hl+gdl%0kTi;`3}t3N
zrJ*?)y=$yH*xK~;x#6mO-4HmL?Se1m*}OV2_ujXPedoFXB;o0Nbv!E7z^-CvXJ<V>
zd9g5UOw<u2_-;buLE6DaBV?0JNN-R7OHyXzHEjsKs5=6T{=3BibM*ImEJLFK>Qb{S
zKxi)6D~DNj7bh_3XwBUZ_GE2u9Mjy*e{>F&OzXRbNxro77>)sHb-(&gM+S^Xgasr(
zAEBUlBX?kcTP6d<kqnWZ_!tQ06EOQ$_+z#S!SUg;G#$1H72O+DGb_Dfk-fC2XT8aL
z<>k2FCy&#KFkP*xv%gU;)V7S)lo483!(nl(0}&?R+4MDj<7CJatRQ3>3?0`n%Gs^c
zqT>$Y!7Hl-a;$MdA_?ON+zvBjqDemW=qQmzMLJLYi~M9QB8U*i?}#52E}x1R_0m!D
z7_jV2A-*@W*{`C;6u>n=8;1N&3P^!HxCLU_ij7)*Pd0|r#tiJ)q6=8v5fxxPJOi=d
zm@As2ULL=JD2z9Oz|NS0KgaWti%ymDgAtrJK?#y0vT+<AZNU_J(5<#9^m{@1T3r2)
z#`)!vppnmS@x#%>`}Jimqi+IOAlm)l8SF>5=XQdn?e>5YIB@DnDK&hTWo))!jVT3n
z9XL7~Q_oqEF{{L=3ONK1W_P1*$IphDZ5w_Qf2;DY8N2#y?2*IEVUj<N_43oiuu-SC
zrZ|BuryeTwlGJwmEA=UvxSmu!=7FJx*d!xW5-QBo?gP`+`K;Z<>UeUBWmGbScIne;
zeh#OiUwhvR=Pyj|l-c$O-!>2=7HPaL+dR>9a8_yU7kqoD<PcqdirsQ3XE8B;%lcNT
z%0TtTEcx@w+VF1OWR48|;TIq8N0Rw<wG6^`M<nxiqaW}Yea>J26=ytbRK2mGE9`lE
z*En}yGIEt%9YqlPi)iP`QaOX|>-CivSNYHU94Gx|*9{XOZ%L?0*9l@EU|pxTWagh;
ztL~H+6YO5qvAR%CDh|q49kg}|x}11BVz>gIoM^=#bZQ9}SSo8K4&XD_`)YxP|0a)<
zvdL<(Z9906y>_*zmAYL{3gcI6^F8RXd1xj1J=P;n@hmsV8uLHa^Hr{ehTOZ+SJ&8E
z*@3MzKD&S_GNRCXP`fhjM<5D2zq`!>T%{~f{YzhLu~PphGS;#K(r>i5T?7Y~>f4Ar
zx>bSO^r;duETBsj-kBten6=-d*K4u!h*=gV(r?t>S3ETBI$jGZ1>w356&h^1UT#-w
zZ()NBHr;Xx%TY!@o6vO5+1kK|O2o1r`%=alw)&~5=}aY2PodCY>hj<SUPk5Cckv}Z
zUO<mVdzPMGKeQ3>Tp$qIUSP%+EfPrN_}bgZ3y^S^?i!I4km~aJ?r>^bFa;L{UZmdk
zmIW%sW!f)>r5H;lxdM7muIW@4l?>Y+Tt9plI^fwkH(IhOZGRFb>&*;{lnTT`ts0EL
zoY_7qYh`=BT9a^fJ<mU||6*Z#GWGCBnEbm>)A@p{VKRNgc{JNYAK5?pen>c|R@6Ck
z`|uM{QhN5HhdAY><EMkdakOXis`fU6q%!MgV)dyY0|^V$ql~sX0cHv)`}}}Gtir)%
zhDt(?v%s9o{f(hm?PGJH`G_f-=ZSA<1SsZ5dTVdFi69jlO#MsPb369V*@lQYclQH6
z{C%NJ#F7ZFgM&kk)E%#{5kTnSKix99X(oA13%v?0lPr6?i)Ec&o&gWA2}JF6!{?wq
zs(V@Z@m6&oH?dFd^vzT@o91Qh*D1WL#Q4zBi7&)XYPU9teR>lm+MGe2+Y|XU=HbV_
zuV?~8*Ay2jaoM$Ny<LZS!GTbVIxfNbFQ+*{0)*->?)Q7};$89Anbyd^fAg~=`p{LD
z<9sn9!>XWT^54T%9;{&CP3Z;<;yaie!-QWZQzBYN#XdMmop^fiJOVP87LgES_6j7F
zLZ9p%vWf;I6UMg94hu+3?IB$<G=XBH=(c6Mrxm{sjm?tmr5n>FX^z_<4i71l>#qr&
z3<b|SDJf4p$3$41_kz0H2@l5zW$2wBuD67~c;MZJV0;+*n+v(@AUM%{!cG`Pc*u6d
z0Ney8LhC9>+KL3i8`2d{`ZJ7}v2SWBaU?^W^%=-c;p(JZ6A=0L$&T9^zVhVMf>d8X
z5nD!H{(UDg&r@wF-z99?q;)>go6GZ5=jQA?c<)=t_=@1YPv1=d9ysQFwZE?R=R8%<
z6rLIfpfQHCkoG8Gz@ihyH6>$@9tD<Cek-hvJ6?5Zp%C7W4M&ZW|1xo>26=}Cn0odL
z>*D#otRCa?8@t!0Gra*1I;T!)F7MY|l}kAnEEYQrx&s$nhH^~9fSy`r_ZzPrqts@X
z&+X+pEAuCEfFPXUfcuvj6#YvU{qbe1DBS2qebuMq#^1$Wp;czNNDGr1Lzq7$Ilu-F
zECT-8(f7`$OKnen!s2jG`(#vk`TLs*&#|`UMr(5d^Ess$-b_U-QQxgdFtw_HYPDSX
zM#A|ut(KedjAkmtu?zrlQ_&Kl&Q-Z$-tyUs<|`-3_Jk5Rz0<Dd6x|`zQeM2MB#wiG
zjuyyyMUzh0G73NeakiFJ?u>lLAp`!xBpJ2*r)Nv(GBs@uOe1Y&H!$W0t>KYkc|_kA
z5^lYhx=;-AW-upaRRN^Qj!CC>dp;~KUrALngud#VeAFX_0-U;2O!keL0@NGxPWwqe
zMDQOn8FLo^?eK~$^8S*I=JqD`gXa<3?)Bi6AT<|%ql&x<J)?zlfQR!ZTIQ<H%pdnP
zp3_uIlPH{sL2pkh(=}hA5n5(jMG`Yx3(6dYu}aeL{f5OBs)Mk#dIhAi%<^XH7OH?N
z?cquad?nHiv-j(lX9<dN>huw7Gf$S9m0(TuqKwYTJw++SLQ}(fMk6L?<J2dk`7Sc4
zgwb3b4)m3z`Eh4`%7@Rr?eKBK8L(RTmf|89b%A*N@R`7(8!g(mSdJkF-$KVSsKdzZ
zT(V$$Qm*5DB!JT^@BOvT$AMsJ-*zvupHD><KJ7f-JlQ$i2b%@ifDYv?M9t0HiA7-O
z?CbIUe6XA<s;6P>C>KvYHyZ1tOc5;8sndztz1oPvA?MJDpy0-<6nEK?ulumXzx8r$
z5Q@~>vd*0zWZE0007WyooA0J^<+g6~M^4~=BQ(4T1pU2i_4PTuRgz=6NaTCl8_%41
zdho6ne}AjiZT9|NxZ;9nU?<by4cUhC{gjVK`oek;>C@(t|CR~@G=k^YpjoWW0@S|(
zSvy*!;>GNbQ(Y?L=mN#|W-*zUw=C$J>322edwOTDaBss1-{^0l5(f0HzOFEn(oF0)
zTO`oJE`(#Tbdz?=ixBKUL=!wE796A~x|#4#6s~Lt&+ypRwn7Pfh`Is+PgUgT-BqPV
zX!hi;$A`U6vt0ST*Q?45SRfie_Q+b2!w59EjvY4C@DkfQFK6dT`2hJ#@|#@waHjsw
z;o+#~S(0iJUS~Cy14fbGGU32Zy(!u{kf(KmOSSBcq7Y6S@?=p=om5^C7c1H@WyYbU
zw*Pi;?|DHQ!PD-Pd8v(Z79WZOJ^5XN>t9{aNKv{C>)v|*mw{X)X)s%ygsl}ob@}8_
z9)6-WJ$#|__RWj&A%)-ir_h0u((#BYcXI}b_PkgT)+QT*UEM{IIE!`V<3g+yB@<WG
zPqcm5JkcZ&8-Z#>LWN{0X_Znsdh1=E=N}x`6qR(`<%W_2KH)PoYvJZ^5dK~eee?c}
z3+eAo@~q>YMDc+#2Pe$f_p-b3$4(H+aX*@K<=f;rR5p{xuXKCIY2M82Plo2=OsNc%
zKElYif_ra2$G;}YteD2_L+|TAGC`3pLz%<U<p%eOjJJH?(d{K>mk51>%~sxZ$OyEj
z!O&a3Ea7C5dfhCvED@W}5S)DNO^g$t;u}X*$TMM!wwJngzluTRoNqLyM`>H7^R84M
zTbi<>`uQS@qV|rzEYVF}Wv;M@kfE#!uPL7yB#ibnnv{87JXgd|&iz7zNGjKA31X+V
zqTa3P2e|cKmMgdn#PU?1*)e?76EQ=#)w|7-%DXD5ULTPDp7wGo`c{TeGRtQk2Bz&M
zO$HiWuf6ZDE*r<|GTS?)yb~nXi7S9lIR@A!iI0>%4r9eWmXFhgm@0%bSGx-uOtq<>
zRaT_T%{;X}jCNwzD@AkE6d6GareekkK=`)-F=Fh1Ja_&jLRtZ^EPk&ARy$PyIxdAq
zB-M23r?H%7&r|C^I&XQ2`{H;x@j3WZ0W9H<kOJBxJPW4HAIzUpl6#WQzw{CN_|2FE
z!|tz@zJJy`U>9|hCs+F~*Wj4q!yLDwVU4-WZMEZ<<deloOe3VgX5UGN0D6Ct+o7ij
zVwVkJAQe+#${05n%QPno-6319e-ZVT))-S_6mu<`fF_bqM!$C4iCm;i5N8KBJ&+Ue
z8yD<Acj&HJbSyg#>z01q3ziG@#iBr8p`uNqjEK7xAwc~kgnPq)6uqA<2dQz`zSWms
zkfR;1(xn@J9K%Ztd^Bie`A3ui6+kP2|8!?6J_!aR)Gjw%TE6t@rvvoUJE&i+=6=1E
z(Tn!dck^(@rnyeiMP?%SeE<AjrMh-#wSbF;!cP9Tqh2`4VgUZgaoy-oPZ<nR-+hs<
z`1ycsK*SLpS}uKp1`~Sm<~_Bj=O->ET`lvcM3AfL9VoSA)+xbqc4_O%0e%-f<DC-<
z&t@1SFsRHxT}A8Ttkr_3I9M}`ybLW}tP9ln<95@pZhSUK*g8)@{za${YH^GgSIwJx
zdoGgl6!~oUt4qPBz%QD4CU3W+hfjX(e;gDuWZ`)vQ)$|pupiqju?^_)_2^&yAhcE2
zb+mvNEG9!Y!mr&x)?#IoRvO_Y)7(61Kuk%VLr}rb$?m+GJi{1f)H!~z!XQO9MMkN*
zkW~G7*bgCk`RFUgYxx1e{<k*`nGxFq-MfP3rf$!ar8nY)0_6i4Wx*MkwK=pN0)to4
z!`D`4!L8}h8F(L#Cw#)%UJ!v^$DpInX#_s*`^e{eXS^@*Gu~acbD2?_|B+xz+UM9c
zzN)aWJeNR@-z5}D?@H%D&}MKvRHIw-K0CS-m@>C374er6J*;>uV9x&4t?kg^MX$Qt
zyS2jjfV0I}1s-oh5zq-u1RNNqNC3j6U%rxBnyoiq=9D$3IWoTBK=W*9Ucn?kncXC?
zp>?35_}yKyD!@Pn&(W9X=IiX#b3B&0Aja7nL?S4aX1xV~aLM|kh)|tO5Qg3x?Le4x
z-0NrT3!AFcjor)eq?Ii@`*p0?;@1Nt36N;ah9C7>fm>zk$NjAOv<#%)U5SPN)yj*M
z7GZWbHVgJZN#+o;ADgyY_g6&@AZS9`K+Wz#_u3jC^+v8)2_-RI_{i<8Sw+9_7$m>E
zA*NhwX_UY<<OO$)Q6>N}nPm|3d{&7L!sa7Yka{*P;7^O=8-^@HA8d<^!S7;Ed(y6B
z2dgv=?G}(mto$glMtRe<nxO(pGgBn@0sUKz*EysC7#~q-ONg;hOk@y6uPYyaw_VWr
z;9vH-vcukqA!gfGa|u}d3Hb5v%0+&Pu?0p_^x2re{7d;PrmUo%p#P*~8bXDp&JG;E
z8ns$Xb#VUui(D=Vxc!3P3TBQeMGT@5`V<g2rxHVY1nE@5S6L>!gR~q3QA7O9L}U?d
z<AUJo<nT;*<xbmjjrW10KsAq|@VgcJ6c80@rkJPX_nk$J4}EJrj+PEzFok~Wo|wl(
z<6n<ev+9r#o#&yMpmbq>CS0|OjmEzk8>B3Mou4R@`o$&bV^vfhv<xRu;qGyR`SnHn
z&p1STk0*aGFd?(DPjWc2B8A@gZ7+Xv{9Px>-r#iXJCMrC*d0L+B@lhzAsZO2ppK#2
zV$A(bw<;w)iVl{~6FY1+Ke1fHXjXe)owlJyaCJG~hs1BvPQ<;gA(Ps0ks_GKj#?zL
z17oz>`CdbUEaiJ6ujW_rp%k6>OU~DszUTzL_j<*RZr1^jf*xZcrB2Oz=?<D7<n5hE
zq1M63D1eqP8eo=~AhkT=Uj%EEom*duqxW_|T3l5@0VJZHs*a?+XUr9^VIiI7Ja>)$
zWxgxRD;@Y-){hAo{ja4XD0!w=(;*&l82FFAeAiY0f@jD$bt1)FYb*?cKTMbE0s+<-
ze{}vcqWkMM(?%I4Edp>2h=>d+DM=x1oInn3J<9Akltv$kVsb9pR%R9H6_O7^4%-N8
zw~Y>-JeD^CL7j!KRubXfB32~3){<?2s2pJEmIvZmfuD&TGp+Q-cQI*f_IE1MXR&fm
z&ZE>5&_{PNx3!_1nmOQYG1v?W>E{x<=+^>?@^mwrkAt8%vF&CrtR%y;qpgUG#B~9G
zAKmz>x$!<8<vbdpM8n!a>wUpowdI278c#0-FsdhDznB=i>RgwVNuX*lmCuf7VH{!g
zySr*=brtOz-m*q7d9fp3d^^iU(%T5m!;HxE_8%hY<27LsG3?P!e@5^Q(fE`TrAn(!
zst=7Yo>A}PB#kT}Eam?bEg)`HC$hr~KD<O2#=LxjX-=}&!V?8Q%vc%q82h{*ImF5c
z`2LytzkQ*N5tC-~zo~Yl;m&MUPE+!pseeq<Mf7b6xQu)oNmPnujNQ!YWK&eVX*egp
z7uw){Ct>~((@5q0e)Fi8^_#-N&l0HfcV^-*Qt>F>+1sD<0WuH*(VgXN7d4J+IX>za
zt!HV5ZtD)<s<q?6A#$_tYtMOwHsGpzCuVcgUU=6j{F_QVQ--fTXx&G*&ZVNwV#stc
ze>@Ezn*wr5K1q4s4`YoTIu$bBQBmG;E$I;m4R3VZ)-p$zyqn!%E$-Pw(Y4qkhTiT|
zN3_xuGwSWhsLhFyB1d#z*?oEJE<&nJzN;eVj}1Z|zV43_aNZh~d_4HN;@}XC%<z;Q
zEieR03B*M&=^dkssQ;!M;txW;wfueSmEQJE{y+QtgAf_6^4DHJfWy_?l`oZxeuIJ~
zf^Oj=;fCy5vBE_M$Elzf0%cp?A?=l>5$ni)w6c~bp3R2!%F#K1Yh8S3zlJ;|*dq${
zyK&;G#jcSDG6H@P1|0<w<GNr4VY~t8O0KxO3fiPVUOX(IG78lUlw>O+be%WsL#slM
z*{V#fI_n78s%(r=HAl?m^1EX(smi}a@FFV|-zb0q8L6iu=7?K{s8d1f@zo=-Kovw7
zg(x7nn@hTwFm-hD-2E~+>R{HDUvla9B~Dj`g#Iv4ZF~B~n|DxWvH^{-d2K76<-qTu
z(_?W<zF8;b35_IE+<NL>Y3a`pTVSEv<3dib*+FYX7RxQE>*loYGc=ffv7_c@i9yYI
zSEe#BYWRM>43etdA8Qo-&AoRm&v`PM)tpUm>Qx=i^>ym2^wiF)gwr~^Uz%T24e)QR
zP98B-aht1fp3X-05;Jh!+s>;vu{;4z9ph5(f4`1;*^wd8bNj5Zjt*YHWkShpO{QxE
zu_V8(Hcg8>2=lwkZ7A}A6?}9++>=EUdmL@06sRZnCH4$~vcLPiTU0+2Cl%+efA+RQ
zM2O|>&U)46P4S>$(<T}z=e_pT5NCwNy&8?qhx6sxFTKR~^3O;o>;$gfX)b&+vomZy
z@j8jMFn-bQAfvsT5U+PH=_d5#V?o~CN8<oNKqo=u|I#gt(HJnwJ>;CMYJ5fR;6u^n
zaAOiaZ04{oz4)lRde8{#^LI13hWv2aF_O#rZ`<TQaT5;<3S9coawbeK^fk~|>Wyml
z#i~b2k40s|-MTc*^w)R?M9K%y_)F^BaF$@)q3oKxe-lc7LUdxHk08qqmBy6mzd;8C
ze(tziW?q`#jYN{0Ga(}CackMby#*EcEMe#f*$ON`h!7x!ntL9Uxle>@+qH5iuN^Vy
zGg~RcbtXqAmPvE&02ycAq|{E}2*MmgKmlO|y>Y@kS-{w1_pN?9z}T9<{j)G_Wb?fi
zMiLwu?_`e80^r@=XWWezk99>-#Iw|%d|`_%O<T&-iT?6pM!lcUF`q&{fWA~dV85{T
zCm>G3H?vgz9D8*$Ig{<g(#}|?<cDvH-gO-%d<$uo@WII|WNal549$ZsGMy^PGZvij
zd9&ExU`NZ04;BGd69;~`Vjv4yYWPvB#<JSK<hNg+(Ftk(sCy*)^pQ?3ih|R7SJKeo
zT)<v#W&833nw9Odw!^)%U_w@s`sI0697bfdZy7O-F0Eu!ubXo3?ddvt*PA*UT<(lI
z3~jX-8I%+-SVXx%r8=PM57(Y~1(M-LDM&r`v2JVOyqCPnr^$Y+zAliSvlq<DVUoVL
z8lFs{WFKusEtxOOtP=bB>N*Kq(e>&WXx~bis7(Rl5ukDP@$p|<U;!2=2kXZpj>X4_
zB{yQGe#TU=XOq`y`Y%W${fEsGmV&=kX@C11{w20ve)897RHljaRd)y;2>xg*oDPom
z+HT<55xWi3Kim8R@{N5CKK%(_7Lq5{)rsU-4&e=EB(m3R_DDE)b`X9&PCXdHGK%UM
z9pPJJ5|p+JsV%}#bC1c^z1B0yg%D5jF+>C4SOR8TmVVD#1$rJLvB?y0Ao^U7g<ryz
zSnq~tJEd5{n1f9XL+Q`m*lLLek=(ZtVz~;Y>zfM1kQ?b-GGs8(qlXR6?DpzZ5HJ`a
zS)80Yi^?L#Ec@CVSZ^uL@@Zfnk^|$}JuEy=&zKGT+zoci6AJka8vUY1enIlKZ8(3<
zKW8P90I%u5^7CH72E?Krs9(BOlrB>WS2YfOkK@i(B70@CE=>C@mhsX;+y7SzaDNcs
zZ|tOv@(%Pc@wHebnF=||AKr9E!GoPL6GIT0WU5ACg&$ruoZo=15V5VV=6;t@-%Hx@
zGTTS6E?A^(0fA(>7IZt-XLK}YIhN{8-(4M5v>xru$qjb#0h0g}Vr2dgf547vR7TTM
zz86Xv=%TCJX3Kk2qDK}5Bd7~~zD;;Lkk4iMh>3X-ev1jCPIeue-OXFT#Vc4c%Jgkn
z^Iph+Ml*-R<1K`UC{_EmJBDOoSR!EPS37j(hR>GTR=4p5&9demM2k883k7#}Ss))~
z)PW-VPo|lgj5Pdg=y{gazW>b{a~LNqHvA8X+I-sEHv29qo%G~DcanaDuQe%Y;-0%5
zX3exDW$hePn#vR4Q!*?>q)fg$W7ICyRNk!`mRt?<19X;X?+TOqx{eYE_C?F00!B&g
z7aCloR*rET)K?WlK5gdQ3+*4feX9T>)3URfxc<6ny94)*K{uJGZ~ymY`Ij39y!BVJ
zn40f6ZFQAw2#yRlHARnSJ@Fjp7DcW=h9U`jwe$FI?YC3HMf5}Ic2Cx_qa_oFx&jF6
z8q6iGG<q+>FpZ|eOUTYvT(g-`(4f2`>XEk|{9r5N$6GTz8V5mQuHF??fCnIyeiZv|
z$UERUJ-v%RfY6wy&2qz}=JJtR?IfVQc1ptrrIEFx#$1vsAP^DZ6BjUPB~r-AdNe+b
z&{zWd3p|>vyV1SD1Hk=<9M^e4LCPjm3J6QG2qjO>?-Fb7SA9K?R<W7g)SjnERB3eU
zlLKQ4*SFi5`86AGYy6t!G#G2$?1vNmFR^!<TNpAYA}V$C%a6Ol=_G0)hWSbbk0%T*
z1d=CJ&twAf-fXz(Wp|5#dlp?>g@0G9XKT$V_ebemG+iU2?bGqPfR)=dnDs=;rD&2^
zo1L}!`Pek+o9*?}crr$7ysG-bxZa_4)x;Mhq^X`lmp0prt17^`#qIYQl+&Pr?Wy&3
z{6-iT#o$JH;y%hMRlH6HjaJtvHIt2ijr=FtaL7HaBA@B1I%FI$|5!P<vR6eX)gPMS
z#`PL=qjtxKs&51BSG@brz%*a^xc89G#}D)j7DA`Q%I#aMp5pXnW)=V9t)M}pv)ubU
zVCevdKeRXuQjsTF=efY$x_>i+Fu&h8hrlQGMbm}Dx_jRvK<aqm7uHR@M`TvkH*yFF
z5})ylRP3@Dh<T~i;*t%jC0c)GAyR!!@JCWHUU6uJMR&qe2_>vfL<=)G7#L@K8sao`
zYza|b&l!+P%r6=*P4R^b_Bs*Y);W~_yrHQnd-m3kO%VvSp$yKHaQ-vN{KI2k2xwgg
zw(umvlyz<lJ5PhaDkr<VaSz6tC@Duh`V?}EE<*E=2Kx<u36G0N$Bj|hd=82nt_FUq
zS}^+UM13o=7xbn#;v8~ksYSXr-}FL)L#H;I_BiBR#oDNj^6fxov?sv<*u{OhQIOsW
zJZnWt2CS>Gyqq1A0DmAYqmu(*NoJS#y!_9=_-}dOKa&(=sfV&1fCn;M?I7kRLLgmO
zh@9Q%o-h%d2oxgSA_BPUk!$b9WJFXjclOFhr;(-3Lh*Z{;Ssk0bs?A#?a+SiO3pIL
z-o7#2xd;5(IF~k5CE2*jQwWD8!WSd@I@?S_c}usRkhxcm76a3qph*)EfPV$H8AT&L
z$s?ph9rdYDkaDHAmj|XN4F0^N3J6{~h3y+TFrdTrbnGoqOS1wpBR)nwz27+bzD392
zj)Zz%|3|R&&i)fcw{k8P7Uex)v%Iulo|k)X)&GZiG2!j4DSlBP7?zAJ#8zmNtU84|
zo!xa<+6xi^a&8>{)gSsSkQOL&e%I-x1zfzDOs1{IizUgq-$>KYXC<m<-9d>NkFtE?
zbj3YuK(cm*ty6oRQCmvX)%l70fc#A}6vL-x2-7AQ|JvxfZi9tx-X|Rb9h#o+i{Z7=
zQvXCUl@#e;N)VgJn>A_u!@dV+zUE457{(cWMM-A<NoeWOdvmldbYS>QPGRmuwdj&g
z?}&XB6hY@LgD^hAo4@OCr8yg0d=`Pf1lGd4Y5po>IURGqd@lqe*sHO)GNFOq($p5S
zR~y?y?v>36wr3p3mRgU;j8Os=<JQCR#Hulr7!aFg9<wa<cVL2m$<DyhM`7YWlcHEg
zTmvN|bsL{)qmG;Z+r;DgGy&NB{pFr^AOUxe$53rF7%!3${y)Ke|8tN3S23k}D5jV~
zE@ct8XoP$FyO-Fx><es_@BAfM)cljOS+%pHarrd(?POjuZz6djFxsS~GeT~Y%iHwP
zYh`FB`6PFUY8nV6Oe-qrSLe9*Y|0sL?D>$W(p{Q(Yjk-^F#M75f@RE%wg6m*6a}8w
zU$_n%OqwlT7FUVvP(tS-GcMrFz-%)P69OHa5}^?S+EOGF7U1vr_-W!YR2s<Z11c)_
zo7yp81Sc1@7|Z0VrJU6DLw9fa$MDIhd@V3mi6)JPJfrFz9%7Qtm_g;~=stRplT~DW
zNB@>IIIMZoC(F|^txcIW=g29aAVI-=mp4B27va0>ep~*=ZkjCNRBV3ZgnrG>6D-d;
z1stvDehj?Hs;lu{=AU<W&U?Q5X8%Txav)JGsa$q4v&CB(`2Pwy&!7C<Vllg6c8Ax)
zSV(E!w|pC@EyzuYnAmkIFP=*HoZ4a;W$0*I>nX&R+CcS`$efh6+mut}$HwfG*~w_^
zsq^88!rXr8d35g%xMDPGMEUhrV+CyG_ASkV<A+d>BdoNf)!B5R7e3h7o{b&UI4>4(
z_P<yI-O`$?HV&V2N&~xzmfE2o#<f_q`Qz2YyeV^m*SJYT&~J@E21X2)XC2LlzOgn}
zHk!Jx3i%x5<1g1Rp69KFj#$~6mrw~cP(C=sembC6QmPs#&g!45)1HpL5y5=03<j*X
zn+3cE{HSz6d(3y?&9e@)x5PZ9Ka*Zx4@k}Y-04m`vDOu4tet(tK-!;-`~q8--v;O)
z@@s$6YX1Bbj|sfM7ZaZESGc?daL;QECX4NG2eiRy>>D=TSF;{h-ww<_JW__<SaQ+l
z`Apk05k+I{NY+Ta?SD7?sAEnKL2WnH5Fk{xPb{k!S_H(-f$+5nanZOaB4~AhKvEQ+
zrv?Aj_WQ@3`?p2#n)pFM!*7aSj~&elkty>BL^JYulPNT9w-o^d-bh$UTfptuxpb4t
zsuQ*^Kh^fm*lWuK^$nKkV+;}Ou|ssaE+NT<-8EfqRFB`tsC)wZ)7QDt&rBXC3!=GT
z?aqC?t;ZKvP2XS1tCLAXm&XI4GA~r}Tu}@9NS)Ny?sdM!Gg$-TWuw*3|16PCjl}{f
z&{_Z-q^7;rZuvQ2n4X?*|3=2H73{b%#J*(I?5O~R%o2QH8&(Ihrr2HraMob7b~*Jw
zvp-xo)slG`Cj01KJ4Wzx7rTF6m3iGf9{dc$yD7X<7(vm;V5SpAu*geA;Q-_keKe<L
z%2|y9TfplRfkSW1`TGN;JcWxVfsheqll0(q4kgamu0u}3OsVYa(LX;=F=x7^Zf?#8
zAOO-plg((}n~c+wOD4CTDV%QOGs9Q4EI|p6de1(OMK4$N(t9BQ@#?>xlZ3eVwY_=%
zZ8EkUd~hzh-s7d}us4vhMrn267kX=MgvAcA-&1->nK9USc>@2#DAsh2Mknq*uYQJc
z-{aOLB^!(d$e$G}@6rK{Gx38A)!t^HTj13V%}6%k6IYR61@`?-t5)#|31Ab*Cl3A;
ze=3K=J#s*IVwwFOQ!@{Uqe<XWFztvH0EhU;1gWxzp3w>qjhC(z`TZCw#zvz)?<yk`
zmZ9}L-r<%2`suZ03(scCFVsJ^>&G-Rspss)y<SP>mF$ir4{FM4C4x3D=HPSIjXOEq
zs7jOQ-*@#d7F*;E1TzMF9@6DjY4UqYqn#uFY{{x}Y^9zWE9}{2#Copn{oYYLx`LGO
z|Cg;90G}9?u4dG?akhmt``IY+$#m3qDH6IPjdecv3oj@D@VoI(PiXXu%xF(?Mry-~
z%a8Sg_2zSig&<F^PIBp?<%BiS@OE<5RLl*M${2|t)=wm^O-MxCNJ#Q_p?7_&ve*HH
zK^>KIl%f-dAG3!$){G^>aL2D5q^{g&2-aj~pVStylA^(ebDkl>$T=w<Xp^~6*b`Q#
z`#T@cv%Qp3@uN6i6Ip;+mx*_HxCRAOBUOwmUwDL)CJNNPcF)z?(qxFPT?bcB6ZZi5
zckVYA4pgJ-3!q<sDJ_m!Wum<Gjw7o8gLvnB*^f}$5%bqDoCIRn?4tl^86GQ;TsYom
zX&fviUH>Xhd5Fsc=BrnD!4fDd;c(4wdQY+YfT=*=;2`H-yQaXy7u{lo#&0onp)k5^
zbF?{<A@G7T<Ny5l5m5{e2{YtqH-@X8+)wsSgGk%F?%-tL_y-yZ5JyY@<bg)QP%=Sy
z`#>Wxvd1zG6*?gTVb;=8y6CIIrLpNjsIIg$%m9o8v$p6M-ODf(xU2G7*`v2WsJr0U
zl^`*sHXvS2$a6(OuD6z!l(~1kyEA$>3|9tE{|fj~UH%fvST@GEE0ug0Pu30UumXE~
zfSjH%zJ5S1VxJfL3^W$vHrMPWOef(fsQ>&0P{?{0R&4??l~jOQ@dc;>6SI|Oav6)n
zk{a68f7h}Q|J#SK^CX^I5Y)_1@OKVT1^vUep3xzPK?uk}?{8m#9tz++7EmVdXNy4p
zPAd4%zn6s%{;OYzE7;t1s;K{O9>?Bg5U#(N6N<d{&z!7l4OwGMUnEk2`e-fs_yt++
zQ~fHC0}*|oSe<U@YoNOfF+x!6vABV;FriDJc6oRETXSMm+Xdg*BPLDEZ502*izeP#
zL`Nlfwi3TLggaOTl+{mT<z(>4w>BhN#yk$IS6Up%Ax>UTl0U*4T_-T^PosM1^k9iQ
z2(_bf%IKtSj%NKVRKGc}oBgI9Rr>eMFMtOf`LFOOqXF@=)8U_ho>c+h^~{|ADj{b_
zYjHs%YhQbFk5+X0Pw?X3$Ke*kzoHr<rLPL=e|_4$-!s7Ue%Vb5@>BsQjG+YNQ5H{-
ze_>NVCMU`dLO_YhWbzr6cyO>dD2Vc3C2|;uDJA~$Qo(vE4bHh(wu5<ixHX)p{23X#
z@#PU07Z+%}ZYDkW<0CxSMc_{eLKeX!f}BH{39;e@hqy)1R0(8gd))PlF`&po7p9=~
z;1S<2f%Ct47}v&l4vQR9$w59boYR5u`hW2-82-(}Sn!Bdt!TSx!nmlATAdq_C+sq%
z<?exxzST~~>JWuv^3Fw5tp|fCTn$OD9&i{XU1ji`jp<cvCPFt>HzcLMFekga1KNJ1
zf$)`*=DwzXfX1whZ3j_62u;7+c=gPO%GVDV(UrM#O&uNpq)JQ-KsM!7yB_}+Q`Y}4
zcE&wE+9IHkSI_a(G;o^=vAlLaBP0O$A2%tuZRUh04knSei6!Vl8^XLs{!Te>Gm)Ke
zhrSSP4ZVUdS~1XuUcMHQnnn>X@3!fEUD^eaDMe8mnWg0!Fc<RgLh;C>vNVeBr1Cbl
zG$>AWZl~$4mtYU>rP8lyqOVCO<Tmv%%(s>rLSGU3Heui>A7L1B<M0UOFR0J+Ai?M1
z00)(ImPzSvmW#a@%FQn&EYR~%9g-d*;5*5c2!RvJwRJr}DIpjD=*@lTTU-dv{|<+h
z<|4KSQsg|ohs0Pe7zbkLJ>;_^MI$-aKzhU$QXW`TH5HZ8m<MfpgfvD}W{<H(2M?16
zX-CCw7yzuI0+%}uq+l``n>9Set)6n#{jf<UQnr_h;=Li+UWG}l)f|wrI@<K?qKbY`
zbplQmX-4peQ~Mf>5y;09V$Y?Ab&c>Gv(R|;i~sFFDGnZ9Ff1(eaoYpPWM!TP$YY*a
z<*EJ$Xd8*tDog_4pB~$qKD2*N5@ZodHtTOC0sfAX6#zJn9m7)b{SS=i-=r#ga+D8>
zh@x_Pk;1a;oen2o+7Ogzq&{M+c3h+^B<`LR!FI_Fs*9CC!ejgdYXm{PDQ3?~C@s-?
zPYBh}XeCuBJ=$6mXz1Ov$2Y{<Q!r*Z`56^9Nv{G?XoO7Q2yVNqN97_%{27!!0&Tnl
zFHjUjV?qQcq`N~R^;Mk)7J`EeJi;9#!HI!@azQk7j-DJMjd%@A;zNQlaizM50z=>1
z<pMdW5v@Z{{$7AUQ1`>Tj0~#@j{@$#<1YZBS!Ha}<pTpX=V7(VGICx1hbZ@NCTyEF
zRy}ol5X=g>9kpW}7qW|q*`_+sMrA`3nMe2pxtC3Q!30$k6(z%iFdf&`3F62ffZFid
z7)lR8tJy;pUu}R<bhw_?>6sl;Vy`|vBh7<FE%1;Cg;A4TNE@$-*kHZlaN$Eze%dCy
zyYV!r2-$BP1&e6i<T0k+g;Ul{p6dKZ$%P5FX5+c4xA%BhSZ($HKfOwKg%x{EgMUZi
z#_$1AxV-<4!ljIprgb7GLN%eOgA;%$Q0RQO7=@i@;qTAZUSA-(->=G2AcS#EMGEIY
z-=edD@`G2|<S|=8<c}tkB|PYQ!Z$***~0(M?@VqvJ*ig)$z&SO5kWg9$|}8j%GWnw
z9s}Ve0K!B)B(@WyaX?@512MA?ANa2T8Es#J40w}=Npt_a7DfMM8tnEU;#i@|hu)s2
zGJ`vl&=IzPQSB-^yF$s!lM55JB@qW96PDQ0K>tvH9#2Tvp!r<Sg2K+*#VXN@P?mj<
zuVuPMIrSwwyWITbbo7<$@rH!tqjZVI^4s$L-dm#2<o~clfl>#bdwBn-Pyj4ABj@5U
zg%8=pWc6=ARh}BX8#X71T#FK~M@EE>I9M^j34)7pTk6|h!G*7B2_=#D7GIUf9JP4^
zOwZAFeY9FWSit7XG6zbgsIR<4R<W3A9k~_AHCx9P9-Uie@kpO+7atS0t-7zH<)E(Z
zbsn?DuVDIh$@21SD&jd)bmhWCXQM?f7@!y~m;J=tIOv3wf7RxhUrG$_^%Q<727gyA
zW3-FrSRzQ+M1hKWgX494lAGc{b{TL(m{@`@T;bF^#)5(oFeIwxzJ>Mv7aAL-E$e9#
ztR9v}Id6o^ex7C{{czgFxV`3)G!5f@==TGRIAeK8omv(*?o^iGYdg4RAFAGbW!>fn
zMV%*DBFGQCOJsS%XR-VWj;X(@6D#Kvy`slYP8%bAXFl>w;C6qzZw;#aJxkp~#ra!#
zBWH0=VEfNqUedN!n=MHBWSaW=*@?nn@IU*?U%N#3Cx03e$Os2g+m{{zKYNs9nE&Q2
z*e5##zXJ{bm`VhP2!Vq5O?TafK__$}U?NvM-{W&80~y7qcf#9ilB|Jz{5;d6k5OIz
zD7pdcpdTzevaIho=(^s`G9Jnc9`iVCBDM(>Mw0qP<~@hW9o$RzjYcRsL}~+?<zo``
zp;hNXx<0V?Q8C;Ez3avOksKcIjL!&g+B?5r4DS8?n<71IfN_9V#_PEhhhYii?MPgd
ziNkEBq;GOGXdViqKPE(`{h_I|OnYMb?J9wb+D{ex-0OrbQmE8W_WWQ8Uzo*V*tGJm
zDl5W+wj8HWq!E;KVxHu>mlyHIRor&<q~#Nw&!J)jz|t4CbA&^v9=06BaK!J4eQe^d
z=b%EP>3zW}@5y=$ZT8gBE%C?hC+>ADQ)N2x*XfPU@shMnV+eJdZr};;JstG^`}h9W
zJ{%t5pGFMXDPA(tn2~TfJoeq7h0jU3BM%AtkLV*`@LM8tAEOObj$NuqMjlhuQJG~p
zwG$?Ja4&}P#7pyj`17?aIso5{sH<C9gjYvqG~J_p<MVM_kLz)+bOEp3;vObl8nO}|
znO&x>bsthpKVw)>TW@SMUv1uFAU{#O*O(R!1_6S<3`mKH{P~Cp75wQ<v4H0*tbqk8
z>o&6Z2dGMg>}!{hfDI!5I>pr5MDdYn;A|u6ea*Z5BPAbir}bP)>s$`=b(Fxqo~J6>
zZ)3z`#hG)6Bf38|6G2Ee3n<^+@Jr0(R%Y(){0XYL{hF07R7nI$$H-^8f&tY5cHb8j
z*)EhW6V?~UE^&LM{obq=7J8>xIBFz(6<#c2CKcXbn4<Fcr}fwF2R<aW2ZZGYA#La`
z4_OQNh$ebkG~Yik4jT7gFv;Lv<2Aln;`e7k1V$Ix=x#`&v!T6w7%MFvAbX0K8CqPU
zuC}br^>hoVEh<<dSW`cE%{{LPd{T*BQxATftQ#8D{&9rnBmI&6huY52fE8<+igwuM
zn&_A%{T9OHahTS30-B9bdgmcjI%v)M3usMg0uBfVn~>t)<v{-bUJke(#YJpB|M}g2
z(0TJr4F?xDKQe{?MX_UIzj)qvbFuVj(4tB&Y|j!aiURsW&}M$1Xfc4%;BKy2jc$}g
zeva&L?Q-DhWF^lujGpSvcjp%3v;^|MpRzXaAiLUiEr`+l{fQuPBfD+7!C|9PCF<lV
z*YDv38THCOy<9HXVI%V}6$+mC|9M_w!g)AKw3UAwDaM7?z(u8`4lL=KkYWUr1^kbi
z5GyTOF0|gSd7~v;hU7r&{{G@^Hz>0Wc4snNHz}~}mqEvEVxurgw3fd1!0##7Hn3WS
z)p{(UkYRA9aGM7e+elOhAMc_gEi7mB(pWAZ--hIlDhQu4jLHT|?{o-BCl$gvUxA4@
zL3-RXl2Kn-yZnAZ4$)CT|9?A5u(1P)lkm3QFj6R?#+Dq>tu3=|mHEp;8PcPpks?fx
zBvwwM%Ep{nv7w}j3CF8#1>5`c<-!_^OFJ@0sT;P>r7fbxp8FVzix}G7YiDXRH~cfN
zgK5}+dRs1w1qyc(0FGWFB7=a00L{?%^6fy104tq{y|hv6z77^Bmf?80T0DYiC5e&L
zN9x8FHT${Gh6el;aZb`BjlwxHua3g=V&b1O-d`VnJrS}n&{~X;0MLo$NTXLNlL3+<
z$V3~o6$IioXAtzRDmw0?oDhE%e-Cijrlek&5AC%no|t2M(lG<T)EqKvvUp1Cw4%@j
zdAt};iHyth30i6Ik0$;kv$ynkb?#LIweJ{1*qVcrWMs}L3Vn?1&~b7vA0_TCRp?0N
zcES^K2o*Eohsy^9xV{%@2z*8F=B?}B2TLH|{#@}{3cUTb{M5$Lap3^ZG@t(GC#f>(
zpfxAM`pJ@oWd64!CHSl`x-XsAZlQ!6`wdlqL25ki<NnQDd=23I&dYiUEXdrtCup!O
zBV<ESA9km9@AbsNB&~dPDB*`+1u?sITk`>CCv58hooEP^6pGeY70T_NJj6<%m|dv)
z&oVpT@gJ9?c`>2DDWYi3Jdag9C9sN|N4|v_=f&32wX@4P@(;g#jPA2A3c$suoFV3I
ztDXtT0rHj>hR6+A)lvo5fB$~FMlmz$$*xtvbiewxSR;iaVW}0oM1YgcSxE4+{oDJk
z%91To(*DdkB=T<>8AoK?YQp7)Ur4=b$ABbx>@Ve8Olm())_<Gb3a{ss968;OsL`z&
zmEt89>7`X|92LUTWH^>VV)Y)0ozDGn%z&Hp_9VIIc%J;5r50NCk&9s=?Iq4Qqk?&(
z;|ANl-gma1Ye$ik_{jRu^5(Ngw*Y$-ht->#Ot{#fQRpFyjbsff3UE-expb?>@>P<y
zn$LelIJZXw%0T=LF8xnOiDkK*k{Q>QoCXpe8k<UGC6Iij@bdC7P>3QE?GEzKELG7X
zjm*{GY-xTbKfEMjp3pBqXGTC#B)!hRBohGCWd<|T1&f3SnPPt&Yhe>Jr&fzr;&ywp
zxbyD|Dd*EJ<WP@jkc73K%xXzXOUp~Z{jJyR>5|UsI=wsWi#)<4AGv)kM$VHgr(S7$
zp!?`@>CyFe?p~qB`A=5TmY$c(7E->lZh!QQR76k;wC21M-T>jL=?#jQFeAR6JyJ0z
zsdVuMy;mDUDGgs2;BaHl6D+ArGGes9ag@lw0z?S@>q5HXrek2TgYqs=Dmd@y?hv{0
zYUm@O3mimesFwcN%Ah{-dT_?I^3*!<6UhKR7MJ9;eV6sF?iS@K6e)eKu@%ggz<tV?
zJz|B3wtB2pchl%i1rvs`;k6A9Dyk7=Ka5y2cqiXt2PU|xjk1#HEMP>EY7=4wgu-3U
z_J~e=t!yd*z`<P?RY(!U&#2HY7!>tePgIh%nVU<28=2?1zAM+iZZhypEQA*$K%?~W
zYZAEB$hfk5^!)uRu3Rs-peWzen(d<Y-2<xI;oVnbQQ6XYcJwf0h_}HH`yVUu|9l^P
zM##T>MT=yUrc)P*Q)Zg|!Fn^6f)d!>ZqzGBvIpnv&^eNok5pEyN#%t>0{8Y?@5+!&
zUESTqlWSqOvDWzaFVkd1bb%w_9CM(g!>atzadjNAdxg{H2wB>wb%vNrE(U%F=Nfo)
zC`I6@uD_GwOo4KIv>4uHCQefM#u)i}Bxz&aSh<n&3&JHj(svw_wz9*Xcmkl2G*Sue
z`)@o{GceckyJ4H??AD15xsru5M%z};4Jx&p3TmIDHU5E(B&k5VqR0M`eyzky8cZbJ
zvFvSsX8}~3kE}p%Ek2AD0Q5u6EDYwN;kNLXC7wPCEWCbCb6c#!SAGm#W)vrJ>Bh}}
ze-C9=P8fUdwH768dYdf_ji#NfwS7~f^DUJHS1l8g@|pY1#?V`_A@Wy1V~y_nuYI-t
z_~Tb<(<eZPbI#Lc!W^-Tt4He*mqB6^&2MA~HhJ`mg__55X_#L@gXVA@Z8Idopa0Rf
zllpQ!jH%90&NcVF<?RU5fOfqgxp&LQE5)I+Q-_0%r+P=fSx+>GC5KjV0-*r9qFA@u
z;%XVU$;jh+ziwFNQ|zC}nKv&<MdOJyYouMTs+MUy#bPq&Ptu9mi95$rhYn}#e&zkj
zOB$<3R%45=OPuZG>$^SWp?SA7O=`ohXZa^I9td4bR*ZVoZLkCqca6L{Kd>{4kNz8G
z3jS+pS%Tq!EAg0^8;%4I^p`@Sqjv}5qiC*x_^7Wh<=knxN%=$wVeKiK0a2IekfeqX
zf?vpvGWDYkW3m+#`e&6?5sBbu`XQ7ihFBC~_=t`n+>aR{W@DiQQg<g2lFt7AkPE><
zILDOn2$=rKg`B%#X6UB$`!j_svmr5Gl69j3)fDZ3fEA-jj<m2)2B2YKdYqpVYCF#O
znG=l=R<Bm2^sntAfXV-_rUi6Tc!8%pghS_RaR#Eotv_&EsMTg9ZYs&TX_0s0aQ<{T
zJ2J(_?>o-s+X#yiqZyWcqcS}%il{Uqh#(&ET{a3d$S$;qgZACy$T=DvU%_PAz*22`
zcP?NDAyt_f&zXx54H*wKUp2~$8?SqnS!85VsB~I-B~1(XM;l$Ew9zr3Xs{4m%87JQ
zr@f@j3PLT+K02_6JZs)Uv+7KFIt)JjZhiEJH8dDmyt6D3_ubtqpZ(Gm1Ft}6atS;}
zU7IXiwdKAPKjY3abhtvI-E4ozuE_8jBqX^pvg|if7_vrkuwo4mMYl8Delb8Qpy-s<
zT?ugHTpT3{yOv9q-y<0d>4E8Lh$5|goc*Q7q>hdbn~m^K@K}H+&!JOFwLe!KIz$dp
zp0-g<OmnX4H~ZtCY{KF@bbVv+!|(dXoWkzu4?LFE)!&S>`}nM`rTN^|tn+F8s)%|z
z_gHVo^(c5JMhs+wu-3rIlo(=<_8cJlVV{Gis{LM7l+6Oi=tEC4uH5$-YGD^2`raf6
z*PZ%wmZ<e^rERC>z6s>}z~a7$Td6~Tq)dN#FXtJhCks_ar#jR~Z>C>&mPd#<ZQ%Kw
z%~?`W{BQM`7i}>C9&qFWcJ!BPL&@N=CwOh?8<0}&|9ZAvs*Tchh)1KOHcb>QOpgA}
zs$tZr?c-n*MW#v${v9gJ?8&-pRs&H*irGtH&-O14#7^lRnqv-!rbFv^zqopa4mTuC
zsb5~Ct%<S<;YqExb&Ks0o#ZfrmLU$H8lp%CF1_^-{N=Okz+R4Gcuow$_}`rfwg0D1
zgu~+JuP2XxSCdT#)qOx4@A+QaN+e9ba)gk<`lio6s`N73-hsoQ<<p-mQICW%OWkU3
z;-ywfL+s%FXT{%v4n`x;=kT84bheed<I$N6x|j4&?)5c#mURECJX*s;=~si3#^;X9
zV(F!w1eQEuQ6La=eN>Kma{LLGZ{na<CdY5*^O4Iz=RMdI#Jol(^tr}t^R*&9PV~p(
zr+cKGuIB?!#m*L#UF3(<22Q=%IVOst`7oHv$RQ;0@PN>5Im&~)$r2R8!MUKuZTM$x
z=few`0qtxya*j6&+O0)}unBv41aCZl3@Wt9Ciy&%*l2fu1Z@I+u#Ubp5B7ScrQzvj
z2hVQ9YR`W5SLeGvNt_?<pS{WFO6@Nx*_z#0K1IU=7SmZ@-Zna)!JqsnjZW_`DiMCI
ztUZ<}LlObkZ1MK6Ui$I$e^K_9VO8$yzAz2ah%^kkk!}VpDXFwH(kR`72I)pX8UZOO
zNs+EWH%Li1kd8q&oQFB*-e;|~_qne3{m2)-aP$1{`<FzluX^Uvybm9XMSxO~$*JPo
znHcZsvVsLHHh*9#`1Gm_=C=3e{_cj~3vTMC5`Cv<czu6=fBMUFWxI?{^6r$d0<c0T
z5|VYoDjJt3Z&dQ*45Tdyv`B*r6+dPy#C$4dd%wENcqW%pmgBNjC3(ujB5Yiw@~ctn
z5AGs59N2gMkk`=|a?o}*ZmllJx!rXs4Q7c+e)Ruds+o-}F~ofOMarErUVj(`uOU~m
zI%M>4MXhx5OEK?R#_Q`C9dpFKwAjO<I6slT%9I2Yyp|0h(4k~NEGC<$<`vur_#-mo
zkn#2a51JV4jF{Gz9}S<*e~;K$|Ls*}cb&~5B7NDr>?p01?=sEQpD|>AF>&$sQL+*#
zUrps5ToQb~rqjCfJo0EXVd0MnaN1&_Te4#Esy^$hxZw}Dl4y3>$Nk8;p)f+8!gK!y
z47?0Jv4+Ey&F<&^30fE+LHw5r|9Zd1D|C!tfT~!exXQ3htXAhGmcg#kcp1ZF2<ixF
z`W@n?0b3EWLlDH-D`fUfDJ+vYZ```ejwr|_LWJJjD*D=Y5g~^SFITloxWOukg|w+5
zO7hW0LU#+P!-sb*w9D8~Gqv-e*gI2&6@Q}*36~>gznrvp{UJO#=pMVp_C<5s@F1s5
z>Ftu$8Ur>j8$<9p-amkC{{I88RqPQANfT;4HsG@`U8$^ayN`J%;>s5Zdk?2FxF|kj
zL8JWjs@>W3&PiBHJi(;ZK|;V`!Qdis{GNrxB8j08NCs$q13*-5+wM?2hAL~6SaHeW
zFDDcZ?1D}yjstXVC)3|C6zbr@jT`hjl?M`vi|yBu;h!DsakumP8D@RMV5YPFqB4?&
z8+nFA%9=Ck_fVsF8n2caR}^hKc&afUh;r=V8xVu2B{jaqeTts3&`$X43FgB`--T0S
zey<6jqdoeJJtFz0VIsbDfsC|z^4)Kwf~U~z-5~z2z0u&eQyEV{;A)SJ#=4!KPnHB0
zGDgHEmg}Ph@_V@1dJQ{6_Dha=icA!IZu=WB<&Fjg&cwlOzI;DK>(7plC*OtxOAKL|
z!Q7DSnd9&C{lzD5Bm3K|m2$u$Kp#bf#xHtbkKG!&4Y2Ue4t%Yf-}l@>)C->H&9dy%
z?tixxMq<o;Z1~+J>;n#kJiwX#v>#1k#4>7ji(K|S6lXK2W5FTi4m7pUZ-a>3o&7Xv
zep94x&88Vr4!(OC4L!}vOdG2%XR_p{xNIFsWItD^3xrJAt#gDhxcuqyH@F&+qDN=P
z@r^6myNcd<-=9sv?`et-nyh$i|EkNHYwLp}4tyS&^X}8+DfV>Ci=Bd!!JYu6W}nd3
zvjNAjFvH<evwO5om9Z)oF-IOc9*ESQhkewm_0!4sko%!IofLuEvv#TkKC9a{cKM-i
z%s>h^>g%+i6YXAZ$aA}oy7sAgoD~%h#A_?<eGk#|4_LwyoE7-s-@zF@82>`rthW6H
z^6qV6?O<w>^b@V(uHtZW)>bwzd>~%Of+NhKw;&qc_I<jsS;JC8+>NlHNES&Mh=68M
zI?m3A{|p7gWph#eYUeISEO6ajdBbrNNxkz`a!hIFvQKzT_5O-Tp+?ieE0KirEaso|
zC1MeY(6ngsO(3@cie{4AJd5ls3hyR7Di6a;C3Sx}xE|R&37(x{4>JDh-$3}t)Bgt$
zUi0qAsYa(&s9(&2N!*E^MI)Klp*12ZjZIzdWy-`xrcmX?m-?Z4<D%=K6h2Sm3ddj7
zNx?`}llH0xr7kC>_3wT*;JE4Eu}QK5?O79xBtZ0nt~AWnA{M3(6f+9P#o5GSb<%4$
z*G+G2^2mSNH@SbId+`F2ZXEF2RQb@}uVeoS5xDQG#QTJbxI>`99>0&tB9DJpB#TYR
zrczv!)<PqBb9=x7Qf0xu-NHLT(!S5XaeX_2dAhnx^@e$-mf0(7sT_m8bx{r;J!q;9
z?*ONB@t`W$)j7;3VLoHYJ0CXb*3%6{AArVF7X#+L$x5R{$*-enT)7fC$R9~5wRaKb
z%>i<x&TV^=rQE;Aa|KA}{|p4iP}p}|13q)*G4Tt^tIJCpb4*3hnB+qXkG72s%Se_;
zvj5G#5G*29&|+g}ht#Rr&5f%f2RsM2F0#5AE0Ma|Sz`1JhDJYc)=VsOKf9!Nhm3ud
zc<!L6Lzym55O2UnhhZUEuKp4e=R34pP@yglsecKPz{G;9{w@?#$4d!{k|Xk&E$Ecz
zcQ%kz^k5>rWV_VFb9H@9ex>hS7zbGZn0wV+iEK{8rj8Aa3O-{;f~4lie6@PI)1w~(
zXLcH`$l73y5qi-YW-o|@KkB|1WPpjo`}plC6fZmG=YSaXz03vmTvCSvQr0*@a5$k<
z%CmpPcbf?QLe;_<Ax;SNS(51|=#v4t^G##0>dZfhJ00jAPRJdvJ5bvYmm^@md5U&;
z>Z(7v9%Xv0uR{7mBv3@FQ2k~{JT8wR@W_u&U3#eEt{aiwYc|qZtGzW;2Q9WB1rE-+
zkvwO$IbpO+S;}KG%3z+k@h;p>(%+;~4X#+!IaqBG;T>qw6~ngx(c0Hl@BsfCMHl?v
zQ1nos+HkJn&&&kJ{aBpU)gw}7HvsXu<o;b1?u>#(-TL&IS6IV)JvkAVKlx<>8nmy(
z$A37bdN{7+on6PSe5u=>w7;X)##zrY62KQv2)+NmU;PbC5P<&V$A<LnT4+-GDmVfI
z-3<W`O+nSS;swPi)wK3Te6&7a+52!Xq2OsLbc8@^SM!D0#Co*g9o6Q&QGy30n9k>W
z`V^Ud%=VQn)7IBM-Qd%GJAyJZ(vU<JYIS23|1jmPiN&9l;|uyLWI4A^*^6jYl0|OD
zWM`LqiQ={^*F_(=*w2CAel67-KNkV4`bVC!wnK+g7Rhv@Z6#gY@0Wy?({S74Yd+@X
z(HMMpDPj^2ErqduPAYv}{Ax5yguW@yr5l)P0dG1U$+2SPk5SET+cNIEvjU(G3VK<*
zV4Zc4);@rdMM;aY8n-lg?8DE@>51PyNnqMze0gwtDxVwqVLoT|$fxQ_ihw$zxc{LJ
z{=SL83fz+M6P>CQ8^7Jl(kXp*Nr$X^qU`0?L?Z_bP_f6IT&7iwP@f*_>qZgrmFoHf
zG<2i<Fxiz&+G5Iy;|2Sw?LlVY!9EU}@zwTUkC#wi?3%scP4&UopA?&}P~zJX5yHjS
zAKY|aZKZfG+8$+uOmNPqz$3~;p;?jp{p_u0HhBlK{snv_l?s_+61YH*L^EBy2oxw!
zw}Hk{StW1%>r7)Hfr0}wpMv3GNKFW<7=f~ZSkx)$)A`Gm6oQv#lwG%oane{O7*Hq9
z%WpUpe*VMn+PFt~;80Qt>?6Z0EPu*kt>=4Vcck?gQPHNFEgBr5CsLce`2vgc@z5I9
z^|jbQ8^nNWYmHK5SoL4?OhXF2rF6`}5=E()AGL>(o{C7i_?ENh(s|vI;p69N5vV`1
z77QCCGC5J&GL>&IKu;no*NOiNT{4gR`%HZ?9F@xPvNxCA8hdZ9B^9aCkpJ4MchC1`
z7H6U5J?hMBh1x>3><><%9fORn+Y`0n8N#0#EtTz$I@=q+7L=!PrN3qr`DI?SB@RsP
zQ3cmT3oViF)OESvxt9}+*htbNqmoKg8qpU|)5htt>SUhzn;<6&-op-2X&3rCx54nn
z5L>x4MBJ&(O05rNr&}dZbKvGkXGn}0YrjQno$~#fgQJjk>R12vt2L;_iP^G0MP$}_
zF>UnA3KMP=zBQ8<7}HPcjyyGPl`y+AXO-uxrF*7@<>%=5G2t>jY+$ct>IGETr_M|z
zO$4u{EpeIpXg2y2>G|zvIj%Qz1I#n%q;KI)%?j}#t?w>>Fs^FGfSMI~VF?r<!<($-
zY=VuF|8A;r%UO@bGnUvFjUco)XLZ75T}3&wQ~9ZRZh9TG;Z|vJLtW5tM4@g^UoiQ+
zDo*hktKoNCZO0iS=+Y1G*bTgcw>@G7!3twJ;-zZYqVj49{;3@LGAEm3#Wxp+6z3(+
ztXr2qXLM8bEq~R`SoSNAG9GMA(6Bq2sG8s$0!EU}YG*4oMZ7$1^kO?ga_jPw{lo?_
z`hVfm|EK{1|6+c`>nmM%y277>VNld%gi9oR@NRe<)-6JvRVAW-B%t$%cWvRFzYJyx
zHYBx{2F;E0UUUEgoX*kRPVo2|GA9y>_dGg`a?E>;&_Y_n5VZz^h&`JaPlB2)6y@xI
z)+LBWAQ&TC&5E(P+R~z5M+x;N)i~A1$3t}5L#H^-j~}8;Gd{Zmsy562L=gXfQ9RU-
zm{hZTtk4+YzOVJA$QJWk+FE44!hDwRLS!<&flhlNU>m~jljnlY=2hLY+>-6%WqW7y
z&32!+*_%EhJn5}HpbKdO5c|t>7|E4D_$iInM1;zL@e>BhOqWBuQ@qXJGH{lo*>bI~
zN`B(v8wMu>PT?;H3_Efkp~Cp@Vx6!vJhaJZ$A<cYZ7X~nO(L*4dx0*fVBLcV!#v4i
zF-iaQrm&O1a0DfOQCip5Ylxfx@|iFF3Fz{Ve^s;GXDsK1rULYXlD^<5HbB_+B|Qne
z_XI>UKqUduqu%`%@ZeuuA=D7*vfeKZw6T5pEEg@0A?~z_5KASN`hK~^mJcmhp^~ns
zRJzej=SMTMq-VgdKIoD+2}-;*bYYGVyr6!Woy5FZ)ibQAxvL;lmA8-8CX_Xd693$l
zX(DppK88mJ_#r_|$mXB}7rYMeZNRsjq1}C#xI(}7!(%ZlEiG*yB$0L0r&TU>#b}T9
zHAAuYid)`v>DP<<a@Jkj?zq>QoMQm6;(r2i-x65NZ~wG9;kd^u>IL3-!4dwXm*u}?
zDu3;z35EDhuY=JgRJhd<E|ACWClrQ|cu`RV;YjItfy5-YW>Dc1)NyN<@K<L|-|<X(
zKK~K!kxIZbKd6mMy%QmRj&%ZRqAo~HhW5{|_hkM_Ju#IYe!;_)w2w_wER%JXsm{()
zUCY(cF}5s>*9Ry$poZ}74KkXZ>Pjuy)o8<8+fSPtSta^gRcu~gQ!znkR-n><UhnfK
z{|g8H_<H028O_XP(=N9pjib5`kE2?!KBZz6aosFpn=y&wb`xg_Z20;xRUI;i7>qtt
zR&??tG@%esV9Pn0V1P8+F>lLK3c9Lh8K|Qr8Px}<-*j8RxdYCivRn7{6l?P628I}*
z-ptwJ-y_B|^Pd^KSe`0ZnC03?dNO<Q`?OQf*erB57r*&wt9Vj|J07*bzcW^lLRWOg
z9(U&2RxEDsA1}H(zZF|fvK1*h+HDN{G0Uyvr<z7__KiO81C}dwkmP3x9OV}jFpluk
z?WxzrYS|?KSegi$`erx7=BM(~#-<08xFrtw%uP}_wnzg^pIP$6G0*dot)KwDr?M?A
zEf}GpK!4)A7I953#RJZ73^G?nJbduCE0WKBxp90;0rw>}`HSqDPr-fc!B9S#iF*O8
z?dj38^EKQX&nlCm;wXmh-?#uJ4!C~l4W{EuSiV^pI2lfv1l;uw#f^*Z0<b|11^I$N
z#S7rhpJD&u0Ry@3?Wb3Fz9a$ETeKDb2IWf0&;d|xJepnA=1eaNo)d1mXCU5-ft<5W
zRMVhQ|Gj9u-OEW-iwN0hJ&Z*lGa##4);b3f;zUbru-+wX?uC6N%zR^pEwOcSI}T&W
z0ylKFDE1uD)nIJ3cwMHSoP7Q`KZmdzu?Aj3!gd%3lWYc3b+zqvKkv(feY=p(z0*^O
z#spnWhmUE$>XfBj>+Jele3tpYi_gM|68A*zN)}$alYvBicS!eFYJY7he7Sr(sw}X(
zAW?VoxDNWv%TORGFy(S+l^b7HHKuW-tEuPyE>N9(B>jZAP~y|u(OO)tU*(kjDUtd`
zo-)f^K;X5%>b$`F`%S^2VkB_IsMjBBZFirG8ri^&4DO8s6iS~kqzwKN+u{MnD5ZQR
z!Qh;t>L<A_T~8jWSpN`n`Dl%E=QD$Im`!+7&$Nv!fLO-#wf<!JlK&x>{Rya#F6C9d
zownV?XH+5<4H_g?q!$s%%<7X}VOX|6{o@8V1=WLKo&C?a!H?|4arNJx+79DhzDJvp
zNaZwAjt8yrn`~lkKpyTcO8xJy`#{wm!@2<VHJByBXVeDK*9F4K=pkQ+E##Ivo#`bX
zZuyAVbyN}qUuphuj*jUa*<MKnEw#Qo)mJKwJGHy{gLIJOhZ>*SZLMK&sN!?Gdc9)%
zlX?n!Wsc=Q3ch-V1p8?ckWDUN_X42V90Uzazgv%i{9cR!HfWq5t-oV_X8Ngl?i8yg
zM{TItDX?nEG52r6bjRDe)98!<B^_OuJlel<X80&s5s&f_xaG2l`zJsh*Nza(wM@Bf
zb%1oc+N~`D-^he@5C&#nA;Uv(x)`wQnpOb>2v0QN$hs=?ghl9i3)A}AQr7OLm!~*f
zoIdJr@LVaG{@`?DiH4G30J4dA^*smOGFp|n4pL-`T=Y>e*NxNrIE7rOaGD67E=mx!
zNk5jasQVw_DVbG7#gqE|{htx@{oh9nU31o9e!F)8XIfTtR{@2xJQ${JPRBZf-10&7
z&0-U1sWDj%h>o&gH~5TA#HDS#*wrOt+k2?_fccZ@QgWdN0BWse;1P>Bm4z5Y-p^`~
zj<@_Vp6^dH55BMpR_Ksauvy%K7f{e)oz|=uEznC$bw2{R?Ty>{61@ty{)pNV8vDpk
z34_r=I{*=*cLP(`(<6N(n4B0CROI~Rivbj8v1u$*NL|vak!Qgj@@@0WeXSt*EJ20+
zWJSl2yd;zI#i4&OI4pOnGQ6>>z(gWIlF1D4pQmAfT9h6`WoOEWf)IkDu+D>h+gvx*
z?hqb;K|Jqn&NURDe!tXI!-Vt}SdN166iEP<i+0H~0Wpga84Y{<1<|PI@17^*f)1o>
zKSIg)thO-e(>RSLKeD)Nj;aD>d<+2r0UpJ`1SSIw(8V|Vsxv|j!#RooUm$R+H(>IG
zZRTJu?FRt{C{~g}Vg8tS5=Mf1P*^XC!w0b_Xd>+FZh3F|v(3-9Yl1xvL%h*(<qOa&
zw|hf5?@MgmvOH|>!hBcYA+Ysvr}s2z)TmjB8rPU~`*;a>Aiuj>XWn4_#$~&S_*b}w
z2Ez;V*L9mlusP#yVssmZ+Jr!3703+eYGZh~f<%zuO6S9I#Li7~x}}1*s12^z*I(&z
zMNvG4qp!>!kJ6!A)JrGq*-)NyUAw)o<X%DG^_O!2ol5`ikeNrv>Ou3O2lwTb(YCUp
z^-zYl?1fWq(c_I3YteJX2Yn4Pq(r9@G^U7mmJlfX8BT9d7z|_#IQ>Hu#QtwYf!uj@
z=-vJ9b2j_wuZ#UOo+!qxFT@0^w8yHCcxbJH=?$E;uo!rc(aJncIr}5&(YN}gH^pP7
zmBiuk04h^X9IUYPKIaqU==6y28hJgMrKjCTt_&_svG2TjYiaP4`MQjdb?Qrg-&7yD
z{=>ZwxzTdX*jDZ2>SBHhXLNm+lnM}RHP|&a7T@i@SZ}WzH4@L}#Gn|0R?<VxS%S?2
zwDLO1GAwL;gvQWwjZ)4jB}>{8MHXAQCLhR_*mV>8;47G7+r0_`eOPkcqeR{fNL)2r
z76dJY>}dk!N(BpH-jvX9FLWc36?_9xG-}M{GQ)eatN~YMoy#BPgTC^SM0#}9Y|)Q0
zGBR7`Rc)W!AMO|Xt<~YHK|raZCV<kgXqRbCf3ti!nECVi0fi5!aHt=uv(Y2L1xgum
zAxdpmQKWu1n4p;q&wO((1C!m7TmEu&cAr!HnQ#l{je3Vqe35DnCWN%oYlH43KII1s
zAeT+F5vsXPxQNebNj`HZl8k!y2b?n=@h0?hmx}`<pUrH~y%B<I085lp+>w!GI&|Z=
zFtmrYXaBAK5|jASpAGDv7oW<1A&deS#OuX|cSaFkRi6Lbo`$dcC>;pGducr#h6yAZ
zq(3Ga<l&jMAzUKh*9||HqU1a!2;tcv?7;0z7eNz@33><=S;eE?s&>Y`G<|P~pF;@4
zpA+G_SVL|@8pH9831yB<tQ~j~y&1-hkQFTyEXTuw@khcD0}v?@;AKCdn@jL}e<~8M
zZ%tA4q^3GQEY`IE&n$C3E;nOc8&HBJ(i`y0gKonAmeMu;zoc}=W$WS*B&@IWBypE8
zFbkgtRA{Z^-pBOX$o<;r{}^62>(%RY?K>pe>~j@7!KKk8{E^e7cdN=kbY<g`*n%dK
z71-Mj^)Ntjj*xlJ%&B?zH&5PTE4(T@qtLpnU+~#ls}QUE9TF6`g#Bg%>0ZnJvbY%!
zX)5Xh9Y4+}Qq6ep0@0Go^6u<1grd>rjJbPT$tIm3wMw4Rr|qSE!Qm5>_@6{NTy68S
zdMC%Z@>3tKUdKqV>%0Cb!j%mur1q^Z0z@zlU$kr|C&neux9EjUTbvx)nIgeDGv0fs
zZB=y#%Ja9!Q>V1Q_QKsC+STBzss%~tagN@9KI&o+k>~-}uc--`#{X<aSKtZ444<9`
zA{w+o_@$J5r!(*Q+xC9ISVA(xe7rfP4(x%Sfik__BE3P~VuSq(Z70)9>p$RyALgl`
z?%dBaI}=!kZUWaDz%?S=Rdlu`5|t8K)f*8!nwv*Ncr%f#u=YT;gLsDCX1Uv{O6KP}
zb)BnFqu+R-2IuA?z95F40##JKG^eX~gNu&s?>wSkdG_r-AAIuE*-5IgS?jxlzc?(P
zP`C6jy$IJDfM#X&dj9=RSbh7~{Bf;wmPPBs|0}g%*hAo}2qL^QEm-)5@D?dzo*IHU
z+AVrJXO)94+BYtffX=hDnYvFJ5_gq|SL5)yj5u!1zwY_d9yx9it{9j%&m}`tm?=<F
z<%;db{id313%`dx?q{f%Ra=MEC>|A!p5&PUR5*amGMrm)d@bI2T_HLe_I(T`URwsP
zvE^0$&#A!lzX5%KT(&^xS6>RNtImhe;KQY;^N0x9oy(6DKapceS-e*D+K<FinBEz@
zLziCPD}}l3ZnzKbWQH7*U_4l!1KQ+;Z};k%mxX`5$h!cHaf|8Yx`CpcN+%L#*ses<
zcrfuI9$m=s_4#JhtIEw>dM&uA$ajc1Rn`@{CshNhmSvE<WPJWQ>yh&tKW-gu$;v$m
zsSD&Pd#Z+xJB6){0ap#2fplh)R&Ybr1o{B1XVU$d2UjTMCGnTr`Z9704RFwE5G?%z
zORzk^?({8G<%q}w&v7vz?s^>u31bNu*eq2H+qrH(oY*)1va35v(WYT+H=9V__7hqi
zCJvGE{mJ>|MBxHZXr1=IseAT^-8l95BTKOhL$TP-O03y)WB%I_<yW*`=@*90Znl4M
zqt6XeG6df#$5X?~^X8tEf%D1~X3Df}Z!eGPoMC-)j1eK8K=+TUD3e#mtji^~aydpA
zho>XTWCi;sOMxh7KeNJR02KUQ9Ugf1eZf(qNNCxT#E0@cL2Z9QN_sPH%0liGe#WKJ
zb`|}~<0a2sMtxOVkd!)|7hL}6;iu2pj%G5GDhrT>a_xVj=U@Ol+2(w?wSKNwR)y3G
zA9T64r>${OjpG%^34_CH6Vv`fWgLeOXk`IhA?r0soJ|nmobAB?pIGxG3#WT_z#@Wh
z-v|1E9q=JGmf0{fi&3QC!p5Z;A%R%~IYyL#sPnlwO@H{11gJsp33I%5!T?!jr;G?*
zqf@8(h{H1uVi-1Md}p6W-K=AN%I8q%xQoQPnUmB;N30Asgd5E)0F44h+TGi6LJHx@
zF)$MTEXU@>LIc*n{9A)a0w5Zqx5=deNVPY5W0kmm&y*VvS3G{1zukBtou}3IN{icC
zdN6e&tqT<XN(1iR?9Ut#XCk52i5E<0iNhNSTwG0DfT`qB32N+L&+6@M9W$f4@O_`F
z+2zPEG~I2RZ_AFRbf&79K3S~MadN|*eL6%1qD25~ljJA>BkBK~8^H5ynut&mX2d^`
zGsMCZBMkC&?KQUr9px##tRiDrfNyGMnSo-3A`B#nQIxX%waQN&h`yx1?;RX}kbf|j
zJ&te)0oZH!yBSCD$BG&5Z`>3=@WBqN@HsW$WM0;H9p7zb(%5k$qdd6r=#bNPrccw%
z?<VL(4ieC2Oh785Q&d@R2R7RQ<K(EZlI7Uaw`*BGKNin(iV{~UBCFj!8NCW$ro~x}
zX4%#*0|L68@%)fjRndQkWpjf5l9IDdkCtNreat(&CxlwM7Zbekfl7p{jOj{5<Ic3`
z1{=>=v|Vvc(H#*?cBHM>gg4l^b%Xp8&_y@Gaiw$J=MqE0U<q8aU7a;JFDN<D5Rz+S
zg#h=<M>}Nvz#wiLN-PRD45&pn+<J99+-b-kNQrO_Zb1|?Hw}sVzKWdv=F*(fPYVtQ
zqLcyUMj-s{KMR==u#oM-)ic?it8NQwbFKb9i=OtBzvRL-YGnH)LW|#@EP#jOa4%x1
zxO@3>T_n?L>R5b;(A+M5=+lf4-^@G1WaU(LWf7OF9p0t1QInz&-?;HX$@^CdEMS#!
zaUwRWd;=Wgyw77qg&g{RTb&`{K(>e6LaV#vUA>EaovUtzeo)!8R}SX05e2fA$*Jr&
zoo>flWD8ApVpR8=UMWYcuZs$=9j8C@61_}n?^UPLOw0;Ka-2-OW40NYjD6;GWf0t>
zTdtk>&2sFi<$F*IWQ}cq-aqh1^SjGg_E5eB3%VOm>Mtk7T=PwR%%G|?5<A^*10>#?
zCI*cVtY5nAPS%l*;7w<pp})KeKqJ)K<H-5lMCX+n@8_t7-}{X+1>ZHFY5@e{x<wf#
z61vtYMvKoUk9UiCY@w^JIT(-Go#Ysg<0`JD0nJ(GFiCplD{}vE)vm5pMCu+ol>hxD
z21;~#6u_lt0A&@vg35`4_4oCXP>@kwQXHpYr%~qYt77tx`{RbXL_By)tuC=1{&##c
zWbgbO$;ct`sqH_<_MS>B)T08228A!60b!+~>F6WYM*m0w9IS6)-<;NDGWe|HVxO0(
zVY(N}54FKcwa$D5g`I{8<@x|EEYZl#xA?O3RZ>{6Wx2*;P%?|3JT-2uZLWm3Oq#ZK
zLM&Fmmpcl|_6J^Fc$3#Hg6iJvG?Rjg=qJ}p&Gt_eb?mbX7xkOwPnjfYUgSBgb!k<b
zN;mtQvF)$4OHSIjqT&e`DQD&wZ-WB?Jm(d(M6X<29Mdl^{*C@gx5ne`4HLMuANvye
zGlzaayXo1Ce<KhsgYVxHJAVko$he(Ml8H9m<B9(8to4q_z)e^A^zFuW@q<ZwA)l|{
zd`!XXCKsx*<+2|Qi^(qlF)Vqf33g<-)Y@$u|NFp?dEo^ifY49c^I*YWhc!Wn-A}y#
z`Vcx}-#Pt##V{NzOSGBMa%`svdL3)@D0_9@+T@@wt9CnBnrHfTdv#*F+~O@LxcwjZ
z6WU+*6T$OKBvQw?tCvD>g!G8cEYv!1rPc@*0yRQc#iuMpibWCh>s7+)+*j{}arW@U
zAgfrNY4iPvYDf-{PPCe_vHJ~(&X~>pap**o2&EuY2Ku=gpMxBU3UIuw2_JOhVOa<C
zr2CCqBNBvcN)7o_MmwX8fN_7cazUM<9j9)GQlI}|6EqPlB_$A&{V)3m6Ofg?)2=QW
za1REAkVEyEt|#;2L91)!Y^o`zT&0QNFJ&qyOU*kQCCw|x5Z=r2G^#ebB}ZZjF-pcb
zS{ZmX9c(i<`GeY?DvhXc(vKZP?WzZydT*)2ey+*}a`$o#XuJWr$;RTQh|elR>>ePs
za5$CR`emFfh8>d4`;f+1=g>(mR$iaU=FUcb*Xq-s(0vrL&%Ber79n9WH9&8j1(LL#
z#W<7tbtL=bW<aXi`><lnVR$M{{oA&}jB}_N<hQh%_rp|!rm8KU;4W0I&h7}Un|Q^G
zJ~8>(-%b{_U|yLFz<R|pnabsDIsBs0@t4MWIYovPaVJkC1ptc5OFd$u_><Rf8FFvE
zSy~-+l#?f-Wu^&OJ-N`wv$1rylx*~-J4Zp<hm4TptL*~)u7l;`a$mgd0)s5^SoUMf
zu{VKIv;FCeVqp?Z#foGKThnb+8F@s>XN7<e7c-?&@jDiAvD?nt4S+4Uo$Wml4R~u3
z<J$8UbaaG`0&KGz+96p%J5hLt==QErTBW7Gbg}C1wGt5mwkLY_4m8wD%jEuW=p}ty
zXO$`?@?KxApz+%qs;`AU-FpxMwlXjFr!&H~PJDy8(Cd4@nJ*TnBd4m7q5+1?6#O>$
zYw47!<Kr3tVxb+}s^bl^{cn5UUv$e;G(;f=uPj}$-YVY5R0DL{Ra&X=xyDuOKy(DL
z+u^hwa*qv6Zhd5bj&UBmzPws4mkf8~XkL5C)A{ht-gJGyc(6aQqX^RN{OoQ1J`qb@
z-7DW_4N8G9frKD6>`w}HX1Ae@Fw+ngg8pV9TEaio_X6#O=PswiyeKVHq&gw^=dvRk
z;2p$cJKBF&DAfWsBzB!bK!-N<kzKb0=%#VIX`mE)I!|?u60ER4>Av+#^l;}n$5KP!
z{iWDW?K=IU{i_D`>lLAtdG^{5G;j}^SQFkj&R<Q1?Jv%Ub`*M#pB3C~p1)2IEn{ST
zX{=rIJNikk=3vTfBMuF}A=pzNwr<ifMq{+pP9f7OipQ8Bp2b>k;+MGK>9?pzcFi3#
zngy;Z_ge4Yx1W=)CcU2bDi|9KzY~4cOETz^7VNm9z1uG1eX{u_hUvJ+kxzG=+V!6Q
zHm^`EIiv5EJMdeH-7~;`RcX{(36eoiF48a-#}(ry`xWpM$k@O%lKo8q0ILmXsS~W{
zD2ZFQ7Gk51j&FiD>pd&_7=T+#Ur`wZ{LP$%Pyn&WTvfeIz0YZvm{I-J+AlHwAHc_)
zq;U)2EYBMA4F|ydh0t;zTJ;>DqUOh)OupHzd)e+!l6lN8>bYuUysH<kI?@40zk8QH
z>inx*O}eA|afMVsMqLqc&yV5FdCX!MyX|C^V#I@E!)XAsu~QIU$2xY=j&Mu}bzrJR
zCv710b+xP)=+eb_t0}!Bh?obAek&|wu)*Yn4aq-24Ok9;Wx<rrUm2`t40ut4=Wr<y
zR&fgOsB7V0%q&bsBU4?EH-I9fW0?fxwC{LgC?c!z#7NYaGG6b8gV<x6Iigl;USa&3
z=gOK8tO4@8gs%Y(5IGV~!#toodNNzg)9{CqP__{U=)I3cRujj|I3)hMKN{IQ!VKvW
zISbicAm(E(-|VK`P!)mm;<3ocXTmbD?%BKTBl_L60@Ub@;<5G}+XV3aymv&O-%-d^
zYO+kC@|RmdHR{OVLIu1sWhy@=@+;?1#gp^c*n4%D_2fo<0L9(!BCOXdiPU6wBCi9A
zB?I0#p5DY?r2`~%$qkL8+q;RhgHjXyTDMC$Rhjk!ss&_K!D6fH)63>uPCo;3p?xgB
zB`j>32AXA#qgWwA@P_&ub2Tby^zFAp&G6RlAa<v~XSH6VgBUzk-P?ql7~0_Br@3CP
zsB!Eh$TmX@*&EXYr`d1LC+Y)tKZv&-$<r~j{_Ubi(wgice{|7X6b37N%}+*n9358_
zfQ`AeEZ599U3N3S*})*fdR1P~x=enW41%Tt;(7H<Yc;2#UjXkU{(dO%z59Src|G~^
zwD;n6M@^`>F=N}0(R3bq)l9(=#go-eWZ-0~pCFO@_YpezYlQs5Rz=*!HAo0~qyX1D
z0x(3VfumUnMdX5j!O;NYwgb=99BtQTMC5zb@WSZx#YFTg+VSsWg6rlc=WA!djrjI1
z=t)j4TQvk6q0B;D#3JL3)ZtuQPem0;5MnkrV>`oz;&3|S{e-z7v`fdg5n)`{U)i;>
zAZ8Z$i6mFRTMC@VG`A$7f1@*=Q}aML78*mmS@(y;#zrAvTGVupF>rA`_T_&$c>pfH
zHC5;1q~))Zp&2^$Ip1#=&*e@R<5sf3-2wDQ=UMYxL!}|7R#Jgm7o8Z*5ZVqaSLA0(
zpZocedXk~)C9q8K?b7RyS$sVzs|eo<-<13BWVtV1*h_NWCQW|neuMwHJgrjm@Ztmc
zYrNZU%tV~;C+_HP)W=Kw8x&IC4s6I0`r}-fG(!09u75%3?~CP3G&Aa0AoA-aGA}}y
z>PzLILO20h_&n4zPglJ7It%vgsmV`zw9JD0sI=?XWy6dd7$uIZg8Opz(YJQZBzx_y
zFZ-Z&-$;c|9?dH~2prBz_&@~HZo4I5Z^8FsEPXtu@DYLE<g*)J>QuB43e!azqb%Ab
zQJ)A|eUl>M>CdLoCR(Iu`<x;Ah^%!vpE-UhIqx|N9!Xms*lQ=v*e6ornk6es!gBE0
z^@;i-W(cT7T)+8Wc`>D@h(h#UEuY_a$+?Sbk+i+t<T~oaLD)nH-n@^mBUYOXq=t+K
z%XX~eHykyQ^9p0`4H+W!)KLE(_8T6^LU$tImi+&dwh?y=SK#QIFvq?()Io(ggy3-o
z4tVs?Fr)ZapJa?fNeRI^N`TMs!+%E>n9BgYOZ31$2)p;{Pg?7-oEe7rAD)NJ9<}cU
zeSvdMf(){C5c1=6o%MboHoH(UCz!Wr?1iVyL65LRew>X`rs(q`wNDuuGjv~`=C(rJ
zaIJ6Zt9}I;bDn%t6MKK8TX!+M?4=rX<9WE&O1jeSrJ5SiX>{Y3H#jr2OQudOsbF$?
zJ80#h1t;Y{Af#B_%k?k#7;tN85pBP@OV-WenyMnYG(o^tvW)mNLf@zEA?rQLugSuA
zYT_7yvS`S{5mqK9K?Ip#sn8WkzeT7d1YbEj)35ACcor4<L#j)ryW}<DqTd(g>?o!&
zDoe^KBuUq>#LGVVTzv2X5?U0N^b2&+BfQ5rif}^dk6JY{2!|^)z->z8@cYsG1PTnK
z4_43`@-LHceD_HpQElI4^k$@FBWgDtkaR|6^%N;vGV&lsD#Ty<x_AD^7vS~Z%8f9X
zocmf3P0l`SUQ%+ri!WdUDUcdLnwq3<&ZN}9)WRRx4UQsAVz5ER-H0HPPmkvolj%y2
z9tvB9c3rLE{YUHwJ`g)HD;Q%I)S?+v%LN4p)9TtvEo7Fj1TY#)1g&LoV?VO=(2exa
z)i_=1rgqtRw;}%fDK?`xn~4OkbLxK#yU<WNFH>Eh>+UB*yviru34oHbeeA$_rUV*D
zX4J*IGRe;<vU{ad1yq4_7cj@4YEqF+Sx$2WM*rk`wL<1c*+yDMR4zYK)+*OV<`)~@
z4*(Rlu>4ql$05UT9tRE}q1vE*`opo+&`Z|WtGRmRV9060dz6*DAtA1*s@;HzLOc*(
zqcfw|`si;3aq^4tQbDR^#h0GXi#Uw+OJ-hHY+VHt#nk5+bk~dh+)B0;ADJTeDTs>Q
zD%!N$X${FkLS$XP{0KXp+M#A;cm{0>8_H`$Pt+B4G_vzv|4iVvyIcX6Mqhx&`Xrd>
zNbXCT>I4*1=N{*mebD%>NF;u;`aNW8dge8*4#E4uwd4>9KN#P;IS&VlyGl0yz`4l=
z9kH;*6BO*g`S13dHlrINZtK6-;HG%9!!tY@^GHOT!^b0E2~sAY2IuMfCGd&HA*aD)
zE)qY%VIO(XKH~f4>5a>0=VRMx+1J>I@&m6*8fh`R&2m-q@;WHivj?Kvvc;2rZH0JV
z&r#+k%?Q;>&@t-s`tO>%8`XP-R5qAZRtD`*8!aEaTMvG9_h<059aQnoVGFYChn+3v
zo_HO*qt>fTGzkkorz+q<ZmU1qKf3bOsyucT6x1TwhF?P$fZd;gAXwo|Y|*3d=$JY3
zF=vGz3Ha>&=0cfoqk&Vu=um<814NqmO38C?>mj9qT))26=LL4qigFU0kqZ-zh_aov
zQ92Yn&o;4dg8!jOqB+9<J8Z_+->Vx1r*0L2`zL})ldQs!#n;+w5oDEFDJ%*sH#qL_
zef3co&w5`sV@?{3Cru>B{Zf`2S2|j6eU+!EPeoX1xTU)l9~!-hWByFEuUhZqvuQX^
z^ot=0hfT<aYFFBwsdW+>l-p)WmvaG~2Y)!8G_b42hIX%eS2|PL$a^S-wU&*W8()GB
z6a~1zmtcL244uegrxzE+1oZL=Ku-%`42%MdXX+P+E1vNq%w23QR-<yV2H5$hwv!4X
zne@Tv5wzka521qwgYx9QYuBjcVCw`~^Dg_bcKPRabus*@=x8HXS!(E68<lisbxe+O
zNrCGF8R5s)V^wDzi$QM3J4zLq4~E~p!&3JQ9RxKKm*low<r)I_gxf*{DX(7=k_<Bt
z;v$&kkDx4~uXdDsCQCeGd>+x`!G*l%7d&pa$IcdewL-p{pKdMP9u5h`Y$lJD`qEp2
zTVw6S3^I}5qmLI)Ij}R&-9YQ`mQJp(?Jp*;lwbFq>O37wj0W)I)=WX$1GaC2UsY?d
zDXKxUc1;I|67`w>`jWAIbriesvk*2;8dSsp^)!<-RsQ~$hiwxfAjz&9{_?t=rIejk
z<JX=VSHba0-zgyR%?(@S``zvPFM<7iwz+`bIEh)kQW3iwFzlzDi!@!Z#8O>m&qk+l
zs`x4R7oM5A_h7_X^9=P5+zpIL77Kpbe6Zf6CBNv!J3UZ%USid1=L0!hLAUKnppJF=
zdiF!4fW@*bB_sU=xI~pdJ$1B<@F$miO{m#-e=72)f4}^lBt#&jp>n0v{TA^E=+Tbw
zet)_sv{M;h0^mt5p8);U$Wj}b3Qc*B)@8&;U(drSxG^CMe8)Z0m((cdvL_Sh{02_!
zJFqRT(d!eKJ3e8gBSd8*A{O|<Ct-qWq*Wo=k&i(<+dwXniWXzq;{t$;WEC9lEpE%}
z^}5gDJB#*$+cUR&25}$KlMAU0aOF4v*<wyxs{O~mK*!xb``x&R`<gUO&Pl;rN}%lv
zF%Ojs!DRV!uW&Nc98hX1X$bEm*^LOw3zGyfI8ol_$#MfZr0Ka9Jh+k=n^MstJbj{7
z+Mntq%@aa?V*19QM8fDIx_hyJrV%(35J*%;YeA^T5J4Q#8y1!co3O?w#lJ*Wx#ZCg
zaV_XYHKQTb7!4CPbl9Y{3ebWhbXvsXgP%PGaRBjBY(RPl6BB<Y@{gfRp(-Ztx1(8~
z)N;V3?A{EhN$6!fF?M_zuf)Z=$@EsuZpq}n1OnlW*iyWK!gOA4j;B1@99o+3u(8yF
zH(TRZAt_=n-}_X^$Qq3xo->N2jo((B<dFxn(Y|h06S$V{ly-wJA&PojrAv@AhO4Ww
zS+L4_*KW)(1icjEBx`H;$bG*q7^BHj!gSnuVdMY?w(r^?>t{+m=HL#6vnQq+6_;3{
z7Iu^bVEG2JqVhNz);=Ms`NBv_5uz^wx#gA#Ox))aDR{BUR3bBNZF}&K##nvLcXI-h
zI}E=lJc^^j*lky+b^O#b;~i#wAP<mVwse+fqvQqrX7Sxyzm%x3P53%tTSW!r$W?VZ
zue$zFq-{^eMr&0{dHdlJgzhkl2Q=s?^CWqt2h%HlxtCC@S}^>CgoaPucPp&X?g1(<
zBQn9sP-)dB!o^Uep$U<iEK+|-XpLRWZ~J=p{_Y3usafVd+Q4dUKD9>aaqk&v1TA&L
zPSpSb_q2C~X#GvzchC>Fd$zM1)6m(Y%|9OmO5UY4x;^X^*@t3MRPmSP-)yRzd=8?H
z{bF3^)9?Bx%9bK7rgMUaV}nPFkyY~x#-G7tSv;;!jZwsECi2Ofv0t!_b&yDpX}M5{
zWTkKiR^L%k%s4*v6Rcw|l0tpT2psK2fkQukQ^gxv1*2c%FGc%CtvkzUD}R#kmdH0(
ze%l4a*$5xtVemn5vtJ!yv8>-CpH#<!a=rMZyR)8!rY|-I+X7L%bzTR9ichG4gKTUY
zJNq#NshiA3Ij)HYcuU+K_n+XLzbq|c^!Q|*-Pu1We=PuqI+YCXf9GHQg8(x86~ERA
zICWML5myk#uAGCAO_?lBu>~>uCZHs%lm%Mv!f{^UV7P6FI{^~^qy8t<qhZb%d;K^d
zB!lxHs-q<ZEHkm2Hx)IP(J0OYH<1%it38q@6g}9O5SnUdO2JU9Z~@-`5or}P`uz6$
z*sdniJK5ZKwkgKd4)fmv&`@67S0e_|9mFF1Z2~)dZ4Ru!-GB<tCP@Jh@NNMCy&3BS
zPk)QF$3+bp$^1bDvZ+c0c_5umNb^Cc>mOKuec7o5TI*gcCrwxBdoc#(pM=k8l}TA9
za4iOuT2Pnsc4fEU<f;<IlSH>G9{y}h;*?_BKHIehSEy#H+@0K2-J8fXc|)L2QaIg3
z-$@NA6o7qhBV?$<KZr?8RhHeF4BvG10Zvvp!f)uUM23`MV4e>e1}7i@P2iMj3#36)
z$jrhMt-85Hb{XCrK~H21ms@mF8Hfm5!~28IPp#N{`xS4=d3eAxz~8>-Tau`y-F4~i
zvsR%54pG0U#GL!+2dZ8Zr<Gg3lkFEOqf(n@bFx0S&klvU{+!<}PFwppcAElTKZ1{<
zZJHkyJ+c>yRNFp5Yz@PYG7u2<7SwDA*FY{`nd|ORkH@Q(bg<zXZLYQIv6k_gGtXTl
z%HIPi?t+!eYLX8;#_<-1`{V}fjvX2$>0xO8^W8L0v%xMOKKKhRe<%z}Up~Z{>cPk<
z1P;3=(63T)Ag~hsjmosoHTL!vg}WOT+k2XWS~b~{7xQ$d2jS08FGup;oI2l(4H$|8
z4eUT&FilFJ9unf(dI+)KRx;Oe@#I+Ac*8!0$JBaF`{Y<OGpV-tvg7g)jf~sYIqr-d
zJosF^PM7KzFZ|8*`vaEaV|iM>&rf6nKDUbD_K1PL179Fhp`mTqrp?kSH3kZquk(v>
z?i+;mcLDo!S}e{CN0IR{d1g7x9Q-jGs&7>6Q{VGu)`M2sGYYj35Qv$TUPx<QVmyJ_
z0>L7{4Z)o76lDE)Z$7*MBK)i|7&QQdVCfl<r_{6^n`rs3kzaqA;iXz04+bc^oh-?t
zf)b9CR2oekw<?XzIj5KIfS7}D%l$#v5Fc2%1mAa3NpYZv-i|AP|2q~9)x>a>FI|bg
ztQ?gm>*N!w6;AMPx)$ypb2I$^5sfAnA+-6&8DEMNVSQC3&s8n3XIB<?jy63_`YpwE
z3+Oh5=zj##m;-j~>X(v|lB&lYt&$#lq&Sqq2_SI9t?sW%IVS&FT~>pz=GN<cwczl~
zI5wphfCcG@iAG6kmRe6ivg$^EcSBORs}P_#7x?`LycS%28hK&{sY?)$5S2a;T^!mc
zMXuq?7I|!op1#latiR}bhqV>o{t%&A2pIPm!Vqh(ca<5lXn2F6joIx*DrAtlMc@&l
zr+f%Tq!zf3J+|0Bv**&L`t$8z<Mn#fT17=Dp6SY2I39+!{BSTP%k#1~)YDb>)<mig
z_RZdNwm$KF;@eljW4n+!$+EN;i~yerON)7Z;SsM&dLHg4BXOCJg<4)<{F}XKW<qc#
zN+;GmLrPn2{eo&uo83G!&n5PFy519=%42tfYG>Iha8|k*;R?y+k2%7bcR<r@F{{jr
zC~IgI1g=I6M2d3YtoVdwG&&9&`7J#Gtrn1@y<I5$e)@B;qE9#l@5ZAripwBg`r7xz
z_}XKdUp%fr)9D4~TY!q=huwr}<ta*#gO`=1%y==`!Y|(rl-x1JrYrwwxGxxWQLkjT
z&-;e)_U%e!C`@*tWkp3B!CPrDj#{E3o{uM)Vxi6IP6%0{E9SB?*I0<nERZtzO5SfJ
z*JZOFM8fB}LtXc~(f!$q7stk(`r@k`$agK&6#iy&#SH$^vl`x*OylZn<X11_aYN{|
zx=D}Sxs)r}q643WunYXo{Y4)qcXPoMc4))iX5|Y|5PQ<7#4Sv`UPZ5G3?~W7x}1s#
zPrxeyB1VL&!wr{xnXV$Tn~(n(PG}iP^qi0N*;Gs@@lAQmJOH)3I{FCiW*^3VeGIz%
zcirdjkiWRz^49+whzCtO9j4MK+d#_0yJ;UYfD?4!1WLNa$zmZk(x@0ktO;LAk6TNG
zh*0;9D;N)>w;@8jf%$d--1}hWzTX^*<ho`w<8*jjQIXHKWEG;ZB>p=N54oo!c-$;*
z(>q0fGnm?ijyPCiQ=al)((uszn|GKM>R6tE4gX>F=?MX2d-lfQ$7uODN+rM<NS~5|
zIsjXfy%l>{Ns^Ar)4U{;wwP)A$ZyddgFz7k$hV@%+99XqM;h0#(qG#?@H32~kure-
zA+|Tvfn&eG;i>z^R+lu;F={owGRM4Nj5rtlR!8lx5g?#p8Pe|ZO~SlSFYlF<6f1CO
zcY(qP&M}m7I7Cz*=AwxD$@ZBW=o4fqmD;WDN?#b;3!vbeN`;z{Ex44F!8N5kg{YQa
zNLJnlg}d1^B?cn6Ft5jrG}Ve8F?xNSs|~ht%pfuov2qz>p7MFMy5E5zQ6u2y^Cw{E
zGBcwFryyY*-$<HiHfRiKP=`ZF9KF5DxAFbbmxFicNuKgt?(2ixcjKU{5OpzsY3U%L
zy0NMuCh~`OZjf4LaZK1n#B3;VH@xV5N#dU>-g5QVtq``qi%>u#6o`Ukh9q;vR<sQZ
zdYBT_`nt97xoXzAM>&@O2txI9hbW~r`c7S%e9+KY8TNtvhO^fBF8c2*)e5xg6@P1>
zn)3^7(^43c67_IY;+m;XvcY|D&62-(_~0cu$E?dh20+Y%sp8RbSKgD93)L=}v+Y@;
z5#Kh4zqnKSe)E|B;sxD(i+dlKHTm_OY}t{rHf)Qc>8P11bS~YgMQun;wZx$)sqZeh
z`ztlcL|HF|1BA-`H^qbDmHDa1%}-X7PO!X>=~DZyVwWa(LJ0yZ7^0pReuR}2-eN?G
z=l>NPi3tQp@=qq20Q3En0(b=&1b_49{wr4am%@R880oztT)6x^?&-bP2A&!MCwLQj
z;t$D>Cl<tzMBt4NDnLNwLU7(grf?hW4aJ!=gGt=b<ZOPx+c2e`^Y*jx9&Qob;7#}Y
zXg`b!ToWzM4wzj4uuPKqg1ReE%G%VMeh$SE!M}y|@*AE3M!cATkOL)Fz;L|8O&8iP
z{|dk=j~XxVs<Rt2qWJ#L-&SBM6(Ho<Ba_csP4yUlc2*=Cj;om~;b({?gETAwgMI3g
zRi(@g4qrWgO4dmqu8w<M)-BvE&KrzOos9&=3BUi$YX60lM`K~y;pN%jT2rl4C-SI8
zTKt5I9NN|9p5z*GqRl9cR~8QVC9*YM#9OpCd~puamUtLt?;-HkvvjgIT=rFGFW;5;
zdnZWK>xi(Xl;h+0Z(@mm5N{XyUh9lEJR_%OC?YMp7ne36t#(aF4PbT(yQdDfPd;+#
z8CQK<iD`3NyAJcJYEt^hr1kUQ>}hhLv}dq6B!KD<vKe9zDn=ad_OY5HIe<a3>G8KW
zto0L8`XNcnJ;A4=v62ZD1mA=x<0f0-3dQjTU5)PlXy)w)@*-xLz@Ke*e{P)-&;Fd~
zY*i3=^7eRXB1Mo&EtKBK5_v|DMqo!LU+vc$+dG2}cU7?TS>$clu^k0lXqJR?4!{=|
z{fye7IAni{5r+{J*pDQGg#sN>x$_ZJz4Obe6h@@LV6Q40WK><`f1yiAaP4PWAs=$z
z=qhh_jCHDc$20-xH6E9I%_n7MDwpmNJdSUh6jG5D;1HfQzJkwvI9iCB3X45jzbQ-(
z&oZXVTwhy=QWAwIgLpmyxVRhW9QOD%Hhod3>Hush(K!IW6dBtx%nab3PhtT$<te$@
zz+<lRzYA|FWgfE}yja4}iG_DFb1zOvPJlFJV}ui&ybzK=;x?O3eW@WZqAQ0R0QHPo
zESbl+kD^K%Zqz$bx25Af?;G)(5pW<4q2bi&3aY{M*%-zWY|KIzY{(=*<qh+O0TG3B
zG*aeyUMCK(WzhmvK+=>b5^-)ZHB>~>q%Y<~&|oxP>#T1Y_A&wk5n>{}SpJSuu51me
z1Vl<upkQNK7tpy2*iQD;(+GQ?131rAJoRm1ZE)7a!EOVtbwteSQPzX&ba83Dd0@8`
zx_p8_+pUE|Xkyt;e{8CNfu<FgJ+N+cXWum1W{KXz+&XM)=jA=oIxT?4pVN^jB|&fc
zT`kU6pClmr^x}}CYq=dUdY5DMR+lrP$fDSDahD^ESUuq-CZI9gqMxfgx1mC^DaYSf
zOnbM3m{eAtgHl?76Ey21a~p}*X>n2di{EV9D|{&GopHF*Qjd-L!sYv6uD-<Qol)r%
z721*(MOM6uTix8(CR6G$Jhi|*=fyqt)ai=+YzLh3LvSCfo_)eMSI<PD<f)510Y95=
zW0Lh=xqtcqO>tay5p8ZvLA>Ol${+T$NfDunW$7ZeXT`(%yxaFFRn`O!99F#jUE42h
z2U=!M{{}SQ{7?bil$YI4$S2cE6>gQr$s*DQ_o|>>m8b?Z`|$}cA7zP^3A*Y~U;Fq7
z`*-`O<`Ir5a%uJ+>D>w0X%0Q;KV%0Sud}+_EM<qAPxlG69kOW86s6Rcz0|4->nA2$
zx$Hj{{q`3hBP^rf5E&%Z>NESGbFk8Wc#LTeu=7)IB>b-VpVUb59JCD+5FtT91`j6G
zALEN-iC+4SPo;9~68*<^ki?Jl!*fMt@+NIWA?~k5q3olv28r^8q$WwuMMy};*qx>{
z^4QHpIGof0j3#D2;y=EIMO_CZ3!~q_h5KlhgvO9Zk&ZJg)}l!-PqMP6fE-a+%~OV`
zXeUa?6p~O0!60H16fC~oYQS#p-3^xojRH1xZf?VO$by$g5HR}n9x|0ggk{kG2+8ES
z{s)W!@<m%CfdOr~-YK6hdP#DJbs-l`JW*#mWr!7=Y(17ME{S7r$w+|$WJ5iZ6t0w6
z%bZB-^UTV#7D^@?KB9?;G2rK2Ao{!@s|AfosZS{q(hmC&EP^H9FQ7nm7#szPW2gZX
z6oV8(2g`W}ufM*<Q?U0fUwCus^GE1(yuo5|>xX1}y)z%7WgzS(d1b2VJ9YDI$>J7U
z=o~||1#`m-?2}ec)K2<O_bc!A1aGR2W@$Z~tqyIYnYUnirpPsjg?RnrV||ZWl)C>z
z*jGnI-L+j4LrDxJAuxb|prD|VGPIOPib^*~w=i@`Neta7sdT4wcPR}+qtwtte?RVc
z-tWDid#(2`T)JF~nRCu{_TJY%`wABcTP!XJCp62x0Iuy-Jx-U$p4^ryHcUA163wPW
z&ns(T=dhnDQ@ZaHxCn;HX|KlCQiY|C2@^zazOe${wNZ#*&32ZJ)b;D4MGL8qG|kyo
z>s7i?XGwaTFF0veXJ(`CyX;tlB*T>#PxIzn9HfGD_h(-u2_A!)E`bbuH>H@j<RXTq
z!b<x-re9)0d7@*(g9c5hBL=fkJ&;GdiM~_{MSJC{z8&0j==7%dGZNIBNbD@hbyrNk
zh8blBzY%t@e;s{|^k06xxL3S!xZ6fl*igDXB=CH3!C_BmWA5Ur3CpRrSt!qV3ympS
z>hm-`wUu&x!L>=tqnI_C^*1XI!eZWD*QZvx$ged&e}s=If~s(*6$vT6^oVVETU~tK
z%v%cn9*+#yz!|=qsFFhR{wh^HMC1?oWTN9|>K{1Q^c9D$IAFPHI^tf_`QK<4gueF@
z_?LyVIdj9wj{sJ5oZbk-H`}or8s4}n^rJ+J!gj#k6)Ih78hz_osxL-x)JiW(+X1AM
zdOyYOsw<gop**S0kZ`odbPP50pqRRkjlX^*41Yg;;QRL83^gJbX6TM-*(PddX&LIX
zMBz{iBYM%@uI91QCf}B}!$ou-8=eKP;sT3>g!n8AZ!hItRM2&|y&w!_@9064$vuNn
z$-v5xw|-XW1+hgkA~Ra>;NXCbm9@J9ZKCD3Yht9J^WI28j@{jNVlJ^*S#N+`{~!hA
z$a6P6qU)d)B0N*l3>gyTVQG?hSZijbPTNae9H%mmqKidWW;pLvt!%ne{+6*)N-tux
zr3AgGvQWitFLp6&R8C4j%ka`0Z=%9Mk!LR`JBlJ%!78annYST)PnDnOw}9F)AFYPs
zY`C=p)gm3)!b>F8U7n<Kr=6`Ms|G(dzo<upn%ouY4Vy@tO0Lc%yg=5d@jR8hJPB1w
zYUT)8amwa%&u>sUGB4(f`;tSZNPb~nH5QL|53cN2LC$YONSUWKRoIEe-(`Eq;hfTz
zmHE?kYu4Da?@ToY>THkC5Ae5jp%TTZqqZimR~W}C;Ci;p7s2KW&xFF;+?=29WMpKA
zO7?5{mN7D&k489ONaP6C^qX$azKPxa)_g}Lj8e$0=TyMus6Foc4`g{#rPX4<dWj78
zPB*o&h4n(dT6G_H=^L$M>E7hl_DJdx3c&_rS|{yeXR6vLylYEl5d2k3WTu7~uyd*~
zDma$?Z?_pfeUhd6Kj;+lDUh(gWC)%Jt9c_0ydjuqOB8`&AwXPTF+EZ)Br^Q)(U$Bg
z5tuFU0wCm?U1d-p1qRKOIC1=it5%1hNVUI^<2siuw4;TpCdf}<p(690E{yEQDa^Dt
zxdqoI;6s_+9C?@B{)_q{zOL$kL^B*Z43V_S6@4OLD|(ZEDajf(d`$84DLB<*|5&*f
z(aPpx*MEyLk=qi5^1guzyFJ6hd4(La?V7{)mzsWvrKkxl(L6^?8Sgt544iPiVP$Pm
zVufWib6JyAS^vEE$e7Y(^qTA<cDI=FCg8e~-xJSA2|cl9jr~!R(o=Z{HB)dBw`o-Q
z^!rA*z0(Kw@89C3SzruLfG2Dvm6xiIUTCW73+Sa>=@b1pZBHlv!pS$3`zBuDZ7DuJ
z;qw!lt7_qL7((2Vwx!#+V{HDQ;ja6f074!Fe_GP{ro(|mVNrNt76xy)@A`|ZLoCi8
z<OlMO!&(iyn@karo_=iDe%X=Z_$B{S-rI|5tt|3x{M6X+gC}GJq>|=Nq*Vl!acqw`
zn{8r*&a`5AN9m5o9#S>KPDlqU2gc>ujuKfPJ{;5_mHO@#%0aw#imZ#0VHL?}_Aq3$
z>ZxJ@Gx~v^iHR!RF?nTKJ~r!YDd*6fmg(pz_KPycd(>EG{k1tIZFT?=IV5-6-{N+c
zH3cvUF>@!VP$)jnOMN`*d%(d~f#DKE`ndRfSe-Z3S7PStF@4U<kb-OTCG#fa2f#F;
zO2+lnK@eyh+6rcqjsT%vzJYM4Oda7ryCEUKG=o4&k;Z!qu5?T=M6wfVgO^BGG_>VR
z7*tgJ{JnQDeJQ#KI_0(_T{RGMT8?!Z_D~YCv`PlN)g&V2r?}kt7`kMjDe<;#>$k@g
zW7@H(_lIwzZTrd~&#9nYAmQ5Pb^7EE6;(L88(N=7WvM>BwLT`XIx&3{akbA7Z^8z@
zDP!r%F)Y*&VNpjHN&xYXYQ<D?Dfz2T5m<!W)~P#p?yx<7K3>76(_T<gzM8b^*QSea
zcD$hFAZZjqHRF~)9cZA43?d52O%P^^A(X)jw)imCX+de_UVCU6_{8yWh_K%F7z3>A
z2uF9^?Gre^Dv4fC_+IX=^}uHn-R@;J>`^{hPN<ABZw}upGxxYoR92gp)P!vgXE8A}
z;5&CHb^ut#PC3`ue3x9D7p!fd6Rvz3^mMyF>Nrx^DvL9CID_K)C#Gi_|9Q8vMxzIb
zvqj<Y17B$wJzRgPHUE^mji=$M0_%mzVc$I{!T`}yWq0hDmp3|^EMF5w$peT{R<Eee
zg9qK4`Lfs^SO?w1L^qtlslW?7d#r(_G}C8P0UD_CDyCtEGt=mHNV>;yKFAwO9N(xb
zD0ee*=L<~Yt+;MXfqEvb5B6|vkmZJF>qIVn17k>RBZ5D_?7;RBD>4;Rh)QHJ8axP7
zRkQWZdnn{gQRONs{x#H_c;A)tb5%#e37fnhDGx{Sx$`!8q0D1iMH?Ubz7}rtIVHla
zX$h{8%DQH;`u(giFM1R9(W?fGS!N3t=FoWZ8(9Zo4^+j*HxU_5SndmI8C@^fHqFFM
z(fdKhi*2T2)MUZ?VTPSX10#cnIq6_JzR8vgyjp<xC+yN>-v{1;_)M^W!P!|ni<T(V
z>rMwugXr8Zp+4S#QeRj&KsKvnAtfdCtvWmV(-YmM?J|#^HeWPieuw6UVRVvnHA&K{
z;rZ;jh=1^Lgaj@*<>FChL!_o1D+X{IXU3bh>XXv5JvGCmMSJjyYY*+_WoPYi`qzL1
zQ5fVlu9RrFHD<Dv>JXnRAi))%t`Sow@PZcqF~~Fa?3?TdoogI4w40kiPqNUz<70uj
zJ}}kJnQI2TPYdE?xFB-Lm&B^XB<yRS*JcD=b6iPiCKr8uKi_$G^oT8g>Y^$H_se9#
zmkf+MDypwGg_wO*F)j~>&LJjz$wqw4)7q$}jl_Y0Nt1OU#uTBA`NUT$u3wGB)laT}
zEExXo%dPbouksvuLySX@I};Ql(T#g<y2pzpyzf<ICJhPq<Gp00jbE`Pm>3zQ^ysSH
zNs>yR1IAhX0v!9#<YUz&GsOI`oqhu^6GMKm&=MaCuf@FCe$jM&zGbgDocJN_;iR^6
zs_HjY*<6LxN5jy=*L7ic9|z-{++)k5{rHV9)rP`@iJiA7;*DU+qsKJWmFGo?nk-+P
zqO3}$>!o&K$4<k|QD=vz9WAUmVviS_qc45k&-6|%o?XDFqpF$YHJGa+PI{$#6H{*<
zd^DekbvF6VM1O;dq!jUNAsZ`olIZPu6+gT#AKhs|(O++6f#qWUzTk2Eaif;Piv#AZ
zX6HQ}YUEMX&vh2tdz?gGvZTvoh3lsuoBJJ}Kj`l|3gZ>AKjLwKZC(W$opl-OsLo0N
zq4aW6v)9{;ANt4Bh1RBDnRmPmkCvU3kL!38{PW|*(9Scv8PJ*G6}{8IVSOqqYxUMr
zjtgS`!nk*aMMtC*eR1ntXCtraDt6YWs5Pr`ZQa2-@`+Xbl>V-(M|QTg!eghNcVQa6
zSMQ;E9-cZU70~@&RL1qvI$;uXU(Aj34VS?01g7mmq~O`q<_F`}xy~rHqHjkH2m*oA
z@}`AQ5sy=RmyI0wch8U*6Z@MtHPg*cm71XYbZ7RzYtpW!cInmvA~C;Tr`W9behSn%
zIyZ$Lo=B)<nS3nhpGkb|tSzEYZ8iOb@ZQg8A3kly9YMO?GkDW467{nU^AcT=uf?HW
zzl3crjpg1L&!sBLdg?UMy1x8OKrbNr`$vi46~5Em9`8eIlEO&)E_I=tn~#)b>}rb*
zoL&YO^R7;Dtu<OKcKc1U1wYt-_G!1%AGnWBrK4inqt+H!tz!p{=ne)4rppl<oxW#=
zP4kLE?}qC0YgFY6m&5qhzVfYY4L6^jzbC3wa!q$Aw^VD8Z92CEI&`SKLiIWYjT)<m
zLJA@}3W61ZfpTK4)^%<ue)Sqx!^BRwn^QKo)q+~w6Rih}?oTy6bt_+wRcH4<)rLon
z4?|_ce{Wx;wt3Ii<n+tGhaa?idM&4OO*S^2U3p)k=G|Rfx5=q(SJv$9_s9yZrPmdx
zou=vRjk>GtFZ|kzh$cNcyxokpa=B;0m#uDDfQb@2sj>uD6&J(foYthfx34d1ZceMW
zzrP^D&X3Uk_t&P2d?4q%56C<8t;<&+$>##a?d@vm8@`<owOU?(JE8O<{C);58^&kv
z|M?Mvj)6~U5-Ua_Z^v*yqKrxI_ry1n{Tv?aIW^DAeQ9WqA`i=C491%iA>7ldQEBu*
zZ!2Ib1RCuNTo`(gKjhL+JJ(yJMQfxT$M>FH{?`xbhAjbYu<@fV1Dcib@UjSekIm75
z@WbbWBOt#xsX2aU71J!+XHNwDLi`8AoG*ep4Ty2P_|gRMK=v67*gQwXN25~E1=?p|
zb6m?%$*{(ex6gAVfw!u;HgmOmSU#g)JE$<+gEK+!9-Z2Q2e8`JaI^8;UZ9nFrBf{F
zbiBs*q6cr7D3y8N4t1!eCfZoLmwK^U<^{D5^8ta`XeO;Vz*_qm(vEB?#_73izTLv`
zFPiC>M`evy^RtpKKuj{A7mGJTRO<ECU-K5|eQ}~_7q)Hdt}X?k2Ep<C8T-A8=?Qhs
zcQYQ(d7RYCsyrvT6pbXD<?AWpyecStBVgLbVn7QdsS;ClDKx%G;<i}T#*1PQCCc*l
zMs99;Fqfe|!CZb<;Xv)XY!2i5+MVSy8s6)=wHw>NO5fG-8Z#{ED=hImIu8Szv4No;
z4XR)3;RB`YUh&MT`;p0;sPolRqjPE9ime~4DRYtm3$9htCf?{qL<8{hSz_QF+~kVq
zD)9{V0?hwtdp=D2!r6W}?K>%D;4FQ5cAjycl|+|}0`G3vRZitTgTdVMrd2O+Ed?39
zr>tnW-P*TI%2y>7icW9EX6po8>`w{7y}8Glm)jRKVBapr$4RI)f}iMLK`SdswJI|(
zVlUC(nFwdBj#Sw_;rl<nv8q^gH?Af*Tm?N~p!fMi{u*XO{-kx~iw+PauaM~#gxS0~
zd!~M{KbfdfN>j7?=(9H%odvdO<9nZ&7Wf0GTe9l^-<W@FQzbiGw$`lW((tOtFQJIh
zend?dIad_YmIiF3?uY3ecm-qVHAqVHb`$lAdZZN1*}_`~8t}PiC_4O9X=Mp|;G^rU
zwu!(IBluJ|>a;SSf)t~*|HOjJ5vIrWgh*hnH$oJhj`9=lCp%M|F1w|C%#F2^$PgPI
z{wE(nu`8nRGo`s|LC7}DHH;2wvPlH_#LjEc)S(23?IpS{q);vUZXi)-?N?rUF|mC&
z<8iX=tc3fbC?U?gqeMN*c#YHEDCA@;U!mPryol53gRQeEnwa54XVx|m&Fg^p>?wc4
zv7-+;_Bnmej8P#!$wLw7m@;^ir!-Hb6X##}_Qnl7KKS9wX}n(YoP|}S+u_$tb-~f^
z2~8pk*91N3P`#++boQIGCG9i!g^kt$&-yIQk1@$ZO|suQ_T0|o13%9GOr(Q$$<7RU
zEXpXF&1rOL33t_hYVgB3G{}p*5xn6=|HY`pb;>H@Ax=@OL1lVphTd+sc{Sl@x7sWU
z-rYa9CW3P;xF19&#~cg7PiZ!2GBWx={)~`sve-g%QSL&?32C~JQ5U+kH=CMSO9!26
zyjw*ho7oxb_r-p#?^)M&$LN<IjA9kTSNc9smm3Gq>f~Jrb7}*f)kk%~^0jmJJ#ylQ
zzP;_ydndTefEPHDk#gSLc{;<Uy^C<w2hVv*TO!{u%&VaUdRV+T#86o#yxXB*e?>QW
z9tkzYEJLJeLddpTlw-7>Z<98xNOE2L95l1P7{R$*r`dd;gwpGXYiXPc24&ydjv`_E
zKD7HV9$eo74<g<4OI4DZN0n~(ur;!GUUx0RCPm$@v`Li^D9K1qM2*OP^vz-ZX+!zt
zscPM%maa4Fz1^So%Vyol+G@o|xmY479I^!(e+gu5xvo&11wz<BwX{+47q2yY1PQz-
zzE1As>P9)H<TscI6oas{)!I<gSX=6So4trv)s}SdTfSlSP(aZ50}(_pCJfl!mbMcd
z|Gn{Df&Z)Qx=nP{!^Q;D!-4<EHA=>?$Ew9dd514o%TNARTJaG^p&g;yp!w5pPfHW(
zHXa$!*iA$Z=&R~LyNnuVbHN*bq__GuvHN&TQdI6d!2hlEDT-`uRtQ7Pd7VWs0ea_?
zUQ1w$hTj=F{d+hsWw_R5Jv-%ZQXxbL#*!LRg67c3dR3p^YE!w?Pnul<iPFM@U<XrC
z{CNa84O~cWj{v2vJLApuX*2t$IdpCSff<EM3$R$~RUDQZBRj;}R^;2{(d>J&=!KUl
zu}P)39u(zhA36n~m1%qO2n#ykH?JPy-x>S*=6g8p=q0Y1M#)!3`_-A1ckgVpA16*H
ziZz~o<q{5eFjLKoWTqZ_)yU{(IW)zQJjU1iZLnbA=H2D<@0_`{6yc%2M#}vrcn_yd
z!V}P2??2B&36RHglGDhr1m64d%D$aqy(f{tjTXO$Kl7Eg!>7SFereU5^LBIn$c<vo
zXGc}Zx}0P0QuO=c-*AlP^3zxOhod=3^Do61=19al`KFSr51$S0(w2=|^1LVUHI%DA
z-o9@&TN5`(Q{6Ybc=PDc>#}HM7u>pz5!E+pve13ZFO|f?_muPCeQ%1*MlXFx9F7BL
zcKa^4`*_nx7N2J@@@a_YnRe67`G&ZlF8Q5HIc1=v3e~5(rf}O#tJ<y-;67%Ey*!%d
zJQy$3{d{j|yvAx#O4QbDj$8IOZ$6WNdR@88%Dcp~Uwq}}E~D8L52n&#+t<E^Fxw7A
zm8^4>aZBlh#IknUyA?XQ-*0k^5>`HZjvnh;F-^d~NN3?6+t_(XKYb6SJ9T3z!EDZE
zLroW6;FGej$NbAx<-^%Qg$@3W@A6RCsZ=H@^dxYu)$Y0TZd|`~ainnplQTKa#+Q$)
zBtRuxE%6!BqsL%y4sHCFsanw2jdNkiQgv`}+Gn9_r1nKxwkdNe7D007)$go-AP+W-
zwP0^jXdl`9)DWF92)=+l_#+oVW6Yo>ifV;$X3Ly=r=7G=zU7a?db$es&hy4-Lg&Y1
zt4z$T$_yA9#1g@+!zryx!KoDBLzV6PCh6Ztki~K7tBH9DIOt*c-0Q$o2zRY@7_DTY
zz;dPJUjG#SRE>5y0_=oqIg%Lw!Jt{jWjHQeYA*xEE#f*xe9NsIV**AfJiYScd>Vrw
zneJDEfWYx{PaW1@yBzegvWEiOTTTwy(>EB!I85;5&Fw8v98LtX7QAtidnM-9_Cc{9
z+<fD0qNMCAt)!;<75Kb&KcVe~i8I1zy;MKC7{A6Btccw@SX)!94us03xH`^D5@suV
zEbj0)9n>CGzx92gk?nH0K2~F+Yxc7+k(q@>R$ScIa-mWEXQ4V%ZH-!;z46o=q3zQ4
zY#<($*BW?EeDLGFu7B!^C|Jq2+sD)2@Po&N#4h5I9qo@ddpW%|_{f~hD>4??HofP)
zqW}8-FW|uUe?s4m$s-X%3bh^L|0{ab_w?Ew4NnK{BTxdwuhD><V~G|jzVH_lK4(v>
zHT9%DQD0iHAsF2TPf$;XxQj;8uOiGru~J+Q70Ai-A(a9G6UAwsm4?d9W-S4%tw<a<
zY{8E$M0A$KQyG3O;rrBk;r_g!s)CI<s`jVKOnfIFtjusB<Gj{$rb&;=KBMUxu(dLk
zzb`GsgXE<;CC#}X4m#C<8u_V3Jl*}H1n-CBhC{4y33l94k4z`q6)a}Mt?XA0`CiVF
ze-!?skiu}03vA@R(u=!QtHGKc-PD6R36Cd3HCEQ}a_n9fd^?!I>{Y3#-;1Lagz2$@
zw>L>-frkljE%*IS6(F(4Ag@fX6~hf4nW%p7L<I;8kF+TH{w|5zY<aEY2@j<XEOT-)
z;7{x#&duOWC-~rMi@3o7N9NwJ*S!}$goh$!5+4P|F2nq`7rwuTxLYN2sD*F5%AsU|
z#XtnZR6*qElA<I^|M}rlE+&cxG4@;WNnq*2g?JEjea+Qt&}*EPMz`?>)9n}2TaiN(
zUhdcv9>~+vqR&v<sCqUrd}g5BCSPS#^qIqSCPMn99k%kDwg=M4GjMun6=NLUbJ+CF
z7`tPjj)pzk(=C}f=-(FJDVom|G}21c`h{VjwNZ=1yz43W;7gde7!=ep3VyMgRYLQA
zelvQI<ni;PAPT>i1o(e01!Wnq6u?;aC?J!g7s!lRE9-{5v6nXD+!Nz*?GJdoHdpKw
z|3mYgtjZ3vS-N`+cj5eqL8;rY_1UBxlaX9koZlTX?LvTabWmV4@_obg#fkW}P*(rC
zsiCf;`^<W8<4JF%g>*?|qjsZE3G5Q$b!le?xLaf}Z5Erfu?g>~u6~yS^1ar+6%<FE
zA8%3i<$>Vh5o-RVP~R1msA}^cNh$^JBr3*Ve|h+p@T*?@>c)mlxIotvb_>nO<}8-A
zuy+@X4JR@ea~NN3ddNP$?tPX%Q`7Z6jt9@KocrmJTY}Qe*<`V@x=!t&v>s;b%JQg1
z`>VG?W#ysNBT7*;D;E#$*R9vajdEJX-K9a&#f}A&LSF?YOFjHwObP(3qSK5SAb5HE
zfoMN@4hU?SC!2=G(2Xg5G>-$H8^nA8&0wZS<GFL08Az9czB$5L*UpW9yo4_@5qc<e
zd9?Q2oW?Kd>}!^Yq4Jv^H$9Q;B+S6oZGOA5!bBXh`(&M|%*Sf{25dow^t`wpWiY6K
zK4c29b#BOk(%<jF((Nd<N=gG}h@iyzc^#Gu_<K!o;MDbgft{=-UPzVzQs{j*y#6GF
z&yR_M_lnP+Ju@1HT-OC{-}`vvhv_q9^`igrMkrgQT7+QQ<q=+Djk^wG@}obtYS04^
z%tloi1TMTviMHsM;N#^Hv^k4N5crTcheE!Xrpvr()AsHDe3#wm`tq#Ybo74fl>snM
zW$iP2aRd-^0E@Q*3Yc$#Xx&l(&FX5jB9W?_2MY|QgyM-oEa6B8r~dx2CEFhGFe)KP
z{(+yIm{^+SVw1wy*iVpi5aUj!OGD0SxVxD(E<{o`mod~sS4ga>C{a&jf^zMBil-MI
zq({5iDFw=(AIY7+;#L@;@5FlV?2=+Srt5=r=U_&uJRZdVsdCu=klw#2uqp?}D`dV}
zqoG+P0J1Iz^=kUafe2<|bgaEI*?duYEwno`M_HW0Eh`d2mO&C%lVq<IH`cjLwxrY0
zFB2(w>>0rZrV_&It0{VZ;B#8~9TmLjK$-#B`{B_Y%p>coODpRe*<mx9XtP23ke|%f
zwT4dv_ssg5sLo4GbdD0?yhoU`Bl<Y-IAEcE0ShH|KfwRbyC(#=d*D6AGSVVyB&L&%
zsdG-(JL}1t`zOieg7!{5QB@&pYb82yuMY_|?*36rzR;mB(7Xj6r6dr%Xl@Fv?W<WK
zIk%BljCV_$|77CdFLXL**zLZa_<?!qyXRV8ERYwa6A8RhS8oQ!{q*L<>u8v!InUVz
z(z<Pzy#qh%9?Ow4-kk#?jbq=tz7YD7Lu~m41aC_KO{zv=6GpK-{QB0`f!NCF#7Vu`
zGkAWUDM;YkTNJV9?;&mbdPe6Ay3ZU1_gHNvi_D!)ad`}Mf0i@A7&Jqq@Sd6INqp6J
zI`X#UUJY+Zo8_QZUlNH|_6_t`^7`Mx#O-kc-69N7EBqic3t)e<EwSdG-Cp9Ou!u={
zFlL`P10%y7;)&ezKhQ#+i7H;Bx!_9~<53qzf`ySAQ>IK{=F*+@eHYY0E7%3a;8VBx
z#vlxoL&8VoXW!c6&-*E(l~Q=qZ(jVnnX7mkwk7BR7dS=WiAO^6{^od>mNjG-7ej<%
z)gPkhkeo~hZ5pA2qRDRG`LiTWD&avQJ9VYs|6Rp{fQ$YSaJcsOAaiQPC{1kz$uw7o
zg4eJQ$&>KhuDXph_N$B2(_BX&noEwsgaHPG`bz@wx(vg@7lpT1eEr9*2Q>?>1!3cK
zP$=s1-YX>1FRMpEMddRKjQp3<Cq_<AxpyULlqXM|dwZ?c>bJV43gzO5qT_Wv^Mgn}
zzh87-ELg`)e6+qOR}V$DL@cM$S#U~hqqQ<=O(!U+r<p0qJ1voUP^r8FfeBoU3clID
zzhRaQI+Zs#t9P&4=zM%K6~~p!EJ*?-{iYh(KI<GMG5axM;D}qD-zMC0x^^KqTv3KN
zVyv`4--j{lE^F)n@2kfuM6o}u;4#>QH2vp1U&}3j+2|C1F|hdg?J?fl(%!E<;RB9>
zL|vA%O<A;~OH{oxkq`TC)Hv90)KoU}GkER%oogg42a_(YIYk$?x9L=rlrArq^1l2R
zP+1GP1&Wl5YC{kJC^ljDzs)sD6c-28qRmD684*tP$T3KJXA^Dii$Uzzj(Frnt}%|0
zgqBQ0r(7)`+G`OHMZrEIAf7XPq&~qqh~`TExx0S3Et3F0YcF)uufr#J42P@(qn3CE
z_uEq!z58&QZejNbOpnziSnjX296}?)(B4ba=qWK@Jj;O(N!4L_l=Xf%BSCgcas(`}
zmnY(n0K9_Q+Wecvp^+fID>JD3IMwXVevgCNZ85&%anrBXdTt(O@noWFkVlyDE(<E^
zD*9&fwd==%Uv+arc3F)Vk^z(OIH^D)x=(^0dJ0U^;gy1pwTBjC*}<#p>#tm0>+|0E
z2I5FY5&|0nFYH6Ua#j<!%|h!|;B3#p95W3s2O)dm$1NsfB*M*d;Pc%O;{zjI!9wOP
zm+h&VLA%$n%$%IkQt+l1*~s?6GLEZp`zzt8Kyp<4c%%9L0ewZZG`~t52pl?lm2Yl8
zE=ky!mG7w@$G)PoXyWvyl__L|nW{Bw&n3POFAotSdmR;>$^tO+1On=(efON!Hu|q>
zh>o4@T|el~>DD`nOzY4{TP61#FTVlz=N`A-bS*;9^}lM0^qzYjt%pR7Ri`>`&Q&QD
zA5~gCJ1e_jw%8!D3DN_>*9T{NT70sE=(*pOT))YEO}&yg{@<#lD)hE;!v+kEJN~HL
zi0U@-Z3KF;kKA^Po0sqxW?i&PG+c9geBwY~VFNu0LLUf5G7To3r8~hLFO(iM-WiQP
zGz?=YAqD4X0AFweuL+|&K$v-nfl)wEApy!Nj+PLjM@$<~HjGv0+5w<lE+GAdjf+oA
zVy8MW*+HDcF@Jt;QzCyQVFZ}7L9reb$LVV|8_EznCF=cW1>ed63ixtK7W>~A;idPf
zlrdmdmq+=p51o-8T3033g)HdrUdJcEx!(|Yc}hgUnS$Rw>J2%qjxY>I(s`B7qQG9J
zsqV)s3JOmjnbQ{8PaDTi*Xgl_hj2FWXVZ}Y3c<k|z(abea<XYT-+n_~hsA8u%^Cjq
z<`UptxhLsnTjDVzLkhK?j$)F<f;lR=CPt{r|4f{RipQ2efa%Ru)KA6~T<p#!E+^|&
zto=G2A1J2C9DEdku<Jz7>^z~Clr-!n0>0OWpAkOueAw_>&EFFiN~-q?x)<wCbv+a)
z22!<qhFM^i)0JZPc&#;h0zV&B7CpOOxG{V$4*d00#G~j3VF$~Vng4a9BX2QpWUCN`
zRM<0LX->cCdvD5b+KAE{m1PQBB(Q(y2eQ|qt<vey$O_BYUj34|M!Q_<o*R(XLAM+c
ziwJZ*!aOc*#nA$-=M3c+as^zGS}?PP=`E*ltq4#Pb90eQ&12O+HH}vH;|1*pt)vhu
zuH!pedsd=j05j1N7f&k<>D7wq&Xxk(U?;N3gklg_l%k<6p6V4cXq!hBpmzI{qj&#?
z>GxaQvVaQ=k?Z!FE;77vWt|~+=(+REsqtqH-0~IAc@4XR^(iA+vv-|RtLuBQUZFZn
zhw<zD)c0p;`K(iDou>4iNT)IlfrrnG@-lc3miBBAeEI1&o$tk!rlV!BJmNxXY8vB}
zu0;vtQQgbWh|w2bH3V~UGzZWS`xH$forMCj^CyrsF=We}C=zYd?_l3qaP<du=8noL
zDezXJi>KYaTZr+!(@%vn>-7&N@nKabYB7e2&`ukiurIg}Kbtzuq^j3Tjz;po?GrDq
zCZno9hE!UPzjHl#(NEjFe4(kOq{LuoL;r3wpE4fN`cjT^D7v$GDEeD)@L0v^v`Rts
z*)W3B#q$!=iPkX|A*NdS?(b{wFDhez2`IQ7T^~PpZ~!;84;@rRzLn%LF%lqR^uEs5
z<nbM13+OvRV#z5Oe2^EoD16K33-d*OmpBu|_7qB4TmYXlKfh0Zr<+gSVgot?mQ8>%
zZ!jn@sOfzfXunsSP}nckYNXF<TPY?jWWM>ZGSftOm60D-8H(>ug(9{+k7mFSv}Eh?
zOX5Yfg|pHEad^&1F42zizhLiOK!H%d{OI^M?eYeGuY3+%$cJP?Z0t$B^|oUQhdd$6
z1{sA2ILqKYmY!KKi=2u!y|kd=hIJiMx4>vHmmxHw?Q8#AOamNTNDN%$TJ3gpu~SG<
zEJB$B`ZND_Ma}0>TwgUOow-%^(>gUzx2GR_U7u<1D}?i?fZz_#53((Ig~#^NR}&5O
zcddN)%sxF>lSFcjmr7S5dMNtj#>b3*gIV@&7qz(g5Zxw^AVRQf(XC*0W9?}U?Q~g)
zmi-P_2|a*>)`L#QZt6+WhBxRNABwnN`A<q`nH%s>+wEmlOWo9*iS#va3Wemd0xb{P
z@n<$CHEDFx2Xkl0mla!~aJ>q|#`M8LL3EKe9t84w=6{IjpZnba<901Oj6e4<-=eN=
z@6tYUVkpvjag8;hcqa=R2aQ)a6~x)f@-`#Pmo~pwlS{r|z#Gzt4lR9|+{S-!x94pJ
z_#h)}e>5OtnD?khOrQ3^|1kj(AF&-92+i+Jw|}PMce8VJkRAgltZXEcP#;P`;Dr$=
z{E6oZfl1fZ@y2kI1e0!t3SZc?B7k#ybEQ|QoDl}d@7O_Ve`${;UjQ}}(i~p=cU>U{
z;BPEe>x!%csh9=5h~F>#ABKZZAd4H6Dw>^<MFo8Yr>t^e=1rPsTir#`i4Lzk@$Nx2
zr%lNQ>~J_6k-ii8ONPbSIXS&R-&tIt7zB^XRPmMGr>Ziwt5hw~DFF^?QGfgm0~VI#
zB_=@+$wS}nDziF!Q%JfE==Gz4-CC(1EYFGP_vPXafW(8<PpDV7@JgUO7LN30(tvna
zBwyBMqqINEAoF#ya_3GE&EF;6FMswwI}N;X@!*x3J587<Ub)c5lS0<+@7k$|e2x6r
zJw!L9&XiDV6xldO>w>KI=paEgfZ%kA)CZgMe3Y*o#BN&c9pkOvHM@4EAGAdqT4@4A
zCqu>mA4G@v2hc=e`o^jFl9dvGPdeD4Rk?a{N}d)VC>rV0kuFS(iOS^MA3k6-{hg@i
z8ChupBhpXK6LabFe6B~@1HliFEo1`75zH49n+!f@zxLJ-UE+|@m~jY-iS4d0VY2%7
z-BniH%o`LH>AT9nBCk`fbAO?W>25MG+vyRY#)s6V@RBT8Gusos+flXOQ*?|L{P>&O
zW2Gw$`6gHygd&vyE`smgK;`AQeo_DT?=A{Z&dO#Gcn)@x=QDj0?u?*i$nYFb`E-V@
z*iKNI>1H2)@7b^7^`E4KAn>Va-1qgOZ9={|hONmG@Uuh2quYj3=Z{CN?UGrx4Ij5w
zSQzyecftNb;**8yhXqC-@-DDaY{lsXeq~o&Ic_pDt43;65m@xgh_ycH9pIkb1If6a
zQzSyqbj&95>$DsOchKMI)_8i*XQJJBR@*F+PHgnD+Qc?O>8JHO;%Lp3h)*g)GF;Lv
zBG}jq5G$U&Dro?e(qX#Q5T=wg66QZMqyya;9d1Dva#m(ot)R{JbcOWpcNe{p(uEBj
z4%W1vlQX^~&|Oc<@{aBcHCTw(nR?t~?bWqb1t#L{CUk+V<?HG$_=Pn!&?-4p4i{-%
znM`krKh<?xI5f`suvf8q<he1sTm3kBb@6mjf3_7D@}pof#(LiU(Bv|J@N2nI@9?(y
ze`|mw;=nyjf+36@5GD=`2F0YQuJ4e*B6n}W67d03V^|#EWdx>czMkxgT=W+1y*Hvf
z(#2m|*VH2$jy{tTVx}Q-^vjQ*kJX)_)lpauPb9Lj(pT0&?`MyH9^Pg}3mm^Cni37o
z?n<xw4e<nV2Qm4&2}ig}E0$gQatqG28hp>JtEsLP?lgs(Xgi%SPS>?Ev;cVi^B9DF
zM3DYp`1=08;Oo(AQfS0p)%pF8u=pW#jNiQ`SsjX>&&JLZym%u+Tz`})M?QdsQau%~
zci!7ALI4amN>TW%V=HhBpp?CA4_mWfp&HfO9Kpd=pPlcFCMCU?Bz}15+$pfvC^lAQ
zQ9}G2)v&g)(U-4$mGicM$!K%r;xbJCs4JA3L0w(YvZJ&}V9b(I<UBBvIcRHK_J@&&
zIG$I`^=Eb)Uz`9lB~}H<)d$0`W=;?91R~JKuzP4C=29%F-7b!+=};>P#jJgcob-A>
ze1Lapwfw*+Ee9V)#^bUJY^SA+T0xk*z_G`fNmm}XOO;?efT-4G5^pN1^*~@w8JfRI
z#(#SF#2;^u_wOgN6R+6-`OxcmOLmfPN$irg2tT^}1{?=-suvSX(}FKbNOK2;#{3~6
zH_VT24t*jND-g|dQv$rik2Kb)RF*-AhEau!>0op-@cBGN8WT+*H(tDc&fS`kbGO6(
zSx+mTGMMhcFcy#{_P*A1bF)7sKQTHakbbrYUyO*n1}t<NC3f#NN{pYkOSDzz+rt3F
zejO>4fb=QHUk&@009m5&qSO9c`(yyrz(@GL-E0!U!bFZ+2CMH5{f~399dS_FUb2{V
z*JI3zg2<+-gL!w_RP}xPovO)Kz+lHlAHWXq2%rZ0=b5|!1b(LS3dsh^%9Wp$-I;09
z=K6ffF%8USTI@{|x*o<l2}DGuet)yuus3rB%<Od-=7~xU`M`BQK6^v9JLUehOibfT
z5x85DYP`F({L+Vqb6;_RNlW2B*3J*NlC__;&9<)0^<OilHl!M{rlwU7MzB~vZbZux
zsMN-(Gtb|vjS=;g4tZx~gcxM<>x#|h2uJLXCUf>RPA^itS|Wu$dtTjdm?fn~4{$c$
zS9kxhc%rIIfL@s21SLt;iUFRzJ~sOH5T!vKOa(*3@I+rY^TYS%6+qAZ^-C|0kO$<6
zz==p`<u&JHy^yjxF8nGDSq;n>6#m#Fse5kQgm{b+>F|-pZChomJ+ITdPdk^&kun0V
zY>$tANt2C_*0o>9g|b~zBfi{hZ$$c+uA^>4yeyM*@1n$|iDgOkiH>3s`k9}v=>J+R
zxqn|RX*mb!tCvEH@-Ge>j-5@SI|AKx$DDsudn{3ZT)ozphhBM^nDl6q-FM(##-h|>
z0Q8AEzIXwL^`3UH$o6r&8PM`Hssz>zDs}vuhfg}A^5McmkcFpbb)nYT8^&Q23gkvB
zmx113ujO%U_3pdLV0r-JNG~H~dnVQfZqr%@2@d{&Kb=X%WBys_gh@n1u`8TTqX$RY
zOOKVfFIc@HTh~OmA*Z;e<9*ydp@hgxZX+H<x<)oeu*8zV&aO$Cir%w9Lg(gw*3|KS
zP+W7WSDmBa8NpTq6O%TC?xWmD?J_Q8P$Rbop}BjqoaV6#akxTa<t`gLi}_eCu6tF6
zH~0mTLO*va7X5QI3vX9bt=V^t@8-5?Yca-?3$*lu>C~XYpAI#dVSB^iHfFP09)1vW
zAOjKajQsctn<$QMs9%cdI9v;KqR+a4t_2EJq@O}`&v1W#y6HEkkMAqTl!E)9uagnX
z+S}Pel$jRI+={W{K!nT`S^f&NVwfnLCQ4h=XTP;5(77w2Jva^$Q^38<*hm=(3<Lcs
zD>D5@Rs>+`f0I%%r07sFAVDti;M4zj*4<d*F%}`{gbYt7<2vruHt*>WOmg8AZoYQO
zBcN$ea9Rm(^vwZg#6JVZ728a($i?+&JI}8jw_DvVEHJ|xSXetdys=z)K8YxV^v~t1
zkaFwka&bV-B{zPOZ8Lu^b7->!DbOu5D0`i@Z9!?zA@<a1do~Uc86VG-C9SLNTO$)D
z<hE8lI892wY9krk8s7e)s-VZ22PwnA{ZP?Yy5Iqx{i3&&O3@<-XF>_N)f`Kf&J>4$
zODAU(2>ygMg=0hOcRSI`L>%kyPd2~Qr#aTAxGdfMNm1)34M?BHN2rbOwZY)9IQcD!
z8xJIgJODtR+?)9)ry;VuZQtwZE?%i?1BdfGp$Z}Kq)8(HyohQcBti!%QjK=|-9=(D
zY-Y7cd5u?M(47&QGKJSEMSp0);pqwHJ;)$zNTv~B0K|P-<gWz!>k)@1#gOXK4myTH
z(Qaz|<!sujAFTm+_L-&Q@pb^4b*+{k$Gqe4i7uPKyRWlC3W951jKg!7DaFyvBfk|`
z#!yHmuL(CTi|9B3|7AnT`ZpYWw-wXHBC%m-C}4ARX3ltz%lUWzfON8M4}7D=>xVR5
z5?e{Q8RHL9kD8t4%QU9RP#yL<m)gz}wO^xIfpd=I@+ZdJQVR0&u_R3H-FIS~9smWF
zS^(%?g({Ed1|}P7ob1@XvH#SJ6K<?tW)Qx;pxq(z(#v@^guE5gPd6fxRlEFbfTd^k
zmC{-wog~CeRlc14kVF&@y-Q061P}PQvg#uimaX%6G0T5&7T;*qL6g~oHJ$LT5FaU&
zNY<J~LMpU8-0gHXHc)zjZCYIrA4|t^PzmrSt`rGJAkf$5;R;bG5dE*UtI4!@U9a;d
zrEn#3fF5s7?wLy*v}aML7wR^vcwV2eJ5~L2RmpvDZ+kSj==7NFQjD#Ju|9*&*IUBA
z*uw69nBF*Hs~8z!IM|hC@<LZKKSc@Yu&kgxj#rt6K7@6>78wiq+qKrvS$CNh9P0c?
zdI3|w*Y-(P{BU8zpI;#j|I=NoDK2$ql@W<R*)O@^<AIGQq)kVDY0QLO#M~!gA;)3_
zpcN+U46Zmuvd#9hjAH#)A;3&<g~dv<XtQq}kN$SHiTzvYmGi}Mh*xDzwyA0sD9+)|
z_nWnfybE-b4+P)Iorjl-)WmSB2A>S23ihWuFZ_xWEVWn!rXm{;L)<OdzTckB?c2*z
zDwe%luHaqjpz@<kdVysV?HcRI?YX)iZ)2yznOm7gBd7wb24k_YasaW~_4~JSwn+Lt
zIjwIW*}%TigXXbo*FJ<f>Js!{4Wv&X61!6zG}Zo6I*hGM!ooVc&sGU&{V~t>8%m6R
zH<V0QU#t{Mr^SAH6rlY9&<<AuBEhfxq=dgHxz6xXd+ilj!+LfO7+|50tMVG**UWr;
znq6V+?%9ex(W=saGxh)UN!iG54KWY6AQ7^+POwSu6!TYc4^hZan_6!s@4(RfeQ$Jk
zjEU02B_@wKDpY1mpgZPxWf`(N(6y@-Y=vu|#elu?oWcnurYD_#Xgiz2e#wusiqOgg
z%epIq#qEzB#A)xa2J*7{2HTexhV`9i!DW|a2@I6E?D!xazGz;ROJJ_0KwigzuzOe-
zLd(htG%q?bm4<1HMVx>@q!oM?7!|XD+FxuY=wH~3>L>{ATD7uTWY;@Z{&mlIb-fa!
z?&#!4xu@KwAb~C6g=US#qyhjiSH5e#mXnM2x;{)Vx0t*hmINdtUAo)jM*{u!uIS-o
z4O%NF0o<!8k`Alc>Q~j)@{-Rvf`Q%-nN_(3oIs|(O1k>f^ZgR@2R#ECP0j2}9kD#E
zOpHxYq4h6h!%c(7btG^78A6;^k;WE-E^is{;6ZGgSLttJm;~u8n8cuN$J5=$oqHkQ
zUl-H$>4D(y0wGUss63M?4|gcv7_aB;xp>ql=jmYn^4RS>p^CRLasQpv2h3|^fuY0-
z>He{F4CcfD;-1j@oS!TMlu_7Ad|U`Ik`S4e4n>QPL8pPC6Hkre#Ct!@H9H>JUt*Y9
z72=}2qJpO&^`A^J0=`lxXyV8RvQh!RCoXj5F~j9~U(AfzR_~}Rp+#04$4<-#40_R}
zFUAIol0<twj=5;n;DD_RME8Mk4<dSe?p<#?2um4%+<X(5_sr@Az&r^T*WUzd|IIrC
z{%xfqLGX@mv1@A=8ZzD02&H60t$~`E652WqPsPEN%14!+`<;}7`;=ARpZZ9Op$iw>
zea!j&D9Zg}r^cUFl%Um`A?Ejw0^sXvqHG+oT5L*vplP_RUSV?fi>NxB6e`}lyubf8
zN4?@CDnYHFBjdZ9bAJG#S5G)yGB7f1o!HD5XI=HfsZO1{TX7|q5j=`{aoc*|Gxbq<
z#ytq;Cr)EoWQ!CH(-SiWZTQhnZTal$o6O0YzDp5@_M`+kWWI7m+??wT8H9vTy2^5)
zRAFqq`qena;~b|{SI;^>5-0)<NTY+0Z?BeZ#-|(2e~Sg2-t`FnggC32SM6Q$Mg2C1
zUNO$Pu5r87Kk1;Ch>?Hl@3x}1KnL^uV0Uu)4_hXON!UYXn70iY5gI*msTJdMKNG_=
z1{59D8o+OV7sciK;wSJ5mcp0!Hb`b>q;>Mxki%UK5#ozO{lu)pv4|l+;6}y&9b-Os
zTQDO9nV)E2Fd8%NxfaLqX8VwTg^jBJ@_b{ZADTE?gr8x?U<z9Bp#;$;BUL<uN}%w@
zoB7Hzzixv0)^-Mx_P;b-$$!&uKi`LSBns~2+1NbDgS;PCVDy^Qp@uDLobB^LB6}0x
zYFz<ie`Tx8m*_n9yJ&2B-1hEXuV0|{S6BoD1Xg<ux52_F2BRCyQDr>HlJ*Z^CZb%C
zhOA1>F5~)XwL&Js*VnzbXNuw1FB$^nuSeMrX>si5?DFu#anzfI->STcr?|&XF4^Fu
zovrgdU_B_|wR$PyLP`A69qH5Pq2S{<_#`669eN)bPZ!JHtN3F@UN$0>-+h8JHHM{i
zW#;)--(vmQb(cOb*@9wpi5K3DO^O^+G|h>wVJa00Q^j{I=FsL>4BtdBCES%n=D+(M
zKv26+WHz-*%vagtBuRW*J03XCJ?*t2nw}P1T?!05`&IehrmVZSrl3cVLp7kU1B}-C
zVjTMrWrF2A-{R&8Wn*w|;eJfn!s5^p9!QA5ZlSm8$_((<^xolL3c=99q*vmCz&T}l
zG6+3ZJ7J~m@rDs0qIBoTjf7lvSFy3NJ3gXe&=hhC@7~Pe&M$=YM`bc-l5`fU=nSEO
zBDi#=<AEzwu^TvE9pYS19GI}J?ih@AQ%x`tdG;u`$C$Xj3=w1O`l~~Z|Mg#>EbzH&
zolOnvvK6%jFCh+j9*0?5#^<Mk{HZ|`Q(;-}ilUrf+Oh_w@(Eu}+)QCi(f{c*$=TAd
zbnA<G_zai?W8gvL2!N_rUj8e~%Q>%Ne@8_`ur)YS$^ZmKK!dGO`~E8t*G}no&%L;#
z28tnscTQyD7@aJ`N;rN6O!$CzF4%!44}YHY<GOFB!pnnOx<);Li*dU)S`4k27M5ci
z>QCxT@+T}JjKa6}h3cp3n8tV8%24s?-<j0PE1dcyoEMasIp!62fG~a!pLnJb5>VvZ
zey@1WJWoK{%05rDp*qXpHl67BeSO%+tj_Noo=%r)Sf9q`)$H#5<AaH1e796s=?<^m
zq2<ms|H;(&YGFmO-P?oXN&2bfyorCF<P2O@{@YHPZI7Wz3mY@6kQ=cydl$9=#DSdQ
z!WO|}IOc$ir(er!OU{76LMUT;WJu7wKt-48C%x+y_8=ZETUzN79X`v<CFB8;K`yBJ
zk9p=tu!7~7wD_?DOLHMEWim&MAdnRRj#}^6qM_`RkvV|1f2Vbwkj`)kzCW7B>1c{G
zbXS;o7qRcZi&(;Be&==Yr&$H--&FhixR6Qh2~q1tXs|?o!Uk{uR=U;c`3yflU=`?Y
zb(zny`e6(pK|9BfbxXeI>r{6af9DP^JbWW8$9b#*6u&v6kH2{$3hAg9B2@sB=G&qA
z%|~11KyXGPYJ_c4$h5(vXy&#%IiYB76eRxEs6Ur5Zluh3<vip1qJTgOxw&}5Yc^m1
zJ^sa?%#(vZKuYp&LE}GC`u*F6_5BbpZuo7(S{S?FNA5Pa|K3HfKue$;5~GJtG+7HB
z^#weHVeU!%(Qq_C1mrZM*51YI&<gHM#lJvy!t(W<FLJkj=>Z8A9Qt~HxEkB62caX%
zdkS}q?l><M)6b|eh~8Jyf`05;MVC3iU1mVy{R{|m-NqFs&IHDBGr+j_CcchS((qY-
zl4p@3LHyaQ#`qU8O_>b@&z0T3^~H@9D*GC3>_$!;&+SJ{e=cTE7TjXeczaCypt(jK
z;$>AZIoZ{E`{{@lEa>Ig7j=ERl>vFZryz>?y2Q}HQ!R#RWpUa>=1z{Tu`CGQGVos8
zIfT}=L;2m+g-vr0&G41r2+Q&nfGTEfyxx(L?e}!D>bmd84I~{?H)#B)TlzP({x1ti
z)eO0%#hqgGh0_%>fyE2Mc*4d#X?k%Q$4W7{+9cQt)rP}Ui`gSaKcnwgx)%-%5u(A@
z?Py6ie3rKr$h)nNM;a?d)$MODzG+C3ugs8;FdGoAmlY=4j)UcrknEaupvE(2h4}(y
z2MzZT_818O2Q%cF0WKYPRA!jnNEgZD$ACR9eG6zt{Erd(-<lEr;z&^f83^b6q__re
zQ5(7bB)xkqqcb!o`&X%tpeQClExmQ7s5IZDxR~2s#LABS8hI&r7)vA0TFQusz}{D>
zx&vVh;?PKB1vl2eb-IgG4tyJU#T0=UT?u_|Jpb)(`N43Gzx~o?TxLx;Z$ltz9eKXW
zx(b)XzjG$3K?3v{l(PUBum!TwaB&eZISpsj(ft?gP#r+kKx-_^<p@wLj(F5^l7NYg
zS{T@~C~u-DYynKv2pl}JoeW5ZU@WJjRu)l+XAh3E_a(;Ww6g~>z8Co-&ap=dPFDF-
zT6zy$_eekAx;l^Fa)knV!QtpcYUC7tF!Co;+(H=%G-OO%h1AkC=L0~R*^H<HsvVo|
zP7u{Z7Tn)~cKs7koA4#cMpt_m{^zGkNKix0bgN;&?*F9%E9Zm29V|b1nBi{8{h7TN
zPOExN%fD|uwrAk6)%KF-#;4Y&OyYLR=7J-)1>#SOEZ%<X>Uzjlzn!!y`*HSrtC?H<
zz`$AH(Z{G}3`{%-AKdfh@1Hgdbpq2-^_E|O2Voi~+tW;sHL4=p!=W=w&Inj+UiU&(
zb9=lTAbe*Tc9X3&9-ffA5Wd*Hd$!-5xc032!P|edNXdPWw{9j>0v!j5@+)Ku5d<i+
zI_L|>ei=UZJcL7;H(K(cLXJLAtPT};lFRGy71oTfMg)bP)<i@L{}h`k6Q%nSju_FV
z+IW{hJFC$WvpA70tpHv=KXuYmJ8zKCoo>5XEh>+&<c&aHkX>cNnVv*C27QG1oSl|n
z*OyX4!I{ubGRS<ZA6uvG;GdoDe?>{QxR9WtI5qXC-?cRp18R=ntq-3}`w4p9m2A1~
z*%MLCgz{rd^%|nEj@LSfF2V`$j8z03PI@Y*1&zNwRL|9>P$6&4s-1JG6L@yEe__0p
zOW0W_IY1$UY<ao7vJXsh!ophGNonIR`cH%XZ_1G2e#;fk5UvbHx2^zszhAH39LS@a
z`O*(4H(1$Y$y{=oz)R3Xpmi_BXqr~<Ft8p0dy@rjdx$qB^V@a>bG9c~To;gHM9A;V
zAluvD5Abk^>6wA-XcP8R$RGGtTR3+!at)w+`SwtuxB__G(e7h4xi7z_pW4I}Jj@7a
zfv+fc)57Y<+qVC>jnMNA-v7nu{*5O9&zt=Aftq!3TMtht#lko(K(>9dOcigG`J<W~
zYF2mJ3<JueMSZ)?KBTb%5;8N7#pJ%z7Q(dQ;Bf6!vbo2t<aKp}s}jA{M=oFK%0cCg
z|Hfk%z*o{O%u%@p7X|3k9FqoeDTRJSb_NZ{`z781k!ZO$q>8;V>j8<Mi>aSTw@)~5
zHY3K1MzQz2S4xMq@)?8hDJ;U4E=8lp?yxQ?fNqUWY|d@Ws(R3_QbG~Z$9fVp3=H73
zdSb2dqlD)@ff!!6-D~``m{nUL!iIBB@>z<zQ)Mehz!Zwa+*gJ7TmIZScmD-oGqJ#k
zNs8F-0sUiD?&TLcg$%?pNZSqV1_hz7@e@iX+Y65Y43lJ7f=cN*{@hee3jMN|v>EzO
zJM{AHCa~M2ovUVJfgyq@hy_|0GcpD-hR|nMDQn%aI>f=(=+z0#%`MzT3v0v-L>{Ot
zlXSy~4~QXuLu%(O@DSz>6@J|aD<#GYzAi^O3WahbMK3VlP%8NhTS}<h5W+y;VP#&}
z0PoS|JZznr_=%~h1Xzryk_7G{nux;#;_dn(rL$@K65k1Mh6w&&ygnNQzW?Q;<Sixo
zETz>Qau@WO3D(hf4`Ty>sG=`{7n}dv*!YqUKwLq)TV%b{lGLqP@vR17z^OsZ3kTgR
zY(;Mo^g19{YQWe<zZ#XIH9x?bdJx|-h^f0NfNhSXi~wI^bco}K;C7o>Z_dG{5~HD5
ztGIH+YWG@Ov}o;XqZxfLUOd6I!d)353SDVa#u@70VWRZ*v(@^I`>6Z>vG&$cQMO&U
zI5BjGGy_P9w9<{Vv?wSIqex5V&?3#yAYBq7-65UQf`A|}bi>dM=fT(SJLi1w`<-*v
z`u*0LwOoroz<Hj1U;EnE-ut@!z=v6NHm+h*R!RPUTWHI;BP_RVJRYV`vX3@?O<HSy
z)yo^s;5tVW@=#qdmdymp9bn9wv+?N57mmI`v=ALRqF^v+bF!4X2kiO9SGLKx%^y31
zVlM*FX%<Gde0>;zw;#;JGu!A|+re$rtbgQa#8mn|5)YKH>LLUO44~~7kZV~CqW;zL
z&HU>@+0ip14u>r;4i21bF8%I;-cGjPQmaB0xgO;ga-vXMji#QsB_V4ohHpd@$PQx!
zXo8>^AuvbY(l|3YQp8qctU5Z51syWJMfSq(wqq{aoA$4neJxEJs^O5$q(X?<@R*xC
zxGzTv6CH{f#h*lnus=3;T<8s?6{goOP(JeI#(ob0B%^`Y`sg69sw{;zd~h*t+ma)2
z+t>fnlT5&9>)0-1ECPy(otat%7kd$p=k#_n=e5c~3C%%^Mx1BQ@}R=Z4_80M7Px^B
zwOI-<unYWdKX>5cV|H(1k?i{#xF~?bUHqBbyv<fP<>O^ws^md;<F;y(ATWFI6H-Je
zBLe_#)<Ui!1|1MHLT4rQB`^PjYX%G4WpEoECXZDTfNIX6fBccRoP^Sj_Pz-Tft(4+
za5=yN?lz0&Krkdbj^+Hz3&00S42@K*4(%;cpukv_!1zdgJsgAK6iFcQ&{tp$MY6^~
z+*BA0Z6{{IWnbrhi?2ej5g^por%B+-mF&`)Ew$(0ojSsc9wtL`(1!1aCo(=FV=P01
zoSV)M6(WNY5%}Rh`w^!NRBT=M^Zqq_P6-U3+yAof>FbSVd~<zq>;PO;42(5;ye2Qm
zQK0d|D1mC|3RqjLm9u%phP&-21P?2*YVe`h+uNhuM`z;XRMjrge^up}1n5lfv632s
z0;PZ%b7o#%O_!bN7n`$!z}_=U{KIowrE;rQ!11a*x#LFUa1I!_x(v(hP9y_RDUM7y
z;@?ET+3Q~<^G_>P>getaueZoYCz}BKFlAnX|8pOP$r;YT@&Cg<40<-&Ra<(S8f}>j
z1~o`voxThL?QVo*;Ysgo3JZ4Isf_nVe4W)xPLv%h^Qw8e{V%xX=dkiu0K5^F{V>2$
z$^2H|0D);GQgk;eJUqOY>zM-V0SfJ3>z_ZbSj%--peI!W1`>kVhofy_3P`surS+>+
zTkj3r41KPT7g5{GfFc05Y@FX3^Z}RhJ{A_%s!)OWBm2e9vLzzB&BFX|(F$O;^YinA
zA<@0(XSYVbBeIYy13exY2*d}dg#2B|cgtJ&z5SoVp8W^YF~z<cAR1r!Jklo*037TU
z^fR0L9l&9`p~7*b5O@152vm(m{Q&3~hJvq4H+6glw=0TYn!ngGTa+h!2-$nYQGx=)
z=FBb!Yxxtb|Anz31mA%0xvn$^POCBXdEJAmdX1@a3QT7Zk;oLoOVankNr_={gF>s)
zKqLD>a1fHvg>(?~$a(2AwO563_L~pjQPcnR=<>Ir79sa7-CXV^0=IN4Sz3Oc?PGlW
z_;Cf!kGrWqdZxQGMXa((8i*MOxRrhHU;G*kz@ry4{@lLSvuE3jz?RTMQI~BYH@}hi
zztt>%!pZkOAtR<NTN~p{t3o<bRA-~lQ6G+PWTSf&xs(KwCFxlJH<G9j>PVKlLy!g8
zW(T=<0c8*c=7IDtiJ5|p;g{p22jD4!LbHRYR9aT91pY(gC$F>XVspVb3b<(Y+a2gx
z+^s;*Zvd4VYTy%&CdrYG#PGpp><H8h?1BIZLXBoL#=03zOvV*acGkK5r(?uz;tsfw
zXUD~nHpVLKOjqv^5eSvY8Ij7K=mG=GQV;HC2}t9{S*h;GDxcxKG_5ZTzL$c*#A2p9
z-!`CsqilZ!dbqJ1{qNy}Nt5_HDNsjh4jGGjJpe~aI>q}4Ed*5`d5SQIDis;i8II4x
zdSo8^7Kx5DQqCX!_Xb5Soi#!_V=2erp~-<=fUr`heM}N)I#mq%>ngQM(-X&-nirS@
zG&b?ueK~+E{=WIh1dn?9fuF?s7|dpTsx9ZPT4?yIS^$>sob;RSrm&ogb*n3wV1VY^
zd$7{KQqs8lEGDS!(^Kd@J(ty8j$>|V=0ny>+rfqk#<xMF+bn(MXHHoASkE}Kl3%aB
zZ%KK&V_CiPQpl_;J~@Q{jnBlw>cc{Vua?dsu&~5|ZNwVvSJKw5cq6czou8Y9|3Fkx
z+uhy0=6yw(`F%Iw@@qMly*V4@2aW_sg@#`nUcX(Ec;v$TYpW-|yQfDPa3Ex6XAk@Q
zY!ZOClPS*a&MMc^+j-yY80{kMK|?_r=VzM0#zp88-5%j6B1I*o@_sF#e1XP0+x!P=
z{X5$1=otuqXMmaKS^!Nv)A%Th*Sc3@u!~}iSmUCFV~>!FA+Pt*`O-NN#_(w5M0vKB
zF9eB2wbrEctS$5-OD>z(u4b!;@U!Y<Lwp0MgQV+?>$33La^XQJbVqHrS#x~i3?`@o
zC?rW3Lo|sJ1A}fy77#8$0#}e99nQfMqrtdUpo`4ygh3!h+z9;V*XvwX0I3=~Ha2#p
z4TCNY*l_q(@uvUva&PpTgLYn#5q1wU!F1;wK6Oh5%Ab@Dx1x|QOO&+3QHPBN^wmX^
zHP^ok>JLU!307h=B(RdrTPaajf>#w@&g_)_GV8+<pUJRxNaa?lj%}h?c3(;nTrNU0
z@(>ZAny`Hk;6V_RfTyPv(MhHKwa7Tp`r8!Koqr3o<;b&Px<NHo;8E3{bItXX0!%5s
z4n+Cg6}b7sqQ~c`L*ezp=fIfnb=6l}zlfgY%NyJ|z2r-J>Lz?KFkKR<+NQr&|2oah
zE&49meMpZNHfw4{Xfm|rlAb!T;DlVC^Njh*)%XTOfC(>3tn9<jmRtJAK54IBg9i@3
z#g@j&q<|mV$9vtPA(t{+CG@_xP*=9~wv8??w=B>pO^44r1NTvbxL4WepZc!*r8xmD
z%(sn|_%^l%WI#uvQ&#wVId&yV2A`<Peg~*XAyz{c8{a=Yax@#A+)Z^HiYDXKohszi
zuhVEgU!`_U?y8Z-t>3Qb-{#e9blHh1ZN7MVb9Fdu3d!!m{9<<vKp!)G0pb+giT&lx
z;p*U{cXMK|zIr%ku}zj583DIrzqy@bVS*?0U3v0O(lR9i*Ese2qv2q8Y1jj6LD;6i
z3fc1Y>3nUAXi`sB5ZLngsDno03L%3fUjg)`ik-G5R4swI%In>Gq@<X^47b;NK5sfc
z;HfMD-W7KKH1_ZJ_I@eQc`;d7Sfmfy*m=*?zN=ko6sS>_LA35!Y)uq@Y%lc&IKsoL
zTlP8X4I)nO#3!Qnf~B554W$KEy6#~%;D7oLZv8)*cB?o2-DzkxV=Gc=6C_XyMj}V*
z#BD00e}yV=K>RR803_i?1QNIG%o>^GyFSAifBeC|fCxhz<P{#apA{s9Aw5XWcDF9l
zDVFz21lShI`D<I`aEkQ8So$RrxxZZN2Vip&*?uWApUxo@CPp*N_qC~NVGz}U7vF|h
zRFTIo;#_5{#$GmT7jTF&)dvbmn0d%{U?+}P@1&8B&DO!0Ut$%#JG|?$VTCCxtl|v7
zkAdnh4ib{vXi8sV9?)yyx|@~#vTvhxm}0OA$jdnFv39rI)C#nQlv1&B;wph2*|j6p
z?|(@FEr%FCI$l#>8GJW?^w?Q4yAgNe@i?Qv8RhNfz>Qap*x|XXa<cDd$E_iUFa}#G
z@HIlSe&GxCwk@N!iT9UZRZm!;0@B^bKE1nn($_V#Gt1p;@5*K?BgK`u#5W!#Uv28#
zT!s#uy#cOKe3`EsZ+Xz^%PRitDG%Mcw#_SMMx(~Z1@##pd)~+mpd$Zt|LHKSciMYV
z<t)7f(fnCup(T8KbtwK!O4I6PlUq5n_=xv*d5`WR^b2}pKC6j=WGVcYwIw_f^j0cq
zBK+&N_zJYnam=n!Nv*snTfi{R%N}5k2p;JB3jvI8DHSwI!#{Zi;a}bq!}kV(O^EfC
zunF`VBeBz8j-2H=b;2dDD&?g<*sZMRz!GBBL<*;@T3mK?fs>=|zZ?9>dWwe{hnO%J
zkKDj1nt81bQg*%HLw8A}Vpj}I9B>BFx=<Kk|ET*wPd51RanG>JLLj;BBjUKducw9@
z#F*B=ixYp?H#iu-mf<s3BkkYTz;@R1p~bZglLwI^?ped!$Ad0QKL^w3xd1lo%9lw`
z*t1vlfr^O8e2b!u@D9i=fzebM1?T?;<C&PUbSRqgh*ly_b`_xXfK%0V;y0ZF8fP=o
z88YGcG8sKSnpbD*2PzbCkboe%$W$wDD+MBmV47<WNrHkJt({kxVErSx%Mb|^EfUX5
zv0eI!gG(xpyV`ps9`ZJNAzVlXj>B=VoL>Zp=zrg7VY25DydaER+q9Qe#&MMGCvR&O
zx6C<GP@(@*^sn6Q?rKjpxjQN!v<eQ~o-Gk59|Rxgkc2imaJ4~>Cw};+@zS58omW7Y
z94b-eLz!5ZJlN!0gs0aN-M#qq{d@e!jaJsRX^Uxvy$#O8n38C@L_RrHGyp?%jjKDa
zm*revQG0(;*W<2C7>c1aTf>QxjTGH>8%B@m8Ou1wWC8w&)&+}&er>sNPuS=E7dv5E
z+v)o~7m}cJ1MwIG;T!A};4XlzDQ`pX?brT{hV+1Wfs~5yMsKV&Znu7uS@~7Xh2YQR
zdbt=&W+m!MMukhV9Zun;x!S#I?_Gjzr$=?C#_EB>KDB3hg~=K(h9qui-br*3$u_|$
zEO9A#r0n?6L87e2`Rq~1Bi))xQ<|u<MS#KX`A+a?i;qv6Cj9~?COJK7oqytvxaY~y
zbGJemEkoHB%4(p`?UVuPGaBI5Fp*UIZc0NiU%UIyIl%c!Nl9scv3#pDS1Gp1n@^xj
zuZMq9pD~_&siyCjG!-%4jgNk{W&CVC>h0Uby$4q)Sm2lJ6Dr8U#$BQJT-NZTz=~6T
zv`hrstwcco28mr?(2$kzp<PRST-<BdRjIqS(`0C)T`RuN0Ci8Nu`BL2T((V(V^{6|
zr3^x7r)q4uml3$uK57%_Pyd`%S^t%zKp+YAeOxLn)Z#c$+rocW&zH9-T0$*|;PYyC
zHI#IO)_*Ym|Eb_)ni2)y?ekP#@aG}6oe)SGieKp6+ka43u7FxGX#5QF&cJbdjC!qf
zP~=q;YVYy-rA*aGe+%93=%$%LW!=zB1!?LfMmp2N$Exh%CM}&*SWg)#Qb{0Utf>Cz
zEGUhQQiA9<JbGmmRSdxjG;8nO9vNji9_HW?8V6+#Utwui5^zicms84^h+Q(dsp$A)
z(1#iP86&3B2IiBBI+f+Om;J0@iGZ22SMBohvMiae163M`Ku|oGbXd(t9@~W+)1|;|
zd%8zF8@ZR3-o0*-SghdI{pawBIng$*gIi_C%5Ip#D=qrAr9Qvg1oUdEmnq6{<8(E)
zo?gsKEsU0(8z1i53})pz5W$(*>g#W+!}I2_r^1(Yjuc!ouC|Fp+;gTdN#ULKv0`vz
zI7swhd3$LnLD@s!(hJy{QlYsB55kMAyR)DYteGizeWUkt>soTV5-yY&B9p%FLUHF*
zcQ8k==WYVf)r0ZO{7+`4szr@!#iJG00h0CIj*(gDE^~FeBJ=wqZqhIVd=xASef$zp
z=mVeI%lS{h7Jm&6wc&sa9MwUQr!A6VH<HQ^ghqpQm~ng6^R5=`*pv(p6v-9x$Jh?Q
zAUIJ(7phr`5S;TBVQQOCQs(|TetC_aMW4!O_=GAqJdLY<nb=~gENd_{O>53$i&Hrq
ziw2#EQlF0I#}i5+yUf}dc^;IWg&;axtjAZUk+dSe+?E;I0)#b((?oT=&lWI>+DEy2
zQSE_S*U$}|f4l%s_<^v`8q3eTlEvIBn9EvgBtftxn51E(@G3=ac{{{n1X+_g$+-6O
zyw9z8Qs3q;T5$kinN%!=e19|LB7i9)L!+H7{u{*FN!>AJf^B%TC3d8_pRYn|B9Ysg
zy1f!jS*#Y4sHtKFW;lisl0P}Q^<%Jz0~sNPs3qoMj=YK3G~xJ>oJ3r!+d80(5U}^!
zcboEPuA1PHt4zGfF|H^h5t`1-0rMY`PHQ}-J0PTq61K}CM5UXeKVqFK>-gX=R$vjp
zy?wUH4P28+D~3y0&H+e)kwrSuz-~Q-0(UsA=!7>lv@eD^VcG9pFr7KEBI0F*ONba^
z_u9K(tlWU0zNB<75UOpPApWh(8I#HFs%o3n{UDZAEHN7s%u3_!k1IOnnu_dlNx^55
z=?V%Xx2Ed2#7iu(X?%4Lla&4$h*%7KyUzQu)-?Ty$C378QlH4~l-hM>Nq301(L^7*
z$Slw!$!fs*(JUxRiVd%=gSh~DwbXMk^raScjO?fD+Is^<DOs$zxwH$#TdhmI<65(~
zFr~2!JSVmnx6_Z&72|~t$Ys>`3x!SJ?$lIgVSx5KBXQ{EH0($*Tr@FtEGNELxW-7)
zxXig-A5Y3W^~;ef(*uDzi`lgD&Y~4F34m$&M^6A<hcg7!c=Mh;BqYRt$<*$-0{usZ
zU-~OMFaxe3!0k~KSCN0pgmG$AsBLv&HyOng)8%2?E}b6=F@Xz`pYO}_)s!0r5i|h&
zfd%M*#TX#<1bNPeF)r4<PXlNyPB4AcHG`X-?Ux`x{-Ad^z9wN+%N6;eiv>1m`!s|>
zkHW^r=JH}x7O_5ZW=`q;{XyScJfns1%<B|3$H`kJqmrxp>(_(GZm*n_{#%e`Vv>g3
ziI*2iA&Xrg;3k9Gy#o@J$rH^0R3vI85;${RE_Rh>6e_<vPM(D@#z=r5(hc%7X?Jv~
zR0)b#6oySWOaSCnlcj)<y@W04l&`?c9}s4bWeJR_A-BTE?M>FT-X(h@AyZ&VLpYqw
zWhh{19T}}z1SYDIXHr4D(sRCG-jS?`a(IA%6|fX1;et>U01*H~GDt-Izl$@Ed#Z@m
z&nbKsrb^(mnu~WW*DSU3-kwgrnjb3dfWs*M>#QBm-#}2m&@}Sp$46IkMeWM7Md@|l
z`7dclW1{1^Ix?uaJWm#fv(y3Ej;+=J-m#k_OxJgGo}q`Th@S5(G+2}6dAy}_8;U5U
zny`4VRC-J#U-6eMls5=wBU;lJF+6V^&``B;Uoc~d@o8O8@{w>}slM~K7Zdp$Sm4Cd
zrYA$&i=GP+%VCooF|IAQuq%s5T{b0f{i^1jQ<U@FugB#Y!wIai9DlmovWQ&7TK_3Z
zi;c8&mWGP*KF@I+cYd!9PBZTFpxG9{6aSNTv917qi&kM_;Q>&(V9a7=7c75bb~8RD
z+`f<UQNSjd!>GAwgm8gS3>%HE75g+e)_6H=F#+Ec6Z1YHp^4c*645xVsE@w)`N{;@
zb}v;ToXDCyibPFFTEuDHcu5*OU1kzsW8}G0eKJMi1oQ(Rd-8?zgRZ2YiChL>$P_S4
zT*(0~g#&kv;CD#_W|}TA20a5brQo}0`*}cGU#@3Tv3fzfwftfD#zI1IF()_lKNRvm
zLiyc47_4_(as5pRO7mc{6?X4ELa`<WB2*g(RP!JLCn$3FH1k_21#!nrn9#pBx-gmv
z?eL=@wQM8hQW#hR>OO)Q!<FwdXv#pfnP!4yHY(U8ZMh8QG6nerOH)B~ENVWTbrS1Z
zhz~)K%se`X39ESt?S+(4cUg%EG-gmad&5mSleiHR*tG<Y692TKzydg8IC;bap~VHD
z+JtdnuJDLpf!+FUN)zQVU1Bg<O6QJ}U8~xDsGpo(vU~;%>0Ade6BsoFFJeaTDxKns
zpTil^dkIkT^~K}a{ZcBQPc&l}6!l#+vncM}okJJw6F`<s6_9Q=y7+p$;iy6<gIIqz
z1B}HL7*#flUBG%kOcXkHuLo|uFEdn9E<cJ(bXS+-eaBt$ZBY)*QeSx*FaE3cwQY2Y
zQC&#h^=JDkT!$cb?g6N@_r+n4T?h90-5{o#=nmgI8>iNO!0?8AhL*CdU&j6!mA`_6
zLq)1%<Q`^*@iKB`KG6GFFlua+=KzXszys~}T$FUIV_q{#kbd6lH0|o3pIgqOktq_3
zgyn3f><P8Eq+OG`ra~%+#Wh4+$fHZVrsr5yNh#X<`r_3KZS4ejFx|hSgRM=5sa)6D
z!b7oQ{L2eH%OqlQa&FN_V>UnQyd#!hJ0>a+G(W-H291~PGdwHxjv3S{mRIfTAB?7@
z+=yxRN}-2^N}`<AVut|6&Xv=HpY#q%a$SCiIJgBcE|;M<d(sRX77DBPP8J$v75?3F
z;a{-UDjgyPn4)NZ#EE{3?S}Ti?UVqFG9A@Q#m$K@Me{hT?{X3sH)@Xhf-H$WO@Sd$
zwGozTo8ri8xfLYgCB$W2QnyrhK|G^qsN0q+%wx6OzDt?<!1Bx<nO+wODR>LR)1M7!
zQ8GLj7<}0j-+?Bb(QaP`UJk6Yuj=R))Sw`E(lVVl`>Tm5ISWJ(raW--XNrYb-XHGw
zE_LY>y<t<+>73_WSaa`v&6*tGf-D2koBeTq>DZvk01OAvE)p?cp{4}?x<#Q*Wjg=-
z>B-jv1>QzOF`sLkn4~CkuXpJVu~p_H?Tl{W-i}oq^K}E$K%G*ibQ>4Gja$Uc4OoCq
zmT2phNbFb&-;|XEV&arRdcqQKFDvNS&@b#<N8!87!QzCu0DlH7So5FGHaO#OVkqbU
zU7nu3^$4l_?}TC^uDcv}ZLq@cV`D4TlZH4dR+mo^nAl;sG(sA7xwyFnv%p$IsFAf&
z&q3YmaKSk@8M|EUKoG2{Nz8q>eyxD3raO|e7Yq3#c5ZYws0F5D=>Ca6K=xqZ@$eMT
zM!h0h`a0UCf-oP?{aCd%@-ybq)#&~n=dD*X$GJ^2DQG(qjXO$u+Eq+DM(eKM<Q<2A
zDR5i8%^SG2Pq3cN&b7}bi2xA{MI0ka1pC;E)hcI8wTI%cByUn!{@$hk_e{3XIw3&z
zd*%nw6#1EFNSn4WSbg`d+Sppvz;H@mNna`fQn5}(;SN3<nr%2nrWuD22q8n~f!X87
zeF)f2p<q~c+Tot@3R*glP@o!By@C%rCz+Z1ODnSnue^>5Ajt-cAhrHKE}RuwJ`HtO
zI{M!i&N@SXJH!HGn^J~u3xT~T8frN~z~$6`%rJ-ODf(Jx+Zbv5djfPK;+Jz@r=0ew
zr$=QZTt8oaXw<xaFQjPuvJ$$ZbsLov1H+DmYf30o)sd8Ze5v(Y@`H}EfhhxBw%6_Z
z<5HLf=^DMxI}J!CS{}Qys!DI|te&Mr%y{*vf}An6UiDq>u^+V_Oc^!<{M%tMg!SoI
zo+=K}Yk_xj9D7ZNY9m+)AK-p1jCbP4Ao;c%0sb<0tlVki*+*eV7F7)AYnacKjOBPM
z`kVWeW`lrv^;%H4!tKrVCl?z#Be|Q)`P*;Va$%w2%ps`wfQoo9^C_SP80F~g5UP@%
zeqb3yK?nnyIbwB6wzeg+%%B|{uDaHn$ZWiJI+(X~1~DBSH_(f$HV{m3nUMK_8~y%0
z2szqfu`}d((2K^#!lr|7>@godQfgn__0-mZdpl_+uSqQ??lUtz7^|^KcxR@@xvW?0
zfw$p!(iM0l$JTspnQW~U6BNkG+!f@-w*x0~n~i)o@xpeC_xop`nI;YS+sR4%qp{}y
zLvBO;o!hK<01M(CQes}IaHJL_AimI~Qc{t|j-1NI01ZSEfvPxQOkJH4>y=aEF)_|a
za3C{cmLam7&Fv**)Fg|O^2?7&CDKL6Kg7<?wW8P0JCn9Km1&yZ0Es7ZIIt}Ur^g?z
zK(`vm{hq%^fde}Fc80IGm0ZMMF$U8yvlK*t^J4q<59sJ>GXaFy6?!zPz~G}SR)HCB
zd!zx($WDVqjitO_orj5CEHHWp#3#Ucw+Bod+83H<N#{NuM*<95!Q1`kou-$>%G5${
zB!yW2)!;ZzBqn4uZZJ*5PM+ab&Sy0;pmqCh^me(PGWP3DC&RGf&So?D+^+qjbQxiR
z_F3`wS|7zAG0@6&YM1`1?*9FyNKv&&i3{_~gOt9Os{`d%6|b=x4e_#(F)^YRU6v3i
zDmib5(*-1*<f0XTb|I^qyJ3CBUW*f82PlJxQRa#U8m78kfGQ7KuGLrpw6%CZKYn>?
z<a7O^`TArgGwVf>c06Fi{wk$006k4<95WHVuq%P`ov_4*$KLf_+wJAM=JPz~OCa>P
z22%uUckWyfv`Y;Q9$bA_=8n(JWhx>Fglm4cy&s5fNBR_D*BY!b2HgIqWWbdoSP~n4
z#OHXWo<3fpr7l7<_QLW<$<m;0U%UIk<)YD72`a8ORE7F4FKc9h@%5ie26v`Js~@Bs
z+%DA&ab1u<-t>u9^wUtgmMH^ra3Fm$7TAWGh6^%`n0;K_ug@;1a=F&#{nP3CUwQ1`
zZIXXY84xnw>BEMr&%W@Y&3{7+@|DH2Lv0oIg)<|&9Qa1%E=<4)TgknH10r0;j)bsB
z+Q^q$$!#tehITsChEVG;DKaF=7$U1JJRl<&_3dP!M0p`1mAEh&uv$!*AMF!yRwaM5
z?ll^?j8CT)><Xd4v0Y#iAX)puFHPhu6G>&0C?J*aOnO*2i+~ynJm!o+5Fn2m228{(
zExsVVi!wL`GSu@*Dj^0FOi!;uAvQYmY9X2coM5DNZKMH&*l4M;Wc`=56|y<bk9y0i
z<hIz@$LF&{iHGWSt<w|v#fi6`=Mx32ZgvcbM|HA861y}rvvJKz@=b0ys({Qzxbd?&
ze`nZ0LZ{p3s_rfOp1cJXSgSeJ6qd?1e7>N_L=&x<AGz1H4z$j8?CJd=?F8JH3_pMV
zeDvm9oK)hI>yw?C8HkB7<_QW|!QGdy9ij*a*ia@smdl_4(?!tFHx;p2RQjRG5fs_o
zMkE}ySTfnin!!OU;#B56MY{+rWLMbSJm3Vrs?ity`e%7+oJ}bx-K~3I7~Ta??Y{;V
zVO-WLheVWtHpQEnk+xOrNb2@Ia^|T$avQFnz;ZuQza6j{vv6Pg<c8i34(X&*CR0rR
zxx=i15ZpX!N#o|XIuC{dBYX$vJ?}2-nYD|6%|NFQ;_bjJFTb~dOR|7+LW#KATOIG4
z1)mC+W9(9&GJ!iH{ypa-e6+!k+CBHwO;nxPjpj>BVT0|zM%4bD{Qk`P(*M?KYm@h-
zY=hX`YGFZDa6&1>P=-EIjQEIxU&KwG1)9hK^$5DU@7-pgl&zmKwluYiprC^kMbQ3(
z0Ro?OVR9V;y_bvlgxZ-);a&F6BBxP_NtU0O<Z~|pzH&#m!*eOEKVQVT->Pf(f}beT
zv+KMjC(+c5=9E$-l$JxM{bfc<RBP|`<JH_G+VP&yl0VM32L-_*z6HK9?`36COf&_4
zwx>soFfc`VG5Gm-sUM0KHg7mJh$eZyJKfvYGI*gbe*8RbFJ$>dyk%TdlpiOlL8F`K
z4blsVAO6b4TUpu?maCSt;~a}GakK`NH?KnzP?R9IWvw|9yYoYYZp(}K;2ZmJAD!A7
z5Lo7dAY{1Z8i&`kHyA~xT1whlgdH(eJzTOMQ0YVA<&jeyc6Re)J}d=Xr(_Z4Vb{{O
zzw>xFqWSA3+4bJdf~b9eK<#*7D+Jc&*_kU788Lr*{2Bg(zxXFFw<ya{>eY>Dfo|s`
z%PGJK-Lu(H?Q)}9GUS!(;KNII$Z(ctcY#UH;)%k$;n_4>K^)~H9sSF(U$uxiH_H{R
zos5yEcheITfDIckd)(rcLQr!!X;O8}V@Po6E)_{=JebU1{H3m|Yl9^92}24mL|*J@
zL`II>zDM$^Lo$w)=Yg3g3akTPltbv)1jyaIZ--K!5N6lU6uBb}__%gQQ0t^VX}4gW
z5qV|mS?|&<6h0)f^(C{L&{E%_k9lTA&3fL6s_k;_wAo?zc+!Z@=W=GA;$ei=E4TQt
z10&a2XZvCC)3_x*r(sbK{~loLWC>_la5?O+1;z>|a2w6tcm4D%<gTF+-iO8kj9gjM
zIE_emy{ujjp1fJ#d%$e1437>|)NPbsF)!K;ZVixpn;ITq{NZc=$(-ljV!UW-RL7Pn
z&-)jbdo4cu*l9I4x4Faev*-6X7pJrhIEU&s4>EdIkLE)or<Y^rs8G=baOA-5h~~P>
zNIj9{?EbF9Tb%(E7?-P+mo$qp%HKZ*aZ<F{Vymp<9(4g`NH$3PR7e(hZc$HCbRm}g
z+=ITIRO4r_tNPM<mb6A2zVr7ELnpqi6%88lbn{vs#L7Sle@bY@oXEWkF-;dJ#d#ek
z0I?RBC-sI}$LR7Nup4<tcRs{#k)=vzG~djRy8l6#VoRAcyM8>$w2E#MxiCD9$D}<R
z*hVIVSZSbm1TW}Rjh~=zE>IG5+<+6pMDtWr$beqroB*A~t@uRg3pFe;H~4u=rO!Q_
zbX~%)9w(6jJ+w4EEmvpHplxSwfz5ah!<#d0y`3+u0kW)M57VZlb=L%8ch&xZMzikZ
zW2?oe{h&?^^(zf}P#f6j#BI9cZRpAT<vmW<;TBOWC0nkE#h$N9Q)NzF-z(H*Jbedg
zVRJ;2vxoWqbSS~;;vQtUk|*6TjMAjGlCPK3b#@bIa(a?O=?*S5_9M}o-Ql8UovHPX
zi=%l8rkx{1<?{vCrk19B-mv}skwuK9utaEixXm~^!3cZ&Q^W-xSU~a$(a|;^E2gr_
z&spl$#lFa2x?t2TsArYWY`z`-cqrJU<HNn#w;5hFLK~2U_P{{j*Dcp4?C;4B*M?zt
z2De&^U-!J;@kR!Cx}MIuk`vSldu6F-n3azL3?bX7#k(sNINyLC%mu=FI>djC2$Hlu
z^$`^s^^fIzjfwAl-f#3-FY$PNG$&v(o}Q|eu*Qy#oKyFC>T25t`7fhIGHyef)b~tF
zM+9of(X)W|zCYP=Q!S+WCd2E5O_rG0^^0!xL+_Ky47RE-qU;EI>6=hW5FStpK>#;H
z)UFIs5aHF&G3SL0VMnuiuI!+`Sk^?ZPBti))UT|`3B+3spAXZ-Zp)?Ei^Fih2u7p%
z117__BE#d)i}$2k&bqk29e<{-Zw>gm`~>GlrIP$7;G2?8!Dp^9x^nw&EMFZ4fz953
zxiEYQ^cppq{6Bbxm6z(np9xjcU0BDJrr<T6qxeb7eCW2<ntyhX<b1qR?(ySW#r$0N
zR-_1n{4Re3=2|W+<?XQL-Z&R^kI!2k9I=LN&Xd58zF0|r`#ABUII|#<k`Y77fj<1-
z5@(@Vq4G%fApQoVBhYxbGGA5_6yKiIG*?h@(3EQ*Q-TjBG)+;@Helj4_$E~--2P%(
zVQ#`OWDMM!^>y)GqCIaK@2X~%@bok$^$)WbtU5&M8LIcIjJX0M)($stQ1+`rFj(w~
ztsV&SqcJ9&ldg34h=pE#i|#p;CGhJ}!4V@4K^GA31R%XeaDR^t(c49P6?U`dwdc9x
zEv2XTP_(%L1h#oIef4YgY<lI^=vT}7jdq8NEL^%>J^i3i$SGr0N3g0v%ArKR_EGa$
zixH+8Q5#c9=JmOemju|tBgw4EAw#E|Rqg!s=ndkWF7%$24(SeCX%f|*$f|+drOcWS
zk9NuI1@cT^sf6&HOXo~j(k7i`%kG*Il=o|(=)6prpSB3KAfSmFA<By<c4muQp&Q|j
zroRyO#RHZbo0cfB1f9g2c2$D7V><%~yG{uQ_|fBu<Omw!M{457;|HBV6Q-Z(#vLt@
zOJ}AM?ETU6J(_NgOSzu_F_YV}v;;K4tfNC?8Ky5!(Fn$R!YQ?e6zIQWvSQdTV&<DX
zpD#UlVrWiZQ@<F>u3L9F_DUD`by@_qVE0O*(cHrxS|T@k2Y4KrT9idU;k4Z&tW%lp
zYKMm4_rwQ9EGIymJG(zo(L*6nd+gpLI;WYMo^}4?xCX=(+K&c4C~Q7k3=1PMS+6t#
zfggc{gQJ8(0=;_c3_(J_^eCT1OWvLjZ`KqX|LjG9k-e<h3<v9l=+JqqECA+WYm49d
zxi-hw!q`|dz}FW!w6MC{-I*o7OWo#&#T4sik`Lpb+IJmuASfhXL!pkFu3=s!FI(=<
zdCDEGMcKL(LF@mH<99#NAS`(77i2h^4QV@nl!QS<%7;|{Ab=N2v2-*_Y_4WW<_*Q<
z=b1LgInX~lJP#+z(GQ25^+n+O>GGa^ES2DAX<1D9N_(c{KlJjr{ir@^Q+2V!;X$YI
z86ONM8=-EGM$aE+Z>i$>B7z_beSaARMIMYLTo9sTbg5L*u3*$`@XRyA$@;P?FOQ|$
z2Jfj90q^u>Yl$>><!<fuDR9ac@o%b6T$>Nm)Wo41xKf6r06uz&iXDICAMQ+p%~E1S
zhzos{cmu;9&Ny9e$~3kC(7wQXhOxG*%jd>DrF2DTEu357&FfFSr^EBt`ED+;H#ZOU
zcP@B0qLlX2m2OEf!Ji{0V-S`JV%+(j(3A1uds(;M`z>8ppAN%LA~D7Cz#{x}?njNE
z6S*(QZ}!{kPTu)g^LB4R<_;RPmR?*oJ#?Y2(+ewa>abGfQe<^iOpSq!?nd97?kep?
zLX5oiMa1QNj;%u$kGTcsd}Q>!W`49;8B)y({q&nm-Ui%aXGqNuZ(%x>1IhgCZd3CK
z_NJW{`DtX0yVKBTPr?2AYS-?FHVj8s8oO5HE-Z9$&&|^9qeLGMsbM9|)Xi24{L~8H
zYGwqyDx`8)qLEaj>R6+t3*B%X!E+s_rN$N{WWyx6pG~6}LBuHEIaLmE)M=a+^Ho@G
z$D%CTED+!CB<WJ3`EfXP33z=|qhjEH0YAJzEc1)i{;D`GViM{K(9(6ddD{LqZ~l7M
zwX2W*VC5KiEWH650a~wx*m86;+cQLOa0k-GMU5(~58sAsK|Z712R73x?63zFBo3nQ
zr%z7m1DedruT69xmovwrG06k_saemxuWeS=HE+%>n11l4Lj9#=6T*byVT6H_uEdUI
z;A{dlHsPDIC4Nq>#_ftlr?l_U{6}{O1NSNWpbm|TY<MZ(t@!Ox#i#wXS1s?DKif=l
zZ4A$byu0XmWPU<-OW2?8QL*hhDgWvFReer68W{BwKRn~>oBl$TxBA{5)>Bo@jdPET
zYX1SVlcWS$HQu7Y`ZioRROC<yj?m4K2wjZVQ6HSWHpfKl(+3H4C$SczK$4)J-3i<D
z!v*Y*KB`mGmAj{|66ZYO2*j`;&dfEl7|<O0{xcmtRs=Fl;rNr{11w_Qg51mC%heI(
zWMEN-zTV*Qp(PD+FjAC^)rhtly@k(mE3CYP{+oi)<w8b~gMSadL>ptYx$&3`E{>R`
zC&?`s1*ZS2EnwMkSll%~L^RWVb@{Xh;LCX(#_n*R3HLqNyRUmQ<vs9795KS6tJA%k
zqL(@-1f3)qSADFeink11?3&5x7TE8ZL}yME{o_%4Xr6B`j55PIA<AGJHNrwy>R&!~
z;cEVA(J~6%Bgisgu_DReEf~WRIDc)9N7@)DXUusm!pphCMjK&jrEVM1hC!7=gGN=V
zCcQNWd@&if&CdwI2t+E0sN2gSne`nk(a)u^=iKgn%BE+Cg{}Y>H5i;P(uT71J3V-z
z6R@5R(-L(Xnblmn2ZVrc-BMnPEjD=DwNps*y-s!6{ETg5v73R4)#0&a%0&0{!T#sS
z!(qvmt|E)+&{A+Ledctj#H1uJdNyTKJBzJ}ei|jbb1oWNkI5;dgu;MT>b~%UO|*m1
zwd@^sdn6hTS=IR(^E^rEs_=SdK<H%VK#E|g!DRCA@C>H7u`eP-@(^k!yWWv*$gwon
zbW&Z)c@sYL@iD7_^^^urPtWZsA^(Q={<n`_r?Y~9RGYqdOENPZAVK{KUe5Vs!yp0`
zqiZ=<jU9FOtUrSHb~M4X(xEJ(o88Fsc{pv0I#D^pwX0e8i^T1!+l;F6;$(ss593V<
z0K_&9XmQ*6RBp9rhP-&s(T9Y5_IS*6{>J1Vm^Xmfvk_8hSYmlBH-}maMoa5PoQErX
z$w(F$&nibymQC@y-J^U3eBGv#^SNtB$q?K+A)gR$O2HE55*{d5oB3vJ2gc&vlw(xi
z@AoQXoa;UY{N0&$zF>j)-K*HRd<RD-yEgqd9HV_>)cM_gKb<ejj<nU8R;f9}+jl-D
zhZwAw;oDF;Zd87_8eoj1NzD<PE;UyorTTS5C_^Agb>%?ia>-rOx-<oWMW{m;NxE9#
z%_q}VOM3Gc6JZxp#p0O&;d@!>tqO%>y#2Zk<jMImfH#e^`gz=Q8)xO*+uT!?%81}V
zn5XgOkEtXdtWxl~DH4@gZ^R^d?ffXls9l(~I;wLUB<hJLnjF56;V`PPS<t1N!lR0R
z`l;@l2e}(f<!Sl;=p?kMiIITQlV{hufia=PtK#gJ`)rfdQvJ~dIOU(tyf2oMGm7&A
zgc}F%nvwOvKnAyYt^(X$GMm8C%bw9sUgCp1vJAPegcR?Y`4Xd{f5bQkbd&wVxo!$T
z;JkNWx{&edlqy_zYbx88=(@*wIBQRMe;mXnhULPrFl}rM$=l`DQG&<hh>9pIRhEi(
zs54fHhcAxqg~42o4MW<T{CiX~;G?*?nm(HhAJD+nUju;W7~oTVTV3kK33!?{p8;2&
zmydPPR)4C0b9@D;m0~q^_WdB0R*;JE4CpsBPEgV&8xB6&4b-U=%K*;}*ByL!^8eN_
z5m388D`c)n<GPp<OIUqAnV<l!6Wnb+oUoB~l>}-{Hm-&_G_bY90KA;{_|v;!5<?Mw
zkK>Wj%E8m^ofZks^x0o0MZL+4*JHo`9Pk(a_YYOd?~r6l#(DNCEj0a9*|D|K#|im6
z^+<xY?@J4^f=?v9Yu{~eo|U@mLq1b#>DBkj^qpo)ltT+;2i@k>XTSGQ<cag2PVZ}x
z%n&ua2o^}4#BOKxX=dvrJr7*7ud0`#$BN@JZ0388GG1#l&=G7=JeYKqig`cyj6cX8
z1ctDvr3%I7e6xvA1_%TPqF_fHT8;+R$LpCWeOaEZN9Szor(kod@99j!gG^#Bi*jN8
z+OKEA_1$d%YquY_r<}0e;r6T1DmRm#@6;ia2A5sstj)|{pT14J5AAx~zcl=~baNs@
z=W{)9WZPftp}-`>FFR)l+!a!*_Na5zOPhZr5|O!C2X^A${r>;__;EeW90Gw^nFes4
zE!^WY$8(#5p-T^63%?0iyFTKLSQw-5UR|}X3Ss_0wSy!~nq%M1bA0GL!Vx8yq0HeJ
zJemCMW6!8rlZ<@!P}1tglWK+weGoqy1maGeVLUd0Mxp>-Nywn^8?~T53m)^(4QMts
zC+X0qw1Iw6r?HvH8f4e)&ey3@ImH5p$fBlzO<%$_XHw2i3Ejp|LN_7KIAWVi5^m&o
zY;Z*JuSiO8wb}DJfrXTcG&#+kPYUzyMPq#P_N4N<IEsMeHw#}JujzyYJ6lb5(&gU@
z2#6Qy_|>o~Pr?eHKkSm5Xlz|VExpi4Eh8v(vZ+ztJ%6Dud8=Oj`Z(q&EZ_Unj{>W|
zJ)_@GZM^}ssRC~!7dX6oPW4DKHqCX?xeS9yMnhSs8j}2B?w!39`7m!^v5NzY9p59g
zvme8dBxSKcfdngRbE|7po@@F9nnx!Irz`We#b~wEFYmR=SwI?)faMN{N?t$BK?*rY
zRXhP5tBqZBCp*666WsHSO0p#iel)r~iT#28HFmZ7mq9++{1Rz!`v`S}A`{ir5n%@k
zY<f->N9;!8N|I+~d&&lGTW&e~?M~ggz2sQ@@)xl-2=)D)3RY?&sTy5rT+VX8^PbL$
zLUzSn<E~HmfFWyI1$A!-F1SLE=w`c;dI>h>O}#a_@0rKRs!{kn)K(lv%p(^Ni>_<f
zzu6c$O{y=3*0*q8+Wx($y!(li>wS%U06!7%k1d^~#LgpY^^z=ue$!|BFSbaN*d_U?
zP%Uvek++NobTX)l((_+-n=5x@LYG_flf}YE_sT3tXw#Ee09V`}W^q9gs92J6Tgx^E
zNv2vafz545diNnTd^C|L-~p2nE8SbpkJI5}6TiF*cSyu!3;nGA6p*99gE=_e{OkL(
z(EmPF3Xh|SJR>7JV5`r2%sjUj>g^|O1Pf<M8h{3ceP=mqciF50>3)jfyGX12Tds*O
zhCfTrXE_U=6_O_E52L_{Q02P`=FV~sCQ`kB>X{xoUeGo<HU76$=>He)tiwK4@ZM`K
zTR4}HPpkcOFA5Prv2icBf2Q*(JAHZJ#Q<Xveugoa_d9!3*!kI^fJCUNRwYky%4rBA
z-Ih~t(i9~GOx?FO*5hPy?gnP%`6dlL4kXDD{v3jw9ey>?MJl7m8_rI^FP}OQt1f(z
zUtYcef_;=|Lwz1$<q>QGK+9eh0NE>hKUDt$MybC}QSfXK?7@3VaSy)Rhh^q%kp0P&
z5|Ym@Kg2zbf+xYk!gpdq@bB8L!Kx(m-Z00z$*Wz8=Rh#@m%R1sQFfVOz1e#I?%t(-
z%}Jrq*5^}~9=#PSfxjaR@Dsn^&*DI*Dib-=cpbw$2RTaPNdu}Z*;%3Em{9e5if=i*
zi`DoN8#q<khGVP!^2IsL4w{hdz-Qu#<@l6JB39F~%|rsdXy%af4=F8jHioK>>!U1c
zPjD`({G`AjYX6kvPca{$q>H@8Mb1^|&R}ID$AgpBWjaBV_V<*^2^>nQdO5@+e^FHo
zPDBBxPpR!63Ufab&>s!pysy{GgjV0j$}C1Qt8;_`8cOI|#+pv#$xM-vt4cUnhmD1P
zH|e^^q8$1&c1L{#kC8pz%rN3U!VS8TeU@lJxB0>KTcY(p^P)k}=5u%(yDp)_veEje
zwzOkr%@JC?p<n&kD4keJ=n-1st20MD<im9m_2TA<Rs|pdRmU`rOPZ%v1GbvFn9%do
z@!95b^S2xsiSI%M<Wl;&C>vgOADp{q)cZe$N*D>(8!z@sYwDp8T~vJ>w$h6WqJpS6
z7WRd~bgJz9dvG)0(TaR!8-Hc*5Hh#%gccpaKktfY<ZaF_P@4l;7$7xy>pzOlbw0dN
zPVB_<Nlx#iy|^b`HC@8mwCSw*bIteBdiPdd&v$0^I4?on-`IeRBc{6sDg9^OGEVI$
zLgJ>P1|`d@V7ZsR!#J)nNDlpun@N(n{NY4uzv`m`=U3n9i);;`?(Qn(FBs*Hf^k!Q
z41)=&Sg{K>L%q{uDRu(uT&PJzAsl&Oj0(+^T=EhIQC#weFSU5jNP{xGkDCu9>XCvh
z49<Hyk|x^buQ+WjpI3*pJtt7KQ9}TW{%)lDR3|}yr4~JN;6$ANm1}`qNQmquusMVR
zQ({demT{ReP&8c$&4(PCdEZXg%sTcImv0wxvR>lRmFv*(hK$kIPL1_vOn<ldRPC`6
zTDOz1rAN78Sn~AKGkrS%A@M1^jiD=bw;U%MBw`Jd8R|yG|MXN^Cw^y=L<fM8q+l;%
zSE1J1J=t02x*CU`2jTnjE{@`9tV1@nNW$`@Y*zbNc@-Su4-cX4dc4zg?!8Jz@{+Zi
z-Vzg*qUsZttpcgj1)U@)S#~S;cm&kjKminfoao;POdTX`420^1qEW{JSDz4=vL#L2
zj}*9kZhKImPAwToB6gyN0^1;SV<N`0{tG4EW&aH&3Pf?lB7HjaAn?q#C*MmhCWL)L
zyKvUx%RM8hs7u8U&MM~9yBn&lOo76ohP6t$=lR3<#f!~XaQm~V$)0@tTkG=SpagM>
zjFjUIE-!Vc{U0#?57~Z%>@G%9qfQ#=<DCoX>soVhqzLFoF}x#`)qM$;;yDJQy^+#T
z(7qxkBZqcBtE9;3PF%Z>AA90ku74<{=gFwH^J`eYSV1L71xu7~BhZ+WgK!*^wMzIL
zYf{cCwVw1!m~W*sUs#{!u(wC3n?l0!iUKk~B6a?PB&Ry~jl>6n8Rqz}KtIV}PUiLx
zMo|9)M!bTzZN2({OUuHv^Z8x6ID0Y~HwWaDqF$%zCMki{EiPcj{EK1rBQpc9a{>8A
zSR;@&$n{E5V1UZreJgqqX5ZV3)g@8t{@Nq;cJi=c81EoH`EQ!+@AP@kPvoB47xfug
z9A9}PJmkQ&8)D~7oG3tjO*%qNmf^(nTU|a}&M6rho_04$z3O~MArq8Y9Tz4vs65zC
zTI$_s$>Q8S6({gbJ}#(6Kq2k&UcNn7s8|Exd$P`)^o3mf?tR`!-VX$j)MI{0qc8#a
zw99J2!K~7&WpLcLGbTjR#Q1nza*aH5W)KB{6({(*a811%!`VOFS{A5+8Y)|5IP@n!
zNC6C;tc2B1;mx}8VpV`TjG+-XWM77Tv_<`vc*tiC@uTi|Kc&B!`oFyUS<LZ|t&pdS
zlHBolg^GH<wn$%*x+}e>Nso}n>nDsMo67U8Lc+PCy%1+&%%w+*xmr;t^z*3pXVr8Q
zS*qdq$@HbB4#oZlc?&1hu_9IjFBaN=uw-zB)^Nqy`^Bh7g+OSQvUyfC3=KG{SJ?>M
zEwo+_^dRL6L6^0xC1CH4x<Bbu=`ZP2{T~lPFA4#X{ntt?AihoeJc!+B_-~TVrmQ6E
z?}fIAd0RnuK5)NVsqOgB(&Mj6q=Dr3+f+T35<#}b7{1w1Cihj(cUL5*7f0d`bUQ)`
z&!KEpFj|Zp|Mgj&!+t!uhbA2Qa!w~jcRyK~Gj><{+U3X7xe_>sB6*Rn`&3ligL9UZ
zwf2vUko1dilTb8tR=^b#(z&|-MUIA?TWSDkohmaX=y#gx>Iu8c_16K_8z9x>z{koY
z|Io!JfMC-<UzGAE_%PQiqtU?k7d_sR6aW-C=9mcqcjnmph!`$+8Jm9D?DC4ks8-Bu
z_|{SEWFSYB8aDa(pR;@e<DD`^LXvr2s0jBfk!nE25?IeciyGBVH6^moyk{KlK5mZA
zb3TIqfYp2YJ0+paakfm=zq|lOb&1V+6d=o5YF)hYxL(ETjaqrjv}s>dAE~FjCaJ1+
zH$AwTc5+B0&lGTsQ{XSFYlU*Iw8Gf5c|TL;s1`<Ml?qiC3b#R^B7kMjf70Xs%lDo7
ziQAO^Pgw1X#Yg-XI!z5KGyBNlT%f2tiP&^0F)$8yOZ)DpEp<481yA&rY6x6@2%`Ml
z$GYP;VaDZNw&{|dFPtq*XTKcXDpOBdIhzbhD~E!^HvRkLs7d%tL$Y8EayR1~s{VN0
zpga+RHeDUwZ^h?F{CWEB;55u9so3uOr^;_sfG?A$jzCgcO8zT<dHy|rRkX^;>hD>j
zkhyJJFB%U2a}>IS|E8mPMIST1B|m2hmcq#-5oVe&R%rlPfS5L0VG_RXq+NMtIM@DH
zo!Ye@gXkdP&ZBJb9)!vGcW%NQ;q4IKu6(l|)jkmstuU+DZ!$)us+zde%P39^`XC4?
z<slkVMv#&Ha;7@CI5GK+VFbejed6k9LYWK-Np~DKf0#h2`92nS#N6(WV#xS+h$V9i
z4&F*r%><BB`2qw>Q=QxT$HrBX)Y=_v1n5+WC1bO?W%2{A8DgjTFePqZ38b8a8sMv3
z449|@q7nt<oCX@6#cPbW7?PE?g&%BUgPq4;O>p(CzwRW&P}*wgU7|&_3))+0YeQ-x
zq!nV^wo$%gXX*NkdeUwt!Ug}&jyP1#bK15$A@yfWiyF2UNfZ6oP1$d(J>>dTTCzVG
zkN>|OtGm~8@ZVmKh{9bJ9y|7eL*>m5j!po+?|qZNPLlczjVh$hOd>addq-$5!KD<I
z)gfBDP}=G54|2_-gM`m`e=O@rOn*_ASY7O49ius})r-x1+`TxRv}*MjlWz)h=^Sm<
z(OmhJ3^hdJu!?n6`P`p-Se+U(9C8$&F($&Zy5^7H-KVZP3O!~<Bmvbz_E>A)U!c$O
zchKiR5E4y_rrcqELRBl|ZS1MXe}Q%3yUrpU1amJ$gp%RX`o1R;J4{@;kH3M>H$*dS
zj{13Rsl(|hhRxux`j&C}DhcBt3cuW;1!R1ntyiHi9abs>oc-iDG{!RHoNqbFgaRo=
zno}H0DEYk511;sa=;T17rEd!0HtB)32vdCfaFeQ#M5*`o-4D(R`SbWMS?GQT8<s5N
zA4Q?@zaJYwn|t{k<^LLNkf#ly=Y{kL_)qT=b;a3i^$2ud-Vl}%=H+bcX;KFb<=Uv^
zz*Qv429zU=syE53U;V4FLm3EcCh{Xjb*LqUUneUx<EOB=I^92ffYQ2nu+hHWpOMUo
zq}0h^o~%q`qjCsPc(h1~JY9QY`!beAysH6*NC7J%89n~;Z^KyNrI^+ZRyQWXh(CO7
z@9%u=e@#<LIASgW?xk=4xgJToLuvxU(TWH0$A$DIts}^a!3={a)!iY|<i21et+cSm
zh`v&qqmTJ@;~dWT?h;p>@Qx9hC{$GxiF?qO&fZ;C*Q*6CSL6OWn6nR=GL%zP%cJ!}
z+L_Mp<+yJ?4vNs~KcF}-U|#GEWH{qbw3qJ47bA9BE>F(!kfNdy1ROpnRaI3xe^SNy
z{sn8){~zB^Rx8B2AqKB7lRXL(mDV3b{G<8^bQaRDyh(PL3(4dzN0*?qIYJL#DPNB3
z_7&hw&KC~NDlH%9DCRd3p3)BJ60x$@#VKMT&wo3j!=Og)JA#7r65JzHs+ZNB#Q5M_
zaedR%K<Xz(^0@o@1n%X~^lnNBSGPyfB6|Kw5))8lpuqf<J8u54N1Xo(ZLN2K_Upf(
zT5BW`(C_RC*|1#wR3UJW_9XM|jX6lD4>b+>0@<?r!8$V2kmBGI<yN5!B+EfL87UgC
ztE34AZGhB_FcB&Y(iBUlI%p}UWD0kF*Jd!Bj85LE1bV_eIw>anHj3U*{Uoj=3EujR
z?0q(E;lg+=a`n{d<*r`ozRNRS2SW7-A{man>Q@03$`9iK4hX1{_ecM4VDQh!|2M(6
z`Tb?at0vtGzbMN+!F^9e{3;VIlT%g7lFTN;Dx8WD-^=$srKOftq$wY3={<$;DvYWh
zi7??%)j@|;AtxMKGS?{CJ`2~*DNM&|O`8Oos-m8dA1oG(4M<57lYv0DSD(;6RMXIu
z_s1C<@w;W4MIKX1a@mmo!Q5MhMY*;8!=#8v3qyl~Ad({u!+;{8l!%Jt5Gq4Uhcwb9
zA|W*hN=Pe6hr-NAr+|Re(A^#HHQ>JQ9s1nQdp}?P$Ke;;`-7}?t#h5{uhz|1el>(F
z#5c4APoW^^?q5Jbfk9=z#wt!)8+a)8w&cvL=2v;_ah1xc(8GZMP9~VLZ~u}gfYZCE
zN<ogzHV7r05^y^l4{EF(m{d%IuBvJ;6DEgKK7@h8!0L|Xu@+YX^q56Col(V<qVGf5
zBSo`AjhLcgiLdH>H%o|&FSNVwUfwX|$Ml3z7MR@UAkZKi_vV(d{~8Pi)o$HHfQcbb
zb_gPGru(R#;>N1QfHLBsy5eV}5_E#;UA&b46WIAH?tR*P4qU>vWZY&2B%0bW=i0Yi
zw;}^@Ft2Niaj2ekWC~{N^zfcXY>;-5Tmh9hoav&SNTV3&;sFF28Lqipnx0#}uxE{j
zVUAzd2qlHVBva$X*)96MkNf~-9!-vG7^NR<v4{622>3Ba{vrdlp1D4G$(n|c)_lej
zO0-V=oWAB@0i64V@D};$R&4(YmHUrdfz@+m9_D)(wNt_G0N7D4PB@O0d3ZKotuQ{7
z#T`s^r-JhfK+k2Qa)Tl-XE!&dT`SQg%r-2ceCC31Y{H_@Wi!}D?3F3A^$J-9Ga=NI
zb=5I6nlqb*y>@L9bSrm!MfvR;iLj3Wd)e9zf<z`NX%wyRP#4k<?~p4!UU*JS{e)Nh
z!*e=azdsNp{9m8%-yp|W$ld4D$(&<8Fkf}A_w5&ZPXmA7-$`xmI;T=g(Q!pE{aI?W
z7$~y-Y3p{<mMqyKjaOw@16FE75d2B>i$aO^x5oPo-KNoJ`RvZ_y7S+p2+CB22nI(#
zlq=8XrYmXz-I+9Ik@uYZijnw?|97Fyf3+jP;n)bFC18nf072#z@n3{qqpJ@DtFiL=
zOuc<Fh7P=gpU(Hg&u*-JI$9Ht%&u{zTQN{JkM01A_eR53TZhRGIIf%Kx*6^1xomQ-
z8cAF*7IaTQ6Kd?ey@Zpk{iS_yH-*xiLY<$@a3bo3UxLeK31Z@^`^Mzp<&y2!;<n>|
zl4M{R71ICqWHe$=MpPYsH7)~A16_Raneqr(V-g2ENhM#`uB(DX(dr@txKqZ(!RhMO
z13N*j7~wcNn#Eg0+7t*z`&UBSHl?j^+u}u0dioHkC>}>x4gUkry|DQ&-_-Lj8Q5;D
zS1~B!T>5riqWFx}B*DuquCI;vLm+fqv?hA{vxtGD6CfW)^>5AnW*+gzOeZ@TVq-2|
zcyp!Sq+ks2_1Dh)$aKzyRYGGSMZ!xpi$`S#<1&`goh&D`(bH3a+i&D6f8n8#oEws~
zcBo+Io$cb@wDGsO>2ZKjLQ71GAji2j$vG?P{P(E-#*_X0G4(U?X=mM{4phE9s^IAZ
zINn=krPlH9Ui?#EO9&d$RXZT&=NIcx_udM~7RWMX?9LFc<h0Xy2ovR^#Z7eNbO?6e
znsg*^7q2UHwJ|UnQTI%YU7E4S?b&cFsxaKs<4B#*RAnKC#;dClFo*Fd%yVd!lS#Y|
z1*@08meWP)?d4vSSr1}Id*4$4bd13_{C@UujAkd$-DWJo@UBm{@mixI;ibq=mSa`O
zRiwHnyQuZ(vfT-N>OgwB7=2}kaCaJv;%4}38Q0kdaVze7;R|m+vIGZ$Ec?w{e#wVn
z&uX%IDj6Y5<mGb~nq$|=6L4lJ0a9$nkAao*lxmIkybchp>GW*BW?C5~345DNG)oVA
zx1#f{raS#Vif8C}uGvLxfN|gCwa(utJFAalTXDshMlRf}s>B1+tv9ZXs4-sc%pc|f
z$E&Tx1Mp#jkMNxO&-<(6^#0-_cAp(;Zx|L}))l9;ffUU&x_p?;rYo9Wq5HU<HS<l`
ztMw+tVI<wQ?RfV=Gr<yebdtrV?l5I1fn0K0?33AI5aB;V#=wMsclKKjAavYS(Eu>4
zFh>i=Cd?-rm)awI<85gLRL741#oD7olQ@8CL~YIk@3_(wt-YQsa_Ky9VL|#@@!<KL
z*$5MGcPOB{R;U&BBgf_*YJ<;Y$?K6v)vR1(tcTdIKdm|f^OGq@lPZ`nJ6BD<eg7Sx
zsS*Dd7rO_i&DwmY4~_f8LIutTe+$db(`(RJtIksAo!1ab+14!G00)49$sm|HVdF+2
zz{58AsUNv(263;Ev}54*KC)C-^sC&>vIwrMo?RC-*0B)cl;QF{d&#rah{*>XA56{F
zW@&D2+|rv$@DmA<Jw-xFIabEnkL=GgLWMndF(W8;@)iq;csJIgAsMtq1f+Py#?48y
zYP%KDe-(A|)xhroUsz8vAq)E!6EhKu9ehF5(j~EUHHv5{e1Ex|UDT@URwQ42q||B;
zRwJNndf(=-q6Et}A*mw;vKVYj6Y3O4N)+8UlAm2za%4;iCC+TgBeNa(_3*BedIk#_
z+upcQ7iU$KMwo}&zTe3<Q!bNfxl)CIW`Nxp>f47P5u!y6chZ^J0*!^+=rTNOA{y9t
z^^9N3RrTI$1O6k1`5KiYN?0BAp0soNQWiHIWAoq+L853PXTS}Bq?IYZeLq|cNQq18
z?Pl1ZkxQf_zJB*39An}!4hO=Y3_7nttJ5A={0dodPO5A7JWgJt;YWCmfI_LEk|{_b
z>_?D#1b|B#&a^I7dn+c!8~wK{t=rZftD`nQ9(9;6Fb>-(Y%EqP=a_tJ1NfG6YJ^2r
ziFF;HtH*nrw*>ISHy#~lZ(B+Av;VHklsc80WRM`@&K1BL3F*w!No28{?6DgOvo~;?
zx3DXq#65}$CN2^MILYn-B9m!w%jJ|CjLjva#U-BLcuxvtGWuuD`)f9(y}pB_x#vdH
zioyEcHwbQ*e~+)rdJ>if9%u@Tj210Z8m9?ly%=z?p&bfz7QW<~2$!kJb(&1Q-1^-*
zOgAx0AGTLZ`&X~F5OIq9jvo1806pLW#lh#pGw6?rh;T>7z7nbSI(;J4)U>EGT&*(N
z*{=@jvELBswH|(V)OIQxan4}U7z?riAlixIB<n({{aBUa%b?}ehfxNZA8auJbdX1-
z;uAClX13SRht4A`<U`JV4@$k}xNUgcs{;I)EP8W8NauUyNA#U;6&x}!6ICC?bXJPh
zHob-Eq<HaXz-(_>_Sebq7FvbY`3aF-4J4D;J?bxvf^}N{Zn;Z1u&izk9OxfUSlzz3
z+g(iF>Yj7v$$0jYnF=Pd;cqS*GKW>oX_|anztBu)Wk&ZZl4<8&bRc+9LS)cp;O4%m
zWpEGY3S5ZcF-G=_dZ(nTE-0h_!UI$%Dn4@+!fLYdxM4-ryhaV$rgN=?J~OZU@a`a&
zOVC~XzMnLOu@;R>)#`nluymM{3hs7J?a^&W_1>)b(-xmqK-CyrRF9@TQcV;?E=CTs
zOsnpDcI1CD={ktM%d+rA8`N+Zx)kZ5xCmN?>@lytjM{&734dyMuOm-BWsYfshaL8E
zs!GV9qE9(pt9|NtBG1doW2koJXy>SS8Mzm(%W|dKZKQsDqYq*5wj9$bH6GV@oL2Jc
zolPIK5(Nk_L*<8&%^-AX{=pN8UK&OlxBXWx^OXt9jS2THFn%`2mReORuxOwxi(SUD
zj@qL>-hMD;ma1c$BBR6DTGma)G$5b#^Sl;Nru6_NQPT#GK<86!_Tm2CdEK)N!Y=Tp
z_1J4i0`^B2A_bDPwtn;{)ZurN-$r;O0HBn=R!RSFORq8(VA7-tN8cTTO<fA>04epj
z>gwPU%XB@>)?!l|ARPEfNSkGe|Bwq*7mTE1L3Q>X%!x{H0~6VoV(Ve92zOl>rF%W1
z=A2Hbc(lUW?gm)@Av6U|9A-y>kgHP<o66!m6Lf7&2&8%53MrTm>7ldLQHM^>AW*0b
zYRHp40`fZ|`9E36K>hoptKT(cK!IU0Z@qzb$WxN@v2nQhc#sU#^D{amT4b$AD8fy#
zYAnX)ie8Xx0QqqG1=xgbF$h{Bc(gf<$Z-PUMR`HiSWDNMi=65QcD@{YUZfT9s(KpE
zBzdi7wY~LH_;fe@jA`=rk*NB<_~J_Ucx!=~r>w~0mf32k3@o=tKY!Sw<?GIz>oIM$
zo5*p=N)Cx9W4EWhRa~VS>;<q-+ducIGo`Y&Z^m{tfjxppuAt&zy2w>C(5p6tJ<>yN
zp$7f+QhPo55aad`@Zec@cpmY?(IOG*XTRQy)D!wIsy2;~6o*hwMwXG(yXI^35>6DM
z=iHF68sEi$HF%gyo}c<2US3RJv1{F1#~v+y^RVhW@29WzUSz*zs+#y!?8}*Q94p?D
zLIRrHObK&pt~Z1x|DwX>J@e$$ep2BgP-ItMQkAseE?PBS&<OPscu~3=cqjVj1^9W1
zZ6#jFS&hlFZj2CqlbT#+_0EFm(}z@+=LN=Kw$01qA0*;pJQ&LBFiX46alVvxLzOGT
zaJLHX2*gu&aHfb4T7!s{*);j-gV*byk>n`S?bZnWy#*>RR;Sx5At1}6uHDsT(6KJP
za~nNOEz|Q}$WctG0e#$X*=DWElS0M4kx1woX7mF2*2ZVn6y}`10{(S}cM8Y2hvS1X
zQd~}prOLM2BrMM$M;Lb4{O!0I3oy1Wo-{QBNA-emB}osHx!7(ofOc~Q=(i7xxAMYj
zma-bbb_HPM*D6JpAgN=?%$O^cH|y}oqn=v*8W7u*OI0IeVQLM1flula_URl0UdcOw
zubbyukda5D-+@k7sr3uO;%vMk!o_^E<P7w74Ja>WNFI_v6#kL6d`6kXM*tCBF(^j6
z7&f`SrrMbS7b+K*12sQ{$}9_M-3e}==E;<O;rE3J;2u2&TBb|*n&iU48Rb0_NHmX*
z_}>|e24Gt##xhJ3#(CmA%3btHY7SnJ8+=?-A90Pm5h@9Cfg8>?y<44BZV!H(6HAki
z@H!YDzt8s#FY>5zBXS+O;y3dn^cbRWur)Q(Fe#>QP`um8#ms#CW7|vM%u>h`zAkPS
zg!-50z|gJc?Gy@xYqM#HXn~#U@phZa@#hJd*L;gX_>n;$Gd&BRu1Ut(=&g?M;a4aX
zXe}G8wvyCRR4zh`%hE)0`cRp}`~D9rpXh_iZ@>buz=ZH=!#Qw74R*7<Y~Zm2mhS*`
zu^=r2Cyk2^md_qEcS1e(P=NZI++fo8aB(XU(;2vALqB0af@j~&qvN)wtNkI{6iOAz
zKh!a_s{U&=mY`3<3A{8xU~F{N;_ldY=nc|uaGV{dU?PL7+1d;P_{i=jEF~Lw=E};c
zOg#<beGEO2RCd-qys2?^G83|lq2l#jS2g2e(D;Cr%0gICJbwh!?h}sGx=fO^a5I+0
zIoFD(b8{Y(9(E!!--kZryZgR(T=~eVJ>1r3;x2pjbaJD}N9AswO0a5|E}d#e7ks^f
zHyJ5OMF(FT$&K%`imU$+-zQ>)+u|vd0E|WX0b-*9bQAOe@B*ma;&Mnj+AF0oQclsP
zLhLniiM-2Ij~Bl@EIs=}-pfhyJLZ61h8Il{cz$Wb{R^5rBN-&tR$auTiP2r&cUp)t
zVD$4?N^3`@Jq{J_nRRRbu@xf5x|KoJ?!}+XX7@Rl)n+pxk&NqoLPxq{<ah~hI(kt%
zEEiUK5o+HXH~xB(`8aOie)bO$?1`#i2W})<mtqeK1KWN|aLorB%8fq+?9Xm|n;E@W
ziNwl<&(llh(p~4U+g$dfX}6yEmiozxvTQyCKW!wxwKm~!#2yi(obTzT`)2D8e(~?T
z+5*Q3IY7hr5B&p`y&$JP-T>|%ofVC7CuK(;WS3(~Ne=1aSOY0FE|xK)ooRl^{x|gl
z!HN3e8{voDDVxQFH^9Ax+XeZle@Hmy))RtEO`Iwq@XlKGYW@!j2xDgomzBMjUWdEW
zIyAEG$GOYs7r3ola!K~Halio0dc{VGe}xUrC2(-_sjs5-i04Ror`RFSC=qRIxV@K`
zwZsPOcw?kUi%QuLFb$=}+oHqlRKlwj7not5XswW?A@npYmmhqgK~6o)ndz`Ct+|mM
zanzV8e6MFeVn6_T{HYzZh>nwHK~!h$Hbl~1Po81gdnt#BwlOb%@T}W?=71d4R3ebF
zf@h14Ydcg48!=TW@(*VeMb&z)<P<sFTA$@!{1Jya>AW>gY2CpZ6Z|t|A&p$g56%p&
z#xa^4rIW`V_A&%kd*0ocml1T^TA)89I)7em`NhShve`ZD)kobJyYk^PKJ=g};j1PI
z-=#M<GU7Lu0*&iYWXA@T;R{Z`+gOJHENZHz)K|zx6Je<M)aMqc4s>g$r_W^*q*C?L
zpY8yT-@$#SgE5sp70xRX3O09l<sTQcy8_F1P0g#O$aGpV&F^K;kjr>}GU3toqOvK~
z;Pz|O`Juhy_S`%6@TOpO`!eyVs7Tu|uzSxA81OoH1Lh_L3(98q^Q@nguRa4H1H#aN
zqc^7{mZ%ev4?5EH=t_R&bj5?~UW-4D=}nw$Xer#c$wz{9S*qTJ0p=&Ip|#2~%eKdp
z{aXwm&ke55Qs;F!3LyL79g)(v-j9my<D2|Dlo9UZzL9Yks|F^~tTn0b$Y!AEj+hKb
zuQ#n8iyt1NaMQ`J>`fbNF77nfdvT+k-d=BuM63m?y_3cH=!Bov0zw}h&a*}yk`HGp
zeKPW(a(18LS?zf_DWGO&dUB~ra-1GPPkRdZFXZK>!(vsFVKakqu!}kabN6{L0Wzwg
zoAU-$wUs*g9VKpoEJHG$o8i^nrg2mAI;CUfWs4~#tNm%={v8;xJ_t3rz>0-#MX&z$
zw_mC@d!(>Bg{K#wIyl035VrJ{Ay^t&jnT_Uap{}Up0f5??=E&Exu>}IeB0HQ5(tY%
zJUnMILs1-SN15rGtsOu9%|;8r!srsa6ut+ob@WbU+1HK>w*jFKAm0vOmmZ;{p3mY?
z6d1K)6!x)ZYTBX`1^gdd84>^yIplYj9znp;osf|5_vU2B!7cB0cei@+IZ5EHd6MJ(
zU6-BUz~(Z&QQP6-Wi;STZ7Nz6;Xi5+_yp=QW-@Bfp02L-Uv6nMMS%qp@9i9kF6Zw6
zsbU7jfuu~dBd*8J01=(CTSwkz4ldwvXgk`d#H`#Z;OF=6`*1K%R7=$yyLV+L|EY}!
zDKG{Na-Qx;+f2uXd91BKRf2!C*b_Zhvs_ra%o+G#Mv}YR!axCxigTtfl5aIgio%s`
ztI!Hjmr}pqQ0w*d5!2>-pUHV0;rSdH%Mip-?+umJ-?bo8ekTn5TZ@#qbI0eD2Jjs4
zX9xvTHoG%c9y2@Ui;OZib%_b80Tk-m!{?VKjP(?lpyp=eYtj^f2wS@%GAm<;4gy!2
zXbYNGVx*-b@jcom`R5YvC%RW$82|wKayk|0R40lKU3Nu9;*}OiInT9;J+<lGZjZMn
zlSHE;;${0hVh_{Bqm#eNqw^;xW>F;x#B-}CGz~iuC3)-|kRwA%c$yeLu;WES1h_17
z4QeY|26ZsABJGx!Z_p<gFt1l{oTCrZl!flpe+-4N#sg+z*N!Ov<1R*la2}iLG<fYp
z8Mjx*uWh_g)4B)D14piT@-<^UnDNEsXZGM?*L8dUvh>+iE?tt66E#D{1wf_uq0VG}
zu;TbLHKs1EI*qB!bt@OHQ&qT7TH0A~>`hnR7w_`DK}&IY7~a_}z?&$BZjm_{;Bqa-
zAI}_?x^y(lSBd8L*yI_Dy#>347l{CTF@!<7zn*jpsn7Og*ZHyAzoYiX0|Yyo%?=5H
z08E8{M>A+z`3mYCC{@5#(VQ0Jig-A6*CGYTd}58={+98eYU*zD+h=!b^-Ers@`#gk
zA>Th>5#_jiyXMv|;hpSwMY1rkN>SbDpvI(8SU)7CBnUH37$jky`NXW@WzKG}g?-qX
zhJDT2Psz@o-Bpzz>oBQBMCjV&EgK{$e32j=Aoh~BeOMa+d?6Hwo;+kfrjDu&d5mK2
z#=K<Y4`%h)LHl2Myl$U}IUJD@61Be3u`}NPMnFOsx|^Zyc$J>(#lj4a4lrj@j8yON
zbmlm(!*EBS(~7qHtzPpzo{&Z?6vYsJ!ZCHOwJlb+AAwiz2gbDl?KIbv^qTDVWfecI
zPb!`mv(8NQ_4yv!mS^JDwU3=l?>Z(*m?lFZ(8C(a$xUd*Q#tp^K;`k?+B|N9&bwq~
zzY$Dx7ItGf5=?``*g6)H*Y|NCqO$BzBw<U>6GvVe7`;wu(vq&A<=q%)Ei19w9oyzj
z7t(tPglXxd+kK^J8%8Myo9|wF#qFxdWf9%d&H0_c$|@xuQY)G~G)b7%$~37#dY)ep
zrYx~y5^_Jx90k{WZ!*K9W@WCnLWclQ*D>lp8<2|t@nf6xx>V7A>W@&yr;9)OQ2rnJ
z;kH76-`KX7m^CP!4Ep}7GuF(B9rr_r78e)9%xibNZ1&v&4Sj|w=6$UmeP+#4p1jwV
z$46`@XKC}^3Aq?lhxutVE@EGjv2P2+JPu^Jv#^;WHHGRsPTwVncCs2i$u{U1J(X(n
ziP`n^S7Qq6xnd&9w}<KNm&?-^mX%C~h6TdqVD^uqj5YV9wzlJ7RqV4B2SbcrC9nl>
z*hmyRi`Uxs_>KI)iUW~-ykE0~sl!;jAM?Dy!r<yIk4^{35PnX_VtRmZJg7692~H?X
zo{Zx;OB7Tec<cFf)u=7mc}hi=+=}FBvj}+GbtczWx>#|OPU4`o;eA<Xq`c$j=bfnj
z3xI2UX1z7}AX__ufH90%#gWCgB;WXz?)t2SO8FW@lJdDY&D!{KTvPn1xd`_qO5`U+
zSSWQ4;&Qn;aAYM)+;XJ;+)mQ-)z0HxrChzfyGeyW(b1?^K=+NTZ>~)pZ44k9ax0(2
z999)gcSxJ`6l&P+r<YDIeMlzJxNU0gzJNZaYJR}itS>dR6djS{_2WqQD8@mNs_peN
z6U%{?k-X&6<K0a!bpD}u7Wa51teWk}4wIa$KJ^+~89EG4*MEvp%(%VNcBk_|t=}?I
z8X~q-vB>G@T*d<6F@SGLw+W)9f3)6TWrjXt;fZ$qjPmdj7p(3LfVEzNu5Y3-&R#=n
zI+gq`<ux3?o@U@@PmTDgt?G|euTtGiB^*^2bHJmr0NP<+VZ4TwNor5?DUAz>349wQ
zYZehx8bahU3J6DSR+#|u^jJ?Hx>h6grs#xZq093#mVEiDSIIqYGFjdO>rsGGNjl?G
zGu}{fx~{p-QOl9-O?0|tYfpWnGbZi5=Q%*-q-6Lde(l|p&Z$O(`ii&0zu1=V%mge#
z%T=69x-KxF#!GIoR(mmRuVG9|eJW!{OE1^-Q~D4!w*QQZn3NP+il?-nY716OWC^B}
zQzXc=SCQUmpsPjixWHHaR*}jz4^dO$w*pl{wwb-2EQJth9>jg39Jy5ZSrT+ORyi3V
z@i~1bF@3}GN4)g>RM~8Msg<;0<jF|Mc2fV@jkm|x$Ae7N!&syfyzW0N9xd9LSM96L
za*h#+fGoYeHvO%3|Gwo=Va-Bi=gSjZY&-qCbB`2DNJeIg75(UM6n^n_YFJAxAct9l
z1B0DUFB`~dMILeF7_)&lgD+@qMiQ_1rkx3uP|IPtm1zx*ZU2mB5<ZiA9kOmVU4d`>
zaNpmiyy-+#2G|g$aI5j`r5@?Lm5aVBnAoR4AX`o#_w<#nhdJ_E1{69_H+rKib#BNP
zFoVSCJ~QTr28+K+get(~ZU3z*BfR-9?}iaDlN8JE09R))aCLt8_JWzlWn<}^RE+~l
z0&Klg_F#Rw4G6@L?yk=xf!Y~8tq53Xj!U7oXH~}w;(t<Jmo6RWRL^P?e7rk_TqJf3
z!>(tKG*<P#gco^HE)WUkO419F0yxDh)_|rt#VZ9yWB`$RG;UPXenbQ!7PA><(mLEr
z0|~f-T3(SrIFqi4^;&f-SL?4Bd))f<{n^2WWh=lEM!NA$1E7^ve-`;K=YqNHWZul+
zzLgq<(Ev}zthpeSq+}tsP00mvx(xn3ylaBwG8qX6O^S@+zH5ILv!P1Bt$j}@;DwZ^
zb&{6pj@S<YqA2$%Gxe6ZH9}YlLX?6IJ;PI*a_^c9$U#ue(>!D<N5BUSxRR-o{}m@u
zR0(JZ1d>^0j=SZpLO71bR=k45s;63!q>?rR+o9cJl5VALZ;uzMpLJDY`r}wZRSX#e
zz19yI|FFduIZSrVjeR(7OCO0*+aG<DbH$+f%e;0)n&Z7u+d~mCbY%OXyw=Szi)*;l
zGXX%|z>4RA+m$Lhql(*Y)BH%v{(IMD+j>L>mn~84wc_@x`%Dym0q4UNSu1276qp~1
zTuMhRF$Xe3vsHE)(spf|!}|ax?GvXVz}%`VVCRE>{_)YF1_VPyTrMrj{#wO`?%7;M
z1nmcp%Y)=9@WX`yMF+g+c6S7myRPf5uXjtvXYMvAeUt^XegkwO_JV5~fCz8?1-EM;
z=i|?3dHb{0vN)f&PMKd2Z7!=bJC8W%*^Ry(uEE3VN<aiA5(|-N5Defpz@tRbr=P7t
z1wF2^xmuNm{EH+FJ3wQHWxR_>EWOd4gNWeLg3z}+j$ZE!wG9;a!2GmlN2JWUuVayN
z0J$)36jC+_+h{v&3pxmYGDa$rW3T8w3@%On&?_IG9d!u*w2&fY9sXF|YIlDlHLmvB
zmek55^3{Ewl;FwcI8?%Y9uo|T6l_?RN+)5i0yx<w?^_WZnW1Qk0HDK)uX_>Al4N_Q
zar8y>awI9E)Pa<h#0crH78MJIfC5i@@tg$2IXv=|#p%80J$!pTU6eB<oHaNt@-)TV
z6REM*7ez=~eSkKd9NsLf?nY=vb3u+}Ah$q(c{V2;!E~*a9Tun>EhQP{dI8tacGtmZ
zV^Y19)|+FTLb&yEglNXDp5Q8;BCBSec(C|G>eVxzQxuAcB|&fU5*|)Y6GYnH8pXCG
zK?*-JQJjmU&H_zUx4I%Q5B1O%|6$<qS)@31@Q5Fg+JyV~ov{hL*F2^>eZWl%a0HU}
zK8mxJ>9&KdedsT{k;Rl_Z=&j-p%-8aTE6`@O&*U2f4d3;Scr|N0IvP2Y409|mf;Kz
zN5hLJK025NB}G|AeVo|;LM2Ek8s3ST&<dN%R%c?=g4nOo?Og+Ydp{SX89&B1Hd49c
zIt%haAAVPV8J4ewo10&1`v+h92LSQ&Q(19~1u7Zwt;#v9Ybygu(%f%U85!h3?xMX=
zG@Uh*tky9QyQ&jgxm+dEo3CA+N3joMTgmfcrsiVq=B?7MTq=*>nJRx`@@s6E^9~Wg
z+?`RdjM4WO=PK}m1v@S(v1#Z7<`hI()mv+@y5yNXDvMD-g%nx$kVRR|j1*3t?wXKJ
zx5uE|+NHA|<7Xwp@dD2CMAzg*V;I{lG?PP-Ng@l!#o>)KNwT?>4W2q1VrRi$aCa=A
zBaTuI(kExYi^#M^#fQ%7Y9m)$$N$h6s{c<KLx4{L_>TuaWU61jgB{~`7QkwO93)e>
zAI%vLtKA;wYFA$swdpF566iJ8=W4GzlE!$|Y6s(HvxvUc+8DfFyCy&6)IP9-R&Sk`
z1kLmOrskaCJXQHO0lGd_`9SHJ*D^3c+m}R=_#5~=VHzFyN=bk`H`d!0aBw7q26&6_
z?SS*-a@V>tQg5+EBuMD^iBD$4ie9%5dY5;jDxY0s&{XQuiebKJz4^e4gPk%#eL)Xn
zyKZ@mX0F{`o=pA}@sh^SD8HB6t0og{DGclZ4^=Eao_QR1@9M2l@DIE{bFRNp0a?f!
z9%R*k*SQ3KnD3G@A|g#j*6;A@Tidogc)IY5vWl$$4A|%ybkB1`gRAp*qbTr=p5LT}
z<HD7xUnI!uVi?j<L2|5YMgapoz&^J&S|BOe7jdmNCru&%_a+k;*^e86NM;TVy8&jf
z=odY!dx@8c7Uyh@OKuGW@#AD{IvO=x0fBBN8NZ5qs3PJS(!l7NC?56@MHp2zp+&X_
z__EIGfBs3)wB}!35nt_~Ho5WG7<ZefyDb!mE&BWM2P7V!@SEzroblKLw<CDk2f0M8
zJfRPAhndu+tGV+lA3Gjwh&7K(c!qxbwW1#A3J6cFdUpV;9+ZYyYV>)8?6Gms^ik4;
z&l_Sx-h$n&TjLbrW)RXC2156S+sd-Kwk^&3x!kSzIt0C1=S^BPlM^NxLN00rhBqdI
zt)7R^jekv7oinCGPxNtKE;RI5^r1rv!93Tf+x1Y{^y~)c_k#Sa;0uOL?M!yY4P$*N
zGeVre#SM5B;s5q30xY41?)!gvk0jMj2=N(lsrFy*9Cq`TlupRbF6oEyP4Dlyd}{yb
zB}_oqe(;X)MZX`^A8TYV+Ieg9!a%k>>z&YxPi!Fnjwg_IRWg<Jhw2#z4VXRyI~fz^
z5AHr>sx7|d;#ODf<Ji8Z0+u4AIg3fds1}e;*y#`ac#~c>M!jV8?z~%8Ok_g2j+jWO
zc<+~c%LxOX^2$DMi@at|9hZ^+a9jc#cC(@6@6h<afjL>-)0(sByh!paR@RS!)Vuf$
zoB*zAp<=cBj{e0%KL$jYXoVky1#!+~A~yf}(Zto>*6AEo2+BZkF`9{i=gh#m+^qrX
zM6{rKVjtl(wM4LJv%H1o4)EChgRq09I2cD!NWDN*z?5Jc1CYHzS<e&(4!$8A$DkRu
zB`3`mQ$)?F5?}PFk;>TeB>L$8q_fL5D_Fm)9eu8S*OlfR18xIl&kSA%^&KJp&gaH!
z5)B$bQNwH^%0GP677UZb?F=U}1Oqn#-T1g53h;Tg3=Z(v4&X}^YaJ|JqjE;YD^ha&
zEo1EK#fkNDitB7CqN{ZrYR06K85xER*I=?Sp{^HpW7s373CMwIhVO5t`Cqw$27V*K
z!KzvKuwz;WVB>W$q=K#FvXaX3HyNI1J&~9sL?`&OvfdyK`{Y30<S&CzS0fhxdbwVm
zl^L(Zctr^2VXoqd6niGM5)BxE=KN>v$G#;t1Ww)w`n+Ps0=hXH9fI@r39*dL^fQI5
z>yUD9?<mC^Wai`F`aECetaQfN#PtlD0y)T7jFhfa<O=TTv@G^h8}_piKef64M?vi$
z+U7s<gG>y&7$zj~g84d#XV{3RZspW5sX)KMiQ+>$IU}IuW*BzS$ZkF3d&OMLkFL90
z1KB}CwLe5pvx6L0clyu15ZtSHxH~J9PogDL9^!)RJPi-JCp6K)P#i<8G=N~^(gM<5
zk{*WHE@pa}+n;#S@P+?1=KRW{4tVZ<!`7Y*=O?bwm5|uo>|AyEOzV{-R#op}{A_K=
zemjz>Jl5N#m^eW({0(z|Jm9aMp$V!NW|hYE+@w#BobukJGgYxr3S~n$oYxWZcN$h2
z&^BSW^i6>15qR)>Cjcr=kP0G{;FbIPM)uD{>y5#wezr(>^2F7?i(K~LDQ+=)cyqim
zLS({}#dd=3MH9R>?Xt#yayI=3dGMP$Oi}SjN%l$$@QZAmg)A#>Tn{IYmSty^OZUE2
zn@Z<i+<Bft;Io1`Ep(Hn-aSuM1xi%ArI-gcUsTlzfQSO1uhI~Gx4-?Hc=O%HX8A5O
z!Shc*hUM5L2i%6)w4nRvMKV@<x5Cx!gH1j5LXe7;ZUqg7rv@|lnXW9EpH$}cU)ux!
z<H3l3iFNRP%bszq6()l0D}vanajnb<gJOV|QPRhq-)bC;63Ivam&AmM1N`}BaLn1~
zt5t{!B;e38YMFmwW*o>_O_57FU9vVBEi-b#IOO2Vy7++J`4<GYK>{#_tGKrzUm#)e
zy#ry5?ZIrmK`?td#ZmAR;8X}D0;l5R|NT^`DLvDF0Z7TNThy(43ZHdbYUgHzEN4an
z;}B{^NGvj4l^5ItC;Iw^E?g)e8&Tl*#HZ?iA+h_!A1}UkrSzcTY}kJ%u=`WPdt%l6
z&wG)uB`H8-(D{-?utx#ptt6L#Yw+EdO!lyccZ!nTBnhg6xAp>GPKwO~I1=l0Q&a-d
zH_T%bh)PPJ9$^pdzV_=5$9YwX1WTDv=?TI>sSvz(LzXIr_bgISGx&iFry(WHzY)ZB
zi^Z%0AruzwzW-+ji6(Zs1x|!MZURoQ<jndIYxI<?)xpkt$8#+)hy?c@Ivu>;&+$Ok
z-vep&qOi``fa|+oeSml<y7UpFZca%ESha&QKBx5xIU^C3XGm}3V1*zuqwrdUC<+oj
zy-Sxaz<x$U>3@)${l;bW9}APpt!Ey*G2z*4VWa<V1%Lc+s_F!A0`O%&caV1|;IqAX
z?m!g8yP`RrVQWqdex+#AV8QI(1%l^QWrQ!u&Sw?q_0T<gbFckzuAIWlwgk~Ao9Co<
zxPWiii1_M5b^9J{Oa}^%?*ln2N+2epe3^9jN`r}x);yea4iZ~4HL>W^C}aKQIk8YI
zulCFT*>ry5Km0Z3^oQjL_+Ws4{8yg<Sp{7}AabS)G~q@#HLmEjVtKU-1eg=rMF?8b
z)gdK&I&zYJ?gBbI{-9Mji*TC`5Y^8|P~)yPXFXEfxTyFk?CbPoVw!5_6Fb`g{@5aS
z&tcR2R4PQfmYUDkH=U@+C_mA4SF6yqJ0@Xf{35Qat0A=iAQApsbH%CiNcAdYP%-Nb
z11S#Bx@xAjkuLg-n+ZkgxZ#$(yZm*XA{1tECmqK2Q`FKu&=f9!XfLz7ohchZMvu+Q
z$UVu+AilCP>kBtY0P1}Te^P}T%oU;MU0?hkB`q@l<vlR^3uVVyt`7YFi?+k{94B4F
z**oADo)pphX7gVfk-<#y{K6nN3gyqmw6?|HGwq@kJqC8TlSYf2sa#Y_>;BB1{g-wf
z%=6+tK&h@w_?eJXAIA+R7`kGi=Z5D_SMbOvVgrBX!j&1)j`Mc_ixHAeqy_@AjV~q&
zn@k`Y7+|zDpZ1(Gmo}Dsbi4n}Edc@xBcD_S@i(oF>Sg>on3i{=?Qgibpsi!$@roy$
zxCRDIXOe=`RFFWIC(pYMcJJrUURcyBzQD_{_cOU<@HZC8KkkNK%o4@&SXJA6cS<K-
zDy6KP>{xm2P39TA3HAt4&O`61JIZpY61$Myteo|aP6l*bGDR0q9T(wzlb6)i?|u|S
z=8WVg+)4(Ez6<|AISDuyG^Ra~W=WwhmHfbe3X%x;N03BkLeKv`Na9~-nh-zW>>pVG
zC#U(J2mRDi_zG(&v{dpp-M{cJiaeD(wXNg~qb6+c3`#^sM7Lp%g97z<B1-N0fS)@#
zFk5;)3={=R!hNek8cWVKB?b)IsLbVpwFs5Wqt!w3x}YX;kY6L$*PXxc<8lTAfUzB1
z$4$n%dm$0v#}9tNkN@h7D>mdG?x79RQOLUMekQV4tKBvu?(-nHZ+DOT7g;Yd7|lPG
zw%n%d^s1|~8y=TmA1|NYm{*^5`b~rcz{Jj_$R?kI^m0D!BCdO3dc5(Dq%N}?@|NTU
zi+92eh#M}d^ZaN!mX)^t#uY1W-fBXo0OeiIQO$E{eHVLcI~B+Txoo5Ll=f}t{XB@J
z((PC7(QEc&Em2@~50oGh&o5f?RUp`S-35q=2?eoDWhE!Pqg3WM2Iu7^0lb8s<Yx>6
zVpL<rte^F#4*Q=SI~MU)Sdfq;gG_YvTlGIRGAr?%@Og$gdbSj#nkas+@GzjzWbKws
zIE(Z~&h3Wck;AK^mZLpNzll7=Y6(x1Vd7_6)qzpPfv}5Nsqi{0pLf`&@e83r_<1;3
zLRiSV5e;&03PyZ#IMZ;aIF^9n{|6z=`@biI|GCdt14se%Af8V8(<Kh8-sU)IdTi`t
zBKVu1B8&6g1*rQ?l2SqzzNclsLl!$^fBHJa1Y!<o_-051iACNImOJ@?#~p7I>EO1?
zae2e`aA#>>=1@G!;zSe^=_Hm<e11RZnlcqM<=zmbh(|tox5deV$^o_J%KM&#6J({*
z5p+2KI!C*!`B9iyl#V)n4nkm|kGxVk)!i3wH|gqd>8$P53FL@C4ESeK7{~uOhDF}?
zc?5V5#v3!xCP3cc3CiyB)!Ozej{&vc8FI=GKeL{fuI^ijpdv=N40Z?7Kb<S8GvVQI
z`Zu8Q*Q6WavnODaYX4N`m;IQ_|8K@o{M6?&{?zC97O2j4rYi!BZ#MX8$0*+5o}-1y
zUQFECG12)N59eU+%ql)0Nsq?jK|vTUe&V#yXS_%&<MqG(;{RR{+2j-QrU33LOry3U
zYw?0a%4JENHQE5j14n_+3U~d=#_Rfcz{L(mtx4?A$(htKG*GV}e{^Osbh>|^(O~u>
zG52$vSz@0%mvP^oy`oytb<<KwAT8}4VGQf5OKi6VF0sjH-z6Zp0lzC3r4)_hv+-h>
zt?}*gzI62RRLi56Eh<gbV!OTTNejSJV9ypkhC^+qQUvx#ryYqV3O@;-*U{W(%dcqU
zTcED??d3G!>AqBQyX|t3tJO`Md&+_YirqbZq?4GINn;f}O6x2)tLy2l`|tCl!Ey07
zncl0icDwD07r<u|3_80;0B;Um%;IU~u$2MgXF~oGKVzCMf<>;CEu@^5Xr&fDm|mRh
zBw7k{wco5Xx~PZzzQi;8P2h*ylOhnp6;e|%edKRT0y%eO;oocd*EN1c??XlHvJkTp
zuGAOiPuGe2^wz_Rt2d!)S9NggI?lG-0`1}PA_|y**SNEtJel3iF1$?qW<+MI!nb*!
z9I=^=)*Ekll#qUOP4`?6`k@|Ux4YFDpEHa`5HG)7i5+*{rq)rD7Soii26t{)(BwP@
z-^VtrX3k!R2<1L!!jK7d)k`R-Az(S=T~TsqI?ZSt_Y9(Sxe)i~&^4^uHb|*;almQ-
zm9o!rm(G1}m1sP)J(FhdTQb?ab6B!7#^Gy_I@Dc8iuqBkU#*tX$GUXyFA=}E=&>sy
z`<n6!7&|VVR|OToKLe2tbf1O1v<(_A{_sJ)_7hOO?g<5}WIBvmcpep#nZuzHUA!H3
zW>Ba?GgFwG-nVHB)`^~`dsPn_MK~@3u^aExWABM3YNZ!yqX^YZ8ZH1DdiHW{J<NMW
zW^C*R5t3<*u70KQBzB`LsM}n{LR1xO&^eB!tn5xHt1<*IJ|ThVK`jA1U8_phg^G49
z<MdWHg-}`xc?@T`$x)Rn_*rifosgT}hS!F_%VfFOgM!4ZAHeUED~Pv;jnsa}zBp83
zYFXNCt=;8IxQ>b1Uk)N9F+VzdZ&a^P)GkII`J^=woh0$`V_&(jb&ojj%0o$$$X^F*
zc@w`PI{7rBvq+UeC&b<+adzUSStpKQhQKC}<snbAqt}xfV|}sw^^|M{?gr7$wbp2Z
z>pFy!cZd+1tKhx0U`@FU*Pfh@W_J2q*CLg19_oRA(0L%EbqQbGWb(}@pBOVqXeW&p
zc)9^DX^IZ=6QQ)BFeP){$(1Lo(XqRa0FvP@`vd|mI<}_oCkX$zX9)JwIG(H7Q(zVg
z;a{wLR4Or=j;jH#G%l&0G)UYCT0e3?U0b$UNqw_gDRflU0^!0gkSoCkHn>Uoc%gLI
zGj3+s9>Z(4*08#|oXM&?{u~JB1Tk8Dcr1E^%4tVujmf2Hf1ILq;f;zX`n4~mj-V-6
zHW)~#mv8>+uqt@gmN0-!_QrX4Gkjq+GVlgY{6LksU;Oy{>j>8-W$WwHFA5Zsn2hNX
z@XY~UCnJ>ZL3PbqiRa@7d`CULB})3dmt{Z*AQhyF%KugfkK%fdDH&sC%%uH=^q3%a
zf%6yh%fm<*z~7=$*%3Js=DHc5tv!JEcEz5w(5+cM#zyV)u3T9n0(xHC-GZeYgzJiS
zXQ}DPf@pgUCJt<O>A1GWO~+t#b+=f+c*vC6czma^$SLu`?3i&Ryz8C~XQ<0YP3HK~
zAgjLg^__fd1Ene}gH6MJ^qfyOkNFw%(}x$0Mi~Xrh3Ya_H`dkWMHh`DJ;>W`M02SI
zd2JUQJnIAGjhFZT*@T|{&7?{y0U=90RlD}=4J|gxGm-?v9PdlJs42Jvyxv)ho^pZ|
z(PQaihP()6RkQOejh&EI4Kk1L%cX`Q$a8+m_8i_1b=qE{`ip5FDDwlkBYpBI);Mnn
zaSGj8FpxKCNFaCQ{YmZ!`SY4ErnPd`IpbYd2Gg+e_>>^-0>=<XK*yNLyYhAC)n7}i
zmm|zWmCD88VXcSzc9|dMO|1+Id!WaI$#kopGL59;Ori8=1^n9X50DGuW4g`I#q`^h
z&cF&icy@Z?5@hCpC)rR-`%Fw0rPk4xRhVQ%4cxa@6QVU8+7HS2CZOvLa=^lKYv%)B
z-_5Ci>Jm=>#^)OQ9b>%6L#moLvk6iBe~mNIL6w~_g+nSQaV9$0I1?6+aUjk_%W<{5
zgX4u_GZ1HzV%~ilh%=cOpT7j#jv5;HzIGM@pgDyJqHo2))+%Buh@`67j9_jp5wrW6
zV(KaGbH+WRP20gHJ!}YC_CP%>Q>gXPVN(S!k_!v||Fs%pk6Z%B%EMmM?;q}mmW>T7
zlbCU_I6b-7a<jqw#D01E=ZYMM_H$Mzyc_(Y^^y0Xpv<1M9qit{mRxYT)-vC?vaSsx
z+yf}yt8KdY=Ujoqe5e1{+$k>g6??qPoGbleV@wJBHO0|?@QN$%h?T6TEc8r@>0D><
zIa*$*=SVblCb{Q8?A}V(X~Do#SH#NwZPz;ZdbEeGQ_`RdLW?M;%+m*WMOJbmqjOZp
zPeW0{qtbPTC|sK~^zFbU4dooeU-u`BCa&=ZQ3Yq6@c<Mt&%d40isPOM7HM_qW|)l5
z53bGek{;$Q>f}KokMsJ>neC@qgI|BN@Dwj8_7G9>n*3z-O%#gQb!|yT!@5g8e&XYq
zZb^g{91AqjnsukCx@)Y;_hbOE;P++X23ad58HC{!B(%Ii7;<U$sTPIYzpNw@Sjk7B
zHAUZJbwOv&RP8u~(MzXwv`bl(!0DfPVG1|4t?J{YoYf$^?aw|wkN|GT=eoZWY1naL
zKd)*k^_t=wrMTzJYzm60f!dJmzIsZQE^w?(C&dfp6d<GJ24OfJ*iJi8&wUasFk-5g
zKvaA-w}1&SoQjtFi@cJ{lVpsZH^IJH)2gAE*wZQ)!JV&q+wk)qKshy;){OUXA<DzL
zuPM|-+&9t#+m(4|i`!=fM=9z(OKXn@u6Y3FOw&M}q-I^%mebQDV9_6N4o=^Oa8`U;
zP6(E|$!y#3HJIIGG}8v@R{)Ja8h?K<znr;`p!(4_&jRDXMcOPM=5^&1C-=@?$YgeL
zqkXoOY&srXnv!*`W-zuAi3>YYpsn1T)gAYXVr@VryP+v7kJv=@YPGgf5!bg?_Rt0B
zO?5r3$49m>8K~&u2Qyl!{Rr3--;e^UA`7Mp%$p+Xc}J3R8vCo``Jkn9TX&)ti)Ynr
z`_)Dw$nvBv^d^bg>m2M?d!efZethhIwK<s2dRC}V>r6EjIjM^vMF3OzzDAK%fmJ#n
zy?iZeG*kA*c01G=eXLMb`;e3nnBaqh^tXmCszDBecm<7y-ijZ_uhWtaoE)K>CuLV-
zQznq5S~R!@Hk?gyu$)-X(4h|H-N=!WTK;4MR4NnNqjqnQaYLc{`?n8AH3$Qxeph%O
ztE4s%u<UcZV(;2?tpK=PIkBJIt}Lr}=d`iPZewd0oqZ>fl0d3XaTmRkuB@v2+^A4W
zfhwuW`q$`e=x#$voO@D{q_(Gzq-x7$d%1yo{7Fax1+k_wJCR7RwQt$M1g&h5;>qK4
zVg<31ba_jG4>sYia~z`+CyQt^KDn*|X`$zDxgMXmOfGEO>roT1oV`SL3&h36+Zs_Z
z_w`Hm1usBot#a0PXM83n^1bcAlzwRtjCm`^ashc<P3N`bY!Xwo?6pW#bo^=G6x_f#
z&*_64lVUZvte8wmt<WIhoRVkVdvp{SceBs2^2yO^A2Uei7?r*HZVC4OWqMofusbic
zZkGC>;9>J*H@PJH-qu1Qxz=i+IvvaOh=)N-ic}@Va*;t~*$NTp@u3(|Eq$r<eqY%_
zmdwr2Cw%85?10BpJR(zsB}q!mhp$HTF9tG1<hfbWB;E6^7CWepzvgk<NTtoIi^1l+
zr`(Tk9~P_S3&i)wHPdV{O~*X*+wIesWijbOo#7mtCalr_v>6GdbDA4q|1@H@eZ#P0
z|2X{mGAZwdg&o;Z8`%G;YNoredCwJMSxFi@&IndU`>B=)?T=TDN9P4-#+`oGm8hil
z5Wh;TGkIt%YI0Fb7$1%g@5E8EsKb|X*9mXb(o4V%7J&yhu4E^-b7d&c-|kBlU)Sc5
zfkB^rrbENC)k%RH*D2#tNwdK#B*e-<?CTXM@T6NU_jp4;N1ojUt?C2SmpPDeDl>RQ
zYBu8T2c{{*Tmg-ssV?KDPEWXQMvesnFf4{MOT{wjzCKL=3fafa2jr9NPKQPdm81m+
z?~7NvQRqRr;fP2@uM3O1rIF#1)=avNBUc&3%$ayJM^~2$ok_;e8?>)_N^TmrOn_H6
zp|Ez*PrN>8DY+3f<6x*Zzj0V;;DK1eTdh{5Dvjtm%=8FE<kKL$(%+e7wTnmq`OJ(p
z3G$`!Qc{DD^x7V~?MU-B?b;fUvd(<{k?&J{MiJY4lB<tyT2_yVe$KHjwADl-XG>FJ
zcdJEbXlf6Km#@`+@~Vnb6DXf6$}vwgSaYowouy2)DzOzd8{m+1)RvKSUZ&IUy1YM{
z9Eo`e(~nN}O^ZJ%zTV!Rfc+wUtu}B|bcRW4vBG&wCgP$W5pwX@wWI^~G8*K<%bl4a
zeksyRdS^Q!fa%xypjc(bOAerBle2s!RqfkwSWQA`y#atprt6{nIMTR$7%J5(Trz<W
zemmgBF>BE{cLLS~MHZ-n75`LW&D1qD7Jz79WM8%@m@p&n=`aZHYvORox^KrH?5a1w
zRCAJqRKopg@+1i<z8Q&H$38l?yEB0w4RJSquiETSX$p-qr#4}U8-E|wej*7t{A*W`
z^RW)!2rtv{tBZeQUi;8{>o=2cL-{JKD1!ta@c{1iZ}NtAV;-P}hIXzeZ=}=@b)7zZ
z{XoEy&>qatCv9uh@>+^ZbhheN(zBi8$8z5YS$4bSmC8nIMv#H&w9KZp*DE7zcRS-u
z?Iqq#G|a1CNg9s=xjZ7CC0Cm(+k*97P#c)($dEgjjpzAL&Hhkgt%)_GwjV@L@MX7k
zY}#DsI^7LJR4zT(3)gE>s{}Ocb+Ov`$);<nQy0BVSn_s%&73O#!tO)_Ly<TfkF)T_
z$+ocv>-XvlIDI#%-8dGCsfC6yK{<xqH+OXoOYg#b&@J@c=_Bde-^@w5ekTlym*u7a
zCQINQF5fzQIh#6r&sm(cwDYJALc_4`>1Ns1Th_E(0eA50r>|45-{UF2KP99PVGfR<
z%oLR*(3zw#_1H86BV%CndG&}3S&{c)@lY{MkLZR%$OSt=;{-IRt;nr791bzYOk}ZE
zNdtxX3w1s=!yX{_Db^jxeY((Oyg_3Tdtd;}2Oh>P@!+t*rk}BaL*!2#X<hW(CLVW|
zdrY4fEFdbM@EPf2*lMD1Q8_ZUOL821tpUzVD~zl-3vmf=ydP<x9bYNzgFg0{uA2Fg
z?HpUlVbm-@E?xQb{a2buJ<8+ViB{fPK!ijSdB7choLMN^Ca<W+^h3&_-B{*my%B7M
zD!z(wXoKwJe&J>}9a(L5*h7Cs+C6w=SZlS#vyjd2NJK_YSKzUsqo2VGb~eK!=M1Nl
zl*~Uo%v=4ggt$?1Gl)gdN8}Z-H9FmdELH>g)Rl{et-fU@3#RNQb)o77h0&1-A{H$B
z%m>w9ydJAV+?}`FVsF&_Zt4d{TH;f*qcB`4j(`PVByiYJb#Fz!&Y$J@B+HfR$ANtg
z%3cuJlg4bwAE3_SVUD3@7ei#0Pt0_sbO=qzvu7vL+uyS{%5ox&p2I!|<+u(+?qMYz
z+DJgs!KfwU|G|B-v%T+|I35LVYCvzDmMZWblyA}v!YCrlgmIXK4Bty2p;CR9fV|^X
z!AZOW^6<QtsZU@7H3f9Nz_cNry9DH)2lpY$)LUVoR$KWx7=zZQQB4#MCs!oXSrECL
z1pUIz(cd$xIsU3P>Fy-K`39QWgWm7ocVuAurW=eqkEYh=Vg@~(o;g*V@zJ!|bybNM
zy~x;n4D>L8$j?$tFa4KXSC?ECTG7{PE9x+-#}ESyF{7)`S9&jLX_T6_USZ*Tz~{;2
z{MZ5h<Dyq$$=Y$Efvf#)y;s(KYpdvi{BB7}#~*Bx&Vy`DBQ|kY3(T_PC3rmd{9woh
z?T;3^ZLUB+Vu6EuXj#k&<lX1};B*?9PKEbmt!c#sW6&t&cInqZRDO)f?y+29oyjhJ
zh41fR8Z+!+Le~U06*{a6Bs<QXfi1ite&NJc{{7=|RXlsaWj0nT->KQq>&5txdr>tw
z)p)M)Q|s?8+5Sbkk{XYduDt6VxuOHZN>^efhX_2Zy<=;v9{^KPBEc(T^K7hiB^xVU
znS>`_xSA`cI%V9VX*D%^B3-#)$^xV@zs_$1lxk^rrhc}setP-<VN|LwbL7&3xK>u>
zJAv(9sAp*SR(7T>$kB%dL$FNg2|||XVMpup^!sLcQ`5`I#8n3GIOWb>dUjMjJwA0L
zX8NP5&Sc|l=W6I+`(2%wtquJ4AS*YkXr!m(v;^4w%L;IY9+cE!7HpEK7i#mjno{F1
z5z|Xq^Tm>rHpOP`;@fN8=e{IvAMYFK?af=v->Yu~mOn@A=khy^?AO7&^F7A=QbZl?
zS`xL^k0ZHft9=U44aA|L=S2*vOX|`kDYt*^cmi;a7+K<TkN}QYp)RD5moTm#WO^F=
zn39fhe9caC<Qt7Tmmnl$lA>OVk61K_S`#MG(l;<s7+@D<;Z@AgybP#q+^uitRcn+%
zGZ?2c1WQ6R;}Wq?Db>`?DU=caz;UhMwu_MqHLJ?-pzhXN1H!%Rg2%K*cqD*uubE|P
z$W@<L^N153#a8C?xBH_-a?8x)ukbx%KOb7InSRgFn7iR7PXfvJ8n1A9+`;4c{RXh&
zXgzy}me)K*n675a?^eySGj~AE2t(iksgl@fEnDm4G3Rw0@X17fs{Yi9@?77Bp*w7W
z1PWas6ap!F$Job)>5~?dXN%AGd~qbEp}vi`h}jL>Wg|IK5#hbSlS6&m#}A5iDjk!d
z0evu6euYRnoq1pa>tTqY{@qI6PQoHaTer#rGUr1bX8$DosVT;rxcyO<Ct22$AG%A2
zVbE-jcmH-NlBX85*@zYXxXU<xlxU!<kJsYrZje;fi3fx~y5BBlxOid%y~I-5>!+W1
zbKI$LMRwi6O>JOrssEbYXsTkXMnHLUJfU73cmo&<7|A?QS%j6!QtFyfo`kQ4?3d@}
z<3(0|H5zC!Tg5P{=TpxNoP|(`vDwub9E^mnPJ4<y(8lb;_OCD0E*@!H-aiLf!y{E&
zE(2~3t?yc@%xZJ-k|yS&uMn^ZyvQ^D$2{4yY8DZES^LKVeR^>E&M6C#(8MEK*NgKz
za_uV?I_HG3K=jNBhz&dtFi@<YsdfxL@3J}zz?wJnriE`6C`gTVuwGb0xa=&&i-c;j
zTm5dh;k;u&a9ahSu+zjK`tIKYdcCI);S?J~2x?*ADJs@(I&e6psWp@wp=Ss0B)cOg
z!C-n`Q>K<IfpBhW`eifIT1kfa)cOj2L9|8|G{dz*!zu_WcD>oZsHjR^rEt};enLaB
zgoaDe&|%;Vc<uiq?k#|#-v9q!MGz2al`c_G5fG(aN<@?tkuG6H8eF=TMrvWDkq+rt
zkdg-J21x;_r5l!7@_rUP-{1M3qxXO3ckj%dVa7R*b7uJLXW#EvJf6>IQT$*cg4WqG
zimjft>0oB6kVqJq*vA7psLvReum2gAWM|ice-17;0f)Ad+hM54=~u=#cd#R;HSMNt
zdeihrS9Vw=wQOeeK6S?M!n1=a1l4)x5ud-4*>hWb2blxfTW=u-*PjqzqPa_)r~J7V
z3G?iN<?@XQj<$!$q3@Vl<M3BT_^9;v4~$bDJ=5G-2)Bt-MjQdtgz}B(hy>^L*nMZ}
zC(37w!l&m?dg_$uj|~H0G$()7B&Nj?V`kw~0yXh_?ULshR8StP<nhcqR7)`6S9yHF
zCc{06?qjvv*8?=|TnIoxSrSC!qSX_GpK<WKU16V4asI^ay$l|^##itB(`^Ya%ovl^
zl<h#Bf^cl$H}Z_RmB6K^iN|#<y<1&flE+VXDYU!t$hoOVesIz_I&TeFv87`;sJ{&e
zfD`uRz+Kw?OTx4yL&7eE=DAlPcsEH`d{k(IPo5wV84|X2dz`TN<-PmZQj6Hcyx|k4
zoyVtS-8Fm8x`=j5%me%kb-AYV^C<gANTkS-&!o}=w!S?6wYG(xa`CvxmYU?WuF&NN
zy2wsGnAPcfbzX4SiU4GcDV(*oKgmlG_%<tp&nH_0`{KK~e=PiJuiSc>j75Ahz}0=!
zdwnAZFa$VooMh<~yQ1&V;HNZn_kjvTeW_Q^5MLD+Tzycay5rEMe>FjUCX1fuvuq38
zsN3l4ZklYrOH;CpnTIcLssb<3t!Yhl)^!t9o2-#H$QkB9-8}?`0rSnLOlX@eWX|o-
zZvN`Iym04)NhT@%v9r9n#$ytom}pMYK`Ud189242LQWc>W82dwu15c<&t7prH$v>~
zhAX_ck$ZwnwxgD%PZ-ksvFUM5okX42O(wc`TTIlujt?#L_PwT*c5}Kg9r@yQore6)
z`Y%2H*X*60(k+_B9@EASK+&F&(;rwZdcLrw{>+k0FpQDY&dd&s=D#OQ_2_F}U)GF8
zk3$uuh^<~nOm@Phtg8lb+3S^eFjWLf<YXcpQm3YJKYqp+KdP2m`h4u%biz7G^H<`|
zzb0Dlf|%V+v#=3L{5#KMEzo^TT2##5Sy~kBHooQz;6syoJ&^^x#L7nwU)aEJY6;eM
zSMN~KLKkVa$(p}Y=bK=KJFXPVH8mViqK2Ql2t#G~A{e^o@nOav6MRHtX*(69iAePO
z4LH(hoo6592?CQwb$PA<!{DZJz?$pe5U}P7|9kC*`xb@3OFGpiGrmgu@WEx*-51<u
zW4Cz65>-rVHg^lA4rrQsRT#pnLcA-1=MAVBZJPuA1+H(F&acDJ1UDL(9@D?#XVL>s
z;a<nwHlLkT!1e9p33b)hes=Ql$ZDZel#rb*h<mb*a--KRY8(A|q1=3EIC(Up;FR}J
zACX+PK6^MFJaqbLHt)H+WaO^abO<wN=h9X)X7X{eli+$=taba1X-&~dB;N>!eV|O)
zs6frB+V*&|O66g~Y2&dS=mzr#*uAk=L-BD)V8ZXeOE;olfU{VWs&l+QP08ad!_<6M
zRWD!+GLI(yRIp)IYi4+1QMJsc{x`TUXhuDyF}!dq>?6jp3o=q^6s!E?>Hr`?SNDn9
z#%aj-WU<*xXGpiB6mSC!#8eg5l^=-(9{^a~1Rl-IWotz=q4di-$7tpR5I#F)Jua3F
zi7B4BW#NIkPxQ<=7N@S)mWmK?s8RB_TLvP*YYM>4EJMn9=aya5E*{wxwU^2bT$f{q
zANo;6=mRJ3Td~QyyCklHyZ*PZrgC2f(69=R9ZzdP=IRXeJ%I=RMiEoeaU9e&_Es99
z8Z8pl-|FLoROcG1<be_4X0b6F$I?yP{i$%t`&Ys;eQLplxx5@J@rTg@DL12e&By{{
zp7*H=I3s868rZt46k3YUcm|Rl%nEbA2KQgD^M5u1?jt036#hfxeLxTpx;rlt`06fT
zMM=l{uvvY-4HdI2wAU7|1s%-vWg&QcA+tdA)hqsXNmL?aF!=`5h{<<${`1<GK#65A
z+5s&)Wo7ip0rJ5-qTuMr854fx;!<sV1)Jw<l(;l)3cTA{do&q%x53fU(pq|eB_8l<
zqcOgWurm+sJ8M2)S@1yofS^hU{^F*mRth8~i+DuBbtS9-GH7OyrYN|r^71Kykk{Xe
z=rL<FpW3aDlS#)l6RyJM(1XF@qV{*2+4U~r;r6>f4HQ>ygtw0zEx-7yPE%@%94MPS
zq<Gf7<Mhtxyk2WR^%>|?=drjhA~YFx`-)~0kj2|wjgoP+>FndO+U>}(Kj{y|Oy^XD
z)lnu-<zCp>EmVPd=T{!PP%*OxX->}(jV|W%8y<S5h+R4n)Oow%?6fcn0U8(8Yxfn`
zE1KzO59z)%>+MC=#Pu>aHSkbOOou++dbt8%Vb}1^UhON-|F|g!3YYudXC<7J23>I8
z;^H*MNUIp+hGZh8zNpp8PC0UE7WoJsA)vXn!zj;V*m^sU$IyZCxzILF$L(kIO|&w_
zn<x~Dr?pqRCA*lbK)6Y-T)=pSkNF}F*H03l1lb>$Uu-hUUT2#~kBI+ltksCEv`k_k
z@})9C2fi7xIDK|K`n(S}$i^7<T-!rw0Hf5KV;Q<B!=F)Uy+(0C?i;#o+jqIlm-hD!
zw|%(m=v|wKOn(lp-hevp`r-Gnyw&rfDty-;tNN{%mwn`xSX)|efmZXanG9cr7l297
zw~Z)X@W{QNJoow5OB}e3d+rFwY^hboXn#^<m}<NC)0*xE@^es}-2_L;xNf^@g-y2M
zNRE*VB_G!ZB6`ww!lMVm==X+nSpow<mZ>Z=P%8r)=00JEi9aNs0gtpF{%eoAVZF&1
zh-+bpyY&qpeK)qz==K~YaHZ(dHkY&3OfqvK!omMVNAQO)n@c-e#o;N(FA7C<ny4MJ
z$GhPJ3-bX3QTbkpy!Yz7MDJ+X*m#<1dt9kwlyu{U$2!yM@X4{u&D2__I3(egX#(+H
z#0k(WUYq`A)~-f;dZCb7BoF~X=J)W|CZ1s$*mT|DFhcwldne6${;LSG{&86j{3@@>
zF!$29kxG4EWANrd<iK~FMBI-H%)l~g5#Lvr)}Whq5x3{=`y@WmN42R1Gw;Af4O_)6
z^`C_R!I(qs(0EH|0vzyz4_0JVtUIgny>UElOnLF9^DmYg1wM)lu&0kUChMa$4|3=S
z(}H1??0~b=9Dd@oaci&o%tHp|4?NN;#DKg4tT1@4rfNK<WCwnezX$kXJ`)ns(mz(!
zZ?L49fkFoEUt~~p!4SKL<o)KD2<O2|7|2=X;9_S|_y;#-h;eW>rP<CwPF&5FRg1w@
zIuGNl8xGNtV;x}#mt1w0f#%^b^Ps>CqH12r7&x0eaxCmAHH@i<kzwa5tIY#B7#EC7
zj}n=KK%<K{xJH~co%(2VUr4VO_YDZ)gW@S4y_2ytoBeCC&3d-j{#y*)F%bF+=yotx
zY=8Jsr9MX&b_Kb$3coiNLOFiXk4UQ|;>1o9!guM_+k`L^7t8FKGy1XrjD9?#Z(+_m
z&L<KU0O-f^wpNqtulWdv#-m>Afb%#QThrssujuf#@$-4!Woc;u=*PP<Jg-&wq_Z8e
z#I>9`&ge%t<xW^k!qCme3Lq>CG>Zw#0`hM#*^&?uAbRv4FwdE??YF^q=af$Ecf7ES
zTJW=xLTfD3gfo<pG3^)17!lxnmWRdo49LTp;TU9%1gT?CMxpx?k@DHjovGm%lrdNR
zIszE`zTnGNwmHkglD*dz_(-w|v_n-NMsQVb=_OrLOXqyy4TrJH3tW;emITi@id!0s
zN{qMsBg}tQ<W1?O=V5RDhonH#jppCu%`l(<(@pa~b-h625%KJ-!APwllEw_(u6%y{
z@r@_ooJCc=74{dIDDlYcF5o)M<A;<s5G9Hn3=#DdpiYT5z6a;BTrhziIezjVviZW>
z3Nj0koists%q*Mq-PQbh5#6CcS6n-N8D)}r86s2GrqE6)tg=IU&=xA+j!*v+uo}j!
z9RJo)0J7Jwe~*R0OzXv;j{S*&!5sR5XNSI=6J2!P(V6QDneppA)Cf*f%g|lAbJrKh
zBJO96=SeajlOi0y(;J%2y~532n_qtY?)4xuT#(E@Bf>J`$E!r)h=k_ka$Vyou*WkF
zfo+A-JMm*4FY@c6YF{J7zYHn`8Jh@012dJYgXFN8<SA_Y5aD?oz-Mp*M3ml|J&!1b
zh=~8MUZbd5@I=v&?qYO22mZ|-tc9mUems<+9AeZL{RDkeh~LmhT83&!)Fkqk4M68e
z0m53#`bq=vEzR8~m@y=B!t4rjD>7BJ42&UWuWS~Ui|@ASdXBl7d&RtgNjEuPZ0%;w
z0~z<E2MtP_C*?Jfn7oumZQNe@SnI_#AZ<Q}^1qQVc`@)I-<cNgE5qk=*y-161LUlp
z=c*0<SyA}L2-EHZ!Jxvj6lXbBe`f&M00!{u!XqCp;G_k^X6H%3R|7Aae3U10(Y6yM
zG652E0nd@?teCNH!o6&UX4AzF*<WSPXp)_ojuZo?BiypkpD%^GG&wLyRv)s4Zg7uO
zB1T5Yt^8gzB#mry<wn9${t<bL*V&JOeI`ud$GL0XwLRwl^&2JS1e6g}1bZ+dH))LL
z#}?#OFV;uTXK&=n0c#<&^XzjGQDKLyI2~LaYVVnMHlO(tyZN6%2kH%=1GnX5(gc<`
z3sXg*us{=VG@_HS=M6J#-r|7>u1*`ZGYs9)7azqrOHkkpb!L1J)T;uOTds{tz_@-u
z)j2v<bg<wqfOJ~$|K;HnRqII(`j8d?IBEcn2QVPR_&(?K$2C!8N1Y_iOX%Mqx;dkN
zL3HHVp_%q^EVCh`s`&~73es0%I{!+p((~_=3JV^)p9dL837TY_oAlHpa)9aX&qy<V
z;|FUt7y-#uF=>>)lB)#HQxC@E0q%JEzk-YbDh?3wnEb)406^#Sxz8XI)>RwTa=_8j
zR0IJ8R^`YC{1sUB{~zS`FaFW@Pjm-x3A(4A0gVVls!w_T)=d9Nsmg6?4Gy}%0QgT(
z7?3Dx@+>@(cW7l_pxxT>4HdsCZY4^{jo)BRTAwwfQay<1$ZAYb&%cowkUBFzU3ix_
zL?!vb3t%E14<y7Pi@rDq=@pSdX1tjiFfmokX@WpZl@*VqSv&V|uEj+4aTAP6JU&+j
z|4hi;czof{rN^wy<Ofv6vt}Y$y{#e#NQXdugaccQML&o`I9D-ZZ=+9fy2OWt-Kr}~
z$}p@tkpE<Kxv6y5h{kNl{2Ke9F=ZePpKk?23~f!-ZlO{!A>wz;cmU}gF?U=YtPUxP
z{}@qJg~z+`JD}NY3LU<4^+c+AMqOp7x(G&V8cTuQ<b&pnf`HrRei=^oU1dtm=z6^!
z9^PE<fmZ>53&rwHdA_rxD`7Z9yYxu_m*mUW8deVBR~Yc1R01<rBK?h9C?HF~ifgbw
zY)BBT1yq%H2|y1y^bgNPDPwRzfBd_*Q^OcLy2O;GQzJLj%;ey+0+$6*P4=jicdg@c
zF#*R4;9=4XqdEeAnjYRGGNh<ER%<rhEFDiEoe+X9JmmYHW8_J>-;?&~z20~wPQh`h
z2&_AOIon5HPA3zebK>V?!JHJtgB$ES8;=x`6p?}I*YbZCbe0`;cFReZig0T^`kq=(
zzumzQc!AYnqh0vm;=cg2*+{Fc1@h)1qOz=jZf_QMkoue<;nwRI<q_;oCJG%2bUA!x
zuq#4xvQE?N!$1*JqaOrZu6r;uBg-c%Q<n82JG$IqYetbG<+4h(JZ;KlbP=V%jGlT7
z&)4YGYy&cT5D;JTf4H=nlhNwz$>4PJUgOol-iKO&)I34m>9VbPTq%+b9aNP~R;`0y
zl$H%kyUL~IS^{O_9&!!DMF$LPk-2NhRr0FaEJP62*i3kHts#dZ!<Otx@y4Kg4mj@3
zsgdK?<%N`DyM6jN#-*MF7DuC&of@>8uLkL+7!3Khl!=Tpy4b%=^<5|%5{8GKSUk<0
z%kB8gOsvi7VzqVI`*xa(FsJZ-(M=ZFPZV&_W1e`r4F}zh-z_C`jL!+TpQk<6oidSL
zlz7ayzK9aIamtEXKGFf4Ga6fv1eft2&GZ=uS#q;Z+=J5^5pN8rzyyFfmToM_n!x6U
z-OHzqo6CBpUS%U0ywZe6OUN1H(EW4g4Dd|G4Zf%Gmaii<W!NDcb0IN%ihPeO&w{^R
zV}ie^vbIje&+$<M;E_ZA9`<~`z8f4(v@+dIHH*00wLeq8hxFXL+%#6^dqPplerO6n
zv*okXmw29Lpr#H_b9Eb%maw<5;HP7U6_g_TolAGwiYKa@m3yHTmkN2BoHt&<T-MRy
z<=SP}y9q!XWi#u&HK#NZW!{y~!L_4($QGTe**n6~yf4!2L6t_pE<mJiSLh^k-})&;
z-XD!xFK=3VoD;Wd&_66ta2U*BP<{HF`rrK(<2mXUO1e0$s(}Bvwt@5DERSqiT8R51
zY*vFCo6N^h25FR$sKEG)WBDbyjzdTL9#!PrMRu%ZHXp9JOyJ=X&c4j<YMMES0#FGX
ztNvaVn@2FPzey;WYJT;uXwcRt#8)r+4*l2&t}UA!e=o)?G5QDB4TyNS=E%;yIV43e
z-W*{CzgD~?|2_;cdnDV(t2g2*cCUq`mV~kZYNGfPl|alM$ydM3v+;(#>3+C{#JW&W
z+ZuRG649u!r&BbxS0s4-&UN?ZQTXS+$I(hB>ZhFyP-+IE%+YMU6MNhF{kb)>FYr0^
zP>nBnH=TArd@)ml*PmX?{*qb41pTLLH5KME3VX=L-&E|3=c|CvRe{ebnzgnt9fu*_
zgLt$w0b>C!l%pvuVXVXV#ROxOn{ls{0i&6i_B9XADi7m6fJ|_FIAFl}4_eF$pv5#V
zv|hGJfw8HeD4Bf#TI^ji!OuI$fwKrS^`FGKOT7<OmRb1P!R#_-s2RYKm?~uJ<kven
z|GUmlvioYQZ&E_TSFRMjrLY%VPJp@T&onJ(qh&h1)xmILmlp?<KTD_UPpjCyym>h{
zCc`=U)k0(eFjoWo0vW`4v&Uu`dQYb}#05-q^j(O{U@i;F<BdZfARG`4P-d<OAk<y)
zzG^;WUd38c6v$z~MR;ZLW7SAgs)21=37^AK<?z%7vB}8ND=>|%*$w&eF}C(kd=_HP
zi?w2lMfBAycY=iM=9_vQgibo88fHsj0WoVaFJlAkm44H%X1JfNS5wWpRKF_Wa~q;(
zwbd_iW9=7Fha$dW65D#_Y8Y!J?LGp$Rsa{iSW(_wp|VT0^A=YVm-<|?Vvnk&iNU>B
z6)O^I9er~)p0H_5WLwNlIhgGCdy<fW_K90%ZOk0MoJ&~w+I5MVqMkBEO9$MQJWMkC
zl$QrR*k0tbT~W3CCrSQa4L4?!Px5}lq(zdUlDzQVjZ>9VLTYYbp}oi4z=B_}UYX14
z83^4{^+eSWg$1`eJ#^fVqR>E-=hvoei_{+(QP`bW0m1LkW4Q`@bo_X4lGf}HmTS40
z=9s<R=F{sA?#&W>L4KnQ(XS5}yzBrX=O3!Z{HAjf0RO1RT2`LoKh`Kk7;BWChk!MT
zB`}FV-jsCooVj6*D_GzO*~)NNhzStLL6pIOOckK5e>Xm)-P%W=kyJ&72nY7jO&|`j
zZ$r<9GkCLk0czvGG{oZd7ReT>j8u$~A;I}gh>yHXu~FHBX&@ACK=02W6flbWhgih~
z=qc|<4BD=YvD*&ocRy)MAs(Yb&26!@+$^jGlLpLgQgfSLBkWTO?Tl$(Z$64vF61gz
zYcKgRdE{UP-i2j;n#fVAP(|z!^;*Wsaq0K_a2X9dc^52&Q}daS2FBzm;n|I@(YUu<
zis3#wwx6GBCjeW{bwtyn-f?meeYb~>V4(Da<<j;vxdd&}&S)OLOGNupMP{S-;+?mO
zDY*^pc?YWwFBhDiJbs333Y<pI<2F^SyOcM~w*1ib=>#0`36mAmZTd24NY`sbZe1oi
zy3+*9jB|6NmW|$ELrp6ya6Nw93f>Hs2qxrJWhIf-O@Q)pnP`-l(pRik%bFG|0eM$5
zh4!n(r;X_eL0wG|JCcCo-1KoXuQU(aBRi2TBqdM8DWD&xh#R?EU8OVYD&pzW?SCx4
z|9MS9xd$*LR3>>I`EN2@^)99fOj0Rg+^FrL*}x&An4p}Ipn|a<-C&|ghewE7g~T-a
zEfQRIUZj65?GP<5Yxx?;AmSHr{vnY~JG~+gRhNOxsytiXV8vMi4^X6acfOvu#dV+9
zn-&%|2NBndmA`kPmA$t65G{nt4NY94d9pN_eOm{*%If_Ekgi1Eu2Gq&h9*!xN^?1-
z_DwFCCjYfcR|8Po-wZ~0N=-_QXq?uZp6hNbHZ%Bg3MVKpATijkWYuGx<6gqvFVK1F
z`M)X=6I$ls5)?}Tve&hxi7idmcW90Oakb?NLyA#=1Y%hC<B(tF-tYj|Qk=n;u_K02
zm%Yk`9+OdL!%rtYL-avn@)PA9mF2fgOVswh?vC_*xl7I}7hp4}R>25k+;1R9G&cAt
zB7K>6^Iq*ToZ1VUf1R4TSHS7&(6`?zPvn=UHI)5|If2`tYwYX(^kbwq#dwzw1!}!w
zy|lL)suV9(zWu#oIx{`u2>o$yyHDlXZvJ|;L6gqOR)DEaR?_Z($^_i1yUO8{M{t#$
zIo;5SPNeBb?6y=mJJG)A?WM7`0^|q;X;JYGWiQk|*RGJ=6`&IH#$A$!aqZs&n6PW-
ztkxt9!)iI`;*Ry;yq~qfHpKf(AzD;NiTbSc=BlC$J0^2*f2P+!SeVNa<q$0$#9>+X
zTU6@@<uh2|b3-v5zHEMIlMQkyy+;p)vf&B3@bp4P63>Pb;Jw4Z;|nHpc#ubl9FY&0
z0jzfvh3lAu?|{#8xcv{Aen0)<DFyCu*sARaJB?=k8yAq}DP4e&eICcgzUfT9_0l(W
z1qCkVcU?W}kvv0o)}-!;<C!$WURzHI_t?DzCszt2Zq28F*EYb(DR5oqt*2~sX;yUD
za+U2NCwQwR%fsye5A@-x{R3g=xf0FUSv%n0l`D_Bp6WE@s1*|QxtyRZn%!~NG#5BK
z9kzQTk!@`2I(ls)ClIkBlTzD2WTfWS-onIbAa(P#4gwHxG?2%Nez~<gw7&Co<x52b
zcWfhqKV6zOUt@MT^aCuG?vR$xJ`*iXV><F#jO@fz`MJPCb3zkt!s}}X+20+GPh^wL
zmfH~WNB$8?FUC#&y%F#K@N1?IP+QWKaz7^pM4~Sk;fBO4-dG@ybi)$D8o98w$pgP3
z{<t>s{&$K8aKNc%`zqQI)|<|;yh$Kjh6zS&TnI(*glf2^7(!(k4Kq_@UpyCMNQela
zJs7){cui57?jo5gHXRzR&gxyLKBEv?MhP8wM_&yq+g1~Y%mqsZMl3F^0Q1)?`sTmp
zug`xwIQu+jyS!@Ww<g5bb(vM3mf_UpprDZqc*aYfzE7&7c-hCD4g)C<f3z&sug&8X
ztrsX2Jx9C>HY=Ym=*_rm(E-Z{s1--C2`5k{R6Fa)BybCk>@AQB^(`xJ`+vO4jN9lo
z>YCJ_V^&r7p*LBp`H87#r)>}rd2)MMp61XftN81_?sSixUROsqft9kCqSoZ*r01y|
ztF25x3Va;V#`cOt40N*Zw!dlt@6?|7HiE}ZV%!2`4nMZjuG|@v{Lwd@+WwapfQhE?
zf#K7RC<W3gC%eFP==yTOT6uunEI)ci<T8@yhe}Pk(shN)tiB83>^9f!U%HN0-imUZ
zNi~)C6>{0|tokz4`Jl;VZ*)B;O2oBw!UA*>sa>GJM>GbrKk_z$EJ?KQ)gau7EvVUt
zvW~7yg~T*9<r2THe#WBl=K67+i2+G8_l$=BTh`depp#?A)Hj(Ar$v5KXC{8dmI6#7
zZ;^fq=5UFb3&7@xN1>#J0jdmI$LRjd<!T_-2M#xfHl_~<buv>D*0gHNC`559P?VDl
zofz^&OaQTwOLJC=rMo2?%U&b$s!^FV1`wIt>Hb-kMKG)oBJ;+EdCbe)4fOL`SzE7W
z;82l;UbM9Ea-RwZ*U^&GZCuP?80sN;`=jNt;(7n`hb_bX@e<Qw+ZCHyU2k`Hi%Qyh
z;p2>+$U<&+A4@}lAIQ#&oS$(=530CCjB5n{(^z^h-`j}d(wC{UM#8}^K{lmkdjXw$
zLIT;%+3E*c&M$XvXK(Y#czhgpq783s0sz=bn^Os;CKY8?pj_o9lAEoyA81SRS&K&F
zG4B4DhjmC&LRJcJ-=3hliD3jBUxiR;@NzC+5PQi2)!rCNh9&yW)tT7CwL>k(9jVzD
zl)j=}g3AYE3n4!%)`d9-H4s`6FIC)ekKn=w)06_1&bk+$(wB}q+9sXGOdQ>|*J=-0
zNv&{fc3fq1HLE#JdQoRY(GEVJ9TZM40d+k^+#qmETa13|7se&>|5UiG^n%H>`8Ts#
z|MOY@EfQb{Ruj~Nayh=)S?G$1t(=r*6V2l$DEf{M%$7poKxkz#tW|}5>+6=2LV~8B
zC}lU$o5qL(t04!Qhq5_+pS@d0-d8Kp>G{v|HJ08%({#^ISv>+`D}Pq%8D(5}IZZmJ
zJJUb&D2*QzTM5C$R!T-z1DG4bqj%`)2g&)l_P;0q=;a*KfAzMQ4bp2f8eUV%igl~^
zIX}y{tUWPsYCyNJHjk06JQmymHP;ySf?Tg@n=@G<<r$!uwDm?u?*o^3!($7=mqV+s
z@b1HFk)5}Mqc#?6?+ws!8&WzYpk7X0gQs(zns>xCifm;3kP@5ZGVb@{-)bQ!BwQ;Q
z9fYivN$Q{MZz@h#ObCgfNFkaoX!(byoqV{Bz&#9}8`_1hyV!N}PkC9qTLf<Jz<Od4
zgtB6!<mBcz0*mxe$&>P7!M!mn7wd(Wci(?qOp}}*cS_K@FX?{dM-w^mki)#(!sElz
zz;1<p%EFISBVId+SiDy;vb4|^!Ge==*<{p#?6h=D&h@w<WVWA_({@=Vr0I4_zs>K4
zq_@Q|s~acQJHW>YtUjnWDj5hp;tx@Xod0hTyL+df2!kaqQ1wygq|iDWb8*xIrFy*q
z-4Sqx|Aw*qcg2m7hY#4s0Kp^X1UjAHOqBnU9Yz=Bm##U|^=03SZiwn-_6Wt_8Z?h1
zDmd<{XjW(6FHZg*@3a&gCFn3vb0W4JDXjr;{@>;1hf%PK<QFJqDi0NN0iLu#w6z~U
z@KJ8IvZhQoke_~B-E7wx=jN!|mi#25*YI@(`>G&~S^w>mqxnaL`?w5O0}@WQ{W}mC
zwZicGdb!@rzVa7d4T$!|$e-4hO?=e_igT`;jUD_8iB=C?)m$+z5^y%(1-u1Z1K1&*
zJn2nyF?%i3aG_l(O-Lg&VD`?&T1t#WVOqEjd92-z0c5X+#Uw*}wSLHq5;HfAIh>Bs
zX@JzwYa;}0K5b8h#<12Y3{Y7RM^m{%-|K-+){T^K$9|(VaVI+)mq5uwk`||RHfC@r
zx6sK%!5_FURetfFXc&+;=Y`W!A-8*pUrOri{tf@@PMz3Cja+ul1!e**sUKifvxoki
zg##Y#GC2{ovPR(Yi>bc=z;?>WrVS-H<!P*U%q<;<22#B{atXzN-l)v?7Xdh%d1HA)
z-o8AY92CEWPw)4X>eoP@`1b>SwSMz25);Fw?YRW*7deIF{P?c83ktHUmq;NT(aMe=
zHPCFWfMyPMT8);X%g%9F+_`Swk-2{}9*x{H!7f>rw+Jd8$%zak{JNSG^>C@48ig0(
zQOnFZ+)`7nzcHx*`TipO8$5_^w0Yvh*4&ms{Dbq2W(HTIa3{T^qCO(HS;?Xnf7NHH
zCwY&o=5lWB?(1`Wd28o@O+SH_6V%oJ*5Bx_W8I5ifDF=jyXkVQ4R7bDFe#wL8FzjY
zYWUM%=Ld6MB~(+>b!aPYviB50Be5rUbilCLQmBaRkXAWaq~gH`_4rmjeD4_R(%KRQ
z3P*_lE@<?nAhy)&$vS*`5Ne*nsM;+w!&A8Xn9I57LezB80O0zj6dU1KM2S+vP02ez
zm-qL9W<%ie?3PM!>V|$}87DdMVm`Cv-A6a{jme)DQ3Za1nEUfb4z?babIqm|J)s9W
z<iPo5u4g*Wh>|}Z38Eo`A*}|qI#xqc)$$u8sSnN1Oc4KQ*PuU|GQ`rm*B+iuMIz?w
z&IoqM?Jm*ri>EP*4*vv#I3FD@Hb<1s7>m?Y1Gs?^`83)jC^uLh5UL+cxX^`usL}w_
z*S#XKuiP6?-$W(o#H~V70tM4U1-sggh(&1WMNT#o9ww+nZy1xC?r#)8I=E`Oolk}x
z<~dVUl4vVrbbVL=;c*bq`FfX2n*j%C!F+uBpz34J4g0mS(&=vm>ls||$(qv=i1kV>
z;*iu%h*zMiX0EUUvCe=P>w7$C)?9PZ?<@Mm5AvfEysVMch2mWv00r^CkK5m^4o9RX
z)lN<o>Yb7cXIin#|85i+{^D%&{n?2wB=5)K6Ye?`6vn$Q&Bi!|EAXA|W8_|IxUyR`
zEHhzD29#XSO35tgh-1u^U_Z{<lsY!^F3ko7-O6azUVbZ+1LuMV&dm2UTZ7S?&hfPw
zm(c?jXP~0oZ0Up67tf$`dTn(oM93`hfZ+6<Yl?w4jB-o_>r!guH!dilqv`d4mUFE8
z_ZtYmh5lkm)0)0D@7lz@QyR~48r20(cptQWykN)nvBB-Rj!$%_jvMRkispe)Chn3A
zfU+Eey1|8Pji!|Dqvkr{DmqY3ZTqQJpt{hMiD_C)CDkzPebgl|@7ux2>fS7Zn0P?)
z>`gb(KFGy-+!ynNEkwQ;<qYI(ExS!<^Rb+qw0u5@$l9w^gsXG9939&bplS|0;d(1x
zg02aQQ5L_eHI%tA4dRSWug7IK0&x*H_+`-=RcSl777u>2ac(&_rVmSoFy~xx8B92+
z#U?sg3^3qRFQRms$`Xz&5Tue=G8uX5!fg7Hrl)C(CL+;ehg(wEflU3u2V(3c#$l4N
znHyn``jIyo`pVTMfkB+rqKVx2JWl6NU}Dy1u`;Z*$7%mPy5zddnE``p-J50AE-bhp
zmR0#LIZMv1b8uLycG;{)z;?uD9c%S}q=4G#(tCu+_Qz|{;c7tBc!a7Rjl_H#Ca~bM
z<o2{K`}B)E9JudJ;~3X~VxV1bIG-!v*v<;DYj132Wr%HTe=!L;xl4yUoG^@BJuKX`
z(i>tH)?OJmrZEiF-r0KQI2&ME%HTqq(!8lYu4F>k$49z;YEpQsE%V42tM|(E^)o~q
z0381Q6L!U&?7IDi{|zr%L%g^9<vp}5k=2X{l{XxaZCqz2#xn$QmUAgw9c!+aH9;Ag
zD+U8(3c2Cm)P&FSJ&S>SPqF(VtTvHm_thENyW1_UBuK}!20XGZ1lIENhaJ~nZ=q01
zii{a`jkN01lu+k6tDx+q9zDPqkln`b%ouQC9+NKg9~$eIFUo(T0;H=$zm$5^1THwf
zlmgDrE741+Pc_LW!$;Di&(=Fj+uESQHc}wej5u4?wy{Qf7&bl7Xj7e*?b&5}?YxVz
zftw(N*fmTX<Tk#}goNr<tn#)m9qrXHG*cO9svu8Xcd{9p;+@Q60>eXV!K3UN4L@C1
zf>)hf4$NHppr<Ox;{g`?VPP}mbl+qB%tr>DaG*HO>MRNm33XNKIG<;#@38vigwa?j
z{A>2>KHzb-ag%x}`Psy=d|!_^>4Q4=1rSP-IgzalN8BmV*?F#4ijz@C9cv)*cI4N=
zM2Q;F{|!rcWAZ_6c2uG<-SmoN#2#vyYy6>B?Br$`aE9k?nf^M%0T=)?#Y9NG31xR|
zWYI8x&GXeuyRpOHeks_MQ@39i^9$S&0bXcBSql3iqYAl|l080x>7asyKp1cb(-GX*
zV5|2pcos<0T#f6(8qIAKeVvwf21&VpAkCJxZ>*U*j0Km|84)TG%ztIbWww&O+18n!
zM>fJHHM!{{a>QTqBh9X4UPar1vrA}iu5q{9M3Wx69b!24bxUR1lFa`7BL-0QLQz1y
z3-7IQ^!quzJ=K_6o6{kl-4Tb?`#Y;*kC3Ax46kC0*ly34J?0DQTHdsHbrS#(@A_)>
z|EpDbBY?q_JynXX%2TW-Nocy;GIGX#O!%f2tE@-V)G{wE%5e~NP&BiZ>gQ~YjUf-8
zc%#vd4mtuO3ETwR`Z>X+OUFm5fiBgmf%wvR#wlbncwXO?RBDGo6fBf8c&5A`PZ(Jl
znuh`vv$sa!Qudh|5({D+6luj5Ji3LIVg#3v<nsX0N&jMU6JTNfS@P%%E(Ox?1Ze?d
z0%@AZ`likgwmkJ^?=EjDb_+3;{3zJcQ5>3kCI?#S$cPfi=FRNFj7r@@Fq|T=R+7vv
z9&G?TNy4;!UZR)V8#ckn)cJUsX(AW8j5xENOp_v$2`2kg=|&ls$Q<Lq817c>&L{CJ
z0~80iPsm2U<6qO5>`g;H+YASvhMAGobz7;>Q5B}kYaOJi)Gqjw*~i*XyKly=178Ni
zcL(U%kK!(X5yr+(@l+ne9!8Fuqvqby^`<KlW%!^qv?6@?UBHbWCkf6F^`n(<2gmD6
ztFGqMCOq>)r@n<8G}lj^-_5#q{m|n%0w6iUm$X1A%s=kZJ5-o8=_Irr(^T&FMABox
zO2)M6rfdRM``ZKp--#0yCyiq{s&CDFSG6qUQrAPPA3O=>q<qbZ;VQL5G&WEWz+uvI
z=`DKNCORslIG<OpoiA-9cDjE+)+Dw}vjNrfIy7qTy;VmFJ(3dKVCV;iamin}t|A>Y
zcry}B$(5s4?zj)^rUJhFwVP@_+f8wr><qH2H}A0i{Fn+o`q8{O>eQt(81{v}v^FJ2
zMA%{+0AU4w37!fL$8n#<*sc=XN|%kJMa^xR)Q7B|EAhYt_%!B~YNo}VL29nkUJx#u
zt%j(R#-e@;63qBmt77jS-t*_wLY9V=CtFr()u*aY6Z;}6B>J|ko{#g6p-1lyvJeD1
z&i~j4#U%%q_O91_#HLXE@^Z8*rXfi3lnI3yCC?)ScoWn&?2Z>Z3zTDazML$(9@@xI
z@0#o8l=tOB`t}b#?LDo(<<>aY9Ycc@rY3pB0&jKJBNs-?%TVnwtz9X+etTosy3ZW)
z(<Yr_*V`UhJaj_QEpir&N{LIGuUKTE6OIl6%HV^+yxtPCS$v9n_8WCNfj8LqZ3L!w
zhD3bpJSC0pm@q)MXDRv2Etz(KA@b)?%X_Yb6IR{P^5SI+$>e&?bfqnGpuI5-$dtF;
zwYi}gXMw7;**tGRXS$xPWIIx>KW6)4FUJ{b(qB}1&=g+<MJ{|1o2}6;n=XMs>VJ|i
zX|$n!BY1k_O%4F|k`m<#)&1hkZXetN=1<6MHoU#`03WLF#D+w<+}P2f3F*JcR?2kY
zGEG>Lc#nFPMEu-szDKj!I@u8~cFsLM2kM6A3{j9-qu7|_AagH^z7|QL`chgL`5Bno
z#J_Cf;==9><7F5c=k}>yQjD=EC<h85a&QyuMP9DvUyKYkW-WZGpIZ&=*R)-Y-dsJl
zZ+Jy2bTavf%eDH1(;zdGp8N0L?7xJEOPAL4q3;njTbeL>uA+|-6!zCVq~DW#tla3R
z(OaW+DK41BXZ&n9ma}#1gR`^IiBI?V4RLMg18FrXEaEGB!&8aF+sn=D5E^kEC;rPq
z&}Fjb<wyC+*z`HjDR0&WmFib_qg20ZkJ}?Xhs`TGzf9f62hqD8eK8!&*}5(+ouw$`
za(dj|7|3^0NTLn=Fa2vAE!GVuB6OyhF$JEzZSDLgol%jsuQ^UD!#;e4&&Ez?!uEHr
zr<gppNu4=HF2xrNu+ZL$-bFnf=?-uF*)3J}g5uvm?;VYEI?Elf01DEMEOGPUKg;<%
z!+7;I)!lxy0@o$Pc!BAqunMdD0}f_+K^!3G(}@xOb+=w{f9ADIu)<wY1tMfLqWtb_
zC-5t1U|IQB2)Z9g3I(>(1Yvp}&w+0|pLgbmrKNX9Jn}Xf0o}L3!k77HNmJr2QK<ch
z0u4urc11w>`%fSxviKnS{>IlS;?m~q?wYduPw6q5CHH|37{Pid4t-^Os_dxE()K5!
zU;0K}xkI95$po_+vfR-kwuY?7u~)VL8zjBqbOt2`OIId$A-Rge_v(^nNq~ekcWB~)
zt>8(U=@P6pp^;!H;WhYEdF*5o)~b`~^%&>lVZ+`$Ft_pd%J)}rke)h~te0!vN4`>d
z9s`J6fw3@TP;JP-rowuWq;%qBxx;n6iV4$0g`>?zOZ;)^zdk?-NFYCl-jjdG)RV7U
z&yVU@ZyI;9P|=@&b;dpBq<38Kp8O13Mu>R6#Eg?Cj^=)RN;pOKN^-Kt4hF+HI}sck
z_Dh)}sePvRTzPa9Sncyq_G)@Lx<pdJCE-cT2F>HmgxVt|S+LyEJV>Rp-FgEH^wR@i
zO+H(}&=_Xqocj_~u{5NmSA$Tx(v3l8Y>%g!xxi`F5ATjwSAO}gs`R`<6-|5!JXAh4
z!HXrWg{}wRaY*`%fUDp+v1q%5FyQKn#`2_u`aKS*P)*R>GUR~^L{-0O*_-XVr(<QL
z|6M3UCtJDEBs^C7duNss9VTkjCDh*v89uT7`UW`u0H)i9bD7m9g9tn`hf)gwnQOa6
zF^iPpFh{BbWReDtfK`bG;Ou%ApgK11j*9%Uqw~TzLx~-jq{_DQa_LWzPTQcdF$xyc
zTz9~yLI>z@8y^7(s6o)D;SM;!%khfomw?#+=sq={c}T;})iaJs4OOhSo1?~MGT5>2
zJkp;L)w&+VOMZvmo!3mgX;5R2NPG05*$B*jG1X3Qe?tO6eQ&%=xucEq=fLAZ6CHGT
zeQ*u!jY}twtCYP@k1FyC4vE4M@ny)V7mz8!Mv-Ex(p8N{Sqf{m6<3#o61{Jq%Qn&5
z4v={f)<!KjyW^@XQ|oW}q?>17KhB@K`PMHc-Sb4tFX8hpq~Ls<Vv{fP<?p%jpb4bT
zeJM)M5chQ`f3@}HnfFHQ_}oz`TuYg0TLAA-J=lzh%(U0~D`d2qS6;5Vnm6v7H_gXx
z+5rWE`{oa7B8*9z!?J0MVN+I0-AM=#x+r$CR|uGlbOV8D|3l9=V8q|;?<yKFvctM6
zWZ5Y4=eF=({Gv_>eZ_o1=IW=$`Hnbp2H26#QR=CV-WG-QiC%<*<ci}{(|$;bVSoM>
zq8OvPtMetEE@#aP^#4&QWg5I1J1A^FHR3CV(L_=&Ta@<NLiRZ(T3O^y^j>Km!2>9+
zt1YNa>bf`UohtBSh|v+UGci|A2}6NJ2lgN(*5h3>Dm%>)p^6EB5Np<MzJGWPPPEKT
z<i!*_&Q+-AVrtcrcq@m4@U7i(fuyrTd!vrp_AeUmUZAWa#Xn&Oz&+vC+3qt=*ehwV
zttJ@<dXXjRnT-fS>kZ5~?z_9PD5?jaB+^B+-1}M^HkENE>#F<UI}V%tC6{VB{b%|@
zTNx=9tgod7+D-4i1e!3g!p{eQ&_awklJ+1&<C?sA-L2s_GH<emnADc<Nfz@YR5!h5
z(~)HL_?RA0-r#^Xi?sHTbOm_rkgfBi(XR<pOF*)cX?lG~%k;;qYqIh`uZ02|uNu(y
zN@LJBmWz<)MOPtq66e>Ym#U8f_s3OgRD?7ax}$iQyK3f-D20rADJsqc&+r-+_502i
z$6mm^2UZgwA9s}Gw_KvlcVaIic*7~_&CEzM!tWvjQOk$D{qOX>6(|;fg@E)8n<G~r
zF(|agcv4V+Iub-g%;j!|qY3txiz-+9t>k^pY{%|Z_#K)%-O_g14U7Uoq#P!~>?|-G
zNijyV4+Qeuevl5VD#N(lkwhj8TD*qTh3~rI(&p;Bi?^1GjIO00y{}q{Kd1q*Yg4IK
zD2gaw_d>XoDLLDn+Wu(0RVkGwf%*VBtS3hvRIub^vD?%Avq}YuR#n(pOD9W18^Pg{
z8OiA3qaqkAu1DDPS^DEM(PX0Og-cP*z+^|cm-CLCxlMvY16PwOMI06cM){2G7)v0!
zKAuw1#d6vFp7PKqb<LR9bO*lt!x@&tjC_eSXVIisGD|b_nes$D3yPA<w@Y8DfPHyX
z6?ZoO4BF2`GXnFjpYwX9>~?Ivk1q#OhsBlmrb{L~S@~g?5U7z*o!aKJS<G%5NWLdr
zKyqlfExGyqR2%kAHoB3oROEp=LJ7^Wri=7xM4awsKcSa{?>IK#Ow{DRJS~P_Zs0jB
zXK5GfcmH}R824Ex&Ed}|^bw?&ltOZk=qJJP2!FGV-IyM$U;jJp%vs=88yWS$Aplk?
zpSU1np8<(KE3#8%eDLEL*{qs-uEA5zh7a)lj_5uAp0J(iF#zYybU+bDTyz;A1G<kF
z!QTlIiE9m0+27DpK<6H?2wjB03XW*?U`bqW2)A16x|fMsbj^$?i_&(*G;KmPhFKkP
za%C%((<F?6D?tx6;(H+@;tgI<S|);~t@%@XV-)tyv6nj|9)Pr136nj_<;VXzMmTL}
z3(_UA`WvU-Z#-g?OyBz6$pjFr*W|w+xr_kpi>g@->cMF+zelHzA6^yq@#TpuelQ&K
zty#KBPrdO5w6S!_dy^$Frh$i5lO{L1v`ue*!b<z<iRpV?+WBSZu7AoNe>n*<EO36p
zNmx$KWQ&@@^r&bP(v+L7A^=5(U6S(y43|Cy@|h3w^OItWFFb%Mcb<l98n_DFe%Bor
zHye&(07l7K6U3~Jns$q3o4tn#uNf>SFx_xl^qJ)m(4{=XbSXe&)9-$if7t_@J3lRd
zcOTpBA5sN3<ZkboRH3JhRlXRtJSm;>aV&?($`@>$iE!M{kr$dHV`b^1-HF7gUz(;z
zqUKB<fr{V$QYtX$6YzP$xe^>k$Wf`zOc4k;u~!@G=WuXVw-FsBDia>u&A($*#;pUz
z2OX$Q1~2>%PC0bVy?M;R3c0NzAMip)CJXCmH(?^Uip5(cH`e|b-@D*aA3mMW`cYht
zh;7^O%5^sHI`6J==$;?5oyf!GswlaseXF%=M6>a>lZzO;zCsSHK*<UFTSQ|gi8&$`
zCzYo1<bBKFh_|^KMMPdK!_5g$fHt2$>d;YhHSNg*(^kHBwNV4Rl$UD6SC_S!7%n~3
zDLbUEbXuuw-t8??3pMP`lHj%&zmKBXc(@ze8S|La1u~7p<+2j;WA^(h26quvJ68pV
zH8#)6{+oaKHI@_Y#m5XPFe(e_Q*|mjH!o5UDsonW_B)+ntiZnErL1n&4@o*75$J}`
zYH#A_So!)GUxxa~0=~*V@<knW8O=&i;Q~QGKV|fa<5P?-#;f?+{{ydL3ezSk<W0G3
zuG}BojrI}&|LwwS;$Mpl!F6F+Cv>&o8s<BliCFQvDp$k58x6jKbmc$TT8L^O8jkCp
zAU(dk%LZ$m;<-<8GMSvyZlc3cxsFA6U0u7`au%W@QK7tgg-bw1$WgzNYoE}@#8joY
z792xZ>eolCZXNO>Fh>2!MzgE10RL@$37YAks%msG|71QwvSQbaU^d-VnAv6P*h=rj
z`DFduJIrZ6D}U~xc#@F1{<|0Y{4H0;IlEs&*Bt)Ost-5*a;(338$X|^JJ_cLpL-kc
zIFh8c-cufpaL~l6o%UaBT40MGyXSVn6F+*6tiG8$frdqAasg}{E=j~S__R_iR-j=+
zEr8a!c;=zpZ{3YeJZe)WYFR^{BfF@@0EexZWPMab9X^Ak4)itC{Gj3qwJc2vtF)_F
zYTw;C&Pha4)?mI^eNQV{rSWW$hForRe*RFBEWp_=h`eL<zB7g1iw}r7oXA-t8&zW8
z7%oDkL6+Rp*&!m*G?QM}<esQ~+6^FZA=5A~G}84GiE{-X?FH@;K+hZ{Uv{0VvSyF&
zI<Eh_M)~zO`GC&{pe;jK&olQa1#>i9rxd(^&$W?C*uyW-u+1oErX4FL0~tu{2F>0s
z^-mK+H?%TQ{Fm^=rCM8m?jhiW-2OZp!v7y9WWWD*LYDAN;xP<Q2b_@AU1IHbG&>bW
zAI@4YkCs~0aDkg{{ay4AJVaz#y175a_^DlBq6OfO+MDH&y)tkUBmZZbw@5#(njC7%
zJn@fQvN~CRJTI{>S4A-8T50JDIvT0>J_(5ho|Wo%VX50eiE~Yd90A4Z+}$PHD@e{p
zv!S0F_eN2|nAhk{+4+4HL6E}w$6%3KYVmXP<0Fjuacv_2L*VVJ?%e3SAb4SVW){)U
z-T*jPsCt5^gqI(DKxi7zL04~k7lIDjddp$6>@{-2)iMdw|D!`GQc&NrzJ$kq!EF{r
zId8*OL}`UBQOpzRdeDXYL?_MW>o4bH8LVsncRG}49QxAJqo8koG?h}b0sfp32V%1E
z!nd1bPI8)J5#}cY{6+~3?7X`bPOdZzfhUFq_QEf(x=VU`baB*#rXCVpah2c{vXFqC
zY$Yc&-GK77N)aBveR?#}&7nc6uXPs9;V*aoP27fmqxr``?|=T&{ma?<`IxK-SU-QY
zJNG@l!ryu+?&9BMCo5vtkqeg$c_;`kI&Axa)qOMv8s|gYJR^XBRM0?LI1rH9(l{hh
zy~bQtTsTW=|3D>1m&fcmMDIcmVIM22MY)k)o06>hK!0Z_YeG3C&(|#|U~18l-q^xK
z(KnJ)H*jA|2yl`7ZxJ}KcHjYgrGOh-RMTDSw83VxEm37@M=W>3_IPFIzGJv_<BGrG
zc+u9j-w!K;W_gQ))>k?v-yL%YA;)m@mO=W5w$EdohAftf;UjV*@9NE_w>`Tu%pAij
z$q^#DnR~BpI|vBYR2)$Brl<_%d`q;$ZEiHsDW)9&0qXf$Y$>1f``KEUe7)iKmow&H
zeLAl=b3UXkpZ=&I+Q{(FgEA(0_q;Gnx-aW*P4ZOKk@TX%H3F=A$-|ArhZcx8&3TDf
zQc9myGG9HtxG1zuCg|=JbXr*$8}lQkqSah=_~-jB)L7i7M+RLTm3Dh;YjJz=0+~cC
zdTx{#T`mqD)!~T73tYTsviP#w<~dXTB@s72s#r<mgPAX{OoY;Nd>H2F8N2ld>nKK(
z*@u{MEb1&QabxqXW{yd(Vw*W6f++Vz$F+$y=x=Q?YF&7+mcV+MX4f0PF34<{)08Rp
z3EZ=l<{iB)$#8PK*xL}iLs2z6`YT=?jFrAj*zn1CStfgaa*$)Gi*NL6vt;?M@9ffx
z$V~<`H2LP2XFaUM&QdtZ1Pdjo!H$$**Pxzq)8Mtqr=!HC>5(6*76$AF@2eTLD2m%0
zpMVcD*C*4fMlL5$zrv%3JNNmJg_2+5f^kEy((mC*Je$t;cwS=og8W|QHEeA62jXt7
zH28JQ9dfZ(RY*v%u`h~={{9a<GAyQN8-}TGhBJc{m)_{%e!K|35PJ!4@R+pq(zjcL
zk8ph4TA2)e1oKqX0#Yu}nf4Mp-|V(EG>A~>j>WzIAj!2vjBW&y^+lP%^pcJX0ky$X
zdq&DTfRNSxBZz8GO3gnWw-uXUyBjR3Q6lLy_^9`inF9QpP5ylTJsE6WhWtxWIg&K{
zUWx46AmuQHW^A|-S)F(iY16j=$|oCmYcH;O_&lmhv9BZRuaRsUWDw~(aDSH<wP^pv
ze~weCk>EzU{$@L9iNWHOe*iHox=h4E-#-H*IMqrtk=By#p4j!!f2_aw_P}AhTlq))
z)T-Xe;kw8YYNvkDoC0jIE>G1Ex{NSdkSA}qojSm=UITF~QboSfM;b~VaY*9&dTbrA
z9%sm)LzJNthdQaA_&&K`-kv>8JOk|g(qIzMH}w4vd&62Bk@2;4p16zqo5(34Dyk|*
zMm#($54Yd^p>px;iKj45WD<?3ej5=UJgH-Hd()XvcA+xL>mgBpK~<Zvdk~tigW`?O
z^G^n~V_5rd*RZxaRNhQHXBs0qF35L<;W*L<WJ<>A=-E!Hh}Tn+Pm56*6Fk5Ak=5U^
zWY#vrlW;p$MUCJNjC6-l|HA8dQ2JGlOW88_-+$F9nsq6ow;_=$D0MgOO=iMhr_gR7
z@sNDqok9YN#i!?IBQAA_qE<38jY~8FT9#b(={>Wg;iH1r1@;og8WnWmL3d?ic1=GI
zNmiTx;8k9wf3?aq&l#I;;Ibgi=#X!1)*mj|zuY3RBvK-hJ67`TUd$(zifXCruXR3>
zr_1&DAca4(JMk%6cNu1>m9W^9uiZE>LF!Q#zX|(s|2|42V}X`}Lo>04;z-)HQ|DRC
zp!vY;Wxm0rT`l!jGQm#<D<h7|VufAp@|7ZBXAgHO=)x&4nTO=pgw_qZn)cc6o)jLE
z`|;vbWxsj$XCH{B7M>Sl9k@d9QY%F_7XPKm%Zxa<Ze^Dzi5cNx%lzGrb5|dzUAk+1
zzoov^h?tIHj1xtz3g2C5H|=(S;dnUWz1sI7&QnyI@QT}i7pLcK3$l-AG!B-eyKF*q
z#c|=m^UtRBj=KhOWF@A!LZRGW2|b(i_4OoHMCp$o-Sry1>0Qg|)$GY#DkwFWp#WBY
z@&s$?>9=ba&p!tFL=sRXIp`+y&BefqiO;XRrBj_kWIQB;k0cg6;~B_q%86D-9SN|H
z<lhO0^R2rCXi8|R!&@li2^~X%Sm<Q+DOB#r!I_=j?!LCkiTYxC<rri~Nm0*>Q^!0#
zy~}1KtG645vt2*W{QY6D;Mt+@jp*Wu*)42%{$ra@?`k<B5AFO&Ko2|bhWq{D1@;hS
zkz{PZqc^)gHrgNjndp9qF59*D*rGwASF|teV?S5u{)}J3HCjuAYZ_&d_@$hEML-XC
zNf(z6yBd;6crag*;X{YLa|gxW0fJw*1zp~sEO@dW`sN`OX;FQhr+7TBhx^tW0i}Cp
z@Q0W2lW|6-Yim1q?>xfGm!#jlKyKS2-dr6#?IsG}<sas=HuUk_S*b6Q#3m*Iy~C>$
zXZoL$qaz0!F1KTmG)gm`S9ffCP|9u5sVSLQ<GjGuMiU;{75^9j8dg5Fu>_RgvauM9
zi&n7^BUOVKZH3MBwHCeknZT`PR%pq%>6Ul2a00)}tCQ2kIg<a96jZ%oYF%(GoY`O4
z={@6Y$A@<6-rY;1i>bPt+BrKiS7z@%Mf^kyUiamgR7;ZbV<lgV$<*n)!bvLniU&`(
zv00>A%v<kOcd!r4%-Ds7*RLrss_EVf9Ly2j!i~3K!OUd;Hd>L|mK;*1>DU1rf)$Fa
zmu~#w40q4I=1#(Fzj1?|(Oxvlxejz8G|XLa1B)?3l1dZHJmt%cp$kt5@lZrPK~OP^
z5=pv!+!cZyl7%Un0O<1{mmQ1dAHKp{{xP1rgS<}k@aAriLetw7PcI@p4=fry>oFqo
zZ^5#3SKZpwCu1C}Iv9;>>8tI%uep4jzqYG6w|OUgFsY-Q_MT4OyUu9E-PKVa$KVGN
zZWLTzlcZzhQEK?~?$3BM-tuOtZ>ir?$iUWD3KrO<9T%@4T21y!QQ-|CdCnA$%zD}S
z4Pt#CO=&h<B?iCUgCb0;8S7QB7DqY0F43d9cTL01a;8Pjhln;Q(4@2nhL2hpQ*$;5
zID0=oqDdwvo;~@$7<<dVs@g4FSbEXjut+HdBm|^WkZzC;>6Y$pB&4LfyGvq$NO$+5
zyQKS_KKt39&OX2M{sl1Coc9>lxZ)mX{a(9VI;LA+TP_uYq1Y^DvcDPsuN9c@KRs`>
zPu}?*1kMZ2{*{|yXbU1;jNbqO6uAq-*}P8&q29kbTE!k|oFIVzNb#fRR!(4rMW2mf
z4^m7x+~=-Uyn4f|LctY&5)*MZ1M{KyPz*R2DXawd+Hy(aD}D!-H;IjAd<k-$I+(Da
zI#X!igK(xdEG?KKuw0d30Pd<fI6Jcsx8P40lueh~?N#a-{=a272a7)-vh$MVzyJI@
z*vBO)5I8C<UxNfDd8nWbFQ6~+quxF4|K)>#5ur2&1rb5uwCk-k`WT(<r_swV0&%;R
zVG?JoUg%QbuAnr2q^~@;TF8|_&2VPZFxq6$H(eOr#}InI%%Ng2VxI|$1EoIicZB}V
z3_Y0-stZx5>87egp!~C*z;5_=Jis&d_aK_c;t1D7tK1rPBQ5rTgD|G+&~Nfo7YUD`
zxHeP_b$0b4nFuyYA1TTu`Q{;#w$2H$%vyLThAjUABPl93ctYQO-T$Hzga33>BJ599
zoUkm(GOmp9F>k*8yrlo{@9=*VBjMjzAd!GR_4n~#oDD`O(qEBj?)G^INl{4U1ilmL
zDTkAU>3&3L^v7_&?hQiU%h_aK$`-gmC%uGljbGfxe$Dv6wZs0Sad6tze=`cCKo%ch
zKj?AWORE&}j<TPlnjc#>fc_O<CZ6>IB4f~D#&2};reJ}4$rPI$u~k9!d10P_KD2P?
zIwGx(zB{%U2n-Pj1kNhU^JLq98&B7N4)Z(e?|K&SIxNX20CLkHao<<???e9|z|232
z3d;yShRc$GZ`whW{O9UeI$ZfvAYqpm+*Zn)SK2#oDNQo{76{zu#Q->)o*R8}gzCaG
zTw_n%C<`mn{#jjO1(A1JWPFr$J;djItNAb-m&!F9`QeC&zpSDBa=jQh9!3fcPa)H}
zbPzUnfW{oj|C<Wfxm;2l3lZDHHqZl$f5URN{DU!&87sy`R)Kf>&y)H8|9CQDNkKau
zwn4RbSuWMSBOmWi`JZPaK(OnP!j1i)!T2|!@BL}|;ss93C3c5J7V28;9@14#Z+?9d
zUvIBfru796rU;Ne-K_c6Z+69T@{JW7G2z8baP=u<=2jqwPSqz5jtf(oT}v^27peje
zPvn2$O>){^Gt2Q0iW0?Yegvs9t0hp^Zm9E54BB|I_=ZjfQpSd3^^+#XQ`0PGWvQ5r
zD*v%qO%bjDn~sRyeci1!jXbTrAl&K<KnVMb=%C-RKx_e$R_{~D54vJp?9uIO;E??1
zp0GvW4xhZRw|MuO<xQNmr!d^-yAc^#n$I9yIx)PU<Po2<U3_EqrEUTngBad?p7BxO
zSHAK=CNmLlCRa#63Sy`8$3udL<G2_%Jd+)VBKKhU%yS<mN-yOOPjF=ilHc@sXAg7r
z_k=tsea;BQc<KQ@*hpqA=`v=mLTPx+U~wW;RwEFErkP#|_{e(oJqZ7M@`?Uw+?cuP
zExE;-=gN2)^sL+JlH%gD<O=+yG2WgolLK3{AtObc8mhKTnhZ%O1kpKO2q+T^Q$>uQ
z?|5RXL$6Pl#0XiSRUu&jaH0hM$mVHqv=CahR09+}<?>9%>3;q^iZ4DXKo~@6?_I!t
z=@$BXR%(@mEo1gXwt7(ia$gi7cO??v6O(?DBk=cUXHz;lea;ys?QYeAG<|Sq8U|U9
zn7+l(*vsL^&$ZA-?1>e)kV1>7;GyK!#z`(@$>^!6wU^AocGVC>*p<|ooU{Y=iNKUP
zOAc>7XC-qq*^gA9FQV{|Erq7t`$GxcX#QGio5gPBX*%sjE4GUbr{;=gXWKNo3@;Km
zeVILdd%l6cwKq$A<t#jEjd9B!ultGXO+)#+tYN5$f%e)=3ojZ4oluuEP2;VWK%KpF
zZ_m1bBKYM)(qXFC4SGxxXO0be0z48Rul*=){h%S>3)Jl0T8qHOYBRxrK})_%BoWdR
z{WsI7*6V=?jixDA`-S%D*pPfT1Adt69^o20v?%{|JZGL};2k27Q`;DPuQub`6B#rR
z7~BK8ZaxnY4@(dDc=+17uVY-n#W_J8o4-)z|M`-g-sAEcMJ8nr&RDAW2Luxn^f?WM
zysx*t`pWCdvK`Jq-}Cl!t@82xx8g7RU*%`ClAWmIOl9w9te$36*MW}c#K8ffkzD0h
zZM(_v$P910o=C;Uh4qLK0D&<XfeklU{IDq1JZ&4Gd1UZuRO+X)rqhPs=T~`6xS8s~
z`4VT%4TjDKU&YYNGFXY<ZWyx(37ng;0mW=ThdA|$4q|*gIC8ndN#z!_lk#231o5Bu
z6>>L7)=lbPbZRF(LaII;Ki_!xPT+a*N49)Ft=o}`_+$rS++_$8i$}dTjg^<^S#@^o
zUqBE4${*{s8)X0zFuA{ZMgIyOJ)gmNg`zthZd432BcsT#v0QS{M^)VsV;mFy7{V+}
z%=l|H+D0jq-wQ(Ef_3W`tx_qrV7D%EyQ26WDw55N8S-QReTD2nwj7hP8(vevUUw85
znoT~J_N*Anj%qE8!uG&+I6;WIs}#fE)SE|z0|`_@*-ZEkgo$SWhWHKEcGUlla#6sr
z(VB`3z9Qi&z<I5U_9=CLUF^YPb@-E>O|99O2zpv$&LfkNq!tiwwn*i_POg`&Oyx};
z`sQs)=O#`VSujh{UOu*;g8PZ{96|`|;R9e5MV{WzAWBO37%U!Sc6gkC(4WZg{%=D4
zu=p#<W+$f#XcXV+<2_v(at}?Y*gb#LZltIm)ZnH0+aH_%>5oyuVc7hO#SS)VeZ<aw
zoc>qdAZ3m*9T+;Fu8=#FYL1I)lgd|KxzR&RWdk&1Ua-!x2g2Zv@zu;w45X#fa)SAF
z0}-$b^o>Ee4>F0V_;@tm&NV*+8oYbm>QSgaFr#ye_<^~WLsaHpi4)vPYJ-+2$GD2k
zgaA5a^I5Y`FjrOg6F$T^MEvZP>iV018@$;6ZtyxJ-!qmPi;q=YrfTv+k!hoT-k~Tq
zXk9TX=(&RQh$^%jqPWU+Ay^*y-%=Y2x@&shuAF3OTa1m`INgrJ45q#py&F<68PpZK
z`$L#tvMH(MqT%>^`>Qc4d`wg)Y7jKe;V9q4oZ9{a2y8Z%QFJado6ol%Y}GWmN)x`n
zxH6N^Wimt{LNA|LxJFPZJEceMmXEB^uddGcIp{$BZ9(77ZmL_vnk&PXJ|;L^_-=9-
z;H!Qh{(1lVWf0e<M9=xg_1D2S&;8(M|8L{Fn7N{7XGqL7_yFZ1Obv8yOvgxq9V&xX
zDCRqQ|8Sgff81B!5sH|skmqPjs1YC5d5Ii_#@Rak9@tg3AbftTpyUNYWzg?{o>c;8
zALR5es%}_2!taD-{g6yCejwxT#Vpx_U1aNp$^X29@e!Ukuvb%m*!6+pUSpjkKdcMJ
ziuN@!IFfQuR9qHVP8YK=Q&E|$X%yfhbNsh)JN%EvomZ9ug@HwS*0R=V`YxX(N-L=_
z@clJvgSqP)iQR#+H}V;>F0^Liy91KUcC9m(KkwH%EiOW4)E9=u#l<J@E*?84%UkaM
z03c043j`(>x&gp9kIN)GCw4O#Fov;t_zE&H;1UE#DjFoq17i(k6c#`*5M{h2F8!D-
z9_Mv=wWQ0;=^-GLj)Q?{mL^W+rdcGy4vgmZ!Pp?B03_R^^$=1X__zHRFc>g^Ez95!
z{y2fh8D_O4#|5gY`k@>K%wi6jXTHvkV0MpTbnt_+r48mN+*}^(t!PRM)_97IDx0@R
z5ixdM#s`Kihcg4&uZRi+&ZeGqQH^L!M<drq@&108Wafc*9-zVgJb^|VL*P>`-fWaw
zC*-xB2mpbnXahn;siT7bA4MQaLd&8Cr=Zv;k05CvNI!J)&M_1I8}AjV$^iRz!^jD_
z_Wr4a<6F-GH_5K;uls|Fuid_Ui^~4NgwN3GhCN8G9X6W5FEJ1rJy~<yw%?z09E}^<
zpZTp;k_MEo{gp60hpvi02X$9IqmX#)XRn&N$Mf)KUPtN$rlLWHT%~7r2R!X#l8FAy
zjB93nG06?rV&CSMI3f5LvJYPsoj5b~rLcWpNKHmk6H)LOVmE}had2_>k3KIyoaz7R
z#VbH*AW`0DYG9ydCw*<bB%YSUnlGR|d$;vg!N1!{uAOeHwUH%5z+1UNSDZANW&G}v
z?90ISL=*z%oRQHiAqB~s(+T&e5|>DM)qE#+-m)-#&yBf>wJDzhN2c1Cc)&B@^Yvqk
zgMh;&R))IxJE?}>HvMW9;OznZFpQH``NQW_Ilfsw$yZ<hcz$pmN#ibRu$r4JSl$c^
z56{tRu$%xg3vj&CinMOx#W&vK>dSP}%x0=ivGu>gp>b;7g>RC=3P>t95`K6^sN1Z`
zv|)hdhQ3Maayi>DX{ff}Y#K30>*8`93kZ|CQjB--quB)SrGH>(vqb%*I#=ho%Zl~;
zs~lO(HYYH0W^LSnr@6<iM^x}pcfxZ~dW{*l^f~br=h}nv`Yfh0;p)9#uz7v`?HUR!
zchuszndlQ))qf1)zsWxvmG;D-FLggI@|N_{=Nw8A_^u;?!blN3^W*D;-nQ}5b5Fgy
z!s7acgNuJzyaCHDD>P{%F}^>nT(#hf5|iFL*50j{LX`v04Mih%8zK9k#^Q_3G6TKW
z#wdG^Ishhiw#Asb*L~RUs=@7O9Y0MOWXeUhUn#%WYjVu`4VQ(SP9KK*v5P~J)^@E_
zbK$@>zG)0)^mBfMlZ%LV(d(>_sy3(JC+@b!GGsdE*6+@GPqi<#+hrY|KL6UaTd9PX
zKHF?Z)bGmvvJR`-;b!Pu`DbE%^HaUbomQQ3*0UFcj(K3uYw{0qw6i>XjEsQLjx&4m
zy-kSKRERFpUeG=BP9)R=&;9(#n{z+k$9Kc=bCoD##G7;lZx(8cn;heM>!C`gs`lWE
zo!KlAuaL8a9y9yU`?q?PWf4;kE6E==pVC=ThA71c4pIXTqvN4%`9|SH-cLf_*Xhrz
zT@5f3wUr6bx=ki6Dc_K~Qim7P&ZUU@FM4S|bzR-kcg(8IVRUF>_U>-KGVi^J(wFK~
zhPgD@FQ|Lm3=tpA7U9i}J>D(5WvR@Q^PTw2ztZl~(Wv9Z*P#9W?fk)FCPY9zBYsmD
zmml^?d(C1v7nrC0hyDL{QY}VR*HW5K<hTA2X)fs-F~}@;w+Gu|Cd?2*q9vLb7pVzE
zq}@AAj0a&9?Y$Pv{970_-&;rj#&HNn02kKvx1S5Em&l}G)FUqf&f&0Zj!ZJIAAyiB
zntiw%1NMBE16<=Z<d+OfeI`lkLr-H=1^TlP%!6i%y%AqqQ!p2nyki+ZwVP}*2uzY(
z?Y%C6=4)pZavUV)1eHtUnxRSKFq039Rt$Ol-@#ai{6Ax6zIVq(afO;z{joLiNc!WO
z44zD!ZXs&w0A8nk+nKQxHveewwfA|ir5nB}FpOq8GvM$ut<5GexmZeIFPvEmTYjhl
zp<8{F%^!(3^Hfc;nfYP4>f08|(k{9&D!_TMa6>jC{LbR`v8Uv>gUAXf^JP}J6}ZbD
zuf(=qj1CZHGUP8iuolZSSS{x&GbDw2LeTSo>&-p(CYe-=75!H|#~jNK=c<a5^*k6W
zOiK<*-&@L2IIU6t_bvce!5bKpvHgx38*V9nFF!gTU>#|ez=U4GQ1766?&|4_)I<dT
zI&CEVw`_f(JE0;b8I5cyWDkdVGF?I6leYqEl_)C@zFCY$AGJj<!73~au>E^6dilPE
zFQ%o=&*A=^$^+6f&OW2Xr~7LQxW!|kor=*6%hG2FPkfEzen?mX*Ml-~)Dp#>Hz(88
zhGK)2QZV}7SNL-2dr@{;c~&c}@16H2gQXJ~L~ft=7T<3p&(<&AX%R-i$LQa$96H~+
zViFxRVBot1^c4%e6!h8JCL9Qa1lMc$rbzQ)y3u<;b?=1J5T}27AoK}}1i2aCa&{Z#
z$N!wWFl8zrc%Y(q5)7?2Uu$$JP&I}S>1p`<WNvh|(~BWdXA#qUbwxKcle^ouxpQCN
zj~<nI;6$@MnzAxMzH>E2A!FXF&W$`Y075N(vbCw~N4v`?W!#@8Cb^ljOx-gn%LFqY
zD3JyS#}9vQNa@IE?Jqf~-LtjP%;25roPDznA6ZPZ%UU$>k}nr@2!Q+pFYgA`UBW=b
zf%OsWsVQ1cgSg9-(UD@)&)nG2^bc9oMzY!+A`&!h!pDbd69;#eq-j-jtOmFlMsM;k
zuF4J`f6i#{9WtT{9JF*rKAd@uA@yixFtsP#*;CkioQ4WbmAN)ee4+NF=b2k8ar~C{
z+XA^C4zt0gSO`hpg%X}wrzx=`t7HB!_oiUDuVl=A7U9q1RzE>W6+PwnhXG58;s^U9
zGFUb$W+3Ru$H)}=zzv;Rd#YSx#cZ&WqICZeaB~drtV_&h*HSWjx46BrcaHH5*|-)J
z<dRivMGK<#Pl70{N>KI-XFj@=j|ElU&m|Pn^KnfaH`0;iCWpw*y8AHZg0NUuSHBzK
zm5hD*&7$=w@8##CxVm-D)lZVpC!JR+UPyv-1;JO|le<F`PH!2V6Fx?D4m51<LAd7l
zaCj6lK1e?KW)$@EAj_-Fv|0`R26p_d=3cl_EHAm5r;4Ms(bPRbdg%9BZu+JqS=XfR
z{<!%xb>9>TjTQ(ISl73WZ~v`JSK4b1)wzG=JlWuX5I&Q=YcZUHHdF@eo6&{ZWw@iJ
zbQl}p<<IKU2vHCwV-RbO!fS+L+k=la`@Ucl)Gp>J*xKzZg!I7+Rg0w{n5Z0?)Qz&)
z@j7yz<!<@28wU6rI9?C=8J_V_q9IxnY3k<a85keL?S}Udt~qap|6!#^40+h@$d&|e
zepx%~G1<!L^XHQ{!Gjo=V)5hQ7{2`%>H`Swe{*@e_2QohXRPK`HJ)$Q5J-eP!cfw?
zeEFp8)+YZHE;&5`^h7Ebl?kaIzx%}m!qt}DB=TwG$!NPCxfZ~5(vw)i$B+oLDYkIQ
zO$s0*&04F{Mve&_!G1bpSlQw&ohnKT^{qS0YK_b2)im!1L5zU~a-3eb_ER>g?nP-G
z=jZ3sRr;NrODT!%tK_nC(>B(nWEXAR6*&rEhczB<bgku?&_&>Wy3O1I3s-|VbrO4D
z?JM`*wTEdVdbeS+>kke4cTSc)k*S@9QMqgs1$4DP;A0RrLzGw4n|QzTK@pAJ8?WB$
zJ%F4L!k8$poKBWgA_#AEj{bx~mFmrv4i@U@fq;=;Epa|Ql^fQb*qPMn<3`)|6I-46
zcH^`-;#;haxlPk<SZ%EUgi6<sBThOA?8xA-TYviM_5loh#*I6Q3}j)k`_*_$!;t~Z
zFuE$VG*KknbIOdm%a(maOPz*t$ATQ)1{>CPK99njPXk<4O8#JTHK7$NSSijMUta2h
zoVddoEo`#{3i_0GW#)T@Hg@=!`Y^hkc`T}p<G4|U$j$wGjT6g?Tr{3|-FeJaJ2;;V
zxfk1&5vDi28)tkgc69s`Cvw%oqo{MqqGtxM_`f&B&p^MtWMN@)yD0;rxl_*ONJs?j
zQkpEu$#)PMnujA_^w8CMjrCtjb3F_)s3c(7vheIuXb6ihRMaJ$+*+*ONZ3M6V9mUQ
zZ#_;zk<~c~>EFad&)4S{rf-r8^{Uw9zku|sY&Hq1W`mfE*uJRVq*A{-_w*XS?A(jN
zAN}>J?L`mK-@cXF;fcpF+yvHW9j;=kbCkLptMY;ICE+b0%m8PpauV3WH&8?W7K1aK
z0@Igmk*zzTYIireW#}gk99tOcT62apfh#V3(TKJ}7G}z>v-LGfJk)`Df&4QaFH`8s
zFq6>3s+(4mdzRYHa3ZDK?Spn!A6*H+PkgccheFcIX%{0mXbrT7G1Pn2k(9m3KDX3a
zIjUFh4_<zChG*|*=a+`U^OrFsSKgD=_kx2J9H&2!9fkYDPmtK~rx0w@A~b*dy3nzc
zgvt2i2U0ZNFJv&;;d(Ot$d!2{oChxFH8Ob<WNRGbSrJo&+({}YG#lfXNO;x}h4ciq
z(HzqsBfxH!y;dUfH+8xGzbx&y|1s<VV$wV6QHzR05+#dd&xzgs95_zb!rcH3**GI7
zF%|x|!U@%`gcX{tVWqRhG7=HJO8JXlVFM&qa4ktAw|!Z&dqS_RGJayU=&+!RF<|r8
zZV!GUN4#m0ADgJ)vluD<<m5FcI9QmeKBiF$->a)188KZdjWlN~EpgUgAX-mYt~qO&
zRc7E*>SJf7D_A8u_HuDGrcQ2#<9d{{>B*mkCBB6f1-JA|xVn7^Af+y6-ysYp=UAN{
z31fe#twYTSfrI323&gm^GE`R#ZVcp^4|PTYaS@ZUnS#_2Te?`*bP%|ANUgA_6Kpr~
zu)M&*X*X=J*lg_|y`%I}$ak5<YW>c2m@@Tt#8c-|N-hA2sk#{?3g4)?(0X9E*zS(L
z%}i9*__;(TSSEo=9|S&vd#)ZCuKZTUxIb?w`YR5l!7jTzBG&OW2pm>9ktc~+s^8hs
zig8|n9yQ4P>C>mnMXRcP8!b&U1^?a=?zQp(s<3B6!eM7%qQ3l1YByiTQlcXxf$H{C
z40ou{)7bRjSW@#`KIU4L(^pSdryvX~3}>A-EWnR`i%K&L%=<t=&mn`<9oS@}Y+JmM
zn~qq65UKN)n99vSn}GlL+jD6WOX=^=Qyu|C&YH2cLjrP#niDvRp+)00uU!9JelArn
zByw)9EN!@T+xqp%<Fbs+h42dwfAiPb2)TNv3;A|R56`?*FzWr-6Ay$f7lh0I?pfln
z3A>-t6O*FItHn`y1?6>#bA|M**V*L-X-|=(E7&5E1pg|4>P`HN4&B?&6=U4glnENz
z4a;se?&OI*q!75E<QUHaEXY|$daaW{g&I8QZFw@Ia_m7ujEdx^_7y+w0Hnq_VR77~
zr1x(W_vZsz-3mb9`Hx29@veWcK#{-mcB`^=Ev>2<oZBTY6A#}c&SJj@=;br%Gq*+=
z$I^W!!#j~6zlAs4`^<WSOtPmPHPI6CT`C9dLKG=XlhRG5&U=NeYHukh{fG6*tMVf1
zE1l5|sa6t(KK)y%et(mI0&LH;_9&jp5_H+y*e0?(;IJ1i0u;n-d=$X^t;78Y;|=et
zgO}Cjz~|L^6<-W!*;gHGOQpKpXEYvcepoh&hvx2fvakzdA?_0^2(PpEQb7||w+p|i
zcv^j>T-Ow>8`L8xjLU6}l6PXq8;McW<|3rR{VXHcCVSYNuwvS5t0QUda}k14*qKjH
z1I{USRA3$>1oVY>8bR7tWzDxq_wiZ#ry-v=mrtfbCDQU*oHX8Gkqve`>{6z%@a27f
z^XNt^o0ddgcak3E+ZD}<^j5CKiAFc#N-69xOB{0;2jkd?^6n*zmOSF8vZg;w_}X2Y
zi@&}3u=%T*7s^~43~_n$8`bZfA=2&4nhUuI`_Ug8%0Ucn6jyg&qi8<CV;rczk*E^f
zf2IhlC~XzIrpZvhy``pu1Ri6Of67&gPnybn>MQmrz?m-4ML%rwUP*%YxL$Fvwk)u7
z?F^5=x6f*FyIS=cqI{u>7&hkSZnS=}oBz_@5L;m14<9gm3{_VJ7@k<<e;A%_E`L-%
zimy;CXjiL{emVrzfE*xY-}(A2QJP@DG_ft!V3e!~-E(11_Xt@#WMgo9G_zhK@YxpC
zYRzX32^N?qU_-LqvWY+sBAG~`)6;e2bH>VNJ-Cn+(0rgkAYXoQD8?1mv<Q-j!2RyE
zseZp$Y^n#Cv`Hl8sfT;-pa38%2MdmXr4wQB#Q*hO1pQxhSE&O|S0}LLOh=>|Ou+AJ
zx3MCq@bfEV_*%1FE+$Q$q^~;>wTtr{hZ?L!S60srWmSW<fR=KB@j4)s`2D8S268H(
zqa0+>zg)?FkXx7rZ@bG7<cX1Ps3c&t>>1Te-{&PekNnFAla(u%F*)0cshyZty{RT2
zx-44D-wF(6?kWBeFnId%86Hd3pu8x?J%(VjUScRQ3L3}Z3ihxW#))W_;^ArbB=x?R
zGUHA3>P`i2PcO(TE~9>Wj5ZVhw(FDnDL68D)|5UPoO|Qhn5*(B)6v%9hGtq_;?~bk
z=j#3#9dCg8syjRb3gd|-*Wr5(1MBcBVel#qr_o~JY|Klg;=&Xjlc9QLD5gqK^e}mN
z;_C6cm@Z-J>O@-M(<O2lY^u@NB?dX~ICImoXiZH?mb_5a?@rSZlKbssee;=8HN6KG
zwbBH-(CG^8xi~UF&&N&`^yd8r7D$!P`~|0+!$=3|Nyd)WSaRhFo_DphV)=M4bH?NF
zhg0BcMYG`v+18;2A#&Sp#S@osU!{TQCRr=NSR^6n6nttakodT}=yv~g61H--qy--h
z&xK-q0U11BxJ}XUZX|%zV6D{0tqji^hJDzdXsQI#Q2(U|W6_h=q>cUB9}l%;Y;Cn$
z`3lSvA<ytVl`#zKd?e3R6V>_r0b`$8EWJ^#D}FQVBionRwL;Vkr1hd<32MZ5-tK#}
z-A1Ll)mW(AW1e|0)fyj?<N|sUsof0P{6c34u&}Va?-_Cx4(uo<zhFZBrSM!B^~>M2
zPgG<S{C<ExB~&@{xvzIMvx>|p8JeAn6;iJLra1Q<5@FsQSuB6ydmf$SWV3ISv@|5~
zw~D=!y5NZHy)T+E8;^P$$V~5tj4Jg>UYo<oMkn&@In-dCFpCNcbo5=wmBl01*th|q
zk3S}OU3jE#RlhkI&cVmlE{HjNd8uHXOo-3TtJDfFOb71d)PVU14GKo?I#2kgGhy$5
z)=GmzrUwGed2p%dIgy-8Ai1Fa8WRqdua9s5DG$|^U8qkJSDu#TIeG2w8Qc9@Hr}67
zb-nJ$yA0Sr-JAL^JMm39V@`Gb?gE0(y^DGvcvy`%(ZxQ0W6L+#63=i0J$T~6qO`Q2
zGhZp}zTwjkQ(Sp>+9axZd?a(zBNLLHkWK!|o7-jYM<_GFXVegR?fRc<b%6~Tgga9T
z?HZ%SYm0Bs;6zdF9P5N_nl5{P3AOO5-EaDN0_?)F7UtLA<28(PQ>wp1y5-O2T`A-4
zAJ-UoU89p*!fY+Ig72LiU&emDFWL-TBL=f5sDr@CNE+u%pf4Z)0y>zTRW4_!0+NsD
zW*0&t>)CBtAV0<#Tx@P;%d2J>EC;0+58J9*aN{j(5bJbZ7p9X`il;#|jLZ0i+c@T*
zYLOSLTNBc0N6%|i(kz5;>`dlENWvTZkLTI3yWzRuc)&^`5(1PYS(I+Uc~%7i#wKGS
zG={;q=9n<{)=1=!IZ-{<p>1^q(*(>@CXodHZ_W4PFOFU#i#o0*tj5p&WW2G$7iMzT
zoXQ2#bF;b9YAf8NBZDh^O_S%wl8m0w>8gWjs}VaeX&*S~#wU7^<SxqCPXSr*r&u^t
z>ABEU{TI3~h<H`r;3G2lxYIrD>y~rq?=>znHs*(&WL4ei6K}#q3zzl+Oy>aL$~rrh
zCl}Yf^9{PIC0Fi3V<Z8ZWX*Vo{)8%b0Rc{fFtrPjDwl+6?>aRuju&ntm~CdfGPTW4
zugIO4r-AZWcRtAGR=Dw{^lod((WD9>I2t2$$fNdQ%2a*sI*>_bsc2Ft(QRv_p)wxJ
z6yS8&5Knz)7_iDdSYoVmDvY*2UqjRTD}S|d^Jt;&a!lxDJeEq#yf2zuo<&riuoONf
zr|ww^n{KwCdkC!rv$khQOr^taxCnB-uz$56#aq%yT7xP@P4*c^A))hjiWU<$9twja
zzIN(+Q!Kf@y5n4;n=z>Ma#Kitd4B4svT&Q*1#9NS0{`nZxkYzJScC-c+gXu!MfI7n
zvmY^7cc|$~u}2Ul@L;JRly1waP3+EAI~|Drpy`JTB_D-eoiK^5XiT59<GmhQ_=UXR
zoE7Ss9!<`^P0(e^gYXQuK#AiqaO=z;?-mf){|6?b1rF#)Tj7j$cn@JsR#(bV!)Bf7
z&3nSGj2&xYj{$f~LA~YB3SO+;K7Q^s1#Q_n)oBuXwk8qjH?6^+&boG?*!cMSnmAw-
zdF^2<Qo%k-BEKjDLTehh4KQ_}@Y#nRE@gk5-*CpK&l6*_M8d<Cnk`W$h)dXPN|<qc
zXT*L#l(LWJ7U(<4Whf<h20}5NCb6XzZ+i2EEFg;&@}|VI-cNhS1Z*QqyH%bS9@CBT
zugfs`AaGmg))w6XR<xCgE-+MBoivm@zBfL5Pp-O=A^7_x4{6=1lBdFWnBo^w3P<oU
z8iJX!LvwD?ugg4qAD`t|BNLo<H-(fZKPljR^?16~W`)P}#m4m%fgVUuiX(#u4x-*E
zyrM_a?`BBT|4pePoZUmnb^#N+;U{oeY}WB=Q1(Rev#wo|VPvCJOU8rD>ajwJGim-&
zmb3MkO<e7}Q-1(1EY~Rf4#8GPQn?O<k<hE>KpOBsjMdE{_Yzh_GSrp@$MMc%9vE0H
z=CRHj{y9U?u3!H-s0FJ%-9ymrU8nF!!(8w$ADi_L9}BP@H+U$3A>&r&^dCbhU`={O
z)7AMc2nq>vD{N3xjaFvjMS@p)RrrlUqDa2LN2}fA7*Vw`z>_V?<|X}oA_g()9Hl#_
zjWdTt2?#OPSQ2C6Xy4ycS%D9|hY(;sB*~@WW2oOVO`(t7)QszVaC&Q#$&O?PE3>OW
zmj@p++9R0rmI?3ums!-O{XMQR(vI=nZ*M*_BmML%Pr3wwn+>(A^aQx(_H!j7%@1cQ
zz^(Nt!1{^_C2CpHqoH=Ub0d}QAHIKvYVb55Pg@++CSEJ2dXjv0(&F2~P8j`nj=SbZ
zU2{%g)_M6LguDCW(|nxmGV_@O1dhF;x#i<|#F7js+sP33{#cEeEN<2xy1z$STmq+L
zQ>9Iq=<hx25d_SsgZR`|K=L&UxWD(~{?m)iT8DtZ<+vCgHl#^>`J|d5=ms(O=Wqre
zw;DxG<vV2Xk0G~)*u<n^-S&mePtbbnkzOfZ8n;VT1&K5T6i1<Nr3nR+i)-1D*9bS6
z%$0ew4OJ7)^U^yF`h{OPV(J}cYNvPUnlR}D{63xixV+K2cOO-bxn}sGV7L@cR}z&7
zI{`LQyMzlm>*Jz+>srQaf+D^<DZ$?!T{JHz5`-y!@0Z`uZ`oM0yEI)Xu2~*D&sMi|
zuZ@8S$qOmt5dhe}_p0IwDY?1Dw-?78w*zhEMWZ5L-i5#SO58f!jwesk^fo;IPM9OG
zX7@1jV3TTMW-91j&C$CPGs{%eJKL5zUHF$Ay>~Bb8lO`kNFAwoWF_oGJ^RqU8)%pW
zP7xkP@);g`yR(V+^ed{^@0j!VTTTDMB)q8T+_8sc(?pRj$nG!&^0e3E{?q`$7wTl2
zG;(@Adh*=oAJ`%3){ir~2X{5N(e@)BTpj!~)^3P!S04&y)WNbWOTL#u(ZySwdhyV8
zW|UABK8{xO#u^fvpD}K4ABlfmp$5T8y|7uz;5FbYXaAyon{w(dek(#t+|P;MzFQPI
z;VB&P>I1BiUc8(@qxhW;Hn*mmAcL_yvnn2D^ikk(XGd1%vz^%Ww#;BY{(MI^C~65m
zZi407rtJtGGYWGT6mUnW93C&e-Cx+Wzuu~s_3ex#4qiwk)OR-i>+V-yBp;!#z!mW|
z5osQBz<6HiyPE5W_EL@{A=!c)uh4N$+I!mk6BAiy(9XLmJ`v@wNjBlHzcDWZ*1e7Q
z2NXsR<^tn~qJQn(6p4GIv34xiSs#(Jd1ylCAI(tG_eUxw&VtGX#k+wI0j%@6=yMQ?
zhcA575JZy`iaRu9zMCdCOq^r4SR(j8{BH9<p54c#ZRGqe%zJ!z5Qu(cFAzTXfx8>6
zdl(O5`wW2r;+Q9fQ<}U$?LRq%|G16!SD+}Hj3K?_fSH4!=o8|<##4KI%72PN-2E5*
zJnFBi>Tw@53C6NMJzO7c<w{2Ek%w(SYTbJ*wVvz_#(x6T%fywORt>S-qw^0G2x&9@
z)!RP9gufo(`W*4IFh$e^Zr>Xgi7!1DFt#76atX;+S|hKk7KH&`G&}IH(O#|9;v)zg
zcTjQ@9f+ja(dX<N@(!`h?-eXHNX7#mvk_JO{$Q^cFQYT|#Ao)b%3=_T`9w1KWalk8
z&(y2`q1x)rRcDoRsVN{}XooR;HPB~jyTk*`J$?81oM-0=^kP$1o+Vdd!w^)7vj&$6
zt@@WcvwOg==k2o6WT$UAksEB{yE9BoBILsl?6!&c{{8z~?`9L(fKc3pb>$sn0M7#i
zs1@6QT0Uj_N_#sv>B++I?Y;3i#rIiJcP+v3Sz0%Pb#zpFml74h=*<0P*)mr$9)K_6
znXt_r;#PH-$dI#C%HeMzKa|^UQX!m3S4)#iT*ZJ}G7|nN@&4Q#s_0fday4D<;jUZf
z@u25nHZ6-(7F7};a#n43>=(BwQ1^cQouLolZE{@Z3Nj0mdZqblBn-;G=_QtVJeh9U
zD!s=)N;;Yq|I^-61|U*?RJM}A#|V!g!Ep-JVU|_c34>};bCW1Nkm2LdL&mgs43*2Q
zkq=z89V>dS@c_1UIZv=%lyWy-Pk(>Ln2+T7vB1o#%9VG{h17Q_*~8}`fwiPkTkh#N
zX*8oVJ5QVEJts0au<%8~w|su@OOkYGKNn^C?VC;}>i((-O*{rRn#3;FN8>o-uo`q!
zY6nVcB=|XRofj%f<MlSg0GqsO?}W};lT(rVPD7PPo1GcK<Onx*DI4-$BU`E--uxe%
zV~GCq@@M9{ha?9Je|T5mmpBE^N9=e{FO6`d^^_bJ%6GQpc$-S(Z_1cqsOY~dOzA2-
zZjDS&rU4Z3BAUmGZ3OeybI0QvU*}ZF?Nbuwz&{pTEZM(!*<a{!X&UC5*%3Qdw2RSx
zFl`xO9<r0pd*%#V`Kn2b*kcE?3xgLF{|@NX=m*V0TkA+hV`)Ci1hy2_ffUyx9rJW6
zc!1yBl+S4pH-vSIiU{w>z6;T$T3fTKHtC%qoHJU~3T9M`exu>qGT5ZjSig{Bx!w4k
z;m~t7QX-eU#Muuj?EhaHgsWEJ@_W@fTQQltQz2A~r6hD92gIn-NIg^T8d#7i=z#W`
zp1f($v~Y@CVA`ns!LFYSAF^2-vE}eS5*XyCFkc5CA*$A^tF=C-AXR`!t{vfPdv^vL
z%}2tJuS-?8qp<k@v`H&pxZYdnyXD#)ttb@!8A9Qhlq|;OUrnoxrlvlh%i2qJ%`Dum
z%OH$)x4phxfJ`wK=06A+SILxl3EL!3zdY9?gS+k)X7Q)KZZ?e4y1mVM@9*Q@RHd;b
ztN}pe-Lb5q_g1sxVc0b0y(nWvW9?Tq^ln`z{c$v$#=p@>1Uxuj({EB0pTz@yiF|w$
zYCBbV(OY){$Mb_gBwpq7fRp3ybZS(4QtR-53MI`Wsu#I{KFQ88D*kXg!g%1G(ZACi
z<(Roj^XKc<bNJ@=c9jK-wxF!cj<i~dR=b6Ep&d-E8s`|5DNg6Am6ar>q_7DtO|M>R
z>ug#gpvsfe26R~FBsSSpk0Axc3^0}U_1_I0tEsZn3AL})i7~aa0_?h>;N8o^h8m}3
z2Z!OX&aJc7W#=@2ds8uY>~KIo<7#CC`lbsGE&9tl1+ovPp;`xv^KyG~irQoX$JM=Y
zl5pBou@^Nh*KI_DfYmFV;=<z;lc)oSf0Ya{qaMqi4P3=`+$e7Kf~*DKH=75Au{C!f
zNe_n!AG*FC2Z&6S?e7wMFDI9e0>9R9=Np6a!hg^*aBPGM8N(67=7$M~x2Q#_)Kk<^
zTwK`FyE$nnsVtW6Da2mpsv4qZEPtr8(vH8(#cw1`zlh6PlLYL`opXYZJ_IA5e=eAA
ze+Eu65@uHRL*`TO6x3*d*6k;c0d>V9^yQ`?&sYiNuY1rw{GYB>kvi()3`Itu6Zb=X
zWrWBwDmyDoeu49YrYj#~2b4{Y(Wv$r^BMN=e6BxJl5dZySgdo8!UN=lm4CvD-h|!+
z`Jvc%&2-pV&Ec#4b(df2I*|Qj%YfZF5XK0WAG^|6Q7sWM0ETS|!ny=O8a(THDk<Mi
zJ`|qyW3>(C$L0HB5Wss~RLdJfq=^H8GY@JxqYeRxFHdTqb?H17xB~6<A5*#?_CKKt
z1M#qk<hNpT@2yQ4XRGd#XRGAf!;FZFWhXXXmxyd8C0Q;PQ_R?cA6@&E4`r94bs5cA
z;6b$aW)ozK4aQ}o`W38yZDInzizZTy_Hr*)-sMvLsfRx3>|i+Klj*{&Cod!T1gX4c
zFaxZ6O<x==7hc?t>FaR#LppDSw66PjFv}<3T#|Ao8p$M1nT92>)xOm2zn{1Y55*j3
zM4C(_hHU<fD7^4$V-mcpBTa#iY02v>3SaB6<8HCuUb1cbX4|q&z@%AiP%7w_Mj(FK
zWj37ltG9QbSxa?VxoWBY@p{RQ)AQQejzv`2j^#EOz}@vAFklbQuL6Ifa$9~_AF~`n
zTW+KVggIlviu?&*-gws|T5NG5;e41RVJ!aYu5ZD%Nf}7dlL-Zfpf9mKeewI<2sbxt
zs7&xa;i9(6(jLG*)Xvfh<PFz4nXD;5Y!ge|Flu-zPqgIov+!TSpwxuNltny6u%=R~
zGGgp*q?9+Z&FQ*w2)-yYmjF@_W*rogX88MW1Rx*yyr>u0AI|*cRm%<ZbP-eFwR?F6
zw&~n}(U7RE1)7Ep3*%+bMjWe7X<>7mgHEOg`+)-K#-~KO7q6P2^2xFo$OFELqkv@u
z+R>SNOP+(A0IiOhKA3l*Xg!11bM%ml4`BIX-GvX%C1n4Q<l;WHTZFQX?1kN<{o(o~
z_+F|H1kW3V@ZIbFtzl#wlGdEYQnBiE){U1^3uheKui)UrMw~Fl!U~pR!g#6B!%HY}
z2ll>vI`8rMQ_nMd>c#{776*Y{mhp7jX^|>CW&=8UV}`%A9FI6F`uVdS8!}j5UQ5`w
z^4r|;&-TmaW8Q<w(I>%W-#vJNvvdx_z$H{jU|g&!@P&5c{|!CWtMF`yeglDTVXSfZ
zs!zszBYv<#pYyhN(4d%Tm*$~xCiVX4%Al9xkK~ujA%|UT!W7k#<bUW+UmZ^%VSt);
z9fZexlw#PeyELF3YO~zy@?LfFHQi8mO0-uGM(^G+AR;ig4C4l1bem*?y1K;QQMK76
zEX1Ukk;fndDG(I$hmXYFyecuiDz9foqKW^CaSYb~72|^cw;0!q9nP#3K(acYx}H;!
zHa8VfFlso!Oq0l{_%6iO+PSXa-g^sEeKpf!Y8_(`e;Z@z=VjGhz2lN;+kl#y3jhW?
za-`1QDZn>cg;ywrOLLbf^d6{R%;Ol}PuWd|dn3s@#&`iRaIb1AHsM(2_m!e{efo#P
zb;96sY;32Y>KJD4V#zZ%lPzG!lOSvC0!?SZz#3-t?-QBAT~d9OnL};!sdjDdamjWS
zLD!D@rX?MWfWCC4wW<8mN-{{`ZqS%=3|OF8y+RAmolP2Lj;gmEP5j}<J44tE=lK&P
zF7vq(75anm9Jt<J&11NiZ+AyC@(N^<510)atrxlvV&%l*pmDPV%sW#9*F}RP%p8-8
zxxJT$YdZ+^Ojz)k;>0j<@ZV>z$qO@ZA)A|eO^R`_?)6!Qb;+g0cvjA<S%iA!;%N<h
z*Av<_X17!Pa!Up{{KNi+ag81+8<vqlwW=C6VurQ@U-X?uQ(vQ^KLERPTH04jrRXnm
zsdsg>7u&A}j}p`N+bl*qgEKqZgTbWB9{_**aR`Z%cF#d8caewck9^t(l{=s>-LnyT
zwKJm?5kh*R!tAxM;Z7dD@aR0e^-qr(%LoZyFUX{Dd`OQ<C#C<v4+5_-2J|VdfxxCt
z3Hm!MdEqj%JT0Bnt^Sy6mNt1ahjnrHe6N=JWc3^U2a9bWSTM->-#+<P75vRX7}wgl
zn%D;S#tB#ULv1p|zl@YNW`B)WRGjgIGQfuaVxg-+aCQlheA6H?<W1OQ{=tbq9Q?A_
z#H>}lg4USvN#^TL|3@0cYsFv0H??zbC=?2}tz6t~sNpfI#-)M*(K-nEZ{V$qZ~@4>
zSrPNUB9kpQ_!Cs3F_Vhh^%HK{$wB{@N<Tq0lD^*);5RPk0A!U9C`@IaNQb$FiLbDt
zkUYrx1M^<_nh)_N&UD)}r`w_+5o0@z!fjsj6Jc?_!<tUR<y@k~G)1Kg<cB|=i0%uz
z9`^Ddy^P@(78ZavxsEX>$3jGPuMG8_Ux!}nm<IF#X#obOZq0v{>+t`5Sg`P!$Yj2y
zHq|vMh<rg8pFfkR-R=Lxz4A5LgxC2|ir4x3i;9+I23wPP3b!LQx!_g?5p;pwG3l?p
zO%uRhvX>VL45G|yzL3QZlZB=pizVybUe<aJWUyJh!ekIqCdZ2VNSb!lu+KgmFHR>-
zq77iGoqk*~WA+9nriNpN_jaqxwUVY={c-i4O{WT@gba#)C%}#pOo~WC*1}*^LXW+4
zlo(7hu>z)<akZt{w*{=3(hD*x1FMx*Q=5#I3&%<of`9|I-N&KIu`Q!&dszGo`oT;s
z5`~=4Wm^2({Jf^RHY0)JT%~T=7+*ihYQAcT@^qb<La9noG>zfb-b5a!)$H42deu%b
z(Ksl~&ktlB<PUUnNi^c(eq?gIpJKc|{AS<J5glx5AyNjT%4(W_{a!dTh)YmHB|jm`
zk_a`_&%Wj&v}O$`f}})2#n0TX8+TMyY$Rbnx4P4N^SDHhK3q)Jno6{Gs(G)_jw@-2
z215-u{ORY{RJ5zlu9?^5b@M-Q6^=mZPxI3kR+@y6iu)OcIIxI&19|cq3*2OE6xL}e
z7{AL;MADMe!-lpLAU8rG>ijMcSPp7<VSoeJ1Ju*o7FD1%&F39KIU>eNg>Z`RZl}z6
z=JgKlMDo-A@*8q9%Nv8r_bCGmkRDpW0*Tl^jy%NdzCXo`TRW7AR|w`%pMAP!8Mh+|
z3s0C*5kPG7-@|A)p4iP8<y3<(3A}Ege4v1SW(6O<q_Rpk%a4)}SHup!BjpBFB|z)t
z<%9H;Bc;^#NeXGdu_<JY_d<Mv%Zkp9$M|AZhyF+5(p<EHq>U!Xddc|**q3>`ZP}I-
zK$)xUo}Q1-a5b)r^%mERQlEjwQ~C8D`-gl+w9SE3tD9CAoPiPi?Bi!y9Ha}3X0S~x
z5_i`sm=|=5n!bBI&SnO4+zh1oF<`v{BK-oFXfQ=0xUg<}4DB#8^t(r&EWZ#LvlB5q
ztmDxUl@EM`7$}W(IgmS*OQHy%&QY28p7lVXYiRO#i(QD$SdC`YN-?}-X#8^!f~$oY
ze`YPfr4CD-{;x~D?tev9P-c2RfOog$FSc2w<g{I@`^k|#T_-l~INELNHaIisn3q%_
zq9?u6YD3B6N#9DcW{32%V-M(G*Pm_GbdINRlw|Qc{>l36{1?PA15OJfdtlB{^3nJt
zaP4eNH`S_NbntUZ=F2GA^f{7ie#Fc4x2>9$%&+V+1y(nAhY4khFcoX(yK9$YgkmZS
z)83bQOf&u7Q{;*J*!&(>2NK7N>m8C2QN$b}A(%H$j9V!9Oyi!X?a6>wePeU8M8DGq
z%v&V7+ozuwLVj9Px_UbC9phxdHgRgGxpbyLQc#-s5SmPp@9aDBsVZgeei-PC^pO=B
z!jr`sRhfNq9*x92_LOoN2T$#&d{`fCJMK)|(PaUTXLCs+SjZ>+<mZ%7ZuHD8yP0Sr
zl+A0EJ@riE;53Z8ts(DgFN&4w?CGL{9h$;tDGR+k*elDGF45to_#ry~9>&X`pI;_U
z9rv>2_?WZO|4Ud9)*KJMAb#6rsudz$ISw>*f*8uMJGQ}dx=_3Q*hS)s@qQ{>-9T{H
zM_7_IkbgUG`3hSemI~<ST|ZY(d~jPL@FrEr>I@K`8tn`EoJMZ28^_#F`hc)e2i`D(
zZY+yOsT6*gpt1Dxa?9S3WUD3<eE<?cQcV{c<eDIi!6;eJhD7n=0Ka}Q9Eu%w*L7Eb
z19DrYWRdvTY{$mB&>Sd8%$;23B@OqUB8wN5CQld`qLwKw5p&ISBEBR*4L?@G_(LK7
zH1ESVlz)GpX#7@|R3ON`lARV96*R^y`JEjZO;SERA9kXpp1ks|`MNG4oLTDh(tJ$9
zL)V+zMsZCEf_1@XT}RCqP;5lVv4MX8oPIu~!>RL&y(;t)-$)}qI_n{LhlsF(H)MK$
zwg)7{gbqxx$+A2yxEE_zKKzknz~w)C@%xne!&F%r$N6-1#qf774&Z+*BmM{ODeb}%
zUeY>$Q3@SUP?XU97(wy1*mx4&XDc-yWy_%pNqM@>_p#*z(s&^Et7n)$Z<nL*fSn=+
zq_-mA4j}V2?7p`i7`z0@TI>GyWE1AXL-fjNlsA)Tuw3Iarr#^HAd$_+aVpI9>mToq
zh<OlGR2l=(o%V3I(Dx<~dABAAt%1;cBM_^{=9gv+2o*n56b`EUcYGrFU*nSo5W3@4
z5VV{v#FY?-nh8zhz#(7m9+gBa`**^1$mlTQ^FG~^ItM!Lj!q5_bh?Y5_kVx==Dp=?
z5=w6~p_Bq0{X{)-p4Ta5yjH6Yi$I+spe>^&Ewcri9}Xipts1_GJUU>UaMADhH2)Ey
zcB_uUVlKCy5PR61%-5U$hYRbSMw8rOfzA?X$w$nasbb2IVXtdl-VS<*<#+B%q6W9q
z4{yGMm_#$6am<n|9v3$Sc-Z^DPv#-;_m^?nd$)1Z+#T2zc?LB+B+sia<sWFK?8a3i
znAPMD<uiFcW?i-fW1KE+8}<YrF0NF9Uaus`(W|@Xr({4O@xAExX|}DO;yEJ-Sw}7J
z56Y@88<~@+Gtd@|NPi%rlV}EEd-*yYFTT%6f|PIpX}9PT=#UZZ*rJyfQLD-UGPFpw
z*;jntphUSA)b3?ANMLAJR`GGRm|0R&xcv6{yfW1QW~G+7Z7*=m)UiSSu=tn{APlD`
zKM1T?vqYN&L1A`<_?-#2`$M#?#Q&w(iZ33OwU`l&u+s=3Ij+D@DZeJ@61t6w8d3Gc
zKf<;p2ts1J`-<E6wFWg;h!O=JULiM-2%rqBSE87D@h23D`6S|`Ov>>9r4LH>dnd9m
z1x1A~2i7=BJqW_8ngDBnM9gC-$gI^Vw|m!H(5UOy<~e9|H2(Qz92g9n7ioG$>HB&3
zFiXSu-*A0)=IPX2-jSl37wn4`ShXTzjl_lYEVF}L#-88#yOeZjVm%pmJsoiWh`5Dy
zpmu{d5<EK|GUIAz)`Ip$NeBGlRw0~RR@08X!V*t#1N8oTgCnx;tI*eq7lytmuhD<q
zv4xvx#`>yAWG*x>rG)t%swR_`PVEV4j7P&sk{J#IUDn5@hk&7ZHQ>mQaK2GF4v=j6
zWdh9}O{zM!zB*Ns-Sb-UvNsusOymC4_<Xb!ShM<bGyv_-Wb7xsyt(NL#qU~WHj1mn
z@8i>u-uAkvg~fLNnWFHWsy8`ok-B0^$}Qq+d5Ni+lt1%Zr5!XBd?Do=rQwx~YQ^xi
z_mw=<N}HWE2LhQLkD6Ac`gKw26|2q3dV8m`^|%3{;i+4HQ)6V<fAiRHf~joLpb7;2
z_^~h35OK7v^E_O}xw7)v^JLB<x({%?vvrsW>ygK*vw>4x*W*3EpN!{5KKNnU<798b
z@rJDk@t};SJR2le+K29w@Dly7iR*jy4hxMWfIsNUy63nVjSnsTc;a`xND9J>Z8K)R
zb)SvWSa!<Q_>xTvdQnTF93%dy%VLK&qi=PhJnb+RR08d<{`U*GT<QPN|0+!bBsuVv
zXtE;8A>ZAOKp8?~zS^3HDV6Hjyci|H7Aj{6o)6k>pT)6nHQOv1$@#v9eU*?<36`Ey
zUiCULM>B7-KXF>@v}@YmC>?GCj{GTb<W<}2;B==-@cmV5?CwZ-t6gx(mjGIB_gf*(
z+)Q@v`!ePHRznQ`k!JANc(Btd2T%)^N~(|a8!~ZRPYUV?3B99F7bM#VJid$97Sjju
zVH_x6f51{yeo9gM68~+}RB_`3evni{^;^w*llJERJ|D3RsJ{bO%dz0W*;yq27v3t6
z(#vRy`1zY;b21r?5{bk^?mVyH$ng_7t36zaS8T5vzkGJ4@hQDhrRvu#+fgX8DSOH3
zL2fH?YZ=7TIM*uGd_asC&J<%M5?pG{-R~aMXNJeL>ShXLAPYNic)FUK0#;SY0!8qB
znfGAeyU9X1($=e4ZAQ=Qqs1%*^r~tlN)TAn%jz@D)x$0z(N`=0hd-h2z{9WiYqYf?
zz=Lw#U4ABMa>GB?SU(xUXBG!?2{o%zNVxHP;dVY8x(AQs->eRL-73S@T*FMC59wf4
zS=4?-1o(N_3Brn;xoua@roQ#}siy3-7!yftx5On^5R{u4#G69(+)?SQcBFO_wiDEi
zW(Se{m_lq|@DyPlPkN&scA&448bLTLS=M-5;3_#5loE<JnwBPbyt2@ZX?*W^tS%S(
z!HYj-kfHnk<Ls@X;#!+_ZKQE`Z`>_NfI#Ce!GZ;s5F`*FNN^{(y9Rd+?iK<8f&~i@
zXxyQ3hdr~_djJ1>KO1}OLq-D!J)ikh)m?X8CEYkRa6T_l-P^?L2H%T}(N&yl!W3h-
z=$E!t6C+r0LF$B!!1__3OqOcjq~bI2lb~;EPJhgUJd2u^YMUyj;jE^?0_>7qujlCI
zgnL-v;$gXD8iD!XHSTwd#|8I=;xO*k4{r*1>yS!q%gkr*i_4LwXl?VR37s>EcSx!i
ziwg?;TYduV`ZgUi+X#YQL_q@-GJ|iXC+-LN$!x+BbjiAtwhOamcJU(6k#H!y56u<<
zDgyksz!yFjJBPtLC>-x*1O%Q+t-&<Irg`AwOgU}CThU#@bSB)h)h<3|^02dYm-i{}
zfuRaLe~C};ki@oKB@2hCtMcj-cU2&KN-%#M=Mm1NMAhs`8IX*Qu-8!_xjg)XSlMd<
zRX^l|$%F|eb@MSqu^(00wF5*=ZRSo9_rbz<uEcZHad!fTFLLR*2Z+XM=rD*1k@wod
z-z7DV^Ikc#vQ}TwnAY0q&TVt<=~L>mbi2M>7cNu<RD=-(ym+KwWjp2iryVD@GCFD3
zj_PvRqaH)>t$72AM&zQ0a^%29k;6*o&+?;-dq*W9_a!!GKsJdAE(D}n-0HspQGNKM
z`u7sw0zRWKNe21}IY(srLE&>TMb6Y^mH&)azw3!X@;DC?-uOi<_5%_tTh>T5Y}fyg
zKSo$j{p$03@E9K(DgVb&R_}ndaYR{kksP=%VmLVSg*<TV6-`JkR#0D8E9a5-YH;po
z--D^xkvyj6H)N4&d8(D~uHvdm=Bc_f1AwaF0+h-O|7)rIXQFo!n5bY8#_>J6<5DIb
zD;5=LF0zF~KxwiK?;y>lN$!;$7&|nu-d*!pF0Wzp0WvM`7kGYtJi%^{yR9wz(_9YU
z*QUQbovCW;r?xo9MogR-mwU<(*i1C1Hagk}w&Y4}WirG%(zu&q|2Do2w&j26(YGYV
z%>@`t)jvv?c$HavPG#927!t8NPFnlc0kFj4zTdY20_iC3<s;3kuDxuXn!O@Cpz-Mu
zO4lNYzz6!!+amU@h~S|}2=oRLHZdwHLlM=g>AgFRfR|Dt><Lg#w(6ca9HZc2#FQb`
z<A<E?vN?cowA>stVi(F$sdmEwWATC+JFcCysT`Km>2@^q<iP$%Suvg3H9esEms->Q
ze6UHV<|KejJ=a1uQMpYdW}%BrG!3}8;rcOSUgEVc#7yyO8|bZSZuC7~eT2EUt=I)Y
zP${BXftN(!GK^#*J<yuJab&<Ixmk8qVw4#dga8&Kt~WVa7!KO#b-kSaE76|6L%aHr
zI~FUxjhtr06^P>f(V@E9J_2(y(oi)Jh0d7f6bVD&eG9SFFbTJc!&0J@OV6d*yxgdw
zcjs7sRrC0lL=!*PYwW%h9))a>hJnP!s=_Pa6gTYuS`)T4x+7;M0k=_Gapmof2)gfh
zvvyn=%9rcsl?oWO*s*usZANMR0Lqtmc3m!V0lTz8cLHF_x9jfcY(i1;kCy)@bB1Er
zX9_3G1T^|OBDe#n90c#104q3SIL+x*IBNgzWNa0SdEh1Z!X;u#g!w#7(crrP&O2E!
z%55l>X2YsjT56BaGr@qxo)jON5YcgIOCJK43=K&GFs3)mUk`^0d7MCwOrRK6m}g;d
z8J^^Gx?wixaUIfwf?ba(3KBDoB}AZ@muhi14?v8yYW=?&ADWy|f%RN++?;xeHI7`n
zE0g@)H+QKVBTCLB=eJ$OBMQdCo3XfA@tdkXJgIEkdj<Grcu)anIgd8G=7pEYt7XlB
zPOiS3`H%jORFH_vn$w3*4oCA+T4#?Ur4p^0NiTm3$p&Mm(4UA{Un)~OPSSI#HxWMf
z+|?K+-t?fq%hH#~zPn|1S%0UdQHZ{XlMd@~b`iKKl=rIlAHt#%c6p{9EZP)`Lw%H|
zT%lfV_+oFRyE|0T+Z%GRKQ}I&YMf{I19|!(d7M?rK5>Hq<;t!E&7@QM9m)8I@>x7F
z&rXrFh!jPS(`o*%YY0v6Uq24IXI%bdew&;uNbjB{;+$b@?N_C}t+w^A$;Cu6X_TpR
z+RryX4ehuI5VrbGCkqqLhf*!>k5~ce)ES;rGG5F#<3jMFv#6}`*;tUUH{EM{4PI^e
z71Qijp1O$(E@!T+c;YG)G8qzj-p<OX^<WSlq!qJQhmpPMh^NA}OZ7&maYmUh`>JUt
zY=AGgce@1ol9{Hd<8nkxR6fT4WU~;7t^Cu~v6$WFWV7?Dnl}g#Z84WwO@y+(OsL^S
z=cZguA#(8Qlj|yaY?Z=%s%SpK{!b-tlwQO)G<H~s__2ccxUc~o?D@B}qLPFGclyRw
zoGuH8c#3a#x3bwXMs2B9dyH{3CzU@oeDG!ipi+>Fm>oV`;H{r_wKr5I5RJ3yeeOBX
z1o#X`Ic{55I`y-c^I=7!RthU#mE%hQTjul`MAM|ndKGzIXUn+JKlbo>))-O#<fw6L
zx^H^>w9`B6>(-<jm=Lcr`OsOSwrn~=YrZz;wlm@=?xyu>Th(o+-znX%-k2h?o3*+c
zUq-`%2O`789G$r$DGyvqNTWYP$YD$IeSM5h`FQ_k9Du9QnIrl*&FYOJT7?=`e1Cra
zTc-g%Zg#4DY|ZmE>sUJE(>TnnaTBYFlebxw=MF;#ynz*W>-YoM{4jo(&;^Dc0fr~l
z4Z?rh$KxJ9z^n!p<gJ4^jll0#hBCxmIIh7yYCdYhcwE1jsv4xyNpE*H#}+^>z#Jh9
zl}L;cqGqApcd7H;ie{UE<%jU;y8%4bWV!bfcOPHb<ylvGIQ7WrH?E#`=(Z@!*zEsq
z7bI~Nww-1nfAOI(R2BU%^V%OPO_ebH;&cE7oyu1@9)x>@%9|hze~2@J*6dVX&VHPf
zX=tW^$|w|~`I9*{S%3e9rRaT+ILL#m9=mHk#ooOG%6a|r$e+M1rb9e1z;S!abQRIk
zOVrLIZYanFzNf`6ht$q}Ee-cz{?kJraUbhdY*Vdl`WagVQqu%am<qlbHOu<@bYP+o
zb@g9#FrfAJUnUCwTnVNe2-)T%DIF0)2_nL=%WiA)lk`bhNy7XNxp!ECet=Ac^CAfd
z<7ULJ*k?dj-er<BB7(KF$>N3D;LIfpK)%tOdhGeMxh?insQ{g%aDFeDMN?F}^W!gD
zB_*M>6p#7o%^@P$YTG%Dx-WC;s*Tx`k5mTMkqgzEJH9LLjjeu==QY1!)h!US94C_n
zwyzfiA7<JtW)!*H7CuGtm?36i@*m)_kR^Tw>?F8hD0=h#=B+h>6o!migaPlgf<Lh|
zZX)Nq+Gz}u5p}LBq35IwBE0S0VWBFgHmES|<t<1s!b2Y>WG!rZ^If@Bi!j1R9-q}X
zUL=O`CI*Z6qcY)GNmCcA3M^ECZhq@c%_$LA;ENW*YT2{xyuO<c`<w=sl)}Z$a4&M3
zE>3jwiO_Hte-r8$M8gw6+mvt4W=Mwk=6wf?#;}^+=+(J4N5DD89NFj(0y5G!3~HKt
zLiJ;!QKZPBFzZCT*6Eu!h2sqirg!MLCAb>Uqy<&`xn2vx8nM`}F|O&Y#971ssxgRQ
zi@<6p4nm+H{?cX!^^oR#A87IAYKD)3J$0k5y>`Q^=z+Pb@0;WXa`$WG+J>^$eLBiC
z1`MCY3To<FW{r1`X}xtzS9hw$pOMHAA(){e<j?qgE>tt;N@G^?`%KIk7wT5({Kf1l
zz`sKtxob9L_-iXQLz;ktsE^p29I$R_cL$&@Xmz$_wp2Pnd@_z&x4~5DjMIZ$>wo6?
z*(3j&=P&zptcCI@^sEliA6i2ogo0an*WRyJ6vv-d89-<<qJk|cveRO0=_<N!774&r
zJes6CU(1O)gL`Bpp6r5z8`iZftHkD!yY7kk$HHI|&(w1$?_o3^yrrSF|AO24V=Wel
z?ypi`T1R>omIjH2YD<dln+748egWG<2@phfJXBZ;V$l93i_rhCWD)#86afC2EcXW)
z!UeR;oQmq=9{&aK4RDm|p$?Q6`l0-d4FCYFM)tU}v7zKgNl2J`oi5f%W~x+`JDq>~
zDY*MMZqg!QM12>p%KpKybOf+TvwS>CD&=E|r4Zzr4(0so3!<U<3pa-zn7!ANsI9j+
zxF9YB!mYgFBk||{%Cas&wOCO}ypy)l;|+fqlH`Fr%IUAR#YGeA`BTtaqJ`?#!tALw
zG-uF!Ph&eZjp}-4;@P)gclA#T2>6H_;rTiCtyFuah@#utvk2%GGNKA(aLLT$tU)6^
zYy=pTM<n*GCP1<q{4<M<8xO+I#(eeSUn~HI)rWuhzaE4DoAxAi`7b=?A8Uf#Uu5GT
zy!dk^dgzB%7ho*4Wok&-@2|@Ya47W@(E=zX<$g8&KhUNpR8xZ#mPXDIaf**SNK`+i
z#}4Apdl9oM54)JSUKcbe>c7n}Jh>l<Z7fcc@^I-6lH<xx-uH=A|2Tl+S=Zm;aW;h`
zesAjPce<gtH(QnYFX}840<8jUzWUps86-V7fRGYyi`-w_AEH^k4%tuV>v@A!I8jG~
zvmEe;A6pvau=TvwxQrtPq1no$Hq?{@7tHMKm|YFq49UJWf47iJg_DC4=DDE}+G{=q
zVcE%(CSzT!&3Q;q4CY*v9B04+z<lSumdTBDZZxsNOd*a2g}B!tF!#8?FM40(qRD^a
z<98fih0D?9MZ^7#irZHF8x^NU9gy|wgJ$M}{cc~#8NdAvKau&V%55}Pz3k=C0b7<m
z{O^&I#6j(A#mA}-=^|Z9OY~7ztKLVVFFl4s2@+>o2ag5g&vC=FR}1RPeM(;^O#3zt
z&d}oD{*9anerz4Cq;~<j(zXBtsM1Pr>mMpb+-p1l`SVt^o|jva#s#nC?%SW_L;K>A
zePsRPtNSi#&&YbAMoX2E4BK{b*;9X0uM`xNn2tjGekqcW5y3AM4ju71?rUfjLC4(y
z9!@3QJeiUHFW^P^Uz70Vb@s316NDWT$qn-S>ES+rG+_um%n}ivOfPGre<8aGnf7}l
zCO5jH=sknF)k=dAR)i*yADjV8H#R_b^Pqg2Gl(t%M)Zd1<y|R0OOHZbScUe|jwAL;
zc+_UGB{#{F1$}LP_Ww+t{pZa7l!vw&d@ivqs<7#Q&Hj|z>j$iyn6zxN)@sEQU22Lw
zvYZkp0Dv0jS9qPH(!*LuBF{_ZejChQrTsMMFPS>!Y=^k8cwgx0vK3yw{?^smDNX@C
zr$BBTNiIZA9+_w?ihT9BW7q)HxcX(613BHEk<0WVGoU28X3w=EjhPQF(#c=6R1TeT
z3=RPRnTuLkCe2`i>Qn;OqRstUvRFL-AI;AM;u$J0_%pSt#DVE)H!o4jn(i;xDqhC+
zwk>J$9}65n9G!7&vW)rX6-!40pjdB;>sg)gKz)etOzwrx7XYHp#2{cO!<CS^m!~mk
zvePN3x2h9krTgol@%?E(Lpk*&VE({hw5#tHd|DyqbN~sQr06gd-;pAD#&hRdDh$$s
zWIfoWHCrIMXWHMTjuQax>Xnul7MLjT<Dw6#%{lz7FK)a_LvMaeYN_%~OkszHiKR9`
zV({=8E#>uFGdL}sIr<Qo-ir`y8Lat!5()nv9D)~deK{i#aM6@-((6>|&5@bP*UrOT
zK6mQe_{FIIsP_netM^yuc<$|sB3i=54FW1c84~076M_$NslTLvQ6UoOUN!#tp6DS&
zc=u*iT|Q7cQ#iToV>?B|;YRn|M6OLFWWnfr-$NCJ`rUr@#lmJP!S+w;r}qllPpacF
zE-wx3*gDI4UdHQwQq9L|pNcX5ME%Ob_{dU#(c!_9RFRh9Z-^~w_)jNo>^|V;_rT~~
zFNZwv+WhUv3V0?59b;-NweY#}i@$ldwp;NXqt9J{*9G_(6$J#q468+pEHLot>M{aL
zsWLAqbH|xR=WsUCToeAmMWRCfhTgtyc}1lHgIj@Xu9E!b|7;FUn3&1HpSdT1VIrm#
z-5Sx1s^l6$$^iAWR+_9UJ`}sD7~q~_QA3zuWGYi(Y_^gVmmV6M27z*9cwo|n(~Z1-
z2{?R^We&%_hMcHc)`(Kk;V*2<pWJgnaD8w+C|3cHJCFiI4Ec2#M*={B#*h?^Wmek6
z>$PYDaB=+q85bw&1F}sjlT>qc*Y|OHkT+3z%u0FxI>WbM#17!;k6<`fQ~t8n>jvvE
z(+Ilw02J^ht1j;bb|Tie&_7^z&y7e8VcSo?w?3pw^1}ZLMv4~^I9gUCQvQnHq+)jC
zHjI$b<U&Q_It;=Ybskv`YPf>&0CE&0SyNC^WljEcm(S&CP<&q)2c@YG$ERImnFR!K
zR$g)ND>%Hvan_i^{X6>52dil^QD*?}1|RUMGl@^~ivIJMR6^n2*KUo*EY|IPItano
zeYBeRbo$pzAmNWYR8%b<R-Hj_z;Gz+uH`U-PcyeB_qX4!2j#E6vJMvDX;&=^DtK?2
zO)dU9T_pF5=iv&2b1v8$<cMer5DKNy;Xgv7giRw6y^-c6TSTux6V7A+r<eOyT6C`k
zXN#^D+j8FdGh3!pz|Saj!~jBHcl9(l<WOEU02-ScrHH^X>x3?@axLLCzfKL>`vp)(
z*$y$>VBt-;NxuXaXvPDZSPF1v3#bmD^Z=oJh?o;S<k>%nc(ua6XJo-SsHnZ`r=Oeg
z#sMoN-ASWblH*1T-TSEX1)E>_AvO{<1}>tg_~Y2Ozbg(mOIdU)pI!L%{1(5*p%hQu
zs$6>GI#Z*pY^z=t#T69%;TT1FXT#>vUOAZ4CZ@DXkGPciaI7M;u2_o&EPcE@7yqS{
z(}2dLqDtp&DW~pKLUY@SO7cu1KSy)7Xc$+iudEdHUmVof331(Y*Wf>`>X)eC^b5?9
zj(=rM|9SfXO+77cASkv9ef^Cqv{>N1`+5(SZROSaHzCa~yfF|~JvJp0DlzIrjD4<j
zjt`L5d+xN13RsW(^dtM!Vx2?83JtAdqFXVGt9UPrphEP01cQY%Szo}uJc>!Qn5d_i
z9BQCM03;p5jAYOxiN5Mt;v@~AqhlX6-Wu%0HOCSO;Ko3Cr0LvS;L+xE{{cVe{--Z@
z{MIE85WV(e9)C3iEmOS^{h|7~weiLCN==<bOwuzHPUG(9FYhFD#Qfyh%XRA-+*(cT
zAwK7e95`2ie=)yTZN*|&Zq!JrV4J&nc-J!LXeo+Zo3rG{R?V{1=X)`cA52%xtvL&G
z73KDTMen7k7-_~I4lDNGyy-k=)#|4B7N7}XN$YNCZ)Lp2{?}CwxQYHViFq~Rub6x<
z?xhP4o|%;8#^qp8a6}ClU~Fk`Uo>f$Ahy9E)MuGkLyAbElVy{bL?{A2Xg6?Vc8B?m
zaa>utUD+pK1)w_L7{A?G7|m*C$n;_U2#O&YxKJ>>@A&)z#r+b;814*f!V17xrq}xu
zci}!K_PLfewW4UQb^7ewi<8<$4DKhPL~E$6u;Af$sg=atPH@asmUPJW=(Qd3$hGKf
zV~Jy(j;`?Cu?T_-4_NzAubPGlXBI?*jX+&5W>+aM`v-s=6F>#RIfi(gR(AP52;1q?
zu_D~#N)tf(u8?g52SW!}=wnc1V~MlD5FhB&2G4JE&80LaqZqzShKy^s%n5p0QSaM0
z<1TaF!gQ<LG0`TGPU0CP1moN<Iva$|r0+MtBM;d;m{`5^W)Sw}8sqiBiWDfWIvNFy
zv_BzUQp;eBtWXAMyGzXHF;M?e_+>}aiO$-C5{IB*#$;89hFQTyX}Zf5Ej71wcDqBp
zZ4#dmn*}XS4rs!vzl35PD$L1FoR{G+{`1=jm(Ug?3u`hTy=$eBTwAcbY+~+gvAraH
z&v7(=FTHhE!^H!wX`SAi4=-Wvex&=O<nYn{auW2pcoV4BAbx~@`>_u{096iF%5dqX
zVdx$g97=l~2c`u19>t<)x-XAYFV|xUH-%S6X+KPaa(hn%iuh9=^t;W|Ky<d@t>E8j
zTC7HrUiW_@CRx)GKR5Di36jafi&>984Se>EX<{wV)XbJzm?H;`yu<987H9iyr@Ep^
zRO<q=3vB(y9Xsk5iqAC+lFxoOVdi_CH}2siqpa^FGlFi^D@}tJ9ztzy4J><DmU#)g
zes)bVW|(G6RQ{5cpr}yqeO;8IWQvgUA*Q`pQ+gA4#eAnVk>6L-)~M`x(aVgOwSnGd
zagFZ1O^i0DCuGir#<-^J`ES+Oha^P$8al_7nkzoGf==iC>}<&e{4?yDu?b(*%&#r5
z_ym6Bx`xh@3m<JLrDlq;o`nBsL(D;$C|V@3nOabE(P%K^eDQOuP;I2?*-MdEmCh@z
zKE(n$O)PT>-ag)&wS(u>yEh}-S`Y&?TdjON##04`i&zBkWDZ~Z?(uWK$O883Z4oO}
zJ4xFzlLq+BrVS=s{-yl(*JwOzC7M}jPKE1JqRQ4=Tj>FkN;P`3=F_=)(tSP~^a2tG
zzue!@KeSt>tc6p%OqPBu5Cb9LzPZ}{&F@vQUdKR#;#xHqjqk;j;#)5;p5JM8HRP*x
zt65^rWoz6Ow}fB*F*icqQL{6S+sC*p%*CMTj^(9ct7r?ZQap>;g*8aCPP@4Jw`{xG
z&B=syA_FYfIJyG$gAV^m@fOxadQ;SqsHc?W2g`iyMkVF$cAU@Z`x<;vq%P{e5Q)SN
zHZxbYCz3@TFKvIG93P73y+IQ#sv1lha9Nwe0#;WRpDRgNbiHQ@eTS1(=%g%Auh@)7
z3GeGrhz7>Hmy=!He9b}JD)JH*)+hMd;pKmSOo3Ayz!D+>qIcKAgBs(K#8~(ClbrC&
zHZ7rW!!ck1p5P(kYnYx(k+`>>=^&{g!iFR^z`q~m_RkqC$1z`x#qdGBe?|@R$uVL3
zN^(tyg5f>Gvma#d(Pv6^NLkBjOzPloM9MN29m4Z$sqa&y{f@KPj=BwEX0Z63B#tKX
zKF>G9TIq>3bErF}-`BE}gTdBkI5|<pL_`psU34%)xB|yNcL@`4m)s3dzlp(~>CWre
zJwtvm*(rZj+JAj1AV!1IP<78Z5EkR$v-M?t1N+uzKuB@9_KI}c`&p4<f?T;l^h2QV
z?2ZhLT3WcMBK(`6Ez8VTMUViTuEX`^4iD*{f~p0{ce)Fi%Ql7u5?HocU+QdP%Jxg9
z$`K{U=eobcePL)Gx)xa1e|2#`Q|7iRTzun}y)4_!NoHw_$Xeplh>gtCKds1eSQkXC
zK)Nz>q~gKhkrkr&L1iV~hXFn$``}8<bJwBaY+6-d<Wl)0gC)MR@L<K)F;iIWcI^Cd
z5hU9u8PmYJTe3B?IowTpf^#G8OP+6ZKbeQeNc0A?L#!JtlKK8t98;kBspL6JyBhlL
zm6BGOjrfsxrcj*lH?#z#Qz-$km~=Pi63yMqVZz*7#>y*?ACHOWcT+z!W4TL@cy`;1
z37K_NT64C!O*9VD+$*H{-7jrl(S`GOJ>b)Oy4;p~qOoO))K7Qdo<t+7X&Pl+Zl7OE
zb-8IAB5I2_$Y6u6JL3T!bIctPk*-aho}aI6puAQ`yQkW}-F*Lc5z5Gd=vaCYi6F31
z>J9`)gtX}Zo}4@B2^12zKF<bsmQQ-~j<hKk*6}!;Ah&C&xYR*w+DoSc42s)PGpP3v
zJg_7lk)r0ci!`8d#>s;`Pw8@32Wp8hP!uNC*F2`h9HM+eGFL;zKu&hLXT6_zFuWzu
zHMTVl(;`FeY755eoMcUOVWvrRP9r!xz#Z=Gkk91aVR9epEu5sutZJ{a_HiWkG7_C|
z?#QS_D?cODLEmIJj;^2-WU34s+RBdh<Ug?aYSihK+>|q~g!35^#}FNuUYU#qg-aMr
zvn;?eIVuIkN|RvYmpe(QPp(045D8z!sipaws%ctlma}S)m_)zATf6$a$XyXj_+3*5
zmt~5g$!TN|`L*I#M@%!6{Ys6VTT^P<<-kIG@%P?jhittTlhv~MACx%BA`7gsZ>aJ?
zM`)levO;W=Pn_2w*r*UK_|~oc@<U9b^3R4X9+Em-ilp3lMI?8z$M202Mcq*M^y&$S
zM)fAbR_My>DNjUH$~rVPTqRA-rm$zNOo|G;9;l<x8Pb~fla6;kbvK$LDlpSvN}3|J
zA~M95vd{?nyPW9bK99LI!Z;!3D^gMl60hNWkoJV2DUkTN+HsVZd`)o@GT$7Td_s>p
znGM>Za9UEdaXLWTn0=5xzew}jY<qU3Rl<>c-q@8>T(kJ{teGlUKf<;jkI#dzFpp#T
zd*R`+wutp^HNAsq71owxB%ZWZh7(E6U2u~4EAqZczBo9igHiu<_TB5EeVuL7uD43r
zDq87I1Vn-FCx2SeNWKflM1^_L0~Z_SM|RulN%Oj)Xq!O!>5gtsVA;q26q02`L3F@=
z55MQ$#)jhqNan#8(r)i6F4Nh2zT&Lf!ib=QOGxLdBS3@N0H&A503RHwMuRN|CCME>
zx%AoiI?$ER9<r@H2$L48oMDtSwFWuqg^|bd1Pc}~Xp19%a_%ufg7>_3#B7GuJ$jVy
zpi26Fm6}KKxeUlY%MIy?-L|0Gb|h)y&kr25HR?rFmo;7uCCz~10n=B&#2fXa_L+83
zaZx52L6jbU-#JTkOWoSA5=BXM4zIU-<LwWj!2YP7_`!r!;Yf5m?*sWcR-@g$-|TV_
zVYc2NEU&SHyt<LWAq(R#g=12kk8=hYS_ud-bY#ChqvG#swEd=v3o?a5>@v|26lR)1
zxRk%9F0TZRt0H861U?_&;fdbpsK$r5Pcm0$<*s7*6)}&i6RALIYbN8-I~l)Miq%FG
zSAOoRJ_RO;?7u~z9L`d|(mgWFd#v}PqulS)l%l*jwM?$rP#?gKII#IJ5sT&fMwiU%
zbXd_MOA6QPQzIqQAWe4eFsgzeN9ir^ml>9z=~Ia~Phr*jPxx)jg=U{0Y~!Ry%e@?9
zVml9+Vq{nuV&nqud964Pg`C9n6sDNY@=Xlpo*Q4af92tnmPw9Y`}OnZ$BW_b?zm#=
zL>o=O2CdQ=arGI*8Bv;aq+at0E1eH`sPy)7KMF6`SV$;$5nj^`Q6+-DOA}SM<?T(#
zZ^m%TJxa$SBA75=ig{8#$Aw<W0T1K(k=2`}kOA}fky%jeko(nn#`AyA>i)ysc1?;G
zW(vZ&2qSicbCm3%mY}ADV^Vr{499@6Gg;o})}Zng3b6|kA4jA*!*J(nl%?)K%n2RL
z!r%nG>$RlD$W#tw$RjciZo}okrGSQ%VSLI7v3Mp&?W@wzL+a!24{^u;292m59Y3RC
zzNdw7kM}$0hQ#>rY>_s`-tLICI&T|$tbe-T&KS^BW{~m!C#Qn|sj2w&pi878!NN;^
z@V9lAM4~&LVCygWuOl?uZ4ZmM!S~JJ#o;8#WS%DqTUKk>IbwiW&%vX>%^@1Nef*$U
zBT}ARYz!5gM)A%+i?}e+_vw<)e0r><shdKZ<E?#;qVgq&#t*YuKixo-auaXy*2vP9
z9`QW|9oWv?HFK++*0BT;PKw*iLihtoqO{iFwVOHfi8QwO76Kznp!~7gt@@VMEdBFY
zDt%?Yng@|*F#8&V&ZBQmFW!FrtYutgGgKmt`K32LQl6!V<`cuE3J0IRBjVaiJ#@Ab
zLNBisqwikpk1sR2EppgtkAKh6zqh-+TP$9CGI8-_V>4;EcO~!iCUSqu=FIiZ;W&4V
zPWa?88Pl)Tsg9kFqQv)QD6NJ<Q>g~TTYP+O>2lqt?IfedpD|H}#uyfwZy{sE%tfO`
z(5t~EG==nK{VP(uL-fHJi4_QEUZZ#Y^M#l7ucLy4>P40?xNgBIP%MkPGyV4z4&q3|
zG9WG{P!y+@{dcv%U#F%=K2`$6=nf4d#(gmz%D!_Jf^*P1evj~oOWgtYRSM@3UP&uR
zyr(rc6bHn)Ht{8A5Q1XYB4q4hiK&lz-{WB_vL(e8m5u>Vb#I8TLQbx+J}sLE?5!_I
z;B&99Ej!sGt;1xypK%}iE<uCB^))vH9LRGUHdx1*g~ACEh>?x#m{_$zV{@;ZCW3(Q
z<Mpm-KQXxnAHSRi#a;!8gJJ8E;vk0Tad|)DwQ#W-7*g`cYt`GDrS8CD`?R;@nf#KP
z<R`7kwE@F~rJ%eBC0NVT{Ab~)V^ir53Yq+zt`#drpE#CU<DQYcx!$H{d5y5ogOB(`
zAu-5Jo$7hVT?!3Q0^@om1a}^{N3DcF;SWS!9w?Ls<|)y|%6w5mbD5{U{LWj}|HjE1
z%a;T10kUl+?DqX6pp7pOCB``Y{s&=DT0rZM7<C)<O86O8L*22}D9Ts7UW&E`H_E#O
zzOAgNL`F=mO|7Zrq74r^L<X<OR(QEWGdnL0RZqbPwj|#bKkKk?@C6a_IBj0y{+N2L
zwa>y-?lwQthFD^Kknr--mi$+fSdn6e!d^P`#O6jytKFKbY|(LtKiarK<1#aeQ{p0W
zJN|)FetatIX5Z4}0YOt~Dj1vu$z!%uP7JgqzAO@|tE`c~erUDBs9N|ag*ExW68z6O
zad}1mOXtrysRPc*7mAY9ivQ)DME;eI6AG>+w16Wq#xEtOp~RGMkTVo{f9}cWpe4jh
zd1@zXyr9UtQAQ}G^(uzwcmVM|(gtETT-`)3P9Q1jwwy_{u5(}L96~oP27)wwT9jZY
z<(_#h+3d+DcRtF{7r#)1FN44qcdlg?8wifLIjOEdnl+9nI@_Orxr(?dkh4<|ILHQ+
zi);8c-7sm)moU^7gHmA43ChzRlCAoBLbT;+{oJ_b=k!i#hz7MJ=J)d^gs;@>z9pQ4
z%|g-Hvo>v#Ox9xg>DO-~9ZGaG(7>H<&-2d&$Ld&zbQpyi$j^|FI^ZU!G`@IP6@an4
zln~3`xa6E?7rY|1UJik-M&+*I4F)}459`BAcZ&B=T}wMMjHGrqKmQ%T7bt%k{C#(B
z{1XKaAO?sA75Jm}eOVpy%&7H5z_DMbsc0%Ne_b^rV=5u0Mc8_`Qyp!hhq*^oCZBw|
z<_?(y(iDT~cb_g?q52zlSKyU89uJRi1%eLCLj~G@5^Oh3T#j#zPr(Ua2N$s_mRhdj
z7Gg6w6`lt?b<fYXD{2pCi#1Z__l2(fXy?DJKt6q9EPvlXDP{z&LkPWmuB`EEX?Bmx
zxfz^GmQN?|7-&mlM4nXA_j*cE_n@dz=xB49e&0#Dci^4ZhLryb1(3IFv^i+lG!jS3
zzLzI}<WCvRcuoFSIpLqcL&HXo+lvYo%HPayfx!z55kay2N&J-$BlD*727(A-aPuhC
zU)TFc`z#t@1Bk>&^f9h!vD96TkgW~(M(%0N@pn?-YqQb7I<`xTnYRnNiRr>Z8mCeo
ze3+niXJcf{$U!8tXVCs0rJ!pBYVBAt2g=rKwU-_RY?FvxEi@34`-wr}7lNlSQlA~C
zyg+$jNuU*k_Ft;BUK5{Z-FrPs8Ew9{Va(*eJu&n8KpV_XkYQFQl?cA{317N_a5DLf
z?uT}QRWp%DCD^r{i}FbvnRaC$F)mKAOG7mgPC{m^-4;LGU`UkhMP@_0e$iJOO@jV*
zT!{J%h=}Q60MLmZE-u{5#QfGfB-JI5Fton-mwJ||cL~wEU*BG9Ufqhb<n(?FD)&to
zW+%e3$8-0_bbNocJ-0R=U((Aw3r>3e@&oyNez6RKd?B<nDNdntt+mYNSKz08z!*|`
zw}KH}w{TPLsaPH-7^(GR(K9aM%}e<*e1lj!GG4Fehr*jyEfsI##8ckq&D=@rgd-Lb
zP~PCL{J!<`SxWfPaM`0sUvnfw^-s-|?M42}&Ad);BjI^3JutP=Y5YIssn#Gdoj;js
zK|%`-$v{XbDUG2-8VQ^jT%uU6mB<-dZV=Rr*gQ=fE(d`NhSfvXGaxXTEYL7f)(S;|
z@Af|{%R)K#u!pU8mCJLCqfn;?6SNVY&!#hPg%lt;1(FPUwwx9JiqSKisK|9v-4R00
zl%_oG#SXv{ZkUiZm8L`-@_fJA@7zmrg1bXgH;}AKJxNy=Hf{1xxUl~#T=+$P;Vd-=
z-&4{r_`DU2*0&Q?Vq~a)Sn|fbVK|Ae4wLTSgC`5Bkc5(w;=<`kNy8*6^@$1xO4Tf9
zP{+!eQZ{zW{1P8EU5_V4oj@vHGn8*1a1>#sCJkaUM9Ca9B5H@Zlgjd$M+LTB<<kuf
z$@gDe#_=;+J&MnjQ#q)xtcBGtHS8^5OOnMW<)?eUD_oww`)D)XwbamelUm04%u7}Y
z^|jU77LbXTOXp}*krSMj;%xsa(6Ag%<%!P1B@%GU5~AMNVC`7@nN~wip6oP#v-0xJ
zx~b&=NxmAoLxifJF)1^EHB=WlS2j1z_+{Lr*u_Q_>;x0|RJtNQvO96ZZin9VQ0zv=
zmd#gi{wmf7O+Z~lhV7q+UA{<MMEnkT*uFr?sq|Z?lk9(f*!j;8@RFbbQ!pq3(2~W5
z=Ym{WX`+ZJSW(?i{9$`qOz?*KTTBbh<~5GTw$5O<ZG>`j&G$${TZKsOg;ZUNYQ<xv
zr|DSu17%O7>Tt(`#JI7m%B{qkEM5)uQ@~FmJerQ@YN8wfYZicKjCzpvd3~mJA~aX~
zqC+62H|>nRxuj(ZJ#g?8STYb<oiY>wC;oK;6btJN%7C8u!fFy~(4VvaHe~XHox}S)
z4C>P6lo~a<Y4Jy1=*9dpu=<&Aw3Upw_BBEp$TyQbey~{wr$F;`aCe1^*o`{hJ>u$D
znLDiwqme&r_Y1q)iC7P1aG&Nf3?(Oi$CCPS{`eVIy5PK7nc^CAo*AbfV>9XXq)Z_`
zMExa_wzi&rG?DxgL{NO~lTTJ76(F_srAI38eSrGb1DS^Z<KiVXPJf~O>m$DMxc0!2
zRzn}3c1W)Xa_$~dxk)biw^vSYlp@W3f^5Q9iszX^$Em5v!!3-KidMhZZ*PD_QdU8I
z$YB)aP<LXjEp?<Dr>NnF!GL5PM=i{lD6<>yui^r3vm)=rYqWyPNlufufIV>&6Hl{3
z&NH_PQGdsE1#bfSn<-A>$fW=vuAfqrrPll_t_Q@D1|--hzv@c5VMHLfiY}2K4-zjw
z8wosQaf_PY*!Bw`&Puz&D=2BM{@0(tM@zhsp$Wu;uwh_whwZ1gq#5|*Z1j@&Ltl&E
zw)EggV2C)w!*oZXsI^ott`U7^NM!If0<)2PC-N+h0@{$TC2>SPn1Rn%EqtIBjJdV3
z!`je}zULc(xfAH-H+|-<vECi5=N<*M7JRs~JYm3z=q;wn3OYM|JNbcf+f0Jztt150
zQ%CJG6w1T(99zxE`zs!2#W^15e^%jospTvES$cWj-{>v4nN#2d*;Q=Us75P4j~8nO
zFXcNhyYXJsFtfd!CLm&_DMgrc9ljFq&0};wx1LV~0)Aqv0Oc<U0v&@`-a17t<OlL6
z88Xc`aeIIOH_?Fg^T4R@%{?s|7>SR#Fy?ucd!{xce3`}9ZF>81+fopY9jUgU1VR9u
zi0Mvw(x`F5g3hDnr#yUAW5gK@t&(An3eqy)oHIVEc9&_=?sq8iKcQjdx=8X+C`(ot
zhi!6nQG4vv3VU-VSF>~H2Q3aSPn-7LCm_b(#eQlolV4WESNdH~!d-Ugz_+{lszVSf
zY1ha=eMt}6pl^A)N7dlIjRziXk__*dZi%{({&psKq@H?mW`EF&h3sdX$y=nv_Pf^i
zKw5cP`tRsI-QFB;DGx+<E^S~x@d(w={U{aGuMw$nYwqsf8O4b~d%E0qD{3;uoifvn
zOp3`Vn#JQ%lfNsEN5FS0MSLiuD_6gcX;mR=N;co=X0F~FGx|9pF}_gNDc)K87$g`t
z?`8e-G}o~4kP`uoz*9YiLyS=Hy&YmA7R4pSL$2`2+5l}NW3o%D#~FAPT}df<_~j&$
z2L;ikz<UU8Qbm(2E=V1X$Dpa<v`ds&=H_g>CdAhA#A2`nt+7Ns%9>|qYn|eJCL!*j
zdZANBjV-M$E~|c|T-D`qxmEo~P0)X+hbK^jQT|+ICo-nJ5cr-zt0z)EZaCrUt=<l}
zEyUabBtICEDZn25rgDU9WTQI>ljYElk;Vkw(Cnq%S^xTMG5Lo(I7n;z!Xqq;h<TZ#
zISU2DoGGGBVV2K73*Bi5J^-t)3l6axu$aw5D{d{guFcLcam`z}+dlP>v_dgfTVF8y
z5}f1xtsLWjR=E3DqeJRXzvj9E6h^{rGU5MpOAs@Ms;Z8(cE$aao8};%M%iwMX)oUT
zn~B4B>6#yqCn7fAR&hN81lu)*_-p|&;OM?lO-}dpo08u`Xw4ld90TQtv(fnbqayT%
zz7@(Dc&h8(!~ZLWDa5lBm5%X@>WQ$q=O&<l<cnQuo`NKO&6vFcLzO1reLEj_6-gXS
z-h9jy@XDY(7lPv5{66J7|4zV^N+fk%igU4CennVQ7DLcQt2Kd&p$ODgMMUzWEo^#E
z%k^GHSa-RP?({r)uU;wb;OTZ5|B;0ypz1{`)hg3@IyPleNZ8j)T~A@^@EbJ_<dILY
zv(1dYg2z)I{eC^l>W)N&l)YWe;=SCgn47kqSUHWx+*aFIRrT}zsi1=w@c!%lMZ;$4
zg%|v63V&Wnzo!r}XG~hJ1|LVY&J$GVx19Ii0S<Q{d|#FI)lFQem3IVlHR&4dLO;sm
zX`Al_zNpXX{4SPj?6#zFc;bsj5kAAt1NoYPtHW5J84<bE#pLciT7%Gebrm8JE?_Z4
zUlyy<?QjX#=HWSam$6qcyid$7@cc+1N}>gNHICWCe&XBeHFE&L{9?c4L%PXsVsJ~j
zSQo=|VM|f?G@)G*b<<Uo#;ULq`+L5@M)vmnC)>v5<wHY+{Qj);?ow;+cYfH@YUo?7
zJSljxHlI6Sxx~+VU}4}@#&i0pyH{Nu$vxu&zvWaw5rp4W5mB1rBhRk<WG?m%#2p-6
z8xPTaq&GQ;-pt2wBHGLmEsiT@Z7Of3XUOT^8D&Tw!F`vG{EywgfT6%Yg<I~@bFe$G
zAGrfziwYVf=*<wr(*nn_Q%`*t*bsoTX(mmn|5Zv4<p^#&80@n})`LPa9%5Ka|H*!6
z@Wn42fos#zvjS?i)s1T8GdjG%$ggx0+XMabKgJ`8Z7mR&<T1BbiA_y|_6)NSJyGwz
z;566f#f~haSX{1)>1)9h*#G+iKeXG+?&EFI0C$qaeIpno6ZT_J;OEK6;Ld3_`v%$E
zs{|x;OEv~^@7tqehv)&VAG*mIDevD#IjM~71yvxCL3uV+G84PXZ=VOCBDB(daLrtp
zfo;YpA^*wS5ol=&Pc4F|(KZl=Ms&GtI!a~g)F59`b_?G$B}|vTu~<i}-@HrtDNa!m
z3XFyLvH`yK$m|j~(ibri=)}!OiwSDCf@BgY8^pUo!}z7o%(z<w{e0{n)M5A0XVdTG
zprP{gU8IVN>jvzM{<%DUgAk{}L3Rzv0%VV`Cvj!(#aryPc59j4WeCK_B)Sw2()DiF
zt|>R>X=i<|R*jSr8L&2CCq=ev;Pu8la6ckG8k>JE0_K~M6VCHN^!-p3Qj5nG{aAr8
z(Uz;<C2j=b6DRAhmDN!<6fzThZH<lEvO3)|+f)rrTS{ONY8G9#)*Z!u+3hJ!RN@Zv
z&c({IKAGBl-!8Q!@V`bX6AbV?yH__G-;Lx~*>FEOO{V$bu914X$*R2u;uVviB=~8!
zL%1F@5VI@C*Vb4#()?hvJc}53?KHN9n(jH^{HkE;2J6GnC+QNK^Ze*<`Yz12v_SV+
z3>cC8LRiLXFy@^!G@_68{TG#(*C`8?_?OSd8TWQ08T2@o7eDqnrptT>`d$;_!lug}
zES~9c%zpk1WN|!8hqWsfQ<Ukf7Bf@4C!17vyHhEmX5xwrW2N|z%f~x{-@<2RCyH5I
zBt@Kh%5?5V9fKX@5p2$Qc#oIe5Artkn><GP)@We)=9;4KomDgEDg+osOzuL`-*j+3
z3~qlmSP5FxuPkV?_siq6s!mI{a{;;-i%s;?6>}hp_6J}q&jDWe^?EIhdtuD+%WiEd
zW<jruRieGx1NKybN1u};t*?uohDULn;us$kpjJ3sj8kL(Cgh6&NpciaL(%Rjuw|NF
z{~8`y2<fd)ayyM-Ar2}SSVha;<3uVweh<F^P=^>W8(Px5dd_-rJZLU`p2yDgXCV3x
z^|!ttxdSbpt_8X_IIFKeV|5^Xzaw4Ca=jXe^Ubum*4PEhtPj}^Gs*`nbk=bI{5sg8
z$%*SZjxH#G+GGgB$I!_m5*=LOii9J9@A0q5A`L{A?(6k{`-zlpT(<Wh#<zn3-ntch
zn<H`DrO#e>=B-qAx%5AM3R(dQSNs;{+^*<5G8vT64F{r<nw<OqXJjS(Fuu<pXDUEA
zcZNd5$b<c*r`4d?%CdUR@uv&TR}1Hoyg=bt;S?Mp+KV_=+Q(D7J5I=)k{oQ2*z-OY
zA0iCsts#6Q@!*f(W!gAj2`-HRUmu!BmDhr=T_=5*QP1CNCVwbV<E-i)eXT5_nw9!B
zTwIu+j#~{TA1*-sLYbbVGS%loIguBs)Hl2*6h(YoU(svakk|0^Ei6S_AZ-I4afs3>
zslY(udWYAni^18Eh5+Ioby*hX#T64Id4c<!x;)f{ct+aLm!yno?g2F#NR4mPfb=ag
zId9sAgSJiW<MC4A%XcZl*PlK?Pqu!%(}!5KRYT3{%xloV*xc+@sPG8s@3d)FzUd(J
zHj0MY+|}st+M`N|qo=|Emn4b2$Xl*CDRZA!T91oYD?JLaUc=|FciY8DDDGO^*J!fm
zTLW?+9o(+)vD8z`&jSjL`zH$vbYK%890Gr-GW|Mp(gDKYw?o+?IMx??zg_lIy5sF<
z%7344DEjlb%}#=FtcGN$eDrW3(|c3}A7cu!NM>6>>c}-w{0(c4WCa!OEud_-+5caX
zp~?Cq{Kv8(V2eBD-E;XRVR6BQkwMX(y+#x;93{XtTHhhs>BaHKBDssG#_|iefHfY|
zV|$|7rIDOT|Jgv3a)SX|C4~{2TKG-68VRLW#6w<iC!Co^t<=`4U-lx@!7rmOrn%Fq
zAg2u#d^;f(8w%h#g?Z<{+pwe<B2P;^D;WQZ&6L*j?M6oT(jrnv7Y0WjZ}?Av9vu|{
ziUo9ZR_AgRF2qQBIqWQP8g^X<Wz#AsR4D;NkQ<lFEBqMzc|rG`gVxvD=5_c-z(rwm
zJ~esC#Fkml(a5rxuq0#S>GkX5`_ExFKWr?qlXsxDNI0WJqWUhj+I7#9M~$bpX!-2)
z&Xx|H(UN7gJai9c443{X6|+JG4p-hi6$5sUg`L~5bxYoGi?=jmKst2$J@}HK2uFeK
z;qj39?w--)*8BWrbc{et0m3lwpRB*<FluqU<GF3K71@0+2lI#D$hi!xU1cwKO+$<6
z>|L0nG^7Xbh!;UpI;M+MNp{h_+f|6=*XOmFS0NV>(*>Su(w-?mPbxoASM^lASru1-
zh0R{Y-M|KsA%eKSSYy9@pwxW1l6UQ>fAD;B$IqV4y0i?)+6_9IDpx2R36<+jiig;!
z4uX`ZJ2SL+%I#~xoI_wECXU>9AFFUE=ZS$HIr&OmUFjCWV|lATDX;zS_xf%7VXZ;4
za%6vJMtU2foi3{!zdL+Gj)3$XAW&-DgTh&?misoyR`l^uR0na63layh7dnt3>&%6P
z-(M~4j3Sp3;`6=buxUH^7;||rZ@qP?vCBcsehFON8ow`}Emi`kIxNeKj^rx&hA=gy
zF}rkG?i?{pVVbBajd-X=n;CogWO1g<-f}eixpO^lp4^XSciG`r-}G(F2dDgYM*|Ke
zx2A|7fx+_R1Po#FXIrnsWq`eOot}c8mj@oO04DH2<aXhQ%}T%f+Be??7P_Ar!gX_%
zbb$>DbYIRji&a#r^k?EmL_N<t+z;(O*z7tjtxZgO^0CS_ke=`TV7C>k;Su#BgLn-2
zCZIJItLL|b7q+dRy1Cr+GJ27S{)bmbf*yAj_s`2yj*#oTCWVCKh^1wYl5=N)8P3Gz
zh}MJSpKJLZe$CM=igbQLDmT=TV^9Kx0$g0CEjFR+fdcDYYN4SV-}ld%LlCJU@J7m^
zxWz?N!>6+vK+D_=c?Opi*=y=36gAjEs~xLnSPPG72cq~g9?i3@aaXzioR(O)!S;O=
zZRPrlyur`&#dz`!m9z2<aQQ>*|9DKma<YLa3TT$zBQ2O5F{f~}5~BJ2$qY21m<F)O
zpSTd%cp5K3(Ycc1`!yD9$~mWt<>zfQF{o%qVbO(wjtM_R+94^MOVRbg7P1HIB0|~M
zR9G~g*I{RUe8r<Ik6&({XSaL^tX}T4&A2eC6MRXoIt)0jK&;H|^IrNIPOJD-9gfem
z(hA6x04r4zuZEK31}d)no?&rhGxt;wGftn474=mEjdRlZ4;sm|22YgiZeL~<_;4^M
zx#fwf>J48}Uc>j!osFW*23^WuA0Fi<&KQ(;6pNhSG8{9Y4V=7mdP9Vxb<lH*ZPLPq
zMrYQhp3Nae_ji7Qf=YRP0i8aPKIiB;k@nDPWHo67d${YfTOdJvq}Yic_D>(8Xu}ep
z?+Frt<nGJq%C9Pols>*+ae1Mpg;A@=@j)4Apw1i}g#L;Jl6=xEX%S^D26yk%2V6Gw
zvYg<JmeQq@kK?m>A%mMjKsy2aB@n58654BDwEl1xTV~t|?nMcS4+UCMcyaM8FK1L{
zfQP=nZ@;~H$nUbBt6tR=??7OiRuCU$<nx~Kq0Y;$^mK>j-*h$Vzb$txm%fUIYRl$u
zU^p%&oo60WGyOz{a9dOzxIX)#ZUJvFuIq91kNRVsE6#GqDUl|V3$eaR;H(nwuX^tU
zMf!V(g@A>LV^*DNIq`$#p1t6XTKC|gu{Ncxt)S4dn6P?_b&LCzK4vc1{cV>-$(B4d
zO%FDpqYWmJ@882=^a1)DHCN0w4*^8SHZW9z86V>xJuMvgdcvO8{RgN2wBDnkctWqH
z4?|L*z4P)$Z_da5s-z`gfU%(<HXRDUa%5Ov-KnqQZ(_|2IG0RTq^dj$PD{E?6_2ZU
zU=XK@(Ijw~2=OUA#ST~lLZc2nTQ7Y|s0iaoa&iZ6VUgE1*W|^6{O&+v$%h|2%i!<}
zIW0ZU2lQ?!;Z#x{3(fo}u_Pg9&kqEJZS5u86WNJYyEpWY2faK3`cXTC%mF3*9mL7W
zIil_icTV`<N|J7z6Xb^rU9O_2e~Y;-)V_G@G;OM;7phzAa=v%t+~%`J`J2GQxU<KK
ztT&7XGVt;ZBVgjZ^K}F6JUTaTwK^t~Q<dy2V`NAPN@~p{5adz!1qhm3>R1(LqX@Y@
z@FP;EBXhzdbb2<HUx<6&5c)Y<aPj<JPcWc-M?~rVrqnP);?#dh%s0g?Hl5>cFO_7C
z<x?vBm~-EpXgCSuSVj*1>#2#v#jCHKFXSIdxPDSd@?(dW?)QoMFqT_OCTtC<7?b_f
zv~!}SUTJyR9FvnWAIwm~_j$g~BC;GbWAk`_6O4{jE<+G}w6n&WsGHAg2(?mt3zn%9
z#V2EEfOB%8X80uZ$Q@qFpE@K$A&MWo+U_gA6HXJA8E#<r@k1n$9_?s}DHg(a#1B_%
z8o~{xY0~qNR<TIN0UK?=I)SNyNRr**WOnSX)jXI=Qn2{|+S04em<Lv8-~Fpof_8za
zK%NJ?6e8FLo@kffxZEHj?!EZ{kGSf|X5@D7qahEyySvqx;tkt0lb{CG-gS(ti|uiW
z=^4y3Uy6d;m@<M|Jg1IG>-Y6uh42UO<7Mf+39~{o>FuKHoI>ziKDp#hRDrjjhCTOJ
zjZa}%N~5UY*Ix?r(23B8=q=n1gp|B4voHl>g+pW2KCr3Nuv`Avlb)+tDe}2foy?QA
z+N72&^|f93!HLZHGDE$*_Gq!WYUW<E5SH-n<Wf~hKGrz*a50ut;Q4abF>6dRhb99G
zEjAIiMAi%M>&i;&Q&`^FXR}KW_oxfYk(89vjqfrxKbcB01s`@t(?=RCN6a)tIYHu~
z%S=k2#ga$kU(`1Xeb^?;7WtU$&vTr*UGHEq7{l15wpL(j;KAry+{!DSkCH+y^*ZrQ
zV)!SAr;sM&CA&=iVEW^08*43X_OkWwk<9t{VaW%0eU5$ONH{n+6dxa2ORy+C|ASMv
z$4lam)LI_FX^cC!-fS?yeRhvge=7%e5IQ-a?(0BoqDS_bKU{x-OpW~E^Ln1e7{VIR
zjAulFe{q6LNHTavq1g}?Kts)S=e}czuX>~zw-)uoG%qXEpvAQAdO9GW&)jUAXEE18
zRR1Coy(w?gd#E4kKN>(0LVDeUZ03hd;WqEc1G5X_KDY*<$v5l(X2-ChKW4|Le1fiz
zJZ$)oJ`xSl_Ww^|kP7`<bIv_EQ;CZk)-8J{RP|aQ6)fsf#irTH=DHEjt?*%O$XD3u
z+&})<;z$oLe?ElC!Qfi+=dbKpJ>KsJQCxt33pqwr9|$YlEbyo7?VESl4^fv#PR7!L
z8HjE!F@8cE+dQrr?4MjAoxCrk;6PEl;y%pIqMgpM0+&>>mMCs{^C$MyYf##!E^({4
zm2cVKI^I&n5-OH`jXfMP!etxX_}W3_a?ox0mPs1rgJR<f$`H?NN3A8v@b541MK%A2
zv9At`>TlkrSp`W2lm-Fml3uz|KvF_lx_ikbr5gn4luqgHM!G>d1caq~d5_<Ep6B}2
z``2E}#qPD|oSFH|+;h)8n~-L*eo_JHx4D+LNKv<P+FwD&g*q`61g0lf+oTIqwoe6H
z_K#MTn->$~!HO>EUhhID-^`A${ls{ygZK6HYpmd+$}r?A1xp*TELh;1RZD5d$K0}g
znD9EJx<$IcX+;6+@^O}{LRx-=msHZnM_LKg(pZ4TKC1C(x(=DKM!o{6yMgo|{q@$-
zQ&xM6Nr_t%?97yW*Dnhin%8}Sz3d^3=+k+f8CY2Xg`D5!%~;7b<fT7H;u<ZEemUXU
z6PM-O<hc`@XsB>ROw%~6TjMvMcq#Xp=?i4nBaVPcLj;hw#5ysk#l1@B7gksi35iWH
zvf$o&W8XZBJ?%B}cB*9O)u2vea^bHe3e&fH(=$`Oa)5;1`jFPtTHOOm7A<#*aoFsy
zSYh<>uMKflXG+dvHJ12A9~z?HELx5mQs}gL4pDx6UrwO<^{p<t;M2uA^1O7xzLW)x
zLOk$7?R}tB)S*P>4?M(q^vj(nVS<TNK4J~yh3(-lx~f#y6Y+L_B;QO*^B&=UPU<CK
zN>c0Xyjtxr+M7G(c_DBi;Na@&S$cOqYnUwPk!GH*C`2!PzMq)Y9a5Sr#7P%z#_vKx
z1JbM;8Rq84VMN0;le|9R^H%hwzNj%BeY7`esvO<7w<?n$6){M>Qfx(wi;N?P_9U4_
z+e5ohxj_;>dNA#90IZ`vsLL-LdHYCgtnJCpz=o#2ZObla<L&F+$Dt-}nPIf>``fnm
z4|*)-l67P&PTPl>w$nw=)Jj85U-)u(-8jjjU{39(|Kk+Esp(rG`A5$rZ239_0|gtr
z`2PBl;j2Eg4pg5A=oeI0L@z)fJPLzvjKfyciDn1E9fsB}ERqqpkyPIP0rf(Z?2H}z
z1}D4$ThO7xj^w0)iR8e7V%?4%At}|lnLS*SwUHYEkqG49TRE16y#}!&f@6~O;@He%
z3ym2lMDt`yV>;~uSajwnv!_AdoX!U!ztt>82#A8}+h4qiCV)4w7S@)o^A7|2&+ZYn
zRClv{TCW&TD97~j*y>JZKYZoC+iG^pRoj2Aha*vhh9_ayjS=ZSF7~lf<0U3;$g1xk
z$ySgPc5DEG19Cqxjwwo%#7{32w@m+nIfL7G@@VvWx?pcH%SJ4`fKRufpFajZp@p=g
z8%jy6W=42P4-NuJ$)eL%sL~Jh3|=pa-qy?SH{&<DT|Kr#dpnP<XsYNSB+1-*+fU9a
zex)U~g+`8Gd4e=uL`>yg7G<H(Q3Uy1A#p@7FvN&WqfZOhK$uCt0E817o%!teI_DgH
z_6GZ16l>Dg-zjy~3ikQAJ?M=Klo#Tce2G*&zpRwRy*R8sIy__{`^cRwm%tE{pI?)V
zA&>&3IyJXpOZZ>)2~%8-+JwhTIX$eLE|3#e?T=3EHEX(l1oiFIj^?TCe}%oP&7y>)
zuCPZbl%-j18u+0z>V{eXn?Zi%48+mJWn_>S3v28H_+o-`<NFbt##REaz8Kr%(66;Q
zi?b7(KlGCS@Dw&hOrATPxCSddj=1fJJ!_kAv4t@lut+Pf<8~<WW8wQV)GVanFII~#
z0q*C?`X(FzoP|6s^scb;3zKehf>h+)=oID&k_;vV4hl`af)3uaoVT+bz;Qv$&c^H3
zG{&3rzB^`JXqde4U_Mj%dKRrFForGn?%J?F8fx}yp+#X?WDnst+jezxQV~4~PYRFb
z<f%l0Eqw!TEC=rwA+Klh$<@OhrJmP!hHWFHXYcCeVEwBJF`v|*3bO{HG-GCdb$p<^
z&AR8vT`!LxXN<!nuk(0aDp<$gT?icQj@!doUpP%a@dmj}{R_1ap<ZS94Wjw#RkMUK
zihQ7WF9Pd8!)G7sBetrxBGf5FCP(NjmHX}!d7g@Fv}q_jUbFVPLih}}#&fYsLV?cv
zXzUo3v|C8{w)navy~A_sk-t4BS8?U8k&*_8;E5=Cr~AbkFIg8hhAFhWK#1tuDi2*c
zGV`rH(*b6(MbM8QiAMF3Ez%jk$Ap@_?;Ce5LP|h?d5FOO;vpm21t@tn8p2L(t~&(Q
zAMmf*7r>qBDFRTXs*l4A2|xjoJ7Sr*!0*{j9o9qTN*)6un<rM!DLF49a^2cq!*_Do
zGI1N0Zmg1gC4Y?c=?N*oAFAFRFYMJ5CJw;7Gd}rZOe|p>)U}TIz-WK)hMQg5cmt6v
zl5qZP@>NPR0DExzpg1^m$hkY+e+I*m`{H2YBBlwH@0fEYf^}2H4v}A?kcj(jXia=(
z5&O||^8A9DlVH5C$1=Qf0kDWMBfX@D>zux9<vyiG=voyx@}=f?CKSK0-?X-rhI%)`
z>x{Oy*ep7EyX&Bo?k~*orJ`z`_H;O!l$+m<*h5G*Fc@sQRf?dwEU7`wVVemTx8Cn}
zyZzKYvAWuj^3lxp8C6s4Or^2~WVd-C<n??o|62<HL+nxTt9CZss_yu_ND9?lIh%k3
z60VT4gE`yqUjhJWnB8}@G$(&^oRim&R&-%%Q|EO#Sd50(PrB9fx<K=rNj#sU$p*a0
zF}tycP|tJi-8_|j<A=?|gh9IzEx4mw_v5DkWf<r5X8H|(lN-Uco}lb;v$D$PlMJQs
zB=ydTV!~0g5!Q_0Mf@vGmAj`u2LzSm0_0?4N_q48)x|Ezmw2UjYWao6S{>JLS|ApJ
zd+3pXxN~mkGsfgY;<mn=Z1l){#N!Qdna_TC{cj3^)!-)ha$kPj+~3V?omS76DWgT2
z&9~W9{+wv$yjGqO%fr;}V0bdX7+6QrBnK10saUDffAJd|R5l$H-6DocRxTeOC#a!u
zvA%IF-{AaS5!VL42~!3#f}()i%OgTHHup~qGnzP&Kl@|4ZX8jBcyIg}-B;@)>^De?
zFH1DP1*IpGpeFD+zF<&&RbinrT@*+!8QSe&@%|ARsPoeA`jqF<Nw@uA{6I?b>q0XX
z$sF@Z`1)Le0EaNt-ZhGvVfOAVyTmhkP(Qxoj!mXN`Y5PMlEY%If{8{p)@Rv#s^GzF
zJiD})h%{lf1C1n)!}U<g{%-t<!Kjp4(Nhk1l<4KWssG%x*_Tz{4aHg;K?#cZRWY^L
zW;f0-3&K3iPBaj+T9H$X2RB~xF9||nY(iEtD{&4W1t+oo;Tf%v-Ynxe^|;7#bti<=
zuP=#xW$<?B+IFe_4Ws%&tn>`T%}u%N!}IRTNIwTMfbUAOqw>v!-RKC?vU>GAxELiW
z^e5wFBI@G69DZGtN>t7AN8xHctu4zpYMg3`YeR~xd5>=4nDz`nWzVj<p!CmC_rjM=
zf&vK-eAj(pC&cJ2+Y@N|a3$gkHLtPlc-~HIyStXJYiPFnh0#Y(jF0XBryIa~ZVtjy
zE%YKbh%VOLB!eSF$Qe35W<6s{rrRcHcqiyj7)s#XjEKXGfFN=1FZi{7Obn+h)6OOm
z^%)5diV;CN`mLyL@QXf|IUI}*Pn-*JK`~o^rk3}J3@`CNTEZZ`yYqv8+6?-C0(fjx
zk=9^r#UhOsUxT?e560^>0poE0{nZYsbN7+%t{3~|uRLmPGlvLA(Frfc=cr$FAWMIn
zO6Hv^_-=2$sJkHO9y)U0KVQOPT%<cQ_5~NL3W&vWm%!ns?oK1MOZ*^|XP@rPz-9|z
z{qUD4c7`569AMF0c0=$DZ;x#?d^C94a9;jttE-!qbpB)YmNz^dPivz8I8NKuxF$Lg
z@oJd*9n+T5T273re#@JMI}z4&@DnG<;x{^oh?gb~`9=5_hJFO!eLo@l&u3@b=a#Z|
z?L_M;*%X2vu;_Qsy51VPb;n7!vmG1G?hZJ)EedhIFGTL>sk3LEZ>!y@D7jgi(q!MX
zQkP!pa!^JF&#uBj6POsf=*G9M+;$lCt{I=bmcvG{N&#^~+|79A>^*du*F~B9m<&Ow
zIt{XFL#;tUK`))-8Q%kwf?+%Q*dP#2pM+Ju`Q+E}2x1<R1CvlZ>)vQ-WpZ<e1fUmi
z&bYu^`vesX>ZY;r`>5J@k787$GwGe#L>TqL=vLt-`<&jffk4>{7@JzSToLggoV0%{
zdL<edYcX9ku%UI3rW-Q5kR=`@5$H?rbaQ6zaq2-@w=1Box>oRVhqvB<UY?w&%7V!x
z%;K-^aq<U%f&Fkxu9H08Al?){b&LnWWr*w!HKLJCTWAXGb$uAF2$Jk;?F?n)!zSSs
z0WeQHnr`D2hpm$kKA{BrI@^_eMWLS?4p>-MfAp3862^9b?BFurai=mispA@t_AC<$
z|7ndwYDXelX9KO!tOfL=5?-QWiX2|u?CV(<MPxGcuRtFtWVmH>g@cKm_wHWZ-u0xf
z`WmSwzrGF;ff$(oXy&M~(p1B$oO%4L)i`c^rd%)iaLG-CnEQao$pT%vZa8&*mLeyd
zOz?fQ3U@I})k@wQYI{!^-1)u_ZFd}T>SdUo%MK!;Pj#~gr7dsPf`ot0FhkXc2y){)
z{1;#FM?2YJhP8Bu|D=_X$OtVG&No}@WWF#7vX7yfH(zb%Rz70r$Pv9zEU^FnmBXcP
z%ld3+rUUC*N9pzO@guI2Cptp%FgsN9U;jp87SW*mO-zM2dWe2d><+}H%Wkojq~=3~
zaAKw-3Z;r;kbhW3X5$lgGA21#MbAVfN32}MX~#C&r37Pp^0D63vkBjgQGMF+A>WZ2
zT0)8RWxet2AU~~tne@8aPeT{e0p<rt=5hBD_Yw&e!M<i0>3t9R?0B-h*5|bu2=LvV
z0m(eR)hGfa@+&cj&9E;9G}{mtCF)u7DrIzVcjv!`zXDj{?k@kphHOZ8KV<O1M7WCt
zo8eUn`_ZPJ`+7q^8l3&P%e77RpAPp6Si3X4%dI}^6d%Yt{VH%b?Zrl)T|E!X9NkOC
zKPEGmdz$I@ODMsz)4zbGv{3+u)G#<`wB{=&XYg|ql&Ho@X*?OEXs->^@j&Ork1Oyu
zho7Ik65mo*ev;kIqD1fa$AcF;2*E9Qs%&}#GGY#E+co*7t2+KlpaLU5H-YpPH<-#q
z`FJ)x+d`7Vd_mBKL1$qdk{Tq~LCw4FKM(Hs8R}vqPmi$~Z$lwQk_zrKmjODpTy$9M
z41pgs+R`rWxh4&;R59pz3W~GzR*Z<21zg98ib*~E`pQ;N*o~VU4MSgO=gQz7!)ijV
z7%$Fxy*>))`qL?<=K@K<_~CMsCh7Wx4v38Kk}1!Kk@Uf@Sz@trGv;N)hpsi6w($gi
z%hWQuY7}fP0|416bUWQvtu|9S)(1yg#Ck!YP_0HM`ZZoZpc5i6*QJH41R{I?og^nl
zPji5BkId3*IZkfIno_L*HeRDlFntFsuJO7Fws`tQEWvajmgb~}q-YffO%jk&`NEl4
zXuW8FRctd`#xh-O>ZPbJRiaUARiO0D==LJVuY~OfrS$jIpFPJcD?~ReI=vI{`}~MX
z!Hyj013lKuC(Uh|HGA(jJQP~o1$W_IB<~NC!(U9tczTrz_lcr(e7MLa)A{CcX}|Yl
z`S>=}EBqd^H<gZ7xYgm`r&?!&?UDE{0vKD!QeV<=c0s*wF#S+4(=MvUp`J5y-4-wY
zp3%?D4?U+q`ya~!XuL1~Xj)4i-22tT5*hwdi36WPPivz-2Rc`DMfFwWL}5E8C3SdJ
zTPToW)J!=Jxo?VkgKoekqz;v=w=nk(ZM!c^zAsZryxyZ=tPM=}P@(r;xLlo3jidiW
z@;Sb1X1{{<R#IM7V($d9256^e3=k7J>hZUp2S$p1j0NqqYXHVwD)ZyyPYOSA;s02Q
z){f9cnjE`%yv87lFa7X31qq3y_{E&me5Bm_64{XE5~z9DAb9~oHVqCUkY}Ug2Cl~B
zD9H~hH)VgAZ29J0-MUlm8>Z%-=kA*Oa-KsbLa?JtP?oBQqcLMGBIy3ZGF9?1V+osd
zl++_nWtEx684CmoT_2y0Y&<ZG!md@;8S82$m2HmqvB@&F_<$ZsTxV>h#q+^)b3s1O
z^L5l;1PR<42eXFs@5Ak?A8+0*SXV<knOwNEfb;W2XJJnUg&oOnFIf4`9-oFHDt-Xu
zrs*k1kh|INZGiIkh;|U?Bj%2rAHyr*@x@J;xk2x3h>U7f?ng7|T|BW17jAbjPZlf0
zxrn_IR&iG!d_H?kPgd~lUtL{|!PdIyZI<e>Sn6z+@uaa!G64&3!8cz6JVGGfDaRu*
zN}z=K;lW*L;w@?(Je;?6YxwJPETIw*h9CNln0CFL!OhA8Tw1g0BRkpKZ=?^>Cc{aT
ztTQc9+}Su_I5C|1mliyui(nb(26{Up4~VqvZRYs&`AV<yqDZhyN>D;DiMhi*v%21S
z`%;T-kCK~Z>K3cJy5I4(6P4=-Ozh^k_YK4)0@a3QGbGr4X-E&soDF@@F-=nQ3wj<B
zK$^%xYN3k%bLuBgXC?i26zTk2xdaWIab=~6+(%D%RCxztd!!?kxHfK&tYxw2RcOEd
zob-4Of4J4N)qK|dE}-I@=QZnz8Bu){seqoR?GjOEf95hUp>Zs~W?SggSmod+Nk=#q
zj|@c@)MT6C_&KN))HxF>;(d9)+w_doF9h`3-$|_^^Zb^>Qt9~T-SsJWs`Nk58#d|R
z=nYI1>j!vagbQDwV#jP9X$Qe?2VT&N=*G-4HwCDxUUf!UTM;q<%XCOR*|9yxO@+G)
zjpn|bBCsHBo08Ar^p)<uaIC;~XkY?iMZm-=-u)msfl;|}R)}kihJExy-QMA954y)4
zb>9pUHZ+jNAigsS#mMaV&VRM4d>$;S0J>Wpwd^F~ru}m|>cxLdN5OjZM*qdpmHLC$
ziB63-XEG2IRCT@0^;CenJZt|w>+_9hr=<-y<-8Y9FlWz;3C8a|wx{ti^P@_PfDTP0
z_2Z75{yM#PMPdzG$$nIXo0f|K6A6`{Q|st});LPhyLtv*hL57B?Htz(9UDCxbk!QF
z`0Q~EIk&MMc&z3NV;=P*F|V%MtmMq!5OQN+DDQ+X87jgw7HWgQ4mABAU3w*+PsmNk
zt`E<6*qYWpM{_-pGXs%bcqFY<$ART~P1fs>&mk#s81E;?HN!@hWN09BZDene12~}Y
zyR%6};rS#U>xGw5`VH>qMG->P{;rge65{dHbm4|0wi??A+kOF8)I#ApFP1bdo}P}N
zX9n`H)#ggW9y~Riy@5A^-?zm{5b=QcUhu=0feZjmre#HaD!qr#J$Alw_ths|OVj88
z=A7w{UIa-B(HdWGI_f3rtIw5*(I8oy7y=b)91G+>Ynuus04kFBzI3}xIT#28QJ5}v
zHI9=Pu>Bz-7fq2q<$MGWD!_B<?2Qy{_54yO9(1$05x3M~QqJqJ33fi<9hB0kG!#{;
z%qg4#RyM~LsXi)sWxPSujW8{xUQT8M@w}L)XoQTX#+&j6R-Ce+M2()*X21AiIq+pB
zO;6m&6%`L%oVO)HyXGy|ZkVTa?%{4J^^`~2uW<qH@BiLE@R6cGqWGQiB%r>$8pAqp
z0WbD_gW7>8AlsuK5Mam@L>qn)_mb})I}O|Fw<7F_4S|uj2`K1p`u&9tENnJ?>`3l3
zc!lq>c4`XT=RV*$BPAj|GqnmP%qz&l;%18Gc`R`2Kw<^+O~1{yL<r>S#8O5R*&1;S
z1LIUTlUJu=GaI_;jwg60VRTxPjL%9}%g9h&5_$&%<2Y*`<iL=L=ke?}%fD4gfW?Hr
z2V6UXH5Q$oSG`L|>cQOd6<plyoR90}%_Us{qtY7@=>3GB4H#)PKwM<Rg<l8*VgB+3
z@H}~Ak)<6)6qt42NSiRld)>0~rA%kNogR{{xktXYIxl`#i#B~A?njC)34^%rkTKh?
z^t}#}gUu~LUoZ2!EWj8i+c@2f-&%%?FEn`cMsc4xl;~#1Qu`G#tXrj9Jt<PIy-`d1
zLV7lPL7Mfe?QSd6R1&aTS;vpR;r^kgwz@4}uSZIt>?3^BwY!(ruZ)HEAJ!O~u*SaQ
zLW@mu%JIP8nN~!CzJBMl`pysJTs}Ssoq9Ssc#;~?U?5Ue8fKn%<lWx@@o=TA$I0^?
z;z4M1Cj=s+aGfgEPApO{Wng&pl51OR$A}7YR+5mCF6d?W4N8Wo<sPtPxONmDOb~t@
zQp=Ls`}Ng|6d+s`@nU)R=cMUZNb&S5si^cf7z9?Iy=mTTXSaF@C*Ef=*tQ#-?JC3z
zdhq}BfUQ19QVNiyfuK02hRDHomp!Pgm)(<8`fFA4KdF8*u_xAf=Wo=<0^*R-N5#9U
zwM3^<r+@Bpd58=k1FrAr-&+4})Y)(F6W5+>r9nrzQ^n@o2>qm&t)3KUmMLz%^(Z_1
zDw=W|WT);}C)FxnXb`EGYro$!^4jy3bvsh)-!gQ-VErh71!`p8NH5?tA&*{|bs$3S
zYf+^UysE7>g#OfDtRAb(1oRj8uu1hQD0z6%z^r{nEL`cgu%LD<(J+lo{gmBLZ+T_=
zn~_^^`oeaebcPC9X_Pbq;)`8W<5k%Ed>M_hS5@R^ZzooX&?5ReC2uIu{I0(|?W{DT
zJm}gN#gOwzUTSbGZ?>nl(pXQD848mAV{#c#3DQkd!UwCqcGCL?J6QXx<JImF%^=Uu
zpjuLr#cO|tkj7iJLKIZ(@31|+&qc$-^OWEvzk?l58lU~k+%@uu3}AfQBLZ0KG;!Zb
zQ^sWPvPH|!>lXR51!k?B!ZFLlYJCB~8x67dP_Eox^ZwG+t$l2hBo~8;h@-mzQuj%#
zbQM?2FQ}2c<g~pzIwrgVtE5j?)vvGEbky8nDEfhm+pfo5!{<pqj;o?xCSBdpgr$7|
z=1a%Y(D*C|`gtfTO4PxZf{(10KW|q-WIZSy)IHjMeE{jwe6~Q=LFYnG6`+`iQlO*K
z9UdM|6{#^Ys1?Oe8fmGQeV~srce@xq)<sDqzbKG^z0#^KfbcFwbC`a&at9B^P)nPw
zcZWFLI8k)aza(nd!&hQ}oE4=dBBg^lbXw(LNdi@bzcltBOR?tqp!fs^wd~jC69Mg$
z<EWQrz(&EyPi#{Xfc=gnFaI3CR}FG`3iO@~yREdUKBhP^=z{YL_D%y2n#5qdcB`+Q
z55Y4@ub4n&s2C<Wzl%mb0J)`&j98Ej!b-~;O0*5x*h_z-y+6ZM==2j@Wh({y-Fu$O
z6n$>q=6aO1dl~EXzV+kqt~?a}{Oa3Ru|V5LR{?>|7#-q*%3B95oKw!zu1D3mE`hCC
z4xb-$^sUEXn4kED9K|c5{}Z%+D=R@(KoV?z($6YaFxESR7$D!Hu!-7nA?LNItO&ML
z<DFQ70bWx?ze$|DNWs3Dfx;ki5_I7+%8Cs!%<UbGxi$1P5;Jx>tEHWq#|dZBKpm--
zg&jj-56CyvLqvCtaGPaynd&+}S|Np@j3ZUnSJ-ip8KL<i`U<xPkl%o@*Vp_pkObH5
z?QM2(g2K*I#m}Wtv2noamH@Yd>OZ(0tylZ=|ELAD|0z&WK^4m+X~Cf0h}R{)Gn$&|
z?@>o=^q&7OW=BM{F^0*+sp&Sh2aqLH{LT`}erE~o???LcRVf?3T6$nlUKni1AIW85
zF%L9p2$=&oPi}vTf2ebf5Rqd7N|piv0ZNpo^$QgM-!@)Y>>Hi<njuxFQojTx^b2yo
z8_C(PGjmS#Mez%4d3t!pm&an|rGEv)zCAKFR}J!UhpP-!C-(vVTWeH4G&dm5^u`N$
zxvK!Z4Gf@*fD1Kf<LU=r^!6*jWYYM#fL+Srj|o}cRvPuyu8>{d-cFuu4VhmY%%9yg
zocn(BS0Q4gdfZ_YUl4A09dF%(`U6;nFq9PeK;33k7Wb4d(M#a3c4-U0#0IvK0S%A=
zx597U^ShnI(zER0G8aG)4N+c7K4t~<s%+=`5~In+t30n20D=CmxP-uk#!$CqrSeS<
zr4}@g*V_xb(?{ZdOI+((R5@IFp0^Obn)L&_`+XUSMo5=X;x~5CxAyR@(^2%ej1_0H
zR+U@etXEAhG<T|5*}5zL(VVSlGK)_R-pJV;d*nb#w>fJqZnj7>+qx2DqfVOlubC_$
zZl3*NgT|Hk&4@c~$sH;0oB52$eB=gwo-`wX4v75Uw6kY3o{#w!Ac~Tc>r+HJhLp*J
ztzc__vQehA4Gd?j3i~W615mapF@+Y-1eVL-yr|3|GP3IRju{hGeYcA28@W+I`Qd7k
z2TjB&kCZuJc97YL!lIqL0RaicZES}C4&FWoB}^F%^LxfdQ!1u|hsyP^Q}n3VLOx?h
z{W-9Q9o530LklS|+im?PzwX5TQw>TYB<^Jn-d0g)Q1Of&B8wha5x_7(=OdzJFJ)Ew
zuTo~q$AGnEVrh&F=vn}4pd}cPrB%+y+q&x3@<LXlQFq$4d<^Z;^N2Pr+MF;^gaz*>
z_!LF{vh8NRtLAKR*G}tv_bC&f0N66o*}q)noBC^jxiZ7{=99k34hsWcr`n0!H2eK4
zn{^3}>%&Tw-{kYJ_-a>M(xFGz_iY=GJG$UPNwm=75&@%l(Oiv1YqsS!$n;c8@0IA&
zA!Jm+kPEm_+8y^Vj&l`8NWiDn4p~7#!9oC2RBiQi6AQ%lKnyXMEz?=2gb6zxuS<G-
z>*A%t1>2;}i+eoW4;VyTgx1?{gi@34DFHFDe`sjv2h)SY!)k~BYvaME{LXvv{`X#g
zSndy7FSFbA=U9v907u7t^oPwZv?L}>7wlmXzEYF(i0nJAY6x+T)V*{V0iRH&Vtl_!
zbPW<zNVRw-Aw9%GhRGr_l2m#;hI(iI<0OkFhlDD;LpnhMdIu;`-`chqU}O08-CZNo
zdwcF*wpJSLgvp5!N8DMM&XlyS7vu&g8%x1tJ~QEtq|Wz&%oon+8olq!*s;t_=c}^(
zHlMv|j3RGKG8xe#o*1LJ2NE0K?uV%idHW?8kFNx2VqMM<Hp@V3u^u3MHs1CGIqJfK
zuZ>>N6VBFc9!saF0IXN2J6@?akAvKkRF41j{l`fDXrcc(b{N{SHbR6CZs$QjNBD>d
za(t3$WKboEnYAj07u<y^iW%U%wC*q7EjAWJ-rFA`<OEZ@=OqyGJWg&meJ4w8_zIOh
z^|X}X;}btI>Sng?lkKc7F+k+h=m*n2`=>OX??zd%@PnYr<iuYTK5Pj69gdl!e294E
zQKAG4{+oybo|F}*?y!N!zyu4xRz;BCeD+s1i~Zkbv*ZFD)S_an3@U|RMGx!ke{uYp
zYo?u|Y2WB}&S$78*(pHYPt%^6s9_|0?y}4P0_ZS&p|1dzy^+K)8N7*3RzgkclGDXN
zim8Cp8b#9e8_*5f;J331D#imMO~B9Q!llfz-YMeS$Xo%s(dS0*1&+jE1>9xAJ+kp4
z3Jun~8BSggN<lyIbLHi?&|NvtI&i_iAqONqo<#WNec9N;p?LUQCO;P)=GUa-5|q$4
z+LvR8>?)*7YIg=7E;m>=q3~FL@}~xpuy=6)I;IyZI2`%XS1*m<c~_FHJL@;~eNgkW
z!}N{1&3;FOW3{yyopab%FOqMuRDlVrIs8j49`8wvD<#M@@t0bs>njeW0hH_<BY$^x
zW}3A2{ZWdyD$p`3gr1+!IF~1nbnNInr670n@Caf+Wf)84bEH+Rq<o^xD7Dn;l_7MU
zt0*H*dIyLFK36MvC$d*(Eg;t7E^h}@dd9%FaR*4}f@u4Wq)7$1bG2$hhuFS_61+*o
z%Hy>!k@(c&p{P7t)>f?2-zvB2^Si7Y>+tujFS<a-CR15%6eMvFVgI9gYnNLG65#9t
zoTnPNCLMkm1Cx_Cf)21%=x<&=7N4ir<Ya-;3Xwa|8x1d3f09qcUP4}Dn5I!)%;9zt
zwQfdXxkW@m2=t{^M<2a+EQB8cx&f6RZ3i<3Z(dwGj`C_by96c+b>ElBR*kox%*+m@
z(bW}>tdEM5VDwQ*hK^mXKyC3IuV4myi*@A@3fTWSyh!sGS~sHj0dal3h#2x6TcQ1p
zz2-`^R>r*%rXU2}tV7y|6u#?7q6H}6=^J+?ZmgoOK|B$1xBZ_OZT5IpbS3Vro>yQQ
zU7e<86T6E}i~&lr1a1(eIRt_rc3VJuQ$Jk(w~}la7$Iz94O<NT;GpXil(2>l?G|HW
z-I2n)VG8aca;yLb&Bd|qdgHW!+530Fn}4I%xd1u#zo6H<MzjzJgf_D(!7?le8?vbA
zv@`Q$p~^du!ZiTyT4dGgM*jwD9~YlhuQfC86#j@yGvG)~)%y3$vKCN|>$4|FolyY{
zuC2!OA1#WOO;56SqSE`>Wr(-XoH-fZikM5m`nBrtj2|*;@IaB2^HUEz`Q4ABUsUlF
zNcF+BEmUO1Xo$c7LF{wg5%sM8L*C$S>t=el)=z!MhjFL<IdkH~--$TRdkVzM!#CMt
zJ`c#m<Yc)F(P0zs8DzD+$%YtGe#6T&9?qG$^PB^oCP_U`QH%qmhqB1U+$hp8nfM<;
zgIEVtj;(H|pFVrd$E6E`;N|JxcuJ%+_t#FzkKeS_3-7z%iBSYsbO%3=8+>wTp=$o9
zC>g`>2YQG9te+eiZ)=`mPw2XDF<mh=DiAdW!&T2t*BnNG&q;g9jgqhMT<FgzF-kN)
zML{LmXLmdK$Pa}#f4N1U1(0WSZlS{W+WfUfX}+SJ9}5c>bJaEphP39_R(p+Kdx=PQ
z!P%XFv<F$^@_;YB))xC^RaSCOJiV}TiN+OgoLSeW`>D2vr}I0kE8e%lMM^nIB<l}v
z^A&O=pM7$)`mDzERpFPYY^=tB!{w1M!82Q{M<fQKFe2gBVqOl0gw)F`dtr+eAz1L!
zSNK3{DFnZ1=riF?`0BU<37u-WF1<@m8(d4`0Zx3?Xe>S$9|E**X8a1+A`h1;@1F;5
z7}{B34jnG{{R|?9J3wqvt!~e*!)vYhUPjt3bQ~%yy3#<N9<#`;5I%@*3%l*0+}G1a
zzx(L~2yG;yE-3Y2Cgd4^uY{U!uR-57rxw5N=;GsE$o^Mu)P5`eySWQ{LoL+Y!TB(2
z%L?A<JN1qh8WY1d@cU$eERE=g4Z`gc8p|4ONUA^-bOM`@cN>w-Q6%|$W$v738K*RI
zZ*vs1{+?3r&cgk{%bKb^L(R$^!j9s?MAF%QB31F|2f`gh1!hbhY1}m5n6;sCW#k<`
zPso4vYxXfY=sirMlo6tV7*e8l{imQ2DWFjCUm-iMVU(z<1PdiUiIC@b?sgAkE$)GO
zgsDR@_f5Ose-Q&LfN5(gvQ-W8iZlVQRIFbZa7zElk4j5`-0LP5HC_j`v5+3ajTbSp
z8;PXDDNNS=`rtXuXxa#Gl|Sil|Kn^DOtl9igq}$9D-rm@kFIQs9(H%g`6r5xFR4^#
zWdL!a<5*2SXbO6GmbSE_z1Cw!KlFa`gdTgW@$O^_ir1-nO$mpoLW+t<R~S`ta&l(J
zv`19e{qpg5ed0aW^26kh+b}w)9Y;<!7jtVX+oWf0V*v7{)8!eesEDjIFg^9Mj(6sP
z=in(x2eL$XG8CJuD6iT3I<)MNPa?{8iS~?=2a7Du(N3W{xR~9(!ipV@Hve-8%>zav
zeOUoVY^Bz7|JTU37kYh@HG;m-Wvn!Yqqx<q?{UC-EQ^_v=f(rElAkxw5&dWqY@dg2
zk!1u`$?G*|jcJmj=<!uYl7%w7DEc7SD2u~#_e5=robW|?+%sg$=k7Hd+8e|t-*rTb
z?p&C^ZWQ+wo1}O<)oG|^4Kan1b=`~)0=VhV?LyMC%^%KU<PNt7x+iT86T#xBQNRxG
z$<9EG9$dn(-E&KLQor4!Sp}f8)!M_ToXpf<HG`$Yk%6h~y0bvTUvE$taX-~s>U1Hp
zIvSN#tnmC2J@Trp(e>Q35pst>qp&63W5Jb|RJk!oQgLpn5kR)mW*Ns>ge~?4_DUB_
zXFQTTgO-j*;}8oFpbMK^eqojBwXO6<J|;%TLc!acdQ4{5Ehw@TX4H50O0B3{waMkt
zMiTDai`To7aCnrJ!#Yu{WEfh^;^O0XFzp5h!ZcocC9*)I6ZH4XBC8b!oTxDd`SOv#
z_+z!Tb{PD!%|S{RhXokG7YV}7$otU1A%iD3oOBmcI6d;b16kOkg3k4lSE|)<rwp$s
z8#t-I0LvXkTU!YLYId=w7xd;QLZic0?b=Y<U!{+m3`JduUYlac&s5&;194TRiqbuy
z*K*<cih7c9w9K5I<A?@%R`Z_R{I*BAszs_cg<VQ~012h4Qm@5*2vKjEvXOVjR&M$E
zCfTIcug$PP=6>PT^gTQ)az2yb99_fS3U!-D@>l@1gTUn1f2B~je{Olj(%iAVbALDV
zGW~u78EKn16HuPky2?g1MPxojnbJ60!}d+HUh3@i`i?66VNVN!sW%)%;Nr~m=@S+=
zX#E2omJ|JMDJQFjH6ON9$$P@U70mxdbs9Y|T{IxwNWV|O`F~QKmcCjqH=X^P{f6*c
z!TI+LZJ@>ZK-c6&)~k(;lI0p}L7f(-j;Yi?OY~R+?IuqlK~J13glV?jNQri!#mf!U
z5fzzCHH^xUQe0aL2W_jqiwSAyD3ny?ufO1d15mp|Vp|p#qrnG`g&UWWuyzuF0;c3!
zo=hwOa@M^Sy0`>t?ezgtwdd18Q;$ha99MWDsmhxJEb>6MxEf!X%!`)mrgN*>dv!M9
z>;$<%G8!g5%Dyhp4<g{k9-jYN``(2#GmRRqu7-7wEeW$r(oph|StD4yiB?iVpBHRU
zQ0<))8ihTw^h3wXXZ(;-rGC|!)Ya-}u!F>3NsFTg#_Y9Z%a#M&hE!^lGNWV$eSmU#
zvF4P{971eKL(vUrmlBSqzxast&9nF`T&g%ff}2y)TnG4nqh8N9^Hq7ykBD!LGyue^
zABn<}H;!IKeQc_5yU)!BG?68KuiwVQCvf-@K#8uPyrv4!l*gnO9vu%VIEq9(0Eei%
z!*qA7+)9S*;u>-gzKSdC4R(8p8D}%MQo=<Id2Oj4B6M4l{G~}HZJwR~;=8;9cmB;h
zvE2EYxQaRanD2r`G}8Vz|JC=Ptu_kHYLD@@%(D=1mm2fQUfPeN>4LA##wdu>$o@ln
z8}tU{x6CJbx6V^Q%ouSNVf6Mmj7G~24MvCt`2py4T8~veaqlkj-t8h3GV34}!MZRI
zu)-lLKGpZ$RQp4A`!qFG<y@THB@rb=jGCm6cnj4aTpkdx=;)jha>C@k`Ac(K(36NL
zfgejaJ!+>t$!jbK+fZ{X2<x;9+iTcgDAG)pG15%)Hxu|JRpnni$CQHquKoIZSe7@P
zMzph?nvy6bbAdEvcI+IO_tAO$lv(_T5P`YJl?cgEB^b6=Xx=VeR;^K@A^F-5-R2F@
zNuuwg<Lak&Qw}AdZY(*mdzet;7c%9exe@l*L8O(etXborj8%FWOu@Jve4YHPCv}6%
zD3a*NRy}++)lp=4E%}?zED1YCsAg<1h~d?jVrWZ^x4iYy@Zs>`!FSBmQ*cYdcRscA
z^S8E&v$Of14`Qv3*-vUZ4)0SAM<BF2$1x!Fd1zwAuMjG;+-LJ}qp1|~`to#=O%sJJ
zO^Lh;=OGiq47|q@O+=8+Ym1`TnOe!PctWe`&QizDj<RKkWi>S#<H|^DIjQE8$Xl0o
zOSJ{nU~0&4GXJum^VmZneSWoU*tSh+&ZQ$MoYt;eijz*!-d4w@*s|_U8PRn?s_is}
z*i-&NgW1S?{8{3omr>il78|aUxeKe<_z%Vj@A!&uXzNm|cc*?>Lrlo)#yU&Y5^^QZ
zuL@SgZLWrF>73^|*gG~h;Mh$^Xszk;H642%>2`{ErPQ+ufVv~If)`Urtx_<E>_s2d
zvbz0{jd!%=Ol@3L@<jFE_4W2S`Z7=XGV64#q7fiXjZE%{t)qTTT57#2t!tm5?T}Do
zsZu<li{G&ki_GoXw|o|0xlkhcEal<4fCXE}rRsJ!FTZ$dlsPbImEGll)h5I1`;htG
z&t04bqG<`3Pp;kR$U~8onGc)Y&%Dc~2l%D`izW?=!X7WD$eg>{x6`uPABu0!J!S-9
z?P1ToN~$MfeL<odtZi={nvT3(q+0sOn%F|n94_YXmr`uiBb2NAsY;jJc1YK$Q=Cdo
zgsm4a+ggUxY$CWmzp2%uOdgP>TYe}1{+R>~-$wZT@+aal<*gT)E){O~=Zat<?E2@N
zCPj@}avx*-v43!fIN&k*jF@ITaZS{5X(JZA*{L7cs3;Uz4<;?1npW=*TphRQP(QpN
zJ#si5;mk<ka17nQ?-n|QLbu_V@CoKJr{zVKXP+ESGVzz|+B{VjD~f6|ye{U28fDdx
z=0oVxccsFKl;md_FcHPuaoAA)czN{{iAD-GH#llpFFcnW=<GdnC7TdLh#)JTifF?-
zt_1KSS@`i5=tp<3?^co5OrrP$+!KsQt;TmKqB5nSAh&aJhC6bnV@gpTmg3eKRnnQF
zBxQZ&7fG}f&k$)=FyFSn%8L-U-rS1fr?O^~J<F38l)#);`O3P0tBi%1&v7%?b1@Oe
zdSkvGMdOJQ<x9}fMXl7EEs04Dd3yQsMMqBkmG;pmA4+`?5z;3goUxYirMEB~g3ko#
zzM-L^WA^lTU_%K4t@?<Ro1p_4CQH(9ZTnU5WKrYYd&P$GbZ>U`(Iivt&$Vxy9p$y_
zFc*VF7JS~ZZ$u3|KPN82n~)vq;SXTe-?pssN#Qdtn$&>5@5<Jq>_bk+@L<w%Ib8LA
zSfk^y*-bPc2|S=C*TZ<N-Ot?oHqdgvuILK~{5{Xw(9zPSJ^qQkLE|S8B^-@h6~ek3
z69F{C?_)gZjpVJ@&962p6ga8J3MZ*-PDf^)PTgWxiES|-X~%w?8U8Tj!epf*hJp!V
zJZiZN)T_2#>hDYwsB<hXtqm_O*xjV45$V+Th`#|T@w%UB4BD1X49cH>>?m&|`z3U<
zyWgCBS#7;I5oKvU+i;qVsOKrd@dfP0tyCpsG~DM>_w2&SqqJF(?dRoM>0v{8#=wAQ
z&jn-CDR&FoZmYnJwd>Cm=dgPFSsm8~0%nuAme!j+M%M-N{H@n$Dja{>#~)^rSq6%K
z)#0BaI&&xa6{pajV;>&Oi$18LsFR%UQ@SU+MifnD_h=WrL)TdOr8S+-LVs8n1@tOY
zf!^CJC$-KD$Boz>`3aKt6De;ZUJ9krEbsMZT3Vg*&$}_w6)GMFQQU)gP85ON_uaqF
zXq3`jPO6OYRl-&&;|F=CsXpO?Lp|^!R+T>7mAd|IxFVq4AN?RQvF?dpsm$zbDksf7
zl&IG%+ADV6c;{=SZ*1+2gm0Ki4UPZ)Ks!zn22K+6$F=y=sL%3RpI(un3ssh<oYU<i
zat&>1fB40)!16pE%j1TE$6)X(`3XmLt;O`<`O!+7+?xy`ucah*qoOIzj=$XoxGlb7
z)b9*|fOdb^dfA5w6$#h(BBzWvU4I;W<%@NPf_1hIZUo*~WmVZe9v6J-i6r3E{l1;>
z;-nZ%ALQi49efipw7XV*LHtJZKFv^rup`p9`yw7!p2;Ta93|W{liXiKkiSYP6pKO*
z7R5|OdNPrv<1K@GHkGlr>7Nv=19*!lK?icd;QQ)e1C*%sbaB9C2%-DSfB!}tq(&YN
zzaX5->Pp;#f&!4v=NBDD1G>cZ<ibSjIWBKY&<5~`%Qt3tvrpugg}r31O|b;0N0PZ)
zlGD|GJKLW=*S_Q{av7ND@P@VdZPGyswVrF+=vB*1h;4N@ZP`51$Wj#aZdd5}ty1aD
zm#4KW-X60}u1C4%yJJ~pwkxf&^BjMF(eLkj{YSJ3t~2vRZubqYMno5pcyx?Kq4`en
zMc9I@WOp=qO;)*g0e+hgzh^JK8w7n%+W4AEvSR2X@r%=I|G36bG+gWeL=HrG2C@p!
zvokv4t#SNuKkgGBA94^e(yXaJ^xR+Z4EMa)dkUT2c_NP~g;z1Psmxor3lSJWi5pFh
zXGGHK2AW^?qnUtRg|(gA{1I4y-v&ZlCVt)_9;~r22U1m44Pbc_v08b0KU?p%{iRR-
z1y}F=+0R5uolG~**~-3?virdy8aA(s)ycylI&>Q2p?}>mAIdQeP^$Z0p-pMc%;RNX
ztvn!TpA+e@(Hmh|a#%|KZzdUr`%1|VUAtOj(?^_wvK$%9_hg;0+Y{BQFbK*vNxn{a
zWmOQxhcc%mm7!{UE=np5COF?J_7_S;ubCuleHDfws$;bgR)di?BWc(KdAG^;Iw>hN
zYhZ$4vCr5FmEoqc#q%-9(@tMpPEhO%_`+(DN;nr}T}9Kw>TESGh7yJNe}>x_TyQ{e
zUPOe5q@?6-l`6o3Tx0j1t2;c$D076~7iPq9nQN!~6`lWh36w<O*t<JD4Q^I8`Tmi#
zv-@vz{OvNIq%*rgWG#2$($_{+!WxS~1zGVy+$GmtxMOSILfQqRh1?EgdI;a?Vna9m
zpX%&Qqto-4Uis_!Fgx#3&QYAm3)Mz>W#?38CVtb`Ak5;F$09%0m&nXiz^97pkMvwn
zz^9XsVr1EP;|lN+osrb;-;F|E+6x^<U2MZC{Q5+Im{LE;`BzMV1L`;bdrYxFiJHb<
zS#<v3>^56%z0VL0b2&H*X=}>56#gmI3fE2GX8iZZ{^uF>_swAAp#b6ocCFd-pHcL1
zpAU$x6toJjj*7(v3;Ri`v#@P_{5i5Y=o?|6_#o*_{un+_HAC{-u+xe0t8UFqrmQH9
ziLH8OBO#a1N|bN%&=@)ce4-og9<!m1`hqZ>P^`>g(6NyH0n7C(Dk}8skX{LFCqA6>
zKr**p`jsoCFW_?$3E1xTBQjQ^|LaKo&u2UZBI`6rciVB6U#xb(U|w2~&o~Q80o(W`
zGjaXfnea~r@y7|FUncSU$!cH4u}zSx+#{|b;j$tK`N)t@73+8+?cs=J8BoI!z<`h%
znjISN)b_Wmi1@}iLigZ$^y;2e^ph9F?t^{jb49{VCwP3rL%X)d&P)_ClnQ2cMbbKP
z>H@VSQ+=5Obk0=<4s&Z!E&<$oGKJD+T?z<Ay7c^e7&-^Id(o)(<0G|Zk2K=X$(aoS
zt=Ibeaeyn(v*OP4_s<USIgAKjrIcW@wnAZGU=T~plY^OAflGw{<&u9~&A(ln9H}oJ
z4dm@<Juv!6jS%GH?+lIMUqvmz5<CgMLaw5G79_%5nYJf03I8b|WY#C#NK)M){!Z^C
zF#W)jJNQiPJAF?9OK0ZmcW6|?n_RpT^}6mBrpi?2V`$?@UUQrhdzoqc3pfh};%52K
z`97~0X_*c82zl<kxTh=3_mu&35U{Yzfkp-w{d<(Z!o2AJ73P7YsWPSx{_k1de_mAq
z@gF(g#HeAt@Yh@a8ZRseG7J@G=4QaL9h)Me0gcmB^A2sn_~@IO)3ZkbE|0vsRd6mT
z$dKKqEWu~uWTdFc?d48}Wqx-z+e@f9jgM>j5oF!&%#}B}loq~VX7l^LCH)186WbK`
z=F(K2AHen=aFnT#Mo@5{T2pMS)Zx@ynYJD07^^iqmWi)4!|ehdlVFGx>=g@qrs;3@
zp+Ir6{fL51X=rXl{Z2^<xAC=~v=O!K{quiLroWTIzd!Mg2SqM^mAz+F3XDkGyC7_Q
zj8vfV8v7Wj8=YcHguE5ummOL`4wFOBs5)a#7+K*#g(xhj+jvVhk5yYa1cJ@56F=yt
zFc=d{Tde9$6rJ?ym_lu|yD~;UpTj!X3DIAM^hBJvu~Nt@t8Wt(hkNkWifVTqLr21E
zh)%oO=^Ph~rUekC6zAdNf95QY|C^je#`w*Xf6KoA^SS)>?v*nN#Gq6>2+Ibx+M#`0
z83_vlk-;8!3*Z>|b9_K}<OYZx;-P;)T&SQ9<n$@PI`r{WR@4O}JMm5>f?muOJb|9j
z3oeXhmB0sPWjDckMo;hrp2Uhq&#sP#+*^*d@_+F;nd^^IZa4I^*E<mvq|*5b<mg~&
zt)Qy2csd%tK0G16K7_E<wU}2%e_hIdmwx?!UJ8|8AAxsC*FCQ7I8qw|t`c=xR0BqJ
zR{1MZJJgTpiazeg=-t{dHQ(s0r_gn>0$ZzN|J>8c47mt7=<b>{3ifrZLMc7g9#ti2
znnmXoRTSv8(|8tUljONjN-&7H`cYCfdHi&_)28-C{4}jzCAlPus3N*|p<X|E*e!_6
zpUDqBI==ib8w>g`Hg@nsMn=Qp?-byF2<BUU?S2?61weSaW{-mPeS`+%AQFn;@=45O
z$~U9fqX14!=8fP&GI2i7aECgblk(k2T2PGmzEdTWEDMYB85EIpo{e@P_vqVoSn8H#
z;dd^X>#P!MmE10bFFv%LFU>Zh^;ty>1@`iSum`V{ph78EYQ!I(cn(B12a{4IjnP7d
zh)=A>4ROIDIm0~!>~b3@fRDfa?jMoejAO<D#h<N}Tx8o_$%h|)eM`5H6!M2hoQt;~
zM(hP;XcGUA9`nb6C9*+{gA1O^-V%V~bRd_BzCF-i&fuVGMYux<P+dYg;C*Ezo^pjU
zMP1#)JvENj=OZ*G9ppJIAQIHtWAHBH1^W5P{<i_%HO2zlJx4s}*f@vW?o}=vL<&75
z@zwsf#`3Sq`ab(}<4PpGTc{JEjyOFjWjA<}CPQ)}|6B=bF9`U~v350P6|)wSl<{rf
z@5`IwLh0y|p0lTx3bz9$3N6jwPDmMmo)Y;!`MlCjS~(rlvH$Hzc!Lab?~8P)e>edu
z88eogt&|5Q^!1>Pv%=8j`=in1qdWq=kM?!5QXT>mGb368!MrO`w?A6Wdr}V5SXX45
z%GGuC;iyZELuLEwXZI{Ea~w8n1Q<;NlW@BY+DOM7+Hfo(A@XxguBLh<sxkMW(w*%z
zQ(d05%SVTCYZ=`mpLtM<DD^L;rYR~P|IY`=pH;R!3YDKUtMHT(d`!C%hQKRPg5yS2
zb~)mU?}J6+`vvZ!=hJ-yL*0~iJ5wdR6xSs|&3NM7&6D|7zz@2|*FPx?uTV=HMT?0T
ztoh1Y5H{9;M(gne%?}MZOE6vEN?dpN#R=$Q&iw6M*tM9>K?ay&Eb`rFXtpy2QdslC
zaTHn5ff_-Fkq<5y%x}z88?5w#t{f1gu9?vqO0(~MZZn+3ehAQ&UEA(2w9VgEljUew
z7S?<?{W`LI_GNN;QLj}8TXD;@(X3c<IaKF8Ci2625bN@s9+atA_VrW)F#WsXjoDbc
zzXxwwk(ABf5ppWKbsv5&%a=B9n&Y)JV`&vvrZC9reUjaDng<8|Ovyjw=8={mVJ=&z
z0>8SATHaFp@VStFs<h^8;ZQ~q$#2tlIF)OX6ryt!T_^lWgd64H$17qEb=&{LQ}aSa
zBw#?IX9oq=&>~^`I70)tpujp+oFkE6bJ2wP6t($SO7U1HRZ%(4tKt{C!QVs%DLqa=
z^xPmg@5IFmbbh{&)ScW^(MNu$$O+_m1srop5uN(xgmxA=3g@{~Z>FRepjWuO(R{p0
zbD84AR9?6eX894~?Zr=Y7BuM%!U%w{>HEsGQZjsk*CUV?;&k+5SqHKfSSA@pV6)jD
zb67p8;D}GBz!WhsWB;^{%sIufR-cn+ANuXXktKFPPU6Yy{f0r4!^{z^$NtdG(x7Gh
zBOxVqeDeZFK$xKSmS~vg$n$VPyv(^YrB0T=G-S1MKrW?T+?_Oe7mv%Rx8!-5bRAh_
z0ZFWfUQ1sKYz>2f*8DQ^O#`Q^Fy}apa=#gQ<;G&=N@MNFVk3=!YCz>jY6uSV=G?DH
z#~~M8?qk_r5SiL)g&4u_JC4DoZvB7UahfpTj?YbCI|K+*Kq_j<Pw_q3<7L;7#%L!{
zngfzlMbnYVbJv-#tX4^ac@Q3ank8J;8H0-{NbjQap(h~Yt-`(_374xO?T@MDlrAJl
zKHt^yLzL88M%58zUSj#uYM=E}SDAM1V(<<IEB_4X7R`v}Dd<BJJUI)*a?9Ht6vhQN
zy88u5!eAot-{z>K%u`#$@WDyXUPfWC)VJ4I%}+cb7XnM1B)v9QHh)`>m1FiQR)qA^
z#4at+gU4*Z1#@gn;MBBuknM8oIiE6=SyDlkAl>q1(_*E5aGEHcgHPc^cqgY<i=iz%
z7cqkBK_Ul8202H_ra{ec5!!}dyNhuJ#5uiD4yHd`rD}fZH&*YjCv$5SXg$SkWkt7g
z_W`!Qhp)y+bq{{Qsqn~oTcDBlcfRtU<-$Mj;yI9TJ8}}qcg&z~Pti9YqmcOo6Rk5v
z7m<=4gZjHLL<}9!oQo2%2j^r*P4Uyk52D<i3dC1QR*@BuocRmHC*XPIaK;*kt70t2
zccowP7Gq7{VZJoU*3WhS*{@)1MZ{b9az=@?P=<t^J-~f_f@Q2AjvSZYP-p&?^&xT7
z#qPLKJD}k-&PlI=VuhRYTH%8@4me2@sQ-dD#{p{X<^AOfdTbxC4PEy|+;_*PQZU|E
z6ehS}6R2JIDm!u5r(tFwl$IT(dlfs!M68PDlD3xjmqYLNZkAl$axFGRHVNRcaY$~?
zyF^$o>k|JWwer6CMd!LFm><wjJe~Tnu>K^o_`~hYFLB0z$o-5aA*;!WYFPjdib3Tp
zE(`TpCRXd|lKXd5;lDYy-`#+6L1ff-!TGnyy1p_*FdT_!nf!J9cTu{YSB4ci3Lg+-
zxC-K;gKR~#OTrZAW2B{Bs$#H=Yp$)RzU;;JCUa1UOy$OvMu-+0<?Z4L`d49{y+&Nz
zg1P*y1rSVt;M471jlYumv_mShzs_o2E)ZW#(+}%r7`;u#Dj!hvl6<PD<KFzash$pk
z31qMB`Bo2qSZL!XDhOMX&2l4ZETE}e`u}Kq%djZFwQrahm_eiwl$fC;1SFL11_22r
zlm?OR?uG%ROS%O?8tLwmPU$Xzp&Q-{|9juh-p_j+@BQih>ZRkvTI)Q|U#<D-MbF(q
z{$n{c^Jp+;%kdCbbMouw=Wr@42?Rx7%fS6F`*%ii7UMW#q|FnZm5F0>mCtbL&El0q
zmXzLX0=G(Wcv-UlJZdzX%t=gs%~5>-o&B!qwS`se^3FMR@Ronus2Vu9#RQ3cyJ0|W
zlbW-49-e~1iK=1qEQDiw;ujRIVBQ{+^OPO<QdER6!g%6G9O=b~@cFo#Lg2yE+qhS)
z)akntg6iv8vNVKM4T;=_`akJk)K1F=@mT-5KW3S4D#+ca?7fk1tL@*sm-tPcUO4$n
zqkfV0KHjoVy~-{7OR0M9|F-IZG}=}_+~4iDY@M(lr|lTSNkU}ER4#}NqB|r83o-<<
z_#k3NSBmcjNqMG?QaFPB6n-V5222*iLasT}`8X$Q$?xYtGa0hl;H#vdp$%I5k7})|
zeyu_1Ij%cjztYM*OI3euvXjG@o^$lQH#?8t6VKhc!*B=BJwqWR!|!EQ)3%?1h}U~(
z-)>Av1fY;9`}<op1kyOOwij#F*LmGvFBZ!ubM7ZwHQ5-Y*aRl?Cl*zcNU3McUc&h5
z(-^HbC?Er`6xMUsy}a&Nm@aN#AJUx<CRo)I3%RiEdhY=CjZeAqb5&-R#5W6v`a)dh
zC#+K9z8<%nc(RULX<q`QQT&>re=Oba0yFyC&T8FKH;KMlAvZU6<n8Y#91L<-g{jU>
zTQpCeW$wizbJbgU-q2=4F~8IoR~KqH&75l1tL{bbQQuwdvYcO6F3y)KrPpr?xeI1{
zuwg(u`JGUrhX#sw_cv6F)F;ZM`R^N!`38ya6O&#l%7C2DZjI(pON1z5YIy}RjI64a
zr{Yl1jpp3ylG)4xHM}md=7hK|^2}zlEb@cY5AV1RYWcJqvb)gEv#WW!ygpB1ah7=9
z)pfgxR8NkY1=(`?g2QgF=e94fDY=@hW84g;NhycY$){#|_nH>=GjEowVRek@byOxA
zMu`rZ=@t=K>xVI2^{dz936<+f+mlVnq->V{&50v>UoicnZK<5C!=~M`Jv1XipuMUb
z1L<&@=C^kfw->t+`-$gi*i)~Tm3z!4%Tp6hr+RM5W|xE?MGL=4W;OA>UQU3`X|c>d
zlZKgU6d&8}FKGw)Q$+<X-0sW^qyM%VyPY?EQW#*yJXIgHDh0f!hBo;N4_~>$T=${-
z+GTCa)Z8f<K`Hmm?eoA*OKSu7BqE58dsSq;P7|X!=KB#n8K>dT;Mx%ECz;Tjz_7cy
zdCsjWTrJx7pDzr~C7%hGz4+f+LldAi>{j*6XuoYu{uRjpfMQQ6qtS4NdWSsW+KFoQ
zzX8k_;Zwq`kZS&(C7C-8${UC2ALsdAm7Eh8IEINWwpulMlINB!qOp57sUM4{zCywx
zf~}41ZJ9={xs+q;uHnf)Cco4d-I9^=9Fd4s9-=&wOp(G7NpSRm>)iB%4S^s$+zrs*
zo+T7Z9-$krRc8<IepME;btTPryy8d3kE6Rq<%w+7VD9qx$P}b2)Sl2~^)K<IEj8PH
z0xFL~2O4C;J3>2yeTVotE@YY!T(rfHLUU$+l*#A&HH-{O!KMvv1q!zEg>#Uo&?%Uc
zs;s$SF0wsdd}kU{=ROr;g11Iq&CazF5zJzzDhgWbf)@{g^Si^}{P-NE#-e+`6NGog
z&e#3bZ7a_^JvbMpero0$8h>z)>Ld@HglMR`->yzk!tV7J2AU(4|JZeFsw0FxCktZW
z(aqjG6s5%D^CpT9!-9Xh#K<+$jF;W2R1L9}$9@iJ#uQ_(tJD?kyj{3liwwfGJz<u*
zw6l?W3yMJH6lgi!DU(<sKEm3Bw6)3b-7$XnBeZ{JFwf$36TP)p+nScf`csU)pVs7u
z)VCBWXM2mO39XugX0zJ?A&v`%^K>rQ8%2h34#ov4I-4}gc8(~sZ2w(~DY3OoF6hbB
z3T(b?mOMFfCP(T~ZN%>F7Do2!PGk^$+bKJ3e8-ze@tXMNJsF>}KUE8smU#}2*ecc7
z4HLI37&Li*Y)vJ}W_Y(c?&v<vUF8|fzO~$Fc;%apK~Nr}xYES@zxBz1!ew{>1>1O|
zE_vyPd-UV5Vk4r3+S|^%bYFo6$KUby3L&;n52fk5daF0wkH#|@Un09~>k%hxze@oV
z-E8UbR_f)A8^s9b_?0xEt=7TO+k`oIZj2+7<ktJv|F~i)gD|hAo|eW5o)vTpT}_X^
zeM5Tns8F<iXLuP&Zo<G%0VJ@)q1sm&yu{W*<?Q^rPaVKwftiF20v)hF>?Fpai3lLp
zp7uDMQ1e-1Gkk=XMDiAU@d#4A{D?FVTYda`PO9K3uWdf*LA9&#RC(&|f(b*le!`P>
z(iw43<e##;G|M1Br0lAqrtt=ZmG!uzO#?yY{P-DLM^QsZUuL32+mNKkM%Nwhq(#%t
zMrJW31j<yW?KO?ntN+|8(~Dz6RsnQ5EPrm*&e9}JT<v;3p#?M~x4!G<zTH~g^Qks^
zuhTSAjQYc}Ed|+&3!Iv!H!U02C+(yxJnY$V1clnWvCcIdznnzz|Gt7S$p#j2D*~JY
zN=zsQG!p4VV5Q>xdxT`WBk4S!qr&5tUMuY9hvzUlLw51{CA&fhg9mJDm-x8ZrrW|v
z^DPL6Xp?2(9v_%*yE|Am--<tfVRf!VsQv8f&FEX&Gq2@#Vt>dJ#*1B4yR9|ig_(ub
zim3E}Eo%4lOSv_wb%6E81z;3^<`ix&F)8dr4Vhm@G-Zz1{PFvhm2Ub7g8g2nbm?z8
z5WrfkHM_a=mDwfm@%^~RVLLHpv^3aDFgRzdQ>U7uy%QAnb?LCR8dk$mR}%Fl%#1Nz
zNMbZ&`8vJH$!Ol|bCwu7``mhCiL~@HTsCb)RM@wHuPdA%OvcviLRorb#q**%KoJQ+
z4S!C#WNuhJOEo8A;t54|aw%CZA)L3LpvqUx&dsuO&HF+Ou>f+02L1oQ*8c%6Fdl#l
zXGWJ;A<uKL+4YaxT|Hj2yA>4RRv&u1VOo!LEK%|*1ZoFyB#ZLl;|}H)Iu~H-5I76N
z5qI2@S%Z1(aKzW7RW;D;Q?Rs4Ct{c=Dmmm6uqM&(jmD8%$oaaidvGF$u#I9fpek~>
z<X(tKbHJ1@sXJ9~ugr(DS;QIG4lP3C0qFCd%~#4KrMU}e>t+X2`M}ZO&AU*RCTpWq
zhxlaPIC^<3YA&cvjM-1c!lfJE`xF?_qgSULM6`$Tj~Z3?)~jF7Fe@1zDvQ4Mk+VI<
z<ht%0jwpED3B*5?8uzMKO0)=cFOm}7lK3v!Tv_G?p?$de?B;!6zpIlr$F40VGb-Wn
zDGKdImGigjQh{HIqxi6}^pi#qY|Kbr;ZN-4h!g^j!Cp|J@NvK{fwt?0bs6k*=QU_|
zSZgF3NL^Jx^Ydw|!C7KB2DHG?C)tASn_LNvJ^5}hiP?SQL|;Y(xUT*to~Z05Jz}m^
zpPPLvMnsPf`-lxW`y2De;Q%t*4+s4*7JB(HIn@?Q0#S_d>h{5k{AX2ymW%RdvRw@1
zx#AT*pKw`@3Yk;mF<Vjf2U5FL(=dA&gBI=8DQI&x#dth^F#(RgySI2xw$|$JSl&%d
zTuCy}I`_cs?6wUOXr^u}2e;2th^ow^1al~;h$zLuJm;-;1?H6E;?Jo1DHNK|V<QOJ
z`Fv&M5RgZALpcmFb7*PwIlHF(&1KDD4);4cGHa1{xnRu=A`NFYr@8s=z!%ickZ1oJ
z6q_-QCymE1d2zUdWrg=zdKn3Bh9u?9|NVEYS|c|qm06@R9VJURYE|s2AE$fhR0gNc
zldG`3I4M{wOtjA#k;$3<OMZ=pYj5V`v<V`KqG-*;p!6^!KQCM)`i<1-5wQZk*0vH$
zbf7sXsZaa)magA$vovM-bBUbn!${6^sRR`@Er%8bp<DlOnM5ie;CqzYCQJsQ+XdJe
zY>Bn(qs*p?Q%O0^)hS%$k|lG#we@oZY3>r4)Jj{YEGSc?FIWofa88-GeqLrqEKETo
zD@*{hGU@WY(`NJBYnXu8s3It0omgifwd^`ElawkjWv}}6#a-UPxW%B!kP4?A*uqh9
z_#X^nMB|!=@4KHq+tB>TEZJW5`UjRTB%%Wtx8djaxR>Xg8Y`V20gX}Db9sa}q}3>G
zix{8EIi-HeB~#iTlI1LYSN2D!t%^IA?(N;L(?fC0nGG?7@d}3Q$w;miaV<8(VnAVf
z6Yr{{B-{6`>;pmDLgw0!rIc4~i1eU(v<(dkab&z@lerF?{qcH*?{RBNFF`ocY}K$a
zmeyT<*u+*Lo6hs~B=!Ib=N~`EKknD;Mc~W})AAi{P#0)2n(YY5FZ4M6sz0<=xH40f
zxeCp$VXCq=a@s~t2z*IVUHV*6ZYwL(BD8A1POJon7x!d(MJk%s8FziPgYRvAIT~(V
zNH}OFn4Ti~!p;hWld1pxf1716fmz0YQD|fN?&}|%$;+8XXyHg}5M?4vb!*C8tsIQ)
z4$du`4C*GIA#{xsO7|XsOoBA=lxBL(jD|x+`3%FT2hDzLnLNH2R{uUEBW~X#!=v05
zF6KvF;=`(lrTV?((N*j-8gWc>o~O>{3Lhvk6rdh0;tHRzE@{_=hda~6n!dM(i=7_f
z?Gm)2`gT(Q4Ia(EOKceibf69h|Mgib+sX>91A?CF-YE$D6lswrU=lo^&EmhAq~6~p
zBWC8v7scgQxBk=`0fCm4J{OHaOR8a!@>(u-*+?3G5APgx-WOM-FqPKIQ3pjh-+ZZO
z8`8QLbh?7m{$bjfYldAqZSfIi2}vU0E%0)-RzJw2?sLAE;_UGE1x&FxqyhtF<d34m
zQ*65$^Xl=0BBbT)o;z2XEYO-iY4_#NDta$Ikg=bpwr_(f=;~umRZZqnXtqFzy=|^H
zn>&lXkQ6C5+LV61#N%{tcK(aACz6ra7cACn+#E`v{CMhznvmN>OdWLS=_Os7DP{48
zlyttU69p=C5r4n0T!?JspZEf!05>b7M;cp6Rv`}qC!)A4q}N~H2|8U{)p@~DL~j+F
zZwEUgwB)Li;2Et(*BQIma=*tw5uva~dUJ3*{Y~nK5v9eV(@>M;c4s`sN5E&6{rNxS
zCIFe^(<Ph%IO@WKj*f?hvegOxnFM@lpZEYNK(*UrdG&#=ly;V#6B`zg0W63(2*+Xs
zr4`l3s)e`N8HtU-kJA|$vKPCI-_vCzCg6a}fqP_)SsKHVUZk%;%6lZiCo0giOd0tm
z$n{Dskx;4egwgEyBGj)=y3U|aj#fZ=C-NzwJ?_q@Zhvm|OME0j(%&+8VJZv`H@w0y
z9dGZYpa$g=l}TrJmHg}<b!Z+<ZV^y{G#H;F(;ls12!Iv8S?ze@x70dtzTkFX4M4>0
zd*%Hvu~53&n28({V9o)n7I*5j@~Lau<X5KZOVm2aguY9MIT7xMr4imyp@jNnCj$b5
zw*3y;FYV8#XCvym7hISUiVL?1Lzk3(*mYAeh897r5HOpu<q6izHVE{Y9!5vV!z6{b
zf({C={B-T!9Xgi?C#c2epVo(41mR{`rQEiB*dG$Qw<rE%mudDp$AmqFPp59IajV)p
zsx((NuPXJuJ<xnPJ)m*Qs^7BOqdn6$gQ!4+aC(klqSN3vHcJkTX3EU2vn2CV*4<WK
zqSKr&ppP}am8K5%i|JrTeOci=ni(|t$U>{Ec{;8#LG}Glb?=>J|3uAwIy0_3#dkc0
zAUI1R4G_nFu@|)8msi}AvvD?K468mRnf0&Y3Tj-obY1=y8Bvn(9!^L|k}bq0W87SL
zcFHvtSKY_fin@Xe%95kP$HNm`F;)y&^A7_cNJ=Az|HiYhf`vngm{Dr8DX&=<|Mh4+
z0Es3-=&q+h#l_`Uhol(*%!UO3;Oo}<HQ722&<MdFfsxqJsftAgsBvdd(!5b*bk#1A
zsz%W@CKv+vyqQIzi?N6DX}tCZ0yA0JVg|lLxG|FMY!J?)OD7de3`qw(tj8MYL5khN
zNEn$vdJEFn_mXP*nhKIU{;2KY3WJaY$H6X_z<t&V6xRj*Ya)DgkUbKGSLt+rg%!xJ
z*BJqNEzC~sv2Qo$todp<|4~mV{9-FRq7KL(=6d?Hh{$vud&|XJ!)@E4fcfL-wru*{
z)com@)(7H~VFpm^YS*TfwUx&q(Y|S%C{uiSoHy0ePobhpyTT2kWf16>qNkgdF>#zW
zK_|Sgg^zC)7V9hag<h?Vc|G8(Ipt(h2wluaRaw%Qv*d@VqW)|&Oj%sg?98Rzp}Si$
zMmt}=W8fKeJ2=|bmsGo1i(NA~rT*!<Q<Xt~e%1B-S|>?-+nd{Zph&7d_#G|!?U)na
ze9^FBXXK`WPIJ*nwL!CdpUuVwwG~EA;EAZmk8F1yJpQM4)o1f_KFQ>3({JdQ2LQ0=
zs$MOyNjdjB?i3S`rqhfAIi9=yt+D{a?3w1}c-+LGtQ^95D-1k8b|65W{QQ#J7tFgw
zg^SErwf%K#J(_1Ho;qghH6Wi=g&nSQz%FTGw(Q!Cj=jy!a9$jq`&<zm2vCkId}XDK
z!7P0BZ&ky;i2g?*NKt%ATqR#VNuPwfY))iP`#Ks7{_9~M2w6LtDg+Fe{yv4SDba5J
z3N(SlqWn{*((vxSZbT${MT9wcawD%`E6S4_?kv+qwn;4o#&RB&Q%fvi4<NvG<zhHd
z0`9)kFpJnjS+C0MLP2}fAH02Ag$n6K^-Gn-R20&O*&=Mn=u#hIo=0FC(qbqUybFP~
zL<?Nu@){?EUmdChL$9W(P8b)bBNKFG69wN#kh@+isYw~lW_4yOZUAz+QEs0wc?7(d
z99Zf#yHh1P*+{J@YlHXGGZue{5y>xEubDl0=|aG>d59`6S_T#0ZaodMea??I*{o|P
zvxr0A^&(WOL(K3J!=#Tz4xKd4#@1mj`;J;pYWF<fztA7=s~yzrrZv88&yTijyqdI3
z4?D+l5GQh8It3S04iyE6;_)A~vV?y@mJ!Eu+rKy<_i1@@t&f*3O@2JkUKD8O)a+hO
zdU5C-8PTlp{*TwGXYVAF2;~)lS2K3wGA_86COMY0RqR+DfF|ZG)0irLM;AVy{g!V5
zU+h|oHYSbH#DKDOZJWirGwvykP`eYq!jr#fI&6$tdxM||!2}V3vid^P8=fACf1+f^
zY^!>0by4JE8vRU=q#sy<GVgc95W4(dFcV)JM)U1Py%Jl1K#|d}4gQOy8E%18TF&#D
z_v^2Pqoe<x`^*0-N<!dk$Rh?T9eu>vea07@?}2JWkB`FO+kNi&wBfeb&l&jXsY@4d
zw*6vx#X(^;7OIBtme5*oBEwI<i@^kHP$3R4vK10JPxmTR%=4!%q}N%X$u*D5UV7Tb
z_r<|eCI}?Z2SMDpxdq)UF<<D0TJVqygIO`;k%W>6>qeuBe!mJ!=AFq>xR1@1CBHmT
z9}yvJHHefz<b!fF@Bi}Ing)QF*S0$b2!ph&FFInr>yp$D+c&Edm|Sdk(9W?N<p_1H
zzT_%-72LMdZKY*2Dm!&Zbj9gq1&&kzMWAiyw45H;8CfS-U_ywfhh}#hK4tPYs=MwK
z=hep56%)EYzH=?DKT9+lZSVg+XSbdPuxvQkbqWpFHEwG~Qnux0^tpk8PBtL;302j`
zc(Dqflgk0)+|9Adod%$zjI>dyGVZ1G1&{Dh=uXh=GFLa5E!KG`>bZrb)7_FCy~|T@
z81h8?FazN8`a?vrsFr*b$Jy*Bp=K-yjs9j}IN8oi&y%dkMfVBqVyB@601r)qK;4Bd
z(>847>D7FLymixbE5?5FJs(jpHXtRe5)*l3^XpMK`wivMqea<biI5f1lel&(UZ!B&
z7|d3KOj=AHD&roh^x1Hm=DS0)t+Jsik2&7{8|t;i%Oi59Tf4+yt!IJ61;Co|mbN>*
zO8^N)K2~r+tVMG9N^w>eE&PWv4_PcD60$X;8AGd5UB&zfw$cC3!;BkTT@8nhG=<p3
z+6p{Ulul6!D(6!0xwKd5P|W0YCy(wPOxybceXa9}uv!Y5@*zlH<7kVzvQVDEqOYIW
zmF!Ucy(Dosh+)rWhByXLI{i7qf3{(GQeBQtmUkqUy0a=$X8J^df0B9#ge)U$a#V0E
zkKs)fp&OJD!=RA76AYok{holehs#fNFp}M!0mNjCepdbl*^<hcry)ECaT2vs(>n3r
zo{a6sYgjVucX%AQS(fI_dKQtlasTC&l<x^%g>NSTjA>}4CQao;<yJChF(Y6JJ5RIL
zvF5T|slnBvIr}wxqz1E<kCUb`@$LS`XtjgAa1*K9rB5H4*nGK%b@T-Cgzk@Gtzk>Q
zn6~4sc7uQpF;>>1;-+O$?xbzEg_rx&SE0>IM3_0@h1hj<eP?)*h_7LaTEI+n7AfE~
z=h$8QGhUmIKIMA2c7K;mM@1=|nzB^Uvj;1#Bj?4Nrcwacs%x+2r+RM|`t*_b-Imh;
zIfluLSt91ps*mGZ4v2|XXml^e?F%86Ose*DTRLasH^+moO8ADP>J&kbvvhFL#FQ7!
z;%`35*6wS$Z8^yJ6H&p1+eB?CovjCX&ZV#F>LkxOZXAk_t`x-c3@mwg%}(N%C{FVP
zK{yaPCvqpJmBX6)2rQR+t~V5AU;b;r5k{rTFtU?G7z;=K`uX>XAspxq`w%%Zkyt)d
z4<JkxLhRW#j_5`^hi)Sb+jsIdxV+U>$WYv3{Ck+Ixq-mZR+iW!CnVUY5}yUU&qm0a
z2thiV6^eDW^G*<WTDke0yCAgZEy090{VNP*Y=56lmmLtNi~Li8o1W^O5j@tADiLzm
z9R&}?cx)H@(zwXH@VgOe1);T>EJKWDiGIv~=~!V=fVN>SX8I{8;_KYupNgTnWXl5g
z{w7MJL8?FB?!pd?7QA>}`uXM;X+7)X0oI(P!0cmw7;#aF+u6r+pf-HdGGv+xT59!$
zVS0%}xPb{Wny0k!ap;dw<_7gR!7<>f<PYi_zg+z<Qymr)_ajUqY3+!q0!lrQQI?md
zXRj_W>+V$6_UT~1^wa9YE%+|C$P(z<!-%6=V|(cPS(@BO-vR`C#WK8^@lqV|b&t}m
znA2CvHS^}?K4r7%X1U)Kdv%MG#p6n+TlHmhjN^^kbhQ6b{$chUz|R$YiyU0Fk{Vpx
z^z~nh?5|}3Mjje)Rz?Hw$;6MfKmsT3-l7GMQ}0ejF&&#~oFwvq?@d!bET-lLMQSeH
z$Bk52BQV96ytjNbFLNR&j<gIxQNICEQ*l-9tX0e{fm(u)P2x46#)x6x{Hc@9hkpd|
zJWQ8KVxAr-+6$H(&{2l@f&~fj0viA!QtN-}dV$lU40pJK2QZ)!txQJzT_fv04{KzV
ze}*2*NBQB@z!h}+m&K&$oN)++fCkG3eYzZtJkEo2I+HpxmHh_!#yINHzV`*%cp=%Y
zZm1yGg=T{@Qtdu-<X(grhTlb$prT_|fNiEC$H2;fB}Wh$;vH>;POBR`+Lg(9bR<R2
z7$k@9v8{xICw3aU9qQ;G>&Gi@cv2R+mrJK}A~w+SpAHjP6ws~i%a|McO3Cdz-(7Cb
zhjA@z!Rbea%desf>j|9r8BR_|oM4xoH3{=Qht%^IGOOfPStZjB5)b>i{+oPKU}nSX
zwdYRVp||YVj0;tMWs5g$^81Ze3z&5eS_*Sm%gRwLPTIT9CSaFQfrRaayKdOK1Wvld
z(#A)6tqjU!%OSnV8$e(D#2%PNUr-vQ91qz=Gh@I_5#{NdrLaJk@-D0JB1zZxC)P}V
zd`Fm3EwP(My;eG<hz4f*KL4>YXuoi#_AH#Jc1+-R48C^!!(}qY8yD)aum6W~n*VOl
zz{j{9rIIM&lFSDd-B0C}>P;_}t&h*|y?caOVog44|G1}2zW*1hrLB3bk8rqtedcMr
z221Sa*tt)WieNPWPdja16-CfQA^}ssDVN>k078q;yuZEODe#P8EDv<_BpnZ9Egjo}
zH~;A<D(7A*ZjrtyC=8&7wq>vuRYq7;k;_SQxHhgvBk$|Zn_Yw=H7No_2xpx;!)}Q<
z!L;r5e-35m>Dq;4`VP;(y_{m2Q8<YT1ftPg6izPYZdvsz0s2w0#j_;r>@AhGHM!u$
zm7gdnDjWl<bJ_C}*9ks4QsfbE-Ccff-Op4d$NqMZ0TfaBqhf%!c&MtIb|U?%7B6>g
z3@^QI1CtYH>Rnav2aDbbvh*U<lHJs;$%G{v=@K#ivt)EHV0_p8`G1{z)D-`md!W)H
z%)hgH9|W1&Id-JCJd*_Fxvwy~FBp$s{dg0ZHw6hok`wz=S{nKiDa&>AwF0unL`+}A
z5W3ry2qILduT1Br_ibI%i>+l$t>1X=ih1uXNM7;sgrY}Ph~(spDCV~2C(5ug?C^{$
zq)c_>>`veVh>)Xc#O;VApeV?RtVyp)ZKCNLd#hcAdD$R=z<pzwvLluA*BL_^y@VIT
zhyM8KQZb;FlcaR=&X<0;P#b0C@HX5o=9^q!{$Lxc)t=MQ2zhxg4uPk2-kgVR+{P7<
zxV2qXz~{1AaHqe?q&s+Jr>Z`kBS_Hdca_V_yneOs^0s^r<dC?w$FgY4#qT8cjkfyk
zh_r^fnwQ-m`31|x(B*z8tVyv<+oWu0-ZnMC&S>98b&{u)xkE#zA<_I8aHY&BK1p6e
zu3cq^`530T*e5Te=;`eYd@nW}i&xpvx&1RCtePZn)u%}|6GYMjisVV+dhV^BtLU`7
znn0uBpMTfqS8ThdQLR=s`TAXK<i{`)X%XNDcH1CXmAQteOVzwO5NKcIwyjrZIBonK
zyhQ26kqf}V?<pE(=z!d&m+#)#M*@Hix0yeFmOJiVHm48howP4lygfOhrn3w!7LVUl
zZ1MMhD{>zNd#3@O@$AU_B$j`Eb(oJb08LrUPCD$2_x`k6T#!%U9tk7i=qqdLIA(l+
zb?9F{s29LL7OxEgg(&1pn<65+Tm(9Z4^hdR2FSKLUF<ubw9r|+xHhMVpuYfFY{5&s
zS^li6xhP|AUzy{ck|vC_!wi1Ovi=Cw?H1N|>x0u0Rd>6hND2-|h-5&?XXqusxX&-T
ziK4JwwufH=CJvw&IvkA*R6863y5kQZoQxMbBN5UMra3ZB!iS#dG0E#5Ew{a&VXoN{
zz}3)G^PNRA9`X-L(H|5QOH)Hq`xo2Q>E%1a!v5j!9C<n)hFw!vrZWMa3ykxQdWH8U
zvmTA}V)i&MtlaO=6&^VkzHze%D4Lq?y0!E3qQ&DL4*Ftyz<7Sa^+(6{0W%XtNH@#F
zuC;$RH+trK@d-5+h1!4nm9hc#JQlKSkwSg1!fbeWcCi<skQD@kooGz=ttAL~9(vfH
zS@pA!b8W`n|7!kO!c6dHc}|3#$FY93&9{)Y#l`bvPwJ*hXvm))_kmqA4#9x#vQ&HG
z7l}C5=7=kbz_BAsZnnKz8t!JyfL8o!+UaQaZv1@ibiSfw^*Hd;Y!+JE_+Ix3Ca=!5
zF(x2QwCtHW%d2Ak_z6d3bIT&U_o{Idya=f5_NgoQ(;PCJH(w;z`f>6}zbwqbZGGu{
zafACI?~NbJc-)fG;(@%!UZT{)(2s3)umx<NH*m-3G*ZK?xP}cOv-1X9Cwu|w$-*Iy
zPzXt*tNAj(vDwF&Y5z&57msMTEp-F9x|N+5zzu1yZ?u+DZa04g6_^)B%S6<Lx#vWC
za4k%gJ(2zRRcLWyQ_y)5k3k$?%B5~Xire*9Ob~l-SkNM2Q=W-0q)yzE$1S2_ekLM~
zTyAfHp1@bt=)WdFMPty3IUo!Qkes~vtCv??fW+dr8q}`G+tu$?B_Esl7MqOq8PBoR
zjeV21qm~M{AL2j$;&!>7K<a=+1r1uq@~I{FfiFFXw!z-0K0?A!Y;QRJ3b}B3M%rtl
zpD`hDN+DA*om)jZPoJxhcSi{9@rb@7i4azz@JeP27mT+%FG$4sanXA~uS=a<sNN5!
zo5xBih-3clK}GSTFo$TSW8F>Q2#!dNms}~qa>iv6%FS2!lS`bm941u2Hz)6YVU}wT
z<@n<bM@VaQ0Cn0F^0!U{-0(cdl;*o}(n8f=A8QBGoLf*nCU01X=7g7}4YRqn<^NFP
zD^Gp0Q2H6!)<i-7E2lx7SByu^^-fU9jYMR>?y|&e?|b`vYwBA^Z1D(qwSP~ZRF(UO
z^3`Z)P`F@0>tn9FbBu*e_{Uym8yGM#{3VP%qso6KFI6+>@tZ%P?j=GK-kVXTq7jBj
z%L15p%w@8(=+L9$-7MLfkyWXHJ$lpjy9I)MSp~|xBl7#k_S>`W5<QK~l*h_%q~>jN
zGzwOgPs<kpK}0(dIm6UXqKee|k65JYdMXNUcJ`^K?KLWs>2?M-J#QR-EZTnhM1e0t
zUm#E6U~@<LtI}$o(ma>l{DgR47WVyH)sToq)7akW?ZWt!pa8TuPF#$V1S3jzI(mz)
zdA%2I)BXydPyhN#yQ1`9v{-hlUTark{^pLqbp8=d1Kz6Sr-sQcJpMG2n;f^j<0&mO
zulXQc_Khhv$`D!pF3ltJ=82zLYE!nE@p`f^;O5!4@oWKt=ZETF-DU$AF@hbf2&oNR
z{vMJ4_L}$viV6>H07*KEkYt^I4o?CfWeR8Et!cM|#`#B#KO$ej#ZPJgM0sArs!?V9
z6rKV$4+(4`^M9zO4@NjR@lemMw^&BQ1sa<Uvc}FbY~Up50!@%~#{e7IF?NJb<%0*@
z2sc8)!{sysXH#VFB!H!FMO2Y^_CkLzOJuV6g9>$3C|Dv*%rCh2SrU^)nTKYVYDN}p
zwm;7sE<&v!;EQuKU11S=6G>Is8&B9gUM7JFoT2I<oCxt}p`PNusU&R_^nu7<0k<=I
zx)f_s$oBoHEK6-+ab2I~w-C1w(x2ybFJCL9an6u?j_HN*P6lrM5Clb}-%_11)TeyR
z1EPl1+X$3)40P0dyHjlUcR=nwUv{n=zn6IUd{cVO1;BfN{=*)qQ}!cW%8TK93;9iu
zq{jcAxdb6wcViIfWya^azxeOKC4?|}sch^mU#%aGAomRC1)v(VYmnwOpJwe2U0m&s
zaQc<Z<wVYFZw;t-Yygc~t%mFNj8d-9-GzY@(B9Sqe8bQsUH@leNGt@XR)E5El-^73
z3?R;NIbz$8b${vbqm8OBL8a6A9dlMN=~-Ti`~m;9;N76zw6e6s^Pb?IGB1vIo>lhe
zIX}ASd8k&yWwb#N&a705ig_4OQlF#*E%Jh6zI8uII7G9C0};f+DnjG0pfV{D*V&D=
zLhvQFVJc@x@sZpq_2`4?#|w~1m)_3<mm&QTyuYz?>U^U1v=%UKP5tB9(hZkt0q)+q
zVW`3&`7aa0Yg`mpPCyWijxs&>_le>+2*4_^0jF3^JC4wtMfJQRG7p>__>St`R>QdF
zeth^b@|hR{jTBY=Xp)<8vYnS+XE!7;B@LaStRNE0$ar91{$<r$3dQ^%s~$rK#syMk
zZ~~H(H@m1d7%=P6_rG<Zvb*>;oOVWh9UtCBH;7`0;n5NOvfJX>aaqNeM;~5rDB{tA
z{KwbT2tFam;4Q*ZNCD{g;MAJ52HJ2IDqD&qW4ROI1XLxRttg|EqcB)p@4F$Ot+7m7
zI`jn;`uK)SS9m55j^2(3DYN`+mBoBVFu{Df*U<zA+rLp?5^SIseN9%iG4o$Aj_#o3
z-yO(ilx~##kJR{*N|6bGNrT8Y9H;eySm!9k0&DtT)(_ks^|SEccr5@{zw3kM=2?d+
zEwK|EYL~UgsSG42r;@<TX1<&ncGCUc3!eB~8_69hO3<CtggD(Bu*`ob#q&Z+Ljk*^
z5<z+X?dMA%V(1jeVBX-5oSJs$?EROaFM8TVJ5c)rIVMCIPC$~fb66--NUnjsV=%lH
z!#l&u^9LctCrav&12|0d=#W%O2+2<aUY<c5dVKG;K5JBhnEI|FJjEZP0P@NJp-yLN
z)W1!;tXz};End5AKCAh^_i*~S{P7<?C*tCz&_iMF2oZRb(iE}u%*uMC7&Zpo%8XIE
z1J}I$yUTRKWFJb!704$F!1gB=h>0YHgpCZ|y<^)Ja{&D69k58O@7$oAf^Oov0&4l|
zyn_q4#c&=L)YrX_*tPt4hGJBWCyy|@4&pbu_7r{t*scIe-}d)Zncy4|QgO@^y&=Fs
z@3><uW}J<knR;gkJC}e`hV622&g3g7yH}rd;{)s922fyoOy+1l)W3(O?(xIN|366x
zr((JRFB$8|CyU+dSAh}JE*<+m#}hl$*y3DL0<>7i`R`Q%9L%~e#f8t)0Xzy*p-KMl
zV(}rtH3o!C?#I72i<_=Y)s&Q$;d5R$hu!tS9XgGHc#{kEKnOrB;9Fk&mBRBXQh>yk
z+_d+3f<2&qE%H#r%0KLedDQ^X$(*`48^3Y$p8=ghS?(0se#G{ju0=ZP0+Ktj|4+{&
zG6br$L2>J29inC*$C+#MCt4CHnOeTEL&6RLc(*t7lmzECQcB8)7QvYMwUDB@iN~om
zzu313TUZ(Dl|<+wAL~f#m;aXHofKbN4(?^7j(q%k$0I+TBQpFyc-}Ekhl?0X^QnO;
zr&GW`RLT7DpPe%kd9U98G)<DU4v(J%%XGKr-}@sU54oPJ01s;R`aixMpm%3Qsk;Z{
z**-oTPJfTuSRNFBf|$1LXVwCo|0t$;X|4sNy!OV8c$&tU0dwR|bITvA%(v5AcNm;@
zi>fDQjX1$mI$n){lv|@}L-IS_j?7|{2Wc+LZVoTX@sLpStY!V)9B>)H-rc!SeAh0m
zSxI9RDVJpmsi3>_JQ?B03)EJF5Lmb~%9k%cz4}(r6!VOsD`;>W;XWThcjVv8j}@gp
ztYN1YsY((HD9`BmZLR(P;#&M?r4j!3N&_-CO5La_N4QOnH-61$g}x-g_H>Vc2?^f7
z(>TK*(X;{L-qmG9I_@@8Wd>4Qp9h|v0|#(TUEpu#wVDAn7pfK~r|sMXY^(FXK6t=8
z;FK*DVHw$rhzzejtaCcp|5$CgCwPMpJD*U~nW!xRq}G8pPWy8+@=33IJ<jKy{_3q=
zy|s<H(OB17&y&oGw}1aM+XUh-vRRAkYSz7;IUjnvP5$^`*#XDf-z{mzJ}wFsssIj3
zpFjy<<>v|`mg>p|d3U)Nggt={cRrTk2gdtHQ!7Azh~mY?YQ%iX6;sr8u3>@}r>2mO
z*R4an&M3HLln3AEK>-WAD}jNqYqb4fxi?_s`E<_7)3X%2L5%Y(cWn5A68-jkef2C*
zZ`y9pM8M>0p4WTTqP4{<-48aJue*<KEcw@C-_6zS7Vlo%d(0RnOl*5`uZ07vL0}Fw
zDxS2L8aQS5gh13cl)m5wltXOs>5Rw3!lK%)qX8YrYTlfGq&E-2S-dr<M=}B~hmVJs
z6?HDM$Vq=LD+uOFk@Ch)2wn05o+Yy%BKmnE?2sc8e%>-Ktlw&M1d347OQlCo<yiM+
z1@onH7|C-hqc){?730&6a>RiaMGEm$_uu7aMx6`8@Ue;?_B;fN^{9mBe=O6slDco}
zxjx!-KA8tp0I35)jTu6~7>p};($CUtpYDDn(}nid{pL9RUd}z|6~DbNYYP<vW(i<%
zk}G`ASW-Fq21q^O!s!S%4QuiZSV2Zb2c;+e{yyunncCh?_tPkev{~hX?QW;ABrxb?
zRHd#31+%7NNRW>(2#a)Mzxm#iV?8#_c_lbJD66&)jaYN(n*ts0H>1t#CKvCn74B{<
zuDPWn$|1YA++4?ld?nX6$3yeLs!6bU$~0<THS;+f?Xt48=62xKP|c9wHPig-saTBV
zYeYMaQ7TLN0dT`<&;dp|PHC~se$(}SgCK#s(33ND1HEQ5{~-yc$PDMFeeLI)>0S$s
zz^mKXMQ>X0wNX&>%N6ZaLqGX?-d+U^jn*GD?OvpOj(MW0-bxOXSj+FZ@de+q4$G<?
ztIAYf6=Y_q6sqp`P)9t**>YAV#+<$Z3h%O=ko8ha4TC}2!K@D$x>Yr;ANoZv`LO$Z
zX(QRH_z*d5kj3tHs`*Z3mkRFnIeO_b{r;wV9_U#ny1Jb%fX`!phend1J!hzy(hM9j
ziypwjn=l`H0<_f*HQKvgAL_{UiS3+$FMmv4=JH3ype_HnIQP^;J2<MU@fir1v6X^(
z*jt#&@L6%&lD}pHZ8(BXQss3}cYOQNNzHMk^=p4032ims-&$pPB3i+GT6Q0nLT5RF
z8d*KS-Vd5j&~|0O?Z$8Cp=swxyidJZToT*{KFU|I&!5NgaZu*64~2;Il7lzX+=}0Z
z2A)0>6Ppp;p)y5nYwQBVk<*G=iHp1x!(Tx-cVK~wCBJ3QSc{L3E<YGAHZ|7Yp3Sx&
zggyMV#=|aB2jOhTn3r00+(gx(3^$Ggn|HNe4N}LhEJv~euFsqA$+fX{E&0K9D<O<W
zb=qQ8o7K<GMRC^;xttM8<-RW-`WUd_9PE&Y*|i5LkK&!+#!Kh8jO@P_<Zdy{olY38
zRjV4BXrS^<uv^AmX(vwfV7KQBju*F?K`?-RQ~A)h%<_MGkOv4JPXHEhD)V7l-BC|P
z)#lMCtC@GI$?lz=gm@qU@{3$gN-w5;nDFDiO-h<B+S~wdPw~mK;gU*8fz2?01+ZJ-
z`T6W$cJ^306uaMzc$k5}Vsd*+hpOKNOt~?*xSwJpa6FMJgp>As&~bj9&)F$}Td9fl
z@S?%_i6GzhKE{%i@83U?<7`nXA~;Q%s6HrmqsohRZ%{Y<iVEhA`UMWH40(%x9_A@e
z6ILnYSy};Mq-p^&dtuP8ovymSG?bZGr)aI3ZyhOI$G>sWRdJb34wNC%3U7dYw10nh
za4%Y!D93rxW>k#Q>Ni9eeV`8YI8mBv_PRIKI{Y>YEM(Zuha4*QxALOmp`=y~Z27_`
zysKYcu@@Y>wCTJ&YQtpVUdSjcWc|ii(!fsrjCwDovm2PCUTCGp))w1Ym;v~Maso4I
z=E`rg5(htYtiF8wbR8lI;9kyj64=utJ{&4YGy}~k@H%8?Jak1ePp6ox=b}&D`p$2G
zKO4AA>e0C5|1&_t-=S$3ry@FrCq)*e_9-4H(j(-qTbYS-pMR{@IUV)V8JSnS5!1C4
z1u6od6GVJ|eg3reQE1SWPMb$3We^!Y(NZZ4sK*p@eQFjUoGgEvTS>`qvP5DV6%Rtb
z0E+L|<3Q8HFCNc$_BCWg=$xi)8L0WpAJ~UT%ziAxmBEMjqq<@k=hxT!^;RoTeWqr&
z4bX!uIK8ur29PCAe(as3s=W#9A?8UVz@@|}w-Ib^J_2wlJ1O19(@D%B!UfmeB2yOP
zwf)@6*TL`}JL1MjB+9tOkejpudHClJJEHyUzPPzoGHSpr(!|{ci@BvlT0%kKJWliO
zA*r;1(=Ao>PBM<S<79-1)d^|@G-tTlkGcHGmom=QxypMmqUS<Iq&N{o#xYGp8!Wc*
zCAqv)6zf6kG}!1b$RCgzyq_a4TeZDA9LHsGV6lhUckY~rg`E2rhtK(r#!bvkV}uq7
zyTDk3RL<U#iNK`%&?(SGgS~oy-xgo_KUD~wA`ev~r1zQluI`~~n|6(PL^*K4eSg99
z`nW1QOKuqVnxb5?0Sgu9y~}kvm*GhjUDRu5zw~^l$ugGf?eMoSH$`c($gU3=Yd$zQ
z&z$XaL}`%D6yyQ<%+jxGZd=G^6zj?%3OIBtLDN>|2^U*$7C^CV$el$0_IKIRv!3H}
z<hSMqwoAK)&(BLep8yDbv>YI$Uscnv;P_qxOhDXJKZO-_kps72u9(En;jcLwC(^l-
z+S%E@`yN2JWt>^tj4&EbF29A@^<>$9vk(<WT5@j!U<=zgj`dgH4>b6R9%}>WYDf4G
z=JLvzI1>Ds3I2P)D0wz*fKz>ccTKnIlMli`X?W@-ArY`UQJ7#bOMv0#NFYz6vwS_|
zr=y1*@7@fl=szbrLbg)1nU6xu2yW-Zyq8lRq#UtuK2~jWB3Fh)DYnFbT}Ul!^hRN>
z{Gcz39}|iOOZpl-dNklOIe^K^D)1N=UoQ=pr6qQYCd8gl3WFDT2d^YKeOub%nehKm
z&LEzlNkN1fQu{S5@S6@EWWymcAL#++BDST5cDVsxqNfA*>nz^ztr1|pXfSaHa>!`&
zyrt!tj%-pldGQx@CL|FQG<6XDbkD2re!?H$wsGLEJQpV})>Gg=6z9Qe&M;VC0FOf3
z?qid(QlC_=@~7gNgdBI?4+E8mLt%Gr;&PsF5j^iWYr%ofZ+EWgE)NVe_hjR*eSwXE
zOL@HHhlAbh&~x1QA_t*CtgLFW_DA&0q^X>`<<u^S+!69fJZR0yFmuS|Y=N{2O&O}h
zFr!RJNrKg!V44IQYJ~6iH9n0>q8O^cGU0gxxW7-voq((#u%_SFX=FNKIf+BGB~*%>
z6(Zekk!PfUnQ8~`w?65PvERKgY%5zVjrS4=G5~^eWPot?n}RRc?r5j-9&fYW+A4>~
zK5am!LRyn0&{IS~dl$z+xq};5Mu+oM&L1P!<6V3@cL{G8h6a#C6Jyl}-&t|hmf2ZZ
zL?`_O%&cGf&i*~K4*%aFG%YCrau)L{&ExDtck(k6J)fBB$|u3T60=|JlR|MkLC35R
zU-UMy+aRcG{u?zW)%~AS+A`#r+5FNyG!2w_5K!fDsZSA_elP}<$=K+3cLJI&R8tr#
zqCW5?+qr`#6d-n-5nZT-?lk)vj^?riEIWmQso$uBaxkE{R!<|S+!44*)z)L8AVO<#
zXXo_2u)E6nl=tB>Pu$VN`Sxo}$56ZXsuu;tX>q?X!SNbJ{K=mMBejT8LceTK*M12&
zV6b|!UG$r!>4GX4H?!#xAo=d-oSjeN!*eZ{Fj;cK635(zI*cnVGh2o+C!7E!!l|FR
zT2pbIat?zoCDMkGP?`?tRdhy)ul@7R!-NhRj-x%+Z6KNGS<)Y!A?V-5r$h!}n@)YP
z3@Yy(JXhaTJ?%Y1l+gNuXX)Wcs50n3*adW!v0g|0q-Z!SWhWBt>YmL)#FLCDu=nOB
zXz!>u#91>hyOPUh0;V4H1pw73GwTO5P{KqY`}+^TjgnElHEB=YY}f-YD92pZfsg!>
zh+^#i-J<{lo(tf_o~Y<>BFL)q&;Mu_0;C944t)Y-NshokjNAGBo7?SGM(6E|^CI)G
zH{#+kE0itA>3dbPFK^=4XONrcDID-mD{1k?4<%E)hHi>it0`DyD9m?V<{hRCQ($ZF
zJ;!W{Hu<^_N+JZW=<nakc2H3{pB#yitmE9~2Y_2B^#ghcY#ObwQ;-g^YodO*XK*H)
zKg@uaN>G7`&fCbmFv3h7^Y^L+uD^TQm0Dm`1pwYqu2efo2zJ2Zooe}1(keWxOHtWP
z@WrC1-CU1-4ew9c*w@nTgsb4&7%=EJUf+>IB{qofhT61S$d#;I?E?@3mIxJg_K0B@
z5f-8Xj5l)xz<pmUD^`vFlN%!XaG3W6q_ocQ*>5d5zapc|al9F3SA#WB4r%A|lj-jI
zo@ne<9-fh+%pi@5aQZ~HW)){WK_vTG5a{;;mulQq&03OX`9>Or`a3OpB$!8j{iO5=
zknp226;INL)B4B}$R;g~e1AvhG{DvkiUY3938+@lzvUjv;*Qe9CGuM3axiG*;Yi2)
zE^0@IA_`39?;q>!x=0g-LXZHHbOuWkOhNGS(?zk!<7wdcUZt!7fU^}euPeSKZ`1-n
zk!HV->CW|}i|h!A##i(C!b@~g(F_7Ap=2_bwWwvjw{CST#b(p8&IHMg%@r6R4yfr-
zBth?qW9FdMA5=LCKjZ7%nj%`?AgFEjeMrO_r0jVN;#mM|_s=+Rc>SMp*(X1xpe{7Q
zilQA~zWz~?7$Jsl*leac(aug-d+727sjnPsr85ID0C?G^E@By<fprMLdr3B?TdGxg
z%U(!PGZ*(FywxE9FO=qT=hbz-cxaVdkhHGG{?I>EJRFn1d=jXgZA8zH0>pzXFl)Ii
zdzi1B7cvs~9`T&HyO$Zh|F(#Bd0|Yi<*9(UHc7Uq!dSzGuZk<I&0tz^L{X5K%3*CM
zEGjizWF-W#OT1X_#IUE3KzR5O9syVm*IFsboYVga&S(n@gX8_Ul-0)%at51r2lCH(
zR|+^lZqU{Z6;in|r+XKSP{GLQ)_B~X6rhxy=&dlR-)VIX;)BQRA+d>%{|0BUs;5yw
z=+R~C`~D_p#3?}CyF!0Uuj2@zfZ4%xk4Wu}#d26mP5vF8k$mCD{E^8w2%ME7@y{}>
z==p`!<|ptk)%-(f5<T>xNmeRedyGr>0*MBT3jbtpmWv1D@?uFoh123UNMpf!SR<Rw
z$KQh0IaJ$1M(X=xy>bPf_1HCJETPH|(s6*wQkTU6s!PG--({sA3*0a;@RMAk%9u2X
zD~iAwHx%KBg`*Fo)IPPMO2<Who@_lR(V$043YGpPAC1$BstK7N^bziv0j4l9oY;1@
ze`f(`dx;|%1H7__i0jgE|G5c(TBw~<GQS4gPX2=FbZO4|T}37<$cRJ`SWv?~?Fi%W
z@rZdSa}{LY*5v{d9T(#%Y!TenYbNrN*6UN~Abv#hO&6(EN`n$ai8-8Cync{nimC_i
zH!ba~I*?D%vAU{9y%ufW=9C0>7+_wY4V9*&3PeyO1|9wsPz)UI)VNKd^XFk#3kI_L
z{Br|^S1B{DhaOm%N<X;}cVLuLDaM%rWPG`pR(+z6jOXhdN&{H?h}<EU7k7;B?{0&*
z%x3gM&}fTg#XpjAn7*GbelaV6fytpsj0dl0+E~s`ZWhv?W!w*Cmngcf@BbnrXS}rV
zy=pI;Vyv9L!-M5M6n@dc={)d?{FcEW=;!;Dgf02V)Y?0{)Quw7s;U*QR-oD6s|i|$
zM&~;R;J3|G%#&=si@L1IyWFooh<;fvP(J`BxU;HY&L+}NBtFkH5s7efxgFBK=G*wC
z>IYvKpdE6wz$e;dmpFgt*2J*-B|8D=QjD3HYKX%E78PzBsjo=54?p<R4;dw0gXgI`
zIo+)&*r?}N%Hn)vw*StYRJ_9b64Uz?Rc?LmN~2bURH5a$W@0@6<_Wtae(ist>2cKj
z$}=sxP)lsrhWxoch0^<r#a4lb8;-=P0cJ=}8d_^^G&*d+oa5ygbHpU0^mug$=1`g0
ze!vHZZf^w!kjWZchCra2b5a+6UPLz+p_dp<Py2GLk&OvVe1k-RMw1^bDBY=nm}%k3
zfC3jXpdR7R1eZ`0``RPK((X<vLduWESH-CKZq}LE2d8lPu#tr~3&uL=3m1$EkwC0)
z;TUFzHN2TudM9XPhT`8(hBLzjDXkPEupHL5G0BhKjO3=W|7SW1pqYFSj1c|5V#x+f
z&LuA+Us1e6Sd#-e#jijz&eHj4Yv!{J(usTM&ZXhPPe;T$xI@r|zK+dS_XwsQA^8Rk
zKtc!j$N%mqJQbY$87U6WE$tT|9H?mt5F@K+$}b#|jcweN!*8zk8U$@AhqRLc8Yg=U
z#=drAqDGkW67s{0r(_W=N}=Nm-Vh&Bj0FQ1A#;dOp$LoN^Q(`0(*#&+U+#Bh1?7#O
z?$6JL0idI0aePGfXLn$nQ)#+q`n`?(Jk-+tfJAwkRwBv`{<~e{(@%_0p^q2A!3O3y
zS!P$ixE3{c^;550uD0_Cy!;J}THj_J_oKD|6x#IscDCNIP#dbDC~0@w0sn%kIGn;B
zz;kJT`J-&KzG$m)v9qj+u#AmJ0scSB>@dzpwA*%SF|e*OV3I+LbK9@Y>*JUK(l;p8
ze#&D;Fs+p}ey|u)88#A$xi{OII-3{2huvRD)l3p7by01pc6z;ivHej6D0}-e_KO{-
z4~80yap2zr4mA?Kdm&)uH+A4V>&cCz`QYC}GbbiG@PL>Wa)^<c18{P_*M99LnMNU;
zUidkvAt_5p#cqp{;=;s@m8hW{Pq{Qe4nj$jEbI;MMV~n-pAcIZp42oUzta&G)T4}d
z6W%{wxo0-Hb?;}ovC&bC`9#Gvjz%!p3}239W4{%gHm~?Kr!33-BoGzZ-|ZvO8t0sK
zds5)jeOQgq%N#P#i{(}=i?3X?M29z%ns)NlT_k25Dk?{j{ucsI?ukTO0jyK^MXl(A
ze0zm;R`tlOLb0v!s_}e*RN6Z-slsC7<025Y8TCvALDxq2>17YOxQ5|wx4P?Diu{po
z|6e@dCL-9Oj|0bN6~I<ys(99&p3ORn0`bcrc7i=13Xge;LG>9>qPEsaC9e&VgecS~
z5J=)36fBnBzeh5$1(C8|(;6w}xn5>`Nj-6|dXy(}HJ<JF61emjW{saeCPE&4#&SEY
zllz(3YeJ9N^jWfj#?j*tA68?3{;a>TI44hQPRfEE=w&JN^Kq27d*dB_h^pVEPsW9f
zZfNHfdg<~Do15!!GKhgrsLQLsc!u0R%Sp-sh2xI>jPAe4rwv`uMx|ReAo1rX+_XR&
z)$zRS)B=#x-VcKlV5^Wa;o#DX-Uves4PK^-0rt!uv2B2#)R5=+Hfz=Fp*|-X%*96e
zRoWDA5#tceIWLt|D{jBK<^22|mwy4^YFM?q`os+2^x~hTF{7lNL-CgN1TkL#3&{Uj
zNq)Yxrm*M|vl{f{CpT(NrL<6X2TJFYgGHg4N!rGI7GO)^@>3|334k#*m~M`&!#e{{
zvaBdEpf(-R4-pTS;auZVYh~R~St5+FwGj8~FCK>;J*P7ga_p4mHaCcOn+axQ8mlKe
z6a9xsWXo=yUuy&3T%;z1E&049BM&$bTvULFyQ?uw(egX5TbQl2f?<zw2F)rmZ-Klk
z+2ZFVjb|UH(_|ba5xDq2^Tdnvm7XqNLoy4#adiQx>>tFWz}5f5)>}qJ+3xS(bV$t5
zC@>5SN-CucJqU<MDIr}VQqtWufRu=$l$3&WNQrc-bazT4&Ct&|w|np3-v9Ny@m|X}
z-fOPwI?rz$$LH`df)fuT_X31#ng|#^!lWP}sw-rE33nmb)BYpK*PX5-w%=O6W6M>f
zC86I&#nwU*=-DAKpby!(FKy7`(txcTY*U8!n*G=j$q@Ikmx4-!{tRpzjJNpqlIEJ+
zW3M78ePkXe&+g|*>eqtUpimTHg|pqGD~CkGhvMo?`_#KyuDr0}<+E;)?^eBOPE?wm
z{JcF8KRZ*JB7#q7U(c0p*KQEt;O}erlLkp~ShG4A3eBHWDLhz7Ih*%zB%_3^+yITZ
z42nShfLGxxvey^U4yN6YBRq0SN4b6hc>9>{b1B}R8&~n`jl5FfL;HFhdwO*oS>Te^
zwf+Q*y>t7llmy3FB6fKn1fD|UbY&E){XtBlZYSL-1g{3|uaa@GvJmSkiW<#fP0`6@
z-}@G7gC%gyBFd@lirH=yM{gI510Nbq@FboLxiZGVs_BkpNrFf&{`{^X5fFyPJ3fjB
zj*F0Vx{R0eG%rhzdLgP5>;pV0*dACzS#-?xN#Kh?6$GY*ATIju8`Yv;?cazhCii-i
zljUa}#j(v*R4fv^UwbWByW_!b8&fX9gUB>atbRFnwNlvDR9*t>7!mc!TRW=RcH#_P
z4e6hX?v16X6V_DNIeYJ#Ht+7dm|eIu{nKc`{5scJMTG?W0tJ?Cj=dz1%Q#z8a`~)_
z|E#u20|^xWX*m*8n*6?P6qG+|#6YKt_eRY5r97xoLit!9!mV#BfIi^07?U2f_b^aQ
zy8d{~Ja#TY2_Lcat`!fmjt-33!IFnW@@Vy%SwAGtfbmc_;Tn6P?@J{>YuNpgg}0Ll
zClKoDt1CLjz9QyJI50#Bx`p8tB59sQ4@_1L=Zbn=5S*Ky`SE_l;|MVoLn7END<7mQ
zf|`W_JHW{<>O~zBBUQ#isY!9Ku&31EuDlm*sNPiX!A?mWUhHD5^o(8o$!}7YI%}Ol
zQC8W!G=FNz;F^5`Ti@)Wdb1$hDBay-?G+t7NITuezV84I)b6||KLQXxyKGA-1=LMD
zGD%4?Q;cW4dXJ_;cde6gGCh}0O*Fj=2I-=#p~URFK172|4#aG!wmd5jM}>Yv1u8Ae
z0_UJqX1CYgykK&`*^}lmiVHtK1P;7sjJM;htF>Ea&3pJ9vaBdsUr-ow7zsZzcDOyR
zXXwfL@oIQ?C9miX4?&Z>c_^cd3v0+(nr+BTs@nuljoWU+YZO?GvD|*CYQCFx6+xsy
z2ruEe$><7Py|<4|>LPv|MpnH%NpmFYR;+egyNp)=DPSioUkIYYDNoPAIz#TtR#pBn
zu|ir)nq!B)Nf*C<H|^7(s-wiLN{+<@;~}N<$+@b#3mp5r?BL&zsw$Yj_v?@=VTg{Z
za#Dl+rl7f_9{Wm?3-0*sK}krfSn8j!MwYbl%Z58W4@qX*nsJ(=Dzhx=;%XzjI?Mfv
zc&2QiWRbH3ot^P)`AD_Z5eyX=%*U87+HDKIGEAS9UqA$`sb3`18ga5TlQx)E&x{;N
z<RR4*$H}5L*jA|bt~V6(e?J)5@z41baj<zed8x^n*yytBK)RHZIcK6>B>tT>dNw$K
zxFPsP<jcpri~!&>;z#-qx(Xo}O!ZO3UmPDCWmZ`J0dGXfFHKP?^W{x3ZuUh=mL%a=
zlHEgW>x70U9KEtvtMO)8Qhf;ynvPT!!Tg>~t2QvH<m5hULwtQQ0_GcW>v`s89|SF3
z%=)wQ5{i6YcYo%LP1{2&>@-ol%TwG5l;Q~@uSK1Hg2dooXv{Mo)yHkg20MuH;9{<+
zQhUm7tk;>O@t6^9+I4HQ{}Tze$^UCW8&VQ8I|QfZ(BStv+FbJ};WE92dnEO)Rm!6g
znPf){NoH|f4P-f7h3+Nbs}nn{apO7HxkgQt=hW4Fu38+gh5IFa;PkNAZwhwkB;|$e
z$lRfAFBFR)lKqifbV4+lgBsFIX3D6IAcaSq{hpUxQPiK>fk*Rco0;m9XY#;T&9s0;
zUGfwjB8GgAX6)yjpJ#G--|q16i=Dlaxh8%(w(rb~58jN!L6VV}x)N!K7m%G9Mw%Xx
z@aP^-KdR&P<7ZtjXLQAb1rv>s0_;HfT5jO{OJv^r4UEzvi{q^>E7&x!z_J^>mCbKv
zi52NV7Yk=jl?;7lU(9(2@3edR81|h+gBx4MJR=R?KSb=6UXmg@RR{cEF|i)+Xd51H
zPbVlyuA9%BqQUwRN=v!uMMN<1lw{z35yTh-b&RuXQjKcMiK<*v*okZSFmQU`I<L!J
zzG}De{AkiajW%yB)|pz-C!(WKggz*|ht3XR!4>p?GjI&s6sb>}(egFSCZ;PTJYZXz
zM0E$O!60Aa@CQ2`d*9A97xGgemc&3Zc^&;Pkz0eJyywzO{sEm1b;>Ylf57W7h^^U<
zr2bgCl(6R;#vL&Cr$^!s=?NNQ9*79$Z{GiQAj(@&TFBD>Fz$mAzbwlkSIlLj=naEK
z^D?gWsfOU_?Ttv~9d2rXA>oW`tP!w`61Ow%ykjH9^*&pO7`A?kswpc;7no?Ih}XCn
zw9kJpN@366X|XbB4Ya>>hfj*ar${ER-eW0ro%egb0AH4$SQ%E#5*1SsXCqKqak%#@
z(1mVT73m9(EcH$}$p=`8AQw%%MT+##?}sEMMeV#&ibs?L#-&~QzlloM91I-Gu+tM3
zHM-VWahNpmLns;p+}(v76V96|v<6_HSO!>o3wUVT&U`~Vpn%>#m2V|oR<>kg8gB*N
zPhMTdmxH{r^^*9a-Y`FFc?T)~CDrX6QRVQZeF}oip6dYwQINegHy075y*PfXSBT7r
z3^wD3M3QTr0<|d3^}8hsVm)VOP#0@D6l+8$;WGY(DINqOR|7U3q9dwuHH*XQiz`jc
zz@&;5{C}4f5HJ3n=Q?30ibV{PT%4dyNC;t#?d;G~z;tnVUoO+e5K%(}39jh;k-Wy6
z`M@lw1FVt^IEDmC3J*Gx<5*2ru|4YgezrI&Q7ZX@+$Y#=3#FZeEy$P;dOlp)d8!Hw
z=1;mRZxcBPwmZBo!TQmn2kY4(ppH6y#Uu^Y)qP%|tlPI-$V02WfPa4+niH_f=bYS{
z9SI82-AdxR!d}(_Byew2y`K?Igz=<@i(B|2XeU@Wly!v{?iBWZrA-=$+Fzx^_z0NL
zmzI&DPzUSG-F6=Kw+F1cQ?L~p*Wg|;2ai$3^>q>_8i#q-H5BQ+2H<6vJW%LfnA-PU
zCYvxIf{n2uQ0B?Oha>vE6f_Y|ye2q#yfFjVCwQ!k&GNcMcWQ_nPBDXFSsMSim+YZ{
z>6Ti|CkKU`vD`|`KiC`}EWLm@49af?Om&LfOD+j$pDeMhYMmT5N#%UrR*ANfecy@2
zJCs!%nx@;kw;cQTi|NFF?@|#N25=pzSs(|5w2%`AZ~nZ-K^9%#daN$gl*P&1Bpxn3
zbqmv9$x;QGG73HVJ*BudZm3GZC;kqoq4fpX_6R1wFWsD)-1Ake768f|gZo;jz>o<=
z7yo39QdnXQh>fS_Ja1ro8qP{9Yr?UBcYM!4tL7?T=e07j_Bl7>g48fwRLa`fwwWn~
zzgrWW)4tW53!<08C(_S=@OVl29&||7*$?syuFj%s+_Jb~KHBJMz4TcWbO!DkM`;;x
z*85>Q@d3%j9uwJ843!!}=zJE(q4H^$`*g&adOX6z2wSKoMHyy*dxF?6Kwa^iYiqGX
zGQaO-Ql+2Vm1PYg6eGzx7y8A_>L#dAjs21w3LJP#fjjbHmE!o+F8x}`6TVNHH9nFX
zp31(fJ1B|zQL&idRZVmtz%)^!wO+2wJkX>w6Xa-!#5Mi+fzeM`Nj2=~%IXhduHa^o
zwAAA@1#H<`B3fSE(zYOt*0;zkPPevJdIjOGSxWs(P7SJWrL9Fm?MQQh0FfJ1=5+W9
z@gRye;DO>^`k#V^ADhed)5mVdTi%eK6QAv#pYQKjJ@Y&M{_S}BWcm*3;6^qP?Lo1S
z37hwXia;izT^?5s`5Ayd3ZE;7lPMN_CX=GrEE!GW)&iV?q$Js`Lr$77ut^vFttuBP
zJH*{G9NQ{))i2sTSMZ-vp$Uzgi6t0Wze{O)%XFTrnjj&5SPI!(kWALg*0dmpneFdz
zs(7d*?=%uYYCYA{V|QpJ9fDcc?2OHV2uUVDOnaA0*JNBv7#WFcNDxg1uEN8zTnI3D
zI3Rerkqh+-J2Hx`c(BeAL9gK_I{5EQ9#T`|u@XCIv4lbqa={LlJHhGr*0hAwt^qe^
zo|(>jJ$aq<I9{MBxJ|HAlxPKox=Bfag)S(?dV*S#pv|-_iTimB34Gj!>#Gb#A7g2u
z;dMj~4itrKe7rL9?U6bDSC?IrZ3u%D0I@Ylz0(^+JBzzi(n6j;dc+iGRiWFR=~dt0
z8BzA}3LgqZ`#o^+14hMlma?5N#^ys*+ZV76FC8I*WyrRJlKC`5LCz<#5BK^)BQE4&
zLBtoiB?w+7+2RkyM_^J~l89Bgwm$*TKu;_sD5Q#g==P^yd{XzGxn|WQHrwkUwK8kl
z4Z${acKP+CmuQji>E7<(7}%GatX<Ky<|KJA#8&AP8?i*<-lN4|EIb0gXFm*es?Hq9
z{rPGf`LKK9MZ}65?*bcCVBw0)an_(%L4C;x9)m+OX}PQbHwn!9v~?ey^3oxC=vnfx
znpCY1&?7}CKfI0V=x`qRpp<!HMtfFZV>F4D`Dq?iq5B{d|5!in6O@*hMwS<*Z+AlP
zy&cLIq-ci{vG?U6f<eQ_-crax1?+duglD}^=$VL0DyB_>$$~z3{}j)BZmYlTx!vJ$
z+4yIN*NK0Y%@4v8WWux3QjW(;RCYmg#RhjroGc}{-k|K&K<>>=$?kQ2@J~$<Gh^!^
z>wV;LBwO(~v)k#rIf+5$2psHmo};IVN!eq8&Us4BE=;=qK#|J7<OmXuW%|q501J&x
zWJ$hm@uyt2FJcy6AK()oX$7~(mg-szM;w+q_`qL;ry?&)6^lz-z1FY2t_}{maliec
zQgGyutRfTc5Hjyw;e<vw*}t%4nvnm+lI0Yh9s)qBF-p@c76cJsDO+9vN07I!`ow-G
z1RI2N{5L4z4QXH^aE@?V?KIxII3ha^sM|FhgjJV9U_~7eOWoXsj?$UxMN?r1xok_Y
z5hiz7$XFr!SwOh5Ci*4<CeunBQnkC-cYH#Lc+Q{*oyQK}_Ps7GeGu;i_YjQnFyj3r
z|HZUFP)sjg9Os(36*=&_5=CqIi#s6BHIT6dMErT2w+w}hH5Cw!N)HfA@&g)FNGxv7
zreHtMe7)?<Jgt0((xzZBNhOX2TnDjHc740APdw0(zc50B`oXe4<K<SAWp5S46@1lw
ziw>TtiO**UHYfi)*`8Hhx)I-wE|S`9rK+jTeZs6r+x7MP!?#qGIDI0~tE9UseD@lF
zU5fTlSGOO1U1Idx1TgjAkS|ya+@;Ymh>t_SJ-FA~tUssz20N>7CghHf(j%&UwbR6g
zbP)<3T9SVH(?l>m)cYQc=d1P#*hf1+G}H@2R3L&vr4($oAIO;6+BQ%+IMY!f;CI*S
z`O_3xPCNn`g6u|c3}30R)U$XDxuo+MLu;xDkpugQswpvycW+Ncy?H&?-<Q<dZ<w?Z
z&g@6OeYNQMc|Qm4VL{>kAhZp$4v3p~_H`Rf%K5RoG7Ih#mj}lD_)Woj&JDzR*G*;9
zbKR!cR&HIK9lL){_p1%&ZFjwPn<b>qjHKZgvbEU3I4<$fED|+^AW>GdW!r-_G@!|_
zPDq-)c<@O*y6~qw-FAQ$a@0m#sEbXNwD{fP{THNxxS15$gnkKf>wQNqtLTgDcQoxW
zObmYRPwsWGDeTN)r#723T@PT8xke~sfGdu(r2OP_-N6DWsz5gK5x~uqO5S5y*Qp~|
zvsN>9^j^MBA}1*q>F<-<W3^9px2xByhGt9HLpAQ{-jL9khx%&2kdo%Ol7Rm%dubw*
zSDI*E+T)L1;X3#X6Yv8y1{n=~O{C}zh_$}|!I5!{P+gvtW(@!VSor#rKH<k%7IFgK
z%nr{IjsM`FZ%uyJP+fZE1C|9+)v+2$&u2{P0rN`x<uADfFzaGC2&w;#tz?&>=lLzD
z&6=;GjP504+Fee$A?l{!M~h1EU2XPdrJXsxf&MH-!UCdn(o%d@Lh+JiY=|{`4OlK0
z$Q#TxwZ6K3$0F^0Sx3!`aHx#DfBtLg2RkUXr+~R&Iu`n#2&;|Lqc6P@t{lr2(<9?T
zp%BzhKbY4nn6*;)3w<Z-(M4SNT>OTe%{s-*jYVw15JK34@-3x$oa=H8(H`Z=gfG7m
zBL!hgW$<=f9pYIw!(@~X?)6O3`LmBwH8YGa8?lE!teyk>wjg_LccPBb`#2$ZTj^@>
zj=F>&Te8iqSsNvTuXcsa^`l_4ZM1{C8x@{UBn29Uy14KIN3aVTtxfvD%1V+`TltAC
zhrWre(b}Ih@IFR33sA6~?+l=7qDHyDPkULvg`nhsK<k=<lI@xeoyMr{%ll4tshQkF
zP29cGYxswt278(Q#YQB%zChvOb|N@t(^|-%L07Af3so0#8TBYw2ezVAZ9BPw6y?n+
zsmPK9^aX+GV}C7A=N279c=a8n^z%PUGn_MNkYr8M((z}>9P#$+4|-6J#rd)HTtCEk
zalq$5$|x>4rB$xPNYd|8T?Nt&t(@(zcSzoC_^$Yho`Q94H#wa<J(}iv*x5^|K{A$e
zPO~Y4^qN`PKp4}Kqo2+Gw`41ZiQDnsF5NloL^99kI8rcb?1{JOhfNk^C8pcei!mgV
z_<Ib6njyJ4CQWyjju{&E2Q+Lvw~_3F1^fCu`wwI9uzGB6Z+topmK4%kvARVqj<bWM
zG>_lN49_vjCLV9KD_LwLh{3XhYHqUl<TAM}QLNv6E`RkxMbZuB6F0!++HF%{M6+of
z42h=kH40h;93^pK_#UZT!jMVg=@$!1Qp!wxJHU3EN-L9r!pCqq-Ij(STcz_deS?7Q
z*54l^?{$LOikQQ0GEc!McxnC`Mh4Q5lC2d)upCLq|K;eElua`X-|y*Mc{TbwxXoau
zr(c`7PFa~cpkl}TDGA+qF6UDuzp1ImP_9N-u*>!n&t&Pha$wX&D}*@E;Qr$T@Yjkn
zwezMF5VU91V?5Fs)&<ilcDklu-=(kVyipQ}f?pgM(PG&X+$fPl-xe+%>E$m1A#<|S
zG*TTtQt|$m`|0!8MhGzqtVq?Jxp^LkW|q2APq64`ZwYX5YNb#Nx!o|T2&8sHmX-^A
z%BamaKj-)1W?<+~rmFXIabGW?^`cy`b3vuP8FyKQzlL3?m@iWCVW){cwbU79rFcXa
zpAX$wC9o%KO^BF$S>;94#SpVq44(W_*l7$CTII9tel6S-Y{N0C3d1rl&Sfl%)oL#y
z?0|Y4j9eGptrY!Ap@-hwdzRrN+Z{Yd5k-M86Xtaw5+(=7`;f1uuU9mi#T&M!V7_vf
zUM2AZkBtU{{mavh;{15_8$h*C<Gd2mC+2;$`E#o%48pv9dbmLsh91#(s2Mf#<{N#P
zzs9mVeE)?u?zYb1^*Mj%qE#IqBc$N6ztTh4o8r??Cs4!<N&sPvpq064YJWca>nj<X
z8<++M5VfLpU#`sDdaG%QJ%4iswhcboSK)~|X}uzgieQ^UXA%8k2$WtZ+qx22;cKVA
ztM2Iga(`e?u$rWmx@D`QK+Ww}?(}PJhT(O(-z~=rwrfoKg`#hLRwQN}^NZ`ff?YZs
zuqmz&vSV#;oYc$KX+K6}J`2?0>0<gIzEb?j=JVz?<*51cxJ#0+919%1dFY02tM^Jw
zHJ9#HfG!k=I3NvOVTJnaU0fNijz!urQy{9Vdv^%&Ab9oI8|Mqyt=XjOUtlhehqb~~
zK8i{uza)ddfDg6seN6Q^u?4A*>#eFXT(WA{VK{lVeRrf1l_}ONjdS?sFhbk47@j-3
z2XKWtlx!1$>}R)!s9*^g_FOqJ+EEI22T8WMi%&%4*LkbW$TGSkKPaZmCI93c>#F29
zUfh5uj584<wFT<Lij`mBYxti8u*PmThsO+e<@bwtesvH!!~&(0UlSQgdfoi%OQD~l
zP{hml5>(2DDa7G?KiI$ttl+}!lpb^bdN<dUOkXz~v$Lb@u%gAU6QuZxOI$89P_R)x
z0v)KqA4I5uSCQM}E9K4YY&6g;f1|Rve^SpjKO<b%dNT2A3Ml@2**rD^IXqx4eX_Ua
zZsN#*84JQB1eL+3KG2HY!?lDRO{3X&-6|#C^P2zhx7nI#(FyLM(t!5kq(92Mo9P&V
zHx~u7hYbnuLT~1#buUgLUJxEckpxqvhf%a6L&)RH%%s6_=H`*xjeD0pDy12B9mrwC
zhVB@3Qc!U|Bw9v8vdN{ce7K;30qDlyHC-1!peHb4b_mQoTHDW7?;R>nBpLn0`*1u+
zp<glAN<-jM%9-y~1m7lEkG7beoQGLa8)eb(?ugE6^c;%|$b-9qS<k$1Y><+CC(#sK
za#cGb_a;T$(X46E4k)dTgXh9zDu2|50Rv2B*Of{oTS&@4Ul~~Wu<MhQ5GB@2<x6Kq
zkkJdmsZSczZ-R@#3};^V>P!#udj^-~RJkr%SN*>Kjf6Q0*Kcx_k9AAv1z5!!t|KoE
zO#?)@0Wum_+){lQD9G#Gk?j^ykBj)GI<7X1udZ?LJTw7mD@CAj`kL81QWE!t1<`t3
z@Ku!8Ias(07bo;GUC3A|k#ji^D}=q1iFG*@taINBu(V|@*I>DwZgobD)!D}R7131j
zjQW6lli^s$Flj(icEBQ(kw(wszx<ehvc*=zl30JvRfKbda(OUM-bc$4%Var`pcrmv
z`?+##wef9Mjr1z>neKD8^wRnfMD#lA4SKul^g%I!Sd?;l35jX94*_4#=&Rh>Sn;|(
z`W04V_$gc<Js^8`9#>MX0Y~Fr_YE}$3|>zc6ULQm@{hs)%@I51&Hg_x7;xz5j&Dcr
zu~<Fdt{L$0EqOTspq_J_I_xO0gpl%l89hIgrzKB~r?ePH?{Q`cZn>_><*}Xn_ekDa
z0}UZmVL?oWIQ{2j5~5^?ReZ~HL()%)kqV*P=?mP}gVT?1l}~qND<@Dg>30GMg6R2b
zFBc_)D6cM?<19kl{1zFpZpm1_63*&-^Lu4DQJd)Zw=E-s@u)~3!X8S2CAxskq()db
z{u}?HF8!Ee2NUl0n4tfR%+q1OP%J?~6DZK6j(#VMfU6)C7)F=|;K<8^tT<SrC%l(C
za3hHQVricglU)!Y3b{S^i#1-X#QcnHCM^nok2pMJcFXpJO_Rp-KyNj+<=1|tpoUqQ
z1$gi=_#(s*VZEdluEkj(t7e)H_&OKDG=FjuLw)mAL~r@;&J~?eYQHLS7()D2Y8&-<
zc2%B6x<SHN404uZXTkfChjo#(!T$9#jOV3J<B@)kiZ;qGbr#@q>h5;R8y!aVf-A+7
zQdPcr1;Ripao2bg{})3-2<=-aQv~X!b(5M8|NF^l_t`CvA1O9?4q9^cEDs_=+T@F<
zFh6<7`)3)2obv{2^OEdRmBupF{Qdr_%~r_CUqcoE1=-jg1BP2VoIxGHH2v=SjWm?{
zls;?Sg(`I>FE+_{8nQK3zsspUJjkgIJMZO6>*%v+kSt0&z~U%}(|_eK&b?e8=zVwl
z1RlI}YuAmT;<|*}<~2O460OI4w&(_J+aSU#^BP4Z6?xNv27mu5(C6SEAGiV&Q>Zun
zc2nGkPJVDGbX4-x^bjAmz0owIaPV%u8scTWm9FR~L9FGcZ@mGRDy9X~;q;i(lM$=h
zF&nX|a=G*6ORD+HZSa#7ih68+V+nI^fb@R|_BJA7GRUg=)Bu$6^I|xo1Bn^~?2zA`
zBKiFN&f52Vx=kOQw=auvr(0hd%TIKkPN#`z7sdNPP&Xr9s#@khlx*pGp%qh$nrouv
zn?ms(glF29^vt$0FOY*!AsqGl0yy}PqB#N+HKQ#hifa6*3t5J*E#w&wVN$Z2y>wKs
z2`@Ba3s0eRviG_xbke2t-J7fp6=dQT6oV<_gUUGbiM5`+#^OlTJST!<lVk*Yjps5K
zU#l-|;j#l_NmYhD$_dR+kTvUYCaLO{b!+_!59*r+V7W3TModGl)gRsfEZLz_dwTTT
zrzQos|8~Qh(A@vm1WIF&r+1W4I_N7Op2OYq-(n4VEEU2e(VNDFghwKWDoj!Syt#Wj
z_xOaG*bLu2d1m?szjsXMVOi&kgptxef>CXh6bNvxEzbuv+jaPUCWfKs`FhaUQV!~C
zvIVDkpazTx4xjOg%NNz-X}}-Ls`|voS~gKgS_=kFy*|Ht)Bik@Jbc{I{NybPEAGj=
zoZyfQZi>*DP{Mj20%yKVDq=+kcB@0C4PC|{1N8*K5$4OCaN@j7PJJRSjBahuv$#t%
zpPZ6#G%g7qZy6pF-qnC^6JVT-wjg%Eqv{GKEIUzBg;=}#*|2%^L86`@JL7@QcvVk5
z5Y3qxX<M#Y6pl3O@``#@Q%dL2n<iQQq39#q7ZPMV+KW3YX^zEEm1oo!w#f!2wVy=4
zm{Y7TC8%llYs#A>Vp@>IM*sy$Sfhju<y^eth?{_uw&Q5fQHH;qzhXo_Q3i^2z^-_B
zxw13?<CrlbFcZ3>8W^=8dK0$}-JUcv6S{ZVAiMNYvKPx0BExh=f*PQ95C~w>DA4`5
zawEOI_y>5;TPm5*^Ztr;-<dC#*q9{T3M!{{O9f3SA`A7zSBS1|y%z{UZkq#@>JKls
zUw^KB#aG)NF301oKEB`h_;^vTQt#e&{}Ir$$-u1`<BA5Q*3V9IZ_%!y_@N<({(8=V
zGR62R+m<z>@k94_0t`s!-}G`d=)vn$s;}nC33xtm2ME;d1bva~*7ibbEg6xO6%l0&
zzba5?!-<yNnVX{2d$d8NcU&Ms?nLut$$t#k=5(Tm?E59US8y9iDB=>TMP+X<7z`g#
zs(BrpLza)@j0`$iC2}U;=-*Y?Nm}R-n`H*^y2NvH*bEyNaZ*bCEUeF4U0+f|FEs`q
ziLQ43ct^yr+MmbEFp=+IyDBfbDk7yh+-6Soj-~SO(+Nq;2I~j$1i3d4VvRYKuT#L6
z@C!g-p)mHQFpsUJd}OYa&D81;t@r6?UVWFPLGv~O4ZK)Kq2Sst-Iv;wIH#p=ToVkG
z_j3<Q*9p3bV2OU5cIW@X*(SzdO0iZ3aRgG)!~CLVCG8fPnr6Wq^dY8KhW9)HvcQ?2
z-hV@X_8R1xr|t)iP5(=<;U7lJcwJo_tIc5j+H1mf&m`T6OA4QKsM~G7_6z})T1j!%
z?px8!W(#Q5P;vSiaR@%EZHcHk*(L0Aw!zP85WH7obSHqjI8-i9P7x-U#Y>tUDyE@p
z^6TkGrOBcKOWMJK_}dFLWiOjso@f5tdAkelP?%0}T*G3F#21A#-6nVrpig~`4*m2h
zl$#f+81P1Ga;7_u)T3N$GAX;(orIQ`Xfk3*RN$+*BbJ#=9Xk7-W_yQx2}U5SJ<8K-
zG{cqZqNz*Y$tttc7lYpb8k%hFE1z_@eo5cQ(2p)`w7iWKpF#ywjeqCa<qh~PIMv}l
zDhgi)n}YHH>k)+K{;S^Sb@Wun6e}zTn_!A@w~N=u2sQAUCKc&d{isbtKD#%u44*d2
zvJ*wYHU;k6cRyV4zf&Nz16rx`xj|1;;1Z+=#=y<4GN0o^EiR3BrQcN5a$q|9sOFpU
zA($0pGmaEV+fdY8$CYWO<(!HpX^`Tk^HjCtd*mY}2qEK387H;2!}pH<U_BM+(KNZ~
z*jiixWRv9=??TSL9bBm>5lt}+As?h5Je}Yw<pBu_J7d6IJ9TfPAj=-7i(QrS(M@Ly
zw14*2rQp0je}4vCOYXzlA-V(){TouZkbLo^4BDx08dm~N+ME9{DnPSi&iqDhRv*U_
z(>uP2D(K`c?JYn|=J1qZADYNemt%&&qhwM#wGG&W+cbBLO<H`zD(g(^kok*Z(HN5q
z9^=KcU|X4N>MpR6WmZzT*@0y2{Jr~~Qser@p8n%{o#6^_lGq-0INT3;?$2xgxxps?
z=H+hP;1c;|)vh<bsl*2`cpP{cg?&o{mdUZ?pPBHfc<_Wg!ui*qJD2Hmp9fe6ka)54
zem#R1vP$<0kE5I$r5}4>FEGTdWKi%_X1}CWI_wp*S22RT&(%rQTz~UUU{iddH1`$X
zMZ6~SJ71xGjqtuAxJFCR4PWyD?2f$ij|oyRYKusZg;FL@P9i0o2MO<bbU3gg5poiE
z$pr}40eKr5)ar1US6b`Q-p@@(gU*LH>HbP;ta~#6y8C~V&A*9JQbs2f0T-I71no@4
zoa}9f<XscdixIuWSI?HW74%2qf&?J_B)($}PHYuUy)ji>cPkVKA(t$Y%aNBF3^$-*
zx~l<1jb_wQsv+vVGzwXw{vyNs0d~M%>W*A62@I_F(A8n6;rPL2o+v;I(-pJ`j1Gkh
z^5ncEvp8d#*?g8}OLzz<D#L5u-3$x7+q-?&8<3CZ`8-SRV2s*(@_!HeIe*-!CTk}g
zeaUHHq#h1fd0N*-tN(Z~hlE#9PCX1sq4KaDMfb)$S#IA}qH)BSC~9{1tzD@==Jk?7
zy3vsjAsg<eg{Iq0aFmh<?GN-|^JI%{7ABDX#S>mZ3YNetJw-;xkRJ*=C}k>fljP9a
zx{u0l@_>!^2?H*EXI$wJO#)b73jCxpj*E$S{khH7CC&#G$x;T%Zak<oDuqlanNbqX
z=Cj2jG@k^bo<6U>Fm#_5u~lT@I*;|ks8SyeXqq<B3i<FW>?skv>9$b#InOv+p`gOZ
zfsZ3OIi@_2PEpA<REaD#-W<e(os?8O1D;#%kqNT&EihyZm)-dmuo|#l{yRR=iVsP%
zy~`j6Z`@Cn9ati-+za~H@q_ieU9MV+LZoD%6U%hx);M#T!0>XF87Xe7I4UU7Ly9ek
z@GM>R7?5-t|AXTE*hEHViF93k9G+GQUMJg97C-}sjOV7^LoQb{i<6+EC(TqCHitj^
zw7V_k2`IgMOs65C`WQF!C%e*JC;{zCKe6rJ2SGP<yu(^42E`djE&;%y+&{t)q7{h%
z;BYk-|CxEuwtryMp?~kWp*_@B|F4@Q$~8fqbCCDU6!>U1)#Aze{-PL6Xk<Y&t#&+>
z-Qa(t7(PCP0vDP*6#|v}cvbEw&3pU3ZI0s7%}VV!XrL;I-r6d4nJI-Wa4uhqnvaCN
z1>+r<CVqCMFbkd;p++KH&mvyK9H+(j>U5MnGOk_y-5VhpPMOs^-@iWCOh6GKT}-ff
z_;|U6wuN~V+>m-f)lG7D9em_(|Kylc#3P{H8ow_Pqa@^HG5q$nNt>umsm$#;#6Zl-
z5frh`8pnE9Iu4@8`x;Z+khF!=2od_;M?FvZ`u&=QhHk)HsPe6u5dz7@eH-c+q8pSY
z96|UvFrR18p59dVnd^I@=YjCE=|6sAuiiNm3WcMB^)zqt>b_#nB#3u0!=X&4AS#SA
zXZe&J+RK)%!X8zP^Zp&tb0UqN1+j%1w?Gnh#vS-g6JF>CxG2R|*s5veB{VC+*vKNv
z%%r2{Vz}6kriiA_qDho*?T?OM7BBOy%;y#qgX_1ORRq2ZBWafX_y2(OI*T9VolNKc
zNc1d40s=m-@!Q+VjOfiZ47}qTbIW6YC;=a*33o3c&Fe66T-$LRh>T@A*;Yxsy4v+i
zEWZQOa0$2{wpXG#!-QF{EG(jiY-!j(_rSlTEj*;O&2r5~b=b5pj0cx9C(s(_Hii1$
zuSUFAn(N|^6^$<YD6E5(%>*P4phDT79~uv_Wize<FWqOuDI>Q%1!=yE{!u_9u=)2a
zQ}0!B$hEMX{&0J|*!$sA_x|&i+@`=}XzD(+{y!xXC@0j}fum*|iWkdDR%%kkD@YZM
zQs74s^G2>vvTAdGV2Fy!d1_)|Lrrv(h(ZzmX^1^VfVNm!PsS?n+b7R%x}!RZluAny
zI=j5EhNQlX%52^#5QJ<xkMUq&<TU6(h<>C<h#<eu0J9>0-Yw3WbDGT04k&`T8|HM1
zRxA@^m>H+s6lZ0eMy=}xt&-ZB^E-Nl_uwQDKatw~Zg#6a4D~7(Bqq*Sd=^kr(rd=?
z@R2IBefW+c5M(;TWeq?;EKt0O&^ZyZ$3A!m*bvi)o8#rOpPG(PwtB%OOw^N<?MvjY
ztgi|C_u)ag(d&F*^1={Eis4UIQ&7E^%O>wGIAdF2<c@_Ie)ue-zotA-AIJKV*ri3$
zJ5f2_MPgi2(TI`Yq=oQa4mu;osVP&Ida6&X5B@yAsS|=r$bP`M&edaMR(AL-g2^+)
z?Mv*1hN4~&*+1mTG(euzQZusCKl_I~N%NOH`M?_!3jXxG(EZ;%j||b+_xQMyp?~vM
z{)sdHC<H!gE|O->9QltKC@k01K>zpwGNyaN!r@3+>iB|CM^?@f4#MUmvnzCmwA{?4
z1zQO~Eqs#=CJ5~YbLDTcM38aG_ao5f9(a)etOX*$Ft{kj%G*uSlVwA$lkH*YjwfCg
zD!B&YvU?_H<G`a^?89Zkrf&%a31=-A*`LworPA>Gf0>Mq_{j6d`#$)a3y8{vYbe`{
z+gm#sb5s>A;O~6;VZJGf9W46kWSM@8x82ZL=|Lf3t`rkOan+KRNUEp}a|vuV=CRLQ
zG0_v;Z3F<w_?eHDVXrL1$WMu4b12<qufKit@JK9ekvE7?{YlY3hKmQ@<bQvs<8I_f
zOXsV~hu?ZkcZ~Vb1HX)s6+(YC3+v$^)^r#mZS%izyXy3yUIQvw(`uTQQw*jk9-@0t
zck(-#LFA{D{uf3drs=BWcooQJFt&}KNf=cYr>?8?2650QzYX-VI{gErct_Lj#vQa^
z9Nb!|z!@gP!O{oj2AR7CFPrd98TeRvFF;3{=lnSNL34lm3!;JtfP)q55k)JG#Yl9H
zz4sCe`3Cw2>ovaDfbL_po2+5yx#Zr5!H>>y4rx9w%`@dPCoV@(oANTQRSwaxi|6G(
zhwjSI!>Rfj7qFw$W&K}dA>3G>sw={vJGiFubEme=*-gxZM}Q0_y&f!FnNuL@Sf4t2
zyn8koZ7Lw0(5>*-f;i!<3*POYS0FrPd5Ws6;!foZS>P#K$;Ij9xAaXv+m{D@QIJ}R
zw3@8TN5ABwiER74OF#siYSpIykEuIUX^zGMgOa*#<wiCKPSR8JaK(vIRYgJyt8Ol>
zDQAa^9~UcI6D^sQ&u5zL%C|-BOa8deJq?~(-_my9r}bNwdVdy?EVOYkXWslutX$&i
zJ_*J1$-_!l)nsAU^7@ozqvX@4(MQ_WnAhEx;NR#wj6qEh?*<i^hl2T>kVfxzzG5uQ
zKHc|-I8mVCyv%z(l&*{{D%;Ti;&8gE{;;rw>4B<YiQpO7x_$#3l2MCit}yJ%a??_h
z%%8xY@;r$ihXyPZDG~7aRL|e<?J<5adIR7&-v*G4TyBJI3FhdNz`2eAmj!lWhXZH~
zVG8bYn*i$MT7?4SD!_N5RbXfQW9(hLimX?XdEM*sh+q%i^&iQE-r|rip9Oq0Xz6ea
zr`qXk=*Y_4eyr0ve)CXYx+|QNl#<b<Nmmv7qk<-PtYd3gOQ!W>ykhKJFk)-GQUSc|
z>q7vXw>p*?N(3_prai^XpueWQiG;B}yjVbJ7${wt!mP2I3x2)7^SOe`Heja8z)EGb
z@`--c^DfroKUMih4K`R+H1nA@;YIc@)0<2S9DH{wlkc8oMbcLh=oNsxYY5AfxOF<e
z@00`N!f;@`_|N%Q@0Av1kRN{6BRqH-_3KzEKVi?jSn{{M-@>Qtlpbj6atd==GKdu)
z)$()g&fkU{UrOcQT&-lRPLvo)2!MOSRo?PDDjGQ=J13dt;dD4|4sS@V5B|sL`cH34
zjr;c*(y!8PI|Ju}69xqM<1L@F*o=oc>Ri40U2WCsSZNl(H&t&;)!a0*-i#2!%u4M+
zi?j_`EV22#>9~Qw{L>LAxQ`*v5L0n3mE8WyqMk-1%-*yJp1AwBKRqNDJ=0*oN8A4i
zp;i;!4b2q;SUz)|Kel1gzfXd{`l4<;n8zyl4N;T{@8!ffmN{CoR=imsu(v?jNyt;V
zI!ws(518*G-@Q%f1bk0b%AWa*S`C{SpO|cel~gmKMG-+y!QE#qw?y%kE#3aV!F-92
zK7L_7c%?|WvUr*6umNH9c=eSC7L<Qm>mOCVb+0if!vInkDp&B;W|Tj~8L?F{I5kz{
ztRm6nb#(bT-DqW^V5FVz!YfZ=VXtLlt+4C^WYUAqeTZK%RCe}oYIgd2@2tKC75B(L
z>F~a!A4Kea^GlyPjh5Pd&K?8}xP)DGf`r#7Ge;%zmu{U)DZ_vRitT3jQm@+VZ<z0i
z22FyC%(3g#^|SQ&F-o2#U*=Nty`No0&XYa%TOw!SrAvS)&bJiu{r)#s@1kThyE$I$
zi<i1AgqTN)ORx3+JW~9>kCc0dcC!MUDvwrz3N;?^#35>3)+oaoRo()tjwMgHR=n^2
z#X^c4$~2jlO8(<(=yUF%j?1$@f5@fl);&*w5|*LTnuRhh0<anjd@#sBdc(nsruw6}
zTXW>Ct;EaU@2x7IC=xRV+}owB2D^*SjYe7+w&mA^M;7G9H0y(xX0v7e=vC(QHym%w
zmESGa{`A@89fXH#;?kJP$#fd{iO3T41`?6l;;HBE_?JW+E{Qn$SmP^DMMuEKoZ#yu
z5__+->UXnldmb<XGt>$(EcQlI3!6q$=07;|Eey__usm?d5kUQFj<zPh<!OBmQ-YP8
z-+B_MK$(k&0DD8<x8RRx6`=}6Qi%ay?*s=OXG5x{#V_dwTlXrqp_>BejuZRyid~l3
z)>$vcimcep(s7h}Sc&5ilhu=blJ`rsi$!Xg_CL|KVG@f+gF0j6pWNTg6g~Mdatlc@
zf98bl370s}ZUYJxWI;+OxYsN*%<lcWM!=krq=9C@kg67cEpj;aMC31vEteM$#2fZ!
zD^t0*U$?3T3yd&s(WT|8vc8gj0C<#<!rV*Kbr&xFygP&;B*;1}G#0DxN1g9zyj5tW
zMMg9lcZXVhj-m1ToO)_+*gZ{`Qh`mVBP2KKE@VkWbn{K@VZ{^wF=}=vUG*%7SHq<N
zaLZ-c-~^bn>eIuDWf|sR&h*uxu&#b`{~@-a3^s*Wbwvqm?(dqmLv)oDihV5kCt2xs
z{gfrStJ43#83)qZ0Z=eW<LdH4o;{YF>0ULD)+fBziaBK$(Q}TE(Q!b6wLJTT&4afi
zGpQ!fhRn%|+2iP%A?CHU4*zrO_F4Y8{g7)%ao^F~{we~T``XAfNwL|%i<jZSa>=nP
zKAVreZ@qzfn&WfbrKV$cjo26|)z1y>b6jrG3A20p^sB8U1;WYX+4?%ThBRCe1s4BX
z5*vV$7{icUu$}qGot7(%hiuLLGeO24i%TxvD0#lqLQ9-GAcZGtB>zzc?>4ZJu?7E!
zb;#=4ND@abv4=wAzL3939yrmumHn4>_z`wB=A=#tk@fAOb9=E;E^ATjWW)n@Tci(g
zE#Z6eM!{^+hh<ul36#pg_fWKBEom&8gomrS^11Ti>S|Xn{?ZN)ZzNCrLpwZHG*;X3
z2-$hbnHYt^BOP;QI8kuN#?wX3HEm*6=}NH8`MaKbOO0cEB3;4wbH9Im_$Kgsg;@in
z&OtIsc~86|^h@Yj%@>uqKsb6c2g!7E+t=_j-p@p^WtVNRB1Eqw{xM8`Yd1wlvUj#!
zn%bQxeVS!68JRksa&{1<W;jZ2C@;kv)S*Mt;3Qphnq*Qe4dpA6-VRfLI7&4iyvj5N
z|2*yXg)K;t1*a)XKAGHp^OpJag}@YDz4?Jo7hbGf5nZTBVk}?0`q<vH?em&jNR6(~
z-))m{N6#&<2DkEFym+K9>TtcskmX8X0Cr;FYCGBg>%AD4VWJ+0M*shBX26qU#kRHo
z`{dN|Zh)lsjKL~X48?-!pR(3>Kkrg^2{LkY*F%uR7|XOU9YJ`GHIQUeGWKqsDA#D~
z`<D-etbytRO|c%=7GSOBjpX0k<)xQOH9iubcXR%}*;!oNWBmNFPDUYlqx!#Si(Z_;
zN?g!>jaWe*q|}m50>=}HlO_RAa^fvOTV%R--X&^3+P-my=6l*cLCXh=xJLW;woo$z
ztL&czYUxtjRSg$~8$)>s-oOGs4Ir!Azkel<K2LZSxb3a)-%B?6oQ_ZyYeaDGu&lTx
zXtu*y+U?5oY<9bJ6!zopr&<hW<0V(s3avMOgP4?Ca5DN52@=xf&qXP0itoa&lf>g)
zp7r=oKZ!jT8Y?Tg*kDdud@5`+U-iyodir#!@V)t@=}wIOAHHL9*U^}o_U~u?-`>0N
z3T{_z`PnDCR=FD9b5X90HEC!Opz~Yf>`9t~d7i}oJj!ABc(XS}tR!{k%x>SeO8H8a
zF{<@fvZ#BCz@_}x7a9t99;<DH(@ecn8JzhpUg#3GsW4wrQu>t37311A6*_M@OV)q0
z0KRRF%+6exoH1ATY~3T2Y>lLw^=+C)k41~{S2I_2>|WmaruuyUXNVPICclP{(4M9n
zNXZ`02(Lfua3`wy-Jd^QPL)pPdFf-d>OOrrxy{@ypyoGj=yRo@Jl!WNKb<S7Mzq{p
zkN%nJ(XyJ&Kl`x3{$i>^?b8VZ+w?@W40j%T5aB2)*iC$lY)zVd{7cvgRlLPf-S~Fa
zICBl5u?O6sH$+9=sC;Crrl{dQY<-M?P%{osbD}`tSq(($s&RU+^;h6QM(v?j(aQa8
zFBfh~>(vwGUc%db84}FDMYkHPt4Ex^Ctr@cxbOPX)#w@bXb#e7)~`sTM<COEf$|Bx
z6tkYE7mF?aF3hLGe9(HdElZV@r^*;@?$rmIZ&_}4yAO@a{riGvTaUS_H`)#9GJoJ%
zwlf2{Hdfo?e=P_8^&bVySP;Z?+-Uth_Rqx;i3B`g8qW#azAzbOicYx4&tqIlwSdyy
zzLdSa@Rc7~o+vuF6c!KR5%0PN#0rbaQi(oWsrD70TlNbs{c{#aJx|T?e7-zn9bw}B
zgJp!kVy9BVlbh0{N@~9rWgqmmT7w{x?je>kd{S-ArSN;U=H92xF<m6^3GP4T5hhUr
zwiJha>G__9%*;*Z1qwSv=*OGP|N00_OTkA_<o;(Sa0!Tltd4<PCqxMrExY%8y6o#W
zmnBPOyZ{3wwKJCw=0Z=KaTa?R$h3rbZ|`&7r4*p$B9L;v^8d3>;?Lavr9`Q=<F_{+
z(^53Q)M+F3B$N8D`}nN*P$4Yq(J!grevjG6`+-_$dsCw4+Q~tB%lX2xy*yn0o_p8D
zyXtqG4N0PoHf~P`Q)_3fm6w8PNb5Du&$d=D(dohNh}N$TswM0zUWNTXN}orUShh(p
zf2Z2pW&Zu^oI&$}?{*Ri$z{hMp_?oVOCQoH5O3L>w`Es2NG7D~*A-*<)GBtqPkp&b
zn76h^D4>!r@44ugT%FM1?$jwyba#6q{nXjf#|SUB#a(J@>t2Z6<rloa;px2%`%xC9
z{s)I#)x8a;2G6z|w;%^0dA@RbcCWj=EN=3Eodl-PCqJ<(Og}lh@@_brY}kINZk=nR
zU8nG90m1fP>FGUISL?-X%r#U;_gGlUyJD!Fm`Bf!t-dqo%fm8>Mt>yfQZmR=&Og0$
zP`mbO`E1f}*4KUYIA~qX{Ax++81J>#k)#NfVkP<MxGI;W{l(ySXGm2x51#AF<_veE
ziU0q0bdj(A#zAz-Bzyl;#oQzX;N{0-#O3oha4Z8r03-c;_#I%q@Myd>XOhZXgtc+`
zW*$eW^T(sgIC0&itQfcK3We+_-^`9j2wYq7A-ek9&(Z5UT<K+5<P6>G={~8*9um6(
z*?~vF7SJUT4_mXV8x<IQXPW3<7PVlK$<#ZakBSzz7;=UHzH?IjZR#Gtcdl-*dAuqm
zg8e;(D->Q0q_wsK@pX&QcQ5?c9e}5~&8hl<S_!tl;_NGs#(3wPYg?be=ax%j<_sQl
z-a_=6Lv`=DHpFnk-4WNTV3FV$#yHc-$r`W}utn~2X8Yu8-z`Hs9tvU9Z77LD6JTrm
zO?Xsk%tp(JM1?r?cntnZksQQSkA#L`cLKuGPZp&)qY~>vrPf-iBq^3Gt2!IVW^JNp
z?8~u}YwVf6o#6ji#*5XzKl`CZ@J26RPx*4YTJc>`?O_Syb?2%rugTi<7;*mhgz4#d
z=?@R}-5wstBkr9M@!EtJiVOZf;pw&igs0=yLX>Wt&(F+$)A?NY;Y}BGE2y`SH9N((
zW86>Z%23R!xZe}xv!qC0m5>-5Kbod$NRgmOR5$vQ#f>;@*HV^zKX&DC>1TveFAZ#v
z5)6m=&8Kcj&ZJyELrFCszwbp#X)1$sEJx&y%vf`T^M3omuHUV^oHX{4w!n@?81D4<
ztM8-J$LHU>4j6*wnmk93?utKCN!!&jb_cDjHj2IMlXN<AU~Q{dKW^SE=st*kwBA#(
z-^WhoGGCnNUQvq|&Q03erFQ>q+@oaMDHC?qLP5uoO^TwAJ?*@|PuJ@XdL-w)nl`#6
zj?6(NU>i$VOg}i1;P9iD*0o<yR*8fkwxYRw@RAvZ5ZMR*ml<@`#^=i`8Mepwhv!Oz
z{ud}8|2LqCn%Kk{6CD~R6N-tHz9#7?YAgRS1k=mR)TgXtW<O|rBFJ$0o{&4Xge`Rr
zZc)HQ6P1j>Wq=2X5e?Dx2fJL`fu~u8u=UUTrOZC?8&rOsY<o|hbT0}p4J$0TxqKmB
zj6!5$n?;dXj`0orL&#L;0)$LK?S`^=hp(Rt?wv$laI+G*-P?((|0?ZuLIlIi%i<nm
z=4E_lZ;lv2s9Uh!(dPJ_IE47I!}r6t5}=Eo4#FpiBxrmhT{GeOe3}I<{l{S?EVQ|x
zXRMihM$>du)$LIKgO9Y(-yC&baFg1co2}j&{$dssep9!?f_H*-u9E7Ck@f<N5z))X
z{?}<O){?idFaNwX_Hg=Pb>qQnx5Idar|3f4io-?@HUZav=cq@$-IK1V^;u%`XvKTK
z{?+wX;@(tFgb?%3)5g@ZDrW<q?eW+1T;Fu!0pY3ioUsEBVy!p#O#jtM)E6Q{E|1K1
z?OEHex3`B&w`m)46nL*z*xXC#V_tdN8H~jQ`#fySwT_~-WeGpOL`kzxHh;tfsmG^v
z()fNmB5R{6Vpt|0U+c3-Dbsi$%%DWD%i6F$V4VYK4((Gua>F$NbxuErrO8`E4t%&j
z7r!Ty-9a^IA@T2>yuJFd?iE@u1oBc|wAR$@Fw<dz)PD&$NbuQ?vwADZB6;Od<a-+p
zmtQ7m<&bY!%1HMbNsiBySlq)9{@_P@WBl`~IdvsMuC}OsKl!fJr?WQICrI*fV-NJ7
z>8JU{%lBu!<kZK($hY+W8TlrGBtf7{j*KWf`3FS)aSsYU;N{di45cjUFYtg{dfSh8
z@LZ^unHAZHACQEtCSyUi-l_ssl*uMP*ui~{nZXve8^^&WUDDiEDb`otfwZ?(sOBzR
zi3USEGIi6p@@0Ivoc<SQGEF~j`2~}YWu^fSWvDb#We%SZC!r%I^InljaW5)foY*et
zxW<XBYF&0kjj;p~yY8)&v-HyP(Kg`<V`Mx7v_cQOSs4=MnjU%|tj_ykbe8}$TOC2g
zMew?#Ig}&{r`b}`czfh|!~LDq8O@Jz<W~oS8wSb46CqWNvDN20b;+NIVRaOj{$f|n
zd<Xq^*xOh=3TAD5-A^yALhMoWPH$%iOu9S%<W1HMnlk)Zh?!W<HSr&H_IE$N@36J9
zl}#R&gK+QIa!i?hYc(nOf=GyK`eo)7VS7WxGNr)CjVANE{MI0{h)8N5*$@Y9S{#)|
za8XA<WvL-uBOCTim2DXa_Zdd9ZYsW^N@$Rxikfa_?2Mt;4_BoVeuLA-ocEL0+p;rt
zX1v?%YL7l;koD7j^1C<u_4x=ZCRKecp(edr%Rw?n8}zQ=t!gvyKPMwLT0!UDPmoG^
z^TwuI7_Z6vhij!v*0Ge33-0^PF&jV4t*3fgSJr#?i)?0|ba)IHC*K?}W^A)89QLjy
zfiwuZVAq851)9mx(WFkgy6K#+cAMbJ#`5`ra7985DL=PG<>5-KX6^oXxC8}0lG1-0
zs7W6KHR*rfyw6GhX4pgrRvw!Eb6_pQ!GYC%S4UJ+{-Xh=3t~G@(tWjHXIf1lR2uB+
z-;F}!JVn51ssh{?-&o5t@dUv#`0ayZ?uio|1WJl7na^O09`KO|pAL!l{V49`Tns7x
zSl_?GvCa<4Kva_r97m=eFRW4HPNGPLt{Mm^KtfIBtY>keg?oNfwIlUiSRFgXERh#&
zUX{<+wyK_Ax59uPZ~hDPSn;_8Qvg<aA8&&K@T-l^F<2i}W1`i$;n~ErROgl4{{Gwi
z$B)vY-rm5FLjNmT{Rb`I^yHt0Wq$9SGWB6-RL<^H#7CK-xLTz=pu|UCcQ;S_mVAGD
zlkG#Jn9FvR`-SKM)3ttPHV>o;ay)5H7B7^_fqn~R0ad9}m)gCEh}lw)b6I*bcy+EK
zwl!$2Jm^CVOPz4Lrr*{F`6pRj#(h$&sz#_7@jx`*CiWu1X7Y=jCTB~f%4qb3mfUUS
zGe2}r__Xn*r}tOUH97EJ&>#OHmy~8{d-==j^w(#}yp6P<ozlAqhJ_DpCpC^ks>-c!
z)Jy1od;RTC+L?+U=9-SUY=d9FT|A~=hHJrN(a!E?Vk_GvsSpwknZAbks@c0&%qt0o
zvdF%<a^E2uMb1NzyLsY%#Sfy@>3tGFwEFbrCT_1A?&uBb0x++-|En9wA_HZ5TVPM`
za+;M3;%QUhNmaY{KC$>Dsw&@D6T>7?3wq7)ENd;;{D*&8fC|=$c=aoP@)_0&Gdi#I
z7!J>uoM3wBG|V)~&^O`wW$v{#!#3Oi5;%suxSD-D-bf6r8;z9ytG|US!v)~d-R4}W
z=Ne#nhM!KBO}56P6+6c)N$7dn!EBcU*o!Z+oC<mVKibYRtjca}*Oa8RG|WkZ0SE}v
zAt2HvAc7zzAT8Y;N=zDQ38kdFMY=<}ySsbtf$v&xd~5CF`?L2i4*11<=6J@q$9-Sd
zbppY<ZN67_9V)V@kC!4;;jx97b~pI}`nE|X`+#;giEWX`fv1SzGmV>EwZ0f1qR~Yt
zKTg?g^dkH7J%r*;qFduq)k9$Bih}}i_`D}}?QZ=g@txq|$-JE#t0jrhigW#Qy+)Fq
z^{2@=SguQ^k^a<<Q>%13SJRR|+9k;WX+0RZ*LL$*i_P{F<}tf{v!CLz^y?Ut`zx(u
z)W%d%d#B#RpA=WO{SBhWqJYaL`wFQj4IY0j#?Xw7PHEC{d>%?tRT4-D41Is1*6rHj
zLZRPQsjetSa}0A0c%BMdm$-CWBGT!~#6pr>vev%2a||b0U#?4O#NC+YZ?9@deD?N4
z!vD%RkS3&bKeTosFkV(hyW-rlXZoW{%DwWCPiRds_sud7mnancUp%*_I@el#VopMM
zpS21eK}4HPS^Z1b;_O>fE|-rnl2phGUUAosh|hET2TQTjEvGtBhATKT^KYN(9ST01
zZeSGJ%y&Q92m_#XW^qi+=z)~a^tNjZ0cQ!i7V5cxTV;)oq!be<F76BCA&;Wv9RP{j
z8tuThm*t8A+Ct~6{qG9j-{M15H(d)hKt@$gwCQB|r!|~`rWcUh3#OkdmSES(<wK%%
z@NBAeI@BtnwJg}+nY_aGipB>sa{z3~zIhu&i~<oI_8H3If`2*k0OFA2g?ow$6yc8s
zx972_KMjg%Z!a2Cl{UrP3-?gQDPw|#brO3}uAi)767aHRDH+^t^!Dn6Da(?YbwAfF
z9KzxmPK`{yS0`?laS#CYLXPtxl%O!n5;I`)Mn4|1o(p(sH>TVt&7xTfx?i~B48W|?
zxwsYWJqLtfE|vN)LGDMiGa&wc=Uals19DQT*A7+zxcpP*TvltCK{2)$&6}_q&E#5!
zRc{D4C>1I<!iB3>XPj{y%B02qMO<I{^qaUY3U;30ynCw_nb-~=BzG^;5qhFvnU$RI
z56{~(;lL*r@<Nq26likqnMs5bm-8?1`=RT<cqMKbX7HQ1KGyNBap4XI{wlv%!{*7X
zJ$%mpLdAl2yeE;9GwNf#b)zl`qY@uA4kbKuacb&Z)spt=;IhG%;s`=%c5YKXJ7NrO
zxKg~mS~go{UPEEpSupdc0?^vf-x<wI;r|euPN}(`;defB)hxG@{WLj8ZTJ?sKJ{+e
zobndA9+4bn?!sCRUm&_gu2&w8H&ml(J5pQ3EG>edg112;C|fhxE_$&FE19E^1o6y|
zbdFaYlCI~OFxLo%i>bYFfve&sZ>cLc5qODk%Dp4v=|@<izy1~0{;%ET9<rDH_5?4a
zJ>gHs;j^~`X%aiYU86jp{QI)_UII{b_E2Y^c!=!1H5XoCI@4p0{E7HQJ_ExMW2pt7
zy3Y$&d7s#o+X|<@iNe!EL_5uX;bCAdHN{mJ(A1zCE)Oh<rP-f7d^J?r?_auzhhWQ^
z#Q0ppIVBGj?*}!KocTbw<WMq(KHD5KowQ*9Hs6JBtyeM-cK#yaUW(R}5i|q|v->Ex
zz(L@ZWdP!Bof7L>86BIKLV@F<ZQ7~{I|K;@D{OZ1!o@<e!E-jJ<)GqhJ&_q=mhyA*
zMngFt1qR)<4+ecT9NqTyiuuJmUBV%CBWyf3oTl%~Hl(HY`sc_Gr?xA4+;41@iIi5(
zrQp`;0=9nsN-R`G<LC~1p#dmV;dgc%S3=((=e|!(1z5T{63JFi`m{j7r$X-f)K)F^
zubP>ZB_kQ9^mrULBP*(l3PM4s=cVsLp$x8E`8y2Mft0h$<8)!jHg()mgfyL_F{Ex}
zZMlB8gCilo&ZOA<(?;v1xh!j7IeSO!9KL>H{kx8eL7M9T-(5Z80pDGUS$?T{&@QeN
zc`HhtBXBaIPop4COj|;2CUf|GHa?S4V=KK#Hu>B_?-p^psH=rONtWa^N=9S(&#aH%
z^@%Y(O5jqXm{Xb~n9%-iRDG{TFXX~n!U>ux+HglTTk#ipgAdA+8fWP4rO%y;ud`=9
zXw<a+JXio46dZ7}hluxeC`i~-9;DUD@{gaJamo8^4d|<sJyu*M1aWo!>mmUe*PN)f
zMZ#O}iR@nG>iUaL#Rwq3rM^o}1AD<nQKmx1kIL#Y(aasifX6<Bmc2C;<LpH|nyxUJ
z6QaC;dP2rw(Ed4&Gk2!**f0|Fwr93%J$P%QiR1?{7(x!ycB+>9Pq_&aaL0a96;31O
zuaXN!=4%Yq^fovpF<|CG8|;z{V@Cj79WrZ%ruYH{M%<%W2`UiEspSbI@0NBOcSreR
z#U$-Vf_4qRe)mF!em7>9akXP0+S<#y6xcSBZY~Ceb*JYpQ*ZjFqt$El$cQedz=|IX
z8jDwn#d$LY4XteFvQ#^s>q3V-YgL_0`x=H05nuK6RE;#NnX%8;Y_n+e-R2UyUicNJ
zQ)ukvvi3DVD}bA7O0`Xk%9cxFgvT~h?3$ap2Ce4lw}$!NhD!$;__mzA9T;rUE}es5
zA)j5Q*<Y^t7}3&uczE4}X7x404v}k2Lq*t_ld06$zv=2Xcus!145PFX)-)HL?CNkH
zKv&0YOt<SA0Ya_=_KoVr>8*ly0T9V;-cD?)QoDBe>NdRcA+(p#-taD4)*o>78(os$
zaP{-ej@vb(Wt{L&xccw0*B=Ujf7Q61*j^xvEgm{c>v7vR+7@Pf5y+f+uNOf8p>wPy
zfQ``^NL5CD$UjSV&!S_~bZvDCY034WG91s++0grV`SG}H#adfkw}0c+{rXpNgCbHz
zzfh=OBOnk=D)rev%s@9s`Q)0*qM2>=j9F#~_b~f4nWhl;)=PyXbmpUgb`P|a-DM1j
ze#jiYRrZ)LXtBanMqmAgW`j_O%3Dk95&~uAwbyMg9b@s#vA1n!_38B^8`)SYw<^u_
zYx30mZxXY%i=bt{ceom9fx81#6H<Y!1eYu3=Q6PR@2}~B=~90+#NM8+0Z==Clhkdl
zPEh?YeLqE>WKVF+b?hrtQ$T9!6-gwD*hZ=*KP0hCuhl8s)ql57GNtD=WZj_@t2Qb0
z6CKippJNK9X^;=A&v&^d%z3SH-|lVldu-2~TPbrb@(>_lEla|!4a%H}>dP7{rr4W2
z2tG~F`Xhx4Tkzbp`aX#^4T{O9s;-uKu1!86#=*PppVKF!XmFKn>mBfIs8mYDArxkt
zicTutK#l32Q9t_Vyd(O<bj_8dIz9gDKx!lmtPX$#=UttmTi@JRLFPUkt;*RH#!m`u
zIH^n=(De2hN&%Vc)QM9Z7bDG!Ynlq^PIQj+VXs>Mp@jM`Yk(5e5$r#4?Q(?28j{M~
zc@SS82B?V8<2s`5A-%|6^u(q9;NRzO510<rs+v3~V@l6qSx~d(Sb+d^%8XJa=mJ9)
zt~KEhs^&R$i}vB(_?55x=7ZAVw|l9GQn1iQ)1OgAA0;B6GijdFM=Qud_?P#C=6vW_
z(?M{U;C*lG^54N>!&d{dgaQz*-!cv`wq3S9AO}9!NPdX(8jZv8u&U)a2_Yh}uEU9?
ztMJ(Za);*3`pxZ}#^ESm7;__@`2C{9=0iyJl%VkXd1-F<2M}TO=&Wc@($`bJ;9QCW
ztvpsY+%9i;G-t#peoj=AIL4_2qU&xn<L-a?URT$JiQvBPHWnT_E|b>NPMs{fY@7{L
zxJswZC}_(<lNND15*H*49y4%sUet{!Y;=%<3o+%s!|h(tCAicJa(`d2MWui~%+w^b
zw$8hWr@Du*KzL`YQ{v!o_lnj`>X}ok5$BXiMN&>q825e!{VK#&``L%~)hTrl{~Up=
zq?qi!xSqJ=pR#>uQ!(z@gG8)Qw;xsGK6St>3!uW=RV~^$KYM)1EY+F>Sn)t40@{at
zHzDtK7q$RbbD;W8z;OuJv3*sz+y~vcu5$sh6REgE=cX(Q!r%OK26Hvj#qtAjx)ru;
z5Friygf2FN4*Bx-Z((fz9D4lgw`gC`TBdeAc?l-o0fD4=baNzyD*&5*?wBgYx1lQe
zNfAw0<Tn3GKjndoUH`*qi0}d=OkIP|wIilHhsm!(@o^P0(+Bx_JYDM2VG%A;*D!@q
z{an$A09(Yj8#;td&+9+q*%ycFnBjk2<24R4Pdsg*9#dw+lOPDk?R>t6uBdtgZkSPR
zypOnVdBOq*?Q}aix%nT&qnOLh%1_?y7|otLt!+l1YVQYGD^#zi$T_6A?Z++39W}b4
z)(p5Lt$`smk^cc`QBH-xro(u@J5J)Xy7#23*O1J((TwF!+DsR^Ap~_b+-w%)lofz{
z9dtxIsxVw(c>nbIRuW*OHvvYv<!?qhsASQ(LH9tqovh)avEUCzdPd4>GcKcZaVuq;
z8lBh$uJtV=y-YtJ<8O@gmBct**IN9B%Xf5Bci@#j<iIIkZIaaaRmg$;1kY^eyiXWu
z!2gah3+o{-k}!RW#0?*!l2-_*YY_5<DV@SML76BLYW_N(%%)7g3mfIby-95Inwj#{
z?v#>Uo|=hOdy!Mbol*BqIt+TxE=cVVi$*oQ>!`cH)#tsWRQH>-b1&BxBXOj2qaJzF
z9MVBB8KI?l-QtV>Ak&R+I+A=Q_nl4-lEKp#mfgo6_hB_)TFCw{NV@f2s5@2yUF7`r
zO)}QBsfv!wzaZ&lDVucIf>#H0$!qM5ZjU$JDje4rJg;#bc$h!_XtZ-&ik9@_L&YN$
zVEi{Ey+G5W;<#F~M${Bry0wGXZ!SD0)yKbD$t5`~OXt*vPP$|Bm$rksx+z6|rWs+$
z`a}YskmjvSN>5YAyKSHbvRyRGd&cQ4iIi*j_a@D6iRV8zwlB|aTQfBmG<|z*&ph__
z61hB3ED#creeFDGov4RYNHQPM>IEfjyxdjAkVK>^Kk{L58XsU~Re>oSS>dnga1Ra0
zXZJC>wWI3X!3x==B}YddUfN3+ea|sOH)tA0`>Ma8)iY$vgiY|KD3j$lV@S84@WGHw
z05;lNX$1;b+OgDsWkr#Kl6N5M?eCJe7BKJiKHMg?Oo5DyaponQFd`{xXq1MCz`p;$
zPSClIL43*MZp}+USK9!sgU0W;Q}-*9weqR~*EQtQd_n$3+s>oNhWK$Tt0p-y_ka<;
zK!3q6eJY0&uLkMzB#~GKxcepVX<GpNv+8oWriY8e)=t~I5h<sc*<L#^42b!3EG^@<
zGqPEg%)jW#Y18XvPLu4isnfv?eZKb!M>QJ*62|YoOIP^B<R^<*w8*Phg$7=xvpH^D
z*S{=<sM@8ARmdd)UoH-TLvm8h+>pSQWX49cSR(KNtI!6)XrBI4B2&4NG~c0#tQV-X
zjv3ROa8iX0u)=zh_E&#hha_8&B<1dM$)Lah0%I9XyHYr8<|PQI$>@^)i1XoQdUZ8e
z@mUmZn%fYg>o@X<ImCgz_!O<pF(hExDoeFI{;xaKUw>4zxSbzB3VZk3VsGX|1#!dY
z@u<I_2Vqecqy3k}BT-pQ8U?p43TWU2i_VKzy^=p5b**XcT=EmEEY@gFPo>E?c(EVk
zKl`7dN2SB!*#S|AS%de9Dl_77v#-Jj)vhnh3e(xd?-xu?60;yiblw0R*a43RXu^Uk
z4gg710Zd)(Ksyz+j6u+@c}c;f-Vfq@o0?wU#&?aosL-m#MPBG?X{t9mRpyq<r^zp;
zb3|LOwrX}8PODu~8Y*7@9>J%%e5nOjWoP9dHp3gH4}-^CaZi1=J;KE2en2=nQ&OsX
z8g${l{AR071m2Pt<`-k9BJh(7*wy6uy7?P}60449r>7p*Jt0np!7m?XJeo<aJpCwj
z6gGzN0ZTK^^=5dr9+rG#e(q&$r+v<Qljmk0r--wP6VZX;5`u%3ACItlzT+{wTygF6
zSlsJuJ#g`vs_fpFMcsCUG4}gd1!)u_-!%lkPL2iycy|K)?xg8ZrW<a<wt`#V^f+4b
z_E{}W6;Oyiw!7Ycijv^qTI#1Wd%hV~u&rROS(+t?lC*fjBeZ%7`S}xQ_7>?%rXp=p
zJ;qvVD#5H?nl<+;USsd&Wfe`ad>N)r*2Bj1`q_kwawEuf^|8&R*)f&E&=`+6zf@lc
z$AI+By0B@5pq(k8pqKwaL9f9X@4?<;FPz|FJ5jioR&#UhaJtq7Yu$JcXcwxkDu9B%
zt>d6Jsm9QSGmRsp+(E++Ss44oz|x(vyV1lZxD<Gg_G7xhb%!L+MJ5xjVaILp|C@qd
z4k+lGo+VQn4k<+Oz42TRyZqMwm%RxfB=}L3QD7$So`(%&^ofX-h#6K0&0L;zBFQi9
zI%U{;0ZJ=ljecpLYBK#-s(XuRSn|mcQH6SP5jF5EoV5jPxN-}JM_zr8)f|xb8AZuv
zP@wV&LGy!9`Ors)937A-5NM94l1Pb}qr`X4(2w<_rh0@w17g|mkXEtIhoyB-#|<Lc
zLHOgL_ahwHP7*Ll2Aw~Ux_f)T0&vq#aTlxU%MX&|<2dGgmC3*9=9%S5fNma_+`oVG
zos2i#KEo}mn_}dV2+k3`zh4l^`P1>tYALAf0f{WD2R7S;{5R0no3?F8LLQ)7V4=ZW
z4`<OC*{LsYORbsp*Dbo->e855E&Yu*H!am^G9PlFYj`m}&~q>u^%zqADSYV@(w7UZ
z`}hRZC>dX)R`U%HW0Lzzyf%N86}Z4l$aMKXO%%cpr7AKr!_?1P+bnHu>f@=(i8^yd
zJJ&!jWpiY`UcPPuS~%^c8x$I~`8ueo_e~ylkXhkJCI;+1e6tI??bVNbJyVh;Mi(rS
z-_U6BebXjTlu1^hgR&K&rv7;^8SCLOE|Zcu+1oaX3A?s#ZVgY5%~K$_qta=qcp<*+
z3}#YCMY9J66HF?cAE~Tw@#e;}H8Q*X4zZzc-q?6R#w_j-%m7J@p|&aL_ME~`g--i0
z`YJ<js&Q$Sn#Q62=0*$bE0IQ*c|vb4Do^qz16Z=Di|ckPf=$+JpfJBG2kp8~Va5Oh
zLUs!d^8Cjj^`F0q@Y3Po3)r77g3+nd&D*4uJjFS5kx*(sk{{mW_m64i+8Hf;sL;3<
zD9+4`gX(DQ6=76-D6^EMsUBrtA^g71o#c{<qpym7k?y4yrS-{(+qQS{aHLTMph{|k
zEJpJ0$_1DwQDpSuSQ8%}?*SWvm#3i979t^kIM-4_6mnzC3FSdZs6FLim!z2j7*<0T
zgMUXw{Bi1mNT$0K+}%mGgHkjZaF8oG&swz<5fQ=PiM*_8bkFXMadL%KGXKg3R9!&v
zn8wj{gQQ4d{B(#<7ojn|fGG{^BX2Mu54);z3HPvvvVOn#mS0Swy5;`rseLif&$es>
zp(;8kdH-aX|MN-R{`Q0$D4fLr<$X^?ydDpin$b`ZjSV3+nL`VyY=8WVB$^U34cm;5
zYbtDVk&xm%4I&D&bVSc67F7T)mj4kC6L&;CK*P>FM!hx^ieX*47l5!gj&j8Vg^{J9
zo}3|}pa|1`2-7PlZe3IrmDH#nc6I*|Mk5A^<F#P~36h^Y!KAtEqp7jCBuKNFI$urM
zWt9Cq$6CEcOXm>PW>Wq~idf;`?T|eBWN4!Kf7x2TApKGH%=hjEphNbs=MY3%MVh$+
zols~se-J8CV@!O^6`>I4`)Y+|92C|T%2D5u-o#z5qG@JpEQ@L2<@TsFyziBcY4USB
zeC*)zcmY0Cq!;Lha%#?<L#7K4TvQ8Vsg^-b2#seTmk;i-sJ0NI)FNTKc669PRvU4u
z_M)ze?H?*`5tS+sQG3AS0P?%uswMF|*n`oBU02Ndfg;4F?BD5VK)wuaQp^0A+;soY
z=KiBe@%uApQG$srFciujf_jd512xFc3aOcDZ@lmX8p*52ig;p0I3ynwVueP0DRzZ)
zhB-#_9ybM{u5(Dq-$uk^NF`IKSt3kc0QGoXZj+TKFZVGXs^}*uqi7sPnDU4Lmu{yl
z!Iun`Pf&azW0(Ru)lk>-Ye&^VTp^R7e))P6G3{rXokui++l5V%Ao%T;@{CEudfTud
z1RfDdr*6<Uznu<x(iA?poe!$cKboZcCk*^QzPT@2_%HAUzMWb3o}!uFfKLMujL<=}
z`+VXkA(c3xNVGTJX8qv-oSt>)6haiX54!wMHYgG*v7JL;FYPkO6Dsc(<pK4il@`fB
za~yNpOXn*Nsm?$Inu11CTG?lWKjb5Ss&QuoZj`i!$eR~sdUO^`ND2w6nc+$7k%G!G
z2O)b1Op}>moPc0|z=O+;`NuclRZUYUy4z@xl6Qg;0(D>4J-rwj-B`)G`+B@ii7XL?
z;-89+|9rjf-Ch|<cPQjf+thFRof~-2zF#+C;mvy1Rw9jkDiL{*ECTw7C+NpHD$+>@
z2YytN0>sfpL>Rsom0Lln*nAs!6GZgye+#6!=6Ir^wu5ax-m>@#0_Ajl1_)9jCI^G(
zIVd9GlEwjVBcn6rooT(%#gE>L38KKX#r*G*J+)bp-H?5750ORh<=US4*xz@~dw0Av
zO5PE*6!89Y%=_PN5UIbrr%pf&4I)3RuwU&4_<XVRNHZ&(Q6N*I;u}uQG&hrkXqvs4
z$C(r3&dJ14;@v^(MRH#(9~(q<Ili9)T&^2f1eC)PL-K^fp3IyW$~eAWEl##b3AK#D
zRPIZf&SogVY35&=<gc@FJBw*xA%WCF6bEZ-Q2v7>v1FFl7Z)tAA;1xFtF8iKuf2|v
z)V~e_k%3z|!@w7=w!eSCKW6}c{jvYYZ`FF;=Ix*;=Cfz2KrO-x@`byFhFAg!(Qe$p
zb@oiquoC605e>;f<O=ENK^u2zi?%CDiX0DaXRb#ldihz{<MPT!Zh7}X5Cg4NzcyhQ
z6p>-}uIvwHRsLZmR7z!A@rkz)V=@5{TK}V{^&n^8C}gTMj+4@$$u9Z&wal-=7Z<>z
z!}5nm2L)HNIcqM`SnN-{Ge9kf0WU83U1^(t3drC7@lyTmM~JToD!8IUK%B%wPeN*C
zG{cc=d~2L2Nf>L7%9p&Oj59ikIf8!NXa64@hZD5V@1zJ&RR47nKj8sJkD<A+c(MOw
z`(oy}b=6evzW*dle2J`wvU!Pal1QR1(u`}?tp0<wSN~zYXFKtOUpP=qAyNL@zmaSY
zv^m*Q79WpOZ#-qy4-me%|HAkcNh%fc$4QmvPl_sq$_XDIc)2+*jAex=n;|H9drZ(4
z)38x&EC<|cl)si@f||R&u>=ayM2S6woK7O}(5GyS=>GQ5{UvW7Iz;8r_&*-=U(4tA
z*|*%zkD7s-8*xr#4mT_SnL5qa5iRUde`IT#vob#WlBC^OF4T5`B&&dj0e4sAI_r^6
z2D<+Ii;A!L)QwVaBzywVej>zqq7vi2^2NMRp;#^=9Ps8l@brYDQy`5R5i(-m@Dxf8
z6<jDmGm2OY%RU}%*VgwjCYW)ofaHr4jYw&UAH?hrj=RSz|Jqf*klvaO*cG`_{`uej
z$8U<Vx6T5y0i<wb953Hm93HgJARW9zBI6*)O(TIvpw3*0DAWesSdz>|IeIw25q1C7
zQ0!CO6D!C|1%Nq!#EMc~8NxQgN^Y<nGt9DJzyGD(#`rNiuUO<)RtspX$wS2=*ko6x
zXP_t@4AO|^G*Mz6bQMNBl+AMAeV|AJ3tylW3`$|*cY?r{^grHWAG)`v&#`f`dH-}5
z|Hnb`yR+d%`@6(De$%tpCg4$6j;kbgpAuo@szrr5<j#)QR>!Z<m#&D*EewIzZ-cbm
z!hY)ffL<X@q&49v%^rlXS}4@He#-QO{kD?wjV$fq<32~hw2jAe65npmoHSX;ShsF5
zbVa00bN@=1wSgwM7~XtdDZDXx0p7LS7k4hFdkB03hqdEbfBOb_ZodI*E0%WC$Zr<M
zcJ}rx6LvN!bxsYJ&g=~mrZJ3i2{nxMRK;7S<s@A9O;I+iLc5%WxOv3BbmJROd^hBH
zR`Z-u@xQgGy`pa|;b7<eb<I8t{kyY?Mt;vK1UQ@KaQ+`ToAASt2gLQ$(8?(Tfy`Q9
zG0O5POBNysT+J6w-fii(00IyLnQuCppD&ld9m<9%83Zz*2EWQor%43;EPC6BbP7BM
z)x0(<j_40SBQyP=FSX}y?I>~_#G0oMD7%LFzAwj@`NES@9$g&JogSwZd>}uivX9@a
zI);f|lW=NVSx(uD$ZM+J6*^<f*Pk_iw%^l0xwRtiCa)S3LxW_;+h)jvO%Xw($c}Dy
z*>%6;Rm!C~oZgb`zwJF<(P+1p`h=9>|A%HJKc3V^DOeR_p7X?qFEG(a1>5&cOU41h
z8~7Xj;c1dt=}8Dt+V{CPurw#=*1|wbzhJP!c>UI-$Yb?o69nH-+wV2@sS~qlZvoyC
z=P59?)i}&Jwe^5|^XxZ6L!%&4LpwgR$A7)Sih5u~p0!7a$40sG>tY(seO`#n!$JO6
z2u{Of*>07FGW=>6t)?J$@298*h%_VI;py&)JCL=|&%Muk`@k5WrPx34vfiRBY?k5+
zJUA1d=>Kc^{`nncAp#eB+cQfX5`<6$N+g_-Yl#g{66)K`)P1xe|Cf|VWgjP{B;!vM
zX7@!w$8Y%y4#fK+f^BQ4*T|<`d%0Gbd{}}s3bj*YDWDcrl6=!J)0xo^Du<GG=cLdm
za7a{;@AeVV4E<`DXmbmfe4$_0`i(nk89KWSG6%re1KbC`^{Iv8h3hwAKegS=ugTvW
z$<y?~ru1$DnRioX`-{^*{ake(h=J9`(Hj{u_}5H_9PRh&T6LU%oH9}dNPFfVJT;B_
zu)K2%-i108%vLJoSpe(zLvcex#aJ6HHNwJ1VCnT2aZTd(mh?*!pveF0IQz4DaH0kG
zN&ibmmFuE*=mPN%3QsoA!V!gJWOoFUZmzd*{e6n8Hh|?U`orBLFL1{icY*zFfc9c4
z^k9s+AI05}(1vJfRDG50NltFR8XBSczui5E3`rubq0YWiGzT;d@sF}6I&j@PY+e|{
z5E8fg=M~DWYq+&%+bjn-%9YEFO~#AA9xmD?SX5`50%;7?-P*Su0#ISqF3ZvCl#^9a
zHDk2Tt5RqvQvUi7PZ+_O{E}y4|K~g!yuOtsQLyYeQ;GulW=wa=n{>I!9g6Ch?dmGt
z$P|0{B~;kxJ*cgQf)#j2h@C^btZ}#*#l9%x1s42zzLynM@^10J?8{#s|8}Z16Dc4V
zeZN*AW<y9y`_uBVcP9(f?4iZ&R6;e+$TYWJ42N<DLXKJt0VAZKJcD2#^0N0~^kHo<
z9i5z7_SIu*cBO@XdOm3F+Hvj1T2GGrs3J*3N|k1e2pfrY#%MP*g~q!wV6Zv%ast53
z#O?y#j>O2ZeZc3d0pQtYH-HzM|19WIu32gv?(@A-bO^WpUYIza-NwCgoArsZTup$a
zd2_hJ=ZBcN)EB`ZGj(evrFEw7<md*Eku3Qh*jxTAwW99+URM{6^Lxh$4V(2Bdu2b9
zhD!YK1vW6gaDghqE1KNvj1SD7qAy-fF8#-;1+wQr(Ga}%SoZMtDvH_6kEiRyp@o3q
zG~Pi>fTo4GVJwMhgb<Fi<LR2l#FmW6O}Qb1f<2Li<8g4Gf1@z<`%TBF)KYDKD>?o4
zlNYF>+CFWJCk3J|b{vSyCUnBmh(Q>wZXIaO;^jb;Oyv|>ifuT4AG9H1^+L3*pFf1o
z4Sd(&thhdz3}~lvuN^iRFE+Gj4JOmoj0Qe;G+Ie69|6oT{)^MyT~uuHCE)T~U1MXh
zY`89WS@_6A=?nXCPqUf1twc|s<4;AXmog1Ft^MerY?Q!VZ*}$E@Ava+c}`s~Ptlk@
zW}M*hU-n{YA8;WW>w&$^5nf<<Jx4-K`#$54-NB1*i|GHNYyZaL9%d_aqGG7DjLUx&
zI7g#XoXyoD`i9;@GBf^heLEczwx$_#lSMdK8u&q;e29@JK1CPNA+EWChQl4|Tax&q
zg}=OXj`FStx<I0ddOp6uM1|$<^I+2XYKPt4&xBrM?7BZuPJd#tRf1Fij)BpqzrM*)
zRDiVWsu18W_sfE$c-&OEfMhKylPakqV)pnaCJRj_Bv(`qpE{Ze{Ix$9exgzQ?vfr&
z*6KK4$ch5{*7LfWQLZF?fkm@AYeuuQbhoU^bW~p-9`x<wrQU_dbS;eOOh@bLDqAkT
zawPZEW_x6!Id7yWHUYjqxswZPt!dHe{a&P7LTojkyR^wr%dEuEqiJVLb!oZkmQUph
zHuvwE<r#%6RGOt4(`6e|eO%@Poh)x2w^`a9R$V#|kj#@T6&H|Pn(SXzzteR!ADQ~m
zXR1H?-O04#ELUMFjZN5A$89g?(s{o&p29<AGDC`Uw0e^--wj5SZ+OWyz2KBm#h~N1
zweQq)dFgluiftXLZZ(1M>0NG_jrl0$R-dIp#p_GR&gNIFDQ|^Y>qB9e%#lM%huI7b
ztkKjywVNp%s~&aJU15F8YBo3egs)idXFf5Xq`C%`zq#h!*^GBik#W8J>dv2DK%#nW
z@#drqF7)V^4IVt)VxoHzU+b4JPuB9)Q}g<GIOa=1%s}DTji>UP94Tj%9(A_52UPmO
zRF~$v6ZRk;%IJh2=GSLFrDJ|hY|d9xKK;~wyBhc8P)qqZF6pu!Mtt<1OW?zKs@tWu
zrtdD!tXnVkYF7WQ4l6wR3#{N5$U3~RGl@vxKi2S|dgVS1SXWxmNn5g|*Y;~W#J$dJ
zZ)@6965_vpG#&All)lklsF+*eW1A-jpR#|CYR;K^)&ol7%H6?ou|OAR@GZnG^gQ&5
zd1pF1+<-%?Emrt4upxHeK@@ou9%#!y9Qgrz4o<ut;gsU6rE@&`&EfFEgY?oQ%avyI
zt8L2LmeZ8;C#K`QZd^9C2~Q8PD{NEYvxfgUoyPuj`z}rMC$Hc_j@HnyaGUOhivig_
zPcz}(xbKh`#2vxz_RjSF>dtzR+^R_cBsJ9CiN^~f8W8rW!82syhvpag6CusYc8ye)
z;VRVo!ck5U9U_KJ;C8~8#)r@5s*1$uzF_}A-v_tzOvyti;Y^jP;l=ttmHR3g#UXP8
z6_z?rUC;K2$_;L_W?eS2Gs&aI5UGnwK`^yAwgyPp(6<8$O?q5f=cveg`uK~Jvj9PE
zUNJodkh9ywkC?Yo?Ra(d_5Vg3RK@kuo1)0uVs6C>0O(9J$9A12Z*5}vl-iT0r#oVq
z2CAyR49y1yi*`pWek7Art0g(Bml#@`EF?*uOO*8GtS|nI7jvH*nYpel^n`47lo-QD
z-n3P+y{iq&$6O|b<@gOT=I{10)s*?)7~k=L);R_xaGa>exo4P-6o+gUEi2`(a<^Ty
z6fG~|4?j$TcTDaKZ=PQ#v#?CqTl>`8Wb2sCvQGXmCFzdeVw`r+d*#~2;T`zPVJzDs
zFYoy~RkY~{r;@4COjg@Oekrj;{qy&Mr5au3`F%LTHq#Y6u6{POaJ>8VpKqY2D>^vg
z?^}TEWz=*srCNKEYL{9uV>xmZzQZW|q53j~iD8JoN7LbhV1-a?$|@b_b=LOOefS5<
znIcyK!K<^&-IH8J!NaeUQ!e_w5$l-iBTNm^XE&$k$R!;j$DjLJC8u{WNGbf>+h&VX
zE)RTS-^5~{d_aDFHNth65&P=ZclW~+pZ5%}N;iwbH60pEr`}VId`e()>K#yr))DtA
z5o;i8U3y%~j@qTWMRHrO)JgVPaq83ys^yiQKg*Zl)Tti!i0RG858@h`o<5#D?(VkX
z+>QKbv7iI%rgxbyJtjw<`qm~ue8rZ?*;MB`jB_(FGUB1;zF2+tM*dv``+Wv(=3!n)
zWgkQSrvEIq7@5cB>2&OFL)2Z<q&imHY3>=}sLWvgHn*dna#C+p6wJmi-~t`}T^TNt
zReg*CpNGOtewn|iIN7s#6ELHx_DjWcjcdM_F88?1TM7BWhVz?2JZk&zg1`s_|Mker
zfNS4Ul0^Jl*Ma^&LlABot*=6F8+P+$yXVNy1+{!Wk2O{c8;ZLk0#HnL<(<;SFR~OH
zFYu4h?W`$mhZ@<x;BFPsKo4;WQ!TAYhca8ewB6mf$(UU49ES+8DKR;bNj|@;jf{_Q
z!0;F)G(P(~MhmztC^C7Y6G~qa8)sN+*K;vWq|t@AsXWh6Nm+9$9-9Q!Uj9_M@i+)c
zo>tQ|Z>c1tiD(9R^HptUlXKWkM)8X{fIX@K97q#CbIn#56#9a3nP56Cm4qEsH%s{q
zHy0xLG|HVCMoG)WtVfnsaer2_;HXkWM1OKrr*ucQYDO%W1cxGiZhyQvU@F5Uw7@PH
zs5ycRuI+gRjrfT>wJTlMFtNHH)ngxEIph}b3m9ptb~~;pKV6LLdMP_dic8FZw&hf>
zX%mSYd11>~hd?@7QdqYiep*9pJ{mY2=fo2APPLenzdw?{_yxMjWyLOUZ%_Q0)$8pE
zTdJiI;k|F&ccx4C+X|-F8HH`%r{#Z~3*Y6HPaNhHyBCCw+t5{}WBMX5Ind%D4W4L#
zkrI7;+K~VJ=iP1M-sBbSIt#}#<~v0V+;32;LqaPno*#s6(L4`sICpta4Oj`2i5C{n
z-gVGA4mk(`8^ik~XQyWapJs8f-A|S>1qL==5anhj8xI-<L?T&#bC$4QZy>?lA3#`9
z%k>>Ao7Z)y0#&i<uS~4-i1xwl2YFg+cI!SKI*Z=~okUljxLn%&a=9er*vz1R(6;$>
z`w+gBgz-75Ww;!dwmlzDTv6=~ibnGtU*qPiENj>&8Ljk>F2T=*(`Qp!K4%S(M)n!Y
z2D{wTvKEt&k{uG4P!z{QG$he5h^7*USI_P{WmabzttGqdC2l^**%;!@%t@}uT#s|U
zc-j7gT`s4zm(fucV!!b&MU~ZCQcg%8-2v}3Y`ss%9BbB%XIcjig~`m8sJjdFG72e*
zpZxF)7OhE95#RXi2)Fh^=Mp+Lt?H`ClXO&|L4KieFkcconMo)aWvDz~^r@DOomZp4
z*=@Ix9$88wXx@&)uGz1AD?PT0<B(nQ2fh^wCE{N{+6ix^a`ctm*oHmo`FjY=v}%C@
z_?zs0Sf1DopZCm(&hET0W21OqR;04!2$@)5_I*cy20aRVzwi)ZzQP2lUmK8GO22t$
ziWDXJSqm$&ff#r-{2oIrAD^VMyu&s}@wh5wylQb3C27mVMh~VtB1$o%Lch6SQC@6r
z3#Iiqme_MF;>H(1nr}wMg;2(5J~AEoUPPkp*a%D~>fgVA8v)UxmLTz{^4q{`>3?Sd
zn1k4RGR{znr>r!I2F#{eDntRK0%(C$wc)k!n@cH1TqdWNv&buU72w|ax-FE7Fs<k6
z{4lbco%3CHR=vcTY^W#Q96AqDs0fwBmj^-=!%EBt+V-wiuOQx%o7$30vuh*1JrgJ8
zF>i2XtIr`fZ747a!jwX8w@~%UrMW;;jk9--i9L&p@T-oMT4$N}l0IhWl<_h#vQ9LV
zQ-XS%>fP8V0dMlI-Gwo;SAsZ?tsnczJz|F(&Q`ZgVDs>ktx%iZR9;4GGPHZ*8!?dD
z*c1?q(u!=19bS1D8VFNgwjYoc_&vlDx~raEiZv>~Ws+A-mDtP{GS^6i>4xkj^79xg
zXvSqKCIHz!KWW}M;v3>2cMY!GNeoE$sMtKMU&^~n@N=P~k8#YM`zhr+|A&vs(b9gN
z%xhRw9KP*hbbQ&$Idbvvb;X8z;4#m9fe9%Najd+{eFFnBk{6_#1X>-G138yrRz&iP
zcb`dY+#}zAdNvT5=VOMagW{@b&(BPGLu`~BcYR=KK9kU5^8ja*?dWnoh^(0c$2Ydm
zd3uB;DULEC;^T7-^_++~rGe{a6qrKm6-&pfbhiV}st1$?9_eikgLRKj8q-&o1jeRK
z`p?PEL?h0Rt}~lVPok~rx+3;$rr4_pi~`&<>Cx85QE)kWw(JH>=QYaZVnn{dcnhWy
zyH(k=Fv|4L;-=~y*p=S>oR(x%FE_5!)O%BZ`gMjO-<+00x+2{@<yo`K;8v{V-Q93#
zz+Kk5$nR2}PfGRF+Ib@BkmY6fJjGj7A2|za!N-}(A74@KySJJu8w8kvw^CTYom5bv
znyHbdx4GHWVi#H$z96>PGrJuVfzff%SRZ!gmU8MX$adzCwhHBt=y6;p*QCn7vq=7(
zRV-hs*yV96oWXLs*d;rI-RuD^k(-X51xM*#A_EkDTE2I9T40sbvmuf{*~ACEIt+#@
zv|gKp86Dug(&k6{_wxK>l@f`7YiC%az8bUEfcFpNxhd#;w+=xh4=G8er?sniar;RC
zn<ujjn)!L#INx4V47rl)`%hOUbVJ1q5Qa(U8_~46JG>IDc#i;nEUVSGrPb^naaX11
zFH*ksY>7_v9`OR}7P`<MUfqp8KhgMG@g~NCsDJY4-#Z#2|CNaC6Z-J=dpT4XK!Q(T
zFeT#chz^%OqOlX%@mx|0d}2D%-7el}_weemi2PhmxM5ce5oNA!lYfUD13)3!v=St{
zooFm}evXgH;cH$4{jz6U#zR@*a`8{@n-u6MFn-C21TpzNmC2GEE+=L+XA2RtX9DXP
zkz0Uo9;p$bw0zpYc0(79nqR%~VA5gMZCFklO)P-e<C2jIJp@F8&4%(U1d6M&P3?w0
z(kO236*%n9NV@J*QSh@|ABK6jmas_g!bsVF?$n*m5TJr#T9Jx8Z_7k!$2AL=b`wna
zrcw~-FBx2wL@=vB{vT)SZL0g_R>|OQD;0<#+drh$ee=Dq<=?wzRxbL>VWafwPm)5L
zp}07_k7f%$h-O>Je5=2Qj#&`J*NY&2H=jVXQ*7@AWs@)-8Kbc4yB9{ntV+X{{mmEt
zmhw+6h8fmv)?^rwuiX>eexRNs6TEmt`QRB29-^NZ4~JK?x>d7dd#q1yHVt9Ns%b3K
zgM69s%`|uh+GQj}fL9BR<N1oz^kH+&FDJ!r*Y#L0>H`sfYz;1IbZQWHG9K`S@@|Bj
zH}att(zyO&{MU8e46|Zfc&3oG7fT$UbQ77~wRyb8HL9FUT{7b0fxu$0)Pw$Y=jITv
z?hkqdUZW4`eV%=lXmbnxPEU|uGWR;@Rb_I<WsyWd${>!xr90fjbR^$$p>=aVJzRY(
zGxAdh&t0?0^Mv_?JzNHERf)Z<;M;fIplBY7v&)1lEZDiA*k=!Kw7D;}u{dg3!3*!b
zx~98a4a*H^N7--rhK-q^^weUS3oluyv)_)=)#>>O48wuSLW_bx=8BIwBoJ?M^w{2M
zCia_vy9Kpyh<rtFG?fc{t>?(ibUgrGcpsKT(b^zuUT!kbds2ONT$&OjKu9Q{+|>X1
zbR~&<Rp;q=tPb&Yo^{H@83ieXaTHj*d+WZ?;wbKTeEY;bnP56~Eo&-5xUlG6ShKLO
zmd$Z5)umC`4R!SvKDr0({g~(EDz%@6z^m*1cKkSNA;mOb|1~>*<k7QtR4e!xrt3o_
z=ADGA>IUOGo43cwzfP4uejtLARrLMqn=|CbbZusKVH(>=A+CUp;B^vycbfDN2mo1O
zvS+nuqWAdPcqTW=pP`j991!Yw@dn<fSm=MQekAi&Nyq%CIjkM#cqb?k%dQzli||>C
z&pf#PC^Zf-Oy28GylYW<NP7x4a4D_P<=ncIR)9+h0<|rT!=jkTDKbnHA_Ia4Z^^uN
zz%%gl*}iph8WMFx-B&R?S{s;$A@3iK8h6A$qHw!BkwZtydtLUxWG{LJB&C*tsWO>*
z-uKsKIWKFryg<_l;L>gSUSf}XHSHJU^r<X9FnhU7^L%%vZadx-Y<DkeK|<(BuFiF5
zPQ=pwVG4ZeWIU6{@8FwU!q#W~(EDC9@atLlIBU&T#6~QePO|7(Kd?jyyxlj|-I=Pg
zZ@fcd%#*&N(OMvk{p|7laRdvU<hz-&1VL>z=-?or2_?siFe9$kolHC&jvjX0%&XV}
zZ~zwFVz7OT%RQZrN)4DR(vx^sy*P%D);XeiNUThcD|n=MP^~s=`tN$@OFoTabV|qu
zIq`Y^)6qs<lX}-6hu((_AIpl|oN(ZacOlISB5psY112XJI2ZacbdeT(4W{ytRcN2V
z7U2=vJwGejXY2DmuRV0xK$Ux~0hgJTR~=XXZm@{Bzc9@o5Or2=Cp*_JUo~5MJu|Wz
z+&0K7MDAo5K|`zRIy^?%n120~tvD1-E4(_IsY%)%H7pK(Ru&$YL93i>q!oUyH_IAu
z^}xV{mrg0%?d?L{tQ>BXeogU2O_HllL^eu-*`UN$+tYJ9PhO`pvU1lI({(F*#&BE+
z?_`V|T%9lfxgDOs1AXCrL@v{S!k?S&=I!DHGLv<^z#e?9n&#boFwYs=(ll~;B$+1>
zaPu+LXJ_95l1PxwaaU&3ck>PEP2W<Kui}KNv_bp)18k{$Q`0Bq3`5IKI`R7pY)IDY
zyXz+%&xe-hQkdLk^xLNSsI@{=4!RKDz-l`tj#w`AnwfymP6J-c8X`&uKxTKJh7s%C
zkDgPKNiNC<nDGyTaJsqlhebk3M+S~baQbobQLVD6WzXQ#kTW;R^pCiSc!3{V;BDxU
zBQ;b+FZ3%I-{jYOeODh>Cfk*{)Vk4*cgp$xFzmITtfZj-Otgj9^-}d96W5;qAGO@y
zUOqG8-&HNC2W4lqBDFd-<>#P4Q3@P{vQ{z9fkrfumu*XTZ3iEsy6JVWK2b?Vb{<b}
zL#0Am^&|x@vFtBm#fx1+q3Q-zaUNXr@47$WwAGht+{4lxKsOh69&ZN{+gFua`yb8M
zWT^Fwao&%2#9Dh1!_xc8qhTB08o%i`xL<2)6}}ZU9{Fw+2sN^JRqI$Jc(Im#a=yB6
z%5h5GG01M*PrYcx-+L22S^Wyj2_fQELVZBXVukJj*aNQvaHs~U!APzcxKFzSRaKOS
z14!pE;}8gXeK80LBHH}1s~X%?mF|MHQ_FD$kJpKqagLy7P6e5uOms()3PL8><WJiB
zsXVUs8ItKV)S^O=^*KO9Yo8#QyDgu?j+p^&2LnjpjGuN~3}Ntp5IKi<A2|^VKmSqn
zQiKf0$reao!?V@=<F)Z(A@a((F)L+Qh;g%eH3C_!zd%d4Oi4q|8Qki$IZKh-`+Rgt
z*c45nFC(i|pzTIsdn}plKIp^b*G*>0a9zhnrtg|d8h3N0+A;h{im6RlOX-K|9dz|l
zE7c(SRv)qYeGlG#>R^K|*4e6F`>{orkIt=GDGZ}7?f#a+oAGgJ-IP8}cnuHUNxMG&
z{Ln|z^;Ityy$VE_jZ*|t_nAK8I`2?m$|gQ)m2CCyOB_RPCx0_WN)fi}y%fZS9(ai7
zCKHcijI7IsWVz!9ivxRDs)tNLf85Pac!}R)KnIJ~uA*&!P}-x27pjB;4@`x@A;DuR
zTD{(X)|wwMp5Hbbo+Vw+xH<kMS4g&Cx&5Xpa4~9Rro5|uWHSow*L)GYqC)gu2Lrp;
zToOMHGgpZm>#VDt0sXvmoU6g~0k3K?>I!S$C4n#r-V^$3_s0+F7&E^6T$Z~eL0$-q
zGFnt|a2gN_NV{B%{qPD@i89a5jnOSzCbRj9*+=5ESn(#(ZiB0(jHP<V^w&i+_sXe6
zoHID-8anKr+};$ggl+5=CeOdX<qDqAA`_2BFGcXw$Wd~}nmgHnQ;U{g6%HkV&t9Hv
zU;1sjtVCBQKD(J^;R?k4pNCbb$R`ovMtytIx%YQ&#zg10QiNI8VbF1|fZ;uaetnYI
zez{|i>-^m-$(Q@^@pH}c=HSWKGKx(0M~3N6WI+tcOe2=%a(f4uK_ksMA6wLC%M{;l
zl_CZbBX~C32ix$;h0M_@-qWndZX(6?trtHmgf7q)rL>X}9*Shn7n|a@PM~?W$#BW-
zkv@LwiA~89CP<IuG&dgr<u2whlYd=dVJ=B!_p-MeE8%dM=g{g^gYCL<wsam%r0cHB
zY{@sd`L=r)^x+=Yn>rh6E^nv;NI|x&EXX+YxH{WE0pzlY9PcGjMvQg}_WN}F>Ehr{
zxZUS*V}p?hVsO-4*1yIoyRjCFu@<A_gh^A!E)nFP&To7%qX~L(Lc}5v#Dj%CZdCb>
zQ4`|2C+Y&&41~uZE7(~0Txy`gOOHRF!CUcgzYVHcpj{VigtzFu4V=pH;YD<#$$~zT
z!8sLFf84n1anN738Ktvex1G%u3Xstj?=$z!d&ut1;UzU44T_^#1uM8*C~IEb{Oa)H
zm~D-bt-M6ntg|YvW_Zsb5{7}KQDaaY0^e-RmJ4pXY?9|3JY6NI$LM>y5Dqq7A<B#!
z7A^|M<*1{{+3|5f<@A1V;?UhN+BtDru2HOacOCOZLWP3f`DF<7RX!tp$1{pGDqEZU
zZloU%?HgE3^ictkMi1QVr)>kY%UvorB!3doCUEDgJx#Y5tk%4cJK+&)R6C}a_>m$n
zXI@g`l7XD6Gz@}6#&9=7Fd+Ndi*@~sE{Fl`?VsY{Wv2ltF{S4Ln@lIQd&8wV$(BdE
zPu5&FkGAy&;hj#q8T-W)Jrmo>5mGZyVh)nt2lcP%C(CQEg^UDEdK|W5WBKE(&v^4N
z@Aq9QpKwukzG}RncF8-pmG1q@ZCvf<9F4BNXeA;4h`kZ{pq1a_Mvc0$2PW5PHX378
zyJ_uFZh3SjU#xZwEo^gwzqqK)=Z7A_2R8#F;3~v&MV8uvf@H9QsZVgYUSa9zBXx^)
zX{N22N}4d~`s!YPcZshYGqL5{Puj<DBjma}%IbL{hp&oG$GZn4-~}tic1izv-ax>4
z)5!L)<e3H`rqT6reU4bP6=wSqVjsc{0vGqaGXyKbX0#tL>x-d1{5?N0Pb#;gz?DWd
zYZH}w0p!pguOvN{pag2DnMv;3K7l(JkumCf!+L?KccfrA2&K9gW6f!D>5+MQ$`8MB
z!;&VNB?3i1UasNmy;TT#>%)>(73Whd$NUDVQU9eJ!R=hka`t-1ffFY`#bt*#LCDSd
zmG<}`n1=BK3hQ@WF`mb9#lXR_^p@%X743*cv%HkbevGCnpH4jnw;i)j;4s(?AUEQ?
zJ{(PynHm`c1uh1Yb9I6`E~cq^Lv`3L5D+zBzDu{EvM#Gh4g?WsJnH3BAaS`IREe3o
zOmguC>Tm2+Cxkb@mwX^#rB1eL30m4J>66!nZX5<X)h$J|K|KUI6w}xP$=KtNklHVI
zNj$m~?8lFRT-Xu-eGGC~LntH2eOmz!Btn!_FF5xjbyzc!(_-~zepkEJPrmWUWFJBS
zz2@5OKdSCMJBcsVLKNmwXRT-fWnObiBf7kmZ2?hXMAR%#EQVW7c;L%NMYVQaJG=#u
zFww4jwD^4I4x#2x`;ONatRL<xwarc>Ul$~p%}_{aU~gFd>OQG*)qAf)Jf88$Thc!9
zbVqPPt%of%LB22!6>W#Imd@tpn*Z|bIKp7P$CsI1s{{qMCu5mz9%4#g8A=^%#oSY|
zsqV&++H7?~gvxMNS}APV#|--7O3w9SUsd!-Kl{D6q~TV;aVrY!<0({UgXY-&O}$}F
z(t${EOW#UxkrZxnqQ%qR9i=0iVV2<aKEiB`xR4~=%aF3^!yyjy57;sG*vhPv&urZq
z;I^%)t{U~j5_`;}m&$4@@x>?jFt>X9xVqy2L4DO$B9<x+Ttm{;wV@cJDcuH{?@c!n
z>UZVb7LZIUw?o&7xGiUf=sG$FtoN2XaD;C9QX)*t#osR2^=(2w&gv6NB*VkjeS)FU
z4AJ}b_jDRxj;I>FwuuD|4$dSIf8TrS3;%KPS(XI-3QFR%ZcIaSW?DUwU{9SNal#E|
zzd}9MV?wP%l<0yRaq-y2%r|(xU5F;Fsmdr&F3^<@$A-Qtjt&{Yvh)(X_Xtxviw<jG
zX<zT7n6L|_CQAf&1{T7vs|ix>2_A}{^4y<-Skhh}E(Wo<m5FiUn+;%yhl}6yanGZ6
zOmscyKvfb$vYr!$pTnxZju^=QQ1|!|OQZg^7Vu?J?~4S?kf$?>HG`{&9)K=+?u!w0
zft4QFbOHdHkJMJpc0uX~85vY9U(4il)}z5~-_KqCL$&HhWzt6)5&Y)nhVm12E~ma@
zqNtJJJ!0_n(c)1z=mkuWAsnHzxhE)N$saoJo22BvtW;*uRLgrEUCWr!upFZqd@TJ<
zKIwg|j`mR-*?Lw-D85eP&rMHL7srqnfd8GKC714QVwVbh?&?ie+qg?nS|8eJ0_aDJ
zoAn#cdn^adr~$Sue82pU{pZIhk=MqRS?Slel!WB(`5V(kBV$J3gA0~N55+`I8b>rM
zj<8T*L|=u{KF>9BJ)&HZ(i7;k36mZPa9<@hilG2q$EC@GPtZ|Ae1UR{+9d2sGuQV*
z^U3LE$KFyV!n3V2!p?L(_V|QK2giMe>c3EM^FFTj(y(l1R}eKI8FgM+<Boj~Q=0E$
zJF}~OQH+5PT9UG2h9eJG>&>y41R}=FCQaU+#a!+U)qfwAvLh7eEP-+>lcB&y1Yw0c
zvvn+u3<*5Z-eqymUC$KMVi;5;&0Eur9p`Ic6P>WOWkbzCrC(I`C3zGT-Cc&)pJ2Jm
zpJIx&Ex!`<6$6X1QGE<a)!boOSL9s@b_8ow*e3#k_XlKcyLYsNN79-+mo7dJW6f4z
zI6zH8G#$zWd#RGll4KPO-V?A}Z9z{<Lrh~B^uh5_GsxlCOghNe)Y;0F&_|!Jt^T?J
zZ2#2i`<#JMcGJNl!0zcNC)#go{g1vm=%RZa-0BzoeVJ{<G)O)LQla=$d=dU&wNqfZ
zU?bA=N~?+H2Qhz$i9nWeC=#Dp9Gd1taTD7PCu=XVN1(3z_+$T+jegnHFzg~s;>0ex
z*XUToD}m_}?>R4mB930u`(j>w)-nC@{NzjqMs;$=A>TuKBAxrQ(soP{LB-iM@FS2W
znWpu9Mu)JCyhMe8Y&H7PT;DSKVypRQF5nJvKr3bS6-&LQx!7@E&jp0&vq00tO5XJZ
zH+|p;Y<_k(KcFKZ%lc?Q(g+V*+x>jm_Z2}~Zm7K-RL^#EgoVb*b6(T3&Tv%6qt5My
zm#v{Qyx#4Sk$utFzzIKmDLXS+I*@O_azZPG&KswIAwMd73N+`Qe+xA`UuN?#xxPGA
z7!KoyxE(27UhjIOY<y1OGLy$-aJ&L6l5!d`M;L2N5NkE_Y8h*><D3^72Z#HupC;LH
z;iJJA2IN`Czi#Ya1)NC3*W<=AKlm7*4B2~2CJI5e+OAl`n7H#X#z!|MzvK!_L(u|k
zC<mh%w@-vP{|{|%8CB)lb`8_rB}j)LEwx0tJ5{7XV9_iA3F+<*2|-%A8xiT025Ezm
zM!FHdbGgHP-}`x=_xpZ)W9&cIcIX)Eysq<zdCWNvgYoC8ye(^!)@YhCL@Sz^fgjg)
z&uYXhNqoP1tu)f(`Zo2_rF4f@zr|BpsegH2S~80-t7=V{)EilRL@M4^^kT~fyF+@r
z{A*Ql{ibKYuHe4GLP1Z=Lza`i%C9uMDW6_%)l~9Ryx10NFqz7A7Ri$Hy4Ui6LAo)|
zzIxa5)(@`g!^9V`W$&Ig?Yhs~&I=b~wu!8_VrXe9m@mJ+p0r;uH{N#n5V$!@*#=PL
zC!kqJsQ_w=qm0ukG0PZ*8Qn}A)PoahN^S0H&{({<y|?`->?`(%qgE7G$J4x;ikNw-
z5z;$uAe2)PBd36I!vF?~=FZB`FBVQlte2kceuG~WF~tnJ>Q}_X>+Bvr^^zSdOw6po
z&w6X;MDS$5&Af%XenYU`BkPF_6wH|^(-)hWD_~M$?%Urbimp^APJRD59QrjX%KqiY
zf1@FA|7B8+L&NA?+f7sVtF|Ttq3nEh6P44PR~SNGOXy+T**YJWDelD(fZ4JYA59-K
z8iV)_Q$(z&NBNy;z9(_r#3ZZa6t(o6Y!3@1Y9wGU9p^s8^4N`)Qm3(Ml*KzLQ23P$
zmy$IhK~mo=S{Zo>RQ6IxKfaH(d;5Lv9br#&^fRZmAw@FXY4Pjv%qYAAQdTg`se(_6
z`WNFX9Z>ru(W#+S5q$~Vnt3kU(`Y#oNY#E_XI}mEhs9CN%y>Ng0f*^duY1$(+h;=G
zH7$iuvL&q^fE~J&x_V4QCu`~U%yZhyrOH6A^c21yXK1J5-FIFic>_jdMaJYu>bNg>
zD|HA<6HY;U{mO9{Gbk2pxLyDRq<F_4qxCaJ=>&R8C}AYQL?ZoEQfP#j+7<9MGFxN@
z>kMuV-w1UTl@f{71nJE-azbfpmbxO_qEmS`sQ*T$@aQy%0WyWaVa~-PCRnzM68l#z
zykprj6o=<?ZoT>Cv=tg9VE5!DwFBso=K@~PDD~A)evZ8`OE^LNv5cQtYprhdI<4F7
z<vN{9I@zvWWu&qC%6spHR=JWZJL?Gx?d`L`<sB4&6nh%&@2v_Ec)fLnb{d9ShR)g1
zA|c8k1tGty_We{kw8NZ^OO9hh*}-UZ<SLBc@J#;4m)rb5<Z{2%nMB9S%03ilPwW7s
zi0YoitSJX0ms(fvvc0n`McCKt&(BVKdK0qVu5>+&s>P&%T~gT_#<DSFc?oR}INZBV
z<S)3rVD~O@*YS%uthD?hMlK#8#-NGNWDi{In0kQhbNF4mls_HdxAe*#xYpuXXruOx
ziF?YbD=L`9hqkjt8&^beCt%zpH%%9$!X4N9ff0Tq(ku<^*{{~7CW$fVU>lZp6JJB;
zEuH%ATpHrFb+~S5JL3}0$$`qQrQ6eSJ<!)e7*CL;dX!~b*8SdSUm=M2xB0C}YoPBl
z)Xqkz(O<?V6=a_LJeWOxf10=MV-v{?y`g5tUB&(Def>R0?Z@`#2<Y2ctXpH3$7^3g
z)96AZn2~5fvWYH^#)Ube$xLU<vE-=k%+h2;tbJR2a`eIY;9I3m8LA0YPHzhNZS3J|
z8Og(1u7*za$QdMx{9*FQ=vzSd8u=zU`Z(g2{hzv5F(lO=K##H*c&|-?{xDw-ns=gc
zA2(lGkjgiHh+>L!cfP(BEO#iF9LW+FANUf_rerPgK@bd5cq?20k{c}w)=xG(qt>;-
zN(9_MCTRR_!vdEJ6U`jW;ee83H=#V!8{`M#Kd+BY$K`2@^*LQssZDoUjz(NTv&+JS
zu?5zgeNAfk3Oa}`A?J;_UU<vAGp%o54=|;f{?`l_H)n13XP=zZ&wDM#mAwvE2V4Q4
zm3gMrz_Evvw|1&jo24@S@qtz#$36I-+lK<9*I?<W>g9~#3qv$_=yG?s>&cXMVZ+<o
z-~!zornEt!@dK?f%rjKxo6mt6Tg|Az8mNnknnwN@#bk5QYdiw?g-CABHX0w#mal`c
z^+~X(iuZCr)HY=^t!HWdm+~+UWw{W?Mm(G+*EUJ4xM|O5@$$bYCGi<u2Xnez?OBWW
z=82%hI%q3+SSe_<jt|{WH}gm7G+_@7PB1_Z6vh=96{2=aQrQWVA89b3{F!umWbM$?
z{e4{|C2fS(Gi#XYM=5A+<zQgnb`}}+X~UPoTgLA@Nkym@mFNf@`lHJ9?(qziXzC@C
zUUa)a;!Wl!Q|6b?vSL&dwy9mkJ)s^}Lr)<cE0m)6jfQQWp39A<Pv|&s*_k<VR@>hi
zH!<XnyY{qE=EP#s9J4pmnmz_HA;I^pXW6g?0LfzJSbfSm<>7T)Wrwa%I9zz5Iij-S
zD>8k3yOm%V{57T1VgXlhp_%IX2)b}B;@~!VrIzzhGe92Z(d%h={WYpgyam6-eX_!O
zI)As>YtU8rn?Fui&4A8<mOlA&Rm_>B$BmDtuO7axXY#B($ZW0NHfh*Cc=4v>z*n??
zn(I~k>HDRBRsO;7iU<$;3+RKXu|l&Af|2B(PHGFZAlz)o4hwq7K&tUwN$eIe76UcR
z@Ry{0X2<2o3m3bt56!8BWq68SCGynQPiHIb9MFeMM~HFLJGi5WN07;+%UUJsm2KU7
z`7A(Is;)hyv(wD?5=qkUDSql~cv;TuM-mwc42_=7Scdne-KWrfzeS#AW(=CEy|%WK
zZSCXXcU5?vr*oFDwV_x1gD>q&F?zvvV}4$@m)c>9WXSO-Vbarqo0En15NJ9C7Nx?J
zfG>W1Z-6P+$K|7dOPJ%lbz#0|DANHTCqJHd1wDctypf)81M<&$Y_6tu)8$YFN*+a|
z1pHB==Y5FiRB@NJ4XOP(h`h1Ko0IZ}=|V0ghM>e!^xX&ko#H=_#5c)T@hhz92j#3E
z-t7pC;s$dUr7r8N;@tKSVi7Eqq2WIPqXq>Oo2^<+Lo_~+63S6xA>N^SGmQ31CL>~Z
zE$M`vdRy-Qyj_ES4LQl_@OK|2-c!TRrVnWGUbsKmC1MB4ec+_eb9ysQIbG$h-%0WH
z(r29;Jy`m?dZJG)^^3;OQhHaQyjHoqM1RNAwpaS>q5WsKcLdwRFPYkx80X}YRj89p
z3h~e@DQ7sUM`yTF*oyz00SZF2afdEhUXd#KG~79o{Sh+!;e6V4*nEGHEm?HrO8)sq
z+s0@4gl%_T$fXtwGqV32<^7N{RO7>KxA?k^CiSL}MH83kZHj`JrhB`6xS1<4SJ&-g
z=|4g^0c^WD*3Mq|W;9s(dBw$r*JsNCL2b+vau02#hjEWiw;?yZqT8$O-k>8Oc4}2@
z+dlBcgeKiQSKkIRd-I(_&>YO!UU=>Drk}TuIqaveoNY^l*i1KQ)6KPH9e4@hwSJo4
z(|@gHd3mJ0(D@=VOB+kZ%5N#g9|JYML<AZv_yqP$yZ4F+1%a2E^%3f;V7JfD^x8_g
zr|D<w=6<}{zx^!DK$nQ}@ykWYviZZPi=TdH*aHC4n(e-d)xN)f(g5+9e%ma|yPY*d
zxtCMjy;ptB(e`3gQ?Vd+mf1RV=K4Ok%3n7eQzdaq?N~v}KAow5CeqpLd)DA={WdHZ
zO19-WJ!yT4S3?(%JMj19`ES||TL|E`3G&ON0gwAihR*wov4SLm2t2HTJkA&^$4f;0
zEd0cxLSeTtU^f~>eNwbnd?C@MJTciUkhZ&;gJ_~RTaOqg23r;M+uKm?#zNIqvWB7D
zMASrq0k{EbK6YUfbR>ubX#;`+x%DM-KsfGcy4k3_JoY5|{iU9|#vP%{2^jJGJCuYT
zY_O8~Y0Eqc11kEtYI=f9cft^*L3#Y>6!X<?W7aNMJiV!BIhqMGSl849#(802^&%gx
zG;^5m^$M+ZTUjqq7(L_EC;Q<@M%4cG=JftTlcqSWx%-n59aSpR?o~X)$cGQ!u|`Z(
zo5^2+apLO`6!m`6gDK9H-soD}inqNZVC`|xM|!gw4+B6<MRSzztOpKUu}|(7yHh_G
z9RwplGc6`iF_HxxSZ2C22cA0(ayF7{pTA^=r<lu`ws|$--IpONX<6s<S<ZB$;OZTs
zd>H<-0B*2Cu~`OJ51k)eJ-Yz(H^IS6bgeqAINbr;k?)M_Ju-xb1WDutQGVKuq)OaN
zHH+4FX&}o$3z1$leaQ7y#P@83zBh?^Zep{&6jtcJUa<9z&!{?hg%(fy%}_0K`MU+R
zD<cw9SH(VYw__^(Q_Z#YcjU2$L?R7)-eoSrm-w3^i&f6^RtrK@^NnCg_E`MN_`YtT
z{Vl`9I08bS1p(?udEdCcHe|AV&U^OI3FGbIN+WDIx41z|&<{3W5L&<Es_|TQ_#+E<
z%^g}2?}8*^Ce#Yg@eZy_mqCk#2njtMfuHZY%QOc^rg@?Di(9F|#5C2LqokI2np9Ju
zo%YJA_F{kNG}|&tnwUVaSZ}qw(qQw&P5+&90qoq)hi=&J*8Y!%`mcKl#}diwgH9u4
z7}5lxOp8@+hy5st<&D<K2Ikd<VfpI<N3?AX7xtv$q10y=IWA%wzTYyT#dMMA=d6LK
zf!KZcwbZ9~o9Fq`7d0z=kSUKRttoic+PvEYtPZ1B^)N&$LFR=)iN@%iARU1Sq@UFR
zT%m)l%g|3ozoZ*K?q^x=)6=r=G&Yg8S7-UWYJ}!%bG~tJy(Cr1p8FCz#VgWeHSh7{
z5X-l0E#h{yu&aG9*ZN&g>cqui5dzX_FTJSzwJ#61KDZepVb@T_?<yb->`sO=N?Bz4
zhH~8BPB>HKU9_XEz+XP;_1stu9Zhh5sEyW<*C4V2=C~%iZSI=QZ4Z97&WZW_IIU9;
z#G$#qR(x*0K{q%;v;LF)2gSFatTo2_H(fuqRFN(!tQWp4tzW(g`qU@*Dws_lG<L^x
zpQ<67dn^_l=4Z4bv^U1AH<T2wvXXs{V;xARJZu}xjJKWJlQw^{Tzkb{O}6jTGzl_>
zy!dyU<^cD9k6Rq3ZI#*gvqwNjFhxJ?(MHXXfks|R=ExA8hnM={Gi2B_A)hnURp+VO
z&2Lmi@2*BhM@)Bjlw#b?d4(kw@rv!pdhs#Mc<27q+ITgAgw%#@?Zl*|0j0Kh5s~NI
zc<hP|6;c=iUC3H?d8*%=6U^0YrM3J~!3GTP@=&ir=)PNVGoKsOaw(PH`r)h4H4cN$
zr58Z>HB}EZ7+r9T>vV8o@#!_#g|7A)umGMxwfp5kYNpLoFmhW$WlWTJQ3MbK+(mWo
z{R|urV4K?l`_>b-jY+QIJCFdbd_eokzU4p;ENq-9gDo_m4ao4?&E69}=qBb1ql6;Q
zRha~;BwPY+L0#7MDhq{Bg~D82IPD}uc;$D=E~oc9HG{3UIdw^u_M0{f(xYZFgLde^
zZ0Th)>I~?Mpsc#}C9C&qNUj09p#JjY1Y)tKU57zY`DaGnVuJ$`MHR~=mBPOQgS<%W
z{96TLo(^f(w>JGKd!Id3o@Zy-zD7QRHy-mrT?5046{U~%!w>rFpY(il$m-%p$pv{k
z;?dL*yIvb@jFJLU&1l8?&H9t{t1Vi)F)iBs&;Y1|pIbcVTu}VZK~okW^|?(_djxOu
z-54=ADeRQG(xc#mic+_!$2GWNfURnjbZ>pP!Mf}A%DkV!D{4BjWAe+!sKA99q){sG
z@?ly}dz((g6Qf)2lPzo%(Q|u;QYxwiBE@l<#9vy07VWNkxvS{bT0eU&<}1s?{YhB6
z`WIzm7H}Hy+eReP6p(imXRPz?{b>Iu71sA47+S;$UcSDNk<w%9;gtUIJtLG!$MSAt
zw%WHAZc@YvdV*}B9plTrdLBChaWS{mtLhJIfs2yu^ZQZyd#+HrmTT6}Ur~7NIqovV
zD`K*mrF8k0KBMEq`EZfhS+5R7AtYpX=^rm*3y^Q!Q?^YRQL1n?wI*O=5p{ds%SH)V
zppW{RMO4I#mWQ`j{fOpDQYtF);}_?D!E?{yia$JT`g7#*W(|lBX^ayvf?USSmXoIy
zZ9%rd6Iwz8gG`@~7@rXFRI8J+@Z`UZc9>9cJ3mT5KjMigUn96BDRg;mJ2J-6Mo`zn
zKvxu=CtHn3CU2ZR6|z~Y=$D`-;E14n(;YO{(PYQ3u8|Lh*OB)sO;@D#1Gv4{;%LNo
z!Xz)FGHjM3$ao#%!le&CK2o21{lMuXXRX*?%Td&=#vmLV16xf^e|{oMHJx{^u!L~N
ztR^ak*B)EN#hXAQNxUpsQnW1RLIb}Npq~9o1F$`|h0kE^wC94$jN#-tU<vS6bW74n
zQ8^9qSWn-Y9&d77fpRoRSAz+u@JbY4LiKr;`1KM#P+vCc)XzbV5)^|}IN!d}2tY?6
z4r8vq-VYV8yEt0!D~USIPcgDtuy6A&@m!6U-+6x)(dzOFe0{b~JrShE-mZaJZGw^E
z#3D6>hpm6|XAPyZg~TbR1niXb?E!yQACrcP!JDYU;9|A>9c&WK+dZd_+A+9)ykYRg
zv%8M6Xf#x(>h9N{;@+B?jv#dA)kWQY9#KhtS8*Sstxmwo?z1#m?2~siY$<Qn7C!OG
z{SdjZ5Q*>+_u{z|@KSE1y<*_>vvfmjmW2U}pBNdhhu8Vew+0WBXGACzTtXp%pQ%p;
z>pw~>>@Vr>L^3C<t;X=Vbsv=)YN6mSAK49jYB#abMK8`$!)mi^eeRUNBel(@J5!s1
zmXT$Ej^xgi;?S;2hSwTCYRiyZPFb%|p{aRF5go5++FSI~GHaOnR*kz?tx5p87*#`}
zEZrA;C&}As#0nAy+{DEGUz&1k&ok~iamqTS_=?A|_LumB>23c*V`)wyJdBwI67z2Z
zF82)TVsd@;gUfiSJ6+fvnf=^ei!!4mhmPXBLY(fER($8v|3(TzV4m0yS&VyK#){o}
zq<b<Dw6nxnVywHTj1+7+`QYh7kvU5-@77eLSXJ<z2s8`LtG3d?KNayU=F(OMXwC$?
z8spSd#aBRyq1)zUtdwkUoH_ax<Gfb4+QIN4oUueA3o(pDuhr|`e!fvT5%pNNOR<3V
zi#pwS{BQvqQ`T92s?HO=WX3SP47`>P^Cv%jdOZ`TRn}sQ8MzcrnKEYnlja7VG<Z-z
z)y*eeCy?W?hR-75g)(wQ@2!z1eJmf~rV{#UAL(3H9#SJ8D(VWi73#|rGfNZZ=kpq~
z_5K)fs}eU8$<cISY}gpXfM&e+6_qBZsN)>8#hok$LkAY5YK7cAUQh+3fC8DepEp(`
zMUN&^IECitY<ti&(PGD^GCEeOy6|ehf9|60Hy$ts-~rJbDL>f^l%i=PclqP1D94^2
zpoKK6eO~`IW+ecdO98)mVA%l_sq3WJ#(1Zc{C-(xU<%g4HfR%XD+%VCz@zmB;2D(N
z0VOZtnB7~SJ2O-U?8kw$*L2jECzZZ1a0}(<V<{_JFH7QC8Obnk;{x4WgFk?s^sfv)
zCk$4?FBe%DN43MQ#_6`f%6pF!Ufz`M=RhhoSEQLolmv#C(El1<dK(^;$9Ov}_K}jt
z5w?D+;~B7u$1i4_nLj7v-CS+N+&r4`5}JB4n)T9=RE8_-!?YuLGpx_M%&4Cn@gO%B
zeVhscnae2@n6Jo#lTDjUO}a{g2O3A{J69Q@o>tQp>h1Yv0VnIg!oX4RV>!F^=>spj
zPaY|8l6i(rAM(nlz81JkebH@OPGssD1kBTK$g-x$)w%9h@5>wqxdyhMfrbkP>wi=8
zY%@yY?z~8W!PO*~v+Ps4y<fcu$YTNq-wkCsfLFFO_nmAlgS1uar-hM|ZDv!&hR?E2
z(s+o>*c+D_CNpH;+}Wz<n$KYqTd0Z3^d-=d5B@H-)Mpqo>@HZo0(0}zM|Pd0YPMZR
zOYNWYuQ4QAt*QFW7V4}lDYd=RTEE{9ISbl)$+=5mKWW;xszhdVb^TM;f*EU%bLx)y
zd+z$LXQqK~!Z@hJKNYm?x0!R48uicDI34<;?bRAs)>|y0pY{|CaTJ@Q?9=}n%!8{T
zIvRu5f}|pRGR`8M+ULaEa3bv`^bHX0HoJ7Q$<*mi^P@#On|dGf6*QQd$cNA5iZu+$
z54(NB|NOX*Jiu6zp)cTdEAMo`ik{<`A}u!-uWZY~Yz%}Fg$Xf#Sov1SLj+lx4*U*g
z?R1_^6Dr{Ib7(>pAb;ia<1%`!szvA<K%^U!IrW(zCFuI85L{rO#A0)|+*{EQGk&fh
zcCfNTXRt20<AuuG86*5XHTBk3`W{vOoK_#boXU5%{SNe9z?tEKJxD58_aIsOa0dV#
zLvCW|G&+aPq!)%jEe(10Nzf$`kkMiroJKiY)2Y9$4kS7mne#rCIG2VoLDQF9QkB7^
zhF}E{S{mh|Z<_~6gVD~J`9?c3ul#8}4@;KU^X6wPZcvG}KhbOdN>-$)gSib`taAGW
z0MCJSBCRN~&x2fV{W@UqD7vt{{1fG&`uNwX0&(9YPOnCrw)RIcvdxkIMWxuUmx0fY
zi4anWv9#;j1r}Rq@d8GE4AgYEi+jtMZSaTCd<BQuFsso;O&4OvKSG}CBNE%sSk2vV
z{<QJ|DPKe0u$!b*_F>p{ohE0PSbBB=9S6s=JmT3pU4973^4*4G$)p+^x-f;RNw2iP
zRy!**Mg|2dW|;F(EH!K`XPA^Tqmz^~uY@nK56HHXpe!0|L9t?&{h7Wu7gaB>gGwRG
z$lpIS=LV-92+$0sa5tjUxY4!AqLu<Rmwr2MecO7b=Y;gxV_n(k60z>E0qU+0Ts|ci
zOkJro$hZ77)?3D2)o^bJtpri??l3}%Ra&&S4^es1T{xhYn;(lGl`Om0jv7c=q%g(^
zGEg&2kL5_iJ_~($#FsCEf?tugq?c@`3)qD30aqR3ed!DwC}MZpe%@GzP|T^l;_azu
zTE=o2#-w2-`SK@VaDJOnV=61UwRh}t8q~1aR2YICc~seN;U`ZLg)pbnOm(Q7esYLw
zZV|Cn_4Up`y++#N?m{aC@77Bn)v}!y*MAjOC4bEKEz(J`tYyuRLguNwq$UU~{n+&v
z5sPTk!f7Eyy_;Qk=wk5Hm+-=j{fvIhq8ab7*2SkTNa!E21TN9qh0~kB9E~rBr_&s2
zLv+#Xl&~k!pbXh0>A{%fQ&F1ya{32j@z#N=rrg5E1UzpFM-8GFKax0yUCyrDx*Alg
zPAxRKn<0f3n*v^xJOBjv@M^&~YPf<G?e(oFFQx{JL*J|cR|Svo9h}Xlh`1FW1dlap
zK9dZ9uQvhuM8ntV#K?2k{ObEVdmA*&0OZT9xDpZg>w@my<;;`Q+U1*>Vuva`063qL
ziaXf3x_~FV6PJ@E?(5UL9W7a74mXvgfX1a5`^-Ii`z;yA*aay03Rl_aPKuQWO&`((
zpda)P!Pix$xnrXcS3D2L)KlB<rxd&YXPr75Kv?u)R8Ie%U8?}NB}EJ<uNO6C+dXu^
zL%+n+1+FW-%11OR&g|muGs$WcNd|?TNt!2=klKPJn@YT|mySiA`94Q|qEGpZFU{xA
z{!M}X$bs|#gT`y&HJ@)htISsR=v#4y5&!v(aq}aR{iMv>F^B?4DSqw**HeXZH9w$(
z<S<8al34PM$HbvLm#C3snpI{FV+??WL=qZ|Oym)<7QQf_Y8R+$BTw0(zTzm-yP;g=
zPeL?G9Ltv?1M9UOY;?5J_C$-_QK2yFjp9*iMG8ma4G!YKgY$`+zg5)oIjE#VhCx6{
zdeQRbz2^cT(523cgRTvpdh3jnqJcmJyZ{1*!}m&syk|nteuArmSl`!#38jxKS`I(p
zP*Yc^J^<^L`Hn|L&t89a?4zb=PE->bXkxybJjnHKNDTZL$TL1*rd_T`{7it2NP%?y
zwNp&9wV_b}I)ws!2bu0nh2f-*xd5HHp~q(aotkcN88|+A4gE1xSlkYf(DH^{HZQJU
zf%oM9LAD)}CMmK|-U&Q>!T@usPh_v=zwsH@vc~=xYd!b!Uh_NqcPHKpcPxb7Nu~G@
zqCqF;&>E}J2Lnq7uG{6jP<NIg4S+{!mm}uWt#OEXUFB~qk*;%Y**LBVrA!OA`qC6j
zeUMs6*HJAJK#LvzW0P@SM}%eplf0*03=L5fa;O=NOQ#eT%gsPJUt?n6SVXwPpXD6U
zVeJwM6hh7}7W-C{1<@IR0#n8rKV#%M45X!%-z~NQf&PoygKsZ!gu&t~7ix+?cV|JA
zr>ZC0*tcsfv%IG*Mqa-PeG#vC_Wd((1nf_K!~`-&9R;67k$%35)-OD+pA8WF?UTOe
z)GfBe^Z0TAn;T2Dhob-$w`=_3VQZHbN7saQHyMy_MyIRI`X1N&{O}OIIbWIdspuW`
zIpqV6IZu1X0JX+8%Qy3a#u>Ezme5=DgS|ji6t|@PWZm=YmC^Hj6+lvWp5`cuqig#N
zOle=Q5XjsB)__G+255>JOnV}53_rO8Wjb$#k2{#9+MVw&Rfqq@u%z-A3YO6Ff|NW$
zI5&ldyC?LED6x7cf?hl^?PQVdPxS50$~~#g09SyKQCa>p<bIjKQ`$D4Q20E^BuxBH
z^S_C)FiDV-y0z=}I}B;REa8uZAZJS}IUm#O7U2y)=2zon2wAqmF&hq4yG9x&ovB-*
zCcxG-t}Y^+mKr!UFQcPZaF>k?yVZ#_=SN1y4a~yU5@Y9Q!&TeKVG?wBLAd^uX^LLM
z(keTI!=>*0=mJ7>6DjJS)==KAfR!RQrd~bLV_L-y!DtjDOqDH%5vr;8BtN%ofmxs4
zJJ;QZ3j(&&OxhL>sYXQ-TYjH%0uins;P?VcMO4?ZS8(8opeM>J#sc-M%u3JTox0OI
z%E^!K#=IV=LBa%1Ph_X7iR!ntvzi`rBSDup=4uP;?Z18j1M+7E6euQP+fl9OQucgP
z@0hcO|F|F^P_bh?J_Q1(8hhhHkU>7Gp=?drKgCV+V$`Mj`tqmOZq58#4dW#6M|M0I
z{B~a;aU!m@a-71td9a0r^=<LmlR@zYfylyOTYctdJXgoj|9#&2x8ZS%%13%N48PpE
z)WNE#;T%;+A?TJLx5qg;r8Hq19Y3-65Pf^B*y+bDTRi8NZk%cCMZ}(h__r;&Ovi8C
zLNeSTZce|&%o*G5bC7HGs4g}2b<Qv^*;Svn7b@Fnb(<*`hN;GWj$G_mVYK*6!(BXq
z&Q!1B@JJwZ>t8%D1$T&!4^bD73FH-JUZSEb^^6QoWZHg`QSqN%l5cRX@dOtUwig<?
zhaY1Nvjs5ngxzrm?Q&Uj7@wA77(C{$2vOBMX!_?b(?}sGiWrr1vHscm<X=_i-_ZWA
zP*UUxH#ej=IRUgHYAnku0)fX}h>JtCF@8!`1xRNSr*#K;WX2LOM@*8y+xo|8fwymv
z4i;}MM&b#h(8^R>_uV+E*wvBk9FkEDGO82|6EYe_VUTRp(a(onTUlONuY8=_j=1VZ
z5|lYoV$l%5_n2l3UpVlJx{n)-L&L3gi_l+jd#cnD6e^AnCw~H<4z3j9$4<zL5&F>V
zyqT~hogAo{CpTB3Rmyw(R)|0q&UuR}6{-r{=}e+aha6lE8G{is6wU#n?)Rwn5D<6L
z;`RyCJR!fr`Q18dydc;I2ePhvq4X!T_OC$t=ZrlY174Jqayr4uwX0Z7$UKfxyV+{T
zAE};op+Cj(bVDY&47>a|+sUXwDZVMQhEeiSJmyxTdEN<OXzKBuW3z$peP$099IIZV
z<=?H>AY?`f8Y0#9t9HX;xK~A3H6U@0o(|R$QnHXyJRpGaGkZK$mgK+XTBloRN-Xa^
zzrhOK-=M;v$twYSe%4fY-6X0cEWlVpUHpgh1lMX-2rebF_WL@!Io_^lTRk|HYE>xG
zKwH=04mO3v=~rLhl~5+K3J(e~FSoa06zJ5>aVJ|-?3jHmfIi~cF(<QJPcm^T9VIpP
z{7=0FoGU&%K?qJq|C0Fh$AkZpmHj?KxPn=q2DmV;#8tg}kW+?>+KI|wgy4k`FE(at
z;DzLjp<T1cc!MB<isgkYh)fb4Y{4rR%@q?&&F~%3jAz}yVLBvZzM95(I8;?K*AMl9
zw|QAODvCfiCa0RmP`!2Zin1haU_t7}YS-F6Bh3ni*+$VgW7HB0LcVr%^jasGFA6o%
zyBsGM<MGaJmQ1(Yde}m80jE*REI5<oI_w|l&-)ZvGN5syHNiMk-A2H}36PD4sy>^9
zQkMaWPQ%obfcjbND?&!OSs>4jOhT<&j$&ZfuBZdxZzIB9#H&ksxt@I|``<Tawg&#y
z=uaOVfBxt7`laRmhyF(p$rv5wIf2M@uy+@#A0tAjU#DcF0`)%9o30oL!;PO;Q4f8b
z`VI@Vi>udK3^tIWCwG#IpbY2h%noq#6UV>5be|GTlGk}j_Cbmyw1G^XdfU9v5ppL1
z&5VGvDpXXu0IzqPC*)oTp2_QrZ?vPblsPdcdg)&LECSW(p%bhGB1VgP30p6_@Wnqh
zyY8l`TmWYC0ze0>KYk;`pzNb^iV}c*i01p22&TzH@ZKT_=V2>`g=L0Ia=q0!gJUSu
zA)rSj-GAKS18&Ye==Xp8Lj8M4H*;3lK25(W$Nvl-|9pCi4fnXDy4@hi{_+Gcx77Xb
zBN!trm1RLtfu+olz%0R~0}4Oa&+HFXTFeB!ce)AHB|ina;etgE9;u-ah!Ve~!Cbci
zY7cJL1-9BZQZFq8Y1p#$qq=&{-Z>nG3<;S&!OjAL9t{InvYt<bw81<H<_$*s03P)L
zl=4ad^D<^BA~~hxDk51j5KZ^z&5y2QS>ETX{i{l(C)mXUy$8egK9g`wq0}c{ySL0C
zwub#Jnl#qH6b<f*QM`>#>-7}O^VF`iKJQCMmB`Jm{4I<=o3)Yp%L^bz!ceLmK#=)`
z-)vp}yI1)`cky>*A?TOf*CGD6jQO;4PTczdX^b^8zzuO3S>sVvD0jQyYeeT`+3y&`
zMBaAQdC9b57zc4pm5}<KZ_1p6dy+p|Y*0smiTH$t7CRm`5%C?PH^+8Vw3t3yd?Ed0
z_xeBP2u7JuF7VxQwKGCk%5$|U2;i))lI3qSeJ-oq7uT_pzcsQr4yWN?tR^PP9*ly<
zdzbd-x%~WS<9P8SmIdFRKN7dK4^LQ!V{jPE`eN$gMuJTh`2_w3HD7(M^0~mrQa1eh
z0pG5+OBX&*$!(L^z26iJ7fi&Tc8^mslPjnGO4k4U8vZ6MIiSGh<X$Y!mZ89Yx&`R~
zPq2UOQ<fO;F@w$h;bYdjpVxZ}ZewPeTgVA)%t(gs1fYrCX3RAG5W{1)Ve2#hjR=+d
zqi$rkdL)#vuHMaV`CQLgqmWXAlD0<^SI6#NrZUeMBOr6z^jbZtfo%+`!EAWloUtfG
zjLN>_(3lv(foVHn;*e(zJSgkmeQzFYy(ZFua0A&|l{CKSN7paz>%7t0l}0dwwi=7!
z+qSb+IRe%b-Ei_=owAcbtC#z8!@=dANM%0O9C5g;0SH*b0furiV1#ui{3V8-Unf2f
z7Q&$F@8t3{QXYIKu!Gpy`y0QH?LWimuO~Z4cm|g4T(bx!G9J$jbS4y-_5Bp{7CXuW
zzeD;OVQxbH*Vb;yBm&;Y#Oj}rhoiY2<91xB3j1^KX0F}pkgkT-&ZQxIi=T~EtT!<o
zmqWSTFgq6dnZettfT0u342fA7uQyE_62a+vdHYeBU<_}-(?ThO<7jDXT3)xP3UEXm
zbXRG~aP{BXXx6C4tD2K({RN&rDl_5Tx<{Dra48|PK*O65MW>w1G4Lup4=kr|=)w~w
zpA!deXqkLh=s}4fs*>|SSP0}tM9&OXO32@|%qc+btLv_SfI=oDw)Xo7jPIjd*N)HZ
zhF|&oKCORzFMnMh|K?aY^?+l?!u4*cG$B0_mmlx*u(FO+H_T;>uwPD9J|uG?7HGbb
z9(J`0=Q;A^W=3vU8-&gn_2<~L{h<Op%vWO^1|($~>Z%p8Y<S*==?s)){b+TBkPdR$
zfHs8iA7g_t<j3K87gBuDr9Az&yAx<(gu*iSOjHh}_^h2|m7@X01b?qE(}T@c5emHT
zAkSA7T%4xkgk!`mH~95KqWZ@Uz<%J_I_pV0P`^F2ZwT2LFW0|M31nsUHdAa}nW^d`
z2SFeYaY$!<?s&1-?q7Tf=*%_1>-5c!Evs{SIhX7mr{Bp%!cY<(U6NS7ZU1Y;^k311
z9?YAXa?R!L0tS*T0kN>)Sz&A#H-b1q4p}3DIF`2;d7j)DV^ojW26>=aJJqb2wD21B
z)2>g;_pH>)k>4+eSTw7hW2I4g8|5k<^2S8zw2(0%kSIUJpn6+eiQD7h#Tpe(*BMe{
zhRwg#R6eUm+`1vNtYLQ<wibC}Fg`(VFwP6uyjCWfg&D4TXw|u895g4}GmSt*!#mL8
zUl~*{4hVV;;1G(jD(F3SK>g7GzA)(d^B!y_{0i8>o#@*FHMJu`?C%eQhiC!l*9epk
zM(^IXuo<t1k|)I$%l|$B|K><PUr5Qj(Gl(MzrUpa;WBysTG=R#ga*plBT};ebeN$+
ztYK*9mN5043lsMfb&a4NmDZ;U9#dHkvzN$n3_6dHPro3&B`Or|_3^8ns^1|7_Dtt$
zWsk(zQ~Ibr$|!HENL^{l4qEYYOkW=3v7Q5KWTX`g?;aL?^Z@av^tV~WTuClUxdZwl
z1ppChm55Tr%|S;meklDuH~r%x+L^BxBQFx{D678*;)_)QDCfbt7Du&V>n@F`$1I#p
z9H+<xuSS6T%q*Qf6#Gq_=-DSetys$YBSwI6QfF+4E8bni^~Rg#cY#ubjd5<7*O~b+
zu;6PIS~Oqn9E__ot%nws*k{V&UlM}<{DAyF{HM{!810JSQA#%xUT+WUQ!3hcj+!4}
z_w9!XN&Oqg2E6O4d^wV=GzQ;!jC#9p`GHe;?YO&cl4|ihDA;5rf+ajA)jX8JJ-l3t
z{2^){Tc!}nc%zjFc`EqwTl8Rj*C_hB>RXaQj=gs`iUkTDP@++5*L~DMi!E=M|DYyz
z0k!?y?4qL-4-7T<Ahy#SE{V-x*zO99BrP@$-%i}=q90WQaz~;Bc{?%di8nzCMB8Zs
zwibYCW1)mbzt9Jy{TgGWJdQ09bw3F57&hKkf}ZftYh`Es$01pz1o}SPMSE=+H<^C_
zyN$8kyhdCs<=<KUfB2ffKOnLBD>?^G?P$=sIW*875lh|QeAI>bR0Y2Jm8UoW5={Pa
znKQ_u^8tZ9!VMxU{JLA4KwA>hoilu0M|f$I#}Xpr)6K54YEZ=JLxR}8I$>>goR2?@
z<0rFBV0G97DnwsM)O)|Y*rMbN1*#!TcVtnq*>MXlJFC*(CkF$I<2(c+HXMQSsb5_`
zIm!z!1rb{@*O>NT!H1@H;M5+@fk>U`D1Z&=3dL{4r7T&C{qP4DafUnNC)k^$GF@ji
zo-5Or#3fJb?cAFNN7;c}rLuQP?NTWDRCvSax4(``5UHLTp4VrIp9jdBxV^3aeTu(h
z)Zdk_1}Qij{0gfJK@z+ndTqLW1etCEOMktU*)VwtN+SfKM7}Y_O7i1_L_@0rkx)Sz
z54$mX2fQ#u7Q_rJAtvG09Rc5XEt;P;ogymAsolR+@8PfR3Mz6_UKGdTV;3fOLnbsM
zy9tS<#|DAuJw(k>W+M#tEw9;{e1|*?hnq{M3gO-DM0)8EozYCu$&2-o3|r7B)p-K}
zN^-_~Gw>@h=S&Zldy2iy?>FWC8g`Iz>b=RKx0hI!__MLL7kV;1#apaN3ULn841VNh
z_4|4`plnegft*+DWLpuA3T&Bh*>BA@o7did7j+hzVPWh;`15;{?ydX(hsx*gOvX#l
z57r0EH=pFRvlCz<(*$4;&DEcyY*a@oZveAIp0QiH$Uy1rh1_eZZyI4HK}mTrAP+ef
zAWJJ=oyZR~7f3B@>C6?VwlR$t4Q$17okx(t{_%~k!R+<+lOjSdlnB$ol+wp;bjNpQ
zpN{&3MRxn#w!|?Gx*(YAFtVZrnw=^Bjy#|pbM2+DB<I!}0J8oaut1G9{ypQ23+rL=
zP%~I}?>cJ|nom!k{0R*Dr}O0T;o+-<<y~KuNmm&&KEdn7criTte|`mV3>t}9qZ6_p
z{pD2lQ%^>>cN2MhOOZ=Y?+E?xO(Yq-i4-=mgzHx$5=1uu+JNv%WRaqsHrpZ!iB@7Y
zYD}dGC8`TQJDHmbX=fu(o*%y<LOC6Y1yR_>iyz%JLY4DbfLB$1dBFQq!n6xTSqz$U
z8=j#8AYhG&w(dFx>oU{9y3C$8PgGzu!u8>i1c5|n<BZ+ynYTO8^ktm*&`}|>DH8kS
zT9YWTRe0N<-!)PLVG#c#_r3RLyu9Z3_5lzv4-Cp5rI*;f2JOK#fFGQ(YTSTWX}Bp@
z_7I?uv%WV+v@W)9$T~oAY=h#H_3?KdY(oKhrlKnJWt7;?{pWwGi~ckE{{58A_6xtI
zPbK{@uOZXzM^2xBYJf1mj1puqG2PS)5Z6WYu+$t#y%!XaHIk_zLU#+xTb<AQA^ek$
zJByqoD&OhC0w%d+bTC@=ipZ34=I2&sZQ|?b4G~h;W_qhMIUqq9b|km-WduAT`Y9bL
zg;Df-Q5hV=!Ns&C?O-hHL&Kx8LPo>Me)XuS`5PEn3_W;4`r*$eN*VVDVCc_*nc!3C
zL6PEtbX1zm*Y)2+8vpqakeJza!U3{<ujSMH?=$%8<Ny1I0yGcn7io;Z_u1WTDZ=VN
z)+NIB2NHs3EQ#nyEPmBZYY6(Q1VF-ulTU@}JR6}bq3?R$<Gn_*j>v^{G#J8j{W#P@
zDAX=Yt7;L9?xl2!Ao5h457SMk;*BCnIsz{0(d?kHZO7exG{(Kf6kSHvTI4Bslvq!d
zV)1zhcgAd0k12>dR(j^+z^`VB|I=ee-Gy%p08A9R{3Xh80Wxr^AAUd$O$4c>x2ZY5
zpa;C&OU(1@-|0ak9gYeFi{i4lk%Xk^|L+Mz%l{XDwmXaR!v_t)h4mfd(23W7tkm<y
zGONKKmKh8GO4sUVsu^iX_YADm%Ncf~N+%DyPF2r6pUs=mGZi&MRTPnwjEUz-pUh=&
z!D7PxC`XY}eofwm_I^I+RUB8HW-q-tLWy$;Dp&ofbh+EdbDJ+?Dm$sznxF61*@7A4
zz=zh$b+~ux$>t=>lRLWrJn4k{-ANg1MF6V8g8O=grM^Ax>mIn8kCMkU2=?jj!^a0Y
zs&&@{dCVGT4}RC_*$oVfS<zUYygVH%lZxtd!1ZnZ@|zQEcjDFQw_BIg)m5p_{!~f-
zoqzusGvNO-m_AN+Kb5*=#*0)E1=mIpVfl7DVYX^HPksI^O9?2hjWf~wKAx$kGg}m)
zehb=WAls|^r`nXBLG^TcEIj<lIIBmo0jp+|3yXz4wt-QT7EwJl)|^<PLYE(F;*AcL
zRZZE2j~T4{Lmk&N#QBLSy$E_`u;MFIF>A&>iqb<SOu&H6m#|X`_E+NIORhKRLZjrf
zhyrkTN9%dO?vo7Xo!N0<5oia8kW!(W6Zxm-E3pgBF3|2`>g7=K`jrRgY`;J8Y*)Hp
z$AZVx!MNbGvUTTl?o*@P_fp>~KztjbIsKEr^mpd`@3VPK1P_T-1kay<R%hvTRvEGb
z3Z1-$lLlUvUz%e)GnDx1seFf=SKye6J34{bYA7`<XqLn))wngvEcBk5@pjmP9=od^
zGEd!;WB2edJh+GGxn`Z<JY45#s)ep)s-E=^Ed!$G{46rFD}(J_hyZE_nQw$?Ue`a>
z3=krbEda^}qT$v6BQs$RzDLO%x~6dYiKD8L`C!AsGg~mrtaJe~AejxI_nF%m&x3)7
zJH0j!K}jJH-a7h^kDwm+kCK8la*<G8?{s|dblgdMu6HV{*T(*TzdSJ#cj!Qu<1CAf
zf{)UQxPHBi*!>2lcM+8?YBE%rE^AEXc*9u~6?Q;FqU?Ie!VHP%5e%TAbi?>qBRP8g
z|B89edhRG}#3zI6kA;EZ2r4ck?{bzRB5Lo=2FzVUa5>I815mNaE0aJnxFx(A2mDW+
z8zhcX1s!u$!2rDfyQa~cO=)?QyT{pgScZ@b8=(a#I!*zba}J=2+NNcYyWkE%L{N<b
zf0t4p$v_9JX`;Rim*N1I$Hwx8FTJ-R_O`fbbv2g@byXhvn={eNZrjPj4KIGW7I`0B
zT$<)L{{Gzlhi(}RpE`db`Si1&H2$RI@}e<RNDeU)%ms*VvBsnLEVC^JDiq%Sj?jt$
z+u0yahX~o33167z<qC9IhAA@Uq+^IhTqLozVHF;Rk%d%3Vq=0gxU-JybvHNpDxyni
z9}l2G<;7!49Wm&T{K(S+8%&WBrdi?w+3vXCi{b2Snx15;8(2Je2pd}LFuem-eW=#h
z05o5^B+3Ic#}LaAs&sMRR=C?k11Ly`jg~rTZx-rpQ%EYzOkxapf#8x!zdwSM{W<Oo
zRtBx<$B4eg$ls-vzg_^m2Yih_?i~8r4shJ{wvGCrC`^fc=&_lJIY%+S5BR?$&ac@0
z80QaQzZR~au6gD#0;pr3{Kr&AZp?V)hwik~JPWukB5Xp2su8pyv?95NT`v)5YiIFX
zI^d<oI`;C1HepPJ_hml_w7v+={zgjH$xsyD#k^6AXmdGV22sgUMSAI663*C({tt@g
z5dpC=ig-_NXakDz<q*B7&nzLbT$ft?Nknp00}&Fv8Ib5q1FH)nv+95-O5jDRSn*J|
z@mWT>aod;M5IZZ-A}c8+GOLzJB4=|M0-jDXfiRc`7&dEiQh;2ncSz?&@sGsB4o^%;
zSLC;0GaZ*1!a@W4IsGepRHnAKeBV_3n@IPcQAk7b4_aN*f79w@#=`m*@6s)E_oGq6
z1d(SWs|eG=)x75xE8kX!d&60ExATAry?Gd{%i%_9@h}WBB1bqiL7bvGA&-Iq<20f)
zA)ZHYohNCi$&us#A6a$vz_@1odE`yGc4g)+&;bs(utC3M8^4l5i&IQwo&)d-Bpvk)
z?|Qic2k3z@JpSo_2S#|H*gDLz`b*Y~tnT}rO<!;+zHaZ|0?Oat6TC1fB8SJ_F#>C~
zyQNa&?qJp0>pxblHT<z^jWbhXM;T-u$LL}U6<AMa3{j=5x1B7KiOBcRXUl;xJSc4e
zGBLkK8K`~r%h`!yMkLS(x0PyY7z9}XNhv*r3nw2T0u0v!R4X866Nv><bh#<`D+D6*
zXWKLO$MD!v>NJrniw%ZgDIlIs={4Btem`;g@ga(WkH{4LmGD*?0ErAeO<RqA_a#M2
z@BlKE{qF3JS(!;~u8IdRLl}a4zfGyEYW~jF|3jScucZHDg!AJ;j)N?=0rJSz65Yn9
zEDp-oXOr@vcQ@NOh~Bj_i^LW9glO6?w-QCa)pyjcOZRLil2dHfR;?#$*NV?5j-W_{
zK^*oNJ9m`1h!q1na$y4n0)vqY!#Pj0806*)5}_GulT^p%(v*~hk^oK1GxOQ8;=_kk
zSZ$J2SW#%D@VW&Ftdmq2TQ4+}J4P!5+I~4`Q8tq}3`T)pjND2ySrV8Z!$&oVoi1E1
zx*(2niHjP)`+dzHL*Q?|hLAI2X1$CBh<kq|jMT&$HQ>{T7TPc$7X7{%|M{i=dTO5`
z0P(<4XZEH{L$Esrlia2Y+wTqi2!eJ)J2Jd(5VvE3XBi19Q#uFqZf=Q6AG<*A(`CjW
zix2J^`bU%URd+&y>U+F5@9PsciF6js;@r&pEo0p6u`ofBtH0)#GyaoQG)gXnC4&QE
zXv&(V%Acenz5*9s^!z_rMO!v#u~j}PV40Wc`QSg_nZGIgJ@b$yO_elic>}Yh$9ZyV
zvF}GMVs5m|gU<y2FF%mRp9(nHx&e}GrC+U2ywFA$rbPDx4N6g=o4C+QL6mbPbyX1?
z6iPe7-?SW4lTx>Cg8TF|lWE{K=gO|`Y1$RbW{>Ccf9OQS6qP;d1{b=agQGq#da>|8
zkj6~bS2bS?ip<KC$wFhv6+Ry|$t>0yEj?!n-tT)aSbhl}54@Hz_8Cx~{inRlzc~=j
zekl>gczmPumn4mUzpl^+8p#r?a`hLZ>_0E;?<fBVH~@-=8*dR(Lr<bW{Cbb^qaz?r
z5{g$XIiv&_e(HT<fN08F;A6$SN_Zy=;5B$Xo#OAJV;C03R8WK?f*`K@q&l{e0cJq;
zsmkHY84)}d)3@S|5@q@nKYQzu86)i!@OBJ3KWC`P;iT*(4ZYo=cBEKHuG*;sC%@EL
zf~9cH>|h4(2zZbS^r2LyZP(wp;boA^Eh^);zduv2-CxsqecrI$v-h*Wj_^&nF%S1T
zMBhcY-vj48JM{ukxxaT&HoiK<_OW;Re+AnCf=F0sI|&1<9g#p&ss;H*Csgbz*7bHF
z8KJZzKUv7bR%D?A>~7yMTSp@P*vkmn=qfU$25vJ{Ovw-IdNM5pWEhT8*pat5q)?9r
z<Y7B9bx)%pbOh*~A?>)sFCR#YV4a4oO^Kuh3N_?&XElB1*bx5Sdi9-~%b@8VzafyN
zBzcEbe|BA%Gko#c91i4yl`sq3J_p$PjWIY90xTf;*8^Nrr%)}UOsn)s18@aN2a9#0
zX~kRD$g~D6zql^I%6)&{Oy@|W0~M@Zz*L%m!IajEe4uh|0Q4#y;2{SCl@!GH#w-1v
ziPWJ-(YtqtNq?4$;Nm9%Sxn80_QJCFk*_7QvfhI|YJFbwm*s}nIpsfhh;5s@o;7?P
z34~pJ>9gQ)-%7kSmw=x+I_BQ@zjmk&C_wLs7l4jlNAO^gK*6fC{eg!5bwFz6|4(!f
z?aKe6gXCK7V<>zyDUOhilDJm<Z8093M(4f*>h4oO3U6<A*|spCNaTdCn2Dwqo(9Av
zw5w(8_p7z#yjSa4zOB?iVjupHCj}JkaD=qLB<9OXtoXC13>TmawD?7lGoJ=yu+q<7
zN5d2lFyAQg2K*?B<_KrHZ`VMp*id2kPA?Fxg&O(+)WF2KeE<{EUcSuuWjZ9Zk*yLc
z;@K?<pvdQXU%{Ki<dN4gI|iN>^Y%VR>90@cKEPq!5wN0+DNiMlnTdvtRisV)<{&no
z$Y=lO_ZMyHLKQmIoj?#-S6A5!bp07|>BoG*Gz<fGQVdK%9T@-Wk7{K#7-4Tf;K0|?
zC~~1Q`dDn_Cq7rYyaY<@uzuR`&5SB!FIxDJ$A$8DYWy7_{sBDPMF625*v=2*n!DV6
zy(|4uW7z+0Xn_ODV<|q;CiI%ZXxjR7B%AH0T0BL-efUAq5;p<94`S;^mGr(ONxJI}
zZC|&XZ3mY<;K_AD=c#y>d#Wfl>*!%}K0~aO&Nb5N;+}?%Svm-zRz{m)U8!r5qaI9-
zl$0_qlVi+*eae{~j2-zSH;=SJ$-OU+!E2?`_+?4$6lZLRWP&gutAcYE_Ker#dP82k
zCW&1;*&<;p6rc9laQc1qD;oo^gPvcQxYINo+?=X3zBY3K{8NfIv0f-OAK-4ZCRiQF
zHIn8SbE>~+ht^DCZg_Ytz;TLKzuDQkr1f&E6bAl=>Qyj5og`ekbH_lh&I%Z3<Fe<3
zraJ9Zzxs;CWkNy%rGE=Xf2x%S29OEwi*G5ea4za2b<sOoddAM<vZon)UM#y#i)oQ@
zNQj{-Jk=t==izj}3%`#IbaF0u{y_D!09T|}c&_%xyCl2ab93ltuKIC0VB|JRbN`t^
zbLITRqUbg-*>AKMPMZS^<lVzT?n+MOtNAkNpZ0<sjBWB?&aTw;fy2ijzfq%IgEw4r
z1H`6-bw`r_x3^3SDw{xdJwT_o@#!aZKsQ|ung@no#j<NpRa#J*kdF!H*=C<FDjNGt
zgbxvhBpmFxIoi!xdG8a1DH=<OC-3-(B-c`-4_a$PJj9+FsPKz>yKEE4kr{x2V~3)_
zOK{V{JUuzx8=haeXq(3~j;xQB`?e}?Sco)TaBEUdojy>09bk&zP-6XLsFMM~cLU&x
za1#803n#!8Gk<Jpx7BRXx#Lg^@mEIOJ}_Gu%arD?c*d&Qv>X^k^mY~4y3|tIxA4qp
zl$4gE;NuDJ0abHL|Gjw}Xe#g#mo+TK3g`nXI+79<I^IAHFMup7eI)BMT{_xooEHXr
z^&SBG?HWHlS1zmZ0+dkK2f%Bb?;4CP6QME#8XkRwMS&PzR|{W9=qL=_mV#X%?6_~(
zD~O~mhEn69v&FF3peO}EgK%grgG(6)v~-Jellkc&Qg0IXe|owWutviwzJpzal6p+r
zL{bF>007NOcPgzpD7m(KY#v#$)UFRSRh&XN7&5V94Z{t>x?L79yOp#%4bLtN^o3w`
z<+}X&A=(?0|9jJ~58o~hI7XBIpEyRHw7#U=Ya(OQ^Hq<8r&mfZs#q0Bs^|(UqXeqS
z)Nmc8A`@wxV0tU(56AklS($|{svca20Y(YAkj*iM**T`frj}XW+o~Q~$xw$;8OyfZ
zI~}=dKOX@nTxez!=;w(O`u;M*r7XeWYqj<PuQZvCJ?_M0zRntLJcwQ4@^sf0jLL^}
z(7%4<+%8GVx;ZX9Ew}e(?~>1$s1dSy-P}MV;&U-*tOwXBEBW)E-Z~|2n-#r#vG=Tx
z`h|hVUA8Xv%9aUc^~~4K1MNIe42mpX)Q2h@fcJl2i36(PkMWWd7{vJ&&)rW=xlm5b
z`mtjPaae38>Vi`HyLBXqw4edA6phZc%k9cWV5KEGWgoNX2*{bY(W|P#6cWyzZPED#
zlg}Pu05sJtZ|6{|=L$+9^zX(1>T6f3ov2u8`n-o!o9X9ILUZ7eqL>_aD7Tu%Hl4&d
zC)2odVDMwld(RWo<lab=*5oVYlFM36E-Hid-@~K-xgM2}{Y8YiwcPP1IqGfOU*sq$
zz)sPKl%~s?RTtXYIUl0F)f*nm8O-B`k)~W-m1u0m%;0%)RN}}MTXmQ7W$;}*7-x*o
z^q|^`oe~pnFP+{~e9sUS>Zm+391|SsPQjx=@ytkw6!msJ<ibkL#?(n(woqc<&7>m$
zHD~joHepeWbX2COCo@xQZ#3ls5IiQ9p=rq;Hhpj2=71&-xKTl0JpdeTUT1@!9VgTT
zQl|8S{j=3(W>Y^c_e9#im3^pxkB2d#%$0wKi9d-kM|+b2Q#OL6xDR+4$wZJ~E5$R1
zDG-vpDRFz_MBvZlryv}Lh<BZR%rBuQq31_;UU`K*x-R@RbI#Mka&#nfzh&mcy#B>|
z+j7s-Oi}G=KHDVsibqQyC@3iH9i_1l`vIS;Y>jJEg4C_szK(BGZaK&p#1QMTY{_rU
z@C^gYTNyREY6Z<}#y>A-HYb$1%U5*udx{w_T%Z`vf=SZ7^;`y%owgbRtr??fC185k
zNBR{j_5{en4CjH7j;&E2pk4Cp0*-XP{tKj7^Atk}oC_uJ15|Bkg5E(NEs$!F5V1ur
ze`>8Vl}u?Hi^6?H!q4VgtImyL-S&Xdtw&`igy<nr8L2<IuR6g;TngSdgCZRJ8v)&<
z%ukX2kN=Hv`qS;tatN#Sj|yLyFh-+)M({w4dhc^6vVp9SH|=M?l}D)Y?sC24ws8gu
z(f#c(#*BbEbVt1IY@;I+s>5?D#<-@#9K#=^_q7G=owv%zwej!0WXWWfHNwu%=y_T~
zEVE_Rr$bUe9#PtpP?EJwkRlQrbwj?mDo=f|N_jP$xE;C{eazFEdna+@={CMUx?8fb
z&xO=#M-m0dc*gXHZfAzair%&}22QHc1PCSwl;^KMX$Zt5TXwNW@IQNhGTkgcR9V%D
zC7@l^0y1WgA^`~8jud1j7)bGQPe_FgK(@pjSK#YaybD(_MYH$KwSgoo@c;1k9zaoM
zZMP^n2$D0>gjO<=qd+&3C4&f(1yQ1afReLhXb_N`V}pVS7?7ysoI@LcCZmj`R%i)l
z_sr+acWVBbd+t3|w~Ct3Rn9c;yY~~;v(|ciTLr9qr%k0RvlH26Yk!MltEp{vL4mbR
zgWP~RUr6KTb9Y)=R%oXxUhwec5jsKC2HGcoaxggwj^SHMgq%YR$Ac>5des>&xJdg7
zdIt2dxjrqn=Vk+|2i>pjYWZGPc;?u4w5HP%ZFCU1Lv&VI7}A`$o=EPa5dIqxBrj@;
zy%W+dYqe=)P)00zDzkgq6X@5VF$5TcFzyML+eRy6%w~;^&8PBfS*2MHt?k-LYaex$
z_TjJl;<S>4YlAh2Ah?>+0tgxsZdIe--XMZlm3HTS$tH$ifL{`kZpT0Vu+djRcNTN|
zCdkK9TSEsXhz%>xqB1YT1MrL?$Kw=Z$#=pO2IwBHxMfPT9i~=u28*F@)jw549#3ap
z=lv8ppjUqGbGo<Zgnfn}75!tnOZvtb;XGvtEK--9*VfL{I0iiy?Qw8d_}IJ!Z(Q>z
zPvpF<{ZfLnTlY%*lj0*Hgt&5N>EdMl*+FIpA>0b%1$+xD!0bbon`f~xsg1#ZrLeb4
z;QTmF{*jZ$WzX(U#0p00qBWcW8^z6)#@Fk_mvRuGIT1ct;e!eMe8Hq2<#gM${ym)5
zFAtFCoa?)pger%lzu;2zy_Vgc7qzdlw{mhB{jSTNf<+%w=ykq(5M@Be+XQw2UnQEr
zLJ^oq^G1oI5_?MC_wl~N?|vADWI`&rBDwx3WegD{=q==)*M*>?qqhKV??>s{J>fQF
z^t%wW)7g!=kSEsJuRx}F7`&qNEFhk?eLwc{bJ)qI0XgkdH2XbsveMVJaW%@f$(}v<
z<U89ntsg|Q+PZ<8`r7Y748MKr23oJlhi&`$ja_s)>1bsv?N30C)B`k~x9^yp8RXT%
zN%_u>kJn)ac~Xz+D?oox8T<ZcKua6NIDqjl7`5<m)2JO_`k)AgWl+o_H#pChzI_D?
zi3^o?;|$WxtZ6w@iNB@d$`Ky}uz%KhscwMd;m3N0X3tgDS)E&CZlfU9`i!ZNW>gB*
zKlmJ}udrG6g1S;CculfU;Ao5)W<%iys6{RoW%Qv;v7KLj0;|V2vF~BKO3hE*QkHz^
z{3NMAzgntJ%Mspj!W*9c$6y2E&{!Cp#?LL)ns-f;fZB&x@$3FR=liceO*AODprWuN
z9yDAC&VHh`nSi|9QwD);Qtm8`n|GDuE4$!WhzOY=e%GMARtaK{rUqVZnBHUUl-ZTt
zxLxT@E9=W}e=L8pvEB0yxhM99h<-EWYtGP76HKu%Y*kyn)4^OBU<C#{eHRtxuqKQ6
zyfUw8{5`RqfZJnoVf6EWY~x$iRxpZWW>*_0y`^D$!6Otnlkv6ri<o@e+GKWXtFH|J
z@r05goHVzo@aM<%X5H-JDL|wWR@&(x?2hKnEq-R<J3LILIQLu$Y?2u)o>Th>S~3dp
zZqd-A!JGh_{jG{Rne(fMU{|Tn(HlccNoOVSYU7{2S<8|BL|eWyr)zfuN+XN8`2leJ
z;^}3ozpS(df#RlCUIp5w*#<mp%jY87(rJC{te!oB3l>yemkQdUu0(s5lYJw70e0uY
zxNV%KNZYpp^}&C*grGLc0u+}CV~M+^oG1LBzy(uhr5ymhi1Er1=`LABO10OE$;`Hc
z@~b<=GF>9Eois8s615ay{H8k(px-OZUf07r)V_Jgjz6)y_ogI{-tbOg@ONVd?oi`X
zs@)%CH4lAnj_-8B^8D1E79_TZxBvX~?cDD9HrbD!ssR9sNtlHPTQd5!fTHaYJjTQf
zO>V{H7h7dQFNXh*KTgTud?XGj@viXMjF+5QWg12niE}nlG%nSdD2$!!W6sCJOi*jR
zN~t-g5^aoLifr%UHMsCJ2jNu_sEdf<^oorUm^m<}Kcg#>W%?;z#iT2|msPZsK<zjs
zQaw69adECl&#v0bbLRD)ksGG+sZ4jg*W56ZZV)3Ppbf2Xu`o+)254OAINVM|cJQWO
z3|_W$TdCje)@A{-TKgltWmc8`wS{#ccFLg{zGpS<Q_V1A;rh%6P=KB^?W)CY^{}#^
z;as?N7P64>93=4_k&{<wu6uTcKI~b$#8>ycQ`QUJ^_$h;%dMQ6+-KzueHTDX6R-(>
zFN^*GEEwJ`<En>we#hl>y-ufT@ozxn`MdtSxiP)&oFbd=Jx7NIBg)y_4){e7ft-?C
zn76u!r@b&*isRhp6jxN^W6@ymEk8&wCGU|cv5aE+K5)aHg;>94L(B;Vr-DxqexWrU
z;|IV8UGA<@{SpMyq>_p6I)naEVE_Sjt;f|M{-52VA*sLyQ-LwTM5Clb^*m<AR_P{x
zvL;Obr=32SMjB^D5LasNp2kZH?HJ;9akfxdNMd$U@TkW7fosb-{DdI8r$TLj-(-4D
zi=n}WZ6S970325jcv6#&E&VyI+?%H3g+5Hp5>Tgly8Lr>Vn^_do{Ru7?Cl}5_jw3G
zMC*j_!xPiR)~KDyl>~ab39>PSW}CcasjZ=sLWD~}$OQ9P&uh>kB8*~EueFCEuws0u
zmBNN+xe-k$lStRc@v5w}r1Tlo^tbUXVo^lWvWy9IDRYR|dr%Tbv{Or-$a3}hdXoAn
zL`xJ0UDjBIpS<PDo04umKX~bT_d8^Y@RvgpgJqxvET>^i`3$NBNnN^I-Qe$LJ#kkP
zU%X&uCE^#VeQiH48Pecanj~`#R0-zuJy}5?a)z6#O&^gGsxNjb&f6&gcAcxf0CEKO
zq=vnyUSzGt^`6F}ky3c%d1JLpEVDeG(_x7om~X(4RFKK}V^%r(NBdc@SCUGS3B{v3
z#1N1_iK`xL^AnhclYaAvtvLO5c3LNQGp0tb1nve3C(&*~)9_xLCqh(FL>fEIrJVxS
zZvU>+nOAKY5U?F2Vq6wDOE-d2h0!1N68s`Oun%qxm-O3G?7|md`LB&m@>hwd^gnf!
zEE3R?Q=66a0fljVSD<&}7CB@n`%5uRvsVs@nXh&p-TEQYAh>#st?uwuWINbR(x<K6
z(thh1nc^uh7-29Q^t?(oJ4Xpoe4U2{(ClLt6Q<b_jF_j#43llpF8UnXE+FuhQm71&
zu)B?Yp;xSSuv|2{gnmJu750xmQU+XPfN{HuJobzrwTR^ZYPV1*0d`no@e;422(Y_`
z71Zt%^oEFWrk{qI>}-TR9h{tdMi0ZN9dzCoiAz&&G0NT@^e-_Ro12n#WGfR`U+h|u
zro}fD?xeZkYjTq!9&@l-e4a%(irCbFg3WnI=;fN4?bb!WxgE&r!EtV1+*URYep=@w
zz~cKACaJSXD?22|u?%3@+*{GpdP)R6rQ3m$c)2&NM(NtiN1T~K4;W0F{N1&#_2tLg
zWsmh<f_Pd!=!vUX=={WPem-p9c_dnv3e{xFeQC{Ph?E0_-GnByYS;K_=^qz&%t5hF
zv1CB`j*-f0HEh)+GJeu_YUGY!ZgCLQ<_1UFdB--61xv7k+EkRuyEBts(~Vb52#|af
zakbAuL*dC&D>QGen{lkfbz1d6qs8zlhgY8-q*-S3Ou-LgxC`K7Jj)g-{AhqDMo!Qh
zC>F0ttSaTS-?eIpAWNLgy=}BXqfX^w7C)Bq#!Te2wInG`)jablDx?FSl;K@!T?nSe
z8T8HRmR@M$$|rp^1KhRmX^Ln_U2iPXc3(VL78I)OOXV)Z?O8K@O8cr&aAy%?<AN-Y
zH#e%X4+mBXtpAt}K&Hq-5=C@4s1@|B96Rk(a^NAN80ieUuph0Q2+FSb$!3l-+l_O!
zhN1Zpg<EL36=!I7tSV<lMs<PAo)bf?rtS2q+pVqA`v?L200+w2IAl9-Bs?;148qLv
zGevX$oqeCfL1p2ZTBv|+Rl;sRVfcf@7iez_QeN{FP^lgJP1!n<D{5FSrf&K?AEpe~
z{76~zuv^2!>qTl$w1rd~?XYZVpeuN`Z@siHbdg-}5VdqFev~d^+~ywwQsC^AQPnwW
zlAI2xz8~0hr3>~_-s7$^zx9aG*f;5L?=8$_s%q|;PFwR@0f^Y=o`Y05+chrx&Gcrz
z4rYb2EgS|KpME8;q$wa<H+^C`EJf$%$*A6PC;T-Kg<eA|5ap5&T_dU{KyZTVwpkGP
zn?R-<i0oycIDRUuiLb@r*JUDjZ9NZpI@0sfMftSNJD&7g6R^>{ljG5LR9)-%=4LqQ
zRXTm*YK!Asb@p$8yA9N{Z@DT`wDI2cL-Nq!+*T@ZW|0DgN9Kg#Jvb+##*WB#b+X0V
zvFbEQeK_FQXHriVIp-SQ3w-&~+RjeCF?(h4h2E{y%{ERK+2P++0J?{N?m#6oK*INo
z=#4rCL(pg{_`BFp56=n)^$QamRr$&*<pQxM$oFHS{B@^O+6{0@%;@~pNVskznc|iK
z1X@n5p;+J&I!sttq1%~zzjnFHKC24N-<{FVvq;0wtMhiirBokNHZ?e<QSH}7d!g$>
zi{U|MjEwGSgx(c*MS?n+Am>6!nAJF%UQr@e|K%x@Usuu#obQIU|7u2Xv`3QkjMfEq
z&ZHWp!=4Aw<tS=wKL9Mu^5%*qav!*TZnAd^qs=_u>V&(=;LrM}A<VX_o?*AWrh6NI
zf2UrOay^@u&p^pn1`5pvpIhF(sp?DugJ7bfgL7QES$l^jl1n(XzB6D#o9|NaWLrVU
zGHyeS0x%kw>}IyL%3~{PD+AJcf!gF)-;rE6A3eC(b6?j3m{?~Uy68n{7`ZNYpuhQ)
zPWs4Qic3?t-SSFpx$NTf(%?6_nNQ9a>O&g{sEI6axnQuK0ebqWgo=~M<^_P9%K_(K
zr`z7<c>DLp7FX*`%?5HR06qOAu5tOUiuZ0I>5jopRYB}Kj4;4s^jYibh#<R47$7Ux
z4+Q|`fViBvI$xk+e}?Q+%$O>*NN~>ZVf>Rj+Vi9T*nEP%w3&{KfciJ3!YocA$k%Pn
z;H#IJ9E}_C8aH99<CoNoi2Wh<Dc4@5G(1{^H{&0Wt~hS-wMI0a+vBG4@8Lrj-6#mK
z@0&vSiR3E0LZyZ%5_Bw9Prl=?QNHVn$6eJyq!)~YB9JhO9@|BJhij`b)V_&vevtuK
zL(PKv9qu{?Ta<uzMDLjJ&JH<*RgKRw^qeA^mz8}PNCEr{OKKkGpJUg{tJ+!1r9++s
zYRNEUtG0wZs<D3!tj7w5)zB|pFtwt%;x3{iau!x_uhWzd*NA^wO=gp>rm5}Hkm31K
zpJCuT{E1xghHWs$#<u+3=o5YHqUE8bds9Y*VVT>7`xd%N;QgOWfYSQrBv^WRC1bX2
zir$Z3c_C|EA#Q7wjQtgEtC6dHvmHVHwIUERyUu@Zznxd)HD#oUG$+fWQMd(q4YcW{
zT`WSvE`JDAp{8wSlZk#&KfLt~EdaLRifX3!!xi$~nCon4MCh>tdL_we*Mk|;qL~a)
zYW7hX>-|}(+kQ6vgNIyhMKcfBfHm{;@P7dKQ~WWU-cL%XY`j?u17ZCVFkrbjPS5=+
z_n(t;7TkeAWl@|bP$|Fg)qdn_0##BjZW4bPnD%r*gE~h5o~|>)UBk+N?jl{`qgA)0
z3YF8T<x)K}Dbg-@8ayAe_1q=ssMl{&De%yY@cZJ8SMuL4jh1C;T`w6k!t5ErGB)3L
zX*-T<`j(N2OOYUtr!pU?By3Na1y5^kw#?t%5UhDBk@JPxEnlaooe1I;`Sh*cDeEPF
zz;UWH2*plH-@?H<ZAZf%GgI~4paklz%(!8JtT@FjsKMoDqgTT+AIfmubw$2jVE$Jc
z_Bppi8Jxtylgq;Ns~8P{S2ztn4Jl%#OeRHMeWa_jS*e+=Jiw4cS|B&2Zb;s1A-Q^<
z%mdiRNHZoJ0OFZmtjlxo)R*HXZ<UD8ehXT^vI$IjlBc`BjQY|Wt<cL_c-5X$JS4d?
zi{FnPeYr(5@D-F0o1Y#YSvpIGkMM%}-gBVqbKtjp8!&hR@9uZd@$_h2l)0Y~BHDUj
zYy^r%FVMJ`gV9st4&V3Lw(m`Q=NM{3kG{WZw=US}{B{Jea^tV6y|_(3+%p}?08EOq
z)1A>;>?CJN>|Q84goK%|g8zl-kLxEK>q?74wnx{v?>T%TW0LPy>sg@ECqM_s|6rcS
zIb7*0?sk>*1%4B#%rlOEFUN?p-x5M$RTqtdIVUfCDVGF?^c%E9-bVKHx9Okk1FxC<
zrb@pl{9AGd?VkFgxcCn=Y<Qe;=lKr=Rk-2}y>ZGS!Xilf;fnoD(Pv>@Y=`}Ql2q<1
zt2#qjvsU+gmo#rGvt_<?^2#$`a41-8mF9gGo<Wx|DJf-SeB6i^#e~;U!nvL2s2@MB
z33iVK&wDKr7Yk3yJ&k#7{<ZP}>LUD>qHYMq%5jSD4vRy$W@ov>*(Nne?H3$MV@tn$
za0JX_k>r88f~pPydK}KbEAkC&ak=dNypzPa_z_ceHVM0p?J=5paQR67eqrek$2^)&
zi>>+D_EXHk(P(I!+f_rju>C+a0d?}BCfP(^R$c9LmBt9aOKj{BsIf%Mz+@(i(oJ8R
zp%dWcqy-e8A~GwB-Lj)K_E}rg{FokO&u{u4>1BV!K6(wkm)vu>%YHWH-9*6G)Tb}E
z6m@yu(VCu5Ul){cmb=Z*j|O->F?Y)qdO-z|2xc0m5+ZGfXtp*jc56+e<&_9>AH4AD
zj!1$1pIsU!pb0bEztR;bLO^^|l7Ra2!fPh&_Jo$Y-JkD&u7hbs;cE?Yhao^G5NQQ)
zgEN>NZiW$3jfpuAqzlB+XT3HSvSE!-g@%*P#?eH!lg4V@P8|J7epN*H^n98@DFzxa
z?|@J`_X_kCB3>wMhEPS|EZ^;{+ENEb(Cx6Ce6JqXZj8Q<TV_WowyhuQ#|Z;-%eRcB
z=Q)*I1{ZX^%z?WnJ&aSrMssdQmT0+Y`8H$gOQV$%<!j3eVRsDf8!fcy|6Fb`vK^h@
z{TF;w`6I4oOwhHz8ggiG?>ly!$Z3>Q4}U>l<8DW&aoR*rBm|C5%oD+mCo6|t4<4sk
z)XWahU0$qtz2eWp2{XnzQ64koUPlpOIx|}!$4$w3?jtOXuQY7+qFHiMwvZ0DEW{jx
zN}R{1`pv|(8BG#;4gAseU5jy!TjKVM2JuUgq80V5k?W;LHx(!p?=^Nn0lMw_gID7t
z%g=3(&|E*1d*E+x?xI3|Zd<ZNpBErvz?lM!*EI{MU#hh5Y*tsoX+9J{#&yZ5wtrGR
zSKRYPe+i5SbQZ8Pui~Es5Fiw+U8-|dTfy4%%3;}smI@C^Vwt_Jn$%F=Vbk3xtf{Oy
z5(XNpN#F{(Vw~1fC?xGUo%k8ycVZs;(LBTU;U-Wa)7$4efgB<yy#jCnep&J9Gff!Q
z=*>7%p^|sZ8B~?jQOtDKT}?a60bqE=8%6)(4iV(=-plzr*7_F4l;yi8k<`_hHUZAV
z{p@=V5IP3K*|ZKq&f2wjK5_MM70e0AEh4^z>mmhiB{FI;v9f4#ZWogQQ#%|pBHo?e
z&vT3H-TUh9NrP&4r3`SKKcJ<Uk4)?CsIbH?>PL4tPE{ER&Xj>Dvgq0QZ{>sq4+zd~
zSiQbc%cE2p!dp>lOiuWHI`k<G>NeXh(C!^oPOe+gdu3ZtPD~u~cqjFHSj#|lDI&=G
zrCTrcr>lc#stA6$qZ^5FV`lWzx8qdWP|0<?+K_SL4yiV#XoP(|*YQf}>qGl%$n|2;
zHu1uQM@u?yx#%(v<5lJf5R|uWJbwU6hzgd<2V8y|qRVq9&t83QMlKz`{QeMtVo^m`
zZ=8@wV}b^cfVWzvPUZn9wQA-MYVO@O?WVFOxGsp*S?ql7HjvCBULp~RRhKaU^nwFi
zT|NDftZ7=h_dIIxhL!2_<GsO5Q@8Z{E?Q7a&rae=z|k<C9EyVu>GZ_0;SlEpdXnq2
z)~Jo#Rq$G46ROC(jHGIvmVShpoOk0LLxMLdTr0MO6^wFc`9^Nl|IYLh&cVm1GE`1G
zmO8`Spc2JJ1rHN*8Y^N8C*gbKDXoLq;_sr5B}F^y<|RhGrdY}mKBxFvKl(6Be}@Z!
z7ujx<r^RyxfEkm3c>XqTu@hrSHpX=GcIvwf%znrju<veJb7Ag20x<o}SY0M-pK;?2
zl|nPy;3hxLRn%jr>2?@y2<HEe#LUI3m7<4HGS7(zsPp1+S|97rU_~TQ_0E#{_<hTf
zanE_zn7)^@4KNYM!s-s>M-Ta5UI39KueYSpzAM+B1w@ZICd<>rRx%LFV1hna&yI|0
zY!|#;{+GffNRFH+oRmEzZU4PAA>9K!Vt2e8!o=WE{k@JcqA?~DDWXGkrA*U$wQ(t&
zqiupNC9|-ROpsnDhqRF_QIr$4W0KRb%Cg7(U{GWIa~AJAwI(^V(_;(D?W`?nZE|52
zTkj6d(*rZ5-Q7yj?Xt$D(X$-FqzBVW@9Dp6s_Ih0j-~kq8nqY2Ef%syeP2>t9IMq!
z<o|8a?}5xI_QMRIb)j?LxdlkSAgAXo!;O8rHEtfr1Av)6H%3B@D`@b|m7l=0vTz0r
zJJ^`j&c!l5Kipnnt0Z{@(ox+e;Dm@;$d42bH{VFqQ>`U+nwc?oNV0DUSkUDoVSj8D
z*&+Ms?OO}pWHW1TXt?HdUG{M6Yv;Z!OzqebVjier)0OwLN7LDByh2vy=Wg4$U7nGY
zd4Y@YFXARCaHGr@7bIan=SQ2ZsCU6Mr3`0a*DUWr&_IO$S;(*O0xTrx)vS6`vw5w|
zO|TOrVPPI@`YY26XQ+%5ZWxl-%)VP7pho{Jc{VNdI54_SjWSl{HC43yR&ZOT`H=V0
zBW8VS<wH9G)CSVv3QB${>PN3%lKGV$qoE+Q<?cg5e0@Iix616k?@&sb|6FN=k6Z*H
zB6pQhcVzU-(Jha{)=()TmcY{q0^Nm<Je!JSnWJ^LTT|K#a7ccbed@KlT<-Yd{q!-4
z+u}0fVwm>^6^Dl=b#J-?P!=!b;5r<Uk`xnVSAEGJle-i*4w{`fy^jf4^?pgzeZjdd
z{wk4*C-IZEh*GahG~)r<&hOJlGWOd_%Hb39XN?Z@(0kFl+uNYY^sKmwz8Hg>=8C*o
zc^p?G1+1_i9e{e2X@O)tcj=`Cft9O7(BY1Kq%Rm~=@<ZUZZnAY35_!i-q+mj@a)-1
zKA3xc>rqvt$H%kwvma-41eCcr&`gSl=B#1~yiI86>_g@72SGAhL4r2n|A?n$b-;NI
zY5jDEyz{Z0L6^q3)E&5<;j`iGmARPMU?v2$o~m5SPgHD<+%U+8pe_=Pzup=(uW#Qw
zbZB%aJ5aAIdI|C)Lz(=y(7<tnv{?e@O9t#|Nogeo3gnkpN&~tR61vPnrgX}Rl^Z3a
z2jndaR#XaP;;v3*GC(g^9Horp7G5&5@n0QxdyG=Zgxh-9ds1Ghv=x5dLC_sXw)^^U
zq<kg&j|&`RPHw)qyA-(k6WcHzgD5g{`)6!nYynpU>??kcI&R74{C@Q29B~pJn#|Y-
ze}Zr9@_1B`J?F9CPU>f_yE>o3^}iNz!0hc3J5@P*%xqD+@116YSR~h^&5or=?VV}?
zPDu1y(l{bUTd*!p%t-HsbFJVpzod2)6D@)GRN*5sg42~<Qr-|cAmkCCjut&3zFse-
zIW44dunq(>5C@Up$4;pFxK06bu)^F1w{BS>E3xw4f4=^t!B-Y(m+Nu%z#Btoe`F|!
z2(>%!G;?v)H2a@BE=mT_a1{$#(D>QW**>2|P;tdeIF(YB=1W)*VEX65<Bwuv@!y0n
zBw*(Xv*<i&bTpJ4eaOo8iVIE!)9kA=&kj&br<dR!PZh)*nQZbe`kF^YP;2C4>$wW#
z_n>(4mhTWpa(AQH4(Tb#SiJg#x@{eY`OFx7(@lSj=h@0EAVb~*_JZcyXR73W^rfn{
zat0rMrRgFOqT~505p#~IU!VMa8eNq1JF*A4b3f$b4%{P*1W%lV_nFERt(IH3KAcp0
z%)o*eYgcU&{~)Z5#s8R}K3CEAPy>eP)<mq8MGEgAG&s8xF-mKjQ|yyn8#Ub~y=q}m
zMl<%N2H6iptX8c|4+jzV6Kft2aTyQOs5+#W>2f)TMC8?6OC{Ytv~P4(-d|DV^Gn<%
zhZJpE?el7^w>>C$^Xrnp7wC8HmbTs9iv1VxYWotdD;$>)hk2}PR^EF)ir|H#!t+GF
zpO$7pUC4JdbBjXCSWw~4N2zUitpp)i*iybC8%Gp>sl1&c%6XDyf~QJffg~y0I?#h?
z`<XZ`ugbLxOe)#LDaJGQO?CsqbcWR?*DjDNZy4wjaWoEUF0WMIzhFPXTv}*98Pk&3
zmd>h*Ie3p?*<ZW+yw0hgLE&Rd-1w4cQKiSPdofmtdl3d+3R%B?v%eoGIE<!>OtEL3
zyK*f?DO3u70{VObu@RZBpi7W67p;Mx7NVj&m8vNj)=tDOZ#jh&=ES<Xf$Os+4Kr>^
zb-6bop|<exV+-wT*@{OEhPueDxIrEP2ZF0MX|q?&qnN@9nI6>zX^A(@=F0kRCBT+V
zo{0Lc)OdH3PLe|$f+Ko;xbm@evf>JujQtk9U*`m+(ovX%QkfS2^OL_9{3vZO2#^_E
z)*TpSKN=5dmm>XoQc98XIjp6T;P6@lTreM~Mj)Yy51ItZ_58u2AifcG$Dm6D#Y#`S
z>o-(6+*wX6Bz)bFlcXa#p1O=N3e&yX+!e7Dh>(?=lqp2l63ndc3F2Q<yb7Z%4Dd%?
zAC2RR&}zkwC3iUV;qZp<fEHLfJX1^ro_eupUZwk=Po0V2)NP9CDb>FWx5hMZc!>nH
zqyPXQBWA)*LZrx0Ki1RFfBz}q0B}q9sYzLEYA-XVaU)?@3IgT){(ACq(eL^X(cqog
zta#RwzRAyzX{%%_LM*Cy)w0!7_Uv?p*O2sXdlq6LT}E!x%TO@|_eh0fse+y&Va~ln
zEwaZA>IDity$HqSl=b*0w~peUum?wI@{V_|=_OKy>?->H%Aw)Pe2b$R!O2tj|MvWK
zanFBLRj-fQtWga=dT)b(##u|UBST~Z@A}Am38%6IN}%)-7nQhJBjE(IVcq?MV$stH
z=!zTC-_T~+q-W<+bmE)t7h8e4<R97779&nAfN&$-UQ#^Vs5-=@l=6b{mYj2>I=W6_
zi`KrXK*C`>Id#u42AK#uu@5~GU|Jr^F1+;zp#;}rflEx<?pM(K>!`yEahF(Gb1Up$
zF0nhWp(cPsAYYG4h2ZpDMx>74PsE+6(kly^tM$X%s`)NK1frjWd5K|bbmL8b_GilM
zO^ESB#H+x~MDf3J%TXKT@FvxZXv;}Ni+OPwjh1@Z+fbx=JP<kovjvmDL``-_R^`~M
zXu10Q`r8AO`ZmGqBmRYp+}Akr0_aO4Z50hZ{R)yrqvVGTMukvc_^<cy_fdJc%<nVN
zJ;UXFNf8(sbZ((n%fZG1pB%Q6YJ=aN)oW&fA189Xf>Ye*kmv+_7H)n}sl0|<PW2dJ
zs_=4Pg`vgubu@ZEvz%mUeKla6d}kkcKo5*Y+nYUurZfeV;pw!6ZCOEmgE0@fxx05J
z{i;n8{qKKx44!knawp;SeAmxi8y{y!BxQenxs5>FasQ#qG5+fp`E`r`y0b;CAlgH1
z#v@3=NL68Q$oa4+x(N=h7exh9IVkN*U4|E$mg>B!(%Rq--RX+Wgh{0@>{IQB9#x_;
zUp`6XYL(XG(T^g3fu|>|Zys^3$AoirH*G9JW)GNYMx5^)o;TK3RfHUiC)IdKkvJce
zKUEfo+17G4?+pkcLI7swJu`#ET~lOT-uhqHlnVE~sj8Eme_h<a4=Tp=34C!uJ|czi
ze0X9DM2d7k$Cd_j?w}lU()ktycCNb#*edQKPRC<y(;1ct)~+{BPruq6miilBaiC3j
z(MU7Zt^8)L!TGUL;7mYQh?nt-zf?B=$e6uz!iht6TSR>5*SahM+Tzqo2$aMh>+PO^
zPO)!9iVAokm*biAcUZQ;a3YRM#>3ucCt)4=XD4$Yj~^<t<y}VQ^e<oj4lc&<P)p0h
zJl0}bgQM84jpIFE7tJSdmTDy3<1>I6ZxW|xKH=)=%nV3Ui4O_)+o0Ic;Rwm<x#6~D
zABti$Q6RQ&yzTQfN&GpJW}+M`Ie*n;M4!W!W*f?Yg74Xg(33b~Io;)IeM4y~%#bG2
zzLz3ueVF=>y!viB8wuk2zVH6iKcfgr6&KgvPaFOD$zNyW*Qr5i{yyM!G!Cg)BlRLi
zh4bebDw{E5%E<V}2*-G>2~hY0EdEM|CD5X$A=PPFP+7+9X+QCoK~gYhCjLPds72F-
z4_j*v&zq^v3rZW2^AcVCExv|W!XKh5wnJM~!hIHL@RYCa5q$!^yqJj8Ii;P|PAR64
zm28~#qvPJxJ{U+D-(2)6C5B|Hk>-M5;jnvu>d!BX%?pFEW8{rVrG8z2zcX5#BmuFK
zB|9c3z3|mu<RU@ZWjv)YHO{FT>i~i+vD(|khy%h62EQ@(BGyGC%Mm8`vZ-$K!&?)O
zF(zs16=~VybOh#L+V8OO_PIIZO6ol6=kOGsys?$*h__779$@zjIg15&eYf7DH#8Md
z;;1TUIixfxgrF**{Lo1_XcG`1f~8<OAihVaYo3+rB_`WM@%>p8*bxNl+5t!}_`>+o
z-=F?ppA;z(ZILN{9)T7U!=Yi(VKKp>aSCI^RaCIOtOLR=Rl)P&*shRj=(C=?Uk}Ih
zgM$2>RAY+j87N?w!Fk}=bl*fEKfpov8oeV=ysAwaGzqh3awXxhR-{Z6(_v}BQW+Hr
zS$e)SNg?q;kJ*_VlYgSrjO)FvQgQ7#rO=RQ$MD|s!1LK^1f`G>MhwZv4)}lmGmvjz
zv<ug~vZ?U$SFryd>Ka=xXF?3$|44r1&j<Uuy<fx*7Y(bQH7Fnr#&e!x(;al&WOp(<
z%X_vvt<BXq_>0_nXjFG3M&%#mGe+dKO5b!YJ~ua4Rsl1!QA4UVKyQ%1Q}<|M#9nNg
zT*9}}LHL{rW%snb_OyXltWJTZMkuA{UTL!5Ued{0KSp(*QN$rgT?Zss-tZXQIZ{9P
zuXB`$J4dY7NG1OH9BuI7!jH-R*aW#4#rIPI9cFmG{(j2;|6EN_;XBwW7<~K{;fv1W
z3d{3`9237j`E`5$>-;?X9fwsM>pIq246lVZ6OzmDmlCI>W)=p~dKEDiC54nZk%&zf
za~SGw_3PXXqek>RhDZLtAp6z-Ka>4H(C-v2__NjG9b3nJ@zGR;qJKF#xd0RD?NgkJ
zI_?s(7-P1hbGg26^cK#3s!&e&EDT#rUu1*D-(<RP%H3ypP0^WJN+vii(sW;Z#0Rmt
zy8}%og2Z1cgqRQ<^w`5lrX%9iA@ou~yTysk>KJ*;93u8ca$RYv5jHdB_-Ci-(-r38
z$@)Z{2Gm~2!YeS<z2D1XQZ0p9@ZI0k<=_3)Ci0H`z<u{`*$O59^4$Y*-+j%$Ebho(
z5=N(1L_8LEfzB0#%?K$1I$uMCa#)Ds-tauzNyOIfv$ai4B@OoF5|biGDHSv$1fLUg
zOTV!ncGpst*tMyWWcDIs%!^vR)ZZynWC=w{;DqDHom6FViu2QJF0Z<=nz_eH$Cihz
zT>6^Q5hc?-IL|Z6C<5vPHjaCsPS{}{oBXRzFu~>9N;!_9e>pRpWB`scYgEkO)Zc--
z(mdh-^w6F%W;kb9>NQI^3^H&~tOsJKI0`f%PO1(aEQiag%3D-Bx7?grq$<`_TG4B6
z@&*Uz*mAKIx{*?=7xJ=NczN*RFR?_F-V=0HsV`_#Y}Dp@7g1eLoFuizXz5)d>PzRv
zFXO1Y+#OKsbW_51Tw=zoks(pxJ10oGUv<{w{!F@!;o!<P;a98X{5{<K`lJy@X-MoN
z)EgO{q9#!z?SS5T)J0zpF>Ay-ETZ+3g>6tg-w@^8B;DyXs9%;_DL3ky=Se;wNq6pz
zt@B%9we><)>Q^WW{>TC(#-1)|am?&(8YeMY)G~ns=brIQO7DgNY&h=JxXqAx894TA
zvp9CIH&P{@8jY!3+12*{Z_0}Q|JMSOT7M@ig|pVFa|~aLrL&-2I9{&ois6;uzS8qm
z-z%vm1vplwVFfcPFifvz4zV!=!-q(nAcaB<nqa;n*%7a%1mvL4?MmWH8JKz5r}=fC
zJP0>o_pRp3;D_E#2xt<5coC6`lZr9~@06a0BHD>~8Yg>Hm5?Z@wY)dLkUJ<slVrRf
zG$r~W7{!ObisA4!UR+b6H}!ABAC%tjO`(I{5H$hy_wS|3+{_6Q0&IAwi#hO28g<4M
z|5z2lPgh5natJp`#yC?lk%b56CItIp<W^V<6*X@9Z^dz&z|c)@TOVCCvAbk=ku~}h
zrBAVW&+sqO#uOn%LsrBwcRLR$E;#BUk1wL_bi3H6CP`s?iI1rL%MS&W`TQt{38)Jf
zQ&oIkfajlPt2FvofnSF!@NMaR6@UFse-HiP4?uwr+}!*X0IUGYAHa&aO6km`HJyy`
zAzB$Hwlm68$=_m(lsH+*Y$yloCQBilTVns)9NqPJ(?;*O20zzx8%_7~@m*uxi-JZQ
zWdTQ3g#!n-&}A<+N)?Rb<VwV%0x;#)V^t;>$Q4TJu>AS5vQ*ZN8KxRI%gDodh4e6x
zvko5ZH~v}spswP|w}@$#KR@{^-2X}dD6Jx%ovf^w06y5pxr^`MF<jwl(isI|0mtaH
z1a+_lmtW_p3IG%X2vH@X;^pF82m}_12;#31xvNuSW+w9>GZnAoN}-0kt_7h`cjlxd
z|Cde!3b+$Ko-lTN#8;?{C74LPej4YrrounC(^-h2VNN)i(q|7A?|HxS3O+*JPk<0-
z@?)_2zq!F~siZk6g~+4vThyq1O=yQUWP@^yI_1ydVIxsvDK(d=Etn4&RX`@?4h1j!
zL!`(LQztYX7Axn<CU~o;+g31(b-pWZ-ZD@EYOISyVg)&}K!K@(*rbOqhe<C#h%xQD
zc$438I^Sgt$^!EW&;gv5u_zzfv*n8WDd!ZHH*_p(Q2gJO|MmV5m#ZP42M|##WYCK*
z;1qD1oUlBRJr^@}foM)F9=T*XpDE{2m4WVR45E>A#q<V8@Jeoj<CTyNCJUyAG-{pw
zuk3ZB+H$IRbtw0y9|TEj@5NrmtJrN43t~thUgG2I?q)W*riAEX7e{bLi*85or^ebF
zzU+XiW-&gs&Sg2<<vdG0#fXNqxcynZ@s7z8p)m7R-ei6ie|`A3?kisd++~fGHwGi4
zAC6L=N3Iq@;of2O-$u0|EID{6`vKm%E-+IJyeBAqZfa>zuIr*G4BIWSJ)u~nZWnYk
zrU5ILw@l{dv~P0r`7OzrNB4RHG?^FKJ(i5LNlWjD*p(&?S}_Qn*4a?8MFFJVwzpb~
ztc2wd-vo2O&8uO%^`F@c3UI>NINY}9y5_)GZn7^GI%%ZIk?}cz!Yga|uUw`O3BLkn
z5j_J+I{$LL^>L_S?yB_PW)WInRkCA#O(M8j|1*eCVR)31X`@oFEsk+e%tlKxZ%JnY
z-+CP;QYWN?=N+${>s6OkBveGw9~@P4x*r<W1&uJ3P>7^|ujW^CkI<>cqzFO;f~D_`
zFoh&O637=?4_4O+FKam4{Gylmh{=K<P&@pcYu~J0!|{zKQ#B#kpF`Ng|NGY>Q4ViA
zb(7jL`^x$+X%Fa{;yWDdZ9vkSxfo6YBChlKF|rgs)7-u4%sX)V^Tu1ZSH?NiLbnda
zc)4SHOhbF(PcnKrOF5wBq6)-Kt8U&ET*Lln9}+@xT9~wi{K4LtJO<2^xVG_86>t;d
z+0Ed@&^cF8ZzmStA7N5H=yXcsR}0n_KXOto0|010kkt4e04Waw0RU;-pykYp;T_dy
zEl9}W68a)_f}+9!>1|&9K!64q6k?|;CxD^i1YqXn$%!~klxc5zg#NS)_+g#TCQZ*b
zhfo!Ly%^y<EIq5MbNS8f03cDCIQg+qSfM|?uo{vV8-uIa!>RFc(-ka@-9FvF#KSv0
zIFX&AfdjaPL)1Ty-KvXe;TKfmK}(1)uuZ4e$M?MxLCU##{{G7;j*TuV)Gm}n!wrHJ
zYr{??b85U@d!Hp<_Xb@lPowHx-X;L`^hgX*j8L<ozg3d+bOPYqCU+i_iSR_t{m+Al
zjb1Q=i9O9*4DWFG`jlsfMas$00tETJ7devpm066nmCm!gJ@G=d=44mo`j?AiAcA{f
zO{KoK=iQDkU@e)!pm2jO7IC(y>DLU72$nJ1W(~z+^A_moB=UiW{U#6`q_}4D2+OUe
zn+)7<r7Xz+t7T-AfarM>P_In9?6s9(!$=#;)@Hz_fw_Yj2PS8r8PKl;H~<k#{if>)
z5L9Hh0n?63u)5~iK}{xdWF1hUglr0{zol@iPO#q1OI^!~{E%Pq;Y(e60Z`S*GN#!(
zQ%R|P2KpGA&YZ&V5BJ`RC;>8w<AO%E^o%()7*k`brz6sI-I^H<0|K2*ps5le;`GhG
zzshK4m3rvFi<2>B*S3DQV4MWJ*=*qH@b<-!>)a>Tk>-a$s5KD^gtyLTCqG07apJ?A
z;0K>&EMEbYprADv(@Aq2eUisfH17|JnC-Lm#VznfL5?q|t+_n>1M(faLa>c>n|eF{
zI`f}P^)T`S9D)jL)d;9{(G)Ve^YEk+N>n5A+7-t}qOG)sM-^cT!9R0ae5m6u!7zLk
zL>H#|t15@0*^oh1cY`vt#is{D1KcTaYL^=vSrQa=`i~WauXd7_>Ytd?P`|AuNNKoq
zoQ3e#s0xSGH&SR{*t@oCc5}bqRGq_dy56<fjY;jJm{9Bm^Tq0#+8miVJHi)wthwli
zz~9!Sjc^{BpUVP-cLq4SS}$t~_Mc#bfv(G|zU4gc5$#A)>U(VRJW0P3+v4~Nsy6q*
z>sG++Ad>b~V!RAkNIt;1;GvhRuRhTn@Mdf7ietoWB%K8R%v2M94DRc^I=`WB^$2PG
z1YpM5EJwdyG;7D2<^Ve7K!6$%PCUWwWaKFQWQ)BV$I(*&V{dk8=7_jqSxxh~260Xz
zUfEARCf4g`M}=odS}UB`^U;Rt$iiWd>;#jgTd{p=f#*B&O(12dMw;A0D1Fzu^7VOS
zJh|5gu#RzH4`-&aW;y)P^ZohFrwt>5Z@palag05-Q%8qOl8yuEK7hkr3232jCeVL$
zTyPJv#{C8(kF{mxe@R=sWAr%mCpDOF9mw1cNSyF!9PB38)HuSsm^O%Z08ld_I7Vuc
z50L$X?!INs?9&zGds!?}R5?J3?|?E95h{h{zw`8)xIrag6lmB2{xvVb#l#m(H30*S
zu1kXJ)$eRW(jBatpm?-+_D+<0@#GhW4N|;K#V<RecDwmqmA_)|NAzahTqUBG$$%a8
z8@w2)_zEWN+HMsOo0|gZ35$vBiHHdEn#1}WfNI|T{Y>>@Se)ko5ud#0{phE%pjdg)
zf0kf-eDzj*qpbIq_F|_qE$0zgdqGnVFRaoJh+~fcTNS5D6s;wU^BMtm(NX~i8|E3+
z^)77e3^ccYCvaHMqMo*&Vy(9VtE*&1u?HH9<DUfhPcH2og3V3^mqCEdF>wm8@8}oY
z^+_!_{`5zmJC+513uBjt%!~Z%BfbKl%}!Y)-`e{2@y6778I^G&tt%-F-JfBOTbGL9
zh%gzv(7i3q_<H$%jk*KkgZI>>`T@}_2$_K{N5un@Ll;VC+UuIEp-I^dt+El{XNV%w
z<#HG(?Cz)cr|m7OW~&oi^Su@ag&*{Cs%U<#^SbX#O98{p-k#1KB)siDC?8^}uH+2q
z%94de-YplhXxq!ED<fvM5)-YF4+I*VgrsBrcD7=ejQHt3?5u&te+V$ZyEEvlu8dz!
z)im|&_|m1#z_MJ|`W@9h%{g5oh0|U7fc2!Aq9ID*y@hAT>w83x0R_C}wF1;e@9NzU
z2w-02<_Jkvd!rfB+5yvk)qHvgSa-b5tD+6YF+~nqzXgllm|mgfX*onENc+|s<{OwU
z?gGl$+d*fUX?k9UI<^EXpq^Da6#)AG1f7m>DwBm1_nXX3uvRS-XDsxhWi7{brj<9k
zkiNFVAD8vEYU*rS9+<AhN*ta914-nd-|YbCPSHY{cG?6WuxY;R2>_ym94*Xx{6Za2
zFSjSYA0;(=dwm;7jP&}K!CJm+JFznUdn@PwR^uoq9P6FAHt*d#S7*uJd!H<Ef)g<8
z?zyw^7r6I7ym&($&{VJ#TK;E736!G^|J#sKl^G5@GSsxG2NJ&mg^X3z(4-`4hrXDA
zVZ?QJuis~sUs3;=QS$%SjMA=&hN+l%u~L0Wh|^ei8?&aDk429jrmyO79Bo<ccKSs0
z&V>g>kg|8;C>sbf=I6WZU?G};?L6sOGAB&gD_FD9AP7w_UCDb4ShspOUM^Udbl3YQ
zNQ+u-SE<#$)afOzX4<v^x#=qD><6wrc+mLl4aN)CyD1W1fwkmJW`SMxv>sx7H^FNM
z%fD=(z!f4%l}f%WfhEG#83+z44XyRxRLNFOyosYU>lK5smW1;MI9=>X34NH@hLa9~
zGo(|ij@$0Z^yMk?xM<8;_{am~S?iGYYH?b*>yNmszL+q+=r{%(Ro^712rr-pXlJN=
z@1#5N`JCd$py?H>H}uc=!Zi;}Lx4Tavp4r*exw7Du+gyZq#%Vp1r{~iHq*_#G*P4y
zIe^8j>8aJK@bT>Vz4R>-AUFY~JhFeT|F2T`?_FUI7q0Ae{Gom14~3z=Wv+#_h5{}I
zdrIaFJX_G-mea=u_b(&HYh}tC1?tQXce~zHC5p0<8>1ZLnFke6V%VnJTYq5TOId<c
zzSW4dtGcwzkdwX!I2U<3Q(g1N>jde3Q6VhKn7u}10lf-QY5=B~BfK}o^u|rcuQ;m0
zhcHQ*7TAwBqnDzH-Cj1}51C%TxB(7LoZf~L*x-4OIlLEmbL8QOY7<E{yllY^1-p`k
z2ic=Ry!9$SV_?U(;PHxBZ~DkIQW~Qn;NsEkdLK~9f4u1{W$nZl<>JBbg1x|%BHVc?
z($Jk04^u7AL6KKwz`4hCI2iDw*W<t3L~GOD&GXVwSPUZ^*af<trbh55GFDwGXiIT5
z(Qb}feIM{2c}ycCMaM54Z_1J8>KaYSYJ8(tNF6xgT8Rm=dq;m44c^r8?d>k~^C|#5
z;X8a57~eM=B?%Ltx-n`FT4rC+vUICgznRRcCK>xsPF`)smH>gPUWS3#o9*SHwTjgB
zCQ~zOX?NhE;3!=p6;kW|$E}mpD1yP*hv#%Z{mJI|)!6^{ORd07AyePnF%FUI4;%LS
z{&SqmX`#@BeC}IiA~H##$;e<~Lla^fq%}@OqpIvudEP`mITGlW>P6cQI&B>U?;Jo4
z0(CTsK4)Q_i*slknN$WM#leD4{pbCzorkoV^`;0!2NzB1Qsu4`iQ`d9Z%muS-juj<
z$O>zWqFE#5pjqQbIU0i<yX7B1;pWB$_G}fb`2+&O*aWa>m;@5)>!uY}A3C89k$J<5
z)$TwF7pLbr54Oyt&kfa~2q=V8G!j`-=VG6DiM)q1_#gb>C8;dp+FQg@)$ReBHT~8-
zbb@7#vt$Gz5pSl|eeGdq;a(22Ku0Yh0%|ov;3i}yKfTpYKRkr|`cXByum)^Ccz%HG
zg`K9JerZ3gp$YbEev~Mr@!F-VU^<r?{R9Xx>Y~X@;(*7sZy<qwtGN}Uanm<o6Xguv
zfKW{-S82gmgsVNSTbXFjSyD^jzgKh;3r*IdR@{MaDgXc>!qb$h^5sTZ_9>m;j)L&Z
zIr~)wn%q3!nVXwn_lfYAJn`QHsuO4UDS%nqGUWb6McEPa2&cp^m{sHk^n-WG?caZ9
zXflUsspjC<T6Jiy0&?^c6i|t<_-GnJ)hk@V_!vP=PApnf>FUUn9MV;3Wk*U|`@Sqo
z;hMJI^o~|>rf&O<I%580xPS~jJxlpkrOZIQCI_GUbFE|N(&U&=Hq$0|gp*3Mrfvx?
z8??_A$_%H8`yB=xJrBCtofYmE(W}Akp8n$SqCx$P+@r7>xz{{VdM2!u4?W{!zJlsf
z3)FwD@s0x(XBD~kES>=EX!pUw*~x-q_Ru+@rztX^hLh^O&?4L<L!%CnbfAwEGN!pc
z1UozK{^A9eSA5UzD;1Q3;}uUz%}HtA?m%8{>F@VzrB>MBVoYef8QzOCVVJ*LXg&E5
z80BsbNsbDC0clo^xpvm+b{e0-J^m3n;Qo;ybZ6=ZuvgRnoo!3HoA?!Yrs-AY-H(r#
zcJ@`a%6!)^wHwb60;9PprBJMaUsKO!4SR33csX!?SeBj-ZP#)N<iRC5@e9N5FXt=P
z9yYoib2WgB``}IT3TQ=jK@01+i2{86A2IxoxoIM3qoC%Y<chdmd+38=`>`GBtMvK|
zxAei5mWN<3X64nIhCgm?u)5tH!1Jqlb0zto`;Cs6G&l$F9PoR-j;IZ54T)3WcP+Kb
zi3BaAk%6^oCERv8jSj(}s?#CIa$?HcU462m!|rdt1e_>~^pXGPWQQv1|BdW$gzq2G
z3-rAyubYdmsHRH*o2#xh*h`Q_E8nc&(VpA}^|@fJy5Y^uUghugIW?l6!AdN2HJMDb
zJR`xbCPRFctuAR!pTiw@dmLr-5ngx=|J<==$`~`qqCU*@w1k!J7ksCTtciG)$Q5bS
zzP?^gc23iiLDD9OLHNwoZm9`*m%s?hz($<MZb{s1Xl$~dH2FIQB0(}9;I>ydaR&I>
z*M3)Y+#iyX<W%F&dpw?I>M%|3w{v9|r=Pv^88EVaP~fR2>`Z^L{Oq@!)CeS&K3Akd
z%Sy5c*V*sAE|%N8$^q;S#Q19Emy|BDd9NQNf_uuPc2$a@&N*)1s9-khOC`<~?$T$o
z{{UgZs&;pRWwvtXuf+e`OH0Mw)F-nrW!U6iq*KI6aH7;&Ib+HWA?GIJPJ(|>IU^UZ
zyWqMTH)MQ?zHtfKN>IOH5Ikhk3th+)(OvG52<dXD6=CHFbU1L}sfRi0QY37T8`v=r
zo^8F`HR3Ek#52+e!rJd~12!{b48wbaWu(6NTEDx_8ogo<RT;e^uC{FKVRIGx0Xc8f
zgR2YN5<^ynY>~U|;OuS{6m|3fufkJIj>nO=5-FdNXRwYUV+pTaxmb!CMADT$(p9}N
z4v~d#q-tw!>N#euO9TNs&D`o$PC>Y=?`bONfEHls)S2p`X87BHQ&lh6B*CX3L+2N?
zL}9B1GNc*nPXJSCgXT@uXzebduv@MU1ac`H#8BQYF+3F01i9bG-5p1}WhVNp=W1T3
zFV7sfPF8tN@4q?bF}nlBK}a*O{pn|_%%)_Ms_NUP=F8d%oJrdaWHb19hzle>AdhE$
z0+p?umA=`n-(G;zSw}PK29`Wu@<6>V076}(`!%^d_BfaEftNUU78;Ta#`6-Y(Ub!E
z;mC({hOL)lZ-bNE9H3YGM`i1j+A#z$0JPNhp98$VA)Mg$UIM*htXX3b{$eM<mo*8w
zilcl;V}+~uk}4h;lZ!KGx*xh;*eVkpGZ(*v_zk{X_0+tK#d`_)z5LlFsFsIbS}gso
z2w^xNGgR1AD5v)?^A!_2{>6&{9-@Ee#pKkfunzOC!=HoX!S}3b#=^SdU52n|`|*A3
z*SoM>@K$RtZ{rqebC1QI$5lr5(3b^X&xk5z`?fG<AApHv!EeBt!ank0fj^k;ejtel
z*VD~j&8rvKjAYJLeG{rVM-_Vow9q_?Jdr^8UH4l*nHBQgwBQS1GSMV`jZZHp9qiVA
z-i3jVZw(8)G}})if=s7fKAgvPQim8m(3GMLl{uP+0gPqvfn!cSwS5v!mgHru2X&RX
z@pW{Qi%fvw6<g#25}Zxx2b+&I54pl)+2kD~CuMH8()w}ru>c8;mxKXBdCS3s&L)6&
zaDv;`cdtwrJGqXzHKruNPQskorabti_djQV<DVtc_Ye9~ktu;B&=}QtH@5%ttqdbq
z!wu%$9za*%s$9^znDg+}a?!C0u4X)}G2(Rb)3KH<(4PcP(TpJ^PRj&cDYyL4R#G==
zFJq`(?h4x*Jz<Jyp5tQ<lp)zp>i!2-kT46(euv*cNh4eF<jqTUFD-G@RA&0?DX&6u
zK(K+3@%eFmg)bUO%e?j%BVvkGd}94zzS>gn2>doW%+@|ede*|8#=X~so-x!kuh7PL
z@(>YFCngcp?k_S6q2M&Qnn#1-A<lx(x^_M`C?Ksao_Ncebr0;{R>WB=Y41*v=|8pG
zKmGL3KlB<BSW1*4lr8f7j66oB<&|*$Nnnw1?YhO(u4HT<H=7xr7o#QvhuAt!;NgQ4
z!Vf5BJ5N?jlhI9?=Tl?4auj@Byrxr*wE(<0;serhPjyaEtGP5)1_5k;;do6}$BT6M
zdDyGOH*TON5}eU{6O9zKc>{XcNzg#qV1L|LR0-R17MR)%<WRTYR_u)2EfE1ILR^~R
zUf?}1!E2bNo>$pNU0EZ43~Z<P#uf^bwnpDI{SOW0pI32A@prH$P;Z2KLF=vuXK}Q#
z6y5}75&dFZ^mq8_%BH4ZYuMfd9Y2Xc+mtJ%@c~7bsx91hYv;YxOM4UMpS@XN^JnR$
zQU{MjZ5OYu*tA>X)sqp8EEosfBeFy$eT#fEVR{PBtRB<+B9{wW*$CmaWxvMJTrl5e
zk!%1JZ0W<u^D<+3gHrKV1_ttO0=I`rPz+6}yKxr+*BKsbuyjoDf%bgtXE1wsJ5>`g
z9N5L-_C(Y2cB*>(@OMzpOWt5M>)Lr1DARN|8ncdjr`Khdf4Xd&cB%j;AO{gpDaK8z
zmgL_b5ZD9-i%9hrxgt)wkM>o+?RB^mQ#pV4>_;8hY&jPi`9neGN~K(;OF^2HpG6bc
z{ww$i^n22us`rai>2*Ofn)y3!<2(LfNW8ZU=d^tLt$C}tDPOJXRkH6u0m8uNaGjL{
z_(@rZR8lGKbaZ_)hAy+^`Q?y=;Bdrx5==I2IH~xd|KdqWsx-7}W1CSVsptBM#GtUr
zkZ4<>6g1H&wDOM^mPCHAez&xz!876CZsLFZKmc8A?x}`V_yewUeCeap>PiCd$Pe&<
zS4_v6v?E~d1<hY9|39INZIe($ePlGFxV@*CWw!{0tE--p7MdH_Q1CJF-M6!FVhC6q
zjC2-0?vpg==1m@0L9Q<mA<AELEcavsp1(Ig5xc)~xe}-rHvy<BI5<sN$sP2YccBgr
zjcyVE(6Y63ZkK5cBKWI=uPkbQ@ocj2G#vtVNxNjeurf%L2su7H__=V_cCh(PH`E;$
zQ|rKJ#%HrU^Q9vayiy>)jC`+D5G(%R>zogEvyyqgnr~+=_{XO~dP%uQ_6@@_H8<Y_
za5Y;lqs-;m5E5nP7{weLWg)G`sCmIlP4CrK5K#QKHg3QCz@!XpU-VD_u6H>;n!mG+
zsTHa2K?|n&DfHty&{H5Rn(eA#q7lji%1!BM(Y8Z-6i|4&Y+YZx`{aw0@*Zt#`D(lV
zO|^GO_VGK{0ZPaLd}Tab^tWSoKXC8Zx#M!TJmppKE?JBl$e_*ulc;e<pF?gjsz_5^
z{=?(tK&z--nH?iXd*zRR|DXQ~Ps4GEZvQ4B0P|QS?Ih;4#tXoUQ?4?5dq6aG%gP@P
z2o+-jJJ01~?u2Q&%Ud?48Z$dA#b^h@EvFPyW<TzjD9=oqEiX+z7O>OFSN5FgAW(_-
zyqadB^qWN^YI9IEII1FR3Q3(%%DL;*>r|SY9*`)5B`?~_K$q6<l*2SQdaR*3=c9xy
z=86`;CSx3UW~7@KddT||NO<yT0{$0o?-><U*0qZgB}vX%at<XKk(`l?f=EW708vQ_
z3P_3MP-Kt{MHCRh0s$o#S#l5&l`IqnKu|!yySBD=pYE^w-f`|Y=SRPzUiwe(z4lsb
z&SyTs&+W?MHIQ5%eXYY+ruXgfmnVzGcR$MNG9x`{A}BfY7=XNo`w-}{$AMsT?n4h2
z*#9O`DL_5SO6rWH8-CNRS)KJ!%KOJNn5adH>A~|nAL<^=7bLZ#@)8$7q2$dq?sydx
zxB)4e#b)%(?}2MT_mrS_@X)Z(m*~KUOQDyfLtHt4os`x2WiYZBVj0svyP#qGS#8;@
zWuTn8H_yrczC#$P75B548vtf3OEFRux<K!#p&2$RJPV=_Gh4s5eCmE6jRKvfH~R#M
zK#mEmW%vzZCFR`;WBM0F?kC3qjv@0Jj67-|^nITvQqcvDHa9MQqE*tT8-`Y`{Ur4o
zaztB7TJ5InKY0zm|AHVzDB>ZXJSMZok;a1)2h4oUD95)A#cuK-^;8O!i^r+Tjo+6k
zj(Y_4Bv5|9^kziL`Vq4j)NY!$fqM2>ST1e6LawJ++LXUYepyCg+)6^F{;^=RG+YCY
zP{A@AVGD8rTr^wW1$PVDMLH(|ess%fsWMAS>P$IZgiB!;xlN1Y<9PDx4{M^bKL7yM
z8q(vCE@<EUtZ}`A1P`fv0DnzO%Cf7qRtx?`Y2fPpvRCG3Chsi8%Zbazp672_h+rKD
zH4sPT!P&PTA-Ue(Pg$?~`Hl`Sp|Z;hZTD;dXiO<IimC?INiyohvb_rAxM~E`Kq8wD
zeC8y)Kxui|Ahs;DyiTzU+>(3*wDIr3h|w76<1W8^9!)VdAw-otNqfS(gxE4xGq#f+
zTlu*nMs6>6J~eIco2H6UGSk^x(l5tv#hh3ClDMFm3)ZkPma(lDa?XUBLmGkk&E`!a
zVU`;9xCdn<yW#C&-q@Dm$tN1~YIE3U7vF+PK56fz`niTVpFw&R$|&Qamd#GG=t|E&
z`V#;Jz0Qw!y>jE@Z_4!VTk|hJY$3v#Ym+}1Z}wB({-(5cVwB9`{cg%THBMEQ`V$jA
z7Y|O%PN}$g2Kmzve>lCU3mVOBZ;D?u%TYxKOq%heu;A5&$@eNhO+*uci73Nw6Vc26
z?<S%fracyN=Y5aP)lOZwBvuWk4l>TZooX%DT3^WTY9J=G<Xs352!LmE7|6W|K*dOj
zdHuL3uY2h1(#6~mZ5Eb?-#GbjT485Y!{95>Ip+-cfa%o>Vy>CEx8TU00Jfs0J=md~
zhFOowIu$3fTm}snW7d5eV@!qgX^a7I@QsQ&OR98z?sAIZ_><P%>N-khj07<;lnh_j
z+T8HKs@U%ecb7Z6NYx{C+Pr#kVEO93r8qf#gI*t+Tdjiutl;a`t@vxj;Jz*r^<5kR
zaQvlo=Liu;Dk3e<jCN-*j5N-^HoOb3W<HeP#kl_9eM>qM%<e<*`Raee=lP)X@11-v
zl#7!)^Q}zO7RP_0ZC|-1A^yrbu5RFFS)MZdL^(>-TW9!&&{8aC#esWDPJorDh_UdU
zUA{EEuLVBl6?f$Tn+|PN64Q0BSTj1s;w)JiGA=}TBBGy>DF@_WqZe3@n8{QaBvf}y
zmE8=wGOPfT<q~Y4^k_5)jH3|O@9MxhMDyk?!m#}BWnS2TK%K06gXdsIY7Sj0S4Vxf
z@tlYr!tcMF{vm-?CFtM}x&t+g&od#sLlNXXd)Q=C2aGxbA3{BwuQb&xHFCbm?aq)q
z@?GvtyOa4~o;&OJvFY9PWH+8=3}%9zDi_L|j!sc~gbiYeJ{_`y;-t5v?w&bMt@*gt
zDNZ|(zyum<s#i_L)ryB88{d@Voj2~x1Qkk~K$OyA1^)+{fJf;7+mnn3>g+dl&66Dn
zPyegU>!spe4}a3W;kl|nfm*Lj`}LnR@IU{8ScQ0vZ?EQMa*`;`oJ#$udY60vj(d4|
zMZjK@4NeG0aSCPOp7Vzj(WTtl9%r8KLe=t)79eM%$}+csLBaq_eo)Q1j!Pe^^u~=-
zF4HW_1O_<;87f+4keTEu1D>6gsANJ3)1>ou8q#dG@3Td);Gl7!8K^PSjW?Ojniy^N
zebxikefm$|*BK1)iwX!`KR<Zp({)nA@2{FS9$vB`q0w;-%d2EKakn0DCt}V(?yG(P
zPW|%A3^eQSrTx}Mul71?9756MSxSLE&A@JVA`|A^yyr{a<6f3~!MqKaqn1Ine`x~w
z9RksuRW6=m;BaAe{9Hy#VE4Q*Lgy?l$%`aEfvQEtf&dm0SW`oIz0=e_FtvIPzJ~5J
z#Vu*7gYS!sZMrXPZ?eBEb*3LSJnpKOJLLHMZLV+c$dMywe4>x)rk0xa`FT^9OY}7s
zOBZUA+_mRBq6MJA5;LhOBfzeiEJ@e_jM^<@vU|9%kbT-k-#d5YIbH2kcJ>B@VHc#3
zph1Tx@Og9YKWH0(_5xA(-Og~SY{f9_5N6(EnF^W|AB|`;T2~b*2w5X2&j?X%4!1c^
zd^(kelJihTHgM5lDkx6sweu~d!zc}<bG4-zS$i2cEwwX9wR#7qq+SAY`yfHoRny?%
zbpDPLUU=s5VUK&#kWQ5<F7-ke;!w0|n-dx~*muJtiD1Xfe3Z@d{&>yVoX%8BTK17!
z&y!b-KwQ6Qk@V7)b~@E{+J+mM`RON*r+j9|-T^^44vG=moLf}AzHbP!_$0w_BUS6N
zdtV~gx-;iDt6Qu0-AGM`fhmN?!)SL|yRrkWN6Y{@w+6~=T7hOIsHFqWu@d7(wkbbl
zGWhpZ?eEC4^m*>i{T2QH+izELFpRAYv>U@fT3ZtCQM@%xif9>linv<#%l|FT@o8=f
zeqm1jK0Gv4J&DIvVMiK)FXqggPA2>bJ%cY#Kb)oVlq_9L$#|rJM?1UabB*Gc7XY1v
zTP!pmXrxujWP73X9(0-{-dgB4d0A>otwL`f41sUCTWKaU9dbD7cIO68T;@~fj9=A8
z*G`%llz@-A#%kl7eSyGVE;unNc&{6ZA>v8pf3+cKh|BT6WlQ+kfyMlDft|Pe3#Bm;
z_!VsSY4UO4uk5s)6@??1?m(Q-Tv0{Xed}O*lpjG%_MnGBdOXwe%20Icq#mYlCKVRX
zxY1PP(tr5jpca26hJk?oF!Hb{Ry+$b0H%9;AmRGGLj}xxT^C~_zP{Nw_J5}zNq@nr
zFsj3y4ot;vE@ovg+68d6fIfR2LBt_g^CoJma{tG02w2|8Og*{<>`iIbLFP#6@xJlV
z@nqw!XGb>EQOv8b-}2;>*8#O#y50Vi@qajf6_EYtv{?n-xnYTn)^GxNs}cjar-94~
zhf^Bw(<)IgF2m6Z&k@e?p3n`p$Y;`|3YRcRZ$eLhVA`Swnne-u-O2t*os}QN`picD
z(&Ma?c;x)*w{0BWYjE)4#Bw3W@<Xy{k4l0H#d`!O31(pfF(dBmOh?hkirl@ut@5@3
znt-^xi+Ge6#wdk=#)i<8(^PKKbr4_47VLretk?tKf~5$~dTv1dvLQdBx_$liAQ*MC
z9E`T_(ttsomj>rOe)H0ELKqGkU;JJG*q2zAg-f#)N1wRnB#qlI{u^rG1bJvzb_xS4
zT1@_x4VdWy-3kH$=O6{8!x|26oWqDc!!P%{u@e^G%p=!RDYmhk#v29s2TMFdctXg7
zlwRHSFluDkY<_b(xDv2*Ii{SFrroA7WGV3;bintn9ZcY+QuI@k9&5GC(j(>XhGm8^
z#whQNMI`mw<5ADLFBRrlc(M2n+w{_l9o@Q1RFn`eJ6lfWx2Z831K|zpVB`Xy2{%{K
zPGGw)Pg2bM6j&FIgL8wW*{6OG45{-@kh4sVmd&<LiRIAwfXkQRF4E)nnSnPLIMMmu
z-@>Rh3yAUmzliSsvaf!=Do#7-@VHni2~z>)$*+(H_iRK5is3l$6gZx(I^FricU&hb
z{Kyn8!6}MaRqv?Um4=Lq=vJQMTugKe+GcWDsSt=bcNF!auEMvHeZ7|l=j200vA8;{
zZ^Ma1I-<L!z{0$DGM{is;(98qWIH7)3`Tv;#F!KG?+nVzX^WM6IrA`SV6MMQfb0k(
zDub-DC|^GVOt<gCnzR|VH!gNfz~pf+1B=UZkB9iro$UqFiLbOU>dVq~0F_dPCX4RF
z3S(W;*HAi0BdB@!%<kGgk<BE9Gh+MuesNWRHppO0v0dW79wN`TL5~Z9Q}Q6XtC|g>
z5I>Ph1F-Asmw4E51O0HVemSVA00*@S->9wf-blqb!$7A3y#gaje=C{RLz!d08vf&{
zP{YO>-qFGRQbPPRX@Y)cEkTpUE(em{W*2u%*()^JOqtYM4#vgdlR^mu>|>U}sl(Z<
zkq_$zgi%d)#cfs02%<8F!=q1kjGr4==07`<RcVX&0>%YK;0*AK$G0qn@@CedO28U2
zq5)=Sw#8%X;B6Q|CeY(5-<j`xLFq(f8E?<iQEKwWO0)(fi9g%y)C#VC*Ji*FaF`wL
z`+J}8kLDAAqypnvj=_KD4-`lcTM%&2`aZw89rjR_`=lQ+jjo0HI!Sr2gmRPBPD%oz
z4#P6p--nVb5IdkTXTjgzE6+dZi>#aVX+YodcX7VSd}5;-=1yLfYL%0xc9Al~8b_T>
zhzK}^**sDI!zoNb@h;=Nx|5uT7b(4KXN*6f#~7Xev<j1YOvn)>O68|JxiuhCBdt_{
zvQk^&Jf_XBv?!`At*mKPZ22axUb_Az2^qv&4P3puQHtYlzaM{fKiu1@ebB4a=0L0r
zQ47>}YJt^zvBE0VVk&rW>O%HEuTx;009@3`E)dE5ZkPY@1I`1E(HU+^DJ~WB&!{dA
z^gE<>DEW226Kh1e?n{o3ts!r#|6BBQ2=v+;x)zcG;N)ot=a7AQIi>Cl9`!@Jz<2DL
zhV?z}Y=9`?YYouKfe2vUbLKU;b6lF_?gJCO)8HQ;7fZ5V0K!iRjcOasqjr?N_if34
zJvSv0%0K8N@2=Bsf4&gHi&16bN+Ptz*&ZiYzLI45bXFI3I7+0OS*TQT(5DmZB&-rt
zeD+jL9d-?xiM`s=crseo7<P)(sFmKCduK{h=Au(#O~X>9Ub2oKu4H^o_yWpGlvrh7
zP`owM3>(P%CMxhyGRU1#bCJd#COC2$5BY0w5~w!;ETfC!wHrWq3DobSu7m(&dmQ+&
z+o#`7sclEeP0scH`C%XA0V!kf$Rzdeli|<B|HZSIHTXe$PvU<ExNW*K9rJ6%qihcE
zzikMUN#mls>Btd(VZDop;Rf!HNu)EI|8gW+E0cB`ge_}SrN<Aqzzpw215|p9x1MPh
zHszU2V<p+T8j3tzuer6jWc-WuKKf8GC?K@r)&Ui57xLCn^l2|}Pk!8?Rf1L%Bg9b$
z=2{~0ta3)(;zElQnjCc?%|g)@m`n<P5w34XS<cM;h5G~avEUK^zgPzhY5<Yc|7ytd
z?<DxLvpxJLN<-oV%!3XqUf4LD^%A?tR#1}AalhN$tgxA&pP=4kcBL`|y`a&ut#Bzy
z9Bnd7?SgaOU!O3;VuS0vlG%aBT#ncw6PdZ9<?slNCAtc<IPSovTbQ9&g_i%EE?^&f
z0Y_F@U{=1<ZyZwB$lwY{!0RA*-B9A$rutO%ixmiM_ohGBBI0z59i5#36mBOHvVX6j
zvLDdrt!aEszTNTX@%Fb1Z*+@57!Mg8sAjHWl}f+@F1lhFRUTDv9}iU8W#LB4=vJg8
z>D;6`s+iDVtu66134SQt6Av-voy9X(^-k8)lMfxDP7Up8a~`;FTc7%{?CxAAY7N=#
z3>0shwk&pQEsCgM`NmB+XPGqskt1j=Y>eMD>LA%^dn`!z-Oxz#MK;(0VCwaj*<S&<
zx9)&vTh8nW@g6G5^7KEoR+?Y-$hs{G5llc|cjLj<Q!v1r03kRw<~okRzE%&)N&z@y
z<hay`(nLuMtq9O5M%QG$`|ZXvDh^%y;&ln0|DjD6|IssW{AR@$q5e;^;=kv&NFg5k
zHh=%Frg<Yd`4lhprT1#4B_8e!CHkozo@H95udC`mp;f+{S)?`SUowboh~f2ccaIJ(
zYq+AT$xZW;=lI>xIpr5kBXjqge5GFH?g!cDDZC3B`5H8$@acZsnXmhEpLMGW=TdE$
zYU3D9R-D)}4hNEYeD4zBA#1C}HlsF(^Mkhaz~xEV39@wYka2~}!a6%+z=Hoec%Knh
zUpFTSKkklYj06C^+&Pfuk%-SlJe`az?&rU<D$yD}_B=HA&9B3x-~sVoz3KVjr_ZeZ
z<Dk?%MIEq4Ek>*)87<NH0(s&Ty~}7MB`ROm8ZSP2>N(Q5Ow1(XqgoSP+17GuD1QD)
z6)!*H)R>NuLl1MOZIRZVS5Y;*<(O^S3D>24t;H@gL5#zmbOO0|HIoEx)D+$!I`1KK
z;fbwfJUnd<tZ=mDrH$x^&vh=fra1+_F4Ty8Gm(5Z(hP7;8qLyUl&L(RacoKF>9>H`
z-?_Qa<+&?yv5xmP-5|hCIUsZ8%C@ueLA0o|L5wX#za9%w%02%=+fZvf@b^~x=MU=`
zRlpEsc$Y&bL81hN^|-2W2tCh4EHQ6`sz5;)FOxtup0nh>MG?D$sCKUW>~d)Waz@(7
zIeM%_X8jb|qPcUNPwDZAjgt>BXvrRr&wEzHiITIkRfp*H7d=x64@K>~(G$<GIyl@@
zCl#cL{8tD-=yu=Od8)usV^Qt4=`L?ZP)U}G0UNwUtQ3X=qNd*%(6RMKX~&a#R@Yys
z{s;`TRc`YdR41&hO;uW20TG5elH=^@^K@kh(Ms=IeISOo6Aa1dT!(X*K`vi?mCy$J
zr);L(Menj-MWZSa4|o;q?^!>7_*XHG>X}oC#7Y60gZBc4kQqj==ap5maNLQUWS)M_
zk5Moa!`*_@N+-}Px$boa=-G$bA%KI!811}ew(=|-WNe2VNJpZHxAj2AwljQ1dt5(0
z{1LJK_9K$nE1}5^vX$f$t|_6(#(aUidk#tp)PAEiOhdG`;embqR&m|~Lzx!2?N+nT
zxm1>-cyzu7>|R^fL1Pp&M)J;Tw1phuA<q@QfA!>=mA6)dUE5h;5V`<*&T=VY?;b#&
zV38lvxRB8pCU<;aCAccKN7`TE=Eu4BwUQHM#@Dcu7nNUQ;OWaof3ej$uV2t)cl*2$
zg~Wf*6${8|`$&}D#2DnVN_^Z$ktA{;z8ys=3vM~mz>9<7bdu|gbYV!vEZjxDnW(Af
zWerYDHBHfaz4I>)1gCT=>h%T_dwM=@$79kmL8dk=Z94^IYWEP_Yi6j4e?;OHD(@9z
z)otR0m8icO#A2ZIdE$boGIqb7CTtyericaJ)ft-Xw$A{r|Dn3Lg^`+^XfVxY@>paP
zv!;U(%gwFcEA>}j!BO#@fLzAGZ<NDy3?_gKgxq+{K&%7e3vVX1@v(3QZK^Q=KfU>T
z$WqX-WF?pS{_elgjCc$I=!{RrUmgGT$o-HHZ4m04XJB@m-3DL3l_<==QIQ)gkt-0d
z#;HY*iJIsOBqo$R%--CoH0BF_Aj0Tvvu-`fpS~}+4givin`4o%ho+^7cowv@6`7I2
zN{7ImW<d<yde4MB_6n6QUdhlP<5N-`J`P7V?Q<21!P~x~5vLYkFda|_RLKDRuom>q
zg*ApR!R(oCaW-ap2dIl8?|lFER;#r~y4i1?1`MZLM(aIa^qlk@p9%cD8V*>0XP@Vf
zL@OFP(**4CYfo(YG5kIhu>4>rOdTD4#{O$9Rlq}|OTf{!bS%&l%Q^rjZ+we9q4xDi
z!7%jO5gN9;V9puRQ|dh#%@;xQ0Y`xj6>=b1KM^=8nkdI^AnF|TB4^-!k7-$(sdkb6
zxh1rzSd?sb!E&dSm36Pc%w)aeVjpU5n-+3e!sqnPGWH$4aMIt2Nd%RE+>b^LD@_J!
z4z}+_rQg2vjR4u0z`aC=geE`N>muVOBtUMM%K4xJLV=r`Yw6Q;H{lXdO27p#E#_;H
zl5jAs2CvPZ`&+foC?YJZyXM&aPeAz}Z9zE$UNo4H+I-#%*osQG2rX1>#?fRMKe^e*
zaJ8zEz4E2J)8(BBZQ#A*;{YcrmT0fJ!O~^5SfUd(b$3<Gf5@1XC!Ou|bb7<ewMV?(
zd_5pYd#N6Gg<5b01ZktpmL_#B?QeRIpUCFTQx38rhAo#3*<|S`Y&m++jx_oy1zQXF
zUiVLU(H@}RliA>Rmq<MhBxs%5S)ULPujzr(wyp!O&v2KE0s=SDJ<b==kwMW8;VDX4
zouJ_%jT!#^-HzHp*V^rUqg$N+UPO5lr$V^_L|+fb=&#q<7g5Z@@g&_TEP9a=HRcXg
zhpEdb%qWc{Wluo8Q0<|7_qa~34Mvz#q1aIO8+vudw$+__1-U0}7Tb=}s>le(CsD~*
z)!VRs=HzU}m7v%ozgxDyh7#jBVXCWWJUe1@dTPDwVXabG@lfu3Em*@yK;P8%92o$K
zy~0X|n_bWc9FTGcR8#TH<J`3n*p?DhHz^0RUVgQr_xrATa4*!J^?bkFz~FC$<ev*F
zNCiGwo}$my0b+5CMI;|TIj6eYLjF!sz{{x-ZB&KETP`;KU^Rr{Q4G4P9IaTM61%mp
zs4U*y<Kc?&?k}f}v_sU(4lr3o!;EM>pD}c+mzN!_E8(HoFlAJmv?!QxG;UGcVO%$z
zhAnJIgpyA?XEnuGp}D)YehFAlWUO31$?q<_0S@UI0CjY24|-N`J<o*k%mMr8!5w+P
ztbEDqx6yOsRr*lw897L5GWbmJEyl|Ap3(1GcQgT7#4`u3w?zJZ5!Hx!@sK;zV&hz@
zYDMJII5hgYH$(gIR9rl;WvNsefs;DVRyzcOag}{a`TVc0dUDjK#nL$2RMcK{b8by6
zHk;rc4}YX8TVPm1c5G3yxxRQmdkdonSIbtU>I-wwxb7@7SQFT%r&Jm!W85qr6w_S;
zb8pevp`3Aw71K7(Rg}4K>BR+*#<96ubLVVTGxkAvxF{$_=0Kp&0*G}K*~32Y`#7f{
zY7-SzF>`AdOApbEo!nSGS^$^Oak}c_AetLIH-8YvpOx?MkLUgtaqG;#KQ}J)@oq*j
z++%|iU!L?Y<T9~6xNA|<Zgsdu$<S?ZaMx<+up%PV8y#I{amiQLE%Bk2Z};ZiW3{8_
zTH>?g>hlJ@7}u^+Em^jUY)J_Qj9R${v3lLiv!cQ4d*~r6H9lqbnw%?{jdU#jB${ie
z;Q)?TMM|km!7K)Pc0o!-;j~KHPr&~`lE?P6S%B(w0$x{Ev&Ym^a0cIa6Zq~uwJ0<e
z-vgvhdInJ|lPFe62i3I~HTH$J4}f{`(|aSqAAK(}E^z|ZikYKUt9}n1pVh3Yjg<Ey
z#?02`8>ru|SWx`_#@BEN4rj~pgHl?wejVQbIKAL*1T=`=M~(G*TUoRcA3yL+nna%n
zy?qQP@ay*I808SmQ(O`(VKlL|##<@iWoTP1&}>sx-XV<Z)?w>mI@eTDwltA5E|iEe
z?P=GK)AI0L)@9_uB~<o`QUnM5=OwiFz+v;aY!gnTyWKv)`OouiXY0^;!#a1q26%(n
z<g{`}nA`%T{CsFT=yoBUDwv6jEi557jyWhfl;q!o8JQN+zG=NQyRNrca0C$U*_v;<
zRr<dIzPt2Lgp%m|dRaaNnzc^9Cv`x<Z8&EWM0urQg<rkPW^zIK6#QAhoq`Iu`Rw;%
ze;)9^yhHxrzNntz`B}hW0*2xl7fI3h71z~w*M<*M=PYz6RrCEw;6!leNq_3W_`q2f
zfqV>Gt;oj6I|E-4?Tr*yL_?f{+2E6j*5qC3i~XUBIo9QL#CV0*SEf!}RAS_3!VsHG
zQ@2s298`ENEiY<PoMSZ%=fL<lX2r^5tIV7FQL_GrHlUOhBvtz8<%Q@4ReBew))#%0
z>!CaO%*RHaQ#*aEDsvEJ+B2T(BVdaP-Le=2Sr*KS{xl6irCB%loZ`WI$O_(C?8Y_i
zugkpVBuaJ``5V-NUxU7gmSCM0&VT<RzC&NcX5*{qtt>_f0>(@~+%;U@gZ7$`b&^d-
zrQ~qfIAwwP{9=zn7Owbix5u<jw8W4W8~+E~;EGj5^R#vjwj#ea3z?~XpGMYWblkyE
zlq!{204={y@}hKV#7g{7NO}cR2D_5ry^P{SQF8Ky#3}cX6dVdikJ(^Ob96;k)77YX
zEl`Uoxw-(IF-7pVFGCEa5+#}(V2(-(#>$s+%Y1CW*nlN34dPLmKD9v6)N1oN>kq|$
z0#@HOm6f7x*PK$r`_+D*1?<ob;GcL6{p8<0=RQQKk|25y8_%L~-~^*o*S+6@`^^t`
zn_7oGB}CO(u!L(g(=R6|b^q{ruH8;bzLogFC>LCOh1+P!Nk*!>()L1s0oVfWg`%2)
z<(}TD753g(H**^I(a6!e0h3ZWlWHo;2a*SyjyqCqJ!QV8y@KX#)1|Pil;pXyOEQvR
z)$fCmX;s5Ru5HMb)(XHpu?*Zvy^)+DnYptaJeF7~G9@f(N%_yEgMTOB-Gu7YZ*Qw^
z{+A)0I5FsIg?2VxNPyhctD1A8djVomj9<Srjgl8Iwl}^o&Ls=Rad{wP#q?@iZnS2r
z)|##|MRhGBNk@^;opM};D$;(#kpqiaD<Ihx=ARN4y!KxFEt5qB>;hw=g+RWnmMv>Z
z{*c2q$!X;Sm~6OE<hIA-RF*QO<&t4OXQdPJ-3vUxJJtJj(H+$w&B4s+5KRLc9oN^*
z{)3tvAQC=FM1(qUQ!pj(>{v0~E6{{90&RHHW{E_w&<4OHGm)2muO$EI7_rs-*jVe*
zo0?yH`)}Jgl*B>>L4${E^0pi&d8GF7Xd+!4C;t!oE`9Mu5VeYQ#o_ddKI2q>A=_$E
zP$mJ!wEqZQtyr=slWqp0R%cz(^zANOrzswKV8Ow_%ctM0yZ=(&`Es0jw(_~!_W5Fp
zl|s<O<~2n*f5yWwO*z*)V3K34&iRT6AmPkF9>p{PECl;5Ftwf9o&h8Z4UPOpo4U(-
zU*IX_4nR2bNL?OVj;b36L3}n}d;lYfEI+6dnfMn^iwBXR1RS9Is>##3UGnF{|Gwyd
z{b0{h-Hu9s%@akkmE4YRi|`|&xiV^WZQ0B+$LWXH7t%Wwczx}UQV0r!V8@xMx=NaO
zM;+o0`pR-*P)MfPto&PArOWZ`Ra&QwVlPuq)(1F^A6onQG0GMwI;ZOJJ(_V)%<!4-
zrK@C0+cAAN&^vk|2b&6xDapKfSaWsGJoSr|i(fy!?t)TK9s%FS!SfcwplXR@mg2vD
zEAl-EvLFUKm>gtK_SL23L_yZt*MVy~p^-`L?=~t^wVn#BvvsSv-S3Y3AB&n93e_zO
zV{*!N>pVZ*!In2FZXlEw{q>2gapN@vLhT`Uup$xK%jLZJXrip_+U1VzU1?)OO+)3y
zJ|0A*k5FkxGC`V^VuD`1qmOAXLA;d{9CbBG<pLSt1q9=zjA%SsJs;=espPdQWSONS
zaF?nq_SxRk9?xnZm70Od>)5UUHZp<09qd$P4|SVQ7G}a&g`FdKXY75FSa^cfOF+c$
z0^Q_Qgbry*=K-**rGdl=qa_9z(8UxEpgXDO`6_LlS2%tWiGD1km_a)pGJWO4%iksI
zKYyg>LTx1TV?)p$1?W*B7*qv(fJHdCl*Wm5ClQ>%I7^Lmtd&{^g!|D$FPI}aQOLkC
zq50lT$JB8NDio|LTDiN|W67|e<&0U1XVQSiC8VM0NwUlMv&7j2NQ0gLT)o}}OEv{~
ziq7Y553ZU6D$hiNrE-HAE(A&N0AuCFTpcvJpH|80R77J~iV~2WQ!9fKToTY>H-UO{
z(r>l(Dtb)Y5$HM@0+t&sX5$2&Uhbp_NrEcXmwFr@E-u$D#)w3a%gspwF^FJQ>jBnr
z_4$3r&U+*afIdO~*mdXu9jbRqqrmswuw!L)1L@(bmAAVf8IL+BHRNpjoXd|E+e8*b
zat~;){HIr|IzZnxqtfM;x=A?T<YtaUUyt0K$H@vP+XYL+9M~AWdPW7#;)cYv`khoh
z7-f<q+QOI^`x9X#<rqs9>M0Hf4BDon?bLjiBnGARE_Ucq#lBD{P;Da6xGsIp%S&)q
zDm_-2M03VO`ALCkssGA!7E`P-H^oY6ynE~92#@u2g4}j*L%k0AvWVS5_yn&kxToGv
zCTPVnl-O$RDhD6j1JtHbJS4a9;B@F2Bz%Dv2~hzmC9K*%C9T#EO0+3~>_&M}Fs6A~
zA@=+R(0@ll`tgqFYv8aLt!7o3-HHrZ-vi*V(z$He>gAc^qi>O*np8Y<<PuK5Gg2Z?
zSJ(^QG!+Ex%_X7HtC;f^M|(@i$2zZ1Im~aV1FsEBt2)iLr5N?cCy*@GZn^%X>`0a4
z04M~J>_gBBSkja02NJ*{e-nfmFm^gT0(2z0S>n>YuVxQr&}~5+U%pJ$gvfjvo&{;V
zS7tzZRb75%Q2SzcY!?VQ=>k<*Ui6jiqpN?IxTz|D37cSKvhzQ`?B9Qxr9o)U!$co?
znmt<;x8+pC5hoGh&C@-)S44O!B0F!+O2<C5WGkl7B3FLJ=v|?Byn2;uAX6Bpxn9T$
zYWQ&3Su`Z*biC<PXpb|ccsjDwbe|e^;Myr;J=bdjQU_hjo#WhcrZE9R_M0X975Ipd
zxQ1<zI@lAi?Ti&r@wQ>9X&~T0y{c>HdH7Ux$@Lhx4bcOSo~)uhNH<t+`v4K;2<EZF
z%>e`_KSWd|!~9p4SC8tfeBb<VDCz>Lw%pfocK{Qv6HF{<vo0-fRoWj3pL(YI0c5ug
zx3*PC*njV2Y^5>+jrp@PY~C2^!pRYN?l34v@IF=XbOB8QU~@_QaCjJ_Rj-iAIBA2`
z><f)rg7`tqJv=`0dYTdKtEuKjrjIW~JMh*+k%NMAMKP~$>b$pf&gtn~mtV1V{;Kl^
z3ceFOt6rFY^xn;Jv_MVk*o+)wb2ta3B$qUQ_^FkWMBE~ge&>n0C2-@v4qg%q2+qE4
zel6%(fF{Tb4E=V@^!3oCB1)qfe^)A^S(<P?ASlylz2M>2x;y*wXn<4k#e(JW04x)q
zv*fn-O0Oq7Ce;PHQln21Fo;?NTF?wQnahyIme|ivMjD6cN1t|*`CMtg-^U{#rLd<B
zi;zE%#9|(|`sIY0J}~yZk)X^mHePDH8RaSZ3gp|;Uu`@3QWQX@6m#8Tat6o(A5ZW!
zs*RA})Z`#z6t5DrR0zY;&_Y4sjy-YA{-yJ&(SS_)<iZ^xMDNw79~eS5!#F2ny!y6g
zoRQ&>LimD}t>;7fOyBF<C!7+V)$1U7LADp2d{UXsH=woY1m-;0-iB1N=iWcBVnTrW
z(U~+l{VV3jZ^8#v(NT@dBwOn99W6%|`Bd2#v&|IoqWMSpii=3GHp>O6SGg`o4I%vS
z*C@*erHz{L-RbExIIz!+*-_PsAwGT52|@9)<X7LT_IQ)vqzsnDkEN?zh{fUR16g~{
zTc*9zdE~Sb*U_-F7@iZHzT9uxN0@>p@5eHMX+TN7EbzmN!b9HhzN<55um>DNIP(G)
zqq(;_ezpsE#;?=!dSKVyJd*@rYsnpu+8MZm94GCbKV52JX5=$JndnlR-WXjwrp@ZY
zjeKbP)OAB^c@$9JnlUs=MJtfVB08qGZZm+v$fy4mwGLhW{8*)B%_gLwm>wa#i^tKs
z4c=V@l#qRr`82QJ$2f`yGsA#)|2|2rLaxo#s9=7Q`pQb0<?uc7l4<SlVCcml&(2rh
zAig<ImjS9KIa?po;E!`n3&kl$!g~?qtPD^PmwGno_vx!ZsI&<zG9n<RcO0M%T>+FS
zEf~aC{SuJJUB>!H$ux?CNDuIuY^Nc8{fuJe`|h3N&-q<|O<}S5^gkBQ0Z#}VL>)ZV
z{Jowb&!{XRh7cyv6=8OSxsIYci&J`<-fcd`UI8v;<XvlB>~5kI;^ZgAFodfx$~!+1
zPC59_GAOHtzYbB8^$77O`SrRLRx~EOSfB31Kn+v8c{WTm$1J_4PbV5UOPL58Ij=LK
z(VTabFF0G=K*I1NDA*Nm0h{1j>m%8K&3Q@R)#pX#9P+_Mw{JFBKPhpFZ$_!_eY*3V
zRlIqvNVGT^R^l^qU&`(N+xOQhvYCiEhQZDc&0GXS#Q~!_)xOcmbVdU~l_CTnx!VsF
zZ6|6_kN_QmeUfqfv-Twb1ye$?>XW%?oWVP$@*s;=Jw%20A{9xAtmkyqCZHHd`A^-_
zB$K;thQOD*34s^ylOzv)jN<cOLi}8Rm$F;v6&>{QS4;FBO}x~wJkP7PW-8gOqk+$>
zFlo@WqB=X@zkJ*!2T0cHujv11UIZgm4O(P9;9}se3z7%neqCQ)UCTeanDCv`dU=dW
z=Id|o&%f(ZRC{?7X#>th(J^4iY;hUC9*=S9o=TE{((BrdZEkhDcKk@M!}OO+Q3g-a
z8w{f#)@$Bm(76Q5%4nS&4>Y?I=AUfoA!<!q$;;TQ0WVC&xZ@Lx;e?iW?|g2g*N=nU
zEt+E)q{a)<fDcMjejD4OTbJLR5Wpq<>=?5_zu~Ec=*|4Xz1epo<J)P6_R!H9`?j1X
zjs4``y`H>on7LTA3+~HE@Qq1tR9rq6o$834tZ)SL?2_hzFZIiYCylIBcS7M26K0U0
z%UW)I&@O--;ZLGOEa5*(cntXNu$_;uvpvKH$&>=(jBlML61d5B`+Y568|7T)0vdUU
z(rN`{ABOICFKjY!oaWmR3xL8-48-`yz7z|<ii85_=EbX8s`KYfUmTON_MHy_-+UxQ
zu$-?pXzW-QhPW|g-o&cB4}|4ye|i^4B2EyampxGj>2=`xajSvRN!>X<Z1?N~JUcD%
zjQ{9A_!FrCZ2DBEcMm`&85FliL6QYw5FTePt{j`t`jR^3DR1p6S10m=F(MNWQ5L*K
z_)@PlW$l1Lt^po}eh-{tF*Vz4mLQ%hy;*#*9%&^uk`A)Et;8Z;qEGE_;i|Hh8lLl*
zl~(Q+7AW!Y_>8GAvWsK0ac5Q0rdSk$IYo6#%lTgSj%!ymo%g<Wn<3zOS3noJT-+tE
z?4Bhdd1|MNRVCM)Huq>WInLhNAztc<Cj*6+<Gf}>Y27m`?xAmopGVyC?->R|&;sNF
zfE9kg>5VBv!T|xpuoH^<nzVn5vM1J@^m*x$K_Aa7<$8{PGEu5+^?-m1!Xoy7ZnmKC
z;(dFK+q3~epKiY;1J+#<v!8)5ECXg!r}_P@AofPrAG2d&{?YN8c&xP`((TL^hPDe#
zWGsl)mc2n-W+bGKnD_1eHD&%qO8H$Jfgj*T>C<%%dbZkapnmcm^_<00o{BXNcANm?
z(V^3JAP8IhLB3IR$uj={P%GgT0m^+C)?-jdyjxg%L+i;X+?z-*HRqOdTsVB&uXed0
z_mjYd*y4mWJclfMRvv9uHw}!Cc3^b3g-1xV9K14*P&~d{Ac`hhkmv&i*Cr0J+-_i%
z?j<P>h3&;y3GvT$)=T_70HI@rPW92pZX=uEK6NP*l}_v9kw}MTW~vy3_iPW~{+ja&
z>Tx!ye<$J$qB_`FPwD>5M<4JWpuG!yJV?<UQ%6pS!IvcQb`RVi9F2NBgJkmCp2J@~
z+4#yLugB<mK#t@$6r+Q@Z0EPYQQ;<#4R6`^0S1Cv0y&Uubpe>8^+&qA9RYz|1ozHB
zrECCD=}FmFtDgDe9Trd%STEI^+72%5;&%=M0dqZ7C(i*@;3iA&UwW~Be(ca-pv%b@
z6s5feoe5S$egp@TXTdb9A3sHxM{JBrm#x##=wOnwk}w`m+39{{&Ln+rBigV{ni5W(
zX%E|h8M#z3teTu`V(~ip5QC;-crcWt6Fcd;*1P<Io9~fBNHv0~Tgonk<Sdhc3STg0
z!-{^pU*{yF<cix#1=!-Iry+VhrQO-@wsHs0N2yKz@ADmD6%9g=xkVS4^JD}XR{PW9
zmqKx1r#tXgAU)Fa+gxgPqTWDC6G>|%Uzqp}q)bvm{&Ize!QJ$pd8+puB6PNYVDHBk
z^WH#HdhhI)8Puqb3m{oiMtNZ$LslF2GqqsEYzc~3K^f2LCS@Ri1jSPWG%4&1AfT=?
zoSr7+uy(Uu0s1t0$cs!}PVUK!xdzfeop)z@P{Iw5iGWk~j|0E_o~mby>;ungfa57W
zjGH(+u7jSEc@y#{nDzIO4C0G1C_3Y!p61fkyxdxRqYRFEDu=nx84RjUl(RNFLdqAU
zx&(sp{r&I;w1jdIXu=A?dx}TySA85<dV@}{2ynDbCm*bPrk2o3dpOWMUqzQ=Y;pSW
z>P-*mE4_E9!|Zphh8#RlJMXl(u(US%A0o~61(_=Yon>O3#m~89SY7*yg!kWIYpVir
zeT6qaz42(c)h|Q~$@5dHQ1w@x*aGGe3jp8f`_euXbQLQXzCy?mgxv*}cBGY6b^7Tc
z;N*W<!6tnJoE)1d&ZBb450Ir|v>KKsAO|SLfSmD$AcjY9-U1o%9@JB$_(T+<_qLAC
zfJ>P4!(kRHO~4^Ac%bmCs?~Np1SAGdinBYT>Q7sMl7)#o2gHe&swG99y8d7Qu!?F*
zf0>^OX8(B8*AP)kKnBt%=|pfRdurwfin#2U6dBd{dqD7yH=bZ{N?<dSN^gx8O9MA4
z!cCmUiO2ER->ndDlg6j<G+9hB)w!A2voxW_*;L}0sM1V|M-gu+z6hJX>z=!8Vn1y<
z%*ZDsARF(P!%#ywZk(&kFj!hIc(?~Q7uF70F_6=%THFMQWUw8n$l2PgZpjlU<07i~
z5)2(`*W?EB#VjyS>V{v<d~$OG7_2d!zN|Y|K~i1+4#K}7%J&+KXRt1Px-mK_<_s0j
z9Kc!z>%xm_18WKJkU&)jdk=_?;f-AsXGB3)EM^17x=`nHpYtJ9@a(<*6k_>~gyToM
z3XW&8!16d28DUYI#z%b_*A9XD{$$IoUExOH#Pxa;GH)9lu4&>z=o}MJa+r6{UjAS@
zU)b^i2pbkaPv^nkyy$8f`~^d6eDCpy^3ztTuYaIfs$zK6TeIcU+}P`Ez-O~wlSAcr
z3yEJN_J_LfpXUqs3wy>7a7QQ(3BVn1sd$~j0*I9r4SALF9nlyH^0RBOF~Y`KF&=6+
zL+3S?qEHD**|D(aHMKjmkz>Y}Hgq?vMZB%?70?$BT`nHoq}?#;kQWzAXlCeCueVAY
zOY5F+O(GU!8tm?z*g_K`0<akT6nX4kYIZVN*GPc)N>8u6_<kBs{LkdHGaxkPqm1AB
zIN#HX9Ay^#I|eM=1V$e-Rq}QCW#a314C5^3wqCtGmIXreJMLNWqC?4y2B$Uy03*E<
z=xo_|i5ow)eyh_7*<7RB?@3^14ahMF1_n;tQ5t4Teu<EWssGdi;H7s*;<<(Zl6t|)
z|6<6{F0hedT_J=4q~XdQU;-BTy?g^jd8>a5XS5>*en0w!xixd3lhb+#+FsE)FeW9m
zx0~r4c?*G$>UT8K4v!&|nCn2aJ*{vBObQXGcN<bhJ2T+i>I4zd^bio!9<tjq*$8s|
zc!PkMis?kjA24#NRzZn9Yg&Cd?;2D_0ldYnqpRN`N=b3RbGx(WSE=-m5gOP<KW4|E
zNXDZ)*@F9ZzXV@bi+QJztbMI4K3&xrw6;qc<p@)qOYQP2F;hibJJcwf9p}CNcnX9W
zZ129km;Sks?iT^9TWJ8B3~`#h!r!JFWJMazbTYrEmg^z?5+6=&)~>meb)|j~u}qWz
zBbDT~Zl9Xovsr6nKNscM?P|J>4bV00<O!LVj=trJ%ihS@8)ZRyE)0F!TI!X1+0T9u
z0cuaMXx^YYA}}g~dkn^sPz5~uGV7^T=6}kh5tMz8C*-76kn#(r0$?2=ZR&L}l62R-
zz}+y9iEEC_XEY+!Bi|z6CjGfm%D~Sp9Dtv;8jrUjO8Or75-x7q0_nLU&~S*N_vIk{
z3d5{>$(`@Vh<)>Kz^0T1(n;T~tRUpr{ZJ$6&LeX6_Lt|GrO1a|gq;io;=k?+!3<S{
zR$1YjhWm}r`odmFMkxvY;ga@0ZyqGHc^Gc02-(pD6v(OKhce-(OylIAb0YJ@!;JOH
zyVEm;^_Kyvi8Bm~8FN+g55kmsvlzsQSCgE@|4f_D;+!hAD`(|q!dDs&yO7=6s~&o9
zY}$3D%&<-N;SE2Rtr%49>UkABD%?|%s|62pRL2hp6q-r7DG#E%*IwLe?p6f8AnQ^2
z49oF%j83jilb;;&1Q;gyTtU8a@U6$^Ce7qvX5PKPnULu!!0%jN8rz9-A$rB1zNh)2
zj9j&O7T~S=D69l8>VSX)!@V{NXmV;wrZ33TAbXC&Yj0ccF}w$bQOFINShjtc7kr`6
zbO?VjemM7xC;Y;8W5y2rhEXqsgpk2F?;Lu6u}ZniQ?Y541^f(ox9$NqRsjN|mx=Wa
z?MuL=iR|^$&lcq@8Ybr0w;Dn)Lvs~8UC--3Cu5@_B<M*`lnY+~aHwF6J|Acp0;<y&
z`&pMSzWW^TAGdXb!B5PUWNR^rO3^>;+aVmUm_2Ih{hK?&tGe|B7?k0tynbAECD{^R
zFzzkFMag&_<4}9A12~A~Q$ARqU>oP|6JQTpjVnxJ)f16n=;Rjc@8X9)#?=RGK-qKI
zhIsnQY0F9TVOz_ZqiStV(#C%Dt6T!kupS{fXOE9JCd^Zivc7#I@{4^PucwdM*Qv$4
z+v=+<YwR4;&R-yb>i<KO-nGLQq~a3@2ZxTkgbM&MC6#g;KB@GuNTHhVs#Eb|LiW{W
zq1!h4z8{1n(i7Xm_JR5}p&fO0J3R)6g(koT=q|X9=NL;=ue6h#QljClhvfW_CqRYB
zshOAHK#C38m&u?BILRh2X#?ES5GlfDOKt?M73{vxVLz;JZ5M#qHwokulXR_q>H-;L
zbNuCTCB>H?0BbJY#4H{X%8+#or{AkL2euDitg2Hi)?R{RuZ!}|URQ~!OgAVasPjS&
z*17Y40Uais5YS=q_Aq{3gl~JEm(nQY@t@<Bf6P++*zgn(y_dY-`7VXq;TYmD%AIb4
zd_8G6`4QT)k=nRRC)k$&H?P@8?-7DCe>TSP*(!g80$rH>2Fk*|S=OC-e1^%oHK~zi
zqUR-DsaeQ&o!)F_s0pK$*F8fxKE~Y~BOF0vey75JX~GJ-TwHDCV=_{c#&iyBln+q?
z0h6cToc7t@__c*U?k#bCXKKuv*mjBrP8KM`0F0hT&ed9QeJ~J^lal~}&_>4DWS6Fu
z9pe&y)}kEX3#_0700T$B`uQ;>AP(KW0TsHn&b<sxpFLNpPuD!#)@K02v;6noBL2~l
zGZSFyVxCGlMqO6C{6ZcK93-#KHNM}8<<f|_2XNU|-8r>n=q{?)qzOb2aVG*@S_6DI
z2p>ku%_rK{7EZ=^LQ~ZLWip>ZCvfw<xHhcf6G36bv=4%ob71f8(HX&+6?`s_<y{nb
zdujF8YX{SUtZ((EO&fx8A6eh-Y41%xX+2N;Zs!r?hctTQKN`J)BU*@xw0IR_&-pjM
zl^DVWr7xfEHC1+79KHz7Av~{jg7Ml(bFsSz7UN}#3dND9ESnkX0i7=S7|otdn1A)T
zhv80|3yal->*BhAG8jIeqGC(u6yvw#&gL&m%!?RbU5T|N4Bc^z>xweXO>Q_BlAuVX
zc_1ATLcrlk&aUVuihT0uyaZzP8B}UrZQWmOUuPfk3WTW%Zwl@bBY`)SIOH`P?jpmx
zd5UVR9n`WHn9Q%1O!l+I4b?Qw1p!eOMcZDtgnq=GcW2&`K#nB+t<5>aTmeDPG10WK
z>f3n8qG3fA$QvbbuQK@vav%$T4#=7={j_IDas4eg0DQX%*}x^(J^h1eSRjp94f3+M
z)icnd)DS=MVDC;z+(Z*lfQZZ3{wu2^l*A7QqUZ(8iOYQh@U=Dm$FC^0Q4lpfIv=9q
zbuM@^I^%+>fA@kw`c^8OXw6)FcOtT)dTVbC;v=$VUsPU{Df0nrmi23?N#!&X@?I8X
z<NSh6k__q7Tea>a(tejoyq8U{BBCCGhKGCEjLm-=PEn5%ky@0dq`>ZF=G#o(k5mMM
zVJ6P=-ZfVjdm5@qNu*!=xQ-!i9%N&uwYJ<%P|+Snc<S`RJu>DWVh<m17i?#yg;6M$
zuVIK+1kZ10QW=%>sq}=;9A+8B8wW2p0x@kDuxPvA>^Y?aU~&287PjoOw{OTF?=5wK
zhQVIsSv?_#C>+e>7Qi_sa?8S?I-H-vNN)j6_an0yfNRp}s=fe;d#}h6^RCtxG98<_
z@_2}}5DGr#k6BK-^Xpdrd$#}zAw+@()S|^b<=~Xy7)vIVv;c`6tWiJ6kA65d;|x^U
zt^K<8iV^ZBKZF9(kZHCe43HO_P$IEiNLa+7#}dSSJQ!~{Bv0pkh?z-c;FlagkqgO@
z@hFQ$PZbF@r6T141?6C~KR;cF3EM24F8ipj%p1?}Dz+-V()Zbk9^jR?M}X8Vim6uk
zaehPANDEKej=W;2A?vxWP+WY{&!8~_P7%5@*V>Nij+=bnxvIyjU#~g^rr4r&JPCU{
zyrlaUSL^!u7c!j_NODe^Weh>QJHSWV4M~=5TMZH?V2@=G8{i1s9@U@>2j6JP=3YwB
zmDZzhi6Ac8tALePc&0vn4&om|SSG%b{zbkr1T=p@v^bN0sF}=JC*i=p`{o0L{U<p3
zlkXhHIjs(5Gd=|5H?XmOy<yNJ`#zI>aG8BKeqlMmcUYlJcdlb;Vpwsbs?+%$mi5zL
zs2_h{4a66%f_v>@v`UST$>i-L8u;jHs|x=RuK&T7l=dyc9egJ50gbjDQn2UEje4kf
zJ|MP}>_M|DwZff#$P`#?SYVOAO>5o&YXk&8rW!4w4B7B}gU{tg{jT)^B#PVKFkV(;
zQN|?#W3Qh1r7e-k=moQ3tD#U+rz%!wCI9#jEE@oyP%ppXiOpupN$4OyvTM2ZgSA9^
z&f*LzVtOR+3GkA81WL|GFs*beb2yi$q;Q7$GLY$!lV1Z-C16khAvyO02y1%*uF(bb
zYqXHvonkn<SluI1ezm!d5J)8S2yFNhnyzz`x4`QRGOxtT<M;{Yo181CuxF)rv;V@d
zm_R1pc<KrmNQYlmI`I*Lxs|RC{Rhl5x&^TdsJ*{l>yDKpqc*&LhZA*fW1K~IkLz`)
z(^UE+5J0v)Ch!5r|8R}u5qc;HU1Dlt%XjQ|<qE?N*(9txIUkTe(f!)wF&%Le7#0K8
zlu+&u#_y;F^_?tA6)FaeqjAZ`BDOt5Yu!T-2_Hoxea1?SRGLyPx7dgt-B9Xqt^(sN
z-Ul;TOZ+DUtlvNCqF4aey#{^C*7$FpwIVlS6^t`1*Z^ZN`RrXjiI4J2aZ;T?zmVm@
z^6n2T_kU_o%1?t6kSAD^j0IO*l>U;gAcE2mhnp-exgjnjj%`OTMfH;N=lv3iM3o3v
zg|t|<EWivIC}Emz*J>M(w)e|$TX!8`-EvO_p4}!Mg_;%B%>?x5&pnHLXonnGLuI4Y
zUZc^*UtRz>y0a3JG4Zbn_x2=xh^hR=4*-|Yv_<F2Zw6?Z@})Ey_@Er@0bNDLZ<8C7
zZh~2?Cl7`lDvoZ%q-_F*K;;6ctsXaOsQ!hP{kLjHm59#-%-DFWseFn%iLuS24CUfm
z5j$jX0w$5yxc=@mYhA`d6KpIs$?~G{)mxKrzVYUJT$t00&i%3Z&8x_%?dnO)N}N%z
zyRWEfJDSO8&cWMS>1fAsppbk4Wxq8klVGOsdCA!_J(lTY<0Lau{+{ec0cOqF$acA_
zm&aPZ!x{Z^>1MHpal+czlHaon3mBqdZ#Js8a?6T=!<7H_QoLMezd%0Q>d3oK(!UPi
zzYjRT<EiTK10rIC7DdXUjDX$Ts|@)tvraNJZZja7&B7^pOHC@a)(9;!(nEpgT@Bz3
zu8|hIIfx#OOPKT)YZMrXhOBU3@GLdDgNoAxlknO^v)Dr8z>7If&h;j4Ifv`6bQ(e`
zCEE)E0-faNsQLw{iA?(A7s|v8HeyU)o;?AWnM@!l*JkQsLw7dQl-_vdZ(k`mQ1{RN
zSR>%0!vwq@aX;;CElGLz<|rJ-yZ$j${-3_UwKV`=6CvUEfy84=-2!m~Cl|-k8aeb_
zC;Ga2@EYzY`E?km7|%?xE%%P4_k;$_4)n@TP2^-{(5S#pcTY{Fw+$y3r)4srh8_hl
z#}?(2sqn>4ch<Yl;%T0YKq|gvS}ElfXrj7NoL!3C^E1K@pcK8glXSeWdre@4Oy~mU
zJP|s)rrp!iSDh?i`VxPJgRxV=4Qk>cLJK68wq8tt{I;Iwoqfta`i0@I`1{{}XWfPR
zPL9upaW9?)6!NqeOQ=$w8Y2qXU3%9UWs62*bXapkF{}XmDq__ILXJKS+k+E}m&a8`
zFR1trbqD>RNj7TVmwi*NW)LO=Q$Wk#qs1#9qBT;{ZeuWJZ_|$!16r9Kk%++Y91Q$i
zm1}U$V#guI+i{)4e}E`B3!k&W?AeKZYBA>62gl>-;JIXUCH~Z@*;2tpz;k_h&kRip
zCE=q5crWx)_`^ZV(!ZF1M{W7H=lSdZJq2y4$GKxH1)Qn^r>a9W+1K!vc@oG{jGZ?P
z3{=~;7bcj3n6mB$g&eYIs5ob4lePJs++O05<I6a=<&kmzbUBRy;{bQ6j=#Q>S=)h?
z(k7h-y9QiM3CoPdBB!3OO*9VI=G_%2TQ%!HtjG^SZ<)UvSBhI_u0eoF&Is@kcbd9T
zO)xuP80WF_&j0t0LQCLzt<;Y(g3|Ij(EBpX8-YtC2RJN3R3gIse`mb|o)r$nF5#B#
z?e$c9=9ka;Iuye+w^1g3gu{0<PFSE)@G!)k0+7e9>B<#^iLQbFw4wycYDAp!_E=<o
z$n0~};bg6!U3K)<KsI68w{W#I7H5C=Ep)x<N?PAtW!W&(-u#5FNO$u|!WiX!Lfya#
zS^rgIa_UGY!k8N&Hc!@AsW`_@_aNbVTj%Xts^>7}%070rzbK5Vw|Pg??O5XhAjt+i
zbQVBSM*>hXA4sz}%j5IuFFN19E)(?IftdvG+MmY@@ui<S^Yz%h>;NaXfs6ANzTb&d
z8_eJ+nP5JK9c#{}NazE*z_)L9=ZdSBevjT))oj?fg2}t&g%?WOGFGMitS61|%)AM*
zO*zC&3axqh)XAKf4YFbDTAZb9Xy&kmE~ZM@&PH~U`sKK3+DgK%nmPGsu6%KhvD2pj
z6|hGjK$GVD6|Z%EZF=XOKd-D=5=6t7SvcUelOmP2bu$4Va)#@@*Z*+n{W{jAWxzVs
z-hO_T3xacK=NFaRz-3b+6g5?4lz)ifUQBFNJzg8|!;j;$^G@B1N*Px5(ieAHvlLZ!
zXLgbqjHEOd1;@3G)&+fHQzLG%WGk^`Et3~cQF@;}$HsE^>zTU+k+XLT4NoVqg{Ez4
zC|3*QhIymYEMe%+{y~2*8$tBgkH`4{9tZW;$HB#Ill35at9h;CUng~nOwfm>bTv5W
z85nZtdWPO8u&~V{JQ&3@!KPBo?vyPRb0#90d_*qHhOY^q9zq2uR)`>HQ!(ScHY^!E
zS~t72@9K-P_ZH7eHcdw6D>C{sL`!PK8ZLp14Wv!xJ@$bx<EFKVfI~Das2MH3!t*s|
z%S^dj%X0M-%V7QDA2Fs6UI013t+tzXO~1df3EE9hRH+*_?tg^Qe*SR?f^p*Fq<+v0
zy=yp^-n1DlC-#LYi@Q^l)AESnAK)BB_0k#Zai%8-FrbW&(S{!_zN?<p7OKWqNWHXS
zV&}x_r9VA`$g+vG;1FXFTrWCbmO37;*u%8#?)yOG$)e!>NO`A6(h<tDcgwvo1~Y}q
zJtFp1uKpZji?m7wrYo!2Oy`fucea05Jz1%Mye%VSB=8^C3mg{|IAJ>IXUq~%erY1<
znZ{4XtSNRl!8qDgXyVDK3vABomd~*031#UMG%=$(96_mY3c>#V@T5q^kT{#TE~I-1
zs1DNi&!?m^+Yvic$4#vYrioP33OIM*Atd50m7NHpl9_<?wsZBRsB>m)vk$FAVy!im
zw;*LLlb3B=_YdA5=<+sn(9oyT8g1JBR_QO-Og{;b=A`iWtN#ax=KAB_@2=qQ_WU?5
z)YLY@=SY}^yT5GDlGR;s#)?kDossw1DTb6*HEVqRpwmMdO2MZOD~`GHk@7)sYP#%^
zY8WQnWgTyT<&m`RVvOH2hAzB|#Fov4ekW_M+_l0sGilf^r5PO2_$7*{*zND1a9nm!
z>ItY0H(QE6e+dsc#t@(>{Qu{_9bC0PIAZ*2ME=tFbru-J1soJ-HI8$+fdEzI{dH+7
z7R3?`=PlR?L)@jNVizF$T<)sbnsmWt0FZdgao<Mf&52fU7!ixT9CEcP-!6_#Y*#O^
z5<R-xXNApF9nb%N_<HYfxZ7@fTnmvX5u!6hltk}+h7h6?Nr)N*(L00DB6^vK8eK$>
z7KA9H_ufnN(FHT=7{mPLdCvDe=XuZdd(U6ET-VI_xbMB!-g~XJqh)Y+B2N9@*UWVD
zz+<7x8TPr;$=+wI>SuN%BjY$chFfAvq&JK^Mbnt2lZF2|Md^Q?B5}R#|LYVv|J5i^
z&UAR%jFY*!fTp>-Y-S3T2^DgP7)$dXC~&YPJo|3LwL~CI74clMN@%7(d5@x4!<Mc5
zspo@T06oVw;jfo}&BM?_$KnfO4b_AUy#-przaR_rcauS{YsgnSd7|jV>UQ<bAg6?x
z8g1f~*C?JZst+l8*p{DPiPH&By|OQ_m2S6c!!?IR(75?{%&Y$cC;lsB{!cVj(4a=)
zrWa3flHU%T!U!iu-oS~G>AsSaWd$oUOz`QJ=Fn^U-BOPYJSCW*e_ol(9Gz$`L%9B`
zsB3~+2`AI*rOI!9Qpeiu0JIzw(ZVtI1bA39vYu7nqm9$+-7tt%=a+GsjV->6#<4Et
z1&4H7x4xRM&~z8Vi+Lr?y`cfLV1>T_qs;mjZ~ea`P`UEoBgfQgHnZCgAASCNYbh*g
z>}jB-hUt5%hHt{UWWb)Uk8>$OUS~yKmPo54+=k(rk&T#jY6IU_pI_S>iNPmtv3=Lg
z_$=jQMnHz)6^P5%al*GTwacCxT4=LkcH5()-x_Z)<WF1Q*?vb!s@oT8SKeY7P@b?2
z0@*C^X`B9oi2lExsKVfVLk?Nr2J3A{XiW<f@y}i-U!DK{?0<bTW^ubYXpje|PxN0r
z{hI{yk*RSTN*tUh_6?Bx4SLrJ4^6;eM1cM^c3X(n0<RH&{e2nqMfft?8E<?b`Vn4&
zfMRuRo%WUu^U0q2ReqBkhh9OCA(sIKe<b9%229I^*Y$SK_Pap$1h@X-%d|X5QrG81
zsfCiY{G93mwOIETBCluFt6?L3Cpk?%3w(jZ|9Q?h_Tw*?8^&c?uo$_j?Z1_aAu|tg
z6^BZXIBo*j3|5N_aU}={5yPJ<ApI-t#-eu<Z4?`c+zY%s38SN?I}G=Z6ZQRWL1Nqc
zM!O+6+l5mH)4ifcB-e2goGyP9Xg_z1S@z{O(w6<C?Qf{9AivG(q<KISy~P=EOkKu|
zI)U$Szs@bu5PHgz?-ZgM-eLZ~LCwGDFGevqQxh$vWddm7+792pKY63S2r*R+;W+6U
zKv0=#2VbZbKT=HS@M@oi1ij0Va-SbRwTrzvy>3$ORKp?ITsrcV>2<W$wn9uBCLMY<
z#fi`*^ZlEXVR(-zL09Q1sgw5ILO>@}C#I`%&XB+Dh1~uNIX9UwCHPJvK<HFd<>X!}
z`b(-Xy;~RWYY(+mVH`>b?*C7xeQe_Ir@j84pLUnHX7ubIu$xPcKlVMo(BY-VUAAfx
zr|x?>j5tB1j{*o7n~9>b*L^~QyP+>aB75$IQsH*}0fg=K-Kn7FMz`aj6~~&SgE0au
z1^vmqrbGdJx_EuLftO;eAmI8d=(M8;>YnuuQ#bu^FTEzPr!b(F^Ik&E3;0X#C>Ty4
zy$$Cu0#C*NLw54N{tZSce?9%q-~azl-<<$T>c--cT5)R_Zvjue7H@(s7e7~9EoU#O
zlo~n?)mA79T<!cOuA2#|omLg-`AmodW-n}=()G`Fp1h~89T6kBwq^+B749K5YUJ5V
zlPG?+)~jmF6pKr-I|gK&d$h_7B0U?=7zeVp_{52;7|Q|=jIACw^*S6-ZOhA|uje&g
z{$nr?N#T4OqB@EA*_Ll7CG+n_=C!nc0W^^c3mlcIE9MFlDKbWrOPH&y8IrB=kb+b&
z@>OgPkH7=Fd#c8`4XF>Po1%%BB1zTl9wdc+W*$ha5N~v_dyGhSdQ{#uajS%aOkjcU
zb7OFieLJNxDQ~S>w`e%bbF%sit~30N^g2yptT5hAd()sqe1fZ<2#`(*;s$@0KgQVh
zYG+u7sJ^rKPk+OE3mnZ@jNBFHIS`&-^FJ~EuLgm`U$ndbnK=I%!*}&CU%<zy%XSF9
zX^qiej+3yVKba|@iR|w8g+OB#?aN{-K1i@<aR77zd*HqFmC_8e?>Q7XQ*zQI?pg`+
zX<ws+_>BVj(|<x@jfKq|KY}raFwKDbj`ws}sM|c_#VWW;xw73%=G2i!J{z}2S40n#
zKwmjI)&>9VQvM^p^Q+@TErsp4`#tIUKlTBC_Xghc|CMJAO*+4U)mB3&@W?~7ofK@@
z{^Ik8+bA^RtJ~eDFN;y=u{h3SdH#$QB@WEuh?_Lr8Y>Xb(|_A6Mp~WDU==xy5nmR8
z;VR9^4*D`RWc5A%_!k-7mEHRGW=<-XG^!;UmywEiA3ldOAB%##1md_;exfbwTt#jd
z`FyiVE;BwoiA)g=RKDBrA0sBeR&M!ixfJM8MfqgC25try_J>>&#%Fe{2HtPEv^XSa
zVVRro9LO0ZfF8e*p7=*akSjy#X>1o(wHxegaXsA>uFcCwN6UZnDAqr~8yqmuytk<9
zJpDIf)r_3Qfb$--q4J5eg0+u6>8z7^64Lt-o#J)r&jLjEilCVr>m8NMIJyjVsQJm`
z_!pNY4it7&U(4P%Z}pg03+ol6+~KJurR4<q*1d5`lThFhc1jX9Ay80b5+*K0M&rV=
zTLBSMmldN<mT;{&jw*gnqv8;`2Rd%X#ejLf@&6RS-V*R@<l^QB4(Gef5`Z~*$42k^
zvHY0jj;kcaDey<hq7h0#?R09hbTL6b!^KBOdVw;E)YG^O^YD|l)jz25V_Ua9p_P+R
z^aSJDzU+6}*9u><YWgSXK9ZPcGTfk1r<%E0`xWUn+N;RdCa6WQnqV8?;o|moAc%lm
z{o3sVD;~$|2-W9!Z2;-6$0^Kq0_%MRzI~jgAGY$Rp5>fUZj4K*3-lB@C{^Z4KVUN5
zWD0#EOk4U)6t}*U&4|n?Hjp-+^pj<BjBr==V<$6UTtN=1wp?D<{yW?FYY=cVW||MW
zm0yk<B)r}yWpxO$DQE-j7%Xqu{{Bx7M%RhRNMp5~sgon(oZ|QQ+5BKf3k=aJi3e@O
z7o?G0_omX(A@4^sQ`xZ16xeT-m0_snU47d<CC{+Dax<JY*>vkIvws2&1<`w`<CK67
zQvXxRoZbFV!KZ2RM{vZ=pJ(%DYR#QM&n(tFkD_$CKPPnjq}$69?6JP*p*DtUCq=lG
zJ{;?g)bKOCX3tiDJbur0pfb$_-E&>VD7~)?IKg&)h1`_B7NKwCB&U{=r#1XV76DMx
zs!vUxMJc<soL*hiq0}xJ`HFDxcKL%2LoQ8?4;xk<Q<Kj~vn8@ebh5}Ni!uikON0q1
zZo3awiZqItFU-hczTGWd-+@0}9S_>L99@!hdj0z<KdF1hw#oUYdc!KyK9RDKq@0vE
z1`YG+{0h}SJ&W_s`BxG8&%gVxKxWJ~#z;WP+W7-}UKU<wVER}|$YS9GV$PwahLXz7
zQq{gI(0hs1V!vJf2Q?6k-E~1GhK%Ys8Uv(oDxK?!U;o_kqkBMf?+x}ex7}h}>vjLn
zV|Pp&7JKdF4z`g?kI*SM-rnpyXF9)=|I=99u|BZ!JG>`Ii@ddAJFDsucKv43!s$l}
zuIt);OFNnl`d*QjO)>27%Hsp^#g&~E=0ScVdM9;6p`d3h1-A`oqV?lTgXV><LNv)j
z*IPR>VBE9LT&caGC7wy5I7XUK@%S*|YT9o<X*JEz(}nTXx)L$FefKxn>5vQbPF>85
zN&w)n8tX87<aqLzo>HxBPiC*pO)7pumCTEwoeNvZ^!(#`Bj25ovIw}hd5sWg^tH^@
z3@LCpU!m(?6#D-)0eXK$WMs73Ze<wH3iIQdF=Sprv}~79b|kvcwyK##8>NE3_AdVp
z<RaT$5#NXH_cN?TNiwn;JTdS<;jS!2>Bv8o(PKtLuwPayh|bFI54MJ|Qkaz$?ub^9
zv^?aiEP|6ZjBP^}DJ-CR_*2E3V5J<JMJv!k9jAO|_z<y51jR{SUI~!n7wmmf@L(Ip
zE0WG7T)ofPt!%KxA;FR_^^stLU)PpGj5w*w)s7W1B^;-3dNMg96PYS3;54}MR_%wn
z=7+odnoL}&w8g_!LQY1UZbo99bQv~$$ooOXlsOnXH928=FHI-)@-PTvuaS7XjKlgK
zb0t29AlTwhEhSx-z1J6<Yvqdhz1O?6(7``5k0H!h9_#_3euoz}^FFQ5>ZF`DdT+sE
z)m}VD;j;Z``8KoQ!uX%L1c&gr**)5x)_OL~sY{~F!ujumz)|_Px+qPkHu6`mN9T0@
zie3Gc<9n-OU7wgDdW%+=i#mUuy%PMlF_vBS;_mN?S<&A!I#?O$*qrd(y4^orEi)7x
z>(#@>A9Na#!@5zcKFNn!>m~Z3Kb|2=+Ezefb?-xNT1-E7lK-PE5JI7nTX2i@syk6u
z;!TtBc_3-!Ft?8R9M3vN{yZ$s&l+CIdhL8Jm8!z|zVIzh2G#C;SPOwPkKCCbU+R{h
zURoD}i~Cj50HY;kfSH3=C;YYstkq`TH5&&EcRR!CbsT?1l|c$&yp{ApIuPa{TZDH@
z`2y+p=2IXMmiRBO6-xvj!P+uXez_iGXz0cL^yR*f-L-gQI8!{+XL-L(?jS$j*uX-J
zY6ZgYdfkP5<RxhYW<9yBA24mnNyNH{Sbg@fiv~A?oFRW}M={5BjY6skvT5N11;lP`
zeEY_4mFX@LPOlD`8mIe|gQFk(iqtaC{hU*sbSj*B+-rZ?v#PEY^R}5bU&cv7slUMY
zr9msLprR^JfMU2OQRDLO8DyF7nC^FFj#l>xn{tt1MNnYeCY+d1CHvyfj$MuZ4CBmC
z4cyj_@cX6ZlPcu5%@@I&j^mXr#vbXtWk+II(3}PZGc)nU{)_vx9};(w47x`><|xQl
z__D1{QmljhHfVG~4)K)wYW>Zq&A`vX?0o+1-<G;i*K1*>cN!2x_m{Qa)}R`G!C40f
zq7ba?yQPlSlTMr<QcluBn?t2OXN;G^To3|el?*qf#qolWYZnWnTTkZt0owHUB}@{F
z;=>S~JoZ<!gMinc3Qj+1%^b3tD?1zRG^66x)<&V)ytRYnWRbnVmB4kbH>jyEcRN|>
zT6D!~^RdeUAJ)SE5l<B&IL1B`P%iV_%+S3aq+hw2JN+DF;2rHVW)NNE#TNyj%&h-Z
zKOP+?xj-@JR)_L=K8xs?x?JmtSm;@KH_VQBzH)({*{rO+WSaW@{kJGHGl-~459^Gd
zC*Pyx1;p#(24Y{fuZ-S)GJ;xYtJ=k*P@+KArp|}7k{eH;?oC4B5tQLw0lSySL!;b=
zMM1SZizlbCh?u68iXB4IxcOe{RLA77lgCEO9(=RlqokaI9>x{fz{ApERA7MZ9OK^r
z>6L#u{p-~To`uC*021JLlmevn7>&YzW_)&UF6e5y<Iwnwbie25F%-YeX7ozR6|mlT
zwL%VGkMJ!WR-rhcj-PWnbDUqS=ee*w{&P*DlJIW$s53wC%qWd2h{U3^euo9d0unbb
zkw>?_nGax%)4^gn%mX%Y(IV`~?7}F3J?>D}wJ31jziPjHnezr@-TjBKM}PgwPf#bD
z$vNoHHF&fL4LdTfnB-_mC5{aOVig-~hMMBGCTL-jab}*k2xZEvOyq7^o+0W+%Tof{
zAiI^veALi=f8cUP!|`&x1}x?KTN%oJs@1a*t(D(AP1u3`Oi^dnoS63wzJZuF&enai
zG%0Ro+9U31pJUa|c_V>`RA4e_O+1v&-t!|Bx>MQ9?ColqZ=T-OJBnJu-EJwIy;cG9
z=_D5=ZU9Jy^DhV-Rs(n?Pl?m!@R@VRW8!dZiYnX~rUYq3!(?E3`RYEeZnXZFt#1OY
z>h5>GTJGkLgi$5ZdGi?1-fR6th!`mN++^VpCx^YfDjc?7<bJT;@VXwS4p;!J49wJA
z;Q&e$^7PKsWh8H)2R6wypgH|S42m@o61%bJt}n2FKO(;=@_h|S()0kHl;p4QG+)&Y
z6AXZ1M@`w+j%mj0!)f8I#s}P%!^>GU@ZvctsEB=;C0_i3k>E|jN7IW@pH5jl`sDgU
zJ<h99QAx_?20uaAd+c%8=(duKGOh+)ck%GJBL`$!VQ$`%-KXQibj!O$FtPFh1M<h-
z=yVf6{Z<?P>1CYTN&WtD2#L#3kPB2Cu2z-RaYDTv-YSO<R6Eb0*r8n+G8dPf57oES
zz%2`6r;|iGUyoj`6@Y7s>dK{wSa+|?WP6#=PM6gC=JAr|qw_(!-W|%OD=|$u=~N@l
z))3RBI?Q4*_L*GWQ^2=}tHV5d^VlD$_T3T@p*izqu$DlDhKWx|V1I}~^NWW01ziZ|
zb@Cm#pNxwfCjKvnBdQri1tl$iN^h3*klCFXIum6#Uo4#6F#mYIoG*(hHIT(zrIg>0
z^IG9=2L|Y_UTB#1jP6R;Yu)yFM)gK~{sOiLV#ag_&VR--bwB7e$NIe)3|jre|8x|L
zy~T7kNhdmf-%EQ6=$Ojbe*2(j0lWN#4(59~7-q4H4Z1Vp_v9}yDVu$xDNnWOgJeBD
zf|J0AD_)~RTLw;>NQyr49+gz+7gFUQQFqANA3n_LF>(lWVQiXFhhJh3A0V+<-kz&_
zXAIbzAnC2FKW6U&+zuV&@PT8aB!RXgZ<x<vr`^$Jq36a09;iX6Bt^6HHc*6w_+=>P
zUo49j>h*2_BeJMG_nE?T#SDP%UfM@i`?4uY;cI&M4!q|l#C|ejtw<ms1aq381Mkc8
zja2inCqtUlzA6=L9FQb7us8ZhtI*FirA-$P^)~&BiJ;5(t3m1u;pUwVa-2cji|X-L
zcm=*RMf8_;+k(f))b<6A(ID@k+Bb2u16_IL4h2N4-``JX^M+5Jv)uDuDBi-IW$Pl}
zSjuLF_Y@9)utSD~r87$KyC|P^xA0*LmN%=&VZp9#w}W4=!_QJw3ons;30)LQuPIUi
zKb?C%PQHV=yCM7WhmoR&Y9p;VKkVczYds`RKd!*2SbYmlACt)qSYk(KAq%DX*=80Y
zsz@C}+-XkR-A1lZLW6yA_2k+oUi0$jV()RkyBOmC2OPkseok7O)E3$*4P|`oak6V*
zRrHR#qHtfgg*k+z+I#_$Tm}DRwl<IS=wN{xVSCI}aY8u#>!g60nR+2a{{krznWeEV
zTONw`ek^mg9ztgmP7Q}kGKm}?G%eZJIZxFmwOL&jAEHj65)>HuFALwn_H87{mS@^L
zyVG_0<7kxYyxVLmZMNS@zUTAAY2F85SND+|SgT(Ctez7ArFPm@bz=##z2?G}|IU(u
z5%Bb|+VVLFe<X_4yk(;sq1fze?T|qf=#5ydI^hzBs@a4=Mx}m51qEGSeSdj+y;1!v
zb5EClW{LoM6mKZX40G7Ty=bQtdO@d~IIiVLgctb3e%2$~*aov`-gw;=r&OQpc_r`;
ztK)mr88)xA4n(dX-4~sV?J0XNs#*angB7$6<RPm10WL<-!rZ`2I>3XRz?@4V;i5Ca
zcmUr~ll!(&2Nc^Z!+XQry!?mTu8t=3rx4}WCHbF+)G234f!D2<Gg!^LsW)CPjOUC(
z9eCIgtHhE|?;IXjB{ja!=hw}UI>ko`?KO|JjS5Qa*|n~pW@ve=_7;6crmfo>iJK>f
z(s~^yMi}lH$)0S_(wT0)DTx|n_!2+4Rja`N`MzE_TQw=9>%S=`^O+KraW5<xD`f(E
z-ZOKq-|MNN7}jC7@~UQqh;xHY5FKfD0D;Kiy&sK4tP8hFjp7A1#3%zV_rxf*&)=(S
zn7`iu^#?cRfZj@FJHC@MRqm}92B|_0Y0X{5dTZ!paclBcw#ms&Q&|%wG0oc<XZm-w
zE<L6)e=TIRh0@JmWW(Ns#|EjIOXb@o%@1G7zu#NLDcq@@a5g4L$K*ObL~(3}>Oc9!
zo@504oQS!=&}@sYvE{SuA$+9!E!Iq;<7@d@GEulzLC(rFI%oBu82697gh;^v%2CL^
zt<!4OMO|zsDU<clxHUd<&@8oCcblXVoZ0tEwF2-u20HVJU2H5aI|a9CoBpGG{Y`Aj
zz2)p`woG(HeYAFrBJSl;k0C(BBY64f+O~XZ!1^m#;S=0(6tf9*zUPro?+;qmj49HQ
zv;miOyVqbVxo73llb^YAeA4NGLZ5OlfMVnhw|R4}Pp{Cp3bUhnwx!};s#9d!7>XXT
zp8_P#w7$N{D_M(1@jQCNMI>dH*x`Q@>FZtk_2O}c(JH$@!N94!|Bcp(OA~w95llnL
zDzzYo((iyCBou{g%29RwR&;qh7rDaPWU=Np-o<h`hz+ytk%U&oW3=qFaBB)zs2v9C
zeyRu@?#9Jmu%4!W$icrk@K9Gpf5S85H95&5f11OZ1AH4Y$$nZJGGplPpFwTGb74@~
zc#(zpO`@JtY%r32vFttV>L*8&JAr$#FZm@_xGs3A<zYW^osR6{oO2{snNas+{X)N7
z!PA;H$X)aR-wkKrRXPU7S!zB?xy!-Lg>!Lx$*SHXqJk!gvo(>SZC${5G<wZU@o?yI
z_Ppc!Hf6b^Z$?{$Yk^NO;Jii|x^)LCP3%Qq2kG5@2xObslDz#(_|m~B2W`izVh!Da
z<J?vcc8MsRGCza5bZr9^i+sEE<wjuQ*;`aPXYZdl4Bd@uUcW#VtIKog6-Z2{%iuO`
z+hUjQLA5&<j@2G$odpO;>fq$FRY#|_(9j^H`gjJ@rhCbj6t1#q#ff@pPF(+)*V~9n
z;}me%o?_0TPUu_EtkB`v{MupB9`NfglMs?j9rT_NQXe)Lfu*~}3Q);Ybud!J1uRQ~
zBdmqTRuHpy6m)+$2zHm9=;1x*Aq&X8KYQZ^&!n*}u7h87w%j|nkSNiUM6d^*18E3}
zWW4ibd(SOB(yz&(>u`hPs{vY?R@WpssJo&<uskE6-qqv_)VP>IZ+Ui$7CZ&X`N<<k
zarF&6F>sCSO17^81X!0N;!r4y*u<AWY+}*Ai3HSj-sLKN(vB&*Tr^5HJqr(NyAT*^
zHZSM3L~kMqbVjEDAbv7h=)TW3=BpywxGuI`FjmGVrel{y^4h^3bqNAKIQz!YlUe<y
z`IouJ@A%%6%z*umwN1k_{D@+#acJ>fH@Y6Ihjeo2q_jE%b`-4@yW**E@tsMykCje7
zN8Skb60)a66Y2JlZoNvqyyW<$Opr%>)6vjy&E#j8|B{2nqC*XDw)zFRZ`Me}VOjfs
z5w~b6rIDZ6LILywW@ma|@3x(We#9}ilI&XdlDep4dV*zAd-6DPDOm_q<eV^!Wq#@U
zaPi91+ir#yEVvrK4A*8#{xAO4ZJJ+(MAs$+ZC~qLozf+|a0Z&Mdq*Wzem(A2GHWYT
z$(i|(BcnobS+d`On;NX;SHLQ&f$zI~N8<y7y!UVL8E41eJM0ia#n|nA%-)?iO1<=+
z?FFr)lqw!;pS+K43l&EW@no}b{^kV=z_U<uu`gygfctTDMj4cUvRe$_9x&fJujekD
z^<Ol`!NX?n<#(G^HaQ~}dKeD)60XiVVohU|dip&{xG4T5d}p8)XofD<qw$n8r)=8Z
zWx(0D6|At$7YEfNmgswn=9+rp!-d`Xq&XQv+<&!6Y>Uz|8K(CG{Jihnr46{R4ci@l
zp<zFk3Z$fZ1{i>$W9fECjur*KGeF;4YYF)Z;I<i#4T=t;0mL0qq27BL@~=N0O=n(e
ztY5e*qTsD7cd>rI#0+LSk!Bdak$6@6Y_}6s)uxQR$mWeL$;HqzO{4dprnMQYHm{Kk
z_c*gWWLQY7C(x$kiD!>$OlJ)>@J=i9wR68@0*nD$yy$(MJ#Q5gu6#q!-Gfj``Cb8s
zgS*B<WX}fAd!l%7yV`o0O?^jyd)+iYv0X_vAS-@YAMIeu%^KRscGKJ<?PYd;t#Xk5
z#B|O3Mv~ZC$(KlFTN~qdXDzvYn>J~&An+#scffGlsMu>Q!Q7f#+pw3%9)=ZdR+Ub`
z#B26OGmKS@kDQFVI))+0ZDR4yc1?xqD&op6r=T}#nF^YWx!|qo=mXyw7s#l%4;HR{
z1bdMs7NB^3bnW7WF?(L$R|JRMT*C6qF~1CjVF~Bn5V2`cP7VW#ko4K9v^gLx?BwLt
zY=#4~zwc9wX<v->dAnO}<+HryO>A1`wVXtK8yvNC72#!Ybf30oMTdL(<SJ@1onkJ}
z#ZVsj+a(N@KV;Smyii0<;ue^<UO9uRw-V*_;TIOr7_;ZXVJ}+iV+c|tz==n53D-42
z#=Jgdmo1{wfzkVoL9pMBs)!V$h=6|^hm0i=8L6+jK`_S0qR5sA;n5L3znZ`ce*a#{
z6NlwYTk~d%QPEjns~|m$tdWGv_-7+-FUXZfS+5mZHUfn4H#+x;wlvOnjKP)n#_RH2
zf7WHWthpE57bT8l$qr}5{j+7bnK1$|Xy=u3a$5YdjblI&J4QHry}CWUnT4?<0O?Yv
zzivlg;JDarZ2go}n|xDGAd*wcZSDKD{Y{_~X^1NNh8>zQzxmy^6N5T2$T39?d;QS=
z7-cbZUX)Br&qlE3VsI0_+3CjHy*+UP8#P8&Q$wD9xt_Kj=0~hn56Z-STL3nDO(*P!
z9n4gkld)A~9uMu-W`R%GmI+p=5x3>vU0zUJKDu;6224Ax``ebC?PXr`Y5D~Po-c1}
zByQUqW?6F{Q5N)rv4?3W*hWgI!*c4zcFctv3bEdTyj~4w71rPNUo;{-QbqdvorK8$
zxG-r@>%Dr4_1o=qJrWhAnc3K6L3+m5f%HA7Q&!A_0&7v%Mp~$xyFd9Ndr>VNN2&Hg
zxMe3nsi#;xzx^hkPX7hD0YZ4S;mYMS_J^M_)-ptOX@xyd66AR}_Mzbzhwj62Zk%2$
z5n57SI33D5#C&M`JZTn4b(AOe4vn?DnW_%&*}Telq62g>eh*^2Rgb~Cyb1GIjPb`6
zU_bcT3ASJ+#`U-vBVUD=WwTVBg5+_*-PN#AWk7ZME+r2#2>taTj4$>eosL;thQLgh
zfPMPmYW}d)9;1%wiQ0;S00XdYS_c4U=uJjPX`S@iTo#5Ufz~IVYiUyO$CKY>yI!bO
zN7sSd^&Sg#t#=lCB*gHVRvznHp0eXnDV!BqeQVCs@pvxQQI@EPYl;GaUkhs6*v{+U
zIN+kAmh0ptx8CrmmT#<iX7LkW;Re1L!w`}jng-B~$nJeCxC1Nz7~LNHU6E0KU*KkD
zwF66ke7(51BFoDCMtv!DnFrG|YEIFL9zo|BG$$AhWC!l&BMuzwTTu#b@jS#zd8n3^
zW;1*6`T9-dlD%{7$p`c)D6aBB?|7(roMgHNLr^mY5iZ$bt%o9DdKv#a^Y~@S7;%8r
z$Xv7QBibB$uyiz$Izv#Odo%T&{l-7@;BO0hNYCV#vAKpxza&{(Eq3T!)@Q758&do_
zPX25^+$*pkvWzhmpr+5lrbdNH-cE`E(W3GddotKDR0a78n+Bvc<|`M@4H9N0*8;2$
zxX<42>Fnpo|4I{gbdJxtx5nje+N1aJwzxWj3k5=z=Ze>QF%5J+TloMcxp=#Ee@aUu
zj2426Z4zc>#;9wg(vLI-z0CnQxO6yWbGwQ?gyd<31FS`nYo|2kZV=Ili=fMH(R0+B
zMR55cvXtufrq$(j|EMpT>!an|iWBlpJD~yZbxoIEEceRk>xRHxz0?9_$_+Foyrp9^
zqTJwkJ16IQMP7<CX5Mu;UHk^&<1%&M5UIk9jptIe81N0s);(S+m@zG#I=Vy$g06*c
zwCCCGEUPns^0}8#uh$GmU-)YUxdaKrsymBDj#FTEUykzQJ~KS4`S(Hp{{vCK3(PT0
z)&+k~<4-#&%`Y8us=S)r=0CZ0Vn54r(}l78GH$JU-HZ?J3_S{~UoN}q0gva}*w^A*
zaLfZSyDMRkzKsqSiYE{e8E6P49a7TR>Odpof<m^Nk{98_b5!G&U!HWm)x<>G#dWq+
zyB3haZACFtrpjAii<{pNNdp*H315&iNRA%?ohCb~k8xh`FU}RAo{P3{HGl#0h`+1&
zkw#%asmEltYB=9xH<L)`M@UKDpeBg|q}guhwxgaQi2D;;MG38}=$+tmBlOOLK)OE{
zL4vQEPfuGqIo9(56;VeAdo3G~3G~ep-{6%}y6r3~BqxUQ`<1-?vh3!!<VN71pL|Dd
zHMpvnW3H7|$Gmh949%hb;X5iU4rJ@{pMdN{Q8H|TG*f$KUTw=CDIx>-mlIn~N7U-J
z#u`qM91$j$_88Y!)|TR%4kZ%=(1~x8E3Ic)50h>{n}<a<A4Odl%w;T?PK;LdE+juv
zHU|blZ{FZN{mG&oRJeh?d%^%7^&aIh>t>BP8ognIftEh)Vv7dR*qFkB$uU3U{MHg8
z*4sWe4|Lo*rP}lBbQXCdta0Ep=p6zib4vLBZUb020*D`jEqa35nZ(jOj+&HOz&Q(d
zO%x$f@2%e7xfj8cZK(NcN9Sa%x9E}~Lg$3n#nW1PQUXS-72_-<ME6M`NiF$bT<vBy
zJCE^KQdyxccs?u1p}@V^Ny!k|{w`tGe4RoWPJ>3`g;;h#>K1#UZ8LFv^J<uXQ9uLR
z>#{+V(l-rE#4oA9&tH@J@+y`PPp=~M8rlHzn9p1KqnYpYcI{Xjk-d;jpbOtCpJTZm
zs$aHh?)k?(N92$1`&7X&S|?FQf_+mCmo0^ocl7e}bQ6;YJobYM!!)omYV2?3U*f9C
zgkT?gOh6xHF40{GV2&v$a@L1Xd^n-f<38S1PhYE({KEtX=mMK#vOhfBw~lpltME^%
zASCbk#<={=B7*EE2Jq@nh|a~go(w!WeIYDK_|0>x5tRA(#TVpa73?@WeNt6BQjmut
zT9EI><jO2UogYgAdU|(xV~2)tvZea<o0~~ygI>6tv>J!_jqarA_J*Jr_4T0fIq7zf
zv^D1YaAu8E8{cL(+6Q>@<<AQL9Rpx>m5IsQ0ciCCH#b|R?+LhoKMC5aGxgql`WUp;
zv2L}At?^-XO?mL@rd5cHLulg>!=ooW1B`~cWVfRk2rJ$w-y+;&y!`>w9l2xKmHgJ^
z0gW)**AL$x?st6H$R!~7%xRkz5`w*4mTkSzUV#ao`7ce9x1Q|&e&w|{CH-o9x*Y60
zEp%3u5u$o6rT^~5ENUX6(%rq~(t~f|k6shk6DDvKxSv#Dy?vME%)5A!s4%Cot4Was
zFFlr3$}~b>aPO>gIewf(NSAQWi~5E;;z8ZsfO1983t1>@2HBW<hMCR7o9UH|MOw*Z
z@%rCd#H^6^u2fBiJ(0*VrbuhQt8#pYHpVLA@Bi;h$Zly!i3kFGHdEFWaZe)^@WVAa
zBG|Wc!tF;<Fg3;-JaMNRbOg>t&136y#H_VFtyYPKpyx}z2ks-sanvsK^<M-UOWcc^
zLR7P@M7Dz(EaEToj&53{IDya8$X55Y$`jmV;6c5+pS9ZDwr6|Pq8JJ%aCz?%e(oGZ
z*4dnOc~#XWE-*AL4x()>{I$8_(ohLLpu;}=oKhC|tzQVyM|>X}sc&j_78b?yyjDam
zEZeq6HOzd$t23p`c^0oOQU-jyu-0s7KGn#$e;+An<3vK66HnO}=-o4H)j-|^BoYNO
zTT#naawbIhv3#KAcr?Q$O4ihqrBe_h=uqyn*p!QD6Rgy6vpmqS=f))#Ehw)}{;NQ<
z(=Rs_TK58451$N<2+T_4sY;H{UHq9zw`rB5eUL>imoAR2Ye=`<4>LG1p({Fp$;8&Z
z`%^??vgU?j&8S@_kBv3-$RvGB2pwsi@L6fH^Zg)Bllivgq@B)-i&aGBx`bR#hIfLP
zwT#X-)2$~yFOL^)@pj-|uqrXl8h^b}HOHz&C{ZeSp1`${Fp=4{3w~VH6gi9^c$jYa
zAs@W{h_`uqjBxZR8-?`PV{(grb~9?;&AZa8U8%LJ#02-XXr7m;e&PO37A1J=ewN5{
z`sN8mjkMYqVp2QpgsZRKn;_YM5gdjD&3F@v`OVXlRK1<sBl|rM__>F>c#R$&Q4?#Q
z%wb>cCz}U!eoSPQ8V>Cr%*~AhO`JKbwy&m5J_tt3#~9LKMYP4n+Oib8dZE95V9<EX
zv%`PRqP>hlNh%CW%xe-P?<BO_@DLNWN}X&?%mw4$PKmtqjJS?(v>ZBnk}rfP)?wbI
z0Rh4vW%jj&ADp|X_nQft7d01#KnTb_W=QCk&5d9a&T=e<Da06td8}bQw-&35R!krl
zNlbT6he{qW>-8^{6T}x{w)!P`q@49x?;`Ty#R6RSr+6n?FXx>?$>Qk(e$QfO)S;@0
zV(bY4J1&*{+P`+<-K%&y!_fg{d=D)mPkLj@ZbjO5yA|%}a7X1fNkTRSH>sl5;|5E_
z&6mA8?@u3q2Ps<}4@9u){`3ajACZ)_am*G2df#L?Lu9&lg)ZktAXB?x220ev6Q&D{
z7<ci}aW_B#?4C4QN0QcD>X0$&$&&eD%`;z4vXF~j1CgwXdwO3Xqie;qvR(c^mMr!y
zBnhE%&x-^O07vIXmgCWaE@_KFj{<F%PSi0oby2nw2n9;ajA3q$XJB`z#9pM7SEwL8
z-rmyBs~r8G)|?M-Hc1~!(QY>chDoG^O{RjJk}XFY+&KmBO~tXk=LoD>p&s-<Uk$OC
zpo6v+N&zd1J90y#Xo)G$CT;eJSxrJbl{FiPnpHLTNbu`Uv<@Z(+-wD$egr?OuT=3L
zoMklOKPfd-Shk9&=0;2nsxK@$&yvf5BU(>q$c^vvWWtUM8nTHwK(zN!!yZj%^Esxk
zemV^#n*Lj)!FPDu=<v3bOwN1JJ<Odya`N`C7X{RBi9QJTiPsN7DMJ?=2)z6x)>jw|
zYO8*Bs$~vU({gJ15K?+}Mz8@Q1&!ZG;x)iF9b8r#tZP9D(#HM8y{$fSL2;HBX|ni3
zf;j<r29{OL3joc65R%zh=KCHB?K5uEcU0&*@%Hgsp_KWk+ax^ObJgslb@fBl(!UaU
z)4aNGiw(hrU-adYm*_Ep5o<GQByuHn+HFP)%9VfmFSFD^GB=Y&*>YQiT-t)0cM^}V
zNF40wy51;(c}Kdv{Ke%yQ*PbTo%T^&ae*!P=Qh7Ni+?(@YF`NfJnzh?ALz83Fpn&|
zV`#OGmzK4zRO!>>U*M$%<nyzE90<^NGy1^L-vSTti*8%tJp}LP1v%95t{!{AJ~m}_
zDPWY@b8?5g^eeL!!_|DR4{N3PZAYcVivWE+?$>5npLC?xLZ2_kndP2I?Vy)@cDrkG
zz<Y;+mRUxC#3cp?cyFe+hh1B>iAgHvI~Y5cpOB|Ni4Fp}(>~C0W*TY4R9IdZb^>T<
zr(Hzf14#<7ul@{{XzhEV&vNBt$IC!<A*$Tu55mahdg#HfrXjJMx{QQM9lpVl?ZkK#
zO?bqOa|El=z~+J+{JcL8T=6IHq`%j?tC6r^+VG{bFspeXtOhz@jZcuUD|gWo*nPf7
zpYLx(Lq&MZ1|C(Ai9J_J6Z7kkEmWW(w*ExiCTKJPS1tL%Erqf)<p=^B<MAUA#JM4<
ztQGIyNNOXb>>6`c+wm?7a9bQFC5<W!#U;k2ZLk~kn&%66@yA}bD#$HDKlPJSGZE)G
z+E5Ts?+(p1d~+k%V+uD1pTMKH9br!ydRIfJAMN=#ZYAE|+f_@GyhC~vG?ffL<alS5
zOz!1n=oQnYM-Sd;;=c3?DAF+(s^Fu4hJjS_8(q*yBu9vx5(H|*FA>U+m(Ce(`7yT(
zTIHkKtZwq&4i1}>3Hi<=6Sq^{E|c!v%Zfkg0*Aw!iDV(`7-1j1Ej3154m<CTVWJB;
z<^sl0Ox>b=DxzCRZAooHtf*xmc51b%*J^$^*xL9LRVifF-{ucqT+T@KG5r8y=PJH@
z7Y6}ZOy*wgG6^zWgjKxa6SYNtxKaeYmQA+OC4@rAmbuq&&%)PvX20Nd&L|C}?f0-(
zvz>IFh7p}n!U4e!{(=67P6Z$={3Theb1*ugXud7<!fmu6yCm?@le5hWH^)*~N-=J|
zW44}l?xeUb^SG}oGxBjUll&C-o|X!zwji^(vkCVzICwPlM5*KXMnqBk^;cy(*<beC
z84KJSmCP!%3Pt+KLYR(VbzMPT)Zg*php_%+3GcU2^E$DBUj%XO|3DpofseEb#<VAd
z(BT=QCqPHTy3Ej^U6ey_R>R}#Q-@V#Y})qWY0`ti{h%g>*vO-IO_NJ)p=af(JqG9X
z(=!q`;$)~+MsrQV?}PGID;_GPPbzjcOU-6Fg1I`wo)n&7OT-X;9?vDtbIz*I3w^hl
zy`l4$yJa+2E%r;aR7}4dcV<GM9t{%&t5d_<-KJ*@wd~^dgwSSIKQ;QdEkNLt^e6<X
zE=2&w0ywKS(<9jfaVKIf!s_Wjag-x_M~a1`4iVp<y=~TQJr6x|1jdf{mz{~XTg*lr
zFHrAq2|~*{PDoWYQYnr5LI(y(lIY3tFD0}37$s*(-Neb;8jn33yr|@gvS4!B`zkj&
zPm3;z5>l&QGXuasNJstlPOJjFr>X`ekmEY^3%9+cH`av#ounYsr7!PR+f~z-VvmbC
z&h`f*#$^rqYi0@q(z(nWIasIxsyC|N#U)Jbrju-BR#lu<g{NDG-|Y!82_oCYxXX2D
zi~qXRHQ{zm^Gep(JaeTVHrmhTbu8ZzSIzcjuXsjyJ{!EtQHhOIC1#zYKva!Fhw%t0
zI>sN?;kE84Yvfs*vk}I>{(!hQ8cre5+^uwe>oa~f2_up=z8QhnhesJnYqt7alKtL^
zB6XR<H+`;tqCjin^da*v6rNEA7P;*@a%=_zxu4egZ$xI;hLjkQZJX>YC>gE7R6r`V
z+>HMk_oytgu&K}`%myB7RV`tbpKhX@ZQN$CcGZ;-O(w!kkWpEq*)C4Iu(Qz^t>|k9
zr@Yk4Sjaj~u1uX1;IePD32(}M$FfkH_c+iu^rv%PrsN%2C*lqgcr-N--Sye}`ccAP
zH#~0C2jFYhuAEPb>IhiLn&r<=m(b0am~!*~)1ZPpKPVFvmU&}f`Dz4@c=hF-iZk3Y
z4-5&+`&vjj?bRZyXO7`}qR%8Cla(KEZrL1ywmJ^T8%oI4g(OYHO<|*Zm}~YoH}svN
zDlx?hV=WaK^-gg<k`{|l5koAC;<?podkF{GUWjuM#1%c5;O0lDTdxk{moYJ|=}6Tu
zc_4h7%zm|fmJc#bAQTLe(`NocD&#2tE!Ow5sU%`Ym1Lei_vV(e1Y*aVQ5#szxgDZb
zM>dzXgDB7|Qqu+=S2>?n1@XzXJfzLVJ9O)I&ChwW#``ZX08*)TcrO6*l()3ME%dqe
z@oMA}+ESU<f)2XeU3jh*uGQrfwCc-37kQG(arWb(O#ScqHjetqz8&wWqJ@-x!+Qr`
zSykT<&IF6UB{%%n7p1>*x*=cSk+uUZ@2KflsWU;J6ab~TH4H#VDe-FM?1Q*E{M@B?
zx)s}CXGUH2xLXUC)GL(N?31|q%=ajNz-GF5;V8CM6zZWt#wxWu`@1LPa_5V7<?H;T
zB|zrAQ+7oGE6~l+!LK*I>Lj&64Lj%SohuGEEPDXFY02+(?BeQ#lv^#W3#t45WuILw
z`HKDwy=?1)i&ksbj|@ylh-VG7KbFX}(Z{CU^&37y`ZPt0o8R@A%%V&J%qA3?@B9Rw
zg4`IRfIJS31S%q^;B8LhauecyUT#Uk&I-30x?l_1HuMA&Ob*<#`iPh;g$uq%27D*$
zS%03a%AM_Fj2s7;>23KGVlvc-S+93IGp<Jt?1?5QDaT!3%>rd{%h-bFNQI>GuvnW+
z&ndfu8vTW5QKSCM`MCZ;&X3sR`d!s!WxL~>9$A>1+U451vhSIC(ajVw>u~ydrzdcM
zXs*`^<U+68icYDFlT2d{KYCYyE9_)H#^OqbuT(7-`=m$o9MO)tApwfS6L|_MMsxSx
z5mF+4sNHrU9aH2;Jw^G7JdfXr?4)c_u*>a+LJ9^wzaH&HzcKh4-j_*{CsGys((TtL
zRR1?VOPuzClOk+QRSku2Qox}Ni5d)NQl~WYY*@{M<9`~Z5JhRywcm$!cNqy{rY^Zf
z@iGP4h>^TjojF#ObMI{leyEY{RN41DJVrO%3B_M7t2e8YJ5wyW?HaK~Q9XV=iDmL%
zd8_}cfr}tCSTa0Wu;{+fr>(ua#Fn8SByOHd_7S}#m)gyS>3B&3e5K(v)BUWg!U*u&
zkUg0+3CM=^J`L0N^R)YJ8~w9%D^k@hzp^Cn_y;6^I)QT5a6Q0P6rZRxDz((Uw*=Ak
z3K3{A6<aGqgD$7y%)8Vma$>1j_C85=Gt7BNF0*}_oGoA9hcu3titZMsHmgw_ITF20
zeHO6gLo-F9d5r@-syvBLgS$gR0|S%i;@oVz9gWSWg(9D&1Z$l`Oo<^<uizg&oOd1c
z18`ZGaJ`H6I}(4xHM&<b*hG@+1?6L_t%Nw<^Z^}Hqkv1!m{%zoEZkardf#h@{`|Ox
zw-7ei>}a@I#;`GgJ!P|zpXxt`3{#iNB>v8Ik}$0p0$`+^6RhZD@wt}}&q#G8{rZR>
z)sC9bO_6$<Fd?(Z<YWpCNGiR5ECa3KpS8^sBe4-?joyk)ccupK2>f+%LuIs0NrV<x
zm6Np*$)=L_0pcgeqo!hqBRkr=vGxC5|NmDTfVPgfaG)*J9q85h%dHlz-W@6oyP4_7
z@>Mj&m6h}oaNn<B$*9T9ZtL);MpO?_!=I!MIyba72j)fDl)B+kGh7tZQVMTeiURn*
zsD44ejF+E2Dg*#JAMp=RX0HC;r6E}NYk$cIZa(mZ@-JV15Fg~*OtkjjtTu*Zd$3|o
z(yVf3eB+nmUhjX&7elNj5o<f(3Ea+)FooAfs}j6z=;(c)i5Hv~LT2bbZ>Rwt4l<ED
zY`jy>^7Qv^?Su7WBlk+EJn<VqX{83mk1f+ty0xeZx9v05nRmqikSu&_edEN%A%Kq*
zTS5$^wPvA_$A{=DH;GTKCOG2~YBcX~Ik4@|>2HJ^6VoL_Lke0I0D<gpSguQjL{NLA
zrni89^_1$zMZrHh$FAAjnQO~G5lpVUKfsM6b&bo1U#C#^q`Ok@<a?~osg`sGoX~x=
z_-(FnT8x^q*4!2{=y6lR3t$3#njj$~lwtR$Y26pP+>vw=LCgtHRK4k43ZEdz=n)zE
zWI^I4*JZ&G3}C$6t%`(GIe~k}?&>h^i1e*0L2lb#kxURYco~x*22`^I-2#-R_s_MK
zA11k4zknpYVX(i)75W6P;tmDngd&GzGbg_txJ`-VWlg$7K$sKnbGdKI@9+@`xVPww
zC_;tRCq*$J{RB}PzqpqmbdM$pfpn_pE`XF)Ts!RnpGlLV>Q$rQwRrr99n)9G>O+xk
z$^67Z(IpNXy@p4aXxGd0zK{P+>+E-f?{qBUZL@@qHae$<tWKErvXyq{n}@QLu<AcI
z^JW>H_H4yD6A9dwXOr@Ku?gHU?iIh=-V)=?Y^)N+Q=iP`-a>yl?U(rh3o#PeftxQQ
zDss~DHheqNWR-;j8HkSQP+9LH%~^R&y*mTk$ThCst7%OfyHt9*Gd#c9=$tMg(kgO~
z)ON0jI1^++3nNfBR}#DuF@j{ox8nCU`m}@nJYO<zKHk3>ig{LzZ8{xUuadIPpERHk
zWdtLpr;T$?=3bB$<+?HsXa-yCvs(|Hwe+`HIWteH$<|wEHhjJHL%VU;W_A34HvyBZ
zDuL)Tpqc(%=*KF>b)l+f8{9bF2>?}FS9@lDC3FpX%-!Qjp#AWx=`CD`i-V~cbI|B-
z9C}y?`H|a+{HD<Iebv1Q*Ed?OgQs^p|28;dZVb#TF+0te@%;CUI@9*j;Vmu&&VBkV
zx|KL6C|`z2+lgFR(p{8=7S!IXMOAa1m6jU_(4i<myWz)=hHmtAST{W-5pxG1!N(&_
z$A>DnHhh<g$>feP8Ps@lUHgw(h4o3LlZjac!N<|vMdtO2kFy{rxa~lnbe`{u_l^_=
zxCRK$qVC`zW5R6fr)pe}>VXNz^>acp;h#~7c&K8x{kht8G>PKm8M~3}$HE)-)QSG#
zeQvL7Ezba+ps|V3#&5?DeO@{&Hs8%b*ON2GSs4z39s_6O9s(g#Br%^sUjzV}EOfKk
z`$Wz&&&<8Y!n&6(baJkf0|!BgWOAsq#IsFjM3WFEZ<uDoTc@GXcqL|4X&@1J-t~d^
z6W+VSz&XxMFSxW*K#Mb1*6QaAeifWAj&fke_rcwbQSJ#JbR_r_#yxI|#C*bWbv#C=
zA|ro&Qe(?Ade+NpNjn4ZsFjg=Mn@FzvG?T!7a^4H<KiIP!c|!{be!1#T4Y&wo5LSp
zJZaUAs+znlr*o{20$<Lq9r;Nml%`|8Cj{3sn}#Q;?B)@E;(&cj)<NEym3;nug_QRG
zXVQt!RMyX%+xi6yh)Q26+<A15(gbN?5Df?=AXRN<oe14dFc%Pe@gA(fdgi$r*2&5L
zV{X{gzn5?neO7)UAhEq>p_Th#o+ntagOY-Sl55ekPxywvRlt_|2gSl$8WzGPd5YF=
z$ciRu3)9OAsXF-Cs8P#?^iT<lG{D!Y#T*B*3g$D3e;^4}cvqtQ8A-x4Dq@-*!_#CG
zmw7fA6CCL|7=K|OFkLaT+<pg7vmlp~B1nYbLD<{RB(s87P*`BoD`~?~76WcHmz-BP
z2R{CxF$QXGCK~@IjZ!|(Yp-8f0wmn<cAXTm*5DAGkM)F0F+AOHu-)smv;RCZ+{Q5*
zX2wm(cG9XeI~X<+f<{{4Qs>5xWKA^2091NX?dCZXVgA#*dbn^9`sKTgH}f<K7OsDr
zSRA21!6NVKPH%s(w3ar@=P!ebD33_W%LpM+ag2Zkz-N#(Edz46qHNK7DlTKs*VnuD
z0!zgmg6EYVfl2NWy#UqMCRG+0Nae7K;*K`i7{gn_7HzA%tb&$Xd)Bv7e;+umwwqQg
z?#l2CCAIJ6Zi=-06SvN^ap)83+pUHW|9Wv&K5AP2_9T(oDhSgmd13vn-Dvvcry)%4
zM^+{BjVF+gOs?=QF6v)vY$U3`aX|M4nVOktO%&1(N#fqw<ni^=#m`xih!+3s&-VOx
zf-0rGVcN-wpQU}D3E@7!qvnKzS-IShXDazdWvf;G2Zv9$1kppcth8IFkETauN?_{Y
z2C#?zX=iyRBqs}X$G-=M_tRV6dq4u>G8)QRCF{ZFCdSIy>Dr2ls^*w!@oq|;gJzsw
zl?=XJFB`ZrY+UrzwGqkN!DIJu7N0T^s#h>f%-RB+7!Wx*lfC|_bNP-)`)YG)#Y`w6
zAazKT31kg9pN-o&R%`rY*SwmzT@Q2~+C>~O8)zKS8E6tN&0K1Huse0Z0nqEiC@qc$
zo<r(p^+#vI=BiwepQ_v;ovy!VFAmBy!H<pBS?MUkUxUe3GWATA(X?Vt_`7jy8?APj
z15T6oX)7T;Pr4t|t>{5^nt}l7EAK=bHhU|dyf*ckCc$~sYm&*${b}~e(>%ZMt-2dI
z|J*L(o^}*<wh$h9Mz~nR937s>&)}5PF&^zeWPEW7dz2$bTf%D#j2r<1ajR|c6Cs0i
zYjp1cQ&+<g(OpuiY*AvPFYKXDh&-Q=AyIC=@7{z0icda*22z-JK6XN1B!uNeoUWFR
zru-hb^D>6`ax9AW=1xsn>r6ZVbh2H)B<V08K~Jtl-KWC(E^1P)!meuBhHdEWgB_d2
zc7s2V;R-8gxO`ncO#@;@nV1oD-M3ZVywQKR_bjUxQF?!_b(eRVO7o3bV$N03al%*M
zd{=CK1z(_tv)2<<eWMDM@mJ)Rwa{j@-v{a8a`%)ej)^WSdYlO*16AWS3DI8~B7?k7
zY=YzgUu7W|S_5kMYf*;NpH;PB+EQ0D>%joi$20InNWk-JFfIi)l4FQles0Bv{(M57
z+V?!*#T+z*q-YJ1jTeL&Wa*}In@|C{+GT2Yyv-+VD1~3TIZfc&W<T2y97>5dxU2Um
zgbpIhaQ5!YWH$&`T{16`$^EFNq^3HYk;eiql2lukm}lLVz`YpDQM&S1&3{U!&m{6C
z5QZMzoP&Z4ecZ1PN705?i~v^#F<n9fNx#*2`NIW5sewf5ESdJlB+m0czUQdpPJZsh
zU^?o|O+F@mqbxPqebZZ(3G8|#av?}~>T%>E6#s<70lPyT-jY$(GokpuyTub?Les>o
zH1E!&ZXs|Nv_&l5_Rtf}>GQzW%-aC2LXi>u{c+1_32$-_`x~kvb3X%QhKKWD@w2V^
z5Y+-S<cudx2jKt8x4csbG=zhm2==L6Bs-13*uHvlbm<kgnf<pz{BWIwL&eMC-hBfQ
z^sl5%RmF;0AB`7x-!c38g=Ns?Dndu@m)9L4gY7LSQ=Rwo$NpLG6G5N)ivs>X!oEAI
z3GG=Ih=SCh^kM=cy(3C75EKy=1Qex-L<AH>DM|?~0qI496hY|%A|2^H^o}$I1cgB8
z2qd8;5Rw<px#!;BJ@2jc{xU8WYmwQrXMf+!H#2M%+8r0~(|U>I4aB#nA$GHCzIa5H
zcEQC>ITTm0_vFh?LSIVprQ`>05B<WL=Gzs=$&#$ZL4+_ExQ2xuyix05O0phOHWe+^
zxR`c+?4E*A^`o;!!&^&~Pf9ow5R*HJ^UbbyCOA8ZT=FuF@jCt3S=!4+;+ww|l%&{r
zCVk9Dy~)2^=SbR|{^k9NYqt0NC1jB0qfY2&=<E_HTE-8D{gakneiz@1e1Pn-&i7nb
zE>$qIbVzJyYr!UFVGHNyqIl_2=wI~Mf<=ZSK_Tn-kgUWsI@Lfo;`*vUXy0eiyzU|0
z#^(ZU)AJ{7GU2&bwAJY}aeymNv`yMVJNR#yNa1@wh+Ob==9XsDoPW8rIFx;QWrx#P
z7?NY%%cNM3t4<IFoE}{X0wOx475T%8D%$Dus5^T;73O{2<|h&F0FsuEHl`JYxxDoe
z3Y)P5EiGHW6|E~O-Rw9o6_{QwkGaAQAt*=V=B$`Yk~mWZgm%lXm$-h9{+yJe#<Ko?
zZ?{r~?F(?+n>2Ahy2<9N(M`&P^OpR~Wb5QYY>kV>^L^s8VN<mt%-YG4M|5wY>+fG5
z7Jn3(t<tzd%bMP2Fb`^6?&<G}JRmBpcWO2K_<U(X%kBK42rhr37}kjEl})^<P^WEl
zn{@Z^dxB<${~~jD`^I*#tkrozqUQx%p!!7U!A$}ACv*8=KjQbMZqV<as384I?_2mW
zFJROMyvmhGY^`?WdXl~U4^<t(?=9GM_gkG~Y9*VFS_{PduiwE|r&}LQ`rZI(3c%e;
z&%<0!*$9~fj<jOj)^Nq%u2U*+g{p#>$_evr;pho+A~#Hn`~)p;B{@VZ>NM9+UtctL
z#~sc{Q{|dpUOl{F;l?oKHKlyavWB}K^)M=Z-SWEot^^!hO&c6j_xl?oi1aMUhVcSU
z`M+~F|Ajj*+yO+lwE(p6$_&?qoaKKFJb4ily7yqqx6E4iF~I$e(&OckM3((9(Zek}
zE_|y<ms|S9)C+A&3lmRgFA-%HBam`JaPH2Oj3(dK?i&nCaRbufp|k|7NGa(fX5A$X
z*7)ZY-=5snW;4J%`Rjd3#HWc!luvQ|uH1OliK!bOd-6HvMb$DS{GRzg-`ScaHGF!K
z+Rq~gjWE5eapj<y#9D7NSI~A=Nb=-53;rP7fCfSLDxeJo>U%pWiO*yDovxvOqOF)d
zy_j;D)4!B$0q$FmBW1TJ3#^>!S(U8s`W+hJ6Sp1IP&BLAx)-BVjXe?H`??V^_;GW(
z8FABn<j0h2id=qt|Ne&*d6?VOI}xE%%;w&&9^A<Ixy5t$LT3whWpZfJaSqW{ij_G;
zoegk!c+#%zC)|a*E5t6@_-+d$D2?aAmcB-4>R6B86&vTRZlzO*HDg)=)~11<`Q<|n
zq0P%3>?ggum^Rzn3i<~XYB^%v{c4)3Z5Lx?v=~9M#g(JQw5^Zhg~5_Svlb}5&F)Fb
zYh`TM$AG|A#%BN%K?JVWx9!2zOPiPwAl%Md1bLI2+5z;PchCBv%Woo#m=8(v$I-b5
z{t+uM$)G0I8WVUws2JGd#H`f?Ok(fMwNIB@v>FR%!bb}p5$W+KQ7SoZy{EMl{4?(e
z#;ApKpHvLR4pnw=Jy{MY-q^pq>*uXt@gyyL<z3!sx1_VM*RMXd-1auf`xrF%;*!tT
z;m*6}EnU-hMnt)QF@*0Ny)A?B={CHDsj)4O+`NQWnN?rBEd13$UD!#93PyNcoN;dn
z)9`L)Sndq-pXnGB3f=$qz{Kr#{}Ir{p{vbh8hWXuIKLz+>z)5h&~nYGOAlYIzEolU
zlycHln?A>dRvfX)bc&5HRo5B6#tx}X4|TQ*G@7`YiIk(#E&LI@&c<hR5h3!{MdRE{
zGsAD2tVdsevX~4h$<V<c+8l*O_nDr)b?rbu%`WbiZzjbdM8c_wMveHm(n>A{QAKSn
z2lCZ)#-tf&DU*Y;-}G2{W9r3CdI#OJ7?;5`?SL!YrBs0))9|czHzo_`T#e4p&SqT@
z{vlP%(&hNG9}&y6sElg8eo16X_N~=oPR8Fgn91H2F&KLXKGrZZMP&Ahh5#eT=UtdD
zWlEQF*F;feftia)BahSR`;+s(6~9bLPG$04PkXK@zoe`9umRH{dVGsTE*P6Ql{(d|
zw<O+2bJ+35SOxl8w*LKwo=`<kv@PYpC+3pgPqfB<=4kAeg8-}_EO0^A1Yb`6uTsmS
z`e%>s2*7WEt@DY$YJDB5)PH1im2tFuI@(ff%!U+m*5^GhGi-I=9H0!54}W(uKm9#r
zb#kNQsR|zD1wfs}t|PP`o%UY_swGUEmW9Nms9m(&e)Q^*iYSaLP`Kn=*QrdQ6@-tY
z!W&?`O_H`@pXA_6&GWDRe6GUCC267u5z8?jqMdc0S<vb$9HGJ`S1|QCAK0w6ve}rZ
z^J}#SPUM5YKJ$!qBteoYg>ZjT)x~&w>BnHpa;DmIlk{|hkBuPBsNF1ohKuWY{64Dm
z&bh{K#{7cBklq3Aa7as;^Dlp5oz?Rm`ekc(Z@VLS_x6O8*R1WExX;s?7B`hT(B!YY
z&L6i(OSvZ*#V?$FZ}SpApP+v8Iyhyoo+Zfla17`=qdAx76i^s%L^!a#q`iI(h=>hH
za`9)3><3qjnr-sGFqQlw1ZX4`BIOmD!?Y$>*=ZSg%HhJ1!rp)lah25$Y)_u~SPQY;
z$n9@BAF7c^rIlPf6itvc_>ccMFT+!EPva==a`Zmp6*>GD)^sOa!=JX@18NlbDMPby
zi0jWG7UEkljUaq{+{t6!sL2MI;rvwxd}#v_l+rW)uF}ymbnk>Y{zsR+gFffV_)Z9X
zeljQXZoh>tq~$d5Vb!~l>I<jt^*3KBzb580pI1ig<3WxFt&<;%7SngsiWW%9<aHV|
z_S~qx7DTIS(T~VzDF}4;Fj$Fn-%nmc{#Mehyi13_E2xgYqe+v8{M1u0uPnm<YBJWI
zpXoIJRq2F{xn|1V*x2GXzrcQ}b1!00PGUGsCtl6y1E!Ki0)R%nnOQt1uRn6Ti^fY8
zkJwbpJ|bKb!h16r4b#}z2qTU?bLj?siSz)5w;}Oar@T#+>9`hXW4~|jhkot#w^w!S
zGmfXfQeEkCzygIuzfP?WVj5pZPdaaphl#1ybR7BeJq&Pd4U7%a)TcN&P7#OjY2=g;
zcX}?%%xUvYGzxptGX(6xuktGZZ8HbId2UFulljzlEfH9jt~&5~AqVC4=x~sF;LD>Y
zXvyl5rq;5shT9h$<Hq}2#9ZCtDY3}|)Urz-W*t)0ofa3@D<{X4oz98dUmB;MaN<mp
ze_O|dcVJS6-{HFNj^&qqTftC1XMFVS%3Gd>wI>Ium)sf!x>M<K4(~*CFHNfN_ccV<
zP?yCud{d!RUOB5WxUwnjm8V67V6MB#fq+!0R@XE&QvU2LVqaphDR=aTJuTOqu>Mtf
z&Qz$=FTm5f!V`aSKV)tu^>Ibj`)2}|rNFy4Z3E)I+aC?AUNLM`Q5AqIx%>RCE=-&l
z^WXU*+s)sxSeZsXuEplLb~~A>RNLK&!fLF0Dl0~bWdHN}cdXvfB=7tvbya6j&EU|}
z{t)<Wr&GDl`G2%v$GKp*hvOrf5+c)KT;Iag(rNQdXA}g#E3&<}?&_CWJGVz#G|do;
zcPZ>1lsBfgy|P)<zoqb&y8Y<_0j??phlun30Y(@4I0(<KUweRPxbnz^R_aNxd16$x
z)G7SV^Zk{%r(ZvR22R{KT&(TFqABVW)iWiAL_&#BbqP;cb#wa13D$_xQtu!;5Ml=n
z&C1m^l7#n9P_Ex&rB!H)g1YhU@`WYaX+0PFirH?98Ebon4z?Vj>dWzV=f|)9v>%nG
zADGLaMs9Z(%{p$FbUKgTDTyCm+)X3<8G8h*%sxIFS_NyZK_YeCS6oX>%RIK3hed1%
zKUw>$&roTR#AhRVcOgyt)tq{Ej-e@xx@1=wJX?9?t5@xD8u_)en*@W;h18w+0G@$s
z1a-rYjY}uMY;Tm*3joeDPzr|2D(?ap3~lu#T_Sv2(<QHJ)er?+cPIJ<Sg;OC1+I4~
zr=BSTkv7$(8(lcN^B0T0Dm0a)J@v;891TfBS?*7*t}ST>j4yigG=RNUTPz=PqX%0)
zq&{LONy~e2`WLr(`l~p<F-_ULCNS>fu2o*+SIV)VOxnrZ{8&%cRk*so6%jw(?iP#M
zGe(Ua=$|B4r@q%J`lg?l;o|QC^-6k7+s;8<r|m{Q+RMz|P6Hdq@OSZB_UL@tET4<i
zqJxj(=m`A|2W?{Qz>!EZ_Ygc_`1?^0eO|3F$Buf>%Pu`H-5YLdAC26luDxEo+VA+#
z)>@jw^3R!UVcd_m?z#6^dY)EKSxT@2MQm{5@}CQa4UEhZ?62)4d-O4>yt~5hQ#%h;
zBNw{t2Vu3TEu-ttrw3W&S3a|SFT43D;Oe<XYXP`ajHj^KOMdttu$c|h5xIcDxxe@I
zuMZ<iN+dqHriCIU?&SMQBYiC`M^NL&65=@04t^2zJy1H#RDj>m<n7>uWj!&eTYg$)
z;7dm1m5+^gq(G^^+%^+wnz2+v--(Bd$}d3f8I#d%!&!(vCVk%3bsoaNeHFft&P&|_
zFB(<}^3D%?kHrp8N85642cI2zl2q=#wRG3^VoYk@#)ZA8cZSo08In-pn>`~To?l!~
zRai#35C61f{<h#(L3|cI20O8f{Ya3IQDvLCG1Sh?i-P%mFbg*#(?jda)#v^^z*p;#
z{y1Q+Fub)~sMVt%UqQYN;=?K8FPDJ2ds$qX4uc4lyrGNb`N*!P8J`~2;Zcj1^O|ud
zBj|KvsZrSTRs2^|a7ksFPlBR+%WjjG<UE+y@^}`!gX%r3gkfHNo!%WcC6rI0-fmm6
zmJ&p@B)7{>XNb<apGv>&#w%<TfyJ$v;YrQih@Ip`Lndn3u9ESi^)RSU+qEOea#TYf
z_H_oQ=eCp3w-*m1&uJ~3q;n&c%)8{BT67M03n3lM3rU*693vE?Q^T7oUR;y#_p(g(
z9rG$o`XZ4XzQ0-d1V6qEsZI<rMSW@DFD4t{x-G6<e|D}w2yWu-)PLf4v*zjBzwiHc
zd6wabMevZF&~){Ive}9)HzM^yB(FDoQpdlLadxR$X9QK=tjw0}V5qRSrVJhS9kzaW
zG$vWbDXA`Bl(H|J<yk?Ts`^(UOTqv!*4kp1T&{I-i8jh0DX}l{_BbCR@}2$w`)5Ik
z>6Yx~%^&x)FI^@0pNHwvs!u)cHUo(tMM|$Ukzes8dgrK(T%4O6c!GULdKWmxR!@KZ
zvUSQY#f8Bggb{9d3P?^ds@%}uJSf!uOl~n)SIx0~fm>j8iN2V2*bgkQkn|t*ebrs0
z=Q&l|lc&`Zt#Jv>g#H<i)h*jK_cL_99Z@F_rex-^S=F5FY7r}4EdN{)kSdi5ZSEsH
ziOcXIq1tpLN=@B9u6E+Hgx16QI|Y@&5fF!9hi<ZsWG9dM;pf6M=~pGtQF>=l<lFey
zOPwEuoEtP-<B6Ydchb}+m=t3^iI}XD&K%!Sc`s_8JLO~(E(x4qujqP6#WDS~UToKo
za_;i&H^?6R@dLB)rm)p>RB5jMdC6Kyh63wDwVth}72hiP?$DrXM*ywdw6_ZDPtcQo
z(c}773-7Au%9RWoq_n@!W98ST84K}W%^Kcpc$FKPYP<?arSZ#!qhw;4SFj`SWTkg%
z<~P3qLGTuy>#?F(&0rcEuES2Vj)t)>lAYV6viQn255uFU@}4>uCwQ1T_>ak=vSXIl
z&Xp*gQTl9`!4RD6w~74nE87kljSr{$2<$*~XFLWLgH4cA`9quGOscf>eraQV=H)d2
zN>SYbSsr*ao(4sZBy;si7>r*d&FpSZ@EFKfeb1+D!JE+!W)e~qMt!q^vMUQ^+`-;T
zp_hwrfCjR%15={PZ=Z7Qce&#9bKp^|q-I+OeVmv;tw!3sL%WW!5Z{Nl>ML{vHst*7
z0#Rq5X~X6V^uSQhDkM5w<-{8tcSuZr8IgXTp63Kq9na|+dMAuU5m9?9o+MRlEv?dH
zK*7-GUSR}!6uKV0-SW4O{88i2lP6%vq`tEul=$<}<JP9oE$wq}A<+$nm+8%3X88U-
zSnSv;;$N@<bD_j|K_@4q2c__J7uLfS)0UXfhx_hA<GYols9@jJHr$lB(XCnq!5lW9
zY+QKWj71~Ga$}uU<t#$t?(IBz&hBe*&MrGV+O?WxmVGycb>f4V;LVqa9>VW>-R}#O
ziBMbfX{iIy_^dFKT2YXGbpmw-SSnTK;>u7GfbocN(FrGa!a5@bve24;72ZpKd;QS#
zwN5@%u9K|b_?WY6mmF9{+HYUEn)mj*k8}+EkWhPBKH^!}{I-@`+nl!N*4Hjoe$XSw
ztX|8{gOtyTw(1o3E%TzwPux<*dWP<JZjrmhv20N*l^UVB3Jt3s9~${vFdxf@RQW%H
zye{A{>yfCAd<9RfThQ+>*s7FhGMxSvz|40^w(I4hXW+cz+WXsu24bfKV5$OII)0Oz
zLP?Bl#1uLkx$CR_N%b^=eKKUqfH$9!CTU$M>7^@n5&qF}xB~rcf!k#rzBm4tNbN>&
z*2K{MjDHNhAW^f-;;Vsmq$#yL(`dQue-#S?FDj*}Dvhy2h8M>!v529H3%C@A>gLnu
zYt@|+H9GFK*$csUOnJu(2sb~Q+G0kA$o1*o+AQ<fOoXWG&YlVmbI)Q4&ae|?Bf($|
z-Wep`vNlKT&Ep%Ccpu%4VJB)nqTBwx?;;46dh;-&aO|BIHT&blkIHwdF+uv;l=MNI
zv1%%>$<UY~bH(^eM{CQw)ZuJX3QA++^UHu7P{srYRk4I&t`KW9>~U4Ovb_Y@8a#*`
zU)yPAu9bLH9J(&8zEQ)`UgTb}G&F@&bUe_J&fqrmWgAj5q7_-KUZGc&KDH&lkf5K4
zL;6g-so$huAmHAKIvd|cdw=ey&}3&pb6tG3XS4!}lKbP1DwCLm103oU<M3lYGuR;q
zgUhz=&)_q!#;l}Q!I6xhg!Nah{?=cQxXfFox%f>R_j?SYx7#_t!2Tvgf_ARNvqM{r
z$QryLt>fUF-7om>;uiq%z0vWvO*f{#957E?_Nmd9CRJdwmsZ*Kh4;fA)AmPp+jxxV
zWuW}aZ}v(krOEjnPI4%fCETXVZiW!rf$Wv#_e~(S&)G4FN1u3%LxYPJZu_o;O2&Zc
zfU)nHV*PtNd0ZFzDfCm#R|XD^%3@<<ZmA30c@ps+{82dXb(|4Py#GkwN#h`;dQSGn
z2+1PJM)4bX%wo7%AnaBf5GKtnI1e}bxwZ@QpZ>+3ahUG+I2f)O)Vu^;ATHhZTD+gK
z0Xy9FJn8LhbaCNZ(alRK#C?4La-(>=|IIxku@$D|+Os=d?DIJ>i}NQ=+>ewIhZw)b
zMsl!V>E)iS<EGVjZ5KoHw4YbZ)-B%20>klBSP8!8Dvy81?Al5`di)ZX0x~ZWo)}nQ
zx0Cn`i<x~=wfJ7nYr6T_^UDJ82sN(WH&~(yn|DdW3!7AlQSEd><q#xS_?p+04iCyj
zAxsYJ-#aDV3&t*-Vsp}Lt;i_z$7dy#T-fA3!|0&a{RH2vEU>wApBbX8H2Es8s>`C^
zfY<k^ue4Mi;pb8j6xkOQ)Q?8&2#c?;yadk&l|Dp6=*5t4t}?8(Wl(&d;=h6)3XErC
z*Iy@pdt+*^O&qO?K0BXW2k9*mg6kOn{Gw<$zt-Zq7m6)$fjmBA730$^te-scZoJ^d
z2lQ5Wt09POob+n68+5s3WU5HAnU+484WBH0^QAx8XfATReW?2~>6d%~{gbhaMuueQ
zQ9XN?03-fz1G&TFpas9p|C-JdfZ9jHi^uZ?cWa9pY$W0KRz}uyAwpAa$c`Uc1%>`j
ztsz2~LXo$GB7o$uu<(mHFJlW^>$G`LGp!w=yrsZuBBxQdWVfKkX?FTm#b`^+{$h*c
z_Q8wrM82x+N}%zUTN|g@??bfjZ)M5NSP)j$s9|m)&816j*Q;Umo_w>-+{c{Ho$9<E
z&E9>`XxA<$nT4_c(8>LkYaKYx4mraH8Q#C}IR1HFUXeusrBPjPcloQ2R^oLK4Y>qs
z40|!9P81;YyHkxBN!DsUy`F=T&TOw})owP(X4G!wGNLhU$SmwKoDeQS%VQ<4SClX-
z0g)q56;@iRpsI}2UwYJm$>&9-RS|&2IbD|VC13i3gBqe+K4MD2GtpT=sdi=OvXgGX
z>a{tWmytsgV7CGV&s-<UG_Z8_R=Z%&1Qu}Jh_tce=Tw|Ic0iLF!n>N66U;xnVt5wt
zFt)yS2&kawmhvchR}^pe<^bDJWDZdr&srl`Mq@gb517y2ioQG`sJ3b;OSUMgk*oQB
z#%}y`e(Cf1uhqZpYg?Bf&Ql9MpM<`kcmcqjnA1aiGvr#rwM?Vk_wlchTz*0sntNbC
zsTR94MmZ)+croQvqw-9hnoEt<^YcTXeTD33XFEHetu1c^RpQ($nNh=zmTbh;$njgf
zTc3z8K3_HNfm>Q@P)p8E$Qf6<#@&Q=t?Kc|UQCmiO}MUp;5l<qJ8aQzDzfbh(eNJS
z(~bSDuS~?SN!Eo&bR4HR{k7QSlr8MukYy#UWp`pcwx~Z+qEE8&3ez?}Q+6Yy=(?AC
zM8t$l0|Ark6HjFE&!s9XBl?Z#ftADb!-0Tn!Lksiud;!Rsn3DjVZ5pKm~b>)n#261
zTO{8bT+kZ$2IRDe825>jU)*r?OvVoCvQqtbHQm~xVVQk9m|PJy6`TS6d@1&>{i$<{
zQOL7kxm!LMy+TOKtI*rk$Ys`X#CQYRt-Fca`JhE%y2$De4=ih}rD=B<eG&0y;G!Iq
zG>q*x@|wt;E8Ni+EUT-teZE%Am*)iG7BhdlTL?x?x_+3jcJF;$p{|F``of&rsN8du
z;c?0MjbG_fSJO*mbgw!f*{0>;N0l>}$!a44|M@iH*EN=tjQZ1mCZ<=iZe9xGM3NWc
z`=unp`D=C<nhPd9igWuKJio53tBS65gtu-<e06UcRjG)c#-67)is$d}t>c=d2@8U-
zb6<dT@$u;H6E4GwyQJI6zkZ#Kr%C`sdzYGRONy@Or^ak2slP|f=D}JX$7Nq7{jLK!
z#`?b$KRMAV$7`lH5w2_!(5?wQmtu5zMGZ9;>OxDY<O{FhVe>r$8tx(I(DYoh`E+b*
z??34BnI=SEO>4gAyIo<fIJ5t$>Q?>1!+dd-BP{=oH8K6za-HjbLn8ICOl`X!^&4k5
zle|A;kLK@*P5q3<riqqjGa@E#>++!<+IqE~18w-!c>8=O3=aLToN9|8e-$H0nh~_(
zuzvgf8^v$3V!-AFGD)B&CS~lc6>?sUmhqi`QG;Y}Tn=^SZP2bc;E$3On+yb<RO0)k
z5Tmb@Y^%j07Bq(SU@FlD*&~V?%ZzGorY<Qzz2vO8-SBNTv1{{@AcjbJfent`HA^!^
z?6f%v3*%><Uwm@Gh~}qOlXG^Hq@hL)QsWUCKPtieod=xG?LtM(_v$Y7GC{sK7_DpF
z_7U`YBSb1<E#}f}k}Hv#Ut6^a9N>BU@KwQhyYJ#tiH-_NHTJ!1qB(uV`>yc3i4JYL
z-Qw9o$tPDem<GipjiKU3Jde1**`Ir@s9;st=L|viGHN@|&p(4W`Z=DjlDb-7GK!!V
z>%MZAywRkUt#9eMJ=+5-)Mt|&Nr^)ixgAYfMyRbl{GQ=3<~5hM?D583;KP93AClJ*
z8ZIXcjDE5@UdYfw{yc3DV0BYzKIO`qnG^s}9jT~ZX|a3S@+9n!FecJ)1s5!N*1u(r
zDYKW*5)H)<N2OQFgpcZu@wM1n#Jb_{O1Mdj2qdI9NT<!KXnAx1YNd3$gnRB-?*)so
zffw}}9YoGS6x``nZv;2|N#&WC!u_OD3NvO9Te=3L>bIFJJnMeC%TXnYFBg^79z08p
zh(-H8deq0vSI-))SO+Ek?gzV@&5q6OCp&`FZ|hF9)6#F9t$deUyq5enY9B8ox>J8#
z6MbNY-g=Ofm<KgXNE*9s%&w$Nq2;ySf51kcX{J*#RAgVUQ&MKcQWiAJ_&wY7YaGvB
znD*E()_T+aFa1~GMO@GK7BO1F5EfCo<o&B$(-YrK^?OCakKQA{yo5_;k>7v+uJnF@
z{&SVoSDicBjE&xrhRQz#SL|t$e^MFbpGx{8&1acot1tAsR>x=6@dvBPi_P1rvR<3u
zgWsFminW;6=0hoJsLN>epE``fDzPnLf$RZo<)?exiC<Y;5@8;=wlF$7jyYO$kG(KL
zYm&Q%^F(^!ujk&GjX|dV)%WSbn;1L*LCI_hQV}=v29^!ynC|H~+)`C3;ADqO>m#hC
zxiteB9%F)JHQ|eHCk({D7~&HB1>l$`A|OrpnedpZ^zAGd7=rw^_FzZ|vrfLDCLR0c
za_!pd!t3fU=+>9)jL~((?%Uga*?StGXjA#JdtGq}vQp0%l%=7)As-%xyy}yze*RE7
z_`cfvS2R_CxBI#lkUOQ*oI%lNt?#6y>#17FosR8~rI$|MR`VWAd@qOiI>Ed6E+F+Y
zzN?g6B~*uflF8UiZ63T89l+LX<Brn{NRzk#h}OuiAeW}dnfrUzys>hqb(oV@cxvtS
z^e5;!k6E>~a`t*&nY=$GXqm(v^}g%R*MB7;Ko14sIRbFp`iq92FBk7u`rlW-PwNBC
zANBCoIF^kp>N}?veDwsIZ+k5-P3mVeKF8o4$v+-G`TN|0uBf)$w^n>}q-W$+uc<9e
z)w6!n(<^h~?9xE`{a@Z~C3$JIb(DU3mStvhI~>nXP2T8~0;maDl`W$-=&Bho?iDbn
zENrin*krk-i4`2*(<~b9*%m(UPEU+e3|%z0ai?FrQq9IwJw#OrTZwPflL(E~ASIq|
z>y!I%m;Q!zHy0DTQ-Y0ChuSBrFhQb2k*J=IqC-Rm9Mfjrcpe6c=wHQ`5<1eL&Ha7G
zQ$86DTYhDR!nJyg;Tz5FKEF;&YWz8t>wVS#l@v~|f3~FA!3IxJ&kZWHYUWLe9QLQ8
zXtZ;u(0M(FL-8E@Uy9%@>FAV8^$aPy8Pv`nohajmeA$Q4cNy@FgU;w`tNil;uD_6B
zbEvs@SwYX$-P(TE@G_BC`t+~2f`W2~)4ZFa!saF}W1}TXuqyv9ZjZNQn*vbR04P1Y
z0c``x9mjyi0aNS&CU@RWB12QtMy+cnkDf-}@+3|FyR`vzp1tgi=Du<?-vk+Vk?En=
zPX~QH&S)MsMSK4s*=dFz<(Ws@kg;oJIrNf3w9dD3=@t`!h>d$3J!}OYVa^X#WoV1b
zSdBCXWr=bG0^y5%zjWw=BXu4s*{WQB(nDJuM_+rF(RQUpI*p~DS@5okyUvoH=3Hbp
zCMPt>{xmO$&o^}4DUqWYV$ijntkGXwKjH;eryP1iYdfO0NgBMOZfbLgQmdLt>KEDQ
zl+B_w{(aVF@~Ujoerhpz-}>umxQnnWF`L?y?>T|tT~{#u^R->DQyW_QsrHa!-OX!j
zD_8%W&O0FhFE7MfSgPYG#Ohy%d`13tNX8IeH#q6j|KcHr18U?-5g6`dKIC58owAE4
zr_7+^Cf2Xlf1SQeZ4HlFCZo4~XDl_@%gJB9F@%{N{%PDyHucSkWtunm+)wp^;Wxz;
zoU5==70&<DFY;fp4yC|8*<z=Pu3?8<&+fLZOZ6ExTkI_lPxA<H@7Q{>r##ob*0w77
zIcN;Joj|zeoWXEG`A8%2A5yNwb&<BKwSVBF_cRz<^IHh(`@TP$?ZEfp9%H${u#xs2
zeOrFdaGwjE=1*pwJf^8XrDO47@LqAyA%k13fTFMyJOU7&#(?;#5>j(wJ~Mb(N9R*y
zpvrYC=976b!RBEq=>xBP!`wooxkZF+i7WI#6UR{k0F!hk=SFTgBkibhaqpP$x1B;C
z$jqD1ti6HFy!hFzwVby94Q2xAC~0UF<p0VzFOf!g_C~T4TC+e#BO5Ae2@*zpf?9H1
zd3AC;<{f2`12i<MAI7Mj8(xUqj5Y;teifaFA>B!`evF^*6_PW3{`H$m=NoT`30eB>
zt%eN{I_bg^lgIUcxA5N_4?MQwBO!RqbyWMypDN=`?_<_KE7q>1zc!U+A8*MNw9qyx
zOh4<{v;Hyo*|(j4-kX+z?{qaHa@)y2J0BvUq_PS$bxZ%cbnPZx3In^uPm@-KKqUoR
zn&u^#xA79Owbz9X3e1E*XIknO17SMF8pWIb@L>#u>CQTx!ruDa))B0mK5F2VDZ!zX
zCUxVqM!WqjI8mnJE~hPvcq2=cfo5kuMk=B5ZeT*7tj#bdWBKb-vRVkoQwEIc_H9Du
z<N{O3d&?KWPn)Tet~2OK73bg7z;f_C#($o(|D0R!ci#eSiov5eRa$Hvks4<NWshXn
z{rmcx<_G024ah9X)gIPUNu%GklD)TXn}_`I_tme~YZ;Qz_9I&dzk9(Q0Vo5Ca0muC
zMK{smKLv%uTLGFYH(70}Pu9js9_Ump(A+0ii~~{b7Yq5R$oZ{uc>m=2kEf)9?a#!r
zYCjEA%51}!FXh_f@6y+P&NTwa(}7o*4J1}OV9*;&#VW7kV{q^NZ%B&Eurq>sEZnin
zo7L@_Lq60i`-6UNUrE4Q)*bx+%w+iQ@AjWxIOyi^U#EQUuTx%n{I45jn8-N)<o)AS
z4Fib(u5|cj!0HwdD!nkn3-khLR_W$>mbokXF_4(;SWVQvh{R|GvZ-F<`f{elPKYjo
z4}q8D5X^1=Oc%(TI*RF=5y_)BwjkfgZVan2L`|!;Jo=Q$_LR>?CI-&M=#~p%5!IZh
zTglZSa~p>03EX|p{iSR{N;pTn7#FgbM1RUZ$XBrUJi-W42;GpMoqVynTRpQ&sV!K%
zlAo6F|2fPK!Lp!kM{Zi#d}L+B`d<bK${%hyQ^BhUy5RS9ex}ff`t8H)Pi#c2j~!&c
zyPYFoCbt`vMh`^n?K%)eH9Kv^M6jIX@(Y}2s~L!`@z-q~Tl>3o5OY~4-QF4dj-DyW
z%g-BB!Nq%t#0CbPIr()oKeJA$uaV_?7A~6ClKTWC@Gd@{!^BsbA(cB@)v|fC4cyG{
zP8X78D%eI$G#?RVYMf6%>F~UY76)xlzL+I#G$mPhZu8ARx9Ej@GI@gk30D8k0?>T4
zCC8Db8p^&?czkj7U;P_FNIU=-3|Ps<T&8{5#zj{&Y0TDD;|m-|Nx8bzgm-I-dd^q|
zz`SD*d=8jA>{d(xkaGZv4Z*l-M;xHz>5TYGQ6pM&rnEM0sn?cBSFRBYHuqxc6C<DK
zSS-V7J!JlK(?TrM#6E7PFZKpEuPckRueC~n2eB6dB-6QBkA3nOevK7v!i7Y}KW>|5
zG5C6`F#hs_Od9;AHr%a;Nxjo<Gu6S&>d49U@}J-evtKKr+u6?s_V0jlUy}dLlm4^3
z{HXG;<K1!ke;;q}_m)3Oo5>XmMf0sGirT9~vpz%1%fDZ(dx`?FZ%Ozv*C9a9rNA<_
z3rcH`GBVUhq!mnZN0=1Ue+oPT(gkv9H+1xN(LwzkbVba)Saah29TPc{!r}BZ9D6^N
zAosD^`iwDm0`sWs1%hmmp(qm+Wxq6#Z9)@AF}O*cvJ!Y2@sx5pQ&hWORL*J?5V9z#
zh2So%%o!qyq(A-nI8uBZejwCL{#=Lh8&6q+;)WmZhv#;;+uaE6`+uD9e>6$5Wzhfz
zTUKKNO;LSQV9)BG*2(`0!()bk=<c`VxUag<r+Y==Tg`;Ztu*|R7Wx5%C?e_o>nt!#
zD77u?2=Sc0TsibPUCTBdrgA95c)O?a6PMy2x{eg}(P$7T4#=Uog5QqdZ6tSK`KY*R
z9ser}JMym)TBrCKU-~)}*eE%|UpBJPHfqGP7PFMXtwAv=C!+d&M2u85Y<yn7Cz1i2
zyv4ZTZMxcFh(!AROTOPOSO3OXHZ*%v=3h;uQ(J;N0jwYekVMtLd%6GfkZIKTF5nW5
z|85X#{50)}_f_TlR{(#MAo!>Pv3DlAvjVQXJa<@9H~-x=V51x=jE4p_%Y$MrPnJI2
zPh&#wZSip26JU}ZneP_SRk!|gx?HIBOl1X%_04mph=EINA>yWcCwhV<MJ$P5C1W9@
zQXJ+4oUEL=`{^IhSUA5(MIWsp>i&?r`)~VqM!ZVHS?3ez(%0{nhqB>uNzbiAvn1If
zIFj^th$o}Ov{dL<c64uPZ^p4^I4sN6tUXIX!)MKl8u8Vo=`gvvbJsz@t}1y}(6^%C
zssGP;XC9w-W|2Y;O(pb6`2T8|H7yFVjMn1jdm?v7PQ%Yk*{`*!=J(4^r*3PHUwNn$
z@8v<YYi65~%gyk~_@}d=*yuqz9pP9f8ACnRCP!SnksoE)77#c{uZqD3iw#JiN1wHW
zLBxu=>nFNLg1DrHY9Tm2Fd26)#y?TEF*ry?-J;;q#ns*unQSugeW?jKY`RQ0E|q{B
zv)U(hSKV&5(*`ck<}_Gy2N{AEPhw*SjD6xDb;Hp~3s(#MITZoiY+6el6*S&#4)cs(
z-e}(FTi3>F7?hNCnE|yMiGNN4<;@_p_R#q(r9Jpc*2#AQzrv$Bd0~x^Rje&||8&8H
ze?Q9q;E&@wqUuBogtx5(gr5RXt)5HV|M|1yKzU8I%;HaZ#fS|BX#6LN5Ay^scbf4m
zen1A~|A?}omg>1ZOZ8aGMM4M(J~hAHUY)O1J9eOqr^aBu7Kc($b#2F7ccS}=!+n)W
z5spB!Rqil84~;ZpBvXlfyb)rK87q1A5}=RC|DC?|*#yZ^8RJhpRNs%yZI6HPxig)E
zFASK_&E2zzi(*te%nCJP6ucV<zxKUNGpO_vt-bnN_|Ci-h$Wtb69gsGyY&HWiJX!o
zLNv%K213Nc-$y>pSHf91G!@&F&y^RxXvC*Fn8RN6=9*hLjNk2-Lr=~dd`qiZNUaln
zf~T46@#f>`p^x{+S4M~JvLZUqu>$qY@>gr=?w&#gok~(=UAH#kyMDfN{@v~k)al;G
z4pyV>z$B*6^x~+yIw#r&{pHa2-tD5x((g@K5NCIXwqW0b!hcf|Hk@44epBkH?$mXB
zcsC4$^d)bQ!k1x&BJ+511+7PE)&|M%a^$c(KlSBSv>w%}__RFlHjrGj3$aFAQ!R|L
zdy)Tx;+PgL?JthbF<v*eBPPIabOw9y$aNi(PsJj!kJs-lj(xO9@Af$=JowuGOgAMl
z>)=R>x__YmGjDOfYaMJu#U1GH$j%_~l=%JGMrzMwWcT*$E#S)hT(X&*1rGO`3G17v
zPQe6`yrGScW`46S{YD;={ahP-X42Gbybr7RtlbAm=q7v0#>BcHuhDukcnE-3!p)D^
zP}_nm^6mHVo}<^{{k=wSC$2?+b3UHB59O#lc<1H-6oI&W4re4`Fu<l!Y1f0f(C|x#
z%+a6k8uBm#q4c)t%z%OLlhP0T6MWkGgjM6ja~Bw7<U;|9vVyp~>MU`#OmKK~=|U_o
zI^jHe<$OFGB(2q$Ydt3p_DuxZ^bF}F!$)oZ1kEw23j|7Ox%UB77Lybh?lbt`5F?2s
z4p}1zZ2&MGnhpw~8^!R!$Z^tWN3nt&@l;rXT*k{AUhh%DSJX(u7Kp=r9lmk&v$yMD
z&(^JsX>bktzR=Bn3?*-^d)4kY)t5YnH*er@B(LKo<sD9fkTu=kcS%4&0=az~I2f))
zeTJ586iP?JDT}1M<&1QsFXbkf8BW<CsbL*Uq40(m?EL@C0yyeiH{PgX+Co<CgPSOK
zy%HDEAId17d~8ZdiVjX|R8rIWZibQwA9^ixX}UwX`kFQ(=8{`46CC80I)25f?G}K2
zJpi-IY36IKqr%*ef&?u%rvq%9@WGP61p&xhJ*2JU3P36by3JthtJPu3v8wAs4~$Tt
zDtv(HU2H+}bFkuK&zaP!$Tt#obsJR_7Z<?n!nf#|Vz1ls*wCOhyGQLj3g*yH>T$xw
zD|lOdI?2R21bDF|(0WFHCZxNwR+q$M0}(4o(1X;<fm7kq0hzacwnb+UuJNOZ(UJ#4
zm8XSG-xeVgZxcEVe8g-Bk){y1J*T9w!s<r^az(r_$scZoI8>{kOz&EH4$>~mUk2Yq
z{-PEb<B088MHEzvvPaHd<NB`9<?$?&l(7suHtBD~QwVjB{9xVbnQ0RRv&%%Y>{07F
z&<1Ie@jw5I_C2D<o|rmUWe_|-&3f^X8l3@Gp>$>(LPrZIG^zD)wBi9h_mDb^@L}8V
zP2_MeHt0GX?T-YU0u1MqL-b@xaiM?Yz}mwMMCtI+C#0OyAe)XzW4KU1(8dW<sG7s1
zXi4mhRwop_4E26Y*M29E7@L)Pla?W2`0^-3Ij2SZEMkQe6w-ZPG*VZCE{xl=^J!hq
z75APqQmz=hfi?mJZwqFTsH>&J_o><T@+r>WVC@m-K)3Y-zm@DaC(SXNAjp_GdL3W}
z&3~nBY6DjiV7(F?h>)yKMM~{(VUZ<OCWCkm3?=mw#PvODdwSvhHI#=`GNtaw{cxSq
zeN@^F-e}drilfk1ndw2u>SZVhX}--qC<~$D$kFu}nvc@rj$<ix!IPJ4gTx=iPs}z_
zlBZ9WqJ58kte4s#cMl3DU`>>@UGufTsn<s9s1B}f-PzUy)>Q)e0X9mfW-nQ<6d%4O
zq>AGPld>kP_|}m`nu>fBaI4!&+M`3(h{SeJ7Tm*>u^22SM~TvvXz^6SNrHs&IJ)z&
zMLw!bb+P1VY>h98S$61C>&f%bWdq+YQW7|r0ByUm*%=208;IoyCo^rF%c;D!h_{te
z<`B26lP&Su{qbj7b@h`~8Snt0)LSLg=wfBhX-?o-JdGwqqWZ2_s4T?2-GSVikS-$6
zuLbdD{=&otL2!L3ZI}cF+RP-6`5^y5+OIL|TlKb<lFtCs-QL5Nbu9>sR?WM{O?1{&
z8LjWLLS??*WJ=XEXAN|tx_zVc5WJ5DA>rg7pqlZ+JG)tee)GQMpPRZ63V~=~cYB6s
zh&r_uHpPKLYf;Rob0q!4BiwOp7=j*~JX_0!KyzoX>f7)9w~q4u%bP*EKC}R_?~#U#
zSFD<NhQ3HWZG(lztpC`bTT)Wh(Aq-lxaZKXsL`#oo-XxaM1XV%KZc|p@=P!3TBs!0
z5a!;ENzcw_RmLg$uAB#_hjcQbi9L#>oVgKD0Q|vX5O1Ivozcq#UT>+`%&g>VkMVaH
z=gfz4xb=e)-A7L|=J1n;KV6e^$V_~kHK7aX;;JR-CtL}&zvj^^Cgd2%1wNv){MlEy
z&~D&!Uvouo;yhTO58(1%5iuVsOX`SJbUH;_+VrLTSBEL3NY{;FbnWy|J>?y{h}hUI
z^zViCig<r~+~p#Ff>glV#Fh(b5RKkZm^dh1BVJoauV>6cYoPbfy8Y#@u70Ugs+7BE
z*@>glLs-`&2wAzTR|nm=NW~HRv8C3qU({ME{vdb))=0tbf>CIJv-h#i#nfjrq7(m^
zxPRMt2)fD@2+1Ij8xAQ?AHt%2DYV_=Qp?Z;Nfgg1XQ}`+cHQ8iwo|_Dt?V-R=q4u+
zIEQ16&C)z2Z~MkFB;0KEoucMkrq~-Im)Vr2ylNZ@9ck8hk0Ron+q=jI?hRL1rbqW(
z)#lc(PvwQsy@5OYbXX9fcKFAtdRnOfz$)kY1j!%JD#hw6^gI@oA@nR^uqb+c{zjR1
zZWwW>?9E0@n$s!W@CQk(XVr;w6DVO?ioh*!R96yF3mKNuZY-{By8>9Ki~=Z<Itrh6
zm?9Q$?$my29VCcZymmbG{b)EC;O3Kq>@~N3ub4{g4F#{BV&&Ga-7M`7?;^;awW8kt
zZPw&|<cy|{M$SmqA*jD7wMU**T|W_%X`R|lZ|Zc3?C6JXFbFvu=1X40^SzP@tZu~`
zqo5b5W@#o!nF*}xvH!ZD@^{h~yi^ahD9_jXu*T1#C#c6>eC*C>2z7^i0hM_CPsJEH
z-kr<S;oP?ID8p^6@tUws>Kd#hz76|&d4wF^cq&0{gf+t{q&<*9lPF^P>V$w7r&;}g
z;uzyi=1+zFe8xB&=h;XM;4O_P!U+jZdcm@!T|zXY_NRCl$%MZ~+8D#caN-Ta%b-qE
zf%As+@Gr@hmZ#<DAtH<Ku0z(NxHD8*+mOR73*koo5y5JbFBklSnCy2AQwis4B5mAd
z3hmoiq43dO@Qgd5l^LTim<Wcb_lb}&bZ#F?+iK}(Iz+@g=C;OMQjmHglOXBc6>Ban
zcLv96tQ;Q#3Af>#q<Q)D_ppMBUE>4ZtG3oH2c<ihg9#T_GG%v`7HjioXfR=u>ZtaM
zYPJ<jDMS~C|E8J=;)&U#!N*p{DZnXMQbzYI2)W-tnJ4N0puJ&pH1r2fNBU4KtHKYq
zNKdO~O(w#ZXA`eOs3kiXQqA%#s*6-J)fxF6DPY`^txYjI{CEf^vlM2S9k%=HBOEJh
z*v{ID<Vnh9bC7fH*Bcu9F5X_vN+jdnuddm`D{3Al8C&>mXbf2*ViRH)ne`7+!HEKm
z<eYe6>qMHcCKlA=)WP*D<cdJDQgA49XLwAu9Ct3;>vrSo%1W_k?tc*mj}&{zB(K=R
z=>p#8sbbCp%%LCM`h;xZgb1JlUdM>Z#+INU&}V9-Yt^RmiIaUX(KfO#V=-Q$Uy?(Z
zbfQClkqM|pcNXOsp=&k{MJzIdZXcM}ovW8q^e^|f|GsG8^$?*T<{sRaC->g<sDaWM
z&6&B^vIWZpu7yq+R9&e%dUbfg`fzJ+;s|vJ+edFA3n^Vkt{1zHp#hM_ypp;zXaH=&
zrjerTb=149ID`(fMDCEsjw65;%`a&mRR6MM%-1H0NflGm((H;;GCpbV_4<b5mT2rE
zc4!;Cu{;0lZA1p-3M;TJaK?EQ@PSX2B^pKR!HDV(WsH@x&g~C7OB-nuIVz>)bdT39
zgdQ&{AL}Cv)KC~gMl9bU?M}DoJH<w_t<d4-ADjy#<z3}y53a<WR#XXx^|81z%X~o$
z$K;vh1YKvjD$o|`y(FuLxplK+8xYGYtRSMz!Wv%aAEm~4<{kq<RG?$^bkjoL{l*3T
zI2A3pND@rldexxZHxcaXsHbQS;VJcJbYi9@&r23$6=jg)=oP)-fj@k}6)7m5E76#=
zd-vA$%v&atgkq{SpEb4MpoO+KVQ2$24cmv(%;%LUsBsS~+{D*5SFmM*od27Nn7xv0
z;_`)Y?-8f5TeR^uVRwxolpUh!kz7fj=D9|<+Jegk#+%6B)OxBXb$z#ck6(8qcj^WT
zJym~&Z%ubNo(3`y<42`4!b4Oo$}=?IhSiA6<?q$L<W>B#AqW}gJy;<zO1ZH4Oa{8%
z>#L86h6NNKz#LVK2(3>A+Za~nPGXji{6s(-KjJxj)8#I_mbc8|iuGQ(rf8`FYny8W
zb8bnHI-)a(5$P&R<<d4gIVUrNcp)eq#NvmS(cuJ%si23+Zy7M{x?w#cAHJQ8!tm$n
z7G9MCdg`?TY;dQMqc{NPQg{b&0nWyKqd%MFGu(~XD%<3E9%j4HmZ2Ir-^+!_4F}V`
zC130{Z}hnO@$-(1><%&4pe-i|`5pz!g!d-NEoQ*+N&cf)CT?R58mpk8wz!P3q<7Lb
znkirW;SZX3w`d`ON^98u3w=Fk;@U(O;B`%@Ta+3t#IVKn4aoo&Pss@|nXY7Bi}IHb
zqo9tkd?vb$;A8jxc;N!lo7$gpCujzUf<h?6#9nM^fdVdU0WXfI0uzIT6_4uv;*ZCe
zg7zLNqR<7+SC8~}y(*U_K}a8RE<U&$hHA$1ZD5tq(2dB^iSCSNuLwjQD;@PJ=+2J<
z<glZ^?9}rPdX8ZWCg0uN+FOdGVHqv=U3jj(HiQSLNT6?tCbC&-Q$Fut?v4BU64esH
z0y5}@b}+h4!^Yv<ZxfLvU1iH;ix1+j=t{VAYOPPn09oimx=UGDIAUcHA-lR+hDRnC
zJ)xPq%`#G?VW3!NoEcQg{extAhiqPO#zHXodNYCdd0*pAKFkFXCX>*3f9nO)5xr~(
zQ$bERpa1=^R5SNs78E%BfSoz|>3h_-R<&Mps}*rOP8tA5;jFZ8ohqa8g~|V63tjuv
zwab4~vk43qcFY;fg!*cyx51T|V~U-QbUNGqA2DGFy1u?8c-fbRxoA-~9SN+bE*(>d
zFF@NFt(6175r@(UC0xTulC%&iIfPs~#+bQ3{4EdHb}Ms4IehUU;S13~b&>6?A#wJt
zDB9<j_=!cP0`c(5;J$E}0)R{)8WR#-3De&x2HsJ@K>s<QhYXVEG6u<yVz~#GHv90N
zjVDkkl#YP<{Q9(_LuAqlGgm2-=6sGJj(Ep^DHz=O=^mYVWlpyuDej~ZA<<|qLQq$+
zkuZ7_@m`!1_ok5Wg)Eaq7{7<WpLTV;r#%u6>jScjsI9{Yi&u3a3P7Je@-_3MQb_R7
zPH^9lzx?ch_mR+{HtEQDpX*uP_a`x*j<^(SkS^5UG(hD}#Zn|uPmk~S;I`_)sW$=;
z$`_A!vwSotyX~^ZGW>#4T)%+NJC3%;cbfL04!TPmI|dBXb}lINbp2nL;2``L3{KhY
zNk`)SoWH3L91b;B&XhV*vPd$jf=JBJq2l)oy+b=HAt7pHI>$Kxrl7g+K4b?2SFx`j
z;3DaKPP%gn-ph@g>p(#m`jDoi$fR@(fZhLQ{G!2o_##0TIgE<{D+~>b%;{cWVZ?ok
z<vzi=Wi7&^gJ&@{InwE1Rsp)z&D;CZ@@_UzLG@8q@XVQx3k|~&u&#I=h?_W!=L+qF
zk|5T2J|O<urA#<kCVn9pa80*y2L+CmBk83)Tm?_Lh>!}3KJh5dk3f}fk@}Jr&Cyl6
zu+^*rCp#Kmh2`>~k%^$|BeYxYG$B3ZTLX1LeqmzRr*62vKg=jFtEJ}eU?@-jSnIsd
z^YR;n{-RdTO7ARDCXN($buzspFHIa7=kn}B6G6vdws-6GRMek(YVle`m%uagMHUoj
zB;`07d<O*0i2qGB?l<w&4fP_LWPf_nPAXjU6T)nxz<Mq#Yo?Z3yd%=I^6)COy&5_%
z{Ts^?o?1_d>?bJa(Czr0$v7G&G(Sz6<r2gNF7)!Gt3qV59XnY;J+5)gFCLb(H7&Fo
zL&U*q!SlT-<c_E4g&?seYuxA^xPgd^<Cmu|9m87b#7?Q?=|hFf?E_EGjNcdG=CCqq
zY60q`uo^awe&c{bBNjqxa;3<)AdrcsbgXL5Ng&u7G@ld`uHJiHTL%)ms7nhm(NGUh
zk_?kaM?S+1ZKp(VuQ}+v5TDOIo6(!;^UU##fgB=l^KMcxiN>RZ8>MqER%xeI;ZMka
zc?J4=VcB0XitvE)shIZM?Urg{_R+t!$HEG!9|rbd_1#sJV&og3^<SJf|KLdtIuRsc
zN13B^W1&CjkHaXAm~AzN|2telD4$$+jvaR`sCzb%<9Z!6>#JnRKYlHFSkcrq8#5T#
zNG+@mshpy2R8b35gR7xU3$M;bP2HTj*n8)Q<DN4tSsnL`HV~cEdV$jNLpS;5ySIex
z2qp#p^BMk6ny1wG9vU%$?de8EX%kRx7Lx5r^FVf>?JA7Gqwl*whZ8SS$BTl{9&#iN
z*B+*OdorpWNQY;qc<tiu(gn8B-;H6plcaB<?nsk=@Pw=6_}@P<@<edq^fyB84L9P2
z7Jx5kVFG)>b$(8iG+*d1B}baikNR~UESS4hL+&{(4@uM~a(>HHk=}|E*Bq4zSYig^
zyYxQqD54jno7!$gJCp2S@bwI<k&?U;B!c=KQG1ksz)&^lf-LlIKL%QFG+sU&hU;#G
zx78u>$4Ki<QvPKS67K~k>pi6V8ySQj+ibj>Hrg7p)Poo)phy`ueA?^8{`GJ<D#HE9
zdh5e3;A89S9)%xc;P2g>$Fw=IJN0L2{ii+MxnF4bF+LE%8ds?m=(0E;-QSID!dTB{
znkQ?1T!vz9G8uU$DrG7m@b`oj@U=a(QGy^`u$d35D>I9(7-(2}>o(!$7mu7fHmwGT
z_aBaA6e|PUhH-R#mlf3Q>7yj&QWutY#PLcrUd9j7kSE{cB=LNM#v~{RO7og$g7tj{
zXpQ$)<h%BJ%N{gI)GxUVRSX8j8p})KfBK!oInFhsk~aE!0w|t795|#hXApW4s8i@c
z8}pq{p3f#2TvAX^Pc^IW!4u~I#^eP(LZUZbgzW|*4QMNZO{5ioG(hOZ2h?K|JF@=G
zzs5FoU(wJ&(w&fFgWNy579f0+zX$$CqxJu^_wI2qb#M4^2_X!k9GeK~I0zj~4MHfA
zP>}{bJ>>~W(rIRPN<xQ;bfTI_LMaJJHIt4yPA5{)=rB#^X=bXKJ%_#DHF=)z`}_a>
zy#M{&KFz0Q&z`;4y4QVQ*LAPGRvoR67xX8xYCtV&@`eFHcFdx!k2(G6OP|+4I}IpE
zI>s~ZyDi3tY>nYP_6bfPRQC!0Qx?(;<Pn%UWP%>29Uy$)fhdA>a7Mg)D&Y5FD`G+)
zFVsHi6`FiU(dANHG|!k?_N@7nGs(#1z3I8P9%E6KbxP~fgc!o53KfUzU-fP%m^8=T
z6#>kDYHsf%b>0c^NKVfYy2r2p*Ah8<$<}{@Gv^w8Qdn^O!aVyuJ@b^D4nJ2lKk`L0
zWS{$idDJZIKKw>3o^(6m-Iv&7uxga(5MUrTzABeSC=Z$D+_?|X)Gb?87sA@f=T?T8
zJ!fjvH3EBhTEy*#fY7z{)aJp72<)ZbIxyhM#hdQpZs`*NU+Def7fTbk!QcOt4BqFQ
zDjJ<G_$7R*DkQj8YyE=*l#3eypeT>_f5_`89EMyl<W6A@e*gC4OyA)$7a$-c$d%ca
z&GY0U;uEq`aNtv<Os*cDFaABHp3Cq+AAAIAA~kzTnZk4#BQT1luTM!_Up@_p_urEV
z8;{*-`8}&i6|+uW5L%&J;{RH<J{c~zg5!z2GCi;=`#VxuaR14tH_?PTMPl5XK%O1E
z30rtXnGkpOU7ADcUFEx>43E-9i?*)ac~^T?M%wb|S>IH2*b$psl><hsLb0_m+YUt6
zQEKLBFY!_<n=C){z!!e5XoaUzc3g9^xITHi;JJPG{0N5sOX+Q+L-9Aje%&1DdSXPt
z#PMWlQ<l_e@r{QpAP}t2slh;^?`&%|o^JTHhcx(AWRsrD(4%S_9AR>Xb48KSsT>|<
z(0ti3ZbJ8Z2AU$FWv6du$c1mY4l73ovDK_$Fb`x#`VMNWll<mspINz^HZ3g(kY!$~
zPif!KlrC@b03anKR1?3wxYfDexl|qt2Ca5#^F&!BgQ`(?fBCyYMkjZU#nhTl0re@e
zrU0pIa3v%tY9HAr6ILf$ehTY;M`@E_%kUGfqp4lQlJK@H8!e63&@eM3yHY8Aesj;P
zUi+}lF=s<tL-is5+p&6q{4D>U&ZrN3Pw5cO3~c;RdG>1B8Qzi1Pd1oehcxxPH!cjl
zw>{M;8w+&T6L>?O?iNGBMy0@D|3_fCrW}8}OQLNRs%BurjnlESoXaZQkDsY=NIj`j
zse;E}`=V&dn3y~FL9X<*O|42!HnfrGw$$KV8i@Q6SZ~n^HJry+tiOVT+j0154gY|$
zi`mK4z?uYnUGAhI*%58iZ6`3GtI$v6qvXa=nas8bb=W<Gw1bvmbY2M)H(u={Mis~d
zYHR>?IJBBIsjC5HEqdzWk92871j*Y=Z1X|&f1qnv@0(ws^w+p?^Sx>H3D4qJA5UKB
zRTw&MHYK*oFq)wZyVHfY{VG5d$9{<W2VmmJ9Q4BCNJ`NM#$eGUR2|Q_Cnks{mJFOb
z&3fURxF>G%;bF%eV5v_$sRzI_*pvn;_R>A^xvB@#R_(WV|8>C5>DnoZv%fCu+5EP|
z9koWJw8}Zl)ogioqyszqq3xz16l~XX*gntwKvv9&$B&-xQ^~!*&+&dWW*wfVv|uEB
z<FN&y2l?Ne4$%h><(dpus#E&cXr|t%IGozj4&MqtNLqdx*>8)dXjFz3oLXOMH>4<4
zxODQ%BMXi)G;m;bzq>5LB9nRPc5a<EP4?9x;K3rt8KOnO$&gc5LOPQIpKL1bqUDV-
z{!rW?Dg?l5746%goF4%uFq7^Zyg?a3^u%JXU?G#xc}2-}?og#%F5rvP|I}!PVE0^^
zGLvDB{%x~qLfR$pVluj1M~Akf{j$Go+C34aMLxI0BHZyelaFm*UH;UhRIx3(O}bl%
zZ36Q#6u~^fwG>`xwD&w}OJDe%w-e8_v&G?{bk($FG5$hNK8by<_-;84zP$;no;oY6
z(sKJoD|orA$uLq|>CpwA!<XAun<*MPN^@$CD6dVMJaOB(cOt@tRGNOgMHOeUWk=sV
zrOX@Ry_HmxJ$yEk|79qA8TMM?LrtQ$FI=HRHzdOG;MuA<rCAFn;uc<esAL<bM0pl^
z$Lif-6$5H54!)(U7+B+cci5UP`nUPQ|6{2b`6!nU?HMs82x(4bTuKcafgOTI^-I!J
z2MQiLY)ze&AP=K=_RYehSKC#G=TYTSmXYv}5;!Al&nVoTeVoD7XG{HSB!q9D?2hc!
zPKDlAe@lfLe!|*50~9gAV|F1DnuQ{d!&R-io(7l={Z5A+{Lf05FnJbc4^yd4HW7>q
z#?l|^CG4oU;P`Bp4PWWoYSc1!8vA)+liroa2)>HJ_PeppH8G@#-lg3wwu#DTc#F7$
z3Inlobc+{cU=<upbfULnF0|M=N`pPTVMWo?3NT6T+%b?3DW4QBO=^Y2F61pUAG+6S
ziKp}yZa!#5znfdDkixfJOQl?H0Z-n1yI)P~d&)hDd?uY*+ke<Xy1Xim<k6gat(KT}
zW)hTK3Y>3*lK2ZSX-`O%PLv|VX(r3R-c>VZ`^~T^`h<2HrPZa^p-8X&|1B4=K|*IW
z@15%r{Xk+naeD<vuKvAGY6CupN1t6uJ37;ropUsW*tSLTzqJ*g&5@JSp#NB^k&D8P
z>iRxq@L?-ZR{*@W)faiiI|}@Epz)}Tm{uK0o3B)ejF5wsgKa8R8+rH3=R6|Rgo6j~
zdgfC2v}(b9RgHjuih6X;&f=-ByCD4(+`9^GpjzSYhHCGj97^XejVHCl9+cjTT^MKi
zZo@w}GzZfAOyX6;HuTlv!xA2TSx*eRqZW4ej>+-`d(w+*G3Lc-32T8wG8YkTqD?k!
zu=KxH8{T_ulTzs7-uYTenhTfAC1jI$cZi0B3v7FM{z*NriY%#acv(}dZ3b9JVbzxh
z$BIWlRWPu5_Uq@>V^$XY7+2UM`yBdQeS|e-%Z+C-^2TU?a?xwL{gv6|d6ZI_7nesB
zJ5dR*&H8x6A}UYR-UrQ;TUf(jVeu<}tpkWj{yP})-VZwu9ej`qrOo*xSlde$MXvDB
zUj8CVa}@T=P`;t)kNnbbeWuD}bds|;*s;$&aCKWF!{dqg6;zy`QU(1U86)5ir%&VR
zf?c>^R}B8sW^OKGd0}uu>3JN!>}wNZS{!hQ3hEo|6&4~zwrYVrYPI&#Mw&Fnz2_7q
zgZi?0R;0G-mU(*=H(4(@@ia|QJK?a~3p?jDYy6RiA<s>C?U*CQ6jIF>_ATt>v&vI}
z&VVCNC_Rcc(ah9JZrz~VJuAbm<7Br!?Nhxa+;HihcI0J^Jvp`e@$7K5;}$%S?JZ1d
z9+Eu5%IBn6&h4S}XG!CUL;7uh%W&DdbD%L{4VPH-##>_lO{H@I;75i((c^16$HF8)
zDZwNEP=lK19LItX;51YzZ#N(85%Pqn=oby78Vye{OMz$1Rg&u{*nif%!iY)`hZ1!;
z8hsNG?hJzISjIHb!?=lBP4jrxf)USr@+I}llqkPN#kaCA_hwLt1s+LE=BZ!#FnlIg
zNWH0YrgQ655WihQ@KEfHyTgjv>OHJgsu+lVQF?CIeJGtpGYMmB>%+efC9p%4et)ra
zzPZVo9ga2exTeTXxOUEh_YmjeME5VnEWxnT%c4b$+tG<vB2pHtpS>WaH)Bt4e?r+|
zZHDu5#dSl9{yOWfo=bYh+#%*IPiXz^&^@7LjGlj}ya=wt4+GEZD5?oRc;k~VT&wK;
z(2A}Q<`X#aFw?(G!H}J>5KL~XtHzb|fzq#X1tXT!W58%h05TtEMp9+vk(J-1i#H5a
z$ypO{M`6XElyg<MDit~@m=AoaZ*CNp?^q^a9wDSc4%~?Ujp%{IqvSii^5V&iTD{8k
zf8*SfE<1D%Iv?N06ouPwR~WJsTrOPgpkpJM?|~c$%rMYWtJ<4uBCS<1yzrs2g(2#D
z9d+<de|W*9O1rhi<ijL(L_z;yI$sNK7`%3mf7ln_cTh}p+A5tb_F8UdTwh>gLwc3B
z90yP3*ifFYl)b&L6&0ypsyH#*YIX05Dz+jH?pdc5dn1_A_L(GlpwWa$2g<+d+DlIz
zG|IcgPwzQUHYz3c#<63MZ69H4*Uw(zGONbkWqIQcU4n@mIV2qMo(TDPm|hD4I{$IM
z9wweJEwu9b4izCq$SH0Hi_jhU8VLJ+K-ISm(%;jSW??DQ<=&UD^7;rvO6v~W%4y<L
zahk<x?lNwSoGBg0oixUyJSla$Ob#kNR1`YrPOVz)5ZvxxK7x%ie2$qa7@YS@CwtfF
z&l|htmBZn%;X85>WgL<VtuspL%Jl)#njd>y4^kO*x^_sbf6#IPro>Yg$(UTz#^$gN
z?dY{iWG!{LvAp+|{|-d!*kseZyY8u_h|%^ry_Tz`wV121q(LBC%`a6iE)yNbL=Adx
zvL<<Nvc0stk>3OI>|aZ0vZcvuk1K8t4|<|Uh+MwEM?ohn<j5_z{nFU=d!x11&0YMd
z*UHS}`Q*gMGTk-a&m01q4>#AU_z$`r+H1x>N_vDj6?~OnS#a3Gd!pVg^7l&E)UZyS
z;C~l$NwF;WXEqKLYwt4InRu{P7xRhe>nZq2f9!X@wQuyRT)>O4T+DCanHb;o1-v$u
zYs1j><j!-2a$f&c#{9vfPWB)P1+M~WJ6x`#BS^xu>NWYx=NO6O@Ej8%qmT1qwcpn%
zIpgm<9%bzBrx50SOI~vQjAmFUv|*%IA-!|{vZyoQG=4KZQH)r8^lCEM2NaZ3jHkTQ
z9sX5>FmvSfj)x-=a)ICkR)I>WrZR?nnrsPum{kW~mh8<85hFNPQm1DLbl*zNJGhGR
z$%)>(KSG73k9D3aQrS+9CGnZ9V1ERMfJeLPEOUUO%J+f4#C}Zmn$Vha+h&etX3lJ#
z@O{Hpn{mEpD$||)59vH#YJ2VE0)@5IL0-fz$(QvJ@0(`bRvheiD5f~~1}D2yiaR!a
zJd!2d^CX~#szUf{FdXA$y_pyC8Su2)+gJ@<!i9`mwsH{xinl$JW{5o##{g3Q8RUn!
znj7hojlPafS8iwUxX{at*^^qo&+#Rk>FP1BZ@ED2J2SU)_6OBMRGA~!_{Qe=-m31S
z@;=K2eyUii=MuN9EoYhB3vK@-rLI0N@hm2x*6>{)fS*`-32;b0SE!GZ9AG&9XVy^k
zFEsn0+P5Gr28U(88+{Qq>?gBy?GcOfryo^D1TMaE6)k0pX>PjbVjc~pwMtdG6~^4-
zZSi|L^|7m4+*5&n-c=*`YZg7vWukE|H<}JvAZw{azI>W3zR7mA=6PL(Ifqw?>|QT8
znz&DQZTE?%3+ft<+`hB*l47%_rIJcdegwmi`j#@AcbkQJaqB<!oxZ9614EXesF1!R
zy=DErOQ$cr-B@??yn(LvUm-s$4h^mK*_+z%7;>t>2aVVG{W5y=MH)w9x9FZN9Cy27
z@g&H^{xOqllsDisY0A|=9QeN^4@v!-P?A*-n!hLm>L4Q8=vk?$E5Ba(K7+s^u(5i%
zx~Q3t9)I3M>CNYSCZ9V@Z9Vv!@S1QpC1%?r3hW0=cuAGb;t1P^sl%9yVF{+QuS``Z
zon2~`MR?74O~|`_Ks3(~vdZ9|{~-0bprpoTv02bQBr4~h^7q1Cg69&0glR9sn#a>j
zaU10Nvbgsm(u0>c3NBZTn}4=_XHJW^{``>nSeiUXhkL}r^DC{`0il_)eHK3_i3Kpn
z{M~d<|00s;OV_PgyD~ZD^LH2&2P?!lOYOo?Bzojy>Ig-Y<ddTIkb3%Ic;5BoF19WE
zdV|rbBJOK8DPXL%Ot0A?poeYq*LG-1YQdr82)dPBJ@{{uIQ*WXhfJwwziq(R$FBXG
zpbH9W%T^xlp&@QiCaVc$LwrUhjV8d3mL-0!?jqYnK4|y11kKmH1yO?{7?s44cs}(M
z6c5GC*cgjdOJF7kG7E&;oc@M|Wxwk`$(!idm6yotf9;I-J8`}^mW3kY;wM4=M`iBj
z=d;%3V;{r0V8@7&iK8fn!DLFa3DG@^)crNjFdy@sT=i?1D=EUQ-E3Fr>t85|!Yp9d
z2l0~}=%ng4)iL43`-unZW<xv2wsXpr=3yGw8sO9?6rD4z^q9@$K)ZU&<h```1j`ZA
z)BqdvrN!N<#rz_u)_pvcwBiO|lgPfH0e5qnOKN`6h!n1GZ2@!j#$%HuICkqXc=Q)L
zpPNdWj#%UFGsdUBau@kf`0A(sG!a^d_CpZ_W*>o<FDlxqpmK3DC8RiAnl8<f3hE2%
z7`FP^&|+gdnc_U<Q>ekYmq<2Jq4t`Q1&m4hEH0X{CqGPHaWQ0Qk8&EFk<5F?wIxHt
zlGb<1(sFAS?RTLNB$-BOxtB>5Rw$PFeH;y^nu>qCsIXvWq~QBe#Cx6~+o$nZ;}H62
z8w-8JG+jnEoc<A>K$l#hzS4%XAv5?q&2TV0SZ~Sfa<$Z;jll|kL(r<?lg!?*01as^
z;j=BC|M_6u+J5Uq*MXS5o=e^yxaXn;b*s1s(dq@4m-QX4l`^+oqDfNt9bBUZpB~|=
zp*n#EEPx4TEI5aMayHUTVVV#rN83J4aw|eyrVB_{5Kdc|6s@jeNx4Hle1^6f%xdNI
zL$>;S2F#*nqxT7!Oamm@aXEC120SWn3$h%#Phb)TBv+=)S;Pp2d%X^l3f^WW+DpyB
z+kYGmy{iCCIKj$+Ra0Nl?pOwAAJW|sPp$~6TGp4OG|J~-eI}&Hws0o*Gsh$s>|}h{
zLfC8Wzxdst#3m?fE$0`<C{V*4()HFN9xhtOpEY3K#Ca?jRor5@!n)Kkl(zHU&fwdL
z?kRab^c%wFBNJN>+KsmAvIyD7Erd!sw36(A7i8wcb{^^2w7(7(vEm0%4#AUgLwI*x
zrfIdkg(t3tQN}X5^faO0RvHTpFZdCLmi<meGNC&9+if|*AJDX)xLj(BDAV-Vkbp6`
zhvyC-8of9`t7P@&7FxkpumjAg1+fxJ049{Vv6H?L<)(LMA|H$#xfRLXMbqlG7`cuh
zAGo#*M0S|9FFC*qIa(rV5n{jhSVm7ZHj~94&DiBC+p4EneUHUV^7VKwl)(Jl?-Uuw
zEv)}C(jgJ#FV0<QI*_b`UFA$d)7*|;tY5W+>|ucxw|@}dBa}R_UCtZz&^WnuQ01Y*
zbwYjN<~{X6+q5x~ez$d{0j85FgwMv5=`>CH$@Zi!wxsukJ{vpA>^H(3g_9c+WibXE
z`bJt%z(xoRx!NTDkc4C<9D;NEjf`pFN`7$c)P$u!wP3!+<r?(XUJ;s&4iTplP*d_H
zL>M5|11?cVIGyT#>=beeaYD#dHlvDNnhEcb*3;3ihN;+<C!v$jC<_aMFtA3)6S7mX
zFx$!9o>T5Krag6g=Yc6dV8SG$QxFQXu-PiC$Ee0tUcBBf335^6z-|U%(Ar&A<796E
zW-(vH%Wy$25MK5AqTFeXM@)^;_MqN+D<(IfJD>l{h|Q89&u#u9uktP@9;F}O%fFQr
zu5gr7vzEkvH<xF%3W<yItZx-=x;9t60)O36P`Q2DC3<|!=FIX6#&Qthouo8;3|Cy?
zU7N}Eugzqd)@D*|22)9mJ#Z$8=2mN`;W@c}8aT0lSmT$nGbf=Rt8Yu1&?ZP*3E3MY
z<=B2Go=}Pl2}9!4NH<ZgnWr<%s^)Qn2j^=~=*|ov#{)cND41{n3nDl6#uw3%2+DE@
zvt;o*WI6V2k1o0e^`Dd}{<5?6sAJRudwZXclPdC_gNT(cEt0Mwo~Im+fVQ9(drV$R
zgN8!Hx2|401zPzeq<?lo$gDVVrEkz;DoOH9tdcF0ecZ0mgQ1cf_#HuUXu!ZEinut#
zqL5z*FIE7$8LMx6G0G?{OB(t?hi2iE%JuQiT{42K5BIF^$}9;dcoGi=Pw$>r=p52$
zzIC97{AyBrrRt5iN}rV~FC~{vsXGq^jJh`ZIJz!=%qQ108vXr?T<hLVzusYqr7%-T
z^#Q=gdpp=g2?1Pp3obsK&cEUZ*FKgc3n;%vWbf#8?u-QH1L=f=-{n?D>&HB8kAgLv
zcMO^g=eQTl@<oX2o;9GUyE0gL4Iq$Du7_Dq^bC--TWuKgbtC2Jtv4($UCfDG$0o9g
zKgy+7Lzu~wli1g?LEEJ8`wW%vb;f^J8lti2H_ClNC6?^YD-fNP8h}O)L`qcAA428|
z?S}OY*jh%raG|TmRJCLTBW?8T!;pqG1L`Vh0aebOW6dQG#KYZ%R}mv&Kn^LgBWM#V
zVD^`4HujQ<G23Tu!uChGs4r%uh}8JtQDvfl{-_wntq@x#V#duyCmc^FDfmL$viAnp
zn!WK!{z5d~JNl$us<f1-Aj#<uG7Q3l?$Vh{{;Nj#kR<AH!xYzAP5;c&XP&Z<+$<h9
zCj@CS-<4>yd>(8IFokM4HUUG>b5f%<MU)9yE9Z=2*VG_tbI(6O3!@2K9{M{8Z9oV9
z4O^87e3l!gV(v?j@08XB*D{U~Og%;X+zjl%vX4|c;cYr_DWBmZESeZO4F+(S?H1H|
z+XvufMbe;-eN$Fb;jCA$8SzJ#XuHb_+fPOcS%eYWo81X~FLaKINTY{a=rD!#E2#@-
z3C#oqey};R-Y<da^7Txa^g;rI#vymdGLyRX$f6kA5xl!g4dDfZ32<dti67RQ&)a3x
z9a}p8nI~Og+N4?ozO;TiK+To?Bf&Pc#4W|>1xY+NUWcgBcpmEsFqrZ6s9Vy>B!*C$
zx{W(^gznyTUE_Y0<Mk+BH{)S$&!|v*1|GGh52BGUDYFL}zHb(bq*(kld<0~=HM^9a
z;>1R0zw-&1NAI<^rFSsVWJfmC6mR5hh~y?OwrPrzdLaovKZ`;S=s(#y-l+znS}mh7
zdEf3N)ArE>D8(@j`t@_THN1|W%g-Ist0mOQjgdt-oxG9wcKq!d<Rvf@pJ4z^t(-5|
z$v%)?{b-_33B4v5{ZP-X{wL&twXK2Or)1?nHPC)-hwrFQ^bWV}#y<bB7zQKB;mPf_
zoMN3&I`hU2T7NUT{ju?c+dkZE#SI$|O_J}?=M8LxS4p;CT2HG<s7$oLMKb0+-X7g^
zrGao~B4c|liRrfZVtwZ$Eu2rIo)sJ!2i>6E6=>A3XheAJ?~Yo(=$i=y8qN1G9GRPn
zJ|Io|6W%wKaBPu2A?9We<Fbl=+t4It$FY{~Wbb5|!G1nLC0vT$KT!3x*OtezL&Cl-
z2Vk;)-Ep|OcC3yOZy{<Wd14bFgZ;G(wZ$`IZy(ILuO)}oY!(3;)b{!<!JU-Z0n5Am
zRGcMNVL8eQN59a%(cB{yRC#G(uL<glu};u^1MLXm!|T!PrO@>xt0}7VeGrnq@XQ#*
zU!zs1{5@(#B(}yXn)5|Phc=xdtqVGSN0JNjwqMeyWZ7e5@upe0+X{<?>*qxd*={Az
zub#ZpAo)8r!hDHj%Z7o7XI>Y+sGR1V)Sb44b-xs2{Y{iEYEayagb(;+1Z321t|8hi
z$iK;jOWgejKu8Z)hWQN~^axySAiWLoxg*#gq<#qoo#gt7X_GjaHJZx7Rx)~Ng=;@<
z6R>(A*cN8_$}$_yYkl9@uTCAtevcqj(uP?Ai5w?{)8Ee^O&@mhkNq<=1rU{QTv}w(
zH!J`Qw5*&L^c=*+F~-_*V5vaJsP8i_p4>!CfwZVW{!ye}P3E2mdQrHniR2=Ve)EAN
z@WeSVa~FCWA#?{i$Di>tLD*knbY~@(&o10Xn!<REX0csrRNv2E;zReCHko@F9#S#G
z7k`CM{0{x?jqKv}DdQA0n44D$xGK$NemA@-R#MnOFZfQN({@FZB`UNakY!DrYaCv9
zN(4pYWL>Cnx7BWTD|!kRrV`+Gu{zQ|Ep(CDqQ}h3jSqFO?nNXv{S9?s^V<B+NYBEy
zh_G$GEX?dW4h_u}`G`AusgHda$RFgGnQH(bqD+_)RS;-bUCC;7DJ@$$>lho$#t!ah
zq465$Rgbg4DmspLvuWbRWY3vUO-8Wq|G{-00<BryHix_=@C4#pwQ6*~E|U7`?)*2O
z(=BA<qB=hoaSFxPS;N{RP2U8lFp(jYIZ2k`Mrd6Zsxj?x1a_i&owW+VJ)nC9lDeL$
zg>nU-lvgpLKJ{?-6*tX~#@#i&8{`&!I;{la_&XJAihdXqU*6Z`p1jBbf#_!}Zh=Ol
z1Mde_zs8pxg!(Wz<s%gFk~bsWjKumW9UA?89VZ)>3gl$a)HR0{%OF7fy;to-ILUFe
zl`)D#1$fFJO8~f9FE`10>E1emOCt*6aY{3V@|<Wa3wVUu%4$L2{)WAw_*iTw%O^Fv
zKm!n;K!zOY`_R1N>c@J>PZZfDlebHjY&n#moHT8R*dbpEo`Hz2>^Kkn<}_ob0xOcz
zb7)X#+s;5!;mQiC#g2AjE#bE@l#a<mU1a3};7Z3caj@jUIl3fQO7LuoBMl6E=IGH_
zwmP)$Q##nB829F5-Cezj<%B}k2iEuN>8z*$+?+yV?$QbJ`jF^t>07{4ASz)1viM>R
zTt&{iKNs+lg$vs~k_2nrHn14GLO2gd!egcYE_Jl;{iR09Zl*CAb)3tC4@loXo$Srx
za2Bq3WKp3pnBLErl-3Y@SN;TBV7D+vxpb5q0f?JE?IB8VHMIB+GPVxl%!Xi=G}ly`
z+s^{6h~d2u1y%j*<=#60T*C@hUV%QNL;7p~L}rI^N2KK1gb+hU&0NFsRG~}JZ+<1Z
z$3wIq!O5SRaHu}ocBgGoC!fmDLbu)=UPG?NPRZ7o-*VwIM1Q$;3kZYjJ)3f%`d<;y
zXwXJl3C^6c>)Ixk<RIPu`sU*Cd^e=vfic>3^}K4mY~C^h+&wk7iHvP`#(9bsIIr<=
zT7dA~V$*6Vp1QLtTPvtG6LXXTC)($Yw(i>!5_Bnvp-pr5UFVVmOS8SlNaKZp$T=iW
z_KV-~L^@%MrgmKdS?dVmJ`~1Nfs#<N23p!*!lRae`?lp}id^ecOnt#?s0;7{jU_mc
zs|XVhVy?jL?=YmwdTn9xMHHEJCl<^(c}kQziTWY+#e>#}aB??{-SG@8K$kDI<Z%M<
z28bcP9^|rA6j5P0A>Y7^D%&(5qtG0nmANdi0T*>iz2vgFR~4bLn>8{CWm`H-Go|T5
zrUvLk|Ln7uGfB)uKHG<T^sAq!r-2e4#M8(bzo))famR@6T%RRA6K@+=_gmjcHC<6{
zT3xJ3+(3FLoP~W~=SQy1j0t^2kkoc{>9CBv!lfc_TRNWFpWhzT!8SvV@yRbbC~Sdk
z;dCq`<_B2k(c%P(_xad^MKoiOlC|ixD0Hm262%=2LX0G5{hvca*t~H&BuSR?aUTw#
zJp)AlNvT?Wocr#P+)7yL&hzWBlgguyJgvuLw;ow(JwavD+_TdhW=2fm<!1m1bvbV9
z8m6+f-sH!BNG7D@f{ZR2pHhYnDK;b;OIpXHl(CmkuQ+B2piI7lsj|GP{I?Vw>h_EC
zt4oLWAtkv;n`q-C<YK0cD0PWzB+(UQMYYmE9`S%e>YI`3wy1Emi|ejqNyU2_|5`NX
zd;_!~%whyJH6%6~qK==(4g|2D@J}s)9~~{Kv#NT;>`v$}XO?K8Xmo_pO_**ilIL^u
z4WDZc@iD(K3(!fNWVi2mAq3Ni0>GSli9CihQ|blVT{pb`gKrzN1-Q9|502l&&ut~7
ztzpyHv>#zCNR044a&18d`kz(Uy#y&bUo6Lb?o%9=RDvCTL99VV>!v-2anhQKrA3_(
zdhF*djY@2zx6DPxl}%hxxJZSggrzR2UEil%hi}r}v!ZX*vc{_@KVxuC$CFaA*$OQ8
zPqUJJr}-g6<x0)rKSn6+=h90aa!a-_WznV%`^K$mIjJsMEO-A1l7D45SkDp3jp3GP
zqwX>ll-Z6>Vn&#VfB@6X5e`7Kv=nFs5Fjd9eCi)#D@PW}vBz)-t1E}jNU|*9ND_hs
zK)nEH&aWz24)aXSrtbSY7Bvk(4#jhoZ&b5#o5Ck>RX7OkU3~>8jiDNef_nP&3&h0Y
z2aUo=lF5Rm!=tb5We8+G7_YcoMqhUa*S{S?mROP*nW)zTT&_C+wRk)}LHL5yD7x9-
zKrUJ%@wWU)+aoS3;h1t)ca0V`t|hxHgl<`4!}cET=Vde>K|Vy|1<wAz-J&DZ$6X{o
zSWoBN2h4ybLz-!0f5-q_`&T#OoQY=fg&Z13&j{|xHMEf(Lf<i@%v>0idhsxX+Brwj
z*GxD0aWt75#CSN99z;tGJ;dtd#e!*$nGA>)+GzghK(jDBXbM7%(FiO%<OTGNv)v2I
zX=Q@+0vd-%+RNi0LbMlW_ZNI*@-M1Z<Wp0x`U~<C6G5Pri#(!`UM39-O-xlBF(g~~
zOcVRi9(O|C7qYMyH<Da4r$O+m4sUTsFkmJN;#HS?tj%;|Q5L&;G#0tPA*Iv0M~y#c
z)Sc~ND?GbjN*3(9ww&IS9`wYuA>pvf7>-F6!0cNrqnH|ag3C|hKn~^D`%U}0Rjwzw
zV=Fq6Uv1dR{n=d7Y)C}fzdAH(!9QCcVYulW)B{efF2ByG0eB4K={VUjO_Q-vK3ezv
ze@=nPr{*8LIM=ye{jI8$4T#~+sc_CZt2h+9vLED97+d`1*)VPD*Mc2#^4;ggE11D&
z`BDkYg(e~pBtz`VK7PK88{i_4WCAT2n47Uw+CGdym6bP4bTFyO1qgC0DDVnW*Wty3
zfIdS6y9FkE8s#Mf^e*1cFo5^4rsq~i{ZeWrI8Qj8d<O6PQKypBZ*k`PW(Ha0#M$)N
zW^1OtIMs-gC|tN=FI;aq-o?%zcM+Kg4fy6Pqi$X3A|S-JGkrfIgOG%m-c@gVnMY<5
z*Ps!M%<(j<f75`VQ=BD=XvdR103z7UfIP55I^hwN{4sQ09Q5d4%SJuyXopl*Sxv;o
z@WQ@(byd;?LQJpL)r&iL?=Y$Vp2{Bd`-Ii}n7nwpcZF$R#J2vd5vYGJH;$U?gX0u0
zM;*h_4@|D?Ue_t<shl?}uX{nu=hxD%wQjygKkl+HSQ6qtSG)x0ys3v>Z?0QTA2)q3
zK<SED_uuD*B|vifsnxGhHQ~>FpS|~eZ9g(fcu1gC=Ku1ur8A#Bq&y<^(+A*K)RYTk
z6p;TXpTLX}Q7qbkQVt?^|GMNiEC<pIriFbykX>+}0i-zYB_OeVD!fZX5PQH~|FZ)*
z9l~5XbP7r?JT=w+E_|3d%%r{~n7bl|GA+JsqyGpOOeSdmP7r8x2p)@D&HYtHQE$FZ
z!RE@TUe-THX$R>W$Iq2&4^AEpyh1)$Rg8~|OQ=YeEL!kBMljL0Wj$EiQ8$D%+J|R%
zYS<fJtk*ysXS={03U|zru1Av~E8yBgh@=|={JeuDk;%&$L?DzUfDVaKO+a%tEDPGY
zOXHWebK;XJE7*O;I6*+<a{Y5#tK1_PY^=Bn1Z_irW&p>Wm<KdAa7TYk=1j1&z2<bb
z<w$3wX*@nxhJ>@+xMyWJ=@Mb(ZhJas3oTDbxj`dCt}Lt-{s-+4eTmERQlo*&iIC%~
z+sPKhY=LrJRCu1xXdec*qcdHyUw77<{X{r9mn6(821Bx=E}7qR3i;`0!y4QKCkyQO
zB`Ee;ETp*v$xL~vZvO|-=1XrQK4Rc{kkHfpJxr5Q1MUYtK+OfYGa$uY`bB#+A%an-
zy#!ZFXyncwoTr|+#Q0OwiX7p0^#Aa?cqrTPIP&3|Ny~kHu9DJd2>#WK)D;BZ2E#9x
z@doL~D834_*nsqDYo2kfzS3Kr%z)g-UITIP;*~1wfTN(-<v~xtgzX6n5^&2>tZysf
z>h}cALA(mCETBpN_;E{3o#VpfzD+)hJ?V~TEfi$JPbPKb+Vk}q{+d-e0mMjCqEjSC
z_745P|CPCrosSh?hLbU%+OiI@26yYG-oYALV^I#uE%B*xP58$?ISfPYy33hO<f2Me
z{#O*FP`S)9`+Z)o7ig6tI`M)!8=L7R*TrEM>+CJ4BVF-{0*2wr6xFPzKZ+0NY9~Ql
zNnhy!w@(TR%hC?--+NN=$6h(gE!3HgMQczs=sttnK)9<zTnh2*8LQC-^_!^Gu`lqz
z^2i~}_usv083q73lRgpPRnzKKiRyD+=@)OE$KLho<a|=i#OHyv3&!fyvI3^#%J<L3
zj~aC(^I;Y3f!zM07Z)kFN&Op9g&xgo>U|T0l+~Z3kcgw=g10pY<=4MurSd|Jlb{f1
z1MbGt?i8RJr;q^Pmt4LV=_1gElmi~J(!Z`}iNVD2EIGxmdj%>;c%N+fkQ-(i6}&{u
z$d*$Mz}nl}Xq}z8V(jn+!^WhYEiR}gCN)dMeJpv!fKvvs)?0Y2{TtsFuT?U}E#tej
z2-#oCix>3#F#7Eqt#%PN(wL8Z=ZhQJqT=e2z;tvuJAKXK$WHD9#Qm$VFqSc~SPq5$
zRC+2|T0^1-e)8}a2N?#Hs(nrP61uj#n00fY1;<9&EWJjXdbzCn$D)5<k6~>DLLs%+
z3$O?WFdF^9tvO6(`WB33Z^~j|Q6x{U<!`0E`0!nnEe>*<Q1Ka^N}u$qx^)S~t8Uly
z|9eqovgvK1{A!zXn^3Kqa74=s>3u;y{C=8a-OGj@%mf)-4$U1|Atssc-eNC~TAUpi
z1DUVdU!%tFXhuF;pG^*v1~HjNGabST=O(F%?+USiYbSs@A53Gr1LZ3B8)pErwTZMo
zuVt9qKma+9mL;80GZ91RjMUuqC{HL2-l$f|%D@6nO=Q#R)M^<-p)6mV4$PJ8n#zFw
ztMbjd1-i)A^XOT@Wqu~h4!Vx*u5+&wN;@n9Dc@Zr5-7DNSF;rEI#kWEe(WlasYxfF
zX+OErf8+O`eT5U-!drg$EtIRu^IJwk2lJr>E*_s<SnZSKkSgSe4EQjy=`HHekJbw0
z%D9~WhFTF+^9w|%lzTs{k`d>y93|2~5B-;>{`A*F>sUe@ggaNdc_IqPlX6x2c#u5m
z@y-<ic&fTC`0l#lP}7fjx2J}=!(Nm7er-W@0mwx(rj(IYWxJ+2#J&R!LdqtzpfNB?
zR&K;@bw372Ut;@EJ*!*B=yBCXdnEo1NDo1Az$*hT0~@rSA-+7Kh$SySdhuA;wT8SS
zZ!W%TTy5~4s*QT~`@|pS7aKG4ez94%!F^!XC5la^HpzlIkbDM!THsDIv3Ukawxd@u
z4l$fvQBLw0wwCiUi#>zNvZTK4#2L;J!4TaVS`(@DQLc1UED<casNA2Z_E>xZjHp{P
zhYq<W;(#Rm$Nb0FM5TX}<JyYj0)va%{~im}*_mVcxnXmV<+`Xd%*k0()i-3@v|iS*
zsCXMC7v&{0o~^je=)Ir;);*m4c6)WkhHctUZI+Qm3y)1$nrwzoS_};NIBjflKt^q=
z=W;iSHgdTSNO7s=qrNs!ZL+LCR~X{D2vxehcktdL#NaFwi6`#GrVXK!#`WpDqlav1
zMiLO{G2Yuig<c2IxU+R0L1zU}1}ka-BFB&v)lVWHu3Nv|%6d-a-mrlNlJklGY7*yW
zw7|Xn_<JTgORfKCr>_YUbDWRMRc7z9K$Vj7joZjM6twVKqT@-(BA4*xV5ek|nJ!Wy
z`0txs*X4j-T}Ys8t<E4k@q~`jl%)<|-486?GQ2WtUh!YmR>JRom-TQ4`|BR}7vJ*A
z37R{m1$Qfxd@$}uGw<0{>F#v(|Fm_sBv~;)P1etXP@oah&0L=#DIdeu$;X%!7ytGv
zC}ip&uT|LCx?Pb^Bjq7t{2Nxzn?wt6(wFpfpo@3?yl--1=4Nhf(%;T$J2;t=J#0CY
z0Gv4vO4)VGJkhT+9!eGs?v{T%(xBpjTr%e4^e@=K4h-mk^5g?5-wSav>>I8p{7^c_
z#%6!fW8!J%jde!tadWUE{SZ2v_ZwZ!3t7LSjDBBnv#uh3i~fRb?c7aejakl->sQ>2
z`p!H@zq5^<k3v+H(8Q5i-cKEBp{d0k91xKM=uFyw_|XSVM9p1mHN2+1d7O#``p;JZ
z2+17FlDe{!<sSCOn0=OTdy@Sq{M}+R>wL853Ap;+LAc#Vq6<b>?~^|eMgEf`5IG_N
z1CN5}h18mNf51s&U}!g3JA@Z0Iy|^Zq8tHBd`B<?1hJ6Ytty;qZ@qd0k||M@Cp$9v
zJieXoz)7>v!Xud*0+*9w1U`(~ODeSMLzdh)u4xfzJR#`mCezI0_CN)<F>f}6#{IAy
z?*_sRUT{BdBMDVy{DH;b@dvc1_ikS3N0HCK?271BgtjOD3*SP%()YC$7zvxD60wRP
zpCux<dMz-^fp9lAR$hjb#yiWLWaa{^zJn~R7-<rQ(+QKZ9+nK*6Fu=*Xd9)7d&4WA
zP87doK|X5lAvEAH()o=EEmn+mj$L(TU|`dhoNcL+mHBt}weE}coBmZ$FyVi+gFohO
z#?)wkh(+wNFQR`i+Fu8~pQs;saULj;D}x#zB_J8lzwf+bs-y-y^;$9_L3HG*@(YWX
zdJchoj}7oH^$jB{y9bBY4rq1)!TE`Jt8X1r5>Ixq7xp_;ukAV(Fhmp^PhA+kD+_w_
z*-C|3++_ZBC!iDT3}}ee7XC_Wn)_i?5aRAKxTZo(hia|S7RD>gaBW=or@vLG>OKOq
zJI0^Q!cFh)n)zOvU+<t?H(Gm}Is9!~^+$UbaRPFboM?6tb;%lUE;GX2z+8`_%d1ZR
z`&t7@@$V#c@!UK*m&R{lOnME6e!iYzoYPU(SNXFzx+f~9@xtO~M`ym*zF?H)*}>WA
zYfJI1&l>}6kRn!gLG@p)MfDZBLR~kWFC*&STjV1>l?6T6KN7ZnZ?-iuXDP@yoEBkj
z+D+5!Z3WCfIRrxtVKfU(EMD*l!4sjS2Uth3wP<Q0n4LgQ^wu!fLE_~(A3er64#YIM
zU7;TxIVH3Gep1#fq-7{eD|WqRsA-&6JO!ni*~tQi%j=DrWB*{4VdjwfYeLqH)ohZ;
zm`*_v$^oq|&^Gn3Qp?%;C)_ULRd$MFNbjXM@252*W`glNt_Fvla{YJMKvJa8JoSSh
zaNwsa%Ut4ALtyDS?gi6NY>R$eIe50ycax+ji8&UYxNV6w>4o4TyYosb6$_;ERWYN9
z8IWAgVC%ogZOXKHl9$^s3ikfdL6ZVQ={6}RpyYxEWKXZ@X;rJSg&J3~MW~Oy`kHGG
zTNw84rTfpFV0h#+G7Rh;D%>yrO{~AE-^%D^wXK<S8-672E50(Vyo{Om*yJDg2@lsw
z28~?!BRKXEl47Q`dHA9Dz!RClm88!bEL4po7eBftAuG@q-Dx=c%r-3NN<LV!FZx@B
z-R@AUHF}6_w61b)^7Bb<+p4X032nH}QLjsNZ7X<5i5=?s>*@%0==WHm(d_Pf!3Rrk
zqIK({!f#IZ+`?KI3d_nJT{vKdi;K9iNS(rQbl2do?d))0L=oXbP5oEa714$YXWogv
z%2({s^nRW~{$56ql~dlYzcFjwh3uTsU6rf*i-UUnqJ3e!{IrMB7Q*EIb^ElNmR1ed
z=GYI%3C6=%h3QWa#j?G<hjvS{-9R;BI(^Yo)xmkLzHS9N-!(bER2V+lbsW?8`50T^
z-cPlMmzdp(-g?^MX3B}&WIvp=aow*;P=Q}&1uo7&Qir_7UYVD&$7*GkUeVFKxu#qf
zQHtw13+B@lvOn;5jJaKS)BGE^^FF^Dc@)T&U*7oO3n=(nj7dwQg=YtGI$Y*qIXK&_
z%<){umHRut1_;ZzYe)D2i^oRGV?huYyk~a%SejPkx43heZF)18XJsr?x*b~a;O4Yb
z9rBJaYPIIyTfAzYz4uS!B`|$F+_9-3C|KII{ntKb87?!IxBD~eAo?j`%WR7|ig7OK
z*RpwA+Tcr?jWw1d^h+mwQUd&M21tJR{LISkkN8^V-TUSKr!e&TK=1m0z;urAW24C5
zSZ-kX%Ck?V2a-P@7@?2iB$f_s%Dx+cn7UW{CX`S0E4h$7LrEr|*HLZoj?Gv9W8r6x
z&M5F!aC@rPxNF}_ioB{^B!`>>UE<l~*>Auf<P~Mfd*v(7!Rs1qA?K{bk?^{E&{M$J
z@NUvWemUfJkxw_d#6jz~HF*k@v^FTj3s`fWaVs}3ybtN!jSskMPFF%x3<lVhKYmZK
zR(8T{!9ssu|Ju{>#Wfhn_hP#?=UW`$0hZc)g7Q4&vFT?iK=`&8JOVP`Ec{IxfXLfA
z#x7OA7THVpYTqZ+WsXjtkUD*~1JL;xPgwB`)DzB-4}jcPGj3Z)h#z~Y_iuuv$%~qr
zZ>z2Q_QS#!49N`^f~tcs$@3pQHcNYR5kUua`Iw#~9f5`W(rMg-rKQqwrB2a?Su6Es
ze>bvAC9i$LEsOURua-2E&H8i?eI+obvI~W;L7H}Oxcn`1lIvd7fKZOLv?WL}8aH9p
zD>b}|zo+F|op0soAbD9RGdE4H&55?nhQ4Gl5<fq~=^)7~&s~tK+W+W%?6j9lKxAFn
z4E40$c+Ois#jVcYl$?=w(mcr%z_3$DM$V7dw(?tQbpRC*d2ZkE8iC>V!21-+f?0E0
zZB2_UiBSrQD%)Ry)MWH&n|6cnPI_#DZ#zC>8=ikIC4m&90{PC>Z+oRjwyXYapX;%{
z9j_;=LjXHTaqh53{NTmpx#?17OOhVBxjMuiR0CH`ELr3G3N$&mUP^B*Z8<ctmW_&S
z-pXk)?}-CawlX4BE?UjfxR`G6hY-vh_z3!KT1%KrIxnwr$<22Ku0C%<l}*dcosG$B
zPKXDS8j*3Sp!QDlw=J{xo!2Zy-zkS5<!5QRC(&I|^$W4pvubzx5BcOB99cC@eYK{+
z=iC$b@_rZb#GY7fSVc8&KF|@*{jLR<KX}Wc4dJh3%07Fx7L;-R9D){YUHSwl$?D)3
z&;`v(<d-t`Ha_F`tV_5^_sbWKjt-V9A5HCLmhHb>YK;d%HTK5dHSZWZHls?0G1F!K
zVzuuT(H5kp{u)%@rebb!g^_$TslQ(@v3BJXW=@mo_1|fv{~t~h-F}&1sX2S!|NZ}e
iTj2kn7C<<t;%{81|Hew^A_eg8@PT9dbM~DN`F{W~ibco(

literal 0
HcmV?d00001

diff --git a/clang/docs/coro-generator-variables.png b/clang/docs/coro-generator-variables.png
new file mode 100644
index 0000000000000000000000000000000000000000..53a8c9f4d509b79f83cda29c24193041f26eb1d8
GIT binary patch
literal 774476
zcmeFYc|27A`#($+rO;-H3{g~=5VB4p%R4opLbf6yTe5F+DiviJqNp$xk$0BticFGy
z8T%Sz-`5$#tmoWk>izkA-;dw@_x*U>_x<>~oS8Fc&g*qv%k{cm&+B<z^Yrp1qdmJ0
z?c(6z*kfXBV9vq8+rh!HV+ZdJ_=<bSPYn)^-51>S^)H*~>q}m~=Y8AF!<mD__~{Ev
z9xIC`p>!K_DQ;eavwzK96inE6_Ag$A`x{51{xaV7XrI%$%Y_-yyYlS}KEMAY`RuS@
z;Y~}Sx|Qofd?!vWI@aV@?d0?uw5(Pqst1A$!YiO%BXA{6i=$(hX70xS_s+BVGH9Xw
zON2woNv%ixw;eglc{qskLiJh2X@m2RxvX{!;gbW^Ip2E}Xvmd<gOEe?%gZvt9QO_M
zQjaC^-ILPg<rtNTZzgd_{w3wDtBU^n=O?MY1U(+9eDjIj2ffTEP9Lo9a@j9j`<S|u
z<Ced!!%NQlhU!z+w1>Mt>PL2NlYM$w{=tBNBktF)rgKrV=RCMP$5Q=1=xQzVq{ZOf
zvi95|(L^Jx4qkAO;)$B>5(hURmBUcw)QsGYna@wI>R1Tv%ZL%3SvJ|6I(76~BsD@t
z;mAHmPmasB@=L{*j<>Up$k;n$hj^!KQ&he`3X0K|ec4ZF;H7ci6cjs`s@U|DceVVu
zsW8uR&l_NevX2Ck-%%&(lL6`b0Orm9q9!FS@~CODj|PkBTb)bb`2eb)570LHfOT_u
z7Q~{bIO*NCJ@WHP6c?#(=Uzegv*J>OxIK&ICdV7640jJS9#Cqwl9zn>RHg<z5uSV6
zPUt|lH(te3S5Pb9h(&AkT!ST9dGg$qFF(?Gso`L))n~7SNBdE4rJrN@5K-slcJW@R
zi%UHJoN9FtyJ$jYI$d${(Q><V1+TM$Y~Fc|bX%~YiR{EvKQmiXm($e{of&NAa{{$a
zlE=6vH6y~1ygaso99CyJ-4CVgF1lY)`l*_4G$-UC2cq%p4nfXH%<UTnjt*FEdnY|k
z>i#2p;|1c#tBVI8gf#+(f1J=*m(2dEd4B`c{qUDs&@LZEUM`I_`_60`(G5vX?FTVb
zgv9#IgWpB&-MG35g<|fr+V4v4A0^1`xP7aWNB-%9pB#?7N>Q%SVq>}$!V>%in9rJ%
z9>1I<<5o8xWj|8;A^l^|Ij!+_8J~9^QOQnvT0H^_1oRM=R+INO>*NYHRNwz&jEWD}
z*!{}y!DlLvse=cLeqr!0tZ(k=1Z}uc+V3!0&-QQsusefl=J6!1=X3Yn!&l5^=nc$|
zVHd?()dN4f%rSo<ija>y2QFXm*c-s16`A%^@y10fof|hB3U}mg*ZTWs#K{#C;xn-n
zg)9Qj<D-gSTNqH(Y}%Z5Csi}8`XilW&`nB0N(UDCeKk!@c-1iR{jmPe2@~!(X{+{2
z?fD#XTZrD-<%@pA>W8Ons(r}-&v!<UxA$tos_mQ)$L5z>^pEk?)IDNe3U@eCQSz4?
zNms@5s|3fABlwBB;Wu`-KG+k}AX-rq-Y&rX@Uq~}iL?7sc)t-IE!ytLdid&wgxt2H
zH#Drcnrn`IK6vH9$PLs-o{M8LXZRE!oE+Ox!Dor)3F7=(qqQcgC%EsNzDSg=<3p<>
zn^7u148!8j%7{PJdmev9hATmG*R5m9h~s+Kj!ZWSWgZ%g6gRwdL^xh;ik~DnZ(uKt
zyS)3Vk^g1so?VBZ7MbxMdQ$&xf5&dWUDxj`o=tx5?0WPoU!FeteR7AGs}_CRat%k;
z6D1{!R!%1;Tlsw#>bsOg)x=KL|5f-`u8{j90_{*=oc~RP&yn`;))ic~HBv#^ir=%x
zir0?@Y1lvRAJ1NQr|MUUL56Sl?%uh{joyYm6&J6U@|WC^q<v-k`S-Se56p>Ji!+u{
ze%facNL4(xw>U~A>Oz!QRA`iJl;_WepA!3?W}=H^gA<RKgqcg^M3uyqMEj-sz39=S
zpS>piCO)l6Z|YXjBU@v|qpLCqX@{dIX&}qhT>f}^lJT>6<+P%hqS!9uFQ)Tv!?Ny}
z<>id_oVg?9FHstE{eq7EH>2>3tgPEvGw+1*j$M26=~`Bq<#N@KekpcgXdz(1mU6)V
zG2NVUctN&j^&|dWr|Y%TsO!oK*s`Q+7q!lZo@KnP>A2f7=Ekc!<9fxl$IZ4k$RxW%
zVP2eW5W@K_33DIwYV?CiXGz!89g?bx`!!dIzN!y{8Fd|pJ{-(>@u}3nJ-csv-t3T!
z=gH->p6@odE$0nommpM5J<*705j-`cQl`{?O4Bb&vrkhyO+Ia3+K6VJ=E&EfuZmxH
ze(m-<>4#k?Q&Gw~nAp&IQ_*R~W%L&2(CyPPDyFAB);|y3w2b`LFP1->E!%anZ#W}R
ze>G9}0_rl#26&A528MI}`ra-%bmQ+D!w8ifcD4RbLiY&oMoLrzRPQGr-=*D*8Q)!-
z_e9}|j$yrFjiGslzhQRF*O*uJn0o8_PqA3hz}Kft!%Tl3`*bnx4YQ`p*~dyFSGmva
zT6xYpx2T*Se$j59=3P|Wi-U^K73<vIYRj4NAp4PfXWC{)gpLV0Dcru^W5v8}Qh3Ye
z_VxTyZ5P^T^U#UTJMXT3>-BZlaZmSu;2%rT5qlk5(01U>UvC!63~b6g%a(TxpA7hz
z_Q@{G?(v9oXwPVyy$8Og+LQa{qXyolsOG)p;ys7Y@YDEJ`2`NYiW`r697l;m8eNYu
zmH6T~^(BFrYMXX1wM=8=@CTg}epkjn)|(ZiEvIg8KiJ-H)NRNUo38Tp=Eh0kP~n6Y
z!{p)o-Tj66e0Fc_@L#`FHvPIYqB)}Q%VMOzlIXbjIb-0$z^o3h1SsLF{aRZ$<ZJLu
z@X8jmrKRTNuzPktS*g#-z^N=Yc*%37ZEJL5WTlp5LK$4@UU6S&*~;4ppK@7|-56fk
zig9Xbnpm2s+6W{i(-0(pK$&WvNay{&?IG7QuIpT-T$5ZaTnO%0+nu?4x&3)6cMy1W
z_nzb3$!ERyUbM^4{cC>*kGxxXS9}f~nG;#}<VB>;u5!NQ$LewC8=l13sczaoa(*Q9
zUU#1X>4@d|QEXA%>wW$*uSE_YwA;%%efvslYl?4S%z#OumJ_j#)}RnwDdxSuAj<pF
zTiHyRE3#GS-Nv@B26VlqDy%0JkBZ7XO+576q-kf<aAIgZPn?5FwRD~82f67P?!}$m
z$81wvPbs-fbee3KFim*Ocji3J(Xzm=Nz-01(@)zE(M!BIEk9pdJ2xk5tFUw+IQrVN
z^beug1*Smycv|6!43%#6;?u<!S`Q{iJ2*eS<=Gs9JN)XKbfvnik_%c<LtS<2vw_WX
z1r43SAW}(_r+KJbLP_|u*(6nqug*7%mhTzp>W#){a{(JR2h^V74hWFLNhe7t(~Bt~
zH(vLhZYp^>AJ8p~2CfNSS!>xH_T+{k*|hr=`L=Fq-K$b(HDxW&0J1n)gKx&Kbx*pD
zDpnI2I$H4f(&LkS#CCKlvEsXH&9CiPU>BH^Wt?Gr&CmT^|Bs8&Yv(7<=S3SK?aQe)
zg`cU8Pc2N`nv#z`JSq%GP<#go-!60~mnr(%l=*1*e7QwyJ6L`4?`wDR<gI8yy#rVM
zKUsf@w#3`1xV1etPwu+tp5uPdeYoXE%b`B&T$}2Z3&FQ(vAdvWI*WVyk@gTK;%;qk
zEJn;-UDWf$;D%dq5P4al#%AF}-iZt4`^sm$!@nxr9RA`kfWP9cK44$OTI6qv+x$D_
z@9y4}-d?Refr_8>+z+UxRhkFb4Ichn`L6F>K#Rmo`txr$3LIAv4D+p#Ux+e1je~J^
zS+6BxQ9z0RV84`q_u|(@mBnGP=ZA*)BPG1NHtuNq{akO1JJy}`EaR(D^5F~#o4dJN
zK{e~?`@8nr$*Nq^^-Avv+v=&X=tmpjyma;l%rFPv7H7Bnv>!&ThF8x&oPAs?JSe<~
z{9^aFyN{>F#;3)C$m=857ZocVyecMqJ`{SI4i|o^wnW?WRRnBmuF^k<?NuN0swY;3
zpPatN@El*B4ZOEfMOT2z34ybt(!Wkt7X;%s9*jpc6FmW9_evjT<L39;mqvxy6<r~1
z%(?+Mb)n&5c1iXJf@%6yR3K)wN3<t+8oLhoRmE3Re_h?k?!zrQ3{`1UlZg%rBa>gX
z?3?T<)%GMqs4cEs(fdv4`w)q$mhdfF_&N_u;3{GeK?whgxv@m<L1rSI9X#=B7(PZo
zfqHqlWpna61GD|qTkWAtuC)*zjuIITk+omOhYK_jn+Fa2vzGP8P&a!;iwY_@FY`h3
z8I@ziHU*ANFRt&q7P>7sqTX`~&T@8iX7DV2x6WVvl_PHuq&6VVQdLmj!+i5zGZ7ry
z>JXB<YWKZk(}5KhA@A)@K*8R{GiNK4+ZQi#oPf`HIk-6wacqarIN?K=Q|!OzhMaO7
zT)(ex<KT#N<KX`1o=fmM`!60o*k%5_b3K2;!2^HV2OmLixBcgC-j26i|2f~m_G~!x
zEc8uG;CBnBd(O^YzOLSWCWn;y;0rtN8e98va0ne`ADkxUGHY=C18!HW{H!jTYC3s)
zD&D;1?dYr+<aw7}4~KS;CVcAY>~~W#$kW5iS2IZG$nQHe;dAz7<Ppi=xA?j19I?81
zSyJEop0lKy;wi;bM|5{dN=j<qyLDUB+`#ak;_xS(Bd&gacQuj7z`#JoKov#rdoD;N
z4Gj(CDP^RxvI2aEf^V>w-_0NeFJI|DmHek317}~Sdv16A+`PRc+4bIZ^!E4DIdX*k
zp#T2;X{U3L+y6Yt%lDsd!5u`hzd<S~o<jb&Zn&s6`>N(;w;*Q^YXdh=c+B8u=qjn6
zR@VMq;Q#j3|2*<vMXmm)sIs!E+P{nb>#P5>=v80md-~p<@I(D{|7XJfDg5s*|0$@A
zWcU4FWASI8e_w@1T6dQ=^1rjDyG!M@I(sU`-3-uI;CEQf*neDXrIY*f4xeu)>ct~_
z0ysF%a+nzCT?yh``WBq)IeZoO#S>q@TpSzA`gMcRxnrN$KJJGG6(@E){}F$VTMp!C
zIw-ko+XG|q?R!%n@tPkod11Qiyo{M->=Eu~(g)<mcPVZ=V(>z8|En`52M?&#hZFHG
zD3p3>^|zJ4KJCTj@YkUPjjdmy)lzv{l}ZgM<nnrP-_~!lBQC{H&q;Fs@Bjaf!2fSY
zVC;nST9g|J`#Rmop6h|K!JRkG=20Qkd0A9&6n#A*xraxWmU981x&XP)oOQljtM^y-
zEdjqDgAyTU&l`O4i?@D-HoBk8&sIfEZ{lD<T<a`Kw?*ck?2px^5t-g#pfxf-aTC`L
zWb7GJKfZ{jWDM~+lh)3bwV+6C;wl2nk|reaqs_OhW`j6pPeLS*d3u#Ju{8nnnG^YW
zFVg2dCT!{;0j_?H0A2^W&!sd{#zTDUtdZzNB-{$SzUbd=DE2b7v&faA<wy$%Olc+t
zPQB3(dIeYQKp%#T#J=km;e*pEvaF2qg%=vj!L!cmvW}_~r+wwYhI?+jkIwQw`tBL?
z{c!)x?X%-gOjP~jA2ft?$0z%J8$5vpoy4XSx=<NX_KtsfQV9dM0P-OZmV*1^RQIEh
z??A(Z`iAUS5@hGGUEA;u^s2MN-qxb03~C)Hfl8o?hTQ(`&cn7^G~y<Lb$qiE90osm
zf0r;k0j6}`{T6fPAkaE+e<yktUw5t1?WCjA7gjmT5_p(a0x7!xx55pn-xc0}sv8i?
z=REvV5qbmp>)b@ZBV16;87h*)axtu>C{Ot9We})}&5`W>yg2t=;OgBf&6^yOs4_F2
z!kcEu)w$1YiE(;ZiUmc@s;|&O!e1_tzJaLUVfyy>=+Y43(?jJG(j@#cY8l@W$-k&-
z&U}!Zl0$W`=M?MAHn(6}Gd1#K2mgA8*8=>&IcDL~!Nq}XC^M$rZ^`pM)vN9Yo_eh*
z?j!^t-LuFIH-|eMpP{0E%gD*T&`+Tr>-|N^xk_JOtw&<2PP6T+;?ZsHJoC3}Z}?<s
z_r<pNV*0GrZWIdKYaHgsi}hIv;KE;+G-t>!jvt<mb#@+TC`&-G`0^*VU2>FdAIFc$
z2#s++ePd-Uq~GlHX1T{1V1Jk8DC?D}?dYacwOQO=q`V|uu)G!tJ^WO0KXKsc@#9#l
z>jDL=+~VBbj7E1YCS1!2#W9%x$G@7TU-s3yqZZ{)*6Nam_yH=i;u+tWr)eqANl_mt
zYeNLhmgdjcs<K)H_)inE{`WEej}Ds4QxqvfkoSS$8FlWv_L>SW<~r*AS{Khu%bwqH
zOKd(Rdyi<3URKWspoYuCFjoQVKk?U_1E9)2M}5s1w>D5ni%aYyB^hdh6c>M}7G5Ib
zC5xz+*Hhewp<%$~g2>h{CDaf<Z4)P2ut{V~9b!(lFNd)w2jgt7N5<2LqCACChsh<+
z4hIjtWuie@s5qCW|EFdfbiZ*z|EE*F0_Y9BQn>lcx1Q;$H8l*}t{MV{VG#!2;;gb)
z)HCTXg15IpGk}Zu&nJGZ-6sO3+#*OQ8t&5)Fmts7FZKD{_dPCG?bdf4Ipy^d_j(3u
z;^Z2VOYAz9kj=04endGZY_rhwj%K`b`T<>D>$NPz>D$}hz$@1F!q8*)zkm0w44M-#
z4wejm;@R^hIYrhJ+WvR~J5ex!4m-cT)pytfIu6=z?%q0h-v@mhl!wK(k8AOVzd2LY
z>0QBh?*oA1dKtrGUHv6{DeEx!`nRi@F-xhYTL<rUW(xraVNqCo;6JW;9DH5aVL7J$
z4F3!tqdwxIAPlrFp0~9^R!#`8_?FFJ%X9D?hip!*4CQvd;AI{G3iI$~QhfN7C4UY!
z+@znmAx6uZcxe=zB;}lmVix235V~B~8T%I~2QwTBF>N?_qT&MHUo3_0|2&tU^Jhvn
z9m0;4jG{+eC^=4|qpASIn^92q-ZQ=oD*7t4ZU5ankmqly6Fy5fWldKQ<1oGS-?y#0
z$_^!MmZ%oF2kdZQpnr=%HOKW;*%cLfQsM5^*LYSo%gClv&3G8Fvg3QO^Gqo;dY!RK
z+R<XSliMn@a~aFA8XWG_9_vvRdt#q7GB{16YP7Yssl{mFiScL=Z3Ag_an0j6)1|T*
zd|8^|`PoZ`l!w<yX@#a>9k@;C04tYr>cu3TTBUU9=&l5|ZWKO%Cr5ywNRRBaoI_Dh
zeWGT-;$(kxr69>lI$w(PvTf3(3EKm27Q>(olRXocSd()nn~E<zL%}2e+nTx{t(jkb
z_AKqKBNp4;e;kyu$}Ajcj_DG&H~=`9G8JY%NC|FB7Uxtm7ne3@b~+sF9ojru?;h-^
z(bG`f^CmgDCo;fTf>dU0q`{I@anj$VCOBX`Ro9xNMkYO{cknmqkE$)oMWPzzgCh2D
ztL+O`Qu)wURh6k*qECu1Z_{iZ7f|LKy}|V$*1*2i)J+A6%n?|Q*Pik$j=M*VXnm+Y
zaQ23Dg0PCkP;1MD1$`|Uf#BiX3~^Ahcx=!AKI;Dq0~pr3&XYW+rpFNx=jF+L*Z5b-
zl-)h8SAO#7IxZta`-etHu!+S88rt@xG>$otFX5L*$!pnvfnIQvL&%&O{nH~bBitFt
z!!rq0ihnRg9byu+VwV0)S`6~FJJ$R*&LwlAV0-KYR11Ef7XOz0c&p!7B0LCyt(0T4
zf7KVo{musgAs`U6Y8RW*1Qde&K+P(=AGpBmrF|mzUp)iFGRBIellb60LYSLZQ9C|X
z?+Zn)G+h9Yv>-V-kTx>X8ZcQUdPy(dNi0l7G=z})h8ud+7|*j%%t8T)<@(bMr@9jG
zGf#LF=85JqcTBFV_qT9U&tjw-Z0;k%*Id=JXozd3AWvvV<ebaL?n~br|69ilKtc`h
zU;~_9LpSTDfd98tVygqG5WWKXlHBdwoa(lZmxprw%e)$$g8dq2wmxi-%?jM;F5KT;
zVu?}eZ^o@&23n$qO~=>Yd4<8|IiTqs|B;IT6}x$A=p`-uI4jI>IMG3@*h6!n1uVjM
zN*IbM;gP+Y#^a(04}qfCs(XgvgASYOL$$8bziWTs682d=YSSMwbJ$E~#csaMTPhO!
zKk9G$Q`4|A9{B-Ba1AB7JH5PU^zV9xB#vuOMl9=<C0Mq9L^4`H!%z{=pUmncjP6>D
zz~+w5>J`*K4JvtI7vS{fxVCOjEaA$pE=7$Vz07}FWb?a4qMm4sXKT^?zw5a&Ed3a6
zs7|9Asacfb3=9%o1u|k&h{;_9p7iyaQOTix);|r3C@X65@=$r)z+)4wsI;xR<bw9k
zlUk>Vd!nD>OewRu#^HB+1ef=DZXOkg<?F{~JdUiYT1#-~&;GN=RdWLwtA*NErvZ-f
zLl2^LUueS~!znk&dg&c2u#m6MWW6D###Ok^;g8hYWa_3t1FFwZS`Z=NJ3cpdVpUY&
z^L<S~UagbnR2Zq${`1l^d^EoBL8qeTZz$eLi?Di60FsN7gFbr;0TDk;>xdU}t(iK*
z2D8|@V_s9t@k1xS$?#J3YLAMxzd-C*<;sq(mFne04@T7HQ6xN;^#8g~UO%{cMLXUJ
zvCyN3*v28@JopmandHuWY=TSC(QU+s+Rc-zllpA^5HQN+U+rRki(!QWYt_U+qXKBD
z;2$K?8tTSa6?ZxSh{plZ?9B>x7J}Lrn5C&Y8ct6y%o5shlP-p7@YQ1P6$Z1AfS@zq
z{rBa8)jNU%omrQ1|4f%v5y3&RVHR1%Gm8EB`db1SaZg+2HIV}5bgO3~GH+j>^bViD
zXmCGypB+E<8QZg?mu<Xds!lGvU6*{AxUTcMe`hRZ<3sNLM3*EDRpLJSYD#^Ro}H0a
zdt#NZ*kV69dv{#sn)Lp2`IO9(b;;;Jt`EOw%K>y*kENbx$s8Y;0`#2gKd@%+GCRS_
zpQ5?R6WC>Q7z5pq15+nS6RW|icO90QN{py>hcvt<;1AoR($p_&F)RDo9|OP@a-=Ts
zYVthuCJw(r<6Gp!M=5O3x=9*bix~vmF~MUC41ArUSKll$zq1t6CvWggSSdC*M;#qv
z%akU_?H4WY+FFfK|N4|y<i^NPDO>Aa&7GWMcAq~QW$RhLPp`X#O5+I+s#gxL$B;~W
zhr!IgdF|XvQMzc0E-Qwx+1>82+t6qsEDc}r0jLjVfwd@xJIiyVyC(UyECo-)iIy{0
zC-jGa+izT<G|%@`Cu^|*gY|}SDxT_Ih@F7C(;ygepw7Ofy01&sIgnq=fhWKpJD#D0
ziI-t#aD<jYZlJO*%ofz40^q*ah0Kvxk9u8i=&To9l1{_V7T2FoMSa<rsuU@7;qFUx
zz(MjY<)ct|WC^Yh>g4$<$0{xZzg^}I`qEi<<i5Vo&3xfx%G&3OJu|j*?igAaR+Hnl
z!DwVrmGB2d+Crdh>*N_0>0ZU`HTh{fa`xdc<Aj|xvloZV%9Xnj;2^Oo%L?fZOuY87
zD+OVMO@tW<hpOqDfHjPvrgdTGwE#P@+Kx!W&y?Pf_k30`#~h>;lU-)fVcU9Wq4gZ{
zYq1#xOJuw*v_#I<1%AkG(6}T>YnaYifpD+`eu4V8aIy?A*|*=Mok+C~wsv4_BAgTK
zvsQ_sfS(>ObpmNT^5WVgkL!Jo6{pLMM<{kFUr*V!I^pdQKN?ZZSNjG6Ex{0qG|{C8
zt6d)?O&<~IBdgyv;7I~SBi0=5Ho`0i4af9m5UPy6qT>o{Qi!EcIn=IOLY=&ud!1Qo
z>k()-?!t!OB@q*Z{|b8VUGG!3<-qg=&g6DPbU}{`dw)E?yA$r;Y=xyCxWZf(X4LPi
z7s^UQX~CLl>v*ESY0juWcv8_~sHtGfVIjvUR&Q;M-N=_=PQxk0gV2f95K4xMrVDqy
za9I1_`ME*DZ{ED-bK&NeXgse!im9YFY-RoO(Zd#veYjX{N4oUhA+1@D=hmwO&k0{X
z;z&VHU0bwS`ltmdf!L|7Hiz)FvoE_*lY2oG{y}2VmtwWwU0o&JY+X7!1gz~@#uuv|
z09t*7T@L_l)epjbWhO5!$|s(yb5P-D+r`|+?Lai*IIMv~r8xMOP{Vc0^<><6AwZE)
zU!^$%#H!~N+cGtz&6zrrmw%%Owu1H-VcELr-$m}P@QNulja8Dvod%zK=OIMvt5yd4
zI$eYzw+%7jBPyCsa8G^e-xpM!5x773qviowinF59)irzR+`IIuCup%?)GeO`Gj)<_
z9hWXDD&FXoY%dz`qn)%hpWXJQBN8_vPO&g@=uT`$vG%*}f8YrdXL<PLdjs^n;YSva
zz<iY_WZh@T@Lo4s@872m$IOL*^d(p~br?N=^pb5tt~<82I~1drVBM_z$1J}5_CE%)
z#<v((nX*Z{fWzO#+$G3h-fLq<4KVT!FUjhYH*6<qE+@QNGUI$VfF31Fy3Z5s99Fl0
zd4DxITRXF1K2lawpF2pSb5Qzi=b}u<_igtqL{(R=%PF3nnlqkr64h`TJ--JVEUf6?
zVMb5*puKGjix%MP`+B_cXp7N&;F4_bYS4=`@DE>vV-|#-gMBI_ud6^EBC_z*lWiFj
z1xMTKFJ*?W7Yw-B)ZzXkEfN_5y+&n_;=ujKp}eq@RR%fk4+m9;Io-FN0I&De#WccP
zmOk-1Y*qfqm!DSj3*oocXCGc%cUdo2;$F-oH|%pMRLdkkLs3jJ<5wp#*HK}DtftAW
z@W|%$zsqjX<e<P*%ut5Px$nss!-@P75p`uD4!)b;Zm43q{3XWfHR)uz?tq0pmRrOU
zkctr|Rv0&cjXuoAdo{;_5nQ$4a^n<-D-hJAzbX5|(t0s~f%|~*o44&iNtj>y8%I;h
zDnx}#&H8oGvVj_NW*CYw3Nu70ZYZ0{7I@(%?n$kO?x3g&Q6&`h$$03M=gn`h-%rNZ
zq3VD-d<*SnWw8A)Y8XIcz>}cW=01;))K4(#t)z;fI$%xZseViD_uyH<rlFEkYN$Is
zB$<YA-(83(dB&_65%mRQrG9xm5SFdMI^xoKqde3wx)=;XVO%_bK1IJF76Ek0!=<!Q
zSs!Wc?u-rT2yjhr&CICT34$kZ@lpn{`a}Oyj<-hYare(Wcbw{|;P|+8bNqQjN0WY%
zo!sR6;@G6k&fv?l(6zTHgRd9xELUPSv9N+8f#-N=4|+{chmLqHF{Ph@p{7m7&RK4@
zbpge*I55xca1j(;f?pE=1REiF26}OIViW(HEMe=s8?-Lo3%jxxZ13)!U4?z$e~HEl
zdrM)9Jg5t;LYjc>GPyyUVWR>bvp?ch|0H}f9O6e9An>)>RV^SZAIu~|pWgW80X>ha
z=ML~v?_KIV+UL&eI(OrY;`8;lZaQ|hl2h}1*4Yo7TBj~97bv!DjMU9qRa7{pAGkLj
zZ;AI`Tl?6b6cB{%Mk16O5!twaxg!+cIiJz2c^r{(z6xSfD}h;g_&!|qY}FPH?uHQ_
zM)ysyM=vCHYDL{S{PVS-kHdjrynuYu>N5x?i5yt@ZZOv!T8Elt(DY%aA%m#Y2H*b-
zVwX@+&3F2C06GW>K?HZ_>->?`NS`!6aGn{d;0LDlH_Rc~ST45mB0&?ZN3Plc3lC?j
zHW1TLCo%^|;S`NH6!_{ZW(;)#d@i6dHLi}{l@rZWWHn+hJ-t7cUSzYhf17l`l+xMV
z-H#nBDHh!?jYgGv=e9%(DT*<jqLP+&CB<fXiqE6F3M43@CiNlp2AI_H>EaZ8NDq7Z
zFr(BW#nvqlQO#cj4LTG)2~42m9$#`vhTK^Mks0VVUo+Hxrb1CJfdJ`>N5ZVWFfguL
zdtJ1N4sWo;-$WB(P+(3v_V^mM4ySSgKKSc!k*H|jycjpO0<#zJ?C!0r7K?v)<4#!F
zjb;Uy_ixD$(%d0hqE3=gZvDpXpTsJy2H%(T^*!;M58~G4HQPh;a6SgvXV1o?JExnA
z?<sLh9~RISd2gWQjG&$}{hZY%kTTWn`a-MFBltyu`0z!$18XO7Rri+5FQ7b`%>Bp1
z6|m9X$(Z^}cptEJ(U8&&wY%TTzO~Av`I?b5bgA9=Gp&ok&^UhJ;O3)??hg6VcVW&j
z^(Rl&bs!Jr^pr^`+M3j|lFb+Rd{e+5fAoU_R7riYGmg3X=mA*Da<NA)<LeN04$-(C
z)EIKap|KpcY+qgCg#h`*j6aK;%!Bb^GK<Om`z!_ZH)xi#$jJ*%PyYZ$1xy#^0e*?u
zJ4*})+y1Jc=mKMv9+NeTH1LfEagJ_;Vl4X5-IBdGP#ejvfrMit{$Qk~Zax3;Kt;;x
zQ^C8PxG35#mDqaKLoz!~JgM9ra{9(tVsc1Sk-rG(`VP;vl`dofO98z!tee@GiOg|&
zRm8KL-C3}^;e9o%4jGjm^cv260;FnU@lU9z9r_y&&0y<XpggE<bsCycO@L69)C`ul
zy+g7B=mx#%4*^loYPHL2hydYwX_c%3mH@sBgw~+%b^O2JCB9hgKJ=Tm3RS~F!oY#w
zau67`U!P^p_f{F@I=Si&i}f>8lD7g_Ozgn=``H5B(VbQ{0c~aiE0Y6pKM<m%ZcF<G
zspdYZ?!dp^XisJ6@bZqP&-^XRefq?n>#fE)<U-7p*l@hQ;4TE;LuhBl?pRwpi|(3j
z&m%QS+VPaR4r-1%sw@$ehnYInUMk+)VO4Nxp$)q{h5m%dz4SL?cf7#JZq|Y+Ky~L6
zc7qmL!6KBW6ho^@9XAU~&#)JMSqI&nHDb&JUQ;dKw7?6w<nbTsH!RMa8Tgfe8v-tY
zv_#!<cDTmI^*{XMj<xNuG9a>PU;#Y$i<v~=kngWM>#~z|s0pz&F6X(TLDXa|mVDMJ
zY<A$EMnVWmtj|wH){4@cI*>tsp?_#u?|X7#HtjFV$*NNl!rZ~aniXaO0(@y9#u<Ty
zxr?fXj96;-?N>=gtT^W$tUcyQQuLV;{kZMbg`+-)vx)0*bpKl$I%w8JuGcKkNw4Wt
zIyJjN*z`AB{q5cmCm=G+K1UBgBk<PrJ4#nh4b=tMdvY1PJjRc-L+oDsN5a9OSwbE0
zkAy>hN<_T<CRv!lStvDcje)|42zVNX^?t*51>V~x>yQs4h{F~<YDs~;(p*J|s(qd>
zO@cq}7w&R>eE)vl-zP8eM(vq*&2_(f+KZ-YZa%d1iF3#c6>Ny{yDT6T=qJ>daIRS+
z5A?sRyr7O{7WZIt6I-FG4Heu*#1BXH$|WR|0I<Gz)dHIdL*ke&?SPP%AWv`}aM_hr
z4!-nP%m1Q;O)CN0Wx-~6UbnvZw8hf;9AK0D2GYTjf!`D`A&mbJO#BmnYbu*$tqimN
zh?aRABW)!ck1L1`<s+(&SxOBr8)?#iDG30J1(beRA%d;-aA$YT8<9{d67+tUzJJ9y
zPXG*I-k|+DKH0v<DMVru>(RgecqmDiwPtU<FY_ed_qeC-aw5m398dFds|fRmRdPyJ
zE$WuVhR^XG=bFqsY`<9VfPKd=p(YqSEk?0u5RqcU_?qqTj8{&Yw=(fI!26z8q`z>c
zRZ-9<1lmw6awsn(8o7i$cn`DaFgRobH*I#*WQgwc81C~)P;i)uhEcH#j1i1rX&^~i
zt@o=jHOmRG8%Jk8R=xILfW?McUzQ@-2(7*mGds71`)R69XJr(@PCGuJ&SBuO+CBTt
zNd9GSW?cGu>SIV>%?aU*t`n;UKEI4Mt#6r>n4!8=k{(SyyUkri9e@A6Pcw$c>9K1=
zlxyvn@pQ4hXx9FgmvYiBrw!>Ry3ZT!IxdXa<`Y#FVtbbKg`?+Fx@W^Yt`tLS0G$wv
zTl2(T#Nh>X9VDxv4d@}*%3-Oihak5Z;1KIZJR5KdLH+bM;SVuF+9VXY`WgN}aySDG
zZjEo&N=#O}?0R|m&(uyh4FeJ@s3BkouoAuuGx_74lvMmMkX-sm=iomENd#sm4g!U;
zA}$-Wh6%9&WZee>Du0#14w)bc%DLHp{3>Ux27%f_nDKs0iS!=RqQJm2eotB?BxQK&
z(whB!M*93U8b6DKWi{yb?FZI$rH1BbQ$JP~ai8sUynkqyq4M)5YKQg&(`w9W9rxY|
znz$gg!<CgZGLw@Xr>o6uj)`+=mgsQX=*Ia-5rT6m(&!P{UIEBtW|c}XhcfAc0ZCiq
zfCL>Kjj~FcW_A-9h;XA#BAoy&j{;<fJUR;cmaW4>Fax7YR2>R7TnNnB6X0=hkV3n#
z?y&B_CLJthaqTE7XH-&njmYzX2b#^=1G@;;;b8i2K*NqhHRW7Nf-TsWbXGrEHz#zf
z?`;;)&R`aJZKS=xI7_X{PYO&O5%{vC<#>KqUO&RWgw_#Rot{08a3eB}(oen;9y_T)
zb59iFGT*&Zx=XO2P+SH)th)!hgMPNB6Fr%st+{7m{OI(9eu2=er84L`<XX|F59xR7
z1I_W8Nd)RUJNo#Cw6e*KZp<N8qaV{^(|}&5k6~maGlLgV7nlVu;5@3zJ_L$np*Qi=
z6Mg+=U;!mC1YXfikf8e6NxB=P0J34EJp6jYp@aV!eiMgpV0E*SsDCjBT#n;5m}!vR
zk|N{H@<(_j4{y;Nz-)2>4p$1n#-CLVM_=6IGX79ie&E-`1YEt=tSIz$Ql8)t^MVr8
zhr?q@ed_|6Zh`1bQ;9<a0St&Lmxq)A%u``|hSX&9b4lB)9H)-OMpf;|GrMu>P@>h~
zZDOh3b%Cs@c>>pX)vuo)^=Cd3@yUOW;|!o7DRaYPq^1JOC#O7w{qUu~iC<0$Oj%1%
z(7ll}1#zXA&qd4f6r3L@oQJUW+F(U*nNc13XbzMuBC{CkgJPEf8a@)iTM!80=U7N6
z8ES#1A(-P!g!koQqCYqYzz%=^tiS-rX-vHZ^XfmM%6QO@w_GG|ZQ}pPuruT&fsr^T
z0U1`Io4M@js1EY&At051y}FZt5E-_vLp0uO7o<@bF0Fk9ES%>is)-e0hR`fHz&UpH
zXskWg=a|IK`-WUxvN3xx!TzRMo27`VcvfTO+uq8Rh|ZO6v<`FQd{)RsBWuN#r=Beh
zDUy^H_k|Snd!V10Bj$WK8#0{1UqJclqpUhG#TyacbeN-h(?3}v&fg~!%pwHmLH}59
zb_n$;kCf9n(lTs+VX{CW#zn|uGK7E{=#x7MJJ(In=M(HW1NjyA?=9I`hYdf*`0C*d
z2DY0Oy2-cnA}p@%wd7-4WYYRO?fChMmw(vP!d$G+o`b-zDk>k|Pjs!nJWiKvI3X}|
zy7{XFbOBfV3yLa~s#hCA`L9UTkH1Wd(4GV*LlcvgZ)qmyci<m}JkF|0J7K7D=tXes
zcH2g5y)$A#D%RCB$MPkmN3W;&jFn9DL!Kh6XU}Sv08+a{t`_WhV~KK(+Gw9QiwrBP
z@gCM?Z9ItFT<ipfXq72O;p)mz_kv5@&?E9ih`ZTfqNZ}t`9fd!j)W&rQGLM8fG!G8
zI*eLJ6y=NYqPRZ9r=!2=1vhvhtJV*cx(wBiepzJd;U*jc5@D-Gm_Ua|9bfzr25$`N
zW#73Hgy5kOahj7Y*~5J732*fdbvWOc!C68i-g}Pupkav#^UP061D8Lr_x*wG0tO+U
z>WMK#3ML)I;qO087mG!SuEyq~=(wxQ2=S+@-LZv%wB1gz88Zdiv+7xMoufU$Hrb}&
ziM`QWU1!=7hqCls<;n(KP79JI9~PT7N5~H;eBzR>Pmv;#%&y@nJ#N&@#^)mY{Wqk%
zt-~#P%SK3wEX_T`?bo4d+GOjy#5toUDQN*9V<Q@%yNKy5up)uYe!5NAe4o!nXwDl;
zg6YhREj;X<B&&9TDxojz)-B;cF<VwCsSumS^x9w&!R~*=mrihChw0%g0I?eiXIG9+
zF__r|;4`0R1_3ow`r{BMAFqe4N;p*d_!a|=!xNO;h^W>@{m{GkI%6xXmlUP^qeIvs
zz%|nA7Hgg$Qrp&ny>wTUVi6!eCDjtwA2PfYA#U<&2sd3=P*v{S{Ig)6568wdy-P`)
zB!A+mW^9dG%Z5>$nD}yuNs?4|0whCEnAUr7h1IO2^B3O9SzH=2*{CHTd*voWDBalE
zkPY82Gx8f~NB#{he)Gn{JR)j&_MJm^8mPb=N7dX{xCKOGYnA_SYJ;o5GRoJs7mS8+
z`*E0T_+uRv!P`hl-G~2)7)<7(0ND4SEoS2IvjBT5sCC&tq1-QU989W+<;j(4$&6=_
z@O+rDZMtcIz_3Wg0SI@IDe$tx*$PNqCpfaCT~Y<yjR6(sayILSi`Jwg7u>R!WpJnt
zOO34lNk|-$@hZR!joGZCUZVd7Hkul%6q=I5Y%Y1FBpK>l@!aEjdZN%Xj5amiO;wUK
zmX<E?Nr*hHys(UK$MW$Ec7^0NKHQRH$sc>@#Yn$~3O91h?J}YG&hL1?>PEqmEq5Hx
z#MB$7^Bh_a1bcAFVB#2#jTPEmU@%AmXb^1XfcG&IxCy*Q-w&Z@z${t7>i6DnB#3n6
z(@u|WU{;g2epWB*?~+=~ob{#_<Cl;yM?MYwiC8+dD$PWrW-4Ne`Ks^9+<pE#`i@6$
zvJmc1O4<2P!Jis%E1a{_5Y6=dtl=h{O_jDuR-g6Lp?7W{-)|K2OCM_k4Yzlr+KGi$
z=)Y~rYex|Fv)T)hIAw#;qN?F1!|RCqUq(#yJLQ5pMWiT`(Pg_OU_3N^mH)V>v{`w8
zZOO?9rVqW>6JZf!HrV(sx3k&z5NqQx?tpdUsiOs^-Ddcz1>AsuP@I$ay4jAf4eLqd
zstHA!XSBGhz}}*MG3BD0ua|`yYyXaM?9tXod&sWRL#OkG1Zj4g1tYI+1AMm~;dSJ}
zr?m~+hcIsKp|kxRVMHgWvMLqCvn5Dc#tUHav0A3lD4{o<0nboE#2=a`erwdNqpeqd
z<H;{@8Ul<Xe_o1LIzXFPomi#JLg`wjT7bNJeQ7t%Y0zd(It|6vyBkMa{SFck^}>>O
zg~)*pU(;DUfZq_+0b%&;QupLWMnknlz=iBIRN7qaA#<d&YceKyTIw;nNFsR3v)sOV
z_{3q(io9cGuVZ#7UBe<@Rv{(qN%F@QXy;Pl5OhuKy(Swps>NdSyJXV^cNS({cfm*k
zUo}BM#oiOe(i=h+>Pc6$$1*M+qGZg<v04F-!1U}+LXeSWENDlz+zP3#Y$lEFX3d#G
zF?v)auWb^GP2s}6ACH*Gaj5vMk1f+zdY$Jw%nsoDh()7~(*PTfMiP?K@iopwnkDY9
zX9@`+iicY){K2h5J8npof(DLdZ`R4NOWt7Xr{6KM6_5#22ZNRktblKbN$0vd>lJmt
zGCo@M0L%~x0SdDzY<pep`|a?Aw-cA+{e<>VNMD$$*DnUF@#o8e=aHJ|44f+RnyejU
zk(?*N$O)8hDdq!+ob!sTp2=ivC3I>*pJ0{#tQ?uA_ig0%LW{aHW>LeUjGt>`I??gU
zp$~?6?=D{CslPB9LS1f{Ad`0HC}%Id?9o|BMrg!|1#6c{87bq@kemz5lE*#Ju9~~?
z%L#9W4>|TyA&>GY0&HkQa8_=ptd@d%`TV<xRg>=#wBkOj>1Q)S(ug@7PIWrUgp+&N
zephH2gPnTACcTq=`byV|sauBxC(QrlhXlXGp$@|NQfv`(5dsLj8?>#Mh=4yd5f%>E
zpuCotdbz5Y+0bk2<;2oT0Jp(#*&rj1Q^=brFW8@TP(}pJqq?r+nF9oq*#g_H#X&$^
zAUvTaCCTLQbm9bRGmx2;m@*vAF%uwTKj3OQrOHb_)TDWMkN9YUe2dDVPlo6IHXZIY
zK{+?-Ym$zOeIq?jM%e3$ylul|y0a{L^%&Pt&eJ&#t()^k>6*Q0QP7S)?cYv%PKu#7
zly%Rc?tqOYc4?DVhAe5KE{jyV$Xqu1S`X#Iew)M-fKI3ff0}7=yGN0XtGh=b$LQST
zB|HTOy+PgB5$RJPvX!cY6mSDwHYeE`XIrV8h=i2xP2!*>8^crx{jodIFijYZXLZ#S
z*9GODDmylvkPY)_1)Z0oWM!6a$uzGC!FdvQ`2It>*Wt=7nNI=y5(6*%+Pepmi=?HW
z2K8hKCR?cBI!XoArrM;A(=0MId_6hYJLy|rlhuop*meB$<P70PK+|3xbLTut_g2}F
zV<B44$v><l9Zm!b^Jz~_e~4v%wXZ!Z^yZPW0{W^=#<9mAEsyOm!1NTD<xxyMc<A5e
z4G(M1*G~S>Yk+x5?Mu)Zph4Y#cpod+kkyVD)yg&+soX%fg~Dd)5M{L%z=0Vvgo=0F
zt&lv0Fa*c);D9o@2wL(sl!U|nzxaL;>sjTL@}JVnK+E--P;p^M{2(v|(-)E*U~z5^
zb2a8XgW$M-8NZCrkDS0d+kFm%BS|8*gjv)WPbs9R)(dJmy~Ee&vfaXx$W2_G<!3cG
z?71605DzYXzBXVKL}xn1A)}Ons9yZr#B!)k>&NR5Dncvj1IFtO6E^L>kn>^t0+!Bc
z%5<8Z>^jh@uh~Qo#OX9K$DJj#W!LfTuhvsgIjl~e@X3l)JQ&wAdj{e`l=PwcM|VKm
z`j!nuL~5`*7Whv0ilnFnFEsF-*vM#CV_dw-qukVz@xVL_SrpfWuB^|2@&7Gz9;syI
zs>#$odiIPHrQ67^QS%at*#RShMP?ep**Tt+dK=u-$pgcw4p<C56AEH*$r1r7!v53#
z(>B8d@Dd)h0oaj5aI1ov4JWykQ=LW~*si@5pNr2wlslVt*R0AO6Rvqw`VPnTp~lE<
z69u*I&JQyQ`23r9VQz<A<d5*L9<Ot-JPB@)cRMYM1w-{NxyPrdX%&Uq$NfvztyV>H
ziNcUC?e<jKGoGS82L7vYuTxrxTn3+J8*|)fM`ms*W+AZ6!D9&<<8!HJ^$nh?d5wOe
zLAPf_=%<XgFzN9tQc)@bd6|oHG|rG?hseI4+XaI^MxgD7B#{f>jfOwEBxC5gl*-G-
z2@gzpxn#U6=?T%8X6C-#VR5Us=Uu=KAG#<xpQc6d0nbP-rXiLqX8;K*E)onUyI>Z#
za9}Y2Grgg#KS@dBEOj;~v<UN?*(?`JCYuLhOAGito$C)%(8~A$d&+<OCsp9dg3Iuj
z*}v@82+b*!xxiGdxqEFEE801TPj_LeE{umdGDUsbsoyqqS(t2$Im0N9h(~O($Unb+
zm~2NhR=uJdL95@k-L=NyMPv<hmK}8_JmB1l4Dw4^SxgK1)@u&m_32n-L&(fWPAivJ
zYK%C;4Dlgtj~gO~u=n6te;DjVBf{e8eJs~RmmUGg3N6a&E114~t=!2dU3qifl0O_)
zB!=6J04i?$3=8I9X0<nPw?NrmBsj^q?d9}JBL+<;tgs@X`F?hOPTC8|lT`{{s)T-{
zK*W*O$w_E(f}Ovw4uw*yV1#%|Vsg*|cn-C}6k!Rp4|SFr(dm6X8{ZN6y<k-`4hgG{
z9C)UR7`GTz<=#+2hO-JtsGuPxa`Bd}x?>mUyRqfXUY(XVp}O|<?U~e_b0L%HHS>Ct
zEnMP)i){u%dbQfSx#9!N0`TL*k85rEfRhDPsUI*K`4rg19HL-}fver#sG+pl5b)*n
zw`p_PfR;UBsyAleadzbHeSS_}#$F`He=*tpTb0P3sb~qB(vcyP@yIdqG0|zC-5H#J
z8wF=>ym0u^`2G@W?t$=7Hg4D_zR0wnZZxt6S1gtj-niO@Rh%ub?}pc33~5SwL>)>1
zlEJ-@S-P?tf(S*;R@1tZ)#h?3SW*{$VCnofXw89X0uSK4iVZ*p%SHhsz(<Hdko!Xy
zwyDL#KIxbP8BY#@bLDj~{@~Yu9dNoEcG~x$9S&cS{JJRnb6h<cSht)-d2pNn6+wyc
z^YA{&Ch%l+-%&#=;ovs<lnC+wyfZR1dat5@ieqbu{S4Ns+vMtcviYTGND-(;qbQZs
z{zYQ=L<KG4V3i<kuht}b#`ihZqB^k^Au*#Dl8RY={blJ5zQ{O@>)t${hOqx~HyW~e
znMzgAdf+xzB{DM}z47?6dOJEjhkl=^jkLD6n35b8Mi%fH)!ug)%v#e_By|j^&5RF!
zK<F$W;Nk_l>n~!GL9_O##PDpdVN7NQm=M@Q&<-2(u~&bO5QP_`OMdq}VpeIUfrh&R
zMSUQv!6Rxmn}UXz62aNyD4R{6O&qK3%uQszR(BuK3qsLTm2i+X^Ae02xVF)VM?vxM
zIuYoB1z65%2=6a}$@aj0n(v0RQ#H)k)K;{xuk;h}h7M*-CBQhA@W=|eQklipGx+xl
z=?j{oti}{1X!ZFHRCiqy5{5aygFu_wsg;9uxY_+cyM5qbq}NYqt@_e_=sO(M_=?60
z2gW_aqhT;3NZ+hOlqg{{mb~xi`Y=%o;?67Gt8|C04DcgO>sVol_QGlOu)>8&Ztpr1
z*RVz5X4@dWl9pONM6AyV2|-GHc%yIPGu=huJn<mOmdu3=d+UV;-fx}fwJ3tDs?zOP
zCncRrDCM#9{F%SZP!xAokKP8k@7HydX9i|=Vqgj8M8e0*31)OSO$n=P#LCbm35HC4
zFm-TEMQIz&qvn|jpL9v_&gOF=8wImi@masUWHc%Y8F`E}jv5Zwj8v7M3MsF@w+I-@
zK<blsU$bN(EvEomCTH@Z=SpFc4WM!)4=)<oN2D|mC9Rki%m}ms_=fcqUf`|OH)D+$
zt1-s#p%44dNcRUk#JfGS-f6&$YB$Yg(B@AHGCQ-YH8cVWfMeObr}he%f24-ij~|)v
zK7}`C8ikYP(fPP&<mA?>6Xeg-XrGsZRN^AFPY>aohc@f*jSh?1L=i~tO$e#P<mKVn
zk2r0@b?Ex254}@$(=3t__oB$toneFBKWdYnx|DZj@1BT4Edl#|^<#RENYn|B2nXrR
zw4xm!m2z7Sy?XM_WBO#odCdicFu(E=kc0_}=hH|p#SDo@6wPI0sBZAXTCHC2V4ZM9
zHsOt;4Lh;SzE7beS(t`zaHuejCdb-8JWBiIAkpOm8qK>@yL`a;$em<GvmQg8i|99B
z%oyw+APt$rs_FY2Ry*%?Q5uo8j!=5Rpqi~afIOMcP{S~|7y{ySp-`~Tg>5I77|h2y
zA6S99M#8|2JuS2MsCy9MCcilecvO|1FzWLOL@g)~3?#_?PRmc7JTY;yZ@bR{flPz~
zn7Ym(yi_-kc*WtK&U!Y<u&N2`?3c<7Y<*2!0oupUFkIvt=8c5H7!%0)N4vZ17)VNo
zE<;R7cNgIATHfev+OEfHyf3olLly>R&ny_#%em*_blaQMsA*Vw8j9p4dT*k<^?t#g
z18bsMhT|<hiaiNibr%N}nl7-6UoJ-!(o`-k#1l!^@rE3TCubRoERlG6*3wJyf}+*b
zJJ_xQTUuZnfWs{G`(0<PsbIrlWsMXViiED?sfm9(oxCBZA~4)Xc33Q5)n!#{P3Hiq
z8Ol={7g&rL95{VA5{fyySx!yZ%hxX`mlnVpwuH&Pj4cG~W6;P)Xb&_<c<KTh*TV;)
zQgDVkM0pOq+A>(XKLo+#?&lYi?GC?VS6geMz@k+aV2IP2i5(wfJFgqXna_PLfR+X`
zJ*AT&cObXlGEXKR9qn*hkRG(GuhKnG{U-}ygot_;z4(-UBipAvbCLOOB>vp&G{8#l
z^L-Rj=;j%ej3J0^7Vk#ha+p%uoLnN(mb(Q4@>R@DSeeqX^rqrf#A65!(z<xG`X+-u
z;8Z`|+YfBkdJ3AbYG!@nUVXKBBC?TZym-D={Iu)t%e5Wx4FS|CRc|fU$6{2`Dc(li
zMj6zhqUD~O_csv~tWy?LVLXQ_8wPsNTBIB?2D(mRu=)auseNc`fLK--p68IBu>l7(
zFgsisiq?yprI_J<dgoE8EjeaCy`PF*mhB}}#&@w3;xLeUJ`(;tLS6EtzF`;~H?db;
z0qFhV4BWEp2vCGyLfhQ=&2_ZU-ohv-ve=Qa3MZflERtoZ#&w7^NE7l07ny|&GC`^O
z2oX-Vwwx`X{W`x}DId#r^Z%2z6iY-A^JmeUm0}~%fId0|e7GgEo=t43B~bgQ-HL;{
z^-zzJx@#(e7MV5sk!obe7{#*^_Q2i+Zhg6Xl{wEBJ2@;2oD!NfP1k+SdX>n+c6kHN
z@yatc&%~K|r(Nyi&pPFbJS0o@-Y^b+Fd0ocnh<wSu9nA*!`Ch0q84fFyln7>Q{$A>
zP>%qrs2)?I63}IaVx@H}vevNoQj5t%IL~9MD=x+i?`c8N(+Q^3^1WK=A{0!Es{<R-
zzIi|@e)9Fer4Nt{b@eqWNIV-?c-E*5f>ZTWSkY~(A&|`)OiWka=B;GH*)6xgz+{+e
zp8{euNcd=l&x$eQcXgJbVy<>b(Pu*hz{!AZP`WON%gg`m@!vlSBdJAQn3r7TV-M;<
z;4q6u&L^Or9@m7BF4gULKA|)~3)-SJgw2c*+Z|Md*mQ|_Ao$&U<M2SV(i7G%mjYQ(
zWHKc&9q7>79L~o1Fs!N7EbpY^3n;Q7m`}4<jSwd}!kLD2&o={s=}-d9RkT=PYZ0`{
z4GV(RI|@BJj+|_D`AI)5@qIF;J<U%C%DFSlPyD|q`|@xo|F(TwQI^pv#E`PIP?6n?
zQYMl4qN2o9h{~G8SY}A5$PlFxL-rQ3WM?edV{BPt?8{hY7|cHNyQk-Q-}n3d{q;PD
zIS$gy9PZD3eXh@Wo#%O7a{~6n_n7$aQwjHPO`FRgN;ZFb1)r_9Yt<LGKxS9Fj|Ak}
z$XCMH>sB!NM)}d4sroG0^$e8DOBNAEP-nNt0ZzI`)70q*^v#RJ96~O**5_b5;DjA*
z$+4X09b><|s+|6+T?A_Lb~}n<6h!40E)}<Kf$7DG=E(MFG*``xXm_Ed2rn1Fvtu!U
zwQ9|E63R@EzKI4L-1C*1ot&)}@O0oV6=Eyl1(T$U9Y=F)?UlEH+9jCtPGJ~nDOo(g
zjlmx%2;vABEbnNreew>2YPo`Y4<P-aA8zoWpCgtx(ElR2dC*{a+ce6GRrtf936bDq
zaRq3j8Fw=L9{ZW+!imO#NGj8TYad4F(^Fkw8tFT>^7R^czBqNzhch@dcbEd(@?qk9
z5+2tKkZ2aOCQzqgDRiI2N-w---2zt8ucsame}v#eiGs^BR(mmqv&U@NOLQftnKG;&
zIawuQBEH{C&d_jSw!)>gZS1JvRulP7OQ9%arQ~cW<y`P>ndY~>&C!R)sOw*%q5zKS
zz|kLvo-CMy-M$V2iP#AnbYdDP1@(Uch@|DvwQMp#$0S{Wtdp(f=i%4oN8q2awE&oS
zsWk;31#U(00F<Z<*a}+wC+%8*8@>THRbaUqKK=^5QIZw1q`^#H>m%d?=&Owibzdpr
z=m6!m@vWNG<7L2UM-y4TEp;8l>v<dHXch<#URon88<0U&UDzgYaaK9-!hbbrO@XG5
zQ}@{{0ktCZUmQg=ZcWg79-wZ1i13`yvGZQ;=c?*#;|6VG7R~V3jY+*lj_+=ELU~jO
z*CHiTti5WA;A8oKS*EcWSN+qUz?!^Xkqh(qpIHD2ey~o27>pHfg4jq{j|n_6YMc6e
zMbZkC;>@X*6)}<w8Kl@2Easv|7jHl*N!os|k{*XP)|Cr2be#I7dTVW4LQYB%C|KsW
zhs*)wGwlHSuQzM6#}Im3v-th>uFOQEIJjd4u&KZy<TNa-LVfq_F<|M*H1)}1fX|Bc
zHA0W{z$yozS3Q`^y?FeEu`@=nP6ARd(t!fCg!)!EvZq-+#c<s8?=xIL&2;6|s<h}_
zHCkpHP>cKV)qiqv4`;r50R5+JnI~mYaIhvotm>d=(5TB@|LeB+m+PWqK0JgoFpoji
zzC6Lh{!}{ij-cx%8^&AG6li`IJ)C3=g9chHfm5n#(EySRf<F4q!H8!#5vVxWB6q0Y
zi$H&Hf23EP{bT9pbL~}dE`gM4_S(RuY=+uFUe`r8hg@OdR&^L?U@y<;M<y)f0EaZ-
z9ednKp#7vbs)9WvUFNpk3-Vpu&FwJu+9p#kCoUl9Eg>U8NtZwL8e3GomEkU@@$f{r
zc0dojiry2t;)P&Z08XQ&+I}VBCxe`RwQ%kB_FlI^#$+*f2rwm`!seh_V)6mF8^=%s
znrJZ;DXvG$16CJ-3WrjN_ILuM)e5?3wrQYk?UF}1%;@<XvjHxJj!PegOe`@e>`_x0
zj^;JM=Us7U&z(XqHvG8o)-S-Aaa)7VMP8mkuSrCs-GIaX1n2GG`L)4D{7D`a=JZn!
z;P72XyU{ON%5nRF<&*-Ni539iBf17~M<09;zO5gyN$0HD%olVamJU<`j&S@M@S^Yg
z*ikN9`>07B0h~MTv82E)=p3@2N}6U(8mzMTp(Pz$Yv>)woI`RI8MeQq88N=eQbRmN
zc{qP16pnZf>oHPBb<xO_gzAl}5Vhbb#e$W@ucJEL=d<gt!g19~H?^-CgZWk9DHvaA
zdb76UVbPrs^-S;NOVYK@FYnyfKe^!W@LrjQ{!P{X2O>@e$d}!$T*3ti(e+D=?}?})
z3`N#27C9Q<G0kcztSdIh`QBK9NoWmxP6(W-*DOr{Z(@beDc}&rnO9{NM3yOguI2H%
z&PRk%>tvWIotQb!8nQ(7gT|+sIi0A5C(ZO6t~)>>@d)pxsDB9{6#20~4Tx6(*24%Z
zIyip-a8r@S&(_cjQ-bHQjewgI-xT=*n&fB03In`Uyst^$Pn(cS(lSe9zJDxEUR)1u
zQ_d~X5YF&Ca5thsCpb5^UQXfL&RSk!`7vmsqk0{*aKDOJCsh-t8h!XjY%OdB-TGvJ
zgbjQY%gD3I*^G@STA68~0j5W;C?S7Sr`-y#ib%1c3bNXiq5;P%onmfC9G&Op3!^vH
zM%Q(@8RRH%5q7p>>lM4>u-7($;Lwc=nw~L{Qeo*fAS+n1#sdeEc_BfU+ar=UABHe`
zz+TuqB%(HB*WDp-7L@5|?bc5^Fu;iC$BABNx9dMgwH#63${I(eBOGEnil%{I%&U<C
zIT;+Yrh2?_UU=mu>z?d92=wsvnJ#oc2=Bva0z(#c*;D8@47etD1IuQD&|kT<%_QJR
za$P~&1LneAK+6SNG~k<hKr@Z!P67af2cS=em%UR1{<UfN8xSga&7W6UbTS^nWO;49
zbX}uASWD`$#o~Akar@C-oHW2Z%)Isj=TP=TXa?C#!o7U=OhplgHoDJ^B9*ME=wtdY
zKQlJ^Y8}z7C8>Jnh%y=<|AG-VS&P%m4@JU^HuF_9F&<kRGx)eLGnKT~12;H7ri%tT
zcH_kLxD{I5&%u|bSs|E>3#mo@JjCkDiH!(ke%Wk1rsvh3fnqq`kWulL7FLT3Enmvg
zyz595snvMiFO|=K2>+Pbrn#PM%XK@R@N7j?IU-IRXLl5Pr6$|vR~U*3U_VS$ph(5S
zyjSAwFL9L@Q=<GfjueP}qcn(B8zz!c<((mS0}qJe9r^@&utn>y3kN_XUCZ``2vc_J
zxyb2{V?GJJ+Uv<t7zki%2Sc72W{NRB5dwR^qn4Tr|BCvZF^+rad4;R%NLd8^t2^l(
z^-Bm7#5v9F@bqCbu|bwh!fu05R{)yvR{!DM``mgZ^`H~<E+_@S2Vp=nf+Xy$lcC=$
zJlGUPX;u3}a`A{Ro|@_mJI>J>5Q+vsu-DWDk0*xCV#r<+?q5$l&iTC$ecyF{5#8-$
zv;~8D&{UZ@FV`?wBnCO(TH7JL>NFToAFi)nKsF5cY9WU?eZT;TN5?++q)#4QfQ{=q
zuIJBhvRX=?uNX@ppFMU<oIm(gMUId>{Tg3OPRHt{D04D#r<>{UU&7Ni+#4#aZe8@!
z+#B<ag7E6|Ci(|<)}5;rdl`3%`S7G6f{*7o8y1${h)AsONOy3w7Qfr>NuQ_>osnaX
zVDx89W=<$|6n-YG4cW5Wzk`a>92oIux`3eAb;t|s>|?c^(_#sdx+~A!$bqueG=e8Z
zvD^>yhV#CutQi$3vj>CP=v%F2uBsTV$76z7PV$tuEg@5TumSxk{^Rs=nBmkee+$3l
zyy|!BHiJ99sIh@pRitxlu#Emv_pvv(8BwKYKrTB_Olz295y{T-PY!|xKdI#n8u0`5
zC^s8ru;<Bta?vD%xA^`3clI<9QlJ~+Krc`6(lZ#ZtBlv!GPTF>!6VKx>z-KF<(ME-
z-mfz-3qYfiJ-3@)MJWps1{&yq@3rX^>Z~3bzO+8wH_YYSBXMR-Kt>yrYTQX-a4?w-
zG9=ekLC9gC>cy8{!P8vo5{|P}5Yl2WzLvkp4si~Yf18T21SQ?qJ-NoNB2buYXmgj7
zQLv$VT>nb~*QP~t92)1vW&`a4&W+&^m-^2fwV;bomzJUdX!|+k=|{^C*sW0$39e&N
zfo@|EqHTWg*6`zM3oc)?i!x%r#`eDyL%07W9(x6{)RI&$`g9`Q(d1)lls?l&{^id=
z;>q)^k6pepZ6aaAnY%rR3a_IAn<1r_o5Cw%8P_j%*_g;O6UxLikm{YkH67j~=X8P_
zb*dMNq0D03`VM$NgA@VHW&2xN-lC*~Lfk08U2~=sVnW<C^`cf>q@RR9v<`-I%$l>(
zP7Rgs6}&GAw37dak0?npRQ2Ge>b7m_nrntcE6m7mpBA~~Mc5JTKG0?NZ1B4w3qrcr
zYQ=nN=QNm0i^?W-qnG>&g2uR;6eyR%-CFTNGYztD<IsWS=cfw>$SFpICJYtC;*1Av
z9~V}L_>x0laQHp?dz8WKR-3XnXRdmTnzGlsqYN?rQrXL4H%Z`=!s|;+;SAIxims0H
zrGd{TdRtOTOgj*CU0v=}UB{)+6Fao>&O6?EC(P>SudOq{#e?MKY4_&SL71{?(=nUP
zu=;M0HAmIIA1;q<FLVc#PY>c?U5`>doDiRD0%g0INcbh@MbVnx1+CUl$MLjMFJ`G1
zYSkCZgkBb9t9hX)$vTE!0gbZSq~LjjvKaQB|8xO?b+(lb%Q0HHmppGoLMFQ!EnQD`
zg))beRUm86N$=EqSshsRFqZwU?6THvO8w&GZgDhm#}t#k#zZeQleqpNgK#;+R|Kx_
z^Sdk#A^KQ0*;gmD=(H(&0O@sW8HNx+(STVIy$)3HGRBZ$oPlEY%?XlJL7aHEVuBsI
zk77yA205Pr#&x}Hx~pF9b~4rh(jI!8mKgGGz-BJ1;{o%^_pc^zaO5fT*GX5{3u$7C
z;nl&8j_K~BiAE<sT0!-j*8t-edL|tkaS_bSGhvg-hMs|`tP_Ow69{?>mj*^JKC+q{
z#!(D##q>@Bt8+!8)4~H79sx|Q`41m;5<;t$OZlBi!EH(IoNh3~BsmANnde0ZaVMdy
z0w~oNO7&8?#;Hy~544e1DiThPLyZpsivkVRUN=M)kf_|7&1bi}h)7GiFzvVUe=s<L
zD-YOPuu_Iup3baeJwO}`p+mU0?~iQ3)Te!yiD_W1tuhbKiXU8kT8Gy8u)7x3C+i=5
zsTR6eyD3`eC~xg-Tzxa<lG0{Yd@sl;v(sGd0ec;OJc=fl%$8QtKa&k))%5C&jm9WL
zKh*~ZHIT%~s-Qr59d>_z4S^a$pc-L0i&)O9!Rom)n716u(JYp)MW5ZD*(bpGlmFyV
zo+v_A!8+0!q&#cMqByb*DTu)2EOsC`>)7Ch^eEby_rlpx(OM`3p2qgX>aQx(Y9Iuv
zFE{itGJ9DY@6TTMX{iems#*ni4;lc23b|+BWfPNN?D1n{PT2YYW76O`DqukrH?4eC
zUQIu3lPyYgS3>uh{pHCpj^rl`Y`Z6PV85~OEn*wRm}doadG>f-Uxp9Kq)lnvV5mj)
z8?t6#Y#Sf*_K$KVXZLFn%dm_ECxdA#Zf=615Fr<BkNhg1x-$E2(W}^qnxzr42(m=y
zFE$kf{`$<&y!`Qs%zv?s2E+K!<BOkhjL$llkYy(>9?V?>b9)w<G@Z+LASaDj`=o0N
zs2qO(ZO18WK=;?Tu4N{nk2k_(e~j_D#9K!$z`)N8S-+tGUuR7M*J%>5FQPV%MKP>!
zMTs6bE2v*reURiEY0cAV-%%|d%%6UHxPI0;J|ZBS&bLCILP>^vJLL^)o%`~LX_lxF
z1?!i6r2svMTrByzY2yk8WP$AVhsOsn4AKWio(Y8UuIy#)PhX4IIViDCl+9xU9Talz
zI+QF3B{z}+S_~Xn?Y6ZKDQg(t$>Tf!vv~Md%#`mE$|g<gASh=L#YjdUmSYaxa)NFt
z^=b65>JO8Y`ZQ6eNOJr*E|pXNHc+aL#H|6<k(-0~+SeV!TvqoiS8|Q3*qA;L!0o&v
zDn=x_7`Z&_7lp)zjKPro7%EtGG>TzvVmVPHCK7jG&n=9xl}1`^T;BZ-SMbQa46!!Y
zJJwL-t^fn`j6y6UPX%w19Fk~o)x+hGu>1x^)fTb|zbx!zv!BfxR^IHu!J!WiZxpI_
zBL_DOJs8E8(QDyg)<hY%y@1tz_r@2St9$>87*O=yFY@OADqwsn82RmsH-fTpaILoB
zjt%mxA%T#vo`NHA;lnF60>Jf;Kk?rVv@ub|TkW$k!Q(1vvCQppMXkQvIs=Ksw#F(r
z)Z^8|h2jAXfcGNn6MR0jUIm<s^$)-^N0UpFFNtlPewCJR+SGAx+K$6gror_BkgFbT
zdhZ0PS(&v5Mm&@-hR^iXvvNTxC2a=T8`IK=T&Em)Yk43wcSY=Vq$$wjcR)P=HI$eW
zTWi=4o#lZJ+^RG_pqaa6j@vND(1uzd+E*|9hiCD>p62?z&cT1|J8~@=!OAPES@F<d
z4{{j@E;wPCYKaf;n>OOF#YeGF1g3C_Z(CEKjYe&^M1jBL@#J<ATX3`6GMNgB6SmSB
zN2kR?+D~OkNXJ#SeI6{02RkID1A%`*Hh7wKuh_6XoSOYAp*k=8ZKB}N#c5GRylzeK
zl&GXLV~0BOVq9JsS09d+ca$iCJ&}?@tmk4GR?yQY^@)oHz`ha&YCL^LqZfksIKF6W
zF^6*s9zwK%a*t>r+n`)JmP>bqlIsMkX0Ay6S9gEAh~ZWtW(<Ho!HxEuVJOp;z*zOi
zF{#S&R5%FtXp${Rl8aFT&IMaO`PH3&MdoUf<OCRN0j_n`)ZMBHw%JtB)*{xfteRYJ
z0`<!$L1TcsMLawkAEy`F6xbV&=TvPNyeT9_%)P5~QEWcs{p46crs1@Cap#&%B;O6x
zyiU96_U$>xtsR97vjGcRf$6C;9?wr3Dy94PdsUsD$dY3Z+&hr?+HCxBOCdm(l2TK1
zLgxKZ^VHh8GX?bgvf9nB$#B%}0oDIz6y=1WWWbCPxZ)k>+|Rk_+J$J;dJc-1gSykZ
z*0neZ*F}QsG*(+kT<M+Eoh%MQ-kP+%RS@ep6R%_y=>`KIh89g+-2w!;;EgrU#zdp?
z&&Ki}HG<cH?#a{HpFC114of*=D?n?RVYV+Od81%YwHXnVx7nnrj5w{Zc8)LDof<(B
zc(2ch&lkV*Y2=mRn@LJjU$!NY?jnsEf2M!2A``L{xS=_21)YsQsN_df!Tj_Ati6qm
zZ&S&MIfk>o+1RMSrUEXb07|bHreDqdFRy-AIdip2hgYc(s6f_Sm(4+>)5pvPqjI8{
z+k&yA6uwnH@A-PB@XaD=l1c!76H}NRuX91-y!1zp%jB5g4AyrP393rC!j2vF0etxw
zrvrYP>g@p0!UV-UAh1D<u`4nvewxAp7bQbQYttA-LRZtu<nq@Na^#<24b_t1e1xR5
zpFZO<^*>5iOrW&OA;|WdOF3ZY8n!7r7C=M>+cy(>g+s{4TGo^Zg%-meXc}Zi7PltL
zogb{-`VQUr1I+W_>4*NST(1ikl3#lUb5A8dRdIJDKapZ-6a+LK<1A(_)$}z|8Nohq
zB9$G?g`(!+sA{)qDR%a6u!0wElg$sX@GhWu|6OiTAO|OOd+Y@xn_qAGy}*8Pfu!3Z
z+y;n<@pvv)8yFf`#4ua&(!V-(@Zp3GjQQp2Hw*rYHw2xpORA0Hlg{5y(bxl#cfqhH
z<md|;^(rq<pNg?kX8(eZwwkkB4fd6}AP@9eF(zeqI6gA=zl?Vc?$ZhBBcL}wBiOTG
z_5!Tz_5X*K57lu)%jZ*rv@t=yr?qED)s!yrsLKB@Bqkmnszblxp&V`22#rl8;OG;=
z(DF`1SC96mzoRD@l+lx{!7S2X;?P^J=&dRht#N%LkOe*fTo)ufE-R_ke$#smfCc;u
zI^)m@NLZ1(ZLGhzYKI9iOPbhXusad<v3j7uf&~4tX_JjD>QqW%=}I8o!c{TO$V9zT
zL_lgi!Cj`sI5jHCxkf{T+}xr1CWmNZY~zIxZ%z8--jN|cV@JBIu@v|GxOmd(9{~y@
z3Nsi+=P@byfD;A~qT<poj3IHt<P9Td`P!fnJZPk^c48Qh+R~_|q>!qYrolP(|2zYe
z|CCalyn=d#ixR>7{c?ey?Pd8<A;F9UzGM*kElxx-;%^xHrn1m#`E+cjYjCq`@bYE>
znmt%MEk~lEvkS+dM$>WvwXIYMK_EZE!yW@CBqWpya!Yb6E<m;qAm_u1kc>dvus{;{
zuX%PjKr<=#z^rjp8DKogtWRo*UYE~JA`D;#z+}SS-Tlew2X6^&OWQV@WZJH@BY#ui
z^T84Rp`>%eoUMAk5WTGoP~~Jgh`t&9#Q!S+MGSNOivGkSu!(<l<q{&!rA%8v1@h(x
zU02_6Qb)M7eE*uI76P@FY6bqs1^Ecc(_3Mlsq~+&o;-pAtWdA9ua?Uu5#B%ne&S$%
z(vE+kr}7UXhI6Md!(`~YbQPmTGvV+V--&<c-B*~G{{Y|}PwIer&;0im8@KfUw`Jb<
zmZoV5PL{8oo<}|L7ZjNas79Z3DI>5k{@be2#=z0pirH*U0|6ad-^Q521C5_ijK2y{
zfQkSIIzGq+n@S1#*7JG6qr628S$0hcmU{n@b@$c8sqJ-9!MB&B+U)YeCpIH&9?SC`
zOKX*1^03KSf8Ca~Zy|kA(aXSZ3(!5)3u4Mjx-$T;cR3t-IuSR3`5nTTyb3MCZhe%h
zS*RjxWb1@za0M0ro=M6_M9BQi4XB(1#BILULh5Tl>18eE-veh<kYBwuC_Ykjz0b8l
zw)gMbZQ`LqT|pRhS{hGd@}N2Dj!aKgb`)t7S&pt7<0lCR*oKmr6=-8XfSm?Kbgvo-
zj<njl8o7|n1Gr&Dfo@co!2lIp?VE-Qpf2)+8$w|owuq_=6d~~T@|kqG>+KmJXPi$y
z`KGR3y{N6B+TO!{qRk>BB0D$vF5-F)EVy2megZk;L0=T43ryHki%B<ls)89TWI7;`
zALkSbu&@|Rvj8`@1*z${RC6W#IFU{g#?yK=*u6=Yp=0#RAj3MWRrKFy`{o}NxS3i(
zs%2`kZsy{2`wBX5UVY#;{&%!hrX;{RDxT!_j1A?iVZ2aWDjH|5RW=>%Y5BiTD?2Q3
zGXT|Yle(^fp4LH4!?Jf&hXm5L$<z+c*YXj#31DC&1);4q@Buovsxii3lDw+dO!9ls
z7&J``Kp8U__E*=Ntr-p>HjhW~fX-r@%2)(h%@u-!xBNKmZ{L@em~&NOwW61$>pBmq
zkPZZ|#lW0){X9b^{{=J9;70)<^SyB`pS4>s|3G@s@&hok=5-b-Ku#lYR@TMaK-BsT
zG9WM9mYrcpH|vQGo-Z1}A8-rmk_xVus%CMRB<>4<1M@y=`oC8M{9wC_pX1)=8ZzMO
z0lhfC{&RGEtMs#oiLb;z_R_Jix(#SD=ni+VX4Q|amvd}wAldnUYWg}C*hFg3>ac}W
zDtfvM%`D?iX$_QF;9)@I&})<?sN8-!n+u=~blnk|)xCA|DMGaB;)xIemq1&7zy!*S
zPxAVTE%G3b@=jr=YVJfzL?IZt2~;47I_s8S?2!u_T|&r<sP%ru=)^$ho4PxiBaX@M
z)kn76IePNRd8rX0i8@tgEBuly+da8RIAjosP^~VF{{&;V_j*5GOtNJs9vM|t5|szE
zRtT|>kc&0fXG!PH8*=G*?tCA99WBqDR`|y*dM{$g9LA56p$97sea{fk#eZLq@<5S*
z=aJB1#+(k0PUP>70j9D2JKiQ<DU!=uj65kn09J>~hI1F8HO?|4U<(Y+cshhnBb&^I
z`tvJ{RO>df`BT>%`H&>29rUUkxkve38A#0;Fgm#eYSc4Dx58l8OHO&VuMcYj|N9cT
z`Ti)L2~yJV%ecvSwkuf5RAQ@P@2ZuMLR?zB)M4Vlk8-&xjR!(|Q7&r2p%rpx+D*2z
z%NK>}1)-Z;3Fxn04yl#y_kj6#$bPS}@9M&go(M;`>X7fzSm@+N^`by0zJ^*uV#mOP
zr?Q9ekJJ8fQhQS*16b2w9J}vw%mk0d%Z83VqKsd7|9{sFEWpG^$qBsPSP-ek-jAi0
z147dprLy#m{vK=wk^#$3;12Y|;65+*fL92mpOw8xU)b=z*F<7e2-k)1k-!1Cuq&x*
zRUXv*hM~LAfIQe%0QYv{yn`SKYVpNhj;tC5w7QO4eY2EHl#^xuJSJ?EVMx1Y!yRhT
z`$}KTxVau1l{kOy-oe%jM@<Vf!pv;;s}kEpw|mU+OMkrVoi}f_-{U}=o5*z2KIgwo
zv)31(#2k4&!?>bZLJmS&CeI0&NOHgg>e6U#kjF4nWaeQZCGnraSz~>T+ukFB@j>I1
zdstTHvW>^zk7{oh$$)W6Y8kR&PwF=$mr1})qOt$jXS}T@$d8IxH-iWE&Y6H2-E-mq
zkNE5DqhhiD+iK=7_@q8SSNb3;HGSLO7hof4k&DP%fz|ePDg3x@YHTPK80;d?M-`}s
zo^%1N9_Ypf36WocNr(T+hJu`d0XL&w$89jER!%=X#(#e^Q>vs9j+lub!9pYrr-S?9
zM(1Ubn@TN}@vfi7*W(=z?3lWsws*mbAo9eR|8%v$F`P@Cmc~R^Q|(6jWdWKMj#Vo;
z>oeShHT&4LrU(OkcLAnIqN3c=Vor6TBc%B>F=u@3^6=N=rvjI+pjJfzV2TK&RuR}w
z)w}uNe_J41(Xc8~Z4jjb%d8kRqu?z6argqK)8~k~5b9W23Yo-NC3(sRlj}d9q2HDL
zznU;fgkQbea8?aY?*{NBk$YQaKsGxHk+xR!oC?bRv_j=hr|^?>ywP=r{z8X|ZW$P=
zq+56kQz%g}pc;_|d|*{BK|esTV>%?_yG2VTr(*u}wfQ@2pce8!oEQcn9`2~x=-d91
z<=d&p`P}P!W$k1SFolM4@}*zsvZ$ow$X6weR-l7v;5MHVdc9i}D|J%l`3m_{OOoCe
zOl5s?_!suEdR@2c7{Vlj1iWdcqmA<yjo+#Xfg8bHQm9EO&bk4C-c<$NREyI4^^eG~
z6$pcKH{b-0dk_xdd&H5Nl{%vYfc;L*)HN~4;UNgCbAA!t={dfc@9%*H1y2Yj1+v7|
z|0m-p2zg!=dR8wF0BkSrhhv4de_M7%Z(m6TWdLWUZgqQc=DbjwT2BVyc$&%wS}Fn9
zaW)F^^Hu0prZ8%alp;ie)^&^tj3C`$%7+Vzz7)%GzM~ibpg~-GN6f)7%EbDh{q;G=
zN7_s>Gf>UORemJ~Amoa{8MzHtOQpLdDY7pGb#;ym3W@GdFd^w$@rU_|S2}G(bjPm!
zh3^P-3(O7A-Q!IAmF~G2$16$dRNzLZ6=(qpvsTxjNkD(i^iX`dgtdzzz@yG6w$J4y
zEt^OqTj%*3TQ1}0Q9V*2Jq6Y56)$Yj-wEZqt0rks!lPf~wAo}5AWK!yozTk|kJ_nY
zwNJ3QV+O`3#YF-u8C{H=mL-%Y<wX7+3iT<~aUa!E$AdK3Sf8mUo4k|KokIq58b>z(
z*<^Xn3CtYYLh@=4E{DKndQ|IjU{+lQ0N+h0LIO%ydBswGmG+NROkjV8Hh{60bhw*1
z&R{2+uEBMoyeX@m-AMbT^ygMhSnQ(pp&4*5P6L7_>UxxNxPem-r3oh^II{@Ij-?7R
zkHt#H(RH|Ua~1fI)rwYv$_9ut0n$dZ3%Hvr<2cN`;SwEwg}@#o)X;gf8RZJ)<S<Xt
znsd<z8qVSrzh>Q_0Yu<N=cBMW99Egk?4zVFt{I@L<)0bw&D7`^zher=QNg+47i(a~
zL<fHBXOA_KyzzC09&;^0JNo-z?AG*ZK*<7cer`nB41Se&9x@%%RS!t1^2qFRopIre
zH^mLsX7^)dPrXe(E1*6U6CP!TQCmsec2zRxlSy!OFk&|{Gz-hlS>Er4><1OB!)js#
z<!4{zbOCuH{8SGJBG$mlML1Nu1t<__j(q>(XtUYp)ei@&cmLL53v?FL07c5%|Njn8
zP#`2qDfqB&IztIeZ@rp~qSUmBkmnm~>B%@`W0ImIBD8wN!)vsf6ha#P4QODa8v48%
z!Rt@8K=a-<1s7#CXs{Z#2Je;b`{&_iA`%c5X}1heGya|z2au2ErbH2+IC;>`9Em#(
zA%Tw6%0Ukfpglw->(ne>PUZ<hqrshn!JTvQn2qVpv6^w#oQ@Y~i@+I8=47sT27eOg
zC3K4E1dry^A<$C@$S)Wu#>abcT?I9z;@N&ghO?&!!&$(1rcyr4e`x8Ceck5L3udH)
zE6L_sj;XgXwbDNdyt%J&>$cinAxrjNkqjq42`M*D)pI&ECko_VQ*f(2g<s&HJTc%f
z#5e&TB-W2~t$R#wiB@i%hSU#%AjD;zc9~5Z(8^RsSCE900gGoIV@$%JCGt_zezm|n
zK;kJ<z@&lgxycLh3Nd@YVMQT>0Ydxb<-?L}-<4Xy<v#+o;CjAIK)(`LZAN!89cy|?
z))`?f9#O3|w+PC^YKu8Z;ai2FhFjA%MEnO-*qkkMo6M~j|A)LMh>BzwrxiSmGVY0L
zvL%fXmZi}^mvCDosEu}RdW?jkZlT3+03SANNSg=;$;xn@Xgoethqotc*%SeQAQsl)
zE`Zq?>OF>Hc}M^D-VDid9-<F^-l|b_Q+Wz+&i;jC*y_ZPGJhDB$A$D4$mYBFAfH9+
zu%>SRNr_4P{G94F9zOL#pIORXo4}zMIq1N~tk94j7k)sw^IX;(PeZkQ!-Fo#XcmUM
zHkSdO)o8kT{4%!<O)29H5GvKP(DNi#%F4K(LrRV%>(ww7MWTAT1mmf}_z3RQ3Why|
z@u1yY<-K1u67ZrSYx>IbQ8zEKGte6tPtF*Q=|5_oIe277-ynonMa<#Z70|Mpa>@(a
z>M8xKpJNbaF*Jx7-JFP;b4A^qcfFJ$fO@wI=d?j9CZmqjP8-xZxmRk@Cj~mAv(K`}
zH@=;$uvI`xh4mW}*pz@4K1Y)B+g{UgrqJ}5fJR#DgW<%$dnvZcj2g5e$gi|u(;jn^
zZnk1VJdl?J%qfZU8%O_z6Knq%@}qmp*W4=D>Zv-)Y+I-z`sy~aLc68BW1Zw*)}wSk
zDVjUBNLZ(^wGXlt4i`-ClsXRfAXDbw0-JZXZ+kixIw(Y-HAsJ6&skPvC;oyH55_%5
zJ*zeJzYJo;&$Dut!_MY`Av7@5i0T)1_uSvB)?+W~C|E9Q<!6<#?*;J*<;zSs>p8TL
zK2pFQ`MPC`WBf`+u$mCR$;Hf;U5F>aOJ%5~NDb@C^S%dfeCK^E#=jnQZM%3eQTnqW
zWaR;TeDQ-Gy@AT(N3~gpCNE-vd{BmX$iSW>@rV%~SXt3{5F6~o;oZX0q|&@;DSybk
zFT|DEwAgBriugF$@#9Z*?5QjOfVPBE&cbzk;K1|9`T|e(1dy4#%(ZB(xUE6X$Xqmz
z7L7^V9L2Fmlfj%xFr;Ugb@6PD`wX<5d`xxFgq8CF9^6pi0@vXGz-_|Oq()-_5zJhJ
zWjjQhmE9uP))22I<ET{5_@auppb?E?rh!_E*1>Js<G#~n(bHbPLo@sQXKiOWKe2R8
z*N4w%FK7hz%xSQ*a3yj==%RRpCgcT;GRc#bzOlE>K$))GWveebUp>l=DFk>pt!8mj
zP?MxgV?zlPU)#`r61PHU!ib-wY?u08V)evOQjl=zu|}qFW=VZV&34jVzzGB+U>v3a
z93=zvr9nd!X#JO=bT>j4!g2mgp~rL71~pwH@UjQ?lf>%9xSSM7tC9c1-ue86_X3-z
z&H{tVECR%LdXNQP7#HxD;6he3{f4&+fseaNv%>iiv3hsqUFZ*63Ul$C?Diau(;?{_
z5wLfe)0*p+m+e2wJ?xr`|Fun~``O<%{7$9rZSRk<*1dZB>4--JE}MX3BOV1`+~6sJ
zIWjxIoHZPohA<%^@j==eXnKcE@blgOV342!)G>cTDaEDInBs?^`X!SX`6T_f{eYM{
zzEU9hkUxlhYzGxMhPRUL)37Y1T`KFM+DnLmNj!|KqUi{94!fflyCFRJUjp^AuCkYh
zCCV=f(biccU}IO&f8tp`#nEp(2MbDKrzdT%=z@Kp&uMUHajb8mLae;^QnloWQK)S(
z$@V_&m#z8|i>I0EK-ZO^r(Zy+!BTFO#=c6X0J}3bAmvv3uwibqk23k-z}i?Xh`<ff
zsOvcMdz%8@L11Je?BWV+Cvs8<B2Q9{i&{2!yuKx81JHF*VjA78CTQY`gms04+RVh+
zcn~|5A8>asr?2H=0|S~1N6^ckQ8fJPE2#WS`8%Y99=&Q2X%$_TF(%&Q<%7gd(ZrBR
zIZCMw&K#G<w7z)cC0xEm33OVK*z-KNl@<7#n3gK4*M)@;Peur{$t$l+<j@Vz5oz>&
zUQ6)Vzp+VMhw2KT-k-TD=yeqxFJy=P*1E&%?T|EDoiOAZ8%ehJ>vBk|xJvl_)nJuG
zizN*O4*)d89FASaeN2YdSQ^Np9y6G=B<8>9VJ;ED{=C9{tN%5I_V>DTztAdBT6jzr
z-AtQ&(gCHbaGfdrIS@Ysbq-7qje3eu{0&2FxgbiB!yS%~bKiAnvpzOh{g_?lDTn&E
z!my}6aPqg95YO?kg0(rPgWJ8{tW&$zdOMgikLGMOh+A->L0$qXu2x^UwhGQF{I3ou
zjB4#L4;B3=qI$EAPyKlNq4M-l{zRPZS;-pp<5E`L(JyXl2smE2vR6k;q&5HJ7j;#r
zSfsJ`?sFn>Pmeo?vzF|Q;PXEXyzq~x8y5J7U9RK#EB$&`@yy3?sy#mF1zz)z=k<!o
zNZE#Ow;LOPKjx)3sfId|ZOsW4ln~(wXTx#12QAZl1THV0`=Y4ncfUZnyZX$z(>n5!
zZgzuF(9k+is7;!Fz;+Rr=jFm1yZl29^$*@xfmi7Y4;`6$`RJmH38xAD>~fn#X?H@d
z#^#~^Ze@d+dp1y31`KMje?;vw*yZWz$^lG!$!H)w%@H}Ap5OQM?4#Fi4YXqC{q(^f
zYNDz{q2rqZ?;r27c$D6w?vSDof5X{6tsNS;d-Ju|%cQ`KMe|GVw5>(d*9%VF-eW|(
z2XnUP<dsRu>6i4_a@uTikT<wP-~*VHLt%6ItEOr1PCxpKR(3nQK|NeJVn_J)r&fY5
z$S1B-?+4snVVv~cd`})~KQsCKssj$}^yB!l{_R^s#V1uQq;(W54%S>aP_r+`$;qzB
z-Q;bVvq`|6`bsiJ??FQh{eczfp|rlwt)YkObtkoAeJ2(=8$-W7&;Nevi+!cymzUG_
zXV4CIYyBhN?mId)56*^`o51RJc?%s3t5h%^S~gd&I2<FZqaN{;jp5QTzq`Nr`3wlg
zECXtV8tVjHz(?@TVsDnwt^Ti33qtP)-UeKrSLE}#J5q8_?XXr*2yVUo*cpRpoKwRN
zl@D<ryS<gXr%l;cJ`HF3FE@$A$Q$W@Krt38G@m}@k@UyPI<M8P`W1+pzd620{=S`G
z?0Vsyh%+QQ<QX4)9}Zx<;hL`LTd@YK3*o9895wL6R>iOjA(_4S<W3!ep~rPfgQ!Ny
zj&Aj;<=w!O%CG64|6LCJ1$%m`u4dTk3b;tB?%kP}RVC>b5HKADW!=74QjJ#VFFEov
zY~|#a-+@bC7x^XalRAry763JM%z6`!T2H8(oD$I$h&5J{^cJ{0o6Fp9`P%9AuZ<HQ
z1|sCYMf54IRd>cl9qL}tdf`;ZfQl|mN&c06nI&LdF7RwUQL(ifWCeCt=+B6j+o4}%
zBIoh@sR<}#y`*D*9t}iPkxe}F=Isd|<wrL7k#xBpR_lYC3!fc1YJHXfVXS%A9OaCy
zwA?3NVw0jJo1`K7aa+^e;-QeRtOS?QR>3z6>o+04__adJc6}d>exgmxvyrc&&2=}O
zdy$)J7%1X*?TgV8LVo~Sc?~Q#e59#hOSKW?XEIgT2oRP?(o|ElQ|Ve!(BV#tvA_=R
ziEAc}<yIMuX=epZxaET$3%CmcuG0+h+l`Fxw^!bOpL{3v<|*^d*PT>#hh6d*=lA(>
zM(?*rm~6@H9<O9}pF}NxJE{9BsPw>rmQo~l>cTGSjho;dkb;x75yu)y@`!z_$VYww
z?w*4&UUoq2-X*2qUK#{RG?o$c#SUnxx}Qoz4;fCiG_8$9Zr)~Bir*f4e`a#lH7PO+
zjgWI>bP&HijNNmua*E*tjty0s5ZQ1R8rt)=$+K1S8R4&(+pL`tB?lT3;+F=#nje-q
z_kyYgWCyr`lTlDL?4Zy2A)qX31gr|Xv*FnB^tNy1t+l@l&HUM$gHdP>1uIuHpAw`a
zaQt*q5aUBr#i(c1*L{4;FF6BcwP#|$P>6$T@VH6;6I#)&`X}DZm9m@%6<1p<w#C1)
zgwSVBBG+d&_Okcellu(E3vb{cyB?+nueX&b*r{t7=tS(Oh&?%g1}H=|!gxoM6yo4^
zko*T0Su4++Y7)A%lmTYD$sEAmc92uYF%^3aU_Y3kUKQ(AN@VHS<Ixib{rasI4V3MB
z$nT18H%5|K6kwVaKm=4+jSh{^{n!J~3_3QcS+M)zt<SNJJ+8YZsN%g2B3dj`?Q&7a
zrTOqEy;U5SW38uA+-GT!B_dk7(q%vrv%TpP@wL5^=Y17FR51>)Pm}8%8u(w7OMf=M
zzAV2=_m$Xhd)$t;9R0i~^j`jKn%?o$vYe^Hqw-%?iU;ry3ugB`L-VOM?+o<jzj)-$
ze5iQ)Lat*TG4Y1cY~ge9){}40XKM~GA)I;+emmm0Eu=rzVSOE&gddUQ3K;CBOF0%5
zTnS8h)qXGMfR5=2yY~n3>M|UV)}e=9d+$DfuCO#S>V$;l-gAyB5^^cV)+2}4pI=`5
zD@Sy@(eRYIw06%0Ez3<4%&9{&dTt3PI>Q2YXsg<Di6);aK8AZNT0@V^@2M0Gn``Y+
zYUojlimFkL5e<(N2g&Lz*~$bTbnOT}$ZodPiTr*HA%W>aNM_^656fpGW^?`CAFLs~
znS2{iy+$yNRBSv%b<<3PX*Vb5`UET7cb<MA+j#7lZ4?f&b*_AE^7UkPls<K_p@#_)
z&7cxa#|%4J9MF1XcevAC{(#{0QAkYHy}16GE#d>8;Li<HEvcgBs|aaKmmD^%x%f{I
z>k1f&@waLs=<T+>9J1J0n`W}*t@Lh+nU-bR@FR%PAYJ5xZ<pTfnb!EpPoMMe>xlUs
z$<Skw8wJy|(E!&NQMFq6`IUNpUku;SS+TR8)oab?XWTrA+B>YAHqH-e;tdnbALc#!
z_QqIgcMJoyWKXz%p-!LjQ&TYg$LOB?n$HEbm0x6bnW~phJUQvDz<;YlwC8PihbW#i
ztAW<@2r?y=?k5xvTo-ZXmq!9XG09`=p^kCg4z00?!;k7Ek1BQ_eAIn*5@3q#7LAg1
zjzho@Y6U*RcDm7iHzdbWZ+7p#zJW+di(M++iMQ4)B1Fie+ZWY^WS)JA%Q^y3H94b4
zB36p=kVzn{ZIcqu{jOd37=5FEc3&+J;!)Yz+5JM-H$@S&bE)qK^#p(=>s}Pm4a<J`
zEXCMH@7ndtD%~3*0-gN<<8#g7s^Z?;?A~76I55D}qxmRsOn$xz;OqA_;q&ANriM0T
zpDz7oW>lG3WOk{_U9|v`DH2$8`301CaWUF)0JoA+(YoE^nR{#Yaw{%Hv@$WvT-s;}
zy5B}lZRKE!dT96Xoe;NPfpD|lu5>fei_KZJ3%-g9Gr!<`y;1sn${MntFjvh~`(RQT
z8?#YG&39WOYB#~rS)wBBWtvMP1E#WsJ!Rv{e&D_iPVReqK<AvnX)XCZ(>5om+JT}u
z)Y##p!QMXyZbxOyO*LD`yZ-nxWsXa1&SACY$PhEM2ttR44=C0COf@jLi3_+#8=i7$
z+S13NCMI@GX67k1%*Y|67@HV!lh2V}F;uH7yU8a@_O*|ItWLE|>ih4yH~mTL<OG7h
zM4RiVNmSH|gxs#?WnJwH%VD~EwD)6U-A`$4-x+sUGI8p4u_ELYE?}VH+rsMXWJOg6
zN%7|LY{9>SHAFmYmy+q>^`AeU@9FDLG3;aPRNZ^ym7ASK$Xb(>{jtY=GAiAV`u6mR
zMqS8!iw836y}QTne0grQ%it&=jrbq3Gu#?-3JV!?I0_5B@X;#6G)n^DvuztQFpwL_
zdG+4k4+?57iMqNwH4KD(e{rN+GCnL~_F==DTccjMkUZ&z&QDiA_g+(tyA&lR-FqiK
zX!Uo6qDF(L=qRtgwO`oncXzmGigc~ibQw?TxLc3sxlXRlH|)?lmrKpEOj`_@*YYx5
z+2vMb<@`+FB-8DgpQPLgehlet@cwmVyb5;{!QD4>ckiTcms0mk`i$SK<b?m9eCic6
zI@-&%dP!X_#!H;{cwi0zyetubPG584${P)YJcL#cD$VUU_H8raWF3fe^B_NvNZo`4
zEYtE1`tfq<)oXn_w2Yn@`??u=`A(e`l3A0UP;~ol`$6cLsz@+D`>#02W`ABx?0V+z
zYnZDT2MMBXKjL?U-Rq};#2IWb5TO_^3HJvFPbHssp?BN$BYcls$hA$G3lnF3etCs`
z714#h^0i>1T=T-#whziL;OqK)W-6D`K8ELFdhd8V2;u+C5s+saa{Jq$wLSBzQf$5Y
z>+xvor;y$pOQ|;uSGG-QNKejBMeg|uEAHD9i>{QIkiQHc#|XVQkTpxwd)WWXn=$0k
zAQ92F>*ZaMw=WNm@9h@qdvxdQ?A~*-H5aYl8O`T;cNS|z*uUvd%3NPP;5Ww&njg5m
zu0GoIS)sx7N|G*Q_3B{((7l7_W<Kw=X|~*EM%j0+<imAMsR$D5@x3vno)}ceo7=yu
zN_RRvmIqHRfgR0f-Yrxk4-O|E4HPKUt<!DReLm<oDw&#0x3Kf36n&fF$DB|2wkGv&
z#A>_{a0st;=FAbT4~90JkE(mM<K65I0)<Ubkl6F)9a^*a&t?5m{31xh3qPOSswq`d
ztZd24NtM}u;>KRP4ez{zz^Qlht?0GKipfo`r7_<|j}_Jk!n2mRN|C^gCVhYL<k8Ca
z!R!->ZmwrpXh0=z3<p6j79Nf|`0ygF6$aV8_msfLu9ACwmsXrV821UpI$i}ZV}X8(
zi%Q@Au)BZ#4ot@H9Qzy=2TPO=(uiu+;wjQ?DEs)_55_i6+BDB7)OpEd0`PPT2zPRm
z2C7hNkI2yZ1T~dvOf9|BEBnNrVtv;XziR=+yquz6C)xqi4@O|-+6;i5AG{D7JXbIJ
zXQMva^(XG!{vLOf(tvTAfM8hUKGy+mvTa)5QSak#o~e6_NM4K5`vM8cRqBo+n(Z3-
z7M-i%r|-6hdzPK`d9MTAHigw~n)=vX4ZDd+WsPbx+Hj1PPltPL5%!=-z7RT0%AgmB
zMAF{)@UD}dDlJ0CbKj>aN27_VI>jfg9XB^}<QjHbsJh=<F1pQZnR{hKoCSX?OxI62
z-agD3x(cO7O&sD@@_i9!+yciNS)T`n8hy&pa5=xS(SnsAo{<<SuUj+`dM_tqmNpzR
zl;%B9-g6t5;=faIMG$c~=VZ-G*)W0IpTuygBSZI`SXPOLkF2?jG05Es#SOD~L$EyJ
z*ILf?!Wvj#op)D{QX@Yz_GP^5A!q#zAKmj!X45~V?H4z+u(zLSo&QMhYZPr9%)#xI
zPqxRdi60Xa+Y?LBw-_o@bFXa*)vcX+Y#mecs5s)3j>!e1Z%5YQF~%9Ef27$C7Kl>7
zse9(QckbEgxq95E{7*dAM)O67$GPr|I44lwrn!O;z14Kt(ewAYkBYm+ckQXH0#33j
zZ2Gt+1Y&Eq_vy1b0fdori=4lOy{#daMd2Z`f~Yo#Q{xUTskzjp4)v<zyi)A@>*(Vd
zvD(&`uSq?W;=+!&8x-EE7uaW?sjV*|e%(Q(yCn3p?LHv>K3lpDx0PH-)0=zibgAd`
zwfvjXmIf;&m#MxnO+2~g_VQ3+-rcR`;jCw#rbk>-j1w;W==i0OMnH{47;E0^&8*Xo
ze#J}A&%LuBYImA`DO{1fgEyl7{!Nt%9Q7!3D4%=(;F5DG5UfUIhqk6aRimjw_jMr|
z5jH?JAyyNgrS2H^cKBUrJi0J<^zpePwGYO$@jJA>V<mxeg8kibarltH?AudL3KHP`
z%%8C>$JQj1LYwZb-_W&fXK*V6v<fU*ZpMTDx+&(_hE0>m=TrOKx)C2j{C2PHbfMi8
zA`P#I%iIX_SKx%Z-%2oyV*PMG`uXx^VqyHsv5*b77VIY6;1DOAWfdiVD7hiBi1k3;
ztT0tWchTbZX)(t=d<T`Y0$GB5c}DdD+ZtZf_1pbY4l&LWH%d`hTyS<~1EpADQ~kCG
zBYwDi&w(s>%G|k22zm~NceK;{l)(#qAP0KcbW744M%{%Dbi_@T0FJ(K6SpDLR+Scv
zgv#Z22e(UrcL{H&D@nkANSn%ZG>LyW;P6RRqPRlb&Mxz#?9aIGrw-5i!43E22i!HY
zDhMyY>fd$N-E3WHoY=MN(=+!MamjJN&tEx}aY^a-Owl{yo_z{my(6#Ts*eV9-$YEx
z;!yVzZU{e&b5*Vhx^qDPPC&p3(+V?NGZ|-!or}FI!o|Az{HyI!QG+;y-FNp30gSnd
z=jUzQH*No&G;5#nMgNvU1p7=x9+{eX*Iw}CT>6pcb5tBR#I7h4LNGrHzTo#BIL9P3
z2J_{@(oLe?c)nPZE)Luzj=VfW{q2ekY3P9gQ_jCf?6IeO@NU>~yFF1Y?3vzq@pn32
zjsdK-mPWw12zEqOtek+qQ(`Ng?*^yif4<Xf<N<_lcP^kxV5J`q?cVVah^?p6ns=W9
z-Lcz=*H304#<c|`jbfo3T1k^IcI@}^&B{mj0suj>!J}+!ztUx*tIO$>Qh5eoH;LcL
zlh}hJ!?4k(<jEiOvOFRJiev?DU8|cD1k;R%?&YKij2;N<G}ov}d?f7~R0L4vkvi3r
z;){9S%{%KQdv{<HhF1<qKw0EE*F7lO%?k?359JTL8(e*9_v(_pz3wivNXoO4eY0U(
zJU(hsSra)&$%BWyaDUY(Ven<IS!Awx&WDFD-aoV{^}MqtQ9=Clxpz~!Y1J|fIkw~|
zHNF<>{}jNpC7b&On^|;9;~u!-((G}g|Ct34@7Z1K7rE~vYq@mPGmn^^?0MfeT6!E_
zxgBV_B4kn{zh7Mn>KX}M{_z+*0SzEeY{*X;b<pQ4nF}!J`}_;zi1XU2DmOvAT9*&x
z8L(bEO9>}g&DM%uxs;YS$4-y~@7gw8>AXJErdFw76`P8jduN=vtu<;Z_@wKhz*#d}
z)b<&AL8j&W&O<vz&tIS92n&7hoH99I?TW;?Mmz}pIw<+^48niR@5Gh5H69HosSCr%
zr*vr!GJ!^%r2QoO(YzA^;#k=ah#BC%`i80|sHmg{+|jo!S<^!|OS{hG9eMJ}Ycf&v
zrR}xQP>5HRXjmlmI5s`l%b3PGG^Idjy9APG4us6}n47vopT=Z%g|Oi{P(ncffl|}B
z<vQAo0jg-laV6M^9Ry7dtp)nY?kdnNRki(@=XZD8_7BsI`Cm(Yzuf1`Uj`T(Bl)nv
zJ#Mso$Hm9xaXY+f>2vB$-8-}*_ICEaGOd~t0+KIG(oN#!V8e$1$xXh8V!;Bt5Us@w
zQH?Z&Z)&n}e?7l3Nbafr>z*8o<c+J5d`eZPMxTtPan?1|1+~eTTe5|mGuk7$NpO07
z8LLCsg>lbwAeBhQq_&?*c28&r<*}x1H+$o~+Wn-$to<81x4#oR^sQCX?0mw&)2jn!
zu$EG*Y>k!P*ACX_L+&N1Z}!?4xeO0n_^7>l?tV;f`sEc`n!Ie0`HR;5D<2*iTq8<u
z_St!WS;UWLZ8;u@5j9XO%!Xx+vQ;Yee$V3$dnHCA9UlMIi&Z|?XTiEy{9v2t1V47F
zHZIy_%ld_l*@@~OBFDM!&uD(oVOEgO3ukomNW}7u%)tmi7=Y}4^wB@=<?0J1w=MXE
z==i2Pucqvr%5o1Nj1838QwO<(X?!H!fg1ew6wb-YG1Btj0c9-YXw2AGNwbs5`^r2U
zbaYRDnd;6}J9aO)`urGb?fxBRj;rfWakxYA;V<t83)f!z5yk&qNQ}vZ0c6Jap`#6L
zn!m95=Wg$L9(m#X{;2C|#QZ6t_p5<-1Lw=T^yWXDd^poe=0UA{x6?C<g9od|Umt>w
z`3*W$WMb|wn>626v=O4X6KoM7XC!@|#&MA6_VVv%c5J$)N2=riG6<rS*tH*Gc;@AA
zfa7?sD~LS*#A|=u5t_n#ynN7d`loAZqFg{s;ij}+xPS1>{VP>)8!phndC`7<9!M>^
z%J8L~r)S(#JMS)g)hC(0_1mOoJaZJl!ItM_;*G!EiLX4ttBMD&`F<|Fv_lJERW2Md
zz7+~g+$M*EONy?VZJVD1w7p>K&Q#nbhJ8xcp+EI<*wP!}`}wsD;`wJqsAFw7pe52Q
z`ps;<c)rP@J3}6t$8U!=S(MJS?kWAwSi_)iR<16%w#msM@onIzq8E=NKT;)kiI&GA
zK|D&-E35rCvOw*j)v*XWAArvi7BRkl4`Xc8!E9kDoF0GYntHvVyL3@=$45sS8=L!g
z<dIk(2e$OBmg|?tcTWH#+)V|7`;y1^(x8XmX?xtb5oi0ZKQ$o-i>f@&%3PUAfLl7B
zT1N_~#huuBy`-#nKd@^RpPF7Y;*n7H)wW;6TR$>>+n0Oajz-!UVT_%6$id%5fKC8j
zncut)$lPV;PiDnDJ1HieOwv9ywAVhmz&$T$KE*$qq&3X7Vwp~(!CdTc(Q`uGb-6=|
z<dobSVKK<r*S5;9BnasBTG{U|Hqq~brJhKx@QB!uJ-lsg9G`odwd!t8TMcOx*D1;B
zRrB~G2f)<70EfpLdEKe{XUqO;Usiab10RjnIW%JZ`1owQvVi#iqwB4Mq6*u;ae);U
z$ps{(mXZ!(rE_TkDFKo04gu+qT0pv_y9A^c6a=JGx|K#+x>n*n_&nct-tWxs_t%+$
zo!QxQ&UN3{^{M-LWcEE?+l~_`LI)uWs@)*rud+^k6{W}fYN_ez8|cLBl=<=Lnz8mm
z@#**0OYoLPcnlmhIrv$_?U36qI@#Nev0n4XiCV^WLC#r~|Fnd6E%)6{I6;WVc44IV
zx8>5hkndJlpNT7ASMq9!%p19?begYd4c;U{wiBSWOnw7EA_F<9Tm?4SYe=`gk4+y-
zbF=SxY_Fb5P2lJ5(#I|Qq+Y!w>c_N=J-{3s>l);#gVOH-g3NEb>pX7jrTo|a@z%9A
z3&pw?@I*!Jqu(TO5@ziLU93lO_ODtlO)r0c&|9&JN=;6;>u<TsU0L~g5sRlE{Xs28
zu8V}c1Z>$8P=_I+x<R2i&|5dbO%$5a)+Bm;p+**xbDg(eqO(ACf(*wc5b<}GCg-)y
zfO>@m)r4x!#dOD$WvI~nY)>*f5o{$h#I5rel@NWW*4v}wLB6<rkC6lLj00qOThwv#
z2+j%g67y=>d82O?04%HQM$~6y2SO*@2`F)mQW3lun7CjLC45T$r}0l})n9y~pf<-N
z7299Mjv7RA_`n&A<0m4cadW*<40=?j!*cx9AWBmAy@`q0ygz_^Fzy9bc4W4oi)q=5
zFj+b<)~hX_?V{8a(!k5i@Y0Tt6$B{T76S20<ETD76&00A(wt*oDf__XxW}aN$T`al
z4S_M;CiU)#RZ^<#<5yi^(pJ4oEAdxYPZ*62qk;W$1>4QBlzu3&kEeknm)G$&*7o{M
z4o4yOrftt61&H$VLxer$5@6sOIi)GJe8cg(uGA^?ZP$~>zYwqi!d5ymy0zX|wGiTC
zWBY!SNkw~-9Q&^5nAXihsltMLb%krHMi%W#hp!v;c#3@P)hb!rM=;^^%S9i?*i6%Y
zO0Uh_l#AG&_=`mge0Xtf&f^xAh_CP0qj^VE7q54&C~}3gGW^f$a`1fi8+3MhR!2^^
zC-grCief~_bwI~B8-M1TDples`@TnXt;T@fKaO|2<f!wMp6?f5`9&HecRsH?B5`*u
zsfDv1ghc}evhcgG`MpX#4T`{rRR5~>R_DlY+RC-)gwB-u;Gy^uo&Jf~p#MQx*N=u7
z!wkeo@|LCh<z3$$tN2Vg{g^KxYE_OeMpSoR5Qy6u0KiAC6Lb`~O>yf!FKM>pU<z0S
z64yL)Tx2Yxz@E!M_G_~2Te{@QIbwL?NSBx(zq9)1(Dzi33%XXX%F|!`d7nKsZ8a&&
ziGpNN&JRb&zT+{?6{*0iSz;Z06B@m1KJJVaWRr-UC06Pt?<iC{3~aRr#2K7vXPhb0
z`1$rB_tfRIkgR95<=z8X%!1qYkkJ<*qEP1~7VQjXuH{M0LumZVaN|T_RPIP#a}2ub
zt(j(jxc!(K8J>62$@OkW;(lpL7m6h|fQ#t($?SMbAyv83T2jE?>$|NjX$J1QS-Gv}
zj5Ne%o&(KPHK7w%p$?rj&3ojiURs%>7)mpxvUW3s&EI0rH3(zhGuyI!!|`ZgR@w=?
z@>gM0YcN%^VL$xj^o@Li7<`r|Fc>a5Owj(PWD8Z#EVrvxdT+Xtjdj1g4F^SRSFfU2
zKDxT$ba$ViM4O$7;2nyIVt6M~B$JPA?+{tc<GdAD9lxKEUzp*S5ti!oavzu6<PGHc
zr{}St9D$?lXrXsMaMX+zYL1u|ZvRnOKzW)GF8^kX>a`yZ67(uwEBXA=S~35sllC?)
z+ikCD1N9n!EBub!>>L={U$K^Cd)vNlx5$GvHz+TB(?_}jccvglNx$HY|J6qKK?kQc
z8(Yn<NAVW*)zbWrNmZJ>`tqOZyAxUj_X+jQH`sM5vSex1s!U@4&hMs13gk;Hc+uJK
zcASgtjJ)$uW|r(FvVL0dcDJ$-Q~dd?ZD=28J|<%S6^B<yciKHN%xa5f4(+3jLx&q1
zAAyJJqIi~W&l@K+7cPV2PPPhS6CCI{aO<?u)g<WQ6Ym3MCJA@&0Wl!I$1pXOhg}he
z`umeEQYP@MY6gtcL0gt0AAN0Fw9Je?v)i80NnKb#>r#U?D6BFRphH?nUy*U>#z8S^
z8Xj-9X?y{glh<&jyo38MY)8sM6SYjspMm*F33}Sy2Q?3pA8lrCiGGD4csj4Ny9gF{
zMZb+);Dp#+0U}q`eo=^1{V>G>bmLb=`1RD}WD@G9=UTK475}3%&k>hw`qr&^CN|(a
z#6FET8H7#u{1GV*ZVPuEh|&!l!F>gwKJieRYCmE#D_*U)%VK?JLuZUOQIZq!7X%BO
zL5)8MW@t8d)dEnO^C^#@m^loYEbMydDg_c4d$Vl)Cjn^rIOsNb;}MS)%;4#9&c9r#
zb?N6rNmK48|M6)bd#|DqZKS|*b8Q!@2tNb1qalyY?<>(5pU%~C*ioyWBNQZO)jjc=
zaQOmfoEpsUwZPW!oTatX50G`M-PKic-94x4+j$r3z@FMmnnMAvEt5qk^<AVZDe?m?
zgE#cA<kR1OWzK!{In55m?>B_EkSz{$m3OkhA_L&BeGNkY$oeNlHECo7VE)&?Lgm|0
zZG}cd2!g<v)6%H&amC4SKCkWX>@U+q&uSmnMcf7Y%`=Xxr@=glW>XC2tL<*eKj3@8
z#mhPgOnajZc5_;;m1(IcpE(d6*XN~X6`WQ8raS#lXw$>Fyj|^(AhuCZ1vBRZ>(tUJ
zheYZkbP^HtsN3{JjqL>zF?&rG+I5HqMB6?CwWdtQD!$HQnK&xXlJcm$3MbC{ssx|v
zIJWgHmga3TwFYz>7tvd&ko%<o0!qJIu<)IYmHmpQPr@)jZmQ%WjGOpm9sl@6jjsH+
zfui}QXmyPUH8e4Dr=-mDu4l3hltxt|cFoT1t!6T9x4c@<K*^^wVRx_A7S;vFd{!m@
zcGuOq+?21V5paW93|g%drSZNPNYoj8tUl-(4TwTpX%K8Z?|=4zy8g9to7OLG`)LWH
zIH>;#O*U2b<?G&_nfa)he>?Cp*i8qdYraw}8fDW544Gp)5x=}+v*IqJXQlZ>2%x%d
z#Uub#>KQ{o8h!xG%#OTHB38c?vY%PSRGzmmjM{)Dnd!%;TD(p)bp4e-mQJ#+pu9SA
zE_fj5x|yZgdmYRNnn>vT7sjF8S_lS?3I>iBGGIx%&aNdbRy8=f0+=IPKL{?#h|OtP
z$McX?+7x)bsy55kM5S?19xjbPVKgmObK(l?4@eVdd)k$lK$KLKK?*n=;uhG9gdU~x
zwIB@A;9}LR^m$Lp>(wZ3Rj<FU!1R~|&4H+YCsL-A$~Cbkgwn~aW2;&6$w9Fr7?Vrn
zsY93*m^?(dwrhEi=KTqW0C*uLwvzqLvQXX@oBy=@RIH%e-`;wc?3f68i<mtFRy>}c
z9t#e(w$DSa(S?HSK5X%P;1%Ig9uM;E_EDCeqyRfBEW%V<X6pTjXCCKJ&0`kO90>J^
zq3(KxHvBTaAtN$RE1mqx;pu&bOPNLr&%JXtWS<3H_qY#Rk(x4asyqIx*x2aB&Ze}3
zy}3FUJ))lBYdVNhEht?s0Mq(Sa`E!;9q4z@ndwsRn>xZqLLRB`5npA6^=Gdk9iD2^
zxl$Z8F}oIkkfLm{m2eDk(%&$Md=Jw|yOGe`_le$;^fiq7&~qoEaIgWC)<)bBN{Mi}
z5og>QeK?6#bNnuO&q+YM?$X{e)Au=WVoe(>J=IAT!lm)-E@^+9`NhXL#K$<>%O{(E
z@$L$MNvH-$1@fi4$Ifw2JACvY$67izkI*tk^3E0b39}-$dYokYp3U8rB%|Jn0l?9g
z*gXw*{Pqg4oj1!A%J`QbiP5Yy9{{Kh`#VD{6&l@@JIW>6D~Xxpd<TrVj%0+50jqt{
z;DOY1@ky(`4sZm4P^RDcT$wWW&yI)^fhLn?J{so-y1Q04IihxVVD~2(8CqS5y2+%|
zdUPqXEvSeGG886LA4SFcCSqQO2iU*l7MnA9^Fij;MqWo;J4RcvklQE;a$5VgkVa&d
zWomYd-N253N-uyd5&Ei4SF=|~sH~trft8utSfEV9Z0Kb#y|F#j4L_^_gUt&cPDbXH
z<#r4f7RDBa%+-v>5Ezgjla;?lpa!$A%Q>jtxUFx5dxX^51FDT{yNYpjC8J~8REdG1
ztX{|TRZ`V={HADjD4I|nuU>`Ri07P<&$v1?p0p@Da|s!adpT0bTHa8)@Fq6P+|JC?
zb~<rotM@nK{%a%0W@q17i<$j`5$ospct(-pGTAk8&~BH<C+7kXE9>YrrLV&}Kie!h
zaKnu*KWJA9w{0@5bu<~^)EL2oD>GK{a!E02FgUCXgk79}ioOpH5fw3W^gMk_O=UEe
z&?bD}j{`CZ5U?7IO?)*FFeJ^1_&jR8BF6;E|4amxx<<{k{b76e_-YJ2A0YbQ4<_#f
zK7io;5Z<k*x0ag3&R$nrvS6<tHihC0n)Q58jAClwvhAaeDuN;<Re_`<pUFum037&f
zc0FdJFR<39rAiLWFberKLJWP);{?i&{*};AWIZ2QUkm6M9_G8#8OuqC>l}lOF5z^(
z)q!?JMCf=BcHKA91F~g#(Y(B{E;&PElXQd-Sj|-?mk=AOjXW8DijLPQydEb*v<;7c
zdK~s><$a}A{nMNh41eRclOG}_OK-e{C&m8m)vcu4eeREDGXZJ=)u@vrpPNU(NJffZ
zS6Fktr#-LgRw$Z}ZvMc5KADNvc!5Z=cE(9;@h#yh+#Y?&HLBa=Nz0}^h|=-W_HP+-
zu`xxV%WHbr;P)IU(6AoRD*MvAYJ9OvIywAy;1`?62>dL{8;V>P?L_I<W@Fg&=%ck3
z-TU8S0ieAd&pl+sYjF*6GgKSW9!}%^6tX;O@}1h0p~GF0G4JYRv&5B&yMf5s^FTL&
z_~5ZZxB~<Pus{72E?KT%l1D$S>^XwM?wYS(O<L7cv*Y+pwOrUQDIK?rG-#(!ZTj_f
zSaQ1V)$|r0*lhrn36BN9vctbicjB^ex8oO_Pkm_Rb1&$ZLFoe{W&c;b;U081@k_l)
zzFVwH9yoHST2S2Ivm!>`1>&bU@QB-;L!$1MKqm+zFUA9Z$69`L(^x=vpEu*Uw#dUC
zp&R@;!~+{y!*pkx7y#_8d%3~ltQOk?Sgf$J)SHi-5vQf?m|vf)3lR>CCyMR#mshl0
zPZ;WaceaY$TfpXVk<`ozR8KpKBa{4uC8+u+mkzy5cOH~Et64um7SFj+!_;hBP~p8(
zyI<drbSomuNLbb+tEJ94lQQ(*GGE}gwA0z2hPh*`@s^DN&j(yc^_x)gwF0BtX)yVC
zKHq!W?PnR3iPQ>oHJufD&lrL%9S38yQ-I2np0#LH$o>A%x;IHdFyp$Bzcf)L(%PJ@
zx4-L|_GG%y+MGgenvqt;(b-w*2Z9B)w2zK*jxW~iqP6uVa6j``o6=m9ul9HXC8c`#
zcat<}|IhzmSKn~Zkjs}tR^k&yi96-MTpBYMmM;=PZ2fg&ZcD_Bz5^>q47~8>`g&7B
zP3=Kr_ya~f(&WZP@7+USJ1Y(~;*3GgJg>@9K!dZ#Nh;`3H}Y3%)!ujwG(C|DDJj&u
zOqI+HQV7;X$J4KnbnGW4^?eM<dqV;`vr}snL&uAMvPa4Us2p2{iGPcE?9EP^H_n^*
ze5~@_U@zhtO6B}8iyy@DZQ`I^!*2Hl12#V;-dn-j%QfOL(DAEmLhWeTOXTm)D~VME
zg9bc%tU$hG$wg29)M^15UpMK9%@5CIl}}G-)o9jM5kil4+l{Z?QiM}M{i+WY^~S`G
z75F%`q+dwR%5g3};lS&3Tr!WaIrAQgn8o-FfAaL*pN|+_!XRqp+#vXOfk=+m^-mqB
z5+0?IEh@u@rE1r|Ai@09UU#z>&qIj%=r%&03V%g%1{IIxpuyAn97|H3V~Hz&3=SD#
za45GZK;KSZeTE{=xrA@Gyna6hg!FA!157T9X6hSP#i$GD9edL);pb4jI!h)fK77<Q
z=dYqr>!ebyNjbADqKienWj?CuH;6Ltj}0eKSD}-YPs*xEGp5efCsSJdqoJdR6a-$)
zKkp)2YUYJ$=II7rq`*5<a(#Q3eIVxRulMzhvflwB$i<*Qpo}D9?|J<4ieh4GZ%|Me
zm|v|om^0H&My|a1b(m%7$4*zYr}|=UWouY?WvCu|qJbx~fdN4;d&2A^Ql+TQYNaA$
zu6ymv2)y9Jmn>DS^?Lg28(7r#rc`5sqw2IpdUvNqD$oGR0tb0D@j|22N&H+c%xDM>
zu77PSzsc9QnXa)l#s_?fVfChZ8ud6&=CtSDzsQj6gjqz`m%a$d@^lkg4~Y8M+r~7~
zm$RqA>~vFJ!K)#+C#W?oY9a4e)+=FV;)_7lQC#8k{=9UeY7Cu-alt8Ra+hv_#KrKp
zsBp{1<yAO$e0(iuD$~|?so+p8Avg^Tw-wIwZNIIO%GnLyS^n&>X#@7UAkGBSixyhG
z4Y0s8)tZ(4%__w#xs&kS_DZPxY=_N4Q>dmU@KNKR9or47v#xZtYqLr6ywRU@lXIi=
z1s!N)$mOqpOYl7(I^@iRecurJ$3l4vp~pR)d$a-AYkktU+iuUQ+R-CPJw(h<94{Li
zXGbag>tK4BW2OqoN*W8K?C3PttXX_$==2{(V5U&DDscJ`8g~^Zxp&S-h(3!SexYE<
zh>Mf}RM(3Mq_Dpi2RR*Qb(|2B3lZpt(3e*V9wyC1&G(9u*OyiGZPA%%F6wIP`sjFS
zf(r;jk0)_-x*<}v>OvJXUg80>!kGOG+_MkQTekRq)?2K6du`&<=g2J$LIB)<Q!<6W
z{*g$MwfMWAF7l4E`+p>JYcRsi+ui$onbop?z6W)t=r8M*NyZ4Cj>>dqES?s_`PC(y
zO_VwQidTcuZIJ;GMbJW)Y%aYiR=ln!_u(Q3i`R^?&@uQfm0tYthH;^a=HLs!8^N_q
zD}2unPXqKjbU#vU_7GXq=>l&x^6PJ;3NYYG2FeVW<i<3#2Jl<%@w6cobW-}vvH7EK
zajSDiIvBWz<?d>3_Rfu=?UK05VzON<(0r+$NthdaC+GpDy2@DS`rEsk3$>-xFI;uf
zA)NW@0PEW}5RI=1i0FU-4lTOlw2&I9a~m~)R;H0&TqH$Dy|7~p!h~Y$`~6lFLdRPo
zre8c<yaA?H^+-_*heHXn_dkK=z`z5i#}#5-Ez>m}a~<Pz58o>YLdPUC>Dfrv=sQ#v
z#jHg9M??U5?UHmZFk(l7>=e&QwOhH5DSpOdX;pl)J-$yJoJc8{I*tEKY#jnW1GY79
zk|0Y5(awsK6SI)umg$=n+IH;GM>U>!sFj4uMvLR|sJU0L(nQ+5{3vc;jj(sdS7ZQe
zWRa*P&hM`;M@zkJXRl<FO1n#|pTv15v#(^KbF1YvI6Fh$KcmfG_LWkg=YbJRI7vT>
z1YD2yAaN6vpc4R;G+jeJah^9Sp5>vhojh`%yl<g(TqoeF-Ymi(VN{0&hiZKzcoNl1
z<320JkBN#;<-9JVWwk){wV~>ar0C^xzkGw}3uZ0VI|zCVtd9m*|5SX0jz=`H%E+H=
z+F&?^X@SYw+Vh%{(6PGmHHD+^YMJxc&&^18fmXxQ?_(<=ok#+VF#H&OMp=fVc9p_U
zNk0W*g=b`3DsWkUzx<%srtng)+1CBC+h8Q@q{=8^p=x|Uk(Pow9Y2E{vpa5(iXjM7
zQ36Sn;^`!x-ka1j<%^-Pmf^SA!t-}NCc29HVDX+cyC2zy37794KY#e822mY1yee&#
zVEvT><W7qpHN#PBJBDZ44$L#Ve@Y54vm__fLEi!dz=T3!8T|HCd`^ok^~qHFg|neV
zq|j6@gLF~>dJai91~<-*3xoW(fK{-~=B-rIIIq<Erpwi+L~kaXZM|k>fDgoEYcNIN
ztBImH8aPU!2-s9RWIcw6v=iu0Oc@r<$!^u{0BT{=atN6*%y8};L$7!E<s(ZRV|%bQ
z{MA>gpq`lMFNXmMU?%W_&=I2-rz@a0uLo1(9c=(iDqGr442ZJzd9}(%gJo_!L6$_O
zj@<Wn2pU35`{wBE)0wtgfp~9=al_Px)=hgYid`LX`vLNZE_k{hEkpoITCG7kDe)l=
zX`v&a9DDg`<HoFDKyq=}Pfv9gGqz6(!8EE{F?cpyUNvL&0B6-#wSmQ6s^{GKRlPut
z_{?0w2B2+JV~ezglTa$~X3pFrz-4&vDK}97HG>Vb(Vqp<j%Vpj#|jk_BQL!9C}-b4
z^#art4c28TlFuCG>)jSDLN5CM;kp^X%Re8|OSK@C`|>Gcv>cJOW7A65_`9M#=Rt_I
zr^N1G$eqMvp%*v3v9vkJ(EK5pCvdbO22!MKZV<noV}|OL4Y!59G9n%j*ki*Bg8A|0
z(nj$ABUj--KVu%1%%<nF*3?XNpI{q#M+ar`5V+S(g`P^FNn7Yd{@eY2XC0+~q#}@J
z0utkDBR#PSIl!I7xzR}v&HTR5#pcRn?axT0Si~lIoHEYw`3Z5%w4-zkTNRUMn#h6U
zrX~Ewid6heVbStt(!J+1f?J*OXSC%?NOQ7<4(XfZd?TH)_FdchIdWnm@)rZ|tys$X
z*(_>_PAmHGS;AWO635lb?8_?rKVau^WbFxV80~K*%9~@|A+9x@ZWa=4{PKgjpc%&u
z-zoy;i<BjNd#bsEE+>eo8bT+q&5)0*4!)Yr^8G3``I{#Q|Hx(o+h)R7vl)VIKA*B5
z=oTTnmQD6I*}&lra~k^i*dN){G$J5o7k@#^5B2?hW7dg??<6+uUgCqGR*m>=uAisc
z8*DT3B4@qK7)N+|Hn>}#w4P6du9O7m8V*(~`60#3J!g-)uuZy3<0B*!d%|gjxNH*g
z5{_WX#j~-MMXSrOD4ie2WQLFaL#2!ij-OB7i)a7{J(jNhJzA6r#~XV4y6+vwU)^_6
zVDz&LOEvlt9y6BF$%B^WA|xabw=*9N2NRBK%>JCyaIbzsb-OOyA#fJP$j;$<lIFJL
zchP_tik~V+;>OF58S1uJ+p(vLE_l#q-@=4tvs<IV9m8z06lF=07}&DvkXm)J+FMVj
zfE64gg(<J~m~$}yPje>e;B|n*Y^il9W`9+ofFt9jWWOuZMTFo5fYY~8U}I5%ALro6
zA*s7RFLF(MeW%$62rPOC&?-p309s!31B@=Y;oP6p*!(H$9pq$OZ^-GIB|k9aW6i+e
zf)Jtsh<1}ym4buO9crc%a6xVwGc-H@(mS^Kw)hsR%i|%QI>1R(?T5e4#rT(|5&rPE
z>S??Z=tI+Dyb_qdR&V%k_<<9laU2J|@_7;)AC84`?lyi78l_nSt2sp<7Z@@^*LOJA
zDYX9V6yPp!n)i|VHc?MW7b0T7X-{hO$BH6{2HS%$I{p9dr$QA}()gI-r7It)Imu+E
zXvyT#M`DJrr?y>NESGBSx%W4)=UW!t7*shiynISR^dOpXv|g5!T|PI6yrkwtg1=ep
zXtGSRbIvNqK={jM=3CW|LT@e0DqiT{uRXtjZ_ua|5(A{$6BnUAf4;E9gRb7Vy4*i(
zx+uwS*!+=yy5Bj-N)4kz?3+f!{}vvcME*#f+ODW2wti&e^Ah>?j}qED|ARyQ-|um_
zwv=X8?u`T9ePN~balInY^E=V8UwdMqqRrP_#v+D)-q8Y>ju#tV)!GFnK|;4f4E?Am
zH38CjRlcwZgEwOubtd~h34>Y?eW0L9INrikJIX^4W2+AFLMhD`maJ;8B;6bW!}mRN
z`mu2PL2+#arEWB=a<d-G=b&T;!`^>Y>oJ)#dER)t@yu>;zEUW*;0KWk`rjAQtsH&p
z+D>O(^%~Ueu3KH|1az7FAuzbOh2{hOJ7L8wYmTzFzVX@L2;I@QdOeh8C;QtaJZf*Z
zgZtV}3zbE}&(x^GD|IMDj%n)@=|*3~JTG^2=pt3s48wn9G;gv{Js5MuO#i$;BY;N|
zCIuq&VvZ-fV4Aux`>IamF#xFZiR&!GPOK!zvUR&R758g*LxNVu6Aj5cmc+z??30qG
z{KLOb*DNQR_~%XWY}dy>riZ8eP|93bi{-|>^ZJ#*V<rx1v`x(;#tNfWqkW8AswDhH
z(yJ4~B|Pao|BMzZ4JG%@?Ge-;$7%>$&EP~@SD?yl&2FTj5olrV(H1u~&}Exs-z7jo
z7o*iOJ=Av*j3_=nNoE=m1sXt;ADKeQR9mKiY~77_*>92>!ra6<4p0P+mTv&Y+r~u<
z2g|!9yq+2t=a9ob(c=}IL)rG8ADm@-q~!5ou6EyvLD~GQb|G(;)18fJwEFCUc${kp
z=&C_orJXP#Fc~P=t@$r{9kC6Xb~O1U3lz@5QFp*F>RMBjSXhqDPl!J|PM9WO_lk~B
z-p8~bD`gLjK6#fqh`eJW(;?y;CPFp;<VDUOh1?p@!`oh|kp4$w1)fz58=rL)Wa7dJ
z^VyEfImTd3%yec_{(X_(xa|B!q)1HSAKX(&FC$;Xow+Z1Z8GIQJkvwRQ&*cE$MHJu
zXHkZzYRHCiPZ3&I^q((E5X$Km$|Bq(-NHnr<i~WZ8$px<q`Pvym^@*xyNFuolp*u9
z=tN_wUQSA<w;S_o-oYfnS|U17-m!41UkGG`Qs_N#^V`A6M?X@b*#G&G@7u`0LRY4H
zxI?PneSjV)PK`uj>(>lg+~_KHgNAGOZH=TIXh$Vi-HjXibP-UZS5o(+W8e$Y`B~Y3
z$^bh<_g&i;mSP+>prd(6r?U-}$z3VtgIF`}niYGQv~*z^-_X*av9fouv8zSC<c74p
zr{a6EH7k5fG+&r{7OUdc-8M>29t+HPMDx|bPq~{XEpC)kVIi(Q*95yc;z>;DSQf`%
z-Pc)Kda<c{%ey9;FgLW9KQa*W?&>^JDq1XI{^(uVpq)bHZ*GF$1D)K>cpD02npNp;
z;$3fT3SxatG|h@tZ?xo<vKnHud&ugF#3yZH&hxu41xvXsm2Yhlxlz5}5<5cV99BsD
z!qsAEsa5AKT*;fJ5Go4EAK7qumD<2)V>YY&rnsq>U$GOk28UnWntrvsItJ94t=o}S
zJ9RgsB{38f#ncUc<h2{s0FHS}p=4S{(yRVNo6^EtJ&iv)8Q;eFNC4V~+R$G4+ong}
z%6n6!k|i9M;du=>7v8Di%E;r!s#`~nS&<u2Vtq~oSH0k6%OeX?P8)_Taa6p%b@fCQ
ze=)h`PJyxe{0}2Z1~FC-V>!hazg*_`Ln@oNM1e8692nw&iUCp)OdP2enCl%X?Op><
zMnLewg4<UOkOQvCT>pyi^RzcB+*kgJ!6h2q2tbTQdDWqWjOn8E>4ShNHai``pNDt=
zr(5|Jdwp{dK#*$aZ8KnHG>g9uWZ#>8`XLuvvyR<y#m=TWjhQpTaT)0>JqgM-{_)5g
zeo4U#z5Nk++$(Wg^O&NG$QnVE@XIyCUvx~Ri>|QJMdc$;UOgTe96er^uDv>$q)mO6
zviee=Esg!#8{ea11qxrw`Rb_@$%0B<CF=5_Sp_KE{G}Gej=9-F#V2YDu3FOtf$Bf4
zMf_UaJnE(8c6e-gGuP5B>_hD?j~@Li=0&{Oo^PKFkqBz5K+_$VuA<)@z=Ng3E5qUG
zSHO%+MJHii1n?2f)RgBZi^?e>@9}4<qWLUFBtR0M$rQ-FTt}s_ia4p$7(%EO^+D+|
zEdhIHG2D)kGBGtBm-HT6Cm~3VpdQ22=u6z{wMym@yv_Tp2^&XWX4bF7@tkzlil>XK
zB3uybSxcUSy08&Uun9Pa)+@gFUYmE?hEEOUaS>Ah+&|UMAXeqgU<I<Fyw`x=t^x6-
zRTgMX3jsixgRQ#-s6w+<f%}=l>z{4Mo?dD9x$ed{n2mJp^hKvDWB_I}BVkogI<@3&
zK?-<!C{zu~EWrHoa&2{MHGDK2=dWo%7Dw2t(oRI>GE~yk=p_%1qo#lk(z@{)yt6{G
ztm(gQbW<vi_5HJb8Lj&%eKG(^ZYPvv`PbVGo$65U*Gh?aAFD+=P$!Kw+!it1y?XwM
zS-;So?1_5{kJQ{uA*<c$B<l$IN74OE<KsZ{phA^pVr)z?6prY8a;n2TWB}bjoK;4P
zzKA*lOh)H*Rif%qG)SsMSu($FWH#=r#Vt16kLnG-Hq!x77S)@Di-T{4n7gUw8jna+
z8O$#LJ8#;WT=;Y1Pu}gME`b?HI;91gZYi7NZml9+TBj}y1&!r@a`g(9&W(7{lS?3>
zGzUWJ!;kl9v&CoEx!wPsE`$sq43DwXFz;$!GM<w7@O<T`!+TbIsM$0C>!Gp;@b)yh
zM4c9`)O<OkSgP)5=b=xq?m5s$JYQSgj>Ry2p|Jhi?d=y`{U-$4Us*BS>{m{AQ0^wT
zowCP6146BmY%QYh|9Z|UX!|O=nmeuMau7DUW4^0Qw%D4%F{CfSiU#--2?}P^{C83~
zsQ7PY)S@s^cBuqVpc5Ap_vxm;97Q7DwX9LE0D3X5?-G8f({^I-SGDSB2n}>>%pf}B
zg8ub*^+kYB$0FThn&T7#cCXPT$Ft6N{?Y_iozBGA7O`UIam}Y**-V^s;3pekCOE=G
zpIworKQN;?)xbj!ij|AyA`e14opZ(N*Kgkp7@pEGG8peSU7DPnd>=PC{Ys=?k%~Lz
z<x9s_w@u2>)8OjI5S%@rld@-9NTv3erFXdNm^SY_zU09&DP@9B->c#^;!nDM-fL;f
zzQ_)=c*ZIqd@;5a^JV=9yk=5QYs;c(OsmwlYgC5r<E!dt30MK-q@gjHh!qnz4&Y8j
zBui`1^*$UxP`?1C;NJASQr(HK{EMC&tcl?F3Q6*A*=#!hcv7lfpw!8%F)^Vgk%c5(
z0FI8(cNk0r?f~RHV7CoUz(WylGf{Cyjo@T&$=0g7&qGJuPH)U^R;ldiv872&{NrTZ
z{H+0M^qx8-k=I^eKPZT|89Y?Ep)b963KT^G&Q`um&((+`zBG=YXyN85@=Xd?+yzO!
z8b*q~i6kZ@=YiPJJ6`cdcX3a?pN@GfBFTjSqzPi0VGWned#RuTG_%h8#1TAdo(C3f
zC%IVhiIwyan%hy*lrm_mbvb78RmG1e!ue@Z)fRc6wd74Z{5PotsmcPoXL9E*KseN}
zl>kmr*FOHwi-gty6zCy3-5p$HpdpBUO&-BMQl={LVp$??Gb{|j=Mk#wZ4)85(>1j+
znZxJD`Zkv(|B)BI3tlz*ostECBf8hs%%!O|(Y8q}%$NRHD$RRfS^_xp(EMhCx%pIz
zJoR@0Y$ute;g)r+?oU4U^sj1O1i{r9qgDyu&N$i=<_g6H?^90C>6qgkD(ytPQ;T32
z{Ohb{1{dZj<f?>b1yoRl6X-9jSj#|F!+OTjoo(OOwqF7}n+h%TDmA#RkzW|~B%y2w
z)jZ8wzxR-CFE|l<Jbe)f=Is*XA0~?Stg9jSVt%ajoFqZXw4@7#R~0t*Hv7?x=wFQ{
zi+)#<U1s@%Mhn5U!^+O}EF{hiEVkssvu;9@Ps=$WpH9d3QhZ6`t)xHTpR5nbu86c5
zP&#@)y^E+fjomSlZ?hQW`D36T>hO++7Pin55Se6lWR}SpHvNI(vMB25kL@X)=OLSx
z;>4ssTR1qpa$DER{pA)r|2sD})3vV>L5Q1H5QiW!wLOuD;xmeY1gm=w@Xw6qN*`D&
zFwaLzV14rf*$Wkl6*+y~E<QZ$BzQNtJ&z#8jeDp;YJAD1GD$&d(omg7R`eTiohmH^
zTf4$(jg<+_XAIss)F3jLUgv7!1%*8Yix%~G=zz^ameE*4N;c^t35R7m#4k8FEFQXI
zY|5kS0bJO*K_iY8d|iE=+KnEc_xW#Lz*xdBVN&P1G2jf?m<G(N7ZlWm?vW!wys%Jd
zdP$CP-Lzf`tIYW7RJ2=y`(t&~d1arPWD|-VLlcuo-tr|AS;q+Plu9yEV6T~qu=K<j
zk)F}TNJ@^jj5T!857gH%CHK)yN@W{g*IA9IGUC`f`ytFj@MjD?nk#?Mjn3{TA7;EW
z&@@tQQg<k*kyN@|Sh`Kgf5FT0881o=nASe)FUq`XDvk)#Q58mIcXG|pKpyT14WQOu
zv!KAwdA^YDWsU*Tfk*d5gw5wTE5WvBmO}~<Wd!t@@8w~FUj9B|zbqh>EiO2q_2q#r
z+HHhfaOjIFvMA@~88Jv;3`934c!M35+54k7o|*T>or)*pv#M7k!YnL)Hn~JREfroY
zd2<lwaEY%FLhpbbrh0yo3rOo{2oGHRiYFvu=_J2BM(~xts;t#!JpiS@iu06x4Bo11
z_a`XL_c74A%51m`uN-p)T(-%m+wboRfjzs^1E*z%UNk`(Q0HLgZ0oVg=dOI$ynTMs
zHuN9c&gXkmZ*sxj|EgngncOrg`C}zfx=;jGyOID_M=GTwVFbT)<S9FIQ7IqEn1kVl
zs?AZ4hPwLA)3H=;@xoL3B3e`}{awwc^_-)Ae7_3nxrBh|g=sVP1vU3AyIV?(ui&)~
zu6@eJJuul-UKL!}gUfc;`O|mz*_SbBIH2LAhk?wc{bQf8$BUzi(&E)^WoSjfxr(tQ
z-!c6dXUC<!opCQI*-SK<lNCR9uUAyNElbS4E{LLavtYN>G`t-v8lGo}6~h)z5+IxF
zc#eevm_s1|dM#3iSm)m{9#Ge5UoL-aqE<HvH)Tv0oabL2k4?jUa%5@#X@yBHJ59{W
zqsjkh{IITjFr9Py`BnROh7Lu|@bG?-7*@G_6Xy@SG%inB8)(Yd4J~+;hQrQQ%JfFU
zcmEjB_#gj8CHSnrU67Fv0SvY@_XgWjT0*D2BSx*0%R;jW{#L;r|D&+eF!$Y#RXv=J
z{N~i~%|HHgpS%|TQ@rgEh36*{Cut)gfL}%;@B91Fhh%T!vQ0faBf;kV&p5<&PDla@
zT-*S)*NEi78G+DpVWR4zI68zvg2B+1>a*|{hf`Jl4G-h#oqRd?Pe86PHq0Ch4y`EM
zgqc)fQs?b9>#=-NHee9jx=)G<tNS5bUT?kB@sYI}Jg7N4hafMl-Y0sCWW?G5DwT}g
zLj<btbYdTb&WiPc5$<Xtl}**|fKA_6)p9)W;EIPGwi@Fw7*%SzP<I80-%lCXATy4n
z85Jr69PLsaeN-Y{<ULiK9ml8z;rv&b?-so(f9O~dTGOFHWaT=%)*`NzLQC1<G_{a7
zwSM$8QsQX606loxe0@Bq|As{`Uvb()e(7y(Z$p0z{k5)13&5b-Map&O+X4NsU+tJj
z1$D+q)Fj}mET%c2RiJ<y1aVklQ@&{=l=NJZ`m=mO+0QWga;b0R!SxJuj&tr9I%qtT
zP}3e7`q$o#;<QQi)bFFKQ2)ZtC!n<AQ}Bh><6M6aOv^9^R-`><88wF-xeDC#hJ}j?
zfU8A&Gbg~`W<goRzcpw5rcE{+I5}8N(sYQpwO}7G>_l85xfXGTA1xRje*<@9K1SI3
zzGIM1ivkb<L3D%AdkRs09;h!fZB@Vpyb)*f<{RLge=N_-LSw(ul`es0cfI<&l<PCN
z#Ny;*;fDMCSnvyJC6TYrScJ!)G{2!<rc}>Ik23q&8<X9lx!2Q=GjZ!HP3dGlyei2K
z=G7>v0XWCGByrZ;8uq(M`YqSKRQmzUv-Gy>1dsGS7c0S0bBm1HhaJDu3Qq-?v&e%&
zsOKIqqRC{rO{!7}10p*n>f_0U+fNMLr0)albr))@5<_VAYfC0kCHOTS_KQg6Sm9k-
zpGlX%1=1tBAI_Z&qg}j=LawKW&Zm74ld7Y&4#^H*r&hf!GWp7&0=T$h`Y$_^{ej0T
zV^=sGr({N-eby<yXecqJo1IgUE?znnbi|p+tj;}kq)#pt<*~^p3@uKZCsto1e^vE{
z`WAI(2yXMYw)kkc+4qbxb^Kqv!fMTan*$l9gAjj5;~aAfxzTknj|?gLu-%dT2K$lx
z@@g+>C{w`ejd&|iVaY!%Mi9M{aG9C;`Rx+GJCwk(vp0*G^&gzN*Q`^J>Nn*rB=$Ya
zlmG$*&dQtK{vN@T$iCf_mk9ZA4?;#*j|wgBm?d}Hk}_TZfF3E%ux|L14%27x*aO%h
z@9%v`D?os)O!+OxR<$S27&?*4t6AxgN_d>U<Cmg(ET2~+DG?s+6QhLmwQvI8YTW?9
zn6bcBdF<ppk<kH+d^vsmwnO^nm{eeqqm`(iUm$wm4jDs!(B3h>P$9`@EcRveG%!?t
zQJUwgDCg}8L4e>5LG6u5qGYfUSk8Vy+AWErwZ3k&)opv6|9tOUw_bncn<RhZE$iwh
z2iD*8g7><dAWolSV0N8nNSVFO47d${Hf_I7a%!G>@*tw!k;-B}X;3D{Ox*=A>v4X@
zx#p!st{68{_Kea5{Ra90wM`1651WA0{v}t$_n0Dv10lO`QL{`ya!<}<EIGg0YZ!Z<
z6+ZEWJ)Gg@DyHir{hGx?{WxbpUJ`Gw2{lJFCCM8W$jC2BOw;EKz8QgM)y!}}$eI=*
zq_MPqr|Juw8&p>mqqaQST3bKIn7C_k&*1rV?{PaiEqtewiN&9(N!*<B*w0pz=HR@c
zf^R^NaW{F{?@`rU?u1{Ds$QY1eBu0&30PK_0Lv->`6PG)Vh7y7kterUCs`cqExoh(
zaiv~ogotNwWl=!wprn7?|7Vc9EfdIiB;5IjK1k|7?}ajB9zd>-Zt?`Mt_qn`>No-8
zXO}wQMNRia5Pxqcv3U6T!J7xzXNLr%(dFMGhzS!ljNG(GltjED_RSn#Q*TFpG`lfV
zyWKXsF^=`v4X%AW%w5hR@`}vol2qqC3{_?bKsV<}N~yK*R_)Kqw$zi|#y#M3ArSIm
z=O>L=9>opCZ7UWo{qcxl#c-Sf^iSTqv?4s|3T3IIY89-iF@3zXHfNOqcAYWuUV90t
zG)VeZ^Zo8mM04S=vl@@l??8A1vJV|w<%oH!GCD?*xHtaa*g09*h>PVY!2co2C!T<k
zU+^?mB7K#+WR@nJ#)-pGw+-e<T>}$6=}0*2wJ7iSrmPdvt8+Xzc2z4sk?Xu=DAV!1
zx%ViVnlucTk`PHQ7})H9JsW)_w}fig?bs36?RfwHlj=i%pX!7CFV*M6^P|56vrv0d
zr%CCHmEw)^U06ti-E>H5jsa=Pji0vX5qEj#fRLRCt?0%7_(uvL#$yCT!}oanZ{D}R
zhv50Fjk@y%Ecz?Ub?X$+k=ci3kB0f*3}f@-Nj&`K_<#dmP1lcP0zRp@U{+1Hs0peC
zsnNz#i@+dOh?a*z#Tr3cl15_RTbx%qPUqst#&R_>9JXD|LPG<PY3Q3@cq6eFwzU{$
zmwkEPLF22*Gg*eBy=tN%!lKc`DXPhja7^PkXZgarpDJsVc44lAZfTF&O6Kn*rr^0X
zW@yCOwo7KQVnkPgyfS=j7a`628N<OYKc$7@vp-^mXxVFvq*nZjNunSQDQ|fyJ2kuU
zby@gLi>FIRaoy*O7T&4jYzEhuKCTv{*#s7~OVMW3PPryAxm1FUa(f*>@V$R{fafqX
z^(sec;gi>$r09edVF(4Xi;YDBu};&c0A8?cC|cMW33PqCwc{;LCO865g^ugIgQ|%H
z6_;0qEpEi_Qrhqbh4CJWivZIu(HFkhz=3x!kos$MM$v@y_^$Zt@`#Z*mO6Cxa891`
zo6iXiYZm_m8Mab^e^u<UqI|cNjcgx(=eJAj^O?UTarHQV8}Hz9mHp_;O<p+7fQOnz
zKHEQm{labbMGK2gNgdmjD6^|ACt%`x{nBo(7n`5+XbYc7CfCdA$;h*#W!6{mmQ5y|
zuO(QYFSucZPa?ZbBh=y?f|Ub}!$pxRLR-JH?8liuV>le*TQd}?cvQ4*Fe{`cVFZAY
zeRed*+&){Sz>$6cyH_bLJRLqj`OM;~1TcBGeT~YTn&2_kr6X@r4Vyo-Q$x2<dHEGx
zbs4b^H6CsBGvx(@D<4q2q!69;92kCAOe~-9W#4>pw#zv40<Zo~73?a(4@fJLA@d<}
z6X4#Z6`P9>=#kA9Vf+;d)KsGRz^If-tU~de*I_P?$tbDG<b&dGye-v#8h9j#v@Aax
z64OK#rm?(o$Kx&X(k(Kd!N%y!4wW()XhJht-MTe7ebudO*FTPDvWP>~L5wpU8Ze|s
zY&leQE$7qkNzEt2f{&}ON<$9VDhV-TenN3)ixEp-1zQD_4e`p?nqG-EzyJ&7$D=8e
zNXL%<j(TWd=d}cu;V*N2?`an_)<fGI+eax^6EZ3}`f3&WH2-oWT)^)$ute>Cy&Jmj
z+uZggeXN&cxgx~9s&tVR6kE{<eaRkF+_*Y6vni{^kiSSrX!b+&Way`?2B{i-7yZ60
zi=xu6b*5%Nh(6DTY`f4#l4b`mDxLG&f<5hx*2n80fL&xnJRq9J4}cJ4w(voeCf`3I
zRsle!5HMD*tK_dpS^5CMFv?gUYjBUS9N-s5RNVWt=S}OA^K_+XNk88Ea%)3YTl@_-
z|H1U%Aaoi}7Y1H2%=Chj;M)U_;t!j0u;`mPSJ3RCMAE8zO2AJ#x^vPob@USo>d6N*
zX<?F`q^fV%fMlxG2kjJuSW-V996hj>Sw_q4l*$-z#-J-NrA4ihyktqh|A`*rk2nJm
z`<OE|qpLw=jjl?-ZjNwtA`CT1bC#qCSx~d^<-zLO(Ttt_lFM>eZ|8fesXsI=zuqK1
z4O5|Mrfjl*hSok-sI5lfhwt_vBXP)Fo2-u0t-1hq3S#pPl+QZjHn=L`sJZPQlwkFq
zRCIF{fL&F?Dgw60jJ~dKEX61zuehI5QC6W8!fWuNdqXZkKRK^Qo&OPm|8n;L@>dK*
zH+FBNU%BTVbC^fKEKs7u5HEc-JLSgaVnFir*{XOmI;s!Ep>zOnz9!v~k8RkaT-B+)
z7x#R-#Hi5#0p4NXG#3$cyILh!c}XgeDnWE=a9Rb7Q>-R_Wj;?7Viy$p8hNH}PU<$`
zfCkrpm&E?L33_<Smyb|a6Fq`Qenjs-YR4eJg|Eg}<89nskKC1R*$_wtmh`izq-#-f
zswW_b!rpkUgtCrnmyJD(kresYERSV@`Ee1}{C^MH0#MWg)5%%%_G-R>$0uxiqNId#
zPPj9C!R;T|_!{(yeymz$UFG>Nt!Y^w#&o%OD{>%kEb=a3FuO(75qPy&Fx~`v0ZdF;
ztk;n-x3jE`9_>lR*+mrAn+JnZxi6^lMOnXY=^f*GPr4u&@?0y}rV>-Rn;u@0y+d07
z-;BmnXXsE1Qzo98X%m$f2x~+eoF7&Rm8zoH^zHiwVcr94MuKIFt5OExp=eXe{E$Ga
z!xwOjo^WQh^kJ*rNm~6`sp^Nb!Zj)8G9obRO%UC5TSF}Mz>XS4o+?XLlFZwPInvv0
znE>B00{ZTf>j@*I88YOdH?qx5VT0BWpnayr*WOz{V|83{Q+_ws!5Y*1Z`8gI2Ko=j
zj{{l8e&MKOC3EeuqPQFI@PD!Z2!CpGOFdW}>SxKx>~o)a^HwF6Ipr)tb9I6TtcxC9
zrmC0C-v*2wwNr+FHo(;a;6meDEF@E$+UIZQ^Ui>?7FjgCKkos!q)!W77N=R}z6*OX
zM=6Z3(gMxd3J@7LL=D_%SCgXw7~xZrM-k2!mus<&c~i5+Kva$h<cS`b24=4tj@h{>
z($#@Q<4N}u@LS{k(x{OkKKgMF=9fbP35Pq`9I)d-bCHc=bGGG`uL-*<#Y2vZlvC$F
zaScBEd>=y0MC2?GBSQHka0%UbXu3S-X3wfw^gUObVzElbtKp>GPlI<I_v)?{fRU>#
z|8@RhA6Oe!fYWB*MD7gVt;Y0C>pM~3TNPd$QWt;Y?E11e>;G}f-9CPrx|`6NBq(fO
zcbSCl5G;6U95_J-u?nhH;5r0#Q4z8PwVoD8cwxJvT<jgn+U8jk!>uIQ)Tu4b<jKB&
zo-ga<cAxVA@|7ek6JJ{W?MvyY^7swU_Rxr!r>MIwu)7&-b(?DZ4X7|J8HTBAy7SNy
zcR-(_wTK&8u*7yCXfHV&&=t?C(Jl*9DnrEGx8l)BkqQsL?kH~LwCtF{rV^F5qWUB(
zt`lG1)e@H{=BhPwf#$ah5=&uIIAl8lSaXa;F}#Vrm=J4bdGuj7mQgt4fAXUKy%vDE
zA{cz1uW8aAh-PE=XvwyO!0>^4HXkIKq(ARR@vlsVQF;&n5lhAZNzNj|=1U?MFF@((
z=^cP3KLdHu_aiW)p96Yt(qk1?L4IRN0<y9aNYUv8EhD6doTff$uk=tkxuNT+38q=}
zc=pkUhXBcdoQ+f1{9md)Ehs=hJmHi3=s|c$HXvZ2sQja;0^+3s%{sN+jDJ6Wkqp)+
ztfS^#NUU!7b-XdizWrPHzvEWchj!S4Ju%m})A4i#eroFZ_YZN#-~E=mQ4<7GzdGf`
zkK<!h3yN=b4BWMwHd6F$cNWG}<4owsoYiW}H||3tE{{{&o*-H#sey(ejjXo`N|RF8
z?%n43pzsi5nSd_NY8kB^OP7pbrzi^VxNtH`hp?_Jwu#tQQv!`ie^UQDjkr)Dj*ir=
z8LS~MJdRJ?twvD-z6QCIzAck4Em6IlQab@MujW$m%TV#FGcg$+SE+^-pA6QUJa96+
zO`|$hwSwiNT0$b|KM}XTctoX?#^LbF$Sj4%VLUM)P}|UNKSC?3C!EXun5vEwrSG><
zq!t;lo7gg6GaLkDBNrd_H=Y7noD<=}H7=#xp!5hpWV_^+R7A1g^j~k*e=nz!AYIX)
zu7jKatBEm^MKA}f2tXoQ32bhMQEj$%vth=~j$W;E*INWVP8o;~3bYQbZkkVQv>FmI
z64h%4;wc|yH)Zmg$w41V<-2TOft^)Px+E5kUmksR1s|TmW{4J?e`JX?X1;E$+zhzx
zi)T>A!l~g92cC@b3J^j00?RAr&sW`Phwo=W|B2O;1s~4&0cVOfP`Q+3cdq<BtnxoA
zKMlyw_9$;~pEU^&{V`u${ZkhK|F!irBhF;5xQ0D69DJ`nbG-3uC$x_FFgp~5r7yT$
z9O{YfjJb^FeAtPf9h598HuzT3$dCS)ZcI=^IDS5DtP*FHG-^Cq++ug)_z6F)TTw8z
z3vjh}9xdj#RBpkK);|FPc#@MpsXoD;Do`_o-XT>Tw2K5Q!mZyIl9Jf#w{u9ME0x)=
zN6X)gN*R%b^*A~7Nqi;BNd3Hz69+9T&2ELM>4C_7tvM!@V?DS?29O3`QK6_&<~!bu
z5Lm+ki2?o|ZiM{3Aajo4|D*N}>zUOuKI_#n1N!w+0_d`)$Nh%&-$ncWf&x-$tmQtC
zLp=Lx6C5N_u-D=~|L^~W3DAIk_uc8So9J{8h9R2WU$D(NA@_rrpst+>e05BS(*s8-
z822jJ&!&^Rtl@D~QSIpp4c(SiXlwm5*v4316-w{#G0WYmz$jMy=ivedzq1kpuXzF3
z4xtF?@i@~|%V@dcdSUU8;Vgc-IYLP2ciTf;5;4e&(d2CAJkDrq8J)lZ{QWmi^t&+P
ztW*2MEgXUh{V&x;9E88GL*_@54Sek?Y9ZxHX7aJ(Kz;$zh$DsT!Iiwxtxa#t-e6k~
z>)qB^SSj~W^{83FY=mlV#MA%v{*KB+mgOn*Md(ERR{u-m|95=5zxd&x<!eCeEQggA
zT#Dbf{qGmwO$a?q@~xJexCo6iRhNpfDB$_p1Uz3Pw-1C@(Ox3=Yd*SlKYvN(0guZg
z5V!8cXT{n-gjgr){{8SBMRxbT6|t9?8h~=PrYWrF#o9Siv%N9Hl1i0<DQpj3Zl#{7
zHG{+!)K208hSj~(gHbxkru79-DqJn%nlk1<+U-wSwfM{l4N_EJ(W6C)$pU%xYs*@9
zWHL@`DqD6objq$LxbpDg)ix?Wqt<BiK5!Qas~=Z+1IV==6`S4d>`Z7X(m4{8zT^p_
zG<Nx4Et?ZG*C8{7Q=c@2%V7Sw)S=#g1wsG!=lc<&$>!*NakYs3*;+=B&6iXA4;=ir
zbe@`maPM6OKok4P>+338JDFscK;Vh-i8JO?<u3Z%oSkikKkrfV9;i>W`vV1?^t7{q
zp3>Brpd;={mRHHBgwz0yD;=XbIfB2(5nM<cd3#%jr?H>Uj6PSLR!4BdksHmr`km#t
z)1wWOvcYkjnDT~aY|kKpC59|;;iS3;pE!)o!^IIAJ4MP_!v!V&=tR_6pyPV|yPA(6
zk;y@c<gMN6vvFB0pL!QMwn3J&IO8U|BAHJvajO^^Tk)y6@@ZP2^ljB#h-mhIU1$G(
z_UHv6InYP|Er!jiFSqZ;W8}2^|9jK^l~fQ1kW%To+wiV0j$Wn%hyoioN63A#O`f*$
z?!ERU{qNTDpWDO_+@{|KE_)|yHML&&zStXt5|VEoTs{C>_^C4$8HSJ!4D{ODCUOBM
z*)Ky0vE`5}#E1-fzw3hVJ^~cc*)WoqW?s-bQEWF@^t^~ce^NQ;NcV1X&~d0(v~@ig
z*0nPS+of%u4%pEhjLMuIpE);iMzl{D*PXopo1hDuByk3^X@-l###my-u5#Nt@UgAu
zM^_%ufuACFDg);73~nvL`G8jcREx!LZx^mvBI>bj^c7i!QkQRb+bZF60Iy9pDTH>m
zf6RHpFnSFD!Vv8LM#cSSeL?Y{uk}RZi28lC98RwWz<TiyA!@(=(=Gl_)7Z!`nxVi$
zEu?erUoegjdzyqp#<hoFJ=ndUpYAni|1K}FfO{nev8EfXKIe`>H|*<V2*jbo$9RGP
ze}jSGS$%{ckxuhIdN}%jID7A~CfDX$7(xV8Ix4*jh%_luLN5YRRa8_;fJhUNj)1fX
z3Mdej-n)QE2}GoWf`r~{kRmlSX#q(Ha30)S-@V`OoZq?5_m5~2xvtDJ_sp6#bFVp%
zCP^96Gdb$Z;<E9iKM4ZY>GA03-4N<kXpQ${l%qasPJ2%%M>xPSBM{p!aLoGT!{v|d
z%0k)7$-;ASLBsrCA3>RQw4bX<QjQM<>s+-;Bk?aZKa714u+X2M)~*gA)z@s$8O~4)
zTZ`!^HFW)OIqJ?|?<NGj@tO}i_r@7F4=u#RzE8jHrHjl%UP5qsnUU~L7HrJi8~vrH
zR_C)doA_U@gDs+T0<=7=26C@k^<}CVW-k{p!FwnEVJx^sWN?h_gLJbub|WR`Z0UY0
zg){r87PQ`ftTaO@4FY1dyZ|AC12Nn^7rY9fjRjECQ()ff7S%WK7NwK&-F%r9xrZ8H
z#*Rl7hh^7NhE1M+FnmVsb7mHL&hgcrw4s6k#-{h(kR#QK_RD~k_MB4xE`cm1#4(T3
zEh67M{YNVVGKTk2LQveR;yVo;4snwY3LFP+ztv`rO$%-s&dkec$dwTvq4%SVp-{aw
z4vJm-3a8;chAig}3B^%Ig_UpT)>!6-K`EujZg5l<n5bapRe+H%&+V%en2RGXS_O{B
z-ikYQs;sw_`9vbX$teISy1Dspzmf+1QX#}nHwN+ci<nk+eezftGzm5O(|9`3dryxD
zj#2B#kgarPm9W<2?Qt0^<LOFMkPY}LKHY5k?-W>b1~3~BsJqVRLj;8x=?L-soflp~
zG+XpTVw_;b(_5zs7{h`UDV$$wYCt>6ufG(>(Jc%Ek2+|daztglGSZ@wc8)%Q%sk2n
zrpk4^UwJevcsJi05ZE!t4-l_X0IKEV%0qmW_4kaMg+qQ6*l$yXhm;53yv!AQ>LOzW
zVZM;;A!+v3vS!P>Awyj@K-d@1O!v4U@Uaq*;1$$3n%A}=@Ro-t3!YVKefAm<r(5nv
zc*k93@{}*gM-cTgWJ1d^xkgIf%(7mn7@7$5e%15A?uO^anqU51qH#~^aH$0P>%z$x
zsCgz&v`zj21K8m|qJtUjFv=QE<2A!~P5JpNADfq1!ei_(FS<-Q{_$Dtt&_lo#rjw%
zhO6?aQ#u~HlHOZlLX+@P@zRj`x`EnqN+ls3g8(tLvJm3uS)eflH4CwJ?hM5C8NO_z
zn7bXc)~+HHNQbG|O-@-C?*Ru$2dyy6@*bmX4(CGr@OM~?-nKRR+jBlDg?zoQq#0tX
z8mxXl=5qsiiQh`w>J61Pp1oXMqe>lT4pmA`f9@4T$;yXxr@j|Kw_5e<+k{uNYcHmJ
zBA}LAR<bkGAfdOba$hb&@Y<soV-?K8=(|XSXsUNOIGHU?mGRBLxC)?WNm-W0$TL~}
zSq!+f=$dmFXJT>l<L?g4@1v#)%qPt19|<ulCQdZ+ys^C(`DSzS2<WWrI;Vic|Mh84
z#uH<W$>qv#-D9C0**#ytXUix^Vi8<LHY?O8kPgRvr??^bEYgP_J0PGmo@tm0@+=H{
zVtMeLEsNT`qmm|I{5(RPcy>49vgl$SKC@IJ(-9aOd1HkCw2O79o3}Bo$qgSK+%Mk9
zzDr%|enVsQuDP_mL3+6WI<t`C;;rTP>_xZIRL{|^XzOxOik-Qx$?}GFoCy!OOrP<6
zdb9ZuKg~mje1kCJ>+-0##3C-(yY9XGt}MLOD>sM0mCpSFlc43V?+2divZ-?sJw6ZE
z*()ZesxsdEFT2@4Z#vm-0n7${i8p;b{=QrIzul`uJ<|f5<Tt+j-gk7Ow<Aut`K)*y
z-QD5`cU%S^_x>Z&0hAF9B{Qg7qW{}BpUsPW{v=zJZLdsKUzYu#JyuO6*$45KwK%OH
zXm%-O=(gzkzBW{yMs?opiYMfhhIo&Mupz}bXT@8b03Dq7QRS^#K^^;y)1(Kn#SNJL
zeDlCzD3P|C=RIl@VKI{4YNRAHY@%#^XZ_h$w63=*>m~1YsT(xlO>nY6XD%~E4R2Vc
z21hW@0MZwCs)|?;scbDS5SJITix4~7ilC7x4(bSR&92VmV|R!#S}O|b5_se6Nf(uJ
zG?Gr#NK`9(P7~Hj4YwtptSA^fSy3?Z?;{DQDo2s1e0xF5#rcAUgSE{C`Sn5;R3#w5
zxp?Db((oThG+MQu4tr@iFXwReDx##axo)%+$~97K%EaCC;U1EAq1KQ6$A6j8{UmFe
z$`wb|Uo)iN0vnQ~g7!%mwr9!01yxgUGv{=|6nEJvZJdhI*&4AgO*3b>hj=#6I|Z!_
zy=$12ydr>kXsUSX76<m_oEpco0H^+5LVgb+KXR;k2Kbx75zy$Gj_n_2E=0du`|7lt
zt#)VH!<g2Il8XG2sx*uA)qC5lWdh0Te0_a!&QUQ9SUb2Ha7h{?T7_zVK@vwzu=gJe
zL#e-P*fmP`YqfK}&k;4*V^EVj5H&u)j^#Vf-50zjt#ymgp#l?lu@uS-UmX441hs#L
zu)lE!J18LT;oy=X%4qDwt>DTpgH^!8+D3Iowc5Ovl+u_l5tP!r31vOvG$e}T5p7(t
z4hft>AR|@6z%`@FA&P*l(=B(x@3g1Oox^5_aBN+`3W_~*`uctd5!*135VT^W7oc<X
zE<Q%w?GiR0atnNnbyOTv(~<um8g*xVgxsrKt-<z9e=>DTG;6q%1pW5=(TiI=!KAe9
zd-T|$vn1m=w7zNFaW)!45Y)}ehW6F@0`Wp{fYSuyMI}rDntV?E65qDoqn&aCuHnwc
z^&%PE0yMCnk!D3BNx6BJ6mDC?XwS<a_FvxdZ(jjutyOy0FZ!h_UzOGV3iI$YbgtG0
zr?{(M^7VbCd<fw4oTv2mEfjRUYW`#^?Hc83w*|!0L-tRX!<D_>hXttyVTJ{5^-{_4
zxoLHBr(=h;j~p18)Eod}@_L30ILMXx;V2+1rCf>BO9yo0d~sMrfr!diOg4?z`%i)8
zM+@2kMTQc6H<66CWyy);k2~k-o9KBo1r3~&tAe#MWIAx4OIk%0P%$R*wt7tAguY(F
zytTTV1Ax$lYR10MhOG^v2?+)tjpGfw-Zg9+-9JK#9w#$7Z}T6I-AP;52mm@6pInoF
zn95y~{ukl&H}CY%ADN>(GoeWqPjmy+|2CVKR#UsQ7x_@PJm~6)Pi#aBES}tgzBi|>
zBiet8CfMf;>ND*Oo%S>D?OBFQK2ua7l}m77d0w3yd2F7`;h<5#L)p7=#C8?uO=cbB
z^FA$$%ECdp!e+6(t1`<u8k}zeI`c9uj)tO@#(5B9lD?sVG8va0efvoqmy48HGdspm
zmJV-0QIhH^2AU){jG*)eeK%9qUaKq9H;u~|e7L>Wmk&(`$d2?i^m}Qok5<};3qV3u
zfOY^Qx${tOA-)yCS*rDcQ5i#2mUhaz(Ls}s^fgd-6j0yl{HwnGX9)S*NmNVKnj)m?
z7jF|Z|Bli%l%#?Kp7eb`TInx^1Ow9?Om`O1uxpC<=9rAWxS!!p>LX_g_P#}LW-aqE
z!X#5th&LeRg6Ye{mY7TqwtYacI0#S)?R&f`#5XGfkcET?wYZod3?1f9ejTy7l)G1P
z*!HlK)0rLn;CQ}JTbgSk!w+-yLdVsJ)S;p3>KK8hhx7(7%C`rupXnCAz)pgEdy2$7
zO%X#kE=Wrk9Ni{J82ES;m!GEC^j_RXIMxgF5Fxp{ul<nh)+=x$R&VXzFe0`&n!}Wx
z0g!ok^-^*d^JQ6U`pb1!-<5yW5&vIXD6zn`80=V=YXvi;xN853x4`WR*6X$oxf>%o
zK2dyqPw1|IlXK=iFtxzE6z@$e1v6$U62T(}-x*&oWg2`V-G1%RLYpZL!Nx49J~Z)l
za=!$MT)AEm0JQah>S+O?=nPbuhZVkC+_`#CuoBJbBM}pISdO9JKo){flvaLHrblb+
z-G<(I36Fm1WG^`mhjDRC(AW>~*)Swi358*XnK<-x>FrT`AVUbqaPM~O<B>*8Y&Ku5
z?pFS*Scqq_g^0kLpflUfOW9{ly@fjC7ekdPc$GXi5q(;K*ti978|}E`w6yB~hTr&i
z6j3@4f6y-cac{Ly4qx>*<|A-|J~CK8S|9yMx5uIY+EoMiu_;E;o|BORRa+#8D|k2P
zmfM9kzF3D4rpyqTD$?8`p4@V^bZ<P;8Cmmi=h$vTac1puhq7WWu-lI{SHKfYuU7MH
z;PwqQ(!6lXSNCqx)dqqpX>Ti^7qY%~lT;}C67QL3X9mDv`q#;Hucxk`nw9OnpOG?N
z@kG}2$K$=nQ$K$yC`fJ8*(?S<mVde!v~*(>JutQpP;N#N`9yM90>9UG0!+4nf2+VF
zwqd_5u5I|T3i8}GA9iz)DGgaLx|6f|iOwf;HaDEVfM)7?|ERIpT8ErI0|!Mey<yei
z9*c2P7u3bkE=j|SzE#g3tgw=L1vme5e&NyF!`Bq1JNL6SoJ{khhR|vgiZ~eh^SAzr
z3WKC+IXxRCgeydW^Ly+Y%F|#FT$P)S^8fUsQX{hzocOuYMS%p?^!F#75vtG#u>j=8
zE9VI^7Dh_&1xe5;_zP{h$uxvK-ZG5~&Zs)+b!_L6<`uM067J0bFgvHl`)M0VkYY=!
zfzR;YRbF!$f|aAA7_y`@4sEVyNFYZmMUV;8aB3KjiXWb<C*xkLYzcBGB}OLOT)Swl
zR5{)TTDl5;94DtnVI)NV^{v^Wh~BDxV)H#h|KjGUVW$B?pLUuYKR&cFV6pRkN4!iJ
zUMVRy8jT#wr<Ev-VU(cY$8X6RmfZEyJ>05@rroQv?_dam-Qb<uLE5gV8e7o^{m+I_
z&FrSbin6>@sQrO+CfA^n{cSTWJv?3KiqdYJ+EYgpf~FGiI^l&XpqNFG{B-*>U+RqP
zz7GG2E17+#co~f>NePKH2b<^q&3ut|!To_!^AZ`%NW^mvb~2)nL+8yaGHxBj>igp}
zz3_UxkaLPk5#%&Xy@NQ!p`DabQ@w@!QP-tb*vP-Pm>rFs?q8Xk_A67aF<GQZjX0eS
znV@>x+@9d&q^ICS2lGThj#$H($i37mak<N;i#*Tpg^qJoq%2Q$TLN`kWMt;GJNEY_
zXb3n7PJU~r!@fBD$IsKu*bbS7S4%K6WdR#dcl8!!ex#<(|KJ}dzJsC$)$=qCIj^i&
zCOJ__|2B+0P^z|F>Gs@>tw$CfXet1&`yl|s6mF;jqX>>f_RZ6;%c;?7C8URA2^Y`f
z8FAw-<|S%{qtkV<;qpTC6Ye6B5lUXI7x7vgejc;Xk?w@yIF*30M6UYkVg+*do8oh9
zDFb7Bt9CRP%>1L{Ja(s`93RQuC$6YMKEszaf@~bS?o>naiuBbO5{@gnvuBX)QAz=M
z%!A0+ugG}v1EWvXS0jaY6jZF^@LNI?i<&NC{sk$$NX#BMC57R@J#vhJ!qgk3Ru-aK
z@i;JaFlmcYko)xi?ZkMiTB>O#`UTZb(s)ElkpK274Y+FBIb)YmbXn$6O#q4vuA2Fi
z@*J4eJ2NR6asTAh2yu<KJW%l`ktf|GnI~&HpJ~XI-H>w?D&80z_$2o83rGM~zJPC~
zFo#c2enue{ulHE}$I3Z0GO;$YyUIb~vHzPIYKT)OF*@!oF&vg{(Gvo(R$iG$nfDq6
zWwem~Y%jFcr=sW%1w)k<7c>(^c=uzma+_J_n2GiZR_Iq2mO`Dx;Ad%-)tELTGt!|_
z|C=)!A0!*A%SHuF*u1=AZTxpk8PH~!_c_wEd3cYH5}dP{`5A-(yd_1lhc!j=|J_lj
zo`U^&n}-fo)_(|$n%4dxYH<AyJX9uNKRutqiQ$O_qTh<}dNwEUPJl)xci&oq(nb$j
z3UBEQ<qY-0Q&ksz2J!`nvXZh6)Fw;GPP!JUaXlz410F3m#v6;p#fhsySq33~)`Mbm
zMfuVfh^Ka7tBKepZgTv*yqvcaJ)u7?a0M$qm*BTXzhpCc?~Yy2(d*pv*6if-O)O4@
zOx%KSQN{~qPdrF1Nm0@z;>_OGpG&|pvTRM(Mj3~wD0no)mmf9%2qw!in~i=!r1T=P
z3nOm0r&d4cu1J-I_;(vk5EoKE4ECmmtC_7ti<<~bs6_E^!@Ds_svh2S_Q4k~{yrnW
zzy5!P-xpFKC3^FYhVm&V<+Cd;i;OkDW3=T(?lSIey>aNss>eoQ>0`Ea;2kW-U-8$V
zo1%qjgt)XwW)a+S-jm&(Z2pubqDNVt`u^|?>kaL)Hwsw@&GWi2hLEbo{9Ib4>|)|s
z>v8QM`3nR&I^14TMx}2!&v6L8vokY8*+Y(kun^R@B-a7)KfRM9+o*WPbHw9gkoB`8
z-zPq&LCB5>W@9C10eB{ClSG}P-=wck-eCo+*B52lDsf@Pm3!a8vF-<EUnl)D)6-eL
zl~ZZVL)|Wub^F}&`Ls`W98Fub!(MCi#KW#O?nm|pI4)?*rJYB(GL&2b4#)o#axCR|
z`xuw=)`V9V6l(?(WNy;_Hs6KwfP*!~2H4a`8yE$6P1sX_;7!ly3+V0N*&xR8!L+2b
zBo-uLL`okLtg{BE+6*~2u!A?OUSj4fH+EC}Yo#KYf7xilEjP1yPj+0tfC@i_@?ySp
zP>BoybyXz*TP8@GC(*~ZDDGB(mLP9<DT7KwKqfGI)faCFx5!bMHGx03$C6}vFc8kv
zN!aVLx@7|)gw)@t60&Bg&9nG%DE<~lV01Ha|B-Ogb7hZ&Jq$}&)xM?iOF!Kmwo;sz
zS&OUKN_bc?MQz`Qlhtu)_&yEVekO+hgPZ)0E%2t-d}ZA3di9M{woQ2@A4U(HMxFbf
z2nHb6#k8=0ta+EO&jfUK&nqJxv#Z<+9UY$7l@}{$e9X13YI##@22jShE?ipYUIPro
z`GyXn{5L5D<)}X#RJ0nUz1<C3;iu)6Wm72AvB=_r;!h0?55q7f+vmEXXniAuX|oib
zq8-@R+7O8ow@Sd3Sh~Z4lxF70Fe1U&x|H^?CHI`5QwI?xPv;a~EDEvCTKy!K7)a3;
zP1Od!$AIu-X`EVW=`$W38EA)WISsUt=@1g|AM<G0U^m_AD()*4u~oLHmhqFXOC|`)
z6anW+29rFkOtsFZ;)Pb{K;lkW9=Vqc_3wOnAj#$U18C9Pom~2_X8pfgu8$Y6obHL`
zP<fBlcFU)eWCq8XY<~M+Uk{};Us-0+YNz)TFSknv{lgQO`(g#qTEI!KaloIcRv5Kf
zKT@_`&`2kBByLj8U%ru#s6^cRqIF<a>_ZwmHjpnagNaif6WTML)piK<Ic*ade$|};
z)%#w{(0M_sFaI@EXw`=OAVPzR2Hic@m!e|Awxem(=$57;qf>8G8$A_x#BVPgdL0C^
z)+fha#Lm%0nmLbUQP64j6`Z|Q5y3#g)8Ho5tqrX=aAGmQ@ey*g+pam`7F{{-50}|F
zja`cVi1RRO<x#1flN0lgQVU#}XE>|3zAU~j4C=N`arPb|qg8xetByGjKW}1&WF-J=
z!7wGp;cmd$`Ev~WB*AqutUn3HrWd3@r~dQCDqUj0(ZqX8H?AD7#Auw(<Qr4nfcs1c
zTgZXPQ&^63?4Mtuz}LsdpeL1b%Vb1P<j<)+#dqUr=wSUg8HjqGd1T*wCybWFC0K@n
zmrKpOK8iX3dw%7|C72=s71J0@?SBdvyYJSgbxD4nC)U}yOrvYVzYE&hOw;(5&&mmH
z#Xgs?m)Gb`7<~6Q@DSDfq{+G*PA*9kGXK!^EjW|CDqw8%G_PS7V;Gk;``Q*G^Fb6s
z)T`~%JUU&*D_nR#YcQj#Ww3E5M|h(8KK_wWp`~fhSF;xF@SJF5Kd`~l%(?4LRrCZA
zbG=+*y2?pGp3%OC21HF)Sz4uYPBtC$lfAX=&skH=^rG}}J_l@PwPJnq$6j}N>42+4
z+12(fE79}WQ{|dDNhQESZReps|L;v`)3Y|QZo=^Wku9tt^wTSEz!Pzi0BjnUHJyq<
zu;5+0xq3hp0Gka0RtBmfGIxTFf<*(yu7_r2Kp-n){aU!sv=U<Y0+<n(_fWSHN0t#A
zo(}W9lxz2Gj1?z=eus^z9>1KwE@uzZR`%q?^x38^iXU_$zq<sYW9|en2~Iuf#dD^!
z!{Qh{U*F4ll8sv3Lm^3>#&uW9-<N+fU|%88oJ_@n#ZB0_;{vhtp5b$n7QW@H$d78h
zR)KnRc<Y>&d|b0unxF2RTBQ|DWKKGijS$~~W=(lqvwZ(VA-?1VN*E-@01OJ7_gY4t
z0K>m6^dKU5RLjZtpU1YD7M_=2QNw!s8=5(A$yP490MHZj>vUi1_S{1Nt8Aly<xEj5
zQUXz|UyBEN24Y<7I_>{BXfqq!HaFgPXE6_0m?P7jBqgYH?EiDMx!WJGuibQ7!(*mj
z`I8ek$UE<xt}1b@sq&3KPqVKJexAWif6+wvF^NB^=2VD_`&#(TZF1ajJcH0KM{=}}
z>xD@86frM~c&hqBYl{|iR>2U5NoL~67Oxyy2)aVyRsA4p)O&o%45=0l=kUcE_YG=6
zb?eRdM9U1b;V&gpQ|I3hg4gp=Vsw#e9C_S~ujD8zQ^B5NuGu7}{ak5BAgzqUMG@FI
zg3Bwkt~J{^&~t3YMahKikj|3MI^xvc{;TUOuOPE#&U3(XTS_c5SL9lq=dkaI<gk}y
za263`_!^ZjqZ}cD`*Vir>{&n-Y=F^iUTbCkel;_X5)6Nr=Ni3l*-a@5<PI{edQnvv
zI@+L9q(YxnC9v-Of70<aR(O(>-8;bye@UgE?45b$d4}Oa6E^0(z;Vx}0v_TN&kbOY
zS|{h=HV^y-$%mcJ^TBo`OeAkoL0p!PmFJma4_7vuqm>W(d9XaomaoQ)3`1oqaVcC}
z+pBZ;%HQ)4W{gC395bUUk*F+GW7kIFW)_VK$ke?ZsEu{8PmVB<Pm+Q6KOHyn5bn|y
z0FX_WLbDy5MCnH>V94Q4=@nY(MXDNhj};*^W2}BxX#Byv@Xi4!vnBn=7O$sN8ScBR
z(FMi4on~U#BiyjwV2_C43iIi+GY`0XM>)1Nr`M;|ie?P}`vQHboL~UZvb}nxsg~(5
z7sIUa2V;QX@jNphZ_?R6Aa?+t`Qy1SRe@K|EQQmIaIk3A=W!2BtoU||HWIgG1_9-+
zBIxvvF!A+ot6Sbw51t(5jw@$N*#^b{^MCxv8oDk~DH^3Vu5YrZeNsbSI63oM^oB-u
z;_~EXb+?vOFD;1nz3>Qbq6pUEQwhEWUHeWG8(2L)V>o6<vsRFke><Znjya6oRqRr`
z_%Y}yS}n~jF@zy2Lyodx#KBY>x`?I#{()KuYRa;YS1Driuwvi$p$n_QukLi2*E4~r
zZztZC*Maj@`nlgQ0e@+ilQ#l+W5L~wH(EW`ycGON@|DF@wGl%*6O{<JOkr5RNl!Ns
zcwG*OjfyFXy0<S+GiTKaj}64Zit4w@zAs&f+2Kp=iPV;}xrn)EcJU0zfGwki+3$C1
z1qw$h($c7!TKr(SCjF)VL;=Ecz?Otl6THhqk#xo@HAM-Q$knd$PBGI$+pff$qIBeL
z?=&y1{GD1}@q!cwUOP%Yu|3PsoxOiKT%J}Nd<N95?Cbw?9_Fn)L&N>yfREwY3{|Ev
zEiDN$Dvby)f2^|i)?$;76S7k%yx!lB5JfHvA8;_{d5<7J8Aj-MW|bJKctu7i1DBJK
za_prIn47F&5SX-z*E+}3yFe#BFWNif>r8;R2z&VF&tMk07`4b^PLADyk;<6LC}*LS
zQ}4i33NUr)@25g_AX(8F*`koSTh=aYrsz^7ewo;G<`vHB2fGK^qZjJuDh(sJX64vj
zSV}F>T2CpZ9b5TQgUjl_Ud!h0L;duP?Egn1cBI?HT%wdN{w$UK+w(xhTHL@ZTAuT*
z!lxyBZl&3nyQH|c^C##t3L1X6eKJ^_eDxK_rpBKvq^b&zfAFYqg+2_$6}bQXQG(2Y
z$f`o;SmjW3c~Z$5;^YbaN;%)@4WBH*43%fDHmMbGF(}PbwYdR%zNM?q(%tEE6gDAE
z^P0=jXggb)IW(cY45#KrYp}V0*HeLMlotEJ?DZm|2Z*hm@82UBZo25NZ5ac4d9VsD
z=wX*he5(v_!6F8CQj<oy)4f)-e|p77wIy2FNVyheslmnX4F7yV1vb|EDg-TX+Nb2M
zwZDfY;>O#-KzE;K5ioxOQ?GW?wC_VNnE(E4SiRu$)XGAIQnTU~xjo-!oy5oxVsy;d
zK2OhABrNRlvcEpyds<oDf0+~?5&8WeCe{1Pq@r-CXMap8stuS_G|aPw{6oyc!~#g?
zEfWgskHdHMh+mEh^V4<Llo|hlEY0Z?uUM&nWb6J`?P2`H=)<K5i4oCN`r5(K^=u9x
zx0%xek8Dj<>MEBj+r`HDM_&_yX`3E25<#4!7im?e;D^;k<#8@qoC@+3u4mfR6n#(Q
z+?upsr@Se;69~J(zy#ff8JXb#ii5w!>l;qq5VFiiakzc4ASHw0bAF@|JNj{8YX~RL
zi`hB=9MaAvaha~`LU92M&uDH(-ZX5Bh-2hNRALLC_MtUX&pA6^>K}CB$ZQW|9`p{Y
zoD)|ju^<e<ocwn+>^Rt3#ul~hCXp)|Y@gBlu?;s0U@(1Mn!~l;_b;N50sVb~aA0`$
zKCR*=3E{QOwWeP#x_mm)>Yw(c|K*~h&#ylFlZQn$PV5=tu1RtEw4~wfUoQF<I#gQr
z<xIrarYgI;yo|fQV`HW;NVphvi&e^2e@%Vxd~zQcr127ng}?Z;Eq&z~vL@%?vrX_O
z^I!_@trkdS9*46pO>DU0Y2CS*91i6(GB)~7k*Brc#QnF8sP1lET%J{+$H>k&0uhLb
z->F0p=W~0JS7<O}+?ZRk<ow<?4dsnEwP`yE-wckk6ckn>f_YQkr|W|4=Pvdu_>$xl
zQwtS(0D9<Wl_4wL#4fl&l8UVG>xhsjYj)4*2*?q8o&^*)(s^%4o;^R5+4<5I%#6~l
z?5kmlS@8;L8G4y3W`{=F4h$FI>u;Uqz4?vG{{JPPP~R17zjD(nKCcA)EbsydD9mDv
zl#^_GQtx~J$`gN|>PM8W0j-A8Yh$esj4R9^>ZW;hN}5TTk8_>a6?=kjDqJd)(`ipj
ze>iA96<jupQGw7r4-91pxNeUW3sY6W;CNd2mDN?855pY-@sx`^T0$!kRvBKHJ!4tB
z`e<qhT8+)he_l4)yHd21eSdh}DZ%5S*j5xA+89bwOh+C@nI2NSnnd*lj4D2N?ELf#
zb3)8q%X`7}K<uP<Tc8nWeH)C88IIGjl%UmZ)$;cm>i_QOuj_GxHH@KugosTG+p^D*
zjmCEn0sY#=EJL%kOTMv#H2`DcNB1|P0ia8iku}vxcKnSOUjTR!NYE^~|H&WrLLlH3
zp;SJZd)ME(y<HLMPuLxJ9CdX>Cr2wvmsSaP!|%v<6pRX&{Lk;v1-C|XqzD+MCZ+@W
z_@00&D+T-=Sa1d2M8^vtnlZTT17&|2q1A&kj>ntKGoTw`!rcAW0?80}-kNd?g?bl`
zhB0qb)w#-K$c*Rm;F2hyvyZ|!Imvuy__(oUR;dK~W~4E%4Lb&tslBP)7h^O<cG#+F
zbY$xr>7uy9NXwJHbHl_7w~HWKFM>Msa<haY+3E5aZJP|g9YHQ(KMu+kv5K>{ORk7?
zi{IOqya0=J8#Z3CGn_3P)aI~^0pdV3bE}%&M2*BH`Tkf+Z%SW!w@a%8R<5-!1WgA2
z4ssDt<n*(B(ktC{$m1Lw{0r5D#GKbS0d0RDJT>hm#~kiHD#*~AKN%X!1lV$!$>~+9
z@spOsnue(h1=nK)?#w6ef6olpWcmFZJP{<dtslxeE;~%s`7Q$&3(UFUFLZuL!yIsa
zrU554V43AfN;WOuoy5BeUy~#E`Vw}LUe^~K%f3kptkQiHl4D6V``IKjsy+|WAt`uB
zSbW5fYnH#FBF_=#rH8)e;N;(J`g2h0>3VqBjknNps+wDyvI2gqHi|n5hXe;BXvSBy
zy>X<gCAU6nR`ituchIxdOyL}&Hk7cQgF6LGJBQQ4L(3ztnX0Uj=w=}*1v4r&<0Z)h
zQKypL62~r&#Wr9+$~Ho*dFEqqIW8=WY}Rxs%p*b^705E4_)08r?0_X0?Uzv1K&=N(
zmQi^lLHr%CxasUq{{^i1UA6u|yv&6YV7>4<x+=I^M`A@J&9g%0>=U(2`mF!p=~H}7
zPXqy>LX&)>3=4T_zAv$m)p(nD+nREf6LizZ4jkKNzSJM>=v4nE1xx3~Z%X8xa4>@|
zn8O>Bqb(Sw%H?wklbK@>af6{*#%tDcu0(Q^8lXBWD}itG;#=sq;db=BeTHAE0G>{<
zkx2}nY1pb5V;6I#x3a}|j6W44oPk!r;o${xsT(}x6lAlR7SC?d^j%L7WHn!greE5{
z`fp+NxR6^*+@G9b1-<4`ZuICFxKd?{y8T3$p%MfdKE~G~!{ycD6W#B|+*v8lECo4q
z7}Snnrr?+ppz#6(Ba4<!h`q*^HXRi5FYEn3lqGGJUbm=XZrj~o(WS%M^@~wT^I!fC
z-{+?lya9-d;39A0$Uz!#AWHy{WpD@Rx?GGd8xf}ne!Ck;DCksZ8zvWoLjv5vWuIt(
zdn%GNbs&QLJg24hM1xterXiUoGx(`B&mwmQetGGNOx|o&o0am;oQs$)S}#IrM;mwb
z!qJc(mnTZp<0Av6=jOq&_iib-n{cQcl_8z|S&}V|DnBojd?HmCLg-7wIcpZE5+s5j
zHx31bM(@YAxWq$M2oc(aTzfCQ^l+G*xfV#!YAh}(<5t!WI9FKh8rDuscXj{961}0L
z9jJR5aJ2t_rsBWedis5ZCUiuvmb))p>p$en1|22HV8iuHk*a0c2^dlQ;_yg0y`5t9
zi0+QrYS^pb*gP{*-!lw)Gp90hq8h;$)*Kweq98k#ounCR$7aQAm%3dZAG8(L<GURr
zXai96uuKhhVR>C;3hy&sYeOt#b>Vs$hb>6Mh(zG4udw$}n`e9l2Y5Tp&djVd#rl!!
zHTs@Pa>bCIA+a(Ag>jMWfTYNkTQj4Lt}7!Ea?cDRR3HSsr^I$OE{fTELID>UkD^Jr
z2==Sc%I<5NK)GlWxDHT@P*ZEMbXtqD7NFbE4rB(ce`SVQUOAiutTp#Eqd@M~99mF}
zQY&gZ=Mg=a?;Sg(h&AP=$b!7bV(v)=3v5k-DE&+w#lB*ulbBS~_$%_Uw$RJ{&I!zP
zK;&bg^M4iU@uY=W0J;~qTP{v1{i>z&qQKvAuTLGEOm9z#XoxL$o-9|Wz4ceFWjQ&t
zG}1uraaVa>_{Nv>GpZu@e`ci6P6N7a7t7`!HA8=pxX}@}KF42!tYESEB}x#2kP@o<
z{Yy5rKv*mu$;S!1-G$&#c4z8Z_QA^19-+4GxCkASnbNe#-#|wzQXcLVJ_*ES<tTBx
z(UG0L^=z4hgE@dImJASK;@6o+$1|87q*gt=FlQUnnv2v1N@}c&A%PAvpV@s~ssu<8
zYaaS^wvpFVZuOi}u#U)?>wlr#LPW9T_#LvR1hnvQ;pvGEFgp%V_b<vO>AxwPnQYm|
z78Ss?eqy%lR-zk_Eu^o_0Qw3>TO1?jl5gxW0#)5J)j=bYX3;<9d<cG~D%D1U2;TIv
zrZT<-oafZ?0Fn8xe5soJ9ls-~<!8#x$@@W}j22$*`W4>hqg^ha`6yojZ<D|Uz{&FW
zMdeR%f<RDGF_Jox(2qeY+oubfPtP<Ns-N+zri^9ZCP%4-WgrVD@}r51xzm0nGW!ht
zXiyN^Y!OZS24-dEImTm?f&YGeX7wI2Cz|HzTKJu4>HrpCg);0Ftqjn2moYgI12Y%M
z`OH$Llfg2>L-KwEr`a!uaAZ2b*mYMy@pq^#3;0@jy*bR#)=rrDX7pCSpo4&kaAv=i
zV_>Vcc~(gy49(d%a6!Qt=#bsrP70A*M2|=$X9QYHF1q2v0d|IRT?}AnW?{7%?Qc{9
zy_4>H4c{4jtG5lVCPt+^Nt>~hJ>%j|ZFesI!I>$mz?pd}PI;D-YUA))aL{|aqNU(v
znujAF?k@vN9ZQvI32d@N5b0l-o0s7aZ+)c>IP%?_5Uyr7>Fyu&0D)cWxT6B>-hDBb
z6eaODFbA4$`#-2a-L6Us5+&D8|DA1m&};rWs_@#T2P*0uS)%w5nN(8G_@VEo?MsVg
zs*9`;v_z0E1)(oq1J3o#P~x2;`GLrSQs0JIc=Js{pZ5o#|9&}v{Y*f3l(;&_)0V0c
zpszSKpcQ#-5((k-JiI_LvOzN54`=TD{64>!@&e;vwDPJUD|9s$we>YRMyHmOjN=(~
zfDN0{m-f%o_dxnwdwW@eF7g%u?7&g8F<jkS`f*vtuU)c5y^KdZ0NX)aiOfe-?NjDw
zp!rfI-A}l$Ur2G5R}u%v)z$O%swO5WJb4VjKIL+ZHsG1cOP94U%9N1+eJI+nKYKol
zoS_@oR6D91f_Ot?3NYOORw+}uCQ<gcV_sv2bN7kNiU(WhHyTEM=BJddClS*J!AW_`
znkSSqEb16nz3P@0^J1nxekb9p3_EZy`Vg~6!1WI7+uZ#??*H{ZRB!Zf@9Db4KC=yX
zA1|0(J=2PEKjD^3Lcp(N_TyKJ%2tn3x_7ZzmgTIYJA?bj_Z54z64&5t!I|3jrsxDn
zn4diP-c5{l+E^_6qAaJ`sg=b~QKEjj)^0bnolKLtpq?+4TjCkjjwHjBvB*!(79Nlb
zyEf^5fljH=l09>M%n7e+m<W$$^q4V>t-MI5dTBMXS<dhqV`43<>Z}Gk+Q86r|JJ9p
zB5H-5oSbNTZFa+C3468)MxBZM==*G~AfYfBuZdVnzkt4*+S-AKE$RxoPDq|o30(Vo
z-QEjyF|+iBP5o0qjYDUTIZ%wo_3uCykOr^qY<tru7ffL5SB?Qjqxt?ziAp{YQA6qR
zx2l(#j0lfMXel{j>)HvY_F}p^g8on0$PKXNJ{%&HC)TIT!g57Z>fdwwh}z$`d7pK-
zgkKtf9XG(_Xe!VW>$o2Ad;@R!G8hez7l>+(KkZc*AUwTw%N2IHdGXNdK~FJ-bv?E0
z<xm}IEeNQdzb{zhdNlH}kl}76;RKOo+WjJp#@+pteL@)>iIn_xV6YeoJBja^%~NzZ
z_jC5fpQ2O_2?L9-&#H-3)4lax>okw!l9|8y0vYv+1LAwO<x>>3b7<92R7eTRLj+L2
z!6WlaaXc5)&9>(>CM)sf?<MT~!$bO@Bf>U7#~B#xX^G+^Y4yHEGP9>R25}%34wlj=
z2s&{&H}P^FVg{ivopot;67m8TEvZ_Yd(fBO^eXQdCub<rc($SpA`F2RYEhazR(#!~
zB3axU#}f8-OY{xmsuX-Nk^+!GGtt1DCuNEN+`YukbsmwTjr#C$<iW5ePcLesM=CuF
zKnR)uLRh7@6lB$rZUa=bi_o<nmGv~uZvVxIyy{><`2Ib=fMPo-sy<1{;O5q;;qMgK
zv^ulR6~7eXeJ39E(&k0P$h?QxDQ;O3c8rv#zq!&6M|0L{KBJVfnq6^NUNE`tVW$#f
zt9WGTrMNRjHe4bysa7$;BbZBFIhnJPe%QshT)Q!WQW&%<f(}tiPrk8JRO3t1j8PF7
zlw=glJ%86!*}x2jM}2wUnSZ~x%aGPEKu$?|_YJNk#AtD}RtKdm?~N<)9HTLP7ooQ`
zvGw`Q(+nR-n$P&IgGDWT@T}bakF{>TRLVPVVINGB-%s7;I*F6NC@w3#s%5+ngzLjj
zv&QQs5-t+Bes(&8AqnJKU~mT3=01ioq4Ln3COjwoRyw&aox7DV6P!UIRAL~EE}?R(
ze>trF2#Ykf<4{@aB<^J~G3^YZZ!YFUiSlQ2RpYcvId%hIT_<pGD=>tdz9JM_<Gi|<
zWsWg^VqJ|%l=ePDLT7s86h)dh7OUNL`d+wv@v3&aT3`T&V5N@yU8nFni=!`?Uq8T!
zqeG~D>4+=|{C>_`y8sowhDH<jv!M{K7~t-2g%oh^SR!OD{Ce1}f^MO-X=!#oY05vp
zc{ud2x0SC80ne^?=R(e7F7W;m$rp}kyERRKz``;8m8hZMX7N9X+Uz7zSM5iKV!hlW
zo%I&)jigmTF04OO<-45LS+BfXA-(){!lXlxvr0IPfJ}q15kGw#E9iG<PS{9V?C}$r
z^j&{$;H>r1eXS~QfMD9F?2PM>%^!wVc{Mcu5GC|TrEaxoEJr(v>3o9XQ#_BIgI3kw
zody}$)xCJ?s5Z}no;JLgeBK1ATwJ}yN~1LCx4_}BXk5YSd|6LaLD{)>Y0+U-$s)N`
z^sCE<l+JJWd*3OP)GHir2`1!+ZTO<9`>zZ{cWg}L{2b${e-v_+BH;GYw7(r><yogT
zd&G;AYJrN-DaZf@oz*-UE=h^n=tOt<H=Qa0aTfm80<a929`L(+r;gNzoE=;Jx}WCR
zD}M??e<w#~w7sKw32DO^Z_71Z(^)G$e;CY;XjV)q<9=y#A(Kl1b}L&j{-_!gaeNc=
zK=uMYsyN7Dj@ZAfk=C4AjiFmYI5$4$COZ$4OJa~a5NW^GD0qs?2rOe5?>Da1OR9sR
zymj}g(?>NR#_J?aGd+W#O&2xtr{4O!y;sJsf{ctQinG@sgA#<!6OtMsOAk49X^0&m
z{`>Fe?4}$~!3A>q7XOOAM9t`#@1?-9a|eJ3{=hjoR8rgP>IXd8XV6@Uk5=&D+P<#j
ztHP5{xQ^LE0ci0T0>;rcZvN?Lkh1|{mrCZ#Iv12TQVQuD(ePn({el4e1=p!fjqF<h
z1C?~b#=jp3J<Z5k>x$VLgCVF1CG~*F+J{fOs{@@s4>cb=j{8`YHR;*DJZyNiw|l*g
zZlh}ABYVaE+hv;rf025h{7)Bkqyydn3$c>Lly=)Zr$tR<z5yXC*m}&wPcPk#cU?Vu
zoOm@QS2pmWNm!cUQJ>Mk;S*2A0L$?>f!y_}x;q&EPYJomR~7n03g0XCT2-dris6?R
zmbWO<*U|>n%Lv~}#EqUlE<bv0qrCA>VPGlf3I6p0+e^(!diyp5sF2M>O`i8ZQvkI>
zpMEz5ljSC8hK0M1XW{ydjwhKdt;#FxD_a$5MNvLDKAc9fT3OLjR&gJ|NE<~z+xnQd
zo<)sc>m4>cjOJil`*^36OQJBvs3g?c|NM?4OrH>?#=*5SBgO5ONp^ZRIAS4_CDGa1
zSX}T`3A<uoiyF%2Ew-9<Ce)a9l);oHbbwgdZKb~(+gfI^m~SW&jl^FGD!J>sX`|fw
z(fK~cAxFviIfNq6V%O3xa5s`H4cWH?B?Trrfn|`fZ|q@iZ-5HV*`Gmszi;%uU7GZx
zp@J6b99!8cj12(@1PM~-=bQlGhxQ}xZ~Y={N@{;R8g*!?3M2hRW7smMMl!4a=79RZ
zfE!4Y^53QgCr`@Wu#0fJKYE|z*i_^LhkexrK<)y6b}Vuo&~LM=`Tl~9-S*C7VBbT;
z0e>D5SZ1`MjuJXBy}1G0Ey>{PH7F`+^v(A~Ob40NZ06GDhy6Ix77VDBx%>3{-Q_zQ
z@-m{IwjYose{nWRO%OZWT4F#r0YEG`lU;JjcAmxi!2|#6GHV;J{n(rpUfb93_32Of
z?BYgxHzwzzrH<45K7HNk?bc&T^>?dTB=%(3OhTyK&@R&li>U2T8s2ZC;?)A(zR?Sx
zM><~O`)_Rc`l55jIaA~n>Iu0SpJYC(N4Gw)tKhI_zMDp6Tv==iiTazn>H{Vo;I_w3
z@>jVON8nz8&uQg+DS8(nbtDI?T!AaA{NApo=zPgh@~DrG{pC;3w~|kh?v`g%jq${e
z-OrZFm+$K@1ttt87A0t^yTp$}+$@dQcZ_Al@{Vl2eJF9lieF>vEH541_3N`xkdb&e
zK=q>ex`cy(Ui&<@x#XM^2gPAt{n38x?!dIa4KTbBLcG=59545>nj;3!zbf~5htb#+
ztuL~&(hXmq#dE7{+4HpwnW*>|&b>;*?>3{6*xnFsK{Smt%X1l~Unycy&>c)0RKGh?
zWNUK#@lLIQJf-xycdK>XdU)_g1t46S!x6>>;}rO7fWgXL-J1C~NvKK<c*77$gsmX>
z;^$ahuXtSCu3ySEfNqGT>?!@3psEq-+pCgLB6l$6x$a`P2=1cA*naGW9#5erk1{&W
zewm8iNNn2aRZ4kh5#(000X50D<2-!gw}_+|NaT;#@6<Qr6Rq8zM_B}R4Q>~e4BAv8
zrVdIyfAqT;lqzD|P@gNG-ae#z)Ns%c{d`Z1ruiOnP@yqZW_+RPQ7Et*zRnZCezbG_
zdja;{ZBkMh{ud8_v}g=WWyt5yZ?1SC^g0(j)`8_wRWbHJ?H>QYyB&4BG?myY_W9PR
zzb_!$x7@RupXVSuneqd_6yRVX(59bQ<@vz&Yu;doV`*&VR6vFP6;tm9jgqZOm5mgp
zcdLOa#Bfi3_+s2NM6^4xGuhqIruclk!BU-L?`3;Bd$a7WvoUxDCgca!<HNz<&E1;6
zS!F-vF*x}EB?XiCCix|Y9rF1y8tvd$(s)qpST>s0)R)#KC6G|2Ei!D?X%$#S9*rh0
zwwp&uBxaQ3!WwT>E(=Cmo2KF0^(Nn@Mi+J&v0HOv3#GK3ai6-()3ygkGW;^W!hZs^
zC^vbCYhUb2YGjD#Thzy_Au*F8x!CZ)p^xbJqEiH1j+joWv~z!olnEQM9~NT&ieq+v
zIKdm2G7a(VHsZpkEVOW%eyze_w6A9zjyT7hwWdM1KHvl9U;Jm_6z*4Z&FNiFg3h`h
zx%N?2h^o|fuUpp@I{WjWvni?-qZUu|LTDfb%btdcTP>b|G?nc7*@aE@zetzT9fq7g
zBej}9n~0a*Mhd*r!t6A3KOO>$4FD+9xRM1`N@=aypNql+8%9Su0*T*B-#d3Ot(Up@
z4S$BNN>9m4W$2ou?Et$I7a9w^-!2c`cP#VhUMziT^uX@bsOO6B?B_y<y8_l9YCgxu
zDN{<P&9GdE`q3uUxW66fuoo#fP-|3IDbXX`d(`OJ?JM}sT*cMQ^o{)r7?ATMzQ=9L
zIwxr%+?Fqueo<CZIQ$+Kq&sT9kKpum&CwE#w?G||FP}efj=6h(=@hlo{)fY_qk$=p
z>*nKMJeIo+B;C!5rYHFdecwEKmC`6Bym<A;D<9vF&i&PEwF6J>mORM=v?J>)++RN!
zd}pAi&F-e@M_A6M5Yh-;7YBmRyKg66vP%50<<TTq_Sn?=bO5F(X6<e5Whid*?4>P1
znLu`6GSk>pSv`q~ZD+2NiDq3=88Xq@8t)T3mf+<qvYw;a1&oXDuQCGzQ?e>~66>-T
zki{^K09EzZ@H1q#nP);IOUNG1fQ+B%o0}4mS>_FNeZC;>eCC+9-@<F6M~I2o2$Myo
z+qfYidAc1bF&Hp4ef`_#3t6<#odh?mcAu4<llzKE^eF|jLtsFJ%aK{JwR+}#L6tAd
zb|+)g4$xm|Y5ggUHlYphkbz#;E^#`0juQ{r=|==guLGJ|27;d7pEJFiL#Q-3Kre+c
z{gsY<(h`^rey1aDpbnvrxRk<kgTZRe#O?%?Oo)tqrIF4atzHv`lfR{U_&Op+cI6x}
z2_rc8Rb~Hn&D_eg_W<ou>w}c_RPW=qp92N`BJ`eZLyge4M~O>YU2_2k@a0z))h?Lt
z&f0m%IS3~UOtJI3ekO~OcZ77{c*OFu_79J$sB&3_zA}&Aa^cdXbf1|YFQ#2fahmDA
zYtv6|`wX3yHaXrlbwJ+CnRuyb4D5>P?d(pNj1Tnqa?PPNMrzf3vJxH1k+z4&aRN}e
zc(g1@c0*-(s5dq4N5l8@7muw>PW(iw=?Ng#OCAji;?oD#B5gG{WX0D0s1XhT)bgnr
zx^Fz2dSwROwBWFRw-CZ%KTls)f5Gt>CG?%F?R2oz3@Ma^jX<}8<5}6#j$9IxK@|#*
z@nO4NQW6(sf$I<SJI3e{vk?amZg&NHh4}Y#jX^7Z?sQ=qM-*P$%{i*}!q^D4b~s%#
zF>=^j{_z??gnYdM_I<`uDCu=(0eUE+uQziU#(?cV{n|2%k%Z8v-~e;b6KNr~niyp$
zFG{KeEJ`9^1Bgnz&?>Tzr)3`e^x4p0exD2H4k7yrEkESBOl35wQgu~L#yNy5D0=U&
zf@NyS?IHp=l6?5J)^8yA2=5g@*FtG;jL+Qd>-pt3+MLVMa7IHpo?@VST^`UAH#ku;
z8vfKNR+|f~XOrCOjHC_R3`tva_|SMnSl*j(*~q5-`sFV`D^lk{m-J`0F8zcBH^OQ{
z)0oM4S48yE!s7zY#;7*0K8-Pual3cNcccF2J3un+$v`n+^yO3HiH${MX@L5=d9sw9
zp<wM@S%Pk+uMehsxhrQX(V}5Pa6s?L(<co@UCs-5_nnL${UFG08*VLiO1!Q2Cua?2
zR!=N#YAdR9uqrRX@nZc#xyJxk!_G|$OKNlIW#JN}8fK}@gAutZX?9xrwcVZ^@xbBv
z@Tk$!e40tRzw3kiEb(wvGGE(-kjA$o6Wbq`6QJ^!lw6l~(}-b?uumF>SX2xZ!r6L>
zrA>-5G}{);N=SZV?^B;aKynO5qPbcgj_DYISviMsIYwzoTwhzaZ%scO!rpY~ZGNJc
z@`ITDVy^zUMkQdQEaNKXY%K#g8J=;1`PrL~w8j7!H*m84RoZdn!>_IDqgE$9QW_QF
zX2%h*3`<9L$_G{&0NqI*%}r>)<t>W`;q?d|CJh6QjaS(HPskLw{KO<a@eQrm-Fezg
zwH}@f9uwi4y<b@a!kOe<!5wxJc$-Yx!%Zo=6)bQib){o#Mx#eE9;TtqrTF}<|6CO|
z->{KLx05Yp3tZdr+V2Y!-ICW$^X*TX-oHQG^gOU7b)<hHtfW_c;BL5EDrH`bKm*y=
zp6gfe43~8Ha2}w7iBj>v)GHXpFO->mNx~<W7;t>8+zs*(E}TwXMrJGfmamv;xmM=w
zL~s)cMRAESz;%fhErkI4gst&Oa?|bFLF&01;nxBmM)+G+cmqRWQ#}aRWMI|<1BkCB
z8td5r?atvZEYN$+<|pG#-L5LeuYT-5xSo06<}hxcW7SaqxC#p$uJ8?GxWlw(*EQqE
ze=_+`PDVOy2kb6ITZWceHyliE7y_%w>}KnUz?PX>-p)P=psw=V8@DrA0K7k`q`ne%
zuxQdq7+wa3!HXYbHa<-UIxKcYvo#YhFTK{FxsHEuInBL8XgSrRhXo~}J=5^ld>Y|}
z>AgR585$=uRWHsg?{yh7Sf#=tdN`Z<DT_ofD_YU4^N7yg@OaI!Xj$KHYj64L#-ZPX
zNBB4e0|c7M+-D$b`Ln|D)OD3q)`_)%dl=gdg!GG+9;rrrYpqmLu<7z46Xyczz-qet
z$wIZL3iJgZTx+{)-oWBMb?oHxm5a&gl+thQ#sVmk6n2&?dHN&Gt^NI)(Lu3yr(IKS
z3-(?~06u_AxX9WjwG^<<vD{#)5a_9K?E2;hu<=)kpik%tGwfCn-Oa$Z>JitqH&RBX
zJl$b6DoV3V%X^ML9-)|Q0Ya8cTLt(7UDiln_j7;9*eDNJG?fXsY!sWxWM|9wlxd=E
z4X_!{Z~XvH>(5>P!>hFDN02+0^uv8_puz7S%2-D+^7G{41PKu|O6=z|>$?%g6s&ak
z_TCJmy>|PxvtfQ~be=;<p}9^y?h?EZl8-!_3(xXuYgq%8Bdy|R9SbXW4qRI>Ydhvn
zfJt&foZAtN0o9m-A{Z5=^G<?%v5D3p9`@{Le}!)h)M_bzZZ}pRbd!R6h`y6STF^j+
zWYBw{HFKzkd4V}RjP@30_z<_N`KYwiZ^zi384U@{n+c<)n9Cj-Wx9U%s)fjy@EF@D
zaaKzCXK@_Ych3j4%3mCSdTzPfCD7x+<7#{(yX$Q6CHx%%pigItR5qn|DYO37eEqC0
zH%oI4ewWVvZYs*m4<BfnI}AnXUQN}`S9v3bx0+J2#=yupEJ_c}$7n$Yr95YwRWo}j
z_m$a1KvDBa?v})XSwI0_SNKDjI%KPbv50NU1=#ufcK&Q-V3>V~{|}E0J=t%r8h&$Z
z(}^^Sn@ue$(z>thHLbrVSg`rhFdc<oP<+Opkdk=!LC_%TTkh__?3`e(#9V8>on5a^
zMTWnKk~>;u0MIgi9g#!KVE#2{)ilFf)-5xU+bug?r@IY^)=3@WdxJQ?h0+JSIl+vt
z<{dV6b5~oK*@pqpNS5#)(5T7w3AgQ#AcW5K`h+Pqb`8ID0uyncpc%2fOrQF~5M`@W
zwY^owRHNf65aM}<5o5%j-)rI03J1*Qd6!3z8h^yszROG4sDkEwcD>jq@~H#kkipk;
zT~DkcyIJ3h!Qcrv8RFYYl91u%S+gfAIKe)`e=*slerb#dIrM`qu~5`;gQ&e<+xx6f
zZzoiQ@%z4z@Yc>uA|RHRKi4j8G##|nBl3!ZCVx(b+%*kaz@^X8Jlgzv6mUSctiW|7
zRLE2VhTG2M17Gr&cLUrAbYANnG)h%UwAUS&ZX_$4>b-V>!da6u*#ER{H8>VyRV1Dl
z2+WMonx}z7Xo=4x$-Tqpo;79yD<TXg>*ggmy~6{>;sSBY!i}FLGPT*6=B_llwLF*c
zRdWCM(vixVmoWaLZ*Q&?e()XB{#A}DHG<dh8G{h%jU`R&6UkPRbJzKB4vyDVB!F)C
z%g;R`;qoHgMupKaR*E}wMa2&<?2N82%c{7Qc?c$%IaA_O&m!q}Fj`?8eme;sYDR&y
zcm2oUizx4J@E5eNW$Tv>9A;Z0d|5*T*HyEBVZ@N1knlD#jm-1l<l;TQ>lkZ9k16*~
z-Q;R-qmcQ*YRZd;vaC#d0ER3Z)$pMJDe=r4I9&ne0(@=v*2d4{pQn(u0rJpy*A;(a
zMjI+I4-SDvWI?dRu~fHKmI&mQeg4&vdwlq6Z(vU+f2E$+>MnP`w54gm`O+v1Tvh53
zE)mru{J)5L%dn>Z_YYJ732FFBcZdjrNarX?0TC%F0V(MmjYvyNH;96CcS$21(%m%}
zFgDnL?R@zC&$-T1dEiCte&6>iy|nC|ZZiETzmHt*^;AhZBTuw=*fiXuC;x|sPtxI_
zuLnUzo^yu9j}Z}|TUJK@v+fE425qchEZ_zG@1tS^Ar>e3fTm~a!H91MBMx2S@duJ*
zS<Ajy@CvBoBA=b|XXv|yl<li^8D8nTgD!Qrd_|Y{ep6A-I=m}Zz!)CwTvQf#9;?(Q
znnK*P%+s-Vw+@N5v+BLo5(g%CBlGj#C-dIsOE-eWVD%q%O8JLzQtrEpn6yf?6_i={
z)q8h{RxZ#8grWrh4gjE%e}k_3o;oundD3l_r@A*|c`MS6wcFvu%glkdF@4(wRE_oY
z!F#XyeYIPUE88fpic!z~2W#jr;PY2GII}u@TeZut)~f(OS*ufd%5{P$uh!Ig*~W0s
zwutAZa4i^kTeec(-XFDVneeNf?F|1voL?0Fj47zFT|Af^Ax>!9&=g2sC`^l%%@Io-
z_?agv?Mu?v*fNy`co{keF0irbAvO2Uyw?S7^;G<HOFF4vU%C~OLq=CN0gB84&c*K)
z$F5)If6cPzl^gRU$G}{H*K+2xPV8Dnl8WTsL^Q3@5d+16xNj`HR6T){r=vor2+M0?
z9C^12wq;{9G>@_DW#A}$FQ2VL!5~`d4)>RQfd{c>OG0Iq@dn@XBqD0g6-aBW|N6ff
zmjoVqCt%lq3$&N|>6-2Nkc9B{<|uC8^IEufzsLwsQC|qhuvv43*eAuZ3zlLB!tR1c
zdj*v5g#lyaX907cu{2b3H~n%dR2TP>l?63&rCMe_8t_U1&LoLL833m!e$lw_qhMP*
zzU#W5f7-4%NADyQd^x3DoT7iaFd(HOM4V3bq2Io6Q#0StdRBO5psh>#g|KUbsio)o
zI=_08iHDvqdQrRsLC~8R^(A^@0))A(UKeoxlpCBVSq7w(b+ARe>Sd1;W6J>&xY)t9
z?6%fY-!6a!!w+>HAC#Tz7k3kNm!#iuq<gQ|(zx&Wz+hcYk4#iI$4{yMp%SQ*5C}gk
ztK;?T(LvOF9Mt@Nuyh^~GH&;BSdZM6hF&rgf(qWIXsb(pWwWQ_GruwMbGh=?v&7UE
zZnAbaTYnyR<GC|Y6$R}Ks_P>W2uSQCv)vo@ewE{Q`S7ryL=?{1pRkBf1i)QSsA?9(
z?V~W27lp*8<8hG*30m!<MuWtSBhe;(Mum0qO?JI}rW?cvAX5!cho<-3DRt&m;Bmgv
z*?Qp3mdC751*e<P*Q<hQO`J?!eUpYI0whomQg*5ZPf^@11Ov+{|1ZGUR!jN}?l63R
zVEGQ%R{H`2#9*k=20)u(sZt?!$n+G_IRyFK4KhMY6Du2A`~ii{WQs-BcQMq~U~sbQ
zux|!62-Yi_uhE61O}l;Mxj;5SAuaOF%PPCpsf#mkfIV(ouoGFMFc=*B%hi?#F~xU`
zxGLkNyyW>4lygKWe-b*>9V*5Wg0EDm_ahfK0L%FeN}c;;YI#mG$WmNcQzgf{FzD}x
z?|;}_`<we*`7URd`{akJ7(Ms9y;(ZRw9M2(Z-M;-wr|~pxRv_oW?h=Q=7dq{JPYH0
zbAKbcmxa>%Rv1j~xe~8RlYACSvNh)TMz7V$_JhF*pGYJ)97)peB*X`4+Jad(XBXyp
zEpd0C;Z;@P?xO;O95To9SixaY0RE>(VVi2=wRjoAe4EcUdS>Ety&1vPC1~id1k_uU
zg-E#Fo(~oSnMl=?M8@px@QMVq4xEq`Uj*oyvFWwmjuq83Lk{cHoy7a^utz-_N(c)9
zG@88`vc21xrnOgoFp&-P@swnFZk{BU2`s6q$cCx43QS@y%tpQ;xDT{;SMA{zOsvW)
z?R>I)r||88D6Fb^>zmSeMbnyuG*IUn>-9dI3(L6oj7IG=>Au^%1ivTY81=jjQmZb3
z+)4Uh-y4b8#uquxDL{Z%Wn(i{%OCY}+&G651q?0zH(10~Pz{m8_7J|ldf?cT%qcYn
zuX@r;8ZNm4X!b90cigmRt7=%p+t$kcj%$~R^}9^W2epM;FVb?7(3gc1zgm!qs)RZu
z*X!#5?<raues<7-Q3Kulvqwo~uTj{BOZp5%ft7#wKsuLafHip0JoT@VC}y{uTr1bS
zmOV_i*H{lg#4h735hsa?f@D}@3;lyXC(sv@192P)HY`)T4t88v#0Dood)RDYx6h~N
zpo^myQF{SUKeHu3r7*LbW|rHDnyo5A8^b)omB2>8_$s7|6l-k5az&FCXt#%$IXxI3
z#oM82!M=EST}(RuMG8j@Ybn1@qlB0<aDnww&Og{_1C+SnJ*55YufX23lR9;FX83d-
z!><Lt{q9(wZS%6OY&_V^e!7}pKz5p=qup`@vnL7c<9@T7=LX=aQ+h>$4d)Q&_Lw{A
zWnu(G!%vHHTPgbon_E5{#zu6m;i*14i_v0aJdd<uI>JXUr1~Bmki*hAs4Zul1NYzV
z{m3&oij<=d+S&@4d`jX7OWFQ=g!?yK0x_Au1;h@}o@nNqYnMMETypkXUZ%sL+ln0~
zq*Lb;&#RC&Ru$r-XrUP*aI!(J?8~GzYcJ=FW{V4IWQn}Ef9bXE>q=(m33>mZAvPfO
zLgjeP`*jqRhe76@_6T1>_^Oe6S^ADlq$1$c6(uwAqCE6v@!>uQ<5O7G_E?g#+KA%K
zMSS38;c5n4ZK4u){q7W8hI9$B1-cRsQr2Ar=m5nG2<>9O<h@UP5u16jE%d8<<^B!O
z^Iu~C3ZU&WtWxcKzuj^o0U#@v4-lB#m)$p%nD8X)6unUea)ce991)sb%clXK*CBEN
ztl&<Tjhpfm9J&j4_uwodULjb9F;`Q0%U<ST4%FWUzUtQfh}r|`L<^+iEdh?rd&aHw
z^)ZV0n_C23;9YOk7kMfs04pQb#~Mp_-JTgCk$Lr^=CU>b>kx#5-(1ycs>H8(y64^Y
zc<->f0J*}PBs)ON`5PbosYKDg=~MD@8zxX+{f1+rzsMll5MmL7XWNlK0~8+Ao*fqF
z<$C}3!0o<ac&HtIoRa(F-d%g>w|-*^2SMs4WU{=ocVAPR{7V}7f&w5cmTRSU_Z*tu
z(ECOFA@|KaVQ4>f5fKy<|LEPRR!R?M%aa0fpL3kLvds(ocU<vwk|wBu+9}6tEM6@0
zrK_i|F7lwG$*O0#O!6?iR>1n)^)O)l#^&`UD-^^p&2Dmgr-iV`mzH*${esz~3g`sS
zlT61b*5$j#*XP(|Lsi%&Ulzx4L7P2xct>7!y~8_uB@Iaq0muZ8Tae$r=gh@+{`m6{
z{$$@aa@z+{%pNsQXN@Sv{MKN(m*JxAtarQoL8>c8ry~uf&YEh#TM;t7u>h>#E-sGq
zzaItfeUXYdL~f9427c}}+{k}XC2RV?7boxwp2)&<M2_NrsFK3M_%M;Bg=3w)pK-*E
zdsE}DCEyDbX3Wc0e5e{_<<%PUByS)DhR;%aUv@N}|A(oW+MrS%2jd;2c%mi)R7MB;
zxn{#UDf`+breBt^osvC*vVixsU%-PuNtK#@OIYTslydY1W;yw3jVan_ou)xNvmw46
z8-@ij)1xJ_rcv|pa;KPh39oBcu~TEmgBUH|?ZkCzlc);hxpLQgnK|1*WwBXsJ}nUz
zaaZj&ISEkAd{xag)vO})4qpg!(}l@0z4)GXCSj5fp!JlqvC@I8`i~_E<UHhn*CGi1
z?N3h}wtZEH*KV&5KTxZc-pty|1_9=h{-W1agc+Juia*)$2_747LUl@G4<DZKv!-Yw
z+~B=r(wvm1<<0Mx?1o3Cza^zYar!@=C~fxy^56@tqvo0<0vbIsWh=Vv&w=Z*6rEA(
zN!%=M1^fbVi7apcO@HUa2aWvAr+?-<5e@YK(^v$wed`fUVEeSoi67>G*IJflaT(Xu
zBN*INbW=sNnLpecmMd{GV@>@&eh8>L$@=>0_&j1pq3KRq`flU0zAtol+ReT^OYcM8
z@sIxLv3-o#-OT_}QbCBAZ1^?SL#kmjL~WX`JCN>gPq_n4kj;OBf7T`3mcZgqorVdi
zXAA73k;1uVzUBfURW0WU11HaG;seUrNco=~^8hd4B`}LX_v$i&3nC0ueuw!D0oK^Z
zAIs<(b+(u~?f*lr0W7<va`z)ZMFXNH0=p(sf1Zkl#qroacN>`;&A^Fh`jdjnYK3tC
zb6!*#Da^8#hT9f){MJXw|DY||p*6*32kyXys$=Y~>!NyeD&y5up7+_`%UH0#bF~iN
z%RosA+w^|`_OW*v2X40v=AdKNKm(Jmg>B*tyGBnPa4g2!Lbm~e;mY8}<JA8{miI%H
z#cEDTNdevCGjbXHVNz^|4J?*9Zt~{G6rX-uD*r900MK33hls0>fww23Opb%rtLc3#
zpt~hNGe(S>)|2)beH~~K!{^cRfk5Qi-fTcIBH7X{`=Ei-h8ym+*z~7WmD+Y8Bv{}X
zEe9nvO>m#dS@pKdk6)m`dbp^bp0NKUxq8Bh;8UDr0Gnj^`n!TFs&U(WwC@>g**S1v
z;_m=;4Kw2pkf(##r1@Y0$P>1lr#owqp)PKXii)4uLK^sS0Q%-?d?A}`S0#&w)&SRM
z%t050lX+a#B~!B<^9s<N9f@a`RmouHc`w})4VQ?4Ayp#<#N|JcTyYiVF;dMJKQ6D6
zNPN+paZ(kRwaetx+RZjOnjz4%6l@J!71hjwqPOJM|2Oj~Y@0X1z}6$8NxNKCCrAvn
z_iBH(3rxn-3xkZ>E)X1$r{gssE_}KPO(!pu`H7U?Xv6O>1mbfpt=w#us&g63v{zL5
zc+dUU{co1nE4_+X=cs|Rz++ZdpIFG@nv@Js=>VXbHB$As_yviS)Wl@a!#yp;l%%s7
z=6d#O#@*d|LVu5rBa?tfOKqmAqdqCnTsYM%5P5XYRKZeEV=yxH&Ap!(*J3tmHlRDo
zqzK44PQjHB^{U;e`_9Nq{hvK>fCmJIvi+nr+fM*uRoj|+NMa$ii_b^9W9+xPn}d(D
zKX{tP%;(jD_Dc+zs`ueF`NyhW|8-l-RpnWWt1-HIYovilp4}XKdTAS3rRY8v#)SCa
zZZlV9dDz{1yk_n5)s>^4w0E_XtbbF^s^5b39KxSi-r*MRVDMFhPSyG`q}ja^fAdE>
z?hwoR@vS(Nrd+qRNC54Bp@I7gu;R)Ed-mZtA~JPt{(k!3L7jPOMiK%5twC9<GvhjO
zn(8rmUE8NUPbWR%6)z)YomGU`Go-Ld01NHxWZT868Hhz=$+ltmI?<0=eUSOKcTJq+
zhi)5%?UztEm)BQ_6BnJ=n>}Sp<HCTuV>9M@Ob76mMe<KR3MIiB!+P+cuayuBP=>4|
z0R}3ThoWCRgPeHZeMrmxu1ox#F#7?CqNd`Z!fN1_Zym*VB4ga2Yw@Iwmpae??eeJ9
z*2KSK-V(P4RPmhpWv%Y6QpwjkM`n9X>jILg4yI9eXY6N9c@pwnmd}q7`bq(J3i$w>
z{fv=}^P{9~F!fLCjg9y<nwAg!`5c<kxF1vaaRv*d?{?Uq1~9}8c9wsRNowW#W3Z!0
zljJHTPG&qzjn%lYsjkX<aj4t{t0pyq|C{1D7zsqxTUmMKyvaw}0kS>p$_sK$h%9z6
z?&jX5%hM@um_G-F?e#AGL{ueV{L+R+B0ojwe!aRuf%F0no!fooOq7XzE9IczB%u00
zB7rxuMTgFbqFd9se{5Sb7_USCfRZEHw|?$kI0^AjOw=>Aui6?fm`#3-uska9z@wvj
zy};CwQLQr&A#+*qSs)>B{6<@uF$E1wNl`&)n$vLdLU57UpN1&PogScNrP6n25+D>t
zQe#k*dDn_t(szZ-ceZbt`1k&Y^*nI-^APVn9<+lr=(=~7&wTY^FBTWH4Hb)=M(gCE
z-dk)EnD4g&MIDupIF$T7)2v5^^FcaH10f|=(LP0G(lhS6m3TQ1WynAa#s^XQ@>9+)
zbG)cK%FOT(<XLrx_jiCHA)L@hrpJ)C)xZlNTG?fg4BXhTGU(IlaM{LtU?35&*7kif
zp5rBsuDfqunE{}aW$)}w9WCh;AnBm!GhXO@Ti@8qbH@2&e-hR%0O>EyZijj^%f~6u
z5bnHZ>h=M8WAs{2(Jn8dR`rY=brT^hqG!25z{NEKR6aF*-Y*5pS=uD8`bbD+w!$Z<
z_xdr!2{z5h$lG<yomsK{003nbPFDcL*Q{_^p0Lz6yU%N^gyqhFwu4XH=O=bLE1#6k
zKFk3DA(?(+>4^BPRY~8nIjosWmQO<u!?^+o>875Vwv#Se=1@i^jmPGU<*X!5ExW0J
zN~spepZS$rJzXhk6*Qraq}$O>uk!Ba$t(&<=sg}hhZ%s(+9WQoq&B+%Hssej27RUb
z8)zWv@{Jj^0o^Rt1A8uK5dI$@OQHmNZ+=!O&D43qmhLQf90Ec%bub6FX&srUxYoE&
z*Bzp42rWew8i-?ogXub>>CGJy0xmNBCxpzE+CJsxCUYO`D?YD4=vt3T%k-eqdxAjI
z`O(ssK{`ZoAyZYc2(&~i@FMAn)5ou5S*x<hAj0<6r28M=><yjUJpKChz>z;?^dN)J
z>EZU&p9tnaZ#deXs%(}<bYx16u?iR!NK>sRse`1jmT8D@cuu({P`cGes(Zcu-AsgZ
zRuc#ci%q_vsgQ~wFWU}Ummuj3e{slv3<r{&j_}8wNpXXL9aa4o<Jx(D{S+bx+<!9>
zwD-$C&{O`aKl!V6@`ETv!Rfz(*%`n~Z~{kWxXWZmx!uZ0{%WhtOAWlu@D9~kCxd+M
z*w6LTo44?!7`w0gs-KX#;XrGl2ShukcH64#f1>SyLoWz|dqfM+bZla^w}gP7?qam)
z{<;23d^W)9=p?26VPf_hOqQSu2wSKQcK-5FIftV1!EI<<mVV3k{)J2Jc~Z%vJ|U$_
z5ga3bG6!-8&S}62aVQ81JdkD}?HnkWsL2Pv0d$Jn=IWYiP869SeV&}oC7sM=spK62
zn@WThzR=sm$xRkUF-JVDxZ2aG-yzYAfw@GejAw|07<0apBk^<Y?YH}%$Rk>RA?>mX
zdB>ot!g&I<M!Us~o&UsZtQcUDeCmU-FYw-_A}HfmA{)tUp8_V3=Wj|ufFkdJVVr}G
zI!b9cw)nzT75E1+E<lcC*}y66nae(=VP^%1Om-|D?(bzei#Qh7>f&TlkUHhMcgLJX
zK36r(p&;!Y05HgO#Bc5BUum7_e!FN<qVapsgY-#oxOmM2_darEj=&djvZp*|$0@bf
zz4k#sO;#n9pz#m@GVqjEED+Wyc@8I60z*%JWov%$_DA*=HKFfVDE&Gg1A3boW1AUa
zMhLvljLX97UdxWF=&hWIS5z+zBhASZuTlvkd1-1VyL;Z2QSAeLRvd@kj0^xW9LrrZ
zB>k3xINFJ<jm$}}j#97XMxTB0!E<1#Rn0fwjGjMKqIZ1XL1psRA~;fcYc?!Q*y-)r
zV*l4yRm;p)J#Pcv@Xpmm$3`dlKZUjK-u3u#ecA*_z9|>>N#4eV-}SxbgGB<{63~~p
z@0xBA#fP2re8*TzUd-2SbXg}=x82k;5U#e)N)g`^BiMGnX`(d<ZqN$poA5Yi{*A?k
z-$yKy)kdj8Q{}gMJPpJ^83s6;;~}r^k325wPQeA@?ukUroYx_boT&V>RFMZi0A5&Y
z))K@TDnv;Y;&V$BjT#HRl7Jd{IUHyj{0;)1&^#6cp``B)f37Oxb)P!|hVefhi&l*L
z2!O)WU`SwWrKGxKWw`}F^wUr8_KyB(3jvDbp&w`Bo<KW!5zxT4Tbh|Fb@AU1dBDtk
zEwj18w(+n<g(9;1ETnwuG3}7g;-@5y=_=rV8hDx6k@_b$Ya&6GtaAdf9)!_siy+>S
zB2O}lce@Lhy50yOpom?UzW;5<KVm61Cp~YF+>~rzBuUN|#n-@}*J_{h9`oXASMpZW
zsdFc0-2+;n?3?AAODPdofGWoWK*uhx>B3iPVdIkEb4Q%h*I~yp=)6oJyLa(({-T0K
zs4Kqt@k`ZD+c&wGRfqWyc?+yGYel%8h4F->fuoEwl4#K)nxtp%9_76w)&5TG?QGfW
zNyH&FBp9eiJl8zjESJwc73s?ue9`bZ@Be3JJA;nrxo+3rccnepB*rr0URkzB(3&g}
z)vtZV8Y?|^&NYRhx2J3Q(^u?lwLmE7<&86Q>S-IiR1;-OWQGGs0jM5b5ZQ15Zj~St
zK0#M`)PdI8vP(1iO7xdm;5IlG(NkIvSko;Kz};K0L=60a9zRIf<_CDvBRnHGRQm3N
zE^cs<t_ngLa5<vl7F7_y$M0s}$4w{NOrx7o_Sq(p3P}M8^`F3&2M1)T+;c6gVOn-t
zydL<R0ubd0vI*nl0Zt|$k8@V79^8(qo_uX0-Lhl$_9{H%yk_?$lgKX+;^64+mx;K|
z1^bdwRWdRjdB~jbawzy)5;cw1x0=}*uO9gKD6tZh_oV{A3g^qCG0Q;5DZ<|(R(Adl
zlai;ts!V1yan$srHv!tVWlWN=AMTyW2aJX1!xVl?3$_zEH-C{zYlk)18cJ;HfD{51
z!4oy``p4^7*<NA->X5}!%nRq>m_=4#lQi$1HLG(gOfjQ}WnZ_d<eN_wwJ|Ck$-qTK
zuaUYhMZdIgkB)gS8INAr|I2l%ows~3ig-(Io4x(merMHo4PIM6{ekCq>97wBom2_J
z6nUIfIjg~Pk8iH>2yft7ra9j)Yp1dpZHXQQ?h*&75b^5Gh4tsgimo+}^gIw<Hy{d@
z)lTEq*`u^&v9EcsiG1#e%)RcR?ws6h7>`9bHZB&ul0N!fSZaoy7WfazDTR5!##>|>
z0H4+Exb0owTuk<rnO|`HW|j;KG$0H9eQEOVc%$FKxMfJ3jDompH2lktGsB@D4rTpU
zhB@Wac~Z^+$3|uS4qh+p8$S;8h=1yP_rLDCZqmT&ksg*%p|vJ>jzG1Qc?F+8e~k+A
z9>JO+vGZ)RB_+9snnbb2rg<t-8w0M$O`eWvRIA6`xuBhne5Gr8l{-zzZTT<=42M@q
z$;d|vTZ6NR_Tx&@y*<AKfwt=?rY)K_YnOm<mx}ik`YdDBM(TIxq4zN*4TZ6a^~2&A
z<fx5IlEGQU2+)9+=574O#9+t9{G0usi#yF*>%K>o^vE?H#|+AK$vzjRolOa6;A5(l
z@J228pqqqxtS^jzO9AWGSnPy^UHtsB^W$iTzE1+{j2<j6Yy9xB!sFtbcaO6tguu@?
z{m<^qw)z>maoo!Cnsg^`@ur_EJ{zoL+Hz7Idaw1@hJC)om`jMAG^Melq-m?Yx-6RG
zMI;a0RgMP0&=~1@H|cP`r@u;oeTrd5pG7<g?rY?%(ms2!E_qplt%j7?&AyN}4eD~q
z(d&vs0aWB1%R<&*ZJU2W;bP)1K4Z(;8`d70qj~H5rfkZtm=|I55BeBQZFX-6Aot>s
zz(m`X?&3H~jGx@)@2&|}Pq`kxun86~bGxT88BjYFiH-7?Hn%JeY13*kBh4Wu`_d#2
z4IKW2FgB5ST$*E7-sTYyxIF8JNxnO^B;xdnmgnoNI{GpG9Sxx1unZgm(k!nws9~O8
z*>VNjB(KP4k0sv;(*13PC|~X-6IavLo#crkaly8=wr3|lWNyj6DAwI;Z?XNh3hWM3
zLHxHlufwr^)%Z~VdI1g!VxPSXGEF`0*d11qd=4XD_%uj7lLcz=Q*d2#?b4b@pAIPb
znd>TH1>-`l*OQo^xW~3ltMTZzF}j0F(K(_tVzAY}q|g8MSE6>|0ft=Q{|O{3{wI*c
z0~CKz`tnhLxm%p&xOneAkA}}B(HLZKAdiRlAlUMYtuiA$s^eTGClGGM$oQ3nl^@~R
zasJ|N|3h3D8a3@4c!ZjYx78|%2V!#v^#pJo9jenTgGuBf81F1XNVJ_zjMPy#FP<KP
zzd#OTW4fO-C3&4a=-oN6{CdzLsjiym_>ki%;(%PuUtHW0W-fKPFzv-RFgq`HN!d%y
z?OH7W@|#1-PtVb`lnHp9M24m3NuhKZF?T@dKz0{$D1LHK&UnATQBOG3Cz#P|fj=c)
z^*&1ARv#c4y2SR2KA2ovsoc%8IsmyFyJ_<p5#=XrQz|Xz?@ybjXGnxr-Xi9!d2!B;
zO|4!M2uZmo_mc)D##VQL`hr{Qzshbu60DgJ-_QQ+=h(=-Kc7F8%&k*u)}R+_eyc1$
zAREbLbOQfg-_MjKWbHE?F|OurU5#zxZZ4WRX4>SsbU~c;Q`_=dC7VB9Gl<RPqL*iy
zeWn%tJ*ZQ6K9f)bJ0n8-Mn1nA4~qcvhlQ8u-1pgT2Q<JL{w(aBepL|WW*vHY{2Ap#
z><EbdQ@Ek*Ym%#f>+NQy?-gO1tAG_mHuv0>VhmqsrMimWDMi)b*4#WLsQIg`4qM%Z
zL^>nTjfwqq>MR;OWcavE=3f*Sg1G)NU=Gky5pZQE+%NtE2?ieQr-0J$-HZQV&FL3+
zqgcZ6Ip*dW_Ftm;JDWc&*gTvxvw<xXU(6pKES`Vq|3Ko^%3Vd-thB*El=2dfYSr*y
z|He*E)WG293UF_gtqNBH$T}b4KHQZ)?{xr`Yn$JW`l|EqWmEc;zOMyL3U&TG<qhUb
zw=W_*55Ll2%PG+|K^`Low2bt|Jk4s9(M<G!c>TQX5(3FMjPi4;x-4^Cswo_DOPbSC
zuV_~D4Lr@h{4>Ug(_hToF1qTsnnyGb{XW=J-hMGm95loRIoK3_wXP1que-l19e&H=
zZ=Ngul5OPyfB*8PSH2P|x2BvB${Ak$XY8hg!~GvPnMFnv-W(qBC*&+M{NsryU8_V!
z#A_pb!o974o}1(eVOR~}leRz4{JpBPG`Rgm=D;|txbX^!*YYHLJ#5mF4M<u%flPaS
z@GC`9Zu+eRQJN??2qTE`G<Whcc*wtV-vKEcbTeGb@jX96@h=h<@UR<qw1vA&mainp
zT8_`$Mj{MZ-Am>RzH`Z4x-52>_1T<y2l4oMR;D4dPKN)N`T7qYq_pyIwd0Xh*|^SK
z2ljc6vZCtq?Zgj*OnwfQ40<f47r$S)hT)HRe(%A^j28SC7<<&v=*z?%$<IL*dd2if
ztcr8$+&5&$@Nsi6bC2IojuBajEqDQtY2z0Ur3;;ZVBof`ADll0!EoHP@$dtcikj=|
zWcw`T63jyq3CU{H<niIK8pA$L=et2%2%=Y5u<PLUbteXR&L*#2aU{IDwgTR%@d)s3
zlf2HsWFX&sKhw;?=&;JcT6%3>V%F!P3>5?#0qx_X;ghVwMd5M@b!+{(W5-Ud5ZReq
z;Yom1Ya9EYR4Dls7!h+>%pAbC>Snm_NA*ji?VrAZV@G|@(Q&cO{<s?H>Rr6haf$M>
z)q%jcY`*-*d&MOn`Asve-By|<5va4NMoYY;74-5zZGhJ@^b8ud(VKuHBCoR*qgwK$
z22Z3mS?O^wF2l7s_I%Sy+SZ!h-HlVef#?3jJ$K!zVB-Rheia;!;cNm<?9cm_=rof_
z2`&Vrb|hY@W*>8si{m#3da9Lc+)=#1>D8?Mzv!eAAU~No5ZnG{$O4GL-RQ=miMF!5
z2(GwWP`CX%s}`>bf<U-%J1_sL=f{UMCj*{!ngd%wnUm2x-4sU7uZ)PeU=i0Eu3~o%
z5|PV98wVL0`mYtP2fdnRCy)t~6p&0LY#KN&_e7QLBFwt_e%`sEtC9qU2?WeTrwJyD
z;uhko&~K(y?JSvnwfH<8#%$g!U(@7Gr^odN4qsFQX>{_St>~h1N@{_w6B0*d2Md(^
zR3ihIC&}O^A{bp3%_yxvE>5Q4(n%Uk0^W64bB_kUb7$GHmLY{LYHNbHqhRHzSw@69
zit4ie-4%W12|&a)AC3f9|JLUFNCd#U^fkO2vPv$&*z&}L3crB4>3;E-OTTOHDWAL5
zJh_u;&^=uF&Xs(uPAj6#TPKJGI@3vE+xIDKHcdlvZ`Ibc8u6{Pv=-Dh)qEhF;3F0Z
ze6{FJhL*=>5vc2d-a^s(BND0x1C~wdpDQo-x3c;@LW3H;gIb0nD`V8N`dz5@3=+2C
zW5>j7X<g$Llw7!K`Q`%O&3t*<ixxLS%+MDj3qTKW8_Ki#?q1-&mX@tdRWcHG+zC<C
zS!axd^?pTujQSrrZ&1G{m^B}$6-p|zZusxhd7ebqZ)Jl|doUXFnvCPP)6lD7U~KBX
zQ0f454;l)<W}D4xidyX(r@Ut}+vq<xYz%`V5!A*ma<5sA#)EVGn_UZ~s~&kp(#35x
zBq|1Pz8Xj|tG8O(iG`lN%2tM=Y+BL#gQAx_<sI5f{ul-NlUldIUtMq9a|5Lu+Xp4p
zJ&RHIay=)po8=sj07sYAZg4nOV%?&B-ux}F;4auH6RSMfwG3T2+yb~m!S`^9&q_=i
zxGF8jqxTA9%{<ZjT0O1{rg+^Y$EMZR_No@{o~!hp<MvSuK!6sXcj1K64!wdOi}|xg
zRVSrjMfFWQ^5Cs+d#2GQW9spE-Om4teTTyd8BlK0;!d!`!gQiTU5R~L?tu8s3pxIT
zht}u*yEm{?M0DO<5b9F#*V12&8+MMw=blsMP4&XAXgX?t8K=(I@>34-mG^H05ni78
z<0@f%^<VWzQYrDvTT5Z{eDPh+AHCC3HQj=bVw%4<SMT0EVx2b0&m?d09%WG^3$yRL
zkkFQHt2oQqX>k$>@+#^k+4N@!xSVR$=inF|x%p!MW=}7T!mo1wnhz2!n|vO`c$=9R
z9y18%>(d07!g&c!B=tK=Nrsebx(MdpW=pd>7-5#4g~NCOFYC<(@}zL(Iw*MlA!f#<
zgT33~*`Q)2Fs_8RPEx8C<qLpu^&S{km4^4@s+}}hwr&dHtR~sM9c_2sZmG<DwH{~^
z_-Ur#@NdK^_pGD-zP3zT`%RhMNt)M+ddbrSXz_~oxz@DVgxFI#ipK;v;gVR#jxAok
z+XYpsQceWO<wv4LW<keS*h~n-jYE#<T_qAzxN<SSJVR@i-IcZdg_4Nn4?w*VJqUl;
za<K|vR1f7AocCV?vJXtqO0Vp-Sr|t9y1*IS6aU(q;YCk?>71x9+m!#2PF^XRYcSVd
zdB_eehP~>N416+WnV6~dO#Q3JNuB?|1J%4AP?(kXl279LEupmC{$%_y{r*@aQrsb}
z5mBIE8$XVRoT8n$9mxN=4k*OhuZLIvG(qO6jV@?HM7-+9oQVc4cr6#^=A{mzf^F5A
z-|ZbwUY=Jyyg8WryCO$35>?jpBH`eX96eXm+TVvXj==&XGE7TYWM23{m@Z4>6nRoD
z1L57Cu#%pzW_If<lDi{yBRAikk-($9pY@foNN)0N7q0j8m;a;(_9X!27!?j({^2ct
zF^L8g4P4J)G5B1mC=MN!g+VizJU**!mrphXATkMQd$25J4c>?y<cH7!Z<Xj<tg%-k
zyfX%hTZA1e5_x8N?lVazjY!=Sm$Wd|{D)cn3$fNsz_NYiuS#dK9J`n4?);nS@E<*q
zN%;pF96Bd&*zwy?Gk%yDXV@cmQ&Qsx@_QHhP=o@0g2xcs{`ug9+0UF;Rx$#3epIOj
zQruzv9DKawd=?|h--(lDd%XkdaiZ@Mz-qxbeTC1OZ54qH?Z?cGL|zKS;?IxTX?lt=
z)(D1ayZMLsno<bcm%tUtLm+fZS70m!b-Hb@5?S|$eP+#>x8@#XaDM}TM!2Q`e>6Wm
zE&-(+a=f2GuKt)iq`7>=nPEShG<4beUC`}+l9wMR3SBY@))jWyK?X!mx!TBxL}>xe
zmfE4((D!fF+We!+!J<1J6Y!G1Jg?;Nn>F9GVW(LQ(^REcrX+A$%TwqpK}qS$s;+%#
zZH--sKGX<aH+y~1pEk7q5caC3XdAb;5WJr41gJOLb`GO|aA^Dk9DA<UanIJNA>RiI
zKPJ0rGffZ0NuJv3r+sHMyoM+rNy;&J{?nd~{V!*sqr5GPY4<t^FyC-8DTG!hJXKBh
z5~O(O8ybeZNPL>`ZJ1HA62RUE3e%`UwUzrGVRj`9zKX41L2`8-6ONO~C!<a!juxMM
z0{Asuop+zXd5`=%Q}P%S9z2mr>V9_!jblTB^&2nR6ACp4wtOGS7leMq*D-Fp()X3n
zBC8lymOl@8AXmd@ynd$gG|Vicl|4~HZB5I_q2~233%rJln&6d{&WmAWJk>$mGb<@C
z#>C`De{Jro=*!qPW&hJ|4)g$LpyJ-ZufED8b3Vka9hNvQ*YV=79v>Gyja-0Cy|H=_
zIfst#gY(b7P(vDGZ5FS*R;3^L{&MK14`XuR%sH@UM8ny(eQtPQALS!&u`gj|u1aQ0
zMRjqK*rXX_{_oyIQ}_Ia3Q(6Zo2K;GScuD@u1zmGQyZnp(G9e=Kmyhn#deyiq?Wiu
zMc3GAPWV}#&qVP(Hr{%)vP}W^O6{kjPeDi@t<+z%W(nsEbckYv@3s!wQ&XT_G2VZ#
z$f=@y=CpS3Gp=6o@a4$dhH%?KC~j%Gw%_MoL<vsUXPs6|G|!&^5fOYH(9ot|oT(5N
zXPC%!_n@Zwj58WsWl2<Ies8JbTL-Q#TCq6H#9EfDkzUGG=keqJUQe;dz7^8_{Y<U!
z2iv!A3(=W0F>01ZGu|aX{=Y1M`SipE#fTOk5dp12u=niEDB};UDpO&l%&gyXe>;@O
zry}X(c9+8NwO%|XBO`zD?MJEYKvtLMQPZ*Hae2qY;;Oh!9xTAJ9MHDy&5t6@ZXg|q
zT#WEM3wbI}a5$9vmXFf;vHASy-sz3SumqN5;lMFh=e|eVVv?{3lNEi{NHbFGwDL2g
z|2-}iG3fGQV3lL?%n};V7*DJDFk<WVH($Q-4I-A++k@ix@68Y#g&dmXK3T1->qhkk
zCwpb1DG|h>jl-at1B#6A2P0wUUca)RL&t*Bpj?+o>Orh_>CDm8#O-ze?E(=m4t>~5
zF+~<ehkrWY>NgL*SOoe%B9e~S$S>9;>h1~JcT>!-upA`#!Q4IXo!sYM{%h&5cVZ=u
zT5^l5*Y#saE)R`+X)mtXV8mLDQ6^|lPzG@Tm<4r2L7e&$%2Ys|OmwhpuZgzbsd#^+
zb$&;vWd8QHj8X$OF`<b&Ew-7<om{5t)8L}{#iK5MV1IL$LN83<F-AW>$BwUP_W7ri
zjAWTih;Ng;3r=BnTr82JO_lAn_%FUw?pk&~#KdP`a(!{JHv3V*(@nXy{&(M|UiKx)
zf%Bti82tYQ&$?q~bUw?+qM!kr%Ck?X$4|sN0A4_l6ozQSCXVjrbzy~wJaVVoWJ?(I
zW##?PV>wafJ8?Wt<nnZtQB#O)@9NG#DRww0f!`bY+Tg!k63<TtSj1paSne<eb!>(~
zNm#>3!H#t<AsoR)QR6Gi{xu(i{l+qd8H-Ih=$Kr|p>+E-_~!w?DL?f+!AwvAmpdcr
z*p<6!A>?tl4d~ODw{l?9AwLe_?#tP7a#(qd$((O5S|%$dmlODWBNaPt%=H?J(J1$V
zY0?-!Qif31tQ;q^L!a4D`iK)_hs!&rEcQBts}qeW@iJ^F4eX=z(hx^w*yxrqBd$M>
z46MIT3pg7aI}KhMV>D*bgtyLpOv78#`j4ungXBY-mtzNgWB}PLOM@q6;KErzsj6F~
zq}o|V|Eb{WJu#fO3&ZsR3>^!GpY|I@j+d0*HS}AE5`!TdnbFMFWF8Qjt$2|RuROcx
zvbZZLuiFJcDta7Z-|hq(j2L`?G+Z}65VfXs_gajRys))dT*=`4?v@#lKoroPfUfB?
zop$;sdYtfQyD(D8)6{~=6y$m(d}Hl*VoN;|IIfdTI3MFmolGNmD&o;gEc(w#Z!qu}
zg5V?w8~oj`{MOZpQy1pDC+SuNMYM-=W(y4t*x<J{u=Z_v{_Mj@pW<9jWg`YJ4hrLQ
z8*gaWvA3>TY?sSn&lH)Yi#*tw7uPFxB&FLXxZCIaFRNM&WOi6dOaACZRl&3^9B#v&
zv5sFaJcY$aliA!Z4s%S7hEHCcI=6Mcw;_<PS#yBVcD$9Gba}CKE48!q(?`008^=HX
zDn_A)q2HNgz@7(>b<%%(#<?mNH3+Rjv_n~E<el1gQAg@l^BMG2=u@G)pCU}Y!-U7H
zHombK9_BmP(RwHPs^iVmMrdaQldd%4QIPg3^dV`L*P3V3(L8-+X(y%&;|H(f@$#3^
zg6#Mb-GWwZijT;P@3i6hNr6>MO;WA)GtN%sEb0Mu5V3lz8AoX;YVbU_jdY{$^kV!8
z(M40&l0gmz5mfGkLcU;JHRb{GqBf33n?O}d0nm~x#<RAqpu4=Iym92Pu|mu$AAwM>
z!l;c6AZ;CVO~4;Xk?pmQgBF~hZ{G4zIR6*rT`S3BZ&e=kY$bd8M3i)noOQ<=n_s5<
z@H~C3pEnqY>7(5bcT>p^#>=lpkj4YyZJx~!&mNOqo;_gcnT*d1s9(NZaG8t;o&Vdk
z&wf{NGs=v7ys&8#s1BR4@mEu1b|>dQs&)<h7qz)y;lE`bu|dk;8|b+p1f$QoRWU-B
z<jP=P4Vi~^zj|pYuA;^@`lYZ_NRmsP9wn`f`b^#Z4hW-MzSUVO=^DVb4Q?_64aTpv
zT53n|3rHRATtvENT2^aek(}2TkWLj<#8tQ_l!uVytq!|-w@@B`4GY|(2dTJ4WG%l=
zbA@vj0x}?B?roDMCzC-b0wKm{O78$+k?BKZ!H&Jz&4&&~8OcA28T`PcM%7GiHy*W9
z`LyG0^^k)}2OkZJjhlL=HyPu<)Ix){;vVpRhCO@qsj`jra#;AZvtKsf(67gQmeTb0
z;B!-lo@UnRPcmC3<dCvnmG5lXQBB>@9ipkLqq1(OpY#SFJnuH|l+{ek2JWE6lUy6Q
zo+8;4riZ3irYvKsdHT7Fv)?r}tH5hKd&z7`#CH<Lr3^xMESfZ~LGCpGjOd6DFAh<x
z^Ez3I)1i;TefI<W%<}pde?SQez}{!hX>GzL3_c|T?NnwP(bUmJ47~3f3BZ`Pm&kj5
z0R5_;`h^z*Y>G>2`0v#fO~{qK3w$FWlivo`f=vu|JZt7Vt=+E-h;&ag3-n>wtCt{*
z;YGoksUm$wUz|1yj5Q0fR9l!0&BHC{WB_QkQ3g?T*hF^Hua>{)IT27g*ya0;h}Lye
zb5(u)s`Y5i@7h$<1Q?jTo)Pf|Td0u1V#N2oEkVKqGL8hAGX0oFnM#-`AlCD2W#bMv
z`QaVgL;&XN&k<tz+Z_FP{<F+d^;lCH3D)zrEtYd~RsTk}3WY^=mQA^j8+XAIf4n1f
zv&=YvmxfdtC<$G5l7T!Y2Tx_j{&`@y;huMs&3$x5#;p6O25~u~RduuE)psc7Sarsp
zJRdODxnRBj+*EY<_jvUAGc9TOzsy@2Ph;hJX-)1WBEQ`0_MKMZ%_*@}ujn23DCXkC
zJty@Bw;N`r%^gku#J<;}(B9k0Z`DRb80QX-)9l_;jq{@9h4~d^7_{viZ_rdcGk}+9
z0?}2VwPN+nWIRsd=vf=;`C`U^GLcA>ykqbJ+UP6sddHUM^fLjc5ou<Ly}|eaV@tUY
z`@*}POzT#QSRJbJ>G7_d!PYWwb5|G~t5iKxNBQQmPB<n#K$trU+tU_%d(XUDY__&K
zMdYC<e>vaXj!~&2*y-C*R3!+mi)oNRop*q=f>0F``evv}doJHx$B-@zfmT)S`H_2R
zj@M@H@}b)<HApXDd!=NZwfO2yVDzdCW@Bt$KSJtYO7LkN65LfO1Dg*rxI+}>O&nz5
zJZhV0zqqr}!>f?a7hGv;S~UqcXHU~xzgwRH=bh5jzi0BF#xC8*m`zlA8Fo|5COfoy
z>UfSb2edD>`ec<p;!lzlyyaE_(hX@y=*(LvZ2v?<Ps;_)?!G|(t|+b?OLA~nIW_Kr
zH9Fm4CXMqN^nF47Jf_-JAeiI}#F#QKsqfmrE;S$zOkp@4sqyjS6T`i|?vntVMyhPQ
z-%AU3{_vp<!ax>8OU4_oI|5Vp{*jm+0jWRG_ap-aq@2q2JY(Yq_3Tug0w6DxINH|P
z7((Wc{E~?*Jh~KH*oqdzgXyp#vepsro%4cdZ>xJ`85lf6<y;r`qaU&`y40-`XrsPz
z&whv`{gNC<E2+inyG6ck+!*zX|4y5MnKPiT;9v23dsZp|hPs-DV#Vqxq@CXd-1T-L
zX(WA)8$Lxvk1w?J!zS!#vUbP~Ef4T(kWFPnYKh@=R4T{hO4C40f7LG*|1UM^5ob#=
zJLy2ZPj%hGpVf5q%cblqe?V+{(>y3$Y5Ml(c)Y0pf}j(%0q{(%00N#oL^HV;x$W>$
zgT4Ke<r_Cqx2ouGMNt1H6BXJ$zEWV9TJ*enGW*v?y3}{Wa&9yop)tSmE&v|nU$%5z
z)^fM7-GhD!b0NDsmBm&a7!>Qvv0f;NROW@VuY?g6JTPo>!V3<6{c7luV-G2Gt!8wa
z=bD^}He8mHS-roKY30$fhBpMuZ0o!13~>Sa67sR{CWb;)Xl(vJ@F#Vme}UP*FgAr8
zlR|?dPL%C)hA-p5zZFu0h0gxq$y9rFljOuFbw@n9Y_5<X+=S%*b2N9AArKmeeZeOA
zop-lv{I<Hwu-0HSj>7$pp*!~~AyPqbPrj+6@>79J&EKdhZ}o<Kxri~Ex!E-Tq9qrk
zzV*SC#C2q?`%*yZvU6)?%12m!amR0nth-yoF1V|t6Q0;KcE+rDJ`V1zQ=qfwt8MBJ
z`Gl+lca)uC!7Ss%i!7-uJ(Jzb`;_H`!O7m!NxAPEOqj_tnR5*;mAp*`CEFIY2kd_@
z<;R0=WnqA{x&!fL2*Unl;auU>=O&li2RJoh6SXam`*c-aX|p`3Fsv)mlk<&jWByDX
zMdr>*DtqY#Zr!UCalN8ss#~zv=ebZY&WxoAWej~9squd!$!E-^7}OTYIvQ*Lv?}<s
z)%L`*aZ2$HfzUx~ujSQv<!9m|k!7y?Pa^ukc`Biy>6CdVBtYUqEUR&rYV}-~q5MB_
zKmu+r40<!53{K2r{781XmXxoNHSq~-@k3gF)BSfIrb*>z=7^tGlk3@n60`Ww!lFrW
znoBI9IDs8{B8Fny)RIP@c|U6&cmCCAENmFVEXm^fBF<$S7nXPI+53AS6DL!fwMA&V
zzJqDj@D<F!xiLw0=B{CiKe&k2Zh3Fnev3%)`>&^D+#~@*ag35ndsicQrrIh`I#fj*
z1EUR_mQAlJOmB15OH=$H%bw|ZK~VKF`=GOi%Dbc`*J;?gf%MT;8+65YTX?-n1{OCn
zauYiunj}0UjWE&e*y^x(e`O+lw_*yfupcIQ1^PXQaZ&J`SQXrFoEg6=xgSJj$}-2-
zpHI#s(x3v8djW&aqWXFMGwb?YA3*LA{}}aPX6nEsO6Ll;_9nuMt<EM8`Zz#*<R1ZT
z*mmq|H1q46xR4HbH}@7hc!1HiF%{V>xv@&~g9zjS_;%@=rSSgWEN?vU7YdL^-^)}1
zNb%40^gaa}AcX%wEXJhN(;r63zLc=+Ix|Rf9*b13YZL(0lq)ksVu;hewr<BD4*se#
zaqP|GG0G%II->A^DDQHzkdTM2Z*B64Mr+?oqVRu;Oo4KrUE)h<;;^Y)(y9k-P4U0o
z6OD6ppeaL6vDa@$O_S#E>m8B%Ti8X&?1_@zG1N1%H%@g59XFoWfQ=8*c_faKAx&W0
zmgwuRrpr>)Sv3TpN;fbKh=_z8P~o=X&;`D+ZXyno+`TnV50lK*UstmJ$n_u)U-9~U
zHY{knK81I!?pdUZjq|4prFW!Y`9bE&n*B;?d|Z6O+m|0ASSQb#vlpo>jy^48JeJ8>
zmv}z~6yOUfOF4ML`_gg^DweW{o4kfrmrqOhFc3^h;AKkQ_;t5q#O!D$;f-TeOU7J>
zFz}$EkE+n65`azYI6uwUP1O@H&iwN<wO^ASwQpRu5TK4scxa&VDKY8fD$(;85b!Oy
z7A}*BxE9k>LH$GfU*gcA*Zuz^iO#>xYaobkZ~<p>OxCjs0W8HkjA+z@sBrhKp3GLj
z{N_@PgYAn90)k?RjT!}&sj|f9DorF~4|-NnD<@2k*oZ=DA_cS!Y^4A<k_1K)ajqgr
zb@3#y&nAL?NQ8mRqxHnQP%E`i`J(`(?OgU{%+bT*TXc--dWV%eyb7~jHBy+q)P%nd
zY(r%gO-)y0_#!{Q=FqfP1`}<_do)!;ys*S^S}ulp>@BJq1*qmfk-ge`4tzIAtc;Pu
z=omr*c?(&0a+y<$UW{<OLsr?D;m$a30^zD$n2G+i6y$09x(Jv()S<(?HyqWPIN`Ka
z>z54e)?kaSVNVRcDi!(4z9Z|EBDY}o1Bp*yX?S6f#wE2$;91uKQ{m`43!Fa@-x;u|
zbmg&vk9{A+Qd9cj{C>Y*DuJ0bxZJekgRaTfN9px{pN;FdU$HZJ{uh<G2jsa}^5CmJ
zZj<U?eL7orL(3ub(^2%dN{`Qpy_r$KCZXA6Lqp6bGOz!3Q14m*o#%6=vKSW)Tp!v^
zi5QH1zrE~ohg#q*M2Y1yEIR(%R#4vhW86d)Axr9x<{JwmDKCi~Y1~a|*3);wYB9@h
z@j7^N;p=K*RS;C)l>vd_!8Q8Wk7VaL{+!0E9<jkRoJ`jrZ9*2JX7P<LGOWr)msRr*
z(GoP_K-x;EvdAj>n7)-OeIu8v<JM%h2B@UjCkni1AS_jvK~MvQmld$);`W~syH-J~
z?h7EyT3T6<FNVi&>5auc59so)7LZ&!AFmDsp^b74SKUs!JdaH;Q5}f&I^V@MvzH-?
zy?>fkY8qNm7kTra(x0?d{Pr?w02afmlv!*|1D%sj(h=nu;A{iG&%2H8s&Z;^I$s^i
z!pS_`|C|7rGjZh{g=9u@WCiK|OVdo2%@?m5{j4Ak<cP&Rv`3f2dx>Q?2=hbmL*gVP
z6g9CtJs4cD#dZj3)YsD$ZY<W$ML`lc^xlL!$_HSF0XDNyD9!9PHH?yf$x<0)*--WV
z*e5;DIPz{h(HLc^o)uycSNPsu+nH$JJZPOWN}09K+=VDoRg868k{te3HA~?onA?-f
zw6x*CGHSY2b}|lGp~JR7o3_Gma3)CAnlrMgDjV;?b2!69zr~6gWQdzN|ERp$D<A;y
zZ40AS{EkObgF8d=`N&Ioa@*YV7#oj9tGz+tK(U-?+2$ytzra{UlG%C(RGWaJf8Y}`
zqkkLo^e_h|@BJYCG3G!BS~w38<OKaWvjitE;tRbrKng5|nYqzhCNSC>5S`b&<^3cS
z^q++TuV!@!CDvd3+9``C)Q1tQxtDd=W0hrk@8hMnLyzR*CEaj2AQJ9}gIy+fr+dpb
zmt#d|w<Ln3?w1Pro3!~*??FikFgI+w(ZMcK?|81Ftardfaa7PKs_Ik4y$Yc&AYcXx
zh~j<r`E=2ECX+D$;Lp2E#b$VWPW}#pzd?Gn^X0amZ9esdQR~_)y1rPt0(Kn&tTGd$
zE+?X$=dm*~T9!g<gVsio(TKv*oBP3{v7FA*R*fH1{pAFsRli%eQ9v4CtI@eB7^)m2
z7jBmXx~T&luPlx;LU}d4*lQb*#ZFTAyw+G_-&U_mHPJH|;~HIWoyOIg8r0^#%)2`T
zDo^8B0+Lv)M=Av;EoBZw^v+fVA-3%TwryQBdP)3Lng($)(yq>!G1r4RVPF~dt?iJl
zN~&9C*E>*TA0wx=UositUZDO)-3y8mWDtVxn4KA^5;3$0O<y1$UoBx_l~uB(T$|Yf
zGYpVC%&z55xXitS*vx+(68`?7Uh?*=w^(|fo&tAgp{Q@wn1(C9_?Xk5&rd;t<3)8H
z0o3j>4l{w`k&<rQQM(w^P@a@iCAVu}4KP_s=1-Lc*6G*I!XNh#*1B#FjGtyrh5P#Z
zlhKfyZMuK62Zb>v0zX9Og>3EGHaZNc8plUDCiqfZps8OwE;%~wI`5rtVnRJ_eGH~8
zp^ROwAum1A7D0+RBb~jsg@Qe8s9Dx}Z-e5Xw1s`m#}5YnJw)crlde<y{AAk6fco!C
z&eRInthKEG(=NV7XV)0~GmFaMixTi*eBhYTA{2yI<4C{&z(MXB6VA#DqgT*e$Tvts
za&hcc(bK4;2|efJDH6a3((_*Q$c+yt6XXqhWW|-5BvIn??_d{my<W;A6QtRloR^zc
zRPM7-GV8v69hIA8^i8rR8}$ui%;<h(D81ito_buW)rsE86-k<F`3%J_#awqo$0Z=q
zpxf}H(h(WSvQ-NagBrKAqrd!q{+?Uz%%&K$^pHUn@|3?cNMfI7*4Y`o%5rLX)pC$h
ztRDgCP|n%UGy<IRFQAyh)F-<)(50JGlZZPwBUn!h6NCx;$4w=~l$RrAojNhT2i8OV
z=xg*Gaq`m6gCF8^1Y1imjdVN#4k>EhK<zlm;qF!$MRGLlF3wwDbET5pR>ZFl^HYbU
z>)m3>M>U(WgTWt1x=AyZQa*E0i<7(Sh<5pmS}ijyNN74tQsU(xr(7ZLNp25yM(zAB
zkLrs_nJ-bIfLh#v%wMdU-T?pQb5^Pl_lx@N{2bn2jhc<WB_vQ$y?rstQKVcnPm=}I
z8X61Zv6+L$tQq$BXMK$?{~yZUJRa)zdmp!qj4%de$ueZ^l}Kb8W^55cT9w_DB};~E
zVGu%=5kmH*BD*ZvDZ|+JeaSLH_TA9n`<!0O`}6qy_xpD|)WbdZea>~RbIx@E-Y>QS
zqbvFdtQ-t1UfbA<PKb$?K}aIEc15sT<q-sfY-9=a_)-^^@!Xt1J5$(V_W^rb^j9|C
zsd*EBk}})I_km9Z^P06DB2R&xcy@nb#E+0Ld0yG?Q_Z@ww90hCt58<obB^QA?mx&o
zc8|NhXS-jqljZV11Ful@-SC=*6Li4rNnJIO5fcx#Q6lb*7boe)S7^=L__u;J*-%7E
zjAXkviwy+1AGI7R0WTS^vp2Z#wAaMil7W4HF-Do%;LQk(T^9_;<3>P8@6wzQWmA7#
zKQ{jlAuZ0?H6<6`tS7nNy9;QoR=dRS!cJyk51__Sj9&vV4)WqxEa`cHNlk^LES<EZ
zIC9x{6$M)%8qDf&p)CE`w5W6$n{J*e3vaz;JYzL0Dc=4$4cmWOiiX;R90S27XJ_;#
z@3(&ATg&c|_7Z0;{9tVpZe_n@O~<ik{F`s3TfAnHUlSVQ$Cp26>?!YBFnXRZ^P@6j
z+_t%J$n*f7`MBx;bN%Xp{lsIRgUQp!yo+TP(#o!(EHLZuS<+a<(UL8E=Z5SLLKWNH
z(PoKtD>ipo+X)r5hOzd_EKm(;>(QXDVe+<1jGb~z`c;_|GiJ9lZ#V5vjkVoeQPu#)
zdiHhWF4U5*g>Ut*rdQq*dIH|v=&3~lc6<TcZCdmtDG)v*2kEF>!NMm-N5l>hZ0=ys
zfutYzQhZMi%Q@#Cxr_dKSH$TOwJ+Ko<|)7&dMd(u_!pWb3GmnQ9EZd&X}G3VS`iYt
zXRq^XRXqqqZgIN`?3werhg$?g8w7gUwdBh93%5K6bw`y#rWBdktUL!3v2$=jxRk<^
zFAM9MT_&!4{_bAkj$bEThJ23((h7T+_Dk!`HhEf4*-aO_KWa%64Bpgs*JybMa>_)J
z1iK<x2#$yNjNr;{h-xj9>K$vmHt)A_VPmc>-d}bBJo&^z{!|A<R{x>WFc-DPd0B!O
z56FWFYdH~JZ#eRRC(MKXMmkz7tUQ_C-zxrH%|kqL@8eKY+Ftt`O&P#JQTKE@lBbPr
zWl}@3(7JvhV2D~j`Qmk1!tqkH{=<08+!ftD+*jB^rtEzY#hQZ%&(+57dSW_SZey<Z
z_wcaaT8CFge#QVUOFIzwrWlPpDI+iMt{JKFSr>W$)-h=3*l;}5vaAC;e!TZQ-eju7
z{@ZU}PvUXz>3*%q!^`eyPkV2*aqp3-p_S9E=;azN#|viBG$EbVi=qG@8JI|uFs%OJ
z7FCye@+VKOCw#q|)J^3-NAEzYXaFf@;oBGE#M13A(|S_}hS_>UUl2F_7jIPf5b-Dd
z`6x@Qsn_ae_O9rV9`bqpJQkKa@1#h)sxg;Q+EYQ0%03LUwe~(@?Lhz1#{e6<Ndp{Z
zU1quEqrnY3nbOF8wa@?#;hQeXkl2Jl&RPlzNc!36m`}Jkx`9R9aoLm1$f4@}(Oyjm
z$%a1M^Gz<>Q72Xvsf~Zlhz`-`L6gR_no<=<(>3jr%zTfEC1r#pn}hp32ZE6bV5yLb
zh3w}C&fJgJ-75Xy#q6rl0B##J5q$g?Qt`$Y?Hy*li!DqzF=f7}Go}aCBV$9C4l^ce
z*bey<V981Dzs)=6sKbcjy)KIcTEwQm9`|49Ta$Go-hnYKBnq!_+w8IY4h*0j$j?W|
zAHG5sIWu|wyQ#L0^<B!v$!8`;Y+&*((@U>Yj{_-pQ#Eh%+4!1kgi;|UKp$gSNWxRA
zuDFqD&0IFWlh)s=*|Fvx)~AndC;)~GGU492TGNe_6JPCN_doN!-RQ@597O9)$>#WT
z7J84}XnOGKg~p4hbKRwa+&3SrDrEkNNdbisa^Jkl;b)2ynAxv9^4>bdnw<4eyceqm
zk^dmvNDTU{mF_t&0F!qx@;TAumnMSg*iw_Z&@sxaeLN;9HNBn^^LQ&$YQ@AS8MxEZ
zg%$5}gwUF1r+Cby5GsLKJMu;9-1k!gc|F$R`^OD1U{qjJvVNv)CH40`kz@SV`|zS<
zz5V2l!LN=-Ca%rXC-K`k+0&iP-h4?5-2z=fUIAn=n3R3qo<bT;Ue6Cxu4CR}5bP6M
zf$KX2im+3}N$?brc<b&QM94c*CDxG@0p~Bhr1<7d4<zVRAFR?^;V`kR$x~jZH(C(T
zqlawp2+YCGd7kW~nE_+58!I?DU_^iU<pk_zx-tA>U8|VxMV670F1(P+HJ|posdw;m
zM=brWNx0-)e2PZqapHBmzj%?&dT(yVY<|-{i)gnJhmYY#kB2qUrmb!3oW9#4zWrJo
z>&mN-+nBzAH%?cFS%fTQIz8XO#rIy8JrrcP?(Xo0@CM5?=f%AB?&fzD{K}+eku7RW
zP<x9ffG6Qx>&NwbxIOM^SSIeYVL>1=ljoG{M@XYPcx8N3P84i2Zxgfdn`R^d(C=Qf
zPWbYh<)s@|g}5%vEXLBcMDU?A9<OiM@`YH8+I3wntjsK3;jhoaC3x)cd!PQCye3jO
zF?Qw4NVj@8-9`F&A$JEVqgFM7a325-5T?vADQdu`d$B_(VmQgo`?ZL_#=TGRF8sYq
zlyA>e+0n})B$zC+TQ!jzu0wR$$LR6_@yi|=?|5W~M`|{V_agS~dbKncD6yCad-)@7
zB6HUoV7_~`%Xu*0fe-qwEySLe2hBr1OlugWo=BZEk1151ZbKFN^@iZ(eR-#%Wea_!
zPl~BCn@4?mUaRoEhVJC`aX0t98r8(vKD~!NJBfg=EN1KoQ^T={gb5}_7**UaI$nL4
z3!C!junaWMk=z;^(@5X)LNoBX4qqv^s~EDDXY~gffVd~}c2t1J#;c~+&h}S==g+aP
zy8We4NwLWcm4|FI7bhL{D-Kz6dC3Fjc0?!>)mfmq#>$aYeG*&Ng)LxN`^-vjWN$-I
zyJYR(-$aUC>eScbmeu?QOECuHAKy!<GJlS+4+VaId^Dlv9vpQSYnIF<<3mUJ`aG>Z
zH>JBd!#VV|LBf0vzyN_5jZkpCH5v*phhnUVhS4tA4RD4jIt<Z*eK@bZMi}LR`J8pd
ztB9@O(hII^m0ua^ZX3eOzn$h-&Xm{>_jxK0kw8W#kE}(u^&=l>QFYnOxW2WKXSU(<
zB(HC8h&==JeV(VdeU)ncLeYc`j-rKPD4u+KT>}NyZehblA6>F}=!vGCD&jMRE5a(4
zz}nol_Z(|l4lvot1jxH-umy53Ar)NM-6dvU4fDV`YTRq*sK2!~TVU?Zb>CqRG4$9D
zMu(i8w8-Bie0I0m32-igKiYd=z%A>)&>K8ir10}6_Va$_Mx=ySxa~P09#vHpPkZ~Q
zCshsm1hOtJn4SDIQ*fLg1*Qg<B`%*lpP;MCbNCj;6RQ}E3lM8wNe-eir4_5Fl53%2
zp|@<Wb{(LTCc2CiU>}jC3$xes`X`^huH3r*osAeCGqTYH>~Sc34da@(XR+x;ADja1
zALHDD?}cy@OwCz`O82t5j`pw9u^c+-+L2P_H=C`jK#3=~rs?tg^|=;DUENt1@b`GE
zfZYUV!LOLLEkKwqJI#yGA?-0A5Q%x>y1G$xnX+~;%`4wd{MguatpVe?4xu)5E)aZA
zb3oo@+jFI<(b#T+&=b5*)NNU=E}_09uSgV?6TRgCJ%UebC%8Q(EYIAj&g@pw`u6v^
zhsG4^d!qc7Itzh(1A<?rLqSm?hsohjEZK(mS&ni<*fJg!i}13z%lXeZ0ef`{tDHh5
zbq}&)@cYs*EZ&h98kSks#e9c3tpNOiB1Ad7;F2y2a}9XQ6ILV(I85DT&6h`>9eK0$
zR`>0TZFk=T_aX9uKU3ItKDQ{ep81`EkmY5%`HKDChG&o0GTNn}$=%PRThldlZe25~
zkZ9U_t}tcJ+Sp}8c#Xe8L-le^amx;mmfVpO<CzR3;uS5+S;4<PM(}|wrc*5V^>0QF
zi$s6zRdK@rbdZg^iOiO@=0|IU3*Kbe`;1yOp)Q?^pWc~qXZ!e-rytXX=2@)5;dXq(
zq~|=##wE0McIC=DgyMRfU6)6<=jr!hyC<gXQT@gDH9|`y@sG9@7OImE#D5g=_25o_
zn^si~FL?27D1mFtRTR%mqqSIUT7p*lLpx5No|}l<ZLY3Z_K?t`ipaSo;Xb|C_-ye!
zNzAzLQhU~W$A-Q50md))1t{KHO$Th4Z7pR20Nb=br<l3}R(nW#w%3QXO<9c7Z7AsW
z85qcgI~pY<IT_JjzxdOF$;X)HbH1@(nFeY71}wK87{k}Go4#|$88w%y<en$<ytZ__
zBItU1+q>fj1^I55N7M+>x<3_G+}*oLyfTH}JDl-7-9QKunS(n6O+8w+M|sOr>P!kt
ze8V=~e|eBHxsBSt#Y+wE^S|=QxW$v9%Og%f)!;|1B&3?QA8lcYl-Wh~1Ydl*o8fSB
zr+~)GJ|dv5ere9X2o;KI)=<$;l3U0Y6!zKV3eSjSQPGFsYKJd%6~W}8_9%uIWrWJ!
zy(?$dCM4zLpeT96J#>_xK=C+pd$r%3-4voRJra3baiz<w`qSQuew#?qVYy2(E(03_
zbtU%U{XLwO-Y#UnV2B+jWTleo!Qpm*truqu_xO8#<9$(|;PX^nhg#{EZtn7ntrr-5
zEhK!)RGvWTD4anRR^BX221_fqv+W@6Cc@Olu7A8l62)Uf&}hJvEr~-c=gSSByGQ)J
zPH*Y78ey2<0GHe(MUMm*UFL;fO)5^GB@s)KYE+pq=~nTaL9l{40WQn8$Y?b&%eUII
z2G1hvCe{(--R0cu{tG{9HVX7!&I;rPCvP`$s&Nh#N=X*e{|1^)IO5c@7_U>RS6xrB
z=+z07=%_>Z$UWnGvvr+#`r_VMlS2m-KN6Jq$>}TFJKwZ4?Lc@sdmJo1y7Ty_xVt+b
z!uf>Pv}p2I8t8(-qj8{-g+`-CIa#F;)uA<mkMgnGIhsnjKNv%Uq(YuW0e^d=#jF>3
zGoC%m!$gA$^6~|A#$)OWy(-BptmF#<1M=>9)Csz9TW%&N_&#27@##+k-MjQ0rTi{E
zCPsOA^&XC{1N!1=T`!g}Y%Bcz)s>3zep7Cf=IqeW)E58wwo@*w(X2v?9n(=#*v{75
zJEcn6AyY)}GQJG~F;Q8ebrx0FX<j&u^As7POb71*JO7LE-(N`+G&~wY;5D|aIxWoT
zK#q1jc4#3#aZ|2EbZVhhJZLMC(UM{088l)p%O|ASbvw^=FPEbN^IECO6L)FEig~9(
zQ=@rGzp^`ZMU_<c!AJVFyaW`+rzV$Ofp<~4x<9!+elmbO9<a^g+E?6?Z``YK?$;72
zM_BCxlD-AcL5-0kra&|~=9h5dgZ{+A*X3%KNA8l6=YK0st3V6y9-e%&TMrS4T!MaE
z3SjtFWsm3<=}t+}^a$5KXK4nQqVtfJ0@A<P1*Yg3rpGyHz`{m4S-<mqzs^0MN%ZX%
zv>$r?lc`3cZsB@S$)osM$3vJ&joVt8Wba{{KIY4uSLEC&+gb>{i*;%^&6m_7n&Tfp
z*VLUi>`i1zBnxUMJa68GDU*Lgq*k_=Re$HPb;V?qa1VT=_Grs=bUUZxW^Gzm!}N(@
z?{3)Wc4<7A)jG29^TzOk6hECbb6vg&RUNZo`2pq7-P0wXA!dBqwXUcsckz?Wl{~JZ
zF?5PlmpPquBnA+n5UD4g7Cy*Ntfq^(RI<~_)L_heA9GTZ>$RS0uiC~UcSAnNQ`PTJ
ztY96Dkk(4KrTEgBWC6%<&=+07r~I60x{2^3_nwsOPNAq7(5U#(V|^|qnX|5ty-kVD
zk*~i{-Io2KpY@DL%oBs5MTWA$4lNd#L&lcI*VP|jJ|$8Oh%&%pjzJq|INYVr1)(|V
zlbs9FHM8umPrlH?&_ag~m3N{I<8lqux(N=YU=F4>+bi1|2u8dJ5eXuDp|rFlFInp8
z39B00uVzI8YGvm=e6$~0CogeJVE~-;qY~hbcrL$O?glJnff?>a4Q_klM;X{G=z`B#
zB;d{hM2L5wM94XT&zou~?jvqqfvllK^d_JmxFJ?qI-y+)AN#gx(IoS8Qg;=*yl=3L
z>Hxd|O(di}tkJ94ym*`p$-&4^Gi_06!!u>%CK4cQeQJLv;{B!BoluaKTDG>}%_Uyg
zrN)(WU#P<bSI4*BJgilXKk9oxl7sMyxgB>+lXSDOwTh7-OU6p{!B+7#Kq)33NsLfU
z_4FuTJaM9QZ_RN}8Dp+eJSa4O6aGq>dZ8suK4&L!UK7K%Z8$Sia%RdjsgIqCQJu|n
z#~`GcX-RDRI{U8Y@y&EkJp6~j^xG~%;>EW?Tbl~zBiWA5WJPeXb5%45H=@s8o>_sG
zZM0W4vrVzcBG9j2;fbnjO1u#U-<TgS&V}i>tej_o!hJr7nSe+QAz=@iy4hMzfQ9}a
ziKOo<EO$QCh!q>+F2>Y2zYRNu+h;1Lq78VJ!rs?f@6?Am#VcVMXI<M*x#|5_F4zT-
zq>8||fQ_Wv57z6<$`>T{iy-prB||=LuP;xFCplJS9Q6gk^45uaBnKV{I$SdWmx7yH
z8r6+hp=Vd|G1e<9`Qa@cc4XD&dR@>QL4Cf^I+chM8SYvrWE(f!K3yrlH|dYVXfHH5
zim#VJe_N&;(6hyOCR?8SktRsBX{gpJQYXmbS3PzNWc$_dnu&F8tOrhD8Lx1?$XeEZ
z^tRG0oqhsyB08of{;Fo7OH$fa)b+_rv|NQhTaM-8ReE${R-0WFYQm(8QZDibt?;{@
ze>3u;DK)#2n!YNj2Ik4j_e3jUV{cT5H>E|-ul3zS)3K@_*<GlYlT)v&obFOi2Y8!U
z4x9rA8Dq7cm85OlWH<9J>ADb4JhjIA)AAbnJ`aK49b?yN<svm`vOpH&HjL2Tt7qpU
zgZt86a{Vf_ZCYXJKUZ<qleoI)8@O^8aq1~%B1?%x5Ry*u0O4%3Zm@F5+vv^!zUhtU
zQXdVCJy5D|QoTk<v@)Kv#^0%8wlS8wvRFw<lch1aJ6+J_rNUy92)2SVC-6|ZWyssC
zm7K?J9PPc%%eC_*?tN*JlC76;cvOCHGBk@<tRFv!`u#AeT@T(sI_StVDmwkupZcO(
z96<Sv<e|O`>793_=%jZ*yMZ6*Ko=@7b@kld(9#y{?V}qJ^j2~dWseTBnRtRHgYFdU
zRh0DU)l)i<wxk|>cDo55y)??ABZ=guIfBk{#9#o|U;;<i=K?|t#9ulv!VTBoqTqwW
z<10Uow*~WT0Ys}Su5u|yAhP*o0rxypXz+Pj8*UK?dy(@x_9@aQgOoQ@1kWAGC5i-N
z){+h#?p~Z9N{yBQl##AUy!2~<%m$7H|6sitDk0e!%oFUOJ>KHNT~>ZwxMJ6CqiAEk
zq(?4xxAgl<^}|`BBAXz)Dxh(cr80FRlQjETHWli~_qsUeJJyvH(rHQKKCs;y@$oM)
z?ytw#9sW)|y2-w=bL;6pgIzEbu%K{Yi$<mpy{f>vQ&|jSH?lmn{OqC$FLGf66#{y2
zFEbjZOJ;l6KLPu91(p_$g<)3qbv-GAwXm%wuN{2nf}Do~vGpoj@c4GIQM5Z-X&6h5
z_n()D<P!2fQ|2+>R0Hf{9qqMPLO<w$%~pG7EcxhDmnu;ycwTi2S}wQb-WJ6A3+s|3
z_|aj8|L8m?&0#P1qkSx(q#EW!n<yt$5n@(8tunO!8)gj4+lLodTTcuFQ@#_qM!qL8
z&j=AKS%<YlRl?@7tgX_u_#KaHY<2wUo16qx97ivpq_mV+HsRnzZe-E5c-I&A7HmHH
zrsoG8LHr);$z6|yQX5jJ0qvgN>m$YkV{`$UW6G*)!mDzWlELyCv!@AD;(X0pr#wMZ
z&cxa~qgA?Jva2hiQ&0(M8|d*(47FtJ!9}<4sxQ0V$|+uG?a^DE07wO_@+M`lm1#&W
z^wQ{+%LrcU_PpFsS(e~Rwx~(RQ@IXI4}G8u`<y5C^(|1Lu76Gk#9}@6`|oP3Yrk8<
z4#=Qs3g)FWPH#!~L=?m#JWrCb5D_jUFuM)z_T`re&;?hizFs(eqq*rZ6~Xg;k1y>$
z>9o8~Wc!dN^un)--v(2UPBt5;JY(@X*WLBzt~U!{i6nc0ncBmy=Y?huy&r7XGrQlt
zh7se>WC`22Y(PAn_W?(P4>LEBAT(sw=Y|+SzcCG}KXlf7)PV8PWwCJ2`~-}*SgoQf
z4oO`5?vy`P4(q9h3Fl6q2lH&m;S);O+59UuqL&hn934&oF2A&R@oK3ERjBgCDDW*d
zDsQ*$AmxGD!U~69`AH6YkGj?Q=%l42A9W`J9o@VcClmR~4h9HC?U_Rw%(tImTrV*;
zrQv1^n`$#>_)2)zLUrKd&D_L(3H?_H$j*v>sGivzC#%W)wc8`6r<u7v<xO~Wh5&Ml
zWbqb3xc!D2!@VFr#OZJxqJkL@AtXOzB*ybKy<v6w@=9x>+S5kUYgsQGtqvatBXG{!
z6n-gN5&XYCz8K>3WZ4BK(6aRum#J>?l%!qT|DE}ry>91YH#cATZOsr)eAX48;Y;#m
z%_CX&;I8=bWGvuASF}c}K^1O9raA*n1lwV$8UfZCoJv$D9!d;b^9DWeLWNm*d<DcY
z8K~$3#Sx5BSPqGm`^_kPr2Im{g|lqii8YvFo9`)Gq$unr@lDlg=RK~)v{3cN(^3Og
z8;StUN5!RVft(ndgD^#PRW{I}e+mz1$qOhOM?F8lP}MW9=%0D98CM0Q@?1--Q7Uoj
zup7ny)Sag@ycn=Qp5svB5qb-4a4SHQU8PsqP|;c>stP>M$wuP$R=w@N`z7l8>9r5^
z+O5w^*QDh-4f$t^jldgHP{A`@&4y#lpgUmohc{X30!=jdl$;o;EncVSv(<DidfHk_
z=I#uJ!*kudC(qa5aVf7GBStucuop-_4(cDC`bPfs70ox`<U+MkS{t|M?89I|MaB9d
zzp5QjTAn#7@9^duP;p!l-s*emEsK~9WL5Z7yAg|jI3nocm}>s@a){T|6G`()2Ya0x
zX`sZ9&cb+Etgz(kTYqj?;T|v)S-%tWmepUKkNGAjju*IL5bg9W*@_VDB=LP~A6hE1
zNLC)8<FU7QaM|(2`@Bu1(4^Z?#~kyAKdRcpOm{PnzCs-<XEah`n*%5o-*G)_XSA32
zE2}z#I|=C9A@dK6?gSfANKh0~xI*N?<0@8fVSW&<Dw;B8hk}mljqFB&m0i@x9WE}T
z7vs@P2%1;j74jwJg^cojG03BF{ZP4hz#D`1Bo<tPVi?~?5e#+81(vuUYK}b*qG);}
znS9G-1~>9r;n?q$9$!MSgErIeOEjSh$FJ0RRC;FV=~8?mLn{u4JS}bT34j4TbRs6O
zdutSXg=L4eDS6jGHVso2y5=FOq+MQkS3%bGvd&Z8L*`3>Twi~1aa4ulRCS~#0#LrY
z@qoqu0jNbQo%5g&(8K}w?Wg{HM^EeERDgJyudYq+D;h(#JvqaKN2OM}!SjulwC-Hl
zVh$h@l8e?K=w0*EMTdpSC|ZSUhM~VI(mP-*<~wrX`o(ws+!xi@49(UJ7tv5f$UM1Q
z@1yFmrn>*^2V+B7&+uG|3D7S_%C2^P1(d#qDLtg$V7F{F<~tNe8c%vmf#PJK5Sz3=
zrUzp2@8fUG>YRN4)8V<7(XnV&-;K1%o9w0@7P`?EXW~uV9s!D3XWrpFZqF~QFP0nr
z086`f#w9(C;b<ZaJ<>=aj>dy7Ft=R;?g0z=IwWvWX{EpC(jyelSt&V!Hzn%{t$l1U
zNoQNVni!p=@37HR0ed+R>6H$&wLD*pIv}ZWTU!ti_Th6DwQ=bMM$@9HRj{M4-OYKA
zY`DT$-X{4mxoRKHi^o2*R+L1zIt-%h7*-af8axM4Jr)ew_Ve1KR)uF3AP(VLzTrVw
z&tjN1STSGD+*01=j8zhGknp6h`5QgtrNeu{mu_xvKRz+mad1HC^Nz=sF)xogOVjYU
zv6vMP#mg;m>a5D9RV<W>Ew~nbN<pQU^(bsHTeh(C@5nYB$JWFH;%6xn`yT8JEmx|j
zyKJCtZ$*7QE-<EYKHUWakh}C~>Gys8$L@E^@tC}2%I=XuDP~4?I-uQPz*&z0D6!!h
z0H1hDXQ}2`T=wC;67{8wdGplX7<8c*C32?KFb%+|pOoGNO|DDt%rf~_V#LnwTc>(j
z`D`Y{vDePfq`=k>EoS9=JoO3)eHW0xEc;dcM<BpAF&5st>w8Tc&?$JfHeOyXuu;|X
za=7$azhOA&+gr^mwwluefb8@#u>Nc--SHAg3E`BKC#QTxmr9isbZdUzZozwQRdr?+
zoq3D%fDRrV0KMANge$U4p;ysAmc@RB?TzYD>8gvb{7EN*RH2QlU<R^IM@jgj@fRh?
zl>z32=PbI>)}IIs36Fsl3@gr&BY>@@T_J%_UpVpFG|^P%aXYhUm!mX!{2TnRC)>bh
zQp!u&>lus=J2?XJhqFQn(}v@^b@HpN{=6w434MhxTw73L+N(bm&2=!|L$ZgkU5iB(
zhUJCpG7joBpN9v{9K=x5$txh<cUNuCG^rSCD~F2E^0*ny%`aj1U%kPm%N$1p@%p;l
z*4U;;GaskG9q<%0sbxz&a&yMorEI9)blA5Zbdjj*_BDvUu+Nb}^t9wm7|;HKyJE{=
z+Wn;_7uv*Es%hV;CaPUm6ilnAb!p$^3sM!3zi73Y6t}U@3j3Fsavvax>C0bs>Wz-q
zt$2^+eCNJnSZp+~^4P)IKKZ<dMTH*rR1?2T1;DEZzNZJiMWY(Xr=EN>r(<$-(sfZ!
zN+Jq=93mI5hmggshuYs=&I6(|?GU;D#kubNt?svD;m}>+#Q*_>bc^(3<Jk1LvX;Yw
z&=PT7g`xi#D8=$r8;dUf+R6+QZ|C`b(fpA<s3n9#z5AecQOWf+hBrl~h5yQ%-qqHo
zLdkYuPvcqu$$HxUsih*9YSu4(fP;53ZZ}B8aV+F^Ut7M&wp(fwk9B&J^F-<2=e!g_
zk(L6idzHU6&+M*#+S&Ul?D$byC`E(h7u0WWaD&%^0^XZ%$~RIu-GVl8rTv>pGl|$q
z38VNyA>?pKH#`e!WQvkd4!!b{RGb}n89K2$kp29#9;<y@MjBuRQ<904WF{mA^vlKK
znwe@o<p>~X_m2p8Iqt9>vE0WYq3PL?X)=;XG_#9t`6uuE0~48mJ)U$qXn;T5M%z{I
zlf2gW`0Zy8b6vT$$@c={H~KCJp;7@7Y!$(&TN<xY#zw0(@_hA)4Ux=;n|gOWzqNfc
zIzw8fe(*eMVE9Z5K~q1;c;TyKQZm%V?J_<iNZI9^uI$vk+#Lz>8^7xQRuqu0{^yOq
zQI<M-yT{%abX;4!%Jj8N4|Jpgz=WWSy|rYIr_`gt><Y;HDO-<Jr}jBbKnQfw->TSU
zkF0nmjO+Y~FZ6yfV9t^K@ZwcbD)JYd2dG{axLJ>gxcO>-es-(zjQ`;B0am40E1XuP
zz&8z`Zm?s{J0XIci-p}RKKYvPLR7S}z{N+xe<M3Vs#1YJRh1R~(iIkev;Y(=y{=sP
zaXzr2Q2}X3;@+gnpNIsp9x-bZLD9N{7GbF@zLuywn1VVSkkx>$ann{f^OR*vW4UQp
zBXmr-w8y(k?`7{XYf4(%6Z9jkZoQa~LHei2N2uAaNctKu3BS6Ya1hH*c5tMY7IzKw
zrw>oJR5O{lQ{Z5OXMeTT_AOLO_$eq)8m`I!@fI!5a>TTt>h9UWkNRTFVB_<1d<~ZH
zg^WnKhs#SLL3&0L^VyN3`tXHp2TTYYjOOdjzv3Q+Q2>Y+K^JzmgW&?B5p*k$jl%6G
zLd-Td8ezpEDIRm*Pd@Zy?FoLo#%w4ECi+&~1|<RxJQWJRKEyok^Ap`HO*_snmv|RO
z5A@F%d=aBh7XYiW&kzK0<_$bO@%!m_BR~Os8IZknE4@4cgtV&~AR_Nk0$%Y`cjLOg
z86bn!lHlO1n&!RTHl$;A)c>*(8U|ph14cz6&cl4^$oUqPu9?#n9fBi{qtRx+yc~er
zVrDY(zy4q$7Jq(d0=@s4B+-g16GxeUg635?6(d#G>$z_b>HHl5#cQFy>f^DSAUWbz
zB#V;JB!wJ=dr2-wV4TS0pjb(>9O7@zb%Senlckuz*H9(;Q#j{xu5HOCKfB~P%MRD8
z4L-4>cBmvG_q=;#EHGJlw-^3!Ipi=WX28t|W4|CqC2t+frbesN76>v^64whn0q638
zp{!Z!p&4mSbohkvJC>*uD^mY`dYO8P^81>4sM{|son9zLQu0{-FuY)9w5`5HbF;xE
z@SGY%{ddCj)(pzn{y>x#kQ+#Y_fp^|h3KSR@PA!lrG(z3(yfO8S>L=6EGF%5RY<s1
z(;T4;1VAqgc@1niRNvdO{AowrG9Gz?))lNdFm1qu%RJQ3`5P)2@{;GeJd+nKtj7df
zpiBVWV4oDvV-v;oyw>1;+`s8Y`C=~(CGU`TuL|vb?`dEKeq{yEFQc2J&N;5jgK(&}
z%C{$%I&c72?95q~)N?0Q7-{p)YRW?n8D2Cor8A09WTaI)5GkXcpYt6Z2*%6_;J!2Q
z9QU!EP&zPq(#0-qmYU|K0Bgy_7F=reF)@0Cv~VdmlE;TToYBZzE<6Z=bd8n7*t)d@
zF=b+1T?SJENc_J<adHf;bn>=y@Y_epfqr-TnW_RYrW7o?6d=h$2@tQvJblDTLK<<D
zkXL`tuK1T!gG?;<`nsy|c2=p_&^VS9a>xS%)XTjC`!VCVyViQYZCP3QgFP>u$LsRi
z%MYKU>hjb|XU?p8_Xkhez5|Ce0v3Ib(3JI_I}4o17q(J#AL)MsQA#kmfSVwAu`zz-
ze%^y2vq4{AL_mM;oO5Rw2zUcx(X4Nq{sCt<LDhUwphWS;4=*oJIp8>4<K%GFl)VL}
zb=AOpM@p`dW7S7YZIaLbB1~xGEzpB}LxU}^cHlW)PJP^`@hVhdB~H?k4D`uqqM96b
ztk31yb(9ovZ&FmmQJebVn2F5XU<F9!+0iCC@)uyXCi+t#_L&2q$U{g7M#$6RK1MQb
zp^RUQMgtYwyr4N=#oL~ImGyp?NFt9Ho^K<IrNkx6-kn;Q{aPOV%t}3u%UDb3dn7UR
z4F?MF)c2&LDv{&h2+mB6*5X?R-RBY73iq!ANNtC2e30XojEICZ79=VMi`h5m-v|{O
zs$Mhfiy1)!pf$GTuc64I!x9n902UJ%jWJD*jEPPl%8BDky!v<yDy)&Ru-SW2OKQMo
z5UrtYBKRDL*!S$~lw010al<0-4@Z%^K(YCL@rtIBJb06^PtMu^>t0HBBp0@xk?+3|
z3_Jn0sY`8?LjT6;0pO{0Ny~ldqwxo#MrV?b!`kt|bc68H^U2DRb}ox*zvN|d_!tw}
zeEP2z0QrrW!>Pz_jm_7`*S=1hP5~OQQebJ_JK+u;;3xk(|6aacd(k-eI5kshp<sFk
zzyynbsjux)ZzRI!%Fpw?*)UgiBx;s=+E2#ysLP+1$Kg0fproKk=rF>+p~{lmviH;U
z*0MhARVt>1(({ZL6JCo8iM_KndK(Yx>)DTW^J@TI2;vC#momU<-M+|7;i?yi5kWC1
z`-{CA%~ldRmi*)fy?qX^%w;rQuN;1Z%XU2a?lpR!z8|FMM<M<YxOcu0+x=h)0mLqP
zw53I{yQ3;cgvp1v_BgWqDQ%()8EL7*qCts>M(qLhbiU$^`rpH=dt@<ct784XM1ku+
zC>2jg1-=L8Vn7%5A=y9b_=eA&mK1`qH#vlCrNnR>0Js*wrpgzg%s^OD7g(8s+C*Cx
z-?jeyyflK>`Gk^|`!-sj2QUM<tXq2P?*K%86=$ecd3^z6>EREY2&u84PnZ52ugx?W
z=ja1JUnkpajWq*P?;e4y$<!W<`8#SXZ)yGXAQVF1^{(vJurj=qXC4e~X5QOv;<>HQ
z6S}|4JVH9U`^+pi1+&UgBssa%BqKyCgA+dEAlQ=CgMOP<(q5#DH7{y#`6wv!%49w~
zAqlYqfA9i#O=hiLNH6{Q;@aOXV>gsq+^K*86Z?dm{UdO=#KmLKa7=Hfgbk6iWAv!E
zLOeY-M2H~p`nY9Na26YhA=nz-E9do9uUSH2>~rkwY%#x445orL?+BY;sJlD?5UY~O
z1fYAE`A1v8Kh0dl8;}gR&Sq3$|Jk=TQQaK4*P+%|@QRnfmA@GXCII*3Sc14D;`lZa
z;B3Qoar4E0N<8whtpVpQY<AY=MtTHCR(G^v2`{RYqxB*Pa57`n)&BRphyr)2o2U8g
ze|C?uez*ka7K}(Sz7dYL%s0{qmN8lFAr~XN8hlWjZn3zLIKX9ihSEd%mPJ<@bvO0P
zWxx}Qf7#*gcb>)vB@W!X<9ENN8NNB3m0r@k6?&a08u^w*i58=)vf+`{prAu)z4TSp
zXh%JjMq%gih2RQH*qY&2-}Gz39HNbG*$aWUrEKS1uJS`#`Cb|1mgi)vq&0HIoQU?B
z)>V}4;k!vo^8BkymM3!VEjPHh=)~gCydP`WeurD=jSpVhl5+e!?BHIOVP>Yix+&1W
z@RtkDb4hbL*O}YYEnQy1{ZB6$SnHt9m(~SU2sBy>!}}%yuamv3<m%(f@&rm?ZOak?
zm#@&$@RJyx<Y}K0w`cBvNM8zVyh^^n9J~Fp{~~S<BpL1_TXclfCZ5wSi>26=MqrxX
zoF7u5<++z+KVJ43BNnv(e<HN<{WZWl?tFvSj(q!L{+d?VkHF}h1LxWVxi$kHL@$@W
za=NEMNiCUMnMHoGJ8=TffO;2{ZJN$U;h_`bw=*?o7;Z26$trUam^DvceR!X_?8LG3
zyi_*+Br@20_xc3x^~6l2b~)SXyn;EwP&9O)Oj8pQR8g_C+Jvnp^mU^x;AIK+qslE7
zQK^C=KO0f7eX5{T8P>b{Q=u&S(*Tz*DWd>{Ai${!@P9|Qu-@d=fBe}^`BP-7Tx7>6
zi&1Y<!#%qU-CIYUAG=Ar%mJgj?_o&D4Xz!#U{g40%zvR+HC-M_C#@>`^%?*_nl2Tv
z_zP2!3#|96Q8vJ!oD3QcOoDy#)F@2>Ghiv;J)2oF^PKC>RkLU6Sw9F;HSq&^Z+25>
z;~$i!Zazsq_(`xt3en*MhgJ5EeoonmrH2KN?;{^1DqugD%O+N6-tA3`{yVBffyLNC
zmG2{_YYt5yfM&hXGM8D>&T|$^d!ufv>D3;$|8DYMaU<`@t|klEM$VZiYCdZ}gqHaJ
z-S4!GaSGm)4x6RZ9J~C^biD$xl4)C^1rlycTSGq(-k+Q9MuFA1yk5kj?Qu7XL$^AE
zc{8x1+kAsrJ0T^VA7L^rg0Bf9!TXJ>(H21}K>EvvI}s*6g=f~W?t`EYQ}%%#X-~Ce
zahAAT93Zl?1RWow;iO`aplHd6#4xRPJ)cS*l@mS+@lPKamQe_*mfN;v-yvusgLl5U
z0YjsSSf{^4cP{~#n6+41pL~g_4!_aDjALwWf3sTjyFIWk@Q^AVI1C+D%6ju1Oe0}R
z4%#Y3HfBD?6iZvDlqTwffB=$aV_o>~8K?UKp9~<aI>xfr4yat?<_7HZtlvhNppQ?`
zJxT}n$in-WQ)s%_zeFM<6-k(Ewta7M@b#u=Yh>?K>qp&8a=J7AH{Drj0^tJi^sIK?
z1dFtC=Wp#K2#*}~eWQ$`<&zJRF{Z|`@EkkQ^cq|obhvu~i=Y!6y|bv>#QpK{Z`+~>
zRG4JRjSznx1TeMVJ-Hd0Syz%oV!{VA$t;w08xC3o)G6RYdAY{JFKd_b{I)8<({(?a
zY;NSx3t??cPo#FcGm4h$Tas8h;>t^K2fSm#Vs0rNv9eMzoN!ZF=@F8q4VzzFSUPgK
zrX-KMh)^KQ(J0urVQ*(oCr1WF2N=#TDvNGqgdb)#Y;6VGl)7C~{QisVA>ngy^L<Y(
z7?c=aJ1>-COg1>k1I{6TAa&zSnEGLB*r5huhw%+X(IZTywiY>iQGfv7!b1g)>wrLV
ziLUSe3Athj;M7Zr-EEe<2C{d;U+OIO+UI3Y0`?0lO*fiG<M;mkdd%hd8zTxzm3|a!
zg_Jmq;qsE|ZR6K~pDslz^Ys=}caeoOEb7u+F^9~tOVDL!&Wd-{$&^P>dFCUOouWKb
zxy2#1mgc%YH}#xAS(ko(0&JbKWcO=P&NG-C6VWLFc+)Du>F)XEmNngn%&bN&iwWEf
zC^zrs4{X1G8jo%em)viaIG*lbJwE>jVf0fPP*nu5d8j<2ruE%z5gk%EBZN+rq`3U$
zBRB0n?kcU24>1X5wzA6%+XAJB1@DAPP8PJF4KB81rE$+^W1Bn^%*+<WC|f5>(&V~B
zd2AxZK#2`WWtwtZPY=k73Ta)23&rW*c`gI|=Ikqk^Y<H>06<>S;g2~8rEvGzYUq5>
zZVYYo)+ZnFPk)LUJ@mtlg|4JOCYO6-?H$yi$v-QBL118toS-U4_^%NEC%hO%{s4lB
zYaDaDOv>>tdJ*XI*T>gk{uiuc!YSt25F3UAuN0fQrRu;E4nE6bNggi-WY@l=0FP^m
z8Mp&dVMDGzE9<zODCQxz<5`vIuzoZ$%TBzavru3L^yp5(Ax$4IGO!X{4@T@-gc-`R
zy@Ze~Yx`FLV<dH6pR#7t)Bu8bt^(U(Netc`vXiAba*i-d?-?9T8f?QRq*d?7E-AID
zDJ@8JeG61>Ivaoucrk-}pdV@I{#~!7%|cV@RlB;9N=q*<{76x2VNhh;Wv*9%XwG%q
z$*e=NU9p}ZeawqNF3jUB18j7}?QM)?Tswmii#HJC+#$y}KV_FuLB#jaP%$=^F68MB
z^5H`>pX+LmuL6d<bkg*0XRo3EhY!Rq0@hjDR9zPpF3HFKdolUQMGA$+gT+8)*;ipN
zJ2>)nNA!Kmy2x<h55_Bwg(gHS^$!<ye+ku;^%)Q+7xI7doP3s{=|hMc`PyE`Xsdsr
zTRC1@&1Ql#d4NXwh>eew2MN|<-o+n3m`Tz)96?cIi<O^Vx{to6g)h&4)eF7(g5QlB
z<}>O9P3xDu!0z>;>%@lLr9Hm4zH+!eHsmmA=l${;p(tjh>FMjw|LDeRnkQEZv7+yE
zfsQiRKY(w2CB3={S;1Z9E#c<cDn=`Q9GCw*$V}Rtz1mQ9iP;ht;|SRuPgk>qZShUX
z@g60W*s=ywdx52c_2&eV+biuJ*|0ET9p<)l>92^h0F!qaJ?aZ&B>K<kH2VRKSzryC
z!CC<S&vB4p)}M}^R@W-{g#a$+YxrA$nVmi|UMuVQA2TlvTF`c~Exl_!mhAQKp#j_k
zz}P-0F*S?4^3&WTjha_kaj=ck52cJ$g-ufEg#5LeA*Jzo_!Xn2EyEl$S6%f|p6U84
zZmi{xwjjpiD926L{H~Y3mWc^EX6>JFXlRV5)|vj{szjKQ0**9XUjBli9Npl1tG~#s
z%$;wU&qL0ZRzi7~t~ikSs5tA2A8r>E`)mp;Apzk<KT#wVe_S&TH@e5OS(ep9Li`nq
zAU%v#+omsR4erIx=wMbm0Z>g!1vD^R6qvM+ytm%NjONy=>w17$LKu2=)S>~X`jehe
z>2mtkv36zg?uq95bdaYe(4r&8F1p7z=_cyh$&K`CV4d0PeWy`cm^x5i$hJ!TS3UrV
z-twO-iO`ZS`{&S~l4*^&y<aeo?WvpM2Cx<3X9Hs+dEA3V4?9=?p)+oR7O4GI=&m?m
z4A@uDyi>b<!esb$RTe}A=xo^x$5T5}PJ#g2S5L+=`578>Y6)eC|0Io%4JsRpVbKfs
zPn8>G4WmZ9ngly1O>-Z$TcoEUe(RL8&9Ru(5i(*B3(7FGXj??xp$?$ZpC&1~H`I+;
z-Lh5;*>PrA$WB8ngs9hC>Z0(~Rq(UJH*=s>Z*gS2pep+;uusTCvbx<F9}VVBr(rSL
z;T(`bj2g5g9I?4(!?=!reT`(ayrowmf~&5G3ui`m-Yxj$^1TJ44LL3ri94>i)TNce
z4n@6Hb~yp4(Ll_PpX6Bjpvcr@@wiz6V_@>4@5YY8YgT>5lDXT6z0dmh8;;kWJLui|
zb<u#uMtC9ir}94uo?Oy~lRFlmM9Ejl8UKCCPp*%lGgn+j96!Y6VE4ceWqr!p{`cIu
znU^&DLXQlw9IgYJNx<w{XyYU~QkK3M0p{ZWqP08S<~Af~iq6ugJJZX6lW8CzGx@V|
zqu(S#>PIIeMwC+|-)I5SkY=6wDV3v%HTSa1LxDgjLt9T=vOdYo{f)Y1&&}k*$EL7|
z0`xn@g;Tgkv)mQOTS&uOD@F{XUr|ncX9BLqx$afaP}cF;5Cr?+{NLC)2s4C?0+H+&
zLnlOa<izOvX?9A)_?^E-RgJ%yW<*9D4xu_m0a+SA*g$1bF4PW#V!JOQ<GO>F<fH|J
zd2kTvMPdIlmAQY>N#B*5pZ+iW0i}R2CqUiY+8*mD`KOI9MoZ4>+}+#6h0zEw>t4wZ
zZmpvFPYxCT9pcAuWr=N_)v&z~ASC{Mdb`1YC-S|YHY_%Lb(Q-}?0#wpmCwV9)Rkv8
z!Io*YRQA27CKWMvvy5)NtGEH#xEm%Dme~=<yO(e`V88QMf*D;QEw3Ac%*Do|G?A;!
z_9be)&2qy-1RVy!;WZJaV@uW@lvFX);~WbujFxL}kVlNw>Dq$z9F+zGWOeAS$ZLvO
z3FaF}6@2&0&u)#`k>>i`JLyvO3cy6TeR9Co7-NefQU1Z)4pjzBQpR<a<LhCU8esow
zKZpuenp6BmFwi55`DFw9iNIvxdsY6^|8g$*ZH<vZ{FffHNW|aelld<JAa#87jl|10
zje)P0AcrSfq>X%&^q2pOpIRE`GW|NM*u4t*p5f_*m}=nImm+|3N`53J8>>dexN>~f
zHB@@=(LuZi73CSz#tP>~dOwsoj6`|#6PhFP+8E?oCco5#RS<dw!BC?+^^-89Hhtb5
zJG+L%ts9*|b?9FHh3rTqt<OBr{1nX{Pb2N;HNB?x>3puZI=-E?VFXslzuF3}_PgCr
zdzI>TZuymum~nF~&;C)|GY8tyS05u;B;6*A!y}4BTDN{#>siVR<5h1#ynXiB99E&r
zEh?H^8G^4pFWKTT$7LBkm6Aw>o!?Y9zUFcV!+c3wi-0_8*GGlZXspu(#szVfp90C`
znTS8Ak?+qjwP^so3GcvxCuw+YpTBczWB&~xgXkPgc8_#g3Do0G4AVM@N9Accon`<1
zA{&)IhEBW>0h}q~-!nCL1ESl)swFVT<sM57p}=8<AF4#(Ty4_um+-rt`BgBrBr_Fr
zBsP1#Y-p&Nv@FU@xK;jXt~Z;<b5<dWIwPXdB1ERE+8AsP;P4mP;Q^6yV9zC>ySp~h
zBZKgvkAXel7DVR=FuZ9(sKiNU>aJ!NVnrC>fo+<ZbPB7KaS6Kwohh=QQIUFFW^Qvb
zkt6s{8V6vY=eemOYP%8~NhI>^YE?^02xUf;*bD?SJ?%70+?x#fncTZ(5IF?6FEWcg
zYtxhqRsTPPGcYwHPHtmQy2K0olY0LntI<8qGMPHS-|sj{=p+J{@J|wMwA4t~3bd^N
zj)px+reP4E{(v?qh>8*GR9y&6%JaA3rcj0(!K}UW^68po_}6NP07Tg=OQS?_T7Ts(
zSCc{ok%N9Bp+VZtceR%*snUrby1XF3c2vP3neD3Wqg~Fbg9$v$taG{Mn+)?IrZ)w%
zl<1x|s?JyhaX_L}46M_smMDfa`N~62{GdMCr3gyDH((Y-OFs;GMAMG$#r9@1p@x27
z#<$ctX?ZYUS5Vp~q-5K-zVINQ!K5^gT~7T>h{LwPf~F?&i<kU^cuhHzD6CWNRt`|q
z1*x6r{ttlwL=nKTbqS``aeMvq*E=f%x2I!Ptfz;2Q8mQ2)OMh&Cqzo;KfERfwGsGM
z_A4<fB}{if<WJkhotzed|AdUz{85b;kV1e85ig}8#m(zr06Vv8DF>&VwA|K*sTw&m
zR4n!Jx5P+UVA=ur>LUp&kz0~2dTKC)*wu>c<K=?vbi5VK)#HBg_H?$g?Ky(fh3!VX
zc=OUlj_LPraZaLWpd3TD7C2<<mrMVsQ+iff00Pp=uUC7|F0>@MSiwJ?XU7N_J(4Os
z?E9tZxC_0XgD-h!b)^M|CyfsX^S){zWWq|;JO+OSGrjU*<+6wR@Ju+~_zL)mAyPnz
z7l2OPt#47j|3zXTq9oXmi{ZX(3EH%OAS(ko$6b?a+mUO=v|g>V_uA_sytsD%-y^!}
z40`$2^yOZ&f9H_+(VqtF5xLau7XJYVhkfSW1o5@EN<`)HA^j>`i41+bup_?2+u3I8
z$^bQ`Z7;V}3Nw0vYceaOK7<}Y@oGxrTw$1<L~5fsdGX!RHWW}y#cge7>r0Br=Ljrw
z6diJxo^+AMIh!O=vc=$;M#@)NtS|msc#pffyks5CY%{*Ac6FA<D*R5t+7h8rv;QpN
zs_*Avk)V?$9i#)u>sU!;DKxBC-`*75a#cj=m}sGv)P#<IwubVyXWoXJTJ~>bC2rm*
z`xt8;cm>|eZ3-r5yia@vg6`?>zyE(?c^f&FNB5XU{?mLmmnWA2Kb_(heIt)q%z<EZ
z%K5bA-y~_S2;S9s{c8lHnAM#V+PJO!hZe9>lV2AL_NMWm7`()+Q%PmiLCLO+@e51>
z#?&m+KAgAAi02L{kiV#}0=G;`J4x_00=5f)N?>(HxYet;r(&y1D@j3B%gEPKUi0{U
z!_H98zuUV=)TcBSqZehtEVPohq89wPMI}E=-KM1ARz5E%q8nO3p@41^x;0qxc!Zbh
zsMQ|^XnX4$45_9a8Z2r48ZG9BYze#%l9@BMiNTY0$K1M)1~eh4yBC+Bzd4cwId=y1
z703H9*nU$V!13P9X{Spzewow(IAWWUn=^W(Ff&pn;FT4&;0hH<^{{Nt5cJm&{PKTz
zY;|*I@}rzjt>O0k=TQP@0z68Bq6AJXSg4s{r2*>Fo&0aWQkDR{7HVKvnYObZBhxxk
ze=1My4Oj9SbQ}LB1yqjC>FkVv=zd?E2&DuC{Ah+QJzLkrR2{f)&gt1OBr2;^KZQg$
z(fk2bB!OALLtxtS^v?XEzXXEO5h}Kr<Gyb(=E;%e_g1P)x`MALOpMy8wfS=-882-%
zxta4f7SGLuHc_OA2~pojX4sRHU+$xYt27OdM&9+Jgo3mK4VTLb;O#I=r-bR+1;Bw6
zYA9NAU-PrXKET~Pey(?vGtko}7-tzSH=NM3l`jgBY}jhmPf5VdG3y1t<FV=P;ITQ&
za^wF`^paTRjscez6Rr3Exn*sde4$b?z7Ur)F`;CHaNnx9o_k$cz+E|&LXj%4qL!kP
zVrj_0dC4f3=NhA{{ZFRbvXGC6me^orMGzOxcoY@(qc-j+u24awBBUe-?s(Lx4;b%o
zjOJ#k7Yoo@>aD~sj7ThCC5u1u;DZXEOWI6Fe*5On5vF4B5JpQUN70HayF8?4qP$gJ
zn1ZP1us2Ff^B5dj(dTq`2Uu52eS5uE3i@ClK&`Y*BRC<rU(!%E@@1PmPPKOh8%Z+%
zkS1T!hd(T)W&|bfu)k5>3Aa|vQUKP5+pp#>14`sMWJU36gqbhd%C4Ozu4cVD?`Wdo
zc;ea4Lf6ha%oES#rh>0O6uu;IivG(_li7c*8vuaNP4ZB8`oFxB3<w}oD$qYY*d(Zq
zSuBsrtrr4_Vx~CfC;wi$MhNH(f732u0J|1x@bn7sPmV)p4mtxM3OUuJ?GFB4V<QMy
z+M%YSkxqFYR2&dS#okDrLIVAs{VOLcP)!+P*@h~HrT7f~d}JP+FfBR$3Cr;YXlm)h
zTq_xg9iCUuAO_b4*x~pLCWloAt^gLvI9I%yQRIzXSIq9g6=Cu|*;Nh^ZjpsX9x?Gj
zX5}a-3*|)!ZT+$>A)Co(395hAVY#9!B0|2eIXU@OVCCqjCXbQjFCCbJLnqcmuy#~@
zxm!!-i~@-#@30HSm6QPU+DW3H0vZ9;K(ZWv8gG)jh&ZT9!OOkd5(`mIJtQq6&B&_+
z#!;juVOKH%(sc3t$-n2Je6jf4-)xf3to;J_KNvOcB7i*}y|H#Bxm<{yxt23cw2tNc
z55G1CHS(z^sCcuD<hw|RHqNO6fzQ-X!_9}>pjtU8Jl46F(e)DP;6br@9_m5Oy^Btd
z?=XkeZ!-0A1>eck{_MufCA2}AA~X0lA#D^aGq@#%bf`MJ>%#HS=tVd!9Ha`B8E4kv
zTG{=DngkjF*NO$|4@)v4IoNqzaV3UD?||m_R`DD~1mQ!(+6T+ff|!;x1&YrBqH}@2
zvL%iNihQIR9JE2mwM(`FyIcdcOKo91-?b{3lpKDy=`-wD;4Ms77QW&d#x%L=Sa#rg
zJC<svg+JvofmBy>xRBI$B}7a*)Qaq3D$O1WlstSAv$qoe&Y@(a6~+KKR0AKk>fM5f
z{NFoCqiW8<(si|-UV7?Q(my%(7Bk1|&THWt*Mi{LDd2EMe}L3<6>3}iX&zyg$>2@t
zK?znMp4ntt4feSBoPyl+bmH#i)U2lxdWl0UFJJnp1x*uJ$eu>a)hBSJpu38cgidaj
z0b?!RZP+5?vq!xI{ZL*W40m_-J?pZ)CR5+FB`)ooh<rIpk;@5X1xdkfW!dXE4$ZQx
zh#568k)97FQuUI8DY1nYp6I^<hEw+kLp4qOfCBw)3&!joG1I+%@I5uwIx`YS94!CX
zQ+-%|U*8roH@4v&;{OAfKKyzc>txB-s`}O@j>_{4$yg9@^|ArF8~@&?MrZQW5%3BR
z9;$XD|L^I%2Mh9B2@8Mz2h1C$f(31@JW~&n_vSRs;r<IZo_>&xMY+fut^{KnRZ`S4
z!L<~-QIF28{k)B<*l#IeGMj7UpW#cL46v~?F(SOB2j(&X6&-xVPUCR33{DDZ3<jCu
zq7_O?B1lX}eYTpb903)_qjJM{*1!t?kGHoDi}LH*hDj;uuAxC%1nCB)L24)g0qK;I
z4r!1srBgbE0cq*(M!HL2sG;ABUq1K!J;!?-??2yPb2vJ}>}&6}&$ZUM&a-Dq^+FE_
z_{L9}CMft%YO|Hava;||8EsO>JAC9Na+F)CRb@tj-OP9<z-Tj6F^n%I)SVaE3Co~A
zoq)9Ra`^T}KPNggDFp((AfG~y9EEpnezb|Hz_jpz5w@NKQ16lH$3d-^1?mbk;)*kW
zV+3kk?H7Mm8yKAAooN4FrANsAnbMzr`ZML4;@=fTaa4L$Gk?F4ny@xM)WFAb4bDe^
z%7C<gi|z#%s1}FLKzqV4|0cO73#4c4(MA~)7!U>VdTNp@6)KTFe%qa|#L*Lx(Qy0Z
z{MA&L(W2KDU$lp#BS`=rxv_d3J6}$rpa;NEHGE{KevTdz2XK2AJpO<L_w$9$0uWOK
z2bO`usPzE-*!S-kOu}1SEf7f6WuDsj>q3)(F#_jVxu&0B!?hf9wyI^X`SEXdI>i%?
zmfIw2{DWP>C*~P@7P)-BMXnCB&6Dnd6F)GnP??@2yr4k%qkzayF(LHtUPyWZbPRy9
zqGj4h+N;0O=RbwSfA<m)o8IEgDnClTee@OpXdNH~>-%HG72(R?*i?Q!FZ(#N#OBAg
z#4BYsL}q!vK@#TQ@I?zcex+KH0x~R0L-(hh;>R-PmOh7-W4uYT+$wjy(G}N>4*{aa
z9yqK3k@G&(RlTeTkO%K>5tu;}W(i~ZQvdYno3%cP5d8#xf$QAntTnnqC0~>5o@I)J
z*jvfB(!Sfu3tVvZOzhX(QT)l{rClUIGx*r)7pr`F)8~$Oxy&m-m??jVzjAE(Hmc`K
zrz99?egE?R)@_{g@W<{56O9yr{kuI#aQ|EjX+N_s|GxVVS>~UZ3;f*|Vx)E~o|(iz
z;dZogzxRZD4hYoIy~HscaD3vT{(?56W8aj~%&8%}1=zk(-`=t61PNZ9^O;R7e<<Wb
z$-iO_Q&2ug$%%SPF}oj4weG-{u*EhtEIbO(yZ>T0y22KIm)JbnA8>Ev;v%UJ_@>b#
zovvNvx`$v{l+K$FdZ%7S3HL)Uf?5v&4+(#pa2*droLPV(c^wjs#}(|d9awP{DF>2!
zk<i_p++b`oKc6*`>4x`yP-PeBcgOhF0lF}#JjB18CzRCZZ-rw2JS}0u-U2)SesdD8
zKi8wP&8+_4cmLP*AYny_1T;ny-UCR{4UVFwOi%KeKONkRgy$b*L;2d@9Z!I0d5l^)
zg-w}5>EVuY_Dbl~g!S<soc4ZbTdMDFf1)?Pku7~zmp1<U3t-(Q^#|iH_wy+|6NI7)
zbedfLQ@ZlWX$I7|O;O$`t2;mHxdWE;o2u+`cJJBIC{CpH_yBx!#eP(<rE4Gw1?-KL
z%;N1vH6Rd3dAHM7fo<#fLDr6WO8Jp;xRRSiM<i%#8=om?EOz)M6LLi}`~?wVXda4M
zpt4PuhK`5N_QzrQq+*Qq5BpUX>2>fQ)pk<Sy#GW%|I6xZ!2^Bvxi{lgh?woVVvVDf
z!z$@`8E_$;q-TyOEBp?<xLHvabcSti4j7ZTh(CG5d3RPBI&Zgf5>m6GNVtf6HfVz-
zZGK>F)kWiZ+S=e}O5KNCZ77d|dyn2yMqE|W6dXcsX9snV9EY*kgRzp-peQcq_58Tk
zbcH)(XaK`6DoPZaiNeH6AuRnW^+Z97eHMl=2~KC~gl%Y1M4P;^|AXGQ_9`35_Jw3*
zxpr_4O?<j#$*I<ptZHQdRmZK{U;a<{8MPko6(E!7_&byMBVc0vaqn^qDsrp+U!w*P
zTdDjH0biYDMIdpt^nBe%=YYV!!99v8=Es``7wEGG0rbpTR5u8Bq{L2ee_-wGL_>}-
zEj)V_vZMk1TVQ-I+VHcz;z7|uG^hKRdP;$Ykn7op=e#yR2PMg8@0rQ%_`zokDru_y
ztwJG`E2(kS&<`Xr_>SY9AG6K`(BSj)s2tJmn&9YD;82a<8ltj>4-=E!0ZMFDJzCyC
z@)|693M3-%vB)$WS7SO!R);b^owrfZ!dtYWl9pn&Yen1zcQs*QF&#|_;p>dp5DIH_
z|E(taQPi>bNq=mS7eKvH@ZW&?I^xeQ!d_1Lh-m+3(D?5^bw{+hA`Fo-3;<$NU@SK{
zAU)+2Uz4Rz&RX{?@e@gdn|DAs?jp)gh!!U&zZoU(5kBh^pav}LLaEltLT-<FXqn)J
zu-5RSMUh2X&gP;c0UdSU_y_;=4`JD5x7auGT0P2?KXCs4;>v+mo$Y`z7izAbYkySw
zVf0(0{}w^sz5ny>jg?68|7TIMac8Zke5Nd;ecWAsrSwc!{hqC^x(0^gsvDr@hE#&C
zPyTz&fRDZ_q6RpI<o#(tZr}7o9?K0gR2a@VR2okJd48;*BERX=jtu2kG_1271(+Az
zJ4;0r5d`eJWQ_F;6`<sZzD`~Iomon)>r<JcNFsNi5ur~nKvD6$KA`-00N9G<WF9gf
z!ug`{pmSj%n&*VZXGzn-?WLjKDR-0jA?P4gsd&m?n<yUsk_(2IWJR&t8d7)XMDp17
zF@Ftev9R~)W9t+Tp%k5Fk0_S%!=<|Bw=ppRZEwDvj1LjLb!G3F2gp}H9?4h6pSm6!
z&Wgu!gV}$T^@01gt@qCqfG&Us@gK$@B3<Q#w`;kw&yG`Zf#~gLs~LUQRNQTIzWc)K
z9(&*VhARjr_qAH=zrgr!bkpW2^$6$=Uf8$LpP(}Fqa2fIZ->y+V#B3Vz{v-u>+XDH
zA~|!UEyRzY+uIEh%E}PLJKUu!zdA3NY|#+9o8>YvsUHI-Ua@RKm7dPaewN8xrxi0v
zH1r4kH1OS)m$N}?>?P&m^WOScx)W+@Gw-dCCcZqpKuHpVSd|yy@%aepz-eXnx$5CL
zE{UP^HCicC_E*EvWA3D?^K4svaiA^(cnaJ^uFADjQQmb=Knkry04Ollb36NHW_tYg
zqKBkO7B;o^4`uEDWz}eo{iLEwd>-<|%CY9RE9W1|Jt3OO_bc2}!FuK88BS{rQTI!p
zM@20s@thjOhVBNn=jH5(oAL$!OFU0PKv+QIF;DsdB&9%!5@iLRE9j*!7C-h*kwTh8
zNMW54+DACOF35*bYn4SGW9F@pi7YBEhUt}D{@3zn6tI`b&Fhi(du%6k1;5a3hw$Br
zb_T62DdxkyYfKjJ{rkVX%Ys`-Pr;WQ$_pF$8OTo^fC@BPb26TIBiJim=Sq8?&yP=2
z37zI{2Ug}rT&@2iCAPJpuhi~#Px!5Sz#S}Lsb0`I0%#Z1Esh2Icr#aZMBlzB9K_wu
zrt&TU97O=uR1<)G{$EaK3(l<P<(>7X!4qqh->|!l`6G?YhnWY1MaZsJZMm&^N@S|9
zy7#IORf*5>slD@U^*`*We{o}*qvW3n#qb{q1$6;+`BevvAHH}hVm79hI7rIIS}F|W
z&n=#gxCCeVS%%^YBhD<mxWg!`b!h4BwGaq8He-WNZWpl5GNoG?zwZ}ml1;BX0iqNj
zn4i!2jxDwqB_q@mbU12UZG=b~BH<C2mVg-TJ&Qr$F|Vj7;=&{^?5$8c-7~xMIS8Ac
zux%o`p_5dF+FjT(e;)p;pvM|NhlzrXggqci6U584%bhHEIwAAac*5N0mY=|Ou!7R#
z@YozBr2`21w?NQe{J#YK0e|uD1O1v0VxHUZc3P_C4>ZjZa5M=#lCf~`dBnTg7Azjj
zYRI3EU;Xsa)R^39cJ0Erw9k$oJ=1;0qEze`h?kd4A2X_8K@*hlaJ!ZEadW|Gddd9A
z+f{P7?^0w$aH8eKX?g9Xj^|LLSvwiI9SBVI-EyZDTAaPyMq{-4>B?wnzamgaMU*VV
z)4}nJ*iG#;!od;+yi6NzDCz-Ve`&PR=5Msx9<<s`Zttv^gop&zwal+{<BI;YB<fG_
zePIhF__Aq#bhKi`Z!(tH&{ObscX~N6#C%C-zDn~+8R%m9ezx-bi6_Bfw1~I;{PZs1
zR6yWT2IvR_^7y=^--iW_5;ehS+~&hi`Ifz$(8mBqUghPIXGkoaOfW!-=2rarBA@nb
z-nxR4ckX?>2A|8`<Z&myibjp6(Uruz&QM$<fE|if(R`s>zZTBgelYhvIdj+@kRAiR
zf&<duFa9j3HTV`$p+mRS#u!X@qXFD)G+2K8h*d1HHY@Oj=>(#14I8T%p{N1r^P2&@
z(r*#=aMj3F?oZhD&I)_L*H#@#!otY7M$d9pD2ooseQ=#i#kz7q*_g|-w8aI=A?Nwp
z)lxE|C)xJ{!HyG>G3Ds#c&IQn1IlFIM7c6qL&Rycwu-$x6;kh7W=^V|SYm$pvh+3e
zkT;B+F*I!s@T1sMW95k6U0McQW2%`J9Vn8`&YItpB@Os5BHYQTgysc!K+Y&VCV}>=
z1gbk*IdZg2qt2)JyNe(H{fbG{S)REW%78D=_rs3vBN3i6%3fRVA8b*2#){3m(C2s4
ztvzy@K4$g&2vL4?qb^9)6lsqA%t5&oUYo41mbmS)5pX1<#DGS$Sw9-pK(P#Y^sJzl
zMUP70%MdITnr+T6<mJuVaSk1=4<tk!Yh0SPq~9;hRy#g47Px>7Ihl=}CcJ&476kPP
zj;B}z0HetEiv@ebRlB7o_ti(O$Kn8osu=@M!b8ABRV85Y`j$KMN|8jw!`TXDKMbFV
zs3Akg2gmQt#odvw*xByHg~fSMK?Zv3?Ive(CYHzl@<hK7T9`=_LL5E~{-h9F;IS;v
z6y%3ngFnuV;O+N7Eld!cUqyc-6CR`WRpZ_Q-ycaI>VQE&XBMA+mY>_Alet3+6UQd+
zik>sJnbnSM@!Gz{KF20^CQx_3&JlDT)k?pX6A?_<O-^jGSr8mrfOjo^`$|HThgumf
zy<ezQBD*KsU)R&0!}BV9F9_2GwniUccL5883lIr`Q&+sw^12EyCd?Yhh%br{;Jj5Y
zfNc<F3Oa{H#}w#uO%bbMdEsUv(sTt+cGq1%qeY}sxmtqMrL$8b7{k)UCx%q=J}&>e
ze|10k{f_it@&C5GqS?-P1HzsDWiilj;$gbAzIs0}jOAqDUS&tFbs&Ix_Ipw2>~P_N
z`xL0U%CiQarHx(5E@2_1iJ!V=83=6wBz7EN2%xUgVl3ym<#?XMQS;?AuZ7XRl~iyL
z_HWq9bGx&spc&7CMGB6_yTRlBgYNsIGu<MsS<Cplj+;6G+NTFIuW`IzUc(g5f=0^2
z9s2LI5r+DhGAX9@KQ2mcrdXGwY3aLYs=6X5!FM^Plk(bV!vKSIjk6CwKOwII)^i@J
z7oX`?Q2*G4f4x*T(Y^y`xxctjMIzg}`MOOk_9;Oyu&Ms(sfj@v{5;MF+V|1rw_&F-
z1nX3?f@`2q6F=X;>b3QO?hsEu&bbnHFatS1_N>D*pFzfKtZV~sB|^Y>oJw>o-*+$m
zx<a(XSCt`bzU#-E#%r0z>-VQt_8%v-Kjnf$Z)1w3^8Ct_G2Hw$o?u-}4#*5LQsu$=
zyy22e6^f{K)}^>)EjtWOsx&irZ|MZ#C+DQrt3=(xi(t@b?5o^#lFptesr5Hrd!sX7
zs%R=^A(Ucd_$sv1(LlA_8>1jpnvX~cPr&~hd8YJ{hdS!n{tTy0^ZB&p)j>g~kDy|P
zumEg3i}~4N@8h%f2L(MoW?Fx<w$Z5$tRA1-tzzye<t;?54C0Io=!n&j!EyjDKVTQp
z!Mk7+|GJk2z4Y748mcogJP-|#oAXUDRDT!dMxm$LHCWP)RkHNAKHMGU;}5#&TTjKV
z<{|pxUSDicmf_4g?yA(?3kqL5Z$f0c^U4om{G~d47C`tA@9m|NRifxR$@-HPF$Ohd
zeXQbV-Zu-&SOF+?xK7j>zkDC=Rv`;^7yyH|8_mdt3|tfo=#4KI&0hBa+-kEcl~zE4
zF^EPgI4?@{xD9a?n4HtO7djMi{wGd724?M5TnnaA3XuO7+~}G0Y<FLNgpDpV>L3|7
zVn5+DL^Xb-)3xmDFG{xvCDD}Q7Xc+hNiTS7vLl{x8csTere?u*@Th1+r9pGD#6l+&
zrId%Wap65#B;hI|E<bvyATb|h!|b*8vy~y^O!)>VA9@#HRkLX;g(V1XkoZ0R7EP6>
zsXb4|-tqA7_3FJLEmV&2lBE)vca9@a=LKit1MEf2hG}Ikn{W}+xuQ~7RZ*zDiKFqD
zM4keh^E1|=eG+V2GUAhGPSjRCC@N21UlS>}jW@rFmlpBocj#kCMoP*^dVb<EYZOv6
zOS5@c^{8Y(cn(k)UHA`cCRp~K*;I8ASzZ}H*)}7N+YSJ$Kjj;vf^ZgJ?X8b@=Q9=K
zWUZ(~98W8npz`k?oIq;Tbr-i+xAuhzfIBt+?FN%rQK9f!D4k&B!y`A+GR)$f97T8|
z^|_H`P(@Rg+cZ{sSHu;I1^&5r_h-^o;OfMjG4yRtVHu|g0NOLG$H!F$S3r=Zy1qOz
ztnhHVzuNAAovbUIyTAJZjQR(|BR+Aq*iM74#4~IPFmV8P#p5Yu1tB!>?E&s_Wmp1S
zO(p>!Ff%7D>)zsmT2HEj4}WEuQ@=yw`EE9Z{%~&iFvb03E7xrS#{pqz;feQ9TZ~1N
zFtoRF&MY;5x$x1-n>KWkXU};3<VftLjj(ZQ&8rDRrc7eHmWS#Fh;=dF!HyAmNWe!C
zX7S0@j|UYTH^yI1+H=oSw02G!VIg9-MyfqW7vy&J+0PaRdt>Q3W~)pdyG&%z8#Ja-
z?YyKT;RU`!0KM!|ECHnDfsf|RAOHFqf4>03euqe8j56k(0fZJ`j?9gt32^5E`;_-4
zmB~E>S8!{TLLqYi*;6Ar-ODI|gZU7zD@&356K2cgH=nU!S(Oi?be!Z<xvV)Dyb#|S
zqXLq=onB(*>q*>Yh@J?!x)<86%y$HJXn1CH2qZXg*je*9TVjkWo#2Y--8i!XY3P&0
zOPT9Q40?>GhvQWz$h0SkJt0$?%f}&6%AseYi7`E5A#3Udz-5K4&oi#+b(I66(Img8
zUMv*j(t{;ZGJ9``Qy~mJTW)9086Em#>jCoX?}5-#!cX_u-u)u#?eQudy-5T^uCw3w
ze%m+r=6B-=IRExz>&;?~uH4cwu#7Zm-WPN3F0ZZtnGF{bQ7^Yw`eg{6k$`{N1Z5~!
z6DADM-YIMc#t2I)T69&-F@)31(0;$A4ySNi@drNkgUujFQtlBM9*oLNIzDyt+-Ehs
zk`D`k7K=SxjrgN9oTHOKk7v*WNl|65UK+b3n0tOwR}pzx$<!S2=MB&n{uuJ0XcfPs
z+UHYksk*(7>&K1QzVI;#U#@H%?@dfdRxEjUuvkZ<TiZ6+?kv{%x?L~nbs%b`=~-<j
zvOYlo@DA7TX*ESYzC{wld}P-eA8{*JPb7J=EYEhPug^-|!g!2kNGWN{haH`u{i2a2
zVvdIy9VPs?=rV_2;Xhy$G}j69A5_nw0GgA8cE5dYgP-qS_b014Hd>j2Bz;dJ4B1k2
z(%=97_2bo#rO)^ld$0P>|3;PnV9YF1q(LI(iChse-7hoSekkv-r^2}5di-1idF$cj
zX^KZ*r+anU;5p;uL;c0P)GGw7O8|+)55m2<1+W^ahHPc;q0_3O%52P^t!oA7rzn)W
z*-~t={(0Z3SVYe!X~HX-oD=&$s@a`If8wz+KcB*#3JcBg#~@LgN2)}n6BJ13qk~Bs
z2GDgSMFg-%A@QH@s7-5c@PP?e<f3`IX25baMk5}zw5=-&++8(vvv^b@0PLX^z3ajf
zk>+}dr8e(;!YtYVGqFU`@8uOe484v5U6&dOjsrS>L3SJ|^m}MXDcC;80jv0#FzGJR
zE8rF<Qx>~B3l}83KPwPJUV0W+c>wjj?eQ(tj@@do^19kgo?zi&NzNB=&r@WoNaZ$H
z_PgzPaTWJMQo$S;w+8&ijIS)NXg3d6fr-&Bu;#|Vn&vYN=G#MiUyPWqr5hwolt^St
zS#MpYG;Qmww%ny~Z#~-0bGO8s$Nq#xSqSip<mdDVO3VHM9m<*fFVDzp9J{oc2fv<6
z)^CqbbQ~mSlv$8h_U{?Nz>UF0fF&al=i0@?AwD>JedxiAk>%9D=_sUi?z{Q65@zyO
z3o`}T-wP>(WuY@uRsPs{NacE9)KZ4hSyUYoNf(iEO$1DDJ@?JTj{S;OgQ_?qkt(P3
z*WF(irq%6C4fn&ezDZF%!6ciEvrZ^86KtH;Ba9wFoN+QgA#wVSlmhng8K|stu~}%>
zKG;izG%tD4)jMx}*IXt}@7kRz(c%Z(MkZKFy)x52F1fD)Lo)}1rK=YoOPgF_BT_zk
zMs@t6{V6ZbzqMKMaJIH=)HwvHF$FC(Uk9f1&2cuFmqZ6k+_63!EeaiOIo!J_fMa(%
zHD8@J?3a@(i(YML)IGa5+}KPt%i~7CoLNe>Z}|!tx9S$-ExRwBJk?h#sffH^W<Kq5
z3Cr|oG3h_hoGBiG+`8}m2Si&zWwS>G5cjv9mY?h^@G+p@KjEn>5bxmxkMYjL<$fX0
zy?L#|o0o7BA$FAo65CmS<RRDN?y$5N_kt;&%v-8^4l@4ayw|L_AC)CH$-)%N>wIl^
zVglJI@rvmOd%*HRKQr#B$&#<LU^NrpLTtc*n1i%L>I*x*IBX$m%+DH#;_1RUt~RD4
zelMTx7DfURYf-`t;Y^t<CIO(LbLItY{hC}l&-p|dlvovIh;k@&TvNUk=ln;Rh5)1U
zq;fxv(Ai18YoZ)syf872H?IFyyLo1r*KAar!li_Gy!22E5|FR7O1Bl(BFNLp9v@#w
zQC+{O2WZ|3b<g((iSbEFQ3I&Z^sXpA{`$Z|WJxNp0gNMRz03dl)^*8!`xvmq6RF_n
z?*Vz*xUB{ehdXBU_0b6*{{ZGq<8_x?yeAgC6<Ic*(Z08ImGV(}r6cLM7Q-T!zp|Xz
z#N>tOsuH>y&&CxZ5<l?Hz6+>f!NzVYONb43d@2}Wdlx&Rm1Pw4RacMg#BT<#VCiP*
zGB{pYu&A9qUf<^C{I~-<#ARBCsR@94*pL1%tj_gBBmC>1!v#72g4t&_NN2v^nQ$#$
z+cbNU+}uYk=yDh|J^OW?dKAcKXlE1W*qgE+Y-CcWOxzy-@hvgYp!fGpV&oDepA|$Z
zRO`~J*YC!k8qp?umIJ98W84XG%W+WUyj4j)n%(c086JB@<36RpRP8Rv2^b|T*B--y
zo^ABSBW@T%P1~iJTv1uMju*B8!D<JRmsT3IL2pK+SuQ{NUJV%T={DOn9zrmFr9Pz;
ztf{v*jRSo`yY!tx9VWM}QI+`hjq2CFIQ4c0CPh7p*?6MnS?TjwK)A+-#}6?+OL2$p
z^VelKz`proh^_2sEY*Q2UE>Dzy;15=HQ~){OTsdGX}g83K!}R)iFCuVFSL(o^Bb+L
zYZ{?4*)s<a471&p^|L0z&z-6s5C+cfkJegfY_u1>P3;u8DBS51$XXd~5v^Pj>Skhw
zQ|A*Lu`cJ#Go3-Tw#+kA))mc3<t{$!8jY8m!v0)&g^zk1mJ~Kx{dZ;E9hY;DzC<<p
zY+P50Tw4G(X_#e9bm}0|w-IofP~t1HCbR@a)W6mHC3V{PT2dJDvCNg7Pcmz_e_QLW
zh-p)BMJQz7Z#fZ9bfIaoO?%d^PEj{VVXjupX0}q=Y!$vxRN*Z-XA8`@f%nJj<Y0%$
zN=nCH5TV#Z>M>y;KVZi+MdPUxyS=R3{GyLg(oKy)%#K<o?BHI%l@ZaA=59Yp`7vV0
zq}4VW>qy%)y?x7{>49*XLQt4;2*-lD|9G+QV`($*!2>3^su{Kxp|03+`N+!87tSoo
z1U!;n8TKaNTDSrL_smn{!|1wD=yI)>>;4a>pD3B$24rIoO1G6FXTW@VX3NJCkwzm=
z>w{R}Xo=Xe4u#ixK>#FtzUE8Vn(dzS^Py39eDERtg|FBBCBPthZrm9fzMpy2{g@sd
zw<mb-77og2i7q@Y=EQ^NE%?S)i)dp1zb;_^I(+!=2$<LgFo~|bA~&qT6S_QMhwmFi
z&N%!S6sVB3XU|?mhbT-FdDof*6e^1rN}*&Abu&d9Iui<+vG!MMdi7Jx8q*1NMQau<
zUQ5f|89jyND6hwIUcC`0Cg2(^Fu02?kE{@O#f+ffwXt}5EWxy);67EthEqt4IzC2S
zG~<WV;ulM0<1DMi#Kg<sX;Yfszr)<ZfnbxTz#@@c@Nq|>H#ZknER)=#u9BIcMM}Om
zsdCHG;?-T;&gb4$=N(67<5v6=od-W%9P#a<7LVNIGykoX0BkI(o7tXmM2B96DNyHR
z#8d2CfEzTD--mRObic*dIybrCbH%GHJZh#%3L?`nYBvQ6Vbz+unAO#Xsk0wayyE`W
z0^kh`naA_ejZ-euw5u!%5pEC2HoRRU?vk=QaD=LmX6F<6n+?63DV;S4M{)27jq(M>
zoxb^ek|5W7;sdC+2635mYt+9}`)O=|{e%#!S`z5$173mSh^2~UR<ZkX*hs1f80AKc
zWTnRr&Vb|>bMcqnwVzf0pu7M8<?Vef-Qy^<aUU({NFlc!%WfQ5dZ=(%HXPo2(>jL|
zrF4ybch~Nx%wcWcpigp{eNuFt^dQ18=)T$YHeUhoFuhr{bZkw+XFQ^A4YzRsLM$V#
ztR3Q<X+gSce+g^<ukUHD^On0yB{?)i^&vQ8S9>KDSZ9$_62)loz_8aq;7#R-&&Bt!
z**DYWg1_N^aBlc?q2tf}x}iC{DsTLaF7&%TS94c9%l~N`B*}HWu05+6jtlNnOkj&-
z_l{z{%zzx8YgdZy#jH`L0{8l$_oN;p5v%R2=gqGq&&>#qhQhV$PjKz_)JkejBQU^T
zgOg9`GBGBNK4$EwejST&qWNk}hALf_{FSL(ur<8e<=|Itp_+MigC7^LNU)M!{c5lS
z@BjiUybrBkR;-tlH?Zt(`q-2{+$}#SmH38F#_p?`eM%i63dCkT$Fpd-zN+Q2-lUO|
zKq_{je24u5^@S8iSvy1P1!v<ybunvRF?LM)s2*AS`Or40SN)SK%!^*S=o+>JhSA%N
z2T^<jUnlVFjebb*NBaifcZ8%!(7{@D4!QR!$QZ1?jdM|@b<>Qieuq6FV;xC0W5Vk}
zGHXe1%uyi0*Msn4=sP@s(Ol!`eH7$F;<jh%;U`=l@=S+zH0BX|J5hVEBLXH;B2d89
zO$-+ng^B3fT3g*?L)n6sA8G?fXpkmf(9WvLP!jS2V+QY)DTXhD$?deKD3OT3HTm{G
zcy4@|E<2nkZpcOL3kEN_A0FI;H5m2pYyf#fX@kI{kw<3_w$C#Hk5$b7QAQddL@+}p
z1JQ0Ek%4$Ee`iwdQw-i%=forgq<Swf8c38XpO1GWgWkn%hmhaDx1P{WcOHw6nzxK!
zN%dIx)^spRGm=fk=s>d*;T%dQbcX9ZCdKpQDpqC6Bm%p0YbuKJu)Vumroi&&2i8v4
zCD-Zccr9~MzbCn{r*HjqvNWREMUikWZgc9yS)((cT&ZjGRd^_3;dGP-8ZsbN3g~1J
z)#Pb-^4hzCmOZ3h!sm0+i5r<dtSxJ!uQL!0i)FCArD3DpVC8)ogdt?*Ecle)tSR(X
z3E~}kKTD_E$<>-(%@g8bVzbPZhsxmu^6?9K<`$8ynJ(~46t4~$vnt{oe=85H>*<Hs
z=h!#7SW!Ja#rcAwfd>MetTd-gDu7lc=38!l>(#Fs;jH8S%r}sXpxC%~d()j-=m8hF
z#0gx*ZSUWFog}xvT>OY_cn?j+AU;dx(>)A;dT>2Ud;hf@&*Dp9H8xxcu8G^13ez9E
z2uS3hrwxC>i&B2nouxtmUbu=~75LkxIsUlGnP^v-0QHu15Ivj}JwHf4LdSml!2rnK
zd|fRJQ;^8O?^Ys@-n#9lA1V|pIV1x<&Wy`H#8w{~L;1ox-qTz{uE6q1;&l3Izdve!
z{Uo}HU-Bqrr@K157hi|}!HSL}?xr%nYP>dnzxX>d{?}4K)(XY@2Cz4tfaku(j1s`@
zjkBn~d^Dj?|Mk5(w3(Rb^}Dvf>l(|6a69qi?93ky_p=^!$uIDfa)Hx4t!EZjXOl1A
zT~Ov&TCse8QMU>lSN39RP6@K)jrbB6oRkrYa*T4R_~f)aZ}D~tj=@C(+|iNm!%de}
z%y``fX1RT=$1(f|`IG#SaFUCUG3D!Tqg&>kBAO$+wtLG3)iL=P*G~y4_i09J)=o>f
zX5@h+Rjc&i{W*;7M4+!i4_vXqZI`cEzcC}a{mW-tv*Qb~pmwvBCF;xsO$bFCn00Ml
z_~zg{t0;<o*jIdujMVe@j&<`fMn;*oQBU=3>t($7yVi?v%$iTUeogQGau`|X>k2Uz
zUbDgnC(YS6MhAV!O~zp`^thWlo)H?xR9|ydiYv#lfc-XGpsK8VFYQf?XR<e0w0jeC
z`-STq-{QcWXu;|Mzn+L1n$TR|`TdJmiSa$v3hRTR%;($2&w>hB*)~B(s4WqWhB5Fo
z$hy1z50v{M)^<Q~xmJ^Tq?vTPEdzzj&Byt4NMV@Ir3_}X@UU?8QJCxPf8XAlxc;sP
znDI%sy?Fk?S3YR&xR$-#N1SVDNF;9XrSfo`epK(Q)=|g~cj$_HP&}rwO|j-H>6R}L
zE6RZJsq1i+udZHrnf&3L`n9a0kWk}nPB9>7Rsnfp&4<!G?8%g}xRF|EQ3_H_F2JB6
z20C7Q<V0z%nmc*O`hGFvPo5)Co5gb!9pnXwdh87Fn#F_OIaR|U|9nlmY-$o~Z{zZ5
zQf!Wy@qUu&966I>Rx+4I)U=BU_!+9NWlyr}^Ac<u5Avi^Z|7l?z?J`I&+X7r!>i57
z$b2CGwMg<N3A_^COZhA#Sw#oMFVC|T=*d8+K7hTNawjebU81HQU@fP<{*1y5+~ZAX
zw^J29wa0xOoJ+K|YPL7uy!NS*l(*RpWM5wR^O_Vwo_aTL;DYc@o)i=Axw%D5Dp<tc
z<;le(U>7j3Q4Zk^vW>901N46(8{0gH$NHu;WqJPX(2NJyn*-rP7Qh&50>~U%(CeDK
z^)6o91jP~m^(esnNY3_?>Zm-dU1+V%tsWl2(bvqC8wpEZ*E1czW(<@Gf5k`6uQoHc
z4%bs6F7Sw4j;62HnrFze4i2W=znOIy!7jM)FTunpiC*I0%=BJ~s&u&jRry@`o0#Je
zLs9vhm?E6xh#3>~2j_bswBBJw14rNXaMUYl1LZ8E1qLuh!t=S2JqleBcszdA6P;=l
zy=?IfNF2UJy<r#jf~x#u$0GcO6%}0o{$sSrg{&S~UvODr4FiFT%|tsStKq(IfC=fe
z&&7FGH}poW+q7?>rfMFv@JdcQ{}bBlZokSh<P!SIYxwqJ-hwno`Thkl*H>SwtK>co
zeqo63p5$^L*w|~nTuKEquPl*PqvpB93XiUXrdkrlcKA=m3LPTTfeeRm1{YS@j(Yd4
z0_Xj~9+zU0@4nu0F{-In9NY}x#1qoxs$#R+4hEx&(s0kQ8&YSAyW$byG||*a4JU7u
za8)($oX;3mY$fPIpAg1WP`5tcjjU|Qp_D293^@Fa_<plioWbPeJ^+3XD-L~IhUlOd
z@4d=dQH53-x`_1htViy_pPYY!dE$mqdCIPDSqy3uL>&#-a6{!BpA-3b@rnVa4@9i0
zs~EKL3@>WXldG*Wt!4Cj#KZkYJ6w!>7v5mQ$2e8;1U_Kjax9vq+Ty{-p3a4r<rd>w
z@C3UoSNY8PHkz~xh1A(5zrGtJrov2K^Z~p5P9$CR)}thHrD#;SeR|@?8pyZZ3@Qi0
z`|$`$g`%E#e*cA-KFw(F2vBX%YqI&d9XzEl(CI_$XlGR>PUi`>UK6%XWH#{f&*Jkq
zvna~5MZb!wY`6EkN0Ea?nXxjXilB~&!X8XqV4zRG3PwadOp#Ld%Nse$q`O{*9((ip
z+``sQ+dpP_4}Vf-y+3+x)}qJouD4&MwBcKe1E+YVOKaN~m@m8KlsQ1PkepQi&$jRH
zKyiS{(mT7od$O_$hmHKB%5XBn-t+k+xCfhMAvj9fBX5$mE7NPv11M0SS7fnO2(GcT
z?8)L@S+dq%Wh;4fb|sOZnj&EZO!b}CC{zY9Nr*XC`B-#dOH#}e7=D=I5K64F)F$!L
zdX?UZ?H;K7p-_n$F;6h9&RRTA(1<HF@TXo5TH+2oo2X?Em3x;OsbMvP=Is|H(Pt)f
zVtLh7E!3kpA@!)trgN$xzBV}X(haI_)uaWXufFoy$lBg&D|&Y)cS|qExIK-=c1sgS
zBwN%zi)kT6!0a&7j7Gy}%tPTGDwXaCFa=#79|xf{3lsb<-dN!0B7L`vwQ1XZvfiT=
zxVow}ccCPlHUBOE6MYfLTX}DRp#l{YBw93Vo!a-`_Y$TRKv%mezGT{SG6;-Rur?G8
zn^19XtI5J}K9Q5I;^(*P8<E^-=kGf5)HqC$$hv!_yg{-jaEWp_AbR=Z9>jKv0S~U2
zOyMCko<}*CkVAx0%#%YY+w1<0leasZGwk+nPoFF<eq7d9WZ7&maQ(9xq|Jph+N`QV
zGkQkeBtLuTB(~=OC2DyocDv5R)O8KXWCEg5FK><#%GB}rBuYEb2BUHnIcXK-h0Huw
zh~GYZAJMX{DDY?3fGm0JOA<;VF2rxN-wkd9)#p!c;-1P`RU62RXg>-(x4xP?YW>_B
z>rK5)$;wT`xA3`aTqdNam}iK|a$lfiPJJHqjAk4g|5UMUfq!<)u5P-o4!NIch9<qT
zKO-HfD|+Lx@L2_=zT_`ei^Y(pE#r|QzdTQBlhAJlkW9h`NCSbi$R>{R6LO$4+6I+F
zB9EZ*+{?JFyn?h+dP9Me-A@=CvVqjK-{%qX$bLuShN_C)pG0>srrN}zs*VRZBNr{B
zU*74$$6K!glkuFg<lrONdbA3?+g?A|scIwAY6dJSFx(7v-%dC0$P;Ge+FQy%^DgIP
z<s~{V^-y!wKe<604-2#=%t-)>e35r=T{lll+@{n}rF4-ADGiGKI%6<cFOEG`OMU``
z3I^LCx@PcdoZ$`Z!S~_lf$%+Q7u_Pz+CyaZmxp;zCK9V|6zKNP0#;2~(*5go>pJG?
zgl?p{tmCBquUq(ZKMvIQ4{{lGl93m)uxaxH-TFn&#J*+cWy5xt%X`tm^ySwStON_(
z1Kj?>Bkaq~w?~x-Tg$L+J{E)URlc%b{T*nlG}7~744?i`jSEi-Ap^lrX)JxMbjA8A
z9|-rtr#`v|8yOZ`cRi%v`pV5+GToU4k%j8xl{^+E@ht+&3;q-OFQNeeV#ug!gnNyP
z4|H9XJ!RrXg33g_El?vu|28KY3&jh;T8fkd{uZG&UOvRNO9P~Dvw@*cE=CkD;xReE
z=i}LfNK@ek%V`-5i7XN>vN47nGx!9q>3B!VogBg}I)VjlWN{NM?R6b~Y5oimlr89;
zP=znVDqcchrnJUJon|DOK!ekerZ%E8tnP_&h0pgHjT&Vnzh?2SRfjOi`xJIy(L}$C
zUs#VPq4>dhX4|~B9?g4&uNv~=1h=ao$~ExiN=OgY2=QDUvU<eBE*RA!u22U7Ghh>*
z8QThM;J$hGCOJensp$<S4yafX$G@`;aosC5-^w+8D%p138E_pU#D-C~X+tsWS$9`!
z&xZmEA8uM7SV0Nn-BPBE2v8E?yfxdB7e%i?tJx&jqCbDenICs7m<tOhCE)66GXi6L
zoZh<%yGq=1)q+?h8`Mp)-S5K!(J+?Y4(GEOl$#7FLc~9@r0x_4`Jbz@XX0BF?-q$J
zLom`vnfS131N$(RqBfhP3O{`K>u8_+eY7c`KRdX;V!)pf%#9#se>!uw)A@<NVO+ks
zebyUhOWD3>!i$L*(n2@r0gXN08W;)UGjL-Rrqart`)y*Vbl7}z!3haWx$G{1H^kT)
z!mPXd-B{E-CsvT*VO3T3W^SYTyb-uAz&Mdk2$-yGJ44Q)!&9>JzvwEhLg|^*v=Nlb
zrnFvvCQ+^GDqvXKu)PY*HICov!~iFb)_zebPqwWx@t#Vp+sN=zBcyU&t0jA~g1V)v
zly75=y^smCDHvlriq`Kjn&=g*)1WH-d(x$jpTtYoE+t-8E#!)mth7w`19vI7>YBEb
z6O4ju6h0L`EU-;sJ^m(R89jyp2mEcr$+#<yuh^C=E<Q;+NR?Gz+t76(Z`k;;utumS
zs3W#<C{I8;7jZB}>i<nMbGb>yeNH**R4;hBW-3tA5VGuZGdakO)8a&bLE(&?d^>Oc
ziXOaF$Qa{M^GTz2#O&pQ+1^!}l=B8d_;9yQ^n(ZB8EDK}z&^A5^5FZSD!fqO{@!p;
zIfifJ>8Inp17dakS~1{YZnXj%BqFf{djDOGWJHU;6qd*SMF+QAl=Im_ikxd+%$;`Z
z6L&y!9IuLpa4j=Uyu&;<Wp;!qkUP{oQoi;<=;|RZtu(2}U!2<ucB~=d4&7MOkuDKk
zm{zGhnrTw`JIMdT!CJCInn&Xa>G2s1MX@Js<9ZjBN#FyA5Df?NTPiO|=f(PfS16@&
z0nPT9$LXi-$x;w;3BARQ@aSTyJ?SeM>h1^*aP870D~ZFjrR|ZWqQvSVmZ1d``V@Y@
z#kQ%UX<0L>fK3)f3Wn_w0*nmL4(4==pp}qeOB@90d&`NevDK_7SFCw{>JW<+uGG%!
zaX(Ea>SqhE{&-q|>A$AVBxJ0Pts2yHsK>?_t71c7c%5((@~*;XEUK{dzEIh9mU?G<
zHY=bP>}`U0!v!dr_~TpPi7sA5Gln%tW*Hp|H?1BPc*ybmQK<}nCVX}cKzYLQhP~n>
zBW*||{sx6Cp=`^YTJ2$~_H-tnC_|D(2jT}1(TSf7g5_p*2uQ6;)(MJ=`I)@xqNx4W
zsJ&D4GYegf2Hj}oThRee1vfqL0{8^Zt{^?>05=og<-IzQimfU0!2B}M;kIM|HBt#T
z2s}}wpY(W^d~*a~eP5g$w}9{5c#+1-0DOK)m7_577dZ0Icx<{#>+YcCF6^^D;Cq};
zs{}C6HC&xlbg`$Iu@_m4*q%^jqIWD^l^?eeZU+?{-I<46)bcHIe7+flY4{F2WW+(-
z_Od3Wk)??i$t8M&Lf<Gs6V=)LgMscpP*%j4;2!3ui*1SsXEO;Cvf=CWj_<^?(5{NI
zP3{jQlm$A8R0K*JbJzHDK<U0C!}Ir7C=P0|?Tx?W?+~}nR|gv&=00%}Oup2O&%gE;
z^U;!j$Q+y&1@F;}Zs`3^bKm;9x*9?Sv15VSu$%U6mww9C#3v(V>coTG?<v+Vn?lvw
zuf~eG(qjp_qp1vW7E5B;*l1+AWUubp&J!!xj9jcXbykC?9G6ZrF6m!BuV#IsYS~AA
zTb{yY@){*Msk7@S#%qC)^q{f>t1ALIy=(6&A0(=_HJ{D0fXM$_Vy0naiLO-`T<Q3G
zIj$+2{=Gwi9dwS1c_u8*ua&xHFa3qvSSV!Y`n9GGQp!R~5sd;yDDHB6^<!V7@`O(`
zYGocb=W?KX-<+u|yu7<B53?%%%29!o;<cY$Em67m*=Ioy&#(nu)x5O=u#BsJ?s_mw
zKpSn1#&fl2?w)z@pqQ~<`O>j@eD3jfMB9*RG&OGipnE<C5U!d-8#DfAO)RPn`WFB$
z*gbKh13gLrJkMs%Kq!L2^I~WI+yh!?()iGtc==~^I}W4YT;~frs4mt7Ym{OjwOxBT
zDjQ`jUfC7(;r<-=lbm~F_|Y_G5u&(Mp7ZoOjyM4Xs_AEz>Y1jQJzA|BQr@pCEPu+C
zNFz?JFfS-uSLTgRz1t1gvoMb#vgoflo&1WSC~C~RJJ#)wDTsRa#w*dX4pO83dNjFN
z8Rs1K%A$vl_><Q&M&`rWY^kqOcTsci=^l1yJGFdI<b>#H$qgIwXi2gMpoF;qK`IB|
zf);O48>o{5eOa8x&XeVuA4<+dfr^T~d}3MD3m|Q1TZj~HG8MRc4XU5SaA}Fex9>f9
zmzsa>-Z3B#oI+~7<4T=&Db2<TUVOx)&p+{co8%G|ltFz9r!VRg%6ipbTQJj&9=~B*
zR^9_P0){ZEl(wsTdm!2??KLj0cD--l{<YNr_9~oEdBp~c90ka`q4A;dT+yDDL+a5&
zU$(9hSFnv((xtwjyn93>O5i_48mvbmjT}-+y4ZR1bBo%SLTQ~sOn1VA9yxS<tH5W)
zSKhTNjFZA6iYR=l1h&O1oU2l%O%PFgByfgii-Xy%i^I~T>yXHPcW75-ll}l`ujp$b
z>2J41_Y>5Wiv}2^KQhkgyo~eYV|@PVOCKeVLe?rM)wYx#2QmOb)fCcO6aN+Ka5Nkq
zCQG+o^JPFwr{A8kar{lQ>yHYn`T(ZO&$=$JUC&GLkmR1Mq0g|lvQvAnR~(4ea;qt{
zG)G(;q4CuAYL!;D^NPrtS1~H5k!Vxqq-?x@USb#_KG8p7U)xf&WPnmCj^^J&TIF-V
zKj1ah-eqtzE9tyer~9&M8zX%fm0;?2Qr*6f+eWkdsL`vj&f``)-3;)bRN=oKA8kGI
zJFe^74zVtHYpf3*;;cX-%P5#T9O{4at`7YDS{P#`MWP@4x!+r&)5L|1{s9Cm({Qh<
zt7f?sQi|Ebkxk3hmgP%|^XBgl4C4dpH(w9>AG1S&>i@cwcCa2<!P@@8DC|yaG%X0i
zaL~L^wR(ipH9+%0O!8HIm}Xx$B&_BW=N@xBZMcy;`glj(+JxOY_sKIVE<GHR-dq$u
zc+Ud@L(?lQ#r_Rkwf^d!<nCvl-4!SO%A(I$0Xm{8`Ni2+S2gFN++Pf;Obu)FM=^ro
zDo|&QKaAVyF;fWi;S2o{N!rDyv^Mjjv1~irqN@E=)3g%<dAkZEUGw(H1z|s18NR=H
z%|q1Rsp>VaG5y6<g`8_6l^`2SmE0R&?)3?fTbh=PG*on~N1sO=`1Di_=cm$S$>o4?
zEJ&_*#&Rd*d72WzuX6-0W=$saB85@k7o7p9$*zd{iNq&OyV6Rf33IU<pc)W)Q2YX%
zT$iF#)g@a0Z6paNq4lsI@jTjVgKk=>e|sYq3tb0UkKGdCCvPIIZa*rI;@L6IQuvY|
zSbynKE}jt?i}k!)2--`B58q?lrMK=H2i)W(NMuPbDrz^0=|RJ`>_(0VX^;6@TY%t_
z>vxm3B&XpE-T%+w_}H?&^4zMi88{ropk_YBt2?~vPy&EZ&UAu;h3;h5OtRHF_qt7!
zd{;Ns2A}t)t*U5m+OQ{4%D}simoIUIDbe>dbve(L+giZnT<SiPE(svob2R21@><V<
ztq1B(tB}YP%$UE|;kzb@BUX{D*>M}!eFlaFra~!rpGcGbZa%e@EK|-fpK5)SG|y?s
zAhclHIxi12Ix(K20zXKQkrte*j_sBLthcw^Z-s_r%c?dP=;y+ITUEKi`|nGYtZGcC
zi<AggV@420{H&%Q^~=7qeSeeJq>sD?9pJ=%%#*E(Uu9Lz0Gf)aTbawcu2(K?5tKI8
z6Q5_DMhV{@)MCRA*)6J1bArO>ryvLN%BI&t(>l<nt>4|o@=7*Huh)=LaDD?K*A4*v
zU`qI9(zA{ukNZ~Y3tt71WxgvS|0D;X3oH=JbM%%SK}qZ3Xm-T6>}a7$1@f!^F^yuY
z&iI!-{zzro;dnd_SP^&WxY0jOa0HM*a0pEB%J5!DVe`)-9wj_P=2y_(sC;d5!{tit
z@65_@J=jPUtSyRyO^Xs?c+VLZz-Eo4ScVabfXXXPD^#KwR$YLVfpn)~Xu`X>sdqA(
zuWSmCuV<@@cgU*>d$1P2A*1LMx`4!<8U!%p=Y~F!TEdEIs+Ca5%4S{OD>y^;6Q_Q4
z5Eg5vr;y5d`>bV+1te;@j`p0np_nd_{W_fLJE3=;lF8=#H3eEKNCj`k^#mu`EDS2+
z1%-7Idv<*O%&Mcf7Sf<_`3hiol+lj*br!1Tj!UJdNuO3ld$R!!;|t0$=2p!=7c`{@
zARpEws5~KhP+8!5z!JMXZW}8+9)%7I9hE*G0Jt1q0Jerx{P3eKWgFfFxWAXJW=6qc
z6_d4lIaK)J_SByty=;;kn|vw7<f|(h_67B{tmD%L`9C|QM7q%B@Pxx2k{-eRQuWpH
zdGuF=@&<pQ;~lQQ+9$zSRQM2~>yrHKoeTvPY);W>$g4NyqrRJ(9ZfTakyF=bvDP&Z
zn>vRz(O5#mVzd(1%EkOgXmuE*X-Kd5WEzh#ga+ZcPER;llK40}0_J5~X)(=isn{P3
zjyN(Sa#q_u6Q&uU3*iNjFR-l3b$&o`O2P*-M1;yX5J&?wF^*X+lK@H2kc*|SH%z~;
zO8T-S0$9LT>!RL3cR;Q=_Cs}$oX7?~SmRxZY8ZpC!*WXeP`*rmZFlp_K=|KZ=elbZ
z`;W_OkyTo{a7BsPv}1<Rwq>_b;_6zGlb8^c%uA#t6zc-7BRE^?Vd)xw-WYT4)h-u;
z^&4>t2qh}KbkizYz^H)eT_sXTwa;c=*)X`2`E*Ptonhp7&~L=byt8Q7t9TKO=fu+W
zmt$nvj8&hx=;_z996Z6&`_px*2mYe(d&DKaH0idDfx|#=SGS3xJ%ZMiwGIq)nOacD
zuUg}MpHS}~H_sN%j=Trx1<F~L1nx8cc^b4qIFB4<+?y97AYK39AKticj3Lye$D=Bb
znl0nmQM_1l-8K^6Xrjwuh#Q+B8oW?u&?nG~oPcMd*E{p|;hcJBVF4Vph8t7%^8}e1
zCtW?_S}u909-ZRAh=E#Ha(bk_cb3B3MMOjJD`j$Co+o>uUSrJA;<|Johh`Od%giZW
zrD<U6Cjs)D*96O_cR~g-2%uqWxe*`~TUd5ve3oE3&OAUNj8~$}f#uE8^bbVuh1qi=
z?3qUUt_8#%5a3fnL%X@@w*Xa?9i)^%g}S@kkg78JQMhh^G>ab=ygB!ke#QSBkhlB-
z%6kBaeC$a%hAC~E3;ZW<GV%k<H0Wf~0A7*NY^4#nwp=)#<0|!2Z7&_$>2SyUo0ZHu
zOl-gt1q(ZTkEX`v`w*``pGqn&DyJ(nE5j|kBQ+W@9w-Q>1ltt_Nf>F+b-a&Sw8~ON
z7z$&`Ako&8g#xCM90~fTO~`{TCdm2_Zil~X2Y;3YX-6HlTuFT8wcTe!v6w!b<OHp*
zmT+KkAGK8F*w)69BSRcL_A*l2O=dV#9|CNGir5_l7bFo7UR)~R0f<L5K@)|+X88(;
zwI#D}=7FQYO4o9;c%=N{E9!g6O42I)2!Md~1dBrZg9>ccBs%DHL`)3V96C#MX;$Y@
zt%tOlX<G@y1oE1&2cXVMz^cphV%X|lsHBcHw=$h=A;$UD!Su6fZ2X>IsmZZn+U!ob
zk-YMIF?HHO+xIs|bz98#TJLvk^zu!AAPfP8f2#5(Xg}7SCNig{12#w1g3c7}XbtqV
zWOho;lTtjxD5sgUD*jymCjgYr0+xqr!MB^2=RoL5Ye=*2n;7JGBv;5-6(WFszH9!n
z&!-E3eQ!O^sy%{*BO0#?t5k`b*8B{>P1w7|GxpY3mKEp`z*-+Oa|fU{lrIVmSze14
zhRUjz^G*CwL)&}x>%VBMNlZvj{kOtqIZ=6%5;-yunXA==zucH36Qh|@PR529Qqxoj
z0>u2zwq&=bRt3to8-#}U1cu%k0>$4*@=)(_<_R-jxYuLOMx0Vp&Bkb@+Kj1-y_X{O
zPTUM2h~?azqUvAM*4B%Rz6gFL@wnTaUn4>g)qAFN6np9Xs3k*@QN<``i!)xhR%b^F
zN;3t$a8D#8r|ilb?%sG$3wtGFyD`?4Y-5?rM%g7Zf@h!+HPt1TguPd(Wk_iFLF08#
zw~XNlVMK$(2?;LMF^otVH}p4K3Ox%QzO&0)VOeu`BTy5KFv&gOjEy(DWUeDS$E(b1
z_n3`m2cyh_-Gq`i6P=8p-F7z}d8pVg%A8+=RwJvuFBfN0kRYflG~W|FP}$i?hXb@&
zORM8C0x2OJ)qpC`EkmXLr@RceowW!wQm{kpV9Lu9<0I(s^BDcwni8axU-tm15L=OG
zvl<wE3gYxy#UPwgOj`v0UqY9EqW_P_=wE<qF+JziQ#l9UqapIa$<R}v>Ea;XXthbe
zXtP!(<fnD1`mc3i*bD}3?ABH>q|B;U%c?4mgxq9jG<$c*T|peuaCL!Mx;B1rXAV66
zt39^I8kw$XXH-!}Q~KKxCNZZQ!2g4t!l4bms3D01$pri+fu|7xb6avDd}aS}OaN$R
zFK<Gm`lpNM9eFZgw(rv@DQ0wxM)udE#X$91wqR|pP)*{qR5;0DU<6h_VJ?N`!B&Dz
z;VlB|b5{IDr2%LS5a8+g!1Eh1;s{v}vH<R26>AOGXRp;8n@tqyKhr@AP0ZsTv8l6W
zI%oI)##%pXohe$0u2rU~M}scOFR5KyI7iGJ7jO^_#QKG(<!@#5BRIooJhoY<2w0qB
z324h@x7{<B1CM0?*<q5t3gU99G6V2d1fjWebU;S4ot{LNW!D3xL5lw^?G?}n&^a|F
zg5>K0I`N2lGAH9inI?IfnYK)$JBn`q$!{G^|0CR!-3W7oZgI?)44ocmi{F^FWWErF
zgQ24~!2`9U!fikMEM6OCbxyH7+Tu=|Dq8}N8k%uSR?Y4nzE4>02Lzeb_2@NDy%hl5
z7u%t^?Uj`AfW<{YYx-XMm}rJaQ=J`Gz;<F}Hj)LQq1QQH#Wq{m{*5eqS+^FW&@zcR
z0)0#}PbgeogD6)8{Dg99ZGrx^yH}zd?m?c4JZZMRC}Fb4T29Dpyp)J(Ej|@hcnCID
zm!`o<qH$)O>Nf-9^^(l+2&}`qbuPjCcbOXdk<SwJlkAbs;7l!nOh5pen^Eyg3@EPJ
zHeMgk*OE>h5*Z(<1GJfe=i7pawiP3V#hkuFBIIgAi>%YiR^;`6n0xD}DA)F1d}xpu
zkd_YV5Kx+-C8bL`B_t&zh8BjB4(XOIg#nT7W<VO078nEpK?fwx!`}OS_kO?MbAIQn
zb<Q8Z_53k7F4offx$o<`?(6zQFQEvTer}S|F1wq=k_zL{V`nLlhmsDsFrqoKCf~`X
zkKK(}?nUxeNya;09{%wY1g_hg*VIC6QdG@V9}xrD8HuH=KvQa~{<ZF-b^k5@q%BOU
zFCkmq-j52$lS79*d5x{!y}FNLfF5->E)jF@-kR9;_o{e3UCP3vzu4dS6M<ZB{Rgx0
zaluo^Qgf^2;ttw{i3;tfw9OkIc7EBv%Ao(Q%;q{{$Q#I3e#<+cyoo)Rmh{-@I|9%d
z5?7i7=>ZwHbFgY2rS|&>EqID~;#;Pd_5Mnkv;p55LiI^WqJ#lRuTtT&XbR$-11a;u
znY@OSq`ptKLZtcXxPslVnWcB(0bB-bNqnq!PjHupr;SUftf8o01|o@rM$$zuX*|a=
zbqJ~(jBOyAlv*Rx8+Zh8q_Xu!3}ljOKDUMkJUv>w7R<Pm^jv*r2y(W~`S9I;l;M0x
zsRvL{Gb&%^Er}WVIrbaS?6=0}GYGc)sK;$b+O(p32)J@YEV%{$UOoayLKJp_)o~?1
zE^mn&GDi~;2gQteyVo5A!2zFnN68&_x$`v**2W?z@4l&?zmp85;OVZU2*tJlX2q2j
zP&dh|(@%5;=+^l4w_`Y`^?%(#v<<K?nX()scs}0275_=A<w`*raIguRB$-!POji*T
z_<oCxqcrm(d1jWLBiXxC{O$e9tV|V=ISX{N*q&X)-t;)%YUw?lp(R(W!1}J5{7nZ&
z&njC!Lv8N#d8KT-{}~Z`@GhlQ?(97NOR1j;?(2lo+@8yhbe-&)05-iOgY>3apfgER
zbp4fHzu4u{e8;ow@XyK7*l^$LZT{e=g~jMWN9^k&!Y0E^=QkAa%y8%4n+elzss@b>
zND5nNmD^Nz(b<d7dzmjog}-ogJ{~c7b}OM)lttd)OBC9OeDBg3i@KQ%4EYO=_;1))
zeH#lvJIo~cJ@SfP`YvO0mrw{lh#yo))PZ*91oM@X5dIm}AGRseei>CKqbgM-d{W6C
zVNoD#y?|=0VnF6^48EOVU*zgIRcTPFPowt(CON1DxKzRO=f(5^I>@P8Zea8x5kMka
zO(p4f<9>8b$!<nYC=*WpCN<5FcEljR4g0ZlHN71s4HQojB%^t~je)qoyURM1M<$yu
zc|C-0c@njx;_{J?%ei9ScUTA;ZqJ-xbhA~emVE-iU1n3bu&-xokJrNe=MwDj%Tzid
zNE5v*rdX$@0YkyhL%<+-@mg=WBRcxdlcIu2jLOY;iHR(OEx#xtyvHX1Y8(gtFnMc>
zB#K2cs^J94jPDi_OwT0Sn!q-rxV%cV9ty`YX;R|{kgq7<oNf<|zPg`t>l$kCP?mn$
zz7?SVyhuX_f1v6N<Q5t%h;h`;ts_b$b-2^dMNK~~1Q|9kanWzxh+C>E6(cFn%A1$?
z6S8+ody6*tg!?e3P`0Xu8ZP~lrcm<+jWEBSlEv?9o*GxPnDc@P9k#QQ-)><W>wATk
zW=r%;ULFP3f;9j3G+_92I}~vT+}y}D-Wa{w8k35AHSBKCzUbadsGo-0HYxc-UE38a
z6x)79O|C7yUVjo8n;HsC4Xd)Lr#k`eEW);S=mXIU!)E}L;{2yrhPl5ZmH=+>IVOv@
z9XFyr>kjK(l_uUtC!T}$xMRE?(VjzBl~K4T+x;0By}$Ay%{_ao8B6Ri*k{#P6<0P#
zZw8*1=UiOpBjP^z<j&uGtbQ)7w}!9uh@~0zkj%5g;kghwj_^jlr+Eeig_?=Dj0^Ih
zM8xXtz^&g~y*+o~?aH5{AK7qhte_vV3LCUPq`G5d?&3brVJFEfpR99Jmbo;rsWKb}
zEY>KVP2^Nq_qz#mW%u_74Q%IRP%exNcNn21VQhpPytH4I8n&>n0McjnofGx<CTrdl
zu|I9T1gI`J&k$L4_nfaY{JOS44W_E~nh02^-D5&Hw;WQecjZFJ=Zw8=CHap%uXiS4
zWfpk`qk*&VQpEzMHJ9ek=?3XjzofjYb)Gk?d<_W8NU5gv91{p#J-8bb4iNAc_xue%
zW79Y%)>@^V^Be`1kQbJ_VFB~3o0qpJ&zM=~^Xpflw$ZlG$)m-&UpCh_@BU1~mB%xG
z)0_b`Qh@{$0>A{FBrxo^l}?7=N8!?Li7AkUQp#K<<4~xb_7^5f+3k%cELrQGV8B8!
zf_Iq>4E?N&*n?4z%TwYvbTxBF?%}kKsNn?1fGN?tDYRA1ZDes6PQu$y%ilvds_5v>
z9zDY^)VSi}q8VQ+sA6Wgr~bv`QVk|~5F$q>oJw6kQX)R1!Ie1li7#=8WybT=QPU>*
zRC$)&i`>FnTmXqvaa!MW?VVh%8$1)wCkK7iq0Eqv+Y0m^xGhwtQeNxqO%HVI{yNJ6
zTzhYs`bBxF1r_kux*7<C<JJ>)`Yhw&X5n`(TYlL(0nxf&3n@MP++()3k_6ES)zX9-
zk~HreGpf1qXOI{J;}n>xk98zW`P@gRLjE9l;X0(5rN%VyY_@;d&F;9`^=aNPEgQ=>
zR$PBEY&$rdPg>gSK4L8!FY#7M-<Nc5fP2)0XJn~;;xEpWJ@c(ZfO5fl9AVJllGSLq
z25z{4Mb#lGs_(!MHduROV|hQzjtQt4CHdUK-lwsVZYgUN6cU|u15FEgIPnBGw}aeP
zNPQ8xpccone}6J#mRTtF>dL-=Rp#n`LnQ{zmWxY33?$iYA80a2uplfaD@A;u3-wF*
zTjca_jnG9el&M3p!$_8KpRSv~4hqNH@)%ZO&n1flGt^qEcGH?cOvEj8e7`jeav2bo
zbWC}S7#2G&dJnWPPPyl`KzOEL!b5xN5SF-<V^t$Z(o0t7nqCTtyN3x(_$nhC?`!Vh
zbgD2Sw%)aoILt_|4?N;Qf3q~gc6IwNS&o^kGb6#szx;Xycq1#ri46+YO^vK1=xU!r
zSkC>1nmf8DI!-OG9>&DB|Eys+fyVDj&qucNIaa}+P`SL0x<4sn5F+`$F`u+r>|szm
z<n%{NkvfgG^Bm12Z1l$4pylNful`4KaOg7!)6ZWqB;vp(@r$KLo(EDHuR1^b1!=oJ
zeE$%*Tj8qakq`RDKLD5<cf!a5rGUKr>4tOJt)mLH+H*t2otW2hjiMi2_U-ThG<;3Y
z(up117b<~y>xcv@Qeog1-YUcS&E1bp<gLowe{?Q-r*Fjn0}JiNk2!Yd;U}BPr}3FM
z)MG!?PD2t>g<CLOz!K@i7S%+4DD;xbgvBtLP`4Hg=JoP8P2_gAWEl9QB!<5;HM;a#
zN7MWgdv>t02X+!uhKecnw|+5%!zqqpt9#c%(ok8R5f7by7HCy0dXk`_8@5>+BOb@^
zofTgZ7kuJVKeIT2Pm3%aWvtjS<X~SJNDAJojO)<g;<S4u*-HCC3ziopvO7Z2IrE8f
zxz=~ORNdTh%%Cqh@51tTp-~)Zz(9rg$PCaqrw}EG_k+7<?V5<9sr@2<a+mi2$Ht9P
zPV&@N#j8EHI!JGMDQXxk{av%Qp<Tn-$ajFlv&XBJro!~T-XHJ90JS~lmw{w}r{X{(
z9RxLYweT;IjM`x$>)8SvCyrxN=NZ%rhMV0+411DZKjCj#3yXpE_-v=Y^PMyBqu+Y3
z3N~TR1tyJH>q-YDV%YE+o&?R`bG{I`AdSf&MsINfWXYgZQiO;&&zCXNf>h@#uOqz%
z^0MN~zlN;j7`FsPJ#<3*hL@e!R$Frcue6uH(_Ijnmm;GBpHI-NhjR1d@=LzZ)Xy!Q
zy|pF8F@SJp$HBbD0*P#~pA@B$I2d`OhlNi#)^xn!|8eU|%P6DPtdDu`Qnm#JbZb6a
z3F<aD8w*oA{L#>>M`x%aH2Ao~M89k;Q#t38V`m~-IGBAc)(yMJaqsPeaNAUW_?q=_
zp3H!EJ#VzOHB7d6vzodj<?&%@^n7NB$gf|_@NiBOZafh-pAYn&4OQX@pTiG3*ai%8
zZ&29WF8aw_YS;CG5%6r|Jh=OYafRZXJ%{?*4!4q)1vvCAdA`NP;LGTexJx;cj^xDC
zAF1E*3`Co_Ul!QlJv8eEs0kkoP_*-RAxhQgB^e@4dvi|jx2ToxB-(d9e>%h!ZUFx6
z7VNy>A1)Q+ek&G(`sC6{G&%17u3YymFm`=e2eE1%2NQhvn=dT1$cN9)nfo7P?(U1M
z7gBF?zaMe4-<xJctN;UL$5^(&3_ZYLC&MfqPQ~*a#xVldvMU)fm1e)F_#9-oZIY^@
zRPh<kw|b`%NKmMtt!>H&-_@O5Eaql!8{^{gtIKr%=Jwvo$fPU%iCyS;l7OQo&?|2_
z!4L$>!2u!(M!gv+Ai5@6orkpc2Gj=Vy6KZN_eNk#WJ*4LI#6G*FhqEoGyyFb_iTn!
zAG%k_VwPPrDQc$#b=J@SA??sxMg(1KIe9I`+z!`Sw%Frna`E2rSmA15r|R?~(E2u9
zNF>GW<KAcz7#j0hX8d3~*8dxMcxp<5bg6(z-~2*@E(6GveSGN?aJG%qnHk6&l<IXm
z7j+*WdkZ@1w!L}Mn!1N0{>~73rUaqJ@2ScwhGJ#CQf$61#)|+F)7~8WphV{kvE{VQ
zZTpHFr=jGL+dWMHB6b84%I|yXl<PkD7uIU@{_Wgy81o+5<V`P{{3=Og^#@ne%xAd|
zd|x-|ergjjC#)NTO13>b3vEFXNV9)|6ebW`B;~Y7TlKuM?s1<kQCZIm7>S19(7MY2
z)G<2o>ZK4(T)b-WjLz4ZG}E}Dk!ifU{5>(C_x6(-e!pQVI?6rS&hJ!Uvseh5o~#4e
zr<wONpf9d~A?L8Zcq)^}`+gkmLnEuw@LO&fiRDr8@G{U<7F_|VJ%dNv4X`+j?j={6
z6jBG<p*a>}%9Auf2bl8<fPIGrm*+}(Ht_Uo*%xtmo^@;`w3OuAQ<ty6Z49@1!SXxF
zj~abO8ueW9Cix+#choy_+XWvlput&X#EYv`9fB@EAUF+>baS9T+D|`!4{BzPzwf5^
z@$jfST^m;cA6MJXx>=(VR^4m=0~j3dhtSPavWi?J<eLKcg`v0H`4;e(R-+H1d~(#x
z>4`&CuG8XaMEHoS;yAELqV&xfzo)4L5oPt4ep}k{Yy=8j@>TDc$urz3B5OtNSCgO;
zMR?qX|Fiq4cD(y_Qhc;6KiGT4t7<aj-fvIyg3U>4Tc7K^kq93C<@gAFe1HXx$FOv~
z8LIC>43B7a9r?aFrxmPxuD2K=4Rra>PMhiiYi6>|ax2xr??!=1VG3`jEdPhC<VNp8
zvsr`qbNZ0fGk}LyS^03=cqbUYYG^5Kuw?9p!hoR4=L#S!t_zh0lnPpFR$YHbyfjGq
z44k=`nyiQPIvwDr6|ZmXno79y4{qCDSi?za-@6d~|55D$N+4fI9&jVZq5|#k+363X
ztL5l5KwBioSf@u|*ANtDr94c0WJ_h8U_tNz;IJp*w=!2HvUhyj7HT$>)~ll`iP4O)
zurA7oMMm|9c6@6nW<cUsSPwW05FsOCy_}(!%rUbD#@2N&<Q)&InKpgwpa>7|EIn@m
zJ&f2zW`4#rRZa1dsHJ69HPEyZFZxqXM%f2bE!7rKG!9PKMQKpgYqBKp_9HfDatlbb
zylLslYhJ%-sBSuu_q*PFT)`O4N<gB4IRDIBNdbLa@f>pO!gbyJk+A@WL8D-g&wlLp
z-rpo@!1nMnO%32VR2pp~)Uo(Xj(g<6p3y#S!EZo}qZ7CH4uR0KF^&;CYPNw)kewaA
z!Cl$>=;LO2#=;{Rc@y=9cUV>3Uf$%<PBu*RTa*SImfxz4(S{4D(E{K(=n`<hiu=2v
zVmst=Bpwp)E4JMn;j+xu8#FI)O2F{bN^e4VD?hPs6d3RMQln!jX0^ef^NfWC8Hq~V
zBw2}<fzXC{o|xw|to8CKQM(JX^UE;ac{8bb$$74w&<0Bkftc^b#?xG;z{EFZl&b5^
zq#){q_HEwrP{;e!eb-wBt`v67Ad?{>q(xnte?+dxXX&=!eU_e(ARZ`f^j%ML!r)w}
zYs+TR^NrrdekOKAwXm%b%93>=j;6hM<rPkT8%vGlkCXjLa_`ZANY7gusKUJn=fN8e
z0}LmHqrw|rQ*P+$k4zqxcaJujsXrTBuTK49Q78l`L2Ot2j*Xh*P&m!&^^5<+g#z%G
z5E(f7_{sviF9;VLCd$EGM<$0yixvb@%O`_`4zUSjFChx^$O3#ZcSqvI75rynU&>_9
z>O#Hxy%Ql78&R7f9hW1N;=4L*M4Ze4r7lfs1FoaH#4Nv5=#G^SM5CkmVB>(eOobSU
z(fHHmr&Ctg_X~?p)aVO0X`^HmVQpTH^00U9#9R4PIilyRO_u%EOsKW4NIic*49aL^
z(0RHz*MDO)Qm{c(!8LlZ$aynSsXj>y#*Zwi5<DJeaVm4()HQ%c3B{m=K2rxsC9>CS
z4%ILB9s~H$A-sp5cT4;X|ISpF6-+>RZztx_F@ybSoYoDXRj=Q+^!x(>YwptMjiMe~
z4WP}^co0#!pwm-JT;#6_{BnUet{zKFBcI%_(-|&PClZHNcERyq{zF}bRyPOngv-v2
zhEUwupHxl*_z98HpD*^0q667qT?gX4dVcspG?|F<l@y?$ae6S0KE@>T=G=3ob`M|^
z&qCds|5Nklk5Oj<4T~l2L(KuJdUV1bQ|T(i5k{;erz9E?iG4ZcOHe!p3{V0h*^A<5
z40gxYgMl&4xTg#0&BI>T8I^3Xl@l?{QTp`tz%W-g#5p>al&3P{M^HP&H}c1Qxdl8-
z9Rrme9Q5LSdLMWo;@+_#)yoA6+PU}1&%@i00#h;$%>_D|xgcxHt%~9&Fl^^Q)eJJJ
z{>iJsx2(In=&7|xFS)IJ)ZH_sK&|$TCB5s?SZVIY%><v<Bsv7%QKv3Y;POIq{(7<N
z-deScXJMTT2JJ0XcMCJ?Fo39i=kL9T5C<9@z}ItDOHpNbr0LG1eNH^iqX01*TzbIm
zo1M!`wK&n|`o8B5iL=<oZI@GxGSq-XG2N9vqy3k`IxyIe-*G*yZ&>e@2<9RP9>lIl
zB6lEGepKDn8^b*m7U5P}lXWeyJ$+=b(mMQKLn^S)q2%PZh-ht(%TJrfk0pqh6T0x$
zi1S0A1sQ3({_dWg(=fNYb+c#PARmhKxnx20#EUQ0@If*9*~$4sQw4p1tx3Wlv2OvH
zVz2XYj;=yAsu=n=dLTr+8rBo}kqCXdiZg~-#m;(J+u}Csl7VN}#Es<4(q76K(*?(M
z@zZZ*aXECBm@*I(Sd2w@A&e(g_@RI~K;e|@<|ad+vp&OnJZFGEAzngD{FM0eZoYV_
zK=!w&1dfgiCEETTX}L<D5tMGOb!G00<$ZA%%D9bW)fu;qg3$(ZK>YLt`fEmHUd10m
zX;PE<zn1>aDaNq?vJucvGdA5ng9C~;uv?RK^Rq4~9`4yTx@Fy&&nkvqnEtrvHOw)m
z#G)})jdv*75oC!5wv5ByM15B4Cxw*&Ee*q3hvFl|n8c@%hy3k7)nUSQe+}yY`C+Lj
zUyv)~X+-;r^mvUB!x!|;iilt4Pvd%TT6b~;j8ukN0|cLDA!lHE22f4QGZ^{{pX6X%
zCM)6G%<+2zB35~k+pGnHp}}4bDOz<wm2`CdWoAGF2FL#-7Q7d8QwU$s+pW-`(i#BU
zon?w|+rY)NL%vooS7V}gi0)Fs6wXQr(6w`A^yxUSw=e>8agq8IRH#C;APyEFG3G+c
zzbsX}PW8zhdh^8YEX{wTlHsHfRdPc9KB8u$IN@dm0G_A%@M)Z-kmWCwTOe_=wG-mO
zzHES?6-l%!Fg<&L{zL>4{l)ulQjG&OPz3^Ck21O}Oufx^y?2X~K-A@nUT^aqcHUgO
z0~dyL5&s$hh1f`YyQhu6G&C<-Rk?ehKRF=9^Ly5L0HF}JaN8*n4LE$hJ|Z!J_e;vd
z#6{lC`%mQff2=T;To<uJUR-Y;DfsxFve?=q#tk$rSd=oeM(C+SN{!C(ooYHi1s~KP
zrSK6UM+%^<S6&wX?%NOxoX4cJF9ZCIeDtCl$8KrXV)0X?mZrp2h-7#N(OHV1kGM4)
z!IU7yERnYCl#PnLm4ZWfV8=KRmqcO7c;CZ4xEBWn-LWN6Z~=<?&ljnA2FZohkVK`7
zcYMU>MR8lLePn<cPgNa?`TbhB_-VbixnngK?t>nfFj3l+HtlOQ8*M2Mhu+#?$$>s{
z2eykl5C0EVbVNHJ64(RG#Sb66*c!U+5InFQN<5k~af`T3bn4l{ye|7&3fFH7yjheA
z95LeoBNoKRm&+`#08x&&leV8G{1bttUT4|?sJ_{WerL0Wy|;K|`d7&zX)<yD(gFbg
zPa+1IlI2rIpz|gsIN&cx@4Sw=12V_>aha&lhf$zG3(|R&X6!Jp`d}uORF>X?7M|zR
zf>FtyU=+A?RE2uSo(5dk4|b<4&Jl79Tq!RK!rh=+kE^AZX81Qg86I<&D*+(S0o52c
zOsP?7?c0MNSW0AZ^5lb61BSyw%jXcH3KmsIX1A;7ASd0yDXW6%jMpJkimVc7RVj&s
zlnx8S8PL(lcPHqnWZJ4XM}<D;2$h~Z@uh9e)&{Own*`;_B@T%AR%w;q@vLj0dlEt1
zU^yyshBeroE@^k*PqEC+lwGsG0Un|0K&oV6M>G=XyxjV`MEg60|C3k_x{aml0yCt?
zmRR?ELE?;WAM8w`?S%TDFi&W~%FXR`S}=B1_0k!TrJCjksALXE4))@@Zr?p;RwN7O
z8STQhT>HiI1N|~G3F{aWCbRwqNqi)kVa26Zdim>j<5leyGuRFzvLCe}jFr?6uf?c{
zacqf0N#3jIX@&};@#i<1G+b0>@YyW_YEd(uaz;~%#|I00ly+VUmfB8(B-ttDPatF|
zf_^&RwnU!kKIL9Q@R{hG0;VfedNEdmOcG1j{Wf~0fHPK|4C()~^-!`YFWWeNCJCtX
zX8nN4gV;j^apb6LLwpI~Jajrp5>bpTu{FX(<6sm?uxv@Dk-j^X1OF#Op2aHZzG7bf
zO+cRag8aP-f^;5@?ks{#`UAwfFHtL<&)catzU_iV>J0xgUu>B7Y|<q{C6|>@m9E)k
z7oP*kDfD1AWnSi(efiCkvpEs(lL$gUp*MxjkHp>!J-RrB;yu>TR>fe##r;)gxK<qs
zVBiTt9CdkSIUz2nE)kcQ1Ui_4cnrT3vJ@u8jfjiG%)-x=VcBi9ulz$(b@Inf`(O05
z+dMi4G)!-|35DUWum78E0;F(Yn~(^M#QD}oka}L@=AaRIfcmHlp0H9jrS#y)WJp5M
z_AJYM&I<@hGD7@c%?zXjLq{p3kPT@aOZE4`nz9O1BfGjvPQN29FzNO+kIR)DdYgTT
z|No>fd%G5x8ZZAE%>Lt${O8FT#Rc?O#^b1xd2<*&9}G?kKgrrJBO(g%1u>UtMLrMV
zD2?}NL7z!t-qL4FlU`C_l+BP_LGO=Rv}FerU)<){TX7+X-p7DC6cvrIK?(n=*p2*E
zFE(AjUS3Bz5#6bNS&&Wwk==#eeg0**#t)y6TrNZHL@;3#sZ7}k(Q~a`4~K#|^MX<H
zjv{|6dAbhK;jJ1e2ve8kBvTie?lrkZN0AfO?z<gZ<klMg4f7mve%r=#XrEGj@gLf(
z=gbfsxfsHV*KP-ngVH~VT?TIPB!4KO-DX7Tq%VR0$cT7<W<<3$FZTajh5z=a`Z!|X
zvMWg~A0yKXmHu(Lp_HftOQg)O3L1+QlA&SNc=c^6P1=J#+4>T2MPI@2AUI0UGJrxC
zQSRWQ`vFb;m0=8_gs=@Dm%+3E>(-c`;;+H^yVKHVDp!X2vz_J13I`je<z`pKW`Xbg
znwmd4@^ou+D+DReGU;>~lwq~WaADvCAh&O21$8dBARr)`OBDqr4#KPxrz7(=Y^o$(
zP)c6t<?mqwHh~UJ_s&LtVgxsmhNBui9B2VR+iPl?lNguf<6?mZ^zjRT1k(ZN5&u66
zDL{yg^KGLCkN@r^VdU2UaY~}MfVS2p6MLB6P=aR%oi`|r0>2TXFmd6y|5gLFw7=@N
zUJpkR``NOXo%VZbb=v5kC_+#BY_tga%ko-Hx4<St5+!}jRc9PQ)qEQ^O6^1&iOt70
z4PxEehFC|=><2RcOFhtXC8hskLC}l=Il3Yrs7wL%-X9(7KZRcd@|C`~|4qqg#0Gi*
zfFA54-&V@Mx6nUpo`0)W8wvl3#+`chmqaDNmjWaz&x8IYQCSdfiD8eXE#(8mz}AvR
zHL`F)mndk(lPjm^W<_P9yGiUF0GvIF4+Xy}CZr>}OsPybiG_IkeVlzS(4UbQ^zaG1
zBx+-4)A6~>S{?R$hzgEAp?TMC$xt(r_rbju9qc06Z8Te@cA0lwda?Y((S?$bEwBZW
z<{F_LYz-H!*dqB>zpe%CXHz_~(rmTX;-^CrFGor%iy*-VAKXSz>;I`x2{c5$vQK~f
z{&yJVm6iGR7=T{l<sA0?UBdqtb%;Ovu<0b|&+yCk1h!UuAJk2vfxGA{N$o-LXa~41
z?ii_ZdH`hXJZh13mr->aBN+A8+<VU8Y=bXqlVmp5Xcs}Whi!89Xmapv&*KJLwXE0!
z90+yBkj(U~5qZ8pk{D3kZdILym8|SCU^&iAAgfp^!`Fb(^bhnXVKkl|T$f#KiV)Nc
zUew}DJTE1V3GFb0&wm$KT;-<+1Qug`aSw3eeM*`@?24*OiKQ=ka}S5pr4>cSV0L-t
z=9a?HUq+|BIwc>hEcsYGl<iOT&j0JR!OGebU|m6P!5ceZ*?6S&HH0#Rs14+S+h>+Y
zXm@JMB0-|D74Cmokibub!rgdGY;sI458rrz(8yl05ylHTuJb>GJzk>1a6vKM7rw<+
za|-*M$a0zNRnP@~+XEMQRDfeNC0yc28vI=zJ-O6Ju&Lvd0<M78UAPt4GMWFW%SVy8
z%asSMj3e|fnJaU)@t%s0|K?~&Pp1G8TNGy+Q8VNWRG8U;>ukS4XiIRyz99xPmh{>5
z)lW=iP8ETC478J;BF$Pk(f_=FNQHiJ`TMLmaQ+o;t-S63TY&w~gjAn;+gk*}Z9570
z>vg}UWjq5@1f}IL?=GV|g($IezqVEk{U!SI*p8+CM_Rd1g|rfR)$G@r^qsq0K4S){
z#|az|J0j<Xq}+|c${E+dqT=fZ*ih}V<Dm>_Ijw#S)vj*cwetLcA^pclMWS0wBdUP(
zq_hpVgyGKMW9~U5&ODbvx9A!I3yYUjrI2PPO9{^b0C+MBN;Ki=`)LLE9$0~ReqLl2
zXS>rck$Wj8TsHE#7tt2|8Iwl(z_bko*E|U)G!X@YtpYm&WOzFT1A@ozJo)b*{t;7%
z{){Q!Z(7;^UZ?*WQw%UMeO8?l>gOt-M|^zyh5YMSfDn&SBV${WWzd_>wVDSGU2FC8
zE$uz4->*&36zjgU_qy7=B%EIj4$N6IqczXGQgvFk&u-w%y<CVxlo@_^=w#$EZxcQJ
zzAVz#lnBisjt{=|26#|!(OK^fh>PFUH23_ZNg(=^7bw<tv_u2A2=o`e{0{$FX72PO
z07Y*C>s&q4+MWNX`G(<&<FjkAHs_bsLumPKw*5W%wb^-lk!ST}jj&ot(OaGO0b;RG
z?CIFA*X!u+QJTc>qwPWSAo@@j(mD_wa%#OR!@z=RQuP5UrRIr^_q3v>xKr7aFbkwh
z_Q^YWm?Tc1{F)qsa*tTaXP0HS2&QF!#Zl+)d>p#awiGS@vA*GW$`PP;-)YEg&TTr|
znth60(9HqCptMA-$yl6i9W#a0Q<5n9pwmeKS8!0iWEdof3?_aKNW|BQ?m7?v(%cVX
z!gDWTg87J-NY2?3=&uZ*{e`b3am4=+b^?43YjSQi!SMkFhll^Wo)I$de;NY+_|k}k
zn5uIg^>B<+^9#+_u|Of`rfUPMpDT-nZmb<EkIoM3=MYUj7($2Ok1nkNMvV?uMlM56
zD#1ftX}xXdp1qm#xtSyKIADVI4d;VU31@o${xPlw$(Pq*zGL9`JLBNq)dnZ7&eIE;
z^`y*6*voh-3WJ2LXnTtwUiE;((xX~|-rlhOSY|(aE2mETrBB1(8@?|>7V9Jnf}Qr4
z%U-J9T-N>8ZF}lArs98H=79;Y*)07&mI7@O-*1B{$Hf0#-{;3Bz$=LlU0MSp_=rN0
z$4q=-NDfa#B%PnmD{&MdN*g-TXrl?;;OOw0twk*`$3%2oYQZ=le&56#SmH0M7#g@1
zNi7b5K|#1bjm0&BJ61EWH|!H&z|dHLPpq>9dJO7w1`b=U7Xr}O0ucMt7|x-k;yMZ)
zqXW#V3rNh0;CH&}&<af?K18g3_KQjCl>E>r*gKU(2g<?6X}8QLv4>#m6oz=m)ZwSz
z!i$@>(@o4#E1r}6)oL$qc_l#fuC+(}KKMJc)hPU#+3r^!j{Yw*TWDTF*Kq&?1AV~H
z;|B$+T@UKrZt6K|@4gR7Gru?`)ExKcs$~_Du7>!YJ>aR;=^YVlH$am6|3Z?UQA9p<
zYrA_bcs_}xtiBTD{QaOO<oc>7O!<}H-kHd!7(sjro;%O01zR|7uD^1wzq;~?ewCxR
zUfW_lJ`#@eilhom_=AE^o%<7_<M_yGecaD2f^Hx_rh_7)$<X3{o0z>h)x-K=)ydz3
zTh=c=?1tFild}|KLC{i5NK4W4#P*hA9<%IlMX_2hBa*r(Wt1xnW2DdDodLS7WahF@
z&Ao;HLqa8DUxwh7AUdoYx#j1<gcv_(m{F-_Kpuq?!-V?{O`AqF?40jWV6W<HvmeXT
z7wXbFkq$AxRvv_>WW3g(^2ZHtRv<+paEzeS@w|UZsE{Y!H@L5@?z{!CF6JmP?`>Fy
zC+eird7}og3!jwG^xs!3VX;Vc>;>Ev68awsLmky;2%`txff%az>;I-M3l%M-i5A4)
zOxIBUuAKLj%CJCt#6Z2>ps2RS%=bXRH5^grmhXNhm7wrBjhPx&u_8J}bLNtH*`uhw
z30J9Gn4r*9bDEh>?{KSn-P@!~j~C!}K9Op?tonq1Hn!`<;{q|Bp8=00zYfT=lGf1o
z+I2^y{-E{xaLaZ-rnZ&O#3v65T~gW3@HyA|uL>6Ha*SK#V2hcI^Ll|a_OvHs6dipD
zT9Q?D0-POm3@!^=Fhb5E!GkCG@gKQRS~z=Te&Sr9S=m_=#!{ODwC<%oYw=sZcUI>u
zwVw-9d<8S$C6Xmud5h9IpOK#m!(i_PrY#cZ7Xi?_;Vi!*I%*7|ms4!P%t$RZND*pg
zD!v|j_NXP|lOo_^(Nny9J;j<t>d<<?c^dy0=noKzlffrJv;$uo<5qrr@!z|Be^tJg
zSTdHs#(&~I@1M0h%<?yUBlLZ|p7E~rmzb)FF^eMus)v^D3(r@~*T28KsgWr%{QkwN
zx#45v{`XGEL>oU*-B?hmgNe=V+EF}H_@@y8AgJNvQ}StWBNSZK7l14SfK~Znl;!~}
zfRP3NShR_R?3+D?=9sLTITMf3GBl6avIq~mY{m6+!-f7qJ4crK91@3x9>>R$Mxog8
zXr7lPM;OGum`5PSS;ods156s|-n+x*5Is#Axu;uHUKM3TNzxu{Wcf1#>~+F4U<=wt
z?%HXiG-n4=vBwv??i?MWcN;2?<a4)!%UAmhCBFftAWhor+U?d3v`%iKrJFUxmk2RJ
zxsV}gmwOKZ$+D>lSsI)zVnWGhDMl9e>XDe3b~F}#I+#-D`cR~;k~5O2GO;-@wx9JR
z`t^odzTh?Sw^{xrlC)Wd*<m6Y%qv<NFXF{YHw>H4Z<Gr@bm2Y3%Xj^)^heNwmmz?G
z_g@NrqUgMT1udrX!;$}W&>{zg9tXVEEY3OEYPXtL5$QG9cO!o`nM-%^*m~4V_dC&Z
zLNjj32b<UR%Wppp@@7y(&PG<JpTY=C#m2IwMRNQ;XI4O00$$?Y<Xx-p0@~NC8)LCS
zk6&RW5q#>@%y+*bos9I}piuvm7r+Tv0~q<*e*IxJ*^)fAa*n?N*Km6^Ye-Mq9DR=}
zFZk^B(ZVPGS4~BS{UTr50lvXcB9A|!)YKb#@Ph6w**(gFu{I2amYN2ev^;AlBrL-7
zTk5BaTGV4xgoO(?6|2!**Hg}i`r^7oUC4hyj#3`PR8j42XkHc3OC3{L3!f)Te{)RX
zU=tW(sVwe{DT9nc68NRgweqK)8)I62B8n@-F7MWb=gwKGlvZ_bJoauBQpSI1x5V_u
zJ#-lG5|LUGttWAe2m*{r-loWXY%0wa4D0yj9acWbFDKFQfp|+^K*e#8h;D=y`Erbn
z=3I%AspBGc@@DdjcZi|B^Jf?IP*!4P2Q)cGuS8~L(mlo}vROr8<N>|rGT*SY_4J^S
zhIRQhLo_VylEuU?0qn&7c$msetVfesqvDwXo}cjXCVoncjnxYMHeFBk{wXt2lVyot
z#Al2iR}g#W;rbn!9|xU;GMi<IIi;#;$<w83`@#<%Xx$}$Cj^0*Z)|LAb%kQQK+bTc
z<@LxyRcz57Mr=YCpDqzB{QKE*!S;0l{=zTYe*g2cuK$l`O=q<0bME)_1Du!29@4?b
z{Ko3~A!2oKGu6AD%!+oa$AOi1pA?}tzf$(oFMS)eG%&m)$k0TtOH!J_GpL!LxHlR+
z%~@d#F>=!9DytkNBfqg$ENwxb$WhTqFTO*jTdRk+*l<&xs;B=<P8E9yj6XN@+02CV
zNImBjHV&udgqRCyprApK=HTcEU4UJ%*H~#f61_PO9lJmlD@zI)Wr%ZprwgqhfufJX
zC{zdu0`7bubXPL9<u`(125b1xFq?u=DmIN!|D$nKV)ilVELDecT&Tcl$P>oHP{H`U
zd;x*S46i6`mU=1Yl>@5t_e<#lEyqxmRndYAHWP`(5q!Q>J5i$1Iiklo#O!zScWsK)
zLCN7If-0sDjduk@O+~x!8*%K6K+Xy=sI9w*Y}g__$ka$}<G|Plh!?JiPzIDpHxE^K
z3q>3W?3iiu&GWZXwo$UH4zg>Z9K9>J$yU&zb+C<Y=c{{$t}89aTk<fXio?<;X<Yir
zZ=YnV_Q#O-uQK$%oMs0(6crWOUR|8&AkZHF@mlxtb-A0U>;L_rUfKedloT>QzKgE<
zqzPmE{bWvdA(lr}2Gj31bzku-4tE#(PUJwYR&hw2xI;&6_YZ|m#>xZtxOd(+s_iGA
z92LCKDRd^mJNl)~59dpv3%I-8`upY94|QkJY_xwrp~3Rlpi~tD6Y)?exi8yO{Q|3i
z%%u!q100#Yn1rG@#NZAmXJ^`=%pdpO*dV*U%nz2PRBxr9NMm3T$auaVkx3Q5J0t6k
z8*JLe3ep8hqDKqHb>pysa84O9G(ga}slkM}pHwW^#zR&+Pae}BlhSf~E9JwF`ca5*
zLSt(R#7PLj@m>Z5Sm#C@uqfVGil@HW=Q6i}a6VsJ`F!!X8l#C7VtnX=tjIDZj@W$-
z`k;DWJ_=s8I~&#r5lUExBZd?kVe9F*pH4oW<y}PB7^%EjM&CAm5LLbhH5kH#5rW0*
zH7u`YvMbbqY#k>NtR-hB?NC0onwX6nJV$Y22tlrxAQwk?7BWeC+Q7t!!Yd>y%N~w<
zv!GXJZU&GCRq+T3$J4=7XqJuE{dX2S0t4WITcu1Ecb|fum_J**fdBjD)l+luP_H#5
zVIj!Ug`Vv=3RwziA%xcU4NcL*x%V8|a^lnb7D%xIM<QH<O4TwF#l3rGaisfmKHMBD
zeN}D5sjM?#VK&{_Ia=QgW07*2uWjICNr3$dzWyLCrO(aHNZs|L#{aAm`GbOnhX11v
z-O$_^mmvq&kHPu7+Zr1ilf&eJ!uEg;gvCs1)B{OZvJWr&=+A99D~Bwg7e^5f_Pz*n
z%qsk(gK*QieNK;YqyBqy{p+VayMhoj2keRuk1GLtmLu1E5ZSCZh}-v<rYWc%G=l~f
z+`#O@AyE&9`1XMdG^HJ=699Nlh%~X6Oq?cdICaLcBYt3CC|^_u0*Aq7h>C%ClQf~w
zhzKl5d`gVb_vTx%9rQwQGaFJ6+aSh&^iJI?%TB&GgtU-f{Z8P?xC*nFI?vf!OduN>
z{F`p1Z2>r7NbDIt{c!3hdfqPfII!97dI&)fW3LKh0z0PQY2jcBA$`>GR1Z-f<SJ3q
zod_D*RVL*dl|VZcFHk5?Q;S3H04m7z;zW0FnytJK*kh%Ad8<+*Wa#kg@bk&<@13K=
zDeRv87OrCJYRvcr!!>cn(*IhD+s_Cwj5gtSpsY5j*wFi`s_x2S>_ZL3FjffP9QAVi
zc*uBHX-a6T*<rF1@9$l#12ZY5xyj4j>8C4-<^X$pe#oWSJA+vfHN<8o_iIH+)#q>}
zzel<F*X56J;PpQ<ZgrT#_FIqeaCB&0@Z9OB1CiAZs^>~br`Oh!Q^XN8I_X<{@!d3+
z$|PmiP`kP2w|#Cc4W(jwjclnD?36l8gdX=csGom$CmW)E&+N%QeS%7Qx`^v=s+l=g
z;l|kN>YR^U&J%I|efYp=(k|=M=KHdc--qQa@9F-1{{IyTLdR?iK<ExWRel&U7zq4m
zQJh9bYjmyd2uvOj_$W%VfYu1>1hElGCa@MOiapX@Ob|E#16E#42`Ki!cYh}eL2P$1
zmO%m5^t%|sySk4}c0K4DyQwZ{Kj9b?2R2NeTgILBerepkV4M^ba7ZU4#88jK7p5GD
z_I&q1mf@A+(Oy_4BdQzC<U}Bn&V>;J+?Nt2a6v01>_tT?Da5A1h8t8UPkn)i;9ru9
z6!4~E@Ml|#u<7^F5DW~#37AMbriR(<zBw+OM8#p5hluy^0fI`#aA!`@wkd#uHk%>-
zk(G_@F@{yWzAhBnhKY@>s-=~f%JWnw+!fS?hbt4a!pX_067C@Evb4q562;QN3cU7M
z-@DEZ|5&R~^b|C68sOM}$xR-ex&C5SN5goH7OIiB9ebVH9^fNxv)n+<p-#>5($A%0
zvA8|&<V$}Qt5f$$&%-Op{3?l%XK-``G-pR(9AsDdD7kVx;Q3tl6MWm73u$$a!UGIR
z&v~=?w_-QH4+E6+dp>8p%|R!XdGEgNPc5+F8Pf)*klB31i<Om01Il`K2UF-69rvr(
zikRb)i07>Rn>Q8%vx!;jwRcpCA=0e*!TDlGga@7<59XU1<Dk>}ew<?GZKjLCv9*;c
z%O2Iz|M+j+Zi3KYP3${VA<qg%F3r6y@!8vc?R*SI*MBhZJ18y`17wbdxdI{@au3nu
z!LX~xx&}cnWYr2{kmAdaNq&Loy%9K>$Llnhbf;E>C@Rt_kQmWLDWC5f@Tm+4Qx@ib
zq)7LZshIGJD?NNTg#J<p;)TGO;`iMR7p469NJt`>(On~&w1j~#tK$R5`v^Wt!hq8x
z#uE_PdKeRItPz%Tk#{Wi7(a~eY|5G-EC)hO3GIuZ$7RnHJQ8?Id%PZgTmTFAShR=2
z>wl$M5kQ9HCjA8Yych?~-1M;uGNX87_^?inkCP^HC92-R&sNe?a&mYie7+@~?az;X
zX%F~p>qxj(^=v$Ay535@!t}!nt|q(D^x-1KSe;{j>~Q{Qo{+EBwyOh;|BUnwe@6N(
zpsd#IY?yhzFvA;yg=i`51(aH_oqioM6T14KJtX3>V2^!VX3<@o$jd+YseJ*Ig)}_N
zSju&s-xqE3j;_AEeob3f!6^FX(Y_g{@yr?O>q0|{p#6lP*b7#>fNH>1Q^k}vFYT=+
zg-l9N5FeJKE68CCDO5gja!3h#pPI#In3k$(UR&gHM~*zpkiLdP#Db=J=e-az&Z{#(
zDSgf+%>~=}zdVvcT3w)r;WL*-GajmRzJr<t1V`vySRq}2tt$#+1Wl63733qSH{UH9
zq9onNU4X@)Kz*@Q+)v137iBd7%lWZ(IkvDim3~~2K=G4N7YxJ#-i_XLqdP-gFe&4s
z?r=JyS|xAY*N!-WOS<59v?O6WsVyMB^x?iYfdkzOGfMgRy^`uuakBNH{CxDWk@Q!e
zVEYvjcR9j{G|>;H!U_@~mxYs<#o!{^8qS@BJMcBn_FWso1weT-agQMoDK`Mw)Xji4
z#-RnLdP*Ik1>pf(wk$fMhe9JLDCh-XgrW-6UiA>z&u@#x^m5}#3k%;eQtGq{!%lan
z<PFO`Xsae732AHrgG{#lqSN!_A#?{m)e9G`e;kTX^z^^ZdJdKIXRq-l3>~p%P?jEN
zL!4YH8h(MwjctUU3kNx}aa#5F{1u-74!q1QadcXZL`Jt#uRuK)iH@Et9c(T&>1zN~
z%3}s!YdcyCy^vezkFGC&$USXxGHDHR#>(>gq{|pV&YhkF=*ehIy7WexA)9{~rhk1Z
zK4h2<HB@r>TuOhpvg{QvwsX>(PrhWCkBmi%^T>X9O108ndX?~)(PTWr`A{x}%p936
z^W;46*p3wauopdKdg^Mn`0vZY0>Ndr(3*v?0yO4A^@KV?UW*ct3Ar*2WW}P?MHW>?
z1$&5P<go6fYV;dC+3K5+x*N=De4N}rIBz(r`CWozdo&~CeURO@vvqZjM1a9C>1@0H
zbFqrUipvRl()V4mzYcjz<svMfHdt5rmt_{f9ouz+E<Mn~RCT7snx04I1Hm^oKfkW^
zrE>E75sT=%*Do%-+FZlK!yC<Bydc5D!_#iN8}|7j>X5h}=V;8i8y(fS#rsEznUck#
zn!Hg&^Zg=JsVfvhWQy3_pltyip*>^LW)SZp1*+0dV7vMV22>wfFaj&H^_y<SwQo3@
z3Gx9X(0$BXCL<8p#jt5}$cVydP-N&Rz^6kMFdUWR<n3YDj1=~MsH~M=^Drk8s-hDe
z;+(>v+!XUbl@VPevT&|-W83h&v1yd#P#R?|1dFA-g5W}F^JGYeMA>%*A%2nqIXK41
z?zz*+6ioUZc`+~9C~%KyIXItjH|to`;=&qVYN@`I&sH?1s(7gsR3O{*2C&YPz3SDG
z<sDY^1vU8|Tg#TdSt+pYk5QSgHM`Hu+%ryBiP$nA5E*<*b#$NPhGZ<PLjRxlgL>+g
zEy1qmEI_f6t-E}kWqy}d$3ScQ`10ISrv$eHgSY{xR)(9;vZSf&L0Ol>o<7B<F`%rk
z<NL!yNx$||`}+D+JA-=PXXo)O1BonM*I<{>JgB5-<wb6l@o2o1eaWDd#Ji@q4g3a4
z&ujTATnt@eGC~s$M$+?F9tfg6jGgZrahd!!LD)P81L~iDJP%tvL$JHa@VS^&@S<M`
zLuBvTY?kU7VM)bH`^d$+E;2DSRjJ8jPQSQVvl8nNQsok^VBble;4V;4PXFbyXfRj(
zlqHEhmV=?;j_4<yyQH8J6)jK}-E_IJV}(&e5%l*#v-tkiMbkVvXJB9;83&uj65OR4
zBvfn7?2Em*W2hHyS72Ie>KGj@R@3!hzvM}FWGk-;JQa?QTh2%$;;I*t4FW3pG@$Zw
zy1~FEK&)k|6_$sYYtdh7{}B!Evdogy5v?&F<>g~af^d0lyNC*~NKn?u0&pA1_(Xrc
z`*D{@JqG5wJ3;>1Cy@(AlnhYbe&Qd9>OT)lfE*KUai?#F2M*Xk2bdTHJP<g>yImp|
zw4^jZ{hKd-ikG>EFB~U-gBYgh_5jjdB)`Tz(oL%CDG^HiJTX5u94OK{X>Bu_Posqz
zc3m+laO$T6q&qo=R#8bj$z2j(46~&ke$<;-PJ2nDbsWTG);m^FYN9&_$HpqReo%b4
zmkK&yXvPi>txVv}>3BpEuf^d!)99#s23DZ4UjojUpyyZh#b&qs-q9M3A796}E30Z>
zc*2Ypn32{bx3`(`esK)p+JC%L8QBp~-D{o@wqKK~z6!jKOFiD7+npArb?RhEIS&~M
zFn}F&asHE=gmIf@HZAd<{#kfac$M(bqNpT>wul3giE0;mB6ii3MPG6ZbDJu<3kY_N
zl;Z%Fy}bS=CA7NW!@IM+O{e9)dyF5(2EA#h8R&}jW6Aj0$nRC~BajM1!|gcRyKfJW
z-K$BbD=owIY-21X29H!U)KiPIUyd9fe7yfO5G#Y4i$YHa8nZU1@3I@Xk(wj@QsZ@C
zsm_mqyJQqx9sQSEGad3TEc^Y%Jnxdn>gtl*SnI%DpWsz%8K93w3}xyzh_esAJWMY^
z2~`mCxyN9m)JV?AnJOMP|3oJy+}(Xk_wL(fo%ibHX7!RPJ13J%KEiu)_^BA&J43-O
zLC=EwS?#+hG5(xSwpaCfQnc-D@K-a{T>YwPsSEN-+e48q$Q-%{Wf|KL$lblUn(VPR
z0`j5jN4s4mqkX^7MJKk3f)8`hCo5Q_xD1$$0#p6gSMtQ)>IYuMs4XI&+y@9wIgnPR
zqL+0UEt_BFwMvXa$c}%!lzS!8sR1fNQ~bWj_&gXopVZ*_Q@wh`aKuN6*SHa3BKm!`
zGoOpHiI9M&wsL7hATCzx(brI%i{0+y&9bc|B}jpCvx>^F>#|s8&*%(gUH|LVI!=iy
z`ng_VsT1si3f;1pi=L*;FFH=g8-(wm=jjAjZv=XJxs~5P^DcG$>@Mc}z}TR6h3onJ
zp;jj)#6YOY?1~wA^HLIpWA#NTq4nDJ4k6)`Cmud`8Q0C@%7gdw5`r0}zogaD2@(xz
z_$U_&t?@%t_U8@|<bE2tsyE-2-RK26fy?a(x#3K5r9Su2dM~@}@1N}#+pqQ&BSora
z+K{u{jtouDVHLa|CWa<*f6wAr&<L993Y@(xn2hLbeo{A#T#c#jgB^rLH<d!ECDX~y
zkFr@#j+--YKAO{*p^Ei4WzL`J=}W2V@WYT6ABV|yp0A=ioILt0{N~@AzN?c91B4*s
zJgn<_V1HjD)b5BR_#F}_oqID*<mM>W^n9)6!vEJ=jQ2*_p<6gv7n>NG{<CDPlc*@%
z3bT)JXDRCPL$}%XkXB(019AuJgvZeqGMpa-+gd7x-^;BwGj<;=JxVVx@0~7El)@u8
zou3~8O^4h~UA5Ie#wK@gKX|?GY$~nvm_E^ImJ3NSKUZ(^@n>RdhG-Gm!+J&Xd&GxH
zM-X4&^6Kw=qnsNGw9q~%jz7?$^Cb?Y;a+Lk>?V2uZrQ`22Z1f!e-Kg==_W|_&s{k9
zg%4yuz9``fl%Wh9D5SF%fVe&wLpklD*yGV4GW9qKyqmgU@tH>^yEeIv^3a9kgb!~}
z;Gwk?9B8J;9?efi<cG(#(iOdPsdla4!4HaG@ggnM31WpciF^@4!=|)bZy_mnXxL^y
z<#hlRpiM#lFiaX=x!<?5WHl(pxGw4VR_d!&ts>#0DIDy?aLYlz&1m|~AGUG>zLizp
z0t1o-@&N-o(A`hogFIpEpVuSW(Vqa<GXi=^qrh{dF$Mi4z}3|Zs{ynZ#J!QMg!dl0
zuMGD04+F;a#n<a2EArZd4>H96=%;;1)!HBUM=!KM%l?mkTFS?=sX8N%y}A9z$NHsF
zn*BM6e(&F^{Q82y8^RX;(Bf?e?enYCK%GL5cD*c^fm>>z+Pw0o0Jph!;ZAKw2b#Kj
zp=b|Oss>@cXBg+}-X^wFyB?0{k1jN5i$R37%aaT^(tHQzLFyQSmd|yzzQ@{bw+tg|
z?>47=sZekba}@f<gr^lk?$+ps<XJ>r@qbZpZ8NuM;|0Zq1O<H4f0Xa^V*4umVbp+L
zpVXJRJx{ydEo!2VPA~(Vor~v~o+`1{V1|@Vb8433DR!L-)2nY+I_ohP6{tM{_;?Qg
zKHKH4K2{g?R*9DadEH5%n!)J1gaLcpM8Tb=!)1m?zUA%WhNzRN6wt)^kJ4XKi`4Pj
z@Bl=C?44aL!y7i&);%_}i{*#a^JnH2l``C49KIvP-UWSC`l3Kow_Kl}v%2i&c9xdj
z+|FCdJ@2#itpyrb9eLQ<%<bl{GBVqe?3C%ml5xqAy%V&z8+5T(h79!-yIMm5Hn8ay
zLFf0)9>8qgDCOGceNaS8H?|!HGI?hq{o68Z@i;1vk6ycozN$LKSm7aj{<t4pET1rM
zxug;&Q%VVYbx%2!VsE}fu1?v{*J0A?K9xbek+b^ubAPv^`>)xYZ26r4T0>>;FpB+t
zv@TIxrLf!B?IOT--xk<s!mq6@#W1Dw1BK-~e!F(`3Q1t#_#|InNtC}hm1SG50Vhr)
z3T<X!21=h%U$8;7ZPyo{ryCuSt30Oy`P-u+qiXztX@_tVz_a^pwz$L%zwZEgB48-U
z<{<OBjDTO$+<#9)6HcEm>S0{-3#(4hP5o>#(fP8pBMYXsq^qC5SX+s(g@x=ao<5JM
zgMbfGUoY#1JYPJ{G@}8EH`lM#x&ID2A>%TSsd2v}bOLvE)P2U!P24qu^FZF-vUc&_
z7NOZH8Jq=#wd9{Z=D&4)W};W@@j>$YNew|aM)^HVgb3)=xtqHZ(uM~H&wqc??D`qG
zkj86bu!^(9o0E9|yWStQ4Nvh58X0*#I%ZV?SRK^G(lRoLMvp5pf2o~}Bj!t`OZf)P
zlR(V2{$;1EA%P;|z7bYnUzt>(|B@&DmJqh_fD(RZ6rPkOGq}NBCjV!BucRntV_Ze+
zgm{;|dAZF1%!1hF!(+~$wDuKeUg`HY@kfqAqdlh`hI{2cJec*qP-itkYUA)my_n~?
zFN^8}3$kVMCS$T<&)Z!@K<>s+;OzAK=|)m0FnsfTQV(<p6=?hH)K=>i^A%R9ba5n?
zx*5;};}X(gNGCa{WCejPC~G+4gX#ROtxE@HCDJ(ULP#2}*Q18vOdZjVuTo`Lm)?1O
z5!(xO`QvVP^5`H!a3PgbcP3nW?Zot<bsuiG+kEoB*T1VY_0PNBVcu-I;mhP0tJ?j;
zReQ~hH=Bb%^%MR-sQSvNDBE`J8B%hP?i7&jh9L#%7U`1i?gmjoKq={=JEf754(aah
z?&iCFp1t?`ev3b>1&dk0+}CxUN8)BzDQwqe$P;V?^}hnzg~#AV5VRnPwQRy!j6?OU
z8dvB6yu3<V(DMYi$oeAwAI>;pF17<onr?ZG4DS2zB~|`ou6*RklY%t~bRH0e^Dy?y
zsN)%y)@V=VMS*qvNueZA@|)wR*mufHX?e>yO<5nFDm+MK$P5G2f`PLkawpY0=7rJA
zHaWT^F}&CegMj@=uPXmF<w6a)Fs<*H<<QmC9t~6iLX;lU4@)i%#nyuP<7zKm3`km2
z7<d}%u2Iv2;5C?&d2cGY{gU7U0s{OX^M&g44BBHgkEokVAnwF#-?v5FyV7`d?#Fwx
zr~m{gKlcI7GU|9Sr+6qQx7lSBcUCaqR|>xnC|N%uzdymtP{{uUj0&E1SgTYgcOZLv
z6PYh^d)6FT4Bm0z>hNAzzpuC)^QA_`sWcivS>j6?=rx_QCtcu2$UIr5j74{3z(xq9
z<$9&@BPbm7O2RQ+i?(Z_kw;E*o+wSu7S0k|!^&4hC*F;cg!h0;yQ=f?+3chznxd?D
z2$n&<O4~8-5WsZ4Rr*#P14a~R0m|z81az6Yl?SG-tb9j(aZx{Kz97^)av@~=+ZL!J
zG*n19@;LI%)j2LLyfONKCpMnzCVRV)%Q&ayeCAa+&1Io!imz10ViuwwP2QU7tRU_0
z-n%b+o_u>tvN~3lgr;X%=h(k*tNZ*PaQOKMJ?^!E1l$+oA_xSVR$J3wfath!8<SzR
zaS4q<TY7D{Q#Mr)ckoRk-!4^aX>x8|khNA_se-PcB0Uxee*^%Yc?uA(tt>PVr^(?3
zcNsIdS~JRTSF{`Xi9V^eJ*i1=jzn;wsa|8_N!c}bp>m6rz$1fxOX8Cf{v5^+#MUc-
z-xu?%Zo|Q{&1%#^B0CxVP0kOuGlnUQb=ek<`GQkD`ZQC%1pacA6O#jTB;W8Z7}J$Q
znzWj7?x&lRzVjXc?R;v7`UQu3TK;j9z|<!-Uydx7nx&y*@|j{~QXoX5H_p0<fWE-#
z0h_30hCtn@LNL35)U>CU@bpp8(d0fg$?z3;WZMxn0RNF;zS7j%7srIIjU$<M{$6eD
z&>(w5ngT3A?H>`yeS>85qh3<@VQmtooN}1^f-DH(LCG&_!p1{?H-@Q~>eS@OM3Y++
zv3`k+jN2V=7Tw8Jvac!d4MG0e6+tk~s>+I?X?X%M^vC&!blKv3?tq9Egc~!kQFH@^
z;|{GUgMZkMt5woaK<s1kY|T0H2+p>;{|Dv-$y5Cc<1YxMY)Uf8?_cTHbSyU2_7&x#
zxh}th$%^(|I_xmG-Me|j0T&*#kpI1|1H4A5DN~`d8<)(-C_WHIi&4|jPrh7Bm`yRW
z@1NSoL{Sa1?{27K-%4u3it<fdT?~v&?=xEs$kw+A;Bmx4I09)?@}G)o3Fb;c(9PUc
z*ghHPg_y|rvA48CA%Z@3QM0RG07AGo8ORG16N1XDIbZj^VoH(h_U`kz*?wfh(9BO$
zI7<*x`?SxG>Gt@2S{<;X?*xCnIS-1Rb3>=p(IqMjOlUT|EWg=V*6hq_-Fj2NcGy1=
z72^%>%|B^Zk)CnntFwr=UOV$9w>V4|rrMn6Jizp-L=%T4a<*BS_nAMXNXh5J<GUKo
zcO_ir`2z*puL|U7F((C{<jdP-G-WXymL(YL>VJQG^Y~NG4s!XG>+0YKh&a^-4kyZX
zrl>COH@;?W(Yu6-;^YF(HJ};Easrv_uB8TmQKQ(n==!}yZ&pGI8Gj-qpv1-$VUSKT
zDmKZhgzb5!YJxB#hYHo!g`U!x>8~8VwZ-(S<)5dm=}x#QC-`1P4*FTZ1VtF4!-6FM
z#K7?ly}xC1F>SR0i3rroGXLO#o(H!6enUrqB%r?%2&xLtG3|*mm?k45D`gs6xdjLE
z+=|b1Z=Vg&q^isif5Nk>ARDI7c-ZNkNfPpC==7QWQsREd*jOTT3G`wyWM-OOuor`f
z30La??B5VIwMy1{5;Ydcp6)J19fXOO7uqF7airSA!+jY5!ukqZ{So0;fUD#whR3Fs
ziCx|cac1u|fEGL3c1PYCEP6bzM~-rUxp9q(bAfNt&<ukxc!}&THHW!iwl|#*)fMbj
zBa9gN<ODov7lVaBhc^sZrm=K2nl%aTn3PC?nMaEmV>a%jdOI-g=SU)&MKNJuNs4QI
zHlWe1J$Px~6}rcM6Trm+`qM)pT2|+u^(+~lWj0lk>n&d#6Z3tnKotXs8rc$5(g6|M
zUIQu*8;ir6W+NX)m3Lt7j$=st{C%Ur5h*m`m7V=;f1z1%+KCd6)oAaf!`N7j6X(rt
zukCGL;poF@%`0ZyXn{BWsRT5`k#mXaiBn7eHC6+10~;j!Uw6{l4KDHA#j|>gLbQDz
z&%XJWchx(^^Ub3^v`{LgIldiAi;3E*lu5tg?gEUlVr@2-=Pl)Fk(EF&{Ke(sT#XGa
zFYi13ME6AXd?ndFxi=H<dS$B|hphiBH+hs#d%ZVbviQqOT@`5d=)V^rw+88lT|evq
zNHhy3pj!-%#%F_biv0>NO!{j#5Azi60@)Vqbpj49lbrxB%aDz91l>f2CCnh4;ST&E
zu*E`yzCHQMaTBRbq;pc(!SS#z=3DlX#V}Mt8jb=wvIgQHzx&izS$8wi$4#Z8^f~rT
z1R}@nMu#W?Ttkw^NxNxq1^Y$2Dcp#2d!F)MRv6~G&z%;v;>ao_rg4Lc&JXs+7kj%y
zAZEDUT>g3D4QcERd}yjo=OVR9V1Wa$-xx`@j&xZN7l?5cR=k5XJYLP`YA{U06A7Q(
z-tCuPit@hK^H&+qQ_NLN;Z-UC-c!4%{FpQu|1LI3^7Zte5L!tBt)DP98r;!fw|uE?
zP164yazO!x|7t-RT#o$wBHq?5@f8t#>$-^8(jKQDm_^e~)B!2Z2_O;XToYuIs>n_J
z#`8v_7|&W=X;Issi^3cr3G7B#1%2S$-sRi82<nK!5Ts?&jA1>-bTO!Ydc5m_YYUU5
z3q<k<b``wOLf#9i?d}STqQ0|4@S*VRQL<@EzH_r#uaDkVm$}Ert+Zggm9o$%Xpl{z
z8XCTavRSOJDe@^TBa8%k-L1TA-c<XIavhBg*N-Is39jovzkaL|!(m)c(^-Hve9GS4
z)4Zo#3U=`@yg%fwA}@M3_h)J1O{@Dc>)6WHQnH`Xt)PBQw|dV!VZ3ie7q(U3kMa-8
zYnr5cS7#~SyN7w~ZGQV~kYAJ*CTfA?c~vMdFvUs+qJv+SxLh825sBu$XFSsW58n&<
zxYtzh$e3|ghx77AMzH+%b^<QaVA_OUpw)5X#Q1tyjd2q#@_)HqRp9B_9;O}0Fg=bB
zjg=w2e#-5azYiDn=$3!J{v~xhyAmlPG+DmKRwJ}nHXB~Y`YnFBb+vHKEplvgTHhlL
zuE=$KUcieHEd0~|(OW;>ya)4oxhdb?K3d4lnz18%Id73%ggAR>R(=-q^dEYdfPj(>
zIF=shGJKs&kXjt!0NheVz-tuz-KZ(cBI7a6WD$D3K#2j@4q8b<rXW3h$AV0|(fR<)
zGkUX^@hpKHes}0D=c_2Nc3%0qg|nhLa-q$Ld0mG?I**0bJ4aNP!?8^#J_^E%<7I;=
zG9iY5ht;QNa=gl#@f=w;A)}0FXmVDYXn^`bwX%p??A_bqcygL4p<{*b%l^81y|Vz6
zs;X&y^ldxixqOcL0AaQfamQk#_nwqoOl2moFZK&QOIGb%qapwj71dH1t}rO^*iRO<
z{5tJ!PR`H-{Xy>N$gyamw2LTBsR&b7vWz^NrXhGVvtTU{4ngJ4*_J*|Ed04RjA&N-
z|HI$CR!Db@u)&t~`yB5u;_n4TBMSPQZ<=g%%hO)HqJ5i$ZPsPzM~Q`t_3E#5Nu<YJ
z{;koc&_W1~uiuwmt=53TOQ5M+=W|4a`pQK}V^?h>XiTF(RcEM^uD47P`9f$YLY0Fn
zU!YOr0F5wk>s|cxTesAf7{g$hK$Ny`C75e)BPQqypcl>;0!UEGO=%`XEisbbRSE>d
z3V4X~4O}MV0lc2gv(825S#9h)lk}u0a2MBs??gnwxu*u<B>e$#H4cFn-U$p(5U$P(
z;F)MNmnS(Pq<E@Qe348dCHe!Mpz}?K=uMo{O9fn=UJz>#i$4e<nUacf?kqe|r{p11
z+`ikw#7G|M`opz5TzLo91*SxM;s26CxVESmC+dc>d;0+^ymEyiG{HG?HCnhkht%{2
zfeL9oi`HrB6TCj@ByOO&i7~>6M#+2CMA+8aNrlbUU=rz;d;}w%<VW<;`xn%dPnq{X
ziMN_xbK-RDzsd-`JQ49%00rrOyEkYyVf`Ow3AY6OWDelZoVR~*ThGZAO&8^-l^V3D
zP<z-qk?MlJ#(vRAW3)?_JrB1Z_x<(h@%Ct24J_+BIq<l&zU&sr25cXG^u92rpKYW4
zPO-GkPro%P+bn&1K!u~FX$gm#pr~BbUgPg5uvvk72kpnn>%8~ZO~1Glf`3c61#|__
z_aBMk3kl9h7p;>h3#>1HY@x0UUu05K@%*zxfLoCNi=36@{;L$rd!=u>HgJ|P*%yJd
z_8*f-^)5F=)GD<lOTQMe+q5W=aA)X)zM^=73oNIy+47Ug+BBvEMCQQfwyKkMybsI6
zg%@q?kH)_)BoQC9<KRf)*jTJbv%*?rqCS0c^jDvF<6mbtpC?NCzLI)>`dI)xa-a=W
z=Y6^|RO+&tU*LNu7gw6qv+w_;a#fnN-zzLMi#YPbzXB1+nOkNCND9~Dw{mHK$m4ZI
zr$Z6;0Jpw$+QTnvG}J(I00DJg_B+ng*^-1@#`{0c*+Bh8)7#<={fRa;+_bALL*POp
z<u|j9JNB&a^JU1M)Rn(#&Rg2Kbf<%orwvFlP!U3k*ne6?yZetn-eDyE_%RU=Bh9yR
z?T#d3#2}WkWC-N_v?;qa^==@+?)!GS{A1H)^(kq<`5!b)%bgs?=cT%1N9A~1pydt5
z!PH%{fJM(I@-nGZU}PsZoT(pX(C+<%1kY}Vueo+WlQwsE6Reh3T|N=@nl*9SRWsz{
zpCB5G+{cx;WleAYlEMFPf}m_70%0L(ayi*8G^i3RLQc@)8uqu|`eyBCztmv3Xc*M8
zJKy+?a#K{4*|!%G9Qi3;rgFdgXtC*120}FZ-fQW@U#*h;yW)k?C@D7(vZFLv8{&#g
zm&a2ZYhYUkxCHKks)z6gXhsdf;`wSv1qu4dj`^XJw2kd<Bq(5<?UC|n7R|Xkm>ly4
z8B+V|#2Afea|Dp`5mES$=998K{zi8D{0vk0Oe%7Qg%-;3U^JK-+|#mW9!*I3cSE8B
zw>TP^@V^CkacZ#*j>3)1Fwwg}s4~J0DHqWRyuhOox`}UfMK<J2QK)}JVhE?OG1UvZ
z^MA7dyayY+(>3mle5)_a<~X4jtIOk<&R9CVr|TQSqrd5G`PqixGqN-_b@}~t3F{nz
z3W!+mn^7^A_BX(&6d$2FRpPQeRD^!MnfmeW%dU*3wP!TfP&B3?HXber<7%s;A&Jky
zpp1orwg+HY5jFjt<?zKiS~;Zv$~pBZxup4jzz;F;!G9^9ZHs)3U98=iL+`_#i9W)l
z>wOKrd_ZuLfLRx$?Y2=r7qxRuAl=j_V0pc!!k}CJk=z^3QpGFxUZz|{GlXGCf5v``
z(=Wn9e8VLv4z--w?cRM79i$6#<mf{fdAL18E@Vs=tGgV(ma|l!7MaRp`1xTR(#p?j
zeYHdR)qX#}r_Wl{-GzjDxIxO*nq`w@YP7LwizQAoL)rb|2N{kSe9VhB>RIZ`m@(1%
zxVVD%24520HMIjXPHO+;N|+MdpiSDSgwe6|vj!sR>KNNFVTo~odOWieq8Djr=uoQz
zW_7e5mbRBB7C_tuo$xBE(zy8ebPlbLwLhCH3Ii^7RFk+YqKX_()uY6yfajN)=(&V+
zxv4KT;mFWy{RM+oal?^=*1VA2vyBNO8K*P6n53a)3Xe@*QNo4RJl>>pAW__U&<zlR
zmk&8R%VK#AO7>VzV{Uy)hD%u2sSF#bwsLTM-RvWp4<(j!>^|S}Te(p6GapEJZDH$f
zKIl0=i@Eo6IB($jaoJY}06=~PxfbxdY`xAW9x{#gdp+}xf#vr=xLkby&S64-Jkw!8
zri5~u;Hr0I5u38;<7XPle3LFtPUGJ%!adf%vh5Y|V$#H>83y>tIFKP;R+{GbO%w*0
zk8%3dq=GE-zLp+=l<-UVSv)WXX1R*#kD1x77dMKSaa|a-lOZ$TpEg{?@3upA)GA5I
zJ@Q759<6;^8aU4SI%%Avl*QFMYb^j}<9|M4Y=~Z97MC-gcJ~XovH7wJs4`AivCu<o
zwXOw7+#;*uX<o_NErx3S?1Gc)O{yQ?le|B-p4XpdTE>yV^h}d))h*Z5rKcYlJl?i3
z8q(fe9_#ue21XcJStEPFvPKLH=4D_hQjjO*Njp3%h~-j4YkriR#R)NH6w?pua0l(~
zM%*+Kx)BM6dJ>Lf`o#$3vFLW@LuR-Vc9Eg%A5+hBMLtlTQ1u9d(u1IoAso}0Qx$y~
zWWSjZ%bv=Jl&_+zulYXtvpX^4cD{ijCZSM9qXnf#WI|pc8ok6an~*_k*p2nxA|u)c
z5xH_~K|)*Dfqq76x_0K6q`OFEb~N8WWRaPQ;dhm!AoO#1BFw1F4t_m?X$k0>7*dnT
z-X|42%NCYMJj=nul!}1cP!yaA=d+Cd1eRhs^yM`KgGSdvW`oqb)!6J9WU`i5j4wdM
zfqK%><fQ|q&p?p==zagTYafsGgi7w0<q=4zq1PtfWmocb)nL-sSP~Gi-VF?wAEZG=
zMdfY(x;Y$qq9Bwz;OPN9U%l1euK~cWC#J39-ICnm)Q}rJ7aQG(edCsHrL^6lC#4#N
z-)qDDKYYj;8a|j^qFC@LTO<iYf{>QCK6zsV7wOa#Ezw~Zn!*u&Y*5UA>KhK4)r%GX
z=l|~b?SGUAvnXvm;)&^xuPQ>(VorKeiJYnL3R2Y3U{3Pk^RUTv$sDza?&L&r&+J{l
zy0<l$Wh|1yVD5tSU`9smTo>8Cv-Q%W`P3hv<fLm1^?8a)IL9v@F08uw!||7~c|yOL
zjwxON_=szF(qRrVIUqtPhCYkV@*HOjHcooC^AuUDC==B*t4qg?)+zuO>&4R<8N*t%
zuFdE?lq!mkvu^DR0C;SPT-WmP@PWQ=AGa3=nebYTNohViBF{0uuMl$>jnlop1D^W9
zfaiCzkrA>yk4V!ZjK*J^ET4wnsDIdhI4yMT?X?48Jeow%FWg@oEza9D71!8IqaKNz
ze@8_PM4Gfv3F{_F!qZOoTr;&u$sZZvw2sJs8P&%#E&R=#*5D`gK#8{eMeSQ|ZZJ0r
zAdon$8CKbtQcAn~*vQt!XOva++qVc>GJ8PR1G5^R1B$#JX@!4DFecHPKHlkby6zD2
zUGYqFYqcO-EzSOv9c%WH>tf;Saz*k~Vz(IbiRw=CXvsawh6t>ux;n$bh(xQ3^d>vw
zTX|B}H$P@JIdkH9*;&3Jni<y&qH8T`eRyisQEL#UaLQ8<b#}-SKD~l!x8fb+GXH0=
z`|C8=P(+g0ulx4%)VerVkcE~5ES8Lb2%Dvn@hF=B--}))!;PeV<3e$&vROp6ayt~0
zRk@S7F4||kv^84_t<-VojiK`1r?bsRYrH=d|6WoiLNKIV>QP@$_y+Dm>VQJ&AIV`w
z@(j7#3K)|exs7K<L{yBr6C5am4${Gjc@Jp!p*NjUFo+?)0(7GdGbS3^I}wiVPn2<G
zSy<i}Md}TN{Tflj<TDN=k)3ZqkPmku=*S;FMK}B=xGgHav>L9-3Uk16HHOb(>o)Nn
z!V^3}Y_^SU2s{vM59NPZdlJ^Nn}G(4#0kg5m-U-t=XiKI8ugh62cG06n=KvTPC}mO
z@yFGUsXt}yN%#z*q!Pu-xeOfn#)YDO*O>22NP&!K9=U5<KC5nb{=!Aj&C}ist-liT
zT2yWTxJ>jpZq`y<e&bnx*+UWF_~V#mD_9^1Z~(aWZ9k+U7-Hul*Y^UDZ_p@^8cjte
z=sH3fks54CO-;Q4%!P5^Y2~d!Jr%|s2u4w<G4!8a73=Z?vNkyY^2~3^LVN2%3>pK>
zkRn$8o3}uJgWP<r-FUUKoZJ}1wmB%Q(QTg&B773~(+z0so*6sX&Cht})03U0gX&eb
z2mqAjS~5|j47u|8zJ-Y<hkM`(q~)-~V~k<9nacke4cc;3{$)jmhMY<pF*eV-|5V2o
z^paJX^cfbC0*FFU9Dk&eg9r_PFYcaB5uJZ{?93odnziDukDh3(jWzdzquaYYX1zqZ
z;GX1~9DX6jb-5+ed<URo^8t^NQ~q3&qx&awvDj)qi@1ht>AmmmZ%sq5_k`#Ix5~9d
z0f|Sg#Z+m(Jfv{CRVMEeQliHev2@F<_;>(ksbDnrur1T~xwqNCR6||UD|yK!HUaz#
z0V@`G-W|_`k(Vyo&yiqY{(!#kv_*40&$9M`mcVn7bSdpnuFePMZb}@AQFQ6{nuO<K
z195|fLHy%2Hd^iB9CVK<Ans}X>ypl%Y-~?!&f-Gfi&0!nr)!7OiCQ}vuiMQgSJCEv
z4wajo?4BI|M{6CAKgoxCUhO@N(V_|<MelhsfXyK=e2{p$?I9}wdjI&|mK$gZ`E%{r
zE)W*B@iO4K!k$MYA14wt538+QFJx=T<|2q1WNFI*&4uq-Uo2*;%XUwyzmgDeApN?x
zHQQY*j7x_D(d&nQcskZ`IDZ0&i!p$}V416868Bzg9S2f@PZ$}n6t8Q=&nEOYC%Ik<
zZ)sfa4yPw>J=VQ_AQDEZ`S4%YwGmg$J<+H7qKq4S^}r36?N@;7<Fg1ctv@*T!*S8F
zYemp-<oD2VdRpjJ+#tEA^quHaV<)M=YZU89(_7o=W}YJvY;5d}@>acJt-EFav{Y41
zat}K}%6z4?{_8~+XHoUAWm7Qx$U+upqb`|P1UzyZd>gJF`Z{O}?0bQd3Wi8Qdn}zs
z(e<%=F5sb;z_1PYd5*IQt3#Aef@Im*1_flQSJng5=!1rDldm=^!^lj2Bl`vS@UOU(
zt*C~xJQ^uvp_~3x_Nvw4600TEV?Wcwnt6-$u-+Ez=t*Z#NeaC)U)zoS984-DTJ~*U
zdCvq8?s-Q>b11!ldaQirnvU`^Mkl)vxBcT*|B8Y0&PXA0R0d7xPE%h*+IQGXoWQ$6
z(wDk`aCst}_44#eR8JF@^n(O45m)Y;eRATsJDwkn12LQ~)oU0d+V()!l&Z)Od;wtu
zI((kD_Hl2%7f^yMD;#}GAAlx#dEl?&jF46Mt~42G7y%J+lh_mK{2bxi2ShO(Lc-S>
zfVTtbq78`n3wZW#cz(K9_xpSe^g|Sv&o<^;6q$Kq-2$CfO`1QgCW|zS%H_wM5dsHd
zs3aFU(`|rvF-X*ysTwdCXnjg4oCoD1++XfnFpbumbz%Om<QG-$Ut~}~h=#23Dq*bM
ztU7kRC#qmyFOJ4N9~F<O*hX}^RKZuTq%-x?6l!q-i;AH^?vzJeOSR0Qp_kr!BC{Tc
zfWGANfRn}*q&hve8XvbQPGt~A?T4a(bvGTPk*?#8<;%T}*rYf|X>8@5#!5w)j5~;X
z%bR_8($IH%OSBZc6?E%KgCoXue>%N!(N428p>`UhH5eYc0J6nyW>H&<lHRYHi<vy~
z`#P~RrBu9$7HB>r<ZPmao4+|F?P|}Wcmx~?ad0E(9<eS)04vf6cf&|?c72@)wDd{>
zsjBO~GsbhE1yPS9J$+g1>*Co=c@(kntT$+WY$0~RPRG{RNA0}!3$NFEq6vKCViGk}
z7)LJlE>rJNr|y@hfuH?&-5Q$_U{Ij$d^w4+IjN<8ovaNWDKThPO!pnTyksHWL<-FF
zI``hAiMdZz$0P8NR@pT_oIA8mIi}e|4BOBC?sK$DdpK{HdHD-$N$v@YMkgE*ABd(9
z&Ap=`%}6?&C?U#AM;6&E@Yp@`EL-H-Mi4vCflFE78$rYta;KIpBd(uJGIbYRj{Y3e
z?RKh|-uqj#(z(Z_>E}9~4DY24ej>%>lt#{|A$h{{Q-)8xaKEk1|Nlr&6V*2kmsVMJ
zOVu>v7>urwB}L!d+{rm(pP*ae9LrSHFuT+D9zE*Jy)1cEO!V3aL@X>dlOM(J21<on
z6ACbTaT3x&i2TA-I41=CLsVhN)TN}`D&k1bs!t}*V|(;|W(E_~0X8b}u_Fa)3XI<`
z++)0=DtUkX*!ZQPg>%8qhUE%q0#4QoA>`WyjXL3tgz88_7XlPB;?d`7i>Wvvn-e&X
zM&E^H^>FF!`(Y4)7rv2|>}R9f606Ywa&?>MaI%BO3o1P6ffE7_iB%LS(sb@gx9^eA
zo0AT&QNCrlkYT{mhhlmY8d=$E>f5^l-$G^TJF`lu8h%=c481gpZdznS{fn)GE%&BM
zi(9eRgDcIH3`7@9m>ukf(?t*=%Wo+H9-J6Q5X`(r&uCS}!6QarZ^8qB9xuRvnP?ki
z3Ht86Us`u@B|ej2d2_Ogje`@*wEOSV`WLF%VM!VM+}YEE*2ejRtHp%dY9N7S4{Y2I
z1m(H!0+n&xny&sT$&eLNTVykU;adRezcJD9{E9RR*CJI5&4CNLaZE8y&@JaZht{np
zBWS^_MfgIm2?e$bZ1-j4Qo6z~(s*nVo&t)sf0~a~naS574+7j)ykWp&{N8`zps(5g
zdJk?LjDnK%sFFXS!20&8EuJ4jIt$B7p8iZ61hr!?m4zv_2jT617;Rerm6zQ`<MH<W
zK7b;j!2a@w)&AX=Cj7SwSr4u<{Sn&bLu1BUxMJfMODVZJJ;kn}*(vV5hUHPR^g3Ho
zz~fQZV-P<hndEb8Wgm?=@+UQaDdxPdUeP`O?{*!p+i37(8!xb1uVZaAW8i7iW|7{>
z7=aQOqzW6fU#$iNNM`fH=WL$6dhm|DU-_0^-w*~tOszie-yhD`6*E$cntRQ!{>Fs0
z*VfhZ{r3C1ErN3v!kF)}HTaD|JugT5O`O+YX)O$7N$%@dZ99v->88DfuT(Gtw}SGT
zn_J3m^&;zNJsNnai~Vx>jYSW70rv(l>~<wYCKo`(=I9xF7JSc{zUS0dp66UI`Mf9Q
za~G*QMS^h*BOf%m$u~q_U&K!lJ`Yn_<TM=-7aD|;Zm`SPDS@~Z*&s1}$C~4(lZNnW
z<vr2@;q)J8bK=MM<UWIh-{E>a?{}PK$R^!438k!#ao-<CBxAqa%HJH{?&(#xW}=t>
zk67UEXI|feu)E~nik*tNmB+m9D?<I#=Q-U}6oXxMEkH#(P+;xpgq<ICzSnxB=-um3
zY;J|6*wkoeHaBMFtyALbb+#e<Xz+qUa>aDh-fzC|O0mgo|6^J*Bt17G_OA|*D5okg
zrDl9@1y9)C-4P2uFr2C^;v6>yZPSt}g!Qfs4fF}F1a$B^9}?{Pi55f=Iu$ANe4?nk
z6QsO6k{{`}g8L#g2>os)lV`j8Qmp5q$8#lwGQ?pfxEDD(d^e)f5}E8S6f%^-OK6dT
z^(sG9mILLa3rer_Oq$K}`A&~r83d%O`bB0c6Ww2cMJ|#^YxBs~Ly&(H`q2?NjGV%^
z3HOGiB3}??_;rV3ZX*!+1_EPq&&szE3v#Dx%kFTp4gPxa9TWJCir@8NvB^L0oC}(h
zTOuk9SH96C-RpegZJ0Fx`d6Gc<Fy1?E1Mx=c(-qSWi?_z-0y|R$wlsVviFSVc(mb0
zL6mSO=(R5aZZQ#QcqN#+t`|q@yEX`8=#;7a*Bb9v9-FD1+i$mB1cq<JQ1SRD4iD>=
zS%vv<=oB31$r-e%zK5d|QQt7)Y5?mq5s!7;(K!TUe{=r?0OG0Ta>fxXrGH%|nw7b@
za@8r2e$o2_uE$&kKVYf`c-`U>>9549nuiX@Exf{xyiD6bzpF0L)V;Iwk6rLzxxAQo
z-@llfg+<OIPxCAO&dV<EcOrl8tg7zzM4!vTe+srfjFgY(7wbV!1j$e}Rn0}ts-`QZ
zBgt85nHtI{M8MonRJl1lGt;C^8TLMEhtcWH$W{V~Be3uC>q1ENQ!Y}a+p5R=PIM`s
zqfnrIsDXJsJ>{PQl8nfVXJ;G$N1Ai_Xyd;idJQuZh<NyXW%3qriznMqWqVljt1E!a
z_2)6PdTX}s_4(wO{QRg>>#%FBefE@Sa0xV=%z{To7;4+d_#I(OIs_0i5fsRWL>Aq=
zV+zpL@F!I4z^&e^&8oF$gFlx2cP+aBxj9YY=ILJmG6j8^D%H(DwmMOj@Q+2A;rHg2
zPAgBW5((SRpeUJ|RhVbft2dC2yb;znq+>T9{Bb-fvMyS95fUQb4-w6IcpU$^&9xWt
zUW<p|bqe$&CK=8C^f8b;7MT&?ulw+*PvPx*x2t>xT<1dGz!{Lldx(L8`g@(_8-Rcn
z78L~uKK&Wm=wp*_aL(lcNY6aqTcadyhD}m*;)|lWr$J54KNqp8&xwZ83^5646CCyn
z=t;s}2GVp%-+>D|1eD@mNZ-=p$^07N$=_->;!{+br#(d6_9pMTk#%6!*~dPH|GYl_
zzWyqO%!R42%f@80*23LlJeKF@e;LOrh~A3A^BMSMmn<~MB~RLxLDQALA3r&1emES@
zD(77wY1Uf9S<CSY!E~nM5a&H!rY6beeOqqcT>X$D+B$mpmpZ>Eu_YoQAt_$2GuqF4
zGw_rEQxkXQJ^Hm}u|>}ENTh4!{*S|}Kny1rQ`iXp2mJAk)9F`ygr!?FWY#MbB{1YY
zJeKYWWjMKKSqi*XhH3Rhe&d{|&JHzx1!81o;~eC&5I~TjnP|a~o%WBXnp)?^$Rx1^
z%FF41ELe8LSmA0nwi2dHzK!BDV;t#~v?0K2Ms!8QE!pLYyAy);8>KBrn11k$I0Tu%
zdX8(iI-f1~XeJQZkiTyy<>~|lgnr<}ZX7B59I0$5H_5VF-P5;$62f+a9DO=!;d=s$
zxS_nDvQQX8WWl_V@Fk_Jt`N8B<~!V_xT?>NejN$K{PluHP*F6+Pr7Ehs5Q3Q#NjAu
z_`nnwDendnr4X!gay>Y1KrTiDS@w&KW~fp)`1pD4T<z`cjc0vmY`<XF{)2xSe1hL{
zqQSgQy9f{sUB9b5A{u(o(d7wCAp{dXDck{Lw;RgIT319da*XH?>c@&HoX}wY%e_gL
z3s5rnJuTOUb|#XcY7`MyH^=4QJKXE(c6;%u6;2Epkaz)s!G;0DLm%vV{*UsYO!?O^
z_<<`(n|*)D2ZnnM8NnYQcx3RoJ{SzX%q`t(_%rg$LPl;d{n0)G^EC<cqNsP*su#Td
z#qIL_ntG?(K=`3=MX9p;>wHl1*yQh=aou75%xB+Qc!644i*2DMP;z?4FS@P}`P~L)
z(n~{udg~A_Hg`GAQsy)n=s@NN1Q?3Lyk4@1XYrJBxkTjpiA4TCfc!X;cbucJw7-{Q
zX#SU{y_37s>`hL?g^v-o>4NgS{?GD^BJ9p+Vy1!)V>zFZ#y#h&UU-B0EiS^=yO2gS
ztY=h-Fa7n{_BS%qs!q;(25G$y1J3&B3!(Ob)J48oV_O%y<6L@mT0IS&%m6N~--Ow_
z=+T;HttlD4MA_`ql-}f7+zXgfUL&eGgWtR>nV|-X<T5m%3W4aM%EG6WcPge$*%tlE
z>5=g*e&Ir#Ke^oYNQwkY(O#XeMHu#qK9{(3iay=s&6&(z9*^{XTP~857yu%ZFo-p@
zuDhb03z&9cSyA+V(jU{|lIovCyM#HGOU9-RA?li@12$Z?^ngsJe_2xd(rj-V7uZq;
zlGRG9_I)3Z^DVs&V-GaD8_G->yp9|Cy{%6zhGf^5QvO(N*#t~*#pfzpOt&Hev}k!@
zUHkv8T((G0ZbUo9ft>eI$FX~O)^D?lB)HuAaMMC+FXv<>)Pmy}RNp;B;K#xODFyuR
zLJtILbtT{=UzI5QxVgjqR__0-wr}WjSYv%Z!H+lrEzmuv1yd^ao%yd(?6B#+lis^F
zez3eVOx%I1So!DTSF4gCJKY1(fsXG_(<%HG>SOX78v&!Pfetnoi0IRg7ju${Pb1-M
zW7?BWBA?{iU#nn}bceG=if^<N95Q65{y>@GfC31)7_eU_I|U&>R|m0f^#}w-L$?PT
zUMJA?e{O)Ko`<BmaNfrmI>y!@MLCD;AhXbi;){Zmi2Zsi*{henOG5o#^BlsnJb2!T
zRXI|(A-AU^P(%jdbMA0JZD+&LpTvd1fKXxD^bYj?<LNF1bO++xDF)2%D}sa+!aFX1
zD#SA?6$Ks~6+Q1%^ud--S~Ind)+4YW*S!f@07QE&?#J;gE*^%5b8-S;S&c!A{KLPj
zZ-HeS&a%Qh5Eu?m#CJ!tWg01y0e-~h1ib}Q%Ifqbjr4{fuyLOP8#lgKNNGcnW-$^$
zGA1cMB~U3330FdkL2u*C0l>O}{qk1UnI7QoSf<ylD;W9dvaQ5tV2<Q@vzRG~{Mdws
ziaKW2tNMj>UMvY=<xe<;|4xPQv^|_9;6h^>hVDPE8c0Y4AI#S&`#(Q=v?o{*i5Pjn
z&HW7t`1hmjP4B<A{&B1Wn0t4-e*c~(2k-`z;!=>&oTvNA1IzO+hZslNlJ`_$3|t-{
zI;?$czY?XiRlk_Fp~yqg16$@i1*W5>a&Y0jr_TowcRD=xufrK(y)gQA+p+X56laO)
z%{%L<w?!UyOlV7_#h1;G-}FI>^Dm>{Hr&-z%R2}#5wDcDbAdM{QY(nSmchm0@;!X%
zV|N7#X9JIHy-Be|4wCqK*+s6tB4)*j0QHuje~vHcG-~C1t%yC^P_yo{)!Z?0OgW_(
z9`Xw`GQ!6=x&Tx^jy1SgFsNoB%!a`M-d7XFhP|@ui&Q6ne+rkBSKZzGHYZc<eqIgE
zWzWwiP9R3xEb1MA#ceL!7Agm(`ic_*148-;d^<<yKx^jp9te)8sIUR{?RvE4*__Qf
znSfxlsDHMehkpO@U1bkcP192C$WAqEGsWSf{A?9#prnFA$;Ce3!XFvH8QuNWN4iL>
zJm=9~&Fq-Mf95g0$#oa|tczf{P_BZZ)yo0Ta%Ckm{H{=-)@AEv)?!>UKJRwuN7@&>
zfnE}a1)9HHS`qL5lLuvO?C)B&9J2x5zVqRLyo|7zaQp_p-CQBF_TU<8n~ko#UK>&1
z81`{sW>Us*0{cWs(f3XAIb0Ic(`;>A7t4;{v9^sZyvg5aPt4}z?Bi-c9NEyeE;ao>
zRVv`RX9EcEEx@^X*I4F~&Duz@Lh#yU`p>r;sTC5v<KMol=bH1vt2O1`MEfC7gH!V;
zAQ7bWSZgR%AXWgrIDY@{7N4v;0q4~IH5|wHIN(Sk;|Hkv<{KC-5bE!18AR0v`RqG6
zc>|}OAv!cbsj+dl+GtA|!urYY4vqr(Kp_Ay0Es<Fxs7~m9O)r$+y14gbuZz&^RlOn
zFcJdwOqU9%K=edr0SXKFDZ6Dd?kwB4gZOt+PAA!Y5M%^s6_!$8(PJ)e(0rWT4sIZ1
z#q{1uk?oSm9FCRH-xKc!Jh5vdA3554KEqQWkenz8b+hZvm?{WG8Are)2t54Ya57=Y
zXLV!$$f~1EKJ|h~=VnufFvEFfP0=SFSOL_(67wLuRUCV`J)aCoz9il9nQsjU00WY`
zqE|O#ETW&;KBF=QA~k?MZ2?#H^fm@WSe0Qj(a_&%Orz4|4UvFzjuFf2jIaG%VoO_y
zUjN+n;<+?GFRp<jfV;QHC2Hn3+CG)t(l70F?>z>q@w~6Xz>)A_zgDZ<ppZPLfx#0U
z>lxS$+|Q3Q*LVlz5Qz!B-(@<vxElwahxc<AM4I;f-`x(QH2>FSbZyZCXE`?P^hPsj
zgHbaxG_IspRN$9=#H+$^V?5$%oL20k6$=L=EH^5aY5%)Mz%47w2UrsYHQmgzQ|AKS
zs&pYu8Z$Ds$r;^p>TDlRizd*JEEq;@AqCRGY(Kf)pV0c*MT>zh4wA2<N#|?j*P_!@
zj9_2X1gRB%>+r-=TMyvt@hLT#6>vEWD*^EX09uy-pmxXgLVhQo4xU^-OF2YnfHJJr
znyz`&a_Vgc2pP=~V5clP8s2^a{e@R9tPI*$Y+7bD5Hh?A@$f0z367muyL`Tkcl%|(
zGo=6}a7d=_7u;_rynk{QHfzX;wi?#P9nvZ-=-I&B=FGlv!TnsGVm<&^f(D|?0c5|l
z1riA1jToiV(sJPqwRQqtj`;_Me9Mad!q(ZHq%%h+jTrZRw`qgk*>xR!LKtN}8ix#v
zZ+^P7rBS79MRIl7`(;r3k8KwA6(vK+;|q|<`edCw_f0+v`Kw&F3pJXCrOwnL>MPc>
zU7wzgQh-)y0$sLGJabl5fL~tAa8U(7g7#`-c<qSgJUtQr2sy6&xH~b9Pa(xKc~`!)
zbuxIE*w*=6T(D5(v~FLhQ5dNg;74LXLfwmmPMBHrl<s~o3KlcJyQs_Q4MWuiq9*Hd
ztGv$1n>Df9cuI)j9|4X(Cy-ix#O*a7h#RP^*VX)ZxN(C3+Z-0vDn4v*FJSp%V;Q47
znpL)p5f9i#RvHkeaIs>}#0LQbBjUV?-Exao(L0QIBn;{t?VpWz*M9SM<;4)isIk)V
zmg}XCv3e&(3-ptb<JNKwm$|}phBysfKCRfPQj#4~r626Ni@N#3Btb7h;$%X+ZU>`)
z-I5+K7)}Y>=JJ^J8`_^e-bd&3mlvw!^qN&&<u~s6t~$ZqPE*a(I<uu)Sfm>@J#Blh
ztvi2Xp-75OsNB+3e)s)JUw_`YbP1P@4_eD)N>gn7|Kp6G(O2NjV;@5L3-6<TIvWHT
z4*|>rtcW?*lTl|)nS9hb{VvbW+t2sy(bqlR5{6M3<=)DUo|>6ohUa+>UJbF})abvi
zMeR~%IWjTJ)sOhcH(sV_gK%YoPJuEFd)6X5Yi)m?!k9e2x`5n4QqTi{-yg|M+p{)7
zN2i2uM{0S!{)-Y`HQEUl4z=qIXANSl;MhhcIv|8+aos^<xoDU4Rkz<u;`p2hYvFWp
z{sLY1DMPMeen}Ur9t>ec#gQBnyZ95T+M^p0=BPS1a}#oB#35^LLYW^$lp!nSd$rmz
zvAM7U5frAH2w_F>yXk4aSpnn%hasZGdz?_eAPB$-KCFHQZD%(|8O~*PkvfS{DBE<P
zt>Y_sGH_asQ1bW@l)kT&h(CD@@9=dv{&lXT$vJZ=MxrYVw)~dSnJ9%w1ccE4txGHl
ztwiINz*4ldEI=AIy3a1vt+Q5<GYGO^j|M2NmYX%SI_#An`v5uEo-1(p{B$kaj<4~`
zcbDJy#!>aTd{>Se&>kK0(*Z?X^o7ykUZ65uO~^H{j59gC)RYp*;bQ;<q%#0op)SJS
zyaWzQdK!G%=!?B1NTnOT`cEnds>J^U$p!ENW~X)1>4qobh6lRomhJSp`j7K<A8O!I
zxkstXLBzCR_T0ly;frG2=`Dj!24G7pni4m4tE;wB%#qH>{n?(oGdM9oTXW69_F3Dw
zTlYZHhF5k6@3Q7K4G1I7`buLSVR3GHJf3Tb*QKQ-<s=*M9el_|9b2uTM~HVA%dV#x
z<vebvn{2r%E5~Wx-69C=*0nt=BfZIrE1r$)P^cO8;q4uJ7CGM8;tfQKDz5ouu;)$$
z^vxegvEHik4nK|=uRo!*wg<}rG63I))Ab^3t4U^-0hbgyKtRNf)W&W#dUN`7f-6ef
zpbIo7eD*c~nR!!FmScPllL2^u`G<i{on+lGqG7KU50DY$JOMut$$f-~XAjJW^)+8K
zY8)_rAtLUvWWTV^YN?wd2{b=hDRbUV6KO8Y?>n#klaPOwU#{^@sZ?*cEN?jaTmI70
zcH#G<o2wQo&W9(JQXDr*Rl()e$1vqaC1=%}-WBJw-qJEg^`>k3ejX2lh1-p_u@a{L
z8WDmbQo+h%3VqJ2-&0-Rfh`z|c~#-9u$9RYXV&j86+|w%g#3W{_mCAjWLRgP#>;}I
zr}m{VT4IRdy|{@<R;uu>!6p08r&5C{P8ATC+w&!PVHASfN;4F5VIvvYs}mh$+D2mB
z8;D_q21EU@G732Cf-yg___a~^hA5|70V-3mPEQy_q&u7xMsmWhHQa(Nz#9{^l5xt@
zaRV<n5rSz#g`|T$-G8&9K6JB^sjOdnf#H?#S-i^DlF*U*!^=;>pPkMWus$fTgK2~X
z2(ux<%IToURpi<mr<Fsl#~_G)WCQ-4CNn+slG4=o1`Nz4r=3A71UC}GSR0fx)X)n;
zL$|aW;f4`-4!UVzop`z-q-Z<|S3bCr<XxA<TwBOQ_M0d#RU-s*TQBVyOQ2dJ(*6K{
zsV59A^{)5|A)gBbY#li*3(2wN>Z|fuzkwJIlNf;VJ;Aa8iZO#(DpYhx;&lvsPv10O
zn;Ouu14}fpMaTmveQF_?naA_Q$YKz)7MClC8Hh|6*X1+*ny#t4<a@4o$ulpJyjLp`
zh6<<h(qEmQ*LDWR_FbpWo~7d>hRb-DyDDHoEG89neS5B&C(O(|INH~buM{;JrX(g7
z$nCJC=chldpi^!6&Dy6RTPC`n43K48nfLq$G$s)FpYSnJyuU$H;GnWZr~icu?qOl<
z61n(aH_w8mWj~{CZg3h>=r_>PS-0BWt!kl^8NAuM+Aq_p*)GqmdJx6=Rd&LuDVsfN
z_XsFZeqEnvL<$j?7G;#gl#2cV)E%BJ-zEH>u3O-}le}H}uc#S_sTGf#-W0i6=4$^h
zRgWw5Xd+IVvk8cKY!`_!hYwZ+Ma{3FvyVL{v#0ZhdA+3$?_jr=nJ<Iw$~ff<mc}14
zb#s3s21Z(W0NzOamh+^zQZ4muva03?t-di1!z62i`vQD;EJdT+sx{q(;fCRmGe8tL
zA>dMuD^6eF8DcHDp!3DI_8TKXakt_-h>jCeIxoPiPa68M30XX+)jE^QCSL?7F;3W}
z(H;Y0*Ro+nx1z2MX+&bBJcSx*wfq5nY%D!h-w1pUA<_9~An)Wk`Ryd!axy461UY-M
z2d~`RdMJHTk<&DeJP?U8Mp@L45zh!AFj6&}uK$-8(-?;8PxBu|tmm-1g5H|a=)%ZW
zw3Zu_yvE}Uo2oSb5Tr?-5roK{omlILN&>Q^*(AYBjQr$35p4dMP?&S`dzexZr})`k
zu}5*g@ahKvI|+%bIsZMIhO_?0p(}-g?(>aUQ!=t)%%VMX3jdU9yZPQjK53DSt>f0|
zf(He)>pjoG?KBl4i{2QUG)9p;eCMCWk~zQ5hookkrY|=+H~AJuR5ec&3#;Dvd)t|V
zxZ{~{<8Rk(3V8~VQ)IVuWtaar(|wC(9zQ?&&T`i}2gq{wzioz;OZ6>;6}$=nI&PXM
ziMSRnA0%_ty%&B;x=W`uc;Zxj`_H{#i^Q6Q?g?iir&V~|*s4lEHq*UpyC8eYut!f1
zIT*3Ini#gFegSJQBLR&#HqGi>(&Siv%W}S0L1}kaN_u=D^wX&{{Tdxt41~crB2@&>
zM<%<s0+0ABMmhue1bzg}8bk=Ex1(OxNbCX<IRj+%ttT=RD;<!7RW#(27bEbH#>ip9
zYTA!r=ib*^`iduL&l@gKEQyaiVo0KvFG=nq%NvLEcHvp<3Xxk^LX4ts1M69bf=;Qz
z(QOctyjhiapFWJ({t%?QFlPOPCOLrE1_BI-6}ck#gKt#BIZwV>&SXOLyR(&}jNwJ9
z^OWZpYiYtU52-9D1S#)2`?gSwGa79<*s!(_h#SgL1mm*1#D#V`2sTNx%@i(9;eYGB
z0D=WiR>V)x@Z5Gr#N*fPa<(IpAu}JhCkh{cO>?fga49*w58enY7W425(DgBch&Q&5
znvN5oes}y<a3?QTl5TFQTf1H3TxR+J*(!|1cG_$VAStK;TG8b_--2Vvg8%*D5Fwdu
z!^QP-fK57mJraQwMC2kSI~sd^+3z|H^y3Yj<2LZM>3N+R>rL-WFc439kAUa*wjzz!
zF8Ssz?;$=W9+lf}zSixh{x8qUSFdt@d7dsTp%A!k32X@e7m4gx$@NdHP{WMH@9yga
z@N>)<>TBoUFL?rPcnh6tisyR%$C((#T#8w1hgzns_$86Cu#Vt2zBH)hRr*j0i^_O1
zvY1Fg3$|J@@X~mb@tXXJ8?FY7`I~G)z8hRpn^_+=QJt-qsRVxc%QxZ~RN>Gt@j~~y
zd>p?P3DWn9Urgy^|Fj7B*p^h%S<(0pV+QQyZxdPnOo(Jg142C|2kp3c{+TiqU6RaT
zG>|wTS%1d{I=j|Z-|RQbm5rUleyOSoB8KZloTuLmRe7`Zg~__8G%<MM{0o=$z;1Ju
z^<*uxCLiiMLo7ZkHF-@hmi%)x3eah1ciw`-_J=6#%11G<k~iZDisjkh`q^4IGN;*+
zvRyp4nq3mYT;+BoQt^j=c1XM&(p~S^@qR&So-mS#x3H{IS9)wb$@<3a=>0i+qnq7-
zP0WrA7L>(-KT7tvJP<}44`$bH@!PmO2}~J|x&B_rArnQ{1@QN~Ygvi>$iLQ+GZ9(a
z5bDH`5eDHfv1~u98lj2&QsB1BR2E>X>dS`YX1R?>n0L!i_-78x6z%hV48U49jfAhQ
z$Rt~5N=Qh7W_H{fVVoe6PfHqJv{$M*nTS#N4uv<G`(7C*nq_q`lMo{T5}Of%`wf@D
zSAoN?RKg<=GSE!Uk_hjDFd%zp8?V3BhJUr{;kY2o5QUpjAz(pi1eJ+pghE{jGkkYU
zj#f;S4qVWIQ9$_y@abzX_we0CIt+dGHws}%RSDXf|NR112<S%3OoGFLS-W|aZq9Iw
zKEo)dJCFzA=N~pUuOvUM$^yLI4}iCW2QkbHq3BXIKb#HN8(|FdY;Dcph~bctvG`ma
z$Z~Z8i~7XVn)-!TI{)ZnL^se*y~om|@&+m16$z7c=PE3jzW2jM?>Fn=3ncr>1Xg{<
z6Wt;WhUQ%L!?_wVfrI<2!(xgd5x|A&?L&(SCiU6PP5iLgAMf3iJ8j^j*q_LzTxtIk
z4)#}J)3q#`dm!8<0wnjku?{VH@3UCVRAjD;GBV1Md#->jY=!=FFC)_adoKq&OCJc2
z4TwcomVXX-%TQY7I1#Y9QiQQvsGIbEsGnFZ5W6-glb~Fh-D26Ipq1HBwT%B+G^8hd
zcl`VD-djfoAg89`sD-SGr^-mfo1@N&Q>)_mM0?kBpEuU*{y(q!_p{j{G1+IzQ`KYm
zCWBGUt<p3mfH*F)Dd4x#ZB2WqwNAJywjS^O{C>KJb9HqUHIR(DC*or<BhNhT(+1V$
zsBGXX`tIC8@#bZ#srr_s$>8Iwy~$#F&%7F=_F!C*`GINl&eX_wHEa2)2*sH~iDn@w
z#N_6^6NC}a#cz-2`X4>w!a;l1seo3n?d1>7HB=Zp)YY+#Kvica8yo#$bjlV_cV8k}
zFiuiB*ziP==)O7x^NydL56fPVonVg=^AH9JF5BOFWrlGfnPKVFh7T1p2LK=Op#>ng
zGuAG6eSy;DZ0my}^e&YdzpaZMia!*Z^qHEX_M{!5JzWfUj$?Pk)X>dM&!u!EoT57u
z^PpFfqLY6oq{k~T+L}EQUT6sK=@dSQREp#xGu(>{x-Rw+in$X55!nf;SL*#znB53B
zqVPHrM65Ae4=4e0EmgI?s4&2<#BY4a;R<+7O)UX0Mra2lv2mJqqhp?zZpu?X(WE#8
zp;=KXgA}C)KyBNf>C;E7mly|g%pieiAg<?!^XP22m_Z=w<OoT_Dbzp^^b!0D9(d;_
zETQ(RV>pd5u4mwZFrEVT7Mto)d~Ui2WI#59c7?Fe!vYX<)qiw(DkJ}jsTnLSww47l
z_NJ@9!81*HEv6S(-zNa!FgLKbiwlJ;;{Gk$0<v<i3E60$(<}zc=KjH;R>p?K5~pZ4
z_wruLv2Y$WWmg#nDRO4@9KU80r}Z_{{Sd71aO)|(cgt9=)2B>Mtfy2|z*1DvrO}5C
zt#a}SMki{aH{3Q3`ET{FULz6VnAN9LK^!LkmN&0>sorH}UCa4Oq{gwnWCke7-amE@
zs=B5{9&zWbBeIs$Wofnx^{T;pt`qaZh~b8;J@E#+{cf8J4X+sRbkx6NuhW`e;sUNg
zL=}m73_^CX5DV0Gp?Ueu0|Iu?9~Q>)`U2WOq~m+p9h|oz(4~7E&tNp)6*%lkupWpV
zDGn?O61WiG=s>P~v0)qu&pDGBjgLTX(Axo7H$@jXcuC<zn9&%=1+^8AMD)7@LppDE
zI>-Sfu;g=UxQReLdVgOCCMJ*{et|6FaN0i1*>0tT63ZPs6bcJN|7;AWp|zitOXdb_
zIy>X<+-k_be~k^x2{e+-a&WN8beK6qaPqG<Rk&GC<tG4g26ZV#uqaFcNYuZrgD?~y
zN=U%cotNr95fthF=VpZi8UR}{4%s^agu2Rtt~)V%_|U!4>~JJJ$1O3lEMggdtlEKg
zQS+Oguc3!tW5pjNniuT<$tmGXoCWL|SYXX?P)^?`z@bG0s_?{)$$q{a4(J>n9#-Gr
zjf#xaxUUi|yspn#$&QAqZ5Gh*D@=BGbrm-6t6!xL15$MNlj#3P*jI-|-ELhYtw@QK
zq@;>~NY{YUA|TQ&C5?2>kkTdPAT@M%hYU!E$k0P~gLDk}{q%Xx@p<2KzVG{wx%h*N
zx%a*Az1LoQEtIL`Bh%CG+yi-6v-`WH%>V8vef{G9;`DwK;=CD~I7$(a_D@lg-hoP3
z)#6TO_};qbAtm~FuB;^XGk&aD_h;B!17V>%HOBvV(f;uN|N80EE@fTelAh3N`S4&&
zTryM7+CYf+pi#wEH^=SUccY(}7IyCsvznYAz^e2s-}eIvoAI8(*Nii)xR}KPqyCd-
zczlE2E*Tw+!QqS0Jw+Pg=1i;VYd|;MX$oNQeY?=!CV028_<Pq2yYMMcM_vq}W1xlm
zvZ~(_na6rf=_K2N=Y#MYAiG}gTb3Z8)S18Y^|*(7DK?1g7yYx3&}2kH_d5-jIEBBD
z;WKBEi%+p4`*h2hyugB%7=62M_&z1nVbcC?(o+&Ln93s40uF?TCBPq$s7WK<OmH}|
zXd}TEB_jKPuF&ESx`zk*)J6vBT%>@T%hQJ$Ss1-WA_%o76W~B!NsaVD7`+zWo2TOg
zd<5KEw6u~hZhZB~TsCA?LOJOD=*h&({X#<@$Uk)>wR+9N#nsuKfV@W%;o<0*X~A;S
zZWV%MysMdE6cUO>S~lnjC1|wvy#cT734dKHlb^`mt{6D%Y2-c}`7-Cpw3dA8`ku&|
zVJ44*oq=w}C_I8<(&Xw{xD_LG>|-P>$tkZNGZFX_I~hMnN@`ZVSC}R?{*l}-+(t=M
z@Fa-FC4t7S<<`bt03La#vD;AS4suX}ffZp*yH1o+;i@vB$@JQ=n?X+SV!LvuYFM$*
zj`a($k3P|KI%~z*eChC<DRY3>WL(!V!g(9XB>`AGY>swDzqz?+zG?Ku@8I;>4?CNb
zFYz^Q`w4U`vYStyK7HLK3M9+uB1S1b_jO>cr9Og+UvU))G~OL9iy^G)vU`TPxxiEn
zN$a^?9!^2`Z+8sd+qas286PV&usOSOFd(e&`UV)huBf%WjxENAnd$!DJe$%^=v!{z
zI^J5*bVur8FD&1Uv`^%*c-zUaIXZ9~cF|_MF|X9@SzFS18#fu#r^<F_(&K1-K}Ntd
ztp<X_8!p~PtjOJ}M$K_{H`pa=L8=WZJ0TJ@5@yjP2GU*ReiX>Xsd<J~G5mznWmJ{U
zz$gxH?$`Iff0nLcd(avnzFtUMkI;x?)KY<HcYL%Jf}Rl<K##@~RRP0Q>-IFpK(av!
z&-j4$QBntUzt^E45>lVUv?DqCmDq5ck0!4vI;qg|g2fr6>+dU3yWGlle<xG+IIzCg
zz@@W?DqCbdSYn;A-J~lv_JQt1{hrdM8Vrn#Q<9|V9Fb9_qjC8tF)j3wF`2fcf65m)
zo4!cU`Fb7wwKx+62wYcA(p3{vqSq+x_u}~tkVGt}{Oe+i^5L|A$C-4nb@ONb7oQB>
zj4`+q_$75JGF4((6pB1ge~tm+be6c%ozoB5c?n-gd}iop6XrRWr_8toFrtAT=wS&S
zgR<6_Yswif3Qy*LIikIjCr13Ta<zW!@UxknNML@*vFHjeTCCi+O1XjZvpybL;rp4M
zYjdtcXMzO(Pb0a1d=kd?1-KkGRd!yLd^I^=d14I5KF*jw*c}>t^_jCG!xYa*cPWP*
z{Y#)s>jIbUnL$(XNaiFTO(U|RbZsgIRmt|fIlqAv8;hA0n;wNisq5>ff&|m-E1yL@
zdw;LK-hBb3D4^`a+UMKi9~Wqx_bE<kdu&5&Ub79CXYC`^b4J~5@wSNN&SFo)0R2#z
zmN2}x_<kwE-vj9U2;jbp@e&@^xW=?-F@y$3>qIMUfvH?ClZuoKYD`3p>dFdBCevQ6
zrNaSB*(lg%pLF=*;KsM?ZDd}lM|&YiXaXUZr9n_Eal&r<rG1R(;F%!19~We$i=_<V
znBMjyAOg=Jn2feTIi@~XB8!A5Job1rS87j1e^e${z%nx^@1>Yv?X&QV<n}!HC1ot(
zL;b_nNb@3;Y_5P2(Pd@%Hyq`n)z5TJ0lwmg;x;UZvN9D#LFO2Wbw(f^R{;Ljov-AN
zKS;l7=Dn`59PI^IRf*-oT?t&q)<CY-VT8mxtUnyAp;uP5cuD(D%jOmWN%394^Su(|
zgCBGj&DUcA7b(r2sO43-TH`TsM@R5x3m>ez@W<!of4T+=lldwTYbDlfSg)97`w9#{
ztYVIqRjB}%7waZ7bRLPAe~SNHRMXk7Bh^eQPZn77-GPdy!Zd7kV!Se^1M8RN`Myja
zz>bYrQW!11OG<1mz?+$s#RRDR&SFH~k%U!fVhkw;iF$URinME(a<ut89gfyiv7q;f
z^8<;^f8d{*TGpw5#7jjZY;s@aOhqp~yVU~%2WE+JoxR{mk4jN*rT7+vm$Z#UL`S*V
z)8qxcli*=Jm-Aj}Eomt5<wWst9-W1B5jAi@4~U}_G7?A9qGY?kYD$jkU>6ALanNeG
zLA(^2y5I|jH+Un9l&q}99YehuVayRuNG+3l=AFog347=&k%$N{WL+>|RpXxLwceTJ
zCrTNX!sXYz=<JV(CI6;qtB>28o>+wpX$#_luF^DcNu&0q@=8%LB*IJ?CJ@w6=(~@c
z!U!h<%tRb3Vc$SK5kT1bJw6rRUHMnP1kA15gNbzj!$uCh24r~`ZPJU>1!{@N$_X+P
znqpIx;$K((y^3u8%pcz&K)2fC!!l-KgIz5o;(s35KR;QEV)99uSPWWxD1?#F_6Mez
zjtA11ztUi@JjXw<o=BFnzo&OfMHI}o@_8;RpEO8?+riOsDUyecjJ*sy;=pG0%>Hq+
z-6U4ywGu#<isV<Ttsn*5d9l$%3U%RNcc&f5I+r?Gut;_`Gv7w+sr9aZROsPjHujo>
zn73?taD;lPzKuMTqxh+F%FsB1#ylq|>CJtPebV_`h+HW^;6#&#qYljulZe=Mog}s@
zq>7=bo9nDVIwqH1%%Ko;0%)~%C>IeYJ}UTzCRTTQQM5a!{GgD<_#BtrxFc5HN>yZQ
z+=DciI$TZ6DYIKzr0!jA2RY4>WeAIM3eyD1kYtY>EaVPa%Pm#kEUDo|HneKHXnD=N
z)dKkC{x@m-MG8EM(c1-&kg9Y6i{`xi;BJX)rY`|p8P(5nA~%hPHLcV|_$(jsv-=T5
zN7RQfWu7B@x&$`nn8Jm(0U>b04?WS`4|6)@6cr;=+*iMh)!0mh*YA6aY~O7g<Q^sd
z!Abq>ozL;^oERg1(RUeIcazGf=^otKPz+Hf2lRe*%uy9V^naa}c(K&)d#YmfAzf+u
z&q4pMhtrqFPh2hJ=|t?xgi({nv5kVrkH=FD3v~s~QfWOukoht&*)gp~mg8<=PY~D}
z+da@MpM%!{C$X{(@`bo$(Jq8KBMFPumkN*W1p}4wKGQtbd=P#rnn&kRM;PvmKn}LY
zcKc&~p@}RK@0UySF((>N?F*vlxT+SP2IIX5i+IpmNL^J>>gKe^l#sjnprO2Ux*&*C
zC}JN<$WERyJ@Qh98m>lmPgR4*{ovUP_RpU~7_{$REibOK8t;*qpFOEvee8XUp2(co
zL5_9K9M`-1TlRTDp~rGNwr>EL!G<|$So3C(?MNXD-#N1W%W}*k8HbV{k3=PXT-h-V
z(B?2JYrQ~gz6#QJHGL9}(GTp{{Ce{Y1xfn=_F9DT=bYDH?jh6sN{!~McK{I27BKOt
zB%l#8NuVDPIUZ4|F_E(@th~GALc9JDa9XS3-qZi_=c~44`s+?TaQ6hi{p(zC^#>FF
zr3LWsqw@J<E;^!(naE>lgcb3iD@CstL(o)9*%Ak;uAtq)TW#48Mp~?ITG&c@l2`ul
zg6;z+<kARP&C(UQZeJ&gx{e(zb>12)UvXy~;5a-yTq1am#XhCw{x~V52!A=L`tvaT
zabDf{H-DSekL7%xm&o;IMy1YRlDXSABV~-qfmp%0f%3wLVBs{JnLS*h&PA*iEOAa6
zcBqjAzd}xWROuPUQGAd(SMOt5bQBSoXtb5i;HaQbz5LsKtHrQK<dA-<q|M<1_!-67
zE19r)8GK3J(Ofl12lE!I<9wfhAg7bg(^m~0z9DD|lJyDqAsex=nAB({h6i{vP6j^r
z3gMUcGs*osErFWmC|g5>{pMA^*ZTYKB;MB3nEiVEm{bsl_4k<)szK_GZw%bCa=i|u
zeDwF1b3qh<$jp9vl)31(IEL?gH{h+KcIbW>J%3N5=BJXfvht-^Vpb%GP#FYpwv$BM
z9Zb|5$lgiCK%r)$o+njh!co=cgGLxoO78dVOX;Cu53hFIewW4DXPyh(e|%kiX`*~)
z0W<0MU2}La-#<t8?~?WJn<q?yE<s3RnaS#HebsPGhzWh}xhxjdzLMfG&u=X3IQW|F
ziM}a@Ct<}XbH~ZbcIoyZC2T_ON-^}r8@@k)OttT%Gz?1gu*dL;U&v$S!<#?Lj;RqX
zHt%?AReO=Du3I&<s)%|*+c8p>W-+atGkf43c=P=Ww=wNa<uOs7FJ_{Jp)-;BlSw^n
zMKlIQG~qhgdGz&{tX_WGcKBJD0TSZ)4k2Ci!a@`4DjHttNhPI%MvP?2N!x7de4F2F
z==GCvR$7AK)}b9rhi~p0U-GIGXiL_QXaL=hCX?|Ojd&@k!en~P=alRM_0JS^W3#a#
zm_(Aea_Ew@97fq@uak|+(j}Z!D2B&1tM`=^oO%+cUw{01)-6qE;P9A?${3Xg@{(%}
z>wJ*{@9Fi8bn90;R+BLX#~kl#Kt^9`5S2t$qi}NpdyXk(85+{h<@&P#8ZH|{39!zO
z*9VQ8(*0=7F1~H+mc8?}zpDz=GDYr(tLksic&J{v3smj_g>8j$rnJqzA|KWJDUS!g
zGvip>2=_lS<Idli(Q`L_;Qx^s6N6j$F_hnrEozNr>uIn(W-n)z7NyN~cN<_Vsw+ae
zSPV@bQ1Mi~$LhD<4`t@N+OilcuxHz<lb7kshgqF|^>RHD+ixea0e1L0S#YXEhCfoM
zAgVvme~;7bJE5`O@T>nk{%EfEWy(Y>ysVR5douL#62oBNFu!gM2RhJ-II3&Te77r4
z!Wg(Z-ZqdV?G)KHyGw`$M@RqQgGWW~VNYqoNOpb8V<nwbnb~Q$OL`#)KU&E$P75}T
z3pxr>0Z#16yS2Cas0?G!*+U|~y=c1n<xCJ?$D<E#vIQy6q79CLYx9Ux=sb8e!%J0t
zg)pFyEivuDe6Y7zidhF-ZN9jH@yUrus_NY~GG-sS0@E`v*8V7f8h$Qa#+JL8k;~zQ
z%df?E6mFUe5>zD=;}}&lI3B$RvoQ|Wk0~m3FW>+nsRFG|#nar)0n?3j%Xqu0Avx9*
zljAKsj!J-ZHv;$^MH~s;!Nc2`_b24D@&C-Hn$B%8#N?mc=I+C*aP+DBHV^O4k2CO}
zSL>Y&ecGw?rKOny@)Z=1fv1tMqKVy1m|J>)tX{{XeJxT`MIo=};^(sG?zW))TTAzP
zP@9>~)>wkUr-#)G)$t_ok=U%7;4$-ys$<7imcqw%)(Tn&U2msy;TrkExXXOq_y+2w
zH`ny_4mE0=T05o9Of0jm|I0qpMIBQJSUcRwe(_POk?*)^Xb^(k<@G|}_~I0)QIz))
zU!qlJW^YFP%0p0{)+PUi!{+eX;NGfSlgEN*!rs!&LD4|(OrvWpz(1o*!mI<Tsxtz!
zi37E<uUbpR(5lccz#fa)E>P^25rG(W``S9Vy@*Zc@ro{3m^a;&zlnWHA*@A)$mts<
ztO}%Q(C(9(4>DE^LQ~a6ZXY-*lZYOVW(;4disfY}x~qu^epn<nT9Kh^y3AKHp!-?A
zAWy4(v>v?Fsu3fqPJJEPMJ{GNdM^feV-49qcbM;Q77{FFi9y)I-I=Z^7<;ojD$OtR
za>2_)8%$1&&l+xr=1QpoIU1|XvQIIoYD+3RQJWJqvgXEejtlB7Js%{xe#Bvs%L#GN
zTUOyH4iZ^+{x~dAy@Q(zjO393iPx68nrT5Z2LMmut(^G}r?!FrbFAFBE5d7|RM!$E
z1}lZ1Kly3i#=*<0_E!WP6QI8SBLX}bfC!ktd=PCGgZV&~N7ja3$Xe-cZ1q`d?hAjB
zLJb2twV1JE2rdtEp;aew?v`1#u}6J<e$;28YRhvgw$e2B2U#&{Q#W1c?hEAZ=M=iy
zaYCn}q7Qck&~<M6UiwKd%$1LLL*uh`Nv?@?DB(|wH*WgX`Mk&fc&q;XWVft7L;#}s
z{f<TLh0Z~(WfHF~RXzn`DOevDkS@q*TMZA;L7yg&Fh+?Sbe~ZlW<&LRyO|)spw9+q
zvTEkME=-a9u6vVKr5yvD;JKBz5KK#Y-__>Ug7*PdYISa71JDHBqxyNWJyV*mXh%dE
zOb1;!1TP$(dap!BXdZlu4FfOIGC@Y6r3cD*57j{hQM4(Y+Bopdz94V`klKuhtB-mD
zI6=#AXq^rupx?h~x`27c5ej%h{cOe4Lj<=A*m~1DOq7x@57MJ(-XMtE60odZ7s|V6
z`%2O^if2Tnt@W}bz8g;7Q-f?c8S@ZoqllBfmnMBwJdCC>keu6mMvjUl`NprMbg0Hr
zefC(o7+h}8duN<>fo@e>D$N4e8^EU#8Zl=k@&N=gLe8v=HlA2MoEFbmxS`eo%(4M6
zD3Tn0Nq2V~P(84|zC?AFhyWQqv{|v_k%q$1#;=;na!7B#{?8lR68RD+Yv~bt3mU=*
z@#KXo+AQpI<mHQvoZYuCUKscF1Z=@#hALwAq=*Fq2L^q0+ad|@4<2+HNEPeCU)wr2
z*dXWwpe{lMJ?|vZ70%R>it!tHiYKp6OYOH5_nk4N*~ow39?lyVhB1~M6Zwb?J3x;m
z%PT4zbZSi8k?2j+iF|Bjmz}5$qwSZKcGnq4Na24~-2X1)$_^ivZIv{tTPf$>tmo`C
zbzAD_eeKN&zv{+LcKb|9UR)Kxo!dT8TMF8PCfvw~ok!b$6_^vXmmJ<(XyZNt)<~*>
z>yYU6u`)Y0@ww?Mkpl}rt4-&@<~1pO3#7nT81Wik#Z;crjkt&N^*jcB6>^>c!N34#
zf&i21w0Mxy@HDPN8x7HI8DobG4cl{E!F8Z6J9wV+c{MB*euh7Uqb3w|UXXiUz$nUF
zALma-=K#Fc)4^%4wpKEekpSSJOtaFJ`|9Q5W+=;R4u!-Ji`QP~JslbHiu&_uP|h&P
zfm-PFo(atgw6ybVD`)7r^!id?Jp1|{$v(D0M+~{BquivL5<GmP5Pv3I`(T(l=I{;s
zn4HK~TqKvrDB%nb!S;&|VnxES&y30REy+N|Ln-szK5GcLCq`0I`Rs$5@AOE1z1#|V
zjfZgke_={3#Cf;?D%i_{2o_*`>+b@47(`<6<mGd)9d7%VF`LQU2)57cw+()jIc^8G
zD)L4e-+}u1`5_mO&$_ia;u#SQTelW8Jis4$b)LC9&F}9Gj&=|uJ#d}o8pC*dm1xw|
z_2Hdx3U1J~kSgh&({1Wto~_z-+S7d~r3=b^>pu_;VH|W|!d^V~@<+pYt@>A^FQfsx
zGWzgcLYMP+{;44>$6|r-dJq^Dl5kVe$HC2q<|C=1t84uMj(Q)M8i9WH<K19JLG+o~
z_XI=mDKF&w$g;T0Uj<hZy+73Mqss6xjL`Ky-&UlZ5bFr+0mqL`zn~Uw@3ULr!FDJs
z_)F`FgdKXBJ@Zu--sZXz+pIqvtPIqJIPfDi8T*m95I4e}bHjumoU|B%M2my@Y7o={
z<D5yD*p@2waY#HU>BcuAxuHA0sBpTS@jyP}t@lbMpT*QRD5Ul?(}!p7UntU;J`f09
z#>D7SpJX+QGfStv?_wkOSXA@plk<AWnc-HPQof2d-(b<>wi>M~Pa&;LfQ=+x<oWR9
zQ3l5$qQ5=IBb^FD$xR5pUwWPx<q(6npfmHM+7u`8d;wa_3IkU#53qdXIb6?->eMq{
z4BcF(`+edxkXvUJG`IeB=JXiAi7(>ak$nzE%UM|ryL~(;yugUjfRDZ7vO`4sv^Myh
z)X?=yeWjW8{8A`BvH7e>^%NFNgMM)2(Zb#1<y=Jj^4SMtWAwpnecMfnnd*hRC%juf
z?AIuAF3B1Ljp3V9n=1xER)&|mMH#w+?pv-GEwdJL#xJSq)Y{<f#s)L<)~=w%&p`$!
zHh4Yw!mRcplyC1@#z8+*G}O>EYF3WzSfetqYV1x%%4kk|`=89E8Mf{9eRD>&+P@dd
zg*yj{7P-@BtnhxX%^;Px8BC_EdAdBCFh6BE9ytbvo~XlfM#u(|NJ_M70#jg5J_Vw%
zEEtA1Gt_Q*Lm<k&j&FD^r+JR?hbDQGn&6?_qt-V5Uv;T?mf!I8g&s_>F4>G#jks(k
zqN2O#9=Hq+jWfL`X->9`ZBAr4#I{jXRP3}L5*S&bJF-Um#P;V2{q6nj`ydichE+|5
zK=fF|^K>Gpb**w_t;7TTg8)WyMERy3AeeYBUEGSac=f8r3()kDJA8l2s+4LCXw_DZ
z7S{Hgt@fvOD=ybLp?VV~(<sT(R>A}#?G`&&D(*l8Zh>F4<mz!iZlm#w8~l_FqM<QH
z9Z0=)2&bXi*PZYy?S68UU+T8<J#t(mO9NkfG?$r^cfXHpBd;T%c`?M8HW9DbGSJq8
zW<4Mb=nZ<p^3*M>K>ADUmX)zK%6md)4$tfr64iys3IYOb8#w7ldzC;{!Q1(R9zIn%
zVdCm|vS7yyXXe8gJaLP99?pIpK5a?Ck@3V}HONX<r0nymUWKQS5C`thghCy$N^_Ta
zpyCol#7hXgewzZ9hp^43Xv5)XP)4UzziDWA8QMlY#bc(p3N^3@5*qp49FC+X{gik9
zcUkEB>KEK?6U&xycb6{K<N>qYy@*fU)(^{er*=P89TX0Ls@3ARL9s3eijrwSf-hfz
zzPs!#V(9_R)qNe>2#5Ssil2N|wPl}Iz2hQ&q$RQ>uOs9d<NVVTe4gxzb(p;n=So|f
zVfuwB8av`nQP+IfN{yDyeEme-8b7GC|COrm@WR~J7MGnNM(rHPe#i=MZ>=8Q>0}E@
zMPDJ|r6Bb!fMPH*Z#Y-0z9Zau#wq63m*b5Jp>ZE$-Vd(a&gz4we&C$-^sEuOXM>&U
zZja<Ly^8LT%at>ho{%XOiC68ozCCxH&ucOJ<isqJ&TLoYVjdgxw#ssR5F2)JC4f2C
zbz>)y-A>&xGUn2Z;W+`n^32V>nV+Kzli`=+dN+ToMq^i$H+(15lx=(e0ycnP#sS7Y
zb;ldStoaoHD|#G|K1)eI{Lj96<TWM|WokSG(4-^2*r=TJ`mrh|Yo_Ph50fnfk%*)c
z&>>oHHdnG}@oI02{hY5{Iw)Q15P*+6Ot|3Md0SHWt|pgF<}WOUF}tD3x&^d{m7iNa
zjB6pD4`A@t)_=p0^wG9z!QX2JkMjU`PmNlT7?n=Gi>5QAbiAJpKbTiCaBjD6F>sH^
zqXSct$b>!8w|Uj*wbR1|8tMKK66sNu7<;hLCj~i<J@#*~d>ONOhva?6J(j$$N5m{Y
z0&rEq>wV-@pzv5u%#~bhenmlx*mnVj+TGuz?iOFJ7lNus2xr!VXhe@m+Y+>ON1*1(
zMG|xmrTrpx*Ov>qjyD)9xE06Diq475R58#M3<X*@e1W<^<>vb8SI0y!tZ<u9P&%B#
zF#ZY_ak=@bFi27{EuYPGdVJ&0b44ZbyJDheK;)T1@mc$=?|zFbRz?nH%zeXHN@MyY
zKI^@9?DWR#g|QBm-BXT|#;&}aZ+U6MaL6W-Ei$RtLRf{`X||a3yFm5>=Nn!iPMw6_
z1@DZp9VcqmyOoeunZsr$7==I4ya9pFI_;Q9=oD0j(v($%=f7L=P8X5+bzKlzgY}c*
z^UzskSsQm#7%=UfRGap~zV&8hnFolv^(nQwO!qx;EFUhd>zVvH%Rbo!zsfjX7ehSw
zm}2rIrrK}~+I;nqF18p7_g5;v?t+m%8<jaZt}*%0<2S)~IacWR#HUj6oB%XRAhJ*-
z@C2_JblJsA-dpuTcsYZu*bat~>9x@+c!FRTn{6>OQ%=4#=)&@QJzkd_zX1K{dVLD4
zcvYK^PQxK9f>iT*6%8ZTcEe&rUK1$*3x5z;3i*`owG_d>J$G~M8u$a}KMKA#+{ES-
zE`&1))}i#3k3CK6k6*{qm?Iy$ZzIbGtJhxm7x8R|P6}K;@}tWt(yVw^W%_w4O&Qtb
zlsc-@^!%r7qqWuwOopv+<Gf!cE$|+IYLk{>;I~52wqLeEv5zbkZrgnmOOxXKiAIgX
z;iP*+j9Tx*Sy{rP-stw7Pr8pv3xfL4ksqstsZV2W_&s4H=Ne%Ebaobn^5E80XMHlp
zKvQU7l{HE(f@E5+RzDBk9L&w>BpvNCtCSMAbrHvkOnl0YckgV7BqQLRlHU^|Od`C@
zQY!xxc|66{I8Js@0W3u<A!9z1JPn+knd5-i>?F!VnG>uNjVjdiavE$hNpXsAYcpn1
zSkfOTjg5nYmG!~tniUxRdx>8$`t?0`OuJ}YsSg$K)-b(*9pMdoz1`n{>E=1xxAEOS
zq57+}XF#NpxU<M5%bo1o_eiFxZSjqrruizPjxPmIP-ZaAQ`)9t=P5mQ)}Scn7wEU&
zt@M(fOytr{3^5zzI$e(gR#Eqr=cy0l+4y(QU^`ekQ$KOE6wmG{4ngKU?sZYQ>N7Y8
z*era@)mppauKle4{DLj%YC>lrA)CQ!?(mJ(Fte2G!*9#VN|WJH(piq9g)pn)#_c59
z;SIqv;r(^ia|5^wtoU)?ba}4H=yM$*PxRGd4zuesvaK2e7Gk68@Sc-zeNh3%5$akn
z#>iUR3);Ani&2>mQzYJJQXa`O_4A#=fI+nI<lb&r@Uz1lr7A%cKnKJ9vv$%Cy&}Mh
zr_4%8v~kDp+s1txVb{vA57(X^x@2%3h<AG?pXs!WI#HzZ)NVIPk=-ub;v$bY3Tf4L
z^O<&y*V;tv`r>dSk??Z$6|k+BYvEbpII73_QP2KusqZ-Lby9@KLUdNWHU!jc06892
z`(Y0-<2pHG05p$nZwQub)#xgJbt>R_E^={L?3yd;1zyS?YtSncSnbZ?@Z6!ju<9M3
zJQk~C-+|6h3OVHgeq4_v`U%0QkdukKDTM#4xVR;g_MYKWQsYnIh*vqakNv*qY{zFQ
ze7J)E<P*>8gO;G5N){X+cfDBQw|L-HRsNhBKbgukON-b=!iLD@6}#4RG1}qwC@0+-
z8g$-I;*R>7J0o`4UJvV1MbKG&<iUZ6h)c9iJ9MJ$Bi=wdG?cU-jq+WWGfG@tj75Z@
z(s`@cyCV6onA%S5`E(5%=k3ggY^>KurmUYE3QC1u#GkZvc1-Uv2g=tYh*@cDo_kZ|
ztV!L?pg2}zd{B50l%XRL7j)a=g*Fe0SJ_(oBU*OHk7hTd`z-`i)G3oDNgtQ`b!z|H
z7&=rBPC~nEcaXyIN@o2FlNa6ckp2*|hmpVyQ^81&v|I>6^!`W?8OqPu4(|zh!>A?p
zq>au<3&VMDWs;G|oVFkzGZ7dkAoFumAHdVbpR?aHqes*g$%qq%025YPlHGrxmNUOl
zO91dd5E)9_%t)CZs?S=SmW<_o2-SV&y6R+AE}vvq^tz+b;*<U5nz3~&G~YnUMn52B
z-0F13H7acB%Kc=kET{N<9VT)g^e_h$3O{eG<QHlxkx91kq;_+dH2YD<zkNtP9xL!{
z+jVlZMjm>VInY~bHsN_%rt)gVVTAOi$^-u`UH07NVQq1h+XZptkEUzFv8G+ajh_DG
zy)Rc+n}@^3Lag4WLR+rW7LWDl!)HXS)8g-MbQw^VxNLm5v7kB8_1@k~6!&xAt0}a8
zNk7ZP%(?y3=`5cdS=e@ysBW-7o@)&~Nm+DTspvQf>QlrwNZu5#h?{ER=BUXG();}2
zWBui6Ko_#tRDMX1qw&hns<|uGUc~T7*yTonigtX|c@AU8k7|4rp9y9FbCwkt%415&
zY)IL=^3(I&`MmQyu1L@tS!SoIejwbG<=l5MeltKimIK+M5Lh^TY=PcS+(m7Y3%iNP
zgYkR%iXN6iw}mxQlQ`@;-1)OECQ=Ru#OAOGX_M1`IiBt8chL<XP`|6`e|IduK3TG$
zQ)@MGP27L+9;jFxtZ)*0w=Z!s^_$K+BTM8KJUP0#x3!llc7EaY58zo}$T&b%Z5Y7t
zk~ZNGeURHb^@4G|?#vI%dpjJ}8l;H|VI@*k-iyXJSW%|2UZnm61%Z~8R@G=X57YGz
zRUqJvSi&Ss(Q>3om&|c_p%>o`hS&;o5Ew-$b;!udAXY6W>P}(x<1CM^L5+koPuuCq
zEBa4iVIlT%S-Gaf!A|IELfQw9ajqMP*Lle(mcNyVgcAc5xuRf}QfH5Jm>&OgrzK5C
zDJ8vr2|-nudi_g&n!@w5STSJ&;g?}4<hiSI@`^@4m7Qsi*rAQI^C)AXT^Iq}9EA_R
zbF<gSKe%z*@8j@ZC;qAX@;9J&`_0#XwA0RHK#Nzf(H@ZK<==2RIqq@veVYBF;6S3_
z=;xrI6L&+Lupa?>?EFDdbdTMaM%QdMjOIe#M~R*U`#jaHE9)dN$G+JvKPq7^opw2I
z5Z1CEwn5aZ3(O}C*71SUPtSMOblffM10DM!n~LV@n<E=4)cSUg#q<*l2WY7wqjrU`
zq;vM}ty-9wX(1a+u1S|o1eatCEe>t8;&EeGRBh?nYo(}-5kj%mx{Z9Lmr3V&*Sk13
zMx`J(v%n~9B2A&4>ZrhzqZIxr&jEa;D7G!94aZNAwVKA^)v0FBn0^>YP*l;Sq{%yr
z7vF?$x&Fqr@M>)XXK3}hE5$HZZ@*9=@V;Z)<HrmtlMkKX&WB>w0~iD$1zIFxwPH7W
z-?Qo`A#hpv_4!s|Ty9oYJJR>PmDlXej+kv(PXF0v+jiz@CsMCg?0&Eh{*L$0bEz{m
zGsPn3tCi+z7KMYI#3`?Y)KlKM#*WWV3*0=9!Z;et{q-DRZi}})m!peoW0gD@X6^so
z$ow6~zBG|&U3I@=oDWdsq5+Y^T56=;SqPlhx(|<-b=&AzEKN-Pg;9fkMew6L<d^8F
z$HNZati;<f#Djum{W$qhhAKtc`*+G7M}jg09P&5}!k?)_M#ZBd%NN%fd&-SQ-x>%8
zYZRN%Iy~zv0Y+L>ezd}eJ|h0j9Gito$t4Jdtqqc>ZjGh5nmgWdRlJh!2?BUhRk+$r
zw|$(nab4J@1<|l7a2b*=Ey`IPdp}(9w`{HX;2St>VP)us6I&-j4?@5Dfi_Cd87e<v
z(0H}ymGb2cVNy6SBI|ghbA@`&<wTv!h`Szo2P}{q5sp>SnBTc`C)_3)5cnp`Nr(}2
zT%|C06(3~tUH3UL;ulfQWO0%4U%&(aMkV2{HvtG`f$Po1w)ZE}pA+0ECN{|Z#hr2e
z3MqpD`7Hsy_w(~4#cB4#c{*g>+@;NnT?v2yxwHt`Q_VL^w|Tns4^1uus7SmvKC3K#
zWxJ?ci(b=#`l~P{ksjtwc>mN^5_t1k-^)?>px3)|VUpMM<bXw1uuJz!763p8N47`j
zU^b>5Q&HlMgQYjNqS$xCt2Mw)Ew(}zWypc+Lw_-lQHn=aw+ZZOZSG{i03tTU9LagJ
zlsQ%|14D9Z!@t{#G<gSjP4q5tTt?G4|2WM^bo>cBA1)o@5W>ELb>*)p(&w^O<631l
z5Mc0R_Mo*+_WaudoC<CQi}DaX-y}Rqo*vPly*b}4v^gqN>Z?+z@&0;~Bb#7ozx(w%
zedYox6P}smarE=SIG|+1xD$e-A1os{a{z_qa-H>$dY4(mnq_zO5HO`*Uvcm50K&Ou
zDmky^8r<jp4AK9tD_rhjp51LJ2W-BMh7>Cp45UqQN1mbg*Bhn|w{@IU?NPgl|8Vq@
zWmvqGM#AqyW|&L1J$W0uNr3pMrz3vEI7paY6vDI-74a1y8)&0~Y$qe>4%G;NU-PQd
z(K*Co(8PQXQaH>2Ig&P`ivmQPYISV~#xrI?#)1f7=TF0F`K@iI&eI+&Yx}(hV8=Zq
zuR~Joivivz1(L3yfL;SGbZJh8j@z0uAGgoZBLr67+IBosH}Dx^u>3+NXre(d!?ZYP
zr~yJ@tBn95%}JSgeujXSYL7xizxb2iz9@Ai5Jrfo#=4{}ST_$ViOswlm;3xw>z;SN
znr<>`OL}6<<FD3ks}$3q`Ooax=l4W9$D=f;$zul?XBW?XGg>?==ldB;$S*r-J09t>
zx6qY!JRrI+j#N(=Sa;R{W^GH06Ta_J2P?b}f{jZdPnt0L<(nt&`>vvIImm7bWMVCe
zRSibXsxC}MHgZVyzGIDseh{pFN%uGkHIcbNW3GQO=UB5vtVlj{M7ZUSXc36liUrk*
zflC+b2^vpMYj)E{?8PcsyvQOo&U12LgZkIGU9AK>kJ0;&N<r0P25|SrMy9psXgd=B
zfXW=?2Tmc5wynly&z6QG!DpK+5TV}dIi&&lsZ+sAP2t@@u`Vvj6f31|8kcv2<4-JV
zQwk2nZcg#61{tLGE|%K{Y?gy>kWsd0in>cN%zZn#<TpYmc&FRe<FhXA9y=QPPMO7;
zHP1y)BL94Ae@_!e^f1qlHjDw=$QoecRPfX9jf}efFXA62GB+TM#(e(Px?e&ow>O6X
z;rd3HfImUBzOgnOvqdAL{2&b~2ASw#R)z@Rhce~{^Y(B);E~Zj(rq`z^XTWN<V?|C
z%#$Nnk<1lZ2ZHB7xlkmWxq?ga#I*En($hEDm|K1o<R2gSbPsg+)zSMi`Qo)zrGofw
zl_sO#5zY|FSKcfIQ$D5@8<iQrw;<veRVqnw=jsdsW$@KU#+f!L2&%?%L9`d+_??y#
z&L<9A$;B4)II-h5S+Dlvy<e>V6-_EYG?6=72aEwKm(Qx>?vs*A9o${Js8<d5`9%QB
zF#_xU^A*MeUSR-nKmK_^tJg5)bw#Ma>W-@KvL5XX?V-=T+eF_K*3%7zl`}hEQslO_
zwg&6i&?Bf_cZPwYDXdvg5Yo)H=_K}q$b4)^oZITV72nB(@Dqp1hvT9o$3*6Q)rI}#
z3%HMI7LGP=Qk(Q{^H7_O9HzanFEN1Z<;*zmbdZ*;Q9&=iW2d5a%ymwV!HpZuV>hnR
zch44Jw1mQ=S-3oO%_l{oPFqp2tFJCJ=B_HLH{v|{RX0_A4Znqpr=iBmDapp#8i-_{
zk0&G86k_VPPZrdXjRsW;m&-HaC4yonuB&4Fv-a`M>(d|(`-3x!F}TE3nK4;2^Ycbj
zt}C)ij@$Ug=hg-Cv;<Cnf;vnYE_rp@wDhMv{k!pZrt&q86zl{v{{VKh8u+`_jI`OB
zHUys$aj7Bhs$PdIAa}TvVRmhYJCFHn^(93cX-gne*Fm~eR110909LLK-<K$nWeLf|
zbR}xOlF&;v`TZ(fdt^n|a^O^&dw}iz%Xa4$8OL;Hr5Z6IOIeCO@)Ya#!Ex|%uKR+H
zjdJ-W%UfEO{G==?i$yuAYxFy!#$+s_5cW85DE>^I(yDZ>ZoBx~TOu8U<o441pM)AL
z(vz^ifIJS&^#Rn=Pyaa6XeJ>UsftJER1g(Gz1*2|er6}PoOymA!6~+g8p~6mN8Bxa
zjG5?*fuoHimf6B>)@2Xt;v0=aWwWZc33<o@Ji!Mp#$cg8KuDPc(Qmr1D&h4>qjSZr
zAVNh}_w#7aHh(vek@X`9MGzZxEV|g-cAKlr<o68~KBoeSL?DRjv{?}&4CwtqA13t1
z!G*0F8~hws_4p!7i{?5tkEoBG#vYQMGvDU<AmRZPJw3fCL2&AzhHpFvCtZF%7#(iD
z9JxN8Dy-CdLd^|byd!H<3ljU`aQL>%2)}u!!0k0jm+%bV;^EQ0;>{UKitjqJmhHO^
z4fD-)s)uEVFv?nS7NFfV4tbmLv4W=7SevhH1xoUDY9^(d1SfwsETva@AwEf1BboDO
z;(L30cRVG$Ki9Z-{m0|9NrrxjxB>EZHDD()Rvd6)SqzX(0|Rsb0x|VM7U%N(UAqtr
z`XAzNvgJ4`hFfu3pdj!qV#DBAJ;Abvi}%uG%wvz^VAG);gremK&<M@*`h|PMfn3+?
z=CO{9oPJ8rVVmhPWzk8Rk{(VCG4-nQ_Ggb+1$n$Xo|RO*$!ciL!bGnh8E=m_FrLin
zHozOhr!RFnFbi+C-ekk^I>L&aG)dmfl%N<U?<RWZcIz48M%h*?K-JJt0Q0a~FN-7k
zG>o0@owssv9>pGqwEf6Fo}UuXrLpQIR`l&QmZ7$nOnCe1%mI+PgVh2Krt&K`%(%bt
zn<uxrNB;!Do7}&HV7I2$dbAUpHdd%6@U_y_R_ZGfTGydBeVp@40TVKrvlWyotm>OO
z<aYxC0_?v4h@~iI;;(X77_PBTtiGo2DgywW?QTl^t57CHICz=E^O`P6Car#N+`CdA
zhmPiO;LAb$)pgAA&5YVgGZfGE>#zr%$LqTRpAK{T2Fk7_ew3w))$?w?3Ju$Ch<z)(
zqRM>JX>@+FW7T|pT*|*%p-i=$G&D36ac&*5KP4vAMIf~36aHy2W7Gv`^N@+W;NzH>
zFpe8e;pOg;pCX0oWPsJ!tkC}^?1|63Ow*~g9Rnofe{C6Vj<rInc_YNz2za<#f$`=i
zLATBUocE$zGB`NmB52Vn6k8aj+D^!YJ879x>%g2HjFhDR1_e!{kK)cyuzI7+ZF`c>
z7?xY2D1aI1(PXe(=w-Zc(oH&w*gwj$U_dvk4aa+fN9pM7*?WDt9$HqLq-|F6I=Qu1
z)NvAci`dWdw{Kf1pV?ca_qh3%xUJmzxk2HyyvQ=F)EP^Hes^CN{WWjA!EroW-Fu%B
zH`^Rrg8FVwBmCoyLim^r<fFiLL4G~r3Hfn7k+u;KkFop69wiaUq}D{ck0duTPOZ0D
zvuX(l4=ea*-58&XY&2R^-}=q2>a=$#{PVbu;A0-m0<@jc#yNL<s)^gVsVRyFi`+LA
ziwrIiGjC#K1e#F0Yu1C~c>l}+)|9@h)xtk2W8$ZF*JqcB71=(6ZJLD~#RE!*=w?uH
z&d5rQs)&mLHkY{wsM@-Q8hXDy`DOR7*+YV(@nK$>IaY6^=SdCQPw;c?u@XI!DEkHx
zYR=L0saa+I)qY|C91ts_uQ)GAlui@e{3x-r{lS~t*wFhc>c)C|o%6ZVjpI1}ScckR
zUXq|q<ke>HO@A3H4zN*L3EXPY{0hKxe)yN^?0=n5DE^N+!iA&7OwSb4a-M)^xXnm1
zZNXh^0M!wdCKZGrho&<fbx^hh29u<_f#ah<`}bsp7HNZx>4lBY_wEAI*r-U|P$+wz
zQMdp}dp?K{lg3ZqF(TAis5mP5?73LK+~IgKZ}a>;DHnSVlkv+kE5#o8_STTKmPAJJ
z!(NHP+k>0fY2<t8`3!b@5~bAXR7>VsU>AaxMd>sd9>VkGH~8B9FOEah=#p$pY|$E)
zvyb1%+kethH9+}4>5L{6hL5mD%5S}aIJH~$w<9!mvQk8}RhAID_i%Ymce@#5CTl-@
z$W3ucTv@G1jti$1odn2vHuz05nCQPB1qHN~_y6dRNdE4Ryc?q^z!}w(**33vPL@t>
zhRz1~WG6FLIQZ6)vBT%&so?Hcuhet9oXb@-H7TGwn28D`=8+8tUVUt`#N@M0Yfk`}
z_0q+&hv_!Y{i~VUD4+>tc|S5>^U17MM2zne=^r)vzI6JA)e{Bt{=PSuR)Cq<3OS!J
zZKf;XaXCGnE9aDscpX(ZggTC;)AowJUovoc%<nSfS}Iq%a0*Q|ab)q<imyrv2juqS
zQ=Eq6l8y?y4S@=IA9pR(T5WW+)L`!VymYE{P%Zz0-?LNXVukc3Ho8P^82#d^srl7S
z+FGiZs5Q{?7zQdiJbTxIcCn%4ov1p2EH~cir26pvzd}T;=v~x~;eg&5;0dIoE=)dO
zS@Qfh?N3p&JKT<)5Sn_?Mx4f}h{i604muhNaun~AaD^wwf}9we={J<9(P3y0$sS@;
z!Gq6x15)LuF-N;J1oH@^Ww;KQ6-9zfs9lUKI*i}okIF=G=)^^7b_-XHDE%DsS1dos
zl`0Jr0#bcjndQrj%{%kVtob3aIv$ox7wz=g;bcLPUiVbV3aGNE54s4GY-22+uxt!O
zNJJ6&e|WCQ8^gXIPMTZ3iKEisRl&kfMkfg$X$>l$w5HkmmgB4(#YN5_EJ%-Vnv9ka
z;QcT=9Y}yb)wH(8UHb?STP_2n8O_&s-qWZ3np92PEt~H6^P0MN{GB?QM`?sz%e`<2
z!$W#|U+v6sibR*=;b-}^MN%6zR#}ikLe-0_psA3o^8hRNz+YtqI<RS!DDK$Z;i_4A
zqQ{_L*#);XPOXPD&Rkie<9D&ncy07_maxY-mY|w;yj4Sq{H_C4PHIQvx7$_K(DAFi
z&b>Lcr+(-ePXJT;Hv6}^lNt5gyb<9iElUGcK=o2P#F%RbTIy;p_m8A|urwA2#X*<b
zWb;+)`;6pN+J!}bFr&LX>y$4xgNZIZ7kqM_Q)pXDsq#J-3OR&%zu$4OBIz<UT{&q=
z7n7fQvMJ!3!=S3W_8*mHr1%an03tR=l6gqiA$<cmpSX3Faz2&h1Cof)garTjLLXKX
zpyEv-fERq!{+Rsw><NccW*#<%b+AKEj(c{|)|D!b*!hSCjEs0NV7doo*cKFaS(TW1
zuo%TLm3kyUcHiLT^+TS{A%+52Q)-+u-nFH@4pD_~AsOd8O`eSDAy(hP*3k+g_9xh^
z*N+I|#b2STXhD2s@sE4b!OKI_-9b^SBn^Xv2Dc$bb*cu{D>9MG3L5xF2QPw{Y?l3<
zGEwgae)K1Z=$=WWZG2^nvb$5g6U-QiJ1pC?JyE&S!wRc9*u#SLtup=<0O0S_!~X~X
z{eJ~O*Y>R4qQ#o_kA|S#uSz%IZJ8RU{qfHRi1w51!Kp;v4_`Qreo%2vUDj6)Ej3I&
ze)=?AzEln~@x0!B4CV8XYC49t3oKlGaS-1)1QD|8mAC0!hYo#=`6;9KPLDv5M(<x8
z%JWNJ3Wpi)ay+etY#qhuhSr*kw8Nslbh8|rzBQ%l4sLi^j9PwX@>pCva@mzueya98
zYm&Hxtt{Iv=pwC{z@!WV8)oErw3)<#+Mwb@V(=S8)0(*!NTgXz<%j8!%JLj<W`j;7
z#wahw`uLougKErYpH3B9DQzK}YMsUA-}LocQT*)DZahc+oKTUC(y7^cHx7FCTvUB+
z#*D6l_4-U6e@k+zxQq6rQpf9)zLl267s$c2S&dVV!&L3u?5_z;mCZ;}=1z$)?W-uZ
z|8HmoVoBx(77FN!jIa6OeEOP2V&;RhjU=`P_oR_!+~ITTC2mmOdol6^2h|;+-(XU7
zbVlEUb9t5za*^d<=NaGjdS?1PiVLL%vws_N*b=18fYGC5r3L}^-$peI;FjeYTB=#S
zw_m2QWKa`{RGBO2qZyL4FodtKKFf*vq{8pR<sQ7!@r_ufH;C4tQ#<ScTkU}e02D%K
z;y*^5eYM+fr6(jmCfdd=AL&v$mf+Vj;3pT8<gF&qw)WW12f>%E>VD*i6hzSsPVcuh
z%hd_($f7IceE?QhDggG_5TFNP83j7eBPBoF_38Wkcbd63GClr}C{zC{%5>HUBt9)X
z%AM8)d8u-6=O|P}&3aZk1PPh^3a7$iAe_E-_V47;%`|toF7!8hjjOXK(k#_r`R3?f
zQ461U5o+7E`Q2Ho5oAifgTXKHg$WS{jn`6K)Lhu!TL~lYWK2bV%YL3u;2W46l<JV0
za@n*s)Yze_j&qfR%X6kHc4=cZD^x9f-*|bolCUxxZ!H_!d40EKiJ-=jQz5$fiO$K#
zfn4N=(i?fx)vD{drN~Noj(R51oWq46y2;|&*~I@=jR4?=_|D7UNyPWPW!*eki^BW_
z2pz{4&f#|JtObuKNPHTr<5{N?S5>%_fHJ|+@;8{SF8JXi+m8Z7XZe9f9fmD9N!#gE
z0beKV0Ti{!m*N2NmZ}k0qtO*Uj!RN6KZvFAUi?Cs;cP*0))N`Q%OKYrmvhHkt?Ewt
z2GNQvM=%y+l{T93dGL1olT;~7<%4K^QE`BfqoyRt%YrR7X`<MH?K~M7wOC#lMVOI1
zZSAm-2?vipmyvW-jCel&F>1R?H?$3=TLF~FiAV0YxX6KCY9K|J{E8qJuq^JZ#Jm#u
zRxpG9n=UrGclhW}$$_l@s{^rax=1uYAhD@ks(3%t_hHLT#IG(9&enVxX8G{0urT!3
zlW~+1AwIklTv<G|0QM$1A)>v?fbcE;0D#j<W?=*oUPz8(Gf7dO&oFzbT;CDLQ{-65
z<65Zg70XO9ha7tUx(2Kq+t3U;8$^?&49nn#KQMOHcdv<$cL2Mt#4sVgI^4jvSG=#w
z9oAkwCDMmOwr0k>qJENkh$kCDlcU<e1`VZ)G(PtQ0EWWOx+BLhf-ES-+E&o@jfd{I
zP!RRRK%m||ePevoE0ZRljRZnq7%=BC;ISg@(R|jy`II^!Wnq7b?gk00=YIZw$3Y<0
zQBIJRRGr%iXDY_P8L_t8Ejb4~5H1*#<D({+vao~D>&0_E3s&rCML|S9sKx|xzT82c
z#3ZN==+~?=e5UBg_j)7c`5Xrn9%78Y45)`cf;yN}>kjem$&{kH`Lzrni#3{Yc3~JK
zx0c`C>&=EAJ;1{}LVp3$X7XT{oyVhCc_<A9J3IP4?j(r~AEpo+9S`%NKn=ax3#jvJ
z{s;mUaEije(}+60H83_pSfnlkB(1vzPPXR-u;kbSB48zZTl{c7k_GWh3tI5k!;vo(
zNc#_pi!k9|n^wi%N`gcAT4CyAb#lXYUzOYi&`ryWwd=uK<OM5~cmVc!$MT+U&S!}R
zg&ded8l~5i($m@(sXLIp!7)>q7vD~p(stOXe(j<fSfd(}?L|*&Z=x~|A*}c+jip{?
zl;5AkH<;sf5?#!pT@7Kaj8(I$mv`LqUiVs<tH?1NFXG}ey*y&qUVxm=m?<Rgd1l3h
z(}~TN)UQ6kZ~8+P``3&`JF?~Hl~RqVBa8N6MVY4={q!2It)XdQSS!i2M<-XC=Usu(
z9QdgTQj~`?TtlakHpujhoYTMrV4YT9&fsJn^^r?UZmunNo}Ez$EEmJ39)WA$-aFt4
zc2~uraqjV_-fMESIUX{t*&9{n`=CEFrOe}3mU0c?=J?I;5-Cf7$|#Fm^dbnag&Fi>
zT{`uaE`}SXT#_5sSz1?6nC>Jb03`hFlTvvPUtT7s*FiK*(#Nmsv3$p*L4**F)xeg3
zcN_=lk&AmWdmV%?px-m$N9{2fawn#_1TKcLMNvw?p1Y1jOstY3Uw&?BPX!vg^s!bQ
z8Y2$*_0TYD5E_HN<o%A@PL}`$is$Tz__8NcXg$tAs9(={Z}34J3<k<4uuo(|x%;l?
zD)-W;ma`lJ)1z<FurS@Br692kXQQP@xg5ZDvk!^6PtHl`Z%p%KRsvT4rx^tnD}Fbl
zI)o|%Ri?PW?4=V8s1zw}r=}VK)afMEf4oo+m&a=C`_KBd)NnZiFVqU3r^{wS<z4^(
z!e~%uK&&9RG4@SkoB;+@eEfZ_HWv0G^B%vl&(-R*xZHiK3guENpz=tZ<aa0q*YfzV
z_oiob6<bd(j)s+NycBcuJ`7sy21~TrRXyKF^-p$wU3us#x8G~dsMMCjZk+Pf@BJX?
z<jYiM@s5|L47$Qx>(3g1NIvOL2%<Q?Hrrx+^LsCyRFM#h%OkQ2MUY1af(EXnM10XO
zIDKrK7O+~lKH*=Zx!B`ou?E%Q4sorvN>f9$F=;Nb5As09+wCeap#n~W2{o9eWc8i~
zty3_t0-1_O<s4^lh+9CR1ewTg3&Mj`zz-HQlp#ktPnn8Kp7k)Tqkkl$YQgnEM8K;;
zV+pU%7D|!82D1-)SyrZ0Tw_M{BmVM+7m2V%)mYuc*wvSc97Ye@>{KY0f)K#p2q)GC
zhJWl!S$g&d-*Ue8oUf}IaK3S(em*B?Eh;ii8Fni~`|0ogQ5#GG*&LNlyRe<v`9vEu
zsRL^^ja^kNases55hl(f_rgK|8ao0g0J?D#dAFk*GT>n*MbF7O08q5vS&bEy1C-Ud
zSFiZIsJnlJw$<oEfA8z*Fz%q(B+|;to+T!Wt5Ck=mDt5hOth!79Db@Ow87N4*NQi+
zLd$QPe02bAejW8+0N-jl-x^Q+vaf!tW8iVoHjZa_MHW-72#zZ3h(v9R{R8n-O5=#c
z>T+OHJ1D|v40*Yq`TTPHJ9w_ab=Q_oN$<h2$ZSCHaE?_o1&qWc^&wEM)B}4#Nfs6I
zT?wrGoML0qkRhF$(08HY+<8g{7XT;P8esfa(=9(}@OuCt7W^$dyt-vY^Jpzx-K$U1
z{5d~WEin3W`XDki<UuR1Hp`7w@JiD%;7&c~LxT*xu@ABj-o?^6e-Y*->cpiE%u;eO
z(8UE;qUhF{dPLT{{omjuZ5EbnhYlwsu#=G)=nXRUNYEL+)vdT~5-q(PTQ#f^ZcE{X
z8>Vzm0kJ?>w%o4x6f!!Lpc|Sk#i^`d6bGz<_UICs=Skds*{5m0-mP@kPa%tHG+Syp
zQ#ZVvSR$#2F3|qR@=zk8e9aDr6@ezTLng{pAdi_?=J-~`*lT~WqxxbMwOxA$eMje~
zWpm@6E7$mBS@i<5tJnWw?Jc9KY`3<51QZZt5z?_}6a+-NK~fPAQ9x2ex@%Dj5Q#;n
zba!{RbhGI0?(SOe#l82l_x(Kk`H%nm;q{wiIEHmz=Q-ywkNG<~{aVOAMb*NPRy`le
z9abpNh<6a)um*|Bz5ND+c3EMntaAc8Vr1;!CFEk~<vEBR<`(_(HnJvjZy&~6@d=_z
zWflp*>d5(Bj&IowfYC9(ySWHfTCwTWi_+9T*?<@MDxLBD+gtO`%?(jITy&A~e4?NU
zP4xlViDFu@CX2~{AzIVD*wK@FL-mPZ?63EVMEACQyRYKzm&z*D57Iszx!Z>K)zvuu
zhR>M6M1dKArRPY1B>_y94u6lb^~9C;(Uc&X74d2?DC!c+Pp1!)%+-CjyvJXYZ?rRt
zc&&x!s|c2^pzCHtT)ibV&ULOd>nXvS%s61PU{8q~pm+7$roXFT046{nGWJcM6iLnw
zEj?;0J1_EBs_;~}#N%X!7cO`*S9B|ns7dg>kDeeg(S-OYvbN*HG)0Xlfx_NTr|>u`
z!RlD*C5V3+=Ng@T>S#6ni3)`EHr#295zM`YuLxs5!yS``Q5caIV!Qhs7K4|bA7xj#
zorYu=;uv1G9M|p?`p^7$NT;a$(H*Add;ncmM{_UStD7zwxVPh<dIp;NrnD|6AKajH
zg0neWQ&wR#!XO`@y<e!+;FR}a7=u&&v6R9m4CGz&&w#>HFp{NE^%!%q)~<K;_5{uo
z<R+?Is8Q3*j-wvg-@yD}SN-clEnFB!-&}YKT-YY*c~J|1rr27<DpfY4V_w)ejB%p~
zqX23Chwbd!#@l4Yvhf|l^6~NCMI{|eX!RNN%mwx7yn>&F#u<X@MWzKHreuv1>m1E$
z%r6Xo$i)$Q9^Ck?1dY(suR$)CTug5-N!&dbS~~*cOab(_!K9UbAmCc-VfYYDGEI2i
zMO)OV0bDi}kt}g)invF5Iv3j&<JpQHgW5?3^YxNC6R}QW3!M8c_hWrq%1vTH5gahl
zL%7lJsO@m}0FSb@b)Kgt=vTksg(&A$^x;|RL%$@&RLsw8tW#au^J|XK+$a=8xG4!L
zfLFVtn||&|qKUS`T^peS8PUdMsS8@3c9VNV6k+ETmti#cI<MeDOLNbkt?{)`8qi^9
z9nH<oSTq61f{`TP;#fk@B3b&5M#}p{a3#wN{PPd(PqbuE=46SLZ}?Aak#=U-=_F1E
z4PYmyq)3I1zAd^g@4i0OlO3n^#;n`UeU?k?A-76=wbGh#y)1+Jg&$N)hwxDS{&#}O
zv;RggdHeH)2saBUnALW&rr1ghd#FX6=lz=~!dD4Dzs2Vl3A6l4I?fg5Z&xG}T1X#M
z7+}Fld%?(_%jVXxD(*8;f7&0=5est7Wh3JU5GK?-sjB@fBAZW9+}9s54oX0c_kbOd
zVFcgUw3q36TODN2xaq3jBL7`DLkCKqNnuR~(EioN4DDBz(OBLMlI2Uo4yA0bdF%F$
z)a}Rh_BnU>R{}WlU2e{;gn%Z|5GvltMYw?)Ti~9$k&Wf<dUxHv5>*@wWcvW5-~JvA
z=CLxDogry3#U8D6Pym>f*$yHe@ei!cXOg#|d<JA-e!=bmI6yE~xv@oVHk!&%f`sXn
zN$$2sxNj@-Dy<|H{$_6d+n{6wchh<@A+&AozgPf+uiKrLE;ICXs$w2IrN1Hw3(C)h
zKtW1~T`vJ$=d}oNl;(_4KYv-%-N~2k5+j4k`Ko#_@=^suf8?4>Iy^%{HZ9yPcDdbX
zUF`DG($TW}iv>grDYxl205V(!zFjOAjem)e9-YZ;zfWT_UvEWTr1m8~bYoEb0^=V4
z?VkQQFx#0iXME;I7wz32tNJxs{N-v~gyeX=ZXGY{wUaN)t#Mez_I&$Oh>;HS`ao5C
zxWQ!M%|$tvo;&$xHNw3G!A;A*svPU1Qkmo}^~pKNoSh0;m9@%^(=>71)lsii!$SRS
zREkz1mr;p70h-`@K*IyXw!IS-aPY#JjfM~a8w@cpUMbu7$BW0urNP|tm<TB~;PUY@
z)1Q~3#Cf13>MaE<ZY}6($;f+R+Dl8Fa0)X1V_Z<DXoWb>v|*Xuu?p*MZXRh{sKUwH
z;3NmEl(<CvE1ymVMLFX^>GW_B?*6YwQ)gUkYlHDed8HvgOneEm+d1Ur)%-IeiU@nj
z(_05<!bLecu$w)JqOCp6Pg^@A17w){!~rP+IpF`A(oz>>OhXvTR)8*bS&AaJ0W3fl
zbKlbvuJ4y(12dCcW>nJ1XTIy}aTeGC^O@Xs+n;qAV}V)nl}QUAQy60)10ykdo}R22
z_dnxQgFom_o4NmSDVFt9owvn_KGZT!yzct=qBT?h);0cq6=v(8@6IFAi&y%XOvVde
z?-qZ;>w3ZbDoaP#M0Jppyy3UWO5%-f{1M!h`xTzP@{{47hKoi+g9N+ZT;}m$h{4NX
zfzA&}!f=38jR#3%^>Km(yFVXE5s#~;ebyEWKxSbZniMVa6SDcTQFEqCE%ffH;Y6K$
zd}nWtap3!0NzgZp`vG&e+4qq%FGiV}E55kDdI+Xx2Hka-LEZ&7L3bbTI8Qn1$eWn%
zOY5=}eN{3cMCV>>&#)pQ-G#!MgYShCd1&zrpPX%(fL%^QE4BqjNxkPBgsTzL)$>Yj
zhrK_=beS=uUunMP3A68c|K4<fkv1(s(=Y@DB>4LN8wALQwbF?DtJeY6B|$2R)om(2
z(L|F76){zQn%yrU`+%uh<UY1Xr^}@iSf>#2J<%SMIexw>Z9dkSn6Hi3u^N{BME7=)
z!RLZ1uA`?YHeidkJb*xfwLtMGUwh|Ey!C74yTL#jvZFo{0hY#1Vij3V-)*|?#1Mw}
z^4A|eBOzfYKc!=dCDeYD?d7l0@|H{GY%LhXZH&{Y`X^idqhxdD^^51yc{NMol*f$S
zuL5ujyx8qNgX22X%9cwM#cPakZGBAp<mgDoI^~5s-XR!tW_1AguS;}6wT3YpFmL$`
z^Z>I34@w69GijX<F>mO@3?tdJwVW-|fz{t^adbdr#-7_vFwA)vu<|JV^{6`UPYrM@
z&1ca7;``Koe$ju<vVl{STuAk?NbAlLFN|`+8!aDLlMmt)yqD|F$)Yre3*RI;yb8A^
zGok)Nl@q@sG_<ig3aT4Pw>Ah~i;0WF50>x2kY=;npPhOyP*E9Q@6EjyQI)Udqg}_E
zm=+hdG6|BFez*NNRW4p!Bd(mRGQ?}rU3wT%J;K7)^k?4jx>+XCJ>O%uVnRzNE8%F%
zuHs*aHjHYf@`s6BaT<Jz+E%LCua(}~#~i@8nQvbc{qSo7EV6?qzgAj;c^!~9Nk%r>
z3miH|liTGrB)PZRk_LIJv+f9FS;snqiPfwKPr*bi+}q4W)`Q08e$r?PzZ{JVQ^;<5
z3Y;!eM~L072Cb=k<XCLmmelpA#q1=*8A14?-;H8kq_z(#um-ulc_QNOU1V*Ncua(Z
z>`q7>WqTaxeF!7h$StbPm6kWfQ8he@2vRP&%yZq8YB2tu7Mu|Gj@8$iZX-57>uS~O
z=MR^)G+`I(Fn`*3elH3`%3}&cWV{O1hms4|uv!nG;YrgC0lfHG2D#WD6WVSKe7+2~
zXq5AlT$nSMtY2%=)rxkd90Omm@xUVsrt9ZiT4vsu%vU=`Y_EIrBs~MT6kqaG^!iAI
zY+f+A|EM$b;+G@kDrsjXgt@&sQ`lRdK(!xNfNF-_-rgQQ5n(Len|fz?lAeiU_-)N*
zSILU~K&(n5m#k@xpvqOk<H@rC=dPxNDeyeEav;ch7rC+mf-s(~wz@3-+BkJV!^6XK
zOPZ#`s}fE7ttxnw6Isq~UkvzhyY}vj_RS31QLf66a~jK)ea`ln;4tnajyrH{vu%<4
zy(g<X{y#1-Idt;zmDe*FMkdZDCp`IryQ^XNjY{UfqbzI);X%DW`-TC7@y#3ZV9!yw
z5i!JGha>1PAVn)^4K_I(7NM;5iM*Ocu;&@bN)JRA>BRRezty#Kx-IlVdWN#$M(wC)
zjoZHGl6T<VX4;WaiUxiX+(_62XJG;0WK4b@J}Gj~<%KqQ<1X7dcjoI%ypCAZE3pLT
zf43d<8qGXE_c_@dPcakp)`C+3&|)L#MF+c_Kpe0EYUuqK`d=6wn9BbHj84>bu!)P=
zDZo|mlO0!Br}8dYCRTa(yZn9g&M!{fweO!!GZf_UERjOIy4h?;Tpqp8zdlw9(840k
z`{Hsu6s2`k-aV%Y7>AY>%*}t<)>2PyhKuu%QKa&h$9i1%&^C1Ot-c*(rldG)ma)q9
z4gOwE2}yPr<(S3b{6|-wjyTC}m(3X-b+obWWvTKkI8>W#^Tpo-T7sEE(C0kPtpc3W
zK-Yz0_Xf1~4^@KaaWd1JQzcmmZmW;SbL_IR*_heNhQg(15@_$gpO)!WDO2qBgck*D
z^N2XNCf+T6H5p5fct*zCcMaJys96$}^@MGoL=99W&7Q3A%HO-&9v7rF={4|QR1yZN
zd-D7|OGUJ-VeT~)&6i2f->Y20g54nFy;V=NP5aLLNs?yfissD>C?Ju~rVtJ$Av1p%
z`Y0`$t|y98PwZh(IQIO`q71szMV|c%|L--%EDgOq^~#CYbFK}hDRbv-W?M%c{5Y=I
z2X>wb+>4whQ|WpRD|v1V{mO(zSSE=E_#5)2H(GU7!<>CrWOwpUPr%;;$*kUJJuNDV
zax-gPaO#o@1b;CyHRBTH<-0v>1$GwApZK~v2OaW%ZE&)?JqSh+iU4Hl_fe{v$j`W@
zp5j3SRzd5{+`n;NM}v{6R~v8k^K7x(F7{@4!rU$q&7G9a`JWd2u*L$5UTmBXA2n}f
zrv*KofpK%7bK<>5tpb=ulVle4xCc4Fo)<RB8#+^hY1S~6(*<Wm#n2M^4U?h_;c|N2
zQIW-;D6e*%h+q$etN`rjfTxK5{%(B7hp#W5ZzR!ioy03sUD$`xrjLC{`(!<_NUfqr
z`2JFtj6?s!v`T`|#q0>Sz-}=wxsa2sAY7&Lhcn;?X|qeI+emZ{%d~A`LXG`#XPau5
z@UY3*2!Op!?K*ht70Q5EJ(2CpkNi3<s5_G-X%8zy%Re7sImjS}W!cPr4@!_bc%&UN
zK$?F&UYd9V??AY6_)RRkKgc$jgbQtI!e{>+?v5UlZH@d-3d>&wPxc8q);`B1p%=Ix
za!{;tiCz0@_qmcj{`!aF5`c+g8pOMdegkxw1?}^lQqF0I0U30H+J?~!ZG`e9_>jUJ
zgA7ULf;la;4%+0&_9xCGhvZqWZY>?;NEi>IUCvt)O6KebF)EF(-ozaa%~%24QlicS
z5f}(e*}2eYt;(yG+c)pH7JjKIxR60JWhHXi3<@|9UvS(=Fgzd+@-ZH-k9kHsTGeuS
z0NoHcs*`NwSeFjh9#E&;C~<1GHCbpB5=d<mgjT^XehaLWt&z$k#}JCbM8lzf7!y%g
zT2M$STQZ_Q8-CKYb>fxA6cKx?&96oor0Ais%-$6w&wZU+1kEktGbYy<ISY5$%?(Zz
z2GRNwZHVzjqqs~x_fJ|+A?~;|$tN%OvCe_3wp{xvbQrrFPo*kLR=s%98`Evr-0fww
z9Q#V=6qhiCG2z|Mu2#wX(*R#3EN)vL^QXGkq<P%JxxTDlInI7K3MH2sHLiYo#ockN
z2)tb|c9%eLo^xLnY}+=ILSV6qbVZzU>t7rvU{6&vCDNY~Zw#2QR~0E`$%jgyh)2Tw
zcO}Y%?CB?Q*;0}c&TVE4E?mBE<?T201Y!M%#Yf4>q|uT74Zd3-aHr@O@z}|DPy>k!
z?%ps_)di4W)N0yoD~z<ooFy)SBSeFE4}xsMb7e+r_WO2L3Ry_J+rI6G)j&>YdGvPQ
z+uQ(tfH-|K)9GJE-1^&DkWYwkd|dqr5r+f{w?9V^LwM_(Q#Xfm?3-`j{eD|rtX;Gv
z7`-8HN#-rCZn}%|FB>BWDiT`ZpQ#&}Czwu0EyKLGZ1g5EJNdN^IVoO{IF|FKW_MT{
zQQ?07=Uw%<pLv>8;O;L>)-Ec|&zx5C-w-EBh!1g-nyY}ICm-zC%94r+I-vgc1GDZu
zury~dwQb32N+tlHcDearfxy(CZyoq70#`{09Drj+i=nU*STPp3pt^5nM)rWe+Mg4U
zOU}-p*G61V$n=FII750~W|=;>zn^otJ?dnpv%;bj<Tamr*ZAO8n&kW7NjmmXeW(H8
zv?|cK`CL*j{OE;A`cQ?ur3*>N7QS5MB?pucx+mR{rfjvFsg{IReyA4!qnFR@S24;v
zOq~G@bd!Qjggp_PmZ?R<neyopJOxG>%kU>XVuwGZPbzSlw|l>fHn<!RdD?vtw;Lj%
zi?*{?onIUflXC0x&If_u<uVg9Q_9O@*$4pmFuV>=N}^C>?)(chspfbC@q0%EDDrkd
zCzDm3(1`x;Io{u>Yy1JC3xzIb`^z<u%Xw+lcSi4^1IMRm3K_Ud()Q5Kgr*60Wf5}c
z&!xtWm#(FBA^OUyt|yhs%Js9>q}-KHp0$F+#MR4xxRw)fS{StV5~-$T)=7Ntuim&P
zw!tR7rZ5?;_HA&0?^!{2OFBC=>#pPp`0jpLOm0WCbFcaH9o4mLjbur?Ks2IQH3R0p
zC%WBV8H(JOr@d;ja-}={{2UI_ze=GIP#J-U?6&a+8yz$duQXjQ`%fOXeu7jip#cuQ
zhed_2j}~H<zTZHt&>hY}F{omlC$1fbNT(UR!Wxs+gEq}PA%UtLn-Q=`jZIv1zBY~y
z?=D8Jo_qy!RPZ(61k+uY@lTL-T=NgQT7-QBgReqmg|~H;0)X*(zBuJ=R}rvJApM(&
z!bQSHQG^LtVQd*Ako)LZZn!p{8phJ7#(8x5NQU2sM(s7kwA*3#7+zS!&hj??)`*05
zGD692NQp><&e3L&BQgOni3wR-<`e=qOXQ^~neJ)W50`!=yQAj$*BW&&m<q_~h82so
z0`$<wZi;E`smqNS^s((s`Fd-Q+sxC@k=`q~8br{!uP3lAl!!Z#<3&9k()<2RqzdcR
z#(2M4RutkiJ8O5yo>DG&3-EIRvSFprG+`#VM?OJhi~%;Mg1zOK&1F1jhywU_1n&0g
z5NvXlOPd*?D^k;aMFy%-5KXg>RHue$aT!Z5at;u8H(S#*gQB}S8$_NI+T|X-Fnp16
z*STxSJHKTDqhABeN%f`|4X)awZV1l<GE~Cu`GilPXr3s*rYSJ(0wWJ^Q4@)~|4McC
z;hCelYUemb0es(OSy6T2dPI^b+?=m)3m$b#Kdo5W3daODd@CdB_iM|fyV~f0xsmL5
zx7nJS?Z*R_vC5nMx=#8-wH1%%{I6xETf-SvJopd>qnS!G=*s<|I4C3PY4@|^_DFO`
z$K<5v_YGwHwuVoRywFx}Z#vmOg^=+oqv!Z|rf7uNe;Z}&CWa#A7^2~~)6HymFxr^G
zg~64;EFqD#yD1dd$10gBGccGHL4W%W`I$7mCh#aNp?uqBdHMqrU49X}BomMa%yfxT
z-ELRFTcq2-zj6qtLjqNOki#FWtz46dLd7VTgDy9Kzb!@PR=p9<s1StcIGNraOc9@c
z;=7)cHSYGafYp8dW&z>au|ubs9fmy<cYk-_rl8)eF3^>{%Efd%8Hs18%AvXmIEZF3
z@9V;`lZdn&s5O(Y&!`7w2wBvgBizC?p}-}VbP6r?wKy&cEcokH=bmp!&>gs~@&i8l
zjq0Svr~lhW|490$0WvfyreQ^Rp`>KweD$T$%8<<4TrGKr=VXu>EdaTuTGQ(4!NOP-
zDu+0%`7JxC*Fagf9L}-R3GY!<%sY5Abw!GYxzK6uYicA|_vfT9u!0t-#)3qCpVG!x
zqMoswtli8PufU6OxE}awk1uI@4uAb3`kCxufrJn}A0F(K|8V@Ym9z6v!?911s6S<B
zVB=}|A?A-XsZwD`zxavL-8Vm*Px;%uNs$hos#ilQF1DkL${J5iH+{1tgM)Zk3m9#A
zh?%k7+IvCj6`spu1T3Qd1knLwDij^>+^*G#Ph_syrF^KP2;r$mO0H{2wiv>Hq16tv
z6ojvlqlIY%%!Uhz2ue09w662rIi3R>bGN5!{&Ekruha5eG2y8SO6%kafTFY)Fk@QM
zRZ9a%0uvaVvlf)8Z&M%P2enr(d6O~CPJK6UFawT&8mO+}c6}&r#Y%oNOc+Oy^>*d4
zb)Z?8D6P2B*POfaZbT)Ue&%FbRRUsR+$t*Qs9(_<jWkGw-*gU}o&2D!kUhkZ&Mda7
zL>=D1690JmMJ^?D<q;XXlf<sSDD8r?cHM*)*-7r)NvktQkRo8GyX&7PLnO}Kc{ne(
zE`b>8E!-UTK~H-P4{C1b9Z*Nt@eExr1DSXK2!R6lw5~5D>)ht9Qc2#mQq_AsL--uS
z4KuL>zDH9mTzF=`Vr8iW&)Ip2orjW_9k4^16^S~i1@<&9{#3jf+ddDIWql`N>IRH(
zO}9sL2nu05OHSf}C>EFULT97pML`Q<DI97e9d;Dxq(Gcv6UQiUR8zQ&S`aQ?h!w25
zfZ`dS%4`WVc6UCSDY}5{54So*adVIv7UQnRThui+?^-Ha%<Wc~Kh!>pz|ox4QbKN|
zl}f*l`yCYN+*$N&vR~F3U>bCS+ykiRM1#Ip;bXM5|2qjF<wA?#F*X%4;j8D5F-uY>
zxtwUfV@%et_un4KS+B@_;OY(}7@bT@RrJMEELJgnBtV1w=`m`0Yw4@QqyutSwlY{*
zZz9VNP~XW_y#R+o!c~KT4&2er;K_G<X(MXj+vN^od`eF@D5FpaK*Of}2$<{N|A^>w
z=K0=+u6sdC7f$67#ACHO5!cj&xI5^UXCj~<gkIDw$U){Xf?usvv{)&{w9JYht7ayw
zr1M5%bVrD$X@uBPVHi~rA8$o7K{_yklaYDGsg6g0H~c$!3cZe8D-5JC-Zk{~dfX3s
zTr##G6)GAYY;Q3UZNn6CjOmLStGVLZ@kIia1{v4gM^u7}1_##%{a-&7MIW_@5e{l}
zX_~M6KCZ#6+BnSrjDJ>N=qeuZK_e+}!fbMKZ*h2(BC5REC8)aHMOdYEn{UqB&IDTL
zFo2bDnv_&F^(ZOLWCW$rr_*5LEVk0J#7-j-ISDY(bk%(CZuzu=2A*!6G2t#Sg4YX<
zotw_|J)JvJa{1)!d?yQrAD^&nom|mfnUCh^3k=@EJY3!D91c6gnS=w@Q{j)Res0_y
zA`gn|hjwX``&s@s*q$fA)>FC4_QGKX1HM{$-2xQtvo&$yF&L&hj(>KB0xJ|qB0EkF
z2iLwv)15jR($M6=wT)Ft?W+(6G9auC*<<h-o1mMzXwPliUh6lLj%<e|C{qn}vW9HZ
zZq~f<Wtik-gnrTHpCj(~P<CzezOoA+G!iad@lQ_@&e))Fv!=Q9KI7D0`{bYWeYA&&
zj7TAszYfM4D!THs%V0=6RVMk}Lc7rUWUDLiU+hTuO1(!c6$5;_e$s%wK^dkySmx2o
zW$y+D>(Gfn3~Tm|6-lQF(%9Ifj=SgvEj4f@q8(*7=-H>xIRkfiDrbu7X^*ZBn(kCe
zh)E{B*4|5(i#_#!Dko6mi5g3v7dZ*qzi+%Cv1>OpnNH3*D5LGO({)|Cq3MJkuA!Z0
zF(NkYVnkMzl#%-h7`1eyPx*(lUV@q>&txoHMa&PS74-(T3iA%+(Iv=HA{;%HGu^&+
zkeiL=$YqsQ6P`FCdN1}8wlcs3O0SYV#bkagYD`3FIpv_jhsSXEsBEc#1#s2l%;fSH
zke<a$YE)}%X$di9bb2D>XI2RU_U8h>M47t01k+*(0#y<dhOAE`Asady$<kn#@(+4^
z<31~abipF`4&x_gd_h}Xv?EPQ-@386QvgQ$rG1PuI*CLWu*oqb7f%}yauFo3-}PR@
z9ncbgpd-NB9~S-a452LrB9lR9#jV&Uv=HXu%A6E<&^7RFiB3Ciroe6s6?9m1I?1$N
zhgmAvepY6vkrD=G&{aTA+couqntOEQL26SVjXT4_!H@d^k%;$K-GJZYe!i9W<Uc={
ze|$eb#21s7Hl~*qGS$O{W~Pt7p5%)sq14Y@^9Y}IHl8>#CACrrt|H%D_HO`wGH><f
z>x#-@T&>1ihtdpe-qEV0ri<KQG9VyZF6K=;Hs31Ff_yAz_fApu@>9gsMQ!9<e&$Sn
zrqzkEu)<PpMWt(h&#F&W*m7LWN-ViS=lRxnhyK7<;|*-Sw(;t;qz<8roQbNv!67)u
z?JwiXvwitmW2s~vwAiVAwF6cMT)~Z~KWL(cCCwDck`hpie~=01?eNV}Yv%Iksh12t
z$e;GefvM44)W%e_I!1?~LnA~@Voq7_t%2zx&`S<0h0F389#QkS1hxlS=1Q=E*V+iz
zX{79agOjW=e3_YIU)OLNF=Vn|ZNdB{iT*_<9AK?iZQ<`>(+)hEkju~u80?H<RvKJl
zTz=N*N6+T8j`vS8q21i~iq$MW&%cui?dMQoQiX8fP4(^fVSj&54VaWeMa%Z)kG@gj
z$;EM-1)!}U&a@E~{nh@bZ;_w<K$dxR64k$8ZdX;K>mAEwyni-<xKlX?hP(JlACUX^
z@TF$Rk{UuMw=`_FFU%IOZ?4OPn%pk9=Ev)7Eb7ktk}mI67NB|}H3+o@`1wjAqJCm~
zyl_WOWzH(2K&`6hEpoqh$6Y%E{5Gm;f%i6Zb@F8wM~GoaofJQfpMX^E8w}1E69b}z
z&N*Ba3pwZBJMkWo1bWO)$-C3|)1WsQxR62-#6+ZV)UmHC^dLBZ;ujMWHKb;h=~<n|
z@sBVns_(@kGKB)lzhR;_z;@RS?Q}Gox|A@Gh(04Hq=MWJ!GP#^p%cD33YDfj>h$}q
zS3@q=Hl%?5!xJcX;TL6h)yF-|;VTt~nBAC5CNQ@$s%p#MA723P!^Kg3p)GU*FnpWF
zPx$|~1?o6V>>^w`y3k!?)L0CBhRFaDj$8g#aD9qCPt1p7yj8~ea$5R5ht_>FAe<ue
z0;5}6D>O^wv`X=kv~MJ%&1mB#a?a;#1VSdcdxVTQi{kH3%HLlYJe*DFEbe%*ovgaB
zP34%`beo#({x>e;iIQIvnU`~QWZb5kGl`w?dj1CRVpg;>`_&l9AXKH^#5x3nz*4FS
zH18CsTgwa-Rxp>!S2}|)7=p?-$_fb4u&f2fIZIOmz1}6#Tx7IGb)+}_QT-(N&LLW(
z4#Gla>%e0&Q(AJ9dZO=adV8a_VA6CaD8@JeKfhar41vdhY#G>G*<3q`<{^%QWjl-4
z$sZfHOC<6MR7NB+l}*M3C1*FaBT66X-%5==)N$J{g4A*WwIE+1;5M4^tuU$(e?=~v
z2y~UK^0G|6T`@^@i{lDB*_Gq3p`*FRKpCp-^!xrLxFV&X&jn8<(vv5;{sY#>mN*aW
zrjIS{I-tA9Ozg;999lJl_(i1Uv(V!lySennaryYaAdWhMC+HK)-devs9zi^L`{S|(
za*N`8<<2xFZr<oqUcS&e;{vb!V385Gw+j#$ba-`q4Wq=9M3y_6>GJr@3gKDIj?k0?
zPOByyF{X938j&wMiv5Pzg2c_HL{sK1UU!s#Q#r!%a^tJb4<>~JH*Xi`KUEFan)dm_
z{q*E{o~>Zk`ey#}?M0UgC4U*J5F@HF8HwbtH4raNmyNMcaS-d5fvBkzI`3)A=`gAw
z=GB|Ql?J^+{yBUt+u9rpWm)r%Ow8068`LiHB?shJ33P?&33|p>u3O4iBjIk=sL2;x
zuk3HM4G>j!<}-X7m+waA%h9x4sWz(nkSYULFLEPqbVZ)K!g=vF_o^1o^Unq&3$!Vb
z<z$d*fP-E}2`9whoEqeI%b&Av40r7>_2|x4oPlKaP?pH`?z8P8djK(&RMeHGCzIot
z_BHN{s7sP~c!t7vr%KLOYf*Yk6>Dct_?|3j%z=b2lO+&`T-GyYRaXgq5PD=eBh=Jh
zwx(Hd>u*Ol@lQ$)`DzS4{P%vk6%Urg4>t3GXH|-HgR64|GkOg1?=x=IXx#OqYBH>-
z|F_!e;X^n?AGP#+ZX^B1Y|5$LIZ%zutd~Kkg$4T)i*)|fE@aGGu&i6i`9<)l(2;50
zOmd2x({_sW#@1+xSO3@IdR50p-fr_zGVVWcA7o_U)N+i<870>7wqO+g1fY#esG^j(
z+DH`0Jfdt;TZdwu(ZVbshZc5-X`7AS(?*)S!K}vR2o4UdlMpQ~!!JpZ^`mWUO6h5Y
zME8D-L5_y3<+)q6-TZ@?L|2}FVI7^yS+rK-r~7{JJ+`o8K*OI%H_FgWQH$PTaCwxY
z><Vm6|4<07e3U+?hFYp7`s-V&otI<x<K%LU*r;_DGL}^GZ4<e++(zq9uv+7YIMr*s
zHT_p$TR++)^f$Q*dA)WGADi6k&aF{nr3qPbFALGP%8xdhPyqG94>i_ovrP99=$Rwc
zWgEW2BoK2|_m7LcKltFLghj1!ufEsPcL|#lanohTt-BP^MviAVe}8LMcNU|Lr*1?M
zq_^zAuo}}1pKL7>`kX4@8GI+Of3OGQYcfb^Lg$Q;X7m*efA>3w30bb6lxuId6<!&R
zEyOoP&#2Kwq%BuPhHy+P(5QOzc}8-fwkt^FMFXugyZB2FXRZ29t?SM9PlCstZRUiX
zV7K}b6EDhISQ$(P#WA&<Hq@GakkfGRk2eUs-z$pzlRQ)os3pjl^luKDngEr(i%gck
zgV8yC$|x&ApF#j~x(<-Wr!d*-KZfNc*v~t<Zm^LvC?xLMGltp7Brk2!28lBHdv<zT
zmE%}!zYhy}b$r+gx}h)!&S!s|MWfL{Q&NHtBeLB30h)UCI?d9)P)*wx9_>4`=YT0|
zZ(vvW<Xy%m*An2ZyqP_z9_B;_(x>tWKemB>=Czyog#WGh|Hn<e8-`wPIMm{16$ekf
z`ffDG&|je6&tGYz>n`Br>OYn<@#l<ZvNodkmgx4fp5?tO)yIBx&T03|KPhM+BzLqw
z?W!&nIPVwH|Ja3yut1Sbm&uZWq#TwbAV8QIn}S9JSX!BUD*`OUX`o$=<OcC#Ujmcf
zB@Z^;?LitpmALWDy<@Q~!O@S}d_#)C+!~A!XibZ&>OxEpBXS;eXzK+-Ukm+*&yHKP
zAL21_vjJ2G8#j~~_AiE`c=7COTx&~Qab}V0Tm_ah&50WwX$6<|*O`P-T`Yxh0j>O}
zxZmkEyS(HZNtKd-UVf!mSa%G@n?{Ob_-P+K3vC5g9@|$C@I(Tq@Kg?~n}hsOi#$PH
z>hB0q8w3|98uGU0Iw;RCRIEofneV^{QLJ>6xUJ)&yS)RsX`7Mt<Q43Y5m;RPkeu_2
zHogVJ18&t2TosPZ_mc1Jdk7@z>P)wZyNeS~hR1+1a(>K*Hh<r4HJYM3K$0V1&WuUi
zm<|yTB{UNhl)>bjX_jaz>DY!VtqMe7mhH>fAnq=ry0qXCqGiBzhe6=oQReD^z`-nc
zXfhGyN}1D{u`ae^U#;Daz6ToU{NrzLZSMRapcZekd%@ks9HRO@43o9`Pb`lDnB<~|
zx#*JibX8pox!;RO%Md7`&Wt82yyr~bhgqV<3hY<6g4dqo8S)MXJL-imwZh0GSXRf?
zE&A8L132UoK%wiKp+XWv?fG(<V700TRvr$0uQOobvZG#^G$T*&66DU%hI%`xlga6B
z*@CNf#d<yS0?+XHPJaF1Rw?Iq5)`;M4}=0Z*uco4qDsQE-#`D)UX+R^L<C*KX)6QB
zz+ytg(3|PZP?{S=PmfnSGBc|-jX#|zs5moGo{kDnJgTdJCy9u-5mv>)%SrJJ*K5^N
zLAH+LaB+sjlaVYh#y^h{KM93cW|4TYHI~r_rB}rTQ=yia>9lCSEM6Rof!H320AM4+
zf@+LmzAq&jZ6E!~@vJ$dimpqCQlo-9RyP3p%qO93+5q~=Zn0jkY)Thn<M;w$VLig9
zUp{kwRU@Q3%wTLH{3V1+AjJQutz|O}AE{i;GFqvz{4(2B-__vwSFqO4;|O^kiK`s8
zAt)!>))yYr2<^sI%)j9oK|37M9c?d)cGPt3OHlfwt5Kd=t`==l18+>U*J?~wqEpZP
z^+eP9LEv0-U+>%Gt%?iR^AQ`@ZH|JO;sI7NJEM~PHTpsF$^#wl<5k3I4OQgr@sKuQ
zFB!I!Ci`?Ow~WZ`{ZNBQIp8(eXhe}Cz&C8UvKAb!L4jAG2wcdI%Brwb94xrVy`h}c
zP1bZA{^A<%IQx0|vt)vZ_OaV|p<`WqSrgBSP+j+4rqY+;JQcQ`nW`c#!14Q%-TB<q
z8DLU{TTNnMaN^AIl4f%n0THAwH#1wht$u!fyXP}ijAmo`Uq@Jh&~}eW4*;gkEpesX
z21uLukK1StU;2{+35XB??dv=NfVev*r!wejc1wE$EVQU}I|51$5gEsLhVmkJwle^Y
z$YlT|ayzKyc4xN$zimbln6IbVb>(!jy~g?$l(bCD<Z0un4Ly=v0J&iizy1UCR3Q>y
z2$AJjJ3rF<Y8n^-e{<)s-T{%N_>+>Nzq&o}!F(b6#%HEAB0LPvx{kYmycYrwM-!5t
zYVf}t%_Q_gJ;}!Oj*Y?cyMfx#ZGiD8Nrq{*7AAQQ;3_ZZ2iw67>mnXNyKC;sYReOE
z$3;7cWmEO7vToe8fq*9X55^*1(j$()krHuSTge1ciFa3}yCli^cT?D)C>^zjl7>UL
zS&cgkXTF$!=OoHUT<WZRH^0Pd&CuBN2@m4#JMiwV_Yr%Z)@baY>*)Hi{0+e53L#9F
z4A;hQ&y+;}sJ0s-TYQ3yGlVLNvo?n5o+aRvq669q2fKaGu-61}bB{gbI!O?E){jbO
zjoZ191HR*l+w3&8RcS*#gAc-6pOqjTTa3zEL;>5w?>Q~D7kkI1a1AMgP)WspCjq%-
zULd9u8+w8`JDEfs7%Buj8Ikb{H8xIa*0dnD{P=bK>5(i1j%iQikzS8VicXHhhZBTZ
z13lGyBF3mhm3Hg2X;Dwwq{E}Ncr#{(7Tf2@fnA~_xXVUV?dExvHJ;$6QsqA9IHn!G
z;x9GtSW%!_*D{lP4J+=gkY6uRt?7sw6PYbYFHidgD2tzWMsvay9!ASEK$L}+v9H>G
zfNIc2XmQ!-_1O;B>0XRM;#P1(*c&`UPl?(wT#<mw@z!t!;jgnPp16okU>v+~r0g1e
z091K<#hqQDTC*s0C&bjYz{lmeN;n_#NhD76*nQVSG`9Mn@dnyt08i}L!HeNCi4+2`
zdW}WkoX<VpSR|<_7P7$n<GH|kv~ImZrAB+$@7I%4sIfrWUptUdvTIOgx%Ad8Uk5{J
zCS&xk`w@~e6RYCCv_J^hMvu2E+q;#KAI=~KRmb!HrKHrN9=vEPD=_!)=HAMI2|DVJ
zj3;c?*&poNp9V?yt;6^7^-)QMe<!UJzO_HHH1?=leC)%EEb<Fetn;zTBRAeh0ZXVj
z%pz*Gc+el>MS{5=a`7m~!=12O9j&z#VZp$VZ~$F<)eV#Tm8fc3x&lO^eGv&qxF;Ql
zi8n60b*+E!O|l(*OB@a>Sr3qRnP5rE7QIRxQABpv(D)D_OC!YMFPe^@Jv<bqz;`$i
zC|XqVhYAE)(+S8HX0hs_b|GdYV}Z=cOq}WV6A0OHf!02{sI6Jg(J7xDF}mrKLp%uu
z*yQ`eBjei)wPK}dvvyVie&GwsrPYuwT|iHpixI^SGw28gC+!M`bXM8Hl-K}SC{#iv
zNpSdjZv?)vM4XDKw@EKhYsyuf51~6g->{e|fX{yF)0$r7A-Wysuh?4EoWH(hsoRTL
zmQEbaR~#t>U+=VS^uy<ihwz65-uj@vG3X}0ney18u$Hl1IgGo-eIN59xGwh_TC9!-
zYx#z7^rn4VcjW+JfSF{8GFMlUNp40MEZo<e&E{o+^oU}?b2qfimIm?qPkG6Qc4V9{
z{O$9TEuEb+5kVch0VDN_Clobi2HM4@?)|o%GG8QkDSgun9(-kwA5Tr-QAR0|PEiX+
zK6!FqZM6d7<wEDgF$#HbmUv>|hebbZ)y^Uex}yhz&^a}0ZP}N%CyEa28M@DS6a6}f
z;&$bUv4Dc}e8CNoJ`B)wOf>XD4~a8?P$m)qsMn`7DIk^5RMmR)Q>C+-7V<NJX)Lg?
zV9mu8MA2w8JCt3zL6|qwcO@1FQlvvfMi)LhR%>U{R2I5iAAX~V=gvTfaU_5gidp)R
zwX_U_^I`MN1}?-Cvn+5>uS-r50i^Sq09MbbKE53X&0-WBx=4V)!*GP-RBf<}Uhc@@
zMhd|E#{16a?6ut_h%Rzc|5DvDQ)N0Xk7+{QXFv^srCp+CHw%psZ@B`;*3$)bf8|%D
zEi}TkxoSF$4Ugu~eyjc%-Uv56i9BtJq34en`s4hQzhZmRJCH0uSBhDQ#y#`RP(aQ9
zB6V;7MAB~5HQ04{u4GqD_C)|@446B1l>2ym+f9<!eog%J0lfHFgJ~W_qRb)_XI6e$
z1Vu=302<LxiS>4g4A_^}<Qh!V&F8(A;Z@F815iFkxdiLfXu7+xMaw1Bn<X5eE{@Bc
zVL=l7tX?~OU`|ZyH^N)M(y4&=#rp*a-yp?@P|V*GNhZtd+r;<ZC?Bc(sY<((2%V#q
zTyZ|q#iHm!N&Z45hJMgC21to4rz$R?U%T9~MZS-!?0m*}Bz5_1WtbQ|HdI?qxRv7(
zSnag`z6O>8YNs$H$8!W$w#w`94BsC2mBEEg0Ov!`mAk(1{C$Y%$t@auYSdB5Os<)9
zrmD3Ovg_VSf{N4a!Zv5Z2EF{{{Z1Vhxj&IbVDcH!T*&IR<vHxUYeyF=RVr7LsBYJ@
zH(xvtdtR$Nr>n63?y0(V^gMey69%VXU*nBeU6^Bmoxn={Mf*9OJDS^J$VJTEnQ7eX
zwXbm%m$K5CUw?!Io*P$0F;eZ#IawT>ob?p#!7gs8y5FHesc%J<-$v%EeOG8WNvG|p
z-vOJaPiN*UR5e2S9(AfFOxx`;ULe{Uul3z8`X|6bPN#=;K0nzJ7M7mRQ2W+;qi*IL
zCTqhIgpDWU5fkdu*c3~l9~PTYZmJesXDl3j3lx%sU8Tbhvl-^I(?8{)3J3XIU+)wJ
zI&zuC3Nr*~8j*4uF1kP@k}>RmP%i?xoF_Z)p6(_q4P8>C7vb4B*b+Vlpnu;Ny9Ve&
zWX@-86oT1*JSpCIK}Q$v+3d3Mf?f@s2`AMSO5}+S@$IJj8WANbe%A^TnWZv#mnolM
z4A?@Npn*WdFX<{(=(jfq0@s!oH1nHpXMZBKKkoLJ;5<<d3Pa}{1^H^+HuYk^7o>uH
z>%iu%r@U&)PA~W>j{K}hyXza?{^QOcUU*^sz!HR<Gtdl(+YC?agjUQ#X3TsO<*Qfl
zSJ)aaTXc~~{2c9fCx_q(3uzP-Ba+asYpMRM@3n|%8YloCS%<tZdQ(wI$oaGhn*gwO
zQ6F@L*t2ju*^qIT<a67!NRvr^$@o*fm6Y4qltsOwpRZfxWTSc^=d=mThju{o=o-NU
zJWnUyS_tq2ncEHxkBCi~3-65wleWwq|Jh!ptaOCxMvp*I_H!Cgoznd4-QY%Sx47W?
zoozgm$j)MBTtl^TV|}#qcZj6Ngy9R<UE}$=yxnDj)g96Qsjx3Dz>t!hPe@h)Ya_xX
z+$X~R>4WrP<F0<HVQ=M$ACDqzy29rcS_iOA8L^RATI5+KjgF*)!Ti^&%TLcHm#}3N
zz!-NWq2gx*3e0X3a=QR9p+u0kMTKvy6HS0`kXlFBRAVw5uF+m38oXHKP!JJ0i|)FG
zrZQul6S-|tco{BRL$dK|3E3KZp7f6=CvHXbChV#LX^PVds+xXFK>H1e<CLJ7`gPl4
znno=~k0f*44&e^wM>PqSt8A7!U=Zw&N&#7mEK+h5I`X+K9~cCI`-!yMvm1AHu(wY<
zOVVyS?pv+UNWb-JKRj!p56{}=iWe+V5IRFa2V8=c^G!`x*@4bHH~W+X)Po;S(#7RV
zQ>0Ja9J-OI*D|v#%3pG1(aC>#m;BDpbg!7X&qp0*kSm88yNC7ld6V0+{dxE7sX1Xv
zP0`HKhO>BM6#3fo={GlZz#VR5Hr0W{=76p>-MVvV7-P!!#urI2gapW>K6<ui+M*l{
z<oU`4k@e7h@?_uT=lI^{^Ks1=1Lt`U4M8BAY+&v&-$DIJKF~Z0d=LTBx;;fbY#4m$
z)sgq7037IQH%ZL7AC6&{9bh(?0b;~MV=4hhO#;g;py{ZtBVZ5e$--QE^vU=m8*eOB
zF_@-e$S^wRu5H_ZWJWiDciQuPHwAr*ESE{~2}h#zBa??H3cDe&XoxZNsxN7(=oRua
zPbPHEu+&r<g`V1nUP04~?twAkW+t{A3I&{3=i7Nj#;S$NZe$VX79VTy4)n!@&IP)h
zcHJxo@)8e6X@eG(Z=)OH!&e<efdMJA4{GdfeJHj2dgc+&w6u+I?y=Wsf#!siHpJQ2
zj>%tir1qW3O=~eHhA^ICY3p30xO&~<aaqBQt5Iq}y*Yp!I&ZE^GA5q+&vX2ZN9-p>
z<kjxg1EyL|x&DT=EEY&TwzvNtRCgH?h?Z1Km3i2gggJoXbWJmmES^3QO}LNLh!+@v
z)^NMkE6?h?eUdC?wJHH(bSL%&JV%MlGbog&u@=Mt(Q;$mBut!dC<oiApCAD`$^_E&
zH4SKO++8znMP7Sf%=N-XEc=nqt;Q=KW_*XN_ot`gixEnqfR~R^=sn^$8|^i^Yc4#R
z=#fC}W`~!FfmQ&JKA8P%BDB)9>?O?jO{Riu{Bdm|SV_c2>vCC|{H#6s)7>Cg`-;c*
zF321$7Pf-T`47#c2l_3#Q<iCDRQwoA0BIK3)e?vh<S6bGZ7P$AVIbpe5Mgd`{?n{!
z?M77*gVvDLxPFg2`CWi!&{A-CLCf_|Cd-Sf7uXQLn)}<GyIpIcmUpr=ofQUJCURuy
z_DG?LkR!kL^EyUX6QEUnHpYP5-zUkjSIgccV@kl+@~%^mzPl1KUPEE{&<tlz4{W-5
zQ#2m@{1FwuM;`y{s>~VQX(K$Ke_s2rAQ{1^skawb1>bHZWo#5_{!>+2IR{rWWPCwQ
zBuf6R3LTw}9-}U{9@s&i2_-JQ6370=^Az$3<MAhy)dAq-3VtR-%qQ55e<gISBhiKs
zxR)a-{zN;KF+wHXT`G}S{WkF&()J?nHy0(%qm8+Brd^-oW;C$sN=G2`bNKYzM?1|P
zXg3M^?`deQJO=<OLDgJS&aCtO-ZKoG9=SfC8`-j_3u84Er;(}Djeffjb{(Q|eQ}T!
zA<@3@W)zkkDp!hYIZ3|(^e88F?HbnI^m;UdA#pd|=Cacz4EdcqL5}$vHE%vM-VOW6
zgcwi_d0LMuN?+-Q3`SI<1p2%TnV=HUdwOvPe2S+Fk=?g0t@gkl?f!#)(Ah96Z-`fU
z+GqX7;SRu{c+un*i{Cg4c1=<c*yC(kdB(rqBQH%phuZu;Y{&egAEx?WVk0$E%|)+x
z^3-xr^wq`+sDr}r(@T&8AoJYU=3(O^u#ukuzqm@Qw2peENdu~T?50IG_)})e3shwj
zF`#)-iCM@y!*4goYy#AMKfLIfX#b}}p-vDP$``CVq*9<((OYC^ul)_^sc&I*V9eQy
z0nO_{b@=TPj!%6m8dAX0({JJ}B~T!GQSf)l1J;wU4%$t7k9>B910y6;;YRd1^uu^f
zV|1-s%?9f=CNN!>`4`h_2E!OfzzOS{we!-A0eZASjqBp%uCY-&${L^ZQ)7O^-fc4&
zDaJY&wt@wIC9-3NCpHqk7jCaDoPgB~y(D4}`HZAsbvmRo@+>{bJa&hHYD7A?Xgy<=
zrZQQ@W#X*mf$S;EKh`OQZLO>1vRq9ESaq*bqQFPHqGeUIg8{xkD;@RFO7F>|#)b&C
zdR)hu*BaHKHGFa5v6*V7i|-X;o7ZI?e^mueyifb6xqkn}Uc)GFynsn6x#s?$>5UJ~
zgEwG}l}8$T&$RSO#l_xW>om8;POw$%{n&=`N7aw(c6W8;SCtl>+!O_7WtHQ#MkX`G
zC4=||gsh03D!l}vnOpmrv^e-t|F&M`VqB5wd{_-oht#EtZZE1Pt@p<{|LO$vML{d%
zVOd8?)v_{~je!KAP;c54-tU=oG^ij1AGZ$$GZnKXC)R49#$&tz9T<f^xeBV4`--h~
zXlcjJW}b$1FA*MQdPIl@mHJn;W!YMAh{&IKS}5=NQxO?(*&O>@{@R>$(vTb=FJ!f3
z1(68QzAr6wey?gzIjw(i@ew2!dF=kQWhe;M4P8Co9x$lWQ<jyd36`E*ZxFvYTzqEh
zsO{fV#i54IZpt?U^el**j=PnLq&*9tP+I9ollL^pK&)^JB1r6WET>VJTc9QAvH?W^
zT`ac_aep%dWD)BoL63bAcWf$4R5HNJae58X(5-q7#6O#vn@vYFp5QMDfq2`tN8cU1
z;&dB$bu%$j6d7iB@jBzv%cenK2nxS(`COD%;33jt<R9p45nnJDa<f1ZQF1I$SUqAq
z@xd(s2ZOVQI+odeeG&K_zaS2lKluZhhdGB5y8lya`W7Yj4M_Mp2f#wtaB}+?$#$>b
z81BG8pJbas2(C%K)W+e4I)D_#i?odO4Zm9wqIQXNISgZ?0xtt_Yg3s(+m$PH+Y;D+
zhnPbDP}2Yj!b3yee?d&670w1}C%afEJx@1nz5Wg{Wv%*{p89u)>BrTg$2(6#RQRtr
zI<&dZB2YxGk)_Ruv#vLBI!+(IK_%8g>J#VRtvyt6Is1YS?#CeG!#O=3BTn#av|Jt8
zDHgVxtSP62@@0K1a_|fnr-n;{d{1$h;8A6*TVS9#J@)|L{t>xc$qs3{U1JkS5NxGm
zew+Mt8J%^PN{O;tD*5WjMWj-)mnf1^Gr$+HF69zGl6kayt4IluHfpkIjzUF0Qm*P^
zp@04@dU7D4=tkB);WoD5rB-1)^U?rh=N6T*pkIH#?4VxB$@BL67~Fb8MJD4@BXH~N
zLU-*m`~bb>zAlE@UH=ce%U)RM2L0nG*82|1nvHj4R|^Yrwt%5Z6j$Is<QPp-ck{X0
zE4F00cwVb0Ed!0&bM(&~KvzB3XG1izF226l$`fQXT;^(PiPVOI!{^1Wfg-`eziat;
zaERuVEAxx6DNV?tEZE%LxVkVnya~!#^_sB}o$ruV>P(a8D;axAmv#*>R*nU;UuI`s
z_et_Y52NHyB!WS@>jdxDdCT{T!K{|B0eapSFv7NH-#*3QkEb5448Lvti#Te%{|<}a
z+A4etcC_k=BrX_*q$Ibd;<9F4*e<Ff$fLvq(^w@c@xDI&qP>$Q{FWRtHmYG5;eV}x
z&ca@<bN8B3;K~<f815&{%lG_P<21z3o_er&r~SYU;T*?rmwN}09PX<^402m#83bEi
z;u+?)Qux~Z*Kle3*Ux)yV4RZMM4M1+Tjg<sed@(sJ?EY1V*Fu@sNdr$pqIf*Yw-=Y
z$8ro3W^5}z*Zt<Qm~0^|()eLBi|tfK2!Dm&1IUP_F6Wb-{59mgv)E7$LKhtyP3V^c
z&&hlpj=Y+P@(Sy_9*X_93<x=O?yCfH4x-Lb@98c2k;)RoG-sQB_3ctwA57^O*AmOx
zeszAf4MCuoE>7HchE!3S!@9U0o0#@QDczRj7Q~fTieS|>v+27mIgvR|jOl5sk_b+v
z1SdSD7r3@3-X-H+wcR?k`bCecJ?S1ovHG^TDqZvBeGh&I%mvqwa6sD>v~UHAJMYG_
zxZjx+p*ODq1}PmJg6Cm1g{mbq-9-O+e0)UDBo_amfKCNo0yg%IF`Hjj%3r)iffueA
zrC-C|7jP)x3vLjJ{89$1KEO%`&)P{WHQAOWUQlYUr76^-h6PolU3x+~auo`Tg=bNq
z1Z;6i^tb2$3J^^83tOzH#;IL7VO&}s8k4l1e`Pcfn`CoDQ^!kGYT`P{l~v4kGtODV
zMVE`ENU0|0Ls-mZ^oT8_)LmTAY0XkI$s<xQh7ZY~%$;&b5ZF@VGAvojx^4py)wezX
ztMqW#sfGM6R4G6roXimep0;vq09w&B{sGHn40~^ND0b34(;s-OS^llwd5N|bEUTgi
zk-qE=rEr9r$R;=et<F;gPN{!*6aSb80EcFJ$YW-XIQ6~yHM<8K2-Q%#gpLS1E@Zk~
zi)E}0Y)AkHN1>?}&rprqTyFxarUTFyRFtD9YH0mC6i6)a=aWLl=1c6P>&{P23Uq|#
zKM_U&+k%q|j6%k6mrD&Rarz)BP5x;r1eNYQu`V0*6zV&r1SHv=*xWOg7zjMw;@LnX
zu2oh27Ykr~n#UsP(la40sb)$zUbAw-?qC3Iihxl=ha5NdrDUAH>8K~*os)z*VsJ82
zKuZ&BF8N}ViX(QEr~BcvKM*zpZ1n}X=$s2T^A}j(v5te?Zd<`;n`mEMPxodfo?w%2
zvV_P^y0FC^9$XjewED&955yZitZ(>ladt(B0(&}UGiAEYaU3>{pDrtD0D0Y_af;t|
z^W&=p0G|>`uC3aKgZ3X+laFYH*y80%a1}6K?gUS1nK2MBmMY<7>!Ze=%%yJBm{ioA
z&VqxGT&$1{mv9dm;)UZ;)Ob<JX6YMtoLt2=|9^p|udxM6%9q)*y|>e(qh9t%zgneG
z+aAdh{`D`QY3sGQsxY;pq*pgAe6+2bMTo1VkubmA(CTssB&L^AE7n8pc3Zr~kdyiS
zt(!TylP58>m3ItXeVK$=UU$qsGu>4+oXj6Yqiwv@s%n-QboOneHoG$#Gc<u5I$dKg
z*n**Ytp0E-(G0F$P~^o=0hYbBMdGr5I$Qq~!v1*!HB`yPa#!gp@N>L;qcfYPT<og(
zW<EgxI7e(v|A(=&j*5co+CCr%ihzKG)KH3wfJiqIj!KDi4&5C?4u}Xt2_hoWCEeZK
zDBa!N-S8c}@B4jy;$7?eTb4_iGw1Am?Q8!oUIL@QP5VL|zEcr*3KQd_R4rW(jf<g-
zVq2d4rC@kZoWBF<)k>wz#ufO6Zp<=@vhJs1uYbP?+DWU1LlZBLL?1vtIWPDssl;X@
zD!0O5?#4B4=cgOV+h^F9TILbOBho>$e>axxCfK2Lft1?FEk0^_{SBcxpVu;#7h;NP
z$@%=~5i=H4H{OJ|G)s|ZNi$X3&Q;edit}|xfzX%aDG!wbgApJHKM(sB0Ms5GfQ%@E
z?4cg0SyqJ(*GqGvb&QwUnuB<|c(q<IDGot5uzM0IYzvU$YUgNjH(RYq|C>E4{noXr
zvPX&elON0;ak|c&nq~Zb%<A{t?DBORWJo~meOG7tDRnO)K-f=9_-iDP1?17RKZslm
zEw<Q`qdZ@hgCSJK#7JzAdkx-rBqzok(1j+5MzX*@w&|cvwa@v$(VHXD1GXVLmvA+v
z>|dxV6h)4xiID(oimWx7F;Vd$4pkAcUdDKbCHeXQgO;P80l6D2VM_-Qj0jqL$Q246
zC04M*Jz9vxQy-a4w!Gkpd+Q&r>92pdrrgIs9IVtM%IPS4>Zs*Yp4UN$zW@1p@;vXY
z(915vJ5RqB&D&~7ezE@O>QnEVH<W$<V}7R2!SBBIOUo6N<Q>H$)za1QV0t6HIU5Hg
zElTEf{OvKo;$xOU?vV^C=fjHl8kx^}CEu4OHktZ}B?}fztF57|_Rrz6JlyuV`rzI!
zXlJ6ckP76}Q3`dhWE(VKjxf%mMAdJl5j6`UcA-Ck%21eJMVx3p&(6f<TRz{Z7kaot
z-7Wm1?H>HaeZ<H{n=4eRvg2V_RMd)r0N;LLamUz$)4iKfVe3|M#AqEQ6yZx~(AD6l
znq|Y6CzITfC1%xABa8ilDWwx`hu=oBUG{k+Ys@sCGpB_x0;Q)UTAY!1gqthKa`0(^
zCojaz6OT$gQ(ghzh!?(o&6oI=)08J(GU;qOsO!M9YTW39-xjr5xhJhw!pwJzg}{j5
zyAH#@+Qt9z_N8O}Rgiw;8PmW3>tKQ;{O`BFNJpV^gWO>=s~i0yH*3A6c|e~f=%czo
z4Jqr9AZdQXvWa#{a4rww`$29OVQMfH?$_}bz4Xxx|91ShR^i<~RRxFuuh;?Dt598E
zASQb3fJTz)`v?;qk4ZIzM0glI7mKc(2jCy(`*E}+sdSvIx9u4hM1xp#v&5aEpP<UP
z4a>uz547oG5D`w=pOOB~Sn!X^t8*!**^&K29Zh($QdyqFk(#dQK`bZ34XP~jxq@&p
z+ZH@`+=o+q1V$9MotT|p0V9o*8{ORg9}Q{hJ#^rHr0<r$9i<W?EJ<oJh@qIFc*@jO
zGwIy4*mdzdV{0>268LUL20jVfPLy4lQq+7Djw!v&gWcB)l3eM{&WP^W3fO>u<{845
zkmg?&+ro8PB+<Q0QRdI_kBz;sg*L9NBNQ_=w1vvAkl`zg`sY9Q+(Ic{9E|oWZComC
zPBCxIOyHH^w`i;$c!_(#2cnhAxkj8_s-s;JN-u@7O9a&5+jEroM$|+T{O8~Fl)@Zp
z99uRWF9Yerc&GSI$e_}z8%XRhmcrPelW@rg#h^rSa~|x<G(F1}gnXnmF&oVc!uv&-
z_%dcc_H|V{ZTDNP`PN{?3ixRvPN2$cX_DmqXIk$M)`q$=YEBl(U8X%SpKE-vrthYI
z+bI3-P0U{hMIQh9jsXn*f(eWg6brPjLws(Oefv5{EfQ~Kp9Tfu*1B1K*>Ly#N231I
zdS9D32E3j5*$+X~7U)F^FphhSxHsTt`4!nDyI)|=9hh{SgYQ4_7xp*9fXip-p)`*X
z5u?z^4nxcdwQAU6-05&$-b?y@3Dfo+Iq{$P5g#8xOR0$ix=m2QfypP#a3szYZ-a8~
zqcBnLC<rX{e<wXRYKZc)Uv2z@e>!pZU?Jlh4{e04v%IU+ajW#tB<}OJ2NlhQ>91o+
z?%Lve7)XwL+*ICG`s<cCqG}F@{_k?dTPLjFa(WuSBpchtK$`yn&$>~L0n&Fz7gE>}
z>Ew@(f&MP-D`o;O_!gFIm|YhLU&!soK8~5SXKpy?zNW~xv!Jw&<VYQ>Rnxp$x3Ab*
za4FgQ5O+fsYI8AHlRpCM8qDn-m9+U4>Y$mYu?cfMBe@m6RFsy(^Q8kZcA_UYFA~R7
zaHRrok54&fH@!UwHUrMw2B*AE?N`T`8Z0?py~JwK&HE{1`~};?`>g&VczpL)$6?(%
z9=qh|brGz}WxUY>q-CC+M#3yG=&qK24Q0wZyHn9kom08%huE$=+&g%A1b{Ff#mxTr
ziu_ai10^&r{dcVqZxL-fQPix#e@4>v0#e(vOC$Qqq7Kuepw{cIKZ|bBaUBbmu4qN+
z&ECgI>>VDXJg<Y55E7W#7pu&&3_)FH9bZ`xI2c-j<k{Aiq%fr>;mv}UeHGPR`qEzu
z%ocQ0xz(qQ3s2vCx7E5UjQcQNeN`eSPgt1c?tG$n!q5lw>i4o#I(BVJTrJ_Q{H)qh
z&Btlgn&^$KVTvDE8i~#y?C;1TR(A9^m%~F&$p}5A_vmw08B^pQl&xgAaWgE?c+*7<
zdo0GkibKslC;Dv<JFh}oFaA+($y1^I9^*g3>-!}Mg}A2|8HKxH2^kLBxm;Cov1!29
ztbxe9mILG+T|Dm2PN7{_p&W%Zr4hW%3%Aar<;{%E*E+)!AEV{-+dhSj>q(KZ?<Bjx
z%3CJCAHL))xxHdCm};EJOdh>{Zc{)P;7WS;Woj2c{R7H9dY9ovk+_o{W1(VAmxJ9I
z7^0x|a%VH-bvjw)8vI>DjTBdqdx3XH<z{orl@6rqV)#hnG#K0CA%C^+ga>oYMSiBS
z%!+M#^vIT^WyTw{Ye&7=@>{_*TZkmL4P^_r7kvd#f5w_fVCx$c4>(QwAB~j;vqb8L
z0X1mU?QBEoSL@bLrPH3@3j1)bwr4q!2?%?Vd3|nPQ!^y!c+{s?nb=wDaU%<quubF5
zft02g7)gXFei)4}{`0QL?ZY>knVd6!s%{!N;qtX3@^;ol^$oLu`uoMMm-0ek788g0
z2Wx%qLzcsD^)|*z3X4p!GjnshH6k-TT!cpR^(C&nJOB;n*Zz7S%l+-}0dh<EOPBd*
z++JqYTp+p+dl*z#0Dt~x&HZ%<GYUzpu+N?_ptn;O`YKKLYqAL07#|UxR`M{!>NE_d
zRsGFZ9bXoWJoRfp3X<50PK!JvIe81UKBhaKMH=C>Cr0%WO5oD3|CFYHYZDH(q>PKq
zwtu$TozW8&KuCSVFk(&bs>e{q`U)_tqA2(*w0^Vbyl$%%**WCdMDDw(fv&*ABbdOu
z-(yU*Jc9Gm-ltE!tqU6+_<>H3SuMe4qS=1gb}*IDIer!)Ih4o#{&}Pfm3qF-7*Yb8
ziu42QE58T`Cf$#<qLLDj=J=OaI*Bgs+TfJj8-H+%oYO>yA<9Pj%lX1m^GN%`&o9>N
z@?AhKC$r0=99Vng?vh-NcX7#YLCzZY^ytoP%az7?SmQx204?qjPnuY;%yhDJNjf#h
zIG*inL_c}Kc0N4AsWs&x*tegPJO+t5M|?TB10~=Y2p0c*U2SwxT^j3*$uf|jb>K8y
z8i5XeRJMI%U8d^j>nB^K>nmr`>d0f$M;Cf*$2gQ#2$?+c%uK^uu}ResI5xRHh&UL`
zHRm=*b=J-;3BH63WDp98n>b{8n3Q&(A<XKf-BgHu+3+VxS-9vCZ&Y4M68vXvyKaX*
zdc<4LnGIX>VY|r6MWL1QpzHosm{gE_TsCgi5L#^1!&R))L}uETTvoN$DX&&w5RF4>
z_1<)7EZ}~IIoMkq%+(f!W*b`qvrl#q-%^w<@Y?hwi1Q(&^0Y>-?>4e(42~fU+U}2#
z`hNkx#&v6KqfL{x-gHOdyY$(gzf1Chw^@DICsD>f&T~qbYQQic+HqA5da%G)u}bKl
zN_^Swi7SgZZM4-+4BR!Pk`CQ|t`~+2Vcyq9EHtpFA@+H&Uo$?De2%X1zFX%8De83j
zWSAmgw?c%@iPp*|@p8*l-ha4DC%b88Dr=!*@5wIIl=kq1ouR`kO1)9UJ04hFQ*!v1
zN$Cpb-)g8f$7}rNzG@@v(sM{2m3wclavz`KwLDc}QvOBj(H#H9FyVU5Cd817S(@vQ
zQ}yJ9usb=}F$k?SJ%jWQm2Xtk$W*&cs74-l)hJ2D^LIvoQh7b25BReYgECoaXBKXn
zIc}$Z0^6%Mcxc88L}ku)R)$VV;RI#-hxiYg-M8kqRJJ;eZ5rppL(Kad&7s(#r1H3;
z6^k1iB}-R>(0u<k_ep_XYA@=~h{7>!Q~eJ8P<1SJnzp3OlO31#4SBXr*xVbOTeh}h
z3flOw2>r0n&l)?0*YAFns!yy;2l*3V_%T9*q?k<!e*L9v){ik0Fa;+$0sRErbgIQG
z-_%HMu`hTt-~3yC_}|q?xC{C_WUPMCYMChixXY9jn0FOPEU9Q5C4D*BlGz_tcbIrk
zY`1dTwD(8wl(B7RlsF(x-WZyicZdrYM3gb9<Tnx;VReVy0@gD_e-f5rAZM>1k}4^o
zq=RG~G-RKdLzX=W1q#)op$$=Y9SN5CF#vN@7cLKcktq`qfSZ}j5b3Q+kPL<e63fQy
zKZ?eTJWu^Op62oF<@B?Ug5ikuFOrwY{X%W3=J^+?o>B)2O7JFQNy*OoS+I#k@3c%O
z=_kSI@wC^+RfG@GwCp{D%su{W^a0oX{R0L#z<Yda2HX%PXI~S7ME*j*TedIa;&EBA
zm~@-s>4aFZID;+Z!-#`}Y*?FL-Z596!nufQOX7$(UC+i17QS)sgz0yIe1d??t^e`*
zr&9b~Amhepe<RzJjG!^ZDH<(uGSxn8LvD=(7{`8O&fB^RgjXu$s#x5KyWwS+{ZwK_
z?2xG(tmq6@CY0h*WQn9Y4C!;9)rCvOl{MoVIlD|98B`jU2yyYIsFrbG!I!r(H45N@
z^S&jqxrWIBC4pIdqxMUeIYX>Utqc^V@``O<!nS7%4)iUr1a=omBl8c)efiRhTsL-g
zpk(@cUEohuWBb<k-3*V}lkkn|6Jx+h=_@prJKJ=qSzAJ{TPYs1C;-!C)3M^LXm>%_
z4_k<q9?xVO`IBQ%@I1lNx)b!j%d`J<LvSDAEL9H5I`5YJrX=ZT-oZRw=+Ve)^GHEN
z`HmZw@kDvi!Ez5B)r>qvwh2c%xIF@`B9@ptu?gG6Dq6M^W}Cv=Z0{mpTpnB1F<|ME
z?!CBmjy7Vv;Q2pX+RQC)K`J4zq^DVW7Yi{!M{;V$Qy;O~_oUfAM12;Ts=1uQk>*w^
zXx11<=S|#0>Qd&_NJL~~^g+^3kdWwoMBF~j=iOTXOB-M<>V5--2>(^%qp(@u2Ie?+
zymc(jre{hgKRdZAW!pUixjzbuNE-0-q}7FnV_&_^Ih}7vcBNf?mMoZXy4)KJkK{T1
z0ffENMCiB7*&`+<;A5Z(76LOYa64R%7jXCeF$qt{Mg2V~3=P_$qU2S*q6?HF5Zaq3
zWgP!3c*G~$EL}w3t?sz*1>bk`Jf8?J){3~;Nh7x@ms<dXluEw(72hb6h1v70iyD0r
z3XWe$*MddzibD^`*g^};;^J^+%YI=5_4=)Wcu!~n5%;i&%4KbfbLQe!nPiRp6u2)k
z!|drY26OTu<7XF{WoLF9rb;o6NhJy#9^SNJ>NdXDhP0vUz_9hiAi;b>Y3@4bgVhCl
zz#dNmuF@5#ic<@PI~ddiz863G<3IMd!z@k=eYM2Q!ZkkizwA#lv}k$K%av&BW=y4~
znU9v%Ayl;dfgSpf{po-oF9A69+=AVN^^Ra6vc$n+sC|(|#RBG9)IVrGY;@yj&8(~D
zsw`uMirsQl2WNF4?^C4Tco>g~+D!AO?3yOBfgDp)?AwU3Ol<^&%W}mgHb+0i+u|~i
zXnHo>FnYA;+))=lD8BDQbqw9*=Mt+qgV>aA1S;Bvjl)v~P9FDe8cyr>LV6Z%_Efc@
zjXP?}PqU31ZdPkDA)()M><eBhRv0jqjF|+(hH5V#yDToG6&sEjXbv$Hm9U)kwiLLI
zzbqL{{|07LGm#8?<4w5iR(BkmWw3=%W^UHsxcdySZD){i4Kh3JG*z1D0W-3n@n9gF
zWTC-Z^T4<Pz-_VD@Eg1|><4&5gX$jI1f1{8yhB*iM?GotWS)1wD{<#QA;@t6%0SNR
zyPqC!-x_24lm3ZlYr}jY7HY!zaG{ZFX}9)LeKQ7+7`B53i#9WpGB0W!_rJ1j#fVO4
z#>cz;Z3sZBBJeeTbnL+9OUPTKi=Op)vE9{p6C#!uL5>Vgxp}oXhZ&YRUQA?1-|d*W
zyZYB!o#sR__sAcgeLB|PhZ=7iza>g9Ypn3dw??zD_F>hj`U1%a#`ENNk8Au^3vWoh
zVpoyNZO$%Ra^ayv423#W;8pcnvgG_N(+pB@A&v}8)njpFIyT*QEF$+$B2_M)h^kZ=
zn3DrFXw$Q8e{2GFdEvfcxO6^_zzTC?yvS&AGq(Nqt1V5Q89f3Lb4oX6xpm=sIh0(z
z2tO=iNl{JWDabA_T5}GAOkL6G4n=(VAr~oq=x+&I>R4a38QBOc$GdrK6jz!nfm1O+
z&F3i$=KT=9SsyxAf#PTGU|V_}7-%@FgM9-i`34s|qu0)oE~ZOU!1_n#_n&~K@+RQA
z@)E@p=s!J@AuRjO4Y_v>=>-T}Su;xOe_;#!tpQk~<C3{$GhOPHbBq*ZF~xFp@$U1~
z$~az-Lq<o7Oec1vKDoMVf5u*)<BBaw?7DA(u{Jc?{s4R9XApzh>U&Yd*@;WeX(^cG
zj(&YCcOG0)7y`1iPYpInAGNUN+1@-XPTM0^3#Mz%oXV{`-R$e(BsSmOYVj{ND=sfG
z85zY{St@6mV11V{L^52H+*Keradc?D*NlT0hZc~WR!#|2*C18{!^yfC^)bA}i?tc?
z^%u&~#@6B8E92FnLnPd@*iJOVpSR|#t18|0*({IwJrB1`BGoU3gf4o6Sa`BykG8tg
z%Z}8Foj*y%UKJrS9FBCXm7@5&2Q#OD2+YBpTr)rH7NaLWzgSydtz6ysKfBwROu5!w
zh}9jVwd!9Ooq&aYWS_AM1tTE!=|SEx4byQ98I*)OkoIi@SlBUYzyPxA?kS-sPQoq3
zdLcnG7QhwhwRi`*Htx;%n^a_%^cbwp=co@WC~^4E#V&eL34u&e=>y+u;;!ooU9T+c
zfZedka4;;^#;Ef+RITv64FeWhkuXz?HMcR-H^pkzr^eUJT@})-_jR2qMwPW+8|<qf
zoXk4Cv4u%<@>6LDs&(zB@MbLgdpYDEV=A%WHhUs(aR+`3@b75<5~ouANV?EM5?{T@
z$QLW-b+91eFJ)_h84>8|zSkxyp=e<>Q7DG?>2qtWBAzc;k#pZvi8B798C3XNvTtuH
zbpsCf<HEf!Kej4&5}4GVaoum}<h&M6Ao2`o9hzB(#ju#8pL$;wXo|<5pA(z5@@z{_
zp3It8?49t1xUSk64-L4%54+e4cke2HnSKt>=AL=8HkOrhWN;Lkj1zHs=~u+K4n4Zc
z4NxgATL6Md)z%6;qT+6#?`c=X%W}(h^Ro}_#hGPC^%Le6WbTOw_N7FEkp2w0E-*df
z%?pa*9=+t)tJ2HtjAcG8C^G7=$8)#|qKK-T;R~TFl$MWpEylS4OpM(FX^Cf(DW~$$
zeF%v|uK0^+76(1b>>VW*yU_Uzp80gGM^m!HkcPtu@a?KelKi((&cVyLJ!`0P!yuFS
z$*&Q`5@SNso-)q?E2tVtC1=xMmYN}Ob?=*Z>2~Hf6#yoa=PBRrFWQV1ALgzJ_mF6-
z9**jPClz)pIsz_-h1X4T30R~KYZJOFc;+*CSW34PHEJnTGj@kQ+&=LA2eaYAHqJ5@
zse5G;x`#$1zaah<t{XsBwoBR=C=a((E2A_VPFmc`ai7)=+^7m-$YIZh0qUuyArm=Q
z?x=;u8l{T2ZmLNc%a6Dmg$(a5wGwHj*p#8kjmk@iWzv*?v1!jRg0*&ijcA*PqNk-J
z#Hr9?^pUUOB%L&Ort<iLC#IwOq7>yq#qn^pVvVLo4d);Sycf#3ylpX9tgm}i?OY(h
zd5H*u=3E>W3}h&U9b?>%t^DE!Chr|57AmgW#giKRV@MVhnfHQB3B=hbECi(}R_J%}
zEtUxhN5xcwHRnlP6+E41AGZeOt(popzKz8bO{I~=X`cVs66cz4oOzz!_FWJ}DeTPC
z;>z+L6lpd+#?&1Ei9DYi7XzY+VRiwG^bum>U-YD35k6o*JbZ$Jg83HX3$c`c4oZK$
zIJ0(qM8@fE3nn>-kM2_~`IfXwEjh$Jzr=y@Z5q52(dXE@Vy4=Xlmjivd4rjbX8mt1
zKqd$+xh>)Amj`(vLDtJ%N@#p{sxKQrf%WhyQ?M`{{cg&@HBO{0)`i%R?2m?bfab4G
z--dOw?5eJK%MJ77hqySo`S~#NJn(j?n*LDhH_(@fq!W60-?}(b5h}9Ss5mKmv3q<W
z<Q8V{@a=jOPJ_IYSHpinvXucb+~@0)-mp!F<o|3t2)b%tWR7Mm7!xcH)Nt6m8l_@7
z+%DJ82pf^Pgq!Q+q)P)p7h>Yq`IVz#rhvf7isS)qOW9o`<-LwT<sZ>H)Mev`>sc|9
zAu3ofIGV{=X0*WZ=+yI$u}|*v@RF4xdxyxApP-&wGlNgHGrksa4=gwxX%J$1xp+T!
zOPJhkeVg9Z@UGD#IGDYu0dyB)LTXdAM|IOvRS$INz>?i{avA6bmdXO}`W^o9KL1mj
z0S$5nu4ir8Z07F*3Fx=_5yE6K^wsR$Pkd9(iX(x~hMZ+x(oZCMTOeh~s=&JX8!+^V
zhg>W_iX#d6%2E&7vxK^>f82}VeYv7sBJs-~B7L;8&h8)Mc#Czuv;QYmarjQS?uo(b
zNKdjw^ypzdW?zWC{c3mp;?qafXC_CR6Xu;!Jc=r?uidWsMZYI)TrEzUv+H)jqWi+J
zT!Z7!j<E<vNc|a5Sx|(`w$Z!UlDi~Cb^_8~_(zwTajv97nk%$tbChk;1g#sp2~tk?
z%W`F5PNANPwsQQujs(BtZ0$9slq?1sNTQpMe<HtO%UX2!bG!azqEAf)(=Y80s!y2+
z#S2MKXJ(g?tjD+5T|?FOzSf}B!UTcCnfD|Cx;^&}F`yt|qvKgC;-6zz=+|^Dtgk5$
zk|Wl~C2d}Ry}_4onx4Yy!jC)%4LaT5VIYBWZwUM~aT_RhZ>0PGPz)C8U3SBl)F#Rg
zTzgeut_Bb(rwSYFb&(Npo5Q5+>ADNe#@j7O;i5Ko2qBeK92ZQH#nWsK$tN~z!^w#o
zB|~m8g|fMMwb_?z3Qe20(Lbxb9AEW)PD<S)d93w;VTRqZe_VPQ@uA41_la-W!HC-W
z1<RC7v%<*9oR_=JRD!8WW@Bqb_F>CCA~}OQxLa<e92TDiE#4&4Aft4y^;Xfq*xRo~
z9!;IhQmPeMgpM<Xe9yONP<5G^BQftBDwsIgA;<Y3qXbo7ok%MYGBKIi^*Y>R6_S{h
zSk>EHEO5RsIn$Z3iw*gnwwF9g-Z%NG{>W^kj{52q_!m05f3Vh53c?jwPt}CB&5uQk
z$BlF$)V9HFo<%XT4rB=#od<puhu;L|qmt8M6_O}*Gu{r>9}a9HWfmRIx<eFgXIrj1
zcoX3WObXXoTy~Ks#ec}pe@!yaU&eWiA(E5FLiPJ@9=X+41!XJLSd@3=4}WRGe`~^(
zy5++rUOaNLZarLg4S7<lnUDM^>51t0AP&-b`4}_%+Bu1{i#j$hnii$J1}&ygi%n+k
zWGL-_Vp1)9gqpGCYpAvdr(E}cY^ho|Isk9YOBwr-LRCn#X_RXA+|kNCtj0?!hJS+6
zNnWdk(jisggnsLO$#Jv}8E2-cmsjMtAzu;2{iN2K8sl|%qRQOlT6eM4mb=ebVOnvK
z*+}mS*_npT@P$_wUj32i$m-VZYkJN7)o0Ng%p1aVo9`Xb{*~RV!9~;8-J*1kdJ0l@
zNf++ndAMQ;`k2fFu{aMXWGEZ%a_oqI^{-z1Bz0h{0KxS#c?0#2OSzc<`sMWM+4@s$
zT-W=^J&~}2N5-8VFWx^)P?z+p-k+gI#Y9p0ey+~7g8f!!AS_mAU#D9+Yc(XOdfzX_
z%QfLNTg@!ySa<HB(dKV-v!~PJnze@@AB&=2iE2eO|4v@JsOGL5B;a03MEaKGEk)Ni
zSYTL0-;E-12?DXM+v9&j3O&L5yOTf~j~!B))%GyZ--5XwLW-RS{PZaJn(%>IqA5Fp
z#V(mXC3tbcOkZTWTb!o{EOB)euo5@;$W9moebkrdk}*6ZFQuN2?eITXXcc##ec`z=
zj=J=ahK;x*ck0uHIT#4ruKA2*zR;*vdk1$q&uUp8Ir4`z>_yX6>$Xsq3cRLQ8_Xv#
z^1%s8Y5#TbkKsXQb?;`(l8+<<ai7<XO;fPxQDtHX#B^%|Fc)mnWLq1`DFJKabWy5l
z7tt@`c#gjGtslz;J+Y^Xw{9*RIYh=zlBN3{xbOJGl(#0U{8uahxb&LA;lDEbk96|K
zs#fYG7KLLQj~g`HS&Cdu_C>t8I!w#UbV#kBYV>61RG-moWr;0I|A=`oSHV`TVA)hn
z&@^Vl+Dc|jy|4*i->>pwoGvi$v0U<SCJ~25U{N4<-_&hv>fU?YXdRM#rKXHyU|VhD
z@@(tqJ%#mRQ?bs9NR9te`9Q{cEuBnZLOsr5e=5x7Y<s5RqDL0&AMeZ+bn=dZC7_ug
zeN~_kQ#$EcLch;Qpjf1kl`u+Al-u*G+*|_j3uEn}rt6=$1dHhcxob<ZCh*@ja+nQw
zZjrY4$b(^pgBv4do<UYetkBfZNT|EKC|U;vk_$b;tm0?wE#byn!-t=O*hx?}QAkAo
zW(psp_fv;C-pNl4`6Ot+uvGpU(Sc8%=QEm{qdbz@f|PG;Fci+ASpAu-e9n#s;v-?Y
z3vuFKB@oR7)=%q)@3NePmMQc+$JDddd#-9#`b<ha<iS@#G_0pLZ=urSk>vHVJ(Ol0
z?xrb!9I#@VIeMJ>usk<e(3+k_Fu$iIOx>@kSy?bW2;xNAH#BUIh;>{LhE^`7pQb0W
z3Th+H6<|f(9CkyRJ6H@zkr+S}1_wK4s+IfVk0<yE#@`1UEk&kxY5PU7Qp?w^9L=I$
zP1oXi4U#80CYJ_pl6sXDplCR$`4#Q0b#z%{RA+h^PEDgkoSRiC>jP;y21O;Nlatv3
z7pkT`sZDS=JjrQ#;B4cdOLMAIG4%GUXP>p{OUiY>PF~sgc|R!T(zKDwACvTzzVB^@
zPNsf6Tvg&zXLG2Rb?jNBjpV#ybJ=WIy0a^xvH%P`d4zaWW=!N|mICDkD-R&QE^95E
z77cJh4wp5dV;E2z#zlYY9zStT=UJn(1*e+*>UE>N3zwYNvriJ`ax|+ljXQbQo&t=v
zu1+MsQ-0N@^9YR2^=zjen?-VV<ySP{P;+3%rKPM>@uA>Sz1(^u+fAYj*-@?07Pj@S
zp$W|dY>4yTC#hSuWW08%SBuGZd7u7MR!kQ{yK7`JWI5gXjNCk$?&o%Oz+5<4k#T51
zioj(6rSrhE8o9NEHi+6pvQ%ao-LTmct*~$@?<$Ef=0Si=iaeDP$WX`$b}^|O!B@L&
zzR~NXSfb6gHkFiD1wcBSmXrK=?H*8^;+&<9LMjBN%Eil1S;mov#o8UKABAlJR9yMR
z3y&(WGd6jZn&@0HoRbBsiMf`#Wxjaa^}JW=WM;!RpL;{9C2-@M9}szs3k+EbJEM=b
znSPMXf=Hh{y4k1JVq{cw707;zs&$bDms|}E8=CGr6x`j107?8@G+(Rk!}A(EgB02$
zRqn0_v0Kc6Rs={C>GwWQmQBnk8*aSMc|YCY^$EMG^iDqhPs}?2%Bv_wG$a-HsUDA<
zHg-SN5n)4j+|3|;aGwpTx~|^N2($ZzDTFR^S8$gu%(cHInRLPa2@J`991J@ED=v>%
zcu##66RMTIEn#{Au{>_qsHZ(k-0R<{yxRQm6T?BUk3%DUSr~x7-kw&zoaPT9n55Vb
z&`%NMkq-jl5*gL284m>enn|nZGjp8MnFWVW^fKgtF^&7$n0D<zIumRj`O_P~^S3vE
zf?xOB`1R#3m<pVXg$V#wHX6fpR4708Rp^^>D4qN-HO?+#z65a<^xijI4(zfioY7|C
ze9S&`Uf;5xj0N-Sx5fH*$p*690Ee3ED{a|$4W}idvX4sXEKm)Pw(B~*V$5|eu;oK#
zzSfBAQL=#Rm}Z*3|AUMf!RNyj2a}u2$uXB$DR`7423dTn>FG)IXi^G89lx$IO9p{{
z0W_V<YKiT8fIBlrD8nDG{ols?{4e6>;w}|`QW{?@?B%Fkqm3l+gB+qu6C>;Mwk&(;
z^bFyRgn@#)E&NGzPtFUH$7Tpcq*aoL_YUN&wqCAqzbG+Y;|BoTg+PZ<J(IklG`SXc
z)CBm(C@137Zp<(iVFn85&EXWB?+(V-vOb_$s{ks>&j9dx6`0R9`s(jay#aMX)Nemy
zsnch*>HFKXo`4%WhvA_K_W_i-yVtw8fj4*;@L?Xr0<`3a72~E5Y9S?MNX26OiC0n2
z<DB2nSo;0MoYTg4vC7tubK@Cehy{3op_OWA!XlIVoQOrpjEKNbNxl$;Nm1*$BFvr?
zdAVQoY!Uxon6Y<P8pHWp|D76ZH01B6ysyfF7_8lq+z);xggDWW1C&Zefh?2HGVm-U
z#j^d!z6NlYQny|20)ifI%7xUB>W?08;qM-A_t~d#8&PUNCw#vu9oJ^AFfew8*I<!5
zD5ZM#y868B=i#)ed&VT;q_`m%Y^{*<>9yXyo@zvn>sJ%CZ?eh$@znlf)i@WO_Mw6*
zGak9^ag<1`JuESmWP%2yZB@U`w>nXBV_h$9%MR}w&y#x9p?W~kmehSPJ~u*El<DC=
zp<Fu1Nkq)ui4z2$@YElnOiS`UBY27Ql+L}M^JA(7K6&+)QMqE27{N1fW^l=EI(&C5
z-znsi`}zYJzmZb>>W?bL+&-jWI(B%7%`DU+cxC5}INt2nu?97#6<^=Ia=Mt+(_*P~
zWX@EUsM^_jL6>9~)rbJHYkbmi4ky-9>0B0Ih^}UyX-+d2<Ik!Hfdt)Cby~k#Y+<}`
zaawolNA-E<p-@hpwj0Rz{;tQg@j0iv3W6pyjqT8bp2R-owOS>_$!D2^zM5w%$|J`3
zHZ%TsJYO*>i|*QK<2D2ML0P#4?J|3gShy(?orq|mTjx20wA)E{3+|(3AMIZsIFbh*
z@MQ@n1ZhzBK)Y@A38Zq3rH|9A?aEmrFtg5maQ4<7#`n4E0YOZ#UVUCh8ht7^W-&wE
zWJfMQ^+acGc6zT9%ufD#Lsx|I_+ipfZhxp@_#U_a{^UiTHBaBBzFv?Ay0qDi9caqM
zFxk))JgS3JV#Fo(*Try*qstj3e<0sB_gN?Y5`%9xVp{dgpd%j?k$iX_fJ3T#kpa7C
zKLlxX=*NJiG86-hrQi?$qm}RxzND^ok%kj*Olrd%_GN>|@{(FNYEQio>ywGJP)hr+
zQqjBv5l6grV{2R%#$FhvS(={B#rr>l#--Ug%;rvuK>Og94I2xA<#Jh9coP$4J3vK~
z9N;JW;j)6TtrrYq*9Dh<n{h%rD74H$5yffni_{Xb;~5W#Q^8WslN;>i@atkp$aK6l
zn9jWY>tjxUo3Rwy`DLjhRbtwiDMND4zhb{v!h9fIVGxY&rli#>M~9|z^|FgBrzRnc
z;vZ}~d0IjYiJ6{^WfxT&#a#TJ*L$&!X{=3PuzXlLCpbpI%6j#<(8q@*9uIMhR@hr%
zQU8gWYaBJ$ik%oDTDJ3|^+l}WL1-9j5j$*uzsd6RWZD;s)r^?cjKKZ=`FFUguB}oq
zU2D3iCp|x*Jb7MaKC+YxEh*|EvwnULC7Gone2pdDPG@v_yh2F0mluB0_7oIgbMa>;
zo`c00_X&B`W-Nqj*>em^H`>YR4CUPfL*{smgZZD|t5t`X6eZjuM7i+>bfOyzq*q82
z!77=Ql@;c?N(y1l)pmSR#=+Tisu|v$I)*n?e^D_*pj=G<Vgh+0`uDj01qI<2iU)w)
z2mn$Y`CIp@S!Q>OQ%<SPs{8&1!&}XlgP1YEvVKYazUpvQA>cck{iXcbUa8PR4F5~{
z+W<up;=L2&vYBVl5pjZgMWSKXBOdy}^1r$+T+wkioKpd*tu=7n{wtBiQ@e_8v;34v
zm*?THVg(UqZC@S?YE8{eoK21@mRc0?(WZ3=8w@IY8klFLY84Hn54EoLn`#wlRJb(_
zO-|;j@G&Y^HLh|QFZgoA@*@hmc$UW{vfL7zIcFwoj(2V`L}{&?oQ#_0oa~H{MC*_b
z5)X&PG8F<za17X?3!3d$dYgth49<^tCf(+(FR~QI77v?{d;U%KYl9txb|8*#d2ADU
zXXjw4QlPFS4;!a92PRY#iPOTfk^T-`Ev|A##T8o0b)wVZLgy${t9#=RL6kAPx?8o)
zM5nKp`X^%*D=}yB7ry4$v?lZVC4O6NaKTz5q!Zd9ZAHdGQxp#tkU_2u#O{M>+^d9)
zh=*4<$a(C888b7+8|ii>f_jA(5;+S)%VcKTzii(O3G+pL<7nGGL-MASdp*8J_`%CK
zgJwn&=UD|$PWqb9B(jx*oyxW-Ktg{O<Mh$gT9l=|4uwx7YEfh8Anhl=9a~;n%$hT-
z`PU6dQ%3b-V${3{S$W3X^pxxQ_Loqz2j9NEio+*3>07;qOU-u)__O^{amKCQ{mTe#
zPulV;i4I&fm7?IPX-Lbz#a^Pp+rB!aEjM7EZ+7tfEEei!LC(2(x|DCJW`D2>ivLr$
zemk}Dq4dy8AH%+6sZF)iQ|=V!i5%|8KRSYcM7edWYw8N%{x%7DPY^4mNK?*{rl`O=
z4=aAQzDxY(La{c9Z8q}7O{jY&ZZ1=C<COCVrsj}hS5eoWx-!XCsvE`qB`xz^LlX^n
z5`tbBjO50fZFvCtk2(27C&xsE1B=`^2<|bx-eNUP)?6FR%7O+_f%mo6%I5C^qR;dB
zd6a-lF<9HmyAxwTQFX9cv6X+LgZ~;Xz;ZWOVY%Vfdw$yL_nN|1H}-GJ^SXAkCd>RJ
zAG8XfB;QdF|BwUFi)lHI3%>$}_<tzjs1Bq&VBy&Ld2c1P$*X48SC=XRJIkcpA|H3Z
zUL15xyR_+{RObFe+$o%FD$;do(QS2&9LB0E%BjSQ<*#AF>7lW%M))IriRLKACBz<?
z#Gk3Z8);~3+MqsS3R_Ha`uWoT5%N@(FfTnPsvrNgw_=%Z5{s-wGoa=<?&1ff3T81|
zS_!32)oQqu(M3_I<oivP{eR6_(i^x9m?^I>2g>WqLHe_X$=-}M#t$R8U?Q`p+xLI_
zP^}C*=@<vIbGvFz$Eh3_E*m#-c3PyN)C5Lw-uXPOe|fNef8Mzv_(nYmUPQ#is^E4<
zjIH_94OVJkp6RqlD(1qAC~ekP0K5p(e4<@H#x>u1uGlb_&nZ1mJrGO2kJ9Lv8l4IS
z4ez~w-l+-A*X=s&YiM9S!`+ca`zjpBeqmz+U9FP!^4*j87R);0(=@?qAigZfQ(9AO
zD`_s&Be|O!duW&S5<|7Z@OV#;--#lc;uEQ|7JpQ>d*+;KBT|@SKZEtgkKw5wZ#`i_
zeDuX8_3*<(oRnuDX;W0+{TfR)&b{yhr_QE9thG-3PS*KyhTxv;6c~fS_xqLZ=$cr2
z#gdv(48{i;W$pI_V@JDkBnu+M;LX~I=5##kmmJ51toE}95dKm-KuH5&F%epK)xWHx
zWro(te>}MIlz&-A({bm%m)Tv<wj2P9px4@OP~-Z^9y!=FR+s(HU+-HA4**)|SIkn?
z2bRr)jSuxVe=8Z-3x6mXoEuFoW+0xA)BW;%_SS6bDD7LETkN)!%Im_O+nEqf;;xgG
zzTNk+*q5kG&Z?!TRurX;_;Eq4P*>CiPc>>vp1ZE1!FbM&w~no4VMRtwQlC51{1#}v
zRp&o{R#V3u2>V3reqto)#=OeLFpwx%ulzy7bf8=w5AqKr4^?IH{nNmRr%c!c$GwCJ
zURroB#Dk<!@}8Y8=dSdAXo%WPYLYo%+v5u%aotDznghbn+GP;Fi-|n1a>}(c&UYWc
z9yz1x43W_^y}c3at{ltr{NYO{jUloMe@dg~MoOdibnak{^m>T6r5)w;$FC{$pCMuc
zyFok7_2Iq&eu`^6!Qqb|`r~5&>!f^W;Gh<OXF;XG8wW8~KIU<FP#Hd$xJ3<|8+jb&
zeUqG-D&}mO7sxEls!=c#SDOgCw^Ty(dj(=KnF9=Ji=}1e@wrjeXW6v8A+@B^C?BG$
zo40S&XMJvDe0vAJO9y62l!x6U)u_4}vk$rx=m0%SYL$>=QAzq5_3+eQ0cIcdWT&CI
zX?**Ew?tVtJ_Dus_bHqZCw$LGFA)RV&AaDuM*2Rx&uExl`9G|A=qKfWH?DA)CQNGy
zgF4~Xd+PQK5)qA*K5QHp<=FFMLPoI2m7AAV?P^p;oliZVB*<Hoy*8H`lv=$Ar5vCY
z5@F=i-F=-Bk<a#!K?G+YDb#4Pzb-6eZyM$f-U|h<-MrEp9an~1{L??4b_xL}BXjRp
zT8TW5VIpker*X}1`ron_4=1n|yGsW6`f|wX<jB_faU6cYz&`wdmVowQpSg<bYA<@B
zgsY;Y#eTRVa%s;gJUpfId~(6Ua%!scM_lSN%HLm$+qOSv#s2V-J%Xa|PlD#T9e@72
zvnLik!7a-m^az~#d!PR0L;ItdN&Si?Kjvz?HXQFM1t97my;|k*kn(+!P+jy0LEO35
zo3k{6{3x2Qw$a2WZnZsFE}ecdjmPkwH3~&%2f+4TZwKR4?WW>#TI04#Kp?wFr%>eS
zHoJ&4nw$DOh~&QItdrLk7__RZbRl_(>DDfRd*XLR6UmkYmKKQXzw-xh7}~8?ShR^@
zydjPbAi$cR)zMKWSTR;Xr;Qd44gBWX?onfb?)(Ykrc*8>p03z|v9BENN(1+ZQQq21
zfY>n2UQWw!dV~aKuCTRa9R-N{u3@t_)F^Lnwx8d)?R5Zn0{U&s&7C#bW_QZ`#ZN(`
zuXUU}`|oZs&B%ed^G~1Vt}qDf2Hs!(?oT}aAB1Uxw=KYa)4jq3ng8qf+mLu92J*JQ
z3?Ri1(rKSilQ&@d`2wv&ucUNglOV_A8@A>RS}GJ?F);$`+w(}NJd-9ytU18Nq7y3f
z@vyA@E(jkG4hqZzsH_>~Jd?B>ipC7Z??{S(Jkx5w7Vo&0PJU|s=^TkYJ;^BMA*qvY
z1}fR8=GW%8E+5FMfb(ok>}P~u_zLU2ohlZb=_L;eMm`p8<hgMkl4l_$T94EecYjAe
zlf=h{_2qL!+T-Wg#oe$2aRVAm-g6&WSe9H8*0Xv{^04OP)WE@{qE|AMW{m?jN(n>u
ziBz!#&o+hUa#M4ETUzC`Ew!T@|M)HF`@EmLwzS^=Kqg0o?IA`xb3x~kteh*p^q+Oh
zUvl+41scUjf8EAIHoex*QITp<Z%1f9gHv{CVIMNlgfDG{JI%2j1^MFRh_c*>8Ye!S
zhEW0k@Pb7e9bseb<-Bf%`6DIh<K=K`#;IRQXw6#k`Y|hPq5CC-tj}gf)WYW3%X-&g
zc9vB(r5(m@x?@I7)1(JnA7HKbWzEgfbIKCv!fe=>i$7vNf^VOqJpAFC+9W3QD+i+$
zFxwhL7IH{Y6G?(+(K9VX3=keKqii0R=#XZx2pi)U{R*PUXGB^M>@4Rp#=1tgulYjf
zNt#my1@{*+6c6<AY`1CC=CX^cvMFZ*t!6o*zJf*l&#ByhnfjXk?cw?gL36eQD^d$c
z>tB`H=mfqWuOBX)xqDgv*Eh7)9=#mCu^OaJ1|kfzWjH!>zGBUjdL-wlb<$t&9`d7T
z@b@o;3z0-&?(d67y+FPKfmq|Ht`Eo<@tjZk!lKm)29SMm+%^fw5a9qZ+aKax;_At9
zL#_>uEBypeDQm|03|^WpwCL4z1*&E+R7E02B>KHE-h9<Uyj}e8w!moDHjg@6ayt4+
zQi@E#dnM^v5!slx3@|&t7MX({ThZa32bK4i@fW>V1o!caW_Z7R<0ohJ`Ckl<DU^KO
zFR3Yrd)5C%(&-HEJ!DpP#b`ODQH&2?H#hy0+~3Gn`RPc~lo&QFGfi{bwPJHF(;|Jw
zu>?0lwMy5#4o&W{#nQF0wyDuk`}6Og2=H4e(K#KKR^XsbIo1FXzXEF2u4O>{XeEBC
ztOXnzB}JxtH3lHt9@s~JzfL27Y-y39xUeA>X0AV<rBP9Q6f5NJ#u=76_6qCX(WLng
z;|2BmRa{{0KB|I4!9c`UtUiZ>z{qCn+ZIU1DC0`wPPBP34XD&CVU?_Z8gKso+};-5
z<zCNi08)$KUTm3SR;W_dz#^%~BG)Far0vji2V;gi0((f&TFk3}vb)Vz7>&AEExYjp
zH*Rp>*e{#L>GnQprMvv_icLCn_L)}KVvefckZUTE$yU;$6bCR>oa<!NF<se7RCo1G
z6BE5Z=DGH?y`(^NK;0rps*{L`7|m%Hw7-<UR{i>rYIK|A?ehJCXDchF@$+(-YtD%8
z<k2?xH(q0G--E&$?@tJ?E+>~++}1l?ZcB7}T;jvDtN@C%XAc=T2fj!j<OO?<56j1L
zGyA&A!`zk^p!|H@h1Td1b-+xq{KOywDW>2nT=0{*V))1Fkf(yJS>JO;)T8Z>C@aRo
zkumxMTth)?`{A0r+XRtCT8ib*1JlvbBF&4#H$ZFAq`C%JjAoOSPA59W6L7V)@v;K*
zye49>4#=dIr{mQT!SS>C6Zuk(D=PWhbr8*33_Lf=MJDH}T`U07Tx?cDF3r0_f4#(u
z_g-%gFzJ@?aM{6%LEu2DY(t3kHEF(<pn{y-V|KFQlR%AnTFTLNrVpdTkJS#k;8(}u
zVAFYX;tWWIRW9!|*}%uWE5f<>M>qGk?~VywwZvi*9;a;^1j_`$&4Y0-G}OxNysqBO
zrYo3cxg2G4!q@MCq1eX8m?gt7bA_EvY+`l+F;hfomrF`{ah|}%eg##b{hFI~EH;-F
z>;t{qr?dK5mQ8<u4s{`xu1uA^fUZQ-Aqy?=?6O`4Qi3>>j&Qb?<L3`x-scNpuyezo
z8A>^aja1F{tNpCa8o1oOWF<!WbKdTHjNh1xCvbTN1>JU7*YY(m=eB+{`IDSEx+eah
zyS`D~EZ}nE;6~-&gHGSE^1VUbiNYDIu9vb<hYBXO*sVzJk<c6Dw9Kf^A2~Bq&gT`M
zx-I*L)|fT+F4aZ)JPX)?LdiR&@c6cG?WIvUqm!34d48sf&VCkV@)#GgK#yF^A#Hv@
z?Dtji`&VJ=u+xarT~kSNzw;uk36$M84DP^Bkz8#v4TVj+zSeX4hXs&|GbazBtp8pu
z?2XcZ{od&nQiJ3&J*V1aHjzggfTcC7$r3}lNdh@Egfe9);LKT=Q_h<v6MGT(GQTu3
zuLuL%4_jrWHLO2kenwI`r(RE1Hc1A1{n&P%N2O5_=)1g9_13p}H~{n(V<k<eB&k@z
ziOb!vnqv)O8v`3_S|VbP=;D#KOS^g^<2_gbT#79GA%RivblSTc@C$Cx5BnDxV6mTD
zeA$L}fs7DiUonw%S0?w`%Dqm$ll_r<(Y*HgF#;~h;lr{I>HF@9#qk{U%VlXdcnf};
zioSt~EBP~oha1F=WC05_IB>UqJtQRo*UE>-i(;?;x>ial6>JDjRUm+6<8Vgj@$hK#
zwNuA+v4%bJ{m&}o_X7&9lhWha5DH`cqBGk0C*&Nf4A6u$eTi>R3jqA8t>`CZnL#K+
z_pjKR#$esmF>U0AzUd%+Y=zt9$x-0)JFrs@F(pyA`asv2Y4hXzO-|cIe$#%RiAJT7
ze0|2=-Sv_w=}z~k8)fm}Z*#a#-z+Up%+#oWU$UtR4lPK8uZwjWR=^Mkp_VSd(%r_|
zr#O&@VQc7^-;v+spJ{p&Q*1Es=Ik^SnyLTuTQLChZV7>Nnr=_$pF28x*J!vXdg-P(
zk&lcxB5)!>92*{Jh#B#Ca~6xCm4bf`dGN#VVtE?H2eZ~6mt42>rjM%K@XYPFMDb-)
z$qd(FG3vT7!Ev+!D-8$(Xq7CqrCssCfeHYXqKSFqM_TuV&UtquH$V`KSY;zw*{*}}
zb`S~(281g;3ZLs%G!jzRitohuzsN3Wq;~VO5Bu=u+HRqGJ2ImtM}SP~_67C6{~^0T
zfnP`gLRsApUDj2S>hSvJuin2TV&of-vxeMhm*Kz4AOx@dNn@0_)6)h&k+_)`lfDcw
zMj#f+p(b<n)N*(V5GFazhM$#Wrau5-b_!VRl)QGiYuOc@06A+3qRxtiu1>lxN!$j?
z*Q|^5!FB50vA)=cN!r3i6B~HaPvvqYtCnl$!O&f3*)us>VrFTH*i+=MPkZW-BV2YI
zeZ%($_m|gkGY-U-t`GQ!m%cx!7O&l6z2$xnhmu#mA(M#}Q0I51#A-c+;65|U539Yp
z0oZBGc`fT=M{*?4N{1v$6vQ7bxouS)opa&k6RYL~fGQ-uz}md2%7NSgUM44XT?s#Q
z*nF)J`5F|$5`pLu;}v`0WV;;ws5H2)V7$zJk^eDj`CS~Qs$!k5$XxJi8?5v$a2j=!
z!*|;RGBvAsEsnkyxh;Kf3o%rgYd)ADAzpJ9fc^M*uP<GJt%eS#F~XpOOa+OXHO~AT
zI}>r1WxhF%5vVNJ7X%Vu3kvxg3tT$5rhaW#bW%8sTQ?hO@Z<F{v+?82+iJmsXZGg7
zAshh${njlq|MY>95xM5ELl-K&qO~$74pU>su&2d&TY^iL6tG%760%+K*U&g_kXF!|
zPnpF4fH62+uhGe3j`pHp=UA$CSP1L;ie-X$72Bacgz=89p+SN>1g~ziA@!XoFh+di
z@w(otgAK6xca>(+3%ZTr)ZoPgY%1rD0rGL7h*+I%eYc2^^@xJSu=^~XUh5H%od~(%
zFbyZafiI~~#hi>dXjCUq_R(UzSUYb9?zhj3yHYUucl|<Drcv{Ayu~9AENX8NI>6u<
znL+-Zxw4RIF^+pfG0~}qwjD_cTh*QG<&}r!sTN=fr&Er$C3585kUG;GLYyr}c`<5m
zJthLAm>WO9h+3vkx(MQS9RK3SOv7sWb*3UXhT}@F*oQP-vc{g|cmN5nXG^TXbM6to
z#}{g=*=mN-JQv84Xidy^5su@|;+G%3tO_WomyVBLanGiK0Px3q8Q{Jay4Vka?6Ix+
z&;@QIiu+uFC8h6Dc%>&1xluY7m^Pde>o#i86%$Qn2nW%;TJCr5odS~aD5#4fSKcMh
z0a&UWw{HHY_rLpxj+ZEW-&*~yTdh;@Cg1LQsPGc4<Dr*(A~9%OKizhCO8g&3tPqRB
zLEYiVX%l*>oSg9VT`bKWkJY1PuBpn1v1sp>>dhBc&C1mS8J3$t@DJ2|#pY^PcLG>m
z@7}ioBs^d>ncSYgzhRSDic>V8o-IwuR~~KCOvV=}1KQt>sE{KwN6|cmwr}7at;Rij
zys5g*tSLE*+dbs(|Bd9HI?AaPmC?*yD^Eay?_$Ba2w`c|(_C+)ZPaQYk%g6Y+p@+!
zizQFCc?+Qp(wX<Kam0-%kvmcEZ;|1@Iq(*-zx-VM#K)^rR046@50#ItJ~!@JOTfLz
zttS5_`AS|@dv}JO!`end=2LDChAFEmh1<U9Ta{Fc&o-?PA+z$=>1p4)W66<JG&iT7
zNpsYHb8PnOQ$C1(tRR%x_NfUon5Af2JCjqc#W~klqhI%zSF1EG7Q4AKkTw_PYw4nH
z({|^+`u^=`%Dtuv41q2wgzQ39_CKdS!8a<61r;wi4sd(^AB)@!$^~HL*(5#<L0oE6
zs>1iR-JSNkIGr}<XOn#{6)1qojUTYvFCfgqtfqrcT^7R)-=#)bH{VawFqlLfX^?|>
z_4Z_GN3bB}?uI0yzc@Xx9rLi5>F5$Tit6~znn*qLya$wG;kBe8W@3Md7M!%pdGcgv
zl``Vs;=%bAMG7{Dq{qEKJ@*BI-lJzNwExi|y+PK>LMip>UIuJexG9x4HDQ?F^`M~Z
z-nR~ko@Iu|%{U2l9UJ2ZOF2#kNyE*KS3W@B<B8mf!Vq(w{pwHZYMdt_@5#z3R;Jfi
z`DKt_XrL}g@L(WCT+EKffm_Jz*eW<y(N<pbyhndjv?e$Cm|)i-Uho^S?8UDG`mCm7
zC4a`3uLPBtG;5cWAC>IWJX0-IB~B36Cw@!e>~RlG?k&RO70v;GJk9rxw<3cU=gQL}
z-#};rNtp+f%vC>9%t$HW?C-une&pPJQQqU4Yq)42NyU6l(_NUA3gtHU5uO`d@WT9h
zeFGXAzEQfb27-!B(8)IS!P*nQZ?bISOHQ|Q$Fnuq71?Wc+^~oNQ`M@`1GAZ4ui{OT
z%L)<=vLRD5|1cKjkVHSSL1Uwgq3q&?jN=Q<QnN)jGY&8_f<()gI*d5o;l2<WOt8j|
z&Ano}Xs2gwdBKrB*1E_NzUB68FR|?WBcQ?cgSsIsiReX_X?*JOP>#mb)KS7|3Q@+a
zr*4qIARw2|$xcUNc;3W$6K&Yeb*oe8%Ju3KTJ>Wq@QLe*X8>Cby!!>F695dZRl*ia
z?%i_)@fQ^v!Y3RS@TF*nHyRcj-dOA$CjHxnRm2D9ic$pX!{V8R707{Xjj<4Yt|qRB
z$~S7<sVxooo=BK7rDE_EYdolqbJ2Kf)^ACQ`n$xsZlf2F*Oy5f5F^f?F&i&}>*lcy
z5eqX|Kb!<(o4|iJ3eBUTQBFuRtv?b3s)YJ?>8OU6V$>~I?EIM#%h^Pmvo9bdc#-uI
zyw*3TNtZqDah9T*(4=dLrch+BVL-&PY(58#FDGH^CILYm_91pv7is4yZQHTVCpUUd
z8Uj1Gk0j{40WtBtLobx204C?6@-W0ndXuJ(Y%l?Xl}Ch+jqrAN0*4doM=2ElX9<xF
zz&tNKfzmur^Nj=(O7R%t?0VmnseMiU@ivf;4(#O#IA=}E{Yh>*m8K%Kpgu{0Zs}X)
zX{Y67$293__T4&t{xSzU4`5jboC~sub)Uum$JtwkRr$8t-U=un(g;X5A_5{^6A%!P
z5|Azl>5fTHM5K}KE<uoPn1s^Z-QC^Id*T0Cdp#@m`(c0e&_fTo@B6yWagOmD!Q{6m
zGb33gSjIjWq^8#j`h2Ay#v8Z6b)~3{%(Xk^5#i)ZDIHAzTq;i=*LH!1+smz)=Mmp`
z0I{`<v9R{d$$U-8^iq0W#z}PkPML^oC8tKk!SeH#JT-W0)-C@?ee&z++p~_lX-xbZ
ziE!VE0ar2$WGGL2JcqeqD08S-7yd--hW>Vg)oe;gY+AdQX1BcSbtfjd%a!mIZ1Zf?
zWXmG|&hsYEJrE=L^_MUnH^{CX{OSab7jH`GlxWlbe6-qvz^k?i(S(eqPm6yHLVA=V
zaI>JOwwCz*!{a1)Uv(SNsKxqLvaQ)2l~HZhUO7nJS%mCau*{i`a1~nCoo!h7+6xSC
zj^-wGmfE){z*jdsTpA+ZtZz3o-HzIgX;*x1V#?RfcS&w8(K~q(!l!C|*B+8+vmRF=
zvPHUm#V|J^vd3>gzvoeMGI@gXOTNzMdje$i_F!1BcCNavx7592sx8+-xd-z6NK$#*
zM!L*+Z&$@*t4gVDxGwwjOtHQauH7)CoVQq~7Ku(-G@`7b2K=0g%x-+T0R50-bF|vC
zlNlj<xbz4O|9noILBHQ*vMlFjm8(J0WY%Sii5SQ}rX<i_$d$RA`of9LnAXLej(7d>
zMD%^XQQya_gZnmH!Rvhy<%CGkcj~%ZoxKAlXBgSJFvW8Y(@2|0Jz)ghOqD|I-KTSa
zzAJy@p##*CInKzrl?gwZ?W!hbK`a^Wf`A<<O#XeLDNFc{0{ph<illi04BvUFhuVF6
zLGGs$Z4r*`k^(4Fi3E9*A^e9Dp5F*j`0_ljTB39E9#Td-6cP7>PSTGQtbXNZ>jO<m
z{?q-xMH)#|=Rojm`{ry;uIESQS@1P|qV?Al@*@|}G^~{S8{Y#(IM4e`_Lp*TmthOu
z-`6<6`R-*?9<Kj4op)f{Iy(cdrqgix6^?t*I*Wn+?Eg?0SKg%pE7Yx1##cx}{`9ZH
z$zNTp_b5qedxB|%DpdX{@pGa0dS<7G=lj-x&Spb^*F-4T(*K5z<suSMz<HmZfPh?Y
z$rnRsL<PMipy>-~!gj%yckObF_R$}|?;M|~F`3E_?u`UJN`I+2hw0*eMi9TzA3Q%H
zQ1d)&)SK{loa<ldv?wy(_t+e!AZ0X)FQ6=|IKb$SdL*a|!R&7yXj3_fRMgko&p@|(
zIhvygb;AqMa(Ba8SvK7A`N^VF-i<bp7?E+zu|Uno6tlQbAG1v~luPHXEz~Ur#1}jK
zgLvuAGVx#quI#C9QsdqFD@)e}Z-P9JyW7!9E1h53NwjOIn3<2C63$m@bqy&8CzIhQ
zDIuUch9!|x3|MD&00`Q(EPKlgdKy&&k?cD3E9oAyKs%E$Q$yX(a9Pr43g_#MZ;aCi
zVx&)=u(soI3EWu`voYOyUDqQYmr^?l4z|4wHrYhsrr(L7d1649_{K5d=3;<xa2TN4
ziW%jue-$4|3cs=YBMm>@Fza+vi7&<A>%!JPw=qsOrdsUxo%4vbda0jv^Jy>Gg)8ND
zKoGLZs#9g}sddt<bQ%J=D`VQE4;ksO_Zdg{LyLzZ5^DV<AdKb0!{cu6W<Q$E>O?eK
z(vEJT-JhD|VuP^y;<PYC$3%P^`fX=Tf+tPr_r+ox%iAb=!IO<%*P`vlU``c5=W>}8
zE+)9hunSV-%%|~3E1t+?5@F=l^HWd6No>BIC4r1NXoJ5yy6N_qthBNcCyTqj`?dqF
zxo_H;a4ImoxxenS$=%KI>;h-BOv38Yk2x#Qg*hR%RCKc#$nu0X3G@dele_i9ZnktZ
zK(extJ?C~fNAYHFDN@fp8%!mwvI3MQG()~AQzVj6)r>v&1keU;31*T3l(ff#2ArO?
zhbLn?YiynbqFKNuvQql_<!;@Xwj+M=EX$f2<@Pa%zB`Z~zS<B#c^&Hi=FuP~ANt1Z
z=SdF37^)p(#dz23t9P^L*d;}kK%@fgzT7p*?f2HhE11}g_}^tl`s|+o!95_*6Ubwu
zusc*5n~a!^IyAsZ!mLi412AEf_Z0WH4{}ApS(zis#l}qgZ{&)9meSbR_tNI|+K;#t
zdCO*oW`zWlzt=3g*ZT{1IF1L~I{e7V2h?Tz2yboCASj8Cn7^so;0-UfHTk^8_n7b(
z&Q!#(2Ss*`m&YA}g{E-JN!4NiP!Ckq!^^&uUD*NiJv;fR5VO5{h%JL}YcH)5qQ`MO
z@PAxx;;NX-H|iljez^Pzr}IbsF%cTlF@i8eoyLU;U>^uqDBs1nIoKt3wBh2mngpe6
zZixv6xtY+OH!l9xc}q?{5<{l58l23NQ^?A?&MtQd>J@pFUUIpQV6J8IH(Q?~^7Zp;
ztD&&>@gF?}s&mCe>pef^UX>)@BCnUf0S=kxn5{2|5COp`w-emGSTXM?2KGOIx`^Ur
z(WYReD6LC!r+|~md==9;JN%wvY{BpQ2Ix@sjVdMjtr{7X2EO-no~u5Jmh2L*Q0@dF
z=evk<ktQ3#Y`j0a{=ii5CvNwDd*5<X?bY6<gbJ~l4!)Q&Nl2>o*38Hbs)#!>noQ#(
z-*kvWB7$00pG{v~*IIwcJD9E9tDaE?6|iRGGd`qRp`k*H&pz+>=@STQFTWDxMnI=Y
zmIkP;jsRL1G8(j4n7d*FXy=z&S{0LejD=mGAiH+NqqnMM@6!_Lv|uTIcL{1~D}2`Q
zohBfN)G}y-#>Zi-10PaL?h>?%!!{O+kYLwy$Po_DM!wZ-neUwY_}0bRmtN$_3Ez|?
zq7aAWWXuNQ)sU~eE`|C*7nE2I1#a6c3=QLw;$O5>@6F1Qhfylp9ggM{-(>xsqBOVy
zCe89y+}-i^#LyldwE&-K-9alc(c~in&CpGP(S@@=^MAY)Ekk7|nBFxqQ*YOQ#fuSY
zVrHN!6grF*oU6kpb(UWyv#kds3_gzQc+N^UTVI*!NVt_GMWgjL+0i%E2niYmP6-Ln
z6Yn7z5gsaZUmewOh4v0A5eLjG<fUv+8~ex?*&O9OKmBrE@^uG-G+^31X&`=ThE+^U
z?CGPmJMR3hk?8)FbnMl2-<AME1*05R=ety%>;0zhR}>3pxbP97T)j#{XqIjz)$1py
z>_4g<!MDJAvkuE+zqC|V*cKdd+3jNY7@WDDY>K&n)G^nL&SMx0fwk478>87|p7)5=
z__!&2sV$mGOHT@)sZ5P6YqIgtk*#h34;wP9fbyBA(9jVJOBU?5&Sr*{$4l2)26SBk
z#c;YyBMOBbzB|)G#A!Ki)!0`$Ge@Ok<>dHh8m?jU?thF*;6?s4bgvE*)!*=sXn_Yx
zg2m+&li*9>+U+8LzwIAt6pO`Y+~s3kA)1$*4DQewivM?<Q}f%!lK*F%)2-0|4d>Lb
zthmnaR%8>Y_*d{4|Ciw5j7)@Wtcr^HCzxlIa49yh#^ESf;ZG}K>3u6Bc=!Z-o@s*_
zHy78qDvZDOn<ye3iKOj_lF9`&$IELO=1&hpbtwKwfFyVu`Bp};dho%<bVaGKZP@mN
z2@8B(h2UagljGNnEVaj(0qO(~&bk%-%?#;5#D4A`3FYdgQDMwPC8g)P$0+=vGBEAy
zQWbxLduh&(GMM?TAxomgb|QlRyTLqpHh9|IcZRPWOM8Nbg_s^%s|uB8$&NZNeI@8#
zP$U^V$&5sveeaPP5;BT^>3g)j23_aTyTK5MLyB*>zI;Db+!G9mq%L?|>tr{1gL-5{
z-T!knHO(vN3bK#sr6B%V7jD&Fce*hyx08nU_%g*;r;57F(shNCytP+^M1%J0;8DtL
zU-H#5&yV9a;h6(CpW-sHHnCvuIqT<;BpQTJx?b+3AUqqSH)7|mFPb;<N_*d6^N)Kz
z9u$3n#W$`bhJ<0xX4HG!_N-$s(XLmX%YuNwk0h><`E_CarD+x~$0~QW7IdN|Bv2$(
zlGd8kwD$V23zm&H3f;`f&XV8>H6`=l56NuZ(<d!PvH7;-Nl_p{0V&}Nf`)WWyNe?c
zT@1m168(4pvGHk@C+iuKrxVgc$;L1lP{alI;j~?rxSIFERM6*bxY@C~DV9BELuK#Y
zk`{k3xXZI0*@VGF0*_?K!&Q*0f=ImGo}~8VKfc_$@JKGk@Z;UiR1U>koApGlT6bt+
zmF4g5!$aYp7W_6g<bJkt<JNtg|3d%o#j~VZADu+ku(A$5g)bA2A8XxO)^XYsLq65{
ztPAZp=e>_8)x|BKbw6p?(h9EmZ8~aU-jJuEKgaJ8VRms;b$$P8Von`)YZlrl@|_>w
z9ns?V#+x=JSJm~~MkNU~?VUA+2=oVUyZepgtI&J5)mEcz%Ngk;5VieZ5RiE2f6rNm
z(&TMs0NlaM*l41jcNTP42lg1JPY+l=MZxEZf3t?lK(DbkIw+aH+6Pq^VT-5W3%4P2
z15?$we>UXkgZo8i!R4~>M>LNWVc(w+_eGF}6~ExX%zl6H-*8zi_~FsYu?8qL57Js5
zzef1&P{)$Lxa=heJ7*jPF0M~@r4JrazDu8;7)_sU-h6BV3s96fG?9%*<Ei%ZekuZ7
zTyi?j{jBedJJM>(f<)lp&&oJAU9NxBVfkQR^nUWR#;|#wC-70A+GooCs&eNA;sxM}
z4LkS^G(n;z(3g6GS@Xu?nMvMXj8?6ghDCYQjCB(_jOzh}-CGnF(w--AR#8-ayEW{g
z@ry~rnG4EaeNavO1Wj^ckN1k%Y&jU~ZSw89iR;D@fg+~^x8Az!8vR9s?;XnyH|ViP
zho@^o*57z+N9c*Zxi8*Sev5!6&Zpal7fi|9hiwt6WTrgv+%YkELxE0zwJ~wD2AOkv
zQ1Roy<T7T5kCr{nBLJSXsLQ{(S#Id+d1DhLe1Qfk3zcim{|LIA!Tajy%gwsa<Cb>z
zyzW*{5Lrl;QCa7%kH61cH_B)`22JU%X5Pr2x<l)#?dNIh$#3n>nFV%+y&_bf+VKqu
zwKJgcEcLVHuU=-921M*dvFO;qab=ifC91D%r_uRzC_}l&)v{7Embl~nsX@=2h4WDl
z*A2!i{_}ZN3g{D5N9#XUpx)thTt8%>haOaq+kH4IfFIt5tlGU2k|)7=%+Q$eh44s-
z@Xfmypmna)VQ+Tk*IS%-buK4C`UN!i!yY}dNgihqGBs2;%mBxjPC9npF)|{l0=Pgc
zmVzY}CV{wXdXM+v@?tz9@TZAL%pf5K95;Ad5obGF%S9c|YBHNqumD(%z0}yQs5)bI
zOTxFW#c%VX#X%K}!KQu(M&70XfkI8Re%JeP?j|Wm=~o4*%SWp-wroT75{_UujhQeD
z5w$Yw`Z~U4vAa@+VU?EpTRC7gS#Z4D7ZflX$)(rP*?YRK|7)h#WrYD4bduMjb#|FF
zDl7Hx_LOx#ope4sU0<09`HpV%K@F9?wvuO+OJJv8$DuO5e9!}<^j0DjRp(JL`F|Id
z?{ggi!9u~<#+UFoIeP-~$lnj=bs>FM8%>Bp|00boiw9~m@kBBX>#M-jogNK(yKgjJ
z4gzUDf`s<yQv|s~9G^3GD+-BEKYeM4*4tP7sH<TNx@6}61vOUe-4dhBE_v859*Y-a
zj_{m<V?i2@FZzpkwMJXjO@dw6pZs??F5kuaKdiBQ06y&d6HB;~rX(F3D98CzMC)60
z@nU~i3Lz3PYo{3Lwx3&MpbfWpOM8f_?SKMi&R&5s_}nb1Y~C)Uk%<xsd<cVG9Il)K
zUj8_!Q_|5D+#ZXk@_gqwaveqyY8=l2Xu`0od@Y-nXU@Q!hWqC$trq9LLPGT?!!YHx
zVD^yf1Npr!#PcD`yL|c(12hcB_s;7{zSoxYhiItm%hk(if#?e2FDjFr6GrSYXIiCi
zD9aH$dv~A}T_JKXFZ;{djbI^LXTtDNkM%}RWxE@ZsG~i;K71yPNJExDVKppch#kCc
z`e2jPWN5)SU;mVMbj#B1^l<{eW67S5)3A5NS;dUoT_AZw(Ui6<<f#N26me$mD++sM
zt$m<72SgI*!CDr21RtVMvZtsQzx5w4W%xS_@K57Pf|d$WJ-N!*6VMZ>Cwt2!!wuP-
zgpE0C35%=#c!X#3Au*%0F*2^OuD^oe5)i@5ldKz2a|XS`Fet39?-iIU(&oG8qRkks
zO}?0NJCP-6q!ygCo<12NWG?HJ8l^s!M9v6l;_&4nd~8?luP6vlO46wFic$bMo`DMc
z8%-GbTLg4*3E@v2c)^2<5==-Wg16`TbC~`--APMH-|8Gy@8l$GJElG9%G_^G#!d&s
z$-UFOPn#|X%Fv|KBuDwcy%#Yoc8sjChApI0&yjSOxKL#KUcR66HgvCkGPPb8rY-Ue
zO3Cy3Kma&u@+(bJ;yc<GQYPgDZ|8Mb_?u_yy9!24u9mTbxxbSg{P=!FY1Ib*`03de
z@CG(41_-52RJ2)H*qv#cogB8DE*06^F1J=$fckUR4+ZBOK#1QXn{}Fts`y;Wbcpx7
z4(w<i&|v(V<M2PiP<}$#*zK$cGUQx4U}v(x940+J^2qW5Cc!+m!68XFCg~x-Q*{RE
zb|YW!qZni>X>aEO`*HINf_hzduH%FAG!@IkfCM3(lXh>2MsQKqWm$lYR*LUGc(#xv
z=iGniv-x{B|M(9++h;vfEv_G!o{@5dm?-LaE=Z)QR0yaEwhYPK+X(MZn)m~B3l3wF
zY@9McJjn*d{ab9eldVDuBOAeOn)V53YrUZ8#OjK@^xMp63Ldso`K3J;>}K0bb=yGu
zqhv=Y!(rf%$tl>L_i{Z$5%bmOwW<`*GSqRd>?(%)Pv<$`7*Jo97ClYQQ~73oEe5Nz
zTk0ap-i7Eq5Pu;2l_9rmwxoQ<X|tjBDt#qz``R@EUTYAz=P}Rl7#pdBpXwz|f9?4M
z+G%%NpMg}Eoa6Rv+l2pB^7=^%Jj2^k$D`~6L_8bCapL0a=&m9RSKPpkF{+>pK=6L^
zt|gb=F!PATwIen-W=6-C*KI|G+|cf<=e>rBAQAbAVg=BEZ?gfhdG0gNZmEES**$jC
zSQupQX+WzdUNt0BqjDg=9VxQA*y&{@XiOp!#n?C?!%G&LBtho3|0gdq$hv-h0`6#K
z2H>4fG$tlSmcbNPe9yH71lzR{65(cf(Fp@ZY-6W7p$81>XEG9a)nM^>gDs9sV*SNo
zXL5u^A1nrvi{dS0*59T-fgj&vvEVOR#OZ`*J1VZw#Mx~=;=TcJ-1~Ay_mqsZ-_JZh
zsP!Lw*WHmukdfGS26i6v)}ZGQ9r@b@NBb1Wb^g+9V)ccWlu2e45|OD5KW-a4WgOwY
z%ap^IR-MIQrsS<Ev*6G9yY)H{)QG%&cbn^YH!RC082sE~4i<4|bWo2>X4#`b=h&RU
zXO!j6eS9)VP%!@-*zk0O#VGpCO@C5*8Y<3t2FMG72<q+-lh>{83Gty#u&Y%uG|Bz)
zRs}Zg5V9=m1{UGIO50-p$*%rSc-JDu{g?$Hh4(8cPyOF)F!l3?D40;E@pRv70Cn?^
z_<|wW!svVK!riP2csgJ!5E-Jru9U)=C#O?UM8<j#p~_dhAT+0HYbPgX9w88K%u#Ue
z_rAu|^G~ba#4fyCHxQu@)8;6{m<XJC<6nYx_?n05?UxsVoJ>UI%=Y8eMK;111C3f^
zQn6BK(Xv9XF}8lm1<q&J0-}}8@H<R-Zx}Zh*KvAc^UwP0=Mk8c+DDDS#(5VK+1z?q
z*KEaqkmd7MmD@IO>I#TYz?eni6&=|rojO3|IB7!vH$4XYTw$G+B+vg8KXEH19TXG8
z7NB{#?rA8dCa%^Sv83QM-+^dG2E`!ZE_yYzCR$KeIWxt~zljNBW?Svs7pH&bpr>o@
zRHjgIo(w-NQN-YFNvKaaOwt0(ZidQGA-l6J$B8yU8b>z?9=!eqRk~oKx7t^m9IlIA
zZD{-K@bt5ke5ons+1sik*O`l$^2lhR2uXDYbJzm&E0X6e1M(9FB1rK-EquOA><_r7
z@{ct*5oA<#4v0--@-+>O=5FBV)gKwAv|V>TYvTB^;L^(u7xhPz5`%J(`H*f;Zp0je
z?pRjHZmJhX3A0vs;n{srD5$zB&AZ@y=1F&C{KMK7&<~Gkl<cuA$`E^4+eq|5rSC~W
zs1i6zcV55B_W;BHRZ`OjA8{ssAzU)_DE#%u@STc;9Wc;ngF;qgS1|6lrz~Yq^i!=D
zeU1Rf*)$6*MdH3fz#(whR<ODdMcggw^h{n7g`ET!I<D(`!DFEJa}4KQ@YALBqE3px
zKGGlX`&!@<>X5yJ_lLI{h2z)Gb_)6q6_WNPT4(QXwRP%TqUtUizIqfgW|dTe6)y`b
z2)HIFO-<)kGbZFYh#Oi;a5^H*6P1SxDvnj1A)6^Po$a0^l$WpW)|qfPN-urQnVY)q
z$Y63U=*fe6(V5U<aNpGrW6P4ydU)n{a5ruch#zZh1fw!BGHjz{o9?c`N}*Fz%1O|Y
zR;H=(!3(?XwH#dY!FD-|d{=j)h{+S;AMM3m1V(o}HEQLi%5d%Q?kblRTi=;Z^Bj`;
z@_gt`>#WaQ04<t(!vyD)`c%?s$zz$yk`=VIiM*8hO8+?nkG-Bbp_7ENeVc}=vpA_~
zqEMIb3F^KAn1@^Gx3wmM6IFe)4AxqZ_M4%fVrFJ_O$DQ67W5kYj?pq#3uXyBn@3mm
z6gfkw#Y#GIggO<*$%PLeTk!6gOl_A{GC6L=Lsp3sSfK`^{FE5O<9b^b+x>G|8(sMh
z%O`hQXIE}N#`4s4DQ}}rQ0P(E%eO`=Fz0T&lgZA*r+6&CC)sT<RJ6hPTmeNk72Iv3
zM+*BMw>aYnYP2^|o&KBg{yz@RT(7@3PQGRO8Z&?RIR$e1)4uBqea*kfx;0P5b-AY%
z9J}cuNy_QSAJ46IBfvv?h>h7MRIbIP#HL8^vC>#jlPk=B!On7go~8)U#B%(c{-%pk
z<YB=8Kg7cbDXqVb6tQb9Y)*d^$t{fa^O5O6P?MxtaoORBo+LFX6u+a<_|a1+3f+Gq
z>qONlWdY1+?GIV2o>35Jd7Ys+UKT5gNFexX`;#tz)gM!z`LLg=`GsIT$_a<ggc-M%
zs!n~`IJ*(#!hCA0#)=%+$M>4xxk9NnThO`8d`sYssssq(3kWYNCL8io!cQbaAFxF+
z{TF>TJK)d4jkx0jlPF(i2)*>w{EAl*D7|{gXE`!ZaN8qTd%4ykI3<Qj<If!j3z}f?
zA=Sr1d4u(thL*NJvG&etdtD&p)C6*E89T$bto@^VOnc}|U8+pN>4vm$)-vULkL-(f
z0@bH2tu+0^U~T==x5%z%!!dCc=#NDr=*t8&H0#s0tNEj#6|2EaCJcpL4fLj;r;D2(
zonZEILrgoK%z8X~bri!~6c?PSq+Fu5Ii^0qcDv}2|7@nA_CAAeu?<n|5qR+SzI_M`
zQRZ<`OOAv18Z?#5<Zlf8z7umXn^}gDy-k0w+=dX@LaUnyR&=+HD`4H7uTcr+DyiE%
z6iFgq5?j9w-OsvwiTkeAHcQJQBG3TcO1w;VBqCk0WTRx~R&FB3gd!1A8VVXa<ajjZ
zDvD=~#UT^W-;z8A_V=YEbuP4y2VWDruqA(lEza_dJh8#502){Zz~?r>yA!zjlARV&
zt5wdG|Lz6sD{;bGSm~4F-yiOV5?CyJb<Qc~&7Y(*?{mjEY*N}yyEf}nNgfLyRh{Mx
zlc;y>03%o-qs~if9xF{pX;03lBxl30l-XIM$@Vl(vu^r|E>sWO=I1w(@lSnKk1nV^
zD#S|ZXr67|<turh@tvpp4%vI7pKEa~UoMBmdSBqT6wFD+)sK+sg4gS2q+*s?@@;z=
z8>Sz%2zf0f`Nc}ml(xM`xz(x&*`)~dDJEpcR%&~6=1Q6n(mhLTuV{!WID6s;Y4z?-
zfDCApS;vujwCs!)O^j+2+#>#2^(JO1X%XyuIugu3G%Z_m`LIx94Vp`0e}D?z-|#4D
zX1>Z=&H3UG{OM=_*Kxe6&-Z<lDc{aW)9IF-aQ#(alh0z2Vu~{bIYxaGc6g1t?+j+U
zibqH7#OYv9OqSnU$nP86MG7rlrt0MS%e+X9&$RtCyyfK`xLMElx@dk-_9ud-Q)n}`
z_=_kj%8|oL`^f*Xv;=;+4^de6cV->m8w1CK9opLm%U-??TUTX%>X6t4T|rFdc@zq&
z>3(`hnSU9Tfd00h=?6LfM;EMFA@^pI<X=Ud<(;0g9P3M%A;K31L#h`_5v=a_vtfrA
zNxIW$;mJ!`9>QM`RzLG*Hlyvb^K^a9mjg8KBWUd2#=qd%V3U9NfcwkK&@y9nw@0b?
z1#*X~KUZ_>uMhsbLti9BBOI}L$uj<t+<VcOgUhX5<TkXsk#Inr5b)QFP4YMx$%r{3
z6Yzz7+gqna!-3ggS)b(c=vjtxUBUB%<;0e6v_EpOjLWp4X}VlQ&>vs|>Ye}T4UEc0
z@wTRa1s;9;a3gKE1F<Wx6iZM%%Y8*ZRHmz_KZf10x5qb!&nyKKQL6bm34AR?ZRZrk
zR5qXPnhKdq>Ue1#rwIAYi|Q^{7=PA9MhXpCz%8MF!U!%&ma;<1J@`*TPM#lekz5QF
z$%<H20jY)4O!(Ay?t(h7U?LWDg%UP-#VT6vq&TFH9oMyD`%b!QsNov#LfUlIx%^FR
zpctLJ19~q5;aA@W=k8YBPyeiAvQHZ@Az}Mbp`2OurwYSV!b)PX=B@>H%7)xt+KSvV
z`u(JQg9Mx0_`z-Wf*CdA1yVdP3#(_#CS9|?)x&Pkx!<f8vIJ#fhpRruBunIdnahq8
z|9Y*9rNKPZ>5Vt3UfB_51-SG&eH-#6hZ)a-#&W{|gf^^nig%LZD+TO(i~+8DAG!E*
zQ58@NXRODP7I!l<YpW+W|GDU5klL7&|5|DHY9G^aGl-xI>WL}avYg3t&nB&f?QTCm
zU4T!7q;wIe`akm*rSfl^uKsZ;s6&Po^~}@oU|X`amyP$3)vf@GYqtJ^tAcragiZw%
zJ{VH;38gbgC)cf#Xt2{KscPaO^ynJte730B-WVeA6>5@Y?U}MrnJ?3BSbCqpYWXo<
zAVTP<hgfdHQheox`Mn*~t2m{0LxR~cE5|k2vAldzw~}3};?KC;Tm@!Il|TvuP5-M{
zjuv2SkLVK*kX}RsCO~XnF|)3CAw=QCGrj)}yWuCK^!DDN;|4bjfGbx?En^Av`$jy-
zegOF@%GtWUfMmVki=O<zrS*6L<3%znDBr`4>7gS%X>#I{zF>^j%#|ZyEo;t!xHFp%
zgVby4$TVIEMl#aTu%_17S(L`fzF0Qv?P1!=+enQgDvUWCG}QNu@%+xQsrXV|qS#D6
zuqy6L-KQ`Fbe)w^{uP!7`!Sfe%i*&1>)6*-A9Wm*LK4R*iuxBBN%QsYJp(5$BhBx~
z`M4r9aaMi@|E%9M=^o2&y6hBpzdVol6%3y|u7NJ&{@w1sR{}LyrjTO+Fl*in;PS5O
z3Xt5sKnyl>0tTtf&T0UD&E@r^qv_iZJ+8kBhr2CLqcb+4&N$BAC>L@jNXEJIt}4Qh
z%J~8kNfFRRiIxRfFp3s%ebxd$$KXJPbL0gsGS+L13d;9pDz3xBK@H%)MpZe)hx3W9
zy_~_QosrO<sfqHN0;!8<qMRK;jJr<yIL5IWRfy7@3Lowe<%+f8w2pbdJqAj|pttU8
z1LY;4<Yl|5uNaY+eE!It<CYP=@?*JfKfv}jI}<71ch<SxCwRI{&{w0}^fNFOxjjky
zmkOw`GzjI^^UZ*U<|t-QqOsI|_|&OM-7j?O8I<Px)2jX=>pN+JTYK*wP^`n}GS}{|
z05{p7D@G(<E(U8(TJM7i3j0C<ClF>TLj-TR^Y;OK-VD4uPVvE7dW{(*9``Hxu4tyZ
zzRoDdK$+g*8!BPlGfoRCwa4J*P;^9g)Gx4DK2_)HsPix#pKt3U-M~MRq?R`!qVQw2
zC8(_HUQdj9$D&wHvj(Sy?hGc+8;;xknK6sq?c<r(3au}PC24SA<<%hUIjV*6CRi`r
z9<u1>kk8H6|H61SQ=W5Wi_4;?^T()5OjBM9^s3!Jf6jH7&h1#*v|E;g<OHv?w@u5C
zgwZ?5Wd*7Mzg2y4k!Q=d&|@e5HY&<sSE#A@ZT9t5$FPmaa2yA$OorsZzbA>mmy~q4
zcQ1LccW1V-um8Jbs*SnUt?G%C4zEW+Px84JQzAzXY0hxxUZmnLOGL&Xp(U@eHMKfh
zZ=I<=#Wsy3ADO(*zN9z3oM5vgeb`wdc4RR;)%Rn6-Nkrsc=ye+$H}PRL>{WM2X!it
z(1Ngo|HLm_=gY$jcBYSLeoa{iX8yUk>SVfG3--RG<ghIIEG0fa!NF0`AjY=kGG2HC
z+P$*-Zn!=x8trD<a5{E|5UHze3n5~kw-Q@Pue`EEKIGs#imC+WPG@S8qkkhj{l^5Y
z^w1L|2*BFd3RH~9gz17>2$%Z4EVL(<AQ|rtc&0ar?&<LJ*Y5f7W>PG<2c(UvwGSYo
zLM)_AF?w2DVp(Y+h3AOJ0CvUN$v}p1f#rrZ(nzjzoC`cgy^-o@?knNRtj-?+qNJtm
zw+gR1Qp2*Zlymt2W5Xt6FcHe>Yh>Qm0DVm=`9YjVg_wh)kx~&+kVKk!YeWDzl*p)m
zK&$LNftNwx)oQAg;&%K)g`Y_->^?x1vg$9$Y55hzvByE{Cl@hkI&WFL*JUTQz)2Iq
zLzk9_l*><<+#}7tkn|JaItQj6C7{yE+uZFJt)vGszZduro;-S!TB2cD-9kQwy=j+L
z2(ngjLb%|V1nt1^wPaV@h0(|>5?O1F^_2ddZy%F{MUW?^sAu1Q=Njg5zKfV)ec$Fp
z5NuOp-v0x}75_R*?hFDQCTlp%y^va+^1kcgYeo7jiOSxrJ$C0Zl{LX>zk^$ObEj4K
z!(*#_(r{!u9yXTc2D!Wyfv<f(tS%%5O^9O?j%pb$9r~se%)+0-h0HzYaXJH(7`OA_
zy~nvFKr@-ijJmn1ypxS0WVcUsovBSC@=c2Vt}!_(Q_=gr=*dmMq#TapF=PHudBZuG
z=N5JZJxne_JWjUuiuZ^ELlop~EX09~SKoY{NgQaLm&3%(2hwYihX;Q>qoj^7rgGJ_
zu#qGrp7hKT_IeCm^rz1%Qme7gK_}sUZSf>K*(Dd?0{geJq~r9qvGAW@(#Y|wL`L>4
zcSP*`h|zvu9qA*4yGw+`;B%Eljd?DT>9{wWLtXvxHLk}k7VqhJ?UD$^pCX_1*}HC1
zODDNMN;nL=b!cWn_=anB(ih7Sa@)WR)cMhfv=ZQP8srwa*~p|}`+orM?Srij@Zp+<
zs<wuZryPPYAW2x24mQI(I|_(o!|k~3??TA5=-|8?_a}7bCl(vutEx5#vidJh|Bt~b
zYuzQ!x?<Go9;4?U;A><Z>o<^_%zb|~X~yvEqa>fdC{Cd!=yiWPd%yf|cW9yQm$q9T
zm1miYj<!9ExHIhG-Cb~yT9qxv+`Dp*7qGKFPf0E}-IAR7yBJY<)nm^vWJBj^cn2Yv
zthgeH&Rc*LV%ouTg08j!bYWt|Tgza!_Kdv3Uii{A{BY9qUBoBkxP?N20^Qu1jZj?k
zYl1x@q?iwlq{f*>G4Wh`)sv&VRGKfql)T`9FLwp2QW3G-aBN!I9nV^BLmp}iF5=p)
zefjj@Mn}t5(bOpMGnz*QeIc(v4}FAC6RWJgV8<0-%iV6nMh>2h$#u-$#8t>OJ(D>1
zc!`56-|3y@$xA<>5N()(aqlDx8bS|Xidf$FxR$Q61iZm94*05Y?El}LyZ89kYEK~>
z6!!Z(n8SL1T03qEw@o_w^Ea6iw+}rH!_mHg_PjTMV4YAMh@aNy6|j#9Ypv1bYLQC~
z*-v`~Eq6l^Lbpv*ls);ZsCB_J7FoFZQ`LMp^k<>!_ne27o=!Gok%;7%B@Fn23#o*y
z=Vo~%Wq!6fg_l8Uf(N><j$Ri3H1@}>5~9m^<cl`c%%Nu5FRm(jh?`{prM*ck?~A^4
zqFzy&h`l0UL8_J?$jjL4%2?7-34I=N>-eE+9<-8ix%?EBg^15#la3LYcacK{G*^8;
zpbR-a|5b0Aw#4vhD;M@HqKP79-}(ua4EPuhne)G1NjL(QYRyegx%ehJB9NU`qV4~-
z95<%pF+&bl8upD(SdWe}nUD&p>xNpSob7a5%~XDAxZN<jvT1dyY?4Mdn=A1W{F+n8
zhp}EZ<zZ$vc0k>A^TB%V_%kwbFy32=cI(7GRmV()`%KcgW!n^tZ+p+;I->gOY0V&(
z@f%4JYP#Se`WbBbAz1Qi2s7{1erf(lG}xh=O;<|y!Ot``k`KNcSiSP{;N<jNbrN^k
z+-aA?fqLrRTv;|0TFjZc78;Jl)s%0lJDkZ_jP})ubbb{QNw#xMHoMvzPb!F^)WIwc
z@i-r+>n5@4A?B*IsGJ?$`D*u#{`E6xceL%t)#xR63SkMS7kxIO#NnVHh_D$6Rh$G=
z6?fMI*TFExt57To!fg-*&GC&9n@G!ih00!P)K@|7aV<Z@KHF|oAcI5%tAK%xOY3BA
zwbgc|#nkBi0G&|i(e85-XOzrp>oZvEY{$2eS&OVyYS?oKKkI)h%X2>Og-(TmkoI-1
zeBnR|OnKk<rb5ZT_WhL4F#nZb@jgs4mJ7b<N71r(@tn<YJ~<qu6uYWumg8RQeicRm
z7}W0`eGjTLK_hMQM+9t|NNWP(<4-7~!w(zsD`zKx2u&Yu{Vh@gqKk1!>6!K7j5)|^
zx~jOCTJFVgnzp=$)n<7%$+Xq3J|}93-y%orKrGM^i%1=(2jE^j-{^wHmL5_!@`H-6
z0d$=9T7AlYRB$|mkF4E0yyiym?P4oWY@4+>d*k!FppS+KJuJHdD}0ETlkq`kMlzGo
z;yT&t*+P%|MNqHD<poUm&fFBCXQOfj+b@e32bCsb@x3#SaaM>8QZ&5a<!B`6gky|h
zDsC%W*T^KfI;;MT9T3h}D*Vy;Isj79dsEOJsqV0)p(9@AiQ*NS!V7Fzh=0~P8tCS_
zI%Qqwb7FvGhjE;%hFfpPH=&6gKXjQ807eQ2kE^M<!BvIfhs`r=ef*{E4f=ucTnRd0
zNk;rqp5TAo+WOvWXtL1Kl+An3ezhP$-z{YXMbJe?A}jxcn}c9X0#4>*&3$wO>S$J%
zl{DHICsZt3KcgV@Fge>u?UZR0$g#hod+XxMPiqR=7wP(tbIRd1aoBQr$mKX*My>41
zRs^!(;V%&R$6gT=Cm+xpKI;s>7Fie!p!><MUS?}1uNzp1FT?NtARHsz4?l=hgD^zv
zHtvKYOUy4qkVzj2Zkx;OvbpH{jSE>oe<DW?V}u$xW;3_&($v4NjP)H0>Xun@ZXo8V
z`tBqh*#Z*Dg}1LUfH+o=+I6$$U_M%DDaN~xfdDN@Hdkg6V5pR6z)Sx;Hi4uPyz*ZX
zj<cd{4j5hTCTgmDpLus!S@Cg*rT%e*D^G#C?Q0$fQAObc7LS7xfs=)Zi*E4bhHj&X
z_pdvCx@?;6row^&r;su+1agr-F72uo#Kr0P_;65QZFURw&BSV-jk1csfyz}^VPP2)
zF-^mS$F9KIc=$7^&LAN~*0mJ(!r}^<DZ_-f@gQZ<jZ6RbP?SjM(bI8ZpCoA-pj%~)
z<TP#g@1py;Vm*@2e22~%-GvL)UO=g7ga!LIf0@&AoQaA_^{Wg07WN$B&)Sfjpp0Um
zP()TY;p-TXRqFw?z3it_wpBn{Bb-~D4Gq-?-};MS%@^pNJ}%p#h-QKbACBtIX4%?V
zCX;@olB%7~IO<GURL+$AkQ}PJyS+-zjL5S0ijad$;F`ZViPMM5Gd}U8V`HAG>pI?;
zsw@PoA(|g#Y=%K~C!-oU2R^)5^#D+t$4-EvL=O7$;)5T4dx64^Bk>N)!O2Rg)KDg8
zCnr1kjW%&;F%4GY?;`f}`;56F5#Es=pZU}Tbdn=PDB7rF>*lbF_iAzJ%=~g#(z)~H
zv=o>YPNL$3N}Wz=uGSj}R^eS7Tbw0i1cI9<q2@l*)^3oL55)|qK<>9+oZFL=K@JyR
zwPa>kdd#~bnW}3<0Ci#r2DS|D-F5hx4zt1&sIzC@*H>|ZZSS5s^XibaX;ZqAxaW@K
zMexg>-iSXQKW90DL485h%?vXSIlqfkJwG2gEPUto*YX@f;HtD^n|@LPn7d`fQsr+n
zt)xU!H->W}=<R&Xmg_zU?*YQ|`2W(^MTHQB`j7i}T-Ihx&+#BgNA&qQLP9l~F!J1k
zThDDQxguO#ZOhn2?o<`BgJ;GA)FlHk$@nl&^I?=N{V{%a!4XF~LHZV;G@1B|`$dtv
zkBE!A>qU@D(^#zgXN!dh+GwhWY^@SA${ra8{e;<mZ1j*+P#-`jQ}ygnbsy0sxf~j6
zj1^yRBc$}vM8JHouLxUtV2=v<|KW!!vR_KXrqsI?gVXykDp{GMogO|af^mFuPactL
zrcV4q1GnFuo&fIO!(kB)=Y#nOD=j<e7BD*gn56+I4<!z;kW5}!nA@qG(_R-@E2$N}
zvS`prYmP8KsRw~c6<>I(!BJP1#v0i%2JwyOm0XX<7XL)|O$_8FbZ7^dWiLsk4qrLf
zsi7H8tave(I_y4Fd4aygUUtW~AU9gz4F9lmMM8??{5f)Jgf1bmYR@0KJ=>v$I9}&N
zDjmRxTFKUX-`mwqtr&ZR-lvtax1K!sslj@pvHK15jOR?XsRmu38YNs_eq|w$?11m$
zm1fOvn8%$@2z`6^#5n1~@mNH`n_!72-bu#0ZYLNGQ@ts&RgKU|Lwc**J}E?ge+j<+
zAV<EIYxn&*g|vGw2ZzP@aQ(#wG5o?iLE#da*{R=j>}Qf(2x4J^A|_<S#a^Jfkos9!
zJz|(|M(O=_E}_&|HmmyWci?_7`lT&!&7o7aArh-u>r_k++wYQGvHlMhKxhMZ2_=B#
zl1Y%<`zof2>4xY@0>24qf>jqWRVlD>_+fVfdA!?G>*IZzFhNl82F3q1vOAP$!N^e}
zH58Mtrgpy(P0cP5{9Uk^=zDG<2(o8~mPNZxgNeS7n8(&&a^UTDG$Lj6W!eW^V4MGG
zZOFv;kaGL!Q72>Jw}PBoGS}?|9n838dY*j?JE(}hQM~km#w9RA9rE?O&NgpM1TN)|
z^LwR$B+*TT<z0o0auz%T8%NFhi9^<=q*|+;y`0Qrmn8tUg0~RAR~a)_K!M=X?^5jW
zftH8K>E{0h<mgH~vz)6xVy<wHsZH9Y3nteG>YN>1l&Q4lglHVf2Q6P^()fWRoF2!y
zl096N3%%b(yL_hW?|y#f?S691p=M=M$QaSKe_*HAe$r-m5@xH2W6ZnjTtsT!*#dh>
zVsX4rTgQ>FSH2>as#a?HL%?>zza<{67mWAWx6pU>`q@w3PoDX-<w5JN_qxyyV{Uyh
zbt*$r>cvICe6Le?82jCcO%OYKXx3Nt3`TGfOM7HEHKdXf70|a0hIBfrjU{mH8)R$w
zUwCR**gtFTVKjhpBKwzea_}M4*Hse$P4M??Q!wKPL=3~)bw~oNUwN#AeOEmUjR<tR
z#Ns<v(y8whXZogk@e|@=Ph>`Vp36dhm;=#7`KBt$<o++1Y|?eVHxFS~aLDRr4TEsD
z^N+}uozT>>bP{#x)#eTs^2_muk4RaA)QVcmY7DZUNgNlW(Ojaq6iBC|a`vF=jKs)t
zy*o;cA}`j3&CuA#hisoyxiq%@FEH8ModRf>(9(w&qE73bTov~Nq#NcZXXEpfyKDNn
zrq;I#(K%{Bd&9Zc-fC}7r$D;D^4WP&vGgB|Z~^GS(x0E}Gxtgte$`SqGtcQ@I`X6B
zwK>ORn_4+eU89JdX>1`4t_c8wEU-c&Db_!2<k?ltV!b6yHD?_h$<rR<wH)b~<TTSq
zi(xL?%ZBYOeP-E<zIfvpjJZfizxvdZJ|Xh)RT}kNN3?Ck1_iT&<<JB^eCG2z-ovM7
z=K50Tj>9TzoLbjmY9rH#w;M-UQ^P6QL`YekRDb{>>>nNeU0NYsl=&yX-rdF~M>A<$
zF2Tloyr`Ydp)3YyWN%h$gdXS`VKbx9kendN|Ey8~JRLb{i{YPP2XIi+DM{=dRMUd5
zac*bA^Tt0ul=uBh67-gVkDvb7D54QCvj@+8eV`cop#jh%VDyAg+ONyP{LM*Up~(!d
z)>=!Bcy~wh69i2I?#ScX9K4SnSh@XKTM7|8JtI8C;|-3`TO+D-T!>|_e|QHzt<fDA
z9)I`R3#1IN2Ic(8(XH`_8Oxv!LHc(Xcf)kR?yQqObGmD3G39W4skG~Px5dK00n0it
zk{pS;&$#Hsp|TIX!zc%%a{LLM3Lb(?<!nPKDJK79p>8#wp|N$ezE27BYY{bVr1&34
zYm<r|g47x63r*|@=ms5uoYHzv*pi`JN4v4_+@E5rb57sE#?4MU*v_&N-y-z*dhHbz
zr!3FyyrStZ>cW8U;L(6{AM1^zJK%j1rWpSZjLiLvDkgldR&NWj4QRPtVQY(?KM*Fb
z9nf3CbXFH_j_LP>knaz?->YG;M5<oE@cWj*$}_;o<^GN2*hERk`MILhk(iv$_2Z4C
zmr)|_BZ}@LVaf5Dox*{b)f;As*RO{dMz{L^4@LJ{ik5~s-?(vkFf%T;bvc`y8DERr
z+0#qA{uPb@AKbdBiMNwQ=lj8!%#Y7O{cSOfiv0_K^j#l7%i^+%GpQt3AGGnyVWv#|
zaEPSQ`6vDsO|rxLHo`gp-gtt1sy>^w#z9S%0$05wCyCk#%h>O5UU+r5ud(hp=UzF7
zoyAYz%az4acdW1ls_E+AY=3EFe!%}=3iw_`T~Nx4x2(2Z?kAQ&milq>{1$og%4yp!
zI_G^&BAm8F@6zuxV(+Pdn>3O}M=Xc|{j~aBYL4HzN;A(If@c=hBK{f)_RpUv|2TVg
zF)Sn24dA=w4RR$eCof%hF6c|@*K!y9Kpwnr?pV_KE?>p`wl<*7E^bs|5bQ6{`+;?B
zht&8Kax1;%W*x3s;&y;Q?6#6aTeAW_oN<Eg;Q}5@-Yv<B5~*sH(C_Fq=ijK-xOecy
zQ*A`$44O6iCPPv%nH07ctJ%zx_|g>=Kx*h3!n0eo9Gsx8VWSRpnIfdw0G8HXE&uxe
zEq+14hrshLbz{qrXF!X@3jx#9=oXl`fA*2jUmw&@YAq6c!n`50qrCJY1^tS)oiWOg
zV<mB3(;P^xFQ5Zt>9y!nAiw43Eze~UbXz8_I4_lLkxx<Gh(m+wwR?Y8jyVocv269b
z4$oL_IFu)z449_3QImA?>gZ}#wf}!(;|~4b*tnZDzbp#3-U+Z8Exat8PXUN*;jr?i
z%DoWY%@<4DX!XM6Pv#nZMUPe!tb8_Y_qM>eLfOCn{f^kGb^t)2_=sPB`u*-gsASqo
z@6~^(zW4uSag?clZ_>OHz@F^uFbQ#voUctWnnJc}Q2*4JA-lc?&03TrC@qBC{LYdl
z(n+;adVTL>>;^wu3j~>~8O$Otm}^{vN(*iLx56dsv1s30YiZ;6Cms$RGNZLKP#C}N
z`D<6Jei;Uh@rnQ39#<V;rd3PafMf1b`|Rh4Wq<qFbB<%@t3_{5vD~<gkKjjrA49k~
zO1bOS)DCM4wc`F-QcDZm{=^Hhg8w}(2M)z2_lKg7)h`(v(E0S|8J3ziB=JEz^&(oT
z$oykJISNw9I^j^ZHyc)Pu|9P&qvHNE@H<YP6(+_V#QPrS3jew+TmC@@i=!efui97P
zx9JwKOH+~=lgUS{vksF}O{w)Wr4@HRAR_qHBdwe_f6}}BX)&HX-~QD{G`pFP|A&ej
zk-`6^Oe0W88BGF9x~$+eOZ9`uXsM&Y+Rqtcks|K-{;DG?vYUQ1B7mwq81sTl{~}e9
z|3WlW-d$Y_1Yj>6rbi~h&2brUZphccT>vMw@pk1;H~H;8GT;0c#KY=(5ZgK}^tfS6
zsjtQRhcGhVUW~PUT53tkas#*`|F-o$#QJOb?r6%SsVjgxl%6I*_tTEnddkn{T&jPw
zH*C(#kvepy5E^FG-*&f#+!P4S<$jU?BU2ps#_}Wd^lQi!EA74*IE*BPO4MiN$a@4D
zLI(xOq@!FfH%raIZa&9Go^PRmsqcEpvEqD9?c%M-TS;bI<KXIR!O=1qalyNe3`k|=
zovTHCIi47HL1kWuwfa(Ufa`JzE?QFS+1#3-Z9#wPQ(xd(F#)TFlie9UixqOWeNRDU
zR+;DjTf3lWYWUww)s9eU<DUZGwEYKFgue>$-=dN{tim|V_Lie4@!xVGhOC?T`-*=l
zT<oVj6bZrn%5<#g?v_)?-<iX^$UR?J+v$yI6Xc&E*5C1t@eeb({qeyMpUVIdaRK!n
zN#`hJos*_ZS*<sek|y4$Z1SkMAGI%0a0y-`JR`q6bkN#s;&!8aA;?i>(<w>(b;XB)
z5}AOt>=1M^bc=)aBW!#?QHM1sr$x-&zg3s?(r@%VIcar&hl>8@khp;n$*57@rCV-3
zk%Ok-z)`#vSpqzkUD6+kh*G``+5lPt1#b^R04RRw<L~u>jQq?t@62$#jEuhwIK`(<
z>1KbiKmVASl%D>BVHlh_hmuy5YMqXc(W`9*8#<qOgzz2O&d#BCYQtr<wj8=Uh$h?i
z8bIZVtkm~kdi3D){29PN7Lek>ENsfLMd!$WVKD!>!uoRD*1h8!644WHcw6`NfrjhO
zX#*Y{cISa!5$_NVkEq{*MqhO7yvBXM<AR>QcM?+5&>Z0ru=d8&a4w>^bmvU4Q7+|f
zFGOdcPq}cu%d#b+gsy+sUeM_BZc9=SHCU9$N&av>AH65{6$(25u@!?JV;7OnPs*6I
zYsYWT8t$GMvq*`UKvOrs9Yzawv0^fGpQzVzKN&Hp;r<T`*S|gFU^piAB%$e-hQ@A%
z*e0KpQa>hV?$2KF;$=QZE?YNQc>LRD9!DHx!&8=h@tiN_@g>ADkmf!tdy#1aWa_XA
zle!IVrd@8CX7h(Y5**vWt<=ksfkhVb?;U+&?yH$D!{UTeMG;=0uTJ1{c$izm-7Vl1
zLP`5WH4+Xrb$purAM~PVf5^TDkR?6P(>)$QeW1?7-OZq*?)FiTt%1$?k2s6a#rtgH
z^y}dmV@SN$aV~KMxDgzBrxQ>>^RX44a$sHfp2Sh>>qU+^<k!Q;Jq@j=Js)$wr}HI+
z#wK~(UW|em`NyyKpM329+JL~x3xM~EoS|HngTolSulD{NoU>+F40ENdNzLY8rH$Hd
zaWdR=^+8O{#t!MV{<*CI7MeG%@u|R(ve|e^U7OkEPe+SosmjPLG_W@q%0JsPZ$muN
zUel@NZhu!7k#gMssVf>f&Z9Cx?07p8%@m))*@k-uYtR*6?Ta<&o3qx9zwtvWD9^9h
zi62~f<C*2sqaOV8&3LXwuQ;Zm9o_en%=GeKDnC{WG3w#R{1Zf+Q<FXr`;6gwCNl(d
zNx6m{u{PoNu;qh+2*-)o5Z}y^Zj#7HKq@jS2nN75l|AW1m4AEQ|MhwM5O(k4>0!Wt
z`|qlkkcote&m&p;_kD>IjDFsPDpWKyMhFv{wPAOzMQHgn=IeY<Gltq=tv7LfaGzWz
z-{?^07+`pJ;*WOZ3%xACmC_n%qtm(2UDAV`<1~3=klU9H4)|Qm^k4bp>-FbAF5bVw
zV?vWB16v1^fn{-)1boU=<KhEU%;MSyqU;Z`M2lT0hwXXm`ip!|9z87{T%zt*N=ZEL
zx1~LMCAazy%lKh7=p&COcla21jgr3|)IpzQ+SwbMh{EwPMmgPC5m~1I3IJy(_kc5U
zAEiiAW7Cn``|W*glGCDpYh~kK_yA!3l#SY2RZf~R!k5FP%tHiD+f}w&2z1Iol7i!(
z69LZ=ARE5@7i7aroO=Q)`1KpDt=T1ssS>rokLlH=M&oizZQ}u-rGQt^r()-nPtEI{
zv*=-m;P=vn6pz@$!rUZqzmG|lLW(y9pqr@~mlZhe6ggD+{aKR+X6B{+Xug((q7P=V
z4s1bAx75P*dUSFxf`9Y#y*+JnW!vNExaN>{uTPdzq+o7emU4>BKA?P#Wsf_#rIdd1
znqh7#_j5-^;%L6wXIuzubd5341L)gWZzzI>e`;|>!Df&BIh}`%IIT93%%)57IjkDI
z%4cCJX9QP<r`sHB<LNwX3ib9-zLJ!kK}IWrtcxM~iSfSKig;!tAamOyMdS}&+r<er
ziNcS$drx_HgN76DeWZf0gXEq#?$%LmDw)!XB!E@f$#d3ol`~eW*f(-L%pFp^weR#2
z!o`XDd2keYQ1NLLPopk;Ca(v-JJw*%9fVMi<bYV9$uX~IHe4lanvYI2>X=CnIkX`s
zslIaz#D66_v99T`&|q;<LL+PPm<uph4XAM1{>#z)k1bV6?0!&jpLUO3W+q!~LR8%V
z#0;w7eO;MD#Dq)?0nJC(7l>g8hPzbCu(EZU^LC$P)|kytxNBc(kBcJ&yMOTy`(}G;
zmG3W63Dzu=7Zxlb2^52lWndTk1p@^Rf-AJXZY_7AJ#h-KL3uRYJZID1_2IT`(R2jb
zC)(sRSV$PSOW_i7T~t^bNV)c9%ZeD+pyAT&)$4XIEoVaNTsf&kTM6>kj(34tT!Rib
zs&-;>T0J5>V3S|1$@T0P@Y4vgVeq{TOd`l$m6VdzV`u5{63D9DaPIMITVuxG=*mmS
zGM=8U8wRM*VD3c}jWXjfpc}hxm9LDM*I%qAm)T94WIv}#{&4=~8qnk_0o~F2pZhBa
z6Yvce3;o;6|LnoP4!gNL^>VW*UGbaAT$M4$M^AdI4GyS?FI&;nLY!Z6=h~{@3p%PG
zzG*sYW*~5TZe0BEZa&%0Vxo2fQoxV$*Ncr&y=?PKn=#}!j%>*bn#BHwA{d?L<2%DC
zbF(_wT^~D+jIeVFwsCHC1GZe<Mci-!Q(SR!|7%yUX&#tNum#al8cg0DTeAj@v+f@T
zq*kvG(7z?=XkG8+*$Gt}%$ncdkQ7dVg}?eN&f|HC44x7*&yJ}^t%Q!cbjapJX?7Ce
zyO?ycG+Yl_RxS9jYmwYfuK(j1=?oUa0I5K}?ijdK?f=KvS4LI2c3o2fN{e*2v`RNf
zi3nS|K|n&-Al)g_!bVY$+LUy6cXv04G;Dg){oPy7bIy6k`;2#dzZ@R)499(4*IIL}
zHP@Wzx?FBJ2(f1T<IQ%~C>s8b+%?F{!~2hdY0?DO=+ObS_RDDvR55@|kq-H2+L4>l
z=zw3OYfSC%q_gmXjhf;EwaAg*fnO8XK=f04U@W^Po$MxKPtH=obRH{XM%UZcuJbre
zc9@<HY4jc4bOq@e(xYzr5HLKegL?g|(fL;Kx*v{Fr-%Pz5>v*p*m(M~H&{0sMsJB7
zQraM-ClkUNVMq+Xw)neG5SXQbZus$diyY++L<Ui~h+7zU%=vc?c&GOP=lnj_z$FeC
zAk!7_(}W;I1zdo2l(|wf;V`a&41Elb7YMb(2ad1)Z7g+9>W@@7*p_rPZntj7imGZm
zk#7m8dw1(pjAlNTEM~c|^}w_PzaoT$D}lokrXKDRcG<vsz3!6f-X_=NQ?+k#tCmP*
z84@Tn<^o|T@UHr)S$0dDW9m_SQ)*j)Q?uk&Yeo+!`Y;BeIS#+)E^`st-)3%4?%OII
zq*CO!Z)f4(Ih^B|vzJq<w6%~oxaQ7N$y3(-6|cd*8_;E2qU=d!f?(>|$TAK01H2)_
z+KbV0FIlqj_oMUsNCDiA-shu6BupI1LkO}ywS~TM1#E5C>7j6e<5`e_ha8S~ix0Ut
z2d~b4(@)cg_$qkq8Fk|fv)74Ke3DdF176AA4Hx_p%$VqftUOX}Mj?`NM?*!H6Jh`H
zYot@F>qz33ZE%XCnlUi(a2y)Db7+?npJk1|PQ5e2n)hFS?|YT7TYoh8E_B+)6=*X3
zUw8^obaQnBxsWUd`8gsL`EOiD$(#qg7Pd#SByh26Q4W{zu&3W7>>`4d92tikU%p{H
z?vgW{&=M`UHV`laM5;pr3H^&4av^vw4k^-rN_cg#N^(t@kw0~d(V;FoXAQyi^T7}+
zQ#J1aBC49N3HBvS`jRC9Qy5R^lw>%WFO7XTCHa*oX^7c;J1|KbLLRZaaVC7Wjm5kC
zRk4PsP1_s!Wq7uk&uWx`kW+Mse6YdA>bANeNR+<mi5%dY2T1$ma(jLcAi1BsQ?cv3
zi|P82uxRPW>L@Pn4wqvyGKTim=`Ns1?V<@7x+DM$hC5nBgRBx5Y+KH*&8nUOyZCT<
zxj>p>m1AK4wiFo^eHy^xea^TPFAEWi(q(jC(>VWCh30<vqs>0^)80iSTd&EV&vLli
zYAVONFNVurgq-8JAo84bE{sS(4^T>r9%4dK>X0o1TNvW|Gv~J|KaG-y@q?EmF?4Mu
zKveH}IwOzeS3dg0-3;Vvlo9W`map%*F}x6rdP#gI%05=m8bOy^zB4=%Uv9e($<Te~
z8#$0HK4N}%ES>rt+}ojPj-sTqd<iJh=m8plYOGz6RMx(?N6Bg{)Blyxp>gle#(Oo$
zfkb4u8Fjcr`etC#$c}oLhu9vSioT(<+z#X5n`OS}<{HSz#X&)xMmb=)JCs@(<>|xY
z^Wn4&dM>qL>qg^xC#z6~<Z2?gMpBf8zq0cL8=h$>_)}Zs4wF?I3e(GC@yCu***d%z
zHQM-E%m>QfM#os>n%P<GQy+t}qq&7UR<bTPYv^`1AO6HX#yn~zp}7p1wWCr{ghr6c
z@R5yWF)0I<8jy-~9G8yKWqHbzjuh1!JF2j@PU;z(1zcXxDquJa$bl3u%2fk8K(_H0
zW^29QPRG?iGk~~>O-I;{h5H7C4Vu{*Lj};6yL9KcFs6t4tY_=Znx^3sJ+W$3t}Og|
z6~jJI(OJ#Gix$m6O5;4=+Y+~Xz1?~U_m0d@ND=S;6{WL>|1`(cH|MldS(0BcUSA_r
zhHXt%FnMbPwguC_(BcDlJAJa)Seio6?oIqkAc>*nSQZP!Mv<poO6J$yj0oP7oZO=b
z-{#SnpDOVI+V~}@)7FnvWO%cuP2}6*RV<x`Ppo!SkRzDL0V6aSb#-3yoqLavGdM=2
zrte)RFH-$n+IxbfXKw6rb_LzA8@#x5-;eJf^gc6`ZzixqqeTAam+W=SuYgqPzS_z5
zQEBX`g0XQbpnsjEJ_tCklsz@=q$E%D#SPF(mQ<E<8lu5PPDDEGp_u9s(?>oeEw<*S
zF+avkM4rTUL+wBGTBV~Vco)QHKTR5Ut)Y0>k7++WdbS!gm(tk!?BR90bF+NIlW3L&
z8Y(`<owaaf{$=evnu{E6FLn+#%@LVi>FC-Q!e7ZDa|qq^8OJAEI@>9$YG7@-OE+7m
zURi8LqGmNpYo}=^<ec<y;uzK_o%Aj(??1Zf5*!T1AZ2haQxeRS0i&h_@EgbeTQ?n)
zvh@g)7jNyMe-KN(KH&a^?w`r~%&hm(3KkUnS>9@muUL-aq%4zS&!!@tkw(ZMUi^SH
zUTlOo(szN_>*~Wz;}3@yVsRcLN@-x@sGOkf|M3CYFgMFS6I$l>66N`LGd~R7d<{4o
zHdt>X)Y6#0O#naAgn;jEZ+5?%Mk)yT9Cnbn^f1N#IBoo&2l4xNDa>@QiBXiymP->Q
z9c8kgvSJ%9wTt<p4I)WXBH5!Chpy6#@OUA4>9mnV;n|n6WAMx9qa0&8)<faZdL`$n
zESk4~#V9%SL&%ezL-{Vs{fK%@@k6iNVCoh*TN8q<k$3HmIWfxzD#{Moz8%HfRj+@f
zJy%2V*VoOnUz>soalxnjosqlQldfl0xDzIO{b{WI(}k#kCy0NVfA1L@sSnbowD1l&
zK#ZEq4TX5L%Hg6lcvZ-=7;{o?2Rhe_+fS1T<26RKZdRWWnbIUa1>Gs3vCVnkeDD}}
zVQ2{D)G1Bl*8o(WUd{R5-Dp|)u?!**-ekS|(Km3t+}80JsLK6lJp+tyZxt+W$!(x}
znY`aAhOQe84aj#9OJQx|VAp?hB<|b3?t39~5&e};RUkA`QVo50e&U~toy&=G30?&X
z6wgq{+cyBw9yWpy`_4%ORFkKElnOhb48T5%?bjcHf&Ix)O5k7ODgtl}Z7z?u^6;GS
z{&^gKpHMCxqG`NzpEtVo7m@P(&N{V3(Yw(GA16DccGO=%BK)UzIQRpb#cf2AU7w04
zp*k1ETj(N<`Afvva@unC@K}gP_(nSk9fnVPwvTq^pxi6{)uQ>WYrGSAZ1LP2zJ?dU
zc(<cDpyTu6tfJ~4`zw_bg)12P1&;S|ds9*EkuBL8jHw<-1mTiHNH)W?3V%Vvq~y7{
zo^88{3eh#=58=Pb&AHAk(r>9C-3;3>fh|XcU(-*%POgA$dyUJ7v=zp~0(EM#ha|s;
z8b?k+x=dLl4)JXsdIx)-zG>0F^;GPQrys^9ebS68_zOt{(D7FVLAIufjLNmRz5I{n
z9fxKBby0JPyKBc|pA%wE-RVJ*eXw*8cFDEZ&*$I1j-&&A!zB9nHnfA+7el_4auMV+
zX0Zm94;AE}#=C2%9sQIpFieQZ=<U@fHQeR(><@YATM=#hP{^N|J$4{k5GB%&;TI<G
zizJb}WO4*>pdH+AiEJf4zmQoF0b(at8X@2S5wET(ubiSIR(ice;5*I8%BFj`5alE;
z^7r5V>wn_M@LKJ6D}Vs`EYvtevhBua?<kz6<rA(kCzrIbb~uk84gt&a!*ezo>t&{E
zlgE4hg6O4X7C3;S^x@@ZcZS={2ez=KSda;6E=|nm6zSo-sTU>sSx27<jV{$dFe9<N
zre6<3cb4_L3vY|_vd(o+*WUH=e|}7nCfaWmqC<kz;79TzuDJ!T!QrFXqptp?F-Ig~
z5)`pCaqVbnzV8zOFv2`Oegl02B&WmFqt}Y|9=Z{iVP{g}JLjr26w({7y*pt2?ZukA
z)k&hlgb<yr*|T2mTZ1PS_S(#5O8AzvLkI1oz~v%(h2SP3%q#(-rZ~dLc^q>!4xsQ4
z({{f7QfV_c%ULyV(vNuyNQ{DI%4^P-!x_HdTmn-f*KdsR*M)U@;Z#GomdIL6FE*_w
zJ*VbcX@2H@T<!T<s{p*7mjTm+|9o3XqUV23CXh>)&zcW785Re?!ofhB1(1g1%gK)f
z4o#=q<9{a@JG5LD#N!bXO%VmcSyS58tsR;+55Bn@njNxsoBNv}>E;37t+r>&B<(k$
z(oO-1+8;^5=k&UOZy>t;p_ehmghi&f40T60iSHrhxa(`#nruk28+LB$<>p96IGg~>
zJvi<%KL_xTv%cU-pGv{qcDpI&OpeM<dwz7tJRw+<ktOF<fV%xen9?VICB@;j0qt6p
zgwM3oXLYrw_P$({9v?})R_t>Ll0z+u1a~zgzN?<B@YNKb)50@|#dr9!wgvI{lfpfA
z$U=5j{Upb7-n#{=wUB!S5`qH#ACHxqDPD%6Eq4J>ky=bcu9gMr){5|mx8WH?C|MYI
z?pCnQSr;{!i>9JCpqb!6m=F{v^_-fKpex;@U1fz`cE_hz0dwFMS8mOh_ZJv}&zjM^
ztw8m6l1>SuOBs?=PRTJLDI~8*c)^%9CWv?xfRfgK9z_nvTd|@)Pw4MQfOt?7ga{Z%
zN&Fi62pyEN<Rfu*9tY?!S1<n7VSd-l(2PNcK8QjuW)I}o#{Oxh+!ktzH2u<?q94wI
z$*^Bt@t3+h4D_{8B0p*Py+~IrG$Fm=617gA^W27NUiUA8habm~soWeNF&3DeYX!4Q
zcfTwYuJ;>}Rk7V#Y2N%5FBF3N%}~;Q^PVc@_b)Ik87oUA+%<++4+a`>7z>9Q$v3{|
zluxk%9cJ^7l!xJk<eO^&xS@017+Z2TZ$Lu>P*H3^huO)HqR4KUl}BLSE}-Jw7;|})
z&h2@Z#B9D!*)s~$S;QhsCxwO2$-`ALnCRX0QAx;Z=^FEuc3T0+K*1<3OlRLvixreM
zqD0R!tiC>RH2<G)14If9{9y|Q;6Hz3*SxuW#rSX}^>>z-L?jF)wmig-h)bbU|0I|H
z<bHoxD1z-6iTN0WX{tgxKV{L38iKdgPUdIoN7aHZAju;`1m670z`@XhF_KTY${^se
zwOle9UPl`#ZwH!a#Sr;HVKQNTDg0gaX>Nx0x>M+xl=2|cvnVfT%?Wu3u8McTTAq%W
z%%ItgylgKy$(a_zsMNu5>O?(KE3=<0X-g1|C-Mtr1Ha-=JMHkc7WT-m5J9h*ZH``w
zzf_7(av8zxQ&`fgBn~QoDjU9{_qDnNhuVp={h_qF+QysCOd9ctu2`>=*txGjbBF;(
zC2>s=Jo3&4-}1SE^<ty9jIEAElJhiiG(tGO*&j80>*XvUIwmw(WYqMnKaYM0Ts1|T
zP(P@WgH@(#XU2+>YH=BEp*>QxQmQM3$ji|h+G274TuXHowBwoX`@9;*D4GG4grPD{
z4-nC#!wC^cqI+S$kG=)wZ#e%Cfcgg)c_2Kh$f1%vblBDR><Fu?AR$xDP-Jc-8aL!+
z{E)g7($jNRIr~+nWQP<MU=hruSsq^i<zGUw#$@;1d$=1e0U=o)KuDGc5R$zqesz*d
z*&p_e#(jjLeGD(nF$ZI!D>8zI-oEKaCgokOrO<OVx?Tc_DBR9`RQzW0&q>-Hj&G=5
zikGHhST1X?<azYwh&?@j+U3Z`pZ!Nj)*2xsYqI^plGAA5I52x-weURd;B!)3s^E1G
zQ{8EQZcjW7tp|bsdg_Oo>sQQc8&{gl>D@8AaS0RvbG!lYd(F^>$%YfpoFA?rK6kmz
zoJ$7FOn#WCG}H#<-pf1`&{e*kQT&U7-EC_ppLh~+r{5B3g4Pk~^UvGYc!ccD4@427
z2EU1vT^V|Uq*lajewy?*{82Ka+jjpa!2DekWJ7<mh(i3}?N8Q&TN;3XTipeD(uNfh
zt#=^xi{lBtNr%etCz|UIZXCCz;nuWC4rsC6!DM>%;@0rxyjf!%Fvv&dWQHe%NzOi(
zM*XuGl#u46jjTIRc|(P3ZCJeeht)5L;Z3)Ip^^1+FUL#m^=54(A<O4A<uQhy{(gqE
z-SWXQMX2|^kkdumExWZySiYyZnJnAS0#)E89j#nr|3fdj!e89R5fGRngu5X<i>07~
z1wHAKu?WU)@jAnyO!LMK)GLh*#Ebj9c=?<tI7+rv3ob5z&R|J@{3kM=PJ|iF+y1HQ
zE=lXj?dngJd>M&Hi*+`)v7omFiN7I<b(9DFKNv8(^)GyXAe+BjzyjcVGz7lK+WH6I
zvtj8BZU6KA{}Wv#PyR@l`<pwEZXtvoYtI0oM>=sSZ8=fYYV-rlW&Xnl2T~Bc0DX_k
z2iDM9De;cZjg&~Q!)VCPG~=o)P1U;sK`(8dO&otLDQ<Xb<nn?1K?j}`$xwR5ar+n4
z&?2)eTo+@)A_%A^uze=V?dZ^a#beSc26gjE?oi7;QX$fF#i4jgln^%pl}9YYhhxGV
zYQM_l4p<<t6g8i%@jBSAbe_i&pa2!B<#G0l3*?r_#{$;Zn|dc27!=m%;-f3U5^0Vm
zD;mlMSB4iIH}@QgO~c<RiTekAKp+o*DCE<#taC|1-y?-V1*nMVy5r>ENS*2Ide{X|
zLJ?41jRvnMO1IT|n~m!?s0cINBKuDSMV3#2G5pDFms6#=P+9;~6d>SUVaoq^W}|)$
zB?TCY^ZT*I#Gixs&TeEy{gF_)Q=VYEA=_cU4Uq6NphWp2$+tr`9b!80DJ5W0l6t}d
zlH>*hHmKLw>tKZ8*~GWK0YJo^Z0A8p@=eIUmOUJgH4ye9-CQ~OhS&7!WF%JSg|13!
zkKiWCM>2|LOd#rd#SyX4TpzX>l^By<AXheji6m3mdu5?wYvSxbp1iCctq)UZh>DiV
z|Jl$%E%C7eo?;4l5-sf<H`BEn7LCWh7BK}F+*~5d#?r`bG-`eF&VEBJ!LFb)hZg^C
zER(G04O<?s&Fnb*Na%}>0$My+)O$VKU`}UfPyuD|OwFfE#4uTPQj_5hEFLKLy9g>&
zT$1|F-2*b)z+aiI3`;LC5)fidvqb~|l7av4%!ZA~Y*M6lNY!!4gBp?;MzpqV@@W)%
z{hNHzpqgcmooU`S_U`f+E<+<>a~bz9b#JM`@N}*QHB@6rvvJ7PkD`viW`U$Tf8~cm
z0g<AZ&c!*6u--z!-B8vtyYsNHct$+ykC~*U_uqb&TP3V2)BMCm>Vp{!AygV(!Yf{F
zSLU#u7=`MG5q)OLrdLri`5LVA^iUt)GFuwEwcpTwvm1o{LFos@gqhrGy8ccHK^tWe
zP36c;cLq<YHKklIu*8aRfvdDsnMv+PXu!_3V9@l}yks0MU-11nVe?Z6&blqZNikJ1
z7Yu$AT8X2YI=1m>rqTCGw1T@Y&GJ+x;dW=Z^yAmfyaXBL6jlOyfxkm+q$JK?a5#X=
zBa#XMhpB*Kri<%S)yn@41pg^+?*FEvXbCDk2uN6`kK5dOI=>%G7&Yi92>}F#g3htS
zjPYWTeEfI%d&3Qc$~$98*IES(CBW%!*Td`r?n}>U^EDv)n>=eFQ{(YEPmSYxnS4IN
zI&PeqCje3u#na6YW4ZD!Lt8p~*+5FGs7&p#Y$Wx7(JnxgOOzUa`Kd950g|LF+QX;4
z8E(tPSdUNqsai5gJ-#*V?5vQybRQ-NE|v>XfUvOek#5x-K(>Lk%IU8l8LCsjB87&^
z<zrX2)27HySuoZ*d{Ix8?dm@7{O~j&v!gKxxPWobfy)L|ufX`02=6}z`N&sm*Gs`P
zJ6G=lCo2~RSPgWDuTHGYNN{(+?#_Pi1JyrDO2=$*C)&z3Me2PnPGtX$e1(vYpZqKK
zYW@{_C(H^|gMlqyl_mfNtHur_{1=SyJB#^~e?mfwZ^ILthQpoMeFp*#0Zn3EU_lTJ
z=m^!fzEdR?^+%6H7RZTP!k|6C{Dr*9Ea3viwtqOhl;yS-MNQF4O?~g;Ko-DX;$T1v
zO2{y|0(|Jv{-ijsa6Ki=q&Sq4!Pkp4mpfACbp|{o{752%Dd;j+z8;nyjgutfF)=Lk
zmiy=x;S4@}I1!gl@+(aOZUi1Gl?ysQ`YJw^C<jC6#;AV=A^y(FNrk!Gftulje3qb9
z`bN{^$-|$m{14X{<xbXypkA1VFvYt-t0JERHSd|>rQ}GQ8iJ18svxP@P0zKBGm`!#
zK1+ohFv<Y29@tpmr(1e&0k1`XWHj^`5U;E7g~Zzqcu>XK>h3lUt#j_Y4`xyIQH;5T
z(vPZtm5eT!65L`28dIvVJtY5%7y$&ovGDjWnhzCT$AHM{RRE#oBs}l{{?8x!U8eu5
z#`yXW8Iw0F*`;)CVq*6~72SMfchL7I6UxA6y4{Zeh(IPmO2a<8?ofLZq~_N?bwIX!
z+x`n<<RhPp`F&3`8g%}dl2itoLpy8$nley0)+*oj1{yeP1PH=&xR1|6yYORjpxyEa
zDIL`SKKi8Vk{pYUtSK#{-c;xzdsP&pUK%RwU}r^XySN`dVZ0?JyhHg6m)CRWy8Q?>
zlM;x`#-{k`cWj%|CvDGANIgY~MLf}bJGN0$S-3wW&0o_#4z_w`%7ROCihV*Aqw)#9
z68_rg<_lq+;e-qay^%S=r-QZoyh(`chYDoBU^CTRcK`LyMY`H;Tq3aV8Bn~1l7gf%
zU->Jsl#A}oEq=$_ha&F#&gti24=sJJw!{)HP8VM;C@Mt$PJdtU9UA_@!{dy9@G#$2
z?#84JfQQGUfW)_5>n8AjeD9CI{XV*VHV&H@>(hS;=d~WKE&*h0o8AWVDypcY4kQS0
zE7ieI*ArBK<K?HH7F+R+kg1WZ&1HD>k!?@g?%_*FeoO%F@-qKt&F(-$K>AFBVNflK
zrbdItvtAsuO&cr2H@$)(;P1BlcFYhEGZC}?7}v1elflCVJ=(4hQ^P$)^hLVmM8!-k
zI!UOm9)!kczxZ+-H2zpf2M9Sxl#)a*gn4-yQC?$^)O-xc)hbwn;PG4GcNsQkE5DZg
zpmd`0I8vu!Ntzno)x)tvkQ6B9tWJ|~DqOZ)q9R=rN8l6!K4$*%31-Sdb5-rTQoKKW
z7FL2s($B!bI`K{NsC_6wXJI|bxnFe%@Kf2Th)i-h6TC86MiH@voa!(PKN^5qjwMs+
z!VF(Jj3WT@*N;)#|G-_K$@&L%C}@dQ4aNhRF$gGC^rkxM?EiP=@fS8uVML=H*0&Q!
zBVP#O?itH>KO&w?LFV>kP2vFDP63~-8r=<w!X`aXiV%k&=mAg=(g$w_i6^27-a8;*
zEG;1cp%^0hO#ix{Vlkv!A5#43bX*dWB)BHWlHjxqvZT>hy$xehJ~?Cv(n-8$Pxl@8
z4({IqbWVsMkytXxFg}j-3Wq%E#?A6BBk_K0nVOuyNI|bYPR8u)Y<Ku@3U1%4^bp)O
zVIRTCL(*$NT)ekH9MHr-L*aSOwY5q?)+(|32GG37r4}lYdSl$FC2ZcY3^>V?ZD|lv
zSnue+a!~ln@YA{;dYMClvV8Ila+%+wz;&CqH|{!Zfk)iGX7P;mn^MX<fQk+{j4Q2y
zoBfb00g1t^Q^07bbZaat>omh45nv@%0*hj@dxXrdiq({bn22=M)!!Cih`*`WJ{(Re
zRq$z;)g(2XYtKuF0zb5jA>t}##iLQJ40`6ajfh6i{8rfh75nY~#D1&!jG;JG0C#7T
z0L5ZI)1lly`0?+;<*!DqHWLb<STG}E><qJ0^qv?6woOAeORb8CCO>>Yp%8J|c_4c~
z&-G^<Gs*x#i`jEh50r>%)aCol43g8~B$6we8K&ls`8m|yl24vL48XHqGuHg%5Eksm
z{Q<Al@b<h^j)W!*2bPUnL$GF2&{)DsQw9r%_!l|#WBQCss%u%BPiX1a`-oyR?yPvH
zvX?!67l&$!Y$a1;8CG$8$c&7EB=r5d^;rdRO1l-OM8{^Zl;=P$NPKdX9ZCidg$0|H
zh#z)!%D&#OrJV*Eg(L{!f_|8k_1b6X<sl8qaTwsK(elDRHx$oWW&`L&<6BbeMx(`N
z2TV(d>=XmsjJ0p91AwkC!dvadu3}j>AbTbT^tJtWr$ilmcS^dz=Lbt2&qc1cF8spY
zaG0}=F(;sr|HL{uYC2<AyLmPEJ4C(OtRDQQ+hIlir<ES>vgWu^546&)IsyFXs8mn*
z&q@8=CI9PSGy{PUMOA4HuCOk;^>Nt$siy~w8@0X_e&3BX%A5v<aoVq9-yL0S^ev;s
zCx5^L^e;fM{)B$R9A~$Hhyz27e3#_+C6X7~ie1>je&nkUAyO_*r6rk#z{k@t5;I{S
z9-%a1*W`z_*w7j#Vyra>SAfGhK;Dmo#BOKVF?iB|X@<C`J;!p1ef<Gzbhc&Ohtb4$
zau|^`uOjkvTD2McXrI5Jq=i2T`34oU4{Hg+9g#I2()FC^l;<1iuy%M#0AFUHu>QRq
zz5^_WpLg8np2cv6(Ib|_C-Pu<k*DYF+bczvxq(~~pCYX<iIh{#pY{w5RUy>D;P5ZI
z5`Y1zOB;^Bqa&S#OhB~RWxJp%bAAw57r#1Rfv9;B18@>O`2D{xa&{%ApnvA9;VheD
zF;KFfz|i8^9D5vUN7_88-`S8?R0xHe&o$k;x+<T=_YIsZTKYNLlr(Z0)=uAE9KO)I
z6fs&`4Jz+_Z*wC+`G%tMXSXh|rE$bIUrC1|@!b6(>LdyrkALO*zti|Q5}Ly&NuI}&
z;zEd7leh1}6^Tlq*xA)g8|pB3%>g>;H0frCSyY7fImYI<U>%un#jhIpU)Ezd!Hpx@
zx1%Xqg8A%ohYa=a?O?Bc;}*Vb!Sl!e2)XWNqA4h0m2(eZ>*#5QTmwUYYZ>>E61nm4
z$<L*tcdx6{($K%j2W#3=v5BwbNfQo{`~3)rNJTa6B8GHv-5p=f@-VdbB*%$;Fc8up
z5pzD2uI?^Pwcr*3jNClVp8~IKV(OJwZG@PILVe?<x`9{m6kd8?yiXuu$JiP`c&{Ab
zpW{&FHl%E16kbA05OsH`)$8;IhMKzw{gaM6YA9^9$=6q}_>I)lJ7-+)a{VGZwirfm
z=>;0OZ=jlRZKLbIl0nKc;&hftJ%YmMqJV^fHW48?R%*|#mnD0(3BKou1-?uyZStsM
ze7ikW@P0SGBD}whWU#UZYa?creCtBYE*Qu4@rcE7Rl3d0{^dl2OjV))_@|Fzo!eAj
znz#|wUFpKD{UJ-i`Zcmcc1z>JX+=wea+q-K`yH{<egV*$jB7&qma*23{lv=#$pfL)
z5mc?wy4HW5=|49q#Q`z;z$vnef7Dt3q`Hec4co}284D237Vz_>6=^3tL=PFm?0eLO
zoCFUkYW_t1IiljY{{ZW4x?H`u%Sr0(a>WR%CICTSi9S3tEx2q~P1d$&Ely>Dl8Ff~
zlL~{h3(K`AZ^L3~Dj(eMU}qEd!)1RmV_hJ;vvQm7{Nys^szg47E*5tv_3%IIpqOD6
zat9Vl;*%)f`RGL)=L>1rOh&fr^Mz20EKDNWkIG`W`3rzmv{9Zs^ozGs?%n=|MRp!`
zHjS<Cfcabr!2Z$!wgETUzW`!HfRE3Kx?b%gZB0&jD(A0Mo;xKSi3oEE1obhjqv}s(
zSM%M-oqvY$Zy*5W59`=u{@m1>L(0);k7k99Qg^m663=T5Rt$@o?gKRe%z#5*MXukl
zYg`|J%8Zu2HYEE4hI@Kf&LX;H#09O^YI;G7%vLk~Mu71X*_h2_KUt;I{U+`@On>0P
za^_L^ucz$dm3mjs3kc6`zF#b>`&q3hUw9yD*(Eh;anz75y6<%pd9z*MsTo%$wSew2
z_v=d!qx8OD&x6=Uh0o**Dcm&<yKnZ&CogQ$G$Fq0j}xe%&sS8%(EvkVQ&%dxWWM_#
zSN}wZ`7!yK>*tNZ`gL%KcaUCiCRIA~ruA{VtIg$ycl@r0`kuFS7E`v4h714ug7}kv
zM0Bljafa_GJ6oSrPYp%Tpj)F0jMOaRaU*$O_X%CBpeEBc#x}vbiRaTqSKmf~<^!sW
zCdWTbO(}Uh!BuWjUY>9?qKluyUcpl{VJqRk!Y$2}Y`wmL+=qMc3`8+JXMnS8d33*G
zV#+~dhUKBU`d4Y3OaWz|i{eh)$3Trb<X>Si8I`0%ThhrDrOmKOwk1OdhW*|YB$4xe
z(g&G#^D$s_g2}sh5x^LZ>3tpO&pWqsjMhd!C7bZ}kx-;dk!}|Z0DVvTp^s2{e>BJ4
zimBN|A;%wV{6}WXK+$R2MdhmRMv@d+-MR+&)qV;Q68((+u4emUvbZmjowB#47cc{=
zO8!|E_x`RRF)1H&41p5!o7=VcF#PL<-7D43lG0E?31S0{&}-)iq3cVE0t*X$uQWdE
zpN!7y;h?)ZB~k&_inGAqV4jcQPs7!v&?nk_z@{$g_Dux2z`*z7?CfE=hTHmqgoF!6
z&(0L9Ld1aKev+ixB^DLS@TW_wuM1bnz6H;!4GqO8?3cSAdgr40XAWI@+jLK|$Xj?F
zUTy3ag7zLC)^7!QM8wZzUO|a-Hio`iE;cwD)hb5J?@u_t7kQVUmR8pE?3Sr+Wre_+
z2;+qqgt~NDLC~Vfp;qbp&5G`BPxwEByuWL=*J=o*a2!xd9De6}<L2-eKM!!jX%CR`
z%|EMuCm=9d7SFGV8|3iT&a5U_4HBV!7<syE`dC05Q%IX9uMu9F6>}*&qzkOX1pq5?
z)_<+UVXwXE44%)gfDmH*h=n+ypPAHzs8E@(F{0$sD~uQWO}1392yCAlk8~uE2yU+R
z(TyCvCE;?b$1OSU(ek`@R*9Qo$X}yjHMvXa0<}|k5h)rTA3v<hgL=)BO;X0Fh?&dy
z!2EU6bQ7c923e^*kAX^rm6ANrIkusQ^s4!j`{h2}>zTdnU{FS1ZZ{8`xa0aOK<;;L
zrwvcyRl^rnZtfxeSxW{AKoIm~XGX1%Y$MjtgOz?sGVR^GX$s%Av%XwY>Ekkml(@f|
z!e2AI<cxl|Cgj}obp{A5;}^$+@DzLwZ*IQs@0JF07p$}3-dU&$3<W_Q%WcZWh}n}S
z06`9^<p%&l%J%;QV%dBSDFohGdG3kWRfM`Zta`q0cm#c^R;;@i{|k{%Oojb3kG1OZ
z(a4_(<WCL5#@DA7f6Tk}E9FQOX5*s8i(D?+6u|Z5gDK$i_WY(x*i_CHj_EN};EmLt
zp>ElxfieNxU!DRaK8DoiD-4=d$_<;Wyz*l1-#K+#w^yek^XGm9ER?>73y*XWfGA5{
z$PaV>Yl(@l+ih!O0&c|elV5apLo_}}0w*>5LnQR|h4+<__~qt7wJDL;aUGtt)GEps
zY{B5%biv=dac<rwt1Q&07XS;DH?Q&f;PZR0&SXcn$jkPZ$%A*za>_Uk9ouXIhrniB
z9k3ZULRt0ALunbTaE%fj+lj>FV?Vq^5_`T>+;wx0?I{YMm|@K@VG()1JsiuOpksSy
zX9Cr4JFR17i@_(RI6_8f8Zg~dTdbc7sXa~&4^=v8eFirJ(VRavJehzve8K9$OLtsk
zm+&oqGZ-eXS9BlPpN0bsAk&?#tmH~S>~Ws9r~MvEKd?jbDS>vO&b101$IVUI-u)cV
zENmU<VPUy@4w^)uIosMT=DfMwgo<6)vhg)2;$qykZfPEGS38`gNh0n=1pZ+Sz}gwZ
z<Q1!e+1|Of+}CVs37h)aQB*}}xomkp3j|>+wm&F3><(?Gyi0ttR_W^T$bJ)d8RVz$
z9AMu$a_;pr&CGhvB}=UBj)h4g;jYrozX3j<PQ<N?^NbOgIXnUo<4wnWc3WqqhDqMz
zUpM=5b04<|U((diADu?;71z)9EHdQ*w@5xgX%{Z`1;l=1KZ)lum}l|zJ|0j2V^6)C
zgBsTrRLQS4t9iTC4$Xd%IRYv4oBc-%pwMOEr-1C5yUp0}21ZeVW_{s5eTu)^o_y4Z
zaq>azlixHLyL&R4#yBiU@Q`zq<tNQJ_{oyOUc)681K1X{jldkuO|JGAd+7sf7`vg+
z*vbU3P}h=w*^gML+X5EqNYB}!f|swk`*X=+avW1gxZt5vEZk|jJquxOyM4S_EWkRR
zgN4Izb-Z~@WB}}qahxvJivf5v9uK{zLlp|GSb;39H9BLX{aDSid(XX;e(aNsMl(~n
zuhP~@e-`RqYn3?=_UWe0@2mwV+piK{$(+RT0ZcTQ0Cu{mTi{sRJ;#gpT)zujK;B2}
zcv!lV!(Y@9Fmu#ZQTe<JY)HENbO$8+JmTKtvNMYV?3W`1sFyKX1(wlT*gsoxRsTW(
z9G1%~{^}=F>#jRjV)DuYiwwsW!7(*HnMl7Q2eXsD)&fEHJG6KnE^@G*O_dI;G}7Y)
zb$h-dpkS&%?S~_Z(gSiM1pfs`?hnhNgyw@er|!(2_bG_U-FMuY-==qU94dQriIczY
zC#9LE3D=6bt3OQ~fa+Xky?vyp8Nm+}XTlLczivmiKp8Nr+VlEs)3C=+;8*1<I&VmI
z=MdKn%vpQvZ02k03@(DL>4V9zu)y7pU9mg*Ju!m*>#^<<bMLs#H$*<Xw4EFKZwZ4F
zkuV~ll5M7HOUSebNTfRzlh~<fkx2R&7d%H9N!`rGWHSd+0CW^Z@gfSw{S4UV;)Cra
zCc3l`e<CkZ5cizD$YCjf+Y|PLA#5sfoOu^6l_feptSdC40ul02-Z_7^4A3{;uq9*W
z!)000Ex!jSTB^J2Fz^maqOpnRuZ82#oF3Akt!1H;C=11{ya}{<+1)}Lq~nvXk*up5
zY_=LqC3Zs82GzxeDV9T~j3e+F0HTx=d;)rzPcSpnwOyv=9m11~?9W_sMs!!_+bIfY
z&XnHgRo_~utHPn+?}tqmSlh$<;l%ll8-u4C5_dBm$D4E-o$izQIwNCd0JBM>pPFSv
z6ma+9_c}l5hDJ;Y-n@%-aq(!vRcT}`dr)vUVe}}^U{<Npa8X9=_xZmYG2-zVpx~>L
zl??~0EX{f{x&?UP<O4}Rq0Y{>2G;<3-ibp#H)^K1TlL(qPg}evFWTbb1a7nUK5xH{
z`JrXyd}UU|mHCGo&s9@?<9Wuq<4(u??H~jCVgO4tCJR9v&?x<3fqvo?LjIX1Qx%FP
zsB@)f$|_$p#0<>K3M$JADV!kJK9`4{S5EzPMC|%FX^822ScOtbj!~7*aa@@BIk?ke
zZ1knijaOc>)?!Q`3*LZ^Aj<<0%c}rzGUr*uWO|eL(4NOip=E=uo565QC}+fw-AXGv
zpL>D%(ib9-(13cG=6OZAEp-#LkV7$sv&5`#*@z&2Ct5*G1x0B=*qL*L%Xne-J$?hV
z)%Cdjhz>D_VN{NO#+DVv^Vv2G31ep~Cf^B((=7&@y=4twu0lfbqbZwd#tAi_aOeE9
z<S(FN9daKUcZ=2D_D>m0<cA(vUl_Tsu~!49-qM{fLCSUC^H^8z1!*>^VCe~ZhTRA+
z@gx{Md0M)aEWC6UHH!>-+p`>a>Un-c8$E7=|C57i@ak%msq!sflf*c#1-Nk!eX`!v
zHN9#^!q>En*}I{a=WTB2-ckTH#BinEWAlh*`uN_MPNBU$$^d!rJeKf(#_j(k(%1AT
zLN8L5Vr`(30d_^nEGBlz$$UIbV4hzzPtq;cvF<f$8>g6Vh7(_eaHn|LVf|ph`0!r;
z&(0w>ERVRX`u^xWjgr514$I_=E=!-L$3jk5EnXX0yjAu|EnV)HBAV&(Px8;EwzjfL
z&Jh>SAby~sKF+~|@$h^!+6beC8bgB?lNPsAmqd<sv-vO)Zq^Ap89TyIV?5u)c-=Uv
zq9*j+*%(t@0)|YLR(q&(v8I?*y@`QeH`raJIxY$93l;f<s2F;=qhW(m+$|Ba-d0S;
z_}iN+`6BRg4^JE#ic+$;ciI$xh4mDd2iw5$*4QqB4iD&J@p$CEGB#f4Hm4f_HdUEr
zp>)0l_2`qgDOZ5#$pO=X?pL!wEHG?Eer8|CW2zQ0Y&YjL9Dg03fQ0HwYB~45JfOmM
zb{00Sd~4Coj=RnoZ(#(w>@~bx&9GB5)*rVV0&Kdn7a)<7Am4+p4QB(MWO#P-&@R2R
z1E!}AdrgC^>um#JJZBO-qY;;I8_&J;jeN6}7?<`!dF`T;sz_UtH$Q^695d%;&Jw>(
zxG^b~Z(GUldHitJ%hH3*&!=1nd!>5MDOQ;?5>PdCJ*|AVp_KlJWZu`uDcY=SPh=)l
zgvyGOyej1Qh-9D&(&hcuwaLfT=}7Km*-;5di$CJX#>{80-Pvtd0e;pk;ga?59+_rT
zIPgkKl7pxd-rG=s=XqW}JBd{|Z4oK5cr~3m=21C-TPI(cmiF9psR!fP<piS{{pXO#
zy2|^!-u(Nf0WPAvu(bq9{_4}Xrr678`o70F=ya4d%ZGjGT5K7Fmf{yngd*?uQ*)-6
zS%s2Oi`-#W34*3+>g)r-`(D*IsT)xCEtUGC6Oq%Rs=Xddy>F%UXylk1S;t~bUi?b8
z^;uKrR&XtszVJJ3PS(d_h4oiBy0(^D`_2ZB#Ea~pR7c|kc}f#CQ;n4hi_x0fB(C2D
zZdg3`le0QBIwl)%8`@{{7uiR>uR6u}Z|a^_?v7~bHC&r_Tu=8yuOqze_LJ2vsW9%^
z;>!)VXMHcv7s0btG(W()lI=6=A;u#vEk)6&JsWT1!UmjC2*7>*z7><(?=@Z(4HQX4
zR)DA}%oW4kIK8$$PMIFYbG&VrubSPKsMoD+=I{jOt$ltT!vCcY$>xWsac>_*KBN0|
z(FZ~E{cv?{heEo{^U<sw?+qRcQp=#O=kl_^N(cLEY!^6#9ZraJu-IN-1n89foW6fE
z1i_Oy2@T)SQoh#dQ|55iFmnFDwiX;A^&VISea+?6uFyp}L{pj57$Ws3y~0YS6Mii{
z0uF%>N8$dEjy8%g8KXmc%;VC{G<|`LKY}KSzczt~vX3!}M}F$&4POD}9~+bTM<|`F
zwe`Qab=P>teKPV6x9@Qz;C_@R0m~-7ATXbqi3SNzd`)Wj=`MpaVHT799`>tGyJ)|7
zh=${P-G`5*%uvbnpvlg}h^%4t@_n$lcaR}K!5m>%Jv>KfN4qa6F7PYO<v*LbT8ylm
zGpy%?&opf8xm-h8WVTrreOsZ|zj{8EOi?=~6%gKjcq9bxTsDHfC%uIqElb79RN}S9
zj^14?bBb*EIjXhCbr)aLyx{e%-RiFHWg`cb`snRj1n+ujm((8)v>uRT#fp37+&nwl
zsGcnZ7%RsljgLW;;?tKWFbmX5v6Sx+e$z%8(>u3Ya~m57)N?wBia+f1qzwmR!@o&Y
zP@{8nSPfD0(LEsPcWpc!dZ-`>&&w?XIQ6h^M~%SXCZ+T*E(7+Gjmz|au+vK!1?Eq8
zlecGwo1gCMRa#ls?aXkSd0u|@RNaoReZlypy1)hnCz%e|mNs)Db3+-(afPbpDxwfv
zM@HY0UBJoU?Q<CNppT)1mdqHf(!rAJg&K-Yoy4F4wYSPQ8?2h;ytg;~50cYeL}pzU
z-cAvPO<Rp?rSJT_yJpX1T__bri$6i>S(!)ntweXG$S>_~J$Mu!PSZGT&iTFKWfPbb
z!`7jg>gN$iB^TmDZ3@^AgDc*#CrBRM%;9Or+MM>ifTjvTpB(jSU}Ro$;P_|m1~u5^
z1#o{dc}YLai-mS$6IuW4gfUUwzg$*@_Ggo8*Sku~pGEIt`sbtt#9DFpo<4pS2|X?}
zkf=RgE$t`k%i5i>b*yu$Vo!SmbT2QS96$aq@1isi@h++lPt{20+k-KhjUt+{{4i2H
zZ|xZIov^DPoSRw<#qziJLWFK*aRHK>7WG66^B}Uj&WiM{o8eljxDi?o0On%(E{YX~
z3B#3KywqvxV9~J^0AWE9_af|$PVY&^^^bJoR;cLC1dnh*Td7wX3b5%RcOT8+spTiS
z&(Jt&+R^*!Ti<)KD9^W_r%V~XiNdIZ7t5<B#|f@QiBq#s@VFRs1LH9m-%T1@FpP&?
zOH}jZkj2-7(&ZzWh(sbRIu5W);y!6H)vn#~?wH0f?_aTCWt{+aFIney92+LA%MRg$
zMKd{nQ{*O;=zUR?G>ti(pBd^nZHHE6ykJ*sUFmANTGPAQKL>&BX`P(F6Frp=d^u+-
zb@cLHp$tIAe2PzInN<azGQUcGBk_vry@=D|(^|S>VL&c>*>$TVFKK$C{gLM44Lqsg
zYC@!Q8|Cus=VClRn0&^E?YzgW_mw^{+_2~pwZ6jeMY|Ti+TyV$zAbgWIo-~BUg&wL
zd$kC;3qRGOz7W_R%~b5wjPJN#MU{N0Aea$4;7kKdx-N;}w?JF>jnZEfDN?vt)%@h-
zqFwak9`0!7>F*=&&0bCWEGSnl40>jQxy4UXh=}1L-UERC5&@Nu4WXqay)v7>q+0wG
z*tUF6?sNn7_F+=$8(aS*S6>I9{Up<QFjTeF!~Y(mweNi(md(aq-L$5GmMGI|4_<OP
z$?a9We)vF?CqK3`W8Dex=&JT8ZKfXi2Wc@ac+C&Q7)e}wR2AoLI-SWIu~0Ttm}L;<
zEb%S6%^8~Vnrv9#%A0V=7}(fzgW62l2S2`(PL_BB@jdCbvgWfKWvidH?AWiyX(fCr
zfD0;>aeLOcF|<&wiVKxQwa;J-mGDOg%fu{yQ*!jVoqW)?t(ov~QDSoDZPo#Crmzmn
z@c4r%3rN{sRqfIl=SONE@}pPBjEQFcDHFoi`2|g5De6W%EI#i{Z*Qi)M5?`7tZS=0
zjZNYTaZ=j=m8cn#s`fW#^4|!Mzv%*hu2)V5QCP>`Zlv*P%x?c(>HY)ieqsKgiwZ!C
z4^8a=Ch0}~amYo;1b9ePAE=WA@4?$LwFj1o-T-?^G6)K_7wTz#Ngld~(+*J=dFVb-
zUfgA`RJ~QlQB;7K%SoOa7}I0t7<mcV7w4o0Di~4x>Y|vLS|#yR!A9toNrhj=-7+zN
zUKow%A%>(d6<Qr`I;9_q4%%m>+y3blp*0L-*);0ul|DlW+TIRog|m#zZjR~Y+gy<E
z`DaurpLWXvp=?z&(j{PyA|{@j?zqJ<s)j*ff715ixB-5I@nBt&T=At6?eMtG#K7zx
zMWZCgQV~<2?BvEl;NmgKP;r`3ml<EhZw_Q$ag6ZH1xC{Q1Sc551qml>spGy_mwA6u
ziK-|lD~AFalg3i=7GMCey}<MJg^^!*z^6XLVCjYT8KIFQB!hV#z`o(i(e}pN0Ud*C
zdumoIp%WAnH2QgbAkVrAyK#b+ln%%l3a}H6ofS0lQ_Ldb>ZQ2th^$H4yQ;;lNzaQh
zGsCOJJePgFGr7yL%Ov83;WyD*B17ckinkq4jB>d#eW?-^Zn;IFjdnd@cLjZ`Y8uV}
zIVe|a@X7_KhI;?vXFdH$xWHdI;Aum{6enU4({KSFBO|n(H^rKB8{<@~6fpGYT^~qF
znw@HwMZm5kEVTG=yhcHM$W-`c4PU+P(HM<{&Mwo%N~JG02xFcbzQUL8+T^ZPuFNvr
zZu8Q^qlpJx#+4vDXJme3LL%mr`YJCGV^^^I65QyzT9%2)i*YPIeEEJsBDaSSi`(n`
zc>=$uRf#y=rM~@U@gs#5gbzfKud>e|*ZTL4fp5Omo&2^*UV+=)5ZkgTAGeJD8H&Ix
z)Ll_0nMYd3bE2LrK@7c#49Y2Vr1XPbctKeQ9Cu>U_qRVT+6&vxbopFb-qT^D0Hw^(
z*2fTkz-}e9{n>D%t`~Kx-Y2(omY5VYA5$w%BW9N>b)b_3p0;ABvl_Bys0x(D+694_
zjhriMtHHAr6E<(Fc0@+w`Cc?W-0mr&slpJi<4#}_jr}h@p-4$Gd_T!D<vO{HTJQPy
zXSbF1CLJ{B1%>_9p({IaQ#`&^#1I2~TYI2h7z9iN9p4UvumiLNTbTT&n!SBq&@UsW
z5Mua|Eh%tL#~sRZE29AuSb$t;hw;NAqWz}iv(^dG5XW8dtC>SJZ<8gNUwMhTD*;KA
zn^A7!U~KSsmewKrFvB^!p@kW<zz3s5p0wF%b)$wfhFFI(?y`2g*XzI5ka>T|@tT@Y
z4HcnwXLcSgrNFg5`HkRxUPjC3dhX%vFPG@RT4geA%_$j2FmNo7JM&GNB-G7m`WvWG
zifAiTew75G;_ao~r&R{)qaJF(Q)qi4d`893U*8&<y!2xn4F!i~V86DmVtOlaEz6K+
zEZ?=LEKs8U+lk5Vk$S##-R#E&5tk~g7zr0jcxMNm0a&yrgIARdjr>;iv=xJ^Sb>K*
zL-#S3e<sJF=+pHmyL8kze$V(M?;)L>c-`X}llLN4XX$CXB1=8IRZY)SGXc~2Rlzf(
zjYpmX4?!t4JH>tJq)_oTHOtYAcRphw-2t~(fRu(Ao>BbXmkIAHJHj=;pQycwjymTT
z)!Qc`IG@Tm0TwfV8gvL~Pv#2$)t)rynY5WINpb;9>`?sC>6P@Y09UOmYPagOd&~K&
z3sr`tWMkpc_;2%A9TATu9(XK*TWbT{Kg%)X7LseR5f#ToU*3B)3s6a{gtt(}-8Rio
zmF;ew>=i-Lr(#H#@1ug18rsA}m#LgicHWhTz8(YD2f9gP`?pQiHSH&}w8Pg&KSUyu
zjaPr(&0D88dWe|%{yGkZddgKoI=$aA7%4qednRll<*wH>SKz(koc3y}(e<16lZ%^Y
z76sTS1JhZB6|74<-tQPG(Bd2JF4d3UQoy{)3<E^A#`<N__Lf4w*Bd^FyF95oZDP^7
z6J|cRXVKFv{{_@G-uP@i21jp(c(<zja;d4tIl-w|?4hS_K>aEWisUlc<@J63rl@^c
zgMUV?|6-p^q65T}c)oY)xud@GN3`1`He;$6%g4JP-|vjX(r|sd16B@~S}wAl_+X(@
z`=J9p^^KulL4+QY^n;tsUyo;b+Nhef(i-3?u@crZJxnUzjjvTGg+|&GXywwV<3@PC
z7`1mZ2&HQ^9lCAtt|%!vJZ2G|kSa19>5(sk7sqyYiZ{kI=sVU+);#3Ednti$g+wnA
zfckXQA20VS)S1$4?a}LLf;EM`FL%AMRMuz`6Ba}FmJ*B&s?TS(!o|e%2?h=w89CX(
zy{Yj#Iy{6~^~PkoImtgs8qW#jy}O@=z`AY@zxn!TyW#>(f#OUte{{PjIf^~&fhrNE
znw(W%3tzhg&hx#FmpN$&u58XDH?6qi1B1w7Z_izdi-Y5@#t_4jmv_}A&oFrzjHr$G
zgSaDB96auL!0EUYI>sugN%m7)_G)h6+aQ<26h+nA(sm=ye&<f3Rr<TzA@9?dLEPn2
z)^$)Ek>%cQ7OLcPLQA)2*bM^)f&%af2}Df}HEDyg!1OPspTwyi#WN2d=zb_e|GLiy
zW?2+>%HYYwRrLoXj@Xc(!9e3ivO?ysHfvwsw(jMQYcXFx$jAO*uJs<eka^3syUKEX
zt+UNl(pjmd%`6f7F;J|}jT<o&jZKs*A#Q#m78<$u{>_@-QJN2P@YIPuyt`v0p5>7t
z|0diLhu|8hE+~GGzCHg%Fv2dR(`9r8u<zR)mssA77Kbi4?F@{>cm>EqX~)<$_yAhf
zO!YSd&+&>JsI9N_4^ACrr$~wHa?NwTI<;-(%0)N}yxr*Cw+Dmp<-wc%3St~R869rl
ztA`XKCu7gFy7>X;sB4|m$pQJ;S_nWoB>$M@y5}-$kmC{M>Sm*+dj8V%f=xf4dG}%E
zZDg=aSm4tA=jpb3hhaJw8AHFY2EPPQQ{ju1s9p7bzni|H6)`DG56gh^TBJ1`u56#e
z&SPy3zbwBkH;DuW=l|vIXFDPIBaW7W2SdgOz34a3^F439{83r_#88d#v}iBZln)!c
zz8^}Tuf_ABDkD3J(ztgAhw!$=3xYqry3NgSGyf<E-tu&4vw3X<?CAI`3;#+t{N5A2
z2Z<zqcnb7@DeUtnAlKt0jYPymd6x8RsR{;+R4*gCY|gPBAMXf#KhAf44QJ)&VDJ_d
z5C-&4zC8HF8p$UNuJs!qk;UqJ)SW-FlGpe(K^v_BCLgxu>0qM9=>_OF@jFCOB*0eM
zqPl@CfM7$P_9L`-Wq-+|*Gf#oNZQAY@l<UC2j^8kNK2WP`Zjg;1GPrgTca(;(-8Pb
zE+oI^qE(u!G(rSSN#ymczHzhXfINNlCD65zP`(=B@;GQ6je%SPH9%?SqW7%W5_inD
ztb6rV>5l7dR!;SJp@ztbptCK1qWcsXYp|d{y2&R)p>GbDScZ4J4wi6k=`KL?A@(E7
z^^ZQ_GTSu_P2cq&^S`W+*xut&E6QTikYr5)%Keuq^8<+a682vK+z4K|D>98+cgh-L
z*>cUe#Mh*<a&UKat*fGcdo=6F@45bIxtF{WP(U9Eb@Z2P+b9uHzO*=w11r$dVw1%&
zl3>r<rtH>fG>w0&0pzB4%96#)1d+`R-A;zc&hlyBblvAQW%*R$TmPN0_s16wqj_ie
z!bsRq2+Joy@r6YDrp)`idd!vOYG*>6rrdd|olo_@8Oeh0rY)+SEoYy;w7=PGn%h*^
zj}bBeKGeMx@ya`K2lt$-?8k@-cy3k`!{{~4!$H5&DHLFQ&{S~R?@gCSFVqj&eD>bj
zSKN(~$Kx0g&<jGX!z*VH5b?-xA~e6VboF)!_SQI9d&#f$U-5gs(C;fMBY5%SiJLm>
z*)@S6Dz-gRJ6SV&4;r=KPQRCZFt_zN3Y<>aWvw|VMA45LSdv!rabPEwIBxGX;y2N0
zQ%eZ~mT!QCdK{7GuSAoVhZ&~3l{o0dgd^OzCR8cKdlhg`ibzk_h{M#?Enm_`DO&Rt
zUeEG$$=(VVb0N8BL`!&Hn~muT9hh?3@pi2Mboy3kg=7#?8y=GPi(ms(--#(*zC#tz
zbhm#>Y|-f^V}2lmYx&AaXxreh`@tGWL1Yh6>{|4X`4al~x-5oqs{}O2y;#s($amjQ
z)htr_g88{XnuOTPv<z54M*w+tvrf_9=;PCYdM-SsrHUM+Aa#3szPOX8ka0K41FV_g
zHQqGadLPC-OA-Lai`LgS7UHinb+Jge%L#+=lsnq*^KO=8DFP#bx|LO5IoKCWaqD==
z=%1!sJHRFhmR5@=!2I*q{{yt6lk!G0wu34=Cg&^|`(7LGbnVY|CIn4=;eF8<wwhr<
zUD^?DbsGKXVW<q64;Q|r<!UZNVP)mOten`*iT@G6pW1TiFm3}c5O2!~-(DPC;q63D
z?G^u|09#9+l0V$)Qa(4L?YU-HBLs*Ox2Ixt+YxnuZE+eKK=@qr@TC~O@^hQZ(EWeR
zy=7F?@7g~~Nl1&*Ff<}k64D(aFo>k2gmjm52ud>`-6b7ED<GhBcc(NA-8I1f3;y<_
zdq4Y}wa!`R#d*UkSUUIIab2J56Tqmqcf}pv>sBc%jC#?hX#F*=!8=2xzBQ#>Aw7Y8
z_08pKmB13cT1^SC<)-@rG{5S&XgW=PsYFTKDSlq2@BNMc<!{*Y;k;Z4NxoiggOgc9
zYRZH~9=)cc!Xu&W){K+x?`g{Jx8u71;Zpn$ov!H3?OtmJV5N^gnUkp;Y$1)N=j1sr
z(9lE$5v)&^nkh>VGf60%zag#1<Bv^oGJ|;VU+lfZ{wfMgCWE6IJz69J{}=^GyDbI=
zg}`G3Qf@M?1AoeJSE+$VoAmgPWuMw@Z7{QLTz<2vvUsFZy!3=TS^{kmX+XuTng5{|
z76lh2+rT5h4?yUVU%0lm(b&F=y|@o`D)kQ)<#ScNnhzEn1CRP!NAA%Qol6yE2m@rg
z^E?P%fDZR4W`(^{nlC%St(peX4)$84o}#<Jx!n2*2o_IcqZ41G4^EPu5}PWx;gnAu
z|00ahyCw3vrimCFHhN^jWxMe5@8mMbrf=-)cP2KAJr7Tmss(iGXQe&)3Puv%Pdh#r
z(1-qv6j3Z$z~D((&RMdl9#hbg$)UA*dfjR*h0ab_^a5U=J8$A-&Umx7wz_6rejY;g
zGxli(VNZ2_Ma~!U0>D#H4dz{~-2P^J@pRogR((^A6%813WAfnMj>61C??zz^?nDW>
zy+w`sxE6DnG2CWjGr{+XWT0jj?+zp{tKJr1^DeJH{JFXzv(u@&@FbB5Q1PU!-S=IM
zpM_MV)a{ZVJKp47Zl^${Cp)OW$|z<hz&GhOCkmoA%s4k1Y&`dkVNX;T0j`}9zwPvL
z)JxvWM-aVK{<Q+akPt%XiNkKeqMV_CVO@pQ8f%?&iK54GywKTsqa1-+4<m5;Z0676
zt~5U7TB${gHZv#iOl)_o%YVdNo49-*xAUk#?!I?Lrmc`Vqk!|=WZ%^FOh?%G<COh_
z{6fsm+LT-gfcyith)IL%>{fwv%OXLuvB&(W1OJ=x1)!&nce*384+PD4zMn3yX4Z&A
zK9Eo}X=;CJDkU^QAfo~MFYc=fkG05AVbj@M7yo+P0mia$L0t|Wmyuy@?Iu)F3gsmL
zvKo0%C_d=cO9!v;rzi>yr@6hpfxWrPZVz#wuDhVs6>1$XXI4xwe(+8LkZNIbY$Bx-
z?IAG{@SbI;hnI#f#`Af&;rv(>PnQ`%v#`%Yt#EtgX;9Uc6H<uj!z%1ZY0iZ6ECj~v
z{YuJixjCHeILL6FyZ{oek&-|>RzUGJO2*^^{ZQ3zj2^u1ZQo+4{7VIf0Uy!Z36p7-
zS8U^Ts@hlAwUxYR>nH!_-GVx=S8}%_XPvro#$Azb%#)nK*G}v4yX_Mqsr_)`OL^}1
z6E0{1u4&3N!ww)J3&{7&j+fR~ZpRhP2Hvg-awv%l64Jzq-eR8`9h1VYjgMYlL0?7!
zNfgaFJn%30O8K8tUsn_ecbQKYY^phr`3!mbrlr<>Vch<58RN7IlmIlV*T7xzYWBCj
zrYM0KJ7X2sGYTrggw>Krl;&pJ`>=1}$*oZdt2#QDmecmT+#?0A9r*6q&D7<5Sh%iU
z3v(>1_tvwz<p{Yful|ei1m_(ek!gOGv7nQ+x%VZ#$u}02t7$OF^vR-bV_+7&yeWkl
z_i$=J0Q@q=!HVr<GCeJ&RP}&KRFBX@pQzE*D?zJE*mpHmG~Kme#M9Dy%W2GV7Mu1)
zG3{Gdj_2X^%d(W}nQ6DJBzTSX-lmbe$#N%C8qaKXwXPp)_=y~{*+x|(D7pNdp&eX}
zajJoAizwS|DPY|4Oy-^YXA>SbDw%of%c)<wooV$4x|^2Ai?08zcHFqznC<1@`+kiN
z20gubk&0R|-hPrnAoAf(MmFX}nSsX%Kc)2&n@QtE5k?e$DvrhDfnOYhDI_t20<a4B
z@^B+%I1cE+f&nh%&ISpshOe*z?L4baG(@9Apl8HH!4oCLV5L7)*N?ww>A1PtLp(G1
zes6n{%F=RdD(QX|PEqr^?O9p4UzeDZZ?weqnf|cUYlYU9s~X#Ne1pBP_Izm%hSd7S
zO7*L^FGnHc%T0@Z9dVjxrQ^vZOM(-Qtgtc@s`4mgB!%3Dg-76f-0(+WgeX~;D&?z_
zB(3@}vNyUFU^T|Zn9-6XPz@`yrl}m!wYl7?e0dYe{M2H;A@h~0LV}xQ?%f8($mkkw
z*K2OU4jjIAE$GEj)MN^?tn#8V*h-@__<nw#<b1mM{t@QpTv?rt?cT>$#~R6WzOsJ2
z>Fjn^CGVNQn*B+#;N#j_1%QMCUs?4ufeSoo-$d`Lz3dr{DT%13K}n3V0hv1x^%cLm
zlK%++0RF+ZUPkE*zcJ=FoFezMPUbpm>39Nz6|UZW_T1~+w8*i=QNNYmy{{2ZH+%Kf
zcb3P+Wn{SMUd7g>UPFlPD3@KEWZl-FoE6aRs5M%9x-ZwuZ2j8Ctp-f^v>0B?mJpaq
zC-V-$R`%0XD;#!&rg<%7hccBg>vq;Rl~<#zK?YKC=hpAGyK5ErCs-~SFczicFt-QV
zai2IS1GDzedj%uo(524)cTu_lx&~HbyN~X9UP=RgeA5(H?=9x+mL8uJs#|{z0zu)V
z1!32=q4d>q!uKAhhz_UAGEZ;c?8tJ%Vcckw<)V&VvD)KnIU}1<C7`SKWI`LWmf5Vz
zsb_^>Ts#S&cJ}|Z!O;-7)5MqhGy&Bk687n3O?!W6pk>pQmt(nMvm!DjqA)s#Eg{}U
zkTR>m-n%wdCG3+4yp_)ip7h1ca1}I>v61ma68yoBGyB)23(Zs_6>ciJ@WJfqC9nO&
zMY#?Uf9bNGKv-@XqZvzcBfmC@xSIXw!o5(|V0$?bnl##YkC<cL5mK-OTg2jzXcaxG
z*Af=?k3lsfKwqsSqMuNkSA@S9sGrwh&Plr*sOLB;$_8?LF(WPf;yDb>-Oc)BS^pQV
zZ%##H6I1%DR;gR1Fc9ZQ2W784jIJxPro{Dn#vHHpmPp7}Kc8~5!RzvPyS$kdvY6AG
zYHTv^pjNpG(#3QdO)CMf#8Ahfb8S{{z4gY`3zfS*UKKjIm>92O;c-ZH(kih?gYU_{
zOamnM7{~lfFXSa;QdOS7mBXU#egY1ySrlEwF^-_CUiWGvrZnbqW?=WRrl#I@F9pKC
zyO*xU*kdxT%7#nOXE)E`X{eA*g)yOu*8(~8mLiGc@KIEP^Erik1-ID*9n4~3@m{Q6
zW8`g;)(PA@4OcOjZ<Go{0>x%7&vlv8y6e<=R9Rih;wowrCDG#g=)J`(-8p}9lHRw@
z+hI&qs$Ij?7lEfxo<`}UdiX~7$;2hZ_(+Sa3k_A+9J^^)DN<RNcQB_fO*4lFHe&JC
zJtax7f44h!idmgL?SX16-6QG&ORrxw)qM;CGAj(%r}USw6*}t2y54chJEiayXtM(2
zz_0PO?~KLHDa*$T$4cL&pO7`}+Tv0ieH+bhRi`y>7xw(ptCQ^c+1<=dfos=%eA`ob
z8PI(UQLPI9m(o&&^3FzwCfQM(IHc)UgD^6ssK*P&J_K^)Ep(o5h>6q%ZzXa8$s>kv
zpz#V7sAf=(2oggHLh;7%DuARU8Lp4qx{U-Mx2M7j`h{8dh4(TDx#Q{i3t?GtI>&=$
zu~1{j6~aeM??2$>Nk{wHa5$QBv-y-_yv3Jxw411q?#Rddkt%rWHIibzA^~A@Pmu!T
z9l^?!TN5g2g0Lu;>ya=J7M!@##_+SSev+Fu9`Nr{>TRnc*Sjfu8p8lm?(sf-!?it`
z*Zze*S&&;h?oFG%Bj8lwuMl%BbGI{{Sq(FF)a#B0<igktBBeftY!&ZGCP2Nfr;K~k
ztcsJWt63gsX+ZbJlhUsD@@KERIy!K@13{m08!vy%ox^53PH~g2sAmET-r<Q8vc0i{
zkFtN3Lm|E3Guw@_%w5q7t?*tMR?k%DyJC0xUS%~m#)Ad1+N~QE9?MpUTW3pQb6<Wo
zL(>V+!#|6~O`vvz@I|}_J?gF#Hm%^%c4Obf45u?U33<{_5m6f;4RHd`rTit{tR2@n
zl`DLGam@`ixWt<1P5dg~0Ja#h08NZ^4!<j(tXQ3@v+;<Zy1Y|t_$K=NCTcCJC-u`7
zYksXDxyL;J-Dc)X{&xAAml~9^Q8&Ifg|mCk<9s8Ss2G(5=$RaYwP^;DYx~Wr-x+ip
ztY4qcuT4P@y{d}G=i5H*=)gOP`aO<9g~+6}0T)^a=bH+vD6mjwEMtH=(!OKLyT-#p
z8{<jnxR>*<>Q2V=r$9twiR&t<A8)iQFxRuZwBh`bN_kkRaP9P(A+8Aa$<nj@_)Vit
z^x*;<Wo3JV$G4vI)9X!THH-7z$7#ZLPA*|X>(5s5(0z@YXRnQP-pe;Qo0*+mtsigV
zas~>1c5>S?PXa}FkTTZF1C#Rif-Sez?PLE5jvL{5V})x?rr=(M=3S#%XTBalZ;zPt
z+*6!G>%zD$e0sV@;F|k2B<Z~VYNhIt=a);hlR3Z^CaiSrZ#~YxRGCM21q9=OJUX&9
z@(haSPee4ID%9@?9^Yj1^}a6<cTmPMHTXI9v_!^vbom0CH!1`XgebC4sG`UHkQoQr
z*|I#~9o;g0<55eQOsO5J<+6SbkMk&Qw+giDXOYHvB$1G~;1v~r98jumP4Ln_IF&Q+
z+J^?m(sDh@B;QBgc-zB|fCw-PgH#34#xj;^$=rMx!nqp?a+9YB{6G$Zxdwjx@7iVd
zC#u@el)$Xjtbt<(qw;!zZs48ZGz`#W0!eUlns&KGe^Fx5IbtRezA5FC`0jqA*Io_K
zv{+$vd=$qhz!t}1_X9vuO450@J`{LVUcfqc<6Jp=vS`H=N9!y!bb^5rC6DB4d?xpG
z67&^hx>>2gXx|Ccr@Ch$1KqfdP2Vy~`4?0XW2DkICJGEk0w^cb^5f!!T{|gVF0bWt
zo@5IwW&W%lbxw+@HSr1$pWD58xxE{HcL%AU{njWyYD~l?T@7ID!2%k2!b!F$#tFEq
z0}H^X*Q_!#v6?E5ay!6^nFAdsDkMFtS(GXi`*iC(Lj$}Rbte<O;Zw(X4{IaB6&nT4
zNqlolL&@#k%lgaJ1-)t%UeMAQg@{%t1%x%PwS)6@HAELT#u~J~ERDVfNQcn)t4mQ!
z+1=cDi3vxmuMIlvdmp0R=4=kPfC)#oZtv0P7wFck&78ud#Yepq8{QLXi>Xqxm#2*w
zHzBFooK9m8El6}}Th_D#WvmjbgFH&ySEm9Wr`^x{A_XXCwn3M4dUeN}O(f2X4(%~D
zLpbg8MhesK6<V(@mTr)0mKn8PcMPW?o4sqJOF9@8U~<361j;Bl;kH#=)b5LHZ9$uy
zFkB;N^RFA#POuFw`wq$Ug~BkQzHj|7ZgYm`N!PQ)%)~c#>z?J@rekeXeU-DO(<4!q
z!uJ)Cubi3+(gYp%OX4P1x1)^8tIjVNa7MG=5Y=VuvJ9RZKYdzC-A|FmU&!`fYG+%N
zv8VsM2VMrmht0p{8eMQ^mSo}=?zMEXoKe{l5~G)$m5<+<^;piZmiRh$;frUI_e?Av
zzlkE!v;2IfTMx)2QOjB$gOB~5l(#$lYEI?t6J>SehuAX(ocyF$MfN^|$u#U=XcDL!
zD>XesYeVV~Bk0PmI-0^nIYJT>=BIO`c<~F9c!gS8qM?@e=fhS40?06>$mlpOBFDkI
z*k4IRsMin1&hi?L&V~fs+=evW0q1<hgMtQoRMjWXEk+@yUj$qB?lY!}9{LcN`V_)~
z?>K?mx)P~A&s?BC2MdFrIhE&`MhX!&aIG@2*zs8aH(TNH*WkjYC=26Bh5lKirQGle
z3rJ0gBp_vJtf9fvi1-a_ah$JMlo=IHsBoo@%((CZ<1<Uh-UzwSM@C)y1%L=TH@-U+
za^uP2)*55EpWBwq>ruK8$13Q$)~CE1JhFk3>^QHsfZCoNplb}w7hXJ{oS9#p+Gc}P
zmJzg-7!{fW)>SH1%&S+xIN`^X=<jiY-4T%erkXR~Odt%Lgj$8Pi^GB$^Jo?TSXk?&
z2M9@#B?7RE5vQ}?@Rk}LIV+iCf|KPaQ~4oR>`z+77w6`YEI`)lRbOduZFDN0{M9ZJ
z$z@_LRE?RwkMWqqeA;oEA?vDJR&NTszxW3NqpqCWj*mk~r0FwMHuVxFB<z*zxiwU+
zMyaoPKyG-dnMe<5<M)suh1*zJ3bZ)*C8AN{>9wn7Q0Eizbm|S~1t(~)fVH;4^~OA;
zteWg7d<zKfT7Y^@IabffXuk=SR5oaOt3lzLV^`niyz7fA%|yuZ#^_F2a+LyA8w=+u
zo*&ua`*oe?GB_kIaJ26?2By{h%Zu}`t@{r!77z(e_dUvVzp+ImUL*i0d7M5LKiKN*
z$lT$0M`!FE*}2ft)qJuuCLUolr{$0|52JPlYPO4S3Kp#k&50R$JqzLkAs%^Iyl{Mv
zeTjkALsJ%^vSrali^+UHoe4mgs|*;R@X*b9e4qT5T9vqhz#j^8utk){LA@TTU<g2<
z5oQbJ9nf6Lb<)WBVG{N#K(N3Me4eemTi^%jONR?$OFIQBLu*BWfthg_$g{8k_SUfX
zhm0G1PsSgYC$(zs7J``3V>jWbegVhe63>M7+~KS<-;Ci-=Hq*DsoRNPzh=^Z<e?QL
z^fvgZ$YXU<`G5>)pzE9C+7(E7FrK3|Kms}G`PcF?taWuZ2hEf>2MYW~aiH#J?^f9s
z_D`bJU3a>8Z^_u3b9MxqBehuo8N0pV>U=sct}Aba?;kG%dxGD(vsS30?rd%KA?2lJ
z#VU*XOH@1Kovk+~`)D@P-)kJy%EyiFxD9DQAw)_l?BYKPP4ZY73+taZ)su${IwhP;
z*EUj2T>db<=`PaS$_!wYpR6Rt)75XW2#xsK2#3s`o_Iccl(fr#zGK=SwFf4)^WY#$
ziaHghcdOrc`APNhOSyFFR?4IueRrG5aZMg`vnq?taRbg&R)<?!Y5vP?&8qFv6r74#
z334b)H{Ev_ta(?Yl1Rk$r<c|>N}Rmc<j8*X!_?Rh9`J4%=TQOmXg<&8)$Jmg`HWjw
z;whU4c#Jva1sCJ@gQKgu{wEwJ$)bm&%wMNlAbO7npGReY&nR-o_2b@j_jQh&cbZHN
z(eqr-$}|nzdQ}$VfUlEdB-5#L)V>MNUSd^)>rYtrFW@{TwF>9<(O2vjc3Pi(<DpXl
zr|>B3ndM)m?l&f<lgfG`tizf~X~ajC1t+SzSeWB(^5(mk|AkdY?#q3bdyIq2nD#F(
zs!yg(ag;>~QvYT~wIqkTbx|(qW*-<rMd6ML*dK4o5jgIVEgnxAGNxXPD0!ZDm0T=~
zsbG4qhy?(4mWw14d0B86ZC<7etecfW80%>NqANfz=VHD4DP3{ifDySPVpAygTNF_Z
zz!?qjl)64!fKHT#!V$FI%C?6-exFnE_<32aC`{3=K$i9Ri*83J72z{1qK`se=oTwt
z?Uecq0kCHkkZHik%Cd_L3IQJ<BL@ST_C8NQO!w-i;MB`t0#kp<(FCX4B1Cm9(KYu^
zWAe)Nx3;Z3z^?uR8!0H~BX^Gt`IY!Hi`7uk!~`omI&C%STTibpz=^47Cf{NR-*obu
zs>>rlyB#~<6Z=%lU>F(0YX}+@s;&#z>>uEcPqY=+$z=DtoU0^pnJiA3W{GpVrYMJi
zg^HYAt~R&Q8iRCcRp<P%n?TH5yQLYUxQZ$t0xQ@fPS${#TX556tr{ct=4wE0zu;RZ
z^U*TbkB3`lCgJ7T-Dkl@+g;gpI{HG>R;lKTYc&fHJVN=3$q?g7D?>E~k8E<!h5zQ;
z6rp-IIx^y&@n<LW2a3z2^9}8Lj(hhOc|7PY7Q;#EjSraAwILKq#$IN9|Jw>{lgQx&
zo&3``#S79pe+`!hN0F@if)gEnuh5cUg(X|T!>RGlFaw}@d9nwk(x$=GNuq>ilEQWZ
z7Om?a76NR~cn47^t`X$GTP@{kEZaa@z46h5Q95bY_(uf&_k2I`w@~$LJqieiw@^(;
z)+*^3O9}(TFfxvw+y3fUo;3wSXNjF3m@Ymps0Q@U0HOB8>1gMV6AG2UKVP9=t_hO>
z5&%Hn2fDGzvctedbVu{CNo$z9#TU3ym-lv;Qrx5Wdph~}y!I7Y+dp=YX>T@V^bE5Y
z>(yZsU5=LXE-AKQHf6O=ut&_zh1oc+T<0oz2B`D@bx5Le|ISVM5jd(z<Kd?l3~p{%
zr<V&TUk=Gz^dm%svGz$GHdC@9!jUEuVu<SOO~cqYx<1-`_nMI22cTr@gQkz78ikZK
z114Sg(0uL+gQjOOa&Yv~kjqW^0+0xZ$6x@zIn)$VKO?>ve4lTGN}()A2{8{f8-mP~
z<|0VdV%Wq}i{KNCPG=A;x8I^q^a_S7@Iis&=!rL=5MqlyNT<aCghNr3iePj0A%_Lf
zc%krlDN|Y?|JJR`EdSPvEO!1Qij&D(0og7K)Aemimy_|G)MvEDcrO1BOA6@O@0_T+
znkP7+q_;Y8K3-pAz>~Jh5&dYVw7AN0qANJ|(IlD3ceProQ%wwCh3S*Za0P)k3q#`W
zhwbC)<xmBw*(oQh@nh%b=*q>1T3Uslii>UEWaz+G&n+rgpxgg@--?Eb-$uy%vAY6X
zU;qDQcNLJ=#6PDsA)N_uc^lx9QMlB)p87JC>LNo-;lwC6j+T334-4l=QFStRW76jJ
z*aODX>0AGLYYQBDzbpT_i}M73Hz_dzWhT1ZxBJ-M7OA(u&c5)^SMi@avF3v)kSA!R
zRb3&CeJD43(InM4S*e#B+jSeD%46IUPwRK+&+d}TC5RaKZ03+i(f1A9=ZsWXVkV6k
zPL&83abH7ZP>rlw$fEj87yKA$=zcIp&ztwy7U7#E7!Ke(i<U~%&_V-U9j9ZWdmJVl
z>6!7!j~PV8G6T_HfETZi=T_@fPEkxgA=U*rq~_)nfQv4yIUP^0a##(wjvlO{3v*eP
z8%oFsFSa_RQ+~i6OeAjqZH<MjcVMQozE1$5#){JJ`o|iJgN9;LRjByQ{`v7rF1aSB
zmntEm#tMp3I!|6NCxocDcc@!Ei9XD6V$ukU%V7&o-o=+mhM2)?!n7r@vWZ_s1S=}!
zgHU*DA32c4Q7gLM8S<9#IUCB#D+RZ$mWAfz9H#*N2U+klnvrq0&LD~hO4@o2M>NC>
z-56X}?;9_=bqR9&u$zl`eWhPKRD|B4*{K$*%ztd!eCY##Gz>uS)2Y9e&@Dq1)k%i~
z44pxYU;m4c>Ys7ytH<anpF}I@#=l31He+yQNhDDup4|s@hr_h6h5NP@AXWr&L`i?L
zww4ya%IXUdAFAGHpzHku+OIG}$fkoP8aG|RPjZoO7>hz3IW-4`3=VDRLJqA_a_%aE
zm*ZZKa>;vCMX(4W!(7HfQz-Qudx+#J16yT3H&U@gCx^cI$H{7nO9*hXA_JVP!pw_F
zyFnn!5vuSSAElJ^mnccFtXJgqW1nzcJ3vuy;%TQa9bkFFJiS#Wt%g~j?1yRB@g_Br
zUOWTvH;*2}n@{=;7dx{eyPS6X`fu!z!Xh50>fTv53Xb=v!~Ps(IUFF=2%wC82Cj`b
zh$uspEIB~-5C&WSS<nA*hyE87@(S-UJ5p~3VjM?CFNVT0-t0g?6hbb7@CQkr)kWDK
zm_KmI1QCyVNJ5D4KXoRPu9p$y&a4|nf#DdvesVCV#-a*H#9+Pk4Ia@4N;2+fb-r~~
zz5VrHvZdf+Zmg)XJNGFrz<uh7>mT>2!l!f^+JlRo%}7;m+z5kBIYXCvIl>M12u#nO
zd>v!*Mx?@9$G4YdoNvMNitx+1C@6GWZb9|^NU686{wSmo7NDQ0u3#Bj=D-0E-ojmq
z?SzVR!RUdT%%EeEs3>j0ru^O=b?hA>J5p%p|Mwf~A+c|c+>dugru@FrGl=^rzwoJg
zS}}bveRTQV5)GQ!>4(|d)ZvJ2T@p~@m4~>#9_Ye><_i!uw^3E-#M_tViy=VXQ8B<E
z8ZK-&BR+d61NkL`Ey>NDqwd+-db~t__6#!qv1BOoOAJmZQYGGsfl_ETgb0Hdp`&e?
z=+lOV&x67;_`@tBzf~AuvlyDKF6ik|8v?Hc!#jJWBEUo_WQO+Z23Zm+0lF3g8m_?R
z4$ww-Yt70MCT0QyOp(wN;6wlpP<BWPqM3nru)cB&(O}%ppItQi>F&hkOi>+6#z@|t
zIDCMbM`ye9*YEr(=kCtj50Jko;saCZdylCGqMd88k`W}6eWY^V8J(l=A({rBVNAY>
zEu^9lB1bZhkP}3m9PDh<5X6MzjgFIYw@c#$wM_(#CUeZ7F1EV9!to&P&e4EZ;<U22
z>p85X%S6i=j+(HBOOY2|=|d~?OtX=*5D2rd+sTHTsx-uj?`)Z{@LWw>UiK&Rw!W1J
zSkGiPDXKAF%1T^cOw^Y5|9;ldMyzMgL`dcUB$j(|EJz)ugW1aM(`j`rR6R!r$<w|G
zUl%|%*I@x;Q=o;1uO8dqd44}U|9G_WN6;)<MFr&B0!u;2D<t~}ufyn>Y&1|B0o1_%
zd?Ws>&K?@otG6hLe#j_>5=;2tnbYxlZ+z8whsHEML4>pW=-&}Zc}U&tNQ`I+uMyo=
zGKiGuv4-2FgHC<G!l9Ahglq6UhzW8pdRkR~fR%O6GYy5#sPR7RMS^OUvp9HKAKSdj
zPR4?FKrhbNrK5=`3?(fFd6m>w&^{8WSBkC-=EP_v1-pAxnJtnaM>yh1RqAg|EV^tR
z^h2S1LgLBrjRe|>7-gqHWg-#(kf&sBtL4gR<OKmB1t)Gb&*PT0>G-U;mg6e$Xe-)f
za7OV0#1<)hk?^i!vD1^$13U@e7=<VP%M0LNTSx^LhzJCd0AN3dD^w_b<r$(-0DbYn
zgscDm^|R`Yf^-i^%D@L3iYHN$o}NiX`?4XhA&8;PBV7A#;Q^i=RA3e?<#1&F6?`E1
zNr;BbaATu2ib{l~1fsJ!aIrYG9<6@7qNphN;F3el(;#25=-b1I0Rlw`C50W=HY1i?
z&!d3G#gzc7GXKR8<|!W@X7&V&BK{sj@3tx8&A_X8+Lj2pB$MGYrbhcf#|QEl9kuAe
zxiWhu^1^LM@f<V)o|fYw!5T+XY=$R4mJ*0*?Y!C!T64-^-Rf>`qf}{IQ@DHsoA}`R
zD}1o%>+g4bR;D?Nb>=_#voA~nNF4)iWY2iFoG$8!PNUBV02+kz{1i|g{O85|&;9%h
zBA)&Oohtx^yzL_oGuO^>vrIZiPS9f{HUuFf!XS8SM&R0kI^|>_JuW#C>3NG#peJ`M
z^gs!dT1^<3qNJ=&p*4_p2Y3|V5+P9&D)q-}pS8hpx7L?sc6iYoCie$5o7o>ZYN{6$
zZTX;tFn4A2F0&=W5A;JRUJYQN^Az#bCvF`f1#vk0wh)Ucb~7fv9ZuX+_1tjOk8kHo
z02V=q4mh0di{xIfK>92P5Qo;|O5JUR`Y~C`)-EElVr$cFI7*_Acx1>eM3>XS_JwW!
z580&P-ra+xQ1xR&w!r81!O8@d=iZ)R%b(x*kCpKiLGVSQ9RTXIjsfEUR5n1P`QQuz
zz5WH$QD7ij8Lr_z@t5|vWOG|I$tJOOZ4ay!>3|47j(Qj~n}fMo{W?j=JeB9GY(6eE
zp;2{;jeJ>U95;`SUJVC3+7C}gZ=={-Z>?um+H+ruFSb5niPzTT!W>S91FuwO2oam@
zKzE!%=Gv|KjJC}AN+I8MZQfB2e%sm5!95Bunz>(v8V8vsEM+>!%iYOk7wkRx^diO7
zx&h{8ii?xEjy;2gUc@=PT<A;O&YRb+(6rpmd5QG$;~7j)9F4f>-J>o^)EoErqkbF9
z3q0x%fLx6m9Haz!hW*DVNWF+?<Pk;h2GYfN?3X*{-_#`C>K*y<HQvuU6>_GmJ$Z`)
zL?R|3>87yV&PxA%5ubbz^##nFFfAxare+5IGHk&G`LuHf>G=qq7=k7CWLn$@wE0%#
z*g$Osv=mh;pg9R`5&2<_ec}#ZJP5cR){P4~Zxg!SQnnjmRJ-uFM&q$Bh-5AS;;3m4
zy8FsSR<eYvRbAv4VV}<cyOxiLFFkJUTI9a-ZIv~O(o3hWwgX`ui>iLY42=;)O5acV
zv@H;9bt-vNom$*6h9#r@Fr58)i|fF|%5hHX1_sLY?D8H76*wLbp%<FBb}jiJAV?H?
zoq5`lfB4{@2;jykuEd3WcRdYiKhw+mbKN`@LphoQun0LkBSjt7+eIuB@XgVVpIJvA
zaX|t0+!C8ztr+9Lmse+h(ev+C?Xf1xeSn$=Fj&Qb8~o0^1nWP|OE^sAaZRXF4cny!
zS@dqxz&UQyz|Z$JZ_~h`7e}||CFg*7iRrC*$yi3!K%p^l&ZQ`u_A<epdCBBtrb{pL
z=o(|8%AUB1eW@)=qAovALD)xqe!>XY3M0r(@?^^rUz8s(FDXQ$LuZ&$Pma?@c&RvC
zTWLMHj=wUX;ZCD=923P)(+OBUmxa=I^=b>RKU5UPH&RK{Wx*oI@Twos43ObHPT&pm
zcKCl;M4Iy>0nguty361-zy?6a1o8a`m$5bYc4Ijr^(Ol;3L>CTB81hn!KkBu;QMS+
zdApsV&?Ncm8#JQ{3GMHp@2QgWOM0#Ei<Pzl^Iqs};I=x29r>QR?n)Dy-s(t(%tNSf
zVeb@R!s42jQD&c+Md|rnaqDbo(N0?=zJLL#(0p+6F=0;|{ZkcJ656N9pU_o+(t(If
zxMzvz!-29NW^VEkhlYOQeLmBcfo@0{Oj&M;T^?SVmD^wtuN=Z)FGrkn_zlO<pBRJR
zJ|hacI${#q(I;5auYFA2mxl`2ORn{$fKu7@#ffVlG9%rAwFdPc8m#|nAi=-4)o4(`
zS^?JWwz7K<#BjZG(344DrT&GInD}0ydkO$hy2RU#|5OEcHIwK*)hi#A#6d(MaJm>^
zs1`&#vj%J<MIVMmEH^u}U%Ue?_-{iyWIPa8{IsYj`0zXoBXQkaFzh@>Gl`yy@!Pl5
z1ZENP(Jd2>iygnq0!r+}s5FM_09cT~Pi=9}ZB+{;A&<O#SF_{&{8IFe`)JXNd96bs
z@ShrP;+6PY9C_=57*srM#1uT-R=j;odTv0<hln>j{RfDy&p`~BND_!x8+5Bg-B5kL
z6QkF#9CA<Kywm~`VF09{LpCd<S6d$25NkFMXxdZ`N<-j)L1m9l@I%DA(-)~Ls#pEz
z^r<laE(pv&jz*~>-4+Bc!0CICUQ<Z>3K`!PNdfzN-aE@q46b(wOXh!8_wL$C54>wo
z@DXbxGkP`9GdgH(kxUmk-+lu9AO|CSNGV(0LnagoH^zsGa!hnWB0_i_CtAhcV$F*l
zZ?wv#t1E>0Jt<byPS)egZ`N=&$S=H@SY{Tn<Qns1<KS8m4}dxz*<G_Mj7wo<5{IeM
zasb7XR3Sh_yk{mc#dTiw%5uuzC$k-=21S4k0bc<DMM<|F4~CMWYdW!XvD}`fvC+(q
z2$b{P0ja2w_ox9yem-t9-K`?u!gEB4<$(66oRcxlv4Bay!JcO1m5y7`m;!}|**21I
z4&d{IqGWe+x33nzdo}#^$@6so*;gTjCWX9r`^p)JdE{_V1V~pjxBJRPRc=0t!1QH%
zYt?`5D-P=0l2ZcNIZHf=h*Z$0CgUe2o*jx83bvSxU!jjkGQz4YQeX^7h#%uGu{H@H
z*5-vCyaJ_4_sIT$NP8XVN2N0oCT*SN2|JJWuX+E<{k|LlJp+~BQ-(28RN<{I>z8n6
z4fA3ZGCX-zwQd<LN<|d*agc6kxB^3x-}ecH(0Menw+{MIA|}?sTXo9SK-e%rF8NTq
zUHPEl_(!Z}qmru3+>)^j@&ZMuwtjqDf$;TmKcqQ)z;LZK-)Sf+inv&A(-hCq!XLP{
z4rF-m(FJSVO|_*w@I5#FeCk(LKyczMNQwPgQAHNzS%UlnAd3Da(%KH-izM*_gNu9X
zLu>`0T#iN#Px|+VkldF>EFA%W2#_5sBnY6ssi?n2x->LFzeT#oi;r6DLh)N8)3jTD
zJb&9qTxXsx)2?k78tGD^aXg}qv#844CxNp%QKj2zC^2AYLJ?uIr#{S|K#)uLDIeEx
z5x><ebyRJEF{l98Qy(joB$Lu*3=qByOP5-3b|F5Fkiwf>R)h%kF^QzmHEUbE4P|qv
z7mW_#`@Acs5c#IG^}2xV87AMrf=p-|%<l>P4|I?vSMR<O#kQP#WGeqepcY4^JYX>z
zmnKRO@z~&Z+p;fpl>K+xLjAjK;dvpXM2~;lw)g{zCXD1d0VHlLL_if?imBP>47`Cj
zyw(5<`riuCV3gZpi=6T;83ifC_QP#_>m?B1+JY>~h=U;11ZXdv9ssu$4RKTm;bvJg
zMMNscgG)nNAXOz*_gX)GYTvNFE5lEiD>rvR!rGo)X3=A!k`-2=_K)_`J)05GUiOeE
zSy>i9t8TTIoY2Y^vOLGVCIrGSAM@H<c>AE(9XTJDnx$8u#E4l$zqOZM|I%K@TRaur
zz&|d6ke4$Dus+ydFJ(^i@Ov^y0O?>02eg;O-ReM@Lb>)!>NQ(sHLxw_y#AjyzPz?o
zu#qZM1uc3+2V?naS5EJImr{3rq<g`@?D_st(^KDf06c}^17rFFkl)o1zS*I_%dBb8
zBT_~n4E2>?e3OtjX)@YTs8q68zzd)Un?dxTPh`e*JLil|(S9%*Fr3b7AFLq}DrxaW
zMv<0PZ@{A7?z<x_C@beK3N`*N3Y8hxA;_|4sccIXwuGemvXuR?w?y6~*SVQxAS#LE
zTYr6;A<mx+zhVg7pO!Tq9lR@XfvbRoi2Mn3R7B<kHVwid=ucrwo+u79<xGH$y@|YH
z^1&zlXX8OU6OZH|AiEJFzn9#Lo{w>~t_G$)5tzCGdTqoYn%{jNzMiCB?O#_r+3%}e
z`$e3VwX4OP!6PlN^R0IOjbX2{$qM6D{uoZ$5;MT{+#p8PD!Wa=vg=FQ_F9n0)Cg15
zmiJBfO@}Ht^WBeVpcm2sZ0Ht=>XumBHS1VT6_MJ!P7*!O7q=hbIs@09=<Pe<s~b;W
z`jm0tDdH=k>_7jByZ*a08vho_qSn7$5>I+Wy8KY`6uyA>1LJK8+e&=%BRmLoYIgvo
z5R!=xM;IIC#*)V}zPH$yg!GquSQB!G>MX4mLL<$wB8<;xP>Y$d2!1MRqj^*kx*aTE
zW@UZyukje{lqNSUmhsXgY6xL@QvEF`IzS3WW2UU>V%e-Gnqkt{_Qj+q?6YE-e*F5<
z%zAVP@rx%uxx8(FR1@j73=z=0Mv!DNLdLRZXsXlIuh`}N`S{B6V;u`1VdTqgITtO1
zbO*pFFCz}LpDa4eaEH-^r8D>aM*5P+`_=z~>_<cHFdBvsi(<{Hr^5|d#TD1m)lK)E
zUY9mf!t=2&jcxhaYzDqd$xU0g&tg9SH}s^5^j|byB=;tB2+dda^l>K0m%j(PPdI&q
zT)Hbap6O1Wi!HaynAyMt&qRgAXS25cEUSqmp+f4yTm)b2uk*dmyn7QUp+9ZvjgV8l
zY!eUxNAFhm{P%nd4+^JCBd@A|sk@)|$w~C9JG=`4DxS7BmOpp&?~14F4h&&M{K}EB
zkIuMrPV2#*5rj<Dhrko5ZIrtfBIrT#?*XNr+KT6$#TxJd0%q^Yz;{@RVv@wGXIWX0
zacNzYG~P5=hD>RsmLU4ZV18?jik#iqV@AG8uU2+w`70y4k5Yhrh*{A`=El{dvJO46
zv3pcd#{-k_MoUs=G_TNm;TOE%<3Z{k5=<j>_U{N90ge-+d+MHHssx~73^G2vqk~cM
z6#p{YWv%^hKtQS46mA2yxWzWa_Wt9CHFsmNV7U0j!Jku~u8B9`p7joJZF2wksOV~H
zQ!BtpfP^m5BOeeANp+7}PK)fOf>Jqej>peW#w_}$kIa-Ej=EuV0K}#R)Qzorh~8Yj
zIbs!~R+q{<F9`*`PaNIr(dLzFRM3zvlRcEE2FRnnke-j`I<LKf<Ik2;14#JXrpHmQ
z;`9OL&F+@xAU=<Y^!ZIA&}q9stDrOg(fQ%sVJyvu@dGaY1V%VJ`-kxivzsz^BBMy<
z_-1a=k*sDMbp#5bCJhI-m1YJcjE2y2dnN8B_8>rGR9Vf-*nrNrM=O2Y%sxxT0W50G
z;W<bX#Br9F(^e$K@39vL-jsykI*dJ%yB4gz+Tor1waXv3kWM~@N8}^5!lseb1jz|&
z+egyb?|L*dOzgP4NW&Sv)QN%k$t4j%Sz$%pbLD;sUpjMYNPdU}>p_TyglAg;^hyat
zjZ<fC&T`Zx<w+{8h~=p*68>9bWCX_tBYvbI`dxPFz0hm;b1kS~0(Vno5rOH6z<S<m
z#ZdeXfP6brpgMgbTq>6sAS3K~(&@giuRvh6B3d-**iG@;9;aG8)mwfeZbKOOPA>1h
z6V%?Zi2==$3e^XD%|td+Bm48Jg5O95fo~a`DmD^`NCS#i*avB#&ba64>S6aEd;Z4*
zv0MBG{;q$00HAmQNs8yKkaU><cEUJ)UE;qYMGY6}BOF_V@Y+S4S3?&_BA81c;o8A*
zEP0=a?HB@;JuT7sQR>GM9}~hVMeG;5FDe;|^(@9NKFHCO-|A?_0(2~b+soPHm21K!
zN2vk$`QG4fAx%D)?VbdbTtB{Lu;Ls}?$i2F6k<vd+L=Lmq<Nbt%JDl<)Na@-FPk?-
zEz7BOlM3YQkq-&W@yu&5s4_7iE_AkJgq#4nJ_}T&UWse;TpvmN-*`?STPE9IcF_gc
z09K(1gdkpmJmiBn0K4n0!hVsm6N=}h#=YvYN5J|QMQ*$$vnkVAylK3-GKSWlMfWDw
ztSc$>Y89DJ?^eoa=6dsfT#f4#$vu4MbPJII(Ox+&x`;r6tNnNv(81$q^?zT+1Mr{E
z&10N@%ADVtx{r4aC=MoX7)BV|qT@U$9Kp~IX?+3z$Gx{$Yrd2?KW7HM;5Pf~;aw60
z{9qIflnVU$i7m6tJ|h>OXbp3i3+<Tt;IVYWS;$(t6PoJhkP9G7CY=;O(sUXggUTY_
zujlv21I0aOtcZ{sH;g@eGr(-rUowRGsj%)BLa_BlK0c&M-%2{E8DfX^glzQQYn=0?
z00kmh?neZd4hsXcEx*VBamWlozqVasp%;7h@lv9;h6mpHYfR88?9gcoE$Ku`Xq8LS
z*<Li=Yze?yWB(m*{ZEY5&=STLwY~W>ZNh(l>4ELnc~>@z276%UDX>!u<+h#@47zge
zAi}_4Y<Nj#S^2R_&*Vyl8+K`9H6XM5<*ya~uKvSbt&qq2Qt7qVrIqg$Tjhb#M>pn=
zE;y>#^Bp~j=8Y_8+<Z*=W=Y4ED^)xgwPrl;dd7K<18n@zKz<G%HSAx&p6m2h^DZVD
z)xMo0C$l|edX|0lL2r!&@?P^n`Rn1BV(_e+EG$3Dt6Xobw`wI0`fk&@Dyz7$mz8G;
zFt*$N@qW8b3`A^H9o%uBd8`75%*JZ_`D7bx%*K=!)!Rry#A`Z7ktt=ldd9w6t3p*>
zQCacI^}`i^+HT#;xEvOBT#MyR9M0kdJHIxCD7Vl^9UPHap6O#~rwja4V*6+JO9KIi
zDK;KPnl<`9?q4!elnWz8ssdV8+t|gfNhdtseU?drJMLtMql`QjurIIi^QKGfT#654
zz-OE-8;#Mh3hi&KPGnp!Ef3fFz`Bard?lp@I>6H>3(IYPSfycTWl^_4SUuELIJuP{
zWz}>#Khe=Y$IK#?X)$25viFU)q3u}`$4c(r?&<TwDsc)e_rgYviqUEMCA~IC^+MYB
zcbwi#SI2QE0iZCg>gusgYfI*@+i!=PEo^5Z#7>m+fCCc)e_s^u#>`zd<pg6GelyH4
z{b8^6o*&PqjkSg4x9naVh~9<%x(N@E81wqj$E4YqC5Cka_xp0m5k9zo-qG5{f4Gud
zpiw<XF);!bdqkUTUAN7n<<QF=<#wBO-<!<GSzJ$t)~jw2^#$#wQzTrWZF_8fgy-Tg
zbz!=5Woz?v9=$Pc?zxxEO=^(_w1M05v!@~lW3uLR&hOO`)f58LZd$uWz@J+TGbnOV
z9-p-OsCOx}?-p{8O;7Oq&)YBuj8A;CX{^jh!z#tdYGCFraZY(uyf2Q#(Kvo~rItEk
zRr;CRH2Uf*2MeCx`FoYbJ%Lgn)s8O*_%R7T_fX#j5*7XeonBd@0~5K`0iSu`P2=n$
zLiBh%M1(|>w6}nlvZWt?wjj3@=5~T?wBt)NM3Ad}wp93yU0#wq{;fpd0w&^QC97rN
zt=6V3IFeFtPB4q{6D1IMnoD|g_L!)CF%L9t*!V5y?7;!9K2uJps))D9#1s0b#&#dc
zrygR4h*5t<h*C5YA{)#J43fHDM|q8f<$T_9e2?asn?Yu)uqqN4q=R?9z^pItl?cmH
z#%!hw(B(Q%^;{8$40KBiHxv)JHL>>=^16+^Xa*rEXnFf9i)M8Ep2;l}4aM#L`KY5G
zfO?MwkfeNK=B_3g&Ru=0NrKcHykLAydAa{>r3lMqi9U(TZV)>+hV*$^%6k#f)h~Lb
zZ+(SsB>Tx{%XvGEo^@6OPu9t(-urO^tFYcv)=~W}6X;|d+njEJLYI}|w_Y{PY@KHm
zM=jiAmD{M)^TJGJqF%$^r0+mNBP@6X_BK3UXB`!_Y)+$A)G|`Fcju<7#2aFqYS$O8
zQK|2_es7l=^U5l=%Az=BElXn}Xqzm~z2dIl*Vd%yc*c0WDP~at&K>IE*{OqW?>N*>
zB){Rb->Nbnuioj$;tG7;5uf3=T|JE3vp;AQfDGXP$40_W7YQsoWUit%NWO}gS@Fl}
zYF(dS<zA(h36H16L0K<cT~QKyPxyv-W>$VY7}rQjg*U%&uQS2Tf~-tvrD4yX>>Y<s
zUYTr>B@OL|%cPkkC{!!1(xT#&Uf;ZzFb_c^+qKIZ#Dj@G<V&Bn8q1R^oM0i5pmJHQ
z<wQ1HZhufsYvUn^r=cc)q^LK|jrW0nk`RqNDdLBHGBU=}G_KxP#Cn@<CC-h~R&e!p
zMsBWT@&uxNTV1`)R~r#GTD9CIKh{Pzytn1c1MewoX4N(aI>)9Q10FK+^~J-Zgx)Xl
z`C>&~defFI=SBP1&v>tTnRR?-7?^S2tsg5;J*gE<f1|l;XYH~*Z3MwHJ@d4kV(~0;
zrjltz%HQ3mB${q#6TvT(3%#+IRG&WX;Cyg(hA**IUE&RB<y@($e=H01W1}P`DbBSs
zgn^=L%8~oGthIKWZ1ZoNjX$0Wy-#!h7AZpLm7D4)adL&>UgMJ1_wKVnpulNE!<c>8
ziR}FjK_RH15I8Pez;XTUxXybWeiL@14!k-b4cx&|_}?{ek8y4>@?rXPo@bSO8FqSU
zsJK+tLqVJf?Fa%N!f+w5wEzwtfJpu+=$BufI?h2PS-qoO(a{;znH-Q)w-sSMU{-v-
z^tS5_$l;(+_)uff>6Zjcr0;TWeSxr>BY>qn)jOUjeEQ_|3JUcrOeGwArAj+Q35Wth
z2zzEob25iG7+%HaX5ebKXJU;+F4WLdMYMul@KEqvU)fXW2?p7|$A{&QVw9xEcw6XY
zBkhVn{aW1b7SxQZq_FKrnWXDc0AV{ns}^Tl{wC}!X#og~K4nzhj>!LSk7IF?+-UiF
zcJ!BLD2c|sT7ZAXR!UK5l3~Bl9OoL_>H8^9Z;OZE#Ev+kRF4U-s~Ct-cZt!n1fxTs
zY`^3#y-v-2lN9!z`R*VBg%w5i)R#~UuEtOb0|m|Z(fN`hSGkeCXP@iMm<_23Oua~j
zX4K5;Vq-Nk10)@K7ra?BWPU0vE|oMqr{|1@J&ZnVl4TSE&lI-wZdCD1b%7feGPt&4
zZt5m4jxXNxo0Psx;@+OQIl+DoJr8c$Is{yq0-*eyB!^i~P!bh&J-aCD7d~*wD3C-0
zbJ)sg&%PQCdXMO@<Ndm>Bd84%%<b8LfT1zjCBJI&?IB&sGZa#=ux9owKi~19?EExa
z_5QpUe&J_3`E-(jO||Lk(!E_P>Q2Ua*Qu8M6Uoyk`;od7`<mj09R|MIA%<q*b(7s_
z@rEGa^U4z&Dri<Su?P-|Wx8%1EIDie8b4oJ6B;D!u6vEwq~Plz^;kaZ&gJ8|o9Z~w
zx0)~d84PKnoHje;T86V>T}O?V-)Sw+Oh8|p!7%=%aK_@!C-fM)`O3ti5w6R%4weIC
z{q5#;7w11WWcAptPNm%|`n~P95^Z0DPfbMRF!<l!Yoca!iDAgbH<b=0)?Z6eKScM<
z-kx{*yOtl}x*G%O34~&Wu}%JBu(bjCBLC0`!(BW3A%_<rI&%-;D=2at^+@q5;T$dh
zOw;B%Ltm7B1!Uq%lr6HqZD(*NW)Rmskb@q-ZE2${jSNVgIs=A*3{^qE1hTbD(HS0O
zA{2O%%?;qKmRtBiSLKOM;3|Vp{BwXTX`4W7+M_81hql12mM5v$7Rm>a*g~S1FFaHg
z`M4QTMJ{-66+=}^SuR_oOMZ_TiL{6$4bKZq!UD&VQ&x5W#n6V?Z^aN8ni%SeSVjma
zhR*lfKYNS$XUW-7rQcDn!Z%k69sdUb^rim|GUU+p#Q=4CwVT$oeO*cvor7_Id0d)1
zm%=yW0H@+%1}*wS{CO|K9(tgd|I+((KM(UwT-a^@MCJsLLdWaG4#&nU!?0fm@FZ<n
zhiR51^f1e%D}Cnh(*?|pJ&8ML7Y3!gVFt%ZEVZ3oogB$jwE1<f56OoVW~G_6+I7O^
zdN5b}tJqQ-rAo{c;XSg3!>&#^iv*_8r`_+HWd(P?n=^kRc;~!hU%+iXCeBc|)Z=G4
zGVV9ThfH}say2Qofo)(-gX^}$M+QIX?#znknOJ4x%jMbQ&8B2DmbJIbtXG5?dtYb~
zV%pER!LB>=xz9us&({ohvmyEm9FNsO)e_(Id)2YtbXtB(D6W-8Qz$-6JK+lCLx=cm
z+^h@bEBHa@Hsb_qr4C{1iUQkQ(+>T6@d-~lZY{1OByjS@jjUgop^$cP3ew0TW%wz-
z@q+Kj6^`fS7uSwmjCqAnM9Dm0Q9x4o7O4Hp=nZGHVS^VbUrP>Zx!RKLv)6@A`~l+&
zFV*k?#|RS2pq6|Nf<};m2;{miwotWO4F{6<A(?Y~4dj5%wa>)+Fpb+P*mH2V6YsQ`
z+iJ9r|D&>-V=dMRkxk7HhS0N5`cF7~rMr~;L`s#?8f}<IifvNf3v|HHEa};1JiJ_<
zj^HUo$;dlRJiEL}iPpNgT8<Xo-><k)U#$H6W}(bHt5I9BAa8?l06lKLN!9^5)$PY8
z&kJk&uF>#-WFtH<Udh#p_cV=mv)*#*a61ZVJ=pe)KBC~vlgF;-bSzVWK`_=#s}KYp
z>;|k#qYeYlf3*txqVrCTVL`~J&r73*2`XLWhqEtjJ|CWy)3Vn0@^6>Ql;@;*6Tce=
zRmt^Ta81&~8)moHz3GN)7Owi~FP&raB>}I52NB=?&b9K*l)i1db&%q-B$m!Jgo}dK
z(s6$s?I9W1V+x}u-y;=&+R!t)q|RK)%TsM4Le3hj6pr_KIlWz^q(A=qS%EY0YEPB(
zs0o-hmbf+F>lOV8FQc#~t~(VG|F%p`w8Y`B&tpG8^<%X2-x}IlR5hbZhkTU~_%!_O
z&7Rh;A|cj`MnHIibjco^AxXOiDn&tA!R++=EiHD5Kq9#tJ|3-}8%6`3IuMD`CUczK
z9#@rRqCjc^#^NuI`d$D{7K>RmF1pxHj1C~CTmbs+`eadL?N;XBqoVpx4fcPAWO>5B
zM@94;#aO-!44v|6Jfph1mnxqlWCXSwcEviwj%plMCkojFjom>mF3;MjZ<G}2)s+sp
zfobLB7hn?LRV=V$UsN@1e?yo41cj7xiA#SyGwS}CY5gHzeN(T)wKW~Qb1a~LD#GhV
zp)LE$bimSS`Z~Jq!7zMsc>9>rL$T_JE)7U`NEU`&y3`^BGOG)nzIho=a(3a+iGi6f
zK}_zV_I}zN`g8+Xyt_D;GU9>W@_21Cs&iY4Vs;AxCE%(nd$5x1u2-o#{Z5v$qIt=>
z<m^(dbAv$U`VeQ@_HwIJZ01GdNk__7T$)!0ZhyjI9wX(se<&QLTz6R1%9ykK@Ufx&
z$)w42pTMs6rz#Kb8}GgN<klDuB!x}euC9`W%151hkjrT;+~WbRZHrOepVuBoH761-
zp<~aHb>)w-Vn+Z=(u2wFg^{E<H?zbOvK_cO+4X)B1|Vl+C2wIINKd4A!QOw$+fRko
zI^+^FYIPh=HaTu%a*9%u-Cb4RWSDeT+r1T_pElk7UYj(4ZmGV%fm(>fWNbusY;9<3
zQ#Kq3hy#{(>uTHxth*~Svu7&)Qk)MS<>T%msOD-9V0gKkFaW@#r@V{?)jh>CrK+EC
z9s9|*T;In0wbB~5AvM*Fd+*q!c{j)5a6xtzIL0_vlnukWuBCMP`~7GXHprJkNe?#1
z<{SkC7HZKR8u!mSIZ++J_4rMGEN!xS8Ho;Y-h2nlgZD;FIOM;KYHm)6?0A8dFV_&0
zNE+u4vbFl37oD41J>brrIBGcA@`!WXK}mdgAM|hi7VYgc7MpQkPhAlBK>uXZw|RQo
zQLdZqp&O7n0Aj2I+BmSDzMu#f{2|$?SjyQVBWSWrwj)fq+N_AzgLqhjtSLZZj0OaI
zjN^?mzZK9S<^GCt<x4KU91b02H<%oVrXO$5?Gj&vMY!zm8yE^1vczJWau)Qr-P6E|
z1`_Ng5b)V(f)`Qvy+{M}Uq7E?zFt<MILBiI_-{_cS;PDT4y2(lyxgkKX+-GGk{qG?
zhO-4w{t4pYy+}$CIQ}-lG(ct}LFd91@I^HZR=*o6wz^|`u6Sc1Z3`G%whL8PIq_(8
zCvw{u09l}Ak@I0?igQ;pfU<g4#8~NHI<>cD_yg{=w=uEhKn3XZe5PgW4cn#qcogH6
zk<DrM+}@1&rH<vOn$_p!-4pLg1~9RW`&Cs)aVe5w0r%xu{WT9O{Ly*&)7yjC035{Y
zCmQ5Py+xfO6VvZgv@<o@8_8|9t0$fhDiU8bV7o0&JrI8V@PV)K)N;_+)i0~c%1V)~
zCd%r4@Fm^hZX>+q`U<x}PK)lw&B^qok^A{I<;ex@8}R`e$kzIvoV5b@qkh|x67LGN
z9*o=4^2X!;F!!F}aIf9_HzGpx5=0w9NDv}=8-fsoA&3x+lISIB)Dh9^1krmWdhfk=
z(R=T`ml<_<zLLG~d++=B9sk$QyW}`{VdlElTGv|V`B}#^$7urT3a-b8OAe>4ftGf*
z;qh@R^(^jW9#|N^z!&ci&TDyBr+f)PmQmUc8g*5JN3Iw>a~vo*uW6xXn9=U9jnS>9
zF4EFSCXZF$6psd*wVx5QMg$(rU)dd{27?}xy7Tur_$FGKS=E{?@{5V|rmL4r;_JVu
z`*?Rq&%?O6lka7|rSHVFhO2LUL^L3=;dA2<luIkQ9tvF>*Nj6wj)71f*b8)~y)$LA
z0EXV|tOtzo(u8NzmV4*rBp*?zZ8P&g(W}IDC%kcQmnTBn><+yw{jS3MqSb3(!p2xb
z22Q`+xtQq69etS=FVg&2>n9GTm>;YoexIt+u09ypDCl^&^pQ$AT<WBa@zcAx3#+a|
zer$MJ&87bf7Tg)fOFBNi;ZK6#10Vu(QGx#{(n~)5J#5lL9~emS7jH(va10zVr}vCr
zaW3LuQaYj^NqAE1*Z03)lDy||NxaeN0gd++#HY3nDh72G34v=`5itfVEQrk1^TP;j
zL}zdY`Bj%EfwEg$%TR%`%votjvYgZXB6(jzd^auV**;q?_*@QGmO}mQd%P;Nly<DA
z81#6gQ0LapJMNeJ>gIqV|8q|(2ALP7zfDvb#R~HqikDj|@up!Avq8-cSw_IUg^0-p
z8vy1lniLk_%|UMX4&eLZTM6DuRdK5oAFysSo$5^x7JH`G<n`3<@tZ&P@mA;CZZm)~
zQ+T^7g7#CJ-bPfxnc1n~e!0Vrclk=T+h2lUCConqcsZS%E+S(`sc^a;9_uDTq~*1G
z&WdTS#TM;V7!#fFxy!{e>Z;T6#Er153Kz7nKheH7*+JYaM-KJjxw!+mH_2##!dIO8
zu-T|-<Jn2nqfx`98FQ2<ME+Q%-Q8$!4vKt?;VqSXRx-yux^{jfw{5i6tXMh&?*zYD
zUbAN0IZrLVCbu>p5HeU3+c%{;I^I$DA~iV8V|Dc}_-dDYuxVzc`))V1_Dn=nBx|k9
zvrO^w(A{qQJ+<(``5hJe%Z__=xCo__yv*F1M$n7o$5smR-I2g@=FK#gu=d-^*Ujm>
zrTOP#(2bw$vOpND{@5h{;URN>5QuO5eAZgl`!&F6d@@>ti1p|6itB?q<wya|$|FR?
zgy98^iTx(;%Qh=S&Exn7M{l03%DhTG<A@JCLQ3+d^Wh8aIe)dNI$7wnn`{9!8T^n7
z+g+BwmLWFYtym(+h9>22*?d@uP%qQOg>1$MSnE4VkwcGGy4+4?jP`0zbn<KfuZTz5
z3Ga2mbE`U~2L|D4niH+H<l69~PueVNg_d_aSo*c>*B#mn=2;(%)g4ES=)4?WIq55a
z%*i$%3oD|aeKpTY?WYW8mXFyV7g*GOigfKrO?AZ7W_L;W=wW@1|BO*2EpVv3D?H+j
zW+tYxRf``nvhmJ=zwye4k8%H<x&Y}yoZ&`-LIrLJ!m}_m?)JuN|4fJx1nhoVpT}(}
zJ8k}QxjvB1;%n3Dyxj#M%one6WY^Y$3OP$T6&0J)nIEQdn6Yp+(JV;nZikEY!d|}8
zvh8@L(epjYnyzH|d4{I~LQUPF)kN{tPwyViJmb)1y`X|evE~>WKZws-9NGc~ZZ{L^
z9dX^5;IznW^jRVcO-&RzS_f03GMKr5(MYra1t!%dk!L@*I*rgFsi!XKhP3?#@!2B^
zTl1Fwc0@Ce26tr%i(UMO+A6?b{=MQ)Pi}f_K)wE$XxyZ4XM25!oVFAEsaJ&hU?*dl
zEH*M%^9Sx3a*KErRTBoSOCtdPCAxg{@~`@g6Y>F|;N;YY=2kyPa+bTtJnO`&xeD7M
zr<+kt?+qxUk0;tcv~@GT61j?Q4U-WVA%N372fMD>Q;S*JJ=1yPe$zVG0`zyMY`WIi
zJkVOZ^x*r)a{l_+IknyvhbQtRsN*E*$?B6t<tCW`i3j~Pmkm8g=RJ2b6O;OhJoi%1
zzG`p`B<5*vp5SF)rp3|Bwc(uv6(o*urorszAuz{ytwEy0fZd!5XW<T^fNAK1AOVK8
zEXgu91eR0CQWLQAqDWjEQ*brX=(S&B&ADovke<0v{<^L#v9W;QkG0AP6)Dy#^0QXV
zSPxEQ5zsbJO|!XZ&T|{z<TF;ygP622>>Zb$(7m&3dJUZXV{hM`BXxNaYa#Q>VvD(z
zUre;{$i;8*hs*E>{UX33<j2ZsHHZkq>g%hEolce{_8!#_+Q9FPaaB*Y6wXD(Zb9=2
z<U5^#XF(w9vN?kU#^qsnH9g@lFq<HE`YkO(t-4Hm>?CgRReknSsI`cY;$TH<mV@jb
zdaNece{B%Fr<uB%&4m1vxNFjPRqe95b?d)H^$nmh7Jj4N(1pQqEEJ6&#H=4v?w&N`
zbgT&`%PhkQ@xjk&nrVKU4L4RiwFP<Fq9|9I`13$%vLdkch`5@M3`A{NG91T0SNzOO
zU*$(XeP!zpN^vf%NB%emh)%?wA(YNiw%cmtB%FC)C2`*?-x!C6F@C~5zCmaRv8`W4
zGgF``5qn4l**s1Z*v63CY6#I)!841f-~P3P`%`4yPm+?cWw|{(oT}tf>$Kf@gayR6
zLaOX5h#(%3i9#7p1&PgQhyr#PxL6a8-#@Dyc|u@8t=WVd1c42eGjtb)9la1RNt^l}
zZiu|PfNm$J7R#WDvs|oT<5<TRA2*)#v;C#>Was-M_6SpY1Nf`jV$A?lr&SzNjHsER
zM8JSQ50GSWc*0NwLafsQ-B)wx4)qk~hd<`r?R@pk^pk*Do&_+NUR3`C9AD4GiZ*7v
zCud_<z}L3=qmXZYSEXA^i2-O!>{*Ap4WAs4m<8kT#7{N^Lp#y6Im<1OJKs&bFXkBg
zzpe+QgA32ucol7DKW<whhoXr$E>Ez4rqP2En<-@3UZv>kid`{-0izIA4UfjK2mQbB
z(uqZ#CkpSqeJ&dnEk3k2n{9DSaqr^N?)V8XI`2N>RbN?-L+6~hfX^jh0SEY|Ve3<S
zt!X+$4^VWw2-0`Hv9!KSu0$cHc2=UG99Hil%77okb=aa*tQ$~|tIVXY5Vw2otm9PW
z#+A&J$;AxVV9w<&+g=pkOg<FrOPL;<hRRH&x?<ob#z!`13U*ZY9hVsGZYKsEK?WD>
zI21Sj2P5{@;kWKl5B-cQ#To<Os-?0WLI`NzWg4QaznT(lj~hELg%F6fviTm56Rqw)
z>hR?RI;Dqtl7QUU+O4gq72%hnr>>279?_ypDkyz*+OG<w@lTdJN)9{&wU9`<jWFsU
z20DEdAJm!}gNoFvjEF}Vmi4SBz0j0l=94^N9=maqapSL&B?nJmG4=k$muI1mZb$H@
z$?U7>jL4)5n5M1`Jy>$I(2Mo6Apz<bBSo;6V=mjphV6jXRwD$l5=e0EYTgn2=#5!6
zg<T9l3>?Sd!2!Rxko>K3FNx}>s3Rlx!+FMS^s8v7T8k{iZguS9LFJl1b+f^vH?PQ+
z+OOVI#1FfL4#@n~h`SB{<4juqnS}?~Vc%GoXYYJ^9J+j?Cla<gRzMIL(f7a9ak}F2
z{8-E`57-IHfYv}j{arv%GZ$N?C5oN{^g2<H>fy4P>kcNE<>|?%sSWpBqElN#MTp|J
z$K6g#v?pcF99^QJ)-Ch!?L1pn#nH1=Y@P%m`=TEHxtg5Zi0rkw-Qqs*yFAE?AfO&~
zR3(P9hlzY#g)NYQ>gqD%*&k!{eEX>1F*IhMJFQRd+z#w>$rEm7zJ6LH;iMndwU#fc
zH|KUN?Y)?GEiy~*E^A5xXg)l#He@*_K`4vz8N(;<%3i4gtBB(xkaX!h+VQEZOVQ%t
z*8)ETps$jhGA{1(o$kEYBVDE-HEwek8+MMbD1Wtrv=DMA;<kj^74<$z-!?j0JCvzd
z!G#yb@xWf;q-(=Pn&0A*h_!LRYKE3U?USW4+ixF?-uq{VQvidP`j6VLAMgbP_0MC7
ze_Kct12K)vlhOa;U2T574Z-8{V-6}Dl0RO!3!>tiC`@4l9V`m1H?#9eZRycmuu(XS
zsNuVIrrW2qLqvR@zOs1?Iv8illErWHv&&IOlo+<IjYTEd826m2X<+I`kV<(>CiC@)
z=#=oub3!d5c?R{2!nyi?6-qu`xSa-Q7Ebk2i5Um)ZYcqJDUQ2H0diqmO!w9;j1(5m
zFA38%MBgNNHwppjux;!pt&eQb;ZmRybn79D!%Hb^x3j@OT`x|S{t|r5SRKV=oR?&U
zzg0VTw;u52O!Wf@7ETj2ZH?UhF~{I7{<6p1X<qKbm&I3ZnJU#GBe^PLX+5#Xi{9?!
z_FO{k4+?P~uD>8NeW680be$-D_>d9(M9U#O_1YwyS<zqZ@7KX2T_D{X%TB*!owrao
zHUS!^JSLOVS2bq4I5^iAb5_VrT*xxfdn$etL*d;rsOEKLyG!I5#b%}oKIkUF``#pY
zZwHK?MPg&VnGf#5%U`HT8Y)euE#L@N0@(e;__qF?#<3xlhx+9dd$qIYVh+$8R@+OU
zQ0xU?m`^#aC3I`4J7Ha23!Kh-cH3rteZt(&UOsK(@GCd)r65GxHWqcb6n%HaKVebj
zU6)X+;a;&-ff~SJ@(QLDXgI%yB1?(a^lA?1e3^r}eo%C9s)a_2A03x!TXdZ&-;=W)
zA0gD3@lL09J7^n19&i(?KH`|mWF|H~JV`sNJ(&gcXdfU73#FAMvr8?Qv4@ID0iXrd
z(bPm6fZ*@*79NfqKz8j2{#-v-_xS;C2_`&Hi>N|SPe%0Nz4DiHULIb7xW31y;#U}q
zL_Dg^8a)>HcUE2$zv=76i*pN=Or-edqY-}Q68%n>*8td9FxCyfWAg*??#9*I;hiWb
z`kjgk@*KAO#%<dg<)oJfoF@_k%B4H}9*&EufVEIvVKs=@c{P9~3Yp=E%tx^*3P#^_
z<R4HU<lGk)ty_=gj5H<=3}<qGuK7stez6idn!sn)uO%kVuD6r<IBamJM>KgO#lYhk
z&uMJYQ-z*KVDpX0qzE|Y;)4#qiZK^Il_{ZE1^681v1WY%?b7%wX)^r`MTLB!?^Qv<
zXVL9f_|2{e;MsK&^DmRZPnX+Uc}im(!pk<<+Sz4peEMGz&ec)Rbp(y4JT1}{g&!Kc
z;jv%kbTFE8pBgi$W3~5~A*^O)s0qSv@0vTmd?H(Ss~p_#uv-#10xg75aohI3+PWH#
z@PXFxnDn<0Ok62V6*WheNm#&cdqGukVWrmL?Gp){GbUnQ;K;yhm_M<n7V1<DDgV~M
zsc4V<ViLKv^R+@`|NPd6-Jdly-n#EHttG3APf9mlD~kSz>xr&g=Uk2}H#QZet9XEV
zpb(NTGBxKsQ@qYO`{QKxj|eUKDbOrQ{b_HT`BeFln;LL2h6(l_Bf<Oo@lsH{^9~=v
zZZ}AAuXBRM3_ic?<!Voy8K>xsFpHr^{LQ_Xy15r4oJ<%!`L;$H_0)u!r;FK*N8f!Z
z;NnBBrF^5gYt$QZ9Fx^50x2GQ<5|q{?Dp1s<+%r4Pr@<d2cIrCUzMHhy34P|$J=aW
zUQAW&)m_bC0JX>Q*S;NO-ULlObwxCRw{)_eFxS9x_!?bFBZ~F<J|nnv<$z7GU?W@(
zmPAS_cQ;17c+W}He1&f(oK5sR5S}wvtFE?@)qbH(0{`fIoYX$~3)<!Wtq)jhsq(W_
zohO3`n+HDo(ZlU<Q{`b~jZHQnM!$3M^Ze#j7N6HL7FYRN4Fg{z&+R}yTbsp2&OB+W
zuNZP68Xi{@2266@2HsedJCGm_*dUYd;B!W6vt#@*M;AMbZbIs(pVs|Z5{*kKz;ywb
z&Ax=?>N=0lu(O#eumA1$dSrYvxw17zGkaxBQ~$^Y8*t_*VpvrzFur_mz>$bqgLYZa
zNGjeJU+74z7yT>KgHG7GD_D_lqa~?dq~ga_z(YmE0VA*`@U}I>dZDtUQ-r4u;vSqJ
z5p!!8a1Y*RE8koQRJ6TZk%UNoi_={9YcB7S_ySnx5=GdR;(i*X`0+Ih%+f3gV2OG7
z<lC4f98#cRc+fFK*&Qn2E!+__&dRUKl2J6Du1#xK7jkZ0%s&Stv}&Z(2HRS{wiEWB
zVgtxL%_V0|NWwhzm^yfqo0+B3(Mu%`5CmI^c`(x}xBPZnYE)4anlz_5T;P0Sdf^*F
zZF*AY3`ZS(?aooImHx@GTsO8`dId#t>bb1v_>wkZNyW*#Kuy(YoPUN^^HeSlux(F)
zd{&}`@@5CBXdU$=YC5^t<M&tk>Mv$Z_6Xg?ct2g3@lWXn<A*z4bIz4q=u~r-t`G{`
zUaL=^OIn5!C4Pc6CA4#nW(u91MXkEdDAH~>nH9UM3GPH~bS}j$o{!Di7v|17j7aes
zA*YYI6qDSJe<CUt#R{|Ge~C!%QvUHnb_S~6$G{&DY4Vp5WcUzjAmHYsV*FJPN?4X(
zf68;8vs(;sITX3w$amHDWk`)py-8`VZDtKRzg$zy9@W|Gpx=YK@;Y*Gc=$Zjx;`X&
zS4R%TeQOgQ7dN98@%LDTq&Ik<8a~ww1c;_Z``#Gd=9{iN(u}@0f<>6dG3fdC=aH)h
z%A-(sw<+`Dp00PAF?^Pv>yLVcKTNWj`azNzRvYAZi`uet61^lpI<MDwvc<9@gfGu!
zauh`g$oO2i<Jphg1oW6AtMI8N^u|QU>P!~d=!+$o`^|0pACFpBZ*Ogv@9*WQmvaE>
zs~kt+dsLhNho6{V#FQ<kW6aQybIvIf_4N42W~}I@A^z*&^oQ%hF`Hte1R#f4(4k(g
zW;$wR=#Rf{$R7P}kGvkek)81RURMrN-BYuy?bAb!ap%39HtLHuox}Vx&n#?of|L}P
zSzW!G7$W(FCDHN08vLVSZ@h^NozwdlQH!vDmqWL`ZeD;@n8wmRe8M~ew-yQfSUwZz
zap<6dFbCviAS&jRY454pPDwlo4OUXrgKPqPKXZN|Y|^?qeI`x8--=~lMbXyvP}Fvl
z6;!(!2=@TIM%b`naaDB#=Mi-pULw~en)T@REeUlK<w3bS(&(CgHYSCG?SYsy9Q4%T
zpiCa}<Q7x*IQ?V&ey!TgaCqTyK)SE%ci?)Qnk1YWEIoLcj{7B;OVF-xfFVmVpVv4?
zWH2yZk^{4LaX=XqM3x?V`bZie<WK=I2v{xqkBb64P=aCwx-|JrOycD{_d9;#sIZI+
z<&MhMzl;HeycXNYBbEOOkxGu?GLm+jZnPTBfJJ_m4B?b<H`!4)v&!bxe$p8SNlsY}
zmYMrDO24||;bt*fqk&w*cPJ@+qs`{O#1l(;?IxiqJY=J=1$%ok>~NXT{*YyqDWC>a
zZB<lFdzH>evLg&7EU1Q73iqWtrU~zr*3#N<KJWa}9ztbBk>IRQb*hb=Z441v+3R$@
z98cQrelEGqd9nTVmkd70EGp63?LTjJk0$9q&gw%UwMg+Ho6^xT1cAFOsoQDq79VVS
z-LCGhQT5LwK;O&0cA%K4cc1zuueATJx2{uhcyjE@ag)kcFq~-aA1~YU>2<htI+|Si
z`9FrARCL@-w)~6YcFxs>PiwYa1~bJV1=$2M9PLKeLmZ_RCt}YAo;z1whlSq=mTX;l
zI?W4WcAPFNk&Ctb;~n`nvQ`I6$@}iRdD#{XD%Vk7e3pe-+T%w($AyY5vreN7{<%4z
z9sbEl#i|q|eV=J}`zTuT%@$C%O}_FuFvRLE9WAfz)v|lAI&OtEY4!<9thhCA7W?A0
znYU2cP~nXE@cfCz_(lGcXI=wyoYn1bOcNiXjxw0&V#vDb(xD%+b7vl_-FxQYL;8Xq
z^9e}N{snUgl_BShL+F#}zRtPIFMHC}wC2H@B}M$Z^&IULOxQeJy-Q8@&MSG{wx+9)
z$lJj1dFWclE*>BCUcAapv0iBJO1)_hvo4Q4sis^2GnQ8vcnC7$vi{8iNS_{N%f0xk
ztuD8A@VUKj$8GWU0vAMb2heeZC~QQxOK4vVsz2WNIu{dP<NXaGEP&rj4IEQKqNIPa
zU6fj|$>=={RX>wrD43*oH{gi3;LKxE;DDZu^mF+n(d)6(ib8#qJhjHZsTfT3tEj{D
zqR&l(16rjm{o4XY;;CF8F(vY^tVc*1Oppa8jivw&0<%1tW8kj+(rUW$hZ|_9N4$8E
z9Z!#0L}?)~8%>_q`r4DLiMeRbd<8>_j;atNM9?~*(T8B*g<kJl+mSXI-ho!$;-vjz
zr-RU7uDa#d?0l<$b>mXMDRA|9g3kLDahaOYtn;i?L9n&kr5d!f|G+$Y^hP%Qmkr)<
zL#HEJoK$`H%n;&&C{`y?t5%!5{k===apOwoNm5C=m?NtLKe4g?hx~H7vN{iJPrHef
zz@5U4#7==Eaod^Dv3FamxujG3Am6EbZXD%v-&4c-#O7VMhVyLo-LIC`wg*1ZDYRWy
zY(&%HZ1^5DTZxkS@pE#sGhBnu@{Uy#xc7husX~%!Q~FynU%F49cMir9!d75Mt>q3D
z<x^qSC@jQw$i8tpfMS0L)k*qv8BsPXZ8lY?O{lOP&Z1aY9CBq;Xg)NyaidUdcy{wF
zzY!-2Ef_9EPVLZOqo1$qjpm%OZfqI!Zfo!X8%-Euz!q8f=j*fz;jqe&UmABW==9k}
zd&0ab8;?<+BrUp@lX9H`BS$TJwwDM$UI|VUxvvFpUqr<-b<f}UBdP)a;|4{&h5$~X
zayPs5;amHeogCS%5TIQSXWaJXoc;QEBR%MBGA-nqW5mLx$)fDFV}3B6^CDH;(__!~
zG}muG>^w0K-(P?I-p~<m^K&&MxaVQp6wRkVUQPQRRk4kWKJ#-(YLoxnV$zvkN$umO
z;*X#tuW2!<EU$FVex<)D2fMR{M}$K0OHx4fAC~GthSTdhwar+gdf^u4KhxM)2t@M*
zdg>h4)G7gS^1G?q*JGTunOaoOnUi#n2HmYn_%AM5dP2qK$>7<EwBJ@QEgCT{t^(l}
zHUcm78!xv{T%dIqro9S)M88$2kEOo=E?T8yk!?TzbY?I<X2`rH{g!!?Uw-baoPv48
zD;cxuM)J-jQ?Sd5GWF@=gF&?CaoZEEe3Pho9#|J)s`<DtDo1hVylm9aod7d@8wBl0
z`3l51YtIB^`4gnrn{NE;+12q+pp$ZcM&HSNkC#K=E$)~NSSDanTtQ-a?ZXh2$I-({
zjbZft=AVq!-??nPz#P0E?p~B6<8GwI={2_N3z>-vIx7Sc^Ee9=Fxox_1l+lZN$N?y
z1ttnnP5%?@YZ2Yz&jWT^@*Rsr#xJpe&49sP;l^0?Gh@<(_stPhe0R4d8b+r-+P?d_
zqv_whhJu#(=7Z&TAD$0cZWsW<V;-5ly>v^L9Cd(YXhV|<bf6zRbzT3jDmFwsl=1hV
z;WQW@Xwv?st8_47+twW$5D=o#VPaT)_6hb$#F`W5(~n72r*pPRkvVNp6tOk;b1f>K
zK3|sJ*LxMe$xWe-F0A0R5*UJ*_f-Oc&O+tER0b+RhgfQ_lSEEKp@;*$phuiSZGNNQ
zt-t4V5#9xc4jx^p3T)+k_fAethg!I?=r`yCjGM;`TMx!plAusqXD@-XA#Rxhw)D;K
z=^U_^0QrDBD&c%tNz|qv$p0hi5ZAh3L1$Sl=$5LsYr)a}JTG7gKJmgK?XM(z{Kstm
z(*<3gv0voS!OG=eh{Be9&^|w}Y-RG@!zz#FJDTt{n`=4NB%m>rkzRz&lN+0Bc`dXF
zmDQokwVQNT<W(&6e+U4KDX^c0Su`sIs5po{j8Tl^?NLNQ9N2|X(lNn1)2KcLRS}`P
z7!dmoy0~pUvy!)|gPB-vVjW-_q~6aZvVKI<lKIgNpIjP<fYSJr&WZT5y7emV+$Eo#
z@B_2H`VWLND6iZ~B$dNi&qVE!KsxDlBrRLi$zOGp_l<z;CNP}iQ%&Z%7(vvtJ4Ryf
zSvuC9H<3Qn+8VLa6-mR9P}Bf6+jsKKBcis)J}xE(Y5<KpU7)z%$oEkoyh<-MuGiRT
zdsY%I;|^$%?et8wNibt0-e@$Ji7p^Zq_5DP^;DlY^r)+M*9c+ru>Y`V?{#QGTt`WM
ze{AK!1<bO0AG*yJO800oG1%Vf43XhE$VhP&OuKM5ug!Ik8=h9G-#NZzv0cD<I5T*b
z*q&e}OlR|2s1M{ixp#ld7Ugud`8_F0I$5KzQLb!EV@xAF(gOC$9S{pTz8VYV2PY;@
zh9!AzaqXl7oc9k~EvoC7RODxCseQ$~{1ZUX_}Lpp4kTW+Dy=c$hrZw5zG^tT;-(Po
zHI^kco-DVrau!AGOz~g3XYc)z&&koe$6tfAh$WOz+4(5+TI28#CAy7}*=6lE50p>N
z94jb%?rc&ijVyMF@g}E&!NrRk%iIf7Ir>CX3{TfUlQU~{S1y?<F15~x?FC<u;u>C>
zLY!ecjqR(DUY37Xx2+z3s@vf<FD%IfmrGad&qioaG##`r1a;`n6K6IA!d%w4L*JG@
z>o~QjC(FP)+TJ=lpWZG~X8eQPQ5B00<9+<*H*`lL=aMoTEVN0=#&MhPxMe6+MutY=
z0;ie8wsj%j@l5X>d&=oz0AF&?-B+tr*#t-9NrilELi%E>*j&qls}n%X%BT6v(I`Xl
z{NNb-DyXylEgHA<`*iq@vodj@&c%mcbOL+dxJcHF*rDcB*>$cz)79}0JM&OqZ;84`
zwlT&#Uoggh{^9s%O@&>)-YdnF!v<mxnj>=#K`ay>ZF!-}i9m2lFuwcQWbWGS&bMOv
z7(Kqt;DRaWFVtY7%p+jeC^E2eE3sl!`dUN?N7SaB(PAvvLB0(da^95Z!g@k!w&K43
zEAb5*4Mb#b%*kPUr~M@@42Rw5BoPKZAJ5#9yrf`&u#eGL=ay<$T70kCEZpULKmB!F
zYvrJ4A>ZX4PZx{}8pD+2U5`<B5JSyM8n)zeSSd5fCv2Ix4t5nLGR|{v;G4(_Pq5#>
z@wYrLx$iKUGC|F~Wv??;r5#@T{F!YVyYR~V^j2i}_}|~<ZSOxfjLMI*ddF7DJ0Fnd
z)Xee&eC*p}x$1LelQ0Ui!IDZ<**<6-gi@>*49l;7f(;L{>wIwIt?~c}0`4!&U3K}*
z?$0u+OCJJz_&~O)yp|E7=o{hSsHr}Wj6}`|Q4gO}=LO=-tvg{(OK)g%kO7Dx86mwT
z%=nW%u{gcb?lZy~(<honQ2yHdGhck?BX1I3*dDRs-T46~ZH~GLzG>LWG1IGw;l6mG
z*1D^Ce3ym9E%VV5C<(&D$i6%mDa3SNvZ-QgyLtP9Q%w1#A|$w*?z&-<-!yE!^V8WJ
zc>-SOQq9Mr1&8X%Sr+YRB0EPZ;&lI)V8;}=sMFTZ*wJVmFW*7Fk_E*(<7pQ(%0k4<
z^I{7Z3`oVjDRWc$=4JWfH#_F}?pjvyyQJOuN2`-lgmdP!W#bc*=3i0MD#{o=eW~@O
zYnCD`is}noUUUf(ho|>gCXZ#s-Uz%2iSKGQ$fl=^V6OKTqlyu6@B*5j+(=Xdx**3L
z9h)d#oev19vFk?aW`D5N+IWoJxQ)?NU3ZLs9S@13|L5e-Qq3k}wB=JnZ*s=yUBk=P
zdAsXF??gWENGm}+!B4>E**;kfKPerua$}Eo_;ykCj;KSVHeXa&5I?Pk7Z@{fN#0>;
zsXn~yE<$ZswNKEsCxnCfSmy`_=lmK$Q{9Bq<_7hsXLIq~Yg~$4kJsxdl9X0c_iQjI
z6dk54_=OKzNNr!iMUl~>=j*A0LP`JBbIoIu-T7?_y(I*~Hu7$(g`HyawDDBjc75qV
z@fm!juXwen)M9(a1~+GT5#}4ubGQ)LQb(o2od<b>ncxc8%7uDL30-+333=5(9GDX9
z-xIki_gj5wx;zzO>x(qMfF%ZI_|2;QCfb=rMYl-U^B(!~h1Y*P?Dpky5PvR%XZ!y`
zOqCEgpAl29jlGaaM|6a6O1OEIb|l4P*8#{s^~&L;bV|fwt8_NCrZ7uCCRu>iZylHX
zi9{P_tVwaY#+ZvDzv7q9DEl|ib`H_?B!B0wpvR+#*&$mk;9(28(!cmot@z51Ly+LZ
zD-OZ+*IO0N?ZcXy7C~Q)m}I719Zy@E^1tNJl<YKm#1~sVTOd}75)IR4xl4!N%J}{3
z<&YW4ivH!g!@PIfQZ6go9=_9<qtqL<m4@b%A{4=lD9*M1JpbrY!1cJcTV8n;qR3s@
z9dTAZ-k<Nof~0kr=?W1e{gOWc5xX*9qx<yY84~n&*8IDierWx(!elnb3D;~qqY(A%
zLzn`7aGWVeB{uMIl>%4>T3W7U$Hbq%4FSHJ9Oa|PGJ2v-mZ0-qX*bc1G=;>9V~S$D
zH|%h6Mgd3jAyczYci}pwA1*@@cpNH%uN?|{a9PkISh}Co*OY46ioSa&GK;N$=SSE^
zIP*7(*JONt0@-<`b-ZQ(s`zczeHG@(Zy@POh=-DD$Mqch2C4B?Mj$YN>Yd}2*IYj7
zk`~@g_&Q(bdcu35cgpYHap*`!TR#_DZW{Ns6L2nwF#Otgu{7!Y5T$Jh%+ln#dlH1l
zs_Zsi_veq;75=T2_**jZJ2O>0=Y0nB_^pq)t}qC|XcF*XYLBYtj~C^)n<?wiYLE4z
zmzg;6cR#-rhz%#Lwc(cTXpGiDL_`UemgpP@wOV}zXAII%WhZWCha3<8czj0`ylj|L
zSwZ|fVsj-metc1@B_rf;N#!BK!D0aUD~>0z-X@B$wgba~)<6n@^_6bs0ox*63X#0j
zOTY*2uC404if!O(a`uHljM!>ljooRWIC4DC`S3&UTQ|zp;jpApqI}8Hah8K^6`|F2
zfDfud)0EW51pm0X^4rCE@5Z6rGg#{b^qki#QbtjQZ1URTz&tp;U5kuX6!ICHN5$w$
zKYhklGgDAbW!CYRLYZl}nml(z<`S2JNS^HLd__RlHtaG~e|6dJ-dE&5R0K^+s@)ST
z>(q>6$~!r^$ge$UrM}BH37bkND+uoue4ugNf)j4uQlMP6f}9YAAqS#xQl#SoJNf4D
z&t~&P(tTge{$-#g6ZkXP$Ea80b#%HBV<iAx#r+q^TtSR`;>m#X<L(dZs0ZS@8V~x(
z8du($%5P5;r}G*&jAd)+U2PZlnQj6wz<*WdzbQ1$7_N+xanw{In7e1qgrAQZ+}uym
zjwC%vKL?Xv$WjdZi-|gTIZ6qxWYZL7hAd|MJm05VJa6fMh+rKutPf<WyPYin8te8X
zH`b8z7P+Lp>jr5>UYl0?h&V?n*n|c$-F3LV>@HkZOPd6$dsy^T+@F&`=u+RG-&Yh#
zk<o%ZeCfC%0jV+VZ(pW3d=M};8)Jr#mKR9gUL4}cc{93xdSm#rKl-S*Swx`y$~=q0
z$Dh-7)C0b#2HjFtv|&I<nr*T4Grc845dQTI2Hktu4@8CM8;sF%E||({I6O0iTb41A
zX3TN0t?cscg#0_-a2skqHuda->e&`-N5R(xJ(E_E>vPnHiKyzmgwX;^$?C1;YEpfC
zI85j&WC`ek`uS{<(J`oL_(S?K&DpPF2rPC2-u1E(3%Sp_3a@;%<g?nX8AkTS^_C%N
zxjQdYKVePKj?E@by8iW{D~<or;N86t8d)Cs@ND9Q2<U8es!yD(Vqby*Y}c7vkZFJ~
z@TGP|<eY-3cun6EitLM1J#|5v3t#Rx4^<NW5BY`gFU($azd#~c-scY1pao2Ex&>)!
z1sbdh^r{#ZS_ivdE4cqZ$DV-+pIFjj^SHq-3lIC!d(#e*ygz^by)9%zCm2|JePut4
zKbobH{oeYd9#&t=b&81heiI=+^!HP%5=?pSF>^4uBk05?WHMrIe=(+fEjR6iH;nU%
z&Prs`<V2weF0?6JkdJWQS_nwX{yL}7eq0c#p!ueQ-{JhU8oo2SX63IIL6xs$bh~;l
zGj5Rd6PYUn?z@vPCN|8oagbg7fJV9)d?IGRDH`K4L*n`O_~vir*lj1LNdFfffM#V9
zV1j4nD7P9Uenbj@d%oc)1akxMpEm8mxtac?t78`7r6BIt^%uL<edoxXxL&t^-bi4P
zr0b1YR+^>%7tYJ3TjKNR{n2JQbP_l;=r`~+UkH7XEG09aYyS`_Ash|bnj*RE3#<Wq
zx1D}|`Wou=O~t;&x&i7GfeTuUCY6CFhX?C?(FU(mF3IBu12B2#W6khGR>C@}c7$h@
zsYF=oJ@Q`?580J&*lb!2zu9d3c%-1!1Kj6lnT_5miszI$PZhjrc=Xp-+TS8H;B7bK
zei?{c>(g}d;B%6mC7~eVod~Y~2b=9Gm2N?`9_D5?SLgOt`k?E}ZQLMEIoQP$7!KzQ
z>i4iel4JKCuyL(w)DQ`GyAEX!S}O&8YA<Z_=O|Q-5HH@)jPsc`V#&POT9<ykR>GP@
zV!J~;rV$r0#Z;N-d@Va=bmV5?J;}z@w*v;6;ufP?*m)x?{Gd;PrN+bG2XW+u8`Fzq
z^J);TW@F%Lb$uB)rCF3+EtsW8bE~Ad+c`dbJ#5RNVFJ133d^nftMnqf^(WfuS7`Fu
z2H!;6Xy9V5A`sd@w0!}PB<{V)Iz<6QrpvJcR^hjLx9bdhx-QQypBDuFJIHb|{z1q=
zW|Eh5#H@rW`}(5k64CIl&_$h{E6}Ga()<~dsV5GJq2gy1IL!Zr$ej*k<ZB);GQXry
z6|D42`Cp8jwA<(p=V6&>`2w^2$*9c*NrfKyY&gXGh4>qRWHOm2k%{;?t{XT5(gp3w
z-(^@nk5d}hjSrVB2&B7i9H^i!FB`vMgPqa4)P=N(R?_RFwg#wvsLxcW!_LAu$!pOH
z&ET$@saOxLE5K&&6@w0aQh4!#NdZF|qhhkj;1B<u@;TUOg|)%x>)~gvr~AwKudW`E
z1pm3UbwM|;UWe?`ou*rya(<6;g2Ih}rZF^24xq=ryxwuHAIVG*efVMC&tr6HBJOaA
z<>;JkwET&XxgM&18=57`eBMH^vJ!-172X}r1C|2z|0vSqvmC5=4U$xgk)pP!t4nY@
z>on=OXEgV|**E0XTKALk_?dwVs&|>h^$&XL&doEl-4AFpXYTy7UIT<}`}t}L+Ilyl
zqg4RV=IreT4H0hzKottDAG2_Dcq7D*^3{DGvQ!-UK3v<Ya!{yHt-JFtaRUJGeD!-&
zEA@6AfSa;w75rbYJN-6(z|-(I>dpXqiuG%xMLt<ZCLXvzqbwY~`SGU!P3rAtR^2A1
z+NN)C+vNU3pERv?zwi)nFDPxJ%I`T@YN07pTh?W=vAy460lme`IOclt0EIZB<l}LM
zDBZQ&^xttl(|2W4V0?;4NHo)pv4~x@$bo!t#>Z7}P~NBsbf-+Xq8lZ%^{|X8bSpK)
zG!&E)l^}TbLq%aC#%S-m5%g)6Jm)}FrYC2vHhur`6(cqeFlGDk{Cno(!`<q=$@C=H
zxv%-3?R|C&a2%c?LxtjWu;ViZM;^2BUqVKFMHWJq;T*<o@xVhE=x5P|e&bYo*#Fh2
zyR^I{50LvmbKLLf%bi<8neS&bj_S5zMVeHm?Er7UvV&OaseC}zU|=aGRPQQu-S_z9
zt!EYfprEIvC}%-oc-!i_Y149^Onx~Q{uax-7(s(bZJTp@K9fZ<VB>Qpt@|5_ckZe7
zgBCgtqxeWvdFp?&<6Y373GD(jJGQC`AQ_tX0Dw8f-HSRJkANHEW59wcX^knXM;fyh
z^_=Iwt}76MmH&+5-=1XjJLcT}o6X0>9F?5fsMY{dM6(q@Z&?iNPYs_3cEYVF)0!{R
ze2wA)FT;;m2@wk>E%BI_#>;@x6USM<urma{X`qe*xT~-(d4jWmYle4Ef5HZparG4G
zRyDbWf}0CWzhkHc_DTaFq=C1T3J*j@<HUcEW?c8pIyGphX<NAC3=+3su=?o`3nSk)
z%$l)}6XhgJ4;a@OZ&Rf*bFG%P1c+ib>ZBGX7fRnQq<2vI@hCa@IArcBY0Gx}q?a~Y
zI9Rw@bI+=HOviq2Ey&7n*J)R<h)V^+>o?}zkXPD}OaL3v3(gSZcFJOKiDv<E!?Zh+
znay_Y3A>dNjwrbsj@&hk&Aw_;5fdX6YYIvNDei>F#4Z~{C+~df8pFx|Smmx!)07EG
z0=KcnZwBJsf7UV!yV&_)LvvG%DeN?XNR1<hW5(FD$FDKsgv?AE3$loeA3082wolc)
z+j&>uU>;wudqo1+jAwU=gtMlUP42b;&L4#GWNvAdw%gI|(F%V@-v#%9wVrb6;8xFH
ziRV|sbfs%au0O#N^G3gUc*#19$#XZ{=6p_DSrEq!8ORnxK^Dq97tg4)gng5N88~79
z0qWxz`335KJL5lMvIdLekQldlHUA=ztidwGs=<;HK{NOBV_GD8M37{|8>mjSmcrPT
z3NG#Nuxg;9t#{UKZ_{*Ie_8V;&+t4O>umGhSPGWe<z)Wh0mJw~F<}RHi<xm!PUjG|
zDV3)p_`67M@u=LTe99)#h~5$n*#JXyC?(p29nqywkSW^T_ry}*R_-OZ87oJfXO`$Z
zV`M2zTs4q43?vT;)!1)6*aQHByamKEA$C2D-$-=52*Ez1EgA6MpT!-DAlGGH97fVa
zaog)6M+}8m$uay^iK1uA;d;XMGMZ)PCKr44?p1m>++244u0d=bqBlCH^G+_8JM9*|
zmx3#chkeBt@0zjjEjE88<?PYAbqBu-Gl)|7RH{>re5Nz<jZ*}%iprlYH}VxVfVy71
zs1w#cV$Be~n2p%Jy@Y^hTQy79#+`4)6**U{?QFi|C`s!%b@#4pTFShbLl)E?|Jte~
zfc~XU9>%!=`Lb}R)Xu&KqHymTW~{SY4C-S@!)t&Y*Ylboiyxo}W%5T268;+^Qb5C~
zqz`BW7hVV9c&ZCD3Vt4IaD%;ku!ycpdgt@~1=%O32qwzyMX6+!K;o$bfO1Gn>%VlM
ztd5H4%Q;O>gDsBdo31|xX>xP>rSS2-WwaJ`ouoibWS`y}Cti%mFMg&%&A_o5l)(>1
zjOi3A%a>|UJUKuQP`rBRfp5YXYn{@Lutz&9vDO#JSd3uQJ(m0-f}S1tbq<;wny#6Z
zOEWrn`w{o|7?iRkY^49i0}wTAM8*8+Lx)4+)>gqy2ws&wZ+88qH9qWRtoHzyh=<{P
z5e%bey3#fmS`YjERh3&>u*83q`1`Ap16=yNdf#R)w-rXKoVJ!P=M>LI4SN#Qw%(eL
z!z8yVG-cjpPRQ{sq~mn5{d5d_m7*>)a=t28qL*eXGo=@0VeY9gRxCS*(%rk-lXP+K
zz<&2M>DuDcUuQof@a9r*=#-w9t^pyE1^_Q3wyMreXgu_9D1L>0BQuTwcXGyTP58fb
z^grB5FL8fj;}HCdb7s6m@mqxAo^k5aM@9`!4IvzhczjY0xiJd_L~H|^_62No+Aveu
zVUVKdVTM1gB8DKzPUwAM&l=apJXG6RkqI+Aaq-O~00p0Tn-4LPBb4*-urS2_jE%P)
zpH*!>>c{0Rz199<qmT5`;QhvLlLMW^`&qI&>{+B!WPu82Etz5t9M}*$e@-l*-A62s
zC-hwqEHhd>E<^A=F=rsAQ)LE-W1<6C^)@m7jPNUcrA&8Mf|Z3O<$JWvq{dhrz@rf;
zOSJr1{VYk&*y6L$^dG)9!01_95-Vo|ovS>|at!!ya!}>RIG^c&E~w6QIM)XJZp|tI
z;vO<CxGm-;%MElnX{@wA6FA=3_sah|?*D1y+2eYU0E|x3ergcb+~^p8oqZpO06iXU
z6wixS3;lp{u~a|0kntl{|NVWe=q2aj6uJk4ctTF6l-`RY@@K!WolXzOhJB`z-EQ*|
zFSgjzOS|Z$DZ)hfoL|0WBad@kPB9Y;7^?HjEy~bP!_6d!8-6jEQgSdZQ}em0H`cQ=
zB+v6Tr1H7Kwlx6pd_ll;uR(>b%8$G1^YOq7T+_SHgd8@Fp;=G#aUBqZrVny5{UNpQ
zRoq>Nllj017C3`)&^a4H^<u{wKo`4%(L)bd%c}zVu7E^ppax|GkTuZ!oTTo*G=zDK
zK!&Y?-QK#o0{DeC*uWt|nQgEDyP&@SFwU$mIx}4Wb6}=Qr|#YVR#rq90C0TcK2m@W
zPiLV;)E%=IO_B!98f)Kw`GFX>E1Lem0`4Opku$Yhpa&f24qk%0>F2|5IIPDz)oNUq
zzc;H85&Q5Sj7KlkDpN2p=cU8OiCu@u2_+oNsjQzMUej>_BIg~bU=G_8p3JO*#~$KO
zHYO`J&3b487wvVm(Kfr%97kKaS+sfmBq;=9mqZZlJ_XVg9YngP=(?byg)w>^)X!`d
zu1Qj`DgF4pY+eAd_0gQ>xI)b7^u7>b;ghEI`Fwyps44**0-XL$4Pe9C=EQBp(qG45
znEEF8S#f+w5Pb;*KO|t?6>`CGTl)9R=fD3eh-Uy|9#x_^_s!>l8RO}4S(7Gva$eYo
zPDCiaOg8!vpQvYizxiH6TTrrDPmc4c6g>kut)+)ECU2)ojebda1#=-buOFif$}=`G
zIQ}hJ{^Z#u;SQWee^^DxB{Z0F@xu_fkhD40tu?GZ#7%x==@tiXMLgO_zefd{IjeiK
z5ScX|El)4SdY5`GQR1p@Dmle{(cW}sOv=Oi7_UUtpeQvquhq|at{iC`1f?0A`uM1U
z5$IZ~m;@aRF{~4<7I=l{6+^RH;T{i9I$xG7={#kN$G96o1Y8`966eT8ApAQ#M+yA(
zHzTlaP`7dN8b*X+z~rMApvv{?NX%9I_Y(f+LjC>?b|dU>lpa1T{XxKTaH`!C+a%~E
zwxKz8RR!MB7mi*uuv9$u3Td|U`m#LzhMx<4h{t8lz`}hcwq1CzW>*wvbQp~_a|dwH
zvH38ok+GDp!VTA9rbv_tWU?NagYy0=wM?-l0ox(0N0MzFUcXMO@Rt~2$ZF2P=i0*1
zrP#;FBob)RQx8sNcKSsFuE)}~W+-5syd%Ur?5W|R<ZknGX{dGNb^|$Kg0D{3(2!|K
zz9VS38N!Y_;Ibn?sOR;*BB*k1hi$IE0`ftvc=A~ku*scX`3De~Up8#&IUX_h?3CY0
zNQmEF;$qz1NP&zstDJ}a-2?Kd@&zu`<}7cvm2fY>jtT`1*IZssQv3f9CI6!|-DJDD
zUxIw|B78v`dh2y!H0XBL7|}Q2J#pWo20UL~OUj}<ybBa92g?lmI7BwYHv7bZFV&xH
z;~g$K>&1*Fi%HR~Jc@KYpefk{_T&sC2^>%kOQH%5x+z8&DGmv^Q3D(B46&-km_oDu
zJEFUw$OzKAYHH)IHL@1*lSM;}qxQPA(!PuYqF520n;4lsi~h~&fj<(v{Ff>}juYGX
z1s@GFCUzR-isX)nRSaHPLpa#E{PZ_o;F<ND@%4H_*1m(xupWr<()Z`=<lk`j0Ek|z
z!Rr10-kEvqVXT{2jQ_2+J?v)xi~khBBb4ZL`{$4R`xJd6_#JkLzuu?Dg8armp2iS|
zDpbtD6=1F{CJl`ASPXZ1F&+pz0im<-*XhH}Mk>&E%|jEY#@?a;f`K66YA}YFSUZDu
zj}XL#x5;e=&yp7@b9JYoI3a^d95C>9evlp(yB5)~BEH`I_~OSmoRNjwK^y_5-E_B7
zJ<V3>g;Zg;R`e)day?x^Q1~5xRm2}jjYLX|>X#%Xr%~LaWq{N8oM-cMO9Y>O&%WaT
z?9b5$A?Rm}#ZHgRiqi-@9kF?6QTG7Grko#O&qa?-|K}g5gnN_d$;FZ6^1l}a!2mQb
zG4m%O3|=4caApP59EU>hwcZtZcaHz_Bl;O~lkJ_cYNUO+SnzhYxaZMZB>Akd<|*f8
z_8am0@sw|a_(rfzAGFZNs(CsOw{g6SXREHjU3B!?FQMQ%WtjPzUAwx~7_piWWfODs
zaP|(bU{r^Az=*~ONSIFw^^@&QG4d-W#RWW;ERLexjIrqXet$}-6S@}!AV+<H3yflX
zjjcj^u5h^wY^P@nQH6`>YizM+QNMcZ7|@)4<*Y^mKBV?#IK(Yo_nnvN_$g++2^ub`
z=ispRbKa!+b$Sf^P|IsL#P8UX=A<2Nyw^fJ_bjM1L7Y@6!wdLiNRplYk>v$2N<Ra)
z$`t**RiP9npx|Bn1g!PYvq@)EGWh+9NJcdMAD`3kJ4`uX0R^eCv@d!&nkB{wl2`z3
zoofG%pGOoufky>ym!3krO|(Yf$M00*nUW%<l>5Xj!O7+4*?_}VuK^CZN@kYO-t5jp
z&9}7Q-&gjHnQh5-R|=n9&crIVxyK@Quv`q1S{tMr28n@t{aUOw<R4zC4axmTdz?iK
z^1H<d>?$|2lY-eq^m*7nYVGi)O}@=kxS1ZS87@g<tEE!e7=P}s42EsA&}VhFj@}N<
zd)u>3JdjwNJ<UzbD8?=6)S@B}%Z(W!O#sm&A7Jz(<B&4z=BsTCrNeoxW<@qS`HRjG
zTk6@U?8#riy3iN>NB?E&HV|jrzsYRe70x38j5jlDT_C>GaKr5|vu=8zsw}ZOe^m$n
zT-@QYa_gojI>yU@;}}R&PG3;;dAM9+bJu*1o|eGFF!J#D$`<m)aB-fZrLv`)Gta|3
zi&!Fz($Lk~-ENa4d-i#ch6+g8eH728Q#lcEhA+a+NK#QBu}t-boK)p`O9&WD2p0>1
z!YKGco+aN{x$!$`DyC@x&PW(X^(Y2<wYX9cfxq^h8~lrX$AD4}hvLVFXUi>4nxyGh
z=M3~Rg}+Gj8S;oYpR-7#KCndA8W=$aOPK-qr>r(Yy+>EhEH&G`hGm4oS6$!$Gxj3k
z2LA%cp8}e@Q~&kZJh}O7-o125I{Y1vfVFxn49gBO$%Z#|cF*G0$UlGjuV9@4|2wnc
zGA4=N+IVqtwSe)og$3KR30SF&2B$~*tBd0~=u-|44ypG$k>DY|QTH{*w~(Y$o*l?$
z1d1YM)#<p_@sK_Z-nyItDeyx-8NY`FjpSpY9Kw~u;`-XK^LYwBYj=ItCuDg9{`?>-
zzOq!aVj6K@Wp@IOhKxpf9S1v8Z2H3&Qp$<qEQ-!bD*FS(i<zllpu5DkqwVH5a9=ek
zy)lE(X_5_bhTaIz;1jameuo;l0}}G2obYF)8xTA=U`<D0co+f~>T{X^z-qI0bSn90
zGB5SMxofy4dn$Cl7bpN7IuWmF9c9ePGD|ZBYTnWpV%O$N%<x=e&4|Ux5_5w%hw&2K
zU|@1Qv2W!=aeU9re7GRgqF@wDcJkYOmF;R^m{TTIc>p}~DX2dEGYbBv%l^&t&+BE1
zhSUd}XL>IxIq;=tubVqDgubBxH<|*g22B9#uo2Y2N#L<POhoaVZxj!xuelk==lPXn
zP#Q}tKoeity|pX<YWczDi<(Vc$zf1}D;D@XqYg>@N*0V-i3+B~LRQpo602v3A-{Wn
zSG&h|-;+Z;`T#u$4cnBYJ4d9I2tsqvg>Oo7XB<PG&rqrqH{nHT=g95Bt8eLv!-I(>
z6sFHQMyku-lKLiR6Yz~<bMp>I#n^AFC+hEKT26z2=k^^k>OIJ2MJxF`yjTD=pk$_g
zc~5b~x7?CMAyK6AJiz(kKdbA_N5JF7xDD^B&W;#^EvLWXjdpD=Fd9rEHXKSvy)QQE
z`BvRz8yLx^5g3^pcKRZae1b4MYcK7SH^1_t$4&P;1(2a!N{o2y%-EmL02K2HIJ)zk
zvrX=H#g@-Z&Pie|F7k``QMF6o<8`*tiW72|DXy7QiG+tg;;EA=Q^rYF+JI_K%Xs?#
z+{LT(Ck5Xt6BNP&*DQIW`+?51Zih<ggSdd}iOucMGpv$39s)`-biD0tH?eqC#Ko8u
zuKi(aUWLZ(>}wbuwizaYBu#k!Qq=O1*n6Xpt%5WH36Se%e%&D(#LOmuoB*mez?|tB
zQt;Tv^@c&v+5)p%S^q)|EaHte{u1o?8QA_~GcY1=X8KrNDpFTYO0-j%)^Mezl)k^;
zm&Plm^bmk?(O@RM<lpQLdL&)?c8Sqf5GZguCo+C=>zUn`c46s9dvUM`Nq$Vn-S?l8
z3#iqS`EL5l99{U-d?mlmsV98mEKP!`coh?p@IZ!8SIq+!*%Hg*EQ`SpW@hDvz9nsa
zVw^=RpmU=thlhbT>$b>P`cENJYQ%5JF=H)+EDVgcr^@F?v%|%U`^YcG2WCQK&-YcS
zS^CMmYIN>bZ`b7S45F5|dEF5jE-35kIk>aE2kPnn<ScPondBC19@nM!=9IQgM4eI)
zIKaJ*|LmH0%R%a(<>@?Wi(t+INdxECBn_@pHAx|WfJt6xeO>VC(7!ova@ZT!BRLyX
zT0oISeDbX~Cq6l?v<0qIFQ8-e`uf!Mw4+g)fcrb|l^m{jHtTRd)&!Xl^}WxN$=h6C
z1N@KB8G3QLmod;-AAIdlezFqo3l%p3of5lt>E#DRCE2^CsNSJ=!WNQHgn@ZC8awh|
zxSFfGZ!V~a65GEW0wTWoX(+i~31R+6=LYD)w|V7ZG2p)6<skeQ{AKj!QzzfNN6lM=
z;2$3rvZ|?k{=G8_7$*WpKf$8ttgLL))Z$1UnCf(ihfxBV{uyvUpMouO1u(k5h&S(Q
z@|c8z&kT3e#>#971vw{5ZpbG)WP+b|wKBU=uKQLTcQIqbNob_qMSfWdvdAZE?s`}Q
zOH2QG?QXt)CZb1NfHMCh)lGHjVbCB7GdxAd7Qt{t11#c~r<99Kx{QivSq#EpKsm$J
zqTlY5ktjgA*++9C6UDhJ^8^cq7@>J3O8^cvAcNpZqFO_-(@F@+qOVijhM=epb$4@M
z3rt5IosjS4i$3!jAul0wMPDNNEI7oM*~bH4>=x*jjUNHsu+*ZaP=Ptd7yD_b5Vu79
zW|kP=$^f;n)V*UzE4>&$)3DU503!uaFV`JhvG1iw#<;_d=YD<dK9o|GlkbXkyMcXX
zXuy#Dlnepsj(oDyLJGlyH}o|L0lZO{i=FboEM@IR0DhMLF^fa0Hsrrie)t3CarMCA
z)!lT-5Fv(Xw6P?#5BbI~0KYc>rhc4dp~b{9lpXj2@f3iwP6PZS2>y;Oq=8tNPA`~>
zuPnyaa9m{%tai|18ng`<_0(zpKib|ZF3Pow8%97v8dOwT5L5)DdxlUDkrI(^>5`Nl
z5G6!FMLLFV7+~m*p&JGmT0*)T2AJ=Lz4x>C6W{xLPu>I1JkYtXYh7#oYk56?Jpi=*
zIDlrgB^p@k!F{{nKjD^8iIvOIIacpl@_^xXsJkmRxk<$mS0Xn2_*xMulqr-gKq!!P
z`8ei59);1J<At_>5ch>W$<$MP{vGb6iJLI3<6(92_SD3~&j~`-t6>V~9bw?*H%dkW
z6l2RU?{WC<9-3W|zr<|rF^7mmI?=NrktEVSk?9iBt~?F8NS@-^rEj!fH5+~f7&MVZ
zTjKHuHMcR!rrp{_g0tl4&g?n8M+kO0N3S<2>f(Y+Pb8H|4&IG`f((0-NO`3~SE+*g
z_0bXfFw6VYDo@}9GKgK=*ic^y$9gJrY|037+urq}Hu|!~SgY8}c0cLg+kG6MO=Tro
zO_qiJOc8Z#<wGAgf3P_Z5rC_`?E%SU05TB;ftgkT0;NJp#;&jQ2cNybJK7Tk6ZF|V
zz-^NFzNZ9yhmp&8&lYOuo0sj|M$_f5><tjxUq4&@M<MZHR(AiZRUtHXxiiP!KqX-&
z!`FQ4*G}g9YX~y(S=Q(u@hG;z8>NN8w9<L*2(jxeD%$5eZRbm$`Ab<=8)Jp8w)ke2
zuI92?IbD|8>~!A8(7m;qPF0c-pqYQRzzh|iH)Fj`v-UVhKqcb-3C*BPc2KX(b*x`l
zITC}(F+sO{L@L@J(7y4AC~!5}9`F+z!Its*_=v)DHB8d2dz^+Fne6?_)G%cEN#088
z#c|0<Ohi~1eNN9;1L*~0yZaqkVDHtU(0OftXEa9{YR)%^iVd&SB^}^2Mrz#l(jGB9
zJXjlrxd<wUOaY20qmU-%-5DSR*fLLR=QVKXm6EcBOzE$j7Jp>iA5dRAw=Jmw1Ip$e
zBs2rX);v=hG!vr4dEDzkA6@q)I|Vnk*Vl=6HCSmqSP5M0=D4I0r}nm7_XzYbdkJ_%
z&Sq5px(-TARudyOHnlEmnr*`)MZ$Zg6UPi~6Vx-_hC^mbTsISqwVBRhCcKA9N8EaP
zM+%%pX++(=dYw!~30SPA#aFEA@mcrxYw=q}{M5ZkV5(}SplcK?<PBi|0;2J}`hkWw
zf4C7HKgpRJ%ra!|D(OO<S~-=OhQy6!0kndSY47iqzFSc_l;*B<$W}UmB1Nk_Gh^>N
zR?V^y^L0Zg$6--wwTb>L?^)ce%RH~(8&6gFAnU<2x!u(&(;nAt+hZToS*bYj>L=6o
zcP}#7$YOZSLOC=V+SpA@T5hSAw1&_u37Pi9M!-X<TY%zh36JHsjh#9|c7$6u)>O`R
zl#|%Z)47x@RBxK_ddFJp*Bxbqcw#m_q3#?#R<c8;#W+t}%4$Q_3PwuY<k6PL<Y>=j
zD}nPih8r2%`^9P_=FroOi>Cv(ySXq+Foo*Fl_9rMvme?ew&DA#$+hkY789F}uEl4Y
zYkeEgC@818NMHP6cu$Px1>{KVdWAxqfQ49RiN%ohg(ryJ`9F!H%L2c9dHyqGg*s4A
z7e6sM1YRir_QMgnzD>chTXb6qTMx_r_!pspSqP`zGv#o8@xpmFrP87fHk$MKv%**N
z5%P(r;j*M~A)@Yk$IHo8IgNQ2oo#qZAH6JLk_t}4Z^1@I778^+NYUuQ+5sX_H^f`5
zhstULi`mVy&KjlyUzVRP@67hk)6U`EE6U0DdaQ><fNRQ9_w-ATdvR%2>cwiVHxhyG
z`#DBqEILBI1}Jf6VaHByIay0tawxX=H#5(-<rF9N50=9e6)bmxy2ob;`Ebq&`AmWT
zdMSwjs^P7s$t|nl95tTf!VnrR&pW)%t8e_w`7DtSwMz(G<z=rVo#~DWo_`6Rff7-;
zPTs#I(*?Y!zhCCtwC+#RoB|&=_ETI15j7`)pyCOQ0J~xm1?BHPptrOKmI`@(UGRNo
zzYr>$E;wT{dUkT$w_c~d$sR>KWRq&^GH1(ns~ai`DVW;sJ|Lu!J;-s?%OAFMQGnH_
zKj>rF#zz}!8A@SG2G~I==qeN<Jq6rO-=Ul9d*vO|C%sz5Uw3ua+KE?w&Ht9X^CcvI
z%iGHa9;$nzawFUxic;viXny>%J^P6-UP`kanb1VlO%%Sz9d6T4bi)T(@51;clPpFH
z5)PIdmqKR!zCj^n+P0PHI<JYjW$B!M=9i9WWeIp5zxO!#9YrOMfo_yd^j5iS#2S0=
zv`}x%hk<*GEL5KOE~Z?A-T8ELVRntP_@I1ppltFyY5OGzG^qImtVnAocql$;s+drW
ziq4db6h5%Sl#~i~gXEs{+04s&8<k_~GJ9fLs2Zu|n<Xf#c7u+H=SX-C_ht{nc&oWc
z150$Ppzp*qWgL!LGErvIdUBEVwJR#x9;r>Wx1eA$nzK*h?-j-{0yjr19nH{d`j{LO
z9te=d=>q%{mr$PnqbHXrx;oWnD9rNHeE?hC2=<{w{w!>&h3sG-H>+@?wu9Jw=qO_F
z=;DrXmH|lAJLeHS_5ryO`2kxZsseV(9_4yU9-MVs#AW0d3DrDrX(Px$`jat*(g(V~
zG<ypf{(^4f4>Y0MBGMj6NqNpDLpUC2S@g(hY+eC7)JKgrJ&j1&A-xO#&ho<is1~n)
z+`2l5cY*4FEm8;%wEAoQetQPdN3sMVU2E&WQ{M1^7wU9lD-0NN19i~B3Yko&s*+JZ
zMM9=YKT|)kRw5x^F08KVbLKY7!reN#GQQb36K0vOFJ!VOe_rild$O@w`H!8qk@!H*
zqLO`9@b+aiwS3Z}5?~9{O{oW?ksa?S{iw_x?1R+BA{NL367A@k!i^(Y{ig4j`vs73
zQjM7Pfy(CnE1&uENH%V7Ym<#RZh_U?)l4R!9v2VXR!KhVogd#?!&qV*tft=&hLpXA
zM^Iu^rJ*Hk&Fz;(cEqN-AmKwhL!L!y#!0&42)v2PfYaj_588e)O!!bL7zpoU+|jY$
zhkj)vieU5PzW#&cH5Cw)mgs%DvZ87d`OIM!?o;Q5&R~h~#las>llu%ccoQhG*U1h;
z0qY|qx_DmS!tP1qPR+AvQ#7&c#0pyMt92RTG4ppFLa&P2w6u<8t}Y0rw$up2H>y<0
zN;+mjLh?)#$|f7V9XIM<PwXA(+WBqO^XsOT=z55Y9&KW9CyPgSn$@AD*VGddTHp9f
z>}?W-jrolxLPy%wZ%~-movyNrZq@9{37+tK{$oGZU!8;m6Yk4z5_<|Lw#%?m${pHh
z3VZC^5Hv^Ebsw?@34~I6L9q_bSoAD;zn|6|cYBxu$mxKMR0Xo3V&#ea_M%OxU}SkR
zi46+oEXwtFI{7Idf0#@=g0RZJ*BxjnXT~vmw$N570yX*)J6w{LR%0NLiqnQa)LN=J
z*iS*)X!z|8<LcA)hy;GO&c*-|z(5*OQ0h*VnuwZ9G}2a%x^vPWlpg7@Q}p)hDHS!9
zUJ2>q{{37$f!$K|19gV{pdd6A>|*tE$Pt^fhMxW`Vm{`>jMs+=^VAS*;ua!?!&+d)
zFOb=_MGe*IxqFokY>7A0^>@cAk?>5So(4cZ)_((v1<hBMO_$t9G0+GzOu@;-+645?
zcETUED$tw3aIrf_!|RG871WCrXPp&2Z0e%;$)3eY;oj35b!s%E&LGL?jyC%Cx(;pe
zF~9n>s$)nmmamf%-}ST|Q@!;;z~-=oudeE+uHaNp=*+=!8m&@|@O*DEEUl5S==UHq
zr~E{s^HGd|mF?K*#d={Q=?KgH`YeGUc@Jy_KMA9Xodp`I?!cu<E>cJ%==c;zyTUx#
zX55}v)HTje^#=j0y=pONlFjW~6c>;jf>4@Bp7zkMHG_HYTJ9RQ+Qx0zY8O<SD8|L!
z@81>#>qkp6A|$>(VyTIA>k~hErBhS4ayVhfbu$uW87`5k@YQwiIn7Qh(Flu}ifLb>
znMUsRWATn2GX|3x&mZVYT$pFoZpYD(y5tUg`=U!}Y<o0OUazG@bF|h5jT5E_x$T-T
z=$vT^4Na@+#!qnZ-Ee=lg?JW1(<j8|W=Vqgf;UQhA8b{*n-c^b71Q-v=M;uTbUKZV
zjA&#!&MiSXT*Mif7L$}>$S>j`+oVX^lVgaV&G^qPN9!G6pA`4Kg8y7=6e{6-|8nL?
zZ60g=+{hkZu?6$M+|Mao^afkGVr}0dcf}$0BwnB6X_w_Z!&%R1ev<N1d1&-<kGul+
z@{396I05(aLN?gS{?<0{8}%X-ip-wi6QdCz<!gD-mPtEIlJPTKlJ;O2ddKORGmz>u
z9=wmxh=G{vs14w$$p*Z&p?jPdG@RTai_bX5_+`1_c&Fv$_!e|V-^%opSPL&)W;-|I
z)F_k7o`6~tw4V1n6U>hc6Y@WOsSfE`SVC(UM<LEjl@1=@fuXDgRG?S)+H->}wNLj~
zjC<mmXSwHTG|x{Da!Wb3>(+5dg&Le3AJp#v&G8R)ft^N8U?dD573R0j!st1sx}{Ui
zykj=g4C)0O5GKyoYhZtmgFh)RX>?#C7YtxWJB?mbMFQ%Bp8z9l=rpQt>{Y$J-2(G;
zn{8g&XcT04LN)pH8<zX#<PeAmW`B>z;YBDkGj4f>AN&f_$VulU*UKKHC1_^@SveV%
zkm>kgK7F!lQ$62OEV$<Q<*iYXk>9P2GF3YzXqp4inzAUVc5ofUc`26b=VNjaXDuIm
zD|U|ugV=Fr08HyqhoJ?qlC@m4n{@i{R~o+lam9kKytNVI&6Z+QU@b$ZaluQgi}Mbv
zjiDWAE0f8-5h0&Wxwc~E@IC&8YgMK&$Z6Ho(Ts=VYGMgmu574PG_wpi>UlN0oi1}h
zY#Cm*wFe^~A=L~X>qR4FHis6%o^p%B>iP<fyAk88{oD64nJ{ibQ+SAo+{9DH2W>S+
zgQ^0{uzIEQhf{r(gTQ9LXvE=OUoiI0@4}zQDGj6vg7BH?U5!(CKSs%HTowlA`XGY2
z=rywraTa_r0yleA${sq|c$-thpiSQe?mXr;gQzo~l|pyzc%YQR>lF4<ja+d4$^y9Y
zsAe@t9mJFhrZ+8*51%qlM(tVyHHhqP`F|gDHHyC=jn@6A6>VH8#Gcn&uMr3w<4qzW
z8RiJR9p)$^AGKYpUJZmbe5LzNrt4z5!;;k(h@Kw>m4FY(3Rv`<hxK>57nm0wlj@!q
z4M275A8i}5LGOwR?A{SkQtrx&kgeIh6##oFwU_gSG^zQ<tC7Ls0B72;0>7d#p-_G)
z(o!c^Qib@{T>lqrjS)V1!AYK`pIn_*lZgxf9%K;(V|eUPv{thm@2eEm|GffKWwaxw
z4#2j(dGyF|{iS3)m)a(aU%UG^GD+z~P>uKu)?&V_kpSN@A<N+P4(hoZv=f`*IV(dv
z^Kw3jKZHiG*B-M!q;$BP8aCb?!|S+STH3F0AE2vm&rSb>aea9g82lD+V~f=<S0!_2
z6J93Vbpj3U)2b*sx0rmYL`uKyh3LIgXx#Fxb9{$xXEr^kKW2Umf(l6Ossk;3+aN8~
zw?nV7kV<ttDnF8Q%ux;M;D+KC_st}pX)C(vsp59v6_dQC?_gzHiF(eB8!HNNRDZc}
zvc!%l8EvA{ztK@7%3$T;GRAICLMM#0h0K;o8D{5eV3Q~f2%ErE!#xwV92RxGY<fNG
zo|bF4wjg`FY~?y=_@5^w8*fe5+N4Jm!+Jm73CG+VV^xS{fw>QEZdNYQ#|h-0W!@TQ
z1^D(N9XBU+8;*pq+HH{u%djV!t7{Zn<N5V#-%ph<4x5&)O{~j4>zi^+sO8Jr=HGPg
z2<cEYY`=TZIN^eJYfD~y0guezD8`?8rGNdPrBKkO-H}Wi7Hs!o9`F1>$73tK7{(%x
z<E;YhiVq8f{TDlu^z#Z)n-RLhfG3z=IZDRh>)Y$2-Ei)GaokKA^E}1X{^4BdoSH1i
zAzNAj4={CUaBjvxsg5$KKPuR-hS9~1h~O6u-jY^22q5zP7Hlq)dAo=kg98?5Q4uX@
z)bche8BNOgsDOSM((?FbR!){bX?VT;Dmxw>ILUyDoL0lil;kAlI;DuW@25}$vjr-J
zZ3`ttPQ7*2)4ciEyzdotoIg(Ckzhhn%sa4J*XHvb`}_*%VJ_Ta1eBev(=unwM`Aq1
z0pz_^GWBos#n6kpYrsu=ng-y-Pd!yvOGowK6wER#VHJ_JHFtMkmD^5H@+i>qdY`#^
zob9xQkeOO(UyWR)AvH&ie{TYL{Z|_?3=l2<Y$LvMDzO1>tx4fyQV(=+IT3!O@l;wf
z>mgd-Oipl*y0?60Uo_v~9lW)N@`rysxv*(pT)s(Z51sm&c@)DzTbdqI(~DORH3zK{
zZ`fo3xP_Wlf#oPY#6~M+hNv7^ZZ^*P?)t?k=J{I!2C}@`@o|@afw9p8Pw<9EwL!y)
z19g1xgMO!(<Z8OY17ErPfpD3Lh_JJ4gQ;NBdLYhb;L*-7i?3+_oy$`^9+S?SYcutb
zlr^YL*;uQ~wA<1<LVEy0`SA%8vVtlyR%{>DhuC-)WqjL4Oc^ZzBb7#+%SQN5LqBLC
zet|IT)oEf|q9HnAzZ!ATd_3UN6XcP}o%^93#jxqcqzm>Qx-}cb?Ro7x^m7ZCczHt*
z)m=7&$wCMMXu-OY>_+tR3C3x<DmTHs2dv5qUdm~)O1^OZt(k2n_tF_?yrp;2votQ-
zSpe{yyTH7N-}=Xv-G8x7jDL!D04k&X7n$egp1c!p>|KUQ-}L~j;2<u0Vt2o%(I#Y;
zogqmAkf&chrSkI^<(6G!rQwOdPjfk~+%4)+1AsS9JbLin;IPv9u&j$a&l|s(JC><d
z3Xq`nFDeRW_8bn#fcVFR0K7i_w6w%RAcOy4V7}QXSdi;|A$z2>(UJ<Y&UQuED*Icj
zIcJp^P3C0;&+c^^`}@=6xjL&xW?V)3=Qq3V={hPqx<69%2s}#Gix2ngeMDom4j^tf
z18z$$-BLW_0zmdTZOByhhjRW=9V%kV+2$fMsF#2nb(Qt41i@)n<9V-D_JW6X;bVn{
z&Fet=W}bPFW$@5tVSGcArq}+r6lL}4pYJ|b%D=z+nocyIGL0tPPIa^A$uJ^)tV+=M
z0``>kqA~gr$0$9yhn2{ON^_aTI_Jx3hr<Y1CsF>7^YM9%ZElIqgY2DlZL$f-G)75_
z%X_X@o`Iyy7e@BVOJ4Qy%cQZ#@Q8#;NXrXi`@ZYJw^0`5X3*PI6UMa~*gY%gM8gS}
zae+tN3tb=bW`fV>9I~f9`amPj&Fe#ImOmt#b5oVXop<iKdF`){1qZzWq?sRgV$BvH
z7&MgR?hkZTp;L3jn;fYmx6SFbjdb7g$d#H6P{72%l5RK(5-&7;F=<uKx2TS$Gn75Q
z;999lxdAg2fz*d|l3)(*MxlBG628qZ5elo*!BvlWet<-}?4vZ_;QMC0QW5;uQTl5{
z0#H_k%Xxee^*#>(wDXBCmgf?|XIMgHlSct$4do9ITn!9uMW&1h=F@ovtKm0K;70w!
zvI6eeV8QnJ3&WVjFKrSepFFK&hxP3d4+Ro~mf|~8!Ij1EsA!CDSnNII7j%xmN#I;@
z?t1-~0b2CqJk^RSmbwO{i7~B6oe(N8_yLIaLP#&^2eb0to*I$p$!a5QxPDgfLwz%|
zd~vJ_PObPp&>Z}e$^D#0Y_L?~;grzwz6nEB^f8hvOYVu)q2$-yVSTiU0yii&02SKR
zMP=?XX)lW&h81tclPxKNp3?A{e^dM2`Z7~7VZ8wdK9iTqx(kpDsAz8E;$W>)^{Yf$
zc^?m&zX_Iy7MEkXns*=j3eH<YM?m(2?fN{;lqZH|9C&;swc$OnE^1-9s<K<!N{R<x
z9*et(j2w=8!dN`zl!TamWlRfd?y-7)K&)>@ZI)rY^mz4MQ6IX_*Ru-7*xV3liIEE0
z;~{Jq=9ND1(jlhyTxh<{sg^H21(>(gTG(^HM8q=oBR(ZJHsenu3ErUcC!M)?FS;GY
zHj8BO#d{Shx+ye-5kRoScsc+&t{Gd$#f3WC9F13d^SQvWIj#z}ZimqKa#;Xw(^WF6
zW{9e2I(I<Af$Z9;Rb7@w{F4v~=~8xjkI`5dQiQ{EqXRvjR-~W4mi`Dp;_pdq{--RL
z*4HHHlG{FRwza-f(?odb2V8!S%@`KQ#Yl=XiM2!cjv0@}%E?p+XXhRmw0rkNDi8P@
z`|5R;i%V7v#ku(KQ(o1mM8xl4>wP0~nCC7k;IJt#QBtALSGVFJ>!y<Kv)ba^mJ=;%
z{v~Sm$0tz+mWB2<v6=#;Ieypqhp(rZwmkB``yE`ZMNF<_(#ZCWABPf`DuKN~Ud)Gw
zw}fj$8)W*7KW**RLQcKqpe)N>(_vO>-IVdXTYd^BNM?Z9GYvsdTqJ|HxK{zDhwN4K
zQ^`ZEI5aSUcxrs!_NAP`&V6Ko%u?B%cGRo?v|VASSI%cUS&^c>8$Gv;q^ALSy6#qG
z=zf^u*UTm72O4c|4l9!pd|%*T+^VPz`o|C=^j9(Z`JcsTBEQD}`~(Hg3<Z@Rl0tO$
z%&c3|+Z4qC8N(LKY}}UY1eE?lwX%cTcd?-H!??D7jhd~s8f}QWwRL%-gZ8OboIFDX
zvMzxK#8fI_D|tD$M}b{6RcwHQG#l|M;JVZe9<RfF2mQF8Gx6##d!l)nXX4?UhlG68
zAL<b!R^@uHjkmX{aePgc6~#H`)WwJSeEsWZ-9A3y0dUvqqiGK+aqrrpqZx=D@CRZ9
z>xEdx0m+dw^7#|VZEUj9qE;E_^WE!xErY#$W{XbqK|H@Z-c~cbYtwV`3PtD60t*Od
z)Z_E!abszZ{|EB-!2j|L0gzwa?`vdOvp6^LtX5)~el}3}tR=ac9^>AR!{y(h@JopC
zbj3wHCZkCaT1PM*13Eai$cvoR*j46GoSmFpEyTCBSTR%FIL~g%Ga#2s@rigEI~mP+
zbSc}g8l6#k+qqw*gVgt&8#Jb=lmb~*%LFvkPq8EJIJMyM2N}{~7_&g)74LI8X#H+Q
zW-I$ilEUZuhoR>$$Sf%X7(~w_nq&2dPvuKO(MilA>>>h|hN^iZ`?9hNW^_kp`4{A7
zNE~bhR?lyINp<y2+451GD*-06-G|d2TVphKHNILU>p%={>sp*^n*a>*QMtK%Osmu~
zV&vXa1BqSR1Eg~EmrEmChQIIO?mtQ}+a;zA%tRiQfS~7gcx)ly@lOL*S9hjVOC3Az
z7w~Q>(k+o4`<*}C2yoJ+Fge=3k*QZ-(_{f;8o20T9J>^P^=F-inbpSu6o*r=EGP6&
zd}lAM8w0}VyUz9Kx=aOK28WHYxx85<6ark#{ZSX~HP`pn_eH(Kmxp(hig06x4naL-
zYc*bj?#u1TE@-0iTaz--TE!wdK@|7S<#AExIs)Qq9tcf2kLq>|nJa-#7WVB<qV(K5
zKtR02N00n3MP-D16YI`dbdDFL9g)<W(RS49^>S)u(1n6Y7mn&*P%h2p<y#|<PrDOF
zQS4+>>Axw>E_H+SfNt>bU*t+u)4(H>giFY0jLjnEi#MwWR!W`)fkk-5uHl{Gn67uf
z$N44nw2-@>q9u?k$^AXiyHq?W>nC3tNx6K5@Xm*WH79Y$K0eC4Exe!N|7m<qB&*)D
z8^JDPwaU8;<K-D*Xk=?FfTu981}G}!6CJQ+G~H)pk5^_?jx$-5z}9zp4R~m{v#=$x
z?4LU6-Z>+t6Dr8oFf~Gg>%HGUkoxM4b&H)wT+L6D+W*;xtVe@?mYrB`ozV(aus7Du
z+ShMJ%f1gmo3Qh5sa1$FVn4Qi;+!p|q?Iwa*=dDoBB;5%=dq)-2@uRt*ds7OyuKc1
zS65_~=+=7siH5<1ZFRo$My%HdQBb1dCBT|A^5x3UYI)t(?jP=z2cbURf;rt=9uK@r
z9^3T+>N)>Hsq28?fD&-OG`*jiT)J&F620XZeTg1y94P5?(Q1S`&mYoER0hOHxw>5+
zu(JhooN@O;Yb%QZym;L|-1b$7+4S%z(5WXB<Z5d|LJl^Xnz>$f_Ua4GT<oHt8^z3A
zN+au3ue(y7pAOX;O|idBvKUGUIY0k|kq^Gen6YT3&yl_qSzRJ0X~_5!$GoisvnNIC
z$rE&@_;w!eDR~!x^p6&4$dfye6^f{_Dw|OSPkx1@<c{yM^cEWv<w~a{bRv9AT@p!>
z$ML@+wQc1t>XeA(kIj1cM@vWqC!ex?<XU+ZC&1B`@!W<A5M4OhR3~A&EbXZ2N-BGx
z+9y!-t+(a|)MTDLTCfq$Y~jlu9R5MCNPpwQvtqnz0H{N@hMlo&XeLqu8S0*g$x}{m
z%VgYkevc`MAH4Cq`QR?U<xg?U*`C(v`aePq{@`CjCDJd4#2))u^3>*jV`E^EwBL3A
zV`52#<7;^bvjpBw`Lp;42FhQuKmDMm1Ufjog|lJCvkZkJ<XpaYY1{_=Vh`E43ShB_
z)Gj4XuVH<FOb%&oGq6MaXjbO1Ug&709VP2>(qCZg{9ZsG1n0r@mxw%bMtvhqlUCKI
zX~~o(P&Pd~I{@5rSQH9axGK8mjN6<H=Sj0+XjQ1I1JM3s1KtXCVqTp4L{hq^#zVA7
zmavP!Q!B{63fC%g_bxmm(R%d}*vj($zOue)(4hi@(cLv*6lyqqm89wRJ7VK_K8zYq
zz+s_%p@Ta+`C}(%^}5luT$b2-o<i{VhKCNjj~|0vt@;@~Rw^_d0fhBUp~G<V)!?Ys
zHpu>WizzWB`zQWf>hvFjqwU!j20cNi-dZzi(kE@>Id1IL7YLfyTQ+?&rdy%+Vhhq~
zi{>fHFmLf$<OA6=C#Pi#?#02tD|lAJPtZ<-fJ`s_R8+st_(P4xnN=`w24^MCy&VJc
zQH*Gc27nkbBdb49E{%svGAdv8|8gAdlxCA|?z=zZcE8xwP792DsISXL=T-NWXgVHz
zuFR`+3tx0m-HIEE9%t-#+MFCC729q|TLR#ijlDi_d4Bn{h-Mg_=m@>n{(uu${&g8}
zMdP{p$-Ah{K|s>_M~hO0Bw%lu*ij6>dGvY3@)R{a93LiL|7-fOk=W)S1hrVb4*hg-
zws#-xv2|`Uh=Q&EF=A>;4FbE6*S|1-D7?P6$XT?&Ii<w&g0KB2;ru#R+eRJYcsD*w
zZ1=?}ACpl<;?5H2_wUN7zh|GnL_xj}u2d=XhOn7W++4)2SZ9qb4DOThBjOMtN&*2q
zo$GTuaDOU^g1P7-xxnva)vH-#AbZlRU`67C`Klagi0i#^=qXjOqPQpJ`odf|^v*#W
z>;kZK%)N=a_vIEwy6)G@`FE+*O|c?*rDoQHep<pRew`*}a9r%tJJw&Bz9+;;GZquR
z8H>)&yoXqmG8ok|P{yxcEtEk*W5kBPiT8$w-L2<*ryO2-_$8rkJisyw47a?_-=hYZ
z%Z5S&7z|tc$TAN*UMV;L-CXMP!h9;dStp2>_h;3%P`ZtNX)+Sop3Y#dI&;7{oE_OQ
znWxD+0a!sd4o3Br<hbn{fR@<2{&W~dNmc2GPo`Lxd>e>8>*faF&<P}>eDqnNTnK;K
zA2pP~fNSmVYwdjFPgpGqu=4&pto9d9f$Ufd-E>>o+?LI|_}Wv&=to;WL3nsEx)rda
zcQ4Fg?0eLhNG>c6<sEwxuFJDL7oXXcH)6?Oodbv}vJ4MBIPJU-Gp6=-r{p{pF80>+
z>5KwCedme-z0dX%HSw%-%U&}g;KR3Z{6<%HJ}aj{AU!JP77}u{+4WJp0&o2wh)D@m
z(P2!~ifFyWgkuFPW25zsx7|Fpd+?1vg!Woh*@AvY^40ZQ#VgJn5l@s2m09T70IRsp
zcVo;FEl40tkv`w+aQ6U=kw%F-I}mvtyC>O;tJF4t9!iTp*d-nnZwe)GOaHaQ6K7+E
zzb`zRSh=S~bG}m;AARhsER<NaVd-&Rg#Hmdk_$7WKE&9aYOy)k2`@z>)ThB~c9p3K
zE;bkh$=$Wp#A+0Bi|r@1UcnTgMV6W;{d=PNOMZ-v{oqB?U8>M#=I1R|2Br;G5(u0I
ze_%nbqVL@HBXzjz_2Zubp%0b*pZj6agfzJ=9ujPk<EPhDI7SFS8}iQ2O&jUka_ifl
zARA<8g$McZRbTof3w#Vq<`*~!Lbth`{$hs7-HF+L6M<gaqcV5>JxK4dP7xY=t4>Y8
z`)rPk^JF{E<7l(2^_~@Kc_1}>l&1wjP2bm@n)saoayEcRd`H<|%rDHY*m)I6$p0O3
zzNRl%Gr}*t#`+ylD^r@l{y}0oU%wLiP?_Hov^AR8D#^_6_t8Ix7kB0a6V6<2;nkeY
zzV>5fC}SKgJ>AcfAZk;XUAk3aTV*?v_nF^{lp4&g%Bu7uwwrmH!SbXm-siW$cYyv=
z1rX!2-~^?lXfdzi+QSvA!vh2?&txdVy4L0HTG@bNM{XisEq6a28AH74lOX})C|Y|9
z%XvA0KCf*CwD<VYh+zb28D<1OZQ-sha_?u{UB}y)n#`c#ylu<JiFw$X(mW=uH$`EX
zO0C=ROsDH*W40!qZBFw<W}0rldL&<myEQglKU{5tji*R_lVB{+`7n5}qH0&@_4E|S
zFP>+%ZOb|$;w^t5ueM29@vwG&=)}dvLh0ek@g!F?OK1&*lnfD%gCi-syAd`sUcu1n
zPsVsm(DY+%Pp!h`2TmENziO*rR~~JBW`-A(ZRl!gHA+%FhG%8-?Nr&)PeK~%M+OpF
z+H$czJBJJ@2qwRMv5M{MKgxN&5@a}1B5b$W+qoPz1Mdr{vz+ld|GX2Mc?hr}z1km2
zBgXS0arf&}U|##JL^sOSMwShhW(0-2ekd?~d<AM;9IM9IV!V>{wV3rS0PT?O!2;T%
z=lL}>415C<$8BH35#JX@*dtSO^3m&JbL}Ks*a}#J+b~P?-&ch7((y6GPHXk*j%Rcu
z!^QT!ias*AOW*h$-aCv$ALtrheIl-(ZajgLCAX2NH>mxaE`udv=Qu0(qSNfHr?M*8
zeqEja*Rv!mj)XgwsyH;1qs^IK1JK$p`o45bdM6_6yrmR#?UHZgYmxdM5B#-qdaBHW
z$dsLSxd^Ra&<uPf5-)(^v}cA;H8b6U(Oyesxc{3Fb?1&lR;Jc*yDEq2$;a4KB4yKO
zya!oK^_B*sWP}1|Ir%7qm+K46UxtL|g^Wt{jUrMncH8)u`{b8I0V5r8ULVl@j^xML
zz<UIK{b;}37BZBnG|CfG?XqDBJf}7()b|Iro$lfB7$FG$ki1-&EhT&?%P$*$%ztS+
zKLk*~w6)kbSFHwrQ1NI%A3@1hg})7a{&oiin1*|>?%4H>7pMT*bNNF0x`zSnvDP#p
zUypW4i5F=nWh1-8fLTwxE!P?PJ5Tb2^VHOcmd6y&;V!ZCwC7kIow{f}epGpYiA^71
zdb<sbViyrU<_^7*_3(pHjry}BeX-93d=;0Z;o)-e((An~h>cIJSjzg$dwGStcMS}r
zCw?|4?^Hf?(Fc0Cng6+{F7<2I4JsJq@mtO+j8vSEvr%l>$m5NT8tL9Hrh&9x?3d38
z`NT+;VQsp`Hd-Z*lv}KSt>J8>hv8*{=D1e9RZf2<At@g1-R_3BvhSL%j5GH;E%!Hu
z3e5$YQ=960pP1}EkENZbyzT5O(R?UsTk<)<p~p*@CH4Vrmk~AzZoE9bYqM7Vj|W;k
zB%kE<2KXSH`kvhp;RWkvaqES$MxXL7Wj+%XT%b;pFK_YpL8C6<#gg_PxujRZo*uW#
zR`V9BPbWq|CCE^KhbxLa5~%!h@8iFG4?x2wG4Lmbhy^m3C*yX-x<{`1_SmGsjr;>v
z6LCBcc(AU;+f2u(o>IDMpe|N*hWwEQ*17&$g-Z0i<GTTy|G!(e-I2-))~kNH9xWMn
zdUBez?lY`uMAg4`oJ@Ls-CM2`Om2#m0`kM*-t{HRK!(!ez+$m)fKx=q)DHDqilt-&
zRUsof<cXxIMAXm(uOe1Fx&c}!^P1+mDj<dp>3x;BipQ_H79?;S(e;YfZ9jn4FYC&^
zZACBBwhipB>pCUmvlj4PC0gt3Y-51L*FbbmL^n!QPxTzFfz@<jx2j@_`6v}$As#(T
z>DZdby{CH(x`vIt2CJ-d)+m)O9xktUomlLSQ>8A`{Vl%R^D7)O(d;tj5wNe_G~f2B
zkVvV%s<X8A&v~MuSQ^Ou(Oh?X*<FXvpf*d?NM){f+GlJs2PvJm1tfDZmj~3Y_Ao`^
z%vQ&3^LJBX$Ge@&G&!yh<sns?FknY|&`o<h!FjijHx|A=;$>|scr)W(YqO55RZ@To
zgo}J^qB1MM961ZP4=$s0=0(gz+`^{i7<%82Oqng#=;id|<pfJeZ{Yc!^&4l>m!VBP
z((30Ij~0T!Tb?JO;^=8!`KdSux2Xa9SuLBpi3af3t}m6(ekPzVJYcmKD{D6tOcMhg
zJEt>Ihk(GS2!4ms9YR;o1^5RPJ)31n;M&mo^jA72Bnq+H4O2~n<#zg3$iT3-#k)XT
zHwJ|L<o<qbR_EC~gDC<SZFXR*Bm*Mj4OBF06sfLr@qUY_%spToLw5IgX=V(K;gGKz
zcY!u<EbkS_eNj9&G;-C|-1J7~OZxeb?9Hv^clDxw>u{aKm?@qQfZWg~=p#BvspgRP
zOPxQs?x$!GR&RehG^A~OlzBR2=yd{p6!@%+->%fbD0P}448?gs{km$sB#h2!{4=3s
zreSKUu!jeFh76S|sHgRiqiCagtIlOLJ9F{4(w_{ZabeZs@X_mk>D+`b?}R_Nvo(O5
zow)r`AdrE_<U`dE>s#p0&PXp^^Ol&`1@NJZ;7nsV>V%2RarHx8Kiy?WRjzS=1ii~d
zrhgGT{+^};2Ke`A(Nci}dePb83}Ff$#Fmt)8Y8>k1AqWr+4#d-$eiF<wbCD}g>t$q
zbT4Vbyb$xUh44j#;?%Vyq^hklm4^;ZoaOgswnsjhVouh_PS;jagKr~mzH?efSpMI8
z^OIgP_M7jiCAit0@b-y?N3IN}2XEA&H&<e1&m)g!QG+k|syC~0KeHXOl<vy$;nTkN
z$*;dSwP1jJR=x3=-5JZ*kxZP+lf2HPc!=l}Y6qCFa&f8)#%~KwxHF6HeGJccBf8L^
zHL}O<F?ynNML#AL+`n}EyZ>J8tGjgk-vPLvv)D58!6u%K5hB=aEe>*td*c<4<^kn`
zRh($O(Sn}<$ArcbzhGfm{sb%R_B~<4clkBN+5LnNj|MxNU}t!}HiTYJcfXts^Y&rh
zTWf&n;jbaJ61SSFDdo^8X<$9)8)#Le*Um+4^TqaBxs)kdITdq*zZNOAd0FlOC3jR+
zmGMcCgsQ4|SFZLG-;5d&wNmL$O-L7A-f+%_wV~^2AHq&%To77gH<aP2KRj3(3jySt
z6N_<niHEo|iQ*o&J%?THE&Q7?^iSLih*|`kGvdvWZT{TM#G5BTYQvp5tUz8LB^K(F
zm{}b1U{ZYv?u8E6y<>8F!b0Y5d+=~(nL|*sV!_;RW@@^`kAY9~g4$YXw?e82xb4Lc
znnQo3mbsD_8xI9qWk>A>swh~RV>%6WEt2KbJ}VI7TFQ;Ks_9!T0``5JHEd;dffhKc
z9%v;;lD4X0fOaVxEbxiHO2QwjnG6g4jieqANQ==G4;o0^R>5g3(8w>6>;KH$ClQJc
z4}d1`kx$(JO4_x0OYw4B9<VC0@1DRr_F=a9wy<(Q_rS+d(JQisVhW)BQK0X+^BfIK
zBSX%40LYFC1z)&YO~(i9MT;LGRa@B;jK%;|!3OBgO1J9Jy{?_zfEI}IXh4?1W@Efm
zUfQnmcci69j-plxEs*#Oc+-AkEi9(rFRTHUpjGb@uN5D(UbhFBYDtaBtF(BD1StT9
zxaGcTkRTFhSJrzKu<=I%K+{XRcy-k&Bzkc_=-#OKIplB$6~?aHN5y_=fPWmIt=qmj
zQ(s?d-q@z8Y1m`N`@fvbZ*E<hO@W7_$Z-1xTRZLK{L;k3s?jhKQXf~W`<aFQRzBvf
z{tN|J0;agSR;I(vY}kZF%ai@;VSu~;i~85Y+oLSElkw~qvLMPs@b6T?b84bcO!E16
z>92sEm8+OSy2g5(&BaIq4%65jEwamY_iRYVi=s-pCqho`qww2KBjA2$vR~^7w(d#J
zt)?MbYvkPh)V34kCoE}@v`=>CYHq#d=YHD$h`ttNOY-f5=XuHCs?#;kecASASmHPl
z9Pr!wJ`HcCJqp{~QH6R23dEQk+{o;o7}2ugAUapcRX9#s-)Un|TFVCtG~pu|f3CXY
zrbLfTVU8iUOQn4?sc83zj$;qvaJP$#Mk%I)nqIZ1JYjpKRJFzry%4UX*df7d)>{Hp
zPEl}a>&TT>@t$ABG_~F|0pB^a@=<LDZ)RVyhwZ(0X1YkxB4J<F&#kwvI^)$Re-J9p
zhm<2HA!a-=3&a3WUMpX@`Z9Ms1`SpF9U^|dIdnL!-t4fty_z;d?^@=(f_hNY7c0tJ
zQ<uG3>kxaKB6aDomU=o|ne;YaYYgZ;<2F4G*T+V3)nQH>|LZQQNnYMXAUxsKo%{Xo
z>!IkgIoYSi()}$azdFd~h<`~Vf|Nl;IRT1wxIaiw2)K)&rKhUwn`B2m8o#@!U&)0E
zQ2~lO5H8I}T@MA~dXy6guxDPBm~AL|j9f^)xW60h$W(tApzra;`2_Mvb@JI&fYXS7
z^fm3c+gI@q3&xl65BxyU*T*TWAV*Pb;3V8zZTuPr5F+X#pjGB7EYw``9)f?r<lYAR
zH2%T%kl)4e9g%=jc6BpWeBD}Ou(K0uT`^pbvGB|a$kBSVUqCTtnVWCj9nF0^T<(%Y
zv;w-iLA!yJ=N<T*-*i~St}p%aBP$D~6J^cVZRh>c2Q8m=UyJ{cv{XO+2wYb+S|xWA
zT3O}cT02MaYa$aIfQjR^Q&j2j+@7v_ZiTjQ4SW7u?SZQsOMiDXXeiNZ;ep5EKx3KK
zgxWf0^TZe%aC+wKhA&iJ)w`c~e|*?<d-2FNU4ZadKwW;=T&P7{k$h-HNhgg5urz3W
zruo9=(yyymO>fpiBUr8=YHL|j@PcPKL{h-@7@%);uHuGnHx{(Kt$B+suRpgL-zlix
zEXJHIfmg==I7ENJWPe!baz6fH)HS<H1E|cLW*9=Ts)Vj}Zxj5&PHJ{DSZpSo4&v}o
zrt+4gRiej+Vlh!O;WHEkIt@4VGIAAAM_Q1aN-wB{8U<1JNuo(J(r%vzQH4p0BN;@?
zJXORUn6{n{F(d$pYbtx<-D*6qet;&|6i!xF``ptOhiuK}5EVDl#UC+r5}NBbRu<0U
z{AT$uQOL9E#cFoEP$E$P9a8{GNJ)Gy#&e`|NbB5>eS_g>p^4J(VWb|Sa3tPtUhLS3
z^Aw=ucU5t)d&VzIu5|`0E;b2E3>o<BDF;KZxo7|VyCP0mfGkkEqZrsOS_Z69rh)yG
z`tqRfTc2Q}>tM*cUk$#k0l!NhT;0WSDNlPSySt8IPh3eMAy}~x<FUInQYb0CRtKQ4
zeQ(h$3>j3y7ON~C7w1b&uHN=Wi6iDMGA{jP_!HjR8)h7_rVO&@08%=!uo7<54~#Gg
zT+<yL=H(8ee*5>C7Hd_uEMj^+n-9qX670_6UvKq=>DYjK&K@1eYXTJ@&bkO#kvB^%
zQkOq%s|MtWnO9t~9RC84H}$)!+&^CC?E$z_`iJYIfR<g?9buw29S!Whh$}B!Q>>|b
zRm;R2pixh4%VqpQdV>+@9L5{nG@UGvfm>fqOaPM&+5a@zxL?Bg3SsYafbxA6^%4fX
zOELBdeU4gala0p{!!{y!`1ORbI6PBpfxPRhehPiw4pYtp=wZYGXQ(Q}T>lDJ!d;4i
z&*M86+Z4@7AoyWc6~egdsLkG9Y3IRBhS%Xbrq6)iCj8*`4vbX&*~=JZI=zPFl&rFK
z?x!LZrlcv-zXg@)U3_oisOmv{Uy!XJmBlSB?2ybdXhKVOl>v&~cEhZCcV1>>AXk;U
zfjY4O%b^O{V922#xL&|&^=aS|r&pWF2~e=>#{p;$kH*@740LRuRc$}tI#T_?ZU~=)
z+#0~CxY2+LI{d@X7N7$w+i9htR!B^4R21EKC*~_7)dR#cKI_toaSjI*w58Y|6vH8j
zQ(&f^?r7fT-imXl>a%FDRwlp5Z-lvh8`1fCX_)>HPr^z6c}}9bc3k9GCQEwg3%8wS
z>J2E@47FxAr||0QiL7&(Ae-5g&n}y__2pZcUeMNUmbf*=y6Lx8rCLTF^FxB`hfB8d
z`+yok3pN!hZ!D<)!O?0sPpz1q+w_^_kDkKsTjTxTrHA5AFOzFPx(eVwB2Z1QguWL;
zg?X332XX+C@+6nNaG{O^`&y#<-rMJky-=%5{aLgGj-l6JiJZVf7t<l)TMO(~!Apj|
zeA>$#7h2oI-+#lve^HMOyI(JhgS)>Kw+OMGW22WBo|unoJ8Lf%EzTJM2z2(pV@X80
z33e#wfC!B=8tZYZI(i*pw6TKMff=2A4VlsM1D<blPHzv|LP=C0D_@4c)(@tzMsSi-
zmQoD+{7RO#+QTM*V%;`;MjgN}BBh{<HG*V{Vj!(6NO4C6ndHN>Y@eq8dPE96T~m`g
zZnvYw1K&X!_LDV+eUpdG0lrTQh_22Y`2kWmdS@j-={zW1FI?X<2QVX2Y$OQHY5Nzs
z02!Ql@LrKiG`w&sy-Uj~-vl9T<lB9}gRKgnl7R{_mZu5-L8deTtR$Il4N-Odya+J7
zquu~Oxr6|n?++aUK>HHP(mgru8SvX#q;-t|p`FHc490PJwQ`hvHivv@r**cnMlG5Q
z#2t|Gme*c~fZa7Ej;<RlQE^JOEn572Kh$qO(J|;F^%wy(4AWJ4LoHP{HQ`blVPrbC
z%=I)e1vD@e%2wnz(2Mcc?5@OD_X)J~wc8zvPPv>uzyw%;QJCtb4`s_{7enQr795|4
zfgiX($K1VVi#vI|MZ<niFLMrnY%)2c>_)EOpY;1zxsC8QSz<~4SLeh85Y>`5=#5Zl
z+yG91<_(F>e;T|0l_2u)D#*bVyJz|7uiFg@?bE-syV(tTe5fUeYq0DI5W32wR&zg7
z#a5dvlu6c+Oo>{=#{>vN=19qS_lN4VcxT58K-L+ur_VU#*F5opG7!0PJPgiS(Nl}l
zto&jOMp5jJk)=3VkTv1_kI0o5_{JmGUP#C;u&JNWY6QIg>WR7*E9PW0ti)f+8X&Y-
zk>TJcd_a}zc{Ug%`6fI#pXk1BQS2CXiny72rP+NMZ&Ux>z|6=qE#6!<5wOJibKF=-
zD%MvGdc{p^QLZGa+`E7@e5@XGzQvbr*zE7J@LhJ1+&)7wA-0Cg18`Sb=sdrbr~wCJ
zq$j?xW81uSwLt<7ryP^~o~x%q%TLUP1zW+Api@;ku{dDH^Whmd2ukv?TOo7uUB31^
z{hfnRS&Wg`?DIj#sKhdi06t5X@ZJ(i_(Ot?SdUEcWqt#pM||{?`jZQqYS)J64iEOm
zbMtK=n+H+*G$ogK!5Xxx*9G++Uv$q0{^Je-|2H#Nx}RyyIdnhRQ06^U7A)|^-j_iF
z#oT-Q@xHJrhPc%bAn?rH@zFWs)iq`zSpL8aL>rJHilrEVyw3at33y(C&yn-LB(48t
zr5Cz_TzS{MU)`DHTbQnQC7qm?jh?-ZSh#IOE<{B9Nyq*s9~FK*WA6h!ZwK<RH#juh
z1a%c)5$7Pat)#`gh}?_3y9+;3#SX|a5DcR%z}`h-WK+goR-~mlozx&b!g$pCCe2Zl
zqTWEt$4g5Z(ts-Gfv!vbb!->kq&sw!)ub%j*T1fQOHJ0$jsNv>w<Bn;gjMnUKw2Wy
zDCPm8sen=MqvF@oF2yGvZM}(URtluR4%(_!SZw3Z5gB!it5S|MDWIq%<U1Ob#U{a$
zNb-HbY%!b@aj-nlJZG26p;OThv~nDNXCHwrz$1B@yW^|`d__L9sbL_k<a_pd&dXO{
zcU1yjaqk2-7UR9IU+uB2EBrik)urUzy?Q<T3qXO<d$=9TXJe+}n{XNSCJSEl#5@E(
zn+ksA^?GyFqnzJnl20@7{rt`fmFdEa4SXuEZ#vUFMy%0vCjU@DkK^<A%+~Gw=m#B@
z0f}bXre&|BIF})HA#q~1a<giO8}$blWxcCwH+7fJVKe0)>RdCIYtkPG`I=e~{9&Q<
zt@;cgx<K;lRF&<4MLA&b6b=P)%7)Yq;D58uf5K{ivGraue>C#`an2`~F0iU2GHM?t
zkfbtI;x!!{TlY55&W*rEMPffK<g@=n+|5){08yjfO=*VT6k+7@v>!PNm~~wovV9R%
zXZwv@)^it#H(C#M=qT+#)7I9qqo%?>q^TFQIzB3Xk~QngNOFmV;J_3yU&Z+Sq>Nvw
z%m`Ikufps)>+$MS@>Jd|(1+gP0I`jT(=(rUTX|dsbg$%;4)ZTo{m2p|IsQHIM#Aes
z|8{J~H5@z?5b-JOaldAICCQJJC-DUp5jWK9FvjcRWa$tvhb}IZm=83))wbcEa2isu
z0eV3=9to~ZS=0W?P`J7bHy6>BKM@H!1yq9TWX2m!5M5<cDztSajIJKN6Xexzg|iW3
z*vE3pYGr3hP50d^r90luF|UuKwrjztVi%SBWQa5TxVQG`2uKPoR{AQk)&RFglejz#
z$6V4o30#vZt%-J7HrPD}wRVL7m6KQ>l3oA?>|JOy)hH)_0SZWcxEUPXc4Y%m9{=Ok
z{ycQf>8tK~S6p8W=%hgUCg8Zg)ph^=m62?hx7N|R!R3!3P5&e3eG(DDg6CqsSU6k;
zdUqy?1L_$W&aRzR4RDC^BOJel@Z&JJEq4Qc5)vek?{wUe3HCzj7wzVPD+AprU?M2;
z16ekxZx|>C=7p>JLkV(X!tC!Wz6o<j+*%E*HBUANaFf;~B63qtEM}~au?QY^C2j<j
zY>k6oEaF;O!;g@=DjQN}DmD(zz9rz_*=(gHY#X0ri5-}l->du|BX_j-o{IRZGm~5;
z?#Qdh0!h1j$Y>*%8$qViViypEk*3$d@P-c#Y2iq|HkZJn#R8yBXk+YK2)CK*x2rp8
zO@v&Wun3fmW)xh#&<aR+?Xx7!!uP%j16xk>2eK!WV5Y^Qa{AJ?Y40b5!tW>i^4faI
zU%;DkBX7tx<DF$DM%06MKA;)JK0nY8tlV5!vl)(ftDqa6p~yoZ80@_W#ypS*<Tjkk
z-pTnBE>jQea(s>;&{~Ug%tXCUrX?$Y&5?lnt{&7R<ez%BQg-;+IvY@e1DvsYb{%`p
z)BftS{{5B$D&+D@=Z^x+XL0$dssA;6tWZgi`ZXFs3LqF*mAnqC>8md9!5j9n`L@g$
znyCLu>KUuH4K0@W^%TK<BH~S0xy^!-twT6(tKcV-0a_USlZZ@y;papFupIMnq@*nB
zrZmX+$zY(75xC01sWDJm$nzCnql{R?YbtQ%@@hJN>CdjOx+#eZS)_1>IW2PYWogo;
zZY?)dmJ;blr8JrY6PcT-T6sI)0my``>s=pmFs87SIMoYivkJ#nBZ1(72|y5N0{~$;
zZ&sj?i!s1%Kt}@%WXW9ee_Xl?HF955Alv*fZbFg0!Z%uC4<Q;h!Hn@>2@*9tqO#uN
zIC>PVn#hg0rEE_N3j&1pKUAn>#(w*84mkul0-qtWBF5yG6Lt>|)|B>@a8B9pkNbmU
zY}a!Sx?B>3*usr{(6pfVUj*a7GK5~@T_IN_XGMnHgwiZj;^O`^L+`P7-AJnp^IeVZ
z%<(=9;Mc)EzP`(w%jrG*P4S6v&U5QY@RM-Qxx!ol=UFMn<yBdZ_0N_F@pN8xwk!*d
z9cCm~=x7WQ3}OH|6n=01Y93NQC^egNgTF*l;4=`cktFcYgE-A-&Ql`Lv5BV=%jXLL
zi^CiFlZ9Do<#Bv(T&M#ockZ_to&Lk&Mj85#q2I!xN($~zcdaNGCWfKz$8VIL(E)-?
zBoKp~_WaI9GkZM!!vl4p2i5t>kSn~3CFRqLaExV!6o=dJ5F6Cfh{0?BJJ7)@eU`GE
z%DB25&dd3aabq_{UCNtXgv7@7%T?=2J@l*0vhnW-8>aQ>H()@EFRLRa#%7exz}kk_
zFR1jN>`>X=*6y+d0({_7yO`q6O%u6D=n&>y>-C!*5Kza$V$80plXB;ub!=BHXU1dV
z!%T0-HMe`LN~JoHgQZrZY3Ergsd-4KSk)8z-RV#XUD+hVW>V*Kf;!9%#=`fr6*r!_
z1^!#XYxqI#dNlP&-l5t~`06bKWS`b|07YczH9@SHS$swSx45>P!Q|~pY)@H8k?A@7
zjgFz|9{u%$z!qTO5cfn4LxnfPIQ1vPX#al`e<V4D_tn9>!whkrD3;>ffL<oZ0B>9t
zX)<Zt5EE|T>ikf0kfA|yUqhP@ZHga0|M!YW(kF2~26RVJy<f>QtAHRIxBXU9mFwcu
z+8V4Wu?t!CbDFATO#0}iF!TKL_6#ux^0~btCOnmb!;vQ_9$yJw12@)wmiA8&SG6G-
z8k={ydBtB!Er%c9tr4*VoS_0X<0bF5efO525L=?#%k|GqxF$5Og!<^wWZ>i3(m2D1
zj=;=g`^Xru=8Wj++f^*U=j%T$s1+A!^TSh*dANqq&b+HrjZD`98iIM%^>J^DOn9_R
z%Ptr3q2ru-Vp%Sgs8ua$2cuZk%Xog1U0Rm3lm=Rtj4DY70>w<T$=&{_QGW@9M4ZP~
zhfRS<Vi;#8Ln}oV)g`2;sFU&7Ztzlg*-!M($H_o3X@}wC?iI8EE9@IE>?XiT#Cmpj
zhX|oY4)!T^IwrOk6F#n}3<=er^aZk{O)tZO*lyDtv{R4zdJ}-2p(VrS)rjg(bMT9h
zLv@8zZ1mE~;aWMeAW)F|AP(L4kn<eqVcyx<#TQkQjhKp;!B}}tz7JLKk+Yg@aU+H?
zw{n|b%Lts&^5DhtdlRsp`DDu|hYNWUJ3P}eWB`mf3n{<C9CzR{f$DXf<SwiCkF$Ho
zK3Zn6@p|tD8us+p`2pV$R77xIs&6?L@a*Q1`hUd*Icxl37K2a#rUQ$SV+{NQy@89G
z8jW6>g1cQBSx>)*`D5&q{gnL-zUPRUk9^GHPPoq`(;hN@;=HerMlh&C`9%Wz+MoxF
zHyh*K*YlzH|0C?JqoQitws8a`L|R31L`o$_q#LC~1Vp4`2nm@1q-zF58iWxLkVX)Y
z9J;%^B!=$pt^vNy?el!M@9($P`~B<Eh3sqZ^E%G+JYu^$zWrNlbU6=HtT&<hc<o#y
zd?6CNT$DfFTg6lj?a~5X8v(Kpnd}lmJ8UJLrj@LQJWIWKkjqn8<(_v|wsmPufYs_=
z>XC;NrpdYMoy=aNb)q3;d=46LdRD7vwv9N|m&v`^KkdAxIGkdPv{+6CpRpGK=h)}(
zD*k8OOm(B0A3&HF`hID;z_3a2bnSeU-t+7^u%kWna?{fQ^u&5hTZjKn6qC5GmsBmz
zYD9Kvc(O^eB)s#DniRe#w|1~Jd6;5mZ+-K-L4_59^{{~$YGf4ql6@eE&I@G$m=&Fd
z>Kx{`9!B|!@iAxCGf4bnR+Lt~m&NJ^`0}|mhbx%96I~v0xtmV6p(epW85->D+S2m{
zEPPZ<!KV_-Tuz*!Vq1`=asx~7)040E=BDtQNhDmhg!pPG?M@H~HuxZ<`jbk0(ktB9
zwUjq>g|3Z$q6^4&jt%m7A2Ek_px7hf!nu4Esq$!Ydw<vgYuJ}HV>Zw^x)y;xCPso?
z`>Yq~zdkbRkGgL8&x15+v56;1#P&u?Txz?+Tgr*LM?EobOJJMh7=iJmXapab?$qgq
zCA&^V3(4xuJ)#1m#arjiiSAZ~_qqx~EQQCcl`BWJOJ-1BACaZelpCLG;}Wy?F9dCm
znKt|MNEBY9JoaKPW<efp%j!=j`MpYtdp3nNM5cdgg>UTd+|POuL}`92_}B8E)u20h
z9~5Zv2Dcr|JBH#Ve}6;OBu<k50$p2TN|cQ_0As{KzB&Egf4P!0OhD8B;fqG`Q!@(#
z+!*7=3|8;@sLv1m>VPyP=DNq9kPkwGtDPx!uy-2ql(EFzhUa9wO$yTSTsN8?3kZ+?
zr1W@kJ`;Av9w|7~io9=XbFazAphcE=QH{iONJCWEnc^TrzwwzCd+FM2kdhl_v9U|?
zTdekqe`9IOi_c?$F>Iv7Yk6I~fUi#Wd}Dry@eZZ=$&=EvZ?#xt1YHE>Kf|q!2qfC-
z-ynw6#B7b`X`!a@RAY6H%S=#YfIeS-eoIet7yO+5-t%uvTYP{Ax%SQJvYI`WL_VSR
zW~R6^KasxM$0B{W%z>H*EC8Qu?egJpq-3LUlrx3`Y51bdtBcb5ij&M_DnBAiUHe1t
z+wl+K@s`^;$4YHam#6G<6%)iB?=F(R9PzjkSfz&np`G1FN9opV{O+)R4~6p~^4eF&
zy}|IF%e}qoRdM};_2J%J@zt1jhX)>s1?kg>v3n7m&tT`<c^(6;A4=n5&hG5b>G*AY
z$kE=_Fp;gW-+q72P-Rx2RZ!NI0AERb#<DMh3NBhzH8Wj>134Pk?Ek)xe@>Nm0y>Bm
z>165q7PX)FX-)2E42t@>1Kgajj&F|-e|u{Ht-R-9h~)Zk>Uy;=)h`d+j*gn_E7mHU
zzrju*ohPY;?z;uHoud&dY+|5QfD108pW5To*x;Y!f;{Cu<xzpLxUgJTzf%VicYb@_
zNCVl$*JLLxH2S70io3^n@*vYkqGm-+qtM9faYg_*SvPw~fPctL9kI*36A?mP58jtq
zj~QEufE?bX8bR8THR{`JK$%}@r_2mJw~#J34=g^g3IBxiiSD(#VRY`zNBP>F%G_rr
zefouK;mG2HKr>%>@|j`u5ZfiXj9^WV3YQjYyJD0g_0CUA%sfYQKvm+BMH1z_rwaOV
zCq3pQPu8AcOA^O(3Ghy1F1xcyijW5c;T5&s#ekY)w<ne)|4L-%>g~hVCh26Jcn!o&
zv;=}}v?)`0I*HXLIejNR&+enH#t&zAUo(yDN$q-BU-Z?~ERO7ty~))s?og^)^}e;)
z{J!jXby_WJas=YqdU+CC`;NUO{y$3THPZhSM?J4J{ZXJeUI^6nZW9cOUC&e9y4F(w
z0p=Qx1L`lkvS4BzAJy#tdI13cZ{8Ai?XrDdN$l>jk3eAa=@u6b8x}d+Cy}z3GuY-y
zloHP!%kT!c8yS=E%A1V8(#ONHGuUhMUax1#!I~b%%oiJ(q%ch2%{9BcA%SalrfFgZ
z{fyA_sA}7%OdYAkYlX|0rq=ueHDi^28a4t?-fKq|)p?6Znrw0s#uqPdleye)dexgC
z2v}YFtt97}<y2eDr^Kwx#pu5Ft~zOJp@G;wzDMhJ`b*rMdsgN=NeNtFo5ZAF4df}A
zBb9YQ9zz<O_Lhw+%SgbF|4#Xe!nIQW!Uwajyrr~tyD$&-BYxKpv}9?3tan2rk0f~V
z#r9g5+UZSg=>@i`ZRUPg+HLEq7F@yCmX9XdJkUBR)33H(+pP&G7gm={vFCSd3AVaB
zjwpxA2`&=V9H2YDVA}TXc5WW%C>NS^&U?s>NYm$^lRK__Z5)2MFPY%tv2pJC$!?Nn
z`Jq10F#CfOO8+5fN!}&<r$D~D!QrF@2C#~0b={ggaF9t9*CmorFt`nPcVz9kFCP8d
zlfALN1~pSj);m75fF^VW3*Ge8*_(3ue7(|xTn)XCkZ}92;Rx*Z0bn>H%Ynf;=(si<
z(M&@IQ*7UehXVm92wSF;JXWQ{(V(nP@jt5eWaK1H7~!D-NK6|GC@251tc%x^#rHFE
z?nAA9e92WSnoMgl>(W}`pgXxazn(Al|A^4>3p9f}n9ph9e6=T&u{YR2&pB`G_2!_&
z0E0<P^e7WLLe-&Pl}f@ph=bPks6aK5F4GSr<xs4S=x$mbL#ISF9S)_h&3RQbfAIuR
z1AMIL!-{U;og4prF%N-jaU4-`RIJE9(Ny8IVO2=gRUlu;HttP5p97=55+9Dgy&_&$
zazD?9Z#t=5F<PWqobK4r=VxridI$UeHP}&;l0*e(DkKTnPAiqPI&NmFGdtgO&!`r*
z4ZXNnJoWrZQ+o$xd&>#-^ZKcV>T60wGJVDOL^#6C#;pc~ku0@=9wJlc*<F_x^+q}>
zbsi#sF9B$~B@MtB#+AwL4F49hU!>QE2Kq%M*L{sK$dxD?G|BKlqJfWa{u|nj!5L*t
z@E4GblhsV4!?*8Xs=?-QmF6EF;0a=T?k=kfsyut+VPk3Ek#)9b#>x<Ip_wI~Kgg!8
zdkeM_U?z(U5%le$j0D1&{A%YG{c_Y_?2i4h$2ii?A~{DV7L&+=@<DFi_ia~q=x8Js
zHAq60K0m&p1%!5Y5nN`sw`yY*$^r(tn`fyH?^XA7#_;Sovkt$EODu-xf5<u8i%8Jo
zYh$bIks>*8_5S_Q!<+K!9n?Rc&z)_)gPwoQ7?M5`tQ4y_5b(0mp_|<i+PyS<+z!LG
zBIg3(o0H0ADVae`21~d!SDYVlWAL!gx5uWGgp%xnNkpKRw<=ddNBJ#uU4rNAynu%p
z^m2XzaEi0h*N}+#x2YQByslR|RY4irIM>lBd9im|=AHsKf;hGppOiGY^Nkh8!j}tR
z5}B%~+CAVMcbOr3+Qzo^d?TwE9)Z;MFC#q35I=K9+06id(3XDok&(=ZygR!+*t+Yg
zE=0+FA;c~(iXL=G6Pb~u3dty_Afgyl|0d<vZW)K!GYqDG(?>`^k}X!~yU~Rmz~|`M
zMDx72jU~oFRrw^d#=hd|#(dmHs9-Hu*jizfm<@@<FJxa3W?mOy^Z-^5>WK>~+3<Ys
z6YyIvK!$Rie*7R%>z^Ax@bro6_FrG~Biw_rUzxxj^sCw6KE{WAqwEJItm~u!PM%ri
z6Q=_tI6*GfG;-9k)aA#NB`yYQFngU2k><nM(bpumk|A{+EtGzBwraXkiFyCSSW%aP
zU<R?V1lP5Uj{*!+7G(rLJtrVcr1xwIlC*5&d12BY7U4Ln#{+9Pz#!2P5H(X^D0DM*
z?igU>>_Vg@JpVcaiR&}Cg7&=;F*g9#uGdK<m#qv{{wIwBCs+i9FQ0Vz`-AgIp}Y5g
zwnoL#e8iV-c;2cHzr85y_tlH9aE%@8W|09o$o@~%uXa|#KT*H$AK~)=QNKCW|G%hT
zCPo`e8RBpS#oynL05vD*p58=N-E%O6Bp^Yg@?-tHa0eK&xjQ0bVv5Xbi3(+3*`eIU
zgEWUxiB&f_4uyl=hGQC^Wkq5Jr6&9=Y?S7N((O0;v)nb)1V4JXGR)f7N;D8^25S5{
zh+`Vp<-ZO>@cJNtu9Z=zxY!N6op|TUcHnL0a=Vb4)qAg<K#ZK6V_||G55{Rr+OErG
z{!2Ndd@R4^#y3@^e6iz+-!THqIWJ`O5r=A(t7)-CdQhwJlAL&8;K%R0Gm~AW+sdL4
ziH>)g`T(xQ*yd)mZvlAtJ-8ZF74RU><U@d-tu1n)DBO&zG<|v@WK8Yb4ziWp%`uOQ
z{+S2JjxFX#sj|S*7!dGa;iZ4kd^Bz$FusxH$4vbCWX9oqBRKtuWeXv1M8mx7zx4>n
z-0=n&Qkl4_;r7>HdcHyCJ}usr4k-inG!XSW<0UsGxtA7p70l><;|Pru!|A70+GQxx
zP&$3q=~uk=QjT+I8K=L@9i5gx$Nd$Dks;o6C<-LL7i@+RN}v0soPAJ^A%!grKG4wB
zC=ZZZh#EC<?~t>DW$nc@`GlQdhex$wvD65z>|DDC9HAje6ijdI<|By2b~CTt1KNTf
z>6@9M7_fAyI78BrCme>c{<eA7S-&->+dl?VN&m_EjSK+0oz=<hW}@7`@$3`UYK_l`
z)NKRl4UcU9>3JUS5v^|i>ro{69Jo<afPlH9@oKM0d!@vmTRI@Fl5Gg@4{fpA4DXDN
z^|<TCO1rCMW=MD^0(?aV21~D{WJ(ej_TxbGH4FzWE>yq;2<<#^HhpQW+5$42-S)c$
z#@SI-Z}X)n3xG?4P!A?y5iV^|$~le(hipa0U3fsgji);G{<=rv<yu<QskCWm`Rdt~
z<41g&E$K%34;D{=&*-u3A#LfTC)>EG+j|0Dp4Q-ks7vSPsl)&tc<cO!1oz){NW`^y
zb%cRMm=b`88|lqYlEUN!cb})UJ;`?AA#{<3-$0NN3P@UWq&DKyyA5}J#G4aygf;Ql
z-h_mzX{thYJECb(;y*QrO^3`_!F;=&cUONk*Fkoh`u$uD)w}av_j>A=wXx+Y>q{3u
ztK{1q(}4CUs^`oM^(-XW@O5+0;}6w0Jkuk3*}WG|%Jb08v=r~bWRA=OC>}EbtFT5~
zlrC0B*Bi*14&Yrc1eodit05MDhP-^%)uG4%4X`bYA{A~9SH`6tG8-o7l>Rl%`o-N-
za(`wpLzPGSO6wcJp9^JiaX;;^3#IP*LeV2LH%A;>j*o3p9Tnc|<^k7EU%U^9x822M
zhqNf{16HX3OA%}%ORwf9sj9DUWnFtwf&grBj@sE0HKVxnHm-;7j8eXj;&U|f8+rui
zZ^w$cIc7pLi2zVL#6JDBcQRQfz+nT;BU+P_Qw$pwc3Bt$II=la@E@F-PaD;92hC)N
z3$NB3hisakjF-U54|Qt>guxIK_6A5dj}C8Ph9MlCRvxj;ct^cd){>BSaNKmB;(Um3
zvXQ<9IPxzv8dg!0JijA-5}*%(Qzx`I1OZz6vs%2VzX~g_7=RRhALP2IDNoc9@K)d`
zYOR16uo`G(@!WFwq#k)P)y2Y3c^cWsMwvM%X~kEf1D1my*91!WwUHv0y4PdQ&}y;N
z5F7f1-fZDH!8Q$1{L`n#DI`1;<2z9Xz~di2c`F6Qq^1bKc7@bHSI+b!q-ik$b(vcB
ztgmp%7L!yKO@jLbar&DZ{D7N(wo&|2=8e1LT*G5FyW8sMv==0J++BOdIcI|d!W@(m
z8!H*8`$vOD{u30DtKzwhG>o#JLv&XbvcwLxm<kG!>xCK!f1b=6o@u<I4XffF-l~$x
z{xXnj4ZS}&`480n^KRd~Mq_J%l2Y@28>9WXr>nl6d0|2=MAqP<M17HV^&Kl<pbxlP
zE3of>XF0-Wa5i}1UZ2N>tyt@Ru>tTFmFE&%pIkAqkcb_;5n25iJ|GORsq(}Q`lMEC
zGPr<e`VIwe6n4r|x6GD92InBobANCnZhxI*T8O?$ls5i_8iGqf2V%N|vL|ke##xFQ
zpDDPj1^U&*rcGnuj8Y_W%avl+8Q_p$&G(R5e?ey6c5x_KC%P`8_Zt9#gt%SPqP*;1
zW1eK{<^zhAyT-r2astzeHjP%dznqv_0)e7SvmTLiQBL!~(-bav*9j0w=O$M6?U+XL
z0m^t^f1Z_Q`k8nZDY_fgzrP`;?y!SEw_eJT;7)5Ucroa|(1jF#Z&v&oLTAG@_e+L^
zu)Nvf4Px~BZWI!@6#cJ$xXvl%MZJNTsev>7NcoC;H9(0^f<9&2N;1gtb#m<CLxJ4i
z=l{w56}%6%NAudx0J*=>K<@9vKe@kWH@~6mD;{(xBLdC<mQt3LD0?luor=hD&v1mA
zu{lX<0W^{>chvOAT2uS{X&Fv8jypch%p?6j6?Y1{68~4l1&leNDHY6peHW{<3Vnsj
z8=(vf^~+0RETxJH#}3t>o)#L^fA;R%10v{I7q`piN!G_Az`sWnljP$vcjf_ddLh5%
z%u+1kWY*8Ioh6ESJq^<cJY2}d(yY}ZOA(XHI1ZhPrVd~}W!kNyS<87`bZN*D!!y+4
zagyAfz*{lL9GWU4uLBJylWs_VxfqlUn~iH4L(B}z*N{w}jn19&ShhrMkCu_@`#%Io
zRpvuuuGSSnmSwAC(9w!&#dC~aEbrRk&(`AV#@LM&4)#cJar1uKvYmoM&WoNf$N2Dz
zu$w%Vr*AJQN6$i!HY-PcX-P^<yH5xVCktR&(TWL}(1-|5-BLT8gHOD6gH^WB>M*3l
zFsEIE9%O5KV2rhNfhlZF;RgZlJMj%D{l*DGBvlc!FA=m6!p2kEno)2o$bsBk8&Knm
z35<1AkWNgD*>70vo4qa=r4%pBNDS_Zt#;|8fNF)ER>n^c1-e7p$E#$rKANROsMnag
zPw75V7;`APm$e@Oj*D7aFCDTPHtxHpK2TH087DNF_Qid~RPlt@n&I+GWKpj{aTQ!%
z8>MsAo!vVVpQBegHQT!mn^$>Bd39_$MfcXWMxWN&xSD^IbrsqY1sWezmh-NaTIyig
zp2$_`-D%fZN-32o<hg3`dVXbDL(`L=+8xW^lNQ2YXqO(+O$(a;)EMaoGVyXmi|ob=
zIK)*uPOCwiIrXaQA|p)1d3uit(gC;co5{LM+xKpQNw_<s_D3uJKats70P1X)X*zz9
z<VkL>bQ=*&EM+|?!{A9-N0!LQLFta=R{gl8N84=*(_&55#Qx^C;~vwTa8e_~d4qrf
zicue%M|kZ6edGtk+#IGOKms}%kjmw1Iz-qxNJ{H9wZrK9LpLxyM5qT;@&VHHAh2JX
zkHT2d{Y?>BvMBFQoYlSsjTE#!!HteD@yfn=`$Yc<T84P5KTn77ZrC99<^kwq7_ECB
zmI(%#;z{50t{Rj}7vcyPTw}^aUH(eZUXc54Vx}y<`cBpHXhHhdkBTmeLq&QxwHII`
zQ$}fyt76l076x$mnq4lcp#K84M;zF$mhVPzX_RbcNnJ;G&rzoo>rSLyMLG-#YG@pF
z?tc^tqE?R`At2ls9qZ~L*Tc1lZ_xnezgS?C2vFc}F9sO#H5;)Tw%<414_E>klgm{%
z!!@=F<4J@>s%VGDXJ2pZ?KK~5Y}Z)Zu`XXd)Vo`9aYWrBxr10rquiXaW5^E?!)s%S
zXGDuST=bV13^%=NO*UC~EZL?m)j7VQW4c-#E%xD?m{VuQEYXI{TNb-*$)eL?q*d>V
zdQQPC)kb6(b^)D+HHp|ZYB*y=ww>fzFT2y#vqT-hEMv?e-~FN_!!ZG9X3*$-rZ24`
zcf(<CYZ{ai^djAp-VRMg`A(PXRFsxb&-N<~MZBOe|AkodjJK@SJCRVLc3DY1JzD9!
z8?LfI)q9uRT-biw*e$^}pm?*&cZRz4`Il47rU&}GcZxF0vW;9qK7d{oJAYxQ65Lw=
zwHv(a(TzDJ_P}FcIHS)PiK%s2DjqJLP7829$nq%M-LAE8ix=l$8EU$*pRQcwF;R3>
z%c8SoS7ce<%MDnBMo?LcjnrKHp4L9MqHURNr}^A%S9@L#BvblDdfJyRXGHnx^M^ix
zCh(jRxt)d(;V^=)61@(p7}xrPLJF*}rh|unxJ_r%!B~-J$m+|etT4H>=9c?@nvhY~
ziMFWCO*!}2vIQUxYrQl--f^s1>v0S6s_XI~9W%nw*)Q;Kn=(Vd`}`U_;DTN6BCo+i
zt8YoGPd+bTC$MN(M?_|Gom6dHr&gCJP_c2E=c5THUpm3L4}UL(63z5djKkS0Xz|0?
z=r8^d7bV`v^Z|KcR^|j|R|1+@MZ`0adWUwWpp(a7p#}C!KUn>@GkG~q@9ML6y(>j6
z>9Wb#e7E%JnaOZTg1MOoc<czIg*~6p^PGOe@z5hU7$ds9XQ07w<iEy4nNBy*YE!^%
zJmPb^^F2zxUsVI~$Zw5@T|+Z1KNBf&YBvu^W6-CI?-IKfm}L<eNXO>_PXX}Y{{I3G
zfS;2u)xg;{#VO5Fz=K+`Qq*SUdJOmlxXK|@Y!6cf-*U*+0ltpeMn4^Sd3oDNz%p>s
zs%+L*SVBXJeFX?DMZJ8TL^#di=agBc)(y^TnD^1In&*fWt(X&$U!X9L^b^TAxUb3W
zvXEGsV5QoMVdEqqG)?^N5jCO6s<x!XLF&}$vRA=0@^G5Y=`eQqq39Dr>Rn(AUrTlM
zjZL_UniMhya#~zpbzxli=okA!A?Tw<+;cC-qNVOr&7?DI7t@I{@>%j1aOr`aR8?o#
zZ!MlE&B`ZJF09AWzj0iahPN!g$cd=Q-BJYHLh8ZDRZ}VcUJO@%r=@aNKGk$LtEJko
zu{d`3k{i9Dj>mr4!EtW#$=z8c7Ua!mFFDj$`*cs8S+i+yW4NuwDcgkH76CdrOs(zk
zCBri%E>F|ug4C8tvnOFu+f{989=nDXTjjgMN5$KE_b;n7E|+oM_S*?u!8rGb1B|e{
zF{_l>#=VyxE+?AN<ls>e<RzDa9wl}+n*8dPi<Q#ey9h}hlMc=Ec@VfGbMr52wrgeY
zZJ><rPJ0KEnSa8K+5IuXV(33o*VPy6xQo@txYEDN6{Q6X+?yUJXjNgu0*!+oDV6W|
zS=S}-ISxr8L#pRAnRfN_V=_>5O<C%3esml8S&(Kx++mzfDqhkIkwO%(DKT-Vf`*#_
zHM4k1PQ5b4-51~dT7(`YOFd=c#u`|CfOfdQh;>JNUiN*s1kaIRI%dvkVC@EN&Bh^7
zvhGHxF2o{C;S3Eqlrd$*7lsG)Yumj^#l1i475qa?yh$PbEI&c`WWF(dA{E#d9-q?#
z=_@iE|I}?6IC-_hUv>Nbb=`i1O$<&#nnRC*ch<_Somr|MRCQc3SJ9uVp6J^JPT|Q;
zmRc2iUYrhGmR!tqQT<^5e5ITl4U7O%L~><&9I7|~>n4$2pl56k7ML<eRUgHaC^|Y~
zD>g@o2T2x=S+0hi9?cRtjmAAr)T-sIvGO_=6s|H}<ji70D!ZP{A=KS+#hP(rzJG0;
zv^4#4QMaTFqOF->cu#9p5)@Ii`5s`9dF`OXjd3z$aPQC6-rxNa>~`g%Lrrk=qI9a0
zfBdlW1NZ<xfHhLA)=BRK^7x!J^Ck})dR~KpF822!|9OJd_~zC8``$?hUr3&t1^Ys>
z2C6&TZn=27MJ1kw#}sZ@Xr|moYr7ah>fQQcH}u8fVgTD#fxt98_slawn(iLMIPZHK
zlcJVapEY6=HqjmWmQ)_Mi?+VMpfvx5&RoQE>_2uzYv6q>oY`SiKPjEBz&9K9klWQ-
ztze0g{9S(h;9R{-<kgeeV>jKh_0A&hZI-65Ryg=<NU=4Br3%)*CJ*&xrPG0>*Jq+G
zbgJpZ_&%*qETNcDT`Eu3o_y!2*Rg!=MVti4)V018YiEw%CBvDDZb5{+3!<3h^Tx#E
zVd|F@qL=gxhX=B@mo+*YMzq_QoP%Bwr0QqsaCUA_JX+~m?pYFXo>up`uOIlYTFBdt
zSMzdo^CuW8PgZ6cTkWW4ZPe)94oCPgVuf<fMZ?4G?)`Be_}8Yh2<0>UMt>rw2|cKK
z<aw2n0zU1g{yS(B#O9L*cGrBkxrE+)PqDiRZ|v<F?sz`5esXdr4EflFvWwPkwTak3
zwoH}Cagq?PT97<Dz_`hE15U<-SdxnVg2Y6(XPpI_MS5taIbv2c_-3W^hV`NFHboNq
zmvPF6`6`L}&8Rxx8I*H;MDzrKd((q#wb<&lEDigGw^m0(ktbL~B>MD{QWnxYtiv|$
z^pgC1A*A>bwMMvK+ym~g890#Jt`qUjeN!mE+o%r(u#tJ1gl$_c-O8cDXOhajc7b0M
zV!QsBU(~4lSug#X7&XX|V8|IfS>nM20C}c8Ss$o0C4&Qi`FImmzs|qjg`S@olSX8H
z9`x;CpY17`jxU{B6^mA3*K&v&&f`plEWKPN>E86r!qfaP4SX^t3L2yBRLXgwYM6(-
zY<l%2K0vwixHweT){Z$Sg^fNF*=?c>F3$j#&`&Oj+W-=#=Fe8H>FEdkFVCzlhdye2
zpN2xglZUFYC;jZedh|WHxjL=*$@1pWl*BMu<wEVE{&XRD3NaShl^FfYQEczUIIL9i
zD`A1f&?~ny-}x`Ce9Z$n)At-_dbdfp8&1dT!dkg@PgvwfY9RSuJ?<}c+&-?16!tu9
zoRvx}0J;yBaj={y6gB1K`TAxoV+pLF-Rg%Lg85D1AmEsu53ZTv5<6=%R!qpsb$ztZ
zYmhIiQ>^xbke9_1%cj1FW%x-~aj4cT7q7s9dH8bCT{ArpE={I``FNLa{P6EH`(Ai%
zsr~4a%8dkSepgYaj$B<^LSFpKJ$~p&u?9tvkyd%N^Y~3XEfx)+<Egg1$eG_f)hPfy
zE)n0Blp_7ApkKRf5&KN|T<JBLE}qB5$Mb0-(uEQlhUuaua;0bPn1MzYMD1R2BW%JA
z^|L8Fp5#2#l9!q6{b%4_qiS$*(N*QpJF|kFC%EXHEnfuBj^KS(JfhD7ObuKvZRWK~
z43-vniJg9qZkt6{HYdS1r`W?=g;Gg`f;LG+NBPI+ze<bPO+0d2MMhjr&H#plSctzo
ze?laQug74-lF)~Yz!;3uiY-R8A>c}_o)$vv|G0`9NSv4K)o!A;_dgf|YzzFS2cam@
zruM|*jfJpZ;&FJ*p_F2vmt;&T$gUOYro~&XhqSE<>Cu7~fjW$kp#YbAEAKQ4_wu6J
zTEz7`^;3kn-lX7WdD)VB$4G@I)_dMBuftc%G*kTYNkeJQNF_m&4PqbIs(jdQ_xxl<
zwC4T_`B?)M2%(({WX?Zjdq~UeZ&6^@{W@;X7hof}TR~3d<me8YlimiSL6B3wD>*1m
z1HHNL|ML<d!0xX1w7v9lyj#l8RAqE!bD}(};M^R~gm5b<^2?<*m=rP7Pw3StYX~`6
z6pVeEb4kGJ!`w&by7I$uK%Ol%eW_njw4uEjNNK%$bq|0uQ*KB1&Sy?VhHv>3vK6ud
zl}l&X_Ew-ScK0{s>5^sZb3^fwVtckrQ|Sj@@23Kh@TKEfE<vZ<MX$MaHa)0UNhf@_
zXFjnkNRdMhav2y=MD)7y0jS0~Di|(&9?HqGDLmdERU3ys-EHEV&Ug7;^YYk7+kS=2
zp9;Ybepz8NKUG{%P`n-EuvK1L8p+AiRJV5TtP_%3x;rnCW2NbHInJSF)!N00>eHuh
z+-@|EV=WzQeYPH=3&Ev%a!hb#s(b3iFkFsUVv6@sP#-%jRV@8rcfwI%6hO#3(Z4{O
zt`{kMb-b&y8FVF@xRq6)IgJ}b?om70QgsMZ)0Rg7U&Qa0Dr*#GHEd9Sbgqs4GA686
z9J73J)ZU3)a+uykLs*@B>f8SOirk*Ql_OQ$qGVRK3rYK1c_r}`Y-7iX)vdC%XQgyV
zJ{`8c?D__czv1}c-RmU@`_CYh(m#tTvbgI-)q(Gu=~F*tW=bV2H^h_2Muhy#Q=NX)
zV%U&Rd|m=cT{hx-E5XkWcjdn4bMKvIMb1T&&*d#{wRiiaud#Db)+_7c!v~3YTx>7;
zGg|Xr!xuXB`R6pVKyy2)5JN3DmHaW&w+u0ZG6O#boneh2k<STY_~8k~B(KDqeNB>)
zb}6b{M%unNoB>=#63g|#!M?xY+0AG^SNKqrZ4mmi{$b-Y@}q^OH0^;5aemvcw6SZH
ztzX;Z4j29O-J9(2V=jS#EilHy)%)K%Rp4c{7Kyd|sj+*v$VIKpriJ$I1f4-p@T(1Q
zBc5g@e-(ybWPMHks9_xH@mTNq+UG1HnWRzpe)s0}D6jSl1`x7t3@Cs(ookz8`mSd_
z<s8_GWp0fZ%bPR|i3R+GMzH|>p=5kpPCmKdW?^=6tKcGqIYae<PKoo>K_IQB<to}Z
ziQciEGGlt&qxy*pj507=Bpdg$VyS!Vcv_`9+iv$rXRh4s;%>)Y(XjwIIWfmk_YeB_
zL>Mr3^IHNhM;k_uc4XYx)?Zw$3|l5SY>WST@FD#*waNanGW*Ni!-)4NpHA5H&AXUI
zo$h2(2d@0%@$7l<`J?4C=Ds&+_KlH_tB=KU6y=wg;?^0yn{311o~FbB$XR__bhre4
z{_E$w$!w@065<LUL<~i)hf6lbr!&-UcxUoaxB$OOLe*K(N4X5GP`OJJzbypp3NjH!
z!?^tsKTbMX<jJmCtwLh3WlH?9;%gJ4<*{j$s8FymIbc0fZHCr%>Z><P1nC)1o>{jH
zh<hvxxgA@VvaG`;ZUKo^<0aLb70{z;5!JLW#ef5-bxm0AzcWgJV9?G4R9x08_BV`J
z*GmT4azewM=MSg<^)m<-5UL9J79UN@ylX-XkrZ!g&!x+nW=_?`J|M8uRX%Oc(Zuwr
zLM{u)J7WYU2j9?Lu6&=L%42G3I=$n^P%mq>`{1WsI{ETj&gR*axP>=Yfrb#{A1^I^
z(r~fMi`3^kP-1o*dn29>H`>_aF{AU~*x?RO5Y#h8C`1)uPvAC;O^toK7VChK-jNss
z7$6Ka$%z><;eXbz8A}PzyBC6S$}1DMK~oEp)?grG?w17VpI(Fp)i49Rz);9b>%Zch
zZhiiwPdfy#tb&&1Dx-^yR02fFE<tS{FTpQw7A%KsZXj#7qBLo7Jkx-{KgaoC0qvlB
z(WOs_?s4S(R+g9qP2SP-VC$1MSl}?m?hag3*u1p!TX23~at)i0gil-E<J2!*+Bvna
zQvsB3FlXWH@^;J!ScCdF6q*IAwEzfqonWUf+YjUAQS>(bb{SkzO?FKzy*lF|Vif0W
zD3a1EQFqHVoi>^6saUOaCE9z;i`iQ9u%)j2o9)FlVRYiC2}k`dIZ$yFl}J$m)%d=O
z($ISZrTL1HU$^%7O}6)2MsM}7yET-n>ds&Zr-jYNfeBX^qiQ0W3&rAuyoIcA3-IM^
z)i3R#NT9y8z7=!izGAeSd|NRuUb64ht6gdwg6}Z}Iz1%U8v8D(O76Lw#YS8#?&b7q
zt|i2|eaeORJolYV127JsgV+2|F`lLXQ2ew6WEF=}fLWb$BM;;4OM#p3NowXHC9Wi@
z$zC`Q7_h9r?i!MgVD*0?5lOU7{+%8VxD%?^09mp;^R(f*O?>_ri<IN<g6RASo~#L^
zbgu+3K{)m_wsbG?9Rn=5p*e9I<d{^|#FdQBgc_sv$<`8wTs8ankO;v?ek8c%UTL<p
z!#{S6SCVY1BsHpRZyxjF(U+gd7}B?fhbt@kn-M%6dgnf*kvkJ}vtq9NdmN63Z73*y
ztxz3df3l`ZJB(ISt$xbWCWF9hczxSWS0Y5ieU5SL+Zj_F_AME%KmDFNV>RNhevja~
z-_yY+?w%MK%BDocv4@URMT=>*2jAe5C@iHfw_e)iYF0|Rc3+va?c`K{`!?*sTVP!U
zc*mx7$BUFwc*i6F0U>uG``^pn)|e-V+0Q1p#8)2C!|>xT3@j>-sm4>+oq-R1Z}NiW
zD0rk2eynj6Ar=y-rl!5j)JJwQNpxknS#h9b`^EL}hTHNkG>lIeMzQs?Q4cokcd(Kl
z3Jlw7A7rE}Urj7^MQeW5U1x8<+>aU?xwkuNxVg9*lCyE>c=pJ2@^tO9q+QDoJy@b-
zoisp`0kB$aAz&xOk{>Sa+NobFv48&kQRay;g2KF|_R89A{m1R+xq+L^RzNqoTNQmY
z$d|71815(KZV<as;Sm8hP6uY$43|T*`EC^<guL?-&fQb4I|3P;PCyort~7l6FEwx7
znH<pctb^u_a{7Ql7SJfb0Uw7Qgv9iIBPp3O7X4=>T|BFyCp}pF(s2aM^I7ohfT$f=
zoe3Pck%-$6n}%*;!j4)M?0G~}oy!B?9{@>@Kp3~$Rq;-igD^;FH&x;=`*N`GjEF0s
zjjP9JV~w=ZHE^T=rKDONAnil8Xs9mL^(iOqW|d?IKFRJ%1QSO%UDrZ;_YF;5f&>3E
z14WV{gBIV!HOeeL{yo6SVddJ%LFmL=_AnxmuEQ^$W{@r9hutr6VT^;Rk!F@7j1Ae-
z#Kuhv=(`~pa-|yq5AXe9GS&D`{go7P=Qcpldx8r@0*081NQt1Ndk7~&RjQJEnat=X
zyL*IL?eK4OrR(wc>(f%-7dT`tcJWpnFR|Pt^*He=ww@MWb#D&5dlUr0G_WegX)33G
zC^G>F@4LnA5BoSsaAP3b(Ie%oxG_Si;)QG|k!RL5x{F^MdkgC!BK#^~cF6nQ3FjBN
z3R4#Y1vh6yxp_tknxzv?xRrP3<L~t(y!W7YoYBtrdHeC0dh3NMK{W5GN6E+8G3i8W
z#JV|j)Rx}kv3g&f$K^gaR=oP72joA0uRI+JrPXVYV^Iq~R0POYs1V0@_rut?;y-K$
zKmN_XO()T?IxcaM<FLl}U^n()SLEmnp|Kd8@wyzSP$`g$*y^hdv51IXDA14Zr-c+Q
zTU~1D*Xk}F;U}8LZbhCJmR7+7#5~r8%DG&QJ3H>-H;e;oAvit_Ti^TV^Z#S=6o>~f
zUhZ_KX70;g^W(~}>b#%c^Zv1YF-wqV`rsc2iM%DsSEhovRfdt`z&_#mhB6)CAkm*5
z+hov4)vsON$J3<v1_6)oIwNzr!vvXYvh$zMqwM5h@Ur+N?rLA23~zcdWtb)oU1M6g
z!|q{vKr?6vY*@!*2m>tGc|ighE$y6Qd~?Ml*^Mh7Px3>aFD6w;b})Ki@>U;aq!}4`
zaM8JX--pi&4*8qSjj}7b4Uwi5x$A}#1(804Sj;x++vSGr8~iT+)tcbhrbGg`uUwET
z)xDkg(?Cy*^jH4XKsP-2(?H+8jY_s&3v(HXKvj-3==4LKX=CL=PE6TokLJ7RA3v2i
zYY;s@<W${v9*)mU!;OjL)L~tq-jejn256<eB=i<3a>(4urJ!G7*DC}~ZqKv=IYDBy
zDd@cAn8(TTR~MKyw<{WxA?AV@+ImvhuY^CUfuxP(t*~u=R=FJ)&u&$5D>5=?d^BI9
z>fTh2*;I>!`(i!N1LBID?M3DM)LUk5Y%_DW2Gx8tyIU~SmHP8%H8D+*arbAtWbrGE
zq7DN?egwOQ-r*ZTqopt9B@Ua3&U0J<_hNZ?QQgEX(S&_7J33<GGAl6p<m*CAf~BSV
zer6&RC`?CFKjJS|^17yr>1I>OgEmM#I$3QNM6_Qgjn1F!XKzPVvRc`te<}93+*&?;
z@nmSzd4;-Umup%Hp5`t*aEUlJfu5GGqPt1NJr1|+Pn>3Fe@tt2$^}{{xByFw(hmik
zxSpn4gCBEavmA3h$2}Syh4#|rHzU|1$5W`(gU!>Y<^i59H_iXJ8<Padf$r5tzqDed
z0=ODOh|Lnn@T}pdHHoFDF8?;xs!$m=5q=Es4Eaho7x%||WUB!HWZyAq;Zh+!LX$0q
zDI!d=p_q6062pTdG1uM|6E+^`9YY|IUF-??a>aHtAEK)YCp`7*V~(U&xtQ30pFMPv
z(Wu7so>Su=?}|sI2mjZ*;@%rN*K6+z6@=$Y;+*y$@P)V53ypFY-X^XpT1-vkjR2qB
zRJM&AVm~Aa2HC%aF@XRDotKz^fZN{F9Y0B)I?(K$|CMqBpaXOBsE>s{iQcif3*sHC
zn>&+x);KPvH&AtU?|XO9!8+g-Xt#z3<N2Nk)ff^L^Pb)Lu0HZDCAK0Ef3$I2dsTh7
zHpIS4x7T%7L)VZ=E+o4(oOfdYXs%Aqns8~VfT?38Cm2$tUgvmzg)SLAK9B}Y_p^-T
z126OSq8Avsy!uTn;h~{ZQ?w@n|JiewgG6XqWxZ;aHNc@uL>Vo2RndCAm!jU@SvEbn
zSvgc8doxv)50}QDw>VTL$kb0n0r6cxM(SZ--wYB!gVqlQ24!V^M0zJ^w<J~RwIH}L
z^!!lEWrmT-7}Vw9@?^h><0cM|rODlp`;_JY+J7{@77HJy5g)o;JK|BzT$tNKT`KWt
zqS)lPOHapCna&uzP!3ht?k*`sd0ZvG9>aSLM9UUO(VujV56nwA?sAtHX}MIOdW?XV
z8razm>SG#RB1%-%k4*H>J@MVaRZBC*^<Q2mU{WuRYr!AZ3;&)VhDctMy&gV=NBc-#
zH=>{zT3S4i`cCfG`t+~6UzOr+2;!anq@bO^>Lydm`=J_=_Zk-@U1<9iIkA!&WSk1Q
z;NlN*_DWnc_Otg=s$R<VdYmP3w4i}F%5Or<NO6XGE~FP`0_D$t4ay1WMngHCyq(8o
zd=vZ#AD5I~yPN0%3|rHiT#xQ3o7{E{JCBVtA7i`Y&$K9o1iQt{0Bqu$%I@fQiK4(X
z#tvj03TcxO!xeeKaR{&y0Ot7m+??O9w4VAe!8HDBg{%<y`)4vC>&9|}BJL;q8QK5<
z$GsD%`-ixPdEt8gZwvm0?3%du2A3vgjZ9x}n@qpi`i{N|m;m*&)!PYod$&Uz2`iR|
zo_8)@<S0V*Yw&CP6*kVPl#2>%^*@iG2E8Ti8i@ta0O2dVg%P+M=Yx;V)=?wd&uN?;
z0BR)M6Br>(g!+gSE_677;lpDNqfb?O>8?V>?0)Z-bG6r`;nI9ouDaA4PtsblJYT+Y
z8PNBbX{h4M%*1qT$&GT!VV>APD&E&9Voub?&0BBZ_#y1nnqg55caIl7BthDoy%JZA
zaG+#`8w61;*Uq<#pVJv~YVW8>A9~k{0DMAuor3o&hu9;t)fZuljtdhp<c#0Wls!dx
z(rA#@53LYn=KNdZ9b97Nei20-aNfgJ?^MUpNuF$YwD{Y%t*6W1C4VDk493s>G*Uyh
zCG1Lr-8Td5ArZ~2mGlN8ba2->?Gl6Bi;K+gkz3>Q-bC5k_oSYk0k7tddL}=ym`fLs
z$Q<OTCq)3|kdXNZ*h-DUkW#Qp93C`R3{Q7L&DNJ!<G=hR?RJ}w-%rwKa)`RGt!1(I
zs(oM-_&Y43&;Pm}GS!q7i(V6YcO)2Fe1VLZkCX@X53OmmpgT91#o={~BsZ*?O-MVv
z?LH;icWtyQh%ZViGt9L&&q!g}Fzu)zhTdcg-bUxQGeib_jlv9Mfxe&xarpP1s?vH4
zk#-hoEE=e1W7>ik7!%yd;uXYxi95pL$OA5(#n52h_0xE#F?NuH{xR<$FbHl_%AOO-
zBVhrJ_Tli_WbdkpEe<B!$9$iU3nX3}3_;H2LxV|^<8)zqWI2mj(KTd;aib=QgrL!A
z>cGZC-2!NDc1-r^&15P7@qQOBcDH6xR^{T8ll+JHv~=I~=f9sPFT?da`V}`OGnpsJ
zNx^Mq&2u<Mb&2JSDm-$`hU!acIgvB_+pFs~4oK310fa+k+FHOEYO|RN^7^=@4GDbO
zd-I?9qa+LAs}%D#atcemuf6ark1fy7<VSv<Q*8@&vLg1hE09#hK;p3wMwAxACyu;j
zy`zy+6t#t^y|sS8F*)TZxRTC4mk|p)i`s7H{JBotFd3s%b|kSHb+9mD;MFyLA;3^E
z*iv=jyjCerg*by5(hDE_QV9^UV;u!zcr5PdK%TeuJTA@YJP^`2tZ+`zQ*g^<x@ygE
zeQOTr!gDz{0)M+{r#nVZN6GU<jr%w$mEH_GT}iV5G93KT|7|VmMN3v9B=n4UYB1(G
z?-6+0u!(%uV-g-QfvK??n>CXR2`AZElE+~^w2@;qW*4T!A5uJl`YItfp<-5g`v>uh
z5$LhLp{i_c*8KbKNqkGIarim&`3&e7-DQ(f00VXc?`P@GT=&L)TeeqeY4ObmQ<@v`
z{*UZo$?ogfa`#8*EtTu(#}zXTsHNfQ$reNuzgp$}k|MLUe>1cX|LX=dLH<d>$QKYu
zza>9kPZwwP8<RJ)(D(f2L#=Qs2rSGQHKmp)=A~(LC;jYs1bUwCB#$kG=POZsPl|5G
z55z}#=nPW-qD>$d#b+~gk0smD;Ep6o%p0zp)Dis$NmK-T{fzzVK$1z_BT{afy`g5p
zH0Ffwf`BmX9i$ewsVz$DGq~w4h9!Gf1+luD>U41Mv-qefNi}f+pj_V0<Q0GpA>H0G
z7y<eZr=;>}Lf$&Hsy`^PYNSu(Z<KhwW^qsiTGIS-G3P@p+A_nXPV{PF)HwGtvb3G&
zg!GOBc;@5bkOu11mZzvMRUxV<)TA>~>r4N2z5-^yS9#NIySnt^q*X%cm}99YKDM}b
zvMWJsyhms`PS+uP9H8*eDI_Q))dQlKVE1|Qc`i=iXbovM^E{l*x^&bFoA=EC$;U=4
zASe7q?1^XF$*oak!Kt%9EXc&uQj_@AeChRRmvfiuh`WwvVz*=dFnt9J33wV&5_~ly
zQ;y=>J`S(UF<W)iI#XlUKqHnn;=bA!++P3in`1)FtnSlh=EWnnEnyy;xvRD8-WT+>
z%wI{A64gu*nWxu(Zl_#!ejAkiPUDt*(rRCc){BJEJX+BH_ijS(&L2*%OR4R$Mzpv*
z)1;2YORjnwC*#$xv~E!ir@0>`D4vT>7@BV#>{M1~98NgXt`C?o>Ey$s#T@%pw4TDq
z=4$L#rH0{$^nNE|mzd3E*wg{xFrrSQVZZLU>+IMELrQv~q#okD-_3(kv`D7CSexFs
zFxw93R@1gF@9i&H6mMm&0q9f39#fe8C(W}>E+myJ;!v#t{@`dK_iu{yVhu}*v{YNp
zL*o?(lW14Vg5YASX`x-0`AaBlPR=^@k^?BqAEpYLYV39|p#DxbmZ^;;N@VQPXq(vH
z56D^jL8Dw?fb5InCC2aHnk0x_`~v8|i$9}O@CBY?6JO1n8Qm|(rI~uPQt45Kbz5AT
zeAzR)KDGnhBED~^APbhTHJO6H-`ji6>K+)poTLN~2t{t7nHOY1<3;Lp;@Q-$IJc=j
z>n}*rH_swxf<VbUkC3*nlwn^Yh6^bDl+aCwyBYak%gB;_Z+?ymKFl8C!0$7Bm#BH;
z9uA9G{sNh16JWn0=$U&qKuMc;x7syA|5Qz<I!lQ_eyx*K%*jhdac+?P^z>$Z$Rl4q
z_k*0XOs{LNkk#EzI7J~Ikl@<!Aw@E&9EjNYUt90McE-Cb)7Hl%j=zNv6#M_Bhd&&7
zb(o_NGb$E!SZXN%jX$v&`9$d(Qrd3}JvI-06_neksjY1Rut_e2tA@XKc64l<%%Aph
zQVZ?W;cNmTfc}zKb}laE^iC5tYI~MlO4qn`tpzvcgoN|-;mO`fqQ{A+;+2uKxLtA<
zv8ONYB)*m9`Q=Ut2Zf?f^Jhan+?Y$9Z4<Y6xUqU*hVdozs_L+aWAbcDl4NZNsyZL4
zq!v0_*gD=7ZH$Be8Oac+tnFPw8V#`$;T$$|znkbud0B~2aI2BZT9Aqe6k{*B6DX-V
zV2e{JC|dI2w>$OR;#@(5V?Ut#m?&DK1ltkv+V>iNFOk^!_AO4OKVXmCd}s19>ElS`
zBx&}eF4JI;<s^1AYbc&l(vMcF3xPsZqOHxn0M_5|%S4CgNk8`Zr^Wm6wzarV&HqbZ
z;*Ca1s=gg);VPR!YWZnaIeY54S@IL|ch>(;`jX)_eaRbqDI78dOnu;W{NlHP)Vdl-
zLySNje7ZMAtBG>zO#_i84IUh43a=B-154!6|6rjG+|!ug87w&5MId}aY=byLx632e
ze$~KDG!T1$2DElBN$Oo9-|lVLpo;q;`8#>hLz5^H5Z)OeznzUjiJw^z$wS$BslB))
zzvtihf<@5T6s2XeR}YfF+@*~S02vT*tbQk7>~Xrc$f{2D^5U@)Uwdl8AdwJt(+l6(
z<h)i=!yM5SG*j2Y8$Hvbd_CZwrd8P>>@!i2H};s)*c*U7{w5CK)k&lfWhC<l1Zbj?
zj{i!F@R9z*e*P_5Rd9K6im*h*yS%V)cB+6y#5;~kyDfVJ5xHbrECWGgpC+A+kEl)~
zShR}FL*wlS<TE+}rYzJf&PKKZ)nU{g#^TsO$o8(eSmhj0B+PZvQkviEj<>jSn^opt
zs9g4|lv>Gwv6jI5b=?4BAFMt*;4T|c<abFPJB%FT(#flfeQKA3ff41Z7Cfu{?wt@F
z>+(XeRkH>;6uMG*%3v{Q$%NL<u@!QzR0eI0X~kAMBFK#E`)oa4viUw<<T(Fm)9w8<
zC-b|{b-W&z2OrHc<ki(%mTj8BUXi(Zde{M`SvxP&m*7t*+6z8@n701iFl8}5z*^dM
zi3pm=EaJfRWW|h)3bi_x#@B=~>T~JTbmX3NzCK^+j3Ec_lbZ{^jt!ZF^-9vLq6<By
z3OV%(`ebX5yb*gSx&1Mx%PlQ+V0b-RA{hyg5x?werZb^Vpe^de*xhnq@%NzW69dPd
zCmVAjeO(EHra7%jYK$@uDESKYkmBQK*2Ow39#hNX3o2LgTVaK~WGY5Nw&;|{jH|N{
zdWP76Lk(pbnys{A)dJ8}J@jbUt5)9fF6WtLd6uPH-Ere&ZiV*#(yVdK%64+^v`f#_
z#ki%sIl#vPOd2im=?^Zl2LXXk8Mp4g<^$KBb;Ghisf8NP#oWHuI5*<Mj2kuY0PR^F
zciZBVBrh=8_)h3bJU5SP_kwD!CJ<6=>U{Hbc%b<()mZi$TVdxyJ*&jpfCwMi5rH+@
zIB~)O<C~xfh}lEgZ5Ut+&wqI-9;#_LHh)llUF_31qCUv%qRGVD{8jNm5bwi8bdNT7
zg~16U9RpVM5YHu+g3>`#fjZ}Ad#a}C0mTrrtjqVil*^wbrRa^wOpgMk<lGPa6o1kV
znaD1r=x*ebHv(x_<3G)8S6{E3c-NvR%|9iV{h{}lp;Daw8tM8x|MP;|tx{KStXP{d
z9Opf&8oN9_4BsplD5(9&-qkLukbmv^8Hjqv<++sKd882MW>w@xjCS4LL5Dr&-}0hr
zU@~1>mT<A(;O=v^7H%4I+#m;bfn7^V=MP1d77;qnJWQMYo?t5!O468*7UTeaB&_z8
z9g$pmVYgrM2CHcM)&^B_LOzJ8XWs^e!3=n`VFrab<2L2u@)?Uih@K~%*4OL7rsH{#
zr1LEj0-~awQ)lM#!6I4yk}2L~a?UVS<Nzn-XN}Q@a(=gl^x6IokhatYY6_+6z8%;H
zk5*;OQ`ok5+&%kJ*pP>3J;SYW4P3`?2W~l4#ef);`FJMO(qZ*+c;jxsYa!1TZ$@ib
zWHPotkh<tNq$crbOypH6{;tp87x6kFFtW^MGoXDw1}H~`_$7rIzZ5(@z<ce4oIQ7+
z>RaBU>tot$^<xzGGD0`9%;*(tbae#j2^F<O-Ct}F$OV<ii@L8w#8v?!$Kgylu7gwC
zCca`*_1Btd<xR*p)o6&>+lv!8+W)!SZJuPYBO0OrDEq=2FW%-D!0d<Sn>gNEaU0PG
zP27gDx>%AjxD?zSsm^nGM2`+V>Fd+(yGGP*Y|Im-imv!eX)-k}w1kJ#e3{TaX>}c9
z1Mx3Xs`!Z=k`KxG(a(m*)Bw6yQebyNAsTq(_s=%2U;l;m#J>&?xBRcSaW_4euJv?@
z3?pKjmwO^<;}^ZW6X2-<Swz16b)(q4qb{A4A=tnfBH1XFRDv6`y*1h&GMaAenpc^S
z0KIaqbac9*4G_CHmx7XIAAmfUF#Fbp!#U;c_OgJ>)^oRuL$jj8(SLs|@Hsxf2cGf*
z*4+3V&;ROk0TI~8+b1E3)b3c=NGt}dpES54{Jw&OD?6%Hwl9&yhG5&<H7os$9c(?`
zYdo!oTAlYsn5b|n*DOdfnek%VxPe3YaW?l7S1kr#uj}f~^q_{=oEj(|w!%D+<=eyP
zR%}KC_WtU(1n|j|F=%^p`2CeUo;>acek(JE(J$i^@FSHZ^N{P%?k_XxRhdX7n?YGp
z6(5HQ9W-llT+^=_%xE|%qdvk=u1G^-;Bm-(+$&w!&htB2Qzt01R8L8}L;%&n|F<Wt
z<|A2Xczn-s`Yukkb#G)z87wt8KKnRs-O9h!8W6;V2~S<3IVxvs=4upVT<=2$Z@t7a
zP5jFX;N5NDXFO?X;cWJjEL8$l_o{B1$R5sY*y@wmX9|fUIp?=Uqs`t}U(~V|wq8}&
z`PRNUHLf3yAHw!bE_!0MrMf@>&``#T*A15A>=axx_x4z3w|FFguJ-?0T>VojHEFNE
zf#3k>GXmag3fzw@!0We`bB1u|2g~$nP<kDQyJVGbFWEVz7sy&y%h)V3Pj1K1ARO@Q
z+9M}zl>`_K`P|EE3n*Q*ng*XgQ{veRWTNwJ!{{Sv@wt5Ycj17!!0G~0QH))@wC3zB
z$>A4Bun)5j1^pX8Gmt<|<Fif;zf}CyJiE60=0|A8?>j_kbcWF<hQ?$}1Gf;r+%;LT
zjFHCqg>pL@Ra|7Qdk@j=-Qdr*zgT%3=XMNx-%JBs)<+UG1cyUlq5OUM*Se6Ju7~$$
zJL%2waIw2Dl#&Qv-pT)?eA|^d82(H7W^k>1QzPKL<0EZ%Y;s{a<C|kK(Krw8E+ldn
zC~nzhK$l&2tIZ?7ft8{0eB|5<EQ;>tX`r|IKh(W-RFrM|HcAO74I<5e2&f<<lF~@2
zNJ%4&bVy4MAWFlibfbWDcZ+m`)X*(0IWY9T=6T*H-u?U5x4ymB-hb?WSS}oeb>G)@
z#&I6!akvi4JsRD)TKN>Z71KfBH2w(SRgf#=Fu;7XsoQ3BMJ-t410M*dqK1Qhf;Kna
zoqt~lKOwZIi3_gF5GB|SMQ`_t5vCwH8nHw$#(IOyX5(W1@0vJe9TC~Gy{aD}HR=#g
zT^~yXhnyBrk{v@eE0FF;A}cguqpt^Ojm(PN9M{7_R?mfs7Nh(dHUdp=my#(BAmaCA
zIvO>;vROsmsbc1Pt~kbf9|HqV`PygVzOW*Uk!IvG-2XRJab{enY#~Yf1+Ww-WTWmT
z#-DNmSibA`{z=JcYe2&!J(t`<(Eiw-%C8Q(ZJtP|<s=uMi?O$ok?Fd+RJQ#FI#5N)
z;nuaih#v<w*%s%pz|##2e-%RF*T0=OB{R2UchREPo6CY;+GW>&Wmr^S)v2*IoV9VM
zb7>PNK_rd$i8_~G086Ds$*<!#Cyrt~t`7gDZkkO9j=Wlc;&3IJ^jWfC^#VVglkk(&
zJSLMpxtqXsh~#ZL&%HbI*X9F>QZPJ@@=usl_qKAH!;-7L_}5!g2gRzhE<;#jH<siG
z%LEB}X%X2)_1(?+vp3<yDLCG_<Haw=x(hr|UxyiePO@E<GaFP0L>*9LT2GZ<sT?dX
z><V%6gx?k-+(g;uFHxTZ)8Fm_u@xu=4i*o&0riO)XT0vR%727zk~w%9e{yr&Lc?tN
zj3w*&y1jfm?|8VEgFM4Gb0C+SHo68UH{}&K4)C)s0bjc1g7|;mhUF6b1uWX?2$cO^
zh`<{C^lUhG;(3Emkz(2zzZK{ZRQFc4t5#Afk8_$7=x_DKTfy}j4chwc9B@J~{61^Y
zX|1}Bo>iHgPOntve!p@BMKC^0`^KQL7?uL&Pmde_4C{da!A(F?uzVR?rxR#v2-cvb
z4oq4G!B(xE0Zvr$i|ZWqbvJ>+jIY<Ak;z?tPSr<5)?RH8!vc}s5EaEe(nGsj5NYII
zhd?19*tVq#p>B7#>3Ex{dYg?>L9h)#&v1U&uFE&X(X_$ARR~#S3TRkFj6aPkmu=4T
z9|>|Cx@o<>#0JD{yWzY#1j@!0F+>lwDByaYeQS40^!9o`DycyX&Z!Zd{^6la$z1sS
zW2kksx=d=kptSYMC;er5N9Hw3c*Ln;h9Ka;a0`*VP5YCa^}na1>tQK0)9y+uFoHA2
zdt!iz_nj=;3H@MKKaR^w0zDLhE&Y*y@*^z``XRlhD);TM^u<YVSa2f+>;pYqMU2~u
z7;o)AeZMv+!47-3Wlx!55+Ef(<QUw+BrG(crK$-?{9~GMM34+`hjm-GtY|e+Yv@55
z0K#GvN>Di@ZN(i1o-2m3QS#QK=FkWkdU*rBTa*P3??|8W2-$l*<SR7GX{N=*A$wx%
z+u6dT;c?V@50_3zn(Bk7^|m@4WK^1JeA)Q+a+@4@GN0v35QrW*<X-4?=p>$t$ERp)
zhP@4Sq`I7^*WxKEuhHs@HJz+DdWqyq=bt5NCp4q#UnS}_bcqU{0;!<rv;PoG4@kjj
z$^L`JN9iR)v4xZHwMK0=`=0`h*AU&Z`a%^&tHwafDu*Y?7SPO+gqk&&y9mwxk+O3(
z(Sq$;<olh@(9k4}Dpr!;cma~8$!JEV{^OCiO_UqS+HwnR@Ma`kb0D8X?ts4TwJpMx
zXsFO>GszH3=CT~uiAA^qcei~0#zKdVCs1eU4vpY@_duV$JK;Rd9w;K*hZ7e}3#o}L
z<w;Qt(E4Wh-~Ebh(s8T9B-Y2ydmC^Emj`+on+ub2LsM!|%(|~92FqfTPnVTN&#80s
zDmJXy#Kzkh8}1YqiK$aTwpfh_3BiU8=vAdwUuJ-H?5_bJJ-rn!1!D9QNspaEZd$&X
z`sZkbVE(Bn#I0M<K4AhdGBKqZpghgKB7Sx5ba3pAbZ4!vWGRM`S<KeYoj#bKpu@r9
z(0E}(>g`7H&zNQ&|0WL>5VYPhwwv=bfTNQ<5!^n3wEf*fRD^*v4GL=)i`03SXdco@
zyeD0cxybwD(TZT2D&rny+z?A?NlXe2%+F>dXwyU!vH>j&S1iee`j{<#Lcl9FDCE73
z+cLigq_~+Kt_7K_i5S(4={uk}^kE1q)|*8IYZyZL-(MG1Y!t&Uv^Nb8$<9jTf|z58
z2fpm7!d5VxuZz{<hnUOmnk$HnVa4&?7Iv~3hf<cTx#*82`E~G=$S$7euY@XCkH!`q
z_U1~8O}H>Rrj4^C1)Q-Y`DVC*gU0t7tFK>p$3`lUxS}hPeX0nC<VX~LYqy8>``^2=
z{<43ZzTq^eYPwzjb>8sP{nG_{KipvS%J8!3JvUWguOXO`GWgHaPhdh{%y(1`A#!#g
zdYL?#iu|EPk2{BKkuxkd3%ZL1#SndcEBL(aOVIJYCgVg9D+xUo?n$d?katB)(Y^`+
znE<Bvs2pU+h1jbihU7#I!iDpM^J_d`&0hEk1ToYFpLQTVDqTtkvS^gf8yxByZzX{c
zn&aBTn(#;ivHTM38$y`Peii0pACtA2O4_+TYO$=*)JRj<Y+N@Oja8%y@J@;pC0`wv
zdFg#r%HF?#ha~g(<deh!#Lry#`!*s)LZYGkvYQWaKh(Vw^kc(N=>-Ti?IS-kz_hF6
zD6IZ5iPp0R4nzc8)!%>hC5{^!?o8K|5iyD&aEa+H3SL!J0;~HuVBw4Xs^+u!%jg~x
zxTslyPQ~+NuKL|40F^%e>1|7umEw`h;EFJ^ObDi6pM=8>)13{dLIVb+^pb|H|0Daz
zOAN*^VDB<~$>Y}2JrD)LH2XYVaYE3g=gnhep4Blx?e4$)z4|Yfl0+L?7|7Trp*9oD
z@xd-M;$h5tOz69v6V}uH&)Rn=&7@mn5rGwMgg=+$6^W{`Z!e^oK*a(a)1^9_6SR@h
zv7!|tP>-=ifu|5POX31`B@nGVtR6-A(?A@{#`LFy(w-ytRTOO;xQx~0PWEy>eXOqh
z8qk&b&GLokepv)bD3_i7d|E6tP<kL)uX^5rVI{Wc2(|h3QJI|9QcowPGb-54zsR^N
zG)3RIZlfz!)Vj|oUR@rx!TejQLXdqgNmb}*4M-EJ(ocTigWK26y;&jRY~winoXt#O
zAwflso;c-$bIb3#g;-YPJy;zm(wwi!ya1i+>x)xl&lH|PeK<Au58&4#^r|96T+zCR
zLP-R;kQ{Vyu}~R!l{A20FgfNScF@a%7C(M^@FG=gM)C1V)c+SRAcA1r#D*HII!evU
zKSyEOGoofwp=|ED6u`hR453t;rg$FT=E=mUqIhyY&?Yxs&?-l)KCoJhd!GXC=Y{X#
z3|3QMs7?W3)0`BNE>`k$bzi*rld8Y2gWd`o<0k{N*pqvZzLJ7&YjUdRiv%z9X|bbF
zi)k&;7?eH2#K#YaS@(r&aviqj;X1v5F+;z)B_}s{VA$)G3#N!Imc6&V-QA2U{3Pl0
zSLLn$m4->A?$fD<xF`k-a?X%b&>&l_;m}HOJ$Rt-TY9*}MD7eQqbbPy-dz)EEc(3y
zR^}@kjwbV-e=r|Nv28-NK>;f?-ueVcpFVi1`?sIn2nXOJ0--RE2112e!6J7uec`cj
zYdt;R77}q0?FjnEkC;p34Il^In04cF+C|zO_`GyivJ9Gsj{gqN|45@sY?J**?}wR?
zaREnD;x*-%Jr)x@2+Bl^J&mz<J>&ncx+7nWnFcX-(<J@L$rZ8`lef^m;RaWy<1UwN
zKV+!%UP}cK=8Pfr8@=40piY3rdKqx5h`bi~=C97d&qMCd@?j_TYZt#VSUXAr`lAy~
zAVhbyr<eQ&Wr9{wernG@;rs7@*&dFjWvmhNnqs=RVc?5k{3Z2)F)n%8yL(-l|9Ver
z0Evg$;sV)xo}hEk`QVbjVdw>@4*$~mm<QGU2a?9EoR+Y=`cATmB|Uh_`^r+*E7EeJ
zzfehtJ=50BYS_ck7dRSyUwKq`dz)E#bL;cTXw8|hC?R9}Z2=@_C@RU5oaPUziodAy
zixoc56B@^@bm$;+GF`OTV2B}A1MaPyHqsha&S7ueK)xqoI1Dv|U7DU7XxT8xxOvej
zsks_9mV%|*`(bZ1=USQ!aIWv)!Xi>8p?>(iuzvsfCJ5f)G(7ESPFd)!2f2}X#hZn7
zY#GVZ#-^sGN5Ir#pdAi*BfM4YkKCCP-yC{@8b}r%R}@@$pRWw)e9a4g)A`b7|CE{>
zIm@&tC5noPVKaY=rEpXPvSsO>MLv+#;vM7ITy^}fuIdk1A<cnh5CiZ}u@|TPa2M)e
zzn`=mpXbLfTz@E?BnkbEL5~+8h?P%v5(G{2vM0y@yz)1k`;sJ3=E5SK-bJ>jauI@5
z4kNb%-%l?JWj6vnt{yqheV*1UWl;xqgnV{USuWLROJKs}Y93rET1G<SF%vvJ!SvRN
zf5qYZH710K@=!X31brA_<<bh-C#g2}qH<t%mxj-AAQxEOnnHS{D_M=w7uk60>H;az
z293+1*Mv`sH^V*}4>Iq=H-}j~jiUrO3%sePfsSMoHjzDAh3%{H?Ll9D9LlG>dbQ3a
z17c^K%oQWog3VC73+*f!#wUP6Wjn6#X`2s(G7@MSx2F!|Ov4H3_<Mn=M1h)$Iu_9h
zvCtaRlzFOmzV&$FSgniot9myFN9(JsRM@1x$9g8OdA;CYQCuAu|5m9gK5pZmvf&Z_
zUA>fV&;)x{#ADW54E$C_@~U|6>PRq6f?R9^Y^ui6<w%zLg(BG$O3@-^Dc(ogiH}U5
zFDGkjj7Sx-yrmRc{wQU$JKy}wfR&ZC>!f;kF!4=eP5-Rp!O84jw<Y`fpK&Z9ZI%0j
zL|S<{?=zsCW5b&)5g9ILG6S0M5`As)*nbN1ze83y9U9Vm%$+96ANC<ng~}aA5ktxg
zw|5sC(Q;j)xNM9OhOta{-Bp?!(FOk)D(H+bF+kyN@wLhu%E~1HCFU`9uKuDVR9Q^2
z)&@|K#8KY}43~U^B5zUVa&CRrLBC2eRSq>KZL=t#4K865zpI0<!btnm&)H%>PsK*)
zJ%(vJaSe`e6UOZX4E(S4r!p3l&ZjahjN|O$m#xnsG1n0YH3P9;!XQq3_Z+yQlNOu9
zh@6_ig^FB7C;VYge~Q=7oNrrOUBSvyF7d1zi9)DHxPV4%52V|^E+!#aRZ7S#B<S|>
z?$F}1k_x&dQDl5ZvX0bkS5B4wloELie3#3dCna8<^u2+Z(AlJkr^I}e2q2<kqsqJC
zhilvghLT3MQ<Vx!j#OUFj;HHcQ>CVRO-5!XogBcuv2eTT>aynE=ZhdRmM}TBl^Ypf
zLZOX_#{KA`<%FLXx$1f3w}*heOu~lN@+Ky^5~so0<oFgkH{E)$*yL(VQiiV2xN4R^
zSM<kiuL%Fud#AYqU<>V;9+)mSPVe{g=g&I`5G<*M{MhK=?X^xoK(V0|#`V|7|F-tA
z$DtMYLkZSK!c&)8zQq?$nGv@hgG_E-?jfl&Vf77E%XPzn*u>UuxO*Q}3Ju`zaKJYV
zm*mAAKgRNjEyEAKw#1%#b<sGH@0xrB8q4H|8>+5eYLL1OknDF=^&LAk&q*fU-9tty
zQUjivCo)WNvp7FKTLK+5vHWxiX-%GHnExUTa>h~@GTXn0cUeoiRzXF*+*~NKMFupr
z6dz~6mtLE+7ZT9Kbm=#I%_=-qCI*tTg~cyS)FD3mtL;W$vFo5GM=ZYUS=i}I$o4xs
zLl7}9{)w2fSVYCg+fxF}L-r9W>9P|asPrx5V>q>;n#;fi3hm7a<q)VXG3hW{Uqp8n
zNu=HxOWaD++;TvjOaa(&Z-H-?6Eg_i&G_NIzue{8PGgSfjy_la^htOB#@n!*u#6`o
zX#Z{?khHr&7{C6zZm071Or81q{YeQc4^Vr*bh0C<C5WJ}oG?pVrnNGdARIIN{fDI+
zTB|7^`e&)UNnkWk$A$A;UY>#q1ZS2Na3Mu#H_^Qdvir2=uk`jWvmT3Se6(^b%2&`J
zE(C1+`;Pkt;iW+-W$ut=Nx)F|HOkzsA#Qfis?eWw$q;&BALW<i;d(f4QFhP<ABT*-
z@LplkuUHgWHiYY4+-IbF$0+PAk<$GCVBx>>c96}!c{}536a?pHu!kYwA6~q>OXYcI
zJT~EW8Az$>2Cr{9rnSJS3;6KVpZk)DA*d&p(uIU#`)`Z76q9L^%UXgu{{n`W@SpUi
zfkia<253lt`Hdzk4jk-L-C-VzijJl{&(#H%A7~NAO!u0lZs3ZX^xL{ri^>o85y4wT
z@U?fOuj7FWj8;#~sY${fwWx=)BRkc}>Ss-GyzKZsS}qDNr>*LWpG^#e$+0twyj%y&
zL#44(Ob5H^YKLbe4IaquZf?ir&Slz8fvP}VETW7oJ=Y1D-x~V*DWq<3!01{DZ5*RY
z7YfW|W<m25n|ToaSA0}POG|=o=fjhc12w8Y?q4gof1SX%^Z)GrJ(2MAq89may&tn4
zw{bP_&TU2vAqR{&vfm@`^xnRcgeu}#J8~O~x>jTVER!|%u>qu{Hl7~PX$S*R75dtt
zz@uJ5^a}dX(<G9_)30J^YkS6KgFRqvUqg0o+#h5GU^HY%gl>0V<0LsGZL&EJ+u~`s
zMSH7IKd`p{*1orpwyzoHI|Ti-h?CFt#LnAKHp%f`l`+ElfkZr6-=Q4wM6?uDu$wa?
zTB%VodJTvWQgvjDiRG{4X(LVHPP_6y*}^dY3yoCk#i5<wnW=mG<m>uSdZ0sWET{pS
z<~2Wb2<%a_cfGqtu!S(GA(*6Hh~Nf(;={Bu{WlgnK$973#lbSTRlOA9x}F*Pt&!6$
zY-sR{l<JJ$+Zj_*E#s5f<3{t^l$vGA^J;=VsLzQ!9gkU9bX3$GxXlfpn5COXAU)66
zHC-G@zXfhylW!Xjl1CJ;#&w+(YQ=Zz55nXgK648jQ;}oH+!lf?w9eQqcBrbvo{}@(
zs~D1{vrPQxUfuq;v+eg&i9hPlGO<Bq60ku9snP+&Q+}QQ(>j!%YLJl%7a)mvX_(v}
zcFt++83)U5Ip8wfZaC_-$J>pzC=yv)8PIWce5M9b^k3`@W>m!DCP#LLrAX=)rfD`g
z)H^2xr?WkdqDbCI(=1wBPS<uv$iFs$6qyzc(Nkrsyc?CQ!Fxg^v21S7B+Ioc8NH$1
z@tq^f_eaECC3KTz;F?c%u9hlF_5PL2+K)E#Ea(x?&gSh{?bI4WL`m^Yl$o(9y0<)<
zP!+F&dhM}9oEj_Vz_#E;=TiYlUwgcSg}?77`NG#AxN!%=Tir)s44fc?x;ql@xPu$H
z<M8AUQ~-urpp5Bdol%ob^AmjZy7MD1RV!7{y(Fa{Ced#7FFD~N`$03n&N%o!#Lo;-
zY;-jC>P;py*hd1Dl{P&16f^?c(@p~l-uGYG2gcF8spod-YT!|_BdK|t*45?$tXYp%
z#jhsxz4qFcxgJ{@8~k0DxcGu$C#YM9zaNpilidYv-XVBH0Uh8Py77)B|Dh9yboyJY
z_@8!cuo~ThC757M{(<%^-;@TDooHwB6u;uYKb*;9Nq9VP8ZoSKldygXLfl&oNG;j2
zUlL;<d0pmax*?@RZE8FnTWpmLt_Eh^eYLgqHhrE&7b28H80l_p(Q@d^D71zczbTPT
z-_!Z3B549`JsNMdFwRGKVzvd$Sa02aUNR^XcBp?a*Cg5qIZKN7vz(8QuShYk8`pQ)
zITo&YYu4X$`HuF3paa7nw&$wU@Nh=*v5+=4;NCO|Uj*e+-UBO*nJ2j|Y{-w!cWp(G
zWYDyKd!DC@7KAHQboymrxphl*1SpYw6!Wl<?30k~P|QEdoV;CWsCktpYowsywZWQr
zx|EJvv6mz6P8e)FV&H(FloJyebOtbHV6p_un$>DPG}sEX#lT#vWbW$ncu)Nah|;g`
z7>*oL)Bqywf!z~K9y`xxE5PWul2Ozn<C{psBb<b$i)~;wEwuUi1PwreYM1quy!&D>
zyA5zm9(nZHgsF?^!*cWo0;)+rJjS@T=L_vA==duSYTFP7r(W&}+7GXcU1#0RryM#M
zmI?&=68J*BUsYH|AK%F8Cb!_KUHhsfP&6?>uuIEr{2sYEJ^(z28cLr`|N0AI8o7&g
zIst<8NpWHE6YN)}6{7``{+|s<z9}UdmznqVlH4~L&36y|SQSuEzMnhBdCocH+-AlP
zt+#SIJ<Q>^?Y?sc+sa0rc0PK!S#U45r>#ubkL?P*-0LN>vC$0%e8{f<Wtz^O!oNua
z<imZB;vTwvp!nD3FUjZYA{c*qnfF6*D5X-)bNxujVljk&Bps=GTXDn+ZY|%}7jqn1
zxHl-}-fCi^cpLy>W-rJR*}@mIiaq~Qc85`zKfFA~;~t!PG$Y`sh<;4{R)U%)B1$4H
z#@SXESz)FqvnN!{h~Z+YDq_935?M&w`h|eL&obrIL`o~2;5Gh3FAPMW4T>*F(`S9-
zS=iBgI4!<wn^m~NJ1+V}9MNqhMp!_<luhVEAruGL)2slS0V=j^gBi>5=w8OHV5s!t
z4^7Hwm*GC3CYgAAdfV>pv&@fir5SV<H{O_?0H>kt&+=T)dSoupWfb4IE>9T=gAi}y
z*XvTue&33lK6@N(S+XbDzl+g$THfpfBt}=sOiSmnGD~r2^;wW`pYxV#1@mW*d@jA}
zekE9M0i;FuW!%|c<&d8hb6P5T%z$Om5<v8z>QZ*hefU9vg9%Li?x!MLZ=04t!k?wR
z?(pAuukyxhyAQhr&5t<G&X=yr_LgLO)OJ>Nxv03o(DzcV8JrEbk$_P#QIZNI{}z4K
z>A49U(fOuHd!rI~CaML6+&3&MY%0<7OrbvJUF)OT+RobH$h|l*`|FqaS8b-HkACR^
zbQbOJn>mu0oOT(+i!6V>y+F)k1mQ6`-mtF54KlMqo&2Dn<9SyI6r2~*#h!2qO=BVP
zt$J1KjmxXb*-G%ibY1SkKOs3hN$qCQB;dDh!|9DjRE0c0g;Ot@4vUpgv?O?EzN}XC
zmLDC)=HjbP!WbR&bHlL7wQmu+=|IT9+efw5tyw^tvSOHii-bnBFN+O_dY=aCg{Zp0
zg*wOkUw(yNjNcfEE3%=+L>EJ&2CM1fPZ(jy5&byz&`ZMi@$|pSeG=fQr^^T92T4so
z?xiXB01$pu154p~ieDI3{A=xJ%w6xrR#Q9+i6CP2(=nR*?wPg~qZ;u(==k0te#!j`
z8Z3cLW)aRe)@i&h)%0Y2s;UTWogROJ28CuDJUd*N{mg*>xa-=NK(msUH{SHeQ(T&&
zm;>p_XGqO6IP`g>6LE=Xm&vT}%cp(O0Ie^&Iw-T7sny;bD^PD7d0`v$qTaZ>?XUJ$
z>lXf+dHN|p^GB*n3Yr$aRW`RL(%ngTl7kNdmw&{KEAd84Bdhq$<2l$U3<Y};-*{@)
zyW+Ul337(dKKRm+&Y$1kv}k2L-Y7L~b;ZtD<_A48_~#s44SRsD@w$T&OVc^RPB^h?
zsP^wv^iOV4z9PN*2#ZKyFpO9V%zSrNJz;w+aV5-ayF}CnKjumNTA#N9<8+5VjngA8
zKi-Y-JI^0lpi$=7hKK8P$3o+b|6`pl?b%k?AS;K0A7#-SPoy5*X*$Zkp#1fRZ@OIL
zQPjqMo?@oijnVhOq>aBdAJ13syVJulQ$LnEtM|om9lZ+#))==)u!Ns9z6ZAH`J&n4
zH9z;(7d9{r<pcYrpeyT@z&3pb*rrp+`*;y1OI<|g@kII;&gP^F2~h|P=F{QxdS6h2
zRa+e(%$3bgZt-%z7rZ{m8*c3wxN&+oe<__^HFMFmA}S$(YaH0Df89ks$HpamsNZ)-
zo80eLJKdY|{$%;fHnzQ$8AkAt0j+e|bER5oea~&ax|3gtOJ6@q`0t^;8||E)t$?yF
z)V8#XE#7hqQQ|sjnbn5$KAtf@O)`voX!OKO=2nrK4&ZB2F21D1QWz7wJ1t!{tSCGI
z6yU7^+TlJs5#$LPujitHymNochouSOuU)N<zQyYE3%uYbeSM;yW`jNuJJ`hVuKI85
z9gy~k-uPi#JEjBv>s12<vxN<|r6>FgR|5x|a;R&|O`$g5%ixp1XL7Mt?1~*^fF*PW
zQ2iqVShau*P_b}N{S6#UHW*<k|6-&5=RmmN`P~NVVw4{lD1g!WVS_k5rn3W9+ydDy
zo`ka?Nw3#}pa9{7KkdmiOA&W`FRb^v@9Di-Z~s&$5caPx*Vi)RTO;`-_c(-AUj%Pk
zIL`RNDX_uv_sAA^wfXSXV|l;r9?!+lIy_svb}_yL>{N(uLt@k^Zm4S_ylS7>CQX(O
zvy8eJGROw95Q|=m_TRwF30Mqn$qLL!>3+84ZjT^y3{p54VM*>XFAymh6|XM6v{Yi`
z<PI`kmm#$7`@rAtueACVVvT!SoR4YH`A}03Q+^xlU8_yu^WJm+=5uxFdbV8!jm2{S
zbD|iJ(Ln-ey`j(5*_=GG>F}q#Ex=HE$^fYwdGsssjaeltP%+3PubRzpJXwidZTJy1
z^%{0IHTR8qEK3RE3I3<_0#2+x03hdU7U^p%q9vz+?M+{uOn}#hM6X|3<PCu}`^bpp
zFrNNm&~01OG-8|;q4AZ8T6+@lOH_KY+hXw0T&_CDlv$!}sg_-%`_;v98DG)&ci)GB
z^K&o1)yUoStiKgei#bUL2V{{zD}HI0;*pzqf1H^C?DA7VlM0n}UH8V2J{cC#a@;k7
z{bzOBLuut!4WGA%F4B4r7d(rms<~OpT)fpCYjj0w!TOLm=6@mxNK14nFOf$;3~^&&
zb>g4EF#l&!qWl&eANoFFPH-F9V-w9CA25jX+%CO7cWe>~fjH3a;s;4~`jz>1DyCR{
zlH%ln1$z{?2WH23O=<?ebx1lT4OH}52hIRoBZ|j?aB5s+e1+U7E90v#;^Fe1A<!?V
zkF-wlWJsupQg9Xd)#$?gPcQGQ1yViqxSRXH^F|nPVZ8sE@&}@*Ks~0+?@{9hOh2WB
z_|)J!lXS?SA=tYFQ8nlHB97D>MYFy5d_MyV84;BJM9;iO{GHP;=Arq02||R6%!AJR
znu6DES^@KyuanYg&$@V^jVR+gMG})*j6WVcm>zq1ch2X^`wr}3QPr%+C&3nl`#CQX
zc<(iSyIb)fSfV8_j1g!m@rumSJAoHOSJkuy9N7V(DsYv|;1;Krp{U1Ur{j$-4#zP1
z$3MkRmMLC2_3<s~57XHM`H|vq<KLTJiZy^)Y8rS~><QhqD|>u|FgZ9<rokdA%3f4g
z&p8COAOKY(EUYF<O}!paNz-+K4#fQGd{w;S@Nb${whGQ+8Gz6fi*r=jCN%xUO0O?<
z$b{AXE-5M~wQ2)=F!zUgb=xL+`IWMS7iNf9epDIB{m&X%Cg`^gh<p2xgU|T9)~;*R
z`1*9OI$XxM@h9fsG`3Q>-gc_&O8wCN1FJT(zI+Fh+2=O=*1rgsDl$x;_3ZN872w|E
zIoi_Z>A4Xsj#qVYvU({)KsB-ajKV}WF?wo}U-<Xy?J=DmfH_bdA2tiWnR@pwkx9K=
zrbnaFxQueivAx{>)$l+Sw{2rzW##XH<N{OF%zTzT#j@stZhF(1hNWqLflgIPc&&|D
z4Us_k$(Q4Csm9U;1LBCYyFAR)%^5PN3A%VbCk@wG#mz$VS8tlq`K+cp8)<}E1Ky9f
zS-s&FJRvc+8wnDpiR#?xe>|sgaS~m=L+CX+>*4OJJ0Ze)9uc)@KxMEnVzr`P&=u0R
z7@wthK6I-$kxt)#YLI`{=<SqqmZ)ueRPYg!eP>c2wGr}Wsr$S3yPwsTJX3~w3_vmM
zZ7F49RrwCCw_%HzSI;-4%bZ?Lu@2joK7-FW;tKewGMADD6Q%pK+e>*KdsYK$4<_^J
z&%UsDG9KVf9gXA3d7lRvxz1%a!97k><A=W?lI*c3KEKx_Kd>^q-eldMz$Y)C`VSTG
zZwU0R2TlmE!g9v2c}=a0>^(juY<#K^XtPMT+d<|%5kg|;YZi1LRSfI`(@Fg;G>A4-
z=;~}N3%5>9A6_gGI&8opKZ*FaE0jqJ>n}>aHJ+9!vd-XfACl8Nbr8xsHR%Y*ZgOt3
zYBw-diM&sRA3PakwL4DBpp!@?fx;=qXYmlsA0o5bz%2eoT;nBJZK8_=@&`KRDl>-N
z-&N_wkXgj8xF_ex2ohWu>eD<5*n8bmimXe?zSGSG^1$pYu*Itc5ks=acqsTB<wTq=
z$(6rd3|fU+Gb97!hU^tE|F<IWV`Ckt2N8JB<!Mi-NV=SNAK!SZCV{2ir+Cp6moe?5
znYVK*AE-X7R#*H2;8wqJcdZed)2+LQ)xT>ux4>WaiL8)Wl=?o3It-wmp8gG{ryY-O
z_wcI+0A~R2%b<S93+6$40)_ivT2a6E;q(6@q?W4FC31UPF}9)&LzfCYCupr*gS=&7
z!o6Z=YYp`&NQX~7D~lyy{BU*Hb9-=W>Ogs_;jA$ermyx3RbsN$EPQ3un{)WvDLwdb
zAJOo3s%Z|!<09u>R!~*3G$6>cLF!N_xU=<=n>t%c^tBPw<oWtY<W}`us!`&bGQAw{
z2Rh}0jCHGaXS)q=jlE7&%eWdxgjUAFzm4P78>U{Kc?&PS(;stgBY&wtD{ORrd=wiB
zeXgJq_|$n4a&@zdKQpO$_cHMfV(EBJHKr#*aPRe$-D=^Gz+tH5(TN;n@=KGb7`f<A
z9DIql(<mysJ>>2PJ<|G3(=dp0%xuw)F(#<-2Ldvsk<%`pB!)Nr1PGtp*heWe*jzk#
zTmInfT+zs?$^Lr>*B%<YsotDO9Jw5t(%zj*$5fZCdiQ&f0Eam*)3@h)7KuF8vMEhQ
z)!K*M=3?AI0-bS2{##?>%X6-|??GE)IL*mYlXrZN|IlAxao^|aG{a>&duuiXnj{WG
z(s*VJoP79Ik3^F?i0HU3d2UKq1H*G6Xgk&a$V`85h-84b5^lmLqwf!V6TiPb!#Ak@
zV*Q<fDZrq!m6`Od<m3EcRbn3Njh9M%#j1gBR^0O}q=DgDNKKNJkQQG^>bYe26TOX|
z!i)LV4w_w+!1&qLJRjw<hFsh-M@l)pE&P)$yrG(MJY)wP&U4#RH#qagmHP1g46}Q=
zpBVpG9eyeyx6H12La!D0GRzBsnIPh9JgUT3!4@d^tNfdz|KkISbg%QD!IEl`fdzQC
z^-2;ZsJ@EE=I;+khn@Elu8=7`I04*zC1){-Eg67c_@EZ>wzw+LwpVJPp1tS(D_T>(
z1gLxSHF8<FOObq80@zapIPeWm3Yh)DpG7nEKS=NY#OH0@4Pgu7m5+R7SJU>LHTnrw
z-|}w9)vX6QQD=d+moA(|5eY&rdsJpkr=vw%pxwQP#M4K{^V=pkyo$&u#YyN-7N*O{
z_@=P1`anBob+b06p5pjFJKH1bAn~Ypvkyigy{r=?T8NmM9?uusn_O!$jw3o<_CEW)
z$!uCts96d6)B!+tJ?}^%x2QH+$q&7<;VAm-vt$C3?b?Wj4NUlErS>|hzn8#{$9Z}E
zB@I>|$AGxCCq#{(+fU~TwyEH`)mW!;aEEo!{#B2w@b*hK_kcUFS+SkvrGx!r#9N<D
z#?xD$2M*r-SQMv#6^k`MIgy@f&(q>X_Sj$LY~NoQdxt-A7B>8R#zK`6t1nq#&QJ=^
z=aX$&A%z8u{-U5@*0N!%{=<zs3Dyx41%veCq;C{!PZh04UUxMK^d<64*`3Zb@;jWz
z)eCLf*7zs8de8M9{cI4J$|JcHeU+X+>!2sRQ%KcP20c+7UVYPONTTbaaSQ#U<e=iF
zaD5+1jdae;j}koMq!kf@i$SGNX8y)@{^`9WRBkK3#z{!WewyAgZ=kt5)8t0%)d7Ea
z5Nhz;jv3dl?7KuKYfOdN;yhJGO-?GkH2>)LZ<nZ_mun-ew^HxuacEv>^K`Pxxo<#T
z45f}S1@O5i`RO2>8Ao3d6-YfD^LN{tgc}cE{_M&t&o<1Yh{UlfzCQYyt8NSnvra-e
zj5t^ij~s<1>(V)-oX@Ce-{$4`Q=}kR)yEsZQ-!nay;cxMtNdWFiqAx_=?-Ed(z&3k
z`J4Ak%?ov&xYEhBjnv=ITsjvh6{jT*487R~oi|3h#teMCcXMNQJHCMFpLk<Rmd=OV
zj{t=W0paqqIob!wpYsPK!sprbRG+%}cLbc?4ay5JG${b|;--wutl{_W4C=Zqd_emn
z@!r{xV|348@Y-!2<J2wCze}q1yWv>h6BOYDdKypDXvAS&(@JQcD?n6o-<uWz!)XW|
zzl~N(^0lHuAkUjZc`i<(Sq&y?<978W%LX&#1V(G$u}P%EUyd;XF4E1KB|`UrskdvY
zD4~0&5^D>i+Wxpi-8~0?@8Kv9gYF!dl}5={vR&sbXt9>4oG{X~E_*qJSZlh{dkqyi
zu=kpIvEGmD`3$mxA37^ct77$OH(c4&>paC;@E>RK7VwIzuo`A-+_#uwcw5P~VBWj(
zEX_##eCtN=hjIFFYO(3|2wKYmO`Wc>nKUP_WEB~F-dxEBhjh6>S{9pMhwGIZ9g3CB
zY>DlfGZ{DuM+?V4ouCb$ckk9aZuITe-qD|G%}aGfxCl%cE-5Tm3g~+brTA=pvI+l8
zZm~T7i_va%AzD%Ra=<QT%229iYg^Nx#GDbv^eWxvByYRG-RV+I`F4|7nw%G7e&hL;
z>XKQa8meKz|Ltt|SwkTkK5zX#Uqj!W?Hb<XDFOFP8-?c{*x?(*uwzQP^ZuSO-s!t#
z9Ya~KHY@YGOnrnmAJPH;J^+Dgis_wfWliLzp~Ad`b|2Fnj`^bQf8!F~Hl5n3Lm?aM
z71zJbZO3Z*x52XXEv}q8pA9L7zf#;GdsEPiVfkwS-eCSfQ-}4`y>EC_ZkslR#E0?z
zr)je(ow^Ev!Jn0=RBu9K;z#R&B}eV)SG`^@8@+ba6ZzN0bz&Zat@9UJ|K*;99$tdg
zW_)y4Yma8@-MG&CQU7p8ylIQOl^x$|nPzu|L*(5EyFMm2BG^bdGyz{opjla|W}Uv5
zJ4Udr6Rz|`Z!1Tsc0WkcyjTE-3&&H1&^t;_<O!z(er0)t^F&uB?&ppDVZ5f|XJ-_u
zreEw8p4Of?m*p_%C9Yryg`GOtW6BcY?v<viN@0A0-VWKv5pwwHpWp|NAXQ>OHh()U
zwcOc|3hPfhJy%sgGAnszia|)vgRIgiIvPg}ym!Cw1bs>tTqH}TfLtwG&PXu1r8SaU
zi~(lH6!1IBJr8`0p<oK;gYA%0reRJcV9L_=B5i%FATimg|M4El5H=ohIe;4;@4)kW
zD2hrw`0S<;wa)A02Ad;=dct&%9beyRCM5BBShT)+-7<7wHg_Qxb)A9@Q8fpnTP;1R
z;HArm1`_0guz;q^Q)~aAGTw#fM-M6yKfVQu&-(yvigw7VwGPv|4VWD7Ar%bqQ~sGt
z?LUzaM-M!B7e|=~&#hKx3bavQt0nz$$LeL|myT$z8}wh!y(*f7i-$$#=nvHBa-{&(
zf)fZp;`ld8<C7ix-Qy3It~>`APshbeD{elW=t%OY^%h;`@D!L{``lex>nAIU?g6T`
zGk48b!Ug7jfPDA5<ujdTROGCLfB3lMjMG$sdy&A8+4A*}g$ngEgs*vuG0n<X>nr2D
z!baH<^Y^LiMcYXBL>|whf!WWN0~f?W0wcxm1Rd5V%qM$khRfnWCjBUxN>`-$oRyVJ
zA)dFZP~<9UMDMxyRY+ce-Wl=G>{W1w&3x#EeA`W0t9KIEm}Zs&dpD0pJ={qx+`@<P
zpkjqFpQ;^4ol;Gg<COG0`xzU5!{Z^Ds}q`&#Ah+noj<Pn>Q@f^A1#Jr3EK>|T^?6v
zm6px<v_ZB)s|-F1CtkUk`&d^gUZ=4-3d?al`6AppFy-KLnP@wP6yUZ~VG_{u)D=kF
zUpm&-*S=BYu^6q?*$EY|oa!%}EZ_zlVgkLY|1njPuzix&3FzzBwqygDu|QuhKX~E@
z-#HtK*9LBI@>Xt`2S0qNFghf}PJ-oLX9ZN_mzJ2#vyddyG~+fgZx7IlpB$P)Hj1=P
zMpK1?JjjZK5Tr;s_EbN(SZ<Lwj%sgK(|IXb%lDKcumsn=0dM&EWN70?;tNSCrd||=
zECvo=`y&n|?p6x9dPjfSC5%^wUF~7nuG;1*wvI^Oa`BZM_KI@$?GFirj?68)%32hG
zX?eu_z-60nBsSh<Lo>yAH^*)gl6bm_9a7*<o&1Apo#cQO<{g>9Hh|w`@J>1%WS@A6
zveS*cDVK3Qa|4irfL*RQ>c2qHKM@u1h{ElFedQUwvzpt7JM`yX@!Z}C4F{*83q6<G
zClMa?-2iw+59_W-gFr%)G3$>r@)6q%JO6$oKZX?m!};;9d+}MChEylaA8-<~7`I-9
zpdwRMN_E7iIS<YbZZ$EkLSqjBY7(m;TX)$0YSC8kwb;R~|H7^KHUTXl8U&lGUG^Dx
z*#G%?Y(Zru#ijDT5X26#?sA%wQx|n9PiFBOY<xQ!Xuq<GycUpGSleAKQ_jZP;1`gn
zRVd<>^>PJXyT&$^J9RlEX1ptRbnJxcco(_?g_49y({W<|ZtT+cL>;yDOAG&6e6?8c
zFs|`p%e!vr+=Svy@MOLz-Pu9<j#<$=+RMGVqh{ZZ?YP-1Xs9=;_FyY=Jf&}J<sNuM
zBs5k37(M+NSR*97rO>CS1Ls^Wpc#5(k1<bu(~+{cnd+%`>(fJ1P9E335!Hf`2uLbe
zVK!Ya&Ac!*kyXojt^N`(b#bj{cHOU7cK%@o_lMeQ($>!eO7FaHKG(s6xR9RCtNI|L
z4cv7iRV^2GM=5PT3$5WN8h5gPl%Z95u_31!!AR|ZAE*?}h6uvbvRY)P(rYOH7ULLS
zB0myIKWe0*s6&87<lQM5r_Dh^hwPHv4R%g%><uzj3*{4Yj2pf*qB(x?Ao=>Mo`;j;
za#($(w+Arpm)1Yy1;}ySb>8{X{cetqanvr9U+>?O6D^Iu>Vj$^oP@b8pSS{QeVHzf
zCpWI`6W*ci<mEC0QS1kRThEpJ0UZ4s70Y)+K;rg?J0H)(_BDGxOSD*ryEUkova*78
zbUu!c_ArAl<TE52meXARyU#PS7^{68QeX!pF}Mppp2Ft?c4LjhlSAUDVfc1R(@a7e
zI4b(VE~BaYZzPzD6;R1r0|HSWIhiIhiavGUVC{y=q&=%tzG?M5?F2PyzV%6L707%t
z$|c)*rYbw84=63k>ZC3mcN=SLek>E8DT~`WJT!ok2((8{>n6^Y`_IB!xGKGmTa14n
zc8O0=25!fVK8q%*ob7^*uQv9!9;iLgCm22M=HZ$99F=f;kluMv(!Ge|SNrg4st@d0
z!OXv@@_(HW6=qQE(q5{!$36V@p850jrl<RXkHMUwXY5IezaU`<dw@z@3oRF>j|)(<
zbJ@!aa5nBzc>u~+BbtNYmcq%A4+MT`d60_YSap`|d&%05CHt&`AzxsY_5B%_>nFRP
z{KM(Zq^-}@Z{5(;%>F`Dde52vK1PM)gentd<S*<e2^2AVm>BnOf9zCwuo4+)AYlnX
zk$H6+6@|p6x}2t|;nBI{3q4UZ660HMzMC)OWu@t}v7jn?N&bw>`jlk@Ua$;}HmgU}
zd%JxYW<hRKD$3D2ymw%^@yN9MrYC>^)Af_DB)TU6kFt~;8~6S3XrSgeCkZ+a12J4i
zEjN;XQ`-K(@1FbI%9Tocmd9s3`(sS2%-?0gQP_GUdwHCo5#^J4MFFEwz_=gVBU26N
z#xScN(41t~kRpt_%YM&Ca;c%va!3b!V2n~71fE|Z2mK{Ek?iWu9NN0i(`YVcs$6?U
zHD5Qf-eXU7DcrNl?(X(sJ2xCh>UcRRfDcff&gm##(e2biD}gj}8=2B|UhLmB6Z)QO
zJDH!WE|ATxs1xt6-snERkGsy4durP|h~cQiy39p}QoqHyd6q7k4)%SW>eaYEEOB);
zASo{b!CL<-RGi>}o>hoM*`Xri-$e9(H9QV1e>6N;Iwn8nacKo{y*dU6#W5CQ#oRDh
zuY}TD;c#X}tjDeQ4ESD;LnHEKBA9Op^GhSwKImP<4_~hBXI^v~&1)L;HQ#jF4W>_b
zGp62vMo2$2bt%p~Cvoih(scT%=n}Eq!uCl|)S38%L$oY7fpnwzF~bvIKmcM$pvL7t
zC-fq`^vLta<9n_H7OfQ|iWSe~&1HjB^^hIkL_ACav)`rse!!WY&GvYs$Zc#ys&l+?
zmqo`~y!0NN{ypl6O^{?%UIQONr7qK;h;r(6-bi+(%0}-qS>N1)R9!CfvmFCRJsuYT
zr!(-1j{h~~LtEeEqUZ1XC3*L-!i9F9XEhuNx6%sOkt?Lj!Aw%m&unWpw|jSRftl;R
zI2NgeP+??Q8MKDe>3lC^&dkF!%2XyT2R|pH-Ww<zDqUXq9L<aG+=cQEz?w(LCpNan
zddt*b?=6um6)5cxlnq8n@BqNab@i%_`yexf!r~?G{cl+~2_0`XOd<6sx13h+0c6&7
z*U?XPuetG(-Q%~NzYk5+e(eSb!br_|4m{sIk6*iMQ|UbXLhx6eO#?5$mF1;6Fa8Jo
zqHOTTV1OI*_>)mDK6VO*G)?0Hm6D(KxvyH_z{mDTDSBxN=ks3jF??=9ul=PKS<(GA
zmo&&blA)BrVYksAMU5MhoXB$MR%>tv>4=s*va^F-Mad%#talk8hXD~9Vc?MaMU+C*
zl+0J1>*JjbUm=lA=V2kvGGiC5?#_Tzsc#Zx?Ig|bN#d+PHL+mw5$tB4DPN_v2fh>U
zYa5_t)<5x(K;(qZXYoUDMwOM!&X>tr625rj@%_pcE5X00FtAz-l#V*m<SfINZFR&}
z$dX8N2g>u_*f&QbbpfeyRBEv7-*{{`4SHo#EP_uV?Tc!b%UD%G)edbU7k<&5_2Xsi
zl{aZAZm1ayV4Zx49VpSV8Q9-&7}DF!@t&zbO(TZV&urn{wWBOmrQ9u7W3T6F?n{|e
z+b122Im*f6dM>urLzV^9J$-y6bKd43_P>3YQd{j!Np2fs1<Sx4jMfSXQKSc@*c!_2
zE76)ZhigNWVTD>+F4uWQ`>R`Be>>>f@V^~<QHLbK)6CvWiXRTAGwqHnaGe{7*l=-5
zIjfU|Ahvz-1+@X&$bI+Z$G;BfpNxPFZmd7BG+J2A3YE_u<veB@4@<(rjb0*bF5k<}
z+dmW@mP~D1|D^@cmNdft^F~E+D7{p^M~29?HkDTwds!E&ANVTnY&0hbR$pI^WKSK-
zElL64vDsL6R!o1kSAWsLeI-6EfUu=^%87Zl54_;y9~&jN{VF7LLm(skB}H=h4UF%Y
zYJM3~JQN%lF7}_~1+$Ot>+QA9r{lTjUkf|MQN8K40he^|ouuO)`2u@xW)!~0nZKTX
zi2}~E(Qr6pZeN(8)UwH&+g;h>u#FPMrB?wiXP{kK`~;dBoQlh*r#IM}{Jvd^ycN&+
ziyAqPKk|oJ`=yiqEt~|60?ks<T1XV1_hnZj?)5KX@?5c@9bnVoG7AH%?*>P{p}_g(
zZ0}yIyvM}*oetB@ko#2n!yOv0()9>^&Ko_qfD-)O=-O2np4PFyhwRyi{lOaLYO>ya
zT&XFbMS%bKsREgWK4kYD((`;s%gOTU@m_D4?eK%D5Tm7zhzdHc?S}HGM;9)^62nCX
zO(^Yq2Wj%$v#_&mS(w6e@V=+ECR!q@=TECMKloX=867L+HZ(+Q81!(}@9<e8p87EA
z=$2k3dvAqzTvi~8ar(gZ#puxcpVznOKXbivZlm7oQp=<l%|wb=NmwX+@)NC4c1)<R
zpw})=ahWU?^mev`0?8JtOV!q9zl-hi{TTC|P)#oP4JG==y^EYd1kGXKx#!XLZWMXh
zoZs#hs~As{)vFkVYuV^f6j7Vuew{r0A<I|-6119Tx%)=3BtuL*dehv)L-3F{P|!~{
z-^w~QgbD-8kLF8^O-OjeF2w5Lh=KPgJORa2{E&i!uFNqQ&Hk$MSd{7*Xvv6?=KI<3
z?V=oV8Lu&EfLwzsDp>8^$cg|+aNs#^LL+9##xt%}fk8DA$$@`M)Tf!Dr4vpBxd^cO
z?&lTSjYQY)_O|f9;cA*1H&}R!4G6I(shIe@ofTDc@iSaM|I^nUU~?2de`D)@Qe(cA
zX`adz?b7-z3jyFs@@>#-Pt<iZSB_uewR`u+@%M3wuJ*n(&j~Wprm(;jxyWevxl@yA
zm)X2_979`H?prIHrfu36;ks~vU1oAT`Nk<x{O#VhWPJL&?i(Du#j_+0rnVY-EM_lK
z&m)Pzq_J{lHn5eiN@&K>gM{w5PgA}ij@&|)<YM~L!|YjjCfhBX-RGMT48mq(G(2XA
zi{rV}0=@s175H-j20?}%;tpXJeKK0ZrFFn-rSO&WExrXd-7#Dz;%K^wtYp7QvTK}U
zI)l<V!YU+CLi9bgbVCsPt-4by4Lkd`l-Aq19}2yfRDH~lR1klyi8!*wHNWgEXEUUx
z{-k(;B<H5H0pk9H`#S5bcO>al9`JwQ@~nOK9aDn%TXmXQEe5M*?P63qiT_x_XsX9r
zDMu>Je()(C-<#Cdr6p#I9S<hdV9uAoa3U3m*|ucuMkkjeaRupCcZaOc!<Q)@<mvRu
zfpiui^@4@$|GZx?ux4KSuYvUl+GXw@q{CDa=#usTGsR?qa0MzZU5f#62cHwVY<5PN
zfCMP2uD;8p)y!{f+8|x6Wml&kJLQklWc|X^Fm#!8r$XS;#ZE7BuPwp18k?>v)W%E&
z=B&VQGWf`tZ7{#i$Q(U~nYg@i=~3W+J-CX_ouy2JNtaonKdu@DLSCWfldHCdJb&wK
zxB@W13DA4Z?&M|izrbRO5J8Tp;<~4^W!4G|7RQQV2k7_*u=>QDtgU2i4Q)cR_iw=)
zmNlll#(#5Pp;1PGq>&jI{Xuf(SrU(RfF5j(xYCfZ$}JXJcO=rxG{h7MqqZJrtn)l+
zmfxLrUrinZoPm0LyQ^gJR3GWJzuEi$D#Z3!l;7hds9^Hp^P0|ZVY6e9u`^w)5ODe8
zaxb!{vDj0-Hu)8+?x8F%f_qZZ6{u*?c8zbdNOX~fYA9)=I~oqNr|YCzmE;SOj8&>>
zwS;sVDb$bUHiu=$I3vybPP&wwCN!gE6!LVL?v&Cn2#Ut>7xw+cW>_F)^>OdS3lbu9
zyhd-mALUO%_cibbt)4ZKO;)rdQ`9{!2qRkQEHlP&uS6s4RF#Ld-h!cbLP~?kX4$%E
zqex7LLW*G`SknjjV@xEB`0_?p*I3n?ISd)(omjwhMv@-d{JYAwLW{mf!A(r@d25jT
zr5d&xr8|v`#ysDY9+a)=FpQTNY?pifq7!$Gx;UHho2YHPe{Sb12zW9SOjrucMz7cA
zp<4!5JF@=gAD^eUhi{{X2g;#hCy2|7qr{vZV&O~asm9GK>8W?S-Nmzh?^`(6YghHh
zmO4$!%fT5b?mCoB2888Fg;)ctNZ6@T<(1mPwD8+0me%vfd{f}CWC#urW<<#kTuSAv
zMpXqDgH<O5qdo(VWO7x;rkq&s-NmKM%JS_ZlFF8W9g1yLp9Diqh8lVO7jPb~l4MVM
zD%+3xzZ+3%bf?X9?%X=dwKmpgNnR<)rUMVUla-D~9gn>kZU(H{HBHXKHfGnX%XEb6
z)-K1(#)Uq4PEE*38@OL-`}8nuk8JM1?Jfn}kVAEoBUk6j`ffF@;K31u3;(<4{WF5Z
zFriEM7HptZHlJXAY4~;L64^mA1~mF;88N)B;lMDkA=q`*g!`Wet}yC09;(XM<KRDD
zE^80G`|Wk~T<Rn~_u6ufxU~c2E{~cf5N;rGz`qDqp|7~f!jS5yCJ<-rvq387jbz7V
zP|>ql3p%<jN}IP2q1DtZmmD4XqW&uIxw)~6#r_cHQ~sdaLJwhHmSV6rsv{tW&aAZ8
z3+UsyIgN;QdHT;{^jrBWS>k*na9ToSbm3(}7xWNpsy{okPTi0vy!>XA&w$TMq7e5G
zbILIm;DrdWvn`XpggyS8jjq;U3fu3#nC(p-dI`HlX&L}EqULum1y(Krv1k8P3M!x%
z!Iv%hEPswv&Y6<D`hwYXKkDg}`g0hr86Ti;CQ{nn{rDo4e^l&Zp5cLQWM5SYk-)4-
zVU|*|aU|PChnZrk*})H6Trl=6<r~{DESSnqYD=%b{8`5D(w!ml%3F8YvBu2?XjF-N
zlUWph6G$vhd2Hf=I=`5b^%51CrX8Lf(|_YV<=j)S^x#fCf6zEy=FdWJJYj%J@=#<%
zR_VJ>fc0S8M7ze!f~$jV-WzvavHCWw3oU(W_jO(wefWIU@Z>e>_H_P%-glGz)>NB<
zh~)ehp-rt-g<9&?+vlsvUiA@Ft00!HPn+_b%{*Q;_5RhHV$0HWB1HcA)-`hceVwLN
z+x?7S?Jx?9-fm#lF`3Dca)QvX={Nn(`e?sL$p#VlW)el#5<V_cF_#u<TZ~j31M~}%
z?-#MDMh5i<r9TR6>xf;O8;Z;XCZc3M-n)aSw;v}uI%T=)w7PjeJdwXFFXe(aujcN#
zf@v!iWHRIWlFh<ERR<yL*p%*QXzD|byES-1%?kwozW#q4-s}&5GMX_K;vRkrY<_!w
zyL-2%`VtWs4sBn6bV3&i9<Sr~gf02%xwL8;wnuaZatZUCW1S$RCLv8nUoLWd?At`a
zq>Xamh1iqA#lcYs-9;B=EEdFdKfc6-1^(p#r(#6My_Lhj0hP<jTU7X`ti%I7Z>_Ao
zH6iM=r%PHa%gXYBtAnjMjaC@7@2N#`R(M!i(^X0IFw$A}d9q;M*_~~L2Vb3V^#|TZ
zWFUtqLzQYzIW%0&kZCFBLL_{}o0N(q7vx3u??i@yo#rdc%xHvCcqiD*%Li1pN{gY-
z;^TnHM^@bs^iM4-h$Vv-ka*50e{k!;_V^it&Vo6w^)F}7nmGz>&~|i%$t#P&t{%|w
zY)r9Ak3cj*c^l{g%3B@)20YNIEa&0cvbD~y(SRw_<dx#{zR#?xjQV88(sa>2c3ql2
z18nVWImFs|PWs6)niA3O=3Ra^btkhxv_1A|oEFM&P^_W5@3j&|VbRSth+&u&URkVq
zI-Mx@^&S;s{^@mIQhP4gGK2MuYen26fsah?GQ;OJ)4!6T?7Al;Q?CQ<hC0nx#>5ZH
zOIn&bNuFz*B{`>T!=z7&UY`92&|;^_90eK!13Cd$dRdMLqd@gF$m7h%h5rMa^kvzV
z{{wp}q+p<G&wq$2OocYltPxgU^O_@pVh97BjK#JuPsjf+#=bf%>UN8kl8_is5u}Dt
zP(ebvW+*{LLXl2E>F!QpkQ%UQ6p?NaLFsOgp+&j|kVa~T`;L0P&vVXq?!AAA$OAl&
z@9($Yz4zLCtreloe9)FbC42$JasHNB2DcW#9b$9ocUxmrY?4ECmLuMz9Iikv&ao%-
zXW!rvK&dm^Ml`-0GPwp97_(6q1f$VMEge_SWff$FQE7a1LUR){!_!PS5W*xw3A*zB
z=0T5Rj9K4sj9ds+IO{uhwiwNq$P%?zpC{%#aOiz=qzroS<i--Rh^|@VEStz$CMM^l
zIjEF$i}&fd!-{)ef_eT|GNI03sk1$29sv%@9?m0eQ_l*pL9hMWwosPO&nqcj1i(uE
zr6&(){p-Mlb}Ulu(dJjJ(t1Th+j4gs-E>7qOy_x{@ttN1=ty3Cq(kWy$r7xN3)m3L
z0}w>2*X^`8i;79t3O_2?MW6Yo)h^#n2-!mar;|q~@#=>!uB#*#SX%5=H4Uxyv8j-s
zq?x{{{?W1`;y%~5ug8%@d#R6yRMg(Uaje~I_iB~c%8YWphW!LRcFU!AM7bAc*pK2V
z=myv-qw&WFnLPN<Zox4RQ$L@iP<ongo)|q@^ICPV%bcic7Bcd2Tequtso7xvd8NjZ
zFY>X5xg5WJ>1@vj4`hh#yrx#Sef!;2JHLYB^)SdliE2&5vf7W&wz+PX4<>izhm*`w
zYrgSG7Cz6YZn)({{VwLaU#Z=Y+xk77tL6)+_PxO)!u&~iVX5xGdj*%?(!w(4y=z{~
zXM4ZzxgbkuLg4pqf4hTJh6YR~RPY{<D5E~D#v0EzpW_u7pCA4(nN4>iShcZ5WovKG
zzgB6rU7z>Lo%W&bL{Yw7>s5!E1Dx({Fi#8Ze_czCL=nuthmmdk``TwW*!NS637~in
zT1Y$sFVJjWik3z2)$#bK3)&NvPgBK4TB@k5=0wM+6DI3oG+zl<ml>}Di|#g-JGzw1
z=u9k{LsI%ePW>H<^8WJowF>UdS)ACi^WtcE)$+#;05*TEHzve5kZ`!P{Gd1^kxp!c
zwv(-Rd)SD@b4+QMHdlDlI}7AaSJMlj!O>zg4&j=-ZpEdqP=)>xkD?32v%IR$WlFz0
zQs?Wr1zUU`E$qGvu?~}uOxr%4N%CruJIT{<;0=N$VM0pK@6a`oxge;Zpv-#kst+w{
zsy@UHP{HS4x)P{V_I+&p18@k^!ztKXV`)Sng|@-Kj%e#^aFOFQL85?}p6^;u!>PlG
zs)eJWp$Gt_+XF>~1GOR^f@pTVa{IGJpUR5!KPY~3BvdA3l(c^CC%BWlkyD&;Qb0QT
zFnuHEO;|L;XoWmN;Y&1nXy^ikMBcN~T}g+BDhVQ#h5CtBXKPNrDf(l|yOTRtdLy;b
zGn7J|r|oWvuQ*hOlp8&w)c<IbqC&qL9es$_NIY*8ubueL=f`*Vt@eYa($XG@&XyQv
z{lu7QAGiM$83^)@k%<G&E~k-Z59wVR0}*u|*bTycd8WyuMv^Hvl^MDzwOJiO>KU5j
z@S&V+9rNb2T%_&Jy#P;$)k#s8bF*0*u^bF2<P?gTo=fa$a#5Y{Pc;}N2mb3>*|Pa%
zh1SoImkS<hZW%XQs`}9mT`|yj3Nt~M2*})mQi;smy9GhOqv^8dNKp;GgAJ-R-u&Ae
z*_q;5x&g__Qfv5iyM14L_u^760f3X3{Fu~h%;Q;FZY>KZY?s_R>QOf={BF&m4Lqk^
zH+JEHtKK)!zA|7gRa154JL@sYTk^-R(6@&R4DE9BKSVq-?@6-qm~hjsyab%4c!j?I
ztPg)LE2a(Mz)rmrQX33DZs{}xw{$uWmQD}iYI3@9_mc>`I%f@2(#c!eEi%rd%yRu<
zxFMj*jgI|OtXLEN0;LiJ9&DWE`bfZ@o7PAcD8*ibCqiT&xK9KEHCk9gFe%728VWs#
zl4H|)2&nel#=_*=<&e?A5M4}swzAsH=^G;60^!;)uT^=U2U@*p=8JW_2IrpbW%9l?
zx|G{|pr>uk%cGS&ml~2?-%@U6J}Q<)bslwxE;fNSlipkb`*hGW&{aQDW{Bb@^&QuO
zGTw5$=kag+tR}YvIC9twE!e?o%rn@|VBXl=Tjl$S<)%efklyTVS@2S@lG#_QOV3OY
zv=9gNzwF2CeoLqOWEi+RBeQERCWtwicGLZc_!FE#T?e~X{ZSXNx8LaPEup*IaW^xt
zFy4Y@gAbcox!gdlZIDYJTgl=%6||~oV#Mc$V|#CXUjBHTT-o|Zoc*Ty1V!M`cZLa>
zM&YszJZeuoilHyU@|S}+fY!+7FRc;l9h#8jcwbMVrmLVc6I*R!-TxxC%cW&N(kz&Z
z6r*a&EwHAM=dn5D=ja$VFuk_PO`M4CldrNiO^3R8s~$I9zXvw?4K&XNX_l`(<q-i|
zBewV5`e{UZWQeW`EJZXcQLOBQq#f=|)n{|MoYvAYm!l*W?~0Ju4)DGOv5sm)CSyS>
z$rsjkO}2Z;3hMfpiA==$fkxiYj*`-xMe?3SK9<@71*ArJKQ}A~%^x5rfdvf8M(3vm
zj5WOu-yf5w@-z4t+d1$S9_*u=-s6rBfjhp;e>=XtN`S%J0<P!H1tuZOSVRAV`Ty|>
zk`vqjD@V*lv&0OK*AFX0!`2V_e@Tbb^Zu3&Sqe98Q@Rha2a04e;VL%-My4;l!wg31
zekB^siW_1_8lhs<^$kdNUm@c?n=MuR!uFwPuJ1%PWEeNCwHui5H<?3^cDdd+u(NU}
zsVk<5bn<MGRIR>SV%^tJFE#_xAr#DvFiK{zd~k2fplXZ!9dEEHLS>pe-jlpN+<9dQ
z9}8JUC-&qN>Yy`%Q@!`y6mJTQP&a2{MmcOiYn3&0ox(oNYs4XKnVe<t>Ca(BZGe^;
zXmmt;_cv^-d1!xH(>;1pGW<7O7Bm(d=r01MM0Q@LZVsuxJLFNI*8oLURe`S?`jyP5
zlAnowSRZ_<<(kl6YahP{f4IL;B0o2mZR;E!(pQ=4&(VnCQ*2%cW;npl+wG+Hcg>$B
zk*SFb*G^+PMkzJ9)rG;MLmle$<UbRc+wMGLLFK`{to#NqvKab1FGnWp`UNL6Pn#8q
znzlX2db#z`OmzkE=mVcK_54k3<WgQofTiBqw-UBDNxr<vE{vWTU?aLL=-+l}{{S!C
zl7KZFjSlI%II<(#n7Wq<+nnoQLG7DtWP=sP3x|>TZp%m&7j-ih49)H^b}8KmrSy8<
zLPdHMrQtt>QdoLZ({*Mo5!Q5lhzu-g0%0F*dQGpMc`yxlv?101N6ZB={Pb=oeg7)w
z48gB6gTg?IGJ98B8<gs~YD2yA|6+Uofy+!N{#?c&E>j8>;G^)w0~v&lb>`-6-ujln
zTp6h(JRrH$cIo_s7EU%r*lJEXY`;l7jaNbdrM8kog_*W`mqau}kIH0$k-0l>wyQ-Q
z&T5~?VM&~Flqs7dPS}hcA32rulRr>l9vTxAS(eRD$dVl_Q7$1OD5tlLVv9?TmuD>@
zyFdCmJCmVlMb>0k&qST17G0!>N{yEqjFs+whP+VA@=$$N&!<_{cuigym8x4FoG_mQ
z-$e0sH6r<E7aOzvu4_DFT2nr<q_MR@Xp2cN6*{6B)3Q?LoN;WU;D2Kq$#mm%zZ2^F
zbMG_b{{rQZZ~hF>f7bD_{|R69;{)Hy@PCO(1X9w(DI^gzPiqvIZ=U`l<*ML?$~-T=
zL8+5Is|{Hl(cy+L-x}^1yc00UYFzzA0%a0ye54735lQ;*;d_O7Qm#D{_F|CjHtlOy
zdY<X_r2O<rhPp~Py2yztVu`xz=IKCN2~vM%MPa;CC8X@BGx3|5!nB!*4|QxHOHCz1
zBnss^9HGTK=i*NpGZbRkOVtS(o#t*1Gu^p>ued*DW~mZ&xMUy^xw=!YWHi9<3@~bv
zm2FBPjZD>YoOF)NUFl_1_A|`kaBWWnps!xPZ15+5Yf{M$4r=~%g(x>+5LmCx2TLd<
zV1@%&po1^YTB6a<{*f`RUhxy~6~JHX$7N12k`0Fa6zJ1%PrQ*Pd=tfw*{5plgX?)!
zi6qMDvzg&v@kjgPgjk?%gT@D45fJMIt{6;C81G?ULcA<RIZ))hwK6?Wv(#(UTv!)F
zUFe(qB7Th2>57Fy^=5YMUPQEBPIuLprz~k-+TmP9Y<3EQygOwzRII5YsuU!epPh-K
zU<w^)i@ZrgF{c>3ax%s`9$YgFyv~s@3KgEyrj)+8RXnXN-At@e28tZs?z;h|eYCl{
zn0Ymu4K}OuikV<@PkwieIAL#Xd>l}(Eu$Hh1Gw>@FgyoZDY~`(7~5$%n02So&ZT~U
z-@Nh@T>s5o`uUek90y?Ens@ba2S@rg)>KO%WI)Gj0WL6XI%s=gEN*T<#-!@-A;#d(
zJp4O2QFcFXu=_GT_vydN5JF@KO;1lA2=22pNLGal9y;r}=f_V|QFHg}xfOAPd&>G2
z3MeFOWnD3)6hE^TcLC7i<mwAB?+;3>IFTtcT9>u_?xma(*&WQ2$h)*vtm<}<E0jr@
zbAM{_T?65Rg+3GzNFaVl9Ylk=TvA%cHKlnr;HuT<)rd#=zB4GOB;+xpAqp-nd1R^*
zMaq*f;6mha?%=+>v~IfKv&<^lXqySk5aY_OrZ}a5VmTCtG-7K*7?Q5>F=*k<p*x|9
zSj9E&Gu>O9y^cT4NE!MZ|9;nk4r6}Z_kJfAkDV8L=beiyK|}&Ckw40g{~4A58{{Au
zn{9YaOZ>MyK|)Yb7*3v`N{d3!ou~&-g76!jiyl&9hA&*`?41h;`~clebcXL5!hiqt
zFe+?3&F_@(X{Q+p>hx-UjZA(PM(K-rw2>;Xt-f+-vz2j^mOd)4D)c@=1<kgsKIVz$
zFp_y-(Ov&gX;?Np^J|)-j~-*9bsXWXQyb3_yfaZOtbRLv!3jZ8G075RudIR7xgjQm
zAyim+Gy9b-8i`~;wPa%F1TIG04QPI#&2;cpW<Y@2lBEDi#88)Gb*7t7HU(HmJq3LH
zwCO*<^j+p(^~E9f*8Y4?YUjME3l60t{Qth?Ny=Iu*VB&sE+XqLn1mCu_Ta(d4CXAt
zATm4>zWB|tT(#Cy@=cfOV)!#L_fy9@E>THVM!Z5AxO$#uZz$qqVRWeyMYA`(+eJYn
zgOOu1vTZRc9QheW9UO|n*>5o12)7rZoP)F?m#DKY88T}7<;QE>{Y;fdB4aput~bH+
z;9V}U(-MJ_OF|}ggnDQRXRQsz+hb$&r<@ryt^%_b>FIRhV8dcPvuA1Kt(>xIN6!$c
zZ?>t&*`ug-H)&O;zsBd459xgXN@9;za;^f6$O@n<<XBn#;mSnjPnG|Oh?T!%N!^$R
zuogiMC`=7B#(u>$bq(MG>&1Qe|GC9*++?y4Z86z}DM^2$fp_bN#r62q5Sh7iLynZx
z-6I=~v~?{}X*S(?-unuyZ0DKngS?4^7p@oyjOpt=(|OF$6qgR@NIi<GS0l97<gkc6
zB`N{*mE`I+MT*T5t5mS}+g)?-ia};g26`F)6@!6lp!JfB6rPCY#|O+2VqsJX3*@fz
zxmu*X&P8`45m6}%8fOV84k+;7r{ol3b7<wZWwc=s4G2OHMOPe#T|yUb_e0qrwcgfz
zT{$-bfRQwn%NV@7Rn2dUZEOy`!kyGGx)SwM{2djqgeLhT1g%+r2?bD6OPA}B0s#2+
zI*@Ll2ixUHB|}57Nm~H>XUQ`+#^0dA62C$kaEYi{q=N@vU<~HvZTy@TDHsT$;gUOO
z%3+Z>ICE2Ch9%sL6>j6|NsPLEWS0JNYo7f5YmOcwFEkQTA7(Oals;{d*rTL^<8Z&j
z*z}GWlRjau6{Fj-3D_^sy#MNC@%oFz%lAR8)ip+4%7W*P&llT4fDl{Fp+WsY<9_Rx
zrXer=x}0Q}OIrdaZ6J+jyWHoslp!^tmF|~ul*Zz3x{O2u&*WNXH*@E{LCIYWC5sv2
z1UH=9rk`J^hFCx{_0PgoVp9x5{l^QX@{0$VL-sCH`vNX2x<m%aXw1udzWoyh{rd-S
zd0_YF4mcBS8uUMy&2^__5ZiSW2orny5)$Dp@-LoS)cZuFm89bP;Wyspo_8nqvZ^lX
zvhIVjBc0DFfA*vr^WbmR+u>4`BHmRGSRFZiW4A5zbK{)=SKPW-fv5Q@!w!fU#G}E-
zav%y*?e8&WepubP>Bg*s=${UkR|5{f-ZD9FJZCnR*|Ia8zj(4Rxj$wy9gy0+X|bit
zt!A?eDaZ>Ldq$#6@m9y1rcux3P4xEEK;=?vLQtC;C8-ee$`N-GUdD%C2KROWPiq4H
z2AZ<Sew~gr@Zh^kO9Z;`J77en*|*K~_xJqSvg11UmvCI4Eq<MqpA&Qnxc6t)LQz2k
zd}6yb%b`lhf-gWS4s2;h+cmB^hd*lJy9*iGryhK&c64=<H%0?G)3?l*jI*U3TgGnC
zG^W8y_p+**%(!<lovhS0FP($lGvgI}iw#}8vZ*ywD`9Zy-tAEI27w~vM)@PPp#FMc
z7i#95qM)rJ@^|N6Zyk~=c;CznL*G^QNoNrlQ&TTRM8w1@QORQl6ED0`JnS1J{A60h
zmW}j|wJ(DnhnDMVAY`S85>&xlFi`Fq-OqPM56kLz^!F>E&hcw;6GfV2^zQV}&;nWi
zYT)9PzrW|tssY!%925K=A=I~l)x~<iyY$3cj#a^Vr-BVaM?xX_`}GJFVIX@$uTW~O
zjvO0#r0nvD&O??7Nu9FGP7Hs?=v+Q5C;hR6Y4;wl%bRycX`G9Iby@re>vD+2)bY_t
zI_HaS<=JwYQC82TzOm1E4P1dY=<Ai3Z18WYNC&1V!U6HpvpEhOr^I8Ee8Vl|3Za_Q
zKFYZ8c7u}tGkyjCH|H_b$E20uodil8yTgr_sP3vIhg?X7`J&Z$b`!9x8q9#fX1U%(
zGkHBK2FQC~Mmd8GMQ|t2L%#eeh%WrQ=ZmgezL67%E0lD>HxYQI{E`3u<G||p0&&RK
z#~|~E^p50HoEsA`b|XR?pwVYKbt+t*^?R_h&e=?+`6rHV8Ah_@rON3yXquwaDh_iI
z&dm?%kZRa8#W%dWkB0pnXJ#q(vl{O>%nZV_ibcJ0>hlbm+tsB0ZydaELXIv8JR4U9
zf9DsyL2j3u+YroYr!E$wGZXZ{2AYeWMrD1{!IQf%|K%puoc6{c!ngvR?p2!p6lJXg
z3wA|+^RTV(%$LV{g1KB0P}V(mg6>A3kw>^U&%QQZQIR6;V_uCDo(>^_^bBcgR=KZP
z>)VxU-bodAc|QU6$~CWPu5A56AtutdDgPcq)LobPNq#WD5V&%Q`!Vh)oxo928?em%
zcmMahWv=7<1tL-fMp2n5()<44OCg-x<mlQUxaUkRK$>)lSgt_qbi$Jm<Jg5We6=5m
zOR^P7p9pmQqK0Wzha^<IhrPw-U6^~CF0~7Jzoce!CLQBQZ2e%s>%OvZ85Nr@1o)W3
zBUFx*xeC9$4Su0|=S)^T=Y8d>RXuW|RRUQ7iHq*cH>b$y2icV5i@n9_X}8;NZI^MX
zVs_bBm5RI0n6ed6^T~24Vjgk5tTwYy`%=zHlDK!va96wAq{BKrH=v4ZkPUQyPK4bd
zwHJdHz)DKRbR(j4(z+cuaaaJkET=pHpFLY2Afl72sn)LX{!W};St(GBbth2S&l}CU
zn{EpP3~xkcICTBpR$Ie==^~Y%=nf5?1$pr!FyloHaV;JFjaU33Yy+;hpu@1=Z>24L
zC=tU=zQgZOUZ4URoTiw9Z$Ritl&$uez_jRWW7<o5W7$Pcdh`|1<Ph<eGnw=Uo42we
zrFU;>vj{9v?1j~Zn(#%uL2iXBg<qnoe>oJ#D_Mz>xo7BqI7KI76=0gfbFP5wmT0QI
zNO`(VDU%H1ZU7Z;T~<A-k-MukDeKk^g}W&-T&mvcf*bubGui1dApR+`X6P5*f8l=W
zxCsg&wf5rBS7w7FCt8*1NGo8s9?`?9=spv!<f`iXxdTvstUae*UF<8$Hhsw^A2tHC
zH?2=Fe8@C0AaX-17(^VbvHq=k?P$HW!l>F?*N&Bwc5$F*a*)Mm^#M>pOA@vn=}YmP
zA@f;?x7Z@|{R@3S$^Xg|=;=njryPsmOwR({^LD~byT5UXKcJkSUo1NJs~Z~m{=3c~
zs|pud@?GbfAtR#y#Tcoh`wDFI>iDRFR#DV+vLZtizH@H`IYy>L)s-;1vs|gz9Z+s}
z6k_Mt(KkM$P2^F~WM5*~VOGr^%^iVM>{h2D1)&ev>I&xuj?=fvZ{%{@UO#0R$UjR+
zk57%F@63~GrdDIx1S?BXgtkGK7zN*sJxab&-abh91^YZR1@$GRjL!Oc+u)~)|0ieU
z!-#1lg6qy}YScVOXrZGX%rJL&9+0JqflA}Z+Z%N{zJQ#LBawhgcXUwO*zTJChqC^3
zrE#1(G-%+j!T4?ZV-iH{7;Xu2=cS#Ln#A1^wEFT+&-L^q*!uky=jz+PGmq2<qheEB
z=AqZQLzK`xH1gHk15?_a03YY<PO&`OxW9h{Su3Y09BH_5obXTEJHMZX4Ccm9&c$Oc
zf$LH64>oSFHX-E-f^V{C^I@(t*5EZn84B*+=PsXZ&5VcBzsTr(x%Vpi!rkPjnzxvJ
zBhvW{NHBLJgb5IA(N|E-p{y?rJkPAAtACN$&kE@yNj=OhJnk4sNV-jqC@ipihLTkp
zIVTSb;`tJ8dJkUZyi*r0FXnq&s9sU3?%r8=<q|qWxcqr%l;PWyIUwuf{6&x2a!H=5
zp-azeBukAdSPeMGKT<rGu9vqN*O;^i>HKJ6y?kdxFRcF?nUGAe>-3;J=IBSoDe}M%
z9~FMR&rPiT25y6zLqHY<M^F3w#C)vGs+0MSy>*c9y_Bw0_@w{wrc#L5?lb~A2A-C#
z(Gs8aBv3V8i{LJ38*jwv+v23^OaI<-&2sqt3@W&cr53yAot*@m71BoC)+Sc-n0v0B
z`toC-b@Zgh(R=@f07kgt{%gknT7daWjUj8`r)rkASTvcLDm{Dr(jGlFoRBLmb}<e9
z{c;LDv2U;tLfF`|kq_`01ok!MC5q>=&KH9zPt-xv18qBWL#RyOAX0CnGbe8Ec~4$a
znt3^}Wvdl3vbcA$<GbHhuSg~MRv3GgJT#XZ)ki#fPJ)p(a4c>0N^HcfZG5(Fvd(=x
zJX)HYa{vQvZ{$lWgB+8>=8LqOJ9{Pk)PW7E6cIC9CVy(ej>)+pc362Xe0L)|C0rs8
zr@JF^d^3`N3<u9`Dc3Yu^`*%Yi)U1oEWb}2t6C{w?Cb*K+eQEkN53pnjAqdN@qMCH
zBGq^63Gj@P9Ieth2WaW5L<}1^J``udP`>^+SVmxfVt%-@+)3L3I1nN=TUl>a;76;1
zpY~@WxfyB<W>b8(+M_6%Y+7(CYTFA=-1@+a)6l&bNHf8?n@$R-fcUrVTEt)RlK1Ob
z&B5Q$YFU~9F_S7`%eumI<^!23#iOPBN<5m+KIG=vck9=+^Tn;V#_)_WGM2Z_3vUbZ
z-2;KOmoGU&#q3@-q%y9vs3}rHsWr+UGFy0p4cKdG3;Jh}|M$~IGyWVm(W(NP0@C9@
zz2M+5-~~4lu2Ey%76W-ei468`G6mWhEU;B{Z_=L9E50=)D#F@BYpVCJL)7Nzo6k0?
z2RK~}zLFbqZ9X7YnF=$A^xbXcgo->_MZa-rGhU8YK|NMSbT3mA0863ez9#;nX^SZ{
z8LIyJ5Qa<n8HAmjtU8&P46rkjU2tfo!BSl;e_vMZ$N86c?>}SO)t0hBv(4EU@6+U$
zLr0%0QOWtTi8Qfx41P;ErOyj+jQvZ1+yU61^&up!lv;~$!%yzz`5(rvzI^?=b{V`v
z=fVi4AV7qDy;NfL8RmQ<P9H}m4gV96+~9g@3F6Ld(*MZrVkVE;8I?z1@gttmrAKP}
zPRP4^cfJ7ns3dlOHoDh(vc_)~W@8;NQRP<Z(rb8J_`K3DTM5Y`tod!_uVm6m@e4LN
z6Aq}_mX*7rB|VAu7yQjFpH5>3(~x%iv9hC8*H1UaSMRM^#X4W)gKZaqF)cj8seRs=
z<}r{vKL>juSOFh**U!`!f-r67OzhR?hqBfDuMz-XDec?Or8P5x4UWdwG{jQ);n#f^
z`BDf<_wlo-QrV)p5Mtq+OZ)Mc@@P!y_vcy>(cW|02pKTr-VvZ1J6EW;Kd3F;mY%W~
znk)tYBNa?Lj-g=MPcyny=s#AvMSjGsDqLnUlfxq+^Md^16NxmcUWth^s?Ize=9kw?
zrWa$GqIBz~5GD;IcR-skyCKj<V-*T2GwZyNe`YAGJ)<9@K=X6V{U#_J?B;Mf(==a?
zG*RFI3YVve5=ek*O6+BMFnJ)pEc_4+pkqUzHT2=jZSH36OS|7E##v$4Ab?CC#Vk1l
zD1)Cy5)z$6MjxKF`SK8~td9-MzcOYqUVi6hnEJ^L^o522E&i~djUoCfZh6+LB$}Y!
z9YZNvPAT&3bJzF}z*JhUYOVRuk(0aV_c%$Wls*PD+;m{^`9Sk%Z7tZ)yVZ{hyGj0O
zXZMWX<`n~%|D25fRhOk<aZiK69t_i3R{5K>1GxfwqAbI>`9&!{;?0nRJD$BGz_t6N
zOzzq$dn<J=%M~?QCzMveu~mQ(QuZ2<mExqb5B7v?S>+$1+ZmW2nCJUiFGqFYoaMIe
zhm?b5;t1clQj6oRCQ~!RYyF%#8t)wC$1V%^k7nWFG0D*0ln@duCQv7#t9%?{UHFAo
zjRNK!+p1TWrN)1wJ`+p-yfv?yMxGg+`SAQucuzwD42@K?N!C5sUwSjvMJl2I+v9u+
z1}m8?D?%wpL*ov`&F7JKUh}hMbt(bV8mO^gy1hSPbZpikPJOhKQb@w8HHMkvi8<hF
zd?>8NN_<l&mAWYe=%GOLa~M~V*>U&ytd+k?2Epbf0?QShaMehOJ|(*BNKVnj=xU#h
z6X)3UzHT2XYy8yF=L_~|-2pH^t$o+(m-^G!01zJjE=5C4dT_|VrJDzn;`6=TXMPl1
ztuGJM3jkUBp29m?EuDHQsSTNxFBy^fz&peV3F>WuLojcr+XLDm^AmZExrw;XK0@uu
zZ_fjzl?Y{{Pc25LGp_2me*ZceBlT2U{AF7Jb{mS5m$$8$a4!YgBhoc{HaIOrpm{gH
z=8~E0UIM&!*lMb9nu>y^tn!q=X27#WxzP~g`#U(nZW2!I(4X+9uHAar>jo}EO#+qH
z^5IfD_mT3VB=I0~fr|hd<i(X+(^r$%#-t08M@*u2d6@OcOg>nk&409=f0U*uL0nbZ
zNM`}A?JA4kj(Jn(>Xc&JTJ@yJJn@^RL%0Hyg(be?&zHU@5%k~c*lJfi5rXrSdd}Zd
z#<X)Fbdye*Q~K^`8xEQbeS40iOd&c#8HF&o4Qjzni>*{Xo1!FNmsIzYOL{eF>0VYh
z>|e-koOaVO7l~NnUs|5-8(^9tVkpkZ_q%F@M@nRV(T_jk+U9fA+Za$z9o{_)B|2|z
z`p8U+cM}aoeU_dX(DizadYYwJHC&du*Eo2=Dqt`jE+L%V%$+0w^f+0z7%d6TJPM!u
z8RWKTAT52dzszoYC=3E+xg}I;-kmeedoX`hE{$y#da{p@-G6!vWQpB@R;~2~3sR=9
zDTKE&c!KV@fag38sF_AD$L&8(6OUV_M|bQ<gFAbu+Q-9Y^h|;Nldm3=eqn}R3XS2}
zPO#;7&v-611+9a=md@d@o&3&upkrBr%hR2bN2{eAL7Ze)GJ4_#TdBn7Vx@dEi${P2
z0lT<ja~4iG_4ycKA#z6eo=T!fId|db>t|j**AVMXg~R;|V8<Jca<2-1FVLd#f8I20
z(o}FZpD{s;JEeT^leVGBtjKc1jJbu^In<U3LRQv5V`jrMol;o8%r<!!@HU8VT*NEA
zYEIBy#a(DId~&$(5$wS6_`4VU7q|VdLRsPGR27lQf1LuXKX@Fmrx6Kr<lzKY@O^c&
z>23F$&ep|*-wBCP`Ne#I51xYrd6No+AD&gS9n;3VP?quwzj2?OS<=QA-J6G<F(cA2
zZ$wZlK@_Aj1aoVxGcW@TO(m*7Li4}8eT(a+%V-8oHI_;;6Tvl@D|Y)9UE193R}lWb
z_SV93bK9q=LSOSXE&$3nQTU?jTjgU_AVg3JC+`tJp10tH#1%-CL?+2!Bl@&e+z*Wc
z?M%~UrNwD|FR-u-bR1dlhJg(l3iX#kWKT~Qy3BMIypczYU3z73#^J0kalUDL#2L%e
z?boP%ewI}FU&VkhN~&|a;Aw-8j&G7ZzIkoFWFCEevVnk)0OjH2gxky1j`y5Z!J0fJ
zgHRDr$L?Sdo;kV5qsyAa1jJW1!(M?$krQ$Qh~RE?ja8Bg0XutyEJaRupk(S5x?gZ&
z&|xTU+cZU#DB(|4I<rG`2KeC*<4&SBlUL|$sxf2aLvBdF63DBwme()i^2q7o|4Wfg
zOL`6nuHMI^B2dw-=&U1?X(F2<rXw@JqqLpjYJ+IYikOQL?WDc*q}Vch21%}oAh#Ye
z_Gwb$eJeirf*Xq2XMg7F#Bx7`-u2+Qa?;)#rlBUAe1@)Y>s#6ZLo+(>Dpzwm-oDr5
zr9LwnMU}*IhjG|^IVc7%n3U~`4C+BFx8y&t6n684+@8cil69=F&_QjQv*1owrf;m6
z^stmzn{XKWXa?-|C%i6$Z=ws0MiLPH^~TfZ)GsT6Oijh#S=VI~Q2=h>{joIA#4DVk
z6piB{2g;trTkSZlE8u))`XO|4wsrhUX40AA&)1hgKy`6sI!c}{R51PKyWKsja^-f5
zdzV1#Yokl`cGte%(YV|M_6>UzL>B{6GBSXCHPD(Jtz1k=qz!s~-J}lhXm8xXZrrY-
ze63;<hD>U-c?KlOZ-n$UaAyEn(!1z|1gF{CHNLaP8ZO&Z_Y8S}GRkOJfL7hR2aREy
z!uO?+N>D!`he@EJ9bXMJvs-f0!~{=1z3>HwA)BS|%6os`$Ge0$u7%X%%lxsCsAY|9
z7H=S=kegrpe(ke9*};$T2gYL?Y$16*$mYTat_0ntSzsZ@V4qSbQoyPEZ-D1N-h?32
zpZgF9lDw9%`d1mg>xJvs8}}b{|G?x;<JW1+Oi}W02oyzjvO+62AVs-UrQug)yNwtx
zeIpcUPUEl&u;yq=dzOjKYu2pmRX1!lHG=~+u;B1N*ap{CiJef|V9-o6*c=vP$@e9_
z`0B|UC$}4n_nDdX%H6VDIkga+Q0BmgICsU_)>jL&+P)70^5bc?&h>UN)E6e8Ba>^I
z*rXmF6qUYA#iG>VGxTU~LyYh<<0Z3b!>4=0u)$aK$106FlAsp9ZF~^yqI<d`dS`W1
z&_=*<N`_I&)A2(=&X;l<?Q!eeeAC0Oc)>N`s+MD&&C^RJbm3g*4J)v6cd=ilZA#E-
zI+Tw0S&cG#pG=?FR3M@4q~DKd(V2IQOOv&=6M!4of8SG?{2~(Q%XYga4&3I>e+cgN
z3Sp}T#=+Z<wfSTNWm$V=-rl;j8*-}p-gVOA7eEa!7oxyIRv33?`P4X#5>uQ9VNWMt
ztt~`)C$@p(!lg@7j*9j{SDvFy@174LE`b#GdE<zu-@jJI@rOiPY=4@>@QxFl=!3zZ
zf7oRj`-W+^`&H!@b8T_%payAQEB)2a$w<orU*UQ9QqCDm1|dt!+R+n!8wP)^1b6^g
z`Tc&^22}(UEjxlF0N-6r;H>44ed&jZvLD8?u*{x#Yc80Nt$3!>@89)D%KxjYaD(Co
zUrfx@KgT+Mdq>*-^p1q!yd$@Sf)kvWin3$&Pj`*b9P~Z12s37dN`JxbmqXV0)}mD^
z0pZ(xD^7^Y5YwIkubjCz)u;;Wd2QzeTvO6s>Tx4I=eenj%O5Wc(lW~+XkX`L<U&@2
zSPV(bpjJ#akD08OH<%K{2DK?=UQwqb(tOPyd%B|EkedKddI41kp{*QoGji>zD<@mJ
zsRuPI-hY={Yx>AS3xaMh+*T+2WHh}KTeH(zoClcJ!pNe3tpytQ&>|(!m(3i>$qUcv
zoxgW8?G4Esg+f#0$D7zjo7XX5b!}EywT%!W73(3Xe7oT8*k&u#0m0-P{t{>2ceA7(
zuigeHXzfca?U|u*_3Igpo7gOhpoO=Qyd@wRh;z4J@XQz(P=nh6@6XQF`70Au^OGhT
zEJ5$9be;o(+-3{o+U~G1U%7C&Go68>YzhXHe&r-k-#$eoOR~j}-(pz+jsigU@<PR7
zq{8wu?H4iwk2=?EF+GN+^^lWwN=8v{g^zX)H^3zkCZ-3bQAv^UG6N;Ils~PfHP0=l
zr$}`z@al+&%Dg0emiQ;b=HF*jml=K{j|zBU!*UzV7mZCDluXhD70w0G1u|ccRM?zR
ze`nL28{<27kAw<8#Jh>zD$lh!hozd@nKm6eQf!s)Ld|U`fYJ!~NKIN9ZtiMLb96H5
zLuaX6SaQc>;htu;GAS9G72Au4r${}n3gN$*?$h(+zt)!Vrt+d}++~ZYSThc36ibYY
z7!{I~<;5E@YPZXMD!iv3`?zQmCA_a>V31?Ict3|!3!m)P)ogux3)o4{VqxX*w?x#K
zPlk2=*t%#--a+>|Y%(D7lXdk0odBwcN@TXETv%*{t3@|=j9g?99X)&XoKAsGV`mhV
zs-IckCgEujXEB!ty{o*K)nGs(i0z|U&Aa=S3}(STgdOwc@2R(Es<iWio)KB>SL2Zj
zc&r<h$th<$-s_b=ubL$Oy=BI1b3(`>1qea0_1$MZ|MYW^Iotj3(&N3Due~p{KhMxI
zKbbdN_ty66)c@vOD5M`QvpvhO_O1GpjN-vwMePKldQo~ketTC%jP-Sy^J+0Rb*rz+
zWw9sbaif(`t>4id{aMU<zzNg6UNNGC&^FGYg_Yuc3C^2WV*~7KndQ(1qn{6^NQKfp
z-gJoRM4hqp9f*4O-u)`p;mnynIR!3n>=X(7vhjWcU5Di4L60-e0yMinM24D_K-T0G
z8#O`q48-ROQNmG@qv@#cw~cGA@TyJ3d@5`YTOdm+6gewJPa&}=bk>r7V`g!%kVo2m
z|3$`nHF|5yxWLm!)B_8C4=X8A5GyrN@XufRuYc6tmN)4D`es4q%7&j-g6ZlPk5BvJ
zJvO{|T9SP;<L3r`L3NI_Xz-XL$#L31sV{}i*$}XSKY<;Zzr(lK9K%~H>m$4?j9Rk@
zII_&->SS1LQfHhf{TgnKUTe`gYpfs1bflB|`Vv{_D{iVg*AV9Xy(lFIzQ}xLBgZ1f
z&P&DH=5#Ec<E@7+-kjeM9X%Ihxn2ri9SMKtDW6w~YVP65pY(5WR`e)DCmFSf%S$4v
zKXM&>)nK$(-7B<4HXi0|)2h;e_4~j_HIHQkq0f9ezdW_@l<&A$6Kqqg+sf7_lTx|S
zK>n$a#(KqXiV$jvB+0077`)JZ@(h0PtYJ7l`gBu5xy|t2f)lCG#ps&Q_k5_!)S9dr
zQD<I?IKadRx|a+SzC`9+JD*O(&?dc>=%W~!1nS4_Z$p9%jRzgvg}$B=WdG6vSgj#m
zE%<btx*7V`l>FC`bI=4)Tb>WdQD*CtTARciIUoC}E=Bmy$glHCI#gy4$fxNtE!La#
ziDxFuY-?E9^A;YrpNtsKp&4s5v0g{o9T0cFS|9mYu8L#%%9ZcOyn}@2F8$M^l%W<g
zrFjjw%?Gqg*WA|fBLkAA1p|IsfqsR;@J`%59}^tgF<G3KlbV5Vd470byxNjBuEbcq
z3R!=%W{mVJkRrBboilT(sC8<hAOGIuqPD%HR<L&$O9Hd5${O4&7dZ%bY8=>5GyAGP
zDZ6qm<#`$8brl8i<+a9oMFZ}&`HhOr;M*~2F$?<Sk_USx4()uX8(lxXCne+8`B~b*
zT*^GMfrUvuK3p!vrLaK4FTN4+q}N|_%~V{FL||cj(kt)QwTFg=nTwP{%;(9mGz8tj
zMR&fAJhfo0INmir*=_PEJEcS>lzKeJ)oB6Dp?K$j^1K($<!yW;Ca+$br~LY%IsMJc
zK0mfj8mZd)m7s=Ng+_*~k6A@0eO)S--bP9!i>~JBmcAc22=NX$coi_3de^hF;|O>Z
zi96_DcqJpxyy3T3nrgUzSO(<+$gJ*<!TT+?-+X~t`n;(CFs(nhx>;!8Sbu7M8WOP2
z<eM~sLFX3o7J3inN|{4`J#gA!84!{1B_}c=h{RFcd_U4pPJ3f4w-)dU=6iRW0gCkI
zbtVOINRP-G5kr{N8jP{i3xciO+965RjuDTjz0ML#SG|2OzbE;FnP1`eYxLidOA}G|
zoW_zji{BwP67b@<S#?h_tK!Yk91u9uPZ76q^(j(_^en9&6RP{PAd$X9S=8LRr;@P?
z`Yu9~;$`C@6WCQ$2A5O%+h`-6nQktmcv9#@isssSkfA}8)sY5W@$iN#>*Ti1k!5l5
z(;p-nA@$4&hKzK(rkZPRL9U`b^zI5L8?Jb)lSAxn-vf#2n|#XMvQMS0@+>;r`>lox
zqpHtV?p1L5U2HGXPc(3C9!rX@H;3(BY#T|P&>U1<>b1G1noE?>`2o-?%2<No2h=BC
zy*@^hPb%zv*KCYedX+Wzs}fpad&P!_b`=vMaiXsu%ao!roi@C#d6~1FX|LCo*l)W_
z(5*jtIP{%6AgivoZyob(z55EW*TS&>+VH}NA{l?If0j?t7Lf6vdo<e;?I&g|x<g*O
zL5{gFuiNXb71TG-agS7Jp-{h~KnlBw<O9wa_FfHVQ`CnF7cbo>c?|J1I{fl#FRVKe
z5+hS;!1Gcr&FSXN+p$(3AuA{iwZXwP>O@w^{52sLlXf;-R5~_s;BITAClf*UnA=L{
zK<dHqZ7u04f9ct^`Nnn3C;JL-QXzJM04N1<veOuuP;QKorqRw4d50j&R#yp{9y=ag
zd4{0dK+5^_ulMOcpL~n~(fcTwSw_qK!kuY+)(t$Jq`?Wc=XErJD<)Esz69-@nrpsI
zCKg;!7X0Fc2l7#ksuaP6(WUTWXXYr23M3QBOuo%E)A0E_Q1>R^A8*{2M2QpKmfqN_
z*zTA)tUpQB(aNpB<l0!V>nL}!m<#aJY15;smPkWNsbwz_Pw4nIIobzM=cz$hl7vU~
zi_pb%)<ta(M5!jQJNF8$=@3v!O!xF1zL#?^bM_WLKqTa1f}{0DqISVnkOF-11Ki?B
zKB<_5C_xspaj<r9Iy0h)0Q`Rm((d%3o?5{6>mlcFUP8$EGX;fV19r6oG+8zRNxCk~
zR|FhYEK*lvk9$u(CwVQz`*i_#8V!HUa6mOYJ<0XUt$7Ro1q%sAhi7aHk;ccS9@yx~
z{pb>}?FH2zNYGTi{X9+Sa2bazF8a)K8;h^6U7e-2WPQQx5_Phz-mwV#QHiw^+hM|Z
zX98Exbg4!0UZ*5`;sx8#!cx;3t(TF_AhWkhEeJ3d?A4-ka&If+&Gppo>eps`8u#Kg
zfs*}tS}#I=M*j-PCO{N@y91+!c~UU)3iczrWUZ?5<V1)lq1lNAJbM@L>`z)NYxPpF
zX+U1@cw(6MT58pIXlIPR2dY+POS|nRF8lv0QU95>wTKBcZl;au#es{G^hCW;-*P_*
zn#`PnM~=mpba#g_J)W&=XpvKjVuKd5d?m00R1CXj`qhmy;b#4lMyt+e%QOd)^yc~^
z8BUjrn^n}p+*eyX-@_uY-{K}9N}E?kvg>=~RuQUdz2v)G3u&Q>owSHYi*mt*$%jTn
z0pwTRZ3&v1`-;i%KGE6RFm$R$VU$+1wYi%12`%$Py!Aqou@T#1A?`;XKJ7+lsBvTQ
zW`4xPnl1&=PwsiJIOi#^hE;hIc4w@S^oe~}X=k+LL>55LOgJ`DKI8M})3j|DgdX-G
zIugI_yl)tnqYC|cHfZ54()+&G#EOxq#krg2bcU{X9VCG{+(Pw>g@IJ8tJiko<X1n<
zfku|SqOFAC&+nOlGm#$^QT*gV^5k+4us-tknq{06Lrj^niphM`z1hOn_j#6i{j$w)
z!!yPX0P+pT`LcGf25gW7sIy>}lpx8wy}%gsXJv>va6<YR9864NCa_T@ZYx7L3yfFm
zzKfF5R~}XBM9HUlVOETdSq|Ei_5?S}b!)e{Ywvl6KnU=cUV$9rb8`7gmWRUc*Moxr
zKR~n!fC$JXAKrOq@CK%AHyES!ZBM>^Fwzdn@;T9f#Kt)6*T*;?tsS9<^}b&*J~+0Z
z(2qMZ;a~bL-3`Q&(Z<Y^2MvS{>c@QApEExH$nhv_TYRBR^550nulf<xcYN7u50wqE
zgqb+r3C{tA=Gs1f5+;onUS+Y5AM0?=>(s&&)ZbGq?*@y=u^rDbVGl37>tU;6inrzw
zcCLIk|CUr0AfDI+9NpJ|B9P6tO-~z3&Wmi76<6%86sJZqH8HvQxb|cm%MRQMccX73
z_p(AuUWW0prLFF>y_#Aik$FoN$U3x367)mkC?u?QP0eqneb8u-%GP&JG%Wd4+^ILb
znBq$9^3HU)X~1GSvN{V{w6b_d+HMckJL+LS&{(q?;WB{G0n$b?F1JX9<b*tW87&uP
z1hO@hQ^3i?z=^lNVefTB_^|g;3m(!cj~D?r)|b8hd!<>)jtkM3(@UuFi5ZOvPd&Ne
zwe>9ZYxSv??Ir)kWRLSh=s|3%`xWpk_7Aygh;wKcrWzH(g<Dz1vAyzF96(EO+PL-I
z#tXH8lOF-=PCYfibbJf=QPfskBZ&lY4)I%(?}VOgLVx&ay03qmh>MAl{4S<4;SW%P
zmy^fclSyWsKXU^$QH(E<T$=9j1-=(DrJl_PEH{OqpO0q_VrU;;pdg;G%Z!(C={VHr
zG?v;fJj&b(_K>Rh>h&tXaIO39znATQ-v&kO)VP{2l6%QF>ti9WC!P<e;_Kkw_T%A9
zWJHYQ1q7S>;*Dgi!(#5Xrv(m+^jl&3b4==V$@J(7w|w~kPlg_g0Mq_#gj&5SFAUQ(
z0<Fa;9YC9OUoo_Fob+iM`Xz~Yvn6FWj>Z>fYr9#QhGaZ~loAE$@erF`85^x6daI&d
zUaZl}&~(IF<a3?^Jyk(v(S%){)0Ov0%tSKRLP~NbMs<q^hQ;$91i0I1miLwestmNd
z5m_|5h!_wb&At`mBHnsvEt&2gI?((_Lf{#|C4{|V<O}#wDO~d1+=Ui6JwZ1I%Vzi%
zP=`mj*mr8vZPF^{A1|k3t-Sg|SWpS)QBP)T(ziY@N*8{fyTli-WIhUDrQwsoLjT>Q
ziV+a3L3oY&_NvmmsQwSzJ(JkI3dy(aHGan@wS-$r0Xs^e6=0BdIf(e=UNJpFMjrZY
zX8RqNG}&%z2T+Co1d!%@@wi_ZSx1)?)Fb=buyaHS6QAt4p<X8q7r!=hNtQ^3)Te#w
z1Kn7U#HQ^0iO(TpmAJ#mZsXS0kJ{BXZ_-MY|8=vs$d~}|(YLPr6R#KLDvu{+xD=jk
zxWXCnQ!Uei!-ht(iHMljN`7Foe=<-|!r#ZFwPjSw^__I$ifVvOtV7`kDq*o%52Ni(
zwvkOilDP{rhCS`^QrenF^sZYp+ftBx<^5-J&!|`U#prH)CX)O@kk%bJ_2c>w-Z@|n
z3!5F?tPqY434AL?XkjXTL}+@gr_GW_`cmnX&FmUqd?>?6IET_ib%HlR6rl5Xakjv|
zb$+tUguu3jP{LUbX_aCQY1ub<SlRk`@H(#Fx#qP1W-^{iAH|+Kq&V&T01w55l-t2?
zMZXg?xvCSGo<t`wLhr&xHE5G=YN~XAf_;KUqDT72*Hg7IPegk*TAte!Ru4V*l7w5R
z`OcjEWCWy`eGV2xYjtrZ%&tzK@rl`^ws4ifg!W}GY#j+KZETI&m-VpBx&H>;3k#{)
z3ohRtrbF4gz%1~?6y}L;skw^Rva+$Bh{HF7p+ba<=(%sE+i&Tr+srEXXd^)}2`Wu3
zBhZKUWJ`N{c4(x0sqVLEB(8a4Ul2I`s%9tSd$g>_#Qe!lZ*=>WUtL&??a!D#hyk74
z=Z-!wl5(1#lt!G;n_q@{_hw8Ci+UNg<8JQ)I5$zkio=2Z0UW-Q%Y6_x|NS4+dvM)p
zC0tW}Oo!25nc+^_FCsT0Ox>106GsxLLIvyw6ZJNZIO$iN+557at=IOasVtS0#yovd
z<+TT^E!yQj&bEFY+PDEt8P#o4Ph?1n`@jNQd<!q_z%iB_YCIlgipUvsUMLoY9Z7PF
zSnDQkk9;vQS)Id>QOvP^|JXNvXe5&wa<=hCw5XEFG23-D6F<^)%&BN88{KiH9yCEm
z%tt|}@Q!{XLIEanJS6xbK|&IAhITQSC$geL6WGIXhU3`0{FR+U;sj-?c}B|^VDR)k
z#G^)z_|1<c5^S<r7aT~>KX(R0I(J>)HA)ARPJs-mXPZ3BmsbZ(T(G+}*xkY(i&l4!
zW0%4h?O(r)Tj(wZh1^z-c6H8CLNbW`fXfb^jf_+Wi53f#!m@;LZm@9UKKjwg%tMJ(
zFt3@EX=-z%74{)DaKxkbXaaK~RJ0Xb#K+xC<2FXy2{BC9TRme*YQBXRCV2x;K&PN`
zA)#TIal(n@<YZ;))EP=~&_lle82ZS7=X`;)qn7t(7*i3j&#W~r_&5_mfX{V?=;wvZ
z%?Q$Y@WzGsNKo~;Y<{dz1)<N)Owe5&>WC)(UkSN{^bT;CJof&yf@9rp;2HQ9wOU|#
z#2_R6mwHU(%Y4ick}`QSqY@+0O2cZ`nz4=yUFn?eVq|ldd)i`>Z+y!#@%ScUoG9tF
z^IZQyo}V6|Q5{w3BTC;QU2<29zkCzXxfg`Fl}^{tm66>$BbRq~DmS|#{Tau`(8WU9
zl`;{t;&`<rO!2miXS(tce-TF)5c60H_whl#SqwxnHKio=kG7ZEYkm^MC@r$T6x&+j
z3+cexNfozj>qQQ%Fk148KL?R6-Pz8SYLIgDb@BWyp^^87u<iKX=fDcCQ$zec^o$B_
zA9G;E=tosdSv{jJRcoTf`{ilr72V#exbInv+G$twFOm5&1_rQRhXbibDTjcpc$4pR
z(+j%KQ>Py7*u31a_}x;23pb2(|5N{MuD(2@>i5TU8Yx;a5(&m92<I;t)w|bx+}E#u
za%X0}u=aU=Ve3V2!22JQ&z_3*@x=6jxJw$i11P`k5Wd=zvSAOG1u$`ecY6Mhg!&=~
z*H!N`gs-fOX}&rU{+#;DUp79}5fHeLZ=@L;W0<F*6#cr6r$@%UQE|1bR__zMsR*&5
zGi#4(w)-HHOYvilj7hmU$<$oKLG%_6GJ7ngJ`kReN<;~KsPYDuwBiu(M_UuDNwIcj
zn=^;)uz6h}aqw%JRVs2>Fdgf!zP#I&Z*AdVV^(@zJoY|buaxT(L72?peqYX52#&(l
z^T_VWwUIUBK~*u0v#Hf(qfn@DR&+0RuK1kUssoezY}+C&KO=Y0!WVg!v<H2<?EQBH
zy@+`*v6cyBLS5~Q!S*!=3pXY^<hpJ@^{8GkoY20=yqY|@8RPN1qI%7agfZpgzq(HD
z^WY8FTP8|yTM>Pp3Eze@?ex|cjJv-?`kP>f^?0>dK|>RpJ^4(M-nGYyv+p=DT8+05
zD_T3)?X^oed9@a^c~Z6?I`g~!{nxMk>*A}UiTMR%I06{s<S@GTm`03{M;y<Yc;l9H
ze=ij@P{(zjpd~hQl3|~^l4yix??dWtwjHn2NV~8Rv~p?V+{Fe*Km_xVWOSo39rNC6
zjG0RRAVs=1Vqka8CtfK^ID7hd!O^Is?_p(wda%qjH<F4Pi#@HV9}O@(w6$Q0;N~GJ
zUzVwA>SKl1UTc`6A{}s9J;vilh#d3ItQFd;nI@4zg{fCtyj$L<Mb4sgDlqLprvuhh
zP`FYi4@?%zE%-`h-p0FcQJ5UE^_QKsJS2f;$Ojg(%y7r=>y<*ed@qeMPVTpCZ31`H
z*3f3UUsne>fmNlt$!IN!AoI1V-Fb@u>?snAOFg3{Z1YVgM8bbCXK?<)NlI-eWL|!~
zvdq7GmIcY}4Pb)_OeQFTckU{^0(N~^d#|1B#42&PcQ>Nb*nd-n`!Iff>#yVeyHEtk
zZ(WDtZ?IS2&^Pq#Ln_$pqD{ha9+Z&@&kOIVhFGJp%-7;+1b1LGv+R#9duiV@H^Xlr
z^i7aC?p`g7v$l3BrZm>nN->z+vB~Hpt*DB<%O{NoZEP(m=&rZG<Fs6y%iUB=P@6k_
zamWMd?NjeVVRW&M!6=-hPCcE_=gN$ZQCT;g>vq0o;}MfBYKASn;sEs+PadXRw`%2g
zq@NqsgFgWrPH_aXS32?mu17xbLyo!Wip<g?{YWg&<7B424*`idgH!pk3rsPEjyZlD
zXBFxoPvp|#S+fvXDBu&WX|z}J>IJX^s_@}{W;|MEGhEP+_y#v|b_V0P7tE6GOmClj
z;rtDu)~#|(tt)aT%LpE+IlDXX(^FJUXFib;{J5?=e}MFr6@So8HXn6F)lsGCo4&4Y
zbP^Am3gZ)T<mfMS{=fV4q8VI8PI)poHdJ-7Z^H|oy4{I1X$?#3d^Do%CQi6dEl#s=
z)<)WO3+!PzL+0q07Ne+o1NH0zY*E*ZjfLIBojKb|;V>e`;0ayHM*qYg>72z~i+6g~
zyLz<cCHnam<!ZO5J>F{#*VcA<GKO-p4L$H+b%)})2hqI?tz)D5B%CugU+i+w&k)!g
zQWqmeA|bq5|D9rwOPe_rGE5SfZ}<&iv~ci&VbOZ=H}z~@dex!8gOg<9TYjp<w<n@~
zNtyti*md*`)mUSWrzY^fd5)m_F`i2NU-4`5o@T=%o!Zj;P@rdL17y9#fCK<}NY7?=
z6v8(}7p4zGSK!7*KjKe3Ql!KF`mz5wuDpvlEX&2e*<asKa%IaUK#X9?nP6m^<`)%>
z3Sr`4IF>uMNj~-`6%Cainta1ZzmVmKVF%G>7E|2R>Eig^8g1R5D&|G8R|G6J(@mwF
z+185&pvl;9On<aR>-V=1@9|oeSDfec3iy2`--T%COG@iako#QVQ)2Qaze=}@uiFTC
zXa}*YR;rA>A&oJy^0tW<NO!K(Qm>RXV?)1_4_uI!^i4P#+Nd9@#c#ul@6gGd?JyfZ
z$b5GC_%s)B1P>G+e(`jW`GL-RxxY$${v(jO?ve_Xp72=g5`&Xs^aCg5YdFAXf0$9S
z5@R^uVkr8Y(PL5v<~IR<vHsO{Ew3S|@_)Tv`~tWm+n(_*wsYGgW`M0P3i`V%l#r$V
zFdIN3#pt96OsSoxcv#2ULGoxSnblsczW_D8=GP;<!xCN!aa!8ZwotlQrT!@y0u|sv
zmrAC)J!us|JIs!yY+Z^f%r@keG(G7*qNp}IUY>8c8Bd3vKh=UTwOhN!Z}c@e(lfiC
zc;NxvyYsJ`56Ri3QP~fqt;K->rBh1I9)y;e4axKpv(_D>p6<7_S7%Ql@^1Go<O|^J
zLG!!gC3t}UWKs%<+RI%Z_u0Ui2x3_U{r^5G8lvNg%R~uJ>f+gzWlZ?#U3CW((sUby
zpZ+el{`*>^4f&Nl>Kw9d)C_$KdV@*&c<r2tr|74YHtH!IftEQW5yS(TK=>!&EG4_u
zu`z=+XSz90)J6^6|KaY<<DuUFzyGl-%aF1&q;e{a9N8J9sH3PP`(8%&C4=n5kgZZw
z_Eh#|Fk~l#grO5-?E4a9XDnm5-b0@{pY#27zTe;N`d-)Vx~@On;--0dy<X4l{&>73
zHiZYD%;ly(bYGVv{vPjpyH75&rTPR7&z#A8i~ygdzq{37QW}%s1~?%yp{Ge9Oqx;9
zv~QG-F$JVa(WJqt-o6K?W+$Z>`^j!T7MTMyY4AnMLEG7SN!AoHIs1gqL_xmh3KMz;
zIim6Dow5rsvKk&nYVuoZFV;tQKwu;e&1xx<FAk;bhv^L$@l!M5Q%lYIFE0*IXHiwX
zrTs;kIdWd;{5@LUA(i5H&<}LZ9Gmu#r#*GA=)?m6eTpmN1oUOYD_OU;SZ7NXLYHu|
zq1Yy)tH_LlzxT_tNGfF*S$F<XMOV*)qU^E@gHM-5hRibX-|tCKv@1-oOY|FSfeT{1
z<$}q}<Kxj+_+(1NMzs^5!f0)jy0<Vive26fI}_&T+CKf!^fR|Fp2|=t9GxPi1PIZ*
zjlK@pE|4)VW6YAcF|QTvAz>A$g|;x+)ujs?T9_F){FR0xdd=IqOjKT3rR_M7#Q^#<
ztC+Um4z8g{SY4#Rj~IIGD3G53NA6AGdptKIFjav9uTe;-XxjYPReu|KvWigasHc{v
zUPPhuf%2-@-;33!DW<T2$)@ETM#z~{pRwHa=l~!r>zq1LV3r>!ZP)b#8Y*Dc4@s)P
z^i^iNVY@U49j`7!<)(UTU(5R&#d1%NvUYu(^3A=+(joPBQB@H!Em1mOpf!ZItVeMq
z(Au7VgziFnbJ7xm%p>D53n8>-=DBLot@;oxclWJ{60hX&;#y_e3GUW>HYS%4Pr4*-
zlAUN=FG@1D0`6KUC+a@$)NVU#m)>8S)FMBzbWE!%aB7UrCcqBSxF7ij;V)Z$UKcIG
zKtL1jyvF+VHZM33m@J+I2BU(G;jGmE@%T{g<+{;}2U)>XrF+y78=>mWUS(rYoKH|T
zLXqYJ{Su>L%7H%YO`H>;54(Q8OR{V*r##{$mYckJbk@?zcXo<B7;7mFJPB3Itp-^=
zkcZ)LH+!_8Qcv#mXhU`^zY;~QTP}`Xy`tcwU~i}4p${FoN9bszi{up>Ljw;&1$>GR
zvrJCq$S~~|^m*{ea~98#u#T3i^bK2P#Z5b$Ckl5MWOaZU{V;tq7>P3Sd@wlaawJ6>
zRwW9n`TQ*xFp>9Wyr3dk3HaCxceX7^woUx_xOEe3`?2a2^8T49WF1fv9J!dv11`Jt
ztCs1AvoLUUUay{IHJSt7DVZi8uDHbe0JahDNlis>T31ssLRTlz$>0yF<)@zSdJ>Tu
zCCLQwnjtVsZ?bful;JxQUxCrQ%$-_t%VomlD)&NRBJdtiNDK_;5p6rW@~yy(PeP2~
z;6KM+2FRwZm@dbkW$q1v3K1yJ0cH0!7A3%$moe;4KnW&W$0_DO2t#vKM4(&NspL|;
zh;QR0<(Q-lCbkES62sLx^Ma!!KWed%irgH@Q#`>P6+j|E++aA`Us0hoTyp>GI|T$G
z)AsHi_9eA9K)hZ*WWDes9GgEoh)uDVq|i;OlpA+cxHB@?(DLVY`tiN{q1$<Lzug#e
zEc~4N;2kc0P#7fLnC|NFycljeKyMX2nDh__v?P|x8_w!q>A?|unxggLp1invIKS_x
zd?(FlWSxbLmAZR58zy>1x$v}ufPI(hOmN38vD+yb?%Ko&tu=PJGMg&G-1w*Hr)TPs
z_sKI;n}E{lgOBD<r4@1I+BvtAMYnZCi6@%n2~#C#q0UYu|D&Yh?^3HjqkMbWt#TWA
zAaVqUy@!Ma^>d4l9Mnv}%z^a3SsU#<0isN=U-T>3_bgC_?Iknyt|O4-++Di%Vws(r
zncf7L3yxdGNB;Zg|9P44vr^)4gPyEI4iI^sUGwo~PT6QU#62lzt`R93dNQS+AM1Q*
zglh&~2KBSc<j{;phdGdXtm3u#5q3IUP}(3zWm^lgO!a=9q`|~&B6>mtUl6sSMRZz3
z#_-!u6pAD-iyQn_Wt<jD^ZQZx;KQjjwoZW;-nTP6lFksUWB_G`2|cwsw+iT<ZeRl;
z^>Ef=-hvc2Qflsj><NB>B72I$9j`(614dZ6qBoI}WOkBOLriYLT;TfIL};s?th;9$
z0kWUmwZT3z3;w)S*$!^ipw#zW`un#kjP^CkT+{iNw<;gyRvkT?SlZplXEVaR#G|aR
z1ELkAQUqO*V)XM7lLPcZlEb`x%fg8#o7p0O=&B^gL4-Sz(&y|*HzPuZYZkh@CP!lV
ztrR81NUcvZ-`B8~#_f#`td&8wh|TZ832z1kht&4)3c=bJGaH;uFRJ9mbZBxkd<+E+
z7M&a>LJu8V8pnk@#g^E=J4;}6!J(C<S7)LG-W-0YV5EewyzWYHQiKCSAq6f!O!2CX
zB;_%ao4{ixZSJM$Zs|<g*JVhRucMO1{s*ZEiBZz?$HzUddXSnN3VMCZN|;h5Y_i!j
z1)Zz^amoB!r3J7U_WMnqkB2dF6^?mKls7?OtMM6iQ03I_QOT0!2{wi=v;xYNA9;jB
zL$sA)_FW=o`7~rUpD@X^vSr*yCo^w0O{M9NT8WCG*$8y51`O7iyr!a2B2hq2?A`gY
z+hox2D%_}uapW1(g>ci(RyRi^u?VPA4L;j6beuFwl^X}dHsF^)?gJQwJr%hhB1bUm
zUPv2pa$9y&50e>Yc$8GZ=x4hcY|i3cG%gy$jTM{Dw~e>*wPfE`5SLme>0X~W-c+P{
z{&*Zv4wxRgz=c%NdHT{~dH)vK6t-UbaWc#;4{i~T;0qzYXsb}Zw~jrElB@xvKU@W(
z=cE~xAV^+ej8zZF@tvtA7_NNCdJoN5ziODqiD;3)^rcUp=GJf^jXRMRbKth8i6hte
z2WY*(+lofRK!b`Mft}K6Re0S&t}gJxjo9yHBi_XLST}WOiAiHyk0AB6o_-1BfBt%g
z`3xw4CNpT_U2pg?6>(_!qX_w&ii2k~66u;?-Cc)ZB&+i=gS}yCtm{U<Gr=*H&hKN$
z%U(q6)Ob1)-OFwt2D6SmVdcNmH9pYflsV1d*jMtc2&f}(OZ}e1xW8G7w`nCker}el
z2b<;Uo7aZF+8@AZcjVJ$O5hdGzv8rig{X~C2{|93s5Ne^xy1c=cfw6Qp6h1!!Zpgf
z6UEZfzI_wgp83MQRpk@CDYCBkNwH5M=FJ-5IYWTxn6e5Mtxv=nc59WDlEWIOUU^#K
z4Z_ghLZTEc<HZ1^PFB&KyR)@O^0%>jo&Bw8WpbE9yeJ$(Yn)Dn4bN_SU$WG#OF6T?
zsMdF0<GK%<K0ds=$q`D){NQDxum?Me*KAMYY~T(gxsKV_oy0Oh@|Ucu=K%Snl`l>E
zNiUk~mn}mNl9Ey6%kKaiC2)j)PfD2oJo*c)VSnRjTsTP1f~wy^hgC#8RFXl8)Q^g*
zKp#MOG^!ZrifL0GXV?W*xZ~SV?-glhg)x9rL9#x9=q^)MQ31TofV=47=tG2;)%q}7
zqERB4FEpqO`ATxT9jQzzY;ph_;$V-O!G~WD3f7NE>qM8SQL}<vCC0P+=BWS3uW*^#
zY#8Ys-0f29afEr35M1Mlv7WZ8Y$aPNOKCa{z^e=>aHV9Xkw)bNzpXI7o$@Hd?;XjQ
zm*c*kG%U2QHG%y7N?YN<4YI=~`S9nakoaEQsJ_1VPn#l4oiYOI(X9ir`!xiQELB({
zXlUe39i6j9Y)UF5KS&y}kNYdqlksb5t;IsDs9MQ~O&Tnh1%O^#T7!d-88%qzlWGqf
z>Vmakg^wv21q-0!DpsB2tiDV=RNk?QxZbQ~e}ub<sf^3b8F`XUfQt5z@Hv3>f0~md
z=>-fwLpN{8rhAvlNDwhN;)KQVu9NBzc~sZ$ToAYwLEBi<bJ@6^tX4`dy^_?uOI*Ho
ztv|(+)XzjoIDnX<d(_%^Kfv<%KjBgS8DJR?0_>%cYu^WGz!CUcJAihdpP{5)TzBef
zjes}mG1I$#MDd#;k|`T*p$s51sz8~#kIf&U@v8g0yAjgl%^$j?f~aR30qxum-p*?~
zJOV~tgbsszYQH%5sKzEmjIf`>bC<5))df=Ia3_2GFlpYW=mk(v=?AoCIUY^G$a;M8
zF^yrfrcu8pS^;JH+UEi2Px_k}rt#*Usc{*<l3F<8k>Yr-oXBS0eKW$^N!C%}xdyC(
zGC9jac}$<ETUM*El@7z=UzFCm^#Po_-K>225gTYP%8H@%Q{?fH{d{zWrVan&)7D)&
z=$)K8Z1<S%7u6eOWkfgwPm6Q{`F{Uwp8;3OvgY$ZagaEuo_-j#8|W7%x<dcSO&2{a
zYhE1fLx&)Q!5+*LR!(Gg*pkLQSf}m0t%~!>Q2+;Yf4p;E8%l0@p&1BP*dpLjPn%a@
zx=9|=F`lbY4P|0!Tds{vo(A<3xsDL5gJK1jRkU=p<V99=m16uU>^APngIgv7pJ24B
zz3gMaq!?NURCYw%A_eEJ7MJ%JnS0gwv=ml>+U%Nl%0(c&Z)n$Uq|qa4KYn`#v>BhY
z<saOUBxXskpT|M&;5f8cy#C|I!u&0U0vlxH%6`jIJX_z)tS7%>GZzA*W=cAFlhzc3
z_$2mt+4Z-+;$_?y4XFDFTVbZ?MrxF<4DR9D#fejFaDT#luQywcBl9`|%tbBC<)V=x
zjXJkSqbxn1Ifxs{P2q;;*^1*6b7Z1(ZkU%_vWQE{%dSVMHiLh|(*`klTYG$ibDemw
z0|3Y7@Teo#m*hB5#Ga|((E;2<WXZbk72<SmEmOXKC&Yqtyx-74d3xP(0ee-zEvm6u
zK0jZwLvA{e&E^bSvDkiwC8pfi_LmIn@Ii)kE9^b?7Q5`XM^>NVP`OG{#9+RX$BU%0
zA>Ep-B&ErXp^|E!Mbf;UjyPIv<=4UW)c)842(58IXr&Vm>exEA31X}~aSU;3Op`Ad
z8l$R!>UvNw@Z?c&6b0h7Qp*g)I{MGKzK>tE)QBHWt$w}DJ4-CC^)2}tdyfTSnHjZU
z;CM-D9buIlUXXV=+Ba9b?b)M`t;ZPN$EiuagwLjz#W&(u#bxnPlh5#8#Z2I;c`IKl
zC5}b4d()Ar9Id>^>@ioXgmK2Rmhd>7ld_dudZnJbmiTyLT2h1QK&;O~R+J=yq=#;{
z@a+bs{t*DkTlf2aR=%BM36vBptneY611HD(c#w_yBu)LUZxM0Tt3DFI_3<-5|F<vr
zA0G4>FPm$c?2NN3(!y4e5YTjB2YHlc$qe2iUN_tX^ztwwD}stfU-c1Io%88YrbOgM
zC$XRzrYA@v#kPVf?;n=6jCAlAE6?y+6V)UAwnqbzGU>7<87BB(K8)Vq%#c{fTTFF7
zto=tH!!;9nn&-q(bCLWPMAyxJPt}teH#Am(#-)#UVc7)N?C~jY4=ffm0xOYDk?X7M
zvJ%CO1VDs4%JuY;YT8B{F!0)J1S;`WF!$>Gq9X1sXHkCs*H!x0tP!tv8}_UIZ_O`9
zf0k*74gzc&-CAaPG<2hK@ATwMI?{h0@6~$!-bQ}^6n#`8&gp};-16I4ja(*(Q_Z)~
zqSd$rE=2x~x#hVS=w8QJ|2K(+qc4k=MeoGd-``nx{B8JywYC?rz;sT+`^Ulk`wG`t
zT>9%!!I-M$%#1U&P|PQ;Ck-GO6<DCOWuc2Yd>X>gFX;!UlGQ`>f$vz88GfC<g-sh2
zBoN^lP^ECbV-9PU>`pMRwox$xG1t}~Oc=8S0!u|$b!Z07R&sNnMnu$$D&?2F%nO}_
zz7LsPIkp8%Z!2Az3PrY&88o1+QchI_!wJSyLpgbx=Ro)1a`A3+RvNi!3rU~QwNl?i
zu7(Z*rqpL~Q})c#t68F|O}m8rY{bPqw0qK>Ll@#50O`I>dx~Uu0OUxBiezkzxZ2P5
z0Eap-sR4Qh)ZT#X?0J1K<c{F>C%4bpx(tl`dVcGglSadvvoV+VZYBtt3GNhXkn!9}
z!<&72pJi%yI|Pfx1M1G2x$doeW?CHfLM&G2*gwSSGF@ssO8cu5?VL$Lxucv?dv#m_
z7y{~O%w6tyAa77>fbxTv2}_4gB-}=?0#4OQ6sPLfNEQj!$|@QkheUGt3xEQx^qr7p
zb2705jH=<=g(6dx;5>$fq2%ziW}5@Z^QyL`LEaFtApc#s`MGo6FB|KO|FF0*_eO*n
zdy)19_XDeC_J(q@Y9UO?(;0WysXn{eG_(FVa!~dG>8oaSwh6x<VNC`pS*`!fD7!DE
zA2!1%cNlP~mc&Vtzp#^m0xWSJ?9wF2W<w7YnnRS1?>{6(Ht#rWqD|1*ikk$DIHcl^
z1l^JLOrU4=_*1oi&7yDg=@kEFwMuoLsIO`rl`BI|uUUT`FgC*Kjvlco#ie=CuzZA*
zt3pCMw!ZX^9x6P6HIG*SH3x=7g+z0UnAJUWW-7T!BV2mSs<1)RhSVQn;us`tTY~<S
z&Lwr!w$;55pd!bjmaqB7?SM&J;IzbJ+unL`6hvZet?GuqC95IrN-75@;{$yJ^u%W0
z)#c-xMihf;A!ukcOG+yfz|q_j#qOrB=j3o$?oeI0lK{qSujwT=^OW?L4dywCtsB;B
zEA0%i-(r@=Y?dlZx}IvT&^^aT0RrNI33Z*suW!N^TB%+{CO^~?`g%mm2in*3!~J(^
zYRTJ;pMe)i3)2UDzD7lCk8>Yi%8OBagB<&!ycbpE0r5A9__r(Z-@f42+8-)Bd)x?A
ztAjD&-!Z`tqYTbO$g8Dv?<okQ@bGUdhuiA4eJ|zHP#{4c021^%M8MAgJ-ZrZF9%<{
zBQg0Y#@m+gGHqRwtdE$rNGN|Cy6tPz5U!xv59-S1);0e{E`EjKx+pNR5)z+ym$jI>
zS@kHOXdG!AlmY>Tp8^Hx_dsZeOcubryQRwO;i7upx|Iw-d#I_fSg1&->z%Un1~7|U
z(;zJRtf9Ep*%9Q#tc>>LZ8;JQ@em(x;^Mf?)+%2N<X?vZ)x)gaHukI4)vbH8KSG0f
z5m2pCPy)s2klHR9@BX~9=LKfVjSF)xwJL^BC;1vvx9@FFcf{M(Y^~Q=KYu5@)DBjD
zb0i_O9LQ$OCXfDVlnoU9Rr^JM<FTl@**-lk#eiz}dZs6`e?HSJM2(zb1B(Vo?onUP
z-`m90?3umGmX7>_9F!;4i%thr%wTTR=0!y5Z#R6kkvoJzWdfcIbd_$iqD{M-^U9z1
zgs>G|=kM$yYi4K1OTww(v(5->`&nI+*Ld^$y$xcjshkn@`i~I^XrK+pZ3jYEG5s=(
zk&sLmEWer8z=Kj&@WOQtO?|U!UJTB)^+4Pqthuw=0VKYm?!4Q)w&J?R>XHayb8Ga0
z7Q*(~xG@%0(}&G88-3tX^D*nVM8iQ>bxEyl=EvEq!yI%~{j$78+FqEEEqj;6dptIG
zowU8x%ZFQl4m1Jja3}=e9jR@u{%XzYsv{2$?AgV9O;Z{LfOI+z2Jsz#dPytLWwXsa
zUaxa1O=Rk3is3|^%t&PPupl1gA`KR}-!WD3b(nXeKPOL(1-qHl<Kx4K*lDA<SjW4$
zp6bx;hS4YaU=%yl=8o*X?L2{zXU1M;7q5>l0okk`->i<@_ppA`;fdIBxIQkF`Wv?H
zl|FViKtDqAcV12H7O~AfPm-o}8ol(`r<)!H?~_3yuuc8sa8%633bgNAa7>kyJIgFU
z!eIGq&K!|E$ncnzF_C}C@R+}o6D-XsHyj7OR{4$9QeV+DTgW|9anm=~{`zA_{;dky
z(ce#2ECdjClreoGjVFr_7JT)RAV^vcL|}BXtzS$qFKi80$>7XEk0TXQ1qAMoFW>Ch
zohs%t>mJhPUo~vt^7XWav#UASI;;0R+)RwvZ97v-t#GkwYv@H1;8%SPjv3k{Eau`f
z?>S(tG`#~+u^Xc2p9FsxzvwjQP36m~#mpr&#2>hf{G=GTE*weZy0DUa&%sSLhv_$k
zF|1ElrxOe0$QFLdr~_|1OPCO8Ldy|F0dar_Y0k-RNv)_ff1eWq;O_rybwWTU(7-9P
z8gCN-&U5H3hU@iK&y|K4Vh~_PV_l$zu-Ub~Gg)O;^WL~JN@hv@R}qtqmQ|F}Jdmrk
zbTKih3ZxedV6r^CSA+>#4G=IHW0t;D2}gXO!JJ9X3m)`*K<>Il0B6)I+WwI%tl^@L
zAc(D~(^amPZuJecq-P8qwwnLCB39VjGbo>U%s0`<WbYZVwEo#)f;Mk8n>F;hFDI9!
zDm{B7&~-|pfd!VG+P%}~Ip5?eogz!Jnhso2=OUGPz@b>o6>}G-#hZgkGsU4wa_Pmf
z@WE^ZeAp&<*cdVPDZQ5YjCaY#dGIh$Iu`%{<p~Nv`H$^`ttU{VilpKELgnI_vLRQ^
zt-fPsnP%+kgEu{}5kk(5-jF*uLEu@|;l1^Yy<58rwPc&8pI-jV6aT*HSdUQ_&Fg0M
z#=84_Hk0e&G5D@C;|Q&@wQ%AVgQyqtd@nbGz?ew&fU}B<SSsiqx)&ft7+yKDgyMW$
z7QaPc+}c#{7+`S>21-$mu?cy$aWO&{PgL_%Bp~dHoXm|;d@wB>BRicvyV7Xgns@hq
z4erEpVIOy5vw7qw?Xf?DmrgNz|8wMZDk7kk4@m1EJy(y|^vzbinu<sv-7a6%+Jg;0
zOfO^fnQ#&`rq9S*xTd`GMPM?+e{_}-Gu{6wW<vLmscNGIg{Fz!bEbs{oMceJsz|w?
z;PV7+_m0xAMX;I$_<s-yRV3q<AnOxH%r)@+ez=U%a*RcpGyoorp1$&^UPDouh>!+s
ztV`AjNP&6{2QRihRC%8v@%I>RyIT!g?W0w9(^_0Z(g5mumetX>gOJC)7~%`+>b6#d
zkF94J1<O9DcX3)J#ITG;LO-#YZ&I1FI&7L?`ppKDF|r&mTOoYZdgOIgE{$EO(s^w`
zzqk!ygs(})<k`eChb5u=)oG;ir%1?;bo3hSL3IjC)I8Zqc~5ilJ)gfmVh~{`H}%r$
z^Rj57espceAva!Q?{%5adbK;{a;5xJB1;93@_gg%W_VmWanHeVW2o|Td`)SN&vYe8
z{&@Q&q&myT+pY$8ALfesxAfyLUyKa>OOX1HjMB7Odg1(@UX~VsDOG19lzPM8aSc#=
z-$u7e4ROQm?2bg-2$ppPI^MgE6D5{aXaH&7JZ4GA!ix2UiI^t1S&D{Ax}kr6=;*tV
zS;ke>JaU;jcQgce5JpPAF#dS-<mG-JRheN!sj7~8^>c#1KTOcR&Uk*Jn6@n$S5}L*
zo#F3npSIX`&b;g!F14mnR=bg|d{NNPTb`*BJ=1X!-(ulJD%;x3(uQ41Pz56BoAcub
z5p*-SZS!XYy>t*kryAsdD{w-`?@g{DI-4Q|aorf)L5~F_dj3gn<w)N7dT(Cdb4zWj
zIi{vyU?-hV(b|kQ(BFL}dqtr@7Ol8Gz(Vek<Js<$aT0d|><|Qte^xF(e%bfl3O_Yv
z*EaA0Ir7QCQ2&?XZhl+rgk39v3gb{;-FV3x-C7fuUTWvAPbd<=^AtcS>^?bhUOTT;
zd4R@Jc<YmHrmg72r&mJ2lbwr8rZe~?-4t@WaLRVqr8R}(RhDXo9ard@g2EQIYj!(N
z5~r002tx1zVFHB$h9l+NE=+;xgS=9?IrUme|Hdap$H<ambE0Fm!@wY;$|`N6hO%y1
zJq3pKrLV!eoBBfyyHgM0uATFP+gamDoAC?9X9A~cr+^Lo55X6}0$^{)kJwD<y|gnZ
z^HbXzPFb$nmOi%o`R3<1yBdStt%#H)zUN;RX-&yRI-fODylacBrTp(DoUJuGbis)!
z&>H%i+^-sRBc}VEsp+0M#~eGjiQfWP17xBFQc$GcwO|#%(kSp#n>yuIXA9J(Mu~!#
zAsL@8f2!9Id*gTx2sM2V*YWG{4CPCLZjmQVlY`y#bo3&2F)dbwNHWTb@7UZ^CNE^t
z?tK44f?2z-|4L!yOfXQcHNQI)5_dMBhqc%AHb5^W@nc#&<?Lt9Wg2%e++`ULZ<tp5
zs1+&apa^DFxX6lfNLK1g^1)7dmHNA{llzku^)1<~zFlU9Z`|bDsGqK`Vv@e3EG5)w
z-=-%^)oLHHU-<TSNPPXdePUJj3*T2U({B%5cz|E~Ao>EResNNyFD%Nx?X4|?+no3{
zW9nMXoL#rHxsv`&DHtp*+u2R<c;4Z^Ub=8L@u9}^zlWTk5nL^lG7vH~7O%;J4pHXp
zK206bSENT*{p`1w*{6dvRG2TaCemF3lGo!%eZ<aVCE0Pc+;<)2p+ht-L}At>V9*zK
ziOaTZbyOtuvM&N&0$s+dlO2rvb!r|hOQBouX%Lhjrlfg9DpwD1pI*IKT~)VvBR3Dw
z{zO!|v5QOf)hQB!x~Plw{Z-#0<)v~p&cOknZLCiBZrU4>iKh)kwbi}swk6;}L3~tw
zXbEgqwZcVkCmk)PgS-sbY<~rMR>wkkxb_216Xb5wPhR;{(D#6Y^$u2XYchbxH^X*Y
zOp5!RhV@{cTaBxH&3}q44T~#P4(y%SoxSsUeYm!IXH7GI!S|P1A0Wj6AIwMI6u!$>
za$hcoF$t<vmbCB$SOq}p8D#2!VJFNU2b@rh&;l-15RJ04b3?b9JQ=^)Kr+RQARnmZ
z7e~)_cTeM6jfpq!Pp6*Rc*cA;*8>e8b7AD}D0h!gsONyP{qK&Ze2YSTLY|)Ykz7JP
z)M6I25SK){C$C`v*$6+^t<jOQ(YYFesKn|B_na(-4p|@KM$WUPr?nMOK=l0v5(E*B
z-^KLSX8Za>pe|ub)eH(S|4jnmz>Onp7&Py0vcpBAH(A^rm;>McXLqBQ6Pt~e!g#Lh
z0>_?14}p^SERsoO?fV<=1;ocwj))&MQ}SO07l<!zVcM(3|M+#YYE<=mvr5xs-=Vt~
z_EGsvTIC2G8;!UMYv2H#n=A*MRRMlT6j!GuFqYLA6eA=PG(;>zmb-E<jeiA@>r74G
zd2BET#G_6_0P&E+379-qw-nP?mN$J9wcu!u27v^&u3l2To$NEES`{yehbY4q!b)7!
zT953Od^>WBbkwedYwROqB^OZ6%!u~*Knd@FIuNkxex^?{_$n!fL6u3h=S>Z1M{aF)
zhpXncG!O$b`R1^ZVUl2Nx5Z15euy|PjE*KSm+*eGN`nytirv8)Uj^bn4;!W$9s&y~
zxIEMRq8PILGQSs_;B+QsbE$G#Ci<PmNCL#~lAP_{Fh2QVefK>Qcxxl@m-<rx8QSmp
zaorXx#kt#=>}WnyWPpup;7937CO9qGWSPVs8Ot4Bz|@P|xP|wK_oyUD*<Rslh%T6q
z(K4}|H|dsx&x#PI+m6TcuM*6zC)MU%ja+qRu(eo);fg}1dH>uH60BV1?kDPrI@zN}
zfN34I6!%BN**+yEYJ}?fCQZwYi$UV&al*~_a={rD6+W(giXE8VVF<Nr<UPHxkPjED
zf&f)2nxmC~e|2?GQVyROTR|+dmRd)DnIFd96>!}=EIOp(28j3Sq<-CKVY8XClD|*g
zF3~J#g!RTTwPx#9-7S7TSL5cs^?6!ZverdRiTYfi_1EJ3GnE@FJ*EHYiUHu(PsnO7
z6Z1^_Mt3wE)B_S!or~2|VMbW$cdg8yo!N;u{lAv70@jQbDCm^Tt>l*dHpGXwh`RSW
z7ED$j3sl<bZggMAoZcJPft*#?4XN$d;8fe!4}>fIm=peODu@JAW?+eCPw|HFv;gg@
z*Lm{K)YbA@3<u-+C2C?foj52u5VlFPImu>|5o%29kZ7>^Ne!-1Snre?O_tkt*+$i(
zwt)edrTAE0FbhgqeoG8!9@V(r)`{2lC-;R}0LhwB%!iO|D~Qo=6~-2yI3G}1G&7v1
zzR?Yh8&tFk6OHzC%js*RDmA}vrIB0RX_9+AAN*;ntymPD*kX{`a=d48XzUt)kfVTo
zB0ItsAD`qlC)2_0gL5ER0c2k)9@p)@{r-9abP#2wp(|4S_@Kp{qkZ1(FOl}rcQL^F
ztkk4v%1ViA*O%R6Z=T04c>DpDdi&C(a#LaR24cu{HL<KYas3H0zIkU(6oKXQ+a$ZL
z#yI{~Huz9e38EXVJX&*aoCkcSnZLp8YUoP!qT}h5y~Vv53DR*!pE;438Ks!rEvM(z
zq|F?o6Tez^)e-08lsPQZSx%$biahA93wW&gH$By7SLQ_2mmx`bRHaUbVV;08%)^t8
z>04jlZXcd~j#;}wels#%e6R2d_>m+vGA)Xyd-5Wi!sorZ_a*LyHyP5vFjBu>tygjj
zkI{!6*GA#2;OO&5HB*E%c_rSjrzeclh_H^Xoclt{w+zy}zBBQrzf}_5_Ttm6>q+=n
z{<*@0uecyUl<CR_&>t4*FgZrpRT<(vjYt$H+-IV&j68>`<U?0+l(M)hou}nZed%yO
ztB~ZZw|`_I=D&aM|K%FhrYVwN?Tx0Y4!ZYaSH6I&iesD(U8r_ppNvcJzYSXt2gfNm
zPAfyh#X{?{L?K2n-{szF5+S9!s3-j|HOt@nH~g^u6#M^SGK``JLeJsiQ^;Ff{1=LV
zTa5!!Q591=Ox8Y^Ah_|?HHAZq;E?_q@YXV*n4Y|t%O~S9&rZ{^c{cdN@)XB27QT8R
z=4)~iR}*tM!ybQEQvOT~fBdmVZioQekQV?<j3_;wXsVRu;s&tF2zwpPQmu7EOw379
zDn6GfBQ*usqv8o}OO%ybt|<KDE&x|P@O4nOMJ@IBRKKp^_jzUoKzaB4<&JcB_0;25
zloJzni6X<h_%#cilKuwo8HhK0=_;c;H&&A|Y<=15zNJ2Xsdxz`g!Ej_M*N9e`PfGr
z<koMNBp{5fAU0bcI8Zyma{xn|tI^~rK&t-8XZmA|CqmF+>S-{W@?#c>uySl^X&?Oy
zl~p2}jl7RF5xk!#T0<=8|B@$$0eNB`U<deCNQsEooqR+}){i!-%+Hl`bm%SfaO$n}
z@CbBn=ux@MsbStwk=}aG|M%bK=KTIV)chRXyroxkt7Z*tGTB%#ROIQZ`1fk>=O#rm
zQ|?h?ees%i&>DZhE8-H*7M<&{2?pmo$d1mY*kJIRL-Ky*G`jOx0;X{3n<yvW`lo3~
zDZ!*0+YGan95URTw*#+aP8;LHW1DYu`<<G7>OTU(v97&k@CtL|j_G*S4w+vx#yMM$
z#fcq#_1sfcyrW#8ueFG8?%pd^K0jzLbvutTEr(xS?#*r}D<p#d>e)66AbA3f|1U<;
z;=<~9!1^XIVjM$9>rmz5LeUmi0aW?J(^}qt>L$uiQteTe3r2H&Ch<OWo}d{*n1XR2
z(9)-S0udU0|GFX(Cwf1`5HD9OLbUNZ+;p>lce8(&G~3IoJ0C7FgQ>s%WW(IQ16L*r
ze*mDVTU{&BQ*Qo?cF9t*{k7RiGa@OkmWw$eqE?GA!f`rx9|$qLI|Xo_pb=2Ekk@^L
zC0%EPCUo=-Hrr8AVsl1S5FUuqmU0j3*yV1s@{qR7R3pc=edJk&CVsQ>_eriU2JD$F
z91yg8Z(@2+v;b-6x@_4p-|SXkg0w@0Jz&7B8`XB^0HM!?)wlfxp+9oa?0J^?9(RjP
zc74LCl{B=L>e`&-5ue~xUA(iT;qhF1x5KE_NW3;}pmDhK_^<bw&OV4>YOLCOrzg4M
z1hS4sW#mNN>42EjnQ<{~YEzKQ=Ex!Q=}2ttwO1Xl1OcYa=u^h^-AB{BL(K`A=SUnH
ztd)s$2rk&RQ4cQxYYmh3Jks_pS9kt_U4M+eeR8-=(@=L;hq^dng1z!?TAgd-Xja2y
zwU|QiaZ~#0YW{}_YKf`rb`H}QOhL4P0{0=2D7B74PIrQn8EM~r7y(dW37S>p9C^69
z&(I6x)pL!!^SJ=LVPL}1w~IS49|rF3-2YNLHvB)Qxd6nE>vmyT7a)Y*v^S&vL)ZJ`
z;S_Q!Qvj1Wv$W3&ZK24s`<<v3h)Nl2B{}!>f$J`LQli9n-U+(ACd`TAlt*AU2_269
zTio4U{Y%3CV%qY$iB5KGt>sBf{dmhS6sZ9Nic>Ak&It*;`XUv5YnDQ^!Rk~*A+>N1
z`#<JnQmTC`nNS$i(oJNaBCP|A&1VL)RfrNM!bQP1%c2o2a$Oe-6Jnxv&F>~Fg-bM(
zmgspIFT{ubH-MkOR5m{+0FcPf%>V98%_c`xRew2<Y2@Q6WmRQkItPL?<BaM#FXnC3
zH??@UD!-*ad@VAD_p!VmX<QB2uTqo9VD}_Azr%Pev4J0p@h+uTusr8JJu24xrhVOS
zvzap?IthQ`d?c5n_|yoi=3#ki#v>OE12P+q>nXrj+Uk)k%W2N)8}?3?gz&uimWE52
znvt*vmRa|pFl-Y(<wM-kSov!b(w@bBLLrAdKmO`vsq!+hjH}e$Jt^+LBv|_Okx(58
z+t>4p=LYzfL2EvScaX^hvp0v~fOd-+4631~b_xiI2{C~N!(2xp=D;?%^dg&R#&>d&
zDnVlAN`Fcpqr?mVdlnnh?TF3sMnrjqsqOr6dOY)9q&&<mS2Ld+rnYg7S>DpbeulB;
z&SIqIR&Wyl>XbysmzKucdRSIp*qQ;oWqA|E)f7|E-t4qv<KIDG^lB=0Q(rRrT7&<l
z$^TIehfFH=+tz9D!=MN`j{Y7i8or6Bvkv~#CE^mBJzehVCp(RcteRQvpYmKj5L|v?
zc!~NsSO`CUAUipVJNhfjGD-_lLu~(ArGVn-Vkm4p{x=#kkSQpLrG{z;z@N1zSe{I0
z+jh9rSqp3(Cg9n9XZm_4uii-MCN?kfV|$2A)+Qvr4lUT9O=<nWusQZ@JT(Nd0>E}6
zF_XU8@{9I@%Ts(vCxeTtlaINUih^1U24RW)o)5&E_yO-bz@gO33wk|qJx}=C+Y0$7
zzKm1FBA9d?ut@;GEfDgCA!l4s-ewOL>Ut#8L>znp5F)F8Sbv^COilJ%RAV6k79}UQ
z4vPxCGC9aBH+Za9ek3dAu0Uo9sM=y1F1+~v$YuVOCe}TT+;4l`M%C2ho479<DErWI
z%4@!1tO6097#tm+yio!l&r6Rno~;u$zp1^T%9rDqyFL^0>ADbh7vEpJA#`Vud$tz>
zrM(Cpi`H_mG}>kGvv0atnOAS1ECIm3riF5PPW*b5%aPaA`T_9vaBwu_M&Th^E@Iol
zpB>OEGn;gW1nRN8Ctr`aowA%SXlH0B&uihiKSz23pp*Vw#(SyCAH|x=(0zW`xxs;?
zSota5h!}TclOGP6)&~2Cr-9$&S3gU)mq!l}&$n#XlYT79zdrR(w$J(QeojXbrsWU1
z*GG)F`O(2ZAE8%61fFjG*_7I9)u|;WW#Y_mV$^(G4ARhtXR}Q$?jNp5{5DyI|1v3K
zQ}Dhfw1P}<g3PboH3&}PwoUhr<GXpidXU?0%+Btq6=aU1_(a;0Yhz&6WGC~~tL>Tj
zdnj}R6Dz{1`~+f%0GKC{^oR93Fr*s**9p8`tWL-0a>OZ2D!u~$>fL&Ohdy>@K{x4Z
zt1R4Rx{zz*M2ffVc;w)&?!1e>nS0I0Z)3n*w(!4!Yn;~0E$c#;K|L9LrohS|EKLfj
zfnj9s^#s?5)3|Ig+ZekSI63dWSf#`a@r$}RslnR@kdUODjS1vs+`<pcJMl|2l`1R`
zd{x|z&GCJ{7Z~#zYK!*GyAHD)Sh}$Dm3sr{(J3DydUBrTJv5l=50Y~sd(R)<ayt%H
zsn0|*)3<ORwQ7{`Mk`G2?5ouZa5%^rVwljuKW{RN;;?hBi+M6VkdvcjHnn6iR&1tQ
zsgG?_joboUyzFi<8R+$V1AuR2<F&ofea3aos%`1VEm-&HyXK14=JU@i^v6H{*bO55
z`;067Ye3C<Ov{<>L{4hMp#a6>W-6>S1E4M1zOecxo+JUR%Kw=rhPA+;8<<2C0SZNM
z-j>^`?;@*}<$E(fPJsExgKL3<UV8GMu7v|bM<1H@<ij$GfOZBHQCy+$=!h(}vp^&q
zsV&s|HajVXn?RUo5D~&D+K0K|Cj|g!9)fnRF~fN8yhwAP=bT{#x24=yq=3Y*uZ@!P
z2JsrRlDeH`I>g&s`{9vR+jU?PaPB&-9R(SopSpE8*pN%W!kP1UCMK0H0+4{rgM-$;
z)dv@!@@BfDM7+qtDMz5(v!=7U^J7a^8MShW-x{9{-0I#PAgXG7x><OWDFK!IEhc(q
zdVn}>?;j>f3{a5-u#q()pdA07mjX~s?x-jxcPZ<uoV>7U4LDM??wM?`$pVO<A&#Hw
z0fu{v8W(Jo8pX?q%Hs|eFumA+U~qRcJ@2w$b!OTPqx@j<I*P+75<Xiz;g8o|poNF*
z4s{z4G?<l?#Jf^n26F}~&OD-{7Kk%hVW>dds$$&byiLH<hDD@VHB7I3bhvKS8(zF;
zL!O;|#O)7?>Yk!O!N|v^UFheD4jyxSn$>rfbilP9kB$bjNYu=CL?*VN9Ccd{Ebagb
z&H2CKnh?*@cgmW>kzZ;G26EIr>j>cU1Homf82uh}EYS%4@8Yr^b$3tw|K_rEIMR^5
zy~n2CgHgGXfFsE>U_@{8k0W{mW1VxqY)c6-@@IW#tfqg-SS5m7!!P3}MVwEmLlAR#
zhLl8_**6?&<kEci8L)Qx<O#OsH?Z}k)A*Lgj1+Ftc-h`>d%?1Ew;J@fAlunFF<;fi
zA4$5)Do9GNXG5>s7nTq#;)HHQG@6TDR#1pnDLIrajhHnE_Jq(PI)YRMc5tJS_>?3?
zI~%9fHPrZq>K(L#!db+&JW>is+VirB<wVis?R2kAcErw(v%IK+RErCATHnk8-c{gS
zA!{~z`+r(iMPVNA)OqxrEC@`ZE1-JH5Ww>E%JGakS)Zwj*0VONo$RZb+UHzH_?*?7
zoxlXVEZ`$n0(Tqy=ESlx-;GP7hwotRU@%$*zo`I4wW?M*xmX|zg9K~AER(0x+z<KR
zvN>WYKtKH^EZ$th5LDvDzGK?N8$fM9`vtD3uv_}xrRHjAu_S<f_d?k0T-4RAzMXzW
zXU6B`-y+skr_L`o3#j`z%rOj*R!Irsvgy!e1>dEoW$*J<0ohRFf0t~iFsK?0<kyF=
zR8{n_Y2*ojBun52As)$~M>ejX$4_2L)rsitTW%2OjE*i@-psV~U+s-5uw4|5_R1TL
zfGHLihownN$%;5A`Cu>mTfZ<16c8TrS^TW!@aRLFq@|M&(c_MNSCRBI>RojmzJ<iW
z{YiDeC9Q0|IsnEX8XCdt6kWHeb~m|hn@yd>#OsACDwRv@rh6{n0*Beol!b!Bee5^$
z;Pz7Ph!ytnK7JBMuw!l|dA&?(Q`o?$=i55KeonAlI2O*sx!)-Yg;=WprBf7oFj17a
zqVwB7osTd3=VRwE(VgNC;R%9<o%WgP_<xIn`mdGU;@Rf=su7_KA-^dw5~Kx!bAIH>
z4^Rvx+A}L4Xt5x9rXi5GQH=Zw*NOJqIvo~uwKuA&+dKrfNBNX=`2GXaqz(=#M&Z|<
zA{P{!<hW3+oC)X3p15=WiJW-HU$xFIegiM(CNV!_`uyS49dvOS8f6f{f8|U7T(APe
z0}=A|vC*AZRd3c3m@X>Ca8eA+MH92aJ52isF)_SdO>qAe{l^kS8P-ad34rvE0!)7H
z15TGR36=YU`GgK}oxh9=E*}i$#qzT_X@8wnMBn}nDS7SVG$~sm&h+}Opb6%Y*oikO
z0x%G2W{ECRSi4qgN!ayJp$Nf2c}}O_c&sqit_tS*g~BI|Ui;t1CpCBLI+0?LDU-fo
zCT2bNDPE8tBM-2Iaq#csKR-RkcPxQ1@$@gPL4f=#+VfwqOLwE5tm@Q=q+iYZmX2*y
zJ=GHiA-NDqV;YPApWg>+Jf8hQ2Lab08u&>6!3R2GrfodDN&<_+o#0HPcbUM~W|Id$
z-EZ`?ivc>vG5p^_BH_P*M0<N65NL02WcjH24@3CsLy6Y9Ueo?~-iH<rocHeA*t4l0
z9jK!VDZD>+MC$NyUa4>`%ZsO4u03msl)O5n%wyGdLhFdt5lPKA0o=webT`?I@zkHk
zLi?CBWfU%4Xx~*HHu7;wd6@GsE4%ES>84UjN7ia0Txr_FY%Q;%w#Fmkcema#)RDHB
zE|ji74D7wr)=Sfjpy(9YrUrL&yVVvehZ=Nt7JxMkskx^D_DcirSyy<7qD>XE7{i=w
z@@|{3|D61QB*L|@7c#hu4|}H!{IEPmKmL`4KQZ8ZYy)AJVKq;CH&LDwg1}p36d0Em
zWk0=n*?jb4$RN%McpUAf(naFW?jmhqn(&B6y7xjk+(s&eF$8#NwzbH)Y_K)XD4M)o
zxV@mz>T9=<^yil;Y=RlW;DdMXy`I6((F)>X;30p110B18qS+?b)z5sbA~8#6|M>O9
zN{BI%liDM%ri}1`ywmLa)ti9QD4%$ysg=cNVA|~KzQiE2Kz2)km?VRwX%lur7_y;o
z5ahV%M<2^P<j}6_Vts`mX?#>ZI>^tJK;fOd-^6WgXJ@qr7OSu}0JR^*kf|CG+H9ZP
zp;WBAF|j*XOyX;cmrNTbKV3{VG^uree?iBF+fu+<2E&YZG#sJ3|GV?2X2+;(gw)F{
zNqw5a4}ivYBT$RQ`~0{Fc6TCs9I)}BDnvJ|mPiS;;)k~Z1^i90veJyUV1pYT!dISb
zK>eiJNZDPHV~?rO|752RU_t#D!x$k0_+9agM!d}Xc_P%*>L{)sk*XElcZ^;%YO<cu
z_HVMdm*6nc8PkFuF0a}dsT_7Gujg55cZ-%SA`~xv#qoJ8DtW}aDQ_&VZ;$_N?)q<)
zTJf<!0{{}$X}o_~AZIRsrq1`ksOjZ_QIlI8Gje2CzC;h*A(hON2E7Qq$O`k$1YQA9
za|~L~&MHv#4^y1t)=XAJN`sZjuO=n%4ZuqzERWYeg;X^FzTn#;_%`4tD}!M(4to`e
z$v14?f1K*SUg2VEm!Xm4Z4MqE4O4ScolG?qvg6j~hlxK^YQH~T?u=)54Rcr}e8El>
zOLw5jIXR2N#EFkbg1I%tE%g<^eX?L!$7N+4^FHN7aGOa(CfC?@pYeIhez`6IunCEr
zp=&Gs*}_UY4s29Wg+S{$htkuUtUl^;BO~x*S1tq>WzoKtb1OY3F28;6UdA@&6-x^Z
zc)$#{*YZ{7=n=DD;wFHt#0$nYz@E3|*xO_7OSHTT8*VWvJWuUEk`D-{fPW{PtO3GF
zf1hyjjq>Vg>_y5x*v^p@IbxDIxB$2{G0#d?YRnypap)XwLMxGJvi1=tb%L^Ivc%$1
zzlR@AzrDt|zC=BK8C6Q;<OVOasVa7v)2tRU-C}Cn{$bZtE;GwM?w8$XcKg92MOSWO
znT+JcDB_!bno0@$6P$+96`2KwJ#z<KmjxR*(E6t7=<S=_F3YEY#+Hbma54F3UHcp3
z=HN_=R_fLk$FQNNYuol+okJHicH3H8^DC9n;%*I(Q=aqbiJdDOMrdxxX9^$&vIrX`
z8Sw%heeZmhCE-Mue?Xmr>N17Sw=!PHxIGYuLzS{F%t?+=m8c~oQpwV9vimsGBnoGb
zG*|<N>mu`*J<IG(zRFT*y-NRp=YN5nyal>)ayn4)<!!-#E(E_$)cpMkD!Cijd%{FL
z8C<W@gL)mZMQqj-dfD+3A1xR>KlQ`h$qq1g;@UTNg7H_4eR?>--r85JU-tUpQe*?%
z-|WSs7@uS$bSuPgO>IeJ;!lA4)<Z8YHlAcfWlWnlO(Q#YF}Ixhn|=1povJxBc7o)I
zhbXku*w}TzhedtoqBZHqA*;K=wp95g7I*>Vb?ng4c+1?2{4L_Wdo8NS=O&yBL%lK~
zD#hj`fmRpgjfxs<x1OnCK54g`T%lXqXFIpD)UcEN52_SK>hzF0F+fO6`}&=G-MJC4
z<3PX0gjAt*9g`5fX#hWUInz|(DJ=hLrlyjec?Ropmt~#C6F${k^1MYpKmE?;njV^C
z_Lf;CsXjbLawMHqvO2uXUzb#E7~E?-o*5w^oI(~#ZdIa?PJ5ZXO;2Ucp}v(ogJ>41
zdkcY^2vm$38f&q3Fb3j{rkMuT&WSV*J7v3yQ%KpV()Z=v1Eg>XNUgj?m{d95l+E2L
zsh4Xkx%tj`2S7SmhwhV3cGdvtL}e*{#0INh?s}5=DeU;j{vxhXTlfEE5!sh0?N=vp
zk3lc*a-u@~3tgGPPK%vqR%SOj7b<|E^|avfv!V_^&58lOxV^W=+wH5!hk3We{vn-i
zqo=h7%vzs6Zds9E$O`t}TjBrE9CQAMT@*LPE-LZwc2PV(?4rhT6uYR`s(-PI+Gya(
z3~680Z{^LmyD}W?Nmyvcm)6RA>2NY>$)T>M(kB#BRJ=g}CTx+*Pw+}lMwlYqKe=$v
zHD={QQxc^+q+Ey_YAVsFw!<hky)eDba8xbAvR45?e7o?P-+tjQsM7+)FY2&pvWlA_
z1$C-8*m@hh*3&;%ZRz{gYxrulVBp&$Yo2j2wHJ7Y?Q@RlUWD)N)^xJTLKPY3C-93`
zXZ8b!;!U=fz?>fVMZN;Tw7gfuhe$}Kx+nhuZ{qG+`F3_j@lfc)sZuv74-|$c9g=d<
zgpj8_sA$>ALtqJHsyB^Hi}AvUV2&Wo?dCG=MB{m{<jiG&{ZRKw9Fh)cOXuGX`c(Gf
zq8F^?0+1+W2CnnA=firUwOFW*n;n1b6{MB;P}h6u3;S4>6~rFbGc`1?76~teX$Pag
zUJEa_eN>i2aO$GbzWu|n%}ySS4S@Gx+{VUrwMcr@?HU(Tsa}6M=0wqeR`i@#_*s92
zA_In_k_oi0TiB8Svbdh8(A6VDzNj}<BefAHo<v5~Ob`_a`hs2)PhA(4a=eZH0elrY
zpS>R!63#@o&OK3SAp6mo9A}Z=AfHTD2?*p`QW0399^sw|Rt_*@4Mf;}y4>0HIuKA&
zSkv<D(9hyir2JOCX+ubSIm5dX$BD;><ne5z^w^F%4ai(&%)cC_nD-r~WDNWXI-HNO
zxAAwGDgw@r({NK|+H$qpCre;BG^)yh#vaJ1utY)xmrM}Do$6tl*rjky1F_y|r-F??
zoi{#;!wApd>YI2i-iP|W0x;^;$a?r-sj@VQMAax=5Pcab_;7|N=T2U>?cLk#OO!B?
z{*bPC@n@KL34{rXt6!!TaEtW|YM&E_ZBLp!)s-<ZtMT2EX_aL4HSHbG@{Z3d6CB@C
z-kuSeOs-sn>KSjZyOJDawC%Yy{~^M1Q|%wn_G3lw+y{&p_(8Jt!Nzp<OLT@1iNiir
zk9aC2*K>iYf7~rp@BFL$y!nDC9uyXDn+v#~&a@7F3oKM0IbR|XTw%;;$*={%KQCpK
zo)ucF2i~)qmFbX@)8S-Q)^UmLQq{m8Rr?7l(E*^+(!YU9Qhx!J&I6#*SV7yGFoT7)
zgP`<mZ170z11mFY47VMsNrTm`zZP0qe6v;3!ste`sc<K#(dHnho1c3*_%o+#P~Fez
zu3_BH(7yKNg4{^<-F#~DloC=a2^HSGj!^!cmPv0tvzs*@?Jr^#tXQaj9P1wq_J92y
zin%E{+~_f_v$Og}$P*kXg21kmr3zMLAcQSr&QD0bY~Y1b?NF23aE(qZ?v^bmP85)g
z5mv$*935sqhNyJTYn&s*cH4MoOD5=a*Tn5GlF-Y^ExYquOHYHN_^mNgyZ4zw;Ags=
zgY6F|MIBD%>l&iIe=Pyb=%8hS-~!!XS3TZrI2DUDOC4bFqfL<`bbFF>`<d<u*iQ%p
z0^;DUuD1+Ug`CTIfc;7XTq#^W_gf5*1^eCg6Y%cVll8q&;5@s;-3u9_SZv-Ew~+p+
zwPR+d+L!mRcV)5Ju*y&LPNu^7rPDsKob|hPXV@M7s(2o8@5|Up@>d&1#l3;z)}*pw
zd129C`jP?@i}MNWlpD=-wB!wIvI<RefcH+@5)GRt?Gi|+TWE7xV&a6k2-e5WrWsin
z$FC*^0MzRf*YGLj^D+4az*5osk>vGs;)AtaBZ1{5h-<ugy)BP7+(nY!*OEV#DNFm}
zkIoa9__s3@i<d1J-SLS?HnzF()lSU$!cq8$l?v<fi0=H`yvacHA=)`0K1g0nbf1rD
zSA1v%m|6}k<Utisr{st-$x`cVD}#Wg6`J(LT2PL0IT+6t{Bmw+>Q4XIAE{ec7%vFG
zL`V($=`P`9Thq^USD2FSve}#pl0Qb>UYxudRJ2FMuyNVb<m+X#Ez9zGoO^+P$R)S6
z<pQzQDW~DaLHA~d$((7Q_Z{K?RCKFT?H3DfjrR8iHoB;^L2n{=sLKzp(CFthz-`7`
z>M6!owF80K2s>-wt@$;8fO$C4>YfS-0=p>BI8>nlWJ?fptHakqR7+4zx6%+N0!gvd
zewZrKhc6%x6mJESql!blPdWA-5BLi0Jn>Kk+)*)Dg1je<Yq+J(=f?_wM!W!M1TzOf
zcrIj&YV_M<L<(rcMdzOoks<|%AWek`U6+@162L)HgqFw@MN56j0T10u{yC0_&SCN`
zj@km-<6IJP20`p7MIZw{*_p^m4LBP;YTRGNrOR!&pR2fE-2$t45m-eQ?Ud2e7e2cW
z*H#$&PQHVZykasIOiVZb94D>Am3N=6bgJ!+#}D!UQzUIOq}*qrz<bUG<2CXVX90i)
zuAVr0c)*Oo7_>!o#1W`S@B2QxlNn;-%8dF8cm(c}LLc?se;U4u!8bq8NgFrAC~3Ev
zHObRHy_4Jn7<ZQX>neQFfO$7{ToaSVNgk1rh&>+6=bGMi37$?3k_|X>Mey}2eX6yQ
zXPUB^-nuf$+&*hUG3Yvc#%p=tLS&wjsFeB69q}%U<x;q#e5C{K2#8E?Vydq`1FTGE
zQI6wWUF)Wlz@VX*XSh*tCFk7lmW-aUKonrFbnbQhah(?$0LcdsF*rskA+?qa7DFoS
zHTphvgL&UEa9f=98$6@zKW5a*X%tDWaP#w}Zckdjdf}h^gU+A%UT~EibI;9Fb7cf3
zX<F2Y)SHZtVYrnO0O*g5Q68{>M?uBkWDmX;#4pFf9qhMDT2h=%)V4j{T8fojCeF7S
zJYlMjj1k14>5Iwj&Ggzqz7;*$hzj()hM05@hOePbCHK0x{RF#$Lt5Rn#&#Vf%R|Z&
z=$_nbMCTa?R+k|4T=ZbZGPMw68zZ`er(bRsY<vC(gJf-qme^;I`g-TX!2-%$v4)SO
z`l?HT$0!qjx@rMek2dg$zj^(a_FlUVaUi2=<Ye>uWf&Q?rzhy!Wp??YTSnDRx9E%c
zDeT-@YmJv*x!;y{6T{tO|IjkDmY^;MJemIpvAr(<3sheS@J|fZ2q=u>0&#&VrL9AO
zl%eIq4CXThSSpsoJx8i2J>tfgVmKlOYp_#y^m=0TF@`e>d@1Bt^Wu<TeTOzimo*hY
zdS~yy+PCY#VrfuD`Y3%5HS-(*mSil?Nr&WD51tqS2{B0KXIDd0C$jnA(GdC`EHOi{
z0<rq4KckX+8Wx@*&83>a1w0WhRRuf|PC8%7;67&)zyCt`7Jde-KGhq`?WPFUP7PFX
zQ)dL4=<J;PN!&*+38TLxaiKsG2bhI3e!I<hL1D4TNN#phk9Zm8KD9T$t~z<IKpxeW
z)XZn8S-vyI)cW{c4@>CXzxJ~J6;1yKckdb1<hr&0A|0uU3Q7rziUkFXfYhKU=u(L;
zRC<@9q4&@Nq7*4nQF^m15D2~Z4$?s&l+Yox&`ScO@P7i9wfBDade0c=j5E$SA01<T
zSU%pl?>Vn|&ENI&oi)2eoBKv77xjX{$1EqAn$A5%ze3N$aM_!VPnJZ_pvMR1r*(U9
zSI`$|_&c#$njMs-E+_UyH*w(}<rPeVr0fvm8dhG@m*3HI`tXcmtI_>37))Fk{ih2A
zh1JX}j_2HcU!SB70K58wnXEad8-_*C&oT?$W`1_y6Nj(vBB}|@1d^%?syQ*7Umln(
zQcxhm!~OBH#7<}6U*W}DntUdR3GJF7-18-|9LcP=%leBppuW01a$sR)dDbfQCZV0W
z1S))a?@_H2ds^^&36zcD$D`T~JgU={rFz@NeJgqW65A8z?V_+{ok{zlA;1d(GoV-&
zlM8ejwsZecLpn}<7GzS4oje7~Y66Lr5n{yGnrPy`<XE?ucB)<`(Ji2j!oLDj=KP0i
zf?3CXL-WWfAzZ#{@no$QAjUc?BfGz!(=H%0<VvV^!`Df0W69JY@|t5`U7|1=h_D`m
zZMI*F16_?zDY}kxwSc*Qbv}z$4nuM%4GO(>H6(2fdXO48@h)w_Z_MSvF#1Z##k~M)
z-KU=YTDWA0<bo=HZN)s^<t(DJ@kHr&yh{T94C8Kr(}E7m)#Oa={DV4A0fN__AmVl)
zck$PmF#P(X(#--YU1j`Tr2DM_)A1#}k7*f2Ur4h|efQ3lbhop%bcvLe6e1{L$q`I~
zMU@7s{|$~nt>JP7ewS@}OoXj7%~d?!Jl=-BHZ;3RkUDNS-IPq~paXxFmTCnp*!d=?
z#MumfxW_z?yjsdh`01Cr_iuiwD5_s-9G)s`WVP>?Qn|4n^_poV>qV3Hup$cZOK~<R
z4qihu%k->K6P<B7%HQueISIKRYBg60Pz8RNG@pfcqI4?v{8H3N=YM5@nNb;FGXMh&
zAV|3BkJF$2d~xr9Ic6F;|E+4{{Bgj#Qx03;_45$$5mw~<L4}P@Uu;XlZe3Awr)741
zIQClt9IjP_5!7o1l;TSUvP2VZzQ%tlLH4h(^@fAJ>`p7!ES5HFsS(MiUsthKaYWsc
z(C4^SDm5az^X)GVep<%ODZ#w#`37yIe!vj@!~(Kh&z-}#H{@eHdHjaZ)T#E_p0fp5
z#8{&le?H~}YLk0XpPErj=;cdJePEbm%+3ievis$*VsdU<=<E>v>9FEG4LGczb%O>m
zJbq3(VUpj`gi`|v`IO5=6u@DnA8Z55Cl6(sJ2)rwNc3rX1TW6_FK5+qu=*s?N;K~`
z>)RL1mVe$hFRHSBH0xmV_5y5xF!GX)W${uUd%{^dbXn-LwrF>^evMI2jpB)SK|7Mo
z;X2k@`l}_<5$Oddi1}0F{-Dc!`cCj-Tqve_2+rDkfJxv_=YG2x#Th=HSaoatre*v;
zzOAy<x0M~5sHF|)=~Y2vij}k=8l&(pHNm$%o|$xFB`ie3qd6$v=UABdVB5TCJTkaP
z$-JP_Nvpes{2!PxBX%<?W=#9vF=KfzYrsdP?_H8^YtU5(l`DuAIIq9(9oFOsj1wuf
zt;Q&#rO}eB`j5=e%(B_R*Q-@x>-db@{<`dp|Gu+mE(&v;Ne#uB#=2V&jKtdSD9-dW
zoG1knD<;J&=gVs5^2%yhG*vXp?f|ggS^e4t`CsA>uOO|Lv|A3h#dMp&12tDHpVaYw
zNkYcu84o;)WtHox>xXW%&N{4q$Fx2N^r5A4Gk=q*j{Bcp((}}pbUo={_jg%WsH{Bw
z(snJU_d1}SZ_$)xu&B`}s_-SJN0+*q!8xad(O1zYb{0$o_w#t6p>}J$U;Pamr!5ha
z0TKmqfitnwT?x92z9deM9>)i1i!du-h}`hR1QKFr?HPkochv+^oaS>Rl%9!kNKI%z
z<Mgi#;dx|F?<78|-QQHm<Z##P<7MLtbMr+(nxP={bS@Gv>NF52sR^*@9Y!5e)eP|C
z_50W{)WBXIFIh88`&*gpF)Ry^$16<U?;$o!Tc>|lzbC1)9jau$RJKihJ?+9C0{``l
z=Qwv*gF233^O=;~Qkm_GgyiqKc5}C|JEoBMv7jGec>m+nGMJY7&1$(Vp7c~-j2%08
z`^j~GZW{Q_3U7YKq62Fzd53&MBxS*|KF%Q^(01_klTUIE%97g(rwuEJoz|RYM6h~9
zd>#91DL07#=>&QFxPIiKRxgFWzrG%@^g11K4@pO3OXI)GYg+!%gk(_n4JmzQX2sv*
z`#8Uk;rqZflgLA4lQbuq?K}V{#t7iV;`ec4W<_FzbmZ}DzY_CPeH8Q;<muyPu&|&E
z&6s?N{?_Dp`r|~f9FQ|Wo<V230FxR=Z1V>bG>QKX{oYBttzTg&`}?F_`Ek<x*kBr=
z8nTO7-Iq-%FBV&bKU{O@i*q>=ugm>UkHY6*s-C+AEa4ngcclSJTt=9aIrAA?`Y~DF
z4z>iPoI8)@Kob2?P5!D%D0n~DWLAq@RbRtEf@U#C0~n(kO1Fy6KPFL^lf(%&u2_$B
z8Xmw%N^J0=0z<r77iH$|h<@(HP<ka54x#2uk*3*39Dij!1n6Lvu|CWfBxV~Xg#Da|
zR61A@KnJs~PHh3`U_$-_|4IjwEa#=t!7~3w2a}}I!B7|q>IWU{SRc^VpQh5mg0uF!
zgNlHGzadkv(vzr6?oOfmAAdcv|LG@Pd8uM}?65_+Mircvy{6nN{zk2*m!p|0En6VC
zAIni?XZY=ql4|f<F++tShQY5sT$tAq{o`ly$cy!&(%1PU2eqfVs8Og`*?t2Mc(tJ@
zNl}^NPo7fCdNRALH5^0^_?V1q+TqS^I`t+dR{Tz4jA=N@CnN3%=Zi{)qjFLQCJbl$
zS7hXE0dk<(!~#UKvPAY|17~9PV3`m)xpKm5h?<3(3uK|{U))OZ9|hldUF`>Cp_(I6
zdseX*vaY~t4DkPM8=SHC6}rm(2L49Ml;8gfE8_S6`lqjOQ^hd+kj2QK+4|az3P_{N
zaC$W$S%=Pg>i}w;UTCXJ54d|gK7YF?&IH*!RLs%8QkcEMt|>QlM%?6c!oTKx2E`31
zK3<$$k?7}>m=Z^s#!F2Xc<JGZr<f#EhXvytWFZ!^Y}p)qYf6IAJJv@M{J9>Ok>1>0
zSwHZ`f>tQNi7eEbA_cJkwqbfi;&m4vFJ5L-Bmzy7_qP`hA}9cHa}e7F%C02t<$Q|c
z%<u2zd`3qzD=f62ifjOcSAXVV3pGvv;ick$-;)&K<Q&|uzrA76exWr!5&itv+xQ=f
z*Y&Z$*VNxhtgfB@lSVqHD!QUd4%Y*RRkiI|q!@J>CdTQr{J51aXEWz|(4*^SYpn%{
zV_^<TdXC0lN74Bl<nd>DvGfzXJ;{QK406SQQUNfa@t`RLm$}|^-iy1+22PuHgR`%)
zk~m<5;QQsN-Q0Zl#N0*ia(%5gG-10X2ln6dikVTpVq)}wqEGdT8HMyGMD2UUBqj+r
zp5*J4Y5*`U5IPHzo1Z#3Gnn8y&pAD=8o9bY?wHWpbGp&97`feBVB`6!`x%_-s{AJz
z!jJLU!T(0Ap#E}1uI}|KWwc%f0&6ECkFT8Ig46VK1)jD)2#mEda?4%-a4iYsPR}W(
ziUTl5622-lBp*I|yHu(#ih>pz)wdFOp4G7I)z%0+O&y9t9bC({38*HANBPL>A|Q#5
z%}A#ckn>lmhhru&z~4}XY8K-n3H~yRfvILO#XEla?n)xz`G8q0TZQpI%wp|7%wn~Y
zfLTm8e%~x+B@l$;woq%~29Q$1qI>1!OEKeuzm=2h`{m@-e>O<Tf20a~(b5`O5q;Dr
zhQA6nrU*HG2>1?DG~&YYDGZg+7uA<VXydel(V?IvZ)e9ZA&shf#F!3CkH_v1yM?|J
zinFznU|-(n>b^ipS36TN+o6xuFjNJw>bY0c`A%kFgZqL0cX%&dar6CO3RUmJKz9w5
zt?Q2Sxao}2%{olh**=(TJH!jQIn9FNG0Lh;d~1FQ7E`OOrxbO_9Iwg_0E14wMY30J
z2B#;r12#O}7!!nXXNXflC8dB7^sd-RG!>dCqW}eXQU7=EPU@}ks9OIC6VN*|jd@4b
zC?FVPnHFXbn`fOI)C1QF2tBd05?E;Xj7ttTMqFc`eyDEDF7w*`X+yh^&4aAcQ6P#C
z{P!qASsnlUPe`XUTmpJXD)Na><2y#qmr=^<3DrKlZ?4mSPK~oL_koV32UdQN(l5<M
zwE?8W{$e5du6oMizyJ@-_;l8_!kMp2Y8zU$@u;q-y)43fXkN9=Ww6~;mjUAkg+Yf%
ze;a*)pn2d#bo@VcBA#T~d^Nc*@J5|$c|Y_o+9zu-^iPkkHdw$1?A`7yT{VRM97<xi
z1Mcl{;9Y?pvk_Y#<|MDMXQZ$fR(t+LVCe&X4_;)5;6a#4_M9;1FqwCzA|Nii-!kbH
zs`hO8ZNcV{Ld6M&gK{vAB!OYyKRbp}o$&=yQn0slcNkV$l_RcAGJHYuq^Bx;o9$1T
ztUz7LW}w!vEU@Pjxe-#t$<K`Ws{;x0e#XJ%kXmd0Z9t|>_#;t9nEu2^zP5oqnR0rw
zCF6IYHoh<R`USyK|M7mfurL19!*5Z$bax6s{_^9e0aD9O7tfmeaVm+*dK@tM)@{S?
z5ui)E!0dqO*y)jqxVA8LA`_|W=z8$5ktR1Xq<Hw>)A;y*rSVbPj1;UsHr;!5pM^HR
zXSirn4_s(m#)PMRpb@i*3hYa^W}oO}Xe@5ag-E@WED=)n9;d5P%o}&%vOzexAD)a~
zc}Q1nje=PoEV<U6;)Y3(C$V)XFOX#M|GSiN<eK_F8HnV^_cZ@_rQA;(K-U`O>rN_5
zYH}y{kI{UidlOr8BICeBFgrx|fMqLl3Cs*P9;S3dRCb>`1s(yGE&k>g8`$%Uajrpz
zKr(gx>T%^3!hWt+OWbl2r~!?OzBqK@!*Yc&72iH?Lw3$O219jUKKz3^33-bcvDs-L
z36CJt|F!;>ip_co%K<+RKktF=9em*vb^)f6ZQXlf9Qj|xvi&Htc!I}<>bF`OD51vk
z(hiu*%svuBu~PJ!5991Iv-@Z32B5}%1d_J?=>cUy{xA+;oo*+NdqD`>SXFTo<xcDs
z`3HZCvXkM|GbA6lqq^k7Pd6`&`az3&9+jCH?dC-<!v^f|Wu5FFhMBvzPH%s`BT96D
zE{!UbR-X7C=q1CmGYs+~0JJF3MR0!Ljand`xb&o1m2)?(*zGVMWkqM$qx`h$dvVo3
zoZYLc&>~L9=l#CNL3Y$UI}l*Hq$zNAjW6(nrr8-RDt`4(5Zq{NT;az{_kqKFt(VbI
zfsQ(E3w$PhH0KOJ>-k4n@c4-@=l?sCmy15W^T*V-9~h$P`4h<F%!w^5hJLDmGzuV}
z!2B<WxK91vPy=A0+P_Lf={NF;4M09Q4fG}cB0do_k}{5wfmLclPNGSp)H$>;&l(3n
z&!vvMhF#4TICc^NoPoXDjgL_WV{D}78dB2Z1k;y%6oUPQIm#J6_`3O^jO6XM%5LMY
zCWp+_7^!|cS<sfNDi_u<DBlN=eJc={FxI<l#-il`MvL+mlk!{gJY-EU7rDn<DHjTy
zZqd(|_8!r^OcUq7iGED`k0`&q`aex6<h%AB(Xfjn7w5-Vt`05F2tvT823`_@Py)c)
zb^z(k`;z|}Yk>0_>J~?K>(){CtOkG0y9vY->lr~mQs`8uDRjdiMPuF`b7v&_j36el
zG6d${93ftj<XqeuKxXOyPsi0jIi+8{XZa|oZ&-E?jUJ($td5fj@IPT`DCm6sxsR5?
zFLV<@MZtIoK+guf4t=8vIQH;nDqp+w9o-G38Tr2Nd}W$zAdi7iZPBu9Kn$H|^Z%-Q
z{$IK*qSY^q-Nk9@FcCJ7uh0yByx&g4Pe)=_EIQE6^?1c7SLLzXNk+Q@#?7xGdO9&_
zdP^_(ktaq7rx+n$`0R&qiTZk;q2;Zng72L{JyW{xe1bPo_{LE7?8=ht_N9-k@$;N9
z_arkzT!g7GCJSjXr7e2R8fL`_O;#bva+=4Bq^Y&>v7@SJVWcKHzP_Zl$Jd-+kw~Ow
z&vm3I|9kda5H)+QqdK^QGl3lV*GK_!8wJ#~0qy{diNR@s8yGYg)$I?twqCX3f4?3d
zh4w~;ezfMVfzjx{Bfx)E-Xiipn)h(@fkGN<S_1wMoE>$1g*L#4N624k>4gj`5ZoyL
z3$En-Kf;xkm_!uYy^*r58)ruV+ofBWIlGxdNH<h|dx2?Hcd44PwQ8sGC4?{c-emIq
z<hO&#nx7T8m{cMaXkd;vA;r@UhMcCsiVHNC-qa0tL}xUe3@uK7W-UqZGHz|mRAV!~
zf4P9?=CGzYKA>nOyLStG=yO7pk>G02_vPq;_*oC{{vi5j@RJPPK!q!xehjV3y$76s
z4Xwl<?Kfl(jP7Uu_irJz^70n7AyZ3H63d*7mn|ZQb@;fGn89V<;X^mrXw7|m-dt3z
zj=HwRV0C$Q2YbRISA~7fIC3=~h*Y_YDo>NOzVBH_K^>oZXA)0NOIa_J477v9`e^sd
zBlA(?C4L(U0gX#u!B)hyFlp25G3qqsc@_&S1?F;zxu2n$mospfj-{eX)1TizTR(Pw
zey!=*x-PdlVlQb9_4lN?auajtd`46Cv|<Vj0ehluz&A`;j7KSRWZF&uiuWreI^BQN
zZ{_c)|966i5VQQ-AFt~F&%Hr_Cs!~j<saXVZ$I7-^#7+_KcnUK(bwpQW&1USTYIv^
zz&Nx`yu=m?18`3e0L^H+H+!~InNz`}H}&!3kl+ind(X(DOvAsqzS+3<pOJfy)Gz*$
zy_jkLz1EPo`_YSii9p-g2-wmHd9m3aJsp^*XnbUgmKp49KCcBZjvHs#G7Ycj>0k`_
zz{YXey_y0Z4P87LbP8N(W_~{@vj-8Z*B69#>0-X4t5pPEDsBOjR`PW5S@Y(aaq|n-
z?__70V~l6HpWYE}$n$b^C^M`($zO3M+E7@~FeNZS*U`N=DZKcwR%!~j`#Tc>GlxNK
zE{j=Zj+)j%dCMLVrfQM7hKkkz3AAwtl7G4Yq33M=Io!OaAThKYaEra}+8;Cy_DPif
zZP4huH)uT05ufr;lZyD+JxM*&R{LS&pzP_WtqtBKYKq*{eu~^lAVm(npCVWA_Y^tP
ze@&4~Kv7fV27af=-NV~-?4`(w3umlCb<WAFfAr&fsy=J^h-Nquf9$r&pofx$!+cID
zjrVKj!!qbAKoL!gRa_C?YBkDiz?l#0pTijna{@b_$6J%nD)9Q|2iPnE^H@dPogSn6
zh&oFDitswFKMffFeFiiM|Hf~W{dt%>OYZ8gJJB#;U#2+;gr;2w{;;ssbH=oKiFjXo
zH9jUHywtkH`cH%4e?JpHADDjKU!R8BnVM87c$sY-V3wzKrFrw5i?qdSe>m%qd4*)&
zq_Sr1=S0D5z}1Vc^&h6R?z4aFdAAmEy7%=s6q_U(xNEa2`2(NN;|yWjps9Dt2;y5C
zgyl}w`N73;nGGo1jJuX=K1+u4q!7As=3}6dVQoiMqc{IVb=u$^ofbM6H#6D_;bivm
z8g3921K()ic*sybtfTiMRSpKE%9V<d0$_yCZSwY96HM7B872D4+tLf_0s7BdD*Z<V
z@4x5TAGjMC^4sC1d~@Hm&*T(zsX&y~1ygQWjP1`9NL#MR9V+x2trm%Zuk%nnz>38V
zbZEXzgC~OUmj-dCt{d9Z$CFuivlr|-gU&i-kd|&?o1vtEFBQDNtL;iU6q#$NKR|rd
zAwWz+UoFgCcoEc6OMr;CEqwXU_D_m)Z%#4o)(lwa)R_>izUPE1l`>wCXwdVdhZ!Lv
zOrV?GjwE}g)WP(qK%=p4sHoKZTd_VLFJ{F$a^ml}I>M_sAb|Pt;&UyehsSG$IiFAq
zlS;fp$22Eiy8XmbRME2_8nsfTu-O9&@;K*i!SDf1&b!C0wq%~NUWJjk?rnK%eFnlM
z6d7gSJ?Zxc`dz&vB&*9_Xy(;vcb_bEo@%I-A~QNm1njjIVAN(i`gqYE1r@!4AONhe
ziqt^eI!!YcmAW<h=v%<<6ySLMU#fjy$F%f<3qcY5kK!r8warqP);YThz&0EK@*v2O
z-5$t%Z2&ob%Vlcg(?hIwgD%1PMhd%%M0BcRx8=f>y_R~G`UyiNz-C3-F6pUnB4jyN
zpNui>FsIzBwEoZ-q_lpsR1uoDo|o`;7AZ#SF5smpGf{|mrV_I?k%D-H9&#fFKR+$=
zAJ?O}<u3=2FC?c)1IYhq5587=6i_dVXx8XX4*}r1?w!-=4_He9;?-i;+L7ST1?1I2
z^0vldfNEYHeXWRPO>m>Qph7|Gvv4b_Ejb(rgIsslUq@9B0tMAg(DuS{2spc6v*Y$q
z-M!ar*55+@<OUpj`{W@DGk`qg3y_BrmmUwh8n1Nu%A;vb5}wb~Cy*r~8y7}?1m(f%
za<iyGdBynFy`a1(YET~j>>VJmAf>S+#Na`nyMWv5N046r^T&^cKfm_gJO8~GBRzkg
z|61z#hdR@?p)r=~;rv^{_Eps#7v5MfnmPciO3#hA_INE&6SoCpbp~#`bm6i78Vs}`
z&pA?95=-DozIS+YO5EyU!i&7kg7ux+%Q9;rrlYk6z3Ct^+BWXR9r>>IZiMD1`6cF;
zO6x@iFD(CKcF^{6Um!*ne+tcay3OIk<h|mrc#UD@&=F4*_6}f>=K#qcSU`1QZ#~M_
z>86wOT0@20PmtOv+;$oOct!E#CSiZAcG_GzkOB(EZbpnjvd7aC9W#IOiZb?jMav=j
z{Y}EogExGlUdJSJ_!F{PEsv~<&n}L4V)!TMLo{Kf&`BVIFY+zEhF5O-c-k!iBBsx)
zr;f?zRTSW$nLZ3)Mnf^eiXF}MU&4(;oHMN;gj&c&D*#+bneF*<EYo%{S<z>DL*6Ip
zltj(`<aq0p8SdBRTGjb;!(d&cB7bEtX|t(;`6^~POMSo!nVcb3UI-$!&TK#nioFJq
zEvBO-iG*8P{z}a3xGdhD@Up3?oT<M^dSBlIkuF_PBeDsFbK~Kt5aFz0HM#fojTNp7
zVp??LOl1?1Ih)QHXSDm1kw2e!kaK(jvay^H;)k_PZHnE~|I}W@-KoBDpozM27q`N4
zmGl<64@e?VunbiU;*5NXaN7PufZwQpiZfBif57Xf--uGjjlt(v7T0w*LNvAS%E5rl
zzk<EYKZzfie}HjoFZ0iIc!?GBzWx){Lo*bR-+!b8?;PeWoYnB~4?U%($9+t^vtGQq
za+pd#$MXD`jv>Y2`fG$c^7+Ng!2D*bGiAKK!LQ$cK;eH1{e!#G0-=B48~>=nZafv~
zESvsPNQfM9?-<5L+Za3v=XbnKwT8rbn$Slp!s(iy=ve9LZ?h(E+>B%GAnF3<%NXf5
zt5Z&|I<qJ+5N+E|c2tH4@X=32ltl^Iwz_*|F651l=mp{`oo*FRA;_aq@9+M$)%|SH
z#*%>n$mveeG#aJEsD-(G@RIx<c&W;d19JU(Ujv)Jw_wk^#zatzq9gnmI=9l@QBS#4
zu$6g}i-b21;VAU+kl{oNG3g}~*A;U*XmT$Po@fmKbURL%%rUEsnOcZx_8hhMkec+t
z_!iqq|BbyQq#*~}XD@xqQgN~R!CtEW8+!@;gS}J|O$?&4mr6B=m_|tnE5yX2jhLl8
z*nPQ-3Qzy9D>MJYwg+*4+e22KwylB)(u^mQc22*zK*-e7saTl<%PK(T#SC>i&SP4K
zEBuW|+M$>HE3pVioIglY=Q63(0Oxsu6SAQ+zNy9+eyJH>x~$_oGBbMQ`AGSEBQ_M<
z_nx?<4Y7NMVUTk<ovbPd|0?gX>uE}$L!Oh9-+YD*$6XdH6e}$$aL=>M9VX>8UKhKt
z?mUn?m~o+yywzNDu~>V+t$m2ndl%Slv>E5`clI1~XjXF<c(kOy{n;J|%{a$7<gO7@
z@X+%tU_NHbUmszL*k&bMaBvy?jC1M>ft+(0Ss~l+D!8Bu36>>_xZx>AevEag6?~vY
z|4F+daltK4P@s8+U9>Q+QoZj3tnws@`S~QsZLDIglMLGt=!hyb8OS-j@P@cto-4oe
zt=6=3k;gfqe9?I*edr;?rBJN0M4mL31bOweWoNWU&0X+;b826VhW9YMme4G4v7)qV
z*NjBUjmS^4F}a}AHjuGUvg=UN(XLQ<d6^=o>CkvT&PExMM1)UPW~LVmWL#v=kP~xL
z8L(<{Xc?9!e;p$E2C?ogs=F5^D-<r|ET6<?Jg>z1M;x&oj%*#aBP<~}on4!|=XW_!
zsk`pBR^-uF2|ZUSq8r^G03X^hF@0U<A-hI<><0~sqjZ$@uN&@{3bDDjHzv|+rEkCl
zbg(?~%jCO{jM~Uv=oOl=i$Wv`e;=gJZ`xKamA3UeL2?=NHsHEQ2L*+_FlMgVr)jz1
ze~tbImqf$E7G5WhY3c@;8lzoLAw*&}z$yEq@Y1bvu-Xh$2+(JA$_fq+3Z}*dZgx(6
zrh6kr`E$}|brQC|$_MEml?302Uu{JlBBArdoOcfU+rHg2GEYyS^kjkT*odi^(J0b&
z5NzQO%%~?dMN(eFWfTzfJ%Jvp|CBKG@l<gn{!N77qL*T&rM{dm7F|RMC_(5`UMZjH
z$cW0Mg{;Z;;Z5R5ug3djy)eFyUXeTsgqh|__;zN=&=>e-TT62v#BtuuB%*BNKK7Yv
z{G%pu(vGyg5Towe`0k^#!UtkkK#;Dt$A!w|Nu6W{dtcfEGxHsk$MA76GPbDXjm72d
zZ>6yQ5(99;b}B*x?}pCSoBtjJ(JP25^c=uNHEO0NoRrelb~wpk`TCx}Qf{|CQ@C&F
z?uVSocqpiFxd1dn?n7)ZV+;NoebyGI7?aWQvdyZ>cBPsrLS0SfFMh|)>%+yPTMg3T
zOdyXewn=%eH=#S5oSVa9TUBz+o-P@?HVjOH<nWx`pwZ5H<*|}68?V1Q6y~KV*-8sa
zmx;5W55Bnk=t|q=k<6Tp3FwDq&+Xl!SjUm)F;*?Ph;dHYPw1xy3dvKLVJCgi@fp+I
zxo)!g)ZJycmCOXss1Az(liE2a@`su5GzdYuw_*uw(|HeKBj$=pa5@d1zrHeP*WpAS
zAiP2lGS(&x;Jeqp3p_2!^jy*CyUR65_O5ODm`NV~qjelAH;H}g54xb?MS`}peR#Rv
z{T?{NY6_b@_8VQQ5$?<7g}S!Y4A@q1hbR=kJ4$(`pSJPu`M8Fs_D3?q<F946BA$>w
zU3kb?;f>&}`FjlLqn|OLW_vN9tt+e>U7r_`qn<^T1=3Oui#GD$?c03*)(HDKXol`V
zkNzg}hR_gsznPTzm-+Cz7#seI<1B6F7X+(n51xfN5wol(yA*hd7L%I7Qx95d9V{kL
zHtb3o0jsAXJA%28Y6^}AR8%*>g<g<HQ^tg{*;t={1*Wx9foUB8Fb(JsHvq@gZ50hb
z9vOy3#QiS*VmN(<0%8Id#aZ_15E;C=0Y>_#xs*WvdSvzVW54SbT(a>`9qM#O)Q>F`
z-0uDYhMwB)1Mlh-V^t%N>vt26n-9IL$M%#g<tXNy>9qA75eeQn5MsYOk7>EJyMn`3
z=M$vRO}%qm7K5=(X&JftF5}0sC7B@z+3(cr(^<Ay%4KP2%}=`QBfmqdgeh^cPoXzp
z*66UsSHgD#V(HTPo3j@wn?cxUG18^^FZQiAh$0aacH^PL7E*Yvad3mV)s=zOl)KIA
zqa=~xu+X#XFn8NU1uSmY_WABhLjuO_Ckx>CV(TTJ=}=rHM!_j*vv3DlxxKpPn@LiK
z&2L0`jcC)hy`10^*F|7ukZ;`@m=timm3nvp-4QIUmb0@e-F^YGH4lMK49F(|f0-ec
zU1GlD-1oI?q$f((z6lz!H7R)I_s8A87FVYBO(B|@jL7%XNUI^7Ke9^6{Pqza9&dpw
zLB?hLubr*RoHpLx6}U#vHh(DShZB;yxS<~;C)F44tLbF&)f`d8#atW!mIV^*fgF|W
z<Eo1Vj4*OtQC}+lMVselL&}9Cktv$BwQSb0I}M4GOh+tft9|sqavNaQ!?Ir2Z>VO0
z83WI54ejNpzA_p{KMl#-EWeF6PY3c-A+yLVp1u530&XC7Sj4GS@)*c>hH2TQ-^l-!
zlDZGC>06~RjcXxmk6TB6OVUPE=NYhfaZ2_cZQDm+TSTjDDlwzUs&GX2vi!0GH=knQ
zrLqjO<nqRq1AdFn{rUWkLt-ugD|LZs?Owa|j)3Q;yigBPO&OA=7!jWH2*^Kv!H0t_
zSL6dS1zmGD($jtS`I-0<`w@e(6eaBk+O+IXUztXD7LZ~h$e5+04;qw2i}0)<6tqYt
zTOK<mEVmw_kzi1<K36IJC6E3@NMT=}>gtiV{<t(VJPP`G0H@VBd*Ncn)}O`G>!8^_
zoVvDsP4lw;qYP4ALb_O9Va)n%&*rxKefp1<P(%aIb!dqO?WKOFhgxgj!x|M<Uy69-
z^|vSCx<SvCQYzD2=%dcu4c~x1c&2P!j16DQa4AodU;8FJjC<?13dhXIh>3M<4Y)X2
zZHdMaE4VS?<s8}ASCx57xppJF@^F5w)H9utd8C|<4<AN}DunkRlFMf|HzkelI(}$@
z>@;!*80c7Lo61bPDE?_6)=)`KK?tSL2bo5AlvJ$EzH^)QFVPUshd!Oib#3zBP0URB
ze%K;wb+iF_nPgGC-mn}2bZfn2!lb(Hmd$$0vfQ&8T#MNb9L93>O-ltih46u_zcN^(
z{`AOgr7@bzBtqcI)0Fky`9(j+b)juo+oE(hZXW(*v@Fm4!|_PVN}`u%dmm&YCN5Lj
zQpFV5hBFD3Nh({9TNcGw)>fY36NBL{9uO;Asj=UgOO7aAu5d2{r~T5Qeztv8W;dvc
zJn%0M_%RE`%9xkdFRmQmGP`rWHJ~m+icz3~CKsQIA7Xx9E3ksRlYuqSjNyHGVzw1u
zcp2taY_Kr9#+Rp|ZlFBug)c-pyInz5-($<~(rXF%nlxe^EV-JW|1BH285kA*z%^;Y
zF9n-45D5c<a{cSk*7~+td@U!l*D=?4_JVTtxR-C4nuP$wB;F&xgL1{vN4*+yR9rIV
zi}V|c{hdMhiP$LzH7is$KnzykSWg2++-fZsW4%os_s29#lCAB(e-q<BeiO#BqO@&)
zGyhI#Aa`De+r%4W7*N6j*-hpSjj53DdM)}6M{DE)ois_{>10isiT+9fWSQ<{o|5IW
zS24S)<Q+)3%zC4KTi(L=+!mLP?FS&>U%6tZazuNnawarXckP{l9>MV~D`X3X9l_Yl
zKIdQ80WH~|j8&3<$jEXvOhM*l-@Qw4mfhQ_6nmi36gX~Vl4<`gR;fMdu!WGV($%qi
zxs9lMgUKmFj*G9bm;{f}r$%v$t7VFdNykJrAE?RBeX;L(sQO`hefxuF*$`rvjGYJ)
zyKEk7r&7E<9n&Zm)9ct#tNto<w;XX^$5IAAiXT%Cu(a#Pd1Z=d4`)wA+pj{9A{lG|
ze3m@H)Uwu6Y3LSLcuvfNNZiplKJ3u4wo~_M)PnREzirR?;fpQe_90?B#L-#B&sJQS
zuVmLoy%I%u_1p$aMapv0&U8MQc>yru-*s5ewXCF`$EPXP_yl)11Em3QuMdh*^y(+p
z^|raac2PN4w1aG%$mj|^|5QH$;ik&};_Aj<yQ}MQAs|wr*TQlvdIq)ZKpx`Wy*sez
zJ>vrR-4!6zOZi?oN*l6hK$_r*=qsC+G+mvjCp+~tmRSzutgRvuWgcXlflXW2?|<>1
zXIxoz&;0j@vnVf=i1YTZQb24Suoc`=X09w+FFCu-u!XsXW~+n5egi7%`zBE#*9(zS
zZon?3GO&tD)P=M4D1G~X%W6zI^QvN@HYl*e3rL95%H=d}EvzJFpUr;*cM@&pwvZK7
z+1b}h=UO7$e!pnSH}<z-=^Zucqss}ymGa9Bs!r^JUR&d9#ZZqg?gty{LbC&^99h`K
zQlZ-`cLe0&(*_Xy=p`9z{i4y_=5^@tbp_jHJ?zD0#NFixB|7-n-DNGpV0y&vOoyX{
zuk@S&o+)90`G_yHlJ|6H>~3FWj{-xo&7-3MJY3=)<=y!PJH(nWRjJe6Cp#^_##fYc
z%2n{My;d*9mKP2`Xi}<7g4K{*^+8e&TjTyNna{M#Tu%jh<JN0Il&n}Mv=(-)MJyqU
z(8SiQz?rXY-@4e6aUsHE3W9`5xQ2)b59HBD`!5-&SCD2*zN8j(j;UKRLZDt8P2+b|
z<hv}c$ZWlI3HMCcZElx5egHU)D_T=vnR9P!k1doy%`4_}5s$66@DWVPQY9muGwFZ_
zc@nYfT<nfTmbhJNN;|VuHXAW&0JK&VJ+|GpKMu={%}Pfkm%A>cC~>V;tWgXWJQo=Y
z<H3c(<F{W<q{wpiCf~HOj^glXP*_dP>8rr!WBW~zm*f`n!lu3a<ne7bUb*fU?o;qx
zwTJ)1Dsb}D9=oIGb2`;g^*5|14Ik}~Q0QzGb3brr#_FVSg2cJHKb~AquG~iRx4M7^
z7ppB1X7k^}imn69Q^wWlutvop)?9_O!34U}BHnI%@@3=9?A{WOKJ{p(p!?^yvkjkT
z*-5vdLVVq_Qt>59vV7)j;7<t4d=8-m^r>&{-t^UV4*5<g42i951Z>47u<ZVTaBpgG
zG<xG>W^R7dEe%xLAg|^0L!@i~IZ7bzGbM3%NEV+#AC1h=E&kJe`p``P7qcz=oms!k
z^0!{gD+{=|fwjd`d9|lH`JG5<&F!TNgOZx~NznU?m)pV)oc*LQaz*b(?J97*b9(tl
zDs*H|HCwe*bRQ8Uey|C;)G%GEPau|Omd{=G@$2i3C7RpJnw&nj@<)sVWy7g_IN6x(
zFbF6MqvfnG7!WbpeXSdT$nzZvbCu!4;qsX}R?V<jzw3zuW=r@@Cwcc{@n`?klGj@X
z<bwGGzgl${*PZsAsnrripwDoDFT0QG6kx0uP81SLH_<vR#WV3bisU(H1QTd#`Dx5$
z;7kfQ_nYjleKI)iUe7xfw|-&!uQLV<@NZpMEy%9iuo;>VogA)|J7G|%nXz3ns#dZ+
zNvTyO_962#?FOy=hnu^SDa(rf$xN$*NfO72WuMmvM!k^r3Mo(9;x4xI&2yV+KV!Md
zv)jGQqpu-#ZjoEZRp7gmft;uK(;>Iigu-wohiayghfAdnk2N@BtNFHf{MFJkwYAZ6
zs-y|`=60g|=kAlW*Iz5g?<VbLit2h;B(L|xgS-~=)IdZD3W8JUm{qCaOna{}<}{mL
z8~|GxZvAvCsH87KVMeASYU3zWL(l2)_Gie>XQ?y4UFJW<GQ$SyyeS0OZqean<fQRo
zsxnMpY87kSch$Fo?)=Q$j`HyOczPeeN;c&;;4?=UV*%g2B}j&&PNz@1j;keFu{B`3
z-30AqZqrxSX_9qTYDc1Q=8H&n@Dm%NM&HKGtfUBq@OnR)_cbc%L7`c8j6Vsi=tZXL
zosF>?t*~N-w9D=v8GZT9COa3QVw<VXc3auqqx|R%<4yy+;otn#3@U%sd>C(Twa;J8
zEdqKj;s4bs3@j|O(YBRsV<Ncqw|6SGG&;8LQ5S=DoZdGbisbm{ldP&_9~J6BR<6{t
zX%Jid^Gqk(VdFf`cS51WVK6H+%XR7_Q?tyno=dt8ZJQNj+qrmSWB1A&nnRHyuqVTB
zTN9f;NG?^Xok+GT0b;5P3^e1t7BD3mEZbk$VI1vz?pEkYC(n5Df_a2XC2@GNz`g4h
zk5i^abQ8oesvSOfpWkzS_dI4%T;&~+qCqj(Zjy#{m12@lT`Jwk-E=bA?Ib<|ejY9z
z<Of>LO#>wjZDlIKec~s}oP+z2Crd%DH{tDk6o+Iewo&MK@KCtY`iDx?)Ay1lPu~|T
zoy;V)<q$qLI-S-3T!(wlXnK}<0=o|zYZDm|>ld*Vr49Nmn`L3UMS#AQTY72BP4d{K
z3|}IqbEhOT?qNphN-+B?wt8t%+WVzZ5oJ?nu$!3(Pan&qmEC0<Pn?dP+3)nxpPMdh
z3_e=76Sed?8@MgFmNwTjVMQItb~dQ@fnD$8a^NAFYaDErZ2w8MrL|+*!31epzbJTq
zLL-RZ?V<9=0$nB|&gAR}Z(omj=P39}H>f(<kbs&EAa~Bqx#dvmEOVHr&WhYSJZlyu
z^JWYtQtvxvbNwhiby-25m&C?7guWUc#Y^NwXJ)7ctkPMaq}%|W53BMgq8%r&U%R)j
z0KMo%v+AyLTVg{g*oIgNvPa!r*xyzFZ`bm_It5I%sI&j+)pCxx-#+YCj(vYj9Zmu`
zVJH0+Dn;QtC-qi|=pmg7M||U1sn@2owB0AK0(E@_wtjgF$C58qFMiy|?9I)ZwD5vK
zGtAQ5sj`V%t0_W!r#gphdo`LLCmeVGBM{ymWUx{a+F#>llBsseOqM=c5ASJ4Xugn8
zp&ZBF<O^FqKSPF6R?$;yu6hYg%vZ}&r}qlRc(_^A((-(_3^b^9f~TxNzqZij>}`QJ
za$NCc_COA@`+U<FETSZF=kpmi-PKjX`^Tfiqzt?DaT_>vz_hDB?1}c3_ID9w*`n~d
zoy@+XQ*Fk8a4$w_N{g9wdVV+2XMt`m`2)3y|HPJ=`=Zq^Z)S$&j{JF+X9*LIU8B2h
z#WU!maeCi%3L|Sc<G<F1IY||=#V9#Uy;Tk0EA%SKcwD^H6h%4lP+;r7OPXxldqjo1
z3YhP*OYnu@cdBFb0I*}s4^(wRjphYtSdv8ZP5N~it;32s=;JZ?-f9P@rd+_S$n3ZO
zmd-vns@U7pIBt?9ZQNv}$;t;Gj0Ar21It1Db^K25j_q@##5wUgS<7#9aEtlUhaQ4A
zl%iIZFIRUkDScw`7D8|OiYE_}631<b>Its)`5O6jfYCj~$V}<!^{;mkmhF!h*JhFZ
zqvWne{T|iG&D^ATDffBv`}bc8;!U3nNqYvEE@%LrA}9fAf2^Ifwa;-99L2p^`t`T2
zT2R`r^vVRmXF8RhxYpy;&g<AH;L|#KM=SDl3FT=bKaY@TztuyTwKzm~-^86GNcka3
z`Dd39bwSUm@f_#Pl4M$Vw`U}MG+*Y<^X+zXioLD@PmqBo%hkrRbe)W1a>H6aCnYuG
z&J3`BS}^n+XRqs-e-upnY{Jh5QB2jTOGR97tBv(4avgfpc0KWU!)BD6i!~gB8F?af
z5Vb6+W<XhsXfan@kD|>sQ#S>sVg*ul7sSI89lIbTmGKYP-)IXJ3YA$0^l{ujJIQht
zq0Mr2G*Z-(-{<MJX~M-mues&YqU7?eI{T_Fn<dA1-u1TK^<|MZ(ABln^NKU+dC)|l
z{xaiDDExTHV!y30LF{ag6z^(2@!Zu}eAt(^UdcpSu9*%u;<CUGyNoVv-hb0lFwg+W
zO~-79NRMbL;<J$-SjEb0<0Q0yIoN(ZdCYr$q^8Z=E%UC49IiB^hd>rX<q*6B6iWXh
zq+0A@-K06Yh_~)!WcK<0VL@iQ_5(2&l0Dl~n`|h;li+2%fE=X&3yKq{yX+=d#B5rI
zq8eX07eJC*jL%NdaB?&DrS6jfzHeU3;ExvO(_8^k*Lxf8DZsdvUsWw>i+}YUl|>yQ
zf&t6lqE)=28A<;efC(YIj;mOzJpl~(o;)V%61kz|YI(p&=cO3C6c!ad1#IR?R1V_P
zGi_xr{ja-ZH*cH&kqDpH4{AQ7Dv05skdoVhiAzUXq%fbdq3lb)O9lNo6<*Woo^zSx
z#mY;trJ@uAo75Z`=Bs=juew<_kE(SLj;i4hy<x<J3qEF<<^npdaYtz3(=~r|uJ%f!
zpN_0YLDb_n=SP?B#rP_LQMk!?P6%nmej-pr&2^BGq6+fN({&kk!m20iJlaMVN~uLh
zw$HSAUMqxNUky}-tS?hiEP*oS@>FW>sDbrXy9OIbx{&@vfZ$!R?bd6RL3Ww8&-FL+
zFz;qIr;`0aWtG&rQQWk7{T|EJ+O0D7Af@h!9rF5oT7E_TQm?i;qZ#=0y$B_+%}Cjj
z<b72Er$+<NlH<-Uu&wbX5WCgn(x;XMxyJ(kV=43GNpV(|t1Y;$r8H%IIxw9+9l%rO
zJN5RanvdO9t5di}=k!P6>4yp0Q&rN2A)%v!{n^>T^3b!sY1!`46H}P9bY*t*)^O(b
z7T3bo0Mpd*34x_BNqA@~9!N70o-P*5Rhtxf<#!ToG|7Ktm8<nOtG5*Ai_uz5x(7l*
zLOa3c4WsC*8Dx>mO9w#ND6+R^kmMEKyU-@ltwC4fU9*pjQWM`Ex@089(+DeW@n!^O
zBWKn`2j_f|WmY&%-2}h@5)b6o@EQT+Ldu(~{|y5Hd^A<T^<02C8vp%h1suzc1y-TK
z*>^<4PIVqW;d<j>rrp31nI)GFQJsz`%qxAbX~e}sU6+BRFCjq#{PfZIe8tJ=-um^?
zjnU?sdVi<sqZ%G5y6fAVY<|Hyg_#~Dn~fUpTiUm3?I_jtU<0y_X{Nbgca(+~;gda9
zgkmZFMF-?a%6w1e0J)oX08k3ZA`V=%y;wL}5$pK<4tDl4^0NF#vj>76E7kUMe?>TF
zx|UFOa7@F)ZY98V)34?ZdZfRCmav4!^%YX)2rWTg@P!m*t_Y!g)N;z%OI?#Y7dtL#
zAgtFk-i8o85F5yrOOTQ>OpP-}OjLCjz+MzOw7VM^^w&B*O`?xB@)+^El?WecLg+dV
z?+ysC^%=m`>YawUQ<Hq#vK*5BNE`=K<~1ryM78t2A_gCpCSzM7BA^}<-0;DBtbc6$
z)FXag=2Conj$jk5U%OWQIUKi7t<hvP9WYdJ{gc%zf!<e7jb1jN7YGv^I<Po<n-7UB
z!AmC$Y^CD8Ci*ia$8IK;J`wR7?5Xn0+gvFY+sW0~KH7h1qK$xR?NJF3X37pPUs%=;
z{1~3E1=Sj)yC<`qO)oX25=$j>D|jnPtFifC9pU#`rINV|s@^Coz{Z6;v|D#lWz~2d
znAJtO^>o?*$u76E#tRFwdwT|&RD;)Jt@tIDA~YQqGAq-sBPfRpzwWy$82kVG>NvF(
zOacSafV7J~d^-&~AEfWm1qYJT!wI-^d@6=%GMEpOxQljNx0wvwI$+YVE0Kg(HoNP<
zSd7UzjlKMexpKV}laHSlkH?q*wn9!L1Xf-juJeI=9%dGKNWZBz4K8)M6lCiPZx38v
zx`7@l&hc2JTONAgDzhQE`%!-LxY(jGTJ;p5sx?xG;Dec`Zb$*tR{2b5Vf7|yOj7F2
zvv5(ZE9a}nLTo%td>(+$bt`lj#+i8GYE9co0&?qggpppHwBCCD+9H&^ZRlEQ&2p6!
zl2s532*~aU&F$_W-B4GYyI%PE?oLGWV5!YwpO>}7%9o?@MOOr(&hI@`Mu#lI%QmKV
z4ql3%(A)KXeCOT_xg(4fIvaHf(6-u2M>aF;7qT1G`gaENVJ)|K^d7Zen+&#}iE~j~
zY5e&v{S<7Kxb{teZ87sQ5xv#i7Gnpl2_@NfBpjHzzT{nZ+%mwquho|6o4T$JQ|apL
zPRd{Xpo|yrp1O9B@3)6T!eD1cxbKT$a=|t*`}r&)i3)5}><6~;Ok&e(wYwTidN-jk
zYrcDFRVH_QnQGXSddzKw8iZDWkgYDAvGnr~whrNf(;K`_v>XIt|0t-v6*bnr_&Cq(
zvy1|E)*TYlS~IzRfXfPau!ez$D#5~cRM%|c+viHQ3&g@1an8{)*<-yTow4yEosUkd
zXzs0oGaJ-d{?5$dJ@G?C_yU|9evfatE!_zc_(yL6cv6<f5_hsw=7*^MGvS*jS8^a*
zPobj46O#z-C!WiEIM&5QIeF|evN}ZEde-Th!}Ri_c>Dx+i$bC3MfvSpZv$4wJkwg>
z2)ozQ_9S2gt||_n^&vJ#j24FSI}IVWEC$>#@L_I<>zaQJEX*}7#GbeYYd!;_K%I)0
zG?wrwVqCAj5?55>`LDP%4zs>weH9IO0|v1#;j66rI@C^1PR@L%nTW6(yK_$GJs@S2
zV$U;y24<{-m6XST#%|}nwbJ;Vt2VB+e6lswFBn*NzHJ_|V2*5u@6;gAFXHmGM}Y|)
z-=;%BnPf<}4lR2q$OD_uOdX3UGQQ{y^4cbw5{C(kmSJyew?sj<8!>W_gfogfF1owJ
z6)I^9S&bA1`zB7YYw4&8_%mE;=5?YYRB>k_Az0z9g|3nw|622?s5{6_*IXay4zZ=1
zPJcFwE7|~Jjb-s>EKv5xqPc5bPxT%(CnH1eO?t%z>TL`qTv~S?$<-y+b3PwItpoRQ
za=hDf!uvB)br+k%FY9Ty%i>V?-EIw-DcpShTbukbZTqt4_;+<&Wp_fQ(AgiHP(WOx
zx24r2svbw%GE7P2!Av5v0RnsH@fF5Mj9bq~!5fH)LuMYqO%sczir^c971fgY<5ESL
z0Ms^Cm&tgD73{<~nO)qN>MYTagH{zRA)Z8DBRuYn3TiXcZj!b99yIlF$Q36n2ze1F
z$<!UXbUnq~`mvn54qr_EH=i%GuG1gy#;($SZwz|UEBMrLp~ubu9p3+fnq>4+QgU_R
zZVY+#q3<A&%q^%a(LiZ1k`DC%w%?!t@$3+7NN33O!X9RIeaE!#=Z*fN$`-(^QdzY>
z86@d1*%1uqOe0>E5)3oTZSHI3`XyI|Hq8S;-5R>~Z(CT_JIe9*UqydIBDRKsTAhr3
z8ObXZO!Lx5AU+2Ha>PJ0P}0PRqs*e;$_q2jN$a^AEo;E>Vx&f@ea3zOJLKA{(K4Lo
z)pJNsh*NZExwGb?%Nx^(C$v>EIA)rdASLC@QVntuo@fa#iZ|*|4!sQY{`AvoSqMTb
z!nU}&xZW>j&Se;|&5Y{d;;kLpW_Al2pH*82HNRaZ#Z(ARMaE=QB%*dG(k&!C!g->?
z?qk!D%kFru_xDeB=D7>aBs+P|N*<M&dZT=>E{cv*J}cJ4-jlYemegeU=^jDe>P)~R
z`6lzAOK9>kGc<7gnFQav>$L3I9tChLI<{EuJvnr1`wC)w#53?NbXV-?vJC+zVHyOC
zlF1IcJ$#nZyJPQmqLW+Z5f}R+s6L#Pfl>0D>U+3cNV)dX_wErrYn`1qJ14V0fEHxN
zN|XKsM?qh1==lzhUYkwxsoXXM%q6od<K2j$C}ugkXvHysDh~=T47+1Cu{D+M8lXy<
z4?v3(dTk=W9ss(!WD4t)yKPPnX+Qt>BMjI|_IBo1#1a7j2WYuy2D!pS^WQc=5j0hy
z;)P@s)XE{SRxM+7=i4p+WF^k^fX0TQC3eUSHJuLm1!l$PVL=v+(-yFS7I|aT9d?2Z
z2ksf`eX|Ree{g;MYpEf9IlB1hq(T;tcgbL|`Ta6QCm5Ii<!kL$@#O52s4j-%v}_<6
z{73dmUsqxpCXAQx5_y8z*+kDoy|dm<4FFY4jhnj)8qf4)yZl^u?8>5g3ExBXRewq^
zfU8{m55SI2<yA*XBd*Jgf3WGFx?4Cs(u<Alh?jguT7nLwmCb(Tb{mBoY(3TTbkR@m
zqRR<92*tRr%2C|PyTTM+`HrQ}v|j2^*hZGbOyKTY^F!Ea+k`h$uE>uYd)A~yR;(f`
z9}AWwAkO969f;wfd^b3zUw$<zc2~nj45l%PF6DsNr<g9+hA#JootM_#{E*WJM$YNR
z+(?9%Wpq4s2l-i*%p70v+=|)#0$8KK?(3ZGySCe_eY<}dK$C!}vcAMrTHe{9SDNdK
z&<dL?8a`$ATjvp1LT`efjhQq6J?h5|U>sF@x-%=m7l}|EAgpM7w0>lE(dR6}rhoRY
zyT43QKMOU8=?}9p>Z>opiGjNE%Bd*Tmy&D8ts&?=bp&n8&Sh@7HGa_0M5R*?tizc=
zf7rNd{8&F=)Q(?m_@-<DI~?@D_nQ5JL2tX7piDZi&Ej<caw+W3>z=C9tzQRoNSJ!p
z810#VrO4Wouh*)itBRdovzE^8K!4AepTZG4HqOCm){<>>>bNI|)N{MHT^Nt&H@r_K
zoHWGeRglvkD>wRBEAn)+s$7~k$%T}Vvk%=x002paVOFrO_J+Ao>lAPVF&?<E4g)zJ
z>Wb&9Lp(Nz*~63^0GrqcZnbM4N#h`jQSov`HsX-ngt+P7u-jquz?zD}0LxW}nKm&o
z_ujx5k6t)!8*l*XzN~h*k9V0a-wKi&^_w_*Kh%z&mTJNy?JmSIxjw4jX@J~J_4cOy
zgXh5cPsX>hKlLj;5Earab|xE?8jA%8VDb%)@CgYVt%KdNWbRiw^yT)4B&FHMyX$f|
z_ba6XN{4ArUm0DGZEe*sAIZHV<v!ASRdLlGQ~bQ#g1{S^^Y&?LoUuW<N1UIQwsXA2
zlyR=Gu+o#L-IjBz*1U%DS0gN#q9t2+Fe1=+v3vm+lhJ_KFt<GQAecen#9uDXrOl`U
zPQfM*_HET`_jh;F1C-QFcDcixMNV!nt5@dR>^R+*d2;@8j8*c@sDlMi49;q17^l*|
z70RzrFMsCN>jlHjP?<iw1^V(`&u<E0s`!FeVU7Lc5sKWeP<n62L)~OG#s>;86O^ZN
zQ6A!!?QW0aO-hRh%)5rG4+*)_cY@bgu9^^sOgE7-kY|K5*yxE+-RPZLmFHZX4T}2M
zk(K?=`EBjP-P0o<A7`~(f(ly=M$qN7-p@pMtlPijwu>RUt>3u~lJ`iue&GhkylduS
zj_QPk{I2BEvh3y@m<N-3CaP58md_IFsL1_}qTWBkF^^Nf;5*oHR&OyO<~y^63xsE*
zX9fy-#%A3!^IvZsuDoc5+-X;V3^?5Va1c&Md%(wANl8eEi;J0=o}PB^j}^iXs+g}X
zD1p26PIa!7CCrdk=<2wY{1bg`s`Al#6KqA+rzdWn6NF$tL(Ayjg({+I8VkWsJngqP
zqqr+xN3^AU;=RUMXPZ)iQ+dI0J+oNlb)}OwNM_a4D>`>NSqA3Se&+B%eeoN;_1X-X
zZO?&SwXjmZo7*RwmD5<dXba9>8e{lV);2&jlRtwGn|KELoTF-O;Lm6#%G3jPZOghe
z#q4<#VaO*w@Ak?(L5RO|T34CHA#AN0`@G5IciBe<+1k0xN~w#Orq7buboT{R6X6Q{
z<5O?W)qV26xqU0~yg^0T=4@B(AK{|UxX<3pl3On8ev+xL7jWpHI_Iyipk(NNLuomr
zz=gIA@7PgTn1AfzW2sYP?+1%>jyjcAp)@yQF1a*9{&N2opV0s1u*F%gtv?-a-mBMk
z*V#^WFBX29H25hB(lF&sLdEG_4&}XIV3j%BAHA8Tar5SZZ$~hH7W9$_{)CO}qA=<9
z<-VUcg3N*oyKIIEn*LPuG26fu7WLIb51mi_9?!&^iMa)C)M>C$=Rc_m`fPc{bm~El
z?irU#9N*K6%%?66!w_>MgRpOp{|{kr9TsKVbq^~dphznqHI#xNAksAh0wPk<k|PY=
zjUWRe(lH|44FbZ@9ZH9E4c*-h!vNpKecw;K$MOEY_h0ca&hy&mUVE*z_ZI4o6hMAC
zbBD38hWW8ChH=3>X^P7i5JCgWww<8RW$PXWw*PU}|J=M47v94cSPC-!7tdYyXQk|J
zwa}(=Dc<4`#j3h>G%woU#4raD$aI|!fX-cz`G2EeAZNg#ZU{hk`6dSoLQ<&Pgnkf(
z%WxmGLPc?VCUetuhuf7zaYo+3kJePA&jWnxB<0}fVHvfQ28{W@W;!9fv4MI;;Z0d}
z)6%7^#lli_GL<J3f7Gx{5VLlXTJmn1e)${36Xg`4rpny_5g-(M^`OBs(c+9_KjcwP
z@rH=1?kuu1v=uj;t9;MkC%<~Tdy%=`Fzk<Shaw>!FJBXh;xCmdX`;Y`VDURtJI*8|
zrf(xz4Ir0Cs%2yf^aED)W3pLqqpXniCnfQGmanC<s~uM-GG!yICd$lpY)R8p{v8qb
zpJBP%KE&hoJ;L!br+AM&NMd$>%c!~K;O=d_$L;#FEHmf6(?$*@39`ai3JvnC%=iBD
z0sW7t`bGXK^)rcCuS<90GN()Gyc?wN%$y`#;o}TK(Z%X){fo?JqE`<R?@GBZL<UoO
zm>0{b`Ec_~RT~Y}k&enkMt&o}$0Iq?Qcet&5Qx>8U3iLyqnbWuAZr2+Y0N$&J#bA^
ze$J7O&fS!K@3^lX+%5fvT$4lp*IG7f<q`OWc9k}Hx^x!#I~sTIr8^ONaPKz+$DL^&
zOM_D#{)m;untg>V1hLG6n2mgXj#q@&+4(r`=c$q&)aUfmW?+=6BhOPJiYsC>3qPhr
zn)|-GN8eZ>_jttzfkO)rY*yrjT%Ey&p`}lYM~<SuY^~>{RA`r5w62@}i8Z)U8$~s8
z;1?$@y+W&IW=yX`igu=J+hNW%PV0s}2?Ap;CS+Fu?`KH6=W0neXQ@VkwlrHWOy<`S
z=CBWe&V~9tcx)&ge6St)=`pX_x8tpG9=Q+Q(Hz@brF|7$QS7G0X>)=Fx9{M*pKX8-
zO)yN0qSrEHD-@H2#|N|IBAl*Ll~TYO+46DkTkB$>-~RpHJ=DNHnqeh2yT3{M;E?kN
z9yXSOL_Lp=i9U@^nb(C#h#u^lbAC+iV!|12m4Y07zZF%+9LayZlmGrmu(0mpdt<#$
zU0>>t+u;nuo392_h~B#Xu6VPg14Uvbx}}D%e3x2y*>3vHp2RR4?s-S3zA!|?-JYUl
z-)_gA!=p&DhC#zoVOvC%CPO<YOW~0h-I9?W<DE`TOLZWKh(|P`4HkU<9YU2*HRekD
zhQ_Civ`U%fW(xYZ49I>lhJ0v76yIXbvH86D+Qrb#-mWG5fW)5*Rv_(`#D=5fiBf~g
zJTYKusH&CnWQa-r36lxP0(<7|*4V?Rbuq*v!@_38>#aHHv+u~3S^i~Z_*TG580bor
z8%cu4OX&NG)BR24EH>4LaX>=f-l^RUbCGx~V3RoQ*e6!;M3&ce%HE&Wp@U(ccD%^I
zr`&pypDzD!DMD!?`O{8r8i!U1bKpCsW=mwf$s=}+;eL~zc;gtTr<>@B2(fHeG{<Is
zIv6{7eJICtfQVL@H<&XgXujEhlBZ9UtX2|rd-V?C8xKKR>XP*kI<bkJ`qP=`qpw4$
z#+!*CiJAW55#izCnTNQ<-Y!$&AfkKgBl$7j(*M^bp9>&qgJViwv*_8hzTGma9X7WR
zdDw#ynO`NkvgCQ)7?EM<Egr!X`$hG?F8RMw<wJVx`;^{(v}W10_%^qvu#>l$7WiJQ
ze4+SERCmiBuSM_O0~qFEh9HSz<Q*}$Jr)WI4{Up;9|h>VwD(}~Ux24{Tu*=$FpjzR
z;4<-#ehw8aTWW$H>j`uzBs%o7X?T__Dv2ISlyUY*92di2p#i%w{Y&%tK`RWrgvx|o
z63;2qMzAP{WQ<re_Mg8G{*JI(yR9w0kni4=%guc@fEIeRIb2K&YK3WnKvF{0<24K$
z-I^SZQoO4ztZoWt9g5vqf|$-HwQXg2vMChAjwIgMeRXb!{kYFE@zxjyqUrl@-OGH?
zLA51y2pqE%b9}j_Pz~obFlsZbD8aTt=-K`wO<e<^@|7PXpc6#}JzC64e3p~q=}zv@
z2c%OnTY)eZk^DN$L3f;l*Qv!=q4kz$gd)(D!Se~&K2#EW5FvhN{cfToljF*tVz29q
zSy@X$?D#2qN{@dK$y`d%=Nam|SUr$qLVss#c5Ezf6CH-e<Be!m)!$*&=#l4lR~66x
zcv52Cki%Vq#8W#7<5K@CdH?s@`z1*HHdDWogW$fMQ+4gf6RzYteP<-vA8&)J?}D&<
zu_#_WZwMJ#z-|16INoldAAE)@Jkq{0mZPNaQ7nroHdIgHrYLS<RXk2&E4dKbGmxjT
zq^A%rkg1N8gsqZ}{Q}jmFhLs|&@Ic_kuS!)^=}Eqes##@s&B)(T`&F(K^~}&PkM&+
z+AUz<33QsqG-x4kf04IM>G+tD{&hncsRu*h99J3wu_1@xa+K*y93G3Mbc$zjQ-=yA
zvGpoIk1vIu5pmPZw-EByV|n9L-ACTs5M%#m`DQ-?CQ^l5m8>nm>RDQ-S340+KTt$=
ze~FSdO=^3XR@q|or+FXv;(0c++eU8A*5&Ce!g;-edA&mrTYSR1E<zE2J=QliLUOX&
zxlFqGmij%{;`2>pV+ibr$}*sv6)9)Dmfj%e&_+3r>BVdqvTGKG;K`Mm_EofnQp=qF
zYWMvd&Ls16TQI_=#{f6}c|VTyKLLYjCh7l_A)b2fKf6{`R}d!dvnZ^yEvXo~p?K{c
zpn%z1E&%(#v+HJ@BQrbGF&SB?#&I(_$f<WwuR3qNE-s0(l;+NWHnx)YV4i92{}%e+
zV9O0))uFz5d4a!seCI8olDZF+dR5)hT_CHnY2o>7c?<EYGEnm~k-JA#N7l6FS}ECs
z^M{+2inQkoETHjX*^c0u%3P$N$C@Lz05&JKV>prXLx0E@E>ApR2m(DwCP&z^+pS2u
z)zpSWC?)&0oHJZ4U3Q|~(ecmm`wi7neDPm6*v)f^!5#4E7o??>6UEz=;A5e`a}aY%
zbR!Zum_hTMJRihpD#j`;O}NFpaB1~!ItPCwfRW<==p3u#Qo}hNWD%gNfYHFvjKmE=
ztR8A_@jJY}nn&Nv<VelLb0mClNCL9~hx-Yjg}i2tciub`if<e&HC4)2%T-A{Y6p3h
zaq88yEaG!k(Q~BTH6)S|mlby1vD%&}A9}%vIa@@{3E#na5a5Ce4<ibOm)yrrNb#IF
z@^6UIbBQarnbtOcE^?r-H75)t=<+0&FMZL^KpuPE68-yk9po!*roX$)7$pCcJ!K#^
z^JsYm`R5)gH2=<>6=Ry>?_K}*NAh3wggCwH-|$l4B5sG&+i&%f6wNr;zgpcGz#>}Z
zOYuvY@n+3S2aeaRT0!i=2uWS+bxKDArZ6za7edcFZfYsqE(^Izv?DRdtvsdY>Dp)P
zM8Z<X>tubtp~78AH$nA>O7QWlW`F#8DD-?!P7?$#^vz6QmmkE{``m8`Y<iii3Fj(U
zSn`2*p0STQGeLflzaJc6>`JoX=u-G;^KB(ZaD+K#CA0clMDY8L7gG=0&KQoOaA{#*
zqiOQwx>1$vieOaa7!sj)e2gWnIibgQ3TVvW!__|5X?ovBx5bzz2C=F3a-of$Qi<Ms
z+%8+FfPRL9*PipPJ$z#*`*vE29GwcQxFk5gRQFfAVNw~+touZr!cJ?%`5<6FpIL*7
zd^~SwVdIrM!D95Z9)>9%z(B@Ough{;_|ZUdUo1B)JtN;xnm-9>w-}_QE&uQv%Lfyo
z7zV(uHPq8tms$8}Ywb8TRwzP#4fpRd?$^mbFX##SKHetb{{xo(tM*$Wya^+r_8!lg
zT8%1v!do;xYK2(BXIKpcpnJgj81j1UgJ&_w_AmvGx-U-fbqxCE03#@$9DIlXIme0c
z5x3#d;LNg=BlNy^U4Kg+?Q4z|#d9voZP@546budQEH66id8F`N^oZnX>yrd=a&swg
zLsEmGQ}dqW`#L3F34h41?_?syN9uAghiB2`^-_F+%5>VTXVvum@BAshMi(#iho?|t
zC%h5JlMC*UsiuT6`HdoX{RR>{v-r~37Yi+bxOwQS1H_Fbv6*<De!V*oH}Y{W`Q4Aq
zYOk*@tS{a59nP69wn~*Ehx%M5%yU|6e?EBpvPE#?S(4KhY5<_Msp8P5yS6k!PuzgE
zy7VLyUX4#<damOVxU`uDQpxLM$Q0+@8L?3bQbWRo@4!_QHlQsm$4iV=CfJd6FSm=8
z9Nf3d$G*G#T%WG3B%l#|@e&s+^xb{yf8xWc4{xnr832-f$S*~}itzvcu<0dX{eVs7
zLLR$ZII=_9DOKCPR|irQXX}%o!8u=SeggI}q&)mU;px3q))lNLG~6CTx!g1c9Eq~1
z=UP-kogD))I!}i4Xx-tAZzM-OWV5Zh1@TU77HX*i_kyDWBxD|Nq;sE$;s*+7cBI{V
z`3x<ucFW?+;DvZsI|3IQ(ngR$ed|YnIOFT;he|Xqx3k<n$P_*EtL}ojv|WX&&^fg*
zxF0CVTZjtf{*H>r_ktqPOU>CH#Kr-}@^Go+0RtI9p6qy$ru81Xuc}{V(k_AOTL&n}
z-T^x>bAb4PYgAZ`2iP6_x<j<Y9=K9aIa87@?N-h5RXgK#XwU{)hWc`^nP}zB2ZvHR
zF?ULA>ae&k*o$&Gib>Czn3^@EFMs>svd{)&r>y{pq!@T7wE&wq>>7Zm0*nt@gLrRZ
z1$FL6YsXB}>D2Ws$1HY}pVr$1Rb<dsC_LWz*fcTRXahit$Lm8ZvJ^P~Mt7|8f8Wx>
z>{{{_s{bAD|NA$iTyCB=0vW*`2MUq?U)*z#Asm=n*qpaSZ@uBebFtE0U`}mT+V5fz
z<NsRR!F?8F`dQ(;CGH(uyAcc@$G9&oG!@oZB(Vkke(Df{*`HT|+cPJ!_4@R#dYR1(
zYl<zXgOaS)m<RLyMj1X8>O4&by`p=XBaKJA+@3AH*z8LvLc!13!Ju8W`Fde>@JkjM
zamXclV~FwW8FC{-m8N_^QY~$_uZMt1S<z4pdKJ>?PmFXkkl`Km$W*1_#=FWz&H1ou
zIKnK$#Av)%Ld8=9_nwtOZ-R)_>{dQdLY)B0`)g*h2@<B)OVoaSVyB5CKa`nzPp>X^
zCsmcijAL{i0?W;Zo{mC4&ikGK&TgG>#=w~KulLBdvJ-5A!k{(te$<AI0Puf*ak9Nw
zEPT_;IPJEYMD?0E75D@As9ZNkIRX=TJTV_@m!owIXPs^?F4^nKEgy*CM7HCqJk+;l
zMWDPUl!R3R|G+f7q<=z(6#%O@@$_(znlhXA|GKHJo15wpvN<1`E}zod!=~@rd!Z(J
zS9|{s2JC?|n?i}9kMD0!{`g}<Sdo(H%+4T!mRk6Lgsbq*F|k#^iq!0>Cg)aOKGoTG
zrNdW<l#dSpa!LifP%Nt!L3Kp39*f*C?+sEa;DRjoC+>Hb;{6h3*~2Oq=SVJwm~%vH
zG-y9J&us2xgr`1+`G07_mlx6%IB0H@4tc}=M-o8(HP29&@3%roFprSbUz6+bTpSKx
zy(1jSY*nNIhg9?0j)fXVf}mf~@mxa1l8gtN-7y;2OdjLj+1X6}NB6O*{G}MkfkZTc
z9q-sH)P9p&7;x_`Pue#22s{+|8xI5@_5frlXCubpFp!V2^^7@kBwv$SBu_k92&kAB
zKd!8h85@0AO?;{lZ)O(pRG|bwf?z6;^RTkZVyHkHe^~AUC5ua7r=o*g>bzif&?EM?
z^|Idg;oXY#{Y#`pb9Db)!S$QJ!NFB{rNHz5m2mnGG@$>s`&R^s^#5_9kdY5S-Xi)8
z=13<m{6*#Z`)<FaJc&&rThP8WW5GB4dLV%E{Q6YCfvVL(wNS4D>Ik`Xs*ru31aVRO
z67!#!yv|tgtM2#vOM_I8`q1V2t<P{_Bd*hLjF^&zU^%T6w2M8Y3Wko~GZo*EJz;@A
zWxI6?_ubv^<O;#aTf<G#oCay}`RIdoTxz}m*z*r+?YWG<v(c;%W%8@%CFu?>jL6#p
zMw?=V^AHNK0<C@2kF=>WJYgyRm8aiBHef6u*lc46wWeeHY@LwTmB$!?%6pQ2$@Bu*
z)csB#Vg6^3{d}#`Y%w4M*>^c_j!G{*x;D!xhP6)skD)b%uAPKbd^k_-;7xx|#l=po
zoH5PC(QjN(3ru)RyLn$hWao!1ORz-)@S||hN#(4k$jmg(G7KSR0`kJx8!4SWsl<+_
z(Es>5Pd%SJN4v%;*LkaBx1~-NxEz8?;z$3woag^sj@BvP&!_({SpClfSCmBzB%%1y
zX<^f9%ko;Oo5>)*WIwCk0!c7Z!}ftaLUaBE$+z6wj#`85rp@FDj!=QBHcln7GH}QY
z0nDqF6j&2WBFWy1RlsRA{7!#4RUO`%vG|+wa9CGQV1vDKV?J0BhFQ*-P<+%Es=XH2
zEYSDr3jh?7MW5>jXBnplkW0=&Y2XkXh_@fo!7}-P;ue-S`Mzz42U)Zf=fRJLjXXM`
zVu=rFt(wBxtqVwWVFIs=l{ULTgf8jw-s;BKkVjDMt;U7}<rJ_`XDf*!-xN?rH52j@
z2EKak{-&%Pj~DEgZQ7U2Z#Vag<I7B{gircnS|kQI_ZwJkW=14-^bLM8ZVL1a)IMgW
zlii)E-~L(J`<Ah-D4}Pf{2PcTOFEd`W!ibnm`xYK6v=Njr^CenxPdeUEC5l$YE1u=
z_4i)T65zT9vKe(I#l70eM2D_tR4X+KGN}eJNugrmb*D3*3e1!}J(x2!k@}-KN+1GB
z>VH3Hul#{Z*m~;}nx>-`Yxn<;8GxS_wf=QwiZ*FO(^<4;A1nIq$nRqxy@)9Oc-!vw
zk`EzujG>;u)1m%$P0VdHK)F*cKO~Yk-j`4Vd3jvc<RRZ<a(%ecb&vRLt@4!e(F?FS
zc3O@vDRfSGPu-U_;jQP#mozOeG-wt@1HZ34JKLpKkgAS-2w%UH#R~fz1u`F|NNvgE
zqW;j&LRZ^nF(B(vyI_7ft+&^T0Bp7$Jqq6x1+b~%7^l(!hCC%Xn)HT)H(U;c)x<|Z
zGBWTNDv1^g8I%S7v}W}ENB92ytWjZ8rHBG%KBV^b)nz56!*s1{W&Ozndv7BEh4Xc*
z9eDlB1}m&}E)JC%lAg!Q<f_;;{qcab1(S1>9QKMJY`%+Loh<_8ZawpKHqP)eB}<s&
zTb62|Vw^J{NS9X03+8G-t@I`(094YLu4AZrp4u~w<L${Ypw6WT9U?IkcHS_0{P=OP
zmxXD^HwIZFptO~hW)1m=u1GCnNIiY+Ev|~%(T%nJKkn$mB>lTv_jq||NQuob{@)jb
z`WY9ni+qByIX`ncEwFU-Hmk3j%@e=#(Oux~2vg)o=cRlUPtbxo4tpe8s`-#JfYPa`
zB5-Z)>@SG2;)0KiRiAst`x{a&B8wgR8N0F?Hh9R}n-C-DqE&4*-Wx(9V&KcAulYj!
zbA}@labkuR+yGJCmbZPef~_AU{nxTzWFw0$7Bk}4F!59JU+c~~UW1?x&&a7^=Rd%n
z;CX3SPMB?pLn@QLqdI(KfD+UQzRE*tuf46!cNc8xOncV>Q0*1G-fb*>b$}-i#~vov
z1)wVu+P=s_oJ?deXALXA^Y+9@Jij#+;<C3RoGCO(-(w#2G|YA(HhZcIIC@Ij?MO~e
zz}|`XirCj9jvMBu2SD%!wjBYRDZvRh7LBUjAti|QZvCl#CXnkuasB_|dbS_kt{1ou
zv{f8GA|Cy}FhB<_*4+^Cp_v}fidw;@z||}F6R!08Fgk~qez>=wx4i{^`G(@xS3M{p
z!N0y$f6yQFW1(ihSb0G#HG&A<f>g4>2BBMW!7%$rMUKyq9s}GU3^6VB6Np2n4BRS6
zS>I5d2!f_slmsd3I~|c&c|ufHmhpWBG<hYK2hyW-f)8{&%FPzm?|gG2mM;6Sg~EK?
zI#4P=a&x>&#F#&3LPe=M52#W=z9u}Uc5f?DO`*y22gPjU0k-WxrE>1qT>G1r$$7y<
zC!{-h%Xqx?t9>a5&NmFw;=u7U|8@NPZPX_K9JaA969flpuN$rPr=0*KdV4ky26XMr
zi#M4D)*d$U!eX+_90p)F+3z+@U+<YCgoV%Md?GL=y@}?Uy7toF6@35@7C`;t!{?~y
zS}M=d$)|w|=X>TIqK=oj?Qfme`Wcd4>HC=_$p3|#i;@`sih$CjhFeV~|EsR}Pgu}M
zcd`C!7QhV-sFu1BMg*J8qidt9gH17vf3J1|)J9;nlS=^00s(POw=*p4j_@)G)SFz4
zks7{kK2Oo{nxGNv(j$3zXqln<bE7*Y`eVy!1`^$(lJegESy38GGP#)WLRA^vSkFrh
znya7~P(=@EkZqyaZS9}GjD9w3yg0dBw>0?Ed<qL8mgM9}mR`)kblqMf^>8A4s1!V>
zl;Ux8_XS+aMSHf8;!*Khxjy=*_hQ@{lcy*8pf$SzAXMG8N9a?ww26fJH}P_2=J8nQ
zpVi$IIYHu!5vJ~^JJz~7a;RmKoFtbnm7To&!V-YvZbuuo28{!0reuq?Ib80Wu^HTL
zCC>CGQ1BO9bbf!*COp)+90Ro}&@K;BSQ0+`!()AH@g}A7I|Wc>DBD}9=3xNn&2KTx
znjq|)$7ep+5(f0UXJ$n2r3l)mPun#U*aI(T%e$|N{~Whf`#(izJJ*+@qyMYu<oqjn
z5ZY`wPL&Vpi7tIb{E|+QlOp+edyS#=%@P-(FC@D0pyTy(w>e_FaHXj`vzqX8Hc&aS
zJ=vZr{OIca<}ctEz9}68OQTvCE!HEbp()qiv`Ltb?bQ#@MZod-n{8hd4}+qrx4(UG
zlol$B9S)3uf{%*t+!wz6`wL(8$P<wQ@pOb9{eVorV-~4Gk-6eoUV~E&cgSU82nnRo
z=KBUuw3ven^atVv#rZ`w#yuW6nyqojj4)gd4h`nuve*Tnk+=BbXVyPAck$)sE>O--
zYiSKC9(t~anP}I#6qH+zamsbZ=(~63spWppOaZzX$Q|z2)*tQ8LISN0_uD~}0IrID
zm@iRoF|y@Lq2K?)qIA-ig$tXCw4_uXpw=>`gf6kX{WFp=w3%4mPjlApHvV(8XDm#A
z1p<(Wx(%%VT`2xXUjHB20V8?y@uR#=0NSM*Umv^g{0Y#CuN|&tMnVGh0HGJS-ypbz
zoPX02-vjuleeOtV+E;|=vOHv6NN(17dJdPT&ynJz)pyWiHK=oD&eIr&{sl=JG*vM5
zyjrn7xGS)yPda3S6#{l6B?Z3yrlrBmxY)L(oLw&RB#4{0kAA~*QZbQQyXH&bE-q;3
z{+L3RaY%oA47+;&JP>&eK~K2cR~9DKl^R0U^PQ(NwGW19Fm6A{H<^>2-?mfMZ`9g^
z+?jL-DtxZ%+oFUL_bIUc){0U@@60a&veop)uQIFgL4YaQ$gH6$cN$U<c(P7u_j9n5
zB_%Nv#BUWVb~@!KTXV8KISKS)CS=y0OaLpLmO5pHz50<*2)ka5)6a!ZE6{7Srl<xB
z5@1C7fz{G7KW(OK-j)wU>tz&Huf(6NJrx3|mT!3f2h|cQ=ITl?S+ByaQ|IKP_pv+V
zmh4~G!5WFjo2N6l(^z4_d&&UfEY`%~VtcX4x{>@!*J-8D1$}S1z+*gJqh^@U26vlr
z)O5|Tc6Gh_u1T^1<G-)C<KAD{WuV8crgpRnpAM@JGmrbrZLS*PaVyO73)$z;?$e0g
zTqZB=eYyadNY#cehJ+2)Fzq~|-QP&15KIe<9`uDSNqZy)1U@*{!D!_y{80LNl#3y<
zS{jz-Y%FvjLv(O)zx9YhXwOD3f&&lbFx2w;PjOhUdL;Z)xULq8F0QbmD81vUkh*Ju
zngrS@m{<Bz``RL~p4-2ihcF*k=8>%c6+j4xk_)`M3x!I0D`^RV3dcYtYWzoZPq}Wx
z65J)riOnzGQ*=k+t907cjbq}k`sEHTVOg@+CmXVm8!3JGbh(uRS_VYgBtXOf*a!ub
z@==-oMlX%KBAKU4`@oibmZKrgmZLw5fy6CFm$NzLd9gkEQ>&DDC^skBVsq@UR7e%z
zsT0dY2y0ss?b{y55duwQTPSW}hdlYmitoGGXh6rnOE>`-4Zn@7@e<o5e_RM;*7=+N
zVUk@X0%YOuPP}RWruzv9ybW^yVm`&iQ{Vjq$9~x8#BTAwN_Fy`wrXF<nhwwLnkczD
zbZAWe$dbQ;*5$d)=2&6LlTwxyx!$Je%s1hTqeTWBlLr2@R;@wmC!5Im+k+J~{v&zY
z7XF|HrHKOoPnHsaDI-S7BYi>Mr8??GpRVdZm3_1I(l)0pOwd20c%*UTrLl^^i)7Vr
zX`r5>XUw)j?Zwb~tT5|Tyh7Zs;9!Ox<=RHEZP-h$==ym?8VHE}$t9Ec+j|3ijqg6*
zyunm1sd3AkVRm2G=}Ydz1#gR{K7@+j?EZ3(pr-||Q4Zz!^2{NC{e{Ww>U)p0e8I<$
zkcqkb@%~4768Y$|rL3o(xs0^sRePGE_J1TD)6^BlLIEgP)zOYr(w8bH&G_>%)Nv&H
zX}9{`Z12*Cg?Qcqr{`+;j?-0K^dntL;Ie?NK;@_669v-r`FELT_RaM?^&M`ra>T4;
zxplt|X7xvFlNIM76NgD;{WPirv$&LP+72XgJ|3*o=5GvbI849!0qxQoq4i!-a4E%8
z`FK+aV;K7*xu7%jagpqry*Na4Ry3gxjW!v7DZV72<}U-dI30jX1!NA-QuCqgaR4W7
z_W}L1M5O;Ng+7bB{GH60nba~n2rJ~Wx^}!h7nFjG<zi5IQS9H7_+L(xAM8|EX0d5x
z>$<UP<wX|g7EeWU^%R{un(DND|8fW|qSku#B1>ZKtIli?f*+35N#LccR*eFDJ+#j1
z2v)Pp_^6Zl44U!!OW9JB(4RV$a_PQYS|w8?C$?O}^t`q+Eo>UkDGCf`>ZJKi{`3jy
zE9I&@VZ`b-@ZYxPvrwymdd3+;J-BBYuH3&|_+!Hrqxkd+v}?x)WOidlibx2ASQHcE
zmh4yJ+EK<a<qCu5MGngrBFmASRDLI*ueU@PJvQn1W7y)(-V`kRs#>+#T3$=AKad+w
zCiwyXlU5<g4aX#WVkQqsFza6B_yhky$mqgalQAM(6&i)Ch>0qEH*$q`^eNCf@#ly?
zkLbBLchre0rpML#A}5T!Q)MV1Qfb}Vanw21y)`Ien&mi|S97RXU{(oj(9z|AjpJOn
zO<3rZ8Tr&-+WWypEwrrg|3$aq;XTB;k-cAw^#620t^cMMy~WwKk?;GiDM9M0KN`Z}
z^)S$Cp(dOFbX3;T%=AmsevSA%jbis!!Y*JfP=UHdsyesW|AM+j;e!?^$z+04BSJO4
z=z-VP7Uc+zANhy5m&HdMun{$ETpT9}+rIkV^XTaxsVDO9+}4@ob${Whd8~DhIWvLU
zuJ9$!bAN}bDr$qC3xxR6=(hBCA%Lm}1kh*)&0)#tAOm?0&TAxf=%w%xNkJ4?*dZ+f
z**2Vm=;%cu``#Zj^!~uXju)FEQMCOE_*qDO-vfS@Mm%0|<yseO%bB|BVrtp{SO{)F
zs_(tO%ZwI1cA(>DTe;GEd*ma8T_c2mlkBCzl)y=M-y-jy3XV-Yo(qt}SM?uNmK9*f
z)jwss?VjeA8jjB$zcqao2dMwJ+VX;XyisgXl_EQW5bdd-TKH4;#nrP0-}RgTQ$6J-
zsyHj^S#vw?J*_IcK<72PN<V6AgvgHg=3thp{NO<nL!U1EI6@1L>j21mJhatW)qX-2
zvu>;Sl}x7B%(J5fM5UHUF?QJ;3HpA#z#hxP#p4P$o(`&Qh#B2+^L>%s?8jP3U$Xld
zi>{xT>I6C%Z}kySTOFw)QBZ0O&?C8B;=pfzT3W5Q%i_3_5IR;hhBF>H&s0*pqNi7^
zK_-%VnWRznKCnzygA?8J8tkg2t@F-?%6p-8ij!B8>OHrsD5=AOw5s*Qb44eV%Ry5;
z-&U+-*K4v^9<L-?$C)K-fw*`7QFG5fi{EAjsK0JQT&l08ms_uz6K<o+_OPp@5+%5H
zZrOgX?zdl8DD@z<WqWz}o8wy;4PHY-2d-Ehc4LFZKu`ibtq?}HMLC#;)FF5?6<sp0
zzy%8J?0a(MzY!?%`~54pL%#)Qo<9*EpQ0uxg(mp(AA4~s8HH$#B_crHP51q9Dj|(f
zyjA=Hy@q|OfP8B;l8XbJml8i4$M{=@OZ@z84s|+aIK~Vd?{k#5SqWgIgYz#*9;kAl
z8yxt$FOK7wV1|)n&Y?VpDDP|~lsAsxjXtmJNosbVi^^c--{1g93A@1CHE^PsI$r#q
zJ71@<c*L>_A(1W{l}Ky`<W%$aa$&K+kCMs9n`6}+OXobys?jFTE0nHIqeboqKp<Zp
zlHZshx~Foj6;g9`Dr@TZng|9KfVu%zpfj7e=(|$|Qx>_#L`)S5$!G_X#$}2;>3HUy
zt)4HrU9&mC{@l9ux7bm`6*|Wnb7ro!1sEY2?0xS8V_|gy5pq;~dDgD8(?WR^c)VI4
z`XUS+^=;kQVtrCUGcL7#k#kHhN>w=%Od>p&Q}}%_s>Eg&+O9H#Z|aELR-pouHBpl!
z-j$z8&4QxXEZZ@)rcPfou7xigo84iWRLp>IY;-WXJChZk<nX6gc`sm~d;7bH$qI)u
zSi1gNMK51ZY+rvqw31@HzVEBv&f&d@Z~x*YD@bl2m1CigK}~J+HGzKzlO0x7i|MaF
zpt_LbG#VE))nPly67@Y7DQ;9J;#;tw6ABIme=zzfSBI_b)RKGSa9Y4%x&k5a@xZLY
zjr=gxR!>w_EOkSR-7^Z|h0KiO!&(DTE<@thA4sZjZZ0TmJSbgM{tf<MdM7f_aEK9`
zmBVEw6t;So{-*D{E1v>uckAd?^UfFx<f`TEZVd?vFjs-gbkW#xb;B|_qs|$E8%%k~
zb7h(G4n%)fZ>+~)ZuVBlaVI&ST8u`zn<_4}a3V~6NQBr7TY28<_}h&EZS@kr^uHBa
zuN&{gc9Qte$<M>-LL&4KBYT?=&=-Z;V$IjjSELc$nN&X6j0_}^#Zgi8sZY?Z$5Al6
zP=bV%capWYFZMq>!yC4>Z+J9aTveb`{wLexWW_{DkY+yevmKyZ7YB1%&=Po}*EOkm
z&6Z4H2?^K<l4fhu7mE2RPwb_nYT|*vYpYCE_S0qo?n06&z?yA_;m}#6c!6h7QQ3~G
z5fN4e+S8F)kxHosmGSe<GRFI|>LHZ-_3-P<w(AQxK+_blq|^!D2ZjQxVIT6(B#!CK
zu-S-iltnLZr%~s<Rt)ChM=_WxOnekCA{WN4u*3?j)j6J|F_jM9gsuIV)V%D!EU+43
z3pZ?zFF)z-N4yNxw*@Ajp$4@Y-Qs(8N5M;KmUg<UtRzMo`?e69Ll#eS>QQ95;S<-A
z8X2du!CJL)pwHLzFxa~N@y0N#!R|4N=hcU1zs8@k34vj#UYNHvQ%aqFU!J;R9XSYY
z?p=@CJ+<Zuf5-Q4Lj3&ejcX`zJDRUqFHyRi%cCOMhXN2Mh%g=w_k=k;xfc;h+=;gg
z_odASWcR`e>JNSp^4k-3$bmoy7j9Ah=sjLu$Vht(_`FEApDb9x#X}8LyODDM*;Cbb
z=+MLvjVC@ki3roigB`pWO&kC|RX4In9Xd{oZ|T1U2M5~AX66`^1UO--3j@kK?o|!R
z8Xr03((gji5R#FcCyGLur;A1zt%cO<H3RP*rIqx0B!$#cG^%w4!qnlI&J-$t+O*c*
zVAaFSCoJ6YF)PIpLn96{g2(q|1cjLI7XwcepqniuP-g#2w7Gey8g5|oq=@&*@Cij*
zY~lNcVfxBp>!M2asVs-gK46l}@%n0uNRNEnQN-oh<(goQus^MHo%&3HHZKM~FM5Gl
zoPX)-A|gc(eA8_Tvox=oQzfRYR+G-Fe!<@@xCET<H5U(5X_dBxbFyH^%W+vRhIv_#
zoFCF3cd>&?Wg|?aVGU(gZttARA|ATVoPUS!OTRqtW$9mH8F)f$_SOBWJmJv2yHU%?
z<iJ(x<7I*L=<#?F)2pY#*C8{&!`ivS$!{||aUy(t)=_7&{*u~d-Fm!wr8YSufo(+W
zMVH8Fj?%V2!}MYO>o>9DbHM_eI$8HU>TFL??D$*f;K{JE1v0Z5F-V_2Z5=Cj=iYxl
z2H%I?SZ`9})>DmYrT3}Ks^|FV7jCCYN->k<h<l<Q=4;epLBhc4f(H@7lxH|0>o84%
zqcuwQFz^E6QGPVKOQO!=+^yp`W$jA-M=Lz{tv`1T9$eM%Dz&w&J4~be!TSLVyozRr
zdr>zo_~sTZ)^zE9M)0jca|Tp7<?^k_mfBAF1dMs{h0yeiuJBX^p030@aF5OciU0@h
z*G1;~UF<LM-Djc4(k2YJ!k7SMXn|@gHl+g-s*3k@j<6>jszhtmVo$$KXM-EBZ-D!c
zSP0OjuRO?_hyRPM>k#-`9pWqtc@k{ZR8`FHwUdZ9%M@m9T)P#|gQc*)@MLRgNVx5E
zd5FkwVnG@okaWvWewZo{fkmkbbt*^w2sWM!xlEID)jN6^{ho+=+(v>!9YJc$CMAM%
zdA{ndQ_IMtqZ#(*$GiG80I+HvL8s+RuC3%&<Vy%P&tx(c<icr#YBH=a5C$ID8P|T|
z{dR7(ALA&O+;T*LUit8f5|wd&^q<NC>C|!Ov83q%!$8QX*R@#lXn~IH78;8iRyoZ!
zAuuanf^0x{((7Bj4XRpv5g<Vy^89^~;D%<N8hC;uOQV3T!lXwS;LySVL#dfgZM2tr
z7aNUhChUs`yHIIxI7sF})bq}##U@^uV%~#Upd#Grm!t+N!)<H1=;w&hqiNTOA-V{S
zJhgl5&}7wkGL;-f_1e=JFX=l))q732MnO6*zFKu3wL~>=u?R#u9#Ee|iCtrK26T6-
z7Cl7JYuVv)ih%fZ53oq5&}yyR%|c87CFT^N;0NUU)5AU<Kx(E#v^(8@>Fsyn+%G|3
zodVyW6m)>Lg(WTNv#dAzR|!lp8C_vo2jF6uy{G*43-X_?FS*+QY9kpTWa8y72bbZG
zHwSsV7>(u_Qa)8So8aQJ)F{3~Wc7i7YPf9H^Fk~hx0cR&OR)aSk^3P<>*=usE??0^
zxvK!W4!qi<V+3oHFMG`6@cPTReLLgHP`}^KsyX8!_o_mL(BY65Ah1TDIjfrJ_<nXZ
z5c~|d04JVKW9x`on=Gkht`Y8-ep)bH^Rx0KKdXMbt3BXi{*I-q((pIv<f0`vntNy)
zQyjiX7##DAc9*lnWZmInX!}=-sLObMSc0mgPOVjSHxCDQPUzD9>TP<FG1T-jYN%_g
zBA+7;EE3<{<ty|FrT5e6+2sCiin3)4lDD8<Wf84WY{S7`?5L%~eeJ4qHe{qD{K-be
zoZy#D4BB~$@L5!WQ<fg%@xn76tAWV%3inQxYGdo^DLHl4m`uo3pvWRqBDik<w(D2b
zgVQdk*H!s`A}6(sxcUzro_MlN9bUJZ%l&R7&OS3s6XA;~cBA?Wp;TCAm-_59o4|N?
zL9rpPf2*pgu&!sPY*#3n*K98A3A=jNW!*WaP9s34YgbsUcLYPLW}`aKjCDRR`DN$0
z7PAo$28((=-mg)(NWXqvR~eT3K2T+Jt)F~!E|tl;^Go#!uy>i?t1a@!S=bJov-X({
zs_6G9a{BBOtBcdmk&f+}s%wJ^%kl|+(*p#*`If1D;g*JK+FQQU2Ks%}9HP3EvGTh}
zRQY9?jqxX5p^g=W>(-}XSHmlc39))XWoVl*wQ?RMHqdzXA6*y=RFtn^cJv(%u!%xk
zX!u{8WkKLysc><$7sN<2kBy6^f5(>d>~kRGZ6=w(XYzR>`mM6F3<{PnV0+yLIuS2v
z+3QoNS~vn@-kFaM%86>+Y;(b=rMw5X{Em@xUR!LAr2(j15dXPqN_7y_o9;JsUz?qy
zngl}%Vl-Y+;TRdy-Din$;OQ+t!|!n3o4qanhD<p)uptGC|4o6c-Cw#3%8m7!i?Q|I
zdl02zM~*HkK$TN_Tvs5MOWd5Ke2K|_tIJ!Ti_{4frPHAJXgyG7DaRS@d1lsd#`gNy
z_|)-TV`QyG4Xv3wATxe0*bAZ~y^EzVU+@1v@cI%6X#FUwCo}7coF0$nl6z{n<R^Zo
zGrcEnwN+px;=2A3iGyM5Y?1}vX^{`1m@rj=GZFm^?UcxfaV-t#1j+Jq%EmdpypKlY
z&r=LnocNrNipdz2%UM>R6dRp;<MjuTi%q2~tDc7eJh(}4MAT%xP4?g>P_N^k&dwlA
z0uOc5cLFQxlk$SvcbzTCP6bAD9oN>|29$D3OwJ82DxjJoWrwhE^S}8DT9xSo<1I4Y
z8s|?vob_eaN591Qglcm{_kL05Ze(&V#p&1YggDVWa%#FaHkstA|3QYX8u@AEPs%_X
z*5j1{nOUby|E1~Jz52~Ud>-0*x}}&QrN-$Yqiq!^Gu?rJwRWF&844nId45PS{zy}l
zoWuh29GF&+p5%2E*kBO>cW~bNXiE+Wpj6HNV=wc{r~$`OGBsDjTJzI(`Jdx2kW-1K
z31E}STY`+}u}k}Qy@X8m0AQMnEJD;lnpCSZ<)}XmI<bg2Jych2H0{%QkdFWI)5$Bq
z0;Dp$iZ6J*2_+IP86@ED#1YGkKkVBjC&{DC=>mv)t1;;8%r<E@;`2+fl21up8T9ot
zv8vS$B9D>W`qa6zj`4?GQI<5%wjRl3pKIlnZSVJVj}N(OAgng0il2t+SWiLtWV2o$
z|2*f=L=ir#dS&?7t&eQ9u6tR-DhJ!#5M^9B9v^aK6g+)8J4twTdRJqZWK+dg7Ye!h
z-g9}<uBOThK1t)OaO}xb_EI~gtFX+dsaOq^G5+nx^buTt(HUD6&L5}BFS=-C*}wDk
z`TiMFJJKM<8QvA$*Zua}@gZO1`OEwgXvQ19hB4xViynl_@buXGK!>;%v2)2+=SPL|
zuV}Cn;Jucs9c<-8z1~&F`C}+e#FYTYK&H{TgUf;A=`(nkwrBQ(osZ7$&{IeFb?}7F
zHu=30nLM)hA$6;r3Z!-$Wui?dn;s?)8|<2lE*88^w|{&pEWht;k)7?7nYeti&p{;p
z(UjXpO1Z>H_Vi-W>GW`4@D#4n3&`K506(u(K>}#y4=7*IckwxIsBHpHNG1!<2<sh%
z#~U;*ej|{l`wiTSmC&vHWC1HSqu>dp>MaeCgQK5~F`68GMHIFT{e_+;@6^1e<JDa(
zH{m5o-RqjJ<&xUvN*5D)>ocYhuV$ff(hqc=*D-=S(Gk>>&Rh@5-}0Sws8}}rKx{{e
zaJkmiKuS@A&$M~jE&FCwnO@p^sAlM1CCbI{AZq~sKf411=Re^6KHJ|!{5)^7-j?_s
zB5{hANJcLxW<D&rIDiKY({~RwcWd@WO2L*TxV+FMfQtxi_voRF;t|GsqcK+kLB!$C
zxu9~ler2;=E+<;>AY4*tAeYd+=TVZ!FdGpp&?@AzVav_p63gxFp6k1fFYoRUpI5&p
z)@0jLe6+cK@rym@b5j_rl40Z-8}#)N3GnuD3w)h6a9YHa^KiADtqtLqUkN7)k&Z`7
z@;x2`(7c<T0F^4lS^sQ7hUS=|ojz~=Vv*hJY5}?KlDbaol6t$aiQWCpO&{7RonPEP
z4&NL9=r^x_NCjDcGaEeKmXzTWEB1N18T#4K^ClDdizgcw(3A1TxRKnk7b=%KSGA{y
zN;8K&lh%!|hsb{Jci#K{t7fEoJ7dRVCtRN@#mRRB1(EQ4FYFowKR<Am4KrpRt1zJu
zUAWQ#7yC$mAE5rJRngI~W?x>ssNhmkjUE5gy0F+nLE9*1W;?QIRP3Xo=!+R`Y^n|{
z>FHjaaoL(_1&>)Zte#Eeb>BA1p=9^_u-ECd@@V}<f@}?-vHh~CKb~Sv!U4XnKZ!z+
zr#h_uvn5P2If|!fwr(XX1(k(H+-jimbsqL4xgGe9&M@WU0s;49T!)sq-_t*>#<6ez
zMmkTJR|1`0?1t(RK%2)FYEak}EaXW^rNo<cM`Gm*{*Kbb?KYPF%qm-_v*j3wu%RsC
zIhY9iQL9W<j$AUx@N5rBtCg!1G|Jc%i>EV*K5pB207p+nMc;}M-Uq|MWprYG*?AFd
zh*_0sZCX9f!As#!1xb6zXR;@u1Qv9j-)KJ?qjzBo^bKb%RAKG1l{%f?nu(Tt2thMg
z7UN`QCAcVM#)Bm|1z6t}QX+RLR&w^{w+%mN8k6cL%Tb#_<2E+Z7RE|V)}YkradtQ1
z`tqyQv<EN--?N$Vm6GtmgL_vFkO<rqM}@@v`E?c{(mvV!kFcZ^T*Ar}+y_d)(0eR`
zuh}qA-S~D>gsh<)+sWzYPnYMHN|U<~B?Z$kr(EoKf-vl|1SUmemXxUzv$_an7x16}
zpdAB?Qmpa;+xlQ%B8NokzN}@wMnMVC(J=v7YS%sKY>hPO6^1S4G*LZabqlTU4|@_$
zQBApSEK4nc2mR|)$I_Oltyn9jaeJT@>f06`uQB_U#kHq<Y7|nJZ&H-<>}vgR?)YT7
zU8^w$4xr#!wcMy=Y=z|FU80f(#FV^igi76el8b@uvM1)BT6r(rHnLr(BD_2+GA?$F
zm%E?PNdWZh0onLB_J6e<Ji3YOW-sO*dy2`suM{PY{(uN?C*Eq)#e;!qND#k4-Tvfa
zSkTX4ipGBPOOK@ie~#gHP+DtSR4=p~koAfEIh^HfstDp^fZpVkD#9}}&ax%hW0@SG
z8k;31Xj=)14D^tJ5Yp4_EC5xra^!hN6fvwYn9gfI>j!bc7K4#L6@zm`dIUL7zu8@V
zouk_#u5w$nb%4VDwZJ#w8uLPGA9;|N8!y)~dGXyFb(A@zGf<&CQx$HtL@&TU!V)IO
z2m%B?_j8gyLS8-^fHMSW+M1^EjlT-=-U|wo|2DB){cWm<cr*s!NBTDLZxW+Os43Ho
z*W|WE#JRX}j9xy9q}_y+0DNs!b=mMif3#^bE&B&kIE}07_1W*S7%n|axm5*);1+v<
zI&+ceH^lebh~Ql-m_38<(8J(s7!cn<&OB`o%6(Pas$fz<TIF!<7MZ7WFMv{cT9J1e
z2CTjrsYkfT_(X<`d~;<^nD91P%{$)i#|K5#7M#Jt6wbF4bn2bb)C;|ewg-OdS!I+0
zo;o&Rhb5u`MHU(*$5Abgkfe9ar?b~J@FQh3t{d1J$BWKyUp$;vybY0J62a0}5%j2A
zy8T!!U!xP~uSH2LMye79@47bol>>TVoANpEf9G8VOhUFPaQ@!6F`V;wJnfU5wajmB
zqpo+alZ)<=Wq0!lHw3{vm;v9G9T5}}?6+lia5UrtRAFKF(91DeHvuyR?7pb9HHgt2
zeEPP1K%FJY36_Z7$nPuTw~m)t+-*#;y4cp_s>mwrWa+niFp?cW=2gR_gF}Gk?Tyhh
z{hQR1-;)1L0?kPrd?GUT9$IO`_i5hukMoO^t4kTFb_;t^)qD;Ai7Q|j@a4tnHM8*5
z+xscv&BgLWV#D3rS3`pU&B_t8#`>dn*HdUvBH+~?umeR01`^ISdKQM+m^{r`L$SDY
zn9p9JJ_;~lfC5(bV!s`_-nOtS&a_<NUSDnlkJbCMtrh>etohI9+dFR}svC7i@}fd^
zvw#T?g4(J;Nx6IFs+6z8v;P4z9WJ*WZ&SKBMZGB=BDB2hzX<c@|EXh2h9)eBJXI)7
zvQ#gv7pdS2JKujo3;2iB=elj4lHoS2hFQnMjPo>#adMJ~cR77%59FLO$$T1HS^D3c
zZpuJw^q_6Eu*kRXGfK6ekcflp)+Jy9@oM|BTqxAO3A!CRo(Cb<@fEXtS`bM4!B2nc
z3FE!26@#tS!M-;~Gzu*PGM3Q=0d{$wW<zVSznrkCUL;+PxRPYc(b8>B4}4s<*fJru
z>JqkM2I<cBL|^hln*++qF1#-QCoC@1xxt?K;Xx06eFm*n`tMojsL^u;IZtzk-dqi&
zdN9iQ_n9pJ_GO>dbl%87u^d1uQe4wO>jj%<>KDRL0_BUdmY|5N(pBlI=P9+ny0-Mk
z^XSTuQF&FR)<NQ`HBRQIwVTgfBPY3mhT+j9I1p|i5*<LROcaj#m^evmM)zrpJ71ms
z@Y0vGIQcP;NmYvDrS<R+Wfj{JvD9Wg4fDP`jF6c-9+)$s1komk2wBJVcdm_++k-Ot
z!5JDx0HKd=Fgx^2G72RU>5VT3i!`(g8q1`fpL*eFhvLK`Yaf#}lqxsRltET{%Y+pO
zD_|n}CNw0zKV4;FXY4nww8FjsGbPS0*TaI*8%g^7))-WPyrWfncfneiD2^Qw<J85L
zjt*ZjMAd~pB*eVILm<<8Ta9gBY;P|3aC_O?D|3IMge~lkPe*C<tP(Y611uXZ?0z22
zK~?f^Sn&b>uL%%IQx4}DNzfwELz%e;cHAqKMS^z<sl%=MI~BXBaXi+WDLx0J3yw$K
zp*}dZJ!KVur5uU_!-)&tWBS<z_>5IS$yj_qH=VGymrbC6P~b;M%0fr*2PT2yfTK01
z^J+CY`!iM+AzrW67rmwLj2wMsi&>Xdpk$h!q2}=LM)IH3;`-}LeUx6~!<vMcHGG<K
z&%})kNy;ILj=Do&EuLs*b0NMLU@_s_A4n_;zas(oR|aI)@MW&1`;b7x3TFZCY|@na
z&FF`BtV@8A0kCIw*c!!p{^j`iyS6?5@>iFsf`*_YFuhjFZZ1tmi_f<9975h(Ig+n2
zp=jK#b$-9LMcVV+Em46oCOMT@#M(5;r*GZlktJWChM4J8JO5P@O9}m4gv!Sc$G;_W
zSj4RI-erW-I?VXkPo4VEy$rKF@MR(s6u!;S4j6)MEj6F->y>P$*lp_U`(!GRn8oYu
z6-6PV>CJ!YijILlT?J(_Ww4DwT23H`6;-37%>8u-U3Rcl%6rIZP5o*<Fq`S`FRfwk
z91wqGt=YVBn3lpg_4Ho^kT&pIj&=a?_a`>6L0AsdXfw?jc;v>=Zx=GOHy=mDq1D%)
zMx@pG9aFtaS0nP0I5TxC(`I@jY|mcO$PaJ0#<}+@U#!*xKwD?bam(GBT9Xx(QsxZ2
z_2>=jflUJ4$@1Fi8P5yvv*Q@|;RvPFP8<DPGz`%*b<8-DZLg2VOC$T_75V95-Qi9t
z(C)!*JnXX3LdaK;4xaU=nT!fAW~92nj@N|3`)KWt^)s)5gwQVYf`IZxQ}QR9H-Hsr
zYko~Xq8&1k=d}KPfDkNPu|8?*RZMN`!5_5Wh*^b<wJashdiB|5#@Bi{4ifTGRxEFB
z7q=Idz)uUPdv@DFIYLc+s<MtFpT~GX!S;2iwxz?F<AMXuK*S)x=ysq;7+419$`)z4
zvz(W@2-!eW{T%D>nOuM&NP!{|VMcGongXX4w69OSW54oTblX}VdFgkmToqAs1-i0%
zvtk*v#He8rv<qJHyXa;IU%1%?<!5AMX4hQ=hJF%@Q~O;kE8|)Z4tsp@XOsW%kb6It
z@K9_&w|0tK-k9lf_>|`M;fS{^nCc9h%87dPXL;XrxHNHy$20A^*b(r`y99J&cyT;a
zXxFZCb2j3PZ6fI@|Hq_7P$SdRyT<+rt2WlqO}a%)Z==o361PT6$&9MI=)+w`Y`VDx
zxvaM1?SM&SkUA7HKC}{o=A^rzO%Z3hBtkxTqg{Jc9%f_W*dA6{-c1G}%gmVu8Vr~g
z<fm(#7gPL84tYNGCJLR5>e!C6q<DNUD~<BN@B>a<Lw2<v#3O);QoUV9271KO4NS_|
z0<5}4GxeV)7g~cOLT3U_dHfSUdDY?5W}inil<eY&&be8i?}T#In?_mgJbv^=^zc`;
z%p}!mgZki70NGZZNXqC74)5%%+y0ZSL?<Ee8PCE5IR9GINBJD1^_MO>DSheD`$SBB
z<J8mNy>QR5slrU%GY<nR>`stXGEHH;0!JNF_PMLHF;wkB%Tf(%s<UkBe=hrK5|?^&
zl`1XsC*oM55z~N>GoVp0U;O$%68=N|8{<N@kEBENjnD2SZr_<uwrn4I;PUwNF79Wd
z7e#49T*l>VMT`Dn<Y`Fmf&&9GZ)_-9dOC}$kh--K3XV@mA%lLUIU~tH7g8fbDWOZq
zsk-wt@ZmSbs}OZn_zEMf=Td~?p%j7$wz4d@fvU7F>0{>CjFiMToR?>FZqk0}sjK}1
z#MS=@soN_SCLzNKGMXj4?e~%Mc6NLh65S<Tw~|BYQ5{z!v=+xBj2_NGp0g*T@$6CR
zs&3}h9nx^luhEp+hHA5jc16#H^CfG9-PZ`ov~R(Gu~!U@;RfjcQk$3X{~lbTH4kjt
zdi~g9w>oCqb<Tjw#JsteM~go2p*j_fg@TgOH(dUode$J(@73L37*EVUB^9$jIA}-C
zPR7o-zJ|m1uM<A0w_#lztp^(gO)`o+ERVLSOrQ-qSj`wO<3dkzG3g_=sW+D*#hVF*
zwj!C%E3V-cb1?%{ZQIJ4$3-e4nqh4~(_d+uSclm8Qo@4JkD)SiF_{NdD$M~$FHC!P
z_unU}VJXz;L6#DRTpWS1L#z&?TAj!{8xB$rWuzIXnW#R@Mq|ep>T#7c6HX?W$2O98
z@BnQ~K}U>MXV~A9#b(O|W<Ekn4zp5;%~YT=Ng|Ln7n3(iU29{zbd{?8y#rLh)N{K!
zrZA<{>YbXLZGTYq|3}$d0L8U!-NKD0K{~k81Og;za0t+l5a<8_LV^Sj!QH*F;MzzC
z!686!cXx+i!QI`h@wYkm-E+=Y^;Nz9uT>OX4TbEz=bCekIp!E|pf{JZl>4l9SG#xR
zK#}!)xbd)qdZ^$vj-0^aCGtVucH#bl)^<M~BtT(Dt`$!_y9HmBDjMVPQ8Qw^ylvjA
zqY+jzVv#>iDRfoT3NFbWw_hm><hU)sK{S*#Y3;<Ex<WOBlsW8z1D|rk!YYaf>Lc3C
zlaHrV0b{hV(`<m|F7Vj;aR9-@ljZcDuncJd)IRY;JZ4MBhM}?AtLKLgd3HV@-<E6Y
z2dR6oH-?fMrqOKUPj8e}-E{<%y-rRvY;akTs8MZW3bCRhuH8zH0#4W)nOfGh-SGI|
zyz(#K@?UA&I1ttDRAPiF+RaNIulR~B6Kc`^!tr%lc1V(n)~%WS9PS>@Fk(2}St*>b
zMG$%ee`erz5gK|ikZbJrF?zKpx)hK4<wOLG#yCToVc*PWzeK<HXWF+36hC)427m}F
z>bTqyO3x(=h!D&U1;FbPSnp*4ywI>DZI9W6_O-_2G$%xEW0$8p8jsWa$<HGSZEooH
zL}chhYfGL&TjZ>}TZ`<fj~oV?IIUMaJ1r{<=XWU)qt*2T>lLNb68KV;X#M?#?nbrB
z`NWfx&TE^wr41gf8&THEg&G!)W#$y^8^Q7^<orh>uJzk@-9(j@r!>Z^EW0$J{?*CY
z^p^}C9Gzj$6YuI3$Ir5oXh+u!D)YbbR8weKsY&FP<WAodNpqT(s65YARz<w%o4a`0
zTHs+NS8#-}{x?UO=*?fdMpP>x^YP|hvd81|0j9Jtirs^kx1bfZZT2~{3y+W{sc^Z5
z9vIHchfk^N$=m*t!a4$oA^(-af|k+wVT;ZeRR0_YW}XN}Rd5>m<ZZue+Kycl48<O2
zHi&wRGHKZ!U6ruN7pM`e`ac8bGrqx|h%56nTtsG|!CY3YEEMq6j?&jatQP1lZ;n&s
z@kUeH{gSr`bD?}k^ff3O*D~9Owg++_PotBmdi)b9{s8C9?e}hWNYnndK^60PIhuT9
zrt!k%2Rw!Te-oa7Chegt&c7wEXy+gQCEC#a$Qa1yu49&FvT-yCys?>!jV7IN!4~&;
zk=7AEiwHWQPrdP2ZEfF&3g^EB8!*y@V^arkNCY#4QC|GouirEJRm>%^8xbRFnZcqQ
zdKvzOwlh8IW@^@~me-6X*5(*8FUT;66k5Hm)~T<*`+Q_A8#j{VUs>q_J0wS3iH}py
zA1tO_w<_ND{@?y_)YuL%#W1PqSf5bQRl5&0W0)yGd+X3CkO7Q}Iva%3Lqp!hb7c1P
zKN$oBqRCtjnzdA=FN6~MVoL|{zt-p<Mto5xFuJCeSnF3`2}`aS-{rGS*L2oj;^nsA
zqH=5-p&wH82Ya*y)l#!O4%7Z>M+d}UIIvHgxW*1wP|k=?_|t49j;*oS)}rCo_IQf4
z2m^YkVhM8S#6&0Rl~5i!4$if==|cPPzycs`C3}wKj#KX${cMxdy2V!(ahWl5-HC8Y
znM!S)t*}~*d<L$4#NhCSQ&)u{*($v{tA69>*dE!BHt&l+;|=mDlVgRIu6vti{*WXA
zMvM=w=9M?&0Gsf2*<2l*{7z1sMIiyv+PCLoQgBNC{U(gu{dQ!|iH@-h{%7?`?+?yS
ze%K9HtrC}mbOfMFhSzx7vq46?BV#yF{_ti#I`_(xos%iizGVL*ld^V-R$KU_uKx~d
z+YhLY9lfY@oum|B65iUe-q9I5`Pnuw-acX=*@@OIIsI#$e#lz>MDVAPmyn56<jWZG
zf`<2u%NrSX>5ycxQlOjWdVVy${A0!;C}y&o+FY^wzU>#Z(**lYLT&%&;Dj*EUG8Yz
zSFCg&!*C*riHdv_xg4XtJ5oz?FY;tUV+{al?bQ(!bbkheEv4)OI{@5DCuYvX^m|F|
zZO3ZOc7g5F);{{XmHvUE!{ts`Cl4?tsJf+59xs!kz}dwO9OysT?E-^(A3AqmI#P|h
zozJenwG^ESYnPcEQfVY87hwG%{BjDT@Y4Mu7y0~nvWR1Bv29}d*sJ{f#lx64;tuP&
zF%8#`J=Vzr1o@LEXw9d_VrI->iOvdoGJuHO!fdix=a;~9yvN$^7i-4?2=@+vG<8~E
zoZl;jJFD;x-k1D1S?qY&oOHTvh#p9nxk+1=<rI6Qp)g#x5lm?81vjIE^a%OUJ8DzK
zN<F;`vmKkOadYkwr`1A}Vb{=^{hE~o(RUj9B3L&%`|dS7laD3;&gj6b^IsBeSj()A
z%~@l%9s;N*?GICC7ue@MjjZ=SV59oVyB9fUPw(!vFHiMz{=kJ^9ET5%e;SP~LTc>}
z<qz2!`2e9qd|3$=H3ek!(1lKo+jZaLj=~V^O5YHlx`fL9WbUu4<Xsydj!IBpy1ViB
z#fKZ&`gUJM{b}oF6XCsXAjxYtvPKS_ztT@drhUY_NNbg(Q5#hZ*&hfMM|6!NKH@eq
zBo#05D7&rFA+)0?gTZv#7cV+u{)a#tYck*11DF7c%px$Z3@{K{+#szx|7IXS{=+~h
zS&CIhmG$npQuVIqzKeIb{3FGuJEaFGwq``nW#{dV8bNaF)XS?LKaCSCbr0=K)djc>
z?8+I<I`~9X^=<g?A~j9+b+PPgI4Ewb-Sg}324f2blFedmW<uST{Xfi~?XNkwUA!FG
zIHk=O(|Yy&cd~)@Sye}=!F)f!nlp*gHB06jmkb<Z)uy5*jAqYm-2OdAH+Zsj>Q#Sz
zRAe_&+~Brftm!Db^8j_yQ&a8(Lo|?LNj|vQzMP{%#WmHD&9-TIh&3(Ce|Qla8}K|E
zN?|F3+W%MQjXK(iENeN8R@i#yG5AAsjIM9^`MgKpu1yn+(SV}?Sy8JEfw?sx(c_|7
zRHmP2+69`&k94zn`K47=C09pJmXc0gtTVlpgL-v%xLvp;e>$GA9aq1(<oy{Lqji}j
zRAJ=s=R8oiI(7g!K<UUsW2G970dv{QQwM-YK&7)&lD?X!RX^r_V{c*iZK^9mbb_O?
zPrAg??1jUEmU6+nCHf*JEzQpZXLBcbJnhoaWgoSE&*-n@9~V^OhS7#g%tS3a)F+3G
z0`W<BXY2UpQ6RO=#D#N{)s^$Y!NfGvAbfmT_Ubf^LpT0qEo86_uYPgK=m+O9|JBAC
zmVQux*yFE9bSqD^bAbQQIc(=RgZ^ekQd((d@@#KnEQfzul#yAVw2^Z-=*!XT6yv#7
zo7@u1)@SV@@{JXiN&6XgQf!^F#ol1l*asgJUIH8SwsHYiSY%J2@ZI&XG9ZUEVZYeo
zJ5<090X*>-5k^7YC7UaxULEoSS6jfCi8{dHk97F4Ih5uvWMWly(fO8f&O?Lpg4%gC
z_%O`Q^gCM#zWf~v;283s7Kw_ae#c2e>u%Gzw>H;++gr|X-EgJen5o1}1_yH)S*L}=
z-MpL7z;j*Ta{Lt9GXmq?3GzOLEgKeWejf3-WIQ{4$a9|PR;7Op6|Kk~sooAAE1KjS
zvf8=w&fBLKbftD1EG-z#iKD%!11Q(C2k(`KlTB7s;={z2-Seko=JSiTX%ftXxt-qO
zKmD7X=KGiIk>GS{H$*&lrv2C*`r@a=8$O^$c{y3Um>H}+h@@?MdJB?2z$VJoXiG^u
z^(gKg$fl>vx9R@ntRvp#1&nr{WEVo&=zd~z-d-0iwo16G^(#0Z!eUI$Z?5B(WV+^k
zC5y`fwlWKtbo$#>*w{&*sA`lup`^wCoEg4qGFmiI4IqhD6SLmHhy=fLIK`mj{lnyc
z6d_|v4}r!ZwmbNvp|7bpHF654${q+_#}`m+ta{@eny+}<rCofM@9+O1tbqi2#N=UC
zHqVG`hQxo0r95BZ=X&>FjiLLA2>2^NgA*9AVo3mSiEqyohULtr=S0w@t}i$N4kVws
z%^23Bd&2c-o+dwYg~KidC3Hph926kR|J75>mm+j0{FngD_fd)qP)BXg*|9f5|1<M2
zc@pdm=|+u{x4JmWFg4q(GggJaM}pJF3i^r=5K;bo?VRC(JFQB->@}wyLObPa&gpwx
zW@61zpks0HH$ox@ARjsJ6gTjhO{m=REHyMnbq3(itBw~GcJSH5QE|}`%V9E)ef0)p
zY6a0Ls4c&#jWZ33i2=B7fntnP9os`5K>sc$(Yj-3NQ3tEy99yOv*|)D9{1DH^PTeX
zx#0DSMSnymSP#IQqti7F9s%A`(JB_@6uiuxw!e)H$MP^7nla8{dr*a6nzhrc^tux1
z=i14MFYWEyvP9`_M&z|kaVf`wE>$#!W&tq=;#0R4h4ik)n98|JVdHhGWU-HrwJd>S
zQ$?cF(v!&ozvY~XL?h&Z!sKPw!6xOY*|~K4Po4eL;N87Dy*-yF$$!2ty9iWZEgCj0
zgsEf#VpAhjz!ZQyP+iEm-B?dI<G0(eh=QHQ@r}3|Ie^Yp)}F{tW$|)J`U&8i;vT;r
zCcH&*^(|LnG|W}N(x((4JpaOh=qzhd5{*wQW`0O>aX{{>v{#w;c+3VxpZ%7gO!ffk
zPqx~51N~g(z#eb@s!T!v7QdOt`torLU!4osHvkBC(x0EPVP1crK$c-}p>yc1xIbQb
zKf}-pzt=Bd#uDH8m-F$OYFHegIJHbIxQ0)D+Jjr6_nSi@d6^1Pw}esI`+mu_#UuIS
zc~@!v9=<TRBp@7{lV5bDZ+oxOTmB`tbh*<lRqb6u+Q_fj?G*ampI6#JW<Zdug~m-O
z$)CwkRz6?tc)-J7@$Q=DUH>AXQNx{H)@hBX;r^=zcxl6~nimYkK;-gH2YlWo{aY$O
zT6@{g^?RA^lwujTJS!wKfy*{D5q|i`iR$G)X+H0-G|%L}Z0DHoKnf|dZbm!c*lZ*B
z)b*rp#^e`cdqZrt@vYUKz8Om)muiKc-(~aq(<UK=hLO+`n84f#w-_`o$aXW7j)UkP
z<wA9}ubmgf*&@7FGRt$IE_MN1xg6T%6UVH~k?ZK}?+F#~=2T6G;_5&xjGhu?L;X7M
zAWt$aK)OeddxO0j)1$#64I$+utM)l$!WTLp-HuY$>1d|-Ii4O_G2Fg>J2CrWj{Sk?
zD6H_ZruvVV;>Ycjc1c`7nk%Z%r}pIa*|si>$MO1rW4V9=n8^J9g2Dp(K{-ss)a77N
zArjEYn5y-5&Ab`d_|hJ%o;!k!hdxPmfANQeLoG|kh#K{loh!DJhq!pT7AFMZcm#QQ
zk#d0^!=YU>hHv!ia;+-)O^WK`10MMV%4xo_!(+=3v5)5gc*^HpZF@|V41<dKmi}5u
zBSjlid?Ol<x@6Y_r2drtB|1hJ=xcfeGt|qE-sK(;P>j()R2l;Qb`G%W<v6+CujHMN
zi?4#xKNVB&)}Ip;659b;H(=UWKdcYK0E;5|`skj%x9IQMX*=_29U;nP*8T{({tnt=
z4B#7gSK8-BOc0y1y_;0)>ko1(z_=44EIHB==v}`k{3W|B*fJzlF|NfMIxhLr`SzkR
z&*M&hIQGk|`oI_`%^n>Dp=J;>%ga)sKmJ@jKd2mjQZXUCvE$Ks^XE74FhaA@zII#c
zmJn!vIOfzUcaj<|mWl`F1sZCp&%4;Hv(F1_2W~YwVX>4kjkhzOb4m2m?T;K&E{f09
z&w7O*8K*Kzub-KW91s2Ybdq^RF0?$N5Jto^r_6rcI#wtRN9<^n#!)*Q>nk9~Xb0a>
zooEf?tKZhid}Gng*Qyd#u6J7ut#Ch$bIJqYmW!M9Y`&v8CjBWR*CrIKD&57Gs(=7Z
zX?T!8N1$%R<=O6bfNd0C98)5vD1aW%+Z}Z?!8-YsvLk7!F3JTOjDfa_#d>9od92XV
zmXA6n%s@-@&XxAU@NO>@QQ{`wf1P;Q8otK|v?}wZ;!X3^qDY3WdhTZD$K)GTZz@z~
zbo`SYw7%(XLa_5p{nw0-&4C9Xtr>c@e}@yYoK#hppRW!WesuKkB;sz5PeN>?Jm!!>
zgTVf==3hzqDVso;`9<w|tn=9Ru7t5cxxUhPg7##=MwO|?N`PGB#PvM?O(`SZ-)cm_
zc$C;ZUVzzgXy@2$z7WGYaE5>Qh|Koz7YO3?@Bsa>H7QXmnP~g7u)39xcqc2($r>65
z_y^+tg<H9PU)pt=!ekxBLw)L2VziE%144N}(}UT{4^;p?!Sx)fvY=>)^ALR-H{{Ls
zFqI;pAX*=BcoWbYm1UIl^$za?APTBRmQUg-1Wc@dAs+Zz{_LX5k0vEB4N<uDJhP*1
z0k*xanD?>M0ZxR_bb~)$T`}19%L4X-KJ1|c!n76MMS3B&n56@zm>r6d97Z=Su|5g(
zIOy?Ozc?g4oG#_B_Q_kz&p#ag-9Gt`#GI`1CxGl_T(|F~T=**LSvTGJ<1Z`zttE`t
z>LWRtOy^BecZ4sl4F0^0=hfYs|32sm>Le1gdPO2f&XmaY`M+8Krr}2aZF{_}v^cf7
z%5wdhK=8bd<JQB$gMYz2UiA459?oePwDZi&U74twA<Sy3eh9x}-<V(WV%)+frBuCj
ztwOstp1u)C&t7j2hGoP>{&3aa;>%4^p90*OSSS7Q+U2n0-Ho(($u&i~cP4$Q{(SdR
zI|5Su1vI4E*H(6{2plZxKZDe>w%LATHK+X@a`cg3OVDJ;3rHsc?k}}$u&ytlXJNhL
zzZM%c#<XY#_|h#jUFE@@p*@`2<NLZmKe*EN12kCv)zd*_)Qo<N%}^K*Q-W21QttN)
zD~CjSuLP$rUoy$&=lADtJZjZD$B5}hXMDGJvTPZz*2-?EsdN_p@Y5j7#X0uA0L0di
z&MhJ~<v!Z(w;vo+1P`AB<kXCy2tod*gsUHG0sal0(9Y+~(;nfL%^TC&sUzojm>E!S
zF1z9ucZ-XT+3`<}z1(^%v```!0bYqcFWlY=;8um4j0-Q(3LOO5(l@e6q#TaF&|NXQ
zTC3(C{w-n*`$<K!Sawen%iH!CUh9=pEzl5zG4B9`L1#TcoG@U~ZD&2%WACl*jX9uK
zFqt7WYzbThK!LCvnDm{c;5GT?J0Xtm(0#AfJpFuUq>8b&j|!NqG^*RXn-l`S$PUWG
zg>(yg(^(Temf$qh#FuE#GnLEzeA_?C%hab6T5DiUY#U~Z7l0!1{yjb*s7?1aJbgji
zl6O9FOGu>kX$ovhcpx$7W%1UGkcfz;c5)p5^jytz+d@K&@yXBgwr1t!m4~cieEajx
z320MuM&(=l6_ImyK2YiDdH6V*dE-do$rkqdKq>awM$(Agc0FEYy?o6??=Z1ZPIZtV
zfV7x@aU8LBomhc9>I_UFO4P?lG2RKwzUre^VcN1rp7R-R?NhK<R>Vzvk=-6m?LF)0
zQ%3ic@C=Ulg8AQ-V}p5wBi0(9d$<?3%znG@XuXL>^bIyUEqY}NnwRRr*CL6SVzrYd
zQlE-c;bW|ZK}l#r;Q6aycd}%t`;d~^ct3^dUeLe~Y;`leOk~*VYO1^)4`j@f!$}PR
z@AmJIV+izu!uw%0I`GqC7Vd7HSX?wT_=|@ZAGG12(dbC&uXbf`k!s<9pmP_PuO>Y=
ze)-KDP5hvo?g^1L7=>>*gE$r23d0TG&5lSLZWk*AUrTT3m}nXiKFmcPZGH;0ybOZ;
zmU1u3-e+eX6ed97NzdJi<O|%|8Ef2Pr~55zK>M$i63O-AZk4_+^Fhk<xWN*TQH^Dp
zZ~l<8`MzGqiSgMSB?pOpl~3u_JlIwVziBhxjdz8RQpBc>K+3F3>}Ji}e0q8Rr2F&n
zhR67xVLX*xHZRAg`Qr|-`=n&uEo5H)ZsH7Co8U${Ky!KvqC9Q(EDM65b)bAhlf5{j
zr#w30ZD=fmWaNvD7?tyy=QhORuJu*+o!b|j?%U}Q<@|Y{kEx~wihJelH|<M0_fa4)
z<$No5Ts>bc*05B&(SJ9;5(-X_-P5KPEB)O3T19ptfo_`}JB<}Y0ipqkyk6R%4tw<I
z`AKZ<(Lf36U|w;@mcxKZESyyXS({rDj^gKQi+!aiA$Wa&1KJzcUa!Jk!xp#BzKq65
zsopN&&X<XCfUW_4hvuZ3c9z*onbF@FJb!T|Ra!M%?Kjp_d%$&kgKo}hyA1lCxqCY-
z+iy>LQVO|{2G7bUj%!MvpFhi^;oj(QH7m&tkNow0wq~MY?-`u}g;?q71=Cc{Y^}@T
zgd^r8spGnP{dYHOa|sk4<m~b7cE0576toRCe$(kr!Btmfm9gZ%5Cz;DKEcHorgxS7
z@DPY7z^A+2_DVRcffN_oZ_T#$_B={)zNo@BYtCCreu$xQ*L7popLn<C&n<~4b!2Bn
z7nky{SUyvqlI%Rv*S2+EpXlwp895&jZF$7AiZbYP%QvY<D74^-W){jj*7Z;;md7}<
z{-}rL>~gyhXd1$2;^TqeU<@!;5Vh?j>MQ7gdUa6~CLWa$AW>ViITDPTERZBcZ1iJj
z`{s<@CWPswh+M<G34a%8nuOz$KD*((6(&E~<;c8W+E<}DR{C+@se=h?VJ+jz+`!h5
z7z0A>(9Blhj+m{}q7Xwn-im3gpeW~R6h&ydR=C}+*=kkhiGC;cG-ynW|J2B>d_EFe
z`MEg1(eMHU)z_3~zd3l8oqKx|C30dmwEi?kbEO@p?f<S7)BgnYFl06wa}eY*G$(3_
zSM9@OrFo}=3z_hwf1uw&oPq|SW`{2l{seQ8dX(U*cb?U3NWpCM=&oD9`dhtt)R!84
zxTx688tLLxld0g;U>bCele7lp#1h)0t4KnuY8%21Smun+a!-oY(W0I>)J^s~h7o7b
zKuh!sxK%0#)D8aZ)dY3`oe%@H2r%p{1$BGicKr-b_<~g4R)xy04D(_=O1QmQ&bD0?
z{B|v>;q`pjsxFshy9$-wN%a8()MZ0ljfv})@KY;_(Jdtq)W5bqpbsj}x{?~wa6kTg
zy}uw20L3lYs*!+XJ9)=TaQP;CgqGiW#UwkonNJ-Q%J7Y5s}DF#9yj@147uKwW{b+c
z>nM)uZaT{pHY%qyU?#+Iy=7FlM0^aDWeBj;9Z2L9O#5+JeTYJ;+WR}+W$)|c$Zn!K
z;+=7o$8AD#;%_(RL;lOw&fTkXn&&@bf)&(2vZreHdE=AvPuUybYid`U@m}4Z5@|Ap
zKigCAowu%Hq&zQRZeNQp)V;Z+&dr|0hiMoSGRetwY+Vzh@BkyU#iUwYo>>!b0P~xX
z@)?`*ClmU{#6}MW8}Bd*Y~IR1u&r>E#DBov4nSR_c}?2RUHWXiQh-O}S6uFIKt+lF
zqN>jrA@aL72fmEai~yYCTP><rzxmiUV>s3>%SdDOge&Ny7_}s@+C+1n&0Qt)Iqt!I
zpWCZXg5rAI=Z3MJ0j?zB`{r(@`7eOsfo!f`XrpsR5nYRWW=OHle1C7`I)zV4^k!Pe
zWGyK%hKdg@0X!Xr$8>*udhFKKazDw0g@eYefXxO}QSw0`24Pt@PVLB(7Z5x~S!k%9
zWFN9Vki^qICvcp@nz&miJQ7zh*?epu`$Nw%>Ep}y`ro5IKQ5?e1;xqA-};wNo9OOm
z3%tkg6yCMoFITR2_!X&X4C0|GZc~tXEJg<6lu7$aT~UII_ze;sVk;Pa|Km=(vnq(<
zt`3^H-Ge^(Ow1b-S2#t#cDI82(wKrvFCh28UOz4|q!-vNww|BQ=@3I-vp>}$?L<KK
zJ;UW)58kCdbu3)p1gqz}gw|X2bq-88T8eyngHL<N*Et(OY&x`$t#MlKjVmUruYjw9
zTRh*ZT`drcXcKi=#CJWbkO{8ZKOcUPt6E#IzK%MWaNKm4r$3F#k<6+8^HeAgh7ZZm
zVPygJQ<GMfYN_*|d?dTv6`sEP!<iBLn`_8<eew6$FD2Q1AylL9QbB1WKfM04RujC0
z(FUoOzKhOjq7D|#bNn2jd_IR9%p^WQ2jNtKZpjzDnQqBrLijzpxGjG?x<HC?oV5PH
zCU-bFUBjn=5cnOE(=(AA2Bsm0qXB<xwC5z3M`+*=@XHC?C-h<0rZi!zyMY|lP8(Lu
z;N@P3OZ8GK2lryx&$g$n=`2smy55EmEYxVlC_4<lf{2{xZ`Wuc^jK7vhF%{fnz9{*
z<^xjHOhWB#_fpggQP}EpvNj*d(y)eNAWk7q)Zow9*_Pk43vJRy&A)&Ow~wPeDom=K
zswGH719Xl7-Gv+z%wbSm;0^cu&#av7z~eCTfPbxL@|aUphb8>MKwy`u0D&E4z?KIq
z6d^E7YoxT$cDpd<QZ@`WcPtxKw#-beNB(K8$RgkuC+JkFBcv%Na=&u9%`-mRGj|Kp
z)Lb5|txChQc)QZ7UERuv<19ix@(>a5WqEry9?Tu~;Tdi;iI{hxEVz0`seiJmR(}q1
zeTA-z_U_LwS9vU)m$I#^%3gk+@*<}Nv-T>B{B5RHxT<1Qxdm}3Y_hRjvv}6W0}>t*
zAlz-7S6KY>*FN<p!?5b_+RR$2C_I3^Z+;^DXr6GMinlF7{3lR!Hc?N0L)mB++N;_I
zsCa<{SA6KfAOe()52{4Uj4sFh%?_udcWguU4?JT<TnlmoY|?H{MmPk`UC!rPP}D1|
zO_{Ijad%m4d_J0Ey#Qovb2ZyJpm<zYYw^|&9psgi(RRZ=dyD@2D_-wnG|$AoSCydf
z2ypBlJG4A>ot`L;+8tF9?T$;s+@x<ln_NeY=iTNHw5>qQ{u$!i>VRn(6uQ0Wm8(AL
zOmKM;)x)V6j*shh@SO7I@`_?=D?Z;+Jz<WYd_)S(NMOnM+_gZ?lUCGCum$QcY6cof
ze>nS9Yx<M-sM=J_xo89f9tzbe#vspgBSVp^#6%?Z*d00L=~ivGz|J3wnZcU9(o&Q%
z@?+(_@xUWgqnG=8x9n6kmEcON-<D@Rhomo?-h_j(<|Y85M*c1dCF4hCT+d(YqwAld
z!p!UYkzi<Pvm-QNf^_`we7(Umge2~|eVlLOvbOY(y5*9mlp#&jK~S!wPfH1C@Tr$(
zrtA6MZ9pa<llinMN~hIt5}+fK<mjKWpaRx%mPz%1==Rx!E~0wASXI?sh9a+W-FVEP
zJ}Iz=R9?&YBg{Q6Wc1;DZqsr-&b~2oJYM4l-}Y^#W}SLcx$E6zXT#mkgKE6hXRAQP
zHRQ%EwAcZQ%&r$%m_OUP;b&Yf8c84Bs<<j%uhC6vD6OQa=O&s2iZdLyhTPHokNur5
zuWAC%uGP6qm)Hkhp%1bSn00>QMElR`rAKqdOcCtMch0LtVm-hCk$LkdqtM*Z`f=I4
zS-_xGLz)~iX+MDoa+TLP4~&v>gmBQXJr#{r=m2wV;>T2PL(n@OTiU%&v=cp+?8-Ip
z(&tKONlt}SA-w{;-vQPbHj~C7vax~fD&Kgdt_S^p5XI<8a|QnK8L?ptQ0DKm7b!+Q
zHbNy7v3LZHSVlB_F%IFF9A<9Vc7cQxM?lmjeHxJ9P=PjIBl2E6n%iQpa2<N>wpVyk
zum&=J+i#&KMeU?)hqMhngN&#~aQ2)N#DkRV;YtoU=>M$AAEWS02Ij3w4_G$z;}tLm
z-ZH!}9jjCN$>TAnp;GP1QLJZlmKV#nG6Lp3*{u+9EMH?&k#!pRC0D_j)>wLB>rv{0
zOQ9N5u&vtKAxm(9<eZ--eGUn<V%WE`Yd*__Dn7+#pzeLTntwT?7pmqsJje+47;vXK
zt<L{Y@0Z_T0XCg0cBP9K*kQZcYWHB!wa*#N(%<FpF?$vWn=T(8>OPp}-;=$x14_GD
zM);wN4VAd}_g(XNT}yd8(9nVTv9GORtze#^Ise4-h#^%iVk4rHU=GQt+I_rYU@&O{
zkXF@YQ`2j}M*v16kh1f+s}DJ<_w_H;!?K$*O8pD7>57b-UCV7(8rYQZ&B>MOkyMoX
zZ&VysY+#1q&NZUkA~75D&B1=--H;OTk4QyzfY4h4=#gQm9uF^w-U;>sl9lPH)~p{-
zw_*_xkg;rX67Jy3tt|JmX-nVe_gP0kX*&*JTJU#^&;TpGR=?1S#n#R#OiH^F`?1Z9
zi|c0vKV=`JyedrBbqyB>NlV-24XGMZuY_4l(dn+ZbNK_zbw(u{K-h|!QW!b?X0G8T
z$Kmc~ogscAZ`~X0H>sk1^Vzd>OrtbZRLyufv=&L<8t$*;!3_NUKEp7I{^Z6%FBQzI
z99C?SYYWZVipuiwJtKhFcXFJ_Xa0-YX@Q|zZDVER_^gYbbkZc3_nbBb@B<l}7jIHD
zEu3wxm@?7a{*x^mc^>I$wTQ}Z-6<gAq6-^f?mc9zopH$?HH}4?)e*Lxd80j<t}{J0
zRwv_iPI~(m%_!X7s*~7!Cyb2%1(W=A>PnI!s=e{fB2=jCv_W~gDBcyf(4ab}QxVhC
z4;i04*EwZY;q_|?h09aco%$+GXf8X9cXO~qt)N!9q++PB-cTIk@^T0)bicRRR8JpX
z9uump=wADKn-3ed=IO*M-4x4u2D}EgD@OF7&qB7>Pp?*MA=eQyA>F5r1C6ivgU19v
zJ7SSAU#s&j`ycYH^?Oxxc@3%+jC8*|V?{sfPvo1`61o^>y|OCGL*WTh%9Nu9q_J;i
zOnbuic<rh*J>2$!1UCGg7eBFtRWb(xcjO_~XUc6*n0(cET{A;A3Cr?LJMhRG@opb*
z+WNVZbN^vPIL#Z#lk1+wRaBhC@0iF=+eb}QdIrunE|bNJ3h|S1%2eZcUxvm{GRV6p
z)L9(3T#rmd@ZDrW^3B)n^R0th&daO%F;cw1o|v?u0n$9Ja~`7|9zuOvHRny{wRxtd
zd6n7aCXK&T$S?PL&*pRGqG~K-Fgy)%akr<v$L$Vodyg)^^A<}})!n=|G_9pj$1~s;
z6&T&4c#%&~4^#Q?ZpVq%D|)}*Ek4M6xVMyS?)46NhJKe&Iu8z`4Yl<IOF^g$%5Zgy
z(BVeU1oN-yV!)?E?UD_HxXZPvl`wDvd|71z{W8d(16d9C;T+A34&8PUMJ{<`skLLD
zE^^DagU6XDohAlfQ_yqpbH)Y{K&yT{CUs-%Y5P-S(E=uXo;aBF)JWmriPKZ!50T;B
zF+0WL1#h0aZ@~EalrAH=@hrbxH?`vRz`rUw(FZcQ!nxL@uU<2)ut0Cat##;A@N}2N
zgTNfe1!H*)+m+TZ0TB{}6`h1q44vn%t+`r(<Nua-_n&)u#n1=!)=RW*Fe^s(t_-<l
z$og;0D{l5PGD~<&YGvcKYIoOjs5i6&SgprC|K3qOJ2M?v6<&KTW6eNg=6apj!HM~#
z-tBl1&-23(zkhM+BiYcY)7K+(H_qIdQ?T<Slg5GdUa3jNTzB1V=c(4oz7HXDjoVJ~
zLT(>^PTF@u$E=U*qpWL=J6oKU)0Edf`2a_i#Q<8RmA3=_oK1111HW&Tow8IEK>Yxm
zs0X7iH{Gzwc3Z1E{%y$U(u+C>db?UNUBZ7H=~?iRfNb$}+>(_uGZk_Pm?ZfEdUU>?
z!cHIvAR!P!aPwYjrHlS*vM*L5d6LHyGg&6BJIbTD6|eS-u?fw}Q5#{5YO7zhM<$Is
z(jH($sgz#7S6Xi)JeTr(quTQ4vns051N%i^7!itj#;<Uk7*fw)bp!Dro^GGuO)I_8
zK&H7*=^qq0xA-;=*~jXn)|M-mIjlPm;0d^wJ^Ct{QuJvh56#(8Q(lb{QuBVp7W39T
zr$0Mw3fF)R*Xch@$yTVWFn8N~I0nz>9oDf81D;$zF-mIJBKsNl-UQBes=M3Fu{ke9
z$}q?%N2$=Fc6N5aWUTkPJ_1sHjR$q@%BEms8#Q#PTF$4a)D~G>-D0%?C4ku*M+JW8
z3f$v4OH1oZF~nkkH2O-;&o||D{os8=Ph)AiFTijFTK*cS>2aq@gU9+|fy$wvlfR66
zUS+?W1G2qR@Np=V1e{@@4TiUl&DO*il3=A|Pp1^$%(EdTbEseHQ<8Gu|NPr=`y#fn
zb$Yi_Che<H_NwXw5jD<_>c4g}E!c|oO!eKt$?(zs_3y+&$@(=D8~=sGK4)}^C7~GM
zSxY4^u;1!n>H^4L{+zJa!v$P~bR-?DgW(;j@QEG=WQpFJ%sZ_KQgg8Q>c}6bVQIwg
za8rh)qty?)dMU@A)zbUj0@-F-YX-9At$^|pmR7)B6A9%K3(Z*&Sf0y??vPk~JG`!s
zEh-DA1spX(_wDfo?3YAZD+XAQl#3lYIohH*{&7PFn?yeFGgXI5GRbRscVbA=P5^H5
z9caVf)>d5pC@^$MCYke=c89mH&x6QUw;xNSJ%(NQWTd*8|EYR^sADAtp4jpHgOtOI
z>=IER;(F$41P=clangPF5yw<&QGZ!+OoOEdM4W=xTE0s1^PebAzqV^A)iTRV8U<O~
zB?^RZ0}7#)5J~qaMdk?Bib?W~qDh(xheBO~wVmu`vjq_*rsc?&qo$+9q2Q{nIQvf=
znNtYy`Lachd959hELIS1^R#(YX?|(%VL?sAt#ALk%}Q;?sh9x`Q+0@Kt|15qm9~s^
zBi69MCfb0D@~w)-VK}wh$)4({azM8vHL<cqWBrCL_3!8CgH}Xx`*Vi=OBrV)#IL@a
zED!-FCx4onrYtoyz^>54^=g*`E_z*j{5yV9aQ)ySR+atzo2*=Sf4tnu>UEkgitn>O
zW-1?r!dD2HZp~m+*3pWkQzfWIXEVojd5?H(B(LKwUUsV#;Um7gavKlx2rc?h+5+r9
z{P&*9J+Bof1h=wsW%N-uww8hsWw`u1X7<#Bo}qd}_2&(GXqa``I>R4EH{wjgD!+am
zd&sd<RFLc*x5v`r14?k+6YR`VM7Jg)hJAQ-cYCfnR&A?3FS_6vyc?r!Z~+xm_qYKd
zv&;crK8qRo2FUHiXYbz*-3&0mRV8-b`@JNjsXI$Sw5Eq(AKCLRRX91Rvcb)<Qqytv
zXBRiY@0cyAan19rn*;Gpegp7ieYU2H%^EcH#yWNPkE7y7$pm!ahpe1%pPK#l7v60{
zkTSMGe{pgnjj{K>%fy+>fuyz|Ce{G8gUgxq8?W(Sre&3Ezc)<n<N0x`d^N-DFP)KZ
zDNlQZ`1C*Nl+*K6ID!zrOdSq-&<d-2f8pHx8nlus1jcW(QcJl!<ZZmHUZdqKnGLRc
zq9X}=HtwK~NwPH8(S;LV3pI$8Q!b9Yn0Ec|+`?tpm@J7^A|8E)t&3<V)h(Rk=zB8H
z3t!|2$!)T1+l&8FlQw3~{EQu75Smy~&i>od9LwdA+udf;ogtWoMyzVD*>x=e?4CV%
zyr|?z29t|^^TgB7wXI%ls7*m*o2n>2S<?9Uld3JIaIdvoZb399E3fVF+pD9=?R<mX
z%ZkyV%BMPjbbV7+hstExWt{1>4(WAoT)9I^C4<<(B=@fiW-cJNvkCq*-pkY8jcl}3
zGehp4ZJX9#B>^grET~NvIunLxQ)v9th<T4hhK93pz!Ck?@s3tvsUF2i^^%3UYw+&P
zL9e4$3G6bFT_K`EVuXUv3IuaGq`1p8cOHCuWlu6q%o28M|4M!PrV?kvYHdfNVs>$U
zddAtz{7ul`FhEuuc^5a3df0<?XWJ{4kRZ$yoV%(=nPR!3bvnuX4^p9g8#jyWmtDpZ
zF5+Hzq+Hz*&8?8LmY0vguQ83~PQ=|VUKdmad9mN}a3D7xb}rP+#a!zk-lhL&O92OV
z!ifQ5v)jvtlNE7W$-<d*^zYNJK4)Q=9EP+?nHHA~kAAVHr?DX+XN=>t0Wk`HK3c5N
z0+W;Sq2|8;w>xTjd<^%m=pq;2=+)wX0Cj(|f?RtQ4sK7E2%~=amj%ix-w}fS4ZNQ;
zxy!VMC4@s$O)ri+xunby6bGa8?9l)D=p%q;F2<8){tq<M_#VxameH4kjm}Cc)nnpn
zw)=G`XIhBFUrke?*@<7RCtp(%7(45ETAK4)%udFY?V1GNellvjMTQE|>FhkJ!pi-W
zHu{JvCA0PhYWZFE=9YV$y}m_UfAtyqpoY_;2%piQ75M4Kq%A!-eeoF?^8l+BRdRn%
zkF&%z=>4R_^yw@o%-p;ye1A7eUef8Yf&{$at$J9NnrXfR%?j!Z<KGF+K*l~D*Nvnv
z`96MaBGd#s-!r`o5(SV<>u6tti65L%*Vd{amz+nVx$9zI_a5Z&nX66Z<nq}PhZ@w{
zzdM6{L?L4VyCIE8O-F@~BPQ|J!%7j2tmKa)!W2fRZQJB{L4;XxXqv!t)Zft9McsdP
z9btIkE+v}fG@PE2FH~T}>tQxrx7j#Fh4mp;P^aW-#o}TvSq*o~`LkR-Nq?S8ejML=
zX0rKNTk=k2cC@RHob^SD+PhLY>T6DA>{$i5kJMHHx{=Cc)vnWBv!u)?FDe0a^OC++
z;P>+n7u-!(-z0PMb#T>*k!g(U00Wkr31Ra&8)MZ`?Nu*hzH+4?p73$!K6ZU3+qpp7
znaC>Fe6sMGQ0?Z#Yz#%J&Irkvu{T@Z#nvwMgRBBsYSAe)gI^((<%-iiACLp9yJlDY
zlrD}p$?+B{b4o-Ss;b4(+nJpPxI23+Q%qaIv%OPgn;sK8^CXk41M-zH?YM}pFHC=*
zmQ$GZXa_(%aBqXUTBQFQCk*IHN+DS2&k+J_S=EHP0l`JFpz59W;qp>Gz)0p~ODB6G
z_sRvS3}4ZK&zW`mZ!<QLU2b>*nO>-U{L?LQ6sKFj_u`$y4BnokQ`7pAC~!y6eofNT
z)+9mtObJq&<wp^W3iwQ$4~>5!(+A!}gCfO0L_cMVBhh4dk4;+T<7-l|B?F<>YikNC
zCP)(qTYVV2)EZ8mzvhSMegOHZN+zy`t-JIL$`x8}FiB7E2OoKTK->KzUW_XULFEx5
z?uM6XUJ6(-i1O)6?^*s3a873q_X02iRQ~63oPYn<v<9DJ4QT<4)Hz7k^M_Ymw-`G}
zcJ3X!+|&DS;ovH2$2`ryE2qBJ=#W)2st==3N_8%uG((%dK7r14&Ln3}n`UaLJ@FV4
z0KWfoyWYY^?tW!C-{cq;1iHcIdhjJIqQ}x?GS3g?QTk<H-9@l@np*>}iqUh9jUh^*
zDaLIQ5rV_&N`ooEaoN6L!49ea_FCF-zVZEq&`k8Zeyx3`!)*0H0QBG;>ON)@eZW&)
z9LPHmsY_SW!zRObkR8t*azbwsL5FI+rH7)a#UJHQ^n*FtVPHx%9TFq^C-e>1Jy`i`
zjIK@#Pk&^tZPhez%m?|i3x^KHgnbcG=Hma-+=(}9Lx{&L9VhJmsg6XZ_stVFKldtu
z%T!z!X^NVod}nFxhwllDQV%-b^3!)CTM$bu1xt&B<RB6?yY{qbwoBmNs@lH!Rc^z5
z$PkIJD$o(`_eV2qMIOQ6m&i6!m_WHLu@Oxuv5^L2&I<{UEOCRg_0~VfGfuSV@Ar47
zOi>cXiRPRPyzahICt@Pw4ksnwsMWSLtyN=FFS07uv~}#(rZL?_nCF;AK`GauA|o)%
zSp7d)AE#Yrol-~5$0oQX8<5}KZZ?jci#r4Y@9VT395auFV!qp29lpz6L@9<v+mKx5
z>?z!4W&DPeMt2^5WPx7~*X1$Qn3aRpMJJC@q8U!Y#Cwz#OjQdFr)4Q7Ue)rQDm4n2
z<Y*<DoDJGGP9!4a)W=fO?*1tQrThOXdF_KKx7n3abS$+&MB}m`mg{3W4bUghW|Mda
zgZj8^6VJ&IvAbu(^<a-A)K`2^RAxgg2m!6p(P)ohyKH+!8p&}P$IsQCsRUo-93)1N
zyhf0Pe*sKrkR{m5{?I{vs^8dJu~KrqW@KbWbVu#!8^I8{$dzZs!>!Sm($6ZkWF)yZ
zbP*c5<p*z4HO|UB<%uAgCspB#SmgY#gl8Y?oeYii`(ZtYvN?ob9?>^62_nBP^wQ8A
zc1P?G5mHYYi72L^h;aiU@&ETmy<g=j+I!-v8YANHBFr#O7!YH)Y;{t9VOzm#Ha#`A
zQ67r&(<{8u>7=>>Kv};A69KKZa7%LJy2pI^uiT1aXIq2Gja0x#UV;8PB8+-$uBpai
zjvi&e>UtO#cK*9^v-9Gg;Aq3iYr<K1;%Sf|g9VDy7}Y-k2fjd2w8Hno7{riAqIWkl
z2f(oXJjsv*VXthQB8Qj?PBi$NpBjaA75V=0bDNZw6fVM@z&@shYz)S{tJpB6DbWdO
z0OY%|yJIec20oDm4}N!!jwdRc!Ap1oc*nH<<sHMhA^!B0XZVX($i)_Ac*jIHCb{ri
zeqT<vP-05zb|2D?6@%AE2^iPq#*$1$7nuy~a0nZPgF?u`CTbOp<}5CEi6yLK!{=~F
z1>XJ@X5L<L_<$+Kmst*9>|?v-=LGjYv42*E9&i(w8Gw22KC$R7;DDhRI8DSv&iy`v
zixwm+6@|%jd?<T9C6D)+@cI2_CCYvOc4@q*W7w?2@Vxn!4@CMyaLGRp=nt=G*1xut
z!HheYqqPs=)$P<3>3d(!tDBH!6i*e;CS0p0%U?NIz4a!(ih6R0mZ9cUS=g9;vjQKa
zi$fkQFH^quhdKZk=M%<?#u1CAQy-SdTbQ&ISbJCj5^J_Xs@^V+Xz17&^j{&RaeJUT
zao%@yO=~3~MfeTErq)oNRV1Mj+>r!*NB~X=APJ#TH9}}yZXzj0UQb9Rf9LekfD)5J
zt%tm;x+9wnSp^RdQ=K-fpBG#P6_Q+)g>b?Z$h40YHSEHVM`Q7T6WPYM=L|8$?X(|m
z4S*|kbUxP>`%kaB4TrT6=MAX;sMkJ(h`Z<~_rXRQ=`>I*9FJ5Tw78lLrc}k&|J;`f
zfpEVc<!-{4+oq}W=iM6|%NZRstcX=ouwRRb(A^nPhm`r+9&K-Y%Sk#H0HVF;chhfJ
zl12P(Niwh57*{r58d)R94?}v)dUJ%9+pk>dR@m{sI578Im`B{<h9HVCtGI=N$-OTK
zZ837d>J^pNsV;jFLe`Cv^nl+pk$qhD1>V65%n3sc8LoHIo=Gf+dZ(cSUy1WyBkk~G
zY7F9VQ`<>>k7!uZm4Tc2W{6=zT-A^VMa#o38kJq29DsV!rHlCudf_u;z}zLW`;%L{
z4%ODkc*_r1Rwr2trnZ`Ju%!I$s(1<^(~;DeY`6%U57g~?NagFMc;twmjfmnz%ec#C
zDQ1}APq6RprVMH9We|tvp)oO<N2I9;(zB30fhH6t5D}Rp13azv`<e;Pe?O~DM?GSp
z6udiK^oTORC|U8@iea*wfo1)vvM-#KW%)shs&Z_oyYAo2Amsv_KY8k%TYQ!#qo(5}
zs-kjdoqYKk<N>*%<VaJ!6zgE6eAR7-3a25uE9(`NiVg6Y6Wzb>Z5z(Y@4t3io6>}h
zg6h35g%Dv>QaAP@jGc3DY)?qTP(;HEqQy_Z4i{^?cmp&NJz8jdYPJZuL*!W>B-s%`
zpu(Z$cy^;Pc$~x#R&*b>^zYDyK?y9#5t_Wq!D~ne0}N(sKkKt#to5)okD5yH+lqLw
zlswhla4I4py{XlY6MO83a?DDZG=bX+@bLhuLSgjvo$YIt!}gKTAz`BofAbA({)b6x
z^nx{y51*{jxxrs=_#N_SUwjBoGW#P<O0GifXg~AW8y6_r8GbU~7wxOHr0f3_?L|A>
zAg8q%+ktYn%9)au=G*IId4Ren1hC_^a*GOk9DvqFbmGtk$G?i1e-o1adHs3yFEl1g
zjC)M;cm{+0tD8H%6DUAAg-GpfQ_OpRs1u6Xp(cN`<O1c6r<ec*HsZ<rh8QvhNIFNo
zKR(5Az%N_)1NJ`!f=`L%4~b#$FE-s+Y%h_3WmXY>SoA>#eR9`p?TKe$*J&TwTdaY2
z?}%aSZ{z6@y|F4jXz6AVJjm#!Tl_fbOG(oa;jw8F(qO^NrW*jQ(J=n%pXABE(L>UB
z5G(UcMR=3zSK*j_=@%(^+Xw5ZpYjM@$DY&ZB7y2cie{@83ILx30PxB7zX6}C5njgw
zqVpAVE~fXkK(g_C`LCZ~KYEl0ICwhrvQ@5VGQ<HD#$bME$G_{2|NS+K`90OeHlP+?
zcecDqbK&vCF%|^0=H=#b0HPXy3P(A5kKiQrfj*t_?l^j`skS+&LUkCBfn1<zL0bHn
zc+}|$e&Fh4Lagnw@s{6g$S|>l9$K-Kc?jt`$i~*z(YWZKiPSRqcJZ9LJT>o+8<h%)
zFF#r1yO4Arek>Z7C%(brG+Qj$6VI}ZT`#^O>c0&7Es$QnE*YO9H^qCZ!OvD^NVBLm
zGZ+PbA%>@u={#=Zw!{UVeWw?+4)oSJkHuhTL4_AxObsb6dnOQ;L8OM_CIkrE3`^|<
z_&;IGHPFO8Agl`*{FGqPI5i~!df1QJYxiol%@Xa$iosB1QvqNw_vj+W42a?Eoe)Wn
z|NUzH`&|~;{go5Phy#H32TVn%sJVk!gdnFkPAEELO_&elX=z>_0L~p4+>gT|=v$Lm
zWH%n=y-r4!#0-?xuRUq_GRO>Na}NyEiff}MA-yc=408tLse(3Zx?e7^a11|OUMBsJ
z*OxV@wais+Y^cwr^)m#_SRgeL)cLyn#klOHbh_H6ls`$NFgNqN@SO5z-6m25Ai{hW
zplL=R-J9WTJQoR24N@D%eDarinXJ(xQ==`-HiK8<i|rP=X*^K7o<Z~3$4}o}*X)^W
zgLn`?mt3CQ;#!q~8wfY0ql$K^&cFFktd;i}#zw{-=f0Ntf35czf8_zCl-T_b5Ai^#
zSWWGp+{!`HO%FWn!=Y?Ws9v%f<Gq7k^1&hksCN2T;7A(yB3`(!OAw-<2yZx3iAslR
zFh=|0HI`;aTdJ>KAzv$-hO2<2KbdjzK>&=hAnM14jqlBi2Eu5>x4a`&!@M6af(bYk
zE;7KJ*6eSkB>WrfB2rrg;Ez|5$z}vD(s83!{T?oQV^gYa0`<rjKz&;aEC^rf_2VqA
zLbtG<)uZZ21?$j%_2+OJ9{qtldD2bpe@M;(Z6a?6iG5rMv%0htZs`U9hmjso3Sj)*
zaBpx2n67>X2=2$7@*$M`zX5xi55D^|C5bM;pu9QYdsObQJ9Y1B;#RQs8zW22J{xdi
zt5}OORhzCbHwR1z`J9jRC|}wc`6B4EYsdq+5#*oT$v^2wE9DvV#{px*EvuwFk~w9-
zb(hV^;ClHFs`GD}$Nzkd#5SM>qwp+i#+4U$YKbZtO{{0iW6OEUG?61)@I0*-2qZnB
zPpKC%R={RnPDsx|T&b4_%?v(=#KFoF=!XaRmn~tRkdGJDKVh}Ugpjyx8ijbUFNzY$
zPdx0PTQbd$3tmbUe-~=o4$J(L{^t7>t^ZF(8I}Vqw?odM*WvzBzF_%>Uvy3upNwQ+
z_(L;ff|HgqZ-gS4>`p+^DnLbk!PM~fc!;V^Wu_?FV{?h)=?YT3fEP8$3rk8gWPplX
zm%g#U6n91fBMzk)6?ZeM=0pW^(CQ7KAO$f`6msO_vm80;gm_Hcr(8f;rkqVHPusgp
z`3#9%MpVu^O!srI9s{FR#U9tokjWrHNq}<lZL~mBC;~_=B4;hw98Uqj*)+fW`@M0R
zDziV0OBQegS%Os}UIEioJ3%Z3MZgYv5xQ4ah=EPwMBA^VqN(q3+btvhBoVL>uV}>+
zBI4>9Y~2UUjbcWYfVS{?6Vob1214|9q5#nCNw@?C2TVrq{fR0@<b+h~T@r*<zEyct
z&e+Hldou5wPZ}n-S*`WOI!x&J0}XPMKUq3}8EE0A<0Gh8;zv2b_6`JuqkLlR6(%s(
zEChIsXC!w|6ziHFEHpd6$Ssji1<Xgs(qM?TgNzuhl8a-MZ|kHH`}aib9F;-~V3zdf
zEHDOr!P_hlpK)I)evkW^_5TXi`>z)f7~Ufy{<Z3yW16<#X9&Dt6F92`Nhq9@Ss~?z
z{s|2o3uvJEfU;C($`z#lfiJc2N-Db(T1QQpwG1{`2p}g0A~ZL&p+lJ*f)#B_@OJCG
zHvrUUD^5zue(}{m(Wxxega6%{FYZ&yQ{q<a-ed2S=vPOrK4r9Q6nAmYxY%Al%#gsV
zLKg&kdNMvmkJJ`<P);QerZ{>5y&dXQhL4|iOa4L*r>>%Qp)STC5B}5?RMJD!k#P`f
zBJ8`z`70rZOi5p)f0HFY<|K+yPIo56(kuuADrX~r&nBT%I%?pNID2`8@opy>#b2?1
zl;}5w7-q`%$4&~*`uBkR*)IpZ(2Eywb*P~2G-~oh0eqoa=^~@#UUp9ea0K4lQ#X3C
zCSpP0gP_rGejqdeK(|-T0U3d86)_bO*8#pO2Z<g*2gs_$0C}=GLU?wuo@}XZS#Qt9
zE`t@`t>wC_)RwN0CVB6#O|pmvp8|MI!3QHaIU6%B+xe=V-^;=}KwtL&tpmU=eZ5V3
z>h7xXt`yMTVbTRXss@RI60+=whJmi2w(CxD=WFGi^;eix9I|mdhupyMAVj$9Irx{-
z+H<qUn==Q%Uy#v(Daq-J*4J|6teX9((v7gp)m1#J3Atm%qyZ(`0{zmt{mI4<#rK@m
z{XW29*p$+|ECTBqMe-W>vv+Iv8%hCfpkYS2gip+(7b)nE!xKPIoQJfuCHHv}O<r8J
z;FeSyuG4sXY04gPr`|Eo0|#X4$_4F}iAwwNIj#newV^GQwrd-Ay~Mm}WRCj%Vy|9#
zX}{c8kU74!)9?SN;rMrE{?{?^pZ6O60_d=6>q2Zv85WySSglB_KvtlsV=Q`<=4I4e
zPpD1DXVju-8a844myyjekoH%MeJJjmY2J5(h1ZWE34Uvt$c^Dn2p2xwEJ7+QY>8gw
zn&%cx<f7f*&i5|7FOWdwFaU$I1FYLmo^<(t2>a@=DF3z3p+QoRj-flGdq6@ul$4YP
z=?+PeE(K|6q`P~NmIkGz8-{KeU>|<(Is2a7y>|B>nCqe*;2ZaSf2u+s@;3148u^Co
z>&(r~`SrSv9T8tm0hmbyoCb9`AOj?OEI4&CxbMr_Z&N1lT9`-t1no?Wo1w(C!772y
z88EgkOyrtTu_FZg!3z(*NY@Q`3-PPz`I_c=+fZFlRtAf9?SA$8oG>IqSOBFBw0yk3
z;?j~$Ic`9Rd{38_mk~)93(BQc<YXOr{k{Xyx~Rt)_>AOdcKP+SsjkOA;rAl|&wiW&
zh{6*NVwK7NFy0x2lq|zFUp0YwUC>m*E6c(2{VAT!p=Gy#-5FqeYBJ6PCqXR?a<f5T
zJi!!2ist<lm^8iXXk49C;;L5>nCn5y0D@O30ns#JCaAwU1BgA2$Wr2L?ioDw|Gb{h
z*>)_o;I$q@0CuDD-gu70{7x^oghaAkz<z#%4}p4MpD?lZG{b9A4DbuK2Wrl;V@)RI
zl=pDM0hC+>EHVT>LRDwCW8$%Y{_`ft<M3WbHv^;p{uloruR4txk-X8fNsr=1#|*pp
z$6bPK13x1LqWug*;|$dHsn|a-S7S3k;F+5UGiNg12^M^>3~n~<4C~kRIL~on*RStM
z#)T2God@Ig4e%S!(FL#2Sy2UAjnDk>=IK*S-e(j!uMRUWbPm;z%@QiXiU%`}O3`q9
zB6>zZok{e`AnGe_Jfbdgyz*_`fOOU*QXuUl1xenJE^fFlcj#nG&x;h4cSV)gh_x$7
zR6essmDAPupy<=!XeBeZ7jpZPdOb`yJg^U~rJY*Y-t^kp&Z$LRn$JNa6|IMRudoue
zP(mC}G$J+<b&RS*FcIjIGK3~Kg3zd#5m9sYegaj3{cM%VLwN4tVk;n2U~T&rQCpXn
zGOPR7W5M24wWO(pKF0-E7qebVK%`=KF<E*R1()o?4<%w_!&ii&`Cg;K+q2=%>EK`P
zrb~6WBI1e0-^kk)6bNe0QJ4mcs{paq>eW+b;Y#qe@R$>hhS}WkloDZ(zEDk+RkmTP
zhpgM}#l967aO~_dCJTHD$Of2@R`j=SMMBO1rP|_TjXHn_s0xR!jQFg_J7Y!mm~H3l
ziYw1GNeSNx@x&_X(%voyJhq22H0)QjR-ClxG&(HO`dFZh0~wpkfR&hbk6PM9qvJ9U
zg74(zkk9am9^fXPi8eNzE?D`epNvS5t20XC;o3|UPX&4WMKTbkL&`k-v~<8g`D3DL
z`v2+tvYzVAv~GinzSb!Gh4(m}{8eMP@-jjouP)HIl;($5s!<T%^OcEyVx0jw91ls~
zTlJziEN~>(N4?!v2U<C`lWG)#pbFoTI2;$>&S0s4fu=9dvznh7BY>rhF9Emu4x00-
zpbe$!7sjr#XvU`_O~WrZvxobXuu{G-<`O->ABPfAr!vOkM1AY9`$iH^y2n~=Vp{b|
zuEULBBTAcRW@L{or|yd${JFl%O<-jhGL_3sQiYi}<nqgT-KaV)<p{Ci9COD;4(?j*
z{+}ga3DvPLUo%SP!v`h35%@|@rFxSB9?r4|A&Th7SaLF;E}&iII>?RU<cXbPVKW5I
z_^r$#cHYd14!=9g6CtYb5(JQgR12(TC{jH0(|VFnnsg7mi!K*5iv@-ltkK#7i%IGw
zLINia{Ob}k&~K03GrD~ZL5)6718fvbKEhA>kz)QDmaKt3<0Ay-ihK$$!J%0Hwoi(%
z)15Zywwb6yRE$^+kjr$lAbvH6baItdb6l(#lG23H`VP$uL;i<<n_pj0%*)@T1|6s3
z#yPEBrEbHnmc1Qyeb$=DVUb7=G-zUo@|Em8)pCIyk?Cp{;8lP&2TDmy5}J%%?gv=)
z-q;{Joxk%#nAr%kCp`peFIw9GJxh_uS=J+*>Er@P4!kYV0I{lx9@MdSZURM6#zk5Z
z$)<u;{giesaKFxs74XC&uDf^m-`{rR5+}AxMS{{^l2?Oyzlb8Cpr9B53%dD?p!q*r
zy8rX_P11j{$WM7k4K=e(YPE<vUtGeQJH<K+xmK{m<b7jo=7c15@~h#U$q>I^Ck$_W
ztDe<#GXd(X*>FBNy=#Nn9~d25<nvL5M53l^(vVda)$pu9Cd&HLZe4oP^(rp!=J}VB
z%@pk1id^Ud7+n&?m}`nmOV#+0`<ExD5r*=o1gH6!agfiMYrI>b>JzepoCcz2rqMWl
z^7c)!(fR6(;K*Moz>xcNzr|k}e%>}Niz~Wj8>V+6?dSHxc<f@K^%-AW-!nc=0<Dxm
z%<OkS0Sp6H&lR~jA<9IRjo+jK)7+@&5scRxNQ=7LW7<^M2g>DSlsb?Xg5)Qa9X@`C
zzq3djQh38aWne4#*GXD$s>OZZ@+s@SYn%L5lLs)@kn6gG)sA6-RGq(}CWry)(x}XZ
zd_B&@b|Rr}J2_$6Bu$^8R+-+6rDS6(KZ6f|I|y6)<{^~a!5S!N<hsDTmeB~|ED@8o
zoE(9S*m^fW_Inwi2!9t?7;wLDeA!3t_yPm>dh-b2aS}b2H!jrxmes;hg$TA&Cp^cX
zO&Aa+iy_1$POL?qTY`G^5?Kp@UNUPC^XsV~k_`yE-|vJhpH&HDczxdi<ipnzA*}3y
zAFt)QkmwO+E450ssbmNt34w1wC@s*0j(Ge^UXWeU9&%t_6WxH9zvr9<aR5BUDBWwT
zC|h>w8=m@G{J^cx1gi2z8mTX#Y@UCe!?Hx7VIbpdpqd|h9Y=K#r<F^KIVO=U?Qaz+
zIvCD>a{Ig5tUo_;2w)*@-T(Dm_UpJ9O@^b#p_uYs12y3#z9i3+@qVt)dW@jnSf;q&
zsE{efT2qs4MRQ*$4&`1yC~*MnvWbRJc`^PJCv8+J&U^jy=m0QP-x3b=2ko7Rvaw<Y
z7C0_<^zkB`w5l;weCdM9!85SdKl|K_$AQv@4Xs^!hB|lw<({vo6^`+EI4~@R^Qx<>
zuLQparpIG`(Io%R&R#teYlAd!i@Gs1;&3f}DHV4NlN<3pdM9!q!pbL-U_wdM5VPI!
zuCPwb@ohv6N%Grn+z09)%N?_@Xtv}I(_fiHT{!Q2qL0W9C)*5W2z<Ikvh$k9*O<z2
z<49=xCPSD2;RQc<*T!f^JLN@LZtO;w!XBf|n=~(GY2%1bV<{mKYQ~toS$2jqXVS~Z
z8<^xCzVex{oo?v@a(RkHz8RHxWiX^AE!o!$^QVsQm@ZEr6Hl)cTIL#EH(JM67Kn%e
z?Jjf2`HB+mN7nH=*K}(n+Q8SvrD^uS30v0Sx;^Rv9BbO>GTo@m%N|FIt&Ph~n+tHR
z_Vc&R@xKZ<&oFFkmT%FHyPT<VE1?e}R$7VY+;VmmwGl(1mEke=_63o|4K^<^R-$}C
z<Ld5#BO1@ickJq*K)|tSVyjAJa~&Ws#MZ>XUj(l32j*fRoZXNest~YOT*zqVKHFL7
z5*8i<aHe0#*)KG3yp=^=364$ug_50vNZ^gRzLK?lv>qqH$*H-qt$->8!UkbW64H=J
z|Mmq6#o9pIFiMXUN;kYCQC`3owybRX+j2V4Ax~i<{8rClCO#1bG!Z&hl!T#6(8qhn
zb#M0A6t)$DPF8hi;Gu*-pJWmkRZit1pA$cO<$C=h<@N0c*B}L<+h|4O=F<>Ls3ZIV
zkPjL;h%r~&&Q$ngQNI$#dPR<oz`5)A))Xc(oL-_j0DQo{t%6t%WY5{O@@cK-!3Zh)
z1{f*Dtj`Zz0f%+w&bFQ_t+Kn&Ef4}QeP*zP=?s4_PZNlWw*qt>;K;KQ@KTg+P#-%0
zaO|BJurO%=D|3btfsbwTATH{ERk!@FV~?Uw|Il1=I70~j_6j2Nk|U80gPIUbK%YsS
z%8)l`g*m)HZo9$6czCFyzc!Ib>Ddu{!v&(59<P%)e2%LVJIrS%(F&=lvCPg_CqiAj
zxjw7zsGY1x4;;sE&_l!wbGUlrDMd~ra7T;<qK8zFi<!Au7!Zv2Y}ZaJx$WgAXFj<j
z9kXSgljd=y-kt_g`7y}ZTNhp*Q}>-GnuA`7frh2UHd@AYd@l$IC`T|zA=KBCQzV(;
z9jCfO7v0yhG~1It<K;&c|Axrs%gSo?1EBE{_nvIOBZ0W(jaNu^lF5bmBR~R*V<7D;
zH4xj<8fhn)WWfW-*()mr2<Ab)t3^4HlRy;nJ{#3Q96XeKVxVZSyU&hOF<9XgMbh<l
zw~)L)Cm%x+Pm62z3+jumH-Z*F&eN2TvTc?-{IoZ5GqG7kLE0FlTAkaOVT8gd%L#y$
zlxFXHG55Vcv|}jn$+w&!?DSQ;%rNAgFt)c3R@BJj)(v86>SQE{@`o9oOD8pafqlG-
zxN~o-Q9;;RHUi7Ot`UAH?3vOa{*tm!*r1p9MsHA9%HYWIsOqR$nq~<9_5uLmO48B#
zqg}UuazU^Lb*op)?yr#0mpFy=EB!ETJpREypek0<A5FsjDJKh)I%jXnoBRunq)fDM
zg`5mK>jh{usAx|M*jz2c!OOdP^<JWH<uzdlZZZ#`UtbemGM<6j&VVMFOlvrj4pv<7
zW5%Xr=Nqx4h5^=fKh?*wN^=Ivt>NXbzCQpY{%@*o5TkL{c%q{bP$KwGJeRCJUA${r
z`%i4~e}7$)`cxtutE~Jp2kxYRoji8ryT@!-v(QomcCtkOB$NEGN`uA*iX+Me%x2*<
z1cAtgw|1<|q5D3fyGn9GpU0V6@F3&euTOxpt-70$es{=|>@nxYBskWzgexmDq{AzX
zsbQAe(un3qe~w+TABtg#DaNzY9294y6ipw%Y}Us>&)T{&f5@ogyN@XnuJK><+Z=HS
zLg)LjAD>wY;A|bf!c%4WLz(Ht<e^+}OkHVT0BXE=4(`HfHvPa^cPi`48BRn#|I*(&
zj^c5)wpMF})xU@Cy8&$pkJbyx?)kYF9f0`3k1!ax#cmxR$G!`@7TX8|#ANtaNxKy7
z*S!R>>~r>wb|5^t8$jdxzSxyT!K8LXpcb$Y7J7rhmZl!@6X6&BI|VJ&$@lTT2;)`G
zgM9%$-6HS)#N+g9JGT)`k%N>HLxKArJ8-M4c63mLEJ~3`Qznoq0De-Y;#$(FzDbks
zHcTTO>Ln>%q><!LCaLk=UYD{^C}UtO4hv9I0-gjJl{kn_bYiNpbBI*J+uvQ`J-<O@
zMwnjME7t<yX6Zy+Mlb|U#MIB1PjlN9+#u_sQ~G06tr(y4oz;7F8<a0pl`$0WWO@m2
z$RWN>2p@tbfiEv?stUT!I9O<`MBv0=!Olkb-u2vl7<@bi6eZt_kf`4N;6R{az$&f(
z?!A@j4~c)HA~)5e2vn_J(jk<wf0dwzF6?9{vVjkv$A}%kjQNTzsl|{8ly?3XFz;TC
z8(_W2*A6rVShev~z+tzt8b#x3nVp^8c}5q=XysFa?7D!nF!#EgH$d&^ZjCCv_9YT)
zIetQ1LiB(0{JKbIt1ez&HsXlGpHV&DzPYm*Rr9V*+q%%3r82y%;pCXEDR58I3)aK7
z{|Ux~?GQhF6?Bazi1?r1A)J&s;9p?I$svAV=)M2tO-NjJ2-*aiV`u%iVHZ{t;0s1;
z9+wH9=r=-^mzza$lM*)T?bjgz!w_0(l839?u+yn#omc=_(<ii_GX~;j7;5|7Of7)<
z=`@_rYGka~E8ZXl^B2FwMw-TKvTsCs4gWuFxSmRQlDX}qVq5$-HsRUsWZanxi1kUr
z<*S;e=f^Z7!4SxcU?Nfg8Yl<ss&IEvJ2AP+na*=Q@Tchr;lgQ{P*1)(+fYVw>6ep|
z-$qgnp5*bLXr1v>1a7VVYv3{n8n1%$PwRF&(resj^HHan#R~W75Sj#E15?)8$NQrI
z!cp3OGdti~ipav-iGMjXk+T;RyW9DF#6ut|2k@92eM(MQ-bUjV?u&5$!GaANAO>Nd
z<qx7<y&}KcZ7%Fz-SW~Siv9hxdKnF0p;jSfQ-<fgYxwIjL@(o`K^Y`)@Z)?fii=Dx
zI58Pm8WDungXycAj>L%80EldUchG38&5)<Aib5R7t4=1_&8_0tcJg`}agy*G2<r4X
zK75c}B$FLEOre5^5+n%BD=!jlkh3weZU7+w=!<`tHwU6q{Jx_bdNzVNl9!we3fUIB
zBxU8c2u|co_@_^Zm5L$Xk#NP-Ej!k*-wTW+Jmb>b_yjx$27>IYv$9dU&&7_O7nr2*
z73&RR_NpTywWQYR>W!&G3183Ov#+#)jIKJs8*tWWqN}P;dATAetpGc1{zHt;*?Z-~
zc}GnD+G>sdif@8dZw_9292w!hyU;3G__JIvi@`-%`*w@w<3y=$xAwu}a{u0Ce^;}B
z=j#VMOUp6W-k2wi*vP!+89|ERnH4Zq2PZF>Yrp%ER&V^#@9Iu}acd;iLL!E;`LAie
z<PaB<ie9;c$|8!&!>WF|M6%)`z&`*geD^)Ux@i>K8;sEukzfAqIyOZrc?}-6N22X}
zIvSz#KaPwv$!aYsYE_3EJY>g8qk7(*Z_|pdMEFG>)6xA0rkuTJ@y{5%2}{+9czI`{
zLxKe1cD^$oMGiz12uVTHdx@;ch_~!6-a-AR9|utlWP)iDjAbbAck^j1#BgT}n*iKn
zyu&#sPwcl!!ROe2+7TX?>4X)TbA7z}L*bzcrA=DN_qHeOMOMeT&FVfa4gg`1C&f8w
zzA-&rrRR6{jZ%xQszkBn)}!z_@6NALU!=bI0`VDeOL(CnkOL9Kfqcep$;mMN%yAiQ
zQCPhu`UYPj`d5rBT0Cd<xJjA7h?)369p@VN)+hRYfGgwyP;MsQ))&`+_{@VGPp|f8
z#vE~yk#lz!8XXaNOptCz1eeRc(QhG&fGicjCmKob?LN7{LRm%LE7d3I7<u=$zIu29
zP11k7bZpsW1X%Ko?PBn+s0kYcfuY_m^n*(ukL-c&J3)ez<@?79NPm9SWju5H)(-@0
zp{<aS%P%;P>y-^=614fVsaSfnq8`SFzs;__gC!3L347GZh92)|xonwcToZ@TTwf8Z
z%jCmHCDFf_xSj&!is3XT-FK09Ex-ls<qeoS99urXZ{9VC#f5?Wy@)EC5D0aILY4ot
zfAloAF9sS#3_Mw$dIkjRd}&eVHJQ9Y9nNP6e0i^9Y5*LDLn&tce7@dx+nc5n^?gii
z0vXV40;1wy?>ylCNY_C{xk$R`tQ>);fusAC&_ny{pUY^h-(P7?2yfsmomnx15%@Im
z5Di$4vBp%rkdwAD<LrUc`cM&h{2Sb@=I4By1sn8d<QD85biS<_nrK%i6M68Vw6VTV
zC0##(apZ3Red+AEal!n#jfOp7kOD~69%u>wIpT3oDFKfvJ#0~@^kBO?hXtleKfT^A
z38HxnPhxExf&dHaN5@)?GOM4l{>3x?UWtApCkcrra|{*LHm&Y<XLJ=OB4?JW^;ZrO
zEi(6sZ)Tlq#mj9gm5uzV_-7#j@YBtt(0aim6TgbK{{Hl{eJ+o=2YvR&*&CuFkBs?-
ziLL#4)O7b+EgeUj_1BNq(vbW4j$61s5{Y5r?kBk=UP-?3^_eW&;KwUvEhE*CZKFI(
z>;=08>_uX{YxF5t*w=Ntu9QgxzH!INvTFN=5!8mIK3o0vTla(3)x3k+AmjOzuP0}q
zQw>tLV{gIP*P^gZ!<g%9`;RNRe;=2(BHc%)w->hTd5t?-HZm|_`^CnaZ}lt0B*g<i
zQcLoJV;;wm?5#LbEVyFWZ))o|MJ7nt|8uwg{ip2z=|DpOK!#+OcXR87A=lKg?f#rz
z@vRjz(#cT)pT>i#inl(7_`)_Y{oE`XW%wh0l919Ho4}*#&oqEbh01TOR1Fhj=aY(%
zg!zsWx0;@t^(L{OmyLH3ao-CgsoT*%3zJfjacxpP^lXaP<GHCMQK1|97zwHsNg>v6
zmV~Zp{B#xHrjTp^fIVA8j}91|Q?V5);pQftA65|#d1!tC&IC|)NLPAn#o7?D81uK)
zxJMFx@Wv^dsNU8<2n3XN`7df*B1Se1A*G1haiVn^pN(m=B%d6m`#0w@eDWwkpq5t6
zR`AvswDj<y`yR;vxF2#<D(%nJ76QJKr0*^PH}=un<LLU(9&Xyz3EKD{Kk08!vnRxC
z$ORqj^zG{t5p;K_%hMNNgGO{B8U@NH{oUhSv6Hq&WPzdUW<aTa#_p+^iOwRP?6#R)
znCUaP_op}^MLj6k%w$lh7ue92q0eY&v_kUqx^61U?(rlO_JvWMewaloB93o}G+>U)
z>rSBDeUWlP0V*ck96(k3J68O1R-NAp3iQ3v-Z6gr<&BZo7(kike^*|ag3sz)rr94$
zDIP_4%qQ3Rxx^`9-~BKrjEd)81%s{c8<Au}#g0ZIh!5lgq7Au9hDrd=^OzdxjrRt?
z4~TL<QsO-U@J`xdqJs5H7X-!1)<Y3}s|<HoFfQa+E-bO{>U1&!l=e5ZphojmVMT*>
zBlcfg$#NN+o~kkX4MC-VK5_vEQ+>PI;3>y(=4xzF=wrwDQotDl+CVC4jn1C`6JQn4
zfn=>ZccG~dHCugCu&SaD7!s^DE-{T{S~Ediq*9U+8-TMx33@>(xIENyI6K1z3{ob~
z>e=}kW*yA-T?b$Nx@&%kx{_-9=p7bw<L^6A@fP^goz2!;Eo?hNbbf3Zze_0BuZ*Rf
zwc=VZZH=kvs5}0JCJkFk2eW9U<f%di8s_~IzPIpN4cA1ec8mr^*FvqoY6$Zv4NC%0
z$v?d7wMisb9ozBw!`TCgr2vR#a31R-0h(BP+12ojT75M%R-vz&;6DaC$20#WSaW|z
z30T^v)Krf3TFQDwFb#u9knF9gk;t=&vY+vI0b-5J@t{A0Xb;EO#%<%;zHZd#vO6pg
z7fHcmz8SaWh?=c_lE=s~p?oK*9XzG?)Vnt2k+@*0{Odg>9gb6%giqfVf_VSPY)DJK
z;4p~Dku5;Re3_fxodWr+c3v-}XXh;>v?h-c7RPB;F6&RprjU95g{31_Q7TkC>-E|D
zm9Qu;JQsBRwY<XXiu4|<j@cYEmV)V5QOdYkYk5)7c_s7Hyz2p)5lZoM@}^Ql<nbnU
zP3K8Vt0$N@Q5go+57Fs)&ZwEI5Nusn-R5;-3a!2arkt8W+B@$PR0K}sGCqsNKA6<D
z3FS&*d*uEMM!SpTC96ePY<9K6R+_3Yi8YQ@2g-fMUsfO$A;(*aiC)I0Soa`eL`;(H
z-1D?#)NBmFOV^vB-X{vy#qMvW=TAv1(<NG^=HDw$zoI9(ffhi2pP%>w;BUQvm1;r0
zJ11xa09H+P44gwk0ukn1DdvDlW{HX2*-Ioqg7i!a-Vu7<iG-^=l*A#9*-4@GCEE=p
z%>a~EKdm?902G9kW4MyR5|40K9?pvH{a1jqaU03F;^V_TU`)AmCPy4{#)2W)eUuNP
z5L)98U&ih<q6vXot={GnP~{P6*R1Xc#C{Bx3XFycbtz1>bihxQT93Oh+jlB)vUjIS
zlQUcfSriU`i6&YJ9afg815eA@3X7*$)=G0%_r~tTdV#?A<CpHk{-2_EzeU`4Ei+~g
zGwXlmrzWFH{?0a`TX3GiQdjqA))Qbv>o_YZRjfUq88WyG;5T~OOK%}a)_VL77CPXx
zV&*RMuo&{Kik!op0%fhv8_aF*CF6r7c6)J?i_`^3bMF%6-Q}dR_A_t5%Hdu>4%k$#
zb?p2N2ldUA&Qhz@1<j@$Ms>3eO&gqd{yixMYi--*Fi_fS6|(u9ewZ1#L$%Rqhj^RK
z50r5V&*{~MU)s$-^G*e=rX6u))*%`xhPo^|HM2<&BtnF7yv@_h`!f9@cfU?T!w}1h
zP<vQ|;nAsU>SMpJ==m;^uf)TDPMfZ6|0xmeK190{&ms!2%`Jp=;*ZauaDphE5W4UJ
zxxf5%Cuj@ys+(OU4md>sOU$lEVTvL89FGH}h%e<Yu?D^>9JXX0M+KZzn_<zIVGkTj
zE7@%TtiN9do59)*liggt=pVO`Yz)(a*XZQ>$a2)Vx>@?wP&mSDOliTZpXy%fae$C+
zziiA7W}?`>nf!_ciRg=kv_vvxk(mWk+3#pH#AXakd=+=Om<ZZPNG~VP?Qomo;gWY9
zjjoyb9J^6xP$LaCj<;1VO*!`tOo;;VAr&3s74KQTe+RN2K<y6z<*o|hvQAW`4xUH`
z63iM{nl;r#02P$8QXrF%>Sp6|Bo((L1ByUo{pyhKHR^pYu`#R=6&$b{>SnY3ecvmO
zFNGcq;`{qpC4wnO;~cHXxwjzbdalj9-jrt0oT%zE0@P_m*Kd~cl0X7QZ~qx;*}O+0
zzk8y>6eLaoYW#>N7}4?sM`$H*L$?Sx@o&(SJDml0tv0^9j#fXXdu@Ib9Sr{koPj3V
zM;&(#E^fLl_{zFDKlngo@b>Gqs9(HaD{6uQSU(;QAI}fi%Fb^A_)z^27xeEL!YSz4
zZ(ePn$$iEwdel~Xak#j!a7u^V8BU65y>cRgPJ)&D49E2aHigi5MaS(2iJr$;(L?61
z0UX45JQ?KhRbo9VU}!3*CUzpj(>Uo-R43g_IYAdEW(t^^jgRlmR35RVBhZ?Qg9G5s
zw0Q}rsUXvmNV#Uf*~D#kvKJt2?w4I8@vR8G%=DB8k@X!*&)YXHiValf&upp(cwd^K
zq`~Zt+&d-y+QvOh4J2^`HVBsKneFAWLsM-hSpl34W2o&`5-Jo&ubNcS>#If@=O#XI
z6_fQlt5-sGFPGPIc}s7<15F<^+!uA+ZqeO#|95R=)^;2eZk;Ta>HTPMQEU7__^0?y
z|Lu`z=XH{>952Pg@Xv-$`*>NN(%)6hgm%r9YQ2>Na`PLjiI33tT4QIC$m4vDi=#eQ
z!oLfhVBuV_ZmT<;3%4Yj2Tl5$nQk5WqekLkW9u=<X(aZ|hON#IbF+oJFNQVdOq8c%
zJU3&+o0K?^pL&(mNsr|4E<>F;80qg*iw{E<RGt!kelB2F6n5;u(CwU~;#4l36|=n!
zo6GZVuGT97jq<Aw@%uB{Wx%$`YUX>^T`l2mPs0XQS*ttOLblwNl0O&<Z;H8fQU8U?
zvMX@#Xj$m!owV%_v8lSaqdt@HP_%Ex!c1%8`f=bJ6_<!h=$tf^Yxd>8M|g=Z{}BrO
zS^RVYpc-qXO-e#&m)A6UkZa7J96OTliJGYGCM?Q`#E(&O?3w#f@zo$JERdR)0HZuN
zO8G^sMd@{Pb5-wakH{lkBdS};Fm8`B1moj6s+)M+m<&aHB){vlIJfTWJu@B4_--C@
zRMhMD_@_TQuF2RijuVl0t-l`EQ!^Mxd<EONDd-6HN3@9x<Ye^iuyjx%+T|;;NfEC+
zyQRI|VeYuz;qkm4R`83d4l49s(>dAvU5&qdPAKs`0sdZN82~4}=@G5vb*6p*4wf`@
zfta9)ofPs8!{OH@0<%f7({6k+J0FKrxIF;h-7xmYpTC~SZHuj*V{6Oa+ZmUGY?WR{
zK6I3f+2~q~7}$afcC}8{>-}+pfuuA~#Uv<OL1*ZFM|14oz$}PSaShwJlf<;K#9g`p
z2xAht$|f&HV9*o_TqhWK2%QLoqEe<6PY^#gKtDIXyTetdL4M;LfGe_-EyIPKohlK~
z;cSq?j7<@Ui?=&2jmuw>E;9&yn&J)PRZ)VV$JbofG$fpUl_>dAT=~a4G}i#@O(0MR
zpy~1WZ_5O719Uq@3FNwoDii7DZSQ*kz>ZPm3k2}UT6z$6rEuHj@qF$F0U_j<Na%9k
zP{%vBn?on}xB20DagbA<*u@-?gif*Hgc+UA@IXe{V}vbdF`_zl5Z~?$r>SbD*j#Xq
z&~$lS9Ullg(Y#^NL;on2BoOuHK2%`QO`~-qUI|c(=NN>F8R<;QVTGO&E{>sO7Y}?#
z&Sn|y2hcfq?YD|Q+TR~e*W^Gd9tBsB<D$<e3*X-f`0l*l|Kn~5*zzQ9b74#VZnAM*
z@Gq2-T*|&$u3||DQ1+cHHf)VOsF?{mbZ0o@>Th*yvxce~Hw&EC;MbTxVRavGu(Ia@
zhLG%%u{}@DD6S7ATOSYZTsSO#tX&+orBk*~IQx%e!G#7Y$h|HU>mQnCBQ4&`0`eMx
zvl7SlWeJbQjH|s>;;y@P|LwmY%;ePMi5eK>L{YLML@T^IJK%hGkApJ89;X6GS7NTx
zJdHPJM6+yaPzzC5Lkz#e{J<H27&M=6@s_KEO&ev=rH$@=Oyw|8zm|BkAqJZ*3Jk!Z
z{k6!mtlyXj!ZSAGgVLrx%+uY8uLhMP!_MHV0<I5{!aRJ)<2`g#$wLQmul{YJYKp2Y
z|1^O_I!(u21IDhBT=REmn4P^H(3iM%1}o@Nq-N&;ZqiK*p}B2)^;)gLWPjF(SbSGL
zR{68zb&j&%T#j^BRP(F%G$~PT;dG+sG;R~YfJ5R}fZNfF_TrMEALsw!w^;S^&6-&h
zE(7{Ugg54e`FVvM`?pQ2dK3L|Yw~EKCp*%S(%{T3gscxQBZD)xA_J)KHon;H<n^}^
z8ascP*p`y>Js%INQ#G$&tPe#RhYJE5Q3cqDKtsn?TG`REOzrTe=&M9S#0S)&cp=Ch
zaG}Txco9aivPg$wJ<%wJ9ZhR#2A^xtRPQzeMp}1{=0E$@!f+_@6Ak2&<h{Bum`IjO
zh)+x~I=W*1@CAmNY0bhoV^f}}p9j5m21q#1b^$>Agd9f01v86^`|-xdX!yPxSE3HE
z{qNQpc>g<V@0Cr}o7<gy1Lw&Or4mU3s>e^2RnO$+J-xnR)c{6F*NW1UX1^uGji&*)
zU+5z+roQ(>_$)m|9)w_?p1D2=;+tIrXaUnL?pwipH>>a19AznQ!VhYmE@6cHx2*yJ
zlaZ4EYZ8|A$PN#+u|ePiDHeI<&&=dzF1Bi;6|HzN1aVJ}y|eDFE!_q-7Fj2rSVGOu
zhiKl>D!`FxsTCW6ucuO{(ujm}_pCMEe^EilK;c`mTe$n~k7OUv7xN)RNsV$n{>NMS
z3TZMCw<1HQpC~k?JrBT>?JnEL{>aTG`{qxj_zXGFt+M@+Rly=;5HNT|4aDlK`{HHa
z2Iym>C|pVQ%8LAf;f)m)Xk^&9uvL*anDW>nm)N1T+hMv*b{E7)`AlRXG~IsvK(U1L
zw_hHIUCgNM$98A4jz++YK1KX%qLU`D3w1KoKxw!2`}*Z%-x<H>+&9iI3mmnDfPK#w
zB|mPhThX5$z?Zid#5|)l6#o3xV%YuLQtNI#H{qXp9~E=E`;q+d%u63uLYNpzHf3)<
zRci4dn<4C3&{+Cr1fhIpc0k*4x^K+$G+IHx4W^#X@2G8Kpe+wL*;Le5aPW+sssvXw
z2CYjlCqmQQ|9Wa91rGRU35phCQ`eI12<x5SA?m~XyV%>)+{yJW7_O{nnsv{^K7!Rp
zghYya)5zz#nrrR)K0g<5#Xmy1_IkhjCtrowcDx2?g_ru+#<KWF{O>Gs?|D2@?weEM
zXcP-RI6!=>zA%=&;f7A)+uFXV3gS0vj~e+dHdqitWd|J_(<GisBW}IP-_~r`rYzmf
z)ZfANteXWZW%i?TEa;bVrt8=5@gK~9q1B}8VGFsM5T!JILr-5)Z~MCWGy%u07giEo
zmMWy{%8JQ4)4$uyJZ&c}IUKLfm>a1)my-np3oH*>w9EA-M0-9aNf2=|v!DbB&yC<3
zSz?n}WF}UI`Px3VtSy!CXZruDZWz5a#9vDJx?aBQ%>bTLVlY~&jNBSYPi{LRc6`@3
zwMeY?;Sam-BF;L8TbAE(Yl>q}vWWn|YFXFa7-R%|GK<f<%qkZ|D=i_(Ee}gk<qEZv
z2-bW2MYfG%Zr-O7+K;W7b+}7FYAWO7$3^(Kg*Non>Z5j#&q;(Jeui6^)n_Y){CBmM
z<>OpyP89r8f;m65C~&qZFYhin%8ZnImjH&k@C)LF?IZ1ShD40tiUIJB@pkRx2gfiN
z*YyHJ!<+q>`{U)TMH5H(af{4QeGDjVGwtv{Mqlqk{xznSsX|}@sa)fN{FV$()G+<@
z`rrxIt*xCX$3Kw(pP8`=^Y}d6h&bGHDT(OjFD7!vkDt5NgF{|I%r>s+A!q8oyNVUp
z*Fi5Z{ceH)ova_M`O9;;2Y%dsMJLkus4yI2_{$D>PBI$=&O@Nerl`!L+=4F^72U&(
zP_gfBW5GmuJ|S(~P_cg@FCZ9|FF6~&Av&YCL9V|@J)-6j86h#E;QEt^xB-UuJga)`
z_y5(`GVT$5q4mRaq;qUn!B|Zc-fZk8k=Bc=+N{}tM%JcC7)vcn`%y<K{_0%(nw;Hj
z&6~14Ax)E&CLDW7vm1bff49fp+*W7vyANQ<J(@?%<_7>AX?!Cx<#_bS-DnEoNk<>L
zQNTjBkKIHgB!@4NL{(&$J^)L>RCf?3k@;KYn)0NrYOsbmT6TmYE=@ItgxB%Pm&9{d
zdjpLVsX)jf*tSxO?Fub4B2Uw*>oW!_32Q!I6@t2^ZV>9?Ik#vH6i{`DktnRsZponv
z9m*wE1Dsm40Crq3j&|i}R-ZCIS&7OCRwyA)k#bKh6Ke$`wu>YSsOb8b8+qqg52*@U
z9RnO2>IM)&o|r!g-r09g{>eZ9+CH044mqOrBvi0OGMMUp41imORus<FS}puU1H<kB
z?!-%^ezBa&8PQDVEqSdXgPs-y^mysHmpYQ}nk@=Q3{gnq`K7XNn$cD=!$0U!&>?><
zmBv{W51=_|88~9<R2T`K!8+8piBPlwoyzReeb;5{?|QW>oS%huGcZ+A2AD3frfm+s
zfgA(q&mAY9_3W%YpGAODBVaY_u<vM^0T{MsJn-3k@`L|OpF>EBp0C@tPIstp2j3qG
z$Qk)>ef{pkMo<f2nOuTdAiiWuuhe?`rHVg|2%wS2MORbR`%8}bg}ILz_7_Qk<WdX|
znPMjs>zB|RrDd`9`yJ6Thw_#sI9zVD_LH`)ZWiS&33b|+<D0!NItROPS<mJY=eLi>
z*Y6|}e24B+h+}JZ&-_`mqB9<Ed`~@L^`*~B1w#64kKCf&XWeke$gXyyE=S0(KASGw
zrs$WOl*VQXZm@VpJKVXqcRWB8y)XYbU+#8qGk{*4bQ3u#0CPUQ1Jt?nmPwi{jgI?)
zCu8F3$U2aWfcW_!tuM*H2Ja$;?B*3~oo~p&PG8YLA-=Z~aSwmTOcobaFKH#2vn2ez
zlk}1^AFKEq;H}##6qgMuHx6IytB#s+gc9z3L1i~}k2g5Or_xD&KI0{4mJJUOEB%W!
z*`OKFrw-p+vh>^gLYrFiVIKjJ6)D?hOG`Sz=S`zrW})+S;Et+#523+Y*K(#mmDU06
z_MQ!NoE_qY>B7(l)8vCJg#Jy<>nLvhdIuAoMQ5X$lDh}~PKf`}nfYJ$c2$opi`1)6
zrl+D#u;sDomsUd)xwp%RC`-l~Id0yQvT@jTua!#reyl_M@0$g#1LivW>qbv0j_Q7^
zJ_Oi7bXKEK;f~sKe{2pA$uCAy*c&Yn#$nzostsYSGLN3eH}9q_aKt@~_p67r@$Wq5
zzwLTFVBFVut!4u}t3E4Knm5gL?8fNm_H21$#OT+w6g_?`{@)UdKqJIl*#M_yOh<+z
zCa#+ko?H%HtPs|c^MyBvJ54wXTQG26z56I3T6Xu`$YT6+kVm&%@8bdMFEv>UK=e0(
z_4@9=Tf364PrQt(uYjh-7YPNc6MSe2&4K{l5>)B5Iarim&#hNWd&7J~d^b+jFe@b6
zs<@0ujK%ci{dw}D!RC{6oUZ*`JtT<TczgMG1G2NHGze$%8arsD{?u#K8os@W1|DQ)
z8kVLfgbXV966Z9Gy$q@G?Bn48Fzz?E6Pklo?S(I2Dx9ZgdV&Ggzm5FC-C?eTX`G)j
zJAhkmCuIz>4iCnb8PfGQKCZl>>3-c29z_u!>&uO^%}l;)62(>i%e_3PbzTO9U0gJO
zt^>GO$pLkv(1KaqaM-X?tF&ge5Ria|-5D8<`wT|{?u7045g_?65mMA3E1~+z6KT7E
z45k6jwG2NtqXKfTTM}_#=&{}3tG%hxlvMNDo6avWhXAg?tN+F9n~|$CD9tIDQZSIH
z$VY?XiSaqRgA)<sxy6u6I}G52cGzUcF7<AvzaIf+CTFfLz_xiE*Z^`2>3OaS;w%b$
z!Scup!b8eN3XC75kJVTkpaJPIXl1-rmZS`H5Tar+Y?l@xv;o){T8fCU?nX1=XT^Ly
z1qJ|398Q5iQ7-?MA<e?sj%;f1*|TSgS`)UiTiLYL$yz8+I@SThUup|aR`!b}Vf8jT
zE5tk==bo3FD{(HIk#so^4Q4cfcg_9af#OCUZtszx5}rWIoYEcVuEn|Z(K4Hb`o{Wz
zb0Hpt=*}V8ZYHe8bfY%zWaiZnP}+h~^OWkIpG&oW|Gn5WXTa%cPeYBUt-!wd%tgzu
z){Mun1?GU;z_(${zzAk>QLVaR|7gdL6I=3ZKW<qsp}{aJM&je1F1gnji|{6gsJ$sU
z*vnb<-GD)zQ%AeA--#_Uea*?M<j;%>P@v#t9RfWonPIPuNGCy~WIaJ2j>mMuN3;gv
zyzyqlW{wvHy>wi1QvW&#iT7MSd&<rbDlVywf%5IPW5|>Ce2SbEjWkt(k$wDG!F@?(
zCdkOWX9cV7iOe;PAnP@%$~KiXLn_XlY~>sAeDL(;w6YB28LEL*9St2F3U8Nhi+Os?
zrQX4Wt#t3G7HpdaE*BcnW@jpwbow&Owy(*{U!8ea*uAX^#-S(AANSeNd43Qo3Nr~a
z_~O^TwO#6foTk#SXT$V8!*6-)?r#E)kc77ch_lV;)bqu2BP$AT=bH40z8iZWMQQoX
zy4k0*`{e%XHMyngz7c&w<;6$G?O8M~@pt3e?-czXH^P4d5(HXdd~Nq+;-J?z>kTA`
z*p&oETmyA1zX5+g%ty7LaQ97>?Zg|>?l*4=DSS)}cnXcfCA-01!2E9wLr++N)vJmY
zgHO)LC#t;BeRb>yBw*X3b$hJ_s>5z`v#4KgrgM2?yCz_^gtdPsvxz^)2c@$AIRB%)
zQa;rGbAtNG^m8X<3Iw^%=ejp|MW)DP4{09CO<SKaAxJzD2K?PX#GV)OjTqW_IQ(RF
zFwO)k8AKHXs~e0m6#mnnOtRS}SOE@I5cA(jOmCYL9GcTBk>flMap~coD%4WmVLW3@
zzJ6Xpd}pgt0<Od*=iUj=k>%?%a-cb>c_z&FicvNzf`_|4Cw<S1dXu0Z=@U^1)ho23
zaLKL$gGeTv*1)&-LUE=krEm)?ie=`_S8^0mU>Zh=%<q4;2ggDJ%5?lrO*A@0pnW_A
zklfpyDms|s0u9CLA7fDnz24(n7xc0+WHN^yZGHPhJl**Q`__Rw9+!=QjqoIR1~p)8
zpJc9sgd$Zl=Lm%vzf??KcBw-HKlC7C$cXB0fZx2hBiuWayahCqD#{djZub{6#4h^z
zhnXJ_s}Ycr9=PlU=gbrGjg%216V8Ix@)Z(b0JFu26(3Z6V{7yxh;jJ{XnAXDM@85;
zjSIlw?%?Y2=lWmpR3+}+7F>on%02AwuZLCm9hbz+jGl(56{|B7PsBz*`DXQjtqv(B
zT5!=V!F_j9wmYBWQa<(jd$17V5JHgbbL%8&7PWV@=t4n^AM<?C(ioucqiDv@XPsR2
z{YKp5^Vj>L0b5}I`u9Ov3!ZfxJ_jlDnSN`p(sf2+aTSOkZJ82&(YKe#WbZb75VW0k
zT8*U_Eo*EqpQ2HkB;oe|j19MbKgzQeN@*P$OO=+lj$T9_*Uqv?WBHQ{6BgkKZd{eX
zYSdQITW}P-Ew7J4M{R@zlwy-U_^ZZx;SpF<e|LF-DlCC5C*VgO`=^+&&v3z`{zCMs
zZh}CLLkDq0)zGt=Ku}ybbt#zuT7_vmy7-Pe)6*ZCi@`T2v6Z)N51%>GJE&3`BU_+M
zlYZ^JAFF}&R-xk@a^X-)96K!~^lYzG=TIe9=$TqbxifWk-LM@=xR_s;vjUVoNm8TL
z0m-&ewepw+2~4Bw0*^<RRFcj%=_qxqbwZ0&05th;q%*n0MN93g66*CmTj?u1UeJE;
zK_<IVq2mA3=X)+CG-;z9UwELWKWJ_Y^?>@_P3P_EW~h=s_D#>ZuSze>CG^i#y}C#8
zr%PBbPbu_xq(ODTZ_u5F`8ByO-g*{CE;=<y@OMFD$q$HPzs<APPq5TwQiNGh@L5j|
zH~hWFTcp~ryF{J#i#kXsOMI2xe1EwRlks|x4i-OfcNiIO2-wZvfvZ2pJ3PQqF7J@W
z)HM1<BYkh?Uh02rcPKunB>rpN4Q0I(rw=QU%-*dY8J%u8>k_^aKHoT(FC+v)puY9s
z!~rsdIKtcSp<@(8p5JPZ%Zq@+WyCuGGB?|cN_v0b>Crp)n5>F)s=RPnp5P4EyV=|}
zhJ9YPkoc{Tkrzu~P5V;Hvz~vtwq9^SK+B=UcJ{(;YAW8@8-vf?Hf%BC>LdM#w>)*z
zD7MVZV3PRs{Lrm~lk!`Sow%&oAC-oha~j7!v4M0e=t79VJ9P|nO>V>y8k1jUUA@Mh
z-P|kKA{=lKG8KXtNS+_v!_MHEX6IO=%|$*A+XO4_lkat|To^N0;JieQe5Q~~-yn{J
z69zcB6vdO+{UOGcx;WABoBPB%5*#Z8<bD-Z`t+1SMCQg7@^x^^r7v^!0h=)?D#k)t
zbiB=e#;W60sjv)8a^ej<4ly@^2v?h*lNjiI&VCm*&*9&6E+$xF=XY$wqWr}gUn<K8
zN4)U^5{to7ZCfFf6S`+#Rh@gNueZ&DadUM?V%lqXgCx%a39Rvb6)%Sdm9r;QOqk+h
z1eaz-JYXpl_1)Zbfb3l<)|_?Bqi(RZrh(#P&sT17pYxYgB{;wcc+NYhK{Et*k9y(>
z0}=5&q@Wre;q^d&8sxlka&|^8AwX2kE>`vYxI4XvoW`!In*Hu{{|5=8N&#S~<hl;)
z-rfCOFw2I7`S{~*e%}F)O3@*ODBz}<$*%9bc(+N!gC|wF1_tcyUr<~*JI~MdjSfdL
z_mhQObJC8Gl%uznzMa}=^PJ?c_#K)S1NgY?<$4`hLjwI}!2FTgbhCT1jG+6prs)0>
zuf|7zCdTpi<ltD;+kJw7YMY?z=SoqpIm-209WIvykNWnW9UMVNgFKw#V8->hRCf*c
z_vJ9-RxXvVo3FB0NeaPGGK$1X6Ti_tEB8GMPfLoML+HU!@?j)wwL4JKXYK^{P<)uU
zkK5>7>r&k_z{$eFD{jAbjH|KC;GX7b&@UiC8}%z5oo~a6YTj_`Zfv<qEwhcWuxDPB
z%HB|5S#*k!_WK49KY4d<L4JEKF-al<Yv`^J9T<)<gN+1ODyT=$`kLVQjl!(>k!~<=
z9-zUn1g3|~^K~v@8sQV8LNHaPU}isV4^<l}vnq6@Qgw~Kzm~m&NeC8rh@9X^M`M<w
z-JixEre8JWRkqP77?UEW0m`3+`8peoicMcY8b;{TrlK65D!$JodT<;S@7*gJ$&TKW
z#v9l2=fU}rFv#uOeK`ALQ6aon69rM_;X5bns)gS+VR5Rs27?cF4yah=$O7s#c5wB9
zC<{|AKxyg{Eg)*Do%BLF`xnhGEJ{FI<f6|rLg`;yPw5MTC_LU(OU#>HJ@!91{hZjv
zk6pNe)nzR^BdS1Xo7aQKpnU>+wK}=%TtT5L{W!j?D!!Av{qxuF)_*%&xRFiUSPMRE
z3E1jY+PpnH;X>7gsvVU^Pa1Xqy997TW+e?`eVr}L&K_XiWEqN?36ddQ8V@g&Wb>E&
z8#ab_M-U8_A-J8bb&Qr`LN`MObK2FINy|ajR!NHdcNhiceKX*Rn5}yysLmU|+s4<!
zR4*oqAt$r+B@tYHG9r7WnOkMo@Y_=joDou}OB9j%A@ZASqM}~jSf=vc$w8ech8VZm
zcl4NfgfC}lQjQTdV#ihBRnE`pCZ9Q%fB`=km@~UaANoV&47QY-5wd8rHw|;xhsh6_
z*DtqdaXF--PuHW=jn5J(@!uF2chu36tS-DX$UqD#$Re;t#Ajfi3X;_5YV0pOd^(jR
zL+CmWH7SBapGKmxM-E!61gD?bC=jSj0io5S;+AtSp$_{6w^3BuHDJGE3s_PL#%==?
zh8dzw?alM2eaXQRD*3xaMRdXP&hU3Wo;CGk52DGM;2i}%Rl$h>9h+mv?ROQ5$KcSZ
znh|rJUoAQUh$_lnr02cVt=m(rLw}c5%SGlG&<xV{DoLE0V}=(-r2B-xtwi6wk4Odl
zF5Obu>cu_s2<#g9{nP*Wnp{iXZGX{EYVnsPu%4yajL}zP*9bVsh^^;LxA4?5NBb6s
z3TNmq^#3i;un0a>cCR!Dkf9#e#7Zp(j*q(wpM~3x77-eMXLKq=d$z__akq$`eNF;G
zi1-+I|LX>fmGilom$v)a_80ApL!{g)ozh<`-s?e)7q+=yL(i<Gmv?2-a665b@9~mG
zr=w~}EvuGFIqtpB?h#r+{prk^ix4Yo0F{C-HFm`JH`d}e{A?};bI55=T54*XZ*rO5
zTcrET&;V~K%s%+)r@XT5f_=t@Yu(HgcIP9GxY=~BcB@A5TftJTsJQMf)lyvSx<|3w
zx075z;yd4}^x$!TnCNLuy6ygS`A}z1we}Na0JO-C#hy(Z3Ff4Py?E9Yt#@y2|MDqG
zQJh{fU5?ovHr`_I&ADG05c$UyTfo~|R`Q{`d41S0wEzk6jJf81LlX_gL<YLG738!R
z1)nk^fv9s#(eXc+q7&5HP_G6*UhmbD0~AcIyc*g2q=ug*<a&$q?&W^#I1SR4{dEtp
zn_6uAv0-nqei!HeDx-mFIw%qQX&0HGrA@4^#Z+BCJ4W95P5WW>*)TS-0Z0fj99{4{
zBEp`wTz-s1G?$E9?UnzAwja#--rJ;%A}1Z+Y-hbcoqH`p!!IV$n4k2woHZ$1rh#7G
z+B@iUsnF*8vfG^WaJwE}D_Griu34LMt^&JKJRC69+yG29Q9vu863NM2n-ju7y`}*M
zc1>3;<A>|_d8p3ODA9Y!TC}s8;E*+IenH;>X@7{qG@O(UaMj!^f(XPA=6TuuQFqWo
zrlcizrDLp!%kcdZb?-ehn*#`;@8^1d{&0Klgs8H+B%0OyKX3Lo=rHKx!&849sc3c7
zb`}dHPXs8%ahFx~3~_%Yuuj5q;6^5+LOyD{JL>9Y@VZ)EyoJ3(6}FrnCjHgqks;Dx
zQJGO@vET4Z*BBMo=a)Z;;Op)WbCx`*=8J6PL0c8ek8JH(0X{oYcm9J{yF!Ns0e>XQ
zobUT0_6~I(EFArBEbUb9xP!LZO^)Uyj0X-;zldm>ZvPn>t2$`0%hL4SD`h%ZiX*OY
zSNL6IjW{yknT6Fl^sRi^x!cfgarygqWvltCZdgWug#v4uF<@zCpDY0>HmSiJ>{m0$
zph*A5Iy49D^V+RTIo+A^OEp#%@~irGSI^Z&asg2|*L1E%(zSZH6Fz~pP(`F!>8V8`
zs)~IW@1fcsJhRM4B@beZ8+)Gt!e0)e(N?M?_RDP(Y6at#5>(H3f834s>o2|@A-F@k
z=c>i%mXN;{-e(DH#dnyv`e1vxUu_*SO$Xha&=v@?^e)S36CUe;)9c%|PbC~HtTQd#
zFd13+`M1VUx+!~v<#E&X&uv;6CC@r~Usq}ussh(~au%ui{o}=vY|>^p?LbdF0B+3!
zI9d7XGaEKnw6x`i%SIr|hbjLv9S2GLFF-XPpI1P0T4@dZwytJSi6{PQ0t@th@Xq5(
z1$W_Ha)!0-)lSk5D&*ut&7Jao1IIy$K9_v5$ENViLeZ+(Z|A=@f19cgs{5&`2RrUB
z^~uG&oMIikv4s3v#!nW@_39Om7CvJTMAPA)Z2}Cmx+kB8Cp`f`SA;odRI35fyU7`e
zR>)XdqSB|H<f~_Zo>Y!PyTJGja$3NFdqB(Gkj3gwzFPNl#UYK$gAOQV%foYFhCXF-
zd#*UIU(|CI;dJ>fjO$21H~a6V*5RMJ^QTFUZ)e|4B5hbFlH`YwDC6@LBzbeQ=uE+O
zgKLvuroc>Yzb2$eD`jjwSPx;1g513j1WO~JH(u!?f-Y41JBpqm;B?S7_^Py-g^JP<
z1~{scwI~=jYvCHZk*#xqr*KN#9R5hDQFLq);j@=opKh3(=&^GU$kvhEo~-Tkb6;cd
z`6lY0eS?DV(Cg%V%{J2O&0W9BU1;fC3`79YcpM#*6Rbo5Q@D%Yg-w6!HK%UGGs}t1
zG-YmeFZDS4ZRSDzT;ikD5JiBVjOv{7pUX@&n`Je9Tf_6;=MF1gP$ccdH(GC0k_^e(
z4Pp&o+tt*+s}HuXUmO*Vuo@XY#GA}E$*WpGbYyectS8QDC{AG7VlACx2HHV3qf~T0
z6{I(-Egg@^OTX3~#Hamlz>cCrgT2sJA@RQ5Ioz!JepFoM{TF<|X6u+as-2MtD9<*w
zV2+zV)?M%$fgN&IenvAFgnD+rO)dv^+V<Fzof*WZL-}67*1?ke{!lH#zwi2Mkh;qU
z(ya2>j?uKw>lO53ur2vqpe?KZl?TDj|AKpvi~@h7>--ig$g=yMv#P<?G4+M+pRWoe
z>t=D!o~xQ7j(m1pYQ&{#&Rxm(Y(FlR8Pm&xeyF4Xdu*6kWtv*dlH&s187_kfIqYL*
z`MlqLKD6oLeD=(3f7~prr*hmUGDDjCVJcP9wQDYHfl>cLc_oe8zQEAy)Y#SIh457q
zTlb^FZ~Qw6aQpm+>A?pm|4N4y?1ssh4Rwp~P(Kwp4e(F<eS5z9KN$P!s3^2|ZAt-=
zmQZ3SK|w&2j-draKm-KomXZePlx`4^AqDB~?(T*`x?4IWzCE6M&iUPY*In!TQ<sZ{
zGkd@LeV_M<sDe1!ugOkZPxU?~iRcH86-V+RUZCo2UDKa+QZHKN=57n?FK5*3FWWmG
zFFTy2_Q4W&!rV2^k4}OcV&Q$82AFiwbz?79s~|t+vxBWL@7Wi)fQ8Kt*yL2=Q1Cu$
zt}VI%_I8zE+gw?6p!^BU{g-=!O<QA2%jTsvn%u1oi7&1u@AYO*W&U=vhhIv>NI|W)
zFlx73r*|6(p{#l~K7Uu~1&loMKR4k!@_5AJ2o8H)oF^Yu5|ZneVh3bP<P~omvRGYF
zuE(Q8oyu3pmp!7nq^l2bg-;HI6(Q~}%QZQLDp%Rwtf}h992RE4nf7u?Z-I$|*$cvf
zH?Elc{)C8RGR$V7mph51CRJQ22qzeY9e-8#-En#t112}hd$%9&v^)rM8_Ya}lO1**
z<%%-|>;`7~xTId5FLnBa@WZPxM>c)6)54O?IeEoSi~~f+lqol}A&nSAOi5Q`=(O$G
z^w4Acd_M{!sv-UrG#FZJ<n<C7EVAAXYW4SRPietl;AW_p(IQ?fRC2$G(_)gAP5>9-
zx6(6D-(aw!F(xxBF%T}fe=6r&r)OG7Qc+zPJF9wI?JlTqv%cTSHm;1Aoz9~JghuN{
zKl3>k+C<tNOzw2f+V9X<>}NOfj62y+Q5C!6*&kG`Uc_TvcA&v>ldnLm>#tldx9IOP
z3|HkBzznGTBHU_kTI;Sx@t6S<Oz3FXBX_5>u;-U10vl<+U$c8=u!iwLzc;Or!`|>I
zUHBV%U*~r_1MJ}RRPe4jnU<JL{<!jQ4x3hHomVhotT+voP>by^t^M++OM~q}UE6)3
zuCC4eL9|0VE;q}b&zmLa+PjOmDDR`eZlvD6dHv|*X7scj>+^#*6^47Q`P&nX(q=vr
z3eU5=z9J5Olf3g!S!c6&DxS{+{A6rqw&#M`YA@UX8)j$IQp((YXK#~DjeNe94Vq5J
z6e)x%3wZLy!02e05*VW(%%P$^3ILdpKAHT9f;S)5_j|ff0wjq1RPpwm((ql<Fy#E1
z>X&HyD`N99bh>h%MK!zi-qtfL&ov+vEb~M~XCd@lLvS*JSZa)la|@rK^y{G;21GvK
zyV*?pG_un}Z8Dj!G|9`gt=nng5@`VYc&nv{lgaNK84`a$p+WNZ7Y;DXZoDinG?+ZH
zE(W`geeAGb<q}$7m^r#Sl1A2MD{J^rY$!1fGDnw>49|Ehwt(F#NlkX)T9%60N=#Ov
zN=SAy+2^H;w`BF!S*-E82c!HTr`$ax+M8IO<>aBXir|6Ovm?gJETd||FGBQbcRH~~
z+c-X$3`wYb!JuXnT3Mud@6^%u^e!2%*dIS&B{Imk7|G;V=JS<y3C_#&Na<oYX|Ke|
z0WGfH<s(1y5e7}3A^gJXoQ5r<U^#V&`?xZ*Ep|&e46^v(?Gr07#Rbd@B{%$gXB9C3
z7VzOk@GA9Cc=W8{do3FTV=5i6kNVaI<KxLk+0A#1;@KC`r?)+56}<YRaOdk+0`gWU
z<KEDJSnLr?`M^XjWAjUMs*zpmh38tCP)heNjDwq^v)u|3b7qH5J(T`C*8`OYqa7s2
zx%UnBSoq*i<CBxVK7ty2OZ?puC#_Mzh;Q`GHUw7wjL`q-?!@*dkazYP6ZODrZ6HCn
zaMtqcwFQ1HMtj3CfFI4Q^tpSC%FCl3q|Le0<H?#2%Ebmx0zcXpYCG}ASopGxUxaMt
zka9op)tWZPvd(=7|E6eUZDaW6Md%liiVbHOZkTcZ55cW^p%h0ed^Fg`Mvlm~6v4Ah
z)@YfODL~E32gZS`s{IJlI|SFYS{Lii$y9wTg{|$c0nu}rB<}G6zMCHQANJfen-i8_
zqE-$|3rg8&JB(EbJ{b@g0Tj4=1tW7mPA5)7);4$)xCodRHe7DqYeyXB53zpBFt^9b
z?NM%LA-KB(+mJU(^`B>Za0H;gOO2bol6QZg4oH7e${q#W9xuUpi>k%Droh4Q#W+Gl
z$?f%2>qo9yr8)ORaN~GiH2B%jSx?~iL0JNb?k^^(gQ?_+cqtj}^Dt0kmX#h&3;aHo
z9;H0rXsB2q5r~%>?=4q&xl`pZ{p+)8@7?@JURJl?b+e~#I@bHviT(`#yz%6otD!<f
zv_s^i#QdD<SJ96KuTOTg8t=2MIST4IAWd-<V_o3DtA=cvxjYn_#GmOM_lPp3zTU{_
ze6{F@9U*7`RV}#=1G>bL1=X`6DzbT6=rYU(xx&K`KJN-sKa_{W>we|RdMA6=ZAIDD
zWrMaSC={Anot6Bg_sFNEml2k7Xf)iDFW%1Fpvr!5j4a9PPow(W-pq>M{$hP{@*6R=
z>`hrP{!UMJ%|><*q(vqMTR?9rTMlQuHB8!WT-mj}ZGD<FEw=51LvFksoe%xi9>>BL
z0+1BQaP`+G;Iuy-CR-o1|GT9j_q-?Ggvl=LTUv((Yd7$HJNj|bl_1v9Q{`xf5DOy0
z%wld(VE`rtM<zngj4KK5YX<VgW0mG<jc5$^U9!(bWbRRqFR!i|lXsgWPO&YDmMh_R
zV#A*-YslWcaUAptMJ%h!HT6c@-ur(kKC$9bS?1Ir_+7Fiq)_1yB4qfX_h(^kVtz%l
z?9(_jL4T?~1LGfHmUj6%d|o(TiVLYP(GE!ZouIcnu^9~q%?YEI5Fuo-O_qDdUm>pz
zu>69!c9%N~S-$CB*AJAPdX{~8pem=UBCEw1FJuEuX2|&Fq9vff?g=43=p|G|)5t9P
z+Y8`%1{jaw5EgFU&b!UaB`*ug_g>wGZ%s~H0;yrD-sk}Yr~3I$3?~;wtVreyO!k&E
z;Iml)ylAE<op*Hd)hc6nKao6B(T2F}djjlY&=(hyw8fOGaNlzcK8jo1Upf2aqlt|h
z%ruk2O&$k-B})OPB)?*w?jT-ZJF_i`P_Z2y;pb9rF%OMf1rA**;X<W99wz{41cH-J
zZw`$AVufQE!xYgx4VO>j;Sgtygs;im&R&W8$84){Juf<Ue(rMF&vY=*J3^E9<20^%
zAhA4|4FA=cBj)5BzdvXB5#A%hf`^MB3>x2H0jmj1Xb7BMcPhR%!h^g1qeL}NJCPF)
z{R@SHC?i4|j?4D3^wVCcLu%K)U$g^g{r$<RoHRed?_p(E-E6tMst52PLb)JH3M+M0
zxs8~IOgQfCy|~jaFxz95%8;1fYrlA}Wk=e1b6oy>4gWXWD}U-(;=$Ad1-DfsTTEyT
zW?-iHex@As66G6|hVPTvWKf{Wu_~+U1k!3UlwduRRm*HEx4|VnQwjU9yWk`owLa9+
zX(-GRC31v_Vd(3x`lC2L1mOJ#_ZDiW2Drjwy=s08b~5vd5W9iAhzGYiGo)nv8|jbb
z^dFyf0a)msGCc}5R?X3m+7YmASXq=d+xfwEXKzI?w4wRY`0{SXlm_d(frb0Yd*Mca
z>*{!2pE|c~jTglBos_N1Ruqq76|%as@k!Js)Ln+rhRkXl8f-6{-$#oZ2&l2fWXJmW
zE>~7#+iofUeWBDT>jsI0`3KVIT_RBc(b9&#6mh#(d#~PYw0MfnQOh~eA&A4QS~Znf
zj5`|xrJo*M%P_yj{M#7m?*pvUDR=Xp`1VLYQ?t)m`3>0a=N~z*{VaBOYfJcuRwc|G
zkL@PPG%Ce30P=G~#8WVU2J4V{gaBV)sOt!9=WOm!GW6GZDNd<O<s8*8bAJI1-m4`I
zdwB6O@Lufg{xIcDB%Bg33C~tw*aj{GpXv@e?iSN)WymI)6xJOE3S8~CuF`y3SyzKu
zv7y2K0!7VY_H$>r)E#57ulKRAqfYleYwGu_>W4Lx@ooM75*)o8&ZS)$LyGG!HYj-O
zlBB$f4N$m#oo$HmZ`*lR<o#%T)~Wuz*|(0`ucB)zRXYoIjwZ#x?v;FbW$CZo1>&n@
zf}kh`3)#FCI<K{~a$_94T%@%eUiVUK0_#!a4eC<K6;>Q^{+PzYu2;_-ySl}pE)j5L
z*9%5SIiBun+U3Tuah}}*I5%CHmzLCcy7=YC-9D_)mA90(hoU6w+Ryl)Fw>0;FWIBl
z(e3t<MR}>@s&8)Ex>+HYaO67_Y{4IjI?8JFGl?XRi?Qe~F`QWD?u^is7|RALY4OgI
z_Itl~v%<07qSu*pQAE`e+Y`}qx28GoIAI4mQnE?qJeOjtgKq6uY)&|+rpW&9h`~`k
zoIR@kkE1I7=cv+t7)FEDoNP@OI8)+D5=~W-7dD=QOjmxLZo&`U0*L;&B|1dNW|@sk
z)w>2es=WSpzjp0)dx^x6?WfurjjH*U*~cvI$1lh@bQGKUG{-U6uYCC2H$$&%x|mN}
z_C4>cCl=1!lsV^{r>zAzb&1Bp+bjfotg<yKJx_SHzkhWpOT}P+!$|M_<{G$hx(rV7
z*l3KUfVFrhmsN1oN<N=#`NL%ilW%1wu(FD|%$ew|zsm0Dr)h5>AoPynZM>>$(rh@@
z{_W^yRU2BS;VeCE*@!h#9jSMd@L}_tY@!o(l=Cx#?<ZXq{6Bl>hk#3IC-+h&|KWt1
z#T0nkdkAZ~z6s8|G?W|1KK2z3y}0&V_gn5h$1wYTHD4dA2GB!-c80y%N%PCzM&d_U
z$5hm1M#D6n)<eSRQ&$&PZ0ms$L^3JEIlz9%7_19w5t_73dEt`bfp2WBO*t>X3xa*L
zMp_61`EaGdWJhTl)<5TMGP+GyYMfc5-8JW3^*|8YI5iLS!W$X&?vEQd@If{JVQXRy
zYUv?})2sElu4BR|PT<vE&F33)ix5ghW0ek{7yUIqIcn9Dvl4^|X_sC9?vk}vMN$Is
zjo_Y!`c-cCNb3C0WpG@Oz7UhmAghYC#lrodjTob^AvI6Ayk;!X{l(eo8Jt!#!ot0w
z`CCk^dM3sV8=FD%RIYE+dB&6l#+4~Hv%AyD%g)2&8#<=6F1?qODV#2zFjR1pu;qB)
zzQ7EKRbtn%HXeENM(=Pmj3kI9sjf+wRf{+vDN&<p(TZX<wy@)TJ7eU<tBvK}Xx5E=
zQ02ASBS{2o0(Ns4v^tK*ea&T=FN|>x_!Wcf2nKn9wpQ~*l&jrS4$9M#oeWt-dzQ`w
zI)z#1^_TzA7zK(VI)#oB6%-i&;^i6SR&ZLi;2K17nas3R%vya`Ga!=n#pZkYCSJGj
zzDr54mYt$L=p3d1#%Q?Le_AZSIYs+nnzivtSNVR`%quza#JMHj@+~;P=S_BDmD09R
zhHpx2_5>Nq*FWbx@-z$@uPC#r7$A7u06R{<K1jDY%2c<k8)utTi&CnkuXw5lt^B;F
zCRri$5Q5z1;D~Ru#q~)qk!!ZK!F8Q2p39Umy+^W+aNsPtQ74(-QRZQ?NFbPE+AN$$
zK24;EpJGJtEZQLj7~aV3@KQkYvXrJPcuye{SxTMyeX$~6q5`Rg#`5X|Pk2g`i+_i1
zS&9+s!(5sxM>}dic(3j5HlkXD^l_MV+FYvbDs3^;h;(uTuAO6r$yxsBH_%px^nO<W
z(EMuA7lYSvjip^4&dJ5=x7Vt$qawny(p~$x7Zh4<6Ly@2ZB@4FS<7fJk*u%dR*FLr
z5|qYjX2s$)SXi|umbPyVBmv5_X0w*gpmKIHDVcG%dn>T?mUX?{63|vYOrK7r<-{0f
z(h%@?(tDOJ#2N^+WJ}G`+n>$h(pB?l9_)AGR#aoHJ;*Dsiw2|49EP*o-{&^Pgz9ZO
z2D=}c;wxWM^Qr~v{XU2rl7*4uFI)(cuV4a_nR;=ut#IP`XyBO;e`>;^63&#Ogw@zP
zR|i}xSKSo`O4JLy<BhSLhO7Mui<zZCULpxs-SokAcWv5C7VCDq=y(S)w<-D4T$izq
ziu7MmRjG6h%_|1F9t88`w&(2xtT@g)tZxJGvn~#Le3Bf-Q$o5~SHvF_mcN`JM*@EQ
zp&uy^R9l|ouWEZxP|F@^K0eOw<+puVoUOS*-w+&!4h>Xx?U&%`_Sw(mN(3Ygw4lfv
z1ZL6T3U&PZKiPzzMTM3i6op0C7Smt3smEc$;sOYW2JvISxTf^e=}z3H18_<>9d@?J
z(BnPX;=<tTBR8^H?<=;_lla;9a5M^CpINQ4eB#q|I^W>J?*TH_pJDT1>1w-=0j5Wq
zXNL9&;{_L`bJQ?RxY1oV{^wIV86R2=D{KZ6^9`0`0zz211U{#^veSI#iW&32hR5gM
z6#n>vb(v_f+-fDr^v=XL@$0YIbz#%D47JkjY`4&njDe@l<O(<Bd=@pecJmkKiUN+u
z>m63Ycr5R?tJ63xPE;T~&sb4pN^wko61$duA+?W<+wXVgyzaEvb#J^f17S^lK>n63
zToO^|yywViutV}F<CC|LhY6vJp7Btss0fR2rS>l~veORl;weM9>){UfYxA76xmV~{
zPbo^bdw<|V{ARV*zR*T<mhhgCWpeWRU@(Bbf%>!G$>$<U=LA`<hzlFTpIK+gn4N6c
zrCP*&l^vg6PPml+ny8tkUtBj}j^{AW!v-*nwcO7FwOtESb9b?vcz%;C*R##u0QcXN
zd@4b*+Zo^oPed&j-S@WFn5@8dy5(dL-5!52kv7Q^V16@`Y&NufwgzViB{SH=B@a(i
zi@Uq?%=$=F`NJ;nR9G$5E0^nwO3tfX`HBgr-03Y-JK0!1$0bF+y>q6~gH&FC3yqr2
zH%w$rwIpZu*=*P2RMudht8tgI&O6U`U=Lp3xUR?fnQfi76M9E3tnS}w*##P8UBtSJ
zYN8;L{6uls09I*c;(j%q%~M;?{%MYm`K9{}O1wbS>~84`#}>uft*e=3x0?Lf_JP_h
zT<hXH)N9LU&`M*4k1B&Q6UFa8k^JI4{zV4)nK-$AlP8Ue;Cz&5=T)A_PFUri(CYUM
zm2Z|grN$p$lm1>Qzg>Yh-t04z-5e~CYw*MM^6T^V?sn4U5IMRK5^l_tm1<OOhjgpc
z4<u<b@FylJr|z@<-35El7LEM5$~!_Y>D8|yv6O}Mi7qWssKWNYLt?q_TcWlxgyZn>
zb{ytMb9ID<I25>0zX$(fOc}b`7dFZNLm69jJo7Ot$3ZX2W>&jT<5GYgB@8Ry#MitA
z=eR{2YUO%JlM_wB96VW-#VPnfzPXV%$~(Yxq->=4E%P(m(FPAiCc%p#nv#zaSjZ#h
z%qSn2sSZNbwgZnMv4o7P*;)J5I90`X#wKC@O0KIvaO}nzxr&b;_w=hkHy%hd>@eKL
z+E!3j-k=w8y>T%q2b7!JR$PPIRQByPEoo$A&`Y&w9P;q~)AlzVz5-8CH3pXvyvu&>
z`g3(pd$8dkOox>}>bHjy7-YZr=)G{mT#1V{yTYAk9>`anLNr!$ps{LA<=n1_qi7t(
z<iUN8H9Ldr>#kkN?`}h_kdteI_K4nIGOYDeqg}~77p{v_vcQ4(uQM^;i!9Vl-OkeA
znY9UG89%T*4St@@tAP8*u-rp6v@%HTes#TG=r{YZpkJ9`8IY1pwtvM)Q$5`veM;R`
z=V141Fy3WEY9)!h@Mt;K!bd}`LTU_grNmRd3!No~mJWdJvRvT0d=`RuSinOr9WCb@
zxZF%Nx4Gu+#~Rj8G%<fLG(XMaI{1Wfe$exY?Tz{N10n^J0g9KqsMSklLWjYEoV^<D
zeGf}_Qx$FuUuF*Wa6BNL1<te+S@LP*y~ya~+Jd)vgYRwL5J9W9;^gvWXLkxDq~otn
zkUvkoyuI2UsfkU{%6x-fyYVyKeRnYN<>ZZJmyf|j^(>3pPaTFG<yR1<hKoIn+Do~h
zTfzIcz{GhP*gWC+{RG|JZdO`o&#=;33V>%O>YdMitN=j$BNp+#`gd;QiM+Uzt#Kwy
zr@i_!Gh=%e!PO4r3FF-)aiUANHf8(SgId-XYaR15CIC&4+{whf@(zjPbc$%`oyObq
zeR;d?Xg}m}9s5~oaPL2c1ti4!cNmA=tQj2_$@8z!kE<ng2@fvj!%QDJsuC%{m2epv
zCBO|s(rW)My5dI(K~vkwygU!wrgB#uACBj^idL@PZ;oZNGs17r?Kdg4zerJ_G|4JV
z*vOI~D<tG=P`45+fV^2H^i*otFnd5~xJW>sQ+JF{Kj4F*=(5pELIw?9Ch<B{@FvSl
zC}UI6`F4+|%Kv*b#C1F~2JDZ|btL3m+n;9;lbk$$)M>Xt+a@^_ftBdOkpXoTvj^O$
zjvQR8R~!xuNthsjr-<F+i&vpE5dw@+I_@|x`@<OhGs&mCL#%{ZW=>kQviC|lkr9U-
z?zy(QPC0mxrI8tW@^g`EqHL<Au`33(EpAgG6CU1NNLI{(g^!qhQxq}S2VIZzupK^j
zi{a-NeJWMjO{gCzUc-);j;`2wZ8N`p_A`a`ZhDQl9oYO#oh2D47&1Fn#)_}zJQ2X{
z`ZcqynX8l*)1M?beF=5Ux+ceXE=^$nX)1MZyyCTY22a7xL`#G)KrRz^UU?Vec^dYn
z6%=RePQKCgJ8m4K$GLsZfa}d^>Z_sQc9too<-a0dDQ~k~eKzHpRT^VOE>Eokov)i?
z-Dx_srM5}!oi8?s2#xrT=Xuu#(5cHfoFSB<;`jL8-$}*SPX&VsV)LXrYHtePn${0<
zyfEKj;yljYP75}itq_3MobYlFhg?{uxZcb#^VnnCgYDIi-U8Y|t@7A0gP$2X3@O-W
zw`t?1BRRp?cRG1xC@-=@T=wQU52nNJosVIhCmHWf%oYUi{=UU2F<x#cF8|EWTYP=K
zI(<5<*X!3+G<(wt|HY7?Wr!1`poorwy=^gHoS*1=AvIBD%yQr}+3^KJKNY>MZXwvO
zd3eBz{fNEj_g<qQ`iM@eF9KkVO;bzqX1cw$s+sN^nfNe;$qwLslUPmn(~%aP(vR2r
z2&hQ-7NA6JI09~hYUwPHD3%ABOvyMGg|o-8wIqxrqk5Bx2=KFW8FY%pz@v&=LYd+&
zgU0k=O+n)ad1(!DPs7=X?Wt8h_2`w~E-fGJ518DaF`gY-y6rUW+;Xkl^rkR<>Z&@6
zk$5<``5$)*&y!Iy<srp_Hu|G4AFp>rT5a4e%UraZA1~c^>aavz;I{m}oEp|8;ex#C
ztwu$`gXf|bf$QXeY2`YeO2j41Yu#4upGM3>w_*sU34=6oR!B(q;Nu&=EqRE}v+qzI
zBMy@BRzKJylAN%?cUV3#O6(t0iRt#!85)Vb1nv?E@FKUhY@`t)m8n`LC^VFvXSuoE
zXun2-)FZ7jC7SymDrlu%7~zcMn$Ivcx^$CZ*&pgmK60$|!-;TLPv&_+|H@Hx@?6Xl
ztZxq~j6Bk80i`9~dCz6--&gXj72xw`EEV+)mc5>J@)R4L+2S(#TDSP^g`+N2{~H<m
zh_15}t8*rwH}sz083~OTm!MWx?<$pQKdetAej4fPKJ7o#nr_=9WTH8_*I#_gi~h4&
z07quca?-DIx^$kuRPiVqKh$H1JB2>iUgl~_$>3KXk?1`WxHX6ASdAWcN%p4_rk<Vd
zWwW(5HcUr@#ZD&a^1ORCUR5s`oLrA*e&>A+J{)H{gZqEB#&jWi&&`UzLi0;L?L9@l
z{p{WD1Vfoi^Xy_BF4!8yq?G-GCI37rPurtjFGoiumns_kEVV-5i)(<pU3Y16+MYQN
zAHr>TX0rYb5m$q7NJ@B|0HS3$?RVhhiMX#J$e}(zZMU)Wq)IzZn-}ChwN0i4sg6C4
zRo4a9WNwdMe*UG-cRVC|eU|nh;_!39)P++o`{0ec)0RzR<z7bN`Cg;6vzCs)&(k*~
z?FnuPCwkU1%K+yI->P5!_f(|^{Q)2;!^s~O#5K||9%3XQu^c>n-ioVzp8=^^R!i?3
zIoyJ7iRj5)z6*S7d_OP}V9k3Jvm6i@4dZ9JZDgDT1RXIk+#DM)#DFC!uW<CG&a$rg
zIYDc>`i6g0DCYpgceBo4_(58O30=pb%XcGj|HJ3h%#uO$jr4&sL3k-;aOzm?^*cgZ
zr$aJ3@MDpM!Qx|))=+F}+zVr5h07rsz;JiH7_#0RB<_-SwKWmt3P=N7_c5WYG}RRe
zBZH4`pqq*c_8lu?S>J2Q2>RoB&#mn+(O~F_LbU#`OZ!20vc~8#@$WHPz)-x8Vsm37
z7HPEhroJ)nTU}B+zKSXuXhD{0N&sJnIPBd~u12YS2^L~g!B9IyezKwi))cM-3MRSP
zHAlOyTeI&At$Ovn$UmbkcD3VY-NR6K+Du2}@z^qIxg7XhZ_2s9?wqqi`_crmH$GyM
zagGg<-u|AymF=8+3PIT}-_(iybh}P>zG|UWrZ(4D>=MApZ8H99cjr>WqKvov7~iF$
zY4G;<6pn*_^1Xt!&3~U8ZSg;?q**sFqxx0*U4D=1MK2PnB@BKH7BmVH^*dLKXwQ8K
zh4QY9hNbo+u<H-Aa5$}Sx}+hH!%oL&zsr4gU0D=2^%VvPC#I8bad&7K@T0hPxj}d9
zk;b_Qyrouqp6)gxkEQd5vIF*#_e8YPB^6~gXy6qlnl&O3CEyq&qD081yq1uLt~P0L
zq11Pvc=N-f{n%13%i5m$9a|-smBH<3MY$fD7HnkaIzI8+8f8@8$SBSU6TCQDE}|`i
zJ6f3}m95adIvU}3@SEfTdWFF-%X^Yvi@^2&Vv!^_`!{<9@G;3jd#T9%BdJId@dXhq
zqQPi9b47mTR&E%YmZ`jq8&*q0p9*$-nKQKWT8}Z2bFLqZOG*84Y}tKZ`10mEop!8L
zQ`a|s#1`_kxg}Muq}lb*S^Llyk>fcmc%tXIxylOQLJE9nT@krIM}z$_s6&sFN~EH|
zh(7x{GxvNmGkK?|6-OF;ae^p_jmnAp?B;6!zWaV3<FUJCBFxULmStM4Gb1ri$@~38
zi5pY=S#KSqRh!}k{+Z;zV=f+wQn=7d-oD*T%YaFt-aCcJ__#<EG&BL+cMe-KRLYl#
z+N_T8+ar?I+nNx8CbcFh5}hhm#}N*BD&=F!I7>=Q1EM$`a1O0@J<%b(kCfpb_Yr|6
z^s?Zyg&qe;Jjk?quNGQJmxCcs2?9;<L7+(ha=0v3b3ywP^b{dtVf`pU=2Py>E2<Pp
zcJz+(7ZX@w)ugl+4Py`D?x^h)Uec6?qf)rFWfcM|9+cqy=8kkWXil1)G4qH5I+>MS
zYcwX`(DBh<)Ek<iH!#Ih?`{IknE(8?-oHbe0NLiNH7_Kw{P%=P)pbr+f`qBSB>tXZ
z=-FL5uKZ(+ox@IdH2_dWrbW<)oIf$%9<$KTSDuHf8gfcb-k#*1O;uXpN+K@LdpZ>0
zI9+^m{X0|P9QcRtmh`fK?*VbEKp4tL;7ym~YY(hNej-8&LjSX@fy@5Z2|@S3-q{&m
z)D8N?jlTcLVE|W3j33E%32BICXUt23uQ`Q;!3$blr7K`xg*q}Sq3dq8qU72>+9PPI
zz~|Pd%u_b(*HNYpfIjlz2yO^xf6!&0($&3V1pQXkxI1Kl#O9SES^O3_=UwSD!9g(&
z$l~a?kD9))<ClNi=)5RLnqs$dQOUX@qSZd8d6(cZF4I-WI^?gwviknM-pQRO%&9RK
z5u*01Lozg!6!xo87H=OrE#Y6Zv*m>0x?bccxlS#U*GZn+z+?fH5>Vuf5ftA4$$HA6
zM)2i*x$0reJO~<-J&FuR?Lor~IU1nA%dcV60~P9=`$9`muOD=-+h+eeZ4nBdhZyaL
zB5mOktdNd5UKBQz15<+=mn6BWKU#(Vk}ND>e0}i;^p|Y1R;+@1-lg<l@qfyHwC;7}
zk<{S-aszLZBV)CntAQXS8omB%vsacSV5`;mEjf>PUcmf<X=vF-wSO<kG0cLTHBogb
z>#4bqixR`yhZF^k@}5=|8nWdoR_E%URYKJSSC76nQjI->U8z6w$9m!4gfT1j<D-X6
z6BEO!imv<SYOl({c{bF^9Aht4Nl+7!c@c*3%w*XP&4#Arsh`P<3_0n9Ltpb38E!Xq
z`|V-eTS3C<kBS8w4lVF56I*cDaW8S$kpl=2sQ-TdQT9Km|2Q6^y0pe?_Qxc2>qipt
zd1k<79kk4GP_}p7a>{?zM2vr$p^+Gj;65^xDKFHl?(D#J<>*}d4}#sl+dO0mZE<P@
zVuyZ*ya8)+$5L2U__z;(gPefWp^l+fg|fufrgt%vDzN}!5xhVw!a8l(ITlSnno^HW
zlV+Dz`GdlXx43FI-)1by*b9eb&;+e0_J`Gtx})~OLLU)y9EvlLS%F-H3uDc5meMSE
z3H>`t`t0~0;>JtG*zU+aNo-hANe#rWq3yorsbMxCmQ<H28!2gNBZxbVOp;=@Y)D2p
zBP@v@7Hb^@_d+1b_#G>48LA`MP)_S`aE=K(A4i03_2SlF`F})*g#wvJIH3`mhH(Wr
za{a&8O8?nP=P)6TT*2pt%RD}C<bv!_WKOEr8%D%w-=@9l|01Tqif6nw05cSnxwaY}
zF#nerjF8#=^MSnwB@wKYj68JSRB^Wie7wO(KxVN*YJ1p39~rb$wvW7@j;kdEZR;)|
zhvQjc4$+ob39K?y>K=&7<y{%j#z6V+;OmeshlO$`kvZbKEV<<PQ8vFaKcM;{k;ycW
zt{T`1u*dHgOukmtJ~X|IZ(dV{OS{UbUvOlEh~KCdnIk!-M+xd(87OG+?5D9>j&)dp
z&>x*QT2!JQi+7s%f3y~00fLGa$H8zyQ3NVL@Ifl|23Dt%h+u&nxjjoh`n>O=YCm|m
zo_wtW4_BK-iroC)|GO+hL@NuM-8ZS4c|#wS_Y;$Se1GkCB*lOG1pa;1Q|Ry!+C%%4
zhlDJlmR^0$FWo|oM@rpnEs@$#@pydL)KSHld3P-D3=Pv9b7Uw%&1k|Gq#@>>CSS}6
z7~C(6%&lZy@U0glg17w4YdCdvBO0yOj|ABTb@%0EPU3=W&lgw|*8(UZ9eqqOS5{_f
zx`LVu1c@YzDdehnTpm)Zaw<VBVbWM3%Xey)qM52P**3b`WilY0QG_V>57bfKa71gJ
zfWKe!C~z>(FvqFxV%5by+01R2kZ}FNy_1t&J;X?IQjMpoYqWd`$PkBwxWn|z65Mwl
zyjD*TQdr^|%KQf(P<#mhUxj{i2M%>}`Y5UQyJ#>Iha=bS|LoBIvktwD{8P#KhJtZo
zG1&R{)_EGw@yj;wE|*^PVYyn}Euvl$dqSzTnkH9=78zQFzC_wJ@D0k3YTbS^93g#B
zaz}}Pb^r~0x!A&fDU<4i?S;JgVF~9ifGNe%WCBOyLZtuBl6qIch0bKcQo~MT#?u07
zdC;W}nTV9tAaC5M6JkWe0s2l_+bC{T;^bu1qAL6id|*e;qv-wK=B=ylcxOg@E|~{Q
zLm8#XQ`y>aBh3T;E<XybExqLtUPY~9)t79?N_-Vj5~jN7umVpZn?}HQ3V)&l_)eJf
z{K++c^PS#eBaj4+y3KL3ZUlj;n9wL^D&%bSUtsMYIoV&^^=_KK76eqCUrb+B9nU~Q
zkzB1%j27{m?xf(ZdlKU`&sa)9YA_qAP8o1K<vj3$@Vv3L#9UWJ2F2AASnrRLkNI__
zkzOpZCV8(>C`z$xEi+Jhn`A+qY%fw>+C=8Qr$b%0XmjMWZpt?7<o(>#1@HGV(5b%H
zIXnO+)z;E-Ayt2wRJ$GPz5S479&X(Q5j3lj%E|r)PbTxDq`o`PEz7YK<IoW}s&eA}
zG#@g#(y`>-6{~vw{C#|(o&dazWs7Z1M4tg24HoM5g|O&r1-Lh5d&z@y|6C<=#8s-9
zJ=m(816S!8;wtUU{SN#;uM#`rD#5%_1A)>G2XS5=3Nq7_U=IEs?heHk)Y!+^7xwoo
z^^lZHu*UahWwsA6dEVjsB-RY3JW(?wXSbgwhtS30YjjZz;*vqtS2D-Rqe<LtFHYgC
z1y38kT!|VHNC#!P-8hv@&@`A>iGTTOHNJEw*gVi$8iJbUlWRb&s_2H?{+&dU?RiiR
zD_%AjuD^z<!^kf7hh0Rt>a&mWm;4vsxsGRucj3?5jtj9uY$eTTAwS6a(VgAQJ60B3
zASf~nf-O%|-Y$WC2|=SL^q*yB3JkHPHnDHbU9^4)FjJF1@7JF6KehSan)0XAdPa&N
zmY+D-RH64L6D$wZld&j;T%;HVkjPPYFgQW70d<bHXeLlyU_kv<SE(S?!_p+3uUUV2
zaZ@%6<on>-OLDcD6kJ?uzt&TfE-t$u81zoalh&uYFv-seqohc^-0D@%Ukm3Z`7JIU
z7!q{Nfxk)RU^EP?Lu#S-5GQT+h3Fwm%FcR#;jZm)X~Ca%l@>><LpA8|uF}1vtD<aR
z^xE?3igA(P!bg2fw1~Ft9YheYz{?uUS={_hOeo6W$4d&XF5Sv!%JGL;P<t_(PTbp9
zi(4CLFb|P{AVQ<pS9dRu|2_<!CcZ$Lm@|c4=-Z1>9OdK!?Gs~v`lE&iF8_Cdq4~SO
z+$Whqslw!+mS{88?z_W^1aUxK!t<|9A?-MS`pSwX;R(9a;es3s%Q?o<tG8+=N}|O}
z&$T=0odGhgc_>|Yq21Q;s3^iO=I41XP=l2RDB!5J_sH2_1$>11Po(pJU*9YbKj69q
zX6>RhZ2ctz7=5w&-=Qs2Mbf+U-)Un+_;?;hU<(LgNxCS>@o>=J75vGasq4*KzMST;
zv$gE~gn>(qOAPde2BH3I2^6aZn2;&s>IZ&;)%ug)-g9goq3E*E4DlZWa%sxvr}BtG
zo60qix%>CTuc(0dD;WzH4pn<Z?cD}v$#&E=`~R%HBmYI3?i*f&J!DC`4q9(|>Gp+K
zQ|OI(GrDJKlMfzKsF&*10!dr(c`r+Wr)>|uIsru0%*yq0rCqsPl<VUgH?fn4Hu%!P
zONxS71kMSBkbXaoaID0MH1(eER+dtx3t^f^5f<uNu1kT$kDu{Azej~0^_Jx%eNu%g
zg#_bpm}b~i7)7h;2`*dBN^U&i8!3J^tAvj8>;_G=wyxOsUaW0ISC5Ym<Xtxp41R6I
zMO{83^p-uME8JrG_%^+IGuMruVU^mKHUa)e1?Q3bzmO*(J23$3jvo{#7jk?A8}Vk)
z+-RQ7DsTV&sr~0`gs8lKp(aJB{cy`wz-X@Dm=aV6unDFJ*W$w_4i5>u={BY&bOBI1
zMt(6JPhtAH1cpqf$hsUvh3&xUR>UFGW6I>v<LYl@C?sChU4FcQt&nCCK{Sm73D?bl
z$LIuS2AMeaI)sy*nKL3wf#D8^2l{#ft}-!qjM#DpTkeqZRPDzmL3z;d4ro%J$fhhK
zfS83*iG7B!trP{fP4>^@%@U?>-R!;PDVErk506lqL!WTDUVW?F%EjTlR6mytgWUK}
z1cam|)Z9hsN0m)MhW&5$);K<*Iqt00te-$M$9F(;>@@pXEC283>Yp*;zdoL?EPqW4
zLe{obVkwJ`jb9r4*^eE6u`XevkvbqBwSWs1>S?x*UKAusCBV}r9v;d@!np*LVH*ov
z;Swhg^%#;a$AI>@r{J=UCda4oFhx``8G4`&&q}+3NxXf0Fcz!9>ryq;c@mA=C~`7P
z!c3>}UMG@m0|k&dK%)4NjEgZ510YkjY83;3AGx9_GL%^Umx!snxvrfzyCC4uSF<zp
z`y{BhZS633oKqi1qf_X8i6PIC?&Ypvl7+&*autNho}8eFPg1ETI-Tx-xHQ2V(i)cZ
zKu}c}*9OM36*|G_$+bO_9#n2L+>#-eY#!d*5kmISqW&bNKY4KQQr#F{^qm64I-dgN
z!Zh$8H$oV)+jWsG&V#5tF+Tep6}2Ld<WUwT6i?l~*^$181lP;sVh~C~>)3yvlAjxU
z8tjGg9go&dLD)5jiFRE(gNk?XAx2sXM1Npcw>@s!kLVAIK&6{>JsbZ&`-9ED`hz=k
z627MIs}_*<h1imFJ#ZR?%vvQ-w2;1w7sj~s5N#LhDPR?*yw$3yYOpQyKh&3);)kzD
zXspR%e_=!Mqdx843&QfWfZn1e(hYcr@?+S6s=d`yz#S6sR8{4wB#>T;@)9FzDN=?i
zh@B=GE~~{>qjOIE3`!46X@zKa2PCRo=A^S#KVYJfe)~8T*7gK9Uw26z&nQduzDTc%
z)kc@xeJ`0GWaH`LD{1P%#rUFRlen$=pH@a-Lmc;%(HVx2#En6rkGa0t1*uj*Xr~1P
zk`|pM*1*4X8Z6LyNDCTooPa1&E!7>s0Xu{4H#QHGRjhzg6-N0e2?46QobSVTu3w9#
zz@00rvV~!bC#b@&B5$3lP9}guNN4DaX^^}c!gHUD^Vzs#C!+Q6-173cVO7`M_9^q|
zOX!9ef!R1ZOb8UsKwi&`!{W2R|Ep$#xfva?|60VzWLtz0vYg-dw+~x%6io1NNkVlJ
zyE(TE7z8hhCFTN2muzkwJX+M%B|x6VW%*L)TiddqW_5S2U*WM*nED3Rs=q1FntTHo
zXt?wh#(2<m@k_b!ut~oVQros6nn!x9mEjH_e8^!`QK^Gsao>Ip$(*o-9|kNV3kQjW
zrJyQ`QRyhKv=jPi2xYq^NU-%O-ITlxwfK6NLW}b~3@-{IC%a{{y51r<kqf;jdYbId
z8=M`M1?An5Y_+t*kl;Q!qfH>-e|?Jh@OLM`ny?qB*6I@IcAgMM854`AR0AGt9>}PZ
z+WGoaVQ0S;cd$E3&|iv7Hl<r4)4H>Lu2!T}FEh0f1nX>?Qk_2JwoE}p-eqqA=b#F^
zi4P1Exksz9T0ifQsK2%JqrGrYtH;mMsI3^!YIdJrtXxLn(^0apbvgNS{a&98T-3!9
zdx9RYir3T77~e+lKc~t6-uJwJhNwlY__BX`K+F3Ml0sO5NS+c%xI&#DwL3c=%cAgJ
zJt!m}(Gw*NBKKx>vh_a<c=`RZ47^(<?I&IBJ|Ic;t`dQ_%nIqna%=kKk|dfE3|*t|
zcBn4<Kg<@e|1w+9{MT$DLwvc_2ZxV@lckGV)nS94V12|Zoe(d0`EqNN=KN{lR+<8N
z`LL+scgP^~eL{IzT}7>%;BG&>LPs45_6e6C3gm1UHy7{)?<enVav`>HKnnmKCrs$$
z-(X>~+Gq!SZ~r7-x^iE4GbF>Yxn{J<O)GjBbK$@$&H{WP=GOcQ-_a+*p0@&M9R=c$
z95r1I+S_G2Fxi8AGT()OZ{@06IAJMi666DOSj|il%JJ)1G+bo#e`BK^&uu->m^QD?
z2Qhta6F;N~vqSt$L4tT{e;jHAydA$w++#LGwcI572|D&8eS`?1z&niGlK}C6K6V&&
z=@F+6@75=9XBc{LC!X&goaKLAm7*4e*AA=|1&_A1op^d5jjCtAPmBMK>H+E!)WsK`
zOnOBH8|sv0(g|K@^^vhR+vt+uCr1U1VcV<t!l*`T9c7Jis!$ggiPKe^9>%P#><TSq
zyckEeFeF0u2G0Jm9kB81zKYs(6wZi51`;s84f;qfry7(VT}a$Z{-bq(Mla=|o}YFL
z)*BuXu7lPN@nA`&K2d;)Ad@<i2PLw3H7?nC4@HqQ%bIIq^n8bv!2??T*kacxYNY;4
zWB-`JUQ*1;^<pGY`wJ`IknKG*n44@0=3glG71!QZ1=(T^`>DrLO|n#=irZgAF^m5I
z7YXGDf#ss}FNDSoJP7oq8b#Wkvmjpk6eO{-p6q~Nb#1<taM5>%#C)U_c4P9dSV$zO
zUQy8x)d4%$Q}88s8t(C-JPrUa((A*DiyuKk2z+%8U^FOYj5|Mbes8_cm8DR7{;@#j
z(n*XcDO;}pPdy71PZ3|fYupki*kA65bx!(%$!>eB!uRBTH%?1tc+|&%nCzYkrfH{~
zXG2i7|GZhRA0VpPA2IjkQ}rtv91A2$Bv(~2W_3@nd%!}HK*C@2g|ZdP{2<zVs3)OF
zyY{pzY$m7h0#EeXaq%6FZH{nqf+e9pgDmI&kaK4jo-oStkRp0z8nT1FhkVQp8>FmJ
zMg+Z)bnYUsCp?=k<&lxNJs88wj8t9jq!6=MV`7b~j{|)&JWH5d4>=9&n=m-XCBAd3
z_G3d1!v*O}pT8<2?f_kc*(1~bz|L3Ls~O@$)Fo_O5&DV{z6-#64{50q1?GysOQ(IM
zH)+5ybvR!OPJBN?9Yfa)(ql~#IbglXW>0B=iu3_hv<Fu01H0Xy9C1&-kl0f{;<45r
zemSoj#JmF%k={tL1B+RhhD|?*`1*lIG#O}_@@CE|YYGN1*`q(NZ;xAO@BaK;@+u@=
zrPXpgM`_ydO)^YfFqlkFu-}jbo*o4vu|FKe<mcPPE9F~_8gd$ph{ba+qQ`R@M1o(V
zZq^bX1-$i}yq>T8znV{{C}B;XbVx_t=}W$j5^#O<Yq&7I_QijR(SsyX?DJvv^KpI|
zvZ#XRBfp$iuWelpe@(+DW=n79zw@u`on>;`ajJjjFrCjZesQ%Dv}d-1t6-3GTtPFT
zuo-pg_1`aQbl{)uAUUxhy*jxDl{E%G5}GVZ0?IiC!4zo_12QX;pxHj!(94WLKga)H
z?Jz&)T6VRs`G~Ulcq>U_$9b<rxi@1XS@{s5pGCC~8j#rVJF3(XOZST5GyA&LabH^9
zcSFI}TAkouWko_mVJ4U-y)a~+QeTmG9aj>t*7a@@2`aQL3vAZ{m-k~RyzIR>eKlPV
z{vby|=EeRx<R9%YZNmT14&V0If*XbCw4}U11Z(|OK+FqZVgLi*$RmlX2hv8l@@c;1
z#-kY<RaZ~BfEBXiIfo5>ZL?BA(J4@xOaUdg*&M%QPCI7j6AsgfzK$@6MdMXVSmP?g
zv_)fmB6#4WkSU3Q=|kKp@Xh;Ls|^yecYw>W<WVB%NlrnDjp3U=#K_ZXY`8uqq+2(L
z<y|bSj1ZAXC5lg^@K_u0QPgnD&MF@-(3q}@StBjbU9ZnWgMsq<*hcMro>28SyO{NY
z{ZfxfN%6jN{*wS?);0MLg~Ne)%5xlr{@h^Y;yyN=_p`a?4BiE+?4m@j%nEF6yqsw`
z-$^S%eY7V}!i?ZfeyW(~2!}e7MKB$$50*}9I`kO!KXZ1`DfoEu5TPw92J=vY^PJNx
zNB*?=^yBspoSUwWI-Vab_qdEGFxa`&VJkK$GCs{)0pJl%LBs1}NT}OiRJI@N-AjKd
zXgB!jQ&2f?=CTM74u}5w#AjA+xb)I`N1tzX1UcK?;4`DHK(xTe!u9_)ZN07jYw*-L
zp>0j4FgkxLi}rO0<N@*jCSgTl@o*W^S$=1>k)i(xP=*ABv>c6y(h`=a!*QF@<z251
zLbh`nbPqCd8vLz?C+mu{0h2o@%o7|F47V{MMFm&Yab<ON!2wU!We3d-Fm*D6sdGr|
zYmn4%EHsI30Yl{?;<YRC>%7Yn<h!m*#in`0iF6fANihuWIbp1B>hLb_ka(&@a}1zy
z+%!xBQ%VdsK-D6WE)Rl`>X*9ZT%W&7Fj*3X3_CsYfuh0u0Ks+PU5OSLy>HacK&F|B
z_g!p)r7oaTDsB@4nR_M6p?n1?^MEH`+eYhh50KS;R?e5h<RqWd-GE5ks6<Pa>Xk_G
zNGYmdbzpYgf!gJZO-9f6O=#^K=O-BU!xwEg2Vw3HuB0vPXLRcEWPlL^`Z%KP%KyAK
zCzX7ukSSXZWT!s?C9KIZ0%g$${)S!;?BHInHJLx-Kc8R|tDLb8!lvY}(hjAUfx!V4
z<|D9fbE>&ZULFc$BQrNAh3?Feq-DV0_Gflxl1XksO^&A@+7;E3%ki+pE7xoBtCBgs
zRd?_Q?2$W%K;{Yu&^fb0wOXh<iiG|Syx4vzfFXuYSsd-K25wf`|Lp2Eynu)t*lK(t
zr*1P1zW4<a7^QgZ=N(U{K9YQ>0aUe$e$HtXkZePsixP^L=#XCzT*5jIjD5aL!lA!+
zqP{iU-Z(phpK1pB{ECxFEkSDafM8T(A0B>ShAB-auzxx4w4I;teD({pM~&uOl#D_%
zm*D-}1(&f*Jff>fAEf_Ct)YUzk}Ipj`cV+~WBQZ*w{moyc`|UIOIY2%MwZV3?QQqj
zy_Ng>32b_=4lnPHS}k|1wuK*6HeP>9T|T)hk3-Eu|MEaU!=Jt53|&y|{1(~ESGPGZ
zv0@Ia{#V@Sed}0PH9l&>(kVsAXUuDNBWIhaHmFqB0X67b-Q4j@aVht++WE&{L^~Fm
z2j@k&_aE_q*t=0@><RUY7Sno(FiOnt?+URr34uFA35Zu`D8yR(%Ic$;rt?uy4UC<U
zzx(UgJ`%Kbfl&1hFpwtw@DM8pS$B{5K4ep#-E9>1zWc%MyuJ^U{h}}<5MoiIWU)QO
z_S?NTvP3S4eQwTSQ!vQZbhaY#)2#EP_)j*WCdcdc%+>+UQQ~c7jo?f7S@4qCbWYCL
zNT}^C2t?`5<32RmQ+Jl|Ap)yLAK}eEx4sQ83!_CUC<A+E&J<T#+3k`jY<gdw5ZEr+
zBknhkR^jS@zuy-aioz)USso~WSzp`Rgem0U(BwCHpw-DX0R(S?o=}uYq*_%<nz!ES
zOKe@CvYeM=(OQ6R!XaVSMq{$vplgsuiF?<dWL=nN|NH+EFDoUpZ8Jba4K!DYiI_io
z;dSYkEjZ}Fs_3P$FEN7fsOj$d^Q*VL9{k4Lt}W`fk?}mEu-Ifd4Ysiao66+JLXMU>
zdK{};%oY>26&1xFylF~aBbDs1DkM?2mUEet#s*!9p!}m=CfBd~SG{b-wVxWq^17Rb
z%)CnHQX7U|esKsU_=j|4-E5uYQP81S*-s#&DOQrVn_&nMRNw7wx9YD(xdZ_|(FPdE
zco-neqKAA$l3c?5aHZRRfM@+Hbu$0;@o2Y`<6Gw@WOO))i82Afa9y}^9pTD|L|m^5
zLRs$S7+*_U5Xv~MASa+i--*Mw6CSR0TvQj_T6nJuncsGR#GfymON_Cr()9j?NUh|v
z?RwhvS7e{`fdjc?WD@!%f19$~_%k`I;w3QQy_vKD!d-g(tiu)62~x^)?`RBapTmPo
zuP;TtEVXoo{%W6R<WGf(s++RF=<GA{gSU$o7ue2<uuDdNA5E7gnRdUuCM2T646PzC
zW;j(ycD4eEyi;IC{Ly3FT?wX!47w!p<w8_RQarm?!KCsX_c48wA9exl5y6Z5Xay=%
znQi(s{txTsnswz-*p>xvFXV6{{P&L1r--(1Z}x7jEtj1G_jXTrr&)8<Q~kyc;Cf+?
zSHfL75A$9Mc8-Kr{xo4)R!y+nn__;mLg=zQd3RY3deH{vAn_U=6kqKCON-l!LHD0|
zAF#r5Jw2*G5M4T2N5X<p6Y9R(^=0&hTiALtKlK++E$T&R6zUEx-%in2_}9H&e(L8U
zw(4<U66?MF1<ViFX#J+oAq}O*3JmBi3+jMzU3~&PC{1VRq;K>%weDXJHC*m77|?e_
z$kpzBHaZt%{B09Vn+&BqwC8Rx_A0;E1`FYu=X4d&bH3U1Y2=EQdLQxt0qx`ixM@8D
zAa2%{r;G^bryeisNlSNo<}Z`@g88S&d!>bql~@ICPCiH<<hMUiN1dhqkJioWPwPfX
zEWoKwzDS3IJa{9nDBOAE*0GPYAWU^`sk1CH*zV_Cg5OJ8e>fDFMNqR@EPt*ex)MdP
zI>Er@h3mScm|R?S4_^IMxnbiQt{cV|Dwh=rmW!+~8MYoWMJd<A1qrwDPCsaB-Nptj
zEa)<zgkS4PU0y0CV;FJ%jH=?yQm#0m9K_>?_mk8+dY#0bda>?qt|7+wODl4{q|F5h
zIYCJ^=hO%N!h<UITT!AXUCrq=KwGN1bo42BSbFIambta$U2k5+i;mLo-A?&X6eTvt
z8;m`HwIn_m4}*G|3W(j$TOsdW*3#`4DYg7U{TcE@1qGY$$A%6+?)xc6v=J1(Qc1YH
z_pB7%j(#($%I=e9b5>9?-<SNpi%UCYCBjKb0CAwYZDjfSO{g^#;TphpxDc!7Pipg(
zuqDC&<79+Nlis8T$qf;&7{S^=DCQ0w@&^$<#s@S*MTFhpWlqk}$JTN^iFVy@M(_T`
ztx;<)V^6B%=PLqsxdWRZQ|Hkj3;oVz^PM=)M6L-Hy%03NZ$hPcXQ5uo8g@!AxGwP|
zfvVv}4G|4LoFoo&1RtJ{v6!?@s7aM!nS7%9QFEgVwGRja@jTBd1H>h4H#um?=wRp&
zm$$7yLb@?|e4j7aVGaxNJ@3ZZL?V!B@Yroex6$7aS>%wVv`_D5&^vJLIFo$FR!lcB
z`jKVDTd4Jn871^$-aDLu%4`2J!~7ZEgsqEmrflK}h0)T56~wj<Uq!?=tzZRk#$Vef
z4u6XwbvfGB$@RC3%;bOP&yNbtV|6d6n}f>-zDJ2OnYEi7e?H~oJDW*Ycj|!s8vX(6
zo&GbQ{Ur`$&8uf)g9^G-hTz|5SUEhT2@9Sflnk$f>K#51G<|4ui~kFocXuh>;TVra
z+0Y85|FE$ZPv60L|AZv@=ZsK7KZ)C~lc`TyreoZ{ojVhf4X2wjDF;564#aVowf0v?
z9Rofc>*<m^W#v>UG3EGJ$w|(HsozehK6gE_{r5Ot$M!1&$GGl7As@F+YZRH)88m=q
z;|ypTtiICjr1<=1PR<pWWOuACVJJRW{2@A>VLzQr;>KyUbAdY*Mo4nQBL6{BdCZB?
zv_bO^Ao$PbKIQPwt{yr=Nyy^nxJK>a|6>2n!X2uO1+pf=@I>6<b}}jiDT^H4wuiO>
zh1p4r^KxV~kU`k3pbBc=FM1SYnx>!$hFInU$sGx;lYn$+@zOhOQ8|8QC)+kFV}b8#
zhI!mqM67~ly>t(>-VdXcsd#p)xQ^?P{~n!e`6jW`NlNXjuF4<OA$Jii`XfsFwa|pj
zTYhFQLD$1!lBQ$)DPz4_O~tm<NvF-yyP+Rb!`d!3dr9Iib6B-Djg`DzN=yY-gbHaB
zDD!Y6?i6iAD}lyzCXwQ~w$LLy$aQPMvx^pVzn@lg^Bw1)vX+Ckl#h)_Iz=98f5d$@
zV>3$2x1?d>@U0GL?YlYRuxeAN@Fd^?kLRL2{eHCtB{W%yl7FJX_k5+NYX7Nf(U&aJ
zZ=WCT3f`1RzXtft<zyqhMC_0Ryn7L5S^&@o<<L6M`?T-Se7`M3%B3uRXQRUu;5Xde
zHL2(P+Y3Mf;%a+{$0KuT`f)jgOIA|g;PP<0@o0eWbY1Y4N08l7sGm)M+^6Kp)6=;K
zzeFjqN7cYkAWDOkIJ!|qpw^e%stsR?6nFaabXUzR(BKqQ{9Wvs$Kk4)4vTm6pcwid
zGF-Fl0v(cWJ{aueerXgY``*DVEKGIg`2M<e6;rMfUe_SNeyE6w2RW5Qf^L%oJ=;&i
z92jB#Kd#;~Eb6G;7gj<*Vh9mwkZzF<X{13xN=mw0dH|6I=~5c$?(Qz>ZiX7VYhVW6
zKhNI#oU`9g^Fh75u7P#0d;K!5Z#z!HUC)^{QI<HGbl|yT_zCBs@V~hjIoh8;`f!vg
zfmwk?F32rl6q4<89(Z5qAs+Hg6VEV|qs|{kkP+cys+sbXkRhsc{?!5i2>u2)TmegJ
z(wTVYgaUv|*ji=3K6oXUnBhj|OL#B#+Axh1BY)l{4>f$3+j^FDVNbS)PL>O9^V%{r
z0}l*3fRYyq?6c5|?W}3~XoR?bT#Zu}WNL*P_x+M{6$s1!tx^He2l^g|_r_?>PXJ-#
z;AgY2X4p=ObL9B`q$z}5?i}LI8WIs*gxdzX^$9_3S$J+?LS-pGPmY0<wFN{!4|=r9
zwpY~vZ$c8cD!pLkIvB6_<e)HO9&zn_>p?htEp%e3lv1zwWeqGaE%In0Yg2c{JO8h#
zv^M%5uT>A<la+7h_DjDw0Hr66j2T59>cprf?5Ls#e=M%6+bMRd`v@ctr@gp3${$p-
zz3yS+eVF2~eK^c`8A$6>d%z|f=6I!7$0f@>(8Ce5bkDaE2jrPvn2%MWI`8xqrTUC4
z)|RN>*v)#T@z@;y(!NGStJu!o%4r<pDdWDoYUW41)st+0+<K+^@5dt=wPv5{u3N->
zTY*n$g^*tUF{xI4x{g+ga`DnMhUk2QVaC#Uy?za<p-fS33@>@};@6bRs+rlZYxKUN
zeQgixN~ADYFytoq@X;lF_;bAvvh_6UJ(r2{e6H1O&0wfkey!J{pw96(wk4OUy*P#d
zFy;KXxHi6%kBDoBRi#z1RNA_Lh}Wg*l-T-Gfy%9Sl`~%o<?3+6{WYf7BdkBsQxE3C
z?DX%$mYc$=XivL2p^pEWG?j&BpE>X0ZGTKL+z~}6@-@wJ_KcUE-GOau_Dq5Hun0N#
zc#y9!zg(wvg=^a3&B3zRtZr;P^lG-jgGDD-|N8feB}7X?e~G7CxXHe<wX}IAlGk~k
z_I{_k-ez9LKWX2kIY&SvCrwlW0<v~rQ`avnnA7n=gwdwzdUxv2dS4xSwbzT18TvME
z_rjgR$>(0S1AgA*)_%;_aGNPJ6VY}^NWlp2uFVm$9Q44xe`&nDFSiaJK-fsJz4Jlx
zVrPU$@#|_*PtPM+Ht2r?_eI?QArU`4J7{(|`xs`9Xu*K~9i#h;-tz47EZFm$DEKYK
zuz`mOCR401kHqkQ=L}eF<b$Ur#s_7tCbFlr4myD3foPR0hy)ezRM?BdGyo4|3GZ$N
zvyXOaxq`Xp#+6Xhb`qGwkD|Y6OAFNehKI%(h+I}8zZ5ijQ~fQ%(q&MYC3@GGNcIx%
zQWqvgV3SPHR>YBJ_iIO-Mbu~$-XyonO}TI;T@dw-2o^`s$p=79%t>G2ud+4T<{S0{
zixv3wyZ%qkz~k{*M`G|0xS)|+27g<VIWYh)&UgUT674UF5TWetv!mnYli7Cy`~IL6
z*gLLOZREc%LGc%;s$Gu#<Vgh{792l8VPK?@^-lk|&CCt5{503(BcOaEGz{$l*Pr&V
zjjF5r6w-~o2G%C?qpz!Ub@gT3$-)Q4OVet7LBL_GR=S6+oBo{SGU}^y!+&>vrk@zc
zq)`Q*zH5CSc$MaJw}<g3plO*XGGT*Sx^0U934kA{+(Ik5&_Y^V&z&8Yac{&yA?I$l
z0}KqOhCXo5>6Y7$+pRbiW$5T<70Do0aK|t^-Pwc+Op1JakO3hkM#^Iq|8H<Yc#-Gj
zHqb?82?i13p9YB0{6ufg2qDA2uQtY)X{0`sLK3GjxFyb-Lu{aP=chW!%Padwca?p7
zOVTT_5P|VFK46Z=gkMg?&&=`Qi()04wb>DRIi}AJp=Se-#^K#RHh$~UQ>I21Ogo|L
zb+vB_{8t`-cc4_X)JMxS1_MU8VYsekp@RbFS%Ki(Ig1B&URe2O^lvR~=i9%HBL`vM
z$j=qCjJt0CFvqR}U`O)dzl02HQ(|E0wptzp8r!uq^ls!BE+(s^766~|l;g_jJE!_q
z_ww`v@p577IC^+N)^`7+s2B7(hq@!;41-ZA8Cz=q@BC+OE#v2yiQn)QY%=?z+b1ko
z<!rS4wL;iPctR_Ce}<9x>q+^q%zvuX4Eqt1$rzRy{RlRP>6x7@t<1e%Fdc(|k5m3Q
zkEU{6G~cw=d!3w&ep`3DOc%0~1sB!E4w`gLM~JE2zO<NX%s%kiS{F-a+Tyv{{Bs-=
zC({skLA2EX-_4=X?H3{4+O@K7@mkBa>r&zgExHWXtO(r%U%-ft|6rCmA;w^mpCor1
z%?IQv6(R(CzLzpHZbE`7kM{xj<HcwALLfhwJ^0sWP3=k#1-JQ%_D1~_+qinntt2;n
z_esP>luWZd6w1JSMP5<AbaM+S+k43}8&Wy&ShnVK_UWeUx9?0vZ<rp=Ln*1l0H-()
z!oPmmw0&g`@lfQoVwL9AW0xheMqtA(V5GuleHTx*G(;}coW&P`*%Y|ueW1MRjqG;1
zm(;krbd(65RSj;6RCvf`hb?;mTSOBO<3aT+;>766J9_knO+=>ZE2RmmNdK$5RMnv~
zzId_fi73}I=q5**oSL0F8{TLyPyWLYx#w&GM&ov>xKP6w5kY36yP0)F<7$IQgEDT}
zaP7U#(cLAb*5=)<P2U%{z0Y0@Qe`$frWei~Kb{*le|^w*5T$B$GS&O29(GWwm9C1f
zJbSdzsmITKe~y=N0D4%7Kn-Pw^}V`z{C=(LHW|{q(wl#Dv{(z7-NC26&Yct8q42O8
zydl+A5TNWexzf=W4hy{JZgDU+LVtr4^n?1_e|>!NK#wQ(GmDmS_zM8|qy8l67Yggh
zRPpn-5_!P_Z`*oArXiT0<WGt*_<qhiph<OoPWin{m?9{FwI4rJ-cU1`d-gYf^o{z>
z3{{x8qz7XH<)fC0o~r@2&n<3>2wBY_M~uEmSm=jHY08l#kqCSwJ`L_~Mwa7PinbeS
zQ4Hbi6#vPppbt%N{*ZdgAGI;8X$yBzWwNB1KHHYX6ogB)bZUBshNoBd0D5%}+kIwl
z9)=`1caS^TP_OXbm~d`(gg(rhr_+$mz#KC_<;qp?W89Er2Rb*lsQGZLSa(F)+T+8G
z0H};74p?s3v{8d>f4W<&mK(HJG+~M?|A6+{P#Y3r4q%{z)@t{q6*CQsNi}@sFhSYc
zgvb!;@&RE^zu?M?PaZ%k{ob*(W~N~?ap^!O@{dbbMxLL!ug-3x1a^L#G^fPtq$OX`
zjh|)$yAB7alKSxwh$3+rQln&8c>^eSahvwzSdNAx+TCljec_vwI7W)pp&0Kik+A3F
z!M23q39DZ*e$|qENlxw9mo5p5Q^673rK$(u&ldzs9+l@|0SAo=N$gny9_J=}R@01s
ztl~Mf0|~7fc4#>(1JHVsbe%#KvDrNsyJ}e*ffnMp1cjXN{x5)k*gp}_glOrV+l|D(
z@kWg$*Xg5<4xqx6OY9RS)G<|jB}b6Hl^xq2_x9~Jjc9KGm5&>Ld&zE&2)xwKVeV6?
zobaACi+6;C;IIrz7$;lZ2ep<XV*8Xe0CB-4cU`#QLQ?Wb{X441Wp`nF7ub3F4spC>
zKcLNIn4qrdwsay~k?GyP;<K|@H|%{t0`)p+87$wqwB6M#6$RWiznkzZ&JCWIa97_G
zb}zqVtI}EAz8SKj6y9OIx%#&tz>ZBl-r_t)b6gvf?YuGmqi4$Yp3Q1ES8KN(WYhH4
zpb+so5u0{g?6P$1<y#m!H`v}Vcc2M9hj@T1+QKRhFZlRSB#pN)XsLX(nYTnxtvgy(
z_;H=wZdy7&r$!;!5^3ZoY^~wgM><vFyq*9@S%W=BC0UH?I&XNTteszfhh*EA^3C>f
z!|lD7JGN!&SPJgj8uGFi>Y7N{89G!-m8hj6Q_LR=F}Q8+?<W;FW~jar(TrE%(-r$w
z9oexi9U(Ap;%G39u|w(UXR$;rg!uZ)f5s>F>DvQ&u5p7W!83`I0@FcK&d2LMRzBr)
zTlOML&Nx!ZX3wsu$Ky9Viw|-Xl=vH+=EwBrc&al<DLmn9ZZPbqITuED`tg4V!Ddg5
z1C7AP;8gdAmE(~cr}A3WvEWW9%{Ul<2aW>>lnXAF|4rb&L&173+3{Z=nP6N-E~F80
zia((b>&WkfQPN@EpdS|<&c3(LmVfvY{D`2rjvFoZ7o~FzcnC2X(&RKzqqw3;I={eu
zKdM3r<5UiCNB8{BoM7N{JHW?&W}dz;<8vpUu>*vB<lz|gdJv{Ogutsx4^?0VzieH~
z2pYcNXMEdu|4u9v_?3nmg)#(xxF#yN#4Ff)2n$Sq7!oIrGYAPUXx~xRF@ENdR?{H5
z83Nr%5r5_X!ZpB$gzsQEHpy|<#7IrV0K2V*7)@lq98MBtWR_7RGzSDFR})$Yza(V;
zm}z;7p__<8qCpx;`L)0LM@^SJH#&=|*o$w(wlU1}R$K9EFV_kpHLV)ge`3b68Aypi
zAflk9&7+w3j3WoBQ3&KHK~tYtk+br}hGLd!>!!Ndqz->?4@F(+#&bu#K(v{+74IH?
zAAX*DQ)%Fh!ChkEI4QTl7wI8JQSOP?Z>#vBFZdQl(ZfgYQ`u&dR1N5S(2|Tmw}i;>
zM(vOs;2#`zx_*+im5R$#N@?1RGB|&Ra*j+(6yVJM7rzI6fG!?p67}I4-TuyKGiV!#
zG~Ke@{atbfxR8V*CA$1ESGT0*jj$$@Xz9f(lCgB8c|L082C)^)Tggs-Lc(Sl5~CIG
z*ag5~)q$360QBGCU9`ylW=zU}NY5@f5N_NXbLkegA{dZ;yDYvu!K{6`X;014@e2Q8
zcGvI~$P0)WoggXWq-Nx;+Me5;$Tn7Q-7m50sU5v`m-Qqqyd`0~_Ug9chhvLVar703
zqHb?cp1J5+@TAPQ#~yslWQwY;-+R5(bo}RV?R?B^<a<;^E1&tdv)3wXx3+u~E%$Xj
ztJQONcAF!0FONX0-cu8a!r-;E<KjjIjjEWo<RL|Rf;?-KWVm_j$!D}?yCpMC$@{e~
z>EqI7itlQV_hD;Bd!%z_BawCq`X6-1o-Z}-*e*4IQIfAeG&Ll^XH{ZHb*>%t>K3`V
zp^C}-t9LhV{=_=%Sf4bYI=9O&ewi06dOTfYXyVBC-dMex^Y_6lTqv6Yc{RISFEdqs
zE<{1XUYbq);0dnUK-+w9=)$z?iUTz=*{e9Dd6pe$Lg$CdSF4H{Qg|JKA;Y;B_+V4X
zeOy-@`+&F+71lmNz9;M_$F2MJH$9R6b;f5Y2LyQMJKtW!ncYr2Xk|bB-<)+dWuu1e
zV&fu^H>62chkWPIbzxVu7F%@0Os<>fwpLL!aw82~J#r$|)kWz$gku?xFd%HdP&_iy
zn=0;NC4B6q=kvF9)up@I=4tBXCrXZw-_ypp!N<nxWU|Mc+V!}8K_|ig;>M0jes!Ch
zm;{=Qtej48+!KaX)&HJHIz1CAzgnXGu-SBX)aEL<8$v=`27G(BB-D12W%a+sB(b)5
z+LacpRU#cnbZT9Nu<s(;KPJLQ8sPJYuB)Ug-6qjS-WNDzZw5Yp9pM=rxPk)}O@u3g
zB#)}P_KvGf)6}WDE|(9X2i{}t$H(BinH=u{<&<V0(o*t8gM0NaQ4a@cp=<`Z_9u47
z|BT%*^8$UErw~zXt0M*-h~q^%`tbp;S#yKN&o>cnQzCyhdG*Vym)vfYV%K{4)-oW<
z!W=hVVujFd{$FC*)rPtMv%Ea*FvDm_rXMqjw?3zBM3p0+KP8>)FULc>FF~&<cX25)
zfpEi|ZZvY+{dk8L8@#SImt!}$FAQST3*rnAgj_2Io6-6Lj0qXB4O=}NY?CqJ$i7$k
zl$+yPj0sbkKp!R%`+4XE0nz6&8*4e=+5`1G|C!<oBXgs74%o@6g7ih9Fe_D}Xy$DO
zN_nYk8*2=)7KtDws$vw{HiWglqb~e;e5~Z=WP@q8)GJAsk<hlG^50w3<?_9G94?_o
zQ;8tKD;oXl<K9^K)(oUrd|+*oQcu`#XZEfIJL$+ybQt3~%{VvQhE)Lx8+3(P8irdg
z=G;eY9>9kJG&dhs@CjCLKYtHe#2m~`(%*k_LVh!g#I?f3vj{8AL^AZN-@vV7!L!^3
z9#5`M(9Zb94&;w{I`RG<B>+2>)ZoRDG=yqVLOS0Ur&BH0Z130_=d_`aBpN<WWjwr6
zmEivt7_J&V3UbuG0t`Ndx88D!WDvuQvOf|q<5<OHF`&P`IUuF0m)2(;*FJ_Ij-bKJ
zg>5IC5UNOEu|cS~t79X4rZuU<+}jg-hsy=%TZ!#T&O4C0ut1dj=W>!5+(O3<#tnta
zimTrcOKT5_9N@nfkM}D8B}BGp8LOrTU8uF56Xf6-)6ldj56HPE6r#NChvwiOA1&2`
zKm|qybv7Fjwjpf2C{8%`UoQCK9`l7;HbRUGeL0Z;G16zsq2FR^zrci#6eNE1A1fYQ
zN?Tsr$7nq=4!cDqe64qEhPw}1S#4(c4k_JHxi$O$roGrEa$cqZnGA;79MvqTXzdk-
zrkp&AL^Zm_lQxX4kH4tB)KD4bi-nmd4VwJyMCXQ`5jza-m0cMj{R{V{+@p{1;w|6Y
zCoS7FYv-%%d#!8<b;r!x<4|-M)4hzUQCM|tH|+PSQoh!$)SI3cO+C=!l$<QIx39lC
zc~ltzBF5lwH)s=^>!ZgTu18}TJ$I*7+eEFI?TYG0in{qwO%zMW;0yB=7+50A-3>l<
zTtom;diQC2ci2uvBlge59u^<G<M0M$K|OB}Z?}K-d!W~DhHs*R>h;oz=5cOiHnQK_
z&v)`IND-GO?eLOj&t2S`m3qvam)mzb*O<`LBck+yiKL>=y2xm`+r!=>R!a!i0g%4F
zNtONT>mJcLC&$MM!3YiLug%kiU+O1X-Z4ZB@=TJ!IM^r_y(TXGknZ4MLX0UCl7TV+
zHuyR(uk}#!f^p+&EK+a<v1`6?Vp3vs4W;qj$!mSg)9v;U+Pe#Ou6w3n5BXAbz^0UG
zNSY6&(5mWs`&b^dhH#Q#_1cSRUSWfBP><31vl|AEJw#^%kZMX2S?_f%05M{4%$y3m
z&j5C%VInu}{j&CN(K;^MhbK5&Nu(#9ZOYwp1QsIcut1^n%g+zh53wizye^kI$OZWC
z>qFXoMPcnt2#$iOXD|7Q{qml@%j5lj7bOWiU{Q+D$b|KE7lPV^Z6KMYP4adD-FWML
zL2xDr!8d}nXXGL^OnX44OzYkM&6I)P#u_-jQuDa{gyP$)&I;a@N%4tC7hGVZo8R`2
z6l~l2q=y)fiBdII)$Q%nyy9=y<WoeS)H3YA`qK4`FiEhh1NE0!Z*UXXo9A{sk^`le
z0gXYKlgYRO%!Oy<FdnSxGo6reR&ZS02@>)=F9z8<IYt}&bxjQx{M)E1Bg6XrTs;Lo
z4Pf&9FiRHE9^&mEVsU3W{3~B}`y%U8zC1&E+Lh#(HGyWF9DcxR-nEYf&=b1n9nEk2
zP`TYxDgg1u^EY-5`y&#)w&;FA2{y9^qW7)>jt>tv?NHPQVD!6?+P64X<Z5P4yq12m
z;KZ=@lHbGMlWvP#Rv4Y{jR!#$r>X%Gwt}hkL56C{k2ne;zIAG4YT;98(A^Ew*$`2$
z#=!V*BczYO1QB>CAtAX1G+1EjBu?<rkH4Z}Sl0;=L&p2LmKKC=za#N6ZaP8ht*-`$
zp@45GJ)L)DtZi09n~~pm-J(@7s(Kb%s?k5uyImT$gVOpwYOCof9dkW*NMNXn9`p}=
z7wz-&FGfh%zA`?*n+tkYq{m}A6u)&tdo)sqaku2p)|NQgX0_Q4(9BzO!vhTL5n=?{
z6AewJVpIHZlBJa3dW0L|kM$I9bHY-|A1<UlI&wF}YD%LE-#l%B`Pg002a)TxPit1q
zX)7ewuGm-keIItsMt>jswA@g&lcR>43*Nw|HkZEuF|s@g^l=zdd_XBx@jAntUbM~H
zmtSIW?QTl8lIYwlbvBUe7G8x&`VB6khi)a`N_uU>`r%QhQVjz-+R^|*iwEwVqkw<c
z*B4P}v2sd1FRiZ;D*|@IE7ekt8twwlCzOI3ujLemXOEUCAFp&{4}P`YHF-Egb5?YL
zJp5yY^l0bFEFO3m(<BPXdGyKstGZ>#J!a1?_h2I0RY_exltZMLF!p$*Sx98AGFG;2
z#*C`^)I0R#t3sUYSYk{IRwn1W2fzT&T%wH5^ygj@oc;^r3^&fcX)iC8=&jZsIue%Z
z?pIRoOZ>U~$5<kUNt9$&J>Z!WL})X*>YGB656M(xcTG8NFdz55UBo;d+60wArnFEW
z4PX@g!?>_E<Ws~(Adx<LF`m`T2v|VDv2{CK2d62mPP?D{X4uD&u|%YsnY-kpS_-gB
zo9jBYlRn<b2&^P?!~E`V)_fD-krby(2P>KvP5|vdE<gSb@H&ds;Qn8v;~RQB9+IhS
z=AT1|A?j12P03TD&3?SF;`$F?vCwpUyJwU&0^^Rm{`KG<qEEtYa0MAGlr^m_e+qph
zAF?e?c6mpFoQtAVtxIYmPTHQ=bK)@5>>dLVD}oGvcrpzg49^DtGMU9i3RV!Ob_9l<
z6;`7U2)Yey7Idn<r}pFcpvR4O$XS3S{*04zUyyTNfE&V}2WC3^{O}!Y|MF)PyBJJf
zM!7e0Iy5W0FPk${GM<l}(4cKARGNs&bp%fpO8^(v596V@V`Ohz-us4{tYd>pvR<GI
zTy@=JdII<sbqi*s{V9Ive9Sk12;>v=ISfyR3~ygpDze@iV`vvmN3wDf)d^5cHHQ)~
z1bG^xa|7mu^aFr24pFl}O$CZkyDE($X_fvXvyW|E1)|!m`V*yH64>>cwn~bqjXvw$
z=OgwNGPBxGvqek>qJP>a7?qdz=m7q(?WE%lWCdTz`uX1jW6Z>|$^5b928#2t1QwH0
zFEp^3=vzE6Ezux5ts0SxsD44eOTuWWdF{Pk<NMfav4skZB=fLn^PoO9;VYbO`=PXF
zM_bSeqeGUf-oBva#R(B_Nno~Q^m~CH!q5}+&IZwYqRD9}O(&~{z8R)jQ*<CqozNSQ
zU<6wQbYv!tkAPwyu9vN1JBRkT$xn$p?9PuGO2=ly(ECz7K6X(f({T8GU1nIIGg4*L
zg+$s+-tTM;toKMtzce=Yf!Rpr6LHk~XRWDlX`?hni)THnSq5g^wzG6%Gm%sJ2RPk3
z=_EJN2lIvDOP=?7UAQ92@GLLLAIGe=C6#M8wy$+BuaXc}yaZ~+3n-F=h;@5%KzI&k
z_@M7+wt6UVT*<|Ou|&Up186L9k(64zr~aP2Zt|rBgIA>vj%tgSzO}QSRkH)Nu=Qf`
zA%W^|VC|}!5I@!>uD-4!dP<*L!;+%U2DNIMZs*uh^q#UP(HQ$Q{Ro$ugO=+TS;No_
zMQ3CCw)cDN8zDn0!SK#nn-$)9ne-bWzK01fd;@jh{A&~$_9oxS0zk*zQ1wcj+sda5
zT<mL8LOc<My=HZ?5H?2b4>1?b9rgxlie-&EB=vnH*3%9Uu6{tck2k;szJ|92j46UO
zww;?t&D)m24@NV@B070;A2_u!-@nX>ulc^;n0~*qmZVE$iKJBc^uV~-nD=egkx7lF
z2wPL#=d-Ktu4;#7D5vdygulISF9|Y6CdBPo@cfcNO8DX@etYw>*N=5QMR}K2!r{si
zThUWDHv7gyBD>r~?a8B=`VX5QMTN6w4SP|Njf#htoffjnT#66fDvW~>!`|;;Q(ukw
z`ut{CYiH^<D&8`4;Qs!95zlYvUnriQ3<C2s{1cSUL^MoK8ALch2GPCBVixvO2$7!c
ze?o}n@r>5}({9l#bGj++yF@cfkqrXy$q`>}W@tqb8@FMhqAm-2!&>nM9xcp(qQ7$<
z$PWlM%F5389`6WVQ|yzbl=A+f8Trj-02G@j@hbSar9?9K8P6x1<bY?nJvfahLPS48
z=%4*$EMrG$E8LUlrhqbtJX|KMY9i0i#~X;mlX+T6>Yit-?G<NH!tl1|ZU?e;dt?|*
z^Oc16;?-Oi7wj_@Fh6=Q(UI_ZoNn(zFt}+VfD9t$YVN0zpaYBqpmFzgztWq46!z|5
z5Xi-@#{lTq%NoU|NyLF>Iei5{0vb0!@X6qFHbRCc*5Eg$p#;krf34gOREn*XH6N2S
zi4FMuCcw0xuRM=fHR<{2b-6ncN?NRIVHyjJhoaG?cE%s0L`RzX9=&Ov`-Sgknr(oU
z!Pb%LoC+j-D&{1zpC+slj7Rga0Wgld)7rR5?{_IIFp@yU&}u|K7o!#c62|qG>K4%G
z0VLF3(W;q)f?n5k!8EYz)l5>DSocfrCU$Dy*2_r+>5JFzXA`s}d<UMNo#<z^{9@6k
z&d&lM8Xc6}dXpi(40m?;hmiXIFx$X7#UY~VGC_?g$dNThyJp0SnK=+Ph+`kXk~hQ+
zk@l2F4`mIH5-Zv`QD|!~OqGfGFnPA`8PNyruXA`A&G6eyC-;%*t-f)2>1346f{d<P
zw}>4gf>-Iq`5}SvO8-<I?NI~%z0bn}>rD^MO|u?iRtXqjpkmVx?l+~!OTD0og)YlL
z!YYH{hx;uWySwQ@$AY3_!0GwUoMwDW^osNls#&~bEf(pxS~9krZ{K<%;&DWNbEZ(<
z2s7Ay_!OzL`Vm+PR_d0PuMi`h)T?|MzK~<l^O}~*xn(1Q`k6DypQY?5$t~h-4BH^L
zhd)B=IhYc>b-aS6QHSQkcR8X5k<exNBaJeZr1krMA81zE*@db<wgczz2<~CG+i@tL
zz(W?!2wc(-G+XF~ZGPBy+nP{yEkNX(NB{-8AwW9Zl}3tsfeEHYe}{l&4`SB_GrIwR
z?Kz$v)w4~nr@xbdySGUTKn`co;K|7%Tdy0L_L-!Biy@d2rVfL6O$=~P=QgbZ8G`<E
zdK(nPjRCj0%id^*N7tf;QWieE#zjJI4x)eLRUeMs+A#yXP&d8C0aZM?2IqFOy<X0j
zIUPMGHie8<*Oht1IaDW*`X9oxP3ZKu@2p@d@Tw~r<DtQEBG(x4>3i&Md~~(<jrRVW
zJ&UsQ-^9lM`EW-W0*qNYvvKfFQqt3BJNL-?KSF3uY4~x!lP#xqQz}2R<$r#eKuITv
zAySp4Nw(cdKl<FUa8nY_U$zlVnf}6cyDQ57cDF)qe2OUL@Rm{3bz9{^^i)mPXh0JV
zT4-T9i;L7nw~bIU{+KT)f{jmS@d?_ShVrw+@EaNa9#CTe*g6s-W8K62+Lci}Yf)b~
zR<PjMO2L@+^50DMBTTY;$_RiHURwE=p}a5a&5%l=77%c)?gJ0aI1VPy>KW}VbX3KU
z6Cp|JtBjp<xv?)<v(e%Apk>zjB{p>$#~|?{ApX|SO^;?aDc}rHr!17VUN2I^ftC7T
z6+r1}2|<%jV1Y>jQD2Yb&aINvsxBapHr|bk66=bch}PKR@`1a%0n#J}0TQ6Lnb=B}
z0AIsvylM)UiAuL(%Z-)QcvN)_zYakx0&Yk?24vRQv~PC45((@VWG@3$a(Y7~nZTqY
zx75DZ@c^E22Qwk-rv6LFvNK!QR;SY8qQ*9(^eX?0#zuM<;|w&PIRjts!`Nw$>f{+A
zTHt=5k8AEi=UcV*V$rPJs4Ck+xqG$Pg+U7aI!Dj)`_yY3w-8hrU~LMx8!#;0kz@Gc
zF62CgvY&48!5cstJ%dhY92mnkt96tYO$NZ;gKMGOS=k~NtgqUCi+Gn-_L6~ONSHab
zL)+<|B%|&rU(aq=CYg07&t41os?heL8@Hs5ytWR(vk1`IREUWsp=9z!0ZwUuZvWSV
zCK2y?xnCUZr!^_J53y(|p@xpRUSlrHc`1*KH9Ou2b*f$qX%dZeV4)mTxe=Ov1P%Hm
zcz_RE?XFAyyjM3Je3a+`AOe*~5Zrv$r#sTE1W8S{EtQwXrkLB!^9UqLDv`Oc4)tod
zx?W|=VVLV~7iBp4q;b43awQ4{b!r@=BxgMAe6K&!4>o~rX{Hk=nv<rq#Qln=p#Tr_
zSQhDE#bhP!Pr0of9^VaX^Rxhm4a$|@?2NP67NlYu251Z)?+DOLM2%-=%U@lC16~aD
zm!S9DL^s7+0DA!XSUF&0oxct}aT%7sU3H(7W7wvvJJ*Y_1p_j9J!5z4n}tQh-A~e=
zZFSD)`J5-TYn=bH*Y8}12RA_}8e02)8zW&CQ&p0vGSMkzQuET<ilkIY*i<GFa7O|!
zfH2Ln?Uza^4NL5z|9Esb@9TH|7E`)xrhAv~Isj!z`PI;-ZsmWzlz`m~t94zU4{~HF
zy_x*KDT@x=5Ijanq!ANQkyQ=ca=%z)edM<k&je9E2)aLyKfjn6S3xlKvUKG6Oygat
zvBDeAWKslOx5c6&>+U|wY}DNq(Y70n8#He4_hd%sZL0g)pnD$vLO#cO5ad=~yqtm*
zhk{)0X!i46HI{y4pfCqA;vk#<4WSng?Rh4fANp<1<Q4KKs&J~R!3=yLiQ5PF6D<j{
zE!gq^=`-u1>|#>t+v;!O2;fZ)k-_?YJ0atepr3&IEda!eQ+xKAVno3?CS7=bw?8?L
z_)+ZMOvkT}=5r^o5(Z_osN!3!0_N+~BYi;U)EMyXFCL)r1;kmC;tqb`XO$+w<gT(B
z>79=|DfX@<9|<sr8!vw7b=5K2#7PZav*A){tvrY1u@+DFfF?+Rqsx-~L~&AkRE&Uh
z-ifC}{F(<SanGgJA)Wl7JY&D__yj5qg-^j|XslG)mxl{el|M10vfBZ##WB#E6xQ7A
zPDqLs9i8s`0CPZ2=B$c+S^S}-aVI%ao#~CaNGQ-lF5uSlEz(AkE*68u+<~j5vg;)+
zV9<3&uV$^YpS;~n;2rm4yFD=DxPX2=nk&y}3V5%SDUN|5JDgN)0!wpS{WB$lJxXjs
z0CO4<Q<gANMYnIWLZ!co7zVzN->2Q!KC*g&A=xRmb=neOKdQc3CgAX!jAx3M0hmM<
z%NFiVg35eRu`_fulxgiHcQA!#fpH)Mt<t)z;GO(00gzxBF#6_Gr>An#DvkShS`fM!
z9`xb|+7SD=I?nYwBb^@-v@9BJUp=-DQ%-S1ZoL!40t^em2!tW#I|>T~94e|(H@0s7
z%r|(W9d%FL;T$h%Am&<O(_qfJCZ1)yqW4WZ-`ScgM10||bolB0mt7_Ey48&vs<ovp
z$JAUsKBSS!j}2yUp5|)@9h)gp&X%jY;p5V|K87Q30?+K>e%&Jb{UcMIui%C8%~m#u
zmN|qA*Mb(b!Zjq`P-r%2^4{<cAX9t@zpq-U%~?nJlzHc+SXW8FYk8NV+BJ<MNq5@r
za&BjN0a~=Aeay}d;eCT18isEnJeUvg8))rLfXd&a=TkjGB-j*sxsY!+@1uY|rY&Xt
z=<$GT-<0>jMR~r<=IhKM0@gfAYw0i9i(8RsUs3rA4310Cnkx~d$h#v|Ya#k^R`{`w
z+xhVVTb|XC782Tn{dub)p!er!h#<&FqE@K(Hla4uS<OHkn%gC)YZOBM+NRz<Fd$iA
z=i{?z1%kTJbWZo1mD*mxajJU1c+w@aA6&Zj>sJEwau<8+xeKTX_<;lCVOuQIFjrM1
z>AY2U%@*Y)|9M)-!4@FDkDBHg0{)l>DJg4X+`G=<{&<z)*I}Ht^GbBwIe`)Pcbai@
z3>NJF!)iW}n~C)QVFC`Jf(w+a6PQWQ^pVG4uyZzGQaC<i#G?o^%<rLW<;R;2?ZrE&
z>?V&+7sS6erYSow7R`lw^Hgr;!!NhZklVrv1x4mV{cu7z4L3VJ`t-@O?g2<#ar=(E
zoDIs_;YcK1WxN86M*|)zo$*3J*e~dXp%_VsywT^VO>3ko%k|rLa<Jm49~06!K1jST
zf<$`MSkq8?|HdmhZahpAQ~r&0-^z>7;~YPDD;}&@xqYB6uHq}@6Gm_k@z-wyc-3qE
z7ciZ>?370*(W-HODSN)cXDI)<>A%P$DS5t%R<Z+HddOT9GY2_yNLa71IO#J^-=zh2
z_Xd!DPtx}+-0XB`ji4UKWl-`7HC8h>{e?k5&tK;yu?oIWas%3D13clIfDY4nS*lFw
zCEl0cO8{-4uH#k@h-<fnxub-7P?EJ&KCCsysk9^zg0?{X>p%YT4bPN+!WjJNTuM6U
z3mZ1DiVk{t#a5fn>0md>cTLUzZVoUg(lJq6?DaW_AdNf`kn_rlU8E%dM*Ep7o|U*A
z5Wv!Hp`*V#b5(JnW56wW;}=9aQS$cful5U&o0c9F2(K6;9?MIyORhfj^jyll!&TU@
zEg3-jBi%z4`~Ee$KA@F1$AX2Bon1?PLs^i@$!p+}id${<Vqgvt0zAx_IyE6t$l882
zNR14ANJgrP?1f{(g!OH7C;}93<*4Ax4&B%|CA85-4UsiRq@Ftz8GH});)#;;MZxXF
z=+IQQWUT~DKdcupw#45KJt=D~<Ybs2@dgn><fSV5aj_mlFYZxFl=9EnAA2Q7VIKzS
z7bgKX9iD8^ccbBu%g2z^ZwifuD9O244$R<S9>Hc?*Ei7?qTmy@o1ySA+9QdCYnJl;
zJM3GzNSN=V&Q{;Dmkf@0p5DE>?xUV>^6iTEJ@$h=$e^)E^YUf|o!h1T1<={hbWnAb
zR|frYq!)f|8yA_O=)Uy9<CsuR$RL**C7FH6W~c7<h(n?QW&qwhzP~DjNeIb`=umiR
zRU`=cnN2+)uZT->U>MsnPHg>K%5W)NDkm`#cD<iDUM;TDS=~a-#~C6+8bPyf4EJp{
zC->ZL0*EFhVSJ5m0dOpcK}N0#^qz=C_8fsPVNVUQM_z;8E;AuH91gb<qu+t;!$mWC
zMIs90;>KjbeUoHwgkXTBuKBbP3`k`lUSt^<+ui2eZQ194UOy6Q*zAlpKVF$IM%T7o
z=9Jcy%(vtU-K=U7xOk%e_aK;{?YzwwMObP<G-%xy7(35<i43qf{`Ig-_K8dbcwnw^
zqFS@`1>s7mufQ~q9^3=*OX1jU)8MgS9TyO&;gWFu?`M8IuHSp0w{I(-caX9~3>hSb
z#y8=yGuDaC?}#RqI-w-6H<Pzz_#8oN#e(ArQ%g$I!rzAOG~eM7d`JzT1JNe_!V~h0
zJXG>B2t{e_BfUvdrqdgc!|Iii(esQTOap&m#<2~xso88g#Ae<^!G2lMKtK?|5J2)H
zWXvyu2D?R*&U2shG_0^;9<O(>u1a)UAzbih#`)9>ca858q2bWc28&He^3TL$G}hWi
zYFOJjN@=k^VF$%oN)ML}K&Yh87LmU^%-9#ay*4!044XmuxcHKrBJ}0!CrWVZ>@)_g
zbQ#5F;I(q^0Bea;=j)G^(?AAD5R*=Mg{KlHq_ht3<guOTM3?LW4RZMnQ6fbi7omj4
z4O#tFn@_D4tFO*@rI%$*g)1wZdLHgiGL{v4`<4>t^xX6s7KV{MDV%6}>jd7ukfPF1
z7+`1#^#I&qTP44!MeVkLu3YXGPA2?d8|F^G3j&}LCA;LLQO&@~S-&=S7+7HNSGjEl
zu<b&a*N2%&1^^=#(u8DVqV(YYYnvJM-IjOsqa#3O`7xCZ6+o!#n%i7R3u8ckqJ^fC
zYP^C3RoM+~B6q&{8<7K;5eHr5M}x}5BwI%K)`UyWbLQitKh3v=SQh}S6U5%{aGuJz
zXHLaCgwBQ-RgUGg!v>l2>LA)j%R!k;h~bN#P%Wy|Q?Gxu1)T3e@628BF?^TIMpKEw
z+QiIt5NE(97KlcBo?ik`Ts~K&7kru4b7cZ69U)`1ErcWP#=j<Me@F1G@|zdaKkW2n
zm}@CnJXtw5E^avVTj|p>f{EcPD4MbYC1nYTrgE7Ox4@aAjdqW-R83t+X}5oz{abKl
z@lgWD@OEw7L0VnXC;MAA`j4=g=t)4KCSGdjt)W-yzvSA#f~bu_G<DV~*tCBk(l!hv
zT<P$B0c}uz^O)-Om_`Ru;w)?Kq_+dO2h?^{%$?@@6Xu849?BitH8Frd^B-_OR1mZT
zGL(Ej>ml4)DsOYWf$wkq^#ktpevWoXIz<fKm)8f=B2zO*?skd0i1xtnWj!>?lXlCC
z839c|Nit&<-Gpm#tK3->gFjGSQiKj`DC-LIJ$rn2Cw>|<|4DJtO=~L6^TIfExY7T<
zam?)1NgO{%8JX9%UYSBUE5_ZGIfdg~hNIMCuKv+>s91KWBvNqnKXdvvg}gxL(HjAc
z_=&`NOG@-6@X1e$PH0!nsV~WUZYX(`bYZ|r^qEFUiHRT(np0u>j>Gp<AYn|hB&Q2X
zU59)Or0m2#0Bwlq8I+%Tiu->x?Y_E!`@7Nl6kV>m{t(*wM~BkD?Ux7O1p;^$+-~v}
z_q`lGkojb$2VEA*5J>d3z*uzWg&b2^J=k$r-0ff1QnOy24hNR$hD`N}{aBF)Ane*!
z%!J%KgMDgai~H>Um%JcIY@6zn8A>v;uoJlOagj1w4fn%Euw+NXi^Zlt;whv*=A#?f
zDCJi&Z#y>xZTV5-9fdbc>>152HlscnZ4N-ob@x?5$3Z$xBg^hue;r{~xbPmA4jJPt
zTIEH`w7%WqPMNkE(`-tYh+v;vOWn3b_s>;*A63a-6vpMqS*`;)tB2^Gi;9Y#`^k)F
z({q~sB9RfwmAln{MW3l43bMQN2xKYJxDAxV@**h=A`~jGa<E#@@2zkbdwTM8Cz(Wf
zjc`d*fL1Wcx0KsSl%BAO42C*A#7Lvg2Ac9qBhziFZz#z_0-9&C0`|D*?&CNZ#v47M
zgZ`)U<yvW^sowY3*nE82_{k~7fp#r~;c5n?phRs{zm%eq_iBf1!;41M@)#gucdb!>
z{RfK5_edyyivehTxHL^}Clneo5bBv4%PF#@7KcqwOAJ1-$J>5UdI&LBvG%J)RXWFC
z%r;Bit{>iRIK+{4Neh&q_xV4odu6;Gb!+&6mKvR%b#pYOe)&@79M580GS<%_f!_9;
z0;UdK?-HFM2c2J#JD?uO_W$xZMJ#juXz`Mn<BBe^M@NQmXb}InB%NlkK9GQxHiYv`
zNgI}r!3X~U@r#tzJ|EU}qIgP1pM5vumd=AT{kijmgN5I6xZY^xw$)5Q$VUYiop-rl
zcK;plI&RwZQ~nx+MI2k=fFd4*_Ipv^uiaQT14aR6!s%h70EZq{vSM)m!a!02lb>cP
z_;Oj%jK5PUqruRs+L)Py>lEt3<PDR@G+vTFf}&40eT|s<nrn<I-Yd30_oCnWLp&ww
zG$z8y3)3r7(64haVqwU^6$qL!9|&x@=qo;sz{vHQC;h3h9tG&5KH36L#<Z5-%T>G0
zfyZmnI$^T)N4##{zRnJ(zqu@nytcORq+6=<2*2Li(v~+<o!y`0&NUW}MXD?o9s2Yx
zlx;H~Njwn?8dKr*<~f_6WyljG&%Yc>&}Cq_0dtUyizOwe-fhHDYx&LDx=vXr!^~UC
zmb=vZgA+8iy6MvS`*}mV13$DEQ0eqb=bcsW6+Q!~Z-qfp$S{o`eLCM7=IvqasNlg?
zwAOZam{%>{k@gi%Eb5x|#^Xg;=cBa-Az;&MJOCV67#YqSO}l?n$9`HOMVy~$U+rc`
zxq@fEU5O&>Bji}P^1-_gm&0@{X@w`8zKv5ZdiKUVddXsHfI@&`_=pJPGWNm2D^@L`
z4f)#jJ|h~j^bR^jY<KIoG*mOKAX1G(#{aIQ<M#<r>_h|8)lGR4QnK~mSlZ9!<u;#f
zbPk@3I+{NYw~8{|6SIGabfO<Wy%F%?3@H5q64;8qd{tumzZKHYpA^zT9G7bF&ZP6h
zh%;8-=c_;XpKV~dt5Yp>AV;U+i#%(aZq$B;he~-jSoUTf7dWy3??>}*do&E-<1D-n
zRfRYz;!<yHZnLw0^?%ew$>2{v&yuOYOKA(a-2L7IJSPyRpB0TP;QNxzof5HqFa7b+
zdsOw1HNmw^#xd>mB?Mx+ds`}`z6)P8vET)mWC@O<+cLr9dF;{D9S+am;p|l$bQr+Y
zHI>2K^fo<W<A8|zJN|xG%p779e%U>m1zT^96>XY-txOU45>Tjqxy7h~Xwo79Pxoq}
z>v`iiQqvZ7Vb9B5lBMw+hJHz#I=@L?*7?2RBJmc}uXact$bX;H{<+-|)j_pHC0R|K
zE|T0klT7#fRSkIUcA_1(&|o39Cg*qp2a=i3RqB)^+~X4Ib}le<d~Mqi*UG+p#*{l~
zuzi4(9Jx<)mkgnMDwLo)&cR^A<(hvVQuzP@`N{^xM2Y$1y~nr5loSk+Qv&FKwp6Yi
zbOE+C$u|`~hn4+ZR{Xd5xzrbm{RJvs;cRv2QXOEa*A(OLG(tQpD{5m`gLYaf1@Tb+
z)HtLsBr3+zcl;pKF%f_9idd#Cm2?zuuDp)MIJ^k}4P@C}=skLiAr<?0_Bl`jip2Bg
zpYO~y(8B`JtW$OD#wC+m0H>d^^StMC;mV7fEnqnstF%tx7~tCMj{?~XL;1l`*fis;
z-W59amPB>xlWV`Uxy&0F9xpN(#!l6<k0&eDi0ZGdn`z1ahX9akHZQ+g=}R$veE19A
zP5&&gBt#dfuJQEb@B5fJ=(}+bcM`pU(5MIuu)}h0)-xLQMOQh0x4~>8n;-e|q0S25
zS#Az@?lR&KzC&vb_fh;Km27IZRY&7%4LGgU#gzY2uKti-+&eFK!4zcYSL)Zg5T^s3
za=+yPNzt@%Er1}>8n>NL(kwT8)?~0%*EpJ<A$Pr2e-aQm8vILVsx=-?tA0fMwcgjv
zl%Q|FlBa@zp_)nZm~G8iB15IKjls=^^AncN791xiBudF+oHi#lfNFwY11n}aP>jal
z!MA$lt@o|XTfb-j>XdWw2SjZ6=!~j1UcVjx2IVr1qWIvXaj}%M7PYdj_E>Ukcc`S<
z5d6@X)4EM@-|QVlnLc0mUxdL>nizq9N24jvm$v5JYQYQ-ZSu-8Rlo#M)|gngW+?-5
z(7E2jn{2rr{q8oDaXgp;Y*HSEod3UDJM>AsgUx!+^za)B8XyziG*6-ba3F(*CMNvM
zF4{P|l=KJlgkdntk4XHiGTz(`Ljpe(gMA0g_-;)}QP{AIak%kDI4-j29b<bry>qq-
z>$&D3_oL)yHb`WsP|!FAhZ3GU#Xqqrnc~<RYu8`*_23M}+J${R^Gz}?!w>OXv=TW;
ze^zK80=4OmCDBdgE|cap!ynJ(jNi*le?^0zsVVnr`t+Wc31k+FGN>^GYxT(F!>1D{
z$=eo$)E5Bsa$1W~AVk^+Cm{AA-tnQKePfF9uTo!~!#U%ZhF$T~676bcFi>eFnOTMV
z%{mQWW;v-REG}{7=f5(vO?&{z`HGca8mTcHphp#cNI+`JM3wg%-4bu8R0lJ8ZyiYp
z<z=<tz(7oI{lU7QShMR<SJml1ObMJ~tX%tLUjyf$`R<p>%V0B999!p_n)8dkW2@rQ
z*L8hwlUA2Y?L{26NH+ZVX;T+9K!(yJ1AM8Ps{tq%i+9s*in-_J`fyQXmcAiLC^+Z0
z>b64vbA|yU-TO7S<4diu+RosE%yEx^R{`A9`mAnDwgBKomkcw$qC6#-v)fiD*d%fm
zVYzc7@1KKTD^|mGV3iK7N7r%4x);4V>=GTY)7JVG^J-C-;<D9=tKCNBjfcH|Q7ds#
zv%1*Kjs<_^0hLWP51_J%YnxN@brrQ(?D0nI+3;-_5+TO`1pJs@aKx-jnomQFF#y^(
z-+b7oU%s^Gxkyw(to3~;N5lbm+jTK#@vNd31aP(dQXzLS>0_~PSU6F}azR)u<;E?+
zeyUA>kG5ECgyI3ciF0;W*j|kkh<at4FU)>o-LAIX!q4-hi)RPt^L<Yfy%2w7R%&ka
z-N>8Ek34N|CF_;fj=CQI$D6=(mRr@`Cb`^UkdeCk8q77;%I#W{a=8N#i42q9_3G;n
z+I%SIy#1BDf6<rCTT#qNWXg4%6W0APNf3INdy1#Tz7y_drYn(0_9N-y4%o<L0}25G
zoiaS`tIJ&O=^m}keBm*ho;s}xeJ;Co8WDEDV8zL8SY-|sqxp3E__C4sJzvHe{~C4-
zAmzm#)F~24(*RQa1f!rx+OV0y`Uc{RSz0j?zmMqJmjCNstl<975n~xwybk`alyp%(
z!5#v!;7ot}j?NVEWvWiix5@!`oB8h3)qE;dtI^+gqe6_SglxANG*UA-@e~lI7e(VT
zsTg;_nkr>1U0(UZ|1s+f(Axb{F^<(Up1qw!^fpH8ZDzOky_z%CO!Z9o4>q6sLJDJ^
zq~?dY(d{;}jA!qu9rX)3$4Sm-;RJpZ6Z{1Pk^+pC5CF_mCnr6_OMCYHEDP4dNj1#a
zzCj>)RuSh9zuX)bj;ADlNUlZ~8zNr{%@(LcQc_XLY~)-^d0aJ9rAB|_mnWedkL}zj
z^@C4QJJf8Lt^ObNr0~T*uH#haQMsLeDcMia3yZii4S1umhl{H8ob1^zbvDGP#DF-X
zEj!@aYoYOT?<%5LzLeGns(MVDVx?EblP3Y3a)<;F^|B(SsxK(+8)j{*e5VeQj~eqI
zH}jPKXmz)m7AJIB@dA8uxnZ2qD71QoAZq!9^3?_T0&UzN3ND}-d;E&)N8j!e`ugX{
z@69n~*>?)8Jl57H6Kl5MQ+*!naGt;r{#LY;ZO!IAgNik`i=Ae|uV&>4;3jw%19)~f
z9_)Lr1Za~b2>!`rcDjv^E9hoCAY-}=Pmuc2qeX_<atDZ250U6CJNl9zeXSdPd^GFJ
zxq-iwUWp@&0Cw?bC6#kJbods`*Am(<kwqVKG8(YGO7Y%`A|`{6TUaQ|Vsdo>y92l9
zloH?eucXq#P4#!qzh?hm6G{gzqxBOl>l}?{9VYticlYykKP<B6`x_{qx@dl2ijj{#
zrz1m3j0L0+ac$@MQdJ<&DsRgvv=pa;GOb;KU714nHj;C-Sjx9icLPMeXJi{b^Ta~A
z?pQ-v@ZE`A>9gJ82s!z4H3re2%HG>5q55bZ1^qDZYD0-kV7ZDyEA%7K@pS5OQPPT=
z*gDOo@E?+L`T0un;nr|UPF49T6d|DZCkysJSpbf^cz5fhH)@d$Iij{#YC5o1ra?;R
zR?kzeyzcF|#-CYEN}FUNn1wS{6#+7mlIBtuh8DBVFEO<R=-9`0Fja><rHyy@H4C7$
zas%>x8}0#?_K)K6Kd!OwHk90eAn_Y1pPlXkt~s}CUEzXl{mS%r8=ecml#lj)1W0mx
ztq~2u?r-zH-?5=V5eOlaRQ!*fr+@`eau%|vV&5FJAE@XxUl&iabX2R{vR&mV4KC$N
z4{O*JAGx|7Wvkmdm`EoRPvtj3Gbo^%4UmIa40NE0`B}h|`wHZZsFSs}smrpSO)k&j
z+4jXm$jkVI=g---E{T6DEG}t-2>MA?s|L^LPQ4z%ui`+gwOokb?2tF=EHT-c<=g6J
zbYEi@Z^+8Pwrf<Dos;wrTuEZ|B1|8MoL9WtDM*^93#z|{)Dg#@J=H=s>*XHq#U?Ob
z12R&wD{{AG2%u3(B-5+>sMfwcDxR;q;b~uQjf1maJs!70FMJl=eAE@E*LL*hRHt}M
z<2ZWiapP;PZMmJm_S<&sJwRAFV(g}Spii9Ce3VJHNobMRf+%V}-1Rmoa&C0$AI15J
z(xs&WJNuYrCn_H<R<7ULbwk*O45S{hxZsRh$poh%+~!QvR?D=*6SLOf<?T*!Yd*bt
zUW<_9WiNxPIjZ{JR3XzSY(Sad1gg#B-tiPVYvMGVA^jUEeB;jdAj(tUvM=<_zWxz|
zMq<M#r#zX43m}0qHKS$#L#8Q4bc==4>51KU=QcP!gY8&Zag)N>zX!zche4v_)2#ZP
z`I!c@9`Um7q6NZ+Abp4Ol+$bOKZ){#%n~gj_`^Wf@G|i0@VdCf?tjhRPt;Fuk{vM+
zDc`=9Ul0@_yqPa0#S3Kq3N8Sn51Hb*2)cKb!Dtoy#g#hiU1oMphLlfDC3P5GN?$@P
z`<a>#9PmH5$V*8P8TD^=2Z7Men0P++_EPFCQr!0n^d?2OpOD<Cz8%U)SDF191q4Wb
zXqtAI*+TuL9A@-XIuKfH@;3R4px@npN(TbZF{PXel(Bkg68k<1gSjaTl$4oW0(>qZ
zpM19Sq(bmAzHk3`=>SCzX#|>=Ag}F)!}{3DkX>D1oJ>|1aAgttAlcpqi4jvfY{kd~
zvHjeCPitrQlxTJgeD$T_&9MJ~yU|a01xi?0F}LO9bZOQ8ewoXX>+!{yS);&%md4Uq
z0+&0$6JU*@bN7y^0+_kpJ%yfb@r<U+$WooF!981iTk2h93YSN+4!`K5<n;y~so@pN
zLG8(^fXzzH`f&O3Ws*LojsVYFqlNEB<F)2g-8Fi-{wbpNWelw?kX6!08>)^O(`Xi6
z`#n#)mPQ@t(ks2;gvEw2GewsvQE<PKr!hFqc$zK>_C*9Kx>PIooTc5Cs_Lgf<6+Ca
zKBkMT4#P>CcmdJ9;hYZV_e6swgVDvgtKC?-2a(HLo#S&-it>=b^S@Xee`K}vTmj<q
zdO`Mw*8h9)iUm?X;ZfogZJ`f8^hTY%aonGsQK#`EJ787N%&)&F2q^f8on~}vg$u?_
z2oo1J%1&ZGX89lz!cQ>`{M-+t?Zz!j_L}(Uyep`CG#@fUeAa11!2@*vA8TJ37KPe&
zi-?3scaDH`iF6JEih#6)0-}IOcgFzIASED5cS<Re64Ic+&@Bx^Gjt4a9`?K4d%y4Z
zz2`bV&d>3>^m3lLpXXk8thMe%PYLKO$w%R<!-kiNG|K%Qy?xjeywBP$s6(6xB?Qdz
zo};}aX<~efScK<B5^A#g3fUumw0$R+4-2L@oJb6x_s~KI{tE1kmB3%&()5DxBv5w|
z_$%x)Lt)+9MGCMKeH7u+kOW)#>bxe*E3~6_pf-8|HAWOc2SvFvXB83sN#rZoAu($a
z#^}cO157!wUzw9H{O5(MD7bWIA4bxyy4^MVgTXBAO$1hay@~iQ6=i`QZ2$v}U+zF}
za>OXeu!^Bg+EA&cO~<KmhQ+DA*EP@!uOf}0D%~GM=t(wXZFu6asJmH#;p8=ca*#Gh
zci#gEHHMWNq!YkXk7xV-B;s$%<kfKTCL=g?YVGuo2!iSxULwomQs`K|wfM)^kDHVT
zgQcf#9AreKovID6@E}RAOWN<Sl4E&c*fZ^D!Isnv?J3KQW2bfe2@{S7EF~z*`LL{7
zGwujqfMe;>CG*$tUXs_}$3CYb$aFWq)g48b)Zp6++qaBnGxrzc-O8p!1_<yeDx<ls
zg>39=`LKLa=#m%OM^lEb9do1^6)HneYzrUO)`vssAO{^KVp1VS4xgQ6o+Okiwg;O1
z9M|4T-ib+hCwsob`hLSglY`Zn=XN_IdJi{{A>{Ljz4-u;rO7<zt;WFZV}fVM2A7@7
zbwtulTNIsG6H1f^Sto<vVOi=Fk);MZN|92HL&{m1fxT+A_g659OEny?FyJbk5cXgT
zFk=kMP@xA&k25rpzbu}paJ_2PL=1;KJ01>GsX;=2{o;9h7RYz$P-S4JG2ZhX^NAF8
zvdcbN)oG!1vu{bwCe_iJo|5ZXPJ8b+AcVM8yI!Ky{4+_o4@gCdnCrEHU*hlIC4Kh~
zD(mg2Sreb12UvX<ZH}{4P#GNF+bmVH{!LG3r4hg@Lh;Hg!W)@jpnrXpw51LMqG)9g
zTWf98sqq%fQ}&WOF?WO~NUi0eo|A3ZKL(H2vNV@dVw4ek(8Mq1E<dP8gnWZNY<N}2
zR=vr?^{p78|9!UiYXe2xgm7Gp6SH>+L9pBXf;3o4Y_TYIK{*5JJrA=0v*cQA1e2j6
zP6MINPJ&v8{7FtnLV4Ef$Av}85GQI@`*=gXdX7+KgOde5c6{%#=1?*-8zFQ&JFOCM
zs1AK85mUdvYyLwp{?&|lU0=HlY!R9Rz^uz)W!lsd`w0#DS=!y^D?tQf&)m?`dW4k6
zk>JtNhUzr+N9QY0H+_@yGuoF%_odCeb0^KC={G0Cm@z%Bq7Pqy%@Vr~TF1xtT7VM8
zgyc@sb%jZee=f^YqQ?hIe<U)~^;>500hM|5^ZgA74_fdsMgz4QrXAiaQL7O0=Fa^n
z0<7|oP_@_0l=B6)`X0_ZZK_hAjbjNZysb11&UZ3;zmE}C6TW>QwWtn75qTkH;cfO$
z5>gvlrRQrm7QgdY&7+P5bg}hshGIboghn(+Uwrgs$wEEW_a*Yb?z37zB8%2<C`Wog
zf;cYND#AG)B0_~`NgzlK#Uc#cclv2b(}rKf1;eO0ku}m~Okv7&(Dq_fw||5fmejyP
zFf`NM$e1%PHC5U%mdMOE*8yajc+V}X#ou35T96TB6bp&<VZjy9WM#3!NL&-!7SfYV
zpCBak%N_eh!L5&YXyEqQ(|)5dyyjyMVg4fwZGH7B>vPQ@Tld}<{~@Q)^+#g99kpcQ
z1Gx8lFdSzhb3np2Y$cKU(2T1)OW=0R?v{x^)YpI(eKnMU=Z#Z^Xjy-h81ox^*me83
zG$EJM*ffc5Uc>SDG$1BBS1N)aT*_1N!M9pe_ljsKlJVL7!9yoSi&#K~1sgV?5bN9+
z)}QWnQaW9e3B`MX;aiGnsCW=daB)0W+R#9aLeqPpARo%+(on-D<n8=MK<YqK#mM|I
zH7=hh;zt#S%1vt&;aZq~q>zllt)-l7y2)WxsWDRF-YAt5XTMC#r6`*utIu8B{H8y5
zv|<&no{D&nWvi*bj?ek(_|_!0hXvOE<nUUEtqbYMrk@fb>NnA<&Z42!TH?-*%Bc6H
z<z;I>J4j}nM^jHZkJ>gqmUpfF{&)V;qW~-o=6|hm+)qSAm7=a;p+(+LEb@)yH7>`4
z;CNjNAg))OXV?DJ5vk1l?Twf7>O5*doiQxd`qjdVScp57xG3QSHf_)ts{sWC`i#Jp
zheV`*qzJY<$V|K4B`}M@ykE_WG@R6T?DLs!g~<x}aSeoc8Hqa|^y9P;yxisM$Dq4F
zQ+m9ypHrf5L89dSzyw)%yITR$mhmJ_gdDJE;8+>i#V|yODg-0xk#M(CXs3L+nQ<1u
ztK=EsmZ^{&(G1JYsNRR^*TTBNsP@U>$(Meb--;W<LB2$0g`016X1suz(uWH0VS8%w
zEuNpngLJLQLz`j2x*GE0XLanh^=l0$i3xX<|9<ZNB=9#8=F|SM(n_wP3518Ca`*bz
z_$0Q2NurlU)8!YV2!fAqt;y2xTtlIcvy)NO8RbXvdXwVLh&0*;T^wiVSj{&FO6PMz
z%Hb>FL@Q5HG`zh8Lz!U#yKNVVvAZ36Ee|11`x$aW?Y~|y*ck^>Qs4H^?3KQej>+bI
zE8A=S!k6#}*<*)}Mn?2#U!4n(WnHf2*uRi3D<CkAZ``O@AjuIyAcEE85wTmUj~(o?
zjprci@o8>jq^fds6N~bo?shR@_HjA{j5G^9ksA^mhrvXkmK+*y`n}HMS11SKOv`<u
z$YKDXxu2upgY$&-=l>;G{s4t)-~Z^)b0_B>`;(c?QtrG>BZi@&M*-FNY-$W~A$)sc
z_*~va7=xrxKxHT2ai)AQTdD|2w<^_29*<sflrHU9yiSfzpwyULWDHb_wwpIHK!@2L
z-i9KGNJ4eX`z*<`(ujrbcW1bFKHq-lmmm|;GAn>{E%SIxz}!m&{&J<M{hFv83D7{X
z=Tg=cyLMl^w-nsyw0RCF;P(o9gm4Tkl%JRqwq-}{{zwx-5}sP>j}d(pq+cD1lES_)
zx7xWsq+@Kk>P!WC5M9^;*V4ky!7fRdq_dFSU%Cw(S9nJ&^{_C}3ADKOK#NN&#U<&}
zKe^EVJfJ6c|8<Qrk`V;&j{;cH)Pr7M%(JV7K?q0Z_**abzY}EQIB&!ePhd;Yj+t}>
zMY`=%jQV+DgH}u?{TU~@%aRtpS7@GnMu(~drXgNAY1GBooo)sEwGgMbIwu0A2dXOx
zX+~y0j|FuxUie>^qZ7QZoa!QBru<%`A(87Puspb9oNj()d2l0_61+rZ_Hg-Zvi1yA
z1GUP{!)jia3G9otQj&nwYHSxHM+UYa2Mwb<h*X8BfQ=!ZHz~w007cEqS8iFva>}@y
zL;4-tP&<8f8Am{v=Jx{SO(}OFr@zkU=GFO-rS5YiIe_!w7eZH#PjO2s`LE~0b#*>3
zZzOhIb%>E<v)9z}<eoH>eb^?bMw`b9v0Ko}P_-g457b3Gi2SZ2YLxm06;g~ETR`D@
zC4o(m^a#il29n-HQx6#*E94yTuJV*b$?=KLa_LE7nh{(ZW{Af~b56zYqUhD%Zxd%m
z7=iPSJ{nW;@!G8?XlVH9H}5DX5*uF24-MCEmJ`$-UWy_Ra)3uw`l{2H-h88cqMi@Q
zX%8Z>4}ru!HDgM|wccfe3VjuPebDg`zb#kp%3?q$#v~3Rjx_7IDE6+~V@a}dSbs~!
z?=_sQadi|qsiHJhzrVd&$`w`<>+=-s5d&a(l^S?O_SW_c|Hc^qgjHAX3<uR8yEBOL
z<oPID6vMv>@tgfE#6OtJ^E{w5NJtcJoR?;l6{{?UWF0!>32i(b05x|OL5>LSyo{O=
zQ}8?oW)9f16LTYet7&{NtfphN1jP6!j(-HdyRL5KkdXyxz#~T&&=zo;zbmc${+%CV
z;;jQWhoJ?jV;li5GQEYkMl?d~I6%dTwY_@`X`KayrIwV@l^B0$sZ-oqcq15k!#P=n
z*2Y9Xw!UL~C@~_E);{>tHDr&I{-l5;R@Z87Xt-3Ge?U7O`RGX*IE*){sd&@Bzkj~W
z6~+o=t?$mg!dPzs#(Ks2E7$Vhs_wri1(fJy;aAuQ)BQH<sVpZvMErItngn`4B}neq
z;7amc84V4I(dPLJoEK=$Aa6k0ND2JhfZv_NTokXzl2$>(FH(p3a5i%12r!)2<Y+vm
zJXVwsot4Lz6V=Ll0y_xb>)_kTp^L&bsDE&OeVT7A-H(mjKa<B>;!)EzX%-f!q4FAf
zTtndxRc|N0{9=uhP_UD9koQ){H72b+6k0BdGq4))B1`2~iWTsgQM0&J$QilMxI&0~
zzw`ppN$_wjFZnn>Y8kL=2+eHx&?DqC-&~d8Ub|L^-*1RC*;QuekXY3cS^}Bf6aXU*
zaioyJ@B8*wHu^Jk@^vw~f!}nOKaL;E%-TIWbV7S{-R}?3DWtK)kR&F-MYridOV5E&
zNdj}czw4oa1TLrsM=6KfAKnUG8;gg~I9KEC195FWgsg$kL7VD=8U&cy6;sBA7!{Nt
z;p82RTSX@?ORLz->%||^QNCrNRegMY^u?+S-8Bp%86)42mis_!bWqj1EJM~w!%S00
zuLh~E;$*L)!Xl&ZWs()iTRucVV|~H~h4IZjur5_m`D~O9I?IKq?kpiGdl)L-um=8h
zSDs?BxhqR;juAb=GV@1KGph+GMStgDza?tFt?63s;S=S=<Ch^&*R83X#Ev&K=z&1*
z``&jGpe&C4@z?K|`p-Lc%WqdE{U^4-nWvM~E|#7Ma?0>ZBRQjI4>lojJE+vRiDtH`
z-S4?R<~RcP63F_ih2_r|bycd*OQy#XIN#}2wZed`?Q6m&<~d%hsOg_%8cTSL6<cR5
z%5$CMM=E=hjusKCV8Vd1wpTt=Sa_sZW_dn+9Nx+O`Svi(e38ToBTEhY&{7s<pEa~Q
zbaF(>l;V}uP?wXuy&~mMxavh*1<^)W!8b%#IHXzOJ)WfXc$ey}rIsjJcP$-Q{J!d>
zSyn>piJ;0-8<wY|kJAxGVhq5A(yui#t{xW|)nvTM-`_o7<LW-;ur`F{rhxlo1@6-W
z*6Cb}|9YPyuI|%Vvw0Y_VIQmS_(7Z>FB&h}eXJR@3%?CQ)mEDHG<-v45qt!xyw?rI
z{Sd36oIFz1H^;R0unp?qc8ZmUdDY##K8mMI<q+qkJl>O4PM#Y=`}=uvQIRm(#Hi8{
zvm(gF8j!e+Abtll$gU}CzZlRqa0id)kRV?>(OLgPge;eGs=`9vaTvousPK(wXX2qW
z;_qo<$b8%vpz@9@H=rQ%?ze0h`gnbkAMgr}hnB*XotF|y_v5<nON<LbjM7KRRhX=C
zS#K9yk#d((#c68(dVnLY&WO6t!?_2TW(VMaZbfZxUC+qx1N+Zl;h^~A0iKOXUIVfw
z-@m|)v1x}y<w*S~eCh9n%U#T?wOXHWMq2GN<I|$!&xd*vo)z*KwlPu27=jS@aI&zZ
zKQ82f%(FxFv4Jg~;#zC7Fpy}HXN4Md@R|LHjA~yi3QZrZCVu73iDCAMEr=zSUIe)$
zODCS5PD1!t*IJS8v*yh^YhyQvoiQZk6V2Wz8#n8m<P3&p&~;=-v?86~#idnld>_BO
z4Q<w4gmi@MnSV=6QjCZdS>rUcqUA>QXQ`%1JxoVRKlu#4`MbW<h`(Q3sjKMlUimg*
zdiC1If#|S){cHKZzP7*?=8sC$G!wdfa{W0*-wE^1HA2N04K^YoLnFxAJl~@zUOwk0
zIzk+v^n&CgrJ?&beL5bP|Nis%RyBhYz}3uVWeqk`o0qj(vk0e|(+a(kr{f0lbW7@9
zUetFmY(AMawGO0HeVAy?qgN$2kZ|(@YuS&9B{}QtninL(cn!ZIM-_6iBh)$)n79!;
zBuGrd17;qNcH+r+)rN|KqW$4y{oO&zd3!dBy=>@ST%?f4($O$RaJ*Lvcd|_-cd~EJ
z|9chspReri$5#Je(-V0Ia3W{d>CyX8b68F8vo%mJv`S0jMl{b2`K0ULAw=8vt=r;w
zbG-67qns8sU~UBVu$jlFVI9)90xl3wI2utqzzB1`PhvO<wI46lbJ_IpF<ooe9*595
z(a9^l@w8oas>wBdR=a{bQr}x&z3?SYqjfrNBUZ)t+~8GPWl|eFC&QIIkN3aF(nv_5
z?$eV}*^V|VW^V7(PSn>X?9YaYw}E{wuw~ky^1MAGHdOh<OYnHWl=e8kao18JOo1yq
z9$GtoQ{Y;7*u@p#mGTyH`<-CY*9ZZM8BW<POtJ%j&Jh5*;S`UgvfnrAAKCbiob`n5
zPe8|h@dB5%8lAC7Etmw&UC!&8J>@KBi0W-RDTJA3UI0H~Rk?8%RQ}lx+0#I=lVL-?
z!E-_U)p&o66ZTLg71RJDU5&_&j?XC{eng;#sJGxpERQ+HG!dvh(NcHi`+c3d=dZ+c
z>&19oSTL`TrnBc5XC{6qWpTb%paI*>kTdD4e2<nr{0!r>qV$^=neJ`&p*PawXaRYT
zn}#`)vd~fmQ2W~ML_N9}7p82uM%<Rm4RKln#A=hcG>za=Oa>n%A15S6q8{|(c;V9s
zO#9B&l&5ph)c-GWl0tGYKzy=UP?{Ke#VOSRA9~oULaBcjnZHx`pGCEFAk$UvvRHaA
zBAARG4V}Q(zjBR(A+CwYcm1}H;xPte(y>L^oPX-O3t&~O1$7ER!591iqbUwzXivAO
zS|BkmGFw_57D&)g(Ip$2-`M?0$TERItc@XfEi5$BD-J19zVL%+bOH5i=q`~i-g?Sr
zoA1Z!06z{M=Kw#)+vsGFwcWNLg3V(YH7^IddD7T>m_LL`7%;KC*_PxDPP%exOEIAv
zh^|tpimo;WhwxICO%Vd;04ZXpRvA{quU)EiF)1#|X_dRO$?Evn0#uEGgUVNPga2=!
zW!%5(rT})|>C?cV71R*>9FARxi2w7cYa;lx^Vn+gi%bo^^WV9nBPp>P$~&<nzXgai
zI;e*w-KOK4*WG9OPcaBt6u}aS>zm2|RANzkrq^-we~Lkbq3#=a)S5Al8LRbYkw`y&
zD<)<o_NN$x=0C(B@<=j|7Zc1RsQvf5X%fK8GvD?m#cfPb{t+MIN(zFC$@-M7q_lSA
z;@`v|{%77<J`-ODx&#g>BOA-ls?685cjX6jIjeaak0od!s!8)#OvZYwFB<DYHaR2K
zvW*WwXIvHf@%c9856|GMwvHJ_;?nu!Nz`v3^l!Wy@P{wl?_VJsEKhybQ{ycouLtyg
zBuxxb*G-SFRm@@B9xEuqgQD$5$bCaU_^#p=pt~;%L;wSArS*c{aN-b%yxLNhlU0fi
zoyVwxA1a2s9F4nfTJymX{^fH@5d1b7dFU$J!g0gLx8YJDr}-tc-^ZdMcet-;dIJFh
z=JS^6W1t#W45f3xap6HDQbxm2*EhiK`KCfaeJBD&BJ)c38>pMwb5Xk)i@u$22oq3<
zC#Mz&d6?XZpwuoK1(HZ{+?+&JY$+~9h>ysDjW<VbfrX%fR#L*xG@Hl_L;clza0P$v
znP&Zc1<C%H;q%YDjOe{u%C3pmd0K4t`Dg|<U4Js+b~NpZbVZ;HCW^lmPBwLHj1?%&
z?=)U=J7me9^PB52hXY$r95N>O7}?AJhKPcsU;S(GSkfXERo(>oi@D5KgF%O-e*-m#
z0S_XPHuNG=bRzaLku4SJ_E%26KCjUcd~C1Z#sC$jGFAb)C9SOVLO33$MzM5|rH&Gm
z`2`j%OzGB1*d;@fFf(i*1~)^Qc7mE;Y}otj%S?=`%gam{()PVc<j#QRJsdePKMp!(
zFw*cz!QQvPy*2+rhjt=lS93s>FFWe2>tUz?>wu1|p78_K=Ml1e-~}u95?n1p9g{$(
z3B+YWQ^GYDvhT2X3-Cl3GncM!RoT^!gB7O#R)BrkS3O;x+;>c=GhK@_JN%?NwP3u>
ziKts!qnpO0IRYy`BN;(aL}nJ!#K5$K-1Ca2+fnf!&)q*;i8TKbeBnz36+Sha=&a!F
z`D^L2a(4iVQihEU>EyWG7#oTb%Y?PwCikAD0fQ|AQFvBXF>ENNy10R(tCVtIr$hvw
z&&+t#Z5TrHhF*NZSHiy$KO$D>-02t?Zlzp=oED@E3(d0nDKdn$G<{^tR8V8&LmMz>
z922B!LnG~E46)kHYo+gqOVfUrQOpNWLnRMk8=y9Lcd_5YtsFBvd0@(bDTYF1)hd=M
zV&84Xq6wZjW{X<Rsd=PIZ?$i^74_E8Z~1T9=>P0`^b9~(`TR_~TaTt$A*#-fQfZsy
zTpzY4D)q(>emN<P=33X8+_qJ5>#9F*=Ty*(f4+X;LIVu(dZ=@LpCwBiSVD}7GXLFz
z`_Bsw7B;{{?yWTc*aH7J#A*&><qvipLud$=uL>=iVnF<}GD~L4{dqYxR&x1I9r&)&
z9&3*_q9UrhOE2PfkA_<#Aq!f!xc4)Jn6ac<Sum~dL<TD_FhY&V8qp=+G@e?P#I-)$
z2^VGRj!*{NI;U0*tBHg6Zs-SF->R=)ACP`xlu6$)ieWO?gnXH~s{^@~75?ZP<CBVv
zSfg}Snh|^DDO}sYSA&s*3c_cib1UV}X;w>#+wMVOdLERqu{vsZLDt|;avg;N$gq2y
z@V6B}I+U1<3*7D5*w?aum3f7Pf86a@{QE(r*xTPu3X?8-gawH&X0f1l;+qXsmeaY`
zW#0VTmPfPGRl_@zFBTxR&LcA-2!<bzy)Nm59oq$}+M-+@J5>g1;*IHpo36+NA8KtE
z_`d`y(-n3R?KBqnO;u_oO+9gJa+g*$Xr*!r*|*Sarfbs4tej6z0HPzI%<3=^yfyj9
zM3xAqeoFR!M>sDxLhx60qZ7xfyowPUi`dfr((=sMD8;r6?wAM{{e=4auaeX6Un6%_
zRK*rVBTDnfv6sdS_1o)_Q>9Dt)p>j^kWM|>it|jfp<)OEmW%svUOu2$iu!}3#DpH+
z?MDBLR!*o*O=V&T=oVBrP)|JT$E-Wmn3sp{hTqSjMM?}2=yCqfbJcl)pc8kp>q!)L
z4JYSu(#%uloPA~S2gX|qmj3acds@nKdx@O;+2Ypek(sb9eOi5j;p(^L#1e(UT$}Hx
z6PtF{G_C{^?)}S5zyY=iB-@X%>l*s8bQwlu_C7iG)zkH%nxsjG$&_~tQ;sVUHgm3C
zqXvwGW*)^=F@QNEEJ5}9&87Egd-1pKHJe}qOiWfiKT{lSdu~vL<r0}0J-InLs%I<d
zIu}TR8N8C}*^5Ub^r_5|=*O!BgXJs5LW#o}AD<(jdO(Kk6)Xm{waFLjbkiP5w%byd
zCbgdKQ}s4rJ?hQi&7Q=%+>6UDnagpMcw?BR^+1NDYL(5*rhbC6u*;V`Rr~F4$Jt58
zr%I7C2{HqwOT+qK$iFSjPti6OQ@bY$n6Hh@6gE!UC8bPXjjUY_J-X|KFUB+NNgEYn
zS!|Z#b+q~8484$nXNP)Qc_06s>-P62u&Q^(dYkPK9)=j6$qH}m|4gH~Wrq%ylPczJ
zB9>*R-V-o;VDzFn<GWqIVyh4mFhDP`nS~;~F1m9r;;_%lawc1sA6B+672O9bnb#l@
zf$v1jeVnMXN--fP1MeeXLqLOSE)Q^779Mm_(<s<eW~;no$?@wDLX|Ti%=!6N3oJ8R
zH4P2%PDO;o+A<05kTq#S*BE;d81Lj+dvr*m3$j=v+jGV|!u+nJyRof}j5K*x;~+GA
zA>i__b}+<%HaS5(3eh)RQvxP;;UEVa{F#Hbt%iIq*W*h7M7dZ#zMSHPiDrK&8dvm{
z1{=mx?e)SpV=m5SRxG^XzjBpnj(lW;s)49k-AJ*{O#)3o{966xwa)Fj*QwX+USzGi
zzQ48vtLPE3*?ev(PVf7LxCs!nR!w221xaGgg*5IzieX6+Hy*U?uUGk7u6OJL2hEyb
zpgNcxl1cYd_W&D({K-!{D!w}(&Rg??r%_@kVH7fI=&5`>ecDc92J>!>`LQIhciyP$
zeK0%~=yR(1>*W29OOW^%B_(I-J}cK9+T=|iD(*(sU#bWn{os8k6u*4ochMX;Q?s2J
zyanE5KQ>z?u=ZCEHno~HWykAWZLjotu+D)O=oGMH)^Ew<3A>ULw^3G2mAei9nev09
z<Li%9`CHDD+m5<;9Y6xEx=2OrC!J?o<D1RNlf5ap-#FPv3lYx;GuJ$?7O%GwOPX{}
z_5GUM&e~+Bag9HhI~4cg)te{l)flg&x7%(?>D5V{V+CCrmv6u3&#hI8nwaz!eC_h7
zM16Y+YI#$WYGyZ0OI7mViD0q7U#+r#PauBSS3!AeS2k&$z{Y0hC&Z7xi2>hN<ocML
zL(ISU6LYwo!LZMqm_mzK<BW4pUqtP(OO+2N@E}$>eZ~<=bg6Gb?p7e><*7hU!IrX@
zSdxu;!6f8OwA#NKV$7a!I+yjK6@EM%?BXAICxQ$UfS8<G%#vdZ&2~l~WnEU^>VdAb
zD<(z><*_=22*(1fI|@HT+2FQg)mvO&`$xeBSQp);wGQ)Hd@q?W-#l-y$cj~?i0tu_
z>{W$s&?XWzaGqp^ThA@3=iBk2_k0%hs~4W=Bw?!!T)d<@(yjkxS8UySLLV|R(y@Uj
zoBs5R=~h^y{xVw(Ynn&*`NEpzP0aPG7W^%WV0w*(mqB7X=^GXNy<D}QJJS8}ovyo#
zKQP#R-+0!7V>9yUsk^o1iEt56gQgrUv7T&&EklB9fMgqsbrF{jc3EO~ZqgwRNz%@Q
zrOiNg*Fox??3<lroynq(Q+yqChTD|!LmAI-1S~(52hrI7s&QA1zX~3Wuid{z+E2dR
zYX<}YHCEz$kpCOO>c(R`Ab+;0J9kB9YLqxDEbCUvi1|!>ixYku^o1M6wkOKg9%qOQ
zK>rj^&;J<rF9q{D{OJ)ztJRx#5*@LQl!_cD0G$)!wc{r~7eL~G`t)DX?bAOdm^TFg
zr{>;nw5VHq-{;RBBT@MHzHVrYdGH`h-+Z}Xx)L-o#emw0xUgVl3}UZ>7H&0sL%j1v
zUPuIYWGJu9EdZKO4T(P^q{=%D%Ll=<E=uZ{EpP4Z12M0i1eFE3C<5ztGNoF&NsHnR
zl25}mzWzpLGRbI|@$_XZfskMp>LUuoTB+_Y$yNVI-DHzBdmkei-;DotJD;LF2(0HD
z?hUS6?6;I8Q{(0s#iU)&$Ocni=vGw9@&1ar{h!<Ulzyh;$xZw!(4PAy(m^SbQ}x@n
zj}jC?$BW~RWt8J?C~GY+59PH7z@uv)G1n`xupc1NQpgYK`!MHGT<@aG_vrp@p=A4!
zrxq;CML&}lA<tS*bg7I7vK4GDg!iV`=(Jreoi-@vfzEpw-4CKhsVVDURkp;M;9bEE
zU8+KLrFsT!CVZG1lc%>A{^jIY&suk&tygn-HXOa<i%JuoLeTrnhkA76XX6hmrq9_v
zNRvuhOfH<EwU--%0lga0vM1iggWQ$Q)BN@5_-Q?ftMoLUwkdZ(PjmAw@aLTcjqZu^
zX5*(|J7j5TlwvCS)^3q#TFvE@OZ?e!OI1syf*zj4-IJ+cO$PKH9<0j|Jdwo@cX}zl
zysovE>W!;ntyYm7b^b#;+W<3<Gg7?Cc6HxqUiC@`dS^nDpl%enH6~VBY`ReI(~OST
zrm@rjA?m4Y;r61ZSwPzi@6qG`5?Q1VuIkWF(g36~!gs!<t)S-R)iP!CTZCs{SYEfs
z^b!maAzF({n^(_k#CV5)PGo<}Bu|+x^;-En3K5cX@!4Gqaec)P%z(Fx?!y@nf+Oj=
zp9WDOp}Y%&9|T|#PZ#!h?we4o$hbrh)Lm_*8`9>&Zd_r1*@A28YZ&PiJLv2X5in-I
z1tOL0qn8-w3wuU56E&8z5gS%cd-~N@dXh?s3R>+k+^aJO7X38N5LwT$&qVLy)ujr>
z?~y&}@7Hn6C2(Gf+-&Vn*7eKHV}(zd{T6)sVx^J_fDhhQ&d=JOk&BG3xyu9+&|qDT
zi&b%2FHT^dlsH1|ZG~}OChkWVbL&@ho;!4w5+`x2#=q8G4e}~4BUbTg6dY_ZhIEa;
z3nhz!dG4#1xbJPy!0jPG{qV%+uoqpWcOJDsKu&D*2#)>qHU@k2>=p*E#?-0p^@?io
zwrPfHas@lK`{>MW;j(rFNNKa{*U*WwFHd@xk4#-`RS&lw*?j5SJNYKeAJvQcRbOA{
z?IW!Dw)2y)4L*9$r(nft&O&W;W%eh}sQCt?>^PhgOh1)a*Bw@lkBGl|5d7`XmIisk
zI$m88Nn*Ap!)p6u$)S7!Bk>eOoGFo6WX&qlDvzR*e`TvaL-}KC|GfLtm-Ri?X__TN
zWq15?oo`nqyIfk%jj>!v2%kB1hP#}X`}{N-5<L#{v_`(#6Yugqo3VEp*;?-=odn9T
z9U}E&h+jWn1M$qNFPIIlikr(VI+IlKD#ZK5IlfT10;jsSAyIUHnejDUdE5W_$q(39
zfVQP%vz2_{`kCEo4V3oUsXjKh1bQ{v{v0iQm!U1rP61LIV;Fr(65PZp9r7ULdk}AC
zgG0s&vGDx^q!#{~?4tTxuTYXchKLpwu41=ag&C)aezNdgB3$?2_ow>>QR$kWQ0D#X
znfT`wFiIhnM?+6u?ylYuw0BTaji@NSVafN57Phac_B7un+MyEywW?JpEch^#MiWfW
zfH@<eA~^8gtvzf|_rRaMC~ai1{c36b+3{U%y-&uHV--AWe9JN>tXLR{C9nXJvL}@y
zex}P)P*(U)+Nq=>3&0E7*j3`?bFp}o!-F0{FTLQilMy_AG_&XV!>I=GX0HV)a&mb~
zYH#BUprQnswgZaav`zKa#3%YXABppy{2%KZdS3?98*v_OrPP!=KB_7)xbjC<U%^Pc
zmH4z0{H(jSz<M;-IQNKm#?MrYWq4}g^}xd<TiZ=>r?+?CUiOK2&aHM(-wFk>5X_PA
z;{huxd?<>Hp5I@ZUp3f$b6?skX3BN7>#*C{)$+qv)|nID^)d4~5uUHtdsO68tu3u;
zlqzH!M4dadHWJbf+|}Q=PgJa%<w18W#`=SzuzsqHztl<+Ku-Q-V1}3Z_k64?Nf{p+
zUkFh8bksiCK38`1ys9W_%pjK<{5LJ6=WojYGXVaIigXo>XF$;edMY0nPG{!!tB)V{
zOwNQP^(ZTrjs{jcOEl|G7<;M4*k`Nu$$`i}^1VnXiqcp%PD!YT(>7%*@ASAoui>;J
zB!p*CWhEF7!m~SQ5!oFMtu^voOBf+Fl(}FKXdhrKH2jwO?d#jpXZlnOg8E<Gu4N@<
zP7UNWnax=|4I$Ukm$paT(75^7`rrnm;1Fx@Zj6#66GSRe6=5nIFC|0Er(ak)QGqQw
zkW@mASn|hLuwGnXFI)Dn7ZEnzwVNaG%T0pJk*ej}*`q5D+P+vXE0PC`pqoW@cL)e5
zXc)OerTMqN(OEYKtU{WEO)hpn8b^wLQce;QvY(eL;3!!#%T`GtFS|soG~%V6t>&dz
zOj8(2ocaLWAmjyqeyp_PreM3}4R+_&fnlA)FO}Brmc<HXCoPfUlo;5H!0oVXK~;kN
zkmmFTy7Kn(`i(CcvYU|?*R76s=3JhMsDNL;jT8vm*v<Ih6S%=nLuXXG6LQ-%i6kan
z5ibBUlXuKMuVGwZIdwGNvEgJnTts01%s1st_2>*xzA-RQ8Xf^uLFF1Ox)#aTdAULJ
z)4I3U8R62I3+ETW8{5q4l%);%#j`m8=kN#b>DMazRcghz=J|%+U(8B8SH$5%!TYwB
zhN867L)H`h%78*%df3|JX|-!|L2mm&#4i_?yuUY&BM_)-{iR2!5*VzLbF_q{6`{Bu
z9@j4d#%tvH{tq2P?^hiVAFn3{?o07PeG+@w%VXO^1)35ik=Nv{FQ{Q7q(6Y(gc)zc
zaVB?tcWvd2dzIW?>0JGBHFqv#^<%r$mrMJ~@4aUawb;>n=0+t<QWBo=Ot5+hj$j@&
zis7!jX3li^(IAh-s2rS6W49F}w742<oW{%Uw;b{PvaGHb%JQY)OBX%X1i6nGBmSkh
zX?4n_Pg2*DEbEzudTa4bS5+z~fqd_kVI#H2`oB47&;I=h2c7$^^q1EkCU)$U@hKp7
zc1Se-`fQTdz643-Ok>9{WVi|5lx1i}!Xw@gWNM3c%G>M@<dhFA_-8|07V=KV7^c$)
z^~2A;Ke-ESU)1q#FUa0&3Vz0Ty-U?n%X=R|2&(M{i*n&-PqO=QWYSqOX$d7N%#?pG
zpfBIejf&P3_T_UAS!(-EB;O4d(A%Gd?_z3Uf(7&fDR;Od(F}+Fx4WG!4<s?fWw;KI
zn&<GIwe{us;QZW?`{ByGQo|hyUJ}E@xT3{Tbp2DvmPUi>S}{MCbhWwpt2gO3#K^X$
zZMkw-TkmPz4xv88BG$Y+jvbk?+^L?m(Os15gZ5lMM-&^N?1oyv(Ac(aa9{aaqgPdV
z8Ax#AgAbYQqoZfMn|JDRQx{gVv;mjJuN+|@C+=K*ok|vp-R4+q{q5Xk@`gr-5b}Nt
ze8=yUilHH=d)fPROk2-}C{K8{w{9a|$MEaYdxH^RKz(6`%Uhu~w0GNMu`|YHSa7sG
znB{BnD-dX3>knTX)s-C?+{TwYMHp93mYiJDmV`??+Kg$LmbZ~d*h9ScV^TaXZj|>W
zq$=@Rv!+J-<hNLwT{hB3k0Y`d6KQN~yPhA9HsWp^-~rtR8;JlV94uu9IIZK5dL?T2
zBv{fv&4c5e!&;K8g{>lrSZVRoHgJb<T~~<8exrtTzCG~m=;Ev$U$V_as8a|F;*`!^
z-b#$j_}CynhJ)C2{q;4;rlIEH+vRW0$IrJ(w!~$&goAH|UT*f5KMUvjW_L*6L@R~b
zk*xz>=46RqQz-O)UYsVJThGmwy6+|WWAGtDSvR^^v0gW<?MUxTed!AmC-C0w%A5Nt
zdOpT`0^glj7L1&78|*pU^4xBNo#yaiQ2#PM9P_RcP#$SKYB;l`Wq41O=bX5A7P&nt
zc{J71q`qB4SPhK8Us4!)jN8hXu(QC+7KR{4<C7u|%5|zp`W?W{nm8wrQ2T~@!`G3&
z`E5`o=F_NHbYF9L5A5dN%2~>h&0@ZBWJI>WsX+_Ax;*5gPv=Q{lI?_b)eMo5_&Ch_
z(n`|v@aL}u2Af}wef5W@G`57=MFIya4>UnV)mG2R0(U};eb0TIN{r;fd<lyBD!~z5
zNn$#^9x~WD9u(p}*0V0ZO#%w*igV#_K93&Wf{@p(dgfXU|2oufNZ>xJ`$ct&xAE>+
z1Fbj*jd1oAL{jfwa_H~6dV}bk;P;V7+cMTI4`W%BKc_UG-Mwcw+w_)gune*KTsU}!
zVyp4;;!&6%t^ec1V}mgw)A&^t1>0dM@rja-WQbr6a7dg!D*U?nf%Wpv%L5lf4>w;)
zyS9RlhKlL#sP{^n1=j>G@fB8fd#d?ddY{vnB7fQcg^T`5i+|)wzgz}*Gq%8nG{1zt
z9qug4A~Ybim-EHX8$M_dH=KFGCcbD%C#DrN=qjoFbCDa}eyI|zvEjH<Z>ouo9p>_=
z!^ncKi3^R(3g2`OU!Y_f)4k5On2{soGE6XSRno0oq<g{#*~doy#OkjVz?$=;qp_6J
zX#AE-WT;g^l<_pgAscs~+3YJCI)dplNs$}*rw9LHPtbaMX{m||vS30d1oDjh#l9?L
zxoxa1__MJJ>j?uz^TJopLza^vWF<I|aq}9|U}%lW=m%}V)jf109!B^SP&_xxbS+TB
z008kKLVWM--~v+E`2l&X&e+ncnFjAK(ez5(WTBjFAk55Xye|z*HCGNsXUblud8)L0
z3dFV>`S{udsJ{kfsvv2@e!5)Ltc)>+bG<7t^gKThQ-FRnMwIQ{lE2xnSW<gR?RoHZ
zRc~BrhIj&cj*O}VvJ0&RQ{H`&evj|{!k3sW9rr(WiriqcKaP<fA1a_N*JVSw?zK@E
zm^lV4a5f!*Rzt<BgjFhRfx3qswScYLl$&cZR`$}R{pst;%sKk9tR8)dF}!=Tz2cjp
zK2q#eV{7-N2)xd=s(k#4%l+Gl&E9YIx2T9A13qzy(AvSjxBC|aJ<^<K;I*ol3u4~Z
zdq(rNN%8ZLhSCI|JS1fk?T+`vXIv8b*T}NZx`#8ihI+3zQR>*Pw<urW@>WOR&*os3
z2!O63nlDe=UHhh8OC5hfFy?#O67-+lwf!Z`)3`sN8<h67BgM|B;GGNn8rIlaQ7O5#
z@Qg#a<Pi$D{BrdCsLEyY#NIjvk;T+}`{(^#&i7!PIiQ<%<>C2VV;9~Oyy=Gp;d<5W
z)N|mCb>I32_bx^?m;1n+=RvY#-<Ovy!K@Rxt_R-2qS5;JB0K4iOciJN2X98<c}%(4
zvXe*FiVAKVX44ggZLYdCHlhSQXP&Y|o2ZCLUA{PsRfT?CFKJ{L34Wr*<-u$8y1>Z1
zB#k118IICEQo*1O;a?Bh?g>OEa_Ee%LZ{kFLC1ERvBf@A#izFdf_ObfDblg`)#0lR
zCaUXSEiUQ_Tl7jB&02zC`m=-4egNspv;Pa}k{S(@qT>aOPrLkxl}VCGUj>?)l3*{O
zaxZ3OY)x;AmQ|O6Drd!FeX?O?iu7EO$gI}~#grs@;$3Uk_BHA`?br<chU>HCXkxRi
z`b9ZUXL$Wb<&-!e1MSl>{<N-?Gqx(ym+2pg{qIk3xhq6#+E<nwGdF*Xoq8U=(!>u7
zAb&2cX!9M*?@Lo4At2X0L%E53XrJd5=L>P__QQu@=sio5az|=y!nr9qwGXHqiIm>X
zKY`|FO7?_;eM9QbK$UfMo+Y?QwTa-4eCbK2aB-=W{i|@|_jDgHqWu-v5>5v!x&Fnh
zi9vk54VX1?!ITEO!#y~;l&X)Z^Y~zer4-lrc5~uqaqVyT;COpIG4$~IDDJVG-O%2e
zJ}A*ublOez<f+YRVOp<(d&pcekxTk2kgS+3)CZ{6{m91uVDt+#c$)Xlw+r`b94aH=
z8SsV2Kx}g0rErYUOPz!`M3W6~2fbrOGxZQ`AKBru<$iLhtA!R2uf$t3CW0x8K4-lD
zflo+KVsW!#_RV-C<pGh|sR8Vx^P}CkFX6X3w)Gu{<8K7`PHrP?Hy#cUGkZt(Z86Zk
z&NkqB7)5Wb*?j$&*`elfM<jvHe%<GG4UK{R!9EERS0=v~a6)vvE_~>|8Zc$hd7m)N
zDI$X=gRilc>iFcR)_F;YOXMAq`TVS6Z}=ui^p!vF{b)6ac56E9t=k?4GOoNd{4S@m
zvy@MMB!DYBV(^-7(srid#?vUN!)h_tO@$O1n!YV=%i<c-_iMnMlHcXk$#JYI`o~Y2
z)jmqPJR2(4-?RG#WxYE!uy+sQGr{dN%6b@4ne$$Mz@v0Z{LA_ALR8GHII@o;3V;tp
zI@XUh=V`UrscyenRFZhvw+F<=y6Zh@<joXBDnqAbK3oF5a9cv=JJDvnjk7X%EuM;l
z*aD2cCOGS>k5eVaFC3cuK8_|@etM~A`}LJjP<B(K)}tDX#CF(Bd0lTw-J+{oCQqqt
zWYh=Gt&z%A7Mj<4bJ&$xQIg~drX+J%;6Dnj4K#R3Xh$t%Rqeb!A}H>HcW!vSEBC{r
z_YJSvTpqhME_?c9NVh(7>VNS0C<S=0WucFo-!CIrR0ufovtMZVOfUJq+okzL=vykj
z<i?KKXu>key`K`j$JTvYTX)Wt%KqUxHw)aVwu=S%`t!!d@je&;1Wt#sP;VE})?<Ii
zu}6<kE37y8Wa+Bsy%}*YIO><r4=E4ZM?yS#SLwLFJ8(vUkQGj9bYon%t~!StmYNE*
z_|y;VkI$c;4+W=PGRp7O`N0StdzVzthgECR)&iC{`NkA0bo24oX+NJY&aPvug}~}0
zKBZ<-%z@o(oe7>SjF3%Y+OP|f@Lu_P*dFT~s~PWwqF~xg2;kblpk`Po{@*Y3T}MLD
zte|#Dr=lwXeUEGk$+KYGCPBUGR`>L!H}cUFt(@3v`3aDk&QSPkR?1y=A!~oJ(|K0N
zc)3Dngw)r9QV}nNBcCt^4~An5fip;mZ+KxLBk2;|?2_e+(gO_}tuHBr%)Py+x$oHE
zE3l@i2n`R8nmqL-&pPGO_sx-3P~{TEAVe}|vZ<sm<iQZ_wX4sNtWj&7!6K^&rb1ar
z%CcYT=FKO;B0ce_Z_>MyRF9)@3O}T|7d5QC$m)B*vx@xTlPOFGv65(e3MPD8cIe8|
zU<U|CU}^drNu;>YJMs{ZlX#ttM9WgnkJ242ot@?U9n1u-2e(PZA<LK4^sN*>XyzEs
z4uq$4(@n8RT<+a>bL+z=ybgh7H|1pzh>5~h3B<QPv^{r2wjBat=GP0SVzi>I&O9Zt
zzqJ7RWwGnLcXp2R(vHW+cJDC5&0Hi9L7&nMI7Mt8b_|!%biD)|9Mu@8y!K|<xr_aW
zi620UHMt=uF6pqiYNk?&w@yN<4W`&YO&?jbI^*L@2XV<IT+J0pAHo-ElX^>|l+K`C
z%6Cdgts=2%Qj7QAVPX9KvMp;$+>*~igtT4lXO{1w!E;5$wH6YYO1<(B=WKf~eDto_
z?7KRhxWthdimqrLcWAqux4n<LXeZ;OnXY4e_|j6sdP>Fi`2n9F<}L}rdW_MVTVgJi
ze$m-j?s+M&hlhz)9DbPZ^oz{1ap^W6Du-auLY{CLMqN&*JhJ1d+&j<6%Is!_v+0P=
zxDvMU5Lm)3&p*CS``S3;k)%OYS1uhnaJKq@!cb%PxR5LTXXKVS^&twEcdVxhCE0Mf
z>#=Yl^G4RDepATv^y6z#jn-{^)4YA#IzEDdQbyqM(m*)xy+}$KAf48kcH}iun$qRA
znd&|A5&6}BQ`P$@V(*=g<Ha}l5tsADcAvLa=Ez}Ht?LHb&{3byXVGvM%h@v`4W{j$
zM?9m`Lb^WQwd(CbM+K}{P%U4A?P=PAAQ3;7X0`Q%K=M^9OItYW7lN}5CeO2ogwOUh
zbXVOZU7mz+6_xY?2&aU$?#mlM6>BeZ54?{wFjcXJmwpeP3s2d}ir6@o=T8f6G_!nu
z9HU2DQa)T#IY{)n?P91ab|c$bhil1n-Kwg}+GmHoU)usJQFGgSe4?jLx&rFD9dX!s
z5V&~qPC<z+MOqV`R7Vv3??n5L?8{H+NB%H<oQccZq*^Y0&b=`}iU8Z~mulL3J`@`L
z?swb|)I7(S65ryKT~I`;T?`j*oZi-y5y_{K-^XfXDDI;aM-IKFF9hU4tu%=ci@8z>
zP)g^iN>B)`u{ACeJ)z>2by?k+lSv!VkwPgB%NtfkR}?hg(S+Z?<`(iLLd&4>HYa0+
zppY$w<ry^RXh}&o*s%GCmw3LCFR^F~Kibnwws@!NIH2Fz^?=KSa*bcx`?T9Xl21kn
ze>mq=@&x$1R^3S>@_DBoewhpA>Q(XD&sKYZNPl37QV!!RD6F@p`sL|~pxLa2?Bevv
zyH&B)UCi%8+WJKHxQveyP6HPFd2s9c9@(6n6_5KrSvC9N4)3}?2C8uGMRy_3oe#@}
zrF|+qxZD-#k_Y?zH{|<<D1@^%fe(x!?0i^7WKNET!X^IE1A?Cng2=mi&9akwy`tit
zYTg4~K0jPM*b}5%%Ec^;1-)6xhL4ZJc}H$PeyJ2H87=5G%lWwt%v9gfDT^e^-@!A&
zoO0I@!CXsovW(9qG`o#&Rs9{aJ(8QSt2^FdLG#@;`P3AL33EJ-kHl)&vmv>CQZ&z)
zT=t}rZtRi<Z0z_85h@K#XTixx%*GB@L7m7wW-V6=fL9AXrRVlnR(7Lb=a(MRcS_|R
z>@FQkm*@e<lguEx7?smW_VTpeG*81VA{7Okv-8SH%;qV4b*-?@nA4FFTMob6sqH$+
zqp|5ztIDUz_#G;A9&QW;c%X&0@nZ5EHi-bbW!n>NgOT6D!dPJal3LU*{WYjC7<D`O
zy^hoMVv4g>J{8BHX$+fh7quIn2Ql&xyw*==xJFdjrJJYN0>aZ?;rC}1mNId9Ld5T8
zh%?ek9yz-~r4!Fq9*9D9@@J-wR9dMbePyN64*6s0Ub$_X1gAf7D_*PS?Bu@$+npAX
z-r!34W@j7uso;*+(%4cx|8W8(c^E}&TH3g8r7#o30j>a4qeh=cOceCYKJ9#<0!qe8
zr7waF4y2QBFE$UlJe!80Y-A#Eww(uFRSpE{$$Nud*GnC!nSsY*Xv22ne%$$a+-jrG
z$Rq9G`KsIS4G8S@(#l5@1U%!gl|<DH@n16FEcPAjio57N&tF=!v>eD*&!RQ)&ALOq
z-O#Ybj?oM7AfYI;{vzgJha8WMZdKe;>DmfjgJ*_f&*a>9v%Ki7cC&aKS$Uf&=VX5l
z<yEI$M)TK_N59%lTs0h&FzfPH!%A8!D%l7d{P5v(V4JX(fWj_Z{sU=(75dX`qLXZw
z)3AO#z%WI!UolLlNF9t=-#1v8Kjvv0$V|v*;p>(y`XE4oLA^%k55K2PE4EAG|AJ)o
zp|d&B2^)p6RVzIApyh7wNBw6cVize5Ur5Y+Ge7^4`x@i5_4^|wj86HeHq4MaOi^RA
znU_|jyq3V=%X70l*t&8&mrpP|w56qGKtp58zgI)lRL}s-Zwre`%XJJri`(zH&-NhE
z+U|tyGq|2cCu86-!|js0yI51rHXh!|P(k{K^7l#Qy+{bKuKd;Q^2?kxu=Ra4Sa}fG
zGGmmjoqSL*&9<0V$&Nw2^g-QP(d2ltATQ83_MuR|fV`CbK8eQY(XGQIn`)^#3U;Dv
zdWz}SKwb)@tGTx!cTdNyvz6z*n8)T-hfGWRx#-0DMEOm|NzvD7Wak>YBcFgg*wNV*
z7((E5==RXqQyuTLxLjTrjQTo@{fVSG^6?`@sjE;|Fmz?-<4{f_cGED@Q0$)XWOYHz
zf7Rq#^;9wd(@JcYyfhC*mu$*XrN}+mt7xu)>LjlJA|rVn0O=5nXY75kO&aMm$ufSj
z(p$Gx$v#I}VpI?z{pEOmdgEjzHGIo?Q%XJAR-4u}*%dZf)>ld8ZaPJgwYfxrnc&5p
z@CA02NVItpE)Blp2pkv(cKig(7@|6y;w$~GU$&ztunw!R&Qi3En^z=uBYOm0lk#~V
zq>m%Z_^pbqcklc%Y{9qB+o~>9zOX-4aTg>W$vo!97B~u|JCU2n<-(ZvJ{-?`PgOnf
zkKxllnk@M?e@q_;XdVVyI#kQud#OWFea%Y^k2U?nN;*Lqpcd-b-!iVo*ugc)(|W$q
zogvlG%0|&(et!2Y5{xQsb^#+8QAA$@Pw0*rd6K5hmyxtY43#<1@F`X}7(4(A5p;YN
zA?;%{0NE8r%aWLA?0?Kh8^Xv@^@~%bCcrj-OxaF@mLDR`_^^gll@<_!-2ow(J*cMJ
z(4~DfLWAC^m0_RAC~Toi0k33U|2WTMm=z+lM4}$dcR(~OA53RXQ_?$!omc1N^<d~U
zAxyRjV7i%mE`D>3XPR@5;-aK3#z4Eu4|9`c5TNs`zD5dpr!5k8BzZ(B7x{NkE2bBO
zciNqQ9UYsR>!fk%J_Z8MOt(BYsX~;JY^cw2=re`mq#ci622+b4-j;aOxOphPk&|Fp
z$*&5PoV<@tQ%6+J8Dlc6rBg18FZQTq>^W&CKdrhN*GigaB(jOBX0|ze<D5W>&84e7
z9#rEKUyD+l+gI&>t$KHnYsj<KyZ-LAe)Y(c!;Cala=j{*`Xn#CB%1R^<U;4B$`*Qz
z!nvsh4>oVcy%Q8+3?A+LBzG0!_;3vkP4?Z;^pO_HNTldu*(!$oIL+i*>Q1dV*P?RN
zg+_1O?0b5?cw#+~k<f({MTOJaB0!v>I9XqeRjIgYHfg3g_tqS)=B2g69527I9jz#8
zZW~yG=lX5k$$PzxyaH6Q&a)8%S2{h5F6OM56myHSr%!48Ouz1={*X1!PQ($|oN%p(
z5Sr<|?Zdcm$=`c%@N;3VJ`~EA$Z`db8ufW7Qb4;4s_pjk7CN5u(S;sV&;FbnK@Hqi
zRZ(BOPnW~(e<lp|R+IW+I`n+JbZc4uz<>GrurumcrDf#AAS_iZFdF^4>z8``&nhW(
zlh&_k^<OxmD_^6P`g^GQkGQtOFcrC)G_(+@WHq<p3)tOqYgfgEqRq)lxuKn=lZgxQ
z{6Xz@TZ(=TA-zW~ifIUj2TNJx9>aOc>w^j0#qzf(T!yu1p73ENb@J_Y1S=i1#b6h5
z5!A+A-sj>Q(vh6kf}~g>?hF(KjWBqQbH|$DGZ4fMGGc`68V_b7=VXFYaA_#go|(^O
z&@lJp+)!e@O__HvocCZy^ZBrRo`^+O8o&PjU`^ml(xp2O50?5qcwr+GDO}`?d*sF1
zi2P>_-{Jv17D3^Q2p$S-0apM;=QYa>uX={X3&$TX&laRAdnYVki)&{;ZI3)}=BZsI
zrh&q|3V9I1LDG%}w_Pjn7x5Rr*;#df5!I7uO@9Z76NinqjV8Ks%3_u|<z5**@pXsI
zVV$Q*LZI<)s*k#DNU&YbP;eeMs1j9|V<$n-vH$3lpqihSeb<C`*KYR{e{W|9Ih$=`
zR%3b6jdxor&DAAYEtU9nJY^(!8l$qF8z8jD-LfJ!qRuvY;#&BONjA!uBHx||d5JFP
zXKlDP>Pgj(RQd<+CfiS#xA&5Cf|A_yXe!Caw^xK2KE?OKSMuq+2qh|PX^r#j>15@O
zat}|<{yopL6xT3=&-RKUBFOC61)`T<h_BF$xNptD`y_rZ>H!7UZSxQxZz)FjWBj@V
z6f<q3mg2y64G=)<wW&E6LAbb!D2AN{54lXyR|XN9N#SIF_;Ot|H!j(09%saOqWs~8
z7e)+;`Boam(uKZxcfB|=V4-+rK#77YVc`&X*`2;kuXU?J9a;DdR(!vjvP6>Wvqd&-
z0h<AiC307Sg+~&LUElq!e|{)0Z=AzipDZ5SG8QB;6I$EzTo*A+t-!rI<9(*w#e`rq
z8*KO{yr@NUIb&bIahRa*;l4gq?3rsjm|OhyBkSd_kll4;puAnZ@ZwkMTCQqEF#vZ;
zhHwP%>gM>vr3Wh)P%dlryszsjcEo2BTA<~njkI)8uhz!cFX^+i{tshs8CGT2tqqd`
zA}m5eY7r^|QcA}Hq(lUyl@6srx)<GzfG8aj(u<N3luqfE4oPWPEW&T%^LX#~-TQHW
z@1OfP;Lm+{G3S_Loa2mm;FvmXkk?q<ryrG=tdJ_wHA{0I6N$8aOn*ZWQ6j2Y)|R;a
zDx7m%_^0cEH4-LldE)fYTSzg3#is@5Kh70VUv9`|1QMGaUq`zEp_u;OOzLy5<)Qc9
zO|v%0<FnmcD&9uB{$`J$!o>mHeh>=2Nkk?SbZ=4U8RTn_nc|_U_eyg~U4bv>DG%o%
zGPHxJjyrLmI9eeqX(S{1aR-3W9W*@c;HNCXs}cIy(rA1_5+wQ97JCuHa+Zk7{ZS)1
zCR%6m_ZWSY!dRqw2HnJUgAi&8ZpLCR2*B`NRm<`a6StU<Y<m2-qHe@n%d3GIGzD_u
zsREvq4(79&)W$%F6o5+N6hufeKH2O^)WBl$JRPamrXM(T`2zt&Hn|Z&tLJQlfk>60
z7%r^Ho6O<q<7MY+pl6Xt)Bu9b23vF%1qojQ7L_AD)&ralBD?!U4n%k#zI?on(<hg~
zg3~yxDAp200JSkqcMEb_AG#AnzQg7J=FF+>Riequ@CAm|5u$MMiMo|sYB0mTCcI^o
zs6WU_otMh1p+9p4!Yd!`F{e)m{zywq!bgEi^a5Trx|M->u#6qmeA<JKLtd0!@)ZC)
zQ|^ZR*8_H>oqv8*;1KvFqU|~*d7}NgUX&;%txE?W0JX?w{35`1=2p_ZeVt}#=XEDl
z44;tmnt`T{*NccBrrUPRDU@ez`cWe1w%X=7WZ6kGf@vO?fkqTp6T=kL`8&hLaCVS|
zz*y#l@yU$U0SuniA&CjC#-S^6`=CNK8^0sG|KXhvRJeEK&QcTIYbKe9qMn@l$s3^P
zWX}}ca|Lty)Q@xs+Y8}4Yopbe3ZRut5|&P00(}!!Ql(eziOt2)ScR_hWbOP9R<V9P
zuT!%6qvPqN{zkA}X!th$={0n3HNO$$vXtFEz8Xxd_-o_ED?md1l&;I{E#rds;uY%}
z-eu0)!tt#od@F^VFk@p`f>J|0fTLL<N@gtAEL9s&J9*YyYs6LW3m>KTtSkzpf|P^l
zzMZDz*Ntu>K?etAadT%oO><>m7)x`Sqx66svDFA670*4o^%u)3#x>1#aH}0vl(dmN
zf$_73lO5)5mrW~jg(_k*uAgp;Ib)XD)>-;+eG@EziXed#p)wbl7vZmUC2!V&)Bniw
zxccS()d)c#Jg{ph2q9_H^uZBNCs)ST^eMJo!1zKeE>)AM^wh$?5Dg;X-abJ*`bDv1
zBpR`it+c5`V7!7m6G15n6|2GP&O=dI(iL>QgJq$@m?`W+7GE@t6$-2v6roz_%Zd9_
z5W|b%QFd`sab{26EaI7GEq~~j!bY1%D-a*bGT(dkb)wQ1P4apEYC+@q-d7odMM`|i
zQM`hC6Kc@FMKfN0&rZq9_}*s9K`lyE*&~E`5ic7ZHE(<}xkPYh`**1~%J8ja;}ld2
z`C7x!6Sf&%8=byiJOq4+*Cjyf)#s3ne0@6OGCgWvtMfKy2}TH89<WMrRunwFt!+O=
zOuAlz{!z*9*DCev&L|ZD)WlM=;_Y87&WJ;aI>6;=8IH#}C4!K@^S)f~3Ui-nR>YOR
zAKSFE^=8*_zwEdXhpud=>DMfv329l57N|!&jea4gT3o*0?Yzc-BRkqJ_R9)>GA;Hn
zZ_kO?Y=|j*Xojm?N-j?YmnWQ~E<e@f+vH?#v<T{*kc&$<jil+Y0$=s#lPR`>iCY^1
zaf(m+u;Ij5&(b`*Kk`|O<oC`#$LK?J&{+3ye$we}DdFAHQ++a#6EtHHb2xjDv@pw5
zwJp;6Q>KH~;py?*P8F$HH6B(Ht#JuTk9zQ9YvL`147sj5D_iHPMjMeTp;$wd7z!x&
z0_zh7=LgefEd&*{yv)A2wcdJWq68PULo_EPe2fILbD~Eh!^gu)=L)K-&J$i9n-knh
z=LX|>43e`23nLXjhN{PsEKN}0M(DV@!psJY2Q~VzsG?qP2)y1FJA1JE=35WvjcU#g
zMem(>^QtPo8^Qxme_cFAO-&`Hc@MS2x$F2G)Yp#AK9HKRE;(~nFPB%3%Oy^qhdXyg
z8KEwde@zMZL#~f6Ie%%pSb7AeX}?rd@Ti6E$eYh&mGEZ7vjfL>pFTi3q#{qZ;2u94
zlef1`q3pYY$8Wu-{B-x{OfkO+HV26uJoZifS^#&h)QuJt+mMg79YVvEJKGVva8b58
z31E7;(|GoCr0?uMzd1fkTm--2EYYXokd3>+^2<bE&`!Y;z}_XkUCvFeWK=!S0ihN_
zKa5xpry#DgdrELP5vpqASS}}f!2`ST(7TN!57kO6$@UGSn8Rx1nchl<I=n60pZTi@
z@AyC@d__albI?4Z$x}@@zr0o%U(^oCr}cP1ZE(4(WwGkH*4;2O?!)1>K_z2gD2QbJ
zw94X*PETYOd)VH||GMi3V>Fo7H)2VtvB<SNbF$U%Ulu^?upL@+tLz_k-BFHmEHM}D
z3ubOGhp%MA;P!6{6>92&Xw9x633<>#2oLiB&rYAVc^lbwClg`+n{vz>5Ss9ne5R-}
zGEE(Zg3|iSYaV+!iD9}H5v3b0fmO-X!wP$Zw*&yKgdFT{>R;Lszz5!A_co^jd}+h~
zRS!EKq-KX|=JIIAdzyfMR*gvtvdB&2)Qf1xL>S4*-E2GgnSn`IHmcb#X8SSnu|y*n
zTKeb@>Br}6v+b;utyNgCl4{=CR*roij^aF<Z-H!|<8u^QzraRuqzX?@nx@`zGH~xS
z43>SQ>Z?k5f4Cwq)%M)(z5d=e1+ci>s_Wrsw|jv5t4iTQrI}crMm^B;ppV}fER<i_
z2wnd%P2_E+EFwIjW4|YjE!#B;U`m6FqpkkoOUnih5)F+SDN`9zGnGg4c?<XGI!zy{
z9Kxr;RjJD!H1?<Ck1Bies*iW93=<GL)YQvVpQ4hidI@>@E)KGEFcchS>s6$voZnFj
zIp%*p?=IvG;eqo_^U#ETdfw{1MWH_d0^wewf5X4e)nr|-u)qNUORAG3CD`S?cD0AT
zxoAAlURx|ss*2XJl5P>g<M6-L4;b-r$C!FuhI*$w(6*cwwd#;mioW0sf?FkS=_4;m
z38eK`lj<Yc5sMbrxbqaIkB<S5;kd0#$*L7cvU!f|{Nv?x&&Rwps!NTo9UIvy7DLkp
zz#OSqp5tx0uxjx8&nuPz;EsI72gNjMX~_XJz-ow#IOP61Q$!oS4?aEAaiZoXb=|G^
zL16rzm(+JNBg;-%WS9@1_vd;GY%d0L5&;cijDbH(=aBccRkv&*YSDO}jh5XH%1bn=
zp0063mN=&5FyAAPN>NuXE!RW%6oZF&#Yp8aM~N9v0IjomnWFB6z7I)Z<N3{O3jcNs
z^>oj}98D4SWm3>*`ME7dM<#=8_aK98CBhkC46`v{LAb#deK?||kF`do<h9`+yQoGA
zMTLwFa~v`Zd<UO1M83`IYHj){YRQhyYkAObhGpmmK1WT4uA{`InDF2@U<@CmU-kps
z{elt&RwmPd6#6N-ER(^wXbfoYxfZmA@5k8QrkS&EH4J`L|15*^u3@L<HXu5XK<D4Y
zEqRsTc3|vf)MPeCv23|w>@A&A=#>eh5@&U@Gf%zbo8BvAEKt&Q5_aPMLVcQX9ai9d
zTYsSFT?T+Cp$%ZNtu1`GyMFQs>EtxIqB#-s>AH*h<7gq&%DDb^hChVGtvB&Hf1r3p
z%kgwuwr6e-E&b2hAA0@*YEkjjzdGQ@cvx`+6(o?$C5E#ONyw|EIw1;OTVM7XgG<UB
z(7Sy3*0h_2;S$wZH{WpZAYElH8r&4yb96A;dq~=V0m{x(g%#Wqeb4@RO3vl6X&d|w
zSyAdUPvX$*cW6>S9r|Hx!hSO7pyQbMxQaB6j<gXw-}R@x>D<9+D)=B9kEizM%yy-!
zQdcf=UhI<i=h+wV%+g0WZBA8fjT<Fp*KlruTu;HD6UGs#Pl=-FFkd@UJbJ5RgeScI
zfz&!`cS`=w%!{IrSV^mwZps|KtDkCISYoeX5bRsJ5<qBCwVzSlY{$bqo|^jB>e@C_
z-;|Kw;U}VeEZ;E$(lw`pPfBZZ;Z4%7kj;e3#_ds~16LfQGsTQ9`n_*AWat6kGE6b!
z{nszQuB{x4o|tUbZn$$FoN<c2@)o<}J+k`xS-}g{S#mGvf~aaUGV|Ky&m79ve_={n
zJj%c(F_XbMXBdc|(!HjL#;AxUT=Uq~HUpm)j>ING4Gr$21yC(F=ca7*1f|vO<InVJ
zI;h>}vzH&yENUN_U%0-vC~^-6cKAy5zE@JP5#>@T!hV3UJEw8up~*#S9-_B!Fe@uk
zkqF~`1bGU_Q_}I5A0?oM#HV>L0?}fF!=Iyfyc1|MkPKeS<;pEZSS)2HtqmoeGQT=)
zKz7PN_3c^6sf|zbVO}f?(n}fNY|8C3nDiXf;iH3;>6Ng{f>N!uRXj}!v1XQka|CTN
zoc-sKAHbgJRXWyUSEubr52vBL6ToiPzF-y)W8k-b$Z?K%#!Hux2KF4h5fC`&deE7!
zP49e~X@C2|<kqyyuXW*@MHR<W?f#y49eEP%$eEqD(ZXgKrgS1Ve;9hgv6AAHM81Mt
zmQ2SzbR%oo-j|AUKkCZZ9&ei0`(?nEdMm8fDua_kwZqrGSz+`^-~`dW)JSf8^~<5Z
zi71rhJ15qD@0hlA0&zx~+wqIDv*J%=1}-4BU?R)+Fo3Azn{;Ud$|B+gFyz`U^_~<9
zhyDv(g_!ICr}F+Nk~ii3Cdrcx2hI^K6o6fD24rrp9^i^p#TtcmiS!1Jub28(4ilMM
zdAe`iatey3G~5YTdO`19l~kC({@D?-Ia8R(-mu$j2V-PgZk|#lX$K=k#ia`&*VM{e
z?J-o4?u*rQXDNzSCTBMmFEh}o5gdF63rP%a3${AtniGiF{^KIVR6!wCeSe6^0T9hh
z6U)-Fk53%qFBcLHcWMH{Lg)tv`p?$od+jf1up&WJnCvgG=BFGshRa0XBGs2<l8vhT
zX80*efq$m;iVsoWpfBbxrS?Pxqi=ViWqWj7umAh(^?;IiAD>LY#+@kGKCZT5;JJ<N
z@-i>Pa0f_K$~biECU)NHa~L4xZi|n)Jf+xMc?>9gEa*W=F?8wfD3(oxTC~*(_iVP{
zqQVKk=q!8W>wmp%i~tSsn7hE{di0c<To2N=uRS9i!ed!nW%9Kj8p6|$wRlUmw90&V
zYT`n!Sy`zU0Y4+0wDXwARZLByEk0=_{Z+}?m~u(ChDDY6W4rg!)rA}a7xINm#)1?7
zl#@60>7Fp(VR51*7Q(+5kb&bbyqpiZ&L)FPMYLN=b6AkIjv%<;<uDMJzS-v8e%$+X
zo%TQT*S`?o6Td<T1o#XcYmohq(#|#~n?hJ4+-CbJ>BKJ$M;#k6`8W=Bb_rZ@Tcz(C
zA9hQ!Wj$Fl>5y3nCWuw?QhTlsdAFOXl(O-VToYY5ale~__TVeTmnb0myz>t92*D!b
z=FhtHa$eSJv?Can)v&h4loXCqo_xo7)26`8uRB|s?Gt%IHPT|Rbr^?!oj%O$>jWZa
zZ(>DXN8ea0K;U%vflL`&-bPj)hZipY3%<jMS9vf&HHa9;sJi~qD2cw6afIHRv2hV~
z3h0(Yu;re3#;q@so{8PkS6K3lFtHgGiPPxlcmPgs>F7ohot<*`$D8P@i};*nimChc
z<~KWbcYk%Wm6kVcHR*f4(_@82)*f#ZHv7vc8u#?94Q6kDXiCj8Otn6<jUP!cG>ufd
zCVE*jLv}gbbmxhKNN1_FE`eY(o*$NQY{;@W$4^LWu!e6y{7<Q`%3@nEe^Ihae5BvO
zx4x*o=4)~<tINWrsrpL{t_W^a7QwXcA;sV8U3^PQBy>wt`_8OPJhr~=y&qm#R#hx^
z(#&1gJJnNN>=)SP(e~W4X=AaBxMJp2Pvf*3k;q5-WEk{;y`97D9I|P^5n~wdGW-+^
z@L=_nYJ*<Oh2Dy9d?D-~<ApYutD5%OlVhLdFg0#44#YRZ`lOpie#-0vh<fNFEpnI1
zxAMp*2w@$yQx8I)d%g+(*wUIi8S(?X|CWX!-LxUMp&v)r<ZMROHNN3KI+=Az7u{bo
z9<2I^MfHZ~M?XHX+2V3oqqte!`l@zVrQvkj-0mzpoiokGpfQW~s|582jD0Z-uP3K{
z)q;yiI}JqNYL&OAeDLvU`=$mqy3f=AvyJrl)&5x#saZh$r-%5y0B@ET_BAn?@eLXm
z-HS4{ZVVKj{t!AhIq*pzfk=lF$+d-ZPWQZhVcZfS^3bN$)OoN`n@?o#>EU++e*!?d
zhV+@(E=BTBWk~p!BV7K-cK$Df8T>Ni?mr;&5xmBA;&#>zPjpvwuoS$u3wasr$~cQ=
zSNDj2NgzX&Zja1tkg0qyF^$~I*J|YPrt!E4Pg5n3IfFsmJXBUi3{{|X|0t9!5rf)&
z(R?tQDN245Z-&aaZK2fl)X?(HvxbjZ*k%ZfI0UMoMc^o}G>|)bB%1rtjoM0A^2qdk
zHk=Ik-9d$CBJ6%*@cluGFNegF=Bo92YOPu0?&}rTqD-@*Kb)@jP$Z+ko`2n3-QTm^
zQgm6olquyJl2}n(rS^)y{&Qv)EImSTrx00cJoPSzP+QaQESd45qkBGX{sd$FFrCE6
z;KFnWHMxsZY!rEOPbf1DG@4cJXsHp^&GC6_zbw3y!nsdfa|%SBkU|IS1Jta`0hO$J
zwr=y~s`jN<q3GFnhsl~oMU?H^Q#umop9cJaD$O>`#aerO>S+y!`)u3>#k}M9IJsOa
zn7T7|pT*bOjf|LpC5*!rZS{B}qeFg`^kayw`enn8*By9R0*JIGs<3r^<}qOy+&%*f
zb1*vxn~We@@sc944rkvQM@4~Z*$xR7MfUpk@3u<{R3_FABxbJnyjq;bO-f%lTQ0aJ
z1wBz+G)=bV6JE~nFVfu6!~T$2$v5X640ThBq_q=J#Gz3kiCYX21QgQ6lW0ftWV+gR
z4h5PMl@hCZAlYakoK8$RF-<2Rkd@b^bZNbp;F6f?QeIsB{Xq*2v{MyAKx0OvNhn0W
zWn1a)6`XBZT0g7jUc2i3m5nnX<>Gidd0UT3lwEi}g~E3J;zRt0jtH{p$-R)&XchC4
z9pRtC9X}ApkP9P2=GafOUelMvQO83Z0t#;Zw*caQKa<vDL#R1a$m3@$bG)datL;~1
zXPOzOUS-)^9i}-C44bj)89mc0jK}}%`prxT8lsuD%$zsoF3HcFwgI<6r`sDKckQVO
zM$eE#DPo)@jph1ZmFWs3&JG{;2hdj8D5<|_;y&TPFUiLAm0@J;3x;~WBQ`55{(P)e
z?O5JJCFGp#2Buc4iLwRsv+ar*ZzOE-h!fD<Vs@QP7-D)>jLd;HtO&qgb&LZeWZb76
zp<_GqViTI#u+pAl=}9D9q1huZ{6CLJ^SC3C;`mGhTjVCfcXh;q!c8xBiEGvzzOv7^
zIJ=^+F;qMT_Qjrlj`zJ+Vk4jL$HawuF4BEy-00Cf%?ET-=%{S8qH!^%{?UpgY}E88
zE+Rw=wcJbrwZ8`uI?a@K;Cf37?c;TPD3Ry(s@q=pwpqcwU~*LvyZxSEn6tQq><we`
zBJN_2Npoxx_!dSe%rC9*9Wb_YXx2Q+8IBwjF<X(fO48@YOI>9WZp1wzUS;&QXGG>6
z@#wFpgr^&gbW>1kZrVESj20%6nqd$n+*iyn|30(yby-E8!#w<cT?jiMgo3{=%eD7b
zQUV&et~jAVYV-GU^w^5cKvGk1B(3n>nAduRM}|VDb%DXbfW`IXyW3*luKo#4i%Q9^
zL0oXt@`3<slnjgP1o5jpu`SB6rkELzUl;AM9943H!Y3aDUGMcvD$pVMWukDU!b)2Z
z{OocQFBrFGU(K{AvI5I|{DU*0t=R`cB1rf&y$X*MxvrzKwq+mh60uaASISE${2cLl
zZ#8+3cOnLvi?Utr|K>ZwZMO(<-uZmep54*iH96wye7H`=wKbIEAn-N^?i%T7ycN{`
zlk`Y*d10aekPW!$j(F^s?|GbjKG_0Wse7g<h5c9CY=JvRW5v^y0Usf-b1o2CmFRd}
zHcW)|Ca1ZCO62^<8CjO5@vB%{nh936QmcbL;ZitR)CZ64qwz?lEUofNi&0jifhx<y
zT|L42NIKEGbhNDq(YB%ZlN2MyXLdjBJ3O20cUF@uzagot*G}AOoh%MsN2OC#t=Qx@
zrg)6Uq_Ftra(H)sn5pNalglA`+7WRpxpt>|ljBw<;gBV2b^%BZ@9y;8{*4rDVI{qH
z#O%ITfBK8ou<$h^t}5O8E7k4(^xUWA5Iy&)Ybp4MPkUk&J@2CGh5j>HE9%SBM^`P=
zgs+Kf&E3Fo$Q{>joW2mB0V99m3<!smryP8(g(4#X=J?PCt9F4fBp{nxaFC539zb(w
zuCDKPOI7|rT1o%>+PXh$PiEuB$HtAvN(vk+8EwJe4|&P?8S@`}_KYXhP70?SRrd0v
zx&wCaX3fh(AKt%fU(QSCAlDUgyRs&{Pa^uuy^TO=^InrDZ@WH!!dvb~Q2i1kiQx1Q
zekAMLhPC(d7Mh}`4{}TS9Pur>lpG%a$kZx|a~)%$H9iEG*DXS8$#kX^)ETkjm*3Av
z_h**`o`YPG-c`|{8D=Ub_E_=ZcTTPm;8w>6i>uB$Bqf9Y_RiEZ;EqUV_|kNLGj{4~
z#^-#gZG(_@^bK)7qrJfY4ro4T!4MM+46yssaKh+&iq?d8N7z|wVpYGg@PcYqQ^tG0
z<buRG+~Kdnk&WLy=`n6T1Tr3P`&DutQx%Es`TVPcHxkWCkj)GI7+~wLmu*)*hsG=M
zoW-7Ng;9<pDsWc_;5PIcwHednJQ5MAD&Q3JTu!b=Rw%Y9r$1j0KZQ<oehk+!*Il_w
z7pw0T|BDq`^|AOXoP<B|1OK~*r#GGrCHoULsSULmDqUc+a${5p=+zF8mQq!+oCY?8
zELJ(UCt><*=H~46GU>`w$|w_Fn;>4~+0n>U(p}9=4plgM#E*JlLWjJkcf<=_xlQpv
zsiy9fkcSx99D?ULtPu`#`_qR5o5TEY@s`-<4t=BDdR{9lHS)^b0;%NBzn~5mo=bQs
zRF`c|KR+`aL|B0hpG;rJL#p3>k&v&64Aaih-+G!a#AY<VyBb0susSDJw8fUv$b>MX
z?S~*S7P-C<5T%O?c@A*rD&d>b(;9ckh-e}J1@1b0gH2B`e2EvbuI{gKWZosOa2_Tt
zWMK!om&JF%tB(qkGLwrjKikGCX{9%WSggnnFY$5|vMXw6?DCk}bJ-~*X$X-Q1P0oS
zQN&oRFhjMZj?c}6#HWg-=?k%M<KETMOiI*2d(xBRW<Fa!$k?6PI3`uJ!N%+)^(Ir@
z#%NtWJ{jRTz>?<4*qIM2E*Xe?biuc<K>{}Cc2<${D_0ysa5`ZQ|9m|vV(|5@@EvWS
zRAszR)58O_sAoIKy}gN2?A7$SRf3MUa$Xlln)RDMJuGP6gIS7U`Tv@wzHI*Og8MK%
zqr3Ws*|cVNgVn!aiqE6la%<?b2apJ0*aBlj{B-e`aRZTjo&&=qkZ3ghwoJ+8U^<|I
z*Z_LoFyO7IHA>up?!BVU;ko<jDXo^&p2fjbgDz|>K9~ypn}s$8b2nR~I5nq~HgQy!
zqzrgDgrIE!FKPlpSvlbWp$-hdIT3Z`oVautm~lQb-_F8peO-gn<Pjl<5qPTfk3Qyj
zt;qoA#E+$v?IVjMwRapV?6iVmk0P*WYX`bCjW>p6Uz(<jnHHglOw~Xz83)aRV=z$o
z#^f6SP!D@fB)#s>OZ_bH>Ml>djEgL`OyoGqdf>F7=3zBJpUkx|P1Ok4ALyZMCS1So
zvLi;{jcc<#f8nr#Wx&8=EG>Fplb6r03lGqXY?@L~r5z0C1u}oRvv|YZ4rgA#Hk3&m
z8XJwo=*IaZE@V%S%w>cg?vf0l3Z&U={2Z4bI}ppV+duAPFY({#kqK`CskUN?NN54`
zPb_1+qbiL)>7(7u-1eM}rC|f&7y`n33;A6`z*SM-{ZS>+7EAuxARaxr43w5nSIagD
zI9ee;df8nXg!x75`Rpes@=Kk5<Q(+m!C6=5w@%P+T$aZcJ}tA1j4>Ku*jL(oDBIq8
z@AaBiy)6Q>FSQYqz*emUsC(^qQv1LEp<jVW7BE5iT&bgrT}l(IfO%AP?T<9!pD_PR
z;2?y6dLh5g{+h=;LoO*jOSs8A5FgN%0NA}?peYpeHM!>ZGv|M~xENQ5uEOPcH&EB6
zRNs!3oyW2b?f1Fmg=Q>(j_Zt~YJvE8G7*=Gb5C?d$Ak+wPN?c5uSGb8CW?ng2^>lC
z7b@u_=?m`VaS9_w0D^^A&<pnM4lRk494n7xQyeSJFS&YGrEg?W9;q`?Q6B_m^)Ym}
z1XR+qC!HtV<70p1Gh`I3<CjX|6LJ^L?_Q}MZn2*tC(o8Se1A!)=rEJ|YkMwgKXb*e
z(=R3wwKfuD6Oj$}m`PA4weWM?;;o1-=2VOBZ|_oYdDRj%vfNa0^h<vFia!LCxL^JQ
zZRP>BRrtXR;XlvnXqY%ax^fpmWz1z<+f$If!Z~h&){6BF-Ohh$j{n=1=}Uyf)$8jF
zDWoGV`MBVR!*xez7ek5$Z0pu)=2_<<%s$*HXhwU|W`lCOu>_y@^9lLv2s`DW(EU8O
zgkaPzj|#9y)L8~ZG(2}BL|0VwhOy>_5wJ(>mM6TjN7Uwe<u#;?Z}^n1J?Wv)k>n@4
zER4wTZ?bq;_IBLd9HEDm_c3A>GoHxG!&biN^J9E&$M6@KNC@W6LRGf11Rt8GSXE9q
zF-h`A@oFB9jEcld9+@YwIaWL$?MOdwvzs8++G-HsRnoF2$#yfRv-Y>yvkg%72KXYq
z@>lRFt~a{Doc<|u*NK7Hfo%4WRofnr)!A(DfE&C2Y~o)9+EL=!L}tFgpxclkuKA80
zCObvLH7OA^$B>ZQi#IzV$ws4LnlbHzvNvVD?1?ewe7NCT1-^8&#%+VLLkOc$6@9@F
z&0-#iAf@rRCs{OsM0Bc~c-mL}#MQ2M0=k*WrvuD%$|>z`)=Z6W@&%>b-U;AKSK?~j
z#ugW1$EqjKrt!geffcK8SvfJBvOlc&Ac7YMg?uoEFI7v))Ydq1^F6lqMk$K1A_kMz
zt2MurFaH5X#O~Mj9Bx=awv##gkSY$fokK7i#g%kRcmV4*>vO=E`pVy*_w&zrP-DOV
zMX5Pi@0I+w6??lj@IyKEPLI>yG^k<HSEuFpx_HzL3R!`#f<EJ49K0Sg0y{`j93OJS
zJo2q6ylbBYh!4Bk5;hPmv@vP0Hz;GK(hasWjmymB$vxx7?#xISPwq-{@b_hdh!3Y_
z=6WNhC5xIFvb&lMV^8F5u$tl@W=&0~={rcva;%u>RDZy~C+w<)Mmi)lVw9D>I*##j
zxOLAkb6+#UXE6M+9fk&fW=*3wlVWYWHh07*MMh<a)~YPwWP&7rsH@`yaFdoGJ8EUx
zmp4{}f_Ff@5LKQh=o(vzr}?Q2vkz^~_wN70NpS`-7&ET=C4cJt|DiVceWBFu|1LJN
zX!cSy6Ta8zd^kc_L=JxV9r&a*hCEy&TIbCT6!s~VGG3A3^hTA($!DQ#scy<D5?hAE
znnhKv)rPQ<A11w~;!M?A%xAF)^i*eFd6A*2Th-n2oRNB|lG7u5s!}|g<S9O@9+3sP
zf)@vQEA}lZg8-YG>$eO@ce_JH6GrsiPakNAcojsaQ8!0%6r`7=x^cJ5Vv$#^3;79#
zS#@V7MXV2#<TYeccT54-K_l}$rYMxrvyU=$gl&e&%sMm8ggls@I7vQI*x_I!j$Az*
z`Ll-bWt9SDN8NR}(5mZ-gf#(va9)Df{<muKj|lP^;x|Ry2ECJe0mZEoqe>)t5G;uX
ziU~zCWr>gLjrHfS$~9sE%){Dwf`Q!APNqx0#}6eEK*U?(z9xH{C2XWcL*y?1_iKG@
zj@k3fC77@ze0I06i$P_{=I4!TFPWkmSDD26@aOJ^hw<{p?sLh_+-845s$`ACzjqk%
z;@#n4)ia>Ia?FPHMJm>Y_<q1}4a??SjIg11avXldkNF<vzHg@2&2EJ64wicNBDz8q
zsy?W~t(NKe`9_(e<Qbu(iBR`b;(kINT}<CqMdktAx^64bjK4qWGu{5@`mkMHpD|&5
z(ao#tg8)CIdHiJi>-FiD`F(wEm_=O&*Jtmyf!m#b4BW_JXpM(g25thA37FK39O&)=
ze8qX^%NqB+)<P4StXhYH^XM6G7@rRg<~lHO`Y9zT9Stx|bh6V09IiR$)T+tbA@4B@
z@_4s#*;GFJ@Ui&=7DnyJcx!^b@s!$fDfZ^I7)_zmhwb`&M+qR38$;HbDi5_r)P9Kl
z0E<!&tyfZLt8P~lexyzc?ZcxMBs$z5rA!TIw6!8bnZp;$9Lz<L)dc^??&f-vFAV-q
zZS5ukwKZX8msQ6G)Yd|9W5{Mb^~?WyV{ZSSef4keGH!+c@-E}L@-CD9?Omo7(AZwt
z{oA{29e9^T4n*4RCq}QnU2UaczyWp7Yt6)x=`PtRn+x|Z$pfZq&{4JY+Q-3=%OUf$
zwLQb$1{D^%ylVHeb&O4?k~Rs!N_rc#IM%c>@<3=Jc~U#RV;h^MZNJe#Y<Tihu}EfS
zEXu`NtalcyO&j3%>}a}s-=EIV*vIO(Qf&-t!i%F(xONljst%dz92*V{l~;m-Vb`r?
zX1XkeZ_&2bL9RGxbDCdgD9!!T<~+Dkx4+_VxSp5{_*Vts6b16nC2svWMFM}G@PFSX
zvd}Ad2Nb7HTL2j!c%>bLLnXoqGdVe0Mg}NM(WpTl^x~d64Am-6zaHX2ALY9hvEK5K
zUQ-K7TYHmJ`&k>M-Q5sym~n-$KCe=d4LfQ<1@~TgCB?0l3R-%7y%V;zsMA?;YB-((
z<c7mT@#8U9ua6logVqe%A%-4(0tSxQA>bEsN4;&M_<?rc2FqRQAVZ08G-H%TE&cfs
zVtsYvT82Dlaj;g{unljLrVwP`up<#QQBz4cs#!Gl^agxXaeH|%2@Lh2;u*Cw;JW~`
zV(?x3UZ*L|{PSHNU4558{?FZSQ^0o#(<bpX2@KS%j2jbf^Ex+w0~a-Kh=hz&8|zo(
z1&NwBop-0FaQv;}%;TzY{;LMpkY+an0Q&GdGztxFhhcX542J^^E&!>t#mx<nM-4=t
z`;F*BuYLW$e54pT`Y3{4{M$!rkD_1VfUw|QI111-GpGup>2wkIpokb3vyV#YsF01}
zL@{=E-I2`nF-S{0UT6~QQ&{SUyNWjJA7hWJ4A^DDTcW+Lv*M2us1Tkm1uO8&@sh=%
z9F_){TIZA7Kgvvz^gp+wD)i!dbu^sx{kbR6sR}-{-6Gtn2q$?=TC`m!WNX<+e0Vh4
z7UOL<V92N)UmQ&SSDzQ;LkhCbqBQ5+9cp!S9OW4Yikgr?&J#(pV>P#oo1n%3r87WE
z#)^^Y{=GZS|LwG7a$-v%^KYvq9RA9X&buNzvdqGK=Z;^QzThxi`-Z_NYS%4&qp@=0
zPQh17>MWgdb3M0BN3qeb(XcE2wv;1-i;$QV(l$0-!$D2&e2{EfwW%@}NwN>g<bF%8
ze6e`6T>Q#&bg>)v<3d3jKIP2KmA5c#8GOy2N0s@D7-5o}F)9WYg`7kmGhu@9(K<=g
zr@MomkNE399uhBxEU3UobEP|DXB|g>2j5ySK|fIU;dt2Dv@b6-LopQi|5+se&tlXU
zQNK_;-D@1qG1fO#wPRr2!V97rr%vV+LdFw~WhXyCz4Kij0PmM%_L^NIzPa;MqclG2
z>B)GYL_B-VJQ47^o!U+3I&tTUz4G82T%J^6M+9$u(JR{3pJt*I&)ePampSsSdzI>#
z^b>juQG0a!IoJP#gzsx1`LDW<1PIKs?Ve6LKApI`9P>9c8q6(Rz8Z83h5`f0Jeo{u
zY^YSReN6QCwH$fl910f+n&|no(PkXs57b7y<BQ<La2Mn}pE8pBA)z=1vSvN2Gys@O
zs0=?^Db#QX`L^Pv9BaR%;;kShgiEJYd6(0t49|dArfi|(IocaletO63)aQCMG<F|6
zTNiqsTPn|HbbpD1@{Nuw%T0v5CgX_J@BCk_ejJo4SU`g>%j0ss`<kMpL@@*JZ}skA
zdT#ehD%w+0W`Fzhh`RM!$If){4gn(Fd$m9Qk*`(7`IADePv2Geex#hQD{FbZL#7hA
z9nBrj{qlS<m3h7Ds?DgoLu?lIx`n}##)jH?PM{#2j&GvG{V2?c0{YK=_#Xw~8SlTj
zrGTUjo%*UUKM7;h!!+h_MMr+Y{~a>h#?`LG9Y(q!B{&d=<85!ZmzhyRJfZ;~eO4rJ
zV*=4dSEt-5@3qbgjQ}nweES0`)ZU;;c%Cc(sB=Z<FFVp*SN+X&ll(V~JKCvYM>tE(
zTQ$k7rOe2)Xx9Cnb3H1=2G&ZQibabr%E<3PCkHSqeOfh-;#G}}cC!^6So!iF3d6!W
zN?;(#%D%!$awM@-Yc<5|+o|$G5{Q_63!d>fdFM!T^*9(O{VIr<O$BbcP-o1iIbpaw
zz@V#i<PNgoPiF@9Zec`glNCQ+*J(Y*?`#bv^&S>t7d*AFd~KFmze>f^R@&kHsvqK2
z8T2~YZE9+D`lpJde)3{q9?qmI^PrTgeXYnbJKzz!|KQ!KaZ(C9fbSH#Y>$}kUrhZh
z-Tbfr&3~uzzDxKwOKXg&r41)V<cy-<*Yq5hy}>8`yB`F@)EA#Yg9f__`IYmr@m^vo
zrIgT$cre6|y;v!|0a@U@Mjk;{h>EBUxg%p&096}ITWys+FeF%hly`J5(cX+kr<@9z
zJr3~3G0pUva%sFPnpWr6+wt=7bzXAZHEci6%EyOVF<&USx+NnxPB&JuR3n)0Mh;Vo
zjgE)=P@WDw*$u4m1JYku0jFqVFRiG9ioUVYfO*=oI(+3u!F_)CsM3!QBCWOoq{3GO
z_^Zg+|A0@Vh=YTC6fgeJ0{QH0qlf!=JD}8hV}xZabIG&%(GG7BJUD7n3g&sDaEA2%
zRKa~EPE}S)_dd>4K)QR1T$-0?6c@?UuN~?%YLd5b@F|ISMqltaS>zUuDS|dh{&g2t
zQVzwM`41a+!;~NIc4VeA{RxZ4rdKwd|1yJtwd<8f9<7;7aN#>~%ysGh%LNSEP$i<e
zSW~odG&%_j(o!_&At=f6^`-yQf*|}a76iC;Jkh8jQhCDL+dYc3K%ZS4t&|<u+Q+cI
zJlvJ*j!L?ZKVYPm9*ECnZ@a)L3b)6lUzC<oP7&`=0;XVJ@}lRJ=td>-&WFr@7WhYx
zWt?wa%NA-iFIoP9?MJI@gF<q@9@V6^I;`D|r4u{<fk4Rp@d6+&CZ(1YWGDWu>$zse
zstKBc+LLc!G4=1pNR0_}*{9n(LRhyh1CYxtE>_(yqn>cYPnsn6@lD#xT8=kL@;psD
z%oc_1{=~=|vtC71jZO2=?H#U6C}Z;xkZ*tQ8+-0TO0a#Dm{u{!B-Y<PI@JaG_4TWB
z-|8tp6$s~uJaEKNsA^P!&-{;q^6z1f{<o7$G*@Zt0?AcaJPC?%Xy8X}QwTGW!94aZ
z)MgiBp0e|rH(I>_)=-?J4%RxQgCSECjQuK;)=e#)+j;S$676VW=@!^+XK5kXAVN|{
z>!kgGkwV?2ZdmFOd5v2`3oU`L;!;m!LP2z+6@$5Ipj>VSK9QCc8WS1-!-rTEVaTQ6
zCL>9H6ngcSH&NuuKUFN1ZH%1!dBoa2-mT?f?h8!OY2DajZ$~m5Cg9K-2fU>zkna(w
zVpD`o#Pb(ND;^zTNb>_Ay|{AdiAxkUKZm0(D;YZIGB#2V-2SPmqoJV7&}xTs-fa@`
z{Iz}D(|=CzBF(D$2wG%aFG6g(#tfaPm(*k7{Ca;a^{S~&5r2q3tE2zsZ~cDnn}3Oh
z(U%t%Pz*Y@Xvl$*cZEKLjp7z9vKabknbM_iJ-Y#DV62K?nxUp&SJ~)T8M41Ddnz)q
zEUUY!ND#n{5zKJJ@-rOY_cixbAZ?$2lnAa<8QA!Amk30LvS>f|RF#gq4S@&9*>&BG
zk*11OvdAl?oZ&hrB2On#7+8RmEw?J(hmQ`u(+MMsy_28y|Ed=*i3ED#I_HHA;0%(H
zZv{}_S94<Npa1+@pIPIA((&s1-ZAZuDuwC^hUDJ)j-Q`G=e!dmsi$68Ucjn%vr@B#
zg-09Co9BaqzLk4jy>5MYJQZ8)$AN!?fWO=ix*7y*1BUZ$5ml9s?@oz=V`qyI=0><i
ze7e+f2j*`?*>cB6pI*3l-$;}2Xzz|p?XJRRuwg_8<g<Mo1qaq6{iDvnMzyHQTvTNk
z-pu_(Bym9^%2EM<0}yKNwUA6{p7vQY20PV2nk4$TEFg}^xO)$N{e$O_tN3H%X0BN@
zacL|P?(Q7d#WMVReVD?23f#Twm*o!?U&rrfFcD@(6RFZDDO%W){b&{&Ergwxv80Cx
zMSP6ThY4*xp0;%&logm+krj{z`LtJ<!tvN?>z^m(DxVgQhkdEq<!4gG=-4cm;;J_=
zSH3puk$kb^eIBZuX4SaOF=j${S;=*_JHx-d1exXd=e%h>@HN?s3ZK5CeHeal?!5o$
z_($+@8um^AXV>5Ru%O=-YBUQxtU6obm>oW*=ar~kLOOI2w8y--&Q>~$w=-Il_PVbU
zZV~@ZxJ?%zEl^4x%j4A6iq_GYP}j5u<CPU-M<*1@F-jrJm|rfkrg9m)ssJBl(mNXL
z3Q}bJY+*)Qo;dShZ<T1DOAz3|AaazQ3d-R)QCnswK{(hf$^Rrt9{;VPIG5sPp;GvK
zUPD2&qTPrLL^;q)=o42&R!VDr<N2*P2!q~1bCg-p>Ek>BBck(g#8({UqSdjxYxch4
z4#A{mj4-k@pj{{P*z{cdQ?<m^ueLjH?WR3S8%x;JHi?-J`jEdS9Q>gy9!3tIY^=3y
zI>cI+R|50ss`h?eJZUMgz8WJu<nIntT8@6RnK2Z1jy|sOrbym@_-kZ&@lO%{??Y;o
z_|fl==@BT*C89&xf~|y>b))~hO*Ml|jDEYt^zlXLRWvSqtCFN07pC(&8rQQmd{bN$
zi#khiQ0XE8zkZ|OTI2F?!@v?Vo#a}(PPeQGU#NwOKa7#~KwY%&#a`RdN*+zpL$643
z^ocbrxeW6Bwug$$#ce~~n&EdM9VcGAyv}31MfB9wmqiwAaIAD%Ln@m^)WjT@6uzdl
z8<|GTm%7f-A1+Im#U)et+2`M7cV65z?9Fitwuu0{VB!tS#^qMR(0j$4`dY*hyDM{2
z^}sXLoB8ih>HAau&wo6YNpI~>HCea*NPoE$<gD?+0lry>d@~5|UhX<>ZE=|{ZQ5s+
zbCL&-JX+;T3<A=vU%?(lM=8ZiI#ARadj0lxQ%9rs-*&d3tDE}cGjyv&Zh=JK7c#T9
z*}`9Ci=7*Q&w)mRE)_tHYW=_$tMP#X2zEywyItRB`m|yETOnofSk0f?%a*j&BSyh^
zMN^^ukXWldj5%>E-*v3?h*eRLvJkAiVk2EwBT~ur9V3!&CrPQkxvBJ+^bRxpE-M-u
z&NJ*c<O<=XXd}(i9EO=mg{b`w9BSSW<NF3`%=uXcs_>g(*`wcg86c;s5AcnpEDfL1
zj&|fC#)@2l#KcAmi<XhvfmBZB2oV=;w=u^NAzXU(+(iDb&yC4Xcsfw%TW9xu`gBYy
z)E@fBx41iPBrCOkl4{fDO`E6~Qu)liVF!ju4{GS*&=4LE7!4)}Uo7oz(Rm47l6q|R
zFdU~to5DKWM1{}I&v<EeR{nm9zZ6CN23uARK#kQc>txtPBND*#G{A&PFj5=u-JZvu
z4<<-Mb8kOv3&))i$i3#i)yl#zH2ERhiwYX_zR4(56}}jfNoMM+LBVHnuhf2^pmA~V
zYn(w^lI2=^qkSP=;~G!9KCFONZSs45V48p@Q2(11nC&kc?rGg;CzY<GM6EBb<#AsH
z^d@K@{0``?eC%$#ocw9oAy)QCUGu0jlmG*ty>V!emHcWB@e?!})~bEK{N|pl(0FCq
z>M$$ThogjO)M1w#N0jZxl{MA;|AO|vujPlY*gC3y(uo!b{e^AcN{YFI=!!cveH$cS
ze@tr~$NuTlq{&kM^@>br)9anaBa0T&-o>=52dgZH)a>2ML_%Hb@SN&$n7p;2LvMR}
z{P3$!XU^s6UJK9=oxdXv?*M)Ox2j|6x2gk4q=*$i(1*(g1_w?w4j<0}LsBI)QN{-Z
zaK}Z+fF_;RsH)-~QD^YHe?IIInd_vkk3gz2|ME40k<<EJQOdPODWI78>t@8~hna6J
zvht$y8yAOA=7gTN*rM#RrR1o|MA#pbDs$YG)<-$M@nJGpk)=AqzK7W8xkh;y+EIeg
za(pvV;;PljToBFar8Cj#ADK*_YJ<y%Lav)+ORA;66_A3j#HOLv!$FE{mlB*yB?V+v
zK?mId!ViWr#f;DzwJi$JIc*gmg#Obxsb6(YCbQN%wVs^X$1a~M@W7*QaAv=iZ}mM!
zadXVtYKh&YOrv-;>gSvyaw^SZ-%zZM1)URY>y+gID~$I{r+WIj$JCf;HKKNP{ukGE
zP~rn!k)3~g)=YGN?}??vsWB8j&b*q(qF=-+^AE@3@ZoS=WbPy!7Z=hJz#5@XbiGlY
z9Ty36S^@eNcpot}B4nzv^!a2y&L>}2JGyvGD5{)=T&z!$J7RDAxu6;Gii-ZxtqQNj
z;peNPX3B#y%(%W!W$+a#azic4fn>Q1m-vy24=@KVI*bTay}{=gVsqUXMODZLD+uK%
z?&_2343D-t+`E%#gy4P~y|I)?qQ^tHv2~XXM`)2(6AS~9dvcpgrKo~Hn>1UeJ*E_l
z>tq}NLEoib8+Q3K{1n^(t!Zxx^(?$AZOn4Cs$%i==gTYtD9Vn{=CNr>t=43_w$*Fv
zRdebY<_o01=n>NY_&{lWZjGRe$MS61zP9GeTQEHI6yRv3-}%&O(>}8^W8NR_+TN}}
zLvfk0tn`H94}$&Q2V7gM0?gmaJKhYKkPgj4YcP-dF=y6w<Fu29K{#mo=)U}40weK1
z*W4&4z6u)N9|+4QshdRnD`*&gyI9Ev3yNy}2s5@LD|28e^&I4eo)f{WT4lYK-$bJ(
zZe}-tg~x+krlyT0NvN`(;~{aM^hLf^FU57>dLywin3bR%{zTie%4~5RGsclfy3$~4
zV#sxCpR=TtoBdJBx%n-oi3C-rK~)N|faLqs?(OBO=`;nw(8P^`Qj|p~XEnDBr`)Dx
zT0uC(`SO+H6_=aK=DUC+2U!?6qO6DQr<MQoVyssL?}wM|oMTn>#f`@@FBbkW;RD|}
zN^rH3fN#28$T5Yp?$_b4nuo=JKcCxBGyz^IYdToRk59h=&1|3;%g*#s=2qkP<74*Y
z0a=KN@y`F~8~$rYr0^T3krjx%K7O*Wc&jPo!7lbZHvQT>^%t=i+Qm>;%+~emGZ9NZ
zif*l%8Pd_6GWzaXd$&9L6UI79+|0bptsT3;!<2W2PVQo@5QZ3G*3h|bwHBHu9UUdW
zQOxXU89^4@w73#nEca+HH;g=^?fO%Y*GsrGlHN$(E}|mVj?29~wD^L@tGkqjF%?S{
z>jl5WUM243VVA5v8X$e-DQvqd6Nl?<=JZnuq=mxXtmbj*TWu6JdgkP*6YO`D9K~RM
zj<ThimlZ&lTL}Z-^knw5w~)fuW~OeEqkhXM-E%*bfJUIq-FG)Z7(+eki+vBf>1IiN
zN#DbM5nxu8_{!|?=gjoj3sYJTVDG-&$3IsINDmAZeb<4dTJJ=C%>ZMn^)><OrF1a-
zI2HuY$xqfWq)vi(fUy}@`DAb3z^pUQrgI}Du}jFA;i3!pdkFphdwd6f5Bw_KiIU0H
zpsm?0`yImLp3?Id8-ITZ|J~2kgMRL=&iN0je$)b!?ck3^Nt(2i7JPBYduc{*yL&^3
z^8{Hwqb^BBj{Q)Nm@D@5%5d<L)^6D&f7av*Bo%VLkhd$?6O>dx&aAhvTCk^Bkzf{c
zG$(8?2OrL#<$4uwsd>RZH3^S;yd>%UG=b9<NhpX*W(wlQAA6^fxK#-#rOv)mP>Fu^
zr#xSC1s4iYL+kBkbd~!|e0`<K0g4B)w7|#!*CAF+kl1tBJ*m|GW1AAU#Y%W&p;JxW
z$;N089uIoRlPlzWf99Z8lMMrMF6QwAolev4^MxE5vjOH^h{2bM=aje?zXqS;qu&T1
zdWsk06ZtWh)E!UMz$%FwFJ=G_i|bARP~v?K{&_v~eH6h5&jaey&#=0wgzAq)j^7S+
z`g-51n#|VMzxml9?66&xDH}~^jsmu{co30YahJg6AKSQ&!dQT5Sz^2U6(<<@9K(8r
z3NN?t4Lf5PZjl3HJdcT&aSG+fz?p-b1L%8f=in!=>fB2zDSUI&xc3qNyeNNK`+siE
zZ$Lu)<k5U#Y2$S`>JU!}c6wWQAR>Ca()DY&L8?9m6fN_nTp$VYUfa~OU`}xsc#+9>
zT%#6{>@52`*qJ(ja)c#8-_ATveo0zM7qBO@S;I5y_F0<tA1f(xv}LpgO^l2J_GH0>
z&sI0%VAS|m!Om8!Bp8}(R9pxivunA0dmpE}vMT&31lC^l+#al`z~}$Is_Z#g5c%1L
z^^SFbP_>SmL~v>d|3r6U&DyQ-ZB5wZP;SWsjpXGwsqF?GnL53s-m5IM)`#8D!drx*
z#;Hsx?Wb4d$tJ+tKz(+Zaa!JLEZ-<7%w4ajXr6xEG_##iv!0vW4DlL3DqnUoMF)^7
zi6C^jY`3PdS2^ClY$MaWfr&%-)S({Ox?*^D*2hPGDKl(o9>|oeIB8-mOYSg9w$3#v
zpLEu~l(k?5&JYB$Agt$|h@1Uzl#nIxQTt}|4l@3?clO_{%@hr~WX3J>xX9@7PkVmB
zUy{VapV-npKM96>u~S&27<j#H=D;w)v?!Wi{H{DFp1^4(HcF=(vh#H{y02GL<eCvZ
zGF$cId3lRWb#-f+TX*z1@=5v+?*ZATx~=7IWYpeL&XL9K>E+7iXHQB<Ycz~rbJ^XZ
z%$NXFN?|r<`$KIH->_)$zB4v*_wxk*p*Q*=;hp*pdBxmlH8x2{rZmnRi1$+5W&_dl
zd3F_|=x(=yZSKOmDGIyZ@-2Ol0qI<O_wFKM1Q$o=Tr%XIj~qdO<<`;m=3D<c@O7Gj
zoNM^|YY_Wj`%UnNqcF`es5QIc`sa*SGG@bH7bg%Pzw@d$n4kEB->dteYXV{H{}_Tp
z>-fy|;%K~nS!CK?#p7qK>4fW2WW~)G6Ce~#vjsV>!634Mcet<_p$N{bJLA~>y0t8o
zU@-5`EquX*+yGvYin)cK!t~^=d2OQlBw(k@an(;2?Ebt@|6ObH&s&|(f+2<)jNT&T
zi8Hc|vh`!5FTd_2fF2{;N<kEiZsUW*8Zz{c;&XZ;fLW~|21h!qU~9*!=f~+^hP^)-
z7x<t__+%yTHisahC){y)v03jD`LDccW8UPeylU#?^qLTR-f9z6p{~Yzh{oH48^%Ul
zwggEkLU#*7+blFFVpv}>Q$LZqh%A1OX8wuXw}>Q${+*c~9|r<0{*C<7P|c`p=)Itg
zB+|z2(s#?mazQGHy&Mi@tEqAcW)L4mpM4%*XT*zlY8pM>94qdy0{n+dWr}|l$zSb8
z6F(JeL#E_tXkE@|&!SoyXCDfKz<c4ElUa}8NWQ6i_!x$(z)cvU?{)er*Ye{7vw;lT
zwanlk=&yIA0!!Xxu~HRtUT1-Lq<1Tq$IQ=C#D~paM?!l4e@?g%E;!+XggnDb&*stT
zchPFh3QI^H;nASi5SHf!DAAY873IRL22>XTTT}ZqQ}Sgn9d!WAGVgo>ur)oNtN^yA
zN1OV6tFjygx+hYXtkhMJ7v&Op0?W^Ixd&sThrY#mS=Cz2tVZi5gnlXLtse|We`tq?
zsjjnui}~Xb7AxP#D|ski<u<;=Uf0)1)u&JJHLOe?-Qe)N_v&}fwR_QBVboxp{#DNP
z9eepA6YJz)A-`xWTEsw?hV{Ox?1<9zYF_a<k!d_&Yf6eVuRj25LPq5dP*;Q+-DVuu
z6c^WOsCOn@XpWE4K-NgYoJ1$%JhR*RUPnv5a`wpm+7xgFn!(XejWwHi^{Ru`<^E*T
z!&^eWxesRupnuh6{JYn2yoP0ul@x0CbW7a7Y^<Myr|19#rZa!R{^djRxdzjgrlyrR
zBdSxz6TQt}8>C+oSrvC)6K%lSs_87%iC?_?m_l8_XhD+0=y3?3ZtfRGlXy(#D|mA3
zKfJrbB=$)AQPXOH8w-?gS*`w=m(>$;^cUrl=#m6)-{9eSuG<Ytt$oVKl4bUJN9p8v
zxIPqM{d7{rDDV6fiD1Lyb4Tyi5Sk%sV{VdK8l?Q7si)1_XccNI%_)w$3dCL(fBE|0
zp$tgQwJ8eH+cUyRV)MkIn$On<kY~r7urt=$XrbmnEOA1_8jcuJ0(g_0O5O>mp8f2B
z6NSOz#O!ahWPIc5MY<}k;62mjt|&OFx^2g$7qw=<A7l~e(fPS(6{aB^^yK_Uio-XM
z=6u0v%V%P#BEO~GGpZ$4I^7RDE8?xlx)3|<QW~<X%9p^W9c`GidcOL_BQU$Bo9y#=
z<FB+PH?~9o|Fic3ms#Jrl?KA8Cl=BfAIXk;D?Y<23fc~b<T(irdLJrXKWL|Z{b4DQ
zHL`1Swc+3!ensyPGLH^XTDMW~q{Zo7I;ok^4R@u#axwn>_NLwhS7MTb(9W}=a+hfz
zHwBGj@0Jno9;TV4uaDP;MGK@1pLmfBWG}Lphi>{rd0F;InorfW%4bN)!bXjGd!o}L
zOW#eXYKk>3OM@tQMAGD7N83c%*I;?XGBa&XBs6)lW>;c4yUzZ@w<lgPmE>6hHy)D?
zeF!b}9m<o#ef9*!kT}qfFD^xjCUH}fGk{JxkAmugPizm-p^18~z#jx#NhG*KAw?cg
zvn-qUKqC%%O1v_T?9A3_mAyBrl<Ydhl7})UC1nWSS9!u?vv1JDOHX3P;eruKU(gRQ
zl!UoY(iFNpTfmAWQ@_c}5On;H!x8sQl+t0M=O;~`l>v^eAeW<Y@;+YpJDM&82(}4+
zVQTGJC}HI5X*VRt9QX)!<tH=&Z244*4*JE!L=Fce8Gd4r{eOhLWk8he+BOPEcL_*0
z(jrL55K1=)2oeL*NC+cHNtbjfouYui&<)a{NXr0<bU8>XF~Gj&dEWKBYkm9Md->BJ
z#?5_S*LfZp^r!7d1P-mmoo%+z_KRO_%WJ8+Iz%g@lw*2JO$$%kBCakk$~=qhk?viW
zFv;Ht+qe|o1nHf3HGPztCAUGOlA65v)0QVoV32CDS8;#UUj!OmEZ=OqMjZTVn6)e{
zEM!ZLG99u<f|JhefxXnv$~r6=!L`yyl!IUdrwqC}$}hulI%FrO>vlWdnpg$}Nke5?
zyCAMimND>n;{DZ97+EC9VqjhF1g~EBlMFL%^Q6dd47e>ABlzjcsJT>faA-_fU$H32
zzu5!Vna#<JiSSnI_0>g_2DTJ<izi)<d&v}t@mK;pUW#E}fq#HbIn`?s@bB4OuKgs$
z{~GT}3!-in2RrxqqF<TgXD=CaA?_7wQvnZp^x|wP|0->fsB|EvtEnehmHC!E0KxWw
zX^;DEk19*xbh*r*wf$Z)GJ$WRxRx^Ceu7{hS=r0uspaD-UH1o&kH0|2iq_r%t)(y9
zE_;sy4&oZ=pzvw&!YcUEM{@O*Xb(rC?Bt51_;>HA{e>S_;-5hP$J_;MzTRn=j4_rV
zpBA-@y1Lkl`f+{r=dU4_7+9+xPJT)ptKvwIWQR=N<_y4fYd;vkTp{z_f78E-YYU<7
zN^-x4ouyGTOXqy@rqtKmt--kG3S{PL{joeZ(2)OghFtdX2bR%{l@$3S@W0-)BU%I#
zT;QDFQ4GQ-(vLOZ*P;LYuDZj<+=gB3tS>Jd0xbUTW9~mg@3=nbcpn{?j3hW%S@^Y&
z(HT$Mj^T^(=*Zn-$aT6fs^X>GWxW}s*94XpALk6G#9=roEE_GAJ%(dw5U&`3q;c`8
zl_ZwD$N_Ls8SpR%QXZd9mIb+{<p-^g9VyfpT0n;$qR3AAt3K;Ud$HBWD4W|pD0@-)
zkXu9V&GX_+2rCWkjnx!2-C<7hZR~RIv_3;(qz}h<h2WiN<*t?S2Wk0~3jIlo=kF6!
zg1!%#YP-eXl=Hhs+OHR+(H}eW+2jn)6SSBZ6rK`fnM2RRC2}y)ES8X`-=t;{&SEm0
zKszy8n#ziHW<13v+D5<Z6<>l+ds%wGsyNm6bi6>(3DZgjLu~sM3C;Q#_#t|LiA-WU
z^!CvwtE;=)LSUe(Um%WKW)Cs7g;CAfL@`Y&eV8$H5P7JQ6TspIlKEtiHKCk&=mEp2
zi~-4N_=^4>pFkr5DwaJA{y3_u(?JFCDP`$-u$5$O9}&XT$Js?o$SAyN9yu1O2z_M6
z!tTp#2bBX-yS3IQr-`>jxsP`jX3Mju*>2=`^bjnApceqc!4F6HdAulmhRB<~!^|bu
zl9cZPzXHLZnlz~qOD>fV;9BLVZ2Q%^{Gn-)5Is2Vgx<Mp-Hq|)9xoeq!vni^Fyc@C
z#16&ACQk6=jlA*abl#<U)XOOB@&yQ9CY4-zsq#70+JK0iWG&U|a>r!u6mfuWp!mMW
z|I-(y$&nbIJVen&il)#T%b(x!p*@nma)=)>Nd>Hr*r>R*)UfG?+>)LX4*$6N-Q)p2
zWQm{W%Vfc?E!zZ>afgU_6a8~&zXwN4lUMm#)!G`c@gBdyIS9Kj`K;<0-F5QSyo_50
zm86MoXnArJ`3~>kw(z$&nP1JtF|7v3n<X^)X0i9*kFBGly7{Y`dteQZegFq`4>$-i
zk+6BEGCe}ipTOazCF8%`x5<_t*FcV+K;BM16x+1;!bPyB(fn1$B2fcfIE>OZ>hAv!
zF!evH$PQ)*V~9PJQS?X9&FYX8BFh`>%ox%tDwLjtcB=US#<3>Wm+v?ZIppE{A?}cD
z>B|uyPIo)gS9LY-<QuN0JLS)eS0<i6c4<E>BU98>SbZrQ)*%$1Djozz(z?O~1$Ko7
zhK8FcPCY>tPdZ7e`(gwdeORoy+)7>C#dK(Kx_&;Ace_dC2_EMCB9ZZdj7Ce=*!v8_
z3n_+nH0PBahjE{kh`kZZ#jPXnM)2?SM9B*^!1b9&HJ>Ht$sRBNDfYeB=YE*z;<hY4
z-{K8LkGxl-HX(lL#t(-Xr9z*8ltHIIbP4)URr+1NRI68(cFFfAf8Ap5fOy+{d>!F1
zg!>5LP=-lUv<qzpy@R5k%0Y!jvIF8h_6kv3?`?5$a*N6uLt>()6wvz{iO1~Ay2~wM
zH<aWWlHGqw!SP^7fPWV%*0xr|Pt+rX3mW$nrk-*?LI|i7z@H_t*wM<XBJ{D52&Nlw
zY<da|162YA1jVXE*O@kzrVI>UD9O$e^v{J$wT0A(;w$|C;h6KU(x3!lzyoSIANasj
zG$>jYQPZZ`f=#cPLjm6b(Hmp<Vo7$04{AL>)nMzCjtH(2aDPQ$@%-X&gT*S+$yCcR
z@f(r+W*;x<KEUgXagGrKnl-JL;oQqUV?SGsz*h~^OE%3Zt>Y@87^3XS4j`BF9Xohr
z<<>9*0C`rnNUrhyyilW_pEs5}#@OZl7*umCs*+D3FPP2MB#>U5eqf=yK`W!CIFfYz
z`1$wr{q$Uo@5(E+Q_+?`85agnbJ}rmwzDyTAk~_%_{B^F#3GDb#d7H0XDaxSLs>p*
zQ}?d(^57@ouACwNyYF%R`)l?tH00~3RiF4%IyUoUT6o16l>OnNn^h8^i2re;{-^q}
z|Jx4(z$n_N^mX<B+amTiq+!lxjW5<~n-Fubg3lv~w@+`n8O4cyr#&%lQ^1@H^C|t0
zS5)+PXfkJ}Kq%CUo#=+L#O4v<V2!y!=dX#mhpR+9SmymkObq8SA*Aqwt<|DL)GK2Y
z;W7U2!&wX7n0V_bvLe%wQWQPEc%Omm{FAs`$lMoEM!F>2q7<PW@lNgnc@jnjZeIU0
ze7cV4H6ClhJE!EjaBnK=ID^1g+spT&@+w>hL)zIv{g7%}0_pRc=8gL4$EO^%hKw#z
zBroR5je<@V=+-uW9$l`Iie)aiFR12f=CYwv`=8RO=GFX+3;+Hk0h}4!gH|=!{QwZm
z)H;ltKr-IY$9{A7iN=y6v%fd4jB<3E7g(FiR-IBjQK0?0AZo4KQ%w3WhT|ct5$jp5
z6e46D%($a87jug1!og>nwd;6Yrfh_cnIzGK0+Z(OxQWTG^PP%qM~-M5^0?QcDLmsf
zseRyx=4<lSB9{3Mfuz(d5DXHsWdIG4;wm(UofsmyG(sdJ^YTM@u75Y#$2?nSx~A~g
z$it%I8U*hXFwGl3=MxNhz|^-gB*=8H^uiqE=#zU!B9i!<B^alS)waTXL(-HB;BZ!R
z4_9*ACs9<esqqBr&)dsUEIJ!A)?=GQ)w)a<rKlNlr$bILd=f@|<8laT?VyXN!0fXz
zIg{x?P|IfnU>vPW#%DMa-ONWgF%he&DtBz~ZoyI%1wy!a?NLjPqD<*HBDn1*S)CyN
zW|CB~*2*^i-sDeXH+gp?sX9P_KC6jFL_a@`ffUoF!E%t^eO0PzSE6wo@96324OkBR
z$TqDqW8DeFzCD9IOWC^A%pN1OeD3FWJfYg_RF<3ZzT%^$V)@x;17h<=o{-4LeJa@D
zC1qnUnLLf3<oSs2xV$Gv<&zd}%b<R>5i(=@-`;9D4)j~SBFo)}E;XgAj>BnS`P{rD
zTTmvrLM*NFqJg(RLJjxr2LPn$KK|X8a$CUy=Pa2;E(be+a|>RmG8SQFIquT_)k-@3
zlp>KXHb%#9+@*5MuGqlSQFB^#;p>c*z7v%ZwRNOnkkUKM=u-==XnwK)qbDzRXngyr
z^^Q4Wrx4C4Uh_xO6v(uEaf$Hi+@Aw;Euup62t8G!c|Wn=pOJ>d>m}N`V1kLCndS}#
zbQm;><&15Rb;Nxao-Sv8?>O<tovv@s3oxtBZvrFU^F1V-HpxMS2Cg4V;mA6$rN%)c
zVAkiLxETA`!{K>!p|aH58vR3TdYU?^Ur|i=D;-5-x<m(`w(ZiOCsDf*QpyY!*6Zco
zs4}KIjtXF&olCyIGba3?E-a3Vx&a)(s>zUoaeeMsZL?PpTQ@{um?wK_3J)@nA4VMe
zSdUv=l`Q79ii~c-GP9Q*narM=2G=+>#}|*WIZrGvLrnw0khkN%RKAhHL={<|Lk@>D
zxe!!0CW!{mq8Z;dr&Bs11RcP9sxw$T=U>wNAw9_w;j{I7!ChCS4*L6y3E~?#e<9d4
zD!w4F`$&bs;HHXrUuS+Q`&Balbg`<b_{p{Fd&ga>J2EUyb~b&P2I6GF=3)c(i=7b-
zd|7D?5`$?l?Y}wEpKu6GJ1Z^%59#3}r&1O2G^;B)wvH8ZtoU=I1ST2rU4(LW05V`I
z#VD==YI5|Zl&;ZL7c2g|8?WxWdfW2)t3*&m#=)JT)K?*<YkmqqN<qDo*Z*Z!)*D05
z%Hp2PhASu9?W|?2kN$Swh|?-DRAB^0yM2nJl*6)mDNUP$rNNLWxgs1bMrC{}D&$J(
z_Xt;Td{n^k`a)miLV;($GbOUmr~83zA*aqFGm{i&{gXJIz*GHMU%N6-8}o+SW(^_K
zQHnd<P6qVNyS<hWFmv~nlvTx0>UP4jj>SQwOAlTO;nHxTwDh_6jp5>Knu=RP(-XE8
zp1h6Gf?Med4^zjv0YJ%>dpM;^44o=NoqDR<6{BjEg=J84A#bfX$1Rqz=@m`#0r{-a
z<&Jz;@STEHmzY6Z0OC8ODS}0!LItZ)jHOW9nu69pj#ka*8Am);+RC)>-)(S`c-5}Z
zN6a(@ecqkKvJRH9Y*=rBygajh@ndsG4wy(*zZbqd3@^7d(^A^pBkEmGf<*LVVmz5>
z1!Q+>W+Ur(O&g)D;!sCFjd}Q9SCrO*1@b9_AEkO8<k?3Y+Hg=u+o?{TFuyFxxN>wp
zq^a3=OO1qFS|iyRUkva#HVD7X-0V{L+IuBVGDbd7{WJFx^9E(|13R*9ey+)(t`8b8
z<&}Fe0f)mpAxw&~1l2f9-_ez~Z&%-dPiIIX)?pRm8nzT!<DZgX#<A!#`#5t~r}rX~
zW9$&Bz)&;$g}rvlfpC}BDgZby#kXG5sy4GFVKvC4a{XQ#_8RsbAnUK@3VHhod9PDL
zp>8c-cp(+iTV>krKrXMzhV|6|xrZI(?kQ&05e?e%)^*M#{|nw)&_R9&JIl@Xi#qy(
z{)z$yE{he6Xi-SMn$S*JIN3`<pHr;bkGhw!iY$NmmnJt11E8O1WsXlrECWvs{_-!m
zAt&_b;S{tV2L=q~kMRTh256I+bA=c*Th^lDp_Iz2nPj`M5u7uGVU+lN29Ix;1T^q|
z7_y}LO2S&@U0W94ac*QnjC@DLnEqAwY=W)+J&_&$opubmDDUH+J(g$2COWQS+~?6D
z1(_e_w>YOGY!gkJ&JK$6HIF_vI<(XL=Ez|u^Z`UO?<H>SMGsgiG8s*+<pEkjBu&dP
zj_B88WP3(U$+WKYgFD+bWp3{NA=vEi<bcMph;L!Ao7PeEuz@55&K0x;ZX5&~oX`=D
zwV^GR!j0)aHRwwbW&MH%QE<M`heB*)bf?~Ge@al{6~o9@=vs&ZGwRPVFeW~_t)lo?
zWQAB2^KrmI|0s-@m#*(NKN#JztT);Gv6Fkej9%fhqn85f&D3@y%>sQ{HO%&W$)F((
z2ad^o>2<m2b^viL16-h~*Uey49<1K&>Js*!U+)7nU2F~nJu#P*E3FB(Jxl%WN0@W^
zEs5Iy$BtawcfgoXBq&_@wv-rP)2jJ=>)oCzlh_viN`5ObNPbVHa(ID1NRU8m23jrh
z@cd2x#mAL$U-aT$UTc^nr=fZI?YZ4<!)yj0|CzueFmoC(d`1v_CbUx-a%mmQSn45_
z2SZE#z3SGd&m~6oG>m-q#Yw^FBaVyc{cm*Ecl7KB+D&;a8JDr&gxw43G^=U|@)j`g
z`iMy`6(MeMLxoHF9ImVyqi2$BaIi%c0ulo1`ZT4W_s)hb#p$!-V+56mk^A~rrlT8?
zi8tr?VR_eu2g_mcUgcj*OL}qrx5adh5##n24-efIW}Uz5)EKX_<Hi;ec;28h3hK_H
zhFN62JE7e=G^upVJ1gLzk$BU3!o!VChXQsmFTT!0{)b|2Sgoj*H?2~A?48l~*2=0-
zSpa84ZJFrAM;O3|#v=4ql%0;ugU@3U6;!fu0qR_&w+T?^Nq5%pn=n(wU(Krt-A}n9
zO6Wocgd%AU`AepBXpYUWNL0IP#viGv6xdKajJrlSZH6n!M|huc6x4t@4vl_wzonK!
z*7Qy$OCe}v&ST{@t)&2EQk!eLm71oxf^aO3?*Uw8?zgzz&LTvTX5-?EpVI;0N6{;P
zfh?}#R+a47?;%<Sl-cL=N98pOL@IC1>?a1@Q2boK^m<oLz$P%0;xf^xK_EuX(>U*3
z5$ZW{^g<ayFi0tpa`);st^e6g{<#mE;N|)Zn@eI+p-&h*|JAlvvXex~K0?S}ndJ2Y
zP^ELEMiIEr&bP}P!dVx?$ejTz19t8q%P7C#wi~11D<TIiuE|=G_OX9-q<SXQ7?bTv
zsYBx$Ouh%%qt0i$gw(D+Hm04fGcOwn{?1Y^%{PPniZ#|4{$h+|$5yPQgy)}a=J`L}
z0=ejf&p_306vNZz!jz=@fdb-;HwHxXPuPKcBvWR6)%#!ak^j3KDde|OH@RbDycExX
zCN}-o3$smHg-}W+rN<VyG*3)F_dEm!I~3el)|1)pA`nb2T8uYVIO@~$O~n<y-GB~T
z1?s0CaO^J|1f7$6<G+Z%$7{bOrT~S0Iu+xNx3c|uTBJ^S68EeyC_XVLe)#K*0*96#
zPKYyGtnzzIeXK@Y!6h66HV@F-Ne9n@>QVBr+^A`$*rJFJlNLz}OOZ!{oc@_@4*4R6
zmtQQi65}!7+&{aSPZz7IAT^Z8lDScc+Nzn&IVi09)|fAu1>6e)^aNs==M*{=G17qZ
zP769>^%kffR~jxX?f)$LMUsPQHH>Yr5JIO|d7Y{&KetoGkz@#Lsx51u5}gUe9<6cH
ztmOk#F~?cohJubg>}K22BT`^6kji-E{GK}2v*wP48Zud=0r<>7yP^#k(Rp*fe=pVx
z!WRY4-)0BvqBv1rTC`4P91Q<9h0npFKoTcVt40$?IC!@I=#8W&lq!%nz6w(C26QM*
z>Kl;f0IAEf>y2#po@UQgXC0c`IJ51e0E1-t3P8(}r!l7ynE3<%lC`dlVC)x?F9iW*
zfv6SQF15}Kj5^TPiJgF>%-RRijihJH&p-NS{HeOx@yC3gZD8JT4@7RhBo19m4AUIB
zrSRfp=cXhjsS|Gy$lG^@=Zmz*st4zWKx`Gr+*{T&Jis#S^5`BfKt8*oq&8l2^v?qj
zIxBE-mQ$Sp2T&dim_Kd)+m-l~0$hnUo*)tJ<Vkz|FEUjyRVfi9VhCaQNaCe=;Gh0(
zKZa+W;(TRRz?R1h6%Hi}M6j0w_EyM~DviawpH)GH`D}<r;efG{<?_@ZcCdhP1Q-r2
zsmZK{WYX5kbb_#R1D@!{Skn5vTR_3h-jKh0WU~)TqL+l1@<e@b>B)M<9mV8B#771E
zhQxEWs`*b-yhaA+(SE~IYnRKk-i*_!jQsf#mbO}yIFAGkgG%FCPh{grd&1dxnO}0E
zIBii76jHFg2~;eafk36L(G@IU`TOUruvCq8W&aa~kH27{i<FK?sG}4%aes9N#Fkbw
zF4-`x!qkj6WZ>6`p({_KVi<0OZGsfH=401aNb+7C5jA+OK`kr1=Rh5aA6l=1UGO6z
zb=Bo{b%_C1Jj$GXvgV?9Z8-JGyz`tsc9s`hSd1cC6oCEsU8r?o89~nntIT)p)DX6b
zat6-;3S<KSPa~7ML(pGL3L?P#Cyy+dK-$|dNP8cfa1hDoXD>NM&WT$1%ov*q!7$Eu
z_!9&Kb@|xFO`%#r68zcUDQ}pwkLSGDd2ch!m8n&QEI$IRstb2Ucfd{eGCj!trgUe_
zYwH+`ptJSmSpF(Cp9|$d;hTqO_>S@ossG{)WOO?1S#eXiyk_Odh0){&3$Hxwy)Qtp
zBO56Hkg|d!0FbK`@BDTCLroUa_~7LxE$YM5F1yLEQgWSt8wMbo$}b%d0WFQwwg{OY
z!~rT`n*RsgRtKaT`bb(WQ&ADO>0rq{-|p3+mBn%;4_Fdlom?$5J+;<U?^2O+oQk@g
zaV}Ohu=oZx62|0nSfLXWy`|(TBY$4A%r0L#@U9tA{OOOmOtxVF!JO*snr6)H;2`<<
zjwh(;wGj^|`|@G59kV}U5vw?+!j3)5NInELPQ*#11xH~eGnP{+A4HeJDaaV&c^=kf
zYcPEzV-Z?6d8!dk_xmkB-ix^cWmlF1*6)t8$D6iO+@!R>$sN(zR}>M_w1V?NV50Sr
zPi<tm@*HHMMQ=W7x4UErMnZ1j;YO1`N>Xtk5EIBd6b!ZlJjC|9g6pfyCPOE%v_--z
zYe)Ea=YUFpo>cC5e3Fu(LIV_pDc$MA+JHU*!URnjUqqBjKkUq=j1-21FbAU!?l@21
zW+=D5JpWle#tZ??-oeCwzCF_wm5F9bVim((p|k>j@&nIU3MWgj6da!eoF!H(E`td$
zhS>hvML@Y@lld5?F7pB3_FQ@@L6p8i14e@7f(`8jatNYd-|CUIpCVIrq<QP6CY(x<
z5L*$hLHD4eZ0>7D<>}V|syA>=h@iYBz^j4MCkTlg2Mu=y<UTQSUL;I^4x73zKeT42
z4Kn+?O5>7nwM~2jC#3fIQM@zUr2X%yyg`(tjq@h^_}Z@9!WAh3U=Ufm)OK#~l_f)c
z!vQFHO^{?E@R8fc=+rh&uu)A0vU~-&>C#zPM4@yaaFH-!wekFC=76Iz_9^8JBIf_K
zOOwA;F&xg^UfU}LFQK;rHgSmkg5<@)K^C(x4SUNHCPGu%6L*PjJaHUP0>Y(o^L(85
zK_&M`T>bZSeH`L1wkxZTSA?vEKkA%qM|p?o*pANHnxq~~aCaU~H9`kNNLo~IQ*t-T
zQCkyiBh{_0#*<d?+v@$8Ma8&9bmX#PP9&BX6Fe{P(@l&PYBRCDeTw0>vh*n~j3a@X
zOs=v19&LYoIkD{AL>a12aV#P@(f6Q-m6Z03{G1|S7~pz?&KXt`v@$AE_sBHPk5>$X
zTw?}<9bd35ppH4{WnhRuu_g*U<0{lU763v%C$T%o$I%MEuLV=IS*8fV=pnrqU_L}B
zQ!aaftW(E<Z?yib`h-u<xZI27&1Um4C&H)A)KU=mN&0EYl2kbb1u=vHn*x}{!dnw1
zll@yD_E-$rxbRei={MjakX!5<Bgqnsw`@xhLJ9y14X~DA3FmkdNy6}Tr>Itr6gMBV
zpoqXv-8yfOO0rghG2ZN3p)%Xnm$XL|j?{3vWLz(*Kiijoeq=h;)fVH6rCXDE+Rr|I
zPLhP@teA5K_z1cL*~6jk2@5ok+_M9S66Z*Lk^4V71F8HfgRifa0RgIxU7682>Iou~
zBIN_mOFSZA2~gRj4m;anpAQs{zS>Ma)B3lB66JoI*4bvUKJu`j@7esa2Lw0WE+t8o
zAeeS}N>{<{t1!Ekrc57o?BFj6WN6r$LbU*<awnR&hghiz#C(Ao4Lxjit}%J_&-$qR
zwCx!k_9&3A$taY>&I9^W$JBQ@B2^f>7aeF{B+0bChMkq5w0QKNuvk3~maOLFwI%Au
z=j%8BdM&@xVs?6e5)#87J9_j2HW#RW_y(*A5H|33M&eW*9E6&8;}N=C;4@<Ew4U}-
zG58X!kVm(%vdklfqf*2@W6J2ZV#4I$%}X5CJd2@+{80h@HU;_eCxgzE;G=&LAEk5t
z`bJ=1A*aM|ATpaXy%}cES<4~x0y}sh%+ZoQ>~KogYBdUGD1+jo<oYn=7nur05I8~8
zvM@HLDNa`TgSjh*KRpQC!lYN&LGa~6%j3ylY2t4Kilu)JKDaurNFXVC{mu0|`OK4w
z6i+t#XSNmFIW_tUT$!jEo{aOEw?;ZU?Jw!#@Wo6}Kdr8n_6*9<MzLdtKdJdBPLJtW
z*lQcwTlgaA*j`g`GKtDndUQ&K*Nl%9wQ&abO#wpvmT6KTQ1tYCdp5?D0XBh`Br;@<
zgP3OzLBVmt_xn-eeqayyvX43%!-EBw+C<#MQZ0kv-uAZBE{&T&P1=iCV#fl-mVAvP
zkst14>@iEgzTyLwW<X_8!6Q94`hVRPwdo13FYX5mBo*uZJ3-I{zzWvrCDybNpRqk-
zO>)!$;M_jom`iM1Sb6U}C+LLzxpmyY4Qh-<*tIp|<})C@KQ~ts6&^MpzZkhT<if%^
z|Ij$kSJ!a!m^>~iBVm9K?*a|4B<Feu$2{Im5QXLR@gGayx6;y)T2Je4=9F0UwOyhs
zf~N4~bsP*Z4CmctwtWpt&jPe@s?44l?EUgd1Ul1v<7nmk^B*KYTM#@?GSa!xERodc
z>u$Py6#3Y3187M~Hu4>QN$r`dJ=7F{tC)gOpf{4?>1*a@nykcM!GMEFsGLWu(v~6L
zLdZ|+@-GF5>F%&@zoFgRGqZog6=)BpQ~Ng^ANHjkHg6HMK_0)@E{zEvVw7ZlHKw51
z4ako5_s61|S7)^y1Y#Me!95yFj+tLdsR}~w3vNw<&2<JrzjEKB*=%o#0D=ER=w{fv
zk65qVH@ptD8XQ}@{+a_G-@r_E^rbmH+suC+uYM)~c9nur^f2jWTN`t5&SzP98MoO@
zub@OA+?`@xqqrpC!FC#3TtF!UhDloglQ<XFT&PN_ow9oVsBUgEg70~yxDyqNe$c$d
z=?aI!dSdysyDhZ%;0_+Na4LXWQ)D`U+l&-B;UZhxaArJM?W*5h%M^b@gf1NVUZ`Up
z3$p`~6we)FNkw^z^}us~g3J7S4k8_O01tAuEknJaSmH3(AZR2vsmatEd~jAem;o<#
z&?>KpCzFm(Z3f~t{q+8~RVbKNY#5o9x&vO8r_R5D#ZY@TU78^?#1z&g;IL>w)73in
zTO=zSR9W4dxaqt<2#2*7cF|zd<8va@iof3_Tu-ke7b(o@cZ9_*^Kt+A5ghEhU7Cp{
zW6a9v<UPa|MJ?u#ocCD!QA#q0Y9vp7z+KVhQ$Bp{jL*qFS%fmOHB-u#^RbX(SwHqs
zfYLFON^i0HNbmwwSY6c>4bZR!6Q|)*H?WSUg~_RK!+tB4bA2gcJ-}-tgHt7NUdtzU
znmRo{?zTsrZm7nt2R;XR%HH(`mhKBl^sG<jn3M!U2Ie<@CLJKsx54@|^1M{sE`3Tz
zMv&v>O|h)PN-of?Lf9vIwI#&q%JIqN7!>M$#+OGUDX*n{oY{`NOL7yE`Ry<wrLQzD
z)1kuG&VWSg&DaG+9p>*=2jV};H$7tiI_F7Kp**y*#jBRLM8X!kdby*``naKV%TzPn
z?ae3K{}WPqAzmhYDarn?R}TZfW<hjijE9o=suxK9G$3H<1&J(P?EqU;6Mls5t>=<g
ztmMwaw7B7m-Kg7+<vMQLdtYF;Qz_RSpGvp0@zsu9T(0Z1Zhm<4mRtVLVQ-`~t*!Es
z(jA()B-J1U2OtRw>O74ynXRC#ja5O1<HD}4L9To>ShGsVrCsX}d9Apgu*GhzJFH;6
zqO(9~>gLj6;RjRbuknn-4dot+?!V_iP^tQj^LBdEw;fs+OWl@l`*@oyh@StzFJwIN
zwzv<HuIZ4UEO5G|CYf0VNimu#|8R;X9y{w(4&;W|3I93I_iKFqOJ;yYbZ+}{{lt`7
zLd#)fS`oz}HBTJU+C4al>{677rUaaom~9HLfvo0>1=|Bh&$FU9A;MXUXWCyK_}THP
zIcREL62?TWTGnSLoH*#C08ggTW~r4UIRPszcBZlL_*h~~ipbzVe)*_FB`z%Tnv7^F
z!($(ai0ajvZDv`9ZbjgWAcwq8m!l+PaARgKhI#UH4%Y@p6*w7l`>JFk`k1YZwr=Mu
z$)|6rl-M*SDU-?7TjNL51GY*4G5A#4k3AA3KEyz56X!xbo&nV?&8x&(BcusX)_5$1
z;BjMhlSq8i6SI_J;{Q-@Ft4sIk7PF@#bnhDT@CaH`o!B76Di;AC7(KhmDb(97p~sE
z$gR7vH}M8)H7431>e8;t;=2@5Urh3+jmrj4V8aaA`7DPKkZ{8Dn5qXV1$@u(=uvD7
z@j%p(^n*}l{|>X4qNKSiyx-ut%OJ5Lwp|nB#jTed`Tl(ZD)bfD{_*Ln4QPstF&(_J
zZ0>pW1Dn(`r)m7wUoMyLVJ{+KeS2*3v^&~h9CLsEdGJ3I_Qol(UWfzoEJN>5UbP!3
zm%8{*7JvtKD+_S@ZwLMP=77XEihId6zV8O3IR(&Ihn3P}75bgZi#T5_Ok7J0VtHRo
z58`RO;Fm#2itQLSp-}JomaX1IX)d(k-V+dAm#3-uvLnLgC;SSX(Ox#ylAzVJEJ1-b
za7{lTo})ZP;cP~jN!((~EvUb@N-ya636WyOO<X5MTGq5<%vTuEGj^K13Uk(pOurA~
zXYBA0SenurDsZCF`|yJqlF8GxvYfuj*J3nKuVtcJk=aTNGU2T&1;z1tl*)d*P76V2
z!;$j+4QCU)U6ut~8__1$&F_40RI8e&g|WbUi|$iU_OPXW<G1mKr~Py#Zu>Ut{bsOf
zDD5qF7*f@$X%ej>2f9VL5Ot?lU(MZ1@7vLQU3heQpD@P0`z5ARoOOcIiW)C$%~5k1
z!JY7Od5b?<DN*)N&3OC#_b$UjtH6``ZnDBe-wz#mQdR+IRuy3j%m)geM!HDSt*wGG
zHkjMpB}Zb42dg^IcaU038yTbblJHqME8zq5m{4_*9;0Vt4@gufVH_(bpFFk5eI*~2
zC4rl`4GRsuY&G*N{!Ho{Nk7}j7G}Ec^L!G}cqcL#$nCTf4)`SOd4GQrsMA{7Vi`aI
znA+qr$R<<h7so<s-NMa3h4+*!r4X5S%U;yN`?Q9H9ddmhHUrVVMjlqaA5X5g0TmX_
znn(gxg(?7H<QDI-)cwn9|F=#+1pC<a3xQZax!2zw{O?r2CFVVL@v9DoU9Jd@xyV7u
z)zyB9=gSy1Tdz0y3c`e~FKUvzX(6{KDme3?W^$Lu@54>>-U;Q73NvvPNdm>a<neGq
z`>R|=d^c>;u!iB;{)lezv1<qq+;f})kBuyFq`G}4Gk^XxO;D@z(FALZGLwy-#SYaF
zd;HMAZ^O*Qr*ts<)Fu@wZ^R7&k?q0ER<HNk0%x&9l*a05R4o04X^HVTA(67DtZDV@
zQ3<D~tj96l#qDX)uGDejfv0-&DEOO=2Lhu6kE?x8e6UFLrkl3%kH6yMoydKff0Y+u
z`Gwg{-PF!LiYi>eqY20?Gd_2Wfm6Np4i;eW72Uw9@T;fp<b4AdRz$)Nkahm2m$xh@
zB|5R+>&Ty|b`F>zvXNhX0Q$jsXeQG2r3!R@I&>+^gAefE<lrG*<l}Wx07MMSTKH>z
zA3s4yX3lmE$}6iDeEmy-N9S+QjtH{gq}pZfeW4JS>z8-qh01RP$W{e|xd*`#?Yn#f
z&utUp8F;@cyr$vIR`kp^$`T$!M2l5e70{<-bvttOy>w{+^j;iSQ23#l|ET)eE^4b6
zk4Zi)dj8l%0K4j8bRp*5+dITY7|t&z%~_w=USfVc@3U{kd>M<+2}YT#(+S}MG17C2
z^R-^nC)GjozaQ|DN~6ek@a6r2p($bfEJXelCP6C`Wxnh?<J<~GsI4v47XuaUKntz`
zQ2Ay3CWT{N(6acbE5aei^8v`uaj@pWfC&r}$qb`qadYE;kac5`Xhh3>!wo#5EpGJW
zp8ahLmc>0d<B3hx7HMWPco%j67V*Hl_uAq-kt}W5bE~z8j!4@Nh`{%Gr5^~y7;i*t
z<w`f?+5Dewo@oLs>ASN8Vm@BI0e_RZY0)nOxebdnsp`IH6xf&qC1~XXadq!zesEr%
zCcLxn84p?ywq=T5jW}|+`+<~TiLPcq_q_z4ES4`uk~8`F_9`V0CKK$?gm`;f*)fhW
zOz?$w^h%6qAu3H=!RKTypl_wf%Fp<L@nNCv8r__2Zz*a;O~sf7FLyoI@!Oh-JSrW)
zP2=jo6fvrN`6QFLokocf7pRZP`^$7w2O^1>E-jAH0S0}x_hRBp2|$ap;{vMOkR8EW
zby%Fr(lViJRQPm|CR?4Fie)`h&TWP5)WzbM!`Vf4a+r(-`$AS6!Fo7@uup7_R4Zut
zOdv*(-+BoG<tN?0gU85pe)2Pu@vU|?&FSxJ^g3ey=1Iml_+_V`iZcq70<?1TrN@;|
zw?QPq3EIK9bMnRKmuGdy?|B=rJ&z_3PWQ%Wi_dW~NUptMQ;xKA<8yC6ct9s=ueC)O
z+yt2O;SVQ~|9Oz)asYNp8sr!T{Ph`(KnfTQv>n*T1zOfpoHAFNx#=YNWQC{7bjhfA
zbciK>1Uzd7ZH+lvsdz&R2@sjT<w_$5=xKc)n{_1GinS{A6tmtqYRr1~pb+FA_?-7M
zMbAvX?Td_)Ke3HV@I3_)2=d|bDyxN0cW=f8ts9fN#*}#$4(iB8nrg$T$%4;0N7PAo
zqP<_4oQc_=Z61Bn5fQUT^3h<LdSa};BXB;f-z~UjrZMMS8=jmH#O)OFR$Bqx2OLXL
zF`OVT6rl3Rh7l~g+i%C+1GmfKt%ZX@1VZ2aEgo@U?0!8Rjq`)vNb$@8rix+J1Ni-+
z>I2HuAf}mcVRmy3idQvH@UpHBGS{1{YRIWJC*6Scxb(*hSn4e?N4*Ntza+&#&^gKS
z>H~dS2pUK-1Dd_5C+==#uMwd3hw-7=lk^&JI7Of@1y|54H`grL=mdnSWN13vcO{0W
z2hBB=!VJN|4y)++Pdnl`4mv}UwmC)~=dT|JAEe;kU2O-}amedi;;yEqU?J1YW0T>s
z0@lgp2~`#!cJ1MJErlF<4Ct={#ETEOJZkSlm&xW*>JykOE%<)KAy<HeXTQ^sZ`3Wh
z2`vOs3FhlHvN&@T=4abYXI*h_Q1!GhaTq!(K`(zkwmPT&JqI|Tbp=NCuRdEHrRPJ(
zA6}Vi7s;b;iC(55ii#hnJCe^}50k0g;Nted{hz1=Zta*47}I?qDxrwJVyAX6Qr>Gx
z)x7(*t`S$E?wS5jqJeF1q-0-{9nF07hElfdh;IITRvl`aBMNHBzD0f_BA#pqbjT|c
zjlVmfZF|?3Kdv92eZ*twE7$Bm$9Ww7`UYg9IWW+7_U4IFwPi<-Yu#wZ75-YHSwcYb
z)%nj}x2Eq8E%ll}XjF($Og9=Wc#y#OX5%-|h((}r8O<+KJnQ)N5@5D+a_^JN|Ig3B
z9Q~Ixs?7eqQH-Ba+<s89?TAHU^@UypssfWeMSB!~fvssD;9odjam>iJ!HKyAzKi*m
zVBq^A1EP9g+*7ccvjB4Wbbq(0@>3gd!$AC50jaBO+o_Jm8IzTPoydc7iMm@g>0M%@
zFXcW~O)(h=SD2sfj{_eGH{Of$14CEuom3qZK|`^+y|5K3jYTHlcE1@pZD@+T*X=5(
z9$yy<9j}l>?^5};70S=jxHUasDV}B#4NWCt3PL;-z<&2o%y;DuL^~rTgguMcM@DMr
zE1pb9<OirrPAx44RtP0qhIg$t(*d4HbDEW3dgAM2MeD-8hte8n&}tOWtc<9$_?)-A
z=}WmC$y6~~c5oa`D&Dn{oG&dB0~!D>*)NX0Kxptkb|QK2zur~!_Rhd=%`#Bme=OC+
z@CIyo=HY+-PoWNk3jMucle+(96&iP=7@~5Lj)n9}A`h0x9ZsR3Z&ZQt>e9V!U=j83
z;In;SG=&YY32WyHcb-Be0ezgKGR4R@D~aV63MOM$2C7$ynXm$i=MY&;Uu=B01FR*i
zg3N>zO`yF=c6w*ae}&<Lo;;wjQ4m>d29=hYL7I-k@-%m(h7|lhbzJL!cK)FuU)L!e
zLPIsiT9s5q%DBuZ=xh!pj@wwj+dQ5wNcgC+*mE309jB<DxkyUsfsqt5EcCFh6{Fw8
zXKq+k7`MTgPvPL5CeUCG@qlwjdmu(G@EdmgV~KG2oxbtW2T`}lN(TyBSJZ^PNZ}w;
zR#i3VPD;KPl!Ik7iph}`t?tQ+R$&6X(+)@+9{(5J7V<P29PzEo+JAWeea#(@K+hwz
zoh|bB{^y8*sAMIukE7vPn<;GxZ4L*MT@l-Xv=gpkmaF}!?RN))P2cYBTCQ?t5n5vZ
zTERDph9p2l*&+c8_{M!jN{{{D(-PjXRE2s2(r!pm;DByAQ73RZokd3F?B0WhZH`D4
zpY!y5`5h02ecDY$k^RCd`7>yl=TMaRZ)bj7t|HTN7uouLcui*(%jJ`}l#E?RM!Hwr
zt~_r;Txy#2R5298$%)84oA=Z6#DfkC$(V|=m9kssN8;nXit40V-r#$}^;vveW4Nr&
z9X~xdgV%-AHWW*@4abI%2FDwprK`Gu-5Qy0)iORxQ5|uKUly&Di+*S*-1GhaeQ1#o
zb1FwfDBT%whqT?Qm4E!Nm-DYFFC+nCo#(fq?qs&_@lW&&1cOprl^k@|`6Cf<-fLht
zNQEVy?FE}X5U`TOaz-;F2~FR;G$jFe{)udb8NdsAG+2?4Iv?*GM&Sex=yxlZYxeSS
z`;L5ZS&UOce0N`=^XBomPX+P;%Pa*Zj2aZi(0elC8tr)AwS@@?)Oo?jNa>&vU2_YZ
z^A>88#w|KBSad>s3KQI}SUT`f-y$HLOlHT~R+&lP!h4$H+*OX3|4ebk`Th+fM(s!N
zd%Wh*3iVZ8rk5pQw~Gj-bn{QjtbU)NoDJDehx1azmAasO!Y>Y?Q68x?QF)E$!@9)l
zy$PCr#wJ1Or<)g!#hC~cec2&YS>bv@kP=Ym&%PFFUZPJxU^<rf`qu*k(=KW+__uD)
zBh_l{_D1e;&<obkqRZU>Z{srf=Ebn<Z^Q%6wD;2oa0Gw}O$Z-S5TaJRN$n>07p{l`
zP3VLqBqSj|T>$b|2sm5MG%l3`J5euiG~Mb1HX@a`PM=!PhS=png)2|v)em3K(8g{s
zaa4-Gx{$%TiPn#T&?;Wb_87k<{(rAkn<hI@X~#r)_d5@GLdqmp`1OiZr&S-n&^X|X
z9(!^$Rqg2*ATA-@ZyYpdxRPipvon$?e0Fd%GgSWg)UZeuWJPRHRgBIb_F7)m7FYAJ
z`85;~IM!mNHtvilFU%Xx_R&Q(6vr<q3ux!5+m@ZivQ4TZHCTUg59L#S%#%KNM`XPd
zea`FFWodL;G#l_<6b}@uB_K|Lfir&xEZ*C^>0!P*vjNc|ma3AEf1zU%E<sGf4v0w@
z`sbb1pm>K@qy=vg1Xh&(R_CI8aCF8N*K#Y$MgPYw1ztOBObSQP<Eh`*GxI+n*GcHh
z1gr9X--a97{&M*Uk6seA<RvvS2;g-QphKc%$LTZpjpgd#OPANrzDV`PODxi%)l%iQ
zG(`J=T9X1GaDnObwyn8_O|sxq<+im$&QXfjK3~mxlh~Ad0QEzCb;OeKg+CQdag!;a
z?f@fLXDtTP))RcOg+3dtm|2Mf=m!>gELtBS3}^?7CB};(0m`{JWt)Xs6ULKeO-85}
zh3>Z-E)}h=H+3b5t>$hA8$Ge5vAV(FdXHz0T=xuqzvjM?0grfNY?<ViUDTI>f)7FK
znQll8rKj;Ok>#jRjG0)d<IA%UDl75nZ9Ho4({cMAXTu!>g;Ge7P8Xyop<r`_N_%VS
zb&YY*wzsVqsm9#0%j!?=!YSYBL_wt>5N4M58oyCIf1tvY4<s^a%uNpB#6Ro4z6ib^
zBLK|%3!odBKKg$2znKH29C~9Vw`4#djj7m9mpm{bgaa+3#ADfSOAC&S#rFcGWj?n~
zQ(}H8+yiPH^d#VmUO9mXh_-cPQ%TVC(6p4T+yD7ecZf~$eH!)ZVxZg+>UgVhBs3Nq
zRUnJ+?&ydQG~dsCfsUuw+`YrtlH3+8QY!Nl2HKTvh6Y*cKoE&B)dY<<i!$;DLXL-k
z-sryGM&E?Ko4L>V=<(tXmijN37dlh9wKbkq{$}XANivX&zKD&nrKv*R0;bGnJ>7&f
zAnd9zikT2C#Tytyiv{rW^&g)M83rlT{NxJGmOljP^M4F}6Ygu6s1!WkTV%_-oAh>|
z>K2&vpPJ;@M~%HVFwpj=a!df$rDK_wT@7=AZy(kInPw!K=wMQBIQk+Jg|@9eEX?i$
zb1ju*>*KI%2DG_7YR1G$?vg|lDD(7&4{Vbh5+qlm_kn7lq;^qFEE3X7Dh6;0mS_9y
zrkk1Fc#zxo4GL9M0^s7)%S3t_AQwR*6ybrV(xY!4K-@oo%ZBTtv2J29zc%R$QeV0h
z&)<ze-&F;SRLf4LrfQ1ZhO>$LH1x+umum?#^p}hc^r-q#vt-AHchAJ)Y6*~Jg}m1e
z*n!n<A(wDSU(DTAKSm;ZNSgepts0$etFtxK8442wO)ST4cveUn54nV13_vf^bsOv7
zSG=0xZm}y9L3rM#J?hK%(@<sT0Gw>3@|fu+ewk-bI91JbHQ0}X_Hq;dX?MAe6H?U(
z`4Z}89@{xLV@!jcVc|7=T%hrK-HJ>g7VRTz2V27S!V`h|jR-ol%FQyj0jvxQ-BoF_
zM%E(hXs70RtT9@KRP#mpXUY8)CGjMgUmbrIsq6F<TEUkrDJWLaHU8~%fSfFp1(-Sa
z_kw?3`}*|-*V@ZkD;@zoKvXbll|@uW^BqtvTH1iSkN0EkL^z)03w({IoikB)ext3b
zfxYi1YESY3+<6~Em3iRqZ*cs!7EEdIf1v&f6ayH`Y{CL>e*h)Ja{e{Sp|fJ4-mm!>
z&FgOkv{<2+qSwH<(L+~ho&vNfZI!cNYU3~nfHsQL&>Z?AKKuU$l21Zig-h>`h5I%B
z4Koil1JM}?6+}4#-wtbTrQ8(Jl{=XZ_0+#QE>j?p#Q@)iqkV6$`oN%sRvZWxFeD-z
z;#}&zq1Bi=MwCC>{%a4w-2o<b;(JYC?iVL!mX%!o_L-Frt#0s`xKG>Ayx#gt_|M9_
z;sj@(WGApj696Zvw+4vRn2o5YYArW&UhD;OJP%Vq!}Pk=9pJxx4-r_)ORFtt0LuiP
zJVQO*4>1F^wHE&B_PeqNeN<;vm7}JnvZ_EK-&G$N{B=&_Nyo`oB4Fcl;eROF#%7rd
zG|ZwmEuUex@?xEv7tw%f!?S9Kz|w94{f!Vj<{>S#)fXw6ImWcm1UMhAL(T!v3~t@e
z*(G4cQ@P*ffvB<Bc2~L5X$1m%>E+vjYP<0msy+KMG|SgqSELno`i&ifk2U1go}mK6
z_piT!t2BS-s@NNQgljO(w)&dsB&X<DKiz$JkM&BYWR0#2JfSue>wVb`0Hb&St#-+^
zgpTxf$v(g$Y2lSR->a><ih_Td(6)HHUp-$V;uqRY#JE<-13t!(;&$?@$XhZnSF`+C
z6ieT=AL@EYlygN6WK!%Zyy3e`k2P9DXCKfG9H_F`Us20=`yQ`+g0F@**jRHg&YLT9
zz4ddEy#4;&Ec#k=e39pAAn?f`$d^o9LB7^D2mgTlc*}^d3t5x=K@jeccgUg8IVnnW
ze!xZ(nATd7*#(_?U^k2QYcQlYR<B8ixN!Z(rOFHuaq@pQZsX9e1I*>ZdIuEm{DS+H
z61%+gypCdk;kyo%B@;xCu2VA$9<bS@1RzWp?j?Q@y~ITjU)TRT0_2&Z@A>?{B0w$;
zzh^k;e!T~g_x)%5(V-YhCD`$%AB_Z*rhQWVJ5#WA_kxU&uSN0FY@jRfSIe!B%I99t
z{g=z2(ORf=X{5Ku>iT>(=CtrsFkRq~#@|CV2Qa^_Tz@JPambkw2PEwLfGuX909iOs
z+sr>G8*R;{?$M-i`(cs%NJvRjURo~Xj{ElFR2opy9$@X^HR9V|@Sf7X@u&Kp-_HB8
z9$=}M%QDME&StEX>Lf6wAX3^8%C18keFDa;3^l$nX)-j8kgk<j{;X1Iuq^dZY6>JD
zgFET@1biepH%2APCj0k9v1&zyddo0(cFFyI?h*EQ@E_sn*o^gVD!11veh^P6VA{Yf
zu%hJ3%PpXFw3O;@+Qg0ewf1aWh|ZkFKe)vm*uu@@WT^hqP(1ItcueRC=-&uUp_aZ#
zO<SnzBU$%JC;C@$V_%_rhKqZ$TY@K|$&b0f0Va@YW1lWb9-MT8Otz*H=OsG91C5c0
zhBX+hKHCFgH@E(gHl~oO#Fz3noD~Z~Q2NoYOo;E(IGnVFkENi!GB`TOxeh|5x-|!d
z3$q6W%g;EwG--%RkcSbF-{C_R?pRJ#p?86&tf?fI;-?#F1D$YiDn8y2{LKGx9AiWD
z$1l7|u568+;GS*3H@kzYa{L3eAz{W6{5-XeA0Iml7z`w|K?EkeE2Oo@V!@yE-0Fkr
z;@9&77hSATzKY5eCOK@Iyla1FUbODxoay6_Ui2;Ja%<k!IJz6G<jHxw`w_)ehs4Rn
zJ!mAL;CscH%m%Qq*s33yd-mU)%49i}DYqOJZKwVlvFVS)vq4Q#nng(GU<`4r5N&1B
zf5Y&_jP^zp6azcbmzQo+x-3@~ejgv&-P~CNAIHY!J<{#;FoQGR7)>4IAv;q`b4{i~
z&?{xzLZdH0`5j`}JT$PuBUEGo4$dp*OyiBP^mW<Z*L1`dVQ<qCVw|DytQnjVjk)1S
zO8sD47j8)1=oZxwA0|O)%*K;dN+Jkvi<nTnd0<*JxLQH@6e?Do`LNu`HQ>D_b#|lc
z_;8|cpJH0Rfyqv^!q6Di#|Vy)4UJB>9fK}3Lj0dMU^t}o`uO3Fi6gPwdlwh1cvv29
zqvT@7GnY6-5)Z3BD`VU-K+Wu&OkZ!fe?*52Oeu;Dr#iY4!y@57-mZk)lVPfvk+Y)?
zcekw_H{ZGrFI0{apKHNtKNBZwdqn91&NX7$=GDTeaM8XwE-vAW=w9~9?`M%B2X_!c
ztRhWdm#b5!J13J`ODb6n@#Mm$XMQK@0Q=#j{W^-dapBwP0dshto+E8RfCbI^obiJm
zAsO&3+C>3hoesCtbCSlDNJ)hs0Y^A6n%X^{XpNKKqvft%#zv;Bp%xMQ%znT?UFRt4
z3)N4xtwxgG{-&(P$9R!S3#CZC82&~7fuLGlj;_h};~spF`O`PQ4v((-p>k4~)^O@)
zR!)y&>{>81rKuDE8HizuTs{T4^dT0-9=KQC-r8@zzu4<i=%}Jo*pskKZI9wYy7u*c
z@{0I#ZFO~+N&Kv4?eYX9-<184S|g?T%sQvmObcezS6ODmP%*NfEt!;yKLt>=cPed(
z5b%$SwgEN6alfjMlTcqISBdXsr^}{A_Jw~#2Q&bH%njPRDflBz<u#9L#6G<pJmt^U
z7~rwE`&b$&qwZwVs_EXqB_5g%Xsi__+w-%OX=gTNtP59I+$+luda+SF0zw<!P<WmO
ztUc>83W4;l$Q%M}lIxZEVIWW}#HNHkIy{nBEvR5CcwFq;^^kvFu5?;5DQ=fEfc%b>
zM#7liP!j5rGXzrX?o;Ov$us)c)OdewwXTw;Et&VZe_FtU+Bn*{`G82p0Av-`t)0v+
z)RQdvu2j}t*aMqE&R?`$%12$cE>f0P?zO>q2hX3I%!fqgDpFhx0H7qaXpN;b$(p+I
zwVQ}s(78gPT%PX!^$D0yC_P_F?^Qe^yjFnA@O7E9iZy!+U<FV>#$rzc@me~+{%J3^
zAKH7?`Ks_b<}CQlwg00mD~q2x*Sd$-9AkE;*X4^>2^5Qw=vK$KgYv!@`0lPk^W_!c
zdOcxs|1~El#mbY~&~HhU6~SKxwFzT!L*1tE-YHHS7iRFDhI2*j94f2IYs9O2-o!JX
zal3j~rG;<s;f^&fpUulyO(vTqtOo^9<;RWiELX<`+R&BMFov?%X*89H&*5|yr+Om^
zt%st_Aa!MKA~6qjSg0)|bf`Jq==Y8zi{Hn)Pl?C8VOW<*U%@Rqzwz+MD#r^_;1o^v
zLFfoLgD8DK8}a+q>`mo=A}GD^>yO8*ztr~!Gafbe&#$lX(-d2DJl+?cZKA%YQUl9=
z$bLaNxnd}eAB=dMLIJta&0d9#rR^f(JJcN-PcvpF*0LPQK+T>W5a$D+@G<XK5h6D!
zzo^dklAeQYW`(<t$-8e-!qyLQ1hhcKPuT@(7TgW$#z!+oC<g+^74b^B$zFZp8uw@?
zXg+v(U&i+~%%yZxaN&Z3?PCge$ynwHzZ`ZJ2pxF;CK^K^;HQlM*%=$rTEu;!UA?{)
zHO)5PwZJxw><e@bN9i`C()v$)yGKs1aL_@VH+IsUCm)kJ+7DI3JgI3_=q#yZcIyr0
zjpB9<26}qC(E`R~etcX-_O38InYFFUUYF8_?+@k#Uw)Vdf&DyrzR>cYc-Nw}@ogJl
zL@_$t?TRVAW|{qw$#Y&>x7x_lI6kZXpj-Pxd^9XPNtuY(8Ngt%8VX5mk^z3JsR^}_
zF@0ll&%++&2SZxtjB8hl0mFNbN-tb4{q_{DQAZEcXFi76)|2CV!#v$*j7_w$dCpB4
zZB+SaT#rI6?IiId&T#M&v}>#H(KN7O+@!oS`Ag;aS+yXXl8KW^@MV}4TJ<%e^t5&R
zZK&$QQl0B%z#VCmlZQlhk(8$cy%#sfy=T;_SAUG|9kAg59QH@-S@F6l-OLYr#<)R;
zhjIie%E_W<0_%IBzxa|qNPP3?6$`u?*bwayjaAEjC8*0?z3$7_<OXJ8v!TY%>0C}_
zgsO|tug9*HBk=on#nOb*9|Y&qWEmB9DoF>R&3D(cZOB^9^RBiW4k@Bhcv3>w2R8Gg
z3q>m^uia*Aqw34+*+}ABDOV4cG_l*SSTByaA`^VS%+iQ%TyFrS(_4<{m@%C^RF{ck
z5M$}5>jD(hfPaE;NZR3(*Nt?~1)&@GVpO(TumVL>bYbY%v-gHQR7;`3keEEmrx=WQ
zLY~;~IBF@ZAO}k{YO2K8=KQsaWWty-LWlgcrOgDB_l;9cPm|*4D4}sb3s5`7eU{?R
zRQYkJSL$a&iQ^gL)h*7JBT*z$GeT)B*(vuS8A!cuy|7X{B`m8K7Gf0Vf`65Uaz9{V
z3Tzd#VK8h=L+}R62O%{=<1Js-qDw;$hJ`15#c;RNXES1j%~#AOh?0UvDGZ3E$mbSN
zuad>(V;L$!s?|Vg=*@^uv)B;LKy)U`d9$v(#y_Je7dGML4b!2{*N;U43+b3XJj2rV
zd;T()-uF&ujgO$}2{k2i3R)KV-en|>SDP7<NgozGX;!(pOr)B<zu!+7D4h_2<D+?6
z1Dp1X?p1C>caHkN1CJwUEfIahD|j+utO7K&?BnB2U>hoPZ`I!{j0q&~>Gepst>|n0
zS;<hU=NB`u6M#sM_2##a+0x7t1io%QAJ)$wxld!pStDtW<MoQ`y2f?{82w*svFbWT
z-TYZBE$OmM3Y@*es^X7Z)~;e)yOqoeA8UyPcji|7A&|z`W!$!nFnyu@icSQb93t!^
z(5^7_$v>V6>+t`f?X9Dte!IQ@p+UNvA*4%6kQ$VdR!~720Rd@{5|{xcC5BQY2c#q<
zRJyxVQo6fi02yEyp3mI(eeQFf=d5%7`u)DM)_1v<|9O3`Yw!Ku@BP}8b#Ud&tymSj
z=|UM?@|bY$Un+%iKsk~TOHUI^Oux3eC8?w4{DO_ONfuCQ2lFFz`MvZP${)?fhePA(
zot%h3eXyq9#2_){Nx2it4po~g<wQ$f1*c;@)qS<jmEyjq-$t&VX3*%$6-G;ni`+d4
zVZjc`ocqHmfZFn`%C*5VQp*N<)Uedtf4vbZM#}n(Deu}P9Au<BB}j%FuWZJ(x7PpY
zFVsjpQl;}nDX}!rD*f(0WVNSJw?N&;m9yAcJrx?E4($fy5sIy6>3b=Chag`DoW}BR
zoezb_#HL(Hq!Y?6bPr=Y4%wg(<W1Q%yV-6P@@foo4U`|IJaaND056*_{8D?Qnm?4#
zOTIDbD7$v;#|v*E<00xUG^BfRFX6}R0~9x&6?E&NO|7Bt-Tl}OX9};I3g}s>S~j!H
zt0!2i{wSmLmUFb-DD&pn%qJA&eHL8pK=j8Lg%wk0#OCv+Mb3G;D=#(Y3wfPZpuxh}
zE3Ir9=b$lk8{S5;M1$aZMhgkKn+Xa7m|rJ7Z-aPp!HJaDM^aRt?877OuFI^$69ts|
z+OO9DciVR~Y8v%ETFK{ZRmbu)Wq`IcjXl^rmYg-Hj!p;Ii8_O-N{b@}t0|WVYwnb-
z5otMdz3&W>=F&??u?PgGhk*+6!CoZSO&~RC94a->vj<DNpn#7_dr@$u78bz61aubd
zRr?7N3An^AMv7zqtZ71lPK7d3fbp1G`)rIa`}IIEbmsCs-o2Dm!_S_{L}ukU0AaA6
z))D`4FaSj^9=h7d`}dZaFcs%DMhwXYj)3Ni%C^Gr`xk-OM*Dww3IB~@`qwg);k(2t
zeY90CfborKF{ZDHjx)gH#^|nweyKTSnyk_j%bXlVhV2?;><10FB?AeRV@8I_izmKT
zwnV>v^EFLO-Tz<_GL;o?8(6J+ywv_qibhYcm))nPzFL|i``v^0Ba9FBb}89|r;hB2
zqofPie$5-U{2q1yx{0g)d|kVZ{_S{6tYGY_+Nw30*XADdm65b6toopqDejc5+f!dC
zn^y!VgICz7QswV<)q2uLqK~9S_<@&hg%++%@+N582y%D8pcT0ZY%v!Uldu!UL|(oy
ze#y^XPc_q(`&A+jI6+}r-Y)>2P=}K$CMMKVE5FE&Ch9frQ+e$bLBhWc)LVZTn7VSU
zJRNxk45AS)9=czTYihN+*_9R6^t{h*Vu^eq04aFU{c58WJw*#S)Py|Hf{5$KB`1AC
zIhwO*4t+7Z&9N*EEDe1BaJ&kr9pc@yDgV73FN*kwTFx@-Mo^yR;sc+yfAkjq+hVbQ
zVlm|jaiXNL2*~7l(i1h7YHXZ2D3|~EY2FsgAWO&T3Yw~qc6hS)WOkGy`Oj@1Ts@o9
zHUzuWv?vXdOd)IoUd^uP&leprA2VJ3tza`QT*^mZ6rVZX@kK538o<xf?fr8<+h&rx
z@5N~=0DjBz#<h<Z(#%O?nlI|sX(H4X^Cyq;2W<hmMweg$doc2;pe0T)GB3W*;Qx2<
zP*IkhqchofRl!%rs23v|$eJ~}v~XqT3N-h3Dm->y)R%I}66{voT)2*_ON!>nsChm=
zWiaYmz>?e(uqe0S>8X8=2V4$5K_@|5|N7YHIYK{T($?GYHJdrvNIUN~1&tLexT`Ph
zTiv1ffA7J+uEP@X4`Ez3;jRb4fO&7sv*Pw9bDR?HlYD0KSYH5)BhtSPSkZ|AGfzu|
zp-(-Qko6WflmaNrunqO6?B5@SV7m*J&2Q*;UdgieZelC+{f`yZYIBpp=ku~ImF<L1
zjJM-=0gY3WwBb1oc`v=Uu8(m*1aWcLYWiv+mD2&$^i*!fX8s;i*T@SCbzr>5KXa_h
z-!qIh6VIQ#>_nA@ni7;H2TCDUs2;A!vL241y`N=b{>%U}hqGPGGm)*l0<_iz?ZKtb
zA-=K35=&~2jLm#pqV&jKN5yF(deoskLQsM?_<<MB<tbenmESX|*9#3UWS+hL-#9u#
z|K{lU&>}AgSfK9<rjWT`8$bnWPb8IJe6f;voqVLJ*}Bqyw<L)JK;KSiUR33ZnSB_j
zfU>@0?B6@y0YhBG&e<LRduF~h0bG&nWw2d~QrB-O1oYAg1yqEMGS8F){v%WPuhzDI
z-4&KlY##$K2`fhbOn^P>{LV(=Lwq*{;8lKT&>O^d=*n6#m&<@>oC4sbss4)hTyf0&
z@XS<DDPsnZM>@|Jzl$A?P7oi>EN7PkqdnVaz5{|qVVWEP`Q1O>5e>~nWA#pA+urzn
z$=m51N^imd!Ko;(3xq@S3vj+qybLR62zBe!$1;S+bfXv*hTHIl2_(WPS;gciw~Axr
z-ccl4x@y<qO=*umV>|s;R1O6!Dq#J0=qSM#<jOR0i8M-#(180D$EL9DCo7uzp2Bve
ziXR<9^#h&?QMg85I?gs+3JbZueUJOoW59q|CYeu}k#^-R8TXwRGSR2fTeL^-wOZ+&
z#hYE62EhvJnnUC^nvCnU-VWwJs;EQN3$ir#^gk;DPNA7`!DFL2fMbW;><t(7MI~C1
z39VQDof?3~n;e4ua_RvZO!R+yxpHQ?hiL*LT7aRXFs!1nuP9|dkm44TZ1C^Kz<+zj
zLVt&;0K-jV`qFIc_|JI_GDl3R@yM-WUs2llQ4kaMVR?`77A1GZdp9xx50O(m*0Wa!
zbX`ixw=uk&ZRMIoY#u5mfV@qM19Vy>wx>SRn0GF<OKC|{DxE%(pCSFG)jVjZBTE0;
zC3(XOunB}?aYMhv9s^SDUC#%KbQnHMU`O%Si%zbk!qJTfj-B*W@b(&e)nZ1xAe<d@
zV6CgG;!0F2=cw)zdp5#L(rMvTv<wGwB-Ndk)$LsmeW_L@?IaFsTzHaPK;Y}OgyD~(
z@@LDM7ORO_J>jl@KHL2vG-M+z&H#93OEr{%IN{lMX_K=X__P%-dR~prPHoYVf*%Tn
z@I!XCw#LYB!`)2FUUC{~!oT(uOmWxb7y}lh2N|DE1yf%S7=R_B0N2K9FK=QTa*(2V
z?+BQ+fC7O%90=?d|8HR5S^&)L!w*}ro~&%8zzBdGEUpy0z2JDU_a6<||ISw4Vk16k
zdYK!}psi5lu|~;F6Q}(eYx6Gh&0W5rQRv0uC%*{O_|Ak%tKJJhu2oK22KuDU>k8rK
zXJv8@u>nqVn=GFREq`x1T209hJixWMJyPjR-Z(if6^(WEX;o~3Z&1D4gArIhfbVBu
z7Tc@?KxxvR>Q=L5%IAH>dRuyBu_%bHbO^3Phz=XblW>c+U;R4e2&b}u!b;^Xs@O_m
zOSOwv1^#Fz=)Hl<#fwM|tLaiotDPe{wXobcC3$UyyDQvNQ~m>y9J%nRmHQ9j`l)dR
z=EzPBiRj$p!ARfk2T8BO?~~xaC@v2bZkb7WW>-P7OGsrbKRwoC8C)Vv=sQjjuf1D@
z-lCcr|L2nASXKE;ADaqvi7@|N7TVx0mEpl>uCHl!I;u3wX3?mTp@IwRSt(1>e@(&y
zZPg3%)8&6TSO6aU`)gH)v!Kq%??G5xDNJr;-;zt&=6`yxw#tZAJecG>tlwCk01&N0
z8`G889d<w&o;v|#AYPC0aR94L53pK49RXoFh1?#nh9w;Wl2%rbf!uP~(N-S(BYTmQ
z+)w%4GMfz78m&eo`f727_$M0{M1woD>;??frVUV(9M8z3mfm$@WiK`2P(^Pn>`OYm
z{hdRSW1U}u<E>w<pO^Nxx|;6+lq0;gGyobBi5>b^62I;QW2mNlNa}&U-PrDevYBfy
z*M#&3F6{c^d~EcwqnJS5E7Wqn-dX}h5pjCma;+nb(tdr~Nw4yMkvu#|Q+S>wBlhg`
zs-{0pad)**fY;%FT-;D5iOny7B#i~EJS9-H6RR%oQkkm1ss<-_?Xl-wtO@=)v5#*i
zBlaSPDV<6-<o-HQnU+m&L9{6>R1DVrej|-i{)K09u^H^egOGNFPKaR9ig4=-^)bGj
zK}1GzhN{R)o)HwVMNpU_dB(S~AdjRNleNtU0>`;OF21&|tWu5?wgV}ez<527qWyNy
zq5Rig=bTxtM_Y({rFk4T8aj2@R^~hf7*YBESQh@BAIRZmF^6KIv*d?BkP)xDBD>8(
z?O3Rvl|KEWHxmo&sGRqmEqFr3hb4IeJ_gy8%m#oB$QH1cxb5hf{WuG-11|%o43MY|
zRpUiWfu@$QkNb)rput8KWxo;H21cB6UenRsP68&Ci~AY>Olc9n&;rttPN1x10y>Zl
ze4aCnIE1@jYc08l{^3f&4oN>%W%@_%x0<OBK-WuFs%xe2nIBpwXmbv!qyuzA<Iy|A
zRf9D6tYNF&zZFk@HY`T>{9}yhMH6-=1K|Tj;jED4s18RD9$A#dNU4DT4bMojLE~s?
z&f$5IMZt%kL0?RxW1~%w##T>w!Jd3aM*rW@Gepd^SKG3g2JUNtF$3_5ol|P?2)T-b
z_^;lcVK*5TM&8f<r^)Zor=Ct61L9V{n87;(u2<1{_W+UMv!~~V;eVxr|A(C~)GacE
zcVMbtrvjiOGq6|uR}<QQIaDBJ91ftW#AnecuyAmhrp9OKA<zm__ZO~c#k^Vik`W1p
z%JKZ2g6<?31G)~)-5JNc;XF;an)EU;wneGnKEt;Uxw#k$owt6)m0-Xmd%^@1s>Ay7
zTiCfvwQts`9DP#Qkw<Nk_OP1lUw>Y(t@ni7>7WqCl2Gb^T_OWM;*JpiqoqpVX(#ff
zezTG0aBJqZ#yXbDQn|zd6acl+idfo#EVEGKXe`^kZ%fy%FHAJau^C9DlAxtkkIpW}
z1OH15t;)Z!6_kJjt|cz@f529tU7DtTzDNLb!-MQ%n*AwQ{GJq;tN{jQ!gXMVW()b*
zAN61%_cp`_3DiXve^a)65+fUafipgkgqF|~x)T+L(we>TnrZ-MYaP9&9@pkbKAz|7
zPEhha-R2W43Mx~QrBHfHSu<g9&m|h-dQtopT_rr|?(Jn;b>B`$%*C<v=k6!U;B>|d
zV9&x1KOufP^dJ6%THt!N93tQg|MX|&1Y)nJN2w7oXfwa-v;6)49+3af93=Gez6yRi
zHZCSQh<IuM!#05v?y8Oz?2ebG+q6CL>>0jBp-p&P;dvp?ECcMx1=)AIJeS#&WPrf;
zI)4b4RD|mC{1E&RD8OQgPk|erKd%S~pjAI>zX}6S?sHVX0n{*AnkxK9sdu(ryPLeR
zh0oKOID24nTmJ$J1PV*d9{giRMNeoWn~8OiJUHxdbEW?7Q8?ph41g1UkkYI%TMq6?
zp8ytMIx@h_^qrsP7m&<+xCCZC7C@mptvB}3O_(lV>ZNDI;60F0-eI0V$p|LunH*{s
z0hxpElvg`X8(}*UcAYo$eYjE$GUm_if&1^2)vCseJZ1j={$a$i2kb=f(N<Gqd@*}G
z5=qW@EGVPV-OXXqvN$uHxuVmj+`wn0t#)y8;<P8aTEu4@p(ACaXW4egNv*1&w#{)x
zC@qTbL*_5ai(HzA44+<H5OCjn!XL3htz*l>ahJsaK=!-vjvoWeob`|F5_@{a6sO<r
zM@}iP+;151-x}&I)MFFd;~GAN$4I4KQAqz2Wix*K>m>I-W9O}F9<eNK6+l9&4Rk_<
z{ZIKzjz0E2Mv#T_8GN=`2D9U{Oy_H}qOs)qf6M|*R#?s454UjS0AALVXa`*^bp-I|
zeSL`m%7;$CCc?^@Qn{;XoTwO0`1V#lP1prMOor_X8GXG9P(iH_S7E{Pi4OlTIs-=Q
zYPkw5I>!1O;CM?<&%<u6mw$}um+H{OVUrksDP_yC)$RsmMnzEIKR12@Q!7h^U&6jT
z0Lynr{kc8GDMbIyze__r>+FN&9D%3e=YWr-te0H#OQlsCyH&y`kHszwH@f5Befp_m
z$2t9~9H7Ine@2%KJ}c-+h#kDny}f?<Jdhl_@jLwop?Js^Z5wy&a3s4c&O_b#5EH#?
z6mn%Lil@7i(rhL@W<}#EVfNmZ5tRLd8W^Lr!iyxW>Wc)Hh3G){QXqh4)lfghHBcPz
zXtanCfwa|8{A)W#I=xW@9+|4@Y%y-Sycf#%Po~v>PED|dy1fhG?VHllsYp@=3}azU
z|6bku-vlI;&HFi0)Q&HK^zgOr2o}f(YFK(p+5w<(7x6`+6oJ;YIX2-jc|D>&*6k)v
z^->u~On&hE0xV+m96?9zKZNyYH^oN#k{v%y0Oo>p-@I*Zqp~X}G8Sn8HqS2V&`D_p
zx%=<2oLilA<DqP&u`88Bz4$fxvvH$N7$(pDEUDvlUE)6y6qJ9vXh=re40b=bs~8Gz
zs!eCuuG=Dk(sgf*2STk+ho|IO$<M$Tgzw4I_Z2<BW#!4VRb+5ol^#~tm1?<~o2<N{
z0`?T>kHfm52I3Ge9t{}#?(}l}Ctd*TtyX4{SNIm8Z5ryl@tMi@fi6?-IKCsWqyk=n
zPOIeO1u*4AjyTa@k%EOYC~Hk8+X2~<?@*UDgVvoX&iDPlCdeOK)Ce*2e3p={N&ij4
zdS>L+PZ{{4*veIVJ^wdpR8<s^`X8#`iJS=on%-jICeOb*S6i0ECU0Y30#{}`ONHll
z)Bg;`%JJsPQNI*4DwLiM=s9>W#-T8raG3Ls;uaQVj~lfbjWtaECV0lr{0o?~Pa|eX
zh`QZ!6`QbyLf(A$gFKoQwYNYWK0)Dp(>0@J^72RI6$o@`x3@inR|r^hMWi4NiYFyM
z&^*Vz^&WlCS~FL=`m^NYCF??V(OdNK^g>YyFW;)&D0*=++GxM`(0+CO*P(SY5V4!~
z{rvhSU~%)_p}F(?`Oj<sT9M{F^fOy&(r;*rnixeoybWZZNB$KHmlL&aPw#N1Gq$3-
zc^>I@2;P}~hAWQejWew1*@v;{JjCmS??dE>?n1l5Nxv*HC=$PHr_^GNePAARL?9!I
z=^}DYsO)1h(cmbrONT(}{gcm^lzjD+wK3(_`5*YUlWwS_dyLck4LN)`bxd|>pC=~n
zhT5kQL6-#${GBn!I<+$DOA{nIxRP|--&PF#%k%J;A3yLn)LJfnlmhNni<iC#mz8yh
zVj0sh8e3<vLK^TS@HNd*S+>!mxuA;ukY)c57Il){2xq6Tg3L9(n?AO;AHQ8B!$ghN
z9<CDeD-1W4;5mNT-7_Q_&YTb2w87-_D><pPSpvfsF_h4wcMi3Z&1)8T*k9^HJj+5C
z%^2vaEYuclxItsz>~~l6VMJS4x84*2oecB(DZVKKXS}X(&iP+|OW&NWjX-g4F@N1}
z{+cX=I8!|L{~IoT^-Y{ry=7a^ouwVmCvH0-Aj|%_sDFgTspWaYqzP~D)8L`=pdqig
z1Chy-vefHb%HXXP)|O>f24F^OeuWXb%;;MZA9($lvDtC%c{H1K=65v$6<eZoKgw1H
zmAnU4*PXa10)spLnwdCdIBfUhNXtQR4@Ib*x)$2^Etj}csRBN{3DuG8@5$vAK3RuK
zrl4#^eB&}9wkLrHB~d^+Vkk*Uy;cnBe%R&g7&HZSY0ApL6cuROBXdkqrc*4x5gjEe
zLdoc-inn+=DPbKK{ZVi8DXDn3Kfew-rd%jMUQk25lHiqn1||d9K}Y!%iSDwW)_5!i
zv|9+dIn)6Y*SbEYUz4pBoE^5>2{$zGjQ47I0~&7n2At5wzfR~pYw;p0#^5sFfJNBa
zYd3eH_+S6j*42a<G~(uDu!7iS`C<%mJyuGanKV`3O}gxwxl56Ak5{&wG4Fppmz)t5
zg5fCK_<r$8pwEXhd^?Ctv2Ae^hS?NvIoNG+OF0GKtissfEn3+`RIq+^5bpId1RY74
zbO<;m>->8a79<8<wM#9Y2QSw8?>k!oQaY(+C2HK@dOWe$6CxhGtuw?nwZb_dZ~q`j
z@d99@!72>ruMhmJfYy?z0}{#r`bA5~+vJQQQ0ZI%uTUmLAm$-$v>>KB5FV8|DE9u!
z5>XeCYJ!L4TlI&ZoFSg0$KHu|P=sDdt<J}zgI3D@kfvw@%;@K}aEMn}?%I%eGeYh{
z4LTr!AGf`ZK!q(I(^{Uu6@!c}NEg$I=Tqsv2`SxLexLVA!1^e@X)d3X5*ma`_C1OZ
zN@2I>jg8?BED=i-b#;3*kovkmxhzG{o@>m&)S5}QTSmDHe6PctM~t~Yb+@61-@?C3
z-R+l^3DN+M-nVwy?++dr)n=@)SUEQRN97-P%i*){P~d`hS%}st0BKqKZkA?=EDvQS
zpWD`FtjYo5IFRov85M8@rO9|`;tu1>(aM(WgVG&8H~#bX{ZIcdGQh(bZn~O3Y+?7{
zQ%Mhbu8*s-+F81(bQll=NB6<m{2YB_#}3iI4>Oh3T_1Bdwq+{#xy8_(lqp+6uXn9l
z>rS!`n^5tGsQC3>e-}w?EBe#$<NNe#Cm(B@?3>&sw>P}EB`okFCslw~&nW>qc68Oe
zOoRur9ezO{)=oMur^w}x(?!pb+SFIFywcqrF8m3}U$%j`BlbQs#+Y$EiWCv_!Gfbt
z=B>>a#jg+(;-=9KEdfoPr}1)L2|-=FH)|QF{K0Bzv*dw89l_i8&<}IbG>p7N(lpsD
zLkQ`58&tx8aZdCdG@VYNztcssae2SCY5CD3WD1{GotBY(73Y8^OP5d9>yExu!f4Vh
zz6-C%{@jDra!z$JH_KikQqPN%E3-7flmEd2Xrycn^aGJU&2-%XrGF2+$<(?0QKaNr
z)^-j(JQu^hHd*rnr%8U~KONf!0aADH#h?|mFn%_z*pRa2Cre!(o+C180njObl7Dqr
zlo|eCes}UheLyVz(3^ADHV+iLV~B3c^=xafXKmxWEB-GNn9t9P=S7=}oA+V@7q1qG
zihG1DkKcz|+>EROGFCS&0@bBXFmyXP#4(c&<d`fSedspXRxkk^gj~n=#<Jhw5oupn
z4lZlk0wwM63fo&7VhBg;-8g3xT-4hxZyw_yyDqqKdS8dx30SQ66cwg<x4x1p3UZOi
z8>FrAOA6{XO14l?w#1WUwQi7=1pA7Ra|+YyRIg`fm7;DEPfWxWqHBGt5h`fiQOezE
zG-mWbPs%)AHpK98%56P6-saaUd0$dg6=_nX2Usv&jpedJUSV31s6=VkH<IhilLH-A
zoym}Z6>f1A_DQ5>ztt!I(~-`@g-&XH$T-}C<;?TtjeN+)&xHodBS0s6fJd+54b1$B
zV}R*_Lu0k(KwblVJa!dtuZI1kM#6^w^*7<X<b~si83tdf#<ZT*VkZ2@6WQIeulF3p
z*%rs0stJ|Xjg;-58K_9ZPT`o5ZrGLkjcauR$E=J8iS#3obi7AH0BSmvuqPxAK?)98
zB3=TCai59x#GH`waZQEh!(+peL(PfU-Wk{3p$ll&y$y5wg@Aa<K)xAHtg-q50r(-S
zhc4<E#d~9YJ0eGd4N>|2hKer^-LHGOCNARH@qomDqgZpL*4I9Wm|Jt1PT<2TJjheV
zJoC;mbOjxvYIxkPkc8@6`5dNBT@FNkZ|asl-cAFP(S9?goLex_S&w^$cG}74&ck%Y
z+W^&?V_kk-%VbpR-I1{7FEFL{OxL^*u>uQ7x?&HM{utu$w*t@c_ukiCf<u-%Zs}nh
zY2z-}mi!OEU+R<`&<_2)H$EM=54_$tm8fIz%`xrJcLxt7i^t0gBITpd<EHK8p|6`B
zrZPvVH%Kc>bQ<Kup~rhOxIdM>M&CK0R&<gdy23A(eT2XJJf_=h@p7Wu+3IS>RXtM>
z<Fk4ePRiJ$?(pF<19svr)>@-v^kFZ2pJnPk9^$GVp#c4Z#=iZ>NvTbC#}G{K&D4Eu
z1Fr(drFe`>li`7Fg~Mc94GU@vc4;8~8v%iExE?&WF;0SCe1bm@nx71w7uYF+T^2Dy
z&b=YEU#EV<(qx8l9qITrW$k4TR1a0WCmR=hJON9(D3(8Qe7&PC{$^z5A93tE-<!YK
zmo#$xpO0A}@R+q7y`u%CkhE}`gfHQl_fif$x+tI~a_kUSo(gSOu=GAgMe8V9+N^0&
z+9os+9WwR0c6NZDuOz!86XKgvbmUr=m<SKQ>8@#si=`;8x5H@|pq_%>qqP@B=81`t
ztFvju_vi}V7B}shZaw+Py71%~GYy|5Z|P@?7@|8PZQQm%v{^ytlQ^p1OS)NNrx$4{
zq(5X?Hn^PQTwdcowsR&<OjN92Dd=3C5ntzMvQ6*njafq=lLR@UzHERk-9Q@t`zqdl
zKsf<ViH6D2<vPc4q_5@`FR#8XUVfbw-BR@;g$S>S7{tunoWL+c-woGv4E4o@U!Szz
zoZl!vgz|}724NO&q~;DZ?e>kz;ixFM2@sR~W^^=Osb~spBKd*_=4<Aza2-?P+C{}P
zjqGQnT{7Od34KS_gSV*czk1aky9dHH>=Yo+S<Q?Y<*$z-Ax(aWO3)FIqB)k!^>+9X
z46}W^;Shf0oy*R|gE_pB(QAw=B<t;%IWo5r)RepE*gQmMwMq%?$ihx!8DaYs*VVHr
z6Tz&`OBpTO84Tx5n@Xu@vJ1#XGNisK`y>bh9OjA?<V1?`y!kMg5q$0mmaVZkL1ef!
zI30h7TzzL1UwN|t?N1HrUm1ZzPwT>8Gmv7ztl@rfV9YYu65Sz=?l6~#4`-!RR0#1H
zHb8e7#J1Y%UayQhxBZ<|q05D#C~uZETqV&C?bl61S54i<urp)0dhqP*LXH2zNwy!u
z`(|(E$+*cyV5l0>0`FS_<QQbn&{-DgIIS;vCN8=ELi6RH3d<m1Y|cFWYTXL6-c4$S
z>M(lr%`$B4OQ`Vjv_H$)!r-`!N4<=7+!7J&h^SXa(HhvMJ5XWHunoS>gZ)5F!LO%0
z_tyto(Swua?IN_kL&d{DmL`x8aQ~WQuwe$spxqnU1P=%67sjifor^9aMPrDRw(i>|
zr8e)`$H0!oG2Mmz3RaUUlWoi+Au=w`L7npH;=T(qXpwZHincr5U|>I@`0ZHhO3)o#
zfe==x(<^foVG>F?LU-gBx$vXJEOhrI`ew*O6#fY55l4BKt1hx@T^^ZS<O~IZ?x|ft
zyjBIK*nLzzVs{PVuc@7RIv|oAWBh5iSs;8$^PyP|U5YN0AP!d`(l9UoNqUUzE!}SM
zmUhcB$AOa34`8@+z{5@X_gc!T!w*-IT0`<z3tjyC9%tWr=^1CpIEmE4taxR~^Eb-{
zsNH2GQH?gL39(CR^+%c;V;U(Z&!I~|qNREmd>BaNUi7^rlp<$(?>D)1RW~4-oF-me
z0F8$Z)hN%NcTrWNX}Q1`1NqK$*;Ti{tO+~S6bIrx)cv^j=I>&*x+m<I5q7*XgqemY
zv}{dnz^lgd<ut4}<CS-x51BFyGuFY@E6^X-ea_Hrl<2k<Krt`7Uwa}IKYs{4XYFKW
zf}q+d#W+qr^~PE?0$FXtsRz@F^_OpsM8QzvgkFKu*z1e8tJsf|vE>#;PN*1W^5)Vi
zW)fJN6=QXCW(97IfVSar+?n_peDlHT;zPgb8;OfkgPYTChoBw@ksr}bOCU12_YY_4
zPwbg*pFCJCwTtK+x-1)F8T5e%&e`JgI|j~7i(ik5i!K1(dxMX;gxcr#5GVIy3{YVP
zCSs#WxxcPg#4#(Vv+v*ZsE6BoVd!4?V{E=R_3Xo@@_GHx+bWvri)F~o=8fXz34cYT
z(w+@zj^ySJx9hwPByX$ry7tD`C1!c^@|%X$1Fg;q7o2TS;B_(#X@C#ri5UvPJ1w`+
z+&ox5#2g%oVdh(}{9<4?0Cd3RYN0y2_f}PRxccxG(GtF(@O9I1oO{#!M9xfp1x2-J
z*Td@y%%4`4>wUykfpRYFFK`N9PRzZxqi}{ydMe0f@GSybTGvJ0)0FO(y5`Lk|6Lbm
zh<I(MdnJCe-Wsbh^g~o*UsG{~nsi^H2lt{G$bO$U9r#pa;QLgctG9Y0ZpIOpM!lCG
zYsw&7;{EO-4=;FhZZu|^#j6V)5f}wtcQIY%Y>A%e*zWDrLm(AnxY>qW7kS1~PmJbq
zoPBu=UjcChRk>u|(85*>L>4Z7`^E3MB3$gv<DG{jU7ufB(=A3$Fx#Cc1{dUCV{e@p
zTIhNgc3r`^c-6dU_AdTn9zv*WJ0&E!fqP%KdNChwiQw%@Irm9s>-jlmLFnW0^zlyz
zW$(<Q@Z&{v2JAa3`|u*$-1nT$u}$idi!z|kgvLsNCC04b=K2su_6Rv(+~Rd?3;Am+
zI-AyV07_pC=>=-Mv0oQSbeM#bNFBo^a!HBvzxv63488n8C&0T(C|GM-&$HvEKt3l?
zcepjw13_wH`c^!FZ^!@p!dbbWsO3re<eY??d4>99TQ&>ovf}2_Gb{EJW784=sX|3W
z(-j2xM%cyoRczTC#-J=yYZmscY;HPAL^<s?ZcBa-LBEU;m*d?$;ETR0(85<xrMwUM
zYx22cY<>j?zw$ddYP3fYu+gkiuHx$AC_WJIED!I*M=jI#`g6_6w07_0`-xJP`)?Pn
z(D2=)L+i|u{G!kMt>m5R+C@B0X!&pY;>g?vKAv_g-A<9RXK%`bd`kvf+SK=ev)7dH
zMF;uBmq(Z9(}<^14gI~ID1TTN6416sIj^IfS%Mz3EcEH$>jV?1k(fVm*Yy|gF;7v~
zM;f&B!*xADBIZpbt9+su-~oewcx=A2^D>G4eXQw3y48R!YiK%btsjBV{br(bD8&?1
zeyH#tVK!r4k#x?|Bth!IP0u2VRj0G!jx{kMx44jm3#kJg6uqic<i-=Rb9h;PXv(7P
z9&_&iFBUZlTv7WaAVZ30?o6eAw;2iDiew-oEsaW)1QE}6!@7-wx*fZ+wehHMJ-iDN
zEAC=OtuA$X4{S>D8YD<AR$*v3{IR7uY^MlXV$Vt*+LHYhlrSqnjuRptA}#9V?L$P8
zL+kMMccM53JJB$Ns~Tc~enx%Njx8;^v4!AVyxtY50Pa?J87Q^`J#=#c0mogJixR@x
z8H3t8xEF<x+yX-lwquAZZ<w3CMgJJYRJG*lyo+DilVmGxaI%WC;?1AZ9?Ue-#cb&A
zb>Zac2_vS%iV5do=_r$#NXg{<Lu}fH8<s?I=jBeUg-Sh064YjzgF8(4d_Yn&cHstT
zg&Ca12{@{CrDNLI+Hy3c^T-xAQ{Ep_c0C5NhoB_QoWK+K%`wChx1hu1Fa|^DlC){;
zUqZ_H1>#{ggRjz=&}wkwdXm{lI5qZCN*~{DxPc)-)NR~1q}N2BuobG+S{_;n>K=B&
zA4X&I$lh?y^$E(qxj=Z(42&UACCJew^GmF`1+e}u{B8~wl?eSu?#8XI`9B-4r?Q@$
zgziu3zjahJxWO#mT=m*96Xhqukx!pzr^O@FK380%S1|YfxLc0rXc}jZ*=@xDFFA4D
z6B^q|KiIjS0mFcK%Ht($SXg0gJ!XZ+g(j2yHnAoeehvD??EEdYxh^sjV*Ym^1s6ny
zs_SVwn{C;jWoZ5D-1_VV<!W=@&cU6TduE0QeM1@=yTClw^eXu=*L`0n1v9@dsWr|^
zl`Kas&g9d9AMbg_i<j3&wE{_yQj7b&(SCWuqIsFh?Z$zFdvhPXj6vii`wD1R!=m5`
z%PHK1`4{Y~Db+>WbV@$}9cZcSRSw88TW);0a5<ITkqHM^g}F(m?ba;J1E&(SOwIRO
z+==LrMz<3#i^9S5E%WKw5k{(Tbi7M~K$~P1*APEY>+cxV8DImVVi(@Y5HfY7E)ye3
zDv@^fv3D|T^bfj}tf}0&KFn2*y1}O-iIX5&Bpi@1MQ^XGRW0)fVFaGcplCIj(W5a#
zKIS&LEQnL{rQ|g@BLX4k)m2YIGDuF{3pc3eh8=MTM|y-IAYL?LH#5XxMd-CbM0}E1
z0N)MQt775Vr+`G(07zu(dxm5Q*IU_t&MR>pUDEpGx{ijdT{>QMI5{MFZ~kT{<)>%j
zR9lRlx_3)ME?_hX&)mBraNgO@9Y@;N*Q4n^F%zEx%!M?kYVe&cBUK(Fp76UW#`0ku
ztIJUWyqOOV_!wG#erJ8en?rM6|5knO%G?0Qan346X*pUk<}<RmN5#|9eCFh(u-yHt
zff#Ct^+(C>EshIs>rBC;*sBBLFV7+xBx1dUE$Td2Lkc5KX1DQ(@j`msc(%NOLPj&8
z)8BI!F>m|Bx$$^hxowJ)Om(u4LYnL)u{$v_4+xi($S31*FphQppRe+x7Qf(4s(+;H
zXD0~KiE~7qzBlTdJ&OTPTET*+G|o#~c1k}II&9B<Z&<y>u6zn0K?17N4zF4cS^cV`
zsT7EwhGcs*5M+;P%=$_YjHWr~4J=y{G$kuCckRn3*TnWVyPu?&xVyrtZxZ$5uay5R
zGHt9sc480Npt`QzklieDIIsFR+iVEl|A_fqdK!2xak~uHF+Z2+>}(fMtV$Y7rL8v^
zW4=9HJaf5U3X)do@_j|heB~-=Sq$YBoTJu-ivH==a-CuHINK7q4$Bp=)>ZUfMzWFN
zOrvO|H1CqgX0HbT73;|Ow&!lJ6+zBDC0^yUq}L!lKuFUaY*^La=S-MCbe)_(XxJ4L
z&msgwgwgGW&DXxtN5C=%{|g)hJg(4}v!<~gj+4mlf~TgSP_1ae+njPJlzI!;`PEP1
z=j)02V|D#!1R>KmR&Z^ZULIN=s_NKg%A&AZsj2|_RY3{SNG+bXx5j;R2j>qbJ{(x^
zx<rdgBsdTATqaq&|FNzw7UJ2l&f3!6xTeRP{ETfn7nOW>5?Pre7<`8^F*Fk5sTCu>
z{*Yx7<;~7)PWSwnA2!);GX7(rT8eipi-3%To{zqqUZ>s2B;bfg`w{aKdbN1O4-!97
zi_s|B2rnb^lTG=6RjurfSJ{llMdAb9$#T#pJKnEylgnBdl+Y6Ir0NlpPc!1qw$a2I
zBFHPQXaf$p;1{>~y^IQjhK=QfOX;(#e4i#S1F;A)UFO!@>{wlw_WBxG4<C=N_!lu%
zwC)YIxXeAhFK;!)!Ongh-;<Tvw8mRI8;mc|)>T6=92-4=ka7<$jsX6w_EWF(SlqJg
z+iH~ff|oS)qeu&%ayqZYQSkV56)i|`y8W%q@88Vvw@ai6ay+eesu=8=u~I(y>L{+h
z3Wi9aos-z)!&OMpTUv7-&0piIk-A}QjW&j1OuzOMp7fWCFp8O*u&gjo!vMFrUBK&<
zA8iZ_r4MQ-+-r6Ccoc@0j9zN=`!%hy>#*}G;gvD|W)ctW0;%98v^&UV-4e6ClO=yk
z$Ldy8NNl~h)9Lp&?tPz6wo*8lg1&nvNP^<;`Id~{JuBO;Io`F6zjNzee4m(jyz2vB
zD$l0$D^?nVNv`v^)b|<!u4WgNIhJ`al$dsiTy}JpT#uHf#8a=kck`rPNY}pm<Br*I
z7kl0{;xr>*{Uq?U#;is)iQ>%}5CI-jSVviyvn@W=LL^xo#NGWOhNB}9R69#idmi5O
zmt%~vxzN&|ZT&Ptwkgz4YZoLPxz#%O^O=7v4$!v$lhm20<D2)*jA|4PV#k)b0ekMJ
zSG160+QGaub7PQRh}2B9CpW1?Fj0EJ`^KV?Hpbu)M$uS;zVGdyva6ivLkRo2KfTXS
zQ{dPR0Aywwlq9eneW_f1Dfa11YXyJN78v~wR>`IlyRXv1)%c0vQJEE!c-*`A7PZm=
zCg%C(A1rk#F{7z9FK5>$aHN6n#v*2WjvfD7AnR7ctc9P_dchbVW1GucoM--yC)|7J
z;`%EvUM4;0$uno`{q8qu=R(K(y@nrZuO^+B3|mU=<~M*XEF6c3z)$C^sFTpV>Ob1c
zr8cE#)K@M%n?IC>&kD9Zu|kZ45gvicNr5W)f)X0=<Q;%#VIe~0(^nRqIBoYi>u?~R
z(2naRavJ`x&dZ6N<d5u+#Ncrq{1oAvD}RpB{TlwL0v$~GyQfr#jP=X(g^DOD6{_r`
zMEY!0H?Q6r#eG}M+7b`sVs%+FQ8*S{AaV>3ZAAC`TWRC)@PCNkds5QiZ{)OnAt+!I
z!d+bH?<v!aI;y?cOpnPYg-puOfYbVo>w=DR-*|^L>NlbLA42U=yo;!0nECds++>HU
z%uR~a#R~P1Y2YFbAYKBMgQZQ@?=OV7Oj`=WWi6qiArYSEK&8lrBN8IQWla}ytBoZ>
z@HC}J(Oz0t!D|huPoXBgJh_Er8Q7O;g&e<Yi^U62!zV2jEP+}<+?>R2GwaRx1YJlN
z$KJP&VQ^<|C4-8sEP3z4FwU!@7OIi+ElBSs8k<#46(*Gv@Plwbct2hP|G7ADt7I}Q
z9lwgiiStjy0I~~)_fg(JNs#5w))R%+lc|`CaMD^F$F9I=2dXUYM&>T!-5A0U_MYjF
zEk~rS9>>Lq%VOIjx=-p|d>|GDpQ&KJd~+3hBehN6`y|uBxlo@fJtZDnGvlkia$%L*
zzVuFv_e1z&MW?&fVP6~JeL$^!A_d>a#YuT#Tz4zJYFDRGD(r3~zxb>Qvx=gqlPq&@
z2!idzu-G-PM{HGtmsF{SN#C-EwXp;L6N6=sn}SOCjIa42DTgMD-u_FRBU|ItkO2|t
zAA}cALFuYzi{rMx2^dOTC(K_RFe%#t6*Y7-ip?5lO&-MZ#^>uk9%arO{5d8MTv*6k
zoWgBm2AjPve<x~N&U}|1DMTq=j=(u|mIIh@?~pMY7sm78Ay1yNbLRckv2qVuW$kUS
zB+-;p0xEjTw?0_*030-gEr(YpZ5ig~V8Kivf?+S((L#_7?(5egNyneEM}$DXN`bY}
z>%X-&lv=zpKi{G_G9QHtKW{ow&8FVadCpcAgx>s}Wu|NlA`A^ujb9tvj57j+$(SCV
zO&1GjF^=@cqc*roP!EkJ<tKwJ%8gsQQ6qhyO2xG*lzTivo=I*zv^Zupj`IW7-e33R
zm9cFTn&ewCZwfOf&pQW`awcOG;jvr?hjS<M-dv4b>B*l+4MqT6WYIH-rpRIgI-NDd
zwgarWv3%>Lv^aXG)cMosfx(;6<eof=YSNq&ZP(^F0^e3|Fi1AkFitc6gw*<bVFmOu
zCveH{L~3%h(fw3bCbD?j?`%>TSyS8Wiu3w!4v~_hA|LIr$vu$CJWSqY^OkGN<`4qB
zOPRS{Uqa4z4X6#`HG8XPxxJ~uJ<jI~Q(+(v;iETXO=pzWGy2etL)3AA7>M|jANcJL
z{-H(h&MBkCvl5pOhZDh}2baF+%!}xNRHUQPCD&yr7!~@Q3MF<nqTlS!?d|C8I`n)s
zyUKh~7CIa$G0J8`&#bz|=4fp3LuMB*ThLc5E3XFZCW!$lLUta*S$c86^dPUXjBmKX
zM31r+&(k2oqCB6Km!aE(YZ<CcX1jqN$W@k2B*spE=Q+u^OXl5|EUc!LetMAgp*qqp
zuI0XfaZ+S@PdX#d_t(h|)7(dO_9Ir|J%Q~!Rxs0)=(k`F_fmyMHi4p+sR2`lzcc}b
zGjj78!ds~aMFIRvEto$w8Fb(OEEb+49}U?Z?##r`KW#1obj`!(=q)&AtK$<GI3*^(
zpWf`t_CL!Nkm0F~JJU=pt{YogM)MXu1?f<;R!Zj}yA_TM9?l_*JPcXIIAH#Ev8gDy
zf{tT=X`N+Bfp9g<;UM3_zg*zmy)fuYH}Zr8xGJ~zz`Q+~54>gFti@iDVlHrhj?G`F
zHJNy}pn^&KZyIEKbK^m_*-WMnGbgU|E#=qg>F554J|Zb6TBFM&T^{e`duv*!W2S|{
zmqr91m|3%rV;!E1^rANgdNkLv`dC(#*D$8vPZo;ZP4kdJ$fmz+Ux!MHgOGExe}ipb
zp%)$EPZA;(?SfwJcyGjQ$iCiIDh!^m51Kx;urFDdjj+%L{i?CwMgJoh7z4oo=-6jO
zB3v_@=?DcnIZ;FrU>ZXAqr}?F`Vjt=g5uU?uV_yyj(hFG9^rXi$NZUBk9qYuWMIdF
zngglm$S+m8i5ck5p${Bdtjd@*ab3(Pp$?aR-KqI<NB)UUFyY~Tc`^lTG}2(s^7cuv
zApJx8Jl)A-e(>Z61ccrweqfVDOZAyh#VCp!A}OF0#xKjYT~7>w1fU<^foLn%_7pwV
z5>`B0Hzw+pI?fY9K$=!o3^2=ja<Tylsi^TtvSmLo6lf;3fS1*H9a4359}@v_AkuP&
zO*5qFHIa*rMAx9~w?}W5JHkf}?M0(tF^ayvm7E`JotR+RXJrm?P5X-lpkb3QKbXw@
zG9OX&YD#>`{fDeQUZm`Ex;Szpvhp%?R;MxR?FK`-=tI~(P^%j9GWb9NJ>XS<#eH(i
z3A5GfB+nyr?_V;jF`0K_WaF1foV+DMgmtcGlodU3=$d|VD}<~u@lMV-#77;|V}y9j
z2}*w`WIN7UrYCyBeosLpT~M36s6=?sTHjYIKt~pDv=}fgtY>#@N^t1qj}bXQ+*MZR
zOPHxn4d}Y_C{COI@sKG6sTY}KE`valha2hMx2;WogmzCVn^}j)3AVn`;gQM%Wto~R
zhKnHOOL*=4@?JT9aee*NtU|up@u!@Ede0Qo2|I}qqf`l50M5zJdD|W)(j2acyjxVJ
zKmqXNMsLBp{H`+>(Ma*-6Ha5j-l-rA`hwS;OspsTR!(B}h}fHclmx+e8Pe~}xtC#l
zqwh1Ft;Y<oFb8R#se@-uz-d@4hCkr}?V&lwfxanxYE8pIYM<@b^RO{Xm)V$kUX+D(
zmW|(UW<#yb9*0-5FlYtna8+6S9+$$ua}>9TigRXM7L{|Tir3G+n%j(5*mA|q!P&62
zR`-o49%@T@H(2TNnEko+Sy{>H{vu{yhlgaC2k%HMA1>wAbQCUbJ~8{<9jKrlxptm(
zNi`e@{F%T@*f487Gvg=6xvXQqk^ETeWPvwSUorO)q!^o(4ZD1lmACZgrK0nL?B=0>
z-`*d+--Q`ZQdpfn?+rSh(X5HA6>#^2z7Jx*s_Rv9i!D(gtvAl@^tKbDy70%HkX&F^
z75eiA@5|NtwB^O;)OuU3W>TE7-yd||s8s2ZIm#^Rp|kTCcBpNI?mV7x&9DR`Gc(f>
zlDC=NZ6CcFYp9_G`ezn?jbBvb#joNgGeC|ZcFnL~i2?OI^S->GA*T60YqhW5@B3s9
zgD%Gq%F`QjJk}gb`FVMH64A=aGDM5j%5CfmqSQzDs?RRxm`pH=k<3n^PYU~j@_*Nw
zaeyAKYAX<R8(%H+tx^u>w<!KG?w9H{nz$7EK8Owt>ZKzC2T?0%A-w?iYbxx*_xQ|T
z_0Vg9T~f1Kche_6NCAhJGs5REIDp?7E>kw7in$z76OD1Gb#U^v5T1^;>`^)SG+fi*
zO?e_BS!DQIFSNd2>VR*^1dvy5D;oUd-$6~(czTnAALv%IZ#NZ9<(3zewCWwKb0PW<
z3Ty?6`WkB_3<43(MH>_|o;s!3asIVgd9_GC_Egh*v*wcl+vW|io<96)wh70+??I1G
z;+L-b=59af6_fu<mX$&D<61~^<v>2-%|d?gC+DqmmDa|lNKi_e0q9O3+%0r$prPEr
zZPxg279;?CH-4f@Mt=Q`Z`v$@T~=MgiQSCaN=ai0V*Cnbcr9%|T?=_Gdl#X?EX&>K
z<GhH-IInivD4(~oeHqvlyk7fPu2!sW=t@HYb12_?{PRpXZ-40V)kt5>p^&#%Ey<9=
zMFFszBg4yQ0bApK|B7?M0v01XA*IbZCu^mofYd<!@gdMwkh=29#L=rqB(OVlXY%O8
z<huJm1rW>KaK)xC$es!j6t{upLlgg`Ru7wceWjxwFb6UTB7&d9(7n!2D$XIvt!aV^
zd&*o+3+D%<Ssj{>UZQxlXH=Eq>Ka0044LSoqnIK(o6)*w%jVuEI*PK|D~w>?W1eLw
z*r!xMjt4Aj-;sf){m7c^<<ZIS`T>z~TV!2J(egGdUu2n|cKPc?K~o|<qt$2^d4*hS
zBU!6AyNZ<Ka;y>2u!}FAa7Y9-cW=uRP@FngytL2we!zXokRD?jveB)yp`eX?0F#J)
zR3D-)5H|tS>^!6z0XQ$V{(kf0O$bM_rFN=F)COw3e(Xo(gd@!mVXJ2Q;~e*{rsZ0Y
zE5i$a;-?-R;|Dy;tCCC(MKnDQy|tW#zZ*&;D^7yq6g3b0TF?YS#iWIdVsnH!IHR9T
zb25_^=nw2C7%j26P+llRWTxf`<LM?yIu_Z|rp6E`s`!=LYR}Z}feOoZZMp2%qQKNE
zk$E<WreWj02h6rg-yYTM>^_ar!Z}#3jhbhH?Afif*z@xQagjQl?e>JZuOvBmDn}?e
zJ;QNiqM1Vo;>zI@K8RPh)y?`m6F)oevJ?D#vZC+h_<Q{|{>5P+FU`GlN2YsMKezhK
zMd(SoRr-a6u7e+tY(gMA5@!4)IU{1CZz7m-nOfrR`B4ABHDwC7ouXVaxCB8_p^L9z
z#hk>)4_k`^Ow4~BTgkM>%*9aC?Mz1;Nwz#mn)pQhj%-O~UbGx(DF&1AU^V*IRzVJu
zdlD>=Cyv_oiWfPBb;<^HGU?qgn1#wQf4hq_Jbp#lWp~_oeVK~$8^yqbxc5jw!_LO+
ze#&NHh8v6DkhQLYygbX8B)31u^E=r)M5Cs!T|!?}?}Om_>{3t3?_B*L@44J7EL42>
ze5_C`!W=({xuSZR>RLeaz^P+Cb;clU*P+dNz*JBpWUJ^G<Cd5=^^k>i@yDWl<F&vA
zCLQ5@(i~6lk2q7=^?3cCk6qF@v{~Dwh;pcnukHuxF<i_9lq$TDJ@1<%syO|!6Ls+G
z`e!eN7QKqmRgGfGw0GIZ&P;+&MKm1evJnBEJD?}q&3T1GVzM&Vxjm^fo7}Cg#;2{X
znk<QOs#HVIOSD}%R!oOY))B}t>Pp1D09R7~IM00wX>2VpRb9H{TQWb*%x}9y7xEHu
z`D8We@AJc(C$l59JDFl{?o3E>Of=|jzoxkJ?H9`SH*%_1`R;Rgc~$(us|$-jW5O==
z#-HHAHglu1Bt}l_g{&U3$kta6-Fj`ELY|(m>7`G|3`xQ4)ygir9p;KdY|c3w-HwKE
z9Cze$*?jhNFd<&|8hp&=B|g6V%@Gh{)F#|x>*{J`dLH}q?y3fHU;V+1>1>@OQ$?Pp
zk7oUbA57_cnaz`#4NH#8H1k7)+4vOdkow=?nU+|GSNgR((<YmaqsjGd@$uq6hU(~;
z8|!9ptv}y$ZLT<F7`QZL*J-->8GLGAMB%NC-{#0h@uzC=MDnDvxQ176)`7<XAu-;J
zzzxEsdLxUIGn4YlqM&MF<Zb!nS-<P$K+6fg!l!3cpE0FbJMZclcb1_hrKO<Xs<@6b
zo~H@dPcA4&5M|5u2X^eW!Iu5}c8onC2Zal+9ksuGz#$Ufdzhe?CD%rE^LDdYA%BAx
zzpTO(#9j&Sqy63Djo_eX&rT$V9y>EsbNF6OU`Bwa)AqB{O(95HO~@7+fT=m;uI312
z$m2g`*<z`C=hSv``V6otMwZM=qm`x6A{rK%4=J-lmvE?z+4fWz%5Z~<712kyb77PV
zym#NaOoi$$dFd4%^=)X{9Rbq#TEj=<0pUytDk6;Xz7F#-e=DHh(TEwe_ARY&Wk4p|
zHOTG{9v!I_)(PpPuhCxk`*OcMFQEHWMrYdYrKb%0T2^U_643Pbe`<)zia@Yho<yUM
zHdtV8LX$Epy}Of1cY<&hmh?=$!i9C>44}(CzrcP+9D~9^f~umLNJE3|KT<a3;ruu4
zjpLn~lu%%Y^FTRk*6(saM}7G{&=^O$g=XN{Cwi66BRY=`x{#i`BV2@Yx*@a-=Y}9h
z5qVu#heLI44F-*!TP+VS4x-!$;!Qk$JgR=r<QX*o&J+~qn*ilij=6&ttrk}x%eLHl
zL@<onx!0)wL(+UwQ8<L~Oa1|>vP5KU9Qi|2=pS<UOC#%uZV$8eS<&`#PCj>rZd{iH
z2S+f7JnvJMMM8LA1B&J1PqjZneTM;6<)l>lyL4~DeVOj)Hol?}ykdGI`{#D^!CG=q
zQ1c5L5B8CFjK0>Kq`U_P&av=VKrE<fzD-_>>lmc8CaYczF2@~af^Ei=N6!fjavRAQ
zmf;TnIQRK;U3{(HQhig%2IQF;>{&miCtLnjdzaawp4<z23L@`xjMqxsx{V|Kye`$?
zYMt3yj{th{?HhqC$}j~po-6Y{IeJh3Y23#hq9v!tX`y5BB0aTlz`!R|U53O<D!F_;
z7bZUTfFZ+M1c0H{nvu^9yjXq|IF=eK0Swwu<r&ZPj4P_%)EGTxC<=Y+trqgPl_W%C
z=9YZr7rls0t_ueD)+N#wYm%f&&hlTz&83>IL_5Sx302R7czvkB?_R}aAdPJ?Y=A`X
ziZ|8yn1xG2Be-KkFt+IQx!<6p-;7hcEGR46X69Rcm_;@lDSbNbqb_1PjsQI2w4pn<
z)DN?p8J6SDq7p+wBcOAtL$7SLuW7Q`mWqa3K<RzOZ!#n)BE-)hvd>g~{wk}Q-0gm*
z>B~5NAgKY`tj>Ab`;_qKm$+$iG8L27(fIvSs=ROuxmfbc#oRBrj(txIBDjX|*CpVi
z2h7|Ld{z6+AAgegMX6PxU;Hl6%`%z2tgF^`iK{17HEt5<7R28<QtZDTZ`mGD{C)DI
z`IVJfwZg=gvgM)gVK(txt+;Ysfec%N2L#RGdwehEelqP?PJJLXnN+f+3LY@Z_G+vu
zBN*N@L%cHAo{k0kB7w#P|8;DlB>t%pPIoptnO$*AOQ(mOCHEedl8xvbxkyPY+mG~K
zrDV=t?jLMw`)$!9Z?{*gV|;0LrS8A;2*JAliHgHoKp`g_WZS%${(pFS%ebf>uX}p{
z32Bs05$Wz`KqQos25APQL3-#x36T;I=}@|n?h=OXlJ4$iV1VcF{r#W&!)xHh%-Lt}
zwb!~f_&Rnm@ql(Coj<?6)US~;w%0kMcchlQ?3{D0lXzzEJ$R5`RAN2lFQq5N9lHZC
zoWS`f4`q`KY;a&PGmNH=w3T6p6aCI9{wQ>LseJ?^g&7}HRpx)BoEXec{DYJl-`b1I
zt5rLDqgAQnU#U^KAB~lgM?{XhN6c>;r+4xvnN21BO0#(m|11%jC43}!zN>F<I3|4R
zDDM8`qnlEh16YxIW^QQoiY7rL!*AZ(a4=PUUc_DsV)zSf`I@w+1Q(uqj+5Ec5cgKx
z!-oGRS2X6&{B6UQm4`=MbeeQwglxpG#rF6m)kzAwEU=$`tC4$zF*Nc>K|!IOS|;7S
ziA;SCu9Y#LTedGAQ20UtlEfRa7?U+{9vCsY*>&T${T=wmNQC)&=imSMW1!!BJ7uNa
zM>p-0SQD8G4?Qk%sg^xksq#aN);sr!eDuftWRJ)81lLk31plOkSMc6RynD2^0zVIb
zR;#Vxd|ls+Hmd2l7iQt`h02n)Jc`e~J7_Q+d%yS55XvNI>hH@A+kA@lSLJVXdoNBz
zg26Q?5i^{MP82B-WQN*_&~6FBYi9rpbHCQ74jVX@$1uKMk8?YHcmIjt(HzVQ7Qx{A
ze%`Ul^l-fL)XDv%$ZD+DWHzYwn!wp&Q$c4?xFSbkr4j0W0+w*Ln}_><CXl<ph$}Mv
zq2Uz8;~Q_g(W!gSqMgiAtwN{Yk)uXCkL^!S#^|?uC}x>?JYSV;ENW{M(jW8kZ!q+g
zQJ>#dJGaS3&meIE662_6?}O;^E5B1Uua|KgZjYXE9_L*Jk;q+<#sJ>)qY9PnnUaf3
z#w9@R<rQi}4;N!vpP316JrvVWLR-mLP~!l}$!~;ZnDM$~hao#_OM|IiU;y5~?rKfX
zmU4bI!Pj(KP0)?}Xka8e`3;od2LA9?O&i@{1RKj%tuD|i2k(0e#J5o10o}8i2z;c#
zt*GMz2p)_xhhlvQ*g_)O!uA`g*tnf%eHL6KkSY7pn(mG#4BYUT(FRfjy7NrT$o7Ln
zsg$Y8-KKbxvgV+zE~C1#@Ak~iIJ|tD;bTIQXyhP6#emvnF$JcvmQ~jbA;ysI2GdF#
zDoFtt{N?I|$89w&uFVX6{(_}72`slJht27Ez(MQY^#<gGjJKi0u9CFv!!sQefmx|6
z-R*3U0BOIyYt&|o{rJ%Eqrrd$#J*3f!HN@DM0&Za8|!M)f=0&kJrpRq7fOa9MFk92
zFc}w%z^*fFXyktXjt-h|nh`)UuH8e!eG>7M^a)W`W6hEV`p>FvHsCp>A73q`5sk<@
z_#AMmr6bRUb2|HuF?^NuRJ{emKGHotV+k2{(vk46mp>K9YhjC->^B!;C7BkwiQD2~
z+&2W7B!nx%O-XI>o;)p6Fnc<_$&S@P8K^^Zb_Q4)kpGCRW=AJ<i_`Tfz;jamA5a7|
z8#hMcg={%qABbppf=6^{VkT(;uuA6L$8Ktqf2c&%-y{?y30|i!pj0tN6s1`@&U}7R
z6+K|HuLTIP=ym<#pmHO>5L$Cz(FtfXwzTG1;_TbdblZ${Z`x%U>0`KfckR>(aMXHB
zd`Wayd^sqb)$tZn{x`qCv}2FJriG_c6DREB(oGdSryfE>O+dwvSd$YOO#UNKlJJGN
zv@AL;R$1edd?&l&B{w8*ONuy8F)ub6o0e>R6I&*UwCsa}O&mhjLE*ra>y*66k2vo3
z$bW)Ct&l!Rl^j#IBq2ZH%BAf&#D{T>Wu{Gqmv9<-uYw}bXFiZW4BDX(sN93q-%L<;
zFf}l+Kf(Rzi&7eq{tR=#)^Ln$EB~^)06_-(>Ipt0b-*$Gb4WW54D0B7bPiM(#+HUk
z6rRlq%qPm}ltmF>-1=Xvrpca9Os(Dsa~|_bf2~nF6jxaCG<=tK=`4&g@?9+yk=ZMf
z?WbFKfVn-8Xi^YmbdHs&A071<XZsB^inab#fjFGn99Jc2{nrv|d^L#OVUd#Hz3be4
z+@fLkzf3yiNwTE|srsEd&#wqgIKu|tP!)&;m=F#9+UExj^?u2sS*sRi(EbK{)g;c*
zU-|DU_HHwBm0Qgg|H<;33UL~8*vf5F)oe_dM#wF!><}JNRX?z8ak*08$rz2y<5zl=
z#gbJ_X@e)W0pm93f9a`ZT)Vc)%zbIsv-czlBJJ2uiaQ>N{t`gf3FdEbLfT)t9Y1;c
zmtjB<JdJA6Ov<f#GskUVvW}-GCx&45v#8n8j0$2Va6H|LWmog%WK=fxzm78&+fdZ6
z>vcWm+M}27eyTQ@Vfdqo%Xs0Z)U-i;NqNb$r9S15&v_&BK?T-qLePYJ7Cy)8j=yqY
ztx)%r&xdYyhCiqiVnNCpWo!u?*c&%FhOC~ebu4_VYH8Uj_c5$=2zaEu4MH~NKun?|
zHcG<f{pr@Gusm+S>DA6EGcNZoc*0-udG2949v)zW`FPdaIfuhMYk`?eD!pd+8w6>}
zG%!qgX`*zqimGp)@&Wm-0Y5X0>C&^g%E~#3h>5$vHMTjT>-#9GHQV?0W=Asf4@sxh
ziuQlTSJ}|$$2JR5e91tAqobp!|Cj}=a21<QV9sh3<b=C5?POf`_?c$ef0ko{l}|az
z4F*O_FEGP1nOu0eu$D-qv$#8t`&g<D**Q&eH|1uyY)Q>tr%a6xiOtBovdJ?Hmzh=U
zDa9oXdcAk}b{5m$<m0B&LiW~}XJcYV+dS7RCLr!Elp0L3LaH-Qhe3SBe8nO!Dv^gS
zUD?Z}ej)XNrvRh(zkQX-%?G)@1R=Z8(w|OjX@9aA&43P@(3q@t_j)vUVXJ`)$^L}_
zR|rL5LZZ@|&Cf0Z$7BQLji9`;`rWF_kg09xQ;cNo4U0N{TAuwhp+bzqg0(jqi#LK>
z#DrKZpWEC2DPO^8F4#{UKmxcXFzOn38N{!zBR&*8K>h{9pV5^D=YkXE4itNBJH=%S
z94H;Ei&H#3_T}@hp3|*$0mgF0O0~IimV?mE2$IzwX(uE-z2tjEs#@b^4P{V@=Cc8t
z=)Z;Xnc5%%4Bu{omzk0nbF3Pw!(H2{>TyJrX*!_DEltUL&I^|SKgP%^nX@xOkUFTO
z$fNxvxX?<tf+}o59$#)!@t4CE^qY<p&z~&L@1jHf#+Ma8EjQc+ldHv_QU6!Ui~<P`
zwc&VY;}bJ=zmGoUvD}$qzmX12PeFOk*euGHNwb#S()4YYmm*5cQabrJsVOk9gyyT+
z8J-U5$qcc^A%1xg*hU7qdt!(ugCtbd1q*8$`{b6!N+<(z*HwdaI0r96I~yM~&q8nP
z+t&a)WBy>-CA&QTm5|mRZl=UVQwNTl+11y_ZJ$!6kp(pGiCZFCYiM*MD-13%T<QM}
zwUAKQ^1E?fBWqH}fbK!5U1<A07LoC=`EYi98Zpp_4i4^ne=J@696l?A|3Yg?MH8dc
z%HG4ELfqsIYBuiU^=#N;PyPtV%h{?cw7UNS@zYQh-g8dg+qCF--VZf9-nL7*p36ok
zIna32FA_^b{1dZMir;o;NbKoJUhJ1bbaA*Ma7)vyBYg4~777-#jJikEKb_^3QH)n|
zC0bw2i8u)VoTYu{q(JRpS|}N8rJXTfRYq-ouXWo-4HdC;TzITlzcXkH!TRj|KC<X5
zwPFR+n+1q+Y{Kf6J?$3J06&vNR4`Log_nW?e6v`0>b$YVscP*3s9D1@d;VV!BeJII
zgmM5got9?WW7>1M>d{M9ffgx4h!65gx^$%pFbrWON6`yCc3pXjDn!u)p{89i+*!do
zW=i8BmRf12`C8=LOIL<uVCIV>kU;C8tB(m*5St!zjw$bSmR<eJ5J6Wav8pC!cDk0U
z9781vT`NNnv^Nsb-^QY{TPvb4B+#)!iJ#}Jt`Hn8(|8$9hY%u8KdrrtGl&;7<O;(D
z+tjVBsonQI2O<@SPMm~*vAUlL8A#dsQp9oAM7#;Pen&@&9pWqth%F%+RqtW^y=bsa
zYv_#M#zd#z*F86R0aNp69Y1w<`9hc9UGrqpfAbTvl87g8?t#lKZp8x;-$_wl;~w!2
zyee*>k=CsKlR=mH^aZL{LGq1;q^O>VMIuf$h!W>afB#dopF<rE4XQLmA-HLR?TZm9
z`H{yj95Nxy5uS2hc?O(i@J$xDp*%}P$vu>ML!;%V&`e<N?))gEr9oe=n(N?1PtftO
zCtJH>x=V7Mo`8aE=0x-Z#8JjSEw$4CTJuHYHYXWsva8HFYr99%*$on~6!=r#|B14K
z+^E%bwknd=8#7r&+A$*7?aM4A3s4Ph4QBHm<gHw}7;I`TYxA{b3^i>y5^G*5dvX43
z;KrSI-}(`UI>Mr0<fen4i7EY5fy%yur+VnCX5=D_KSI-0ONP!Blw4nB#n?nyhmunY
z97^o{J(sr84kU)jw`QI%J0|WWeAa}2F6^&g(CXpyJWnkS5eVVFnLdKqnUi-6L!Sn{
zk4(IvU6YwryJNdG-<DeoUXVfE)<gGDOa{7ym8CAfA=mWECk=c$Jf9t81^sTdUD>Tj
zoNlsO>ag|Rw?91~yK(S$ZOQ{`Xq~v=r8PgCDm&7fHrSSirdb>1>&gmh?Cb%yXhS>M
zR_Jf+OdHRh0xuR%&}dNv(b)Ev%>a-t?otFxy66naU()ND8PG$^n$kf(u<5u+*8&Y9
z&bAU7V?bfb2d{Fu6is^1S}D_t_QO-dPqvNV5AJPy$F70Rl|C6KeRYBglRf{dK!RLJ
zvfvF?wIq&^wQuBs?;Tn%iaeYnS~vGKpK$C*WnIox>|FCR>ZY7ajH=!ZSdll4l!B6P
zHf|d(>%P@e#_ly2UF2WASeeb^J~<ewh#Ym%1mV4yG4v^sAD_L{tZ{cOJ07fuk5jWJ
zt_64OA^6d)-yyjiCy&GsAZ0Hro9;1{!<^F+@1dFvlP-hX&Qw*l<41e9A-jK5X(Pi2
z@j^PTAr$LZ%J;~*fl&fpu3lG@Sz_rd$+0hpT<@B05p9}U@heuyWSrN<M3nFKuAY8i
zk|6a`DI~wj)k5WM^Y$y(<>ML!dnR8x&+!(GjJ(F7j9sGleZ@=OS3KP&SG=-*HA5G4
ze8M}-Mev`qA)BX3dwHR|YM>C1)YpTqe=SkFx9K!-yWNI+W8ccI$n^R{@#fyEwC>%?
z(>!=4t+A9Xj6dRpEJvF$|DE)!Yuz;Zw0={44wKi91aHc7jx^Fx=}Z=MKf1~d^(n1E
z0!Ng#5A6nmrl2YuJaI6XOtp>R9>$((XGQ~WS4*S=W2+sb#%$&AnOVGOitk{c=0eHH
zK*o{j-uLqJt>UXgD7|8cUd~k?uV;vq9v`oNF!nz!Ll_Bg%<EVoS9_OpIeMM2D9;8i
zwC<roOcV!7tQ@+*zmY+48Q;F@T067R8|o3iDqc)vp(iQj`%5L-+Zqr4N*z`#(00tE
zoQ{mipkgrVi{rzR?X7Kyp))>Z%R%{;5GoO9z8=_Na@M`}A-RX9DZG&Y48-soT6)5^
z-D)0_dMOKKvLNfGY^&*KSbngi5eVJwKS=j`EGt7YDFtBdk(YGvH##cnXkxXi4+lxi
zJ2t;(eDy&dC^vJ5-lKBT?_EuDxzGooLg>tjL)%F$hS8rYGCRK}Ca`h0UVKeRDrWw|
zZTOE`dQB;TS`89YOH1-TczP{@!=-F45dKneVOG4NuyiU=W4i;@E`pu2&OF7N>A_*X
zrbItqW7DYjbq(iKOcy#lX8+R~<J@bXMn;!~Kuqx3D6uvy?s~ie4UfUMr8AkWHtQJn
z?e$~6<PSs?s5<zQ0dZHUdm?1vzj|JOT@z#zDaakyIfym=r33CIVj$IEgTehu<wxV5
zjPPgy;LV@YWX2w!P4jKYpn#0~H7gSSb;_TFmcp3^O!Vb49+a7c+VY0XJt1tW2a&HJ
zX=~;yJ4*zyYy4CPC}-jp+2Op4=GjJ=vt9agjr`XZ#SVBME}*EEAF98ELvi4u7&bKA
z_ev<41>*OhZ{G3_vhG*e9=b_7s&{-XlBF#2nYRCI#v7yP12g<{)ALr}-+5P|a;Y-F
zJ{mRiFIIt)3!}rUjUSE>nx233Gt31Q&_}8ozfAl{H<+<;#Fowi?=Ym3Aa?6=%kI6w
z!MdP9uC;#p?mb(#0uUuAQ*=$c#9*&X+*w<?5kmO5QRd3Xb7;_>z9>R`fENe=N*s2F
z$Q!>x=K>lNJN;e$3RLT+{^A?c%)b2lnlwkT`!^WU!w?{CbOQN;U8U?hZv^;<<h%lm
zg#*g_A&cX)$U=HA$E`T;rg#_rz8RIN#41i;#!X@mN3PSD_P+&L+on~4?I?QH=DGZ;
z`t0wjBg=da(M)*He|r=!tsB4dWX<TkCEtt}9Ogc&Bh9YdqrM|ti>)|NN<p<=*9=sK
zZ$_L2>_^}Iad!M><>))`FBnr%yD=#X`yW=~&U?tVVY(o-=>m1z>|5Z}QdU3y+my@r
zs#b?n$1Sz<Eed_l&Pa)2>kHTDTn~bhE68MBoQvv3-28*KYmeW}&aWHIH0DyDT)BXY
zyH5V0ZjHJ3=S-0c&F8-y@^hCi{Ia4RT>bQtHaA6Bo%{ln$#qjeil^<VG=;h8b3p5z
zRorJqGcNAi=#f~x_@VXiQMTWv%nU@$^g8|IHGbSrHi;75_T(CU7@$G;!t4@3uhPNO
z$0+V&uV@(~p{MHQ_#K_P#!$3r1bRXn2~+De<`G}OP9d_Vb2tNPv%#L4;)`(<tCpHk
zu@VF4u88F_n`aBCGyJJU`9t7@Ihxlc0h~np`K3aj#xJd8B(atu8&R-J7h1yCX%MQw
zFGAva8!S)Sj<J!>X=HCVw?hZ!5flOh;g3Jvf%FOMPsvC<kE3?hqhka5m^0Ra#VpgJ
zpx0BS^e|5qrdWm3jSvk-#PWr>Fvz;}2$Y<^ZQsD#0FX>|mcZOxw}6W>f2#*2B$V*Q
z7b7|jm&dq2?nlrC4DyNZZB3hq0*UlmVrItOOoma#2SY2<0t>qddwoaeKa%1!WpY>h
zi)PFO;k&xnlaQo|nQ87Cj_S;B<jmyLoIP5$z;+Y@x$Y9KZ44C=f6v4BG8}v6a>5-w
zSY$H~9#$p*B!M(hz_STu(z6MrJ>H!Rw0|UCc1*`zTo?S;?@TO!$!+&lM`MFuh_#-j
zaZC9Ftq%N<Qq#b*R{UUF*A)kyq$*4Kpj2XRqOxQ>cUl`v`kLH0uSc(qhXhRkq;*hR
zV_h1aOX{_F)p8YSVq?YI_xTI=1e-prU(b{}`L;0skecDnW+qTxze=r}Q(66hfRq{(
zz$Riz0(dX(gf~$6%CRh=*4h6*3*g_Ss5sg+CEt2aRfCUmN4*D5!_96{cQEbR6SSuq
zwb+^J3Tx^f>LPDJkw}zUasryC{MDbbd|TSm3w3d4p|(^^WGJ2%G0=q}TDQ0?Yru)V
z*Udn>4I~gEFmZclcylRTPC_1cp26?Vh)t{O*M95{5uaL<nL)M*r$4_A(xzM#CB+R@
zY$=MW>vUxWl&d}cn>5=!ASbhQVYE~|A7;BimL5e|5l+Ad?vHv_D$I+jCFMVi0vV@9
z%nF(=ds~dqJe7XlJh)@Z)+9KfdSd@v2akc1=U1OTdZwAh^_0`y)LRCvc%p}2z??)~
z2CShsKT5;qG$cz4qlDM%657&XdG3zYT+z05XfNt){yDfr?I!HF&r81XyA<ToMmsY^
z<aJEe`ag;FuW~dLQiB3^j~0n<_U}CK>`*wv41@#n8;pblj+HI8We<EUp37E1bSk92
zpC);Z_fEvPtR%S%V*`c#oVr{-27<1cp&hb+Y`-=o<S0x>XFU+(Vnbz};KRC*<I8je
zNz|Pp&!21fCfhhLYPVF+Vp;hhHstuzTyR<llq?oC2}zAm_wbWTXZNslV1-Pl;<LA2
zJa4r4wZE-?R@$_OJRom~TJnym&ha2i))tp5jT0qn?X6>Wa1&;>V)M1Vx4`SZE%%&i
z%Br%-iy|x>UeR6~D)f&PEB5v{7*`{h%-D5>J66uv6isS2mmV>@Dx~V%Pth%4nP3t-
zjyII<cs_0pAEZE(8-L(CNtS6=-v)%6AXC+9z6og3XxsZ<KQmm88TQ^DggSRTA71)@
zY!=%#vCOqg460ua&;Y+{gf|&}+iL)4LM?57&+|+%aVaf?&i|woPkt;U+Q%72DS@`~
zHubpYt<>sg(H9AH1PE=%^&UD^h7F~AD5Cc{*)y(^y3REq7aJE4{?JBNZxJMX9o>ni
zuGA9Li&M|k?cT@1ao^rZ#rS1G!$g^c@%u)essFDaS8Y^Aj&~~L0H&iYgNj~aS{*}l
zm7!bk-M!(?ONJFeqt&`qR|cRGrBz@jIQ0=x^kz`$fALHbLaZ^jsxZ?!ciE+~GAAh8
zXq5g~%L*;O#m6Zs1N3UV|3%JI7Mg&j#n*Dn@@2?@kW~n7_vCe~xQU04e)?QT6{__I
z{nHb3{Q@-WLLtYayfYj!E^2WP)Px1%rV%Rz5;Nl}G+S#FfmMon-=}0QzGP@#jN$ag
z*Ohw9d&wNg6>(y~NPQdLdL5piZ>JW4j)D<3A_U|N6{1YILPuYZ-Yz}d8%8zszTRob
ziX^{qjfh8*ld{7CxvCw+gM&6bGEY{MrPk~akKu3HiwrmCz~)`uY0bt1J*Zf~y>D!U
zLD4jg_#11qlqpwMYb63B;O||T<R;t7&SueIYLIddkXFB*jSOC|%D@g16%%X8iY<xX
zAS&!uw86x!orJD4&QCY4Yl(4+k7O%zX1=SZNTNH2VFp<mV*M#Mc(ms)(cf^3-dxqY
zZoz|n6(k<>uf$ESOoMMWxTL=?l(E?`!%-~)ukf6pf3f!a-ElHUhhg0uEvW*qj~o-n
zc>(@YR(@00=!xjJ`&B=oChh1Gapmq9Jl%w%QCKL!D072EnRsYi0MNt+6dC*%`zv~9
z^6$p^4X%DUkpoT;4&;q=m^4pIg90ffO-a?aLLh2$K!*Bn@cMpJxYY}JBko+XS08iY
z{T$K_IAwT#TVV3QH6ZIkZoqj%u8hXLs2sk<Ed=_Hn|&*=Pm(rip)LgqkFG2%3nMq`
zFnpxpP8{tGES*HUa#w^G=MW|EVPF_JV4WCYf0!FSc1sCfk3Y#3T-5UW7nX&dt~3{G
z#ft|2wM_`3bS)mplNb-&9u!3>fzA28TJg8Q%mV}pd{JY-s4xIQ>4@<{6f&glHlHAS
zRIN7!Hla~}is2Y*VQC8y+$sBGdY)E%Au}6}QDeegK$}IsTUq8@aQN?+?@VuG00~Od
zbXJ%~xBFUjG$9a+YO!Vg07eb@s5~X#%zcgG*9e}i;J!9^7m)6QPwrXeS><?d+AS|m
zL_aJZ(lWvnVsZ32N{64T#+7bZe_8gXw^E=YZ+;v<6NNTY_pG_eCz}?o3frltQTSV1
zVlW34_phyJI@*QgFO+<Y`(II;-WsL`We#Oj?1j1f)*gn3Q<FW^CS}o?8y0#U?Ji+R
zGek?EMLrdUv)+(=q~?{X{_#iYdLl|CzV-BDZL)0SFTspm%v2ERaPqs=2<AnKNoPkX
zx<!WtmD_jcO837u)fNpRxfj<^?AWSZ*ReBC=tFKL<YssscfDeK8g^&2_v#(Ml3D?U
z-GmM|=?0z*XZE#R<#^h6vq}BpvqD-IHUCXtSLh?z`%{xmfBoS=+FXQez`GQB07~pn
zwy+-ej6XhF+hiDI1w~ePJ&auKAq5dC!HlK<fr)f$Fb2?YV4Fr?ggBmbkd<G!-toP~
z#+N34#)Zo*?E6?h_&&1}Cb%J4c3vt`0xeZ9*b9qnQ*7s2y;*g(?lc;ACr&qBHpK3P
z)oCnTwxf$3^BDIYxlY&WhQM326lpzYX{v7_Z}<zA3C-+L|3o5J@Pteo{S^lZQAXbz
zj2uvSW@m5)drKcK9q1j4<)_GLcS=QzdU?(=3AB$yx&Z2|jl%-#FS82LIW&_I#B@Wn
zm0%~)%Y5<9=ouHC^VnU$-y`d5%5!ZXStIk@D)G?+kz$qbo&2mR$Rcpk`~?G^Id1j)
zP!67rJ?jujZt<Qk{Yr|0$=Fu7Cl~qXbSAiZPhKOV){|c1Jkc~cT)P-s4gM3m=6vXy
zaT_7yDn6ozJmU*{#39Ga_s8tuUs~znH|SqHGmjZrb&mL`xg6?2hGScCY}M2Ru&PO~
z05j)L6W*{h4X2a5!zb=6AUwpZ`C+qNC+L?%1lO$Pf_xhNFF`EBx^mI-Al@$01NsZJ
znhAn!y;~=D3Cv8CssfX6Eq$28FAp-x71E{3p0cP5#w6;@3T*0_k15`(OTpX8yaJ6!
z-%+lm!4XErht)BKS)@-iWnRqwWQd#)wy-ak#d|rYuBuO1j)f1?ed{x_wLo=KdO+_B
zbk=K&a>Bc~dhujmg4ip)`BC>=;|mXSJ=n$GJUCeDUGt=~`}Fvgzt`+f71~RrzIvUa
zLV2a{=Q3CVqV&SjbPDHi3b>u?F5WkFXMHMNgt`<F1H*`)qG+2}3wis7SyR5M2Tj6n
zTb54D*s-BJvC67z;p%I=x9>QYd#SwnizSXzUVpf$AagBk7)bGB<K=rn@((<QmTe{Q
zY4;F;Q*Lxvh|J#OUcU%J-RJuW_uq2a&lsZm!`%_4vIg6Wx-cSGlWl~-RTnhEb*$d%
z^bJ5w@h@($f@u?Yk9>g(0CJKh9&W^?Yd1wyyvVOsKNNv<2GB6QxUVuwAV?LYF!Fy*
zl>1f@>Gt;qr(IMQA7MzU{gxPPP5Q_`JxBAEgNLF)HM=~zXRPL<G(Aioq2xu%PIt@$
z>aVHVxjsW~fdfu#GA?bhuIvB&Z3sfJCHq|F4Qz^Q9<`H<2r@$kOs|U+>=OfYGk0=B
z)_eE}c$xe0qo3-kp1pAJA-OJ%Izhrd{Zi@I=&oXwZWCKzY~4k5K`Y~3okA3PC&{Ay
zOP|1Wua0DobE$H@`B8m1W?4J7ugC1Kd1usT<s!)AWS6L$Cy}*vgObkrK~MqQStk&`
zRbw`XYQ0N6b3^tbp`Ohfpt=1i&srq5+<%sL1f5QQd9*Qw`~8xPCvWeZ_1>uay^Snv
zUn-M#t9j4pD0z8QP(CS7#Bnp*8c>G1f$#}f|1S^o5Y!g$@2nM;DR*WUsUa!t(TM(Z
z>MnEr!(v9IV@j3t)y|_nv|G@UKPlTc)1VEW8q4+Iv>BhU+k;R2{(iZ+z38wFEX>@I
zU}9MARQm^<@6V+T(;YU)-Th8nK-T+p#P8}2yXk}_1WQBQ0|3a0^fU<1BFz_C1|9o^
zC|%jYV&<;@gtw23z_P!4T4_3Xm=SCzUj21zm5i%|$A1TDTL<NPHWba)D#8~VFy{If
z3{5z>j%}^tA%ZgS`%A3a(G*HJ`(JOA6}RhRDO|@%xo^Zq9>4waGF;*ZQd7swLat`7
zq(H}Xud|lveJ1g%QkHu*Fo~iec9a6+-wXkDTuYrrmKrvBN{N{G6hi&^PP|3PJ<E^q
z)mY}EPKA(TfoEo`4B}0)Bjj9?*e1LBh~w-gXdDxQxux%W>kLvGxZ^yXF@aOr;U?zw
zxH#+GgR)`bd^Ij4KcZ=#xE=lRlS_>}Gq}v{fKb!qeG`-qbp{`a(|U}f3-Eg8l`^t)
z3nL0By?)hRk5NkCKg~SJ$&@p9H&8Hs&)LVSoAcjPrdM8`e{dZo`f{n1nAdlhW@TLp
z`p3Zr`x6>;vxkAZv#95FSlNz2no^~Tr0&Kat|9YObdl$??iO3I4L(oT+*|e&Ja!W~
z!W}&hzH-5CE3RMS{Z)56^8RLzm&QHSmGP4y^;{OG?SQPj8Xwohqal$`rx<PId!^Gu
z^|lt-4Y001{UdE%n=zkv-ne^eP+rYq=o$RX81SLt1Ftmw)BaM~P>d1l0F=NFiH<VX
zyEzY5ys<Jvzg=<~mT}(*FrryA-2LsiP`h7O(;>>rpr3f@IQtrNMEOWMt5w|Ah6d#V
z-8vr+#hkvI64ak*YkIn2*K$tBkOCeC9|-a7{n5~iXYV%YA>d(XloR+ZzvkN3wW%EO
zr6~XBctS(VcS!-izC7Q)>n^eSn=B)<+?y1Cf503c={=H=RoQSGc;2^9F+bJ};6_ep
z-$Wlwr>2mj^b|M#1F7(dsRTX&PzbFOY_j#c({q4@ra$fTY#Gw7o}JK~28QvE;H&R9
z9g3&IC7$pUQfLm|21Tu{^P<RF(iaJ(gC<v&Su$zO=*whEJI$f((F>sXSkT%wl^)20
zd#5jETmHOqwKc^(J7IO1H^2pNCb>esS>+&L_1+%GVG?aP!&+y#q*@xclJ~Vu8=Q~_
z)eoh70BUYn|EI1VKxFzY70SaknU*U7fxOaLd<W-2&^J;|BF=K}cpBdTi`F+m^sbby
zF9k6D-v+?>1p_{r#Irh}zvx!w;auSl0KJwyz{gCXj5Vlt+sL9kx?z=t`qpJ4Pi=QJ
z0EI|0r}0OaW0M+6pd2_1&9w0ZPSq_n91yEDSLeGKRs759O4N1SBVz8{*tpN?R4T-)
zp(pW>!YI(<_CINwb*@DVMPP3;>9kN<!*AfE$EzDyGT2#V%uqD;=%I%cOZ!3SGHtgB
zAuRfj`Pu8me7u^qkQ*qe=;leb&3H@25wbUV;<4M_1!bx=m)(*F`>W3lmAoCH=!i4o
z>xLOT%+}i<FzdNIGzYAwRp)Lk<z>)TfH$!E>bJ}d4kt<`4f=&Wjbv<YGWS@Ie{a26
zv>CudHI(Rl6svUyN7>&Opl{9EV7yPJ9hj`+oFtH)U3kL|`#9P7j@>n3&o^}pMWE5T
zEN=%;o)X6&X=7%~h>H~~Ov*22{xDYidw(8+4g^{%QOL7&JFb1u;`>qt1?V;6xtb0O
zf`9cDO;U&1v+Soi5r!-NnsB-mRwtE=@;bD+L}rQ@Rt^M~ntpIFv*UUlKI4%41+zbM
z?Z@&sn8EXEl4r^?>$e5pO^UARtto~~Na&K(e?W;v$;!CyL#Ii=9SKMyyC)uQy^%%k
zx?I!HYxVgL(c!$cU2^iLZiB<|TrR)}$oWuX2t9er{O}?_HejQh>b?>0z9^*Y*IskB
zn^+?yU{t8|Ir8pRGe(PwIA$cwau?*&Mr$fd%_aR9KUB){xLvk><wPdi)aEr>MV^=@
zV;fn6G6HqR6ZyEi{k!Ml^G^7&H?{UENRMNi@2lEqFir}Zb*wRsZ$0^%M+ZgbkMS6#
z7Nm59R}twUMH$HDceiL8o+<X1f53u2Rv0l>&!DjF+h-c>l;ACYUsKv!oaW6@VPIj?
zdp3!$b>R9lMo_GJ+k(jb+bly->Vxso5dax&JyHCAi6e}|I9rwWXEzF~5!ISQkC!Er
z6j!U?!7d==-2|=A)FK(qtCpdDbJ=(Z(U5LX$tb%-^@Hkx^z#<K8*YkTg+{a=sdM*w
zU>?=$2H_v*v>=S|>Xv(RWJypSs&$oJQ%&oXt}3?lUzWtRUlEaQyzahQ@IJ`O6~!=A
zp7X|KR3p>jH-KFVK<YK&ya*)UpU^gNn=s0G1ST2#r?L2A17-*}oB-U{xX?p6rIt&#
zCtr-(_KCG3zi=LH)Rp2S6-A`Bu}HJ#qZiq0INl973R(}n*Tz%6K9@7OstFPBOKKMx
zW@?HGutj-!LF<iw9V;O3YV&?!r>mNKQBqcClnIE=Rz=>#Gt<=RUMQ&@3?-yc%#q(w
zJjjwjm+P7Q4))UVTD~fKdrd&(dwO3Z*cd=k-5{Q*pxFv6VUK?f9K*8(kcL7OpFYAN
z9vo}xWYiIf4c6L&)lvV^G`D$ChF^6mdugXNfa$hWQyOd7=Ek%~CTgZX;Q-Q>9&ihE
z7?PHHqyB8BZ*KX2V)*9Pl{g~^*ZP0K_e4RFF8o2LWiK4k)GGxWfb~0pfy9B08kLuX
zaD<KPS#!+J6!;91xp|q7JF0n?9mNfe(BD)x9OiT#J*PhboTi3H1{t#C3A;qk`vlO1
zP?paO2OK3mB+h-Y9A*jFr8}wl#RuO+T|IAXmUnUu=~!o8kUGO0=FD9C?x8t14s=@C
zPDYTLEB<b|H=ZlJQR51dr}?{iG`hl^N(&_BfZ<EUEsqW~RAniYCsdX=m&4#*bl6~3
z?Bh_&cO-KY{ij;$!Imo85N165<DEy@c&Kb!@p`b(B;CEzh4NxIPg$z#FkI|m(K4##
zs4m<7Hr5l`NERZIwck?lZP!qP>NH(K(4>&hHh^%EayVFDGaq(viKss!w{=93Hr(nv
zy@)uA+D%Gaa0@Pb$=AeAQ5pG1^s*{ff0|kNfc<tTgZBP7lsoWlclR#G51fCN<#DgG
zgwu43ux2?L+O>-`Fgm3VsZ+wsRF9IqU<R#5^zM0xp}sojkais&<#p-G@YvDM*gBbL
zY>J*A!0XUl8r+@N64E@&ufeN+?7#r4jONx4O;_Y4%^<#-hZeTOP3g0HzxiStz>z8S
zVc+J_k@*3S7!>L1yu^=`5<$W=-vbKQ6O=M6*-|s~&yf7-l@)A-V^pMn1eM{*3S+1Q
z@d$3K+bS~mdhVPx+HG~#uELB;qqJT8nbZaVxhCZj<!t5g3Nj4U)mzP8Q}gOba9LSM
ziJfTyq;-1NIE@10CiqJ9@1RDVKS5`Czt<TLmTCSL*_*{M;(f>7V5Di~ajp+M?Ny;_
zQer&A=}vKx^MG-7!~vFoUcsc*Sly0TpzZ4+O+e<H;RG`yLe-$x*|lhcei%TM+u*P9
z6Zb27>r4Qn3HRI0k?_LPdjD3?UuDA(?=Ip`%O@i_V<=~CH)vBXAP)Ubo9`;DlBBrB
z$@wW%!K=wMgLt$A*<?B=+V**h>rHf@Bhj0B&bk4jEQU1k48s=OO40xW*$2ZLC5lpC
zX<~#6pJfS-46{D%0Z8wO9xgDbTvY*8aU8^jL)RjZC+<D|v(gU!YqwnOp*kz{Ru2!=
zbz|(KpIpJ9?Hp<N7<b<6X8;R;_9L{4)ub(}fQA?TmF9s1s?UR?pJ}C4Gh%mg35-Sh
z{v=<!hv+;&1y2aSmnBc4@n8^sk$Dlme$#;7)^Pol2jdA7U3bhYu8M^RkGy!&<&n<>
z5h|146-<`SRhEcj@;W4g0)67XZCe5SiuH-I3qz`LiXu>Sm3w$PDJt6e-!&0z`SzcW
z)J@1r8!;&X%1|e7p-xs!O@}nkBy-`u?Q;5DAYf6KMUaDJZ%S+S^VJ;Ax3{<T#2q55
zCYRMP?)39Le*CV2xoby9yg4{T=SA)HIb@&xQRe@abD(m<nJhh8I^O+CQtac8kuo-I
z22NVPr8{AKv}ttvSOHBCrMxAq@Vg7>Zqz_B-E>HiWg+(>9cVR2X)Dou&8eGra#P@0
z+-DbT#_pDia}$Y-gJOjkFHI7_M_$7z%BbIy{uCW3O)eMRjnG4aYtCSp63ZTqMO7GW
zvJu>&lx0PKvJRzXS>Zn2s1{rY{vzHQMv{3HE0P(nJ+RH2W<#=t@lgYOqJE1q*rQ7*
zZ<ulAl#qA7$H|<jeQdQ@$<|6dp79Y(kPa4r%63)Rb~`h*Y{vcd=E~0(F$ZV{X0{t8
zUs?|x*Y+aY_13Yz!N#lO2k4rp$oHyd-p}O)IoT>kMTqK(pT<kc`pj|}H@><Fw)GKX
z1r0Ja6e&2dq--Vm>!Vz=o{4!b@RN=_TjtB=q$rcEj$|HaK3A@wf=Oi`Xf>+_R@T%f
zB<v>nYJigcwzhAhl<L)P>lP1emv~6612z+9SBX*D=MV*B6Z2VctNlW=O%NQs7Q5C^
znL<Ikw+N%+Jm3#WhD><kyPCcbbj?s%q_NYVbxFjex?(HYi<>)-_uEoST4K%EDz*~(
z{>7Q<MU>;~gP1>3#f5YTs4<GPzR4K}ukbK`D()J512jTi2%fw($8W*zx&EpkVg5w3
z=!$JQ>{?sPlcFQfVK>wwnICG>!|!36UUl?e^ED}J&`Z>ou6x4nDw#W4UEl1fd~~z<
zanXyPY(yRwSAicGFz!jxbPF);_lw%oTL6QS={vM^*(udf&opNI6JY{y_?`XzSD7wu
za+oPbCoiEFg{lM@Er=W3Bn{100;tJt#2kU2I!h38QZsME0pk0LT51*Y6ZWa17_y_T
z+I}@gK-+lislsZ{Pf|uhJ%4ST(|2MLF>mGjG6YIJnUeOEumRnjBmkQhQNO8!9l(}K
zCGFVJaYPxgx*Cn^i&R&gPv%14$d)3qEI{OV(xT6pq`=5t@cnuiv*j}djAyiqhB!DU
z3eldlNWp-eP5_DsEfMj34eqcR>7d91NPla><zx>pC{5RE<gARV9MJk-2*??eHz$U0
z9H-VSU~AMIdf?5g3w&DX5P)q)X|%dul;@AfmWzV$7WbdIT7&RrNK%)*qc%O1Z18N}
z4u677=ZpZ&6;^4_ztrR1*|1lQM#?}=3{^C3#vyli{B_Zq`|F|_XM*7z4+690#q5Eb
z82}1Wv0F2>XYud|DXqi1q0aCzH}jeF`$<~f*<sxr;L2x5m4*g<z78TEkCj^1l{UZe
zg|PGg?Xm+%gL5a<;x+S8D^nnxO+;bUX*Mm8G3M~#U*9t*QuIREAb<6t!uozJp~Vd$
zXis%CTNl^0%*yc7r3tygZBiwk$!o05poh!RR~ddHs}HyfI;Zb9yuYoN4ef!(A6@Q%
z-|DPKLCWQc-@pZ(r6-u2VFXxu_egd=dZif%mN7Gx!vl-Hj*gHWG=9!SX+?p8UiV@_
z3pNYz`u~#M@xBbpb7~_r6>gMybHoY5IFh;$4RU@<@!d%Bd{4VmI=6)rW<P#{sW|7T
zI{9K@6Z5Op{K!vWxt20SN3KV4ym=#cdWmnfl`^Ggg!eNl2P80`Kps@h{sPmx$TSA9
z9n8G(+%%hIl&!h9wrVkAb&&PG`<)gb`PfUn3CUYPl-4*eqYvZIQ4fqxa6N_AHYX&k
zK0dOP7bRzO)2;%-3QJao$^CY7$7Idhatv7YDv#1(DIR56WV=UHK7_Ztzc5;ITxu{4
z1U*|6hz~p&HZ(hAC<4rafVZSoPcvEUn{}atrrCua!wGO^?{=+7!|FuS&t<NqVi?&=
z^kb}yD(gTQm3o~P*!xlnM$r*=oJ2@(?>6*PyDa$^QipD<T}ZB}IC+nY&@uXa1BJWn
zZ*w)q$mibv#zT;sl6ZN{>R0;7?}pztIZd1d0D%J^r$sZ}Tx|<no?3fPVa-sLMQxql
zMFFlOWk7pX5u8FiBEE%2fm!Ka;O6F|c4LY!<ZqDLJ;gon+w~I&C9f_-P}(^-ER=*0
zE{Ck&luxe%DITBxC4Cj$E}n}M0M)2rd(Qn5EkmF;0Ps08spK3hc!WPFBC^}#B-V6)
z2CcMbl>g8ap`Gs2E@&=ww!5qTY?ow3{*r;j1&z3n^AlU}FgHobPS$yLba9|xl03P-
zBVT}u8WSku++SVqGy;&N_0jftbC{b`l2T+z5hznV8DK=7|Ea=Qm|8k3j&Q~LQE1)R
zC%b#WX6R`SW|ipUg6xCH8j#DdLSkI5(kY`wdksK-!f9VOKk+uJE#D0`GCAQ(4*n!T
zs_r6Gxu^Qb!C=k`#32)d<xUoNwe_QCs*{@d&ZKp$S{Na<xJ*z#d_5Ihp+=zw*;z1C
z`nHg_0<(2A<DWWvT?(A<@zn7v|AbBGtuT{VL3j{fkWd7_CZq;k%4F6+y3+E3YVkNE
zdD8MK;4xKyjo$dV3V!rK1U~2&^~wC-8ONIv`l8L}ttAJXyL{H#S;5c-3+P$bE%ggQ
zmpUM@N=4SQm#`i-olEWBp4rj*R2yv!h@5?Q_(GJ{<Z(r^QT=tee3Xl8+PlsvYABJ9
z%Wj(VBrl4Nfx)10m?G6q>ZH->Mh?eR-B@1nt}`JSfEBD;3*FUT2AnZU(~C=YjtHV>
z(kBol{i>ja)ZxKCj(eA(jj)9O`5{G)U>^;w&vgDpd!*rYc{06#uXB*7fw!dDXvu?|
zSzs|4%T@LuMGYEjLA`hE{{BB|fqN*fLlwGUeu76<038iYPO_5jE!sbbz^q%b9#i_U
z^<0Za4Z{SRdFm+H{)ooh(-G~c@+wy{Qx&DgsZhhy7|-}T@ZC<y_KEl8^vv6~o=)-t
z8kNWm7^Ka|<C`t;>SjqxW28>AunJ9%_OVRGSo})lI|;4XL9P@Qm45IYoc0|&@TgTM
zb;Zhj(=Dx{E-25hSV~EdH&%vSS(LnRTgcmjYHfVoK}IXSXfs|hvCy6y^+_Ywy}?yE
z%LpU)X0cFArK`jTPFW^lSLY#I&Yr}}K0*2Z;4-9JL^DHtHd+*w=Ul5^c+%u_BmJCT
zG{no&3GbE_bHPH@>}n4C;|FxEc>g8Qo6<FnzLqka&ip%(pEamV{AG!;CNjdaF$=GB
z)Z1Qd7I_@;6YyY`LgLQh=Z^@uZuq6nAz=G6Qt+ej{~ArR-(YT>i|k8X!?n`~;<!dP
zeNt!4!>ZDnjtAp?_7OrR1=4>%koiL@w{#pBItFVO_>bZNgQ!Zn(e0(CrOfRm^HH1V
z?!0)7dyJ@yy*(@h^v+!&zFwfH<Dy;xd^1{`V&W}7hklIC4Q>CEC2OD$*o~=OqR^f=
zL@m`k_j?cqPR@{~*Sq_cxPLmDYA({cH@X85Wks9b4UbF%ZAun^Pc`!2-4KX+WE$Pn
z3z0yMs;6Isk8i<$xf213y9N)@+i337>l^}<9mVM53FvfhMsad0--^mjC!UOwZ;*h0
zuL<?Dkf6L`NLz_n0bLR_p4(CtxR3S2$&`Ziq}}k9)W-7+KJ9>2Cz8rE*%5jcmpWF$
zgbK}V7O8j18d2vR3;@WTuWr4HkEK77FiHd8pDz|um%3%0+X1=Sqfn31vCDl?zz5Th
z4OovO@><=X57<F!M6Yw>2JCX+1)P?itY@owWY#|(d@&~}3|QWA>k|abfO$xai&Y~&
zJiY5n7Yk?4=M^JXb4Wjvcf!3zXMhcIhqCU|FEr~}A=A-27gTGFbaHa={TkR`VAY|K
zfvoKbLrZX-tw<QMZj+rfVIQL{QM7i1<bD_9l_{@B6o+|u24BlS&dy4$P1~wx6bee>
z%;x<(lO-?LZ~iEh3G3$(pf-ug5k17!1zm%K^)Mh{uBqGOmQE(Nyw#6eT&*qHc2L0A
za?$7%QJD5vt*i`UsF*GD)!it0XgoU4Z;7&;`RPKO!8@3`>vAa*!g2HI72#i4hD#br
zg(AlHPSbT3Hu;-=ncIlzl%Kjr7lI1!Q9b@OTVOB$DQHqbkb6*QeMinR@D-+#F9o68
z6|XiMUKDIhk{vzn*19)suG%0VUuGG%{4#wdzM+E(gOS`b%m&K{wD;$RD9`LY$3(-#
z(my7p>^d%q*`G!T8Y>S{d;qCL-HodTrlVv_Kn9<x?bbUN08Y2vMwQkh>N|l5rCH4P
zUr#(H$q6`;|9Hiv2?M|2W+lwk=$IHyj#iuW#i);0V56iI**skAed#pDTr{9^2Tb4D
z(`7p}ppPk5sIaG|OrM_ej0v6nQ`z&x0+b_7bO2^^&^H!aV<-Ho3g3D>*T^&N3f*Dg
zPJa@fYr%L|dPJqkSBu9lJf(V$;tJQ^!!NPF5DFD3@QlaLv=Lk^dz`KDf1wQ13{S71
zr8485I@0=+{xrV|C0ZUf$0Un_sYlRl!f<3wsp^2xIgtr)!az1)`8R*=M!Y<=F>!#!
z)2q1BJ}RyRzIB40IDwTex>VUd$tRH&KWI!o;r22haLA@K*5BeZ{f!5=BGpf~2*A2f
z|I-z~bf3^jBy$_vG5tDs*@D@Sb}|DykFS7Q>b_8mWDxIL#PQnmgp?l=ak;0B&K)rV
zP`ie8b)y|4W~0)$2g2%s*3BLGs?16}8SS+59M1(&f_Lw5HQhT23Xs6nj)I7K{|pm%
zwzci(s{M3vLW`ZL58l7GD}SQ>_D}7~M4vvQxZ#xrY0(RhR+{B?!A~oIuwV6|2IdM}
zx^SpJWSJ(dFh;JD9T}$izO9cwS*V0hpQSza5vtJ``xAgFb}r{bl7Ds3JmIPTod0;H
zn&X^i>4YX+xD_#2Z`iQODf*0CoSK87G4F9oD8bkS3D`aN2vI5w^xNuZ-Je&L2s7F6
zzX9$|tj-MkU-p<cRc9@Uy`Q9pebkncECsHP*tChay1%pN77}l+(TbjKUJ0RsN8f!&
z_xBU~@A*AzIoEK<OLYg7k-@dlhIt93uX@koFnm}JC8hidN2y7~7tv0%EHl=|kr|{!
zra!j4@r{6yV0!hv1tZr+1Gx{C$!U8x_rrBPi!@n<9insPgdrn^PoB%)%)xh~$P0s;
z=$?Kj@n?8!fI&w19|L5bi@~dU8ap<+uJznE9vwyI8YgvZvG1xWUZowg6tI2ddSFqf
zJppQvMUMUY%3Cjtqto0|hMnvarTgBL@Sc2ONiQsdXdbP#9<+wHUY)fSMsaJQbg<3B
z|AUIU!ktXH@58LyJPulxk8^^!KaflN6>KXJm^2)m2492`SK%B`*1Z-mRlqh$Vz81m
zu`$6{3*-r!u?0gW<`yq&ASU1D#wcmb4UW}ENG=s>hy4y+x$R2)7Oo!2_X*Xwl4p;F
zeT@=iD;{u<JZa-B^IlriFw2W*rY%|3w@>&Cuv<-NJU_We3~%`kZBdIyIUJH#YU-32
zr<Mk4<^X2^M73)nH9Voo++||vHf*kqXrlLfkMRBOtOW9~5x_9rurXSyx$_L5U5>u-
z9R7wu9aA_sXNH;BcsK@0$fZNJ(@{sWR%ng`IBwn4c-spX{72n;5%<3@*Xu`WWv9%&
z%HN}(E8V3>n3(Sap5lB2!KEdhVNwy2y_ZV`&b~D<tjIVse37UKW-iWE5uqiHzyt#?
z1lnzRy}sF-(0ofu=dZ|hFvbtC#L8cWGFV{8&jzM|m5%qP(cmU$1|N&XOYS85l+x3@
z7f;cS6>~JNSriS4soNACqwM_Od~}4_xnxg@P&*TQb(}P)8ol(y`oPBRyBZr1_3jz-
zU0SF~8@EL-L72i=ADzi5T~j~O)%E1pdXo32uACFbfTne?IYnRu4Od_lmqG&Ava20X
zvJ8$@Dqv!aSZXsyK7Is`KjKY~j|H3YT45`i^`#wYP2j_()BR>zFkmn8q^SM6xRnT&
zpLuK?#1noo?*j`DktU3Nw$sFJiI3dv%}z%-E&b{Lh(Ms9Q8ynHXe26CWhH?}{R42@
zhrc9YoV4M{SKn`cH-(`=sATCsguEcIL5QF!Z;P2vp<WpWGR<1k+(R%7TI4b&_kWZ$
zMu$N2$FD&1_2X?0UVsB!Y5?rGoJ7$CcjY1gkOfonw&UZ?3wl8SJ`cQBZLZr%+nCA^
zvX&_gZ`%o$1C_8sUi=Hrc8=t>iH$NmeJLv<dfj{aMFC3))#+3XpvU2UfjWoLH8I2}
z`FCj2#;i`rP>ULR0hyvNp88Wg%S;Z}JtR@ty0kyMb8vf2nnc{+#2=aW;6L=bg=Y7^
zicx3(@y%Hi<>|HhYL1FGiE2(qoX83>Ck=RUoh+&TCk9<@pYkj{lp>mWr>i>}Q!Y0p
zjp|tdbd8PdT@tr3P1*DGPS-o&>YRSqu%NKlshxD+GeDJ0Gg#Q!w|&$k51aQG|L1E4
zkR*Hp?*n-YoI6&r%kF{LWNyRU|J|heq=z?u;OI#ejf+?ZHl%V<`%3ufT`i!87&=7E
zf0OtV&)Ub@hl58=L&HfO6S5DweLRO1K9P12RT@YtefCHx3%8OwOTrOva(E3iUSU7Q
zhHb`S1nfg;;=(Ow;)S?+`>LJ|<EuA&Wpn@l2Oyc(K;iu9&a>YZ`YYvu&$&`wMzZ1Y
zzc4?R!RMGLN0k0_<o@;#?V{x;3|xlN=(Wo-JUIS}v!=wWt6#YDZBCe_^u7ei!Q1#w
zKXpwa5xre0!10|wcBks6o(Ie&0&fA>UnXx$UuWEDKAL`qMn=!K2txvneu-loKqxji
zi#BN}VMVP~)qD2+7hw!ZK!+<0BUbdYj<|Q&YCwjAg~=tmm%FK6RCXfKI60YX`Z_j%
zGEiV;BjgyhbeZ4;@K-+VF52&iXT3L6V&K?#C5%wmB&_-|n1QrQdZ|FEATkrUsZAp-
zYh;xGJTLo;jgQyd{`bSKfJQ={HB}#F@b_9A7DD>p83Z>VwBm-BT8wcB@Oilo10HU<
zr7j_0kCV&VR=<-W&lXoigw#xG<2mrW)nHI-etYD0b2`>{l|H`DxNMhO0Lr__A+RXA
zKU~%I-_RwUJG|b;Vrl&9cm0>n`1@#q-7<$YNN@Ad^Gfn;RWIcs5jdC>3`qsfIT3IG
z2gv-OOFJAkk6)d!g<Z=7N)hq@*EaiH1Idxhgn##DD9F#uK+zfkO@X2^;pn<B1rb_s
zMa3<4E9Nt=w+0LJ(Dg2|^v-ac81^Q3SB4w(`tAo_`)d{@D8N*>CNlO_)Qf)}(2>K4
z2K31r3{njRh_#s{EU71x9n&3CpYDlu&^40Q%U0FNDg3XW52?u$zw(2zyd}$73kT-)
z_~Gc|xH#V+IsXog%k?kJ?q7Wy+h5V`Xy}{@5}vSBApSqX-U2GFZrK`bBtWnx5G;7m
zKyZT75C|by2tk82?h@P@x8MN+1a}DT4vo9JySux+opbKF-~ZmY|9xW(nx;oX5B6SF
zHP@_JRVQy8Pvs!1*AK@uq!jMIT!L(WjZFYmd(pscc=tcw28RFnHvIc${J$R)A%FE1
zJN5L9A2g91k^i+%{cr!LVT7JfocGI!+L*Yg!jupqZ{MgFZ$(=*?gFF2Qi6S3cz#CR
zYX-+^Jp!X^R%BtPu>`cN_Yu^Xv>xs|5A8klrwU<I#xkr=?ev)8DaXKUiIs$2==J1>
zZFuHSr5$-R71E9m2tJPM8+4jZ$7^jV6~Ob1eFi2Sjh8bD_JQUmwmx6snbOJnSu$@h
z;@n3!_nD`zaXi-1Y(s1GX`n@WuTtz*UJ%y4^U&!L#!{SK@Ly5_!<Lbt4}fOg@z{Ys
z|8E>CciD2O{fjF9t|A&N#ncd&Eq+V82w+rX;K%>NfBa9=?Y`G>l`wf`3yGfFP3%4y
z9ldR)VJZ^R+AAukFUPKT8pfe{-r}Y+Lizl;FAa@z#MLT`FJ+5&J#y?a@|3~C!h&0@
zqx-~JlY(20QMQDGp+oiH)k1DwhSHY9Qryn%RW5JBZ?E58yMrP?q0H1K$5@fQgBJqm
zI--*ijRv=O_=Ojtr}0XDM+(djXiz!s&D1?X$Qr^*&P*d5C*KL?*m*%%fW%JOi9_W7
zg?@pM0C9oq5h<DB90@FpWBv0Dd{v4?Ahh%iXX$lNBwZlIEl$*Gq>Lni@dy62M7X(K
z+pv!+42{bx020_iZ)3j06w`SfS?Kw;^EC7KbT_!rToej2{^5i5p-<J}_&6NPS0<ID
zBg-$78X6LzBSz9$odzX=SI{|)WA2k8aAoRlWuv9e)E+?--TFoTwVDqj5D@fQCt9V?
z-i3kU*%V4CIC1`$^~ILsC(3Z97yGOHiT*LY6FriDy;%eW5riedJ5U;Y_T?Fe`~}hF
zT?=xiHTE&pPDUH8ppQ9`4=o5v$ya#)Iu?8}((w^KjI}m5JePqAg`Z#$Af^{PU3;hW
zNf0Txsm-YJoY2C2SN(3)qE-X;#Qd=Y&5fx-BVrT2Ysj|-&~;~SbIQ|x!FwqzlW8aF
z8i_MIJ-h^pM1LbW-5tAMDpvHtguI(dzf7lSG&a%NDPtX#+yTc8EulgP64(-b>F3_9
z0=uV*5=O8m!lU>KbLvi`bFi<XP9zTjrhmcqD}-Nr8s;?GTd%Ss{d?8RqW~>BVu1^&
z!>uc9Ncq1k?X7MR<l%kNt2cujhT2Abz^++A%i(j7_d*FlaJ{csMxy+VybJ<Oh;&C-
zKaT=|g=YE!uDdBa5xY_`D9C9%T4fj@7DOp<eH}rVHI<4+W#=h(H*#16gkK|wHN>|R
zSNK@;K44Q8*3qVQvLcBhM+4V2CX0-%HCT;)f30Jq=8OCKsk_41u4b8POSJPD?+1+P
zqMYGag15=IExaMc5N6-xdq#-h%nK5T2>$KLy!o~yq*NZZ!`VsU+qmgRI;wj1Le61Y
zWIM;!2b)HHpb!vZzw+E%L`yxnnKAMB02=u0y0_Y{hobwh<&c5-5;c|TS(?m)s+S>;
zi*<WeCuaNqb?qw>Gdvf`VB|Dy^V?YUJDm<E{y{w5$cY0o9;}q<uP`grTt)`<qCqJO
zypIt=CQyfPcJ+KPQ=hx7w_>YKg`145h|=-Z9UliYmSDI0{Cpn|e)s}~|70rCO$`<v
zz<{{4va3$rMGEgn1ip+S0&?-ci&sqs%jCLZlVQe#+tCTXnr#HoFe^ovldXA8m}8Xd
z)ZSXQiw(YR`@s1TyV;lUH%xVEk7}*;V=De;+V$iG&ogJ^;6!#`_`bc*@*1INn3WdV
zY1;3K?ozP;_Sj7eO)KiBZPE5A*h!#<#5)}f$yj9&wOy{r*Z^><6bFSG=+ghSxj-we
zM3?{AoMqHsn|l_VLlAfSLIlr{@$x^H^WPWs|NChA8RfNS_U3#rZ`5xu%{NF0p8n`u
z<p;CcvB&7kpzSeH(-w5|GK{yL#|Vh?M16C<9A3)D&!EdhcKFooZ$5#Sg2Dwunr1q$
zao?cmoj22=T~Wq$o)-TO*%7M<F$iZZl)o&1?nNiT*py*=E|Hk7n<0q=noc21AL2iV
zCuY)T2Chay@RSfgVM{7SRv4?<MT;<NEaMX7)#kCf+5LQhB+w}pvP#s4$I%f$C7?Ed
zMRva|j9kv<UqR5>E)UaxWj0VI@e%VT+i&@OBs-6n{!0qJN1}jRS&Mn+;>Jst?<48r
zlc5#mhLniL4wC|eTV(&`fl34nM4EqpA6U(e4mlZ46VpuMx1X;t8JoX6xE)h`RT>KH
zp8swA3Bw#O(ww_ET$r<%s~QL;7n+r$DtMa#pZy+I>y!q}eFrwN!x1Jj5QLcVZ1-o4
zW1Fd#L6mIIVcMAF?`eqjR%n`^r3PRof?%EJe!(S06nIDwoBef?Wc}6W!fB<sG`D3P
zpHD{G{FQ${)1fkscp>J#;iG(F00t)_pN(r2?%kahI&HU(5xMOQ+q^%Oed{!xrf%8#
zUR*M&3ye3eYZAwh*)a?zln}0XFDEIc0)HpB!-OD($n48{lDYTn5{W`1sJJF5Ku#Z|
zYCM7;u3WXv1h*H0vPX<a3=Xu%euA{t`k!#6+#2F^#tN^Fb<x9lTXXZ3Qfq~kQ%hy`
z+g~DZ37iZBEBrOIziZk8aNs>9-%rLFi9_x#_O{}=Ojb9CQjR7I*Z$0X`j5~1zkQ!b
zLVy(SjORu@H5i_*_po3Eji_4iE@?D;A;0_kJN1r0BpBd}35;{dMGP%LYJKiW?MZ||
z2qFN%ohmfSDlHMS)5436k<K>)l(|pekRm=(+EI3hQfOWW<ttawowASktp~$5LuF1(
z@S#qab{!dh!_+95zjwzN;rp%EX<$3zfHQ9u!^klX5)_p9A29Yex2pkBG<-Ba{UnF#
zB%#kcl-+iEKC+^OPB20qw}V+-XG`jW&$;BzPX=Uh<am&Y-C)~>Z9Xpplx@F2-P;Ly
zQK2}nnKo(&Uu2SQIyFgQ06G`Qn0z`K#5It>;|OdV`%n@d2SNRs-HH6ID$9jkZj0IV
zVk{Ew8d~X?nrJ2!HFO8}AW(eL6L}~8e|+EmE%7HG{R-WaiPoGYxSQ*fPXF!2-a>=(
zStj}1Ur6!4Ez^DxAj9!M{u1i4>Knkt@KLXFMbY*|)Oo)PaIO1`#Geq14Zm@NhS&30
zkmx|Tbe`xeuS-C6%g<9N`Ba~s7`NcyAVFK%KVi#p+_rv8MSTO`{KL)x`!Oz2bltt}
zW6`Yj8}+8yAFf{}E_e15OYi($3|Q?Ge0&N<grWCh*3_-LPX=Ahu&vp%$G8Q^+gi&K
z;N9X#;!x7a;JDY6;HR)Vl)Xm*kIL%EK=nF4O0ci^vDe1oxQ|4HK<cg|nK?Q~<zeTd
z&B2|WpDoK^&23t?O0bR*q9iuRZBVi5@vFHB10&(qT=8InuyVS3`@KysPC?bvEZ~CI
zXGzD_0b<FzHwxJ`27NIbW@;JdfcyKn5XKw#&qdSa#PQb8q)Apgd4npb-3$UFYnAHy
zJtc^jP5*BJ=<n@ObVHe~Fj-TpG~K8&pIN1o`Tb>m+5$m;+@G_*17qVq2nYyC73Tew
z!PJS<S5u2ko{S+gM@E>~paw)_OHT}9LYNLQO&h^I0_-LY|4T<*PpN#XP;(Tv8$(FA
z6&l&?^c#qt4*pNuo?^Qdc8YUKG7?%Ln52Z8epv+693L!n`zk|>A`<F!PNlE2#MyV|
z(~M6m`^CM&_+~P_l|YmEmFq3dCqd8GYHT+8X+kzTIcsJz7&`QBYMnSXH;Nj-G-mKB
zois{H>3Vxtz-n4oR?RLGTy1}hqw`o?2R5K!J@OuVG_xM)pN2|R&-nu100m6Ge>ra}
zH&vurFO>jHrszEgCFbn3nD0I_-bw{x6yKpjqQQT~C;%P4q7nPpu8{wh-EyWZYj37J
z$4EZVf7e-YbPlcwusQ-&|2H4;pVtLmBf)xd<ug7$+P?ZM>}lBldqp@$(nps|nPiZj
zko8xW`@cWzzc5l^X|Rwxg2A;mkp>7E+<<V6)Ps<W#^ad;WKW4uoldD65Sd}0IBr+R
z(CNKAir64F17c?wdTmkAj#vou<(EGa(34VHNLA}sN}q5cW|-5owTaLqGe->BZCNaN
z8TyMBCMh*ff_!}rWvDd}ea<V{Hec+r`<JxD5)7q}M0wocY`1@a^wd2s%kdk|2^YSV
zVsrasGyA285WniUQtP(BYb|bDEY0I~R6HBLDy;%j>u63{R|7+q_t0|g906(h?o{y+
z=JA?87Rlx-Zu4$_2Q>LO!}INt!!bGGiv&dhnoqR+p`dsv8ACh#{|4D5NSuI2X83h&
zDh8p9XVI*?mf~9N;{IPy=3fMwy2~lDLHzmMvh-&Yk&sT)v2v!cl*!<=zZSB<13-k$
zw|%XG^^<cQTEb4qEVmknL7o_%;NRjjZaNIDrRN(G9OfrT{+<{ft>nISyV5F;Kb}2L
z@JwE919z#R0xzI?1c3<<TM52OQ!H)!UxU|=GXsu>QT2L8Ml=c&a0CaKbYd!fe1alj
z#_T(RYf%34h^UuNQpgySzm;RxE2MDU)`2fHh{`KtZy8%rm6MLKP#w}+eN4Yc`>2<|
zw$++($Dz<=83nufop3=5WeYb0DQ2Vyi3!nuk-Hu>UVTPLUHP5O>V6X3foN1^F}F#;
zs<j{U_S-6J82@yINfZ)hoA|^la(>V6(?3k-stTKCtF4Z9CzX1csS!>BhSjYaH<(6|
z_g_r`SNxTu^s4VauqtQ(*xf$qPNoJ0)&Qg+rS;~k!^L4=`~MAp0uZEqVQbTtJf-~m
z7jq$LT$F%o-igESlH2RB26l7vVS*P3=vNmX=U`D19Dsdy|N0n&!^{DBJ1F=Y-HSC1
z+iiuYok!U_w3%lb!RtF$Cq3*Y8@%EJDO~Mi)zdYBAMao+QNl}cDR!BXSDC`D+t(w(
zYIeu6P(2;$km%zq>XdF#=rlvveTD){tF^c!US}-bKs#5b@Wc1g<JKM~zT>RE*DDG$
zekA0b5gt3HA#h7ZX1yPfIt>%=fMide?OuYW<2Y#s6~P^j?BdC~Z>5<ze7<L2-cAMz
z?Y#_Vhjmn2Mh4JLZUI}`^z_ANa~v|8ZYuaGW)wJZ4P&!EZY^9hp4%cGwb%C)cI#>6
za=E_)1hy@g^X&**G1JjZ$-@a1=!KEm=3054J>a+VILX&K{`)}iOBe~U!<SrCYg&R{
zh-HvvJ+>11HKgpeSE9*ms_5oVw0Z!zB6T8PC3dds(2&<@U92ZENB9Y^+I?qSUqdO0
zT;i$NY*yzdm}hTespcKVM-`=QBKm*XN@}MOCG2x}<5vGG;U(Ude?rCIXS@9(1_(-0
zr%^F}YbC{k>#LFk|Bk{429%)T5Vl2`dhVHL*j_>&#|YZw(^shAW^372LQ3!jk?s3!
zu~s4Dw<)7p(b#SsFTz!Wz=u>JR&DLdyeKe{={AM@>rpX@I-Ow1-Y>F~QhediClP-+
z&1>|vUY^UJ=NuXRayx&p44kj5HNL%E)V}9nkHlRhb=vlO$%5_tr#MqLBQyJ80|Qbj
zllkAh04{^xlXydqF?zJjj0*UEeqX2<PImS+D@~)n(e@;5{d`tr<+jQJUXalbGSV#W
z`|X@c?b<N~TgQXqpuTg+@4+{Ye`mG{KaDP~!6xIse!hp=9CxzT1@w2hA1?gXHewYm
zmZ-@GyZ+8&w>|t?$s~pqdYJ5b$S0vb#(-#SI!hLK7Wm)LngN2;;ic5BLgLch!4xbw
zUikb0Ej9PKBb6(**q|$yj*Q!(gryiU0#~K!l)Xmn_L)Tj7FxE4fsKj2lTg@At=Xhb
z2}QrtPOPN`DAi=#L2k+^9CAq^$6Rf>;#6(jfq7RP;Nd8hp;qKxj$LFu2Px8ua<E+M
zovEravE1{Ie9A7?tUj)pK2YbOdv7{rwi@s7YWEe_(Yjlq=5gE|Y;W^JvRsa3n_-bw
zd03G{5{l*A<~#1#3ClZzyp!3;(|-Ijv*En@!u;Z!+@q=jsLs^aYMq{!goD+YH{*Fq
zbMK5tX5LwjO-)$L&u*6XT9h`Jt8!BtxihYj$8x2d_5vTd{;F!%<f^tXf6575y<k&g
z(qUdNY-?MTQNvqMY<-8|U?ot`e?d|o+GkW_FP&X?;1J#`WlvI7;X1!KQKUI!F;_8b
zu~1Pp?b`45$KVQw@!)Za!=uvF2*S;90EsmcS`k`=4?t3woWj|==*R-<K3>Mpm6?R9
z3tWj++w0Aow(h}o_7;mq$b~GD)DJG|MlH0er!6%2X^XdzE<<N89kKQw%<;-yIXi?=
zdv71l0v9Dce1~?}9aCZoUf<PnbrqHy8JQG^w@>qj&YqVVy;pJlSmltQ!psKRcv)?*
z5c!2xqA4a{v)XIR1MZe^2NOt%GyLH?QL{kl%4*LBcT;fPAg%5xV&03Dq%~5XxZ>Tr
zfJYByV0h&C9`zkJrUvdgwqO2ATCWg9x^C<^O0^(Sskyj}U#KwE(ki|hmNU5MJJpct
zqEW#*cvvzHUn~j}vYld0uxw(=rX8xRa3}o>ZU4RY(TN$b_9LZPl4tG6gVu5`R(Syn
z*S`8o!lfY&kk(%hqZN2IGoH5~gUDZKi4dVYmW~8`Urj>_XLnPBXrOg<N9aJWha#}h
z(KNl3w(+2sr*TzNg6%4pY8^Qptn4GwQBLivqR`7}cJP3oGJ9b{0TTT5HGodtsXBT?
z&=20RC?q&2)2$&Iz}T#1<CO&o^8CK>>@Y=`X3>+WeLC&`!}dOJGMy%$RuXoaZ|~=R
zA}(rD%f&jN&Y|`RGCmbwq6E94!M2|0<*7OsKOIozKh_TrcC#tGuBa>63%m~9-_-@h
zqXL5lf3HlYrYzJuI(Ct}nnzG4{6$Gx9uF5%&jvGN7yigoLoW=QJrTaW4zo+g#lkl+
z#T5t1%GQxsv-}5Uu_I*}rSgh|0Qf=f@1b@-R-C6IKhM3|a>2ovK`B+fUd-p<^<aR;
z@sU2QLdt^|sxD24Ab(@iDG3q<W`Az2O)rz>;M~;KFTZ{*=wr0mSD>BIO$xN;-I>HU
z@%WaavzDbQtZs+2yrr6io6};!ZE9&8OtmY4PJ75P$6fK8dq>8Mk_8YJC-fmnz1kS2
zEkBKQ#=bciCHtk%etJt|%3)Yi{GK`CI5FYrvCyxR&!uwjot*^=>Cr5f(8L7ewB2CZ
zCO31+0iCeD5=+F^gvEHPy}*7wdH<J0T#^dg#ruRId6p!#%H105yQHmIOhKyH>9Z;e
zjROLAShc$4)n`TGD^)mS#xveAo#`3N@W(FvTKYhbyVZ_5alM6e!Y5W8{GIW<jae+d
zfy%i(JEXE>D;w^@#M``tyFpnL<Yn8q)!ffMhD8U<QE!X8w$7>h?9U|Xcb?uP$|WFl
z)rO(M7J_5tU@Zf#M~1lsCaB>s-V>qAqA_!7t&|hwJbYETl(naQp6(OwgBgk$|0kNM
z^ew8QJwo`7={LC8tTtt^3G017NFM|aqHiW%Xu1|dGOFk))W0%rE6cT<4Sq@(BXmBf
zcoHu%IHI<h9rCy`tTrFT;4`RyRsMD;3sC!1PmMCuW~Yi*`|MXb$`87hTi?j=oMl0s
zzi6`1W2xC$KYV@+AhggbpO$sm=jd`R>w6ztPtb_nXRyjKG@72iTi2xteh6zU!oI?b
zznp)nxd^7Gi`sMPuGZ{>NZg)ygC93y!aVpRPJm%Ihd%CtccD!+N}P*Q`gX6X4mbHJ
zzSp9z8$35ZzKOxItV8(FrfQPZOB#Dso5=@CT)8sNcb5o^p0Lz_EDXSIs<XT^%0gEE
zAxcyI4O8DP$7s~8=E|~`^SZ0ChW(rILIi~}4j&J&d<Wt1f>=6t^EZ4j*brf=lz<r%
z;Z3*KY58Nr)b?9mq%sLO{XAdJK%&C9cuAqT+7}LE3}eII?kk3+`H<=D8&f$&js*Rb
zMkRN%3rvIWFTV&^wvB||yjOW(gO~P6@a8hG>(DXkdY?*D=&nkIL<V8I?Wj%dv0+2e
zPUSEy6JewNJ!Lwg*UPQ`$}Z>lDK~$tG>ZdBAtwfHzg0Eb2~9{l)4(Q$=PN51$#(DP
zAi&9>QwP==G(_(m)IzT<bbyxQYBu}gGb*`L>YT^ZgN2J<`^RB{qlPue<Mo>8CiWrL
zE3R%b>-HDoVfL>$4MR{*#(xdu0>`cvR^M&Nf1~}>NX{?wOE^@*`%EyFWc>{jra36w
z7R1m<ZL-lDz37HKyq>a=Mko}`Am?^_S7i<1Bi<U@Ih$Cj?R?h7nsuQvySchQ_4wP6
zkQiy$c+~pG)(~4W(vriGG09fP$mPJ^!Zj)@!&eipUXZ6u{1tBr@0f4X_82OuN_;CU
z<Z<eGQLsgJ)uE%E8Z+^b#}jw8orVrJo|pYDkHy|C`4f*yj{Morr1B=`f!91I5z;oj
zYIaZFdk)uEn4StR*CV)p8{B-h+D#uTQR108`<=2i&n9=L3b~=>xhm{O;VbK{>DnRM
zAX6YqznXe9#&^1n-qp=l+9G~hzTvxYDm)k5%_$00!e<QRF4o$n6Fy;fE!-&L9xpNA
zncBK;BJ!kEh{&LUHdH%>2hL*N$M4xKXSvtRH>=t%J>+~4pG_ka)Oor%vufgGYkeaw
zB80Cgw}kh&n;)_4vCAoZ++1lHKsbo+{>g09ebw-OYl|ws#Omr#Tgi)}g+-&6r&8OY
zO^&{Eb<P$sAKmhoN49E&)%-G*zuli}`!y<WAs0_nz*LB1FKR3F&-E-@Bpf$1-0xPl
zYA#)4VVL7v>eV%G<e(PO?uLXXBRHUeTpkz5P~i^ieZ{zIM>&`5gmd^Dme7Fx^p&7+
zmb2Bua*V7yd)#7b`H^h$%pRd!$yFN|k620dHu;)IIQh2!9g6>(dvB#Pmb*IgOPP^7
zE_NyV%?9T}b)Dq~GD&=A7WU#>*|}Q4X;<c3Mf*y<)}k-hx@%My+G>j&Pk;N*o|@(O
zGSeKb@d&_fFrg-?@Oa?Ezu++f$4|uJWAMzacYn#09R91zo1HOlyk*?)DDV;_ZPJ=$
z>ydy=>ITnyFJ>Gt1x_86Br1gBi|+}3u<|tW$?=QZ)-csLFyBuCU4dmLIN{5CPdV5O
zTc>dBF+~vlKC<46lhU*ajpJwF*V&!5tFBovk8TtAZS6VNH#INe?cnfG0&hVXMwpWn
z4n_u8?s@8jw-|u3#U)P09%a{`lQn{}n364j+~YwrZ;Y{5nJlM!o9s>qac%CYre573
z<CD`out99~LqhM>rruQsqjQ1BGts$Fwygsw+<DKWjUR3=4sW&&NEFXDX&dr(D=g+F
z-7fN>_RMM(-RL(X5){rNas=h^C@(}5WCO!>{vqKRe|oJpGXXXjkQTHS!2hn60)gw@
z(c!-RNqJ8u5Tz=WQs0nx`dgjYBLDbPs+45BXfG_R!T!*Ml6uqlp`)r1%Xe%%EYK@P
z&UvQt&{Z+LuamS(ykhr3d~Vz-<rtyq?fv?qzD!QJl1M80595uH>W9v4g1Pl)zs45x
zVz^8f6m)scqL=EFIF~+m+O@*olHFyRabAC5MHMNvYgM?kxZ6;+Sl3r+lALR|?AOd8
zKsXz>m~Ooge&F_?`T3HM++d(V?%-3@3<{?4TKshBd*(W_<qq=?KvCK2TsjMCq@Z&F
zs&X9++qsxfOTP`{i^OnqTi5KRZS(Q$YA;yWiCIS7=T{;~HM+|`?)I-k{R5Zq$CkdE
zy{$GQ470T-;dV5ysoJ&mHJfy%$lpClM=&muwqsf&rypdTHnEK6bRoA$7=PBjr+3ls
zxG*cewvU!0>+nlK-NRfZ_z?9bNElGo6e*U^7NKC9ENlfo2M-UA1`gdYz8I(@+vY!T
zeE&MZN{JyR#Q808zQR)2+U(gck^|m^Tiu)9k7k=5o7~08M%r1IMR5pr@eZ?neN*3*
zV!uNgl${4e`{)GCCM-Epk%vA#5V@?3Ir!o|jS5^5Yx^obiw`QwISQP*rEWZK((hj@
zI{2xu?XoH)DIOmqge^Jgt<67?z6tv&kM3SY=HjCK2>77o7`bw(GFwR1T8i?mtTUs`
zLFtwXX>@ewqHQ*r2?5Bs;Ol5}`QdSw&B~>1zG8WE-s{4+Gq_ux{L-Qa{cNKEM<DM`
zAk)G}*N^+Xir<E^lw)|Bon9wPVn7<&<R!C9g(T`oUF=U2Rn3amOopCR^CYF8#Xsg4
z<k4Epl(hQPo(s;KcL~eKnVrS^5p2b`PVFBkGc#tI!n+3Q_P1Qf?repr8T<R0K*bZ4
z8Otb1ZjQVPJhv=J1^9~UrUpZp2z{+vkRfjbqxqO=i}JO>O}J~+10>0G$ZHbiSN(kQ
zkWfS760IIEgLSbLcBy%zII@MyIeRvqaNsB!t8WwhQD|Fv=&?JwbWbyi8%d)JiB(dI
zo*~qAS+KhDmtGyNQ7PU(OctJDDlbT;8+^Bd8w}XZe=(`4fAcF5oe3tfkKYKKkYzxz
z$C#5B*Pu>qM9TH^cbjd##}qg#==wgSPdKVb`^!{d`xX+mNm>m=X2JdOYD-*-`r#&6
zx*XKTJTU({NUz`n(0ABYjF5!o#ycR%sUppTgzl^rB{K9jXvJ^i=KXLl<&BQ0+E0Q|
z580P-1ZlpbuTeF;6A3*%>1dE6p^Z4#cG9d|>pHOejPODr#n<yO`+78OV^js<L>lH~
zTDW)c0}K%65~Lli!XKFU^zH+5en)an(mS+&(NK%nlqI#Rbs_THe2hm--5E~v5jdzA
z+wbF-@;XZ;Zsznf?iH0y;I?O0ud*ZUYHu@K>kPGQ^T)F6Ko@YW3xWTo2*da`iKg;Z
ziVNNdKAew*QD@~P07OaMkx0lt$@edpn7@#{pBv~1lV#C-Q>&&6bv@rM70YQ`a{W>u
z66Fzp1+O?xxXL&3SgtD2{914#g~V78r+&&aFr&Tiz8>BXjz6|&SFUv<Tlz?|t%om;
z%z<Myw;5_#XR%nssg11IB|@nvWIUW+PEnrwIAZ|?Du%IVs_o((ZIH~zG%J=f>KD@G
z4#$S8D@`Ztg~A_%ryAxS-TSS@^^OGQxH&F78f<kS`V9_?3`leLv)oTpPRa`(3+F}V
z^zpxY=e;%~42h_-;){{8*6w$^^M4&}3gAdfl96hshk<;;Z<W8!U{K-TN6jxB=TP>^
zlRG;1G9_4T6O@NyrIe*7JejMqhpH|6R~^gs?9{G=I93Kq*1nBWl04rVw=&-|Ra`A4
z-Kgz4g^t9aixu8c>@6I-Bw0#4kB*L4tE3Kli@Q<BQ;114`5;Yv405Khy~>T58sIpw
zXo`AVc~!l)b=N??)$mAPEvWP@-DIn=kiUzfVUS;d&af9xg%&c4$v<Mxcb(wgU^hQ@
z`KlHYktN{j_HZN0)>_y#^Ww8#@l3C(TvJpr&$9edL5XKaw$K%Iwe?K_qC4}PYST!V
zOYglE<`0XO07t`_%%uR5*a5q!0&h8MZ3IT@to%f~`6C!oedwqA&o28UDCr5wAv?as
zNQ)^7Da-}9ky$mtn7wBj@h7XS^LgKnXo&yJmBb0j9m*xE=NmR|88%$oMj0iiF-%&`
zcKRG_i$#&ww^Ecx6C1d5b2aNfaav(~gEw85OOQ#B?>LoG5ek}sS}~2S#+l=;3PSSk
z$OHB;p~9YZb>xp^vSZac<N`AhW~4WrM>*_{$-lS#j#3KuRBE4tleSvIzAj%kAQ&y`
zESF-U**&^e*LP%4#MFG93xdR6Zo5|O-F5(t3DaNWYv=vkB_95v*Kl5MG7;i99}?^S
zBCq|w=>5O6C`*J_WHW2p+x;9ut;n~9BwhbLyaFt>c^}GC$k}`3rwWTQ3=I%3h<mg}
z;`{sKnr6n-@I-6o`jTjJX2BQyku9+l9q;M-5!`n&e4d7j?{O#+{Wo*2w@14uI9|Za
zBP7GZR;g9)0eOwkO-F#;P05F|HhLPNN-7HBkatFQfFHT}gwSi0RQeV7xTB-J=ELe(
z8QLd7S|vfX`ySNWx);aqlsS%lG~9{Ag5-jeU_U5kB|iph@Ql{E7u2@W7j05e1j#W$
zCuPm?DctUJ4t|PNbURxEg`n5%AJ0Yw<B5vvm0-2SbfhceB|2h`morB7h6C|46&zgF
zs~@gLrCDbJSk=O4YSr`FG0FJHPYO3D3)Q!jISwOfrRV$OILRc~3pJURwMusbmw$?3
zTek&(<vJ6%Eo{Jo$DZnqF7~bYVV{wQ6(uJ1`v2iJgN#x$o2*5@O|!C?Lijsol;1TO
z4J8Y3Wrlsn{b?A0GAEu0uHTPtP$}CUb6vOxA2P@{DW$O1l0R$qi5`Pi%T_|8|AgbP
ziIsnD!)-jcw0M#XhZ>UX+2WEXZ7LJ{t|fl}V#Aolx9V}QE7={!R4b#ti25XRmDTmx
zH!=Bij{cu#6mrP6OSLU3^ElK=;R9v_*S&0>45GT^{iyL5vlR2EJx`MDvITV?oA92J
zLySmDbZ0UNBTM<Q3SSL^^1mXzFq(hZl3Jg`ZX4|3?=?MbD*o=t1~}2#i|Wup65OcT
zqs49B#jMwg$*vj^PHD^%rj=xNkw%43{vKKP`?(%}w?8Uhn})2LYzOM}MHKh;f5h}W
zJt}{`XxV}oleEW4iTx!#yt*Kt&(+Qa+b{Sx345GoZH24yM3kv$#RvGowr=nkguP6?
zN-|g^f-(GnIVQ|?H0*U9i-9KZ{N3miFZW8Cd^LH3y|02n0jjpl8n-JQ0>XP4*!Jsc
z>xp?^iGUy(O)z})?2im*A0xTn_*5QkZv41%{$~#0xQMs>?T?E2Qkci=Ub+$H`bcyB
z+X}rOe;`qGBIf_1PjhS|(HtT^J10xqE<V<)^IzuDAXx+?@79t2LNGrGAv03?Qf0|_
z8~3T28h(_Iq*LQ`PAzFqz-Kz0m#?H=t01<-^0-gMbJVC_<|K9inau6lSs8S5Ig3R%
zc(+czCsf(N2#D3UFolI5$f<-~uvk(~BO3=jjy4pf%@*xFT(JouJgr;2Qc)=ly@l$$
zlY0TeAo9!WQ8}UDT^&4fIPXgpsV0tOX)#vh>+$g+qd29F_=irY&0r`al=O@y{<wPz
z%bZ87?f*!;?pF|ZqaNe#`aYbeOGjqeul*(PC=ER^fx$`Tp_MHxE3b@eo8O7FA~gFf
zw;>UVFbVCs2+WL~A|Ubq+~>#;6}S<lXh6`r-DX08IfdYdLtcg%<i}i}dNu0$xgWO>
z)owDm>f^xH`}OiQxA#T`iS=^&>H@Ld_Nit}5?5l@=))L&)&l~|zkCW#^=n1`k>f`>
zrMvEv2_o>KiGnbn4(FVMQh#uvc5ZV_VtjxTiJ55XqW62h9an*QeLF?~3z?6rm=z#{
zGVUqYMv2HeM(QdEP6B-}1t6ijCiupO`Ooo<KMS6G0K-c8oYm^=a$T=i0v09<8z!ac
z^7={bhTmH*H+y}XFq5h(BW$j+Apdaocy}bw%$D`SV?WVuRI&(2cLeUPb_6c~O&3?)
zX!{utKY(QM=^GGN{wAmYg#8NUO*ii&0R6gA0GPaaVxz<rI=STiWS6{SpOIBTyTkbz
zAWbra)a4W|9z;A%%rTrLO{h{`<eo%0=v434=X$w6;}@dqf?vtndGNW%ax^ng;A-CX
z;i%y5sHv$Rf5vn-9r#&JqLNn&Z>`fHPtmvtWC}PNk*E2-i#s@t`pxu+g-LVl7xK$K
zj^fkVXxfk8?8cNHdUX@Ri#$0^&q`+&?EO8?E*ca{hMPIhEX3+*Of8nQj;zn+8hU$k
zi|5!`g;^yR`ljuyCbMuvIA_B2zX^EQy$}E)x4}=OT8%Pn4jTG-)&dfrR8K9gXNsR@
zn8}`G(qj+L1y4OR^1T`9m<pu^0sf9h{`nc8cxZi~UdXNt#;xTi^%*rBK5^&W8fdDc
zZTV>Lc-AE0^#lM50#E1b>bD7h*4dZ)J)8|c9u>UxbEnrdigxyQA;;It7D}0c4SZ!Z
z7`DEyb(rs*YMi_rAesLIR9^;(@2sBAuZ=ct=De@OXQcSST^w&Eku|@qJm@s_NWHmq
zADR34;bWL7OM2YXjX^{nd*sFAt`*j=i)X@#fH{;iqRGFL#P^?Mj@ORnhf|doRDFwG
zDLQ)YanVqrB_KeT+m}Yn);r2BS2Od}$lb?Hidla@k%>5v!$T3RD4SPsaW4E|m@iUp
zgllt)A#Bf-(-a*tVKTADHg(q5bl#R;+@oeWy-`eaCey^ao{?GgZOTn!X$^yCziZyP
zOi8R)kwALxz~ZX3YC+EU-ffEj;V$!{kJHk)3xC3Dk)2f`EKdq4?3ANlELU(S%lT-2
zogmbO;RG_wn94PukwanIKUx6rM%}I0%|%5426-g~WqQ}yS$Tr;?|z_bq@VAp-T!4*
zBql1)CBD1e4pDbFyxSlLL$=&in#AdaSwWDLIin9QvdqL5UB9t}I{K=t)V4n2lfHlF
zwBF|MOIzUS&TonHODrpFta45OsX5Ji?svW1)%`hGOB$FpK2<K3V$v8C!gQ;VH&s(f
zV$-tL^RjcnHfB!0%Nk$8;HFVMF}(XfDyARAe~K`Fq0UEENrtyo{-VLS_>VBN|6fAg
zYje-zLR2<X*s}Z_3-_QOk|#Z;C&W%S;{_k{?eU0@1O-%sih^$%;bccF4Z;CqfbW}R
zSuK1BOJ3FMqXXBS=0f<pPY}T94^(TSA7Pt8tS5zg7Y4<ZqpKvMge0HfFQrf3sn{L+
zY1O{b5OGdoqmuo=4BrlTtOWINS=*^*o*QJe|Lp&w5{QN8gT;TEl|Tuti2TNrU+g<7
zYI-7J-m`%O*P=Q8*;j2Bx^XitHaf;HCS=D2wKK@LjtqeO7|M}ZC_o%|n9%exe0sRL
zN>G;}pKWk1cRg-FO@a$0u53O&r(C&S3m?kUh*B?iVzEwk-XyMu-JGsQqtkfWe+(g7
znf84t-?Ldcr8QR4I|n4}PJhJ;EbD~}ZH{ErlHQ-^2-|0T4DF?5jJX2-&Yi5@%SYg~
zTAoFx4Ali7-(DwhSku;LG!Va0RI?hSYo7s#4zl%A-R>?AFXya=_{cyAf=|-vL9?;z
zu13H94znwgOM$rW9>MR5?gp&f%Q;!7%?9taCeK2q+M$nMYgYVMl~go0ynN1cAy?ZO
z;iDId@rCDk>C9Nmkuf=4O6*;|AukZPqT>d7Pu|@JJ<)~H3EAX68M?JHNObuyh221{
zYIS$b96o%?hDPSYl?$JxB7C65wvgE@OEVQY@XiB@a<nH9OeauO&s@?WCutZ9w^<5N
z)^%%M7y+6GGYQK5RlTfRG>kfsY(eKrC9)fa2*0EvWxh;{mh7GRYAd^7389}qatY+~
zS#mbj>|Y7An$EPRPZ<3yjT1U9sNboR%`ZR<Rq{nMIg$pUaa|PDT0OKeKNjpuxPN(*
z<L5=5I#IslNXnxm&p0GhWZW-$r>6YAyZNguXS7cH65P3PX<Uo{ap$!JqGUjobpqVs
zzP|}~@}s~j<GYxP9^QQ}^9Aajr*yd^M-Xo6to?3nBi8Wi_}lTTg4=%7SB913A_`wZ
zjUUoaEYrzaXia(A8NFYE@Bx}+e!%(63K!obz@zPPl9l)HdMqJ~?F%A>h2o&g@z_(y
z94<(<Mp>V#G8G+Ye{fpx&K6>$klfo4uKgW0mKx#bi4{gOS7ZLsSj)7j?buc$$A?Ak
zd~)m0I+f0v15#9?3_IIGd~M1nMH7psF-+C*#ntuq(jSHPHybq34tv{ZtfEf^V>jpa
z?9&=8#aGOLe+SFKDqHf`#H2|@09kY_0+xj;_k^s|aqHV!)ZvgC);3Pi*|h6t$Al@C
z!O=G+Ulb16yY?&h-F#Dauf3NLkT51Y_UigtRHm&B;+m2kj$XPdV?G&fdK}hXYH6Au
zePg(P=%|^VGC!zuSXcK@6m)aRXbe9CJxDfHQ~^k3fS+XfD6`2(bf)oPJ-baBb8d?l
z3HOq>d+%*2b;D+ZlD2xf(#wBzV*4;atfw=&b{-@Zm2>f5U`2!&4a7wq!qdmXI1D4#
z04bQ_Yrxn%LrOh4<G>i`H?PJlS%)h-qdHHK6mHLH>HR;Qv@mvuT_3m5Xj6NKV>828
z_D7MSduRt83c+NL$5biDFHzx}=yLp~8v&)CI68Hpr#6*3z}#yFHYrv#?&qz-gcz$*
z^oy#`Q`~<$-6*F26g!Oqj&)M8FWk_CI*j_U($a%1HEJ9-JBY|`f6jg<BbH2)fxL_Y
zlAinPQ+4h#_va16;$VSQq%g0S!s8Mux1ohg1B}D-&?J20KbYT7d+62Qm}W<(8Kt;Q
z0wjfOo9>*$!ECL<XFgg49N!?tQxFY^_b+2nq?E557ODsj)T&<=>-s`5jEye{h(syM
zY%c{I_P0L^U5vjKW^Fk3E=G(T(?;c__B2GLufbh-0-B$H=C&EL>_bX*zTL@*x*D9b
zYC3Sgs$a@lpj!i^2sLw-^#kFKjzGCU>0e#$*6Phx3hS7>rKD(nu2?h{qr-dK&3<b@
z|D*M8Ak(z<>${!J1gk+}OwfB1P<+tr7G?t;Cj#|JJx)3YkmjX!T%FL**1|>0c0SuY
z1WsA9lif$Rdqwpu#IgDb3`#om6klXmk~A6^6CG+rcZfAu)&xV4gTnBQPd_Pp^}?52
z=;f1-x2F!a15bKt1d}Eo!YVT?9)}WGQf~7yBE?NUh(~NDU7eUlVO2!#F_7L&Zv|EF
zTc!3leTf1%OfS(zTP$@4d!HfZ`5FrQJ$Alux`=RqO{~(h=b-c)5?(rlz*lerHGYSO
z7Y!*=Ncz6d>1$cCA@e_R_Af_gsFHs;5rRW=-sAD?N5;ZZDAp(hmzo6?BCV%^L^qHa
zxF`tC->NSKviLv7-01{i@^Rr-tuxvJo7rgyB&p6v4`a;N+m=hk-QfUs32K*~o3t42
zw-?@}FH#z68+|RTWtH#<<OJLXr);E=@Gfq4k8Ee7+ch8?`1y+GoW-N521Y#V^Aflh
zBaVs7+=QgGF>Dz80JFGvSs^7lJllbi>U_QT40j92)GpXFfozz`&s0k#vb1sFhr;la
zl6AD4URCg8uJ3rJg`us-YDR2bWd}$l-chJIjJ^-#HaYk-E(>eG`Ms2P^}?Zj*2>zb
z8MLSpFnYEI^RN?GeKloy+;fvh@4|`or^>Fiyr-s&A8D{*;go}t)lt!PMjqsiE*x9%
zb*Mxd+G9qRHaaPrzu#R_?E7}AL<uZ$LBQ}Z-~08gA>cr)a~@djk}jLthiqoggdHy8
zD;*cGYkR|mZ?*_E?{tjn2T}DOV>FIek}hvu==iwJ)5&tG9Ts!6cwn6b<?D(2BaX?&
z%^9?O1{*rcowm6LKgTq#+M4#-6r4lRztGC7xVl$XZxL0v90pjP;>F!Z>*E>xlY$qJ
zMrjpd8sH<8<EXsZ{>1ZN;>aC<`g|qza;QAm>K^pjt6~f6udUvq`eets*~X#&g*OK9
z5<fznZ#A^hjU%K9h2y|W94D0#a={@?vF@{4XD!&@T>595xsA0ZFMN|ru(yp_Dy}8>
z+Qvu=_v|zKR5|vy%^=fhInLjzS2MNoFKC2izVb*0kG6{u-*!`7hkW~eeO~$HzGt#7
znc~);9y$&BO66RZ0S3oeQd--M6(rEaL11?8SZ_KByo;}s<=HyvIXXm9lUJ$RP~j18
zfU1+%I}GsiWohu;M+f_NpaG<b=c#;q1?8k7L?iSmu+>GEJ!Y(6i@ovOwGw=%4?(1T
z=Ys-AHKlr8n}XN@dLq*me5*gGgA@^GN_2c&x6=aV0TJ~^(X_*KiO%7$7+7}QY_h=G
zalt-T81h<2SzdxqwaWY(zIcdEZvuiRiTQL8OWk(5K4-lkDE@E1?!(`DVJBTBFkq0Z
zH3q@kpL?$@kcVGp`=aBM&LO%1HTt$ubh3;uCy&XMst+5^J`4B#l@j-{HZ}MU)##3`
zb+hOF^0OOwH+4;&bc!&nI1UM^XF9)G=qg971FGfcrCISU4-X6Wn6Qv-duU^82$<Eb
zH{{t5I|!jNF>3J^>8gH7uokwgooKq^R@x2B$_xZ9#6h%AQMR}danW86MKEFFaMaAw
z?;0E|az^1oP5S^vk6~2@+hwu@I0u%Af#jU|<#>`|kS>yCw3n`pypRC4{!uX@Jhp3r
zr?s$}5gkj-!CGv{^)36&`DO-|h{FBS1+y!8Bq?6G+uWvi=wpcU?{ILj0Od*Cp`FmW
zlmzWF<3j<SS8N+BOIDZ+n~A7^8!>OUoa^5r6VE*%Z<MXprgoVS<k>7zz|E?|QlIv}
z@`z*?NwfzP0BbM-SIzT^{3Wh4VHMsX*n#a%^cMNnse(NDRpZ$sp#VrokxFVTeTdxJ
zcByXCVkvtlKGAIA?z_YU@{5Gko8j4;lv>)g<kP%?`u(*_!Hv%Lg3Kw&v!c6NAvyTs
zUXdo(psdq&dVA4>mJx3lq7Om1`}ly15Yn#HOK)rtdf~|$3HIknMdCfG{^s%{DG(PP
z`AwaAy|xTaV+oK5aes!-IN5*KE{hpfG@EfGUlgk(=>6xX0xqa<SkIyOj?iQK56&}-
z@;p(d?8d{ptcT^r@8OII@f8PKyIkgRv2(I!oAt6xTbo(x%2@zwxjSyP?+<-c!*z&O
z%9t%HvODga#LCBfX{V_x5zs3mh7JlQQ5Y(G+o6jB9Bu-Rl5UQccH<r9=fVu;hCty_
z1&lM!aM9_Xw&dse(<QnwLefRj?p7gn2dV;U=2@{=d{5R>CE{BjtQK15kCN>8H{ot>
zrCm{3_GiQ@owk8j<X5QadBa9ejJ2!uL#)YqIK^+mpdl~+$)Io=rg9BSo7fccL4|hy
zBLYqWG^K5*pKbTm%m`f{aegDP=zx5QMLJO_mO(T*&%mB6ZV;{rY=_9c00{4k7!c$G
zNoSM?tdBtGdRhv~VZP&|K<U3f93Vk5h{fkCyDjH%Oa)Bi33$KSHim!u_3ca=b;|Yd
zy+K>e{D+@GxKRK@ngyq0d3rVGPvgd{$^&Xzl}us3jm%}G*d9@_8TWSx0)usBppqz{
zHVjYa9|0PKfT|y^o1Vc()rN`;t$WpmH=;tmr!E{4PG-YWh`In&*yRpVCjw&8I3*#^
zj7)Op^@s)$20$#K9U(gbDE@ClOJSZg04BL;Mq*s{^#zB8&st`gN@w7G=54vy9-)?Z
zy_irD`cAK-C{K@QoX0RCK#7U3=Z{IeDin(<q7d^2kVWZK0cjLALIJ(-B|H!XVBYI$
zQ1Jtxn7LuMPsD_hC~pwqcgCxQ5Ms_UI>44=Z1Ayr(cRzVyW_KDO@9Hhz~-iTvL#DO
z6m+Fs!glZd5V#7lM8@XAIz3}#oo8e{4n1(6I_xh6FS12J15HhuZ$14@$sc4@fI~>w
z>$KMca0DD<fXl4AkPbrasao2TnL7A!<jO;`uNqc*mHZ}_rN1k{V~KSoM?e$Y*bCy?
za=8Rf9c-Y1B=}Or*cYDV^EteW@JDM;uje*T9xrZFBinupe3seY_O*>W8D%~Ak)Bh&
zeuMev?P!JArq!T96x3#{a8y)0SMaJPon5-D*kakU3MtI-;EHVL6_-Uy7XMS5!2WiY
zXo>V%m22I5>O)yONc{m*hjO^&udhkUjbY#2(8JhQ7j43WGqk!MM-N7xn6&m0xWHOJ
z|9thz2Fz=@OS%)X@+UnqINpA4=|cF%|HJu+F!o!))?D04pEWP)!#BvoL_do@fI^i*
zyeCzkVdqW!h<6~}M@h5}@INe{3uhV|fe<;&U%>?Ux6~KAVz8JN{-}q2zCRdDI{x-0
zc?-98?rQFc1dZvebol(~ex8}hsDm7wN__L9iL5qRJ!>H+-X>h=`1Ic6Zs#j==+(QM
zxTlq-$hBF2&C4>Rel+I$b*-oVCAfXC7faK!^F5k}hqXouqY^E;(s*vj*4MGYwt3Gn
z%M(tPgBgF^E|tpth@NL%e$jTAn)q2%{{N_4|5RZ9!pBrlevnEqw)<<6CDMPj!C$-#
z!!$|u5&K(&ZPn4$%ml%hp5H;*NW(e>;I#2pF>GZJIr9gop4yEV9q4)8c?$u9zkOrU
zc@{OP=r*1v8pXCw29RWg?Uy{4ygg0o^e}dxPR;ne+n1x8#+tkt$m|djr}AmXnt`dZ
zI5mqgCY6#x4Tr~5GK;=I9H{E>^S_j;+u!IL$0KpNUZooYADlb~p}4Zp%CK)hPd54#
z%OhD_Ewe3&;KyLOlN~0Apj{w%M55?-JO(o%{KwF<5nZ4T7<{ln;p)jPlz)bnV|Nip
zpX}5+kSsKpt;B&q#CNw}ULN)(9^zKF#z4S0ye@tJ{w#~ttp|&gm#vQmN{Jaw3D5}<
z(_d>h59^NxF<?<dN}_-fGExOjtiJ|~k^Tg*9U<r9f<O4rnia?4Ws@%+6hEUdB6{ck
zu-NE_>$659gydpU&<>TsM$uB=Ko=b{CmvNiB5NqFmOoWnzVBcht~h*Bn%ZJ&x`ZH@
z%eY<!1U{|uKgc!FA`HuvQR^rVxbWoc0uh3nty`N_UFZ%eA83LXYGIW>`L#)sgf-!!
z_h>$SBo1p)Bh1C)qB@L`@rz5C4g_*NcdW6Q+%+OVc1gT*XS;YCpsp=~^#{-yE7NZA
zEp2uEnZx{0l5L#T+-Y%J(?U)0`JB5bdF$g+@`J@w8X?4>OTqqxe=kPHdJQpEPFGbb
zrz=9(<^Y`$xrFL=M>za#DZkvC!-g8cD_)AG=(Mp`Zd)Kt!lY7c&2$8)kVC&EEGVun
z^U{nc=KrjKjbgmooZ#MJW8-td@Ch%f4q`)Dmr^KaX~UASS8v@^%{&1o#)iyc3PN%x
zE9_QnQ7mswUMu9dpM_ceu{fP7>T5bCzCAv=qnF@S>U25gKTf>U7;`{^d7hx0eyi8E
zqf^~A0t6P{E|M^R166Ez-qg9f+{TB0N!oVrwbsr1t6BX-xq!QzAfas!m$L}TtZ>-A
zqf!zO8N2(mks7%y$L4a3cdm0Vw@+-BSogv|7P{6^Z3%A<*a%fiovG*@mLjlTJq=2@
z*UeR1eqPigjjc9ZWEeVfJ|EJU^QP{ovQXRNRbzd4n~ffv7a?D1FkTq9#GJK}R<;p+
zqOn@bw_m0bnd@elItT=1nzjAU%wukR&Xb>?!o38$gdz%YCH{fb?|6}-pE>2e6@JI7
zWNrHQb*10`Dz13NJphc8Yf5o*=Wg}G^F(dIi_s~+@FNyMd5%ZrYPTWx8@1S27__4e
zPQ&XZW`e+5ult{C&PW@+wL;oNAYtjKE~0nBfelP>qC2TkKt>GQk0}S_Y$xwIMn!Ac
z2qVqhr_Da=Ky7Z`C<JE!T7t9*Ut`HCy25rQ$=XOCA=phx(bfCllnQWUN2n**g#7#@
zji{n~SOI1}A>r@wi3g<RWFnLXZsrIAIwk!SUT>#_IB2K^thR|k%?LoBO2zoxE3)Y(
zj|@QZu6b){Ibmc1bwGj8LiSIpuw)&Da<0|z2+kov&b{fJi@m9OpcZDzqrQRbt|D<w
z2((sVKEv^1C7k+emF33dFPSbf$3$UVDYYS?tHcC;`#gW8(T^b6=uGDT<XlP6t3{*)
z9;?f)_t8eB2Xobqdy|E>W>a5fi*|oxd~>2u5MqSp<pS^On@0-sWePI|1dNJT%g%89
z9$ew8)lJQ~`yfPZMpNiWC70P`|CZ@kw(Pf}KOrbqeFTs2f}8duy)f-e7Y+Nb;v#HW
zGd3DDghzcrUjVs7-vtjkyBQ-W`}Iw;g9MVXon2IzP2M)l5%un&_>t?|<}_-7jr6nd
z@1_<L+_$ie>cy?Ho~Ir>5nD4V&&^URwP*3^AV8J_uFTLw566u=J-=E&7kDi-ewW|G
z^QFJky~cUrYrckc$i;<KWW_#3)koK9c{*15khtQ<L4cCO>Bik$^J&78J|9iO^xPC2
z8Uz_XM$4BPtv@ckL8cvWUt@*ayf=m}<z-@)Qs+#E;2Usjah?%|OJV)h$@*KCDQ$h@
zJ1YdtB_~3Of!M}!lL+^m(EZjk;o+9zjGb9I2MD^VMgk(k6=RE*%W;l0$$YvywgP$5
z$e>K|E7t3k*$%vU<b2iho2}beEwzqndrXG1A)cjAFSFH@QUFP>T&LZuSKZnrWKXz=
z&`Bas!&W3}yhi1RFb6k`uqZX5E|5;RR!sNyc%H%%R&8@wfXO0X@Y71=-7M_p52YBs
zCRht)R$dozIApD+WEI4<fGJ{Ynw>6(B4TUZ@|HvPY0T1Lp#Ja(mAu5hPadeF!2iZi
zi}i=@-F<@9-&zHi${Po+*?A7{?^35t<-Lse8fAXf)zn*c6v?>_{|{qt0Tt!jZVwL)
zA|(=1L#GnbJxGa2C=COMDBV545CYOIor-jKcO%{1J#;tU<9ojU`Mu|SXPy7ISWA{8
zgKOP$KUeH)?|q?t@?C>f73e@WfACmLwBs~VtJE@}q+e?%xR3vKYnYl!bH;gk#(bit
zQ8dQ^qpHBvI~>(Ai*d_pK2pKar4tUa?tz6fuE}-Y;mSXEQqC?*m2Yo8cKztc)HrFg
zTX^H%8&+{+-$n-ClC#R7bR=T=PqN}C=ROGP5a8h>T*PaXZuT0m29VM05OgXv=<wIs
zyFG11gTtq#|M;VPLfS4bRlxr4g@buqm;@BMVAf@lHsUt)gwHt~`bV`T88PArMgU6p
z=PZk4zbTO5GND=T$R-Mt;Cc=gYvAbxS6LJdKTGr2)~_1PmewG#-S$7W)D&y`ARXB2
zA3ff2{7Q_gt%<mx)5Lar1TVVp&4@n}c+}tjJ--ImSHfNUwPnfabn;mind#3%noeYF
zO@UEY2Jfw23Pm^s`r#T@no=PovK$G*xk%^*#jbu=nLARhI7}P(3!)ryGASvE-#bKR
z5Yc1L@f@Mg0MGkc4n#$-ZOb$}#pvr&jNMi${6dT0a<<~5ZW-(?1LepM78Q2}iqQOD
zfSxU{=OirfgF0w{zv&pqc~}BtBe~zX>69`XrycsFHq6FszNYjR*rvCGE4YrYQOzDs
zxh26l1vve=-R@W0*UGsx=mVhs6XzBr3>UyiOx)taPW8#5pDY+Y4nL7w+5GSs>WypP
zbT%q~pL{Jz8-neHL5*YYx$KP*6#DT42q$SJ4r&)T=ez``{DcV-@uHgkG9`1HWZkr5
zRli?VAb6C#vdzEUMQ-f?0C}l%0N&V83>Y*qng5KM+#bnT0PO3^U%H|5B0<I)VD^{o
z;D=#rFMiWe`UL(<kBK}rtN9w+1O3MvjnbB&Xhor3Br((qX?87+>$wt|(?Uy7%oRr%
z>l3fs9JuXiUuQG0t?Z60x8&mF4~KA=89C=Cub~-ij)vu;<_A|<vB#UKW}(Fl5I<}Q
z^khI9AXYeUrFk}?l%QYvBpbH7DazvMxgU38VQv9{#CdTsv<dL=V!!NTD_(!bAaXFl
zlAXNTUA^d<e|zZph}b(szbv$d|3S&d$4|?M1Qs2L*X)AUW6{5@R8BU!IX9z2inxsj
z&)^>GWgEP(@ei`KdX28NH|N|-XH3-~u0@x%(1wS_Wx>u5(evh_*)<xVr!crDz!!n3
z&C4^IQSLc2?2cs*;p&EkorZuC@5bC)E}yU|gfx2kS|*1NED87<A&~SP;W+)suTsu_
zC0+FM3RaY9p$yh6@GV@PzK5XhJmIlKPx;dQ=KeL8cKN3ea?eM>-gw*?{qI^tWCXI8
zzr&+*zN(1`JjsStqckL9%+9;3mT5iV?(e?!e8K80t(}JpvH|8!mxorZ7}ZQd?rhB-
zV@@6omSeO1GUqp6J?1;YYIi(O6wudABe%yS4l@wk>9|+|LcV4oWCM3VK9qEF#*0qj
zs>P~h*{&{%Q&2K*xlqFzEWVu#g#Dp(-|7|!R{7FZerT|mehx%A+e4+gbUF`Z5C9*T
zTG@tkb&E^q5WlkBE7Lj+sfM)9(Jlc9#7OO9(Pd=M48|1hmO-M_CUc3mQ$<VwtCULG
zL+W=eOHmYI$N5W<>MmBQ!uSUn+PTBL@TgnlE7N{mCY?oYhzW@PUh&3%qx8csC8qnj
z%+ILQ<N>g-)gC(GKJ2`yoEU`tYGqL^Wo}d!-h<G5Y`g697kMG)PhUr<ja{HLw%cBY
zS3KcK>xHY!qme;g5AW83>#^{3;~A8Oz9P_XZ-?^^ETCL;@<Vugo<WTJ7&VT;j9(VO
zVK%$&%b9@>A&+-$uc)GPU7`Spb-kp$iG+-KZ^-J;>@M0x)|Pt0TAvI1gyr;nDpN6I
zF#uc;*8zy+UX`_;#+N)mFk`p>J$cn^%6fIP*;zg|d8=QKM05v6t}$Rw;iIme$8s5O
zCi}{bGWNaaz}-;6yYZec(0p~>>Lq8fij0)A(nQfioaepVJim`#{BeWri5l0Q%QN}0
zCNwo}>YP7BnJ*kN^Y7P!iDEP&REoAfnhg}<1k7HVd%iPzqgE5c{g4#0qx>S+IF1$m
ziWg)cHnCZj+#&HF`t832FU3S8I8|e`qPU_k5XsZO8v~tFvZYFjAqhFt-sR~}^j94i
z$QEYyt<m#!dU(1;mQFuW3{Z@ao`L@|J#=TZrlLP{LeP8Tz9eeMXy;@LY>zNC-Bd()
zv0veC`9Gy(zZ)|PI|&wRs1O;)HrxdZ?<D0`XZk%CZ%hrTcq(35^E&?dhl$k~QevFm
z>MMiK6*iSGrGihoTJdIDKSs56JSWkYqyK&blw<CuR#%d;r=>+xk{VKApc>S$*jb@#
z*a<??7dlE;{x0s+%nM#OT?#pF_Y(spT20vc5_*1uI*dq%g=z$Zit{B_UT-CI-hIkQ
z!;u;(&?=vgRx%Q+At)Bv2N>o1{B>W1J?>nZXL4=j_JR3?ZoYb9!xQpAi`VL3(Y%4?
zXXgM|ajk{KV^MVpz#3b>njX%J<7!|j=))jdgZ3nxb!f+?5X1-CLE&+a&^K1I>dWLe
z8)wEN?O|ZU!z<~dbe>;9!eW?FAy``t=1^4CuD03-kp@nWPu3yzA>F(sq)sN$cjp>#
z(g)0(MQVKVo-+`TuZv&zDwU}=-7Xx~0eq9uQ;@oi5^7HG$BI^ff^u*FOiZmB9fxdP
zZF(3i;!<*Wn*0Kgs2l=Bpx<}98qIx(1hUzUG2;gfJ=j;|8rH$hw8y(_4F2zS5HR2#
zw?rAE{*Km5xu8}cj~SHp>!NdjjRfq)gz93{6G20DMJswhn9dsZuvYmBnb~$wa7!Qz
z?fHkDEdbEf(Nhx!rb(*$z34iN7yzPNHvgp(PweVbDTI;*`N#E1T=SlP!=;Ngl2zZi
z1ky5oi|9_D&`D*+ER;H)bFy|T6|rP~1Zr^y=r@4^mTthToJ5fJ^#}It_wk&qJ{Du{
zp#TwP$9_<h4eRXWFEkD0EDfWtX7VRGjIt9s4S(_5A9;mcnFl;S@vb~*J{j%7jnzmY
z<=tLu(6qIO<*DWc2u-l8KlxHxy_WTJ4?|r!vQMAT)e1phN}LY-t5(FOc+DmTc=xN)
zEcc8(NPa!mswtAfyN|;{=XhYse>_MS_W&M$)$Z%?IslEV?JB&j3&YSg6RT&kYvv;L
zqQ&>Uc6l61SgH?{82%Y0>3`3DlssdtXAVl7duE#Ct#7Z|K@jP`p`s5E@2h!ix_^Ea
z-m9aGy%z375q+rBpjGMik$A%D5zx(eSO-h1J9jEju@sKVv8@-iRJp*^7z!&%{0xcM
zK!-lT!tdeCuK;<yMTLCNV?ut<TLG=!whb_!uPW9Nxv=G5^Lvaji>v|)+kj|0PSEh4
z^906Xl>4g}kH0roxT{<9s4Pki%XUebbH5I?>uv!<V!j5A+4~K#?A$m??#W3nK&7a$
z%ABo+{v&rTJy(1;%MZ!cq1mX>>eER0L1Z1m<g!Uef)_7dXR6Q!AnLD?RU*tmg(!rL
zolZYA>whNf=2JoJ!D-%KbJX`;tsW0O@_t^Hej0}td8|#M>y>c8p3ige73aohDBR&a
z)H&PvMu)J~rAp9&-z^%i9SCQ;fI=xt#~&-6rhguo52xrB&CKm45O)CZu!NFY4S+qn
zHaQ612Z%j~yL_fQe{tQ)UB?>zwQh8Zxw_IL>oo03L|r30j3)nePY_FS4M3r`0Td5T
zI0dvy8GLYEa<|DNSHB#OQiGjw`pu$A0AG40IOFrg07>9;uo^;J=s*s|zj33=dTu!_
zdY}&1*%Cc<S>IYjax5lNpv`n!TIu$=8)%@gJ63Q3z?Yo`&)!X4d%Y={1QvsBZ8oRj
zknVck6ka?CGi$AN$Z|O58#P6T?C@BO%-}R$)vk;79p*9+raH0h^VZu=K>wI?;1Qei
zudBK#R-QF?i<EHuk?+LVaAlwQV6ie{*Z;YMa%7?-FD{T6ZkMJbcF&8VW9g(IjAIAP
z#e`sD#}At(Ew9k1Kg@LeV>a<g;}4{0sNJcOx9j|0UMZe`o!yqml1!&$>Zhl+kTLM}
zR<XE2T_8c<0`*~JRS9$@u>u5^?%P85Wx#04N6~~$ObGvm7MLOlrjxrQn%7zXD!Ar~
zePx;023wAHB=AxHUCAou-b)0gg74({X;^)XuoLE#RFD#zjv4>O3uC}cc=oad^Ys}i
zoA=c?PDxRm;h*4)RNWJQ%DRwB;%na6>S*wK&vUSNBGZNYvtDczQTA1dZOHU9uqfbN
z@JyedMD(q_io5m~*JBpIjK*A<IvU(5i)(tY{c{M{XlfTU{5!uVgi?62@n$!lFJ;uU
zBY>a@*@tJw{Rw}22Zy<^%`)UM)PANAT03fOW(NHzkA}-Wayle_m~U;;kO44hW=38t
z+CKbwl8!{r?O?J}ot~RVO)aYh;<5OK!ur$hNk<odj1I6T8x`d9K^MzQ=dCIZ5;UX6
z3j``V7Ly_YtH!-4E<@rGf4m-oPmpZ3qrYAzVXag?N4Ak^_PB=vc0u>-qQf{K5CkMA
zC828jMioK8l?If<QviH)0=*dtFKZ@n0hmB;sdgO{pt=$d5WZ^U&qPdBGOzC$ixu7>
z;5?I3+|Ll8mHtcBLDqBe`{-eF$a*XA3KU|X2zvKEk*61acibHoM6SBGeBKLoF$Lia
zv9$Q*ZMUA~K9xA=VY<YF5LB=sn`p8@<PUfcU#PjSI=+#P<9WPner$9@6jr;7UD=)v
z@ZV0wk*XqSoV_frQf2`pf4z|$*#uYo&Zb+_x2VPNE84I!SKyMP>%B^^-xTOj_;{EF
z;ih2EW}SrI3c`KLq7TC1^BO&ae(85(E+|9AyEz+EvR6OcnJi2aUJK!V{jOE1kpzt*
zHo#0ll{Sz0_-1A%!1z3jUM)QkD2~hBmwDF+H@GfSO}&DCfd-^#!Uw5Pnv9n8I35Dh
zFCTv7+7Z-|*-WMsNw=1A(1<T&zxy-oHpto1(O=OMZa$H_<w|ucKV>x>D7ZmmKWXmC
z{81VN4A`LdG2sWYP($h<?0cJwB+qh%DIJEgs#tQ@m1y*~ZNj6ndVuphG627I&3EBu
zoN=IZoQuN|;1Sjtk+DRlO7n^e!c+<A1=N7uSYJ_0luhTYyvR*rJa%U{Eln$W^5Fe0
zE5@C9K)p+XD0qs@S_j11ZkUe6T*D$<MlFsaM$T-?@{^j`lW;rM0Tx;KtGlaIDI5?)
z)0CM2H6@|=t_xt)4B!Z?>8I9rg-Me+A8L7jKc!Ax)W5eR=AZz@CSLVOTP<08Ml>kq
zgq&SFi6t`s4k|J}f`!wA5=$>bIZOWJyxEc{W&kCQS)V3P07|C1_laf`d24l^JDyN~
z1G>>)$!owcLOC)|4(DkEr%A`XdT*|!ViHgET0qZv(Yu&aXy2FYKN3qey13U)db!9A
zG1!pusm>u+HxN&Tm7oxZzG-HnP53405z#GN?do!EyI#p2K|=JP_+@bad+Mg^%puVT
z6F;15i#zk{kmO#1oyv(=j7O2v?W97{Z#rcAnf3NRm$e-q<dNGpT)|V;ql1fE)r_FZ
z)z7CPDR+l#PhGA&Y*{1!5zUm089~yV_badi2>)d)$nw`%@akgVeSHh@5faGl7M~RL
zrVem)Lt%%IB8zdx&evfrN{DFh9R-VDdxeAdjypSLXkFW0F*kkgsj~7Gv-=XIGezie
z%pv9!Li42n>4k?$U?>!T8~WA6Oun*--nIFc7>h+rW(qO`KU+ZX<otEA-JR;NT|6V>
z2?6_C?2HLy&U8Gvu);m95@H6*-sNbMz3q&Amy=vBzU`6kVqeYoq{71XBJ;y>>C@&d
z7VytA^J8HWiS|Z68W|nvvp1qBZ?2-ChT5IKhp+=`F(IcAHE=Bi?FHXwX$ng&w+jm#
zc?*vEz1PixY?*&cF{{C3L9*ET4bA5*g0y^0GkXL6)1+D2UIC%JVYNA|fK6FmpTkBs
z3!4|Cs1LbS4UJ&p@0Xy&VWpZt#TY`qgYzLI*@Oj$F@>LXYc5P)&+w_qSylT1Z)JIZ
zB=X+7Ut7%CeY9%0txFu9*YcOG7zx(k?0fD<8_5Kmhoy%q8<lT#+EO6U8{Okv+}2Vd
zS}*YWJ5U6#`DBfGQ7xnSKE+HMfvB=6i0m4zKvp^&VC}$&yJxr_@geQ=S}0O#*aO}$
z5{><}hSNcN#8mMmU(B(Oo17!Ac7TE@Yv3{2Tf**-woC0#SRb#t#i2u~qK&MesK1d^
zIyGARwg@xbU}c`;fLHMMn+Qy?i9jL8-F1DHBP506Tb#P(Z`kgcWa1@Y)k>uF=c6gk
ziwR!xk|2c~$+Ruh?$;74ZcE&DcbII$Z~GBJyYF1B*YKM0i?Q|G)Z}8Y(hU|>+fl|}
z!%n_D-k`snu6MTf+>7_}*|vSJg*gc6iga-ePnYZ@hj_TAA(sdP!mjc>XWQhYFphR3
z*^foW3t=G|QHMYSz#=?BCwzoK7}a9<H2EV7C~@ua$JKl1#oIM6o<BGO2==L8a}Uy2
zpG)Mr2>_X3-Pj06Xih}icVo)@!KkcP=ESvuN>Bp?MI7;U=}Bw#u6!P3?0h)iJ(DoR
zs2RzNvEB?xHvLTWAiOAsa)RI)`5^cHTQbT(uJJH>Gm8nH^j;+-k6xT2$?Un6CC688
z;1x&vd(CE-vZR~4a0`^0qtx%NUdDB^U9kGb!Iw_aad-WteFDMVq6CRX=b~O&A(sMw
zB+$_3=c?3Kvi;OfK1jzx4*<c$>;Md*UERx?`dq0JnvKF<dh4P!IQ7JdE&Flm)y*DF
zKK*f>xDm@)XQTjpKPT(V-1&Z#<$=n+<3sR)Q^8V#vkNy~dhJ&Ku-_qy(YE{YS1rWK
zIR>-&pw457IDOS^-&^g&TTe0zPn{zI*~Gr)REaII<wVEXIbjO;6ia8*cL6iY<r!zU
z#tD1C(C57p?ThN&Q)v`4=uTn<BJHRR*BojHNeK*xMx6>23E?ExP{}?_R{}hUNvc_t
zH=2AaTGck=K13c?ejetZn*qF1oBT5CHu7<etZI){UA^AwSgQKVxMWjMUzGx)><dUA
z6fQv?t5i*w9QG$F?yK^`Wz8`ayuj{#R-)o&f_O*PP`8_32bVNpPeVDxynl)OPzSy*
zryW9t&DB}edZxalGK?8{=g!yrYtK>)SY@UT&!4hwJ{^QrDYzFLVKWC|;up;ksGC}*
zMG7u?iY#S`*k-DwYOzQEyUgPGi(%s>;m$8>^#5|QtbclR{aRn3>Yjrybw)%VU<*99
z36^x@jZ9VrE08?|Mus`wYd%i5$i|gwyHAv|Rp4w)Lsok+L^@@OyRyUe9U=qUN-Y3)
zKa#5H9>wF(jSawWk<_r8><3AqY}2}=p^C!iWt65(x&_Xro9UQ@^^*SY-bi8<n$x?%
z#idaoeYuD*Vk~{^k0&vt*R3*f4vzNk*Y``Y_IYHKj?ZaZWa=tt9gvFBIevwQtp<m)
z+xrhCk57>TWg?RYk-k`H%vDzL)k^X@MiL`nnY-DPhdyjW;mhZfz)bF)RQS}wv}1Fe
zNCO&~*;Rc~$rZ!EyyhjIn4|eK4`gwKtk~mU(s+0BC)&EOoZw;FFnq&Mn?(qiOtP33
z5Hn5Sq^~&kmA<4wnpBP<4`W55r+tD%Ytc~_IJjw}HiCjT?SF$NyU+kYo(}@)CmmUD
zIsreV9ffAV>in9OHS>A}Gbbag&%)!lVKw0A*#}Dl4}f3*Y6Et)eUhTbm%~ht3?VK7
zf5O6Cem0bvmkvOMw+8`qVMjl6cpO&zNQ5@i@Q`w@J2^ZebM)Zn6IfTY2HLXbOKM0w
zpeB&V2fy{7{#5T`ALi|c1R>8=>Fa<M)1r@yZinX}lDwZHfQJND4i5ecqV!Q!H4vEx
z|4E`w$tQ`)=MINga0>N?-jA~qK7HuJCB&?3nOMuq0A?9g!lalS@A&@3(12l>AY-IT
zC{qC~J<{bPz>+-wB;ev@bz0cQ1g?KW4s3^R)15Ch&rxX(q;#zC&z~#tTS2d?QBaTk
zIu|=C!Ch;lHq;~ya1`*nzaQRYMq5V18|JEVk8<Lnd@%$0OAXP*MJj(-Qwz%b&=JJy
z1#L=z6coC4;Wisdvp_+e^2$7}W&lB4e~#U)mCCoyd#~-dTc1UVUXNY{Q!J{UL1z}5
ze3iOhb->eYGhS-w?yBV%b>Z||2}7e^f?K%D^rz)Jjw7NV!15(QgS{cj@;N`^bw97!
zOgp%^Y$gmkQ({*&SK%(BCuljH=Wo9FRsS};DV7q>GaEL^dZ#-&58^=?>c4wHfBAYa
zPh^8WNfIC^X=bKBSgoCCS%p#-#~j|jTrK{FU;+LJL4Q?W+vRpvb!otKdRD7;sJY8m
zm}-3J7s$7n5gL^nwlJ4Pd1)7_?Od)Tx;x8l5H@FWY21F2uvz4A8`XHYINIg(9<SQ~
z5M7K|RvP!ByDeHlx!NrkN`Ac~EYDw9yBQFwVSIQoHxMSOZCpI1)l@gVIl+K*!^*$G
zAQ~q`=~7DxcB#a}gxGWiKk9rw`tD>w(#&0>WM24g;f8^5zpy$}Dv3GdKi#?hF~DVZ
z``fG$nQYF+0BDly$fCF=i2dT;50<{Mnzo_L2=y)%)pxyWW0Fb}KRafcE8(r0l2#Im
zq9a@(_h44hciJA_8Wq3zL=zwE5I`q<M&mHzQ-OXRsdQZ0iShh`p&Pf>j<8EMd){}g
zFWu&3<%^^kTp<?Cgez#MEvAI?!>VC09s{Nu(uXhEzcaF%9O3geh1+Ygw!eK+0GrFs
zpLj~AUiw4qovn)Yo#Gt$>k$^)8Ql#;`nC}xTW@6D&xZXg6u<l+*#0{h%_oC!+pWL#
z{DDRY7<H-*4nV$`j#k%8`r0%tCsh<f*GlSe$k%tQueQnv4lBkKPbW}F<0jO_`sRS~
zCyZO;-U(nT8k~|SOC>4_te`g6Vte*RF;FqnQ09H10N6(!0`{Bl_2DvQc_!^w9c$Bc
zEvOVUs>G~#UYjfa;EV3(1pa-5USy-H#2_4i745yVtD_1-;%PlrU9pu5Qqu*IP0knT
z3X9{gW+>Az=s%IrqR^5giiOhScL7DnibtB~!?j%Jv%DV2k&@W;7yTVyx)zY%8|2|+
zm({=nz}xCrB=3if2tJ=_9xF)U#)M_S=Mp6*GUx5@C;U<S+f}cTcvL4~0Ao8i<;1p|
zatr+_>GH;1Q8sNHvj}TYA5VjEP{zakEfYjKwH)wJ*0#;7e<=>wC(r1RlMn;sl?XbN
z7odR!y)q+jB$I+ZmEZrN8Qtw~IkIjz*yTqVHhR=(J|e6J#Do)qDWrd4+yDNRP7U}-
z8u^wQ%`t#*ZtrPtkVk=Zqu)XZ;#@|~fg-Vc;h^=l;8DwhpMk+`v;p1m*e{&UOle$}
z$&WIwLLv>7+*g^K92hj*EoeI5=zA<n1Sx^t*uQi{dJeQ}VzEow9S^U30|UmzXLg<b
z`bKT0(t&c{<uu-yL`IS_wAAUvsXL8IfJM)~n(G$Jmep#}qqWJ<FV#-cE=}*En|hR~
zyuUM5OvDzeEkS!K37GLH0sjZrca!6vH5DO}wR^vP-g~rP9X;3QP<9!AeOLvomJ}^9
z8KFg}<JjI|CUqgVzE2UFINrZ?56qy^uChqT(U>Gqym}g*vaA_Xf7m1(sG_}f)Q;yI
z`$sA?w7k+BmbLE!RM3=ZYm`Jk>wtd0!~Tm_-M1%*dF$r;`N#XE$7==^gY~^~q*|kG
zU;&A<sO#AgirN4(zM9-vsiCar?;otvEX)WPFojzXy_{c*eWk<bRB8#p!5tHkR;tUO
z@rd9)<aF3^CkbUYSxihGHl+Q7Ti-37#}Nv@$K6#!HoFZPI>XFCABc>V_Z}#L73~;7
z3Yxjh?kQLhNog|HG02ekqkC!0NFK0s6kyKRi9Z~N_mZ^@W`v9F3Xm24zLd?;Pq^<x
z9gZ-B=yejb#1+H$%>%E1eC2jR(YTwJAN9_`>&8h|`F&Kp2#`LbWutz*`<I408p+dl
zOe^8>)l~muKZqy$@U?j`_Fw)~j^GQE`Ko4w?fB4q!IY=|YNU~FlZ_wZvn+m)SmO?7
zK4zuzzpF$4kIx%m#cTN2F1nU4G{MQ%7l{<aiEdheEOLbW^C=~P3sL}a6QIr3=#dS3
z|ElI26n@-xwL0?VuO{M-I-1HhLHIuxE6B3HTNCfd#<}P0!jq!-0$658{ZTZu=zIUN
z>DO<$Re!OL-!=3qQD|h5ip<#@<O`f<)Wl$9h2@`0pN|76bZ&w&xthMnW9$F_-Kvnr
z{dcQ^!u~FF0z}4FcV_9z$OVA(-C58k2LM*7PZf1z+&yeK4Z8(&PEocnVTvugz0z*R
z&C{(RXA8_td5RBq0Di-?s~V|U=V1DJF){^!l_F;rN}KYrA>05bnzs9=X{G@hS&Jip
z=9fx6yiLebfH??s&m$G8@*cZxO=r?bey4{t*x>%vTP>{*q2Wg90r$wC!tU)y97Y`0
zD5cplkcn082Jz*%ijv<Z>^vU@BKbA4WK|K9y2p4FIXGRuGG_H)py+8j?0O?EdM%;H
z$IfmEekmBn=wpXu0<r_Vu4n%sL6I8zq}2d3)|%?oFsA2T-**M>Qj6Z@y$9jM#Hbf&
ze%Lz$I3}s8+izIL@z`96oS&+ehB-@0xct}((`<S`&mdF&Kf!0v3u8DL9-h{vjWQh4
zcw4cX`te_u7bVz6>C)jo^Ipg$R6~gy(Ryc)M1Y<=J0h>J=Td`IgrHd}qXS_u<m+D@
zH(G7VH^1%rH`|ea_Eji1{?+Wb*TVtE-u|;5sR_gBLkz-*E<wz8YPds93!yJxNb55%
zj)-J+p`Hy%xI<|qheP`n-`T}>Yn=7nH$Sg!HmauOtrLPS)iia!Nd<Q-@K2YYO_RT`
zR&F8;Q1HmL-x>njP$)Eh9R0+4&2sZz6>A)|6OWK3WjK}Imyhp^-|P3AchC4v7~J+E
z*ghm;*kLAE#OI$e7*i}VUwivgN`c!LEyI;Q+hoFjf=`IRo;~YI5X!C&|L0&TyPQDX
z6u?IeBmV?o>1D=UE_p85ZpeG7g7O0dWU@q(hRZaPim17P4)n4Oo8v9iOxWdUdEJEk
z$IO>xervs4r<kAkfPOBrbNBcIArcQTtjWCfyjVo(AN}=yi36fp)|ST1xlZ32%#6v#
z{0#|Js*GMj0+r}%=rW#!gYFj%iKoGzd|&)X{|sW+AP5YSM@J&~{^I*@YQd-2cZj0R
zhpfSbF>d%x^1;sc%f_PJJXgbs(-g6>8&Oru>IEmX2te9X7@;PW?U5ip%`0;g<}l>k
z+lm2B=38<<-5kd-x1$}n1s1<ZD0%P3Ldt=kjzKv0h&%3%I{?A}?Yzw5UfJ_rTA>LI
z<xnPM97NnvYy4O+mD?WI_j00C!t7k?^14Ng&y!llDQb6mVHyHBevq-9Crp7Xp!O|)
z&m<u^c;~d+;%VyHUipxdo&4VtJc%u~WzKkPM34<|K0P`E792Ik>tmrN5-N$<ax#;N
zqKYYv+#ZK}LYoK$Qg(oP-NfDOvqDCQ1U9lnx~VrEqc?>5=(Vg@+-$L1AmsF_0`U!U
z`r31`v=Q=2k>+SOW--3m+(TgfBz)dctL{8may0mH>e^HIso)0zbY$So_?W2wd|SfL
z!Q_Fi@bL00rh6N<2z$6*Az_LjyMQ9h@b{)o?4Av;7&i!x8%oB@hqaQ33U#=Tzp;kj
z)QK5dI*NWanJzR>Q)56+S|`0f{;8!OCZ)~`8=kns`wTeVod;Bl30ZDGKcBP#&(<Gk
zNs0T4yc|@(47e_>HFY=HE*UEsq4uvi7J2z`%dUMF^&3ogiA(fb8musUP{1t1Kc}rm
zPbZa_i-~J;DoyYje^r;nsiPM0!I**z^I?B_*@qM{OWacz!|lBpGD1K6vU2>RDZn{<
ziM`sL8K;+6)Oay#Bv#)F6gD$<nNJ_^O{hnoYEY+_WWhF%34V8Yih2O_6C6PXj&JV0
zZ}B;jfZ#Yt(|x&0ikhf5oK#+F7zsJ`0pC$yC!CPI_^?_R1xQ6J-&;2vcbz2+xyy`E
z?9<#qc9$K9H<UbYRD=#{){?8v8tAuJiPy){9vtd+vXZp-_M-tzrxqV?So~_)XQu@P
z&%&8S<BtSqyz?Sb*y>vX6Bj-ay7i{p19QO<^DOD}53P={1HisVQ69R(^E@|@t6V4r
z_05R6{_SkkYfPS8d>!hVWcbBQAfbB1kmtiGr4n>ayTiF*jk41pERlXagpQ34uxFLD
zjS{{F{VnR`H1y$)hQ&(764wAngzfm6{Qc~`;B|Mj;x3`d75sW`>48r(ao8C38-x(-
z1KbW51{!?SM2*q0uts!bdtC2Yy>kxJ-l_fg*cq3!3T4N>_qRb=;unxar;X~-`>Es+
zt035ay;H#7`VCN7kq5+JEYbFEwETLXB!8@=G>f3QUL&28ib?J(J@vp>AT`X^h^M<6
zE-~%?9_HTB&Q;<;MNAH_SM;<us4n6Spi4u%2?q-yGRnn{r=Odz{vrJ!3rEx%t<512
z#<)`<OC_B+PlY9jw$T?vPF+W=V)gX-<;fhUXXwc@NJ#ha-(Ptp(BTDx`N+7gs5`Wl
z1TTb$h3!#2*Wi%t-r!G6{9sgj&FDqP)%Ii=Dpz09)3spbrE{~beM?<^eOD}*D?8)?
zV0)3EJr5R-JV&F$EFe_2+d*3SO3aa1PoU_toQJ)Uo1MzaL7oQtu2N7(_?A1unEomL
zJ&<4-6=#>f9x8vWoKTlP1f)Uam-_d+8W|TS{gvWp!;%X3q#<~yU1Z+qv`?6Zsofv0
zb@M%ooq$YNvE*^f>!et|oS0oon+!UhtZu4&>2I!ivEaNEChrqa0<5zpA<rIx0`wrt
zdsTo=bh9I`fAcZHF6hY={=^bqvJ4U|i}7gQrX6ReJng=K<=s*^p~)tI9Bhsk%2L68
zN1$j2g$(~@q#Oua9&JJ`HhMD@$hhw-8-M}$lRFZ2ISZv#wn}~MEx;7?>HKxb>!Dsd
zXPG+BJx^%TbeeL=^SEeR_9CrZTR?Q5FzrgJkagPIfGLpB#7H6CJ3@GxMSQuok2U|d
zuG37WcdU7($iDv}Sn-#HfEM02o1SG%RBt02i?(M#4B`++-ApuI<T?EIs6R7}JtP|a
zaF`&g=aCdbiH7Xm6948?lXPsa245rlK=YL^&tra^(vbM6rt3wd+i3J-ji@N>vEUIU
zimNhG^ZkFjfJaXx(`kookOzLurw%GKA&Ve&odt+KcJs)}x1k+vRQV<quyR!@j6SIy
z2Ey7D8LyIrxSbBzP1n;kHCnzD-TrQw5nm~pZ)>xf(Y^~))PAjZgDTYc@nA~8p0gUb
zrXw;U=3R>6973C-fx+7ha<|{0%z`h<m61Rc5F&bhh45lf-LW_S3lwt4LW{bKZht}E
z07G`p$=*EL>4NY!0oSSTcrZ;R1tWv78j{WNQZc@*gBcf#aZY~GZ@42}v-G{101YL6
z>H&M}+OF919x+5VK(d@Hqnx&Qv?cy1sB7Cv1Y?nmyBO08SZ_W92pgUYCm-FIU%*WN
z{5jUH6D9K3v>WH}s+}~bu)#rAxIgQfa(PK$Lj8jRel{|Iz06kJ@EU#`tZ#wzgYYdi
z_$rDN8N)XTa4D?mw5DeyWPH!uL4=9wqYGBmujFB~>-87ETRE$$ZY+EmO;{p!Svxdf
z*Ne?w^Jz7R0_|IheGhjx*wejl30NZT3Gja_pgZ5nz^G&=Z{K=D${rtX!=F4PSZH0y
z!BM{re~_i*{e1{+!Xwef95i|d(AdoLly<#BlpmA4hR$}zblyJMHrt5-jhVYQ#?FMg
zF$^CGZyN{a_X?Qq)NWk~b7leyFu7A~+OS?_XaDr4=O*Ri!{jMv96o{{0nv<@RQQQ+
zrUKv0@99*a<2{=w_NY8wn-zEzZMs}^8|w%pB-gq1${$dN+IHFzJ@;g@NgQWWkV%Bl
z<xW4ue7D}?|1Syhzuhe0-@3raMBYZl4L7;9wywVG_%0yKWK;pPBZB9a{UhyI=_~#?
zsUX+QkPYc|g9H^6QK!}ZP{kynYr?^D`8Dg8wv*lAq^^S4b~M|iAKDu>)2o|~x0mHg
z6sHpXzgk!!R)PlwA8v-n@a<KwDQ&NCb>0r=*SMUNK=|c{-VpG&*DO-3389Niv`@_!
zsdyscqq_P+ch6pZzw@EsLFi1mu6Lk&`%&pTo^`ES-;2C-HUDfFlHxDN-brz;jc_e8
zZlX}0;s>9<h>nKNW0k(~U=gw{A+elUbYA(&&4fQp=Xw1{f7IIJG^q&DfY3dB2iz1p
zc?ymK_J*Ujvkt;!BVKdwo_0B+h9AXfFRzO4U|f@-7ERZi1M<A9isJWYT6!oW3I{XS
zH*M^C&jxUFOO1=Mc#@TEfz-?synZSoY&Qc?L_<%W#Zv2yyTt^Y-kgyZajxuA`i`;G
zeD*c(t5FXnX(P8NktjX(sR5FBKJ3F1=?SQel;ixK^820*&>6%<|E^gKFn8et#zL#z
z6K|6*M)78hk}OBN#V$(UOndpZ9^L>2pgQN8YV513>0i@ZN#VsUKZ@#h^N8DI8_@$W
zl0`C(+U|h$>L0#2w9D%%_G()<o@>9QSbDgq9s@|5`fsR<@Y`AVp$k<#w876sIobrj
zVkhufY74DLNcK&G?}6|#(Eo(3Kza0};ELjS)w8sdSiWI-%a@`kSMYj0`qe@l)d{+S
z!K1rO?!cDUm>}PrMR`XcwBYzxFgt6JKb5P=tjFy>j(qjJoqXM9qWNg7j#YxXwrS?C
zE$&|M_v9yCmxqeN`{P&DcxK2UQ(8Oifkr?e&`)vNa|=C;cfkKzjGE{;rOOWoJ}|2?
zN%b|~a==-mUQjx4h>j^+q7NblFsz6}lG4qT2~DVBKtTQ)8e=|`^*&*=|HaW^<rLu?
zy|txoMLU50^c#@8H{W@j9G;rIgBL#29B-a0$Vg&K4%n*qIxKTqxx_AuTY>{bC1;xF
zyWN+}&7OT|B+v+}rfZ{Yg(2hQL-YU`=FrdkcRWqqS8x}g;Y_M=1x>EM*sEi)D<^rY
zVW{>E!mXb6zJTrj@^o!J2cE9YMi@@C{V6J0Ey@Y1d*lFA55%!FlAn?NBFP^o;X7AT
z$8!6$$XQps3Ken_rWmIE{qfh3BC9SV$6QIP4rqujKi}zy4j32BUW!umTPj(ABjuRR
zQcayl3X1;bdEKz%E5?1Hk|#Dao1Z^~OiTr%Qh>e~-fS3f6DxnFL>PyF(YG!pnZSnU
zU1O4Br5;pO1%#d!BB)abD6L)b6a|0oVAJER-Mj*P;jX=c>u>rc^&G}A=++8M?=8$q
zR#C|?d7@>aA??U6NNC<@D`?|?E49}Ln;0B%F1-WTh1xC&clAs6><h51lpQ}3uR1lC
zl@9qN{>B}JjSW@H4YY&ba?$8;8nxT1a9C~gpb+9YuOkiL(TN!kJtj|v_esxq4i?Wo
z5w+o8Tc`cqSeQb(Dc3p{hY5`>Gq1OdZgwh35%^}<e{|{b;Zkw)*Bnp{jFom%%{9EG
z>OTh70kdV#J~cX&;qW$M=7N@^en@A0UfE4)-_P~xceRsnb2Bf&rwKI;1*Q~-kM%m-
zTQFNPyzD}1canYJy`d=*&6qsy{<bFpOh}c_VWua5#e$ga{>NrN6+WQXx$NhA<xAH~
zS3n58YA=qqK^u&erP2z3L`BxHT9ibG(ziJDq0!i@4+X~StqD(1zGp&jZ)^)}NVY2j
z=r7JMTer@Bs3^)=yRVUt$&g#Bx>FOCb}N1)G@0EOq3fZmY#7Dl*-IH(b8fy}h;2rr
z-Z`3yJNpLCR-W*s6I*h3K5<5GD_?n@c;oP*$HTpNd?;G`-1r{H+C`EkPGX#<Ia@d8
z_jPyD(%oi<GbeK3@Q~(*b?~7}xA5rSz1Oe(l)H?rU%taP=~`6@<OM2`|JE`6mxR!g
zER`M#v3J(PMkP~>_^>WGOlOBI&?>!xR*6Hoc9SV+=y-OQHakuoCxX7eUBOk=bX{;r
zmSQ~h(T9o#{B7Z;|3$kcHI*2O$EHoefaV?4pR51a*$UUS0~(gVwwHD#G<m->re-}-
z^ldg-D}=m=f3r>Zr9lV*AJ@_MUqsX=!vRDBn;kEFa41(L4F;!cE00SV>*^Ts2nm!e
zi{hYNpACz`&wI59^^%PpPa_7r!(67!;0S?kf05l|ptKx5P7~u0pVbi9NHo1%a8_#Q
zmvw0s%{Rvl8lx7c9)kQj?g*R>C4Aa5;3+-Gf^}v0y5pJr%SadR3HGZr=SA2nj=EX6
z5pS_l$4WElR`WRc6&CWqaD<*+FT>cRj<qPyRv-48=V>^K6U$^tR(?Y(fNFi7YY$uh
z>dl{Cj!Knm!uMvvh5^X<!a`UaZsyPW_b_=P6xnOnqa1c~v#S~e4t|hRav*3zuCC25
z*AibS<_iD}oR7uRr?iPe7+`ms+Cdp)JvW8Th%EtcWV^*ZR6CG{Ancd#!+8Du#T<67
zBqjWoS7ZCguVPD$-Cg}oaTa*y2y0l+Cq66#W%ri(Zk;}WNIj8|X_HF<J$t^+`j+Ru
zrD<>={!q8*&+79K#W!$8mT!>Eqx;*KVt0mDO;@XV@9+jA60R2SW~Z9eJ%%_rsrEV_
z0SI6sf>S4I154u?ewwsop;6smAGibn<roYP74()#UZTyOQRD)lzQ(_D#83OVv26)O
zht8TGZ<~X$oB0a}y}cafZMz2?7N~z?+kBU5WnptWs5L&`qJMEGjHTgmyeA~5t0F8V
z68NQD$QwNhWBr-fq5FZX$_*SjbO|Hb2%|NA$m;fRW;|DgybRi?q4@NqgX$dLPJkxt
z5Ur`ZAJ9GHq#>cr5y(=EMA!_ULg|I<L#dYP9dVoF8z#f@M+J`MR&KGr5t);kKSNC{
zy^#7W`oG?N2sLLm0e~L3tyj)RlWPbZFD^MhHaUDAy!F^T0xh#qAEB5`n{02qHld4+
z+t?nE<oLjVmt>xC)c$>&o722d$qE(R)uEME3q{-GIo%BoJ74yPV7AGM+wH88CUI>J
z<b6yNxr$u6x#E=R4+ccR+7u)kHXh0~8n)h>wqu)4DvsgBNsOsnpj;p3)$gT-`D>+!
zCA}p<(KpEtXZ-|wn-%P?yQ}`=co?Gg`{PXh=RFeTMTUqu!X+!iFuhzvy&_`OCnW=<
z-jA9Xn4HX+cmgb%E{DnW{o&Jl80Sc+*TVKCExK9G9I^2uQEG75clg?_Y)@s=f@C@(
zP4eoX0iS%Y`2lR!WOYZ+*+HaOE>oC9_<olA*j-S|D?hM~NDqp;(Gg~Kda-&a-g5Uq
z4W(Jdp0+x*!y)4ctz8UgJb|}OSnzshYV6G^MHwvMC+75$Mo~=ZOVFEKQucWW6nY~x
zsHcuU-vRSc$}>N1j@D}m{PrDud<FqaBSowwVQ5mvxg0)g4O!q;?isn;>FDXapd89K
zq%^SJiDY7~(pb*@3;T~GS1Uv35*hc0O1G{?{Bhf_g`1t+m|nlDSE{xbJA}HS{oDi+
zl>I7ed4xOos~F{GNjol^GKBQS0g+;do#-U4@E)hU=5^Ec3ZCcmagZ?bHGExgK(Qt(
zyqSoad=l9^OH4ON#rsY)fzliO8R4Tk%iaz`|IxMfv%-SrPVGBu`c6;mRqKpMDM34o
z|2>{U#0ie>bszA)*0!0NS%q(P33~*-K0;eYquyR7ZxJ6Jd|Y{H?IoZ?#YchRuo+nl
z38xzA(iCkJ!8#e+y_weaN93Dla2KDSXERad-jnL2Y3>rT#UYcsuI`Lv(1mS!oY-X5
z+Fp?tgR4?~R~v)KZOXs(&X+m@omR*KDI(`So~|R}&p^59UB5N19Mg;{<bsA&DSUY-
z-aP~@p6gdiQ`PNHTH?mrul-atsexP{w~Pxo__`;jG(;PDd$JeqBTyPO1QgDiExI_#
z%C0w)s-Y=u*zXiWNyCoXlJeOa20sZ1x{M5AJxliDq*#$cdq8-ioq(nSLSli&*yQaa
zqWQu6W;|PrmuCy``o`o*DLOHge=JM82x{a2w!=U|ThLK%7a^4LsELbCCnuq)l-}Q?
zVboxn<T6LVj)4D@L(m3&`hN9N6HVuCznmj9PztxJ{8@lsvP;$YNBM#EF2_i~PtA~K
zci>C^lOA^{cP|S!V0NM**qeW`=1lgC>_W$?HWS~=TsYgT5a`V|916^f8>{Z%Ds$a^
z0+>988*&{&|CbN(QvtOyfr)^fhqjf{MpwPZ6(&y?&x4ZL;$=HF1TR5)L?2gLbZSXt
zHh=+7&~&|>Bo|JJFc*7Nvj2Qu+9a4<kLON_kQ76-`$2mAr@0wPns^^kgoW;Vbhkbg
zQG$U+DKy<)H?(XH3Coqpz9!+qFIpemHI0Op0x2=rI^)|v`_@5PPt7GWJT@E1I1MFi
zn$+0xG0@xk5;lGfe>-Vh)F`k89+%Y&esC<ROGP+ZUlc`)bnOO*b}f*zZLpcjq(IuS
z8Hy8o%$Yg7(ChP1Za<Z+2C>D)y@w8s=i4CTa!d#`8aQ!u&6-Or=02LA>ff!BY-DQG
zFTEY`fpCB9ky-qIs|F2L<U72WuW`Cn_-fSCc4f~rZXD_K<~n*g@r?!KjF<%lOwqsf
z16{9h>}g`n%B(OW<=3zHCNeLMm`VN}r~NNg$AA28Ln`keNIMn}0I>=2yAU*NY21nD
z?J#5x(>>ES19zQ}j<3lplFuW_32evz>E`4Y@}&sbDYVwx9Jg!Se(}dzIZaIgw|la@
z$#GmI+6dRxy=P#$c0QoM?Cd6<P$wmr6cG<jGr$UTBO-avPBPGz9Gzy{sO50JiiXkH
zWbsUtss5-1A+$L)CHNqw>bi+(=x)0I4Ocvn|B6gr@2nuRN=He5Gi3j?Bs12d=%=24
z<uudro(iS?+2l|=I>v#>>XU5lMl#TvNiWa`#p#_UPW`a^D?t<j;LkqKSE4^|8Z=j%
zOE&wWw0fhJ#I%aARx*h=l3ctifsX9Q@h97Gk(0OA#kKW<$DiYLolGAPPDLdK91Br4
z=sH8nXA<czp1fNj{O1r2(Y&dLLt>Z+5dXaPKPR}-ZxHG98R246JFB5hbUbh8ha;3_
zk)uVh7-)*O6{|%i|9<?d<8gM}B}C-2B7ETIfKb<mKQ~8IyEh%%OyS<{n63^NlT&zL
zID*Ra!j$4XP;F`Ou|-Caymq&4laXq~`V-%g3ix)GZ=Eafmf;+Qkpmm$$IVK5dvUC*
z{j|BK1cQw7+}eO&s*CyX)H=xtZCAbXW)V^g+{94?k#iZ9E#2RWD*PrHk;)OC`$xz<
ze!3vezcWq{(bDaLsHd`z!q(!f#<B-Gk)Ue*H3vY+mwkIsk2m&mL#^0VZ59x8KGS$v
zf&IvE0X*W`iWM+m+94FBdBsr|r{}uUvBPU#P%5+Q;ELm$dO9e~5ozdz%(b=bgQo~(
z@Uuy2vr!yN&`r^r0j@fmLmTlw$7zW3_lPT*o*}UQZN~6F++A<MP~eDHP!j<R#g7n*
z)XN$IW5Iz*HF;5>*C3r>jj#BG2^zWzpqx_(^Q{8+s`v|0*Es)T^8Rs=WFzl(p^}oG
z+3FPgZjN;OlFO>5zxmH<BPzaI2nYMk_<3)zruip6x<!}<+^wwWo4y?*2DbCa+3s`>
z`JZ;9mk}2?JCflh;P?Sh&zLsq3{?bmd`+<sM=j__>Si6Zn$NX^B5Z-MbdsORk-m30
zDM_;{s5SxNGw5<67f~wK_zEvr+0%PIb_g}G{Gj>q0coH4!T0QV@$#fzoEj*Kjc+zM
zJa;M#@nA*(7G|O7HWEl^RMc?N=Ku`Zv$s;$ZVgS46O4dxt(0U{XQ~ZAxjNr^U<b8Z
zSwNqd_Uh&Wt2D{7N-N`GX2>J<U;VnQA%Mw`Km^VH?O2=fF_9ITBSJ*}=S%#bZOf6J
z1OSG*AP;cEtbx^!E+C%Guzi{zEd*YOba7F4<X%RYro~k{kTI!{n!0{Uo=I{Xf^_zc
z;qDtk+e+4yg1A>nQ_!pBKgmL80mm08voG9=x2}{kM45zkN2YFnGd5MYLZ^nZRsuia
zaJW(jp_Q}v`6b42wY8ZBq_4JX>RViK7K>wUW~1W`CZF+Y!e%iAO*)1V-P1SYcFF#o
zc*3r`<n77YJiV`A)BOmc5Q5Lo9oFAu7u)Z0%T#a|Be<)@I$oGV(zOKjR~)*MYuZ?*
z6#vdRux-DpXAk!SLS<8bN+)KfZv6a`i@!MgP9RgN@`NK^q|u*O)$jj8N%Ai>AQHb;
zw6hgoKi-{Yqu>gp*FFo1HvbD)owZ?~ch{<X@pP0(-{DA&eDx1PAw&H4KEG^=6#eJ5
z|1tLgn4l$UA`1*78RBzPl%_)OGBgxVl?qZ=ed=0^(om4qOK`MNDRzR^k5LVw6vTHo
zAHfi9@U3gA+}>c!p_5Gs7BGvHq^X8tAj~tm^}$=~*}ab?RObYnPtF(e%}c4o#vK-O
z#l$3uy(#P<7_ubtL*G&LMQN;1-Edf49gsV^NHf}v?Asj`;XB9W9&Qg7tlz@H5ctG3
zq+bc%d0vBlK3V8?ijZ&8E{Vpgck_MjgFDh(G(?s_Xt*fy9($_`$RnE_SUfMOqn$_9
zbu2qedVsCwPYgO?3(aekhs@cX<8{q<8yq`8*R9)~pJ>+nGi*xRys%<`Z;rl4sBbM?
zJQctx3Qz^Jbg36C0B6<Eo4-Y69gsE?uT5|O^hf)xY|4LLO8jjMV4S&w5fn-He>qw`
zMNC!HMEGeF<JKnDvnbhRVBZw?y1@ztr$3?BC*wT|4+z{ZxB=O)BpMuv@WOT5|AXzY
z$p^za+s@e%z)QO6n7AGY6gS-O_(&q!=y+0c+%98id4(^@9cR0{jktr&76g-akakZp
zcdgu|n}~OayzIfLH9e_Uwy=5w*(sG|Zym&d{KE30-aPl=)StYTXxE!!5`XB(8)`rK
zRHCI1b|sS{_H7uKHBR$q2)hY5*9AKkNPGC|3EQ9KmS;I_u1n<@=(=_~s$l5PzDs~`
z+bR<LYammxmFkwo5Ubh7n+0fj#1&FSY7@9Mfx+D<U~04Yi@YyJ=@pV>azXi^ke`%F
z{SrWd_sda15bL?DQ~;Pd{}*z_{D0Nt6GR^!oAkPX^F(V5!1*UcjL1*}a8ansIQBoU
z{qqU@6Sloe!~P~bw&L@H`K&E;yycnsWwf^xxxEbACN@Mz{)8<QilEXf+Orda__Yk2
zMp%K5!XOgt#hT=UP%$hAPbp)Qep!}I{JMl?JFqj05{xjc_7HwV5kHNR)V~JLmgOF+
z$iDIwgvHkzIfR|sxP10rk;aq4LasG=8IqNgZA=&-B%vm&Bxb*S=mR-~a`yNvi1ZU6
z#t4mZ&mEKbT=&F!L|)oY1*O&%wNp%H^J`G?l}>i2-R;{>6S(fV3+OV53I^Kh78(95
z(BVo$w%<&y1A>IoUmnp|QT;(*Ne{p^O7yr}17{njuDq*8ngLRvc9B(^F|Zxh^v1zM
z<CeqPbu8&4BQbBFJ`j(~vH)<QWV`0X^lt>$8uKI3i??dNzhE@t!2k*ig^CRIf3n(H
z5Cq@ERhn}Z<$sUF%NySz<laW_7Qq_7d(6|bTg@TsNa>RFQ9UffcuA!fCr@?-x&mz1
znMk;faN@bUtZ>e@<H@^8bGN!?RhB3^bn*u!aW|)Ri;t)A&!)97)?eIN7DWr%R*6`+
zlM;rLZn;B*L(=GX{p0=p%0F}{4Ab#~cIPpj_ziI(-Y>tIjHOYzrKpGlt>yt4>Dr%p
zynxzxy8Wi{AN}77FRGiq5Zk4<hm&?+XieaM>cAi$5EPrV)jyCHbu1yc<ftu74x}|6
z%q29jyQc++5|-M4y>d!cu1a<mpuF7zU`)OEowJ0$HU<$|0Kv<X=i#2p#%~N5Z6^`y
zI*c&#u9Lof-2;F^DU<r1_g#GX`PpRI1QbBqgW4K8{?Ew`I|HNWG3U2*2LGJ!|9E{v
z?EhoN%%)<u_Jm0Q<A}^<2v1Im3M3>5I>ThD{B%iAdS*vU8Xc^TC~c=y9wN=XFO8hK
z=^8_%`G-=9PnzT6-PkB9wfoPne*zkVl3MoNa`kJXZGyW{=i;+<nWKC`y2&mh_*76B
zbkk_Da-4iy=qeLLy#8j7i>v6g9?6oO0?e1!%!Rz>-y`6CPE#>O!n4}{)ygnlm{{4h
zBlSE|vR=^rZAV7Cl^DD{Z#z)-EZ?vo9M2HGUs&8Y*y{hqd!BKf(1dZYm(b*dY!n$A
zs7(NO+K#wycK7E~+4AhQ^p$C1V>?0&hBq_CrCnq@`_`Ur(a%-~f%_Xf`8WPfsO#{$
zDrP4L*nV~esyMO*6FeZ+&IZUf{_~`7|E+w_zUg)RfB7H(<&0bY5CAPjx%1-ELai+_
zjtGdn<N4zC*s@5v@k?8D%3q6uqVq`ZL|dB@#Xvk=|5$*R-6~-Bu7F0yHuP-Uw4)N+
zbxNl@znm#7(T?q`QgNUhVyx?S%BfMoRop%6KQV>&J!UyG{Q~Ew(b#%+DGk{d{MMd!
z+TDatgf8}<i!(1YiaG4pj?2YdsW6%9G@BQN5ud}tQHgI|!81^)B*B8-WG+$L&4KpU
zmr~?=U88HV6z5-+wK5j{ZKsL*E#HWF8?7e)KdilVSd?qmK1@kS3>Y*F2!erh3&Q{k
zvJueDR=QQBOH#U#7LgiCT44YY32AT;>5!6U$N}jZ>UZ7#zWutN=h^SGkMH=tKONL#
zxbAhYYprvg=eZDmJjGl^pMF#?-LP|~Y*w-bchT!OLqU`ShG_{9X>C1~980<phuHv%
zthw5FRrFnPoyW{s-nnEev<m1WlKQ3k$biFI1N-e?@3V0tG`LP|xUKy!2lnfT|4gPe
z);}S!c{TO+)q*jOJdV1p5JduBUny!68Z9KdqU=xzyKGatEk@Lhc-tAV83T(IZBCT5
z{-Ay`D4fR5Nuzg&8nb87*J7t{)Ibv>&_QW3t0#+3Y8v3{y%H(REMQ9j`^>tNf&1{M
z0m`MzDel_Yur{@-^-*9GKqBiTo|{(6ayh-!U3=Tv_SyU;(wAqam|C@GB49Dnhz~N>
z$R`nLGU)cPb9y~`a<FhgUS4slf-;1-V+>2mPFU7eyta|tHlD7VLiGA0qR#A6WQNHG
zREv&+%ZJJ30@?+?q~SB#D}L^4r^KX;&HFZ<fZV#-sGEJzI7l++i8=tz=ehCsZ~nT~
zG0a386}{TeFO{bK=Xw3ho1Mt^bM`ahYH6!KjSYV)Nh~Tw;7I!ZEoar!ybzeLfZ0CL
zIg`+STN>+7eWGo0FN5wQtT?^gcFS^cq&8;#0(X;OE1b=%R($i){=h+D2z#TFtvl^<
z@5r+LpfC;G>1Fc{OS9*&LabrXZE*XrezM=XnOA(<M`DGTqgeVlX|}ktRBGLfPjc58
zwbRSl!9*!c!iXR!RAh5uIT>`dR`b+9FYd^(xZ@n9Zo(I}HRoWE4ouSb0@f6&ZphK2
z?Z4ulKKfa6+E$u~6q7L+x|z&eNbG$H008mTkKb+C+U4h^Km~eU>t?4#Si&tQJjJc!
zU#G&TO5QT5WVU3{qWiaH=C2QpuYjtESv?~u8A4u#lU<MW`>~(ApV<>&3Gb+z&<qij
zDu=M438Xy&OPxHnOt#KQ)<sOyYjeR=Q78DykJ7w#<WlQg7`*%NN@Sw$wuI}=)8-wz
zfM_wcGJ5Czsp2l@H-NQ(dz<5B`2EREw1lMIf;?--bg&x%-&?On+p}`XEJd?t#@(DA
z@9~hbRk?kMz_~dp_;RN47!Qip?{vl+=piN2LYA6284fNF-b*jt8HexC&6Mc%m-nQE
z<3$7081dEKY|*JlEs7s>+za)mUk&wrfu=GafP=4?6B#IuY@xO8{d8*!N#<$KzHo+Q
zYt_DJ6;xz*oAP@$;l7XqTB+xjU8*(Q$y?nV4}V1*kr|oa{`aRVGU3`P|M0$2QU6~M
zMbi<?>khpqw%VgJltTnxw+qTf&rumGVs9}<7aw3{F0>v6BuqNOu#7Qq6I(CGZ#&cM
zG3{<PI^Wj0LNj&XEdDg|Yr5exms1b9c<Tzg#j3oAR`Bmt=~f;2KYT*0pbOZoe6Fgw
zvW4L}=QCa9w|!UNoH2}%_EVJcaKf6j$!nzTvig1wKt@;6V6Y{mvf@XD8ii)oSY=Ow
z#pQYLW!4y-LOXBAMA%KSd=1u4FKsy6$&VyP%piId>!5)`mi@-lbM*A*G4_7}k(M%O
z_<Z$Amoa{OX(`(0$gKJVkL#xtAa@N{orL%ucl2z^ALv?_e%5tF9IZDmp`5x@x@ETR
zhI^~4&s^?B-IBBXh1E7_D{`EOGh1nJ5B>#E`E_Ui<><>G=FLi~SKj=tn{D0R!kna;
z6yf$d`4o|0uoDs%0&2^F1(Rupfs+=_bqZQjI@6Nra7l3nkK{2D+^iMNvLaSU$!G>E
zHQ+EDmXpwGOpVJ@9^#+pqTi&E-C@#%%bORTY%kgjn1G(Mf0`o9<08G0KwnUto~+n%
zDT|AUEsY#TEZ=&7>=DiJM)SPl`&^-NIT($s0#%(eN^CjJZ5|_4;Vmw<gG-jt_N4VH
z2!c5sy09HK&e!F)qJp3Dl941|#Ef24=R21K1fqR_v7ROlERqOs?`%PWD4+di__?7g
zJ9>to)64_xf=fjoM=eEi(q!Bn@nFeBT$(rY`^n*0Hx*HrwA+Q;zs_b}21<k|A5wrX
z{r9k3q=D{XttjkmNr4>q$!<W)y!D=9?XVY6<<hCy2@&K%_VlznElJ^wo=c_I2aThs
zg%ZSv#Ees?&Do9^H35(EQg6`M4!&eskxQ=DN>t!vadfrZ$Uw6$@I>lEOCQ&u-DJ0K
zd3d@lIy@#xf4rt)|6WH?9VsUAdl{X?H^T<WZT~-rMYO%-vxChXf%qI9>A7E-CS%Qh
z%`8yT6xND7{L>5I>0pJptU?q%Q;)b7u^2O|gt#MtCVT$eb{?BK4fA`_q+Qc=wfJ3Z
zycQ5c0a9+5K!l6m9jDnyiCjQ9D@UpK+rpPU0r;UlpaX#D{ukHLlA6PyAxh`3-R^tV
zOfAvzh=uY6I8ORBg+k6>$B8-n^Ima32fCO4ew>Wge%>omPaJLa!<wFZG0XT?7sQDW
zv=ZF(r?&N#y>-!gu_HF*_PYE^r#!+jOG~TuZNevxR<2JH<?D?!OTpShqlm6t9*i!g
zuR^t!#__#C8xmiw1tK#<d$P1{7o92Tdf0qv9P{mj{}vkwOk#EB40#HHDq~|X+=-Ao
z55@rtPt``S>N`9_%&b5DVc3!KfZ7djQdXyR*@VlxZj8G=CftMEB~W{L*8Et;dil_P
zL$PoPu~>RAU52Q`2AIr{RiUHV99H+I>+Nl?M>rFy8Jg)n%?$cl048bW^!-U#U_7I$
znR{0SjB$;sA*NFR3?cdbdYW75`<3_YuQASQ1B={0k^a?r$QI}k`}t~cs66b7e3$-m
zsN~S0GP={2M*P#E3WGxp2F2?FX$65T(b_JZJ9US}I@!8k0!Ik)hd&r(i(n)vc;}HC
z>_=_41EM)3{6_~Xf=eA4OU(MyLT-Oi_1-~~af$ExM#^jtU?%bI+LoZ<^Dx`#kCNLc
zBy%uAH{yp&OO>;CS$e2u^KOUGjRq!WP0+F^Eshg7VM1L@Bh8y_e}mgB66$hnl&49h
zo)3U~rS=aO+OF&~Mvn_2LZ#e?5wxxN(K4kLF<6K0USPC~N3z3yoa!eh4=dq$;uc-*
zp1akcneL5t_gjAtis_u3*SeOkt>#)1^T1*&{qk*qFJ%L_IB`(#cv^sov4BnE8HT%C
zU6fP-?U2e_0aXRCRWes*uGio=mDq3b{RPp$kU+<o=tfl<`VYssNCaN_tqE4~%N4wi
zv*YzLWD^@)+v`M@j+6*;)EQ=lLe?|SkER#~O82c2tTMV-M{@Ri%--Qdcm(POyP981
z9nj%K^5|3}90t4Jnif0F%z6wHSM^lEosyc}T`Vl8q9=A1^@d#U>JOPVp^4uSIF^t*
zNj+TCr|qHiN7$aLWjW{RCj=7`_wte)`C}%mU=}rN?j=XkvZz`-Md@(HCrAh}s1+?>
zrQDs&dXtwl6~|q?)0y-@%@#9QJoJd&RMIi%{L?O00TO0$z|V8adGJstN57(_O{nAW
zD$3ZYUrizhv^HKJ^M<#}Yz3-jF5eeeylwsK#jP2m4XuwfwAB#Z{qJ{7%@I^$qmWJ-
zwyPw}cLeGVLnx7P8ZjY3<^)3o?D95oTD~w3&!^bU?TZ)`i6X)bhj<q*;EjLUnj6F5
zW#eSciiof%r1qn1)ZNaEa@gnEp9#r$w<Y;dgHI~PqcBx^_;W5gy1OK(<v7M;W5?7=
z){LQ@bSz?Lq|!GwAh_NGgQew|x%`HacX3~;tC`djLw<rSrIFrx=d?Q5*eaWtZg#Iw
z5oL8*4o(}bBINooKDqEaeh^THl=ye{r=u%1to7$UfcqLHltwOUt)K&(y!rul76)**
z@(;tN;RQdc7d&#oruH&0ByxXJqf*1oY^T9D$ns~GK|;h8hCH$neYvRg;~(ORkpk+H
zC!|hajdAy^*BP1c#Rdzl9bZmE`uZ^l%1XEq+EaaKgg5YqAZ4Ynm-5Zjo0VA2JV@A%
zNH?|9rCeqTG}v&scjNe$OM^|5GK;hz;?ja6rRQDQbRNtU6dSN1M7N#2wx)*)5O{P@
zcJ0~lLU{&S4{HK00(WiCrO19VpQCjNaX`d2rhF~!(K430Yieui1w4HutzNc^Rna^4
zg$7YIE1E>Xn}%f;6S=mrCH%5mqqZo-aEIQIG6=$qMbO<ffsry^1=<G39%VP}k^A_y
z4v#};KSW2N*EaT>|H^~5^8nY#xgn_$cF+nx18nMejI+k94k%~bygf>Hf1T8#1b8j)
z*V9FM{%sBO>qEi5pWvfoQCpR45(_JVB<ZUDnP=-~ELjj|JqZKZYh@2Q9^gvN%0jey
zEDF)+;wW~Z?gncPnD}vi`)Glmpz@U90Akgwigy3wmR!$OXZB}U`TT%R{NOiRl#^*S
zz4+w2$~&8aO4=9HQaDtZd-Fbuh-1it;Z5^aZz!aX2w$Uy!ya4TSilNDyrlZ@VkwW%
ze5OQh*BQTH=@yRdjz~piZ&O<9m$rqf(TisM^TVN;#kd8F5tgd4^VLC4dOzkAGE5YU
z{{XE<@c=LUkmgsvr4F?ksrM)he!YUfe)#LC$Ri$u0HOv%?atMtG43Qc5304pX_60H
ztUX-#S;e?+V=AXgl6A@t{V5Eeq*!NFBdXSmz*h)sh1FKIQ8EyjSj#$QuD!}aFC|VS
z%2+$LMVA<^uB8<kt-)A?7DF?YQD*RuF_l`JA(W5qN9g1eR3mr9V(<eSzV}+O7VZST
zFUZdc%?qbVt`N;hmv9Y7m=xiy%njwPY-MoKm_*R_vumhZ_KZ4B8{vvH5Z3c?`OPQ*
z!uFad(|@(8@(k+NncX>?_da|-7Ni_c@?R|!2W_Gs^b!$=GyW6HYaa>G$%Fa_#w7hn
zqX^gbm!`=$Kut@Cd@EmKmer<$3`yUZzZXqy!Y@T?w7p2X)s|F(@G?j*{aDRj^}niQ
zhSa=pbar0PMtiO<7fmBRxQ>^tdy~6sqf%&Tx6GXS_j`Iw@=UViX&h!jo8L7T{EdQS
ztGW)nopM{c@KlEZCD{EShuV)aH%!${3ybBPHaS~9@vnqB4P|vnn$jM9FKr|f8Z5sN
z<9ywmsFO)P<LZ9IclgH3r(=j}ZMR>~J%RwbrO17mul_<je*N?>QDJ-wO1+0zg`ctx
zHzL}{ScWeTV&Dy9LY~2+>5SHpsw3fMOvq#PXc%>hi)U|av|AO;fMN+JI>cesj(=sa
zoU{(>WU#mq*BTRYIpH9*HCH>1IfBl)aL5BavcHSRCpC-BN?6l1nDFKkjc25o8w!ke
z_Tg|(RI+GprW68ThRvS>E(m%ifa7y1^_|55*~se<|Jta}U963hZg5(U%}c{=on(!y
zgW0X>tcVT^^i7y*#KFex{56lZu<>`vp|`hl5{v@{czf=CAxRbqK*q%~ww@CR!)!E%
z=lp7Vkdt)aLc-Hna8v!|2>$L|=#-%F_@y==fYb*1e&$XAlpJ!%Cbft0iK9FgLHdiA
zTc@ldlk+m*RYlRYXgS&LwjAvMDR2ApSlOpew%%64<`F$25^4HBdfj%{ydy=(oNx;!
zbT%zzPgb_~-<VCrRfZbA{2=fp=w+M`<wb%9!aw94`?vScX0RmaHz^ssGdtdBRHB|q
zq>ONW|CU@Fmn9fAQ9iB6zB%-Wem?ZMi8>q3%;`s3WUCUjQ$N=ttxciY%ZQ!QxM=s)
zdiykiJzz_FQV3RHaKIEX*NH%&)F7&~U4Q+`5kJeBI~mMU{}d?PKVh&rX-%i+JQ8i3
zVD2bMVSjdt>&j?idz>$WFI?P>Xfa9THA5_j73%HbFcAr9n`NwqWH*0geQAphT+VcH
zj_*rQ2KcI5$pI61OjZDIqqKn)`|HxGq7VV;g?*f&hGK`>jb|;>H?2A4i)Uuqnu*5w
zWGQctQO$ZdMc^2ZNT`jRKB*s{LDDYLITrL+FmA#edkXCgaIxoENa#I0zxA?&*Y9kV
zEEa`D3c0ZJR9$cyuqwh;r5!jT4JJJY%<rl@It2BfJplYL2O>2kQ&xaAFt@@4UbFwY
ziM^CTYrK3jJ67;l75&#F^FM!6iScKJr*G+Kn}+0aMWzwR5CvA%5L6N@e(UXz=xh$Y
z+stTmKpfGjXcqyzzhYpGWTo^c`)?}HwWT9}Sk8nehL_?19_@i^Z*9;5#Adh{t(A0c
zMV*85u{2p-2UvE8)4e<DY<zJ_q2u!##&^XG6p{pAFx$}3JEKK<u$xhis%sTwNz8I&
zUa^L2Z@2S|hAq3lG@M)<gIzbtQY=(lV`fcJuEnwfc|xSJvX`K6SMomNtT)Je01o8n
zdBlw=ZBc#WaI|i^^y``gm_6Wk)hRMl{8L;6(mmzs8B1*kv#Mv+R49U5>IA%%SL_L`
z7qPrgL*8XXvC|NWeYqShe^c5fVoqlY^jBiitx7bXg<NoEW>gh^Zc?y$j^I*pvkfOU
zj9}|zmDD1glaz~W$DH;+=?~cRA=6%{x}~2D#zj`yGpLfAk<fAn4iLnKC>47AEmM|L
z05QrqtIBUyDm2uJ-~SfKu%I(8^T;WKYoc6tQo3uS05@yrWtFKXSTQ$yHZP({G659&
z-t1F9OAkZFAEO<@z!ZCX%xS0IPUP@|70=zo7)Z<YbKDOa+4^yre$T48*Q$U^VdpUu
z#@h!e;FDG|ftG}rXx#fp>*!B};R8SjbAd^;AhfyC%*Oo&ZfVc;q3R|=eNf-2Qxr&6
za{!>oIqtdDc_RsEus1#c%|rZJE%d)FjhI)RbqVgc<`S$GfnkC2x!ZHo(m!cPf4R^9
zC6__;%}1g?+9I!`+}74U`Gj5T0B{2<JAvmJk_iH87V27kQk#}0IiwxCN7yy@VSSd%
zYlNwYh}ddG6&)BlnS0GUUM%VFcqbq|7yxgg)b!luN<c)FGMP;jT1qng|G?f{k15YI
zN2lVkCG6g!IO&gdvKy=7^NaD-b+`}wh~d-A6&FPS>`x4n@#TMnx<m`ocy|D_R`~qs
zzVk~k_LDXMO>gSmfMsB{TC>^57*gyxcYO&cBbf$V!9K0KlRyCc;uuU^)d7iMIHS%4
zJQZQ0aZm765DyKUzk$ujyz!4~_h-T_fOEKdwA{(Qw%(JnG;besv9Lo)P<oz8UK0^K
z?mj*0e!T)eUR0)Zpq<0E#q~=qp!$c+8NN$mjK?@>eOMwtDbrK_r2_k#hhn$^!MbSs
z<YX2BTDMk#`=oUQH3TGVp2YiPqnv49SU9>~hfuc?rn+OV_hc!55PGN~py7Y?p&H>b
zG>u;#UF@*(I$jD>wv@vR^1jBme9dVHPIL3vUJT8uN|!-d*s`8ra3Mv!nQp~1q+^A~
z!mieA^W}185*TSqxp{IStmBq7v)opSVxNlm#TYMgOh4DY2oKzh@JYneyy8{w{LtHL
z-b`g=XmitufdeL7cxUO(&I1iQ>P-t*IV^JRqWXC-i{oIbhpyA^vO{QCQ1ivcXvdyh
zw0{4yK%c%7wWp1~_rcuKBrvD=-Y>od)$yq=qZW@(e&g+dhyxUth#f(T{VMM2*GH^J
zQ?DYyPYVj4Ee{kdy?uvfCVTf|?cAq>vAqh%wUN#8uqCi6*6tje^T7Qj;BR3!Z)Vf8
zwpJKBJXv^0Fr2^osWcGYro8XZ)6f83zK$a453FY|1UUR`pZMW=eA53lYrEa%0!+r;
z4&+^EoZ|*4U4EEEp8F(*1Rh`bUYs~yS*|*n?^WV+cU!u{l|B|QI#gOWpGB`wh!|9k
z?y5R$7%D|z%MkkHf!Zx;LRIUZLs;kK^WfFeWKOI5m4h<YB&j&dSK~B5<CWOl`9T16
zgcronTv-e?=?YHm|Nfz$@|@WUAT!EqSn(zsJE`sK8l}Q8&OJC~>(t4Rz^<LAtqhrn
zCf#jbUVl4-?w#azb~&?^ozF@sauxBL?KL1_b70Cq!rs{0=CECFdMLHtK->@L#0I&|
zoIRcZ%2fYuQ(@o%$o74J7#<5=1B7Px0GNM~a-;4wmf!dSv>Qo~gFX{R2r=_pDx_e4
z91m^j#!r#SNn*6dPUoDrs@8mVCPdO4zh1S-0mjQ<E8gU}ysU`FAb$<sS#fsZPK0?*
z5ibELp7T{N2>AmKCSM1$VhX0)5jUZ`!(s1>Y2&X8<!>HJ!}b5*J5CXEY`^?{kuFT2
z90o>1+sj)e(@HTcKYY1a7dC}DB4G+NnC(Fc0Vm5U!|jMUiTv)jJRxQ{ok{8C0u>Q5
zSp@dUioPw-tb2HMgcWt&j&SY(ZGT`(?F&ktgzzUOU2WZ!6MWSk8~y=?M&e;<aRzQv
z&TLya9IFrfnygcxP~8U?`H=J$7SG7b0v?0Ee=o`taKh}m5C5(zH*~MKS$8GQZMPsw
z$^%q!0nG~T9T(NjF&EVvguO2to{O9`Zw(UyuBC<}4dCq<Uy&gf73n!k)Jer>dV%~|
zK*LjSb82UUGOorB;L8#uxCa4IV66G>^6HAI9?DD@4dEr2Z?i8x|HD`p36-ZSsd=Di
z7ng7&uh|NL6D;`covag9qE);JcSob8s2kz2dsAa5d>^iPSb91Cv6man`B=urbWSLS
zn3FxVc4wzqd9u1S-HTb-%k*2tmRw0}zY@9{T?J;^EQ!=d1Fdx*son3+c&i-ywzW4A
z<akRiuD{EfVBH-xx)(+_u<PcO$veTkEp`x86T0yuwaSXX9<IyQExlZewV-nnY>c8^
zs9tC{TYgUi=Ml%vnQT1nl3o{@z@uAHSOsXfi$ZFaFb>IqW`5txGwVQD8XOmbS3V=j
z?{)-{pDit8=UQPV`W?f$y8@+(21fgxOi)}&1w8-bFN~PKNx4LR)6hFE>GuShI+AE1
z-=8&;B#cqUQhqV<W#LFivLO}|MRG$#?sb=%aitCN@BjAD<gV9j=Lq|n^ne~HO}DW*
zJl?BsqI5-%B6*{nE9pI~=1t0A+cR%84AN^t%SoSz%%r-0tKh|5+rMn?SYpqhRvZ;s
zT`FUZA+jh=M{I}g`i)Te$`sv>StX?K4?BFj^^y{~WPxsXw#riOm>!m;v+{EKUHK(d
zZB^Lg!o-v+#9{<*Itdf5|NCYM`A(JclLMer$DJIm!Fw>>Ah&@kH6gl)&qLHmsXGb@
zn6))LYUzkteKIlDTU9TX+j!IOYRGJ~*f)i+HGkuS$`X95+=&pD)hr&_ye8Z|wb2TP
zKNB#UJIgzW2X;ew3`W@WRHpaf5@a~0>KtbO35LerrdNT<ZubluJ}s~IHtBBT1B8V~
z%kvkidyl%_ew1l95#UZ7`P>BvdSRHg#{v-bj#s^bf=L#7GQG{x*Z!gJM*2YR6luuj
z(^4uJ!E_K+WR=--^2_!lQ-GKVOGmy}0fqsH#ig(iW#3|lMcU(@_L~h44|U*1RW$g1
zJ>@9klX)=)#J+GIjjU<Sv9Ake0w;wSc-kEn<*~hkC<qfMVjtQmvv9kl-5kyyYT9W1
zn@Q&U<A@S_=1ckef`be!hz=ag|Dd2TK!n$e`eB;50u8g!HeTo>qMUUBG@wywc#L2k
zy_%ycyZBZ$jaeY$8MdmPB^(MLLATn0!*>+H95BqZPak$^Nv8I8wwJM<7sk+rCfqVx
z@q1Q^vNckI1Ek6Vcowf|PXnuu#LEtu=lz~Q0P|1z9wZ4h+<EI)%zH-=y1lA_nc&UA
z`%)0T+DvnkuXdW+?oNzdA{Xs7xBEw+c@s-@>XKRlOa`HB0a+^qxcb_Y4;0^^+ebP3
zN_JJ^c&K%YwTP$%n&sC0VA+up6O}On9P2V5pU++XBrtjkR)=KwtKTeoObs%P)hJLe
zh5rCE*(X3a`>qcX<2l{{deu%tNLdYyX0p4zemLFOxT}<cj8l2NGz&j<(`f}lL%k@N
z%;p!a^MO~>b|`pJG0-UI)#(kD^y3xteg(=Ke}w)45jERSo{S<Nt^nf=3qXw(NM|W|
zR6n)1-LF5F)aSJ~o`BdX(1;h3^Eenge+oP|c#41=n5v&lZog6C+;>2M=>s#)c!&=N
zG_%hk#*BdXMmV4%Ez7CAsLF$brE;damIXOazHR_6qKMaa7Ccp@_zip*4rG%S?}|QN
zy9#X~;2|%AC=Lf~j%rgrV0xbo%n6@-ycKI(UpCf$XBTAPbsaq*xNkoP?LWwCi{q7O
zU%Jq4z6lI9taKm8KR8EnJ+Pak8wc5KK>XkHa>KrBSM9yWafVvn=KE<gduJS3yQhG5
z5}MF3@yTp^A)%&`I5hSzqnE#L8ZY{Z6H%Y?3Im&n8r;GKW<!inZU`@O_6%Is!=sY0
zW-%i(>tIF^en2!IpqCNFN)9Nc0qllE-lEHT;fA*U)rFRAx2ej%WX7a`dkEx{;%KMM
z{`@uerGjW<&7Qb`v|#OMA+c1~x6$1(K?=1m!gI_O{fpC%{z@%9Pv9Y;zc7%%4#!n2
zbmv~$-zOZ<ViW6>{}*m)s_qeiscUqMX};g0(U7ZbOCp2O1T%2a$Ohe`B&HZDLk?G}
zy_medUd_vnf2dn98w?3l6Bx?aB6UW90nM#V_~`~bjNwZv7CdBmX~6Qmne>%83e?;R
zzKe_NFJC6KfQ>-gh5FU+9e)3^SGQN&Lit07BD*1gMmpbN`i~3bN@pLP-8-(RKfWQ0
zU(8BQW<S?;o55h9gRN#G(vA;Hqn?u(kHknW=V$5abG)gUyJ2^=T2n)|T;@E5*o`Rn
zO~7)`1<gcB{vPn_`~WDeuY?4)fl<u3aPTunP`bQ!I3`wTVW?XVs-u<v2pX{M(rB;3
z?9<{?eDQW{v}ZP_US<t6eoFv!SrU_649Ob;l6YN?aTI7Is4a3nelrWbx1zH&=cEw@
zIfGb?JWjZ=v<qxCAx>vrMeV9huH91O`6e~FIdi!~9m*&3!Pe#N{GTLK3)aYXfe*}3
zpXIxvhmMMa(_2at>P<`c!T8CJQ^fgw7H;>ZRf){;fa|5kfrGsf^YqP6!8Al2-?WUz
z6Th1Q?@et)1*%IV5%73@RO<2Hib+NCps{Q%Si+%R_EmlzIGNrb%?NAiIS=9E8tb?1
ze_=oW(l`DIYIG1n(}lXKBkWksuKXrBiXhoDScdP2VBVgHb|oTp%g0Tz?mqS+V?gQ=
zxjyksk4dqYyAWoKN5GGr%^_;5q(DPIbrQ8NGN{hyTx6G8Xn9o9tIbl^($f(XCF#8F
zd?I5rc$>;J!p!QzEYSXafSa<@V}r*;4-Bq|OK@gv4|66MFA#I|k|1ff@bXh~S9b$~
zYtb6sM_~`rL(W)R4-SQ-k;b0k6xit<SuhF9LYR6DDYIF{!ZU}uP@c<$1*r!OK0LDP
zex>}iGt~tF0ya5!Gm;odm^qojhA3|4^zO(9u7|?RFFQbUF=?Hsd6E0ZGsO%)W1@n<
zI89nZl#?UnflTk*>B-t@A23;w>M!R)i4#|0s$;}{0Pf@F@5!1nUmDw(_IFZ{TL;}f
zbtrsOvQ#JGCM9qAIvc5!S)|zD9f*R42P$eq^&5>a6ItM;an$fARL%}~2yKHFs`~Tu
z@EaemNDG-#(;5uRVcFGVp_!2?q?UCZb5K?U1BX1_5||~2nBP1ws~0%D9?W?%HL^1+
zU6kC?NWlvJwKk`=y=iQoWr<Iy0j<YCSqEr#Gj(oEYSQY+^b!*v04Y{C>?K2=dIl7l
zb;(GeVLjrp0hND#dz%F66}Ig9*CS%b5x|;T0mS9*v~<ds4(}=YsQsQ)4iAwe1Cf}x
zO-UnZj(B<%5U%n%oYAcnzhefi2ng56gO7WgJAdu9Gq_0+yr9*7k3zI%ooV-ZtW>=s
z>MeFnMw)~VY8q(nzD<X?DZuIQc@7gJ{u7`~N{<1&W67VA-=5hE{u7ZUf0BLnZ=><o
zhxZ}L!@ALx$<*)LV^f4ODJ(SvVO(NWBy5oh-uulI#x%UX+MaYC&(0!7ds14Zhq0_N
z2|@^L@m6vOeqGd39pul=I9Q5*nzr@mMk&Gt2TI;`5do>~_|wcADa>-t0}6?8AyrmN
zfMOoPOV0@W;a+jXgkzeXzVPr6_sc*3hx0a8gzN-CzTI^uKxVkJnY9!FxQq9US}n3#
ze*|<i3%eFA7T}~iOuRgt=1k68c^$P)vWKu`wswW|JNL!KF%Hd1(|@hLxK+*qH4RK*
z4txNRJwntN8Y%hAvP?cj^DG4;HMv}X`&f=hmt<DH<Xm9?El-_}tCW-Y!3xVg$J#g&
zf;-~C)Wyx6ZgS=oZ_S-~xCi;!<8$gQ8`wm^+T%Y4(Y2hYYZ{^yDrSW5OF(@@8E*lv
zGJ0WkAm_6Gd{K*AUH?9Zw{JWNt-AI>3ooMgy(`*!qa}5O?rF#27p~}6=WTM;QXc*R
zb@yja#QcKG%g6#XZ|D<qlMha+Kx|gW2lsjPB&w5<imXF?4*rxwPaQfViP*tr^uu#Q
zH{Cn6)n@!&m2Q5$&ft1SOW&&Hn?`x%o(qxMYI9{uw=DEuN;Qcg^rdz{LXLMo{^SNS
zX8dG;DWe}e?M^?nU+`lpvcqKL>_c|%{b{VT`l2N~je1Crnnz6CbC9#G(_)F&FSyp;
z;D1QmEv~GO8X+?;{hK@vwDoJ=d!KF$&6~{rCR4X}LA}NgyRdfkQO>x__afTf?P=+X
zytK^=>gBzAFTk{D;JQB@{Lk$7E8hO4ugHr0N%KaZKC|6qUbI+ergbM+qz&VCJiFce
zq_DQeUph+2l=T2y%bvvZ5v6*>veNSyA*JQ2(%k_GvW>yFTQLDrE<;u_;v}*zgM{v+
zJ}J0I>Zltq7ND2r!brh;Xl8jK9+}Cs^#zCX+jppar6v5wI}?fTafEY>Zs)ZcE)2kk
zUd%8j{=Z{+x3Z+gZ?#dG12M?Igqy1(cYd+`xl*TY*$VwIEQ<YoaNd~x0}5`s9G+0*
z#MFyRhBVQy!3K{46upA1`nhT8jA?S-KMJenlP_OrY6Ywx@$aJggqKZ7W{9<@9CUBr
z5JM<(_8_JG_DM``#ohwm4j+02N?Xn?fgI?4-@*S2RN@J<7r?=QA8k2<jf70p3dgHs
z?(g#<3paG%LINI?^@m<1mK@;9^>E1X@*ljW@z3EFMAUgzNFx6$nFDzjHr%J2gnk#+
zC+f5>DQ**53DJDx#-A4IZwp{Q>jU<w7Wz<w)a;?PLUpO<-#~h9UMT5zC2DEJe=P#c
zB(Y<#>B;(a_ujnAcblP(Vo(B;FugO7?46HMq_IG|{q$^sV#lQ-aW51K4K820zsv>x
z7R<={DYW2aNPqd0wn=PAymr7D_JtV9OXkZy{rRGc$5C&4nDJeH-J(_+SEQ_2_HL_^
zA&y0(*oR%*1yBO{P~@JSON{UpEHVU8)_Y6tr<Z>PGUzG0n&&yZ68><Z(5k}6fU*v&
z>gcSoDv;=LURNaqgOp)2Q~d>3;!%A`g!S(}W+WI`tOJoAu_Dz$$DE=dL#_DMz(#})
z+*AW$Dbi|7v+aGS$3&u{PUpdP3l}~qZ?_qUK7ppua3SM_s7mb%n4LwOe=2-#R{Li|
z?p9kD@goS?+&<}ZSmBdGA@c<m!$FHynE8Wl4r&TK+Vv)W&qYl&E{SAab@p0zT%{XQ
zIbJ(GvD0d2r16{Ql{-+;wQoG5%BG$s35@ujyd9wi6Nf&N5VoHJxR>TLEn#zKnb3Kb
z^$KTOrBpKf*2{(rQOt>7#qL0uYkQ)PB8~K7hVTS1siCY24I@cTe)TzCEFFlcU)UW+
z_j2^uzah`QjOKfn!;=`fZT?)6BzfA@2zcOi$WzTcig!7HJQvZ|6ksDu%kPS9U9WB#
z)|W~1tll5dAfj$TLn4O&^i$BLXn?EErBpHdCUE&-nZ&=nqJA*_+*D8p@UWQQT$(I9
zt`W8_NiG)kI<ndR?fFdU<i8|`e-8`eqn`+4*QctRkOd=|aX@0zO&?|ySrzd)^B&IS
z)c$5R4s{1x_H8^UlK4O4?Jq`f<+U)IUKqL)ou;J!DZ{naxkmc=Ouh<@)0;9HkD(Xp
z@W18UlX7hK57SZ$-=Wq|6ZgOjlT4t9g4iIs!c<;eM?`Bke$kx+wP<?mu~rx^g@~X7
zw0q*$Oya;K4@fa*I#R%-?s(UEIvnmWkRbo~coc4Q?0#N`^Hgi^{47awM2jRufbxbL
z77JvBsAf!GORNQTaPIj(kdAR`;wr7IPIn2*y)B|=Aa|GRMIU}IJqGheb$wak<ev<+
z>p6xy*Xrj<-<GAf3)IVihbs3Q1TR=84BYSwuGS|G>{x!0VoNT{VDRkzjBb|gWA$@S
z8<38T5Zjfz2L(XPrK7EG0X@H44d2v4NtnMV8Q%Q-O&&5>Dy=wWq$cVFutvHhQ?>yl
zHWt-Na!94;4UBk26@nDeU$G*hek6jH$8kdgfv+l}!~2F+U_FO)RnA}H;4l~~mWcEm
z0ddJJl4tCG?qWR*z$f<W@a#H{&!U2zvGRaXpllrd=HH+5zZKFw#1MOq;YPpX^J@}c
zu7<81P180|7^^ELHeZZsz1O{ysT|$FV0<?~6KS--6@7F@MMUSDJ!6UMX2t6bbco0B
z=mK^=Cmr{-`O<!yb3<!+Ph8$OoK5V)JpN5TAn*)jrQ@0^%uBV4!#bKTdEK%78Z;z+
zbmr`TDg=Tn;J)UpX3vH_eFru&=S+x&&9?We`JE0i47Ff3O77%nTJ~BRXU0WVRjQLN
znfxYtXGgIZz5yu53Oqe`&k{xUM0O(pp4q*CdHdXpNfzW=U{+|LF1Hk@??t5M^`5-W
zJk|N3uG@5wDx+1%2jS7Cg;Y&ZpaiNF9n$yC>D2xVnvf*mi5@tgvrE+;ale8j8Ndwh
zrKodL{}AP}`fZ%{sb!x^s+r3yc<~R$77z)Lx##<qWMuQz+o(5w&o1>042{)*E{1)3
zB!bkBbOw|PvJVG!Pu+Ij8Kri~Y<0Y;2ByS)WQcXY(XpWofVVb(uH)%Ju1`J6QvW?2
zxpjInd8(>^eL%(IxNG@UnU;YZg=k*7x5vzj;>Jh7?ZZZWu_<r2SYw-cg*m@*Gq~;)
zqR$++iVFV&_}cL05<nLxQm(6YLPNHN=mw^YHo_PYU$vLvE*|>?`@+Ej88C;@PCx36
z&nzB+b!c;MN0xB7hw09c5}%pd*>fg|v(Zs?hAi%Mp39ztiqXbg6*A;Ii`LHkR!kw$
zQf<@tNE3v0=AGiEHza=gQ3jO<sr^H5>REOPW3*CFIyQ#phgtW9=vfb?yYP#&^ATs5
zh^XKMgPAm+sC{rZKk^{Z6f9D-+jwc^JVE-_gUw2`2CGZ*VWVs2*z4%)DB-iS`ag5L
zFUaxiz}`n0u^IaTR7V%@*@5mr;UV2L9ITwR00!X#{f%g$KG*E$>QvKj3jjt~Bem0*
zjJiWyFdN3Z|H<^p(*HHt%{ktitmjUacD9rTW|-#c$zoy$kU6=AGMn=s;YMC(zO~0A
z`@MtCYw@H4)y^3*3>e)xQSzvBJdy8FNb}rmn`geV>O9%GkS3{aH+_K<J9jt3qBYIx
zZSZdyu{9)0Vw^ABZ`695f2+ca{owVRZ#l1lT7A#4t}+qe*5XQ(XUgW#LZ@b}R%r1`
zi17i0M&1v)`Qo3@NH~N>qL3nD5@>t>j>enbXp2n)IcZ6PO#<#b!4NCo;jCsFEdn;#
zreUetgAx^v#sr};A}g#(Sicc`=;1;O11Fhxi?$prrIt=J{AAvC!KV^ox?Y-VHu#1H
ze)O$MeS2Tf+@SnJx7By+)*VuFoJ?qd)EF(y_jb9!u-de;Q<ig028$hfWuhg6ivgD8
zz^0*2F4)pf(vdu0QCIceigFRNCFmuIy(A}%6Zwdc_AH1N#Mr70DQIvp>{yV?3T*?X
zz{F3*g%l90T#0Z3#IV^t!d<*@HOr+&;dZ__U^(}JXs}Q{V>+2ZPLmeSEk^sr`SLq-
zwPl+FG`Z>aoELd4ljnon_hW2a;`^F@vYiJR7fIPHxvO+N;QNv%cf16|LK5Y}-+5yY
z69C^~*a1}JL>ig(2A2eq%n&R4nx@Cu<f1YxX>`JUygB#G0Q#l^KOAml%*{mk_o>gg
z%_o^BclU$~)Fn>rpk3qjSLHY^)b5rS-&p48p~~pLzN>h-2f}miLU^KTFGSC+U65U~
z@R8}P0recueK^@McaJAdZ}J)my79HvP}2fbSrVCNBmCx}FPM>cd|_c<Xoh@$ZTKE~
zhh^jQt$#?^U?LUl`+q$Pu2!jJOnb?#9n2~BWGOPXKTb?*btEm}bi5SNI^-D<rp2*D
z8~sJ7rwb({LkyH6y|Iu`<bmmw46R}rFb|av$#5;Yz45Z^IQG2J;6k!~y_JjCNKUAW
z6vL<i+-2yG=<@nkxu2qL9n)_0eB$tc6L|PBWUEh2g-o!zYtJl>cs{3Tn32y=)+aM_
z)DD;s(@xiJ9rmn55P<DAas$F${g(P#wKSV1tqJ(yr$3tUm}+xVM!ioT<631Mh&rz&
zA-A%Ch@ML*>gO0LWC+*3=nX6a-`EpT6OH)AX>lO0KpprfSRwHB!p-b+Xm{r8O%$|t
zUO2sW$I9R{!pgh{M+<W4kSxM<Ml=~bsLD9KEIps&o&JQn?Hu8S3m4Ixrl)7~?jzF*
zC}bqY>=hM%>nVow_CT|&pSSc?I9Pe%`1s|4@|!g>fXj?y!JI)Oj#tB0=UPhuu%)Q(
zmE9T84C@{>?gQ^REBNA;XTCOq(YUC^@4Am-Z@=gUEnGYV4GRC*&ia3<_;1<t-U0`y
z=pd`l9JH6fMsSsC;3>!A#1{T?vIz8|wWSrXhdPhrKL9{YbQv;MrRbw;+Jo`-a`*ss
za6d4P*2Qd?b#Z`keJ9gcmgT=)+y6e7q=G|24c9JVv5Howty(!&LQGx8n-FCJ245i*
zna4@A<w>;Fj=T2$nQ}P?a!;Z_?LbEB{Yz26+Cxb{RX`(!bs*%6Sw!7_yOaI6&I8B$
za5;CX!4ibWvsF`sDd$>`X1dT0Aa)O?v)4vSN8s@8(lVU)i<@R}x2x?P-{mREx-{Ej
zI;UGJN#i5&z8#rz-ou`E21Rps7aB8_LmMli%9O^>caKRf40h+{dK<)%g~Q$ml&_oE
zPNL1^hQ}Co68mXavA*Cg2eU7}S-qc`-W#gsgYO7_0zvVghOr6#54sSCD;To>bc_wX
z7A?E0-Fj~4foj>MgrDPOyXW7({}u&V2%!r5gr-#?-$=n@EU%n}e&yG%QdlN(vV%__
zN<}IG9HEPTCO-rtqIT!bM$%7N7R0vH0~1+vmCjSnlQxwlP%+p@6%Y$69EKslJl~G<
z>Z^ujWr_6XP5#I-UHRfvB=N{BWGx4IdA>;P?k6N;jd|j1s$4v&lXEZKXMj7=kV4cG
zegr(%*`5VK0;thPacVD)f%&BdG>%{~Lf36h&OTqOl~6-TmX?;oBugy;2EwN$FRy?V
zEr>a}G*y=PZK10EBA4DjL<R{_Ehj`B_?nbE5BlkvwFNdg*-ImVd&i3(%$mDNqY2=F
zjFOb0ps>D8RkuBS(Mqs2I8ikP$6V62UPj{`QLvWC9=oXU!^3NMJu^0aLM)lV=h^D+
zXfgV#k-{mfuIUmTu42u+Vf3?x#L(?8chR<n#-3N^9~u$QVpc~&Q!E9HSv}<6`Y{#O
zas1vaB-NqJKcs~4)Yo7ahs6siunGaU*c5h7<>Q_rKSTJlJ2qVKY?h&bT3HFF&7SoI
zoB2SHBeDd$e&z@=?fQt5^wWd%XON9+%Ji8P02TLk+@!X=)Ckt@wa;Ip^Gq7bYaIZZ
z#Qmukv0!?1{RS{(p9@+iWwGvg#bFCJ!u-}P+gh(z)MUX&YsC@85yQq`f3(>We_P+K
zMN9^ALTQSA7c04%1HTj-z<`YrgdAs|0eZs44$1h(5CE!0-6;XD=+6;-7bgl)qF!rW
zb2B-~f*)Uzuhs!TY9Daszklb<j>`G}v2k@QBQ=$rn}=l6P99_eO_0CMj0m7>L3+Yh
zrE+-SLX(_y@u7BR_MFa!)Xe8o)9@Rw-5R?oq)_i2z_J@XkKx+NIAcjINr&1_p#Mt#
z2pqbfQ2#Hkt-#k$!#}YEjL^hRRs(Dido589h^W)Kif2tz!Nm2}fxd_m-tTy*Znyly
zs3kE;GRQ-Pp6cwtl=;=^szCZhjkL@Opmt1>jC}I259c~q=m;Ac1Py3+;2Kd~`)JUl
zz!yXBJ%@|zfeL3#ssGl7_0cEjR*W2P1k1P$>&&z@d8a2&bYc~sn6JIsn#&FcA`5j5
zkzL4w`+vXlYJ{O;XRY&Q8j5A;*1BOj=2S;t!r75@igDlPirMV#G+BY8l1ve0pi?F*
zl^{F#5@~3__RjDJ(aBlw!B(0j(o`!OxVdIKJ~3_*-(giS<OeI~CSF=FEQ)4lGxY+R
z96<Zlel$$sm_aE-_7}<2#<ZY$L`*+f14^c4b9(9p*H&h0Le+C2yN6;^OCg<j!}!2R
zw~5isMh;lyf00ZbCL~zT1K_r$d^Y5?sj3M0=lsdsgBu?US9>wU2tj~>i*tJo&$$nb
zn}G};VkBQBBeT$&rvO9GgLy3~E<rnJc%6D}04-?=^b|IR+^~7Kap$3UO&iiO*R0|;
zlCtpG<jd#;{~Do0R^Xr7sd6sW)6L=-?OOXzs0d(2OWASimh-E`X$*NA_M>NsN*{w^
zl8pn70eCoegVAUYie%t-Y)%_Qta%2wfiA+%Iqw|c{J_>ty&*{MNSZ=x3fc}8M9!TV
z@UDsUrcNF}MRzdQ{BqeS$*p42UwIn?6r<yN!NT$zXpj|GaC-;`XoK$&YIv0l6BSQ(
z;FG$R3oSC&;S`fJQR<^4RnJToe*D{P^Fqp+I=L|UBqvHiGo7D!Bzi3#42x>RvK}7m
zgpho3mCF;~Fd!YFdmq)bM$W6zDR+7lg#EJi(CZ?vR@*9Q@55wPZaZdd6u&~8h<oJ7
zEDf9g*R;643E}d9DUBVQqHj=}8b=V3`IS-|5+IJ?{f^{Ve#T~2IkUwiRY@T-kIg9!
zer8XSDmq7w^)g5{w%(X43k#2_kd2b^kCdfVWcRQHJ|pa}Yc;#awlT|jY(6tOu$el2
zV(-y^P>sd?qiPJ>)CzQcg>(*`#ASoB(^;u$PgqZ|eLqkW$q@$xrS!lqXPH4jGB84b
z1YDUtX+vQ4oAePVmoufb3<YY|6{G=*_4O(ZYP6>aIa3LU6E}k$yngfAR9zidB8eOA
zVpJe&iBfNBpGYPVxd|rnmIA=^bn{%Tx5L;~iRwu4(R^B^qoeXec~VI*o$qHdN5y)4
zPGjgDkb;K#4^$`xI)=2&s{r`Wz9hUuxzLq%l?~wEbJy}t(`&bU2Me!tpO(^j=$&0`
zxOeNeMWZkCh5HxYkjfM0Sn%ZwS-w&FeB^wU`Mz0L>8&i1Gx4u`sII*@L-~T_KK+Zc
z<kB6!q%l#i<frO=7B+9Ltc|U8jqP@JJWg{hl06)^8$XnC4NgI|dez)-lEfPTv)3C1
ziY!j2B3i?tvLFS1(x_T5o4Fy`4OX9wGEB8_?&M{0Tu16AqB~L~iZ6e;BUyk$`ujg3
z3VV_}?4;VE2j&$Yk3e!AwSP|a9*i?3JkOCA_zW5XSG@{}ePbW6;$Sp3u`+2v5stst
zT*O@NEAl8e#jdWZppAWT?5pVA+)XD9Z!a@gi@!-aK|mqLfkp3|+p9%V*dM?W`}{ed
zbq#J~qtoMevmCe9r9RXlU4{Wncyo&%tf@6yvAX-IDV}HpCz5_}*9j<y9u9_mOFBsr
zbC_Ft{shRvC(Tb~r<6Fn<z~z)PWK|x>3j^AQ|-z+#aD~|D34tDswQ;wsiEF%keAf4
z6XHIENI>oCP0`H=j>ap`S4x$iHz!!xo|~Q2J#+5i@#GoX#%Xa>yxNgn|4Z8?oe{w{
z_W&gD@{;EO9upk2!2RT3|Dat8ap}ntQu+C2UkkhK33U6p9+8XG_$H4jP!YlX{9>*v
zHzrB$pK<2jz|S#1EmNYfFiYI{cHCQ_T38z{U6PA1^%<mcY!m7}`8<HfCAU<<yO1}m
z(^_>0?F|&F*;~w90bTU?TU*L|ZTqib4~WEu1Iq`>Khcw{_!~!rwEXZRF)}VxjB3_~
zt2>k*U<cgMVP*AlBv{7wLRw0PAh$EAjE7dduWQZBHI~tBjq`{CD<3O*O)Zx3lby6e
zMVfT`8Pa4k#SB7p)pd6j(JFJT$$7!5h{HD=MD}F_={(7Aw~bzCi1}RDI3Y_+IZKP+
z6pr{qFlhp;rm}Oh=vau9ye{zE2S7R0V~K*AD~A$oC3Xl^MU7{OoR)>+{cKG+7h!X^
zv03(^38m&$+8RbVTw&R08CCn-Wl1?Qto<R$tuVZ%PP3zug&v?*r|`-ysdUooS@%wu
zHs-~;rG2$Fvm0OPg7#m8!{zKee-BMhxTv##Ve}cv%rMB&f~f;Kodp!4W6WpnTY&6u
zBX|FUaic$}|CD~h$fX`++QG6`{pbp$zzq!au>;%y0jlE`wy-0wK(~*g(;{A;L~l<E
zYP=EkSsPo~>#o&2h6>Dqm3c_~%<F4G8cQ2(pW@XvKlOgyZE>3Tn+W(bHU#$|chm!T
zts`;g#mCzeqA`Y^d)=o2>73~q`8T==(1bGnqrAnwj4LgI`M1?bQd5lc&242FoLE#*
zLimlV1D_2K2RgY(QyV8}D`{6gGvX7KWfb@v$Ujy`Rl7JA_zhHh<EI!>s^x_G>Cbit
z52O@J=zZ@U=ObGe++CO=Qlci^80{_KL}FEM32F3%X$n}9A*2;#oL7LNZFKCDSLg=S
z;DV38EqrtA`pyC2hR++@Ra497@9hr1lfk6}bVtLu*f&u7`>!$dK}e^&<f2Fs*UJK7
zJbNq4-&uJQ9E=9QXU&K?C!@J|#gwbN{hXf*S-<cfziRYeV!Uy6=Hyn!G$sSoOx7x`
zsd05VbWdp{r+i;qw@z0+#sLPKgKL*b)!2{iA)n(RYU~HyAAoW*4@SpNtuh2uIOrW`
zM?f_y2vkU*lkf&sdlk289lA_?vs;VQ$2^FuNFAwK5JBHx?gySvkJGJjb5I?C+DGS#
ziZT2CbEhHn75C-qs=x-8o2OsjU|qA~=TsI@|J!Q|9@^AN06+|+@==y1Oy{1uO%`XU
zX3H<s*0-D|7hETzPaqT^i1I`16Yqy9f6!Fq_ZJ_e^2BC}a1HPKywq$xDcpzyn}voa
zwU2js8A=&81{;Hi;i;aa2)y>B*9FZK4WHxDP!r?gL%rxqljBkioh#CsCUIhmL`@Me
zW0H(~8@@7k46O$1#yov_B(Zg4A0|Tk`LIr*8XETHILTbYvWj<4L5;L3#>DMPVVGi1
zQ9z_v(_QLA;WG5`JX1h-wX=`)?jG5N@5_%nK+(GzZPaT=aOE(9kI=7%TbD@E0oV%B
zlJxK0(f`v6;GrqWGRxB+U@^gcS;G@pJhU}rl{cZXY5_cml+8$4?~fGkDO;GlCzDr-
z<mT@_LxhT|;>RZXj?RCptsma{)O$RqGSB*7j?{P#svv6eQ6BP4K77P91grrFEkYJg
zzft7~A|nD5%#XXWR^MEX_EFOpsrt*m9hVDn4KnFPerq&m^%}V;OFKx;8Z4b#+#ba7
zvHP?kcfh&iK3OIVarj-U-W}bR^xl3y=|xI5jdW8kBc4R3FI1=;KF$hfbRZ8Y23E46
zWZIT9ybR-p=z_=STdz$mKc``}Lq<^5^d!vqm;k{taTv=^oBX;;b*X@eX+*j9ELTX?
z52M$sBl&YlB}F=z8rrA6rnahJGsfyd!Z;apfJ#9~;L@fb?yU8U1SEkMCxP$l|4R~}
zCBZ`)0mSr&{D|-EF^^_eg=*G9&i5=x87oKd26LtZ;hR0h6vLwVM}BXs{PU_&<)KA~
z36E0%s%xdmz)#|DD`#l}f0*^7pvzkQ{wLi{t`)7;=RAi)3(d+j;*N=QL;HC*E~FLQ
zYo>mG^_loJU)y`nsk4Qg5gY?jk4F#$$lD}gtX4@H?B0Hk@O?VZ`4X+!oia51m2*jK
zNCp>nd#gYNrc_VAQ9gb$&}=RnCSAA`Jq%~@eiSJ@AFm;q8lxdu?$$9PxB2pE7oyr)
z2D<hHe<xAMV7gj!wT_4@W-{{OR7hOW&+z*9+wgyK;RS%aH1ne9;~S|(?Wx>ayQ$;|
z{FDh`1iw@J!{LhRMB+0Y6~Qi^<W_A$EvBu(+8SMf$4R)8Mf|e&faz34v6E1_)IN7B
zqXt#;i6i%mf_DK$^FEb!`0vr4hi3gDYQ46w?o6MrA}s0DnKUU|v_z&}p6tntq~b?Z
zo>X5r*=zEP2Fhhh#Qls7_O#5ELcv>8nxt&ELi?ZxR~!HM`XG1f38pNUU7_x<DZ0`{
zZnT4ojI%D98cb+^*p!=o2cZx};g|c_yz3+uI05bUKA!y<__3GPUnbOKy8}FGa#s>~
zv!nnUk|){6zsrypULgl*=i;2oBlv%LM>xpu(nC_SH!sUFBJ{rc@)F!v^7hj|b3j=5
zZCYet13MM))IZAT2lD=RF%cmuPkh*CCE3k6=^j{PNF{t*dO~-9)e2@>uuxZWo;Nx(
z-{uA`Hd>>USGI@FW@X6x$H<Yl&urmzT3E_Q@~ID$e}u93G?$?zR!x#3cS>IM2B#|y
zBFiwa#z45B)rIoN=2i~nzO#oVKKH+e_}6!R?q=qk@PyO4EMM9$=!E%;t+Yj?%b045
zjJe+b&Wbx9-ZOdp`}z8G%S#MGuXX(CO2?2Q_28(e{c<^kLcnb?zuw$F_^V>lQRb*;
z>6FgC%J?6Z)1R>dYO01%sE8UvmX$<&{~A*a!X%Z+A^5tzGLt<amh4XD)m-tsT@~F-
zdKVBaqgnG;8I^4gXNwO7_^m9j?i#DHj*+;-@cMd_X@$LXPX^~T+eMPQ4`hqo2aRn`
z-+X)%?YSF~o}_JEXwGeG%5xjf;+QpX%RGySK+^Zx9rolTZNlxTm#UKa&mTynF?w!u
zM3*-V=OyYkn)^`A3w@F>%~w25?r37(;454*^YKK>mL2Rbn%F!cVR;i>tMal`&U?8O
zJv+pQa|a;iF@=_a^Mm_bP8UHQPJXAse@H|0XE-SNA-yG-&%7;Q`md*P<!4Fx8WH}r
zvX<(v@=UxoB+o>$BHW0Eb+X#SuULM{C388#>ae6BY&LoI6Jq!$^fy;DSfx4~ff1hI
zY*vVKQKPlbR%v6Lar%$){32VML%Q%zu?>u%2(48=C5wBDy51E*O^2-klK}SDNs*V>
zNox_T9;u3`3_2^DJ&PIwav~9h8#b7aQJZ99jJp-Z$N6C}B1)0c(ZtTn6DPNAR(D%M
zqS8ZDZhU6mF~oWQFpqzEZ?A$K)VC@(KpIr3@|&+5Lu$q(HopY#(f;&f+@Gi6-%to;
zlP7F_7pVT{i2f#{7`K3wnjE0|?#yF1;>a;VccOK2+QChtDI((`Oow1aK3fuLs`df#
z;-HCmYw{*-eh6*AaJZXk>Gz?Lu^G$f=fbrq$Gv9D<jRic%xK?5GI2(+hsu6STD3DA
zuSd)C;XI6nbNT2~9Zln>lnOuR_LL8$fSof~W6Z_!{xh%ioo72XLK@4<jtfH2<DT<)
z7lzp}-i?8I&Zk>`Q;K=PFcub5_>;X1Tej7gPWeH`5^$E|#xF9@&zVInOp}6jGp!;2
zUw{8?a?ykY7uu)6<NU<D1hMBEsQJ(86Of4?P~fMpJv9qfM8YiF%zC2L{tt8S8P`<W
z_xsYMNC`!y6A%O&(yNew2<VN7hy|pB1SAwGK@dU_q)2se=n#rhMmo~FgrZa>(mM!9
zhtMH-uI0?_%=4Uk`|KCz?9YDb%nN0))^+{M@Av(|tCZ-cm=CHo`**X`vZu?D2-gb6
znRbstYK8gzjx$}@xTXa6{ysB}+&3%U`t3#8F3RwubD`Q<N&06BIblZdj__)O+dV8Q
zQOvg~ukun?4jDvSGa>APVb5i%(4K6TI(8TxKH)3I7Mp2oTB0uNMY1hNc;L$HZyr&8
zh&B@(uN@jk<cH)Ouz{hTxnXTsIFT;v*?S&#Q`XLP%ZCR)fB7oRad)YpO|d~<ti?b6
z>pzZ#pRJbek72Eta{gI=oj;Q-Snui+>u@qkWV^>{Ysje))NYjw`)$vCd`A^KSns|V
z!pi<@j8fLJ=VcYLl*OXwLkSXHd3=|%Pv#W&fdJEbi7r5bb6~t?^<#|=Sb|m(I9%~n
z13fHWrDI%KM4E8x^A0_Vd_zNgf24$L^~_{YM#%=B=Ho!tDZ=a5Nk{qdEE{3gsPKvM
zOl(natb9OFF6=Uzw3!u13bq`$KYG+PvxChox`6T7d1Pv__T7Lp@f1a;6F3-7mbLMy
z^HU`#>`xRg|Ewd-Ueg8uFul9pkK6YD-DLjtObo6<x=Fe$b*m90>ebyd9~;<#S=-Ft
zk?gw)63^yAIiv7m$cd-WK(mfQ%Izw=$_?egO0Xu7)~jJV_jg)Q#s}QpHxRd5(}r<I
z`_UkJ0q(XD4}3F`lO8V=9qjqmXSstMHLeijU(l=-KKt-s6p_2=?h0!GNzi??3WSX{
zo&@<tLtsC2D&;?5?f}=SW)@!+Sq9J`q3e-%W-NAU=9B!orP2g3SXaySoZ)f(<&N#t
z6z^^|#wG9f-zvP`(_Nd*zn1zzE4}i7p@jJN6WC^MTh%BDvO53Ex}wky8hp~cint%+
zgyyVS42o<}vA%Y~kKLD7GSs4m=fzX|8qDTPEnaylhJ_Ex*_L0zOZjS<E9vlqaL?Lp
z9{eBNYRU&ta)El%ryOm&&hHJ8o$|K*GWv@mEw!gg+3Z%{O5MDh)^MYli!Za@X@T26
z0)DHIj*7;ubE&8M;<fqRV|2WYUGr?(Yn37v%LSb@O;!0HxN;T`qL#WJ_;4E?r6S*(
zgcYE_kVXouUEFX4iGF>ru)FO8OOORsE%lsAhZXeGA8r=!{Qc9Hu|Pk4xXr>Ze|m9S
z2=*b@bMfqSndF4}T8gP3%vW6Nwz-B-9YJD#gX0~WmNpHPDKMbl-dZRt8^AT+v0Z)%
z{j}0_X?}zI9lPFrwNdZwa==-V%x%2EQ2NQZJ^XBDmqVD$`j^tSqR6d~_?TPnnNxu|
zUo!{e&<h#;LCyCZO&%9=w+j@xl+eR!RxGiW#%!x<R;ch)w)S~?*(sqhX7nXYQAzBO
zBhu0!<1#xYHP&e*Z7ir(USuyxj~l(TEoK|k#HBvRcC&Ss4H)d8u5xyzq`m9!I$`if
zoq)*9`E{LO^W$dx{U-g#iVTMTVwT%Aiaw+`#PFc0uPB{fZ7h`ux00nEMGVbb`2e%r
z_u?hzuM)Q?1>lJ#q;SEOfv3VLE(w}G&ZRqh@ZIchsrN?vhh_v@!t+{=q+2StwI5`y
z*#Wa$PJ0VvmKz9u`v$>tq-b*M0}H{suCy@P5jLA}c*Pj+)!j9uOjZHQ>#~M#8GI~0
zi7sr2jGC*ve@+oQ7zAU(kzNe76r<qyI7goyOKn?$x3LmPxt@3W9p9h|`zEr^4na!=
zH5%aJi?f1SjEj^B&c8el!+4<Lno&CR;{@By$H#=;rZ&;}DxEe}VWJ!%+oKu^ZfxO7
z=Im0bZHbKZ(l8DFJ@?s7A7b8M_Y?TgGN=Yn3!kGa>TELD{oRQlEa5B92UFCSyZnre
z!g3Du*o*oH7V5nE^%4%6GhcZ*ipvaCB%=CSRF(C0FB|hGhf4)=enOU2!T24~EDx<!
zo|#R7*aQN_EmYuv^Z0%o3p(3V*m>Ded;i*R=4G;#^Djybxcxi5766<dS(1^R*W*(v
zDSHVdDg#aakz+>!a!!Z>jGaW_hO)fSL-=vlh0*#QI!K>3)06%AaQ-?2e||I2JnkiQ
zFVy(WZhvOrBcp)%Ha-~Tyh6uA4k(RMO(Ictw+Vfk3~s26Nfu@_pDbQ}X@hC)i_^Bq
zy?p=mj_sGzDm9oqb%vtcZ%tuwt4gTa)upWjj;EVwiww96_U2R8_Ux@HA17@cD)iGq
zO9Q(&wR3ias(D&&YyM~dVyG(G4K_Fa&aaQ-Q<6tZ_M81~B|<QZYO>?0B7J*fLXc%b
zy6q=6C73XL5LI99!@ykwc#d(G7IH2o9L)!N_hEecwu9s?VYJeTu{!d+(K_-cg=Pz9
z&V1tWb{~BDbIK^b2z3<(LDI+m{#yRaL-_L#91;}D6rJxsM{wIPk8QcoGp_D^il3wt
z7wR&Ly-e2XC!k5jE1^;6yESpjCWHElooI_$P%J3a*h@yJ><QWEmzrX>X9+QjW<J;s
zJoj|MKEob-JNk0tgS2{UjM%ndU`l?_mI&(e>8R+N9IW%G_%lAKXpv_Bi2~K;2yI_^
z4n<inCYVvbbWFVb!OkN4{R{YAN}F6+c<|P@f(rOf>S*1T9o91EKE3zC<W5+WMWH}T
zKWR`9?VM{!Ky^x1_|5K)s8I5_+y|e<3ZfbqZhw9jSm?7@vP^jK>(AmxTQ&9hkt%2T
z$gnPCcM0^O9$BN*Z=zFK%cu^7lkI7|yl=+jy&&k!TR2%@Imun<)~03FqJi8>cO!jx
zyJ#5g?n;{Z%zn89`(mhh(y~y_qRm{tUCF)!Ln>a{R%fz^s8W6~=X#X-%`O--5Jj`m
zLwJ(0s}$fXIUQv53<oC*5hi=4-wzMC`=S#reTeZ{VAILpPA`K|<4ED=D3yn**;+n!
zUF!D4`N)BB(x+Avf@a)SNoI<y=YqAib^zbD)x8lpY_OHxJy142R}sAG;?f0fi@lNJ
z#h<@)Gim6J_%xC5$FH9UgsF5f`{wMy8QShk1DdHd3ZV$X467YO#c@1vSul$D8d<QQ
z_GoiMJ_U^`Za0d|`Bu&2Z`J3cdH0)ovlZy@64^S+UdP>>*T0Fh9PKWT&-pN08{2YR
z_^xK^gmD_0e_oi$gx(3$uFdXc<Gvy0c~OfwyV=Z=LH)8+kjgu{KV(r02LA4FD!2Y;
zs=SAN=HUalcqOz=0dZ4edPntCsd4rFrA-XHuMk;M`eNwo5YByJhqxx#vAZ|vit8W<
zwPGtHWnS3#ZqQ{}LUkmbCk8ZCO)xP-7{$MTcoZ*mRxCzkn*ID{{!;}dsFA9fYM?(f
zdM~hs8bw`04Orx|S5Jo|QB1GJFJ3H2?NZb1$Uu~~`8hUl@zyIBT3XukDc&yV#AMdX
zw3XjC4&usPuQnyF3k<B6z}lqnmq$d<8$9aEpt{uTa8X7jx8P1BFcvX!&!q+jnQ7RP
zz4Kr`<ZI#br7)oSWUVB1RF&y8Ce$>f4DQV#I5WCPW4~w<(A^6n%IhDVvNaQo;K9sU
zEYONi`A1GBa2&W_>P}Ba?6w`0NXm-I_3$Ke+Z<<g(FdOg6SBrm`R~sI1AQL3Y6;K&
z{Y3tKivIfp`f+rI{Q%sge+JV|P=$4o*HDiU)mUHB_-ZqeNxjz=-L0jgu1=zLwUw~F
z>|H@Q^y#G{9~`snh$vZCUp~YanWEi41))*Y5`;`@^y~bfZtN#q?Z<+K2xiARYpYDz
zD@L81MRJIP>ic$tV%f!WT3Fw6BIAh-fg{0Emo}^FXnN*aLMBsN5(UMNnBzqX5dq#l
zs)F4Q`ecJjt9&H|yOtuk_VXsyPJ>z0BB$YvykswYg<|UwK9zgX&WD6y^uBiQf~EKW
zXYJ&_-nO?-P;<GHF$JHiAzC+hG(=o-qUKH?m5~}Dw>a6~*XW>B<qgIQLA_>tGH3dt
zGAcAtOAH(P&qhgodqYVdGO|XFTQ2JnFBT5=4=U!K#O9)nzrUNUYJ7KmB$0q*Mloq)
zsc9@!s9D0m2qO6y!mnKT=)r>fa!au+PX54h&hK$XC~2{zM63^g&CDskTX!UHz9dlD
zAVVtJdNjQZ?f9<I@sgC&%qLUo-<20!;({EvBxhr}Z6ewkrH_RuTm-jf5Zt7UkLusQ
z?+!(Zy-}M!0qw@0XT^WoT4{tqbMMr5RmEY3?Y>FRcDvwA<Vj?G3;{Am<Q@!LLma3j
z+9YyU>KgLdDn87N*tLXnnGsFCKKeDa9Z!&(P5q{N+z$Q+=WQ?GKVbM<YQ^=k;E~~c
z$I+HW-vnW!B#ybd;N+qKyUgdgO~Tk~d>_O65mU#GUq&-!@{ao7?jA`m+pdz8MZcuH
zer&BnEmnS-@^vIEFPpb*ejsPME?2bZST!kh=s4PTsw7tSU^;`kbbSz~6lf7f3U6}Y
z*#m>3>jFJIa3F5(?DV)J7Q(mYOC!BA(iEvH^z*ZzrEsC~(<M1bKVfdW`OB*_4*g@k
z&_2fgd!tB_mjS)CuK<lomx@HG5Vt$oe*7dQb10Lk%4r6V{X9R|OD$hs^ZWw?+|Vp*
zP_ohV*2_>P9o5qe=qW%b^31q&<%V&$_W+9*A%-T6H!&mMN{kp>BrG#huaKK1DjlCw
zcuqAnri#0~4n6xIY^MQ1wqPrjBD20v$W!I1nvn1J*lrCh)L?%|D}G2;2(#bi#Sfc~
zrr;|?ya+0V+>2a^&n`v0JyE*?CcJ@j0&d}wvRxuc$B(-hX{`S?4{X?|l2$%XX+X=-
zcNw-HVLC<k^PqgzFrs310*vFrqpf3qc~*4X{j1S9?;ravN&$>-trNWZBN>@|mB>Dv
z%bn!n?;2QZI@!<?mRgM}<3rF1;nSN$3pI6zm8Qd;95LD!EVj&=5#V>29G>(btlSOv
zee$SHpe1EtNu65)k9NTggqU~M*?ucPkKf+f_tj!d;cwvL3#Mtv+~?lDg_lsWNfw)*
zHGCWJL}4JiSxuvWm_tV%=oRQY*t@@7evgiqY}AV{6BhJ<5Ua1|<9gtU*cY6cEYI>Z
zFFQrzq?J<Wz<gJPBey;T*_3!7Chy#eo0c3F3+V$+;aOmdvxCq<v%mvvMT9~nD?u3P
zoczBAy?<UHg~j~h!}Z*G7c0L0YK6p4dlC89le(xz!MJ+h(Jh92#>xHcQmnbtr>_hX
zfKVQ%le;ye^v>@q?3yA(D8B_axA(FALYm~e)$WCDCaep#nGe9>&lnq5ObnZ9WxF-C
zEe3I(&fixCNxw(k+ib7wSv^{-&U(=A;~1Z#{o{%$;kKZ*lM`KgCe6UJQDm5rZO@2W
z*hG_^f>B(aLiFUA!%$xML7WlBivS4a0}GQL3tVIOrt224w_r%+`N+sgg(b&)Yhjwv
zu~M93N(i@2*;%RtG0>br=SqhA^*av2wjol7YNSxp!}Qt^VW9+tL$%_hvO17zArtA*
zVVR&UH~P*1DEYXK@(d~-n-kxflK!Y9-aO*2+Gqo`)ZV3cy*E3B6YV}50RciM(D03q
zJiUO+5Bu{@YOoU&e*s2+o{>51Kaa0J|JTo<&!1;l_K%M5<UzWm!=<L3qsLS|U#Y~K
z1byj!ndbaBnHqVmY{YEa#j>YAs7<i(&zzo_5Im(@e#(v715;bPWZMHj9jVa5D%RbI
zW!EzdJ;HymEDRzQSuTcdwS8?Clp=kCwjZPJs3oTT)P7RC^V2J7@yEet^;=6$UNT?l
zil1^=kRVen_3!@szPaW~PP<;$wxzQo<{Hg4+XTwGN;H+DyN8tex5e`FeMX~cv=}Td
z)&`p|kCZ#{d8LGU=W+&n+p44v&KzyhWo`AF)}IUD?@~B7>Gt~prH2hTy2^l7KSsVu
zTN+9ucsuSy^L;Gh9>__Llrgj`QYZS-?A}j3oEU40T*1D#e&g;2GB6B?AbG5;^6M!C
zx%^pXImF>4494Yo;hHKWB}#zcJ$qw2b^T)%z4VV_S02SV#w`PKkOSmmH_z{a{z92m
z?5gK*sWqt=cSJbs96!{C2*4PprdLPa-uu0{vhFN+HQ1!e!N2deCI%@{nT*CkM&C|V
zn)UHfiJ}<Kh6mGFAeAcBCrYi6R(<OCz(2{&aTebjad~B+X7i1hJ(Q@Prb?&>E}aCy
z2>J9g1O~?QEzvg>b0%YteMNpJA`^iSuhWVMjC?g7VBI^O=Mkv&bT;XrffBYfB?i=o
zvzP);<MN}=v2_HcdDSDA!FZ83NdRNisjJB=!`wceyi+r$+zBuo3Tq4uQ>$!OxxATd
zJGQL<8n}NG5dUL5JhMFU;V-8u%v3Ru?{CtoFCBCtfjgv{vsM~HiEF1%CVG3}6((4A
zi+|?<T2Qx*Ty>_Rj@^@<g)Kq)TDSu);uyO%e5&VLL08x{*EF|Uq!-xicQ5hfX(^S}
z)knPb+^6W-{cUJgQgC|}XWeo6IZ}D##O;>nPu>02)AS8J8N+Cv%<}X7G2WoG1OeT3
zeN<luDazz(itcDATa4)FRtLR^h|*+gI@(5QWa}bbJ<a5(xo;n2(|l_oO|15#w0Wu5
zL88Yv4h-4>aYVX$saQ*)Z-(k~Bg}iv|HH9@2Hgy=>re`q^y<_8>Y;80)g<?w>haOA
zgsVhwS7gS^IgY(%38EtEeBadR1o~>lC}Ksca{eBKk1{hHnj+%9d$|DNTLgXLLdw}f
z5c?4ER*(B=5BI$iT7j7+J|N9R>+osf=V`WfC^?sU)L^`{{x$2*#k|rCuv=d)Uls~4
z2OI}c^J|v%G20=ty*kL+Jt#u;W%ZfYN{a^F2|cEl2Vhhwkbb?iviJL(&ylrnkMrgy
z&GOe=lqP_vAlvP|U1D5D1kSIzSzv$WE39-MY&Yp!0y6kZiNO5uhr&#hTGiJ5x{cp&
z;{MnuEx_jO0S)uu8>hvtAjD=G`Qc@2oyzSYbfR3{dUz+M-?*Rn@UIn2ud3OrdTwoN
z<cEJ;l>hAv`cXe$gt91H9)}p;YFV}dl1p+}D&NoyLxV&^<b*%>1cUA|xGY$xowQ)K
zM&F7Y8YulzIrQ}sa{;rp&ta59VP$`vi4o<2r8f4UB>&AAjl^a;P05tOa-?xRsVI$m
zG0c9*BKi^g*$@Xj7qgPmyF_+=FN%xicrZBQP0NJ&%3#{Q2Km3`$PG%??@MW}Y9`DY
z&>#^9dac;s>i@-gXvW-{3ctc|<q|2R#7DRK2w^nEyo2&Vc^$0{f}C9^ZGU7N#^*5D
zQzT+nrhQ-r5^KC60d?-YbMA)^z~VKiaxZRCav}CAvo61<O4~#q^EK&Eb{Arw&BH$e
znA{pg&y8Gt-zoOdl~#J@O8V|*7ScZ6Eu||ZMk95yESd&6P`|QM>VLISR9fq|Bkw>K
zo1<yuQ3lEAmhl`LZuhE73#T<9Gt(5^pwLM1zFiZ!|H;WD_=b+hN4=K8SQJN@m~*CD
zs=;SbL_1iH76vKjLRK=l$DyA;i%+0Z_uZ1-G`GV4yerrQ#TUlIg;#iVeqPxBspC^M
zdkXF19a0T$QIAkm(AAMWr}SO%MXE5=P$^J7X4qseoxW^iDI8fvD4}*+h|&?Ih6IiH
zp>e+1kLC@LIzGCFX_vN2qg%NeTRGQsgW|__mrV%{1$}0+{qhSInr+s|;5ld5c57s$
z)M?B}oE4YPkSkY?@e7#+N)K@qOoc)*9cM&TC=8O+fV=q$!;<w640>f0@Dlwpouac{
zI$?XeUqV!KKQwLoWpgif)G_ZUm2oMtN$kgS&a&u;lJeu1L%Lwho-b8g4oQ<z8azv9
z#5}Hp#JR=PoBvt?V4j$>68*^J;)^D%%HYbdzb*~1IS&p#a3Zv%&a-OcdKz7(gii(>
z>3B`C4eL;DAxnvVFfRb3$}Op~Muw>np@8#IMnxEi%^22MB;YQcj>iRooeX*+T9Kse
zMBfbwU-R#qR&MnfwvN*3wL&zY&f&4`vLWeIwatpz1T?kXR(d@SIClVVXR%^9ZeQ-T
zet{|d0nl3wR{<1K=PXO4`_&R4_Td`qglGqVfi&m32tftstzYjdOtKk3P7ory&pP{Z
z$)Y9Gx99csPzdsUn3W<G*T$(!MgujoG6kx<Q9w~t!I<^krFp!dwC};&w@uwNIPZma
ziB_K8Nv8qxAejl>_H^~op>7=swasDRJ8GHyD0)Sy{o0?eNe6?U)q3@(4!5|aU-n1O
z8u+0^iis%${T5NHwH4D8%GF&Klsmah<#P_IE(%J4`w`EioYv?UjHb5QWqMHIte;!N
zr{5lA=*8-LvSZA&vG}80&Y9mVQ%IpXTm3R2PRJy`2LnoxrMZ(;0j|oytt;MPc?WN!
z<*bS(Ua`HUvMr=T=b)d6(heMJ<bZ{YQqt8m=m+rx6(N$QY>6Loh(>JIK0dT<VC61w
zYU=p<aYc#k{+ky)Tf0&S#?k(r*xUU_eHSzHWvfVw$_7C^h)F!7bZdSa6_?4o0uU3e
zMS4eve*vA3N!8Ji`koQDlk-(K@+qMq4-PW)jEF!Bl*YuFrN&qBHO7$3WTPOsIM;;L
zzVypI(U3qLiuJt<*QBEiQHJ{g5eS6<??|s1!C{^6gj#{tLJ=zt@UZ-0eEg3dQ+6bd
zZDu+>WJ4HYKn9S3?TDyo(>CFl&XvjD+e<oruXLmTC49P?qh2U11zg@T1@6rk80_aD
zi<0u3@A_EtZ?>HEz?U3(aPXyo&hNF(yU7w$Y&vtC&w8jS5C5B<IY>w5S^ZKuK1(ZA
zAXdj|{d)cba0KO8et|q9Ft3PEA3eaeK#r=-Ax@3BZIuJd`?Q5XU|#*veliI-*+1N3
zz0J;i=FdJ+_<2L|{;19E#Ot^Ihogk`&!eO=Jq+O&82bf^HE4*^xkZTiGuB`^`o0Q-
zn*|49<Cwgnn0L<qd10R{$3fJbNlrJ6)*C-Gk1zU5)yJ`n2jAaKFRzb}La(GMpw!Rw
zsGN0gkxKcbGUD6OK<|e5B6UM{)U`J-_OO%(7%!&*LVUH}r(`ma+s)kT1uC5zK5x4A
z=x9Uid-3G4-TQz{bn>^WB^OOuom{AeewXRf&%g>p^`v$3RVmIZG%#jX>Y4LjAx9sj
zgnB%KU&{bOcN*Xk?SL~iay|t(E|B-V>T>G7%|Pq{BJBGb%(xyHOmY<FQ~j$jrt!2H
z0JWPx8Bchav{+DI=O-`8LT;D-fRy6ZXA4Y@D;--RL9?BS{T2j;&}ZM90adK04zE`*
zCi3tiqUo;I_evgM*yy}2l1zAB18j-*DZp^?WbuLDD=LK-iH@(V!0_YiE4_rf_a@O;
z%lGxBQN{bkQ1t}S(k@HAMtbK9{@ERiOHsvqUDc2pepLL+0Rk<BCG>WfuwMGJ3q0xS
zPF1t2X&vKEK~8PbFeN#0tU;oO#)k~K9_(h@rry(%;1{Xn3iKmlON;*$5@s=3h)w1B
zGZrBg_ZYl3i|P)}X3FA@_Dfls8ccww!l`NB5?6f7HS@hNy60E`C%l0c$b|G!#4v#r
z`KeA(TW*?xXK%tei?$0(^b?KXQ3q1pDFkPomFEcz&H_zqxAfV9QVnUi(E;&X+g4y5
z+$lwk!z>;REUm)MmVIfD10kBjaGx)C@1Mjpvzp`rFQ$$ogwb-ayl;xd5J7-a2~<a2
zPCSm=GC#}A>^?Xao29(`h*600>w9`JJASQAtL4&u2MDkn$WA(rvo~P5F5SL=hNXO0
zC4G{J@LHGyugdtX-vTPMF)0jbloZ`SlzsVeV$Wm{%=@;@d5{^x3He#ezZ`X*iyqXA
ztvT@=_ZGnOwu}jkaw9!8^Xg)@*JVb!(%0xtBh;mD1TKRqJm8=z_|m<8rN*T2FhowC
z@ojTf!2Skg8%L)c&WJpjO<0o*YkP0~8W8AAUDvVCxeaVLXpaDhY`GIy-g=&Q1*oX8
zAl@|Rsg1pQmkaE#L6FG*GR?yj9q*0kPU!FNI`!9~p%K(jd^Bp~DsuhnFYl3iry%@R
z=*M{WSL>~k6J!k%^DZX~8`-n5__J%rm{Chzgpzyqt&!cFCXM38hNGt3QWnWYU-Wss
zT0X_r_EMUMH<)Kz?kjq~>Hgl+Wz=u3%^QH&U2-%f%~lFqe|$d)R0_W-&eJAZ%(Feb
zF4#iwc6ISG!wc}#2`88d@|op?;EY^7D>Gv%I|fdA{$|OGa#?u5Y%{OM^uA+A$&MgG
z(<t(-rLH4bqR%2cl4rKc$DqtJGsgOT8H$s5?+!&L2xlF7Ra?|eK6#1}@uf*5!2Mj_
z;{%5Bp0mKH^~h!aTUPZF>e@-jbX>g!Vr6=Hd=A$~xvT&-)gc9-J!D}Xv{8&7UtCK!
z;1dhk4fqTK9>=@`gXY$ccItilu1RCB&`_~C`un{i3-LD$*^LZ-PxNl$9-Amm7ST$L
zNBVdZb9At_{5xS%T{ejI5FOXDqm+u5!QHDjghEe=CPW=>KzKJ2)$f(irIrZMP0%zm
zLSL)LNe;wGn9so1hLLd?PHzhJW}od}ha_LT<R?b0Q%n5`^Taz#$3}jK;M|OIiULaC
z0S4#Z{eveizzN5-dwED<^)4t-vYc=i0B<!T0VLd4(kGwy3-p28va&wspHb;hjR7@6
zxTBY)r{}%lcl(s5VRBQ2S)jPd+5?}7o<|0?AM|!K*a;LWdCMi_)n?uN<rQ;51Y%C;
z$cvtz-j)=KfBcqygv=zWiEf_A<K^U_BoXG#godI#Hl+;N>tzTbXIdZN`$+A@ogz%t
zr8ywVL$w@US_pfqyK~1{HY?j$bdP2G?X+?<X^}W7NW-nr5sW^BbmMojC#eTdC~}`v
zk;|^7{v4+WXI@Auwu6(Uo_=;UqO?N<R<BYCJETRp47^?6H|U3bR2*ni>tS);-y^)H
z$uJGw9dhtlm@MJTg*hAa!i$iy1_J04Sd+c0pXe?Oi6prJ092q(;>GI&M5x=Mc0RW8
z@qS&g0jMRaUX^qss!8};))n_X=GEYeYmbsZ4_8fk#UM6T`Gv|^TO0pvuEgZiHR<-n
zGoK(20ISmO9mbjCwD1xE?3LEvY_G6Rmc@a?+K$K1`Vkcutq4HAn3jOVTgE7-`Buh@
zGV2e|d!B|r>%Tm{cauQw_o{ggyP(6adgw<+H92#);OSj<A7EOP5p!>(ufpWrKp~^N
zD!y7h`dlCr2D~I8KHqnI-~&65f3gN3_#E1IkgKkMIw?)V^SFW4kf^1i9V4)dHa&RY
zsTUfecc!OM1mTab7bq=!;^1R&kBTeUB&v9!TmH*|?i?r4Ba7{(d>ZMAy65blCq*15
zMf>8DjQC~6!Cwwwb33351yT_+IHdYnHeB{b?UK_Pk-3va*)kvQayYXVvRK#2yqi|j
z*NPB0by*hw)LD3`+h|9#|Nc(KDav|ma%fIxqliL_y9LK~tf1}<V_rL%?@id=aqpR7
zcp}2?g`1~46G<;+aMz9&GmsR-%)B^F?7@o{PFhb`sxYljsN28E|LRvtN3~d0uPAjT
z39}s8sr1w`DEpboYTiY|B4LGXxfGc#=8aKyZ+z*&ClsFo?K;*jShWM=Sg0*4MwMWB
zPl%!uaNS3ifPHv5Fm{{fbu9dFf@V2%F1BXo4CN%mGz%Ofqi~vBJz80%cF36^Dg#c>
zy{5V31w(D><J+e$u!f*KYK5js)P57<4^`ocXZ*&d7;}e>q7z~}#rUozuX<VYyK?wt
z)|F-p7ZF0d&MyIcqKCb0X0ZsX$mt%XRgZ48$@+`?uBEHsUx5Pml70W-(t3&h>LU0|
zU0eV?1halVdJtxf?M)mbjJaEk6;_(kZ_wkouRsu#Z4KcglfHxZJJ8Hc0eIeZhlkyv
zjO2TBFo%iB)xpmJ^;uit5}==rOn{J=#zUY(-u&vuznwn<CJmwr-xRt%ZU)IA5T;Ln
ztCd66HBbo37I&+I9PejLFoAThAk=GZv(13^y{6mz0@$w?OgU#?i2hlkW(GHGfD-k^
z@D2B0o*$~IP_|=DHV6AFCoRr3D*lgcV!oh8h2L6S<6Lle4*pNF;z^Ey>g>G8N=K<b
zWyKrWkgPbU6I28`2t;!VK3#BV-VwQD=Urd#)c3d>uWhK*@a{i}ik-z#evSW4RqO$T
zKZZ0u$)PxzD>Zuv{w0F?nqDHD3Ox`#h-Ga`P~}Ah{G};g2)cS_NA91BVq;xeX`QE^
zqZ34wCh4pU{oji`1NTwYEMVe2Edjt=CdKvzkpAdECtf-Y;Jh~OpOkOxE-{oxo^9IL
zg4jh4UBD2zeGFJc3i2zH@1J1p+yH~^XzK{TzNJ2mx4Vw(Q5}V>@1n2}%TAeoo~I==
zIHRdvt&idq;dLRi&*|A^9X;9&xez*c+6$Gx9MvIM{4#y?=#hr2?yZ-rT(6zm>J~;G
zfH%xjvHs&jF#eiN^=GoJc=YIWyj`9mZrGmx5B+je|9U+dzP4p&HhMfOoxFyV7DIXS
z*<Juxa|hEm6VH-6+WUJ=@S`5D?+4?K+k@8d_Q+Q%YHZ%0{G?04Du+R--+@y3xlcL7
z{z#E&|LY(*PtiX8BqL`<V*Qr~36&q$pJE~U_KG})z2h}YYCCUmQbvB4Kl;*>X_CEs
z?0@s-a`d}&V`qk<?QD<ZExN-;yNgGm?FoD@l=azY<W$G}5&eDTBIrGqj@-_y7;9-9
z(bI&O+c*Tozv}S4B~GMN(SUyUwGsS{7n^c38aW|d8pp^UliiI{l9no>m~T82`}o$*
zAK(hxomb^U`xWU(ls@Y|;^?q+bHv{Y+LwucE(2Qa-qxdCBTp1@{ZrFW2d(T?xM#GG
zrOzS3(zvz2ppOU{yV)w9n|lC*3avEOPo+VHzfdLJV@41@2;<ft2T+qv5S*xN_LhA_
zx&+L)zhpWo&Tu<O`>FV^oeu0Kt-<b#UI3m(gL<JqK1czU4WXiYjj($|b2VBb5PN&x
z9OkJ{ggCJv_K8j|UnQafY%{{`5^`8=F4gT$ZMVdjd#<d0D{j98$jyx~CVG4OLxwA-
zbDrx2tmmX85E@e)Yt~yX*#QUIvg@nnP<9wI!gYJcKluS@z|jZiB&Xw`a8g-lqJr-A
zMdbFWs5mxuA3{kBPi8nr%*e-%dwQ(6;1-?dJFJKHXHo$bnb;X3k}}Y&6RV=;cVv$o
z*!47yUeoZPWF*-d0BjSN!^XMS&D-~IM<v2;k_S0xjlX0Ole8BX$HN0xPxt+@LVeaS
z1FcsM3kYEKY0vVT+5|cOW7dfi4Vt$$i(*8`#eC(x)~E_gG-BCx32xf3rQHuDva;Br
zdDFRAD|W|A9@`?3JSYYF4koY7!hD%HpcasIXI<(AO3E*J_DGAOO5rgMPj<a>xu@nL
zkpqwDkD<;xq$u)?HA$A};+s<YgVYLKg4v`k@|F?d>}0BD8VfX*OL5k@r!b(*J<w%?
zQ=LLtlu9IO)sTQyQqwEt_y4uf+#}v|WTOsXyS_nV@WHNB!c6>736#JNj*Ufm$xw-(
zRV$AiF&#oy&&pR*x3NT7^9vG+8>zY9Jq)~ik9_KY>I!+eh6<NHUoq%%zs$rV2F=U5
z#zb5h1bceW8R4?F6{O4<XlB;=1gu>8q`@HVQ>6DhY<`qJ6F{QPs=}rUI^)(9&Ro0Q
zlrsHr=V6|?lw6~D=6Pr<ON_=HKIG4<hbN6RxPTdn4DMiLj=Jml>gcNX6C!AdIWS2K
zZ@<8%Tpk@>b^@^RcXHL65>G){EWWZzfMWIn>$TYEa3B6~^cPq$UVVDsliCG!B{hv2
z2|M(ZczGkRP;yj9UYnqI02NRx1xGs%CGmnpsULQxXAOATi|t@|I;O7lKO7t%AoLy6
zB?J18Yt&q0%T2s%R?>>^ADECCky#c_TE-a?B5wHK%snJ`T@+8W)b^m=7NIUeuH*Z#
zAWCd~Q{Yt5sj7n(Yai4>Q(9q`EQ+dH%xgJhbbni}V8yhbKd96rH|xx)<a-;n*B$T=
z?2a3I)?RO5(Vsq>aVd4?k3~ub*NydRx0XzTH!)rSH-OWh;@Hjhj|@ufq<IEQmfyA_
zr86y2UcRDKg?Xpdm39i~GAuKY9Kbqvz<xjIq+E|ms)Dx%o6Sd2c~ODD&U5#ToYY+a
z+q!~B&Ly!`DrG$k1zf`1bC&_fn#Sb)4kh1yPY}}?NtM)HXnke`7`~)e4$5Im#NqLy
zvOTb-8fm_Wwlb)N&JR;yeoh33#dod`x(5LfA+*xcG&}nmv5wxv#LiZ*zjHHf0zzfD
z*W)MC^yS%yU^?}sf%7I}Zzj+XvoHC(@|KISnw?*O%w4~C8-RF%1#binyr*vm0pDae
z4vb*mdVG0nBE#I>Eo%+mHYo8HMPTlX=eq!ox?KHt(kjiu7JHN`0`xDixB`IkoV613
z9Q5rxhAi>QUucBQ;@(R5y>7~rDD8Nq(qu0M_<kJ?c~TgXF0&2Dodp$9KI2uTHti;`
zIbnK~0`Q5jie0?XcQy-3a)Un?u?@Fb^zYSki(&-nek?%#<9hbT|GB~H$LBY@@f)=t
zYG7Fsu%#8qM<&u6G#{Hq<5J=I&ty@%RGX<WO4Ou*&9vVB3Lv%{6+}3*%sr614=1@B
zT&qi>K{Na3cN<NO1S#Z(JOS%${DXyUYth#9E-TaCQ;9dLRjxmj;;TT!aX%^9RM8~!
zYvPwtA(%(KO_uSPooEW&y``T~T_z{7jlZ|E*(rtrinf~<OFNodhd2({G<o~x^FxOg
zW(+C0_g20fZ98e-7{$3C4ATSiq+QKvpogNs103|Q(iBVNSwk*12Mc*n>2U-5BkMI*
z9qa;-yEsksDnn52p+wo3O^O6>FAXaI#(D&mh-HV*j?pJL0z|TwmlG)}Na?lT)nc$H
z#~40~YM|dgvX?P#kh~Sstd*Xi)a0>u;#2l(o%y(66{)bZ8)-0X_8HZpaxmK7jbs)|
zMTAU))tULhx7Yh#(W83FdW&B9G$Xc?*0W%|lB=Cp2X>~6QO^Qdv|`HO*RGOulC=Y-
zKih^6Pf$@81`OP52iL+Ekmdr<)?OauKd1We?GH{lv8=o{PgX|iOwX;b?$TN?uH(*L
zv^YVP(5f^mu{_tj)OZNd(G>t50Im17ewugHCS?~d;<i1a(yPkyW9}g?NEOozhG9cy
zzx@xxFlA`$an#(b<&2mV3L?Ad3lf}bR;?LfWOWo*7%1VC)kf6VnfJ&B2VPXjdG%!k
zw>l33A6N9n%*R#`pBioL)8~A0O)3|CQFdF6he9BjV>;U^OcCLdYj}fK#9w}4826W5
zV<-LpPrF9VRg*D_M&IsX!o$EH^U3W2OGAr}-?nMeo^y*sMM0XJ-HliDFHwM?`RJZ0
zcI(WTqUBL<%J#c$s~Yrd0d^VP2~n(u-gRv0{wO=3S$h-5n@u`ikXP*T#Gj_y4&*f0
zxdB8Q3bduX{0S<E30WAeNZV&xX*{&{c$7j+noD)NJ#LXvRG$UP3OVN6!oNXXv&Y=!
z@xjW<m#=-Qk6%qT$6r7w;Io2kb3#-H4i3IWTkc+N6jj|C7ul$w`w9jqD$5Tal~y&p
zIXgnT)go8tWOTwO(g*DNW$VGbg3ICYROK4Iim1|M@rCb((%ir>%i!5HpsAXg=HN4)
zv<d<Lo4QXutcB2zBoE&ay><v(zxL?u-^&%<?+&2yI&=BYzUAMdwEuo=h(kd^E|1qQ
zou8hS{5wR88w%0d{}H0qtxJfQ_U%;MI2l(YxZhJM)gGOS$XJOS$5zx=$;mDbjk}{Z
zUMez;0^|qIsPe>aVRUyGFSiFnw;sU>O$?SBm-mulP3XR(<QsJIM&H7=xp8ZTr}(yR
zYmr7<T1?Pf58hy$>$jUP-X0qaGDApRQBvIMImW^JsoUs)ly;iEdQN?g!7GK1H)k<}
zJ5*=%sKwU0jAm=olh4*{2Y47nS9Roz1ZDt{){FQmOUcU4X%#opBe66a=$C3$IN`?Q
z@WHPk?T79K%ndQI?e=_Me|-CSQY6DW8cy?wG|OV;-OWD0aG$;8B?(rPCl%N6Y&5~b
zQM?d{Zx%=zzS(M9>i|@{bu28tc{oD+Qux`%x%P^?*Zp|5Gv#VOe7#L4O|=5xs+~x%
zXmwLK8x_iLV+4h%)%uhnIJ4-J0^Zw8fzxSA=roNQ;`?5AdAE6wVoETyS(g(@cD>EM
zJ=9-U=CNIZ8_laMT7cr8%JWYmIFIf95u?juW3p4#FVp?BuVmp~^ykJ%L(kJchvFK0
z&=xJ`f7+rwhRy-<GgBWW9Yp<~kk(i@<{3gxuTd1Lp>)Gy)pQHO%OeoZIZdIDskdhM
z2t^Am4<b!yD_aKOmu34?{Pi)G+15T4%(RukQi`R)X9BtU8c{rDRn%w_zM?<f?O7)l
z(~@)kL|l%3Q`4s0O{`_lI^}U$XMGp5{thNm$FqP4)2JSnr7*-NDxBx(zM~;8@fH6;
zdFRD_gpl2eCbEc@#IkLnB*S6>Y>WL5y72MSb{7#Xe*p0{!s^|7W*h#j#}_oepV@(^
zd8o<v&#kK4H7lAauc&MLPRug533<JL<k6mO0!l>AYr;`HG0(ww9sSJ%>^Ffh(|cEC
z{voZjP|d>K0^41lJEEkgy@SzI3D8_w0i>@iZghf-ls;(XLRt2-Q6y+#_&WdA62SH_
z$_?|ANH?xN-HY|vi`~|<!Ym^`JS;9<;9~oB$P4Bk<D<V#3ctu%P71Bp^g4PJx9R8{
zPv;j{FA`<5VCbO%v@92)V<!=SUa<p3ErShy-3~xqVcWZK_1`Rjajv;gfxiX}5e?a3
zK)HYOHx{b@)kphdy&lH{O(?l-PQ7w6<Xk)L-dzw@dy!|F>)OBmG9axTiz@{?b&Xrx
zD`b_y=sk`;Vd^w$#S&6pdxFUd`{k-b>g8s_ms{mH(lTK7FrdZR)a+gRodsXylyF;W
zdtpSKmn^XsnP-5%y~5nK@{#Op!v6f0dC`|dcAljc=W9zNOYbQQ4)n+M`|$_2;td4n
z@sj(`!sSEc@CjPROG|IIhr9Lr=DXB+=dF@Fvv<tJx~?PU2rXQYoOH5K<I`S1ZxN|G
zllfiELr_RLLVOCp*pU&c_gP6RU_A7-I1gf(lOEn0<)NU$Ss1W9JRufhyU_tMnMQa(
zh+X?qHpHw>*dxw~6~P1ixg9Ff4Bpx35<&4^z6Xv~iAyQnGhQIdhPM#zwbs}PCOh(G
z{<p|ggNJQSEUcD~dyjUHJluYmUqY2&PQ{D3|L%1nZr4a3o5&=ym&<0YzJCxOtOCt9
z%5WopIF<-}2NEq;;_n4s$1tBuhbFEcHCh+#dUlM@g#^?im;4Pgg<0ENWG3I}q<*s;
z7NOZ%+(t^tZ~~XuHc-kSjL%(jIkdZ>u}c_$06yNr4*E(+>wXhY`-94Yuh&_5%E||R
z{LWoC{d9qfK6brr?3cH@{`#K|tkFwn{bQ%snBC`KNU|=<RnjEoTBWc&`7Y>Q$}AkF
z<JSde1pB1>xb~MU`D&M7t7$7b5l$N%*+FKlvvxky!4hRDA#+KypKwOY*drQ*tz!1w
zol@DoU?<DBMaXccd~9^zIVHo}NVA1v36&f;lkAjj4lJlXg8hy&RVmI<bZKVTzf!pV
z7Q4iFNr@YbY(?!YaZnQ*Q*77Fb#YB_rqZ!)@`~y9WE0-HKE13ccaTP6c6gkmAN`;W
z05eQqI;1{&a`$NBjnr!p|3Z5)a+p(gz|4R9UJ?zWr*%xa9W5rqSBm&cGa7TJ2yDIN
zvQnPfdZIN)#hx<9U8I1RczV~KcD8Ukjwo%W!R+&EHa^B|x`LjL(M!X&cx%>{hbq&)
zY%p^WFzUG-gsPT`+Y_L~(d4;>_~O+c+kqEh4R+rh-krwDc+yk+q9L0&@`E&L)CuZV
zYjh$N6u;f=@p#1Oakb8uhq{7%gYF#*z=J4CCHXxKhr{>6>X<JW97{f$H7vocBHj)A
z4<2<8-iSD1KWT_jug^xeB1Y&-lz}>zvbuspZzt=LMM2RQ>|TDB^iHVuxxFzn+d?O~
z&fIYSr-kWmRVs?y)(leVK6V(m%2?m&wk}_)M!d@xOwO^?4u+ktbz1=9ack$s%+Pqk
z=1ZX_JA3C(C($Lc)QNk9Xnso{lqQhTe7Ci-GoqCix%f>utZ2>ZKm_ME=<}V#-FBh$
z#!esmf>UqPov*5}r?;wMCWjZG;>eS`YpOX^Ms&vme1edeaQ1_ax3g#8S8tHu{*Mwv
z?}w&YI}Ll~moF|Yg&q{N=T6zV{DxMEEFgB2p>Ls!`Ck*X4H)qhPK4jX==3s#MWIwQ
zEYeq>2;52LBGirdTY-4061d5RL0s8-U+F0cd~qgB%A$@Khv{%+kX`Ii8%-FPZ#BZP
zA&bHu6dbVkZ(c3Tj8%?l4pgXkuaOrTQRqZ{Ivbc6LL)i)wqVb;zBT%n2)i)eGjO}Y
zNUhM(0=&2;3+ndMYnE9+BL1y8YUDP-!FWeivrV^9mp-v5|663lytVO;wY^WU)HYy&
zdQ|dG#hv`hc9Z#@`(zPV)5`)KN5+uhg^zrf_rUyMV2v)edSz;WjUMMU_;fLEm}?}n
zc*ypxj2HC+#Lmx$x<!E9nMZU=IsKjq(b5x)!PbVKD)&;=e0ty>m~5yKvykLl8@~gR
zOx<pl=g+}&p@jg%|Hz$tIr`u0y8oiUDs-iKN|o^ZjQ>B~XddQWQS$!DG9zU2ObiMX
z^T84u$8uZg9qkD!!FkObOKYqPN81)LiK*dS7}1S+b;jpI&b&(nk@4=9MtV4b?@ivD
zn)(+ucXC&nz41N0TZKpX8xN?QKhvp!WN9fHR<GsP?#zcW*#kv%%rt7Pf!&jSiwP}~
zY}VhsNmI7ZIef^%zsyn=AxI~8rup6tm#5@8Ufi;6)KNZ6VY$s9i{Gt6dOp;Sz=RIv
z8c+knO??g6y^K&81U6c0SA09R(SCIUEEe+>05nk8xqKTUXb*xMv_wF<#k`kqllEe=
z1w{f|%ftjx3Uu*10TRp}kik9X24!HBZ9Wj7JQ{VAGLOD44?H}q{Ad35D(F9g+|L&}
zGrClN@e((Xs6pvWoN~@SW;NoxIw=$hnDlMd4U9-%nItkQWEFasJj(He$FbRoEX3)4
za_%F=Q+4F%pc*Qa-6o(x_2X))>SRJR`*kj1i=>9Zj*tgI?I{?@6Ij~O+hWu&*6v_B
z!#|rnoPoJH2TSM*)DL3x;D>2r<z95F(Vms^j$_~5BagD7;19--RQ2Bs9I7^%?qng>
zXKcl656spE$2m~cS5F00ZMM!D5o9Bef=Q!r?2<Y*zyOI}cvO<*-AA}OZ*Gce#`Xs3
zx1a!IYNS}GM<rnp>`4sYGkmALp?(ud+MERmsLA5b4bh<AiVRG&6BYzgE;ndarp@s&
z3&#Y4^HAXS#))UHPW0X!aV3K3dEpttCMiRI?#!pNx3oKd4n58CPlNY$b|JdG?pF`?
zV<<RW6$(=y|Fp{0kj2X1j@Bxam^jZ26M>$ubBKD}SHVJIpyXQ|pur_kE~K-mk*e}B
z^Dv!^Ss@_5L~N~{8~D(ih2uDwF89HGS53m?nCBkCu+3kZA7!w=Pf*Mqm6F{`zo}y<
zUb|FuQ8o3Hf4Ibn<qjT-Pr}(laXHqEQ&{E!V|r@LnqiQPg%4^YH?6j-w>4~>#(?0q
zY*N(QkJWEF>gYRf8;kat${Fu&iWHxo3+ly6>P7SHd`0w~Ez@onNs2Iz!97j@%Y_nv
z)sN|mjs<;0)|oelJ7BvRP+tAmzZ3AH`jyuFWCFBSoOVW^$Poh8BwyMAwO1$9y`p}<
zJ0qffLcs<(9VP|wquqPt)9->10os9Aj6ZLwSs}=6bQ4i#r}&TK_`ly_xdhrxT4{QE
zj`H6H3jE^93&?@xjPd2Q(^(DjUbBkr7qW>xEW~UbQD=AZ?*FTNchmc^oxo|;da0e;
zMp!L%@>DDohsSuZ(R&S)ah`)K_zj>?EguBBcp3_RY9KN~q8$K=6qStWdjw5S%3OF|
z!8SlfB<b-LaGtI|v8{N_##r8?^4S9{0>-+abXkAwX*xgD>4z47`x&1#*s#$0q2BFg
z{%r#OFNf=2e<+CrFRh1ePF*1bVv=f(3`(o<NE3yTCGwD=0umprd9pFMDX`2FEC_Cg
zHJg1a^lSguI{Ln1DH(-M(MtV>f{sZ#$Aoj9d?GbMlX_aEDj=qSpqF@gb!I$B!eh-J
zJ_1=wmU~*DoV?0Czz#M{jQN8I;|NFq`k=xyU!WnL4c`Xy-oLcLp7^%Fjen}SHJ$;S
zm!O}F`Y(<H6=n_rNC2Apd3FOh0;u_XDJ_n#9&$eBE6mVJY7De_(#i0|SHHOwQ^*~e
zk9+WfG$|dYsU0FSb9Ut==VkwzI<qJBm^xYOjrnus@AVG|O7wQym@+1`I(+d#KQgb<
zK|GIQM|cYs5U-|(YxLuYQl$dMcpGa!or%1V`WQu%>Z06Ek*=cw4}Ni>B~x57wn$im
zBGd&NG3B6+14t&W<anh;;GXTRuNSij5sW;N%wE!MWkVk>sN+l>QWYFL##I3BwpBXH
zCi4O)owOl^1m+v_<6Y@q-Y07c6{?ecaZt>|pqW3TVk;Ddo&?T_(Wi<ZUBJ8v1q2UE
zx7AX=r3SP9qb0O@0iIp^i52NzR;p&F{zP;*|3Gv^=`oM*H3|5#H6-{I243KgOf%tb
z3gh(?^X>B0YcT51VA^87E;T<hX~_^;9=q2;>#@6!wr#S1Y^rYmT)vVCU%?KXP18l6
z)XeQK%3^2Is^QZk>_t+i5<A~TVFq5qKB!f$4{6q@m|J%?zRU2uwGeMe=rieWL5Vu^
z=a8j%%(Gg1`EIVKoh+r~<h_g;i7h=-W>yZA-R9cbUfZ8hK3;@WS;Rqf>#eY&dZ&SK
zK^(V@>^Z80={ZHe+3Pw_ZxaJVlp2ZU4>_($uzmQqnj!fB+)?(>W#ypEdxk3ETWU<w
z&r64&wgW*S(9`k1N3-yHYNZPtxgk%EfafxhhfINTSEzE{IuNFEQPGRsvaQ#V(JR=9
zVu{vew0oZ*8!Nvw!&MMz<O*ev=(9zJSB`OGSS%KvZZ}0tWRxk<E)VcvL_Om*!(?(M
zk2=kj?bANoc&<%HQA!xC*fN&tx2U1nCckw*aXmMrKg)q}s^?g1WmgGOHbAC|U(I)t
z#usbTw8}qrw6HDI0?@kD&qnq82Tr&;8NHdu_-w36DZz@+uZq@R01i$3m<se_$&6gd
zNWyLpc6!Vu&pD4Ucu=#3I|DGgbXJn{i}pvI0zEhA%Y^>n!7=ND4!Hk4pX$F`o;IGE
z={YIwy2Vy%8Wld@nG=1F8JVr^b|;UUS!(ncOBup&RIC=CCQl6jH(bhkOGSB=a1pIc
zuwd&EfLmzX6m2oA?*9JRXNK8Q2eX|X&M{S5sk4MHM46Uu_Ln;5JjFJ=q1ItGV}#);
z&Gbl&o@b{x_}+D~yAt#6zOUx7BTNZo^zRV@$kY&O4NH%(2f&=j<m-Td^#-YlE@<VZ
zA*)Era1zVL+Hh~JxMAoKLqI$vsY{N;?a!#a8$3J$59|tstsA%zoRcIB3e8kqfWWMh
zM-B;0Ft&UkP^laj4n+tjLTczrAX{i{*6uI-c>b-l*Lm9kk=L1qF4sLguD=s<o%;I&
zyeA8pzyCMTq<`_%7=FO+mFo{_QFgRdMuE-hu~NRUmu%*UqP}Qi4|NQZ|67|ff97^c
z8M1WY=rIf=51L9~4r!e^wp)YD)C#Nn`yLl`5_O~Xw}Tj+e1J(Y++&sa!=%{uM5i==
zTv0FKoHVxuFG>#2FyQcD;@Be&`5DV{4c%d`4#Ky7vWt9FZg^~}UK3x(Zg2^*a$z?+
za+j>|OB0bGQ<K+%{c@1X=l!$?4sUW97&q#^!Bka?Mq-rM<a+FYBE-~a4EI<ODoeoZ
zRIW4m$;K1gg<_C2CnM%HNl(KH1f7m@P}(82-9@OOM}d{A4TP$n{dl2(XATz{2>1b^
zI-;KcfU-S@plm0LbGZL;*8SsS{<G0($o_Gx{muuVY?Tb-2Td$0v1|<tkz|+16q9IB
z@>EY=rPXGO)~7~@!2EQ4qe7$nBlWM2J{aJf`Lgw~_^ql>u+l7tR*e$Pj`0n9(S5+~
zYvkK*jmQ)06J$>(|379#e5k$~pjTbwx$+3QX@P$u{}KPDxRkKkA3yHaAJ#pv%E0`q
zbOS`O<m>?<&Dg~U@3P;Lf1);s2}moO4F2AxmCiz$wDGglYe>2RQT}pTfA6IJA$t7h
zjr)(0;;kQ#Fy`yOeRk8<kk76T{+G{=**}WXn*7tvl+reV7|wbNSl=!)`kRZX<=Da0
zk{XX9=}SCI$R$GlPEaHV^;Z071}3DnZ>Qf0qqE1_5(|z-&LW@hN|s~i4!=!iMoC^G
zG|`8_numtaW=a)OH0!rW)U)R7uestv_UTV66LQRzZ7by#Sf;&S3@P7n4NwAac)6U^
zmGJ$vVajTWfv6!(<!f>)G(PpR=mqg$kV57IJ-WfxTa{U-zo-Q4f$~E^nrZr;q5YR7
zpkbaq1k=_h2RWHqVzuuS02RD-ZxsgKwko7jbJAa<R`4w1M4=DXP9R?K=i~cNPwV0Z
zXn5ZEX6~7!gP8x$K7%{iR*$eXSp>N^6?}xuI{5r<p#dlwnEmAq{htzA^7FdBTzaA!
zp-H#;l*J$ZnXz)Ew1EA7tfiPs{K4&2|AZth$*7`~FGmAK0^9fH`UaKXHR_R?g!MY#
zPLBr5((>^Qb<#5<$YaExMauV4T+@cD=o%s`4<E0E2rj<Vyom0$7`0evkYd7-NOxEf
z8~4wtZP1s_2Tf?Uy+Jwwomc;OkQR@HPIqHiHym-mrd5lq?ziSWnARM*?1V2cXc4|3
z>6f`wz}6-WMa_9?X<ILy1B-W^5@0jk$<=lNp+aRiJFdC2fq)5KZ2%S-i9k_YVzuRz
z*m>jmup=~{h4vzkpb~pFGtnX8(T6j$px<Mi43YZTf9QYs(MS(cc02u|z-u6p{UwU}
z=xazO=lA3wxQ4oq!JQo6Kp&;X{{%2v@$r^?7|@U{5d=~mFznJg3kH|gD|h_f<O|0g
zrI%eXUO+!V%C9?7dlAet<<`LiD8G*uPek)ziUem$;CPoxnuF=qR?e4-4jq!=NPER_
z!X*Gm^i$#}q2VXz{iKt>$1|YDnE|`35*>6imz4W0foCdiF=6(U%bZAy*Wm)5zi-C;
zK!p&FL74`7@)<Iwgm92L7>L1HG2Z<h+gImkf@`k<frEk1XPdmzK3mz5$4!Ac=Or9L
z%5SF{-}|zdi%78aNrY0Y*{D^$2Z6wOXxN4!Ay}y?^yGd&4l8M0HV5`Yr6fB7c;uId
z-wZ#k6XLqQe&X*I?^y$ny4jt2&rV(Wf6(gwIeq*0OMK7l$NOrP2y|v|7{*BLE)%@S
z_Y7jG!sX2V4F+cXGaq#y3p07ECTrDrS5gn<>Rthu2``L~>jEo!HJvFpCvpHM<8S2L
zQQ{*}dQ>GWu@WS8i#~C#C`JZgE%RyC2Sy`{Fg}<KKx#|^q(&7+S#wd$a{e*$YJQGs
zfg(@-Ms|+HsA=)OAUz{{gaRf7Jq271<O(g47w}h%1WZ%`!DcALYIR^Rx(e-FSgErB
z=9OR@p;7KrE3y~`V6^~2G+?EjYVK01G(irt;rCN{{qj!Jz8@^N+$7uj$=D#YtLZd;
z;<Op^73j|GAeEU6J^Lj)Z!qs?q`Fc(Xjjsi0Gx7;4Rse!^_%)&O8|#FklPlecO^Am
zTUzhJ&-b(fFoY>O|0kUC#1YiufePrt$9Xmz_YX_-t`%eqdsOQ79@LK+-Q!ITW==Bu
zCv#B$Ka{<7Sd{7e#tj1kN{iAd3a%g_-6>cwxTthX$j}JVh_s}lA|WA4r#N&9f(kew
zFf`IVG(!x7@BP^O-F^MN@1MJed+gyecU<>%o#*F7`Zz~@)DY?sDNl*0j5_s^5x;Y6
z!Mw<KVTLyF<LX-lLo%#6g>O4IJJ%CQVgn;}Bbmw`MOz~U-_;0>k<_QR$b+p_7CQQp
z3NUjHPl!Aiajs5;86<%on7JMr&h=BzpYqN%=P)3NIM}#S04K=G*M50KzS)Ch(G_#w
z_nm7|&n#&(LN6a&Y;df-KW<FLTI({{_S`9S9H}mY&jBA+Z<Lr>937t}zm=mgFm!hp
zL_s<N8)H7<&^3C<?okSCXX3!TIC$-ol^7O`H4y@}o1bQ|L%TOemcob=!L7|EtNq=8
z0Ue`55Cf?azyAvNudDjkd0Ko9iua>BGi#z7Ld-n6%{O=SR;TBR;2JTEqUP>f%Lekh
z7HQN61b$M)gE?FbN-PKx`Ss2OTDw)2Yzu$%&igC{-zCR<KU`b{0E#Y}sw~B5$`f(;
z=C8=#&*{jj_p=9r={|gc`f%|<e-{6-XF0|1nQs%CwU7$Iw0gv!49Hr9@kV>NMhvzP
zmYLqNP|KHF;z#rJkxm`7m+~Tq<Gj;(0uA2@$sR*DbIqXvx6Ldw#5X~&&u=5L(R!px
zK^%u8x^EA#FMunHZ8dT0y?)-vImp1qc=aBG=<t;cC+{|pnyG*pGN|yhg1n!FQ%opW
z_{{SNfkdoc{$z5z>WK$ZKbIXDOvAgBOESnXIW^=DpN)oi<_PovK5vO4|FC1hR3j8>
z(K&m@G*D0r7j|`woU^_ES&ntVU^GXnNX<R7s<NtjajRx&Fu9T5iOdl6&LVxo9MGiI
zy6IL)(~RBa=zmm2CBDG933$4=1qqaxlr}w;g0^#9kCJ--F_l!#%gn-=*o>q}B5tbj
zRk1E&mNkOQ8v9evhuXxD?T_F6eWjp3junW^wEONUs&Kf_0K%_XhP)J+{6z_@rCMYO
zJ;G;72*4X&$*~}1>B_{~o|ZqSwc)G+l6}GskX(pjHvn!_??gJdY)Qg<4XI(-Qd#+-
zDW*M<AWnzp36!7%Bd&EN>c<xW5gj6geR*e4*uRC8kDlO!-4CKkJ15CiPvi#6Inpbg
zc7p8wQ9k`y+H}tQoSIvIvtBYNxc=Zhp+VN?bgJ5^qUU!xIkQbKJZ+Li9W0K9;w+vC
z&YrZs@=?ljbWvY##j6#Wf~T!Z#wXVXHe6BS1Q?7DkK3hwtCRk>+tZL3s0bQR$2Bi6
zjy>jic*VzyVjDM3yt%<z%-urQ6pJ#Y`j<7N48QoQ?Vr|=>lqjbDt_G?Ii=oT6t#T?
z>xc@CVYQnQ;x1f^(K{_`oF2k*Q+v+t8o@{!mi~gpppT-B>%yyQg1h(0j!L1=rONKZ
zrsY?;ciPR0S{AKGj889`a#Mpz#XKsDM~)GX40FU_v)JVqFZP&e_%|Q)uX4LC^yk;{
z%r~#mXjfgL+i3+4XhWzsMwROU4B?L;8C{@VhMp6hC1{;zT+};N0`g<Kg7&R$I~*-|
z*!Gy`#|3k*K@|E_neT%?i2vbgSEQNa>ZGyp2=4wQ?E#}?C0bq+X`)sfqirC%do6IR
zwKX>GZj|IkK{^}98+{;o2IUYsgTWmqQjYK1LgL?<LFjQYx8Km?xj8zd25nzB<5~ZQ
zJ>Dad^pxzs+2gs-UVmeck6{0YJw8<1YsvfH>~Rhja~ju?p{m|8%Tt}K64%K@SsZJe
z2Y^ET0};pEjmwbnt_{e6B@6r5FvU$&mRP${T`}YoxNpva)xBIH@i<9!GDPUmua(3U
zq*%w&krt7V@$C`w>ty_WiHQ*hgeaDp$gNLEI}{S75U4Wk9&$N8MOu6#6xG=h7fAj5
zu(V$#(RB>Ey;*2=o|m3w>)k9#U2Wdr?o*t7ivfAPFGh)dYnZ`(VoLZLvYdC}fxkk)
z*g?uBC$%$2%cLu6W_$aN31lCL(mQ?b>I9*j?4o-&`;!4(>NfV*<Fm16JM-66gvN=i
zqH^7xcNnKG=$mn1i)=}S4#XK05J4UrwTk^Y5o1~6U%dlN3ZCHIUa^wvE`{c$Pp`+l
z<nc4<RTVJjFD2$S_k?xqUrO6uxW?I;4_xBYkR4>eCL?O0jARNBP|kv=C6`JtUl*m6
zK_!ai4TGjd+wN>fk`DV;w)$D?)H5OE0u^b)9+Od%gPIOUqsZ${epAjuA5VlLz(*p{
zj6t;ZPqYXG0SpA+^LFy*OqNMc@ih=BTB{|G9~r(%YDln4c+?>`r|p*S!yDPbUq$ye
z-$nOB2g5J2o`6?RbE8J=b{Yz~g?Z1+W@cqCkCe^WP5!)*XibR*Uxjz?oi%DFA^dHN
zm*E6Yi}nkD6^BQO(5@vQ)jITEMTCy;F1w-2VN$oEg%-NN81|_WdPdweZQh-wm_}Kv
zZ8ZwF?9UJNbh%T4#eA1yP)KWW4i35ej!09L%@T===di7CIkf%$aMkFNbBy<JxQ?->
z<#nAR+1>YWfyf(>3t_#@<lvRq*{mW!BCrHNnt9`{P2kHe*$0SQc;zJ3K{R75P)NjU
zhz~SR$8G@IhJMQWqpaKV>u``7$u+4zsp~T6<5q0)>SlPGa7X1uQJ(&X$9}%P+qH<-
zM{Gij`&oY#Ok}X12f_D6+-^$D4C|2JdU|=WqL%7Ul=N)d@_$g$`(ll2DyIfgC0(9Q
zgY0$F7Io@HeJ~=Qu?XaClpjG{ml+wva?>XvCIGUL6P?K*z8=O_s%2U;F_Y6oBF8Vn
za$c6g%Ug0&JXBJN8$Gz(?LkOGY!#$G7>QxsqP7+Kmv%jGZ~zld&8EN*r+;5;u(k|8
zjLJ{!#D1rw8;`&C^ab(Pc{mVaGu019hve||tA%xY3of57@;>%nCJaf3j>l;|AjeAU
z6JfhE>^y=<ZSMzDoVH{sSvzYFDG)@^27a2JL&=^q3Bg+*Zt_d1^wxl)>H}Ec5|0qg
zWy2?#g|8j{1zJZFKwp`%;+>TF&++c>r^2_}@TnP@C~>0iVi$eiIGxQD)jPUX5^01d
zp%Y;-!oupQK`;9s*y{@mUjJsVYd#E+D>3+dFoVPoF;5kX$80@qer|E8PVjw{ZiGfU
z!cpR|B*DAJ$UIv(gO*Eglt?ReAi^4_O>I=Y+a3#FGU-*qon+@=Xsq>CT20q(iin~t
zp)U2-Dvx%5FV&-5oD>svV0tq}E?l^EV-+|AMS1fbU>#6f`xUU2qcOvR<s<F}kOTaP
zYNFq`V2ri=Mnc;(%^mQFr}BCX0gQCf#w-^?;!9%9S{{n@P%Q&#FzlAK>_c$8DhXEp
zT!;-<RG`ay@OX#kuOs(I5%KF!wX3JaD7re!2@@x#r+mk^(-|JCAfU2ar`v40-q}JC
zdY8TDA>g+p$=}{%*6B6&-wum0Y_9C>=v7y6n2HmeZDejvTNH_x+%}^^Oc)siGL<~3
z!t1q_*tib#J$;*LA)rM!5J~DlJ<~G64}u^7z5NsHcX~U+PkOsOT#u>elp2S0NzXY3
z4ur)L-0-r8vr$KFgdL^bRJ9Ins8vkZ-hTq)C}>x@=rMK8Bbctp%&RNH_!<-_TlErW
zdHQGv5IlcysK$sdX<J51%{;}CU&Dk0$Vl3jR>ZfjcFoZaSf-0YU8CNmB&l-W`%CH9
z2VuSkZlwY)2I<yctMT6-P=pNJN*n85`Ii^Qj%yBYZ;#fi9!C#<*Ac@U?*_yK7zqT*
zyNq*~mQ$6gKF-A&vz%HXOcI{EFqAq-COikWIMVRYKE)z)L=>JTbZ7AN4;}I4LVTFY
z9SsU;b0fPsvbq~9krumwEtAJ3Y}lyn?Wub|HhbBucM)D-QKhAh1dyYc&pQYIhQ3Gq
z8~TpZwa^(nSnjTC9`ko%GXNgyHE?{MD(gKHYl9tjMM0fP%E8c+iI8y2dx4_prv8H<
z#nKbMw{-6{w5uz^I-o414eBJiKX%lFxM_5&Zr>UJL8%{rny`g0`3c%tn{9Y9{xp#Z
zfib~7$~wY*B0LRziZXUK+cxIkvRcenS#8-*S*`eg$!dG2(KED;b<R62r$jl5JP4@y
zp62&rnyRAIw@sHaqAa3fRnCGqmhjR#v6IiP>5#r+X!NlP^0>W_PA&uxIF(ztAF|pS
zAgjIgAF|rxnyp34IzU!?6T39P7H$Pp(X2wl+q@^4jon^@;6BP%r4``P{xD#jYIHE;
zc#ydftXRsbD*RhrV;ZFl!1`hfF0;iy*V?}e9#b_=P#<CoNvmE*8!V0)EBFwTNUJ<2
zt07p5)jKvht@Ro)S&$K9xX{35ss&$GV##yJ-<k`4r@;y4aYM*i&Ko0)3j-Vi#L5B_
zk(&1A1_z{mb}d({3mWwmgqM-Ia*Z=}UM*c`jxiQNvQ)?9y_U2|(qne8#sDJ`nN1P$
zqtbp#%Id`;d*M7g!*nIszL{XN5%kTd*ITt0u+|0j<c0p79A<(o<;X#xN-dSDhA%ut
z09mTE8@~r2ZYfxjiMlDUh0j1FAiV)ciqGJ_7|Dw{Lsay&+=1X}Gc-1e&gN5u)E<hv
zZ6-VYs=w_nAzzwUMOR>VyppP6o%M$g#YvXmMJ8J0G;vroVzHut%7Yd|X|O{;R(kTd
zmFmQ0mOz0RC7F_*+YM~7L4i?79}0wpN_N0UJO9O3nen){(E{~K-eOy}*+ZxUY?mfH
zMC@sJx_A|$lD4<!tJu^2o7l7E<T3W5Ut&*;Sqbuo*i&{On0yplQdd$KzWw}Msaj{a
z<>v3adGt5lypEfe;N-AMHMO_1=DJjtIJzHW(#uZen){J2xla1H@pyB~BPrLJN05ql
znv2>qEK_L%G`tM_z!2UYp4wFccr)>k12;&K`^03D=W`TvS-9+shccuP9Q&=#Yq+fh
z7TCPY%U7s>o$|jgG}Sb&&YI_g^nK3iY^yXC-xQMDkV3MGXZO&~kR%|jdZ>;DA`r}N
zb`({<ue;g~R@ny++y>enw<=Nw`PhBLN<=eS3?+Dcyv2@x-#ZGju+%k3k%M(;9u8p#
zQ?b=7;g1E5&C<cVmj=nN7>Sdd#=LD*GvvdJua>XfyA+*<oi!kr*~OYVycg^_-@d(+
zKSFfZyKB$~+m-TCK&FI*e4os7+51NCe4BjD)14)qUV$fKIL;OyTr~&cZ}J4A*8LnT
zaiW9u7PWC-bDRyJtz_kRGk*Ztw|ahHNx2u@UXaQKR><l{kk^T;^L1L}Rp5!<YMTX3
z$5)`S^9V#=7S#Zs2_Mi(;285xeRnf|U5R=00W>x@&W_I0|E&3b+s8lJiw!%pHUPvC
zV0DYF=FQ>4yEl*DmL!Z-pCW%}c9(@SRgp!wUpGUfiOHsKXYB!Mj=}s?URw8O9|U%`
zni_*OEmIJ2o4;sS{}0anb<9`EV>cF(JO=dQcMH|Uwi#HiulqLkm`*w*>amb+nO?`t
z!{+Uhh1<gpz#k#leNC_API8irCnc}o@>v2)s|vyGf*7WfkSgmF0K4=e<|LC9A}*p(
zHwh`+-=391%!Ct>$sIh5?<<&m@L17x0>A`>O`n<jE2oTK6m^~bcos@r-vy4rYarqB
z71y0g&_{f1t$-Wg>ZAz&I2X0`UX{%`q&^#jIS(&i6ctHO-AE&q%a&cZ8gqu6p8qr?
zJgq#~TWovFU}7jxF?P=dP;sK!8*b(w3(zaIdJGw+gTJ`%uIx$F?`piy6Tfl90QT(&
zFeQe9Mqsf=f1m6BYcV%GQU*(ShnP9V2}CgkkT#eWEK{TtM-v#5P9}`gT@cT)o#iHx
z5!qcO3lpPeF=ur*r}42*#)m&%G?gv&osy(+G(ym-@FVtwxMyUjbxHSIOiQWcdJ4j$
z{Ms>6YjkZkTV?)kQ5wVuy8$A=)eq$RU#|X$A5A`-S|C|0!Jv5I)$d&WUV-1Z`qwc9
zL-KQEQb)vX%|y^*_%2#}{?I`_hT6G>oyMjtUp0I(bC552q0Z|~w-)=x)7IPYXO)Wr
z@7z?{LRcJh0VoQDSA~RFU%BBldt6gCd?wlT<NT(~^4LYQI6g?}{oze8c^eVn3oqxW
z+y+KXd7V07zAJjY2Jk}HtWY!Km$m}^Ob$Snxz>}j2I4oDt&;3-7=PL8y8{>0N1Xkl
z&CGh_!OUfoy}zX}ue(WP5+8dzTnX%f1X<q4>4|BL6|ewdkTvY>i0d0oTU~aAYIbZS
zR6DiNs<U<(Z1aKm)$&W#Um)bxG(-K)c|i&aD0+1pP4j7I#q!0gR~uV><95*a6W1}a
z2N6jiJzX{Gx_<*O02m2?<XV@x^icPcnSTE09|!(79gNan!ax4SZ&$>xSHP81#Cv3-
zmlsAsE9FhCpTi^9RE}p%DFj3Zgd*;hM8iQh-_fpSzJPR`q~(e%Vl?Ik!Xz>^#1o90
z-+~2tMYHLpHlK$m+k;cND2Az8xG)Emrr`Z5Rl-M)Zzrd*U2!Q?6W*r+q6M0jGuG}W
zS<WclD|)uaPV7&9#@!Cq+}K%in4px@0i2arBJsFI9Ub209YWC?Nes<<n>xcW-klDg
z-OK7qdJu1^IsH&%e}%Azt=aoy-zBmx-axh&1oCz!3tdlDY~oQr6MG7B*SJ81{w2=|
zx>QLUxgOZVchI}3(&PNG#mDb;T|)8>OPR}_OOObMM)L97z-!4iFb=fz0vtOHq#tg^
z*&Ixp9K2MtyEP3YAecf&#A09S!Dd!`AeD6XXG8k4`qx5Egey$|LcnMjt^JeB%x_kK
zD7V6AJua2?jrIpEZ}xMm^4zKqETW|{0h&mG>0G3a&Du%UL$Ge|&p6}wwhkH>{oQ6>
zdEspF7mwwz{XjX`9pq8feZlheY{oJGa_%clbD8^sT0jt-28=g3Yzg(EQ6s?kC-z9D
z?8j#2Q#CyY0LI16f94kx2MC`4^^mdNW&Xm+8Qi6d>;ZHQv8cN&egp_fLNdSGeXB%+
zS=YwY+Z*A&LHbi>O`ludNSI<`;~OHQWVdp$dvO2`z_T{D_x@!O9;2;ME9palRpXbU
zK$vmESzHb31-x-5NQKo$*q^!b`*+?x(JXEE4xn!rs;~O6D=Q|fsR%TpNEM$SzRQW;
z3KYs4+)o>K#A&B`xFB_qL@fV^O}LI(|36|A__Kj>#~Kz8q5_hFQ>KBzipLhNoHnl{
zY-I^ZF)Jy4FhZCz(aA(81Gs(b#uG_ay-A1)12xw|HD>?%%K&g{Tn^4hJmKwJ0%M42
zpfAI@l<<k{gAmy>5WOUxoW%jvb_mPl;X-jU!7m{k{2!xb3RhKg9(*u4SdU)^e^9!}
zjrlLr@b5AWb|41Y{HK)a$FKBr_kr#vIC7V?%YB^LhGT<C8~BvR4qadcGtPcSr{iIv
zbPdd!1eMJv+Sh4y=`yIRL8vSIBcWSYc)FiKpY&#ER+r1sFbl0zubpE{Qxo0rNc`GR
z{cjS~-rk6AHNhY;H{EJ$yM^m)N&TkK(R)wvED4Ln>A?gk;^!J#NMUUhbF(*BaWy@~
z!dN*KqWPeGN>9Z*DJzDlp5ayQ$Y8B`RuQf^1fNiFz1O%9AD-ps&+Zq*q`;xM{r-xE
z%WypIskXgAkB+3TTXvS~wa>5P>JJv{1t5zd$pab4=J*Ob$m;2FE0qV($;%(Vyc|!(
zv=1yUuYG{63FBk&o+<Y<{KR;U#N>nP&T%%?n6kwSqKk#q4&^2nPR5;o&J-Q_hA&wj
zH-Mr#e7wkhVoO6dJlll+Ec2B#h^7I#v}=ZEVQ)x^5YqU~?lxc?(1wV|=MYb7UgPrE
zL00<+-`nMK{x9HmOM9d33hcU5Tx`xavuno{fEW4*eEXJVE*Q2Q>6(r=8FlcC^#d}^
zClF{@Zm2XBzqF1nuS1rrjn^X~CX}I5<DqE)n(~0a4F(4bMF+N8?GOcKdCMp;^1G2*
zkw*70EN`N^>rz>#>xSH;FCYMS(L`|e*&SmP`>&DKzgt|ODC#`X2~sQm_AT%IZ}qIr
z5B2OAOR-w3BlS|hZc=FUYlQ2wc}L21OfNc}1hK!|=i@`>zQ07YPO(O{{XS7v{kk`J
z9&^uBvKL2>!;feL>4rwRYBj3}ai1j^lqK_{wxCpeAeD~tu-dops9LgBRJ=M&gfQ2`
z5rSFE3xy{s5A{}&YuNa9WlFo|2N}zT8gj76oiU3Fo)-C(wo;ocYWa0`!`3HTheb`E
z{RK~h2;or0LMrz%C@ZTyfk?;3<JNnjCiTbb5P@`hE+b}H8T4ANVGJB9Er%tvZ}e)M
zAUmbb>Z}@&?*aw|x8ddir_XhrV+|nryT#ZWBX+YIk}uXQSqidzD!1t}g>?Xb)7LH|
z#r@jdnWQ#8lid%W{Mo~lV~h2nP~bMm^NxN!%$HoDgg$zq!V?S5-b!!FV_!B9!Kfuj
z5S>_+_36sAj^tLVW9Lk$E~|UL*~qS&66>Vb)?^1F7YL)Xm~J>!&1b$s$;aA#X9R+T
z*yx7bJ%`t?*yL6NsQDqw{{ZN@RqL3pe`$)fxe=nj|8tlB{z~{>kaa+>1*b`;i%^jZ
zrE)C)mdX)jurL23l|%n~Du?)wRF3PI-EfO~^j*4uW&LN2w%SB2pU_RSb#o}=X&qB_
z;DRkZr_I7uI33v!PSFuQaa+Blt?ZX_4cT!kmrYe1nwc293_cUhxlT4aX-#&}GAgLf
z9L{G*udh#gV0{*m)TD2um_TcdRj0(GS5G{Mp{(DEEXe$*DQ|9=WZ`e#8Rb@1e^5rk
z@z8DYsYxmT)L!v~X~>~m&iNngzk1J;#Bc3LNpmJ;^93(5lWPka^h^WVu+1#X<&RqD
z8~xrtJf%{pW@qqNBHTln?8VUw7iBTC*%{IL(Zs?NQPgyjtN|gb=_%-Yc<@VJf^@1{
z@p0!PNWj+Y(81kgN|D`wT+Htl8=?eLZghWHNcZ0f^=CZxxqfz>j0a+;gT9o-2YtkD
z>U@Jcpl0;NUlyo*QNEpXK%6-Ku}?-ZbsZRrUfI=dq2(5xni*55HX)Nd18Jk$Cw&y-
zkA2y2iAMJ^u)MtfdKKfM&Dt^LpN)5@JP&qf5g%Cp^nd}gC>rpB+2{H4)(vzbzb1_g
z|NluEIaH0a-#|*msl#n`>5_blL9|MXq(Lb1d5l}$B1iYIIHPs|?%^JF5Yydk)R-d0
zvUGVvuO%uv8t91s(%PY2q{t^Z5PkA^7o^|GFa+^d_1DYE;LJ55@3cG{a=mtkcQ4-z
zj-CT&d^tGVc?ZWN;u9p-=L7>kuw$g!n<T2Ecfk&BMLpu}?AhSw9gg+y$zVq&`{puw
z_)@|}nPT!6l2mI($&PjmSGu-={?iQr+onFA0%km);A*!P50>xLFZ#t*BhTs<rvXS{
z>Mf1H&GQk<;A#c)t7&eI-FKW{?i*8{=X{Y4A&LWCzP`y-8Kdyt#<Yz$L8F!r!YXKB
zW)9@qMPVHVqw7!BlV@eEI}`jt{d0$L)Vs+4C6$Zy59g|BJ6Zn1idvKXqdti`K*JY6
z0o17%SF8<{^jw<DcrXLj?&^P5y1(9^lmS*N_gh}N4eN(WnAOgrqQoQ#^O?vH6|m9U
zh`tsf3-vLIG?6h%q4KS4ZhxguSH!j?DRx5d1NHK9sNc?BUr)%;SRHu(IlHihIST^K
zV@#_WbuhA|fAc{o;QqRrG)EiQiH|Cy&EkwtQct%SaIC?CpNWxnM=vV@ZtQg#Zv&Pj
z5JU6=<}s6gIe88CsWsw8FZ=J=D0zd2|KGAvz<YV)Qks+*51kFcp-;p`Y>BpS_`*Me
zr=+tZ!fp5@FoG3)0Juy&h+MG_r8an9dZ+rpX7By!2nIoc*qbyd^cxcl^&T{4S-z*}
z4#3@8Xa^)$Mza3=`nMH;C_vEbswW0vGPIHI6b%(Tfp+)^=vN9(GQFgrl<AMc`)xe+
z+@pjy-kJt=^KwdIhCR4ZAw9?Pd)ei3f8(m#^J%=Bkc@wth+{|5U-yojAfPG0gcl&c
zJOrVT8FC5u+1q101k0PO?s6KM#1S<QSj#Qy)zTP`9}W*$k$mv4Lh2t!EWx+og$vX{
z2>1Szz*<73guC5dVNrX?aH2R4Q6ultHU8IyATteF@fml=p7UY176+(h?!^qQbH284
zeT`hQWEV!i2*F8&Zaod=nLI9jimd1FJCo=;9mDt6*!#&9jqF=Wd5lx5#KKe<SZBpi
z2mLhYxqB3ZA=Cu8&XCVaHTYi4#Ypcr>XgLQC50t_%}q$=E!ck-Si3HGOkMF0)|Rsv
z=Yd|JSF;{LIGSmo)@0gFxzVR_LI}UpsxV5c{L~-#_oHz9$L4r5d=|7*&p;S`Z)E*y
zu;3$YO{tgVYW|&pJE+3jK_`s|kf8g#wI4x00#EY<JRvOz!Mx>j+8cw^321+PLTWsS
zPWn}>FwF(<Oj3+*!+4%2W`0-)5W0`u`huIpIcYTelg{-X7F~9&*abw1(*Wd(^Kx)t
zcgcKQt4Vnj4WX@~O^MTaesmX1`N{GN!FI&BojyugBr-KGis54oZHxI8UiDu^PeKf&
zb~r2w)DFirW2GuuQHaEv{}1Ui*Q>v#&)EM+pUE4PVUa3ebj0`_gyOhE8NG8$8SI;B
ziaLDFc|*i-wlDoh_HN{slN#Py6(U5hDjE`q_tu+HpJ#0fEZe>3P=~bj5F}+mR=k_D
zmlvy~gZ~tMhfu*rd;~HXe#Ot!5%M%Bvio^gF<2qGQUip#BPubUXC|}ccWLa)lIM$@
za;=wcf<jV&#jZG@tyg~I(IbehOZO_K*DcNlXnXj)S2{j_e#w1VG4UzcJa6Z!!4Sks
z*8@GZ>UC%WyM2v`9`Jz>U#TAmZ$LYDyO}ip4Kd^ie)z$N(BekzTQ_(>jcVNnk)OyX
z_iEQ47cCXwwrkG8I_mL5^^XDOd%Si(uc)pVc|_m`3!n-jT`m_`GY5nOD!E7l*PYI#
z=d{ErM{Z9&P_JtLamRreX6VP{8{|BhC{a7FF<puBbr4(veU!NrRl1`Z)DKVpsnCf!
z4+@=C;>1&K7t+T46t;J#)rBlnI;YPK>$w5X1C12z3>`vFirUB&PxO##mrS@I6^YO3
z#fEw-=I`$1=CsCehMM{Nkcksldt)fJ2V^<K!Y%##X|`umMn2!&_Nz}O$1lhDZ1*he
z6^P^}DARGF&D>Sm?_x3!8OMgeLVcFd*|s`?3DS5mf(Y^eHfN5xo?=oF`H3ZXp|t!0
zw=sboW>wan@*O|6TwW|cG?jgmHwNV7#6)>~6`>nIB6Lv7>Vld!0s2K}`GGXU9rAXQ
z%54$h7Gi-Fq9ED-U^mcMMKr}XKRQYEtS8B<*7CZof;Cn%Ee8drL$I&@x^*Z)o?$z&
zuO^e<pe!G{8UF%L`MIUBy;t$I`TMQ@`>n-bk500D)`dKf#%zzk`CU5f$@@it$v9f!
zVCS6UXpM0Lc%c{{fEdlX^5xGPAIL=U@U(F;wIvPkCS^vILETw~y4B!Af3rx?>70b#
zU>^XL``~zR^N-9OQz_ybt#>k}#f2ls0S$A*<z(WW+C`(_75YCPtY|}M_Re?a2$;R+
zP_6t^3{M1!;d|0+bJ^<CqIjv-)YvM?q;|s2_Y?#_Q!+Lv5~x0C-<)m9`yR`qqpkB>
zEDyut19H#MgoC+F^`#gGSrK<q-_9X$Cy1wadIWcrC*7XxL2Jg`*lu8NX`&4sz4t+>
zRuL_fBI#p%(mS!c#dW-xdx~agW+3C)du#g?1Wre%$bTq~{AVnW<D~44ECXIwga+RO
zywtmit4DaLqVs`}VMGIm?Uf=*xekU-)G4yApj`mh?Y{0CmwgK51E|sbpLbd$c$WB&
zE%0V2rd2MzYWo1O`M3@D!AZPqe7HI9@kz7&Idz3IHOsfa!FfS4)=__aUOX$+&kLci
z`2|3}Miv9(`Y6^0554q9@tx>YoLS&y1!h6LX^t;z9B2EU*@F-sOaThmvDN{Bth{_#
zr*7lf3dK2J(5pHP$n|iqGQSC$)Ou)=A@}9Oqk|12kMUrrtez{r1w|%|_a5UrnsQ+S
zdfI>{F#I$W)pTTbOFwZNgk8PdANh=b*4F-Gm#PId<Q_PPKGzUaf^Z>iP-`hAn0H9&
z`S@Sn|HlHj@>P6sJm4i%*)DIp#-uVmuzQ2|v)fyvoxQ8BNTE!Xd`MNs6MZ67*J>Y-
zBqvq!U!r^|Jyv2$sh#IGfh_7|l~(l#hdf~BTf4`gPq7=+*GD0|Kg0{wWx7|khv9K3
zkukQnsq7BN{%ZRgne!Y(=8%7n%&8CX>_3!q$g1Mt-UVtmTiqD_<tVS_iqu@Jz!_;}
zb}0(nP!dzRns*l_V1&``!eR#d3zqG``d?ap4Nq{rB6wpNJj~xOcs!$%P^T9(mNkQ?
ztX;`l=ec@0Wf#N_UGsSOLG`kN9uceqN<vbCCj}6!1A11Gw^t<I^U%>EWqsNJD*$!k
zhdM(V^H%TpW!Eev$2Z=@Ip_x@!!Dkm|GaS@a7;nU$p`R2wY8P-2YrsSAPM@*a4CMV
z{s7_G%+20qu<<Cn4_vghZh3Y9x6U@7jr3iz;NXK4+}@etJe~P-KDC({*Pxn>svFyT
z<I!D7;(*gXuh|!5`5#u)xJR`84HZCHBou4&^yl;RKg->G&u^8{JVRe29aBXS>Mm&~
zmAwne7~`1wY9pmnNcKJIBr#CL3Wt@7WvkmS4GtP&To|T2dRC1E%~w^bxcX|bi@m*V
zCGnN*_)p8gUl76Ge%~6Y;0|=EArASa-VF-2CGWH8l87aqAHEkF>b?OBg5NzAImR80
zB>20VT&wpZT!4eh*!ia1dYC0fFW*4JD8+4cvb4VF(sPpC(uLT4Wqv%(K3U}^a6(?k
zY;8{*4Xu#CiA-)ZS^JykY>(l5&?O3f&9!vu`@_=WCf#JBTkX%BEP{EB+f(EWx?gla
z+|2+2XK2=WDdxJ{*NqI&x_<%q(We*5S*ejH0<1uW)mca)?=1(c(#y-ehK&06-h*GP
zS@~(eg_l~SE+3pE@(=Kzztq|Rx@X9p4Ji9$`Jys2Zd;bM@g{&lo{6=v>X;}Bzy<dy
zA&VTw<^{k|!B=n$DDviwy?aDNj}#RQLa2+&e?hewIXwL84T*pm4M#8S1QF7Yqjwc`
znHTCq)8Ad9!`~O$j{UCIwGx2kEmU`T(8`Y8B_eqc0Ay`Sq->%~cOI@`FaHXv`_-%e
z^@7o60LqCbIve^v!g2x!*fy5dlQJKKzlH@caE2-_-vfj_kyhEmyyyl^wuzj#cVjpu
z?u}y0Ln~bvS0}qodNs2z`(l<!dF4OejgfBO<QWB@-z(0V(#kce8!o%MM7n0U+<36F
z=B)N2ilsPm9G(=-g^V|k!YZ;dnqdj@!4PID5fvc!NO-T9=(XZegaV?9CscB#EVHtA
zc!SPD-p|dP2H{dt%el}ZU8>$JN1acS`akXlqGschoyi2drW%huH)b9~$!YD+WXf0L
z|Gu2h$j3u-4)TMzY;;J(LlF_H(n;?hfqo%4s#zg%)Z!za#Vh!m#}12b%0NP&qJ#PP
zCpSw#6$FyQt}o=ZkvzWJ-biQXVX4Ij;d?(oG)VL{PF0XyU<}?f-t6}kOW)EPJc1yf
z!Mp{p-&hm-82qf6k(n+FIayCKV&uJIF%UOa3f$7ws6wibtI*4-GcFqXg`tSEHhsj%
z7af4s+RB?wA#OfM-O9Iqe5YoTjs0a3waikn3$VRVy|VoId_7N=e;<@s3kuAiwN5gp
zs5Zrs{~sG!>f1)niSKcCWgEuD&o;<`hODvTWd|;jajPr|+4rZOAVr*qK}*n}-jK*1
zo&A>)rIn98WW0!g-|AZ)`4Lr<m$OkW7G|cwZk<Lij<Pt5s3_#BWpB2)qtH@@?|sjL
z)OG3!*MN!j2O)cq)%3(m_e>*0o0kKKO-%DuqMLLDHQn5Hr(Sy&hL_fJz+56NZ-fO`
zDA(N6jKm}9+D1!CdENS^OkGoZko97DIXWyO2bh$66{?uPF;Gz%#T^zU5F4?4<#-f1
zl`t8jP$$Hr4c??@1448-){eZm;UV32*?9dSyV*@k;zS@UeBo``O~l^dPqIz^A`3F+
zLyWuAz>UEV_jp5hggDO+N*TNU)tYf`dnG~V$}>=!8MT9QwZAs{mZS^4^&aT$%Ve{^
z5J*|?u5X20m&t(nKhkXgHH=?@0XFpsl-3lq1YAx&@`F*HOaf&BLJM_p=qASn(+gjA
zp#&z_MH|UVpde_2(#cY)zdXJs>7o1)%8Q_KO<8TC3xT3Al&q~%4Gs3r|2T^yj|IF3
zH}5CkNKuDWU+iDF<+l28(x9x*z8guOZzudN*#B`^|M*Xd1H=fvZEt~`j8sLU`M2I*
zoCq1$59bc9QH(J+aNGE<9|>#JqzKgSQSJ0RO0m^$KWn>e9xkG$ARk=SZ;)E$*_<ZQ
z&jMq(SIelvu@R;oc8aPIjlSBaDIgYmqYuupZ{CgNwQ5%$507=<z*rf`R3%Llu(uqr
z8a8A%avIA>ixG$UgrA-{EbY8-9TN}}O#^4cnU>axZ#!_9%LZ<FJ_`9}3L%=v5YIQ#
zF#4A%q{u}dl+hsHnn<);hi3!2ZlwSKj~t$wU^mZ;cny?t#%r#Cg&hw9JzkU-JO)aD
zphs&^!5lC0w`DeOTkKH%>;p6o<lJe=gNGC~(?L~VZDGyWAwkAZ63MjB4;RJjzq<K8
zE!Gd#!8t1waXy0nxbdlg$Y~#ls_gw3oD!{Esgm8~jA~GH;f=2#)yb$IU%sJ}?8?lY
zjC7gkkq-zq&q3xdMb|v4hVRsEu*E0*QGTvV1oixivRkLEQlZM|tL3EfyXAzyPlU~w
zfR)I1m2{OZ#wIJokk~$-Y3H4awOVnbV_Gy`Z;}@b6HDq#_AqdP|ILEQ&J;-fmIVV&
zYj$UqtyX#4^f`}Ujj}XyG7<T^w0CpeOWJ3cRAil-lMKL|y!g;%5GjI5|7dRSQmeR(
zlJ)+(O|Fu`_T0N*il*~s8m#vm*zEFYt!(?_6WkUhP>eokdz=N>z0xk$eZuv1A7+Rh
zANIN3WIiFJClRi{-VW6$>n{a;Z#_=yhNEo`i`o$!rH0#F<wPkyzp4gA15f&dpf@M$
zW-}dvLTrPXD<${SmqcE3a9dvULu9l-?~!#76qY9@n6&~Xbiq0@k*ZKjeSJovzS0l`
z2DSwkIlw4cyg*yS2t-LqPy$J&3yVF4>~v@rUiHT=h@=mAzvm!2=$p@<()Twz@MTR4
z$9P^5+}!MjuUyEgIV$?2Hu|9^`A1*qo1tQ$D8<W}t$g^7*Yeu7>+v1J6WjX<Tf%(b
zvRRyt40>SNcarC&9u*c~ia@B&#|GOpD*4Gd)aZObg`)?h{grwFhI_0o;}?r#{BA9d
zWl$+|_+ZP6{5A~ZL~O@bfu8Up+wC9?;myOZUeH?TD^y7t3r)~aeLkvDOiNwd(+n4=
zsx{PZGqhT|FZaHLs#MDUEQk)^A_bancJYO{z_Ga77A98BjEI5Ha<~9tDjDR{_l@%9
z(H+WZIj66<CPYFn%o&!?J8HY&-l_PYwFe0lW>Bf07yIC85MQgU%^3UYqd(|8xT`)j
zcDl})xthph+4UTk`r>(qC-6p}cJydsIRY<Z(u7DwF6-}ki&iBXaofWt<Zs$RT5fOc
z(RrZh@r0m9UPbB)F&Go5aj_C6Tx#9qX<(e~3j!1#2U5#d+-=o5_atio!X2JTX44zk
z4OtkF4^UI+n^xi2(;9!QfiKEBHv>khd{Bo1Z1wOT6$>LdEN*SYh99SUl?tA++dwKW
z1WbyH$Eg+eS4R<l={({v`GZ0ki1Nvy1M#c$(VK0Rs-k?Q+oIw#b&O&Pk;uKM3il{V
z2gRhO!Nk3Us76Ohw}pomXU)O(y4P{g-p}O3Mbn3EmA@8MmgSYyjlL(NWQ0Sm0h1tp
zXd@voiq)!F>_C`2iK$2{urY*j&hAl^ln&C`F+eo4gOr#$n60tp{CVQTso!!_2I>ne
zsJz9D6&A6|dZa9#7cSp}tTA{dJaVqfQ6fd246z_WS2S|a-<dGmQ=;b^+#L?yA?MAR
z+f+ZFoR}A&;Q>-B9ZjH4j*dMSHO|z9mfDB|Gn7|gq!kZhJoCNpx1EYN{3JJEwKnmV
zJ6Cu>3Br2}fZn^b3(ZZ?AFRwY1T3#<DCn%XIpO_ix9*}!%D8L{cu%wetMc_7nXsgg
z*<@Eu2j=MGJyVAQzIITl1Hum+diZ3Oj!Y$FL`&z|f!2HN{TiNikmq@}CR+Ia*iE57
z?Ma74kLrM1i13HwhLVltgcJ!Iho=(}<3clYOf4pWjm5lf3*!ji<KEdC2xP`Kt_)Tt
zdU<En%aY+h%$@Jl)0ik(RRwv-ZnIEgHP@7BQywzgsj!fsaV1DAw8N7OJ%?`P)$M2M
zUlmyu_V{q06qq9H&#u$h!y2nFAdHp9+6ZSM5;NM*m6cZRo!6$Be=!;#Wvw9><g;x_
z-M1Vnc{|j5{V$1`fi@2-yQu<$)+a!x1@Lso!*B7&Imd<ZzLMR_fuDsE<+6m&4gBKp
z6!V-AVgmH^Bdy@sp5=XBTfuV%j57G3>qHfK@!22S(U}0t7+_Ujwe{9**pPI4i|f8q
zjEN?JNHR<{<(~NDTKc2}M_r)XQ$23rDio&i=^tYQnUpvk<c9F}P-Hp1A8s&EcKaf$
z`$X{oM-u*hYgT`p?rl{TJZ3hfV`arLS!`Iu*@B*0Xa=V7oOE+Fe@Mey*UCbU-xBiJ
zutw^cuBzw;9Jc&!vH^B`YyZBTA;bNA+KG;Bm|2#<z0^UU1~3z{+8zKtE_V>l!7{Y0
zp0sOh0}$-}ovDl|iX!z#ZR(g|8qk`(R(DY}%B%zZ>q5MBzu_9*f!K2fn&_sy;d#wq
z=%fUtvnr?1#{O(P{5Zvb8~~sRY)bjFJve+Lbn3VVp%dLxT5CySS+u?;jFr%Z#%}$u
zDXKZ{`Yk#z<!&zbLt|SV{5?oF<Kwh?3l)UPoR_-1HaT+Z#J!%jwGaB(dQvTRaU77A
z%;*Io@p*&c^6LX4epcosCG7DP;cLg{KgY7X<N+~A1Pz4Ol-4`jn~$WC#$S1(G?G`f
z);yL$>Y0D2MZUM^usqH}C}yCRR)uXrMc4UDJ}v4^qCB>BcJN^5n3rx+f^?5lrhm0R
zPRCWS1wWBw&}s!tGn1Nt1<4m4enp!&b3x<zts~ZDY{WnQ+E-kOGbiqF8`r->BzXj<
zLu08Yka}p?Pp0Tx^lu;K-*;Cr4>zP<&XMFV2;^YBr1@Y6#!DbTSVKTuNKnN}JTV62
z>^G!SVVAce@mo+*?4n9-80_D?#XXvG9iy6B3Gg{^moP4Bj?$(BG?Jtg-_c3}e}gYo
zlO+ehe)d${&5@S=>cnbYmth9a=4id)o~PtJsuFNYiItZA({H$XhMLmLrJ4`x0}tUy
z)*4LF$KT(+<!@51kG`}PAg>{g^@2!yv#o`AyvOZBBPlyQIbALKL<(=1M_BtFx-Y@6
zBwd#E212`}KHg5&RNl^OCo@Z~yiHQ=X8dUPYbNk?hU^M*MJ?7EO16#%F)Y|vqZ$uq
z^%M|!ZEV5%n<Dogm-^R#Fx60pbTkGfx`v^k8#-D?{3sVoCa_9y(%&@Hq@m$sWCI;1
z$%8B1dnJ>+#_}pjfkwu}o?*xWZU#b`GWVyKjjMGKx2!eTY$Z!MT*f5&wOK^&J!22^
z?M=WqvLk8DhnV|y%W-eSZA<39HaE=AO61j!oE#&|cYgFYWLViEB$3|TF_IaouOiv)
zuntK!_n#c*bz60&eeJ2VDoi6-$nfc)eK}yj!v(|^aK+<aOU?NycCM8>W9b-D6o~?6
zf@o~c9PQT490Uu8!m^D0=fY_xAkO>*T+gnvPpEz8RN5X`Lo7TO&^7EJc<mbW$HRvQ
z79bDnc#GsAW;TJVl6z9Mld-~<@hXuIx;YsFm=xUYuVZeb2gglf9SaUvz0t7(eUj*w
zhcOjuS#{4H3K3W{|6NKnCyCLgn9Al#URPbZl|knAFk}w~#Y}Z**WMw5K{(@RQuLV|
z!KWP_V?cU5-0?6Z+KKEi*V*4e{T^3;fcnwcDf1KGttih0RK_}TGDqc0XjP~J#YD`P
zp;}AaR-10bwbH%5InAJ{^tv|C{?Dq<9M^0RuC+Wfax!uBg4OoV_ff+YMblg#34K7k
z{s4ktjOh7wI|JS3KfRomTk(Xy6OmJW{>QBadG=<e-8>;7M!5GrS*hyr3SuXMlksD5
z2+lj{iB+t9&5Y`Wh)PQI3HT=}=Ob%9%5&%0rJRnfa`ksjpQb?#GH%<zElTJvvrZr~
zA}u&tvL{7atEL8v<hb3!8$)#&SfnS6dt@ec4{VfJF3U?fboxMUlYt*@lcbZ!d?F7b
z0>7J0_Lx!a%#AB4OA%~o;VNOmAEkHjv2(3zlqBMU^oT{|Zi{{!d&|ZP<{<gK{H93%
z8*Jof0%msjifUfvK9~mwG4RzHt^E!@ObBkcw(GFjZY^J01BT<v0p9OV#Xo}NO4Gn4
zy)y9U4@5((2u*N~WSxSpFIK|v6LYJCH1GKO$6=Bd+F|F%NH{T7QK9gLD$vG_jW9WS
zU*2x;*?du|7rJ?7v?N^{4x`$=z!Dlg;<3Xpm(0*;&1U7y8DqARD^M<u@-0<cQYdDg
zOg+W~8BE&DWNWH?b<5Z0*du*kJLl6PoM|Ox110zLVwfKq=MKL0pl!ESTY}5oCM+xW
z>zkr&Qv`aoAjv%yTj9_|j9+tRqBqA8I|_^UZr6}mL-k)|U2WU<_l{&uZXDqqORPJ0
z-QlQ)332Jh&rM^>!zJ{%js*kh2JaG4od&)}Jt&d%0}#_n#HD=%TDd<|p-(xXhWIFQ
z>XaLpC{z#(uiQ}!H6RcqC?t?3k`bTe%z8)NpcXj{Lx+_txKO@3^&~M2-&<fQhbG%?
zK3Duc)TR$>x2>;I&AMO~ubWZ(tP2BjC-h23M9Rt(5cs9m+8apT{$U-Iq<%1L@;IXS
z>I?tL{>U!s)BW$bQi!Up+~=(6Pu|`K&~sDk9i=y!D7g}~?dLQr!Ok4*E#uZvoiT&=
zE3uuUSVzP-FWPUWq%20n6YB=@S?s6_E!x15bEMla^Ak97s4UCA=WK($1syqqcQP2j
z%X+#;8-5lFbf!?0Q2(#F?XORH4tfL22F#$KkMZtgL4%Ycuv(@y9kP)+2Hv}4BtjxI
zOJN8PQfy;uqJEh0T)z>WdgO3$E*9I3R+mtdDX~9`6Q;{3LS`%$*{;~RoO);B=)w&M
zZGN_#z&N76RCTxHz$~z2CT=!8`)&GgKEk|nsL6aX_K0VXQsi`Ozvrt6{!=c4do8vM
z*U)~kxr6h~=R%H7oT`hpF2v(kWAjjwRyazrYW0a>B9W3pjERO39;@<hcmDH3>#!sv
z(juolTjOzJ)@~i>7@m)~>Hga>Obj_Ec?^HY`uzKQ<m*A5Ih~%kFO|_Ui(~Q%FhPQk
zfi;2ExdJO`xwVqFiBn)wGo@;}O=oR_#hMsze>G3*v>hzH+)6^^%MFK?;>;ZHmtd~c
zB$JxYpD)Dv0Z#S=Ymahy5mJx4g%Z;Cv{Y_I>1M5}h7))*S}CrbaALG;Mt)+a@@Emu
zo+7jLuR|8cIZ|6);O%j3QPXHH6^UzWXJ=|=Tufwi^TE_DzS3bx^$yrqrotyYD66l<
zzzf&rW*#&ZD|jxVOU|b#YF~Xi18k4o`!Flquah?DMC^(h-4W~i-(L^^y)TN7d|L%p
zU!`r&!)_k;AYX;)wuxzlk`gE=))9yZw@;<FWfZf!vV@KvK`b!GIQZ`#M)Q=hxk+U!
zthpQ?^o?yA-8;`x(vq~|mmQ88M5poGV;k{Ja+I@bHYUmoVBNZk!oO8BO5@QAjPe!&
z&*sc8vH4ME;)S_JY*x(`kbOhxN+a(DfUKSmO1xkY?|}`;Hb|{J)j`v}jPmLae@SZ2
z?t8NOHGb<(<5mnk%l4b$mJ;f{0qqg7lF|j;tmqEDfSG2#!iwk6nW_j5jXu73-(QCC
zxB%c~)F8>tVfuMX|Hm!Ua1#<)`@lNbkR1DmmN&q-jIc;?1Q|r*(<0Ku?W7Tfj`_%`
zR-0>NZ?!RJ=D*Zzt};wo0_evGd%#=6jA9h7arn01mm3nRowDuMVZl$qO}ic0Z1bvH
zZJDdJZq~dB8OS~%30m*s0z&sLIc~aBHy1x~yVg4zr!&_W41m+*H;k}B%oHw9qw;L2
zuXzoC42`8tUQ)A61|;V)=d#kPNZ)tr`Rc4&*^QAl78>lOYjcs<KvVzqCAchc;@OQO
zS@++F#t)xNoM8|e`UNZHIR+WxwIsFq$^PT-{qw}UeiytD2eQfx{w9T|J<w_3O9HBn
zL?-#W$pn%_R28u1F%gCl!`nefSa~DT&joWnxhV3Jnu7VMkEZd}TD_XBA*POt8Kv;^
z4y5k5MT-`PNCb-SOQ0Bj%BJ|nut=%*bN^~y;36ph7-kg_ecIzGCy3#rnmY7JGm*3r
zmAU4*2AI&|x9wmJO94ZPE&8M;2B~g*JtgragG-ja6Ev9owJ!2o8%wD6Lk=i)yK3r%
zScLz+-|1ol@+5OXABv!TDL2`3@HE2c`Lp0!tt~Qf|8+)Q;fL)yh?aO{>i_pf_&%eF
zgJdlOMeYWMt!LewiZ~7kfCfVK9dCA~=IbWH1_0L^V;I&8O(t6E5hYdFrwC${BJBaz
zW?ABOLmgyWNhZD^Fm>;te=%LeQwx6|tPRSR-+8IvZ6K@nRg|oKyFI>YX?<vb)ef80
z`ZPEb(m#l>hf$Wf`?AAmB#XzCK{t;4Zee{c8_iC|)1L9;YIy#awhKP6rgp7tb<qEi
z7M33gu6n4y$~W~l5!aL2xz<+2<h8EP$k}bXe2jK)Ay3I>R^36ayYg9_Ld8NeB(|qp
zKaxe2dJCrxcE-@<_^97@#$9M<z_S1P^Uiny?F?*Tlt3ZTDAD#dR|D`9R^02r9XJtn
zj#FN^)<tTP#&&k)F5?2%Dp}{WpwMJ|o2kE@3srUmBl&j2!OooJN-oxeagGeh*eq=5
zt?4qYyt$zeh9t#DJ^qqjRORJ7xF>8nl5+9cuy8nK%Of9?hn!jwjWbO5-K`oFUlWo^
zPh62h(4e?zmkJXK0zONlh3c!SWD)tNV;I+KVy6aUs+QX}j;^HDdwFARmlwt})C{N%
zzEmYvc>v4*OqJlaaP?kUKj0ulo0QoxmHc^d{vSnz>2Yo+XpjXBHJ||mjm?uDXK%K3
zvav%#y4>poQ&NiTx891e5}AjUlk5``Cv8cql(vNShYlZj>>k0+9bMqS=tmP(QY<v+
zXysovObY#MxJx;c6!FXl+vr(f#@2_mQhS{d7h~fpXUf);dMuHwger-?_#p5#LH?W*
zddysfSxGjV)Vk#}T9nHDm^rnzMM4P@*;%sHI%p!pfTBL>{RS1;!Q9`8d#h`OGe8--
zM%ts4BiGtxgRR>@lW=PXAe=tb{ONuF@24!-6%A(~@$-_wkU&nLj%4GlSUAB&)sHZD
zDKhXD*#~d7xG{<H+-DyLmH6GBmb_QWw#^!9V$R2W=72}z3vT(l(DZ#o+g4JYg}?DV
z3g>s5RvJ?Qx@Y_Vhl2AI=W>Eq*sK*=?JR;$#P$}o(83h%Xn!n=b4&0jdYfQxSW=kv
zorf#2+E@o&KRq$RfSf~|#N6I1w|p1ek5~Am@llrcUvk}%#RCi@t`D-P>*d{Sb9Imm
z2r9R1Gk=Q`m3+Od=-p}#PxRq7hpEy(zDv1P#>PkJ-m23lWslQV)Yu8vh@|{1m%e`+
zG$V1L{4IE@Vkum1^Q&UGlE{hZWQ)^hj*kJX#Nm9+wt2P;labG4iAN+CP8W<~PsMob
z&rx`%cXMbeBbtC(W_OkG7#i<3=!5?_bv0{R*|e0}-JI4A?fwlue4|exv(jED6SjS7
zi-Syr;3@<khMDxH*t9A}HyF_Q^V!R~_q8NAo<+Sov^Fm|b&Ho;24UXMeZu)reMvi3
zrOyFw-(uFMZXYIy7AZ^g!Vie}u2oW_u4>fkFWniDo>}qhS<%#WCn`D{@u%zH0T-a3
zy>rsnyGAC;r*it>pP!=G3bJS$8^e8Wv6c5a>wvN5c8WqW;cakLpx4QZE5!>Ft8~#v
zycE`g4T(8e51}8rc9x7=P`}e!>mTeFSgli7TJg*#a%}VpN)YS=qv~O87Dp<v<2VuS
zJuqwOZxy-T-5l1@V$dONcTA|ihTt`6kGiZMu>yjiFM9b9XKU4Uw%d&*+c?gg{#1|K
zc0CzSd^>5^NB9sH#oXBVn<{nQE3jpQ(mO6;r8<BXa<KO<OF$|NuQ!B}-+9fVP63+l
zHAihUDFQN2(Eh)-D;R}=F({HT_c#f_?<jjr{0|KQK>@u;k8gE;)j_(@aXw6IBfQ(h
zDM-N9<w!M5G<ov*m<lfg$3Ef4G&@b8X*_wrXv-~$XhZ0NL}Sf0*zETDsdrtN-j${x
z^Noz2QOk(Y0R0>(lE&xZ?d*X)49XACG)ASm)XyGcE+UIY_Gc$;n{JK?-D}Yo9%Ut1
zeU1=$f>g{lh!eYKl9X0nVx)!7P)K2Q;i~RT^`&X5qrGjBJ%lah74|dD9W=UaN3k7c
z#8O(J3y0CjHPdsvN%WWj*XDX9hwNfdo}+p7pN>YMbHvJNOiY@55k~^7EXm&fx?aC7
zBs2whUr_@2ojz1R$TaX-*7Q+3-c7$iU{(NnI+U?#_XO$*Ms#nhUzd>)QKqONO$j2J
zy}MslhS(+B=^Vm$fsSb&cm-^>9u0g}g<hWZ@2?NJEo|>&5a;t^Bo5c?z%OSoZ&F_O
zctPCg*4PnPeq%e&5V>IN3vza63hKcb@~Q^?2p`wts}bbE(XYk0(l%!o8Jf$U$O8dl
zP8YV%(H|@LJ~MYDrMQbjm}7*!Kf6hYgH53n+?Tq4N60Ozx2i4WOP3Af-<863-MLe@
zzXzxN=RRR%f|dqF*5mU4b8s$b;#a}-^#VbV`~VE80LB%;@<v;XVmuM2)XPM#lME9~
zVaZ|Qq(%y2v^&JHBvbI$em+w)5ucO6>*}VwY|xGN^b~J3YhBQzL*0uOibL7?4Vlz?
z^ORwVI5bIi_qu5Jmvefz?kHv81AnL}Svxf{CK+C&H#EkKFoT(|>azppS1*ZsePhAC
z*&A<dby5Hl&Ai3B8UpnCi4xvL4`<HO9i`XuTb<!q^!Ro<YAfGnTlLn$xl(pF#?9Ho
zqU3iJ7Lq5Qj5w)Z|C7SO^d@9jK9_uh{y41T1I>Mff0mg*m=ED;7Q*8p?1d9aMIW^_
z!LtCNHUm=(oyZt$^)>S@;ZD0{hz`NkL-iB-CWtI*v4Zp%y^loF_?(OU(#UsiE7vi$
zrtvxfF=qZQU9?Nh1F4lJ0CJ5D<umi2SW7?v5n?DVy<RR)S{zJU55*2@R5{xsvefh9
z0zDY1Ln%ih0#XCL0`kpjobht+Z!M;|mShJm+xhqP_AUk&U}8{y=!OtYfA8n6X771Z
zm`do^?D_pjo$XWH!a>1JL7doer>MwD|A~%dfX1&#kpF#9ng&51LOJcmQF>U%C4&3^
zld_l31$~Ix#Ru_d!0A!Ei#rIrNhd`zI8Mk}7}dmsO5dUroL8kHbWO6ZsnsptC-WMJ
zF`RrFrIM8iLb<TEm6T%nARnYGWvfFX=Ae204o@_r&)oW{$fC^A(XR0GvZw}I3ja?|
zHOiFKZ(Ae@%<ZY>$7P+qv@6LTt85B)(>D!!>%f6yi7--(x;5**(&O?Ob43=Rz_{GU
zxc%|U7x6Ly?^&Id)Ev_J7T@u#kR~y_93OGwu4gb^r|N$cMc?m4wLpNJ&rm|_^aGoA
zE6@MjnZ-QcS#JPbRz$Q+z<T$9P(jgwpw#&~W~trf1m_1DZV4qB@!P~~UA)%mTP#gW
zv9p|OEsS0dB2o{+WDl7SN30DlITqrpVOufc>?~qAmi~5R;^u3UDuXHgw79ZTIg%4<
zyIC4)3F4=RI}kxrjhU==#&=hp6@8AXtya=2+CH?V#2d02C*71=A}h5nDU@Fv%c|#)
zpP<1R#7kNfG+vSQmWb9r;wSjfZHso+Z>@_XJ$h?Rp@Pb%r^NrdxA_;nUgc4sU!xAl
z423`X=SBFtxQq6L5Fo-)=PMi<a+!4d;I&vNwi@_qPy7Gj>n)?A@Y`--LKu`51<4Un
z5Gkc31_q=-K)OK$>5}diRHUS&OQgHIl!l?38M=oUnyEAY`+lGEoco;hykGdhTCByI
z`CYa5zV=nwdW=@NIn!xOF*;sVu+lQZS&c5vakT!oML6!${V7H=5_n>`HF0L0;wWs=
zSk^vEIAr35Ey%bmOdrawSe53PGW&OaH3lpb5Ax~2MetB;{rG%%olFrWo>=b5<1`V+
zlRK;T2SlrgJ$>%}Sx+&;&1C~@6yVzCzJ<b;7XNiz?4%E;-CpnQ(HCz>rmo4<ZPFo)
zhK)B4wWeCcE;+`aBS%x1OPh_NSM1e$KYtvn)X4ii7LKU{yjLa5%Qim4pOB%)xn>`l
zDtaId@=c6j##Y3yX3OlHqj3l6A0B>@Fv4+%L9^#CBU9^vi6x#MI7;2)Br5h*#S+8r
zG;URigSRyvV);J@(PNVDWFa%EIX-_PYKJTue`>L`-yHILWnT7M5F#s?(p)E!8dKa-
zpqgVwzDCU(PJ!;PFP`2>EA~>Y$^Vb9rucJL#%%z$tJBBZ=SB;tI~ytu<47@fQ@B}Z
zo=d@^E`*rlJ0KE{$^Y_!GsaDPSB|DxVb~(`2ms!xjL91^5FRmsz9a9YFiYkmw?&Y<
z67Oxa^)}G_ynn~^XY_;<ufQ8s*HolMNihwds}ZR}tCM{b-L2M?|J-H+(j|L7+9PN^
z#QE)q?pM!XDVywNTe)@rt;t%u1ADZX+Zd(~5W;Pp{9!Gdz@;I~?zI6@y4a4RBD&kg
zWIvb9G3apipH1Q7ugv$q1NH4w^-N4a)qm<e#+hqNapoy8*%i3t1iW4CG5fh==4&sZ
z{6BxP?{!$KT5!8=UH(`7&hN6Cgedh`;=~V}vrN5)SGI1&SWr2DWb*AYo$c~gPaiz6
zP-*D#AFY%mE*w1EDT_;KT|z_I{~KZYH<k1J4#u;eeAp<v(ZFZd7i9S>G2N-P&^Tbq
zWaV0=oe^`*vy*IKPw9W!iLUS)rZ#JBLY<TyM7$8%a{RabUim))Df52|r0yBxhtsp*
zO9p=mfOW>S)cef)-`?|>8SLf&h6oXY+epIeC5#io$bmTH?3r?%l}sS;36>!9k!v}G
zp&A@e7Iw5q4{W}4z7VGDw65zEDzSHX6Y`HhYTCWFoY}-hBsj7f47du_=<TI0%w!-4
zXAZq9CBW8qe{OL!7wiE)##Al9phtRs<a>y+zueS)m_9=@@qi`M`LFeK0<Si6Y3$nq
zN{Y`I`%5z#7STm+9W93u;wP$HKC>3Hd<yW|k0F-V1=%JB7<hjQ?!MQW&Hqm~H7Nhy
z180J%I%O*EtSt=t@;Y_SY|z+K;bsQtydT*7jbp6t%^`Nd&S}ARZTaY~5Ndh1bKm(F
z#%XZ+R<4Z7X0ep^zv;MtRU)#-c=Q;HN@IP`1#|S+3G!CzbI@(2%8)Z13`CN*P|Kq4
z893cmvy<@pg*w#AbO$+vH8zk$qlbsu(OAVtiJ}*@IPN>Ur8VjxPHGBcAL)lZd9+F}
zKo`6btoZV@UTxDn4BXYl8`;HVM-^$3jyu#h_f2T+S=`b#b?gdSr@nvh@mN5;=l-ir
zg;gyO(EdugEMUA?g(;Mo<+F$lTg1fC@Zz-8p*qkp)1oe@X<}l#{?s8^RezZLmEU{e
z<;8=Vv-(%Ne?IBMb>RC1$;TZ%@+NXCn8hVu#~}3fE1IP~8X1aE7-ob-7)jr6$S82}
zbm(QR6Z+EmYPOO_o$04b;H?Fl(~4NbJUYxev+0msl50xEw0-#ZmZ~2puhACjc!f#e
z8=JE6{UZB6_559%tW_nr3k<*aE+7kaY`x1SK~xyDKm}K!nc#Kgj!dR!=B;cRoy1=5
z$EDG04Si0SD0&f*PO6vnTDyF(>a&ypebH~L{}W~Ea#vV=sJ_(HwsFj2;M9!HLa4(!
ze)%KM?T5^oj3mEHHTjqXj855m`FUSG9ZJA@8QknPCR5Nz2#w;t4_Hetc#ajB?T12#
zQz|`))0B-!#Ruj@r+{<IGr!L9Rd%6-)!HMxda_oT#ycPG*L$4AN4-37n;k0~ZI*}<
z`*awK`SG!;{!JibY)l<`p%pploOQr!TNhN-eEJRV9e~+fi!9*Wd{lbbr4vT@3K>#t
zM+aGBjeFT@y^c2vI#`O*mu}>%{Q&wjT@~XosB?%L?2ESmq~c*r=xrC;!`sruzg3=Q
z=h}@<Mp61>khk0t+c@3Kv+)eUOx%K4MMr)axK%RQ>vC}`>$W$64|Dzvaf9v1TW%)?
zM{i5;Ew?$GR)xU+_#%Uh^}WBJrm;IOqp3e46~=g@DqHgX4?B}9Z2PmTcO*U>6K&<X
z%~c#%cGmkJ0a8Zi-6{^>R~31cs;|Al#he~@e}kz3=Iz;W=P>)0l2PRx^MM5xXTjBW
zczIW1z16JB$HyrE1S-3{^2Z%GpzXQ6b(^hFS2Y@kN5Nvy2uLMUT|cT_f+U9fKiZw&
zIBS43sk55f%~WRL>=krI*zq*3kn9x2t;00bVNRwg*k+(HwEf6dn~0rL<WAZkzUS;B
zWm)B^VX0pocGLZliB47?`h=Xt*s0=muv3%A44R(}4BXhU9DqPtb2Cub<Y=dj1SW}@
ziYZGbuZ>7h5hlP*9o-rUyi)irPC(gwEO`gZqk#Lb^xdd}2im34^v4BrE28i$8-A0@
zNnuK}M*lt4CC^H$+kP61ap1{PeCLm^$7mqb%iK%LAFG~i=IuJ_9{-P9UyHSq3xa{Z
z1LW!q%=_klr$c{0;3P<S{)+%h>oT5|Tah`L@INlHMbf;rVy|#!rgZ8tIJ=lKImdOl
z`uF<kYT-T{sf;$zni3n?)~=6YBbE%W9&rTO?@v%lzwZ0rUd8j*-k%w8C;O<rU*gRy
z*OGDv{$o~r@@}y@K9c0Vq(ktNmz!(W<<QyvM$){SvcW5+bE!kfB4*P&8RP$9cgfV<
z&8QVUprEhPz-lTk>}xJ#on_K*g#&Xg2Zx;wIU;l*59n~FG2(~snOSOfP>2yYa1wqS
zn?ebS-VPmR$MXG&J#4JvUbf6+od_LwQw;#Xwl0E-P*zh|G`wD>Pqpr}lQPI*EeeDf
z>4oh7Or=?|croX<9NN4^>U>@56a^N)x<yp*nF%H}rng5Egaj=Hl|qAd0RBgzu9tr$
zo1GrWSxN1_0yvmpS|nqrZe-EO<7+cxAwo5wOE&+D@}*=2iBN1o5+X*K0hJAdy}bMo
zkSgvVSs17kqz$=U(@IXR#JZk?9Wu;-eArR1<MEkh#I5-7S&U?%9?eHpte+p%qyKGC
z@vj{Jmhq16vycd2&N1>l(qrELOmB|t=D53_OJ=S6!dBpF)=Z@xt`?Nx1<pXdbT}@Q
zD8nGlS6tU{JyRvh-wtJD$4?>%hm>BTr*EeA`y^f$#z4J>5#txvCS8;0n&qppM+=bY
ztfm>?EuD)L?uI1?&ow%FDn(LM=)=xx|DG3^D}l`<uNkwU`GzCw*+qW>KsYR;V|%-G
z=nB%-;D76v60=7qR90Ve^I6^BM)a`2$lbiN`Q$g^HVC~jq$$6dtu$vV3t*9>nnQa)
z0=*ZkraXq_&sel)LnU;!X2p*<QT)5=2N15#cR~1q{J2|t_B)YD5?NK|3nivnrZl0;
z?n@G%kyP2(!L6;j+hNTqRnH()#K#GzbkC*jj=zfFgkzX}l(N;9SFMWZi5W(}qC6;t
z72>l@ymfq_1?+p#iRz7B31B%}!p7PfO|%_Gv<`v&13zE_YaxM<`Cmx-!`bu-Eo?R-
z=ZLPUvIsnfL_cdxcE;eGQX$s2CRfaj8A~fF8{e_$)`OvHaNF`mjCa<(IjcYE%nGZE
z_7XXge=VmFlZBu%BoPLQd2###VK@$(@uSJU=}e)x$#iFOOz;H~_&bXo@ho{<ncZ}_
z&wq_lh#rl<pfkK_J<532m^e3$RO0e=9W9Dd|L_LKNVGQKK$X|6&th<#1TS6>*!w#B
zeh{^V%oR}j4|0npn+f0HALSkn(fHndWLQCXQHPTVFGn=!dU*Ne6MJ4NI$rG?=ug2I
z`r<Y)7Yo_+j6PoiU2o6!)MGK2ndz^d_Dy`~{H%pPvR`tlS*0J~?bn{p#!%w!wfmEq
zX@yrl+(@N%9mjt|HQ3!}82mFH$9Eqz17>l%(7Pe?r7H=N$f^c9k2<6NdZA@;dt&<t
z^@Udu$q{te&a6{2QlnEmaDhA@8J(HdxLANQ&UQs*x1tZLHzKxHF7~)ko_9Bsg0O=R
zw@5wSZco`@U>dQNSiH-cvPZO8cY0HP^d6M{W<SgHv%ig=KW0Y!>~Z#VS;@Y@)DD%)
zpvE2k>e*TO0IR#pjy`#0eACNR)u|zhU0-U!usygz1(})hmj0;?Ow}@gV$mJ_v>k1x
zI#`YKM&PYU&;Ua<MTphO$E>#mqu64wzr>}-(m6xmEixsU5AAR=W#K#a@Hy*v8D2)y
ze-;EDg3@nNz2s4ipkwNbyMOFcU_8zB{u?QJlwTL}cOVt7MTPgX-u6y?jl6HVBg^9y
zoQ1+x@b1i{zb`dn55qNAZFVWQ!#Y!2N_Y7JC*`+t=p3QwR@ec--e<Ecqg0e*dowzu
zoz^|}0LV-GWDKch2+Hv5IrO&zBd?YokVtUR1a?%zUhP~ty{Y65tiudG9v?|O5H#dg
zxDScemF#VM^Fgi$vP*G&NMw(K@aTn4XrD5lB8WbBnnln!4Fs;*)6LBhnhF>XB-V9q
zIltzr^xbA(fnpke=s{7R!O=F^-~jB#knaVCv}UJ&46bOt3)<TkYNeoZ)sNN1xGXqs
zJ<Prh;44RC^&w3JTbHdzJnG`-ja*+IHAiJj2Q_2ftk6<2-qID8p98_IdH&nTsRPcv
zIHY$wx$CXmYmcYTAp(;=9;WH>N1xh0p|HrGaz-hD86*|geGC%212mmoi89tO_Rm)c
z2C$!y&dkAFy?iDVAk0%<3#zl(IvuAKa8ekyId+7Q{&(2FNPIMJMVJ=^L02goU*ee&
zj`_y}Eerp3Mf6VC_e>!%5Q$?&Y4I={9V4MRk8|E=uQk|xoyzf?gC)LSDhNKJxYUWW
z@--peDa@+JE!Bxmzn3pF2T8i&+yZypz>ehZ5!NO9yMKxEA;yxuqZeVnS9<yF&#JVi
zB7ngn`lL)AH#3Ak-{JO`DP0ilB)b($E$8_W!t2HMAaqf*cl=4dfQ*YRvQgqY<MqXt
z5buMzI%s4V!zL>{5P5dm%xCQ?O6V8P4hp);9}_iSAP1pseFxv&VNk2mK*JMdO!+%O
z7Of$L6R2E2hSBvNx~jpri9mz*lvOomCfE@7h5BKdJM14~?fG$Vw0h;Sj6a4={E@nt
zhk+?fJKlT0fs!-ZcT|^|9Ynr&ED5?Eq@y`aP5;6G3p~z0$p3EbQ2@RDOFCS<Ly3Ms
zZR$}wXGV#dT}!(?PolCR4q9hVkdi4>pT!1x%S~rjwzMB&Q%U<|hsbcjBQ*Si<p01&
z-&;MdD!~nzMyFlYl*@_86rjn3#GcplI|{hfgJWlw)<B=dhO;4qC}+;XjD}lh?+7W}
z?w8yutR;5k!)h@x{DS#z2=Qc(pIMs^Uo?!Yv#A4m1-+e`AY!Z(*xlm;OYGbtj&6OS
zgFgVemm_nq=72R=NcpmUw(chrbn0XEzSPE}#V`_D)Z|Kf$4S3ypWHGqRX<gcVDxQ^
z8ch^JN+>IC%)ZfRjSg8+p&pcdw))S|D_K;h{&MxJARlU=d8@_O?Ik}BnY}KhO*Zj-
z<Q*+0zL|D<M*t%Zb?C)Hpgwqu8rlK=G7ND9N{>UHQoT7=Z3%XlF8(G+EiL=)z4V@(
zFAe7t5|St1v<tp|c|d)g_(aH^go;FhweHR&3G4<witRYNgu}1uT__x^wnsPxtgqlJ
zsiJPbolB8ELl>0=8cHoz>Wd8}3lxF?e-Cd&`@vx7(tmjYsP22^49X9bc&u!(i8DAS
zZ1mT`ZK5B|rc{6}Z!Do5hTFA75=d2_IpN$|67W1Graf#KyTEj*?!epHDrGG)E_eWS
zPmKPm@4C(#Ro?<}nBA`Xn%eLwXxBerQDhc%brX9i`e><wdP}SXYa&OfHerUHT>Ukn
z#eTjtLY-{AEZ9-%Ug;p7K}b$*V_F+U<TQJ^rPX!YIi&1kTfNJT#Yf#o0V4c~$udLt
z!xbn3iGx#OnJhJ9DP}?diJFnvFcfuHFJaNj4@xz-Jd5{Y7@5sl0k?0ZnX8hV$_p$d
z!Z_IdojfIis6>XscbdYcTf>QD`ip${_)FMhp$Dm(wnb#pJC?F*yLV9Iljb(lpU8g4
zXi{!YsKj0uXrU&bE&N(tP0(l$-0LzYq9TWDXL?VtifbyQK6uO*Ox!*lPhj`Dc#%DD
z*C1NgEC;{P^QsZ`g$HMDvv?PHaQx@x${l6V@uC}?srHj38$nbPCj)uBZQ5Oo3~y_G
zq(vghtGCxZJUn7CJ>q%U?|RY~0VaMK0Z2K|oaW;{Ite~jkrT`7e>j#Luhayxb1##H
z3Ps0Xerbm9CTZ55dewEm8eK3LhhF8TB6>r9*9$Ez;eIyUDz8atWv_&CaZ}NfkcJcW
z5SWKlN$Ka<8|CCP+02^wN+D)+07=o@VFe`~IjlcM)f-r{{NG18?krE<i1&ByQV5Qu
zGh0VJU0{o;lQI|#iFjU*wLX*ng{RQs=8%LJ`5BD$Zg!7RYjRW%Dp?V3_|!HyYsT<I
zXOJ=B_cI05O_J_Jj7!bUSGl0~Y(IlJhk4YP5{~L<E9h-&nQXS}kke#KzVsqm%kKJX
zyH+RMb*^orB5s@aPq<w(W`*tRKtRflYYjxNO5eAqlf0H8I&=rw#x>Z<zqIJnEUzDC
zOY!!h(-Z6C55Ku#L{uE-qqjNJ;Zkg^n+1Vmfx`(|Z`l`N3eTQ59%+7UjQ16uEO%zq
z#LtUzk&TU&A9bGj2sZxhodI%(Vd12M8yjiIcwY~#WNMjhYV1FZIyoy0Tc{pqH(cg8
z-^;?sTz=)>P;b%qpI@e}_qTN`vNPy;*78+tcz4$m^7XdjR%vYsj#`4gjXf)Cj9oHv
z@6G4gebY{zvOlv#lI@nWu(0{PF)H?MhK2slT&ScZKDW2l>>^l_V)c1gBuiJNR3Ob}
zbCDJc%jT&oEerh#-s%G;+1stC$ISy$ES%@#tTcFlOiP)<%LD+3@*K~juJ=GMeVi|I
z9M2T&zMG?V%jw5(GAWn2NJ1f5KUz~C#-Jl^O&ZBSDIcxZ`|@5+-#vAMh)d9_$J8o~
zL@{E#my{<3Fqp?8vj^u07$1871E2C@fMZ6r)<Gk7PQBebml{~?J(wTK?}Nu8<(O=G
zs>rS;e$vCi#QCwF(sWGOz;ZaD&sd_nZMBtA+chT_Ou9&4>rLmBV<SlWdArG;hap<7
zz?OCz!J2ZRkIl}EZw^WrW4=V+<cW>DT<&kt8dC2L#p|Nl_rIVrwxz;G{p->s#{4Q~
zo@y)1*VTH26RIAFQ&9+(!>J2!PxkSDykG@`f8MX>4I*@~hHYDkg#r1=^&r1Mrc&=f
zDlC9v#pxHEH=9dziOEJ%P$6wBoUev(6v4Kmit!$=kvpqBg)et@#+TV@NC{=@;U4uD
z;0P99>vv$XgG0ZM^%Z2TDl{QF!+qYys~ent4Eh_oC<Hd?XtHOv&N`kyeY<d3)XPNO
zt)VWWi$^*C>IFd8=eu%9KB9BfB`0<2Nb6(ijM{0fqda@-;^`(H_2*gf{pufYMAr9=
z4rueZ^zwK(pT1aJA=p_s<vHM$csZkGx8`wIJO-1N&8z1S1)ePWa+sIuk~7%(>ME;B
z`PJ>n!>f*j42w=B9~a~+u7U}hqK)f{jfw!F*s|fZo{--to^D%;9noNzQ{E7j2|ML=
z!*%;aWs?o7FUy#)8TdjeN>|&b{b8X>@PNArY=(67hgK!#a<lbpK55Fv3}B4-fEbSg
z&Xp@4MED=mOy`IP<DToxqWBmuZT(hU=y53O&H!PePu!-0RF;aj=6=pjDUAUFctZL3
zA~Bcs0yI4sN#HZC*!gh^bP;_1BgNlRso^%y*YazBqt5Lw(jXE`#m-Hom^n)F9POH}
zZUsNQ!6%d~@BHo+h+*Xv^K&8;lNq?$#3EU~_mWRUk0OeuR+BgizJH1%0&kU(@kNQ5
zHGH@?$!Ea-GHG0e!yA#8ujW&)(Dth?N?S}x$9draWpsu`9FaE*;yhZohqB#WlY^zc
z+9iBS$)X}vi`k|O!dZ0?CQWATw*JhwnEVI*^Ve()syq~ef)N5pkyiDpa^}Lb-^`^6
zfN;4jndFoDaN=L^@Oy1HYga=NVIyui-D@`a54Tk!D320Gs-%~VfN#|?E?44rS&7f^
zGVe7j+AO7GQh<{`NyIap@1ZuTl?K5;NcMoC)1S6Me0hZj5rMRrIBTboQmsjWvadaE
zW$=zipd3AZ$G)9GbG^2VxnJKzf>_`my*e{jQw=E|;lZmXsw|s6hyGTbxBULuxnpO6
zvgB5}@70z#BZX=#^SY^Os@U++R^8?h5~_L>APlRhi#!PeMFEX=p%xV^DP_VfK`m#N
z(SHHI61?bOr@F3iG>xH9rlK;^@fFBm#wK#)(k@HT#|rVMG$8?oIV)ct>{xLa@0isj
zoU|ClIf4_;r`=VME<X9e6w-i<=$-7Ko`^K@VG6rU4{Fmu_#}<wo?WP&?3=7CiY@j$
zRj=KFZ+S_o9>0bzLbVuMGwTjMaB?YZp6}^+!zHfzIR0jm_&{JGM&#<=zeA1$rIZ*m
zZg3rNl_4O_agON+#d%6yH|jAd53)gGLp>JzG(b{?IsPxwHgaJt!r9y9gDcDp_{?b0
zsViD91H5M8mK=Sjuk$qz+OM<uk6*V}yh)yT<!eEDD~O!)QtoL4vx(PS0`MuB)afob
za7Tkv6rxu=sM52JnTLKRlivlHSDFbb47g!2zPF%;M#52PqiDrseVZf8miBB4p0mIY
zuc5qVENR@9)}@zq3aK5?SKH|DL8z5njyeBjq*06_wqZNH@dWC#;wfP2I%fpGzrO?<
z9*F#}0{dUts)ly=IUp^}flTDjHknh<dYEq1BcpQR0b0>}QM7f@pT9$CKZg^=(hFlA
z`ioiUrYi~{9U9`8J1019&H`vAPe0p;v%$Yn2re{}_6agr@QPHaGnj1rZ3L^1VU`Za
zaT<dN(KzGluma6mfmzFZdW*T*E8{#n0K?~P1;u6?7CplUX$6tB*)Nrj>Z%vlXdnJw
zSlvO3AS^b_K=E(ucPg1OYL<2deT4*1%KR)lHsmDSWCk1}2uCwKv#V5U*HFiNsb%fj
zKGvjrPnAU{lKfuzyw|#xb?J(ik3%Eb?)QWUSAMgg4+;WlZE&cm?nq|)plPa;-d;2s
z4Wr`d`cNwvVZP?Y_0K*^Sz6nl=9zrcM*(_bp(RjJr}~ha=eWCkvHRI$%~d2rZR70_
z|DVg3*H7bxB-*?4hPnBFRSg6zl5Zq*g2sJ2bnjvP5O#7qAShRxSqmYqJYY<Gck5=N
z)S|a$oqkSj7XgDet^!>qNN)&Q9|<$rD8s0rqgNSKb`y6m)39$91AWe}C&_esozF5m
z^$V^?sLFRiMl!{BD6>0!m48I1>3!uhf^mx)G@In2x;RS<;)5AH-dge;9c}0;wT34M
z9k`k>yE55)kqWa;_9xlg$r&5qkb)W|Uf`D`Qj%}!s2h0ule#!)WLmzFG4<G-7*_ax
zo)8(skI)_)%A;VCf|^~wROIyz@!mY&xF~h}Oj$B}`?v6c&yOfF#0#yInaTz&J-VWN
zk5>?@JPSLLXdQ(a7o5GVe4-5QGP;fei|HDVQJcZZyFyP3W0#X3dJ8pjVtW7u7Aj`f
zP0j+bf|iN#d>K$;L3V?8B;C+{%C5F9bKAiE?DKcTzkm#gah1+OPT8DPnOK(^$wmU&
zi`bf#=OPk#sF!OGNBTY4-jx!Pi<@9R^4X6K->GOMvKvzv?P2E{&Ca-l|BV8-E%;_|
zG&H)@`ap}JIP(;@ELGl`uU|qM-em$}+xbNoc6u6sUWY%s_DH&VFBLk0>HS$0S(nhd
z)q$?yt=-bT`dl_a&aey6dl&DMExu>gsTR}?-6~$Ay}(hn@-MCEpGi@yXY(s(WuoSr
z_^rMq*glF7&6DuY#>;c;j%~G(HlM6+r=*9U_BzD3Jy?Bwz5)ZK95|v@i%S>QePgFX
z4t{lZB%{`i@(sV5zwFC-8_Y%%ru~_h)|68ixsvJpjBW|%)=R6b35Bp6pV>3bF3!$x
z+bwbaeG<U=SerBErB5&+ewhDl3P3~>b>hWGdrTff+{je@fyI_O<qCC^sh%*RlD^T>
z!YGJf(sj-0);Q`3NLFLsR$ln5M2xwtS@n`W<>m#~WSbBH%D8!s)wA5dgKP}9UYVIU
zl6KuF>#y(1BTY_~UH1;(H6732*_%@jP(J+Y-u{u8ednm&^5_gwi;WEG%#^AfIeEiK
ziRo2wWSS?59)V392-Ewmo0*QHTe)A14DL#p9iiO|+O?8}BYQxr^b+Ull@EF`+ZQvP
zjh!E8tPj&hLbD%?&1&?~0Ao?X|3mL|x#;~fn&<ZSgf;4Soa5}H4F5q|cP{*XY*ygL
zLHR&O=U$cNFP`tFm}9R5J?KZZma`l#vGMEXD6}I}P&~6%+-tt>ZNY6}j8%I3K0cE`
zpgpxdZ|9EVQr%PybqOsXZB&;f(ZMEN`cWi5r0<Ot3knnbaZ=T8TBr7TGDrUXF$LbD
z2cil!F1$?_0}rG6<!~?;<d>Rk2}ik?6A9ExbbLhV>TatbV@&-`yQ4L5^pE(KQK#>3
zr-zthXJ>H+Vh(}PXV#x<_TrmkKsJ_viU{Fqa-BlBsc%eL^OOws+%>)V=aV2px8(s5
z%O$xVnLPIlIN!$r`^C4JB&zVEL~S!7=KmzEl<nH8Y^9!=j%?R^UuAyUxE2=svlGj4
zd*qzeIBKt@JyS6CPbVgUkr=B*^DP-?05Gq&W)(P|^_vRt{(t?~Igh!_sd|UxlV@ET
zC&LQhtdSQF-;8}VAM-rmdzewoQg^eEy@+yu7Pq02-${q-zKYv3HDKHT2B!R@`sd+z
z)HY(`2uq;`cM3rmzb`Fm;lkfEu+%p*g7;iciz)Fw(q>9T``;nUyj3}o;bge}L&}_j
zUMzTQ@L;o!5O|JPe6AG%872k<Cj=29bRx3(SC`n(S!#~)fi}O)&Zm7?>S>sFBV#}*
zn_v1;1_|@DB;{t$Vj1))E4`<sl0M=M>T-O<8=C%9%(hufN8yZ57VI#i(Wn1oXX{Z5
z$vC0%@Tn5OcbqbqP4oNpG<#A~349;y-okC<Abx%hxY?(3KVc=zaF2Tv8yJlFrsk>*
z*~flZxgg<qqBG*i*OA(g{~Yo)d~64wFg0xSRd_ciIP`Fl<GxD!giT%5w+cIm-#K(J
zBNC}2el<l%+8uN|QB$5_ZDTgexmN#m>V6wv?JC`e)8Bu3HRCvT7O`rIV@<Ygij#E)
z6oRtS-B6b6ZAG9v=GCg?_-l5tA)a@1MQ7M&KZwx8ah<yK92rqpCSnE>0^JeNLs_aQ
zsG0ayv96jR`mlM8Y~(N}NwR8qlyJgSWJ*Xpc<S+^%*&usBpH|jKTYtqI-n|`Uve3=
zn@;ZMGoUuII?(e}laK8@Aiq5Hu;MqV!{j%`mn@G+4o1}1&N3c!G%T)e-kIWDH9)=}
z8o871<2QkcjtKyvTmQ<WGryK73xNinlw`}Eh+OV}x^MIrYu$=JU3r99({gD0$lyaU
zEIR(6N3&A%`*xUGmXIgMg;2i)b{XU#{`qp4iEW43=BFh)LA)hZ06UW9g^*O)w!yk{
zm^I#j+Y45)>5Is+@$H-8j{a-%RhdTvlIa7T!=CuN2E{wOp!S_=i>$Ehpu)oFs-T*n
z*BFzGIXx_N4A<m468fb$t=yWV&4e745oRsY*#2LGmVS-|_qYaypm;@Xup?FK>y=fR
z2MqWy<e8zY0e(E&Ycn6rCNpVfP;VFy1@rfh{vrW=LJEg3`I+sFy^%|f84sUU=1*1}
zO7)6-dG&Lf=uv4<7Cob8=gnP<#gQOw3vgc%>B9W6YSijpE-iooqQ?t4vKWo0HaRz=
zWv)*RuI(9(tarHop>4FORl~;8iDc!U5i8+HEB`A00rs}R_AH}4`MK_E*S9~j#WSqV
zqSEv>jwdkALXBZP38P=(5s~b)9tpPYfh6<}BfUypkY^Lbqt#Mu*GY2uADkv-^4*pt
z#b{hp?a?O*kV*2yz`$=sI!4yQAJGW$sKSZFnXJl8&6VmGd6mK@@1>62ooIX=YA>l{
zX?FxmlO_z@*Q64hZ(#EiCI=&uEm5%F%%k)%)(}DklyMtu@^2@3M$=EXLkip!TW~zN
z1>jAPe;*lI<Sm{t;R!Dn^l}s$b@jJR&R8^ry_#N=65#gv=U9i3eWw*2bgk`6P2l|q
z<xGRFTB*DT{WQ8%Hx4H$54A)~&dyKr34MdxbXns8QnBix`$dH@HRl2%VH7!8VtCDy
z=}!)UyRv-fY3afSC4jf368JoiFqR2MvEMWetOSA{OV)?+OSL07^Rj||{bDgaEh2QM
z&7m<)Ornd<{k9q%W{y`6kPIPX&o7yp1Tc&2<L}A--t=xB>V<qZ|7*YSfiY1*xaOA`
ze?WSC$W+wrT~*FhyVs~Zy{wwWLQu`GY!Ds4d-&AM$|(tleb9=OU|N2s=!<B>V5%oc
zRMFedNF|K8Y|I~a*fHBY@~2|HGmzyL<%NQAKZzuk_y5K0wbJB%MJ5uHFXoPv`v$aT
z>Q0o1=;!;B7xtBmiQnlK_Q#z+j1mO04RHUFU*TIYbE<-S6l+Y@v=c%_>-ifg+eI?6
z)a<6h^~@xZPK}Db-$~(j9@%W-GJ~B8HmU+PMsbtAllW9EXoRMzQ}X4jm96m1uPr^N
zTp5`>*KoKqdkXZT;hSRy%Y3&>{bz(lPM;eQmc`8WllR)sl4Fw-|M=|h#NBW5kf@-J
z_~o<n$i04%1P!UnWe0Yoi+jseYR?@?>ktdDTu-+r40BOyOEME0-qR^-BIZ^3Y_iA5
z(|BIk(?OpF<-Zhfi`4O=R!~ZHd&trpF2b%Eljnc(mU?A(%gfMn@8;crdb{x79^DRo
zCS$7o8#~-V9R)!XbhcN5fODWp$>h0nWaUp5dbzd!l=j3#u@8!GF&PV1s>({9eudGJ
zky}hQsK;Nd_iT_$9aHQ?r^(icO{!Lin05NC2S1_Tu~Zmy)7)WBlf_$U;^QKi!So}~
zRXmWFndEKtxk)na6@l>pUa@kmYu@b8>bEY=joSo+(w<KfvV#Z3l1^fp;6+K3Zffx%
z8*>YdZCT47S{Xk5G=Cf=qa<voIU5qeHW_G5Xow%A@HTClz!$R9p6yv9FzmxY`jbfl
z0&P4LUf(TtUX;ZKrZ>hsPJ(l_VcIi2a3?^nnMlnlyf`Pq#;P|;UGgmRBK(eEMupH5
z!&@h<^e5#1jbr|M%&JQv5*rG!iqhFu-*Nd8IvB4~fXBg@;b<D<+~WAtj(*sB@00s=
z2=m&X3^MBX^o$<t?jHdR3!fF+nPkc>2D}WNyEk`e>ECR$EM<FxMG-@KvQU>r2S8eY
zu#w2ZT%Q`=#&9LmaJ*>R6Q`!;V!W7M$>BbyP>;js?81^bYx19rb`gZC#ggv*NbM57
z==D^$8;DYA!{TH4bCVd6mUXCCltPkTK8Lv10CAza+Gf1co?BEcg<ELbFdoz#CQ<1j
zcqG3Q8Hb+Bu6%b$IWZ2Y%YXaRV*So#xdYn@<>9gN*$;<hLs*^=N!qJ^c&z*oOVpbO
z4onVr{IanAJ^sJCfLG)d#!snFrxR{TRyWw+s(ONIF#|FeyUyYL5ZDT~M`2JpC8k_C
z!%M#G(&izsAd<Wg<%l~`*e|UIZ&7)cIx~HC2(>($UM#zcAB{Hi!v2|-w#s2H><B@l
ztL5{3TH0=~cIWQ2(Lt|42t@y2hj64Bm-}pcaeZd}Bz9$X>s*cNVSBr7YQb*$*7YQ}
zXu;w_>a|Ob60?nGjbGOrhNwCJCWLu$BB+DtnqWXLgx}jGk0>%Wm`|p%=A%~Xz-M(%
zp9~9G`-Mb+q)DEfq;C_RcM)M+NN3ni{M*w~9=q2Uc!)%G>dV;8cM;jX=LtJ`rU;(&
z_35C~lB^o{H(YrApW9mS@uNd6$KO3(a&Z~I_N6ChI(L5~_^ZW#XK+|C83#zVdCL*1
zwf!KX7}!sEyK4iu*i^zt;CVnf<y*8V$wk!J4DwiK!lW`S<4WF^4A=bi%rT3%jqBa#
zpth;l+oupDlWM4@va#n8eQ7YB@jxVEgBPh^S=Z%+<{~9~Y!LkH4RM&YB*K8Zf@uFn
z|GoLo6mXnLoP=BK--Kx0pWyP1jqfDS7ixb5{EFoUe-C6$Moj2M+q<!F(=dr}?}j}-
z&{!e^FyNL6Cg>W(ZU%(|y7xBwWc0YRXwv6!MoM>4mQuyBy0k~4k}6W(Ya5k*I)6Xx
z%T(}-;D~`&wwbZ6gn8OcW*G@2R~+x@lEI|&l%b!b7PbzpT{#Zg67D33fk<WDPo=V*
zgg6LAjklQBG@}VsFa04n97G<$;YNYzf~}24md9lYkzsVx@lN8+Q8!gSVQKZr4DWY@
zVij*MGP&0Lt=OTU>3yk<UdycaxpvMCarG<s3;`5-`9Z>CHO!KFEzFXjAQ3%$rh^Me
zKTQ1$1I%k#oxa0l6Se_xh~qw?5JX9^SoSyG`0jd}9!Ul!eGp{G_;cM#bK)fr4$<uY
z^XcshJuQKLoX3knweU?*om1XtPaXuNhn5kNdxNh<hcjF%+A1$%o}VTwUt~f2z_~$R
zM62-%@<P}2Bg4`jFX^59E0LUzeelr;TfJ(nL3o-<)BFbk@=xNhY4}&#SfD<ekBB9j
zsy-8KY}4SMRH~;Cv|qY*FP$MS==V)=8?bF?R~|gTeftg(^EZdqOM+N^ah8uu3ksf{
zcBK6<hxMLM0?Yy;J?Z&vsicJStJ3aqR$)No+DTRSE1t1C7yF0}tnW?)D#M`o)yH=D
zLCjS#v_rSIarui+!mz{C)3b*L?*mghX=1{fp~B}k%{|=Z+2WSG1Uqqu)lwtYwWo^l
zVYP0d(PNZ1*0DZkI2lff&Qrd{kv`VN988V$MY6)ZGq>omPi}3D3VQe>T+aT>G`3Kz
zOdS-?y$HI+eh|K(cJY4AY{6*r$@RUYhd_F^qwuQY7q*1=g|%6>^$YK=E7CkU01U$2
z$!@-z^yVye2IGZ#vF;VB2ZBDX2J$Uq!r7RiZH7|OZ4PyL>(w;4)YQb)Zwbfk%-;F9
zF(rIB(ZBnc%YEy1S0At12<5tJJLU`Jy>+kq9FoEHlC!85u_m?*&+3MlT2cUmXkpct
z(MV!@3|_H6`+$}@Jp%&Io7W$Z2$@xJ(v2NDx%<nEQxIViOl-&pTv**yUptTkki)f>
zatD)hw+N=8We!Pe4<D1_DYSccD}j9~DbpgP$Q@<BkVTZ0O-ys{5AB6kR2k~!XmNDc
z)@(~;w6W|yo!cPKT%Tdt&w2PR`7}`voU;*1$8dhIR_#!&=KgE^YvcnemYd1SHun;4
ziS>LQQ?Sp10{kh#Xi+8ZX#ETg_^5@CjF$awdgFA@*?6O_ly+Dz&*lr@=|$JQsB;yO
zZF$l`CTp!yZl1C{FvK=TyA^dRz_=J}*#GVmZFXDw8SVFH5WZk*i9_@ohvtx1V_c0a
z&6+Rx8SfU-QwRo{-vq0ezI`;Xg-^{8GjBdo_rf_uES4i`$vNtC!m+dAypr~i_)imR
z|HlNfI&1UUp!aXKt~2Qb6k@Xm0rFlw4jhcwPlKmwMj85I--!C}Q_~YP9Ue6%oH^Tz
z_AgO}er1t;kPAA2=w{I*dW-ztt8<Q3i&{XE)Gz=T^0r^v5Ddu`v-(avqYbz-a8%L+
zleBRsqY=LrQT|O^G!qhGVsdyy@aVCL1GQTVq2MjkqyqSByhl!PA2D>{No!cY_;_N}
z_9h#}jqpobR)93|kHaP|p)cpG8ntTA6oT9)ERps1CUTAfE_=^yez$d0GafI7rSR!)
ziH^Th!dc3*tJ(`JXjyV6Ou#-skxDTZ(~iC<nr9pc^jM*TnFbuT1P$ryES%8beNYAE
z5ecmb_+#<W_kqs8$jtx3TL0lTxAp(wg?`F{T}sym<1p<%kXh*GE80nOVRrxg8ip+;
zEmXs%EW|@A)qW;(l*oKCSqX_4;FtY57z&J4O}#bf(f@5Q&fQNYY`pwId}zz^ox1j<
z9nyVVm+i-~llJ^O9lJ2O`Wo57)8|{bpl%5xR8@d-EHt!hX4t4z`?SbKJP_thnv|(a
z%toyh2&Zr5%w1>sH|M9Jt2G?rKAwHxdFK?ki{(y>5kA~vrhX>j5Vz=i!D<`daNmc?
z#yuvz9*+rzx1G`va8z47W$~>D0ete6+m^x|{;7_Fkq827eTML@&3DmR_!}|Ml0c{`
z#B6e0)ITh?pv@*D$VPmnN&z#Ew_I6J5WRb>#pHV((=^S*uQD3odt342%ARTTi<FjX
zXPS?5$%GzXv>9xEr-yDbI`?9z<j)*;e~Q&$@ia81!g>Ro{sDjXR3Bo0=e+fH1y<*w
z=qT&tcPa)*%T(<bX?;{>GYag)^QYLnl2YWmq0<*GXM5Pi!QzsLKggAMFYw-FRC=h*
z^L!P|J#A%LZxrws?CcF29v_RQdZAcsBxT#~rWvc@;v1N4J_@{fwR#bE<(jLPsvYnk
zGxyu$73HXh^_Eo2+~eeN)QWp`-(~J87OE#jDxABUq6RK~JQyVR5&}=-KlqzP(2DKV
zI#iQU7*5DUsU@wkso#@2zK|h}y;d{`DFoX^;+l&|^>3j1yDaWs{`q6a5NFd?+7-km
zM`h}X`GH#mIn1_;0QGt#H?+!IgdcU9^D&_S9!6j35Auc8<hx!6LqBB_O%*-+7TQhU
zm!8hneL>kEc`Q`P#>5uZ`}b(OKuw#xZCsRr!<qkaENlZM)qoph(N(PW<-MfV`IE_!
zYCN*P@nZ=`zeAcb9oMKx6U2R=ONZYlRYB>xuEBZ62m@IgCM$4~8PMs&wO_bo;CjEL
zZg<$}vv{nDC|m_f;9;n-1_TOivq#Cg=E*%B4D}Lwzxrq4SAFmf<Y<3jh*O~yc;M=J
zszx$!sitdS0)i|nNMDB4;X-@N#%i4W<)GVQbfuDUX<gx{VU{(8BVO1Y_q#41PH@8d
z$fB2xl9#4BYNYizbT89sm-U{i7arDO`V1H2e7)k8n-|&Ew)6f0KZuAg8x%R)e{`dL
z3y;B=BfJ<^@%|?iciRY6n@`HE1Ll~1Ci8&=FwAvOe!{6s)>xOa@Q*x1vUVR79^gv(
zDTP>Dwy0X!gXbA+7X>(U{!}+?&W|4%h)DQb;3=bofi%#ko|qEV1Q6}~6YderTA%~G
zm{+oQUJ!Lc?FZK@Dsh!h%;V7uImYY%2h$Yz7f4a(5&i6TDgJI)Lh7Xn3fT(fWO=rU
zlzuuA-(aJ99;*j+6m^}Ss`_gEOnl$=a7k;*LdW04daJbs*S(A0-G<L=<5}*IFw}n(
zV6u5c3G<)-wimxtSE&6UT7KyT=Gd|3ookm~WT6~zLjR)v_h=g9KCUZZ4#>7^r@WY~
zWsMjOl&$KDCsgR-ewlJoZYD9cEy6g;{034jHUilU$yTYTOZ}aElY3aJH=OYreh$ww
zK4xmnJ~mg|R&o<-S<C%8Xkr27bCQ<}R3J?MlnVH(hR4|OGjh-AWuf?Z)m?WNY!C6A
z=2gUKn256X>in%eg=xhjpLDS&iqxk*Sx~nEw=&tkR8;0!yD}?8I9~WX6QK}&^Tn`0
zbZD*$>RL5=cPB*nLaSd)nN~6rlPXFfnUa_OjH5K4D`n2q?tRZ)I#ffobP6n3UWyu;
z{>hBdR-wLaEF^+Na07n(hc`pM=>*W_ELDu-c|F=P^tJmw7!S#)ut?f$DzJ*4XjZ1A
zUkA|t6Nqo?75J5KNP#wc@p%-bDEGVW*kgi^GX}y$n~GqP{al)MO-r|%27^~bmUtZ|
zLs4eMyP6RT*e|e+k}+hT-e@ALw#7#&%{9xH^cgJ<EQS-T3=AR187YI&gG2H0iUok?
zM7h$o+;5KIvs2Q9ia^d3JVUfAn_i1e0t>_1Dl1={!)XXz`MzSi<Q^S`DTU%+<Ogc;
zp1l^_F`TJA&tLt&x0}^ORAOwz`e-R`;ubD4cDU<kV;qR|KTQM<7F<&Q=^qN*Ch4Z6
zjuB3JQ(6uxF+)T-M=Q%OL{gpbiQl_t9pc;TX>0z_V1|O-Ult$SWNzI<9$67wl=hhM
zV;|oitNbZTqI`AnCn7O`$fr3P02~VkNvOi3(5@v_g0?RZ?Q2Ut#Ck`DI{cxlMiTkF
zxmY*WCcP%PU*N8og{5M)&yDf=lp7;Na(QLyuYB_kI}9P!5JKb)eNih8qVSR^yomXZ
zt7sM16E%Z14KlxV=l}E+23m#%1n=yY;8X<5nd={4`P{5V0iV$sBt9A|gM=9}+sp?v
z?Z}fB<b7F}5zIcq!yyFc6X0VX3PCi_cfR9h9qtd~@0W57D27DG6PLC&y*Pbv(@40%
z<w~L>z`U-q+M0MkENR)V=fz+fEz!MIl+6>CB!VZlr=~<=lOcWLcoNucj=$2XrsAnD
z+5BI{!MQ}R&`SzIr1LA#g5Ok+8-+=I+?BERFr)kbQ@HJT#^WgjxwTm=H@a@N;i*nF
zw>Z*<Bm1qzw{l<o!R;R%&0}%yKaSVMp=xYZx{#H?M3aS(-^<HHv=iIK6}wU!{V>_8
z_wnanKP4=|<pVutqz4CEW7M|tW69m3rE0e$T5wUW04tO~QfhA#Ni4ID=bjgAiQ&2D
z3i|%|*qap>rwJ?1IV1}LF_w`6S!#Wqb2U^($EY~iOYPCNUyFHl25!qJ)y%r#UE>|L
zZNzcafPQCfbipFk3p6~nh84cutH+HqE;Mytn#xjUH+oQUM7)BNmW<Yn@TG)_k5+&4
zs$A{lsGQw*R`t%dntL)|<lFy>>lsr#agG}-%2YASC$s*5c3^DIqmV>4a3znP+GfA`
zHBa)Pp=eOvn_|_)`nsPM+Z0e=OCD}HL#%_aikgP^>w#VU5!b?J)05-@_Lljlb5HWw
zY%~8QpsS7odIj2C2d}F3|5efO8E$ZWnX7jG>zev`y+gdN_vT|Ge%vtZT9g0|-CrZb
z$>9v*LX%baeTs(+`fC1OA~BX{>w2vdA%Rl2_X>JCS68J|5aVHI#pfh^2K$U3C-fTv
zr4;j}tZj;<qVwOf<g86Tvj~}r4kM<@Y|9Flk*sOk_&zlLj_+nQo^T-e2PwIjZ7m|N
zU_@>KmjQsT25W2&Z9Zy=T;mUF8^y){)zC$@IpM&`wt=-D8x-CkMS15>D7F;!+oyX=
zMS$!<yL(g<t7L9@<{!c-Su%XS#&kV#rmYL+5|hlhYQ6rvTq}=E^E$!MdRR4*M-=cg
zeihqTT4ScesoKmZvi}BkiKnbSI0uMxH0pY<jjAWCaDOl8;``$|66XcFA3X_9-MEu0
zhkxupT|`Kpt7i-7q5#J9zaU!P0H~WtFq7R-hH0MrcxTbSi6F@ELY8*6Ln97bvU$wD
zFKR*7`6+nbn&MeQQNFGhK~Tqh8ImAw9btz`;_fZ>TvY8l&Ke>yEY8i*Mi@}DMpXJK
z=QKd&v-XYpr336UcO4Yc$edc%IO6{d@I2fVt)ccH8W*eMy^#-GQbc>0t@@ii05<2M
zzhZH4p38j@7OC>ES4*!(2`5uSwXz+6k{yTj4g;`UQI>?3naU{M{EOH?(8^x?mfnH_
zcwZ5~KrBV}K3I7o+#7=E`0x5!TPLd3s=Lb@7Gj8ec>aF{q{%P=^PZFpMD&FZBs!9v
z`yNg4HI8~I8@d}I`dd{ugHPgH$aAW<eP)&vf+qiz8hECSAVG9&-LUZ#Yml$#&D9jJ
zPG?${OyRFt;QQe==<C26_oE7gosO`*QxTW=xO-nf$t(sg*H3^ZLUO0Nylq*vF9a>T
z@U7jZC{5*<sqnRAb{UMit5F*wza5FHJ>03#>efMRWUss!z^M^MonGjgwFM#A#95++
zx8zQ{<*l*k3!EB888SutQ1Is(bj9`0y}w25A{0)}YAQ_RbIjf1<JyaNc91)<J0+JA
zJ8XVb<!MDTNjX^_bbM3dDL!ow>rCc=r#7>K9lq;7*S9qw3V{G|U(9pyP?qPfA=&rc
zbAxb`uNFK_pN?O8@b|S3QaSyZ%DqnyT?sxFmDCErZ`cs`#<@jxxQ+?J6%MP4MBDk|
zQ1G>@A2_6@evOv-CR#4RxvX8v6(Rs+2e+aMBDU%qgRd&kR{ZX}<Mfmg=i`m4bM~Iq
zT$w_F!E{B@Hk16ElBCAmRO<pcq-OZuMKpXqr#}GCNh6)ZhK72%0{4HxYB+gr+X8L6
z2VaMPR=3{KW_c=uU7r(nlqc502>?8<O}m}$33<Sr7=Ed5Oj06y01otcoHNO<B5Kxw
zclmp9PTx)jE}7#sKKg6YEB0wJ`%j0YP<MUOU{7MvB>NtyUg~R8oPz(iGIDbG4Hz$m
z)X*dmzn&5F82v+?{+%-POfiz>UL`OC4^%fv$&%`=`h9u5DU#O@J7r2}by~k5*CjqN
z{xol*?Wk>M<}EP;8F8@dn1HPLV}`JvJRt@T`CFxfo48g?+{15<r%@H*XKfaj8u9~}
zvC3?=Vv-E()o)b{3F^%u4k%M`@@u2gk-w1K+I`BrwVpq7EqCJD-*ipsc%Gxa3K`<L
zG9L+e`s>NN(XMxhfc;F)8aUO;UnFlPMa%QKjfZNCLQU-#6H0^CbKetog^R(aUrzG;
zGO_;g@?op4z}b2hd$|Eirug8)oYsGaV$oACf@S1k<lr&4=3AkYJh{0w3tmc*A!`9~
zUB~igfTtp}-UGE}<+y%5N5+Y>kh8M3bdwQ>gDHpt7>?imm5ROF^Qux!i385Q=re!3
z7#{!#aFnfqGkiWP#%WXq{NH7}b5Ou7Q~HSe>zK}n#=<1SyFX?si)<v;s^Pyu-;XbQ
zu<V9<@i&td7m6Fq!7qGzkFEM&b6Q$ARGtraHntOvdc9__YIRWfWwPUS)Gs<_?c=%j
zI(MSh{A`n$_doQKI96|!XMBP}aF@kug5zr0TffCN_!WeZ&p!j@eAwTNdt?p#bakT)
zVOl$kuuFhL{7pkY^r}d0IFeA+XzKO4)J813JYp~?fJfr8pgei|!_O3-F5xe>f!p61
z;Ueou6`+E#lHWgNwuDgCdZkd=^Z1KzR(w!)mEEgvN}gv$+Z47c1#D?(;-|p!-|;uB
zF}=vSPo=+pzpwCwv#~fnT*${saD@8dxSJK}7O%}dzVm%LyPuEbAS{b!p{S4c7`bVr
zbAeRUkESjf4<17r9c8%suiW)c+|+#H;}ExUmbBYzcV@)Y-6II27xh(!<iXp>V7rWO
zDA<dB_^SKF4C_CNAXc^B)z>tSTOLeYEs?TFw!7S6SBP;<G;7Dj?Zv(`pJcfnv&Re<
zPQK@i@(aL0fE*l0HTsRBuFjJl;_NaJWH;Aqkw9p_189ubl$ZO@V33p~bt0Uugk&|B
zP=OhbqD!Gmyz`4L=f$-G-exz~MNRk_JOq&WqX6Vx-#=W)($-m&!h)DSwV<SE{&82$
zcUpe`sg}s;V?c1FA(m#3&iO^oY>>@60V!9T?_x@X5~i6%>ZW~KEQQp&K6sncq0%_m
zic%(l<34GGX8ZZ0k|M3!8J6?l;r$eUIWOso48u4|#m>c{lE;qHv^m;7yK6+14N9Gh
zWYOnw3gEh3j!NI*-#MOIQeeS+Ven;06!_<?wTm83z8fO@q1&k4uxX6=SK(6EG$C*n
z;2;XR7~f=TG|;eai@}v7e-XV+mqW6C2+4!nWY<~hH-br;om774Fq#CdilFkRQA~U_
zVY!*JCND*C%$;V(4)przLVLIX#K4a?-l55OEua6Osb_+lX@X!GG2nN1hjhF9GgyCU
zj2{@MJ9B`0Vw!73IY3kXhUk(FPlk^>1$2_ZA`}r8=5`?oo+jK2t~1@7!=#>M6HcDm
znw%#JPpBmkC*V%ar!$crxS;!ju8oU6*L+}=>YzU@L}AbTqzM<u$KOM1|6cLhe7Zhu
z*&pTm*kAeyl%6q<ko;eSy?0bo|JprBk)j|GL69100#YJ~)Id-`nu36UbQO>;NHd`%
zL8K_Xi}WS}DoRtNgd$xDU5b!Uq?Zsv3n>%teeeCvyx*BM|FM#D7Fo~pIZxetA8oIo
z!p64Z_wVcP%{QuTWF}5|nEO&Sol4LmH8iBY-$#%}(kYFZ;fJ8?Gz%U$qD@h6rSq}I
zP~m{-9Ep^4+(eqJ5128N;SrOhJ+^~0p?!hRr0^6dTBEPpH$2i?06RMvdi!r#OX|i-
zv0i;@E<ci)DEUHp#@dh4vPH|@!=@R6smcGGOs^WS(T?$1vU3`<iGmn2Z|?TgjX{R*
zrAcmY10b^R<-G^V_Uc#ua0TE{Gd$LA{0ahR{64~Z1$`;It0!xxZ1tz*s$>M+Yj{4#
zo>#`ywQnB3wN?JPxtsCubWY2ucnLm>VD6(u6ed?mbLxOlex>rD=a2TLO3`&^T5eV!
z3Oop}gj#rtV`JY!?8a<F3lZw%+8TYZF~Lc0WdGe%%Ewyt*JX|(&Hd<Ksk8;EnfC|t
zV*lCTz=y)I&wfYOswiA{hWA23;iIlq%<0llO32FBfml<q6d-E!Q5WkE-8m5k)6j?l
z)MJywP~PTbuqNCnv|{62;|55>=_a>S#Iv^Z@;OhOQ3c-jm}`dZYU)~lJJ6b)hW&U6
zw`#Y4;cFGFllEra(+-EB>issgcf>q(X@EeFNLNJC%VJX{vJrM=y6S23O~+b6r#BMi
z=0J@aVaeNxqmLp&12i5kim_c9l9E;Z^vr)FMWzfx{WCGmKRzl87j$O^2!q#X@5r6%
zaj9vG!!TYJNoCMtz=PU+k!)>umv#k^b0??TWYo>q>E=`1N=gW#AR4r;7ewDUVtPhs
zA9;k+#1T{2&ho<g4xoN?Td9J5<%wGKy%K<gY4Ju+nf*xyo60?6pG6EDsi-a$DGB1Q
zAGli|$vrkW)gj_67H2HP8HGz9zwf6g4%4nsd82+u^H#~`v5PpnO!g|v7{`&gT~Kip
zfR9dke<;p`$Zq*vM>5PR(dDI6m@r7#FT|jb>nxF}mmZL`DXM{;`qX<l1{1!$<kFfF
zb`&Iml{FLPJ9pOkX7<%sRluj%#2B^wWd<JQ>KcIXdc@;fc_|ztwls_*uDvJNmraO$
zNsaRzW{mnL;&h*fO1$I@v!4NW^9lBJ02CLu1Q$6-^C*xITvQ5TPO|r!3>}TyykFH}
zW=>-D@h80&ZCC7ExpR@g_y;@`ek$iZZPm1z(wt;~ICbXRI&IH;bUE=UgOQLxBzf1K
zX{a_!?IqZC_+bswhYB64F*Fq*!95S06$G$qogR!>v}@0`J)K?rkEi12N!rRG(ah}`
zg6>bWeo&Ynn=)srAFv*~WxIjRa*)kZ7irU@m4}_qGa)9`UMp7@KhgO$>{>}eY!pM7
z7$x!_uKL9@++yF(k-gT8m5mB8pW9{wW{dA=g|fWy6nxYiIZ?#(D%-&F-&-wbX<82)
zcyZo2yRerL!R!~RI&>t~4jp`^wK4T#r6UrU{2Z;G5$2l~mCD#^3x(`*OpoOEHM{FW
zW*)oT{4RS#Uwx=>)RYd_y4;e-lvsBXqsBI`bjs!A%;4V?ezs`2(W%x+vX8=xdM(V$
zWXf}-@i>AGd%reTJFWh1dgs3hJO7z+zwwC}^e;4ns9p(j)_CLbPOnUwDIR0<<LgNU
zU{or5=!5b%rw)a)@zd19k)@>x{mU05@kV_freUf>b@fj)Ih5?~2cA)8Jx#&wd>LKr
zx7>~$15zWj5-;|s6${!QFaAj_b3<1&y9+b~2MIL1QBWBh7j*yD5q^ax+4W&G3`lbO
z?J^ugnF)q#*VH<bxu|$g|556h0Pi8F%^$(fCMO50O2lhmxK<tCo{rXY^}TY{;U)Tj
zGOzmt4{WLi#z?+_?eCO3p<c(QoeR2|(YN+F+@qabi+;=|dGe&y@0GL)VEO)fKk=vE
z4-X1%B(tPu{rsff>*cL$*8WL)H%Er=XCxZUH?~k#WVlzV_LjdG1^q-9z$Dwv30N<i
zXVQ}8nd&H;*9Jp&lbj}Hmo~gV_rTB+=w_4T*pFvvWGD*dXkUKfM+5j++K$Deso}Ep
zKvJ%XgRyzGg+zY+*|m&9n?%sRAbLVJXftEMMnW4KP=dvAI=CdP#itPp(J1N=eZ8v@
z^fW{A>R45kS?eJc|G{Ob@W$fLh-LlD8$nHPr@A70aZzLN1am5#@d%g=fDcIS;jGK&
zu!VM++TQ;pHFO#vgaiY|7TvTfjrV)$V@!5aSi-UowDtR$oH7UR>Zz?qnjfnfprGUO
zOT!4<w;JNG%(=6!E({$N_H7fppyzMsQ*}Dmq%JX5Ui(I0iXA(>+rNqvWh}wUGROtl
zgp8+&av^zyBLdUbyIuA}9fvm_8)g!8s)7w5K^>h=o?{?#DRK!qck@mWpjVGav9U15
ziZ>|EJ8U(tJbWYorwC(9v%<``4R9V6=_NiKehRUZbtx|4b|2D9=Pv%>$9yo`kgPQX
zJ#9H9Y~~hbS9zS=xk1(503@;yFUkFBo7a&nj#*V83@~!=zlcv4<8Y(HN66sVK4bBN
zUM`%}^C#_hL-B%2B$J3{sq>Ucae~X=@zm4NA{lPpz#<rWP$fKsBzYS6oBajr>2(qG
z=Z;@l{nv)uUNGL3H|m__f!`C+D(W<I!QZ2BvuWq-d}0*_haw@5;%caqpfGLCZ1vQ;
zEJM{ZiMAC`S)N4W4d5)1Rp9J%8MPrO3xi96Ynr_E<I4`gwrEi%pD(QA(zwp*h)TaX
zzBBd}H(njSUJ;!3F<o4|TV3~DspB{2HU}=za_XAXX^o?4UHGV7o9X1GXDubdUKMq@
z&thR=zn7LyP~k<E;SJawrZnT*ib1@zmR<Ibw$2OSWmP)SV!!GZE++J$D~oJizJQ!v
zeGKE-L?=W&ef^vn$sy^B^HDx5{FU0RgFBf*3Z}!^x?8;*-)bX=z&qi^P70s)lUKAm
z+x&ik0C$qA)$x^2KL~PY`c^d0?ZI}+2V`yE<nm>|mJIVfupy+Gto9yU$VK>uvhc5S
z*QA0`<4u2FupSKmbeCidg98}<hXVMYh|<BZ<q<I$Tv+7$!|ST4;#Wkq%ekyATi0gt
z#L2bPFOb*yj#HoI!m2=Xkc3B_E>mJ4Q<EpKuY0-f9TCwQBmAC0g*HEVbmePGn(B`W
z&2^V2rq4)p|L{TwpQ79A9RPmG=IvV?Sa|ZO@v6^i$@X8~6=`j*!OTzf%%9&h?4B?$
zhflC@9bbSBT<fy0cfwpq(Y0-h79Ss9Ens+T*F;G*e^%2}T;pAyUG6{66-bXI<2|k;
zN*Z<Q)pn#pfTvHXLr|mf$3+g_!cmP*?SCmWxrmvZD{m{39&kL`uqZIXG%g9`Egl3o
z{gCtx>bK~Vz3eCmeX~hY$srFO4X(PBmZ^Z16y{&Q4{6g7Xv56u>y3|g--42Ix%~Z>
zI*Mf2e!!eUarmy)g)z4z;Q{<nO+k;`b(O8PJ}<4ruzG#H8#Aol4fw)SJxcUDKNw|4
z!Vjjz*0loO?;Xg!_rc|TrVJ;I2aaV21_rJdc)(W<T2{N><P7m0zFodG(f8@^w&Z8k
zqXOJ;JVE?<GtA0FMs-3|QF>Zr52KO0TX%op?Z3C{H3?hTezt?pX)K$e@rA5_Ne`}{
z4BK+Ua{`*mRW@7Fk}Hv@?R55wSFy)EKm>`kWU+_0T9!xgk7?U?2<Q_0V*3fu06(_K
zR(LOPp(tL~VJOwjUDrqx);5}&WzS>QwX-&Fc0!va%tu&d1sbBv#iWVa+A`{R0)1nu
zo>1XIYR~1N6j2Le8~v3uh7lQCzZ$0FZ*3~?<mtP0Bgf8_^@{HNaoZSEz%A*;fwiN}
zdMZ{@=f8`AZ}G?lT{I8Myt8MM*O6vaZQiHD0dp~nfzRHbH%O5qXWrSjc*}9AS~<>y
zR7qO`l64W8fEB)m3Q%2p#4jN==eC87LF6|!q~npNQ7~4j*__6sjejQTn6Ek{=;QdR
zeD=up`4_*?2<m-o*MNA!gE=kDpPVIb9mq*W#uCgN<uU#Q(Xut%)eOJ%;!-6m(2L+4
zrMG;iweAR6ylAZK&klx*SSiMW@kd{!*BBJj&cw-IeHO>Af!_66Hb_k{*D<)ZwNU(1
z@cMq~mg=S7Q(0Nr)vG@8LN+Y?IQBEXYLB`WdP?2HjWAaCOP%=rxir0m=ZLJLrgri>
zELksPB?3KoYS^}AZ!L!HzuR)T54#7kn=DWf2}!vmF!&Q>uG870K(g0;VDrQUwZOu)
z{})-_NSjo+#1#h9m<ACMSKq&IzVIn$;o-?ArdF53r>~<v@Z5d_7E4ertueoh6jl$q
ze!-&WOz@L)^|5((83cVOF7uDCPknztwb^^)CYrMHK(lG(w!xX2VPz7A0X;(InZxC~
zY1J8k4T%qnhX!z5HfQu%0F-zeoGKr-EwCzTK%Bjb1u^&JdN85cK?JiJuqw$~lV}Oo
z?%z9}lmV7M?92M%7VuMo$eVOKRfu$oO%=fmYm~ZAnXES64v}fZ9dvMr^|>k%i-C2j
zYqU-D<#VXo`A^&#EBE}dR-I{@Dz@)>Z;r|5JLEoZb-_RFPT?wyt>I7tH#HBO`V)NT
z<>j39slialZsJK(UqMgs79HtSd@Y6{x9xSQP2ZeMW!mrNR^nkW4Q2T3{Lb*PkBGLq
zBYb6a3BmIu|1>+C%TM+EZ{N9Nmg~bBPX1#5fDF3G4PBr$NIsQi&Cv-dt(3C%weJFZ
z42aH`-K}dH{L?cLmo=O+enEI*Nvg%ujch)xk2oS0A#a%KRUczz+dAYI#ps7s`{@TQ
z1*^6rU?PbR&$;wG?yxrEt#yV?9=$!ju$n#Ew9$9yBmda*+i?7GONRcw@Aa5_o;S*e
z8;d?JXpg=>xz|U@vqXO-(P*C-&-~}^&FINnH<!<{ObaRq_*n^P7LJFbIuqfL4k5D1
z$^PNWNr!!JCy#>Ho1cveU#eK^hL+@}mDM3H?1A(z6%%EDs9in$ynLQ<WRai*>U??&
zL;ks!e4!{sZF{r(N+8B<@1-|`(33s6naL&rN#^#azglhM)6dFRh4x%4@Zb`%aSfoo
z(~(Y_ro(;XUUS*V`?T|c8FO+fIgCcZj4sm04)o~esXIJO4mntNu03C8pOS^q22Ktz
zdS<oAJK77@lNlGE{tH9}2h+fn)9}0&ls>>G+b!o&j;|U$66L&nbfn=pH=#(<XiYD1
z$(K_X_zP+8M9k)DU7_D=rNX~S5Sx~b<~O8P-WvIMKi1UGdHOvY>8mp#?Wg)iG05uM
zt)37blox|L{LzLG_4D;QPwkDuGWbbV3jLkhrF&J?VyhpCeg3D*b?bOP&3)?x6$~Z>
zV;t4JGER0Anl1;RH4hQp@q7Vqc584Q!*^_PE_EHnDj?TmhWpA)YVWn#^0b?w49UD8
zrQ$tD1}#v?Dpo8*xPL@XxH(Kz7a$BPj>_-i?jP})dPEI#1eYSs*Yr8?bnB%~{_Rg;
zWUJdxQ_E_yO29!uk=A#=oO3QsYt`v3j#w8AvbS-*c%^GVhhD4~ms*in`5~1Nq?%z^
zmh{b=Z|c3vO5bRuVvr}BwBlE~HCjVB?=EwRT#Ac0Bx|FlU8jm~6FSqz0ST)T0)j*v
z81UOfKJWI7+tTz5S3-$Q+cCz8#v`rd5s3<L1{~!@>DK9Wm?N`R44dRLn-x54@v&z`
z=W6J2JC9>t$##w2Zl>OUcT1jD(mGMh1!9o0)er^=yK=>v;2Hy<?=%}G(kZqD-})?V
zR@q%`?)zv;&+#RYEo%__ks9*=LOb%X?U7Q73ldBVeVgCZrf#mD9~@-lRHTGqH2Rnt
zLL>ZA!jQ{ycSH02VW`o?UKuCD$-ER6LAFkge1C0?K}wTen8Jzce6kO(VK4>%+$Jzb
zafK_u<SCC5>&Nch$l3GOuPx$=mt9;d;v>hlmsK+pXiQ=RN4a0=U(S}z=^hS28>OTD
zI<0~)X>qtuF+M2ws`khgv;4c6kgvEXWe&SAZcaU^Dh-f&>a$#D+)Yf=U&dBuK;gqf
zv<Odzl$ZWgjGKVI>SgAL1OLJs%4jYQwYgz7aGF**MpY|FF(tq_-{N!~?^e^j^-DLk
z0Lfjt)xl3(X_^2Jm0(Wi9V&Q+LR-g9q&LDcB~R9=%{iY!*?Kps5WXvSMiva7Ek2Z~
z^T!R0b}s!k33jmxcDpU?gZ)R%m0(-gq%kx^y`H5c`DUCg#R?eeYUE$5cl2ilC05iT
z<X*YP7I0hZ`IGtpUWfLGw{LcXCYZgyJ=)8Nne~b6`O7uZ%a|@*S{wMI=D2AdXHr-a
zHZW}dtX$3PYSy2FKNoEH^90tfa47x!W~$d2*v=jxw=$D*YbrsBrV~9&<{CLp@ou4>
zgbpcZeD-SPUecS4ZWulmalzWR+vh3M=825UwyLes?&CJ@E{Qp6h-He*jXCD4z%_oj
z;HjsS3MP_$Kz*3$WOQYVjPCeHD=Yos5A|boanbcIQ`hM*Jab7uH2=){ny&28cA(5X
zudn&Qb74P?+B8lkt9r^dXaX2bMlH_@D;n<b+Kh#wmftp-AFb%J@DqcG+i{XPAC(|e
zui8|)BC?ua$1CVpc^3V<Df4O4c%E24AFeFZ0{@G{dn?fLR5*NeB8n*tlu%c1FlRQ!
zx?3#LGR7bP_RQ&SQVhd?ne~`UKx>_B`pW$|;^c`qPRZColIU@E50;9D+1SX%^jUHO
zSIRPUPP@7<L2qiwm{QnA07-$DSbEoaIl!b|rtevDVgeABwHTgr5LjS^oQ>L8)od}R
zd=(HJk}RS&QWAV)^y;N%zDsIK=T|uRI-b3buHpz$E7I1n5lT~NV@&T0mE>KuG`$k}
zii6x+e%mqeT5n<QI4BQWW{Q{4CzhG+cc<dt0Y=IwG^R!fvb2CZ*#tkr(z4ptX>3E3
z2H5WBNNy|oMfn`Gr!~k#6r1G+8MkspPG510#*h|&zNk&m;3xIm%zu(A{&pI$N>kn_
zXB*cYeaC$9K|#ZbdmKS=NEEuN%&VFL%ZdVN{LH<EV?}pB<UIS5r_78e$+fhxzYoI$
zmz;KixUbNaMNPXi7mxCG^SaVRg4r{ilHDHt@orMlVL33Q(K&I>R{asTYNk%trEjsf
zDXjlIEVoh0<*PK>hPc)f#Av1uO8In~S-&Xqi*UC2Z?6<nzk64hoNVB&OKvl=BbQ%s
z09&|x6ez4Ybr0Gs&iG+f-K#i11c%1*G*O5^UvX*}7!#o%-WB0KD+c7U>r3BrPr|@H
zzhISzt6X3Qvr%E&B}=cpy0Z@hX54PvJh9_0jr_uw?fvAT6#0_YhBl*S_#*6KaxT;M
zRJKYI`s@uhxk2+4xNjdpM`CaC?N(EVU@EC6*h));a4_>|zWCayeHs9Q0-iS7iG82=
z@!rQnHBuS*&XeXx^Nd<MuT?Hvnal+R3iQZqV}`9!sah-;MSX!w8`^u$IkNZez4p%g
z;!wt_l%E-UL)U%5;iyt*tIRDU@8)fHx=o7Mm@iLkVPJF%n~FW0qC9Am8yPZYxp+R9
zY4u_md2152sj-#BBA26%+dcGeZrQ602UN+0p!eYSt8`z;21^LGEEazdz15x+ATazU
zR?u?LM(Sj4P{S^3)>^P6y3>T`_0^L_%^mS9Tcu0WeE$GAud=fJIL)Cn>EUAs{_R<R
zqoObW(`t)~?Wu^wz~Z$t97^#H%HV^S3(gi>zx6`ZhK{1Dsy0=<fH%HO_D*Go2wu}_
zIIc@b@>&BGlOBc7rKs7oWb}K#P#r0DuJ+win^HT!JHny-`f9lq2Y<EzemFfx=yM&d
zc~jE}ic(+Jmj86NShqqx2y@-~t|z=hW9U5%wb=*wBA7oo?y|z<1b6fFKPqrCG1x1l
z?SuN96Nz8*Zb{@!g6~;D0yuSL(YJ;;K^|fOzj4>Bq47Vzu;-RrS2cRf4a=q~e#u+8
z7`ew7FkRO;#|m)bX5rtAPaDe)mCe7^eP#dm`8A@l-Hhs5b@c@J@b5~WBmcdApg8tk
z<4LrRPf_c$9F(hfd-QhaM=A2*uN0pTde*er*c7YA8yP)cx8FIo#2MDD1}_f71I#06
zh^S7!poz*QA`eFhtXyPim$m1R9^vJ!{J7Tt-+QAS_SS-Fe(V<?x#TvsQxDGBKj3(;
zdP-5BvSEig_s#CD1+`UQ(Pz5OxF=C4v_1$KbqgHzdf)By>nLsc447KKJ{?#I8)z1t
zsYqe;e^JD)scP=>jzgG^A33%WDV*%R#@j+jA*z$k57C<t+mD0qD}0cQ0w&v|SsZH&
z=~u@tLtEhZ*6LPat?_mcY`wh5lZEed@E{Kdf0;>yrK1p+1eA})(2UDhgw4{K&&%d4
zTi)V;t>F|2g>7PDFfAONs`geY7*Sihb>S>@yn;Fs1@=RlqgnE6<cZpj{`dFa0_t?s
z-?7|_=J`e+_n~&QJI)t2_H=V$$LP~&2;1ctTW%(}_5qlB{Ck!lI7-MWkjsL9-*i>d
zwR}B_>{^uY9u7>`i_CTvW9)#}4A}$Nfn$<36UA*mY9!-(6}aRGuJvt6$Na(D4-gu^
z_KsB`cOd6vcI91q*!oUKeo8DO>~e;jC`5@_9Ua;4yz4iSfPKrp{q^v><!Gg+4hz~1
zJ)}sJfk7Me$kdM^ubBc>nR$(KoZGPQmNdYoZ>+miJrt9bniI2Pb16(5&De(Bkr24_
zi{ImUXb1Lb&7F}YwFibOE8PL57nDV7Mr(N*(oQVou)D?vkG3^>V#Y^I65r2%eJ8wJ
zzn`gQ=PJZN(hqU0OZ1$Xv|Oc8AB^`IHQ;?W*AME?RsCkX%%jfW7r*xF3oh;8c?-zs
ztl!~L|EDE=CxH}Y=}@bnhP)Rq|G>m`6%w_Zclb*0v4PPR-Z#Hyx_z~s%1=ariv4%<
z!@DVun@bAIy~NSs4&C0Lhe$14m+fE$StoW+bggmG|66CG-{yhI)54;p4Oz`LQ6cVJ
z_sJV!i#j{?d~unzdSXFa^V*{Zp{?(l;h#x=srgS<J)-3<cKWyrG{)xkgU=6g0q)=9
zOjN3NwKukVb6sUD9v&6n&x3^~fo7s}%-9a@s9fN@6D_8F<>wvRpsw_#`@9Tj0yIlH
ztozAntx<r9KNeYz)(7NB=H<9Va-)4Kj)m)gmD5NGB?Yh|y&rF#x1^L3-&UVm&BZKV
z*YMPOZyx$hM|02D?!BU_=9Sz=CFO(GCyAn9-*MqPqe4U*FM*f2t#9<Y+6)=WKhxA3
zs_PwYU=(;BkXMZ300tO@_fS~ZgbS3TMp7s6w%1Lzo_>esoskNWSR!So9CBMF3gNuP
zWd8ebixBCbv)B~L1ZfK2S=#!7(VF<!^r7X2sq@c*xcYD&*pdcaqTYGA^t+irx>n{R
zW+Y?b?x4&VJ+`Qb9f19F;cApR!6@1#js`QnC*+fId%cS-i4b|XHP~O`=#xfe@sBET
z#1H-1s3DGPtyXZwqBq{k+wp{3R_r{mWLMKoEgJM^bidcoUMXqm7_I%@uM`&`!lp)7
z_xqIJkYCBw25l1dI{*M%zX!^5%v{&=TB>k#S&KG+sIJau!TT$%Cs~f%WeP*4bn9?w
zEC$2XipTHaM_oE(q!u|^wsE-VnLV++yx{OSep;T6jbqcNCs@`2taC1@eNVPH21D+Q
zfpFRe@`$9ca<r0M1(1O$CV})C!NFXy%q5E@7cV|RbXzpW^-=825-$;>^C0&@b-!Hn
zTeeQOF54kr#snPNie88h=zg-IH4`Qw@>zuK=xXC;_vALA0Wo@;e=cO{RhQoctmpal
zvDgP3|C^Whf{7+_6p+ezwp-M;lc$QiINb3B^K$L&$6%FWX;zjuiJ$ykh16}cx)h4x
zr(Q6r4=~Vh!Rl3-MJoTrE;#ftlobOU9}Q1|n#mtNj*Y#U85y-Ya+_`3At*R^Z)%1U
z30b$>B!fTW5=`j}!L4*lHXAHUxi=k6l*f)k!Z@VF*iekS18(hirG?>3k-U&P<WUQJ
zbqG|zjNyUO1sXwWn!VSf#6UFWmt_k<%-H97O!IWSJYM+SA+f?5(;QY{-I8SyBz)s1
zU7v_$q(Km4+r~&j7v~GGb{<}l7C0;7b@&$37Ty=LwAWN<RRe8$C8F}#)Jn0vjCB@U
z44`vKNADu707-<n7<FxeKe9+Tck@OiB@_(9F^s2-fYz)Fh<s0aqaojRd9Sg=+j`lu
z#I#$8(Eu!Ns%K(KVq-HSo>82DuU|K-KeP?uW|`lM?SEdbnLi|G#$}>3mzr15)3=t-
zWhE_qqwC`-FodcS?62t2AMawL5$&>ZJz88hk<)MP+C%@1sQJhfaTTF&kN8K@+^;rK
ziH#w8OisM@Z5JT6%Ck?O>&zawfgLQBO68|B)EaF5N>i!%?nuOuLw$p}i(|Us>8$`O
z70}bf!jauT-HX$kN0OqijD{bjoc`aQ4+OCN@ZeCoPaYWHxSjO+v5J@e4b`E*^s<2s
z7)`QebdfvOq<Pc^oi3R;{YhKH_L89c^zc^^_5bnvGAo<kJ&bZgtAR_+Zi52#vc~g_
zUhP>S;`F!mbi0GF)t8d_s>4KlgM*;qb0DJ^3D0>6&6M2l8=+W7N1?-Yh}o{{0vub<
zADyF1r-x>gzi%c)>Li8%N(&T;hABnOu5oa{QjtDS>zPtI@V~DBL!nGKx>OyT@D6JR
za_^x}{Uci|C+u*PAbteY#8(-teYYgpr<6U$n@>n+P6~IodN6qfe+zl*mY8Xti=A?r
z_4-<`bFO#O)o+AU2G0T<)}2NBN_ejhYFb|ioNlOB-b|<x4k1KJnx27bK_lMVyX)=z
zxzWR@1Z}J)Z*Fu>kOoa95$dD-FOCKy;|?6IoQ=69o*E7c5jh*z4J%Xab$Lh%9Zdz=
z8CWLz6BFbnZU%I<?1-<B$O*C@=^A|um>HKMLau?v%W_#P`|dzax8$!mNRh0=^$f|)
z=KJ{iahCE@mg(a(RV$_+d_1^BzAswp(rO29o(A0NS#EP&{V{Ch(D`|rg~uvok;w3m
zmi1x8wrhmiaKzhd+uAf=MvvQV5PIUM^B7-A)NxPFhttK}JgeGlv^6{nzs9|&|757b
zxch}Ma-|158rF*2Nizl4p8Wor^WFwHDY0^aL{C@d&;4-!LN4xzp6<%P7y4X04$DTb
zT@a*14{EWU)yoe|39+H+-$C(ZtJsI13qpqN8r~IvgOJr(_PA72O*n`BItr?1dPQxi
zDAEyTuH7JNs)>EdbUT4+1WoY%(3{e(SMG5!k6zpk>Zcg*ccp4HooS(57K|^`*)O#p
z@>?X_?RD0HR$Xb}%v~vu|1ImNWR>w;n7y92l#o{LNamG;x)}4o+Oo!P)$8m#Dys@U
z9z&Q15;jJChZdZUryzql;F80_wUH+m^5}EVhK{N5-brlrv{#%LqdZP~vFGr`&dVTV
zaOsiGiV3eR5aH_Y{}Um}jSg}6i!_JlfN}*XE7qxf78;FQ9#xA#8{{=nG-dmI->2gf
zR?FKj!3x9Md-Djm+I;-(V>-8b%E^)CNXf(4M}&Qg0?+8l<Y6XR+?P)4+dRT!lA4u1
zB~D<o-5;y+Lrb7QE<YEIuoyi%z3kK*kHZdNc)UizWfsyC?_JH0-wgiOX8Etbw;x9M
z6?zPYpJ92kJo%YZTHwP~g#WT|T)Ak2Cb?qir=j=cSyr~dL?&TX*Lo$<liN8OHY>fV
zChr5bZ@T}V8EcO%vi>Kf<t++<_7I>kJylw~r>BD6OUb{d2K0UI%<8LMHmzjx9~Y=x
zZ9>Z8@E~m-nCZqJJIhMI8(CptIxvjDxSI8Y`yusR>!(%Bh}FDdV`<Nxm{WM<WwwPV
zW*VkH&&3=xgJi^7HkDz@mo}KCE9{U8qMa%X<sALr+`-Jx6V=teV%ZrIo^ZHgj^7LR
z{}!~`Ez*%E^=5<(yfwKza%XB(gwb(*n8g6nqhJ;TuD?JF&UHfS&@rtw{iGvVyHl%$
z<&NRH!v1u4`*mK$!YDwYiT^Mlj}Tc7cF*w5BW)dL&V-;%-ws}79-^l)&urUFrj3&3
zY=S;hDzH>7s+r=Gj<Mw{*>WEW^u*ru+|2uIfL3s#4~S-pRqPsmP+&WjDm^dt%jbz_
zfn$I~RpNHm`>!;ayjg&oIQ8@0S39OJQ3Fo`7)y%&7c}=_#CaZC@;o11!70R{be;db
zP%qz?kBXH}>PdvJLJ;9|Jgc|aez*-!gdT2k?!C!_aov3OUr`VBAqGwFDfnFDi8xT}
z*}zW0*MoN(xA%3M?rXb>Fk$G-Cr;?Y=$5K3QR$mee{|YJA<)$#t~M6OprML<^}xH5
zRHk()W0b1}5->#@Vp4cVK$_84h<B1%00mI#5=Pi+dL+PqEI6>2${<+h$f4R_!-c=+
zJ5`rZiX<{69VLUvy>z<z4ztLiTQ?NLV%$}+8rn8>*uIV8#6&0`_FYWo;3FkGh1saJ
zD6V;dUeb~8^l_`BtKSHqfDVj=_uUnC)Ru+<Mi}#h=|=SPL;4r2?JV_;grG!Oa=$U~
z4?KKXSSchj(ia>)ZO?csOx<Dq{pJ6&>->i;|HULq{!EK#F6`!R3WDzIO7@@r8@&E=
z<1!uFk5RXer8{mw6bz#_L-r<?@1Q&#M2YMhMA^xKK3&%jhZZqFq@YuJ79zs3jFe8R
z6I~W0k>2Zt^t$DWb(Or;{iVIykf7Bp#yIyEyw&Er{m*nL>|=~0roy)ChV<_5B=?Eq
zz`j)WVSZSD$=Cq*hhNoT!u7~g6F*ijj`{VO_e5D|C$5U5aO#$r^h?EN?dmQ5xeID`
zpJyr*Gu6DLRn1k-CCs{3VxlrsBW-|k*EW)`kpMQk_mu!(Lkym@vQ;1DO@B6Mp(NgW
zy(mp^;4Q2^V(ieTcVT0*hO+u0k=P|U`k3%qE3NXMqEY`nLI2JRc7Nl%UCvnKX_hx3
z?|H!gu<N#O#GEGN%j&fj2=Fk;A_;LEQML0$_|7Lb;HimYn}I9{UF#{?xd8~RXhK9d
z4Ot7@IvlYhv%mul3njwYbQEUY=kzQn)0B25xj;JC;WE|}eXwKs>Qb$CHrribpBV3P
z>J1@$hOP1zOW=@!JWO62>O|{TS9_~u$zw3`*srsmw`Un1UmH=?zGdZH3%#Dn5v%N)
z>%gPAB?{04oFXhml?XQrn!RJuWfM|~4WYf*oY)m3IHiq`N|T35IBmV<a5$rJ%F#y@
zEkVYn$Z3|YT2rOtZuv`<Fd{iZTw;^daU-VcNe+?LnEm)!X4%H6FXe-qDL+W;ssF#`
z^ZIK(uh20&TBHq;;|=?#`Me?=HCK$5w`awUSEDpOhHcV#{&o4cYvT5FZAO1ZLFhLq
zND|Ziww@&I4VqRBk{L{4Q0++WU^JneT3`fC^_Iv|PaO*(OH7DzInP~gnzj&hV!}wA
zG{AUQ!_Af5ktFcip(<3OAY?xR9O#IeJ}peHVXUjP&@iN*FQQM*8ZWRKBbrV;sK@J=
zij8~pdx}d3DLFEK^8PHGTi&-^!Qx(`k{O?4H4RBk+hW6)S}Wm@F;-hVXM;LkTB{Nk
zOBaB5>luZ$rD5yhk@pfoo+SjGjhV~jjyMhhQ5_W__4Th<mUlZ@giST?*%lD_MnRPI
z%%k<X?7rKo|HLT#F9QuS%xf>k4{Os@{`jvte)>l{{H=(|d?nklKyq|WD4>TiYct#o
zKjr3Yqw;n#$_Xi`asL;3xdnvG)%7!qND4xj|LKy+ZsnNjeST{{^N#CZqh(~-uhOmW
zUMm!;6k{6WDeEw9FfA%)e}}+DM4~q&0e5%y+4RpCm=xVh)^WrqZ-s-$w00|GDR0dA
ze@P9U^W|O|mcr2UStmp@p9Agpr}>xovC=0;hN%@9Fjufw2_TC~jlm&D$vT=9?&5JZ
z@weCa?G(n77|$wpiG^gO7z;?)xz?Mxib+5BO~RMrTwIHQe5;JQl3Ij=`YNXa7!lM=
z6z1KWeYR?pV_GmSmaWD3l$M45L5%!=Jh#vPGBwmpZ_=c?dlE$?{@Z2!&wWEevO_ag
z%PZTX8rcVf&d)Ucj{%@Jl(nU2?`L)M+fhh8*u-WA5uB(vS53$N;m}TbUk67RTg<uS
zM2BBUk<F;+OhU5prPKd0^e7;)zgxN4;NBH?dl!Z|g&&c$fhi7&@GTXkzzOud_~tQV
z-sT0a!Mk0Qiorc?mRQH|D-XO#yzL1LieuXRdX8tUuLz~P-gox^tM0m;l>m1y-J(;?
zG)QuJX+j#=@clN;l$85a*HlQ`CJ0Lt0}n$Ij9H|4eHNZux#F>OE)|*`agHHU##-QB
z`nDpV`nW~;K^zA>*PZ#k_#2U`Ehs)QGIPdrW$l{V{%hsMJ(r0U&sS!&F7*GU?)|S(
z!yd}fq?~>>?$gF8f*R53|NY`Ww^=u61MKhxO{VBlw0++Su+^_;Jjwpq_kAANvV+%N
zdGMY3xc!icz52JlQ-N{?`(fiApB+_6z&4LrOid%{h}BPIX5f7FGk4B}l!lQclhqV4
zAfw<@I+scxPe>qE+%#m|YS`iyigYRY!@<px)>MX6wid#`v*+q-VgP_*uW_}xPMJ2J
z@S1hgnff47T+L%7zns`zE4Sq8Xf}|btGqf|LZ5TBolV?W8j4puRfj{;pcrM9X`hQY
zK|%HlV*PRK-cwdSNRz_mU#7z6P=Uz3z;t@QUbCj6L$fFEMqQ^K*?~DihfV!Pl5(|z
z=M8Gx-qk;|b?&>vaWoe{7kTT)>tmanSKpvJQ@U^e6WIKJlOV^&dHt){Yytc&HWf<$
zW5WL)*l)#5ez|L-P1Yk`^2Tm=V9uL)D-P=0?Owv48{iLkQ`~T8u_JFZHx7`v_w$Pn
zM8RIi_Ct(U*kX2&tMd^i9E!c^g623Rw7o)DYjt?%xR7aV&Rb&$6ykEDTCooAgA1fV
z_E+{JdkSuBeevcT<0!kAe7KbXUy=eHJEC0f_ArIdUSbMj^Na5-eyYS%DC>O2)VXE;
z-63JjnR_+JP`*Ots;f40i7Xs%pSrUz=G0uts~I@P*vp`6{3?<y|L)Mc*%4D4(vW#3
zt>kodUyF1P4{fiQtErGRUn>_TgCE$C#|$sy=SBYTPnKf}iw5BPJvjior&T_W#lu<d
zsOZ1Re!F(9j_Dtk`F|YIuzA^koykA{pJ%d08{HTp-V$<PKqS18S<~D|JH)`24{{1x
zwl<sGZXaXscu6K+)6HvFY14@LL^QG4ILVYFvfYi@p<#?|E8J!g-yGiWGo~5}s~-y>
z%j3h^vRV*~1lq4JG0h^R(S-LIVe(5Md^l~f@G<&ykJ>O^JhZ(X1YyRf>YTx%k#P4y
zmS*@Gy=lw(i>$D<^55s)aVD^uxw)6}zl-j>s7TjcW`U(A=op_dVdgo@oMvBdSyLFp
zkv&qPsANAzzi=Zz5Sax(YN3(%V=nYPw<44d>BQXbrz4I(KI#oE-1Hcu5eNCJfpC`_
zk;3~?t%i^wWY(Q*1Ery|9`*fg73z?L%5Sf6rYw6zOCzs+n5D*4#k2O5<$$=W*JX~I
z_0cX9ZkLPoi)`dzy8l1c{vrLh_Fn$oN9&Ukg>;Pmds_d;Cxwf$IGAGAhZw{YYFQj5
z;HT>!;ozT&<=Od$rg6%nwLNav-rf;BJbLk&dWiXa=y>p2ZSeM|gK!t#L1FtNLHDx-
zH51!%IUlFeM1|FPFoyKQpFdXDBk-vbA@1!Jn;tk$EJO0vz8IHn+b@wAb%NNh4gE}-
zu67-^s6>|R2_Ybfmm9IzNT;CE5lN>r^_WWAeLZ3+Q2j{HRCsozy5TfPORNHLZb<4<
zr^;PsGq+^ZmcZk|l&q9-z8!7R+?6Ot=43}5nzrOv=%Nv7$&5#?V%@NoXvtE2REoLl
zfzlPT8A+1ejj(X^Hz+ez{8Q&_*#$SJg>P5CRKr)Y?(cdUMojq{RmXn#`~tVIunU79
zMg8Ov`Z_cc_w$;|*z@q^Y(zx<9{a^f=Z{+C(EYqGPJq4Qwc#pLs&K%5jh4tro1<W#
zY(Skzm7+(Hi@r|yWAV4I#EO>{8*GBv&b*Me{Oo}(j8HBsf~Cu`X7=qw<cpSmk1}5c
zob>GRA`l14Qq5L`e5ul1%F9Dv66O1jZy4hHp>-Id`|w#j<Pqt@qG|LX(WC*x^p;tw
z`)FqK+#Ue72kH#@^fc8iN<XS?sdjIT#D09!M&QF4r9z+nmumc>4Iz^x^vc@r#C*+C
z`dfLa7w=q2I?kWs8mk>h)yVlavyO0^=m=EVx<=*M<Ut@R?ktd<(YI*)#5srpbpw;V
zwBK4E_5<;Dg7)KF%Wv}UArz>*dx(Ra4qvbLs4k-Odz3PXOl~~;NUK1>2fuRAso}L8
z-&b&_fYkScHWgA`q&>4a#QB_^u%m8^>XzLfEsKxCz!A=G-cIR#=gYm<%SeI_tVDp}
zr_+yzMe8Ax+-`=$ZuI;&lmVJ_AiQ`1H<fXC@sy6u{wl3k@UViouH*P{N=sdFCN2!A
zRbW?In}SV)uW<n?h;qW!PKsmQ@8KCKjLH5mo8IIP?;=@nHC;~cDi)nie~7H_Egt-r
zV!N-n(DzU4_cQf_uxsyBSi!v`XDw|M({JhsxmM?ZO~gkYP8FMFlyl@Q?5npiIzn96
zB=%v@S{iro53JC-pncSF)j&Fqg+HSrO_&y{(H1<yEd0oH2)F2{l<5o|ZbX9I%Qbk2
z+0bXnnq~vQ0K~=g-?D&kiy2q8eO_8SsO?%^d)LA}2ItA>kq?KCJ$5mMsNx+Vs?|<m
zMjyUORpr8$mUp4#8ogD}Z__E+O|*p-RC%9Wnu#Xwfnc(G06^YDP(w05abC*l!Z;0_
z&`_?slq3x5h@7;ObVR{EGe>3bk~83n+*?IE?>E1^!0nRFKG9B7*P&Y&;>!<+$B%ZW
z7`y@{c9#WJOA7veB-8DrKzX%m9gliM>e{&xk2-@fRG`$wv}*$R-BKBdncVq`+WZ*I
zF8PB(bUv4+02ov$RzRkV?cLkn0c5Ajg!I!%qYH{lV-wZM-G(SXBBf{(gTPhZ&T~n*
z0s`02n`=@%1)%7XhKO>xp$b&TydWb7Ot4(sjSm_m!i9`BR=<B3&D@BkA5XCsf=%6|
zg-Lg8Uq7;Ez}$vt=!)oj&RobLSo2v1yKf;W>`pf;FwZu5Q=pe^=fsdIk=1Z!47e7<
ztF?wSIkY>tYZFpPFK!I2maK9LIg?APDUi1!65oj^SqWKmhG&D3PgmKIH?XvS1dXy5
za;0guu^4bPFIm1dA$IBTsY4~=z$$0?*CeBOY{mi%Tc^F>aRc(8Mdd$qjLyV<x$8L$
zkPAdki%xm6DD_*cz_2<EgkxpnkM81;ivCy~9*%2chVo&pEh7L9t?BFxImf;f&3GlB
zoUD}%tOX*44Z=&XAG)a{oDARP{v_uU!x2)?0+*+2)B>0&OIAZ{`!Z=7xIjKyaS9y=
ztd{}RV8r$tECxKVHERb=VmMU?F$u{>O(6DaVkq4UJJo;40y_nn7#hQGmm){n8pEIP
zq9D5`Knz3<6G?2D5AX0z?B6|!)LS@FX@xAD=q&y`qP*Gx!FdTY=P(7k<i5k3VF+so
zm1i;OBI5gzSU2!Z(l2P-=UaczL&eGs%>Dm49cOl;LV5~Fch<sl4#|2#$>G%X8LNXH
zdM19dPlp&y5G$zeT2sSI4jQ}+@e51FH1`kDJ7yB95<^x~m@dVrPZ`!2w41S<g+bi`
zLB)Hn!>>5B2$3u24I27s?JssdLvX#X>b65Ze7d-yKXX0u7l_|)X+Dxec<Ibg`3Y;j
zu{xGfJhCP*)-(jmS+H7UmamR>?&2Pbfv>)^wli#`>$4P6t+#-UeA5=HjDsns@k{Zo
z*SS=sDbl&pX$9#r)0yfA1r*IcxAr%9x<toPK%k-MGBQF!`l6CU_$<}iBGXD@f9=df
z2`-5;@~`)9)Of#=%m7cv*_+B_k5ntq|6bS_DpQp9Z>tE$mI??K)|6XQD=eDYElZq&
z?`eclX`{Jvbet)f7g*POQw4^jt00=@O<z_kt>c3&=y6;kr}r^>f(YMyv9-hsmeQUt
z3F>{j#Fp0dDN2g{+Tq{&qymsZFitdI)j`CybW)huzVBHd*(IP*9eY>|?=HpTIB)(q
z_J6X(yoypQ#C9%=XmW^-o+~k0JdtUIES_kzqV`)HprLh>VyzInxpMyMp1)5%{^a!c
z|Kt-8HQ{Gygqvt9$_|}yZDQ!Fj>q3b!?~E0x#Ke@glz3uJZX9Wpti>D>>N>+VRTo*
z5jrb~oP9y*eH0{Bx)Dyk@NM(Rbbo{%7>Hc*B3}>;Dz<ESMxtYZHiiIV0epm*n$@-&
zJyBpfLE3^|b-l*FkjvmQ7-?GetGqG>FiWT2rc=Hj?qqF<`g~#E=;P8baOs4i1N!Do
zEdz!im7gA;81Nx^)r5!G^3&CK4N*lSAuh<s%;r_{VL{QTdY{=D&9D(hP@$qUcFlvu
z6&do!LRXV&V5r%=U>51>t=^$$0~?Nw;TBZ7_&gDotk7ndscu@{C8O?lQHZWw5w`2L
zw|YQKIsNX#l{hDw(MM@AYb5~H@L1mFoiy)<tHloQSdzea+<K~gUr9b&^ZLDLX1-Nk
zZLv^f<gLPnHW0b^=V<l8jmY2r<SY(%U4RL3{*X8xJk0=A|5HX2ukWQ3XHnm**1C(9
zI7yDV71rMgngf%q@)*>MGT{B%YGlvaY0<n^IT(>G)o$9TF%|A!Lk5tK4<VZv{FifR
zrx_H2l1bMh)7yh_Iz9v{zdAJCz!X5K-6twLCt`5K!l6Fq`R<7T3N=NG<Yn4n2=2``
z)Gqk~#;IAxH~|H_wQzE+6+$MNh9P<mpM@cKTg6DPJ69I}e!6&4i`!~z>d>L3g!^Kx
zsDB-smEXH8i5KcysZa?1_Nr#K5W;NmgtFj}nn&$l*MGZ|x=xPN!rNoZ3YrNv4F)A4
zYurF3+z^NS8cPpTY-Y_-8PEZ1c^Sl)d1tL?ml&nbLkIXqzx#aj&ss9m2ZA9)b=Ww%
z=oRrZ#J^b<kCT=FIrb^uGn0qycWtnM$V(|jyIHjLrhl0Q%zTNy_tzea@LeNEqJ$|q
zCA^4U5{5Hs#kYPKV#JPp<TgV?SlzkBb@Hrwq^c6femQf`yIkz2bXPj!X}e4RmSOfh
zhM;2l2zG77UJDR8y&_Gv4EIuB@m-oltZ&Zx2X&tuVe#qb?CW|u>-)zCA>m7nbh|fY
zrm$UAA%4j6A`|sdqZ9puOs%Q78X}0i&e*&TfoTVg+~Cs|E)2nslr>z617r8k01`v`
zE@i;iXj`kyJ02CInxY^fqavM#ZHN+#Sd;}X8(G<}%Dd$_1`2(=A6K0YC%mK9uj;EH
z!#)mS9<gn*t_r6DPF5h)T@o#wHzUsMyn<32ny>9|%kH1_Xk3WJ6daKg{$}=9JLt@3
zB0DD9J9qKBWIjv*Y|%77<zKNppV5Jozdp-}agsxQc$`V_je1sb>93P%p0=ahB5Y%K
zbau|s?&8D%cge$em3B;IYg2mnj&EEzVLNswDf5}kj@c&d+xx|IyX0|A6zb??BGlC5
z`VIuH)2~TQ?HVn57K@+405gntao%PzT6pBIYBvD?a)E)K+Z4BMB)gzi4LO=sD`H)`
zQzpFFET@bGoHZ?_zAj-*+l^ciV>@SBbWbYInm9jyrOb(}&(l1_u)ru8jui*;_lbj=
zUG7Rwjaf{Pv%Jms3!1fSq%3y!Ev)Ql!C<Z4dpg)%=?iOOO`ut(*1VJ$Bd(?tvv}rS
z5J`q9=o2kXOS|-)^8~vhzrB|adL4pyOjJJ&NwI9s;`SGJ^0z#YRxB-O{srQTHrb`U
zv#v$1FJ*C(zjb<#g4PJAC9~~71lL}H4U8&6Lg;GG8Rqh^mZv??wY|_n!99O+M4JBs
zP2ARj<OU|}9}g=}6cY$>Hoyx5-?P|WZ~jCqm{N&9z3`G+u#oCcoiQ(0iLcopKM~zA
z5VCM*`Oo;sq^r9r3jO-ze8kaD;1k4r<{jAJGg7F=Zf8`?$ap!gmZm~m4*~%3_G#_N
zS=1r@h=A)#63<ZLLAj4!Ll2=3N_q2xcdNa<DG(=W69pcI4y`^QYaG5lD9Z-;+Q7ce
zeVs)FQ(LS$w^2>h!M#70<m-R7u5HN#pxSest7Kcey0$2_C+K?qVCodvZ5z}?B|)iq
z$5Z4(b&KtlpW)3b=!`0_p4q);XpGnO&Euw(9k9~@X?E|BLO!tv+9B-S)?4PjCVioN
z^$*HZwL^H5pMHq`0D-@Uke082$duz@$_(6I1BIyab$W%{-auy;e58D*7B~x1hQq&S
zKzzR1ru#02X`KSO9Fbe|!;g-Z0jQ(DHVs?cd~7_}dVf{Kx!_S}Ne|*+ipP4-)Nz0(
z>IQJONAi%QVxtw`*&eC6n?u#4HV^BBxs>EN;|T=LM$=PSRNSANw|93C)Reg*JWj}w
zq-L^W$?kHEW3iyPX|zDGRr*WY8Almu&0OJaq0!fJsdNQA7KYak=EY`9P72STgM@|w
zAYnAsylc_j(7@H&(Sid4K0FZkJsPbRbG44VYfQoYkS1-`X)Rq{U3Ft=vUe|8lgznV
z2b^~sc&3J&ZFU<t45v&1i&-vM?_#zR;u3BgJX}mUY#!Ol-g7E=qMHPQ6XwSX&`EN~
z-!-BEGmjpg2x~Rrj$f|F@U<$Frg6379q=eLeDL@gZm;%eWiRW{eGj$ai$U%AygtEf
zLz<|;PgFw>NXPqoDDXDTH`RykBNH^I(45f9N%PGi`U9$gQu%}?t+qk?^9$`IwmZNX
z{R7J9!=x6(kC|U^YU?4o1+h)ugX~eJ8l6rwR#4aRHstxib=o}oph+?eaq4N0O8yCe
zs?}vTfcd_nQE58pq~JM!*u&f1bqg)|reUglAm5&SzxJ0&7<se)8oXgCtSBtEWY0;N
z@W2&#SpzHw>2gBlj%FVu96#*Tn(1cQxDhzIq5|<I#3eXrtrp?RXG75tyWp^sLfoGO
zpwH{B;-BztPGGfKCUshe@O?!8SY01->>EKXof6o8E*CKkm8gB$`o`s3U!7UsYiBLZ
zi|N!<hneQotkSEKCcg=|H5XsOt`Y$))NwALMU5K(3|uSRB*cl?9L<-FzTWx{;H{=<
z;U)5Qe=pZqye--M68&Vsjn`!HWp&cXa52fNR?HYk{phvn{Be5IvjyogwZJ<tOrycQ
z<$`~$C#>B$6R<#I32V(ff&n0TR}&=pI*F{9J+CJLl;a6v>9v)DK@oWJuJ~dXhO|R$
zIw&|fLG57bzid+Wg3%Z<l}2nNHO{4IBG7M&pXmtrkj*aBUae960rG`1M-1KA`GC@3
zJD@zxKfUeHN}W3SWofrf_S$h8A3unj!t78#?%Y@jtvO8E`#6tjK{zx@(LC-;ZrMMa
zz)<Zrd{43_5i2BXXM{3Bb<e&G<KVgg+*Ao!cr~f--9ez-p^lR&lQ+F%<tZ|^8^U)&
z0)efMwQwDkK=}RnTDEd!fknT`U%rk|sg}TwHzCl6Z%W|M!-I30E12e0%*pO$%6rbZ
z>fi-l(}439$4UAS#3B51>C9{RDKYQn?f~+grmG9r2S47AADutUdYOhP6MhUoGk70(
z8vI_k;VD2qd!ihruKq2F_oir=`d6b^1yLREog*kN^y<~Fctj6qkf);v|9wTJc{u}k
zM2Z6Pw0znVYW)l+ZSIg7?~{04FGoREZs((q2)N|?0*DI*XPt2M1{!-Jly{AvB4=X2
z!b0TDnqi(_!{N&XPmVrGYE6}fEh83IG;#y$!<UK914-y`at8G!=wp}(3A!^6e2jX5
zezNO2U;2~!5j9sIP93a&0FaSvqS~*N*r)S0#k8W)o7Bx<YDQEVEtXD<ZA0ATgQ<-Z
z-~R^H1Fg;DfEHepB$_(~X%F5u=fronLtk|}L|=!Y9l{~Bf5OR4UfuV7@&`x*#a)wj
z$<~Frgb$EG+SQ-;V>B6j$UzfXo-eIY)ZXEVf*QY(iGp9*cvKN(v(W>wOuD9mLKItX
zVAP@4se?TC$AN)nM`Tdi!Y-HaY*-B1l>4#z(UvQ5{#Oj2R=z@*+z1}ns*E8`{F*?!
z?G5lA3sE0v#>#q9ppdu%;D`I1z{A%#x4DiEUS|cb<DwagHAlafYS@w_>C>J{fO&zu
z(X(Z#u#59C0X-{XB%*By7iu>b(o*m?m-2rx_2zL+oZJ6+d%IxOl3J>0SyHQlEoxjr
z1Txo)%C!p7+7=Zh3TUc;h#*^*xxI)HS<;G%7+|VYu?j>)K(?e3AR<D9D60^Y03ou3
zkc4Ei%>15c@8|pbi@YQTm?!6)_jV=^Wous-{l0qa5)nE1$>&L}-6C31Gp#6#t(KhA
zE}=M4JRUpXi^&&U<<;y(J66Av3;ib=zddh^f3VYzgZ~-An7I9^%Zr=W_>JyjFYv+9
zW6ST2j4#9<v&Jq77wlWCnc14no@R_as8NWowf0sTxrviYQ#v3EBQhg$ruW=qnBSG>
z86g`l4&erh1vS^4Ahhf1fcz$1<kfNB)j1)~DqB`cK$|#*g%)r3gbV>H)f;C0q=py+
zG6;(kZ^o%Ne+$}WusxVyrtooI9J~iQWlbL3dH2@C*-vUT>&j+hD1#=tnn=g{&B65Z
z)+1`%Y!wYb2CT-`C_B_AIYqJqyQZvhYJ`rzG8bEqs-<RYDm`_>W51hq=Vqlcvt2Ah
zZ{~!-KjJf;#ai_d>wRp%k&8E}eP6a1um~X}hcV5rN0oigN9GCW4F1Cg?)@)`gLdVQ
zNsD%ef5B(X=!D-BZ>AFz*-~ExEmgfi8TO1gbPrW#SV9r`Z|tH?oTyCI?_qAjww70)
z)mfZGVn=VMrb1uWf-~m>Mjen<42uuBuZdzf0x6CNiE|Mi)$J?kR&>GcU*?^j*|jZa
z<tKr6p#JIimj8pAG2?-nuwBblD}447t{d-v=7MFqvo_nA(ST)^i)rmmG)D}!-b3JP
z4qgMpY@@dwIm6vKJWej0_Z&X40)3p~U~nOJG!+xT+a>J9*ZUdIP^l)wGEKwW?TK?@
zW~6Np1DS4zUY*(A(en&q)JFsu^1^+qn#1NxuVt~e=#So<)<}gh)-)(qmvLf7ZA9^J
z$<%gOBi2`=f^JK>?@$R`)2Da$poPv_>2>%d7B{7?!wc0Sno?Q_(Xmrt-g$(;(bi3B
zVns)n)P&d5+UujNmHJ%$b1NOpK8PuJU~7;aZ4I#V&dp;DYT{&pt!+*o?lXh)1?|>~
zZBQ>U*$%!3!!&dOerMCD8bMwXhrAKP5QIG$werYyENv-Pqac1OHnC5csZx*K3}61T
zaN;>};+ED%T<w3G!d@fjJMPFwf8C_iJryoZ`HUQLIS{MVwN;|y?uc<TYCaN*`qUCk
z^NF)f1ZB3=(uyt*2wPu<{XA5un<05Jn_5IkqYxgkcCVO7N@mH5^6&_wzS2ROBp)v>
z!rNYa2=_fc+Ux(iE#L@wFGKO)tVCjkf<zovU47>I9&6{NUS0N-BouL3@@-jEv5RXB
zJI9r=_|Buu3=fO<ZEgd(j`7;j9eiHL&qAn{!;hVru5{oqvN^KUi;o#&zx$BuBKmo+
zrJ85%kZI+FWe{aV<gT)ewo<e=CH0jCrKkQ#5W{rzxxY+l$FZ*qHV(@w+OxC7&TOg^
zxnWwJz#g+>%(iD(5;f?4#(~Xoen?0?YViUH6@XLeLbYfLQktjlnFY>Mn8&<2d<uoj
zetC1dyQxqRY^~9Y-eDM7$v<spdJO6{X(tV<wXQt%t9*nHd-sOGyBfEKz^AO<Qf_xs
zw=@JEAPs=&`N||A|563M#Epw_KYcarW%7_uCYyB&i&GmXQ0-vf-VkdvLu9a9+I;)g
z2krx?^|cGWhu#rx`ox_$F>^Sm@mcm7*JQJ^E8`Ci)mKwYOhM-a%!9&2jyLBK11+QI
zLW{(7QM-`;XaU@O%$c7c7o!r+#SB*RSCNNc`52qxO6E>Hloj*zd7GP8WVa%$nP$&M
z?bdepP{lseJbcvXxLbN{xcI}*uAXy~eDJ&_^6UOfT*A&boC8m8PpxAt|1IAYxlky`
z{+F!B@ZPi=pT!auONDa<NyRM9iIC;G-s!ydPNq)#V~K3kf%OHyW?JEooSM~fA?sQp
zw4d;;J;Srvw2<Msl9*_b=f|zYCE3;{1Z(<b+Vp(2AY$aU^Xn^#5At%RwLg0N#H>B|
zgS`&Tt8J10#WglN+5VvX5%r$rvOjrpFI@eXGMwxYuIsco;^snp6z-AoVI{fNEbZw4
zEj1ChmYKrEh#x#_>C+b1b#>!(09d3$umOY1-d8tT&4>+l4w=k!KIoQj3+*lTWLOQw
zws~kAG$}$QSch3TA2ED1E-5T)z_FbM>;$e%ov#XLzT3`2oy^<%jEWqc{MfEwVo>Q~
zz0kdFYK%Pk8+mY4<VI9UB8TK-URhE0-9k5?XmUm>;RoL+cb0bEF7rZ5&G8vK!J#zx
zzUvp{0{QBnJke{$`CLKOqh2-z?w{H2Is`7J6TAI<8ECKbJ3sZXFfP$T1dB7d)r*Ll
zD{4Ch4gcmEQdu`H9{%Ea{wnK|i3WsGwc<{?k|3~iBEkKaTGk}9)akpjDa4oO!}AMR
zjM_f-t|5z_`DD*}1s$@eIK&h~jXY%kf&$A4T1kt<{eYqm>Lm~><FC`m74z`yef0;M
zI^4V5+bFXwcRac25VHl*RJ42X2@vXA%ZX)miFzo4@<sN<=B$83xu}g=iwZei>-Yv$
zafsQ&h#s~O5zr!SXp@)5#1rrn($YbzK`9mt-LoA|O%Js+%MUhK8hB~8FO#oI-%>iH
z0j}oRU<fEVfI04tUYDj#qV$oIV&c7#9&lt<gT*!F-2vCk{$Soo7v94Pq2RJvOZmUR
zB+W5UQR|>7rr&U?XO>)jRW^7`5(*t&ZT5zl+dT365%T9tdxEfajBIVHU5@K_f_1LW
z*n~o|#xkM$J)P2GugSy1gyK->)xr8{6scuQXL<b@yLf^zW$b(Ik3s>tpL+M@+W?4@
z?}rf8G?B^Ow_afiF!YK1D^DA*#jS7%h8=$6Szfw+3$E44L)o`o`$&=xjmO;CH+FVx
z)BO0aqralhBAkWpJLg<--m(&j>oTI9oUTnBWJ?6Y;27Aq(1~H4))x~^NelXzzQ;_>
zd9G5c<FO|i=2-eJB&hrxrI6D<tu*eTo0<fQwg9cT%yah^v_+xFj@f;_ac9pXv2J@c
zswldz*XDj23J97PX^Tv<y2J9oV^J-bdvMYn^!Y7^Q)H^DSm=7|9zK`YLNi9e9|;&g
zcgm6EYhw5vtK%4;nD!|M65wjA4h%M&=5mbCXYyafs>dhL3Z*!}K!`jpmiobUAqlq`
z3HU33VL06v{UXHF1HIZr0lr!Kky^;YE2MbTJ3gSsG1;@f>rNK<XzB}GWguzC4LwLQ
z{_=-0a6dxyp2=h21P{_wI7kl*eC(ahL{=#`PCm(MHF}Cw9H|^k2*_Tw<_aZY<>BbX
zM$0C*j0f{zOtzCh_JEKR$aiD0D)&LqQD?c`E!wa1`C8tinO3Sm%7*V3#{OFD6sqYG
zdK_0=a3S9m8k$4_vT=#bX(Wg7IaO7E(f!%9HfL~9s<TS0f3f<f)3*MSvUo*U(EV-t
zUypyT<L<Y5j|R*vVC^@qq(&^<nKYc!Gp~uvEi&thp~=jULo)4wAvdz?DtLK#_QbE`
zq0H1%GezdVmUDQ-px8#jvQQ(s-rl3&^y!a_NB(y&fL7yJUqsw}{y7Sf6<a=!_)~d1
zngb>mOQXptT_iwrmN?d}JK)Pe7`GhH$38k3_tiY*7<nZlby`()|EVL{dSB9cev?KY
zlEa%uQ5P`iV2!ja_HFeI7}T%02{;;yORb5vvJqQlo7cd#Sf34UmO>5QA-LKat|n@I
z&Nsl507Xja_>kE!BzXC;*$R&CweM-^RL6a(V8TE$TiaN;obk`ukF4HR0rdv+01J<w
zyme-_!a(vAXtDAb5}YO_p8EAHMjR||e}&Vp#uqqTP#aq!=7>d`EmL1wdv-k$x~IlO
z@yQn&SV?+~up!*}rhCHrS_cG~mff}^ZX#LM3KgQ_*ebj6CGJaD4_exPgV(d3($^=E
zF9aD|OnsY1xzUSN8`t0yjVD<n$b5?LL~$TyKbs0^v_jI?MMTfSb>9Xp?vts+y$_&X
zLBRWXVXnkf`0GE!iJvqf5v%NpTRh*}nR`Dez>sBM5~2I<+lEEYnkdzaRofk?5M#BE
z?;i*fwu}o<1dw%7h5o9OiW5#5fr3v@%ZrihDweTD=SE&KmmPG@k-E!`qa_@X-{O*4
z4}y&m|8Dy@FT`S}FtST%xdJ+tQ?ZkHoS884YX>9QjSQ)}{L$y#1UT#uZV*fB9bT`f
z*si*G_WLT7ML+_#WZ<5TBYUzv@t!T!K{pS>!J$vnt12aRm1^m@Hn{9*=M)qZ*ZJk8
zqCvRK5M>~G$p09Gd1v<qJ#6rqTn%6gW`2XwN+0f+wIp*hhL(EwYnA;p&>FuE0*?+l
zj%*vKlMFpjiftIxVzk~%yp<RsmR8sNO!%$H59Tow#8mK_;L-nHlaASB9Ce|<AeBxm
zoCF$9efxI$9cN~+bzR&uQVI!Q+xpx^AoZaZz!A+Ig_Eyvj?BMo(rd`F+(Fq?o%s5e
zBbGTYW+tN`>ufHk$UL1>7R$P{uzKCos$^L?spY;lotTD0cK*&QlTDHstE!Hi=NsV&
zG5C8iZUCxA-Dv^t&-yqbs+K||+}h{sZg{#2n=($^j6WXb7rB08?31q<is{H4{ClS{
z(h@=V@Z)(?uBUT)hRlhfvy3bM%rUIhjM4Y#Ia8?tQ}6$t4jaV5vn%rKUmdiLo{2}+
zDUK1lbwmPr@Iv6aKyy@a{-=leO}|`XLvfGoybGcKY~@XJ!F2of<1_0^3CW)EGdlyY
z>yfj~j0=Ue6APTqthP?*_s+B&J)xCq+xeEEk!CT?7=N4RgI%u3Gm54cNS`H-v*(wS
z_!E%O5%SS3VPO>qEY<2@R8^p{c;ZAJ-rZFuunozY4)$z}n$7&MQlRUY!0ODr%Fimv
zvlef0z8_qT*;F$gXNp>IYxcPftDHW*-!25U8m<2N?doUUhz)?MhByCd2p?BRf@Tb!
zT`DNE-Y_+WcA|fB9=6(ELU`~r|Kwg}n@t2rfjcG+p?z8dq_9j@ckoJJ^D(uEE|zXN
z0pE(W-l6Dz^DLRA&Ls8euy$yDo^iNE&cpI%rrof^_7b)qw3nP!L35JjK?CvnVm#b<
z?Vb@unr?$n^vv)yuT||fe<Kj%+4u%60sjwu&$=ZSR;&tVaRo`Q<*H@Y(cCrWIWhg`
z(%6^U`gHow?334(kE4>-(2(6m!S;Nn*T}wK-S&Jx&HPTz*lb*J_e19ryFu#``ksRs
zmD|t86MY|{^ZYZEA5agqW>1A@a{AvJyE)O6i;TYJ{CMz*>X<)5s_koMEDvWR;&g-(
zZQSR`(uyhCp2qeyHRWVy<`0KVh^%6vhf{5Igpn-M29lK&vZi%UTo-{#{B3E1iH!K+
z{v!(l#Mc5f3+d}?BbxskW4~ZYW1F~1jY@8COOu38);LE$C^E0GjEWO9;`~Z%*fAPQ
zv`(56`G7|pqEAc-ltYh{;vZ=M#=P_`o#!K>>&1Xi;FX+HfLav$fx{ZdV$}a{DDo5E
zm=~#coIP)o-~^_Q(z8bgqpZo3H)fW4{cIECsxh*Z-2mq&_u9lct5xOenEzYH(^A1P
zxnW#AW89#(dj)<j^y`nx6ZdeHs92HN#*Y2L8vY6s2aH3IZjb5YZj8Ane6!`N^-Qwy
zSpd0mFdQ(nHH=x+gO>z^x_|E`QX%>94@7(eBkuesxPU|88m7@(<Ec*%8{G|!@FsYR
z<^ERmXH3(2ouU=5vB;0!qrNiDA(EiJ5%FU38Ej$3x&sLZW6OnVhmOZ4rd&taOZYl-
z(MR>4yYgCpQ*rCNBUGl>%}{NO?)aI$zY1H65EApuIXm{LM@LUo<Pom8){b{Bb=YNW
z&QNb<hxjN)2_yJaMy5dL#=6$Gf4^y;pysuJ4e7NXDa+vd{Ecv|MzKLDb|1=Y-OPwS
z)7!oYdz|t_g0OU4*r=0fQ?C;aZ)VO=2ehBSCXXS&MC3&H4aFii&KD%F#%er|lf7_S
zP=d8#Fra>e8MFBi>Ox3c->Ws;g{!d5_UWlrSnWya0m_d!5NY@u-qQ7gYsyvzHHC7w
z*+%yO&#P90+6sYxOzb{o<NM49l(V#G<r~|-MO*c=zWR<=QU~96^?Gogmj6KC(@lMO
zxRP2b=#Lytx!6;<fyuANpKssSvluqbD>jZ*<B|Z?crm+}BH%A-haR>P4_~&Q|HFWf
ze|9d}CB8=(9*#Zn{mjo|ImclGpxU0Ch7b~X)6{lkC|D_m(HKnL>`_)^zTn-kR-;#1
zxFN=PqyNacAnkP{x|%A>6MI;rR6RX6mzZ18^KcyYX&LMEZ!tNmj~ddwJZs`vr{mX-
zE!{%?oQoHkU#CfKPAqLHCWh7<S{OIib^)5U%e^5i+=)+S870}P$kx*{!Cco6WYOoz
z#4U`6E8Ve)3Pg*9lN<A+V>PKnmPvJ<1B;r#V9ii8D3LXOU$Ddws@s14*}Sy9_^EJb
z*5-IgTTq)j%a}BWl;154FveGO@*AOE%C}{2HS7VKgce8qIW>w@2~A=2<Hy9v20}sG
z4nthErcgkO-d7NNe9Gj7y<_fg`9uiZYj?9$FpKeafWom!YFiSU8{mTEpQ&rR)Fsy@
zhrstnn5x1PrMoW*Z0cISm(7_VjDPiJRi8;Keu)$)hz)QA@)9oWYp|ati>(4{;hOJ>
zzt6&p{qQ={gRo;8A~WK(jzL>;klh1z<hc-k|BQ^VeVR*Q_*&yRQ~s0zORMSzV_Z=*
zx|?__1aT&RmnbiWV=^McZVQ8r?E<S;!RWRjEddFtc`o98YWOje-JjDaV>408Qe=2!
zG+XNY0KMw@YfNlH+<NyPZ%)BSmaYrTEFll%#i#U{q>CzFe^*jf^PJ0I8DEa3<`Y*R
zP%PSEWRERfriOA2)Bt42g`Gtphoxq^lLw|BGMoHkOs%5FtQN9Hv%pX5FW=U8RWwQ<
z2Kqfht@8z&g^GFI4>Q;>0+SXq&Jc~LUoC-(Qi?5-4eViE8yC~HT@5|6-e^?)mPS35
zl6^F?T{yZ?quGn;Re^Qslv%+?)K_t?FdciVe<qgx!o;yDFx<qIv-RvcEWv7RH_@bE
z0M1$j-GA96Y=C17Q&Kv1h%WAiLx36sxW-$^QmPghfY+C(hI_sQWr@$Ck&wk*EJYq0
z=s&C1)Uk-spXRhnsn)6f8Zg`iE0n3sDxKy07BSl*DK1|9v0%iFChT6!?{;TAxbvtL
z-Eb97(4njO-fPMjPVVA>*xUYv)b-oEP!|E}$o0H#oM`gafI6}^(KaXpLNSR4-(PoO
zU&a)Qq(n?NHxmTR6s-A`Zzl7>u^x+NZz;>z7s!~2S+Uc!@5Kk#9Om#oaT1L$T_G9u
z|9;g!>e2TkH-a0%+kikq4%S+}Vm7%k>KW&_nS;6;6vs}@iA=||KTDC_ZEkhb8X?6~
z6Oz<=lIuioVsBOn8U-S&E17jo?>8^0B@wb4-#Ofz{Z03%r^poFTDi9teL}R12HG9t
zep-mqBJ%?m$5*KH+(26;AB6Spu7fQmAT=Zb0Ha>U02P=X4Eb)kQ;r1~_wkLi?2%#Y
ze}b6d(rmuXrHx)4KQw_VbiLYY{a215@RP;?)4rhLIah#&QiYua2tzCKi?<D_<L%z&
z0<Q)rONp7@V16UEBOEaGSrZLXv+G;DseP7b?FJbIMKT}Q6HI(f;+b>0&O!i@XR)^~
zE|UN}T&0*wk2H@b7D85;5cm%Zg&vkAR5M(wMJoybl0Zs42Q12pq%76f{Guf5?fjCf
zF4Ey7z0hwzVSUH#1XI(zU0~zBB$1dUj0i?DUA}EO9Vvs4VGa<&IEN7Gfh%z_vfot*
zK5t^R%jH5c{qmuVv0<Ox^`_Pbh!O3PE45=R(nnZBC?zKF&bGz$-`B2qJa6yk*O#+$
zNH<iwA$GrH5wa#^<78&BOJ5VIg}$q}IX9EYd_WE`$Y6PKu1mpM`VGt0rxuU9qwQ|;
z9M_PI)-Q`QkjzuoZ+h4ZunCD*Vch2P;&mZgF5s3R>|rVX?NmCXM0f}HOT<m$nA(W=
ztk&bmGdqEI4_t}OB;UdR1ISS&O4v{rw$t@C+JnzW%C5O*^9k22yw@2gG+uaj*4fH&
zbqc%Qslk8fz(e(0FE&*DL|q8V4qV><DVUXdWj7#rk51lefG2>3hbR-10N+y+XU(}*
z?1TUwP)ldc)%SM1QcF7yO{l3f=5(JLepr&%-L7Wgcgbn!OAGO?TAndtWA%z){5P=y
z&xS7`SVi+aQjjOgy=F1-TL0L0>S5F#qls}&x;bS1wUHH!6bB@K{=y8hIYOqCY|c*?
z+mArcP6V9FKfKr&hsla*`NWi!g=A>2Wm=>4CwcR0o(lyEFLl5Fh^!b@vZGm%#hC?~
z7m+2_gn#JVcm89~&gjoS-@l$XR|G|<#2-E{LJsZ9mJ)C3ewR(l6d!#RAVZ2IMDui4
zD*=zV-+P|do_iISd$?_haz{>KV}Vc$S>`;*UgLV*GGR<fBic8NBI<PNt0dWFYB-`3
zE=7zZa7P7s0Ejy^cjyV0UnLohUTfC)rpN<_oE74%HA~FF4Dudhy9+7A>O;1si>DT{
z;%XHY$Fx5(YE@da&NzBiP-IpCsq$&hJEc-kZSx8yft5FfY7bb;m1;dGhqnZaj$s|v
z$Z<|2WOITzz>ZVLyg1Nl!D!>WNWoj^Z>zd~)41jE?+3g1V~Odg&j*ESOfAJ}Kwu%Z
zdjD+uZ}q@;@C#*&AowQskZ?o&3QjqNOv2bdo9Wq8M)^~*|Ix9<!Ei-Uu=N*&$C7q?
ziWyaS3oUg*K(>@Ccz)gj;D;Zr6y%5>figKgbK;c+C6&p)qIg;2?K^XxjCJ;qZWo!K
zwRz<GupTjt_17Ur!rq!+C;}i@vXs;W^ayvbli5M^ya^*Q7?)<$rC<|{J#uz(JQBGK
zZ@%(rLClJ+*v^xUe(&wykZw|q6+0sQ#Vh_~rv-<-krW4aX^I6Fc>rT~K`^tHk@DVj
z^Dm8EN^>^VS@vM$1L)}*7faY(;P7yQL)Lt$q;l2jCX(msXFmKx+uDmakGaQSs@LPy
zVCQ30eQR6m8>yB9S<@E6{b@S|n=5X)kD5)wj9xaUm@~6?qTM4o!2usqDt)H8!LcVC
z@yQL>#rOEW(HIO{ZC*LBUmJd`!)<IVMw!|F`bR1KZ9@};z;CJX5HxXHNxlb~mp*TU
z`0M0;EzPF046*RBf(!it^0NZ2{*|rIO`&xDx&~r{SAX*8Gjhf|i5ff?HB)J&)*H;_
zq2@M#Tux0ChrkwT2~_U@%Tc=eCa`)Q6-*lRJ#_~L@~HKi%A`pg+y7yEqMv2onV}PX
zBdMzytFJ!@TRv|diRCNs8Gv7lBwv@m@H(|dHR53z9@(+WQ~yS(@R=5lNQ&#PXxCQi
zg^q~xtl8Yj{9dchyTASDaO6zR%S&lLQW+Ph#yDKW8TFkuZj~4qSh0?aMR!Sur|i7k
z+?TGXY5hLmVL7GR?$&ml@e#gY4l}(y+tuZAV7g@HH*x$$<w8t%C5HT<HBk|7+|m3>
z9<!Ur_`;EhK5iQYnw7LK>^#CrViZjuv_z?bo+il0Y5Ve{aH$6EL#5%D_wm2pkLFva
zRh<TO0+mFwbm*ovxmm)V^L<*5Xml!4UYtmU6S+t;$x)g`%H}--rrf4-6d%&R&7lm!
z9f@y>v^0Zs5PrGb*81XjE8t@6l`Sr0)0ry?!0uiyqb5I+;?<y-Y!n&#JLNE*AHQbG
zs;KwZaWBNl?@56$20{*R{WoATv>%-H>7g3_5YGxY)7l%8h91k0aLql_^H}Uxu{GCq
z{u!iW!h1^a{J#NS$U**eWd$OQmjziEw*s<zw8z-A`GUofgl>Q7Bk#vOB9G%NV|(?Y
z6RJQ)ZPH5apE-eXT^-(HvLJ3vR95B6k}#2oeoS&`W}T*)O$ZU7K)f9C#e<FL@Ao*3
z|89YXsvSh|#qYk$Ilu2x%3`<eyidM#_o=0j3;+0<*>w9Hg}!)2kdK2StNHidF3?*i
z#>f%PAI=N31kVo3x*s?ct#jHUQ{~*a|8S)_G}M<l!#E*n<Hts0NAm*Wc0*`B6l$5U
z?{;GFFCsj7qlZ`01(IgB<;>O{F><2N%LqP*PJEt3&e#A3r;J$Q&eGi4FqzXyT2nu*
zN9z@i&_p_68Ezbz56fw|`!wN>-D=F%0c&WJ=4!KeO3;JWD8(pw0;P{`eLQ_wFFwFz
z9D*lJ19}=XWQdVM<L`k+Ikc(4K@f*CmTQePHB83`%+j+hQC5Lo;8z}Ze4H~rc>4I|
zTw5g#nuJcVeB#vIp)AzwZwNN0Q)s&?Cmw@#`Q$OM5?lwqJb?^ql~1z_x9WlM7OCV|
ztbbZXTJ$Lv;C|geu>_9`8gc?PVjuw+s9Q>N<ewB@C2b?Q^pg9$?4kJ#Blg*cX*{tb
zeH$cALR<WYqY8?S{sG^2HAP+O43Gp-SLyZ`WofMee5y>j%!P0C3_AWy-2|N%qdA*$
zbu&HA7gJqM=GUV^=4q8kbPqcc64mb2U_^N%850AaOe23-xEF>!_g*tGmp6X6;x0QW
z9{C~cyMM8|yw;TaT|O9eJCr=e<^%|!RZBA(pF%8OQ~SIftgcV}+aRB9z%*T7pN>p5
z_`N7(8K0<|IK1usi>fs=V_zpTjl*fYQ6!ELQx2ggvUOv>(@&Ax;HqYc^L0m9#`ap5
z5){w4q_K12DR+9Nud*1EMsum2NzFnU>$a;4;saC@Y)|s%R_SB*kNIywe6<Z>C=2^B
zfmx0C0N~!IO&sUkJ3sKhI{T3h>TC!o72p6UO#A`B9f(by>Px)yf7zfASZ!56D0b}B
z80CdvDpH#mxVYmbOR$_{aK&!S2(8u7)35v)re_sCWQHkX?UD0T`hC|k;Tx@2kcnEj
ziQ>gbG5s5sJ5n{jDg&HY<mpR_-^2`8A+6--B0^t)afCh!x4dLrd_MuxRirw&4VpB<
zrIHv|Jy-goNJt1ZCM8-@^?%53Imp@>8g@YOel_4p^l$#7)~BR!b3Bf^1<OYx4qUjk
z{E|Me_5NRG%Ga7y-_u+ABwJfrGgVF*_adtI-}dwVd{=Cw$|zl|N0zVO;kTBY0h@Nk
zJQWu65ORJxJRd3V+EDFzZDQ2<fs*5UfDul9GR^-l)zS~T*QnstD-GJ{K+$H=4n_5c
ztw6M1JK8%RSrksdd`pUuX*m!3D3<uj=@7W(0)>f8aHbp}KMZO_W~Ev96_kYOfrdz1
zGyJ&rj`bzzg6e+GWX#AztLjKYACzF7G{s%j4ZWN)8zPlzTSCtkLI5C8$plz~Xe+8d
zGsR$dXNm$+JXm+qfK?iJR1`~8Beq0|^6#*T0JXyKMvZ$1QE=d^$o|3PAw-ka`kZbA
z_ZN4ko0nVP#H<gWc|&3nUpN<Azo`jR+_MIAPnAnV<JLc<?uSchN=uJnL2%s0ZYA{H
zPnNKe4`iEQ<`4O3frTg6inCcYtue-kB)KHuVEAl1P%}2+XD%R|6YzcMYe}6F8ptC(
zYUUBk7A0fte?C$zRh<6PF=W;#{q7@8X%Kru>jH+uug*Ca3Dkc)-0)blJTqERWFCuD
zDYOS>jK@>OW$f5^V#KSj8?PC`9u)OXtSLz+oF_=S9;b1VwC>1wu*<DVF4Ee0pzCM9
zxytAi1vKXb8jE!Y|D#dHy(U=-7x;acvt}+l3PCc-R>s8@-;Ns5&je(1{PcDBn%%f6
z<_(20tCPJc3R^pcDh9QAyPYFOPRPBg(<!+EIbu9Yln3}81;@Q>cF@>74J>S%&!(Ql
zw1?F=A5IVq0tvWbw$~3I>L${$(8%y{L|qCQKylH;Qk&rWsN<UL%Qc&wcDBLkt)NsO
z1!Yz9zxqlpD7TdvT0BnKRC;#Sy@OhApcj)fTeJoc5HYf)kmgBWFdV7Y(-i(XOSG3C
zeePT$wA(t$yEOp6CtTdQJwE5LKMP+<u_GPO(%43>09nEY{P(XmHXzNc%;q+VmIth~
zDn{1`r?8w)nKfO=j9E>cm?@=NWT4qqv<6qL)oIZ6-NjDX<gX*MWlCP}*cTFTA0@x*
zCey_6sFr<pLB;?B9W=e=Ia^ok60~koxxIR1cVxn!nxSizH2beu8A7ICYt2X_m{i@1
z0y2)vDIqiRPaHp%->Vr4Z?{962qXr)u`8de%mit|yD>kT*5mOq>n9ABdm3qI6k&{(
zBpr9X9!c)Kl5@gCq#DxDuMYd+@!C$UIMFl9oy;w41{n1lMAkE!;mb%E+hDnmeaJU1
z!ZnLs#e2vpL)yILZY~~I(Dqa0nKBo2{Z(N%bMo29A$(ZE){Nr)l>Tr~U2mruH2eNT
zv$|%{XkR(dDF85xvfeshGF38Y()N>pSS#SfW*;PK=dT;G1rA;uh!gjkdu(D8sIvyx
z#YwT1j@Q5W1|6&Kefxx82YpV`R+ro2LERrbAY<ufwvWBZ!8qFpeuaB_%B(9(J%?Jb
z=?#hTh1&VBnHS5_`|xBH$BGNI(0Rhmaeq(ECmOE*I;D7Xo&4ML{pnI-QH1C=i`Ce3
z6G|g^{r8xq&mwV@WDYtjEh>$2-_#Q>-pPD&n==h=Chp1NVfOy7&_@4C2ZV=OB*}{z
zJ<vbUeanq*E<5I}I$+f4==)b(2(6Dg7}c_JP?fO?+kJ>c;{&u(T*@MR?G%1FdUICm
z^l@L6QL$Ll_QA8d_NK0wt<QIPSyYNEBvZQ;q>AYTfw$wXYwXt4X)av8sW@u*W6os0
zAlAn9m-$y}GWE->W-S8lsISp9ZKWWbVB=9&IosYkO`A`YN$?ya0*p>uhG%FBlA@T}
zY|z+LC0mo5z{16qj(6%3a4X*w_ckto#TJwg*_r3x6};1u-mzk;ty&sv;bTUR0e_42
zYzOnwrNT#v)WV~VL00u4_~h{D9w++)XUy(5<3#WrY()|E)HKa{Y+}`Ds{K<zmEV_$
z{dZ)I*_;|t**Be~%n?uA`)KTWjPF^Eb|FMBMa{xXCw!tCp&$64LU@0$(ZRzKhLn<t
zaahYfHZMt^HJ*zXt_b;|40UT$KHFy;snA5Bb0L;7LK^f>Y~z(x4xIs))|!*DxJf$$
z-D<3zb6n*PoKKlZT}P9<KN{Ba-`g~27D0Q&FO17qedU|6o*Vxp<=6ImsAPoeGPk(*
zVlJtcoFwsbJ&Cn~$x;=&xH*eRCdvZ&<hrm^z?DAAhQc@Qh|)6~#XXNxMxj34El$(b
zK$gax&oat7^Ng0S_>JSJmp#(q0#O;iPmeXQ2^C3=Nomw5_;V<;8P}k>+;+Ss6mK$-
zL^fHk(y<D!@|?8}7{tjnX0f4%*br_LhU(1f@>b`EN^!E_WDJN2*pT@<bBjcK|BUAk
z8~UAYf`!d{8{DWC8#(5k?d<P;$BC<ef%T3Ye>YAWC!IZT_RA0^($TBF16Dx3v-Tuo
zu6+C!=h4M1vcO=V7Ho#$28D!F=#8gKQ*Js9H7SAUFSL^#^B37YmbrVotUFKP)W;g*
z)|iaCnR{6XC(8IQW1^)cf*C6Fa%6?JBhkkA>r~V|VA|xmS5WR*Fq&4(d78u8+<K(^
z);|7|i#ED^BVY$<clZ|7pol$YzVn_ytQgt(`pzF$H-^9XW7m7hFaGdzPKiMOY+km@
zFh-t3nF*Ddi>~-!O|Sp6+HdmmrsEWBF0qPmw;;i?WSM4@V{P*-J@sVkngfr(fV~cb
zQ^#L`bhj0I#3m5q3czm#ft8HO*IpEszA{Q4=}tZ%CL;MrFbYCE)U_Ev-gAJS8K(PI
z%!AD7V#45^SmTDFiB`rFw~vYKI7KczV6jDuZI+|A;a_y$ek*V|M*Dxw?choM@Zscl
z`5G{Grq<1-*-}+!#RF#XK*P~@V+g3Jr4Uaoc+m-&tvPWC?>s3My(NY8ch;>`V8Ap9
zF~HLj;8x-FQr8zZ3kPUmMQp+#waWaH-1|93S@%V|AK{ZsH@*qOut55Z4{Ou3U*W_<
ze&k$yjBIj12s13&ZBB0x1hk{WMnH>vz-uPY$%c=lQL?*~F;B-9;k?&cE|c$0_Qsf4
z*k>R!(R1mBcVfNe5|CrFI$ucG9HX4-Vf)BKZ2<C2nH`hKxsA1=E$|2Ej$BHxTg=yi
z$Ad0?pX0T)HmB=W&Y|!1%srUu-?z7Odb|FdbKPj_JD~<q1h(C!K*A<VOH`vIAgk}|
z%@rIwDXk?a=F&Rq{l*O~G^aF2`&FhpM))JRzxvN%K!ZP+HWpv2hzX4vujoOv#hlx&
z)OexSjwDqeF-bOFl$UFApq7E0UV=#`=@cj==p4b~$_iMdEMAO#!&vuwGX$0>1*pY;
z(2|>OJFPe#?qrU+cS<T)U#<p()QtW4(sq9Tg%03V;}ckjTLM04Q(OlD&jol~Or-(l
zLyJB_DG4%_sfqWt%tY4Pto;A;^nv6=e?ja}!X}G4@rO-~0ebePoU%vb8B_sOvbmOW
zGx*c*ty^(Embus1cT94vqd`mz3z2U2Tm9WBT%zQbfK{V#d#(D@u}bP_KwEcyhI))_
z9WUOMeY>6anqroB#Kf)XzTYYY7eM5-BMwU%<%}6G<Wi8)L)12`NC{scHaRxToXG6e
zrp^Ql%DdSOvjgc0XeWv|td0GI)06u1DpzaEij(9owzNd(_OxCnWIK=S5~M9A;Vph8
zmYKZ#lHB1je)7-gO(du)Ub~ed*uJwk?H5RSIWW4G-Jd!mrk!&nyB;8hzp!YX$bPd2
z*RQ_2&}ncbLlNbU9WHc{j(EGQCTfHnp8gl!_6@-;i7w$lq!bzM>Ae)!Jxx(unAHev
zBQzSm*}<$@Ni?Ztk`tJz@5LJIN=Wi63lh0=&GWzA``HV&OCOtXvty6d52p4&XCCTB
zH9rmC+_bMw-&0wg!m;UhZJ}RAzXNPNu5WSMNxL6Dy&<q6I36E<xT6ZB69=}8NtWCk
zG>gtwxvBst_${?mP<k$>ei_+pZqtYShxy@&k&mNoF}cJSMC*BG+1<$YK|q?Piu3_L
z4elGLEvSm<J~h61FjfB*qkfv&`pwLsMf>OW%J`|+Tl3sY=)OkHnD&=kUXi(}N6_ZR
znmM?R8?t2tA)MnOL5%e;_OBwgBhCx!$RQZlsW7&-C7#jMDQE-wg)3_ddiPkO2;rlx
zKM|J{1sfR}>*Pjv<75Y0`1$(%;!hOq!_TjLxU&U>TCG<O?yPe+?MYtl%St&IFY*f7
zQ6|j@xJ^JHqp7udQ8tV3h97}Yed{=W&m7eV!m3ykC7a-7@qx4T>Xf7fP<sxwmb?LP
z2Twb#Qs(%5OQ;;y?Wt9cb#1a2Sibba<uewu@OND$nUPB{C4nw67#ELmpRLTFFy@Q#
z3VrUNi*zybBe>%oSST}`kqX8w<+%^V4|x+HxLrPSY&6^eW>5;HB<;zU=_Oo{f2loT
z(^75YM#wi_F&lhiCM)BjI3U9K?3Y8^=-B0St>1vwkOCHG-sTo<+S1zuzZk~z^rPKK
zu**>#q=jRw(Vy<_H;b-hSuwRzfM0Hyy&|5K#`7`}`sH;4!Ki^NZot-^KQM6Jp4y)2
zB@#_Pmh2d2P8;KtPqPVpM!o=#W7Hz9gKUt5rLb7N0U+gDwWw18cZ@G&oq2G-n0n^2
zv?XbuIO%DlxXkR1l|YTW5qCkqTrNrqTTHp>;My>=i)1lcI+PCN!$^7}ktl}({=z=q
znZ5jXpWCs%>EA3lxKk*(`r`BC`>vN+7G0$)*!E(a7i~2jtd&n3JziV@ax8XV2gn9B
zS!$z$56DV3ZW^VxbmbFfbDn5~M}P&uL7(`e%6W_C!Y!C9xgacgu@uWA>zWswT3xu=
zy>qoImQlSqQ?(Ii5KFwi0IjYa8aT7z)3#qc8Ky)AmXU_s!8|OZ)`bjg5&lf`PA-oz
z)Z2V0pxcyuv?Z<}Pj9gJ#T}Jio;TS|4EB=^mj5(-{ChodsGAzhw;e<^JtMK@t^8V&
z+VQH$JRy<->O_q^3Jw!!=k;&?d@J?=kS)m1%9+Jbg&IsSoP`UdV4}kKbC>(GMP_0j
zUC(KF?s~yDE`_Py(>5d`zC0HP7VSQ6R{LD6VHuAOpdrsD_86novq!s@N{;g;KOo*F
z6oB~XA_{TDA3}zwuP4S<)PSV!T*y~%&^fZI*v74lqEUj^;LHND^yq_DM_Vd8kx{Gr
zI-DgG4CLC)jDCUXZV(})e?nrNt@=wasA*rCf2p`2XZKI$Zy(pR{S*dJw`s*LE@>uS
z^=x@hv5&q{ELzIi6JCq0UB<vFc(*b#wZWPFOXejfsGLi@E<?L(K_H&jcGuIYG8_@4
z-$L2c{Qe1)$J>O>r;=y<G#UEOAj@!!Tyhjta@ZWHv!3~Yraaa&>Vv*0MY0oGTB+L-
zFRdG#)<d8+fq;@|iO7JLtEdlAk^`;Iw)*$?0r>{_P%C*I_;5i@L3s-uEA%eGzVQMp
zf3zuZR%>+shui>*o&A?`#OBLKTTz7}CIqHR)9x1BEjIwK9q^$kYX$SjR1O_akofLo
zxh;7cPo(2UrJ6V-oLxN`peNNN8X8Q+P}jeedzhcq?HrC*YSrep>7(&g4|w8)S&nR(
zTm7`E8t+nrW7{_MLB=Y)Z-gz57w!&dTXcddb1%;LGLnnykeK!5H{#opw-`;$+r18~
zcJ)K<taCwoZiDb8Uo&1oTEw&r_Z7!oaQ$M`d0c9Z$VJv>;Va)ag}g{D$k+|H{I1@B
zuGqCcCHA{0a_fx$PY}5wNv8FQ&6X)1Kj*?{cW{UZ%JvXVw})-YZv6Ht0Aq*)#Pqd>
z`PLHVdouDMuuHfiAga~4HPNE-vBbABGSyn8T}zWFa(Bz3Kn4S2ToP?4OaS%A@U3#7
z3dBvt8BdIbm^-Ux1tNx=qP0*hfe~x^YhH*?k8~YlkMZMnwTDrcaa1F&Z%cpmE{yb6
zbG6Cg^wM8Sp&#Zb`;unMOeFsGljFRKs>?ZsH`E8EkZoP8L0|3n!6Zn8Qvd(j&-hfi
z6rM5&=(um>Hk$uKDNaocOztxqSU8A=9#S{KPYtA@Nu&=SAX3|<h3QjLNG4TFPivTs
z$_*#Q>$(F<Ay1iNT-X#~x}%1t`??gntd9Ie<N}bwRPBQBb`ML-@PfW39&ZF+SlIi#
zsYg7UO{uQw>;y7jINo-bAyq=k5b$@i>>b6Zc0u;FnZu0u1ujU~aK<UD&5JJvS}S2L
z)hzmRr#1F>`2EZC{jHb2+Ul27AF<(+Bzf~LTK}&HUa!6wPv}nV{dW=}3p$r55<qwE
z(+>5e`Je9*ia6KplIM+OQ{1V#&kB1pTN|NTG0PKZ8i7dEkdWX`MP!oVeHeP23tJg4
zA7~iI8KP+wn-upmUzd>~RXGR_4O4gVl!)bsU(d5llDF$_GMbb|F)Snx8m7UH4i%6q
zIU<fFgK=iV3}p?stvtV?hR7+n4$n7FO#y5cf0YrB8P-g`B|cr(%U%Dka_;^K;zv_&
zMz+=Im3pgxGKi#X*;S3Dn60_YZ9y*<hrop()+e<M9p+>n9k+=LU{CDN!jju>!(|hn
zyv*zt0_6DEyu|XR7B$D{Num6%WfDV;8rD4AkUfRincM3cgZFC<M`vG3o3E&SSmf5$
zrj5}}zvP1CkHvl+wK<@t_}X=d%ae<fE5NoXVz}l`HhVj2e^;kMT1ZJeHOXWdS5k*9
zH`YFd$qzNX+vTgl&5`>S=LA|#&@W)~-?wr<<fV(3UwAycmr$~;YghHKfxX`EoHxO$
zQz*mbS_r?no0$B*+|hwvB{P#cpD&{<I(02f!XXfN*VI~t13FH!%00kM_65E!6L#l8
znRqUMBf`3s)kb}L9z!lwB7?9?%Bb{Eql`A=hGfFNEXGr<3xv5y^Huz(_+u7%-j=Ag
zv~~49<bu(ST1eApy5FGi>Pau6AW<_A*o+?r#*$KwM!n7Q{lD@l{OBcQYidRPf5wIV
z?H$?1$;3q^|B*qgW!@(}NfXxt>&X1?CtKeps2@n7!)B1Aj(_<QB#z=hkE@pA;w`)2
zzrCd)!6^T}frFQr+4M#+1HY4S-!*hlP?BYsqDC-z!Pc#b7{-`Syc`-m&f#<uLlt2+
z#Kqn9wT|7RgyW#YJ4)?NpxJ<hcoBRtZVuU`RxDctw!j+aP_28h9G8+}S6An_)oXlT
z<ZC(VeV9;GCl)+X>%})rU+q77lKDJe$iz-R5qgPcx-8mVjw_J;M%4*??xx3r_q$V5
ze>}76-@XpWPHy_IM@)YIVf^gZ+X5`{pJ#Gc)wfI-{gZUEpP6ID9%a++a7z&~3D#Ho
z(lmvK7e`$eu#$4b!7N>$CxN;A<39?F8g)Xvrpv3_(5id}8zUybJEXCd=toQ#JH^h8
z*3#$*tnjAt8+&Mmx%d&mV<aI0;B9~Tb)yjwa!0#5u*;`7mKaam5;dg-fy;O5kCz!M
zFz+y5U_4a6m;$N27}}FUD5yWq@5xSeVEsFBVSJ1KgyW=(P40OMy0!e6m}|;Ath2O9
zu-)oftIz!oc!6g6zM#J;?D%|JefikX_}dNWYK+gXuIt$g(i0sW5EH+9{EC(zU|osJ
z(J@RYy=QVH4vHbLRp#UsvmXUlL$WY&8@t}LOX?oNFg3sd26S-0RuoxSIWhB?$Y1w#
zO(?j^%O15Hgd;8yqP5S(vwjx;k*yN@WdoyPzVV3$?FbrA?b>kMv8oCs4$`9fjt5+g
zHnykEPorYm8|@VGYi!{^jMYD{`H$QF&zsgdT%FqO2AuJ$8_%pJ%Z1c{rHi#Wq}oIV
zS?I16eQoK0ZhN06OoZ7ZgLGrm#T(E+zp;Pdjp(Mixi!1V6Y1My<c>am%*)mQ5XguE
z1_cY8&g*SQouuK-;fmC*mqVKN1-E#saaoRpo`#6e9Lt<3#^)j=u-a)APZvX0Z#aCT
z=4gb(CTzz9PMEY+^-SulwQjaG+3g`-nF|gluHed<|0dvmabG#w#*vR<u6f)B1@=3D
zbly^FAxIfTSv&or$9Y=oeF{juj^3<z_*P(OAkNN>{D*Ba5xd_5E0LgV4*ZQ5=peLm
zt^tO!pLprJh6KRTFjz=wj1g>?vcgpT<sJH2GeXNc-e8$4*pPEd9Y}lQLWJOb!SiL2
zt!`i!=rdQO|A4T3Yx@c*y2^e_pGq6oW1RzcKu~js)Pvz$dwNfTDWdO-NTYGDXB;+q
zZcP8|$TI?UIMHHBYrpM0s8I!3b@K9KsZ9paH?JZ4`1daVbN^}jN|RgYs>&60%U!;u
z)O`+Cv(Pt7hdul(K%E(iYCXyp=W^^M{#$AZLoZrMp`NNVgv@7YDUkDnnXB_z85;{d
zi3f=Z#+40cu9N#b7#43B`)n{+D@JC?aDs#M8lrphyx~)28IURuKdLlN%;XI<<ukQK
zoCm+8g8C`ai(;^uv-e$D?xPbDO7**P0eYNOIeDP7=6PO(Kv#}*gY+cdt6)XI$-<iC
zmo~gC?fwqnYuiEm9h-bs2hyv`L=Y^wyIfrRXKmfv11WHNd8ss~hVu*?g31nEsrcJM
zdM5@I(nf5P2=}pIn|mV8!jp8ppwfJIQ3-bM)`oH9gjWx2l09$;HnuD%s3WdtnD&O(
znkEHDWs1S*t{y%}dWw6e$}e~yT0LX8{&Z%W9xTs~WFy(#?gP5eqnFa?@x*u$yb157
z&NlD&(G*iHQM$cGS>dZV`y^R0SMAL^9jVob2I$Y>FHgILhsl<;e&go5^V`k<!%kKB
z^PN4{nCT@HQR5VWl0B94WO^@&&B}<7YaI*8CmGo+kSF1TovRtINq6$0jT#9vjp8+b
zhCH%9Z(TlcE_UAU-2_`Fo{<`3_@*b1_Gk6oI6b3vK_Nft;+E!p`H7Y=d+MQ_i%6)}
zoir5;HdE^qxw)hGj|~1IJf#ZQ3el4$?eRk|%!`W47lEw!=urbQzUrYTc>q5aZBuZ3
z25cIRh+nSc6=-gD3~bJ-{8=pBK(J{y<A;0A(&TJl0WtA2Q=cgXv+aCspDlU}Ae<r0
zb^w!EAuyu$$A5~Eiol~KV5K+@B*rIf)kDWk`wZd^ZbwlIGAWqaoYUKGrpy{lBwVJs
zA>HgvK8Z`*$1!hBh1xJaMU^N$tYHeLpzwrdaT&{addw-HMJ>xge~-$PE%hZTbVuY%
z1Z?<1AvVTmBS;<Q!Klq?B)NiY_&yS&KCj|TfBo%1ZDTl67W*JLVOaXzFU)%qLjKMp
zJFgSAItMPM3he&nnNvcfNZqLure(ifno9~L{L1u{NFTG-?BI753A;CwU4asJ|KO??
zlU4T2MUec$^{%-2vmotjM~$~$Nb=mjmMf(j!?yMb{hN&-Be|}8`_bP58E5J_Ga#Fm
z{8E|Gt}BHU(U=iv-eB|H>o^iPyLPWJ&R3&>8r7&)IOSpVtMg}SIL=QrqtcU`dxT|z
zaqIs!^(&hBuO$aY2Ya>3yKmD5V0lVB4m>$&H#Y?4w~K7<b^`u?qhf06?NaHaRr@H3
z_|TxXVn+pVpPI^0-WT8?R)?#k&=xhT3e!@5iiPhSKLQEhyJD|5e%}Eyy0LJ*;RT94
z<_hp3jkY*?jfwFUKdTOxC*paYQIdHO6qS=EwhLoF*{g-qh&#5lkkPqdH9zl%Q?D-r
zubga_*NBNAEXngBOBq*iMuqscQOM;JDrseHuyNvI1L`F9et;K!F-`nvNym!PouX5#
zuI9a0yJEv&{O#R>Ke{3{V>#R7Gmkz?rg38k1791ylDiDrZ7KvB`PLJ1-E)nM-$*KC
zuz6V`*K#pzO{YjLnnrMQ1-X+5pAUDGCJlkieS2I7o=u&P((c6!wJ&u&{373-<qPJ~
zg9|a?+JcQrBF4ay_G;?c6rMg9t6&m{f?h|QfpPIjyiecb6wh~W2+$s$aJg9nZdM{@
z9>YBnM=G?UAs3$e^CW}7!|Tjm`;ikp(lRJAOpM35B)RV~&o*Fh22aHFfgWe|XWcvl
zjJYX6+-jZu0e9QLu?2|*w$^r(cmbg4j<hp3CnqL+2W{13#bO2UE484^=O^<q`tT6M
zd-Y^H?Sk*5b$ko08XIdz!Bg(}ZO!06z?6yqY4Ab!!d9^+xe}kDx^VfMK8}-*i_ILM
z=sG|HmdBG8z|4GBbK@UYh!ZbErm%VUlD`y%=kUn|W=rb=Zo0Qw;$@xvV+g8>mlFrw
z(-46-TtB_!kD^k9624hB&W7Wq(^*AfVxoo7w5xiZpaLPJnBc4rHPxRS&*72Ix8_(}
z|M|&Mk>tB0JIS?#Mp4Ck#SP@^m}GKA!5MzvUn`JVi=s1+v;;<V)FZ8hq1&>dvqFwt
zoSZu^bY_$`AMp(fAZuurVHBA)txQ$~YdHbMhkP^fV^qedN^)pyH_n^ZT;y5OiG&p-
z+6H{zkQU+J&PO=YtwX|?j=qc)ja)50mlY>V^#eEf1na&&(loQ}<}yF-=l279p{4^L
zgr$W7Eu%BWI#%?J)JiiD)gS;AGcLB}cK$pbgEOKE@qWmb>o(A)kkS>Apwl(rQ&w1j
z-v@E6xA&2G$7Tc*A3UkKZ%gJU(0a|vnnkW8Q4%zZoqfsC8hVliFW&wPyw8+FHuG5c
z`yl8{9|B^Ice@cyNh4kLhEo_1^m*_faR5#;<l0oX(TDC_;}AuWj^tA}PU_WX!fUO^
z3w@F1f05H)nMl}#>VBSahLX*BI8iC%&U5{)Fi@<%T!^1X{0>{^kfB8Cu;yVDV@jhq
zhVb5stxL4upsDyjtnPEB@_){8Z1Yn!uk`k-cYW9_-~LxXJpG8SRdqao9IfBWuzDV0
zZX4AuVWs#DqifqYdGpDI<fj5M_Q&W|s5Z{NVNTSTCmg-Fo$3?Kg)RP;<5+X1n96Eg
zg*WfUD^Nir0&;5J&VvD`fRyAZa||}>+8pb@AqZYQ|CEaRF|e6%ERj(KWl%9k{7~+h
zM=0FzKSEO@0#2N*8e+x~o6^~D(%voLe~$yjvS+iDdj{CgmFJ&k{cY|5NqR}0%_e<2
zLbPV($uxf5kV$J<6MVR@Mpdw``|gH&-U?!g6*D~98u@SXFp#H{ON(r}s6KQ8Gi%h+
z$#OUakLGk$B{CDRbXAcRg3v-U0^T)Z1apBhN8vm5-tfe-3x}<I{F)K?A~E=l0V#2}
z1zfVmfYaCG#BM$eCS!kMV>lEZZB;Cdu4-;m-vhqmgd2bJo^q8EWKh}S;M%^fnC#}#
zw4%U-cw#u?GA}OvsZJ=ibG3x6b1O3R%y|uMy>#w+$!5l=|H>u(m%fuOZr)Isu}vy$
zC70Cvsu<yvQNWa6*P%X)Sh9)7jl&Wg$SyNPa%k**N)R>5vT&Z?AD`|6%h=D*;qlL)
z;r^umkEu6-YU29ehqbmv>k?aAH;~k-Rf~#9UDz@zR@|dii;5BzG+IQ;CIU&O7ByPX
zh@yxrqoSZ9L`6W>QIQ=9$i7Ah5C}`iMz)!`?~V5Rd;jO2137@6lgYiG<$0dZa|e@<
zGxE;ll>C9t?Zhm3s0yQ|sHB7<+^338KN-tf)BGz+sPTU=yk(^43m-$QuAlUI<1_iP
zTP62SkhjV(45d7g>M%I_*%B+SO~Y-iVOJxqhSWBUTPDu*v#-c=jY|fsdK$TYrgWcA
zGj;o^e|!=a#7z)uNO1?=#`wRcMYwWj`yt3Zn1+wI`PASxa!@Nm_M*g+0<kC30u1j*
z-O((hE|Zz}wgmFXwo>LFlj?)??=Lc%5AC4kzb(NG`fpgRdZz0neeG<v%1Sqw45(`9
zVMK0Q)_?!6@WkMoK;1uG9${OVP4(SB>7Wu7n{DUhDQECJa=QCVtWPWP^<Po0uTTHD
z;Ngaq1&^IBF1R>_wA`rh>c8d|unBp>dduG0JN883k!T-M-OoAi)47e9<w~}UxL-N)
z(~5u+pEuWP%1Yifx}0L8UDwll1UpnT?O`YGK)NQ_8cf67_h2a5yF0W*w4=36-8`Ew
zV8>0-S0cf;(;*m<B4r}?`R<((^dh3dhEIWC3tsI~<!6PXsTH=neE``m#r23Pm=$yd
zPVE^~SJesmT;noIIE=#)Fyq2;L#wrCOm00emW?*USa$rb|49)>m$O#XAKJ)c<f0E9
zT1^z`C+cmqIV)FwGx`%{db&~$p5Cmw{#l+VX>-b-SejsO%(qH!hq6;^FE}LcQz_Wv
z-hIMEw}PxXdc%;o1CaN=zQe8>6!|A>p^`P(t~*^TAFC9i$_dh|n9mbVO;N?q=&6Mb
zHk|72n(ctq#delo#fH@Hd0qZHBU&zbe~;1Y3|kjs-Sx=XOI7)Zq6~T_LkBsId7rU*
zGZ?lF))lq1O{rxs3226ub~3%j83uX%FD}LCl`U^s<1E&aZ#GU_eQVSFG|NZb`uW14
zv2*TA|EUxCoY8mQs2umR3Im&6CIPu_8<C5hs*U9}M_2S`y}F^rEWQEDL*MP7SS1TN
z1u8TEQ_Tp~6-5(2FlHfyS8QPoH;5v1)h<1YTA`FR85LM@7;=?pwUJONEDPS-<aZ^C
z5+;9xh&#Pvo%M|`xHN=*ATx-=QGY|n93juNojA%O1|TYfwgIZnAl78bfeAa4!^F<i
zd>SX|ki)%B#28CH58P#Z-u%$giif=VtddFy+=cJ{gmkW~mj43#a^u#Z@u~SLHV>4Z
z`Aqxhqysd$M*I|YR0%r}a+s{43L8U>w1@`i;?;M-NpT~PQbjN2N{`@pZH7-|@-$vZ
zP_{Uze+OQy)(vS{p^F&EU&d~@PZxpr7sP<U=vnMou5V0^o@^0PEF39ozAlAYfJGd8
zM)CKX6np;X1t1(nUl8<mDIrPUx-3)aQ_z2+tQ?2B#4@iN^d~f=1>XTv%LJayCZ*G2
zPi=RV*-xW`@=Q<8mNj%nz%>TMKfggRr(e%3ReS1BRr;vE!R=~lL_FDXY#GN-rzxx`
zxV1FDHvmH!{m*SS(mPj6U3G)pfBC{i*nvF}#9arf-Q{@|A(pa;+Ew7MqO_u1$qeJ%
z5IrF-)ZXVui;6khi7s;YC-Ds!REY^UMWgX9KERru=A>al@eNgVP)>#B2f^^=^)@sM
z`bNMHh{`s6vLEm~EP_m1io>7((PbuD22!Fi)j<#AX@8c;T)ev7ov+no@O4#zqqXk~
z6!$X4$mX}}{vXbISa0mmhbDY4b?`%+aS73*N*S<rw7H!1+zE>2A5CA>5HOovLGh@X
z-#m9Y=IDRg>JnNqW9!-GUz4!ZI}gcd)P58FLMmE<NdzO!_eCQ`-@7YT9%pfiZWY{;
zyJ}o*UeH(wfYy02emaI{Cp%?(gDv#2iTkVejc5lD)@=L88aHR{f=x>C*2DG-Qf^E9
zqlX3;LVe-i;(xZ4tf4zyvgA#ihnzBe)IaMP*CRc>8hV=Zqep&V(EEO1V5h3p)?a(<
zL7Y6ee$V?w`%2}iWYsqa!q?5p<@q;YI>VJtmHr#N>7^QKZ+PyV5I8ji?1D4zwjhXe
z<~;}+k^73EB^6I~$9E;$w-NIr<##|=g))Oe4xmN?N-kP8S)aL?UVTFh`%oSH6O*bC
zo{WR?YEc~;N}a7pYx5@khwPLujH2vDIB_S*50s<Qf==T_Cp66|K!kaT0}}3P3s5*8
zUMXs0IrwjqF|(-#-0jmqo**B7l%ETSl-WlHL@B2YxI6lUUK7+HCb$_CF&?9PP0u)E
zq!v?8QB9c%P4}t7)$Faw`T<o<&Ulwt)SECA*AD*x#lF>>livSKT%PuIZkhF&Oih1x
zc$(-bAR0YK`d%+R4L!)Hc3KC7K(r&V^K$p24_M{}w@%wsDc<l3MPCEq^^er%JGup%
z9BqAVpR3kMLZ_<M4b&=j{djF&&NQpsvM|vU2+}MRMQ0-C=d%4v7P>UYCEBJrSc8Cf
zZ|Ig7F~9L*50V>zu~k@bS<;y*%zC~h=ll%VYL$hXgLy<(*Ftl~HT^T%IQb6@QUJZg
z^mo0eu1TPe`WulluYSZssLtBCKFf3vuQZ<KKx04?#P0>50ml%dmVyX!Yo59)fFjCG
znrTxmAJ(_JztD)`^F;HUV1KbjRGNx+q=k=Grs7w(63|YnfTBu5!wghe$n)gV#?Q!p
zz9xuW_W325S=9ouNgp<BRXPf0#Lo%^Yo?d1V`Zb&#$iLN|5`{__?n%iVMM)NzPiz9
zkV+_`|CwAX4H!whEMU5{TD;Zr7xjiIHw*WTq<4;sO!jhAXmY``#75E#&mKWU$e93r
z(rOf)V#T3K`+tMUm5CsR_Xte8hZxtRqmL%&w6{zDWp|}t2rVg*BHJYRj<1NubKtA-
zIDf^o4N%e?v*g9*!CeNY2mQ?Z`nxv>YUfC8#pq$DQ%#xsx-LZ1CH<(O!n*a*s|<ok
zj;KyXs!M(uOphnK-Z<B?bheDVzdj}&eJniYkAXRGbX}83T%fbI-fEu%xi(jwf13V{
zY63#%Fd<R*gQTvJ9j1!{MimsrElwhsDE^Pg`oRA$b_&lPeOJ<ZmVlGp!M#Iz|Flbx
z>v!DK3dQOT&sNkLPKXz|q`RjuQPWVPDKdSsP~gN|dXk}bfU`UW0T&6GmPXt+C3TZ5
z7iMnL3#t7yQHQCq9MrP@d=FumIY$QMRy1ywW{wyr6o;hDpu__~4_vu<+F``AuK;M=
zl<I|NR{mOkRs1+nu-fCYD>)=MwlmYvIWTV(w;Wv>tSCyEY{i+m$x5D16$)N35kM?R
zIcDn-Yd`oQT-cMP*Aldb8%+=RkHrg72>H0)E)sKIo>k4VEL?a&;YqyTv5wka>AJ6L
z(YFU^+rmR;I(?aQcDL|B?YeH;e*KPw9mHj3$@<N!S=8SbM3;TyXd$PrGdTo=_Sd~%
zXZ)6NEYtmNKzh7ReY`RtB2@ntH=by7guFH0$)8am-9SU~U`;Vne#~!aIsZ%*%L3XX
zO~Z)~UhyNHyDK42T<p22HBNuZGKk=mA#_+_^pjI~&zDfGu-*b3I9l~-gxe&Tv76Ev
zW^db_c%Um?g^I+gxMqXofUW;hZ!$|G|IffHFld<iB-8v4!rEywY~m={sd-w_z_JXj
zT*VnV{iC<o?$KiCh+sTm+f&v8Vc-s^LnGz>Ad?MqxImE5A8MrQUF0ZZSS#SgCWyh?
zS8`n4OQm4Y!t5}UlQ%kis!Xz=BsMfyaUMGu-b+Z)`ipJJq~P?vAdcqi7@E0+v$Ekn
z=hp;XuTO+hW63+CcJB<giSH)fUO`lX=*dnz74+`%4RO(8^WG2+*F2*hUH>KR`U1uE
ze?Kg+y4QcRiBrdz1ddfO!|(BL=liZ$rRa3qQ@r#GZb3wC1>K#y)b2axs-xQ%X8Tft
z;acJCkMsQ0mW*K4va9?z3W2ZgS$X3e{I;UsGbjL4M84Pc_gE7xga~>ZHh2AC3%$Ce
zkZ|NF&bx&^LQZt?`7^5Cv8S@O2}gRAr+o%@FKHPMCG7EkK_HUy`@?+|)X$ER4ng0_
z*MD~K9oGBDPm51Bc^K9lsNQH|>9aN3y&l@Dd)VedKw0%p{Fx9D?e-jyn;OCaNAuCo
zZj|nUziaADoMSWmm;dp{LWkq>P1<ded^906*(ouP@Og!Xo^gUF2UKQ=Jz#f3HnJe!
zmE<%y(!<=g%jl!*T7XMjC0<856wP~pj4cX!)p>2MhqkYGJa$?XJUrRVxUMrj1;a-Q
zy%?RvAUegY>fI}edH9tnzbwle2Ao3J)1SRkD0JFDTd~9J58hjwgjEXmw(e$yLU4Ty
znA4ZAX!D&tZ(m08E+50*bJi+e>P(}c1<wU$of6vKK83?#!oiwD!AcMMQ0ZyNEd4tz
zE4z$>Q&dpAtLs$VqZcspMv^V(e1FD<`zvbbe^03$6#8BoZ59NjZHM^^1yf}`dNfA2
z{0^!+P`Ab*<ZLlxxs*tEXrkpvJi5q9R2=B?`Cl_`B7ULN6^`k`6z#5s!H3>>Rf@oX
zKJaDmC6iDbVje7UN|SB=G%nihc#>W}tfe(t{>1cqVbV+t{okb8C-t%%t>5l)dE`gU
zKmO~@`+dmZ`kQ_~FNFMXMi0;!asxkd^)fP1+n=Zv294Ak7fL&;h|nOp7W)(I2knZ5
ze}wdgmG<VT<+@WlLkuSKvo0@CHVkcBS$VSrJjL(-oNFY!Zs^C|=2RyNMs(U>8peY=
ze%&-|u%0Na=G_AQ*16OA%V)%&7?qHkhIe4jCJUz@!`J&cRyzd0UV8p0Q}l}ey6P<B
zrR7hcx$3uzB~VAE?{+n3uxj}X_fZ$^+|^YK4x{T0pFh%fV>P-ox9cp#IvH}ROQh?1
zUgV;<gUtF@s4-(u-wCeRKzJDHTj5l`pZ^^_rg*Pc)e@{u{yJ;mqP;A#4)xYoN#9|k
z^3DptrLay_IJKETL!cvp2yFyN78o?bktUO-lH9EkeO8b|7xH1Gk!(uPWesEh>6Sm6
z6SQj6|NFT1&(QHGCe#o8ugidSmyl-#7Ha9x<g?2-r;&dMum4jj7nY2y^V@Ppj9<BN
z!J4i$@Gb9*25~%;BbxNEb4grZzKB-<`{?<rs}d~ts}@$!3D`0Oyp_*l8l8!2QV8ZQ
zvtuplm_w&C{)EHqll;~w!<!a(h<?uFd}wrO8H@$a%c*k^^PdXH({BK`JL^){?Fq1l
zJF*~s-;J6^`H>{)*e>${FYYy8>u=o|Z<!5YS~GLC4&o5ClG-)-4%A%3SUmW-*Oa3Z
zkUX`H8jmfF@2b(Bq)Z6`!C6Jt8+WL`Wx<ghkAnVf9Iicto6fY@p}diQZH-}?cget3
zcZ$^+Js=pV?Q~w!=fs&Q>=pGojOaIp0|T@mF`43MC6t$_iIm!VIP7j|#dDysoY#UO
z*u2RTaG?wW8lS}uG|h$lwXN~KrYvl@${rA@91>D-k>emKX)zxbgN%(=Q`NO{@P8TF
z^uqtYXe1rRxI#M=C94b-=-+p>-)$^l)#LpSRZl375`nZ$>#OFzSm+E%3{_2g1iel!
z>V%Zp@w`qpbV{A$t*iany`b#hR6~}o_K3<>HyGXntsSk4yK>;CASS~3g3GJ|MNz|=
z$}XR@VtQx#47Zt`=HWjqhz&{Zrt7lqQe*q*Kf0ZF3iw7f);x6TchRS5NvquKohlQJ
z&%Dx{+ARO(!%Jyb9iLJ4@<Pp;wgW4;O4E=SgzvnDoGe!<es<R#`H;dI>@-h2`j&?I
z5F=#~D&GKcJet|riyHo<^!4{gYEzeHj$Dy!cQX`2lg%939cW>^h(V52485ESO(wf;
z7w~4Vp0eU$ThP#&kp~fuxz_j#s(HweTlKa68Isq6j*>UxO$u<l;}c$?Vcd0s^X_|=
z6&yf6Kq$6_S>B$2OLInc><y}eNg?&qoF(L>(z2y;6FvCoVKrn0c_YnQ%ny!~?Pb0^
zW1`M$Lm_h5f#d|$50e8e)>=w~1%emFxKS%;7?0P_CMj9ol}4WN+X9_?KB{AI?#ghE
z*n3!aG%i!Ui>J|V(0}kbZFny%P&~!ozUKQx{Hw6Ye~%opxqEa1Hiy2vIu0pNypvA6
zAYOxL_2&Hj71UM4|K@wOlVno@jT-6m+GQy2>6@!drWQX#zX3m_-R8_s<DYtJeNJn(
z_97JdmKhkETDvM;S9Kef+LI&Av)Hv^wi`YBXAaybS`CNf^`!|u6Z_^7y)4I!Q$LZH
zSqNw5ksYyipm(f*`@Bk@Y|Aa0SUdA%r&x6-f=badISh!zzGe$#IA3hh4@X(9KVMp<
zg{Ic&U_TLB8II;V!~)wHxl|fi653#b$1@t?*q4_P4C6bVdd)GN00B1;0<oXAbcOpf
z6H-wXCL2n{1)dR{()Xr{Uo8JFb)wg}lSV2vsQ+}jX*|d@LRnbw?0+vmskZzq_t;Vn
z9r%Gnm)oM;YEc7z3jZA#tZ&~{etUKL4_k@-sPX-ph~ZN$hBBqCR!qDn_%})QZ<qz`
zWIfP=FDhPk#vaNXk*pNEs;sX%lXg(54(f^iV`2h%3r<i+R`GkMLv%Uh$Z@Nr7|<&K
zKF<G;K|gK&?#B754L3`#@2gb5xF_Cce(c61@lPYmcZFZH{DzJb>W>+<tG8Ky<@9N6
z1-qTjWW!+$X?P+t>vkdS#?#yAbnK9IfUd|0eM5E6ihj2A_U)A)c9J$79Ql|&0ngsX
zS>cggC5KdS?!I24*7yx0UifkK>ZV0`&<0~+gpq(o3-4W^dnFhcfWyWa#jvo^9t>^b
z0<l)kDRGZSOp`&o4|TRTt3aDeCS}*Se&EgQlkty+X8VxRR6Q7qZ@$%*Gi3qGFbYr~
zal-eDT_*fym3+tBtf9Q?%P+d8kN>;ekx_E!Vu*q*+WPWJmycowx{LD7Lpx91P9Jbp
z%rUffF%@QR)s_)iO`8;1gmdWfV%gSb0;NDn8(iPI2>kMCEdk5G&(qhw``Z8FFnz|R
zWoTuVXqXkT?y&Y0w3BhE!bIKsNuIRQPzg8<Q3ka5o}Or*K@N*^ytfR!BWqpe9BpsF
zYJ^Y2uMG)SG?K`DOuphrDr<Xbpp$k~UT`~fBC0=_?scr=K(bFgP4Mov|Fx7;C3tGx
zi30DjuHR3zU*OffZtR`4%8`DLO0$B!)}F04e@9~<%nD!8itNAWQ}Jh~@T6AEIh8m1
zJW{Ww8n`;y1~dBTxW|~G&Id$EU#*Bw<h9ft5)NQqp#omdU6=bza@e<?CgcnsX-8J{
zwzOsSq)LUH@-jI!hdr3UYq-3uXJ5w$IaM<b8T}Fj_2U<T>ktBwi9sA<imj42)4y4!
zum;AJiY(o3VlnkS`Y%#MKf0{DU2Qdt7GdzoCjajjcn`gY)q{mDSp+&nU+Cv<;H)uH
zkvx@|d|J~~dV$aNQ@ht!Spmn9aWh87#^TM?vv5pJyt$)UE9jzGXAUry^gCbl%amO>
zAr#~HwUt!ALnswz?OFNUPOQp$eo!4s)p?b@QXTJi9Q|j?CErpq|2MF#lQfoqSF1)m
zz0)eL)2U0iQJmvGa(L43SB=S&ZH8~#FB{iNU%p@K)2dhUy!wB}1HTRTdy{VdabNNq
z=P?Vhk1P1_1H7akbz?IE>4Sgdzg!&e(#{{eXJ0Fb0I&1D_3w<t7=bH}$17aaw)~77
zv~iVIz=*Is8YeJ3PBeS5qm0Aw2X3lk+`75i`#i0XbM~4q6S1%N${wcgx4P#u484z7
zvv%WNe=KzLiq}^iv%c~1!c07wUJwZA59!VQI7t*D#sMOK6xU;7iP()mDbwR#N_CVY
z6-fD~#gI_@sa^jq-#pOmxV8W!y2C%3j@nHyh7NZCHB};KpOtGG<jAvRxVzy(9l&e!
zsTlC~P%FX8nZ)F7t*|Z`%oXKE1Dh&oC8!Zvdy;jLi{*dhUo$L@u^poGP7GsCPxX9B
zEl4v&Ch}xc30d-cU7cW3@lAX$tL*S3OYN_LD|)HLQYx!=qK(Ua9uqg*_TP%HSX?h8
z>@H3!TPYls>auQ~>RNdH<2RNIu#>-D&E~aSbG*vjf7K&OZFPgcsg-XhEnR)Z(}Uaj
zaLQv=;a<cett`jjEv}_uhI&cMtgu1MP%|;xOcs&CniVxH&d)tAeL|l#)w_pfpNW*l
z7-A+at}x6q)O!UpksM*aYw;8Dsu!Kl9}HV;)g+F)@1zCO$;x#XLzdRksh0R-qd~e`
zL!|)}Pe0|^-%3Pe1cVRVyKdBxj$`|&z0|SK-+J0_aVCcLQD)yc+(d7oN5>_6E7R@R
zMBfr`n_d8Hh7Y`zTT~G8b4%CsqDs7ozT@qVPp3e7>DwZ)lIFLt)Z*U|y{lg8A{7^R
z2uARhlvQ5Aw}6+p(s#0oT8BKCBx^wed#gt~)=hXLx7fOT_bSJ#ds80MgBxP3YRqhg
z@s1T5+oWxp){6pr!{WJHJlDw*1FqC9(5^M!;hl|2<$12r-r{xVTzo9fzBO_W+#$1j
zZON;1B_&fQl8@*2IZlO_sL>bEc3<mBe2FT(+oQ81n0II!BzI{}3{QjpSPXuBciKa*
zXOlrrRqDWtKMV_P%>3GCTY78rWEdfH*Q9#)R-1v9Ia#3J@u2lXKPlCp{C=k2T!Zdl
z;$v2J-vbP7W*1?+W4|Z#XQAf2&WFn`BSUTmQ36ObbcP)q=x6~+VC_6Y@S30c*O`}4
zUNI!aRaw|`LL^r#1hL?Ub1-nQd&TgP36w(8eGlB*`UYc=K1jcMd;mo7&Q0DXqXO^N
zkcmbYtw7=gkJT`%9nV3(oso-DdM$HBd{!^|uE=E=>wE29?R<K(og8yh{qUZc#`2$z
z_Opg)b7kxJhU2&)hF0csC_jBeM;#5sfZf6coNI<b*Hzwqw|K>!&bZ2_h^4=|IXDUW
zC6Tstzbg$jk1aH{#}AdC)4F^cOO$<iZNT}56OBV*&&2%YZyMM1h51~+xR#!iWTPLs
zUC1|kZ28<xBhp(idKZ<f^gd*L-lFl{LH%O~ROWKaK1*hmn1}_Z)#!a{<OkYqR~%}6
zf||n{8_!A&^@#qFvriF<ak+qP*uUB<_N6RHP#M%-=?kL#LHP?7fp5XRHUsM5TQsYF
zP(FesNJMKuJl;MIZayUL0^vi=VC{rqIj$Xfc4f2tJsyrb+$j>n9GFEU5-FcnI*|*P
zQCIbk*3j+Jj`R)<XyvQ~l1enXYOSaCQF?X<5_~2v>|E)r0wp1FNZ^8TID?&1P?t=;
z@wWmoDp!))$Jg_G&Y#k~-st3}x$m}rv?5*R0=-2xzoggDXsW2x9a{#pYTM|Q+etUo
zZ;0PazJMz<vMob*ivGy2e!`mLU$ADyx%_Phv;s%NqM8d9t!^52O->B<GDoxSd8Xz_
z{JHmzW3@-#mhjC^9<nSiE8&zX@-x7(WqI?H$`y0H=0{atNIIyKCPX0`TNkn}CUHrt
zEIF04MSb9x*3p9l2Vg*TN;{Wb8UV_()FY4SJZ)Y>&$o!af%N&wy)=VKR<RU8spa`e
zXqvO-4<8a9R0<ksyfjq@<w_t&K|c))`e?J_@j^)tn6l(i4?qQ=2%9HApu%RkKA*!~
z5~!L|UKS0S$LN34(<7TI2qu+G01cTALv`o2JSoK@ocwMm1@n4up%(jgT7gBM7QAq<
zb7BRI<8a}i$Y>^afHbdyu_t6c7$y6c5_VYW(bOmo2juL9(`Lsy=Jy6Sq~@aK=~`TN
zjzyL}6T?{Ga&!uPbNpa^j-g*wf|Rcp)t!vbR_-)d^s_WRhh`kD6s;tOu?ZtP$nrmm
z>zEikj*-U1OTPinquep&VPib$akO)*-o5HByt{qQ`mbN;PAq9yydao(_=C45vGAR*
zj4!>#JGOH@{7?gpvu?PO>c{l>6kbd96U=TZS^l6qk=ifLv7Rq5i`=0Rk+q_bDme6Q
zSN%lH@O;8@FHYhGSE-rA&CL^RlD4UBS^M2IS!6{;(-W*fD|0z+5U^jguyh{B9FC#6
z@o{>*AB7XZ@i=^jtASVCV#<R0xCDHeOvtZ(yDOXU5lO;vT#Q!@BLie2AsJa62Y+?W
z1u7Y~KAT*%<x}9^MXl@j>Lz*<jSRg7Lv@-MjY3%myM4sqn)AB2m|?n0!Q`LYktvQj
zXaJ~o{cF`DmA;fP&z5t+)p*8ipb75FU}$nzk7l<Z{%J$>j&xn2(NewX<wKZ^N`0MV
zK~?WC`<K0V>3K{PftKG5NHg5bolGWSjLi(2`*iZ@wEb*YZ3u_OE3;W=MWamQ0nhgy
zldMB6Ca48&O=bUl8O?v|Qh1-0_~!0e-HjkG_u|D;3t6Sdl;V5OcYR2<4BQ?wYRC4#
z`|^@7-@ua=vXVt2wFBSgY`qU?o5+n28v6Nd6D6~s8TxnQEBgJXc)YCl!Sl-PYH8MF
z)h>ivwHwcq`B&wjMtVK|IAjwNvad1S&`w_<Qr;n-54#z`c(RlUyxiJ<)W3l8u4a~0
zsB&4dd|5ZlUcF0-0Gus8^;FN&i;ODK=aS*mSD{kmBsW#W%Rq<sT~iF!7|l-olj~}V
z!`+jb#;&?2dhq@bw>K(6PlgA7eM~DF#@Z1Uc5`T7Ufe_MJ9K|7GZ-Y%gI7^~DZ_d?
zIkRULm8DKkvIT6+YvZbcwGA<3VNeprYuNC3NtZ1*Etc24e)y(M<8~ObS8Hr%{N9j*
z7Q2$8cKlT#WRsBBxZ(0suq@9_bszg+*AST@OQtb2uZ<p;w}XK1wlzFix6OIX3GI<~
z{&M5WFMCcdiW*T#%*KyX-}`;V*K;RIl1OZ59QI<Sr116vkGi>rIf?Z6-JpR}BN%H)
zoLz3`N&^3ZPiM)#Ry4*iGuBWaHbpDs8QO3!7bGAPiesnU2n4$kjz4(mN?x^1I;cgr
zf@Z27R;sRKC3%8uX`W|fqB=I3Ypr{v66;&ZzbGk>C<DFFRYcF?98*XMYV0cA=5jYZ
zlAWPnc?|Aje>sdlrXnsJ|C|k#!gQ^;2kQ~HBROaw?2g)v{6;aNO881Vei@ylj!g@`
zCy2n?G*ty?!A+1`;wGg}Iz0qXZwb$up-F$^rDYc14FsX`4tdLKy4}l0Id)sFm0zuu
zw(Z5AU;XZmp+A-^AJK<fla&sk;0tSy@?%rJl<%h=PSM4LZp9rd&%OR)J>TmAGd_Y`
z;}?dUKC*D)ho&(zcCMUS-aLBFYCWTcvBZ#MQQYUTEF|jQx>JX&`F~dJ9<8GZRX5G(
z8_W+#YxoEC@j{+XG%B)yu_16#STOdGo^53*%h-T8R=-2jVr^O~JO8q8Bz+Dg8`)J+
z&U5@2J#Ao~p)-hbQ#&-SAo=jNpN4FS%}@hlXm4v7HKn!F;#`T$=jW?+MO80UL5G|4
zk_Um*nuV&iOPeWa2o(*=z;yIq*oO$^v9?#EU}(4#<aP^<MkO=u4$~c`e6%5d&Nu9+
z3TK<;y-Op~|8M#8#I}-4Ht%%n_EUlHs+EHGKdySaBdNV9IsXLGL+1}gq1<iM1@HV4
zDycoq?}S%wRFm8Zz^0@$d&J5anOJBb`g4=$`Q^6_k!Go?kK{LGnpeI2&OW}8A5F`v
z&Owjutm0<ND;^99I4>j3n%S)dkI|d{F}hmnStN4#%Y>EV7-ij@QpT8K?wF`+c~R-i
zN!KJ65nm;alvprM^p=ZWan#NuHfEN`_U{<icG3Ffa7p0eAX&6_8$w(&JhmFxhHz8z
zvQc%y5N0>5X`%E&=N~l(sz`Aynu&Tc?qeCwvW$b8!g*iUNhy)KV(WxMt$^L{lI*p~
zv}YShYiXgskG?}CsH_rS9lOsHZN_{*#2tQ*IEF^W<4z@GtVaG$Lb9XLr$jtn#gh66
zob_0^gMJ7;cqip+Skdukx)DoZ(rc0YrkDC+O95|=RW_?qE8182GJ2S9+Ry>uH7x+V
zb)YEHO)G4)c}PI68y&HZ;J6Sa8zZblebT2k+77L-dwG@;xEZn@M_XL-dyk&(cm261
z5B-U+SsT%B-2_$=ofrOqNs6EIVo6be4AZyO(P&C~k=W_(M5k*Zi|2Z#8If8f_%S%)
zCqY6Y@#zbj8ZEydM|Q4P%2UN9-L-e$z5Vd+wV7~R?~(blFNP%XCU4&FpQf?Ha`MVU
z!+DMWI?7VEX}95;P$OHYTp<b@Fx00Hiz)B&iuBul$H^E{Xeb((r%CVig5K!(%t|Mb
zAqS}W=xCO4IZ^6p6_1y6iPW&A8&F`nNWhJ^`qVvRT{N2qu<zJK{kEaa!&A@JOXtba
za$FGiA>EcNeHx9@_xaM#mE)fltQjhB_R}T6wDJJyrPoI8zRtYvx&v7;fZaqBT@5h8
zhZKXyvC2EV;?qNDe3lG*(es<-=;|^XV_K(Nx1SO<J)viF`ds;jlgvBzZ0<0NvL$6P
zlo?NRpk=j2=+UTlDDiDn&D%P|IB!Of$1eZ&*%KFZ&!qirPcQCv2wgt&%Lg?(rDK+}
zdy<({s^O<--U(UbW!sCVcr`0~vRCz_8di_G5-%ekwlyahB6DHKtF)PMKb6;ec0@EP
z59#qO$a^^_Hwna$>6vAS=CZBhZk!bKxh5D{3j86KUG37(ZWlks@*jX_EN|Hz^4ttH
zyx3}gC!I2g27M5>c*L=Aoxwa5LI~)egvXE`Cs>r=#$Vkp_4phbkjWd)P(Cj60WN_z
zMS^{S3Zc%GXchTrv0>z$T8WX4x-0_rXor8Y>_KxcM1i3-Q6Qh9!r|@9)Px6XRwJE8
zTEp*Ds4i29Dd|<c8AQohm&d7pU8ME^5u+o7YED1f){5BwY~!Orr`6gL_2=Ts9C{Lp
z=<2w#M7mw$sOa{Cg64uOG~g$A3zC$TAKXio9XKfP`aU)gC{vB%4S9Wz<{m`J^;q)x
zM92Lzy&4WT@Hd)!)J;7pGLz0!t^B_DX4YP7(s2_#sp@j}RC2_5H^aMl`G?11M!A=v
z=FJ*UZYHeuQd;ezWWOU^)yHjQxj)!V4>a8KG6uzcXL+JtG}E9L+uN+tIG9<EaKd@L
zjQgyTd%OT~Ez%e^MRA`M8l1xGO-9EZnYY|Al<Gv!NqhAbRL2?00o7xKe3%7$TC7Vh
zhr7~6Z#I7F!jZ@xaQ8Vjjku7#@Zs|)+w=}p*p9^i4-2yG_#GSi)JW&f`IjZV|7l+b
zHvOG#+Sfr^0Zd$HfIV-`M=dmm<8IW71i9#+UQr(>W{!5oio<ZxL_0w*GQzlzdV~gw
z4_nepxox!~_Za_Pv}v^m+n5Ym$Qdy*_Oki7q1*RVy&QZayTj{si2jZ{qCA(@&u+_p
zO20p3880=Q?#GO-y}x~A0!cAyuy52T{`#`rY}ozm&N}X2RS(7dKj{x?m3A|K+{mB%
z$RowQ6y__;{>Ysw7<e_0L_BGZ06t}Ik~mmqUyDfN``KxZY{RUu?ifPGju6H0ded}P
zwWO7C8$|Ad>#UQ?|Ca6|N7h&E9kn&I-&k=inv%pR;^sfMPe-27K>PG`TuR8*NYepW
za`Cs%G=DZ>5%6y+x>`y#Hr)j8!O@BEQHyM~@Ok_4Unjz*!8eIO1<zp!T{GCVwr7n4
z1k_9mf@-EEK*3Im+dZ}WhaRy@!d{B&d8bFrKV<R9f}L#Y@N$(tgNppo<n2fokOo~F
zoYl@YM3%jpPvhOp$%z_^ar^#u`G+4Y%M3_+@gA-B-H!O(d_2ieC!fi^=B*jdcR+Gn
z$t)gNuHP@M^O@rmvAI#5Yw39)s7heHUyW@VaQ2|h9z#nod(K#1vbfREp!vW*B{fQJ
z2gd1VKk*p#9TO*965Qts^5(B$G;ZV83wCtRE7ZGb*2|=<(Cj3Z-k%Y8>4UEn>rWPN
z57pwss7L-a<zoFB%>LJs8j<u4Wrt-?<#tI7u0%7P<IixM%Dw6Utkj7!wwO|s(XNje
zV$TKXlDp_TJgiJDj##J8F9K53dgKRm)l4NUvWRuCrV6uO1PiL?<4TG9unnH*HbNE^
zt+~j3Dn~9Ym<sPm>E+0(Ev!F-nI;4Ru6c}6Dp||w5%-8~(T@DWtDEJhR?g2y9F?Rr
z6kf)|Q^n|1apEq(Ch~&7ddAsH;{DWDNm+veyDFp*q97~LBFtBc|AuIpqh-=<h~Xd}
z+MKhD+vUF<msDm|9V{&fHM-)elg?2eflFjKNSy98J^cB|Pleo_%8|KSCV4Uz(Z5?j
zYcbns&bVZEV`kXQ#*cNEDR7l{ww{LD$0b{CDan*sNovMGXi?_UTPL*8?0(`QYuvm%
z2%^x2n?c(VQpTRL0W%1P=uS^dGQ8_+$y!=_ttfr1rTP(1$Jm04pp7?1I{oj!lRB?$
z_YO)BcB#&d7<cazoU7VI9r~{#?a(XC8lF$&Ub4CQf{=_F>CeB*b(16TK|_gnr@s<0
zo30mk32ffo{{2(p{{<*R^U~#X%@2mFO3}QM?^mlLl=5-rL%q4fXj6*lLPp*~aD=iQ
z71{kW-0#xs1O^ys{a6ST*Jd1S#l^9<+6Fy4zB*qeiW!L!x8ll=v3VTu@xaF;mE`XG
zVS#Hz*--;zNuZB2I^}sJ62n=pn!Bs<NQ=l)I_{{&3#OfF>WRZqoSCu8+w<_^A&ZDG
zMVa89&6Hg$svHBE!R5yoc^nuuVB|cgsl{awDu!EJbKn=)XV?a%xfVRe?6O7$5ne|2
zw*95MKvWPDk=y|DMv)%w1D@yh;I1WUSpKIZAEZ-Ni5m@K*ugFYHvmL;Zh<xtuv^{b
z$$bx2--^|vu(gea=Uf)FRAbeS2ywJV`cI?=wzx$cmfrxm74B419Gw}oBpm4h@9N<`
zq;cL>E<B!B&5S~a<X@Q3U%QE_JkCZcl@QvaeMeZ;=~9$(HhVX1Q*h}WWdrw6IATR#
zajjp!n`#KukmepK5a{*whCN~T?&3N`FIDkp_;IWDCM=K?v)~&@yZnJSQ2Bhvgg5gp
zgF*j6!7uG?^o+h^&j-)ZXz^x+&iuAmQhjo!S6P_nGV(^!B*V%{lbIcl>CO*RBim|e
z;+151R^CIpWpNEHzCT%<QROE~Mi=<**HolD6;$K0Bp5+S8|o3SyVRi~8|_XFNW%2j
zUkneyjVDa4Tw!b_Z%QlukDUZE)M7?OmDn3#ux>y5$MG2W?W2Px-LQA7QHAG%SJ5EM
zeBZjms|QqV{Q?KnRCH1<NT{!cL_op~Xst-prMI4Qd3U>9|4KtuoXx%U8T!(B#nlV<
zwXi;IVu+u^N@`@>?|P{sVIWnXHS^#8CD;{wWLnC-jI@OPehKvV;99U8K40m9x^N5K
z305w;e9HcW9(ws*PhvXNw+k#0_KTaW->rdfg?G$m%Brc{YXdKq2_J^$E(a&kwr(tQ
zINP0mo=X_Q`B2{Tq;q-Fxw2&6>I51V%s9A3YQPzHB<12JcXHaEN{3n|;<qSfs~#Kj
zPr_^%vS|4oUXMNe()~J&$0%=qGDCtX>qSCccdTR>W6w>~s&jW}fXI6|9~SfY<+EE5
zLKehQKc{J>jHX7H0rX3trjOF8ED=4^*r19g(R|QPNYQX4OC{)HDQM;<j%x|90=7&I
zZ)4Q}c_TzGx3rFk{Wza-xX?%|ZZKxOI_=z!uAcPL+IPu+ED!mB8jC(gAG(|NsO_R~
zyhLw8v`qigFr^4;$8{YI;_o<nDO+5Nt^)sn8l!by(?GsaT<mu*)F_?eE26&j<V}#T
zB%e+W@dP7>!tpfepi!HR7Duk@H)P6=aigm9NtTf;Vi;-0lp1iPg7VH*nb9U4ojUd5
zgeBhEfi>R7<x%yEX2ur(#$EZNROr9Dq2i!!<@&RCy$mZE!F582MrH8M^E|i3G(!Es
zfj0S4vh>|9C)jY^=jW%v0%AWof=2Na*p%)L3Q_^hVqSt=kB34)IvP{}e8=H%#Pcg$
zdRpBXWNN`7b{gGwj~FLBOF~dO@UZ_G_<ldHK~fay69k$(WV0Rmho~Ad#mN4Qd*q^L
z;B-`Bf|c*|AN^cZE}RE;-%B_%6V+8K2bA{W(PBzT=B&UpxIpGI>;yHw+63GfNXtx~
zBqPQbk-P`IV^74tlP^1c#f*8zlg{Z;7_U*bhKv51eGdtlhEJEfa~{2X+}gX&Pw|j7
zFRFyUt-mlZ!Lhaa*fQ@@p3d=++6|MGa`zxBerzbWPqvV8eyot!S^qH{6SyBvrevau
zq=1AK^kyWFo)1LTwCQZ)j6_ifYJkL3Xca&ozzq{f*~2r&R*DD_NoZ>b>rc_kgk2az
z!(2L0Q}+U81V5l~UUkp^QBe?ugM?oT{^i#b{l;@7P_b_!uBM}KX38(gPK<G>ODT@5
zO~`$p>|_>itL~ncR?H0$Ps`)PRoo6#@5YnGzS5eDaLg$N9CI2xQWUxBS7ZA)vO&5=
zldduY^zHvBTzro1Cpi05JN0XiEIelu6PQCwSe2(_7H&-7R7zQ{osEY}!}`MbqCsp$
z=Dppda`~`jb<eu{R}W4Om9ghkFC~Zh%2ObPr@QN-7LmJ=R@@M&k1+^4Ax9+WLbMnq
zfYmS{U@iOt7btNUYQ*UZ1jh9;KqP=@bT}xPtpuOaW4bunZlbO}-H2+LdF8x6KGSpM
zNP|`gD>c@;oqaW0(dUY-2h)Ws<*@O^3#8P9VR|5SO?8SuHzL8kiCV!G%>@YIKr<SN
zpabYxzv<my+m(i$9->>09xiz5Zb&9;g%*ann(CiVr;#$gR`}yuLu;wid$X#r-p=)m
z4VJ^*F@d@$XS*xMuI_TZN|a;DBwp{@s>BUcch)j`ai1WtZ+En0q_%`_|6o!hFB8?l
zoSS4=e9y};D>n%pE!6f=$-HGn)iJ#8b-M+ju{sB;#&GC{a&avh7jJ8fof%&roFHI;
z5wldt2M0B+gQ?Enio)~<bwefBmADH9>48O*1&@DcRTMci<Gm7V9Nk5chKNel(K+u-
z^_W~-gthZ$dY$Zd8I%mJP54w|Xc41RssFnEY{k_zqVPZ+<dk70>>(H1Sr^fFE}~@6
zK>PD!4FUe|zWTHKX01y?eNX6CYHUYurYHfCGM>;WIGH$He_hWGR6C0X>HDjr;M-{q
zQ?(1YrHAKO1q-gx^Q)amhekTpxE*x1>b&)XUX0hxjL_E+p-qcyzTYyQ$C!QZ#jlH`
zWfF@?zCHIQV=LlvWf;(WNt7ByLcglOc!<DyTUBVPX}fli!vco1wB#V~wX>b3_K%B(
z%(U=Rn#C2PF+9&j(phBIsIb6AjebqhQ^6!4Y(oub78pUQhT!>{C$%<KP$Q3RsQ`G$
zdME2KCoyL$brU#M)iB>9;b;*K<1|i4&j#(Hg5IT?deDwe_-jBthXVU~J-7e|%P5O8
z|MgvX4;^)()>$scdYByWE-06-prX0INNWgwoFYa?6N*8i;Oyh7bmE51ct(fO@gHDs
z#`V7f0obqAKNi|S3P;?;8z86U?FKnW3p!_Pi+D@zqV~k%b90YOa@?cS?ZjwYuL8B#
zCH36G;Ms91??R9G-|cIe;y8KBZ4ZRIX5yu=2fWFL7=b&;ku51ay_6GRS3*4&`xaNJ
zZ5bMI3s^~3#TaV7YOfUxi>j@uezc0>+;Yry84}s4^n7IeabP-TwhAWeZF3nIw28I-
zavA6K24rjN^HjqJCLEC5@Tt&*IyLPTIHO7|rgN<x?t$6pux;@^?+3bEW1Uh^xN2lp
z%WsfzW$U$;zx~IvAeXQ7h;bb6|1U`U$Zw%CqnU?e4~%1_iQ*7&W@*|`q;8x*U23F@
zmib%egEtj-IKAc1JS}oyR_;~B_$TqSZ(bQ)=#I)};iQTyJuCQ1;ZLim)i;cibw|fA
zPK<7Ar(JwqC_Nm!C{s4^<jklK<-#L&ERTz5z38B!j%m*0SxAUf!}<tOhQr5@S)m(n
z#}wF$e-ISxMOGx#2sp<aQ@|-8kj;qI6Zr2)Q+p_-9YIB()07tw>(M!=63LRVDD9>8
zchkmiOSlY*#N!HK&B?#L?R2?`N-g?2Ash6-Twf%fC)BoIlrz^b)$7{PDRCp_|4;w;
ze`s<unQqjw^U#qrMZFx&H-nL>q-E}wt_R-6w08W;V^JHjU!JZLg)R+jkQUIh!3$br
z`9ZBDXBI!983`%+_q#i<hMdqc)@oh!DQSQ?V_U@f#1|s|`wfN@9vdA0l35-3Nc=O|
zopFs*&St#R1<a19JsxKhaAMF+^Uz0zU97NBgCOINTgmdfLxwEbA#!AmD^HpWybSGO
zE2_z-R5l2toX7#HQ9sNlZ7$}yj6l?|AN58%0mInno{l$C+B=`})rj3SMna~P&uh8b
z1K=S7q~q;e-=90~zz1SN-0xR@>j7g6Z1J2>>A@&;PQktZAm1CDTq0C3`7^}cCb_S_
zuAj#avtJM#)srDdKbrQ9v$47I9uZo2SBvw3&RoUxzXYuddf7e-JT99pqT>rbX0NS9
z#kF!?O!5_xzTAIO@PLY)%DHX!UM~~7T*YB8{hFg$9=6H={`@Wv54qvtf6)ExqnXoa
zZi)BmUN5p)GCr*4_PcT${i0bnXmduYlXjGu-5)~c{w98c*#!Kl)2a*km0B5_#yQyP
zruM0|9^lET7&HmfteXY`L8dqi9x~{r$L;cQAbm{H2FhlIP`Fb8W`HxuTE^kLrPQ$E
z4@Xll?Fji)yuB5C+>Pa<z38a%?575W+iDbkN^CH1fBK6y)gonC2RQxXsXC)@4!;A{
zP$J`~SVv8>0&x1AnZc>hig_$*3DuV~N`ibNO=J-w(&I_==7@d|c_)od42v^WS@-bR
zN#&?^nYLYVcP>4Rb|~!mm9b1qS=ad*X4t*S`|W2~#>Yy^*CzNqa^U;6%v`$Q*T#)*
zyv8T)6?><9UsABuR-%>fvr9P}5F^{V8q!)I6jG023(lWvn3rjFd>RncAvW7g$N(Ra
zYzt|yNBLmJjeds&1zrob&w_xKsW<ar#DJyLOs{|OA{H_B!S+g6-ANQiAARF#7o?Ly
zyk8Mn|1IsPQQ8G~8;=hz0*k>L%hI{JwYQos2K5VcRI)1Q<Af^1z=#M&aK2c7nJS@2
z|8;u5IO5(*d<AG&m_A+RQ!iwq;b7fr^>or$sWl>_S9qmPI*U{0h21etmt*r@&fO|q
zk(MyAP;B5|p|-vo*E{v_uZ{s)pF<usW0HLBTnhg#3BKjlxK$kGjaVo9^aq`)7o64<
zj%!XBxGT<RV$wH&pgtGtm$GD^qZ`N;YL-7P8U}ZGhFbjGv&ent!D)8WS5k}E+J&Bw
zXAPiMB{ppeef@_cqIS!|R&CJ#0m7GPE+8i?&t9@!@I4iJa9Z@B{8@OFIMO3{M{mqx
zO3u5GzMBv~Uj&-TlZqfgY7afsvBokYy@&1+_tiw-#bC`}i{Um|Uh)f&k*%|BUrsid
zf^L~jIbMM)%z9&c#G#A@mc214))_eAUoz3c)WyEHp)v{oETw=81hilu;uu4HOQrqR
zP*Y)Mcp^pyTc~sR*YW7#6l=F%7e74z>tC&3yBj-W{D0kOcx;<=uQe%(mY9(`C2l4;
zHL%wqiD&FQ60vIR&TKc$i!<Giu$4^i$Z}V5cXA+a@p#cZ!_6y<O<aM@wWJL!C6&AF
zrA|iIgcnVPun@F`ZRmSXH11A3!$<^CXW~={+$f^RRud!o<4&Sk^p4#lE)tV*KYXI(
zx`cA%jL;;qfI6#bJ@j1b5l3CY56HUV!?E_BOYXrrZ>bmwGoRnl%o^n{JwZ!Yed+;h
zEov;4c-$p;u!2Nx&o3Zz!H)&T$~6_&DJroGc@g>BWrl1jPcn;qS`k>VfLacm3^l1d
zt+N~86Z^$?tiAP}kskxE`pw$?v;R!YBLZk1oFg>mhgOvqaQkMO1y1_ozRzqkk4&5I
zmtC2?#y`2UDd~JSwt$SN&4RiU>py#&;^fwwBkvWYhRkh^x~AG^?RV0-lu^sXkh<(7
zm~~{Kdk_x&F*+@1Zo{ODX=ZcMf`&o58r4#@VtBcT1JL7J*SwkylmMG%$}azt`i(sY
zg*ZqqO1D02N84#)FW7moCp0m|pV(r_Dflzfk{U>ulC|ApN3*_M1$#0*2EF0GPF{vb
zzA;08(^qQc=Ee(kKE`HUi6ice84ih}_-T)63FyB~1io$NCH{%7<&)3ZuhB~;>gHaF
z$SFS7zUrZB@!ZL$XD@uQuxu+f;7}{Ki2Du+Id?lHa<??GYClWEbo-l`^N>{l)eovm
zJk73Oq75)+n3x$fU^D9;CY`YC>obA7c^4=-Y<@ziW>kuREa`MT7<aBqe6C1*2Ye60
zK@sRe6}S#GXz9HZvO190wMIS}|MnfkxBo-NV52Da6(=2cUzcyxQimFwwal(Pl!lUv
zpRL?{@FW{;re_=SRmrrHGtXwV)ai)|E&Z7;g0%9VhU%wioy53`v(PD&;&znYt>Sp5
ziO_+NxeX#S%FU30xsAfv!22IrkH&n&cgz$ydVaj;d(v_C{f!d@kh;FIDZ%b)o*hH8
zogBp=mX&;o(NY<IeoFA^{)L^s+aYL6)5AhvoNm}}TuI6@h)Lwr)sRgMvpH+5xwxnh
ztZ_59z~hnSU8-GDW^UFbFs>CwwKX)|>Bn&6mH|t~cS|nKxn^_Tp4}Tsi0p0^VdLEX
zss!C|AH<!Hhk+En`ETc$h}Irf=&9xW2YX?w+)hef-SD-1we_!!OCE<FUy_T?c^(*+
zY;3%^PD@*IB>6M{XTeaia?rD*hzedyp=-Cal;+F9wKpM%JvEQ%SUTrN|M9X2Rt0|B
z%p{I${Dt<=qf)<c!kGJW#!jYf`*lUcdFQY<?-gE`)5zH~yvN5_H7J))!TA2NWQ>)m
zfGn6qwkz$$AcSiSC!Cc-sid|4d|eVp3nhw+U7gAg8QJD@l)Iu6YLNIDFWQp*k{O2M
z=G;#|0ZXWUaMl<*+N?rOg=A<muH>_U!`BJi@+xqycpinTwDxIdlNwZ+nd|m<j8ni&
z8??_^D(6uq<f=070G4BD&4`D|=7bf$#MA@Ryi7PWo98=9d0o{~otGAS<sE8;6<kW8
z#MDjmSDL=s7JQG6%G=h=fIdTt#gwJZu`5|#xw_E9foI#g<k(bH-=WQ|Ld1IBOAvEg
zWC)euRP}ewy0I-&0~t4xcrulCaxvYY$Zdq!{jqJ$U>o-<4c4qH+i3Isq$!DGW?b2l
zSoi4Hmsfh$&yFPh3yRrg=YQ~Tsk~5K#D^#9gevA@-b$EKFcnDm)k&sp(Y_S5f&1c>
zHm($uyvlF!;xHw{8T?EgAqS<2S`hKS!vi|K>~gC^3AzfNnnt}G##0B-dq=ly5zvkj
zGYBQP`wX*sc3UgJqF_bx@8>p0-j&9pOsA}AjC#<B^hSZd*XwD#3_=LgFsl9PV>Kz2
zazHI9G_tg~Fc<iCz(3Dgs#%ZS1cr8U-sozHY1c%Sn5_W!w=<aioQl+eo?M=+It0Y3
zBFjQP`qFlbpziC&jAYdOLSt^p+WA-fy_xeq?CY=94W(bPV6au)n*D9H0*fhMK2$Dt
znjcjjaFDD@TuSTd&V|?LT1Zxs5sO31FMt+`ntjK7%$T`Nn>Vc6vrrrgI~&xXm5#=U
z5F#B^0WVM*TmfMD33buja|~soN(dn&*Va41f&byV-=k;Dt6ip}$SBE#tQw!4LP!Fu
zcKsJfn*L_IfFy-S@#drF&&O}OKUtnC7nkE#(^vX_m1`2S$Ldaj+{b4%<QfB`oJuQu
zlbI3Z5k3@Zm=#pz1k#^x{AtXI8oMu!CLTHJ`Q^eZHLKEX_H6ce099TWmAYdI-m+1j
z`mOPqj!@7w=j$DfuR|bm*6)&F__%i!w^6MUx!O$OKA^H!4@7+z$bcdZr(uL-fzOKM
zIg~Di^nv_Uqke#hq4lCwD!Ec5Dl@&?&j}&DB;KSp)UrIka?nshEtU_4pyNGM31JFy
z+O8y%bZD54YDtN!eX!m=j`LPZ#z@6-gH#K$3Q+bTQ3ohcYR#U~zHk`rr24{nbAQ}4
zR_XA?VcRPgGr4N7eWSaMew>d&@l-*n?-XubWTC~Ry27MWTNu}rbqhps{!GNby64^@
zjqTM*o>gI6%-tKuh}nHusD1}2hEP2!Brl*kDSC!3z@(0xhd}-r=t5MLusqDtsRFg8
z4=n3bWFKC&^FLrqu~&{)K;Za)D+Is`aB1&c{+hcmamT@=?U%)QaFoF@f@P%K_6v$x
zYCEZ|W}bP74|Q&mP5QE*2R(+rViea+Mg#sCLtBt@<?rI|bLT=WH$-f+NZq}{XVmNN
z?B|YZ=hE}Ug(al=%|dIBso=XS_=n@CWcQYVL6*h}`;O<izI3nfp?x)zL~z~DtK1ro
z4dxp_ZrPEgX*8}@ojzp~qdZh+g|-7XH8T$P0R6Zll>okWh7yExRbb%N-&o<Q*B3%4
zAA!Ffka;s<lO&STi^T8mo!Ud~{Z#MwQT3!rA{yHJ+y5gt>cG|1W35j<oF$znwjQN4
zV($Of-jxSJx&3|OqLN8k6l1h0x5ZYnOeU#RM9b~AC{rYaT;^KFHlZ3yl&rTg;}Yr?
zLu4t8do82LWXXCNj4>I@SjI5h^Sozt@9(|;z3<=edmVqxcpm5bob#OX{eC{5@AtsB
z_ST4_C=iss@C+-ITE6Q9c+IZW(l~K}&p`3S`yg0={FD^yJtDf&VFS#*Q$tbx@>Vyq
zm|nimd75R?r>TcJq0_tRRe0&f>;(MWXM6K=n7Jftd@i$MPWL6FqNVQ$iF*OTf?lwJ
z?w2XeM1&+j6@;#;sp!&>J?RVSq6_E_FJX_Ur?1YYDrF4*^P8ZJ7h@|z%G8(~6PE8%
zNb?CER$g49Rb;qQprNTq3*b=S_H-=3HT3}$zMogN8#Y*KJO~#7_w8XdBBxVDEkdb*
z6lkU_;ZE^}&~P4fK%EM0Rvxh){cXT$!ZmD|fqP7@2PAP;6}Rk@X4qFC`yqQ+%YDwG
zt6zMi?jEK7AX_QsNXBF0A?K6gAMt{wS4Li6cVSrChZi?5O13jQ9K&@E&A*-{Wna0`
z8U!>$=SDjv>;ExV33;##hMY+Fij~0el3hzEq|7^&8Pi1rfZI2**k7|&%H?7I#vnnI
zmqM6GDQ1L&5Dhg%kj7`M8Ad;~o0h(Fb(&`r4|Hry<z*8HcO+ls6%`@*&WlRRGz%K0
z5>6X>aFZTOL&q12J`i_P?&Zv)w|#sL+tZ9S^VAaFlyZH&-sfTIh1@)5bp927x$}JO
zT!x4i3XcJ)02~~X!B;Qod{^CM^l?E%0oowy9Q{PE=Rq6=2`>>hlsw)-T9?pLa&`uu
zUPn#`iVQV7?R0l`tJr;pQ>gE7c?N!qJoW4&tX-N2_pr`_akNv_hbjfn>Y4NbLMlNU
zNQGpQ=DD}1J@B7%yC<?F;^|~I^F1gz4L$0>kKUA8P9#XJyCx{}Mx~<^emnh_&B2R}
zn1!RXZHURYQ&934>7%8J{T*JRX{D}ipl1KA&xN->PyfD^qNJ8k7uIOrk&JJk-d;?y
zTW-`~dx@JjNz*JokB!0eZ7lY1yZEmVIzThv2qKDc5$!N0bdF$CFM3V|+W`R4`=n9s
z32){W_JfYgP~#CoXP7a7XZFAYCW)Z>@Y-*|gzrqy0Q3%)sa9nz@{o`NRViEy<1Zj&
z)bqC%+!YPwN}g6N_Rr??C<*<9erfdl>h>SSu6y`41?i%<i}bY6=PhJ*&ScsfkJ_z(
zlXI6gKD@1~+kN2TrVu%Kzq;cUs_81!`{SAtolU_WUN!u4c_<hAijs}RzO#mU%->K|
z&7sHOD+&IsMO^phD#K%m#`CYb5{qmF2u$5C%xqM+o&&RJiRfGi2eGeg{SSZ!z{xk}
zn}hB=HkqS_3cnGbGE)JDln#G_mt>Aii7yWp1QjzX36oGeII51us|Hv(4-*2ni>U+f
zq|uXdaDXwUUX;(neiHA&qo&Ub5f=5FOVZ}(sY$HP$<Z{{FJ=Ah2%T0f^}(SdsOTDr
zdh9V@)F!h-o8~)$15X>?3Vu?H{vgoKR!d;Cy;G)FaLau3X>ag7d|DYoxz!5q!))XA
zQpbX5A0^*Q3pnrj4+*01*-+tmuxT@-j*Gkq2d1D*M4P~fcii3pd@zkjkaI)n2AUz>
znR=Ku2VH>9Ewu}fWigG_e;q^vj&Zxe$|2NZ!l`ibUoHF;lt(-NY?2^P@b?tN(w-Dd
zV$mcMf}E>Efofl4M!=2s`>&bp)Pe9H3Cz29GHIUvPaV_;<*t#&BU=acyDXw^qUH|D
zHAZUY`DQRQM<nj^p^X_-9sU+CZjj^jqD?x8-^As24!$99Ob}T|ta9d-IIKWj)7dah
z^g)dHtCa(m!<$B3yJrwtmynpSc!oF{47o2@0*2HVT6jxngsTM<OAA{pa7SDvyP*^y
zldg(Ku_?B%4O5~{3xi}sQ%q4{xqr*CaHc!D-skxWTDRuPGBJJy3RGDd1|3v@l(J&h
zHLO|91s|~XEF^rPJub$qA#~j9w|*tvtRN-{QQ>4P2guR=0f0AqAwq7MRV89|a*T!|
zT5i+9?Xj5c_Z^T~lh)}cD6UCy<I@>*w@B<m6L3QGA%?<n8pW2s)$XhSaC$O?uyC&j
zm&iQ$?sfwhNJk5~5$iFgp9)+H71N|weuic-KzRV7>x3r&nE=Bj3qhnJ;+wx9J)W9K
ziPzmvThPv0K-jAN>T_BE{knjt1}7id@vx0KA#jH{ZvZn7b~(oz$3yL>#bvTRuii2R
ze~yO&haw@+7|Ldjw78nkMa)zP|6EDHUmzTVlPc^cc<<Gxhbo;;FDuw`&Hs1#$DLLP
zy<-mjiRL9kc?`ji2ieg<Iwc<*uZ92tmIEG_q3v;_%{HTE=<?3ZqWo7(E0M6=*N#2V
zM+<ArBtz5m^|@8Iyqp%A5Y-b+09*oa1l5n;R(@7ekwJ(#2U0l{oc=pWaX=~;q{iFS
z@g9!)Bl5)GX)$=-8@{uQklbKuG#5K&P@*A=cwsKoBh|nfN{<Q`{5Fh1MH}hIdKF8J
zSto?_c%t7twwYFlkkYH9O?iw+rp0!}gb`seLdU8%BgY@C0hd~LY#ywYZkvfkssyVP
zAlK6ANvXdP<#a~J7u`<lvEr<)oJQLac*Cy5XwiBpH%%m@p38iR>ttuJo#uJJsBxmn
zeBKNDX7}*n0_0t|54>)D>YsC$Oa!SbiZ`3n1H$Wp`k@}>eK%|*Zi1-%z9Jc{BY}#X
z^k?(<sgH1A=tr=?fWgMHnm0vH$V59!j6Nm^lS$c^SBH_wd!uVAan0{ajhkk9v%cPS
zXfKR49V;G^K{kNW#Y*nn*!?@EaywiDYB|EXEB(V4_ujDSxH`TYV+_57vkEbkhj$P{
z+}mwZRr!eRpCX5+50&hXAU{;zq+=EDWKq4U>Lfa89KHBC(!Swm!`bi@=+qats~~4b
z3zn?iF7jdaVZ7XRfG~W@cf5L{wd!0hyudLSNCHY5PXv%IHcsuDH@H@7%ajF*GKI28
zS#nfJ_q)E2+Toi*Knkvdg9y*2fDOSD!Nvx+&r-b>Kpt{G{2q+q*Ke;M`9Pw9Y(UvT
zm-;aaxK?Q>9n5DUaL{TYY2L!*08L`aqcm!i5XP+teG3CL!}z%U@UeLQaY^I_EB_70
zD>|tCFshXy`TB6R;n?DqfR~q6%MmE@%AU>-%#z~HwAAj`ScW%EVUD)Z%OxdWj$-)z
zqD#-bLT<nsD~tTAeFSXi@$$eVm!k9722g!Zgm7BRo2F;AFtCCyVG|f7Fpe^){n(ma
zpBTVzCVWE8JQ+>uRdF@pkOS};zI_!*GKTb@ZA9g?kP~kM^1jN1IGJE@UhNB*#hgkU
z<6UVMTzHV>wZNNVV*Oy#vIEp-+XG{fhkJixl}v_igErgXn{DFf0O{)|EaR@^xqrQ&
z7m{UgxP}NH<(|cTCy(d}QNdg3{D64H@D^C!4<5&nEZzve(+VJMg7-f|6A~jjS5yQF
zUd?4QMcR3wq$Bl%_4<A6COg`^Ie0G_5P1&M)pCY|Fn9{5L+733H;m)Zf05#p4+N_4
z$uPpXbkT(MuyonpPEPQD;ypq6Orto@E23o4E=eD1S^O>k&ck{0Z@HRF8Uur(ObH8G
zAao7yT(mZ+{wVv84hO-l!|Pxd%;&efBF2Q{fYTiLEyk@NQ1X-??aqIs;-ShczxxA1
zF-#{ZY&5~<1xxWK!?-hvDnG@1C2F@{dM$0dLjril&zc?Wuq8BTJfj>0nY%^AD3TvD
zmR0xga25A3Xe(+g#`p=%>f7EfA_YK!s6-g=vq8w*O7{76WFG|fLmC@JitiHoF41aW
zQ~=~hh)ozx?5Dja1i=yE3@(6sv`{@DGlBC*TB{g)`$81~Wc%hU7*r9)q)k&O5i}l*
z6<_61g4fjTlD37wIao|?|0a&dvz52Mq*}J;10RNI%ivI;egg+dy}EbyYL!S0#h<rW
z>>z#FXryM46M$?by+XS+KKwHor6bH!<+PkNPwKpY=N3|gvlcf|+mj$xMU*WAwKihM
z<W8|=OA<%bUYC(3<;OmA6`VN?$VHT?q0d(r%_h`jOxwFaQ_d|%htvb&zDd<9br0VK
z_oTwg$;=a$StVrYq}i#yR`QY4uVoU5`O0fsdz(r&`?#m{s^`p~C!5~H%&`NStxoQg
zS}F}Hvf&`@P8#?sVA9(q@1Qw$V8Q?{>s@J@Rb{W$7I^a|^2}u{+tXEGf_0`(WZ!M0
zz_m(+8ashHxe%JyK6812bZP?a*pgZX;5O{!cUrB^xvM-ai#y_X=2K@&jKt{%RG9^1
z4A@nndwCMf^t%X$fG<N^^a9B{I8A-vb)yQR9(fqF-!I9ybjw!uTzb=_&8b~R;+?<p
zIdGBu#I_ZYtAlqev*;ZQRyRjz2GBWa@qI--_Y+EI^?yB|!upsNCXQorYF3^K?{M$)
zV>?!Gqv(EOJrrxTYkXfWNz?{!Ld>W$W5?^-WFH%6f;XZBF&X)0PI(2<#~%GPZ&LUR
z=a$P@dH7Sr#?u^L%|-Oi9RsqP&1956*pH6yJ0R2~{(+Ia1y}j^ItdoQ4QIkW;r18(
z{AcwCqUsCaSGHS~D1U50?bI$cn#rV;a=gf|Vslu6Q&u+)WzZB{XEI5_=O{k(`k>+O
zJEzTVGe=ytSHRO<va3K%X{Hytjy|`a@#XJ6g4n}R{8?wGB_Z>t?uNSPh+Z-kU!a#F
z>&s*L07xyaKH7iyRCp<J12gv3#ab5Ve1hLjNi9}(RbG`FfjYAq#pps`(6FOVJ_~ES
z17-wf(k3)>LK6AMbuA5q$TPHhwBnbG7N9RSO8i!iZ)z_SOkRXdje`AhKY%YyUN5_A
zyF)%fIty6A0Q-oO$DM_3glo&JhS7H1(xK%r5>Y6(!RJ9~;L!{tmC9{+wiD;!-0qis
z#~c<>-BAzIq8^o>byr;hBbA{C$!7VBp6<oZZ)<JrVP1C{+>(23YeC1x=r^=~zx`i4
z4UKz;RKkpwS>%m%T>OJu?qR3%=UB7q+QxJgJ~>ZY=y}c&jxSh@>8kVObW>IZ8|l%)
zG(x#?PT%9kLzYkQ?-2yVM~8Z!mS$b;F+fOqn31Vwkxi02T0S#tZNb$<W&bPChces*
zdfEhBdaf*FPjalba4cI5@f?lc!EAXFQ7CUvsBYkwMTi(#$4I6G0=hDTHmU19BAl)a
zE|^Ho`-Zu5>T$-dh=7(yIqnZkwqe3Mh9c4Kvkui6tj8bHB(piRMU0P}xDgWIhSP_&
z@y=hf2ZRdAwUn4IpzDoraBj_gi){{b6Zk2R4>Q}IDh&WGGU)*ejsFpSfoy`Wc?Af}
z?GA&+n89OC==w9DP{dHAb}C_ZQri+CiThx{?8tYvbK7DYA$n1(pzk;Uta^^}u370_
zXGi)<LE-Ch>^vj{5(s@QRkZL{Xan*tF?&qHChS#er-1oSiM$ib;FheqZQ;{rf$uX>
zzZ}HJL<ATrM{qv-{8WgMX3lQf&^u^iqy{6m`gpo1I6nn*_|cN4b6nU6PaTi3o8e5a
z>)=z9+UdYc!Zc`1+~`%BJW<a?u9}&(Z+!;$7Xfy{9J;lkWR6A+w+9<;HijLxmH>oa
zXuWlWpnWB&!@TQAHp+56hzjE6x3ur|NF`Lp8x3Y-<Yj=Rd&4^K!{0+w$Ag{q9NHOQ
zT?fh15+rUf>!WL>Y%ww|B%a8iAy+Y)>O9FN49!1_DW$G0ZddSeL6FsgB+_r~vdz;(
z-Ao0y3Nu?3j(LXOytxwMNKF>jIb*MDr+!}hTP&F$UFV?TYUH*}QcUyIqBZBtmB~K=
zo*#EtSN9+0vLYHy7-x=a1ES@aFx8{7#fYyTmqMs&I9B%&{w7!DC$q634P7&(64rj5
zEScEuJT&GRoZ&ilZsQ&}?9rnbS#bhc<L+w0%*RpZ@@E-c!7*oLyju{5((B@;a<>M1
z_fCBM^SaXIuTHFV%AGQkC{6SrD7fSkA8aoPYAxBHN{vaZ?h!{g@>Thrr`@r*MU+|W
zcV^bZdOw!dKNe`?NYxX(n)6p`ewe6xl$da_7v%R%6z8jQ8pq8fLxjQa_YQ6gbUFFK
zcK0N)3oq#T@jh-OKpUFI{IqVvPiriVaz?hM2x`aCV((qG=Cc-Rghf>OgOz5n8EI!{
z)$PPX0p#$~<RNW5l8BKYuGD`=s&lZ{7sljBr!5#(eII{{uZ*unLCOWg%N$&G<R*pH
zwM7JHxKn)G%xs<MLkL*mf}i}3B8_t=tz0}Yl`X{#V9!=6Ynu4CLHgTKMI|U_PLI3O
z#+D~tev7rR%$ySwbxHrv&3SNGR}ng+n7j4u?bxTEJy#^eyQUPkT<L4ERvOV{zcy*m
z7U|lF+6aU4rY@FEmo{7o&Y5ODS*`W)lKWcKvFP1nO^_P8=4S6NlRY~M;!gB1<L`E#
zZn4M)Ta^1X1=-RgWU{UV>vbB#E4L64>k_JhFg6`|jEsPWSsF2eO(>F4<1*0wpd`!c
z`kqGz8dmpB1KCob_RZ#ufMfOXAv4NrA7HwVKpZnfZ{20CFWRPbD0&uSsn*VEw#as|
zJqRnT5Td-dz&Mh!kISC<1LV!%4nhi&IEK5Kai%Dn49nk>AWWaY<}J~tMJXaJ#$%tw
z%mhB>eI0cF5O_b3imD?@q1sPdVy(2>y`5}d<6b*n>g059+NoXqLzzX}Qo6LSNMXF)
z{bCpA^ks^xkkVBi^T-Vf!g%#w{L9_9CtM2`*^f7nh8g2#TXpK0UF9AJWJ+|ZY33;z
zzt^BGc@Z#KTWZxA(VOTh%bX}#IZ<nxQ|;dS1F#O~vJbc~w_H50#(4%oYN5|GADj^c
zk*tkmirn0UqJr@52HIEL&8yW*d<BlA-tyf(hW!kO`4IDd9shtL7)mQm`KTt<?@RSp
zp%vX4kDwyxJCQ`?EQF3o2oIhmlD5+PcS_At{n;QDoK*J%+SA5GQ?`^F0Ic$$WXpaj
zNKKAeSI_Ks)U^C7yr#XRJtsv(oiX$$nT~ACq5<#~B;xz;j(E=inM&0zqIG^B;HkiD
z!Y2^Xj+*~&AqyHr#C|7jtG58Al~D@M(&Y1nN%#~ax6+}jLYbtqWSvj+o5k8O3l;mC
zmy2TI%)+6n%PQztIpV$*k-S4SheU(}Y^8HuGw^Hg?E%rtE5me08*N{za!d0~D5B?l
zUip3ax@9lDWF5DMNpAi&&-(4a29Wk3VK+1tD8GLAdX<D#n2gLhm>XpWV(h>=rT0%>
zqyo@u+BkoyAcogM92;wEzn>|$5vMj|6mi;NOYUE&0<H4>7%G?g8T_>@dsw!>PC2(p
z9;~-Ii#pj~8Zf(?`xHdBm?j#=04BTVhp5~=q#`L+S?!^lxK;zv+lkC`B%9wxg+aYv
z%=N5FGdS9IqWmViF{0mb9d3M&t)*1A*wtpa>FVy|9rewLM&q+*Mc1_xiSJwmt!jkF
z=F>`AD&E_;E;m1<1>p6ihH4*d`i;aML`<W0aP%rg(=`D`#W$MYRVGDf5=VZWN((Qk
zMcq)V{9S38R_wd9OU5^^I-kVE+032}YPRYfLKrI#TvLdAbmVID?v2HJ4RTNooAEC!
zZ=xormD3e`3nOceW$nG05_c&zwGF{~jP5`dCj>4=H%A`mK6GvCnGGg(4X^*%d5QIC
zjh44=;gc5+)3>{JvUB*gX4?<}AjUyLy12nr|C;qbjTXF3(qprB8D_n_+7B{UB2ZD~
zxa3`0aUHc--#yf;TG5!>COK*x5Jj4M0aLI(Bj+eAu65UMzCOOY$KZSN7mp7ata;ym
zO8jvND3o6L8+F-;YNExH{68J&GQFe}9;TN}Kzby>G=af?l<4&U90RRurbsf^sNWTa
z^Lgqfo8m#R*_2ktI|4HJXE+22#2KJM8+A-MUgOLj+0ydw@4rvr|LPNfZYe^ThZsYR
U>Qhf!<iO8~qYg(34quD<FVXJi#{d8T

literal 0
HcmV?d00001

diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index e4cb4327fbaa..c35311c88641 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -36,7 +36,7 @@
 #define CINDEX_VERSION_MAJOR 0
 #define CINDEX_VERSION_MINOR 64
 
-#define CINDEX_VERSION_ENCODE(major, minor) (((major)*10000) + ((minor)*1))
+#define CINDEX_VERSION_ENCODE(major, minor) (((major) * 10000) + ((minor) * 1))
 
 #define CINDEX_VERSION                                                         \
   CINDEX_VERSION_ENCODE(CINDEX_VERSION_MAJOR, CINDEX_VERSION_MINOR)
@@ -4495,6 +4495,129 @@ CINDEX_LINKAGE CXStringSet *clang_Cursor_getCXXManglings(CXCursor);
  */
 CINDEX_LINKAGE CXStringSet *clang_Cursor_getObjCManglings(CXCursor);
 
+/**
+ * @}
+ */
+
+/**
+ * \defgroup CINDEX_MODULE Inline Assembly introspection
+ *
+ * The functions in this group provide access to information about GCC-style
+ * inline assembly statements.
+ *
+ * @{
+ */
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, return the assembly template string.
+ * As per LLVM IR Assembly Template language, template placeholders for
+ * inputs and outputs are either of the form $N where N is a decimal number
+ * as an index into the input-output specification,
+ * or ${N:M} where N is a decimal number also as an index into the
+ * input-output specification and M is the template argument modifier.
+ * The index N in both cases points into the the total inputs and outputs,
+ * or more specifically, into the list of outputs followed by the inputs,
+ * starting from index 0 as the first available template argument.
+ *
+ * This function also returns a valid empty string if the cursor does not point
+ * at a GCC inline assembly block.
+ *
+ * Users are responsible for releasing the allocation of returned string via
+ * \c clang_disposeString.
+ */
+
+CINDEX_LINKAGE CXString clang_Cursor_getGCCAssemblyTemplate(CXCursor);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, check if the assembly block has goto
+ * labels.
+ * This function also returns 0 if the cursor does not point at a GCC inline
+ * assembly block.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_isGCCAssemblyHasGoto(CXCursor);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, count the number of outputs.
+ * This function also returns 0 if the cursor does not point at a GCC inline
+ * assembly block.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_getGCCAssemblyNumOutputs(CXCursor);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, count the number of inputs.
+ * This function also returns 0 if the cursor does not point at a GCC inline
+ * assembly block.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_getGCCAssemblyNumInputs(CXCursor);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, get the constraint and expression cursor
+ * to the Index-th input.
+ * This function returns 1 when the cursor points at a GCC inline assembly
+ * statement, `Index` is within bounds and both the `Constraint` and `Expr` are
+ * not NULL.
+ * Otherwise, this function returns 0 but leaves `Constraint` and `Expr`
+ * intact.
+ *
+ * Users are responsible for releasing the allocation of `Constraint` via
+ * \c clang_disposeString.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_getGCCAssemblyInput(CXCursor Cursor,
+                                                         unsigned Index,
+                                                         CXString *Constraint,
+                                                         CXCursor *Expr);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, get the constraint and expression cursor
+ * to the Index-th output.
+ * This function returns 1 when the cursor points at a GCC inline assembly
+ * statement, `Index` is within bounds and both the `Constraint` and `Expr` are
+ * not NULL.
+ * Otherwise, this function returns 0 but leaves `Constraint` and `Expr`
+ * intact.
+ *
+ * Users are responsible for releasing the allocation of `Constraint` via
+ * \c clang_disposeString.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_getGCCAssemblyOutput(CXCursor Cursor,
+                                                          unsigned Index,
+                                                          CXString *Constraint,
+                                                          CXCursor *Expr);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, count the clobbers in it.
+ * This function also returns 0 if the cursor does not point at a GCC inline
+ * assembly block.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_getGCCAssemblyNumClobbers(CXCursor Cursor);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, get the Index-th clobber of it.
+ * This function returns a valid empty string if the cursor does not point
+ * at a GCC inline assembly block or `Index` is out of bounds.
+ *
+ * Users are responsible for releasing the allocation of returned string via
+ * \c clang_disposeString.
+ */
+
+CINDEX_LINKAGE CXString clang_Cursor_getGCCAssemblyClobber(CXCursor Cursor,
+                                                           unsigned Index);
+
+/**
+ * Given a CXCursor_GCCAsmStmt cursor, check if the inline assembly is
+ * `volatile`.
+ * This function returns 0 if the cursor does not point at a GCC inline
+ * assembly block.
+ */
+
+CINDEX_LINKAGE unsigned clang_Cursor_isGCCAssemblyVolatile(CXCursor Cursor);
+
 /**
  * @}
  */
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 10537c94babd..2b9cd035623c 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -2524,15 +2524,6 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// types.
   bool areCompatibleVectorTypes(QualType FirstVec, QualType SecondVec);
 
-  /// Return true if the given types are an SVE builtin and a VectorType that
-  /// is a fixed-length representation of the SVE builtin for a specific
-  /// vector-length.
-  bool areCompatibleSveTypes(QualType FirstType, QualType SecondType);
-
-  /// Return true if the given vector types are lax-compatible SVE vector types,
-  /// false otherwise.
-  bool areLaxCompatibleSveTypes(QualType FirstType, QualType SecondType);
-
   /// Return true if the given types are an RISC-V vector builtin type and a
   /// VectorType that is a fixed-length representation of the RISC-V vector
   /// builtin type for a specific vector-length.
diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index 8d02a50e2e8a..8ebabb2bde10 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -99,7 +99,7 @@ class ASTNodeTraverser
   TraversalKind GetTraversalKind() const { return Traversal; }
 
   void Visit(const Decl *D, bool VisitLocs = false) {
-    if (Traversal == TK_IgnoreUnlessSpelledInSource && D->isImplicit())
+    if (Traversal == TK_IgnoreUnlessSpelledInSource && D && D->isImplicit())
       return;
 
     getNodeDelegate().AddChild([=] {
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 0da940883b6f..f70a039bf351 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -41,6 +41,7 @@
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
@@ -887,13 +888,17 @@ struct EvaluatedStmt {
   bool HasICEInit : 1;
   bool CheckedForICEInit : 1;
 
+  bool HasSideEffects : 1;
+  bool CheckedForSideEffects : 1;
+
   LazyDeclStmtPtr Value;
   APValue Evaluated;
 
   EvaluatedStmt()
       : WasEvaluated(false), IsEvaluating(false),
         HasConstantInitialization(false), HasConstantDestruction(false),
-        HasICEInit(false), CheckedForICEInit(false) {}
+        HasICEInit(false), CheckedForICEInit(false), HasSideEffects(false),
+        CheckedForSideEffects(false) {}
 };
 
 /// Represents a variable declaration or definition.
@@ -1352,9 +1357,11 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
     return const_cast<VarDecl *>(this)->getInitializingDeclaration();
   }
 
-  /// Checks whether this declaration has an initializer with side effects,
-  /// without triggering deserialization if the initializer is not yet
-  /// deserialized.
+  /// Checks whether this declaration has an initializer with side effects.
+  /// The result is cached. If the result hasn't been computed this can trigger
+  /// deserialization and constant evaluation. By running this during
+  /// serialization and serializing the result all clients can safely call this
+  /// without triggering further deserialization.
   bool hasInitWithSideEffects() const;
 
   /// Determine whether this variable's value might be usable in a
@@ -5181,6 +5188,8 @@ class HLSLRootSignatureDecl final
                                     llvm::hlsl::rootsig::RootElement> {
   friend TrailingObjects;
 
+  llvm::dxbc::RootSignatureVersion Version;
+
   unsigned NumElems;
 
   llvm::hlsl::rootsig::RootElement *getElems() { return getTrailingObjects(); }
@@ -5190,16 +5199,20 @@ class HLSLRootSignatureDecl final
   }
 
   HLSLRootSignatureDecl(DeclContext *DC, SourceLocation Loc, IdentifierInfo *ID,
+                        llvm::dxbc::RootSignatureVersion Version,
                         unsigned NumElems);
 
 public:
   static HLSLRootSignatureDecl *
   Create(ASTContext &C, DeclContext *DC, SourceLocation Loc, IdentifierInfo *ID,
+         llvm::dxbc::RootSignatureVersion Version,
          ArrayRef<llvm::hlsl::rootsig::RootElement> RootElements);
 
   static HLSLRootSignatureDecl *CreateDeserialized(ASTContext &C,
                                                    GlobalDeclID ID);
 
+  llvm::dxbc::RootSignatureVersion getVersion() const { return Version; }
+
   ArrayRef<llvm::hlsl::rootsig::RootElement> getRootElements() const {
     return {getElems(), NumElems};
   }
diff --git a/clang/include/clang/AST/DeclarationName.h b/clang/include/clang/AST/DeclarationName.h
index 9bf740b3bf7c..284228dc0ee4 100644
--- a/clang/include/clang/AST/DeclarationName.h
+++ b/clang/include/clang/AST/DeclarationName.h
@@ -698,13 +698,13 @@ class DeclarationNameLoc {
 
   // The location (if any) of the operator keyword is stored elsewhere.
   struct CXXOpName {
-    SourceLocation::UIntTy BeginOpNameLoc;
-    SourceLocation::UIntTy EndOpNameLoc;
+    SourceLocation BeginOpNameLoc;
+    SourceLocation EndOpNameLoc;
   };
 
   // The location (if any) of the operator keyword is stored elsewhere.
   struct CXXLitOpName {
-    SourceLocation::UIntTy OpNameLoc;
+    SourceLocation OpNameLoc;
   };
 
   // struct {} CXXUsingDirective;
@@ -720,12 +720,12 @@ class DeclarationNameLoc {
   void setNamedTypeLoc(TypeSourceInfo *TInfo) { NamedType.TInfo = TInfo; }
 
   void setCXXOperatorNameRange(SourceRange Range) {
-    CXXOperatorName.BeginOpNameLoc = Range.getBegin().getRawEncoding();
-    CXXOperatorName.EndOpNameLoc = Range.getEnd().getRawEncoding();
+    CXXOperatorName.BeginOpNameLoc = Range.getBegin();
+    CXXOperatorName.EndOpNameLoc = Range.getEnd();
   }
 
   void setCXXLiteralOperatorNameLoc(SourceLocation Loc) {
-    CXXLiteralOperatorName.OpNameLoc = Loc.getRawEncoding();
+    CXXLiteralOperatorName.OpNameLoc = Loc;
   }
 
 public:
@@ -739,12 +739,12 @@ class DeclarationNameLoc {
 
   /// Return the beginning location of the getCXXOperatorNameRange() range.
   SourceLocation getCXXOperatorNameBeginLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXOperatorName.BeginOpNameLoc);
+    return CXXOperatorName.BeginOpNameLoc;
   }
 
   /// Return the end location of the getCXXOperatorNameRange() range.
   SourceLocation getCXXOperatorNameEndLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXOperatorName.EndOpNameLoc);
+    return CXXOperatorName.EndOpNameLoc;
   }
 
   /// Return the range of the operator name (without the operator keyword).
@@ -759,7 +759,7 @@ class DeclarationNameLoc {
   /// keyword). Assumes that the object stores location information of a literal
   /// operator.
   SourceLocation getCXXLiteralOperatorNameLoc() const {
-    return SourceLocation::getFromRawEncoding(CXXLiteralOperatorName.OpNameLoc);
+    return CXXLiteralOperatorName.OpNameLoc;
   }
 
   /// Construct location information for a constructor, destructor or conversion
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 477373f07f25..a22c32241ac6 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -84,7 +84,7 @@ class CXXOperatorCallExpr final : public CallExpr {
   friend class ASTStmtReader;
   friend class ASTStmtWriter;
 
-  SourceRange Range;
+  SourceLocation BeginLoc;
 
   // CXXOperatorCallExpr has some trailing objects belonging
   // to CallExpr. See CallExpr for the details.
@@ -158,9 +158,9 @@ class CXXOperatorCallExpr final : public CallExpr {
                : getOperatorLoc();
   }
 
-  SourceLocation getBeginLoc() const { return Range.getBegin(); }
-  SourceLocation getEndLoc() const { return Range.getEnd(); }
-  SourceRange getSourceRange() const { return Range; }
+  SourceLocation getBeginLoc() const { return BeginLoc; }
+  SourceLocation getEndLoc() const { return getSourceRangeImpl().getEnd(); }
+  SourceRange getSourceRange() const { return getSourceRangeImpl(); }
 
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == CXXOperatorCallExprClass;
diff --git a/clang/include/clang/AST/ExternalASTSource.h b/clang/include/clang/AST/ExternalASTSource.h
index e91d5132da10..8fc490b1d847 100644
--- a/clang/include/clang/AST/ExternalASTSource.h
+++ b/clang/include/clang/AST/ExternalASTSource.h
@@ -196,10 +196,6 @@ class ExternalASTSource : public RefCountedBase<ExternalASTSource> {
   /// module.
   virtual bool wasThisDeclarationADefinition(const FunctionDecl *FD);
 
-  virtual bool hasInitializerWithSideEffects(const VarDecl *VD) const {
-    return false;
-  }
-
   /// Finds all declarations lexically contained within the given
   /// DeclContext, after applying an optional filter predicate.
   ///
@@ -434,17 +430,6 @@ struct LazyOffsetPtr {
     return GetPtr();
   }
 
-  /// Retrieve the pointer to the AST node that this lazy pointer points to,
-  /// if it can be done without triggering deserialization.
-  ///
-  /// \returns a pointer to the AST node, or null if not yet deserialized.
-  T *getWithoutDeserializing() const {
-    if (isOffset()) {
-      return nullptr;
-    }
-    return GetPtr();
-  }
-
   /// Retrieve the address of the AST node pointer. Deserializes the pointee if
   /// necessary.
   T **getAddressOfPointer(ExternalASTSource *Source) const {
diff --git a/clang/include/clang/AST/ODRHash.h b/clang/include/clang/AST/ODRHash.h
index a923901b32dc..11f917a1a463 100644
--- a/clang/include/clang/AST/ODRHash.h
+++ b/clang/include/clang/AST/ODRHash.h
@@ -96,7 +96,13 @@ class ODRHash {
   void AddNestedNameSpecifier(const NestedNameSpecifier *NNS);
   void AddDependentTemplateName(const DependentTemplateStorage &Name);
   void AddTemplateName(TemplateName Name);
-  void AddDeclarationName(DeclarationName Name, bool TreatAsDecl = false);
+  void AddDeclarationNameInfo(DeclarationNameInfo NameInfo,
+                              bool TreatAsDecl = false);
+  void AddDeclarationName(DeclarationName Name, bool TreatAsDecl = false) {
+    AddDeclarationNameInfo(DeclarationNameInfo(Name, SourceLocation()),
+                           TreatAsDecl);
+  }
+
   void AddTemplateArgument(TemplateArgument TA);
   void AddTemplateParameterList(const TemplateParameterList *TPL);
 
@@ -108,7 +114,7 @@ class ODRHash {
   static bool isSubDeclToBeProcessed(const Decl *D, const DeclContext *Parent);
 
 private:
-  void AddDeclarationNameImpl(DeclarationName Name);
+  void AddDeclarationNameInfoImpl(DeclarationNameInfo NameInfo);
 };
 
 }  // end namespace clang
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 05239668b34b..5b2206af75be 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -295,7 +295,8 @@ template <class T> class OMPVarListClause : public OMPClause {
 
   /// Fetches list of variables associated with this clause.
   MutableArrayRef<Expr *> getVarRefs() {
-    return static_cast<T *>(this)->template getTrailingObjects<Expr *>(NumVars);
+    return static_cast<T *>(this)->template getTrailingObjectsNonStrict<Expr *>(
+        NumVars);
   }
 
   /// Sets the list of variables for this clause.
@@ -334,8 +335,8 @@ template <class T> class OMPVarListClause : public OMPClause {
 
   /// Fetches list of all variables in the clause.
   ArrayRef<const Expr *> getVarRefs() const {
-    return static_cast<const T *>(this)->template getTrailingObjects<Expr *>(
-        NumVars);
+    return static_cast<const T *>(this)
+        ->template getTrailingObjectsNonStrict<Expr *>(NumVars);
   }
 };
 
@@ -380,7 +381,7 @@ template <class T> class OMPDirectiveListClause : public OMPClause {
 
   MutableArrayRef<OpenMPDirectiveKind> getDirectiveKinds() {
     return static_cast<T *>(this)
-        ->template getTrailingObjects<OpenMPDirectiveKind>(NumKinds);
+        ->template getTrailingObjectsNonStrict<OpenMPDirectiveKind>(NumKinds);
   }
 
   void setDirectiveKinds(ArrayRef<OpenMPDirectiveKind> DK) {
@@ -824,31 +825,52 @@ class OMPNumThreadsClause final
       public OMPClauseWithPreInit {
   friend class OMPClauseReader;
 
+  /// Modifiers for 'num_threads' clause.
+  OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown;
+
+  /// Location of the modifier.
+  SourceLocation ModifierLoc;
+
+  /// Sets modifier.
+  void setModifier(OpenMPNumThreadsClauseModifier M) { Modifier = M; }
+
+  /// Sets modifier location.
+  void setModifierLoc(SourceLocation Loc) { ModifierLoc = Loc; }
+
   /// Set condition.
   void setNumThreads(Expr *NThreads) { setStmt(NThreads); }
 
 public:
   /// Build 'num_threads' clause with condition \a NumThreads.
   ///
+  /// \param Modifier Clause modifier.
   /// \param NumThreads Number of threads for the construct.
   /// \param HelperNumThreads Helper Number of threads for the construct.
   /// \param CaptureRegion Innermost OpenMP region where expressions in this
   /// clause must be captured.
   /// \param StartLoc Starting location of the clause.
   /// \param LParenLoc Location of '('.
+  /// \param ModifierLoc Modifier location.
   /// \param EndLoc Ending location of the clause.
-  OMPNumThreadsClause(Expr *NumThreads, Stmt *HelperNumThreads,
-                      OpenMPDirectiveKind CaptureRegion,
+  OMPNumThreadsClause(OpenMPNumThreadsClauseModifier Modifier, Expr *NumThreads,
+                      Stmt *HelperNumThreads, OpenMPDirectiveKind CaptureRegion,
                       SourceLocation StartLoc, SourceLocation LParenLoc,
-                      SourceLocation EndLoc)
+                      SourceLocation ModifierLoc, SourceLocation EndLoc)
       : OMPOneStmtClause(NumThreads, StartLoc, LParenLoc, EndLoc),
-        OMPClauseWithPreInit(this) {
+        OMPClauseWithPreInit(this), Modifier(Modifier),
+        ModifierLoc(ModifierLoc) {
     setPreInitStmt(HelperNumThreads, CaptureRegion);
   }
 
   /// Build an empty clause.
   OMPNumThreadsClause() : OMPOneStmtClause(), OMPClauseWithPreInit(this) {}
 
+  /// Gets modifier.
+  OpenMPNumThreadsClauseModifier getModifier() const { return Modifier; }
+
+  /// Gets modifier location.
+  SourceLocation getModifierLoc() const { return ModifierLoc; }
+
   /// Returns number of threads.
   Expr *getNumThreads() const { return getStmtAs<Expr>(); }
 };
@@ -5900,15 +5922,17 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the unique declarations that are in the trailing objects of the
   /// class.
   MutableArrayRef<ValueDecl *> getUniqueDeclsRef() {
-    return static_cast<T *>(this)->template getTrailingObjects<ValueDecl *>(
-        NumUniqueDeclarations);
+    return static_cast<T *>(this)
+        ->template getTrailingObjectsNonStrict<ValueDecl *>(
+            NumUniqueDeclarations);
   }
 
   /// Get the unique declarations that are in the trailing objects of the
   /// class.
   ArrayRef<ValueDecl *> getUniqueDeclsRef() const {
     return static_cast<const T *>(this)
-        ->template getTrailingObjects<ValueDecl *>(NumUniqueDeclarations);
+        ->template getTrailingObjectsNonStrict<ValueDecl *>(
+            NumUniqueDeclarations);
   }
 
   /// Set the unique declarations that are in the trailing objects of the
@@ -5922,15 +5946,15 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the number of lists per declaration that are in the trailing
   /// objects of the class.
   MutableArrayRef<unsigned> getDeclNumListsRef() {
-    return static_cast<T *>(this)->template getTrailingObjects<unsigned>(
-        NumUniqueDeclarations);
+    return static_cast<T *>(this)
+        ->template getTrailingObjectsNonStrict<unsigned>(NumUniqueDeclarations);
   }
 
   /// Get the number of lists per declaration that are in the trailing
   /// objects of the class.
   ArrayRef<unsigned> getDeclNumListsRef() const {
-    return static_cast<const T *>(this)->template getTrailingObjects<unsigned>(
-        NumUniqueDeclarations);
+    return static_cast<const T *>(this)
+        ->template getTrailingObjectsNonStrict<unsigned>(NumUniqueDeclarations);
   }
 
   /// Set the number of lists per declaration that are in the trailing
@@ -5945,7 +5969,8 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// objects of the class. They are appended after the number of lists.
   MutableArrayRef<unsigned> getComponentListSizesRef() {
     return MutableArrayRef<unsigned>(
-        static_cast<T *>(this)->template getTrailingObjects<unsigned>() +
+        static_cast<T *>(this)
+                ->template getTrailingObjectsNonStrict<unsigned>() +
             NumUniqueDeclarations,
         NumComponentLists);
   }
@@ -5954,7 +5979,8 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// objects of the class. They are appended after the number of lists.
   ArrayRef<unsigned> getComponentListSizesRef() const {
     return ArrayRef<unsigned>(
-        static_cast<const T *>(this)->template getTrailingObjects<unsigned>() +
+        static_cast<const T *>(this)
+                ->template getTrailingObjectsNonStrict<unsigned>() +
             NumUniqueDeclarations,
         NumComponentLists);
   }
@@ -5970,13 +5996,15 @@ class OMPMappableExprListClause : public OMPVarListClause<T>,
   /// Get the components that are in the trailing objects of the class.
   MutableArrayRef<MappableComponent> getComponentsRef() {
     return static_cast<T *>(this)
-        ->template getTrailingObjects<MappableComponent>(NumComponents);
+        ->template getTrailingObjectsNonStrict<MappableComponent>(
+            NumComponents);
   }
 
   /// Get the components that are in the trailing objects of the class.
   ArrayRef<MappableComponent> getComponentsRef() const {
     return static_cast<const T *>(this)
-        ->template getTrailingObjects<MappableComponent>(NumComponents);
+        ->template getTrailingObjectsNonStrict<MappableComponent>(
+            NumComponents);
   }
 
   /// Set the components that are in the trailing objects of the class.
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 24f3ae78e857..1a87608e5be3 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -6409,12 +6409,7 @@ class SpirvOperand {
   SpirvOperand(const SpirvOperand &Other) { *this = Other; }
   ~SpirvOperand() {}
 
-  SpirvOperand &operator=(const SpirvOperand &Other) {
-    this->Kind = Other.Kind;
-    this->ResultType = Other.ResultType;
-    this->Value = Other.Value;
-    return *this;
-  }
+  SpirvOperand &operator=(const SpirvOperand &Other) = default;
 
   bool operator==(const SpirvOperand &Other) const {
     return Kind == Other.Kind && ResultType == Other.ResultType &&
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index e4d605d16532..bae004896c11 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -6519,11 +6519,11 @@ AST_POLYMORPHIC_MATCHER(
 /// Given
 /// \code
 ///   void a(int);
-///   void b(long);
+///   void b(unsigned long);
 ///   void c(double);
 /// \endcode
 /// functionDecl(hasAnyParameter(hasType(isInteger())))
-/// matches "a(int)", "b(long)", but not "c(double)".
+/// matches "a(int)", "b(unsigned long)", but not "c(double)".
 AST_MATCHER(QualType, isInteger) {
     return Node->isIntegerType();
 }
diff --git a/clang/include/clang/Analysis/CFG.h b/clang/include/clang/Analysis/CFG.h
index e70c70335e59..1b1ff5e558ec 100644
--- a/clang/include/clang/Analysis/CFG.h
+++ b/clang/include/clang/Analysis/CFG.h
@@ -1394,10 +1394,9 @@ class CFG {
   //===--------------------------------------------------------------------===//
 
   template <typename Callback> void VisitBlockStmts(Callback &O) const {
-    for (const_iterator I = begin(), E = end(); I != E; ++I)
-      for (CFGBlock::const_iterator BI = (*I)->begin(), BE = (*I)->end();
-           BI != BE; ++BI) {
-        if (std::optional<CFGStmt> stmt = BI->getAs<CFGStmt>())
+    for (CFGBlock *BB : *this)
+      for (const CFGElement &Elem : *BB) {
+        if (std::optional<CFGStmt> stmt = Elem.getAs<CFGStmt>())
           O(const_cast<Stmt *>(stmt->getStmt()));
       }
   }
diff --git a/clang/include/clang/Basic/AMDGPUTypes.def b/clang/include/clang/Basic/AMDGPUTypes.def
index d3dff446f9ed..a0574c640184 100644
--- a/clang/include/clang/Basic/AMDGPUTypes.def
+++ b/clang/include/clang/Basic/AMDGPUTypes.def
@@ -20,10 +20,18 @@
   AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)
 #endif
 
+#ifndef AMDGPU_FEATURE_PREDICATE_TYPE
+#define AMDGPU_FEATURE_PREDICATE_TYPE(Name, Id, SingletonId, Width, Align) \
+  AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)
+#endif
+
 AMDGPU_OPAQUE_PTR_TYPE("__amdgpu_buffer_rsrc_t", AMDGPUBufferRsrc, AMDGPUBufferRsrcTy, 128, 128, 8)
 
 AMDGPU_NAMED_BARRIER_TYPE("__amdgpu_named_workgroup_barrier_t", AMDGPUNamedWorkgroupBarrier, AMDGPUNamedWorkgroupBarrierTy, 128, 32, 0)
 
+AMDGPU_FEATURE_PREDICATE_TYPE("__amdgpu_feature_predicate_t", AMDGPUFeaturePredicate, AMDGPUFeaturePredicateTy, 1, 1)
+
 #undef AMDGPU_TYPE
 #undef AMDGPU_OPAQUE_PTR_TYPE
 #undef AMDGPU_NAMED_BARRIER_TYPE
+#undef AMDGPU_FEATURE_PREDICATE_TYPE
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 27fea7dea0a5..340f439a45bb 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -965,6 +965,13 @@ def AnalyzerNoReturn : InheritableAttr {
   let Documentation = [Undocumented];
 }
 
+def InferredNoReturn : InheritableAttr {
+  let Spellings = []; 
+  let SemaHandler = 0;
+  let Subjects = SubjectList<[Function], ErrorDiag>;
+  let Documentation = [InternalOnly];
+}
+
 def Annotate : InheritableParamOrStmtAttr {
   let Spellings = [Clang<"annotate">];
   let Args = [StringArgument<"Annotation">, VariadicExprArgument<"Args">];
@@ -3735,6 +3742,7 @@ def Visibility : InheritableAttr {
                            ["default", "hidden", "internal", "protected"],
                            ["Default", "Hidden", "Hidden", "Protected"]>];
   let MeaningfulToClassTemplateDefinition = 1;
+  let PragmaAttributeSupport = 1;
   let Documentation = [Undocumented];
 }
 
diff --git a/clang/include/clang/Basic/BuiltinHeaders.def b/clang/include/clang/Basic/BuiltinHeaders.def
index 22668ec7a339..d6012a896eca 100644
--- a/clang/include/clang/Basic/BuiltinHeaders.def
+++ b/clang/include/clang/Basic/BuiltinHeaders.def
@@ -23,6 +23,7 @@ HEADER(INTRIN_H, "intrin.h")
 HEADER(MALLOC_H, "malloc.h")
 HEADER(MATH_H, "math.h")
 HEADER(MEMORY, "memory")
+HEADER(MMINTRIN_H, "mmintrin.h")
 HEADER(OBJC_MESSAGE_H, "objc/message.h")
 HEADER(OBJC_OBJC_AUTO_H, "objc/objc-auto.h")
 HEADER(OBJC_OBJC_EXCEPTION_H, "objc/objc-exception.h")
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 48437c939757..27f78af16fe0 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -34,6 +34,7 @@
 //  Q -> target builtin type, followed by a character to distinguish the builtin type
 //    Qa -> AArch64 svcount_t builtin type.
 //    Qb -> AMDGPU __amdgpu_buffer_rsrc_t builtin type.
+//    Qc -> AMDGPU __amdgpu_feature_predicate_t builtin type.
 //  E -> ext_vector, followed by the number of elements and the base type.
 //  X -> _Complex, followed by the base type.
 //  Y -> ptrdiff_t
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index d65b3a5d2f44..5ebb82180521 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -2588,6 +2588,24 @@ def InterlockedBittestAndReset_rel : MSLangBuiltin {
   let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
 }
 
+def InterlockedBittestAndReset64_acq : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandreset64_acq"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_nf : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandreset64_nf"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_rel : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandreset64_rel"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
 def InterlockedBittestAndSet : MSLangBuiltin, MSInt32_64Template {
   let Spellings = ["_interlockedbittestandset"];
   let Attributes = [NoThrow];
@@ -2612,6 +2630,24 @@ def InterlockedBittestAndSet_rel : MSLangBuiltin {
   let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
 }
 
+def InterlockedBittestAndSet64_acq : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandset64_acq"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_nf : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandset64_nf"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_rel : MSLangBuiltin {
+  let Spellings = ["_interlockedbittestandset64_rel"];
+  let Attributes = [NoThrow];
+  let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
 def IsoVolatileLoad : MSLangBuiltin, Int8_16_32_64Template {
   let Spellings = ["__iso_volatile_load"];
   let Attributes = [NoThrow];
@@ -4278,6 +4314,12 @@ def MoveIfNsoexcept : CxxLibBuiltin<"utility"> {
   let Namespace = "std";
 }
 
+def Invoke : Builtin {
+  let Spellings = ["__builtin_invoke"];
+  let Attributes = [CustomTypeChecking, Constexpr];
+  let Prototype = "void(...)";
+}
+
 def Annotation : Builtin {
   let Spellings = ["__builtin_annotation"];
   let Attributes = [NoThrow, CustomTypeChecking];
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 8867a9fe09fb..b2aa36acd450 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -155,6 +155,13 @@ TARGET_HEADER_BUILTIN(_InterlockedIncrement64,   "LLiLLiD*",    "nh", INTRIN_H,
 TARGET_HEADER_BUILTIN(_InterlockedOr64,          "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedXor64,         "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
+TARGET_HEADER_BUILTIN(_InterlockedAdd_acq,       "NiNiD*Ni",    "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedAdd_rel,       "NiNiD*Ni",    "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedAdd_nf,        "NiNiD*Ni",    "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedAdd64_acq,     "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedAdd64_rel,     "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedAdd64_nf,      "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+
 TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd_acq, "NiNiD*Ni", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd_rel, "NiNiD*Ni", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd_nf, "NiNiD*Ni", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -267,6 +274,7 @@ TARGET_HEADER_BUILTIN(_ReadWriteBarrier, "v", "nh", INTRIN_H, ALL_MS_LANGUAGES,
 TARGET_HEADER_BUILTIN(__getReg, "ULLii", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_ReadStatusReg,  "LLii",  "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_WriteStatusReg, "viLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__sys, "UiiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_AddressOfReturnAddress, "v*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(__mulh,  "SLLiSLLiSLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 3b969be5316d..4bfc04abe9e9 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -352,8 +352,8 @@ BUILTIN(__builtin_amdgcn_set_fpenv, "vWUi", "n")
 
 // These are special FE only builtins intended for forwarding the requirements
 // to the ME.
-BUILTIN(__builtin_amdgcn_processor_is, "vcC*", "nctu")
-BUILTIN(__builtin_amdgcn_is_invocable, "v", "nctu")
+BUILTIN(__builtin_amdgcn_processor_is, "QccC*", "nctu")
+BUILTIN(__builtin_amdgcn_is_invocable, "Qc", "nctu")
 
 //===----------------------------------------------------------------------===//
 // R600-NI only builtins.
@@ -645,7 +645,34 @@ TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_f16_f32, "V2hV2hfUiIb", "nc", "f32-to-f16
 // GFX1250+ only builtins.
 //===----------------------------------------------------------------------===//
 
+TARGET_BUILTIN(__builtin_amdgcn_tensor_load_to_lds, "vV4iV8iV4iV4iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_load_to_lds_d2, "vV4iV8iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_store_from_lds, "vV4iV8iV4iV4iIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_tensor_store_from_lds_d2, "vV4iV8iIi", "nc", "gfx1250-insts")
+
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr4_b64_v2i32, "V2iV2i*1", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr8_b64_v2i32, "V2iV2i*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr6_b96_v3i32, "V3iV3i*1", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8i16, "V8sV8s*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8f16, "V8hV8h*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_global_load_tr16_b128_v8bf16, "V8yV8y*1", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr4_b64_v2i32, "V2iV2i*3", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr8_b64_v2i32, "V2iV2i*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr6_b96_v3i32, "V3iV3i*3", "nc", "transpose-load-f4f6-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8i16, "V8sV8s*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8f16, "V8hV8h*3", "nc", "gfx1250-insts,wavefrontsize32")
+TARGET_BUILTIN(__builtin_amdgcn_ds_load_tr16_b128_v8bf16, "V8yV8y*3", "nc", "gfx1250-insts,wavefrontsize32")
+
+TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_async_barrier_arrive_b64, "vLi*3", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64, "LiLi*3Li", "nc", "gfx1250-insts")
+
 TARGET_BUILTIN(__builtin_amdgcn_s_setprio_inc_wg, "vIs", "n", "setprio-inc-wg-inst")
+TARGET_BUILTIN(__builtin_amdgcn_s_monitor_sleep,  "vIs", "n", "gfx1250-insts")
+
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f16_fp8, "hiIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_f16_bf8, "hiIi", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f16_fp8, "V2hs", "nc", "gfx1250-insts")
+TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_f16_bf8, "V2hs", "nc", "gfx1250-insts")
 
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 099500754a0e..7c278d6841c7 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -535,6 +535,12 @@ TARGET_BUILTIN(__builtin_ppc_bcdadd_p, "iiV16UcV16Uc", "",
 TARGET_BUILTIN(__builtin_ppc_bcdsub_p, "iiV16UcV16Uc", "",
                "isa-v207-instructions")
 
+// P9 Binary-coded decimal (BCD) builtins.                                                
+TARGET_BUILTIN(__builtin_ppc_national2packed, "V16UcV16UcUc", "t", "power9-vector")
+TARGET_BUILTIN(__builtin_ppc_packed2national, "V16UcV16Uc", "", "power9-vector")
+TARGET_BUILTIN(__builtin_ppc_packed2zoned, "V16UcV16UcUc", "t", "power9-vector")
+TARGET_BUILTIN(__builtin_ppc_zoned2packed, "V16UcV16UcUc", "t", "power9-vector")
+
 TARGET_BUILTIN(__builtin_altivec_vclzlsbb, "SiV16Uc", "", "power9-vector")
 TARGET_BUILTIN(__builtin_altivec_vctzlsbb, "SiV16Uc", "", "power9-vector")
 TARGET_BUILTIN(__builtin_altivec_vprtybw, "V4UiV4Ui", "", "power9-vector")
diff --git a/clang/include/clang/Basic/BuiltinsX86.td b/clang/include/clang/Basic/BuiltinsX86.td
index 67cbbfdec7aa..a4acc72fdc37 100644
--- a/clang/include/clang/Basic/BuiltinsX86.td
+++ b/clang/include/clang/Basic/BuiltinsX86.td
@@ -138,6 +138,14 @@ let Attributes = [Const, NoThrow, RequiredVectorWidth<256>], Features = "avx" in
   }
 }
 
+let Features = "mmx", Header = "mmintrin.h", Attributes = [NoThrow, Const] in {
+  def _m_prefetch : X86LibBuiltin<"void(void *)">;
+}
+
+// PRFCHW
+let Features = "prfchw", Header = "intrin.h", Attributes = [NoThrow, Const] in {
+  def _m_prefetchw : X86LibBuiltin<"void(void volatile const *)">;
+}
 
 // Mechanically ported builtins from the original `.def` file.
 //
@@ -146,8 +154,8 @@ let Attributes = [Const, NoThrow, RequiredVectorWidth<256>], Features = "avx" in
 // current formulation is based on what was easiest to recognize from the
 // pre-TableGen version.
 
-let Features = "mmx", Attributes = [NoThrow, Const] in {
-  def _mm_prefetch : X86NoPrefixBuiltin<"void(char const *, int)">;
+let Features = "sse", Header = "xmmintrin.h", Attributes = [NoThrow, Const] in {
+  def _mm_prefetch : X86LibBuiltin<"void(char const *, int)">;
 }
 
 let Features = "sse", Attributes = [NoThrow] in {
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index 70d928b4ec51..5290485300d8 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -407,6 +407,8 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// with extra debug info.
   SanitizerSet SanitizeAnnotateDebugInfo;
 
+  std::optional<double> AllowRuntimeCheckSkipHotCutoff;
+
   /// List of backend command-line options for -fembed-bitcode.
   std::vector<uint8_t> CmdArgs;
 
@@ -503,6 +505,13 @@ class CodeGenOptions : public CodeGenOptionsBase {
 
   /// A list of functions that are replacable by the loader.
   std::vector<std::string> LoaderReplaceableFunctionNames;
+  /// The name of a file that contains functions which will be compiled for
+  /// hotpatching. See -fms-secure-hotpatch-functions-file.
+  std::string MSSecureHotPatchFunctionsFile;
+
+  /// A list of functions which will be compiled for hotpatching.
+  /// See -fms-secure-hotpatch-functions-list.
+  std::vector<std::string> MSSecureHotPatchFunctionsList;
 
 public:
   // Define accessors/mutators for code generation options of enumeration type.
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index d2cd86d05d55..e3be4ab47633 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -999,6 +999,8 @@ def note_module_odr_violation_mismatch_decl_unknown : Note<
   "different friend declaration|different function template|different method|"
   "different instance variable|different property|another unexpected decl}2">;
 
+def err_struct_too_large : Error<
+  "structure '%0' is too large, which exceeds maximum allowed size of %1 bytes">;
 
 def remark_sanitize_address_insert_extra_padding_accepted : Remark<
     "-fsanitize-address-field-padding applied to %0">, ShowInSystemHeader,
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index fb947965c9c5..676cd1665a16 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -868,6 +868,9 @@ def note_drv_available_multilibs : Note<
   "available multilibs are:%0">;
 def err_drv_multilib_custom_error : Error<
   "multilib configuration error: %0">;
+def warn_drv_multilib_not_available_for_target: Warning<
+  "no multilib structure encoded for Arm, Aarch64 and PPC targets">,
+  InGroup<DiagGroup<"multilib-not-found">>;
 
 def err_drv_experimental_crel : Error<
   "-Wa,--allow-experimental-crel must be specified to use -Wa,--crel. "
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index dafae9983d6c..3298b3dfede6 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1530,9 +1530,11 @@ def OpenMPPre51Compat : DiagGroup<"pre-openmp-51-compat">;
 def OpenMP51Ext : DiagGroup<"openmp-51-extensions">;
 def OpenMPExtensions : DiagGroup<"openmp-extensions">;
 def OpenMPTargetException : DiagGroup<"openmp-target-exception">;
+def OpenMPFuture : DiagGroup<"openmp-future">;
 def OpenMP : DiagGroup<"openmp", [
     SourceUsesOpenMP, OpenMPClauses, OpenMPLoopForm, OpenMPTarget,
-    OpenMPMapping, OpenMP51Ext, OpenMPExtensions, OpenMPTargetException
+    OpenMPMapping, OpenMP51Ext, OpenMPExtensions, OpenMPTargetException,
+    OpenMPFuture
   ]>;
 
 // OpenACC warnings.
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 6c30da376daf..87eb2b724b29 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -1488,6 +1488,9 @@ def err_omp_multiple_step_or_linear_modifier : Error<
   "multiple %select{'step size'|'linear modifier'}0 found in linear clause">;
 def err_omp_deprecate_old_syntax: Error<
   "old syntax '%0' on '%1' clause was deprecated, use new syntax '%2'">;
+def warn_omp_future_directive_spelling: Warning<
+  "directive spelling '%0' is introduced in a later OpenMP version">,
+  InGroup<OpenMPFuture>;
 def warn_pragma_expected_colon_r_paren : Warning<
   "missing ':' or ')' after %0 - ignoring">, InGroup<IgnoredPragmas>;
 def err_omp_unknown_directive : Error<
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 98d4523a5660..e09cc7084e98 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1062,6 +1062,9 @@ def err_constexpr_main : Error<
   "'main' is not allowed to be declared %select{constexpr|consteval}0">;
 def err_deleted_main : Error<"'main' is not allowed to be deleted">;
 def err_mainlike_template_decl : Error<"%0 cannot be a template">;
+def warn_main_in_named_module
+    : ExtWarn<"'main' never has module linkage">,
+      InGroup<DiagGroup<"main-attached-to-named-module">>;
 def err_main_returns_nonint : Error<"'main' must return 'int'">;
 def ext_main_returns_nonint : ExtWarn<"return type of 'main' is not 'int'">,
     InGroup<MainReturnType>;
@@ -1084,9 +1087,9 @@ def warn_main_redefined : Warning<"variable named 'main' with external linkage "
     "has undefined behavior">, InGroup<Main>;
 def ext_main_used : Extension<
     "referring to 'main' within an expression is a Clang extension">, InGroup<Main>;
-def ext_main_invalid_linkage_specification : ExtWarn<
-  "'main' should not be "
-  "'extern \"%select{C|C++}0\"'">, InGroup<Main>;
+def ext_main_invalid_linkage_specification : ExtWarn<"'main' should not be "
+                                                     "'extern \"C\"'">,
+                                             InGroup<Main>;
 
 /// parser diagnostics
 def ext_no_declarators : ExtWarn<"declaration does not declare anything">,
@@ -1767,7 +1770,9 @@ def note_unsatisfied_trait
     : Note<"%0 is not %enum_select<TraitName>{"
            "%TriviallyRelocatable{trivially relocatable}|"
            "%Replaceable{replaceable}|"
-           "%TriviallyCopyable{trivially copyable}"
+           "%TriviallyCopyable{trivially copyable}|"
+           "%Empty{empty}|"
+           "%StandardLayout{standard-layout}"
            "}1">;
 
 def note_unsatisfied_trait_reason
@@ -1787,6 +1792,16 @@ def note_unsatisfied_trait_reason
            "%NonReplaceableField{has a non-replaceable member %1 of type %2}|"
            "%NTCBase{has a non-trivially-copyable base %1}|"
            "%NTCField{has a non-trivially-copyable member %1 of type %2}|"
+           "%NonEmptyMember{has a non-static data member %1 of type %2}|"
+           "%VirtualFunction{has a virtual function %1}|"
+           "%NonEmptyBase{has a base class %1 that is not empty}|"
+           "%NonZeroLengthField{field %1 is a non-zero-length bit-field}|"
+           "%NonStandardLayoutBase{has a non-standard-layout base %1}|"
+           "%MixedAccess{has mixed access specifiers}|"
+           "%MixedAccessField{field %1 has a different access specifier than field %2}|"
+           "%MultipleDataBase{has multiple base classes with data members}|"
+           "%NonStandardLayoutMember{has a non-standard-layout member %1 of type %2}|"
+           "%IndirectBaseWithFields{has an indirect base %1 with data members}|"
            "%DeletedDtr{has a %select{deleted|user-provided}1 destructor}|"
            "%UserProvidedCtr{has a user provided %select{copy|move}1 "
            "constructor}|"
@@ -3317,6 +3332,9 @@ def err_sve_vector_in_non_sve_target : Error<
   "SVE vector type %0 cannot be used in a target without sve">;
 def err_sve_vector_in_non_streaming_function : Error<
   "SVE vector type %0 cannot be used in a non-streaming function">;
+def err_sve_fixed_vector_in_streaming_function
+    : Error<"fixed width SVE vector type %0 cannot be used in a "
+            "%select{streaming|streaming-compatible}1 function">;
 def err_attribute_riscv_rvv_bits_unsupported : Error<
   "%0 is only supported when '-mrvv-vector-bits=<bits>' is specified with a "
   "value of \"zvl\" or a power 2 in the range [64,65536]">;
@@ -12503,7 +12521,7 @@ def warn_zero_as_null_pointer_constant : Warning<
   InGroup<DiagGroup<"zero-as-null-pointer-constant">>, DefaultIgnore;
 
 def warn_not_eliding_copy_on_return : Warning<
-  "not eliding copy on return">, 
+  "not eliding copy on return">,
   InGroup<DiagGroup<"nrvo">>, DefaultIgnore;
 
 def err_nullability_cs_multilevel : Error<
@@ -13058,10 +13076,13 @@ def err_invalid_hlsl_resource_type: Error<
 def err_hlsl_spirv_only: Error<"%0 is only available for the SPIR-V target">;
 def err_hlsl_vk_literal_must_contain_constant: Error<"the argument to vk::Literal must be a vk::integral_constant">;
 
+def subst_hlsl_format_ranges: TextSubstitution<
+  "%select{t|u|b|s}0[%1;%select{%3]|unbounded)}2">;
+
 def err_hlsl_resource_range_overlap: Error<
-  "resource ranges %select{t|u|b|s}0[%1;%2] and %select{t|u|b|s}3[%4;%5] "
-  "overlap within space = %6 and visibility = "
-  "%select{All|Vertex|Hull|Domain|Geometry|Pixel|Amplification|Mesh}7">;
+  "resource ranges %sub{subst_hlsl_format_ranges}0,1,2,3 and %sub{subst_hlsl_format_ranges}4,5,6,7 "
+  "overlap within space = %8 and visibility = "
+  "%select{All|Vertex|Hull|Domain|Geometry|Pixel|Amplification|Mesh}9">;
 
 // Layout randomization diagnostics.
 def err_non_designated_init_used : Error<
@@ -13203,8 +13224,12 @@ def err_acc_not_a_var_ref_use_device_declare
     : Error<"OpenACC variable %select{in 'use_device' clause|on 'declare' "
             "construct}0 is not a valid variable name or array name">;
 def err_acc_not_a_var_ref_cache
-    : Error<"OpenACC variable in cache directive is not a valid sub-array or "
+    : Error<"OpenACC variable in 'cache' directive is not a valid sub-array or "
             "array element">;
+def warn_acc_cache_var_not_outside_loop
+    : Warning<"OpenACC variable in 'cache' directive was not declared outside "
+              "of the associated 'loop' directive; directive has no effect">,
+      InGroup<DiagGroup<"openacc-cache-var-inside-loop">>;
 def err_acc_typecheck_subarray_value
     : Error<"OpenACC sub-array subscripted value is not an array or pointer">;
 def err_acc_subarray_function_type
@@ -13440,8 +13465,19 @@ def err_amdgcn_processor_is_arg_not_literal
 def err_amdgcn_processor_is_arg_invalid_value
     : Error<"the argument to __builtin_amdgcn_processor_is must be a valid "
             "AMDGCN processor identifier; '%0' is not valid">;
+def note_amdgcn_processor_is_valid_options
+    : Note<"valid AMDGCN processor identifiers are: %0">;
 def err_amdgcn_is_invocable_arg_invalid_value
     : Error<"the argument to __builtin_amdgcn_is_invocable must be either a "
             "target agnostic builtin or an AMDGCN target specific builtin; `%0`"
             " is not valid">;
+def err_amdgcn_predicate_type_is_not_constructible
+    : Error<"%0 has type __amdgpu_feature_predicate_t, which is not"
+            " constructible">;
+def err_amdgcn_predicate_type_needs_explicit_bool_cast
+    : Error<"%0 must be explicitly cast to %1; however, please note that this "
+            "is almost always an error and that it prevents the effective "
+            "guarding of target dependent code, and thus should be avoided">;
+def note_amdgcn_protected_by_predicate : Note<"jump enters statement controlled"
+                                              " by AMDGPU feature predicate">;
 } // end of sema component.
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 2c84bdf0337a..0308ae561e91 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -518,6 +518,9 @@ LANGOPT(OmitVTableRTTI, 1, 0,
 LANGOPT(VScaleMin, 32, 0, "Minimum vscale value")
 LANGOPT(VScaleMax, 32, 0, "Maximum vscale value")
 
+LANGOPT(VScaleStreamingMin, 32, 0, "Minimum streaming vscale value")
+LANGOPT(VScaleStreamingMax, 32, 0, "Maximum streaming vscale value")
+
 ENUM_LANGOPT(ExtendIntArgs, ExtendArgsKind, 1, ExtendArgsKind::ExtendTo32,
              "Controls how scalar integer arguments are extended in calls "
              "to unprototyped and varargs functions")
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 491e8bee9fd5..b7c5ed994c99 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -24,6 +24,7 @@
 #include "clang/Basic/Visibility.h"
 #include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/TargetParser/Triple.h"
 #include <optional>
 #include <string>
@@ -623,6 +624,10 @@ class LangOptions : public LangOptionsBase {
   // implementation on real-world examples.
   std::string OpenACCMacroOverride;
 
+  /// The HLSL root signature version for dxil.
+  llvm::dxbc::RootSignatureVersion HLSLRootSigVer =
+      llvm::dxbc::RootSignatureVersion::V1_1;
+
   // Indicates if the wasm-opt binary must be ignored in the case of a
   // WebAssembly target.
   bool NoWasmOpt = false;
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 3d035f0a5f78..69a1de6f79b3 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -64,6 +64,11 @@ struct ASTFileSignature : std::array<uint8_t, 20> {
 
   explicit operator bool() const { return *this != BaseT({{0}}); }
 
+  // Support implicit cast to ArrayRef.  Note that ASTFileSignature::size
+  // prevents implicit cast to ArrayRef because one of the implicit constructors
+  // of ArrayRef requires access to BaseT::size.
+  operator ArrayRef<uint8_t>() const { return ArrayRef<uint8_t>(data(), size); }
+
   /// Returns the value truncated to the size of an uint64_t.
   uint64_t truncatedValue() const {
     uint64_t Value = 0;
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index a7a29e2add20..938385679e3a 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -634,16 +634,19 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
 }
 
 // Represents the 'modifier' of a 'modifier-list', as applied to copy, copyin,
-// copyout, and create. Implemented as a 'bitmask'
+// copyout, and create. Implemented as a 'bitmask'.
+// Note: This attempts to synchronize with mlir::acc::DataClauseModifier,
+// however has to store `Always` separately(whereas MLIR has it as AlwaysIn &
+// AlwaysOut). However, we keep them in sync so that we can cast between them.
 enum class OpenACCModifierKind : uint8_t {
   Invalid = 0,
-  Always = 1 << 0,
-  AlwaysIn = 1 << 1,
-  AlwaysOut = 1 << 2,
-  Readonly = 1 << 3,
-  Zero = 1 << 4,
-  Capture = 1 << 5,
-  LLVM_MARK_AS_BITMASK_ENUM(Capture)
+  Zero = 1 << 0,
+  Readonly = 1 << 1,
+  AlwaysIn = 1 << 2,
+  AlwaysOut = 1 << 3,
+  Capture = 1 << 4,
+  Always = 1 << 5,
+  LLVM_MARK_AS_BITMASK_ENUM(Always)
 };
 
 inline bool isOpenACCModifierBitSet(OpenACCModifierKind List,
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 2b1dc1e0121b..9d6f816eea91 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -86,6 +86,9 @@
 #ifndef OPENMP_NUMTASKS_MODIFIER
 #define OPENMP_NUMTASKS_MODIFIER(Name)
 #endif
+#ifndef OPENMP_NUMTHREADS_MODIFIER
+#define OPENMP_NUMTHREADS_MODIFIER(Name)
+#endif
 #ifndef OPENMP_DOACROSS_MODIFIER
 #define OPENMP_DOACROSS_MODIFIER(Name)
 #endif
@@ -227,6 +230,9 @@ OPENMP_GRAINSIZE_MODIFIER(strict)
 // Modifiers for the 'num_tasks' clause.
 OPENMP_NUMTASKS_MODIFIER(strict)
 
+// Modifiers for the 'num_tasks' clause.
+OPENMP_NUMTHREADS_MODIFIER(strict)
+
 // Modifiers for 'allocate' clause.
 OPENMP_ALLOCATE_MODIFIER(allocator)
 OPENMP_ALLOCATE_MODIFIER(align)
@@ -238,6 +244,7 @@ OPENMP_DOACROSS_MODIFIER(sink_omp_cur_iteration)
 OPENMP_DOACROSS_MODIFIER(source_omp_cur_iteration)
 
 #undef OPENMP_NUMTASKS_MODIFIER
+#undef OPENMP_NUMTHREADS_MODIFIER
 #undef OPENMP_GRAINSIZE_MODIFIER
 #undef OPENMP_BIND_KIND
 #undef OPENMP_ADJUST_ARGS_KIND
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 9afcce21a499..f40db4c13c55 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -223,6 +223,12 @@ enum OpenMPNumTasksClauseModifier {
   OMPC_NUMTASKS_unknown
 };
 
+enum OpenMPNumThreadsClauseModifier {
+#define OPENMP_NUMTHREADS_MODIFIER(Name) OMPC_NUMTHREADS_##Name,
+#include "clang/Basic/OpenMPKinds.def"
+  OMPC_NUMTHREADS_unknown
+};
+
 /// OpenMP dependence types for 'doacross' clause.
 enum OpenMPDoacrossClauseModifier {
 #define OPENMP_DOACROSS_MODIFIER(Name) OMPC_DOACROSS_##Name,
diff --git a/clang/include/clang/Basic/Sanitizers.h b/clang/include/clang/Basic/Sanitizers.h
index 3782deb5962b..19105aa33978 100644
--- a/clang/include/clang/Basic/Sanitizers.h
+++ b/clang/include/clang/Basic/Sanitizers.h
@@ -218,8 +218,7 @@ bool parseSanitizerWeightedValue(StringRef Value, bool AllowGroups,
 void serializeSanitizerSet(SanitizerSet Set,
                            SmallVectorImpl<StringRef> &Values);
 
-/// Serialize a SanitizerMaskCutoffs into values for -fsanitize= or
-/// -fno-sanitize=.
+/// Serialize a SanitizerMaskCutoffs into command line arguments.
 void serializeSanitizerMaskCutoffs(const SanitizerMaskCutoffs &Cutoffs,
                                    SmallVectorImpl<std::string> &Values);
 
diff --git a/clang/include/clang/Basic/SourceLocation.h b/clang/include/clang/Basic/SourceLocation.h
index 7a0f5ba8d127..14543cc41a38 100644
--- a/clang/include/clang/Basic/SourceLocation.h
+++ b/clang/include/clang/Basic/SourceLocation.h
@@ -70,6 +70,8 @@ class FileID {
   int getOpaqueValue() const { return ID; }
 };
 
+using FileIDAndOffset = std::pair<FileID, unsigned>;
+
 /// Encodes a location in the source. The SourceManager can decode this
 /// to get at the full include stack, line and column information.
 ///
@@ -403,7 +405,7 @@ class FullSourceLoc : public SourceLocation {
   /// pair, after walking through all expansion records.
   ///
   /// \see SourceManager::getDecomposedExpansionLoc
-  std::pair<FileID, unsigned> getDecomposedExpansionLoc() const;
+  FileIDAndOffset getDecomposedExpansionLoc() const;
 
   unsigned getSpellingLineNumber(bool *Invalid = nullptr) const;
   unsigned getSpellingColumnNumber(bool *Invalid = nullptr) const;
@@ -424,7 +426,7 @@ class FullSourceLoc : public SourceLocation {
   ///
   /// The first element is the FileID, the second is the offset from the
   /// start of the buffer of the location.
-  std::pair<FileID, unsigned> getDecomposedLoc() const;
+  FileIDAndOffset getDecomposedLoc() const;
 
   bool isInSystemHeader() const;
 
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index cd3dac913322..ed967fd47dc8 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -719,6 +719,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// Positive FileIDs are indexes into this table. Entry 0 indicates an invalid
   /// expansion.
   SmallVector<SrcMgr::SLocEntry, 0> LocalSLocEntryTable;
+  /// An in-parallel offset table, merely used for speeding up FileID lookup.
+  SmallVector<SourceLocation::UIntTy> LocalLocOffsetTable;
 
   /// The table of SLocEntries that are loaded from other modules.
   ///
@@ -767,6 +769,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// LastFileIDLookup records the last FileID looked up or created, because it
   /// is very common to look up many tokens from the same file.
   mutable FileID LastFileIDLookup;
+  mutable SourceLocation::UIntTy LastLookupStartOffset;
+  mutable SourceLocation::UIntTy LastLookupEndOffset; // exclude
 
   /// Holds information for \#line directives.
   ///
@@ -795,7 +799,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///
   /// Used to cache results from and speed-up \c getDecomposedIncludedLoc
   /// function.
-  mutable llvm::DenseMap<FileID, std::pair<FileID, unsigned>> IncludedLocMap;
+  mutable llvm::DenseMap<FileID, FileIDAndOffset> IncludedLocMap;
 
   /// The key value into the IsBeforeInTUCache table.
   using IsBeforeInTUCacheKey = std::pair<FileID, FileID>;
@@ -1269,7 +1273,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///
   /// The first element is the FileID, the second is the offset from the
   /// start of the buffer of the location.
-  std::pair<FileID, unsigned> getDecomposedLoc(SourceLocation Loc) const {
+  FileIDAndOffset getDecomposedLoc(SourceLocation Loc) const {
     FileID FID = getFileID(Loc);
     auto *Entry = getSLocEntryOrNull(FID);
     if (!Entry)
@@ -1281,8 +1285,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///
   /// If the location is an expansion record, walk through it until we find
   /// the final location expanded.
-  std::pair<FileID, unsigned>
-  getDecomposedExpansionLoc(SourceLocation Loc) const {
+  FileIDAndOffset getDecomposedExpansionLoc(SourceLocation Loc) const {
     FileID FID = getFileID(Loc);
     auto *E = getSLocEntryOrNull(FID);
     if (!E)
@@ -1299,8 +1302,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///
   /// If the location is an expansion record, walk through it until we find
   /// its spelling record.
-  std::pair<FileID, unsigned>
-  getDecomposedSpellingLoc(SourceLocation Loc) const {
+  FileIDAndOffset getDecomposedSpellingLoc(SourceLocation Loc) const {
     FileID FID = getFileID(Loc);
     auto *E = getSLocEntryOrNull(FID);
     if (!E)
@@ -1314,7 +1316,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
 
   /// Returns the "included/expanded in" decomposed location of the given
   /// FileID.
-  std::pair<FileID, unsigned> getDecomposedIncludedLoc(FileID FID) const;
+  FileIDAndOffset getDecomposedIncludedLoc(FileID FID) const;
 
   /// Returns the offset from the start of the file that the
   /// specified SourceLocation represents.
@@ -1682,8 +1684,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///          are in the same TU. The second bool is true if the first is true
   ///          and \p LOffs is before \p ROffs.
   std::pair<bool, bool>
-  isInTheSameTranslationUnit(std::pair<FileID, unsigned> &LOffs,
-                             std::pair<FileID, unsigned> &ROffs) const;
+  isInTheSameTranslationUnit(FileIDAndOffset &LOffs,
+                             FileIDAndOffset &ROffs) const;
 
   /// \param Loc a source location in a loaded AST (of a PCH/Module file).
   /// \returns a FileID uniquely identifies the AST of a loaded
@@ -1691,9 +1693,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
   FileID getUniqueLoadedASTFileID(SourceLocation Loc) const;
 
   /// Determines whether the two decomposed source location is in the same TU.
-  bool isInTheSameTranslationUnitImpl(
-      const std::pair<FileID, unsigned> &LOffs,
-      const std::pair<FileID, unsigned> &ROffs) const;
+  bool isInTheSameTranslationUnitImpl(const FileIDAndOffset &LOffs,
+                                      const FileIDAndOffset &ROffs) const;
 
   /// Determines the order of 2 source locations in the "source location
   /// address space".
@@ -1904,9 +1905,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
 
   FileID getFileID(SourceLocation::UIntTy SLocOffset) const {
     // If our one-entry cache covers this offset, just return it.
-    if (isOffsetInFileID(LastFileIDLookup, SLocOffset))
+    if (SLocOffset >= LastLookupStartOffset && SLocOffset < LastLookupEndOffset)
       return LastFileIDLookup;
-
     return getFileIDSlow(SLocOffset);
   }
 
@@ -1979,11 +1979,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
   SourceLocation getSpellingLocSlowCase(SourceLocation Loc) const;
   SourceLocation getFileLocSlowCase(SourceLocation Loc) const;
 
-  std::pair<FileID, unsigned>
+  FileIDAndOffset
   getDecomposedExpansionLocSlowCase(const SrcMgr::SLocEntry *E) const;
-  std::pair<FileID, unsigned>
-  getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
-                                   unsigned Offset) const;
+  FileIDAndOffset getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
+                                                   unsigned Offset) const;
   void computeMacroArgsCache(MacroArgsMap &MacroArgsCache, FileID FID) const;
   void associateFileChunkWithMacroArgExp(MacroArgsMap &MacroArgsCache,
                                          FileID FID,
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index c3bce6e807f3..c2c8cc3804e7 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1034,9 +1034,15 @@ class TargetInfo : public TransferrableTargetInfo,
   /// set of primary and secondary targets.
   virtual llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const = 0;
 
+  enum class ArmStreamingKind {
+    NotStreaming,
+    StreamingCompatible,
+    Streaming,
+  };
+
   /// Returns target-specific min and max values VScale_Range.
   virtual std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const {
     return std::nullopt;
   }
@@ -1851,7 +1857,7 @@ class TargetInfo : public TransferrableTargetInfo,
 
   /// Returns the version of the darwin target variant SDK which was used during
   /// the compilation if one was specified, or an empty version otherwise.
-  const std::optional<VersionTuple> getDarwinTargetVariantSDKVersion() const {
+  std::optional<VersionTuple> getDarwinTargetVariantSDKVersion() const {
     return !getTargetOpts().DarwinTargetVariantSDKVersion.empty()
                ? getTargetOpts().DarwinTargetVariantSDKVersion
                : std::optional<VersionTuple>();
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 7251cc2d1759..0daef4ad1701 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -767,47 +767,47 @@ let ArchGuard = "defined(__aarch64__) || defined(__arm64ec__)" in {
 
 ////////////////////////////////////////////////////////////////////////////////
 // Load/Store
-def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPl">;
-def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPl">;
-def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPl">;
-def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPl">;
-def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPl">;
-def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPl">;
-def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPl">;
-def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPl">;
+def LD1 : WInst<"vld1", ".(c*!)", "dQdPlQPlmQm">;
+def LD2 : WInst<"vld2", "2(c*!)", "QUlQldQdPlQPlmQm">;
+def LD3 : WInst<"vld3", "3(c*!)", "QUlQldQdPlQPlmQm">;
+def LD4 : WInst<"vld4", "4(c*!)", "QUlQldQdPlQPlmQm">;
+def ST1 : WInst<"vst1", "v*(.!)", "dQdPlQPlmQm">;
+def ST2 : WInst<"vst2", "v*(2!)", "QUlQldQdPlQPlmQm">;
+def ST3 : WInst<"vst3", "v*(3!)", "QUlQldQdPlQPlmQm">;
+def ST4 : WInst<"vst4", "v*(4!)", "QUlQldQdPlQPlmQm">;
 
 def LD1_X2 : WInst<"vld1_x2", "2(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 def LD1_X3 : WInst<"vld1_x3", "3(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 def LD1_X4 : WInst<"vld1_x4", "4(c*!)",
-                   "dQdPlQPl">;
+                   "dQdPlQPlmQm">;
 
-def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPl">;
-def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPl">;
-def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPl">;
+def ST1_X2 : WInst<"vst1_x2", "v*(2!)", "dQdPlQPlmQm">;
+def ST1_X3 : WInst<"vst1_x3", "v*(3!)", "dQdPlQPlmQm">;
+def ST1_X4 : WInst<"vst1_x4", "v*(4!)", "dQdPlQPlmQm">;
 
-def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPl",
+def LD1_LANE : WInst<"vld1_lane", ".(c*!).I", "dQdPlQPlmQm",
                     [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
-def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD2_LANE : WInst<"vld2_lane", "2(c*!)2I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<4, ImmCheckLaneIndex, 1>]>;
-def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD3_LANE : WInst<"vld3_lane", "3(c*!)3I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<5, ImmCheckLaneIndex, 1>]>;
-def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def LD4_LANE : WInst<"vld4_lane", "4(c*!)4I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<6, ImmCheckLaneIndex, 1>]>;
-def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPl",
+def ST1_LANE : WInst<"vst1_lane", "v*(.!)I", "dQdPlQPlmQm",
                     [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
-def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST2_LANE : WInst<"vst2_lane", "v*(2!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<3, ImmCheckLaneIndex, 1>]>;
-def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST3_LANE : WInst<"vst3_lane", "v*(3!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<4, ImmCheckLaneIndex, 1>]>;
-def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPl",
+def ST4_LANE : WInst<"vst4_lane", "v*(4!)I", "lUlQcQUcQPcQlQUldQdPlQPlmQm",
                     [ImmCheck<5, ImmCheckLaneIndex, 1>]>;
 
-def LD1_DUP  : WInst<"vld1_dup", ".(c*!)", "dQdPlQPl">;
-def LD2_DUP  : WInst<"vld2_dup", "2(c*!)", "dQdPlQPl">;
-def LD3_DUP  : WInst<"vld3_dup", "3(c*!)", "dQdPlQPl">;
-def LD4_DUP  : WInst<"vld4_dup", "4(c*!)", "dQdPlQPl">;
+def LD1_DUP  : WInst<"vld1_dup", ".(c*!)", "dQdPlQPlmQm">;
+def LD2_DUP  : WInst<"vld2_dup", "2(c*!)", "dQdPlQPlmQm">;
+def LD3_DUP  : WInst<"vld3_dup", "3(c*!)", "dQdPlQPlmQm">;
+def LD4_DUP  : WInst<"vld4_dup", "4(c*!)", "dQdPlQPlmQm">;
 
 def VLDRQ : WInst<"vldrq", "1(c*!)", "Pk">;
 def VSTRQ : WInst<"vstrq", "v*(1!)", "Pk">;
@@ -2194,4 +2194,4 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
   // fscale
   def FSCALE_V128 : WInst<"vscale", "..(.S)", "QdQfQh">;
   def FSCALE_V64 : WInst<"vscale", "(.q)(.q)(.qS)", "fh">;
-}
\ No newline at end of file
+}
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 1b3131be7845..76fd072a41d8 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -121,7 +121,7 @@ def SVLD1UW_GATHER_INDEX_S   : MInst<"svld1uw_gather[_{2}base]_index_{d}", "dPul
 
 
 // First-faulting load one vector (scalar base)
-def SVLDFF1   : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
+def SVLDFF1   : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
 def SVLDFF1SB : MInst<"svldff1sb_{d}", "dPS", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1UB : MInst<"svldff1ub_{d}", "dPW", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1SH : MInst<"svldff1sh_{d}", "dPT", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldff1">;
@@ -130,7 +130,7 @@ def SVLDFF1SW : MInst<"svldff1sw_{d}", "dPU", "lUl",             [IsLoad],
 def SVLDFF1UW : MInst<"svldff1uw_{d}", "dPY", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1">;
 
 // First-faulting load one vector (scalar base, VL displacement)
-def SVLDFF1_VNUM   : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
+def SVLDFF1_VNUM   : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldff1">;
 def SVLDFF1SB_VNUM : MInst<"svldff1sb_vnum_{d}", "dPSl", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1UB_VNUM : MInst<"svldff1ub_vnum_{d}", "dPWl", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldff1">;
 def SVLDFF1SH_VNUM : MInst<"svldff1sh_vnum_{d}", "dPTl", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldff1">;
@@ -223,7 +223,7 @@ def SVLDFF1SW_GATHER_INDEX_S : MInst<"svldff1sw_gather[_{2}base]_index_{d}", "dP
 def SVLDFF1UW_GATHER_INDEX_S : MInst<"svldff1uw_gather[_{2}base]_index_{d}", "dPul", "lUl",      [IsGatherLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1_gather_scalar_offset">;
 
 // Non-faulting load one vector (scalar base)
-def SVLDNF1   : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
+def SVLDNF1   : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
 def SVLDNF1SB : MInst<"svldnf1sb_{d}", "dPS", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1UB : MInst<"svldnf1ub_{d}", "dPW", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1SH : MInst<"svldnf1sh_{d}", "dPT", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldnf1">;
@@ -232,7 +232,7 @@ def SVLDNF1SW : MInst<"svldnf1sw_{d}", "dPU", "lUl",             [IsLoad],
 def SVLDNF1UW : MInst<"svldnf1uw_{d}", "dPY", "lUl",             [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldnf1">;
 
 // Non-faulting load one vector (scalar base, VL displacement)
-def SVLDNF1_VNUM   : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
+def SVLDNF1_VNUM   : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfdm", [IsLoad],               MemEltTyDefault, "aarch64_sve_ldnf1">;
 def SVLDNF1SB_VNUM : MInst<"svldnf1sb_vnum_{d}", "dPSl", "silUsUiUl",       [IsLoad],               MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1UB_VNUM : MInst<"svldnf1ub_vnum_{d}", "dPWl", "silUsUiUl",       [IsLoad, IsZExtReturn], MemEltTyInt8,    "aarch64_sve_ldnf1">;
 def SVLDNF1SH_VNUM : MInst<"svldnf1sh_vnum_{d}", "dPTl", "ilUiUl",          [IsLoad],               MemEltTyInt16,   "aarch64_sve_ldnf1">;
@@ -258,7 +258,7 @@ let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
 }
 
 // Load one quadword and replicate (scalar base)
-def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
+def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfdm", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
 
 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
   def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc",  "b", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>;
@@ -283,7 +283,7 @@ defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4_sret">
 
 // Load one octoword and replicate (scalar base)
 let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in {
-  def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">;
+  def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfdm", MergeNone, "aarch64_sve_ld1ro">;
 }
 let SVETargetGuard = "sve,f64mm,bf16", SMETargetGuard = InvalidMode in {
   def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">;
diff --git a/clang/include/clang/Basic/riscv_andes_vector.td b/clang/include/clang/Basic/riscv_andes_vector.td
index 01019cf86d6e..67e07f8396ad 100644
--- a/clang/include/clang/Basic/riscv_andes_vector.td
+++ b/clang/include/clang/Basic/riscv_andes_vector.td
@@ -18,6 +18,52 @@ include "riscv_vector_common.td"
 // Instruction definitions
 //===----------------------------------------------------------------------===//
 
+// Andes Vector BFLOAT16 Conversion Extension (XAndesVBFHCvt)
+
+let RequiredFeatures = ["xandesvbfhcvt"],
+    Log2LMUL = [-2, -1, 0, 1, 2],
+    HasMasked = false,
+    UnMaskedPolicyScheme = HasPassthruOperand in {
+  def nds_vfwcvt_s_bf16 : RVVConvBuiltin<"Fw", "Fwv", "y", "nds_vfwcvt_s">;
+
+  let ManualCodegen = [{
+    {
+      // LLVM intrinsic
+      // Unmasked: (passthru, op0, frm, vl)
+      SmallVector<llvm::Value*, 4> Operands;
+      bool HasMaskedOff = !(PolicyAttrs & RVV_VTA);
+      bool HasRoundModeOp = HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3;
+
+      unsigned Offset = HasMaskedOff ? 1 : 0;
+
+      if (!HasMaskedOff)
+        Operands.push_back(llvm::PoisonValue::get(ResultType));
+      else
+        Operands.push_back(Ops[0]);
+
+      Operands.push_back(Ops[Offset]); // op0
+
+      if (HasRoundModeOp) {
+        Operands.push_back(Ops[Offset + 1]); // frm
+        Operands.push_back(Ops[Offset + 2]); // vl
+      } else {
+        Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm
+        Operands.push_back(Ops[Offset + 1]); // vl
+      }
+
+      IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
+                        Operands.back()->getType()};
+      llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
+      return Builder.CreateCall(F, Operands, "");
+    }
+  }] in {
+    let HasFRMRoundModeOp = 1, Name = "nds_vfncvt_bf16_s", IRName = "nds_vfncvt_bf16_s" in
+      def nds_vfncvt_bf16_s_rm : RVVConvBuiltin<"v", "vFwu", "y", "nds_vfncvt_bf16">;
+
+    def nds_vfncvt_bf16_s : RVVConvBuiltin<"v", "vFw", "y", "nds_vfncvt_bf16">;
+  }
+}
+
 // Andes Vector Packed FP16 Extension (XAndesVPackFPH)
 
 multiclass RVVFPMAD {
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index c6fd8a1a45fd..5faf81272ec5 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -116,10 +116,7 @@ multiclass RVVIndexedLoad<string op> {
       foreach eew_list = EEWList[0-2] in {
         defvar eew = eew_list[0];
         defvar eew_type = eew_list[1];
-        let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)) in {
+        let Name = op # eew # "_v", IRName = op, MaskedIRName = op # "_mask" in {
           def: RVVOutOp0Op1Builtin<"v", "vPCe" # eew_type # "Uv", type>;
             if !not(IsFloat<type>.val) then {
               def: RVVOutOp0Op1Builtin<"Uv", "UvPCUe" # eew_type # "Uv", type>;
@@ -129,9 +126,7 @@ multiclass RVVIndexedLoad<string op> {
       defvar eew64 = "64";
       defvar eew64_type = "(Log2EEW:6)";
       let Name = op # eew64 # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin", "64bit"],
-                                 !if(!eq(type, "y"), ["zvfbfmin", "64bit"],
-                                                 ["64bit"])) in {
+          RequiredFeatures = ["64bit"] in {
           def: RVVOutOp0Op1Builtin<"v", "vPCe" # eew64_type # "Uv", type>;
             if !not(IsFloat<type>.val) then {
               def: RVVOutOp0Op1Builtin<"Uv", "UvPCUe" # eew64_type # "Uv", type>;
@@ -223,10 +218,7 @@ multiclass RVVIndexedStore<string op> {
         foreach eew_list = EEWList[0-2] in {
           defvar eew = eew_list[0];
           defvar eew_type = eew_list[1];
-          let Name = op # eew  # "_v", IRName = op, MaskedIRName = op # "_mask",
-          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                 !if(!eq(type, "y"), ["zvfbfmin"],
-                                                 []<string>)) in  {
+          let Name = op # eew  # "_v", IRName = op, MaskedIRName = op # "_mask" in {
             def : RVVBuiltin<"v", "0Pe" # eew_type # "Uvv", type>;
             if !not(IsFloat<type>.val) then {
               def : RVVBuiltin<"Uv", "0PUe" # eew_type # "UvUv", type>;
@@ -236,9 +228,7 @@ multiclass RVVIndexedStore<string op> {
         defvar eew64 = "64";
         defvar eew64_type = "(Log2EEW:6)";
         let Name = op # eew64  # "_v", IRName = op, MaskedIRName = op # "_mask",
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin", "64bit"],
-                                   !if(!eq(type, "y"), ["zvfbfmin", "64bit"],
-                                                   ["64bit"])) in  {
+            RequiredFeatures = ["64bit"] in {
           def : RVVBuiltin<"v", "0Pe" # eew64_type # "Uvv", type>;
           if !not(IsFloat<type>.val) then {
             def : RVVBuiltin<"Uv", "0PUe" # eew64_type # "UvUv", type>;
@@ -361,15 +351,7 @@ multiclass RVVNonTupleVCreateBuiltin<int dst_lmul, list<int> src_lmul_list> {
     defvar src_s = FixedVString<src_lmul, num, "v">.S;
     def vcreate # src_v # dst_v : RVVBuiltin<src_v # dst_v,
                                              dst_v # src_s,
-                                             "csilfd">;
-    let RequiredFeatures = ["zvfhmin"] in
-      def vcreate_h # src_v # dst_v : RVVBuiltin<src_v # dst_v,
-                                                 dst_v # src_s,
-                                                 "x", dst_v>;
-    let RequiredFeatures = ["zvfbfmin"] in
-      def vcreate_bf16 # src_v # dst_v : RVVBuiltin<src_v # dst_v,
-                                                    dst_v # src_s,
-                                                    "y", dst_v>;
+                                             "csilxfdy">;
 
     defvar src_uv = FixedVString<src_lmul, num, "Uv">.V;
     defvar src_us = FixedVString<src_lmul, num, "Uv">.S;
@@ -688,40 +670,24 @@ let HasBuiltinAlias = false,
 // 7.4. Vector Unit-Stride Instructions
 def vlm: RVVVLEMaskBuiltin;
 defm vle8: RVVVLEBuiltin<["c"]>;
-defm vle16: RVVVLEBuiltin<["s"]>;
-let Name = "vle16_v", RequiredFeatures = ["zvfhmin"] in
-  defm vle16_h: RVVVLEBuiltin<["x"]>;
-let Name = "vle16_v", RequiredFeatures = ["zvfbfmin"] in
-  defm vle16_bf16 : RVVVLEBuiltin<["y"]>;
+defm vle16: RVVVLEBuiltin<["s","x","y"]>;
 defm vle32: RVVVLEBuiltin<["i","f"]>;
 defm vle64: RVVVLEBuiltin<["l","d"]>;
 
 def vsm : RVVVSEMaskBuiltin;
 defm vse8 : RVVVSEBuiltin<["c"]>;
-defm vse16: RVVVSEBuiltin<["s"]>;
-let Name = "vse16_v", RequiredFeatures = ["zvfhmin"] in
-  defm vse16_h: RVVVSEBuiltin<["x"]>;
-let Name = "vse16_v", RequiredFeatures = ["zvfbfmin"] in
-  defm vse16_bf16: RVVVSEBuiltin<["y"]>;
+defm vse16: RVVVSEBuiltin<["s","x","y"]>;
 defm vse32: RVVVSEBuiltin<["i","f"]>;
 defm vse64: RVVVSEBuiltin<["l","d"]>;
 
 // 7.5. Vector Strided Instructions
 defm vlse8: RVVVLSEBuiltin<["c"]>;
-defm vlse16: RVVVLSEBuiltin<["s"]>;
-let Name = "vlse16_v", RequiredFeatures = ["zvfhmin"] in
-  defm vlse16_h: RVVVLSEBuiltin<["x"]>;
-let Name = "vlse16_v", RequiredFeatures = ["zvfbfmin"] in
-  defm vlse16_bf16: RVVVLSEBuiltin<["y"]>;
+defm vlse16: RVVVLSEBuiltin<["s","x","y"]>;
 defm vlse32: RVVVLSEBuiltin<["i","f"]>;
 defm vlse64: RVVVLSEBuiltin<["l","d"]>;
 
 defm vsse8 : RVVVSSEBuiltin<["c"]>;
-defm vsse16: RVVVSSEBuiltin<["s"]>;
-let Name = "vsse16_v", RequiredFeatures = ["zvfhmin"] in
-  defm vsse16_h: RVVVSSEBuiltin<["x"]>;
-let Name = "vsse16_v", RequiredFeatures = ["zvfbfmin"] in
-  defm vsse16_bf: RVVVSSEBuiltin<["y"]>;
+defm vsse16: RVVVSSEBuiltin<["s","x","y"]>;
 defm vsse32: RVVVSSEBuiltin<["i","f"]>;
 defm vsse64: RVVVSSEBuiltin<["l","d"]>;
 
@@ -734,11 +700,7 @@ defm : RVVIndexedStore<"vsoxei">;
 
 // 7.7. Unit-stride Fault-Only-First Loads
 defm vle8ff: RVVVLEFFBuiltin<["c"]>;
-defm vle16ff: RVVVLEFFBuiltin<["s"]>;
-let Name = "vle16ff_v", RequiredFeatures = ["zvfhmin"] in
-  defm vle16ff: RVVVLEFFBuiltin<["x"]>;
-let Name = "vle16ff_v", RequiredFeatures = ["zvfbfmin"] in
-  defm vle16ff: RVVVLEFFBuiltin<["y"]>;
+defm vle16ff: RVVVLEFFBuiltin<["s","x","y"]>;
 defm vle32ff: RVVVLEFFBuiltin<["i", "f"]>;
 defm vle64ff: RVVVLEFFBuiltin<["l", "d"]>;
 
@@ -757,9 +719,6 @@ multiclass RVVUnitStridedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       SmallVector<llvm::Value*, 6> Operands;
@@ -824,9 +783,6 @@ multiclass RVVUnitStridedSegStoreTuple<string op> {
           MaskedIRName = op # nf # "_mask",
           NF = nf,
           HasMaskedOffOperand = false,
-          RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                 !if(!eq(type, "y"), ["zvfbfmin"],
-                                                 []<string>)),
           ManualCodegen = [{
     {
       // Masked
@@ -878,9 +834,6 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> {
             IRName = op # nf # "ff",
             MaskedIRName = op # nf # "ff_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       SmallVector<llvm::Value*, 6> Operands;
@@ -954,9 +907,6 @@ multiclass RVVStridedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       SmallVector<llvm::Value*, 7> Operands;
@@ -1023,9 +973,6 @@ multiclass RVVStridedSegStoreTuple<string op> {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       // Masked
@@ -1073,9 +1020,6 @@ multiclass RVVIndexedSegLoadTuple<string op> {
             IRName = op # nf,
             MaskedIRName = op # nf # "_mask",
             NF = nf,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       SmallVector<llvm::Value*, 7> Operands;
@@ -1142,9 +1086,6 @@ multiclass RVVIndexedSegStoreTuple<string op> {
             NF = nf,
             HasMaskedOffOperand = false,
             MaskedPolicyScheme = NonePolicy,
-            RequiredFeatures = !if(!eq(type, "x"), ["zvfhmin"],
-                                   !if(!eq(type, "y"), ["zvfbfmin"],
-                                                   []<string>)),
             ManualCodegen = [{
     {
       // Masked
@@ -1385,14 +1326,8 @@ let HasMasked = false,
     OverloadedName = "vmv_v" in {
     defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csil",
                                    [["v", "Uv", "UvUv"]]>;
-    defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilfd",
+    defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "csilxfdy",
                                    [["v", "v", "vv"]]>;
-    let RequiredFeatures = ["zvfhmin"] in
-      defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "x",
-                                    [["v", "v", "vv"]]>;
-    let RequiredFeatures = ["zvfbfmin"] in
-      defm vmv_v : RVVOutBuiltinSet<"vmv_v_v", "y",
-                                    [["v", "v", "vv"]]>;
   let SupportOverloading = false in
     defm vmv_v : RVVOutBuiltinSet<"vmv_v_x", "csil",
                                    [["x", "v", "ve"],
@@ -1816,7 +1751,7 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, round_mode, vl)
     // Masked:   (passthru, op0, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
+    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -1925,14 +1860,8 @@ let HasMasked = false,
         Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
       IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()};
     }] in {
-  defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "fd",
+  defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "xfdy",
                                     [["vvm", "v", "vvvm"]]>;
-  let RequiredFeatures = ["zvfhmin"] in
-    defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "x",
-                                      [["vvm", "v", "vvvm"]]>;
-  let RequiredFeatures = ["zvfbfmin"] in
-    defm vmerge : RVVOutOp1BuiltinSet<"vmerge", "y",
-                                      [["vvm", "v", "vvvm"]]>;
   defm vfmerge : RVVOutOp1BuiltinSet<"vfmerge", "fd",
                                      [["vfm", "v", "vvem"]]>;
   let RequiredFeatures = ["zvfh"] in
@@ -2013,7 +1942,7 @@ let Log2LMUL = [-3, -2, -1, 0, 1, 2] in {
 }
 
 // Zvfbfmin - Vector convert BF16 to FP32
-let Log2LMUL = [-2, -1, 0, 1, 2] in
+let RequiredFeatures = ["zvfbfmin"], Log2LMUL = [-2, -1, 0, 1, 2] in
 def vfwcvtbf16_f_f_v : RVVConvBuiltin<"Fw", "Fwv", "y", "vfwcvtbf16_f">;
 
 let ManualCodegen = [{
@@ -2021,7 +1950,7 @@ let ManualCodegen = [{
     // LLVM intrinsic
     // Unmasked: (passthru, op0, frm, vl)
     // Masked:   (passthru, op0, mask, frm, vl, policy)
-    SmallVector<llvm::Value*, 7> Operands;
+    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -2121,7 +2050,8 @@ let ManualCodegen = [{
     }
 
     // Zvfbfmin - Vector convert FP32 to BF16
-    let Log2LMUL = [-2, -1, 0, 1, 2],
+    let RequiredFeatures = ["zvfbfmin"],
+        Log2LMUL = [-2, -1, 0, 1, 2],
         OverloadedName = "vfncvtbf16_f" in
     defm : RVVConvBuiltinSet<"vfncvtbf16_f_f_w", "y", [["v", "vFwu"]]>;
   }
@@ -2225,7 +2155,7 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
+    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -2379,26 +2309,11 @@ defm vfslide1down : RVVFloatingBinVFBuiltinSet;
 
 // 16.4. Vector Register Gather Instructions
 // signed and floating type
-defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilfd",
+defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csilxfdy",
                                  [["vv", "v", "vvUv"]]>;
-defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilfd",
+defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "csilxfdy",
                                  [["vx", "v", "vvz"]]>;
-let RequiredFeatures = ["zvfhmin"] in {
-  defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "x",
-                                   [["vv", "v", "vvUv"]]>;
-  defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "x",
-                                   [["vx", "v", "vvz"]]>;
-}
-let RequiredFeatures = ["zvfbfmin"] in {
-  defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "y",
-                                   [["vv", "v", "vvUv"]]>;
-  defm vrgather : RVVOutBuiltinSet<"vrgather_vx", "y",
-                                   [["vx", "v", "vvz"]]>;
-}
-defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilfd",
-                                     [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
-let RequiredFeatures = ["zvfhmin"] in
-defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "x",
+defm vrgatherei16 : RVVOutBuiltinSet<"vrgatherei16_vv", "csilxfd",
                                      [["vv", "v", "vv(Log2EEW:4)Uv"]]>;
 // unsigned type
 defm vrgather : RVVOutBuiltinSet<"vrgather_vv", "csil",
@@ -2420,14 +2335,8 @@ let HasMasked = false,
       IntrinsicTypes = {ResultType, Ops.back()->getType()};
     }] in {
   // signed and floating type
-  defm vcompress : RVVOutBuiltinSet<"vcompress", "csilfd",
+  defm vcompress : RVVOutBuiltinSet<"vcompress", "csilxfdy",
                                     [["vm", "v", "vvm"]]>;
-  let RequiredFeatures = ["zvfhmin"] in
-    defm vcompress : RVVOutBuiltinSet<"vcompress", "x",
-                                      [["vm", "v", "vvm"]]>;
-  let RequiredFeatures = ["zvfbfmin"] in
-    defm vcompress : RVVOutBuiltinSet<"vcompress", "y",
-                                      [["vm", "v", "vvm"]]>;
   // unsigned type
   defm vcompress : RVVOutBuiltinSet<"vcompress", "csil",
                                     [["vm", "Uv", "UvUvm"]]>;
@@ -2471,23 +2380,15 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       }] in {
     // Reinterpret between different type under the same SEW and LMUL
     def vreinterpret_i_u : RVVBuiltin<"Uvv", "vUv", "csil", "v">;
-    def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "il", "v">;
+    def vreinterpret_i_f : RVVBuiltin<"Fvv", "vFv", "sil", "v">;
     def vreinterpret_u_i : RVVBuiltin<"vUv", "Uvv", "csil", "Uv">;
-    def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "il", "Uv">;
-    def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "il", "Fv">;
-    def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "il", "Fv">;
-    let RequiredFeatures = ["zvfhmin"] in {
-      def vreinterpret_i_h : RVVBuiltin<"Fvv", "vFv", "s", "v">;
-      def vreinterpret_u_h : RVVBuiltin<"FvUv", "UvFv", "s", "Uv">;
-      def vreinterpret_h_i : RVVBuiltin<"vFv", "Fvv", "s", "Fv">;
-      def vreinterpret_h_u : RVVBuiltin<"UvFv", "FvUv", "s", "Fv">;
-    }
-    let RequiredFeatures = ["zvfbfmin"] in {
-      def vreinterpret_i_bf16 : RVVBuiltin<"vIv", "Ivv", "y", "Iv">;
-      def vreinterpret_u_bf16 : RVVBuiltin<"vUv", "Uvv", "y", "Uv">;
-      def vreinterpret_bf16_i : RVVBuiltin<"Ivv", "vIv", "y", "v">;
-      def vreinterpret_bf16_u : RVVBuiltin<"Uvv", "vUv", "y", "v">;
-    }
+    def vreinterpret_u_f : RVVBuiltin<"FvUv", "UvFv", "sil", "Uv">;
+    def vreinterpret_f_i : RVVBuiltin<"vFv", "Fvv", "sil", "Fv">;
+    def vreinterpret_f_u : RVVBuiltin<"UvFv", "FvUv", "sil", "Fv">;
+    def vreinterpret_i_bf16 : RVVBuiltin<"vIv", "Ivv", "y", "Iv">;
+    def vreinterpret_u_bf16 : RVVBuiltin<"vUv", "Uvv", "y", "Uv">;
+    def vreinterpret_bf16_i : RVVBuiltin<"Ivv", "vIv", "y", "v">;
+    def vreinterpret_bf16_u : RVVBuiltin<"Uvv", "vUv", "y", "v">;
 
     // Reinterpret between different SEW under the same LMUL
     foreach dst_sew = ["(FixedSEW:8)", "(FixedSEW:16)", "(FixedSEW:32)",
@@ -2551,21 +2452,13 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       ManualCodegen = [{
         return llvm::PoisonValue::get(ResultType);
       }] in {
-    def vundefined : RVVBuiltin<"v", "v", "csilfd">;
-    let RequiredFeatures = ["zvfhmin"] in
-      def vundefined_h : RVVBuiltin<"v", "v", "x">;
-    let RequiredFeatures = ["zvfbfmin"] in
-      def vundefined_bf16 : RVVBuiltin<"v", "v", "y">;
+    def vundefined : RVVBuiltin<"v", "v", "csilxfdy">;
     def vundefined_u : RVVBuiltin<"Uv", "Uv", "csil">;
 
     foreach nf = NFList in {
       let NF = nf in {
         defvar T = "(Tuple:" # nf # ")";
-        def : RVVBuiltin<T # "v", T # "v", "csilfd">;
-        let RequiredFeatures = ["zvfhmin"] in
-          def : RVVBuiltin<T # "v", T # "v", "x">;
-        let RequiredFeatures = ["zvfbfmin"] in
-          def : RVVBuiltin<T # "v", T # "v", "y">;
+        def : RVVBuiltin<T # "v", T # "v", "csilxfdy">;
         def : RVVBuiltin<T # "Uv", T # "Uv", "csil">;
       }
     }
@@ -2583,13 +2476,7 @@ let HasMasked = false, HasVL = false, IRName = "" in {
     foreach dst_lmul = ["(SFixedLog2LMUL:-3)", "(SFixedLog2LMUL:-2)", "(SFixedLog2LMUL:-1)",
                         "(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
       def vlmul_trunc # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                              dst_lmul # "vv", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def vlmul_trunc_h # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                                  dst_lmul # "vv", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def vlmul_trunc_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                                     dst_lmul # "vv", "y", dst_lmul # "v">;
+                                              dst_lmul # "vv", "csilxfdy", dst_lmul # "v">;
       def vlmul_trunc_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
                                                 dst_lmul # "UvUv", "csil", dst_lmul # "Uv">;
     }
@@ -2607,13 +2494,7 @@ let HasMasked = false, HasVL = false, IRName = "" in {
     foreach dst_lmul = ["(LFixedLog2LMUL:-2)", "(LFixedLog2LMUL:-1)", "(LFixedLog2LMUL:-0)",
                         "(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
       def vlmul_ext # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                            dst_lmul # "vv", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def vlmul_ext_h # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                                dst_lmul # "vv", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def vlmul_ext_bf16 # dst_lmul : RVVBuiltin<"v" # dst_lmul # "v",
-                                                   dst_lmul # "vv", "y", dst_lmul # "v">;
+                                            dst_lmul # "vv", "csilxfdy", dst_lmul # "v">;
       def vlmul_ext_u # dst_lmul : RVVBuiltin<"Uv" # dst_lmul # "Uv",
                                               dst_lmul # "UvUv", "csil", dst_lmul # "Uv">;
     }
@@ -2642,20 +2523,12 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       }
       }] in {
     foreach dst_lmul = ["(SFixedLog2LMUL:0)", "(SFixedLog2LMUL:1)", "(SFixedLog2LMUL:2)"] in {
-      def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilfd", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "x", dst_lmul # "v">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "y", dst_lmul # "v">;
+      def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "vvKz", "csilxfdy", dst_lmul # "v">;
       def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "UvUvKz", "csil", dst_lmul # "Uv">;
     }
     foreach nf = NFList in {
       defvar T = "(Tuple:" # nf # ")";
-      def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilfd", "v">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def : RVVBuiltin<T # "vv", "v" # T # "vKz", "x", "v">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def : RVVBuiltin<T # "vv", "v" # T # "vKz", "y", "v">;
+      def : RVVBuiltin<T # "vv", "v" # T # "vKz", "csilxfdy", "v">;
       def : RVVBuiltin<T # "UvUv", "Uv" # T # "UvKz", "csil", "Uv">;
     }
   }
@@ -2683,20 +2556,12 @@ let HasMasked = false, HasVL = false, IRName = "" in {
       }
       }] in {
     foreach dst_lmul = ["(LFixedLog2LMUL:1)", "(LFixedLog2LMUL:2)", "(LFixedLog2LMUL:3)"] in {
-      def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilfd">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "x">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "y">;
+      def : RVVBuiltin<"v" # dst_lmul # "v", dst_lmul # "v" # dst_lmul # "vKzv", "csilxfdy">;
       def : RVVBuiltin<"Uv" # dst_lmul # "Uv", dst_lmul # "Uv" # dst_lmul #"UvKzUv", "csil">;
     }
     foreach nf = NFList in {
       defvar T = "(Tuple:" # nf # ")";
-      def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilfd">;
-      let RequiredFeatures = ["zvfhmin"] in
-        def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "x">;
-      let RequiredFeatures = ["zvfbfmin"] in
-        def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "y">;
+      def : RVVBuiltin<"v" # T # "v", T # "v" # T # "vKzv", "csilxfdy">;
       def : RVVBuiltin<"Uv" # T # "Uv", T # "Uv" # T # "UvKzUv", "csil">;
     }
   }
@@ -2741,11 +2606,7 @@ let HasMasked = false, HasVL = false, IRName = "" in {
         defvar T = "(Tuple:" # nf # ")";
         defvar V = VString<nf, /*signed=*/true>.S;
         defvar UV = VString<nf, /*signed=*/false>.S;
-        def : RVVBuiltin<T # "v", T # "v" # V, "csilfd">;
-        let RequiredFeatures = ["zvfhmin"] in
-          def : RVVBuiltin<T # "v", T # "v" # V, "x">;
-        let RequiredFeatures = ["zvfbfmin"] in
-          def : RVVBuiltin<T # "v", T # "v" # V, "y">;
+        def : RVVBuiltin<T # "v", T # "v" # V, "csilxfdy">;
         def : RVVBuiltin<T # "Uv", T # "Uv" # UV, "csil">;
       }
     }
diff --git a/clang/include/clang/Basic/riscv_vector_common.td b/clang/include/clang/Basic/riscv_vector_common.td
index e3d589699645..2148f6f10342 100644
--- a/clang/include/clang/Basic/riscv_vector_common.td
+++ b/clang/include/clang/Basic/riscv_vector_common.td
@@ -591,11 +591,8 @@ class RVVMaskOp0Builtin<string prototype> : RVVOp0Builtin<"m", prototype, "c"> {
 let UnMaskedPolicyScheme = HasPolicyOperand,
     HasMaskedOffOperand = false in {
   multiclass RVVSlideUpBuiltinSet {
-    defm "" : RVVOutBuiltinSet<NAME, "csilfd",
+    defm "" : RVVOutBuiltinSet<NAME, "csilxfd",
                                [["vx","v", "vvvz"]]>;
-    let RequiredFeatures = ["zvfhmin"] in
-      defm "" : RVVOutBuiltinSet<NAME, "x",
-                                 [["vx","v", "vvvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
                                [["vx","Uv", "UvUvUvz"]]>;
   }
@@ -616,11 +613,8 @@ let UnMaskedPolicyScheme = HasPassthruOperand,
       IntrinsicTypes = {ResultType, Ops.back()->getType()};
     }] in {
   multiclass RVVSlideDownBuiltinSet {
-    defm "" : RVVOutBuiltinSet<NAME, "csilfd",
+    defm "" : RVVOutBuiltinSet<NAME, "csilxfd",
                                [["vx","v", "vvz"]]>;
-    let RequiredFeatures = ["zvfhmin"] in
-      defm "" : RVVOutBuiltinSet<NAME, "x",
-                                 [["vx","v", "vvz"]]>;
     defm "" : RVVOutBuiltinSet<NAME, "csil",
                                [["vx","Uv", "UvUvz"]]>;
   }
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 3e052c564112..277c278fd38b 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -63,7 +63,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value getConstAPInt(mlir::Location loc, mlir::Type typ,
                             const llvm::APInt &val) {
-    return create<cir::ConstantOp>(loc, getAttr<cir::IntAttr>(typ, val));
+    return create<cir::ConstantOp>(loc, cir::IntAttr::get(typ, val));
   }
 
   cir::ConstantOp getConstant(mlir::Location loc, mlir::TypedAttr attr) {
@@ -228,25 +228,28 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee,
                            mlir::Type returnType, mlir::ValueRange operands,
-                           cir::SideEffect sideEffect = cir::SideEffect::All) {
-    return create<cir::CallOp>(loc, callee, returnType, operands, sideEffect);
+                           llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
+    auto op = create<cir::CallOp>(loc, callee, returnType, operands);
+    op->setAttrs(attrs);
+    return op;
   }
 
   cir::CallOp createCallOp(mlir::Location loc, cir::FuncOp callee,
                            mlir::ValueRange operands,
-                           cir::SideEffect sideEffect = cir::SideEffect::All) {
+                           llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
     return createCallOp(loc, mlir::SymbolRefAttr::get(callee),
                         callee.getFunctionType().getReturnType(), operands,
-                        sideEffect);
+                        attrs);
   }
 
-  cir::CallOp createIndirectCallOp(mlir::Location loc,
-                                   mlir::Value indirectTarget,
-                                   cir::FuncType funcType,
-                                   mlir::ValueRange operands,
-                                   cir::SideEffect sideEffect) {
-    return create<cir::CallOp>(loc, indirectTarget, funcType.getReturnType(),
-                               operands, sideEffect);
+  cir::CallOp
+  createIndirectCallOp(mlir::Location loc, mlir::Value indirectTarget,
+                       cir::FuncType funcType, mlir::ValueRange operands,
+                       llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
+    llvm::SmallVector<mlir::Value> resOperands{indirectTarget};
+    resOperands.append(operands.begin(), operands.end());
+    return createCallOp(loc, mlir::SymbolRefAttr(), funcType.getReturnType(),
+                        resOperands, attrs);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
index 64556bcb92ba..925a9a87e267 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.h
@@ -10,16 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
-#define LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
-
-#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
-#include "clang/CIR/Dialect/IR/CIRTypes.h"
+#ifndef CLANG_CIR_DIALECT_IR_CIRATTRS_H
+#define CLANG_CIR_DIALECT_IR_CIRATTRS_H
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 
-#include "llvm/ADT/SmallVector.h"
+#include "clang/CIR/Dialect/IR/CIROpsEnums.h"
+
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 
 //===----------------------------------------------------------------------===//
 // CIR Dialect Attrs
@@ -27,15 +26,22 @@
 
 namespace clang {
 class FunctionDecl;
-class VarDecl;
 class RecordDecl;
+class VarDecl;
 } // namespace clang
 
 namespace cir {
 class ArrayType;
+class BoolType;
+class ComplexType;
+class IntType;
+class MethodType;
+class PointerType;
+class RecordType;
+class VectorType;
 } // namespace cir
 
 #define GET_ATTRDEF_CLASSES
 #include "clang/CIR/Dialect/IR/CIROpsAttributes.h.inc"
 
-#endif // LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_H
+#endif // CLANG_CIR_DIALECT_IR_CIRATTRS_H
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 03e970db2847..75650beec0c6 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -65,6 +65,54 @@ class CIRUnitAttr<string name, string attrMnemonic, list<Trait> traits = []>
   let isOptional = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// OptInfoAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_OptInfoAttr : CIR_Attr<"OptInfo", "opt_info"> {
+  let summary =
+    "A module-level attribute that holds the optimization information";
+  let description = [{
+    The `#cir.opt_info` attribute holds optimization related information. For
+    now this attribute is a module-level attribute that gets attached to the
+    module operation during CIRGen.
+
+    The `level` parameter gives the optimization level. It must be an integer
+    between 0 and 3, inclusive. It corresponds to the `OptimizationLevel` field
+    within the `clang::CodeGenOptions` structure.
+
+    The `size` parameter gives the code size optimization level. It must be an
+    integer between 0 and 2, inclusive. It corresponds to the `OptimizeSize`
+    field within the `clang::CodeGenOptions` structure.
+
+    The `level` and `size` parameters correspond to the optimization level
+    command line options passed to clang driver. The table below lists the
+    current correspondance relationship:
+
+    | Flag             | `level` | `size` |
+    |------------------|---------|--------|
+    | `-O0` or nothing | 0       | 0      |
+    | `-O1`            | 1       | 0      |
+    | `-O2`            | 2       | 0      |
+    | `-O3`            | 3       | 0      |
+    | `-Os`            | 2       | 1      |
+    | `-Oz`            | 2       | 2      |
+
+    Examples:
+
+    ```mlir
+    #cir.opt_info<level = 2, size = 0>  // -O2
+    ```
+  }];
+
+  let parameters = (ins "unsigned":$level, "unsigned":$size);
+
+  let assemblyFormat = [{
+    `<` struct(params) `>`
+  }];
+  let genVerifyDecl = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // BoolAttr
 //===----------------------------------------------------------------------===//
@@ -117,71 +165,101 @@ def UndefAttr : CIR_TypedAttr<"Undef", "undef"> {
 // IntegerAttr
 //===----------------------------------------------------------------------===//
 
-def IntAttr : CIR_Attr<"Int", "int", [TypedAttrInterface]> {
+def CIR_IntAttr : CIR_Attr<"Int", "int", [TypedAttrInterface]> {
   let summary = "An attribute containing an integer value";
   let description = [{
     An integer attribute is a literal attribute that represents an integral
     value of the specified integer type.
   }];
-  let parameters = (ins AttributeSelfTypeParameter<"">:$type,
-                        APIntParameter<"">:$value);
+
+  let parameters = (ins
+    AttributeSelfTypeParameter<"", "cir::IntTypeInterface">:$type,
+    APIntParameter<"">:$value
+  );
+
   let builders = [
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "const llvm::APInt &":$value), [{
-      return $_get(type.getContext(), type, value);
+      auto intType = mlir::cast<cir::IntTypeInterface>(type);
+      return $_get(type.getContext(), intType, value);
     }]>,
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "int64_t":$value), [{
-      IntType intType = mlir::cast<IntType>(type);
+      auto intType = mlir::cast<cir::IntTypeInterface>(type);
       mlir::APInt apValue(intType.getWidth(), value, intType.isSigned());
       return $_get(intType.getContext(), intType, apValue);
     }]>,
   ];
+
   let extraClassDeclaration = [{
-    int64_t getSInt() const { return getValue().getSExtValue(); }
-    uint64_t getUInt() const { return getValue().getZExtValue(); }
-    bool isNullValue() const { return getValue() == 0; }
-    uint64_t getBitWidth() const {
-      return mlir::cast<IntType>(getType()).getWidth();
+    int64_t getSInt() const;
+    uint64_t getUInt() const;
+    bool isNullValue() const;
+    bool isSigned() const;
+    bool isUnsigned() const;
+    uint64_t getBitWidth() const;
+  }];
+
+ let extraClassDefinition = [{
+    int64_t $cppClass::getSInt() const {
+      return getValue().getSExtValue();
+    }
+    uint64_t $cppClass::getUInt() const {
+      return getValue().getZExtValue();
+    }
+    bool $cppClass::isNullValue() const {
+      return getValue() == 0;
     }
+    bool $cppClass::isSigned() const {
+      return mlir::cast<IntTypeInterface>(getType()).isSigned();
+    }
+    bool $cppClass::isUnsigned() const {
+      return mlir::cast<IntTypeInterface>(getType()).isUnsigned();
+    }
+    uint64_t $cppClass::getBitWidth() const {
+      return mlir::cast<IntTypeInterface>(getType()).getWidth();
+    }
+  }];
+
+  let assemblyFormat = [{
+    `<` custom<IntLiteral>($value, ref($type)) `>`
   }];
+
   let genVerifyDecl = 1;
-  let hasCustomAssemblyFormat = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // FPAttr
 //===----------------------------------------------------------------------===//
 
-def FPAttr : CIR_Attr<"FP", "fp", [TypedAttrInterface]> {
+def CIR_FPAttr : CIR_Attr<"FP", "fp", [TypedAttrInterface]> {
   let summary = "An attribute containing a floating-point value";
   let description = [{
     An fp attribute is a literal attribute that represents a floating-point
     value of the specified floating-point type. Supporting only CIR FP types.
   }];
+
   let parameters = (ins
-    AttributeSelfTypeParameter<"", "::cir::CIRFPTypeInterface">:$type,
+    AttributeSelfTypeParameter<"", "::cir::FPTypeInterface">:$type,
     APFloatParameter<"">:$value
   );
+
   let builders = [
     AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
                                         "const llvm::APFloat &":$value), [{
-      return $_get(type.getContext(), mlir::cast<CIRFPTypeInterface>(type),
-                   value);
-    }]>,
-    AttrBuilder<(ins "mlir::Type":$type,
-                     "const llvm::APFloat &":$value), [{
-      return $_get($_ctxt, mlir::cast<CIRFPTypeInterface>(type), value);
-    }]>,
+      return $_get(type.getContext(), mlir::cast<FPTypeInterface>(type), value);
+    }]>
   ];
+
   let extraClassDeclaration = [{
     static FPAttr getZero(mlir::Type type);
   }];
-  let genVerifyDecl = 1;
 
   let assemblyFormat = [{
     `<` custom<FloatLiteral>($value, ref($type)) `>`
   }];
+
+  let genVerifyDecl = 1;
 }
 
 
@@ -375,4 +453,67 @@ def CIR_VisibilityAttr : CIR_EnumAttr<CIR_VisibilityKind, "visibility"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// BitfieldInfoAttr
+//===----------------------------------------------------------------------===//
+
+def BitfieldInfoAttr : CIR_Attr<"BitfieldInfo", "bitfield_info"> {
+  let summary = "Represents info for a bit-field member";
+  let description = [{
+    Holds the following information about bitfields: name, storage type, size
+    and position in the storage, and signedness.
+    Example:
+      Given the following struct with bitfields:
+        ```c++
+        typedef struct {
+          int a : 4;
+          int b : 27;
+          int c : 17;
+          int d : 2;
+          int e : 15;
+        } S;
+        ```
+
+      The CIR representation of the struct `S` might look like:
+      ```mlir
+        !rec_S = !cir.record<struct "S" packed padded {!u64i, !u16i,
+                                                       !cir.array<!u8i x 2>}>
+      ```
+      And the bitfield info attribute for member `a` would be:
+      ```mlir
+        #bfi_a = #cir.bitfield_info<name = "a", storage_type = !u64i,
+                                    size = 4, offset = 0, is_signed = true>
+      ```
+
+      This metadata describes that field `a` is stored in a 64-bit integer,
+      is 4 bits wide, starts at offset 0, and is signed.
+  }];
+  let parameters = (ins "mlir::StringAttr":$name,
+                        "mlir::Type":$storageType,
+                        "uint64_t":$size,
+                        "uint64_t":$offset,
+                        "bool":$isSigned);
+
+  let assemblyFormat = [{`<` struct($name,
+                                    $storageType,
+                                    $size,
+                                    $offset,
+                                    $isSigned)
+                         `>`
+                        }];
+
+  let builders = [
+    AttrBuilder<(ins "llvm::StringRef":$name,
+                     "mlir::Type":$storageType,
+                     "uint64_t":$size,
+                     "uint64_t":$offset,
+                     "bool":$isSigned
+                     ), [{
+      return $_get($_ctxt, mlir::StringAttr::get($_ctxt, name), storageType,
+                   size, offset, isSigned);
+    }]>
+  ];
+}
+
+
 #endif // LLVM_CLANG_CIR_DIALECT_IR_CIRATTRS_TD
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDialect.h b/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
index 5de1722cf5bc..fdd56ac1f218 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDialect.h
@@ -29,6 +29,7 @@
 #include "clang/CIR/Dialect/IR/CIRAttrs.h"
 #include "clang/CIR/Dialect/IR/CIROpsDialect.h.inc"
 #include "clang/CIR/Dialect/IR/CIROpsEnums.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
 #include "clang/CIR/Interfaces/CIRLoopOpInterface.h"
 #include "clang/CIR/Interfaces/CIROpInterfaces.h"
 #include "clang/CIR/MissingFeatures.h"
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
index 52e32eedf774..fa73e5bf41f2 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
@@ -36,6 +36,10 @@ def CIR_Dialect : Dialect {
 
   let extraClassDeclaration = [{
     static llvm::StringRef getTripleAttrName() { return "cir.triple"; }
+    static llvm::StringRef getOptInfoAttrName() { return "cir.opt_info"; }
+    static llvm::StringRef getCalleeAttrName() { return "callee"; }
+    static llvm::StringRef getNoThrowAttrName() { return "nothrow"; }
+    static llvm::StringRef getSideEffectAttrName() { return "side_effect"; }
 
     void registerAttributes();
     void registerTypes();
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index d2eeeafc60ba..6529f1386599 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -489,7 +489,7 @@ def ReturnOp : CIR_Op<"return", [ParentOneOf<["FuncOp", "ScopeOp", "IfOp",
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
@@ -1610,7 +1610,7 @@ def GlobalOp : CIR_Op<"global",
                        CIR_GlobalLinkageKind:$linkage,
                        OptionalAttr<AnyAttr>:$initial_value,
                        UnitAttr:$comdat,
-                       UnitAttr:$dsolocal,
+                       UnitAttr:$dso_local,
                        OptionalAttr<I64Attr>:$alignment);
 
   let assemblyFormat = [{
@@ -1618,7 +1618,7 @@ def GlobalOp : CIR_Op<"global",
     (`` $global_visibility^)?
     $linkage
     (`comdat` $comdat^)?
-    (`dsolocal` $dsolocal^)?
+    (`dso_local` $dso_local^)?
     $sym_name
     custom<GlobalOpTypeAndInitialValue>($sym_type, $initial_value)
     attr-dict
@@ -1669,6 +1669,85 @@ def GetGlobalOp : CIR_Op<"get_global",
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// GetBitfieldOp
+//===----------------------------------------------------------------------===//
+
+def GetBitfieldOp : CIR_Op<"get_bitfield"> {
+  let summary = "Get the information for a bitfield member";
+  let description = [{
+    The `cir.get_bitfield` operation provides a load-like access to
+    a bit field of a record.
+
+    It expects a name if a bit field, a pointer to a storage in the
+    base record, a type of the storage, a name of the bitfield,
+    a size the bit field, an offset of the bit field and a sign.
+
+    A unit attribute `volatile` can be used to indicate a volatile load of the
+    bitfield.
+
+    Example:
+    Suppose we have a struct with multiple bitfields stored in
+    different members. The `cir.get_bitfield` operation gets the value
+    of the bitfield.
+    ```C++
+    typedef struct {
+      int a : 4;
+      int b : 27;
+      int c : 17;
+      int d : 2;
+      int e : 15;
+    } S;
+
+    int load_bitfield(S& s) {
+      return s.e;
+    }
+    ```
+
+    ```mlir
+    // 'e' is in the storage with the index 1
+    !cir.record<struct "S" packed padded {!u64i, !u16i, !cir.array<!u8i x 2>}>
+    #bfi_e = #cir.bitfield_info<name = "e", storage_type = !u16i, size = 15,
+                                offset = 0, is_signed = true>
+
+    %2 = cir.load %0 : !cir.ptr<!cir.ptr<!record_type>>, !cir.ptr<!record_type>
+    %3 = cir.get_member %2[1] {name = "e"} : !cir.ptr<!record_type>
+                                                             -> !cir.ptr<!u16i>
+    %4 = cir.get_bitfield(#bfi_e, %3 : !cir.ptr<!u16i>) -> !s32i
+    ```
+    }];
+
+  let arguments = (ins
+    Arg<CIR_PointerType, "the address to load from", [MemRead]>:$addr,
+    BitfieldInfoAttr:$bitfield_info,
+    UnitAttr:$is_volatile
+    );
+
+  let results = (outs CIR_IntType:$result);
+
+  let assemblyFormat = [{ `(`$bitfield_info `,` $addr attr-dict `:`
+   qualified(type($addr)) `)` `->` type($result) }];
+
+  let builders = [
+    OpBuilder<(ins "mlir::Type":$type,
+                   "mlir::Value":$addr,
+                   "mlir::Type":$storage_type,
+                   "llvm::StringRef":$name,
+                   "unsigned":$size,
+                   "unsigned":$offset,
+                   "bool":$is_signed,
+                   "bool":$is_volatile
+                   ),
+   [{
+      BitfieldInfoAttr info =
+        BitfieldInfoAttr::get($_builder.getContext(),
+                              name, storage_type,
+                              size, offset, is_signed);
+      build($_builder, $_state, type, addr, info, is_volatile);
+    }]>
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // GetMemberOp
 //===----------------------------------------------------------------------===//
@@ -1737,25 +1816,52 @@ def GetMemberOp : CIR_Op<"get_member"> {
 
 def FuncOp : CIR_Op<"func", [
   AutomaticAllocationScope, CallableOpInterface, FunctionOpInterface,
+  DeclareOpInterfaceMethods<CIRGlobalValueInterface>,
   IsolatedFromAbove
 ]> {
   let summary = "Declare or define a function";
   let description = [{
     The `cir.func` operation defines a function, similar to the `mlir::FuncOp`
     built-in.
+
+    The function linkage information is specified by `linkage`, as defined by
+    `GlobalLinkageKind` attribute.
+
+    Example:
+
+    ```mlir
+    // External function definitions.
+    cir.func @abort()
+
+    // A function with internal linkage.
+    cir.func internal @count(%x: i64) -> (i64)
+      return %x : i64
+
+    // Linkage information
+    cir.func linkonce_odr @some_method(...)
+    ```
   }];
 
   let arguments = (ins SymbolNameAttr:$sym_name,
+                       CIR_VisibilityAttr:$global_visibility,
                        TypeAttrOf<CIR_FuncType>:$function_type,
+                       UnitAttr:$dso_local,
+                       DefaultValuedAttr<CIR_GlobalLinkageKind,
+                                         "cir::GlobalLinkageKind::ExternalLinkage">:$linkage,
+                       OptionalAttr<StrAttr>:$sym_visibility,
+                       UnitAttr:$comdat,
                        OptionalAttr<DictArrayAttr>:$arg_attrs,
-                       OptionalAttr<DictArrayAttr>:$res_attrs);
+                       OptionalAttr<DictArrayAttr>:$res_attrs,
+                       OptionalAttr<FlatSymbolRefAttr>:$aliasee);
 
   let regions = (region AnyRegion:$body);
 
   let skipDefaultBuilders = 1;
 
-  let builders = [OpBuilder<(ins "llvm::StringRef":$sym_name,
-                                 "FuncType":$type)>];
+  let builders = [OpBuilder<(ins
+    "llvm::StringRef":$sym_name, "FuncType":$type,
+    CArg<"cir::GlobalLinkageKind", "cir::GlobalLinkageKind::ExternalLinkage">:$linkage)
+  >];
 
   let extraClassDeclaration = [{
     /// Returns the region on the current operation that is callable. This may
@@ -1885,6 +1991,7 @@ class CIR_CallOpBase<string mnemonic, list<Trait> extra_traits = []>
 
   dag commonArgs = (ins OptionalAttr<FlatSymbolRefAttr>:$callee,
       Variadic<CIR_AnyType>:$args,
+      UnitAttr:$nothrow,
       DefaultValuedAttr<CIR_SideEffect, "SideEffect::All">:$side_effect);
 }
 
@@ -1914,29 +2021,14 @@ def CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   let arguments = commonArgs;
 
   let builders = [
-    // Build a call op for a direct call
     OpBuilder<(ins "mlir::SymbolRefAttr":$callee, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands,
-                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
-      assert(callee && "callee attribute is required for direct call");
+                   "mlir::ValueRange":$operands), [{
       $_state.addOperands(operands);
-      $_state.addAttribute("callee", callee);
-      $_state.addAttribute("side_effect",
-        SideEffectAttr::get($_builder.getContext(), sideEffect));
+      if (callee)
+        $_state.addAttribute("callee", callee);
       if (resType && !isa<VoidType>(resType))
         $_state.addTypes(resType);
-    }]>,
-    // Build a call op for an indirect call
-    OpBuilder<(ins "mlir::Value":$calleePtr, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands,
-                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
-      $_state.addOperands(calleePtr);
-      $_state.addOperands(operands);
-      if (resType && !isa<VoidType>(resType))
-        $_state.addTypes(resType);
-      $_state.addAttribute("side_effect",
-        SideEffectAttr::get($_builder.getContext(), sideEffect));
-    }]>,
+    }]>
   ];
 }
 
@@ -2371,6 +2463,204 @@ def ComplexCreateOp : CIR_Op<"complex.create", [Pure, SameTypeOperands]> {
   let hasFolder = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexRealOp
+//===----------------------------------------------------------------------===//
+
+def ComplexRealOp : CIR_Op<"complex.real", [Pure]> {
+  let summary = "Extract the real part of a complex value";
+  let description = [{
+    `cir.complex.real` operation takes an operand of `!cir.complex` type and
+    yields the real part of it.
+
+    Example:
+
+    ```mlir
+    %1 = cir.complex.real %0 : !cir.complex<!cir.float> -> !cir.float
+    ```
+  }];
+
+  let results = (outs CIR_AnyIntOrFloatType:$result);
+  let arguments = (ins CIR_ComplexType:$operand);
+
+  let assemblyFormat = [{
+    $operand `:` qualified(type($operand)) `->` qualified(type($result))
+    attr-dict
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// ComplexImagOp
+//===----------------------------------------------------------------------===//
+
+def ComplexImagOp : CIR_Op<"complex.imag", [Pure]> {
+  let summary = "Extract the imaginary part of a complex value";
+  let description = [{
+    `cir.complex.imag` operation takes an operand of `!cir.complex` type and
+    yields the imaginary part of it.
+
+    Example:
+
+    ```mlir
+    %1 = cir.complex.imag %0 : !cir.complex<!cir.float> -> !cir.float
+    ```
+  }];
+
+  let results = (outs CIR_AnyIntOrFloatType:$result);
+  let arguments = (ins CIR_ComplexType:$operand);
+
+  let assemblyFormat = [{
+    $operand `:` qualified(type($operand)) `->` qualified(type($result))
+    attr-dict
+  }];
+
+  let hasVerifier = 1;
+  let hasFolder = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Bit Manipulation Operations
+//===----------------------------------------------------------------------===//
+
+class CIR_BitOpBase<string mnemonic, TypeConstraint operandTy>
+  : CIR_Op<mnemonic, [Pure, SameOperandsAndResultType]> {
+  let arguments = (ins operandTy:$input);
+  let results = (outs operandTy:$result);
+
+  let assemblyFormat = [{
+    `(` $input `:` type($input) `)` `:` type($result) attr-dict
+  }];
+}
+
+class CIR_BitZeroCountOpBase<string mnemonic, TypeConstraint operandTy>
+  : CIR_BitOpBase<mnemonic, operandTy> {
+  let arguments = (ins operandTy:$input, UnitAttr:$poison_zero);
+
+  let assemblyFormat = [{
+    `(` $input `:` type($input) `)` (`poison_zero` $poison_zero^)?
+    `:` type($result) attr-dict
+  }];
+}
+
+def BitClrsbOp : CIR_BitOpBase<"bit.clrsb", CIR_SIntOfWidths<[32, 64]>> {
+  let summary = "Get the number of leading redundant sign bits in the input";
+  let description = [{
+    Compute the number of leading redundant sign bits in the input integer.
+
+    The input integer must be a signed integer. The most significant bit of the
+    input integer is the sign bit. The `cir.bit.clrsb` operation returns the
+    number of consecutive bits following the sign bit that are identical to the
+    sign bit.
+
+    The bit width of the input integer must be either 32 or 64.
+
+    Examples:
+
+    ```mlir
+    // %0 = 0b1101_1110_1010_1101_1011_1110_1110_1111
+    %0 = cir.const #cir.int<3735928559> : !s32i
+    // %1 will be 1 because there is 1 bit following the most significant bit
+    // that is identical to it.
+    %1 = cir.bit.clrsb(%0 : !s32i) : !s32i
+
+    // %2 = 1, 0b0000_0000_0000_0000_0000_0000_0000_0001
+    %2 = cir.const #cir.int<1> : !s32i
+    // %3 will be 30 because there are 30 consecutive bits following the sign
+    // bit that are identical to the sign bit.
+    %3 = cir.bit.clrsb(%2 : !s32i) : !s32i
+    ```
+  }];
+}
+
+def BitClzOp : CIR_BitZeroCountOpBase<"bit.clz",
+                                      CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of leading 0-bits in the input";
+  let description = [{
+    Compute the number of leading 0-bits in the input.
+
+    The input integer must be an unsigned integer. The `cir.bit.clz` operation
+    returns the number of consecutive 0-bits at the most significant bit
+    position in the input.
+
+    If the `poison_zero` attribute is present, this operation will have
+    undefined behavior if the input value is 0.
+
+    Example:
+
+    ```mlir
+    // %0 = 0b0000_0000_0000_0000_0000_0000_0000_1000
+    %0 = cir.const #cir.int<8> : !u32i
+    // %1 will be 28
+    %1 = cir.bit.clz(%0 : !u32i) poison_zero : !u32i
+    ```
+  }];
+}
+
+def BitCtzOp : CIR_BitZeroCountOpBase<"bit.ctz",
+                                      CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of trailing 0-bits in the input";
+  let description = [{
+    Compute the number of trailing 0-bits in the input.
+
+    The input integer must be an unsigned integer. The `cir.bit.ctz` operation
+    counts the number of consecutive 0-bits starting from the least significant
+    bit.
+
+    If the `poison_zero` attribute is present, this operation will have
+    undefined behavior if the input value is 0.
+
+    Example:
+
+    ```mlir
+    // %0 = 0b1000
+    %0 = cir.const #cir.int<8> : !u32i
+    // %1 will be 3
+    %1 = cir.bit.ctz(%0 : !u32i) poison_zero : !u32i
+    ```
+  }];
+}
+
+def BitParityOp : CIR_BitOpBase<"bit.parity", CIR_UIntOfWidths<[32, 64]>> {
+  let summary = "Get the parity of input";
+  let description = [{
+    Compute the parity of the input. The parity of an integer is the number of
+    1-bits in it modulo 2.
+
+    The input must be an unsigned integer.
+
+    Example:
+
+    ```mlir
+    // %0 = 0x0110_1000
+    %0 = cir.const #cir.int<104> : !u32i
+    // %1 will be 1 since there are three 1-bits in %0
+    %1 = cir.bit.parity(%0 : !u32i) : !u32i
+    ```
+  }];
+}
+
+def BitPopcountOp : CIR_BitOpBase<"bit.popcnt",
+                                  CIR_UIntOfWidths<[16, 32, 64]>> {
+  let summary = "Get the number of 1-bits in input";
+  let description = [{
+    Compute the number of 1-bits in the input.
+
+    The input must be an unsigned integer.
+
+    Example:
+
+    ```mlir
+    // %0 = 0x0110_1000
+    %0 = cir.const #cir.int<104> : !u32i
+    // %1 will be 3 since there are 3 1-bits in %0
+    %1 = cir.bit.popcnt(%0 : !u32i) : !u32i
+    ```
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Assume Operations
 //===----------------------------------------------------------------------===//
@@ -2393,4 +2683,41 @@ def AssumeOp : CIR_Op<"assume"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Branch Probability Operations
+//===----------------------------------------------------------------------===//
+
+def ExpectOp : CIR_Op<"expect",
+  [Pure, AllTypesMatch<["result", "val", "expected"]>]> {
+  let summary = "Tell the optimizer that two values are likely to be equal.";
+  let description = [{
+    The `cir.expect` operation may take 2 or 3 arguments.
+
+    When the argument `prob` is missing, this operation effectively models the
+    `__builtin_expect` builtin function. It tells the optimizer that `val` and
+    `expected` are likely to be equal.
+
+    When the argument `prob` is present, this operation effectively models the
+    `__builtin_expect_with_probability` builtin function. It tells the
+    optimizer that `val` and `expected` are equal to each other with a certain
+    probability.
+
+    `val` and `expected` must be integers and their types must match.
+
+    The result of this operation is always equal to `val`.
+  }];
+
+  let arguments = (ins
+    CIR_AnyFundamentalIntType:$val,
+    CIR_AnyFundamentalIntType:$expected,
+    OptionalAttr<F64Attr>:$prob
+  );
+
+  let results = (outs CIR_AnyFundamentalIntType:$result);
+
+  let assemblyFormat = [{
+    `(` $val`,` $expected (`,` $prob^)? `)` `:` type($val) attr-dict
+  }];
+}
+
 #endif // CLANG_CIR_DIALECT_IR_CIROPS_TD
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.h b/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
index 49933be724a0..7f9fb9ef388b 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.h
@@ -16,7 +16,7 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
-#include "clang/CIR/Interfaces/CIRFPTypeInterface.h"
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 
 namespace cir {
 
@@ -26,6 +26,15 @@ struct RecordTypeStorage;
 
 bool isValidFundamentalIntWidth(unsigned width);
 
+/// Returns true if the type is a CIR sized type.
+///
+/// Types are sized if they implement SizedTypeInterface and
+/// return true from its method isSized.
+///
+/// Unsized types are those that do not have a size, such as
+/// void, or abstract types.
+bool isSized(mlir::Type ty);
+
 } // namespace cir
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index 41d7d725a09e..898c26d22f6d 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -15,7 +15,7 @@
 
 include "clang/CIR/Dialect/IR/CIRDialect.td"
 include "clang/CIR/Dialect/IR/CIRTypeConstraints.td"
-include "clang/CIR/Interfaces/CIRFPTypeInterface.td"
+include "clang/CIR/Interfaces/CIRTypeInterfaces.td"
 include "mlir/Interfaces/DataLayoutInterfaces.td"
 include "mlir/IR/AttrTypeBase.td"
 
@@ -33,8 +33,11 @@ class CIR_Type<string name, string typeMnemonic, list<Trait> traits = [],
 // IntType
 //===----------------------------------------------------------------------===//
 
-def CIR_IntType : CIR_Type<"Int", "int",
-    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
+def CIR_IntType : CIR_Type<"Int", "int", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_IntTypeInterface>,
+]> {
   let summary = "Integer type with arbitrary precision up to a fixed limit";
   let description = [{
     CIR type that represents integer types with arbitrary precision, including
@@ -82,7 +85,8 @@ def CIR_IntType : CIR_Type<"Int", "int",
 
 class CIR_FloatType<string name, string mnemonic> : CIR_Type<name, mnemonic, [
   DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
-  DeclareTypeInterfaceMethods<CIRFPTypeInterface>
+  DeclareTypeInterfaceMethods<CIR_FPTypeInterface>,
+  DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>
 ]>;
 
 def CIR_Single : CIR_FloatType<"Single", "float"> {
@@ -165,9 +169,10 @@ def CIR_LongDouble : CIR_FloatType<"LongDouble", "long_double"> {
 // ComplexType
 //===----------------------------------------------------------------------===//
 
-def CIR_ComplexType : CIR_Type<"Complex", "complex",
-    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
-
+def CIR_ComplexType : CIR_Type<"Complex", "complex", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>
+]> {
   let summary = "CIR complex type";
   let description = [{
     CIR type that represents a C complex number. `cir.complex` models the C type
@@ -215,12 +220,13 @@ def CIR_ComplexType : CIR_Type<"Complex", "complex",
 // PointerType
 //===----------------------------------------------------------------------===//
 
-def CIR_PointerType : CIR_Type<"Pointer", "ptr",
-    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
-
+def CIR_PointerType : CIR_Type<"Pointer", "ptr", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>
+]> {
   let summary = "CIR pointer type";
   let description = [{
-    The `cir.ptr` type represents C and C++ pointer types and C++ reference
+    The `!cir.ptr` type represents C and C++ pointer types and C++ reference
     types, other than pointers-to-members.  The `pointee` type is the type
     pointed to.
 
@@ -279,13 +285,13 @@ def CIR_PointerType : CIR_Type<"Pointer", "ptr",
 // BoolType
 //===----------------------------------------------------------------------===//
 
-def CIR_BoolType :
-    CIR_Type<"Bool", "bool",
-             [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
-
+def CIR_BoolType : CIR_Type<"Bool", "bool", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>
+]> {
   let summary = "CIR bool type";
   let description = [{
-    `cir.bool` represents C++ bool type.
+    `!cir.bool` represents C++ bool type.
   }];
 }
 
@@ -293,12 +299,13 @@ def CIR_BoolType :
 // ArrayType
 //===----------------------------------------------------------------------===//
 
-def CIR_ArrayType : CIR_Type<"Array", "array",
-    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
-
+def CIR_ArrayType : CIR_Type<"Array", "array", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface, ["isSized"]>,
+]> {
   let summary = "CIR array type";
   let description = [{
-    `CIR.array` represents C/C++ constant arrays.
+    `!cir.array` represents C/C++ constant arrays.
   }];
 
   let parameters = (ins "mlir::Type":$elementType, "uint64_t":$size);
@@ -314,15 +321,22 @@ def CIR_ArrayType : CIR_Type<"Array", "array",
   let assemblyFormat = [{
     `<` $elementType `x` $size `>`
   }];
+
+  let extraClassDefinition = [{
+    bool $cppClass::isSized() const {
+      return ::cir::isSized(getElementType());
+    }
+  }];
 }
 
 //===----------------------------------------------------------------------===//
 // VectorType (fixed size)
 //===----------------------------------------------------------------------===//
 
-def CIR_VectorType : CIR_Type<"Vector", "vector",
-    [DeclareTypeInterfaceMethods<DataLayoutTypeInterface>]> {
-
+def CIR_VectorType : CIR_Type<"Vector", "vector", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface, ["isSized"]>,
+]> {
   let summary = "CIR vector type";
   let description = [{
     The `!cir.vector` type represents a fixed-size, one-dimensional vector.
@@ -363,6 +377,12 @@ def CIR_VectorType : CIR_Type<"Vector", "vector",
     }]>,
   ];
 
+  let extraClassDefinition = [{
+    bool $cppClass::isSized() const {
+      return ::cir::isSized(getElementType());
+    }
+  }];
+
   let genVerifyDecl = 1;
 }
 
@@ -459,11 +479,11 @@ def CIR_VoidType : CIR_Type<"Void", "void"> {
 // The base type for all RecordDecls.
 //===----------------------------------------------------------------------===//
 
-def CIR_RecordType : CIR_Type<"Record", "record",
-    [
-      DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
-      MutableType,
-    ]> {
+def CIR_RecordType : CIR_Type<"Record", "record", [
+    DeclareTypeInterfaceMethods<DataLayoutTypeInterface>,
+    DeclareTypeInterfaceMethods<CIR_SizedTypeInterface>,
+    MutableType,
+]> {
   let summary = "CIR record type";
   let description = [{
     Each unique clang::RecordDecl is mapped to a `cir.record` and any object in
diff --git a/clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.td b/clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.td
deleted file mode 100644
index 973851b61444..000000000000
--- a/clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.td
+++ /dev/null
@@ -1,56 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines the interface to generically handle CIR floating-point types.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_TD
-#define LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_TD
-
-include "mlir/IR/OpBase.td"
-
-def CIRFPTypeInterface : TypeInterface<"CIRFPTypeInterface"> {
-  let description = [{
-    Contains helper functions to query properties about a floating-point type.
-  }];
-  let cppNamespace = "::cir";
-
-  let methods = [
-    InterfaceMethod<[{
-        Returns the bit width of this floating-point type.
-      }],
-      /*retTy=*/"unsigned",
-      /*methodName=*/"getWidth",
-      /*args=*/(ins),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{
-          return llvm::APFloat::semanticsSizeInBits($_type.getFloatSemantics());
-        }]
-    >,
-    InterfaceMethod<[{
-        Return the mantissa width.
-      }],
-      /*retTy=*/"unsigned",
-      /*methodName=*/"getFPMantissaWidth",
-      /*args=*/(ins),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{
-          return llvm::APFloat::semanticsPrecision($_type.getFloatSemantics());
-        }]
-    >,
-    InterfaceMethod<[{
-        Return the float semantics of this floating-point type.
-      }],
-      /*retTy=*/"const llvm::fltSemantics &",
-      /*methodName=*/"getFloatSemantics"
-    >,
-  ];
-}
-
-#endif // LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_TD
diff --git a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
index 203e42f7c575..4702b4ef08ac 100644
--- a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
+++ b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
@@ -34,6 +34,8 @@ let cppNamespace = "::cir" in {
           "Return the number of operands, accounts for indirect call or "
           "exception info",
           "unsigned", "getNumArgOperands", (ins)>,
+      InterfaceMethod<"Return whether the callee is nothrow",
+                      "bool", "getNothrow", (ins)>,
       InterfaceMethod<"Return the side effects of the call operation",
                       "cir::SideEffect", "getSideEffect", (ins)>,
     ];
@@ -138,13 +140,13 @@ let cppNamespace = "::cir" in {
       InterfaceMethod<"",
       "void", "setDSOLocal", (ins "bool":$val), [{}],
       /*defaultImplementation=*/[{
-        $_op.setDsolocal(val);
+        $_op.setDsoLocal(val);
       }]
       >,
       InterfaceMethod<"",
       "bool", "isDSOLocal", (ins), [{}],
       /*defaultImplementation=*/[{
-        return $_op.getDsolocal();
+        return $_op.getDsoLocal();
       }]
       >,
       InterfaceMethod<"",
diff --git a/clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.h b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.h
similarity index 62%
rename from clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.h
rename to clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.h
index 40b85ef6cfb6..d5d2f5a7fa65 100644
--- a/clang/include/clang/CIR/Interfaces/CIRFPTypeInterface.h
+++ b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.h
@@ -6,17 +6,17 @@
 //
 //===---------------------------------------------------------------------===//
 //
-// Defines the interface to generically handle CIR floating-point types.
+// Defines cir type interfaces.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_H
-#define LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_H
+#ifndef CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_H
+#define CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_H
 
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APFloat.h"
 
 /// Include the tablegen'd interface declarations.
-#include "clang/CIR/Interfaces/CIRFPTypeInterface.h.inc"
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h.inc"
 
-#endif // LLVM_CLANG_INCLUDE_CLANG_CIR_INTERFACES_CIRFPTYPEINTERFACE_H
+#endif // CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_H
diff --git a/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
new file mode 100644
index 000000000000..cf6c8571ddcd
--- /dev/null
+++ b/clang/include/clang/CIR/Interfaces/CIRTypeInterfaces.td
@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines cir type interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_TD
+#define CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_TD
+
+include "mlir/IR/OpBase.td"
+
+def CIR_IntTypeInterface : TypeInterface<"IntTypeInterface"> {
+  let description = [{
+    Contains helper functions to query properties about an integer type.
+  }];
+  let cppNamespace = "::cir";
+  let methods = [
+    InterfaceMethod<[{
+        Returns true if this is a signed integer type.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isSigned",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.isSigned();
+      }]
+    >,
+    InterfaceMethod<[{
+        Returns true if this is an unsigned integer type.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isUnsigned",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.isUnsigned();
+      }]
+    >,
+    InterfaceMethod<[{
+        Returns the bit width of this integer type.
+      }],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getWidth",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_type.getWidth();
+      }]
+    >
+  ];
+}
+
+def CIR_FPTypeInterface : TypeInterface<"FPTypeInterface"> {
+  let description = [{
+    Contains helper functions to query properties about a floating-point type.
+  }];
+  let cppNamespace = "::cir";
+
+  let methods = [
+    InterfaceMethod<[{
+        Returns the bit width of this floating-point type.
+      }],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getWidth",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+          return llvm::APFloat::semanticsSizeInBits($_type.getFloatSemantics());
+        }]
+    >,
+    InterfaceMethod<[{
+        Return the mantissa width.
+      }],
+      /*retTy=*/"unsigned",
+      /*methodName=*/"getFPMantissaWidth",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+          return llvm::APFloat::semanticsPrecision($_type.getFloatSemantics());
+        }]
+    >,
+    InterfaceMethod<[{
+        Return the float semantics of this floating-point type.
+      }],
+      /*retTy=*/"const llvm::fltSemantics &",
+      /*methodName=*/"getFloatSemantics"
+    >,
+  ];
+}
+
+def CIR_SizedTypeInterface : TypeInterface<"SizedTypeInterface"> {
+  let description = [{
+    Annotates types that have known size. Types that don't have a size are
+    abstract types and void.
+  }];
+  let cppNamespace = "::cir";
+  let methods = [
+    InterfaceMethod<[{
+        Returns true if this is a sized type. This mirrors sizedness from the
+        clang AST, where a type is sized if it has a known size.
+        By default type defining this interface returns true,
+        but this can be overridden if sizedness depends on properties of the type.
+        For example, whether a struct is not sized if it is incomplete.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isSized",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return true;
+      }]
+    >,
+  ];
+}
+
+#endif // CLANG_CIR_INTERFACES_CIRTYPEINTERFACES_TD
diff --git a/clang/include/clang/CIR/Interfaces/CMakeLists.txt b/clang/include/clang/CIR/Interfaces/CMakeLists.txt
index 3c155193235d..bc8d94ff9dc5 100644
--- a/clang/include/clang/CIR/Interfaces/CMakeLists.txt
+++ b/clang/include/clang/CIR/Interfaces/CMakeLists.txt
@@ -21,4 +21,4 @@ endfunction()
 
 add_clang_mlir_op_interface(CIROpInterfaces)
 add_clang_mlir_op_interface(CIRLoopOpInterface)
-add_clang_mlir_type_interface(CIRFPTypeInterface)
+add_clang_mlir_type_interface(CIRTypeInterfaces)
diff --git a/clang/include/clang/CIR/LoweringHelpers.h b/clang/include/clang/CIR/LoweringHelpers.h
index 3077010ee5ff..66e99c7e8441 100644
--- a/clang/include/clang/CIR/LoweringHelpers.h
+++ b/clang/include/clang/CIR/LoweringHelpers.h
@@ -37,4 +37,18 @@ std::optional<mlir::Attribute>
 lowerConstArrayAttr(cir::ConstArrayAttr constArr,
                     const mlir::TypeConverter *converter);
 
+mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc,
+                          mlir::Type typ, const llvm::APInt &val);
+
+mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ,
+                     unsigned val);
+
+mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
+
+mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
+
+mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs,
+                      const llvm::APInt &rhs);
+
+mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs);
 #endif
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index d8e45d02cd2a..c76737549a0f 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -72,16 +72,17 @@ struct MissingFeatures {
 
   // FuncOp handling
   static bool opFuncOpenCLKernelMetadata() { return false; }
+  static bool opFuncAstDeclAttr() { return false; }
   static bool opFuncCallingConv() { return false; }
   static bool opFuncExtraAttrs() { return false; }
-  static bool opFuncDsolocal() { return false; }
-  static bool opFuncLinkage() { return false; }
-  static bool opFuncVisibility() { return false; }
   static bool opFuncNoProto() { return false; }
   static bool opFuncCPUAndFeaturesAttributes() { return false; }
   static bool opFuncSection() { return false; }
-  static bool opFuncSetComdat() { return false; }
+  static bool opFuncMultipleReturnVals() { return false; }
   static bool opFuncAttributesForDefinition() { return false; }
+  static bool opFuncMaybeHandleStaticInExternC() { return false; }
+  static bool setLLVMFunctionFEnvAttributes() { return false; }
+  static bool setFunctionAttributes() { return false; }
 
   // CallOp handling
   static bool opCallPseudoDtor() { return false; }
@@ -109,6 +110,9 @@ struct MissingFeatures {
   static bool opCallLandingPad() { return false; }
   static bool opCallContinueBlock() { return false; }
 
+  // CXXNewExpr
+  static bool exprNewNullCheck() { return false; }
+
   // FnInfoOpts -- This is used to track whether calls are chain calls or
   // instance methods. Classic codegen uses chain call to track and extra free
   // register for x86 and uses instance method as a condition for a thunk
@@ -157,90 +161,95 @@ struct MissingFeatures {
   static bool addressPointerAuthInfo() { return false; }
 
   // Misc
-  static bool cirgenABIInfo() { return false; }
   static bool abiArgInfo() { return false; }
-  static bool tryEmitAsConstant() { return false; }
-  static bool constructABIArgDirectExtend() { return false; }
-  static bool opGlobalViewAttr() { return false; }
-  static bool lowerModeOptLevel() { return false; }
-  static bool opTBAA() { return false; }
-  static bool objCLifetime() { return false; }
-  static bool objCBlocks() { return false; }
-  static bool emitNullabilityCheck() { return false; }
-  static bool emitLValueAlignmentAssumption() { return false; }
-  static bool emitLifetimeMarkers() { return false; }
-  static bool astVarDeclInterface() { return false; }
-  static bool stackSaveOp() { return false; }
+  static bool addHeapAllocSiteMetadata() { return false; }
   static bool aggValueSlot() { return false; }
-  static bool aggValueSlotMayOverlap() { return false; }
-  static bool aggValueSlotVolatile() { return false; }
-  static bool aggValueSlotDestructedFlag() { return false; }
   static bool aggValueSlotAlias() { return false; }
+  static bool aggValueSlotDestructedFlag() { return false; }
   static bool aggValueSlotGC() { return false; }
-  static bool generateDebugInfo() { return false; }
-  static bool pointerOverflowSanitizer() { return false; }
-  static bool fpConstraints() { return false; }
-  static bool sanitizers() { return false; }
-  static bool addHeapAllocSiteMetadata() { return false; }
-  static bool targetCIRGenInfoArch() { return false; }
-  static bool targetCIRGenInfoOS() { return false; }
-  static bool targetCodeGenInfoGetNullPointer() { return false; }
-  static bool loopInfoStack() { return false; }
-  static bool requiresCleanups() { return false; }
-  static bool createProfileWeightsForLoop() { return false; }
-  static bool emitCondLikelihoodViaExpectIntrinsic() { return false; }
-  static bool pgoUse() { return false; }
-  static bool cgFPOptionsRAII() { return false; }
-  static bool metaDataNode() { return false; }
-  static bool fastMathFlags() { return false; }
+  static bool aggValueSlotMayOverlap() { return false; }
+  static bool aggValueSlotVolatile() { return false; }
   static bool alignCXXRecordDecl() { return false; }
-  static bool setNonGC() { return false; }
-  static bool incrementProfileCounter() { return false; }
-  static bool insertBuiltinUnpredictable() { return false; }
-  static bool objCGC() { return false; }
-  static bool weakRefReference() { return false; }
-  static bool hip() { return false; }
-  static bool setObjCGCLValueClass() { return false; }
-  static bool setDLLStorageClass() { return false; }
-  static bool openMP() { return false; }
-  static bool emitCheckedInBoundsGEP() { return false; }
-  static bool preservedAccessIndexRegion() { return false; }
+  static bool armComputeVolatileBitfields() { return false; }
+  static bool asmLabelAttr() { return false; }
+  static bool assignMemcpyizer() { return false; }
+  static bool astVarDeclInterface() { return false; }
+  static bool attributeBuiltin() { return false; }
+  static bool attributeNoBuiltin() { return false; }
   static bool bitfields() { return false; }
-  static bool msabi() { return false; }
-  static bool typeChecks() { return false; }
-  static bool lambdaFieldToName() { return false; }
-  static bool moduleNameHash() { return false; }
-  static bool constantFoldSwitchStatement() { return false; }
-  static bool cudaSupport() { return false; }
-  static bool maybeHandleStaticInExternC() { return false; }
+  static bool builtinCall() { return false; }
+  static bool builtinCallF128() { return false; }
+  static bool builtinCallMathErrno() { return false; }
+  static bool builtinCheckKind() { return false; }
+  static bool cgFPOptionsRAII() { return false; }
+  static bool cirgenABIInfo() { return false; }
+  static bool cleanupAfterErrorDiags() { return false; }
+  static bool cleanupsToDeactivate() { return false; }
   static bool constEmitterArrayILE() { return false; }
   static bool constEmitterVectorILE() { return false; }
-  static bool needsGlobalCtorDtor() { return false; }
-  static bool emitTypeCheck() { return false; }
-  static bool writebacks() { return false; }
-  static bool cleanupsToDeactivate() { return false; }
-  static bool stackBase() { return false; }
-  static bool deferredCXXGlobalInit() { return false; }
-  static bool setTargetAttributes() { return false; }
+  static bool constantFoldSwitchStatement() { return false; }
+  static bool constructABIArgDirectExtend() { return false; }
   static bool coverageMapping() { return false; }
-  static bool peepholeProtection() { return false; }
-  static bool instrumentation() { return false; }
-  static bool cleanupAfterErrorDiags() { return false; }
+  static bool createProfileWeightsForLoop() { return false; }
+  static bool ctorMemcpyizer() { return false; }
+  static bool cudaSupport() { return false; }
   static bool cxxRecordStaticMembers() { return false; }
-  static bool isMemcpyEquivalentSpecialMember() { return false; }
-  static bool isTrivialCtorOrDtor() { return false; }
+  static bool dataLayoutTypeAllocSize() { return false; }
+  static bool deferredCXXGlobalInit() { return false; }
+  static bool emitCheckedInBoundsGEP() { return false; }
+  static bool emitCondLikelihoodViaExpectIntrinsic() { return false; }
+  static bool emitLifetimeMarkers() { return false; }
+  static bool emitLValueAlignmentAssumption() { return false; }
+  static bool emitNullabilityCheck() { return false; }
+  static bool emitTypeCheck() { return false; }
+  static bool fastMathFlags() { return false; }
+  static bool fpConstraints() { return false; }
+  static bool generateDebugInfo() { return false; }
+  static bool hip() { return false; }
   static bool implicitConstructorArgs() { return false; }
+  static bool incrementProfileCounter() { return false; }
+  static bool insertBuiltinUnpredictable() { return false; }
+  static bool instrumentation() { return false; }
   static bool intrinsics() { return false; }
-  static bool attributeNoBuiltin() { return false; }
-  static bool thunks() { return false; }
-  static bool runCleanupsScope() { return false; }
+  static bool isMemcpyEquivalentSpecialMember() { return false; }
+  static bool isTrivialCtorOrDtor() { return false; }
+  static bool lambdaFieldToName() { return false; }
+  static bool loopInfoStack() { return false; }
   static bool lowerAggregateLoadStore() { return false; }
-  static bool dataLayoutTypeAllocSize() { return false; }
-  static bool asmLabelAttr() { return false; }
-  static bool builtinCall() { return false; }
-  static bool builtinCallF128() { return false; }
-  static bool builtinCallMathErrno() { return false; }
+  static bool lowerModeOptLevel() { return false; }
+  static bool maybeHandleStaticInExternC() { return false; }
+  static bool metaDataNode() { return false; }
+  static bool moduleNameHash() { return false; }
+  static bool msabi() { return false; }
+  static bool needsGlobalCtorDtor() { return false; }
   static bool nonFineGrainedBitfields() { return false; }
+  static bool objCBlocks() { return false; }
+  static bool objCGC() { return false; }
+  static bool objCLifetime() { return false; }
+  static bool openMP() { return false; }
+  static bool opGlobalViewAttr() { return false; }
+  static bool opTBAA() { return false; }
+  static bool peepholeProtection() { return false; }
+  static bool pgoUse() { return false; }
+  static bool pointerOverflowSanitizer() { return false; }
+  static bool preservedAccessIndexRegion() { return false; }
+  static bool requiresCleanups() { return false; }
+  static bool runCleanupsScope() { return false; }
+  static bool sanitizers() { return false; }
+  static bool setDLLStorageClass() { return false; }
+  static bool setNonGC() { return false; }
+  static bool setObjCGCLValueClass() { return false; }
+  static bool setTargetAttributes() { return false; }
+  static bool stackBase() { return false; }
+  static bool stackSaveOp() { return false; }
+  static bool targetCIRGenInfoArch() { return false; }
+  static bool targetCIRGenInfoOS() { return false; }
+  static bool targetCodeGenInfoGetNullPointer() { return false; }
+  static bool thunks() { return false; }
+  static bool tryEmitAsConstant() { return false; }
+  static bool typeChecks() { return false; }
+  static bool weakRefReference() { return false; }
+  static bool writebacks() { return false; }
 
   // Missing types
   static bool dataMemberType() { return false; }
diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index bda73ea61f3b..7ecb8537d6a4 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -33,6 +33,8 @@ void AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
                      const llvm::opt::ArgList &Args,
                      llvm::opt::ArgStringList &CmdArgs, const JobAction &JA);
 
+const char *getLDMOption(const llvm::Triple &T, const llvm::opt::ArgList &Args);
+
 void addLinkerCompressDebugSectionsOption(const ToolChain &TC,
                                           const llvm::opt::ArgList &Args,
                                           llvm::opt::ArgStringList &CmdArgs);
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 6593c862518c..28dcad8a57c9 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -372,10 +372,10 @@ class Driver {
 
   /// Number of parallel jobs.
   unsigned NumParallelJobs;
-  /// Cache of known offloading architectures for the ToolChain already derived.
-  /// This should only be modified when we first initialize the offloading
-  /// toolchains.
-  llvm::DenseMap<const ToolChain *, llvm::DenseSet<llvm::StringRef>> KnownArchs;
+
+  /// The associated offloading architectures with each toolchain.
+  llvm::DenseMap<const ToolChain *, llvm::SmallVector<llvm::StringRef>>
+      OffloadArchs;
 
 private:
   /// TranslateInputArgs - Create a new derived argument list from the input
@@ -545,11 +545,11 @@ class Driver {
 
   /// Returns the set of bound architectures active for this offload kind.
   /// If there are no bound architctures we return a set containing only the
-  /// empty string. The \p SuppressError option is used to suppress errors.
-  llvm::DenseSet<StringRef>
+  /// empty string.
+  llvm::SmallVector<StringRef>
   getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
                   Action::OffloadKind Kind, const ToolChain *TC,
-                  bool SuppressError = false) const;
+                  bool SpecificToolchain = true) const;
 
   /// Check that the file referenced by Value exists. If it doesn't,
   /// issue a diagnostic and return false.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index fcc66446d5d8..81a3eabd5be9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1143,7 +1143,11 @@ def fno_convergent_functions : Flag<["-"], "fno-convergent-functions">,
 
 // Common offloading options
 let Group = offload_Group in {
-def offload_arch_EQ : Joined<["--"], "offload-arch=">,
+def offload_targets_EQ : CommaJoined<["--"], "offload-targets=">,
+  Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+  HelpText<"Specify a list of target architectures to use for offloading.">;
+
+def offload_arch_EQ : CommaJoined<["--"], "offload-arch=">,
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify an offloading device architecture for CUDA, HIP, or OpenMP. (e.g. sm_35). "
            "If 'native' is used the compiler will detect locally installed architectures. "
@@ -1707,6 +1711,8 @@ def fsample_profile_use_profi : Flag<["-"], "fsample-profile-use-profi">,
                basic block counts to branch probabilities to fix them by extended
                and re-engineered classic MCMF (min-cost max-flow) approach.}]>;
 def fno_profile_sample_accurate : Flag<["-"], "fno-profile-sample-accurate">, Group<f_Group>;
+def fno_sample_profile_use_profi : Flag<["-"], "fno-sample-profile-use-profi">,
+                                   Group<f_Group>;
 def fno_auto_profile : Flag<["-"], "fno-auto-profile">, Group<f_Group>,
     Alias<fno_profile_sample_use>;
 def fauto_profile_EQ : Joined<["-"], "fauto-profile=">,
@@ -2677,6 +2683,15 @@ def fsanitize_skip_hot_cutoff_EQ
 
 } // end -f[no-]sanitize* flags
 
+def fallow_runtime_check_skip_hot_cutoff_EQ
+    : Joined<["-"], "fallow-runtime-check-skip-hot-cutoff=">,
+      Group<f_clang_Group>,
+      Visibility<[ClangOption, CC1Option]>,
+      HelpText<"Exclude __builtin_allow_runtime_check for the top hottest "
+               "code responsible for the given fraction of PGO counters "
+               "(0.0 [default] = skip none; 1.0 = skip all). "
+               "Argument format: <value>">;
+
 def funsafe_math_optimizations : Flag<["-"], "funsafe-math-optimizations">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Allow unsafe floating-point math optimizations which may decrease precision">,
@@ -3487,7 +3502,8 @@ def fno_experimental_isel : Flag<["-"], "fno-experimental-isel">, Group<f_clang_
   Alias<fno_global_isel>;
 def fveclib : Joined<["-"], "fveclib=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
-    HelpText<"Use the given vector functions library">,
+    HelpText<"Use the given vector functions library.\n"
+             "  Note: -fveclib=libmvec on AArch64 requires GLIBC 2.40 or newer.">,
     HelpTextForVariants<[ClangOption, CC1Option],
       "Use the given vector functions library.\n"
       "  Note: -fveclib={ArmPL,SLEEF,libmvec} implies -fno-math-errno.\n"
@@ -3681,7 +3697,7 @@ def fopenmp_use_tls : Flag<["-"], "fopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>;
 def fnoopenmp_use_tls : Flag<["-"], "fnoopenmp-use-tls">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
-def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">,
+def fopenmp_targets_EQ : CommaJoined<["-"], "fopenmp-targets=">, Alias<offload_targets_EQ>,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Specify comma-separated list of triples OpenMP offloading targets to be supported">;
 def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
@@ -3957,6 +3973,24 @@ def fms_hotpatch : Flag<["-"], "fms-hotpatch">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, CLOption]>,
   HelpText<"Ensure that all functions can be hotpatched at runtime">,
   MarshallingInfoFlag<CodeGenOpts<"HotPatch">>;
+
+// See llvm/lib/CodeGen/WindowsSecureHotPatching.cpp
+def fms_secure_hotpatch_functions_file
+    : Joined<["-"], "fms-secure-hotpatch-functions-file=">,
+      Group<f_Group>,
+      Visibility<[ClangOption, CC1Option, CLOption]>,
+      MarshallingInfoString<CodeGenOpts<"MSSecureHotPatchFunctionsFile">>,
+      HelpText<"Path to a file that contains a list of mangled names of "
+               "functions that should be hot-patched for Windows Secure "
+               "Hot-Patching">;
+def fms_secure_hotpatch_functions_list
+    : CommaJoined<["-"], "fms-secure-hotpatch-functions-list=">,
+      Group<f_Group>,
+      Visibility<[ClangOption, CC1Option, CLOption]>,
+      MarshallingInfoStringVector<CodeGenOpts<"MSSecureHotPatchFunctionsList">>,
+      HelpText<"List of mangled symbol names of functions that should be "
+               "hot-patched for Windows Secure Hot-Patching">;
+
 def fpcc_struct_return : Flag<["-"], "fpcc-struct-return">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option]>,
   HelpText<"Override the default ABI to return all structs on the stack">;
@@ -5326,6 +5360,14 @@ def msve_vector_bits_EQ : Joined<["-"], "msve-vector-bits=">, Group<m_aarch64_Fe
   Visibility<[ClangOption, FlangOption]>,
   HelpText<"Specify the size in bits of an SVE vector register. Defaults to the"
            " vector length agnostic value of \"scalable\". (AArch64 only)">;
+def msve_streaming_vector_bits_EQ
+    : Joined<["-"], "msve-streaming-vector-bits=">,
+      Group<m_aarch64_Features_Group>,
+      Visibility<[ClangOption, FlangOption]>,
+      HelpText<
+          "Specify the size in bits of an SVE vector register in streaming "
+          "mode. Defaults to the vector length agnostic value of "
+          "\"scalable\". (AArch64 only)">;
 } // let Flags = [TargetSpecific]
 
 def mvscale_min_EQ : Joined<["-"], "mvscale-min=">,
@@ -5337,6 +5379,17 @@ def mvscale_max_EQ : Joined<["-"], "mvscale-max=">,
   HelpText<"Specify the vscale maximum. Defaults to the"
            " vector length agnostic value of \"0\". (AArch64/RISC-V only)">,
   MarshallingInfoInt<LangOpts<"VScaleMax">>;
+def mvscale_streaming_min_EQ
+    : Joined<["-"], "mvscale-streaming-min=">,
+      Visibility<[CC1Option, FC1Option]>,
+      HelpText<"Specify the vscale minimum. Defaults to \"1\". (AArch64 only)">,
+      MarshallingInfoInt<LangOpts<"VScaleStreamingMin">>;
+def mvscale_streaming_max_EQ
+    : Joined<["-"], "mvscale-streaming-max=">,
+      Visibility<[CC1Option, FC1Option]>,
+      HelpText<"Specify the vscale maximum. Defaults to the"
+               " vector length agnostic value of \"0\". (AArch64 only)">,
+      MarshallingInfoInt<LangOpts<"VScaleStreamingMax">>;
 
 def msign_return_address_EQ : Joined<["-"], "msign-return-address=">,
   Visibility<[ClangOption, CC1Option]>,
@@ -9584,6 +9637,20 @@ def fcgl : DXCFlag<"fcgl">, Alias<emit_pristine_llvm>;
 def enable_16bit_types : DXCFlag<"enable-16bit-types">, Alias<fnative_half_type>,
   HelpText<"Enable 16-bit types and disable min precision types."
            "Available in HLSL 2018 and shader model 6.2.">;
+def fdx_rootsignature_version :
+  Joined<["-"], "fdx-rootsignature-version=">,
+  Group<dxc_Group>,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Root Signature Version">,
+  Values<"rootsig_1_0,rootsig_1_1">,
+  NormalizedValuesScope<"llvm::dxbc::RootSignatureVersion">,
+  NormalizedValues<["V1_0", "V1_1"]>,
+  MarshallingInfoEnum<LangOpts<"HLSLRootSigVer">, "V1_1">;
+def dxc_rootsig_ver :
+  Separate<["/", "-"], "force-rootsig-ver">,
+  Alias<fdx_rootsignature_version>,
+  Group<dxc_Group>,
+  Visibility<[DXCOption]>;
 def hlsl_entrypoint : Option<["-"], "hlsl-entry", KIND_SEPARATE>,
                       Group<dxc_Group>,
                       Visibility<[ClangOption, CC1Option]>,
@@ -9602,6 +9669,8 @@ def dxc_validator_path_EQ : Joined<["--"], "dxv-path=">, Group<dxc_Group>,
   HelpText<"DXIL validator installation path">;
 def dxc_disable_validation : DXCFlag<"Vd">,
   HelpText<"Disable validation">;
+def dxc_gis : DXCFlag<"Gis">,
+  HelpText<"Enable IEEE strict mode (equivalent to -ffp-model=strict)">;
 def : Option<["/", "-"], "Qembed_debug", KIND_FLAG>, Group<dxc_Group>,
   Flags<[Ignored]>, Visibility<[DXCOption]>,
   HelpText<"Embed PDB in shader container (ignored)">;
@@ -9616,6 +9685,16 @@ def fspv_extension_EQ
       Group<dxc_Group>,
       HelpText<"Specify the available SPIR-V extensions. If this option is not "
                "specified, then all extensions are available.">;
+def fvk_use_dx_layout
+    : DXCFlag<"fvk-use-dx-layout">,
+      HelpText<"Use DirectX memory layout for Vulkan resources.">;
+def fvk_use_gl_layout : DXCFlag<"fvk-use-gl-layout">,
+                        HelpText<"Use strict OpenGL std140/std430 memory "
+                                 "layout for Vulkan resources.">;
+def fvk_use_scalar_layout
+    : DXCFlag<"fvk-use-scalar-layout">,
+      HelpText<"Use scalar memory layout for Vulkan resources.">;
+
 def no_wasm_opt : Flag<["--"], "no-wasm-opt">,
   Group<m_Group>,
   HelpText<"Disable the wasm-opt optimizer">,
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 2a5cf5fb50db..5d1fdb153b26 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -651,9 +651,9 @@ struct FormatStyle {
   /// \version 3.7
   TrailingCommentsAlignmentStyle AlignTrailingComments;
 
-  /// \brief If a function call or braced initializer list doesn't fit on a
-  /// line, allow putting all arguments onto the next line, even if
-  /// ``BinPackArguments`` is ``false``.
+  /// If a function call or braced initializer list doesn't fit on a line, allow
+  /// putting all arguments onto the next line, even if ``BinPackArguments`` is
+  /// ``false``.
   /// \code
   ///   true:
   ///   callFunction(
@@ -3956,7 +3956,7 @@ struct FormatStyle {
   /// \version 6
   std::vector<RawStringFormat> RawStringFormats;
 
-  /// \brief The ``&`` and ``&&`` alignment style.
+  /// The ``&`` and ``&&`` alignment style.
   enum ReferenceAlignmentStyle : int8_t {
     /// Align reference like ``PointerAlignment``.
     RAS_Pointer,
@@ -3977,13 +3977,12 @@ struct FormatStyle {
     RAS_Middle
   };
 
-  /// \brief Reference alignment style (overrides ``PointerAlignment`` for
-  /// references).
+  /// Reference alignment style (overrides ``PointerAlignment`` for references).
   /// \version 13
   ReferenceAlignmentStyle ReferenceAlignment;
 
   // clang-format off
-  /// \brief Types of comment reflow style.
+  /// Types of comment reflow style.
   enum ReflowCommentsStyle : int8_t {
     /// Leave comments untouched.
     /// \code
@@ -4016,7 +4015,7 @@ struct FormatStyle {
   };
   // clang-format on
 
-  /// \brief Comment reformatting style.
+  /// Comment reformatting style.
   /// \version 3.8
   ReflowCommentsStyle ReflowComments;
 
@@ -4151,9 +4150,8 @@ struct FormatStyle {
   /// \version 16
   bool RemoveSemicolon;
 
-  /// \brief The possible positions for the requires clause. The
-  /// ``IndentRequires`` option is only used if the ``requires`` is put on the
-  /// start of a line.
+  /// The possible positions for the requires clause. The ``IndentRequires``
+  /// option is only used if the ``requires`` is put on the start of a line.
   enum RequiresClausePositionStyle : int8_t {
     /// Always put the ``requires`` clause on its own line (possibly followed by
     /// a semicolon).
@@ -4251,7 +4249,7 @@ struct FormatStyle {
     RCPS_SingleLine,
   };
 
-  /// \brief The position of the ``requires`` clause.
+  /// The position of the ``requires`` clause.
   /// \version 15
   RequiresClausePositionStyle RequiresClausePosition;
 
@@ -4281,7 +4279,7 @@ struct FormatStyle {
   /// \version 16
   RequiresExpressionIndentationKind RequiresExpressionIndentation;
 
-  /// \brief The style if definition blocks should be separated.
+  /// The style if definition blocks should be separated.
   enum SeparateDefinitionStyle : int8_t {
     /// Leave definition blocks as they are.
     SDS_Leave,
@@ -5188,8 +5186,8 @@ struct FormatStyle {
   /// \version 17
   std::vector<std::string> TypeNames;
 
-  /// \brief A vector of macros that should be interpreted as type declarations
-  /// instead of as function calls.
+  /// A vector of macros that should be interpreted as type declarations instead
+  /// of as function calls.
   ///
   /// These are expected to be macros of the form:
   /// \code
diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h
index f8663e3193a1..78dff1165dcf 100644
--- a/clang/include/clang/Interpreter/Interpreter.h
+++ b/clang/include/clang/Interpreter/Interpreter.h
@@ -20,6 +20,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/Support/Error.h"
 #include <memory>
@@ -136,10 +137,14 @@ class Interpreter {
 public:
   virtual ~Interpreter();
   static llvm::Expected<std::unique_ptr<Interpreter>>
-  create(std::unique_ptr<CompilerInstance> CI);
+  create(std::unique_ptr<CompilerInstance> CI,
+         std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder = nullptr);
   static llvm::Expected<std::unique_ptr<Interpreter>>
   createWithCUDA(std::unique_ptr<CompilerInstance> CI,
                  std::unique_ptr<CompilerInstance> DCI);
+  static llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+  createLLJITBuilder(std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC,
+                     llvm::StringRef OrcRuntimePath);
   const ASTContext &getASTContext() const;
   ASTContext &getASTContext();
   const CompilerInstance *getCompilerInstance() const;
diff --git a/clang/include/clang/Interpreter/RemoteJITUtils.h b/clang/include/clang/Interpreter/RemoteJITUtils.h
new file mode 100644
index 000000000000..8705a3b1f669
--- /dev/null
+++ b/clang/include/clang/Interpreter/RemoteJITUtils.h
@@ -0,0 +1,38 @@
+//===-- RemoteJITUtils.h - Utilities for remote-JITing ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for ExecutorProcessControl-based remote JITing with Orc and
+// JITLink.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
+#define LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
+#include "llvm/Support/Error.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
+launchExecutor(llvm::StringRef ExecutablePath, bool UseSharedMemory,
+               llvm::StringRef SlabAllocateSizeString);
+
+/// Create a JITLinkExecutor that connects to the given network address
+/// through a TCP socket. A valid NetworkAddress provides hostname and port,
+/// e.g. localhost:20000.
+llvm::Expected<std::unique_ptr<llvm::orc::SimpleRemoteEPC>>
+connectTCPSocket(llvm::StringRef NetworkAddress, bool UseSharedMemory,
+                 llvm::StringRef SlabAllocateSizeString);
+
+#endif // LLVM_CLANG_INTERPRETER_REMOTEJITUTILS_H
diff --git a/clang/include/clang/Lex/LexHLSLRootSignature.h b/clang/include/clang/Lex/LexHLSLRootSignature.h
index 9275e0d75840..9901485b44d3 100644
--- a/clang/include/clang/Lex/LexHLSLRootSignature.h
+++ b/clang/include/clang/Lex/LexHLSLRootSignature.h
@@ -50,6 +50,10 @@ operator<<(const DiagnosticBuilder &DB, const RootSignatureToken::Kind Kind) {
   case RootSignatureToken::Kind::X:                                            \
     DB << SPELLING;                                                            \
     break;
+#define PUNCTUATOR(X, SPELLING)                                                \
+  case RootSignatureToken::Kind::pu_##X:                                       \
+    DB << #SPELLING;                                                           \
+    break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   }
   return DB;
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 7a178b5b5c85..06971ff87ab9 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -585,6 +585,11 @@ class Lexer : public PreprocessorLexer {
   /// sequence.
   static bool isNewLineEscaped(const char *BufferStart, const char *Str);
 
+  /// getEscapedNewLineSize - Return the size of the specified escaped newline,
+  /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry
+  /// to this function.
+  static unsigned getEscapedNewLineSize(const char *P);
+
   /// Diagnose use of a delimited or named escape sequence.
   static void DiagnoseDelimitedOrNamedEscapeSequence(SourceLocation Loc,
                                                      bool Named,
@@ -725,11 +730,6 @@ class Lexer : public PreprocessorLexer {
   /// method.
   SizedChar getCharAndSizeSlow(const char *Ptr, Token *Tok = nullptr);
 
-  /// getEscapedNewLineSize - Return the size of the specified escaped newline,
-  /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry
-  /// to this function.
-  static unsigned getEscapedNewLineSize(const char *P);
-
   /// SkipEscapedNewLines - If P points to an escaped newline (or a series of
   /// them), skip over them and return the first non-escaped-newline found,
   /// otherwise return P.
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 0ec1cb4d0c5d..4d82e20e5d4f 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -350,8 +350,8 @@ class Preprocessor {
   /// Whether the last token we lexed was an '@'.
   bool LastTokenWasAt = false;
 
-  /// First pp-token in current translation unit.
-  std::optional<Token> FirstPPToken;
+  /// First pp-token source location in current translation unit.
+  SourceLocation FirstPPTokenLoc;
 
   /// A position within a C++20 import-seq.
   class StdCXXImportSeq {
@@ -1769,20 +1769,13 @@ class Preprocessor {
   std::optional<LexEmbedParametersResult> LexEmbedParameters(Token &Current,
                                                              bool ForHasEmbed);
 
-  /// Whether the preprocessor already seen the first pp-token in main file.
-  bool hasSeenMainFileFirstPPToken() const { return FirstPPToken.has_value(); }
-
-  /// Record first pp-token and check if it has a Token::FirstPPToken flag.
-  void HandleMainFileFirstPPToken(const Token &Tok) {
-    if (!hasSeenMainFileFirstPPToken() && Tok.isFirstPPToken() &&
-        SourceMgr.isWrittenInMainFile(Tok.getLocation()))
-      FirstPPToken = Tok;
+  /// Get the start location of the first pp-token in main file.
+  SourceLocation getMainFileFirstPPTokenLoc() const {
+    assert(FirstPPTokenLoc.isValid() &&
+           "Did not see the first pp-token in the main file");
+    return FirstPPTokenLoc;
   }
 
-  Token getMainFileFirstPPToken() const {
-    assert(FirstPPToken && "First main file pp-token doesn't exists");
-    return *FirstPPToken;
-  }
   bool LexAfterModuleImport(Token &Result);
   void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
 
@@ -2304,7 +2297,9 @@ class Preprocessor {
 
   /// Check whether the next pp-token is one of the specificed token kind. this
   /// method should have no observable side-effect on the lexed tokens.
-  template <tok::TokenKind K, tok::TokenKind... Ks> bool isNextPPTokenOneOf() {
+  template <typename... Ts> bool isNextPPTokenOneOf(Ts... Ks) {
+    static_assert(sizeof...(Ts) > 0,
+                  "requires at least one tok::TokenKind specified");
     // Do some quick tests for rejection cases.
     std::optional<Token> Val;
     if (CurLexer)
@@ -2335,7 +2330,7 @@ class Preprocessor {
 
     // Okay, we found the token and return.  Otherwise we found the end of the
     // translation unit.
-    return Val->is(K) || (... || Val->is(Ks));
+    return Val->isOneOf(Ks...);
   }
 
 private:
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index d4dfd7b44d9a..fc43e72593b9 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -101,11 +101,10 @@ class Token {
   /// "if (Tok.is(tok::l_brace)) {...}".
   bool is(tok::TokenKind K) const { return Kind == K; }
   bool isNot(tok::TokenKind K) const { return Kind != K; }
-  bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const {
-    return is(K1) || is(K2);
-  }
-  template <typename... Ts> bool isOneOf(tok::TokenKind K1, Ts... Ks) const {
-    return is(K1) || isOneOf(Ks...);
+  template <typename... Ts> bool isOneOf(Ts... Ks) const {
+    static_assert(sizeof...(Ts) > 0,
+                  "requires at least one tok::TokenKind specified");
+    return (is(Ks) || ...);
   }
 
   /// Return true if this is a raw identifier (when lexing
diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h
index 777b4e6266c7..7ac933d8f9d4 100644
--- a/clang/include/clang/Lex/TokenLexer.h
+++ b/clang/include/clang/Lex/TokenLexer.h
@@ -139,8 +139,9 @@ class TokenLexer {
   void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion,
             bool OwnsTokens, bool IsReinject);
 
-  /// If the next token lexed will pop this macro off the expansion stack,
-  /// return std::nullopt, otherwise return the next unexpanded token.
+  /// If TokenLexer::isAtEnd returns true(the next token lexed will pop this
+  /// macro off the expansion stack), return std::nullopt, otherwise return the
+  /// next unexpanded token.
   std::optional<Token> peekNextPPToken() const;
 
   /// Lex and return a token from this macro stream.
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
index afa2c4d8cfe5..66a5a3b7eaad 100644
--- a/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -27,7 +27,8 @@ namespace hlsl {
 
 class RootSignatureParser {
 public:
-  RootSignatureParser(SmallVector<llvm::hlsl::rootsig::RootElement> &Elements,
+  RootSignatureParser(llvm::dxbc::RootSignatureVersion Version,
+                      SmallVector<llvm::hlsl::rootsig::RootElement> &Elements,
                       RootSignatureLexer &Lexer, clang::Preprocessor &PP);
 
   /// Consumes tokens from the Lexer and constructs the in-memory
@@ -71,7 +72,7 @@ class RootSignatureParser {
   // expected, or, there is a lexing error
 
   /// Root Element parse methods:
-  std::optional<llvm::hlsl::rootsig::RootFlags> parseRootFlags();
+  std::optional<llvm::dxbc::RootFlags> parseRootFlags();
   std::optional<llvm::hlsl::rootsig::RootConstants> parseRootConstants();
   std::optional<llvm::hlsl::rootsig::RootDescriptor> parseRootDescriptor();
   std::optional<llvm::hlsl::rootsig::DescriptorTable> parseDescriptorTable();
@@ -86,15 +87,15 @@ class RootSignatureParser {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
     std::optional<uint32_t> Num32BitConstants;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
   };
   std::optional<ParsedConstantParams> parseRootConstantParams();
 
   struct ParsedRootDescriptorParams {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
-    std::optional<llvm::hlsl::rootsig::RootDescriptorFlags> Flags;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::RootDescriptorFlags> Flags;
   };
   std::optional<ParsedRootDescriptorParams>
   parseRootDescriptorParams(RootSignatureToken::Kind RegType);
@@ -104,25 +105,25 @@ class RootSignatureParser {
     std::optional<uint32_t> NumDescriptors;
     std::optional<uint32_t> Space;
     std::optional<uint32_t> Offset;
-    std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags> Flags;
+    std::optional<llvm::dxbc::DescriptorRangeFlags> Flags;
   };
   std::optional<ParsedClauseParams>
   parseDescriptorTableClauseParams(RootSignatureToken::Kind RegType);
 
   struct ParsedStaticSamplerParams {
     std::optional<llvm::hlsl::rootsig::Register> Reg;
-    std::optional<llvm::hlsl::rootsig::SamplerFilter> Filter;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressU;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressV;
-    std::optional<llvm::hlsl::rootsig::TextureAddressMode> AddressW;
+    std::optional<llvm::dxbc::SamplerFilter> Filter;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressU;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressV;
+    std::optional<llvm::dxbc::TextureAddressMode> AddressW;
     std::optional<float> MipLODBias;
     std::optional<uint32_t> MaxAnisotropy;
-    std::optional<llvm::hlsl::rootsig::ComparisonFunc> CompFunc;
-    std::optional<llvm::hlsl::rootsig::StaticBorderColor> BorderColor;
+    std::optional<llvm::dxbc::ComparisonFunc> CompFunc;
+    std::optional<llvm::dxbc::StaticBorderColor> BorderColor;
     std::optional<float> MinLOD;
     std::optional<float> MaxLOD;
     std::optional<uint32_t> Space;
-    std::optional<llvm::hlsl::rootsig::ShaderVisibility> Visibility;
+    std::optional<llvm::dxbc::ShaderVisibility> Visibility;
   };
   std::optional<ParsedStaticSamplerParams> parseStaticSamplerParams();
 
@@ -132,17 +133,13 @@ class RootSignatureParser {
   std::optional<float> parseFloatParam();
 
   /// Parsing methods of various enums
-  std::optional<llvm::hlsl::rootsig::ShaderVisibility> parseShaderVisibility();
-  std::optional<llvm::hlsl::rootsig::SamplerFilter> parseSamplerFilter();
-  std::optional<llvm::hlsl::rootsig::TextureAddressMode>
-  parseTextureAddressMode();
-  std::optional<llvm::hlsl::rootsig::ComparisonFunc> parseComparisonFunc();
-  std::optional<llvm::hlsl::rootsig::StaticBorderColor>
-  parseStaticBorderColor();
-  std::optional<llvm::hlsl::rootsig::RootDescriptorFlags>
-  parseRootDescriptorFlags();
-  std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags>
-  parseDescriptorRangeFlags();
+  std::optional<llvm::dxbc::ShaderVisibility> parseShaderVisibility();
+  std::optional<llvm::dxbc::SamplerFilter> parseSamplerFilter();
+  std::optional<llvm::dxbc::TextureAddressMode> parseTextureAddressMode();
+  std::optional<llvm::dxbc::ComparisonFunc> parseComparisonFunc();
+  std::optional<llvm::dxbc::StaticBorderColor> parseStaticBorderColor();
+  std::optional<llvm::dxbc::RootDescriptorFlags> parseRootDescriptorFlags();
+  std::optional<llvm::dxbc::DescriptorRangeFlags> parseDescriptorRangeFlags();
 
   /// Use NumericLiteralParser to convert CurToken.NumSpelling into a unsigned
   /// 32-bit integer
@@ -191,6 +188,7 @@ class RootSignatureParser {
   bool tryConsumeExpectedToken(ArrayRef<RootSignatureToken::Kind> Expected);
 
 private:
+  llvm::dxbc::RootSignatureVersion Version;
   SmallVector<llvm::hlsl::rootsig::RootElement> &Elements;
   RootSignatureLexer &Lexer;
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index a47e23ffbd35..683934321a44 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -101,8 +101,13 @@ enum class ObjCTypeQual {
   NumQuals
 };
 
-/// TypeCastState - State whether an expression is or may be a type cast.
-enum class TypeCastState { NotTypeCast = 0, MaybeTypeCast, IsTypeCast };
+/// If a typo should be encountered, should typo correction suggest type names,
+/// non type names, or both?
+enum class TypoCorrectionTypeBehavior {
+  AllowNonTypes,
+  AllowTypes,
+  AllowBoth,
+};
 
 /// Control what ParseCastExpression will parse.
 enum class CastParseKind { AnyCastExpr = 0, UnaryExprOnly, PrimaryExprOnly };
@@ -116,6 +121,15 @@ enum class ParenParseOption {
   CastExpr         // Also allow '(' type-name ')' <anything>
 };
 
+/// In a call to ParseParenExpression, are the initial parentheses part of an
+/// operator that requires the parens be there (like typeof(int)) or could they
+/// be something else, such as part of a compound literal or a sizeof
+/// expression, etc.
+enum class ParenExprKind {
+  PartOfOperator, // typeof(int)
+  Unknown,        // sizeof(int) or sizeof (int)1.0f, or compound literal, etc
+};
+
 /// Describes the behavior that should be taken for an __if_exists
 /// block.
 enum class IfExistsBehavior {
@@ -3709,11 +3723,12 @@ class Parser : public CodeCompletionHandler {
   ///         assignment-expression ...[opt]
   ///         expression ',' assignment-expression ...[opt]
   /// \endverbatim
-  ExprResult
-  ParseExpression(TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+  ExprResult ParseExpression(TypoCorrectionTypeBehavior CorrectionBehavior =
+                                 TypoCorrectionTypeBehavior::AllowNonTypes);
 
   ExprResult ParseConstantExpressionInExprEvalContext(
-      TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+      TypoCorrectionTypeBehavior CorrectionBehavior =
+          TypoCorrectionTypeBehavior::AllowNonTypes);
   ExprResult ParseConstantExpression();
   ExprResult ParseArrayBoundExpression();
   ExprResult ParseCaseExpression(SourceLocation CaseLoc);
@@ -3750,8 +3765,9 @@ class Parser : public CodeCompletionHandler {
   ExprResult ParseConstraintLogicalOrExpression(bool IsTrailingRequiresClause);
 
   /// Parse an expr that doesn't include (top-level) commas.
-  ExprResult ParseAssignmentExpression(
-      TypeCastState isTypeCast = TypeCastState::NotTypeCast);
+  ExprResult
+  ParseAssignmentExpression(TypoCorrectionTypeBehavior CorrectionBehavior =
+                                TypoCorrectionTypeBehavior::AllowNonTypes);
 
   ExprResult ParseConditionalExpression();
 
@@ -4017,14 +4033,15 @@ class Parser : public CodeCompletionHandler {
   ///
   ExprResult ParseCastExpression(CastParseKind ParseKind,
                                  bool isAddressOfOperand, bool &NotCastExpr,
-                                 TypeCastState isTypeCast,
+                                 TypoCorrectionTypeBehavior CorrectionBehavior,
+                                 bool isVectorLiteral = false,
+                                 bool *NotPrimaryExpression = nullptr);
+  ExprResult ParseCastExpression(CastParseKind ParseKind,
+                                 bool isAddressOfOperand = false,
+                                 TypoCorrectionTypeBehavior CorrectionBehavior =
+                                     TypoCorrectionTypeBehavior::AllowNonTypes,
                                  bool isVectorLiteral = false,
                                  bool *NotPrimaryExpression = nullptr);
-  ExprResult
-  ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand = false,
-                      TypeCastState isTypeCast = TypeCastState::NotTypeCast,
-                      bool isVectorLiteral = false,
-                      bool *NotPrimaryExpression = nullptr);
 
   /// Returns true if the next token cannot start an expression.
   bool isNotExpressionStart();
@@ -4181,10 +4198,15 @@ class Parser : public CodeCompletionHandler {
   /// \endverbatim
   bool ParseSimpleExpressionList(SmallVectorImpl<Expr *> &Exprs);
 
-  /// ParseParenExpression - This parses the unit that starts with a '(' token,
-  /// based on what is allowed by ExprType.  The actual thing parsed is returned
-  /// in ExprType. If stopIfCastExpr is true, it will only return the parsed
-  /// type, not the parsed cast-expression.
+  /// This parses the unit that starts with a '(' token, based on what is
+  /// allowed by ExprType. The actual thing parsed is returned in ExprType. If
+  /// StopIfCastExpr is true, it will only return the parsed type, not the
+  /// parsed cast-expression. If ParenBehavior is ParenExprKind::PartOfOperator,
+  /// the initial open paren and its matching close paren are known to be part
+  /// of another grammar production and not part of the operand. e.g., the
+  /// typeof and typeof_unqual operators in C. Otherwise, the function has to
+  /// parse the parens to determine whether they're part of a cast or compound
+  /// literal expression rather than a parenthesized type.
   ///
   /// \verbatim
   ///       primary-expression: [C99 6.5.1]
@@ -4209,7 +4231,9 @@ class Parser : public CodeCompletionHandler {
   ///       '(' '[' expression ']' { '[' expression ']' } cast-expression
   /// \endverbatim
   ExprResult ParseParenExpression(ParenParseOption &ExprType,
-                                  bool stopIfCastExpr, bool isTypeCast,
+                                  bool StopIfCastExpr,
+                                  ParenExprKind ParenBehavior,
+                                  TypoCorrectionTypeBehavior CorrectionBehavior,
                                   ParsedType &CastTy,
                                   SourceLocation &RParenLoc);
 
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 6c4a32c4ac2f..567ad2d5934d 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -1821,8 +1821,8 @@ class DecompositionDeclarator {
     if (DeleteBindings)
       delete[] Bindings;
     else
-      llvm::for_each(llvm::MutableArrayRef(Bindings, NumBindings),
-                     [](Binding &B) { B.Attrs.reset(); });
+      for (Binding &B : llvm::MutableArrayRef(Bindings, NumBindings))
+        B.Attrs.reset();
     Bindings = nullptr;
     NumBindings = 0;
     DeleteBindings = false;
diff --git a/clang/include/clang/Sema/HeuristicResolver.h b/clang/include/clang/Sema/HeuristicResolver.h
index df60d3359c6a..e193c0bc14cd 100644
--- a/clang/include/clang/Sema/HeuristicResolver.h
+++ b/clang/include/clang/Sema/HeuristicResolver.h
@@ -20,6 +20,7 @@ class CXXBasePath;
 class CXXDependentScopeMemberExpr;
 class DeclarationName;
 class DependentScopeDeclRefExpr;
+class FunctionProtoTypeLoc;
 class NamedDecl;
 class Type;
 class UnresolvedUsingValueDecl;
@@ -93,6 +94,12 @@ class HeuristicResolver {
   // during simplification, and the operation fails if no pointer type is found.
   QualType simplifyType(QualType Type, const Expr *E, bool UnwrapPointer);
 
+  // Given an expression `Fn` representing the callee in a function call,
+  // if the call is through a function pointer, try to find the declaration of
+  // the corresponding function pointer type, so that we can recover argument
+  // names from it.
+  FunctionProtoTypeLoc getFunctionProtoTypeLoc(const Expr *Fn) const;
+
 private:
   ASTContext &Ctx;
 };
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index 0455e1fa5016..a1c156eed539 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -676,11 +676,12 @@ class InitializationKind {
   }
 
   /// Create a direct initialization for a functional cast.
-  static InitializationKind CreateFunctionalCast(SourceRange TypeRange,
+  static InitializationKind CreateFunctionalCast(SourceLocation StartLoc,
+                                                 SourceRange ParenRange,
                                                  bool InitList) {
     return InitializationKind(InitList ? IK_DirectList : IK_Direct,
-                              IC_FunctionalCast, TypeRange.getBegin(),
-                              TypeRange.getBegin(), TypeRange.getEnd());
+                              IC_FunctionalCast, StartLoc,
+                              ParenRange.getBegin(), ParenRange.getEnd());
   }
 
   /// Create a copy initialization.
diff --git a/clang/include/clang/Sema/MultiplexExternalSemaSource.h b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
index 7c66c26a17a1..391c2177d75e 100644
--- a/clang/include/clang/Sema/MultiplexExternalSemaSource.h
+++ b/clang/include/clang/Sema/MultiplexExternalSemaSource.h
@@ -94,8 +94,6 @@ class MultiplexExternalSemaSource : public ExternalSemaSource {
 
   bool wasThisDeclarationADefinition(const FunctionDecl *FD) override;
 
-  bool hasInitializerWithSideEffects(const VarDecl *VD) const override;
-
   /// Find all declarations with the given name in the
   /// given context.
   bool FindExternalVisibleDeclsByName(const DeclContext *DC,
diff --git a/clang/include/clang/Sema/Scope.h b/clang/include/clang/Sema/Scope.h
index 07b9e1bc10f5..757f3dcc3fe8 100644
--- a/clang/include/clang/Sema/Scope.h
+++ b/clang/include/clang/Sema/Scope.h
@@ -158,11 +158,15 @@ class Scope {
     /// constructs, since they have the same behavior.
     OpenACCComputeConstructScope = 0x10000000,
 
+    /// This is the scope of an OpenACC Loop/Combined construct, which is used
+    /// to determine whether a 'cache' construct variable reference is legal.
+    OpenACCLoopConstructScope = 0x20000000,
+
     /// This is a scope of type alias declaration.
-    TypeAliasScope = 0x20000000,
+    TypeAliasScope = 0x40000000,
 
     /// This is a scope of friend declaration.
-    FriendScope = 0x40000000,
+    FriendScope = 0x80000000,
   };
 
 private:
@@ -294,7 +298,7 @@ class Scope {
   // is disallowed despite being a continue scope.
   void setIsConditionVarScope(bool InConditionVarScope) {
     Flags = (Flags & ~ConditionVarScope) |
-            (InConditionVarScope ? ConditionVarScope : 0);
+            (InConditionVarScope ? ConditionVarScope : NoScope);
   }
 
   bool isConditionVarScope() const {
@@ -549,6 +553,10 @@ class Scope {
     return getFlags() & Scope::OpenACCComputeConstructScope;
   }
 
+  bool isOpenACCLoopConstructScope() const {
+    return getFlags() & Scope::OpenACCLoopConstructScope;
+  }
+
   /// Determine if this scope (or its parents) are a compute construct. If the
   /// argument is provided, the search will stop at any of the specified scopes.
   /// Otherwise, it will stop only at the normal 'no longer search' scopes.
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 9397546c8fc5..b281b1cfef96 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -834,6 +834,8 @@ enum class CCEKind {
                            ///< message.
 };
 
+void inferNoReturnAttr(Sema &S, const Decl *D);
+
 /// Sema - This implements semantic analysis and AST building for C.
 /// \nosubgrouping
 class Sema final : public SemaBase {
@@ -7721,12 +7723,12 @@ class Sema final : public SemaBase {
 
   class ConditionResult {
     Decl *ConditionVar;
-    FullExprArg Condition;
+    ExprResult Condition;
     bool Invalid;
     std::optional<bool> KnownValue;
 
     friend class Sema;
-    ConditionResult(Sema &S, Decl *ConditionVar, FullExprArg Condition,
+    ConditionResult(Sema &S, Decl *ConditionVar, ExprResult Condition,
                     bool IsConstexpr)
         : ConditionVar(ConditionVar), Condition(Condition), Invalid(false) {
       if (IsConstexpr && Condition.get()) {
@@ -7737,7 +7739,7 @@ class Sema final : public SemaBase {
       }
     }
     explicit ConditionResult(bool Invalid)
-        : ConditionVar(nullptr), Condition(nullptr), Invalid(Invalid),
+        : ConditionVar(nullptr), Condition(Invalid), Invalid(Invalid),
           KnownValue(std::nullopt) {}
 
   public:
@@ -15190,11 +15192,18 @@ class Sema final : public SemaBase {
                                SourceLocation Loc);
   QualType BuiltinRemoveReference(QualType BaseType, UTTKind UKind,
                                   SourceLocation Loc);
+
+  QualType BuiltinRemoveCVRef(QualType BaseType, SourceLocation Loc) {
+    return BuiltinRemoveReference(BaseType, UTTKind::RemoveCVRef, Loc);
+  }
+
   QualType BuiltinChangeCVRQualifiers(QualType BaseType, UTTKind UKind,
                                       SourceLocation Loc);
   QualType BuiltinChangeSignedness(QualType BaseType, UTTKind UKind,
                                    SourceLocation Loc);
 
+  bool BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT, QualType RhsT);
+
   /// Ensure that the type T is a literal type.
   ///
   /// This routine checks whether the type @p T is a literal type. If @p T is an
diff --git a/clang/include/clang/Sema/SemaAMDGPU.h b/clang/include/clang/Sema/SemaAMDGPU.h
index d62c9bb65fad..f72e1c53d2c9 100644
--- a/clang/include/clang/Sema/SemaAMDGPU.h
+++ b/clang/include/clang/Sema/SemaAMDGPU.h
@@ -15,12 +15,16 @@
 
 #include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/SmallPtrSet.h"
 
 namespace clang {
 class AttributeCommonInfo;
+class Expr;
 class ParsedAttr;
 
 class SemaAMDGPU : public SemaBase {
+  llvm::SmallPtrSet<Expr *, 32> ExpandedPredicates;
+
 public:
   SemaAMDGPU(Sema &S);
 
@@ -64,6 +68,11 @@ class SemaAMDGPU : public SemaBase {
   void handleAMDGPUNumVGPRAttr(Decl *D, const ParsedAttr &AL);
   void handleAMDGPUMaxNumWorkGroupsAttr(Decl *D, const ParsedAttr &AL);
   void handleAMDGPUFlatWorkGroupSizeAttr(Decl *D, const ParsedAttr &AL);
+
+  /// Expand a valid use of the feature identification builtins into its
+  /// corresponding sequence of instructions.
+  Expr *ExpandAMDGPUPredicateBI(CallExpr *CE);
+  bool IsPredicate(Expr *E) const;
 };
 } // namespace clang
 
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index ce79e94ebdd9..788a7abf5f9c 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -82,6 +82,15 @@ class SemaARM : public SemaBase {
   void handleInterruptSaveFPAttr(Decl *D, const ParsedAttr &AL);
 
   void CheckSMEFunctionDefAttributes(const FunctionDecl *FD);
+
+  /// Return true if the given types are an SVE builtin and a VectorType that
+  /// is a fixed-length representation of the SVE builtin for a specific
+  /// vector-length.
+  bool areCompatibleSveTypes(QualType FirstType, QualType SecondType);
+
+  /// Return true if the given vector types are lax-compatible SVE vector types,
+  /// false otherwise.
+  bool areLaxCompatibleSveTypes(QualType FirstType, QualType SecondType);
 };
 
 SemaARM::ArmStreamingType getArmStreamingFnType(const FunctionDecl *FD);
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index c4a986a3eb6f..b7e7f5d97bce 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -117,6 +117,15 @@ class SemaOpenACC : public SemaBase {
     OpenACCDirectiveKind DirectiveKind = OpenACCDirectiveKind::Invalid;
   } TileInfo;
 
+  /// The 'cache' var-list requires some additional work to track variable
+  /// references to make sure they are on the 'other' side of a `loop`. This
+  /// structure is used during parse time to track vardecl use while parsing a
+  /// cache var list.
+  struct CacheParseInfo {
+    bool ParsingCacheVarList = false;
+    bool IsInvalidCacheRef = false;
+  } CacheInfo;
+
   /// A list of the active reduction clauses, which allows us to check that all
   /// vars on nested constructs for the same reduction var have the same
   /// reduction operator. Currently this is enforced against all constructs
@@ -861,6 +870,12 @@ class SemaOpenACC : public SemaBase {
   ExprResult ActOnIntExpr(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
                           SourceLocation Loc, Expr *IntExpr);
 
+  /// Called right before a 'var' is parsed, so we can set the state for parsing
+  /// a 'cache' var.
+  void ActOnStartParseVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK);
+  /// Called only if the parse of a 'var' was invalid, else 'ActOnVar' should be
+  /// called.
+  void ActOnInvalidParseVar();
   /// Called when encountering a 'var' for OpenACC, ensures it is actually a
   /// declaration reference to a variable of the correct type.
   ExprResult ActOnVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
@@ -913,6 +928,10 @@ class SemaOpenACC : public SemaBase {
                            OpenACCDirectiveKind DK, OpenACCGangKind GK,
                            Expr *E);
 
+  // Called when a declaration is referenced, so that we can make sure certain
+  // clauses don't do the 'wrong' thing/have incorrect references.
+  void CheckDeclReference(SourceLocation Loc, Expr *E, Decl *D);
+
   // Does the checking for a 'gang' clause that needs to be done in dependent
   // and not dependent cases.
   OpenACCClause *
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 7b169f56b680..91c3d4bd5210 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -877,10 +877,10 @@ class SemaOpenMP : public SemaBase {
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc);
   /// Called on well-formed 'num_threads' clause.
-  OMPClause *ActOnOpenMPNumThreadsClause(Expr *NumThreads,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc);
+  OMPClause *ActOnOpenMPNumThreadsClause(
+      OpenMPNumThreadsClauseModifier Modifier, Expr *NumThreads,
+      SourceLocation StartLoc, SourceLocation LParenLoc,
+      SourceLocation ModifierLoc, SourceLocation EndLoc);
   /// Called on well-formed 'align' clause.
   OMPClause *ActOnOpenMPAlignClause(Expr *Alignment, SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 7a4b7d21bb20..3d96f8e65180 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -464,8 +464,6 @@ class ASTReader
   using ModuleReverseIterator = ModuleManager::ModuleReverseIterator;
 
 private:
-  using LocSeq = SourceLocationSequence;
-
   /// The receiver of some callbacks invoked by ASTReader.
   std::unique_ptr<ASTReaderListener> Listener;
 
@@ -527,7 +525,7 @@ class ASTReader
   ContinuousRangeMap<unsigned, ModuleFile*, 64> GlobalSLocEntryMap;
 
   using GlobalSLocOffsetMapType =
-      ContinuousRangeMap<unsigned, ModuleFile *, 64>;
+      ContinuousRangeMap<SourceLocation::UIntTy, ModuleFile *, 64>;
 
   /// A map of reversed (SourceManager::MaxLoadedOffset - SLocOffset)
   /// SourceLocation offsets to the modules containing them.
@@ -1455,12 +1453,6 @@ class ASTReader
     const StringRef &operator*() && = delete;
   };
 
-  /// VarDecls with initializers containing side effects must be emitted,
-  /// but DeclMustBeEmitted is not allowed to deserialize the intializer.
-  /// FIXME: Lower memory usage by removing VarDecls once the initializer
-  /// is deserialized.
-  llvm::SmallPtrSet<Decl *, 16> InitSideEffectVars;
-
 public:
   /// Get the buffer for resolving paths.
   SmallString<0> &getPathBuf() { return PathBuf; }
@@ -2412,8 +2404,6 @@ class ASTReader
 
   bool wasThisDeclarationADefinition(const FunctionDecl *FD) override;
 
-  bool hasInitializerWithSideEffects(const VarDecl *VD) const override;
-
   /// Retrieve a selector from the given module with its local ID
   /// number.
   Selector getLocalSelector(ModuleFile &M, unsigned LocalID);
@@ -2445,18 +2435,16 @@ class ASTReader
   /// Read a source location from raw form and return it in its
   /// originating module file's source location space.
   std::pair<SourceLocation, unsigned>
-  ReadUntranslatedSourceLocation(RawLocEncoding Raw,
-                                 LocSeq *Seq = nullptr) const {
-    return SourceLocationEncoding::decode(Raw, Seq);
+  ReadUntranslatedSourceLocation(RawLocEncoding Raw) const {
+    return SourceLocationEncoding::decode(Raw);
   }
 
   /// Read a source location from raw form.
-  SourceLocation ReadSourceLocation(ModuleFile &MF, RawLocEncoding Raw,
-                                    LocSeq *Seq = nullptr) const {
+  SourceLocation ReadSourceLocation(ModuleFile &MF, RawLocEncoding Raw) const {
     if (!MF.ModuleOffsetMap.empty())
       ReadModuleOffsetMap(MF);
 
-    auto [Loc, ModuleFileIndex] = ReadUntranslatedSourceLocation(Raw, Seq);
+    auto [Loc, ModuleFileIndex] = ReadUntranslatedSourceLocation(Raw);
     ModuleFile *OwningModuleFile =
         ModuleFileIndex == 0 ? &MF : MF.TransitiveImports[ModuleFileIndex - 1];
 
@@ -2484,9 +2472,9 @@ class ASTReader
 
   /// Read a source location.
   SourceLocation ReadSourceLocation(ModuleFile &ModuleFile,
-                                    const RecordDataImpl &Record, unsigned &Idx,
-                                    LocSeq *Seq = nullptr) {
-    return ReadSourceLocation(ModuleFile, Record[Idx++], Seq);
+                                    const RecordDataImpl &Record,
+                                    unsigned &Idx) {
+    return ReadSourceLocation(ModuleFile, Record[Idx++]);
   }
 
   /// Read a FileID.
@@ -2505,7 +2493,7 @@ class ASTReader
 
   /// Read a source range.
   SourceRange ReadSourceRange(ModuleFile &F, const RecordData &Record,
-                              unsigned &Idx, LocSeq *Seq = nullptr);
+                              unsigned &Idx);
 
   static llvm::BitVector ReadBitVector(const RecordData &Record,
                                        const StringRef Blob);
diff --git a/clang/include/clang/Serialization/ASTRecordReader.h b/clang/include/clang/Serialization/ASTRecordReader.h
index da3f504ff27d..1472497ff5e7 100644
--- a/clang/include/clang/Serialization/ASTRecordReader.h
+++ b/clang/include/clang/Serialization/ASTRecordReader.h
@@ -32,7 +32,6 @@ class OMPChildren;
 class ASTRecordReader
     : public serialization::DataStreamBasicReader<ASTRecordReader> {
   using ModuleFile = serialization::ModuleFile;
-  using LocSeq = SourceLocationSequence;
 
   ASTReader *Reader;
   ModuleFile *F;
@@ -160,7 +159,7 @@ class ASTRecordReader
   TypeSourceInfo *readTypeSourceInfo();
 
   /// Reads the location information for a type.
-  void readTypeLoc(TypeLoc TL, LocSeq *Seq = nullptr);
+  void readTypeLoc(TypeLoc TL);
 
   /// Map a local type ID within a given AST file to a global type ID.
   serialization::TypeID getGlobalTypeID(serialization::TypeID LocalID) const {
@@ -287,13 +286,13 @@ class ASTRecordReader
   void readOpenACCRoutineDeclAttr(OpenACCRoutineDeclAttr *A);
 
   /// Read a source location, advancing Idx.
-  SourceLocation readSourceLocation(LocSeq *Seq = nullptr) {
-    return Reader->ReadSourceLocation(*F, Record, Idx, Seq);
+  SourceLocation readSourceLocation() {
+    return Reader->ReadSourceLocation(*F, Record, Idx);
   }
 
   /// Read a source range, advancing Idx.
-  SourceRange readSourceRange(LocSeq *Seq = nullptr) {
-    return Reader->ReadSourceRange(*F, Record, Idx, Seq);
+  SourceRange readSourceRange() {
+    return Reader->ReadSourceRange(*F, Record, Idx);
   }
 
   /// Read an arbitrary constant value, advancing Idx.
diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index 964c9e6ea8a2..ee005ec28770 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -29,7 +29,6 @@ class TypeLoc;
 /// An object for streaming information to a record.
 class ASTRecordWriter
     : public serialization::DataStreamBasicWriter<ASTRecordWriter> {
-  using LocSeq = SourceLocationSequence;
 
   ASTWriter *Writer;
   ASTWriter::RecordDataImpl *Record;
@@ -147,8 +146,8 @@ class ASTRecordWriter
   void AddFunctionDefinition(const FunctionDecl *FD);
 
   /// Emit a source location.
-  void AddSourceLocation(SourceLocation Loc, LocSeq *Seq = nullptr) {
-    return Writer->AddSourceLocation(Loc, *Record, Seq);
+  void AddSourceLocation(SourceLocation Loc) {
+    return Writer->AddSourceLocation(Loc, *Record);
   }
   void writeSourceLocation(SourceLocation Loc) {
     AddSourceLocation(Loc);
@@ -174,8 +173,8 @@ class ASTRecordWriter
   }
 
   /// Emit a source range.
-  void AddSourceRange(SourceRange Range, LocSeq *Seq = nullptr) {
-    return Writer->AddSourceRange(Range, *Record, Seq);
+  void AddSourceRange(SourceRange Range) {
+    return Writer->AddSourceRange(Range, *Record);
   }
 
   void writeBool(bool Value) {
@@ -245,7 +244,7 @@ class ASTRecordWriter
   void AddTypeSourceInfo(TypeSourceInfo *TInfo);
 
   /// Emits source location information for a type. Does not emit the type.
-  void AddTypeLoc(TypeLoc TL, LocSeq *Seq = nullptr);
+  void AddTypeLoc(TypeLoc TL);
 
   /// Emits a template argument location info.
   void AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 97679ace8b61..c86019f01d9f 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -115,8 +115,6 @@ class ASTWriter : public ASTDeserializationListener,
   using TypeIdxMap = llvm::DenseMap<QualType, serialization::TypeIdx,
                                     serialization::UnsafeQualTypeDenseMapInfo>;
 
-  using LocSeq = SourceLocationSequence;
-
   /// The bitstream writer used to emit this precompiled header.
   llvm::BitstreamWriter &Stream;
 
@@ -378,6 +376,7 @@ class ASTWriter : public ASTDeserializationListener,
   /// Only meaningful for standard C++ named modules. See the comments in
   /// createSignatureForNamedModule() for details.
   llvm::SetVector<Module *> TouchedTopLevelModules;
+  llvm::SetVector<serialization::ModuleFile *> TouchedModuleFiles;
 
   /// An update to a Decl.
   class DeclUpdate {
@@ -733,16 +732,14 @@ class ASTWriter : public ASTDeserializationListener,
   void AddFileID(FileID FID, RecordDataImpl &Record);
 
   /// Emit a source location.
-  void AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record,
-                         LocSeq *Seq = nullptr);
+  void AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record);
 
   /// Return the raw encodings for source locations.
   SourceLocationEncoding::RawLocEncoding
-  getRawSourceLocationEncoding(SourceLocation Loc, LocSeq *Seq = nullptr);
+  getRawSourceLocationEncoding(SourceLocation Loc);
 
   /// Emit a source range.
-  void AddSourceRange(SourceRange Range, RecordDataImpl &Record,
-                      LocSeq *Seq = nullptr);
+  void AddSourceRange(SourceRange Range, RecordDataImpl &Record);
 
   /// Emit a reference to an identifier.
   void AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Record);
@@ -917,6 +914,8 @@ class ASTWriter : public ASTDeserializationListener,
 
   void handleVTable(CXXRecordDecl *RD);
 
+  void addTouchedModuleFile(serialization::ModuleFile *);
+
 private:
   // ASTDeserializationListener implementation
   void ReaderInitialized(ASTReader *Reader) override;
diff --git a/clang/include/clang/Serialization/SourceLocationEncoding.h b/clang/include/clang/Serialization/SourceLocationEncoding.h
index 33ca1728fa47..5b2485dbc719 100644
--- a/clang/include/clang/Serialization/SourceLocationEncoding.h
+++ b/clang/include/clang/Serialization/SourceLocationEncoding.h
@@ -25,20 +25,17 @@
 //  * C: The macro bit. We rotate it to the lowest bit so that we can save some
 //  space in case the index of the module file is 0.
 //
-// Specially, if the index of the module file is 0, we allow to encode a
-// sequence of locations we store only differences between successive elements.
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLVM_CLANG_SERIALIZATION_SOURCELOCATIONENCODING_H
+#define LLVM_CLANG_SERIALIZATION_SOURCELOCATIONENCODING_H
+
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/Support/MathExtras.h"
 #include <climits>
 
-#ifndef LLVM_CLANG_SERIALIZATION_SOURCELOCATIONENCODING_H
-#define LLVM_CLANG_SERIALIZATION_SOURCELOCATIONENCODING_H
-
 namespace clang {
-class SourceLocationSequence;
 
 /// Serialized encoding of SourceLocations without context.
 /// Optimized to have small unsigned values (=> small after VBR encoding).
@@ -54,119 +51,22 @@ class SourceLocationEncoding {
   static UIntTy decodeRaw(UIntTy Raw) {
     return (Raw >> 1) | (Raw << (UIntBits - 1));
   }
-  friend SourceLocationSequence;
 
 public:
   using RawLocEncoding = uint64_t;
 
   static RawLocEncoding encode(SourceLocation Loc, UIntTy BaseOffset,
-                               unsigned BaseModuleFileIndex,
-                               SourceLocationSequence * = nullptr);
-  static std::pair<SourceLocation, unsigned>
-  decode(RawLocEncoding, SourceLocationSequence * = nullptr);
-};
-
-/// Serialized encoding of a sequence of SourceLocations.
-///
-/// Optimized to produce small values when locations with the sequence are
-/// similar. Each element can be delta-encoded against the last nonzero element.
-///
-/// Sequences should be started by creating a SourceLocationSequence::State,
-/// and then passed around as SourceLocationSequence*. Example:
-///
-///   // establishes a sequence
-///   void EmitTopLevelThing() {
-///     SourceLocationSequence::State Seq;
-///     EmitContainedThing(Seq);
-///     EmitRecursiveThing(Seq);
-///   }
-///
-///   // optionally part of a sequence
-///   void EmitContainedThing(SourceLocationSequence *Seq = nullptr) {
-///     Record.push_back(SourceLocationEncoding::encode(SomeLoc, Seq));
-///   }
-///
-///   // establishes a sequence if there isn't one already
-///   void EmitRecursiveThing(SourceLocationSequence *ParentSeq = nullptr) {
-///     SourceLocationSequence::State Seq(ParentSeq);
-///     Record.push_back(SourceLocationEncoding::encode(SomeLoc, Seq));
-///     EmitRecursiveThing(Seq);
-///   }
-///
-class SourceLocationSequence {
-  using UIntTy = SourceLocation::UIntTy;
-  using EncodedTy = uint64_t;
-  constexpr static auto UIntBits = SourceLocationEncoding::UIntBits;
-  static_assert(sizeof(EncodedTy) > sizeof(UIntTy), "Need one extra bit!");
-
-  // Prev stores the rotated last nonzero location.
-  UIntTy &Prev;
-
-  // Zig-zag encoding turns small signed integers into small unsigned integers.
-  // 0 => 0, -1 => 1, 1 => 2, -2 => 3, ...
-  static UIntTy zigZag(UIntTy V) {
-    UIntTy Sign = (V & (1 << (UIntBits - 1))) ? UIntTy(-1) : UIntTy(0);
-    return Sign ^ (V << 1);
-  }
-  static UIntTy zagZig(UIntTy V) { return (V >> 1) ^ -(V & 1); }
-
-  SourceLocationSequence(UIntTy &Prev) : Prev(Prev) {}
-
-  EncodedTy encodeRaw(UIntTy Raw) {
-    if (Raw == 0)
-      return 0;
-    UIntTy Rotated = SourceLocationEncoding::encodeRaw(Raw);
-    if (Prev == 0)
-      return Prev = Rotated;
-    UIntTy Delta = Rotated - Prev;
-    Prev = Rotated;
-    // Exactly one 33 bit value is possible! (1 << 32).
-    // This is because we have two representations of zero: trivial & relative.
-    return 1 + EncodedTy{zigZag(Delta)};
-  }
-  UIntTy decodeRaw(EncodedTy Encoded) {
-    if (Encoded == 0)
-      return 0;
-    if (Prev == 0)
-      return SourceLocationEncoding::decodeRaw(Prev = Encoded);
-    return SourceLocationEncoding::decodeRaw(Prev += zagZig(Encoded - 1));
-  }
-
-public:
-  SourceLocation decode(EncodedTy Encoded) {
-    return SourceLocation::getFromRawEncoding(decodeRaw(Encoded));
-  }
-  EncodedTy encode(SourceLocation Loc) {
-    return encodeRaw(Loc.getRawEncoding());
-  }
-
-  class State;
-};
-
-/// This object establishes a SourceLocationSequence.
-class SourceLocationSequence::State {
-  UIntTy Prev = 0;
-  SourceLocationSequence Seq;
-
-public:
-  // If Parent is provided and non-null, then this root becomes part of that
-  // enclosing sequence instead of establishing a new one.
-  State(SourceLocationSequence *Parent = nullptr)
-      : Seq(Parent ? Parent->Prev : Prev) {}
-
-  // Implicit conversion for uniform use of roots vs propagated sequences.
-  operator SourceLocationSequence *() { return &Seq; }
+                               unsigned BaseModuleFileIndex);
+  static std::pair<SourceLocation, unsigned> decode(RawLocEncoding);
 };
 
 inline SourceLocationEncoding::RawLocEncoding
 SourceLocationEncoding::encode(SourceLocation Loc, UIntTy BaseOffset,
-                               unsigned BaseModuleFileIndex,
-                               SourceLocationSequence *Seq) {
+                               unsigned BaseModuleFileIndex) {
   // If the source location is a local source location, we can try to optimize
   // the similar sequences to only record the differences.
   if (!BaseOffset)
-    return Seq ? Seq->encode(Loc) : encodeRaw(Loc.getRawEncoding());
-
+    return encodeRaw(Loc.getRawEncoding());
   if (Loc.isInvalid())
     return 0;
 
@@ -183,13 +83,11 @@ SourceLocationEncoding::encode(SourceLocation Loc, UIntTy BaseOffset,
   return Encoded;
 }
 inline std::pair<SourceLocation, unsigned>
-SourceLocationEncoding::decode(RawLocEncoding Encoded,
-                               SourceLocationSequence *Seq) {
+SourceLocationEncoding::decode(RawLocEncoding Encoded) {
   unsigned ModuleFileIndex = Encoded >> 32;
 
   if (!ModuleFileIndex)
-    return {Seq ? Seq->decode(Encoded)
-                : SourceLocation::getFromRawEncoding(decodeRaw(Encoded)),
+    return {SourceLocation::getFromRawEncoding(decodeRaw(Encoded)),
             ModuleFileIndex};
 
   Encoded &= llvm::maskTrailingOnes<RawLocEncoding>(32);
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
index f6a43bf5f493..5dcf03f7a464 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
@@ -554,6 +554,8 @@ class SimpleFunctionCall : public AnyFunctionCall {
 
   const FunctionDecl *getDecl() const override;
 
+  RuntimeDefinition getRuntimeDefinition() const override;
+
   unsigned getNumArgs() const override { return getOriginExpr()->getNumArgs(); }
 
   const Expr *getArgExpr(unsigned Index) const override {
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
index e995151927c9..bd11e0d40cfa 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h
@@ -448,14 +448,15 @@ class ExplodedNodeSet {
 
 public:
   ExplodedNodeSet(ExplodedNode *N) {
-    assert(N && !static_cast<ExplodedNode*>(N)->isSink());
+    assert(N && !N->isSink());
     Impl.insert(N);
   }
 
   ExplodedNodeSet() = default;
 
   void Add(ExplodedNode *N) {
-    if (N && !static_cast<ExplodedNode*>(N)->isSink()) Impl.insert(N);
+    if (N && !N->isSink())
+      Impl.insert(N);
   }
 
   using iterator = ImplTy::iterator;
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
index 70a7953918ac..a6cb6c0f12a8 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SMTConv.h
@@ -598,6 +598,9 @@ class SMTConv {
     if (APSIntBitwidth == 1 && Ty.isNull())
       return {Int.extend(Ctx.getTypeSize(Ctx.BoolTy)),
               getAPSIntType(Ctx, NewInt)};
+    else if (APSIntBitwidth == 1 && !Ty.isNull())
+      return {Int.extend(Ctx.getTypeSize(getAPSIntType(Ctx, Int))),
+              getAPSIntType(Ctx, NewInt)};
     if (llvm::isPowerOf2_32(APSIntBitwidth) || Ty.isNull())
       return {Int, Ty};
     return {Int.extend(Ctx.getTypeSize(Ty)), Ty};
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h
index aca14cf813c4..6233a22d2ca2 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymExpr.h
@@ -152,9 +152,9 @@ class SymbolData : public SymExpr {
   };
 
   // Implement isa<T> support.
-  static inline bool classof(const SymExpr *SE) {
-    Kind k = SE->getKind();
-    return k >= BEGIN_SYMBOLS && k <= END_SYMBOLS;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) {
+    return K >= BEGIN_SYMBOLS && K <= END_SYMBOLS;
   }
 };
 
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
index 86774ad5043d..7af86cd721e8 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/SymbolManager.h
@@ -46,7 +46,7 @@ class SymbolRegionValue : public SymbolData {
 
   friend class SymExprAllocator;
   SymbolRegionValue(SymbolID sym, const TypedValueRegion *r)
-      : SymbolData(SymbolRegionValueKind, sym), R(r) {
+      : SymbolData(ClassKind, sym), R(r) {
     assert(r);
     assert(isValidTypeForSymbol(r->getValueType()));
   }
@@ -56,7 +56,7 @@ class SymbolRegionValue : public SymbolData {
   const TypedValueRegion *getRegion() const { return R; }
 
   static void Profile(llvm::FoldingSetNodeID& profile, const TypedValueRegion* R) {
-    profile.AddInteger((unsigned) SymbolRegionValueKind);
+    profile.AddInteger((unsigned)ClassKind);
     profile.AddPointer(R);
   }
 
@@ -72,9 +72,9 @@ class SymbolRegionValue : public SymbolData {
   QualType getType() const override;
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolRegionValueKind;
-  }
+  static constexpr Kind ClassKind = SymbolRegionValueKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// A symbol representing the result of an expression in the case when we do
@@ -90,8 +90,8 @@ class SymbolConjured : public SymbolData {
   SymbolConjured(SymbolID sym, ConstCFGElementRef elem,
                  const LocationContext *lctx, QualType t, unsigned count,
                  const void *symbolTag)
-      : SymbolData(SymbolConjuredKind, sym), Elem(elem), T(t), Count(count),
-        LCtx(lctx), SymbolTag(symbolTag) {
+      : SymbolData(ClassKind, sym), Elem(elem), T(t), Count(count), LCtx(lctx),
+        SymbolTag(symbolTag) {
     assert(lctx);
     assert(isValidTypeForSymbol(t));
   }
@@ -115,7 +115,7 @@ class SymbolConjured : public SymbolData {
   static void Profile(llvm::FoldingSetNodeID &profile, ConstCFGElementRef Elem,
                       const LocationContext *LCtx, QualType T, unsigned Count,
                       const void *SymbolTag) {
-    profile.AddInteger((unsigned)SymbolConjuredKind);
+    profile.AddInteger((unsigned)ClassKind);
     profile.Add(Elem);
     profile.AddPointer(LCtx);
     profile.Add(T);
@@ -128,9 +128,9 @@ class SymbolConjured : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolConjuredKind;
-  }
+  static constexpr Kind ClassKind = SymbolConjuredKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// A symbol representing the value of a MemRegion whose parent region has
@@ -141,7 +141,7 @@ class SymbolDerived : public SymbolData {
 
   friend class SymExprAllocator;
   SymbolDerived(SymbolID sym, SymbolRef parent, const TypedValueRegion *r)
-      : SymbolData(SymbolDerivedKind, sym), parentSymbol(parent), R(r) {
+      : SymbolData(ClassKind, sym), parentSymbol(parent), R(r) {
     assert(parent);
     assert(r);
     assert(isValidTypeForSymbol(r->getValueType()));
@@ -162,7 +162,7 @@ class SymbolDerived : public SymbolData {
 
   static void Profile(llvm::FoldingSetNodeID& profile, SymbolRef parent,
                       const TypedValueRegion *r) {
-    profile.AddInteger((unsigned) SymbolDerivedKind);
+    profile.AddInteger((unsigned)ClassKind);
     profile.AddPointer(r);
     profile.AddPointer(parent);
   }
@@ -172,9 +172,9 @@ class SymbolDerived : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolDerivedKind;
-  }
+  static constexpr Kind ClassKind = SymbolDerivedKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// SymbolExtent - Represents the extent (size in bytes) of a bounded region.
@@ -185,7 +185,7 @@ class SymbolExtent : public SymbolData {
 
   friend class SymExprAllocator;
   SymbolExtent(SymbolID sym, const SubRegion *r)
-      : SymbolData(SymbolExtentKind, sym), R(r) {
+      : SymbolData(ClassKind, sym), R(r) {
     assert(r);
   }
 
@@ -200,7 +200,7 @@ class SymbolExtent : public SymbolData {
   void dumpToStream(raw_ostream &os) const override;
 
   static void Profile(llvm::FoldingSetNodeID& profile, const SubRegion *R) {
-    profile.AddInteger((unsigned) SymbolExtentKind);
+    profile.AddInteger((unsigned)ClassKind);
     profile.AddPointer(R);
   }
 
@@ -209,9 +209,9 @@ class SymbolExtent : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolExtentKind;
-  }
+  static constexpr Kind ClassKind = SymbolExtentKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// SymbolMetadata - Represents path-dependent metadata about a specific region.
@@ -229,16 +229,16 @@ class SymbolMetadata : public SymbolData {
   const void *Tag;
 
   friend class SymExprAllocator;
-  SymbolMetadata(SymbolID sym, const MemRegion* r, const Stmt *s, QualType t,
+  SymbolMetadata(SymbolID sym, const MemRegion *r, const Stmt *s, QualType t,
                  const LocationContext *LCtx, unsigned count, const void *tag)
-      : SymbolData(SymbolMetadataKind, sym), R(r), S(s), T(t), LCtx(LCtx),
-        Count(count), Tag(tag) {
-      assert(r);
-      assert(s);
-      assert(isValidTypeForSymbol(t));
-      assert(LCtx);
-      assert(tag);
-    }
+      : SymbolData(ClassKind, sym), R(r), S(s), T(t), LCtx(LCtx), Count(count),
+        Tag(tag) {
+    assert(r);
+    assert(s);
+    assert(isValidTypeForSymbol(t));
+    assert(LCtx);
+    assert(tag);
+  }
 
   public:
     LLVM_ATTRIBUTE_RETURNS_NONNULL
@@ -264,7 +264,7 @@ class SymbolMetadata : public SymbolData {
     static void Profile(llvm::FoldingSetNodeID &profile, const MemRegion *R,
                         const Stmt *S, QualType T, const LocationContext *LCtx,
                         unsigned Count, const void *Tag) {
-      profile.AddInteger((unsigned)SymbolMetadataKind);
+      profile.AddInteger((unsigned)ClassKind);
       profile.AddPointer(R);
       profile.AddPointer(S);
       profile.Add(T);
@@ -278,9 +278,9 @@ class SymbolMetadata : public SymbolData {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolMetadataKind;
-  }
+  static constexpr Kind ClassKind = SymbolMetadataKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// Represents a cast expression.
@@ -295,7 +295,7 @@ class SymbolCast : public SymExpr {
 
   friend class SymExprAllocator;
   SymbolCast(SymbolID Sym, const SymExpr *In, QualType From, QualType To)
-      : SymExpr(SymbolCastKind, Sym), Operand(In), FromTy(From), ToTy(To) {
+      : SymExpr(ClassKind, Sym), Operand(In), FromTy(From), ToTy(To) {
     assert(In);
     assert(isValidTypeForSymbol(From));
     // FIXME: GenericTaintChecker creates symbols of void type.
@@ -318,7 +318,7 @@ class SymbolCast : public SymExpr {
 
   static void Profile(llvm::FoldingSetNodeID& ID,
                       const SymExpr *In, QualType From, QualType To) {
-    ID.AddInteger((unsigned) SymbolCastKind);
+    ID.AddInteger((unsigned)ClassKind);
     ID.AddPointer(In);
     ID.Add(From);
     ID.Add(To);
@@ -329,9 +329,9 @@ class SymbolCast : public SymExpr {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == SymbolCastKind;
-  }
+  static constexpr Kind ClassKind = SymbolCastKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// Represents a symbolic expression involving a unary operator.
@@ -343,7 +343,7 @@ class UnarySymExpr : public SymExpr {
   friend class SymExprAllocator;
   UnarySymExpr(SymbolID Sym, const SymExpr *In, UnaryOperator::Opcode Op,
                QualType T)
-      : SymExpr(UnarySymExprKind, Sym), Operand(In), Op(Op), T(T) {
+      : SymExpr(ClassKind, Sym), Operand(In), Op(Op), T(T) {
     // Note, some unary operators are modeled as a binary operator. E.g. ++x is
     // modeled as x + 1.
     assert((Op == UO_Minus || Op == UO_Not) && "non-supported unary expression");
@@ -369,7 +369,7 @@ class UnarySymExpr : public SymExpr {
 
   static void Profile(llvm::FoldingSetNodeID &ID, const SymExpr *In,
                       UnaryOperator::Opcode Op, QualType T) {
-    ID.AddInteger((unsigned)UnarySymExprKind);
+    ID.AddInteger((unsigned)ClassKind);
     ID.AddPointer(In);
     ID.AddInteger(Op);
     ID.Add(T);
@@ -380,9 +380,9 @@ class UnarySymExpr : public SymExpr {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    return SE->getKind() == UnarySymExprKind;
-  }
+  static constexpr Kind ClassKind = UnarySymExprKind;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// Represents a symbolic expression involving a binary operator
@@ -408,9 +408,9 @@ class BinarySymExpr : public SymExpr {
   BinaryOperator::Opcode getOpcode() const { return Op; }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) {
-    Kind k = SE->getKind();
-    return k >= BEGIN_BINARYSYMEXPRS && k <= END_BINARYSYMEXPRS;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) {
+    return K >= BEGIN_BINARYSYMEXPRS && K <= END_BINARYSYMEXPRS;
   }
 
 protected:
@@ -430,7 +430,7 @@ class BinarySymExpr : public SymExpr {
 };
 
 /// Template implementation for all binary symbolic expressions
-template <class LHSTYPE, class RHSTYPE, SymExpr::Kind ClassKind>
+template <class LHSTYPE, class RHSTYPE, SymExpr::Kind ClassK>
 class BinarySymExprImpl : public BinarySymExpr {
   LHSTYPE LHS;
   RHSTYPE RHS;
@@ -474,7 +474,9 @@ class BinarySymExprImpl : public BinarySymExpr {
   }
 
   // Implement isa<T> support.
-  static bool classof(const SymExpr *SE) { return SE->getKind() == ClassKind; }
+  static constexpr Kind ClassKind = ClassK;
+  static bool classof(const SymExpr *SE) { return classof(SE->getKind()); }
+  static constexpr bool classof(Kind K) { return K == ClassKind; }
 };
 
 /// Represents a symbolic expression like 'x' + 3.
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index da83220babea..2b21be771269 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -18,13 +18,12 @@
 #include "llvm/Support/VirtualFileSystem.h"
 #include <mutex>
 #include <optional>
+#include <variant>
 
 namespace clang {
 namespace tooling {
 namespace dependencies {
 
-class DependencyScanningService;
-
 using DependencyDirectivesTy =
     SmallVector<dependency_directives_scan::Directive, 20>;
 
@@ -222,13 +221,34 @@ class DependencyScanningFilesystemSharedCache {
   CacheShard &getShardForFilename(StringRef Filename) const;
   CacheShard &getShardForUID(llvm::sys::fs::UniqueID UID) const;
 
-  /// Visits all cached entries and re-stat an entry using FS if
-  /// it is negatively stat cached. If re-stat succeeds on a path,
-  /// the path is added to InvalidPaths, indicating that the cache
-  /// may have erroneously negatively cached it. The caller can then
-  /// use InvalidPaths to issue diagnostics.
-  std::vector<StringRef>
-  getInvalidNegativeStatCachedPaths(llvm::vfs::FileSystem &UnderlyingFS) const;
+  struct OutOfDateEntry {
+    // A null terminated string that contains a path.
+    const char *Path = nullptr;
+
+    struct NegativelyCachedInfo {};
+    struct SizeChangedInfo {
+      uint64_t CachedSize = 0;
+      uint64_t ActualSize = 0;
+    };
+
+    std::variant<NegativelyCachedInfo, SizeChangedInfo> Info;
+
+    OutOfDateEntry(const char *Path)
+        : Path(Path), Info(NegativelyCachedInfo{}) {}
+
+    OutOfDateEntry(const char *Path, uint64_t CachedSize, uint64_t ActualSize)
+        : Path(Path), Info(SizeChangedInfo{CachedSize, ActualSize}) {}
+  };
+
+  /// Visits all cached entries and re-stat an entry using UnderlyingFS to check
+  /// if the cache contains out-of-date entries. An entry can be out-of-date for
+  /// two reasons:
+  ///  1. The entry contains a stat error, indicating the file did not exist
+  ///     in the cache, but the file exists on the UnderlyingFS.
+  ///  2. The entry is associated with a file whose size is different from the
+  ///     size of the file on the same path on the UnderlyingFS.
+  std::vector<OutOfDateEntry>
+  getOutOfDateEntries(llvm::vfs::FileSystem &UnderlyingFS) const;
 
 private:
   std::unique_ptr<CacheShard[]> CacheShards;
@@ -351,7 +371,7 @@ class DependencyScanningWorkerFilesystem
   static const char ID;
 
   DependencyScanningWorkerFilesystem(
-      DependencyScanningService &Service,
+      DependencyScanningFilesystemSharedCache &SharedCache,
       IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
@@ -437,7 +457,10 @@ class DependencyScanningWorkerFilesystem
   /// Returns entry associated with the unique ID in the shared cache or nullptr
   /// if none is found.
   const CachedFileSystemEntry *
-  findSharedEntryByUID(llvm::vfs::Status Stat) const;
+  findSharedEntryByUID(llvm::vfs::Status Stat) const {
+    return SharedCache.getShardForUID(Stat.getUniqueID())
+        .findEntryByUID(Stat.getUniqueID());
+  }
 
   /// Associates the given entry with the filename in the local cache and
   /// returns it.
@@ -451,14 +474,20 @@ class DependencyScanningWorkerFilesystem
   /// some. Otherwise, constructs new one with the given error code, associates
   /// it with the filename and returns the result.
   const CachedFileSystemEntry &
-  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC);
+  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
+    return SharedCache.getShardForFilename(Filename)
+        .getOrEmplaceEntryForFilename(Filename, EC);
+  }
 
   /// Returns entry associated with the filename in the shared cache if there is
   /// some. Otherwise, associates the given entry with the filename and returns
   /// it.
   const CachedFileSystemEntry &
   getOrInsertSharedEntryForFilename(StringRef Filename,
-                                    const CachedFileSystemEntry &Entry);
+                                    const CachedFileSystemEntry &Entry) {
+    return SharedCache.getShardForFilename(Filename)
+        .getOrInsertEntryForFilename(Filename, Entry);
+  }
 
   void printImpl(raw_ostream &OS, PrintType Type,
                  unsigned IndentLevel) const override {
@@ -471,9 +500,8 @@ class DependencyScanningWorkerFilesystem
   /// VFS.
   bool shouldBypass(StringRef Path) const;
 
-  /// The service associated with this VFS.
-  DependencyScanningService &Service;
-
+  /// The global cache shared between worker threads.
+  DependencyScanningFilesystemSharedCache &SharedCache;
   /// The local cache is used by the worker thread to cache file system queries
   /// locally instead of querying the global cache every time.
   DependencyScanningFilesystemLocalCache LocalCache;
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index ceaf3c2279e7..4e97c7bc9f36 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -87,8 +87,7 @@ class DependencyScanningService {
       ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
       bool EagerLoadModules = false, bool TraceVFS = false,
       std::time_t BuildSessionTimestamp =
-          llvm::sys::toTimeT(std::chrono::system_clock::now()),
-      bool CacheNegativeStats = true);
+          llvm::sys::toTimeT(std::chrono::system_clock::now()));
 
   ScanningMode getMode() const { return Mode; }
 
@@ -100,8 +99,6 @@ class DependencyScanningService {
 
   bool shouldTraceVFS() const { return TraceVFS; }
 
-  bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
-
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
@@ -119,7 +116,6 @@ class DependencyScanningService {
   const bool EagerLoadModules;
   /// Whether to trace VFS accesses.
   const bool TraceVFS;
-  const bool CacheNegativeStats;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
   /// The global module cache entries.
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
index cb5d7e36d21c..ee24e5d1543d 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
@@ -15,6 +15,7 @@
 #include "clang/Tooling/JSONCompilationDatabase.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
 #include <functional>
 #include <optional>
 #include <string>
@@ -56,6 +57,9 @@ struct TranslationUnitDeps {
   /// determined that the differences are benign for this compilation.
   std::vector<ModuleID> ClangModuleDeps;
 
+  /// A list of the C++20 named modules this translation unit depends on.
+  std::vector<std::string> NamedModuleDeps;
+
   /// The sequence of commands required to build the translation unit. Commands
   /// should be executed in order.
   ///
@@ -188,6 +192,14 @@ class FullDependencyConsumer : public DependencyConsumer {
     ContextHash = std::move(Hash);
   }
 
+  void handleProvidedAndRequiredStdCXXModules(
+      std::optional<P1689ModuleInfo> Provided,
+      std::vector<P1689ModuleInfo> Requires) override {
+    ModuleName = Provided ? Provided->ModuleName : "";
+    llvm::transform(Requires, std::back_inserter(NamedModuleDeps),
+                    [](const auto &Module) { return Module.ModuleName; });
+  }
+
   TranslationUnitDeps takeTranslationUnitDeps();
   ModuleDepsGraph takeModuleGraphDeps();
 
@@ -195,6 +207,8 @@ class FullDependencyConsumer : public DependencyConsumer {
   std::vector<std::string> Dependencies;
   std::vector<PrebuiltModuleDep> PrebuiltModuleDeps;
   llvm::MapVector<ModuleID, ModuleDeps> ClangModuleDeps;
+  std::string ModuleName;
+  std::vector<std::string> NamedModuleDeps;
   std::vector<ModuleID> DirectModuleDeps;
   std::vector<Command> Commands;
   std::string ContextHash;
diff --git a/clang/include/clang/Tooling/Transformer/Transformer.h b/clang/include/clang/Tooling/Transformer/Transformer.h
index 71b1fe81b951..ecf7c0912d2d 100644
--- a/clang/include/clang/Tooling/Transformer/Transformer.h
+++ b/clang/include/clang/Tooling/Transformer/Transformer.h
@@ -42,11 +42,6 @@ class TransformerImpl {
   virtual void
   onMatchImpl(const ast_matchers::MatchFinder::MatchResult &Result) = 0;
 };
-
-// FIXME: Use std::type_identity or backport when available.
-template <class T> struct type_identity {
-  using type = T;
-};
 } // namespace detail
 
 template <typename T> struct TransformerResult {
@@ -95,8 +90,8 @@ class Transformer : public ast_matchers::MatchFinder::MatchCallback {
   template <typename MetadataT>
   explicit Transformer(
       transformer::RewriteRuleWith<MetadataT> Rule,
-      std::function<void(llvm::Expected<TransformerResult<
-                             typename detail::type_identity<MetadataT>::type>>)>
+      std::function<void(
+          llvm::Expected<TransformerResult<llvm::type_identity_t<MetadataT>>>)>
           Consumer);
 
   /// N.B. Passes `this` pointer to `MatchFinder`.  So, this object should not
@@ -200,8 +195,8 @@ template <typename T> class WithMetadataImpl final : public TransformerImpl {
 template <typename MetadataT>
 Transformer::Transformer(
     transformer::RewriteRuleWith<MetadataT> Rule,
-    std::function<void(llvm::Expected<TransformerResult<
-                           typename detail::type_identity<MetadataT>::type>>)>
+    std::function<void(
+        llvm::Expected<TransformerResult<llvm::type_identity_t<MetadataT>>>)>
         Consumer)
     : Impl(std::make_unique<detail::WithMetadataImpl<MetadataT>>(
           std::move(Rule), std::move(Consumer))) {}
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index 7578bc37b5c6..ffc547398873 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -1118,10 +1118,9 @@ void emitFunctionInfo(raw_ostream &OS, const FunctionInfo &FI) {
     emitParamInfo(OS, PI);
 
   writer.write<uint16_t>(FI.ResultType.size());
-  writer.write(ArrayRef<char>{FI.ResultType.data(), FI.ResultType.size()});
+  writer.write(ArrayRef<char>{FI.ResultType});
   writer.write<uint16_t>(FI.SwiftReturnOwnership.size());
-  writer.write(ArrayRef<char>{FI.SwiftReturnOwnership.data(),
-                              FI.SwiftReturnOwnership.size()});
+  writer.write(ArrayRef<char>{FI.SwiftReturnOwnership});
 }
 
 /// Used to serialize the on-disk global function table.
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index eba3c3de3d09..dedfb396200f 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -248,7 +248,7 @@ RawComment *ASTContext::getRawCommentForDeclNoCacheImpl(
 
   // Decompose the location for the declaration and find the beginning of the
   // file buffer.
-  const std::pair<FileID, unsigned> DeclLocDecomp =
+  const FileIDAndOffset DeclLocDecomp =
       SourceMgr.getDecomposedLoc(RepresentativeLocForDecl);
 
   // Slow path.
@@ -1476,7 +1476,12 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
   }
 
   if (Target.getTriple().isAMDGPU() ||
-      (AuxTarget && AuxTarget->getTriple().isAMDGPU())) {
+      (Target.getTriple().isSPIRV() &&
+       Target.getTriple().getVendor() == llvm::Triple::AMD) ||
+      (AuxTarget &&
+       (AuxTarget->getTriple().isAMDGPU() ||
+        ((AuxTarget->getTriple().isSPIRV() &&
+          AuxTarget->getTriple().getVendor() == llvm::Triple::AMD))))) {
 #define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)                       \
   InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/AMDGPUTypes.def"
@@ -10510,92 +10515,11 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
   return false;
 }
 
-/// getSVETypeSize - Return SVE vector or predicate register size.
-static uint64_t getSVETypeSize(ASTContext &Context, const BuiltinType *Ty) {
-  assert(Ty->isSveVLSBuiltinType() && "Invalid SVE Type");
-  if (Ty->getKind() == BuiltinType::SveBool ||
-      Ty->getKind() == BuiltinType::SveCount)
-    return (Context.getLangOpts().VScaleMin * 128) / Context.getCharWidth();
-  return Context.getLangOpts().VScaleMin * 128;
-}
-
-bool ASTContext::areCompatibleSveTypes(QualType FirstType,
-                                       QualType SecondType) {
-  auto IsValidCast = [this](QualType FirstType, QualType SecondType) {
-    if (const auto *BT = FirstType->getAs<BuiltinType>()) {
-      if (const auto *VT = SecondType->getAs<VectorType>()) {
-        // Predicates have the same representation as uint8 so we also have to
-        // check the kind to make these types incompatible.
-        if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
-          return BT->getKind() == BuiltinType::SveBool;
-        else if (VT->getVectorKind() == VectorKind::SveFixedLengthData)
-          return VT->getElementType().getCanonicalType() ==
-                 FirstType->getSveEltType(*this);
-        else if (VT->getVectorKind() == VectorKind::Generic)
-          return getTypeSize(SecondType) == getSVETypeSize(*this, BT) &&
-                 hasSameType(VT->getElementType(),
-                             getBuiltinVectorTypeInfo(BT).ElementType);
-      }
-    }
-    return false;
-  };
-
-  return IsValidCast(FirstType, SecondType) ||
-         IsValidCast(SecondType, FirstType);
-}
-
-bool ASTContext::areLaxCompatibleSveTypes(QualType FirstType,
-                                          QualType SecondType) {
-  auto IsLaxCompatible = [this](QualType FirstType, QualType SecondType) {
-    const auto *BT = FirstType->getAs<BuiltinType>();
-    if (!BT)
-      return false;
-
-    const auto *VecTy = SecondType->getAs<VectorType>();
-    if (VecTy && (VecTy->getVectorKind() == VectorKind::SveFixedLengthData ||
-                  VecTy->getVectorKind() == VectorKind::Generic)) {
-      const LangOptions::LaxVectorConversionKind LVCKind =
-          getLangOpts().getLaxVectorConversions();
-
-      // Can not convert between sve predicates and sve vectors because of
-      // different size.
-      if (BT->getKind() == BuiltinType::SveBool &&
-          VecTy->getVectorKind() == VectorKind::SveFixedLengthData)
-        return false;
-
-      // If __ARM_FEATURE_SVE_BITS != N do not allow GNU vector lax conversion.
-      // "Whenever __ARM_FEATURE_SVE_BITS==N, GNUT implicitly
-      // converts to VLAT and VLAT implicitly converts to GNUT."
-      // ACLE Spec Version 00bet6, 3.7.3.2. Behavior common to vectors and
-      // predicates.
-      if (VecTy->getVectorKind() == VectorKind::Generic &&
-          getTypeSize(SecondType) != getSVETypeSize(*this, BT))
-        return false;
-
-      // If -flax-vector-conversions=all is specified, the types are
-      // certainly compatible.
-      if (LVCKind == LangOptions::LaxVectorConversionKind::All)
-        return true;
-
-      // If -flax-vector-conversions=integer is specified, the types are
-      // compatible if the elements are integer types.
-      if (LVCKind == LangOptions::LaxVectorConversionKind::Integer)
-        return VecTy->getElementType().getCanonicalType()->isIntegerType() &&
-               FirstType->getSveEltType(*this)->isIntegerType();
-    }
-
-    return false;
-  };
-
-  return IsLaxCompatible(FirstType, SecondType) ||
-         IsLaxCompatible(SecondType, FirstType);
-}
-
 /// getRVVTypeSize - Return RVV vector register size.
 static uint64_t getRVVTypeSize(ASTContext &Context, const BuiltinType *Ty) {
   assert(Ty->isRVVVLSBuiltinType() && "Invalid RVV Type");
-  auto VScale =
-      Context.getTargetInfo().getVScaleRange(Context.getLangOpts(), false);
+  auto VScale = Context.getTargetInfo().getVScaleRange(
+      Context.getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
   if (!VScale)
     return 0;
 
@@ -12580,6 +12504,10 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
       Type = Context.AMDGPUBufferRsrcTy;
       break;
     }
+    case 'c': {
+      Type = Context.AMDGPUFeaturePredicateTy;
+      break;
+    }
     default:
       llvm_unreachable("Unexpected target builtin type");
     }
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 2b7d67f389a6..90d309e49b26 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -1605,7 +1605,7 @@ ASTNodeImporter::VisitCountAttributedType(const CountAttributedType *T) {
 
   return Importer.getToContext().getCountAttributedType(
       *ToWrappedTypeOrErr, CountExpr, T->isCountInBytes(), T->isOrNull(),
-      ArrayRef(CoupledDecls.data(), CoupledDecls.size()));
+      ArrayRef(CoupledDecls));
 }
 
 ExpectedType ASTNodeImporter::VisitTemplateTypeParmType(
@@ -6337,8 +6337,8 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
 
     if (GetImportedOrCreateDecl<ClassTemplatePartialSpecializationDecl>(
             D2, D, Importer.getToContext(), D->getTagKind(), DC, *BeginLocOrErr,
-            *IdLocOrErr, ToTPList, ClassTemplate,
-            ArrayRef(TemplateArgs.data(), TemplateArgs.size()), CanonInjType,
+            *IdLocOrErr, ToTPList, ClassTemplate, ArrayRef(TemplateArgs),
+            CanonInjType,
             cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl)))
       return D2;
 
@@ -9989,7 +9989,7 @@ Expected<SourceLocation> ASTImporter::Import(SourceLocation FromLoc) {
   SourceManager &FromSM = FromContext.getSourceManager();
   bool IsBuiltin = FromSM.isWrittenInBuiltinFile(FromLoc);
 
-  std::pair<FileID, unsigned> Decomposed = FromSM.getDecomposedLoc(FromLoc);
+  FileIDAndOffset Decomposed = FromSM.getDecomposedLoc(FromLoc);
   Expected<FileID> ToFileIDOrErr = Import(Decomposed.first, IsBuiltin);
   if (!ToFileIDOrErr)
     return ToFileIDOrErr.takeError();
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 81da16e797bc..d1c93e469466 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -4574,22 +4574,23 @@ VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD,
 template <class Emitter>
 bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
                                            bool ConstantContext) {
-  std::optional<PrimType> VarT = classify(VD->getType());
 
   // We only create variables if we're evaluating in a constant context.
   // Otherwise, just evaluate the initializer and return it.
   if (!ConstantContext) {
     DeclScope<Emitter> LS(this, VD);
-    if (!this->visit(VD->getAnyInitializer()))
+    const Expr *Init = VD->getInit();
+    if (!this->visit(Init))
       return false;
-    return this->emitRet(VarT.value_or(PT_Ptr), VD) && LS.destroyLocals() &&
-           this->emitCheckAllocations(VD);
+    return this->emitRet(classify(Init).value_or(PT_Ptr), VD) &&
+           LS.destroyLocals() && this->emitCheckAllocations(VD);
   }
 
   LocalScope<Emitter> VDScope(this, VD);
   if (!this->visitVarDecl(VD, /*Toplevel=*/true))
     return false;
 
+  std::optional<PrimType> VarT = classify(VD->getType());
   if (Context::shouldBeGloballyIndexed(VD)) {
     auto GlobalIndex = P.getGlobal(VD);
     assert(GlobalIndex); // visitVarDecl() didn't return false.
@@ -5274,7 +5275,7 @@ bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
 template <class Emitter>
 bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
   if (auto *DD = dyn_cast_if_present<DecompositionDecl>(VD)) {
-    for (auto *BD : DD->bindings())
+    for (auto *BD : DD->flat_bindings())
       if (auto *KD = BD->getHoldingVar(); KD && !this->visitVarDecl(KD))
         return false;
   }
@@ -5544,7 +5545,6 @@ bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
   const Stmt *BeginStmt = S->getBeginStmt();
   const Stmt *RangeStmt = S->getRangeStmt();
   const Stmt *EndStmt = S->getEndStmt();
-  const VarDecl *LoopVar = S->getLoopVariable();
 
   LabelTy EndLabel = this->getLabel();
   LabelTy CondLabel = this->getLabel();
@@ -5569,7 +5569,7 @@ bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
   if (!this->jumpFalse(EndLabel))
     return false;
 
-  if (!this->visitVarDecl(LoopVar))
+  if (!this->visitDeclStmt(S->getLoopVarStmt(), /*EvaluateConditionDecl=*/true))
     return false;
 
   // Body.
@@ -5835,6 +5835,8 @@ bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
     return false;
 
   if (R->isUnion() && Ctor->isCopyOrMoveConstructor()) {
+    if (R->getNumFields() == 0)
+      return this->emitRetVoid(Ctor);
     // union copy and move ctors are special.
     assert(cast<CompoundStmt>(Ctor->getBody())->body_empty());
     if (!this->emitThis(Ctor))
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index a1d068cc7e0a..1a5fd8678587 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -574,17 +574,17 @@ template <class Emitter> class LocalScope : public VariableScope<Emitter> {
     // Emit destructor calls for local variables of record
     // type with a destructor.
     for (Scope::Local &Local : llvm::reverse(this->Ctx->Descriptors[*Idx])) {
-      if (!Local.Desc->isPrimitive() && !Local.Desc->isPrimitiveArray()) {
-        if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
-          return false;
+      if (Local.Desc->hasTrivialDtor())
+        continue;
+      if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
+        return false;
 
-        if (!this->Ctx->emitDestruction(Local.Desc, Local.Desc->getLoc()))
-          return false;
+      if (!this->Ctx->emitDestruction(Local.Desc, Local.Desc->getLoc()))
+        return false;
 
-        if (!this->Ctx->emitPopPtr(E))
-          return false;
-        removeIfStoredOpaqueValue(Local);
-      }
+      if (!this->Ctx->emitPopPtr(E))
+        return false;
+      removeIfStoredOpaqueValue(Local);
     }
     return true;
   }
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 46e4d0d940b3..c89eca9bef44 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -502,6 +502,21 @@ SourceInfo Descriptor::getLoc() const {
   llvm_unreachable("Invalid descriptor type");
 }
 
+bool Descriptor::hasTrivialDtor() const {
+  if (isPrimitive() || isPrimitiveArray() || isDummy())
+    return true;
+
+  if (isRecord()) {
+    assert(ElemRecord);
+    const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
+    return !Dtor || Dtor->isTrivial();
+  }
+
+  // Composite arrays.
+  assert(ElemDesc);
+  return ElemDesc->hasTrivialDtor();
+}
+
 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
 
 InitMap::InitMap(unsigned N)
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index f25ad8f4c758..4591eabb69bb 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -281,6 +281,9 @@ struct Descriptor final {
   /// Checks if this is a dummy descriptor.
   bool isDummy() const { return IsDummy; }
 
+  /// Whether variables of this descriptor need their destructor called or not.
+  bool hasTrivialDtor() const;
+
   void dump() const;
   void dump(llvm::raw_ostream &OS) const;
   void dumpFull(unsigned Offset = 0, unsigned Indent = 0) const;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index a3eecd06369b..f64501f4a31e 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -63,7 +63,7 @@ template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
 
   std::string S;
   llvm::raw_string_ostream SS(S);
-  SS << Result;
+  SS << std::move(Result);
   return S;
 }
 
@@ -81,7 +81,7 @@ inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
 
   std::string Str;
   llvm::raw_string_ostream SS(Str);
-  SS << Result;
+  SS << std::move(Result);
   return Str;
 }
 
@@ -99,7 +99,7 @@ inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
 
   std::string Str;
   llvm::raw_string_ostream SS(Str);
-  SS << Result;
+  SS << std::move(Result);
   return Str;
 }
 
@@ -109,7 +109,7 @@ template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {
 
   std::string Result;
   llvm::raw_string_ostream SS(Result);
-  SS << F;
+  SS << std::move(F);
   return Result;
 }
 
diff --git a/clang/lib/AST/ByteCode/FunctionPointer.cpp b/clang/lib/AST/ByteCode/FunctionPointer.cpp
index 4ab7af170efe..248862669728 100644
--- a/clang/lib/AST/ByteCode/FunctionPointer.cpp
+++ b/clang/lib/AST/ByteCode/FunctionPointer.cpp
@@ -27,8 +27,6 @@ void FunctionPointer::print(llvm::raw_ostream &OS) const {
   OS << "FnPtr(";
   if (Func)
     OS << Func->getName();
-  else if (Func)
-    OS << reinterpret_cast<uintptr_t>(Func);
   else
     OS << "nullptr";
   OS << ")";
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index dcc458775197..85a07abb5ed0 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -552,11 +552,6 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    // Result.atIndex(0).deref<Floating>() = Floating(ResR);
-    // Result.atIndex(0).initialize();
-    // Result.atIndex(1).deref<Floating>() = Floating(ResI);
-    // Result.atIndex(1).initialize();
-
     Floating RA = S.allocFloat(A.getSemantics());
     RA.copy(ResR);
     Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
@@ -565,6 +560,7 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     Floating RI = S.allocFloat(A.getSemantics());
     RI.copy(ResI);
     Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
+    Result.atIndex(1).initialize();
 
     Result.initialize();
   } else {
@@ -1143,30 +1139,12 @@ inline bool CmpHelperEQ<Pointer>(InterpState &S, CodePtr OpPC, CompareFn Fn) {
   }
 
   if (Pointer::hasSameBase(LHS, RHS)) {
-    if (LHS.inUnion() && RHS.inUnion()) {
-      // If the pointers point into a union, things are a little more
-      // complicated since the offset we save in interp::Pointer can't be used
-      // to compare the pointers directly.
-      size_t A = LHS.computeOffsetForComparison();
-      size_t B = RHS.computeOffsetForComparison();
-      S.Stk.push<BoolT>(BoolT::from(Fn(Compare(A, B))));
-      return true;
-    }
-
-    unsigned VL = LHS.getByteOffset();
-    unsigned VR = RHS.getByteOffset();
-    // In our Pointer class, a pointer to an array and a pointer to the first
-    // element in the same array are NOT equal. They have the same Base value,
-    // but a different Offset. This is a pretty rare case, so we fix this here
-    // by comparing pointers to the first elements.
-    if (!LHS.isZero() && LHS.isArrayRoot())
-      VL = LHS.atIndex(0).getByteOffset();
-    if (!RHS.isZero() && RHS.isArrayRoot())
-      VR = RHS.atIndex(0).getByteOffset();
-
-    S.Stk.push<BoolT>(BoolT::from(Fn(Compare(VL, VR))));
+    size_t A = LHS.computeOffsetForComparison();
+    size_t B = RHS.computeOffsetForComparison();
+    S.Stk.push<BoolT>(BoolT::from(Fn(Compare(A, B))));
     return true;
   }
+
   // Otherwise we need to do a bunch of extra checks before returning Unordered.
   if (LHS.isOnePastEnd() && !RHS.isOnePastEnd() && !RHS.isZero() &&
       RHS.getOffset() == 0) {
@@ -1332,20 +1310,6 @@ bool GE(InterpState &S, CodePtr OpPC) {
   });
 }
 
-//===----------------------------------------------------------------------===//
-// InRange
-//===----------------------------------------------------------------------===//
-
-template <PrimType Name, class T = typename PrimConv<Name>::T>
-bool InRange(InterpState &S, CodePtr OpPC) {
-  const T RHS = S.Stk.pop<T>();
-  const T LHS = S.Stk.pop<T>();
-  const T Value = S.Stk.pop<T>();
-
-  S.Stk.push<bool>(LHS <= Value && Value <= RHS);
-  return true;
-}
-
 //===----------------------------------------------------------------------===//
 // Dup, Pop, Test
 //===----------------------------------------------------------------------===//
@@ -1966,8 +1930,10 @@ bool StoreBitField(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckStore(S, OpPC, Ptr))
     return false;
-  if (Ptr.canBeInitialized())
+  if (Ptr.canBeInitialized()) {
     Ptr.initialize();
+    Ptr.activate();
+  }
   if (const auto *FD = Ptr.getField())
     Ptr.deref<T>() = Value.truncate(FD->getBitWidthValue());
   else
@@ -1981,8 +1947,10 @@ bool StoreBitFieldPop(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
   if (!CheckStore(S, OpPC, Ptr))
     return false;
-  if (Ptr.canBeInitialized())
+  if (Ptr.canBeInitialized()) {
     Ptr.initialize();
+    Ptr.activate();
+  }
   if (const auto *FD = Ptr.getField())
     Ptr.deref<T>() = Value.truncate(FD->getBitWidthValue());
   else
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 93002172949a..db18ad2aa9bc 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -555,8 +555,8 @@ static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
   APSInt FPClassArg = popToAPSInt(S.Stk, FPClassArgT);
   const Floating &F = S.Stk.pop<Floating>();
 
-  int32_t Result =
-      static_cast<int32_t>((F.classify() & FPClassArg).getZExtValue());
+  int32_t Result = static_cast<int32_t>(
+      (F.classify() & std::move(FPClassArg)).getZExtValue());
   pushInteger(S, Result, Call->getType());
 
   return true;
@@ -856,12 +856,14 @@ static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
 
     if (!APSInt::isSameValue(Temp, Result))
       Overflow = true;
-    Result = Temp;
+    Result = std::move(Temp);
   }
 
   // Write Result to ResultPtr and put Overflow on the stack.
   assignInteger(S, ResultPtr, ResultT, Result);
-  ResultPtr.initialize();
+  if (ResultPtr.canBeInitialized())
+    ResultPtr.initialize();
+
   assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
   S.Stk.push<Boolean>(Overflow);
   return true;
@@ -1135,17 +1137,17 @@ static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
 
   if (isIntegralType(FirstArgT)) {
     const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);
-    APSInt Align = Alignment.extOrTrunc(Src.getBitWidth());
+    APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;
     if (BuiltinOp == Builtin::BI__builtin_align_up) {
       APSInt AlignedVal =
-          APSInt((Src + (Align - 1)) & ~(Align - 1), Src.isUnsigned());
+          APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
       pushInteger(S, AlignedVal, Call->getType());
     } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
-      APSInt AlignedVal = APSInt(Src & ~(Align - 1), Src.isUnsigned());
+      APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
       pushInteger(S, AlignedVal, Call->getType());
     } else {
       assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
-      S.Stk.push<Boolean>((Src & (Align - 1)) == 0);
+      S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);
     }
     return true;
   }
@@ -1425,7 +1427,7 @@ static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
 
   QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
   PrimType CarryOutT = *S.getContext().classify(CarryOutType);
-  assignInteger(S, CarryOutPtr, CarryOutT, APSInt(Result, true));
+  assignInteger(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));
 
   pushInteger(S, CarryOut, Call->getType());
 
@@ -1531,34 +1533,21 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
   std::optional<PrimType> ElemT = S.getContext().classify(ElemType);
   DynamicAllocator &Allocator = S.getAllocator();
   if (ElemT) {
-    if (IsArray) {
-      Block *B = Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
-                                    S.Ctx.getEvalID(),
-                                    DynamicAllocator::Form::Operator);
-      assert(B);
-      S.Stk.push<Pointer>(Pointer(B).atIndex(0));
-      return true;
-    }
-
-    const Descriptor *Desc = S.P.createDescriptor(
-        NewCall, *ElemT, ElemType.getTypePtr(), Descriptor::InlineDescMD,
-        /*IsConst=*/false, /*IsTemporary=*/false,
-        /*IsMutable=*/false);
-    Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
-                                  DynamicAllocator::Form::Operator);
+    Block *B =
+        Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
+                           S.Ctx.getEvalID(), DynamicAllocator::Form::Operator);
     assert(B);
-
-    S.Stk.push<Pointer>(B);
+    S.Stk.push<Pointer>(Pointer(B).atIndex(0));
     return true;
   }
 
   assert(!ElemT);
-  // Structs etc.
-  const Descriptor *Desc =
-      S.P.createDescriptor(NewCall, ElemType.getTypePtr(),
-                           IsArray ? std::nullopt : Descriptor::InlineDescMD);
 
+  // Composite arrays
   if (IsArray) {
+    const Descriptor *Desc =
+        S.P.createDescriptor(NewCall, ElemType.getTypePtr(),
+                             IsArray ? std::nullopt : Descriptor::InlineDescMD);
     Block *B =
         Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
                            DynamicAllocator::Form::Operator);
@@ -1567,10 +1556,17 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
     return true;
   }
 
+  // Records. Still allocate them as single-element arrays.
+  QualType AllocType = S.getASTContext().getConstantArrayType(
+      ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);
+
+  const Descriptor *Desc =
+      S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
+                           IsArray ? std::nullopt : Descriptor::InlineDescMD);
   Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
                                 DynamicAllocator::Form::Operator);
   assert(B);
-  S.Stk.push<Pointer>(B);
+  S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 2569cac018b3..feac97d4b1a6 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -335,7 +335,8 @@ bool clang::interp::DoBitCast(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
   BitcastBuffer Buffer(FullBitWidth);
   size_t BuffSize = FullBitWidth.roundToBytes();
-  if (!CheckBitcastType(S, OpPC, Ptr.getType(), /*IsToType=*/false))
+  QualType DataType = Ptr.getFieldDesc()->getDataType(S.getASTContext());
+  if (!CheckBitcastType(S, OpPC, DataType, /*IsToType=*/false))
     return false;
 
   bool Success = readPointerToBuffer(S.getContext(), Ptr, Buffer,
@@ -370,8 +371,8 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
   assert(FromPtr.isBlockPointer());
   assert(ToPtr.isBlockPointer());
 
-  QualType FromType = FromPtr.getType();
-  QualType ToType = ToPtr.getType();
+  QualType FromType = FromPtr.getFieldDesc()->getDataType(S.getASTContext());
+  QualType ToType = ToPtr.getFieldDesc()->getDataType(S.getASTContext());
 
   if (!CheckBitcastType(S, OpPC, ToType, /*IsToType=*/true))
     return false;
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index a849ac71a726..a5a4bd25fe71 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -8,7 +8,6 @@
 
 #include "InterpFrame.h"
 #include "Boolean.h"
-#include "Floating.h"
 #include "Function.h"
 #include "InterpStack.h"
 #include "InterpState.h"
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index f0b0384f32ac..e91531627217 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -176,19 +176,8 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   if (const auto *VD = Desc->asValueDecl())
     Base = VD;
   else if (const auto *E = Desc->asExpr()) {
-    // Create a DynamicAlloc base of the right type.
-    if (const auto *NewExpr = dyn_cast<CXXNewExpr>(E)) {
-      QualType AllocatedType;
-      if (NewExpr->isArray()) {
-        assert(Desc->isArray());
-        APInt ArraySize(64, static_cast<uint64_t>(Desc->getNumElems()),
-                        /*IsSigned=*/false);
-        AllocatedType =
-            ASTCtx.getConstantArrayType(NewExpr->getAllocatedType(), ArraySize,
-                                        nullptr, ArraySizeModifier::Normal, 0);
-      } else {
-        AllocatedType = NewExpr->getAllocatedType();
-      }
+    if (block()->isDynamic()) {
+      QualType AllocatedType = getDeclPtr().getFieldDesc()->getDataType(ASTCtx);
       // FIXME: Suboptimal counting of dynamic allocations. Move this to Context
       // or InterpState?
       static int ReportedDynamicAllocs = 0;
@@ -218,7 +207,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
 
   bool UsePath = true;
   if (const ValueDecl *VD = getDeclDesc()->asValueDecl();
-      VD && VD->getType()->isLValueReferenceType())
+      VD && VD->getType()->isReferenceType())
     UsePath = false;
 
   // Build the path into the object.
@@ -349,16 +338,28 @@ void Pointer::print(llvm::raw_ostream &OS) const {
   }
 }
 
-/// Compute an integer that can be used to compare this pointer to
-/// another one.
 size_t Pointer::computeOffsetForComparison() const {
+  if (isIntegralPointer())
+    return asIntPointer().Value + Offset;
+  if (isTypeidPointer())
+    return reinterpret_cast<uintptr_t>(asTypeidPointer().TypePtr) + Offset;
+
   if (!isBlockPointer())
     return Offset;
 
   size_t Result = 0;
   Pointer P = *this;
-  while (!P.isRoot()) {
-    if (P.isArrayRoot()) {
+  while (true) {
+
+    if (P.isVirtualBaseClass()) {
+      Result += getInlineDesc()->Offset;
+      P = P.getBase();
+      continue;
+    }
+
+    if (P.isBaseClass()) {
+      if (P.getRecord()->getNumVirtualBases() > 0)
+        Result += P.getInlineDesc()->Offset;
       P = P.getBase();
       continue;
     }
@@ -369,14 +370,26 @@ size_t Pointer::computeOffsetForComparison() const {
       continue;
     }
 
+    if (P.isRoot()) {
+      if (P.isOnePastEnd())
+        ++Result;
+      break;
+    }
+
     if (const Record *R = P.getBase().getRecord(); R && R->isUnion()) {
       // Direct child of a union - all have offset 0.
       P = P.getBase();
       continue;
     }
 
+    // Fields, etc.
     Result += P.getInlineDesc()->Offset;
+    if (P.isOnePastEnd())
+      ++Result;
+
     P = P.getBase();
+    if (P.isRoot())
+      break;
   }
 
   return Result;
@@ -492,8 +505,19 @@ void Pointer::activate() const {
   auto activate = [](Pointer &P) -> void {
     P.getInlineDesc()->IsActive = true;
   };
-  auto deactivate = [](Pointer &P) -> void {
+
+  std::function<void(Pointer &)> deactivate;
+  deactivate = [&deactivate](Pointer &P) -> void {
     P.getInlineDesc()->IsActive = false;
+
+    if (const Record *R = P.getRecord()) {
+      for (const Record::Field &F : R->fields()) {
+        Pointer FieldPtr = P.atField(F.Offset);
+        if (FieldPtr.getInlineDesc()->IsActive)
+          deactivate(FieldPtr);
+      }
+      // FIXME: Bases?
+    }
   };
 
   // Unions might be nested etc., so find the topmost Pointer that's
@@ -503,21 +527,31 @@ void Pointer::activate() const {
     UnionPtr = UnionPtr.getBase();
 
   assert(UnionPtr.getFieldDesc()->isUnion());
-
   const Record *UnionRecord = UnionPtr.getRecord();
+
+  // The direct child pointer of the union that's on the path from
+  // this pointer to the union.
+  Pointer ChildPtr = *this;
+  assert(ChildPtr != UnionPtr);
+  while (true) {
+    if (ChildPtr.getBase() == UnionPtr)
+      break;
+    ChildPtr = ChildPtr.getBase();
+  }
+  assert(ChildPtr.getBase() == UnionPtr);
+
   for (const Record::Field &F : UnionRecord->fields()) {
     Pointer FieldPtr = UnionPtr.atField(F.Offset);
-    if (FieldPtr == *this) {
+    if (FieldPtr == ChildPtr) {
+      // No need to deactivate, will be activated in the next loop.
     } else {
       deactivate(FieldPtr);
-      // FIXME: Recurse.
     }
   }
 
   Pointer B = *this;
   while (B != UnionPtr) {
     activate(B);
-    // FIXME: Need to de-activate other fields of parent records.
     B = B.getBase();
   }
 }
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 0234ab02ab8f..d525f84fd660 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -761,6 +761,9 @@ class Pointer {
   /// Prints the pointer.
   void print(llvm::raw_ostream &OS) const;
 
+  /// Compute an integer that can be used to compare this pointer to
+  /// another one. This is usually NOT the same as the pointer offset
+  /// regarding the AST record layout.
   size_t computeOffsetForComparison() const;
 
 private:
diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp
index d022c5c845ae..cd73d2709dc6 100644
--- a/clang/lib/AST/Comment.cpp
+++ b/clang/lib/AST/Comment.cpp
@@ -291,6 +291,13 @@ void DeclInfo::fill() {
     TemplateParameters = CTPSD->getTemplateParameters();
     break;
   }
+  case Decl::VarTemplatePartialSpecialization: {
+    const auto *VTPSD = cast<VarTemplatePartialSpecializationDecl>(CommentDecl);
+    Kind = VariableKind;
+    TemplateKind = TemplatePartialSpecialization;
+    TemplateParameters = VTPSD->getTemplateParameters();
+    break;
+  }
   case Decl::ClassTemplateSpecialization:
     Kind = ClassKind;
     TemplateKind = TemplateSpecialization;
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index 804be89a8d4d..e19c2327aebd 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -904,7 +904,7 @@ void Lexer::lex(Token &T) {
 StringRef Lexer::getSpelling(const Token &Tok,
                              const SourceManager &SourceMgr) const {
   SourceLocation Loc = Tok.getLocation();
-  std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SourceMgr.getDecomposedLoc(Loc);
 
   bool InvalidTemp = false;
   StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 35c41859595d..e0362245d6ec 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -66,7 +66,6 @@
 #include <cassert>
 #include <cstddef>
 #include <cstring>
-#include <memory>
 #include <optional>
 #include <string>
 #include <tuple>
@@ -2445,24 +2444,16 @@ bool VarDecl::hasInitWithSideEffects() const {
   if (!hasInit())
     return false;
 
-  // Check if we can get the initializer without deserializing
-  const Expr *E = nullptr;
-  if (auto *S = dyn_cast<Stmt *>(Init)) {
-    E = cast<Expr>(S);
-  } else {
-    E = cast_or_null<Expr>(getEvaluatedStmt()->Value.getWithoutDeserializing());
+  EvaluatedStmt *ES = ensureEvaluatedStmt();
+  if (!ES->CheckedForSideEffects) {
+    const Expr *E = getInit();
+    ES->HasSideEffects =
+        E->HasSideEffects(getASTContext()) &&
+        // We can get a value-dependent initializer during error recovery.
+        (E->isValueDependent() || !evaluateValue());
+    ES->CheckedForSideEffects = true;
   }
-
-  if (E)
-    return E->HasSideEffects(getASTContext()) &&
-           // We can get a value-dependent initializer during error recovery.
-           (E->isValueDependent() || !evaluateValue());
-
-  assert(getEvaluatedStmt()->Value.isOffset());
-  // ASTReader tracks this without having to deserialize the initializer
-  if (auto Source = getASTContext().getExternalSource())
-    return Source->hasInitializerWithSideEffects(this);
-  return false;
+  return ES->HasSideEffects;
 }
 
 bool VarDecl::isOutOfLine() const {
@@ -5878,21 +5869,21 @@ bool HLSLBufferDecl::buffer_decls_empty() {
 // HLSLRootSignatureDecl Implementation
 //===----------------------------------------------------------------------===//
 
-HLSLRootSignatureDecl::HLSLRootSignatureDecl(DeclContext *DC,
-                                             SourceLocation Loc,
-                                             IdentifierInfo *ID,
-                                             unsigned NumElems)
+HLSLRootSignatureDecl::HLSLRootSignatureDecl(
+    DeclContext *DC, SourceLocation Loc, IdentifierInfo *ID,
+    llvm::dxbc::RootSignatureVersion Version, unsigned NumElems)
     : NamedDecl(Decl::Kind::HLSLRootSignature, DC, Loc, DeclarationName(ID)),
-      NumElems(NumElems) {}
+      Version(Version), NumElems(NumElems) {}
 
 HLSLRootSignatureDecl *HLSLRootSignatureDecl::Create(
     ASTContext &C, DeclContext *DC, SourceLocation Loc, IdentifierInfo *ID,
+    llvm::dxbc::RootSignatureVersion Version,
     ArrayRef<llvm::hlsl::rootsig::RootElement> RootElements) {
   HLSLRootSignatureDecl *RSDecl =
       new (C, DC,
            additionalSizeToAlloc<llvm::hlsl::rootsig::RootElement>(
                RootElements.size()))
-          HLSLRootSignatureDecl(DC, Loc, ID, RootElements.size());
+          HLSLRootSignatureDecl(DC, Loc, ID, Version, RootElements.size());
   auto *StoredElems = RSDecl->getElems();
   llvm::uninitialized_copy(RootElements, StoredElems);
   return RSDecl;
@@ -5901,7 +5892,9 @@ HLSLRootSignatureDecl *HLSLRootSignatureDecl::Create(
 HLSLRootSignatureDecl *
 HLSLRootSignatureDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
   HLSLRootSignatureDecl *Result = new (C, ID)
-      HLSLRootSignatureDecl(nullptr, SourceLocation(), nullptr, /*NumElems=*/0);
+      HLSLRootSignatureDecl(nullptr, SourceLocation(), nullptr,
+                            /*Version*/ llvm::dxbc::RootSignatureVersion::V1_1,
+                            /*NumElems=*/0);
   return Result;
 }
 
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 48c60aa4e449..13c46fdbff96 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -1133,6 +1133,8 @@ bool Decl::isInExportDeclContext() const {
 }
 
 bool Decl::isModuleLocal() const {
+  if (isa<NamespaceDecl, TranslationUnitDecl>(this))
+    return false;
   auto *M = getOwningModule();
   return M && M->isNamedModule() &&
          getModuleOwnershipKind() == ModuleOwnershipKind::ReachableWhenImported;
diff --git a/clang/lib/AST/DeclObjC.cpp b/clang/lib/AST/DeclObjC.cpp
index 5cf0e9a7b259..a66eb7298108 100644
--- a/clang/lib/AST/DeclObjC.cpp
+++ b/clang/lib/AST/DeclObjC.cpp
@@ -27,7 +27,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <cstring>
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 4edbcda9a456..73b272d6f54d 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -1356,8 +1356,7 @@ StringLiteral::getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
     SourceLocation StrTokSpellingLoc = SM.getSpellingLoc(StrTokLoc);
 
     // Re-lex the token to get its length and original spelling.
-    std::pair<FileID, unsigned> LocInfo =
-        SM.getDecomposedLoc(StrTokSpellingLoc);
+    FileIDAndOffset LocInfo = SM.getDecomposedLoc(StrTokSpellingLoc);
     bool Invalid = false;
     StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
     if (Invalid) {
@@ -2021,7 +2020,8 @@ CXXBaseSpecifier **CastExpr::path_buffer() {
 #define ABSTRACT_STMT(x)
 #define CASTEXPR(Type, Base)                                                   \
   case Stmt::Type##Class:                                                      \
-    return static_cast<Type *>(this)->getTrailingObjects<CXXBaseSpecifier *>();
+    return static_cast<Type *>(this)                                           \
+        ->getTrailingObjectsNonStrict<CXXBaseSpecifier *>();
 #define STMT(Type, Base)
 #include "clang/AST/StmtNodes.inc"
   default:
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 063eb1738a04..5833a6405125 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -600,7 +600,7 @@ CXXOperatorCallExpr::CXXOperatorCallExpr(OverloadedOperatorKind OpKind,
   assert(
       (CXXOperatorCallExprBits.OperatorKind == static_cast<unsigned>(OpKind)) &&
       "OperatorKind overflow!");
-  Range = getSourceRangeImpl();
+  BeginLoc = getSourceRangeImpl().getBegin();
 }
 
 CXXOperatorCallExpr::CXXOperatorCallExpr(unsigned NumArgs, bool HasFPFeatures,
@@ -1208,7 +1208,7 @@ CXXConstructExpr::CXXConstructExpr(
 
   Stmt **TrailingArgs = getTrailingArgs();
   llvm::copy(Args, TrailingArgs);
-  assert(llvm::all_of(Args, [](const Stmt *Arg) { return Arg != nullptr; }));
+  assert(!llvm::is_contained(Args, nullptr));
 
   // CXXTemporaryObjectExpr does this itself after setting its TypeSourceInfo.
   if (SC == CXXConstructExprClass)
diff --git a/clang/lib/AST/ExprConcepts.cpp b/clang/lib/AST/ExprConcepts.cpp
index a2cf431a312a..ac0e566fe6e7 100644
--- a/clang/lib/AST/ExprConcepts.cpp
+++ b/clang/lib/AST/ExprConcepts.cpp
@@ -23,7 +23,6 @@
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include <algorithm>
 
 using namespace clang;
 
diff --git a/clang/lib/AST/ExprObjC.cpp b/clang/lib/AST/ExprObjC.cpp
index 3df9c404b5ca..50d3a4474c4d 100644
--- a/clang/lib/AST/ExprObjC.cpp
+++ b/clang/lib/AST/ExprObjC.cpp
@@ -17,7 +17,6 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "llvm/Support/ErrorHandling.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 84936b72bb4f..8a1d4e8313e2 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -4273,7 +4273,8 @@ void CXXNameMangler::mangleRISCVFixedRVVVectorType(const VectorType *T) {
 
   // Apend the LMUL suffix.
   auto VScale = getASTContext().getTargetInfo().getVScaleRange(
-      getASTContext().getLangOpts(), false);
+      getASTContext().getLangOpts(),
+      TargetInfo::ArmStreamingKind::NotStreaming);
   unsigned VLen = VScale->first * llvm::RISCV::RVVBitsPerBlock;
 
   if (T->getVectorKind() == VectorKind::RVVFixedLengthData) {
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index f8446dfbc685..7fdfcfa3014f 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -30,19 +30,21 @@ void ODRHash::AddIdentifierInfo(const IdentifierInfo *II) {
   ID.AddString(II->getName());
 }
 
-void ODRHash::AddDeclarationName(DeclarationName Name, bool TreatAsDecl) {
+void ODRHash::AddDeclarationNameInfo(DeclarationNameInfo NameInfo,
+                                     bool TreatAsDecl) {
   if (TreatAsDecl)
     // Matches the NamedDecl check in AddDecl
     AddBoolean(true);
 
-  AddDeclarationNameImpl(Name);
+  AddDeclarationNameInfoImpl(NameInfo);
 
   if (TreatAsDecl)
     // Matches the ClassTemplateSpecializationDecl check in AddDecl
     AddBoolean(false);
 }
 
-void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
+void ODRHash::AddDeclarationNameInfoImpl(DeclarationNameInfo NameInfo) {
+  DeclarationName Name = NameInfo.getName();
   // Index all DeclarationName and use index numbers to refer to them.
   auto Result = DeclNameMap.insert(std::make_pair(Name, DeclNameMap.size()));
   ID.AddInteger(Result.first->second);
@@ -85,7 +87,11 @@ void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
   }
   case DeclarationName::CXXConstructorName:
   case DeclarationName::CXXDestructorName:
-    AddQualType(Name.getCXXNameType());
+  case DeclarationName::CXXConversionFunctionName:
+    if (auto *TSI = NameInfo.getNamedTypeInfo())
+      AddQualType(TSI->getType());
+    else
+      AddQualType(Name.getCXXNameType());
     break;
   case DeclarationName::CXXOperatorName:
     ID.AddInteger(Name.getCXXOverloadedOperator());
@@ -93,9 +99,6 @@ void ODRHash::AddDeclarationNameImpl(DeclarationName Name) {
   case DeclarationName::CXXLiteralOperatorName:
     AddIdentifierInfo(Name.getCXXLiteralIdentifier());
     break;
-  case DeclarationName::CXXConversionFunctionName:
-    AddQualType(Name.getCXXNameType());
-    break;
   case DeclarationName::CXXUsingDirective:
     break;
   case DeclarationName::CXXDeductionGuideName: {
@@ -314,7 +317,10 @@ class ODRDeclVisitor : public ConstDeclVisitor<ODRDeclVisitor> {
   }
 
   void VisitNamedDecl(const NamedDecl *D) {
-    Hash.AddDeclarationName(D->getDeclName());
+    if (const auto *FD = dyn_cast<FunctionDecl>(D))
+      Hash.AddDeclarationNameInfo(FD->getNameInfo());
+    else
+      Hash.AddDeclarationName(D->getDeclName());
     Inherited::VisitNamedDecl(D);
   }
 
@@ -828,7 +834,10 @@ void ODRHash::AddDecl(const Decl *D) {
     return;
   }
 
-  AddDeclarationName(ND->getDeclName());
+  if (auto *FD = dyn_cast<FunctionDecl>(D))
+    AddDeclarationNameInfo(FD->getNameInfo());
+  else
+    AddDeclarationName(ND->getDeclName());
 
   // If this was a specialization we should take into account its template
   // arguments. This helps to reduce collisions coming when visiting template
@@ -1017,7 +1026,7 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
   }
 
   void VisitDecltypeType(const DecltypeType *T) {
-    AddStmt(T->getUnderlyingExpr());
+    Hash.AddStmt(T->getUnderlyingExpr());
     VisitType(T);
   }
 
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 8b1caa05eec3..de8b5996818d 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1830,6 +1830,11 @@ void OMPClausePrinter::VisitOMPFinalClause(OMPFinalClause *Node) {
 
 void OMPClausePrinter::VisitOMPNumThreadsClause(OMPNumThreadsClause *Node) {
   OS << "num_threads(";
+  OpenMPNumThreadsClauseModifier Modifier = Node->getModifier();
+  if (Modifier != OMPC_NUMTHREADS_unknown) {
+    OS << getOpenMPSimpleClauseTypeName(Node->getClauseKind(), Modifier)
+       << ": ";
+  }
   Node->getNumThreads()->printPretty(OS, nullptr, Policy, 0);
   OS << ")";
 }
diff --git a/clang/lib/AST/RawCommentList.cpp b/clang/lib/AST/RawCommentList.cpp
index 9658c6ab3d39..3f9edc75311d 100644
--- a/clang/lib/AST/RawCommentList.cpp
+++ b/clang/lib/AST/RawCommentList.cpp
@@ -224,8 +224,8 @@ comments::FullComment *RawComment::parse(const ASTContext &Context,
 static bool onlyWhitespaceBetween(SourceManager &SM,
                                   SourceLocation Loc1, SourceLocation Loc2,
                                   unsigned MaxNewlinesAllowed) {
-  std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1);
-  std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2);
+  FileIDAndOffset Loc1Info = SM.getDecomposedLoc(Loc1);
+  FileIDAndOffset Loc2Info = SM.getDecomposedLoc(Loc2);
 
   // Question does not make sense if locations are in different files.
   if (Loc1Info.first != Loc2Info.first)
@@ -279,8 +279,7 @@ void RawCommentList::addComment(const RawComment &RC,
   if (RC.isOrdinary() && !CommentOpts.ParseAllComments)
     return;
 
-  std::pair<FileID, unsigned> Loc =
-      SourceMgr.getDecomposedLoc(RC.getBeginLoc());
+  FileIDAndOffset Loc = SourceMgr.getDecomposedLoc(RC.getBeginLoc());
 
   const FileID CommentFile = Loc.first;
   const unsigned CommentOffset = Loc.second;
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index aacc079f2521..6d819031cbef 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -3463,6 +3463,13 @@ ASTContext::getASTRecordLayout(const RecordDecl *D) const {
 
   ASTRecordLayouts[D] = NewEntry;
 
+  constexpr uint64_t MaxStructSizeInBytes = 1ULL << 60;
+  CharUnits StructSize = NewEntry->getSize();
+  if (static_cast<uint64_t>(StructSize.getQuantity()) >= MaxStructSizeInBytes) {
+    getDiagnostics().Report(D->getLocation(), diag::err_struct_too_large)
+        << D->getName() << MaxStructSizeInBytes;
+  }
+
   if (getLangOpts().DumpRecordLayouts) {
     llvm::outs() << "\n*** Dumping AST Record Layout\n";
     DumpRecordLayout(D, llvm::outs(), getLangOpts().DumpRecordLayoutsSimple);
diff --git a/clang/lib/AST/StmtOpenACC.cpp b/clang/lib/AST/StmtOpenACC.cpp
index bc5072187b89..07e3de8eeb00 100644
--- a/clang/lib/AST/StmtOpenACC.cpp
+++ b/clang/lib/AST/StmtOpenACC.cpp
@@ -209,7 +209,7 @@ OpenACCWaitConstruct *OpenACCWaitConstruct::Create(
     ArrayRef<Expr *> QueueIdExprs, SourceLocation RParenLoc, SourceLocation End,
     ArrayRef<const OpenACCClause *> Clauses) {
 
-  assert(llvm::all_of(QueueIdExprs, [](Expr *E) { return E != nullptr; }));
+  assert(!llvm::is_contained(QueueIdExprs, nullptr));
 
   void *Mem = C.Allocate(
       OpenACCWaitConstruct::totalSizeToAlloc<Expr *, OpenACCClause *>(
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index f8ec3f65b7eb..be02bdde38a3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -47,7 +47,6 @@
 #include "clang/Lex/Lexer.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 1b84b8824047..9d7c2757d6ee 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -24,7 +24,7 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TypeTraits.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 
 #include <algorithm>
 #include <utility>
@@ -3041,6 +3041,16 @@ void TextNodeDumper::VisitHLSLBufferDecl(const HLSLBufferDecl *D) {
 void TextNodeDumper::VisitHLSLRootSignatureDecl(
     const HLSLRootSignatureDecl *D) {
   dumpName(D);
+  OS << " version: ";
+  switch (D->getVersion()) {
+  case llvm::dxbc::RootSignatureVersion::V1_0:
+    OS << "1.0";
+    break;
+  case llvm::dxbc::RootSignatureVersion::V1_1:
+    OS << "1.1";
+    break;
+  }
+  OS << ", ";
   llvm::hlsl::rootsig::dumpRootElements(OS, D->getRootElements());
 }
 
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 2c1158e8f9b9..e5a1ab2ff890 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -4870,15 +4870,6 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
                                       ->getCanonicalTypeInternal());
   case Type::HLSLInlineSpirv:
     return LinkageInfo::external();
-    {
-      const auto *ST = cast<HLSLInlineSpirvType>(T);
-      LinkageInfo LV = LinkageInfo::external();
-      for (auto &Operand : ST->getOperands()) {
-        if (Operand.isConstant() || Operand.isType())
-          LV.merge(computeTypeLinkageInfo(Operand.getResultType()));
-      }
-      return LV;
-    }
   }
 
   llvm_unreachable("unhandled type class");
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 1bdb86ad445a..5c45c596538f 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include <algorithm>
@@ -652,9 +653,9 @@ static ConceptReference *createTrivialConceptReference(ASTContext &Context,
       DeclarationNameInfo(AT->getTypeConstraintConcept()->getDeclName(), Loc,
                           AT->getTypeConstraintConcept()->getDeclName());
   unsigned size = AT->getTypeConstraintArguments().size();
-  TemplateArgumentLocInfo *TALI = new TemplateArgumentLocInfo[size];
+  llvm::SmallVector<TemplateArgumentLocInfo, 8> TALI(size);
   TemplateSpecializationTypeLoc::initializeArgLocs(
-      Context, AT->getTypeConstraintArguments(), TALI, Loc);
+      Context, AT->getTypeConstraintArguments(), TALI.data(), Loc);
   TemplateArgumentListInfo TAListI;
   for (unsigned i = 0; i < size; ++i) {
     TAListI.addArgument(
@@ -666,7 +667,6 @@ static ConceptReference *createTrivialConceptReference(ASTContext &Context,
       Context, NestedNameSpecifierLoc{}, Loc, DNI, nullptr,
       AT->getTypeConstraintConcept(),
       ASTTemplateArgumentListInfo::Create(Context, TAListI));
-  delete[] TALI;
   return ConceptRef;
 }
 
diff --git a/clang/lib/ASTMatchers/Dynamic/Marshallers.h b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
index 0e640cbada72..d44e42a524f2 100644
--- a/clang/lib/ASTMatchers/Dynamic/Marshallers.h
+++ b/clang/lib/ASTMatchers/Dynamic/Marshallers.h
@@ -1060,7 +1060,7 @@ makeMatcherAutoMarshall(ReturnType (*Func)(), StringRef MatcherName) {
   BuildReturnTypeVector<ReturnType>::build(RetTypes);
   return std::make_unique<FixedArgCountMatcherDescriptor>(
       matcherMarshall0<ReturnType>, reinterpret_cast<void (*)()>(Func),
-      MatcherName, RetTypes, std::nullopt);
+      MatcherName, RetTypes, ArrayRef<ArgKind>());
 }
 
 /// 1-arg overload
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index cf7595952be2..d960d5130332 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -898,8 +898,7 @@ class CFGBuilder {
       return;
     }
 
-    B->appendStmt(const_cast<ObjCMessageExpr *>(ME),
-                  cfg->getBumpVectorContext());
+    B->appendStmt(ME, cfg->getBumpVectorContext());
   }
 
   void appendTemporaryDtor(CFGBlock *B, CXXBindTemporaryExpr *E) {
@@ -1754,10 +1753,9 @@ std::unique_ptr<CFG> CFGBuilder::buildCFG(const Decl *D, Stmt *Statement) {
 
   // Add successors to the Indirect Goto Dispatch block (if we have one).
   if (CFGBlock *B = cfg->getIndirectGotoBlock())
-    for (LabelSetTy::iterator I = AddressTakenLabels.begin(),
-                              E = AddressTakenLabels.end(); I != E; ++I ) {
+    for (LabelDecl *LD : AddressTakenLabels) {
       // Lookup the target block.
-      LabelMapTy::iterator LI = LabelMap.find(*I);
+      LabelMapTy::iterator LI = LabelMap.find(LD);
 
       // If there is no target block that contains label, then we are looking
       // at an incomplete AST.  Handle this by not registering a successor.
diff --git a/clang/lib/Analysis/CFGStmtMap.cpp b/clang/lib/Analysis/CFGStmtMap.cpp
index c3a4581e1fb1..028e62ba89b7 100644
--- a/clang/lib/Analysis/CFGStmtMap.cpp
+++ b/clang/lib/Analysis/CFGStmtMap.cpp
@@ -83,8 +83,8 @@ CFGStmtMap *CFGStmtMap::Build(CFG *C, ParentMap *PM) {
 
   // Walk all blocks, accumulating the block-level expressions, labels,
   // and terminators.
-  for (CFG::iterator I = C->begin(), E = C->end(); I != E; ++I)
-    Accumulate(*SM, *I);
+  for (CFGBlock *BB : *C)
+    Accumulate(*SM, BB);
 
   return new CFGStmtMap(PM, SM);
 }
diff --git a/clang/lib/Analysis/CallGraph.cpp b/clang/lib/Analysis/CallGraph.cpp
index 6a2b0392ccea..26646cd6b650 100644
--- a/clang/lib/Analysis/CallGraph.cpp
+++ b/clang/lib/Analysis/CallGraph.cpp
@@ -223,10 +223,7 @@ void CallGraph::print(raw_ostream &OS) const {
   // We are going to print the graph in reverse post order, partially, to make
   // sure the output is deterministic.
   llvm::ReversePostOrderTraversal<const CallGraph *> RPOT(this);
-  for (llvm::ReversePostOrderTraversal<const CallGraph *>::rpo_iterator
-         I = RPOT.begin(), E = RPOT.end(); I != E; ++I) {
-    const CallGraphNode *N = *I;
-
+  for (const CallGraphNode *N : RPOT) {
     OS << "  Function: ";
     if (N == Root)
       OS << "< root >";
diff --git a/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp b/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
index 39c46d73e9dd..d067e4ee6b00 100644
--- a/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
+++ b/clang/lib/Analysis/FlowSensitive/CNFFormula.cpp
@@ -49,8 +49,7 @@ struct CNFFormulaBuilder {
     // Contains literals of the simplified clause.
     llvm::SmallVector<Literal> Simplified;
     for (auto L : Literals) {
-      assert(L != NullLit &&
-             llvm::all_of(Simplified, [L](Literal S) { return S != L; }));
+      assert(L != NullLit && !llvm::is_contained(Simplified, L));
       auto X = var(L);
       if (trueVars.contains(X)) { // X must be true
         if (isPosLit(L))
@@ -103,7 +102,7 @@ CNFFormula::CNFFormula(Variable LargestVar)
 }
 
 void CNFFormula::addClause(ArrayRef<Literal> lits) {
-  assert(llvm::all_of(lits, [](Literal L) { return L != NullLit; }));
+  assert(!llvm::is_contained(lits, NullLit));
 
   if (lits.empty())
     KnownContradictory = true;
diff --git a/clang/lib/Analysis/LiveVariables.cpp b/clang/lib/Analysis/LiveVariables.cpp
index c1b2ff41edc0..375fdb3695e2 100644
--- a/clang/lib/Analysis/LiveVariables.cpp
+++ b/clang/lib/Analysis/LiveVariables.cpp
@@ -546,8 +546,8 @@ LiveVariablesImpl::runOnBlock(const CFGBlock *block,
 
 void LiveVariables::runOnAllBlocks(LiveVariables::Observer &obs) {
   const CFG *cfg = getImpl(impl).analysisContext.getCFG();
-  for (CFG::const_iterator it = cfg->begin(), ei = cfg->end(); it != ei; ++it)
-    getImpl(impl).runOnBlock(*it, getImpl(impl).blocksEndToLiveness[*it], &obs);
+  for (CFGBlock *B : *cfg)
+    getImpl(impl).runOnBlock(B, getImpl(impl).blocksEndToLiveness[B], &obs);
 }
 
 LiveVariables::LiveVariables(void *im) : impl(im) {}
@@ -618,10 +618,8 @@ void LiveVariables::dumpBlockLiveness(const SourceManager &M) {
 
 void LiveVariablesImpl::dumpBlockLiveness(const SourceManager &M) {
   std::vector<const CFGBlock *> vec;
-  for (llvm::DenseMap<const CFGBlock *, LiveVariables::LivenessValues>::iterator
-       it = blocksEndToLiveness.begin(), ei = blocksEndToLiveness.end();
-       it != ei; ++it) {
-    vec.push_back(it->first);
+  for (const auto &KV : blocksEndToLiveness) {
+    vec.push_back(KV.first);
   }
   llvm::sort(vec, [](const CFGBlock *A, const CFGBlock *B) {
     return A->getBlockID() < B->getBlockID();
diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index 962f9e9bc79c..ef24efd3c4bd 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -323,8 +323,8 @@ static bool compareCrossTUSourceLocs(FullSourceLoc XL, FullSourceLoc YL) {
     return true;
   if (XL.isValid() && YL.isInvalid())
     return false;
-  std::pair<FileID, unsigned> XOffs = XL.getDecomposedLoc();
-  std::pair<FileID, unsigned> YOffs = YL.getDecomposedLoc();
+  FileIDAndOffset XOffs = XL.getDecomposedLoc();
+  FileIDAndOffset YOffs = YL.getDecomposedLoc();
   const SourceManager &SM = XL.getManager();
   std::pair<bool, bool> InSameTU = SM.isInTheSameTranslationUnit(XOffs, YOffs);
   if (InSameTU.first)
@@ -1146,9 +1146,9 @@ void PathDiagnostic::FullProfile(llvm::FoldingSetNodeID &ID) const {
 
 LLVM_DUMP_METHOD void PathPieces::dump() const {
   unsigned index = 0;
-  for (PathPieces::const_iterator I = begin(), E = end(); I != E; ++I) {
+  for (const PathDiagnosticPieceRef &Piece : *this) {
     llvm::errs() << "[" << index++ << "]  ";
-    (*I)->dump();
+    Piece->dump();
     llvm::errs() << "\n";
   }
 }
diff --git a/clang/lib/Analysis/ReachableCode.cpp b/clang/lib/Analysis/ReachableCode.cpp
index 739c47b12e8c..4a9ab5d9f0f7 100644
--- a/clang/lib/Analysis/ReachableCode.cpp
+++ b/clang/lib/Analysis/ReachableCode.cpp
@@ -460,9 +460,8 @@ static bool isInCoroutineStmt(const Stmt *DeadStmt, const CFGBlock *Block) {
   const Stmt *CoroStmt = nullptr;
   // Find the first coroutine statement after the DeadStmt in the block.
   bool AfterDeadStmt = false;
-  for (CFGBlock::const_iterator I = Block->begin(), E = Block->end(); I != E;
-       ++I)
-    if (std::optional<CFGStmt> CS = I->getAs<CFGStmt>()) {
+  for (const CFGElement &Elem : *Block)
+    if (std::optional<CFGStmt> CS = Elem.getAs<CFGStmt>()) {
       const Stmt *S = CS->getStmt();
       if (S == DeadStmt)
         AfterDeadStmt = true;
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 8d279d969b61..987f894a2918 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -12,13 +12,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include "clang/Analysis/RetainSummaryManager.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/AST/ParentMap.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Analysis/DomainSpecific/CocoaConventions.h"
 #include <optional>
 
 using namespace clang;
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 631a546b45ff..6048169b5664 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -115,6 +115,10 @@ class MatchResult {
 };
 } // namespace
 
+#define SIZED_CONTAINER_OR_VIEW_LIST                                           \
+  "span", "array", "vector", "basic_string_view", "basic_string",              \
+      "initializer_list",
+
 // A `RecursiveASTVisitor` that traverses all descendants of a given node "n"
 // except for those belonging to a different callable of "n".
 class MatchDescendantVisitor : public DynamicRecursiveASTVisitor {
@@ -463,6 +467,8 @@ static bool areEqualIntegers(const Expr *E1, const Expr *E2, ASTContext &Ctx) {
 //       `N, M` are parameter indexes to the allocating element number and size.
 //        Sometimes, there is only one parameter index representing the total
 //        size.
+//   7. `std::span<T>{x.begin(), x.end()}` where `x` is an object in the
+//      SIZED_CONTAINER_OR_VIEW_LIST.
 static bool isSafeSpanTwoParamConstruct(const CXXConstructExpr &Node,
                                         ASTContext &Ctx) {
   assert(Node.getNumArgs() == 2 &&
@@ -560,6 +566,32 @@ static bool isSafeSpanTwoParamConstruct(const CXXConstructExpr &Node,
         }
     }
   }
+  // Check form 7:
+  auto IsMethodCallToSizedObject = [](const Stmt *Node, StringRef MethodName) {
+    if (const auto *MC = dyn_cast<CXXMemberCallExpr>(Node)) {
+      const auto *MD = MC->getMethodDecl();
+      const auto *RD = MC->getRecordDecl();
+
+      if (RD && MD)
+        if (auto *II = RD->getDeclName().getAsIdentifierInfo();
+            II && RD->isInStdNamespace())
+          return llvm::is_contained({SIZED_CONTAINER_OR_VIEW_LIST},
+                                    II->getName()) &&
+                 MD->getName() == MethodName;
+    }
+    return false;
+  };
+
+  if (IsMethodCallToSizedObject(Arg0, "begin") &&
+      IsMethodCallToSizedObject(Arg1, "end"))
+    return AreSameDRE(
+        // We know Arg0 and Arg1 are `CXXMemberCallExpr`s:
+        cast<CXXMemberCallExpr>(Arg0)
+            ->getImplicitObjectArgument()
+            ->IgnoreParenImpCasts(),
+        cast<CXXMemberCallExpr>(Arg1)
+            ->getImplicitObjectArgument()
+            ->IgnoreParenImpCasts());
   return false;
 }
 
@@ -1058,8 +1090,7 @@ static bool hasUnsafeSnprintfBuffer(const CallExpr &Node,
     return false; // not an snprintf call
 
   // Pattern 1:
-  static StringRef SizedObjs[] = {"span", "array", "vector",
-                                  "basic_string_view", "basic_string"};
+  static StringRef SizedObjs[] = {SIZED_CONTAINER_OR_VIEW_LIST};
   Buf = Buf->IgnoreParenImpCasts();
   Size = Size->IgnoreParenImpCasts();
   if (auto *MCEPtr = dyn_cast<CXXMemberCallExpr>(Buf))
@@ -1826,9 +1857,8 @@ class DataInvocationGadget : public WarningGadget {
     auto *method = cast<CXXMethodDecl>(callee);
     if (method->getNameAsString() == "data" &&
         method->getParent()->isInStdNamespace() &&
-        (method->getParent()->getName() == "span" ||
-         method->getParent()->getName() == "array" ||
-         method->getParent()->getName() == "vector"))
+        llvm::is_contained({SIZED_CONTAINER_OR_VIEW_LIST},
+                           method->getParent()->getName()))
       return true;
     return false;
   }
diff --git a/clang/lib/Basic/Attributes.cpp b/clang/lib/Basic/Attributes.cpp
index 004e5209a44a..81b186f844b8 100644
--- a/clang/lib/Basic/Attributes.cpp
+++ b/clang/lib/Basic/Attributes.cpp
@@ -260,8 +260,7 @@ static constexpr const char *AttrScopeSpellingList[] = {
 std::optional<StringRef>
 AttributeCommonInfo::tryGetCorrectedScopeName(StringRef ScopeName) const {
   if (ScopeName.size() > 0 &&
-      llvm::none_of(AttrScopeSpellingList,
-                    [&](const char *S) { return S == ScopeName; })) {
+      !llvm::is_contained(AttrScopeSpellingList, ScopeName)) {
     SimpleTypoCorrection STC(ScopeName);
     for (const auto &Scope : AttrScopeSpellingList)
       STC.add(Scope);
@@ -275,8 +274,7 @@ AttributeCommonInfo::tryGetCorrectedScopeName(StringRef ScopeName) const {
 std::optional<StringRef> AttributeCommonInfo::tryGetCorrectedAttrName(
     StringRef ScopeName, StringRef AttrName, const TargetInfo &Target,
     const LangOptions &LangOpts) const {
-  if (llvm::none_of(AttrSpellingList,
-                    [&](const char *A) { return A == AttrName; })) {
+  if (!llvm::is_contained(AttrSpellingList, AttrName)) {
     SimpleTypoCorrection STC(AttrName);
     for (const auto &Attr : AttrSpellingList)
       STC.add(Attr);
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index a30bfa28eca7..ab0525e96f3b 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -173,7 +173,7 @@ void DiagnosticsEngine::DiagStateMap::append(SourceManager &SrcMgr,
   CurDiagState = State;
   CurDiagStateLoc = Loc;
 
-  std::pair<FileID, unsigned> Decomp = SrcMgr.getDecomposedLoc(Loc);
+  FileIDAndOffset Decomp = SrcMgr.getDecomposedLoc(Loc);
   unsigned Offset = Decomp.second;
   for (File *F = getFile(SrcMgr, Decomp.first); F;
        Offset = F->ParentOffset, F = F->Parent) {
@@ -199,7 +199,7 @@ DiagnosticsEngine::DiagStateMap::lookup(SourceManager &SrcMgr,
   if (Files.empty())
     return FirstDiagState;
 
-  std::pair<FileID, unsigned> Decomp = SrcMgr.getDecomposedLoc(Loc);
+  FileIDAndOffset Decomp = SrcMgr.getDecomposedLoc(Loc);
   const File *F = getFile(SrcMgr, Decomp.first);
   return F->lookup(Decomp.second);
 }
@@ -226,7 +226,7 @@ DiagnosticsEngine::DiagStateMap::getFile(SourceManager &SrcMgr,
   // We created a new File; look up the diagnostic state at the start of it and
   // initialize it.
   if (ID.isValid()) {
-    std::pair<FileID, unsigned> Decomp = SrcMgr.getDecomposedIncludedLoc(ID);
+    FileIDAndOffset Decomp = SrcMgr.getDecomposedIncludedLoc(ID);
     F.Parent = getFile(SrcMgr, Decomp.first);
     F.ParentOffset = Decomp.second;
     F.StateTransitions.push_back({F.Parent->lookup(Decomp.second), 0});
@@ -263,8 +263,7 @@ void DiagnosticsEngine::DiagStateMap::dump(SourceManager &SrcMgr,
                    << ">: " << SrcMgr.getBufferOrFake(ID).getBufferIdentifier();
 
       if (F.second.Parent) {
-        std::pair<FileID, unsigned> Decomp =
-            SrcMgr.getDecomposedIncludedLoc(ID);
+        FileIDAndOffset Decomp = SrcMgr.getDecomposedIncludedLoc(ID);
         assert(File.ParentOffset == Decomp.second);
         llvm::errs() << " parent " << File.Parent << " <FileID "
                      << Decomp.first.getHashValue() << "> ";
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index dcf0c6cb5428..73f24a82d4c7 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -832,8 +832,12 @@ bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
       DiagID == diag::err_unavailable_message)
     return false;
 
-  // Currently we consider all ARC errors as recoverable.
-  if (isARCDiagnostic(DiagID))
+  // All ARC errors are currently considered recoverable, with the exception of
+  // err_arc_may_not_respond. This specific error is treated as unrecoverable
+  // because sending a message with an unknown selector could lead to crashes
+  // within CodeGen if the resulting expression is used to initialize a C++
+  // auto variable, where type deduction is required.
+  if (isARCDiagnostic(DiagID) && DiagID != diag::err_arc_may_not_respond)
     return false;
 
   if (isCodegenABICheckDiagnostic(DiagID))
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 3bd7e0ca5bcd..fc4ec785d376 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -141,7 +141,7 @@ FileManager::getDirectoryRef(StringRef DirName, bool CacheFailure) {
   if (DirName.size() > 1 &&
       DirName != llvm::sys::path::root_path(DirName) &&
       llvm::sys::path::is_separator(DirName.back()))
-    DirName = DirName.substr(0, DirName.size()-1);
+    DirName = DirName.drop_back();
   std::optional<std::string> DirNameStr;
   if (is_style_windows(llvm::sys::path::Style::native)) {
     // Fixing a problem with "clang C:test.c" on Windows.
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 5490c1812c13..82c1f593f424 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -185,11 +185,19 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
 #define OPENMP_ALLOCATE_MODIFIER(Name) .Case(#Name, OMPC_ALLOCATE_##Name)
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_ALLOCATE_unknown);
+  case OMPC_num_threads: {
+    unsigned Type = llvm::StringSwitch<unsigned>(Str)
+#define OPENMP_NUMTHREADS_MODIFIER(Name) .Case(#Name, OMPC_NUMTHREADS_##Name)
+#include "clang/Basic/OpenMPKinds.def"
+                        .Default(OMPC_NUMTHREADS_unknown);
+    if (LangOpts.OpenMP < 60)
+      return OMPC_NUMTHREADS_unknown;
+    return Type;
+  }
   case OMPC_unknown:
   case OMPC_threadprivate:
   case OMPC_if:
   case OMPC_final:
-  case OMPC_num_threads:
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
@@ -520,11 +528,20 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
 #include "clang/Basic/OpenMPKinds.def"
     }
     llvm_unreachable("Invalid OpenMP 'allocate' clause modifier");
+  case OMPC_num_threads:
+    switch (Type) {
+    case OMPC_NUMTHREADS_unknown:
+      return "unknown";
+#define OPENMP_NUMTHREADS_MODIFIER(Name)                                       \
+  case OMPC_NUMTHREADS_##Name:                                                 \
+    return #Name;
+#include "clang/Basic/OpenMPKinds.def"
+    }
+    llvm_unreachable("Invalid OpenMP 'num_threads' clause modifier");
   case OMPC_unknown:
   case OMPC_threadprivate:
   case OMPC_if:
   case OMPC_final:
-  case OMPC_num_threads:
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
diff --git a/clang/lib/Basic/Sanitizers.cpp b/clang/lib/Basic/Sanitizers.cpp
index cff289e8554d..774b94cedb13 100644
--- a/clang/lib/Basic/Sanitizers.cpp
+++ b/clang/lib/Basic/Sanitizers.cpp
@@ -95,9 +95,8 @@ bool clang::parseSanitizerWeightedValue(StringRef Value, bool AllowGroups,
     return false;
   auto [N, W] = Value.split('=');
   double A;
-  if (W.getAsDouble(A))
+  if (W.getAsDouble(A) || A < 0.0 || A > 1.0)
     return false;
-  A = std::clamp(A, 0.0, 1.0);
   // AllowGroups is already taken into account for ParsedKind,
   // hence we unconditionally expandSanitizerGroups.
   Cutoffs.set(expandSanitizerGroups(ParsedKind), A);
diff --git a/clang/lib/Basic/Sarif.cpp b/clang/lib/Basic/Sarif.cpp
index ed50b73e9224..69862b73febd 100644
--- a/clang/lib/Basic/Sarif.cpp
+++ b/clang/lib/Basic/Sarif.cpp
@@ -117,7 +117,7 @@ static unsigned int adjustColumnPos(FullSourceLoc Loc,
                                     unsigned int TokenLen = 0) {
   assert(!Loc.isInvalid() && "invalid Loc when adjusting column position");
 
-  std::pair<FileID, unsigned> LocInfo = Loc.getDecomposedExpansionLoc();
+  FileIDAndOffset LocInfo = Loc.getDecomposedExpansionLoc();
   std::optional<MemoryBufferRef> Buf =
       Loc.getManager().getBufferOrNone(LocInfo.first);
   assert(Buf && "got an invalid buffer for the location's file");
diff --git a/clang/lib/Basic/SourceLocation.cpp b/clang/lib/Basic/SourceLocation.cpp
index c5b9864dbb33..3e26f75d25b1 100644
--- a/clang/lib/Basic/SourceLocation.cpp
+++ b/clang/lib/Basic/SourceLocation.cpp
@@ -164,7 +164,7 @@ FullSourceLoc FullSourceLoc::getExpansionLoc() const {
   return FullSourceLoc(SrcMgr->getExpansionLoc(*this), *SrcMgr);
 }
 
-std::pair<FileID, unsigned> FullSourceLoc::getDecomposedExpansionLoc() const {
+FileIDAndOffset FullSourceLoc::getDecomposedExpansionLoc() const {
   return SrcMgr->getDecomposedExpansionLoc(*this);
 }
 
@@ -274,6 +274,6 @@ StringRef FullSourceLoc::getBufferData(bool *Invalid) const {
   return SrcMgr->getBufferData(SrcMgr->getFileID(*this), Invalid);
 }
 
-std::pair<FileID, unsigned> FullSourceLoc::getDecomposedLoc() const {
+FileIDAndOffset FullSourceLoc::getDecomposedLoc() const {
   return SrcMgr->getDecomposedLoc(*this);
 }
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 053e82683a4a..b2b1488f9dc8 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -267,7 +267,7 @@ void SourceManager::AddLineNote(SourceLocation Loc, unsigned LineNo,
                                 int FilenameID, bool IsFileEntry,
                                 bool IsFileExit,
                                 SrcMgr::CharacteristicKind FileKind) {
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
   SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
@@ -329,12 +329,14 @@ SourceManager::~SourceManager() {
 void SourceManager::clearIDTables() {
   MainFileID = FileID();
   LocalSLocEntryTable.clear();
+  LocalLocOffsetTable.clear();
   LoadedSLocEntryTable.clear();
   SLocEntryLoaded.clear();
   SLocEntryOffsetLoaded.clear();
   LastLineNoFileIDQuery = FileID();
   LastLineNoContentCache = nullptr;
   LastFileIDLookup = FileID();
+  LastLookupStartOffset = LastLookupEndOffset = 0;
 
   IncludedLocMap.clear();
   if (LineTable)
@@ -587,17 +589,6 @@ SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
 					  FileCharacter);
 }
 
-/// Helper function to determine if an input file requires conversion
-bool needConversion(StringRef Filename) {
-#ifdef __MVS__
-  llvm::ErrorOr<bool> NeedConversion =
-      llvm::needzOSConversion(Filename.str().c_str());
-  return NeedConversion && *NeedConversion;
-#else
-  return false;
-#endif
-}
-
 /// createFileID - Create a new FileID for the specified ContentCache and
 /// include position.  This works regardless of whether the ContentCache
 /// corresponds to a file or some other input source.
@@ -617,8 +608,9 @@ FileID SourceManager::createFileIDImpl(ContentCache &File, StringRef Filename,
     return FileID::get(LoadedID);
   }
   unsigned FileSize = File.getSize();
-  bool NeedConversion = needConversion(Filename);
-  if (NeedConversion) {
+  llvm::ErrorOr<bool> NeedConversion =
+      llvm::needConversion(Filename.str().c_str());
+  if (NeedConversion && *NeedConversion) {
     // Buffer size may increase due to potential z/OS EBCDIC to UTF-8
     // conversion.
     if (std::optional<llvm::MemoryBufferRef> Buffer =
@@ -637,12 +629,16 @@ FileID SourceManager::createFileIDImpl(ContentCache &File, StringRef Filename,
     noteSLocAddressSpaceUsage(Diag);
     return FileID();
   }
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
   LocalSLocEntryTable.push_back(
       SLocEntry::get(NextLocalOffset,
                      FileInfo::get(IncludePos, File, FileCharacter, Filename)));
+  LocalLocOffsetTable.push_back(NextLocalOffset);
+  LastLookupStartOffset = NextLocalOffset;
   // We do a +1 here because we want a SourceLocation that means "the end of the
   // file", e.g. for the "no newline at the end of the file" diagnostic.
   NextLocalOffset += FileSize + 1;
+  LastLookupEndOffset = NextLocalOffset;
   updateSlocUsageStats();
 
   // Set LastFileIDLookup to the newly created file.  The next getFileID call is
@@ -691,7 +687,9 @@ SourceManager::createExpansionLocImpl(const ExpansionInfo &Info,
     SLocEntryLoaded[Index] = SLocEntryOffsetLoaded[Index] = true;
     return SourceLocation::getMacroLoc(LoadedOffset);
   }
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
   LocalSLocEntryTable.push_back(SLocEntry::get(NextLocalOffset, Info));
+  LocalLocOffsetTable.push_back(NextLocalOffset);
   if (NextLocalOffset + Length + 1 <= NextLocalOffset ||
       NextLocalOffset + Length + 1 > CurrentLoadedOffset) {
     Diag.Report(diag::err_sloc_space_too_large);
@@ -812,6 +810,10 @@ FileID SourceManager::getFileIDSlow(SourceLocation::UIntTy SLocOffset) const {
 /// loaded one.
 FileID SourceManager::getFileIDLocal(SourceLocation::UIntTy SLocOffset) const {
   assert(SLocOffset < NextLocalOffset && "Bad function choice");
+  assert(SLocOffset >= LocalSLocEntryTable[0].getOffset() && SLocOffset > 0 &&
+         "Invalid SLocOffset");
+  assert(LocalSLocEntryTable.size() == LocalLocOffsetTable.size());
+  assert(LastFileIDLookup.ID >= 0 && "Only cache local file sloc entry");
 
   // After the first and second level caches, I see two common sorts of
   // behavior: 1) a lot of searched FileID's are "near" the cached file
@@ -830,60 +832,50 @@ FileID SourceManager::getFileIDLocal(SourceLocation::UIntTy SLocOffset) const {
   // SLocOffset.
   unsigned LessIndex = 0;
   // upper bound of the search range.
-  unsigned GreaterIndex = LocalSLocEntryTable.size();
-  if (LastFileIDLookup.ID >= 0) {
-    // Use the LastFileIDLookup to prune the search space.
-    if (LocalSLocEntryTable[LastFileIDLookup.ID].getOffset() < SLocOffset)
-      LessIndex = LastFileIDLookup.ID;
-    else
-      GreaterIndex = LastFileIDLookup.ID;
-  }
+  unsigned GreaterIndex = LocalLocOffsetTable.size();
+  // Use the LastFileIDLookup to prune the search space.
+  if (LastLookupStartOffset < SLocOffset)
+    LessIndex = LastFileIDLookup.ID;
+  else
+    GreaterIndex = LastFileIDLookup.ID;
 
   // Find the FileID that contains this.
   unsigned NumProbes = 0;
   while (true) {
     --GreaterIndex;
-    assert(GreaterIndex < LocalSLocEntryTable.size());
-    if (LocalSLocEntryTable[GreaterIndex].getOffset() <= SLocOffset) {
+    assert(GreaterIndex < LocalLocOffsetTable.size());
+    if (LocalLocOffsetTable[GreaterIndex] <= SLocOffset) {
       FileID Res = FileID::get(int(GreaterIndex));
       // Remember it.  We have good locality across FileID lookups.
       LastFileIDLookup = Res;
-      NumLinearScans += NumProbes+1;
+      LastLookupStartOffset = LocalLocOffsetTable[GreaterIndex];
+      LastLookupEndOffset =
+          GreaterIndex + 1 >= LocalLocOffsetTable.size()
+              ? NextLocalOffset
+              : LocalLocOffsetTable[GreaterIndex + 1];
+      NumLinearScans += NumProbes + 1;
       return Res;
     }
     if (++NumProbes == 8)
       break;
   }
 
-  NumProbes = 0;
-  while (true) {
-    unsigned MiddleIndex = (GreaterIndex-LessIndex)/2+LessIndex;
-    SourceLocation::UIntTy MidOffset =
-        getLocalSLocEntry(MiddleIndex).getOffset();
-
-    ++NumProbes;
+  while (LessIndex < GreaterIndex) {
+    ++NumBinaryProbes;
 
-    // If the offset of the midpoint is too large, chop the high side of the
-    // range to the midpoint.
-    if (MidOffset > SLocOffset) {
+    unsigned MiddleIndex = LessIndex + (GreaterIndex - LessIndex) / 2;
+    if (LocalLocOffsetTable[MiddleIndex] <= SLocOffset)
+      LessIndex = MiddleIndex + 1;
+    else
       GreaterIndex = MiddleIndex;
-      continue;
-    }
-
-    // If the middle index contains the value, succeed and return.
-    if (MiddleIndex + 1 == LocalSLocEntryTable.size() ||
-        SLocOffset < getLocalSLocEntry(MiddleIndex + 1).getOffset()) {
-      FileID Res = FileID::get(MiddleIndex);
-
-      // Remember it.  We have good locality across FileID lookups.
-      LastFileIDLookup = Res;
-      NumBinaryProbes += NumProbes;
-      return Res;
-    }
-
-    // Otherwise, move the low-side up to the middle index.
-    LessIndex = MiddleIndex;
   }
+
+  // At this point, LessIndex is the index of the *first element greater than*
+  // SLocOffset. The element we are actually looking for is the one immediately
+  // before it.
+  LastLookupStartOffset = LocalLocOffsetTable[LessIndex - 1];
+  LastLookupEndOffset = LocalLocOffsetTable[LessIndex];
+  return LastFileIDLookup = FileID::get(LessIndex - 1);
 }
 
 /// Return the FileID for a SourceLocation with a high offset.
@@ -916,7 +908,7 @@ getExpansionLocSlowCase(SourceLocation Loc) const {
 
 SourceLocation SourceManager::getSpellingLocSlowCase(SourceLocation Loc) const {
   do {
-    std::pair<FileID, unsigned> LocInfo = getDecomposedLoc(Loc);
+    FileIDAndOffset LocInfo = getDecomposedLoc(Loc);
     Loc = getSLocEntry(LocInfo.first).getExpansion().getSpellingLoc();
     Loc = Loc.getLocWithOffset(LocInfo.second);
   } while (!Loc.isFileID());
@@ -933,10 +925,8 @@ SourceLocation SourceManager::getFileLocSlowCase(SourceLocation Loc) const {
   return Loc;
 }
 
-
-std::pair<FileID, unsigned>
-SourceManager::getDecomposedExpansionLocSlowCase(
-                                             const SrcMgr::SLocEntry *E) const {
+FileIDAndOffset SourceManager::getDecomposedExpansionLocSlowCase(
+    const SrcMgr::SLocEntry *E) const {
   // If this is an expansion record, walk through all the expansion points.
   FileID FID;
   SourceLocation Loc;
@@ -952,7 +942,7 @@ SourceManager::getDecomposedExpansionLocSlowCase(
   return std::make_pair(FID, Offset);
 }
 
-std::pair<FileID, unsigned>
+FileIDAndOffset
 SourceManager::getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
                                                 unsigned Offset) const {
   // If this is an expansion record, walk through all the expansion points.
@@ -976,7 +966,7 @@ SourceManager::getDecomposedSpellingLocSlowCase(const SrcMgr::SLocEntry *E,
 /// found.  This should not generally be used by clients.
 SourceLocation SourceManager::getImmediateSpellingLoc(SourceLocation Loc) const{
   if (Loc.isFileID()) return Loc;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedLoc(Loc);
   Loc = getSLocEntry(LocInfo.first).getExpansion().getSpellingLoc();
   return Loc.getLocWithOffset(LocInfo.second);
 }
@@ -1048,7 +1038,7 @@ bool SourceManager::isAtStartOfImmediateMacroExpansion(SourceLocation Loc,
                                              SourceLocation *MacroBegin) const {
   assert(Loc.isValid() && Loc.isMacroID() && "Expected a valid macro loc");
 
-  std::pair<FileID, unsigned> DecompLoc = getDecomposedLoc(Loc);
+  FileIDAndOffset DecompLoc = getDecomposedLoc(Loc);
   if (DecompLoc.second > 0)
     return false; // Does not point at the start of expansion range.
 
@@ -1125,7 +1115,7 @@ const char *SourceManager::getCharacterData(SourceLocation SL,
                                             bool *Invalid) const {
   // Note that this is a hot function in the getSpelling() path, which is
   // heavily used by -E mode.
-  std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(SL);
+  FileIDAndOffset LocInfo = getDecomposedSpellingLoc(SL);
 
   // Note that calling 'getBuffer()' may lazily page in a source file.
   bool CharDataInvalid = false;
@@ -1204,14 +1194,14 @@ static bool isInvalid(LocType Loc, bool *Invalid) {
 unsigned SourceManager::getSpellingColumnNumber(SourceLocation Loc,
                                                 bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedSpellingLoc(Loc);
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
 unsigned SourceManager::getExpansionColumnNumber(SourceLocation Loc,
                                                  bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
   return getColumnNumber(LocInfo.first, LocInfo.second, Invalid);
 }
 
@@ -1412,13 +1402,13 @@ unsigned SourceManager::getLineNumber(FileID FID, unsigned FilePos,
 unsigned SourceManager::getSpellingLineNumber(SourceLocation Loc,
                                               bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedSpellingLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedSpellingLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
 unsigned SourceManager::getExpansionLineNumber(SourceLocation Loc,
                                                bool *Invalid) const {
   if (isInvalid(Loc, Invalid)) return 0;
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
   return getLineNumber(LocInfo.first, LocInfo.second);
 }
 unsigned SourceManager::getPresumedLineNumber(SourceLocation Loc,
@@ -1439,7 +1429,7 @@ unsigned SourceManager::getPresumedLineNumber(SourceLocation Loc,
 SrcMgr::CharacteristicKind
 SourceManager::getFileCharacteristic(SourceLocation Loc) const {
   assert(Loc.isValid() && "Can't get file characteristic of invalid loc!");
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
   const SLocEntry *SEntry = getSLocEntryForFile(LocInfo.first);
   if (!SEntry)
     return C_User;
@@ -1488,7 +1478,7 @@ PresumedLoc SourceManager::getPresumedLoc(SourceLocation Loc,
   if (Loc.isInvalid()) return PresumedLoc();
 
   // Presumed locations are always for expansion points.
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
 
   bool Invalid = false;
   const SLocEntry &Entry = getSLocEntry(LocInfo.first, &Invalid);
@@ -1563,7 +1553,7 @@ bool SourceManager::isInMainFile(SourceLocation Loc) const {
   if (Loc.isInvalid()) return false;
 
   // Presumed locations are always for expansion points.
-  std::pair<FileID, unsigned> LocInfo = getDecomposedExpansionLoc(Loc);
+  FileIDAndOffset LocInfo = getDecomposedExpansionLoc(Loc);
 
   const SLocEntry *Entry = getSLocEntryForFile(LocInfo.first);
   if (!Entry)
@@ -1805,9 +1795,8 @@ void SourceManager::associateFileChunkWithMacroArgExp(
     // spelling range and if one is itself a macro argument expansion, recurse
     // and associate the file chunk that it represents.
 
-    FileID SpellFID; // Current FileID in the spelling range.
-    unsigned SpellRelativeOffs;
-    std::tie(SpellFID, SpellRelativeOffs) = getDecomposedLoc(SpellLoc);
+    // Current FileID in the spelling range.
+    auto [SpellFID, SpellRelativeOffs] = getDecomposedLoc(SpellLoc);
     while (true) {
       const SLocEntry &Entry = getSLocEntry(SpellFID);
       SourceLocation::UIntTy SpellFIDBeginOffs = Entry.getOffset();
@@ -1889,9 +1878,7 @@ SourceManager::getMacroArgExpandedLocation(SourceLocation Loc) const {
   if (Loc.isInvalid() || !Loc.isFileID())
     return Loc;
 
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = getDecomposedLoc(Loc);
+  auto [FID, Offset] = getDecomposedLoc(Loc);
   if (FID.isInvalid())
     return Loc;
 
@@ -1918,14 +1905,13 @@ SourceManager::getMacroArgExpandedLocation(SourceLocation Loc) const {
   return Loc;
 }
 
-std::pair<FileID, unsigned>
-SourceManager::getDecomposedIncludedLoc(FileID FID) const {
+FileIDAndOffset SourceManager::getDecomposedIncludedLoc(FileID FID) const {
   if (FID.isInvalid())
     return std::make_pair(FileID(), 0);
 
   // Uses IncludedLocMap to retrieve/cache the decomposed loc.
 
-  using DecompTy = std::pair<FileID, unsigned>;
+  using DecompTy = FileIDAndOffset;
   auto InsertOp = IncludedLocMap.try_emplace(FID);
   DecompTy &DecompLoc = InsertOp.first->second;
   if (!InsertOp.second)
@@ -1966,8 +1952,7 @@ FileID SourceManager::getUniqueLoadedASTFileID(SourceLocation Loc) const {
 }
 
 bool SourceManager::isInTheSameTranslationUnitImpl(
-    const std::pair<FileID, unsigned> &LOffs,
-    const std::pair<FileID, unsigned> &ROffs) const {
+    const FileIDAndOffset &LOffs, const FileIDAndOffset &ROffs) const {
   // If one is local while the other is loaded.
   if (isLoadedFileID(LOffs.first) != isLoadedFileID(ROffs.first))
     return false;
@@ -1992,10 +1977,9 @@ bool SourceManager::isInTheSameTranslationUnitImpl(
 /// to the parent source location within the same translation unit.  If this is
 /// possible, return the decomposed version of the parent in Loc and return
 /// false.  If Loc is a top-level entry, return true and don't modify it.
-static bool
-MoveUpTranslationUnitIncludeHierarchy(std::pair<FileID, unsigned> &Loc,
-                                      const SourceManager &SM) {
-  std::pair<FileID, unsigned> UpperLoc = SM.getDecomposedIncludedLoc(Loc.first);
+static bool MoveUpTranslationUnitIncludeHierarchy(FileIDAndOffset &Loc,
+                                                  const SourceManager &SM) {
+  FileIDAndOffset UpperLoc = SM.getDecomposedIncludedLoc(Loc.first);
   if (UpperLoc.first.isInvalid() ||
       !SM.isInTheSameTranslationUnitImpl(UpperLoc, Loc))
     return true; // We reached the top.
@@ -2041,8 +2025,8 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   if (LHS == RHS)
     return false;
 
-  std::pair<FileID, unsigned> LOffs = getDecomposedLoc(LHS);
-  std::pair<FileID, unsigned> ROffs = getDecomposedLoc(RHS);
+  FileIDAndOffset LOffs = getDecomposedLoc(LHS);
+  FileIDAndOffset ROffs = getDecomposedLoc(RHS);
 
   // getDecomposedLoc may have failed to return a valid FileID because, e.g. it
   // is a serialized one referring to a file that was removed after we loaded
@@ -2057,9 +2041,9 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   return LOffs.first < ROffs.first;
 }
 
-std::pair<bool, bool> SourceManager::isInTheSameTranslationUnit(
-    std::pair<FileID, unsigned> &LOffs,
-    std::pair<FileID, unsigned> &ROffs) const {
+std::pair<bool, bool>
+SourceManager::isInTheSameTranslationUnit(FileIDAndOffset &LOffs,
+                                          FileIDAndOffset &ROffs) const {
   // If the source locations are not in the same TU, return early.
   if (!isInTheSameTranslationUnitImpl(LOffs, ROffs))
     return std::make_pair(false, false);
@@ -2086,7 +2070,7 @@ std::pair<bool, bool> SourceManager::isInTheSameTranslationUnit(
 
   // A location within a FileID on the path up from LOffs to the main file.
   struct Entry {
-    std::pair<FileID, unsigned> DecomposedLoc; // FileID redundant, but clearer.
+    FileIDAndOffset DecomposedLoc; // FileID redundant, but clearer.
     FileID ChildFID; // Used for breaking ties. Invalid for the initial loc.
   };
   llvm::SmallDenseMap<FileID, Entry, 16> LChain;
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 124b340b62d9..72d2e5fcf461 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -125,79 +125,6 @@ static constexpr auto BuiltinAArch64Infos =
 #include "clang/Basic/BuiltinsAArch64.def"
     });
 
-void AArch64TargetInfo::setArchFeatures() {
-  if (*ArchInfo == llvm::AArch64::ARMV8R) {
-    HasDotProd = true;
-    HasDIT = true;
-    HasFlagM = true;
-    HasRCPC = true;
-    FPU |= NeonMode;
-    HasCCPP = true;
-    HasCRC = true;
-    HasLSE = true;
-    HasRDM = true;
-  } else if (ArchInfo->Version.getMajor() == 8) {
-    if (ArchInfo->Version.getMinor() >= 7u) {
-      HasWFxT = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 6u) {
-      HasBFloat16 = true;
-      HasMatMul = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 5u) {
-      HasAlternativeNZCV = true;
-      HasFRInt3264 = true;
-      HasSSBS = true;
-      HasSB = true;
-      HasPredRes = true;
-      HasBTI = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 4u) {
-      HasDotProd = true;
-      HasDIT = true;
-      HasFlagM = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 3u) {
-      HasRCPC = true;
-      FPU |= NeonMode;
-    }
-    if (ArchInfo->Version.getMinor() >= 2u) {
-      HasCCPP = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 1u) {
-      HasCRC = true;
-      HasLSE = true;
-      HasRDM = true;
-    }
-  } else if (ArchInfo->Version.getMajor() == 9) {
-    if (ArchInfo->Version.getMinor() >= 2u) {
-      HasWFxT = true;
-    }
-    if (ArchInfo->Version.getMinor() >= 1u) {
-      HasBFloat16 = true;
-      HasMatMul = true;
-    }
-    FPU |= SveMode;
-    HasSVE2 = true;
-    HasFullFP16 = true;
-    HasAlternativeNZCV = true;
-    HasFRInt3264 = true;
-    HasSSBS = true;
-    HasSB = true;
-    HasPredRes = true;
-    HasBTI = true;
-    HasDotProd = true;
-    HasDIT = true;
-    HasFlagM = true;
-    HasRCPC = true;
-    FPU |= NeonMode;
-    HasCCPP = true;
-    HasCRC = true;
-    HasLSE = true;
-    HasRDM = true;
-  }
-}
-
 AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
                                      const TargetOptions &Opts)
     : TargetInfo(Triple), ABI("aapcs") {
@@ -823,16 +750,36 @@ AArch64TargetInfo::getTargetBuiltins() const {
 
 std::optional<std::pair<unsigned, unsigned>>
 AArch64TargetInfo::getVScaleRange(const LangOptions &LangOpts,
-                                  bool IsArmStreamingFunction,
+                                  ArmStreamingKind Mode,
                                   llvm::StringMap<bool> *FeatureMap) const {
-  if (LangOpts.VScaleMin || LangOpts.VScaleMax)
+  if (Mode == ArmStreamingKind::NotStreaming &&
+      (LangOpts.VScaleMin || LangOpts.VScaleMax))
+    return std::pair<unsigned, unsigned>(
+        LangOpts.VScaleMin ? LangOpts.VScaleMin : 1,
+        LangOpts.VScaleMax ? LangOpts.VScaleMax : 16);
+
+  if (Mode == ArmStreamingKind::Streaming &&
+      (LangOpts.VScaleStreamingMin || LangOpts.VScaleStreamingMax))
     return std::pair<unsigned, unsigned>(
-        LangOpts.VScaleMin ? LangOpts.VScaleMin : 1, LangOpts.VScaleMax);
+        LangOpts.VScaleStreamingMin ? LangOpts.VScaleStreamingMin : 1,
+        LangOpts.VScaleStreamingMax ? LangOpts.VScaleStreamingMax : 16);
+
+  if (Mode == ArmStreamingKind::StreamingCompatible &&
+      ((LangOpts.VScaleMin && LangOpts.VScaleStreamingMin) ||
+       (LangOpts.VScaleMax && LangOpts.VScaleStreamingMax))) {
+    unsigned Min =
+        std::min(LangOpts.VScaleMin ? LangOpts.VScaleMin : 1,
+                 LangOpts.VScaleStreamingMin ? LangOpts.VScaleStreamingMin : 1);
+    unsigned Max = std::max(
+        LangOpts.VScaleMax ? LangOpts.VScaleMax : 16,
+        LangOpts.VScaleStreamingMax ? LangOpts.VScaleStreamingMax : 16);
+    return std::pair(Min, Max);
+  }
 
   if (hasFeature("sve") || (FeatureMap && (FeatureMap->lookup("sve"))))
     return std::pair<unsigned, unsigned>(1, 16);
 
-  if (IsArmStreamingFunction &&
+  if (Mode == ArmStreamingKind::Streaming &&
       (hasFeature("sme") || (FeatureMap && (FeatureMap->lookup("sme")))))
     return std::pair<unsigned, unsigned>(1, 16);
 
@@ -1266,7 +1213,6 @@ bool AArch64TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   }
 
   setDataLayout();
-  setArchFeatures();
 
   if (HasNoFP) {
     FPU &= ~FPUMode;
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
index 1951e0679d2e..f4277e95b19b 100644
--- a/clang/lib/Basic/Targets/AArch64.h
+++ b/clang/lib/Basic/Targets/AArch64.h
@@ -158,8 +158,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
     return false;
   }
 
-  void setArchFeatures();
-
   void getTargetDefinesARMV81A(const LangOptions &Opts,
                                MacroBuilder &Builder) const;
   void getTargetDefinesARMV82A(const LangOptions &Opts,
@@ -198,7 +196,7 @@ class LLVM_LIBRARY_VISIBILITY AArch64TargetInfo : public TargetInfo {
   llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const override;
   bool doesFeatureAffectCodeGen(StringRef Name) const override;
   bool validateCpuSupports(StringRef FeatureStr) const override;
diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp
index 5312767498d9..f6915df1520b 100644
--- a/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/clang/lib/Basic/Targets/LoongArch.cpp
@@ -81,38 +81,46 @@ LoongArchTargetInfo::getGCCRegAliases() const {
       {{"s6", "$s6", "r29"}, "$r29"},
       {{"s7", "$s7", "r30"}, "$r30"},
       {{"s8", "$s8", "r31"}, "$r31"},
-      {{"$fa0"}, "$f0"},
-      {{"$fa1"}, "$f1"},
-      {{"$fa2"}, "$f2"},
-      {{"$fa3"}, "$f3"},
-      {{"$fa4"}, "$f4"},
-      {{"$fa5"}, "$f5"},
-      {{"$fa6"}, "$f6"},
-      {{"$fa7"}, "$f7"},
-      {{"$ft0"}, "$f8"},
-      {{"$ft1"}, "$f9"},
-      {{"$ft2"}, "$f10"},
-      {{"$ft3"}, "$f11"},
-      {{"$ft4"}, "$f12"},
-      {{"$ft5"}, "$f13"},
-      {{"$ft6"}, "$f14"},
-      {{"$ft7"}, "$f15"},
-      {{"$ft8"}, "$f16"},
-      {{"$ft9"}, "$f17"},
-      {{"$ft10"}, "$f18"},
-      {{"$ft11"}, "$f19"},
-      {{"$ft12"}, "$f20"},
-      {{"$ft13"}, "$f21"},
-      {{"$ft14"}, "$f22"},
-      {{"$ft15"}, "$f23"},
-      {{"$fs0"}, "$f24"},
-      {{"$fs1"}, "$f25"},
-      {{"$fs2"}, "$f26"},
-      {{"$fs3"}, "$f27"},
-      {{"$fs4"}, "$f28"},
-      {{"$fs5"}, "$f29"},
-      {{"$fs6"}, "$f30"},
-      {{"$fs7"}, "$f31"},
+      {{"fa0", "$fa0", "f0"}, "$f0"},
+      {{"fa1", "$fa1", "f1"}, "$f1"},
+      {{"fa2", "$fa2", "f2"}, "$f2"},
+      {{"fa3", "$fa3", "f3"}, "$f3"},
+      {{"fa4", "$fa4", "f4"}, "$f4"},
+      {{"fa5", "$fa5", "f5"}, "$f5"},
+      {{"fa6", "$fa6", "f6"}, "$f6"},
+      {{"fa7", "$fa7", "f7"}, "$f7"},
+      {{"ft0", "$ft0", "f8"}, "$f8"},
+      {{"ft1", "$ft1", "f9"}, "$f9"},
+      {{"ft2", "$ft2", "f10"}, "$f10"},
+      {{"ft3", "$ft3", "f11"}, "$f11"},
+      {{"ft4", "$ft4", "f12"}, "$f12"},
+      {{"ft5", "$ft5", "f13"}, "$f13"},
+      {{"ft6", "$ft6", "f14"}, "$f14"},
+      {{"ft7", "$ft7", "f15"}, "$f15"},
+      {{"ft8", "$ft8", "f16"}, "$f16"},
+      {{"ft9", "$ft9", "f17"}, "$f17"},
+      {{"ft10", "$ft10", "f18"}, "$f18"},
+      {{"ft11", "$ft11", "f19"}, "$f19"},
+      {{"ft12", "$ft12", "f20"}, "$f20"},
+      {{"ft13", "$ft13", "f21"}, "$f21"},
+      {{"ft14", "$ft14", "f22"}, "$f22"},
+      {{"ft15", "$ft15", "f23"}, "$f23"},
+      {{"fs0", "$fs0", "f24"}, "$f24"},
+      {{"fs1", "$fs1", "f25"}, "$f25"},
+      {{"fs2", "$fs2", "f26"}, "$f26"},
+      {{"fs3", "$fs3", "f27"}, "$f27"},
+      {{"fs4", "$fs4", "f28"}, "$f28"},
+      {{"fs5", "$fs5", "f29"}, "$f29"},
+      {{"fs6", "$fs6", "f30"}, "$f30"},
+      {{"fs7", "$fs7", "f31"}, "$f31"},
+      {{"fcc0"}, "$fcc0"},
+      {{"fcc1"}, "$fcc1"},
+      {{"fcc2"}, "$fcc2"},
+      {{"fcc3"}, "$fcc3"},
+      {{"fcc4"}, "$fcc4"},
+      {{"fcc5"}, "$fcc5"},
+      {{"fcc6"}, "$fcc6"},
+      {{"fcc7"}, "$fcc7"},
   };
   return llvm::ArrayRef(GCCRegAliases);
 }
diff --git a/clang/lib/Basic/Targets/LoongArch.h b/clang/lib/Basic/Targets/LoongArch.h
index 3ad5abca927b..88dc433924d6 100644
--- a/clang/lib/Basic/Targets/LoongArch.h
+++ b/clang/lib/Basic/Targets/LoongArch.h
@@ -60,6 +60,7 @@ class LLVM_LIBRARY_VISIBILITY LoongArchTargetInfo : public TargetInfo {
     SuitableAlign = 128;
     WCharType = SignedInt;
     WIntType = UnsignedInt;
+    BitIntMaxAlign = 128;
   }
 
   bool setCPU(const std::string &Name) override {
diff --git a/clang/lib/Basic/Targets/M68k.cpp b/clang/lib/Basic/Targets/M68k.cpp
index f0c77f027858..3988cb529456 100644
--- a/clang/lib/Basic/Targets/M68k.cpp
+++ b/clang/lib/Basic/Targets/M68k.cpp
@@ -56,6 +56,7 @@ M68kTargetInfo::M68kTargetInfo(const llvm::Triple &Triple,
   SizeType = UnsignedInt;
   PtrDiffType = SignedInt;
   IntPtrType = SignedInt;
+  IntAlign = LongAlign = PointerAlign = 16;
 }
 
 bool M68kTargetInfo::setCPU(const std::string &Name) {
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 77145e2891a8..05a5dc2d9425 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -89,6 +89,12 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
 }
 
 static void defineXLCompatMacros(MacroBuilder &Builder) {
+  Builder.defineMacro("__builtin_national2packed",
+                      "__builtin_ppc_national2packed");
+  Builder.defineMacro("__builtin_packed2national",
+                      "__builtin_ppc_packed2national");
+  Builder.defineMacro("__builtin_packed2zoned", "__builtin_ppc_packed2zoned");
+  Builder.defineMacro("__builtin_zoned2packed", "__builtin_ppc_zoned2packed");
   Builder.defineMacro("__cdtbcd", "__builtin_ppc_cdtbcd");
   Builder.defineMacro("__cbcdtd", "__builtin_ppc_cbcdtd");
   Builder.defineMacro("__addg6s", "__builtin_ppc_addg6s");
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 2098449dd83a..8a28c0788aad 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -222,7 +222,7 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   // Currently we support the v1.0 RISC-V V intrinsics.
   Builder.defineMacro("__riscv_v_intrinsic", Twine(getVersionValue(1, 0)));
 
-  auto VScale = getVScaleRange(Opts, false);
+  auto VScale = getVScaleRange(Opts, ArmStreamingKind::NotStreaming);
   if (VScale && VScale->first && VScale->first == VScale->second)
     Builder.defineMacro("__riscv_v_fixed_vlen",
                         Twine(VScale->first * llvm::RISCV::RVVBitsPerBlock));
@@ -367,7 +367,7 @@ bool RISCVTargetInfo::initFeatureMap(
 
 std::optional<std::pair<unsigned, unsigned>>
 RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts,
-                                bool IsArmStreamingFunction,
+                                ArmStreamingKind IsArmStreamingFunction,
                                 llvm::StringMap<bool> *FeatureMap) const {
   // RISCV::RVVBitsPerBlock is 64.
   unsigned VScaleMin = ISAInfo->getMinVLen() / llvm::RISCV::RVVBitsPerBlock;
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 0b36c9d5d9cc..8d629abab7be 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -99,7 +99,7 @@ class RISCVTargetInfo : public TargetInfo {
                  const std::vector<std::string> &FeaturesVec) const override;
 
   std::optional<std::pair<unsigned, unsigned>>
-  getVScaleRange(const LangOptions &LangOpts, bool IsArmStreamingFunction,
+  getVScaleRange(const LangOptions &LangOpts, ArmStreamingKind Mode,
                  llvm::StringMap<bool> *FeatureMap = nullptr) const override;
 
   bool hasFeature(StringRef Feature) const override;
diff --git a/clang/lib/Basic/Targets/SPIR.cpp b/clang/lib/Basic/Targets/SPIR.cpp
index 610f4a3eecc6..19f160e279b2 100644
--- a/clang/lib/Basic/Targets/SPIR.cpp
+++ b/clang/lib/Basic/Targets/SPIR.cpp
@@ -185,3 +185,8 @@ void SPIRV64AMDGCNTargetInfo::setAuxTarget(const TargetInfo *Aux) {
 bool SPIRV64AMDGCNTargetInfo::isValidCPUName(StringRef CPU) const {
   return AMDGPUTI.isValidCPUName(CPU);
 }
+
+void SPIRV64AMDGCNTargetInfo::fillValidCPUList(
+    SmallVectorImpl<StringRef> &Values) const {
+  return AMDGPUTI.fillValidCPUList(Values);
+}
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 10b3108b5fe8..ebcdcabe3f24 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -455,6 +455,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
   // This is only needed for validating arguments passed to
   // __builtin_amdgcn_processor_is
   bool isValidCPUName(StringRef Name) const override;
+  void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
 };
 
 } // namespace targets
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index 1af6122c7048..7f7dcf815bd8 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -90,6 +90,8 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
       resetDataLayout("E-m:l-p1:32:32-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-"
                       "a:8:16-n32:64");
     } else {
+      // Support _Float16.
+      HasFloat16 = true;
       TLSSupported = true;
       resetDataLayout("E-m:e-i1:8:16-i8:8:16-i64:64-f128:64"
                       "-v128:64-a:8:16-n32:64");
@@ -103,8 +105,6 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
     // and instead the backend will promote each half operation to float
     // individually.
     HasLegalHalfType = false;
-    // Support _Float16.
-    HasFloat16 = true;
 
     HasStrictFP = true;
   }
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
index 9cec17bcb2fd..4a5a1dd53a05 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
@@ -62,9 +62,8 @@ cir::ConstantOp CIRGenBuilderTy::getConstInt(mlir::Location loc, mlir::Type t,
 cir::ConstantOp
 clang::CIRGen::CIRGenBuilderTy::getConstFP(mlir::Location loc, mlir::Type t,
                                            llvm::APFloat fpVal) {
-  assert(mlir::isa<cir::CIRFPTypeInterface>(t) &&
-         "expected floating point type");
-  return create<cir::ConstantOp>(loc, getAttr<cir::FPAttr>(t, fpVal));
+  assert(mlir::isa<cir::FPTypeInterface>(t) && "expected floating point type");
+  return create<cir::ConstantOp>(loc, cir::FPAttr::get(t, fpVal));
 }
 
 // This can't be defined in Address.h because that file is included by
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index e38faba83b80..0b33f6c7d03b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -10,8 +10,9 @@
 #define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENBUILDER_H
 
 #include "Address.h"
+#include "CIRGenRecordLayout.h"
 #include "CIRGenTypeCache.h"
-#include "clang/CIR/Interfaces/CIRFPTypeInterface.h"
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 #include "clang/CIR/MissingFeatures.h"
 
 #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
@@ -139,19 +140,6 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     return getType<cir::RecordType>(nameAttr, kind);
   }
 
-  bool isSized(mlir::Type ty) {
-    if (mlir::isa<cir::PointerType, cir::ArrayType, cir::BoolType, cir::IntType,
-                  cir::CIRFPTypeInterface, cir::ComplexType, cir::RecordType>(
-            ty))
-      return true;
-
-    if (const auto vt = mlir::dyn_cast<cir::VectorType>(ty))
-      return isSized(vt.getElementType());
-
-    assert(!cir::MissingFeatures::unsizedTypes());
-    return false;
-  }
-
   // Return true if the value is a null constant such as null pointer, (+0.0)
   // for floating-point or zero initializer
   bool isNullValue(mlir::Attribute attr) const {
@@ -366,6 +354,16 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     return create<cir::ComplexCreateOp>(loc, resultComplexTy, real, imag);
   }
 
+  mlir::Value createComplexReal(mlir::Location loc, mlir::Value operand) {
+    auto operandTy = mlir::cast<cir::ComplexType>(operand.getType());
+    return create<cir::ComplexRealOp>(loc, operandTy.getElementType(), operand);
+  }
+
+  mlir::Value createComplexImag(mlir::Location loc, mlir::Value operand) {
+    auto operandTy = mlir::cast<cir::ComplexType>(operand.getType());
+    return create<cir::ComplexImagOp>(loc, operandTy.getElementType(), operand);
+  }
+
   /// Create a cir.ptr_stride operation to get access to an array element.
   /// \p idx is the index of the element to access, \p shouldDecay is true if
   /// the result should decay to a pointer to the element type.
@@ -395,6 +393,15 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
 
     return createGlobal(module, loc, uniqueName, type, linkage);
   }
+
+  mlir::Value createGetBitfield(mlir::Location loc, mlir::Type resultType,
+                                mlir::Value addr, mlir::Type storageType,
+                                const CIRGenBitFieldInfo &info,
+                                bool isLvalueVolatile, bool useVolatile) {
+    return create<cir::GetBitfieldOp>(loc, resultType, addr, storageType,
+                                      info.name, info.size, info.offset,
+                                      info.isSigned, isLvalueVolatile);
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index cff139a7802d..fb046533a91b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -34,6 +34,29 @@ static RValue emitLibraryCall(CIRGenFunction &cgf, const FunctionDecl *fd,
   return cgf.emitCall(e->getCallee()->getType(), callee, e, ReturnValueSlot());
 }
 
+template <typename Op>
+static RValue emitBuiltinBitOp(CIRGenFunction &cgf, const CallExpr *e,
+                               bool poisonZero = false) {
+  assert(!cir::MissingFeatures::builtinCheckKind());
+
+  mlir::Value arg = cgf.emitScalarExpr(e->getArg(0));
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+
+  Op op;
+  if constexpr (std::is_same_v<Op, cir::BitClzOp> ||
+                std::is_same_v<Op, cir::BitCtzOp>)
+    op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg, poisonZero);
+  else
+    op = builder.create<Op>(cgf.getLoc(e->getSourceRange()), arg);
+
+  mlir::Value result = op.getResult();
+  mlir::Type exprTy = cgf.convertType(e->getType());
+  if (exprTy != result.getType())
+    result = builder.createIntCast(result, exprTy);
+
+  return RValue::get(result);
+}
+
 RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
                                        const CallExpr *e,
                                        ReturnValueSlot returnValue) {
@@ -100,6 +123,73 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
     mlir::Value complex = builder.createComplexCreate(loc, real, imag);
     return RValue::get(complex);
   }
+
+  case Builtin::BI__builtin_clrsb:
+  case Builtin::BI__builtin_clrsbl:
+  case Builtin::BI__builtin_clrsbll:
+    return emitBuiltinBitOp<cir::BitClrsbOp>(*this, e);
+
+  case Builtin::BI__builtin_ctzs:
+  case Builtin::BI__builtin_ctz:
+  case Builtin::BI__builtin_ctzl:
+  case Builtin::BI__builtin_ctzll:
+  case Builtin::BI__builtin_ctzg:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitCtzOp>(*this, e, /*poisonZero=*/true);
+
+  case Builtin::BI__builtin_clzs:
+  case Builtin::BI__builtin_clz:
+  case Builtin::BI__builtin_clzl:
+  case Builtin::BI__builtin_clzll:
+  case Builtin::BI__builtin_clzg:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/true);
+
+  case Builtin::BI__builtin_parity:
+  case Builtin::BI__builtin_parityl:
+  case Builtin::BI__builtin_parityll:
+    return emitBuiltinBitOp<cir::BitParityOp>(*this, e);
+
+  case Builtin::BI__lzcnt16:
+  case Builtin::BI__lzcnt:
+  case Builtin::BI__lzcnt64:
+    assert(!cir::MissingFeatures::builtinCheckKind());
+    return emitBuiltinBitOp<cir::BitClzOp>(*this, e, /*poisonZero=*/false);
+
+  case Builtin::BI__popcnt16:
+  case Builtin::BI__popcnt:
+  case Builtin::BI__popcnt64:
+  case Builtin::BI__builtin_popcount:
+  case Builtin::BI__builtin_popcountl:
+  case Builtin::BI__builtin_popcountll:
+  case Builtin::BI__builtin_popcountg:
+    return emitBuiltinBitOp<cir::BitPopcountOp>(*this, e);
+
+  case Builtin::BI__builtin_expect:
+  case Builtin::BI__builtin_expect_with_probability: {
+    mlir::Value argValue = emitScalarExpr(e->getArg(0));
+    mlir::Value expectedValue = emitScalarExpr(e->getArg(1));
+
+    mlir::FloatAttr probAttr;
+    if (builtinIDIfNoAsmLabel == Builtin::BI__builtin_expect_with_probability) {
+      llvm::APFloat probability(0.0);
+      const Expr *probArg = e->getArg(2);
+      [[maybe_unused]] bool evalSucceeded =
+          probArg->EvaluateAsFloat(probability, cgm.getASTContext());
+      assert(evalSucceeded &&
+             "probability should be able to evaluate as float");
+      bool loseInfo = false; // ignored
+      probability.convert(llvm::APFloat::IEEEdouble(),
+                          llvm::RoundingMode::Dynamic, &loseInfo);
+      probAttr = mlir::FloatAttr::get(mlir::Float64Type::get(&getMLIRContext()),
+                                      probability);
+    }
+
+    auto result = builder.create<cir::ExpectOp>(getLoc(e->getSourceRange()),
+                                                argValue.getType(), argValue,
+                                                expectedValue, probAttr);
+    return RValue::get(result);
+  }
   }
 
   cgm.errorNYI(e->getSourceRange(), "unimplemented builtin call");
diff --git a/clang/lib/CIR/CodeGen/CIRGenCXX.cpp b/clang/lib/CIR/CodeGen/CIRGenCXX.cpp
index 51751483d34e..da507d6f2833 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCXX.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCXX.cpp
@@ -25,7 +25,7 @@ cir::FuncOp CIRGenModule::codegenCXXStructor(GlobalDecl gd) {
   cir::FuncType funcType = getTypes().getFunctionType(fnInfo);
   cir::FuncOp fn = getAddrOfCXXStructor(gd, &fnInfo, /*FnType=*/nullptr,
                                         /*DontDefer=*/true, ForDefinition);
-  assert(!cir::MissingFeatures::opFuncLinkage());
+  setFunctionLinkage(gd, fn);
   CIRGenFunction cgf{*this, builder};
   curCGF = &cgf;
   {
diff --git a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
index d3888baea5d5..8da832d9118e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
@@ -85,13 +85,13 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     return RValue::get(nullptr);
   }
 
-  bool trivialForCodegen =
-      md->isTrivial() || (md->isDefaulted() && md->getParent()->isUnion());
-  bool trivialAssignment =
-      trivialForCodegen &&
-      (md->isCopyAssignmentOperator() || md->isMoveAssignmentOperator()) &&
-      !md->getParent()->mayInsertExtraPadding();
-  (void)trivialAssignment;
+  // Note on trivial assignment
+  // --------------------------
+  // Classic codegen avoids generating the trivial copy/move assignment operator
+  // when it isn't necessary, choosing instead to just produce IR with an
+  // equivalent effect. We have chosen not to do that in CIR, instead emitting
+  // trivial copy/move assignment operators and allowing later transformations
+  // to optimize them away if appropriate.
 
   // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment
   // operator before the LHS.
@@ -99,9 +99,10 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
   CallArgList *rtlArgs = nullptr;
   if (auto *oce = dyn_cast<CXXOperatorCallExpr>(ce)) {
     if (oce->isAssignmentOp()) {
-      cgm.errorNYI(
-          oce->getSourceRange(),
-          "emitCXXMemberOrOperatorMemberCallExpr: assignment operator");
+      rtlArgs = &rtlArgStorage;
+      emitCallArgs(*rtlArgs, md->getType()->castAs<FunctionProtoType>(),
+                   drop_begin(ce->arguments(), 1), ce->getDirectCallee(),
+                   /*ParamsToSkip*/ 0);
     }
   }
 
@@ -121,19 +122,9 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     return RValue::get(nullptr);
   }
 
-  if (trivialForCodegen) {
-    if (isa<CXXDestructorDecl>(md))
-      return RValue::get(nullptr);
-
-    if (trivialAssignment) {
-      cgm.errorNYI(ce->getSourceRange(),
-                   "emitCXXMemberOrOperatorMemberCallExpr: trivial assignment");
-      return RValue::get(nullptr);
-    }
-
-    assert(md->getParent()->mayInsertExtraPadding() &&
-           "unknown trivial member function");
-  }
+  if ((md->isTrivial() || (md->isDefaulted() && md->getParent()->isUnion())) &&
+      isa<CXXDestructorDecl>(md))
+    return RValue::get(nullptr);
 
   // Compute the function type we're calling
   const CXXMethodDecl *calleeDecl = md;
@@ -197,3 +188,195 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorCall(
   assert(!cir::MissingFeatures::opCallMustTail());
   return emitCall(fnInfo, callee, returnValue, args, nullptr, loc);
 }
+
+static mlir::Value emitCXXNewAllocSize(CIRGenFunction &cgf, const CXXNewExpr *e,
+                                       unsigned minElements,
+                                       mlir::Value &numElements,
+                                       mlir::Value &sizeWithoutCookie) {
+  QualType type = e->getAllocatedType();
+  mlir::Location loc = cgf.getLoc(e->getSourceRange());
+
+  if (!e->isArray()) {
+    CharUnits typeSize = cgf.getContext().getTypeSizeInChars(type);
+    sizeWithoutCookie = cgf.getBuilder().getConstant(
+        loc, cir::IntAttr::get(cgf.SizeTy, typeSize.getQuantity()));
+    return sizeWithoutCookie;
+  }
+
+  cgf.cgm.errorNYI(e->getSourceRange(), "emitCXXNewAllocSize: array");
+  return {};
+}
+
+static void storeAnyExprIntoOneUnit(CIRGenFunction &cgf, const Expr *init,
+                                    QualType allocType, Address newPtr,
+                                    AggValueSlot::Overlap_t mayOverlap) {
+  // FIXME: Refactor with emitExprAsInit.
+  switch (cgf.getEvaluationKind(allocType)) {
+  case cir::TEK_Scalar:
+    cgf.emitScalarInit(init, cgf.getLoc(init->getSourceRange()),
+                       cgf.makeAddrLValue(newPtr, allocType), false);
+    return;
+  case cir::TEK_Complex:
+    cgf.cgm.errorNYI(init->getSourceRange(),
+                     "storeAnyExprIntoOneUnit: complex");
+    return;
+  case cir::TEK_Aggregate: {
+    assert(!cir::MissingFeatures::aggValueSlotGC());
+    assert(!cir::MissingFeatures::sanitizers());
+    AggValueSlot slot = AggValueSlot::forAddr(
+        newPtr, allocType.getQualifiers(), AggValueSlot::IsDestructed,
+        AggValueSlot::IsNotAliased, mayOverlap, AggValueSlot::IsNotZeroed);
+    cgf.emitAggExpr(init, slot);
+    return;
+  }
+  }
+  llvm_unreachable("bad evaluation kind");
+}
+
+static void emitNewInitializer(CIRGenFunction &cgf, const CXXNewExpr *e,
+                               QualType elementType, mlir::Type elementTy,
+                               Address newPtr, mlir::Value numElements,
+                               mlir::Value allocSizeWithoutCookie) {
+  assert(!cir::MissingFeatures::generateDebugInfo());
+  if (e->isArray()) {
+    cgf.cgm.errorNYI(e->getSourceRange(), "emitNewInitializer: array");
+  } else if (const Expr *init = e->getInitializer()) {
+    storeAnyExprIntoOneUnit(cgf, init, e->getAllocatedType(), newPtr,
+                            AggValueSlot::DoesNotOverlap);
+  }
+}
+
+/// Emit a call to an operator new or operator delete function, as implicitly
+/// created by new-expressions and delete-expressions.
+static RValue emitNewDeleteCall(CIRGenFunction &cgf,
+                                const FunctionDecl *calleeDecl,
+                                const FunctionProtoType *calleeType,
+                                const CallArgList &args) {
+  cir::CIRCallOpInterface callOrTryCall;
+  cir::FuncOp calleePtr = cgf.cgm.getAddrOfFunction(calleeDecl);
+  CIRGenCallee callee =
+      CIRGenCallee::forDirect(calleePtr, GlobalDecl(calleeDecl));
+  RValue rv =
+      cgf.emitCall(cgf.cgm.getTypes().arrangeFreeFunctionCall(args, calleeType),
+                   callee, ReturnValueSlot(), args, &callOrTryCall);
+
+  /// C++1y [expr.new]p10:
+  ///   [In a new-expression,] an implementation is allowed to omit a call
+  ///   to a replaceable global allocation function.
+  ///
+  /// We model such elidable calls with the 'builtin' attribute.
+  assert(!cir::MissingFeatures::attributeBuiltin());
+  return rv;
+}
+
+mlir::Value CIRGenFunction::emitCXXNewExpr(const CXXNewExpr *e) {
+  // The element type being allocated.
+  QualType allocType = getContext().getBaseElementType(e->getAllocatedType());
+
+  // 1. Build a call to the allocation function.
+  FunctionDecl *allocator = e->getOperatorNew();
+
+  // If there is a brace-initializer, cannot allocate fewer elements than inits.
+  unsigned minElements = 0;
+  if (e->isArray() && e->hasInitializer()) {
+    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: array initializer");
+  }
+
+  mlir::Value numElements = nullptr;
+  mlir::Value allocSizeWithoutCookie = nullptr;
+  mlir::Value allocSize = emitCXXNewAllocSize(
+      *this, e, minElements, numElements, allocSizeWithoutCookie);
+  CharUnits allocAlign = getContext().getTypeAlignInChars(allocType);
+
+  // Emit the allocation call.
+  Address allocation = Address::invalid();
+  CallArgList allocatorArgs;
+  if (allocator->isReservedGlobalPlacementOperator()) {
+    cgm.errorNYI(e->getSourceRange(),
+                 "emitCXXNewExpr: reserved global placement operator");
+  } else {
+    const FunctionProtoType *allocatorType =
+        allocator->getType()->castAs<FunctionProtoType>();
+    unsigned paramsToSkip = 0;
+
+    // The allocation size is the first argument.
+    QualType sizeType = getContext().getSizeType();
+    allocatorArgs.add(RValue::get(allocSize), sizeType);
+    ++paramsToSkip;
+
+    if (allocSize != allocSizeWithoutCookie) {
+      CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI.
+      allocAlign = std::max(allocAlign, cookieAlign);
+    }
+
+    // The allocation alignment may be passed as the second argument.
+    if (e->passAlignment()) {
+      cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: pass alignment");
+    }
+
+    // FIXME: Why do we not pass a CalleeDecl here?
+    emitCallArgs(allocatorArgs, allocatorType, e->placement_arguments(),
+                 AbstractCallee(), paramsToSkip);
+    RValue rv =
+        emitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);
+
+    // Set !heapallocsite metadata on the call to operator new.
+    assert(!cir::MissingFeatures::generateDebugInfo());
+
+    // If this was a call to a global replaceable allocation function that does
+    // not take an alignment argument, the allocator is known to produce storage
+    // that's suitably aligned for any object that fits, up to a known
+    // threshold. Otherwise assume it's suitably aligned for the allocated type.
+    CharUnits allocationAlign = allocAlign;
+    if (!e->passAlignment() &&
+        allocator->isReplaceableGlobalAllocationFunction()) {
+      const TargetInfo &target = cgm.getASTContext().getTargetInfo();
+      unsigned allocatorAlign = llvm::bit_floor(std::min<uint64_t>(
+          target.getNewAlign(), getContext().getTypeSize(allocType)));
+      allocationAlign = std::max(
+          allocationAlign, getContext().toCharUnitsFromBits(allocatorAlign));
+    }
+
+    mlir::Value allocPtr = rv.getValue();
+    allocation = Address(
+        allocPtr, mlir::cast<cir::PointerType>(allocPtr.getType()).getPointee(),
+        allocationAlign);
+  }
+
+  // Emit a null check on the allocation result if the allocation
+  // function is allowed to return null (because it has a non-throwing
+  // exception spec or is the reserved placement new) and we have an
+  // interesting initializer will be running sanitizers on the initialization.
+  bool nullCheck = e->shouldNullCheckAllocation() &&
+                   (!allocType.isPODType(getContext()) || e->hasInitializer());
+  assert(!cir::MissingFeatures::exprNewNullCheck());
+  if (nullCheck)
+    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: null check");
+
+  // If there's an operator delete, enter a cleanup to call it if an
+  // exception is thrown.
+  if (e->getOperatorDelete() &&
+      !e->getOperatorDelete()->isReservedGlobalPlacementOperator())
+    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: operator delete");
+
+  if (allocSize != allocSizeWithoutCookie)
+    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: array with cookies");
+
+  mlir::Type elementTy = convertTypeForMem(allocType);
+  Address result = builder.createElementBitCast(getLoc(e->getSourceRange()),
+                                                allocation, elementTy);
+
+  // Passing pointer through launder.invariant.group to avoid propagation of
+  // vptrs information which may be included in previous type.
+  // To not break LTO with different optimizations levels, we do it regardless
+  // of optimization level.
+  if (cgm.getCodeGenOpts().StrictVTablePointers &&
+      allocator->isReservedGlobalPlacementOperator())
+    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: strict vtable pointers");
+
+  assert(!cir::MissingFeatures::sanitizers());
+
+  emitNewInitializer(*this, e, allocType, elementTy, result, numElements,
+                     allocSizeWithoutCookie);
+  return result.getPointer();
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index 9c9c96604c16..938d1436a76e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -77,17 +77,40 @@ void CIRGenFunction::emitAggregateStore(mlir::Value value, Address dest) {
   builder.createStore(*currSrcLoc, value, dest);
 }
 
+static void addAttributesFromFunctionProtoType(CIRGenBuilderTy &builder,
+                                               mlir::NamedAttrList &attrs,
+                                               const FunctionProtoType *fpt) {
+  if (!fpt)
+    return;
+
+  if (!isUnresolvedExceptionSpec(fpt->getExceptionSpecType()) &&
+      fpt->isNothrow())
+    attrs.set(cir::CIRDialect::getNoThrowAttrName(),
+              mlir::UnitAttr::get(builder.getContext()));
+}
+
 /// Construct the CIR attribute list of a function or call.
 void CIRGenModule::constructAttributeList(CIRGenCalleeInfo calleeInfo,
-                                          cir::SideEffect &sideEffect) {
+                                          mlir::NamedAttrList &attrs) {
   assert(!cir::MissingFeatures::opCallCallConv());
-  sideEffect = cir::SideEffect::All;
+  auto sideEffect = cir::SideEffect::All;
 
-  assert(!cir::MissingFeatures::opCallAttrs());
+  addAttributesFromFunctionProtoType(getBuilder(), attrs,
+                                     calleeInfo.getCalleeFunctionProtoType());
 
   const Decl *targetDecl = calleeInfo.getCalleeDecl().getDecl();
 
   if (targetDecl) {
+    if (targetDecl->hasAttr<NoThrowAttr>())
+      attrs.set(cir::CIRDialect::getNoThrowAttrName(),
+                mlir::UnitAttr::get(&getMLIRContext()));
+
+    if (const FunctionDecl *func = dyn_cast<FunctionDecl>(targetDecl)) {
+      addAttributesFromFunctionProtoType(
+          getBuilder(), attrs, func->getType()->getAs<FunctionProtoType>());
+      assert(!cir::MissingFeatures::opCallAttrs());
+    }
+
     assert(!cir::MissingFeatures::opCallAttrs());
 
     // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
@@ -104,6 +127,9 @@ void CIRGenModule::constructAttributeList(CIRGenCalleeInfo calleeInfo,
   }
 
   assert(!cir::MissingFeatures::opCallAttrs());
+
+  attrs.set(cir::CIRDialect::getSideEffectAttrName(),
+            cir::SideEffectAttr::get(&getMLIRContext(), sideEffect));
 }
 
 /// Returns the canonical formal type of the given C++ method.
@@ -404,7 +430,7 @@ CIRGenTypes::arrangeFunctionDeclaration(const FunctionDecl *fd) {
           funcTy.getAs<FunctionNoProtoType>()) {
     assert(!cir::MissingFeatures::opCallCIRGenFuncInfoExtParamInfo());
     assert(!cir::MissingFeatures::opCallFnInfoOpts());
-    return arrangeCIRFunctionInfo(noProto->getReturnType(), std::nullopt,
+    return arrangeCIRFunctionInfo(noProto->getReturnType(), {},
                                   RequiredArgs::All);
   }
 
@@ -416,7 +442,7 @@ emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
                cir::FuncType indirectFuncTy, mlir::Value indirectFuncVal,
                cir::FuncOp directFuncOp,
                const SmallVectorImpl<mlir::Value> &cirCallArgs,
-               cir::SideEffect sideEffect) {
+               const mlir::NamedAttrList &attrs) {
   CIRGenBuilderTy &builder = cgf.getBuilder();
 
   assert(!cir::MissingFeatures::opCallSurroundingTry());
@@ -424,14 +450,17 @@ emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
 
   assert(builder.getInsertionBlock() && "expected valid basic block");
 
+  cir::CallOp op;
   if (indirectFuncTy) {
     // TODO(cir): Set calling convention for indirect calls.
     assert(!cir::MissingFeatures::opCallCallConv());
-    return builder.createIndirectCallOp(
-        callLoc, indirectFuncVal, indirectFuncTy, cirCallArgs, sideEffect);
+    op = builder.createIndirectCallOp(callLoc, indirectFuncVal, indirectFuncTy,
+                                      cirCallArgs, attrs);
+  } else {
+    op = builder.createCallOp(callLoc, directFuncOp, cirCallArgs, attrs);
   }
 
-  return builder.createCallOp(callLoc, directFuncOp, cirCallArgs, sideEffect);
+  return op;
 }
 
 const CIRGenFunctionInfo &
@@ -544,8 +573,7 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
 
   assert(!cir::MissingFeatures::opCallCallConv());
   assert(!cir::MissingFeatures::opCallAttrs());
-  cir::SideEffect sideEffect;
-  cgm.constructAttributeList(callee.getAbstractInfo(), sideEffect);
+  cgm.constructAttributeList(callee.getAbstractInfo(), attrs);
 
   assert(!cir::MissingFeatures::invokeOp());
 
@@ -566,12 +594,10 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
     indirectFuncVal = calleePtr->getResult(0);
   }
 
-  assert(!cir::MissingFeatures::opCallAttrs());
-
   mlir::Location callLoc = loc;
   cir::CIRCallOpInterface theCall =
       emitCallLikeOp(*this, loc, indirectFuncTy, indirectFuncVal, directFuncOp,
-                     cirCallArgs, sideEffect);
+                     cirCallArgs, attrs);
 
   if (callOp)
     *callOp = theCall;
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 56c76c51a46d..bd113293fdaf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -33,7 +33,8 @@ class CIRGenCalleeInfo {
   CIRGenCalleeInfo(const clang::FunctionProtoType *calleeProtoTy,
                    clang::GlobalDecl calleeDecl)
       : calleeProtoTy(calleeProtoTy), calleeDecl(calleeDecl) {}
-  CIRGenCalleeInfo(clang::GlobalDecl calleeDecl) : calleeDecl(calleeDecl) {}
+  CIRGenCalleeInfo(clang::GlobalDecl calleeDecl)
+      : calleeProtoTy(nullptr), calleeDecl(calleeDecl) {}
 
   const clang::FunctionProtoType *getCalleeFunctionProtoType() const {
     return calleeProtoTy;
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index e59a1fdb837c..da8166a596d4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -53,19 +53,133 @@ bool CIRGenFunction::isConstructorDelegationValid(
   return true;
 }
 
+static void emitLValueForAnyFieldInitialization(CIRGenFunction &cgf,
+                                                CXXCtorInitializer *memberInit,
+                                                LValue &lhs) {
+  FieldDecl *field = memberInit->getAnyMember();
+  if (memberInit->isIndirectMemberInitializer()) {
+    // If we are initializing an anonymous union field, drill down to the field.
+    IndirectFieldDecl *indirectField = memberInit->getIndirectMember();
+    for (const auto *nd : indirectField->chain()) {
+      auto *fd = cast<clang::FieldDecl>(nd);
+      lhs = cgf.emitLValueForFieldInitialization(lhs, fd, fd->getName());
+    }
+  } else {
+    lhs = cgf.emitLValueForFieldInitialization(lhs, field, field->getName());
+  }
+}
+
+static void emitMemberInitializer(CIRGenFunction &cgf,
+                                  const CXXRecordDecl *classDecl,
+                                  CXXCtorInitializer *memberInit,
+                                  const CXXConstructorDecl *constructor,
+                                  FunctionArgList &args) {
+  assert(memberInit->isAnyMemberInitializer() &&
+         "Mush have member initializer!");
+  assert(memberInit->getInit() && "Must have initializer!");
+
+  assert(!cir::MissingFeatures::generateDebugInfo());
+
+  // non-static data member initializers
+  FieldDecl *field = memberInit->getAnyMember();
+  QualType fieldType = field->getType();
+
+  mlir::Value thisPtr = cgf.loadCXXThis();
+  QualType recordTy = cgf.getContext().getTypeDeclType(classDecl);
+
+  // If a base constructor is being emitted, create an LValue that has the
+  // non-virtual alignment.
+  LValue lhs = (cgf.curGD.getCtorType() == Ctor_Base)
+                   ? cgf.makeNaturalAlignPointeeAddrLValue(thisPtr, recordTy)
+                   : cgf.makeNaturalAlignAddrLValue(thisPtr, recordTy);
+
+  emitLValueForAnyFieldInitialization(cgf, memberInit, lhs);
+
+  // Special case: If we are in a copy or move constructor, and we are copying
+  // an array off PODs or classes with trivial copy constructors, ignore the AST
+  // and perform the copy we know is equivalent.
+  // FIXME: This is hacky at best... if we had a bit more explicit information
+  // in the AST, we could generalize it more easily.
+  const ConstantArrayType *array =
+      cgf.getContext().getAsConstantArrayType(fieldType);
+  if (array && constructor->isDefaulted() &&
+      constructor->isCopyOrMoveConstructor()) {
+    QualType baseElementTy = cgf.getContext().getBaseElementType(array);
+    // NOTE(cir): CodeGen allows record types to be memcpy'd if applicable,
+    // whereas ClangIR wants to represent all object construction explicitly.
+    if (!baseElementTy->isRecordType()) {
+      cgf.cgm.errorNYI(memberInit->getSourceRange(),
+                       "emitMemberInitializer: array of non-record type");
+      return;
+    }
+  }
+
+  cgf.emitInitializerForField(field, lhs, memberInit->getInit());
+}
+
 /// This routine generates necessary code to initialize base classes and
 /// non-static data members belonging to this constructor.
 void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
                                       CXXCtorType ctorType,
                                       FunctionArgList &args) {
-  if (cd->isDelegatingConstructor())
-    return emitDelegatingCXXConstructorCall(cd, args);
+  if (cd->isDelegatingConstructor()) {
+    emitDelegatingCXXConstructorCall(cd, args);
+    return;
+  }
 
-  if (cd->getNumCtorInitializers() != 0) {
-    // There's much more to do here.
-    cgm.errorNYI(cd->getSourceRange(), "emitCtorPrologue: any initializer");
+  // If there are no member initializers, we can just return.
+  if (cd->getNumCtorInitializers() == 0)
+    return;
+
+  const CXXRecordDecl *classDecl = cd->getParent();
+
+  // This code doesn't use range-based iteration because we may need to emit
+  // code between the virtual base initializers and the non-virtual base or
+  // between the non-virtual base initializers and the member initializers.
+  CXXConstructorDecl::init_const_iterator b = cd->init_begin(),
+                                          e = cd->init_end();
+
+  // Virtual base initializers first, if any. They aren't needed if:
+  // - This is a base ctor variant
+  // - There are no vbases
+  // - The class is abstract, so a complete object of it cannot be constructed
+  //
+  // The check for an abstract class is necessary because sema may not have
+  // marked virtual base destructors referenced.
+  bool constructVBases = ctorType != Ctor_Base &&
+                         classDecl->getNumVBases() != 0 &&
+                         !classDecl->isAbstract();
+  if (constructVBases) {
+    cgm.errorNYI(cd->getSourceRange(), "emitCtorPrologue: virtual base");
     return;
   }
+
+  if ((*b)->isBaseInitializer()) {
+    cgm.errorNYI(cd->getSourceRange(),
+                 "emitCtorPrologue: non-virtual base initializer");
+    return;
+  }
+
+  if (classDecl->isDynamicClass()) {
+    cgm.errorNYI(cd->getSourceRange(),
+                 "emitCtorPrologue: initialize vtable pointers");
+    return;
+  }
+
+  // Finally, initialize class members.
+  FieldConstructionScope fcs(*this, loadCXXThisAddress());
+  // Classic codegen uses a special class to attempt to replace member
+  // initializers with memcpy. We could possibly defer that to the
+  // lowering or optimization phases to keep the memory accesses more
+  // explicit. For now, we don't insert memcpy at all.
+  assert(!cir::MissingFeatures::ctorMemcpyizer());
+  for (; b != e; b++) {
+    CXXCtorInitializer *member = (*b);
+    assert(!member->isBaseInitializer());
+    assert(member->isAnyMemberInitializer() &&
+           "Delegating initializer on non-delegating constructor");
+    emitMemberInitializer(*this, cd->getParent(), member, cd, args);
+  }
 }
 
 Address CIRGenFunction::loadCXXThisAddress() {
@@ -84,6 +198,33 @@ Address CIRGenFunction::loadCXXThisAddress() {
   return Address(loadCXXThis(), cxxThisAlignment);
 }
 
+void CIRGenFunction::emitInitializerForField(FieldDecl *field, LValue lhs,
+                                             Expr *init) {
+  QualType fieldType = field->getType();
+  switch (getEvaluationKind(fieldType)) {
+  case cir::TEK_Scalar:
+    if (lhs.isSimple())
+      emitExprAsInit(init, field, lhs, false);
+    else
+      cgm.errorNYI(field->getSourceRange(),
+                   "emitInitializerForField: non-simple scalar");
+    break;
+  case cir::TEK_Complex:
+    cgm.errorNYI(field->getSourceRange(), "emitInitializerForField: complex");
+    break;
+  case cir::TEK_Aggregate: {
+    cgm.errorNYI(field->getSourceRange(), "emitInitializerForField: aggregate");
+    break;
+  }
+  }
+
+  // Ensure that we destroy this object if an exception is thrown later in the
+  // constructor.
+  QualType::DestructionKind dtorKind = fieldType.isDestructedType();
+  (void)dtorKind;
+  assert(!cir::MissingFeatures::requiresCleanups());
+}
+
 void CIRGenFunction::emitDelegateCXXConstructorCall(
     const CXXConstructorDecl *ctor, CXXCtorType ctorType,
     const FunctionArgList &args, SourceLocation loc) {
@@ -117,6 +258,31 @@ void CIRGenFunction::emitDelegateCXXConstructorCall(
                          /*Delegating=*/true, thisAddr, delegateArgs, loc);
 }
 
+void CIRGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &args) {
+  const auto *assignOp = cast<CXXMethodDecl>(curGD.getDecl());
+  assert(assignOp->isCopyAssignmentOperator() ||
+         assignOp->isMoveAssignmentOperator());
+  const Stmt *rootS = assignOp->getBody();
+  assert(isa<CompoundStmt>(rootS) &&
+         "Body of an implicit assignment operator should be compound stmt.");
+  const auto *rootCS = cast<CompoundStmt>(rootS);
+
+  assert(!cir::MissingFeatures::incrementProfileCounter());
+  assert(!cir::MissingFeatures::runCleanupsScope());
+
+  // Classic codegen uses a special class to attempt to replace member
+  // initializers with memcpy. We could possibly defer that to the
+  // lowering or optimization phases to keep the memory accesses more
+  // explicit. For now, we don't insert memcpy at all, though in some
+  // cases the AST contains a call to memcpy.
+  assert(!cir::MissingFeatures::assignMemcpyizer());
+  for (Stmt *s : rootCS->body())
+    if (emitStmt(s, /*useCurrentScope=*/true).failed())
+      cgm.errorNYI(s->getSourceRange(),
+                   std::string("emitImplicitAssignmentOperatorBody: ") +
+                       s->getStmtClassName());
+}
+
 void CIRGenFunction::emitDelegatingCXXConstructorCall(
     const CXXConstructorDecl *ctor, const FunctionArgList &args) {
   assert(ctor->isDelegatingConstructor());
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index c31754dc11d6..68d7f1f5bca4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -326,13 +326,62 @@ mlir::Value CIRGenFunction::emitStoreThroughBitfieldLValue(RValue src,
   return {};
 }
 
+RValue CIRGenFunction::emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc) {
+  const CIRGenBitFieldInfo &info = lv.getBitFieldInfo();
+
+  // Get the output type.
+  mlir::Type resLTy = convertType(lv.getType());
+  Address ptr = lv.getBitFieldAddress();
+
+  assert(!cir::MissingFeatures::armComputeVolatileBitfields());
+
+  mlir::Value field = builder.createGetBitfield(
+      getLoc(loc), resLTy, ptr.getPointer(), ptr.getElementType(), info,
+      lv.isVolatile(), false);
+  assert(!cir::MissingFeatures::opLoadEmitScalarRangeCheck() && "NYI");
+  return RValue::get(field);
+}
+
+Address CIRGenFunction::getAddrOfBitFieldStorage(LValue base,
+                                                 const FieldDecl *field,
+                                                 mlir::Type fieldType,
+                                                 unsigned index) {
+  mlir::Location loc = getLoc(field->getLocation());
+  cir::PointerType fieldPtr = cir::PointerType::get(fieldType);
+  cir::GetMemberOp sea = getBuilder().createGetMember(
+      loc, fieldPtr, base.getPointer(), field->getName(), index);
+  return Address(sea, CharUnits::One());
+}
+
+LValue CIRGenFunction::emitLValueForBitField(LValue base,
+                                             const FieldDecl *field) {
+  LValueBaseInfo baseInfo = base.getBaseInfo();
+  const CIRGenRecordLayout &layout =
+      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
+  const CIRGenBitFieldInfo &info = layout.getBitFieldInfo(field);
+  assert(!cir::MissingFeatures::armComputeVolatileBitfields());
+  assert(!cir::MissingFeatures::preservedAccessIndexRegion());
+  unsigned idx = layout.getCIRFieldNo(field);
+
+  Address addr = getAddrOfBitFieldStorage(base, field, info.storageType, idx);
+
+  mlir::Location loc = getLoc(field->getLocation());
+  if (addr.getElementType() != info.storageType)
+    addr = builder.createElementBitCast(loc, addr, info.storageType);
+
+  QualType fieldType =
+      field->getType().withCVRQualifiers(base.getVRQualifiers());
+  // TODO(cir): Support TBAA for bit fields.
+  assert(!cir::MissingFeatures::opTBAA());
+  LValueBaseInfo fieldBaseInfo(baseInfo.getAlignmentSource());
+  return LValue::makeBitfield(addr, info, fieldType, fieldBaseInfo);
+}
+
 LValue CIRGenFunction::emitLValueForField(LValue base, const FieldDecl *field) {
   LValueBaseInfo baseInfo = base.getBaseInfo();
 
-  if (field->isBitField()) {
-    cgm.errorNYI(field->getSourceRange(), "emitLValueForField: bitfield");
-    return LValue();
-  }
+  if (field->isBitField())
+    return emitLValueForBitField(base, field);
 
   QualType fieldType = field->getType();
   const RecordDecl *rec = field->getParent();
@@ -390,6 +439,34 @@ LValue CIRGenFunction::emitLValueForField(LValue base, const FieldDecl *field) {
   return lv;
 }
 
+LValue CIRGenFunction::emitLValueForFieldInitialization(
+    LValue base, const clang::FieldDecl *field, llvm::StringRef fieldName) {
+  QualType fieldType = field->getType();
+
+  if (!fieldType->isReferenceType())
+    return emitLValueForField(base, field);
+
+  const CIRGenRecordLayout &layout =
+      cgm.getTypes().getCIRGenRecordLayout(field->getParent());
+  unsigned fieldIndex = layout.getCIRFieldNo(field);
+
+  Address v =
+      emitAddrOfFieldStorage(base.getAddress(), field, fieldName, fieldIndex);
+
+  // Make sure that the address is pointing to the right type.
+  mlir::Type memTy = convertTypeForMem(fieldType);
+  v = builder.createElementBitCast(getLoc(field->getSourceRange()), v, memTy);
+
+  // TODO: Generate TBAA information that describes this access as a structure
+  // member access and not just an access to an object of the field's type. This
+  // should be similar to what we do in EmitLValueForField().
+  LValueBaseInfo baseInfo = base.getBaseInfo();
+  AlignmentSource fieldAlignSource = baseInfo.getAlignmentSource();
+  LValueBaseInfo fieldBaseInfo(getFieldAlignmentSource(fieldAlignSource));
+  assert(!cir::MissingFeatures::opTBAA());
+  return makeAddrLValue(v, fieldType, fieldBaseInfo);
+}
+
 mlir::Value CIRGenFunction::emitToMemory(mlir::Value value, QualType ty) {
   // Bool has a different representation in memory than in registers,
   // but in ClangIR, it is simply represented as a cir.bool value.
@@ -432,6 +509,9 @@ RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) {
   assert(!lv.getType()->isFunctionType());
   assert(!(lv.getType()->isConstantMatrixType()) && "not implemented");
 
+  if (lv.isBitField())
+    return emitLoadOfBitfieldLValue(lv, loc);
+
   if (lv.isSimple())
     return RValue::get(emitLoadOfScalar(lv, loc));
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index eaa199abc165..82b6a74a54b6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -39,12 +39,21 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
   void emitStoreOfComplex(mlir::Location loc, mlir::Value val, LValue lv,
                           bool isInit);
 
+  mlir::Value VisitArraySubscriptExpr(Expr *e);
   mlir::Value VisitBinAssign(const BinaryOperator *e);
+  mlir::Value VisitBinComma(const BinaryOperator *e);
   mlir::Value VisitCallExpr(const CallExpr *e);
+  mlir::Value VisitCastExpr(CastExpr *e);
+  mlir::Value VisitChooseExpr(ChooseExpr *e);
   mlir::Value VisitDeclRefExpr(DeclRefExpr *e);
+  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *e);
   mlir::Value VisitImplicitCastExpr(ImplicitCastExpr *e);
   mlir::Value VisitInitListExpr(const InitListExpr *e);
   mlir::Value VisitImaginaryLiteral(const ImaginaryLiteral *il);
+  mlir::Value VisitParenExpr(ParenExpr *e);
+  mlir::Value
+  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e);
+  mlir::Value VisitUnaryDeref(const Expr *e);
 };
 } // namespace
 
@@ -75,12 +84,13 @@ LValue ComplexExprEmitter::emitBinAssignLValue(const BinaryOperator *e,
 mlir::Value ComplexExprEmitter::emitCast(CastKind ck, Expr *op,
                                          QualType destTy) {
   switch (ck) {
+  case CK_NoOp:
   case CK_LValueToRValue:
     return Visit(op);
   default:
-    cgf.cgm.errorNYI("ComplexType Cast");
     break;
   }
+  cgf.cgm.errorNYI("ComplexType Cast");
   return {};
 }
 
@@ -117,6 +127,10 @@ void ComplexExprEmitter::emitStoreOfComplex(mlir::Location loc, mlir::Value val,
   builder.createStore(loc, val, destAddr);
 }
 
+mlir::Value ComplexExprEmitter::VisitArraySubscriptExpr(Expr *e) {
+  return emitLoadOfLValue(e);
+}
+
 mlir::Value ComplexExprEmitter::VisitBinAssign(const BinaryOperator *e) {
   mlir::Value value;
   LValue lv = emitBinAssignLValue(e, value);
@@ -133,6 +147,11 @@ mlir::Value ComplexExprEmitter::VisitBinAssign(const BinaryOperator *e) {
   return emitLoadOfLValue(lv, e->getExprLoc());
 }
 
+mlir::Value ComplexExprEmitter::VisitBinComma(const BinaryOperator *e) {
+  cgf.emitIgnoredExpr(e->getLHS());
+  return Visit(e->getRHS());
+}
+
 mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) {
   if (e->getCallReturnType(cgf.getContext())->isReferenceType())
     return emitLoadOfLValue(e);
@@ -140,12 +159,36 @@ mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) {
   return cgf.emitCallExpr(e).getValue();
 }
 
+mlir::Value ComplexExprEmitter::VisitCastExpr(CastExpr *e) {
+  if (const auto *ece = dyn_cast<ExplicitCastExpr>(e)) {
+    // Bind VLAs in the cast type.
+    if (ece->getType()->isVariablyModifiedType()) {
+      cgf.cgm.errorNYI("VisitCastExpr Bind VLAs in the cast type");
+      return {};
+    }
+  }
+
+  if (e->changesVolatileQualification())
+    return emitLoadOfLValue(e);
+
+  return emitCast(e->getCastKind(), e->getSubExpr(), e->getType());
+}
+
+mlir::Value ComplexExprEmitter::VisitChooseExpr(ChooseExpr *e) {
+  return Visit(e->getChosenSubExpr());
+}
+
 mlir::Value ComplexExprEmitter::VisitDeclRefExpr(DeclRefExpr *e) {
   if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(e))
     return emitConstant(constant, e);
   return emitLoadOfLValue(e);
 }
 
+mlir::Value
+ComplexExprEmitter::VisitGenericSelectionExpr(GenericSelectionExpr *e) {
+  return Visit(e->getResultExpr());
+}
+
 mlir::Value ComplexExprEmitter::VisitImplicitCastExpr(ImplicitCastExpr *e) {
   // Unlike for scalars, we don't have to worry about function->ptr demotion
   // here.
@@ -190,7 +233,7 @@ ComplexExprEmitter::VisitImaginaryLiteral(const ImaginaryLiteral *il) {
     realValueAttr = cir::IntAttr::get(elementTy, 0);
     imagValueAttr = cir::IntAttr::get(elementTy, imagValue);
   } else {
-    assert(mlir::isa<cir::CIRFPTypeInterface>(elementTy) &&
+    assert(mlir::isa<cir::FPTypeInterface>(elementTy) &&
            "Expected complex element type to be floating-point");
 
     llvm::APFloat imagValue =
@@ -204,6 +247,19 @@ ComplexExprEmitter::VisitImaginaryLiteral(const ImaginaryLiteral *il) {
   return builder.create<cir::ConstantOp>(loc, complexAttr);
 }
 
+mlir::Value ComplexExprEmitter::VisitParenExpr(ParenExpr *e) {
+  return Visit(e->getSubExpr());
+}
+
+mlir::Value ComplexExprEmitter::VisitSubstNonTypeTemplateParmExpr(
+    SubstNonTypeTemplateParmExpr *e) {
+  return Visit(e->getReplacement());
+}
+
+mlir::Value ComplexExprEmitter::VisitUnaryDeref(const Expr *e) {
+  return emitLoadOfLValue(e);
+}
+
 LValue CIRGenFunction::emitComplexAssignmentLValue(const BinaryOperator *e) {
   assert(e->getOpcode() == BO_Assign && "Expected assign op");
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 8b817f3f3d8d..5b3bf85cefbb 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -684,7 +684,7 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
     if (mlir::isa<cir::BoolType>(ty))
       return builder.getCIRBoolAttr(value.getInt().getZExtValue());
     assert(mlir::isa<cir::IntType>(ty) && "expected integral type");
-    return cgm.getBuilder().getAttr<cir::IntAttr>(ty, value.getInt());
+    return cir::IntAttr::get(ty, value.getInt());
   }
   case APValue::Float: {
     const llvm::APFloat &init = value.getFloat();
@@ -696,9 +696,9 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
     }
 
     mlir::Type ty = cgm.convertType(destType);
-    assert(mlir::isa<cir::CIRFPTypeInterface>(ty) &&
+    assert(mlir::isa<cir::FPTypeInterface>(ty) &&
            "expected floating-point type");
-    return cgm.getBuilder().getAttr<cir::FPAttr>(ty, init);
+    return cir::FPAttr::get(ty, init);
   }
   case APValue::Array: {
     const ArrayType *arrayTy = cgm.getASTContext().getAsArrayType(destType);
@@ -789,17 +789,17 @@ mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
       llvm::APSInt real = value.getComplexIntReal();
       llvm::APSInt imag = value.getComplexIntImag();
       return builder.getAttr<cir::ConstComplexAttr>(
-          complexType, builder.getAttr<cir::IntAttr>(complexElemTy, real),
-          builder.getAttr<cir::IntAttr>(complexElemTy, imag));
+          complexType, cir::IntAttr::get(complexElemTy, real),
+          cir::IntAttr::get(complexElemTy, imag));
     }
 
-    assert(isa<cir::CIRFPTypeInterface>(complexElemTy) &&
+    assert(isa<cir::FPTypeInterface>(complexElemTy) &&
            "expected floating-point type");
     llvm::APFloat real = value.getComplexFloatReal();
     llvm::APFloat imag = value.getComplexFloatImag();
     return builder.getAttr<cir::ConstComplexAttr>(
-        complexType, builder.getAttr<cir::FPAttr>(complexElemTy, real),
-        builder.getAttr<cir::FPAttr>(complexElemTy, imag));
+        complexType, cir::FPAttr::get(complexElemTy, real),
+        cir::FPAttr::get(complexElemTy, imag));
   }
   case APValue::FixedPoint:
   case APValue::AddrLabelDiff:
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 8d0db5cd0a1e..10c485c85e22 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -125,8 +125,16 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return {};
   }
 
+  mlir::Value VisitPackIndexingExpr(PackIndexingExpr *e) {
+    return Visit(e->getSelectedExpr());
+  }
+
   mlir::Value VisitParenExpr(ParenExpr *pe) { return Visit(pe->getSubExpr()); }
 
+  mlir::Value VisitGenericSelectionExpr(GenericSelectionExpr *ge) {
+    return Visit(ge->getResultExpr());
+  }
+
   /// Emits the address of the l-value, then loads and returns the result.
   mlir::Value emitLoadOfLValue(const Expr *e) {
     LValue lv = cgf.emitLValue(e);
@@ -149,17 +157,15 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   mlir::Value VisitIntegerLiteral(const IntegerLiteral *e) {
     mlir::Type type = cgf.convertType(e->getType());
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()),
-        builder.getAttr<cir::IntAttr>(type, e->getValue()));
+        cgf.getLoc(e->getExprLoc()), cir::IntAttr::get(type, e->getValue()));
   }
 
   mlir::Value VisitFloatingLiteral(const FloatingLiteral *e) {
     mlir::Type type = cgf.convertType(e->getType());
-    assert(mlir::isa<cir::CIRFPTypeInterface>(type) &&
+    assert(mlir::isa<cir::FPTypeInterface>(type) &&
            "expect floating-point type");
     return builder.create<cir::ConstantOp>(
-        cgf.getLoc(e->getExprLoc()),
-        builder.getAttr<cir::FPAttr>(type, e->getValue()));
+        cgf.getLoc(e->getExprLoc()), cir::FPAttr::get(type, e->getValue()));
   }
 
   mlir::Value VisitCharacterLiteral(const CharacterLiteral *e) {
@@ -331,18 +337,18 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
         cgf.getCIRGenModule().errorNYI("signed bool");
       if (cgf.getBuilder().isInt(dstTy))
         castKind = cir::CastKind::bool_to_int;
-      else if (mlir::isa<cir::CIRFPTypeInterface>(dstTy))
+      else if (mlir::isa<cir::FPTypeInterface>(dstTy))
         castKind = cir::CastKind::bool_to_float;
       else
         llvm_unreachable("Internal error: Cast to unexpected type");
     } else if (cgf.getBuilder().isInt(srcTy)) {
       if (cgf.getBuilder().isInt(dstTy))
         castKind = cir::CastKind::integral;
-      else if (mlir::isa<cir::CIRFPTypeInterface>(dstTy))
+      else if (mlir::isa<cir::FPTypeInterface>(dstTy))
         castKind = cir::CastKind::int_to_float;
       else
         llvm_unreachable("Internal error: Cast to unexpected type");
-    } else if (mlir::isa<cir::CIRFPTypeInterface>(srcTy)) {
+    } else if (mlir::isa<cir::FPTypeInterface>(srcTy)) {
       if (cgf.getBuilder().isInt(dstTy)) {
         // If we can't recognize overflow as undefined behavior, assume that
         // overflow saturates. This protects against normal optimizations if we
@@ -351,7 +357,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
           cgf.getCIRGenModule().errorNYI("strict float cast overflow");
         assert(!cir::MissingFeatures::fpConstraints());
         castKind = cir::CastKind::float_to_int;
-      } else if (mlir::isa<cir::CIRFPTypeInterface>(dstTy)) {
+      } else if (mlir::isa<cir::FPTypeInterface>(dstTy)) {
         // TODO: split this to createFPExt/createFPTrunc
         return builder.createFloatingCast(src, fullDstTy);
       } else {
@@ -365,6 +371,11 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return builder.create<cir::CastOp>(src.getLoc(), fullDstTy, *castKind, src);
   }
 
+  mlir::Value
+  VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *e) {
+    return Visit(e->getReplacement());
+  }
+
   mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
   mlir::Value
   VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
@@ -603,8 +614,16 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
   mlir::Value VisitUnaryLNot(const UnaryOperator *e);
 
+  mlir::Value VisitUnaryReal(const UnaryOperator *e);
+
+  mlir::Value VisitUnaryImag(const UnaryOperator *e);
+
   mlir::Value VisitCXXThisExpr(CXXThisExpr *te) { return cgf.loadCXXThis(); }
 
+  mlir::Value VisitCXXNewExpr(const CXXNewExpr *e) {
+    return cgf.emitCXXNewExpr(e);
+  }
+
   /// Emit a conversion from the specified type to the specified destination
   /// type, both of which are CIR scalar types.
   /// TODO: do we need ScalarConversionOpts here? Should be done in another
@@ -650,7 +669,7 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     if (srcType->isHalfType() &&
         !cgf.getContext().getLangOpts().NativeHalfType) {
       // Cast to FP using the intrinsic if the half type itself isn't supported.
-      if (mlir::isa<cir::CIRFPTypeInterface>(mlirDstType)) {
+      if (mlir::isa<cir::FPTypeInterface>(mlirDstType)) {
         if (cgf.getContext().getTargetInfo().useFP16ConversionIntrinsics())
           cgf.getCIRGenModule().errorNYI(loc,
                                          "cast via llvm.convert.from.fp16");
@@ -890,9 +909,10 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
       }
     } else {
       // Complex Comparison: can only be an equality comparison.
-      assert(!cir::MissingFeatures::complexType());
-      cgf.cgm.errorNYI(loc, "complex comparison");
-      result = builder.getBool(false, loc);
+      assert(e->getOpcode() == BO_EQ || e->getOpcode() == BO_NE);
+
+      BinOpInfo boInfo = emitBinOps(e);
+      result = builder.create<cir::CmpOp>(loc, kind, boInfo.lhs, boInfo.rhs);
     }
 
     return emitScalarConversion(result, cgf.getContext().BoolTy, e->getType(),
@@ -1891,6 +1911,48 @@ mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
   return maybePromoteBoolResult(boolVal, cgf.convertType(e->getType()));
 }
 
+mlir::Value ScalarExprEmitter::VisitUnaryReal(const UnaryOperator *e) {
+  // TODO(cir): handle scalar promotion.
+  Expr *op = e->getSubExpr();
+  if (op->getType()->isAnyComplexType()) {
+    // If it's an l-value, load through the appropriate subobject l-value.
+    // Note that we have to ask `e` because `op` might be an l-value that
+    // this won't work for, e.g. an Obj-C property.
+    if (e->isGLValue()) {
+      mlir::Location loc = cgf.getLoc(e->getExprLoc());
+      mlir::Value complex = cgf.emitComplexExpr(op);
+      return cgf.builder.createComplexReal(loc, complex);
+    }
+
+    // Otherwise, calculate and project.
+    cgf.cgm.errorNYI(e->getSourceRange(),
+                     "VisitUnaryReal calculate and project");
+  }
+
+  return Visit(op);
+}
+
+mlir::Value ScalarExprEmitter::VisitUnaryImag(const UnaryOperator *e) {
+  // TODO(cir): handle scalar promotion.
+  Expr *op = e->getSubExpr();
+  if (op->getType()->isAnyComplexType()) {
+    // If it's an l-value, load through the appropriate subobject l-value.
+    // Note that we have to ask `e` because `op` might be an l-value that
+    // this won't work for, e.g. an Obj-C property.
+    if (e->isGLValue()) {
+      mlir::Location loc = cgf.getLoc(e->getExprLoc());
+      mlir::Value complex = cgf.emitComplexExpr(op);
+      return cgf.builder.createComplexImag(loc, complex);
+    }
+
+    // Otherwise, calculate and project.
+    cgf.cgm.errorNYI(e->getSourceRange(),
+                     "VisitUnaryImag calculate and project");
+  }
+
+  return Visit(op);
+}
+
 /// Return the size or alignment of the type of argument of the sizeof
 /// expression as an integer.
 mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
@@ -1904,28 +1966,21 @@ mlir::Value ScalarExprEmitter::VisitUnaryExprOrTypeTraitExpr(
                                      "sizeof operator for VariableArrayType",
                                      e->getStmtClassName());
       return builder.getConstant(
-          loc, builder.getAttr<cir::IntAttr>(
-                   cgf.cgm.UInt64Ty, llvm::APSInt(llvm::APInt(64, 1), true)));
+          loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                                 llvm::APSInt(llvm::APInt(64, 1), true)));
     }
   } else if (e->getKind() == UETT_OpenMPRequiredSimdAlign) {
     cgf.getCIRGenModule().errorNYI(
         e->getSourceRange(), "sizeof operator for OpenMpRequiredSimdAlign",
         e->getStmtClassName());
     return builder.getConstant(
-        loc, builder.getAttr<cir::IntAttr>(
-                 cgf.cgm.UInt64Ty, llvm::APSInt(llvm::APInt(64, 1), true)));
-  } else if (e->getKind() == UETT_VectorElements) {
-    cgf.getCIRGenModule().errorNYI(e->getSourceRange(),
-                                   "sizeof operator for VectorElements",
-                                   e->getStmtClassName());
-    return builder.getConstant(
-        loc, builder.getAttr<cir::IntAttr>(
-                 cgf.cgm.UInt64Ty, llvm::APSInt(llvm::APInt(64, 1), true)));
+        loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                               llvm::APSInt(llvm::APInt(64, 1), true)));
   }
 
   return builder.getConstant(
-      loc, builder.getAttr<cir::IntAttr>(
-               cgf.cgm.UInt64Ty, e->EvaluateKnownConstInt(cgf.getContext())));
+      loc, cir::IntAttr::get(cgf.cgm.UInt64Ty,
+                             e->EvaluateKnownConstInt(cgf.getContext())));
 }
 
 /// Return true if the specified expression is cheap enough and side-effect-free
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index fd413fe86383..c4efabd6b12a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -462,21 +462,23 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
 
     startFunction(gd, retTy, fn, funcType, args, loc, bodyRange.getBegin());
 
-    if (isa<CXXDestructorDecl>(funcDecl))
+    if (isa<CXXDestructorDecl>(funcDecl)) {
       getCIRGenModule().errorNYI(bodyRange, "C++ destructor definition");
-    else if (isa<CXXConstructorDecl>(funcDecl))
+    } else if (isa<CXXConstructorDecl>(funcDecl)) {
       emitConstructorBody(args);
-    else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
-             funcDecl->hasAttr<CUDAGlobalAttr>())
+    } else if (getLangOpts().CUDA && !getLangOpts().CUDAIsDevice &&
+               funcDecl->hasAttr<CUDAGlobalAttr>()) {
       getCIRGenModule().errorNYI(bodyRange, "CUDA kernel");
-    else if (isa<CXXMethodDecl>(funcDecl) &&
-             cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker())
+    } else if (isa<CXXMethodDecl>(funcDecl) &&
+               cast<CXXMethodDecl>(funcDecl)->isLambdaStaticInvoker()) {
       getCIRGenModule().errorNYI(bodyRange, "Lambda static invoker");
-    else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
-             (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
-              cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator()))
-      getCIRGenModule().errorNYI(bodyRange, "Default assignment operator");
-    else if (body) {
+    } else if (funcDecl->isDefaulted() && isa<CXXMethodDecl>(funcDecl) &&
+               (cast<CXXMethodDecl>(funcDecl)->isCopyAssignmentOperator() ||
+                cast<CXXMethodDecl>(funcDecl)->isMoveAssignmentOperator())) {
+      // Implicit copy-assignment gets the same special treatment as implicit
+      // copy-constructors.
+      emitImplicitAssignmentOperatorBody(args);
+    } else if (body) {
       if (mlir::failed(emitFunctionBody(body))) {
         fn.erase();
         return nullptr;
@@ -550,6 +552,15 @@ LValue CIRGenFunction::makeNaturalAlignPointeeAddrLValue(mlir::Value val,
   return makeAddrLValue(Address(val, align), ty, baseInfo);
 }
 
+LValue CIRGenFunction::makeNaturalAlignAddrLValue(mlir::Value val,
+                                                  QualType ty) {
+  LValueBaseInfo baseInfo;
+  CharUnits alignment = cgm.getNaturalTypeAlignment(ty, &baseInfo);
+  Address addr(val, convertTypeForMem(ty), alignment);
+  assert(!cir::MissingFeatures::opTBAA());
+  return makeAddrLValue(addr, ty, baseInfo);
+}
+
 clang::QualType CIRGenFunction::buildFunctionArgList(clang::GlobalDecl gd,
                                                      FunctionArgList &args) {
   const auto *fd = cast<FunctionDecl>(gd.getDecl());
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 82aa7ec9cb22..76353bae68e2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -68,6 +68,10 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::Value cxxThisValue = nullptr;
   clang::CharUnits cxxThisAlignment;
 
+  /// The value of 'this' to sue when evaluating CXXDefaultInitExprs within this
+  /// expression.
+  Address cxxDefaultInitExprThis = Address::invalid();
+
   // Holds the Decl for the current outermost non-closure context
   const clang::Decl *curFuncDecl = nullptr;
 
@@ -490,7 +494,26 @@ class CIRGenFunction : public CIRGenTypeCache {
   static bool
   isConstructorDelegationValid(const clang::CXXConstructorDecl *ctor);
 
+  /// A scope within which we are constructing the fields of an object which
+  /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use if
+  /// we need to evaluate the CXXDefaultInitExpr within the evaluation.
+  class FieldConstructionScope {
+  public:
+    FieldConstructionScope(CIRGenFunction &cgf, Address thisAddr)
+        : cgf(cgf), oldCXXDefaultInitExprThis(cgf.cxxDefaultInitExprThis) {
+      cgf.cxxDefaultInitExprThis = thisAddr;
+    }
+    ~FieldConstructionScope() {
+      cgf.cxxDefaultInitExprThis = oldCXXDefaultInitExprThis;
+    }
+
+  private:
+    CIRGenFunction &cgf;
+    Address oldCXXDefaultInitExprThis;
+  };
+
   LValue makeNaturalAlignPointeeAddrLValue(mlir::Value v, clang::QualType t);
+  LValue makeNaturalAlignAddrLValue(mlir::Value val, QualType ty);
 
   /// Construct an address with the natural alignment of T. If a pointer to T
   /// is expected to be signed, the pointer passed to this function must have
@@ -527,6 +550,9 @@ class CIRGenFunction : public CIRGenTypeCache {
     return it->second;
   }
 
+  Address getAddrOfBitFieldStorage(LValue base, const clang::FieldDecl *field,
+                                   mlir::Type fieldType, unsigned index);
+
   /// Load the value for 'this'. This function is only valid while generating
   /// code for an C++ member function.
   /// FIXME(cir): this should return a mlir::Value!
@@ -747,6 +773,15 @@ class CIRGenFunction : public CIRGenTypeCache {
                   const CIRGenCallee &callee, ReturnValueSlot returnValue,
                   const CallArgList &args, cir::CIRCallOpInterface *callOp,
                   mlir::Location loc);
+  RValue emitCall(const CIRGenFunctionInfo &funcInfo,
+                  const CIRGenCallee &callee, ReturnValueSlot returnValue,
+                  const CallArgList &args,
+                  cir::CIRCallOpInterface *callOrTryCall = nullptr) {
+    assert(currSrcLoc && "source location must have been set");
+    return emitCall(funcInfo, callee, returnValue, args, callOrTryCall,
+                    *currSrcLoc);
+  }
+
   RValue emitCall(clang::QualType calleeTy, const CIRGenCallee &callee,
                   const clang::CallExpr *e, ReturnValueSlot returnValue);
   void emitCallArg(CallArgList &args, const clang::Expr *e,
@@ -813,6 +848,8 @@ class CIRGenFunction : public CIRGenTypeCache {
       clang::NestedNameSpecifier *qualifier, bool isArrow,
       const clang::Expr *base);
 
+  mlir::Value emitCXXNewExpr(const CXXNewExpr *e);
+
   RValue emitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *e,
                                        const CXXMethodDecl *md,
                                        ReturnValueSlot returnValue);
@@ -844,6 +881,11 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitFunctionBody(const clang::Stmt *body);
 
+  void emitImplicitAssignmentOperatorBody(FunctionArgList &args);
+
+  void emitInitializerForField(clang::FieldDecl *field, LValue lhs,
+                               clang::Expr *init);
+
   mlir::Value emitPromotedScalarExpr(const Expr *e, QualType promotionType);
 
   /// Emit the computation of the specified expression of scalar type.
@@ -857,9 +899,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   // Build CIR for a statement. useCurrentScope should be true if no
   // new scopes need be created when finding a compound statement.
-  mlir::LogicalResult
-  emitStmt(const clang::Stmt *s, bool useCurrentScope,
-           llvm::ArrayRef<const Attr *> attrs = std::nullopt);
+  mlir::LogicalResult emitStmt(const clang::Stmt *s, bool useCurrentScope,
+                               llvm::ArrayRef<const Attr *> attrs = {});
 
   mlir::LogicalResult emitSimpleStmt(const clang::Stmt *s,
                                      bool useCurrentScope);
@@ -918,6 +959,8 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// ignoring the result.
   void emitIgnoredExpr(const clang::Expr *e);
 
+  RValue emitLoadOfBitfieldLValue(LValue lv, SourceLocation loc);
+
   /// Given an expression that represents a value lvalue, this method emits
   /// the address of the lvalue, then loads the result as an rvalue,
   /// returning the rvalue.
@@ -938,8 +981,16 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// of the expression.
   /// FIXME: document this function better.
   LValue emitLValue(const clang::Expr *e);
+  LValue emitLValueForBitField(LValue base, const FieldDecl *field);
   LValue emitLValueForField(LValue base, const clang::FieldDecl *field);
 
+  /// Like emitLValueForField, excpet that if the Field is a reference, this
+  /// will return the address of the reference and not the address of the value
+  /// stored in the reference.
+  LValue emitLValueForFieldInitialization(LValue base,
+                                          const clang::FieldDecl *field,
+                                          llvm::StringRef fieldName);
+
   LValue emitMemberExpr(const MemberExpr *e);
 
   /// Given an expression with a pointer type, emit the value and compute our
@@ -1153,7 +1204,41 @@ class CIRGenFunction : public CIRGenTypeCache {
   void updateLoopOpParallelism(mlir::acc::LoopOp &op, bool isOrphan,
                                OpenACCDirectiveKind dk);
 
+  // The OpenACC 'cache' construct actually applies to the 'loop' if present. So
+  // keep track of the 'loop' so that we can add the cache vars to it correctly.
+  mlir::acc::LoopOp *activeLoopOp = nullptr;
+
+  struct ActiveOpenACCLoopRAII {
+    CIRGenFunction &cgf;
+    mlir::acc::LoopOp *oldLoopOp;
+
+    ActiveOpenACCLoopRAII(CIRGenFunction &cgf, mlir::acc::LoopOp *newOp)
+        : cgf(cgf), oldLoopOp(cgf.activeLoopOp) {
+      cgf.activeLoopOp = newOp;
+    }
+    ~ActiveOpenACCLoopRAII() { cgf.activeLoopOp = oldLoopOp; }
+  };
+
 public:
+  // Helper type used to store the list of important information for a 'data'
+  // clause variable, or a 'cache' variable reference.
+  struct OpenACCDataOperandInfo {
+    mlir::Location beginLoc;
+    mlir::Value varValue;
+    std::string name;
+    llvm::SmallVector<mlir::Value> bounds;
+  };
+  // Gets the collection of info required to lower and OpenACC clause or cache
+  // construct variable reference.
+  OpenACCDataOperandInfo getOpenACCDataOperandInfo(const Expr *e);
+  // Helper function to emit the integer expressions as required by an OpenACC
+  // clause/construct.
+  mlir::Value emitOpenACCIntExpr(const Expr *intExpr);
+  // Helper function to emit an integer constant as an mlir int type, used for
+  // constants in OpenACC constructs/clauses.
+  mlir::Value createOpenACCConstantInt(mlir::Location loc, unsigned width,
+                                       int64_t value);
+
   mlir::LogicalResult
   emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s);
   mlir::LogicalResult emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index cd9096a0188a..1044dfe50747 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -79,17 +79,102 @@ void CIRGenItaniumCXXABI::emitInstanceFunctionProlog(SourceLocation loc,
   }
 }
 
+// Find out how to cirgen the complete destructor and constructor
+namespace {
+enum class StructorCIRGen { Emit, RAUW, Alias, COMDAT };
+}
+
+static StructorCIRGen getCIRGenToUse(CIRGenModule &cgm,
+                                     const CXXMethodDecl *md) {
+  if (!cgm.getCodeGenOpts().CXXCtorDtorAliases)
+    return StructorCIRGen::Emit;
+
+  // The complete and base structors are not equivalent if there are any virtual
+  // bases, so emit separate functions.
+  if (md->getParent()->getNumVBases()) {
+    // The return value is correct here, but other support for this is NYI.
+    cgm.errorNYI(md->getSourceRange(), "getCIRGenToUse: virtual bases");
+    return StructorCIRGen::Emit;
+  }
+
+  GlobalDecl aliasDecl;
+  if (const auto *dd = dyn_cast<CXXDestructorDecl>(md)) {
+    // The assignment is correct here, but other support for this is NYI.
+    cgm.errorNYI(md->getSourceRange(), "getCIRGenToUse: dtor");
+    aliasDecl = GlobalDecl(dd, Dtor_Complete);
+  } else {
+    const auto *cd = cast<CXXConstructorDecl>(md);
+    aliasDecl = GlobalDecl(cd, Ctor_Complete);
+  }
+
+  cir::GlobalLinkageKind linkage = cgm.getFunctionLinkage(aliasDecl);
+
+  if (cir::isDiscardableIfUnused(linkage))
+    return StructorCIRGen::RAUW;
+
+  // FIXME: Should we allow available_externally aliases?
+  if (!cir::isValidLinkage(linkage))
+    return StructorCIRGen::RAUW;
+
+  if (cir::isWeakForLinker(linkage)) {
+    // Only ELF and wasm support COMDATs with arbitrary names (C5/D5).
+    if (cgm.getTarget().getTriple().isOSBinFormatELF() ||
+        cgm.getTarget().getTriple().isOSBinFormatWasm())
+      return StructorCIRGen::COMDAT;
+    return StructorCIRGen::Emit;
+  }
+
+  return StructorCIRGen::Alias;
+}
+
+static void emitConstructorDestructorAlias(CIRGenModule &cgm,
+                                           GlobalDecl aliasDecl,
+                                           GlobalDecl targetDecl) {
+  cir::GlobalLinkageKind linkage = cgm.getFunctionLinkage(aliasDecl);
+
+  // Does this function alias already exists?
+  StringRef mangledName = cgm.getMangledName(aliasDecl);
+  auto globalValue = dyn_cast_or_null<cir::CIRGlobalValueInterface>(
+      cgm.getGlobalValue(mangledName));
+  if (globalValue && !globalValue.isDeclaration())
+    return;
+
+  auto entry = cast_or_null<cir::FuncOp>(cgm.getGlobalValue(mangledName));
+
+  // Retrieve aliasee info.
+  auto aliasee = cast<cir::FuncOp>(cgm.getAddrOfGlobal(targetDecl));
+
+  // Populate actual alias.
+  cgm.emitAliasForGlobal(mangledName, entry, aliasDecl, aliasee, linkage);
+}
+
 void CIRGenItaniumCXXABI::emitCXXStructor(GlobalDecl gd) {
   auto *md = cast<CXXMethodDecl>(gd.getDecl());
   auto *cd = dyn_cast<CXXConstructorDecl>(md);
 
+  StructorCIRGen cirGenType = getCIRGenToUse(cgm, md);
+
   if (!cd) {
     cgm.errorNYI(md->getSourceRange(), "CXCABI emit destructor");
     return;
   }
 
-  if (cgm.getCodeGenOpts().CXXCtorDtorAliases)
-    cgm.errorNYI(md->getSourceRange(), "Ctor/Dtor aliases");
+  if (gd.getCtorType() == Ctor_Complete) {
+    GlobalDecl baseDecl = gd.getWithCtorType(Ctor_Base);
+
+    if (cirGenType == StructorCIRGen::Alias ||
+        cirGenType == StructorCIRGen::COMDAT) {
+      emitConstructorDestructorAlias(cgm, gd, baseDecl);
+      return;
+    }
+
+    if (cirGenType == StructorCIRGen::RAUW) {
+      StringRef mangledName = cgm.getMangledName(gd);
+      mlir::Operation *aliasee = cgm.getAddrOfGlobal(baseDecl);
+      cgm.addReplacement(mangledName, aliasee);
+      return;
+    }
+  }
 
   auto fn = cgm.codegenCXXStructor(gd);
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index 434dd376208e..c1434ee697f4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -68,6 +68,7 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
 
   // Initialize cached types
   VoidTy = cir::VoidType::get(&getMLIRContext());
+  VoidPtrTy = cir::PointerType::get(VoidTy);
   SInt8Ty = cir::IntType::get(&getMLIRContext(), 8, /*isSigned=*/true);
   SInt16Ty = cir::IntType::get(&getMLIRContext(), 16, /*isSigned=*/true);
   SInt32Ty = cir::IntType::get(&getMLIRContext(), 32, /*isSigned=*/true);
@@ -94,11 +95,21 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
   // TODO(CIR): Should be updated once TypeSizeInfoAttr is upstreamed
   const unsigned sizeTypeSize =
       astContext.getTypeSize(astContext.getSignedSizeType());
+  SizeAlignInBytes = astContext.toCharUnitsFromBits(sizeTypeSize).getQuantity();
+  // In CIRGenTypeCache, UIntPtrTy and SizeType are fields of the same union
+  UIntPtrTy =
+      cir::IntType::get(&getMLIRContext(), sizeTypeSize, /*isSigned=*/false);
   PtrDiffTy =
       cir::IntType::get(&getMLIRContext(), sizeTypeSize, /*isSigned=*/true);
 
   theModule->setAttr(cir::CIRDialect::getTripleAttrName(),
                      builder.getStringAttr(getTriple().str()));
+
+  if (cgo.OptimizationLevel > 0 || cgo.OptimizeSize > 0)
+    theModule->setAttr(cir::CIRDialect::getOptInfoAttrName(),
+                       cir::OptInfoAttr::get(&mlirContext,
+                                             cgo.OptimizationLevel,
+                                             cgo.OptimizeSize));
 }
 
 CIRGenModule::~CIRGenModule() = default;
@@ -388,12 +399,6 @@ void CIRGenModule::emitGlobal(clang::GlobalDecl gd) {
 void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
                                                 mlir::Operation *op) {
   auto const *funcDecl = cast<FunctionDecl>(gd.getDecl());
-  if (funcDecl->getIdentifier() == nullptr) {
-    errorNYI(funcDecl->getSourceRange().getBegin(),
-             "function definition with a non-identifier for a name");
-    return;
-  }
-
   const CIRGenFunctionInfo &fi = getTypes().arrangeGlobalDeclaration(gd);
   cir::FuncType funcType = getTypes().getFunctionType(fi);
   cir::FuncOp funcOp = dyn_cast_if_present<cir::FuncOp>(op);
@@ -402,6 +407,16 @@ void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
                                /*DontDefer=*/true, ForDefinition);
   }
 
+  // Already emitted.
+  if (!funcOp.isDeclaration())
+    return;
+
+  setFunctionLinkage(gd, funcOp);
+  setGVProperties(funcOp, funcDecl);
+  assert(!cir::MissingFeatures::opFuncMaybeHandleStaticInExternC());
+  maybeSetTrivialComdat(*funcDecl, funcOp);
+  assert(!cir::MissingFeatures::setLLVMFunctionFEnvAttributes());
+
   CIRGenFunction cgf(*this, builder);
   curCGF = &cgf;
   {
@@ -409,7 +424,17 @@ void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
     cgf.generateCode(gd, funcOp, funcType);
   }
   curCGF = nullptr;
+
+  setNonAliasAttributes(gd, funcOp);
   assert(!cir::MissingFeatures::opFuncAttributesForDefinition());
+
+  if (funcDecl->getAttr<ConstructorAttr>())
+    errorNYI(funcDecl->getSourceRange(), "constructor attribute");
+  if (funcDecl->getAttr<DestructorAttr>())
+    errorNYI(funcDecl->getSourceRange(), "destructor attribute");
+
+  if (funcDecl->getAttr<AnnotateAttr>())
+    errorNYI(funcDecl->getSourceRange(), "deferredAnnotations");
 }
 
 mlir::Operation *CIRGenModule::getGlobalValue(StringRef name) {
@@ -851,10 +876,12 @@ static bool shouldBeInCOMDAT(CIRGenModule &cgm, const Decl &d) {
 void CIRGenModule::maybeSetTrivialComdat(const Decl &d, mlir::Operation *op) {
   if (!shouldBeInCOMDAT(*this, d))
     return;
-  if (auto globalOp = dyn_cast_or_null<cir::GlobalOp>(op))
+  if (auto globalOp = dyn_cast_or_null<cir::GlobalOp>(op)) {
     globalOp.setComdat(true);
-
-  assert(!cir::MissingFeatures::opFuncSetComdat());
+  } else {
+    auto funcOp = cast<cir::FuncOp>(op);
+    funcOp.setComdat(true);
+  }
 }
 
 void CIRGenModule::updateCompletedType(const TagDecl *td) {
@@ -862,6 +889,65 @@ void CIRGenModule::updateCompletedType(const TagDecl *td) {
   genTypes.updateCompletedType(td);
 }
 
+void CIRGenModule::addReplacement(StringRef name, mlir::Operation *op) {
+  replacements[name] = op;
+}
+
+void CIRGenModule::replacePointerTypeArgs(cir::FuncOp oldF, cir::FuncOp newF) {
+  std::optional<mlir::SymbolTable::UseRange> optionalUseRange =
+      oldF.getSymbolUses(theModule);
+  if (!optionalUseRange)
+    return;
+
+  for (const mlir::SymbolTable::SymbolUse &u : *optionalUseRange) {
+    // CallTryOp only shows up after FlattenCFG.
+    auto call = mlir::dyn_cast<cir::CallOp>(u.getUser());
+    if (!call)
+      continue;
+
+    for (const auto [argOp, fnArgType] :
+         llvm::zip(call.getArgs(), newF.getFunctionType().getInputs())) {
+      if (argOp.getType() == fnArgType)
+        continue;
+
+      // The purpose of this entire function is to insert bitcasts in the case
+      // where these types don't match, but I haven't seen a case where that
+      // happens.
+      errorNYI(call.getLoc(), "replace call with mismatched types");
+    }
+  }
+}
+
+void CIRGenModule::applyReplacements() {
+  for (auto &i : replacements) {
+    StringRef mangledName = i.first();
+    mlir::Operation *replacement = i.second;
+    mlir::Operation *entry = getGlobalValue(mangledName);
+    if (!entry)
+      continue;
+    assert(isa<cir::FuncOp>(entry) && "expected function");
+    auto oldF = cast<cir::FuncOp>(entry);
+    auto newF = dyn_cast<cir::FuncOp>(replacement);
+    if (!newF) {
+      // In classic codegen, this can be a global alias, a bitcast, or a GEP.
+      errorNYI(replacement->getLoc(), "replacement is not a function");
+      continue;
+    }
+
+    // LLVM has opaque pointer but CIR not. So we may have to handle these
+    // different pointer types when performing replacement.
+    replacePointerTypeArgs(oldF, newF);
+
+    // Replace old with new, but keep the old order.
+    if (oldF.replaceAllSymbolUses(newF.getSymNameAttr(), theModule).failed())
+      llvm_unreachable("internal error, cannot RAUW symbol");
+    if (newF) {
+      newF->moveBefore(oldF);
+      oldF->erase();
+    }
+  }
+}
+
 // TODO(CIR): this could be a common method between LLVM codegen.
 static bool isVarDeclStrongDefinition(const ASTContext &astContext,
                                       CIRGenModule &cgm, const VarDecl *vd,
@@ -1024,6 +1110,17 @@ CIRGenModule::getCIRLinkageVarDefinition(const VarDecl *vd, bool isConstant) {
   return getCIRLinkageForDeclarator(vd, linkage, isConstant);
 }
 
+cir::GlobalLinkageKind CIRGenModule::getFunctionLinkage(GlobalDecl gd) {
+  const auto *fd = cast<FunctionDecl>(gd.getDecl());
+
+  GVALinkage linkage = astContext.GetGVALinkageForFunction(fd);
+
+  if (isa<CXXDestructorDecl>(fd))
+    errorNYI(fd->getSourceRange(), "getFunctionLinkage: CXXDestructorDecl");
+
+  return getCIRLinkageForDeclarator(fd, linkage, /*IsConstantVariable=*/false);
+}
+
 static cir::GlobalOp
 generateStringLiteral(mlir::Location loc, mlir::TypedAttr c,
                       cir::GlobalLinkageKind lt, CIRGenModule &cgm,
@@ -1530,6 +1627,27 @@ void CIRGenModule::setGVPropertiesAux(mlir::Operation *op,
   assert(!cir::MissingFeatures::opGlobalPartition());
 }
 
+void CIRGenModule::setFunctionAttributes(GlobalDecl globalDecl,
+                                         cir::FuncOp func,
+                                         bool isIncompleteFunction,
+                                         bool isThunk) {
+  // NOTE(cir): Original CodeGen checks if this is an intrinsic. In CIR we
+  // represent them in dedicated ops. The correct attributes are ensured during
+  // translation to LLVM. Thus, we don't need to check for them here.
+
+  assert(!cir::MissingFeatures::setFunctionAttributes());
+  assert(!cir::MissingFeatures::setTargetAttributes());
+
+  // TODO(cir): This needs a lot of work to better match CodeGen. That
+  // ultimately ends up in setGlobalVisibility, which already has the linkage of
+  // the LLVM GV (corresponding to our FuncOp) computed, so it doesn't have to
+  // recompute it here. This is a minimal fix for now.
+  if (!isLocalLinkage(getFunctionLinkage(globalDecl))) {
+    const Decl *decl = globalDecl.getDecl();
+    func.setGlobalVisibilityAttr(getGlobalVisibilityAttrFromDecl(decl));
+  }
+}
+
 cir::FuncOp CIRGenModule::getOrCreateCIRFunction(
     StringRef mangledName, mlir::Type funcType, GlobalDecl gd, bool forVTable,
     bool dontDefer, bool isThunk, ForDefinition_t isForDefinition,
@@ -1572,8 +1690,9 @@ cir::FuncOp CIRGenModule::getOrCreateCIRFunction(
     // If there are two attempts to define the same mangled name, issue an
     // error.
     auto fn = cast<cir::FuncOp>(entry);
-    assert((!isForDefinition || !fn || !fn.isDeclaration()) &&
-           "Duplicate function definition");
+    if (isForDefinition && fn && !fn.isDeclaration()) {
+      errorNYI(d->getSourceRange(), "Duplicate function definition");
+    }
     if (fn && fn.getFunctionType() == funcType) {
       return fn;
     }
@@ -1594,6 +1713,9 @@ cir::FuncOp CIRGenModule::getOrCreateCIRFunction(
       invalidLoc ? theModule->getLoc() : getLoc(funcDecl->getSourceRange()),
       mangledName, mlir::cast<cir::FuncType>(funcType), funcDecl);
 
+  if (d)
+    setFunctionAttributes(gd, funcOp, /*isIncompleteFunction=*/false, isThunk);
+
   // 'dontDefer' actually means don't move this to the deferredDeclsToEmit list.
   if (dontDefer) {
     // TODO(cir): This assertion will need an additional condition when we
@@ -1664,6 +1786,20 @@ CIRGenModule::createCIRFunction(mlir::Location loc, StringRef name,
 
     func = builder.create<cir::FuncOp>(loc, name, funcType);
 
+    assert(!cir::MissingFeatures::opFuncAstDeclAttr());
+    assert(!cir::MissingFeatures::opFuncNoProto());
+
+    assert(func.isDeclaration() && "expected empty body");
+
+    // A declaration gets private visibility by default, but external linkage
+    // as the default linkage.
+    func.setLinkageAttr(cir::GlobalLinkageKindAttr::get(
+        &getMLIRContext(), cir::GlobalLinkageKind::ExternalLinkage));
+    mlir::SymbolTable::setSymbolVisibility(
+        func, mlir::SymbolTable::Visibility::Private);
+
+    assert(!cir::MissingFeatures::opFuncExtraAttrs());
+
     if (!cgf)
       theModule.push_back(func);
   }
@@ -1721,11 +1857,52 @@ CIRGenModule::getGlobalVisibilityAttrFromDecl(const Decl *decl) {
 
 void CIRGenModule::release() {
   emitDeferred();
+  applyReplacements();
 
   // There's a lot of code that is not implemented yet.
   assert(!cir::MissingFeatures::cgmRelease());
 }
 
+void CIRGenModule::emitAliasForGlobal(StringRef mangledName,
+                                      mlir::Operation *op, GlobalDecl aliasGD,
+                                      cir::FuncOp aliasee,
+                                      cir::GlobalLinkageKind linkage) {
+
+  auto *aliasFD = dyn_cast<FunctionDecl>(aliasGD.getDecl());
+  assert(aliasFD && "expected FunctionDecl");
+
+  // The aliasee function type is different from the alias one, this difference
+  // is specific to CIR because in LLVM the ptr types are already erased at this
+  // point.
+  const CIRGenFunctionInfo &fnInfo =
+      getTypes().arrangeCXXStructorDeclaration(aliasGD);
+  cir::FuncType fnType = getTypes().getFunctionType(fnInfo);
+
+  cir::FuncOp alias =
+      createCIRFunction(getLoc(aliasGD.getDecl()->getSourceRange()),
+                        mangledName, fnType, aliasFD);
+  alias.setAliasee(aliasee.getName());
+  alias.setLinkage(linkage);
+  // Declarations cannot have public MLIR visibility, just mark them private
+  // but this really should have no meaning since CIR should not be using
+  // this information to derive linkage information.
+  mlir::SymbolTable::setSymbolVisibility(
+      alias, mlir::SymbolTable::Visibility::Private);
+
+  // Alias constructors and destructors are always unnamed_addr.
+  assert(!cir::MissingFeatures::opGlobalUnnamedAddr());
+
+  // Switch any previous uses to the alias.
+  if (op) {
+    errorNYI(aliasFD->getSourceRange(), "emitAliasForGlobal: previous uses");
+  } else {
+    // Name already set by createCIRFunction
+  }
+
+  // Finally, set up the alias with its proper name and attributes.
+  setCommonAttributes(aliasGD, alias);
+}
+
 mlir::Type CIRGenModule::convertType(QualType type) {
   return genTypes.convertType(type);
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 71806e3c5de2..16922b115027 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -166,7 +166,7 @@ class CIRGenModule : public CIRGenTypeCache {
   /// constructed for. If valid, the attributes applied to this decl may
   /// contribute to the function attributes and calling convention.
   void constructAttributeList(CIRGenCalleeInfo calleeInfo,
-                              cir::SideEffect &sideEffect);
+                              mlir::NamedAttrList &attrs);
 
   /// Return a constant array for the given string.
   mlir::Attribute getConstantArrayFromStringLiteral(const StringLiteral *e);
@@ -256,6 +256,10 @@ class CIRGenModule : public CIRGenTypeCache {
   /// declarations are emitted lazily.
   void emitGlobal(clang::GlobalDecl gd);
 
+  void emitAliasForGlobal(llvm::StringRef mangledName, mlir::Operation *op,
+                          GlobalDecl aliasGD, cir::FuncOp aliasee,
+                          cir::GlobalLinkageKind linkage);
+
   mlir::Type convertType(clang::QualType type);
 
   /// Set the visibility for the given global.
@@ -268,6 +272,10 @@ class CIRGenModule : public CIRGenTypeCache {
   void setGVProperties(mlir::Operation *op, const NamedDecl *d) const;
   void setGVPropertiesAux(mlir::Operation *op, const NamedDecl *d) const;
 
+  /// Set function attributes for a function declaration.
+  void setFunctionAttributes(GlobalDecl gd, cir::FuncOp f,
+                             bool isIncompleteFunction, bool isThunk);
+
   void emitGlobalDefinition(clang::GlobalDecl gd,
                             mlir::Operation *op = nullptr);
   void emitGlobalFunctionDefinition(clang::GlobalDecl gd, mlir::Operation *op);
@@ -340,14 +348,22 @@ class CIRGenModule : public CIRGenTypeCache {
       clang::VisibilityAttr::VisibilityType visibility);
   cir::VisibilityAttr getGlobalVisibilityAttrFromDecl(const Decl *decl);
   static mlir::SymbolTable::Visibility getMLIRVisibility(cir::GlobalOp op);
-
+  cir::GlobalLinkageKind getFunctionLinkage(GlobalDecl gd);
   cir::GlobalLinkageKind getCIRLinkageForDeclarator(const DeclaratorDecl *dd,
                                                     GVALinkage linkage,
                                                     bool isConstantVariable);
+  void setFunctionLinkage(GlobalDecl gd, cir::FuncOp f) {
+    cir::GlobalLinkageKind l = getFunctionLinkage(gd);
+    f.setLinkageAttr(cir::GlobalLinkageKindAttr::get(&getMLIRContext(), l));
+    mlir::SymbolTable::setSymbolVisibility(f,
+                                           getMLIRVisibilityFromCIRLinkage(l));
+  }
 
   cir::GlobalLinkageKind getCIRLinkageVarDefinition(const VarDecl *vd,
                                                     bool isConstant);
 
+  void addReplacement(llvm::StringRef name, mlir::Operation *op);
+
   /// Helpers to emit "not yet implemented" error diagnostics
   DiagnosticBuilder errorNYI(SourceLocation, llvm::StringRef);
 
@@ -387,6 +403,17 @@ class CIRGenModule : public CIRGenTypeCache {
   llvm::MapVector<clang::GlobalDecl, llvm::StringRef> mangledDeclNames;
   llvm::StringMap<clang::GlobalDecl, llvm::BumpPtrAllocator> manglings;
 
+  // FIXME: should we use llvm::TrackingVH<mlir::Operation> here?
+  typedef llvm::StringMap<mlir::Operation *> ReplacementsTy;
+  ReplacementsTy replacements;
+  /// Call replaceAllUsesWith on all pairs in replacements.
+  void applyReplacements();
+
+  /// A helper function to replace all uses of OldF to NewF that replace
+  /// the type of pointer arguments. This is not needed to tradtional
+  /// pipeline since LLVM has opaque pointers but CIR not.
+  void replacePointerTypeArgs(cir::FuncOp oldF, cir::FuncOp newF);
+
   void setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op);
 };
 } // namespace CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp
new file mode 100644
index 000000000000..49ff1249827a
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenACC.cpp
@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic OpenACC lowering functions not Stmt, Decl, or clause specific.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/OpenACC/OpenACC.h"
+#include "clang/AST/ExprCXX.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+namespace {
+mlir::Value createBound(CIRGenFunction &cgf, CIRGen::CIRGenBuilderTy &builder,
+                        mlir::Location boundLoc, mlir::Value lowerBound,
+                        mlir::Value upperBound, mlir::Value extent) {
+  // Arrays always have a start-idx of 0.
+  mlir::Value startIdx = cgf.createOpenACCConstantInt(boundLoc, 64, 0);
+  // Stride is always 1 in C/C++.
+  mlir::Value stride = cgf.createOpenACCConstantInt(boundLoc, 64, 1);
+
+  auto bound =
+      builder.create<mlir::acc::DataBoundsOp>(boundLoc, lowerBound, upperBound);
+  bound.getStartIdxMutable().assign(startIdx);
+  if (extent)
+    bound.getExtentMutable().assign(extent);
+  bound.getStrideMutable().assign(stride);
+
+  return bound;
+}
+} // namespace
+
+mlir::Value CIRGenFunction::emitOpenACCIntExpr(const Expr *intExpr) {
+  mlir::Value expr = emitScalarExpr(intExpr);
+  mlir::Location exprLoc = cgm.getLoc(intExpr->getBeginLoc());
+
+  mlir::IntegerType targetType = mlir::IntegerType::get(
+      &getMLIRContext(), getContext().getIntWidth(intExpr->getType()),
+      intExpr->getType()->isSignedIntegerOrEnumerationType()
+          ? mlir::IntegerType::SignednessSemantics::Signed
+          : mlir::IntegerType::SignednessSemantics::Unsigned);
+
+  auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
+      exprLoc, targetType, expr);
+  return conversionOp.getResult(0);
+}
+
+mlir::Value CIRGenFunction::createOpenACCConstantInt(mlir::Location loc,
+                                                     unsigned width,
+                                                     int64_t value) {
+  mlir::IntegerType ty =
+      mlir::IntegerType::get(&getMLIRContext(), width,
+                             mlir::IntegerType::SignednessSemantics::Signless);
+  auto constOp = builder.create<mlir::arith::ConstantOp>(
+      loc, builder.getIntegerAttr(ty, value));
+
+  return constOp.getResult();
+}
+
+CIRGenFunction::OpenACCDataOperandInfo
+CIRGenFunction::getOpenACCDataOperandInfo(const Expr *e) {
+  const Expr *curVarExpr = e->IgnoreParenImpCasts();
+
+  mlir::Location exprLoc = cgm.getLoc(curVarExpr->getBeginLoc());
+  llvm::SmallVector<mlir::Value> bounds;
+
+  std::string exprString;
+  llvm::raw_string_ostream os(exprString);
+  e->printPretty(os, nullptr, getContext().getPrintingPolicy());
+
+  while (isa<ArraySectionExpr, ArraySubscriptExpr>(curVarExpr)) {
+    mlir::Location boundLoc = cgm.getLoc(curVarExpr->getBeginLoc());
+    mlir::Value lowerBound;
+    mlir::Value upperBound;
+    mlir::Value extent;
+
+    if (const auto *section = dyn_cast<ArraySectionExpr>(curVarExpr)) {
+      if (const Expr *lb = section->getLowerBound())
+        lowerBound = emitOpenACCIntExpr(lb);
+      else
+        lowerBound = createOpenACCConstantInt(boundLoc, 64, 0);
+
+      if (const Expr *len = section->getLength()) {
+        extent = emitOpenACCIntExpr(len);
+      } else {
+        QualType baseTy = ArraySectionExpr::getBaseOriginalType(
+            section->getBase()->IgnoreParenImpCasts());
+        // We know this is the case as implicit lengths are only allowed for
+        // array types with a constant size, or a dependent size.  AND since
+        // we are codegen we know we're not dependent.
+        auto *arrayTy = getContext().getAsConstantArrayType(baseTy);
+        // Rather than trying to calculate the extent based on the
+        // lower-bound, we can just emit this as an upper bound.
+        upperBound = createOpenACCConstantInt(boundLoc, 64,
+                                              arrayTy->getLimitedSize() - 1);
+      }
+
+      curVarExpr = section->getBase()->IgnoreParenImpCasts();
+    } else {
+      const auto *subscript = cast<ArraySubscriptExpr>(curVarExpr);
+
+      lowerBound = emitOpenACCIntExpr(subscript->getIdx());
+      // Length of an array index is always 1.
+      extent = createOpenACCConstantInt(boundLoc, 64, 1);
+      curVarExpr = subscript->getBase()->IgnoreParenImpCasts();
+    }
+
+    bounds.push_back(createBound(*this, this->builder, boundLoc, lowerBound,
+                                 upperBound, extent));
+  }
+
+  if (const auto *memExpr = dyn_cast<MemberExpr>(curVarExpr))
+    return {exprLoc, emitMemberExpr(memExpr).getPointer(), exprString,
+            std::move(bounds)};
+
+  // Sema has made sure that only 4 types of things can get here, array
+  // subscript, array section, member expr, or DRE to a var decl (or the
+  // former 3 wrapping a var-decl), so we should be able to assume this is
+  // right.
+  const auto *dre = cast<DeclRefExpr>(curVarExpr);
+  return {exprLoc, emitDeclRefLValue(dre).getPointer(), exprString,
+          std::move(bounds)};
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
index 899e91574e91..e45d3b8f4aa8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
@@ -75,26 +75,12 @@ class OpenACCClauseCIREmitter final
   void setLastDeviceTypeClause(const OpenACCDeviceTypeClause &clause) {
     lastDeviceTypeValues.clear();
 
-    llvm::for_each(clause.getArchitectures(),
-                   [this](const DeviceTypeArgument &arg) {
-                     lastDeviceTypeValues.push_back(
-                         decodeDeviceType(arg.getIdentifierInfo()));
-                   });
+    for (const DeviceTypeArgument &arg : clause.getArchitectures())
+      lastDeviceTypeValues.push_back(decodeDeviceType(arg.getIdentifierInfo()));
   }
 
   mlir::Value emitIntExpr(const Expr *intExpr) {
-    mlir::Value expr = cgf.emitScalarExpr(intExpr);
-    mlir::Location exprLoc = cgf.cgm.getLoc(intExpr->getBeginLoc());
-
-    mlir::IntegerType targetType = mlir::IntegerType::get(
-        &cgf.getMLIRContext(), cgf.getContext().getIntWidth(intExpr->getType()),
-        intExpr->getType()->isSignedIntegerOrEnumerationType()
-            ? mlir::IntegerType::SignednessSemantics::Signed
-            : mlir::IntegerType::SignednessSemantics::Unsigned);
-
-    auto conversionOp = builder.create<mlir::UnrealizedConversionCastOp>(
-        exprLoc, targetType, expr);
-    return conversionOp.getResult(0);
+    return cgf.emitOpenACCIntExpr(intExpr);
   }
 
   // 'condition' as an OpenACC grammar production is used for 'if' and (some
@@ -114,6 +100,7 @@ class OpenACCClauseCIREmitter final
 
   mlir::Value createConstantInt(mlir::Location loc, unsigned width,
                                 int64_t value) {
+    return cgf.createOpenACCConstantInt(loc, width, value);
     mlir::IntegerType ty = mlir::IntegerType::get(
         &cgf.getMLIRContext(), width,
         mlir::IntegerType::SignednessSemantics::Signless);
@@ -187,114 +174,39 @@ class OpenACCClauseCIREmitter final
     dataOperands.append(computeEmitter.dataOperands);
   }
 
-  struct DataOperandInfo {
-    mlir::Location beginLoc;
-    mlir::Value varValue;
-    std::string name;
-    llvm::SmallVector<mlir::Value> bounds;
-  };
-
-  mlir::Value createBound(mlir::Location boundLoc, mlir::Value lowerBound,
-                          mlir::Value upperBound, mlir::Value extent) {
-    // Arrays always have a start-idx of 0.
-    mlir::Value startIdx = createConstantInt(boundLoc, 64, 0);
-    // Stride is always 1 in C/C++.
-    mlir::Value stride = createConstantInt(boundLoc, 64, 1);
-
-    auto bound = builder.create<mlir::acc::DataBoundsOp>(boundLoc, lowerBound,
-                                                         upperBound);
-    bound.getStartIdxMutable().assign(startIdx);
-    if (extent)
-      bound.getExtentMutable().assign(extent);
-    bound.getStrideMutable().assign(stride);
-
-    return bound;
-  }
-
-  // A helper function that gets the information from an operand to a data
-  // clause, so that it can be used to emit the data operations.
-  DataOperandInfo getDataOperandInfo(OpenACCDirectiveKind dk, const Expr *e) {
-    // TODO: OpenACC: Cache was different enough as to need a separate
-    // `ActOnCacheVar`, so we are going to need to do some investigations here
-    // when it comes to implement this for cache.
-    if (dk == OpenACCDirectiveKind::Cache) {
-      cgf.cgm.errorNYI(e->getSourceRange(),
-                       "OpenACC data operand for 'cache' directive");
-      return {cgf.cgm.getLoc(e->getBeginLoc()), {}, {}, {}};
-    }
-
-    const Expr *curVarExpr = e->IgnoreParenImpCasts();
-
-    mlir::Location exprLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
-    llvm::SmallVector<mlir::Value> bounds;
-
-    std::string exprString;
-    llvm::raw_string_ostream os(exprString);
-    e->printPretty(os, nullptr, cgf.getContext().getPrintingPolicy());
-
-    // Assemble the list of bounds.
-    while (isa<ArraySectionExpr, ArraySubscriptExpr>(curVarExpr)) {
-      mlir::Location boundLoc = cgf.cgm.getLoc(curVarExpr->getBeginLoc());
-      mlir::Value lowerBound;
-      mlir::Value upperBound;
-      mlir::Value extent;
-
-      if (const auto *section = dyn_cast<ArraySectionExpr>(curVarExpr)) {
-        if (const Expr *lb = section->getLowerBound())
-          lowerBound = emitIntExpr(lb);
-        else
-          lowerBound = createConstantInt(boundLoc, 64, 0);
-
-        if (const Expr *len = section->getLength()) {
-          extent = emitIntExpr(len);
-        } else {
-          QualType baseTy = ArraySectionExpr::getBaseOriginalType(
-              section->getBase()->IgnoreParenImpCasts());
-          // We know this is the case as implicit lengths are only allowed for
-          // array types with a constant size, or a dependent size.  AND since
-          // we are codegen we know we're not dependent.
-          auto *arrayTy = cgf.getContext().getAsConstantArrayType(baseTy);
-          // Rather than trying to calculate the extent based on the
-          // lower-bound, we can just emit this as an upper bound.
-          upperBound =
-              createConstantInt(boundLoc, 64, arrayTy->getLimitedSize() - 1);
-        }
-
-        curVarExpr = section->getBase()->IgnoreParenImpCasts();
-      } else {
-        const auto *subscript = cast<ArraySubscriptExpr>(curVarExpr);
-
-        lowerBound = emitIntExpr(subscript->getIdx());
-        // Length of an array index is always 1.
-        extent = createConstantInt(boundLoc, 64, 1);
-        curVarExpr = subscript->getBase()->IgnoreParenImpCasts();
-      }
-
-      bounds.push_back(createBound(boundLoc, lowerBound, upperBound, extent));
+  mlir::acc::DataClauseModifier
+  convertModifiers(OpenACCModifierKind modifiers) {
+    using namespace mlir::acc;
+    static_assert(static_cast<int>(OpenACCModifierKind::Zero) ==
+                      static_cast<int>(DataClauseModifier::zero) &&
+                  static_cast<int>(OpenACCModifierKind::Readonly) ==
+                      static_cast<int>(DataClauseModifier::readonly) &&
+                  static_cast<int>(OpenACCModifierKind::AlwaysIn) ==
+                      static_cast<int>(DataClauseModifier::alwaysin) &&
+                  static_cast<int>(OpenACCModifierKind::AlwaysOut) ==
+                      static_cast<int>(DataClauseModifier::alwaysout) &&
+                  static_cast<int>(OpenACCModifierKind::Capture) ==
+                      static_cast<int>(DataClauseModifier::capture));
+
+    DataClauseModifier mlirModifiers{};
+
+    // The MLIR representation of this represents `always` as `alwaysin` +
+    // `alwaysout`.  So do a small fixup here.
+    if (isOpenACCModifierBitSet(modifiers, OpenACCModifierKind::Always)) {
+      mlirModifiers = mlirModifiers | DataClauseModifier::always;
+      modifiers &= ~OpenACCModifierKind::Always;
     }
 
-    if (const auto *memExpr = dyn_cast<MemberExpr>(curVarExpr))
-      return {exprLoc, cgf.emitMemberExpr(memExpr).getPointer(), exprString,
-              std::move(bounds)};
-
-    // Sema has made sure that only 4 types of things can get here, array
-    // subscript, array section, member expr, or DRE to a var decl (or the
-    // former 3 wrapping a var-decl), so we should be able to assume this is
-    // right.
-    const auto *dre = cast<DeclRefExpr>(curVarExpr);
-    return {exprLoc, cgf.emitDeclRefLValue(dre).getPointer(), exprString,
-            std::move(bounds)};
+    mlirModifiers = mlirModifiers | static_cast<DataClauseModifier>(modifiers);
+    return mlirModifiers;
   }
 
   template <typename BeforeOpTy, typename AfterOpTy>
   void addDataOperand(const Expr *varOperand, mlir::acc::DataClause dataClause,
-                      bool structured, bool implicit) {
-    DataOperandInfo opInfo = getDataOperandInfo(dirKind, varOperand);
-
-    // TODO: OpenACC: we should comprehend the 'modifier-list' here for the data
-    // operand. At the moment, we don't have a uniform way to assign these
-    // properly, and the dialect cannot represent anything other than 'readonly'
-    // and 'zero' on copyin/copyout/create, so for now, we skip it.
+                      OpenACCModifierKind modifiers, bool structured,
+                      bool implicit) {
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        cgf.getOpenACCDataOperandInfo(varOperand);
 
     auto beforeOp =
         builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured,
@@ -323,6 +235,8 @@ class OpenACCClauseCIREmitter final
     // Set the 'rest' of the info for both operations.
     beforeOp.setDataClause(dataClause);
     afterOp.setDataClause(dataClause);
+    beforeOp.setModifiers(convertModifiers(modifiers));
+    afterOp.setModifiers(convertModifiers(modifiers));
 
     // Make sure we record these, so 'async' values can be updated later.
     dataOperands.push_back(beforeOp.getOperation());
@@ -331,8 +245,10 @@ class OpenACCClauseCIREmitter final
 
   template <typename BeforeOpTy>
   void addDataOperand(const Expr *varOperand, mlir::acc::DataClause dataClause,
-                      bool structured, bool implicit) {
-    DataOperandInfo opInfo = getDataOperandInfo(dirKind, varOperand);
+                      OpenACCModifierKind modifiers, bool structured,
+                      bool implicit) {
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        cgf.getOpenACCDataOperandInfo(varOperand);
     auto beforeOp =
         builder.create<BeforeOpTy>(opInfo.beginLoc, opInfo.varValue, structured,
                                    implicit, opInfo.name, opInfo.bounds);
@@ -340,6 +256,8 @@ class OpenACCClauseCIREmitter final
 
     // Set the 'rest' of the info for the operation.
     beforeOp.setDataClause(dataClause);
+    beforeOp.setModifiers(convertModifiers(modifiers));
+
     // Make sure we record these, so 'async' values can be updated later.
     dataOperands.push_back(beforeOp.getOperation());
   }
@@ -348,10 +266,21 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::ArrayAttr getAsyncOnlyAttr() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>) {
       return operation.getAsyncOnlyAttr();
-    else if constexpr (isCombinedType<OpTy>)
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                      mlir::acc::ExitDataOp>) {
+      if (!operation.getAsyncAttr())
+        return mlir::ArrayAttr{};
+
+      llvm::SmallVector<mlir::Attribute> devTysTemp;
+      devTysTemp.push_back(mlir::acc::DeviceTypeAttr::get(
+          builder.getContext(), mlir::acc::DeviceType::None));
+      return mlir::ArrayAttr::get(builder.getContext(), devTysTemp);
+    } else if constexpr (isCombinedType<OpTy>) {
       return operation.computeOp.getAsyncOnlyAttr();
+    }
 
     // Note: 'wait' has async as well, but it cannot have data clauses, so we
     // don't have to handle them here.
@@ -363,10 +292,21 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::ArrayAttr getAsyncOperandsDeviceTypeAttr() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>) {
       return operation.getAsyncOperandsDeviceTypeAttr();
-    else if constexpr (isCombinedType<OpTy>)
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                      mlir::acc::ExitDataOp>) {
+      if (!operation.getAsyncOperand())
+        return mlir::ArrayAttr{};
+
+      llvm::SmallVector<mlir::Attribute> devTysTemp;
+      devTysTemp.push_back(mlir::acc::DeviceTypeAttr::get(
+          builder.getContext(), mlir::acc::DeviceType::None));
+      return mlir::ArrayAttr::get(builder.getContext(), devTysTemp);
+    } else if constexpr (isCombinedType<OpTy>) {
       return operation.computeOp.getAsyncOperandsDeviceTypeAttr();
+    }
 
     // Note: 'wait' has async as well, but it cannot have data clauses, so we
     // don't have to handle them here.
@@ -379,8 +319,12 @@ class OpenACCClauseCIREmitter final
   // on all operation types.
   mlir::OperandRange getAsyncOperands() {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>)
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::UpdateOp>)
       return operation.getAsyncOperands();
+    else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp,
+                                    mlir::acc::ExitDataOp>)
+      return operation.getAsyncOperandMutable();
     else if constexpr (isCombinedType<OpTy>)
       return operation.computeOp.getAsyncOperands();
 
@@ -457,11 +401,9 @@ class OpenACCClauseCIREmitter final
 
     if constexpr (isOneOfTypes<OpTy, mlir::acc::InitOp,
                                mlir::acc::ShutdownOp>) {
-      llvm::for_each(
-          clause.getArchitectures(), [this](const DeviceTypeArgument &arg) {
-            operation.addDeviceType(builder.getContext(),
-                                    decodeDeviceType(arg.getIdentifierInfo()));
-          });
+      for (const DeviceTypeArgument &arg : clause.getArchitectures())
+        operation.addDeviceType(builder.getContext(),
+                                decodeDeviceType(arg.getIdentifierInfo()));
     } else if constexpr (isOneOfTypes<OpTy, mlir::acc::SetOp>) {
       assert(!operation.getDeviceTypeAttr() && "already have device-type?");
       assert(clause.getArchitectures().size() <= 1);
@@ -471,7 +413,8 @@ class OpenACCClauseCIREmitter final
             decodeDeviceType(clause.getArchitectures()[0].getIdentifierInfo()));
     } else if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp,
                                       mlir::acc::SerialOp, mlir::acc::KernelsOp,
-                                      mlir::acc::DataOp, mlir::acc::LoopOp>) {
+                                      mlir::acc::DataOp, mlir::acc::LoopOp,
+                                      mlir::acc::UpdateOp>) {
       // Nothing to do here, these constructs don't have any IR for these, as
       // they just modify the other clauses IR.  So setting of
       // `lastDeviceTypeValues` (done above) is all we need.
@@ -480,7 +423,7 @@ class OpenACCClauseCIREmitter final
       // 'lastDeviceTypeValues' to set the value for the child visitor.
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. update, data, routine constructs remain.
+      // unreachable. routine construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -514,10 +457,12 @@ class OpenACCClauseCIREmitter final
   void VisitAsyncClause(const OpenACCAsyncClause &clause) {
     hasAsyncClause = true;
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
-      if (!clause.hasIntExpr())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::EnterDataOp, mlir::acc::ExitDataOp,
+                               mlir::acc::UpdateOp>) {
+      if (!clause.hasIntExpr()) {
         operation.addAsyncOnly(builder.getContext(), lastDeviceTypeValues);
-      else {
+      } else {
 
         mlir::Value intExpr;
         {
@@ -544,8 +489,7 @@ class OpenACCClauseCIREmitter final
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Combined constructs remain. Data, enter data, exit data,
-      // update constructs remain.
+      // unreachable. Combined constructs remain. update construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -562,12 +506,39 @@ class OpenACCClauseCIREmitter final
       } else {
         llvm_unreachable("var-list version of self shouldn't get here");
       }
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      assert(!clause.isEmptySelfClause() && !clause.isConditionExprClause() &&
+             "var-list version of self required for update");
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::UpdateHostOp>(
+            var, mlir::acc::DataClause::acc_update_self, {},
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. update construct remains.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitSelfClause");
+    }
+  }
+
+  void VisitHostClause(const OpenACCHostClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::UpdateHostOp>(
+            var, mlir::acc::DataClause::acc_update_host, {},
+            /*structured=*/false, /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitHostClause");
+    }
+  }
+
+  void VisitDeviceClause(const OpenACCDeviceClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::UpdateDeviceOp>(
+            var, mlir::acc::DataClause::acc_update_device, {},
+            /*structured=*/false, /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDeviceClause");
     }
   }
 
@@ -576,28 +547,21 @@ class OpenACCClauseCIREmitter final
                                mlir::acc::KernelsOp, mlir::acc::InitOp,
                                mlir::acc::ShutdownOp, mlir::acc::SetOp,
                                mlir::acc::DataOp, mlir::acc::WaitOp,
-                               mlir::acc::HostDataOp>) {
+                               mlir::acc::HostDataOp, mlir::acc::EnterDataOp,
+                               mlir::acc::ExitDataOp, mlir::acc::UpdateOp>) {
       operation.getIfCondMutable().append(
           createCondition(clause.getConditionExpr()));
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // 'if' applies to most of the constructs, but hold off on lowering them
-      // until we can write tests/know what we're doing with codegen to make
-      // sure we get it right.
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Enter data, exit data, host_data, update constructs
-      // remain.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitIfClause");
     }
   }
 
   void VisitIfPresentClause(const OpenACCIfPresentClause &clause) {
-    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp,
+                               mlir::acc::UpdateOp>) {
       operation.setIfPresent(true);
-    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::UpdateOp>) {
-      // Last unimplemented one here, so just put it in this way instead.
-      return clauseNotImplemented(clause);
     } else {
       llvm_unreachable("unknown construct kind in VisitIfPresentClause");
     }
@@ -631,7 +595,9 @@ class OpenACCClauseCIREmitter final
 
   void VisitWaitClause(const OpenACCWaitClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+                               mlir::acc::KernelsOp, mlir::acc::DataOp,
+                               mlir::acc::EnterDataOp, mlir::acc::ExitDataOp,
+                               mlir::acc::UpdateOp>) {
       if (!clause.hasExprs()) {
         operation.addWaitOnly(builder.getContext(), lastDeviceTypeValues);
       } else {
@@ -647,7 +613,7 @@ class OpenACCClauseCIREmitter final
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. Enter data, exit data, update constructs remain.
+      // unreachable. update construct remains.
       return clauseNotImplemented(clause);
     }
   }
@@ -815,26 +781,126 @@ class OpenACCClauseCIREmitter final
 
   void VisitCopyClause(const OpenACCCopyClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp>) {
-      for (auto var : clause.getVarList())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::CopyinOp, mlir::acc::CopyoutOp>(
-            var, mlir::acc::DataClause::acc_copy, /*structured=*/true,
+            var, mlir::acc::DataClause::acc_copy, clause.getModifierList(),
+            /*structured=*/true,
             /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. data, declare, combined constructs remain.
+      // unreachable. declare construct remains.
       return clauseNotImplemented(clause);
     }
   }
 
+  void VisitCopyInClause(const OpenACCCopyInClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CopyinOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_copyin, clause.getModifierList(),
+            /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CopyinOp>(
+            var, mlir::acc::DataClause::acc_copyin, clause.getModifierList(),
+            /*structured=*/false, /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. declare construct remains.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitCopyOutClause(const OpenACCCopyOutClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CreateOp, mlir::acc::CopyoutOp>(
+            var, mlir::acc::DataClause::acc_copyout, clause.getModifierList(),
+            /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::CopyoutOp>(
+            var, mlir::acc::DataClause::acc_copyout, clause.getModifierList(),
+            /*structured=*/false,
+            /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. declare construct remains.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitCreateClause(const OpenACCCreateClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CreateOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_create, clause.getModifierList(),
+            /*structured=*/true,
+            /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::CreateOp>(
+            var, mlir::acc::DataClause::acc_create, clause.getModifierList(),
+            /*structured=*/false, /*implicit=*/false);
+    } else if constexpr (isCombinedType<OpTy>) {
+      applyToComputeOp(clause);
+    } else {
+      // TODO: When we've implemented this for everything, switch this to an
+      // unreachable. declare construct remains.
+      return clauseNotImplemented(clause);
+    }
+  }
+
+  void VisitDeleteClause(const OpenACCDeleteClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::DeleteOp>(
+            var, mlir::acc::DataClause::acc_delete, {},
+            /*structured=*/false,
+            /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDeleteClause");
+    }
+  }
+
+  void VisitDetachClause(const OpenACCDetachClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::GetDevicePtrOp, mlir::acc::DetachOp>(
+            var, mlir::acc::DataClause::acc_detach, {},
+            /*structured=*/false,
+            /*implicit=*/false);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitDetachClause");
+    }
+  }
+
+  void VisitFinalizeClause(const OpenACCFinalizeClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ExitDataOp>) {
+      operation.setFinalize(true);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitFinalizeClause");
+    }
+  }
+
   void VisitUseDeviceClause(const OpenACCUseDeviceClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::HostDataOp>) {
-      for (auto var : clause.getVarList())
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::UseDeviceOp>(
-            var, mlir::acc::DataClause::acc_use_device,
-            /*structured=*/true, /*implicit=*/false);
+            var, mlir::acc::DataClause::acc_use_device, {}, /*structured=*/true,
+            /*implicit=*/false);
     } else {
       llvm_unreachable("Unknown construct kind in VisitUseDeviceClause");
     }
@@ -842,65 +908,67 @@ class OpenACCClauseCIREmitter final
 
   void VisitDevicePtrClause(const OpenACCDevicePtrClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp>) {
-      for (auto var : clause.getVarList())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::DevicePtrOp>(
-            var, mlir::acc::DataClause::acc_deviceptr, /*structured=*/true,
+            var, mlir::acc::DataClause::acc_deviceptr, {},
+            /*structured=*/true,
             /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. data, declare remain.
+      // unreachable. declare remains.
       return clauseNotImplemented(clause);
     }
   }
 
   void VisitNoCreateClause(const OpenACCNoCreateClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp>) {
-      for (auto var : clause.getVarList())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::NoCreateOp, mlir::acc::DeleteOp>(
-            var, mlir::acc::DataClause::acc_no_create, /*structured=*/true,
+            var, mlir::acc::DataClause::acc_no_create, {}, /*structured=*/true,
             /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. data remains.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitNoCreateClause");
     }
   }
 
   void VisitPresentClause(const OpenACCPresentClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp>) {
-      for (auto var : clause.getVarList())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::PresentOp, mlir::acc::DeleteOp>(
-            var, mlir::acc::DataClause::acc_present, /*structured=*/true,
+            var, mlir::acc::DataClause::acc_present, {}, /*structured=*/true,
             /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
       // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. data & declare remain.
+      // unreachable. declare remains.
       return clauseNotImplemented(clause);
     }
   }
 
   void VisitAttachClause(const OpenACCAttachClause &clause) {
     if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
-                               mlir::acc::KernelsOp>) {
-      for (auto var : clause.getVarList())
+                               mlir::acc::KernelsOp, mlir::acc::DataOp>) {
+      for (const Expr *var : clause.getVarList())
         addDataOperand<mlir::acc::AttachOp, mlir::acc::DetachOp>(
-            var, mlir::acc::DataClause::acc_attach, /*structured=*/true,
+            var, mlir::acc::DataClause::acc_attach, {}, /*structured=*/true,
             /*implicit=*/false);
+    } else if constexpr (isOneOfTypes<OpTy, mlir::acc::EnterDataOp>) {
+      for (const Expr *var : clause.getVarList())
+        addDataOperand<mlir::acc::AttachOp>(
+            var, mlir::acc::DataClause::acc_attach, {},
+            /*structured=*/false, /*implicit=*/false);
     } else if constexpr (isCombinedType<OpTy>) {
       applyToComputeOp(clause);
     } else {
-      // TODO: When we've implemented this for everything, switch this to an
-      // unreachable. data, enter data remain.
-      return clauseNotImplemented(clause);
+      llvm_unreachable("Unknown construct kind in VisitAttachClause");
     }
   }
 };
@@ -939,6 +1007,9 @@ EXPL_SPEC(mlir::acc::ShutdownOp)
 EXPL_SPEC(mlir::acc::SetOp)
 EXPL_SPEC(mlir::acc::WaitOp)
 EXPL_SPEC(mlir::acc::HostDataOp)
+EXPL_SPEC(mlir::acc::EnterDataOp)
+EXPL_SPEC(mlir::acc::ExitDataOp)
+EXPL_SPEC(mlir::acc::UpdateOp)
 #undef EXPL_SPEC
 
 template <typename ComputeOp, typename LoopOp>
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index faeee88bba3c..05e8848884f7 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -79,6 +79,7 @@ struct CIRRecordLowering final {
   /// Inserts padding everywhere it's needed.
   void insertPadding();
 
+  void computeVolatileBitfields();
   void accumulateBases(const CXXRecordDecl *cxxRecordDecl);
   void accumulateVPtrs();
   void accumulateFields();
@@ -86,6 +87,10 @@ struct CIRRecordLowering final {
   accumulateBitFields(RecordDecl::field_iterator field,
                       RecordDecl::field_iterator fieldEnd);
 
+  bool isAAPCS() const {
+    return astContext.getTargetInfo().getABI().starts_with("aapcs");
+  }
+
   CharUnits bitsToCharUnits(uint64_t bitOffset) {
     return astContext.toCharUnitsFromBits(bitOffset);
   }
@@ -239,7 +244,7 @@ void CIRRecordLowering::setBitFieldInfo(const FieldDecl *fd,
 void CIRRecordLowering::lower() {
   if (recordDecl->isUnion()) {
     lowerUnion();
-    assert(!cir::MissingFeatures::bitfields());
+    computeVolatileBitfields();
     return;
   }
 
@@ -253,7 +258,7 @@ void CIRRecordLowering::lower() {
     accumulateBases(cxxRecordDecl);
     if (members.empty()) {
       appendPaddingBytes(size);
-      assert(!cir::MissingFeatures::bitfields());
+      computeVolatileBitfields();
       return;
     }
     assert(!cir::MissingFeatures::recordLayoutVirtualBases());
@@ -271,6 +276,7 @@ void CIRRecordLowering::lower() {
 
   calculateZeroInit();
   fillOutputFields();
+  computeVolatileBitfields();
 }
 
 void CIRRecordLowering::fillOutputFields() {
@@ -608,14 +614,6 @@ CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
     }
   }
 
-  if (llvm::isa<CXXRecordDecl>(rd) && !rd->isUnion() &&
-      !rd->hasAttr<FinalAttr>()) {
-    if (lowering.astRecordLayout.getNonVirtualSize() !=
-        lowering.astRecordLayout.getSize()) {
-      cgm.errorNYI(rd->getSourceRange(), "computeRecordLayout: CXXRecordDecl");
-    }
-  }
-
   // Fill in the record *after* computing the base type.  Filling in the body
   // signifies that the type is no longer opaque and record layout is complete,
   // but we may need to recursively layout rd while laying D out as a base type.
@@ -698,11 +696,14 @@ void CIRRecordLowering::lowerUnion() {
   // locate the "most appropriate" storage type.
   for (const FieldDecl *field : recordDecl->fields()) {
     mlir::Type fieldType;
-    if (field->isBitField())
-      cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
-                                         "bitfields in lowerUnion");
-    else
+    if (field->isBitField()) {
+      if (field->isZeroLengthBitField())
+        continue;
+      fieldType = getBitfieldStorageType(field->getBitWidthValue());
+      setBitFieldInfo(field, CharUnits::Zero(), fieldType);
+    } else {
       fieldType = getStorageType(field);
+    }
 
     // This maps a field to its index. For unions, the index is always 0.
     fieldIdxMap[field->getCanonicalDecl()] = 0;
@@ -754,6 +755,27 @@ void CIRRecordLowering::lowerUnion() {
     packed = true;
 }
 
+/// The AAPCS that defines that, when possible, bit-fields should
+/// be accessed using containers of the declared type width:
+/// When a volatile bit-field is read, and its container does not overlap with
+/// any non-bit-field member or any zero length bit-field member, its container
+/// must be read exactly once using the access width appropriate to the type of
+/// the container. When a volatile bit-field is written, and its container does
+/// not overlap with any non-bit-field member or any zero-length bit-field
+/// member, its container must be read exactly once and written exactly once
+/// using the access width appropriate to the type of the container. The two
+/// accesses are not atomic.
+///
+/// Enforcing the width restriction can be disabled using
+/// -fno-aapcs-bitfield-width.
+void CIRRecordLowering::computeVolatileBitfields() {
+  if (!isAAPCS() ||
+      !cirGenTypes.getCGModule().getCodeGenOpts().AAPCSBitfieldWidth)
+    return;
+
+  assert(!cir::MissingFeatures::armComputeVolatileBitfields());
+}
+
 void CIRRecordLowering::accumulateBases(const CXXRecordDecl *cxxRecordDecl) {
   // If we've got a primary virtual base, we need to add it with the bases.
   if (astRecordLayout.isPrimaryBaseVirtual()) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
index 1feefa55eb27..e89393c92db3 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACC.cpp
@@ -95,6 +95,8 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct(
       builder.setInsertionPointToEnd(&innerBlock);
 
       LexicalScope ls{*this, start, builder.getInsertionBlock()};
+      ActiveOpenACCLoopRAII activeLoop{*this, &loopOp};
+
       res = emitStmt(loopStmt, /*useCurrentScope=*/true);
 
       builder.create<mlir::acc::YieldOp>(end);
@@ -250,26 +252,60 @@ mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct(
 
 mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct(
     const OpenACCEnterDataConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC EnterData Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<EnterDataOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                             s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct(
     const OpenACCExitDataConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC ExitData Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<ExitDataOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                            s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult
 CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Update Construct");
-  return mlir::failure();
+  mlir::Location start = getLoc(s.getSourceRange().getBegin());
+  emitOpenACCOp<UpdateOp>(start, s.getDirectiveKind(), s.getDirectiveLoc(),
+                          s.clauses());
+  return mlir::success();
 }
+
 mlir::LogicalResult
-CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
-  return mlir::failure();
+CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {
+  // The 'cache' directive 'may' be at the top of a loop by standard, but
+  // doesn't have to be. Additionally, there is nothing that requires this be a
+  // loop affected by an OpenACC pragma. Sema doesn't do any level of
+  // enforcement here, since it isn't particularly valuable to do so thanks to
+  // that. Instead, we treat cache as a 'noop' if there is no acc.loop to apply
+  // it to.
+  if (!activeLoopOp)
+    return mlir::success();
+
+  mlir::acc::LoopOp loopOp = *activeLoopOp;
+
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPoint(loopOp);
+
+  for (const Expr *var : s.getVarList()) {
+    CIRGenFunction::OpenACCDataOperandInfo opInfo =
+        getOpenACCDataOperandInfo(var);
+
+    auto cacheOp = builder.create<CacheOp>(
+        opInfo.beginLoc, opInfo.varValue,
+        /*structured=*/false, /*implicit=*/false, opInfo.name, opInfo.bounds);
+
+    loopOp.getCacheOperandsMutable().append(cacheOp.getResult());
+  }
+
+  return mlir::success();
 }
+
 mlir::LogicalResult
-CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) {
-  cgm.errorNYI(s.getSourceRange(), "OpenACC Cache Construct");
+CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) {
+  cgm.errorNYI(s.getSourceRange(), "OpenACC Atomic Construct");
   return mlir::failure();
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
index 71f3ccb8e040..f3911ae8b677 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenACCLoop.cpp
@@ -130,6 +130,7 @@ CIRGenFunction::emitOpenACCLoopConstruct(const OpenACCLoopConstruct &s) {
     mlir::OpBuilder::InsertionGuard guardCase(builder);
     builder.setInsertionPointToEnd(&block);
     LexicalScope ls{*this, start, builder.getInsertionBlock()};
+    ActiveOpenACCLoopRAII activeLoop{*this, &op};
 
     stmtRes = emitStmt(s.getLoop(), /*useCurrentScope=*/true);
     builder.create<mlir::acc::YieldOp>(end);
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
index a5b7f0c9579b..1d081d53ad15 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
@@ -22,7 +22,7 @@ namespace clang::CIRGen {
 /// during IR emission. It's initialized once in CodeGenModule's
 /// constructor and then copied around into new CIRGenFunction's.
 struct CIRGenTypeCache {
-  CIRGenTypeCache() = default;
+  CIRGenTypeCache() {}
 
   // ClangIR void type
   cir::VoidType VoidTy;
@@ -49,14 +49,30 @@ struct CIRGenTypeCache {
   cir::FP80Type FP80Ty;
   cir::FP128Type FP128Ty;
 
+  /// intptr_t, size_t, and ptrdiff_t, which we assume are the same size.
+  union {
+    mlir::Type UIntPtrTy;
+    mlir::Type SizeTy;
+  };
+
   mlir::Type PtrDiffTy;
 
+  /// void* in address space 0
+  cir::PointerType VoidPtrTy;
+
   /// The size and alignment of a pointer into the generic address space.
   union {
     unsigned char PointerAlignInBytes;
     unsigned char PointerSizeInBytes;
   };
 
+  /// The alignment of size_t.
+  unsigned char SizeAlignInBytes;
+
+  clang::CharUnits getSizeAlign() const {
+    return clang::CharUnits::fromQuantity(SizeAlignInBytes);
+  }
+
   clang::CharUnits getPointerAlign() const {
     return clang::CharUnits::fromQuantity(PointerAlignInBytes);
   }
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index 621eb66962bf..3e07f6d3e54c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -424,7 +424,7 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
     mlir::Type elemTy = convertTypeForMem(arrTy->getElementType());
     // int X[] -> [0 x int], unless the element type is not sized.  If it is
     // unsized (e.g. an incomplete record) just use [0 x i8].
-    if (!builder.isSized(elemTy)) {
+    if (!cir::isSized(elemTy)) {
       elemTy = cgm.SInt8Ty;
     }
 
@@ -438,7 +438,7 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
 
     // TODO(CIR): In LLVM, "lower arrays of undefined struct type to arrays of
     // i8 just to have a concrete type"
-    if (!builder.isSized(elemTy)) {
+    if (!cir::isSized(elemTy)) {
       cgm.errorNYI(SourceLocation(), "arrays of undefined struct type", type);
       resultType = cgm.UInt32Ty;
       break;
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index a5a457ddafa9..e1b0f805a7b2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -19,6 +19,7 @@
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Type.h"
 
+#include "CIRGenRecordLayout.h"
 #include "mlir/IR/Value.h"
 
 #include "clang/CIR/MissingFeatures.h"
@@ -162,6 +163,7 @@ class LValue {
   mlir::Value vectorIdx; // Index for vector subscript
   mlir::Type elementType;
   LValueBaseInfo baseInfo;
+  const CIRGenBitFieldInfo *bitFieldInfo{nullptr};
 
   void initialize(clang::QualType type, clang::Qualifiers quals,
                   clang::CharUnits alignment, LValueBaseInfo baseInfo) {
@@ -245,6 +247,38 @@ class LValue {
     r.initialize(t, t.getQualifiers(), vecAddress.getAlignment(), baseInfo);
     return r;
   }
+
+  // bitfield lvalue
+  Address getBitFieldAddress() const {
+    return Address(getBitFieldPointer(), elementType, getAlignment());
+  }
+
+  mlir::Value getBitFieldPointer() const {
+    assert(isBitField());
+    return v;
+  }
+
+  const CIRGenBitFieldInfo &getBitFieldInfo() const {
+    assert(isBitField());
+    return *bitFieldInfo;
+  }
+
+  /// Create a new object to represent a bit-field access.
+  ///
+  /// \param Addr - The base address of the bit-field sequence this
+  /// bit-field refers to.
+  /// \param Info - The information describing how to perform the bit-field
+  /// access.
+  static LValue makeBitfield(Address addr, const CIRGenBitFieldInfo &info,
+                             clang::QualType type, LValueBaseInfo baseInfo) {
+    LValue r;
+    r.lvType = BitField;
+    r.v = addr.getPointer();
+    r.elementType = addr.getElementType();
+    r.bitFieldInfo = &info;
+    r.initialize(type, type.getQualifiers(), addr.getAlignment(), baseInfo);
+    return r;
+  }
 };
 
 /// An aggregate value slot.
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index 385bea066c61..03ea60c76c87 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -25,6 +25,7 @@ add_clang_library(clangCIR
   CIRGenFunction.cpp
   CIRGenItaniumCXXABI.cpp
   CIRGenModule.cpp
+  CIRGenOpenACC.cpp
   CIRGenOpenACCClause.cpp
   CIRGenRecordLayoutBuilder.cpp
   CIRGenStmt.cpp
diff --git a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
index b52e9c3e4919..c039bdcd7f6a 100644
--- a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
@@ -15,12 +15,25 @@
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+//===-----------------------------------------------------------------===//
+// IntLiteral
+//===-----------------------------------------------------------------===//
+
+static void printIntLiteral(mlir::AsmPrinter &p, llvm::APInt value,
+                            cir::IntTypeInterface ty);
+static mlir::ParseResult parseIntLiteral(mlir::AsmParser &parser,
+                                         llvm::APInt &value,
+                                         cir::IntTypeInterface ty);
+//===-----------------------------------------------------------------===//
+// FloatLiteral
+//===-----------------------------------------------------------------===//
+
 static void printFloatLiteral(mlir::AsmPrinter &p, llvm::APFloat value,
                               mlir::Type ty);
 static mlir::ParseResult
 parseFloatLiteral(mlir::AsmParser &parser,
                   mlir::FailureOr<llvm::APFloat> &value,
-                  cir::CIRFPTypeInterface fpType);
+                  cir::FPTypeInterface fpType);
 
 static mlir::ParseResult parseConstPtr(mlir::AsmParser &parser,
                                        mlir::IntegerAttr &value);
@@ -55,6 +68,21 @@ void CIRDialect::printAttribute(Attribute attr, DialectAsmPrinter &os) const {
     llvm_unreachable("unexpected CIR type kind");
 }
 
+//===----------------------------------------------------------------------===//
+// OptInfoAttr definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult OptInfoAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+                                  unsigned level, unsigned size) {
+  if (level > 3)
+    return emitError()
+           << "optimization level must be between 0 and 3 inclusive";
+  if (size > 2)
+    return emitError()
+           << "size optimization level must be between 0 and 2 inclusive";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // ConstPtrAttr definitions
 //===----------------------------------------------------------------------===//
@@ -82,69 +110,52 @@ static void printConstPtr(AsmPrinter &p, mlir::IntegerAttr value) {
 // IntAttr definitions
 //===----------------------------------------------------------------------===//
 
-Attribute IntAttr::parse(AsmParser &parser, Type odsType) {
-  mlir::APInt apValue;
-
-  if (!mlir::isa<IntType>(odsType))
-    return {};
-  auto type = mlir::cast<IntType>(odsType);
-
-  // Consume the '<' symbol.
-  if (parser.parseLess())
-    return {};
-
-  // Fetch arbitrary precision integer value.
-  if (type.isSigned()) {
-    int64_t value = 0;
-    if (parser.parseInteger(value)) {
-      parser.emitError(parser.getCurrentLocation(), "expected integer value");
-    } else {
-      apValue = mlir::APInt(type.getWidth(), value, type.isSigned(),
-                            /*implicitTrunc=*/true);
-      if (apValue.getSExtValue() != value)
-        parser.emitError(parser.getCurrentLocation(),
-                         "integer value too large for the given type");
-    }
+template <typename IntT>
+static bool isTooLargeForType(const mlir::APInt &value, IntT expectedValue) {
+  if constexpr (std::is_signed_v<IntT>) {
+    return value.getSExtValue() != expectedValue;
   } else {
-    uint64_t value = 0;
-    if (parser.parseInteger(value)) {
-      parser.emitError(parser.getCurrentLocation(), "expected integer value");
-    } else {
-      apValue = mlir::APInt(type.getWidth(), value, type.isSigned(),
-                            /*implicitTrunc=*/true);
-      if (apValue.getZExtValue() != value)
-        parser.emitError(parser.getCurrentLocation(),
-                         "integer value too large for the given type");
-    }
+    return value.getZExtValue() != expectedValue;
   }
+}
 
-  // Consume the '>' symbol.
-  if (parser.parseGreater())
-    return {};
+template <typename IntT>
+static mlir::ParseResult parseIntLiteralImpl(mlir::AsmParser &p,
+                                             llvm::APInt &value,
+                                             cir::IntTypeInterface ty) {
+  IntT ivalue;
+  const bool isSigned = ty.isSigned();
+  if (p.parseInteger(ivalue))
+    return p.emitError(p.getCurrentLocation(), "expected integer value");
+
+  value = mlir::APInt(ty.getWidth(), ivalue, isSigned, /*implicitTrunc=*/true);
+  if (isTooLargeForType(value, ivalue))
+    return p.emitError(p.getCurrentLocation(),
+                       "integer value too large for the given type");
 
-  return IntAttr::get(type, apValue);
+  return success();
 }
 
-void IntAttr::print(AsmPrinter &printer) const {
-  auto type = mlir::cast<IntType>(getType());
-  printer << '<';
-  if (type.isSigned())
-    printer << getSInt();
+mlir::ParseResult parseIntLiteral(mlir::AsmParser &parser, llvm::APInt &value,
+                                  cir::IntTypeInterface ty) {
+  if (ty.isSigned())
+    return parseIntLiteralImpl<int64_t>(parser, value, ty);
+  return parseIntLiteralImpl<uint64_t>(parser, value, ty);
+}
+
+void printIntLiteral(mlir::AsmPrinter &p, llvm::APInt value,
+                     cir::IntTypeInterface ty) {
+  if (ty.isSigned())
+    p << value.getSExtValue();
   else
-    printer << getUInt();
-  printer << '>';
+    p << value.getZExtValue();
 }
 
 LogicalResult IntAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                              Type type, APInt value) {
-  if (!mlir::isa<IntType>(type))
-    return emitError() << "expected 'simple.int' type";
-
-  auto intType = mlir::cast<IntType>(type);
-  if (value.getBitWidth() != intType.getWidth())
+                              cir::IntTypeInterface type, llvm::APInt value) {
+  if (value.getBitWidth() != type.getWidth())
     return emitError() << "type and value bitwidth mismatch: "
-                       << intType.getWidth() << " != " << value.getBitWidth();
-
+                       << type.getWidth() << " != " << value.getBitWidth();
   return success();
 }
 
@@ -158,7 +169,7 @@ static void printFloatLiteral(AsmPrinter &p, APFloat value, Type ty) {
 
 static ParseResult parseFloatLiteral(AsmParser &parser,
                                      FailureOr<APFloat> &value,
-                                     CIRFPTypeInterface fpType) {
+                                     cir::FPTypeInterface fpType) {
 
   APFloat parsedValue(0.0);
   if (parser.parseFloat(fpType.getFloatSemantics(), parsedValue))
@@ -171,11 +182,11 @@ static ParseResult parseFloatLiteral(AsmParser &parser,
 FPAttr FPAttr::getZero(Type type) {
   return get(type,
              APFloat::getZero(
-                 mlir::cast<CIRFPTypeInterface>(type).getFloatSemantics()));
+                 mlir::cast<cir::FPTypeInterface>(type).getFloatSemantics()));
 }
 
 LogicalResult FPAttr::verify(function_ref<InFlightDiagnostic()> emitError,
-                             CIRFPTypeInterface fpType, APFloat value) {
+                             cir::FPTypeInterface fpType, APFloat value) {
   if (APFloat::SemanticsToEnum(fpType.getFloatSemantics()) !=
       APFloat::SemanticsToEnum(value.getSemantics()))
     return emitError() << "floating-point semantics mismatch";
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 27f4ecb5ab85..8512b229c266 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -65,6 +65,10 @@ struct CIROpAsmDialectInterface : public OpAsmDialectInterface {
       os << (boolAttr.getValue() ? "true" : "false");
       return AliasResult::FinalAlias;
     }
+    if (auto bitfield = mlir::dyn_cast<cir::BitfieldInfoAttr>(attr)) {
+      os << "bfi_" << bitfield.getName().str();
+      return AliasResult::FinalAlias;
+    }
     return AliasResult::NoAlias;
   }
 };
@@ -115,9 +119,26 @@ template <typename Ty> struct EnumTraits {};
     static unsigned getMaxEnumVal() { return cir::getMaxEnumValFor##Ty(); }    \
   }
 
+REGISTER_ENUM_TYPE(GlobalLinkageKind);
+REGISTER_ENUM_TYPE(VisibilityKind);
 REGISTER_ENUM_TYPE(SideEffect);
 } // namespace
 
+/// Parse an enum from the keyword, or default to the provided default value.
+/// The return type is the enum type by default, unless overriden with the
+/// second template argument.
+template <typename EnumTy, typename RetTy = EnumTy>
+static RetTy parseOptionalCIRKeyword(AsmParser &parser, EnumTy defaultValue) {
+  llvm::SmallVector<llvm::StringRef, 10> names;
+  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
+    names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
+
+  int index = parseOptionalKeywordAlternative(parser, names);
+  if (index == -1)
+    return static_cast<RetTy>(defaultValue);
+  return static_cast<RetTy>(index);
+}
+
 /// Parse an enum from the keyword, return failure if the keyword is not found.
 template <typename EnumTy, typename RetTy = EnumTy>
 static ParseResult parseCIRKeyword(AsmParser &parser, RetTy &result) {
@@ -170,6 +191,26 @@ static bool omitRegionTerm(mlir::Region &r) {
   return singleNonEmptyBlock && yieldsNothing();
 }
 
+void printVisibilityAttr(OpAsmPrinter &printer,
+                         cir::VisibilityAttr &visibility) {
+  switch (visibility.getValue()) {
+  case cir::VisibilityKind::Hidden:
+    printer << "hidden";
+    break;
+  case cir::VisibilityKind::Protected:
+    printer << "protected";
+    break;
+  case cir::VisibilityKind::Default:
+    break;
+  }
+}
+
+void parseVisibilityAttr(OpAsmParser &parser, cir::VisibilityAttr &visibility) {
+  cir::VisibilityKind visibilityKind =
+      parseOptionalCIRKeyword(parser, cir::VisibilityKind::Default);
+  visibility = cir::VisibilityAttr::get(parser.getContext(), visibilityKind);
+}
+
 //===----------------------------------------------------------------------===//
 // CIR Custom Parsers/Printers
 //===----------------------------------------------------------------------===//
@@ -384,13 +425,13 @@ LogicalResult cir::CastOp::verify() {
     return success();
   }
   case cir::CastKind::floating: {
-    if (!mlir::isa<cir::CIRFPTypeInterface>(srcType) ||
-        !mlir::isa<cir::CIRFPTypeInterface>(resType))
+    if (!mlir::isa<cir::FPTypeInterface>(srcType) ||
+        !mlir::isa<cir::FPTypeInterface>(resType))
       return emitOpError() << "requires !cir.float type for source and result";
     return success();
   }
   case cir::CastKind::float_to_int: {
-    if (!mlir::isa<cir::CIRFPTypeInterface>(srcType))
+    if (!mlir::isa<cir::FPTypeInterface>(srcType))
       return emitOpError() << "requires !cir.float type for source";
     if (!mlir::dyn_cast<cir::IntType>(resType))
       return emitOpError() << "requires !cir.int type for result";
@@ -411,7 +452,7 @@ LogicalResult cir::CastOp::verify() {
     return success();
   }
   case cir::CastKind::float_to_bool: {
-    if (!mlir::isa<cir::CIRFPTypeInterface>(srcType))
+    if (!mlir::isa<cir::FPTypeInterface>(srcType))
       return emitOpError() << "requires !cir.float type for source";
     if (!mlir::isa<cir::BoolType>(resType))
       return emitOpError() << "requires !cir.bool type for result";
@@ -427,14 +468,14 @@ LogicalResult cir::CastOp::verify() {
   case cir::CastKind::int_to_float: {
     if (!mlir::isa<cir::IntType>(srcType))
       return emitOpError() << "requires !cir.int type for source";
-    if (!mlir::isa<cir::CIRFPTypeInterface>(resType))
+    if (!mlir::isa<cir::FPTypeInterface>(resType))
       return emitOpError() << "requires !cir.float type for result";
     return success();
   }
   case cir::CastKind::bool_to_float: {
     if (!mlir::isa<cir::BoolType>(srcType))
       return emitOpError() << "requires !cir.bool type for source";
-    if (!mlir::isa<cir::CIRFPTypeInterface>(resType))
+    if (!mlir::isa<cir::FPTypeInterface>(resType))
       return emitOpError() << "requires !cir.float type for result";
     return success();
   }
@@ -556,7 +597,9 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   llvm::ArrayRef<mlir::Type> allResultTypes;
 
   // If we cannot parse a string callee, it means this is an indirect call.
-  if (!parser.parseOptionalAttribute(calleeAttr, "callee", result.attributes)
+  if (!parser
+           .parseOptionalAttribute(calleeAttr, CIRDialect::getCalleeAttrName(),
+                                   result.attributes)
            .has_value()) {
     OpAsmParser::UnresolvedOperand indirectVal;
     // Do not resolve right now, since we need to figure out the type
@@ -574,6 +617,10 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   if (parser.parseRParen())
     return mlir::failure();
 
+  if (parser.parseOptionalKeyword("nothrow").succeeded())
+    result.addAttribute(CIRDialect::getNoThrowAttrName(),
+                        mlir::UnitAttr::get(parser.getContext()));
+
   if (parser.parseOptionalKeyword("side_effect").succeeded()) {
     if (parser.parseLParen().failed())
       return failure();
@@ -583,7 +630,7 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
     if (parser.parseRParen().failed())
       return failure();
     auto attr = cir::SideEffectAttr::get(parser.getContext(), sideEffect);
-    result.addAttribute("side_effect", attr);
+    result.addAttribute(CIRDialect::getSideEffectAttrName(), attr);
   }
 
   if (parser.parseOptionalAttrDict(result.attributes))
@@ -608,7 +655,7 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
 static void printCallCommon(mlir::Operation *op,
                             mlir::FlatSymbolRefAttr calleeSym,
                             mlir::Value indirectCallee,
-                            mlir::OpAsmPrinter &printer,
+                            mlir::OpAsmPrinter &printer, bool isNothrow,
                             cir::SideEffect sideEffect) {
   printer << ' ';
 
@@ -625,13 +672,19 @@ static void printCallCommon(mlir::Operation *op,
   }
   printer << "(" << ops << ")";
 
+  if (isNothrow)
+    printer << " nothrow";
+
   if (sideEffect != cir::SideEffect::All) {
     printer << " side_effect(";
     printer << stringifySideEffect(sideEffect);
     printer << ")";
   }
 
-  printer.printOptionalAttrDict(op->getAttrs(), {"callee", "side_effect"});
+  printer.printOptionalAttrDict(op->getAttrs(),
+                                {CIRDialect::getCalleeAttrName(),
+                                 CIRDialect::getNoThrowAttrName(),
+                                 CIRDialect::getSideEffectAttrName()});
 
   printer << " : ";
   printer.printFunctionalType(op->getOperands().getTypes(),
@@ -646,13 +699,15 @@ mlir::ParseResult cir::CallOp::parse(mlir::OpAsmParser &parser,
 void cir::CallOp::print(mlir::OpAsmPrinter &p) {
   mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr;
   cir::SideEffect sideEffect = getSideEffect();
-  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, sideEffect);
+  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, getNothrow(),
+                  sideEffect);
 }
 
 static LogicalResult
 verifyCallCommInSymbolUses(mlir::Operation *op,
                            SymbolTableCollection &symbolTable) {
-  auto fnAttr = op->getAttrOfType<FlatSymbolRefAttr>("callee");
+  auto fnAttr =
+      op->getAttrOfType<FlatSymbolRefAttr>(CIRDialect::getCalleeAttrName());
   if (!fnAttr) {
     // This is an indirect call, thus we don't have to check the symbol uses.
     return mlir::success();
@@ -1287,19 +1342,54 @@ cir::GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
 // FuncOp
 //===----------------------------------------------------------------------===//
 
+/// Returns the name used for the linkage attribute. This *must* correspond to
+/// the name of the attribute in ODS.
+static llvm::StringRef getLinkageAttrNameString() { return "linkage"; }
+
 void cir::FuncOp::build(OpBuilder &builder, OperationState &result,
-                        StringRef name, FuncType type) {
+                        StringRef name, FuncType type,
+                        GlobalLinkageKind linkage) {
   result.addRegion();
   result.addAttribute(SymbolTable::getSymbolAttrName(),
                       builder.getStringAttr(name));
   result.addAttribute(getFunctionTypeAttrName(result.name),
                       TypeAttr::get(type));
+  result.addAttribute(
+      getLinkageAttrNameString(),
+      GlobalLinkageKindAttr::get(builder.getContext(), linkage));
+  result.addAttribute(getGlobalVisibilityAttrName(result.name),
+                      cir::VisibilityAttr::get(builder.getContext()));
 }
 
 ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
   llvm::SMLoc loc = parser.getCurrentLocation();
   mlir::Builder &builder = parser.getBuilder();
 
+  mlir::StringAttr visNameAttr = getSymVisibilityAttrName(state.name);
+  mlir::StringAttr visibilityNameAttr = getGlobalVisibilityAttrName(state.name);
+  mlir::StringAttr dsoLocalNameAttr = getDsoLocalAttrName(state.name);
+
+  // Default to external linkage if no keyword is provided.
+  state.addAttribute(getLinkageAttrNameString(),
+                     GlobalLinkageKindAttr::get(
+                         parser.getContext(),
+                         parseOptionalCIRKeyword<GlobalLinkageKind>(
+                             parser, GlobalLinkageKind::ExternalLinkage)));
+
+  ::llvm::StringRef visAttrStr;
+  if (parser.parseOptionalKeyword(&visAttrStr, {"private", "public", "nested"})
+          .succeeded()) {
+    state.addAttribute(visNameAttr,
+                       parser.getBuilder().getStringAttr(visAttrStr));
+  }
+
+  cir::VisibilityAttr cirVisibilityAttr;
+  parseVisibilityAttr(parser, cirVisibilityAttr);
+  state.addAttribute(visibilityNameAttr, cirVisibilityAttr);
+
+  if (parser.parseOptionalKeyword(dsoLocalNameAttr).succeeded())
+    state.addAttribute(dsoLocalNameAttr, parser.getBuilder().getUnitAttr());
+
   StringAttr nameAttr;
   if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                              state.attributes))
@@ -1331,11 +1421,27 @@ ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
   state.addAttribute(getFunctionTypeAttrName(state.name),
                      TypeAttr::get(fnType));
 
+  bool hasAlias = false;
+  mlir::StringAttr aliaseeNameAttr = getAliaseeAttrName(state.name);
+  if (parser.parseOptionalKeyword("alias").succeeded()) {
+    if (parser.parseLParen().failed())
+      return failure();
+    mlir::StringAttr aliaseeAttr;
+    if (parser.parseOptionalSymbolName(aliaseeAttr).failed())
+      return failure();
+    state.addAttribute(aliaseeNameAttr, FlatSymbolRefAttr::get(aliaseeAttr));
+    if (parser.parseRParen().failed())
+      return failure();
+    hasAlias = true;
+  }
+
   // Parse the optional function body.
   auto *body = state.addRegion();
   OptionalParseResult parseResult = parser.parseOptionalRegion(
       *body, arguments, /*enableNameShadowing=*/false);
   if (parseResult.has_value()) {
+    if (hasAlias)
+      return parser.emitError(loc, "function alias shall not have a body");
     if (failed(*parseResult))
       return failure();
     // Function body was parsed, make sure its not empty.
@@ -1346,9 +1452,17 @@ ParseResult cir::FuncOp::parse(OpAsmParser &parser, OperationState &state) {
   return success();
 }
 
+// This function corresponds to `llvm::GlobalValue::isDeclaration` and should
+// have a similar implementation. We don't currently ifuncs or materializable
+// functions, but those should be handled here as they are implemented.
 bool cir::FuncOp::isDeclaration() {
-  // TODO(CIR): This function will actually do something once external
-  // function declarations and aliases are upstreamed.
+  assert(!cir::MissingFeatures::supportIFuncAttr());
+
+  std::optional<StringRef> aliasee = getAliasee();
+  if (!aliasee)
+    return getFunctionBody().empty();
+
+  // Aliases are always definitions.
   return false;
 }
 
@@ -1359,12 +1473,37 @@ mlir::Region *cir::FuncOp::getCallableRegion() {
 }
 
 void cir::FuncOp::print(OpAsmPrinter &p) {
+  if (getComdat())
+    p << " comdat";
+
+  if (getLinkage() != GlobalLinkageKind::ExternalLinkage)
+    p << ' ' << stringifyGlobalLinkageKind(getLinkage());
+
+  mlir::SymbolTable::Visibility vis = getVisibility();
+  if (vis != mlir::SymbolTable::Visibility::Public)
+    p << ' ' << vis;
+
+  cir::VisibilityAttr cirVisibilityAttr = getGlobalVisibilityAttr();
+  if (!cirVisibilityAttr.isDefault()) {
+    p << ' ';
+    printVisibilityAttr(p, cirVisibilityAttr);
+  }
+
+  if (getDsoLocal())
+    p << " dso_local";
+
   p << ' ';
   p.printSymbolName(getSymName());
   cir::FuncType fnType = getFunctionType();
   function_interface_impl::printFunctionSignature(
       p, *this, fnType.getInputs(), fnType.isVarArg(), fnType.getReturnTypes());
 
+  if (std::optional<StringRef> aliaseeName = getAliasee()) {
+    p << " alias(";
+    p.printSymbolName(*aliaseeName);
+    p << ")";
+  }
+
   // Print the body if this is not an external function.
   Region &body = getOperation()->getRegion(0);
   if (!body.empty()) {
@@ -1914,6 +2053,42 @@ OpFoldResult cir::ComplexCreateOp::fold(FoldAdaptor adaptor) {
   return cir::ConstComplexAttr::get(realAttr, imagAttr);
 }
 
+//===----------------------------------------------------------------------===//
+// ComplexRealOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::ComplexRealOp::verify() {
+  if (getType() != getOperand().getType().getElementType()) {
+    emitOpError() << ": result type does not match operand type";
+    return failure();
+  }
+  return success();
+}
+
+OpFoldResult cir::ComplexRealOp::fold(FoldAdaptor adaptor) {
+  auto complex =
+      mlir::cast_if_present<cir::ConstComplexAttr>(adaptor.getOperand());
+  return complex ? complex.getReal() : nullptr;
+}
+
+//===----------------------------------------------------------------------===//
+// ComplexImagOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::ComplexImagOp::verify() {
+  if (getType() != getOperand().getType().getElementType()) {
+    emitOpError() << ": result type does not match operand type";
+    return failure();
+  }
+  return success();
+}
+
+OpFoldResult cir::ComplexImagOp::fold(FoldAdaptor adaptor) {
+  auto complex =
+      mlir::cast_if_present<cir::ConstComplexAttr>(adaptor.getOperand());
+  return complex ? complex.getImag() : nullptr;
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
index 09926a93eb6f..c6760cf1618c 100644
--- a/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRTypes.cpp
@@ -18,6 +18,16 @@
 #include "clang/CIR/MissingFeatures.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+//===----------------------------------------------------------------------===//
+// CIR Helpers
+//===----------------------------------------------------------------------===//
+bool cir::isSized(mlir::Type ty) {
+  if (auto sizedTy = mlir::dyn_cast<cir::SizedTypeInterface>(ty))
+    return sizedTy.isSized();
+  assert(!cir::MissingFeatures::unsizedTypes());
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // CIR Custom Parser/Printer Signatures
 //===----------------------------------------------------------------------===//
@@ -539,8 +549,7 @@ uint64_t FP128Type::getABIAlignment(const mlir::DataLayout &dataLayout,
 }
 
 const llvm::fltSemantics &LongDoubleType::getFloatSemantics() const {
-  return mlir::cast<cir::CIRFPTypeInterface>(getUnderlying())
-      .getFloatSemantics();
+  return mlir::cast<cir::FPTypeInterface>(getUnderlying()).getFloatSemantics();
 }
 
 llvm::TypeSize
diff --git a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
index f07e234e5e84..e505db50d360 100644
--- a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
@@ -141,8 +141,9 @@ void CIRCanonicalizePass::runOnOperation() {
     // Many operations are here to perform a manual `fold` in
     // applyOpPatternsGreedily.
     if (isa<BrOp, BrCondOp, CastOp, ScopeOp, SwitchOp, SelectOp, UnaryOp,
-            ComplexCreateOp, VecCmpOp, VecCreateOp, VecExtractOp, VecShuffleOp,
-            VecShuffleDynamicOp, VecTernaryOp>(op))
+            ComplexCreateOp, ComplexImagOp, ComplexRealOp, VecCmpOp,
+            VecCreateOp, VecExtractOp, VecShuffleOp, VecShuffleDynamicOp,
+            VecTernaryOp>(op))
       ops.push_back(op);
   });
 
diff --git a/clang/lib/CIR/Interfaces/CIRFPTypeInterface.cpp b/clang/lib/CIR/Interfaces/CIRTypeInterfaces.cpp
similarity index 73%
rename from clang/lib/CIR/Interfaces/CIRFPTypeInterface.cpp
rename to clang/lib/CIR/Interfaces/CIRTypeInterfaces.cpp
index 41817e90b523..8c0daa03b2ff 100644
--- a/clang/lib/CIR/Interfaces/CIRFPTypeInterface.cpp
+++ b/clang/lib/CIR/Interfaces/CIRTypeInterfaces.cpp
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Defines the interface to generically handle CIR floating-point types.
+// Defines cir type interfaces.
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/CIR/Interfaces/CIRFPTypeInterface.h"
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 
 using namespace cir;
 
 /// Include the generated interfaces.
-#include "clang/CIR/Interfaces/CIRFPTypeInterface.cpp.inc"
+#include "clang/CIR/Interfaces/CIRTypeInterfaces.cpp.inc"
diff --git a/clang/lib/CIR/Interfaces/CMakeLists.txt b/clang/lib/CIR/Interfaces/CMakeLists.txt
index 04a8972fba9a..13e4c2040f1c 100644
--- a/clang/lib/CIR/Interfaces/CMakeLists.txt
+++ b/clang/lib/CIR/Interfaces/CMakeLists.txt
@@ -1,14 +1,14 @@
 add_clang_library(MLIRCIRInterfaces
   CIROpInterfaces.cpp
   CIRLoopOpInterface.cpp
-  CIRFPTypeInterface.cpp
+  CIRTypeInterfaces.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Interfaces
 
   DEPENDS
   MLIRCIREnumsGen
-  MLIRCIRFPTypeInterfaceIncGen
+  MLIRCIRTypeInterfacesIncGen
   MLIRCIRLoopOpInterfaceIncGen
   MLIRCIROpInterfacesIncGen
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 5f41e340e247..5ac42b6a63b0 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -97,6 +97,18 @@ static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,
   return bld.create<mlir::LLVM::BitcastOp>(loc, dstTy, src);
 }
 
+static mlir::LLVM::Visibility
+lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {
+  switch (visibilityKind) {
+  case cir::VisibilityKind::Default:
+    return ::mlir::LLVM::Visibility::Default;
+  case cir::VisibilityKind::Hidden:
+    return ::mlir::LLVM::Visibility::Hidden;
+  case cir::VisibilityKind::Protected:
+    return ::mlir::LLVM::Visibility::Protected;
+  }
+}
+
 /// Emits the value from memory as expected by its users. Should be called when
 /// the memory represetnation of a CIR type is not equal to its scalar
 /// representation.
@@ -220,7 +232,7 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
   return value;
 }
 
-void convertSideEffectForCall(mlir::Operation *callOp,
+void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
                               cir::SideEffect sideEffect,
                               mlir::LLVM::MemoryEffectsAttr &memoryEffect,
                               bool &noUnwind, bool &willReturn) {
@@ -229,7 +241,7 @@ void convertSideEffectForCall(mlir::Operation *callOp,
   switch (sideEffect) {
   case cir::SideEffect::All:
     memoryEffect = {};
-    noUnwind = false;
+    noUnwind = isNothrow;
     willReturn = false;
     break;
 
@@ -448,6 +460,81 @@ mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
+    cir::BitClrsbOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto zero = rewriter.create<mlir::LLVM::ConstantOp>(
+      op.getLoc(), adaptor.getInput().getType(), 0);
+  auto isNeg = rewriter.create<mlir::LLVM::ICmpOp>(
+      op.getLoc(),
+      mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
+                                         mlir::LLVM::ICmpPredicate::slt),
+      adaptor.getInput(), zero);
+
+  auto negOne = rewriter.create<mlir::LLVM::ConstantOp>(
+      op.getLoc(), adaptor.getInput().getType(), -1);
+  auto flipped = rewriter.create<mlir::LLVM::XOrOp>(op.getLoc(),
+                                                    adaptor.getInput(), negOne);
+
+  auto select = rewriter.create<mlir::LLVM::SelectOp>(
+      op.getLoc(), isNeg, flipped, adaptor.getInput());
+
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto clz = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
+      op.getLoc(), resTy, select, /*is_zero_poison=*/false);
+
+  auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
+  auto res = rewriter.create<mlir::LLVM::SubOp>(op.getLoc(), clz, one);
+  rewriter.replaceOp(op, res);
+
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(
+    cir::BitClzOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CountLeadingZerosOp>(
+      op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(
+    cir::BitCtzOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CountTrailingZerosOp>(
+      op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
+    cir::BitParityOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto popcnt = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
+                                                     adaptor.getInput());
+
+  auto one = rewriter.create<mlir::LLVM::ConstantOp>(op.getLoc(), resTy, 1);
+  auto popcntMod2 =
+      rewriter.create<mlir::LLVM::AndOp>(op.getLoc(), popcnt, one);
+  rewriter.replaceOp(op, popcntMod2);
+
+  return mlir::LogicalResult::success();
+}
+
+mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(
+    cir::BitPopcountOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto resTy = getTypeConverter()->convertType(op.getType());
+  auto llvmOp = rewriter.create<mlir::LLVM::CtPopOp>(op.getLoc(), resTy,
+                                                     adaptor.getInput());
+  rewriter.replaceOp(op, llvmOp);
+  return mlir::LogicalResult::success();
+}
+
 mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
     cir::BrCondOp brOp, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -517,12 +604,12 @@ mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
     mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
     mlir::Type dstTy = elementTypeIfVector(castOp.getType());
 
-    if (!mlir::isa<cir::CIRFPTypeInterface>(dstTy) ||
-        !mlir::isa<cir::CIRFPTypeInterface>(srcTy))
+    if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||
+        !mlir::isa<cir::FPTypeInterface>(srcTy))
       return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;
 
     auto getFloatWidth = [](mlir::Type ty) -> unsigned {
-      return mlir::cast<cir::CIRFPTypeInterface>(ty).getWidth();
+      return mlir::cast<cir::FPTypeInterface>(ty).getWidth();
     };
 
     if (getFloatWidth(srcTy) > getFloatWidth(dstTy))
@@ -788,8 +875,8 @@ rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
   mlir::LLVM::MemoryEffectsAttr memoryEffects;
   bool noUnwind = false;
   bool willReturn = false;
-  convertSideEffectForCall(op, call.getSideEffect(), memoryEffects, noUnwind,
-                           willReturn);
+  convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),
+                           memoryEffects, noUnwind, willReturn);
 
   mlir::LLVM::LLVMFunctionType llvmFnTy;
   if (calleeAttr) { // direct call
@@ -916,7 +1003,7 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     attr = rewriter.getIntegerAttr(
         typeConverter->convertType(op.getType()),
         mlir::cast<cir::IntAttr>(op.getValue()).getValue());
-  } else if (mlir::isa<cir::CIRFPTypeInterface>(op.getType())) {
+  } else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {
     attr = rewriter.getFloatAttr(
         typeConverter->convertType(op.getType()),
         mlir::cast<cir::FPAttr>(op.getValue()).getValue());
@@ -989,6 +1076,22 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
+    cir::ExpectOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // TODO(cir): do not generate LLVM intrinsics under -O0
+  assert(!cir::MissingFeatures::optInfoAttr());
+
+  std::optional<llvm::APFloat> prob = op.getProb();
+  if (prob)
+    rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
+        op, adaptor.getVal(), adaptor.getExpected(), prob.value());
+  else
+    rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),
+                                                      adaptor.getExpected());
+  return mlir::success();
+}
+
 /// Convert the `cir.func` attributes to `llvm.func` attributes.
 /// Only retain those attributes that are not constructed by
 /// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out
@@ -998,9 +1101,12 @@ void CIRToLLVMFuncOpLowering::lowerFuncAttributes(
     SmallVectorImpl<mlir::NamedAttribute> &result) const {
   assert(!cir::MissingFeatures::opFuncCallingConv());
   for (mlir::NamedAttribute attr : func->getAttrs()) {
+    assert(!cir::MissingFeatures::opFuncCallingConv());
     if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||
         attr.getName() == func.getFunctionTypeAttrName() ||
         attr.getName() == getLinkageAttrNameString() ||
+        attr.getName() == func.getGlobalVisibilityAttrName() ||
+        attr.getName() == func.getDsoLocalAttrName() ||
         (filterArgAndResAttrs &&
          (attr.getName() == func.getArgAttrsAttrName() ||
           attr.getName() == func.getResAttrsAttrName())))
@@ -1016,8 +1122,7 @@ mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
     mlir::ConversionPatternRewriter &rewriter) const {
 
   cir::FuncType fnType = op.getFunctionType();
-  assert(!cir::MissingFeatures::opFuncDsolocal());
-  bool isDsoLocal = false;
+  bool isDsoLocal = op.getDsoLocal();
   mlir::TypeConverter::SignatureConversion signatureConversion(
       fnType.getNumInputs());
 
@@ -1045,8 +1150,7 @@ mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
           mlir::isa<mlir::UnknownLoc>(loc)) &&
          "expected single location or unknown location here");
 
-  assert(!cir::MissingFeatures::opFuncLinkage());
-  mlir::LLVM::Linkage linkage = mlir::LLVM::Linkage::External;
+  mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
   assert(!cir::MissingFeatures::opFuncCallingConv());
   mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;
   SmallVector<mlir::NamedAttribute, 4> attributes;
@@ -1056,7 +1160,11 @@ mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
       loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,
       mlir::SymbolRefAttr(), attributes);
 
-  assert(!cir::MissingFeatures::opFuncVisibility());
+  assert(!cir::MissingFeatures::opFuncMultipleReturnVals());
+
+  fn.setVisibility_Attr(mlir::LLVM::VisibilityAttr::get(
+      getContext(), lowerCIRVisibilityToLLVMVisibility(
+                        op.getGlobalVisibilityAttr().getValue())));
 
   rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());
   if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,
@@ -1103,7 +1211,7 @@ void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
   const bool isConst = false;
   assert(!cir::MissingFeatures::addressSpace());
   const unsigned addrSpace = 0;
-  const bool isDsoLocal = op.getDsolocal();
+  const bool isDsoLocal = op.getDsoLocal();
   assert(!cir::MissingFeatures::opGlobalThreadLocal());
   const bool isThreadLocal = false;
   const uint64_t alignment = op.getAlignment().value_or(0);
@@ -1157,7 +1265,7 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
   const bool isConst = false;
   assert(!cir::MissingFeatures::addressSpace());
   const unsigned addrSpace = 0;
-  const bool isDsoLocal = op.getDsolocal();
+  const bool isDsoLocal = op.getDsoLocal();
   assert(!cir::MissingFeatures::opGlobalThreadLocal());
   const bool isThreadLocal = false;
   const uint64_t alignment = op.getAlignment().value_or(0);
@@ -1316,7 +1424,7 @@ mlir::LogicalResult CIRToLLVMUnaryOpLowering::matchAndRewrite(
   }
 
   // Floating point unary operations: + - ++ --
-  if (mlir::isa<cir::CIRFPTypeInterface>(elementType)) {
+  if (mlir::isa<cir::FPTypeInterface>(elementType)) {
     switch (op.getKind()) {
     case cir::UnaryOpKind::Inc: {
       assert(!isVector && "++ not allowed on vector types");
@@ -1405,7 +1513,7 @@ mlir::LogicalResult CIRToLLVMBinOpLowering::matchAndRewrite(
     return op.emitError() << "inconsistent operands' types not supported yet";
 
   mlir::Type type = op.getRhs().getType();
-  if (!mlir::isa<cir::IntType, cir::BoolType, cir::CIRFPTypeInterface,
+  if (!mlir::isa<cir::IntType, cir::BoolType, cir::FPTypeInterface,
                  mlir::IntegerType, cir::VectorType>(type))
     return op.emitError() << "operand type not supported yet";
 
@@ -1553,7 +1661,6 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
   assert(!cir::MissingFeatures::dataMemberType());
   assert(!cir::MissingFeatures::methodType());
 
-  // Lower to LLVM comparison op.
   if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
     bool isSigned = mlir::isa<cir::IntType>(type)
                         ? mlir::cast<cir::IntType>(type).isSigned()
@@ -1562,22 +1669,82 @@ mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
         convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
+    return mlir::success();
+  }
+
+  if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
     mlir::LLVM::ICmpPredicate kind =
         convertCmpKindToICmpPredicate(cmpOp.getKind(),
                                       /* isSigned=*/false);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else if (mlir::isa<cir::CIRFPTypeInterface>(type)) {
+    return mlir::success();
+  }
+
+  if (mlir::isa<cir::FPTypeInterface>(type)) {
     mlir::LLVM::FCmpPredicate kind =
         convertCmpKindToFCmpPredicate(cmpOp.getKind());
     rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
         cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
-  } else {
-    return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
+    return mlir::success();
   }
 
-  return mlir::success();
+  if (mlir::isa<cir::ComplexType>(type)) {
+    mlir::Value lhs = adaptor.getLhs();
+    mlir::Value rhs = adaptor.getRhs();
+    mlir::Location loc = cmpOp.getLoc();
+
+    auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
+    mlir::Type complexElemTy =
+        getTypeConverter()->convertType(complexType.getElementType());
+
+    auto lhsReal =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 0);
+    auto lhsImag =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, lhs, 1);
+    auto rhsReal =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 0);
+    auto rhsImag =
+        rewriter.create<mlir::LLVM::ExtractValueOp>(loc, complexElemTy, rhs, 1);
+
+    if (cmpOp.getKind() == cir::CmpOpKind::eq) {
+      if (complexElemTy.isInteger()) {
+        auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
+        auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
+        rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
+        return mlir::success();
+      }
+
+      auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
+      auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
+      rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
+      return mlir::success();
+    }
+
+    if (cmpOp.getKind() == cir::CmpOpKind::ne) {
+      if (complexElemTy.isInteger()) {
+        auto realCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
+        auto imagCmp = rewriter.create<mlir::LLVM::ICmpOp>(
+            loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
+        rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
+        return mlir::success();
+      }
+
+      auto realCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
+      auto imagCmp = rewriter.create<mlir::LLVM::FCmpOp>(
+          loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
+      rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
+      return mlir::success();
+    }
+  }
+
+  return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
 }
 
 mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
@@ -1863,30 +2030,39 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMAssumeOpLowering,
                CIRToLLVMBaseClassAddrOpLowering,
                CIRToLLVMBinOpLowering,
+               CIRToLLVMBitClrsbOpLowering,
+               CIRToLLVMBitClzOpLowering,
+               CIRToLLVMBitCtzOpLowering,
+               CIRToLLVMBitParityOpLowering,
+               CIRToLLVMBitPopcountOpLowering,
                CIRToLLVMBrCondOpLowering,
                CIRToLLVMBrOpLowering,
                CIRToLLVMCallOpLowering,
                CIRToLLVMCmpOpLowering,
+               CIRToLLVMComplexCreateOpLowering,
+               CIRToLLVMComplexImagOpLowering,
+               CIRToLLVMComplexRealOpLowering,
                CIRToLLVMConstantOpLowering,
+               CIRToLLVMExpectOpLowering,
                CIRToLLVMFuncOpLowering,
+               CIRToLLVMGetBitfieldOpLowering,
                CIRToLLVMGetGlobalOpLowering,
                CIRToLLVMGetMemberOpLowering,
                CIRToLLVMSelectOpLowering,
-               CIRToLLVMSwitchFlatOpLowering,
                CIRToLLVMShiftOpLowering,
-               CIRToLLVMStackSaveOpLowering,
                CIRToLLVMStackRestoreOpLowering,
+               CIRToLLVMStackSaveOpLowering,
+               CIRToLLVMSwitchFlatOpLowering,
                CIRToLLVMTrapOpLowering,
                CIRToLLVMUnaryOpLowering,
+               CIRToLLVMVecCmpOpLowering,
                CIRToLLVMVecCreateOpLowering,
                CIRToLLVMVecExtractOpLowering,
                CIRToLLVMVecInsertOpLowering,
-               CIRToLLVMVecCmpOpLowering,
-               CIRToLLVMVecSplatOpLowering,
-               CIRToLLVMVecShuffleOpLowering,
                CIRToLLVMVecShuffleDynamicOpLowering,
-               CIRToLLVMVecTernaryOpLowering,
-               CIRToLLVMComplexCreateOpLowering
+               CIRToLLVMVecShuffleOpLowering,
+               CIRToLLVMVecSplatOpLowering,
+               CIRToLLVMVecTernaryOpLowering
       // clang-format on
       >(converter, patterns.getContext());
 
@@ -2022,7 +2198,7 @@ mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
         op.getLoc(),
         convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()),
         adaptor.getLhs(), adaptor.getRhs());
-  } else if (mlir::isa<cir::CIRFPTypeInterface>(elementType)) {
+  } else if (mlir::isa<cir::FPTypeInterface>(elementType)) {
     bitResult = rewriter.create<mlir::LLVM::FCmpOp>(
         op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()),
         adaptor.getLhs(), adaptor.getRhs());
@@ -2190,6 +2366,77 @@ mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(
+    cir::ComplexRealOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
+      op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{0});
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
+    cir::ComplexImagOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
+      op, resultLLVMTy, adaptor.getOperand(), llvm::ArrayRef<std::int64_t>{1});
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
+    cir::GetBitfieldOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+
+  mlir::OpBuilder::InsertionGuard guard(rewriter);
+  rewriter.setInsertionPoint(op);
+
+  cir::BitfieldInfoAttr info = op.getBitfieldInfo();
+  uint64_t size = info.getSize();
+  uint64_t offset = info.getOffset();
+  mlir::Type storageType = info.getStorageType();
+  mlir::MLIRContext *context = storageType.getContext();
+  unsigned storageSize = 0;
+
+  mlir::IntegerType intType =
+      TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
+          .Case<cir::ArrayType>([&](cir::ArrayType atTy) {
+            storageSize = atTy.getSize() * 8;
+            return mlir::IntegerType::get(context, storageSize);
+          })
+          .Case<cir::IntType>([&](cir::IntType intTy) {
+            storageSize = intTy.getWidth();
+            return mlir::IntegerType::get(context, storageSize);
+          })
+          .Default([](mlir::Type) -> mlir::IntegerType {
+            llvm_unreachable(
+                "Either ArrayType or IntType expected for bitfields storage");
+          });
+
+  mlir::Value val = rewriter.create<mlir::LLVM::LoadOp>(
+      op.getLoc(), intType, adaptor.getAddr(), 0, op.getIsVolatile());
+  val = rewriter.create<mlir::LLVM::BitcastOp>(op.getLoc(), intType, val);
+
+  if (info.getIsSigned()) {
+    assert(static_cast<unsigned>(offset + size) <= storageSize);
+    unsigned highBits = storageSize - offset - size;
+    val = createShL(rewriter, val, highBits);
+    val = createAShR(rewriter, val, offset + highBits);
+  } else {
+    val = createLShR(rewriter, val, offset);
+
+    if (static_cast<unsigned>(offset) + size < storageSize)
+      val = createAnd(rewriter, val,
+                      llvm::APInt::getLowBitsSet(storageSize, size));
+  }
+
+  mlir::Type resTy = getTypeConverter()->convertType(op.getType());
+  mlir::Value newOp = createIntCast(
+      rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
+  rewriter.replaceOp(op, newOp);
+  return mlir::success();
+}
+
 std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
   return std::make_unique<ConvertCIRToLLVMPass>();
 }
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index ae7247332c66..d9fb91066317 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -29,7 +29,7 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, mlir::Attribute attr,
 
 mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage);
 
-void convertSideEffectForCall(mlir::Operation *callOp,
+void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
                               cir::SideEffect sideEffect,
                               mlir::LLVM::MemoryEffectsAttr &memoryEffect,
                               bool &noUnwind, bool &willReturn);
@@ -44,6 +44,56 @@ class CIRToLLVMAssumeOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMBitClrsbOpLowering
+    : public mlir::OpConversionPattern<cir::BitClrsbOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitClrsbOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitClrsbOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitClzOpLowering
+    : public mlir::OpConversionPattern<cir::BitClzOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitClzOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitClzOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitCtzOpLowering
+    : public mlir::OpConversionPattern<cir::BitCtzOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitCtzOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitCtzOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitParityOpLowering
+    : public mlir::OpConversionPattern<cir::BitParityOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitParityOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitParityOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMBitPopcountOpLowering
+    : public mlir::OpConversionPattern<cir::BitPopcountOp> {
+public:
+  using mlir::OpConversionPattern<cir::BitPopcountOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::BitPopcountOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMBrCondOpLowering
     : public mlir::OpConversionPattern<cir::BrCondOp> {
 public:
@@ -70,6 +120,16 @@ class CIRToLLVMCastOpLowering : public mlir::OpConversionPattern<cir::CastOp> {
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMExpectOpLowering
+    : public mlir::OpConversionPattern<cir::ExpectOp> {
+public:
+  using mlir::OpConversionPattern<cir::ExpectOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ExpectOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMReturnOpLowering
     : public mlir::OpConversionPattern<cir::ReturnOp> {
 public:
@@ -433,6 +493,36 @@ class CIRToLLVMComplexCreateOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMComplexRealOpLowering
+    : public mlir::OpConversionPattern<cir::ComplexRealOp> {
+public:
+  using mlir::OpConversionPattern<cir::ComplexRealOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ComplexRealOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMComplexImagOpLowering
+    : public mlir::OpConversionPattern<cir::ComplexImagOp> {
+public:
+  using mlir::OpConversionPattern<cir::ComplexImagOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ComplexImagOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMGetBitfieldOpLowering
+    : public mlir::OpConversionPattern<cir::GetBitfieldOp> {
+public:
+  using mlir::OpConversionPattern<cir::GetBitfieldOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::GetBitfieldOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 } // namespace direct
 } // namespace cir
 
diff --git a/clang/lib/CIR/Lowering/LoweringHelpers.cpp b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
index b4ae9f1909f4..d5f132431aba 100644
--- a/clang/lib/CIR/Lowering/LoweringHelpers.cpp
+++ b/clang/lib/CIR/Lowering/LoweringHelpers.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/CIR/LoweringHelpers.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "clang/CIR/MissingFeatures.h"
 
 mlir::DenseElementsAttr
@@ -138,9 +139,46 @@ lowerConstArrayAttr(cir::ConstArrayAttr constArr,
     return convertToDenseElementsAttr<cir::IntAttr, mlir::APInt>(
         constArr, dims, type, converter->convertType(type));
 
-  if (mlir::isa<cir::CIRFPTypeInterface>(type))
+  if (mlir::isa<cir::FPTypeInterface>(type))
     return convertToDenseElementsAttr<cir::FPAttr, mlir::APFloat>(
         constArr, dims, type, converter->convertType(type));
 
   return std::nullopt;
 }
+
+mlir::Value getConstAPInt(mlir::OpBuilder &bld, mlir::Location loc,
+                          mlir::Type typ, const llvm::APInt &val) {
+  return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val);
+}
+
+mlir::Value getConst(mlir::OpBuilder &bld, mlir::Location loc, mlir::Type typ,
+                     unsigned val) {
+  return bld.create<mlir::LLVM::ConstantOp>(loc, typ, val);
+}
+
+mlir::Value createShL(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::ShlOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createAShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::AShrOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createAnd(mlir::OpBuilder &bld, mlir::Value lhs,
+                      const llvm::APInt &rhs) {
+  mlir::Value rhsVal = getConstAPInt(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::AndOp>(lhs.getLoc(), lhs, rhsVal);
+}
+
+mlir::Value createLShR(mlir::OpBuilder &bld, mlir::Value lhs, unsigned rhs) {
+  if (!rhs)
+    return lhs;
+  mlir::Value rhsVal = getConst(bld, lhs.getLoc(), lhs.getType(), rhs);
+  return bld.create<mlir::LLVM::LShrOp>(lhs.getLoc(), lhs, rhsVal);
+}
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 7e0a3cf5591c..2f6d4c414e73 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Frontend/Driver/CodeGenOptions.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
@@ -43,6 +44,7 @@
 #include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Program.h"
@@ -96,8 +98,6 @@ using namespace llvm;
 #include "llvm/Support/Extension.def"
 
 namespace llvm {
-extern cl::opt<bool> PrintPipelinePasses;
-
 // Experiment to move sanitizers earlier.
 static cl::opt<bool> ClSanitizeOnOptimizerEarlyEP(
     "sanitizer-early-opt-ep", cl::Optional,
@@ -118,7 +118,8 @@ static cl::opt<PGOOptions::ColdFuncOpt> ClPGOColdFuncAttr(
                clEnumValN(PGOOptions::ColdFuncOpt::OptNone, "optnone",
                           "Mark cold functions with optnone.")));
 
-extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
+LLVM_ABI extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind>
+    ProfileCorrelate;
 } // namespace llvm
 namespace clang {
 extern llvm::cl::opt<bool> ClSanitizeGuardChecks;
@@ -805,17 +806,21 @@ static void addSanitizers(const Triple &TargetTriple,
   // SanitizeSkipHotCutoffs: doubles with range [0, 1]
   // Opts.cutoffs: unsigned ints with range [0, 1000000]
   auto ScaledCutoffs = CodeGenOpts.SanitizeSkipHotCutoffs.getAllScaled(1000000);
-
+  uint64_t AllowRuntimeCheckSkipHotCutoff =
+      CodeGenOpts.AllowRuntimeCheckSkipHotCutoff.value_or(0.0) * 1000000;
   // TODO: remove IsRequested()
-  if (LowerAllowCheckPass::IsRequested() || ScaledCutoffs.has_value()) {
+  if (LowerAllowCheckPass::IsRequested() || ScaledCutoffs.has_value() ||
+      CodeGenOpts.AllowRuntimeCheckSkipHotCutoff.has_value()) {
     // We want to call it after inline, which is about OptimizerEarlyEPCallback.
     PB.registerOptimizerEarlyEPCallback(
-        [ScaledCutoffs](ModulePassManager &MPM, OptimizationLevel Level,
-                        ThinOrFullLTOPhase Phase) {
+        [ScaledCutoffs, AllowRuntimeCheckSkipHotCutoff](
+            ModulePassManager &MPM, OptimizationLevel Level,
+            ThinOrFullLTOPhase Phase) {
           LowerAllowCheckPass::Options Opts;
           // TODO: after removing IsRequested(), make this unconditional
           if (ScaledCutoffs.has_value())
             Opts.cutoffs = ScaledCutoffs.value();
+          Opts.runtime_check = AllowRuntimeCheckSkipHotCutoff;
           MPM.addPass(
               createModuleToFunctionPassAdaptor(LowerAllowCheckPass(Opts)));
         });
@@ -957,7 +962,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
           CodeGenOpts.DIBugsReportFilePath);
     Debugify.registerCallbacks(PIC, MAM);
 
-#if LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     // If we're using debug location coverage tracking, mark all the
     // instructions coming out of the frontend without a DebugLoc as being
     // compiler-generated, to prevent both those instructions and new
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index 51f0799a792f..9106c4cd8e13 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -411,7 +411,9 @@ static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
 
   CGF.Builder.SetInsertPoint(StoreExpectedBB);
   // Update the memory at Expected with Old's value.
-  CGF.Builder.CreateStore(Old, Val1);
+  auto *I = CGF.Builder.CreateStore(Old, Val1);
+  CGF.addInstToCurrentSourceAtom(I, Old);
+
   // Finally, branch to the exit point.
   CGF.Builder.CreateBr(ContinueBB);
 
@@ -591,7 +593,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
     Load->setAtomic(Order, Scope);
     Load->setVolatile(E->isVolatile());
     CGF.maybeAttachRangeForLoad(Load, E->getValueType(), E->getExprLoc());
-    CGF.Builder.CreateStore(Load, Dest);
+    auto *I = CGF.Builder.CreateStore(Load, Dest);
+    CGF.addInstToCurrentSourceAtom(I, Load);
     return;
   }
 
@@ -606,6 +609,7 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
     llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
     Store->setAtomic(Order, Scope);
     Store->setVolatile(E->isVolatile());
+    CGF.addInstToCurrentSourceAtom(Store, LoadVal1);
     return;
   }
 
@@ -731,7 +735,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
                               CGF.Builder.getInt8(1), Order, Scope, E);
     RMWI->setVolatile(E->isVolatile());
     llvm::Value *Result = CGF.Builder.CreateIsNotNull(RMWI, "tobool");
-    CGF.Builder.CreateStore(Result, Dest);
+    auto *I = CGF.Builder.CreateStore(Result, Dest);
+    CGF.addInstToCurrentSourceAtom(I, Result);
     return;
   }
 
@@ -740,6 +745,7 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
         CGF.Builder.CreateStore(CGF.Builder.getInt8(0), Ptr);
     Store->setAtomic(Order, Scope);
     Store->setVolatile(E->isVolatile());
+    CGF.addInstToCurrentSourceAtom(Store, nullptr);
     return;
   }
   }
@@ -762,7 +768,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
   if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
       E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
     Result = CGF.Builder.CreateNot(Result);
-  CGF.Builder.CreateStore(Result, Dest);
+  auto *I = CGF.Builder.CreateStore(Result, Dest);
+  CGF.addInstToCurrentSourceAtom(I, Result);
 }
 
 // This function emits any expression (scalar, complex, or aggregate)
@@ -845,6 +852,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
 }
 
 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
+  ApplyAtomGroup Grp(getDebugInfo());
+
   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
   QualType MemTy = AtomicTy;
   if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index f7016d87b6e7..caaa9ce156dd 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -1589,7 +1589,7 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
   case Builtin::BI_interlockedbittestandset:
     return {Set, Sequential, false};
 
-    // X86-specific 64-bit variants.
+    // 64-bit variants.
   case Builtin::BI_bittest64:
     return {TestOnly, Unlocked, true};
   case Builtin::BI_bittestandcomplement64:
@@ -1616,6 +1616,18 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
     return {Reset, Release, false};
   case Builtin::BI_interlockedbittestandreset_nf:
     return {Reset, NoFence, false};
+  case Builtin::BI_interlockedbittestandreset64_acq:
+    return {Reset, Acquire, false};
+  case Builtin::BI_interlockedbittestandreset64_rel:
+    return {Reset, Release, false};
+  case Builtin::BI_interlockedbittestandreset64_nf:
+    return {Reset, NoFence, false};
+  case Builtin::BI_interlockedbittestandset64_acq:
+    return {Set, Acquire, false};
+  case Builtin::BI_interlockedbittestandset64_rel:
+    return {Set, Release, false};
+  case Builtin::BI_interlockedbittestandset64_nf:
+    return {Set, NoFence, false};
   }
   llvm_unreachable("expected only bittest intrinsics");
 }
@@ -2356,7 +2368,7 @@ EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
   llvm::Type *OpTy = Signed->getType();
   llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
   Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
-  llvm::Type *ResTy = ResultPtr.getElementType();
+  llvm::Type *ResTy = CGF.getTypes().ConvertType(ResultQTy);
   unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
 
   // Take the absolute value of the signed operand.
@@ -5538,7 +5550,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI_bittestandset:
   case Builtin::BI_interlockedbittestandreset:
   case Builtin::BI_interlockedbittestandreset64:
+  case Builtin::BI_interlockedbittestandreset64_acq:
+  case Builtin::BI_interlockedbittestandreset64_rel:
+  case Builtin::BI_interlockedbittestandreset64_nf:
   case Builtin::BI_interlockedbittestandset64:
+  case Builtin::BI_interlockedbittestandset64_acq:
+  case Builtin::BI_interlockedbittestandset64_rel:
+  case Builtin::BI_interlockedbittestandset64_nf:
   case Builtin::BI_interlockedbittestandset:
   case Builtin::BI_interlockedbittestandset_acq:
   case Builtin::BI_interlockedbittestandset_rel:
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e455e29f3fbd..bb98a3307349 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2661,6 +2661,13 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
     // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
     // handles these separately to set them based on the global defaults.
     GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
+
+    // Windows hotpatching support
+    if (!MSHotPatchFunctions.empty()) {
+      bool IsHotPatched = llvm::binary_search(MSHotPatchFunctions, Name);
+      if (IsHotPatched)
+        FuncAttrs.addAttribute("marked_for_windows_hot_patching");
+    }
   }
 
   // Mark functions that are replaceable by the loader.
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 8d1e57ab2aaf..879c56373658 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -626,6 +626,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   CXXCtorInitializer *MemberInit,
                                   const CXXConstructorDecl *Constructor,
                                   FunctionArgList &Args) {
+  ApplyAtomGroup Grp(CGF.getDebugInfo());
   ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
   assert(MemberInit->isAnyMemberInitializer() &&
          "Must have member initializer!");
@@ -999,7 +1000,8 @@ namespace {
     void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
       DestPtr = DestPtr.withElementType(CGF.Int8Ty);
       SrcPtr = SrcPtr.withElementType(CGF.Int8Ty);
-      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
+      auto *I = CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
+      CGF.addInstToCurrentSourceAtom(I, nullptr);
     }
 
     void addInitialField(FieldDecl *F) {
@@ -1112,6 +1114,7 @@ namespace {
       }
 
       pushEHDestructors();
+      ApplyAtomGroup Grp(CGF.getDebugInfo());
       emitMemcpy();
       AggregatedInits.clear();
     }
@@ -1247,6 +1250,7 @@ namespace {
         reset();
       }
 
+      ApplyAtomGroup Grp(CGF.getDebugInfo());
       emitMemcpy();
       AggregatedStmts.clear();
     }
@@ -1337,9 +1341,9 @@ void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
     assert(!Member->isBaseInitializer());
     assert(Member->isAnyMemberInitializer() &&
            "Delegating initializer on non-delegating constructor");
-    ApplyAtomGroup Grp(getDebugInfo());
     CM.addMemberInitializer(Member);
   }
+
   CM.finish();
 }
 
@@ -1562,6 +1566,7 @@ void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args)
   AssignmentMemcpyizer AM(*this, AssignOp, Args);
   for (auto *I : RootCS->body())
     AM.emitAssignment(I);
+
   AM.finish();
 }
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index b446114ce545..0ba18558b581 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -891,8 +891,8 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
     auto *ISATy = DBuilder.createPointerType(ClassTy, Size);
 
     ObjTy = DBuilder.createStructType(TheCU, "objc_object", TheCU->getFile(), 0,
-                                      0, 0, llvm::DINode::FlagZero, nullptr,
-                                      llvm::DINodeArray());
+                                      (uint64_t)0, 0, llvm::DINode::FlagZero,
+                                      nullptr, llvm::DINodeArray());
 
     DBuilder.replaceArrays(
         ObjTy, DBuilder.getOrCreateArray(&*DBuilder.createMemberType(
@@ -1079,6 +1079,13 @@ llvm::DIType *CGDebugInfo::CreateType(const BuiltinType *BT) {
           DBuilder.createBasicType(Name, Width, llvm::dwarf::DW_ATE_unsigned); \
     return SingletonId;                                                        \
   }
+#define AMDGPU_FEATURE_PREDICATE_TYPE(Name, Id, SingletonId, Width, Align)     \
+  case BuiltinType::Id: {                                                      \
+    if (!SingletonId)                                                          \
+      SingletonId =                                                            \
+          DBuilder.createBasicType(Name, Width, llvm::dwarf::DW_ATE_boolean);  \
+    return SingletonId;                                                        \
+  }
 #include "clang/Basic/AMDGPUTypes.def"
   case BuiltinType::UChar:
   case BuiltinType::Char_U:
@@ -1955,7 +1962,9 @@ CGDebugInfo::createInlinedSubprogram(StringRef FuncName,
         /*ScopeLine=*/0,
         /*Flags=*/llvm::DINode::FlagArtificial,
         /*SPFlags=*/llvm::DISubprogram::SPFlagDefinition,
-        /*TParams=*/nullptr, /*ThrownTypes=*/nullptr, /*Annotations=*/nullptr);
+        /*TParams=*/nullptr, /*Decl=*/nullptr, /*ThrownTypes=*/nullptr,
+        /*Annotations=*/nullptr, /*TargetFuncName=*/StringRef(),
+        /*UseKeyInstructions=*/CGM.getCodeGenOpts().DebugKeyInstructions);
   }
 
   return SP;
@@ -2157,8 +2166,17 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
   return getOrCreateInstanceMethodType(ThisType, Func, Unit);
 }
 
-llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
-    QualType ThisPtr, const FunctionProtoType *Func, llvm::DIFile *Unit) {
+llvm::DISubroutineType *CGDebugInfo::getOrCreateMethodTypeForDestructor(
+    const CXXMethodDecl *Method, llvm::DIFile *Unit, QualType FNType) {
+  const FunctionProtoType *Func = FNType->getAs<FunctionProtoType>();
+  // skip the first param since it is also this
+  return getOrCreateInstanceMethodType(Method->getThisType(), Func, Unit, true);
+}
+
+llvm::DISubroutineType *
+CGDebugInfo::getOrCreateInstanceMethodType(QualType ThisPtr,
+                                           const FunctionProtoType *Func,
+                                           llvm::DIFile *Unit, bool SkipFirst) {
   FunctionProtoType::ExtProtoInfo EPI = Func->getExtProtoInfo();
   Qualifiers &Qc = EPI.TypeQuals;
   Qc.removeConst();
@@ -2198,7 +2216,7 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateInstanceMethodType(
   }
 
   // Copy rest of the arguments.
-  for (unsigned i = 1, e = Args.size(); i != e; ++i)
+  for (unsigned i = (SkipFirst ? 2 : 1), e = Args.size(); i < e; ++i)
     Elts.push_back(Args[i]);
 
   // Attach FlagObjectPointer to the explicit "this" parameter.
@@ -2335,7 +2353,8 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
   llvm::DISubprogram *SP = DBuilder.createMethod(
       RecordTy, MethodName, MethodLinkageName, MethodDefUnit, MethodLine,
       MethodTy, VIndex, ThisAdjustment, ContainingType, Flags, SPFlags,
-      TParamsArray.get());
+      TParamsArray.get(), /*ThrownTypes*/ nullptr,
+      CGM.getCodeGenOpts().DebugKeyInstructions);
 
   SPCache[Method->getCanonicalDecl()].reset(SP);
 
@@ -4403,7 +4422,9 @@ llvm::DISubprogram *CGDebugInfo::getFunctionFwdDeclOrStub(GlobalDecl GD,
     return DBuilder.createFunction(
         DContext, Name, LinkageName, Unit, Line,
         getOrCreateFunctionType(GD.getDecl(), FnType, Unit), 0, Flags, SPFlags,
-        TParamsArray.get(), getFunctionDeclaration(FD));
+        TParamsArray.get(), getFunctionDeclaration(FD), /*ThrownTypes*/ nullptr,
+        /*Annotations*/ nullptr, /*TargetFuncName*/ "",
+        CGM.getCodeGenOpts().DebugKeyInstructions);
   }
 
   llvm::DISubprogram *SP = DBuilder.createTempFunctionFwdDecl(
@@ -4571,6 +4592,12 @@ llvm::DISubroutineType *CGDebugInfo::getOrCreateFunctionType(const Decl *D,
     // subprogram DIE will miss DW_AT_decl_file and DW_AT_decl_line fields.
     return DBuilder.createSubroutineType(DBuilder.getOrCreateTypeArray({}));
 
+  if (const auto *Method = dyn_cast<CXXDestructorDecl>(D)) {
+    // Read method type from 'FnType' because 'D.getType()' does not cover
+    // implicit arguments for destructors.
+    return getOrCreateMethodTypeForDestructor(Method, F, FnType);
+  }
+
   if (const auto *Method = dyn_cast<CXXMethodDecl>(D))
     return getOrCreateMethodType(Method, F);
 
@@ -4740,8 +4767,9 @@ void CGDebugInfo::emitFunctionStart(GlobalDecl GD, SourceLocation Loc,
   llvm::DISubprogram *SP = DBuilder.createFunction(
       FDContext, Name, LinkageName, Unit, LineNo, DIFnType, ScopeLine,
       FlagsForDef, SPFlagsForDef, TParamsArray.get(), Decl, nullptr,
-      Annotations);
+      Annotations, "", CGM.getCodeGenOpts().DebugKeyInstructions);
   Fn->setSubprogram(SP);
+
   // We might get here with a VarDecl in the case we're generating
   // code for the initialization of globals. Do not record these decls
   // as they will overwrite the actual VarDecl Decl in the cache.
@@ -4800,9 +4828,12 @@ void CGDebugInfo::EmitFunctionDecl(GlobalDecl GD, SourceLocation Loc,
 
   llvm::DINodeArray Annotations = CollectBTFDeclTagAnnotations(D);
   llvm::DISubroutineType *STy = getOrCreateFunctionType(D, FnType, Unit);
+  // Key Instructions: Don't set flag on declarations.
+  assert(~SPFlags & llvm::DISubprogram::SPFlagDefinition);
   llvm::DISubprogram *SP = DBuilder.createFunction(
       FDContext, Name, LinkageName, Unit, LineNo, STy, ScopeLine, Flags,
-      SPFlags, TParamsArray.get(), nullptr, nullptr, Annotations);
+      SPFlags, TParamsArray.get(), nullptr, nullptr, Annotations,
+      /*TargetFunctionName*/ "", /*UseKeyInstructions*/ false);
 
   // Preserve btf_decl_tag attributes for parameters of extern functions
   // for BPF target. The parameters created in this loop are attached as
@@ -5639,8 +5670,9 @@ void CGDebugInfo::EmitLabel(const LabelDecl *D, CGBuilderTy &Builder) {
   StringRef Name = D->getName();
 
   // Create the descriptor for the label.
-  auto *L =
-      DBuilder.createLabel(Scope, Name, Unit, Line, CGM.getLangOpts().Optimize);
+  auto *L = DBuilder.createLabel(
+      Scope, Name, Unit, Line, Column, /*IsArtificial=*/false,
+      /*CoroSuspendIdx=*/std::nullopt, CGM.getLangOpts().Optimize);
 
   // Insert an llvm.dbg.label into the current block.
   DBuilder.insertLabel(L,
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 6a1b0f6f361c..428e151c45a5 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -263,9 +263,14 @@ class CGDebugInfo {
   /// to get a method type which includes \c this pointer.
   llvm::DISubroutineType *getOrCreateMethodType(const CXXMethodDecl *Method,
                                                 llvm::DIFile *F);
+
+  llvm::DISubroutineType *
+  getOrCreateMethodTypeForDestructor(const CXXMethodDecl *Method,
+                                     llvm::DIFile *F, QualType FNType);
+
   llvm::DISubroutineType *
   getOrCreateInstanceMethodType(QualType ThisPtr, const FunctionProtoType *Func,
-                                llvm::DIFile *Unit);
+                                llvm::DIFile *Unit, bool SkipFirst = false);
   llvm::DISubroutineType *
   getOrCreateFunctionType(const Decl *D, QualType FnType, llvm::DIFile *F);
   /// \return debug info descriptor for vtable.
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 193710bef2d1..631315c9bdf3 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -982,6 +982,10 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
   if (const MemberPointerType *MPT = dyn_cast<MemberPointerType>(SrcType))
     return CGF.CGM.getCXXABI().EmitMemberPointerIsNotNull(CGF, Src, MPT);
 
+  // The conversion is a NOP, and will be done when CodeGening the builtin.
+  if (SrcType == CGF.getContext().AMDGPUFeaturePredicateTy)
+    return Src;
+
   assert((SrcType->isIntegerType() || isa<llvm::PointerType>(Src->getType())) &&
          "Unknown scalar type to convert");
 
@@ -2440,7 +2444,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_BlockPointerToObjCPointerCast:
   case CK_AnyPointerToBlockPointerCast:
   case CK_BitCast: {
-    Value *Src = Visit(const_cast<Expr*>(E));
+    Value *Src = Visit(E);
     llvm::Type *SrcTy = Src->getType();
     llvm::Type *DstTy = ConvertType(DestTy);
 
@@ -2606,11 +2610,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_AtomicToNonAtomic:
   case CK_NonAtomicToAtomic:
   case CK_UserDefinedConversion:
-    return Visit(const_cast<Expr*>(E));
+    return Visit(E);
 
   case CK_NoOp: {
-    return CE->changesVolatileQualification() ? EmitLoadOfLValue(CE)
-                                              : Visit(const_cast<Expr *>(E));
+    return CE->changesVolatileQualification() ? EmitLoadOfLValue(CE) : Visit(E);
   }
 
   case CK_BaseToDerived: {
@@ -2712,10 +2715,10 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_LValueToRValue:
     assert(CGF.getContext().hasSameUnqualifiedType(E->getType(), DestTy));
     assert(E->isGLValue() && "lvalue-to-rvalue applied to r-value!");
-    return Visit(const_cast<Expr*>(E));
+    return Visit(E);
 
   case CK_IntegralToPointer: {
-    Value *Src = Visit(const_cast<Expr*>(E));
+    Value *Src = Visit(E);
 
     // First, convert to the correct width so that we control the kind of
     // extension.
@@ -2768,7 +2771,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_HLSLAggregateSplatCast:
   case CK_VectorSplat: {
     llvm::Type *DstTy = ConvertType(DestTy);
-    Value *Elt = Visit(const_cast<Expr *>(E));
+    Value *Elt = Visit(E);
     // Splat the element across to all elements
     llvm::ElementCount NumElements =
         cast<llvm::VectorType>(DstTy)->getElementCount();
@@ -2906,7 +2909,7 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   case CK_HLSLVectorTruncation: {
     assert((DestTy->isVectorType() || DestTy->isBuiltinType()) &&
            "Destination type must be a vector or builtin type.");
-    Value *Vec = Visit(const_cast<Expr *>(E));
+    Value *Vec = Visit(E);
     if (auto *VecTy = DestTy->getAs<VectorType>()) {
       SmallVector<int> Mask;
       unsigned NumElts = VecTy->getNumElements();
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 3103f1798e14..c1fd6f22f693 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -23,7 +23,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/TargetOptions.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalVariable.h"
@@ -66,17 +66,16 @@ void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
   DXILValMD->addOperand(Val);
 }
 
-void addRootSignature(ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
+void addRootSignature(llvm::dxbc::RootSignatureVersion RootSigVer,
+                      ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
                       llvm::Function *Fn, llvm::Module &M) {
   auto &Ctx = M.getContext();
 
-  llvm::hlsl::rootsig::MetadataBuilder Builder(Ctx, Elements);
-  MDNode *RootSignature = Builder.BuildRootSignature();
+  llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);
+  MDNode *RootSignature = RSBuilder.BuildRootSignature();
 
-  // TODO: We need to wire the root signature version up through the frontend
-  // rather than hardcoding it.
-  ConstantAsMetadata *Version =
-      ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 2));
+  ConstantAsMetadata *Version = ConstantAsMetadata::get(ConstantInt::get(
+      llvm::Type::getInt32Ty(Ctx), llvm::to_underlying(RootSigVer)));
   MDNode *MDVals =
       MDNode::get(Ctx, {ValueAsMetadata::get(Fn), RootSignature, Version});
 
@@ -471,9 +470,11 @@ void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
 
   // Add and identify root signature to function, if applicable
   for (const Attr *Attr : FD->getAttrs()) {
-    if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Attr))
-      addRootSignature(RSAttr->getSignatureDecl()->getRootElements(), EntryFn,
+    if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Attr)) {
+      auto *RSDecl = RSAttr->getSignatureDecl();
+      addRootSignature(RSDecl->getVersion(), RSDecl->getRootElements(), EntryFn,
                        M);
+    }
   }
 }
 
@@ -585,12 +586,12 @@ static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
 void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
                                                 llvm::GlobalVariable *GV,
                                                 HLSLResourceBindingAttr *RBA) {
+  assert(RBA && "expect a nonnull binding attribute");
   llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGM.getLLVMContext());
   auto *NonUniform = llvm::ConstantInt::get(Int1Ty, false);
   auto *Index = llvm::ConstantInt::get(CGM.IntTy, 0);
   auto *RangeSize = llvm::ConstantInt::get(CGM.IntTy, 1);
-  auto *Space =
-      llvm::ConstantInt::get(CGM.IntTy, RBA ? RBA->getSpaceNumber() : 0);
+  auto *Space = llvm::ConstantInt::get(CGM.IntTy, RBA->getSpaceNumber());
   Value *Name = nullptr;
 
   llvm::Intrinsic::ID IntrinsicID =
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 8091e4e9e1b0..04ae5cf070f5 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1477,9 +1477,9 @@ void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
     if (!NumThreadsVal)
       NumThreadsVal = llvm::ConstantInt::get(CGF.Int32Ty, -1);
     else
-      NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty),
+      NumThreadsVal = Bld.CreateZExtOrTrunc(NumThreadsVal, CGF.Int32Ty);
 
-      assert(IfCondVal && "Expected a value");
+    assert(IfCondVal && "Expected a value");
     llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
     if (CGM.getLangOpts().OpenMPNoNestedParallelism &&
         CGM.IsSPMDExecutionMode()) {
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index d0ec866e448d..ef09b19272d1 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1065,7 +1065,7 @@ llvm::Function *CodeGenFunction::GenerateOpenMPCapturedStmtFunction(
       CapturedStmtInfo->EmitBody(*this, CD->getBody());
   }
 
-  (void)LocalScope.ForceCleanup();
+  LocalScope.ForceCleanup();
   FinishFunction(CD->getBodyRBrace());
   if (!NeedWrapperFunction)
     return F;
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 2897ccdf8866..0b6e830e0d55 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -1138,7 +1138,9 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) {
                  llvm::Function::InternalLinkage;
 
       case TSK_ExplicitInstantiationDeclaration:
-        llvm_unreachable("Should not have been asked to emit this");
+        return IsExternalDefinition
+                   ? llvm::GlobalVariable::AvailableExternallyLinkage
+                   : llvm::GlobalVariable::ExternalLinkage;
       }
   }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 70a09795d02f..776a646ceb32 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -1109,10 +1109,16 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
 
   // Add vscale_range attribute if appropriate.
   llvm::StringMap<bool> FeatureMap;
-  bool IsArmStreaming = false;
+  auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;
   if (FD) {
     getContext().getFunctionFeatureMap(FeatureMap, FD);
-    IsArmStreaming = IsArmStreamingFunction(FD, true);
+    if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
+      if (T->getAArch64SMEAttributes() &
+          FunctionType::SME_PStateSMCompatibleMask)
+        IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;
+
+    if (IsArmStreamingFunction(FD, true))
+      IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;
   }
   std::optional<std::pair<unsigned, unsigned>> VScaleRange =
       getContext().getTargetInfo().getVScaleRange(getLangOpts(), IsArmStreaming,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 90909034cc7d..aa6afcfcf4ea 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -236,6 +236,9 @@ class ApplyAtomGroup {
   uint64_t OriginalAtom = 0;
   CGDebugInfo *DI = nullptr;
 
+  ApplyAtomGroup(const ApplyAtomGroup &) = delete;
+  void operator=(const ApplyAtomGroup &) = delete;
+
 public:
   ApplyAtomGroup(CGDebugInfo *DI);
   ~ApplyAtomGroup();
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 79d0e218f35d..ef2cb68ce632 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -336,6 +336,72 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   return *TheTargetCodeGenInfo;
 }
 
+static void checkDataLayoutConsistency(const TargetInfo &Target,
+                                       llvm::LLVMContext &Context,
+                                       const LangOptions &Opts) {
+#ifndef NDEBUG
+  // Don't verify non-standard ABI configurations.
+  if (Opts.AlignDouble || Opts.OpenCL || Opts.HLSL)
+    return;
+
+  llvm::Triple Triple = Target.getTriple();
+  llvm::DataLayout DL(Target.getDataLayoutString());
+  auto Check = [&](const char *Name, llvm::Type *Ty, unsigned Alignment) {
+    llvm::Align DLAlign = DL.getABITypeAlign(Ty);
+    llvm::Align ClangAlign(Alignment / 8);
+    if (DLAlign != ClangAlign) {
+      llvm::errs() << "For target " << Triple.str() << " type " << Name
+                   << " mapping to " << *Ty << " has data layout alignment "
+                   << DLAlign.value() << " while clang specifies "
+                   << ClangAlign.value() << "\n";
+      abort();
+    }
+  };
+
+  Check("bool", llvm::Type::getIntNTy(Context, Target.BoolWidth),
+        Target.BoolAlign);
+  Check("short", llvm::Type::getIntNTy(Context, Target.ShortWidth),
+        Target.ShortAlign);
+  Check("int", llvm::Type::getIntNTy(Context, Target.IntWidth),
+        Target.IntAlign);
+  Check("long", llvm::Type::getIntNTy(Context, Target.LongWidth),
+        Target.LongAlign);
+  // FIXME: M68k specifies incorrect long long alignment in both LLVM and Clang.
+  if (Triple.getArch() != llvm::Triple::m68k)
+    Check("long long", llvm::Type::getIntNTy(Context, Target.LongLongWidth),
+          Target.LongLongAlign);
+  // FIXME: There are int128 alignment mismatches on multiple targets.
+  if (Target.hasInt128Type() && !Target.getTargetOpts().ForceEnableInt128 &&
+      !Triple.isAMDGPU() && !Triple.isSPIRV() &&
+      Triple.getArch() != llvm::Triple::ve)
+    Check("__int128", llvm::Type::getIntNTy(Context, 128), Target.Int128Align);
+
+  if (Target.hasFloat16Type())
+    Check("half", llvm::Type::getFloatingPointTy(Context, *Target.HalfFormat),
+          Target.HalfAlign);
+  if (Target.hasBFloat16Type())
+    Check("bfloat", llvm::Type::getBFloatTy(Context), Target.BFloat16Align);
+  Check("float", llvm::Type::getFloatingPointTy(Context, *Target.FloatFormat),
+        Target.FloatAlign);
+  // FIXME: AIX specifies wrong double alignment in DataLayout
+  if (!Triple.isOSAIX()) {
+    Check("double",
+          llvm::Type::getFloatingPointTy(Context, *Target.DoubleFormat),
+          Target.DoubleAlign);
+    Check("long double",
+          llvm::Type::getFloatingPointTy(Context, *Target.LongDoubleFormat),
+          Target.LongDoubleAlign);
+  }
+  // FIXME: Wasm has a mismatch in f128 alignment between Clang and LLVM.
+  if (Target.hasFloat128Type() && !Triple.isWasm())
+    Check("__float128", llvm::Type::getFP128Ty(Context), Target.Float128Align);
+  if (Target.hasIbm128Type())
+    Check("__ibm128", llvm::Type::getPPC_FP128Ty(Context), Target.Ibm128Align);
+
+  Check("void*", llvm::PointerType::getUnqual(Context), Target.PointerAlign);
+#endif
+}
+
 CodeGenModule::CodeGenModule(ASTContext &C,
                              IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
                              const HeaderSearchOptions &HSO,
@@ -462,6 +528,38 @@ CodeGenModule::CodeGenModule(ASTContext &C,
   if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
     getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters",
                               CodeGenOpts.NumRegisterParameters);
+
+  // If there are any functions that are marked for Windows secure hot-patching,
+  // then build the list of functions now.
+  if (!CGO.MSSecureHotPatchFunctionsFile.empty() ||
+      !CGO.MSSecureHotPatchFunctionsList.empty()) {
+    if (!CGO.MSSecureHotPatchFunctionsFile.empty()) {
+      auto BufOrErr =
+          llvm::MemoryBuffer::getFile(CGO.MSSecureHotPatchFunctionsFile);
+      if (BufOrErr) {
+        const llvm::MemoryBuffer &FileBuffer = **BufOrErr;
+        for (llvm::line_iterator I(FileBuffer.getMemBufferRef(), true), E;
+             I != E; ++I)
+          this->MSHotPatchFunctions.push_back(std::string{*I});
+      } else {
+        auto &DE = Context.getDiagnostics();
+        unsigned DiagID =
+            DE.getCustomDiagID(DiagnosticsEngine::Error,
+                               "failed to open hotpatch functions file "
+                               "(-fms-hotpatch-functions-file): %0 : %1");
+        DE.Report(DiagID) << CGO.MSSecureHotPatchFunctionsFile
+                          << BufOrErr.getError().message();
+      }
+    }
+
+    for (const auto &FuncName : CGO.MSSecureHotPatchFunctionsList)
+      this->MSHotPatchFunctions.push_back(FuncName);
+
+    llvm::sort(this->MSHotPatchFunctions);
+  }
+
+  if (!Context.getAuxTargetInfo())
+    checkDataLayoutConsistency(Context.getTargetInfo(), LLVMContext, LangOpts);
 }
 
 CodeGenModule::~CodeGenModule() {}
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 38659b949fcf..ef05fc84f06f 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -861,6 +861,11 @@ class CodeGenModule : public CodeGenTypeCache {
 
   AtomicOptions AtomicOpts;
 
+  // A set of functions which should be hot-patched; see
+  // -fms-hotpatch-functions-file (and -list). This will nearly always be empty.
+  // The list is sorted for binary-searching.
+  std::vector<std::string> MSHotPatchFunctions;
+
 public:
   CodeGenModule(ASTContext &C, IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
                 const HeaderSearchOptions &headersearchopts,
diff --git a/clang/lib/CodeGen/CodeGenPGO.cpp b/clang/lib/CodeGen/CodeGenPGO.cpp
index a80bebbb4cf4..98b30e084b18 100644
--- a/clang/lib/CodeGen/CodeGenPGO.cpp
+++ b/clang/lib/CodeGen/CodeGenPGO.cpp
@@ -1423,8 +1423,7 @@ void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
                                   bool IsInMainFile) {
   CGM.getPGOStats().addVisited(IsInMainFile);
   RegionCounts.clear();
-  llvm::Expected<llvm::InstrProfRecord> RecordExpected =
-      PGOReader->getInstrProfRecord(FuncName, FunctionHash);
+  auto RecordExpected = PGOReader->getInstrProfRecord(FuncName, FunctionHash);
   if (auto E = RecordExpected.takeError()) {
     auto IPE = std::get<0>(llvm::InstrProfError::take(std::move(E)));
     if (IPE == llvm::instrprof_error::unknown_function)
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c98503e4bbd2..53bb6e7b8514 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -580,6 +580,9 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
   case BuiltinType::Id:                                                        \
     return llvm::TargetExtType::get(getLLVMContext(), "amdgcn.named.barrier",  \
                                     {}, {Scope});
+#define AMDGPU_FEATURE_PREDICATE_TYPE(Name, Id, SingletonId, Width, Align)     \
+  case BuiltinType::Id:                                                        \
+    return ConvertType(getContext().getLogicalOperationType());
 #include "clang/Basic/AMDGPUTypes.def"
 #define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
 #include "clang/Basic/HLSLIntangibleTypes.def"
diff --git a/clang/lib/CodeGen/ConstantInitBuilder.cpp b/clang/lib/CodeGen/ConstantInitBuilder.cpp
index ce1fe137c191..7b3e7aea0f64 100644
--- a/clang/lib/CodeGen/ConstantInitBuilder.cpp
+++ b/clang/lib/CodeGen/ConstantInitBuilder.cpp
@@ -160,7 +160,7 @@ ConstantAggregateBuilderBase::getAddrOfPosition(llvm::Type *type,
                                         nullptr, "");
   Builder.SelfReferences.emplace_back(dummy);
   auto &entry = Builder.SelfReferences.back();
-  (void)getGEPIndicesTo(entry.Indices, position + Begin);
+  getGEPIndicesTo(entry.Indices, position + Begin);
   return dummy;
 }
 
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index 763b7a3d616b..6ed736ee6235 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -304,7 +304,7 @@ static Value *GetOrInsertAMDGPUPredicate(CodeGenFunction &CGF, Twine Name) {
   P->setExternallyInitialized(true);
 
   return CGF.Builder.CreateLoad(
-      RawAddress(P, PTy, CharUnits::One(), KnownNonNull));
+      RawAddress(P, PTy, CharUnits::One(), KnownNonNull), true);
 }
 
 Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
@@ -557,6 +557,18 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
   case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
+  case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr8_b64_v2i32:
   case AMDGPU::BI__builtin_amdgcn_ds_read_tr6_b96_v3i32:
@@ -567,6 +579,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_i32:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b64_v2i32:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr8_b64_v2i32:
       IID = Intrinsic::amdgcn_global_load_tr_b64;
       break;
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v4i16:
@@ -575,8 +588,31 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8i16:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8f16:
     case AMDGPU::BI__builtin_amdgcn_global_load_tr_b128_v8bf16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8i16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8f16:
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr16_b128_v8bf16:
       IID = Intrinsic::amdgcn_global_load_tr_b128;
       break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr4_b64_v2i32:
+      IID = Intrinsic::amdgcn_global_load_tr4_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_global_load_tr6_b96_v3i32:
+      IID = Intrinsic::amdgcn_global_load_tr6_b96;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr4_b64_v2i32:
+      IID = Intrinsic::amdgcn_ds_load_tr4_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr6_b96_v3i32:
+      IID = Intrinsic::amdgcn_ds_load_tr6_b96;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr8_b64_v2i32:
+      IID = Intrinsic::amdgcn_ds_load_tr8_b64;
+      break;
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8i16:
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8f16:
+    case AMDGPU::BI__builtin_amdgcn_ds_load_tr16_b128_v8bf16:
+      IID = Intrinsic::amdgcn_ds_load_tr16_b128;
+      break;
     case AMDGPU::BI__builtin_amdgcn_ds_read_tr4_b64_v2i32:
       IID = Intrinsic::amdgcn_ds_read_tr4_b64;
       break;
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 6738d4be6dd2..7e6a47fd7c10 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -838,20 +838,20 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
   NEONMAP1(vrecpsq_v, arm_neon_vrecps, Add1ArgType),
   NEONMAP2(vrhadd_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrhaddq_v, arm_neon_vrhaddu, arm_neon_vrhadds, Add1ArgType | UnsignedAlts),
-  NEONMAP1(vrnd_v, arm_neon_vrintz, Add1ArgType),
-  NEONMAP1(vrnda_v, arm_neon_vrinta, Add1ArgType),
-  NEONMAP1(vrndaq_v, arm_neon_vrinta, Add1ArgType),
+  NEONMAP1(vrnd_v, trunc, Add1ArgType),
+  NEONMAP1(vrnda_v, round, Add1ArgType),
+  NEONMAP1(vrndaq_v, round, Add1ArgType),
   NEONMAP0(vrndi_v),
   NEONMAP0(vrndiq_v),
-  NEONMAP1(vrndm_v, arm_neon_vrintm, Add1ArgType),
-  NEONMAP1(vrndmq_v, arm_neon_vrintm, Add1ArgType),
-  NEONMAP1(vrndn_v, arm_neon_vrintn, Add1ArgType),
-  NEONMAP1(vrndnq_v, arm_neon_vrintn, Add1ArgType),
-  NEONMAP1(vrndp_v, arm_neon_vrintp, Add1ArgType),
-  NEONMAP1(vrndpq_v, arm_neon_vrintp, Add1ArgType),
-  NEONMAP1(vrndq_v, arm_neon_vrintz, Add1ArgType),
-  NEONMAP1(vrndx_v, arm_neon_vrintx, Add1ArgType),
-  NEONMAP1(vrndxq_v, arm_neon_vrintx, Add1ArgType),
+  NEONMAP1(vrndm_v, floor, Add1ArgType),
+  NEONMAP1(vrndmq_v, floor, Add1ArgType),
+  NEONMAP1(vrndn_v, roundeven, Add1ArgType),
+  NEONMAP1(vrndnq_v, roundeven, Add1ArgType),
+  NEONMAP1(vrndp_v, ceil, Add1ArgType),
+  NEONMAP1(vrndpq_v, ceil, Add1ArgType),
+  NEONMAP1(vrndq_v, trunc, Add1ArgType),
+  NEONMAP1(vrndx_v, rint, Add1ArgType),
+  NEONMAP1(vrndxq_v, rint, Add1ArgType),
   NEONMAP2(vrshl_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrshlq_v, arm_neon_vrshiftu, arm_neon_vrshifts, Add1ArgType | UnsignedAlts),
   NEONMAP2(vrshr_n_v, arm_neon_vrshiftu, arm_neon_vrshifts, UnsignedAlts),
@@ -3132,7 +3132,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case NEON::BI__builtin_neon_vrndns_f32: {
     Value *Arg = EmitScalarExpr(E->getArg(0));
     llvm::Type *Tys[] = {Arg->getType()};
-    Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
+    Function *F = CGM.getIntrinsic(Intrinsic::roundeven, Tys);
     return Builder.CreateCall(F, {Arg}, "vrndn"); }
 
   case NEON::BI__builtin_neon_vset_lane_i8:
@@ -5471,19 +5471,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   }
 
   if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
-      BuiltinID == clang::AArch64::BI_WriteStatusReg) {
+      BuiltinID == clang::AArch64::BI_WriteStatusReg ||
+      BuiltinID == clang::AArch64::BI__sys) {
     LLVMContext &Context = CGM.getLLVMContext();
 
     unsigned SysReg =
       E->getArg(0)->EvaluateKnownConstInt(getContext()).getZExtValue();
 
     std::string SysRegStr;
-    llvm::raw_string_ostream(SysRegStr) <<
-                       ((1 << 1) | ((SysReg >> 14) & 1))  << ":" <<
-                       ((SysReg >> 11) & 7)               << ":" <<
-                       ((SysReg >> 7)  & 15)              << ":" <<
-                       ((SysReg >> 3)  & 15)              << ":" <<
-                       ( SysReg        & 7);
+    unsigned SysRegOp0 = (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
+                          BuiltinID == clang::AArch64::BI_WriteStatusReg)
+                             ? ((1 << 1) | ((SysReg >> 14) & 1))
+                             : 1;
+    llvm::raw_string_ostream(SysRegStr)
+        << SysRegOp0 << ":" << ((SysReg >> 11) & 7) << ":"
+        << ((SysReg >> 7) & 15) << ":" << ((SysReg >> 3) & 15) << ":"
+        << (SysReg & 7);
 
     llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
     llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
@@ -5500,8 +5503,13 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
 
     llvm::Function *F = CGM.getIntrinsic(Intrinsic::write_register, Types);
     llvm::Value *ArgValue = EmitScalarExpr(E->getArg(1));
-
-    return Builder.CreateCall(F, { Metadata, ArgValue });
+    llvm::Value *Result = Builder.CreateCall(F, {Metadata, ArgValue});
+    if (BuiltinID == clang::AArch64::BI__sys) {
+      // Return 0 for convenience, even though MSVC returns some other undefined
+      // value.
+      Result = ConstantInt::get(Builder.getInt32Ty(), 0);
+    }
+    return Result;
   }
 
   if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
@@ -6499,12 +6507,38 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
   }
 
   case clang::AArch64::BI_InterlockedAdd:
-  case clang::AArch64::BI_InterlockedAdd64: {
+  case clang::AArch64::BI_InterlockedAdd_acq:
+  case clang::AArch64::BI_InterlockedAdd_rel:
+  case clang::AArch64::BI_InterlockedAdd_nf:
+  case clang::AArch64::BI_InterlockedAdd64:
+  case clang::AArch64::BI_InterlockedAdd64_acq:
+  case clang::AArch64::BI_InterlockedAdd64_rel:
+  case clang::AArch64::BI_InterlockedAdd64_nf: {
     Address DestAddr = CheckAtomicAlignment(*this, E);
     Value *Val = EmitScalarExpr(E->getArg(1));
+    llvm::AtomicOrdering Ordering;
+    switch (BuiltinID) {
+    case clang::AArch64::BI_InterlockedAdd:
+    case clang::AArch64::BI_InterlockedAdd64:
+      Ordering = llvm::AtomicOrdering::SequentiallyConsistent;
+      break;
+    case clang::AArch64::BI_InterlockedAdd_acq:
+    case clang::AArch64::BI_InterlockedAdd64_acq:
+      Ordering = llvm::AtomicOrdering::Acquire;
+      break;
+    case clang::AArch64::BI_InterlockedAdd_rel:
+    case clang::AArch64::BI_InterlockedAdd64_rel:
+      Ordering = llvm::AtomicOrdering::Release;
+      break;
+    case clang::AArch64::BI_InterlockedAdd_nf:
+    case clang::AArch64::BI_InterlockedAdd64_nf:
+      Ordering = llvm::AtomicOrdering::Monotonic;
+      break;
+    default:
+      llvm_unreachable("missing builtin ID in switch!");
+    }
     AtomicRMWInst *RMWI =
-        Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val,
-                                llvm::AtomicOrdering::SequentiallyConsistent);
+        Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val, Ordering);
     return Builder.CreateAdd(RMWI, Val);
   }
   }
diff --git a/clang/lib/CodeGen/TargetBuiltins/X86.cpp b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
index 3c2a77ab3fe4..e23d19d2f6b6 100644
--- a/clang/lib/CodeGen/TargetBuiltins/X86.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
@@ -804,6 +804,17 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
     return Builder.CreateCall(F, {Address, RW, Locality, Data});
   }
+  case X86::BI_m_prefetch:
+  case X86::BI_m_prefetchw: {
+    Value *Address = Ops[0];
+    // The 'w' suffix implies write.
+    Value *RW =
+        ConstantInt::get(Int32Ty, BuiltinID == X86::BI_m_prefetchw ? 1 : 0);
+    Value *Locality = ConstantInt::get(Int32Ty, 0x3);
+    Value *Data = ConstantInt::get(Int32Ty, 1);
+    Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
+    return Builder.CreateCall(F, {Address, RW, Locality, Data});
+  }
   case X86::BI_mm_clflush: {
     return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
                               Ops[0]);
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index 14d4cee7c61d..cc3d487da83b 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -544,7 +544,7 @@ ABIArgInfo RISCVABIInfo::coerceVLSVector(QualType Ty, unsigned ABIVLen) const {
   assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");
 
   auto VScale = getContext().getTargetInfo().getVScaleRange(
-      getContext().getLangOpts(), false);
+      getContext().getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
 
   unsigned NumElts = VT->getNumElements();
   llvm::Type *EltType = llvm::Type::getInt1Ty(getVMContext());
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index afa23bffcd07..845b0f4b5846 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -58,7 +58,7 @@ class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
               const SmallVector<int32_t> *Packoffsets = nullptr) const override;
   llvm::Type *getSPIRVImageTypeFromHLSLResource(
       const HLSLAttributedResourceType::Attributes &attributes,
-      llvm::Type *ElementType, llvm::LLVMContext &Ctx) const;
+      QualType SampledType, CodeGenModule &CGM) const;
   void
   setOCLKernelStubCallingConvention(const FunctionType *&FT) const override;
 };
@@ -75,6 +75,10 @@ class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
                                          SyncScope Scope,
                                          llvm::AtomicOrdering Ordering,
                                          llvm::LLVMContext &Ctx) const override;
+  bool supportsLibCall() const override {
+    return getABIInfo().getTarget().getTriple().getVendor() !=
+           llvm::Triple::AMD;
+  }
 };
 
 inline StringRef mapClangSyncScopeToLLVM(SyncScope Scope) {
@@ -483,12 +487,12 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
     assert(!ResAttrs.IsROV &&
            "Rasterizer order views not implemented for SPIR-V yet");
 
-    llvm::Type *ElemType = CGM.getTypes().ConvertType(ContainedTy);
     if (!ResAttrs.RawBuffer) {
       // convert element type
-      return getSPIRVImageTypeFromHLSLResource(ResAttrs, ElemType, Ctx);
+      return getSPIRVImageTypeFromHLSLResource(ResAttrs, ContainedTy, CGM);
     }
 
+    llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(ContainedTy);
     llvm::ArrayType *RuntimeArrayType = llvm::ArrayType::get(ElemType, 0);
     uint32_t StorageClass = /* StorageBuffer storage class */ 12;
     bool IsWritable = ResAttrs.ResourceClass == llvm::dxil::ResourceClass::UAV;
@@ -516,13 +520,18 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
 }
 
 llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
-    const HLSLAttributedResourceType::Attributes &attributes,
-    llvm::Type *ElementType, llvm::LLVMContext &Ctx) const {
+    const HLSLAttributedResourceType::Attributes &attributes, QualType Ty,
+    CodeGenModule &CGM) const {
+  llvm::LLVMContext &Ctx = CGM.getLLVMContext();
 
-  if (ElementType->isVectorTy())
-    ElementType = ElementType->getScalarType();
+  Ty = Ty->getCanonicalTypeUnqualified();
+  if (const VectorType *V = dyn_cast<VectorType>(Ty))
+    Ty = V->getElementType();
+  assert(!Ty->isVectorType() && "We still have a vector type.");
 
-  assert((ElementType->isIntegerTy() || ElementType->isFloatingPointTy()) &&
+  llvm::Type *SampledType = CGM.getTypes().ConvertTypeForMem(Ty);
+
+  assert((SampledType->isIntegerTy() || SampledType->isFloatingPointTy()) &&
          "The element type for a SPIR-V resource must be a scalar integer or "
          "floating point type.");
 
@@ -531,6 +540,9 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
   // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage.
   SmallVector<unsigned, 6> IntParams(6, 0);
 
+  const char *Name =
+      Ty->isSignedIntegerType() ? "spirv.SignedImage" : "spirv.Image";
+
   // Dim
   // For now we assume everything is a buffer.
   IntParams[0] = 5;
@@ -553,7 +565,9 @@ llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
   // Setting to unknown for now.
   IntParams[5] = 0;
 
-  return llvm::TargetExtType::get(Ctx, "spirv.Image", {ElementType}, IntParams);
+  llvm::TargetExtType *ImageType =
+      llvm::TargetExtType::get(Ctx, Name, {SampledType}, IntParams);
+  return ImageType;
 }
 
 std::unique_ptr<TargetCodeGenInfo>
diff --git a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
index 2ffbc1a22695..440b5633fb1b 100644
--- a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
+++ b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
@@ -9,19 +9,14 @@
 #include "DirectoryScanner.h"
 #include "clang/DirectoryWatcher/DirectoryWatcher.h"
 
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/Path.h"
-#include <atomic>
 #include <condition_variable>
 #include <mutex>
 #include <queue>
 #include <string>
 #include <thread>
-#include <vector>
 
 #include <fcntl.h>
 #include <limits.h>
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 2e3333bbb95f..e908afc39ffe 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -78,7 +78,6 @@ add_clang_library(clangDriver
   ToolChains/OHOS.cpp
   ToolChains/OpenBSD.cpp
   ToolChains/PS4CPU.cpp
-  ToolChains/RISCVToolchain.cpp
   ToolChains/Solaris.cpp
   ToolChains/SPIRV.cpp
   ToolChains/SPIRVOpenMP.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 5a2390a654a4..a9a8dbce3dec 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -43,7 +43,6 @@
 #include "ToolChains/PPCFreeBSD.h"
 #include "ToolChains/PPCLinux.h"
 #include "ToolChains/PS4CPU.h"
-#include "ToolChains/RISCVToolchain.h"
 #include "ToolChains/SPIRV.h"
 #include "ToolChains/SPIRVOpenMP.h"
 #include "ToolChains/SYCL.h"
@@ -195,7 +194,12 @@ std::string Driver::GetResourcesPath(StringRef BinaryPath) {
 
   StringRef ConfiguredResourceDir(CLANG_RESOURCE_DIR);
   if (!ConfiguredResourceDir.empty()) {
-    llvm::sys::path::append(P, ConfiguredResourceDir);
+    // FIXME: We should fix the behavior of llvm::sys::path::append so we don't
+    // need to check for absolute paths here.
+    if (llvm::sys::path::is_absolute(ConfiguredResourceDir))
+      P = ConfiguredResourceDir;
+    else
+      llvm::sys::path::append(P, ConfiguredResourceDir);
   } else {
     // On Windows, libclang.dll is in bin/.
     // On non-Windows, libclang.so/.dylib is in lib/.
@@ -991,9 +995,11 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     if (CudaInstallation.isValid())
       CudaInstallation.WarnIfUnsupportedVersion();
     C.addOffloadDeviceToolChain(&TC, Action::OFK_Cuda);
+    OffloadArchs[&TC] = getOffloadArchs(C, C.getArgs(), Action::OFK_Cuda, &TC,
+                                        /*SpecificToolchain=*/true);
   } else if (IsHIP && !UseLLVMOffload) {
     if (auto *OMPTargetArg =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       Diag(clang::diag::err_drv_unsupported_opt_for_language_mode)
           << OMPTargetArg->getSpelling() << "HIP";
       return;
@@ -1007,6 +1013,12 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         getOffloadToolChain(C.getInputArgs(), Action::OFK_HIP, *HIPTriple,
                             C.getDefaultToolChain().getTriple());
     C.addOffloadDeviceToolChain(&TC, Action::OFK_HIP);
+
+    // TODO: Fix 'amdgcnspirv' handling with the new driver.
+    if (C.getInputArgs().hasFlag(options::OPT_offload_new_driver,
+                                 options::OPT_no_offload_new_driver, false))
+      OffloadArchs[&TC] = getOffloadArchs(C, C.getArgs(), Action::OFK_HIP, &TC,
+                                          /*SpecificToolchain=*/true);
   }
 
   if (IsCuda || IsHIP)
@@ -1021,7 +1033,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       ((IsCuda || IsHIP) && UseLLVMOffload) ||
       (C.getInputArgs().hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                                 options::OPT_fno_openmp, false) &&
-       (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ) ||
+       (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ) ||
         C.getInputArgs().hasArg(options::OPT_offload_arch_EQ)));
   if (IsOpenMPOffloading) {
     // We expect that -fopenmp-targets is always used in conjunction with the
@@ -1033,98 +1045,89 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       return;
     }
 
-    llvm::StringMap<llvm::DenseSet<StringRef>> DerivedArchs;
-    llvm::StringMap<StringRef> FoundNormalizedTriples;
-    std::multiset<StringRef> OpenMPTriples;
-
     // If the user specified -fopenmp-targets= we create a toolchain for each
     // valid triple. Otherwise, if only --offload-arch= was specified we instead
     // attempt to derive the appropriate toolchains from the arguments.
     if (Arg *OpenMPTargets =
-            C.getInputArgs().getLastArg(options::OPT_fopenmp_targets_EQ)) {
+            C.getInputArgs().getLastArg(options::OPT_offload_targets_EQ)) {
       if (OpenMPTargets && !OpenMPTargets->getNumValues()) {
         Diag(clang::diag::warn_drv_empty_joined_argument)
             << OpenMPTargets->getAsString(C.getInputArgs());
         return;
       }
+
+      // Make sure these show up in a deterministic order.
+      std::multiset<StringRef> OpenMPTriples;
       for (StringRef T : OpenMPTargets->getValues())
         OpenMPTriples.insert(T);
-    } else if (C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
-               ((!IsHIP && !IsCuda) || UseLLVMOffload)) {
-      const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
-      auto AMDTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs());
-      auto NVPTXTriple = getNVIDIAOffloadTargetTriple(*this, C.getInputArgs(),
-                                                      HostTC->getTriple());
 
-      // Attempt to deduce the offloading triple from the set of architectures.
-      // We can only correctly deduce NVPTX / AMDGPU triples currently.
-      // We need to temporarily create these toolchains so that we can access
-      // tools for inferring architectures.
-      llvm::DenseSet<StringRef> Archs;
-      for (const std::optional<llvm::Triple> &TT : {NVPTXTriple, AMDTriple}) {
-        if (!TT)
+      llvm::StringMap<StringRef> FoundNormalizedTriples;
+      for (StringRef T : OpenMPTriples) {
+        llvm::Triple TT(ToolChain::getOpenMPTriple(T));
+        std::string NormalizedName = TT.normalize();
+
+        // Make sure we don't have a duplicate triple.
+        auto [TripleIt, Inserted] =
+            FoundNormalizedTriples.try_emplace(NormalizedName, T);
+        if (!Inserted) {
+          Diag(clang::diag::warn_drv_omp_offload_target_duplicate)
+              << T << TripleIt->second;
           continue;
+        }
 
-        auto &TC =
-            getOffloadToolChain(C.getInputArgs(), Action::OFK_OpenMP, *TT,
-                                C.getDefaultToolChain().getTriple());
-        for (StringRef Arch :
-             getOffloadArchs(C, C.getArgs(), Action::OFK_OpenMP, &TC, true))
-          Archs.insert(Arch);
-      }
+        // If the specified target is invalid, emit a diagnostic.
+        if (TT.getArch() == llvm::Triple::UnknownArch) {
+          Diag(clang::diag::err_drv_invalid_omp_target) << T;
+          continue;
+        }
 
-      for (StringRef Arch : Archs) {
-        if (NVPTXTriple && IsNVIDIAOffloadArch(StringToOffloadArch(
-                               getProcessorFromTargetID(*NVPTXTriple, Arch)))) {
-          DerivedArchs[NVPTXTriple->getTriple()].insert(Arch);
-        } else if (AMDTriple &&
-                   IsAMDOffloadArch(StringToOffloadArch(
-                       getProcessorFromTargetID(*AMDTriple, Arch)))) {
-          DerivedArchs[AMDTriple->getTriple()].insert(Arch);
-        } else {
+        auto &TC = getOffloadToolChain(C.getInputArgs(), Action::OFK_OpenMP, TT,
+                                       C.getDefaultToolChain().getTriple());
+        C.addOffloadDeviceToolChain(&TC, Action::OFK_OpenMP);
+        OffloadArchs[&TC] =
+            getOffloadArchs(C, C.getArgs(), Action::OFK_OpenMP, &TC,
+                            /*SpecificToolchain=*/true);
+      }
+    } else if (C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) &&
+               ((!IsHIP && !IsCuda) || UseLLVMOffload)) {
+      llvm::Triple AMDTriple("amdgcn-amd-amdhsa");
+      llvm::Triple NVPTXTriple("nvptx64-nvidia-cuda");
+
+      for (StringRef Arch :
+           C.getInputArgs().getAllArgValues(options::OPT_offload_arch_EQ)) {
+        bool IsNVPTX = IsNVIDIAOffloadArch(
+            StringToOffloadArch(getProcessorFromTargetID(NVPTXTriple, Arch)));
+        bool IsAMDGPU = IsAMDOffloadArch(
+            StringToOffloadArch(getProcessorFromTargetID(AMDTriple, Arch)));
+        if (!IsNVPTX && !IsAMDGPU && !Arch.empty() &&
+            !Arch.equals_insensitive("native")) {
           Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch) << Arch;
           return;
         }
       }
 
+      // Attempt to deduce the offloading triple from the set of architectures.
+      // We can only correctly deduce NVPTX / AMDGPU triples currently.
+      for (const llvm::Triple &TT : {AMDTriple, NVPTXTriple}) {
+        auto &TC = getOffloadToolChain(C.getInputArgs(), Action::OFK_OpenMP, TT,
+                                       C.getDefaultToolChain().getTriple());
+
+        llvm::SmallVector<StringRef> Archs =
+            getOffloadArchs(C, C.getArgs(), Action::OFK_OpenMP, &TC,
+                            /*SpecificToolchain=*/false);
+        if (!Archs.empty()) {
+          C.addOffloadDeviceToolChain(&TC, Action::OFK_OpenMP);
+          OffloadArchs[&TC] = Archs;
+        }
+      }
+
       // If the set is empty then we failed to find a native architecture.
-      if (Archs.empty()) {
+      auto TCRange = C.getOffloadToolChains(Action::OFK_OpenMP);
+      if (TCRange.first == TCRange.second)
         Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch)
             << "native";
-        return;
-      }
-
-      for (const auto &TripleAndArchs : DerivedArchs)
-        OpenMPTriples.insert(TripleAndArchs.first());
-    }
-
-    for (StringRef Val : OpenMPTriples) {
-      llvm::Triple TT(ToolChain::getOpenMPTriple(Val));
-      std::string NormalizedName = TT.normalize();
-
-      // Make sure we don't have a duplicate triple.
-      auto [TripleIt, Inserted] =
-          FoundNormalizedTriples.try_emplace(NormalizedName, Val);
-      if (!Inserted) {
-        Diag(clang::diag::warn_drv_omp_offload_target_duplicate)
-            << Val << TripleIt->second;
-        continue;
-      }
-
-      // If the specified target is invalid, emit a diagnostic.
-      if (TT.getArch() == llvm::Triple::UnknownArch) {
-        Diag(clang::diag::err_drv_invalid_omp_target) << Val;
-        continue;
-      }
-
-      auto &TC = getOffloadToolChain(C.getInputArgs(), Action::OFK_OpenMP, TT,
-                                     C.getDefaultToolChain().getTriple());
-      C.addOffloadDeviceToolChain(&TC, Action::OFK_OpenMP);
-      auto It = DerivedArchs.find(TT.getTriple());
-      if (It != DerivedArchs.end())
-        KnownArchs[&TC] = It->second;
     }
-  } else if (C.getInputArgs().hasArg(options::OPT_fopenmp_targets_EQ)) {
+  } else if (C.getInputArgs().hasArg(options::OPT_offload_targets_EQ)) {
     Diag(clang::diag::err_drv_expecting_fopenmp_with_fopenmp_targets);
     return;
   }
@@ -1155,9 +1158,11 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
     // going to create will depend on both.
     const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
     for (const auto &TT : UniqueSYCLTriplesVec) {
-      auto SYCLTC = &getOffloadToolChain(C.getInputArgs(), Action::OFK_SYCL, TT,
-                                         HostTC->getTriple());
-      C.addOffloadDeviceToolChain(SYCLTC, Action::OFK_SYCL);
+      auto &TC = getOffloadToolChain(C.getInputArgs(), Action::OFK_SYCL, TT,
+                                     HostTC->getTriple());
+      C.addOffloadDeviceToolChain(&TC, Action::OFK_SYCL);
+      OffloadArchs[&TC] = getOffloadArchs(C, C.getArgs(), Action::OFK_SYCL, &TC,
+                                          /*SpecificToolchain=*/true);
     }
   }
 
@@ -4738,20 +4743,22 @@ static StringRef getCanonicalArchString(Compilation &C,
                                         const llvm::opt::DerivedArgList &Args,
                                         StringRef ArchStr,
                                         const llvm::Triple &Triple,
-                                        bool SuppressError = false) {
+                                        bool SpecificToolchain) {
   // Lookup the CUDA / HIP architecture string. Only report an error if we were
   // expecting the triple to be only NVPTX / AMDGPU.
   OffloadArch Arch =
       StringToOffloadArch(getProcessorFromTargetID(Triple, ArchStr));
-  if (!SuppressError && Triple.isNVPTX() &&
+  if (Triple.isNVPTX() &&
       (Arch == OffloadArch::UNKNOWN || !IsNVIDIAOffloadArch(Arch))) {
-    C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
-        << "CUDA" << ArchStr;
+    if (SpecificToolchain)
+      C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
+          << "CUDA" << ArchStr;
     return StringRef();
-  } else if (!SuppressError && Triple.isAMDGPU() &&
+  } else if (Triple.isAMDGPU() &&
              (Arch == OffloadArch::UNKNOWN || !IsAMDOffloadArch(Arch))) {
-    C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
-        << "HIP" << ArchStr;
+    if (SpecificToolchain)
+      C.getDriver().Diag(clang::diag::err_drv_offload_bad_gpu_arch)
+          << "HIP" << ArchStr;
     return StringRef();
   }
 
@@ -4760,13 +4767,9 @@ static StringRef getCanonicalArchString(Compilation &C,
 
   if (IsAMDOffloadArch(Arch)) {
     llvm::StringMap<bool> Features;
-    auto HIPTriple = getHIPOffloadTargetTriple(C.getDriver(), C.getInputArgs());
-    if (!HIPTriple)
-      return StringRef();
-    auto Arch = parseTargetID(*HIPTriple, ArchStr, &Features);
+    std::optional<StringRef> Arch = parseTargetID(Triple, ArchStr, &Features);
     if (!Arch) {
       C.getDriver().Diag(clang::diag::err_drv_bad_target_id) << ArchStr;
-      C.setContainsError();
       return StringRef();
     }
     return Args.MakeArgStringRef(getCanonicalTargetID(*Arch, Features));
@@ -4789,10 +4792,10 @@ getConflictOffloadArchCombination(const llvm::DenseSet<StringRef> &Archs,
   return getConflictTargetIDCombination(ArchSet);
 }
 
-llvm::DenseSet<StringRef>
+llvm::SmallVector<StringRef>
 Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
                         Action::OffloadKind Kind, const ToolChain *TC,
-                        bool SuppressError) const {
+                        bool SpecificToolchain) const {
   if (!TC)
     TC = &C.getDefaultToolChain();
 
@@ -4807,25 +4810,22 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
                 : "--no-offload-arch");
   }
 
-  if (KnownArchs.contains(TC))
-    return KnownArchs.lookup(TC);
-
   llvm::DenseSet<StringRef> Archs;
 
   if (!TC->getTargetID().empty()) {
     Archs.insert(TC->getTargetID());
-    return Archs;
+    return llvm::SmallVector<StringRef>();
   }
 
   for (auto *Arg : C.getArgsForToolChain(TC, /*BoundArch=*/"", Kind)) {
     // Add or remove the seen architectures in order of appearance. If an
     // invalid architecture is given we simply exit.
     if (Arg->getOption().matches(options::OPT_offload_arch_EQ)) {
-      for (StringRef Arch : llvm::split(Arg->getValue(), ",")) {
+      for (StringRef Arch : Arg->getValues()) {
         if (Arch == "native" || Arch.empty()) {
           auto GPUsOrErr = TC->getSystemGPUArchs(Args);
           if (!GPUsOrErr) {
-            if (SuppressError)
+            if (!SpecificToolchain)
               llvm::consumeError(GPUsOrErr.takeError());
             else
               TC->getDriver().Diag(diag::err_drv_undetermined_gpu_arch)
@@ -4835,16 +4835,21 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
           }
 
           for (auto ArchStr : *GPUsOrErr) {
-            Archs.insert(
+            StringRef CanonicalStr =
                 getCanonicalArchString(C, Args, Args.MakeArgString(ArchStr),
-                                       TC->getTriple(), SuppressError));
+                                       TC->getTriple(), SpecificToolchain);
+            if (!CanonicalStr.empty())
+              Archs.insert(CanonicalStr);
+            else if (SpecificToolchain)
+              return llvm::SmallVector<StringRef>();
           }
         } else {
-          StringRef ArchStr = getCanonicalArchString(
-              C, Args, Arch, TC->getTriple(), SuppressError);
-          if (ArchStr.empty())
-            return Archs;
-          Archs.insert(ArchStr);
+          StringRef CanonicalStr = getCanonicalArchString(
+              C, Args, Arch, TC->getTriple(), SpecificToolchain);
+          if (!CanonicalStr.empty())
+            Archs.insert(CanonicalStr);
+          else if (SpecificToolchain)
+            return llvm::SmallVector<StringRef>();
         }
       }
     } else if (Arg->getOption().matches(options::OPT_no_offload_arch_EQ)) {
@@ -4853,9 +4858,7 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
           Archs.clear();
         } else {
           StringRef ArchStr = getCanonicalArchString(
-              C, Args, Arch, TC->getTriple(), SuppressError);
-          if (ArchStr.empty())
-            return Archs;
+              C, Args, Arch, TC->getTriple(), SpecificToolchain);
           Archs.erase(ArchStr);
         }
       }
@@ -4863,17 +4866,12 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
   }
 
   if (auto ConflictingArchs =
-          getConflictOffloadArchCombination(Archs, TC->getTriple())) {
+          getConflictOffloadArchCombination(Archs, TC->getTriple()))
     C.getDriver().Diag(clang::diag::err_drv_bad_offload_arch_combo)
         << ConflictingArchs->first << ConflictingArchs->second;
-    C.setContainsError();
-  }
 
   // Skip filling defaults if we're just querying what is availible.
-  if (SuppressError)
-    return Archs;
-
-  if (Archs.empty()) {
+  if (SpecificToolchain && Archs.empty()) {
     if (Kind == Action::OFK_Cuda) {
       Archs.insert(OffloadArchToString(OffloadArch::CudaDefault));
     } else if (Kind == Action::OFK_HIP) {
@@ -4899,12 +4897,13 @@ Driver::getOffloadArchs(Compilation &C, const llvm::opt::DerivedArgList &Args,
         }
       }
     }
-  } else {
-    Args.ClaimAllArgs(options::OPT_offload_arch_EQ);
-    Args.ClaimAllArgs(options::OPT_no_offload_arch_EQ);
   }
+  Args.ClaimAllArgs(options::OPT_offload_arch_EQ);
+  Args.ClaimAllArgs(options::OPT_no_offload_arch_EQ);
 
-  return Archs;
+  SmallVector<StringRef> Sorted(Archs.begin(), Archs.end());
+  llvm::sort(Sorted);
+  return Sorted;
 }
 
 Action *Driver::BuildOffloadingActions(Compilation &C,
@@ -4968,10 +4967,7 @@ Action *Driver::BuildOffloadingActions(Compilation &C,
     // Get the product of all bound architectures and toolchains.
     SmallVector<std::pair<const ToolChain *, StringRef>> TCAndArchs;
     for (const ToolChain *TC : ToolChains) {
-      llvm::DenseSet<StringRef> Arches = getOffloadArchs(C, Args, Kind, TC);
-      SmallVector<StringRef, 0> Sorted(Arches.begin(), Arches.end());
-      llvm::sort(Sorted);
-      for (StringRef Arch : Sorted) {
+      for (StringRef Arch : OffloadArchs.lookup(TC)) {
         TCAndArchs.push_back(std::make_pair(TC, Arch));
         DeviceActions.push_back(
             C.MakeAction<InputAction>(*InputArg, InputType, CUID));
@@ -7021,16 +7017,11 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
         TC = std::make_unique<toolchains::AVRToolChain>(*this, Target, Args);
         break;
       case llvm::Triple::msp430:
-        TC =
-            std::make_unique<toolchains::MSP430ToolChain>(*this, Target, Args);
+        TC = std::make_unique<toolchains::MSP430ToolChain>(*this, Target, Args);
         break;
       case llvm::Triple::riscv32:
       case llvm::Triple::riscv64:
-        if (toolchains::RISCVToolChain::hasGCCToolchain(*this, Args))
-          TC =
-              std::make_unique<toolchains::RISCVToolChain>(*this, Target, Args);
-        else
-          TC = std::make_unique<toolchains::BareMetal>(*this, Target, Args);
+        TC = std::make_unique<toolchains::BareMetal>(*this, Target, Args);
         break;
       case llvm::Triple::ve:
         TC = std::make_unique<toolchains::VEToolChain>(*this, Target, Args);
diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp
index 3dfd51ee2365..e83aee01b75c 100644
--- a/clang/lib/Driver/OffloadBundler.cpp
+++ b/clang/lib/Driver/OffloadBundler.cpp
@@ -744,8 +744,7 @@ class ObjectFileHandler final : public FileHandler {
     // Input is the same as the content of the original input file, therefore
     // temporary file is not needed.
     if (StringRef(BundlerConfig.FilesType).starts_with("a")) {
-      auto InputFileOrErr =
-          TempFiles.Create(ArrayRef<char>(Input.data(), Input.size()));
+      auto InputFileOrErr = TempFiles.Create(ArrayRef<char>(Input));
       if (!InputFileOrErr)
         return InputFileOrErr.takeError();
       ObjcopyInputFileName = *InputFileOrErr;
@@ -1289,8 +1288,7 @@ CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input,
     HashRecalcTimer.startTimer();
     llvm::MD5 Hash;
     llvm::MD5::MD5Result Result;
-    Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData.data(),
-                                        DecompressedData.size()));
+    Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData));
     Hash.final(Result);
     uint64_t RecalculatedHash = Result.low();
     HashRecalcTimer.stopTimer();
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index eb4718909c95..4bd61c2f8dee 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -740,7 +740,7 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
                         options::OPT_fno_sanitize_merge_handlers_EQ);
   MergeKinds &= Kinds;
 
-  // Parse -fno-sanitize-top-hot flags
+  // Parse -fno-fsanitize-skip-hot-cutoff flags
   SkipHotCutoffs = parseSanitizeSkipHotCutoffArgs(D, Args, DiagnoseErrors);
 
   // Parse -f(no-)?sanitize-annotate-debug-info flags
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 5352c4a90c5c..24a46eff09a5 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -1742,7 +1742,7 @@ llvm::opt::DerivedArgList *ToolChain::TranslateOpenMPTargetArgs(
       continue;
     }
     if (XOpenMPTargetNoTriple && XOpenMPTargetArg &&
-        Args.getAllArgValues(options::OPT_fopenmp_targets_EQ).size() != 1) {
+        Args.getAllArgValues(options::OPT_offload_targets_EQ).size() != 1) {
       getDriver().Diag(diag::err_drv_Xopenmp_target_missing_triple);
       continue;
     }
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index ea906d2e2acf..6bd710ec6b8b 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -156,39 +156,18 @@ static bool
 getAArch64MicroArchFeaturesFromMtune(const Driver &D, StringRef Mtune,
                                      const ArgList &Args,
                                      std::vector<StringRef> &Features) {
-  std::string MtuneLowerCase = Mtune.lower();
   // Check CPU name is valid, but ignore any extensions on it.
+  std::string MtuneLowerCase = Mtune.lower();
   llvm::AArch64::ExtensionSet Extensions;
   StringRef Tune;
-  if (!DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, Extensions))
-    return false;
-
-  // Handle CPU name is 'native'.
-  if (MtuneLowerCase == "native")
-    MtuneLowerCase = std::string(llvm::sys::getHostCPUName());
-
-  // 'cyclone' and later have zero-cycle register moves and zeroing.
-  if (MtuneLowerCase == "cyclone" ||
-      StringRef(MtuneLowerCase).starts_with("apple")) {
-    Features.push_back("+zcm");
-    Features.push_back("+zcz");
-  }
-
-  return true;
+  return DecodeAArch64Mcpu(D, MtuneLowerCase, Tune, Extensions);
 }
 
 static bool
 getAArch64MicroArchFeaturesFromMcpu(const Driver &D, StringRef Mcpu,
                                     const ArgList &Args,
                                     std::vector<StringRef> &Features) {
-  StringRef CPU;
-  // Check CPU name is valid, but ignore any extensions on it.
-  llvm::AArch64::ExtensionSet DecodedFeature;
-  std::string McpuLowerCase = Mcpu.lower();
-  if (!DecodeAArch64Mcpu(D, McpuLowerCase, CPU, DecodedFeature))
-    return false;
-
-  return getAArch64MicroArchFeaturesFromMtune(D, CPU, Args, Features);
+  return getAArch64MicroArchFeaturesFromMtune(D, Mcpu, Args, Features);
 }
 
 void aarch64::getAArch64TargetFeatures(const Driver &D,
diff --git a/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp b/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
index 33a655870b01..ee7b0d10c24b 100644
--- a/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/LoongArch.cpp
@@ -214,16 +214,16 @@ void loongarch::getLoongArchTargetFeatures(const Driver &D,
     if (MSIMD == "lsx") {
       // Option -msimd=lsx depends on 64-bit FPU.
       // -m*-float and -mfpu=none/0/32 conflict with -msimd=lsx.
-      if (llvm::find(Features, "-d") != Features.end())
+      if (llvm::is_contained(Features, "-d"))
         D.Diag(diag::err_drv_loongarch_wrong_fpu_width) << /*LSX*/ 0;
       else
         Features.push_back("+lsx");
     } else if (MSIMD == "lasx") {
       // Option -msimd=lasx depends on 64-bit FPU and LSX.
       // -m*-float, -mfpu=none/0/32 and -mno-lsx conflict with -msimd=lasx.
-      if (llvm::find(Features, "-d") != Features.end())
+      if (llvm::is_contained(Features, "-d"))
         D.Diag(diag::err_drv_loongarch_wrong_fpu_width) << /*LASX*/ 1;
-      else if (llvm::find(Features, "-lsx") != Features.end())
+      else if (llvm::is_contained(Features, "-lsx"))
         D.Diag(diag::err_drv_loongarch_invalid_simd_option_combination);
 
       // The command options do not contain -mno-lasx.
@@ -232,9 +232,9 @@ void loongarch::getLoongArchTargetFeatures(const Driver &D,
         Features.push_back("+lasx");
       }
     } else if (MSIMD == "none") {
-      if (llvm::find(Features, "+lsx") != Features.end())
+      if (llvm::is_contained(Features, "+lsx"))
         Features.push_back("-lsx");
-      if (llvm::find(Features, "+lasx") != Features.end())
+      if (llvm::is_contained(Features, "+lasx"))
         Features.push_back("-lasx");
     } else {
       D.Diag(diag::err_drv_loongarch_invalid_msimd_EQ) << MSIMD;
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index d8168ed15feb..e670696cd59a 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -31,6 +31,40 @@ using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 
+/// Is the triple {aarch64.aarch64_be}-none-elf?
+static bool isAArch64BareMetal(const llvm::Triple &Triple) {
+  if (Triple.getArch() != llvm::Triple::aarch64 &&
+      Triple.getArch() != llvm::Triple::aarch64_be)
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+static bool isRISCVBareMetal(const llvm::Triple &Triple) {
+  if (!Triple.isRISCV())
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+/// Is the triple powerpc[64][le]-*-none-eabi?
+static bool isPPCBareMetal(const llvm::Triple &Triple) {
+  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
+         Triple.getEnvironment() == llvm::Triple::EABI;
+}
+
 static bool findRISCVMultilibs(const Driver &D,
                                const llvm::Triple &TargetTriple,
                                const ArgList &Args, DetectedMultilibs &Result) {
@@ -95,7 +129,8 @@ static bool findRISCVMultilibs(const Driver &D,
   return false;
 }
 
-static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
+static std::string computeClangRuntimesSysRoot(const Driver &D,
+                                               bool IncludeTriple) {
   if (!D.SysRoot.empty())
     return D.SysRoot;
 
@@ -108,56 +143,133 @@ static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
   return std::string(SysRootDir);
 }
 
-BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
-                     const ArgList &Args)
-    : ToolChain(D, Triple, Args),
-      SysRoot(computeBaseSysRoot(D, /*IncludeTriple=*/true)) {
-  getProgramPaths().push_back(getDriver().Dir);
-
-  findMultilibs(D, Triple, Args);
-  SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
-    for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
-      llvm::sys::path::append(Dir, M.osSuffix(), "lib");
-      getFilePaths().push_back(std::string(Dir));
-      getLibraryPaths().push_back(std::string(Dir));
-    }
+// Only consider the GCC toolchain based on the values provided through the
+// `--gcc-toolchain` and `--gcc-install-dir` flags. The function below returns
+// whether the GCC toolchain was initialized successfully.
+bool BareMetal::initGCCInstallation(const llvm::Triple &Triple,
+                                    const llvm::opt::ArgList &Args) {
+  if (Args.getLastArg(options::OPT_gcc_toolchain) ||
+      Args.getLastArg(clang::driver::options::OPT_gcc_install_dir_EQ)) {
+    GCCInstallation.init(Triple, Args);
+    return GCCInstallation.isValid();
   }
+  return false;
 }
 
-/// Is the triple {aarch64.aarch64_be}-none-elf?
-static bool isAArch64BareMetal(const llvm::Triple &Triple) {
-  if (Triple.getArch() != llvm::Triple::aarch64 &&
-      Triple.getArch() != llvm::Triple::aarch64_be)
-    return false;
+// This logic is adapted from RISCVToolChain.cpp as part of the ongoing effort
+// to merge RISCVToolChain into the Baremetal toolchain. It infers the presence
+// of a valid GCC toolchain by checking whether the `crt0.o` file exists in the
+// `bin/../<target-triple>/lib` directory.
+static bool detectGCCToolchainAdjacent(const Driver &D) {
+  SmallString<128> GCCDir;
+  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
+                          "lib/crt0.o");
+  return llvm::sys::fs::exists(GCCDir);
+}
 
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
+// If no sysroot is provided the driver will first attempt to infer it from the
+// values of `--gcc-install-dir` or `--gcc-toolchain`, which specify the
+// location of a GCC toolchain.
+// If neither flag is used, the sysroot defaults to either:
+//    - `bin/../<target-triple>`
+//    - `bin/../lib/clang-runtimes/<target-triple>`
+//
+// To use the `clang-runtimes` path, ensure that `../<target-triple>/lib/crt0.o`
+// does not exist relative to the driver.
+std::string BareMetal::computeSysRoot() const {
+  // Use Baremetal::sysroot if it has already been set.
+  if (!SysRoot.empty())
+    return SysRoot;
+
+  // Use the sysroot specified via the `--sysroot` command-line flag, if
+  // provided.
+  const Driver &D = getDriver();
+  if (!D.SysRoot.empty())
+    return D.SysRoot;
 
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
+  // Attempt to infer sysroot from a valid GCC installation.
+  // If no valid GCC installation, check for a GCC toolchain alongside Clang.
+  SmallString<128> inferredSysRoot;
+  if (IsGCCInstallationValid) {
+    llvm::sys::path::append(inferredSysRoot, GCCInstallation.getParentLibPath(),
+                            "..", GCCInstallation.getTriple().str());
+  } else if (detectGCCToolchainAdjacent(D)) {
+    // Use the triple as provided to the driver. Unlike the parsed triple
+    // this has not been normalized to always contain every field.
+    llvm::sys::path::append(inferredSysRoot, D.Dir, "..", D.getTargetTriple());
+  }
+  // If a valid sysroot was inferred and exists, use it
+  if (!inferredSysRoot.empty() && llvm::sys::fs::exists(inferredSysRoot))
+    return std::string(inferredSysRoot);
 
-  return Triple.getEnvironmentName() == "elf";
+  // Use the clang-runtimes path.
+  return computeClangRuntimesSysRoot(D, /*IncludeTriple*/ true);
 }
 
-static bool isRISCVBareMetal(const llvm::Triple &Triple) {
-  if (!Triple.isRISCV())
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
+std::string BareMetal::getCompilerRTPath() const {
+  const Driver &D = getDriver();
+  if (IsGCCInstallationValid || detectGCCToolchainAdjacent(getDriver())) {
+    SmallString<128> Path(D.ResourceDir);
+    llvm::sys::path::append(Path, "lib");
+    return std::string(Path.str());
+  }
+  return ToolChain::getCompilerRTPath();
+}
 
-  return Triple.getEnvironmentName() == "elf";
+static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
+                                  const Multilib &Multilib,
+                                  StringRef InstallPath,
+                                  ToolChain::path_list &Paths) {
+  if (const auto &PathsCallback = Multilibs.filePathsCallback())
+    for (const auto &Path : PathsCallback(Multilib))
+      addPathIfExists(D, InstallPath + Path, Paths);
 }
 
-/// Is the triple powerpc[64][le]-*-none-eabi?
-static bool isPPCBareMetal(const llvm::Triple &Triple) {
-  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
-         Triple.getEnvironment() == llvm::Triple::EABI;
+// GCC mutltilibs will only work for those targets that have their multlib
+// structure encoded into GCCInstallation. Baremetal toolchain supports ARM,
+// AArch64, RISCV and PPC and of these only RISCV have GCC multilibs hardcoded
+// in GCCInstallation.
+BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
+                     const ArgList &Args)
+    : Generic_ELF(D, Triple, Args) {
+  IsGCCInstallationValid = initGCCInstallation(Triple, Args);
+  std::string ComputedSysRoot = computeSysRoot();
+  if (IsGCCInstallationValid) {
+    if (!isRISCVBareMetal(Triple))
+      D.Diag(clang::diag::warn_drv_multilib_not_available_for_target);
+
+    Multilibs = GCCInstallation.getMultilibs();
+    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
+
+    path_list &Paths = getFilePaths();
+    // Add toolchain/multilib specific file paths.
+    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
+                          GCCInstallation.getInstallPath(), Paths);
+    // Adding filepath for locating crt{begin,end}.o files.
+    Paths.push_back(GCCInstallation.getInstallPath().str());
+    // Adding filepath for locating crt0.o file.
+    Paths.push_back(ComputedSysRoot + "/lib");
+
+    ToolChain::path_list &PPaths = getProgramPaths();
+    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
+    // directory off of the parent of the GCC installation.
+    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
+                           GCCInstallation.getTriple().str() + "/bin")
+                         .str());
+    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
+  } else {
+    getProgramPaths().push_back(getDriver().Dir);
+    findMultilibs(D, Triple, Args);
+    const SmallString<128> SysRootDir(computeSysRoot());
+    if (!SysRootDir.empty()) {
+      for (const Multilib &M : getOrderedMultilibs()) {
+        SmallString<128> Dir(SysRootDir);
+        llvm::sys::path::append(Dir, M.osSuffix(), "lib");
+        getFilePaths().push_back(std::string(Dir));
+        getLibraryPaths().push_back(std::string(Dir));
+      }
+    }
+  }
 }
 
 static void
@@ -216,7 +328,7 @@ getMultilibConfigPath(const Driver &D, const llvm::Triple &Triple,
       return {};
     }
   } else {
-    MultilibPath = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    MultilibPath = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     llvm::sys::path::append(MultilibPath, MultilibFilename);
   }
   return MultilibPath;
@@ -234,7 +346,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
   if (D.getVFS().exists(*MultilibPath)) {
     // If multilib.yaml is found, update sysroot so it doesn't use a target
     // specific suffix
-    SysRoot = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    SysRoot = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     SmallVector<StringRef> CustomFlagMacroDefines;
     findMultilibsFromYAML(*this, D, *MultilibPath, Args, Result,
                           CustomFlagMacroDefines);
@@ -242,7 +354,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
     Multilibs = Result.Multilibs;
     MultilibMacroDefines.append(CustomFlagMacroDefines.begin(),
                                 CustomFlagMacroDefines.end());
-  } else if (isRISCVBareMetal(Triple)) {
+  } else if (isRISCVBareMetal(Triple) && !detectGCCToolchainAdjacent(D)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
@@ -263,8 +375,6 @@ Tool *BareMetal::buildStaticLibTool() const {
   return new tools::baremetal::StaticLibTool(*this);
 }
 
-std::string BareMetal::computeSysRoot() const { return SysRoot; }
-
 BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   // Get multilibs in reverse order because they're ordered most-specific last.
   if (!SelectedMultilibs.empty())
@@ -275,6 +385,28 @@ BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   return llvm::reverse(Default);
 }
 
+ToolChain::CXXStdlibType BareMetal::GetDefaultCXXStdlibType() const {
+  if (getTriple().isRISCV() && IsGCCInstallationValid)
+    return ToolChain::CST_Libstdcxx;
+  return ToolChain::CST_Libcxx;
+}
+
+ToolChain::RuntimeLibType BareMetal::GetDefaultRuntimeLibType() const {
+  if (getTriple().isRISCV() && IsGCCInstallationValid)
+    return ToolChain::RLT_Libgcc;
+  return ToolChain::RLT_CompilerRT;
+}
+
+// TODO: Add a validity check for GCCInstallation.
+//       If valid, use `UNW_Libgcc`; otherwise, use `UNW_None`.
+ToolChain::UnwindLibType
+BareMetal::GetUnwindLibType(const llvm::opt::ArgList &Args) const {
+  if (getTriple().isRISCV())
+    return ToolChain::UNW_None;
+
+  return ToolChain::GetUnwindLibType(Args);
+}
+
 void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                           ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc))
@@ -292,10 +424,10 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (std::optional<std::string> Path = getStdlibIncludePath())
     addSystemInclude(DriverArgs, CC1Args, *Path);
 
-  const SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
+  const SmallString<128> SysRootDir(computeSysRoot());
+  if (!SysRootDir.empty()) {
     for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
+      SmallString<128> Dir(SysRootDir);
       llvm::sys::path::append(Dir, M.includeSuffix());
       llvm::sys::path::append(Dir, "include");
       addSystemInclude(DriverArgs, CC1Args, Dir.str());
@@ -309,6 +441,19 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
   CC1Args.push_back("-nostdsysteminc");
 }
 
+void BareMetal::addLibStdCxxIncludePaths(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args) const {
+  if (!IsGCCInstallationValid)
+    return;
+  const GCCVersion &Version = GCCInstallation.getVersion();
+  StringRef TripleStr = GCCInstallation.getTriple().str();
+  const Multilib &Multilib = GCCInstallation.getMultilib();
+  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
+                           TripleStr, Multilib.includeSuffix(), DriverArgs,
+                           CC1Args);
+}
+
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
@@ -339,23 +484,23 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   };
 
   switch (GetCXXStdlibType(DriverArgs)) {
-    case ToolChain::CST_Libcxx: {
-      SmallString<128> P(D.Dir);
-      llvm::sys::path::append(P, "..", "include");
-      AddCXXIncludePath(P);
-      break;
-    }
-    case ToolChain::CST_Libstdcxx:
-      // We only support libc++ toolchain installation.
-      break;
+  case ToolChain::CST_Libcxx: {
+    SmallString<128> P(D.Dir);
+    llvm::sys::path::append(P, "..", "include");
+    AddCXXIncludePath(P);
+    break;
+  }
+  case ToolChain::CST_Libstdcxx:
+    addLibStdCxxIncludePaths(DriverArgs, CC1Args);
+    break;
   }
 
-  std::string SysRoot(computeSysRoot());
-  if (SysRoot.empty())
+  std::string SysRootDir(computeSysRoot());
+  if (SysRootDir.empty())
     return;
 
   for (const Multilib &M : getOrderedMultilibs()) {
-    SmallString<128> Dir(SysRoot);
+    SmallString<128> Dir(SysRootDir);
     llvm::sys::path::append(Dir, M.gccSuffix());
     switch (GetCXXStdlibType(DriverArgs)) {
     case ToolChain::CST_Libcxx: {
@@ -455,12 +600,24 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const llvm::Triple::ArchType Arch = TC.getArch();
   const llvm::Triple &Triple = getToolChain().getEffectiveTriple();
 
-  AddLinkerInputs(TC, Inputs, Args, CmdArgs, JA);
+  if (!D.SysRoot.empty())
+    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   CmdArgs.push_back("-Bstatic");
 
-  if (TC.getTriple().isRISCV() && Args.hasArg(options::OPT_mno_relax))
-    CmdArgs.push_back("--no-relax");
+  if (const char *LDMOption = getLDMOption(TC.getTriple(), Args)) {
+    CmdArgs.push_back("-m");
+    CmdArgs.push_back(LDMOption);
+  } else {
+    D.Diag(diag::err_target_unknown_triple) << Triple.str();
+    return;
+  }
+
+  if (Triple.isRISCV()) {
+    CmdArgs.push_back("-X");
+    if (Args.hasArg(options::OPT_mno_relax))
+      CmdArgs.push_back("--no-relax");
+  }
 
   if (Triple.isARM() || Triple.isThumb()) {
     bool IsBigEndian = arm::isARMBigEndian(Triple, Args);
@@ -471,19 +628,48 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Arch == llvm::Triple::aarch64_be ? "-EB" : "-EL");
   }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
-                   options::OPT_r)) {
-    CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath("crt0.o")));
+  bool NeedCRTs =
+      !Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles);
+
+  const char *CRTBegin, *CRTEnd;
+  if (NeedCRTs) {
+    if (!Args.hasArg(options::OPT_r))
+      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath("crt0.o")));
+    if (TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D)) {
+      auto RuntimeLib = TC.GetRuntimeLibType(Args);
+      switch (RuntimeLib) {
+      case (ToolChain::RLT_Libgcc): {
+        CRTBegin = "crtbegin.o";
+        CRTEnd = "crtend.o";
+        break;
+      }
+      case (ToolChain::RLT_CompilerRT): {
+        CRTBegin =
+            TC.getCompilerRTArgString(Args, "crtbegin", ToolChain::FT_Object);
+        CRTEnd =
+            TC.getCompilerRTArgString(Args, "crtend", ToolChain::FT_Object);
+        break;
+      }
+      }
+      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(CRTBegin)));
+    }
   }
 
-  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
-                            options::OPT_s, options::OPT_t, options::OPT_r});
+  Args.addAllArgs(CmdArgs,
+                  {options::OPT_L, options::OPT_u, options::OPT_T_Group,
+                   options::OPT_s, options::OPT_t, options::OPT_r});
 
   TC.AddFilePathLibArgs(Args, CmdArgs);
 
   for (const auto &LibPath : TC.getLibraryPaths())
     CmdArgs.push_back(Args.MakeArgString(llvm::Twine("-L", LibPath)));
 
+  if (D.isUsingLTO())
+    addLTOOptions(TC, Args, CmdArgs, Output, Inputs,
+                  D.getLTOMode() == LTOK_Thin);
+
+  AddLinkerInputs(TC, Inputs, Args, CmdArgs, JA);
+
   if (TC.ShouldLinkCXXStdlib(Args)) {
     bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
                                !Args.hasArg(options::OPT_static);
@@ -496,14 +682,17 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
+    CmdArgs.push_back("--start-group");
     AddRunTimeLibs(TC, D, CmdArgs, Args);
-
     CmdArgs.push_back("-lc");
+    if (TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D))
+      CmdArgs.push_back("-lgloss");
+    CmdArgs.push_back("--end-group");
   }
 
-  if (D.isUsingLTO())
-    addLTOOptions(TC, Args, CmdArgs, Output, Inputs,
-                  D.getLTOMode() == LTOK_Thin);
+  if ((TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D)) &&
+      NeedCRTs)
+    CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(CRTEnd)));
 
   if (TC.getTriple().isRISCV())
     CmdArgs.push_back("-X");
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index f6295bda0a6a..d3d415b337a0 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 
+#include "ToolChains/Gnu.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 
@@ -19,7 +20,7 @@ namespace driver {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
+class LLVM_LIBRARY_VISIBILITY BareMetal : public Generic_ELF {
 public:
   BareMetal(const Driver &D, const llvm::Triple &Triple,
             const llvm::opt::ArgList &Args);
@@ -35,7 +36,9 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
   Tool *buildStaticLibTool() const override;
 
 public:
-  bool useIntegratedAs() const override { return true; }
+  bool initGCCInstallation(const llvm::Triple &Triple,
+                           const llvm::opt::ArgList &Args);
+  bool hasValidGCCInstallation() const { return IsGCCInstallationValid; }
   bool isBareMetal() const override { return true; }
   bool isCrossCompiling() const override { return true; }
   bool HasNativeLLVMSupport() const override { return true; }
@@ -48,14 +51,16 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
 
   StringRef getOSLibName() const override { return "baremetal"; }
 
-  RuntimeLibType GetDefaultRuntimeLibType() const override {
-    return ToolChain::RLT_CompilerRT;
-  }
-  CXXStdlibType GetDefaultCXXStdlibType() const override {
-    return ToolChain::CST_Libcxx;
+  UnwindTableLevel
+  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override {
+    return UnwindTableLevel::None;
   }
 
-  const char *getDefaultLinker() const override { return "ld.lld"; }
+  CXXStdlibType GetDefaultCXXStdlibType() const override;
+
+  RuntimeLibType GetDefaultRuntimeLibType() const override;
+
+  UnwindLibType GetUnwindLibType(const llvm::opt::ArgList &Args) const override;
 
   void
   AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
@@ -67,7 +72,11 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
   void AddClangCXXStdlibIncludeArgs(
       const llvm::opt::ArgList &DriverArgs,
       llvm::opt::ArgStringList &CC1Args) const override;
+  void
+  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
+                           llvm::opt::ArgStringList &CC1Args) const override;
   std::string computeSysRoot() const override;
+  std::string getCompilerRTPath() const override;
   SanitizerMask getSupportedSanitizers() const override;
 
   SmallVector<std::string>
@@ -80,6 +89,8 @@ class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
 
   std::string SysRoot;
 
+  bool IsGCCInstallationValid;
+
   SmallVector<std::string> MultilibMacroDefines;
 };
 
@@ -104,7 +115,7 @@ class LLVM_LIBRARY_VISIBILITY StaticLibTool : public Tool {
 
 class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
 public:
-  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "ld.lld", TC) {}
+  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "linker", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index bb0f6533ac92..0a9a5acaead3 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1707,7 +1707,8 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
   }
 
   // Handle -msve_vector_bits=<bits>
-  if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ)) {
+  auto HandleVectorBits = [&](Arg *A, StringRef VScaleMin,
+                              StringRef VScaleMax) {
     StringRef Val = A->getValue();
     const Driver &D = getToolChain().getDriver();
     if (Val == "128" || Val == "256" || Val == "512" || Val == "1024" ||
@@ -1715,22 +1716,31 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
         Val == "1024+" || Val == "2048+") {
       unsigned Bits = 0;
       if (!Val.consume_back("+")) {
-        bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
+        bool Invalid = Val.getAsInteger(10, Bits);
+        (void)Invalid;
         assert(!Invalid && "Failed to parse value");
         CmdArgs.push_back(
-            Args.MakeArgString("-mvscale-max=" + llvm::Twine(Bits / 128)));
+            Args.MakeArgString(VScaleMax + llvm::Twine(Bits / 128)));
       }
 
-      bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
+      bool Invalid = Val.getAsInteger(10, Bits);
+      (void)Invalid;
       assert(!Invalid && "Failed to parse value");
+
       CmdArgs.push_back(
-          Args.MakeArgString("-mvscale-min=" + llvm::Twine(Bits / 128)));
-    // Silently drop requests for vector-length agnostic code as it's implied.
-    } else if (Val != "scalable")
+          Args.MakeArgString(VScaleMin + llvm::Twine(Bits / 128)));
+    } else if (Val == "scalable") {
+      // Silently drop requests for vector-length agnostic code as it's implied.
+    } else {
       // Handle the unsupported values passed to msve-vector-bits.
       D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getSpelling() << Val;
-  }
+    }
+  };
+  if (Arg *A = Args.getLastArg(options::OPT_msve_vector_bits_EQ))
+    HandleVectorBits(A, "-mvscale-min=", "-mvscale-max=");
+  if (Arg *A = Args.getLastArg(options::OPT_msve_streaming_vector_bits_EQ))
+    HandleVectorBits(A, "-mvscale-streaming-min=", "-mvscale-streaming-max=");
 
   AddAAPCSVolatileBitfieldArgs(Args, CmdArgs);
 
@@ -3871,16 +3881,18 @@ static void RenderOpenCLOptions(const ArgList &Args, ArgStringList &CmdArgs,
 
 static void RenderHLSLOptions(const ArgList &Args, ArgStringList &CmdArgs,
                               types::ID InputType) {
-  const unsigned ForwardedArguments[] = {options::OPT_dxil_validator_version,
-                                         options::OPT_res_may_alias,
-                                         options::OPT_D,
-                                         options::OPT_I,
-                                         options::OPT_O,
-                                         options::OPT_emit_llvm,
-                                         options::OPT_emit_obj,
-                                         options::OPT_disable_llvm_passes,
-                                         options::OPT_fnative_half_type,
-                                         options::OPT_hlsl_entrypoint};
+  const unsigned ForwardedArguments[] = {
+      options::OPT_dxil_validator_version,
+      options::OPT_res_may_alias,
+      options::OPT_D,
+      options::OPT_I,
+      options::OPT_O,
+      options::OPT_emit_llvm,
+      options::OPT_emit_obj,
+      options::OPT_disable_llvm_passes,
+      options::OPT_fnative_half_type,
+      options::OPT_hlsl_entrypoint,
+      options::OPT_fdx_rootsignature_version};
   if (!types::isHLSL(InputType))
     return;
   for (const auto &Arg : ForwardedArguments)
@@ -4674,11 +4686,8 @@ renderDebugOptions(const ToolChain &TC, const Driver &D, const llvm::Triple &T,
   }
 
   if (Args.hasFlag(options::OPT_gkey_instructions,
-                   options::OPT_gno_key_instructions, false)) {
+                   options::OPT_gno_key_instructions, false))
     CmdArgs.push_back("-gkey-instructions");
-    CmdArgs.push_back("-mllvm");
-    CmdArgs.push_back("-dwarf-use-key-instructions");
-  }
 
   if (EmitCodeView) {
     CmdArgs.push_back("-gcodeview");
@@ -6109,7 +6118,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -fasynchronous-unwind-tables and -fnon-call-exceptions interact in more
   // complicated ways.
   auto SanitizeArgs = TC.getSanitizerArgs(Args);
-
+  Args.AddLastArg(CmdArgs,
+                  options::OPT_fallow_runtime_check_skip_hot_cutoff_EQ);
   bool IsAsyncUnwindTablesDefault =
       TC.getDefaultUnwindTableLevel(Args) == ToolChain::UnwindTableLevel::Asynchronous;
   bool IsSyncUnwindTablesDefault =
@@ -6368,7 +6378,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fclang_abi_compat_EQ);
 
   if (getLastProfileSampleUseArg(Args) &&
-      Args.hasArg(options::OPT_fsample_profile_use_profi)) {
+      Args.hasFlag(options::OPT_fsample_profile_use_profi,
+                   options::OPT_fno_sample_profile_use_profi, false)) {
     CmdArgs.push_back("-mllvm");
     CmdArgs.push_back("-sample-profile-use-profi");
   }
@@ -6974,6 +6985,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_fms_hotpatch);
 
+  if (Args.hasArg(options::OPT_fms_secure_hotpatch_functions_file))
+    Args.AddLastArg(CmdArgs, options::OPT_fms_secure_hotpatch_functions_file);
+
+  for (const auto &A :
+       Args.getAllArgValues(options::OPT_fms_secure_hotpatch_functions_list))
+    CmdArgs.push_back(
+        Args.MakeArgString("-fms-secure-hotpatch-functions-list=" + Twine(A)));
+
   if (TC.SupportsProfiling()) {
     Args.AddLastArg(CmdArgs, options::OPT_pg);
 
@@ -7365,6 +7384,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       LanguageStandard = llvm::StringSwitch<StringRef>(StdArg->getValue())
                              .Case("c11", "-std=c11")
                              .Case("c17", "-std=c17")
+                             // TODO: add c23 when MSVC supports it.
+                             .Case("clatest", "-std=c23")
                              .Default("");
       if (LanguageStandard.empty())
         D.Diag(clang::diag::warn_drv_unused_argument)
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 71de2a7e25da..54e081cf5c4f 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -529,6 +529,78 @@ void tools::AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
   }
 }
 
+const char *tools::getLDMOption(const llvm::Triple &T, const ArgList &Args) {
+  switch (T.getArch()) {
+  case llvm::Triple::x86:
+    if (T.isOSIAMCU())
+      return "elf_iamcu";
+    return "elf_i386";
+  case llvm::Triple::aarch64:
+    if (T.isOSManagarm())
+      return "aarch64managarm";
+    return "aarch64linux";
+  case llvm::Triple::aarch64_be:
+    return "aarch64linuxb";
+  case llvm::Triple::arm:
+  case llvm::Triple::thumb:
+  case llvm::Triple::armeb:
+  case llvm::Triple::thumbeb:
+    return tools::arm::isARMBigEndian(T, Args) ? "armelfb_linux_eabi"
+                                               : "armelf_linux_eabi";
+  case llvm::Triple::m68k:
+    return "m68kelf";
+  case llvm::Triple::ppc:
+    if (T.isOSLinux())
+      return "elf32ppclinux";
+    return "elf32ppc";
+  case llvm::Triple::ppcle:
+    if (T.isOSLinux())
+      return "elf32lppclinux";
+    return "elf32lppc";
+  case llvm::Triple::ppc64:
+    return "elf64ppc";
+  case llvm::Triple::ppc64le:
+    return "elf64lppc";
+  case llvm::Triple::riscv32:
+    return "elf32lriscv";
+  case llvm::Triple::riscv64:
+    return "elf64lriscv";
+  case llvm::Triple::sparc:
+  case llvm::Triple::sparcel:
+    return "elf32_sparc";
+  case llvm::Triple::sparcv9:
+    return "elf64_sparc";
+  case llvm::Triple::loongarch32:
+    return "elf32loongarch";
+  case llvm::Triple::loongarch64:
+    return "elf64loongarch";
+  case llvm::Triple::mips:
+    return "elf32btsmip";
+  case llvm::Triple::mipsel:
+    return "elf32ltsmip";
+  case llvm::Triple::mips64:
+    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
+      return "elf32btsmipn32";
+    return "elf64btsmip";
+  case llvm::Triple::mips64el:
+    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
+      return "elf32ltsmipn32";
+    return "elf64ltsmip";
+  case llvm::Triple::systemz:
+    return "elf64_s390";
+  case llvm::Triple::x86_64:
+    if (T.isX32())
+      return "elf32_x86_64";
+    return "elf_x86_64";
+  case llvm::Triple::ve:
+    return "elf64ve";
+  case llvm::Triple::csky:
+    return "cskyelf_linux";
+  default:
+    return nullptr;
+  }
+}
+
 void tools::addLinkerCompressDebugSectionsOption(
     const ToolChain &TC, const llvm::opt::ArgList &Args,
     llvm::opt::ArgStringList &CmdArgs) {
@@ -1470,7 +1542,7 @@ void tools::addOpenMPHostOffloadingArgs(const Compilation &C,
 
   // For all the host OpenMP offloading compile jobs we need to pass the targets
   // information using -fopenmp-targets= option.
-  constexpr llvm::StringLiteral Targets("-fopenmp-targets=");
+  constexpr llvm::StringLiteral Targets("--offload-targets=");
 
   SmallVector<std::string> Triples;
   auto TCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index b4b337b2476e..6701bfb4cb7a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -492,11 +492,18 @@ void Flang::addTargetOptions(const ArgList &Args,
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
-    } else if (Name == "libmvec" || Name == "AMDLIBM") {
+    } else if (Name == "AMDLIBM") {
       if (Triple.getArch() != llvm::Triple::x86 &&
           Triple.getArch() != llvm::Triple::x86_64)
         D.Diag(diag::err_drv_unsupported_opt_for_target)
             << Name << Triple.getArchName();
+    } else if (Name == "libmvec") {
+      if (Triple.getArch() != llvm::Triple::x86 &&
+          Triple.getArch() != llvm::Triple::x86_64 &&
+          Triple.getArch() != llvm::Triple::aarch64 &&
+          Triple.getArch() != llvm::Triple::aarch64_be)
+        D.Diag(diag::err_drv_unsupported_opt_for_target)
+            << Name << Triple.getArchName();
     } else if (Name == "SLEEF" || Name == "ArmPL") {
       if (Triple.getArch() != llvm::Triple::aarch64 &&
           Triple.getArch() != llvm::Triple::aarch64_be)
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index fce00cf06bfa..34d9e08dc2a6 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -220,78 +220,6 @@ void tools::gcc::Linker::RenderExtraToolArgs(const JobAction &JA,
   // The types are (hopefully) good enough.
 }
 
-static const char *getLDMOption(const llvm::Triple &T, const ArgList &Args) {
-  switch (T.getArch()) {
-  case llvm::Triple::x86:
-    if (T.isOSIAMCU())
-      return "elf_iamcu";
-    return "elf_i386";
-  case llvm::Triple::aarch64:
-    if (T.isOSManagarm())
-      return "aarch64managarm";
-    return "aarch64linux";
-  case llvm::Triple::aarch64_be:
-    return "aarch64linuxb";
-  case llvm::Triple::arm:
-  case llvm::Triple::thumb:
-  case llvm::Triple::armeb:
-  case llvm::Triple::thumbeb:
-    return tools::arm::isARMBigEndian(T, Args) ? "armelfb_linux_eabi"
-                                               : "armelf_linux_eabi";
-  case llvm::Triple::m68k:
-    return "m68kelf";
-  case llvm::Triple::ppc:
-    if (T.isOSLinux())
-      return "elf32ppclinux";
-    return "elf32ppc";
-  case llvm::Triple::ppcle:
-    if (T.isOSLinux())
-      return "elf32lppclinux";
-    return "elf32lppc";
-  case llvm::Triple::ppc64:
-    return "elf64ppc";
-  case llvm::Triple::ppc64le:
-    return "elf64lppc";
-  case llvm::Triple::riscv32:
-    return "elf32lriscv";
-  case llvm::Triple::riscv64:
-    return "elf64lriscv";
-  case llvm::Triple::sparc:
-  case llvm::Triple::sparcel:
-    return "elf32_sparc";
-  case llvm::Triple::sparcv9:
-    return "elf64_sparc";
-  case llvm::Triple::loongarch32:
-    return "elf32loongarch";
-  case llvm::Triple::loongarch64:
-    return "elf64loongarch";
-  case llvm::Triple::mips:
-    return "elf32btsmip";
-  case llvm::Triple::mipsel:
-    return "elf32ltsmip";
-  case llvm::Triple::mips64:
-    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
-      return "elf32btsmipn32";
-    return "elf64btsmip";
-  case llvm::Triple::mips64el:
-    if (tools::mips::hasMipsAbiArg(Args, "n32") || T.isABIN32())
-      return "elf32ltsmipn32";
-    return "elf64ltsmip";
-  case llvm::Triple::systemz:
-    return "elf64_s390";
-  case llvm::Triple::x86_64:
-    if (T.isX32())
-      return "elf32_x86_64";
-    return "elf_x86_64";
-  case llvm::Triple::ve:
-    return "elf64ve";
-  case llvm::Triple::csky:
-    return "cskyelf_linux";
-  default:
-    return nullptr;
-  }
-}
-
 static bool getStaticPIE(const ArgList &Args, const ToolChain &TC) {
   bool HasStaticPIE = Args.hasArg(options::OPT_static_pie);
   if (HasStaticPIE && Args.hasArg(options::OPT_no_pie)) {
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 891789472b0c..381625d9bb9e 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -386,19 +386,22 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {
-    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+    for (StringRef BCName : BCLibArgs) {
       StringRef FullName;
+      bool Found = false;
       for (StringRef LibraryPath : LibraryPaths) {
         SmallString<128> Path(LibraryPath);
         llvm::sys::path::append(Path, BCName);
         FullName = Path;
         if (llvm::sys::fs::exists(FullName)) {
           BCLibs.emplace_back(FullName);
-          return;
+          Found = true;
+          break;
         }
       }
-      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
-    });
+      if (!Found)
+        getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    }
   } else {
     if (!RocmInstallation->hasDeviceLibrary()) {
       getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
diff --git a/clang/lib/Driver/ToolChains/HIPSPV.cpp b/clang/lib/Driver/ToolChains/HIPSPV.cpp
index c86790f66a79..53649ca40d99 100644
--- a/clang/lib/Driver/ToolChains/HIPSPV.cpp
+++ b/clang/lib/Driver/ToolChains/HIPSPV.cpp
@@ -149,11 +149,9 @@ void HIPSPVToolChain::addClangTargetOptions(
     CC1Args.append(
         {"-fvisibility=hidden", "-fapply-global-visibility-to-externs"});
 
-  llvm::for_each(getDeviceLibs(DriverArgs),
-                 [&](const BitCodeLibraryInfo &BCFile) {
-                   CC1Args.append({"-mlink-builtin-bitcode",
-                                   DriverArgs.MakeArgString(BCFile.Path)});
-                 });
+  for (const BitCodeLibraryInfo &BCFile : getDeviceLibs(DriverArgs))
+    CC1Args.append(
+        {"-mlink-builtin-bitcode", DriverArgs.MakeArgString(BCFile.Path)});
 }
 
 Tool *HIPSPVToolChain::buildLinker() const {
@@ -228,7 +226,8 @@ HIPSPVToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {
-    llvm::for_each(BCLibArgs, [&](StringRef BCName) {
+    bool Found = false;
+    for (StringRef BCName : BCLibArgs) {
       StringRef FullName;
       for (std::string LibraryPath : LibraryPaths) {
         SmallString<128> Path(LibraryPath);
@@ -236,11 +235,13 @@ HIPSPVToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs) const {
         FullName = Path;
         if (llvm::sys::fs::exists(FullName)) {
           BCLibs.emplace_back(FullName.str());
-          return;
+          Found = true;
+          break;
         }
       }
-      getDriver().Diag(diag::err_drv_no_such_file) << BCName;
-    });
+      if (!Found)
+        getDriver().Diag(diag::err_drv_no_such_file) << BCName;
+    }
   } else {
     // Search device library named as 'hipspv-<triple>.bc'.
     auto TT = getTriple().normalize();
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index dcc51e182924..38f4643abad9 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -182,13 +182,15 @@ std::string getSpirvExtArg(ArrayRef<std::string> SpvExtensionArgs) {
       (Twine("-spirv-ext=+") + SpvExtensionArgs.front()).str();
   SpvExtensionArgs = SpvExtensionArgs.slice(1);
   for (auto Extension : SpvExtensionArgs) {
-    LlvmOption = (Twine(LlvmOption) + ",+" + Extension).str();
+    if (Extension != "KHR")
+      Extension = (Twine("+") + Extension).str();
+    LlvmOption = (Twine(LlvmOption) + "," + Extension).str();
   }
   return LlvmOption;
 }
 
 bool isValidSPIRVExtensionName(const std::string &str) {
-  std::regex pattern("SPV_[a-zA-Z0-9_]+");
+  std::regex pattern("KHR|SPV_[a-zA-Z0-9_]+");
   return std::regex_match(str, pattern);
 }
 
@@ -200,7 +202,7 @@ bool checkExtensionArgsAreValid(ArrayRef<std::string> SpvExtensionArgs,
   for (auto Extension : SpvExtensionArgs) {
     if (!isValidSPIRVExtensionName(Extension)) {
       Driver.Diag(diag::err_drv_invalid_value)
-          << "-fspv_extension" << Extension;
+          << "-fspv-extension" << Extension;
       AllValid = false;
     }
   }
@@ -295,6 +297,13 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
       A->claim();
       continue;
     }
+    if (A->getOption().getID() == options::OPT_dxc_rootsig_ver) {
+      DAL->AddJoinedArg(nullptr,
+                        Opts.getOption(options::OPT_fdx_rootsignature_version),
+                        A->getValue());
+      A->claim();
+      continue;
+    }
     if (A->getOption().getID() == options::OPT__SLASH_O) {
       StringRef OStr = A->getValue();
       if (OStr == "d") {
@@ -330,6 +339,31 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
       A->claim();
       continue;
     }
+    if (A->getOption().getID() == options::OPT_dxc_gis) {
+      // Translate -Gis into -ffp_model_EQ=strict
+      DAL->AddSeparateArg(nullptr, Opts.getOption(options::OPT_ffp_model_EQ),
+                          "strict");
+      A->claim();
+      continue;
+    }
+    if (A->getOption().getID() == options::OPT_fvk_use_dx_layout) {
+      // This is the only implemented layout so far.
+      A->claim();
+      continue;
+    }
+
+    if (A->getOption().getID() == options::OPT_fvk_use_scalar_layout) {
+      getDriver().Diag(diag::err_drv_clang_unsupported) << A->getAsString(Args);
+      A->claim();
+      continue;
+    }
+
+    if (A->getOption().getID() == options::OPT_fvk_use_gl_layout) {
+      getDriver().Diag(diag::err_drv_clang_unsupported) << A->getAsString(Args);
+      A->claim();
+      continue;
+    }
+
     DAL->append(A);
   }
 
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index e2c88e22952a..26884a5f373c 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -111,8 +111,9 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Twine("-libpath:") + DIAPath));
   }
   if (!llvm::sys::Process::GetEnv("LIB") ||
-      Args.getLastArg(options::OPT__SLASH_vctoolsdir,
-                      options::OPT__SLASH_winsysroot)) {
+      Args.hasArg(options::OPT__SLASH_vctoolsdir,
+                  options::OPT__SLASH_vctoolsversion,
+                  options::OPT__SLASH_winsysroot)) {
     CmdArgs.push_back(Args.MakeArgString(
         Twine("-libpath:") +
         TC.getSubDirectoryPath(llvm::SubDirectoryType::Lib)));
@@ -121,8 +122,9 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
         TC.getSubDirectoryPath(llvm::SubDirectoryType::Lib, "atlmfc")));
   }
   if (!llvm::sys::Process::GetEnv("LIB") ||
-      Args.getLastArg(options::OPT__SLASH_winsdkdir,
-                      options::OPT__SLASH_winsysroot)) {
+      Args.hasArg(options::OPT__SLASH_winsdkdir,
+                  options::OPT__SLASH_winsdkversion,
+                  options::OPT__SLASH_winsysroot)) {
     if (TC.useUniversalCRT()) {
       std::string UniversalCRTLibPath;
       if (TC.getUniversalCRTLibraryPath(Args, UniversalCRTLibPath))
@@ -697,9 +699,12 @@ void MSVCToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
     return;
 
   // Honor %INCLUDE% and %EXTERNAL_INCLUDE%. It should have essential search
-  // paths set by vcvarsall.bat. Skip if the user expressly set a vctoolsdir.
-  if (!DriverArgs.getLastArg(options::OPT__SLASH_vctoolsdir,
-                             options::OPT__SLASH_winsysroot)) {
+  // paths set by vcvarsall.bat. Skip if the user expressly set any of the
+  // Windows SDK or VC Tools options.
+  if (!DriverArgs.hasArg(
+          options::OPT__SLASH_vctoolsdir, options::OPT__SLASH_vctoolsversion,
+          options::OPT__SLASH_winsysroot, options::OPT__SLASH_winsdkdir,
+          options::OPT__SLASH_winsdkversion)) {
     bool Found = AddSystemIncludesFromEnv("INCLUDE");
     Found |= AddSystemIncludesFromEnv("EXTERNAL_INCLUDE");
     if (Found)
diff --git a/clang/lib/Driver/ToolChains/Managarm.cpp b/clang/lib/Driver/ToolChains/Managarm.cpp
index ff455f2c6ec7..0f56f0f6aad7 100644
--- a/clang/lib/Driver/ToolChains/Managarm.cpp
+++ b/clang/lib/Driver/ToolChains/Managarm.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "Managarm.h"
-#include "Arch/ARM.h"
 #include "Arch/RISCV.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/CommonArgs.h"
diff --git a/clang/lib/Driver/ToolChains/PS4CPU.cpp b/clang/lib/Driver/ToolChains/PS4CPU.cpp
index e965786d269f..21e23d486f9d 100644
--- a/clang/lib/Driver/ToolChains/PS4CPU.cpp
+++ b/clang/lib/Driver/ToolChains/PS4CPU.cpp
@@ -379,8 +379,10 @@ void tools::PS5cpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
       !Relocatable &&
       !Args.hasArg(options::OPT_nostartfiles, options::OPT_nostdlib);
 
-  auto AddCRTObject = [&](const char *Name) {
-    CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(Name)));
+  auto AddCRTObject = [&](StringRef Name) {
+    // CRT objects can be found on user supplied library paths. This is
+    // an entrenched expectation on PlayStation.
+    CmdArgs.push_back(Args.MakeArgString("-l:" + Name));
   };
 
   if (AddStartFiles) {
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp b/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
deleted file mode 100644
index 42bbc130637b..000000000000
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.cpp
+++ /dev/null
@@ -1,231 +0,0 @@
-//===--- RISCVToolchain.cpp - RISC-V ToolChain Implementations --*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "RISCVToolchain.h"
-#include "clang/Driver/CommonArgs.h"
-#include "clang/Driver/Compilation.h"
-#include "clang/Driver/InputInfo.h"
-#include "clang/Driver/Options.h"
-#include "llvm/Option/ArgList.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
-
-using namespace clang::driver;
-using namespace clang::driver::toolchains;
-using namespace clang::driver::tools;
-using namespace clang;
-using namespace llvm::opt;
-
-static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
-                                  const Multilib &Multilib,
-                                  StringRef InstallPath,
-                                  ToolChain::path_list &Paths) {
-  if (const auto &PathsCallback = Multilibs.filePathsCallback())
-    for (const auto &Path : PathsCallback(Multilib))
-      addPathIfExists(D, InstallPath + Path, Paths);
-}
-
-// This function tests whether a gcc installation is present either
-// through gcc-toolchain argument or in the same prefix where clang
-// is installed. This helps decide whether to instantiate this toolchain
-// or Baremetal toolchain.
-bool RISCVToolChain::hasGCCToolchain(const Driver &D,
-                                     const llvm::opt::ArgList &Args) {
-  if (Args.getLastArg(options::OPT_gcc_toolchain))
-    return true;
-
-  SmallString<128> GCCDir;
-  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
-                          "lib/crt0.o");
-  return llvm::sys::fs::exists(GCCDir);
-}
-
-/// RISC-V Toolchain
-RISCVToolChain::RISCVToolChain(const Driver &D, const llvm::Triple &Triple,
-                               const ArgList &Args)
-    : Generic_ELF(D, Triple, Args) {
-  GCCInstallation.init(Triple, Args);
-  if (GCCInstallation.isValid()) {
-    Multilibs = GCCInstallation.getMultilibs();
-    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
-    path_list &Paths = getFilePaths();
-    // Add toolchain/multilib specific file paths.
-    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
-                          GCCInstallation.getInstallPath(), Paths);
-    getFilePaths().push_back(GCCInstallation.getInstallPath().str());
-    ToolChain::path_list &PPaths = getProgramPaths();
-    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
-    // directory off of the parent of the GCC installation.
-    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
-                           GCCInstallation.getTriple().str() + "/bin")
-                         .str());
-    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
-  } else {
-    getProgramPaths().push_back(D.Dir);
-  }
-  getFilePaths().push_back(computeSysRoot() + "/lib");
-}
-
-Tool *RISCVToolChain::buildLinker() const {
-  return new tools::RISCV::Linker(*this);
-}
-
-ToolChain::RuntimeLibType RISCVToolChain::GetDefaultRuntimeLibType() const {
-  return GCCInstallation.isValid() ?
-    ToolChain::RLT_Libgcc : ToolChain::RLT_CompilerRT;
-}
-
-ToolChain::UnwindLibType
-RISCVToolChain::GetUnwindLibType(const llvm::opt::ArgList &Args) const {
-  return ToolChain::UNW_None;
-}
-
-ToolChain::UnwindTableLevel RISCVToolChain::getDefaultUnwindTableLevel(
-    const llvm::opt::ArgList &Args) const {
-  return UnwindTableLevel::None;
-}
-
-void RISCVToolChain::addClangTargetOptions(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args,
-    Action::OffloadKind) const {
-  CC1Args.push_back("-nostdsysteminc");
-}
-
-void RISCVToolChain::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
-                                               ArgStringList &CC1Args) const {
-  if (DriverArgs.hasArg(options::OPT_nostdinc))
-    return;
-
-  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
-    SmallString<128> Dir(getDriver().ResourceDir);
-    llvm::sys::path::append(Dir, "include");
-    addSystemInclude(DriverArgs, CC1Args, Dir.str());
-  }
-
-  if (!DriverArgs.hasArg(options::OPT_nostdlibinc)) {
-    SmallString<128> Dir(computeSysRoot());
-    llvm::sys::path::append(Dir, "include");
-    addSystemInclude(DriverArgs, CC1Args, Dir.str());
-  }
-}
-
-void RISCVToolChain::addLibStdCxxIncludePaths(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args) const {
-  const GCCVersion &Version = GCCInstallation.getVersion();
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  const Multilib &Multilib = GCCInstallation.getMultilib();
-  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
-                           TripleStr, Multilib.includeSuffix(), DriverArgs,
-                           CC1Args);
-}
-
-std::string RISCVToolChain::computeSysRoot() const {
-  if (!getDriver().SysRoot.empty())
-    return getDriver().SysRoot;
-
-  SmallString<128> SysRootDir;
-  if (GCCInstallation.isValid()) {
-    StringRef LibDir = GCCInstallation.getParentLibPath();
-    StringRef TripleStr = GCCInstallation.getTriple().str();
-    llvm::sys::path::append(SysRootDir, LibDir, "..", TripleStr);
-  } else {
-    // Use the triple as provided to the driver. Unlike the parsed triple
-    // this has not been normalized to always contain every field.
-    llvm::sys::path::append(SysRootDir, getDriver().Dir, "..",
-                            getDriver().getTargetTriple());
-  }
-
-  if (!llvm::sys::fs::exists(SysRootDir))
-    return std::string();
-
-  return std::string(SysRootDir);
-}
-
-void RISCV::Linker::ConstructJob(Compilation &C, const JobAction &JA,
-                                 const InputInfo &Output,
-                                 const InputInfoList &Inputs,
-                                 const ArgList &Args,
-                                 const char *LinkingOutput) const {
-  const ToolChain &ToolChain = getToolChain();
-  const Driver &D = ToolChain.getDriver();
-  ArgStringList CmdArgs;
-
-  if (!D.SysRoot.empty())
-    CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
-
-  if (Args.hasArg(options::OPT_mno_relax))
-    CmdArgs.push_back("--no-relax");
-
-  bool IsRV64 = ToolChain.getArch() == llvm::Triple::riscv64;
-  CmdArgs.push_back("-m");
-  if (IsRV64) {
-    CmdArgs.push_back("elf64lriscv");
-  } else {
-    CmdArgs.push_back("elf32lriscv");
-  }
-  CmdArgs.push_back("-X");
-
-  std::string Linker = getToolChain().GetLinkerPath();
-
-  bool WantCRTs =
-      !Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles);
-
-  const char *crtbegin, *crtend;
-  auto RuntimeLib = ToolChain.GetRuntimeLibType(Args);
-  if (RuntimeLib == ToolChain::RLT_Libgcc) {
-    crtbegin = "crtbegin.o";
-    crtend = "crtend.o";
-  } else {
-    assert (RuntimeLib == ToolChain::RLT_CompilerRT);
-    crtbegin = ToolChain.getCompilerRTArgString(Args, "crtbegin",
-                                                ToolChain::FT_Object);
-    crtend = ToolChain.getCompilerRTArgString(Args, "crtend",
-                                              ToolChain::FT_Object);
-  }
-
-  if (WantCRTs) {
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crt0.o")));
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
-  }
-
-  AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
-
-  Args.addAllArgs(CmdArgs, {options::OPT_L, options::OPT_u});
-
-  ToolChain.AddFilePathLibArgs(Args, CmdArgs);
-  Args.addAllArgs(CmdArgs, {options::OPT_T_Group, options::OPT_s,
-                            options::OPT_t, options::OPT_r});
-
-  // TODO: add C++ includes and libs if compiling C++.
-
-  if (!Args.hasArg(options::OPT_nostdlib) &&
-      !Args.hasArg(options::OPT_nodefaultlibs)) {
-    if (D.CCCIsCXX()) {
-      if (ToolChain.ShouldLinkCXXStdlib(Args))
-        ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
-      CmdArgs.push_back("-lm");
-    }
-    CmdArgs.push_back("--start-group");
-    CmdArgs.push_back("-lc");
-    CmdArgs.push_back("-lgloss");
-    CmdArgs.push_back("--end-group");
-    AddRunTimeLibs(ToolChain, ToolChain.getDriver(), CmdArgs, Args);
-  }
-
-  if (WantCRTs)
-    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtend)));
-
-  CmdArgs.push_back("-o");
-  CmdArgs.push_back(Output.getFilename());
-  C.addCommand(std::make_unique<Command>(
-      JA, *this, ResponseFileSupport::AtFileCurCP(), Args.MakeArgString(Linker),
-      CmdArgs, Inputs, Output));
-}
-// RISCV tools end.
diff --git a/clang/lib/Driver/ToolChains/RISCVToolchain.h b/clang/lib/Driver/ToolChains/RISCVToolchain.h
deleted file mode 100644
index fa0aa265d842..000000000000
--- a/clang/lib/Driver/ToolChains/RISCVToolchain.h
+++ /dev/null
@@ -1,67 +0,0 @@
-//===--- RISCVToolchain.h - RISC-V ToolChain Implementations ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
-#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
-
-#include "Gnu.h"
-#include "clang/Driver/ToolChain.h"
-
-namespace clang {
-namespace driver {
-namespace toolchains {
-
-class LLVM_LIBRARY_VISIBILITY RISCVToolChain : public Generic_ELF {
-public:
-  RISCVToolChain(const Driver &D, const llvm::Triple &Triple,
-                 const llvm::opt::ArgList &Args);
-
-  static bool hasGCCToolchain(const Driver &D, const llvm::opt::ArgList &Args);
-  void addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
-                             llvm::opt::ArgStringList &CC1Args,
-                             Action::OffloadKind) const override;
-  RuntimeLibType GetDefaultRuntimeLibType() const override;
-  UnwindLibType
-  GetUnwindLibType(const llvm::opt::ArgList &Args) const override;
-  UnwindTableLevel
-  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override;
-  void
-  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
-                            llvm::opt::ArgStringList &CC1Args) const override;
-  void
-  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
-                           llvm::opt::ArgStringList &CC1Args) const override;
-
-protected:
-  Tool *buildLinker() const override;
-
-private:
-  std::string computeSysRoot() const override;
-};
-
-} // end namespace toolchains
-
-namespace tools {
-namespace RISCV {
-class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
-public:
-  Linker(const ToolChain &TC) : Tool("RISCV::Linker", "ld", TC) {}
-  bool hasIntegratedCPP() const override { return false; }
-  bool isLinkJob() const override { return true; }
-  void ConstructJob(Compilation &C, const JobAction &JA,
-                    const InputInfo &Output, const InputInfoList &Inputs,
-                    const llvm::opt::ArgList &TCArgs,
-                    const char *LinkingOutput) const override;
-};
-} // end namespace RISCV
-} // end namespace tools
-
-} // end namespace driver
-} // end namespace clang
-
-#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_RISCVTOOLCHAIN_H
diff --git a/clang/lib/Edit/Commit.cpp b/clang/lib/Edit/Commit.cpp
index 6e785e866666..415b1573d789 100644
--- a/clang/lib/Edit/Commit.cpp
+++ b/clang/lib/Edit/Commit.cpp
@@ -16,7 +16,6 @@
 #include "clang/Lex/PPConditionalDirectiveRecord.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
-#include <utility>
 
 using namespace clang;
 using namespace edit;
@@ -239,7 +238,7 @@ bool Commit::canInsert(SourceLocation loc, FileOffset &offs) {
   if (SM.isInSystemHeader(loc))
     return false;
 
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
+  FileIDAndOffset locInfo = SM.getDecomposedLoc(loc);
   if (locInfo.first.isInvalid())
     return false;
   offs = FileOffset(locInfo.first, locInfo.second);
@@ -273,7 +272,7 @@ bool Commit::canInsertAfterToken(SourceLocation loc, FileOffset &offs,
   if (loc.isInvalid())
     return false;
 
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
+  FileIDAndOffset locInfo = SM.getDecomposedLoc(loc);
   if (locInfo.first.isInvalid())
     return false;
   offs = FileOffset(locInfo.first, locInfo.second);
@@ -309,8 +308,8 @@ bool Commit::canRemoveRange(CharSourceRange range,
   if (PPRec && PPRec->rangeIntersectsConditionalDirective(range.getAsRange()))
     return false;
 
-  std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(range.getBegin());
-  std::pair<FileID, unsigned> endInfo = SM.getDecomposedLoc(range.getEnd());
+  FileIDAndOffset beginInfo = SM.getDecomposedLoc(range.getBegin());
+  FileIDAndOffset endInfo = SM.getDecomposedLoc(range.getEnd());
   if (beginInfo.first != endInfo.first ||
       beginInfo.second > endInfo.second)
     return false;
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index 348e7588690a..791afc1a9757 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -324,10 +324,15 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
 
   // Declaration fragments of a pointer type is the declaration fragments of
   // the pointee type followed by a `*`,
-  if (T->isPointerType() && !T->isFunctionPointerType())
-    return Fragments
-        .append(getFragmentsForType(T->getPointeeType(), Context, After))
-        .append(" *", DeclarationFragments::FragmentKind::Text);
+  if (T->isPointerType() && !T->isFunctionPointerType()) {
+    QualType PointeeT = T->getPointeeType();
+    Fragments.append(getFragmentsForType(PointeeT, Context, After));
+    // If the pointee is itself a pointer, we do not want to insert a space
+    // before the `*` as the preceding character in the type name is a `*`.
+    if (!PointeeT->isAnyPointerType())
+      Fragments.appendSpace();
+    return Fragments.append("*", DeclarationFragments::FragmentKind::Text);
+  }
 
   // For Objective-C `id` and `Class` pointers
   // we do not spell out the `*`.
@@ -631,7 +636,7 @@ DeclarationFragmentsBuilder::getFragmentsForParam(const ParmVarDecl *Param) {
                 DeclarationFragments::FragmentKind::InternalParam);
   } else {
     Fragments.append(std::move(TypeFragments));
-    if (!T->isBlockPointerType())
+    if (!T->isAnyPointerType() && !T->isBlockPointerType())
       Fragments.appendSpace();
     Fragments
         .append(Param->getName(),
@@ -706,18 +711,20 @@ DeclarationFragmentsBuilder::getFragmentsForFunction(const FunctionDecl *Func) {
 
   // FIXME: Is `after` actually needed here?
   DeclarationFragments After;
+  QualType ReturnType = Func->getReturnType();
   auto ReturnValueFragment =
-      getFragmentsForType(Func->getReturnType(), Func->getASTContext(), After);
+      getFragmentsForType(ReturnType, Func->getASTContext(), After);
   if (StringRef(ReturnValueFragment.begin()->Spelling)
           .starts_with("type-parameter")) {
-    std::string ProperArgName = Func->getReturnType().getAsString();
+    std::string ProperArgName = ReturnType.getAsString();
     ReturnValueFragment.begin()->Spelling.swap(ProperArgName);
   }
 
-  Fragments.append(std::move(ReturnValueFragment))
-      .appendSpace()
-      .append(Func->getNameAsString(),
-              DeclarationFragments::FragmentKind::Identifier);
+  Fragments.append(std::move(ReturnValueFragment));
+  if (!ReturnType->isAnyPointerType())
+    Fragments.appendSpace();
+  Fragments.append(Func->getNameAsString(),
+                   DeclarationFragments::FragmentKind::Identifier);
 
   if (Func->getTemplateSpecializationInfo()) {
     Fragments.append("<", DeclarationFragments::FragmentKind::Text);
@@ -1610,10 +1617,13 @@ DeclarationFragmentsBuilder::getFunctionSignature(const ObjCMethodDecl *);
 DeclarationFragments
 DeclarationFragmentsBuilder::getSubHeading(const NamedDecl *Decl) {
   DeclarationFragments Fragments;
-  if (isa<CXXConstructorDecl>(Decl) || isa<CXXDestructorDecl>(Decl))
+  if (isa<CXXConstructorDecl>(Decl)) {
     Fragments.append(cast<CXXRecordDecl>(Decl->getDeclContext())->getName(),
                      DeclarationFragments::FragmentKind::Identifier);
-  else if (isa<CXXConversionDecl>(Decl)) {
+  } else if (isa<CXXDestructorDecl>(Decl)) {
+    Fragments.append(cast<CXXDestructorDecl>(Decl)->getNameAsString(),
+                     DeclarationFragments::FragmentKind::Identifier);
+  } else if (isa<CXXConversionDecl>(Decl)) {
     Fragments.append(
         cast<CXXConversionDecl>(Decl)->getConversionType().getAsString(),
         DeclarationFragments::FragmentKind::Identifier);
@@ -1627,9 +1637,11 @@ DeclarationFragmentsBuilder::getSubHeading(const NamedDecl *Decl) {
   } else if (Decl->getIdentifier()) {
     Fragments.append(Decl->getName(),
                      DeclarationFragments::FragmentKind::Identifier);
-  } else
+  } else {
     Fragments.append(Decl->getDeclName().getAsString(),
                      DeclarationFragments::FragmentKind::Identifier);
+  }
+
   return Fragments;
 }
 
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 5317c05f3a46..def0d73e7753 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -158,8 +158,7 @@ getCommentSplit(StringRef Text, unsigned ContentStartColumn,
       return BreakableToken::Split(StringRef::npos, 0);
     StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim(Blanks);
     StringRef AfterCut = Text.substr(SpaceOffset);
-    // Don't trim the leading blanks if it would create a */ after the break.
-    if (!DecorationEndsWithStar || AfterCut.size() <= 1 || AfterCut[1] != '/')
+    if (!DecorationEndsWithStar)
       AfterCut = AfterCut.ltrim(Blanks);
     return BreakableToken::Split(BeforeCut.size(),
                                  AfterCut.begin() - BeforeCut.end());
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index b4745477b96e..4010f7fbd25b 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1929,6 +1929,15 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
     return;
   }
 
+  const bool EndsInComma = [](const FormatToken *Tok) {
+    if (!Tok)
+      return false;
+    const auto *Prev = Tok->getPreviousNonComment();
+    if (!Prev)
+      return false;
+    return Prev->is(tok::comma);
+  }(Current.MatchingParen);
+
   unsigned NewIndent;
   unsigned LastSpace = CurrentState.LastSpace;
   bool AvoidBinPacking;
@@ -1948,9 +1957,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       NewIndent = CurrentState.LastSpace + Style.ContinuationIndentWidth;
     }
     const FormatToken *NextNonComment = Current.getNextNonComment();
-    bool EndsInComma = Current.MatchingParen &&
-                       Current.MatchingParen->Previous &&
-                       Current.MatchingParen->Previous->is(tok::comma);
     AvoidBinPacking = EndsInComma || Current.is(TT_DictLiteral) ||
                       Style.isProto() || !Style.BinPackArguments ||
                       (NextNonComment && NextNonComment->isOneOf(
@@ -1984,11 +1990,6 @@ void ContinuationIndenter::moveStatePastScopeOpener(LineState &State,
       LastSpace = std::max(LastSpace, CurrentState.Indent);
     }
 
-    bool EndsInComma =
-        Current.MatchingParen &&
-        Current.MatchingParen->getPreviousNonComment() &&
-        Current.MatchingParen->getPreviousNonComment()->is(tok::comma);
-
     // If ObjCBinPackProtocolList is unspecified, fall back to BinPackParameters
     // for backwards compatibility.
     bool ObjCBinPackProtocolList =
diff --git a/clang/lib/Format/ContinuationIndenter.h b/clang/lib/Format/ContinuationIndenter.h
index ac354aa96f86..fe957cf43721 100644
--- a/clang/lib/Format/ContinuationIndenter.h
+++ b/clang/lib/Format/ContinuationIndenter.h
@@ -210,8 +210,8 @@ struct ParenState {
         IsChainedConditional(false), IsWrappedConditional(false),
         UnindentOperator(false) {}
 
-  /// \brief The token opening this parenthesis level, or nullptr if this level
-  /// is opened by fake parenthesis.
+  /// The token opening this parenthesis level, or nullptr if this level is
+  /// opened by fake parenthesis.
   ///
   /// Not considered for memoization as it will always have the same value at
   /// the same token.
@@ -344,16 +344,15 @@ struct ParenState {
 
   bool IsCSharpGenericTypeConstraint : 1;
 
-  /// \brief true if the current \c ParenState represents the false branch of
-  /// a chained conditional expression (e.g. else-if)
+  /// true if the current \c ParenState represents the false branch of a chained
+  /// conditional expression (e.g. else-if)
   bool IsChainedConditional : 1;
 
-  /// \brief true if there conditionnal was wrapped on the first operator (the
-  /// question mark)
+  /// true if there conditionnal was wrapped on the first operator (the question
+  /// mark)
   bool IsWrappedConditional : 1;
 
-  /// \brief Indicates the indent should be reduced by the length of the
-  /// operator.
+  /// Indicates the indent should be reduced by the length of the operator.
   bool UnindentOperator : 1;
 
   bool operator<(const ParenState &Other) const {
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index bdaf264e9adc..f0412cddc6f1 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3582,7 +3582,7 @@ tooling::Replacements sortJavaImports(const FormatStyle &Style, StringRef Code,
       ImportsInBlock.push_back(
           {Identifier, Line, Prev, AssociatedCommentLines, IsStatic});
       AssociatedCommentLines.clear();
-    } else if (Trimmed.size() > 0 && !ImportsInBlock.empty()) {
+    } else if (!Trimmed.empty() && !ImportsInBlock.empty()) {
       // Associating comments within the imports with the nearest import below
       AssociatedCommentLines.push_back(Line);
     }
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 4cc4f5f22db0..40b62b2a993d 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -14,6 +14,7 @@
 
 #include "FormatTokenLexer.h"
 #include "FormatToken.h"
+#include "clang/Basic/CharInfo.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Format/Format.h"
@@ -487,7 +488,7 @@ bool FormatTokenLexer::tryMergeCSharpKeywordVariables() {
 
 // In C# transform identifier foreach into kw_foreach
 bool FormatTokenLexer::tryTransformCSharpForEach() {
-  if (Tokens.size() < 1)
+  if (Tokens.empty())
     return false;
   auto &Identifier = *(Tokens.end() - 1);
   if (Identifier->isNot(tok::identifier))
@@ -947,7 +948,7 @@ void FormatTokenLexer::handleTableGenNumericLikeIdentifier() {
   // 4. The first non-digit character is 'x', and the next is a hex digit.
   // Note that in the case 3 and 4, if the next character does not exists in
   // this token, the token is an identifier.
-  if (Text.size() < 1 || Text[0] == '+' || Text[0] == '-')
+  if (Text.empty() || Text[0] == '+' || Text[0] == '-')
     return;
   const auto NonDigitPos = Text.find_if([](char C) { return !isdigit(C); });
   // All the characters are digits
@@ -1203,16 +1204,22 @@ static size_t countLeadingWhitespace(StringRef Text) {
   const unsigned char *const End = Text.bytes_end();
   const unsigned char *Cur = Begin;
   while (Cur < End) {
-    if (isspace(Cur[0])) {
+    if (isWhitespace(Cur[0])) {
       ++Cur;
-    } else if (Cur[0] == '\\' && (Cur[1] == '\n' || Cur[1] == '\r')) {
-      // A '\' followed by a newline always escapes the newline, regardless
-      // of whether there is another '\' before it.
+    } else if (Cur[0] == '\\') {
+      // A backslash followed by optional horizontal whitespaces (P22232R2) and
+      // then a newline always escapes the newline.
       // The source has a null byte at the end. So the end of the entire input
       // isn't reached yet. Also the lexer doesn't break apart an escaped
       // newline.
-      assert(End - Cur >= 2);
-      Cur += 2;
+      const auto *Lookahead = Cur + 1;
+      while (isHorizontalWhitespace(*Lookahead))
+        ++Lookahead;
+      // No line splice found; the backslash is a token.
+      if (!isVerticalWhitespace(*Lookahead))
+        break;
+      // Splice found, consume it.
+      Cur = Lookahead + 1;
     } else if (Cur[0] == '?' && Cur[1] == '?' && Cur[2] == '/' &&
                (Cur[3] == '\n' || Cur[3] == '\r')) {
       // Newlines can also be escaped by a '?' '?' '/' trigraph. By the way, the
@@ -1295,13 +1302,18 @@ FormatToken *FormatTokenLexer::getNextToken() {
       case '/':
         // The text was entirely whitespace when this loop was entered. Thus
         // this has to be an escape sequence.
-        assert(Text.substr(i, 2) == "\\\r" || Text.substr(i, 2) == "\\\n" ||
-               Text.substr(i, 4) == "\?\?/\r" ||
+        assert(Text.substr(i, 4) == "\?\?/\r" ||
                Text.substr(i, 4) == "\?\?/\n" ||
                (i >= 1 && (Text.substr(i - 1, 4) == "\?\?/\r" ||
                            Text.substr(i - 1, 4) == "\?\?/\n")) ||
                (i >= 2 && (Text.substr(i - 2, 4) == "\?\?/\r" ||
-                           Text.substr(i - 2, 4) == "\?\?/\n")));
+                           Text.substr(i - 2, 4) == "\?\?/\n")) ||
+               (Text[i] == '\\' && [&]() -> bool {
+                 size_t j = i + 1;
+                 while (j < Text.size() && isHorizontalWhitespace(Text[j]))
+                   ++j;
+                 return j < Text.size() && (Text[j] == '\n' || Text[j] == '\r');
+               }()));
         InEscape = true;
         break;
       default:
@@ -1359,8 +1371,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
       } else if (FormatTok->TokenText == "``") {
         FormatTok->Tok.setIdentifierInfo(nullptr);
         FormatTok->Tok.setKind(tok::hashhash);
-      } else if (Tokens.size() > 0 &&
-                 Tokens.back()->is(Keywords.kw_apostrophe) &&
+      } else if (!Tokens.empty() && Tokens.back()->is(Keywords.kw_apostrophe) &&
                  NumberBase.match(FormatTok->TokenText, &Matches)) {
         // In Verilog in a based number literal like `'b10`, there may be
         // whitespace between `'b` and `10`. Therefore we handle the base and
@@ -1408,7 +1419,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
     tryParseJavaTextBlock();
   }
 
-  if (Style.isVerilog() && Tokens.size() > 0 &&
+  if (Style.isVerilog() && !Tokens.empty() &&
       Tokens.back()->is(TT_VerilogNumberBase) &&
       FormatTok->Tok.isOneOf(tok::identifier, tok::question)) {
     // Mark the number following a base like `'h?a0` as a number.
@@ -1442,9 +1453,9 @@ FormatToken *FormatTokenLexer::getNextToken() {
   if (Style.isCpp()) {
     auto *Identifier = FormatTok->Tok.getIdentifierInfo();
     auto it = Macros.find(Identifier);
-    if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
-          Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
-              tok::pp_define) &&
+    if ((Tokens.empty() || !Tokens.back()->Tok.getIdentifierInfo() ||
+         Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() !=
+             tok::pp_define) &&
         it != Macros.end()) {
       FormatTok->setType(it->second);
       if (it->second == TT_IfMacro) {
diff --git a/clang/lib/Format/MacroExpander.cpp b/clang/lib/Format/MacroExpander.cpp
index 37c25e6941ef..85a53c9bb12f 100644
--- a/clang/lib/Format/MacroExpander.cpp
+++ b/clang/lib/Format/MacroExpander.cpp
@@ -226,7 +226,7 @@ MacroExpander::expand(FormatToken *ID,
     New->MacroCtx = MacroExpansion(MR_Hidden);
     pushToken(New);
   }
-  assert(Result.size() >= 1 && Result.back()->is(tok::eof));
+  assert(!Result.empty() && Result.back()->is(tok::eof));
   if (Result.size() > 1) {
     ++Result[0]->MacroCtx->StartOfExpansion;
     ++Result[Result.size() - 2]->MacroCtx->EndOfExpansion;
diff --git a/clang/lib/Format/QualifierAlignmentFixer.cpp b/clang/lib/Format/QualifierAlignmentFixer.cpp
index 8e55d339b238..441a37a4902b 100644
--- a/clang/lib/Format/QualifierAlignmentFixer.cpp
+++ b/clang/lib/Format/QualifierAlignmentFixer.cpp
@@ -635,15 +635,26 @@ bool isConfiguredQualifierOrType(const FormatToken *Tok,
 // If a token is an identifier and it's upper case, it could
 // be a macro and hence we need to be able to ignore it.
 bool isPossibleMacro(const FormatToken *Tok) {
-  if (!Tok)
-    return false;
+  assert(Tok);
   if (Tok->isNot(tok::identifier))
     return false;
-  if (Tok->TokenText.upper() == Tok->TokenText.str()) {
-    // T,K,U,V likely could be template arguments
-    return Tok->TokenText.size() != 1;
-  }
-  return false;
+
+  const auto Text = Tok->TokenText;
+  assert(!Text.empty());
+
+  // T,K,U,V likely could be template arguments
+  if (Text.size() == 1)
+    return false;
+
+  // It's unlikely that qualified names are object-like macros.
+  const auto *Prev = Tok->getPreviousNonComment();
+  if (Prev && Prev->is(tok::coloncolon))
+    return false;
+  const auto *Next = Tok->getNextNonComment();
+  if (Next && Next->is(tok::coloncolon))
+    return false;
+
+  return Text == Text.upper();
 }
 
 } // namespace format
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index d2f8b2703a9a..aab8f054655f 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -1371,7 +1371,7 @@ class AnnotatingParser {
         Tok->setType(TT_InlineASMColon);
       } else if (Contexts.back().ColonIsDictLiteral || Style.isProto()) {
         Tok->setType(TT_DictLiteral);
-        if (Prev && Style.isTextProto())
+        if (Style.isTextProto())
           Prev->setType(TT_SelectorName);
       } else if (Contexts.back().ColonIsObjCMethodExpr ||
                  Line.startsWith(TT_ObjCMethodSpecifier)) {
@@ -1408,19 +1408,16 @@ class AnnotatingParser {
         }
       } else if (Contexts.back().ContextType == Context::C11GenericSelection) {
         Tok->setType(TT_GenericSelectionColon);
-        assert(Prev);
         if (Prev->isPointerOrReference())
           Prev->setFinalizedType(TT_PointerOrReference);
       } else if ((CurrentToken && CurrentToken->is(tok::numeric_constant)) ||
-                 (Prev && Prev->is(TT_StartOfName) && !Scopes.empty() &&
+                 (Prev->is(TT_StartOfName) && !Scopes.empty() &&
                   Scopes.back() == ST_Class)) {
         Tok->setType(TT_BitFieldColon);
       } else if (Contexts.size() == 1 &&
                  !Line.getFirstNonComment()->isOneOf(tok::kw_enum, tok::kw_case,
                                                      tok::kw_default) &&
                  !Line.startsWith(tok::kw_typedef, tok::kw_enum)) {
-        if (!Prev)
-          break;
         if (Prev->isOneOf(tok::r_paren, tok::kw_noexcept) ||
             Prev->ClosesRequiresClause) {
           Tok->setType(TT_CtorInitializerColon);
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index f2ed027b2c04..0adf7ee9ed54 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -223,8 +223,6 @@ class LineJoiner {
   tryFitMultipleLinesInOne(LevelIndentTracker &IndentTracker,
                            ArrayRef<AnnotatedLine *>::const_iterator I,
                            ArrayRef<AnnotatedLine *>::const_iterator E) {
-    const unsigned Indent = IndentTracker.getIndent();
-
     // Can't join the last line with anything.
     if (I + 1 == E)
       return 0;
@@ -240,6 +238,7 @@ class LineJoiner {
       return 0;
     }
 
+    const auto Indent = IndentTracker.getIndent();
     if (Style.ColumnLimit > 0 && Indent > Style.ColumnLimit)
       return 0;
 
@@ -986,8 +985,10 @@ class LineJoiner {
   void join(AnnotatedLine &A, const AnnotatedLine &B) {
     assert(!A.Last->Next);
     assert(!B.First->Previous);
-    if (B.Affected)
+    if (B.Affected || B.LeadingEmptyLinesAffected) {
+      assert(B.Affected || A.Last->Children.empty());
       A.Affected = true;
+    }
     A.Last->Next = B.First;
     B.First->Previous = A.Last;
     B.First->CanBreakBefore = true;
@@ -1088,7 +1089,7 @@ class LineFormatter {
     const FormatToken *LBrace = State.NextToken->getPreviousNonComment();
     bool HasLBrace = LBrace && LBrace->is(tok::l_brace) && LBrace->is(BK_Block);
     FormatToken &Previous = *State.NextToken->Previous;
-    if (Previous.Children.size() == 0 || (!HasLBrace && !LBrace->MacroParent)) {
+    if (Previous.Children.empty() || (!HasLBrace && !LBrace->MacroParent)) {
       // The previous token does not open a block. Nothing to do. We don't
       // assert so that we can simply call this function for all tokens.
       return true;
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 61b84126fe1b..7e8634aeec4e 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -340,7 +340,7 @@ bool UnwrappedLineParser::precededByCommentOrPPDirective() const {
          (Previous->IsMultiline || Previous->NewlinesBefore > 0);
 }
 
-/// \brief Parses a level, that is ???.
+/// Parses a level, that is ???.
 /// \param OpeningBrace Opening brace (\p nullptr if absent) of that level.
 /// \param IfKind The \p if statement kind in the level.
 /// \param IfLeftBrace The left brace of the \p if block in the level.
@@ -2574,7 +2574,7 @@ bool UnwrappedLineParser::parseBracedList(bool IsAngleBracket, bool IsEnum) {
   return false;
 }
 
-/// \brief Parses a pair of parentheses (and everything between them).
+/// Parses a pair of parentheses (and everything between them).
 /// \param AmpAmpTokenType If different than TT_Unknown sets this type for all
 /// double ampersands. This applies for all nested scopes as well.
 ///
@@ -3438,7 +3438,7 @@ void UnwrappedLineParser::parseAccessSpecifier() {
   addUnwrappedLine();
 }
 
-/// \brief Parses a requires, decides if it is a clause or an expression.
+/// Parses a requires, decides if it is a clause or an expression.
 /// \pre The current token has to be the requires keyword.
 /// \returns true if it parsed a clause.
 bool UnwrappedLineParser::parseRequires(bool SeenEqual) {
@@ -3582,7 +3582,7 @@ bool UnwrappedLineParser::parseRequires(bool SeenEqual) {
   return true;
 }
 
-/// \brief Parses a requires clause.
+/// Parses a requires clause.
 /// \param RequiresToken The requires keyword token, which starts this clause.
 /// \pre We need to be on the next token after the requires keyword.
 /// \sa parseRequiresExpression
@@ -3612,7 +3612,7 @@ void UnwrappedLineParser::parseRequiresClause(FormatToken *RequiresToken) {
     FormatTok->Previous->ClosesRequiresClause = true;
 }
 
-/// \brief Parses a requires expression.
+/// Parses a requires expression.
 /// \param RequiresToken The requires keyword token, which starts this clause.
 /// \pre We need to be on the next token after the requires keyword.
 /// \sa parseRequiresClause
@@ -3636,7 +3636,7 @@ void UnwrappedLineParser::parseRequiresExpression(FormatToken *RequiresToken) {
   }
 }
 
-/// \brief Parses a constraint expression.
+/// Parses a constraint expression.
 ///
 /// This is the body of a requires clause. It returns, when the parsing is
 /// complete, or the expression is incorrect.
@@ -4601,7 +4601,7 @@ void UnwrappedLineParser::addUnwrappedLine(LineLevel AdjustLevel) {
   } else {
     // At the top level we only get here when no unexpansion is going on, or
     // when conditional formatting led to unfinished macro reconstructions.
-    assert(!Reconstruct || (CurrentLines != &Lines) || PPStack.size() > 0);
+    assert(!Reconstruct || (CurrentLines != &Lines) || !PPStack.empty());
     CurrentLines->push_back(std::move(*Line));
   }
   Line->Tokens.clear();
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index cfaadf07edfd..cc3cc0f6906c 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -314,7 +314,7 @@ AlignTokenSequence(const FormatStyle &Style, unsigned Start, unsigned End,
 
   for (unsigned i = Start; i != End; ++i) {
     auto &CurrentChange = Changes[i];
-    if (ScopeStack.size() != 0 &&
+    if (!ScopeStack.empty() &&
         CurrentChange.indentAndNestingLevel() <
             Changes[ScopeStack.back()].indentAndNestingLevel()) {
       ScopeStack.pop_back();
@@ -332,7 +332,7 @@ AlignTokenSequence(const FormatStyle &Style, unsigned Start, unsigned End,
       ScopeStack.push_back(i);
     }
 
-    bool InsideNestedScope = ScopeStack.size() != 0;
+    bool InsideNestedScope = !ScopeStack.empty();
     bool ContinuedStringLiteral = i > Start &&
                                   CurrentChange.Tok->is(tok::string_literal) &&
                                   Changes[i - 1].Tok->is(tok::string_literal);
@@ -1048,7 +1048,7 @@ void WhitespaceManager::alignChainedConditionals() {
           return C.Tok->is(TT_ConditionalExpr) &&
                  ((C.Tok->is(tok::question) && !C.NewlinesBefore) ||
                   (C.Tok->is(tok::colon) && C.Tok->Next &&
-                   (C.Tok->Next->FakeLParens.size() == 0 ||
+                   (C.Tok->Next->FakeLParens.empty() ||
                     C.Tok->Next->FakeLParens.back() != prec::Conditional)));
         },
         Changes, /*StartAt=*/0);
@@ -1057,7 +1057,7 @@ void WhitespaceManager::alignChainedConditionals() {
       FormatToken *Previous = C.Tok->getPreviousNonComment();
       return C.NewlinesBefore && Previous && Previous->is(TT_ConditionalExpr) &&
              (Previous->is(tok::colon) &&
-              (C.Tok->FakeLParens.size() == 0 ||
+              (C.Tok->FakeLParens.empty() ||
                C.Tok->FakeLParens.back() != prec::Conditional));
     };
     // Ensure we keep alignment of wrapped operands with non-wrapped operands
diff --git a/clang/lib/Format/WhitespaceManager.h b/clang/lib/Format/WhitespaceManager.h
index 5dd667fb4ba3..6d18db74cd2e 100644
--- a/clang/lib/Format/WhitespaceManager.h
+++ b/clang/lib/Format/WhitespaceManager.h
@@ -212,7 +212,7 @@ class WhitespaceManager {
   /// \c EscapedNewlineColumn for the first tokens or token parts in a line.
   void calculateLineBreakInformation();
 
-  /// \brief Align consecutive C/C++ preprocessor macros over all \c Changes.
+  /// Align consecutive C/C++ preprocessor macros over all \c Changes.
   void alignConsecutiveMacros();
 
   /// Align consecutive assignments over all \c Changes.
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 3beff3d6d849..4ebec8e07433 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -2466,9 +2466,7 @@ void ASTUnit::addFileLevelDecl(Decl *D) {
 
   SourceLocation FileLoc = SM.getFileLoc(Loc);
   assert(SM.isLocalSourceLocation(FileLoc));
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
+  auto [FID, Offset] = SM.getDecomposedLoc(FileLoc);
   if (FID.isInvalid())
     return;
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f26aa5630e8f..6c456d759d71 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -636,6 +636,10 @@ static bool FixupInvocation(CompilerInvocation &Invocation,
     Diags.Report(diag::err_drv_argument_not_allowed_with)
         << "-hlsl-entry" << GetInputKindName(IK);
 
+  if (Args.hasArg(OPT_fdx_rootsignature_version) && !LangOpts.HLSL)
+    Diags.Report(diag::err_drv_argument_not_allowed_with)
+        << "-fdx-rootsignature-version" << GetInputKindName(IK);
+
   if (Args.hasArg(OPT_fgpu_allow_device_init) && !LangOpts.HIP)
     Diags.Report(diag::warn_ignored_hip_only_option)
         << Args.getLastArg(OPT_fgpu_allow_device_init)->getAsString(Args);
@@ -1824,6 +1828,11 @@ void CompilerInvocationBase::GenerateCodeGenArgs(const CodeGenOptions &Opts,
   for (std::string Sanitizer : Values)
     GenerateArg(Consumer, OPT_fsanitize_skip_hot_cutoff_EQ, Sanitizer);
 
+  if (Opts.AllowRuntimeCheckSkipHotCutoff) {
+    GenerateArg(Consumer, OPT_fallow_runtime_check_skip_hot_cutoff_EQ,
+                std::to_string(*Opts.AllowRuntimeCheckSkipHotCutoff));
+  }
+
   for (StringRef Sanitizer :
        serializeSanitizerKinds(Opts.SanitizeAnnotateDebugInfo))
     GenerateArg(Consumer, OPT_fsanitize_annotate_debug_info_EQ, Sanitizer);
@@ -2326,6 +2335,18 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
       Args.getAllArgValues(OPT_fsanitize_annotate_debug_info_EQ), Diags,
       Opts.SanitizeAnnotateDebugInfo);
 
+  if (StringRef V =
+          Args.getLastArgValue(OPT_fallow_runtime_check_skip_hot_cutoff_EQ);
+      !V.empty()) {
+    double A;
+    if (V.getAsDouble(A) || A < 0.0 || A > 1.0) {
+      Diags.Report(diag::err_drv_invalid_value)
+          << "-fallow-runtime-check-skip-hot-cutoff=" << V;
+    } else {
+      Opts.AllowRuntimeCheckSkipHotCutoff = A;
+    }
+  }
+
   Opts.EmitVersionIdentMetadata = Args.hasFlag(OPT_Qy, OPT_Qn, true);
 
   if (!LangOpts->CUDAIsDevice)
@@ -3666,6 +3687,23 @@ static StringRef GetInputKindName(InputKind IK) {
   llvm_unreachable("unknown input language");
 }
 
+static StringRef getExceptionHandlingName(unsigned EHK) {
+  switch (static_cast<LangOptions::ExceptionHandlingKind>(EHK)) {
+  case LangOptions::ExceptionHandlingKind::None:
+    return "none";
+  case LangOptions::ExceptionHandlingKind::DwarfCFI:
+    return "dwarf";
+  case LangOptions::ExceptionHandlingKind::SjLj:
+    return "sjlj";
+  case LangOptions::ExceptionHandlingKind::WinEH:
+    return "seh";
+  case LangOptions::ExceptionHandlingKind::Wasm:
+    return "wasm";
+  }
+
+  llvm_unreachable("covered switch");
+}
+
 void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
                                               ArgumentConsumer Consumer,
                                               const llvm::Triple &T,
@@ -3681,6 +3719,10 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
       GenerateArg(Consumer, OPT_pic_is_pie);
     for (StringRef Sanitizer : serializeSanitizerKinds(Opts.Sanitize))
       GenerateArg(Consumer, OPT_fsanitize_EQ, Sanitizer);
+    if (Opts.ExceptionHandling) {
+      GenerateArg(Consumer, OPT_exception_model,
+                  getExceptionHandlingName(Opts.ExceptionHandling));
+    }
 
     return;
   }
@@ -3920,7 +3962,7 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     llvm::interleave(
         Opts.OMPTargetTriples, OS,
         [&OS](const llvm::Triple &T) { OS << T.str(); }, ",");
-    GenerateArg(Consumer, OPT_fopenmp_targets_EQ, Targets);
+    GenerateArg(Consumer, OPT_offload_targets_EQ, Targets);
   }
 
   if (!Opts.OMPHostIRFile.empty())
@@ -4052,6 +4094,24 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     parseSanitizerKinds("-fsanitize=", Args.getAllArgValues(OPT_fsanitize_EQ),
                         Diags, Opts.Sanitize);
 
+    if (const Arg *A = Args.getLastArg(options::OPT_exception_model)) {
+      std::optional<LangOptions::ExceptionHandlingKind> EMValue =
+          llvm::StringSwitch<std::optional<LangOptions::ExceptionHandlingKind>>(
+              A->getValue())
+              .Case("dwarf", LangOptions::ExceptionHandlingKind::DwarfCFI)
+              .Case("sjlj", LangOptions::ExceptionHandlingKind::SjLj)
+              .Case("seh", LangOptions::ExceptionHandlingKind::WinEH)
+              .Case("wasm", LangOptions::ExceptionHandlingKind::Wasm)
+              .Case("none", LangOptions::ExceptionHandlingKind::None)
+              .Default(std::nullopt);
+      if (EMValue) {
+        Opts.ExceptionHandling = static_cast<unsigned>(*EMValue);
+      } else {
+        Diags.Report(diag::err_drv_invalid_value)
+            << A->getAsString(Args) << A->getValue();
+      }
+    }
+
     return Diags.getNumErrors() == NumErrorsBefore;
   }
 
@@ -4333,7 +4393,8 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.OpenMPIRBuilder =
       Opts.OpenMP && Args.hasArg(options::OPT_fopenmp_enable_irbuilder);
   bool IsTargetSpecified =
-      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_fopenmp_targets_EQ);
+      Opts.OpenMPIsTargetDevice || Args.hasArg(options::OPT_offload_targets_EQ);
+
   if (Opts.OpenMP || Opts.OpenMPSimd) {
     if (int Version = getLastArgIntValue(
             Args, OPT_fopenmp_version_EQ,
@@ -4450,7 +4511,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
       options::OPT_fno_openmp_assume_no_nested_parallelism, isOFastUsed(Args));
 
   // Get the OpenMP target triples if any.
-  if (Arg *A = Args.getLastArg(options::OPT_fopenmp_targets_EQ)) {
+  if (Arg *A = Args.getLastArg(options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &T) {
       if (T.isArch16Bit())
@@ -4688,6 +4749,11 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0)
       Diags.Report(diag::err_cc1_unbounded_vscale_min);
   }
+  if (Arg *A = Args.getLastArg(options::OPT_mvscale_streaming_min_EQ)) {
+    unsigned VScaleMin;
+    if (StringRef(A->getValue()).getAsInteger(10, VScaleMin) || VScaleMin == 0)
+      Diags.Report(diag::err_cc1_unbounded_vscale_min);
+  }
 
   if (const Arg *A = Args.getLastArg(OPT_frandomize_layout_seed_file_EQ)) {
     std::ifstream SeedFile(A->getValue(0));
diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp
index ef7ae27a2694..f5996a8e1e88 100644
--- a/clang/lib/Frontend/FrontendAction.cpp
+++ b/clang/lib/Frontend/FrontendAction.cpp
@@ -1243,13 +1243,15 @@ llvm::Error FrontendAction::Execute() {
 void FrontendAction::EndSourceFile() {
   CompilerInstance &CI = getCompilerInstance();
 
-  // Inform the diagnostic client we are done with this source file.
-  CI.getDiagnosticClient().EndSourceFile();
-
   // Inform the preprocessor we are done.
   if (CI.hasPreprocessor())
     CI.getPreprocessor().EndSourceFile();
 
+  // Inform the diagnostic client we are done with this source file.
+  // Do this after notifying the preprocessor, so that end-of-file preprocessor
+  // callbacks can report diagnostics.
+  CI.getDiagnosticClient().EndSourceFile();
+
   // Finalize the action.
   EndSourceFileAction();
 
diff --git a/clang/lib/Frontend/SARIFDiagnostic.cpp b/clang/lib/Frontend/SARIFDiagnostic.cpp
index e2aec7f677f1..ac27d7480de3 100644
--- a/clang/lib/Frontend/SARIFDiagnostic.cpp
+++ b/clang/lib/Frontend/SARIFDiagnostic.cpp
@@ -91,8 +91,8 @@ SarifResult SARIFDiagnostic::addLocationToResult(
     SourceLocation E = ERange.getEnd();
     bool IsTokenRange = ERange.isTokenRange();
 
-    std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(B);
-    std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(E);
+    FileIDAndOffset BInfo = SM.getDecomposedLoc(B);
+    FileIDAndOffset EInfo = SM.getDecomposedLoc(E);
 
     // If the start or end of the range is in another file, just discard
     // it.
diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp
index b9e681b52e50..ccdd59da89bd 100644
--- a/clang/lib/Frontend/TextDiagnostic.cpp
+++ b/clang/lib/Frontend/TextDiagnostic.cpp
@@ -1012,7 +1012,7 @@ static std::string buildFixItInsertionLine(FileID FID, unsigned LineNo,
 
     // We have an insertion hint. Determine whether the inserted
     // code contains no newlines and is on the same line as the caret.
-    std::pair<FileID, unsigned> HintLocInfo =
+    FileIDAndOffset HintLocInfo =
         SM.getDecomposedExpansionLoc(H.RemoveRange.getBegin());
     if (FID == HintLocInfo.first &&
         LineNo == SM.getLineNumber(HintLocInfo.first, HintLocInfo.second) &&
@@ -1518,8 +1518,8 @@ void TextDiagnostic::emitParseableFixits(ArrayRef<FixItHint> Hints,
     SourceLocation BLoc = H.RemoveRange.getBegin();
     SourceLocation ELoc = H.RemoveRange.getEnd();
 
-    std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(BLoc);
-    std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(ELoc);
+    FileIDAndOffset BInfo = SM.getDecomposedLoc(BLoc);
+    FileIDAndOffset EInfo = SM.getDecomposedLoc(ELoc);
 
     // Adjust for token ranges.
     if (H.RemoveRange.isTokenRange())
diff --git a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index 6fc587f4de9d..443eb4f1a29b 100644
--- a/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -26,7 +26,6 @@
 #include "clang/StaticAnalyzer/Frontend/AnalyzerHelpFlags.h"
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "llvm/Option/OptTable.h"
-#include "llvm/Option/Option.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index 3dd1eb45817d..588c283cbdfb 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -370,10 +370,29 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
 \*----------------------------------------------------------------------------*/
 #if defined(__aarch64__) || defined(__arm64ec__)
 unsigned __int64 __getReg(int);
-long _InterlockedAdd(long volatile *Addend, long Value);
-__int64 _InterlockedAdd64(__int64 volatile *Addend, __int64 Value);
+unsigned char _interlockedbittestandreset_acq(long volatile *, long);
+unsigned char _interlockedbittestandreset_nf(long volatile *, long);
+unsigned char _interlockedbittestandreset_rel(long volatile *, long);
+unsigned char _interlockedbittestandreset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_rel(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset_acq(long volatile *, long);
+unsigned char _interlockedbittestandset_nf(long volatile *, long);
+unsigned char _interlockedbittestandset_rel(long volatile *, long);
+unsigned char _interlockedbittestandset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_rel(__int64 volatile *, __int64);
+long _InterlockedAdd(long volatile *, long);
+long _InterlockedAdd_acq(long volatile *, long);
+long _InterlockedAdd_nf(long volatile *, long);
+long _InterlockedAdd_rel(long volatile *, long);
+__int64 _InterlockedAdd64(__int64 volatile *, __int64);
+__int64 _InterlockedAdd64_acq(__int64 volatile *, __int64);
+__int64 _InterlockedAdd64_nf(__int64 volatile *, __int64);
+__int64 _InterlockedAdd64_rel(__int64 volatile *, __int64);
 __int64 _ReadStatusReg(int);
 void _WriteStatusReg(int, __int64);
+unsigned int __sys(int, __int64);
 
 unsigned short __cdecl _byteswap_ushort(unsigned short val);
 unsigned long __cdecl _byteswap_ulong (unsigned long val);
diff --git a/clang/lib/Headers/opencl-c.h b/clang/lib/Headers/opencl-c.h
index de77348a8003..10a59865216f 100644
--- a/clang/lib/Headers/opencl-c.h
+++ b/clang/lib/Headers/opencl-c.h
@@ -18240,7 +18240,50 @@ double  __ovld __conv intel_sub_group_shuffle_up( double prev, double cur, uint
 double  __ovld __conv intel_sub_group_shuffle_xor( double, uint );
 #endif
 
-#endif //cl_intel_subgroups
+#if defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||   \
+    defined(cl_intel_subgroups_long)
+
+#if defined(__opencl_c_images)
+uint       __ovld __conv intel_sub_group_block_read_ui(read_only image2d_t, int2);
+uint2      __ovld __conv intel_sub_group_block_read_ui2(read_only image2d_t, int2);
+uint4      __ovld __conv intel_sub_group_block_read_ui4(read_only image2d_t, int2);
+uint8      __ovld __conv intel_sub_group_block_read_ui8(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+uint       __ovld __conv intel_sub_group_block_read_ui(read_write image2d_t, int2);
+uint2      __ovld __conv intel_sub_group_block_read_ui2(read_write image2d_t, int2);
+uint4      __ovld __conv intel_sub_group_block_read_ui4(read_write image2d_t, int2);
+uint8      __ovld __conv intel_sub_group_block_read_ui8(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+uint       __ovld __conv intel_sub_group_block_read_ui( const __global uint* p );
+uint2      __ovld __conv intel_sub_group_block_read_ui2( const __global uint* p );
+uint4      __ovld __conv intel_sub_group_block_read_ui4( const __global uint* p );
+uint8      __ovld __conv intel_sub_group_block_read_ui8( const __global uint* p );
+
+#if defined(__opencl_c_images)
+void       __ovld __conv intel_sub_group_block_write_ui(read_only image2d_t, int2, uint);
+void       __ovld __conv intel_sub_group_block_write_ui2(read_only image2d_t, int2, uint2);
+void       __ovld __conv intel_sub_group_block_write_ui4(read_only image2d_t, int2, uint4);
+void       __ovld __conv intel_sub_group_block_write_ui8(read_only image2d_t, int2, uint8);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void       __ovld __conv intel_sub_group_block_write_ui(read_write image2d_t, int2, uint);
+void       __ovld __conv intel_sub_group_block_write_ui2(read_write image2d_t, int2, uint2);
+void       __ovld __conv intel_sub_group_block_write_ui4(read_write image2d_t, int2, uint4);
+void       __ovld __conv intel_sub_group_block_write_ui8(read_write image2d_t, int2, uint8);
+#endif // defined(__opencl_c_read_write_images)
+
+void       __ovld __conv intel_sub_group_block_write_ui( __global uint* p, uint data );
+void       __ovld __conv intel_sub_group_block_write_ui2( __global uint* p, uint2 data );
+void       __ovld __conv intel_sub_group_block_write_ui4( __global uint* p, uint4 data );
+void       __ovld __conv intel_sub_group_block_write_ui8( __global uint* p, uint8 data );
+
+#endif // defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||
+       // defined(cl_intel_subgroups_long)
+#endif // cl_intel_subgroups
 
 #if defined(cl_intel_subgroups_short)
 short       __ovld __conv intel_sub_group_broadcast( short , uint sub_group_local_id );
@@ -18332,49 +18375,12 @@ ushort      __ovld __conv intel_sub_group_scan_inclusive_min( ushort  x );
 short       __ovld __conv intel_sub_group_scan_inclusive_max( short   x );
 ushort      __ovld __conv intel_sub_group_scan_inclusive_max( ushort  x );
 
-#if defined(__opencl_c_images)
-uint       __ovld __conv intel_sub_group_block_read_ui(read_only image2d_t, int2);
-uint2      __ovld __conv intel_sub_group_block_read_ui2(read_only image2d_t, int2);
-uint4      __ovld __conv intel_sub_group_block_read_ui4(read_only image2d_t, int2);
-uint8      __ovld __conv intel_sub_group_block_read_ui8(read_only image2d_t, int2);
-#endif // defined(__opencl_c_images)
-
-#if defined(__opencl_c_read_write_images)
-uint       __ovld __conv intel_sub_group_block_read_ui(read_write image2d_t, int2);
-uint2      __ovld __conv intel_sub_group_block_read_ui2(read_write image2d_t, int2);
-uint4      __ovld __conv intel_sub_group_block_read_ui4(read_write image2d_t, int2);
-uint8      __ovld __conv intel_sub_group_block_read_ui8(read_write image2d_t, int2);
-#endif // defined(__opencl_c_read_write_images)
-
-uint       __ovld __conv intel_sub_group_block_read_ui( const __global uint* p );
-uint2      __ovld __conv intel_sub_group_block_read_ui2( const __global uint* p );
-uint4      __ovld __conv intel_sub_group_block_read_ui4( const __global uint* p );
-uint8      __ovld __conv intel_sub_group_block_read_ui8( const __global uint* p );
-
-#if defined(__opencl_c_images)
-void       __ovld __conv intel_sub_group_block_write_ui(read_only image2d_t, int2, uint);
-void       __ovld __conv intel_sub_group_block_write_ui2(read_only image2d_t, int2, uint2);
-void       __ovld __conv intel_sub_group_block_write_ui4(read_only image2d_t, int2, uint4);
-void       __ovld __conv intel_sub_group_block_write_ui8(read_only image2d_t, int2, uint8);
-#endif //defined(__opencl_c_images)
-
-#if defined(__opencl_c_read_write_images)
-void       __ovld __conv intel_sub_group_block_write_ui(read_write image2d_t, int2, uint);
-void       __ovld __conv intel_sub_group_block_write_ui2(read_write image2d_t, int2, uint2);
-void       __ovld __conv intel_sub_group_block_write_ui4(read_write image2d_t, int2, uint4);
-void       __ovld __conv intel_sub_group_block_write_ui8(read_write image2d_t, int2, uint8);
-#endif // defined(__opencl_c_read_write_images)
-
-void       __ovld __conv intel_sub_group_block_write_ui( __global uint* p, uint data );
-void       __ovld __conv intel_sub_group_block_write_ui2( __global uint* p, uint2 data );
-void       __ovld __conv intel_sub_group_block_write_ui4( __global uint* p, uint4 data );
-void       __ovld __conv intel_sub_group_block_write_ui8( __global uint* p, uint8 data );
-
 #if defined(__opencl_c_images)
 ushort      __ovld __conv intel_sub_group_block_read_us(read_only image2d_t, int2);
 ushort2     __ovld __conv intel_sub_group_block_read_us2(read_only image2d_t, int2);
 ushort4     __ovld __conv intel_sub_group_block_read_us4(read_only image2d_t, int2);
 ushort8     __ovld __conv intel_sub_group_block_read_us8(read_only image2d_t, int2);
+ushort16    __ovld __conv intel_sub_group_block_read_us16(read_only image2d_t, int2);
 #endif // defined(__opencl_c_images)
 
 #if defined(__opencl_c_read_write_images)
@@ -18382,18 +18388,21 @@ ushort      __ovld __conv intel_sub_group_block_read_us(read_write image2d_t, in
 ushort2     __ovld __conv intel_sub_group_block_read_us2(read_write image2d_t, int2);
 ushort4     __ovld __conv intel_sub_group_block_read_us4(read_write image2d_t, int2);
 ushort8     __ovld __conv intel_sub_group_block_read_us8(read_write image2d_t, int2);
+ushort16    __ovld __conv intel_sub_group_block_read_us16(read_write image2d_t, int2);
 #endif // defined(__opencl_c_read_write_images)
 
 ushort      __ovld __conv intel_sub_group_block_read_us(  const __global ushort* p );
 ushort2     __ovld __conv intel_sub_group_block_read_us2( const __global ushort* p );
 ushort4     __ovld __conv intel_sub_group_block_read_us4( const __global ushort* p );
 ushort8     __ovld __conv intel_sub_group_block_read_us8( const __global ushort* p );
+ushort16    __ovld __conv intel_sub_group_block_read_us16(const __global ushort* p);
 
 #if defined(__opencl_c_images)
 void        __ovld __conv intel_sub_group_block_write_us(write_only image2d_t, int2, ushort);
 void        __ovld __conv intel_sub_group_block_write_us2(write_only image2d_t, int2, ushort2);
 void        __ovld __conv intel_sub_group_block_write_us4(write_only image2d_t, int2, ushort4);
 void        __ovld __conv intel_sub_group_block_write_us8(write_only image2d_t, int2, ushort8);
+void        __ovld __conv intel_sub_group_block_write_us16(write_only image2d_t, int2, ushort16);
 #endif // defined(__opencl_c_images)
 
 #if defined(__opencl_c_read_write_images)
@@ -18401,14 +18410,261 @@ void        __ovld __conv intel_sub_group_block_write_us(read_write image2d_t, i
 void        __ovld __conv intel_sub_group_block_write_us2(read_write image2d_t, int2, ushort2);
 void        __ovld __conv intel_sub_group_block_write_us4(read_write image2d_t, int2, ushort4);
 void        __ovld __conv intel_sub_group_block_write_us8(read_write image2d_t, int2, ushort8);
+void        __ovld __conv intel_sub_group_block_write_us16(read_write image2d_t, int2, ushort16);
 #endif // defined(__opencl_c_read_write_images)
 
 void        __ovld __conv intel_sub_group_block_write_us(  __global ushort* p, ushort  data );
 void        __ovld __conv intel_sub_group_block_write_us2( __global ushort* p, ushort2 data );
 void        __ovld __conv intel_sub_group_block_write_us4( __global ushort* p, ushort4 data );
 void        __ovld __conv intel_sub_group_block_write_us8( __global ushort* p, ushort8 data );
+void        __ovld __conv intel_sub_group_block_write_us16( __global ushort* p, ushort16 data );
 #endif // cl_intel_subgroups_short
 
+#if defined(cl_intel_subgroups_char)
+char       __ovld __conv intel_sub_group_broadcast( char , uint sub_group_local_id );
+char2      __ovld __conv intel_sub_group_broadcast( char2, uint sub_group_local_id );
+char3      __ovld __conv intel_sub_group_broadcast( char3, uint sub_group_local_id );
+char4      __ovld __conv intel_sub_group_broadcast( char4, uint sub_group_local_id );
+char8      __ovld __conv intel_sub_group_broadcast( char8, uint sub_group_local_id );
+
+uchar      __ovld __conv intel_sub_group_broadcast( uchar , uint sub_group_local_id );
+uchar2     __ovld __conv intel_sub_group_broadcast( uchar2, uint sub_group_local_id );
+uchar3     __ovld __conv intel_sub_group_broadcast( uchar3, uint sub_group_local_id );
+uchar4     __ovld __conv intel_sub_group_broadcast( uchar4, uint sub_group_local_id );
+uchar8     __ovld __conv intel_sub_group_broadcast( uchar8, uint sub_group_local_id );
+
+char       __ovld __conv intel_sub_group_shuffle( char  , uint );
+char2      __ovld __conv intel_sub_group_shuffle( char2 , uint );
+char3      __ovld __conv intel_sub_group_shuffle( char3 , uint );
+char4      __ovld __conv intel_sub_group_shuffle( char4 , uint );
+char8      __ovld __conv intel_sub_group_shuffle( char8 , uint );
+char16     __ovld __conv intel_sub_group_shuffle( char16, uint);
+
+uchar      __ovld __conv intel_sub_group_shuffle( uchar  , uint );
+uchar2     __ovld __conv intel_sub_group_shuffle( uchar2 , uint );
+uchar3     __ovld __conv intel_sub_group_shuffle( uchar3 , uint );
+uchar4     __ovld __conv intel_sub_group_shuffle( uchar4 , uint );
+uchar8     __ovld __conv intel_sub_group_shuffle( uchar8 , uint );
+uchar16    __ovld __conv intel_sub_group_shuffle( uchar16, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_down( char   cur, char   next, uint );
+char2      __ovld __conv intel_sub_group_shuffle_down( char2  cur, char2  next, uint );
+char3      __ovld __conv intel_sub_group_shuffle_down( char3  cur, char3  next, uint );
+char4      __ovld __conv intel_sub_group_shuffle_down( char4  cur, char4  next, uint );
+char8      __ovld __conv intel_sub_group_shuffle_down( char8  cur, char8  next, uint );
+char16     __ovld __conv intel_sub_group_shuffle_down( char16 cur, char16 next, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_down( uchar   cur, uchar   next, uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_down( uchar2  cur, uchar2  next, uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_down( uchar3  cur, uchar3  next, uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_down( uchar4  cur, uchar4  next, uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_down( uchar8  cur, uchar8  next, uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_down( uchar16 cur, uchar16 next, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_up( char   cur, char   next, uint );
+char2      __ovld __conv intel_sub_group_shuffle_up( char2  cur, char2  next, uint );
+char3      __ovld __conv intel_sub_group_shuffle_up( char3  cur, char3  next, uint );
+char4      __ovld __conv intel_sub_group_shuffle_up( char4  cur, char4  next, uint );
+char8      __ovld __conv intel_sub_group_shuffle_up( char8  cur, char8  next, uint );
+char16     __ovld __conv intel_sub_group_shuffle_up( char16 cur, char16 next, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_up( uchar   cur, uchar   next, uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_up( uchar2  cur, uchar2  next, uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_up( uchar3  cur, uchar3  next, uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_up( uchar4  cur, uchar4  next, uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_up( uchar8  cur, uchar8  next, uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_up( uchar16 cur, uchar16 next, uint );
+
+char       __ovld __conv intel_sub_group_shuffle_xor( char  , uint );
+char2      __ovld __conv intel_sub_group_shuffle_xor( char2 , uint );
+char3      __ovld __conv intel_sub_group_shuffle_xor( char3 , uint );
+char4      __ovld __conv intel_sub_group_shuffle_xor( char4 , uint );
+char8      __ovld __conv intel_sub_group_shuffle_xor( char8 , uint );
+char16     __ovld __conv intel_sub_group_shuffle_xor( char16, uint );
+
+uchar      __ovld __conv intel_sub_group_shuffle_xor( uchar  , uint );
+uchar2     __ovld __conv intel_sub_group_shuffle_xor( uchar2 , uint );
+uchar3     __ovld __conv intel_sub_group_shuffle_xor( uchar3 , uint );
+uchar4     __ovld __conv intel_sub_group_shuffle_xor( uchar4 , uint );
+uchar8     __ovld __conv intel_sub_group_shuffle_xor( uchar8 , uint );
+uchar16    __ovld __conv intel_sub_group_shuffle_xor( uchar16, uint );
+
+char       __ovld __conv intel_sub_group_reduce_add( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_add( uchar  x );
+char       __ovld __conv intel_sub_group_reduce_min( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_min( uchar  x );
+char       __ovld __conv intel_sub_group_reduce_max( char   x );
+uchar      __ovld __conv intel_sub_group_reduce_max( uchar  x );
+
+char       __ovld __conv intel_sub_group_scan_exclusive_add( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_add( uchar  x );
+char       __ovld __conv intel_sub_group_scan_exclusive_min( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_min( uchar  x );
+char       __ovld __conv intel_sub_group_scan_exclusive_max( char   x );
+uchar      __ovld __conv intel_sub_group_scan_exclusive_max( uchar  x );
+
+char       __ovld __conv intel_sub_group_scan_inclusive_add( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_add( uchar  x );
+char       __ovld __conv intel_sub_group_scan_inclusive_min( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_min( uchar  x );
+char       __ovld __conv intel_sub_group_scan_inclusive_max( char   x );
+uchar      __ovld __conv intel_sub_group_scan_inclusive_max( uchar  x );
+
+#if defined(__opencl_c_images)
+uchar      __ovld __conv intel_sub_group_block_read_uc(read_only image2d_t, int2);
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(read_only image2d_t, int2);
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(read_only image2d_t, int2);
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(read_only image2d_t, int2);
+uchar16    __ovld __conv intel_sub_group_block_read_uc16(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+uchar      __ovld __conv intel_sub_group_block_read_uc(read_write image2d_t, int2);
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(read_write image2d_t, int2);
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(read_write image2d_t, int2);
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(read_write image2d_t, int2);
+uchar16    __ovld __conv intel_sub_group_block_read_uc16(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+uchar      __ovld __conv intel_sub_group_block_read_uc(  const __global uchar* p );
+uchar2     __ovld __conv intel_sub_group_block_read_uc2( const __global uchar* p );
+uchar4     __ovld __conv intel_sub_group_block_read_uc4( const __global uchar* p );
+uchar8     __ovld __conv intel_sub_group_block_read_uc8( const __global uchar* p );
+uchar16    __ovld __conv intel_sub_group_block_read_uc16( const __global uchar* p );
+
+#if defined(__opencl_c_images)
+void        __ovld __conv intel_sub_group_block_write_uc(write_only image2d_t, int2, uchar);
+void        __ovld __conv intel_sub_group_block_write_uc2(write_only image2d_t, int2, uchar2);
+void        __ovld __conv intel_sub_group_block_write_uc4(write_only image2d_t, int2, uchar4);
+void        __ovld __conv intel_sub_group_block_write_uc8(write_only image2d_t, int2, uchar8);
+void        __ovld __conv intel_sub_group_block_write_uc16(write_only image2d_t, int2, uchar16);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void        __ovld __conv intel_sub_group_block_write_uc(read_write image2d_t, int2, uchar);
+void        __ovld __conv intel_sub_group_block_write_uc2(read_write image2d_t, int2, uchar2);
+void        __ovld __conv intel_sub_group_block_write_uc4(read_write image2d_t, int2, uchar4);
+void        __ovld __conv intel_sub_group_block_write_uc8(read_write image2d_t, int2, uchar8);
+void        __ovld __conv intel_sub_group_block_write_uc16(read_write image2d_t, int2, uchar16);
+#endif // defined(__opencl_c_read_write_images)
+
+void        __ovld __conv intel_sub_group_block_write_uc(  __global uchar* p, uchar  data );
+void        __ovld __conv intel_sub_group_block_write_uc2( __global uchar* p, uchar2 data );
+void        __ovld __conv intel_sub_group_block_write_uc4( __global uchar* p, uchar4 data );
+void        __ovld __conv intel_sub_group_block_write_uc8( __global uchar* p, uchar8 data );
+void        __ovld __conv intel_sub_group_block_write_uc16( __global uchar* p, uchar16 data );
+#endif // cl_intel_subgroups_char
+
+#if defined(cl_intel_subgroups_long)
+#if defined(__opencl_c_images)
+ulong      __ovld __conv intel_sub_group_block_read_ul(read_only image2d_t, int2);
+ulong2     __ovld __conv intel_sub_group_block_read_ul2(read_only image2d_t, int2);
+ulong4     __ovld __conv intel_sub_group_block_read_ul4(read_only image2d_t, int2);
+ulong8     __ovld __conv intel_sub_group_block_read_ul8(read_only image2d_t, int2);
+ulong16    __ovld __conv intel_sub_group_block_read_ul16(read_only image2d_t, int2);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+ulong      __ovld __conv intel_sub_group_block_read_ul(read_write image2d_t, int2);
+ulong2     __ovld __conv intel_sub_group_block_read_ul2(read_write image2d_t, int2);
+ulong4     __ovld __conv intel_sub_group_block_read_ul4(read_write image2d_t, int2);
+ulong8     __ovld __conv intel_sub_group_block_read_ul8(read_write image2d_t, int2);
+ulong16    __ovld __conv intel_sub_group_block_read_ul16(read_write image2d_t, int2);
+#endif // defined(__opencl_c_read_write_images)
+
+ulong      __ovld __conv intel_sub_group_block_read_ul(  const __global ulong* p );
+ulong2     __ovld __conv intel_sub_group_block_read_ul2( const __global ulong* p );
+ulong4     __ovld __conv intel_sub_group_block_read_ul4( const __global ulong* p );
+ulong8     __ovld __conv intel_sub_group_block_read_ul8( const __global ulong* p );
+
+#if defined(__opencl_c_images)
+void        __ovld __conv intel_sub_group_block_write_ul(write_only image2d_t, int2, ulong);
+void        __ovld __conv intel_sub_group_block_write_ul2(write_only image2d_t, int2, ulong2);
+void        __ovld __conv intel_sub_group_block_write_ul4(write_only image2d_t, int2, ulong4);
+void        __ovld __conv intel_sub_group_block_write_ul8(write_only image2d_t, int2, ulong8);
+void        __ovld __conv intel_sub_group_block_write_ul16(write_only image2d_t, int2, ulong16);
+#endif // defined(__opencl_c_images)
+
+#if defined(__opencl_c_read_write_images)
+void        __ovld __conv intel_sub_group_block_write_ul(read_write image2d_t, int2, ulong);
+void        __ovld __conv intel_sub_group_block_write_ul2(read_write image2d_t, int2, ulong2);
+void        __ovld __conv intel_sub_group_block_write_ul4(read_write image2d_t, int2, ulong4);
+void        __ovld __conv intel_sub_group_block_write_ul8(read_write image2d_t, int2, ulong8);
+void        __ovld __conv intel_sub_group_block_write_ul16(read_write image2d_t, int2, ulong16);
+#endif // defined(__opencl_c_read_write_images)
+
+void        __ovld __conv intel_sub_group_block_write_ul(  __global ulong* p, ulong  data );
+void        __ovld __conv intel_sub_group_block_write_ul2( __global ulong* p, ulong2 data );
+void        __ovld __conv intel_sub_group_block_write_ul4( __global ulong* p, ulong4 data );
+void        __ovld __conv intel_sub_group_block_write_ul8( __global ulong* p, ulong8 data);
+#endif // cl_intel_subgroups_long
+
+#if defined(cl_intel_subgroup_local_block_io)
+uint    __ovld __conv intel_sub_group_block_read( const __local uint* p );
+uint2   __ovld __conv intel_sub_group_block_read2( const __local uint* p );
+uint4   __ovld __conv intel_sub_group_block_read4( const __local uint* p );
+uint8   __ovld __conv intel_sub_group_block_read8( const __local uint* p );
+
+void    __ovld __conv intel_sub_group_block_write( __local uint* p, uint data );
+void    __ovld __conv intel_sub_group_block_write2( __local uint* p, uint2 data );
+void    __ovld __conv intel_sub_group_block_write4( __local uint* p, uint4 data );
+void    __ovld __conv intel_sub_group_block_write8( __local uint* p, uint8 data );
+
+#if defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||   \
+    defined(cl_intel_subgroups_long)
+uint       __ovld __conv intel_sub_group_block_read_ui( const __local uint* p );
+uint2      __ovld __conv intel_sub_group_block_read_ui2( const __local uint* p );
+uint4      __ovld __conv intel_sub_group_block_read_ui4( const __local uint* p );
+uint8      __ovld __conv intel_sub_group_block_read_ui8( const __local uint* p );
+
+void       __ovld __conv intel_sub_group_block_write_ui( __local uint* p, uint data );
+void       __ovld __conv intel_sub_group_block_write_ui2( __local uint* p, uint2 data );
+void       __ovld __conv intel_sub_group_block_write_ui4( __local uint* p, uint4 data );
+void       __ovld __conv intel_sub_group_block_write_ui8( __local uint* p, uint8 data );
+#endif // defined(cl_intel_subgroups_char) || defined(cl_intel_subgroups_short) ||
+       // defined(cl_intel_subgroups_long)
+
+#if defined(cl_intel_subgroups_char)
+uchar      __ovld __conv intel_sub_group_block_read_uc(   const __local uchar* p );
+uchar2     __ovld __conv intel_sub_group_block_read_uc2(  const __local uchar* p );
+uchar4     __ovld __conv intel_sub_group_block_read_uc4(  const __local uchar* p );
+uchar8     __ovld __conv intel_sub_group_block_read_uc8(  const __local uchar* p );
+uchar16    __ovld __conv intel_sub_group_block_read_uc16( const __local uchar* p );
+
+void        __ovld __conv intel_sub_group_block_write_uc(   __local uchar* p, uchar  data );
+void        __ovld __conv intel_sub_group_block_write_uc2(  __local uchar* p, uchar2 data );
+void        __ovld __conv intel_sub_group_block_write_uc4(  __local uchar* p, uchar4 data );
+void        __ovld __conv intel_sub_group_block_write_uc8(  __local uchar* p, uchar8 data );
+void        __ovld __conv intel_sub_group_block_write_uc16( __local uchar* p, uchar16 data );
+#endif // defined(cl_intel_subgroups_char)
+
+#if defined(cl_intel_subgroups_short)
+ushort      __ovld __conv intel_sub_group_block_read_us(  const __local ushort* p );
+ushort2     __ovld __conv intel_sub_group_block_read_us2( const __local ushort* p );
+ushort4     __ovld __conv intel_sub_group_block_read_us4( const __local ushort* p );
+ushort8     __ovld __conv intel_sub_group_block_read_us8( const __local ushort* p );
+ushort16    __ovld __conv intel_sub_group_block_read_us16( const __local ushort* p );
+
+void        __ovld __conv intel_sub_group_block_write_us(  __local ushort* p, ushort  data );
+void        __ovld __conv intel_sub_group_block_write_us2( __local ushort* p, ushort2 data );
+void        __ovld __conv intel_sub_group_block_write_us4( __local ushort* p, ushort4 data );
+void        __ovld __conv intel_sub_group_block_write_us8( __local ushort* p, ushort8 data );
+void        __ovld __conv intel_sub_group_block_write_us16( __local ushort* p, ushort16 data );
+#endif // defined(cl_intel_subgroups_short)
+
+#if defined(cl_intel_subgroups_long)
+ulong      __ovld __conv intel_sub_group_block_read_ul(  const __local ulong* p );
+ulong2     __ovld __conv intel_sub_group_block_read_ul2( const __local ulong* p );
+ulong4     __ovld __conv intel_sub_group_block_read_ul4( const __local ulong* p );
+ulong8     __ovld __conv intel_sub_group_block_read_ul8( const __local ulong* p );
+
+void        __ovld __conv intel_sub_group_block_write_ul(  __local ulong* p, ulong  data );
+void        __ovld __conv intel_sub_group_block_write_ul2( __local ulong* p, ulong2 data );
+void        __ovld __conv intel_sub_group_block_write_ul4( __local ulong* p, ulong4 data );
+void        __ovld __conv intel_sub_group_block_write_ul8( __local ulong* p, ulong8 data );
+#endif // defined(cl_intel_subgroups_long)
+#endif // cl_intel_subgroup_local_block_io
+
 #ifdef cl_intel_device_side_avc_motion_estimation
 #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : begin
 
diff --git a/clang/lib/Headers/prfchwintrin.h b/clang/lib/Headers/prfchwintrin.h
index eaea5f3cf8fe..8ec55d707371 100644
--- a/clang/lib/Headers/prfchwintrin.h
+++ b/clang/lib/Headers/prfchwintrin.h
@@ -14,6 +14,10 @@
 #ifndef __PRFCHWINTRIN_H
 #define __PRFCHWINTRIN_H
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 /// Loads a memory sequence containing the specified memory address into
 ///    all data cache levels.
 ///
@@ -26,11 +30,7 @@
 ///
 /// \param __P
 ///    A pointer specifying the memory address to be prefetched.
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-_m_prefetch(void *__P)
-{
-  __builtin_prefetch (__P, 0, 3 /* _MM_HINT_T0 */);
-}
+void _m_prefetch(void *__P);
 
 /// Loads a memory sequence containing the specified memory address into
 ///    the L1 data cache and sets the cache-coherency state to modified.
@@ -48,13 +48,10 @@ _m_prefetch(void *__P)
 ///
 /// \param __P
 ///    A pointer specifying the memory address to be prefetched.
-static __inline__ void __attribute__((__always_inline__, __nodebug__))
-_m_prefetchw(volatile const void *__P)
-{
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wcast-qual"
-  __builtin_prefetch ((const void*)__P, 1, 3 /* _MM_HINT_T0 */);
-#pragma clang diagnostic pop
-}
+void _m_prefetchw(volatile const void *__P);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
 
 #endif /* __PRFCHWINTRIN_H */
diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h
index 20e66d190113..6a6436977303 100644
--- a/clang/lib/Headers/xmmintrin.h
+++ b/clang/lib/Headers/xmmintrin.h
@@ -2198,8 +2198,9 @@ _mm_storer_ps(float *__p, __m128 __a)
 #define _MM_HINT_NTA 0
 
 #ifndef _MSC_VER
-/* FIXME: We have to #define this because "sel" must be a constant integer, and
-   Sema doesn't do any form of constant propagation yet. */
+// If _MSC_VER is defined, we use the builtin variant of _mm_prefetch.
+// Otherwise, we provide this macro, which includes a cast, allowing the user
+// to pass a pointer of any time. The _mm_prefetch accepts char to match MSVC.
 
 /// Loads one cache line of data from the specified address to a location
 ///    closer to the processor.
diff --git a/clang/lib/Index/CommentToXML.cpp b/clang/lib/Index/CommentToXML.cpp
index 672386171ecf..f54d8be79021 100644
--- a/clang/lib/Index/CommentToXML.cpp
+++ b/clang/lib/Index/CommentToXML.cpp
@@ -898,7 +898,7 @@ void CommentASTToXMLConverter::visitFullComment(const FullComment *C) {
     {
       // Print line and column number.
       SourceLocation Loc = DI->CurrentDecl->getLocation();
-      std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+      FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
       FileID FID = LocInfo.first;
       unsigned FileOffset = LocInfo.second;
 
diff --git a/clang/lib/Index/IndexBody.cpp b/clang/lib/Index/IndexBody.cpp
index 98ce6f73ec84..1979117d4695 100644
--- a/clang/lib/Index/IndexBody.cpp
+++ b/clang/lib/Index/IndexBody.cpp
@@ -69,7 +69,7 @@ class BodyIndexer : public RecursiveASTVisitor<BodyIndexer> {
     while (isa<CastExpr>(*It) || isa<ParenExpr>(*It)) {
       if (auto ICE = dyn_cast<ImplicitCastExpr>(*It)) {
         if (ICE->getCastKind() == CK_LValueToRValue)
-          Roles |= (unsigned)(unsigned)SymbolRole::Read;
+          Roles |= (unsigned)SymbolRole::Read;
       }
       if (It == StmtStack.begin())
         break;
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index 1d5dcacdaa41..6a884f7be00e 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -31,7 +31,7 @@ static bool printLoc(llvm::raw_ostream &OS, SourceLocation Loc,
     return true;
   }
   Loc = SM.getExpansionLoc(Loc);
-  const std::pair<FileID, unsigned> &Decomposed = SM.getDecomposedLoc(Loc);
+  const FileIDAndOffset &Decomposed = SM.getDecomposedLoc(Loc);
   OptionalFileEntryRef FE = SM.getFileEntryRefForID(Decomposed.first);
   if (FE) {
     OS << llvm::sys::path::filename(FE->getName());
diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt
index bf70cdfbee01..38cf139fa86a 100644
--- a/clang/lib/Interpreter/CMakeLists.txt
+++ b/clang/lib/Interpreter/CMakeLists.txt
@@ -27,6 +27,7 @@ add_clang_library(clangInterpreter
   Interpreter.cpp
   InterpreterValuePrinter.cpp
   InterpreterUtils.cpp
+  RemoteJITUtils.cpp
   Value.cpp
   ${WASM_SRC}
   PARTIAL_SOURCES_INTENDED
diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp
index 84feff82c63a..98fc0a5e383a 100644
--- a/clang/lib/Interpreter/Interpreter.cpp
+++ b/clang/lib/Interpreter/Interpreter.cpp
@@ -46,6 +46,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Serialization/ObjectFilePCHContainerReader.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
+#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Errc.h"
@@ -372,8 +373,11 @@ Interpreter::Interpreter(std::unique_ptr<CompilerInstance> Instance,
   auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
   TSCtx = std::make_unique<llvm::orc::ThreadSafeContext>(std::move(LLVMCtx));
 
-  Act = std::make_unique<IncrementalAction>(*CI, *TSCtx->getContext(), ErrOut,
-                                            *this, std::move(Consumer));
+  Act = TSCtx->withContextDo([&](llvm::LLVMContext *Ctx) {
+    return std::make_unique<IncrementalAction>(*CI, *Ctx, ErrOut, *this,
+                                               std::move(Consumer));
+  });
+
   if (ErrOut)
     return;
   CI->ExecuteAction(*Act);
@@ -455,10 +459,11 @@ const char *const Runtimes = R"(
 )";
 
 llvm::Expected<std::unique_ptr<Interpreter>>
-Interpreter::create(std::unique_ptr<CompilerInstance> CI) {
+Interpreter::create(std::unique_ptr<CompilerInstance> CI,
+                    std::unique_ptr<llvm::orc::LLJITBuilder> JB) {
   llvm::Error Err = llvm::Error::success();
-  auto Interp =
-      std::unique_ptr<Interpreter>(new Interpreter(std::move(CI), Err));
+  auto Interp = std::unique_ptr<Interpreter>(
+      new Interpreter(std::move(CI), Err, JB ? std::move(JB) : nullptr));
   if (Err)
     return std::move(Err);
 
@@ -493,10 +498,10 @@ Interpreter::createWithCUDA(std::unique_ptr<CompilerInstance> CI,
   std::unique_ptr<Interpreter> Interp = std::move(*InterpOrErr);
 
   llvm::Error Err = llvm::Error::success();
-  llvm::LLVMContext &LLVMCtx = *Interp->TSCtx->getContext();
 
-  auto DeviceAct =
-      std::make_unique<IncrementalAction>(*DCI, LLVMCtx, Err, *Interp);
+  auto DeviceAct = Interp->TSCtx->withContextDo([&](llvm::LLVMContext *Ctx) {
+    return std::make_unique<IncrementalAction>(*DCI, *Ctx, Err, *Interp);
+  });
 
   if (Err)
     return std::move(Err);
@@ -617,6 +622,25 @@ createJITTargetMachineBuilder(const std::string &TT) {
   return llvm::orc::JITTargetMachineBuilder(llvm::Triple(TT));
 }
 
+llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>
+Interpreter::createLLJITBuilder(
+    std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC,
+    llvm::StringRef OrcRuntimePath) {
+  const std::string &TT = EPC->getTargetTriple().getTriple();
+  auto JTMB = createJITTargetMachineBuilder(TT);
+  if (!JTMB)
+    return JTMB.takeError();
+  auto JB = IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB));
+  if (!JB)
+    return JB.takeError();
+
+  (*JB)->setExecutorProcessControl(std::move(EPC));
+  (*JB)->setPlatformSetUp(
+      llvm::orc::ExecutorNativePlatform(OrcRuntimePath.str()));
+
+  return std::move(*JB);
+}
+
 llvm::Error Interpreter::CreateExecutor() {
   if (IncrExecutor)
     return llvm::make_error<llvm::StringError>("Operation failed. "
@@ -756,11 +780,13 @@ llvm::Error Interpreter::LoadDynamicLibrary(const char *name) {
   if (!EE)
     return EE.takeError();
 
-  auto &DL = EE->getDataLayout();
-
-  if (auto DLSG = llvm::orc::DynamicLibrarySearchGenerator::Load(
-          name, DL.getGlobalPrefix()))
-    EE->getMainJITDylib().addGenerator(std::move(*DLSG));
+  if (llvm::Expected<
+          std::unique_ptr<llvm::orc::EPCDynamicLibrarySearchGenerator>>
+          DLSG = llvm::orc::EPCDynamicLibrarySearchGenerator::Load(
+              EE->getExecutionSession(), name))
+    // FIXME: Eventually we should put each library in its own JITDylib and
+    //        turn off process symbols by default.
+    EE->getProcessSymbolsJITDylib()->addGenerator(std::move(*DLSG));
   else
     return DLSG.takeError();
 #endif
diff --git a/clang/lib/Interpreter/RemoteJITUtils.cpp b/clang/lib/Interpreter/RemoteJITUtils.cpp
new file mode 100644
index 000000000000..c0e663b76478
--- /dev/null
+++ b/clang/lib/Interpreter/RemoteJITUtils.cpp
@@ -0,0 +1,267 @@
+//===-- RemoteJITUtils.cpp - Utilities for remote-JITing --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// FIXME: Unify this code with similar functionality in llvm-jitlink.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Interpreter/RemoteJITUtils.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
+#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h"
+#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
+#include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
+#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
+#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+
+#ifdef LLVM_ON_UNIX
+#include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <unistd.h>
+#endif // LLVM_ON_UNIX
+
+using namespace llvm;
+using namespace llvm::orc;
+
+Expected<uint64_t> getSlabAllocSize(StringRef SizeString) {
+  SizeString = SizeString.trim();
+
+  uint64_t Units = 1024;
+
+  if (SizeString.ends_with_insensitive("kb"))
+    SizeString = SizeString.drop_back(2).rtrim();
+  else if (SizeString.ends_with_insensitive("mb")) {
+    Units = 1024 * 1024;
+    SizeString = SizeString.drop_back(2).rtrim();
+  } else if (SizeString.ends_with_insensitive("gb")) {
+    Units = 1024 * 1024 * 1024;
+    SizeString = SizeString.drop_back(2).rtrim();
+  }
+
+  uint64_t SlabSize = 0;
+  if (SizeString.getAsInteger(10, SlabSize))
+    return make_error<StringError>("Invalid numeric format for slab size",
+                                   inconvertibleErrorCode());
+
+  return SlabSize * Units;
+}
+
+Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>>
+createSharedMemoryManager(SimpleRemoteEPC &SREPC,
+                          StringRef SlabAllocateSizeString) {
+  SharedMemoryMapper::SymbolAddrs SAs;
+  if (auto Err = SREPC.getBootstrapSymbols(
+          {{SAs.Instance, rt::ExecutorSharedMemoryMapperServiceInstanceName},
+           {SAs.Reserve,
+            rt::ExecutorSharedMemoryMapperServiceReserveWrapperName},
+           {SAs.Initialize,
+            rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName},
+           {SAs.Deinitialize,
+            rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName},
+           {SAs.Release,
+            rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName}}))
+    return std::move(Err);
+
+#ifdef _WIN32
+  size_t SlabSize = 1024 * 1024;
+#else
+  size_t SlabSize = 1024 * 1024 * 1024;
+#endif
+
+  if (!SlabAllocateSizeString.empty()) {
+    if (Expected<uint64_t> S = getSlabAllocSize(SlabAllocateSizeString))
+      SlabSize = *S;
+    else
+      return S.takeError();
+  }
+
+  return MapperJITLinkMemoryManager::CreateWithMapper<SharedMemoryMapper>(
+      SlabSize, SREPC, SAs);
+}
+
+Expected<std::unique_ptr<SimpleRemoteEPC>>
+launchExecutor(StringRef ExecutablePath, bool UseSharedMemory,
+               llvm::StringRef SlabAllocateSizeString) {
+#ifndef LLVM_ON_UNIX
+  // FIXME: Add support for Windows.
+  return make_error<StringError>("-" + ExecutablePath +
+                                     " not supported on non-unix platforms",
+                                 inconvertibleErrorCode());
+#elif !LLVM_ENABLE_THREADS
+  // Out of process mode using SimpleRemoteEPC depends on threads.
+  return make_error<StringError>(
+      "-" + ExecutablePath +
+          " requires threads, but LLVM was built with "
+          "LLVM_ENABLE_THREADS=Off",
+      inconvertibleErrorCode());
+#else
+
+  if (!sys::fs::can_execute(ExecutablePath))
+    return make_error<StringError>(
+        formatv("Specified executor invalid: {0}", ExecutablePath),
+        inconvertibleErrorCode());
+
+  constexpr int ReadEnd = 0;
+  constexpr int WriteEnd = 1;
+
+  // Pipe FDs.
+  int ToExecutor[2];
+  int FromExecutor[2];
+
+  pid_t ChildPID;
+
+  // Create pipes to/from the executor..
+  if (pipe(ToExecutor) != 0 || pipe(FromExecutor) != 0)
+    return make_error<StringError>("Unable to create pipe for executor",
+                                   inconvertibleErrorCode());
+
+  ChildPID = fork();
+
+  if (ChildPID == 0) {
+    // In the child...
+
+    // Close the parent ends of the pipes
+    close(ToExecutor[WriteEnd]);
+    close(FromExecutor[ReadEnd]);
+
+    // Execute the child process.
+    std::unique_ptr<char[]> ExecutorPath, FDSpecifier;
+    {
+      ExecutorPath = std::make_unique<char[]>(ExecutablePath.size() + 1);
+      strcpy(ExecutorPath.get(), ExecutablePath.data());
+
+      std::string FDSpecifierStr("filedescs=");
+      FDSpecifierStr += utostr(ToExecutor[ReadEnd]);
+      FDSpecifierStr += ',';
+      FDSpecifierStr += utostr(FromExecutor[WriteEnd]);
+      FDSpecifier = std::make_unique<char[]>(FDSpecifierStr.size() + 1);
+      strcpy(FDSpecifier.get(), FDSpecifierStr.c_str());
+    }
+
+    char *const Args[] = {ExecutorPath.get(), FDSpecifier.get(), nullptr};
+    int RC = execvp(ExecutorPath.get(), Args);
+    if (RC != 0) {
+      errs() << "unable to launch out-of-process executor \""
+             << ExecutorPath.get() << "\"\n";
+      exit(1);
+    }
+  }
+  // else we're the parent...
+
+  // Close the child ends of the pipes
+  close(ToExecutor[ReadEnd]);
+  close(FromExecutor[WriteEnd]);
+
+  SimpleRemoteEPC::Setup S = SimpleRemoteEPC::Setup();
+  if (UseSharedMemory)
+    S.CreateMemoryManager = [SlabAllocateSizeString](SimpleRemoteEPC &EPC) {
+      return createSharedMemoryManager(EPC, SlabAllocateSizeString);
+    };
+
+  return SimpleRemoteEPC::Create<FDSimpleRemoteEPCTransport>(
+      std::make_unique<DynamicThreadPoolTaskDispatcher>(std::nullopt),
+      std::move(S), FromExecutor[ReadEnd], ToExecutor[WriteEnd]);
+#endif
+}
+
+#if LLVM_ON_UNIX && LLVM_ENABLE_THREADS
+
+static Expected<int> connectTCPSocketImpl(std::string Host,
+                                          std::string PortStr) {
+  addrinfo *AI;
+  addrinfo Hints{};
+  Hints.ai_family = AF_INET;
+  Hints.ai_socktype = SOCK_STREAM;
+  Hints.ai_flags = AI_NUMERICSERV;
+
+  if (int EC = getaddrinfo(Host.c_str(), PortStr.c_str(), &Hints, &AI))
+    return make_error<StringError>(
+        formatv("address resolution failed ({0})", gai_strerror(EC)),
+        inconvertibleErrorCode());
+  // Cycle through the returned addrinfo structures and connect to the first
+  // reachable endpoint.
+  int SockFD;
+  addrinfo *Server;
+  for (Server = AI; Server != nullptr; Server = Server->ai_next) {
+    // socket might fail, e.g. if the address family is not supported. Skip to
+    // the next addrinfo structure in such a case.
+    if ((SockFD = socket(AI->ai_family, AI->ai_socktype, AI->ai_protocol)) < 0)
+      continue;
+
+    // If connect returns null, we exit the loop with a working socket.
+    if (connect(SockFD, Server->ai_addr, Server->ai_addrlen) == 0)
+      break;
+
+    close(SockFD);
+  }
+  freeaddrinfo(AI);
+
+  // If we reached the end of the loop without connecting to a valid endpoint,
+  // dump the last error that was logged in socket() or connect().
+  if (Server == nullptr)
+    return make_error<StringError>("invalid hostname",
+                                   inconvertibleErrorCode());
+
+  return SockFD;
+}
+#endif
+
+Expected<std::unique_ptr<SimpleRemoteEPC>>
+connectTCPSocket(StringRef NetworkAddress, bool UseSharedMemory,
+                 llvm::StringRef SlabAllocateSizeString) {
+#ifndef LLVM_ON_UNIX
+  // FIXME: Add TCP support for Windows.
+  return make_error<StringError>("-" + NetworkAddress +
+                                     " not supported on non-unix platforms",
+                                 inconvertibleErrorCode());
+#elif !LLVM_ENABLE_THREADS
+  // Out of process mode using SimpleRemoteEPC depends on threads.
+  return make_error<StringError>(
+      "-" + NetworkAddress +
+          " requires threads, but LLVM was built with "
+          "LLVM_ENABLE_THREADS=Off",
+      inconvertibleErrorCode());
+#else
+
+  auto CreateErr = [NetworkAddress](Twine Details) {
+    return make_error<StringError>(
+        formatv("Failed to connect TCP socket '{0}': {1}", NetworkAddress,
+                Details),
+        inconvertibleErrorCode());
+  };
+
+  StringRef Host, PortStr;
+  std::tie(Host, PortStr) = NetworkAddress.split(':');
+  if (Host.empty())
+    return CreateErr("Host name for -" + NetworkAddress + " can not be empty");
+  if (PortStr.empty())
+    return CreateErr("Port number in -" + NetworkAddress + " can not be empty");
+  int Port = 0;
+  if (PortStr.getAsInteger(10, Port))
+    return CreateErr("Port number '" + PortStr + "' is not a valid integer");
+
+  Expected<int> SockFD = connectTCPSocketImpl(Host.str(), PortStr.str());
+  if (!SockFD)
+    return SockFD.takeError();
+
+  SimpleRemoteEPC::Setup S = SimpleRemoteEPC::Setup();
+  if (UseSharedMemory)
+    S.CreateMemoryManager = [SlabAllocateSizeString](SimpleRemoteEPC &EPC) {
+      return createSharedMemoryManager(EPC, SlabAllocateSizeString);
+    };
+
+  return SimpleRemoteEPC::Create<FDSimpleRemoteEPCTransport>(
+      std::make_unique<DynamicThreadPoolTaskDispatcher>(std::nullopt),
+      std::move(S), *SockFD, *SockFD);
+#endif
+}
diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index 1b6b16c56114..d894c265a07a 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -206,6 +206,30 @@ static void skipOverSpaces(const char *&First, const char *const End) {
     ++First;
 }
 
+// Move back by one character, skipping escaped newlines (backslash + \n)
+static char previousChar(const char *First, const char *&Current) {
+  assert(Current > First);
+  --Current;
+  while (Current > First && isVerticalWhitespace(*Current)) {
+    // Check if the previous character is a backslash
+    if (Current > First && *(Current - 1) == '\\') {
+      // Use Lexer's getEscapedNewLineSize to get the size of the escaped
+      // newline
+      unsigned EscapeSize = Lexer::getEscapedNewLineSize(Current);
+      if (EscapeSize > 0) {
+        // Skip back over the entire escaped newline sequence (backslash +
+        // newline)
+        Current -= (1 + EscapeSize);
+      } else {
+        break;
+      }
+    } else {
+      break;
+    }
+  }
+  return *Current;
+}
+
 [[nodiscard]] static bool isRawStringLiteral(const char *First,
                                              const char *Current) {
   assert(First <= Current);
@@ -215,25 +239,27 @@ static void skipOverSpaces(const char *&First, const char *const End) {
     return false;
 
   // Check for an "R".
-  --Current;
-  if (*Current != 'R')
+  if (previousChar(First, Current) != 'R')
     return false;
-  if (First == Current || !isAsciiIdentifierContinue(*--Current))
+  if (First == Current ||
+      !isAsciiIdentifierContinue(previousChar(First, Current)))
     return true;
 
   // Check for a prefix of "u", "U", or "L".
   if (*Current == 'u' || *Current == 'U' || *Current == 'L')
-    return First == Current || !isAsciiIdentifierContinue(*--Current);
+    return First == Current ||
+           !isAsciiIdentifierContinue(previousChar(First, Current));
 
   // Check for a prefix of "u8".
-  if (*Current != '8' || First == Current || *Current-- != 'u')
+  if (*Current != '8' || First == Current ||
+      previousChar(First, Current) != 'u')
     return false;
-  return First == Current || !isAsciiIdentifierContinue(*--Current);
+  return First == Current ||
+         !isAsciiIdentifierContinue(previousChar(First, Current));
 }
 
 static void skipRawString(const char *&First, const char *const End) {
   assert(First[0] == '"');
-  assert(First[-1] == 'R');
 
   const char *Last = ++First;
   while (Last != End && *Last != '(')
@@ -393,7 +419,6 @@ static bool isQuoteCppDigitSeparator(const char *const Start,
 }
 
 void Scanner::skipLine(const char *&First, const char *const End) {
-  char LastNonWhitespace = ' ';
   for (;;) {
     assert(First <= End);
     if (First == End)
@@ -404,6 +429,9 @@ void Scanner::skipLine(const char *&First, const char *const End) {
       return;
     }
     const char *Start = First;
+    // Use `LastNonWhitespace`to track if a line-continuation has ever been seen
+    // before a new-line character:
+    char LastNonWhitespace = ' ';
     while (First != End && !isVerticalWhitespace(*First)) {
       // Iterate over strings correctly to avoid comments and newlines.
       if (*First == '"' ||
@@ -416,6 +444,14 @@ void Scanner::skipLine(const char *&First, const char *const End) {
         continue;
       }
 
+      // Continue on the same line if an EOL is preceded with backslash
+      if (First + 1 < End && *First == '\\') {
+        if (unsigned Len = isEOL(First + 1, End)) {
+          First += 1 + Len;
+          continue;
+        }
+      }
+
       // Iterate over comments correctly.
       if (*First != '/' || End - First < 2) {
         LastTokenPtr = First;
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index f4d16ecce393..46b04420eb62 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -44,7 +44,6 @@
 #include <optional>
 #include <string>
 #include <tuple>
-#include <utility>
 
 #ifdef __SSE4_2__
 #include <nmmintrin.h>
@@ -383,7 +382,7 @@ StringRef Lexer::getSpelling(SourceLocation loc,
                              const LangOptions &options,
                              bool *invalid) {
   // Break down the source location.
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc);
+  FileIDAndOffset locInfo = SM.getDecomposedLoc(loc);
 
   // Try to the load the file buffer.
   bool invalidTemp = false;
@@ -521,7 +520,7 @@ bool Lexer::getRawToken(SourceLocation Loc, Token &Result,
   // If this comes from a macro expansion, we really do want the macro name, not
   // the token this macro expanded to.
   Loc = SM.getExpansionLoc(Loc);
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
   bool Invalid = false;
   StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
@@ -563,7 +562,7 @@ static SourceLocation getBeginningOfFileToken(SourceLocation Loc,
                                               const SourceManager &SM,
                                               const LangOptions &LangOpts) {
   assert(Loc.isFileID());
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
   if (LocInfo.first.isInvalid())
     return Loc;
 
@@ -618,9 +617,8 @@ SourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc,
 
   SourceLocation FileLoc = SM.getSpellingLoc(Loc);
   SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts);
-  std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc);
-  std::pair<FileID, unsigned> BeginFileLocInfo =
-      SM.getDecomposedLoc(BeginFileLoc);
+  FileIDAndOffset FileLocInfo = SM.getDecomposedLoc(FileLoc);
+  FileIDAndOffset BeginFileLocInfo = SM.getDecomposedLoc(BeginFileLoc);
   assert(FileLocInfo.first == BeginFileLocInfo.first &&
          FileLocInfo.second >= BeginFileLocInfo.second);
   return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second);
@@ -931,9 +929,7 @@ static CharSourceRange makeRangeFromFileLocs(CharSourceRange Range,
   }
 
   // Break down the source locations.
-  FileID FID;
-  unsigned BeginOffs;
-  std::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin);
+  auto [FID, BeginOffs] = SM.getDecomposedLoc(Begin);
   if (FID.isInvalid())
     return {};
 
@@ -1033,7 +1029,7 @@ StringRef Lexer::getSourceText(CharSourceRange Range,
   }
 
   // Break down the source location.
-  std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin());
+  FileIDAndOffset beginInfo = SM.getDecomposedLoc(Range.getBegin());
   if (beginInfo.first.isInvalid()) {
     if (Invalid) *Invalid = true;
     return {};
@@ -1099,7 +1095,7 @@ StringRef Lexer::getImmediateMacroName(SourceLocation Loc,
 
   // Dig out the buffer where the macro name was spelled and the extents of the
   // name so that we can render it into the expansion note.
-  std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
   unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
   StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
   return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
@@ -1126,7 +1122,7 @@ StringRef Lexer::getImmediateMacroNameForDiagnostics(
 
   // Dig out the buffer where the macro name was spelled and the extents of the
   // name so that we can render it into the expansion note.
-  std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset ExpansionInfo = SM.getDecomposedLoc(Loc);
   unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts);
   StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first);
   return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength);
@@ -1160,7 +1156,7 @@ StringRef Lexer::getIndentationForLine(SourceLocation Loc,
                                        const SourceManager &SM) {
   if (Loc.isInvalid() || Loc.isMacroID())
     return {};
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
   if (LocInfo.first.isInvalid())
     return {};
   bool Invalid = false;
@@ -1334,7 +1330,7 @@ std::optional<Token> Lexer::findNextToken(SourceLocation Loc,
   Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts);
 
   // Break down the source location.
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
 
   // Try to load the file buffer.
   bool InvalidTemp = false;
@@ -3228,6 +3224,7 @@ std::optional<Token> Lexer::peekNextPPToken() {
   bool atStartOfLine = IsAtStartOfLine;
   bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
   bool leadingSpace = HasLeadingSpace;
+  bool isFirstPPToken = IsFirstPPToken;
 
   Token Tok;
   Lex(Tok);
@@ -3238,7 +3235,7 @@ std::optional<Token> Lexer::peekNextPPToken() {
   HasLeadingSpace = leadingSpace;
   IsAtStartOfLine = atStartOfLine;
   IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
-
+  IsFirstPPToken = isFirstPPToken;
   // Restore the lexer back to non-skipping mode.
   LexingRawMode = false;
 
@@ -3740,10 +3737,6 @@ bool Lexer::Lex(Token &Result) {
   bool returnedToken = LexTokenInternal(Result, atPhysicalStartOfLine);
   // (After the LexTokenInternal call, the lexer might be destroyed.)
   assert((returnedToken || !isRawLex) && "Raw lex must succeed");
-
-  if (returnedToken && Result.isFirstPPToken() && PP &&
-      !PP->hasSeenMainFileFirstPPToken())
-    PP->HandleMainFileFirstPPToken(Result);
   return returnedToken;
 }
 
@@ -4547,8 +4540,6 @@ const char *Lexer::convertDependencyDirectiveToken(
   Result.setFlag((Token::TokenFlags)DDTok.Flags);
   Result.setLength(DDTok.Length);
   BufferPtr = TokPtr + DDTok.Length;
-  if (PP && !PP->hasSeenMainFileFirstPPToken() && Result.isFirstPPToken())
-    PP->HandleMainFileFirstPPToken(Result);
   return TokPtr;
 }
 
diff --git a/clang/lib/Lex/MacroInfo.cpp b/clang/lib/Lex/MacroInfo.cpp
index c33276ea71cb..8b296d5b2274 100644
--- a/clang/lib/Lex/MacroInfo.cpp
+++ b/clang/lib/Lex/MacroInfo.cpp
@@ -22,7 +22,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <optional>
-#include <utility>
 
 using namespace clang;
 
@@ -69,10 +68,8 @@ unsigned MacroInfo::getDefinitionLengthSlow(const SourceManager &SM) const {
          "Macro defined in macro?");
   assert((macroEnd.isFileID() || lastToken.is(tok::comment)) &&
          "Macro defined in macro?");
-  std::pair<FileID, unsigned>
-      startInfo = SM.getDecomposedExpansionLoc(macroStart);
-  std::pair<FileID, unsigned>
-      endInfo = SM.getDecomposedExpansionLoc(macroEnd);
+  FileIDAndOffset startInfo = SM.getDecomposedExpansionLoc(macroStart);
+  FileIDAndOffset endInfo = SM.getDecomposedExpansionLoc(macroEnd);
   assert(startInfo.first == endInfo.first &&
          "Macro definition spanning multiple FileIDs ?");
   assert(startInfo.second <= endInfo.second);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index c8974e5a3528..e6da19d24f1c 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -183,9 +183,9 @@ static bool isReservedCXXAttributeName(Preprocessor &PP, IdentifierInfo *II) {
     AttributeCommonInfo::AttrArgsInfo AttrArgsInfo =
         AttributeCommonInfo::getCXX11AttrArgsInfo(II);
     if (AttrArgsInfo == AttributeCommonInfo::AttrArgsInfo::Required)
-      return PP.isNextPPTokenOneOf<tok::l_paren>();
+      return PP.isNextPPTokenOneOf(tok::l_paren);
 
-    return !PP.isNextPPTokenOneOf<tok::l_paren>() ||
+    return !PP.isNextPPTokenOneOf(tok::l_paren) ||
            AttrArgsInfo == AttributeCommonInfo::AttrArgsInfo::Optional;
   }
   return false;
@@ -1242,9 +1242,6 @@ void Preprocessor::HandleDirective(Token &Result) {
   // pp-directive.
   bool ReadAnyTokensBeforeDirective =CurPPLexer->MIOpt.getHasReadAnyTokensVal();
 
-  if (!hasSeenMainFileFirstPPToken())
-    HandleMainFileFirstPPToken(Result);
-
   // Save the '#' token in case we need to return it later.
   Token SavedHash = Result;
 
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 709cf3bb87c8..890567cfd324 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -431,9 +431,6 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   // to disable the optimization in this case.
   if (CurPPLexer) CurPPLexer->MIOpt.ExpandedMacro();
 
-  if (!hasSeenMainFileFirstPPToken())
-    HandleMainFileFirstPPToken(Identifier);
-
   // If this is a builtin macro, like __LINE__ or _Pragma, handle it specially.
   if (MI->isBuiltinMacro()) {
     if (Callbacks)
@@ -1464,8 +1461,7 @@ static IdentifierInfo *ExpectFeatureIdentifierInfo(Token &Tok,
 
 /// Implements the __is_target_arch builtin macro.
 static bool isTargetArch(const TargetInfo &TI, const IdentifierInfo *II) {
-  std::string ArchName = II->getName().lower() + "--";
-  llvm::Triple Arch(ArchName);
+  llvm::Triple Arch(II->getName().lower() + "--");
   const llvm::Triple &TT = TI.getTriple();
   if (TT.isThumb()) {
     // arm matches thumb or thumbv7. armv7 matches thumbv7.
@@ -1494,9 +1490,7 @@ static bool isTargetVendor(const TargetInfo &TI, const IdentifierInfo *II) {
 
 /// Implements the __is_target_os builtin macro.
 static bool isTargetOS(const TargetInfo &TI, const IdentifierInfo *II) {
-  std::string OSName =
-      (llvm::Twine("unknown-unknown-") + II->getName().lower()).str();
-  llvm::Triple OS(OSName);
+  llvm::Triple OS(llvm::Twine("unknown-unknown-") + II->getName().lower());
   if (OS.getOS() == llvm::Triple::Darwin) {
     // Darwin matches macos, ios, etc.
     return TI.getTriple().isOSDarwin();
@@ -1507,12 +1501,11 @@ static bool isTargetOS(const TargetInfo &TI, const IdentifierInfo *II) {
 /// Implements the __is_target_environment builtin macro.
 static bool isTargetEnvironment(const TargetInfo &TI,
                                 const IdentifierInfo *II) {
-  std::string EnvName = (llvm::Twine("---") + II->getName().lower()).str();
-  llvm::Triple Env(EnvName);
+  llvm::Triple Env(llvm::Twine("---") + II->getName().lower());
   // The unknown environment is matched only if
   // '__is_target_environment(unknown)' is used.
   if (Env.getEnvironment() == llvm::Triple::UnknownEnvironment &&
-      EnvName != "---unknown")
+      Env.getEnvironmentName() != "unknown")
     return false;
   return TI.getTriple().getEnvironment() == Env.getEnvironment();
 }
@@ -1524,9 +1517,7 @@ static bool isTargetVariantOS(const TargetInfo &TI, const IdentifierInfo *II) {
     if (!VariantTriple)
       return false;
 
-    std::string OSName =
-        (llvm::Twine("unknown-unknown-") + II->getName().lower()).str();
-    llvm::Triple OS(OSName);
+    llvm::Triple OS(llvm::Twine("unknown-unknown-") + II->getName().lower());
     if (OS.getOS() == llvm::Triple::Darwin) {
       // Darwin matches macos, ios, etc.
       return VariantTriple->isOSDarwin();
@@ -1543,8 +1534,7 @@ static bool isTargetVariantEnvironment(const TargetInfo &TI,
     const llvm::Triple *VariantTriple = TI.getDarwinTargetVariantTriple();
     if (!VariantTriple)
       return false;
-    std::string EnvName = (llvm::Twine("---") + II->getName().lower()).str();
-    llvm::Triple Env(EnvName);
+    llvm::Triple Env(llvm::Twine("---") + II->getName().lower());
     return VariantTriple->getEnvironment() == Env.getEnvironment();
   }
   return false;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 7fecbe9eee53..bcd3ea60ce3d 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -510,7 +510,7 @@ void Preprocessor::CreateString(StringRef Str, Token &Tok,
 SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
   auto &SM = getSourceManager();
   SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(SpellingLoc);
   bool Invalid = false;
   StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid);
   if (Invalid)
@@ -566,6 +566,21 @@ void Preprocessor::EnterMainSourceFile() {
     // #imported, it won't be re-entered.
     if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(MainFileID))
       markIncluded(*FE);
+
+    // Record the first PP token in the main file. This is used to generate
+    // better diagnostics for C++ modules.
+    //
+    // // This is a comment.
+    // #define FOO int  // note: add 'module;' to the start of the file
+    // ^ FirstPPToken   //       to introduce a global module fragment.
+    //
+    // export module M; // error: module declaration must occur
+    //                  //        at the start of the translation unit.
+    if (getLangOpts().CPlusPlusModules) {
+      std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
+      if (FirstPPTok && FirstPPTok->isFirstPPToken())
+        FirstPPTokenLoc = FirstPPTok->getLocation();
+    }
   }
 
   // Preprocess Predefines to populate the initial preprocessor state.
@@ -813,14 +828,14 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) {
       if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
         // C99 6.10.3p10: If the preprocessing token immediately after the
         // macro name isn't a '(', this macro should not be expanded.
-        if (!MI->isFunctionLike() || isNextPPTokenOneOf<tok::l_paren>())
+        if (!MI->isFunctionLike() || isNextPPTokenOneOf(tok::l_paren))
           return HandleMacroExpandedIdentifier(Identifier, MD);
       } else {
         // C99 6.10.3.4p2 says that a disabled macro may never again be
         // expanded, even if it's in a context where it could be expanded in the
         // future.
         Identifier.setFlag(Token::DisableExpand);
-        if (MI->isObjectLike() || isNextPPTokenOneOf<tok::l_paren>())
+        if (MI->isObjectLike() || isNextPPTokenOneOf(tok::l_paren))
           Diag(Identifier, diag::pp_disabled_macro_expansion);
       }
     }
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index fbb8c4262d6d..47f4134fb146 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -748,6 +748,7 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
   const char *ResultTokStrPtr = nullptr;
   SourceLocation StartLoc = LHSTok.getLocation();
   SourceLocation PasteOpLoc;
+  bool HasUCNs = false;
 
   auto IsAtEnd = [&TokenStream, &CurIdx] {
     return TokenStream.size() == CurIdx;
@@ -885,6 +886,9 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
 
     // Finally, replace LHS with the result, consume the RHS, and iterate.
     ++CurIdx;
+
+    // Set Token::HasUCN flag if LHS or RHS contains any UCNs.
+    HasUCNs = LHSTok.hasUCN() || RHS.hasUCN() || HasUCNs;
     LHSTok = Result;
   } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
 
@@ -913,6 +917,13 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream,
   // token pasting re-lexes the result token in raw mode, identifier information
   // isn't looked up.  As such, if the result is an identifier, look up id info.
   if (LHSTok.is(tok::raw_identifier)) {
+
+    // If there has any UNCs in concated token, we should mark this token
+    // with Token::HasUCN flag, then LookUpIdentifierInfo will expand UCNs in
+    // token.
+    if (HasUCNs)
+      LHSTok.setFlag(Token::HasUCN);
+
     // Look up the identifier info for the token.  We disabled identifier lookup
     // by saying we're skipping contents, so we need to do this manually.
     PP.LookUpIdentifierInfo(LHSTok);
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index c1493a5bfd3b..6b0564dca6f4 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -25,7 +25,6 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
-#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCodeCompletion.h"
@@ -4957,7 +4956,8 @@ void Parser::ParseHLSLRootSignatureAttributeArgs(ParsedAttributes &Attrs) {
     // Invoke the root signature parser to construct the in-memory constructs
     hlsl::RootSignatureLexer Lexer(Signature, SignatureLoc);
     SmallVector<llvm::hlsl::rootsig::RootElement> RootElements;
-    hlsl::RootSignatureParser Parser(RootElements, Lexer, PP);
+    hlsl::RootSignatureParser Parser(getLangOpts().HLSLRootSigVer, RootElements,
+                                     Lexer, PP);
     if (Parser.parse()) {
       T.consumeClose();
       return;
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 3cf3d4ea7d70..bc238a9517a3 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -43,8 +43,9 @@
 #include <optional>
 using namespace clang;
 
-ExprResult Parser::ParseExpression(TypeCastState isTypeCast) {
-  ExprResult LHS(ParseAssignmentExpression(isTypeCast));
+ExprResult
+Parser::ParseExpression(TypoCorrectionTypeBehavior CorrectionBehavior) {
+  ExprResult LHS(ParseAssignmentExpression(CorrectionBehavior));
   return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
@@ -71,7 +72,8 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
   return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
-ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
+ExprResult Parser::ParseAssignmentExpression(
+    TypoCorrectionTypeBehavior CorrectionBehavior) {
   if (Tok.is(tok::code_completion)) {
     cutOffParsing();
     Actions.CodeCompletion().CodeCompleteExpression(
@@ -86,7 +88,7 @@ ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
 
   ExprResult LHS =
       ParseCastExpression(CastParseKind::AnyCastExpr,
-                          /*isAddressOfOperand=*/false, isTypeCast);
+                          /*isAddressOfOperand=*/false, CorrectionBehavior);
   return ParseRHSOfBinaryExpression(LHS, prec::Assignment);
 }
 
@@ -98,9 +100,9 @@ ExprResult Parser::ParseConditionalExpression() {
     return ExprError();
   }
 
-  ExprResult LHS = ParseCastExpression(CastParseKind::AnyCastExpr,
-                                       /*isAddressOfOperand=*/false,
-                                       TypeCastState::NotTypeCast);
+  ExprResult LHS = ParseCastExpression(
+      CastParseKind::AnyCastExpr,
+      /*isAddressOfOperand=*/false, TypoCorrectionTypeBehavior::AllowNonTypes);
   return ParseRHSOfBinaryExpression(LHS, prec::Conditional);
 }
 
@@ -116,14 +118,14 @@ Parser::ParseAssignmentExprWithObjCMessageExprStart(SourceLocation LBracLoc,
   return ParseRHSOfBinaryExpression(R, prec::Assignment);
 }
 
-ExprResult
-Parser::ParseConstantExpressionInExprEvalContext(TypeCastState isTypeCast) {
+ExprResult Parser::ParseConstantExpressionInExprEvalContext(
+    TypoCorrectionTypeBehavior CorrectionBehavior) {
   assert(Actions.ExprEvalContexts.back().Context ==
              Sema::ExpressionEvaluationContext::ConstantEvaluated &&
          "Call this function only if your ExpressionEvaluationContext is "
          "already ConstantEvaluated");
-  ExprResult LHS(
-      ParseCastExpression(CastParseKind::AnyCastExpr, false, isTypeCast));
+  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                                     CorrectionBehavior));
   ExprResult Res(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   return Actions.ActOnConstantExpression(Res);
 }
@@ -135,7 +137,8 @@ ExprResult Parser::ParseConstantExpression() {
   // C++98 and C++11 have no such rule, but this is only a defect in C++98.
   EnterExpressionEvaluationContext ConstantEvaluated(
       Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
-  return ParseConstantExpressionInExprEvalContext(TypeCastState::NotTypeCast);
+  return ParseConstantExpressionInExprEvalContext(
+      TypoCorrectionTypeBehavior::AllowNonTypes);
 }
 
 ExprResult Parser::ParseArrayBoundExpression() {
@@ -163,7 +166,8 @@ ExprResult Parser::ParseArrayBoundExpression() {
       break;
     Iter->InConditionallyConstantEvaluateContext = true;
   }
-  return ParseConstantExpressionInExprEvalContext(TypeCastState::NotTypeCast);
+  return ParseConstantExpressionInExprEvalContext(
+      TypoCorrectionTypeBehavior::AllowNonTypes);
 }
 
 ExprResult Parser::ParseCaseExpression(SourceLocation CaseLoc) {
@@ -171,8 +175,9 @@ ExprResult Parser::ParseCaseExpression(SourceLocation CaseLoc) {
       Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
   Actions.currentEvaluationContext().IsCaseExpr = true;
 
-  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
-                                     TypeCastState::NotTypeCast));
+  ExprResult LHS(
+      ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                          TypoCorrectionTypeBehavior::AllowNonTypes));
   ExprResult Res(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   return Actions.ActOnCaseExpr(CaseLoc, Res);
 }
@@ -194,11 +199,10 @@ Parser::ParseConstraintLogicalAndExpression(bool IsTrailingRequiresClause) {
       Actions, Sema::ExpressionEvaluationContext::Unevaluated);
   bool NotPrimaryExpression = false;
   auto ParsePrimary = [&]() {
-    ExprResult E =
-        ParseCastExpression(CastParseKind::PrimaryExprOnly,
-                            /*isAddressOfOperand=*/false,
-                            /*isTypeCast=*/TypeCastState::NotTypeCast,
-                            /*isVectorLiteral=*/false, &NotPrimaryExpression);
+    ExprResult E = ParseCastExpression(
+        CastParseKind::PrimaryExprOnly,
+        /*isAddressOfOperand=*/false, TypoCorrectionTypeBehavior::AllowNonTypes,
+        /*isVectorLiteral=*/false, &NotPrimaryExpression);
     if (E.isInvalid())
       return ExprError();
     auto RecoverFromNonPrimary = [&] (ExprResult E, bool Note) {
@@ -555,18 +559,14 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
   }
 }
 
-ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
-                                       bool isAddressOfOperand,
-                                       TypeCastState isTypeCast,
-                                       bool isVectorLiteral,
-                                       bool *NotPrimaryExpression) {
+ExprResult
+Parser::ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand,
+                            TypoCorrectionTypeBehavior CorrectionBehavior,
+                            bool isVectorLiteral, bool *NotPrimaryExpression) {
   bool NotCastExpr;
-  ExprResult Res = ParseCastExpression(ParseKind,
-                                       isAddressOfOperand,
-                                       NotCastExpr,
-                                       isTypeCast,
-                                       isVectorLiteral,
-                                       NotPrimaryExpression);
+  ExprResult Res = ParseCastExpression(ParseKind, isAddressOfOperand,
+                                       NotCastExpr, CorrectionBehavior,
+                                       isVectorLiteral, NotPrimaryExpression);
   if (NotCastExpr)
     Diag(Tok, diag::err_expected_expression);
   return Res;
@@ -574,10 +574,14 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
 namespace {
 class CastExpressionIdValidator final : public CorrectionCandidateCallback {
- public:
-  CastExpressionIdValidator(Token Next, bool AllowTypes, bool AllowNonTypes)
-      : NextToken(Next), AllowNonTypes(AllowNonTypes) {
-    WantTypeSpecifiers = WantFunctionLikeCasts = AllowTypes;
+public:
+  CastExpressionIdValidator(Token Next,
+                            TypoCorrectionTypeBehavior CorrectionBehavior)
+      : NextToken(Next) {
+    WantTypeSpecifiers = WantFunctionLikeCasts =
+        (CorrectionBehavior != TypoCorrectionTypeBehavior::AllowNonTypes);
+    AllowNonTypes =
+        (CorrectionBehavior != TypoCorrectionTypeBehavior::AllowTypes);
   }
 
   bool ValidateCandidate(const TypoCorrection &candidate) override {
@@ -713,12 +717,11 @@ ExprResult Parser::ParseBuiltinPtrauthTypeDiscriminator() {
       /*isType=*/true, Ty.get().getAsOpaquePtr(), SourceRange(Loc, EndLoc));
 }
 
-ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
-                                       bool isAddressOfOperand,
-                                       bool &NotCastExpr,
-                                       TypeCastState isTypeCast,
-                                       bool isVectorLiteral,
-                                       bool *NotPrimaryExpression) {
+ExprResult
+Parser::ParseCastExpression(CastParseKind ParseKind, bool isAddressOfOperand,
+                            bool &NotCastExpr,
+                            TypoCorrectionTypeBehavior CorrectionBehavior,
+                            bool isVectorLiteral, bool *NotPrimaryExpression) {
   ExprResult Res;
   tok::TokenKind SavedKind = Tok.getKind();
   auto SavedType = PreferredType;
@@ -757,9 +760,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     }
     ParsedType CastTy;
     SourceLocation RParenLoc;
-    Res = ParseParenExpression(ParenExprType, false /*stopIfCastExr*/,
-                               isTypeCast == TypeCastState::IsTypeCast, CastTy,
-                               RParenLoc);
+    Res = ParseParenExpression(ParenExprType, /*StopIfCastExr=*/false,
+                               ParenExprKind::Unknown, CorrectionBehavior,
+                               CastTy, RParenLoc);
 
     // FIXME: What should we do if a vector literal is followed by a
     // postfix-expression suffix? Usually postfix operators are permitted on
@@ -842,8 +845,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
 
   case tok::annot_embed: {
     injectEmbedTokens();
-    return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                               isVectorLiteral, NotPrimaryExpression);
+    return ParseCastExpression(ParseKind, isAddressOfOperand,
+                               CorrectionBehavior, isVectorLiteral,
+                               NotPrimaryExpression);
   }
 
   case tok::kw___super:
@@ -852,8 +856,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     if (TryAnnotateTypeOrScopeToken())
       return ExprError();
     assert(Tok.isNot(tok::kw_decltype) && Tok.isNot(tok::kw___super));
-    return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                               isVectorLiteral, NotPrimaryExpression);
+    return ParseCastExpression(ParseKind, isAddressOfOperand,
+                               CorrectionBehavior, isVectorLiteral,
+                               NotPrimaryExpression);
 
   case tok::identifier:
   ParseIdentifier: {    // primary-expression: identifier
@@ -873,8 +878,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         // indexing
         if (!TryAnnotateTypeOrScopeToken() &&
             Tok.isOneOf(tok::annot_pack_indexing_type, tok::annot_cxxscope))
-          return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                                     isVectorLiteral, NotPrimaryExpression);
+          return ParseCastExpression(ParseKind, isAddressOfOperand,
+                                     CorrectionBehavior, isVectorLiteral,
+                                     NotPrimaryExpression);
       }
 
       // If this identifier was reverted from a token ID, and the next token
@@ -886,9 +892,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         tok::TokenKind Kind;
         if (isRevertibleTypeTrait(II, &Kind)) {
           Tok.setKind(Kind);
-          return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                     NotCastExpr, isTypeCast,
-                                     isVectorLiteral, NotPrimaryExpression);
+          return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                     CorrectionBehavior, isVectorLiteral,
+                                     NotPrimaryExpression);
         }
       }
 
@@ -899,9 +905,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         if (TryAnnotateTypeOrScopeToken())
           return ExprError();
         if (!Tok.is(tok::identifier))
-          return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                     NotCastExpr, isTypeCast,
-                                     isVectorLiteral,
+          return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                     CorrectionBehavior, isVectorLiteral,
                                      NotPrimaryExpression);
       }
     }
@@ -998,10 +1003,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     CXXScopeSpec ScopeSpec;
     SourceLocation TemplateKWLoc;
     Token Replacement;
-    CastExpressionIdValidator Validator(
-        /*Next=*/Tok,
-        /*AllowTypes=*/isTypeCast != TypeCastState::NotTypeCast,
-        /*AllowNonTypes=*/isTypeCast != TypeCastState::IsTypeCast);
+    CastExpressionIdValidator Validator(Tok, CorrectionBehavior);
     Validator.IsAddressOfOperand = isAddressOfOperand;
     if (Tok.isOneOf(tok::periodstar, tok::arrowstar)) {
       Validator.WantExpressionKeywords = false;
@@ -1017,10 +1019,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
         Tok.is(tok::r_paren) ? nullptr : &Replacement);
     if (!Res.isInvalid() && Res.isUnset()) {
       UnconsumeToken(Replacement);
-      return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                 NotCastExpr, isTypeCast,
-                                 /*isVectorLiteral=*/false,
-                                 NotPrimaryExpression);
+      return ParseCastExpression(
+          ParseKind, isAddressOfOperand, NotCastExpr, CorrectionBehavior,
+          /*isVectorLiteral=*/false, NotPrimaryExpression);
     }
     Res = tryParseCXXPackIndexingExpression(Res);
     if (!Res.isInvalid() && Tok.is(tok::less))
@@ -1102,10 +1103,11 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     // One special case is implicitly handled here: if the preceding tokens are
     // an ambiguous cast expression, such as "(T())++", then we recurse to
     // determine whether the '++' is prefix or postfix.
-    Res = ParseCastExpression(
-        getLangOpts().CPlusPlus ? CastParseKind::UnaryExprOnly
-                                : CastParseKind::AnyCastExpr,
-        /*isAddressOfOperand*/ false, NotCastExpr, TypeCastState::NotTypeCast);
+    Res = ParseCastExpression(getLangOpts().CPlusPlus
+                                  ? CastParseKind::UnaryExprOnly
+                                  : CastParseKind::AnyCastExpr,
+                              /*isAddressOfOperand*/ false, NotCastExpr,
+                              TypoCorrectionTypeBehavior::AllowNonTypes);
     if (NotCastExpr) {
       // If we return with NotCastExpr = true, we must not consume any tokens,
       // so put the token back where we found it.
@@ -1366,7 +1368,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       return ExprError();
     if (!Tok.is(tok::annot_cxxscope))
       return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
-                                 isTypeCast, isVectorLiteral,
+                                 CorrectionBehavior, isVectorLiteral,
                                  NotPrimaryExpression);
 
     Token Next = NextToken();
@@ -1382,7 +1384,7 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
                                        /*EnteringContext=*/false);
         AnnotateTemplateIdTokenAsType(SS, ImplicitTypenameContext::Yes);
         return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
-                                   isTypeCast, isVectorLiteral,
+                                   CorrectionBehavior, isVectorLiteral,
                                    NotPrimaryExpression);
       }
     }
@@ -1400,8 +1402,8 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
       // expression.
       CXXScopeSpec SS;
       AnnotateTemplateIdTokenAsType(SS, ImplicitTypenameContext::Yes);
-      return ParseCastExpression(ParseKind, isAddressOfOperand,
-                                 NotCastExpr, isTypeCast, isVectorLiteral,
+      return ParseCastExpression(ParseKind, isAddressOfOperand, NotCastExpr,
+                                 CorrectionBehavior, isVectorLiteral,
                                  NotPrimaryExpression);
     }
 
@@ -1419,8 +1421,9 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind,
     if (TryAnnotateTypeOrScopeToken())
       return ExprError();
     if (!Tok.is(tok::coloncolon))
-      return ParseCastExpression(ParseKind, isAddressOfOperand, isTypeCast,
-                                 isVectorLiteral, NotPrimaryExpression);
+      return ParseCastExpression(ParseKind, isAddressOfOperand,
+                                 CorrectionBehavior, isVectorLiteral,
+                                 NotPrimaryExpression);
 
     // ::new -> [C++] new-expression
     // ::delete -> [C++] delete-expression
@@ -2110,12 +2113,27 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok,
     // If it starts with a '(', we know that it is either a parenthesized
     // type-name, or it is a unary-expression that starts with a compound
     // literal, or starts with a primary-expression that is a parenthesized
-    // expression.
+    // expression. Most unary operators have an expression form without parens
+    // as part of the grammar for the operator, and a type form with the parens
+    // as part of the grammar for the operator. However, typeof and
+    // typeof_unqual require parens for both forms. This means that we *know*
+    // that the open and close parens cannot be part of a cast expression,
+    // which means we definitely are not parsing a compound literal expression.
+    // This disambiguates a case like enum E : typeof(int) { }; where we've
+    // parsed typeof and need to handle the (int){} tokens properly despite
+    // them looking like a compound literal, as in sizeof (int){}; where the
+    // parens could be part of a parenthesized type name or for a cast
+    // expression of some kind.
+    bool ParenKnownToBeNonCast =
+        OpTok.isOneOf(tok::kw_typeof, tok::kw_typeof_unqual);
     ParenParseOption ExprType = ParenParseOption::CastExpr;
     SourceLocation LParenLoc = Tok.getLocation(), RParenLoc;
 
-    Operand = ParseParenExpression(ExprType, true/*stopIfCastExpr*/,
-                                   false, CastTy, RParenLoc);
+    Operand = ParseParenExpression(
+        ExprType, /*StopIfCastExr=*/true,
+        ParenKnownToBeNonCast ? ParenExprKind::PartOfOperator
+                              : ParenExprKind::Unknown,
+        TypoCorrectionTypeBehavior::AllowBoth, CastTy, RParenLoc);
     CastRange = SourceRange(LParenLoc, RParenLoc);
 
     // If ParseParenExpression parsed a '(typename)' sequence only, then this is
@@ -2590,9 +2608,10 @@ bool Parser::tryParseOpenMPArrayShapingCastPart() {
 }
 
 ExprResult
-Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
-                             bool isTypeCast, ParsedType &CastTy,
-                             SourceLocation &RParenLoc) {
+Parser::ParseParenExpression(ParenParseOption &ExprType, bool StopIfCastExpr,
+                             ParenExprKind ParenBehavior,
+                             TypoCorrectionTypeBehavior CorrectionBehavior,
+                             ParsedType &CastTy, SourceLocation &RParenLoc) {
   assert(Tok.is(tok::l_paren) && "Not a paren expr!");
   ColonProtectionRAIIObject ColonProtection(*this, false);
   BalancedDelimiterTracker T(*this, tok::l_paren);
@@ -2714,7 +2733,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     // in which case we should treat it as type-id.
     // if stopIfCastExpr is false, we need to determine the context past the
     // parens, so we defer to ParseCXXAmbiguousParenExpression for that.
-    if (isAmbiguousTypeId && !stopIfCastExpr) {
+    if (isAmbiguousTypeId && !StopIfCastExpr) {
       ExprResult res = ParseCXXAmbiguousParenExpression(ExprType, CastTy, T,
                                                         ColonProtection);
       RParenLoc = T.getCloseLocation();
@@ -2747,7 +2766,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
       T.consumeClose();
       ColonProtection.restore();
       RParenLoc = T.getCloseLocation();
-      if (Tok.is(tok::l_brace)) {
+      if (ParenBehavior == ParenExprKind::Unknown && Tok.is(tok::l_brace)) {
         ExprType = ParenParseOption::CompoundLiteral;
         TypeResult Ty;
         {
@@ -2757,7 +2776,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
         return ParseCompoundLiteralExpression(Ty.get(), OpenLoc, RParenLoc);
       }
 
-      if (Tok.is(tok::l_paren)) {
+      if (ParenBehavior == ParenExprKind::Unknown && Tok.is(tok::l_paren)) {
         // This could be OpenCL vector Literals
         if (getLangOpts().OpenCL)
         {
@@ -2781,7 +2800,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
             Result = ParseCastExpression(
                 /*isUnaryExpression=*/CastParseKind::AnyCastExpr,
                 /*isAddressOfOperand=*/false,
-                /*isTypeCast=*/TypeCastState::IsTypeCast,
+                TypoCorrectionTypeBehavior::AllowTypes,
                 /*isVectorLiteral=*/true);
 
             if (!Result.isInvalid()) {
@@ -2808,7 +2827,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
 
         // Note that this doesn't parse the subsequent cast-expression, it just
         // returns the parsed type to the callee.
-        if (stopIfCastExpr) {
+        if (StopIfCastExpr) {
           TypeResult Ty;
           {
             InMessageExpressionRAIIObject InMessage(*this, false);
@@ -2834,7 +2853,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
         Result = ParseCastExpression(
             /*isUnaryExpression=*/CastParseKind::AnyCastExpr,
             /*isAddressOfOperand=*/false,
-            /*isTypeCast=*/TypeCastState::IsTypeCast);
+            TypoCorrectionTypeBehavior::AllowTypes);
         if (!Result.isInvalid()) {
           Result = Actions.ActOnCastExpr(getCurScope(), OpenLoc,
                                          DeclaratorInfo, CastTy,
@@ -2850,7 +2869,8 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
              isFoldOperator(NextToken().getKind())) {
     ExprType = ParenParseOption::FoldExpr;
     return ParseFoldExpression(ExprResult(), T);
-  } else if (isTypeCast) {
+  } else if (CorrectionBehavior == TypoCorrectionTypeBehavior::AllowTypes) {
+    // FIXME: This should not be predicated on typo correction behavior.
     // Parse the expression-list.
     InMessageExpressionRAIIObject InMessage(*this, false);
     ExprVector ArgExprs;
@@ -2902,7 +2922,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
   } else {
     InMessageExpressionRAIIObject InMessage(*this, false);
 
-    Result = ParseExpression(TypeCastState::MaybeTypeCast);
+    Result = ParseExpression(TypoCorrectionTypeBehavior::AllowBoth);
     if (ExprType >= ParenParseOption::FoldExpr &&
         isFoldOperator(Tok.getKind()) && NextToken().is(tok::ellipsis)) {
       ExprType = ParenParseOption::FoldExpr;
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 1ea0cf52933f..8dce2268082d 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1931,15 +1931,13 @@ Parser::ParseCXXCondition(StmtResult *InitStmt, SourceLocation Loc,
       return ParseCXXCondition(nullptr, Loc, CK, MissingOK);
     }
 
-    ExprResult Expr = [&] {
-      EnterExpressionEvaluationContext Eval(
-          Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated,
-          /*LambdaContextDecl=*/nullptr,
-          /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
-          /*ShouldEnter=*/CK == Sema::ConditionKind::ConstexprIf);
-      // Parse the expression.
-      return ParseExpression(); // expression
-    }();
+    EnterExpressionEvaluationContext Eval(
+        Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated,
+        /*LambdaContextDecl=*/nullptr,
+        /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
+        /*ShouldEnter=*/CK == Sema::ConditionKind::ConstexprIf);
+
+    ExprResult Expr = ParseExpression();
 
     if (Expr.isInvalid())
       return Sema::ConditionError();
@@ -3607,7 +3605,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
       Result = ParseCastExpression(CastParseKind::AnyCastExpr,
                                    false /*isAddressofOperand*/, NotCastExpr,
                                    // type-id has priority.
-                                   TypeCastState::IsTypeCast);
+                                   TypoCorrectionTypeBehavior::AllowTypes);
     }
 
     // If we parsed a cast-expression, it's really a type-id, otherwise it's
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index e510302c3aae..96d3999ff2ac 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -17,10 +17,12 @@ namespace hlsl {
 
 using TokenKind = RootSignatureToken::Kind;
 
-RootSignatureParser::RootSignatureParser(SmallVector<RootElement> &Elements,
-                                         RootSignatureLexer &Lexer,
-                                         Preprocessor &PP)
-    : Elements(Elements), Lexer(Lexer), PP(PP), CurToken(SourceLocation()) {}
+RootSignatureParser::RootSignatureParser(
+    llvm::dxbc::RootSignatureVersion Version,
+    SmallVector<RootElement> &Elements, RootSignatureLexer &Lexer,
+    Preprocessor &PP)
+    : Version(Version), Elements(Elements), Lexer(Lexer), PP(PP),
+      CurToken(SourceLocation()) {}
 
 bool RootSignatureParser::parse() {
   // Iterate as many RootElements as possible
@@ -76,7 +78,7 @@ static FlagType maybeOrFlag(std::optional<FlagType> Flags, FlagType Flag) {
                                llvm::to_underlying(Flag));
 }
 
-std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
+std::optional<llvm::dxbc::RootFlags> RootSignatureParser::parseRootFlags() {
   assert(CurToken.TokKind == TokenKind::kw_RootFlags &&
          "Expects to only be invoked starting at given keyword");
 
@@ -84,7 +86,7 @@ std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
                            CurToken.TokKind))
     return std::nullopt;
 
-  std::optional<RootFlags> Flags = RootFlags::None;
+  std::optional<llvm::dxbc::RootFlags> Flags = llvm::dxbc::RootFlags::None;
 
   // Handle the edge-case of '0' to specify no flags set
   if (tryConsumeExpectedToken(TokenKind::int_literal)) {
@@ -104,7 +106,8 @@ std::optional<RootFlags> RootSignatureParser::parseRootFlags() {
         switch (CurToken.TokKind) {
 #define ROOT_FLAG_ENUM(NAME, LIT)                                              \
   case TokenKind::en_##NAME:                                                   \
-    Flags = maybeOrFlag<RootFlags>(Flags, RootFlags::NAME);                    \
+    Flags = maybeOrFlag<llvm::dxbc::RootFlags>(Flags,                          \
+                                               llvm::dxbc::RootFlags::NAME);   \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
         default:
@@ -198,7 +201,7 @@ std::optional<RootDescriptor> RootSignatureParser::parseRootDescriptor() {
     ExpectedReg = TokenKind::uReg;
     break;
   }
-  Descriptor.setDefaultFlags();
+  Descriptor.setDefaultFlags(Version);
 
   auto Params = parseRootDescriptorParams(ExpectedReg);
   if (!Params.has_value())
@@ -240,7 +243,7 @@ std::optional<DescriptorTable> RootSignatureParser::parseDescriptorTable() {
     return std::nullopt;
 
   DescriptorTable Table;
-  std::optional<ShaderVisibility> Visibility;
+  std::optional<llvm::dxbc::ShaderVisibility> Visibility;
 
   // Iterate as many Clauses as possible
   do {
@@ -317,7 +320,7 @@ RootSignatureParser::parseDescriptorTableClause() {
     ExpectedReg = TokenKind::sReg;
     break;
   }
-  Clause.setDefaultFlags();
+  Clause.setDefaultFlags(Version);
 
   auto Params = parseDescriptorTableClauseParams(ExpectedReg);
   if (!Params.has_value())
@@ -983,7 +986,7 @@ std::optional<float> RootSignatureParser::parseFloatParam() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::ShaderVisibility>
+std::optional<llvm::dxbc::ShaderVisibility>
 RootSignatureParser::parseShaderVisibility() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -999,7 +1002,7 @@ RootSignatureParser::parseShaderVisibility() {
   switch (CurToken.TokKind) {
 #define SHADER_VISIBILITY_ENUM(NAME, LIT)                                      \
   case TokenKind::en_##NAME:                                                   \
-    return ShaderVisibility::NAME;                                             \
+    return llvm::dxbc::ShaderVisibility::NAME;                                 \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1009,7 +1012,7 @@ RootSignatureParser::parseShaderVisibility() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::SamplerFilter>
+std::optional<llvm::dxbc::SamplerFilter>
 RootSignatureParser::parseSamplerFilter() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1025,7 +1028,7 @@ RootSignatureParser::parseSamplerFilter() {
   switch (CurToken.TokKind) {
 #define FILTER_ENUM(NAME, LIT)                                                 \
   case TokenKind::en_##NAME:                                                   \
-    return SamplerFilter::NAME;                                                \
+    return llvm::dxbc::SamplerFilter::NAME;                                    \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1035,7 +1038,7 @@ RootSignatureParser::parseSamplerFilter() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::TextureAddressMode>
+std::optional<llvm::dxbc::TextureAddressMode>
 RootSignatureParser::parseTextureAddressMode() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1051,7 +1054,7 @@ RootSignatureParser::parseTextureAddressMode() {
   switch (CurToken.TokKind) {
 #define TEXTURE_ADDRESS_MODE_ENUM(NAME, LIT)                                   \
   case TokenKind::en_##NAME:                                                   \
-    return TextureAddressMode::NAME;                                           \
+    return llvm::dxbc::TextureAddressMode::NAME;                               \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1061,7 +1064,7 @@ RootSignatureParser::parseTextureAddressMode() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::ComparisonFunc>
+std::optional<llvm::dxbc::ComparisonFunc>
 RootSignatureParser::parseComparisonFunc() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1077,7 +1080,7 @@ RootSignatureParser::parseComparisonFunc() {
   switch (CurToken.TokKind) {
 #define COMPARISON_FUNC_ENUM(NAME, LIT)                                        \
   case TokenKind::en_##NAME:                                                   \
-    return ComparisonFunc::NAME;                                               \
+    return llvm::dxbc::ComparisonFunc::NAME;                                   \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1087,7 +1090,7 @@ RootSignatureParser::parseComparisonFunc() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::StaticBorderColor>
+std::optional<llvm::dxbc::StaticBorderColor>
 RootSignatureParser::parseStaticBorderColor() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1103,7 +1106,7 @@ RootSignatureParser::parseStaticBorderColor() {
   switch (CurToken.TokKind) {
 #define STATIC_BORDER_COLOR_ENUM(NAME, LIT)                                    \
   case TokenKind::en_##NAME:                                                   \
-    return StaticBorderColor::NAME;                                            \
+    return llvm::dxbc::StaticBorderColor::NAME;                                \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   default:
@@ -1113,7 +1116,7 @@ RootSignatureParser::parseStaticBorderColor() {
   return std::nullopt;
 }
 
-std::optional<llvm::hlsl::rootsig::RootDescriptorFlags>
+std::optional<llvm::dxbc::RootDescriptorFlags>
 RootSignatureParser::parseRootDescriptorFlags() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1124,7 +1127,7 @@ RootSignatureParser::parseRootDescriptorFlags() {
       getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_non_zero_flag);
       return std::nullopt;
     }
-    return RootDescriptorFlags::None;
+    return llvm::dxbc::RootDescriptorFlags::None;
   }
 
   TokenKind Expected[] = {
@@ -1132,15 +1135,15 @@ RootSignatureParser::parseRootDescriptorFlags() {
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   };
 
-  std::optional<RootDescriptorFlags> Flags;
+  std::optional<llvm::dxbc::RootDescriptorFlags> Flags;
 
   do {
     if (tryConsumeExpectedToken(Expected)) {
       switch (CurToken.TokKind) {
 #define ROOT_DESCRIPTOR_FLAG_ENUM(NAME, LIT)                                   \
   case TokenKind::en_##NAME:                                                   \
-    Flags =                                                                    \
-        maybeOrFlag<RootDescriptorFlags>(Flags, RootDescriptorFlags::NAME);    \
+    Flags = maybeOrFlag<llvm::dxbc::RootDescriptorFlags>(                      \
+        Flags, llvm::dxbc::RootDescriptorFlags::NAME);                         \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
       default:
@@ -1152,7 +1155,7 @@ RootSignatureParser::parseRootDescriptorFlags() {
   return Flags;
 }
 
-std::optional<llvm::hlsl::rootsig::DescriptorRangeFlags>
+std::optional<llvm::dxbc::DescriptorRangeFlags>
 RootSignatureParser::parseDescriptorRangeFlags() {
   assert(CurToken.TokKind == TokenKind::pu_equal &&
          "Expects to only be invoked starting at given keyword");
@@ -1163,7 +1166,7 @@ RootSignatureParser::parseDescriptorRangeFlags() {
       getDiags().Report(CurToken.TokLoc, diag::err_hlsl_rootsig_non_zero_flag);
       return std::nullopt;
     }
-    return DescriptorRangeFlags::None;
+    return llvm::dxbc::DescriptorRangeFlags::None;
   }
 
   TokenKind Expected[] = {
@@ -1171,15 +1174,15 @@ RootSignatureParser::parseDescriptorRangeFlags() {
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
   };
 
-  std::optional<DescriptorRangeFlags> Flags;
+  std::optional<llvm::dxbc::DescriptorRangeFlags> Flags;
 
   do {
     if (tryConsumeExpectedToken(Expected)) {
       switch (CurToken.TokKind) {
 #define DESCRIPTOR_RANGE_FLAG_ENUM(NAME, LIT, ON)                              \
   case TokenKind::en_##NAME:                                                   \
-    Flags =                                                                    \
-        maybeOrFlag<DescriptorRangeFlags>(Flags, DescriptorRangeFlags::NAME);  \
+    Flags = maybeOrFlag<llvm::dxbc::DescriptorRangeFlags>(                     \
+        Flags, llvm::dxbc::DescriptorRangeFlags::NAME);                        \
     break;
 #include "clang/Lex/HLSLRootSignatureTokenKinds.def"
       default:
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index f2849c4eac7c..b79a956d5150 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -606,13 +606,20 @@ unsigned getOpenACCScopeFlags(OpenACCDirectiveKind DirKind) {
   case OpenACCDirectiveKind::Parallel:
   case OpenACCDirectiveKind::Serial:
   case OpenACCDirectiveKind::Kernels:
+    // Mark this as a BreakScope/ContinueScope as well as a compute construct
+    // so that we can diagnose trying to 'break'/'continue' inside of one.
+    return Scope::BreakScope | Scope::ContinueScope |
+           Scope::OpenACCComputeConstructScope;
   case OpenACCDirectiveKind::ParallelLoop:
   case OpenACCDirectiveKind::SerialLoop:
   case OpenACCDirectiveKind::KernelsLoop:
     // Mark this as a BreakScope/ContinueScope as well as a compute construct
     // so that we can diagnose trying to 'break'/'continue' inside of one.
     return Scope::BreakScope | Scope::ContinueScope |
-           Scope::OpenACCComputeConstructScope;
+           Scope::OpenACCComputeConstructScope |
+           Scope::OpenACCLoopConstructScope;
+  case OpenACCDirectiveKind::Loop:
+    return Scope::OpenACCLoopConstructScope;
   case OpenACCDirectiveKind::Data:
   case OpenACCDirectiveKind::EnterData:
   case OpenACCDirectiveKind::ExitData:
@@ -621,7 +628,6 @@ unsigned getOpenACCScopeFlags(OpenACCDirectiveKind DirKind) {
   case OpenACCDirectiveKind::Init:
   case OpenACCDirectiveKind::Shutdown:
   case OpenACCDirectiveKind::Cache:
-  case OpenACCDirectiveKind::Loop:
   case OpenACCDirectiveKind::Atomic:
   case OpenACCDirectiveKind::Declare:
   case OpenACCDirectiveKind::Routine:
@@ -1416,12 +1422,15 @@ Parser::OpenACCVarParseResult Parser::ParseOpenACCVar(OpenACCDirectiveKind DK,
                                                       OpenACCClauseKind CK) {
   OpenACCArraySectionRAII ArraySections(*this);
 
+  getActions().OpenACC().ActOnStartParseVar(DK, CK);
   ExprResult Res = ParseAssignmentExpression();
-  if (!Res.isUsable())
+
+  if (!Res.isUsable()) {
+    getActions().OpenACC().ActOnInvalidParseVar();
     return {Res, OpenACCParseCanContinue::Cannot};
+  }
 
   Res = getActions().OpenACC().ActOnVar(DK, CK, Res.get());
-
   return {Res, OpenACCParseCanContinue::Can};
 }
 
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 78d3503d8eb6..4c8a1f97d4b1 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -25,6 +25,7 @@
 #include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
 #include "llvm/Frontend/OpenMP/OMPAssume.h"
 #include "llvm/Frontend/OpenMP/OMPContext.h"
 #include <optional>
@@ -37,48 +38,6 @@ using namespace llvm::omp;
 //===----------------------------------------------------------------------===//
 
 namespace {
-enum OpenMPDirectiveKindEx {
-  OMPD_cancellation = llvm::omp::Directive_enumSize + 1,
-  OMPD_data,
-  OMPD_declare,
-  OMPD_end,
-  OMPD_end_declare,
-  OMPD_enter,
-  OMPD_exit,
-  OMPD_point,
-  OMPD_reduction,
-  OMPD_target_enter,
-  OMPD_target_exit,
-  OMPD_update,
-  OMPD_distribute_parallel,
-  OMPD_teams_distribute_parallel,
-  OMPD_target_teams_distribute_parallel,
-  OMPD_mapper,
-  OMPD_variant,
-  OMPD_begin,
-  OMPD_begin_declare,
-};
-
-// Helper to unify the enum class OpenMPDirectiveKind with its extension
-// the OpenMPDirectiveKindEx enum which allows to use them together as if they
-// are unsigned values.
-struct OpenMPDirectiveKindExWrapper {
-  OpenMPDirectiveKindExWrapper(unsigned Value) : Value(Value) {}
-  OpenMPDirectiveKindExWrapper(OpenMPDirectiveKind DK) : Value(unsigned(DK)) {}
-  bool operator==(OpenMPDirectiveKindExWrapper V) const {
-    return Value == V.Value;
-  }
-  bool operator!=(OpenMPDirectiveKindExWrapper V) const {
-    return Value != V.Value;
-  }
-  bool operator==(OpenMPDirectiveKind V) const { return Value == unsigned(V); }
-  bool operator!=(OpenMPDirectiveKind V) const { return Value != unsigned(V); }
-  bool operator<(OpenMPDirectiveKind V) const { return Value < unsigned(V); }
-  operator unsigned() const { return Value; }
-  operator OpenMPDirectiveKind() const { return OpenMPDirectiveKind(Value); }
-  unsigned Value;
-};
-
 class DeclDirectiveListParserHelper final {
   SmallVector<Expr *, 4> Identifiers;
   Parser *P;
@@ -97,130 +56,52 @@ class DeclDirectiveListParserHelper final {
 };
 } // namespace
 
-// Map token string to extended OMP token kind that are
-// OpenMPDirectiveKind + OpenMPDirectiveKindEx.
-static unsigned getOpenMPDirectiveKindEx(StringRef S) {
-  OpenMPDirectiveKindExWrapper DKind = getOpenMPDirectiveKind(S);
-  if (DKind != OMPD_unknown)
-    return DKind;
-
-  return llvm::StringSwitch<OpenMPDirectiveKindExWrapper>(S)
-      .Case("cancellation", OMPD_cancellation)
-      .Case("data", OMPD_data)
-      .Case("declare", OMPD_declare)
-      .Case("end", OMPD_end)
-      .Case("enter", OMPD_enter)
-      .Case("exit", OMPD_exit)
-      .Case("point", OMPD_point)
-      .Case("reduction", OMPD_reduction)
-      .Case("update", OMPD_update)
-      .Case("mapper", OMPD_mapper)
-      .Case("variant", OMPD_variant)
-      .Case("begin", OMPD_begin)
-      .Default(OMPD_unknown);
+static OpenMPDirectiveKind checkOpenMPDirectiveName(Parser &P,
+                                                    SourceLocation Loc,
+                                                    OpenMPDirectiveKind Kind,
+                                                    StringRef Name) {
+  unsigned Version = P.getLangOpts().OpenMP;
+  auto [D, VR] = getOpenMPDirectiveKindAndVersions(Name);
+  assert(D == Kind && "Directive kind mismatch");
+  // Ignore the case Version > VR.Max: In OpenMP 6.0 all prior spellings
+  // are explicitly allowed.
+  if (static_cast<int>(Version) < VR.Min)
+    P.Diag(Loc, diag::warn_omp_future_directive_spelling) << Name;
+
+  return Kind;
 }
 
-static OpenMPDirectiveKindExWrapper parseOpenMPDirectiveKind(Parser &P) {
-  // Array of foldings: F[i][0] F[i][1] ===> F[i][2].
-  // E.g.: OMPD_for OMPD_simd ===> OMPD_for_simd
-  // TODO: add other combined directives in topological order.
-  static const OpenMPDirectiveKindExWrapper F[][3] = {
-      {OMPD_begin, OMPD_declare, OMPD_begin_declare},
-      {OMPD_begin, OMPD_assumes, OMPD_begin_assumes},
-      {OMPD_end, OMPD_declare, OMPD_end_declare},
-      {OMPD_end, OMPD_assumes, OMPD_end_assumes},
-      {OMPD_cancellation, OMPD_point, OMPD_cancellation_point},
-      {OMPD_declare, OMPD_reduction, OMPD_declare_reduction},
-      {OMPD_declare, OMPD_mapper, OMPD_declare_mapper},
-      {OMPD_declare, OMPD_simd, OMPD_declare_simd},
-      {OMPD_declare, OMPD_target, OMPD_declare_target},
-      {OMPD_declare, OMPD_variant, OMPD_declare_variant},
-      {OMPD_begin_declare, OMPD_target, OMPD_begin_declare_target},
-      {OMPD_begin_declare, OMPD_variant, OMPD_begin_declare_variant},
-      {OMPD_end_declare, OMPD_variant, OMPD_end_declare_variant},
-      {OMPD_distribute, OMPD_parallel, OMPD_distribute_parallel},
-      {OMPD_distribute_parallel, OMPD_for, OMPD_distribute_parallel_for},
-      {OMPD_distribute_parallel_for, OMPD_simd,
-       OMPD_distribute_parallel_for_simd},
-      {OMPD_distribute, OMPD_simd, OMPD_distribute_simd},
-      {OMPD_end_declare, OMPD_target, OMPD_end_declare_target},
-      {OMPD_target, OMPD_data, OMPD_target_data},
-      {OMPD_target, OMPD_enter, OMPD_target_enter},
-      {OMPD_target, OMPD_exit, OMPD_target_exit},
-      {OMPD_target, OMPD_update, OMPD_target_update},
-      {OMPD_target_enter, OMPD_data, OMPD_target_enter_data},
-      {OMPD_target_exit, OMPD_data, OMPD_target_exit_data},
-      {OMPD_for, OMPD_simd, OMPD_for_simd},
-      {OMPD_parallel, OMPD_for, OMPD_parallel_for},
-      {OMPD_parallel_for, OMPD_simd, OMPD_parallel_for_simd},
-      {OMPD_parallel, OMPD_loop, OMPD_parallel_loop},
-      {OMPD_parallel, OMPD_sections, OMPD_parallel_sections},
-      {OMPD_taskloop, OMPD_simd, OMPD_taskloop_simd},
-      {OMPD_target, OMPD_parallel, OMPD_target_parallel},
-      {OMPD_target, OMPD_simd, OMPD_target_simd},
-      {OMPD_target_parallel, OMPD_loop, OMPD_target_parallel_loop},
-      {OMPD_target_parallel, OMPD_for, OMPD_target_parallel_for},
-      {OMPD_target_parallel_for, OMPD_simd, OMPD_target_parallel_for_simd},
-      {OMPD_teams, OMPD_distribute, OMPD_teams_distribute},
-      {OMPD_teams_distribute, OMPD_simd, OMPD_teams_distribute_simd},
-      {OMPD_teams_distribute, OMPD_parallel, OMPD_teams_distribute_parallel},
-      {OMPD_teams_distribute_parallel, OMPD_for,
-       OMPD_teams_distribute_parallel_for},
-      {OMPD_teams_distribute_parallel_for, OMPD_simd,
-       OMPD_teams_distribute_parallel_for_simd},
-      {OMPD_teams, OMPD_loop, OMPD_teams_loop},
-      {OMPD_target, OMPD_teams, OMPD_target_teams},
-      {OMPD_target_teams, OMPD_distribute, OMPD_target_teams_distribute},
-      {OMPD_target_teams, OMPD_loop, OMPD_target_teams_loop},
-      {OMPD_target_teams_distribute, OMPD_parallel,
-       OMPD_target_teams_distribute_parallel},
-      {OMPD_target_teams_distribute, OMPD_simd,
-       OMPD_target_teams_distribute_simd},
-      {OMPD_target_teams_distribute_parallel, OMPD_for,
-       OMPD_target_teams_distribute_parallel_for},
-      {OMPD_target_teams_distribute_parallel_for, OMPD_simd,
-       OMPD_target_teams_distribute_parallel_for_simd},
-      {OMPD_master, OMPD_taskloop, OMPD_master_taskloop},
-      {OMPD_masked, OMPD_taskloop, OMPD_masked_taskloop},
-      {OMPD_master_taskloop, OMPD_simd, OMPD_master_taskloop_simd},
-      {OMPD_masked_taskloop, OMPD_simd, OMPD_masked_taskloop_simd},
-      {OMPD_parallel, OMPD_master, OMPD_parallel_master},
-      {OMPD_parallel, OMPD_masked, OMPD_parallel_masked},
-      {OMPD_parallel_master, OMPD_taskloop, OMPD_parallel_master_taskloop},
-      {OMPD_parallel_masked, OMPD_taskloop, OMPD_parallel_masked_taskloop},
-      {OMPD_parallel_master_taskloop, OMPD_simd,
-       OMPD_parallel_master_taskloop_simd},
-      {OMPD_parallel_masked_taskloop, OMPD_simd,
-       OMPD_parallel_masked_taskloop_simd}};
-  enum { CancellationPoint = 0, DeclareReduction = 1, TargetData = 2 };
+static OpenMPDirectiveKind parseOpenMPDirectiveKind(Parser &P) {
+  static const DirectiveNameParser DirParser;
+
+  const DirectiveNameParser::State *S = DirParser.initial();
+
   Token Tok = P.getCurToken();
-  OpenMPDirectiveKindExWrapper DKind =
-      Tok.isAnnotation()
-          ? static_cast<unsigned>(OMPD_unknown)
-          : getOpenMPDirectiveKindEx(P.getPreprocessor().getSpelling(Tok));
-  if (DKind == OMPD_unknown)
+  if (Tok.isAnnotation())
     return OMPD_unknown;
 
-  for (const auto &I : F) {
-    if (DKind != I[0])
-      continue;
+  std::string Concat = P.getPreprocessor().getSpelling(Tok);
+  SourceLocation Loc = Tok.getLocation();
 
-    Tok = P.getPreprocessor().LookAhead(0);
-    OpenMPDirectiveKindExWrapper SDKind =
-        Tok.isAnnotation()
-            ? static_cast<unsigned>(OMPD_unknown)
-            : getOpenMPDirectiveKindEx(P.getPreprocessor().getSpelling(Tok));
-    if (SDKind == OMPD_unknown)
-      continue;
+  S = DirParser.consume(S, Concat);
+  if (S == nullptr)
+    return OMPD_unknown;
 
-    if (SDKind == I[1]) {
+  while (!Tok.isAnnotation()) {
+    OpenMPDirectiveKind DKind = S->Value;
+    Tok = P.getPreprocessor().LookAhead(0);
+    if (!Tok.isAnnotation()) {
+      std::string TS = P.getPreprocessor().getSpelling(Tok);
+      S = DirParser.consume(S, TS);
+      if (S == nullptr)
+        return checkOpenMPDirectiveName(P, Loc, DKind, Concat);
+      Concat += ' ' + TS;
       P.ConsumeToken();
-      DKind = I[2];
     }
   }
-  return unsigned(DKind) < llvm::omp::Directive_enumSize
-             ? static_cast<OpenMPDirectiveKind>(DKind)
-             : OMPD_unknown;
+
+  assert(S && "Should have exited early");
+  return checkOpenMPDirectiveName(P, Loc, S->Value, Concat);
 }
 
 static DeclarationName parseOpenMPReductionId(Parser &P) {
@@ -2629,10 +2510,6 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     Diag(Tok, diag::err_omp_unknown_directive);
     return StmtError();
   }
-  if (!(getDirectiveLanguages(DKind) & SourceLanguage::C)) {
-    // Treat directives that are not allowed in C/C++ as unknown.
-    DKind = OMPD_unknown;
-  }
 
   StmtResult Directive = StmtError();
 
@@ -3196,7 +3073,8 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     if ((CKind == OMPC_ordered || CKind == OMPC_partial) &&
         PP.LookAhead(/*N=*/0).isNot(tok::l_paren))
       Clause = ParseOpenMPClause(CKind, WrongDirective);
-    else if (CKind == OMPC_grainsize || CKind == OMPC_num_tasks)
+    else if (CKind == OMPC_grainsize || CKind == OMPC_num_tasks ||
+             CKind == OMPC_num_threads)
       Clause = ParseOpenMPSingleExprWithArgClause(DKind, CKind, WrongDirective);
     else
       Clause = ParseOpenMPSingleExprClause(CKind, WrongDirective);
@@ -3502,9 +3380,9 @@ ExprResult Parser::ParseOpenMPParensExpr(StringRef ClauseName,
     return ExprError();
 
   SourceLocation ELoc = Tok.getLocation();
-  ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr,
-                                     IsAddressOfOperand,
-                                     TypeCastState::NotTypeCast));
+  ExprResult LHS(
+      ParseCastExpression(CastParseKind::AnyCastExpr, IsAddressOfOperand,
+                          TypoCorrectionTypeBehavior::AllowNonTypes));
   ExprResult Val(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   Val = Actions.ActOnFinishFullExpr(Val.get(), ELoc, /*DiscardedValue*/ false);
 
@@ -3981,12 +3859,39 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
       Arg.push_back(OMPC_NUMTASKS_unknown);
       KLoc.emplace_back();
     }
+  } else if (Kind == OMPC_num_threads) {
+    // Parse optional <num_threads modifier> ':'
+    OpenMPNumThreadsClauseModifier Modifier =
+        static_cast<OpenMPNumThreadsClauseModifier>(getOpenMPSimpleClauseType(
+            Kind, Tok.isAnnotation() ? "" : PP.getSpelling(Tok),
+            getLangOpts()));
+    if (getLangOpts().OpenMP >= 60) {
+      if (NextToken().is(tok::colon)) {
+        Arg.push_back(Modifier);
+        KLoc.push_back(Tok.getLocation());
+        // Parse modifier
+        ConsumeAnyToken();
+        // Parse ':'
+        ConsumeAnyToken();
+      } else {
+        if (Modifier == OMPC_NUMTHREADS_strict) {
+          Diag(Tok, diag::err_modifier_expected_colon) << "strict";
+          // Parse modifier
+          ConsumeAnyToken();
+        }
+        Arg.push_back(OMPC_NUMTHREADS_unknown);
+        KLoc.emplace_back();
+      }
+    } else {
+      Arg.push_back(OMPC_NUMTHREADS_unknown);
+      KLoc.emplace_back();
+    }
   } else {
     assert(Kind == OMPC_if);
     KLoc.push_back(Tok.getLocation());
     TentativeParsingAction TPA(*this);
     auto DK = parseOpenMPDirectiveKind(*this);
-    Arg.push_back(DK);
+    Arg.push_back(static_cast<unsigned>(DK));
     if (DK != OMPD_unknown) {
       ConsumeToken();
       if (Tok.is(tok::colon) && getLangOpts().OpenMP > 40) {
@@ -4004,11 +3909,13 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
   bool NeedAnExpression = (Kind == OMPC_schedule && DelimLoc.isValid()) ||
                           (Kind == OMPC_dist_schedule && DelimLoc.isValid()) ||
                           Kind == OMPC_if || Kind == OMPC_device ||
-                          Kind == OMPC_grainsize || Kind == OMPC_num_tasks;
+                          Kind == OMPC_grainsize || Kind == OMPC_num_tasks ||
+                          Kind == OMPC_num_threads;
   if (NeedAnExpression) {
     SourceLocation ELoc = Tok.getLocation();
-    ExprResult LHS(ParseCastExpression(CastParseKind::AnyCastExpr, false,
-                                       TypeCastState::NotTypeCast));
+    ExprResult LHS(
+        ParseCastExpression(CastParseKind::AnyCastExpr, false,
+                            TypoCorrectionTypeBehavior::AllowNonTypes));
     Val = ParseRHSOfBinaryExpression(LHS, prec::Conditional);
     Val =
         Actions.ActOnFinishFullExpr(Val.get(), ELoc, /*DiscardedValue*/ false);
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index bc40b726bf41..8217151a0259 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -1327,7 +1327,7 @@ struct MisleadingIndentationChecker {
     if (ColNo == 0 || TabStop == 1)
       return ColNo;
 
-    std::pair<FileID, unsigned> FIDAndOffset = SM.getDecomposedLoc(Loc);
+    FileIDAndOffset FIDAndOffset = SM.getDecomposedLoc(Loc);
 
     bool Invalid;
     StringRef BufData = SM.getBufferData(FIDAndOffset.first, &Invalid);
diff --git a/clang/lib/Parse/ParseStmtAsm.cpp b/clang/lib/Parse/ParseStmtAsm.cpp
index 182907df5607..c679aa6fe7b2 100644
--- a/clang/lib/Parse/ParseStmtAsm.cpp
+++ b/clang/lib/Parse/ParseStmtAsm.cpp
@@ -380,8 +380,7 @@ StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
     ++NumTokensRead;
   } else {
     // Single-line inline asm; compute which line it is on.
-    std::pair<FileID, unsigned> ExpAsmLoc =
-        SrcMgr.getDecomposedExpansionLoc(EndLoc);
+    FileIDAndOffset ExpAsmLoc = SrcMgr.getDecomposedExpansionLoc(EndLoc);
     FID = ExpAsmLoc.first;
     LineNo = SrcMgr.getLineNumber(FID, ExpAsmLoc.second);
     LBraceLocs.push_back(SourceLocation());
@@ -408,16 +407,14 @@ StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
       InAsmComment = true;
       if (!SingleLineMode) {
         // Compute which line the comment is on.
-        std::pair<FileID, unsigned> ExpSemiLoc =
-            SrcMgr.getDecomposedExpansionLoc(TokLoc);
+        FileIDAndOffset ExpSemiLoc = SrcMgr.getDecomposedExpansionLoc(TokLoc);
         FID = ExpSemiLoc.first;
         LineNo = SrcMgr.getLineNumber(FID, ExpSemiLoc.second);
       }
     } else if (SingleLineMode || InAsmComment) {
       // If end-of-line is significant, check whether this token is on a
       // new line.
-      std::pair<FileID, unsigned> ExpLoc =
-          SrcMgr.getDecomposedExpansionLoc(TokLoc);
+      FileIDAndOffset ExpLoc = SrcMgr.getDecomposedExpansionLoc(TokLoc);
       if (ExpLoc.first != FID ||
           SrcMgr.getLineNumber(ExpLoc.first, ExpLoc.second) != LineNo) {
         // If this is a single-line __asm, we're done, except if the next
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index a16dbe95b788..9a04bf298c0a 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -1318,8 +1318,8 @@ ParsedTemplateArgument Parser::ParseTemplateArgument() {
   if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace))
     ExprArg = ParseBraceInitializer();
   else
-    ExprArg =
-        ParseConstantExpressionInExprEvalContext(TypeCastState::MaybeTypeCast);
+    ExprArg = ParseConstantExpressionInExprEvalContext(
+        TypoCorrectionTypeBehavior::AllowBoth);
   if (ExprArg.isInvalid() || !ExprArg.get()) {
     return ParsedTemplateArgument();
   }
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 18f399aca59e..8834bf80c401 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1606,8 +1606,9 @@ ExprResult Parser::ParseAsmStringLiteral(bool ForAsmLabel) {
 
     EnterExpressionEvaluationContext ConstantEvaluated(
         Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
-    AsmString = ParseParenExpression(ExprType, true /*stopIfCastExpr*/, false,
-                                     CastTy, RParenLoc);
+    AsmString = ParseParenExpression(
+        ExprType, /*StopIfCastExr=*/true, ParenExprKind::Unknown,
+        TypoCorrectionTypeBehavior::AllowBoth, CastTy, RParenLoc);
     if (!AsmString.isInvalid())
       AsmString = Actions.ActOnConstantExpression(AsmString);
 
diff --git a/clang/lib/Rewrite/Rewriter.cpp b/clang/lib/Rewrite/Rewriter.cpp
index 68cf797f9790..ae21a10f81c3 100644
--- a/clang/lib/Rewrite/Rewriter.cpp
+++ b/clang/lib/Rewrite/Rewriter.cpp
@@ -133,7 +133,7 @@ std::string Rewriter::getRewrittenText(CharSourceRange Range) const {
 unsigned Rewriter::getLocationOffsetAndFileID(SourceLocation Loc,
                                               FileID &FID) const {
   assert(Loc.isValid() && "Invalid location");
-  std::pair<FileID, unsigned> V = SourceMgr->getDecomposedLoc(Loc);
+  FileIDAndOffset V = SourceMgr->getDecomposedLoc(Loc);
   FID = V.first;
   return V.second;
 }
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index 2a107a36e24b..7420ba2d461c 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -643,7 +643,7 @@ static void CheckFallThroughForBody(Sema &S, const Decl *D, const Stmt *Body,
       ReturnsVoid = CBody->getFallthroughHandler() != nullptr;
     else
       ReturnsVoid = FD->getReturnType()->isVoidType();
-    HasNoReturn = FD->isNoReturn();
+    HasNoReturn = FD->isNoReturn() || FD->hasAttr<InferredNoReturnAttr>();
   }
   else if (const auto *MD = dyn_cast<ObjCMethodDecl>(D)) {
     ReturnsVoid = MD->getReturnType()->isVoidType();
@@ -681,6 +681,28 @@ static void CheckFallThroughForBody(Sema &S, const Decl *D, const Stmt *Body,
       if (CD.diag_FallThrough_HasNoReturn)
         S.Diag(RBrace, CD.diag_FallThrough_HasNoReturn) << CD.FunKind;
     } else if (!ReturnsVoid && CD.diag_FallThrough_ReturnsNonVoid) {
+      // If the final statement is a call to an always-throwing function,
+      // don't warn about the fall-through.
+      if (D->getAsFunction()) {
+        if (const auto *CS = dyn_cast<CompoundStmt>(Body);
+            CS && !CS->body_empty()) {
+          const Stmt *LastStmt = CS->body_back();
+          // Unwrap ExprWithCleanups if necessary.
+          if (const auto *EWC = dyn_cast<ExprWithCleanups>(LastStmt)) {
+            LastStmt = EWC->getSubExpr();
+          }
+          if (const auto *CE = dyn_cast<CallExpr>(LastStmt)) {
+            if (const FunctionDecl *Callee = CE->getDirectCallee();
+                Callee && Callee->hasAttr<InferredNoReturnAttr>()) {
+              return; // Don't warn about fall-through.
+            }
+          }
+          // Direct throw.
+          if (isa<CXXThrowExpr>(LastStmt)) {
+            return; // Don't warn about fall-through.
+          }
+        }
+      }
       bool NotInAllControlPaths = FallThroughType == MaybeFallThrough;
       S.Diag(RBrace, CD.diag_FallThrough_ReturnsNonVoid)
           << CD.FunKind << NotInAllControlPaths;
diff --git a/clang/lib/Sema/CodeCompleteConsumer.cpp b/clang/lib/Sema/CodeCompleteConsumer.cpp
index c4e5ef0d75af..e3fc7c11f459 100644
--- a/clang/lib/Sema/CodeCompleteConsumer.cpp
+++ b/clang/lib/Sema/CodeCompleteConsumer.cpp
@@ -700,8 +700,8 @@ void PrintingCodeCompleteConsumer::ProcessCodeCompleteResults(
       const SourceLocation ELoc = FixIt.RemoveRange.getEnd();
 
       SourceManager &SM = SemaRef.SourceMgr;
-      std::pair<FileID, unsigned> BInfo = SM.getDecomposedLoc(BLoc);
-      std::pair<FileID, unsigned> EInfo = SM.getDecomposedLoc(ELoc);
+      FileIDAndOffset BInfo = SM.getDecomposedLoc(BLoc);
+      FileIDAndOffset EInfo = SM.getDecomposedLoc(ELoc);
       // Adjust for token ranges.
       if (FixIt.RemoveRange.isTokenRange())
         EInfo.second += Lexer::MeasureTokenLength(ELoc, SM, SemaRef.LangOpts);
diff --git a/clang/lib/Sema/HeuristicResolver.cpp b/clang/lib/Sema/HeuristicResolver.cpp
index 0c67f1f2a387..6874d30516f8 100644
--- a/clang/lib/Sema/HeuristicResolver.cpp
+++ b/clang/lib/Sema/HeuristicResolver.cpp
@@ -13,6 +13,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
+#include "llvm/ADT/identity.h"
 
 namespace clang {
 
@@ -50,6 +51,7 @@ class HeuristicResolverImpl {
                       llvm::function_ref<bool(const NamedDecl *ND)> Filter);
   TagDecl *resolveTypeToTagDecl(QualType T);
   QualType simplifyType(QualType Type, const Expr *E, bool UnwrapPointer);
+  FunctionProtoTypeLoc getFunctionProtoTypeLoc(const Expr *Fn);
 
 private:
   ASTContext &Ctx;
@@ -506,6 +508,56 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveDependentMember(
   }
   return {};
 }
+
+FunctionProtoTypeLoc
+HeuristicResolverImpl::getFunctionProtoTypeLoc(const Expr *Fn) {
+  TypeLoc Target;
+  const Expr *NakedFn = Fn->IgnoreParenCasts();
+  if (const auto *T = NakedFn->getType().getTypePtr()->getAs<TypedefType>()) {
+    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
+  } else if (const auto *DR = dyn_cast<DeclRefExpr>(NakedFn)) {
+    const auto *D = DR->getDecl();
+    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
+      Target = VD->getTypeSourceInfo()->getTypeLoc();
+    }
+  } else if (const auto *ME = dyn_cast<MemberExpr>(NakedFn)) {
+    const auto *MD = ME->getMemberDecl();
+    if (const auto *FD = dyn_cast<FieldDecl>(MD)) {
+      Target = FD->getTypeSourceInfo()->getTypeLoc();
+    }
+  }
+
+  if (!Target)
+    return {};
+
+  // Unwrap types that may be wrapping the function type
+  while (true) {
+    if (auto P = Target.getAs<PointerTypeLoc>()) {
+      Target = P.getPointeeLoc();
+      continue;
+    }
+    if (auto A = Target.getAs<AttributedTypeLoc>()) {
+      Target = A.getModifiedLoc();
+      continue;
+    }
+    if (auto P = Target.getAs<ParenTypeLoc>()) {
+      Target = P.getInnerLoc();
+      continue;
+    }
+    break;
+  }
+
+  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
+    // In some edge cases the AST can contain a "trivial" FunctionProtoTypeLoc
+    // which has null parameters. Avoid these as they don't contain useful
+    // information.
+    if (llvm::all_of(F.getParams(), llvm::identity<ParmVarDecl *>()))
+      return F;
+  }
+
+  return {};
+}
+
 } // namespace
 
 std::vector<const NamedDecl *> HeuristicResolver::resolveMemberExpr(
@@ -557,4 +609,9 @@ QualType HeuristicResolver::simplifyType(QualType Type, const Expr *E,
   return HeuristicResolverImpl(Ctx).simplifyType(Type, E, UnwrapPointer);
 }
 
+FunctionProtoTypeLoc
+HeuristicResolver::getFunctionProtoTypeLoc(const Expr *Fn) const {
+  return HeuristicResolverImpl(Ctx).getFunctionProtoTypeLoc(Fn);
+}
+
 } // namespace clang
diff --git a/clang/lib/Sema/JumpDiagnostics.cpp b/clang/lib/Sema/JumpDiagnostics.cpp
index 36704c3826df..2c6ae8951324 100644
--- a/clang/lib/Sema/JumpDiagnostics.cpp
+++ b/clang/lib/Sema/JumpDiagnostics.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/StmtOpenACC.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Sema/SemaAMDGPU.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/BitVector.h"
 using namespace clang;
@@ -367,8 +368,10 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
 
   case Stmt::IfStmtClass: {
     IfStmt *IS = cast<IfStmt>(S);
+    bool AMDGPUPredicate = false;
     if (!(IS->isConstexpr() || IS->isConsteval() ||
-          IS->isObjCAvailabilityCheck()))
+          IS->isObjCAvailabilityCheck() ||
+          (AMDGPUPredicate = this->S.AMDGPU().IsPredicate(IS->getCond()))))
       break;
 
     unsigned Diag = diag::note_protected_by_if_available;
@@ -376,6 +379,8 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S,
       Diag = diag::note_protected_by_constexpr_if;
     else if (IS->isConsteval())
       Diag = diag::note_protected_by_consteval_if;
+    else if (AMDGPUPredicate)
+      Diag = diag::note_amdgcn_protected_by_predicate;
 
     if (VarDecl *Var = IS->getConditionVariable())
       BuildScopeInformation(Var, ParentScope);
diff --git a/clang/lib/Sema/MultiplexExternalSemaSource.cpp b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
index 9f19f13592e8..fbfb242598c2 100644
--- a/clang/lib/Sema/MultiplexExternalSemaSource.cpp
+++ b/clang/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -115,14 +115,6 @@ bool MultiplexExternalSemaSource::wasThisDeclarationADefinition(
   return false;
 }
 
-bool MultiplexExternalSemaSource::hasInitializerWithSideEffects(
-    const VarDecl *VD) const {
-  for (const auto &S : Sources)
-    if (S->hasInitializerWithSideEffects(VD))
-      return true;
-  return false;
-}
-
 bool MultiplexExternalSemaSource::FindExternalVisibleDeclsByName(
     const DeclContext *DC, DeclarationName Name,
     const DeclContext *OriginalDC) {
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index 066143feb19b..72d95c32747d 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -139,6 +139,10 @@ def ArmIntegerDotProductAccumulateInt8         : FunctionExtension<"cl_arm_integ
 def ArmIntegerDotProductAccumulateInt16        : FunctionExtension<"cl_arm_integer_dot_product_accumulate_int16">;
 def ArmIntegerDotProductAccumulateSaturateInt8 : FunctionExtension<"cl_arm_integer_dot_product_accumulate_saturate_int8">;
 
+// AMD extensions.
+def AMDMediaOps : FunctionExtension<"cl_amd_media_ops">;
+def AMDMediaOps2 : FunctionExtension<"cl_amd_media_ops2">;
+
 // Qualified Type.  These map to ASTContext::QualType.
 class QualType<string _TypeExpr, bit _IsAbstract=0> {
   // Expression to obtain the QualType inside OCL2Qual.
@@ -1905,3 +1909,31 @@ let Extension = ArmIntegerDotProductAccumulateSaturateInt8 in {
     def : Builtin<name, [Int, VectorType<Char, 4>, VectorType<Char, 4>, Int]>;
   }
 }
+
+//--------------------------------------------------------------------
+// AMD extensions.
+let Extension = AMDMediaOps in {
+  foreach name = ["amd_bitalign", "amd_bytealign", "amd_lerp", "amd_sadhi", "amd_sad"] in {
+    def : Builtin<name, [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  foreach name = ["amd_unpack0", "amd_unpack1", "amd_unpack2", "amd_unpack3"] in {
+    def : Builtin<name, [GenTypeFloatVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  def : Builtin<"amd_pack", [UInt, VectorType<Float, 4>]>;
+  def : Builtin<"amd_sad4", [UInt, VectorType<UInt, 4>, VectorType<UInt, 4>, UInt]>;
+}
+
+let Extension = AMDMediaOps2 in {
+  def : Builtin<"amd_bfe", [GenTypeIntVecAndScalar, GenTypeIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  foreach name = ["amd_bfe", "amd_msad", "amd_sadd", "amd_sadw"] in {
+    def : Builtin<name, [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  }
+  def : Builtin<"amd_bfm", [GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar, GenTypeUIntVecAndScalar]>;
+  foreach name = ["amd_max3", "amd_median3", "amd_min3"] in {
+    foreach gentype = [GenTypeFloatVecAndScalar, GenTypeIntVecAndScalar, GenTypeUIntVecAndScalar] in {
+      def : Builtin<name, [gentype, gentype, gentype, gentype]>;
+    }
+  }
+  def : Builtin<"amd_mqsad", [GenTypeULongVecAndScalar, GenTypeULongVecAndScalar, GenTypeUIntVecAndScalar, GenTypeULongVecAndScalar]>;
+  def : Builtin<"amd_qsad", [GenTypeULongVecAndScalar, GenTypeULongVecAndScalar, GenTypeUIntVecAndScalar, GenTypeULongVecAndScalar]>;
+}
diff --git a/clang/lib/Sema/Scope.cpp b/clang/lib/Sema/Scope.cpp
index 5bc7e79a6818..ab04fe554be8 100644
--- a/clang/lib/Sema/Scope.cpp
+++ b/clang/lib/Sema/Scope.cpp
@@ -234,7 +234,8 @@ void Scope::dumpImpl(raw_ostream &OS) const {
       {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
       {TypeAliasScope, "TypeAliasScope"},
       {FriendScope, "FriendScope"},
-  };
+      {OpenACCComputeConstructScope, "OpenACCComputeConstructScope"},
+      {OpenACCLoopConstructScope, "OpenACCLoopConstructScope"}};
 
   for (auto Info : FlagInfo) {
     if (Flags & Info.first) {
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 42ebf2a508a2..6ccd22126581 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -571,8 +571,13 @@ void Sema::Initialize() {
   }
 
   if (Context.getTargetInfo().getTriple().isAMDGPU() ||
+      (Context.getTargetInfo().getTriple().isSPIRV() &&
+       Context.getTargetInfo().getTriple().getVendor() == llvm::Triple::AMD) ||
       (Context.getAuxTargetInfo() &&
-       Context.getAuxTargetInfo()->getTriple().isAMDGPU())) {
+       (Context.getAuxTargetInfo()->getTriple().isAMDGPU() ||
+        (Context.getAuxTargetInfo()->getTriple().isSPIRV() &&
+         Context.getAuxTargetInfo()->getTriple().getVendor() ==
+             llvm::Triple::AMD)))) {
 #define AMDGPU_TYPE(Name, Id, SingletonId, Width, Align)                       \
   addImplicitTypedef(Name, Context.SingletonId);
 #include "clang/Basic/AMDGPUTypes.def"
@@ -2261,6 +2266,24 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) {
         }
       }
     }
+
+    if (auto *VT = Ty->getAs<VectorType>();
+        VT && FD &&
+        (VT->getVectorKind() == VectorKind::SveFixedLengthData ||
+         VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) &&
+        (LangOpts.VScaleMin != LangOpts.VScaleStreamingMin ||
+         LangOpts.VScaleMax != LangOpts.VScaleStreamingMax)) {
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true)) {
+        Diag(Loc, diag::err_sve_fixed_vector_in_streaming_function)
+            << Ty << /*Streaming*/ 0;
+      } else if (const auto *FTy = FD->getType()->getAs<FunctionProtoType>()) {
+        if (FTy->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask) {
+          Diag(Loc, diag::err_sve_fixed_vector_in_streaming_function)
+              << Ty << /*StreamingCompatible*/ 1;
+        }
+      }
+    }
   };
 
   CheckType(Ty);
@@ -2434,9 +2457,10 @@ Sema::PopFunctionScopeInfo(const AnalysisBasedWarnings::Policy *WP,
     OpenMP().popOpenMPFunctionRegion(Scope.get());
 
   // Issue any analysis-based warnings.
-  if (WP && D)
+  if (WP && D) {
+    inferNoReturnAttr(*this, D);
     AnalysisWarnings.IssueWarnings(*WP, Scope.get(), D, BlockType);
-  else
+  } else
     for (const auto &PUD : Scope->PossiblyUnreachableDiags)
       Diag(PUD.Loc, PUD.PD);
 
diff --git a/clang/lib/Sema/SemaAMDGPU.cpp b/clang/lib/Sema/SemaAMDGPU.cpp
index e6414a623b92..5d381229f63c 100644
--- a/clang/lib/Sema/SemaAMDGPU.cpp
+++ b/clang/lib/Sema/SemaAMDGPU.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaAMDGPU.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Sema.h"
 #include "llvm/Support/AtomicOrdering.h"
@@ -367,4 +368,80 @@ void SemaAMDGPU::handleAMDGPUMaxNumWorkGroupsAttr(Decl *D,
   addAMDGPUMaxNumWorkGroupsAttr(D, AL, AL.getArgAsExpr(0), YExpr, ZExpr);
 }
 
+Expr *SemaAMDGPU::ExpandAMDGPUPredicateBI(CallExpr *CE) {
+  ASTContext &Ctx = getASTContext();
+  QualType BoolTy = Ctx.getLogicalOperationType();
+  llvm::APInt False = llvm::APInt::getZero(Ctx.getIntWidth(BoolTy));
+  llvm::APInt True = llvm::APInt::getAllOnes(Ctx.getIntWidth(BoolTy));
+  SourceLocation Loc = CE->getExprLoc();
+
+  if (!CE->getBuiltinCallee())
+    return *ExpandedPredicates
+                .insert(IntegerLiteral::Create(Ctx, False, BoolTy, Loc))
+                .first;
+
+  bool P = false;
+  unsigned BI = CE->getBuiltinCallee();
+  if (Ctx.BuiltinInfo.isAuxBuiltinID(BI))
+    BI = Ctx.BuiltinInfo.getAuxBuiltinID(BI);
+
+  if (BI == AMDGPU::BI__builtin_amdgcn_processor_is) {
+    auto *GFX = dyn_cast<StringLiteral>(CE->getArg(0)->IgnoreParenCasts());
+    if (!GFX) {
+      Diag(Loc, diag::err_amdgcn_processor_is_arg_not_literal);
+      return nullptr;
+    }
+
+    StringRef N = GFX->getString();
+    const TargetInfo &TI = Ctx.getTargetInfo();
+    const TargetInfo *AuxTI = Ctx.getAuxTargetInfo();
+    if (!TI.isValidCPUName(N) && (!AuxTI || !AuxTI->isValidCPUName(N))) {
+      Diag(Loc, diag::err_amdgcn_processor_is_arg_invalid_value) << N;
+      SmallVector<StringRef, 32> ValidList;
+      if (TI.getTriple().getVendor() == llvm::Triple::VendorType::AMD)
+        TI.fillValidCPUList(ValidList);
+      else if (AuxTI) // Since the BI is present it must be and AMDGPU triple.
+        AuxTI->fillValidCPUList(ValidList);
+      if (!ValidList.empty())
+        Diag(Loc, diag::note_amdgcn_processor_is_valid_options)
+            << llvm::join(ValidList, ", ");
+      return nullptr;
+    }
+    if (Ctx.getTargetInfo().getTriple().isSPIRV()) {
+      CE->setType(BoolTy);
+      return *ExpandedPredicates.insert(CE).first;
+    }
+
+    if (auto TID = Ctx.getTargetInfo().getTargetID())
+      P = TID->find(N) == 0;
+  } else {
+    Expr *Arg = CE->getArg(0);
+    if (!Arg || Arg->getType() != Ctx.BuiltinFnTy) {
+      Diag(Loc, diag::err_amdgcn_is_invocable_arg_invalid_value) << Arg;
+      return nullptr;
+    }
+
+    if (Ctx.getTargetInfo().getTriple().isSPIRV()) {
+      CE->setType(BoolTy);
+      return *ExpandedPredicates.insert(CE).first;
+    }
+
+    auto *FD = cast<FunctionDecl>(Arg->getReferencedDeclOfCallee());
+
+    StringRef RF = Ctx.BuiltinInfo.getRequiredFeatures(FD->getBuiltinID());
+    llvm::StringMap<bool> CF;
+    Ctx.getFunctionFeatureMap(CF, FD);
+
+    P = Builtin::evaluateRequiredTargetFeatures(RF, CF);
+  }
+
+  return *ExpandedPredicates
+              .insert(
+                  IntegerLiteral::Create(Ctx, P ? True : False, BoolTy, Loc))
+              .first;
+}
+
+bool SemaAMDGPU::IsPredicate(Expr *E) const {
+  return ExpandedPredicates.contains(E);
+}
 } // namespace clang
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index e992a1012fde..76318a068ce6 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1084,7 +1084,7 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
   // converted to a register of the form S1_2_C3_C4_5. Let the hardware throw
   // an exception for incorrect registers. This matches MSVC behavior.
   if (BuiltinID == AArch64::BI_ReadStatusReg ||
-      BuiltinID == AArch64::BI_WriteStatusReg)
+      BuiltinID == AArch64::BI_WriteStatusReg || BuiltinID == AArch64::BI__sys)
     return SemaRef.BuiltinConstantArgRange(TheCall, 0, 0, 0x7fff);
 
   if (BuiltinID == AArch64::BI__getReg)
@@ -1409,4 +1409,125 @@ void SemaARM::CheckSMEFunctionDefAttributes(const FunctionDecl *FD) {
   }
 }
 
+/// getSVETypeSize - Return SVE vector or predicate register size.
+static uint64_t getSVETypeSize(ASTContext &Context, const BuiltinType *Ty,
+                               bool IsStreaming) {
+  assert(Ty->isSveVLSBuiltinType() && "Invalid SVE Type");
+  uint64_t VScale = IsStreaming ? Context.getLangOpts().VScaleStreamingMin
+                                : Context.getLangOpts().VScaleMin;
+  if (Ty->getKind() == BuiltinType::SveBool ||
+      Ty->getKind() == BuiltinType::SveCount)
+    return (VScale * 128) / Context.getCharWidth();
+  return VScale * 128;
+}
+
+bool SemaARM::areCompatibleSveTypes(QualType FirstType, QualType SecondType) {
+  bool IsStreaming = false;
+  if (getLangOpts().VScaleMin != getLangOpts().VScaleStreamingMin ||
+      getLangOpts().VScaleMax != getLangOpts().VScaleStreamingMax) {
+    if (const FunctionDecl *FD =
+            SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
+      // For streaming-compatible functions, we don't know vector length.
+      if (const auto *T = FD->getType()->getAs<FunctionProtoType>()) {
+        if (T->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask)
+          return false;
+      }
+
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true))
+        IsStreaming = true;
+    }
+  }
+
+  auto IsValidCast = [&](QualType FirstType, QualType SecondType) {
+    if (const auto *BT = FirstType->getAs<BuiltinType>()) {
+      if (const auto *VT = SecondType->getAs<VectorType>()) {
+        // Predicates have the same representation as uint8 so we also have to
+        // check the kind to make these types incompatible.
+        ASTContext &Context = getASTContext();
+        if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)
+          return BT->getKind() == BuiltinType::SveBool;
+        else if (VT->getVectorKind() == VectorKind::SveFixedLengthData)
+          return VT->getElementType().getCanonicalType() ==
+                 FirstType->getSveEltType(Context);
+        else if (VT->getVectorKind() == VectorKind::Generic)
+          return Context.getTypeSize(SecondType) ==
+                     getSVETypeSize(Context, BT, IsStreaming) &&
+                 Context.hasSameType(
+                     VT->getElementType(),
+                     Context.getBuiltinVectorTypeInfo(BT).ElementType);
+      }
+    }
+    return false;
+  };
+
+  return IsValidCast(FirstType, SecondType) ||
+         IsValidCast(SecondType, FirstType);
+}
+
+bool SemaARM::areLaxCompatibleSveTypes(QualType FirstType,
+                                       QualType SecondType) {
+  bool IsStreaming = false;
+  if (getLangOpts().VScaleMin != getLangOpts().VScaleStreamingMin ||
+      getLangOpts().VScaleMax != getLangOpts().VScaleStreamingMax) {
+    if (const FunctionDecl *FD =
+            SemaRef.getCurFunctionDecl(/*AllowLambda=*/true)) {
+      // For streaming-compatible functions, we don't know vector length.
+      if (const auto *T = FD->getType()->getAs<FunctionProtoType>())
+        if (T->getAArch64SMEAttributes() &
+            FunctionType::SME_PStateSMCompatibleMask)
+          return false;
+
+      if (IsArmStreamingFunction(FD, /*IncludeLocallyStreaming=*/true))
+        IsStreaming = true;
+    }
+  }
+
+  auto IsLaxCompatible = [&](QualType FirstType, QualType SecondType) {
+    const auto *BT = FirstType->getAs<BuiltinType>();
+    if (!BT)
+      return false;
+
+    const auto *VecTy = SecondType->getAs<VectorType>();
+    if (VecTy && (VecTy->getVectorKind() == VectorKind::SveFixedLengthData ||
+                  VecTy->getVectorKind() == VectorKind::Generic)) {
+      const LangOptions::LaxVectorConversionKind LVCKind =
+          getLangOpts().getLaxVectorConversions();
+      ASTContext &Context = getASTContext();
+
+      // Can not convert between sve predicates and sve vectors because of
+      // different size.
+      if (BT->getKind() == BuiltinType::SveBool &&
+          VecTy->getVectorKind() == VectorKind::SveFixedLengthData)
+        return false;
+
+      // If __ARM_FEATURE_SVE_BITS != N do not allow GNU vector lax conversion.
+      // "Whenever __ARM_FEATURE_SVE_BITS==N, GNUT implicitly
+      // converts to VLAT and VLAT implicitly converts to GNUT."
+      // ACLE Spec Version 00bet6, 3.7.3.2. Behavior common to vectors and
+      // predicates.
+      if (VecTy->getVectorKind() == VectorKind::Generic &&
+          Context.getTypeSize(SecondType) !=
+              getSVETypeSize(Context, BT, IsStreaming))
+        return false;
+
+      // If -flax-vector-conversions=all is specified, the types are
+      // certainly compatible.
+      if (LVCKind == LangOptions::LaxVectorConversionKind::All)
+        return true;
+
+      // If -flax-vector-conversions=integer is specified, the types are
+      // compatible if the elements are integer types.
+      if (LVCKind == LangOptions::LaxVectorConversionKind::Integer)
+        return VecTy->getElementType().getCanonicalType()->isIntegerType() &&
+               FirstType->getSveEltType(Context)->isIntegerType();
+    }
+
+    return false;
+  };
+
+  return IsLaxCompatible(FirstType, SecondType) ||
+         IsLaxCompatible(SecondType, FirstType);
+}
+
 } // namespace clang
diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp
index c1675a6c67f1..3eed6ad7fe6b 100644
--- a/clang/lib/Sema/SemaAttr.cpp
+++ b/clang/lib/Sema/SemaAttr.cpp
@@ -436,7 +436,7 @@ void Sema::ActOnPragmaClangSection(SourceLocation PragmaLoc,
 }
 
 void Sema::ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action,
-                           StringRef SlotLabel, Expr *alignment) {
+                           StringRef SlotLabel, Expr *Alignment) {
   bool IsXLPragma = getLangOpts().XLPragmaPack;
   // XL pragma pack does not support identifier syntax.
   if (IsXLPragma && !SlotLabel.empty()) {
@@ -445,7 +445,6 @@ void Sema::ActOnPragmaPack(SourceLocation PragmaLoc, PragmaMsStackAction Action,
   }
 
   const AlignPackInfo CurVal = AlignPackStack.CurrentValue;
-  Expr *Alignment = static_cast<Expr *>(alignment);
 
   // If specified then alignment must be a "small" power of two.
   unsigned AlignmentVal = 0;
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index e15a43c11651..a1efa4e114b4 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -23,6 +23,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Initialization.h"
+#include "clang/Sema/SemaAMDGPU.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaRISCV.h"
@@ -98,7 +99,36 @@ namespace {
     CXXCastPath BasePath;
     bool IsARCUnbridgedCast;
 
-    SourceRange OpRange;
+    struct OpRangeType {
+      SourceLocation Locations[3];
+
+      OpRangeType(SourceLocation Begin, SourceLocation LParen,
+                  SourceLocation RParen)
+          : Locations{Begin, LParen, RParen} {}
+
+      OpRangeType() = default;
+
+      SourceLocation getBegin() const { return Locations[0]; }
+
+      SourceLocation getLParenLoc() const { return Locations[1]; }
+
+      SourceLocation getRParenLoc() const { return Locations[2]; }
+
+      friend const StreamingDiagnostic &
+      operator<<(const StreamingDiagnostic &DB, OpRangeType Op) {
+        return DB << SourceRange(Op);
+      }
+
+      SourceRange getParenRange() const {
+        return SourceRange(getLParenLoc(), getRParenLoc());
+      }
+
+      operator SourceRange() const {
+        return SourceRange(getBegin(), getRParenLoc());
+      }
+    };
+
+    OpRangeType OpRange;
     SourceRange DestRange;
 
     // Top-level semantics-checking routines.
@@ -237,49 +267,46 @@ static TryCastResult TryLValueToRValueCast(Sema &Self, Expr *SrcExpr,
                                            CastKind &Kind,
                                            CXXCastPath &BasePath,
                                            unsigned &msg);
-static TryCastResult TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr,
-                                               QualType DestType, bool CStyle,
-                                               SourceRange OpRange,
-                                               unsigned &msg,
-                                               CastKind &Kind,
-                                               CXXCastPath &BasePath);
-static TryCastResult TryStaticPointerDowncast(Sema &Self, QualType SrcType,
-                                              QualType DestType, bool CStyle,
-                                              SourceRange OpRange,
-                                              unsigned &msg,
-                                              CastKind &Kind,
-                                              CXXCastPath &BasePath);
+static TryCastResult
+TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
+                           bool CStyle, CastOperation::OpRangeType OpRange,
+                           unsigned &msg, CastKind &Kind,
+                           CXXCastPath &BasePath);
+static TryCastResult
+TryStaticPointerDowncast(Sema &Self, QualType SrcType, QualType DestType,
+                         bool CStyle, CastOperation::OpRangeType OpRange,
+                         unsigned &msg, CastKind &Kind, CXXCastPath &BasePath);
 static TryCastResult TryStaticDowncast(Sema &Self, CanQualType SrcType,
                                        CanQualType DestType, bool CStyle,
-                                       SourceRange OpRange,
+                                       CastOperation::OpRangeType OpRange,
                                        QualType OrigSrcType,
                                        QualType OrigDestType, unsigned &msg,
-                                       CastKind &Kind,
-                                       CXXCastPath &BasePath);
-static TryCastResult TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr,
-                                               QualType SrcType,
-                                               QualType DestType,bool CStyle,
-                                               SourceRange OpRange,
-                                               unsigned &msg,
-                                               CastKind &Kind,
-                                               CXXCastPath &BasePath);
-
+                                       CastKind &Kind, CXXCastPath &BasePath);
 static TryCastResult
-TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
-                      CheckedConversionKind CCK, SourceRange OpRange,
-                      unsigned &msg, CastKind &Kind, bool ListInitialization);
+TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr, QualType SrcType,
+                             QualType DestType, bool CStyle,
+                             CastOperation::OpRangeType OpRange, unsigned &msg,
+                             CastKind &Kind, CXXCastPath &BasePath);
+
+static TryCastResult TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr,
+                                           QualType DestType,
+                                           CheckedConversionKind CCK,
+                                           CastOperation::OpRangeType OpRange,
+                                           unsigned &msg, CastKind &Kind,
+                                           bool ListInitialization);
 static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
                                    QualType DestType, CheckedConversionKind CCK,
-                                   SourceRange OpRange, unsigned &msg,
-                                   CastKind &Kind, CXXCastPath &BasePath,
+                                   CastOperation::OpRangeType OpRange,
+                                   unsigned &msg, CastKind &Kind,
+                                   CXXCastPath &BasePath,
                                    bool ListInitialization);
 static TryCastResult TryConstCast(Sema &Self, ExprResult &SrcExpr,
                                   QualType DestType, bool CStyle,
                                   unsigned &msg);
 static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
                                         QualType DestType, bool CStyle,
-                                        SourceRange OpRange, unsigned &msg,
-                                        CastKind &Kind);
+                                        CastOperation::OpRangeType OpRange,
+                                        unsigned &msg, CastKind &Kind);
 static TryCastResult TryAddressSpaceCast(Sema &Self, ExprResult &SrcExpr,
                                          QualType DestType, bool CStyle,
                                          unsigned &msg, CastKind &Kind);
@@ -319,7 +346,8 @@ Sema::BuildCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
       DestType->isDependentType() || Ex.get()->isTypeDependent();
 
   CastOperation Op(*this, DestType, E);
-  Op.OpRange = SourceRange(OpLoc, Parens.getEnd());
+  Op.OpRange =
+      CastOperation::OpRangeType(OpLoc, Parens.getBegin(), Parens.getEnd());
   Op.DestRange = AngleBrackets;
 
   Op.checkQualifiedDestType();
@@ -412,7 +440,7 @@ ExprResult Sema::BuildBuiltinBitCastExpr(SourceLocation KWLoc,
                                          TypeSourceInfo *TSI, Expr *Operand,
                                          SourceLocation RParenLoc) {
   CastOperation Op(*this, TSI->getType(), Operand);
-  Op.OpRange = SourceRange(KWLoc, RParenLoc);
+  Op.OpRange = CastOperation::OpRangeType(KWLoc, KWLoc, RParenLoc);
   TypeLoc TL = TSI->getTypeLoc();
   Op.DestRange = SourceRange(TL.getBeginLoc(), TL.getEndLoc());
 
@@ -431,8 +459,8 @@ ExprResult Sema::BuildBuiltinBitCastExpr(SourceLocation KWLoc,
 /// Try to diagnose a failed overloaded cast.  Returns true if
 /// diagnostics were emitted.
 static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
-                                      SourceRange range, Expr *src,
-                                      QualType destType,
+                                      CastOperation::OpRangeType range,
+                                      Expr *src, QualType destType,
                                       bool listInitialization) {
   switch (CT) {
   // These cast kinds don't consider user-defined conversions.
@@ -454,12 +482,13 @@ static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
     return false;
 
   InitializedEntity entity = InitializedEntity::InitializeTemporary(destType);
-  InitializationKind initKind
-    = (CT == CT_CStyle)? InitializationKind::CreateCStyleCast(range.getBegin(),
-                                                      range, listInitialization)
-    : (CT == CT_Functional)? InitializationKind::CreateFunctionalCast(range,
-                                                             listInitialization)
-    : InitializationKind::CreateCast(/*type range?*/ range);
+  InitializationKind initKind =
+      (CT == CT_CStyle) ? InitializationKind::CreateCStyleCast(
+                              range.getBegin(), range, listInitialization)
+      : (CT == CT_Functional)
+          ? InitializationKind::CreateFunctionalCast(
+                range.getBegin(), range.getParenRange(), listInitialization)
+          : InitializationKind::CreateCast(/*type range?*/ range);
   InitializationSequence sequence(S, entity, initKind, src);
 
   // It could happen that a constructor failed to be used because
@@ -544,8 +573,8 @@ static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
 
 /// Diagnose a failed cast.
 static void diagnoseBadCast(Sema &S, unsigned msg, CastType castType,
-                            SourceRange opRange, Expr *src, QualType destType,
-                            bool listInitialization) {
+                            CastOperation::OpRangeType opRange, Expr *src,
+                            QualType destType, bool listInitialization) {
   if (msg == diag::err_bad_cxx_cast_generic &&
       tryDiagnoseOverloadedCast(S, castType, opRange, src, destType,
                                 listInitialization))
@@ -1018,7 +1047,7 @@ void CastOperation::CheckAddrspaceCast() {
 /// or downcast between respective pointers or references.
 static void DiagnoseReinterpretUpDownCast(Sema &Self, const Expr *SrcExpr,
                                           QualType DestType,
-                                          SourceRange OpRange) {
+                                          CastOperation::OpRangeType OpRange) {
   QualType SrcType = SrcExpr->getType();
   // When casting from pointer or reference, get pointee type; use original
   // type otherwise.
@@ -1361,8 +1390,9 @@ static bool IsAddressSpaceConversion(QualType SrcType, QualType DestType) {
 /// and casting away constness.
 static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
                                    QualType DestType, CheckedConversionKind CCK,
-                                   SourceRange OpRange, unsigned &msg,
-                                   CastKind &Kind, CXXCastPath &BasePath,
+                                   CastOperation::OpRangeType OpRange,
+                                   unsigned &msg, CastKind &Kind,
+                                   CXXCastPath &BasePath,
                                    bool ListInitialization) {
   // Determine whether we have the semantics of a C-style cast.
   bool CStyle = (CCK == CheckedConversionKind::CStyleCast ||
@@ -1566,6 +1596,22 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
     return TC_Success;
   }
 
+  if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
+      DestType == Self.Context.getLogicalOperationType()) {
+    SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBI(
+        dyn_cast<CallExpr>(SrcExpr.get()));
+    Kind = CK_NoOp;
+    return TC_Success;
+  }
+
+  if (SrcType == Self.Context.AMDGPUFeaturePredicateTy &&
+      DestType == Self.Context.getLogicalOperationType()) {
+    SrcExpr = Self.AMDGPU().ExpandAMDGPUPredicateBI(
+        dyn_cast<CallExpr>(SrcExpr.get()));
+    Kind = CK_NoOp;
+    return TC_Success;
+  }
+
   // We tried everything. Everything! Nothing works! :-(
   return TC_NotApplicable;
 }
@@ -1625,11 +1671,11 @@ TryCastResult TryLValueToRValueCast(Sema &Self, Expr *SrcExpr,
 }
 
 /// Tests whether a conversion according to C++ 5.2.9p5 is valid.
-TryCastResult
-TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
-                           bool CStyle, SourceRange OpRange,
-                           unsigned &msg, CastKind &Kind,
-                           CXXCastPath &BasePath) {
+TryCastResult TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr,
+                                         QualType DestType, bool CStyle,
+                                         CastOperation::OpRangeType OpRange,
+                                         unsigned &msg, CastKind &Kind,
+                                         CXXCastPath &BasePath) {
   // C++ 5.2.9p5: An lvalue of type "cv1 B", where B is a class type, can be
   //   cast to type "reference to cv2 D", where D is a class derived from B,
   //   if a valid standard conversion from "pointer to D" to "pointer to B"
@@ -1663,11 +1709,11 @@ TryStaticReferenceDowncast(Sema &Self, Expr *SrcExpr, QualType DestType,
 }
 
 /// Tests whether a conversion according to C++ 5.2.9p8 is valid.
-TryCastResult
-TryStaticPointerDowncast(Sema &Self, QualType SrcType, QualType DestType,
-                         bool CStyle, SourceRange OpRange,
-                         unsigned &msg, CastKind &Kind,
-                         CXXCastPath &BasePath) {
+TryCastResult TryStaticPointerDowncast(Sema &Self, QualType SrcType,
+                                       QualType DestType, bool CStyle,
+                                       CastOperation::OpRangeType OpRange,
+                                       unsigned &msg, CastKind &Kind,
+                                       CXXCastPath &BasePath) {
   // C++ 5.2.9p8: An rvalue of type "pointer to cv1 B", where B is a class
   //   type, can be converted to an rvalue of type "pointer to cv2 D", where D
   //   is a class derived from B, if a valid standard conversion from "pointer
@@ -1697,11 +1743,12 @@ TryStaticPointerDowncast(Sema &Self, QualType SrcType, QualType DestType,
 /// TryStaticDowncast - Common functionality of TryStaticReferenceDowncast and
 /// TryStaticPointerDowncast. Tests whether a static downcast from SrcType to
 /// DestType is possible and allowed.
-TryCastResult
-TryStaticDowncast(Sema &Self, CanQualType SrcType, CanQualType DestType,
-                  bool CStyle, SourceRange OpRange, QualType OrigSrcType,
-                  QualType OrigDestType, unsigned &msg,
-                  CastKind &Kind, CXXCastPath &BasePath) {
+TryCastResult TryStaticDowncast(Sema &Self, CanQualType SrcType,
+                                CanQualType DestType, bool CStyle,
+                                CastOperation::OpRangeType OpRange,
+                                QualType OrigSrcType, QualType OrigDestType,
+                                unsigned &msg, CastKind &Kind,
+                                CXXCastPath &BasePath) {
   // We can only work with complete types. But don't complain if it doesn't work
   if (!Self.isCompleteType(OpRange.getBegin(), SrcType) ||
       !Self.isCompleteType(OpRange.getBegin(), DestType))
@@ -1810,12 +1857,12 @@ TryStaticDowncast(Sema &Self, CanQualType SrcType, CanQualType DestType,
 ///   converted to an rvalue of type "pointer to member of B of type cv2 T",
 ///   where B is a base class of D [...].
 ///
-TryCastResult
-TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr, QualType SrcType,
-                             QualType DestType, bool CStyle,
-                             SourceRange OpRange,
-                             unsigned &msg, CastKind &Kind,
-                             CXXCastPath &BasePath) {
+TryCastResult TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr,
+                                           QualType SrcType, QualType DestType,
+                                           bool CStyle,
+                                           CastOperation::OpRangeType OpRange,
+                                           unsigned &msg, CastKind &Kind,
+                                           CXXCastPath &BasePath) {
   const MemberPointerType *DestMemPtr = DestType->getAs<MemberPointerType>();
   if (!DestMemPtr)
     return TC_NotApplicable;
@@ -1881,8 +1928,9 @@ TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr, QualType SrcType,
 TryCastResult TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr,
                                     QualType DestType,
                                     CheckedConversionKind CCK,
-                                    SourceRange OpRange, unsigned &msg,
-                                    CastKind &Kind, bool ListInitialization) {
+                                    CastOperation::OpRangeType OpRange,
+                                    unsigned &msg, CastKind &Kind,
+                                    bool ListInitialization) {
   if (DestType->isRecordType()) {
     if (Self.RequireCompleteType(OpRange.getBegin(), DestType,
                                  diag::err_bad_cast_incomplete) ||
@@ -1899,8 +1947,8 @@ TryCastResult TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr,
           ? InitializationKind::CreateCStyleCast(OpRange.getBegin(), OpRange,
                                                  ListInitialization)
       : (CCK == CheckedConversionKind::FunctionalCast)
-          ? InitializationKind::CreateFunctionalCast(OpRange,
-                                                     ListInitialization)
+          ? InitializationKind::CreateFunctionalCast(
+                OpRange.getBegin(), OpRange.getParenRange(), ListInitialization)
           : InitializationKind::CreateCast(OpRange);
   Expr *SrcExprRaw = SrcExpr.get();
   // FIXME: Per DR242, we should check for an implicit conversion sequence
@@ -2112,7 +2160,8 @@ static void DiagnoseCastOfObjCSEL(Sema &Self, const ExprResult &SrcExpr,
 /// Diagnose casts that change the calling convention of a pointer to a function
 /// defined in the current TU.
 static void DiagnoseCallingConvCast(Sema &Self, const ExprResult &SrcExpr,
-                                    QualType DstType, SourceRange OpRange) {
+                                    QualType DstType,
+                                    CastOperation::OpRangeType OpRange) {
   // Check if this cast would change the calling convention of a function
   // pointer type.
   QualType SrcType = SrcExpr.get()->getType();
@@ -2255,9 +2304,8 @@ static bool fixOverloadedReinterpretCastExpr(Sema &Self, QualType DestType,
 
 static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
                                         QualType DestType, bool CStyle,
-                                        SourceRange OpRange,
-                                        unsigned &msg,
-                                        CastKind &Kind) {
+                                        CastOperation::OpRangeType OpRange,
+                                        unsigned &msg, CastKind &Kind) {
   bool IsLValueCast = false;
 
   DestType = Self.Context.getCanonicalType(DestType);
@@ -3413,7 +3461,7 @@ ExprResult Sema::BuildCStyleCastExpr(SourceLocation LPLoc,
                                      Expr *CastExpr) {
   CastOperation Op(*this, CastTypeInfo->getType(), CastExpr);
   Op.DestRange = CastTypeInfo->getTypeLoc().getSourceRange();
-  Op.OpRange = SourceRange(LPLoc, CastExpr->getEndLoc());
+  Op.OpRange = CastOperation::OpRangeType(LPLoc, LPLoc, CastExpr->getEndLoc());
 
   if (getLangOpts().CPlusPlus) {
     Op.CheckCXXCStyleCast(/*FunctionalCast=*/ false,
@@ -3443,7 +3491,8 @@ ExprResult Sema::BuildCXXFunctionalCastExpr(TypeSourceInfo *CastTypeInfo,
   assert(LPLoc.isValid() && "List-initialization shouldn't get here.");
   CastOperation Op(*this, Type, CastExpr);
   Op.DestRange = CastTypeInfo->getTypeLoc().getSourceRange();
-  Op.OpRange = SourceRange(Op.DestRange.getBegin(), RPLoc);
+  Op.OpRange =
+      CastOperation::OpRangeType(Op.DestRange.getBegin(), LPLoc, RPLoc);
 
   Op.CheckCXXCStyleCast(/*FunctionalCast=*/true, /*ListInit=*/false);
   if (Op.SrcExpr.isInvalid())
@@ -3451,12 +3500,6 @@ ExprResult Sema::BuildCXXFunctionalCastExpr(TypeSourceInfo *CastTypeInfo,
 
   Op.checkQualifiedDestType();
 
-  auto *SubExpr = Op.SrcExpr.get();
-  if (auto *BindExpr = dyn_cast<CXXBindTemporaryExpr>(SubExpr))
-    SubExpr = BindExpr->getSubExpr();
-  if (auto *ConstructExpr = dyn_cast<CXXConstructExpr>(SubExpr))
-    ConstructExpr->setParenOrBraceRange(SourceRange(LPLoc, RPLoc));
-
   // -Wcast-qual
   DiagnoseCastQual(Op.Self, Op.SrcExpr, Op.DestType);
 
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 10a425eec026..440071246d73 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1012,7 +1012,10 @@ class EstimateSizeFormatHandler
       break;
     }
 
-    Size += FS.hasPlusPrefix() || FS.hasSpacePrefix();
+    // If field width is specified, the sign/space is already accounted for
+    // within the field width, so no additional size is needed.
+    if ((FS.hasPlusPrefix() || FS.hasSpacePrefix()) && FieldWidth == 0)
+      Size += 1;
 
     if (FS.hasAlternativeForm()) {
       switch (FS.getConversionSpecifier().getKind()) {
@@ -2261,6 +2264,99 @@ static bool BuiltinCountZeroBitsGeneric(Sema &S, CallExpr *TheCall) {
   return false;
 }
 
+static ExprResult BuiltinInvoke(Sema &S, CallExpr *TheCall) {
+  SourceLocation Loc = TheCall->getBeginLoc();
+  MutableArrayRef Args(TheCall->getArgs(), TheCall->getNumArgs());
+  assert(llvm::none_of(Args, [](Expr *Arg) { return Arg->isTypeDependent(); }));
+
+  if (Args.size() == 0) {
+    S.Diag(TheCall->getBeginLoc(),
+           diag::err_typecheck_call_too_few_args_at_least)
+        << /*callee_type=*/0 << /*min_arg_count=*/1 << /*actual_arg_count=*/0
+        << /*is_non_object=*/0 << TheCall->getSourceRange();
+    return ExprError();
+  }
+
+  QualType FuncT = Args[0]->getType();
+
+  if (const auto *MPT = FuncT->getAs<MemberPointerType>()) {
+    if (Args.size() < 2) {
+      S.Diag(TheCall->getBeginLoc(),
+             diag::err_typecheck_call_too_few_args_at_least)
+          << /*callee_type=*/0 << /*min_arg_count=*/2 << /*actual_arg_count=*/1
+          << /*is_non_object=*/0 << TheCall->getSourceRange();
+      return ExprError();
+    }
+
+    const Type *MemPtrClass = MPT->getQualifier()->getAsType();
+    QualType ObjectT = Args[1]->getType();
+
+    if (MPT->isMemberDataPointer() && S.checkArgCount(TheCall, 2))
+      return ExprError();
+
+    ExprResult ObjectArg = [&]() -> ExprResult {
+      // (1.1): (t1.*f)(t2, ..., tN) when f is a pointer to a member function of
+      // a class T and is_same_v<T, remove_cvref_t<decltype(t1)>> ||
+      // is_base_of_v<T, remove_cvref_t<decltype(t1)>> is true;
+      // (1.4): t1.*f when N=1 and f is a pointer to data member of a class T
+      // and is_same_v<T, remove_cvref_t<decltype(t1)>> ||
+      // is_base_of_v<T, remove_cvref_t<decltype(t1)>> is true;
+      if (S.Context.hasSameType(QualType(MemPtrClass, 0),
+                                S.BuiltinRemoveCVRef(ObjectT, Loc)) ||
+          S.BuiltinIsBaseOf(Args[1]->getBeginLoc(), QualType(MemPtrClass, 0),
+                            S.BuiltinRemoveCVRef(ObjectT, Loc))) {
+        return Args[1];
+      }
+
+      // (t1.get().*f)(t2, ..., tN) when f is a pointer to a member function of
+      // a class T and remove_cvref_t<decltype(t1)> is a specialization of
+      // reference_wrapper;
+      if (const auto *RD = ObjectT->getAsCXXRecordDecl()) {
+        if (RD->isInStdNamespace() &&
+            RD->getDeclName().getAsString() == "reference_wrapper") {
+          CXXScopeSpec SS;
+          IdentifierInfo *GetName = &S.Context.Idents.get("get");
+          UnqualifiedId GetID;
+          GetID.setIdentifier(GetName, Loc);
+
+          ExprResult MemExpr = S.ActOnMemberAccessExpr(
+              S.getCurScope(), Args[1], Loc, tok::period, SS,
+              /*TemplateKWLoc=*/SourceLocation(), GetID, nullptr);
+
+          if (MemExpr.isInvalid())
+            return ExprError();
+
+          return S.ActOnCallExpr(S.getCurScope(), MemExpr.get(), Loc, {}, Loc);
+        }
+      }
+
+      // ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a
+      // class T and t1 does not satisfy the previous two items;
+
+      return S.ActOnUnaryOp(S.getCurScope(), Loc, tok::star, Args[1]);
+    }();
+
+    if (ObjectArg.isInvalid())
+      return ExprError();
+
+    ExprResult BinOp = S.ActOnBinOp(S.getCurScope(), TheCall->getBeginLoc(),
+                                    tok::periodstar, ObjectArg.get(), Args[0]);
+    if (BinOp.isInvalid())
+      return ExprError();
+
+    if (MPT->isMemberDataPointer())
+      return BinOp;
+
+    auto *MemCall = new (S.Context)
+        ParenExpr(SourceLocation(), SourceLocation(), BinOp.get());
+
+    return S.ActOnCallExpr(S.getCurScope(), MemCall, TheCall->getBeginLoc(),
+                           Args.drop_front(2), TheCall->getRParenLoc());
+  }
+  return S.ActOnCallExpr(S.getCurScope(), Args.front(), TheCall->getBeginLoc(),
+                         Args.drop_front(), TheCall->getRParenLoc());
+}
+
 ExprResult
 Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
                                CallExpr *TheCall) {
@@ -2364,6 +2460,17 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
       return ExprError();
     break;
 
+  // The 64-bit acquire, release, and no fence variants are AArch64 only.
+  case Builtin::BI_interlockedbittestandreset64_acq:
+  case Builtin::BI_interlockedbittestandreset64_rel:
+  case Builtin::BI_interlockedbittestandreset64_nf:
+  case Builtin::BI_interlockedbittestandset64_acq:
+  case Builtin::BI_interlockedbittestandset64_rel:
+  case Builtin::BI_interlockedbittestandset64_nf:
+    if (CheckBuiltinTargetInSupported(*this, TheCall, {llvm::Triple::aarch64}))
+      return ExprError();
+    break;
+
   case Builtin::BI__builtin_set_flt_rounds:
     if (CheckBuiltinTargetInSupported(
             *this, TheCall,
@@ -2409,6 +2516,8 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     return BuiltinShuffleVector(TheCall);
     // TheCall will be freed by the smart pointer here, but that's fine, since
     // BuiltinShuffleVector guts it, but then doesn't release it.
+  case Builtin::BI__builtin_invoke:
+    return BuiltinInvoke(*this, TheCall);
   case Builtin::BI__builtin_prefetch:
     if (BuiltinPrefetch(TheCall))
       return ExprError();
@@ -10805,10 +10914,9 @@ static std::optional<IntRange> TryGetExprRange(ASTContext &C, const Expr *E,
         return std::nullopt;
 
       // If the range was previously non-negative, we need an extra bit for the
-      // sign bit. If the range was not non-negative, we need an extra bit
-      // because the negation of the most-negative value is one bit wider than
-      // that value.
-      return IntRange(SubRange->Width + 1, false);
+      // sign bit. Otherwise, we need an extra bit because the negation of the
+      // most-negative value is one bit wider than that value.
+      return IntRange(std::min(SubRange->Width + 1, MaxWidth), false);
     }
 
     case UO_Not: {
@@ -10825,7 +10933,9 @@ static std::optional<IntRange> TryGetExprRange(ASTContext &C, const Expr *E,
 
       // The width increments by 1 if the sub-expression cannot be negative
       // since it now can be.
-      return IntRange(SubRange->Width + (int)SubRange->NonNegative, false);
+      return IntRange(
+          std::min(SubRange->Width + (int)SubRange->NonNegative, MaxWidth),
+          false);
     }
 
     default:
@@ -11684,15 +11794,6 @@ static void DiagnoseFloatingImpCast(Sema &S, const Expr *E, QualType T,
   }
 }
 
-static void CheckCommaOperand(Sema &S, Expr *E, QualType T, SourceLocation CC,
-                              bool ExtraCheckForImplicitConversion) {
-  E = E->IgnoreParenImpCasts();
-  AnalyzeImplicitConversions(S, E, CC);
-
-  if (ExtraCheckForImplicitConversion && E->getType() != T)
-    S.CheckImplicitConversion(E, T, CC);
-}
-
 /// Analyze the given compound assignment for the possible losing of
 /// floating-point precision.
 static void AnalyzeCompoundAssignment(Sema &S, BinaryOperator *E) {
@@ -12057,10 +12158,10 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
   // Strip vector types.
   if (isa<VectorType>(Source)) {
     if (Target->isSveVLSBuiltinType() &&
-        (Context.areCompatibleSveTypes(QualType(Target, 0),
-                                       QualType(Source, 0)) ||
-         Context.areLaxCompatibleSveTypes(QualType(Target, 0),
-                                          QualType(Source, 0))))
+        (ARM().areCompatibleSveTypes(QualType(Target, 0),
+                                     QualType(Source, 0)) ||
+         ARM().areLaxCompatibleSveTypes(QualType(Target, 0),
+                                        QualType(Source, 0))))
       return;
 
     if (Target->isRVVVLSBuiltinType() &&
@@ -12120,10 +12221,10 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
     const Type *OriginalTarget = Context.getCanonicalType(T).getTypePtr();
     // Handle conversion from scalable to fixed when msve-vector-bits is
     // specified
-    if (Context.areCompatibleSveTypes(QualType(OriginalTarget, 0),
-                                      QualType(Source, 0)) ||
-        Context.areLaxCompatibleSveTypes(QualType(OriginalTarget, 0),
-                                         QualType(Source, 0)))
+    if (ARM().areCompatibleSveTypes(QualType(OriginalTarget, 0),
+                                    QualType(Source, 0)) ||
+        ARM().areLaxCompatibleSveTypes(QualType(OriginalTarget, 0),
+                                       QualType(Source, 0)))
       return;
 
     // If the vector cast is cast between two vectors of the same size, it is
@@ -12569,6 +12670,17 @@ struct AnalyzeImplicitConversionsWorkItem {
 };
 }
 
+static void CheckCommaOperand(
+    Sema &S, Expr *E, QualType T, SourceLocation CC,
+    bool ExtraCheckForImplicitConversion,
+    llvm::SmallVectorImpl<AnalyzeImplicitConversionsWorkItem> &WorkList) {
+  E = E->IgnoreParenImpCasts();
+  WorkList.push_back({E, CC, false});
+
+  if (ExtraCheckForImplicitConversion && E->getType() != T)
+    S.CheckImplicitConversion(E, T, CC);
+}
+
 /// Data recursive variant of AnalyzeImplicitConversions. Subexpressions
 /// that should be visited are added to WorkList.
 static void AnalyzeImplicitConversions(
@@ -12644,9 +12756,9 @@ static void AnalyzeImplicitConversions(
       /// how CheckConditionalOperand behaves; it's as-if the correct operand
       /// were directly used for the implicit conversion check.
       CheckCommaOperand(S, BO->getLHS(), T, BO->getOperatorLoc(),
-                        /*ExtraCheckForImplicitConversion=*/false);
+                        /*ExtraCheckForImplicitConversion=*/false, WorkList);
       CheckCommaOperand(S, BO->getRHS(), T, BO->getOperatorLoc(),
-                        /*ExtraCheckForImplicitConversion=*/true);
+                        /*ExtraCheckForImplicitConversion=*/true, WorkList);
       return;
     }
   }
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index f9f7c192f19d..a43ac9eb7610 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -26,6 +26,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttributeCommonInfo.h"
 #include "clang/Basic/CharInfo.h"
+#include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Lex/HeaderSearch.h"
@@ -3259,6 +3260,13 @@ static void AddFunctionParameterChunks(Preprocessor &PP,
       break;
     }
 
+    // C++23 introduces an explicit object parameter, a.k.a. "deducing this"
+    // Skip it for autocomplete and treat the next parameter as the first
+    // parameter
+    if (FirstParameter && Param->isExplicitObjectParameter()) {
+      continue;
+    }
+
     if (FirstParameter)
       FirstParameter = false;
     else
@@ -3427,6 +3435,25 @@ AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
   Result.AddInformativeChunk(Result.getAllocator().CopyString(QualsStr));
 }
 
+static void
+AddFunctionExceptSpecToCompletionString(std::string &NameAndSignature,
+                                        const FunctionDecl *Function) {
+  const auto *Proto = Function->getType()->getAs<FunctionProtoType>();
+  if (!Proto)
+    return;
+
+  auto ExceptInfo = Proto->getExceptionSpecInfo();
+  switch (ExceptInfo.Type) {
+  case EST_BasicNoexcept:
+  case EST_NoexceptTrue:
+    NameAndSignature += " noexcept";
+    break;
+
+  default:
+    break;
+  }
+}
+
 /// Add the name of the given declaration
 static void AddTypedNameChunk(ASTContext &Context, const PrintingPolicy &Policy,
                               const NamedDecl *ND,
@@ -3642,6 +3669,13 @@ CodeCompletionResult::createCodeCompletionStringForOverride(
   std::string NameAndSignature;
   // For overrides all chunks go into the result, none are informative.
   printOverrideString(*CCS, BeforeName, NameAndSignature);
+
+  // If the virtual function is declared with "noexcept", add it in the result
+  // code completion string.
+  const auto *VirtualFunc = dyn_cast<FunctionDecl>(Declaration);
+  assert(VirtualFunc && "overridden decl must be a function");
+  AddFunctionExceptSpecToCompletionString(NameAndSignature, VirtualFunc);
+
   NameAndSignature += " override";
 
   Result.AddTextChunk(Result.getAllocator().CopyString(BeforeName));
@@ -6283,54 +6317,6 @@ ProduceSignatureHelp(Sema &SemaRef, MutableArrayRef<ResultCandidate> Candidates,
   return getParamType(SemaRef, Candidates, CurrentArg);
 }
 
-// Given a callee expression `Fn`, if the call is through a function pointer,
-// try to find the declaration of the corresponding function pointer type,
-// so that we can recover argument names from it.
-static FunctionProtoTypeLoc GetPrototypeLoc(Expr *Fn) {
-  TypeLoc Target;
-
-  if (const auto *T = Fn->getType().getTypePtr()->getAs<TypedefType>()) {
-    Target = T->getDecl()->getTypeSourceInfo()->getTypeLoc();
-
-  } else if (const auto *DR = dyn_cast<DeclRefExpr>(Fn)) {
-    const auto *D = DR->getDecl();
-    if (const auto *const VD = dyn_cast<VarDecl>(D)) {
-      Target = VD->getTypeSourceInfo()->getTypeLoc();
-    }
-  } else if (const auto *ME = dyn_cast<MemberExpr>(Fn)) {
-    const auto *MD = ME->getMemberDecl();
-    if (const auto *FD = dyn_cast<FieldDecl>(MD)) {
-      Target = FD->getTypeSourceInfo()->getTypeLoc();
-    }
-  }
-
-  if (!Target)
-    return {};
-
-  // Unwrap types that may be wrapping the function type
-  while (true) {
-    if (auto P = Target.getAs<PointerTypeLoc>()) {
-      Target = P.getPointeeLoc();
-      continue;
-    }
-    if (auto A = Target.getAs<AttributedTypeLoc>()) {
-      Target = A.getModifiedLoc();
-      continue;
-    }
-    if (auto P = Target.getAs<ParenTypeLoc>()) {
-      Target = P.getInnerLoc();
-      continue;
-    }
-    break;
-  }
-
-  if (auto F = Target.getAs<FunctionProtoTypeLoc>()) {
-    return F;
-  }
-
-  return {};
-}
-
 QualType
 SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
                                              SourceLocation OpenParLoc) {
@@ -6419,7 +6405,7 @@ SemaCodeCompletion::ProduceCallSignatureHelp(Expr *Fn, ArrayRef<Expr *> Args,
       // Lastly we check whether expression's type is function pointer or
       // function.
 
-      FunctionProtoTypeLoc P = GetPrototypeLoc(NakedFn);
+      FunctionProtoTypeLoc P = Resolver.getFunctionProtoTypeLoc(NakedFn);
       QualType T = NakedFn->getType();
       if (!T->getPointeeType().isNull())
         T = T->getPointeeType();
@@ -8465,13 +8451,11 @@ void SemaCodeCompletion::CodeCompleteObjCClassMessage(
 }
 
 void SemaCodeCompletion::CodeCompleteObjCInstanceMessage(
-    Scope *S, Expr *Receiver, ArrayRef<const IdentifierInfo *> SelIdents,
+    Scope *S, Expr *RecExpr, ArrayRef<const IdentifierInfo *> SelIdents,
     bool AtArgumentExpression, ObjCInterfaceDecl *Super) {
   typedef CodeCompletionResult Result;
   ASTContext &Context = getASTContext();
 
-  Expr *RecExpr = static_cast<Expr *>(Receiver);
-
   // If necessary, apply function/array conversion to the receiver.
   // C99 6.7.5.3p[7,8].
   if (RecExpr) {
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index 1594b4423e4d..be9737d9e7c1 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -1138,8 +1138,8 @@ bool Sema::CheckFunctionTemplateConstraints(
   }
 
   CXXThisScopeRAII ThisScope(*this, Record, ThisQuals, Record != nullptr);
-  LambdaScopeForCallOperatorInstantiationRAII LambdaScope(
-      *this, const_cast<FunctionDecl *>(Decl), *MLTAL, Scope);
+  LambdaScopeForCallOperatorInstantiationRAII LambdaScope(*this, Decl, *MLTAL,
+                                                          Scope);
 
   return CheckConstraintSatisfaction(Template, TemplateAC, *MLTAL,
                                      PointOfInstantiation, Satisfaction);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index e1cccf068b5a..4ad3c6520761 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -62,7 +62,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/TargetParser/Triple.h"
 #include <algorithm>
@@ -12401,12 +12400,9 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
 
 void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
   // [basic.start.main]p3
-  //    The main function shall not be declared with a linkage-specification.
-  if (FD->isExternCContext() ||
-      (FD->isExternCXXContext() &&
-       FD->getDeclContext()->getRedeclContext()->isTranslationUnit()))
-    Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification)
-        << FD->getLanguageLinkage();
+  //    The main function shall not be declared with C linkage-specification.
+  if (FD->isExternCContext())
+    Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification);
 
   // C++11 [basic.start.main]p3:
   //   A program that [...] declares main to be inline, static or
@@ -12490,6 +12486,15 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
                                 : FixItHint());
       FD->setInvalidDecl(true);
     }
+
+    // [basic.start.main]p3:
+    // A program that declares a function main that belongs to the global scope
+    // and is attached to a named module is ill-formed.
+    if (FD->isInNamedModule()) {
+      const SourceLocation start = FD->getTypeSpecStartLoc();
+      Diag(start, diag::warn_main_in_named_module)
+          << FixItHint::CreateInsertion(start, "extern \"C++\" ", true);
+    }
   }
 
   // Treat protoless main() as nullary.
@@ -13524,21 +13529,19 @@ bool Sema::GloballyUniqueObjectMightBeAccidentallyDuplicated(
 
   // The target is "hidden" (from the dynamic linker) if:
   // 1. On posix, it has hidden visibility, or
-  // 2. On windows, it has no import/export annotation
+  // 2. On windows, it has no import/export annotation, and neither does the
+  // class which directly contains it.
   if (Context.getTargetInfo().shouldDLLImportComdatSymbols()) {
     if (Target->hasAttr<DLLExportAttr>() || Target->hasAttr<DLLImportAttr>())
       return false;
 
     // If the variable isn't directly annotated, check to see if it's a member
     // of an annotated class.
-    const VarDecl *VD = dyn_cast<VarDecl>(Target);
+    const CXXRecordDecl *Ctx =
+        dyn_cast<CXXRecordDecl>(Target->getDeclContext());
+    if (Ctx && (Ctx->hasAttr<DLLExportAttr>() || Ctx->hasAttr<DLLImportAttr>()))
+      return false;
 
-    if (VD && VD->isStaticDataMember()) {
-      const CXXRecordDecl *Ctx = dyn_cast<CXXRecordDecl>(VD->getDeclContext());
-      if (Ctx &&
-          (Ctx->hasAttr<DLLExportAttr>() || Ctx->hasAttr<DLLImportAttr>()))
-        return false;
-    }
   } else if (Lnk.getVisibility() != HiddenVisibility) {
     // Posix case
     return false;
@@ -13633,6 +13636,26 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     return;
   }
 
+  // __amdgpu_feature_predicate_t cannot be initialised
+  if (VDecl->getType().getDesugaredType(Context) ==
+      Context.AMDGPUFeaturePredicateTy) {
+    Diag(VDecl->getLocation(),
+         diag::err_amdgcn_predicate_type_is_not_constructible)
+        << VDecl;
+    VDecl->setInvalidDecl();
+    return;
+  }
+
+  // __amdgpu_feature_predicate_t cannot be initialised
+  if (VDecl->getType().getDesugaredType(Context) ==
+      Context.AMDGPUFeaturePredicateTy) {
+    Diag(VDecl->getLocation(),
+         diag::err_amdgcn_predicate_type_is_not_constructible)
+        << VDecl;
+    VDecl->setInvalidDecl();
+    return;
+  }
+
   // WebAssembly tables can't be used to initialise a variable.
   if (!Init->getType().isNull() && Init->getType()->isWebAssemblyTableType()) {
     Diag(Init->getExprLoc(), diag::err_wasm_table_art) << 0;
@@ -14126,6 +14149,13 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
   if (VarDecl *Var = dyn_cast<VarDecl>(RealDecl)) {
     QualType Type = Var->getType();
 
+    if (Type.getDesugaredType(Context) == Context.AMDGPUFeaturePredicateTy) {
+      Diag(Var->getLocation(),
+           diag::err_amdgcn_predicate_type_is_not_constructible)
+          << Var;
+      Var->setInvalidDecl();
+      return;
+    }
     // C++1z [dcl.dcl]p1 grammar implies that an initializer is mandatory.
     if (isa<DecompositionDecl>(RealDecl)) {
       Diag(Var->getLocation(), diag::err_decomp_decl_requires_init) << Var;
@@ -16391,7 +16421,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
           // Returns true if the token beginning at this Loc is `const`.
           auto isLocAtConst = [&](SourceLocation Loc, const SourceManager &SM,
                                   const LangOptions &LangOpts) {
-            std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+            FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
             if (LocInfo.first.isInvalid())
               return false;
 
@@ -18633,8 +18663,7 @@ ExprResult Sema::VerifyBitField(SourceLocation FieldLoc,
         << FieldName << FieldTy << BitWidth->getSourceRange();
     return Diag(FieldLoc, diag::err_not_integral_type_anon_bitfield)
       << FieldTy << BitWidth->getSourceRange();
-  } else if (DiagnoseUnexpandedParameterPack(const_cast<Expr *>(BitWidth),
-                                             UPPC_BitFieldWidth))
+  } else if (DiagnoseUnexpandedParameterPack(BitWidth, UPPC_BitFieldWidth))
     return ExprError();
 
   // If the bit-width is type- or value-dependent, don't try to check
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index eba29e609cb0..c5d5d03cc99c 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -40,6 +40,7 @@
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaAMDGPU.h"
 #include "clang/Sema/SemaARM.h"
 #include "clang/Sema/SemaAVR.h"
@@ -1938,6 +1939,53 @@ static void handleNakedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(::new (S.Context) NakedAttr(S.Context, AL));
 }
 
+// FIXME: This is a best-effort heuristic.
+// Currently only handles single throw expressions (optionally with
+// ExprWithCleanups). We could expand this to perform control-flow analysis for
+// more complex patterns.
+static bool isKnownToAlwaysThrow(const FunctionDecl *FD) {
+  if (!FD->hasBody())
+    return false;
+  const Stmt *Body = FD->getBody();
+  const Stmt *OnlyStmt = nullptr;
+
+  if (const auto *Compound = dyn_cast<CompoundStmt>(Body)) {
+    if (Compound->size() != 1)
+      return false; // More than one statement, can't be known to always throw.
+    OnlyStmt = *Compound->body_begin();
+  } else {
+    OnlyStmt = Body;
+  }
+
+  // Unwrap ExprWithCleanups if necessary.
+  if (const auto *EWC = dyn_cast<ExprWithCleanups>(OnlyStmt)) {
+    OnlyStmt = EWC->getSubExpr();
+  }
+  // Check if the only statement is a throw expression.
+  return isa<CXXThrowExpr>(OnlyStmt);
+}
+
+void clang::inferNoReturnAttr(Sema &S, const Decl *D) {
+  auto *FD = dyn_cast<FunctionDecl>(D);
+  if (!FD)
+    return;
+
+  auto *NonConstFD = const_cast<FunctionDecl *>(FD);
+  DiagnosticsEngine &Diags = S.getDiagnostics();
+  if (Diags.isIgnored(diag::warn_falloff_nonvoid, FD->getLocation()) &&
+      Diags.isIgnored(diag::warn_suggest_noreturn_function, FD->getLocation()))
+    return;
+
+  if (!FD->hasAttr<NoReturnAttr>() && !FD->hasAttr<InferredNoReturnAttr>() &&
+      isKnownToAlwaysThrow(FD)) {
+    NonConstFD->addAttr(InferredNoReturnAttr::CreateImplicit(S.Context));
+
+    // Emit a diagnostic suggesting the function being marked [[noreturn]].
+    S.Diag(FD->getLocation(), diag::warn_suggest_noreturn_function)
+        << /*isFunction=*/0 << FD;
+  }
+}
+
 static void handleNoReturnAttr(Sema &S, Decl *D, const ParsedAttr &Attrs) {
   if (hasDeclarator(D)) return;
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 16645ecf411e..f0247f865ba4 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -3431,7 +3431,7 @@ static bool IsUnusedPrivateField(const FieldDecl *FD) {
 NamedDecl *
 Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
                                MultiTemplateParamsArg TemplateParameterLists,
-                               Expr *BW, const VirtSpecifiers &VS,
+                               Expr *BitWidth, const VirtSpecifiers &VS,
                                InClassInitStyle InitStyle) {
   const DeclSpec &DS = D.getDeclSpec();
   DeclarationNameInfo NameInfo = GetNameForDeclarator(D);
@@ -3442,8 +3442,6 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   if (Loc.isInvalid())
     Loc = D.getBeginLoc();
 
-  Expr *BitWidth = static_cast<Expr*>(BW);
-
   assert(isa<CXXRecordDecl>(CurContext));
   assert(!DS.isFriendSpecified());
 
@@ -13619,82 +13617,40 @@ bool Sema::CheckUsingDeclQualifier(SourceLocation UsingLoc, bool HasTypename,
       RequireCompleteDeclContext(const_cast<CXXScopeSpec&>(SS), NamedContext))
     return true;
 
-  if (getLangOpts().CPlusPlus11) {
-    // C++11 [namespace.udecl]p3:
-    //   In a using-declaration used as a member-declaration, the
-    //   nested-name-specifier shall name a base class of the class
-    //   being defined.
-
-    if (cast<CXXRecordDecl>(CurContext)->isProvablyNotDerivedFrom(
-                                 cast<CXXRecordDecl>(NamedContext))) {
+  // C++26 [namespace.udecl]p3:
+  //   In a using-declaration used as a member-declaration, each
+  //   using-declarator shall either name an enumerator or have a
+  //   nested-name-specifier naming a base class of the current class
+  //   ([expr.prim.this]). ...
+  // "have a nested-name-specifier naming a base class of the current class"
+  // was introduced by CWG400.
 
-      if (Cxx20Enumerator) {
-        Diag(NameLoc, diag::warn_cxx17_compat_using_decl_non_member_enumerator)
-            << SS.getRange();
-        return false;
-      }
+  if (cast<CXXRecordDecl>(CurContext)
+          ->isProvablyNotDerivedFrom(cast<CXXRecordDecl>(NamedContext))) {
 
-      if (CurContext == NamedContext) {
-        Diag(SS.getBeginLoc(),
-             diag::err_using_decl_nested_name_specifier_is_current_class)
-            << SS.getRange();
-        return !getLangOpts().CPlusPlus20;
-      }
+    if (Cxx20Enumerator) {
+      Diag(NameLoc, diag::warn_cxx17_compat_using_decl_non_member_enumerator)
+          << SS.getRange();
+      return false;
+    }
 
-      if (!cast<CXXRecordDecl>(NamedContext)->isInvalidDecl()) {
-        Diag(SS.getBeginLoc(),
-             diag::err_using_decl_nested_name_specifier_is_not_base_class)
-            << SS.getScopeRep() << cast<CXXRecordDecl>(CurContext)
-            << SS.getRange();
-      }
+    if (CurContext == NamedContext) {
+      Diag(SS.getBeginLoc(),
+           diag::err_using_decl_nested_name_specifier_is_current_class)
+          << SS.getRange();
       return true;
     }
 
-    return false;
-  }
-
-  // C++03 [namespace.udecl]p4:
-  //   A using-declaration used as a member-declaration shall refer
-  //   to a member of a base class of the class being defined [etc.].
-
-  // Salient point: SS doesn't have to name a base class as long as
-  // lookup only finds members from base classes.  Therefore we can
-  // diagnose here only if we can prove that can't happen,
-  // i.e. if the class hierarchies provably don't intersect.
-
-  // TODO: it would be nice if "definitely valid" results were cached
-  // in the UsingDecl and UsingShadowDecl so that these checks didn't
-  // need to be repeated.
-
-  llvm::SmallPtrSet<const CXXRecordDecl *, 4> Bases;
-  auto Collect = [&Bases](const CXXRecordDecl *Base) {
-    Bases.insert(Base);
+    if (!cast<CXXRecordDecl>(NamedContext)->isInvalidDecl()) {
+      Diag(SS.getBeginLoc(),
+           diag::err_using_decl_nested_name_specifier_is_not_base_class)
+          << SS.getScopeRep() << cast<CXXRecordDecl>(CurContext)
+          << SS.getRange();
+    }
     return true;
-  };
-
-  // Collect all bases. Return false if we find a dependent base.
-  if (!cast<CXXRecordDecl>(CurContext)->forallBases(Collect))
-    return false;
-
-  // Returns true if the base is dependent or is one of the accumulated base
-  // classes.
-  auto IsNotBase = [&Bases](const CXXRecordDecl *Base) {
-    return !Bases.count(Base);
-  };
-
-  // Return false if the class has a dependent base or if it or one
-  // of its bases is present in the base set of the current context.
-  if (Bases.count(cast<CXXRecordDecl>(NamedContext)) ||
-      !cast<CXXRecordDecl>(NamedContext)->forallBases(IsNotBase))
-    return false;
-
-  Diag(SS.getRange().getBegin(),
-       diag::err_using_decl_nested_name_specifier_is_not_base_class)
-    << SS.getScopeRep()
-    << cast<CXXRecordDecl>(CurContext)
-    << SS.getRange();
+  }
 
-  return true;
+  return false;
 }
 
 Decl *Sema::ActOnAliasDeclaration(Scope *S, AccessSpecifier AS,
@@ -16427,9 +16383,8 @@ bool Sema::CompleteConstructorCall(CXXConstructorDecl *Constructor,
 
   DiagnoseSentinelCalls(Constructor, Loc, AllArgs);
 
-  CheckConstructorCall(Constructor, DeclInitType,
-                       llvm::ArrayRef(AllArgs.data(), AllArgs.size()), Proto,
-                       Loc);
+  CheckConstructorCall(Constructor, DeclInitType, llvm::ArrayRef(AllArgs),
+                       Proto, Loc);
 
   return Invalid;
 }
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 5dccaaecd13e..38e80a2a89b1 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -51,10 +51,11 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaAMDGPU.h"
+#include "clang/Sema/SemaARM.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/SemaHLSL.h"
-#include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaPseudoObject.h"
@@ -6573,14 +6574,14 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
   // without any additional checking.
   if (Fn->getType() == Context.BuiltinFnTy && ArgExprs.size() == 1 &&
       ArgExprs[0]->getType() == Context.BuiltinFnTy) {
-    auto *FD = cast<FunctionDecl>(Fn->getReferencedDeclOfCallee());
+    const auto *FD = cast<FunctionDecl>(Fn->getReferencedDeclOfCallee());
 
     if (FD->getName() == "__builtin_amdgcn_is_invocable") {
-      auto FnPtrTy = Context.getPointerType(FD->getType());
-      auto *R = ImpCastExprToType(Fn, FnPtrTy, CK_BuiltinFnToFnPtr).get();
-      return CallExpr::Create(Context, R, ArgExprs, Context.VoidTy,
-                              ExprValueKind::VK_PRValue, RParenLoc,
-                              FPOptionsOverride());
+      QualType FnPtrTy = Context.getPointerType(FD->getType());
+      Expr *R = ImpCastExprToType(Fn, FnPtrTy, CK_BuiltinFnToFnPtr).get();
+      return CallExpr::Create(
+          Context, R, ArgExprs, Context.AMDGPUFeaturePredicateTy,
+          ExprValueKind::VK_PRValue, RParenLoc, FPOptionsOverride());
     }
   }
 
@@ -9413,8 +9414,8 @@ AssignConvertType Sema::CheckAssignmentConstraints(QualType LHSType,
     // Allow assignments between fixed-length and sizeless SVE vectors.
     if ((LHSType->isSVESizelessBuiltinType() && RHSType->isVectorType()) ||
         (LHSType->isVectorType() && RHSType->isSVESizelessBuiltinType()))
-      if (Context.areCompatibleSveTypes(LHSType, RHSType) ||
-          Context.areLaxCompatibleSveTypes(LHSType, RHSType)) {
+      if (ARM().areCompatibleSveTypes(LHSType, RHSType) ||
+          ARM().areLaxCompatibleSveTypes(LHSType, RHSType)) {
         Kind = CK_BitCast;
         return AssignConvertType::Compatible;
       }
@@ -15880,7 +15881,9 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
         // Vector logical not returns the signed variant of the operand type.
         resultType = GetSignedVectorType(resultType);
         break;
-      } else if (IsAMDGPUPredicateBI(InputExpr)) {
+      } else if (resultType == Context.AMDGPUFeaturePredicateTy) {
+        resultType = Context.getLogicalOperationType();
+        Input = AMDGPU().ExpandAMDGPUPredicateBI(dyn_cast<CallExpr>(InputExpr));
         break;
       } else {
         return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
@@ -18319,6 +18322,11 @@ static bool isImplicitlyDefinableConstexprFunction(FunctionDecl *Func) {
 
   if (Func->isImplicitlyInstantiable() || !Func->isUserProvided())
     return true;
+
+  // Lambda conversion operators are never user provided.
+  if (CXXConversionDecl *Conv = dyn_cast<CXXConversionDecl>(Func))
+    return isLambdaConversionOperator(Conv);
+
   auto *CCD = dyn_cast<CXXConstructorDecl>(Func);
   return CCD && CCD->getInheritedConstructor();
 }
@@ -20236,6 +20244,9 @@ MarkExprReferenced(Sema &SemaRef, SourceLocation Loc, Decl *D, Expr *E,
   if (SemaRef.OpenMP().isInOpenMPDeclareTargetContext())
     SemaRef.OpenMP().checkDeclIsAllowedInOpenMPTarget(E, D);
 
+  if (SemaRef.getLangOpts().OpenACC)
+    SemaRef.OpenACC().CheckDeclReference(Loc, E, D);
+
   if (VarDecl *Var = dyn_cast<VarDecl>(D)) {
     DoMarkVarDeclReferenced(SemaRef, Loc, Var, E, RefsMinusAssignments);
     if (SemaRef.getLangOpts().CPlusPlus)
@@ -20742,13 +20753,8 @@ ExprResult Sema::CheckBooleanCondition(SourceLocation Loc, Expr *E,
   E = result.get();
 
   if (!E->isTypeDependent()) {
-    if (E->getType()->isVoidType()) {
-      bool InvalidPredicate = false;
-      if (auto *BIC = MaybeHandleAMDGPUPredicateBI(*this, E, InvalidPredicate))
-        return BIC;
-      else if (InvalidPredicate)
-        return ExprError();
-    }
+    if (E->getType() == Context.AMDGPUFeaturePredicateTy)
+      return AMDGPU().ExpandAMDGPUPredicateBI(dyn_cast_or_null<CallExpr>(E));
 
     if (getLangOpts().CPlusPlus)
       return CheckCXXBooleanCondition(E, IsConstexpr); // C++ 6.4p4
@@ -20785,6 +20791,7 @@ Sema::ConditionResult Sema::ActOnCondition(Scope *S, SourceLocation Loc,
     break;
 
   case ConditionKind::ConstexprIf:
+    // Note: this might produce a FullExpr
     Cond = CheckBooleanCondition(Loc, SubExpr, true);
     break;
 
@@ -20797,13 +20804,13 @@ Sema::ConditionResult Sema::ActOnCondition(Scope *S, SourceLocation Loc,
                               {SubExpr}, PreferredConditionType(CK));
     if (!Cond.get())
       return ConditionError();
-  }
-  // FIXME: FullExprArg doesn't have an invalid bit, so check nullness instead.
-  FullExprArg FullExpr = MakeFullExpr(Cond.get(), Loc);
-  if (!FullExpr.get())
+  } else if (Cond.isUsable() && !isa<FullExpr>(Cond.get()))
+    Cond = ActOnFinishFullExpr(Cond.get(), Loc, /*DiscardedValue*/ false);
+
+  if (!Cond.isUsable())
     return ConditionError();
 
-  return ConditionResult(*this, nullptr, FullExpr,
+  return ConditionResult(*this, nullptr, Cond,
                          CK == ConditionKind::ConstexprIf);
 }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 4a86cbd0633b..f17a33882542 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4357,7 +4357,8 @@ Sema::ConditionResult Sema::ActOnConditionVariable(Decl *ConditionVar,
       CheckConditionVariable(cast<VarDecl>(ConditionVar), StmtLoc, CK);
   if (E.isInvalid())
     return ConditionError();
-  return ConditionResult(*this, ConditionVar, MakeFullExpr(E.get(), StmtLoc),
+  E = ActOnFinishFullExpr(E.get(), /*DiscardedValue*/ false);
+  return ConditionResult(*this, ConditionVar, E,
                          CK == ConditionKind::ConstexprIf);
 }
 
@@ -4417,6 +4418,12 @@ ExprResult Sema::CheckCXXBooleanCondition(Expr *CondExpr, bool IsConstexpr) {
   if (!IsConstexpr || E.isInvalid() || E.get()->isValueDependent())
     return E;
 
+  E = ActOnFinishFullExpr(E.get(), E.get()->getExprLoc(),
+                          /*DiscardedValue*/ false,
+                          /*IsConstexpr*/ true);
+  if (E.isInvalid())
+    return E;
+
   // FIXME: Return this value to the caller so they don't need to recompute it.
   llvm::APSInt Cond;
   E = VerifyIntegerConstantExpression(
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index b5975c2e5782..ba55f8263e47 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,7 +39,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DXILABI.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -120,7 +120,7 @@ static ResourceClass getResourceClass(RegisterType RT) {
   llvm_unreachable("unexpected RegisterType value");
 }
 
-static Builtin::ID getSpecConstBuiltinId(QualType Type) {
+static Builtin::ID getSpecConstBuiltinId(const Type *Type) {
   const auto *BT = dyn_cast<BuiltinType>(Type);
   if (!BT) {
     if (!Type->isEnumeralType())
@@ -654,7 +654,8 @@ SemaHLSL::mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL,
 
   auto *VD = cast<VarDecl>(D);
 
-  if (getSpecConstBuiltinId(VD->getType()) == Builtin::NotBuiltin) {
+  if (getSpecConstBuiltinId(VD->getType()->getUnqualifiedDesugaredType()) ==
+      Builtin::NotBuiltin) {
     Diag(VD->getLocation(), diag::err_specialization_const);
     return nullptr;
   }
@@ -1067,7 +1068,7 @@ void SemaHLSL::ActOnFinishRootSignatureDecl(
 
   auto *SignatureDecl = HLSLRootSignatureDecl::Create(
       SemaRef.getASTContext(), /*DeclContext=*/SemaRef.CurContext, Loc,
-      DeclIdent, Elements);
+      DeclIdent, SemaRef.getLangOpts().HLSLRootSigVer, Elements);
 
   if (handleRootSignatureDecl(SignatureDecl, Loc))
     return;
@@ -1116,6 +1117,52 @@ bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
       Info.Space = Descriptor->Space;
       Info.Visibility = Descriptor->Visibility;
       Infos.push_back(Info);
+    } else if (const auto *Constants =
+                   std::get_if<llvm::hlsl::rootsig::RootConstants>(&Elem)) {
+      RangeInfo Info;
+      Info.LowerBound = Constants->Reg.Number;
+      Info.UpperBound = Info.LowerBound; // use inclusive ranges []
+
+      Info.Class = llvm::dxil::ResourceClass::CBuffer;
+      Info.Space = Constants->Space;
+      Info.Visibility = Constants->Visibility;
+      Infos.push_back(Info);
+    } else if (const auto *Sampler =
+                   std::get_if<llvm::hlsl::rootsig::StaticSampler>(&Elem)) {
+      RangeInfo Info;
+      Info.LowerBound = Sampler->Reg.Number;
+      Info.UpperBound = Info.LowerBound; // use inclusive ranges []
+
+      Info.Class = llvm::dxil::ResourceClass::Sampler;
+      Info.Space = Sampler->Space;
+      Info.Visibility = Sampler->Visibility;
+      Infos.push_back(Info);
+    } else if (const auto *Clause =
+                   std::get_if<llvm::hlsl::rootsig::DescriptorTableClause>(
+                       &Elem)) {
+      RangeInfo Info;
+      Info.LowerBound = Clause->Reg.Number;
+      assert(0 < Clause->NumDescriptors && "Verified as part of TODO(#129940)");
+      Info.UpperBound = Clause->NumDescriptors == RangeInfo::Unbounded
+                            ? RangeInfo::Unbounded
+                            : Info.LowerBound + Clause->NumDescriptors -
+                                  1; // use inclusive ranges []
+
+      Info.Class = Clause->Type;
+      Info.Space = Clause->Space;
+      // Note: Clause does not hold the visibility this will need to
+      Infos.push_back(Info);
+    } else if (const auto *Table =
+                   std::get_if<llvm::hlsl::rootsig::DescriptorTable>(&Elem)) {
+      // Table holds the Visibility of all owned Clauses in Table, so iterate
+      // owned Clauses and update their corresponding RangeInfo
+      assert(Table->NumClauses <= Infos.size() && "RootElement");
+      // The last Table->NumClauses elements of Infos are the owned Clauses
+      // generated RangeInfo
+      auto TableInfos =
+          MutableArrayRef<RangeInfo>(Infos).take_back(Table->NumClauses);
+      for (RangeInfo &Info : TableInfos)
+        Info.Visibility = Table->Visibility;
     }
   }
 
@@ -1153,14 +1200,16 @@ bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
   auto ReportOverlap = [this, Loc, &HadOverlap](const RangeInfo *Info,
                                                 const RangeInfo *OInfo) {
     HadOverlap = true;
-    auto CommonVis =
-        Info->Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
-            ? OInfo->Visibility
-            : Info->Visibility;
+    auto CommonVis = Info->Visibility == llvm::dxbc::ShaderVisibility::All
+                         ? OInfo->Visibility
+                         : Info->Visibility;
     this->Diag(Loc, diag::err_hlsl_resource_range_overlap)
         << llvm::to_underlying(Info->Class) << Info->LowerBound
+        << /*unbounded=*/(Info->UpperBound == RangeInfo::Unbounded)
         << Info->UpperBound << llvm::to_underlying(OInfo->Class)
-        << OInfo->LowerBound << OInfo->UpperBound << Info->Space << CommonVis;
+        << OInfo->LowerBound
+        << /*unbounded=*/(OInfo->UpperBound == RangeInfo::Unbounded)
+        << OInfo->UpperBound << Info->Space << CommonVis;
   };
 
   // 3: Iterate through collected RangeInfos
@@ -1188,7 +1237,7 @@ bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
     // ResourceRanges in the former case and it will be an ArrayRef to just the
     // all visiblity ResourceRange in the latter case.
     ArrayRef<ResourceRange> OverlapRanges =
-        Info.Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
+        Info.Visibility == llvm::dxbc::ShaderVisibility::All
             ? ArrayRef<ResourceRange>{Ranges}.drop_front()
             : ArrayRef<ResourceRange>{Ranges}.take_front();
 
@@ -2369,7 +2418,7 @@ static bool CheckFloatOrHalfRepresentation(Sema *S, SourceLocation Loc,
                                            clang::QualType PassedType) {
   clang::QualType BaseType =
       PassedType->isVectorType()
-          ? PassedType->getAs<clang::VectorType>()->getElementType()
+          ? PassedType->castAs<clang::VectorType>()->getElementType()
           : PassedType;
   if (!BaseType->isHalfType() && !BaseType->isFloat32Type())
     return S->Diag(Loc, diag::err_builtin_invalid_arg_type)
@@ -3923,7 +3972,8 @@ bool SemaHLSL::handleInitialization(VarDecl *VDecl, Expr *&Init) {
     return false;
   }
 
-  Builtin::ID BID = getSpecConstBuiltinId(VDecl->getType());
+  Builtin::ID BID =
+      getSpecConstBuiltinId(VDecl->getType()->getUnqualifiedDesugaredType());
 
   // Argument 1: The ID from the attribute
   int ConstantID = ConstIdAttr->getId();
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index da56225b2f92..ba20c29f4233 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -260,29 +260,37 @@ static void CheckStringInit(Expr *Str, QualType &DeclT, const ArrayType *AT,
              diag::ext_initializer_string_for_char_array_too_long)
           << Str->getSourceRange();
     else if (StrLength - 1 == ArrayLen) {
-      // If the entity being initialized has the nonstring attribute, then
-      // silence the "missing nonstring" diagnostic. If there's no entity,
-      // check whether we're initializing an array of arrays; if so, walk the
-      // parents to find an entity.
-      auto FindCorrectEntity =
-          [](const InitializedEntity *Entity) -> const ValueDecl * {
-        while (Entity) {
-          if (const ValueDecl *VD = Entity->getDecl())
-            return VD;
-          if (!Entity->getType()->isArrayType())
-            return nullptr;
-          Entity = Entity->getParent();
-        }
-
-        return nullptr;
-      };
-      if (const ValueDecl *D = FindCorrectEntity(&Entity);
-          !D || !D->hasAttr<NonStringAttr>())
-        S.Diag(
-            Str->getBeginLoc(),
-            diag::warn_initializer_string_for_char_array_too_long_no_nonstring)
-            << ArrayLen << StrLength << Str->getSourceRange();
+      // In C, if the string literal is null-terminated explicitly, e.g., `char
+      // a[4] = "ABC\0"`, there should be no warning:
+      const auto *SL = dyn_cast<StringLiteral>(Str->IgnoreParens());
+      bool IsSLSafe = SL && SL->getLength() > 0 &&
+                      SL->getCodeUnit(SL->getLength() - 1) == 0;
+
+      if (!IsSLSafe) {
+        // If the entity being initialized has the nonstring attribute, then
+        // silence the "missing nonstring" diagnostic. If there's no entity,
+        // check whether we're initializing an array of arrays; if so, walk the
+        // parents to find an entity.
+        auto FindCorrectEntity =
+            [](const InitializedEntity *Entity) -> const ValueDecl * {
+          while (Entity) {
+            if (const ValueDecl *VD = Entity->getDecl())
+              return VD;
+            if (!Entity->getType()->isArrayType())
+              return nullptr;
+            Entity = Entity->getParent();
+          }
 
+          return nullptr;
+        };
+        if (const ValueDecl *D = FindCorrectEntity(&Entity);
+            !D || !D->hasAttr<NonStringAttr>())
+          S.Diag(
+              Str->getBeginLoc(),
+              diag::
+                  warn_initializer_string_for_char_array_too_long_no_nonstring)
+              << ArrayLen << StrLength << Str->getSourceRange();
+      }
       // Always emit the C++ compatibility diagnostic.
       S.Diag(Str->getBeginLoc(),
              diag::warn_initializer_string_for_char_array_too_long_for_cpp)
@@ -9102,6 +9110,15 @@ bool InitializationSequence::Diagnose(Sema &S,
 
   case FK_ConversionFailed: {
     QualType FromType = OnlyArg->getType();
+    // __amdgpu_feature_predicate_t can be explicitly cast to the logical op
+    // type, although this is almost always an error and we advise against it.
+    if (FromType == S.Context.AMDGPUFeaturePredicateTy &&
+        DestType == S.Context.getLogicalOperationType()) {
+      S.Diag(OnlyArg->getExprLoc(),
+             diag::err_amdgcn_predicate_type_needs_explicit_bool_cast)
+          << OnlyArg << DestType;
+      break;
+    }
     PartialDiagnostic PDiag = S.PDiag(diag::err_init_conversion_failed)
       << (int)Entity.getKind()
       << DestType
@@ -9905,6 +9922,14 @@ Sema::PerformCopyInitialization(const InitializedEntity &Entity,
   if (EqualLoc.isInvalid())
     EqualLoc = InitE->getBeginLoc();
 
+  if (Entity.getType().getDesugaredType(Context) ==
+          Context.AMDGPUFeaturePredicateTy &&
+      Entity.getDecl()) {
+    Diag(EqualLoc, diag::err_amdgcn_predicate_type_is_not_constructible)
+        << Entity.getDecl();
+    return ExprError();
+  }
+
   InitializationKind Kind = InitializationKind::CreateCopy(
       InitE->getBeginLoc(), EqualLoc, AllowExplicit);
   InitializationSequence Seq(*this, Entity, Kind, InitE, TopLevelOfInitList);
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index fe70ce3fba6a..7c982bcd63d7 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -337,11 +337,9 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
   // tokens in a file (excluding the global module fragment.).
   if (getLangOpts().CPlusPlusModules && !IntroducerIsFirstPPToken && !SeenGMF) {
     Diag(ModuleLoc, diag::err_module_decl_not_at_start);
-    SourceLocation BeginLoc = PP.getMainFileFirstPPToken().getLocation();
-    if (BeginLoc.isValid()) {
-      Diag(BeginLoc, diag::note_global_module_introducer_missing)
-          << FixItHint::CreateInsertion(BeginLoc, "module;\n");
-    }
+    SourceLocation BeginLoc = PP.getMainFileFirstPPTokenLoc();
+    Diag(BeginLoc, diag::note_global_module_introducer_missing)
+        << FixItHint::CreateInsertion(BeginLoc, "module;\n");
   }
 
   // C++23 [module.unit]p1: ... The identifiers module and import shall not
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 3e962fcb8b0e..9d5106805892 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -310,7 +310,7 @@ static bool LocPropertyAttribute( ASTContext &Context, const char *attrName,
     return false;
 
   SourceManager &SM = Context.getSourceManager();
-  std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(LParenLoc);
+  FileIDAndOffset locInfo = SM.getDecomposedLoc(LParenLoc);
   // Try to load the file buffer.
   bool invalidTemp = false;
   StringRef file = SM.getBufferData(locInfo.first, &invalidTemp);
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index e63d75c934f2..46aa7dd0dcc2 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -519,8 +519,31 @@ bool SemaOpenACC::CheckVarIsPointerType(OpenACCClauseKind ClauseKind,
   return false;
 }
 
+void SemaOpenACC::ActOnStartParseVar(OpenACCDirectiveKind DK,
+                                     OpenACCClauseKind CK) {
+  if (DK == OpenACCDirectiveKind::Cache) {
+    CacheInfo.ParsingCacheVarList = true;
+    CacheInfo.IsInvalidCacheRef = false;
+  }
+}
+
+void SemaOpenACC::ActOnInvalidParseVar() {
+  CacheInfo.ParsingCacheVarList = false;
+  CacheInfo.IsInvalidCacheRef = false;
+}
+
 ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   Expr *CurVarExpr = VarExpr->IgnoreParenImpCasts();
+  // Clear this here, so we can do the returns based on the invalid cache ref
+  // here.  Note all return statements in this function must return ExprError if
+  // IsInvalidCacheRef. However, instead of doing an 'early return' in that
+  // case, we can let the rest of the diagnostics happen, as the invalid decl
+  // ref is a warning.
+  bool WasParsingInvalidCacheRef =
+      CacheInfo.ParsingCacheVarList && CacheInfo.IsInvalidCacheRef;
+  CacheInfo.ParsingCacheVarList = false;
+  CacheInfo.IsInvalidCacheRef = false;
+
   if (!isa<ArraySectionExpr, ArraySubscriptExpr>(CurVarExpr)) {
     Diag(VarExpr->getExprLoc(), diag::err_acc_not_a_var_ref_cache);
     return ExprError();
@@ -540,19 +563,19 @@ ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   if (const auto *DRE = dyn_cast<DeclRefExpr>(CurVarExpr)) {
     if (isa<VarDecl, NonTypeTemplateParmDecl>(
             DRE->getFoundDecl()->getCanonicalDecl()))
-      return VarExpr;
+      return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
   }
 
   if (const auto *ME = dyn_cast<MemberExpr>(CurVarExpr)) {
     if (isa<FieldDecl>(ME->getMemberDecl()->getCanonicalDecl())) {
-      return VarExpr;
+      return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
     }
   }
 
   // Nothing really we can do here, as these are dependent.  So just return they
   // are valid.
   if (isa<DependentScopeDeclRefExpr, CXXDependentScopeMemberExpr>(CurVarExpr))
-    return VarExpr;
+    return WasParsingInvalidCacheRef ? ExprEmpty() : VarExpr;
 
   // There isn't really anything we can do in the case of a recovery expr, so
   // skip the diagnostic rather than produce a confusing diagnostic.
@@ -562,6 +585,45 @@ ExprResult SemaOpenACC::ActOnCacheVar(Expr *VarExpr) {
   Diag(VarExpr->getExprLoc(), diag::err_acc_not_a_var_ref_cache);
   return ExprError();
 }
+
+void SemaOpenACC::CheckDeclReference(SourceLocation Loc, Expr *E, Decl *D) {
+  if (!getLangOpts().OpenACC || !CacheInfo.ParsingCacheVarList || !D ||
+      D->isInvalidDecl())
+    return;
+  // A 'cache' variable reference MUST be declared before the 'acc.loop' we
+  // generate in codegen, so we have to mark it invalid here in some way.  We do
+  // so in a bit of a convoluted way as there is no good way to put this into
+  // the AST, so we store it in SemaOpenACC State.  We can check the Scope
+  // during parsing to make sure there is a 'loop' before the decl is
+  // declared(and skip during instantiation).
+  // We only diagnose this as a warning, as this isn't required by the standard
+  // (unless you take a VERY awkward reading of some awkward prose).
+
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // if we are at TU level, we are either doing some EXTRA wacky, or are in a
+  // template instantiation, so just give up.
+  if (CurScope->getDepth() == 0)
+    return;
+
+  while (CurScope) {
+    // If we run into a loop construct scope, than this is 'correct' in that the
+    // declaration is outside of the loop.
+    if (CurScope->isOpenACCLoopConstructScope())
+      return;
+
+    if (CurScope->isDeclScope(D)) {
+      Diag(Loc, diag::warn_acc_cache_var_not_outside_loop);
+
+      CacheInfo.IsInvalidCacheRef = true;
+    }
+
+    CurScope = CurScope->getParent();
+  }
+  // If we don't find the decl at all, we assume that it must be outside of the
+  // loop (or we aren't in a loop!) so skip the diagnostic.
+}
+
 ExprResult SemaOpenACC::ActOnVar(OpenACCDirectiveKind DK, OpenACCClauseKind CK,
                                  Expr *VarExpr) {
   // This has unique enough restrictions that we should split it to a separate
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 8ee27694b737..3f90fe8ff843 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -231,8 +231,8 @@ class SemaOpenACCClauseVisitor {
     case OpenACCClauseKind::PCopyOut:
     case OpenACCClauseKind::PresentOrCopyOut:
       // COPYOUT: Capture only struct.data & compute
-      return Check(OpenACCModifierKind::Always | OpenACCModifierKind::AlwaysIn |
-                   OpenACCModifierKind::Zero |
+      return Check(OpenACCModifierKind::Always |
+                   OpenACCModifierKind::AlwaysOut | OpenACCModifierKind::Zero |
                    (IsStructuredDataOrCompute ? OpenACCModifierKind::Capture
                                               : OpenACCModifierKind::Invalid));
     case OpenACCClauseKind::Create:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 0022f18444cc..a989d4b33c04 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15952,9 +15952,6 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_final:
     Res = ActOnOpenMPFinalClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
-  case OMPC_num_threads:
-    Res = ActOnOpenMPNumThreadsClause(Expr, StartLoc, LParenLoc, EndLoc);
-    break;
   case OMPC_safelen:
     Res = ActOnOpenMPSafelenClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -16008,6 +16005,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
     break;
   case OMPC_grainsize:
   case OMPC_num_tasks:
+  case OMPC_num_threads:
   case OMPC_device:
   case OMPC_if:
   case OMPC_default:
@@ -16354,10 +16352,41 @@ isNonNegativeIntegerValue(Expr *&ValExpr, Sema &SemaRef, OpenMPClauseKind CKind,
   return true;
 }
 
-OMPClause *SemaOpenMP::ActOnOpenMPNumThreadsClause(Expr *NumThreads,
-                                                   SourceLocation StartLoc,
-                                                   SourceLocation LParenLoc,
-                                                   SourceLocation EndLoc) {
+static std::string getListOfPossibleValues(OpenMPClauseKind K, unsigned First,
+                                           unsigned Last,
+                                           ArrayRef<unsigned> Exclude = {}) {
+  SmallString<256> Buffer;
+  llvm::raw_svector_ostream Out(Buffer);
+  unsigned Skipped = Exclude.size();
+  for (unsigned I = First; I < Last; ++I) {
+    if (llvm::is_contained(Exclude, I)) {
+      --Skipped;
+      continue;
+    }
+    Out << "'" << getOpenMPSimpleClauseTypeName(K, I) << "'";
+    if (I + Skipped + 2 == Last)
+      Out << " or ";
+    else if (I + Skipped + 1 != Last)
+      Out << ", ";
+  }
+  return std::string(Out.str());
+}
+
+OMPClause *SemaOpenMP::ActOnOpenMPNumThreadsClause(
+    OpenMPNumThreadsClauseModifier Modifier, Expr *NumThreads,
+    SourceLocation StartLoc, SourceLocation LParenLoc,
+    SourceLocation ModifierLoc, SourceLocation EndLoc) {
+  assert((ModifierLoc.isInvalid() || getLangOpts().OpenMP >= 60) &&
+         "Unexpected num_threads modifier in OpenMP < 60.");
+
+  if (ModifierLoc.isValid() && Modifier == OMPC_NUMTHREADS_unknown) {
+    std::string Values = getListOfPossibleValues(OMPC_num_threads, /*First=*/0,
+                                                 OMPC_NUMTHREADS_unknown);
+    Diag(ModifierLoc, diag::err_omp_unexpected_clause_value)
+        << Values << getOpenMPClauseNameForDiag(OMPC_num_threads);
+    return nullptr;
+  }
+
   Expr *ValExpr = NumThreads;
   Stmt *HelperValStmt = nullptr;
 
@@ -16378,8 +16407,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPNumThreadsClause(Expr *NumThreads,
     HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (getASTContext()) OMPNumThreadsClause(
-      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext())
+      OMPNumThreadsClause(Modifier, ValExpr, HelperValStmt, CaptureRegion,
+                          StartLoc, LParenLoc, ModifierLoc, EndLoc);
 }
 
 ExprResult SemaOpenMP::VerifyPositiveIntegerConstantInClause(
@@ -16744,26 +16774,6 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   return Res;
 }
 
-static std::string getListOfPossibleValues(OpenMPClauseKind K, unsigned First,
-                                           unsigned Last,
-                                           ArrayRef<unsigned> Exclude = {}) {
-  SmallString<256> Buffer;
-  llvm::raw_svector_ostream Out(Buffer);
-  unsigned Skipped = Exclude.size();
-  for (unsigned I = First; I < Last; ++I) {
-    if (llvm::is_contained(Exclude, I)) {
-      --Skipped;
-      continue;
-    }
-    Out << "'" << getOpenMPSimpleClauseTypeName(K, I) << "'";
-    if (I + Skipped + 2 == Last)
-      Out << " or ";
-    else if (I + Skipped + 1 != Last)
-      Out << ", ";
-  }
-  return std::string(Out.str());
-}
-
 OMPClause *SemaOpenMP::ActOnOpenMPDefaultClause(DefaultKind Kind,
                                                 SourceLocation KindKwLoc,
                                                 SourceLocation StartLoc,
@@ -17136,8 +17146,14 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
         static_cast<OpenMPNumTasksClauseModifier>(Argument.back()), Expr,
         StartLoc, LParenLoc, ArgumentLoc.back(), EndLoc);
     break;
-  case OMPC_final:
   case OMPC_num_threads:
+    assert(Argument.size() == 1 && ArgumentLoc.size() == 1 &&
+           "Modifier for num_threads clause and its location are expected.");
+    Res = ActOnOpenMPNumThreadsClause(
+        static_cast<OpenMPNumThreadsClauseModifier>(Argument.back()), Expr,
+        StartLoc, LParenLoc, ArgumentLoc.back(), EndLoc);
+    break;
+  case OMPC_final:
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 8c5f81f126c7..0bb4c3274d86 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -30,6 +30,8 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
+#include "clang/Sema/SemaAMDGPU.h"
+#include "clang/Sema/SemaARM.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaObjC.h"
 #include "clang/Sema/Template.h"
@@ -2180,8 +2182,8 @@ static bool IsVectorConversion(Sema &S, QualType FromType, QualType ToType,
 
   if (ToType->isSVESizelessBuiltinType() ||
       FromType->isSVESizelessBuiltinType())
-    if (S.Context.areCompatibleSveTypes(FromType, ToType) ||
-        S.Context.areLaxCompatibleSveTypes(FromType, ToType)) {
+    if (S.ARM().areCompatibleSveTypes(FromType, ToType) ||
+        S.ARM().areLaxCompatibleSveTypes(FromType, ToType)) {
       ICK = ICK_SVE_Vector_Conversion;
       return true;
     }
@@ -3612,13 +3614,6 @@ Sema::MemberPointerConversionResult Sema::CheckMemberPointerConversion(
     QualType FromType, const MemberPointerType *ToPtrType, CastKind &Kind,
     CXXCastPath &BasePath, SourceLocation CheckLoc, SourceRange OpRange,
     bool IgnoreBaseAccess, MemberPointerConversionDirection Direction) {
-  const MemberPointerType *FromPtrType = FromType->getAs<MemberPointerType>();
-  if (!FromPtrType) {
-    // This must be a null pointer to member pointer conversion
-    Kind = CK_NullToMemberPointer;
-    return MemberPointerConversionResult::Success;
-  }
-
   // Lock down the inheritance model right now in MS ABI, whether or not the
   // pointee types are the same.
   if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
@@ -3626,6 +3621,13 @@ Sema::MemberPointerConversionResult Sema::CheckMemberPointerConversion(
     (void)isCompleteType(CheckLoc, QualType(ToPtrType, 0));
   }
 
+  const MemberPointerType *FromPtrType = FromType->getAs<MemberPointerType>();
+  if (!FromPtrType) {
+    // This must be a null pointer to member pointer conversion
+    Kind = CK_NullToMemberPointer;
+    return MemberPointerConversionResult::Success;
+  }
+
   // T == T, modulo cv
   if (Direction == MemberPointerConversionDirection::Upcast &&
       !Context.hasSameUnqualifiedType(FromPtrType->getPointeeType(),
@@ -4735,9 +4737,9 @@ CompareStandardConversionSequences(Sema &S, SourceLocation Loc,
   if (SCS1.Second == ICK_SVE_Vector_Conversion &&
       SCS2.Second == ICK_SVE_Vector_Conversion) {
     bool SCS1IsCompatibleSVEVectorConversion =
-        S.Context.areCompatibleSveTypes(SCS1.getFromType(), SCS1.getToType(2));
+        S.ARM().areCompatibleSveTypes(SCS1.getFromType(), SCS1.getToType(2));
     bool SCS2IsCompatibleSVEVectorConversion =
-        S.Context.areCompatibleSveTypes(SCS2.getFromType(), SCS2.getToType(2));
+        S.ARM().areCompatibleSveTypes(SCS2.getFromType(), SCS2.getToType(2));
 
     if (SCS1IsCompatibleSVEVectorConversion !=
         SCS2IsCompatibleSVEVectorConversion)
@@ -6176,12 +6178,13 @@ TryContextuallyConvertToBool(Sema &S, Expr *From) {
 ExprResult Sema::PerformContextuallyConvertToBool(Expr *From) {
   if (checkPlaceholderForOverload(*this, From))
     return ExprError();
+  if (From->getType() == Context.AMDGPUFeaturePredicateTy)
+    return AMDGPU().ExpandAMDGPUPredicateBI(dyn_cast<CallExpr>(From));
 
   ImplicitConversionSequence ICS = TryContextuallyConvertToBool(*this, From);
   if (!ICS.isBad())
     return PerformImplicitConversion(From, Context.BoolTy, ICS,
                                      AssignmentAction::Converting);
-
   if (!DiagnoseMultipleUserDefinedConversion(From, Context.BoolTy))
     return Diag(From->getBeginLoc(), diag::err_typecheck_bool_condition)
            << From->getType() << From->getSourceRange();
@@ -12015,6 +12018,16 @@ static void DiagnoseBadConversion(Sema &S, OverloadCandidate *Cand,
   if (TakingCandidateAddress && !checkAddressOfCandidateIsAvailable(S, Fn))
     return;
 
+  // __amdgpu_feature_predicate_t can be explicitly cast to the logical op type,
+  // although this is almost always an error and we advise against it.
+  if (FromTy == S.Context.AMDGPUFeaturePredicateTy &&
+      ToTy == S.Context.getLogicalOperationType()) {
+    S.Diag(Conv.Bad.FromExpr->getExprLoc(),
+           diag::err_amdgcn_predicate_type_needs_explicit_bool_cast)
+        << Conv.Bad.FromExpr << ToTy;
+    return;
+  }
+
   // Emit the generic diagnostic and, optionally, add the hints to it.
   PartialDiagnostic FDiag = S.PDiag(diag::note_ovl_candidate_bad_conv);
   FDiag << (unsigned)FnKindPair.first << (unsigned)FnKindPair.second << FnDesc
diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp
index 9b4d82745f88..d5c83aedb300 100644
--- a/clang/lib/Sema/SemaPPC.cpp
+++ b/clang/lib/Sema/SemaPPC.cpp
@@ -106,6 +106,10 @@ bool SemaPPC::CheckPPCBuiltinFunctionCall(const TargetInfo &TI,
   switch (BuiltinID) {
   default:
     return false;
+  case PPC::BI__builtin_ppc_national2packed:
+  case PPC::BI__builtin_ppc_packed2zoned:
+  case PPC::BI__builtin_ppc_zoned2packed:
+    return SemaRef.BuiltinConstantArgRange(TheCall, 1, 0, 1);
   case PPC::BI__builtin_altivec_crypto_vshasigmaw:
   case PPC::BI__builtin_altivec_crypto_vshasigmad:
     return SemaRef.BuiltinConstantArgRange(TheCall, 1, 0, 1) ||
diff --git a/clang/lib/Sema/SemaRISCV.cpp b/clang/lib/Sema/SemaRISCV.cpp
index 9eab0c2a0df6..cc110e1115ed 100644
--- a/clang/lib/Sema/SemaRISCV.cpp
+++ b/clang/lib/Sema/SemaRISCV.cpp
@@ -1435,11 +1435,24 @@ void SemaRISCV::checkRVVTypeSupport(QualType Ty, SourceLocation Loc, Decl *D,
            !FeatureMap.lookup("zve64x"))
     Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve64x";
   else if (Info.ElementType->isFloat16Type() && !FeatureMap.lookup("zvfh") &&
-           !FeatureMap.lookup("zvfhmin"))
-    Diag(Loc, diag::err_riscv_type_requires_extension, D)
-        << Ty << "zvfh or zvfhmin";
-  else if (Info.ElementType->isBFloat16Type() && !FeatureMap.lookup("zvfbfmin"))
-    Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zvfbfmin";
+           !FeatureMap.lookup("zvfhmin") &&
+           !FeatureMap.lookup("xandesvpackfph"))
+    if (DeclareAndesVectorBuiltins) {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D)
+          << Ty << "zvfh, zvfhmin or xandesvpackfph";
+    } else {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D)
+          << Ty << "zvfh or zvfhmin";
+    }
+  else if (Info.ElementType->isBFloat16Type() &&
+           !FeatureMap.lookup("zvfbfmin") &&
+           !FeatureMap.lookup("xandesvbfhcvt"))
+    if (DeclareAndesVectorBuiltins) {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D)
+          << Ty << "zvfbfmin or xandesvbfhcvt";
+    } else {
+      Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zvfbfmin";
+    }
   else if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Float) &&
            !FeatureMap.lookup("zve32f"))
     Diag(Loc, diag::err_riscv_type_requires_extension, D) << Ty << "zve32f";
diff --git a/clang/lib/Sema/SemaSPIRV.cpp b/clang/lib/Sema/SemaSPIRV.cpp
index c27d3fed2b99..76d3cff908b3 100644
--- a/clang/lib/Sema/SemaSPIRV.cpp
+++ b/clang/lib/Sema/SemaSPIRV.cpp
@@ -116,7 +116,7 @@ static bool checkGenericCastToPtr(Sema &SemaRef, CallExpr *Call) {
   RT = RT->getPointeeType();
   auto Qual = RT.getQualifiers();
   LangAS AddrSpace;
-  switch (static_cast<spirv::StorageClass>(StorageClass)) {
+  switch (StorageClass) {
   case spirv::StorageClass::CrossWorkgroup:
     AddrSpace =
         SemaRef.LangOpts.isSYCL() ? LangAS::sycl_global : LangAS::opencl_global;
diff --git a/clang/lib/Sema/SemaSwift.cpp b/clang/lib/Sema/SemaSwift.cpp
index 4aae855a24b8..4000beff7dc4 100644
--- a/clang/lib/Sema/SemaSwift.cpp
+++ b/clang/lib/Sema/SemaSwift.cpp
@@ -72,6 +72,15 @@ static bool isValidSwiftErrorResultType(QualType Ty) {
   return isValidSwiftContextType(Ty);
 }
 
+static bool isValidSwiftContextName(StringRef ContextName) {
+  // ContextName might be qualified, e.g. 'MyNamespace.MyStruct'.
+  SmallVector<StringRef, 1> ContextNameComponents;
+  ContextName.split(ContextNameComponents, '.');
+  return all_of(ContextNameComponents, [&](StringRef Component) {
+    return isValidAsciiIdentifier(Component);
+  });
+}
+
 void SemaSwift::handleAttrAttr(Decl *D, const ParsedAttr &AL) {
   if (AL.isInvalid() || AL.isUsedAsTypeAttr())
     return;
@@ -356,11 +365,11 @@ static bool validateSwiftFunctionName(Sema &S, const ParsedAttr &AL,
 
   // Split at the first '.', if it exists, which separates the context name
   // from the base name.
-  std::tie(ContextName, BaseName) = BaseName.split('.');
+  std::tie(ContextName, BaseName) = BaseName.rsplit('.');
   if (BaseName.empty()) {
     BaseName = ContextName;
     ContextName = StringRef();
-  } else if (ContextName.empty() || !isValidAsciiIdentifier(ContextName)) {
+  } else if (ContextName.empty() || !isValidSwiftContextName(ContextName)) {
     S.Diag(Loc, diag::warn_attr_swift_name_invalid_identifier)
         << AL << /*context*/ 1;
     return false;
@@ -584,11 +593,11 @@ bool SemaSwift::DiagnoseName(Decl *D, StringRef Name, SourceLocation Loc,
              !IsAsync) {
     StringRef ContextName, BaseName;
 
-    std::tie(ContextName, BaseName) = Name.split('.');
+    std::tie(ContextName, BaseName) = Name.rsplit('.');
     if (BaseName.empty()) {
       BaseName = ContextName;
       ContextName = StringRef();
-    } else if (!isValidAsciiIdentifier(ContextName)) {
+    } else if (!isValidSwiftContextName(ContextName)) {
       Diag(Loc, diag::warn_attr_swift_name_invalid_identifier)
           << AL << /*context*/ 1;
       return false;
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index ccc532e1fd3d..1a98b3583185 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -1773,9 +1773,9 @@ Sema::ActOnTemplateParameterList(unsigned Depth,
   for (NamedDecl *P : Params)
     warnOnReservedIdentifier(P);
 
-  return TemplateParameterList::Create(
-      Context, TemplateLoc, LAngleLoc,
-      llvm::ArrayRef(Params.data(), Params.size()), RAngleLoc, RequiresClause);
+  return TemplateParameterList::Create(Context, TemplateLoc, LAngleLoc,
+                                       llvm::ArrayRef(Params), RAngleLoc,
+                                       RequiresClause);
 }
 
 static void SetNestedNameSpecifier(Sema &S, TagDecl *T,
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 785d7b89e778..88aa63d3b9e8 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -8522,8 +8522,8 @@ static void HandleRISCVRVVVectorBitsTypeAttr(QualType &CurType,
     return;
   }
 
-  auto VScale =
-      S.Context.getTargetInfo().getVScaleRange(S.getLangOpts(), false);
+  auto VScale = S.Context.getTargetInfo().getVScaleRange(
+      S.getLangOpts(), TargetInfo::ArmStreamingKind::NotStreaming);
   if (!VScale || !VScale->first || VScale->first != VScale->second) {
     S.Diag(Attr.getLoc(), diag::err_attribute_riscv_rvv_bits_unsupported)
         << Attr;
diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp
index 4dbb2450857e..c2f0600295e9 100644
--- a/clang/lib/Sema/SemaTypeTraits.cpp
+++ b/clang/lib/Sema/SemaTypeTraits.cpp
@@ -121,7 +121,7 @@ static bool hasSuitableConstructorForRelocation(Sema &SemaRef,
 
   CXXMethodDecl *Decl =
       LookupSpecialMemberFromXValue(SemaRef, D, /*Assign=*/false);
-  return Decl && Decl->isUserProvided() == AllowUserDefined &&
+  return Decl && (AllowUserDefined || !Decl->isUserProvided()) &&
          !Decl->isDeleted();
 }
 
@@ -137,7 +137,7 @@ static bool hasSuitableMoveAssignmentOperatorForRelocation(
   if (!Decl)
     return false;
 
-  return Decl && Decl->isUserProvided() == AllowUserDefined &&
+  return Decl && (AllowUserDefined || !Decl->isUserProvided()) &&
          !Decl->isDeleted();
 }
 
@@ -1579,6 +1579,58 @@ ExprResult Sema::ActOnTypeTrait(TypeTrait Kind, SourceLocation KWLoc,
   return BuildTypeTrait(Kind, KWLoc, ConvertedArgs, RParenLoc);
 }
 
+bool Sema::BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT,
+                           QualType RhsT) {
+  // C++0x [meta.rel]p2
+  // Base is a base class of Derived without regard to cv-qualifiers or
+  // Base and Derived are not unions and name the same class type without
+  // regard to cv-qualifiers.
+
+  const RecordType *lhsRecord = LhsT->getAs<RecordType>();
+  const RecordType *rhsRecord = RhsT->getAs<RecordType>();
+  if (!rhsRecord || !lhsRecord) {
+    const ObjCObjectType *LHSObjTy = LhsT->getAs<ObjCObjectType>();
+    const ObjCObjectType *RHSObjTy = RhsT->getAs<ObjCObjectType>();
+    if (!LHSObjTy || !RHSObjTy)
+      return false;
+
+    ObjCInterfaceDecl *BaseInterface = LHSObjTy->getInterface();
+    ObjCInterfaceDecl *DerivedInterface = RHSObjTy->getInterface();
+    if (!BaseInterface || !DerivedInterface)
+      return false;
+
+    if (RequireCompleteType(RhsTLoc, RhsT,
+                            diag::err_incomplete_type_used_in_type_trait_expr))
+      return false;
+
+    return BaseInterface->isSuperClassOf(DerivedInterface);
+  }
+
+  assert(Context.hasSameUnqualifiedType(LhsT, RhsT) ==
+         (lhsRecord == rhsRecord));
+
+  // Unions are never base classes, and never have base classes.
+  // It doesn't matter if they are complete or not. See PR#41843
+  if (lhsRecord && lhsRecord->getDecl()->isUnion())
+    return false;
+  if (rhsRecord && rhsRecord->getDecl()->isUnion())
+    return false;
+
+  if (lhsRecord == rhsRecord)
+    return true;
+
+  // C++0x [meta.rel]p2:
+  //   If Base and Derived are class types and are different types
+  //   (ignoring possible cv-qualifiers) then Derived shall be a
+  //   complete type.
+  if (RequireCompleteType(RhsTLoc, RhsT,
+                          diag::err_incomplete_type_used_in_type_trait_expr))
+    return false;
+
+  return cast<CXXRecordDecl>(rhsRecord->getDecl())
+      ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
+}
+
 static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
                                     const TypeSourceInfo *Lhs,
                                     const TypeSourceInfo *Rhs,
@@ -1590,58 +1642,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
          "Cannot evaluate traits of dependent types");
 
   switch (BTT) {
-  case BTT_IsBaseOf: {
-    // C++0x [meta.rel]p2
-    // Base is a base class of Derived without regard to cv-qualifiers or
-    // Base and Derived are not unions and name the same class type without
-    // regard to cv-qualifiers.
-
-    const RecordType *lhsRecord = LhsT->getAs<RecordType>();
-    const RecordType *rhsRecord = RhsT->getAs<RecordType>();
-    if (!rhsRecord || !lhsRecord) {
-      const ObjCObjectType *LHSObjTy = LhsT->getAs<ObjCObjectType>();
-      const ObjCObjectType *RHSObjTy = RhsT->getAs<ObjCObjectType>();
-      if (!LHSObjTy || !RHSObjTy)
-        return false;
-
-      ObjCInterfaceDecl *BaseInterface = LHSObjTy->getInterface();
-      ObjCInterfaceDecl *DerivedInterface = RHSObjTy->getInterface();
-      if (!BaseInterface || !DerivedInterface)
-        return false;
-
-      if (Self.RequireCompleteType(
-              Rhs->getTypeLoc().getBeginLoc(), RhsT,
-              diag::err_incomplete_type_used_in_type_trait_expr))
-        return false;
-
-      return BaseInterface->isSuperClassOf(DerivedInterface);
-    }
-
-    assert(Self.Context.hasSameUnqualifiedType(LhsT, RhsT) ==
-           (lhsRecord == rhsRecord));
-
-    // Unions are never base classes, and never have base classes.
-    // It doesn't matter if they are complete or not. See PR#41843
-    if (lhsRecord && lhsRecord->getDecl()->isUnion())
-      return false;
-    if (rhsRecord && rhsRecord->getDecl()->isUnion())
-      return false;
-
-    if (lhsRecord == rhsRecord)
-      return true;
-
-    // C++0x [meta.rel]p2:
-    //   If Base and Derived are class types and are different types
-    //   (ignoring possible cv-qualifiers) then Derived shall be a
-    //   complete type.
-    if (Self.RequireCompleteType(
-            Rhs->getTypeLoc().getBeginLoc(), RhsT,
-            diag::err_incomplete_type_used_in_type_trait_expr))
-      return false;
+  case BTT_IsBaseOf:
+    return Self.BuiltinIsBaseOf(Rhs->getTypeLoc().getBeginLoc(), LhsT, RhsT);
 
-    return cast<CXXRecordDecl>(rhsRecord->getDecl())
-        ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
-  }
   case BTT_IsVirtualBaseOf: {
     const RecordType *BaseRecord = LhsT->getAs<RecordType>();
     const RecordType *DerivedRecord = RhsT->getAs<RecordType>();
@@ -1725,14 +1728,15 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
 
     // Build expressions that emulate the effect of declval<T>() and
     // declval<U>().
-    if (LhsT->isObjectType() || LhsT->isFunctionType())
-      LhsT = Self.Context.getRValueReferenceType(LhsT);
-    if (RhsT->isObjectType() || RhsT->isFunctionType())
-      RhsT = Self.Context.getRValueReferenceType(RhsT);
-    OpaqueValueExpr Lhs(KeyLoc, LhsT.getNonLValueExprType(Self.Context),
-                        Expr::getValueKindForType(LhsT));
-    OpaqueValueExpr Rhs(KeyLoc, RhsT.getNonLValueExprType(Self.Context),
-                        Expr::getValueKindForType(RhsT));
+    auto createDeclValExpr = [&](QualType Ty) -> OpaqueValueExpr {
+      if (Ty->isObjectType() || Ty->isFunctionType())
+        Ty = Self.Context.getRValueReferenceType(Ty);
+      return {KeyLoc, Ty.getNonLValueExprType(Self.Context),
+              Expr::getValueKindForType(Ty)};
+    };
+
+    auto Lhs = createDeclValExpr(LhsT);
+    auto Rhs = createDeclValExpr(RhsT);
 
     // Attempt the assignment in an unevaluated context within a SFINAE
     // trap at translation unit scope.
@@ -1956,6 +1960,9 @@ static std::optional<TypeTrait> StdNameToTypeTrait(StringRef Name) {
             TypeTrait::UTT_IsCppTriviallyRelocatable)
       .Case("is_replaceable", TypeTrait::UTT_IsReplaceable)
       .Case("is_trivially_copyable", TypeTrait::UTT_IsTriviallyCopyable)
+      .Case("is_assignable", TypeTrait::BTT_IsAssignable)
+      .Case("is_empty", TypeTrait::UTT_IsEmpty)
+      .Case("is_standard_layout", TypeTrait::UTT_IsStandardLayout)
       .Default(std::nullopt);
 }
 
@@ -2285,6 +2292,244 @@ static void DiagnoseNonTriviallyCopyableReason(Sema &SemaRef,
   SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
 }
 
+static void DiagnoseNonAssignableReason(Sema &SemaRef, SourceLocation Loc,
+                                        QualType T, QualType U) {
+  const CXXRecordDecl *D = T->getAsCXXRecordDecl();
+
+  auto createDeclValExpr = [&](QualType Ty) -> OpaqueValueExpr {
+    if (Ty->isObjectType() || Ty->isFunctionType())
+      Ty = SemaRef.Context.getRValueReferenceType(Ty);
+    return {Loc, Ty.getNonLValueExprType(SemaRef.Context),
+            Expr::getValueKindForType(Ty)};
+  };
+
+  auto LHS = createDeclValExpr(T);
+  auto RHS = createDeclValExpr(U);
+
+  EnterExpressionEvaluationContext Unevaluated(
+      SemaRef, Sema::ExpressionEvaluationContext::Unevaluated);
+  Sema::ContextRAII TUContext(SemaRef,
+                              SemaRef.Context.getTranslationUnitDecl());
+  SemaRef.BuildBinOp(/*S=*/nullptr, Loc, BO_Assign, &LHS, &RHS);
+
+  if (!D || D->isInvalidDecl())
+    return;
+
+  SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+}
+
+static void DiagnoseIsEmptyReason(Sema &S, SourceLocation Loc,
+                                  const CXXRecordDecl *D) {
+  // Non-static data members (ignore zero-width bit‐fields).
+  for (const auto *Field : D->fields()) {
+    if (Field->isZeroLengthBitField())
+      continue;
+    if (Field->isBitField()) {
+      S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonZeroLengthField << Field
+          << Field->getSourceRange();
+      continue;
+    }
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NonEmptyMember << Field
+        << Field->getType() << Field->getSourceRange();
+  }
+
+  // Virtual functions.
+  for (const auto *M : D->methods()) {
+    if (M->isVirtual()) {
+      S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VirtualFunction << M
+          << M->getSourceRange();
+      break;
+    }
+  }
+
+  // Virtual bases and non-empty bases.
+  for (const auto &B : D->bases()) {
+    const auto *BR = B.getType()->getAsCXXRecordDecl();
+    if (!BR || BR->isInvalidDecl())
+      continue;
+    if (B.isVirtual()) {
+      S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VBase << B.getType()
+          << B.getSourceRange();
+    }
+    if (!BR->isEmpty()) {
+      S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonEmptyBase << B.getType()
+          << B.getSourceRange();
+    }
+  }
+}
+
+static void DiagnoseIsEmptyReason(Sema &S, SourceLocation Loc, QualType T) {
+  // Emit primary "not empty" diagnostic.
+  S.Diag(Loc, diag::note_unsatisfied_trait) << T << diag::TraitName::Empty;
+
+  // While diagnosing is_empty<T>, we want to look at the actual type, not a
+  // reference or an array of it. So we need to massage the QualType param to
+  // strip refs and arrays.
+  if (T->isReferenceType())
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Ref;
+  T = T.getNonReferenceType();
+
+  if (auto *AT = S.Context.getAsArrayType(T))
+    T = AT->getElementType();
+
+  if (auto *D = T->getAsCXXRecordDecl()) {
+    if (D->hasDefinition()) {
+      DiagnoseIsEmptyReason(S, Loc, D);
+      S.Diag(D->getLocation(), diag::note_defined_here) << D;
+    }
+  }
+}
+
+static bool hasMultipleDataBaseClassesWithFields(const CXXRecordDecl *D) {
+  int NumBasesWithFields = 0;
+  for (const CXXBaseSpecifier &Base : D->bases()) {
+    const CXXRecordDecl *BaseRD = Base.getType()->getAsCXXRecordDecl();
+    if (!BaseRD || BaseRD->isInvalidDecl())
+      continue;
+
+    for (const FieldDecl *Field : BaseRD->fields()) {
+      if (!Field->isUnnamedBitField()) {
+        if (++NumBasesWithFields > 1)
+          return true; // found more than one base class with fields
+        break;         // no need to check further fields in this base class
+      }
+    }
+  }
+  return false;
+}
+
+static void DiagnoseNonStandardLayoutReason(Sema &SemaRef, SourceLocation Loc,
+                                            const CXXRecordDecl *D) {
+  for (const CXXBaseSpecifier &B : D->bases()) {
+    assert(B.getType()->getAsCXXRecordDecl() && "invalid base?");
+    if (B.isVirtual()) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VBase << B.getType()
+          << B.getSourceRange();
+    }
+    if (!B.getType()->isStandardLayoutType()) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonStandardLayoutBase << B.getType()
+          << B.getSourceRange();
+    }
+  }
+  // Check for mixed access specifiers in fields.
+  const FieldDecl *FirstField = nullptr;
+  AccessSpecifier FirstAccess = AS_none;
+
+  for (const FieldDecl *Field : D->fields()) {
+    if (Field->isUnnamedBitField())
+      continue;
+
+    // Record the first field we see
+    if (!FirstField) {
+      FirstField = Field;
+      FirstAccess = Field->getAccess();
+      continue;
+    }
+
+    // Check if the field has a different access specifier than the first one.
+    if (Field->getAccess() != FirstAccess) {
+      // Emit a diagnostic about mixed access specifiers.
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::MixedAccess;
+
+      SemaRef.Diag(FirstField->getLocation(), diag::note_defined_here)
+          << FirstField;
+
+      SemaRef.Diag(Field->getLocation(), diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::MixedAccessField << Field
+          << FirstField;
+
+      // No need to check further fields, as we already found mixed access.
+      break;
+    }
+  }
+  if (hasMultipleDataBaseClassesWithFields(D)) {
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::MultipleDataBase;
+  }
+  if (D->isPolymorphic()) {
+    // Find the best location to point “defined here” at.
+    const CXXMethodDecl *VirtualMD = nullptr;
+    // First, look for a virtual method.
+    for (const auto *M : D->methods()) {
+      if (M->isVirtual()) {
+        VirtualMD = M;
+        break;
+      }
+    }
+    if (VirtualMD) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VirtualFunction << VirtualMD;
+      SemaRef.Diag(VirtualMD->getLocation(), diag::note_defined_here)
+          << VirtualMD;
+    } else {
+      // If no virtual method, point to the record declaration itself.
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::VirtualFunction << D;
+      SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+    }
+  }
+  for (const FieldDecl *Field : D->fields()) {
+    if (!Field->getType()->isStandardLayoutType()) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::NonStandardLayoutMember << Field
+          << Field->getType() << Field->getSourceRange();
+    }
+  }
+  // Find any indirect base classes that have fields.
+  if (D->hasDirectFields()) {
+    const CXXRecordDecl *Indirect = nullptr;
+    D->forallBases([&](const CXXRecordDecl *BaseDef) {
+      if (BaseDef->hasDirectFields()) {
+        Indirect = BaseDef;
+        return false; // stop traversal
+      }
+      return true; // continue to the next base
+    });
+    if (Indirect) {
+      SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+          << diag::TraitNotSatisfiedReason::IndirectBaseWithFields << Indirect
+          << Indirect->getSourceRange();
+    }
+  }
+}
+
+static void DiagnoseNonStandardLayoutReason(Sema &SemaRef, SourceLocation Loc,
+                                            QualType T) {
+  SemaRef.Diag(Loc, diag::note_unsatisfied_trait)
+      << T << diag::TraitName::StandardLayout;
+
+  // Check type-level exclusion first.
+  if (T->isVariablyModifiedType()) {
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::VLA;
+    return;
+  }
+
+  if (T->isReferenceType()) {
+    SemaRef.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Ref;
+    return;
+  }
+  T = T.getNonReferenceType();
+  const CXXRecordDecl *D = T->getAsCXXRecordDecl();
+  if (!D || D->isInvalidDecl())
+    return;
+
+  if (D->hasDefinition())
+    DiagnoseNonStandardLayoutReason(SemaRef, Loc, D);
+
+  SemaRef.Diag(D->getLocation(), diag::note_defined_here) << D;
+}
+
 void Sema::DiagnoseTypeTraitDetails(const Expr *E) {
   E = E->IgnoreParenImpCasts();
   if (E->containsErrors())
@@ -2305,6 +2550,15 @@ void Sema::DiagnoseTypeTraitDetails(const Expr *E) {
   case UTT_IsTriviallyCopyable:
     DiagnoseNonTriviallyCopyableReason(*this, E->getBeginLoc(), Args[0]);
     break;
+  case BTT_IsAssignable:
+    DiagnoseNonAssignableReason(*this, E->getBeginLoc(), Args[0], Args[1]);
+    break;
+  case UTT_IsEmpty:
+    DiagnoseIsEmptyReason(*this, E->getBeginLoc(), Args[0]);
+    break;
+  case UTT_IsStandardLayout:
+    DiagnoseNonStandardLayoutReason(*this, E->getBeginLoc(), Args[0]);
+    break;
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 26bee7a96de2..758012f894a4 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1714,12 +1714,14 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new OpenMP clause.
   /// Subclasses may override this routine to provide different behavior.
-  OMPClause *RebuildOMPNumThreadsClause(Expr *NumThreads,
+  OMPClause *RebuildOMPNumThreadsClause(OpenMPNumThreadsClauseModifier Modifier,
+                                        Expr *NumThreads,
                                         SourceLocation StartLoc,
                                         SourceLocation LParenLoc,
+                                        SourceLocation ModifierLoc,
                                         SourceLocation EndLoc) {
-    return getSema().OpenMP().ActOnOpenMPNumThreadsClause(NumThreads, StartLoc,
-                                                          LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNumThreadsClause(
+        Modifier, NumThreads, StartLoc, LParenLoc, ModifierLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'safelen' clause.
@@ -4559,6 +4561,13 @@ bool TreeTransform<Derived>::TransformExprs(Expr *const *Inputs,
 template <typename Derived>
 Sema::ConditionResult TreeTransform<Derived>::TransformCondition(
     SourceLocation Loc, VarDecl *Var, Expr *Expr, Sema::ConditionKind Kind) {
+
+  EnterExpressionEvaluationContext Eval(
+      SemaRef, Sema::ExpressionEvaluationContext::ConstantEvaluated,
+      /*LambdaContextDecl=*/nullptr,
+      /*ExprContext=*/Sema::ExpressionEvaluationContextRecord::EK_Other,
+      /*ShouldEnter=*/Kind == Sema::ConditionKind::ConstexprIf);
+
   if (Var) {
     VarDecl *ConditionVar = cast_or_null<VarDecl>(
         getDerived().TransformDefinition(Var->getLocation(), Var));
@@ -10461,7 +10470,8 @@ TreeTransform<Derived>::TransformOMPNumThreadsClause(OMPNumThreadsClause *C) {
   if (NumThreads.isInvalid())
     return nullptr;
   return getDerived().RebuildOMPNumThreadsClause(
-      NumThreads.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+      C->getModifier(), NumThreads.get(), C->getBeginLoc(), C->getLParenLoc(),
+      C->getModifierLoc(), C->getEndLoc());
 }
 
 template <typename Derived>
@@ -15720,13 +15730,9 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
 
   // FIXME: Sema's lambda-building mechanism expects us to push an expression
   // evaluation context even if we're not transforming the function body.
-  getSema().PushExpressionEvaluationContext(
-      E->getCallOperator()->isConsteval() ?
-      Sema::ExpressionEvaluationContext::ImmediateFunctionContext :
-      Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
-  getSema().currentEvaluationContext().InImmediateEscalatingFunctionContext =
-      getSema().getLangOpts().CPlusPlus20 &&
-      E->getCallOperator()->isImmediateEscalating();
+  getSema().PushExpressionEvaluationContextForFunction(
+      Sema::ExpressionEvaluationContext::PotentiallyEvaluated,
+      E->getCallOperator());
 
   Sema::CodeSynthesisContext C;
   C.Kind = clang::Sema::CodeSynthesisContext::LambdaExpressionSubstitution;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 6f082fe840b4..cc2d2ea86ccb 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1914,10 +1914,9 @@ bool ASTReader::ReadSLocEntry(int ID) {
   }
 
   case SM_SLOC_EXPANSION_ENTRY: {
-    LocSeq::State Seq;
-    SourceLocation SpellingLoc = ReadSourceLocation(*F, Record[1], Seq);
-    SourceLocation ExpansionBegin = ReadSourceLocation(*F, Record[2], Seq);
-    SourceLocation ExpansionEnd = ReadSourceLocation(*F, Record[3], Seq);
+    SourceLocation SpellingLoc = ReadSourceLocation(*F, Record[1]);
+    SourceLocation ExpansionBegin = ReadSourceLocation(*F, Record[2]);
+    SourceLocation ExpansionEnd = ReadSourceLocation(*F, Record[3]);
     SourceMgr.createExpansionLoc(SpellingLoc, ExpansionBegin, ExpansionEnd,
                                  Record[5], Record[4], ID,
                                  BaseOffset + Record[0]);
@@ -7072,13 +7071,10 @@ QualType ASTReader::readTypeRecord(TypeID ID) {
 namespace clang {
 
 class TypeLocReader : public TypeLocVisitor<TypeLocReader> {
-  using LocSeq = SourceLocationSequence;
-
   ASTRecordReader &Reader;
-  LocSeq *Seq;
 
-  SourceLocation readSourceLocation() { return Reader.readSourceLocation(Seq); }
-  SourceRange readSourceRange() { return Reader.readSourceRange(Seq); }
+  SourceLocation readSourceLocation() { return Reader.readSourceLocation(); }
+  SourceRange readSourceRange() { return Reader.readSourceRange(); }
 
   TypeSourceInfo *GetTypeSourceInfo() {
     return Reader.readTypeSourceInfo();
@@ -7093,8 +7089,7 @@ class TypeLocReader : public TypeLocVisitor<TypeLocReader> {
   }
 
 public:
-  TypeLocReader(ASTRecordReader &Reader, LocSeq *Seq)
-      : Reader(Reader), Seq(Seq) {}
+  TypeLocReader(ASTRecordReader &Reader) : Reader(Reader) {}
 
   // We want compile-time assurance that we've enumerated all of
   // these, so unfortunately we have to declare them first, then
@@ -7458,9 +7453,8 @@ void TypeLocReader::VisitDependentBitIntTypeLoc(
   TL.setNameLoc(readSourceLocation());
 }
 
-void ASTRecordReader::readTypeLoc(TypeLoc TL, LocSeq *ParentSeq) {
-  LocSeq::State Seq(ParentSeq);
-  TypeLocReader TLR(*this, Seq);
+void ASTRecordReader::readTypeLoc(TypeLoc TL) {
+  TypeLocReader TLR(*this);
   for (; !TL.isNull(); TL = TL.getNextTypeLoc())
     TLR.Visit(TL);
 }
@@ -8381,6 +8375,15 @@ bool ASTReader::LoadExternalSpecializationsImpl(
   if (It == SpecLookups.end())
     return false;
 
+  llvm::TimeTraceScope TimeScope("Load External Specializations for ", [&] {
+    std::string Name;
+    llvm::raw_string_ostream OS(Name);
+    auto *ND = cast<NamedDecl>(D);
+    ND->getNameForDiagnostic(OS, ND->getASTContext().getPrintingPolicy(),
+                             /*Qualified=*/true);
+    return Name;
+  });
+
   Deserializing LookupResults(this);
   auto HashValue = StableHashForTemplateArguments(TemplateArgs);
 
@@ -8561,14 +8564,22 @@ bool ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
     Find(It->second.Table, Name);
   }
 
-  if (auto *NamedModule =
-          OriginalDC ? cast<Decl>(OriginalDC)->getTopLevelOwningNamedModule()
-                     : nullptr) {
+  auto FindModuleLocalLookup = [&, this](Module *NamedModule) {
     if (auto It = ModuleLocalLookups.find(DC); It != ModuleLocalLookups.end()) {
       ++NumModuleLocalVisibleDeclContexts;
       Find(It->second.Table, std::make_pair(Name, NamedModule));
     }
-  }
+  };
+  if (auto *NamedModule =
+          OriginalDC ? cast<Decl>(OriginalDC)->getTopLevelOwningNamedModule()
+                     : nullptr)
+    FindModuleLocalLookup(NamedModule);
+  // See clang/test/Modules/ModulesLocalNamespace.cppm for the motiviation case.
+  // We're going to find a decl but the decl context of the lookup is
+  // unspecified. In this case, the OriginalDC may be the decl context in other
+  // module.
+  if (ContextObj && ContextObj->getCurrentNamedModule())
+    FindModuleLocalLookup(ContextObj->getCurrentNamedModule());
 
   if (auto It = TULocalLookups.find(DC); It != TULocalLookups.end()) {
     ++NumTULocalVisibleDeclContexts;
@@ -9452,8 +9463,7 @@ void ASTReader::AssignedLambdaNumbering(CXXRecordDecl *Lambda) {
 
   // Keep track of this lambda so it can be merged with another lambda that
   // is loaded later.
-  LambdaDeclarationsForMerging.insert(
-      {LambdaInfo, const_cast<CXXRecordDecl *>(Lambda)});
+  LambdaDeclarationsForMerging.insert({LambdaInfo, Lambda});
 }
 
 void ASTReader::LoadSelector(Selector Sel) {
@@ -9672,7 +9682,7 @@ ModuleFile *ASTReader::getLocalModuleFile(ModuleFile &M, unsigned ID) const {
     // It's a prefix (preamble, PCH, ...). Look it up by index.
    int IndexFromEnd = static_cast<int>(ID >> 1);
     assert(IndexFromEnd && "got reference to unknown module file");
-    return getModuleManager().pch_modules().end()[-static_cast<int>(IndexFromEnd)];
+    return getModuleManager().pch_modules().end()[-IndexFromEnd];
   }
 }
 
@@ -9722,10 +9732,6 @@ bool ASTReader::wasThisDeclarationADefinition(const FunctionDecl *FD) {
   return ThisDeclarationWasADefinitionSet.contains(FD);
 }
 
-bool ASTReader::hasInitializerWithSideEffects(const VarDecl *VD) const {
-  return InitSideEffectVars.count(VD);
-}
-
 Selector ASTReader::getLocalSelector(ModuleFile &M, unsigned LocalID) {
   return DecodeSelector(getGlobalSelectorID(M, LocalID));
 }
@@ -10016,9 +10022,9 @@ ASTRecordReader::readNestedNameSpecifierLoc() {
 }
 
 SourceRange ASTReader::ReadSourceRange(ModuleFile &F, const RecordData &Record,
-                                       unsigned &Idx, LocSeq *Seq) {
-  SourceLocation beg = ReadSourceLocation(F, Record, Idx, Seq);
-  SourceLocation end = ReadSourceLocation(F, Record, Idx, Seq);
+                                       unsigned &Idx) {
+  SourceLocation beg = ReadSourceLocation(F, Record, Idx);
+  SourceLocation end = ReadSourceLocation(F, Record, Idx);
   return SourceRange(beg, end);
 }
 
@@ -10196,8 +10202,7 @@ void ASTReader::ReadComments() {
     for (RawComment *C : Comments) {
       SourceLocation CommentLoc = C->getBeginLoc();
       if (CommentLoc.isValid()) {
-        std::pair<FileID, unsigned> Loc =
-            SourceMgr.getDecomposedLoc(CommentLoc);
+        FileIDAndOffset Loc = SourceMgr.getDecomposedLoc(CommentLoc);
         if (Loc.first.isValid())
           Context.Comments.OrderedComments[Loc.first].emplace(Loc.second, C);
       }
@@ -11461,7 +11466,9 @@ void OMPClauseReader::VisitOMPFinalClause(OMPFinalClause *C) {
 
 void OMPClauseReader::VisitOMPNumThreadsClause(OMPNumThreadsClause *C) {
   VisitOMPClauseWithPreInit(C);
+  C->setModifier(Record.readEnum<OpenMPNumThreadsClauseModifier>());
   C->setNumThreads(Record.readSubExpr());
+  C->setModifierLoc(Record.readSourceLocation());
   C->setLParenLoc(Record.readSourceLocation());
 }
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 0ffd78424be0..8bde213dc4cb 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1628,9 +1628,6 @@ RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
     VD->NonParmVarDeclBits.PreviousDeclInSameBlockScope =
         VarDeclBits.getNextBit();
 
-    if (VarDeclBits.getNextBit())
-      Reader.InitSideEffectVars.insert(VD);
-
     VD->NonParmVarDeclBits.EscapingByref = VarDeclBits.getNextBit();
     HasDeducedType = VarDeclBits.getNextBit();
     VD->NonParmVarDeclBits.ImplicitParamKind =
@@ -1701,6 +1698,8 @@ void ASTDeclReader::ReadVarDeclInit(VarDecl *VD) {
     Eval->HasConstantInitialization = (Val & 2) != 0;
     Eval->HasConstantDestruction = (Val & 4) != 0;
     Eval->WasEvaluated = (Val & 8) != 0;
+    Eval->HasSideEffects = (Val & 16) != 0;
+    Eval->CheckedForSideEffects = true;
     if (Eval->WasEvaluated) {
       Eval->Evaluated = Record.readAPValue();
       if (Eval->Evaluated.needsCleanup())
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 8945407cf666..0166e493bf03 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -1733,7 +1733,7 @@ void ASTStmtReader::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->CXXOperatorCallExprBits.OperatorKind = Record.readInt();
-  E->Range = Record.readSourceRange();
+  E->BeginLoc = Record.readSourceLocation();
 }
 
 void ASTStmtReader::VisitCXXRewrittenBinaryOperator(
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c6487c5366a2..5086f8f5be63 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -330,19 +330,13 @@ class ASTTypeWriter {
 };
 
 class TypeLocWriter : public TypeLocVisitor<TypeLocWriter> {
-  using LocSeq = SourceLocationSequence;
-
   ASTRecordWriter &Record;
-  LocSeq *Seq;
 
-  void addSourceLocation(SourceLocation Loc) {
-    Record.AddSourceLocation(Loc, Seq);
-  }
-  void addSourceRange(SourceRange Range) { Record.AddSourceRange(Range, Seq); }
+  void addSourceLocation(SourceLocation Loc) { Record.AddSourceLocation(Loc); }
+  void addSourceRange(SourceRange Range) { Record.AddSourceRange(Range); }
 
 public:
-  TypeLocWriter(ASTRecordWriter &Record, LocSeq *Seq)
-      : Record(Record), Seq(Seq) {}
+  TypeLocWriter(ASTRecordWriter &Record) : Record(Record) {}
 
 #define ABSTRACT_TYPELOC(CLASS, PARENT)
 #define TYPELOC(CLASS, PARENT) \
@@ -2449,13 +2443,12 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr) {
       SLocEntryOffsets.push_back(Offset);
       // Starting offset of this entry within this module, so skip the dummy.
       Record.push_back(getAdjustedOffset(SLoc->getOffset()) - 2);
-      LocSeq::State Seq;
-      AddSourceLocation(Expansion.getSpellingLoc(), Record, Seq);
-      AddSourceLocation(Expansion.getExpansionLocStart(), Record, Seq);
+      AddSourceLocation(Expansion.getSpellingLoc(), Record);
+      AddSourceLocation(Expansion.getExpansionLocStart(), Record);
       AddSourceLocation(Expansion.isMacroArgExpansion()
                             ? SourceLocation()
                             : Expansion.getExpansionLocEnd(),
-                        Record, Seq);
+                        Record);
       Record.push_back(Expansion.isExpansionTokenRange());
 
       // Compute the token length for this macro expansion.
@@ -4069,6 +4062,10 @@ void ASTWriter::handleVTable(CXXRecordDecl *RD) {
   PendingEmittingVTables.push_back(RD);
 }
 
+void ASTWriter::addTouchedModuleFile(serialization::ModuleFile *MF) {
+  TouchedModuleFiles.insert(MF);
+}
+
 //===----------------------------------------------------------------------===//
 // DeclContext's Name Lookup Table Serialization
 //===----------------------------------------------------------------------===//
@@ -4113,7 +4110,7 @@ class ASTDeclContextNameLookupTraitBase {
            "have reference to loaded module file but no chain?");
 
     using namespace llvm::support;
-
+    Writer.addTouchedModuleFile(F);
     endian::write<uint32_t>(Out, Writer.getChain()->getModuleFileID(F),
                             llvm::endianness::little);
   }
@@ -4480,6 +4477,7 @@ class LazySpecializationInfoLookupTrait {
            "have reference to loaded module file but no chain?");
 
     using namespace llvm::support;
+    Writer.addTouchedModuleFile(F);
     endian::write<uint32_t>(Out, Writer.getChain()->getModuleFileID(F),
                             llvm::endianness::little);
   }
@@ -6653,8 +6651,8 @@ void ASTWriter::AddFileID(FileID FID, RecordDataImpl &Record) {
 }
 
 SourceLocationEncoding::RawLocEncoding
-ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc, LocSeq *Seq) {
-  unsigned BaseOffset = 0;
+ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
+  SourceLocation::UIntTy BaseOffset = 0;
   unsigned ModuleFileIndex = 0;
 
   // See SourceLocationEncoding.h for the encoding details.
@@ -6672,19 +6670,17 @@ ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc, LocSeq *Seq) {
     assert(&getChain()->getModuleManager()[F->Index] == F);
   }
 
-  return SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex, Seq);
+  return SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
 }
 
-void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record,
-                                  SourceLocationSequence *Seq) {
+void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record) {
   Loc = getAdjustedLocation(Loc);
-  Record.push_back(getRawSourceLocationEncoding(Loc, Seq));
+  Record.push_back(getRawSourceLocationEncoding(Loc));
 }
 
-void ASTWriter::AddSourceRange(SourceRange Range, RecordDataImpl &Record,
-                               SourceLocationSequence *Seq) {
-  AddSourceLocation(Range.getBegin(), Record, Seq);
-  AddSourceLocation(Range.getEnd(), Record, Seq);
+void ASTWriter::AddSourceRange(SourceRange Range, RecordDataImpl &Record) {
+  AddSourceLocation(Range.getBegin(), Record);
+  AddSourceLocation(Range.getEnd(), Record);
 }
 
 void ASTRecordWriter::AddAPFloat(const llvm::APFloat &Value) {
@@ -6804,9 +6800,8 @@ void ASTRecordWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo) {
   AddTypeLoc(TInfo->getTypeLoc());
 }
 
-void ASTRecordWriter::AddTypeLoc(TypeLoc TL, LocSeq *OuterSeq) {
-  LocSeq::State Seq(OuterSeq);
-  TypeLocWriter TLW(*this, Seq);
+void ASTRecordWriter::AddTypeLoc(TypeLoc TL) {
+  TypeLocWriter TLW(*this);
   for (; !TL.isNull(); TL = TL.getNextTypeLoc())
     TLW.Visit(TL);
 }
@@ -6973,9 +6968,7 @@ void ASTWriter::associateDeclWithFile(const Decl *D, LocalDeclID ID) {
   SourceManager &SM = PP->getSourceManager();
   SourceLocation FileLoc = SM.getFileLoc(Loc);
   assert(SM.isLocalSourceLocation(FileLoc));
-  FileID FID;
-  unsigned Offset;
-  std::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
+  auto [FID, Offset] = SM.getDecomposedLoc(FileLoc);
   if (FID.isInvalid())
     return;
   assert(SM.getSLocEntry(FID).isFile());
@@ -7325,6 +7318,10 @@ void ASTRecordWriter::AddVarDeclInit(const VarDecl *VD) {
 
   uint64_t Val = 1;
   if (EvaluatedStmt *ES = VD->getEvaluatedStmt()) {
+    // This may trigger evaluation, so run it first
+    if (VD->hasInitWithSideEffects())
+      Val |= 16;
+    assert(ES->CheckedForSideEffects);
     Val |= (ES->HasConstantInitialization ? 2 : 0);
     Val |= (ES->HasConstantDestruction ? 4 : 0);
     APValue *Evaluated = VD->getEvaluatedValue();
@@ -7802,7 +7799,9 @@ void OMPClauseWriter::VisitOMPFinalClause(OMPFinalClause *C) {
 
 void OMPClauseWriter::VisitOMPNumThreadsClause(OMPNumThreadsClause *C) {
   VisitOMPClauseWithPreInit(C);
+  Record.writeEnum(C->getModifier());
   Record.AddStmt(C->getNumThreads());
+  Record.AddSourceLocation(C->getModifierLoc());
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 2e390dbe79ec..b8a68afc3073 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -23,7 +23,6 @@
 #include "clang/Serialization/ASTRecordWriter.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Support/ErrorHandling.h"
-#include <optional>
 using namespace clang;
 using namespace serialization;
 
@@ -302,7 +301,7 @@ namespace clang {
     }
     MutableArrayRef<FunctionTemplateSpecializationInfo>
     getPartialSpecializations(FunctionTemplateDecl::Common *) {
-      return std::nullopt;
+      return {};
     }
 
     template<typename DeclTy>
@@ -1306,7 +1305,6 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
     VarDeclBits.addBit(D->isConstexpr());
     VarDeclBits.addBit(D->isInitCapture());
     VarDeclBits.addBit(D->isPreviousDeclInSameBlockScope());
-    VarDeclBits.addBit(D->hasInitWithSideEffects());
 
     VarDeclBits.addBit(D->isEscapingByref());
     HasDeducedType = D->getType()->getContainedDeducedType();
@@ -2935,7 +2933,6 @@ void ASTWriter::WriteDeclAbbrevs() {
   // CXXOperatorCallExpr
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Operator Kind
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
   CXXOperatorCallExprAbbrev = Stream.EmitAbbrev(std::move(Abv));
 
   // Abbreviation for EXPR_CXX_MEMBER_CALL
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index a6e320c7f3eb..be9bad9e96cc 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -971,7 +971,7 @@ void ASTStmtWriter::VisitCallExpr(CallExpr *E) {
     Record.push_back(E->getFPFeatures().getAsOpaqueInt());
 
   if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind()) &&
-      E->getStmtClass() == Stmt::CallExprClass)
+      !E->usesMemberSyntax() && E->getStmtClass() == Stmt::CallExprClass)
     AbbrevToUse = Writer.getCallExprAbbrev();
 
   Code = serialization::EXPR_CALL;
@@ -1702,7 +1702,7 @@ void ASTStmtWriter::VisitMSDependentExistsStmt(MSDependentExistsStmt *S) {
 void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
-  Record.AddSourceRange(E->Range);
+  Record.AddSourceLocation(E->BeginLoc);
 
   if (!E->hasStoredFPFeatures() && !static_cast<bool>(E->getADLCallKind()))
     AbbrevToUse = Writer.getCXXOperatorCallExprAbbrev();
diff --git a/clang/lib/Serialization/TemplateArgumentHasher.cpp b/clang/lib/Serialization/TemplateArgumentHasher.cpp
index aa61496d4aa0..c56138e8893c 100644
--- a/clang/lib/Serialization/TemplateArgumentHasher.cpp
+++ b/clang/lib/Serialization/TemplateArgumentHasher.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/Support/TimeProfiler.h"
 
 using namespace clang;
 
@@ -405,6 +406,7 @@ void TemplateArgumentHasher::AddType(const Type *T) {
 
 unsigned clang::serialization::StableHashForTemplateArguments(
     llvm::ArrayRef<TemplateArgument> Args) {
+  llvm::TimeTraceScope TimeScope("Stable Hash for Template Arguments");
   TemplateArgumentHasher Hasher;
   Hasher.AddInteger(Args.size());
   for (TemplateArgument Arg : Args)
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 4d12fdcec1f1..31cb150892a5 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -2223,6 +2223,44 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallEvent &Call,
         Result = lastElement;
     }
 
+    // For bounded method, amountCopied take the minimum of two values,
+    // for ConcatFnKind::strlcat:
+    // amountCopied = min (size - dstLen - 1 , srcLen)
+    // for others:
+    // amountCopied = min (srcLen, size)
+    // So even if we don't know about amountCopied, as long as one of them will
+    // not cause an out-of-bound access, the whole function's operation will not
+    // too, that will avoid invalidating the superRegion of data member in that
+    // situation.
+    bool CouldAccessOutOfBound = true;
+    if (IsBounded && amountCopied.isUnknown()) {
+      auto CouldAccessOutOfBoundForSVal =
+          [&](std::optional<NonLoc> Val) -> bool {
+        if (!Val)
+          return true;
+        return !isFirstBufInBound(C, state, C.getSVal(Dst.Expression),
+                                  Dst.Expression->getType(), *Val,
+                                  C.getASTContext().getSizeType());
+      };
+
+      CouldAccessOutOfBound = CouldAccessOutOfBoundForSVal(strLengthNL);
+
+      if (CouldAccessOutOfBound) {
+        // Get the max number of characters to copy.
+        const Expr *LenExpr = Call.getArgExpr(2);
+        SVal LenVal = state->getSVal(LenExpr, LCtx);
+
+        // Protect against misdeclared strncpy().
+        LenVal = svalBuilder.evalCast(LenVal, sizeTy, LenExpr->getType());
+
+        // Because analyzer doesn't handle expressions like `size -
+        // dstLen - 1` very well, we roughly use `size` for
+        // ConcatFnKind::strlcat here, same with other concat kinds.
+        CouldAccessOutOfBound =
+            CouldAccessOutOfBoundForSVal(LenVal.getAs<NonLoc>());
+      }
+    }
+
     // Invalidate the destination (regular invalidation without pointer-escaping
     // the address of the top-level region). This must happen before we set the
     // C string length because invalidation will clear the length.
@@ -2230,9 +2268,13 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallEvent &Call,
     // can use LazyCompoundVals to copy the source values into the destination.
     // This would probably remove any existing bindings past the end of the
     // string, but that's still an improvement over blank invalidation.
-    state = invalidateDestinationBufferBySize(
-        C, state, Dst.Expression, Call.getCFGElementRef(), *dstRegVal,
-        amountCopied, C.getASTContext().getSizeType());
+    if (CouldAccessOutOfBound)
+      state = invalidateDestinationBufferBySize(
+          C, state, Dst.Expression, Call.getCFGElementRef(), *dstRegVal,
+          amountCopied, C.getASTContext().getSizeType());
+    else
+      state = invalidateDestinationBufferNeverOverflows(
+          C, state, Call.getCFGElementRef(), *dstRegVal);
 
     // Invalidate the source (const-invalidation without const-pointer-escaping
     // the address of the top-level region).
diff --git a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 86476b32309c..23935647a582 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -253,6 +253,8 @@ class FindUninitializedField {
       const RecordDecl *RD = RT->getDecl()->getDefinition();
       assert(RD && "Referred record has no definition");
       for (const auto *I : RD->fields()) {
+        if (I->isUnnamedBitField())
+          continue;
         const FieldRegion *FR = MrMgr.getFieldRegion(I, R);
         FieldChain.push_back(I);
         T = I->getType();
diff --git a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index 2539d3b9b71e..538104d13e69 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -1118,8 +1118,7 @@ void EmptyLocalizationContextChecker::MethodCrawler::VisitObjCMessageExpr(
   // source, so SL should point to the NSLocalizedString macro.
   SourceLocation SL =
       Mgr.getSourceManager().getImmediateMacroCallerLoc(R.getBegin());
-  std::pair<FileID, unsigned> SLInfo =
-      Mgr.getSourceManager().getDecomposedLoc(SL);
+  FileIDAndOffset SLInfo = Mgr.getSourceManager().getDecomposedLoc(SL);
 
   SrcMgr::SLocEntry SE = Mgr.getSourceManager().getSLocEntry(SLInfo.first);
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 35e98a5e2719..a33e61fabc2c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -34,7 +34,7 @@
 //       Depends on NewDeleteChecker.
 //
 //   * MismatchedDeallocatorChecker
-//       Enables checking whether memory is deallocated with the correspending
+//       Enables checking whether memory is deallocated with the corresponding
 //       allocation function in MallocChecker, such as malloc() allocated
 //       regions are only freed by free(), new by delete, new[] by delete[].
 //
@@ -430,8 +430,8 @@ class MallocChecker
   CHECK_FN(checkGMemdup)
   CHECK_FN(checkGMallocN)
   CHECK_FN(checkGMallocN0)
-  CHECK_FN(preGetdelim)
-  CHECK_FN(checkGetdelim)
+  CHECK_FN(preGetDelimOrGetLine)
+  CHECK_FN(checkGetDelimOrGetLine)
   CHECK_FN(checkReallocN)
   CHECK_FN(checkOwnershipAttr)
 
@@ -445,15 +445,16 @@ class MallocChecker
   const CallDescriptionMap<CheckFn> PreFnMap{
       // NOTE: the following CallDescription also matches the C++ standard
       // library function std::getline(); the callback will filter it out.
-      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::preGetdelim},
-      {{CDM::CLibrary, {"getdelim"}, 4}, &MallocChecker::preGetdelim},
+      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::preGetDelimOrGetLine},
+      {{CDM::CLibrary, {"getdelim"}, 4}, &MallocChecker::preGetDelimOrGetLine},
   };
 
   const CallDescriptionMap<CheckFn> PostFnMap{
       // NOTE: the following CallDescription also matches the C++ standard
       // library function std::getline(); the callback will filter it out.
-      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::checkGetdelim},
-      {{CDM::CLibrary, {"getdelim"}, 4}, &MallocChecker::checkGetdelim},
+      {{CDM::CLibrary, {"getline"}, 3}, &MallocChecker::checkGetDelimOrGetLine},
+      {{CDM::CLibrary, {"getdelim"}, 4},
+       &MallocChecker::checkGetDelimOrGetLine},
   };
 
   const CallDescriptionMap<CheckFn> FreeingMemFnMap{
@@ -1371,8 +1372,8 @@ void MallocChecker::checkIfFreeNameIndex(ProgramStateRef State,
   C.addTransition(State);
 }
 
-const Expr *getPlacementNewBufferArg(const CallExpr *CE,
-                                     const FunctionDecl *FD) {
+static const Expr *getPlacementNewBufferArg(const CallExpr *CE,
+                                            const FunctionDecl *FD) {
   // Checking for signature:
   // void* operator new  ( std::size_t count, void* ptr );
   // void* operator new[]( std::size_t count, void* ptr );
@@ -1482,8 +1483,9 @@ static bool isFromStdNamespace(const CallEvent &Call) {
   return FD->isInStdNamespace();
 }
 
-void MallocChecker::preGetdelim(ProgramStateRef State, const CallEvent &Call,
-                                CheckerContext &C) const {
+void MallocChecker::preGetDelimOrGetLine(ProgramStateRef State,
+                                         const CallEvent &Call,
+                                         CheckerContext &C) const {
   // Discard calls to the C++ standard library function std::getline(), which
   // is completely unrelated to the POSIX getline() that we're checking.
   if (isFromStdNamespace(Call))
@@ -1505,8 +1507,9 @@ void MallocChecker::preGetdelim(ProgramStateRef State, const CallEvent &Call,
     C.addTransition(State);
 }
 
-void MallocChecker::checkGetdelim(ProgramStateRef State, const CallEvent &Call,
-                                  CheckerContext &C) const {
+void MallocChecker::checkGetDelimOrGetLine(ProgramStateRef State,
+                                           const CallEvent &Call,
+                                           CheckerContext &C) const {
   // Discard calls to the C++ standard library function std::getline(), which
   // is completely unrelated to the POSIX getline() that we're checking.
   if (isFromStdNamespace(Call))
@@ -1518,14 +1521,19 @@ void MallocChecker::checkGetdelim(ProgramStateRef State, const CallEvent &Call,
   if (!CE)
     return;
 
-  const auto LinePtr =
-      getPointeeVal(Call.getArgSVal(0), State)->getAs<DefinedSVal>();
-  const auto Size =
-      getPointeeVal(Call.getArgSVal(1), State)->getAs<DefinedSVal>();
-  if (!LinePtr || !Size || !LinePtr->getAsRegion())
+  const auto LinePtrOpt = getPointeeVal(Call.getArgSVal(0), State);
+  const auto SizeOpt = getPointeeVal(Call.getArgSVal(1), State);
+  if (!LinePtrOpt || !SizeOpt || LinePtrOpt->isUnknownOrUndef() ||
+      SizeOpt->isUnknownOrUndef())
     return;
 
-  State = setDynamicExtent(State, LinePtr->getAsRegion(), *Size);
+  const auto LinePtr = LinePtrOpt->getAs<DefinedSVal>();
+  const auto Size = SizeOpt->getAs<DefinedSVal>();
+  const MemRegion *LinePtrReg = LinePtr->getAsRegion();
+  if (!LinePtrReg)
+    return;
+
+  State = setDynamicExtent(State, LinePtrReg, *Size);
   C.addTransition(MallocUpdateRefState(C, CE, State,
                                        AllocationFamily(AF_Malloc), *LinePtr));
 }
@@ -1674,17 +1682,15 @@ ProgramStateRef MallocChecker::ProcessZeroAllocCheck(
     const RefState *RS = State->get<RegionState>(Sym);
     if (RS) {
       if (RS->isAllocated())
-        return TrueState->set<RegionState>(Sym,
-                                          RefState::getAllocatedOfSizeZero(RS));
-      else
-        return State;
-    } else {
-      // Case of zero-size realloc. Historically 'realloc(ptr, 0)' is treated as
-      // 'free(ptr)' and the returned value from 'realloc(ptr, 0)' is not
-      // tracked. Add zero-reallocated Sym to the state to catch references
-      // to zero-allocated memory.
-      return TrueState->add<ReallocSizeZeroSymbols>(Sym);
+        return TrueState->set<RegionState>(
+            Sym, RefState::getAllocatedOfSizeZero(RS));
+      return State;
     }
+    // Case of zero-size realloc. Historically 'realloc(ptr, 0)' is treated as
+    // 'free(ptr)' and the returned value from 'realloc(ptr, 0)' is not
+    // tracked. Add zero-reallocated Sym to the state to catch references
+    // to zero-allocated memory.
+    return TrueState->add<ReallocSizeZeroSymbols>(Sym);
   }
 
   // Assume the value is non-zero going forward.
@@ -1882,7 +1888,7 @@ void MallocChecker::reportTaintBug(StringRef Msg, ProgramStateRef State,
                                         "Tainted Memory Allocation",
                                         categories::TaintedData));
     auto R = std::make_unique<PathSensitiveBugReport>(*BT_TaintedAlloc, Msg, N);
-    for (auto TaintedSym : TaintedSyms) {
+    for (const auto *TaintedSym : TaintedSyms) {
       R->markInteresting(TaintedSym);
     }
     C.emitReport(std::move(R));
@@ -2269,11 +2275,12 @@ MallocChecker::FreeMemAux(CheckerContext &C, const Expr *ArgExpr,
       HandleDoubleFree(C, ParentExpr->getSourceRange(), RsBase->isReleased(),
                        SymBase, PreviousRetStatusSymbol);
       return nullptr;
+    }
 
     // If the pointer is allocated or escaped, but we are now trying to free it,
     // check that the call to free is proper.
-    } else if (RsBase->isAllocated() || RsBase->isAllocatedOfSizeZero() ||
-               RsBase->isEscaped()) {
+    if (RsBase->isAllocated() || RsBase->isAllocatedOfSizeZero() ||
+        RsBase->isEscaped()) {
 
       // Check if an expected deallocation function matches the real one.
       bool DeallocMatchesAlloc = RsBase->getAllocationFamily() == Family;
@@ -2849,9 +2856,7 @@ MallocChecker::ReallocMemAux(CheckerContext &C, const CallEvent &Call,
 
   const CallExpr *CE = cast<CallExpr>(Call.getOriginExpr());
 
-  if (SuffixWithN && CE->getNumArgs() < 3)
-    return nullptr;
-  else if (CE->getNumArgs() < 2)
+  if ((SuffixWithN && CE->getNumArgs() < 3) || CE->getNumArgs() < 2)
     return nullptr;
 
   const Expr *arg0Expr = CE->getArg(0);
diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index 79d10d99e11d..e12261603757 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -83,7 +83,7 @@ class UnixAPIMisuseChecker : public Checker<check::PreCall> {
 
   void CheckOpen(CheckerContext &C, const CallEvent &Call) const;
   void CheckOpenAt(CheckerContext &C, const CallEvent &Call) const;
-  void CheckGetDelim(CheckerContext &C, const CallEvent &Call) const;
+  void CheckGetDelimOrGetline(CheckerContext &C, const CallEvent &Call) const;
   void CheckPthreadOnce(CheckerContext &C, const CallEvent &Call) const;
 
   void CheckOpenVariant(CheckerContext &C, const CallEvent &Call,
@@ -128,7 +128,7 @@ ProgramStateRef UnixAPIMisuseChecker::EnsurePtrNotNull(
     const StringRef PtrDescr,
     std::optional<std::reference_wrapper<const BugType>> BT) const {
   const auto Ptr = PtrVal.getAs<DefinedSVal>();
-  if (!Ptr)
+  if (!Ptr || !PtrExpr->getType()->isPointerType())
     return State;
 
   const auto [PtrNotNull, PtrNull] = State->assume(*Ptr);
@@ -177,7 +177,7 @@ void UnixAPIMisuseChecker::checkPreCall(const CallEvent &Call,
     CheckPthreadOnce(C, Call);
 
   else if (is_contained({"getdelim", "getline"}, FName))
-    CheckGetDelim(C, Call);
+    CheckGetDelimOrGetline(C, Call);
 }
 void UnixAPIMisuseChecker::ReportOpenBug(CheckerContext &C,
                                          ProgramStateRef State,
@@ -223,6 +223,10 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C,
   // All calls should at least provide arguments up to the 'flags' parameter.
   unsigned int MinArgCount = FlagsArgIndex + 1;
 
+  // The frontend should issue a warning for this case. Just return.
+  if (Call.getNumArgs() < MinArgCount)
+    return;
+
   // If the flags has O_CREAT set then open/openat() require an additional
   // argument specifying the file mode (permission bits) for the created file.
   unsigned int CreateModeArgIndex = FlagsArgIndex + 1;
@@ -231,11 +235,7 @@ void UnixAPIMisuseChecker::CheckOpenVariant(CheckerContext &C,
   unsigned int MaxArgCount = CreateModeArgIndex + 1;
 
   ProgramStateRef state = C.getState();
-
-  if (Call.getNumArgs() < MinArgCount) {
-    // The frontend should issue a warning for this case. Just return.
-    return;
-  } else if (Call.getNumArgs() == MaxArgCount) {
+  if (Call.getNumArgs() == MaxArgCount) {
     const Expr *Arg = Call.getArgExpr(CreateModeArgIndex);
     QualType QT = Arg->getType();
     if (!QT->isIntegerType()) {
@@ -332,8 +332,12 @@ ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect(
 
   // We have a pointer to a pointer to the buffer, and a pointer to the size.
   // We want what they point at.
-  auto LinePtrSVal = getPointeeVal(LinePtrPtrSVal, State)->getAs<DefinedSVal>();
-  auto NSVal = getPointeeVal(SizePtrSVal, State);
+  const auto LinePtrValOpt = getPointeeVal(LinePtrPtrSVal, State);
+  if (!LinePtrValOpt)
+    return nullptr;
+
+  const auto LinePtrSVal = LinePtrValOpt->getAs<DefinedSVal>();
+  const auto NSVal = getPointeeVal(SizePtrSVal, State);
   if (!LinePtrSVal || !NSVal || NSVal->isUnknown())
     return nullptr;
 
@@ -350,9 +354,16 @@ ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect(
     // If it is defined, and known, its size must be less than or equal to
     // the buffer size.
     auto NDefSVal = NSVal->getAs<DefinedSVal>();
+    if (!NDefSVal)
+      return LinePtrNotNull;
+
     auto &SVB = C.getSValBuilder();
-    auto LineBufSize =
-        getDynamicExtent(LinePtrNotNull, LinePtrSVal->getAsRegion(), SVB);
+
+    const MemRegion *LinePtrRegion = LinePtrSVal->getAsRegion();
+    if (!LinePtrRegion)
+      return LinePtrNotNull;
+
+    auto LineBufSize = getDynamicExtent(LinePtrNotNull, LinePtrRegion, SVB);
     auto LineBufSizeGtN = SVB.evalBinOp(LinePtrNotNull, BO_GE, LineBufSize,
                                         *NDefSVal, SVB.getConditionType())
                               .getAs<DefinedOrUnknownSVal>();
@@ -367,8 +378,11 @@ ProgramStateRef UnixAPIMisuseChecker::EnsureGetdelimBufferAndSizeCorrect(
   return State;
 }
 
-void UnixAPIMisuseChecker::CheckGetDelim(CheckerContext &C,
-                                         const CallEvent &Call) const {
+void UnixAPIMisuseChecker::CheckGetDelimOrGetline(CheckerContext &C,
+                                                  const CallEvent &Call) const {
+  if (Call.getNumArgs() < 2)
+    return;
+
   ProgramStateRef State = C.getState();
 
   // The parameter `n` must not be NULL.
@@ -527,17 +541,15 @@ void UnixAPIPortabilityChecker::CheckCallocZero(CheckerContext &C,
     if (argVal.isUnknownOrUndef()) {
       if (i == 0)
         continue;
-      else
-        return;
+      return;
     }
 
     if (IsZeroByteAllocation(state, argVal, &trueState, &falseState)) {
       if (ReportZeroByteAllocation(C, falseState, arg, "calloc"))
         return;
-      else if (i == 0)
+      if (i == 0)
         continue;
-      else
-        return;
+      return;
     }
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
index 6bc39ab56504..764e2c640feb 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefCallArgsChecker.cpp
@@ -294,7 +294,7 @@ class RawPtrRefCallArgsChecker
     if (name == "adoptRef" || name == "getPtr" || name == "WeakPtr" ||
         name == "is" || name == "equal" || name == "hash" || name == "isType" ||
         // FIXME: Most/all of these should be implemented via attributes.
-        name == "equalIgnoringASCIICase" ||
+        name == "CFEqual" || name == "equalIgnoringASCIICase" ||
         name == "equalIgnoringASCIICaseCommon" ||
         name == "equalIgnoringNullity" || name == "toString")
       return true;
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 584ae33383cb..3686bd448887 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -305,7 +305,7 @@ static bool isFunctionMacroExpansion(SourceLocation Loc,
     return false;
   while (SM.isMacroArgExpansion(Loc))
     Loc = SM.getImmediateExpansionRange(Loc).getBegin();
-  std::pair<FileID, unsigned> TLInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset TLInfo = SM.getDecomposedLoc(Loc);
   SrcMgr::SLocEntry SE = SM.getSLocEntry(TLInfo.first);
   const SrcMgr::ExpansionInfo &EInfo = SE.getExpansion();
   return EInfo.isFunctionMacroExpansion();
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index f78b1b84f9df..34fcb9b64d55 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -688,6 +688,18 @@ const FunctionDecl *SimpleFunctionCall::getDecl() const {
   return getSVal(getOriginExpr()->getCallee()).getAsFunctionDecl();
 }
 
+RuntimeDefinition SimpleFunctionCall::getRuntimeDefinition() const {
+  // Clang converts lambdas to function pointers using an implicit conversion
+  // operator, which returns the lambda's '__invoke' method. However, Sema
+  // leaves the body of '__invoke' empty (it is generated later in CodeGen), so
+  // we need to skip '__invoke' and access the lambda's operator() directly.
+  if (const auto *CMD = dyn_cast_if_present<CXXMethodDecl>(getDecl());
+      CMD && CMD->isLambdaStaticInvoker())
+    return RuntimeDefinition{CMD->getParent()->getLambdaCallOperator()};
+
+  return AnyFunctionCall::getRuntimeDefinition();
+}
+
 const FunctionDecl *CXXInstanceCall::getDecl() const {
   const auto *CE = cast_or_null<CallExpr>(getOriginExpr());
   if (!CE)
diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index c1bafe858358..2ef98e17cf9c 100644
--- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -895,7 +895,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter &R, FileID BugFileID,
 
   SourceManager &SM = R.getSourceMgr();
   assert(&Pos.getManager() == &SM && "SourceManagers are different!");
-  std::pair<FileID, unsigned> LPosInfo = SM.getDecomposedExpansionLoc(Pos);
+  FileIDAndOffset LPosInfo = SM.getDecomposedExpansionLoc(Pos);
 
   if (LPosInfo.first != BugFileID)
     return;
@@ -1034,7 +1034,7 @@ void HTMLDiagnostics::HandlePiece(Rewriter &R, FileID BugFileID,
       FullSourceLoc L = MP->getLocation().asLocation().getExpansionLoc();
       assert(L.isFileID());
       StringRef BufferInfo = L.getBufferData();
-      std::pair<FileID, unsigned> LocInfo = L.getDecomposedLoc();
+      FileIDAndOffset LocInfo = L.getDecomposedLoc();
       const char* MacroName = LocInfo.second + BufferInfo.data();
       Lexer rawLexer(SM.getLocForStartOfFile(LocInfo.first), PP.getLangOpts(),
                      BufferInfo.begin(), MacroName, BufferInfo.end());
diff --git a/clang/lib/Tooling/Core/Replacement.cpp b/clang/lib/Tooling/Core/Replacement.cpp
index 9e2582ee53c5..1506218d8855 100644
--- a/clang/lib/Tooling/Core/Replacement.cpp
+++ b/clang/lib/Tooling/Core/Replacement.cpp
@@ -120,8 +120,7 @@ bool operator==(const Replacement &LHS, const Replacement &RHS) {
 void Replacement::setFromSourceLocation(const SourceManager &Sources,
                                         SourceLocation Start, unsigned Length,
                                         StringRef ReplacementText) {
-  const std::pair<FileID, unsigned> DecomposedLocation =
-      Sources.getDecomposedLoc(Start);
+  const FileIDAndOffset DecomposedLocation = Sources.getDecomposedLoc(Start);
   OptionalFileEntryRef Entry =
       Sources.getFileEntryRefForID(DecomposedLocation.first);
   this->FilePath = std::string(Entry ? Entry->getName() : InvalidLocation);
@@ -137,8 +136,8 @@ static int getRangeSize(const SourceManager &Sources,
                         const LangOptions &LangOpts) {
   SourceLocation SpellingBegin = Sources.getSpellingLoc(Range.getBegin());
   SourceLocation SpellingEnd = Sources.getSpellingLoc(Range.getEnd());
-  std::pair<FileID, unsigned> Start = Sources.getDecomposedLoc(SpellingBegin);
-  std::pair<FileID, unsigned> End = Sources.getDecomposedLoc(SpellingEnd);
+  FileIDAndOffset Start = Sources.getDecomposedLoc(SpellingBegin);
+  FileIDAndOffset End = Sources.getDecomposedLoc(SpellingEnd);
   if (Start.first != End.first) return -1;
   if (Range.isTokenRange())
     End.second += Lexer::MeasureTokenLength(SpellingEnd, Sources, LangOpts);
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 2868522f8001..2437b2d3595e 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Threading.h"
 #include <optional>
@@ -108,31 +107,39 @@ DependencyScanningFilesystemSharedCache::getShardForUID(
   return CacheShards[Hash % NumShards];
 }
 
-std::vector<StringRef>
-DependencyScanningFilesystemSharedCache::getInvalidNegativeStatCachedPaths(
+std::vector<DependencyScanningFilesystemSharedCache::OutOfDateEntry>
+DependencyScanningFilesystemSharedCache::getOutOfDateEntries(
     llvm::vfs::FileSystem &UnderlyingFS) const {
   // Iterate through all shards and look for cached stat errors.
-  std::vector<StringRef> InvalidPaths;
+  std::vector<OutOfDateEntry> InvalidDiagInfo;
   for (unsigned i = 0; i < NumShards; i++) {
     const CacheShard &Shard = CacheShards[i];
     std::lock_guard<std::mutex> LockGuard(Shard.CacheLock);
     for (const auto &[Path, CachedPair] : Shard.CacheByFilename) {
       const CachedFileSystemEntry *Entry = CachedPair.first;
 
-      if (Entry->getError()) {
-        // Only examine cached errors.
-        llvm::ErrorOr<llvm::vfs::Status> Stat = UnderlyingFS.status(Path);
-        if (Stat) {
+      llvm::ErrorOr<llvm::vfs::Status> Status = UnderlyingFS.status(Path);
+      if (Status) {
+        if (Entry->getError()) {
           // This is the case where we have cached the non-existence
-          // of the file at Path first, and a a file at the path is created
+          // of the file at Path first, and a file at the path is created
           // later. The cache entry is not invalidated (as we have no good
           // way to do it now), which may lead to missing file build errors.
-          InvalidPaths.push_back(Path);
+          InvalidDiagInfo.emplace_back(Path.data());
+        } else {
+          llvm::vfs::Status CachedStatus = Entry->getStatus();
+          uint64_t CachedSize = CachedStatus.getSize();
+          uint64_t ActualSize = Status->getSize();
+          if (CachedSize != ActualSize) {
+            // This is the case where the cached file has a different size
+            // from the actual file that comes from the underlying FS.
+            InvalidDiagInfo.emplace_back(Path.data(), CachedSize, ActualSize);
+          }
         }
       }
     }
   }
-  return InvalidPaths;
+  return InvalidDiagInfo;
 }
 
 const CachedFileSystemEntry *
@@ -233,19 +240,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
 }
 
 DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
-    DependencyScanningService &Service,
+    DependencyScanningFilesystemSharedCache &SharedCache,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
     : llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
                         llvm::vfs::ProxyFileSystem>(std::move(FS)),
-      Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
+      SharedCache(SharedCache),
+      WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
   updateWorkingDirForCacheLookup();
 }
 
 const CachedFileSystemEntry &
 DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
     TentativeEntry TEntry) {
-  auto &Shard =
-      Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID());
+  auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
   return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
                                        std::move(TEntry.Status),
                                        std::move(TEntry.Contents));
@@ -256,44 +263,18 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
     StringRef Filename) {
   if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
     return Entry;
-  auto &Shard = Service.getSharedCache().getShardForFilename(Filename);
+  auto &Shard = SharedCache.getShardForFilename(Filename);
   if (const auto *Entry = Shard.findEntryByFilename(Filename))
     return &LocalCache.insertEntryForFilename(Filename, *Entry);
   return nullptr;
 }
 
-const CachedFileSystemEntry *
-DependencyScanningWorkerFilesystem::findSharedEntryByUID(
-    llvm::vfs::Status Stat) const {
-  return Service.getSharedCache()
-      .getShardForUID(Stat.getUniqueID())
-      .findEntryByUID(Stat.getUniqueID());
-}
-
-const CachedFileSystemEntry &
-DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename(
-    StringRef Filename, std::error_code EC) {
-  return Service.getSharedCache()
-      .getShardForFilename(Filename)
-      .getOrEmplaceEntryForFilename(Filename, EC);
-}
-
-const CachedFileSystemEntry &
-DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename(
-    StringRef Filename, const CachedFileSystemEntry &Entry) {
-  return Service.getSharedCache()
-      .getShardForFilename(Filename)
-      .getOrInsertEntryForFilename(Filename, Entry);
-}
-
 llvm::ErrorOr<const CachedFileSystemEntry &>
 DependencyScanningWorkerFilesystem::computeAndStoreResult(
     StringRef OriginalFilename, StringRef FilenameForLookup) {
   llvm::ErrorOr<llvm::vfs::Status> Stat =
       getUnderlyingFS().status(OriginalFilename);
   if (!Stat) {
-    if (!Service.shouldCacheNegativeStats())
-      return Stat.getError();
     const auto &Entry =
         getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
     return insertLocalEntryForFilename(FilenameForLookup, Entry);
@@ -447,8 +428,7 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
     return HandleCachedRealPath(*RealPath);
 
   // If we have the result in the shared cache, cache it locally.
-  auto &Shard =
-      Service.getSharedCache().getShardForFilename(*FilenameForLookup);
+  auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
   if (const auto *ShardRealPath =
           Shard.findRealPathByFilename(*FilenameForLookup)) {
     const auto &RealPath = LocalCache.insertRealPathForFilename(
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index c2f3cdbb02e3..7f40c99f0728 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -15,8 +15,7 @@ using namespace dependencies;
 DependencyScanningService::DependencyScanningService(
     ScanningMode Mode, ScanningOutputFormat Format,
     ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
-    std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
+    std::time_t BuildSessionTimestamp)
     : Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
       EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
-      CacheNegativeStats(CacheNegativeStats),
       BuildSessionTimestamp(BuildSessionTimestamp) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
index b015e79f400c..515211d47b34 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -171,6 +171,8 @@ TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() {
   TranslationUnitDeps TU;
 
   TU.ID.ContextHash = std::move(ContextHash);
+  TU.ID.ModuleName = std::move(ModuleName);
+  TU.NamedModuleDeps = std::move(NamedModuleDeps);
   TU.FileDeps = std::move(Dependencies);
   TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps);
   TU.Commands = std::move(Commands);
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index d9820ca3c584..9bd85479d981 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -611,7 +611,8 @@ DependencyScanningWorker::DependencyScanningWorker(
 
   switch (Service.getMode()) {
   case ScanningMode::DependencyDirectivesScan:
-    DepFS = new DependencyScanningWorkerFilesystem(Service, FS);
+    DepFS =
+        new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
     BaseFS = DepFS;
     break;
   case ScanningMode::CanonicalPreprocessing:
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index b1495163ccc2..fa86d714ff69 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -714,9 +714,8 @@ void ModuleDepCollectorPP::EndOfMainFile() {
 
   MDC.Consumer.handleDependencyOutputOpts(*MDC.Opts);
 
-  if (MDC.Service.getFormat() == ScanningOutputFormat::P1689)
-    MDC.Consumer.handleProvidedAndRequiredStdCXXModules(
-        MDC.ProvidedStdCXXModule, MDC.RequiredStdCXXModules);
+  MDC.Consumer.handleProvidedAndRequiredStdCXXModules(
+      MDC.ProvidedStdCXXModule, MDC.RequiredStdCXXModules);
 
   for (auto &&I : MDC.ModularDeps)
     MDC.Consumer.handleModuleDependency(*I.second);
diff --git a/clang/lib/Tooling/Refactoring/AtomicChange.cpp b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
index 116f2a757a80..13256b52caea 100644
--- a/clang/lib/Tooling/Refactoring/AtomicChange.cpp
+++ b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
@@ -196,8 +196,7 @@ combineReplacementsInChanges(llvm::StringRef FilePath,
 AtomicChange::AtomicChange(const SourceManager &SM,
                            SourceLocation KeyPosition) {
   const FullSourceLoc FullKeyPosition(KeyPosition, SM);
-  std::pair<FileID, unsigned> FileIDAndOffset =
-      FullKeyPosition.getSpellingLoc().getDecomposedLoc();
+  auto FileIDAndOffset = FullKeyPosition.getSpellingLoc().getDecomposedLoc();
   OptionalFileEntryRef FE = SM.getFileEntryRefForID(FileIDAndOffset.first);
   assert(FE && "Cannot create AtomicChange with invalid location.");
   FilePath = std::string(FE->getName());
diff --git a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
index c18f9290471f..8eff77861f75 100644
--- a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
+++ b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
@@ -45,8 +45,7 @@ bool IsValidEditLoc(const clang::SourceManager& SM, clang::SourceLocation Loc) {
   if (Loc.isInvalid())
     return false;
   const clang::FullSourceLoc FullLoc(Loc, SM);
-  std::pair<clang::FileID, unsigned> FileIdAndOffset =
-      FullLoc.getSpellingLoc().getDecomposedLoc();
+  auto FileIdAndOffset = FullLoc.getSpellingLoc().getDecomposedLoc();
   return SM.getFileEntryForID(FileIdAndOffset.first) != nullptr;
 }
 
diff --git a/clang/lib/Tooling/Transformer/SourceCode.cpp b/clang/lib/Tooling/Transformer/SourceCode.cpp
index ab7184c2c069..922dafeddf41 100644
--- a/clang/lib/Tooling/Transformer/SourceCode.cpp
+++ b/clang/lib/Tooling/Transformer/SourceCode.cpp
@@ -68,8 +68,8 @@ llvm::Error clang::tooling::validateRange(const CharSourceRange &Range,
                                            "Range is in system header");
   }
 
-  std::pair<FileID, unsigned> BeginInfo = SM.getDecomposedLoc(Range.getBegin());
-  std::pair<FileID, unsigned> EndInfo = SM.getDecomposedLoc(Range.getEnd());
+  FileIDAndOffset BeginInfo = SM.getDecomposedLoc(Range.getBegin());
+  FileIDAndOffset EndInfo = SM.getDecomposedLoc(Range.getEnd());
   if (BeginInfo.first != EndInfo.first)
     return llvm::make_error<StringError>(
         errc::invalid_argument, "Range begins and ends in different files");
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 3b95a8ea4859..1bb6d22db982 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -1753,4 +1753,12 @@ namespace I128Mul {
 }
 #endif
 
+namespace InitParam {
+  constexpr int foo(int a) {
+      __builtin_mul_overflow(20, 10, &a);
+      return a;
+  }
+  static_assert(foo(10) == 200);
+}
+
 #endif
diff --git a/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp b/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp
new file mode 100644
index 000000000000..0e8127d27f88
--- /dev/null
+++ b/clang/test/AST/ByteCode/libcxx/rvalue-reference-param.cpp
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -std=c++2c -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++2c -verify=ref,both      %s
+
+template <int __v> struct integral_constant {
+  static const int value = __v;
+};
+template <bool _Val> using _BoolConstant = integral_constant<_Val>;
+template <class _From, class _To>
+constexpr bool is_convertible_v = __is_convertible(_From, _To);
+template <class _Tp> _Tp __declval(int);
+template <class _Tp> decltype(__declval<_Tp>(0)) declval();
+template <class _Tp, class _Up>
+using _IsSame = _BoolConstant<__is_same(_Tp, _Up)>;
+template <class _If> struct conditional {
+  using type = _If;
+};
+template <bool, class _IfRes, class>
+using conditional_t = conditional<_IfRes>::type;
+template <class _Tp, class>
+concept __weakly_equality_comparable_with = requires(_Tp __t) { __t; };
+template <bool, class _Tp = void> using __enable_if_t = _Tp;
+template <template <class> class, class>
+integral_constant<true> __sfinae_test_impl(int);
+template <template <class> class _Templ, class... _Args>
+using _IsValidExpansion = decltype(__sfinae_test_impl<_Templ, _Args...>(0));
+template <class _Tp>
+using __test_for_primary_template =
+    __enable_if_t<_IsSame<_Tp, typename _Tp::__primary_template>::value>;
+template <class _Tp>
+using __is_primary_template =
+    _IsValidExpansion<__test_for_primary_template, _Tp>;
+template <class _Ip>
+using iter_difference_t =
+    conditional_t<__is_primary_template<_Ip>::value, _Ip, _Ip>::difference_type;
+template <int> struct _OrImpl {
+  template <class, class _First>
+  using _Result = _OrImpl<!_First::value>::template _Result<_First>;
+};
+template <> struct _OrImpl<false> {
+  template <class _Res> using _Result = _Res;
+};
+template <class... _Args>
+using _Or = _OrImpl<sizeof...(_Args)>::template _Result<_Args...>;
+struct input_iterator_tag {};
+template <class _Dp, class _Bp>
+concept derived_from = is_convertible_v<_Dp, _Bp>;
+template <class _Ip>
+concept input_or_output_iterator = requires(_Ip __i) { __i; };
+template <class _Sp, class _Ip>
+concept sentinel_for = __weakly_equality_comparable_with<_Sp, _Ip>;
+struct __iter_concept_category_test {
+  template <class> using _Apply = input_iterator_tag;
+};
+struct __test_iter_concept
+    : _IsValidExpansion<__iter_concept_category_test::_Apply, int>,
+      __iter_concept_category_test {};
+struct __iter_concept_cache {
+  using type = _Or<int, __test_iter_concept>;
+};
+template <class _Iter>
+using _ITER_CONCEPT = __iter_concept_cache::type::_Apply<_Iter>;
+template <class _Ip>
+concept input_iterator = derived_from<_ITER_CONCEPT<_Ip>, input_iterator_tag>;
+template <class _T1, class _T2> struct pair {
+  _T1 first;
+  _T2 second;
+};
+struct {
+  template <class _Tp> auto operator()(_Tp __t) { return __t.begin(); }
+} begin;
+template <class _Tp> using iterator_t = decltype(begin(declval<_Tp>()));
+template <class _Tp>
+concept __member_size = requires(_Tp __t) { __t; };
+struct {
+  template <__member_size _Tp> constexpr void operator()(_Tp &&__t) {
+    __t.size(); // both-note 2{{in instantiation}}
+  }
+} size;
+template <class _Tp>
+concept range = requires(_Tp __t) { __t; };
+template <class _Tp>
+concept input_range = input_iterator<_Tp>;
+template <range _Rp>
+using range_difference_t = iter_difference_t<iterator_t<_Rp>>;
+struct {
+  template <range _Rp> constexpr range_difference_t<_Rp> operator()(_Rp &&__r) {
+    size(__r); // both-note 2{{in instantiation}}
+  } // both-warning {{does not return a value}}
+} distance;
+template <input_or_output_iterator _Iter, sentinel_for<_Iter> _Sent>
+struct subrange {
+  _Iter __begin_;
+  _Sent __end_;
+  _Iter begin();
+  constexpr _Iter size() { __end_ - __begin_; } // both-warning {{does not return a value}} \
+                                                // both-note {{in instantiation}}
+};
+struct {
+  template <input_range _Range1, input_range _Range2>
+  void operator()(_Range1 &&__range1, _Range2) {
+    (void)(distance(__range1) != 0); // both-note 3{{in instantiation}}
+  }
+} equal;
+template <class _Owner> struct __key_value_iterator {
+  using difference_type = _Owner::difference_type;
+  constexpr friend difference_type operator-(__key_value_iterator,
+                                             __key_value_iterator &) {} // both-warning {{does not return a value}}
+};
+struct flat_multimap {
+  template <bool> using __iterator = __key_value_iterator<flat_multimap>;
+  using difference_type =
+      decltype(static_cast<int *>(nullptr) - static_cast<int *>(nullptr));
+  pair<__iterator<true>, __iterator<true>> equal_range(const char *);
+} test_expected_range;
+void test() {
+  flat_multimap m;
+  auto test_found = [](auto map, auto expected_key, int) {
+    auto [first, last] = map.equal_range(expected_key);
+    equal(subrange(first, last), test_expected_range); // both-note 3{{in instantiation}}
+  };
+  test_found(m, "", {});
+}
diff --git a/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp b/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp
new file mode 100644
index 000000000000..df7ef3fe5ae9
--- /dev/null
+++ b/clang/test/AST/ByteCode/libcxx/tuple-decompose-for-range.cpp
@@ -0,0 +1,264 @@
+// RUN: %clang_cc1 -std=c++2c -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++2c -verify=ref,both      %s
+
+// both-no-diagnostics
+
+namespace std {
+typedef __SIZE_TYPE__ size_t;
+}
+extern "C++" {
+namespace std {
+template <typename> struct iterator_traits;
+}
+}
+namespace std {
+template <typename _Tp, _Tp __v> struct integral_constant {
+  static constexpr _Tp value = __v;
+};
+template <bool __v> using __bool_constant = integral_constant<bool, __v>;
+using false_type = __bool_constant<false>;
+template <bool, typename _Tp = void> struct enable_if {
+  using type = _Tp;
+};
+template <bool> struct __conditional {
+  template <typename _Tp, typename> using type = _Tp;
+};
+template <bool _Cond, typename _If, typename _Else>
+using __conditional_t =
+    typename __conditional<_Cond>::template type<_If, _Else>;
+template <typename _Tp>
+struct is_empty : public __bool_constant<__is_empty(_Tp)> {};
+template <typename _Tp, typename _Up>
+struct is_same : public __bool_constant<__is_same(_Tp, _Up)> {};
+template <typename _Tp> struct remove_cv {
+  using type = __remove_cv(_Tp);
+};
+template <typename _Tp> struct tuple_size;
+template <typename _Tp, typename _Up = typename remove_cv<_Tp>::type,
+          typename = typename enable_if<is_same<_Tp, _Up>::value>::type,
+          size_t = tuple_size<_Tp>::value>
+using __enable_if_has_tuple_size = _Tp;
+template <typename _Tp>
+struct tuple_size<const __enable_if_has_tuple_size<_Tp>>
+    : public tuple_size<_Tp> {};
+template <size_t __i, typename _Tp> struct tuple_element;
+template <size_t __i, typename _Tp>
+using __tuple_element_t = typename tuple_element<__i, _Tp>::type;
+template <size_t __i, typename _Tp> struct tuple_element<__i, const _Tp> {
+  using type = const __tuple_element_t<__i, _Tp>;
+};
+template <size_t _Np, typename... _Types> struct _Nth_type {
+  using type = __type_pack_element<_Np, _Types...>;
+};
+template <typename _Tp> struct iterator_traits<_Tp *> {
+  using reference = _Tp &;
+};
+} // namespace std
+extern "C++" {
+void *operator new(std::size_t, void *__p);
+}
+namespace std {
+template <typename _Tp, typename... _Args>
+constexpr inline void _Construct(_Tp *__p, _Args &&...__args) {
+  ::new (__p) _Tp(__args...);
+};
+} // namespace std
+namespace __gnu_cxx {
+template <typename _Iterator, typename _Container> class __normal_iterator {
+protected:
+  _Iterator _M_current;
+  typedef std::iterator_traits<_Iterator> __traits_type;
+
+public:
+  typedef _Iterator iterator_type;
+  typedef typename __traits_type::reference reference;
+  explicit constexpr __normal_iterator(const _Iterator &__i)
+      : _M_current(__i) {};
+  constexpr reference operator*() const { return *_M_current; }
+  constexpr __normal_iterator &operator++() {
+    ++_M_current;
+    return *this;
+  }
+  constexpr const _Iterator &base() const { return _M_current; }
+};
+template <typename _Iterator, typename _Container>
+constexpr bool
+operator==(const __normal_iterator<_Iterator, _Container> &__lhs,
+           const __normal_iterator<_Iterator, _Container> &__rhs) {
+  return __lhs.base() == __rhs.base();
+}
+} // namespace __gnu_cxx
+namespace std {
+template <typename _Tp> class __new_allocator {};
+template <typename _Tp> using __allocator_base = __new_allocator<_Tp>;
+template <typename> struct allocator_traits;
+template <typename _Tp> class allocator : public __allocator_base<_Tp> {
+public:
+  typedef _Tp value_type;
+  constexpr _Tp *allocate(size_t __n) {
+    __n *= sizeof(_Tp);
+    return static_cast<_Tp *>(::operator new(__n));
+  }
+  constexpr void deallocate(_Tp *__p, size_t __n) { ::operator delete(__p); }
+};
+template <typename _Tp> struct allocator_traits<allocator<_Tp>> {
+  using allocator_type = allocator<_Tp>;
+  using pointer = _Tp *;
+  using size_type = std::size_t;
+  template <typename _Up> using rebind_alloc = allocator<_Up>;
+  static constexpr pointer allocate(allocator_type &__a, size_type __n) {
+    return __a.allocate(__n);
+  }
+  static constexpr void deallocate(allocator_type &__a, pointer __p,
+                                   size_type __n) {
+    __a.deallocate(__p, __n);
+  }
+};
+} // namespace std
+namespace __gnu_cxx {
+template <typename _Alloc, typename = typename _Alloc::value_type>
+struct __alloc_traits : std::allocator_traits<_Alloc> {
+  typedef std::allocator_traits<_Alloc> _Base_type;
+  template <typename _Tp> struct rebind {
+    typedef typename _Base_type::template rebind_alloc<_Tp> other;
+  };
+};
+} // namespace __gnu_cxx
+namespace std {
+template <typename _InputIterator, typename _ForwardIterator>
+constexpr _ForwardIterator __do_uninit_copy(_InputIterator __first,
+                                            _InputIterator __last,
+                                            _ForwardIterator __result) {
+  _ForwardIterator __cur = __result;
+  for (; __first != __last; ++__first, ++__cur)
+    std::_Construct(&*__cur, *__first);
+  return __cur;
+};
+template <typename _InputIterator, typename _ForwardIterator, typename _Tp>
+constexpr inline _ForwardIterator
+__uninitialized_copy_a(_InputIterator __first, _InputIterator __last,
+                       _ForwardIterator __result, allocator<_Tp> &) {
+  return std::__do_uninit_copy(__first, __last, __result);
+}
+template <typename _Tp, typename _Alloc> struct _Vector_base {
+  typedef
+      typename __gnu_cxx::__alloc_traits<_Alloc>::template rebind<_Tp>::other
+          _Tp_alloc_type;
+  typedef typename __gnu_cxx::__alloc_traits<_Tp_alloc_type>::pointer pointer;
+  struct _Vector_impl_data {
+    pointer _M_start;
+    pointer _M_finish;
+    pointer _M_end_of_storage;
+  };
+  struct _Vector_impl : public _Tp_alloc_type, public _Vector_impl_data {};
+
+public:
+  typedef _Alloc allocator_type;
+  constexpr _Tp_alloc_type &_M_get_Tp_allocator() { return this->_M_impl; }
+  constexpr _Vector_base(const allocator_type &__a) : _M_impl(__a) {}
+  constexpr ~_Vector_base() {
+    _M_deallocate(_M_impl._M_start,
+                  _M_impl._M_end_of_storage - _M_impl._M_start);
+  }
+
+public:
+  _Vector_impl _M_impl;
+  constexpr pointer _M_allocate(size_t __n) {
+    typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
+    return __n != 0 ? _Tr::allocate(_M_impl, __n) : pointer();
+  }
+  constexpr void _M_deallocate(pointer __p, size_t __n) {
+    typedef __gnu_cxx::__alloc_traits<_Tp_alloc_type> _Tr;
+    if (__p)
+      _Tr::deallocate(_M_impl, __p, __n);
+  }
+
+protected:
+};
+template <typename _Tp, typename _Alloc = std::allocator<_Tp>>
+class vector : protected _Vector_base<_Tp, _Alloc> {
+  typedef _Vector_base<_Tp, _Alloc> _Base;
+
+public:
+  typedef _Tp value_type;
+  typedef typename _Base::pointer pointer;
+  typedef __gnu_cxx::__normal_iterator<pointer, vector> iterator;
+  typedef size_t size_type;
+  typedef _Alloc allocator_type;
+  using _Base::_M_get_Tp_allocator;
+
+public:
+private:
+public:
+  constexpr vector(_Tp __l, const allocator_type &__a = allocator_type()) : _Base(__a) {
+
+  }
+  constexpr iterator begin() { return iterator(this->_M_impl._M_start); }
+  constexpr iterator end() { return iterator(this->_M_impl._M_finish); }
+
+protected:
+  template <typename _Iterator>
+  constexpr void _M_range_initialize_n(_Iterator __first, _Iterator __last,
+                                       size_type __n) {
+    pointer __start = this->_M_impl._M_start = this->_M_allocate(((__n)));
+    this->_M_impl._M_end_of_storage = __start + __n;
+    this->_M_impl._M_finish = std::__uninitialized_copy_a(
+        __first, __last, __start, _M_get_Tp_allocator());
+  }
+};
+template <typename _Tp> struct __is_empty_non_tuple : is_empty<_Tp> {};
+template <typename _Tp>
+using __empty_not_final =
+    __conditional_t<__is_final(_Tp), false_type, __is_empty_non_tuple<_Tp>>;
+template <size_t _Idx, typename _Head, bool = __empty_not_final<_Head>::value>
+struct _Head_base;
+template <size_t _Idx, typename _Head> struct _Head_base<_Idx, _Head, false> {
+  static constexpr const _Head &_M_head(const _Head_base &__b) {
+    return __b._M_head_impl;
+  }
+  _Head _M_head_impl;
+};
+template <size_t _Idx, typename... _Elements> struct _Tuple_impl;
+template <size_t _Idx, typename _Head>
+struct _Tuple_impl<_Idx, _Head> : private _Head_base<_Idx, _Head> {
+  typedef _Head_base<_Idx, _Head> _Base;
+  static constexpr const _Head &_M_head(const _Tuple_impl &__t) {
+    return _Base::_M_head(__t);
+  }
+  explicit constexpr _Tuple_impl(const _Head &__head) : _Base(__head) {}
+
+protected:
+};
+template <typename... _Elements>
+class tuple : public _Tuple_impl<0, _Elements...> {
+  using _Inherited = _Tuple_impl<0, _Elements...>;
+
+public:
+  template <typename = void>
+  constexpr tuple(const _Elements &...__elements) : _Inherited(__elements...) {}
+};
+template <typename... _Elements>
+struct tuple_size<tuple<_Elements...>>
+    : public integral_constant<size_t, sizeof...(_Elements)> {};
+template <size_t __i, typename... _Types>
+struct tuple_element<__i, tuple<_Types...>> {
+  using type = typename _Nth_type<__i, _Types...>::type;
+};
+template <size_t __i, typename _Head, typename... _Tail>
+constexpr const _Head &
+__get_helper(const _Tuple_impl<__i, _Head, _Tail...> &__t) {
+  return _Tuple_impl<__i, _Head, _Tail...>::_M_head(__t);
+};
+template <size_t __i, typename... _Elements>
+constexpr const int get(const tuple<_Elements...> &&__t) {
+  return std::__get_helper<__i>(__t);
+};
+} // namespace std
+constexpr int foo() {
+  std::vector<std::tuple<int>> data_tuples = {{1}};
+  for (const auto [id] : data_tuples) {
+    int a = id + 3;
+  }
+  return 1;
+}
+static_assert(foo() == 1);
diff --git a/clang/test/AST/ByteCode/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
index 699746c0b2c4..ddf1d2bebdbd 100644
--- a/clang/test/AST/ByteCode/literals.cpp
+++ b/clang/test/AST/ByteCode/literals.cpp
@@ -1418,3 +1418,15 @@ constexpr int usingDirectiveDecl() {
   return FB;
 }
 static_assert(usingDirectiveDecl() == 10, "");
+
+namespace OnePastEndCmp {
+  struct S {
+   int a;
+   int b;
+  };
+
+  constexpr S s{12,13};
+  constexpr const int *p = &s.a;
+  constexpr const int *q = &s.a + 1;
+  static_assert(p != q, "");
+}
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index 9c293e5d15fc..840736f33225 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -1022,6 +1022,42 @@ namespace OpNewNothrow {
                       // both-note {{in call to}}
 }
 
+namespace BaseCompare {
+  struct Cmp {
+    void *p;
+
+    template<typename T>
+    constexpr Cmp(T *t) : p(t) {}
+
+    constexpr friend bool operator==(Cmp a, Cmp b) {
+      return a.p == b.p;
+    }
+  };
+
+  class Base {};
+  class Derived : public Base {};
+  constexpr bool foo() {
+    Derived *D = std::allocator<Derived>{}.allocate(1);;
+    std::construct_at<Derived>(D);
+
+    Derived *d = D;
+    Base    *b = D;
+
+    Cmp ca(d);
+    Cmp cb(b);
+
+    if (ca == cb) {
+      std::allocator<Derived>{}.deallocate(D);
+      return true;
+    }
+    std::allocator<Derived>{}.deallocate(D);
+
+    return false;
+
+  }
+  static_assert(foo());
+}
+
 #else
 /// Make sure we reject this prior to C++20
 constexpr int a() { // both-error {{never produces a constant expression}}
diff --git a/clang/test/AST/ByteCode/placement-new.cpp b/clang/test/AST/ByteCode/placement-new.cpp
index f23d71510602..670def2d5870 100644
--- a/clang/test/AST/ByteCode/placement-new.cpp
+++ b/clang/test/AST/ByteCode/placement-new.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_cc1 -std=c++2c -fcxx-exceptions -fexperimental-new-constant-interpreter -verify=expected,both %s -DBYTECODE
 // RUN: %clang_cc1 -std=c++2c -fcxx-exceptions -verify=ref,both %s
 
+typedef __INT64_TYPE__ int64_t;
 namespace std {
   using size_t = decltype(sizeof(0));
   template<typename T> struct allocator {
@@ -465,3 +466,23 @@ namespace ArrayRoot {
 
   static_assert(foo() == 0);
 }
+
+namespace bitcast {
+  template <typename F, typename T>
+  constexpr T bit_cast(const F &f) {
+    return __builtin_bit_cast(T, f);
+  }
+  constexpr int foo() {
+    double *d = std::allocator<double>{}.allocate(2);
+    std::construct_at<double>(d, 0);
+
+    double &dd = *d;
+
+    int64_t i = bit_cast<double, int64_t>(*d);
+
+
+    std::allocator<double>{}.deallocate(d);
+    return i;
+  }
+  static_assert(foo() == 0);
+}
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 36f4b72335af..f97990c1ff84 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both %s
-// RUN: %clang_cc1 -std=c++20 -verify=ref,both %s
+// RUN: %clang_cc1            -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++20 -verify=expected,both %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1            -verify=ref,both      %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both      %s
 
 union U {
   int a;
@@ -600,6 +600,118 @@ namespace MoveOrAssignOp {
   }
   static_assert(foo());
 }
+
+namespace CopyEmptyUnion {
+  struct A {
+    union {}; // both-warning {{declaration does not declare anything}}
+  };
+  constexpr int foo() {
+     A a;
+     A a2 = a;
+     return 1;
+  }
+  static_assert(foo() == 1);
+}
+
+namespace BitFields {
+  constexpr bool simple() {
+    union U {
+      unsigned a : 1;
+      unsigned b : 1;
+    };
+
+    U u{1};
+    u.b = 1;
+    return u.b;
+  }
+  static_assert(simple());
+}
+
+namespace deactivateRecurses {
+
+  constexpr int foo() {
+    struct A {
+      struct {
+        int a;
+      };
+      int b;
+    };
+    struct B {
+      struct {
+        int a;
+        int b;
+      };
+    };
+
+    union U {
+      A a;
+      B b;
+    } u;
+
+    u.b.a = 10;
+    ++u.b.a;
+
+    u.a.a = 10;
+    ++u.a.a;
+
+    if (__builtin_constant_p(u.b.a))
+      return 10;
+
+    return 1;
+  }
+  static_assert(foo() == 1);
+}
+
+namespace AnonymousUnion {
+  struct Long {
+    struct {
+      unsigned is_long;
+    };
+    unsigned Size;
+  };
+
+  struct Short {
+    struct {
+      unsigned is_long;
+      unsigned Size;
+    };
+    char data;
+  };
+
+  union Rep {
+    Short S;
+    Long L;
+  };
+
+#define assert_active(F)   if (!__builtin_is_within_lifetime(&F)) (1/0);
+#define assert_inactive(F) if ( __builtin_is_within_lifetime(&F)) (1/0);
+  consteval int test() {
+    union UU {
+      struct {
+        Rep R;
+        int a;
+      };
+    } U;
+
+    U.R.S.Size = 10;
+    assert_active(U);
+    assert_active(U.R);
+    assert_active(U.R.S);
+    assert_active(U.R.S.Size);
+
+    U.a = 10;
+    assert_active(U.a);
+    assert_active(U);
+
+    assert_active(U);
+    assert_active(U.R);
+    assert_active(U.R.S);
+    assert_active(U.R.S.Size);
+
+    return 1;
+  }
+  static_assert(test() == 1);
+}
 #endif
 
 namespace AddressComparison {
diff --git a/clang/test/AST/HLSL/RootSignatures-AST.hlsl b/clang/test/AST/HLSL/RootSignatures-AST.hlsl
index 157d90aee7ee..27c40430c9d0 100644
--- a/clang/test/AST/HLSL/RootSignatures-AST.hlsl
+++ b/clang/test/AST/HLSL/RootSignatures-AST.hlsl
@@ -1,5 +1,11 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
-// RUN:  -disable-llvm-passes -o - %s | FileCheck %s
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_1
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
+// RUN:  -fdx-rootsignature-version=rootsig_1_0 \
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_0
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-library -ast-dump \
+// RUN:  -fdx-rootsignature-version=rootsig_1_1 \
+// RUN:  -disable-llvm-passes -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1_1
 
 // This test ensures that the sample root signature is parsed without error and
 // the Attr AST Node is created succesfully. If an invalid root signature was
@@ -23,19 +29,25 @@
                                  "filter = FILTER_MIN_MAG_MIP_LINEAR )"
 
 // CHECK: -HLSLRootSignatureDecl 0x{{.*}} {{.*}} implicit [[SAMPLE_RS_DECL:__hlsl_rootsig_decl_\d*]]
+// CHECK-V1_0: version: 1.0,
+// CHECK-V1_1: version: 1.1,
 // CHECK-SAME: RootElements{
 // CHECK-SAME: RootFlags(AllowInputAssemblerInputLayout | DenyVertexShaderRootAccess),
 // CHECK-SAME: RootCBV(b0,
 // CHECK-SAME:   space = 1, visibility = All, flags = DataStatic
 // CHECK-SAME: ),
 // CHECK-SAME: RootSRV(t0,
-// CHECK-SAME:   space = 0, visibility = All, flags = DataStaticWhileSetAtExecute
+// CHECK-SAME:   space = 0, visibility = All,
+// CHECK-V1_0-SAME: flags = DataVolatile
+// CHECK-V1_1-SAME: flags = DataStaticWhileSetAtExecute
 // CHECK-SAME: ),
 // CHECK-SAME: RootUAV(
 // CHECK-SAME:   u0, space = 0, visibility = All, flags = DataVolatile
 // CHECK-SAME: ),
 // CHECK-SAME: CBV(
-// CHECK-SAME:   b1, numDescriptors = 1, space = 0, offset = DescriptorTableOffsetAppend, flags = DataStaticWhileSetAtExecute
+// CHECK-SAME:   b1, numDescriptors = 1, space = 0, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME: flags = DescriptorsVolatile | DataVolatile
+// CHECK-V1_1-SAME: flags = DataStaticWhileSetAtExecute
 // CHECK-SAME: ),
 // CHECK-SAME: SRV(
 // CHECK-SAME:   t1, numDescriptors = 8, space = 0, offset = DescriptorTableOffsetAppend, flags = DescriptorsVolatile
@@ -47,7 +59,9 @@
 // CHECK-SAME:   numClauses = 3, visibility = All
 // CHECK-SAME: ),
 // CHECK-SAME: Sampler(
-// CHECK-SAME:   s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend, flags = None
+// CHECK-SAME:   s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME:  flags = DescriptorsVolatile
+// CHECK-V1_1-SAME:  flags = None
 // CHECK-SAME: ),
 // CHECK-SAME: DescriptorTable(
 // CHECK-SAME:   numClauses = 1, visibility = All
@@ -93,7 +107,6 @@ void same_rs_main() {}
                    "addressU = TEXTURE_ADDRESS_CLAMP, " \
                    "filter = FILTER_MIN_MAG_MIP_LINEAR )"
 
-
 // CHECK: -RootSignatureAttr 0x{{.*}} {{.*}} [[SAMPLE_RS_DECL]]
 [RootSignature(SampleSameRS)]
 void same_rs_string_main() {}
@@ -105,9 +118,13 @@ void same_rs_string_main() {}
 // a seperate decl and identifier to reference
 
 // CHECK: -HLSLRootSignatureDecl 0x{{.*}} {{.*}} implicit [[DIFF_RS_DECL:__hlsl_rootsig_decl_\d*]]
+// CHECK-V1_0: version: 1.0,
+// CHECK-V1_1: version: 1.1,
 // CHECK-SAME: RootElements{
-// CHECK-SAME:   Sampler(s0, numDescriptors = 4, space = 1,
-// CHECK-SAME:     offset = DescriptorTableOffsetAppend, flags = None),
+// CHECK-SAME:   Sampler(s0, numDescriptors = 4, space = 1, offset = DescriptorTableOffsetAppend,
+// CHECK-V1_0-SAME:  flags = DescriptorsVolatile
+// CHECK-V1_1-SAME:  flags = None
+// CHECK-SAME: ),
 // CHECK-SAME:   DescriptorTable(numClauses = 1, visibility = All)
 // CHECK-SAME: }
 
diff --git a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
index c0955c1ea7b4..733c4e2ee5a3 100644
--- a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
+++ b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
@@ -64,6 +64,17 @@ const unsigned short ushort_const = 10;
 [[vk::constant_id(6)]]
 const unsigned int uint_const = 12;
 
+// CHECK: VarDecl {{.*}} uint_const_2 'const hlsl_private uint':'const hlsl_private unsigned int' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int (*)(unsigned int, unsigned int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int (unsigned int, unsigned int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_uint' 'unsigned int (unsigned int, unsigned int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 6
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12
+[[vk::constant_id(6)]]
+const uint uint_const_2 = 12;
+
 
 // CHECK: VarDecl {{.*}} ulong_const 'const hlsl_private unsigned long long' static cinit
 // CHECK-NEXT: CallExpr {{.*}} 'unsigned long long'
diff --git a/clang/test/AST/absurdly_big_struct.cpp b/clang/test/AST/absurdly_big_struct.cpp
new file mode 100644
index 000000000000..c17274343d57
--- /dev/null
+++ b/clang/test/AST/absurdly_big_struct.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s -triple x86_64-linux-gnu
+
+struct a { // expected-error {{structure 'a' is too large, which exceeds maximum allowed size of 1152921504606846976 bytes}}
+  char x[1ull<<60]; 
+  char x2[1ull<<60]; 
+};
+
+a z[1];
+long long x() { return sizeof(a); }
+long long x2() { return sizeof(a::x); }
+long long x3() { return sizeof(a::x2); }
+long long x4() { return sizeof(z); }
+
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 2efd0b5e8ac2..8ccb39f8f316 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -589,3 +589,13 @@ void leakNewFn() { new struct Sock; }
 // CHECK: CXXNewExpr {{.*}} <col:20, col:31> 'struct Sock *'
 }
 
+namespace GH143711 {
+struct S {
+  S(int, int);
+};
+
+void f() {
+  S(S(0, 1));
+}
+// CHECK: CXXTemporaryObjectExpr {{.*}} <col:5, col:11> 'S':'GH143711::S' 'void (int, int)'
+}
diff --git a/clang/test/AST/ast-print-openacc-cache-construct.cpp b/clang/test/AST/ast-print-openacc-cache-construct.cpp
index 427f38130916..c1a8b7a66eaa 100644
--- a/clang/test/AST/ast-print-openacc-cache-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-cache-construct.cpp
@@ -1,8 +1,11 @@
 // RUN: %clang_cc1 -fopenacc -ast-print %s -o - | FileCheck %s
 void foo() {
   int Array[5];
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
   // CHECK: #pragma acc cache(readonly: Array[1], Array[1:2])
   #pragma acc cache(readonly:Array[1], Array[1:2])
   // CHECK: #pragma acc cache(Array[1], Array[1:2])
   #pragma acc cache(Array[1], Array[1:2])
+  }
 }
diff --git a/clang/test/AST/ast-print-openacc-combined-construct.cpp b/clang/test/AST/ast-print-openacc-combined-construct.cpp
index 7331ec3786e9..b4e803348a2f 100644
--- a/clang/test/AST/ast-print-openacc-combined-construct.cpp
+++ b/clang/test/AST/ast-print-openacc-combined-construct.cpp
@@ -179,8 +179,8 @@ void foo() {
 #pragma acc parallel loop copyin(i, array[1], array, array[1:2]) pcopyin(readonly:i, array[1], array, array[1:2]) present_or_copyin(always, alwaysin: i, array[1], array, array[1:2])
   for(int i = 0;i<5;++i);
 
-// CHECK: #pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(always, alwaysin: i, array[1], array, array[1:2])
-#pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(always, alwaysin: i, array[1], array, array[1:2])
+// CHECK: #pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(always, alwaysout: i, array[1], array, array[1:2])
+#pragma acc parallel loop copyout(i, array[1], array, array[1:2]) pcopyout(zero: i, array[1], array, array[1:2]) present_or_copyout(always, alwaysout: i, array[1], array, array[1:2])
   for(int i = 0;i<5;++i);
 
 // CHECK: #pragma acc parallel loop create(i, array[1], array, array[1:2]) pcreate(zero: i, array[1], array, array[1:2]) present_or_create(i, array[1], array, array[1:2])
diff --git a/clang/test/Analysis/ArrayDelete.cpp b/clang/test/Analysis/ArrayDelete.cpp
index 6887e0a35fb8..2710fa1e1e11 100644
--- a/clang/test/Analysis/ArrayDelete.cpp
+++ b/clang/test/Analysis/ArrayDelete.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=cplusplus.ArrayDelete -std=c++11 -verify -analyzer-output=text %s
 
 struct Base {
     virtual ~Base() = default;
diff --git a/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm b/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm
new file mode 100644
index 000000000000..7ccc0d61f0ac
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/call-args-safe-functions.mm
@@ -0,0 +1,12 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+// expected-no-diagnostics
+
+#include "objc-mock-types.h"
+
+CGImageRef provideImage();
+
+Boolean cfe(CFTypeRef obj)
+{
+  return CFEqual(obj, provideImage());
+}
+
diff --git a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
index 9e4356a71f1b..09b303961fd6 100644
--- a/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/objc-mock-types.h
@@ -45,6 +45,7 @@ CFDictionaryRef CFDictionaryCreateCopy(CFAllocatorRef allocator, CFDictionaryRef
 CFDictionaryRef CFDictionaryCreateMutableCopy(CFAllocatorRef allocator, CFIndex capacity, CFDictionaryRef theDict);
 CFIndex CFDictionaryGetCount(CFDictionaryRef theDict);
 Boolean CFDictionaryContainsKey(CFDictionaryRef theDict, const void *key);
+Boolean CFEqual(CFTypeRef, CFTypeRef);
 Boolean CFDictionaryContainsValue(CFDictionaryRef theDict, const void *value);
 const void *CFDictionaryGetValue(CFDictionaryRef theDict, const void *key);
 void CFDictionaryAddValue(CFMutableDictionaryRef theDict, const void *key, const void *value);
diff --git a/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp b/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
index 2809f3758eee..68ae1aee2e28 100644
--- a/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
+++ b/clang/test/Analysis/DeleteWithNonVirtualDtor.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.cplusplus.DeleteWithNonVirtualDtor -std=c++11 -verify -analyzer-output=text %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.cplusplus.DeleteWithNonVirtualDtor -std=c++11 -verify -analyzer-output=text %s
 
 struct Virtual {
   virtual ~Virtual() {}
diff --git a/clang/test/Analysis/PR37855.c b/clang/test/Analysis/PR37855.c
index e9c2564ba008..92ed9ffe69a7 100644
--- a/clang/test/Analysis/PR37855.c
+++ b/clang/test/Analysis/PR37855.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=core -w -DNO_CROSSCHECK -verify %s
-// RUN: %clang_cc1 -analyze -analyzer-checker=core -w -analyzer-config crosscheck-with-z3=true -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -w -DNO_CROSSCHECK -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -w -analyzer-config crosscheck-with-z3=true -verify %s
 // REQUIRES: z3
 
 typedef struct o p;
@@ -8,15 +8,15 @@ struct o {
   } s;
 };
 
-void q(*r, p2) { r < p2; }
+void q(int *r, int p2) { r < p2; }
 
-void k(l, node) {
+void k(int l, int node) {
   struct {
     p *node;
   } * n, *nodep, path[sizeof(void)];
-  path->node = l;
+  path->node = (p*) l;
   for (n = path; node != l;) {
-    q(node, n->node);
+    q((int *)&node, (int)n->node);
     nodep = n;
   }
   if (nodep) // expected-warning {{Branch condition evaluates to a garbage value}}
diff --git a/clang/test/Analysis/PR38208.c b/clang/test/Analysis/PR38208.c
index fb0a5a15eac6..750cbc50cc74 100644
--- a/clang/test/Analysis/PR38208.c
+++ b/clang/test/Analysis/PR38208.c
@@ -2,6 +2,11 @@
 
 // expected-no-diagnostics
 
+// This test case used to demonstrate a huge slowdown regression.
+// Reported in https://bugs.llvm.org/show_bug.cgi?id=38208
+// Caused by 2bbccca9f75b6bce08d77cf19abfb206d0c3bc2e aka. "aggressive-binary-operation-simplification"
+// Fixed by dcde8acc32f1355f37d3bc2814c528fdc2ca5f94
+
 int foo(int a, int b) {
   a += b; b -= a;
   a += b; b -= a;
diff --git a/clang/test/Analysis/aggrinit-cfg-output.cpp b/clang/test/Analysis/aggrinit-cfg-output.cpp
index 2e95a7f9e24f..682399399d21 100644
--- a/clang/test/Analysis/aggrinit-cfg-output.cpp
+++ b/clang/test/Analysis/aggrinit-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-expand-default-aggr-inits=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCFG -analyzer-config cfg-expand-default-aggr-inits=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 static char a[] = "foobar";
diff --git a/clang/test/Analysis/analyzer_test.py b/clang/test/Analysis/analyzer_test.py
deleted file mode 100644
index c476ceea9a59..000000000000
--- a/clang/test/Analysis/analyzer_test.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import lit.formats
-import lit.TestRunner
-
-# Custom format class for static analyzer tests
-class AnalyzerTest(lit.formats.ShTest):
-    def __init__(self, execute_external, use_z3_solver=False):
-        super(AnalyzerTest, self).__init__(execute_external)
-        self.use_z3_solver = use_z3_solver
-
-    def execute(self, test, litConfig):
-        results = []
-
-        # Parse any test requirements ('REQUIRES: ')
-        saved_test = test
-        lit.TestRunner.parseIntegratedTestScript(test)
-
-        if "z3" not in test.requires:
-            results.append(
-                self.executeWithAnalyzeSubstitution(
-                    saved_test, litConfig, "-analyzer-constraints=range"
-                )
-            )
-
-            if results[-1].code == lit.Test.FAIL:
-                return results[-1]
-
-        # If z3 backend available, add an additional run line for it
-        if self.use_z3_solver == "1":
-            assert test.config.clang_staticanalyzer_z3 == "1"
-            results.append(
-                self.executeWithAnalyzeSubstitution(
-                    saved_test, litConfig, "-analyzer-constraints=z3 -DANALYZER_CM_Z3"
-                )
-            )
-
-        # Combine all result outputs into the last element
-        for x in results:
-            if x != results[-1]:
-                results[-1].output = x.output + results[-1].output
-
-        if results:
-            return results[-1]
-        return lit.Test.Result(
-            lit.Test.UNSUPPORTED, "Test requires the following unavailable features: z3"
-        )
-
-    def executeWithAnalyzeSubstitution(self, test, litConfig, substitution):
-        saved_substitutions = list(test.config.substitutions)
-        test.config.substitutions.append(("%analyze", substitution))
-        result = lit.TestRunner.executeShTest(test, litConfig, self.execute_external)
-        test.config.substitutions = saved_substitutions
-
-        return result
diff --git a/clang/test/Analysis/bool-assignment.c b/clang/test/Analysis/bool-assignment.c
index 3a104cf627ff..f0942cc4d349 100644
--- a/clang/test/Analysis/bool-assignment.c
+++ b/clang/test/Analysis/bool-assignment.c
@@ -43,11 +43,7 @@ void test_BOOL_initialization(int y) {
     return;
   }
   if (y > 200 && y < 250) {
-#ifdef ANALYZER_CM_Z3
-    BOOL x = y; // expected-warning {{Assignment of a non-Boolean value}}
-#else
     BOOL x = y; // no-warning
-#endif
     return;
   }
   if (y >= 127 && y < 150) {
diff --git a/clang/test/Analysis/call-and-message.c b/clang/test/Analysis/call-and-message.c
index b79ec8c344b6..ade51145e2a9 100644
--- a/clang/test/Analysis/call-and-message.c
+++ b/clang/test/Analysis/call-and-message.c
@@ -1,12 +1,19 @@
 // RUN: %clang_analyze_cc1 %s -verify \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=true \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=false \
 // RUN:   -analyzer-output=plist -o %t.plist
 // RUN: cat %t.plist | FileCheck %s
 
 // RUN: %clang_analyze_cc1 %s -verify=no-pointee \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false
+// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=false
+
+// RUN: %clang_analyze_cc1 %s -verify=arg-init \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=false \
+// RUN:   -analyzer-config core.CallAndMessage:ArgInitializedness=true
 
 // no-pointee-no-diagnostics
 
@@ -22,3 +29,21 @@ void pointee_uninit(void) {
 // checker, as described in the CallAndMessage comments!
 // CHECK: <key>issue_hash_content_of_line_in_context</key>
 // CHECK-SAME: <string>97a74322d64dca40aa57303842c745a1</string>
+
+typedef struct {
+  int i  :2;
+  int    :30;  // unnamed bit-field
+} B;
+
+extern void consume_B(B);
+
+void bitfield_B_init(void) {
+  B b1;
+  b1.i = 1; // b1 is initialized
+  consume_B(b1);
+}
+
+void bitfield_B_uninit(void) {
+  B b2;
+  consume_B(b2); // arg-init-warning{{Passed-by-value struct argument contains uninitialized data (e.g., field: 'i') [core.CallAndMessage]}}
+}
diff --git a/clang/test/Analysis/call-and-message.cpp b/clang/test/Analysis/call-and-message.cpp
index 25ae23833478..1e76973f49e1 100644
--- a/clang/test/Analysis/call-and-message.cpp
+++ b/clang/test/Analysis/call-and-message.cpp
@@ -169,4 +169,20 @@ void record_uninit() {
 // CHECK-SAME: <string>a46bb5c1ee44d4611ffeb13f7f499605</string>
 // CHECK: <key>issue_hash_content_of_line_in_context</key>
 // CHECK-SAME: <string>e0e0d30ea5a7b2e3a71e1931fa0768a5</string>
+
+struct B{
+  int i  :2;
+  int    :30;  // unnamed bit-field
+};
+
+void bitfield_B_init(void) {
+  B b1;
+  b1.i = 1; // b1 is initialized
+  consume(b1);
+}
+
+void bitfield_B_uninit(void) {
+  B b2;
+  consume(b2); // arg-init-warning{{Passed-by-value struct argument contains uninitialized data (e.g., field: 'i') [core.CallAndMessage]}}
+}
 } // namespace uninit_arg
diff --git a/clang/test/Analysis/compound-literals.c b/clang/test/Analysis/compound-literals.c
index c74eacc3aa68..90ef0ceb46b2 100644
--- a/clang/test/Analysis/compound-literals.c
+++ b/clang/test/Analysis/compound-literals.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple=i386-apple-darwin10 -verify %s -analyze \
+// RUN: %clang_analyze_cc1 -triple=i386-apple-darwin10 -verify %s \
 // RUN:   -analyzer-checker=debug.ExprInspection
 
 #define NULL 0
diff --git a/clang/test/Analysis/cstring-addrspace.c b/clang/test/Analysis/cstring-addrspace.c
index d6b455510e36..fc00a7802657 100644
--- a/clang/test/Analysis/cstring-addrspace.c
+++ b/clang/test/Analysis/cstring-addrspace.c
@@ -1,6 +1,5 @@
 // RUN: %clang_analyze_cc1 -triple amdgcn-unknown-unknown \
-// RUN: -analyze -analyzer-checker=core,alpha.unix.cstring \
-// RUN: -analyze -analyzer-checker=debug.ExprInspection \
+// RUN: -analyzer-checker=core,alpha.unix.cstring,debug.ExprInspection \
 // RUN: -analyzer-config crosscheck-with-z3=true -verify %s \
 // RUN: -Wno-incompatible-library-redeclaration
 // REQUIRES: z3
diff --git a/clang/test/Analysis/cstring-should-not-invalidate.cpp b/clang/test/Analysis/cstring-should-not-invalidate.cpp
new file mode 100644
index 000000000000..abd4689b2f9b
--- /dev/null
+++ b/clang/test/Analysis/cstring-should-not-invalidate.cpp
@@ -0,0 +1,132 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.cstring,alpha.unix.cstring,debug.ExprInspection -verify %s
+
+typedef decltype(sizeof(int)) size_t;
+void clang_analyzer_eval(bool);
+
+char *strncpy(char *dest, const char *src, size_t x);
+
+constexpr int initB = 100;
+struct Base {
+  int b;
+  Base(): b(initB) {}
+};
+
+// issue 143807
+struct strncpyTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strncpy(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strncpy(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strncpyTest(char *src, size_t n) {
+  strncpyTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+size_t strlcpy(char *dest, const char *src, size_t size);
+
+struct strlcpyTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strlcpy(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strlcpy(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strlcpyTest(char *src, size_t n) {
+  strlcpyTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+char *strncat(char *s1, const char *s2, size_t n);
+
+struct strncatTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strncat(m_buff, src, sizeof(m_buff) - 1); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strncat(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strncatTest(char *src, size_t n) {
+  strncatTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
+
+struct strncatReportOutOfBoundTestClass {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    // expected-warning@+1{{String concatenation function overflows the destination buffer}}
+    strncat(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+};
+
+void strncatReportOutOfBoundTest(char *src, size_t n) {
+  strncatReportOutOfBoundTestClass rep;
+  rep.KnownLen(src);
+}
+
+size_t strlcat(char *dst, const char *src, size_t size);
+
+struct strlcatTestClass: public Base {
+  int *m_ptr;
+  char m_buff[1000];
+
+  void KnownLen(char *src) {
+    m_ptr = new int;
+    strlcat(m_buff, src, sizeof(m_buff)); // known len but unknown src size
+    delete m_ptr;                         // no warning
+  }
+
+  void KnownSrcLen(size_t n) {
+    m_ptr = new int;
+    strlcat(m_buff, "xyz", n); // known src size but unknown len
+    delete m_ptr;              // no warning
+  }
+};
+
+void strlcatTest(char *src, size_t n) {
+  strlcatTestClass rep;
+  rep.KnownLen(src);
+  rep.KnownSrcLen(n);
+  clang_analyzer_eval(rep.b == initB); // expected-warning{{TRUE}}
+}
diff --git a/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp b/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
index 18f03ee6b010..995db5614527 100644
--- a/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
+++ b/clang/test/Analysis/ctu-cxxdefaultinitexpr.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -emit-pch -o %t/ctudir/ctu-cxxdefaultinitexpr-import.cpp.ast %S/Inputs/ctu-cxxdefaultinitexpr-import.cpp
 // RUN: cp %S/Inputs/ctu-cxxdefaultinitexpr-import.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=%t/ctudir \
diff --git a/clang/test/Analysis/ctu-implicit.c b/clang/test/Analysis/ctu-implicit.c
index c0b28fc25bd2..63df5fb0542b 100644
--- a/clang/test/Analysis/ctu-implicit.c
+++ b/clang/test/Analysis/ctu-implicit.c
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1  \
 // RUN:   -emit-pch -o %t/ctudir2/ctu-import.c.ast %S/Inputs/ctu-import.c
 // RUN: cp %S/Inputs/ctu-import.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
diff --git a/clang/test/Analysis/ctu-import-threshold.c b/clang/test/Analysis/ctu-import-threshold.c
index 62bbeda20d5b..f7e1285d595e 100644
--- a/clang/test/Analysis/ctu-import-threshold.c
+++ b/clang/test/Analysis/ctu-import-threshold.c
@@ -1,6 +1,6 @@
 // Ensure analyzer option 'ctu-import-threshold' is a recognized option.
 //
-// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s
-// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-cpp-threshold=30 -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-config ctu-import-threshold=30 -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-config ctu-import-cpp-threshold=30 -verify %s
 //
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/ctu-lookup-name-with-space.cpp b/clang/test/Analysis/ctu-lookup-name-with-space.cpp
index ef733d5c52e3..0761cfe09229 100644
--- a/clang/test/Analysis/ctu-lookup-name-with-space.cpp
+++ b/clang/test/Analysis/ctu-lookup-name-with-space.cpp
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -emit-pch %/S/Inputs/ctu-lookup-name-with-space.cpp -o %t/importee.ast
 
 // RUN: cd %t
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=. \
diff --git a/clang/test/Analysis/ctu-main.c b/clang/test/Analysis/ctu-main.c
index 0193a7e6d135..e24510e8aadc 100644
--- a/clang/test/Analysis/ctu-main.c
+++ b/clang/test/Analysis/ctu-main.c
@@ -4,7 +4,7 @@
 // RUN:   -emit-pch -o %t/ctudir2/ctu-other.c.ast %S/Inputs/ctu-other.c
 // RUN: cp %S/Inputs/ctu-other.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c89 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
@@ -14,7 +14,7 @@
 
 // Simulate the behavior of the previous CTU implementation by inlining all
 // functions during the first phase. This way, the second phase is a noop.
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c89 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config eagerly-assume=false \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
diff --git a/clang/test/Analysis/ctu-on-demand-parsing.c b/clang/test/Analysis/ctu-on-demand-parsing.c
index 17ade150ded5..2107b989d9cc 100644
--- a/clang/test/Analysis/ctu-on-demand-parsing.c
+++ b/clang/test/Analysis/ctu-on-demand-parsing.c
@@ -11,7 +11,7 @@
 //
 // RUN: cd "%t" && %clang_extdef_map "%t/ctu-other.c" > externalDefMap.txt
 //
-// RUN: cd "%t" && %clang_cc1 -std=c89 -analyze \
+// RUN: cd "%t" && %clang_analyze_cc1 -std=c89 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=. \
diff --git a/clang/test/Analysis/ctu-test-import-failure.cpp b/clang/test/Analysis/ctu-test-import-failure.cpp
index 2295a66538fc..812fa01b6c67 100644
--- a/clang/test/Analysis/ctu-test-import-failure.cpp
+++ b/clang/test/Analysis/ctu-test-import-failure.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -emit-pch -o %t/ctudir/ctu-test-import-failure-import.cpp.ast %S/Inputs/ctu-test-import-failure-import.cpp
 // RUN: cp %S/Inputs/ctu-test-import-failure-import.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -analyze \
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -std=c++17 \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-config experimental-enable-naive-ctu-analysis=true \
 // RUN:   -analyzer-config ctu-dir=%t/ctudir \
diff --git a/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp b/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
index 01914a86ad4a..76e67a8690bd 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/dynamic_types.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct S {};
diff --git a/clang/test/Analysis/exploded-graph-rewriter/escapes.c b/clang/test/Analysis/exploded-graph-rewriter/escapes.c
index 830e082730a8..a3a6868cf8df 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/escapes.c
+++ b/clang/test/Analysis/exploded-graph-rewriter/escapes.c
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 void escapes(void) {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp b/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
index 82b7f3269f31..52f23e298ec5 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/initializers_under_construction.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct A {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
index f18c4c5afe14..3798846c8873 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/l_name_starts_with_l.cpp
@@ -1,8 +1,7 @@
 // CAUTION: The name of this file should start with `l` for proper tests.
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 void test1() {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/macros.c b/clang/test/Analysis/exploded-graph-rewriter/macros.c
index f47342fc524b..32b8d55d553e 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/macros.c
+++ b/clang/test/Analysis/exploded-graph-rewriter/macros.c
@@ -6,10 +6,9 @@ void *foo(void) {
 // The code above shall go first, because check tags below are sensetive to the line numbers on which the code is placed.
 // You can change lines below in the way you need.
 
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 // CHECK: macros.c:<b>3</b>:<b>10</b>
diff --git a/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp b/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
index f2628ee68663..145a56c1c44f 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/objects_under_construction.cpp
@@ -1,8 +1,7 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -std=c++14 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyze-function "test()" \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyze-function "test()" \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter %t.dot | FileCheck %s
 
 struct S {
diff --git a/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
index 5b955a0aae3e..2c1c934d264f 100644
--- a/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
+++ b/clang/test/Analysis/exploded-graph-rewriter/win_path_forbidden_chars.cpp
@@ -1,7 +1,6 @@
-// FIXME: Figure out how to use %clang_analyze_cc1 with our lit.local.cfg.
-// RUN: %clang_cc1 -analyze -triple x86_64-unknown-linux-gnu \
-// RUN:                     -analyzer-checker=core \
-// RUN:                     -analyzer-dump-egraph=%t.dot %s
+// RUN: %clang_analyze_cc1 -triple x86_64-unknown-linux-gnu \
+// RUN:                    -analyzer-checker=core \
+// RUN:                    -analyzer-dump-egraph=%t.dot %s
 // RUN: %exploded_graph_rewriter --verbose %t.dot 2>&1 | FileCheck %s
 // REQUIRES: system-windows
 
diff --git a/clang/test/Analysis/getline-unixapi-invalid-signatures.c b/clang/test/Analysis/getline-unixapi-invalid-signatures.c
new file mode 100644
index 000000000000..c906625b4763
--- /dev/null
+++ b/clang/test/Analysis/getline-unixapi-invalid-signatures.c
@@ -0,0 +1,127 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_CORRECT
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_1
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_2
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_3
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_4
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_5
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix -verify %s -DTEST_GETLINE_GH144884
+
+// emulator of "system-header-simulator.h" because of redefinition of 'getline' function
+typedef struct _FILE FILE;
+typedef __typeof(sizeof(int)) size_t;
+typedef long ssize_t;
+#define NULL 0
+
+int fclose(FILE *fp);
+FILE *tmpfile(void);
+
+#ifdef TEST_CORRECT
+ssize_t getline(char **lineptr, size_t *n, FILE *stream);
+ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream);
+
+void test_correct() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getline(&buffer, NULL, F1); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+
+void test_delim_correct() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getdelim(&buffer, NULL, ',', F1); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_1
+// expected-no-diagnostics
+ssize_t getline(int lineptr);
+
+void test() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  int buffer = 0;
+  getline(buffer);
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_2
+ssize_t getline(char **lineptr, size_t *n);
+
+void test() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getline(&buffer, NULL); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_3
+// expected-no-diagnostics
+ssize_t getline(char **lineptr, size_t n, FILE *stream);
+
+void test() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getline(&buffer, 0, F1);
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_4
+ssize_t getline(char **lineptr, size_t *n, int stream);
+ssize_t getdelim(char **lineptr, size_t *n, int delimiter, int stream);
+
+void test() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getline(&buffer, NULL, 1); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+
+void test_delim() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getdelim(&buffer, NULL, ',', 1); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_5
+ssize_t getdelim(char **lineptr, size_t *n, const char* delimiter, FILE *stream);
+
+void test_delim() {
+  FILE *F1 = tmpfile();
+  if (!F1)
+    return;
+  char *buffer = NULL;
+  getdelim(&buffer, NULL, ",", F1); // expected-warning {{Size pointer might be NULL}}
+  fclose(F1);
+}
+#endif
+
+#ifdef TEST_GETLINE_GH144884
+// expected-no-diagnostics
+struct AW_string {};
+void getline(int *, struct AW_string);
+void top() {
+  struct AW_string line;
+  int getline_file_info;
+  getline(&getline_file_info, line);
+}
+#endif
diff --git a/clang/test/Analysis/gmalloc.c b/clang/test/Analysis/gmalloc.c
index 279e2ae844b7..dae28ebfa910 100644
--- a/clang/test/Analysis/gmalloc.c
+++ b/clang/test/Analysis/gmalloc.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=core,unix.Malloc -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify %s
 
 #include "Inputs/system-header-simulator.h"
 
diff --git a/clang/test/Analysis/handle_constructors_for_default_arguments.cpp b/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
index e20ec5e57d3a..2a934be2e8d3 100644
--- a/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
+++ b/clang/test/Analysis/handle_constructors_for_default_arguments.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s -verify
 
 // These test cases demonstrate lack of Static Analyzer features.
diff --git a/clang/test/Analysis/handle_constructors_with_new_array.cpp b/clang/test/Analysis/handle_constructors_with_new_array.cpp
index d55d354f9c5c..4acdaaf45490 100644
--- a/clang/test/Analysis/handle_constructors_with_new_array.cpp
+++ b/clang/test/Analysis/handle_constructors_with_new_array.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze \
+// RUN: %clang_analyze_cc1 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s -verify
 
 // These test cases demonstrate lack of Static Analyzer features.
diff --git a/clang/test/Analysis/initialization.c b/clang/test/Analysis/initialization.c
index bd003b721d91..32bdf733bcb9 100644
--- a/clang/test/Analysis/initialization.c
+++ b/clang/test/Analysis/initialization.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple i386-apple-darwin10 -analyze -analyzer-config eagerly-assume=false  -analyzer-checker=core.uninitialized.Assign,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-config eagerly-assume=false  -analyzer-checker=core.uninitialized.Assign,debug.ExprInspection -verify %s
 
 void clang_analyzer_eval(int);
 
diff --git a/clang/test/Analysis/initialization.cpp b/clang/test/Analysis/initialization.cpp
index e5e19daad089..d064dd6172d6 100644
--- a/clang/test/Analysis/initialization.cpp
+++ b/clang/test/Analysis/initialization.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++14 -triple i386-apple-darwin10 -analyze -analyzer-config eagerly-assume=false -analyzer-checker=core.uninitialized.Assign,core.builtin,debug.ExprInspection,core.uninitialized.UndefReturn -verify %s
+// RUN: %clang_analyze_cc1 -std=c++14 -triple i386-apple-darwin10 -analyzer-config eagerly-assume=false -analyzer-checker=core.uninitialized.Assign,core.builtin,debug.ExprInspection,core.uninitialized.UndefReturn -verify %s
 
 template <typename T>
 void clang_analyzer_dump(T x);
diff --git a/clang/test/Analysis/lambda-convert-to-func-ptr.cpp b/clang/test/Analysis/lambda-convert-to-func-ptr.cpp
new file mode 100644
index 000000000000..c2ad7cd2de34
--- /dev/null
+++ b/clang/test/Analysis/lambda-convert-to-func-ptr.cpp
@@ -0,0 +1,21 @@
+// RUN: %clang_analyze_cc1 -std=c++11 -analyzer-checker=core,debug.ExprInspection -analyzer-config inline-lambdas=true -verify %s
+
+void clang_analyzer_eval(bool);
+
+void basic() {
+  int (*ret_zero)() = []() { return 0; };
+  clang_analyzer_eval(ret_zero() == 0); // expected-warning{{TRUE}}
+}
+
+void withParam() {
+  int (*add_ten)(int) = [](int b) { return b + 10; };
+  clang_analyzer_eval(add_ten(1) == 11); // expected-warning{{TRUE}}
+}
+
+int callBack(int (*fp)(int), int x) {
+  return fp(x);
+}
+
+void passWithFunc() {
+  clang_analyzer_eval(callBack([](int x) { return x; }, 5) == 5); // expected-warning{{TRUE}}
+}
diff --git a/clang/test/Analysis/lifetime-cfg-output.cpp b/clang/test/Analysis/lifetime-cfg-output.cpp
index d8b8dc4a71d4..0a75c5bcc0bc 100644
--- a/clang/test/Analysis/lifetime-cfg-output.cpp
+++ b/clang/test/Analysis/lifetime-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-lifetime=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -fcxx-exceptions -fexceptions -analyzer-checker=debug.DumpCFG -analyzer-config cfg-lifetime=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 // FIXME Most of the cases in this file test only with the objects of type `A`,
diff --git a/clang/test/Analysis/lit.local.cfg b/clang/test/Analysis/lit.local.cfg
index e2946cf345cb..f08ff8d6cce6 100644
--- a/clang/test/Analysis/lit.local.cfg
+++ b/clang/test/Analysis/lit.local.cfg
@@ -1,15 +1,7 @@
 # -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
-from lit.llvm.subst import ToolSubst
-import site
+import lit.formats
 
-# Load the custom analyzer test format, which runs the test again with Z3 if it
-# is available.
-site.addsitedir(os.path.dirname(__file__))
-import analyzer_test
-
-config.test_format = analyzer_test.AnalyzerTest(
-    config.test_format.execute_external, config.use_z3_solver
-)
+config.test_format = lit.formats.ShTest(config.test_format.execute_external)
 
 # Filtering command used by Clang Analyzer tests (when comparing .plist files
 # with reference output)
diff --git a/clang/test/Analysis/loopexit-cfg-output.cpp b/clang/test/Analysis/loopexit-cfg-output.cpp
index 8e53ce3066b7..26ef33e4a6a9 100644
--- a/clang/test/Analysis/loopexit-cfg-output.cpp
+++ b/clang/test/Analysis/loopexit-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-loopexit=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCFG -analyzer-config cfg-loopexit=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 // CHECK:       [B6 (ENTRY)]
diff --git a/clang/test/Analysis/method-arg-decay.m b/clang/test/Analysis/method-arg-decay.m
index a222346861a5..69fd7a241b71 100644
--- a/clang/test/Analysis/method-arg-decay.m
+++ b/clang/test/Analysis/method-arg-decay.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyzer-checker=core -verify %s -Wno-incomplete-implementation
+// RUN: %clang_analyze_cc1 -analyzer-checker=core -verify %s -Wno-incomplete-implementation
 typedef signed char BOOL;
 typedef int NSInteger;
 typedef unsigned int NSUInteger;
diff --git a/clang/test/Analysis/null-deref-static.m b/clang/test/Analysis/null-deref-static.m
index 06a97644aa1d..2bfb4af29241 100644
--- a/clang/test/Analysis/null-deref-static.m
+++ b/clang/test/Analysis/null-deref-static.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -w -fblocks -analyze -analyzer-checker=core,deadcode,alpha.core,debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -w -fblocks -analyzer-checker=core,deadcode,alpha.core,debug.ExprInspection -verify %s
 
 void *malloc(unsigned long);
 void clang_analyzer_warnIfReached(void);
diff --git a/clang/test/Analysis/objc-encode.m b/clang/test/Analysis/objc-encode.m
index 1d6515c2405a..afa71c3a9432 100644
--- a/clang/test/Analysis/objc-encode.m
+++ b/clang/test/Analysis/objc-encode.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.ExprInspection -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.ExprInspection -verify %s
 // expected-no-diagnostics
 
 void clang_analyzer_eval(int);
diff --git a/clang/test/Analysis/openmp-unsupported.c b/clang/test/Analysis/openmp-unsupported.c
index 85dd4b66ba87..c6769bd166db 100644
--- a/clang/test/Analysis/openmp-unsupported.c
+++ b/clang/test/Analysis/openmp-unsupported.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple i386-apple-darwin10 -analyze -analyzer-checker=core.builtin -fopenmp -verify %s
+// RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -analyzer-checker=core.builtin -fopenmp -verify %s
 // expected-no-diagnostics
 
 void openmp_parallel_crash_test(void) {
diff --git a/clang/test/Analysis/pointer-escape-on-conservative-calls.c b/clang/test/Analysis/pointer-escape-on-conservative-calls.c
index 19d86ce1866f..5de2b1cf3108 100644
--- a/clang/test/Analysis/pointer-escape-on-conservative-calls.c
+++ b/clang/test/Analysis/pointer-escape-on-conservative-calls.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:PointerEscape=true -analyzer-config debug.AnalysisOrder:PostCall=true %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:PointerEscape=true -analyzer-config debug.AnalysisOrder:PostCall=true %s 2>&1 | FileCheck %s
 
 
 void f(int *);
diff --git a/clang/test/Analysis/print-ranges.cpp b/clang/test/Analysis/print-ranges.cpp
index 89a4249ebfaa..8964f68dbfee 100644
--- a/clang/test/Analysis/print-ranges.cpp
+++ b/clang/test/Analysis/print-ranges.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,debug.ExprInspection -analyzer-config eagerly-assume=false -verify %s
-// UNSUPPORTED z3
+// UNSUPPORTED: z3
 
 template <typename T>
 void clang_analyzer_value(T x);
diff --git a/clang/test/Analysis/ptr-arith.c b/clang/test/Analysis/ptr-arith.c
index b5ccd2c207ea..79ca190b25a0 100644
--- a/clang/test/Analysis/ptr-arith.c
+++ b/clang/test/Analysis/ptr-arith.c
@@ -221,12 +221,7 @@ void comparisons_imply_size(int *lhs, int *rhs) {
   }
 
   clang_analyzer_eval(lhs <= rhs); // expected-warning{{TRUE}}
-// FIXME: In Z3ConstraintManager, ptrdiff_t is mapped to signed bitvector. However, this does not directly imply the unsigned comparison.
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval((rhs - lhs) >= 0); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval((rhs - lhs) >= 0); // expected-warning{{TRUE}}
-#endif
   clang_analyzer_eval((rhs - lhs) > 0); // expected-warning{{UNKNOWN}}
 
   if (lhs >= rhs) {
@@ -236,11 +231,7 @@ void comparisons_imply_size(int *lhs, int *rhs) {
 
   clang_analyzer_eval(lhs == rhs); // expected-warning{{FALSE}}
   clang_analyzer_eval(lhs < rhs); // expected-warning{{TRUE}}
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval((rhs - lhs) > 0); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval((rhs - lhs) > 0); // expected-warning{{TRUE}}
-#endif
 }
 
 void size_implies_comparison(int *lhs, int *rhs) {
@@ -251,11 +242,7 @@ void size_implies_comparison(int *lhs, int *rhs) {
     return;
   }
 
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval(lhs <= rhs); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval(lhs <= rhs); // expected-warning{{TRUE}}
-#endif
   clang_analyzer_eval((rhs - lhs) >= 0); // expected-warning{{TRUE}}
   clang_analyzer_eval((rhs - lhs) > 0); // expected-warning{{UNKNOWN}}
 
@@ -265,11 +252,7 @@ void size_implies_comparison(int *lhs, int *rhs) {
   }
 
   clang_analyzer_eval(lhs == rhs); // expected-warning{{FALSE}}
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval(lhs < rhs); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval(lhs < rhs); // expected-warning{{TRUE}}
-#endif
   clang_analyzer_eval((rhs - lhs) > 0); // expected-warning{{TRUE}}
 }
 
diff --git a/clang/test/Analysis/reference.cpp b/clang/test/Analysis/reference.cpp
index 91ba3cd23249..e6986fb8e705 100644
--- a/clang/test/Analysis/reference.cpp
+++ b/clang/test/Analysis/reference.cpp
@@ -123,26 +123,10 @@ S getS();
 S *getSP();
 
 void testReferenceAddress(int &x) {
-// FIXME: Move non-zero reference assumption out of RangeConstraintManager.cpp:422
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval(&x != 0); // expected-warning{{UNKNOWN}}
-  clang_analyzer_eval(&ref() != 0); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval(&x != 0); // expected-warning{{TRUE}}
   clang_analyzer_eval(&ref() != 0); // expected-warning{{TRUE}}
-#endif
-
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval(&getS().x != 0); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval(&getS().x != 0); // expected-warning{{TRUE}}
-#endif
-
-#ifdef ANALYZER_CM_Z3
-  clang_analyzer_eval(&getSP()->x != 0); // expected-warning{{UNKNOWN}}
-#else
   clang_analyzer_eval(&getSP()->x != 0); // expected-warning{{TRUE}}
-#endif
 }
 }
 
diff --git a/clang/test/Analysis/scopes-cfg-output.cpp b/clang/test/Analysis/scopes-cfg-output.cpp
index c082bb179545..6ed6f3638f75 100644
--- a/clang/test/Analysis/scopes-cfg-output.cpp
+++ b/clang/test/Analysis/scopes-cfg-output.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fcxx-exceptions -fexceptions -analyze -analyzer-checker=debug.DumpCFG -analyzer-config cfg-scopes=true %s > %t 2>&1
+// RUN: %clang_analyze_cc1 -fcxx-exceptions -fexceptions -analyzer-checker=debug.DumpCFG -analyzer-config cfg-scopes=true %s > %t 2>&1
 // RUN: FileCheck --input-file=%t %s
 
 class A {
diff --git a/clang/test/Analysis/taint-diagnostic-visitor.c b/clang/test/Analysis/taint-diagnostic-visitor.c
index afe7117db715..7bbc1faa27c4 100644
--- a/clang/test/Analysis/taint-diagnostic-visitor.c
+++ b/clang/test/Analysis/taint-diagnostic-visitor.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=optin.taint,core,security.ArrayBound -analyzer-output=text -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=optin.taint,core,security.ArrayBound -analyzer-output=text -verify %s
 
 // This file is for testing enhanced diagnostics produced by the GenericTaintChecker
 
diff --git a/clang/test/Analysis/temporaries-callback-order.cpp b/clang/test/Analysis/temporaries-callback-order.cpp
index 120fabc10478..00a79c44e222 100644
--- a/clang/test/Analysis/temporaries-callback-order.cpp
+++ b/clang/test/Analysis/temporaries-callback-order.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:Bind=true -analyzer-config debug.AnalysisOrder:RegionChanges=true %s 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.AnalysisOrder -analyzer-config debug.AnalysisOrder:Bind=true -analyzer-config debug.AnalysisOrder:RegionChanges=true %s 2>&1 | FileCheck %s
 
 struct Super {
   virtual void m();
diff --git a/clang/test/Analysis/unary-sym-expr-z3-refutation.c b/clang/test/Analysis/unary-sym-expr-z3-refutation.c
index 33585236232a..f103fdbd285b 100644
--- a/clang/test/Analysis/unary-sym-expr-z3-refutation.c
+++ b/clang/test/Analysis/unary-sym-expr-z3-refutation.c
@@ -29,5 +29,5 @@ void k(long L) {
   int h = g + 1;
   int j;
   j += -h < 0; // should not crash
-  // expected-warning@-1{{garbage}}
+  // expected-warning@-1{{The left expression of the compound assignment uses uninitialized memory [core.uninitialized.Assign]}}
 }
diff --git a/clang/test/Analysis/undef-call.c b/clang/test/Analysis/undef-call.c
index 8019028282aa..3afdb6760d1f 100644
--- a/clang/test/Analysis/undef-call.c
+++ b/clang/test/Analysis/undef-call.c
@@ -1,6 +1,6 @@
 // RUN: rm -rf %T/ctudir
 // RUN: mkdir %T/ctudir
-// RUN: %clang_cc1 -analyze -analyzer-checker=debug.ExprInspection -analyzer-config experimental-enable-naive-ctu-analysis=true -analyzer-config ctu-dir=%T/ctudir -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.ExprInspection -analyzer-config experimental-enable-naive-ctu-analysis=true -analyzer-config ctu-dir=%T/ctudir -verify %s
 // expected-no-diagnostics
 
 struct S {
diff --git a/clang/test/Analysis/z3-crosscheck-max-attempts.cpp b/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
index 7951d2654647..8439236883f1 100644
--- a/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
+++ b/clang/test/Analysis/z3-crosscheck-max-attempts.cpp
@@ -3,15 +3,10 @@
 // RUN: | FileCheck %s --match-full-lines
 // CHECK: crosscheck-with-z3-max-attempts-per-query = 3
 
-// RUN: rm -rf %t && mkdir %t
-// RUN: %host_cxx -shared -fPIC                           \
-// RUN:   %S/z3/Inputs/MockZ3_solver_check.cpp            \
-// RUN:   -o %t/MockZ3_solver_check.so
-
-// DEFINE: %{mocked_clang} =                              \
-// DEFINE: LD_PRELOAD="%t/MockZ3_solver_check.so"         \
-// DEFINE: %clang_cc1 %s -analyze -setup-static-analyzer  \
-// DEFINE:   -analyzer-config crosscheck-with-z3=true     \
+// DEFINE: %{mocked_clang} =                                      \
+// DEFINE: LD_PRELOAD="%llvmshlibdir/MockZ3SolverCheck%pluginext" \
+// DEFINE: %clang_analyze_cc1 %s                                  \
+// DEFINE:   -analyzer-config crosscheck-with-z3=true             \
 // DEFINE:   -analyzer-checker=core
 
 // DEFINE: %{attempts} = -analyzer-config crosscheck-with-z3-max-attempts-per-query
@@ -32,7 +27,7 @@
 // RUN: Z3_SOLVER_RESULTS="UNDEF,UNDEF,SAT"   %{mocked_clang} %{attempts}=3 -verify=accepted
 
 
-// REQUIRES: z3, asserts, shell, system-linux
+// REQUIRES: z3, z3-mock, asserts, shell, system-linux
 
 // refuted-no-diagnostics
 
diff --git a/clang/test/Analysis/z3-crosscheck.c b/clang/test/Analysis/z3-crosscheck.c
index 13f38f43e697..8cbe7dd2975d 100644
--- a/clang/test/Analysis/z3-crosscheck.c
+++ b/clang/test/Analysis/z3-crosscheck.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -analyze -analyzer-checker=core,unix.Malloc,debug.ExprInspection -DNO_CROSSCHECK -verify %s
-// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -analyze -analyzer-checker=core,unix.Malloc,debug.ExprInspection -analyzer-config crosscheck-with-z3=true -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -analyzer-checker=core,unix.Malloc,debug.ExprInspection -DNO_CROSSCHECK -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu -analyzer-checker=core,unix.Malloc,debug.ExprInspection -analyzer-config crosscheck-with-z3=true -verify %s
 // REQUIRES: z3
 
 void clang_analyzer_dump(float);
@@ -64,7 +64,7 @@ void floatUnaryNegInEq(int h, int l) {
   clang_analyzer_dump((float)l); // expected-warning-re {{(float) (reg_${{[0-9]+}}<int l>)}}
   if (-(float)h != (float)l) {  // should not crash
     j += 10;
-    // expected-warning@-1{{garbage}}
+    // expected-warning@-1{{The left expression of the compound assignment uses uninitialized memory [core.uninitialized.Assign]}}
   }
 }
 
@@ -74,7 +74,7 @@ void floatUnaryLNotInEq(int h, int l) {
   clang_analyzer_dump((float)l); // expected-warning-re {{(float) (reg_${{[0-9]+}}<int l>)}}
   if ((!(float)h) != (float)l) {  // should not crash
     j += 10;
-    // expected-warning@-1{{garbage}}
+    // expected-warning@-1{{The left expression of the compound assignment uses uninitialized memory [core.uninitialized.Assign]}}
   }
 }
 
diff --git a/clang/test/Analysis/z3/D83660.c b/clang/test/Analysis/z3/D83660.c
index b42d934bcc9e..16ea4ffb0e1b 100644
--- a/clang/test/Analysis/z3/D83660.c
+++ b/clang/test/Analysis/z3/D83660.c
@@ -1,14 +1,9 @@
-// RUN: rm -rf %t && mkdir %t
-// RUN: %host_cxx -shared -fPIC \
-// RUN:   %S/Inputs/MockZ3_solver_check.cpp \
-// RUN:   -o %t/MockZ3_solver_check.so
-//
 // RUN: Z3_SOLVER_RESULTS="SAT,SAT,SAT,SAT,UNDEF" \
-// RUN: LD_PRELOAD="%t/MockZ3_solver_check.so" \
-// RUN: %clang_cc1 -analyze -analyzer-constraints=z3 -setup-static-analyzer \
+// RUN: LD_PRELOAD="%llvmshlibdir/MockZ3SolverCheck%pluginext" \
+// RUN: %clang_analyze_cc1 -analyzer-constraints=z3 \
 // RUN:   -analyzer-checker=core %s -verify
 //
-// REQUIRES: z3, asserts, shell, system-linux
+// REQUIRES: z3, z3-mock, asserts, shell, system-linux
 //
 // Works only with the z3 constraint manager.
 // expected-no-diagnostics
diff --git a/clang/test/Analysis/z3/crosscheck-statistics.c b/clang/test/Analysis/z3/crosscheck-statistics.c
index 8db3df169f24..cab31b18f50e 100644
--- a/clang/test/Analysis/z3/crosscheck-statistics.c
+++ b/clang/test/Analysis/z3/crosscheck-statistics.c
@@ -4,8 +4,6 @@
 
 // REQUIRES: z3
 
-// expected-error@1 {{Z3 refutation rate:1/2}}
-
 int accepting(int n) {
   if (n == 4) {
     n = n / (n-4); // expected-warning {{Division by zero}}
diff --git a/clang/test/Analysis/z3/enabled.c b/clang/test/Analysis/z3/enabled.c
deleted file mode 100644
index 9f44233b266c..000000000000
--- a/clang/test/Analysis/z3/enabled.c
+++ /dev/null
@@ -1,3 +0,0 @@
-// REQUIRES: z3
-// RUN: echo %clang_analyze_cc1 | FileCheck %s
-// CHECK: -analyzer-constraints=z3
diff --git a/clang/test/Analysis/z3/pretty-dump.c b/clang/test/Analysis/z3/pretty-dump.c
index 811da172e749..e6cd6d83373b 100644
--- a/clang/test/Analysis/z3/pretty-dump.c
+++ b/clang/test/Analysis/z3/pretty-dump.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-constraints=z3 -setup-static-analyzer \
+// RUN: %clang_analyze_cc1 -analyzer-constraints=z3 \
 // RUN:   -analyzer-checker=core,debug.ExprInspection %s 2>&1 | FileCheck %s
 //
 // REQUIRES: z3
diff --git a/clang/test/CIR/CodeGen/align-load.c b/clang/test/CIR/CodeGen/align-load.c
index 06553a307f93..17171d360754 100644
--- a/clang/test/CIR/CodeGen/align-load.c
+++ b/clang/test/CIR/CodeGen/align-load.c
@@ -21,7 +21,7 @@ void accessStruct(struct S u) {
   u.d;
 }
 
-// CIR: cir.func @accessStruct
+// CIR: cir.func{{.*}} @accessStruct
 // CIR:   cir.load align(8)
 // CIR:   cir.load align(2)
 // CIR:   cir.load align(4)
@@ -58,7 +58,7 @@ void accessUnion(union U u) {
   u.d;
 }
 
-// CIR: cir.func @accessUnion
+// CIR: cir.func{{.*}} @accessUnion
 // CIR:   cir.load align(8)
 // CIR:   cir.load align(8)
 // CIR:   cir.load align(8)
@@ -86,7 +86,7 @@ int loadAligned(myint *p) {
   return *p;
 }
 
-// CIR: cir.func @loadAligned
+// CIR: cir.func{{.*}} @loadAligned
 // CIR:   cir.load align(1)
 
 // LLVM: @loadAligned
diff --git a/clang/test/CIR/CodeGen/align-store.c b/clang/test/CIR/CodeGen/align-store.c
index 9ce26fa020ee..88686b94d8ad 100644
--- a/clang/test/CIR/CodeGen/align-store.c
+++ b/clang/test/CIR/CodeGen/align-store.c
@@ -12,7 +12,7 @@ void test1(myint *p) {
   *p = 0;
 }
 
-// CIR: cir.func @test1
+// CIR: cir.func{{.*}} @test1
 // CIR:   cir.store align(1)
 
 // LLVM: @test1
diff --git a/clang/test/CIR/CodeGen/array.cpp b/clang/test/CIR/CodeGen/array.cpp
index 26e172a00645..141b67e0e63c 100644
--- a/clang/test/CIR/CodeGen/array.cpp
+++ b/clang/test/CIR/CodeGen/array.cpp
@@ -101,7 +101,7 @@ void func() {
 // CIR: %[[TMP:.*]] = cir.load{{.*}} %[[ELE_PTR]] : !cir.ptr<!s32i>, !s32i
 // CIR" cir.store %[[TMP]], %[[INIT_2]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z4funcv()
+// LLVM: define{{.*}} void @_Z4funcv()
 // LLVM-NEXT: %[[ARR:.*]] = alloca [10 x i32], i64 1, align 16
 // LLVM-NEXT: %[[INIT:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT: %[[INIT_2:.*]] = alloca i32, i64 1, align 4
@@ -143,7 +143,7 @@ void func2() {
 // CIR: %[[ELE_1_PTR:.*]] = cir.ptr_stride(%[[LOAD_1]] : !cir.ptr<!s32i>, %[[OFFSET_1]] : !s64i), !cir.ptr<!s32i>
 // CIR: cir.store{{.*}} %[[ELE_1_PTR]], %[[ELE_ALLOCA]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
 
-// LLVM: define void @_Z5func2v()
+// LLVM: define{{.*}} void @_Z5func2v()
 // LLVM:  %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
 // LLVM:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR]], i32 0
@@ -183,7 +183,7 @@ void func3() {
 // CIR: %[[ELE_TMP:.*]] = cir.load{{.*}} %[[ELE_PTR]] : !cir.ptr<!s32i>, !s32i
 // CIR: cir.store{{.*}} %[[ELE_TMP]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z5func3v()
+// LLVM: define{{.*}} void @_Z5func3v()
 // LLVM:  %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
 // LLVM:  %[[IDX:.*]] = alloca i32, i64 1, align 4
 // LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
@@ -235,7 +235,7 @@ void func4() {
 // CIR: %[[TMP:.*]] = cir.load{{.*}} %[[ELE_0]] : !cir.ptr<!s32i>, !s32i
 // CIR: cir.store{{.*}} %[[TMP]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z5func4v()
+// LLVM: define{{.*}} void @_Z5func4v()
 // LLVM:  %[[ARR:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
 // LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
 // LLVM:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR]], i32 0
@@ -279,7 +279,7 @@ void func5() {
 // CIR: %10 = cir.ptr_stride(%7 : !cir.ptr<!cir.array<!s32i x 1>>, %[[OFFSET_1]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
 // CIR: cir.store{{.*}} %10, %[[ARR_PTR]] : !cir.ptr<!cir.array<!s32i x 1>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>
 
-// LLVM: define void @_Z5func5v()
+// LLVM: define{{.*}} void @_Z5func5v()
 // LLVM:  %[[ARR:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
 // LLVM:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR]], i32 0
@@ -312,7 +312,7 @@ void func6() {
 // CIR: %[[V1:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: cir.store{{.*}} %[[V1]], %[[ELE_PTR]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z5func6v()
+// LLVM: define{{.*}} void @_Z5func6v()
 // LLVM:  %[[VAR:.*]] = alloca i32, i64 1, align 4
 // LLVM:  %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
 // LLVM:  store i32 4, ptr %[[VAR]], align 4
@@ -345,7 +345,7 @@ void func7() {
 // CIR: %[[ELE_PTR:.*]] = cir.ptr_stride(%[[TMP]] : !cir.ptr<!cir.ptr<!s32i>>, %[[OFFSET]] : !s64i), !cir.ptr<!cir.ptr<!s32i>>
 // CIR: cir.store{{.*}} %[[ELE_PTR]], %[[ARR_TMP]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>
 
-// LLVM: define void @_Z5func7v()
+// LLVM: define{{.*}} void @_Z5func7v()
 // LLVM:  %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
 // LLVM:  %[[ALLOCA:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[ELE_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
@@ -363,7 +363,7 @@ void func8(int arr[10]) {
   int e2 = arr[1];
 }
 
-// CIR: cir.func @_Z5func8Pi(%[[ARG:.*]]: !cir.ptr<!s32i>
+// CIR: cir.func{{.*}} @_Z5func8Pi(%[[ARG:.*]]: !cir.ptr<!s32i>
 // CIR:  %[[ARR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arr", init]
 // CIR:  %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
 // CIR:  %[[INIT_2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e2", init]
@@ -379,7 +379,7 @@ void func8(int arr[10]) {
 // CIR:  %[[TMP_4:.*]] = cir.load{{.*}} %[[ELE_1]] : !cir.ptr<!s32i>, !s32i
 // CIR:  cir.store{{.*}} %[[TMP_4]], %[[INIT_2]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z5func8Pi(ptr %[[ARG:.*]])
+// LLVM: define{{.*}} void @_Z5func8Pi(ptr %[[ARG:.*]])
 // LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
 // LLVM:  %[[INIT_2:.*]] = alloca i32, i64 1, align 4
@@ -410,7 +410,7 @@ void func9(int arr[10][5]) {
   int e = arr[1][2];
 }
 
-// CIR: cir.func @_Z5func9PA5_i(%[[ARG:.*]]: !cir.ptr<!cir.array<!s32i x 5>>
+// CIR: cir.func{{.*}} @_Z5func9PA5_i(%[[ARG:.*]]: !cir.ptr<!cir.array<!s32i x 5>>
 // CIR:  %[[ARR:.*]] = cir.alloca !cir.ptr<!cir.array<!s32i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>, ["arr", init]
 // CIR:  %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
 // CIR:  cir.store{{.*}} %[[ARG]], %[[ARR]] : !cir.ptr<!cir.array<!s32i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 5>>>
@@ -423,7 +423,7 @@ void func9(int arr[10][5]) {
 // CIR:  %[[TMP_2:.*]] = cir.load{{.*}} %[[ARR_1_2]] : !cir.ptr<!s32i>, !s32i
 // CIR:  cir.store{{.*}} %[[TMP_2]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z5func9PA5_i(ptr %[[ARG:.*]])
+// LLVM: define{{.*}} void @_Z5func9PA5_i(ptr %[[ARG:.*]])
 // LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
 // LLVM:  store ptr %[[ARG]], ptr %[[ARR]], align 8
@@ -447,7 +447,7 @@ void func10(int *a) {
   int e = a[5];
 }
 
-// CIR: cir.func @_Z6func10Pi(%[[ARG:.*]]: !cir.ptr<!s32i>
+// CIR: cir.func{{.*}} @_Z6func10Pi(%[[ARG:.*]]: !cir.ptr<!s32i>
 // CIR: %[[ARR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["a", init]
 // CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
 // CIR: cir.store{{.*}} %[[ARG]], %[[ARR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
@@ -457,7 +457,7 @@ void func10(int *a) {
 // CIR: %[[TMP_2:.*]] = cir.load{{.*}} %[[ELE]] : !cir.ptr<!s32i>, !s32i
 // CIR: cir.store{{.*}} %[[TMP_2]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z6func10Pi(ptr %[[ARG:.*]]) {
+// LLVM: define{{.*}} void @_Z6func10Pi(ptr %[[ARG:.*]]) {
 // LLVM:  %[[ARR:.*]] = alloca ptr, i64 1, align 8
 // LLVM:  %[[INIT:.*]] = alloca i32, i64 1, align 4
 // LLVM:  store ptr %[[ARG]], ptr %[[ARR]], align 8
diff --git a/clang/test/CIR/CodeGen/assign-operator.cpp b/clang/test/CIR/CodeGen/assign-operator.cpp
new file mode 100644
index 000000000000..3e509f59368b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/assign-operator.cpp
@@ -0,0 +1,144 @@
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -std=c++11 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+class x {
+public: int operator=(int);
+};
+void a() {
+  x a;
+  a = 1u;
+}
+
+// CIR: cir.func private @_ZN1xaSEi(!cir.ptr<!rec_x>, !s32i)
+// CIR: cir.func{{.*}} @_Z1av()
+// CIR:   %[[A_ADDR:.*]] = cir.alloca !rec_x, !cir.ptr<!rec_x>, ["a"]
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !u32i
+// CIR:   %[[ONE_CAST:.*]] = cir.cast(integral, %[[ONE]] : !u32i), !s32i
+// CIR:   %[[RET:.*]] = cir.call @_ZN1xaSEi(%[[A_ADDR]], %[[ONE_CAST]]) : (!cir.ptr<!rec_x>, !s32i) -> !s32i
+
+// LLVM: define{{.*}} @_Z1av()
+// OGCG: define{{.*}} @_Z1av()
+
+void f(int i, int j) {
+  (i += j) = 17;
+}
+
+// CIR: cir.func{{.*}} @_Z1fii(%arg0: !s32i {{.*}}, %arg1: !s32i {{.*}})
+// CIR:   %[[I_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init]
+// CIR:   %[[J_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["j", init]
+// CIR:   cir.store %arg0, %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store %arg1, %[[J_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[SEVENTEEN:.*]] = cir.const #cir.int<17> : !s32i
+// CIR:   %[[J_LOAD:.*]] = cir.load align(4) %[[J_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   %[[I_LOAD:.*]] = cir.load align(4) %[[I_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   %[[ADD:.*]] = cir.binop(add, %[[I_LOAD]], %[[J_LOAD]]) nsw : !s32i
+// CIR:   cir.store align(4) %[[ADD]], %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) %[[SEVENTEEN]], %[[I_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.return
+
+// Ensure that we use memcpy when we would have selected a trivial assignment
+// operator, even for a non-trivially-copyable type.
+struct A {
+  A &operator=(const A&);
+};
+struct B {
+  B(const B&);
+  B &operator=(const B&) = default;
+  int n;
+};
+struct C {
+  A a;
+  B b[16];
+};
+void copy_c(C &c1, C &c2) {
+  c1 = c2;
+}
+
+// CIR: cir.func private @_ZN1AaSERKS_(!cir.ptr<!rec_A>, !cir.ptr<!rec_A>) -> !cir.ptr<!rec_A>
+// CIR: cir.func private @memcpy(!cir.ptr<!void>, !cir.ptr<!void>, !u64i) -> !cir.ptr<!void>
+
+// Implicit assignment operator for C.
+
+// CIR: cir.func comdat linkonce_odr @_ZN1CaSERKS_(%arg0: !cir.ptr<!rec_C> {{.*}}, %arg1: !cir.ptr<!rec_C> {{.*}}) -> !cir.ptr<!rec_C>
+// CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init]
+// CIR:   %[[ARG1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["", init, const]
+// CIR:   %[[RET_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["__retval"]
+// CIR:   cir.store %arg0, %[[THIS_ADDR]]
+// CIR:   cir.store %arg1, %[[ARG1_ADDR]]
+// CIR:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
+// CIR:   %[[A_MEMBER:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CIR:   %[[ARG1_LOAD:.*]] = cir.load{{.*}} %[[ARG1_ADDR]]
+// CIR:   %[[A_MEMBER_2:.*]] = cir.get_member %[[ARG1_LOAD]][0] {name = "a"}
+// CIR:   %[[C_A:.*]] = cir.call @_ZN1AaSERKS_(%[[A_MEMBER]], %[[A_MEMBER_2]])
+// CIR:   %[[B_MEMBER:.*]] = cir.get_member %[[THIS]][1] {name = "b"}
+// CIR:   %[[B_VOID_PTR:.*]] = cir.cast(bitcast, %[[B_MEMBER]] : !cir.ptr<!cir.array<!rec_B x 16>>), !cir.ptr<!void>
+// CIR:   %[[RET_LOAD:.*]] = cir.load %[[ARG1_ADDR]]
+// CIR:   %[[B_MEMBER_2:.*]] = cir.get_member %[[RET_LOAD]][1] {name = "b"}
+// CIR:   %[[B_VOID_PTR_2:.*]] = cir.cast(bitcast, %[[B_MEMBER_2]] : !cir.ptr<!cir.array<!rec_B x 16>>), !cir.ptr<!void>
+// CIR:   %[[SIZE:.*]] = cir.const #cir.int<64> : !u64i
+// CIR:   %[[COUNT:.*]] = cir.call @memcpy(%[[B_VOID_PTR]], %[[B_VOID_PTR_2]], %[[SIZE]])
+// CIR:   cir.store %[[THIS]], %[[RET_ADDR]]
+// CIR:   %[[RET_VAL:.*]] = cir.load{{.*}} %[[RET_ADDR]]
+// CIR:   cir.return %[[RET_VAL]]
+
+// CIR: cir.func{{.*}} @_Z6copy_cR1CS0_(%arg0: !cir.ptr<!rec_C> {{.*}}, %arg1: !cir.ptr<!rec_C> {{.*}})
+// CIR:   %[[C1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["c1", init, const]
+// CIR:   %[[C2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["c2", init, const]
+// CIR:   cir.store %arg0, %[[C1_ADDR]]
+// CIR:   cir.store %arg1, %[[C2_ADDR]]
+// CIR:   %[[C2_LOAD:.*]] = cir.load{{.*}} %[[C2_ADDR]]
+// CIR:   %[[C1_LOAD:.*]] = cir.load{{.*}} %[[C1_ADDR]]
+// CIR:   %[[RET:.*]] = cir.call @_ZN1CaSERKS_(%[[C1_LOAD]], %[[C2_LOAD]])
+
+struct D {
+  D &operator=(const D&);
+};
+struct E {
+  D &get_d_ref() { return d; }
+private:
+  D d;
+};
+
+void copy_ref_to_ref(E &e1, E &e2) {
+  e1.get_d_ref() = e2.get_d_ref();
+}
+
+// The call to e2.get_d_ref() must occur before the call to e1.get_d_ref().
+
+// CIR: cir.func{{.*}} @_Z15copy_ref_to_refR1ES0_(%arg0: !cir.ptr<!rec_E> {{.*}}, %arg1: !cir.ptr<!rec_E> {{.*}})
+// CIR:   %[[E1_ADDR:.*]] = cir.alloca !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>, ["e1", init, const]
+// CIR:   %[[E2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>, ["e2", init, const]
+// CIR:   cir.store %arg0, %[[E1_ADDR]] : !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>
+// CIR:   cir.store %arg1, %[[E2_ADDR]] : !cir.ptr<!rec_E>, !cir.ptr<!cir.ptr<!rec_E>>
+// CIR:   %[[E2:.*]] = cir.load %[[E2_ADDR]]
+// CIR:   %[[D2_REF:.*]] = cir.call @_ZN1E9get_d_refEv(%[[E2]])
+// CIR:   %[[E1:.*]] = cir.load %[[E1_ADDR]]
+// CIR:   %[[D1_REF:.*]] = cir.call @_ZN1E9get_d_refEv(%[[E1]])
+// CIR:   %[[D1_REF_2:.*]] = cir.call @_ZN1DaSERKS_(%[[D1_REF]], %[[D2_REF]])
+// CIR:   cir.return
+
+// LLVM: define{{.*}} void @_Z15copy_ref_to_refR1ES0_(ptr %[[ARG0:.*]], ptr %[[ARG1:.*]]) {
+// LLVM:   %[[E1_ADDR:.*]] = alloca ptr
+// LLVM:   %[[E2_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[ARG0]], ptr %[[E1_ADDR]]
+// LLVM:   store ptr %[[ARG1]], ptr %[[E2_ADDR]]
+// LLVM:   %[[E2:.*]] = load ptr, ptr %[[E2_ADDR]]
+// LLVM:   %[[D2_REF:.*]] = call ptr @_ZN1E9get_d_refEv(ptr %[[E2]])
+// LLVM:   %[[E1:.*]] = load ptr, ptr %[[E1_ADDR]]
+// LLVM:   %[[D1_REF:.*]] = call ptr @_ZN1E9get_d_refEv(ptr %[[E1]])
+// LLVM:   %[[D1_REF_2:.*]] = call ptr @_ZN1DaSERKS_(ptr %[[D1_REF]], ptr %[[D2_REF]])
+
+// OGCG: define{{.*}} void @_Z15copy_ref_to_refR1ES0_(ptr{{.*}} %[[ARG0:.*]], ptr{{.*}} %[[ARG1:.*]])
+// OGCG:   %[[E1_ADDR:.*]] = alloca ptr
+// OGCG:   %[[E2_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[ARG0]], ptr %[[E1_ADDR]]
+// OGCG:   store ptr %[[ARG1]], ptr %[[E2_ADDR]]
+// OGCG:   %[[E2:.*]] = load ptr, ptr %[[E2_ADDR]]
+// OGCG:   %[[D2_REF:.*]] = call{{.*}} ptr @_ZN1E9get_d_refEv(ptr{{.*}} %[[E2]])
+// OGCG:   %[[E1:.*]] = load ptr, ptr %[[E1_ADDR]]
+// OGCG:   %[[D1_REF:.*]] = call{{.*}} ptr @_ZN1E9get_d_refEv(ptr{{.*}} %[[E1]])
+// OGCG:   %[[D1_REF_2:.*]] = call{{.*}} ptr @_ZN1DaSERKS_(ptr{{.*}} %[[D1_REF]], ptr{{.*}} %[[D2_REF]])
diff --git a/clang/test/CIR/CodeGen/basic.c b/clang/test/CIR/CodeGen/basic.c
index 7ff73ee95f79..2c3c5b0f22a5 100644
--- a/clang/test/CIR/CodeGen/basic.c
+++ b/clang/test/CIR/CodeGen/basic.c
@@ -34,7 +34,7 @@ int f1(int i) {
   return i;
 }
 
-// CIR:      cir.func @f1(%arg0: !s32i loc({{.*}})) -> !s32i
+// CIR:      cir.func{{.*}} @f1(%arg0: !s32i loc({{.*}})) -> !s32i
 // CIR-NEXT:   %[[I_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CIR-NEXT:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CIR-NEXT:   cir.store{{.*}} %arg0, %[[I_PTR]] : !s32i, !cir.ptr<!s32i>
@@ -44,7 +44,7 @@ int f1(int i) {
 // CIR-NEXT:   %[[R:.*]] = cir.load{{.*}} %[[RV]] : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.return %[[R]] : !s32i
 
-//      LLVM: define i32 @f1(i32 %[[IP:.*]])
+//      LLVM: define{{.*}} i32 @f1(i32 %[[IP:.*]])
 // LLVM-NEXT:   %[[I_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   store i32 %[[IP]], ptr %[[I_PTR]], align 4
@@ -64,14 +64,14 @@ int f1(int i) {
 
 int f2(void) { return 3; }
 
-//      CIR: cir.func @f2() -> !s32i
+//      CIR: cir.func{{.*}} @f2() -> !s32i
 // CIR-NEXT:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CIR-NEXT:   %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
 // CIR-NEXT:   cir.store{{.*}} %[[THREE]], %[[RV]] : !s32i, !cir.ptr<!s32i>
 // CIR-NEXT:   %[[R:.*]] = cir.load{{.*}} %0 : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.return %[[R]] : !s32i
 
-//      LLVM: define i32 @f2()
+//      LLVM: define{{.*}} i32 @f2()
 // LLVM-NEXT:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   store i32 3, ptr %[[RV]], align 4
 // LLVM-NEXT:   %[[R:.*]] = load i32, ptr %[[RV]], align 4
@@ -86,7 +86,7 @@ int f3(void) {
   return i;
 }
 
-//      CIR: cir.func @f3() -> !s32i
+//      CIR: cir.func{{.*}} @f3() -> !s32i
 // CIR-NEXT:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CIR-NEXT:   %[[I_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CIR-NEXT:   %[[THREE:.*]] = cir.const #cir.int<3> : !s32i
@@ -96,7 +96,7 @@ int f3(void) {
 // CIR-NEXT:   %[[R:.*]] = cir.load{{.*}} %[[RV]] : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.return %[[R]] : !s32i
 
-//      LLVM: define i32 @f3()
+//      LLVM: define{{.*}} i32 @f3()
 // LLVM-NEXT:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[I_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   store i32 3, ptr %[[I_PTR]], align 4
@@ -117,10 +117,10 @@ void f4(void) {
   ;
 }
 
-//      CIR: cir.func @f4()
+//      CIR: cir.func{{.*}} @f4()
 // CIR-NEXT:   cir.return
 
-//      LLVM: define void @f4()
+//      LLVM: define{{.*}} void @f4()
 // LLVM-NEXT:   ret void
 
 //      OGCG: define{{.*}} void @f4()
@@ -133,7 +133,7 @@ void f5(void) {
     ;
 }
 
-//      CIR: cir.func @f5()
+//      CIR: cir.func{{.*}} @f5()
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:      cir.for : cond {
 // CIR-NEXT:        %0 = cir.const #true
@@ -147,7 +147,7 @@ void f5(void) {
 // CIR-NEXT:   cir.return
 // CIR-NEXT: }
 
-// LLVM: define void @f5()
+// LLVM: define{{.*}} void @f5()
 // LLVM:   br label %[[SCOPE:.*]]
 // LLVM: [[SCOPE]]:
 // LLVM:   br label %[[LOOP:.*]]
@@ -171,7 +171,7 @@ int f6(void) {
   return gv;
 }
 
-//      CIR: cir.func @f6() -> !s32i
+//      CIR: cir.func{{.*}} @f6() -> !s32i
 // CIR-NEXT:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CIR-NEXT:   %[[GV_PTR:.*]] = cir.get_global @gv : !cir.ptr<!s32i>
 // CIR-NEXT:   %[[GV:.*]] = cir.load{{.*}} %[[GV_PTR]] : !cir.ptr<!s32i>, !s32i
@@ -179,7 +179,7 @@ int f6(void) {
 // CIR-NEXT:   %[[R:.*]] = cir.load{{.*}} %[[RV]] : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.return %[[R]] : !s32i
 
-// LLVM:      define i32 @f6()
+// LLVM:      define{{.*}} i32 @f6()
 // LLVM-NEXT:   %[[RV_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[GV:.*]] = load i32, ptr @gv, align 4
 // LLVM-NEXT:   store i32 %[[GV]], ptr %[[RV_PTR]], align 4
@@ -195,7 +195,7 @@ int f7(int a, int b, int c) {
   return a + (b + c);
 }
 
-// CIR: cir.func @f7
+// CIR: cir.func{{.*}} @f7
 // CIR:  %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR:  %[[B_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
 // CIR:  %[[C_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["c", init]
@@ -205,7 +205,7 @@ int f7(int a, int b, int c) {
 // CIR:  %[[B_PLUS_C:.*]] = cir.binop(add, %[[B]], %[[C]]) nsw : !s32i
 // CIR:  %[[RETVAL:.*]] = cir.binop(add, %[[A]], %[[B_PLUS_C]]) nsw : !s32i
 
-// LLVM: define i32 @f7
+// LLVM: define{{.*}} i32 @f7
 // LLVM:   %[[A_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[B_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[C_PTR:.*]] = alloca i32, i64 1, align 4
@@ -231,7 +231,7 @@ int f8(int *p) {
   return (*p);
 }
 
-// CIR: cir.func @f8
+// CIR: cir.func{{.*}} @f8
 // CIR:    %[[P_PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p", init]
 // CIR:    %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
 // CIR:    %[[P:.*]] = cir.load deref{{.*}} %[[P_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
@@ -239,7 +239,7 @@ int f8(int *p) {
 // CIR:    %[[P2:.*]] = cir.load deref{{.*}} %[[P_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
 // CIR:    %[[STAR_P:.*]] = cir.load{{.*}} %[[P2]] : !cir.ptr<!s32i>, !s32i
 
-// LLVM: define i32 @f8
+// LLVM: define{{.*}} i32 @f8
 // LLVM:   %[[P_PTR:.*]] = alloca ptr, i64 1, align 8
 // LLVM:   %[[P:.*]] = load ptr, ptr %[[P_PTR]], align 8
 // LLVM:   store i32 2, ptr %[[P]], align 4
@@ -257,10 +257,10 @@ int f8(int *p) {
 
 void f9() {}
 
-//      CIR: cir.func @f9()
+//      CIR: cir.func{{.*}} @f9()
 // CIR-NEXT:   cir.return
 
-//      LLVM: define void @f9()
+//      LLVM: define{{.*}} void @f9()
 // LLVM-NEXT:   ret void
 
 //      OGCG: define{{.*}} void @f9()
@@ -269,12 +269,12 @@ void f9() {}
 
 void f10(int arg0, ...) {}
 
-//      CIR: cir.func @f10(%[[ARG0:.*]]: !s32i loc({{.*}}), ...)
+//      CIR: cir.func{{.*}} @f10(%[[ARG0:.*]]: !s32i loc({{.*}}), ...)
 // CIR-NEXT:   %[[ARG0_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["arg0", init] {alignment = 4 : i64}
 // CIR-NEXT:   cir.store{{.*}} %[[ARG0]], %[[ARG0_PTR]] : !s32i, !cir.ptr<!s32i>
 // CIR-NEXT:   cir.return
 
-//      LLVM: define void @f10(i32 %[[ARG0:.*]], ...)
+//      LLVM: define{{.*}} void @f10(i32 %[[ARG0:.*]], ...)
 // LLVM-NEXT:   %[[ARG0_PTR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   store i32 %[[ARG0]], ptr %[[ARG0_PTR]], align 4
 // LLVM-NEXT:   ret void
@@ -292,7 +292,7 @@ size_type max_size(void) {
   return (size_type)~0 / sizeof(_Tp);
 }
 
-// CIR: cir.func @max_size()
+// CIR: cir.func{{.*}} @max_size()
 // CIR:   %0 = cir.alloca !u64i, !cir.ptr<!u64i>, ["__retval"] {alignment = 8 : i64}
 // CIR:   %1 = cir.const #cir.int<0> : !s32i
 // CIR:   %2 = cir.unary(not, %1) : !s32i, !s32i
@@ -300,7 +300,7 @@ size_type max_size(void) {
 // CIR:   %4 = cir.const #cir.int<8> : !u64i
 // CIR:   %5 = cir.binop(div, %3, %4) : !u64i
 
-// LLVM: define i64 @max_size()
+// LLVM: define{{.*}} i64 @max_size()
 // LLVM:   store i64 2305843009213693951, ptr
 
 // OGCG: define{{.*}} i64 @max_size()
@@ -315,10 +315,10 @@ void test_char_literal() {
   c = 'X';
 }
 
-// CIR: cir.func @test_char_literal
+// CIR: cir.func{{.*}} @test_char_literal
 // CIR:   cir.const #cir.int<88>
 
-// LLVM: define void @test_char_literal()
+// LLVM: define{{.*}} void @test_char_literal()
 // LLVM:   store i8 88, ptr %{{.*}}, align 1
 
 // OGCG: define{{.*}} void @test_char_literal()
diff --git a/clang/test/CIR/CodeGen/basic.cpp b/clang/test/CIR/CodeGen/basic.cpp
index ed1c6d364a0e..fe6dd938f0fa 100644
--- a/clang/test/CIR/CodeGen/basic.cpp
+++ b/clang/test/CIR/CodeGen/basic.cpp
@@ -31,7 +31,7 @@ int f1() {
   return i;
 }
 
-// CHECK: cir.func @_Z2f1v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z2f1v() -> !s32i
 // CHECK:    %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:    %[[I_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i"] {alignment = 4 : i64}
 // CHECK:    %[[I:.*]] = cir.load{{.*}} %[[I_PTR]] : !cir.ptr<!s32i>, !s32i
@@ -44,7 +44,7 @@ int f2() {
   return i;
 }
 
-// CHECK: cir.func @_Z2f2v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z2f2v() -> !s32i
 // CHECK:    %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:    %[[I_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init, const] {alignment = 4 : i64}
 // CHECK:    %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
@@ -58,7 +58,7 @@ int f3(int i) {
   return i;
 }
 
-// CHECK: cir.func @_Z2f3i(%[[ARG:.*]]: !s32i loc({{.*}})) -> !s32i
+// CHECK: cir.func{{.*}} @_Z2f3i(%[[ARG:.*]]: !s32i loc({{.*}})) -> !s32i
 // CHECK:   %[[ARG_ALLOCA:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CHECK:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   cir.store{{.*}} %[[ARG]], %[[ARG_ALLOCA]] : !s32i, !cir.ptr<!s32i>
@@ -71,7 +71,7 @@ int f4(const int i) {
   return i;
 }
 
-// CHECK: cir.func @_Z2f4i(%[[ARG:.*]]: !s32i loc({{.*}})) -> !s32i
+// CHECK: cir.func{{.*}} @_Z2f4i(%[[ARG:.*]]: !s32i loc({{.*}})) -> !s32i
 // CHECK:   %[[ARG_ALLOCA:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init, const] {alignment = 4 : i64}
 // CHECK:   %[[RV:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   cir.store{{.*}} %[[ARG]], %[[ARG_ALLOCA]] : !s32i, !cir.ptr<!s32i>
@@ -91,7 +91,7 @@ int *f5() {
   return p;
 }
 
-// CHECK:      cir.func @_Z2f5v() -> !cir.ptr<!s32i>
+// CHECK:      cir.func{{.*}} @_Z2f5v() -> !cir.ptr<!s32i>
 // CHECK-NEXT:   %[[RET_ADDR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["__retval"] {alignment = 8 : i64}
 // CHECK-NEXT:   %[[P_ADDR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p", init] {alignment = 8 : i64}
 // CHECK-NEXT:   %[[NULLPTR:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!s32i>
@@ -120,7 +120,7 @@ size_type max_size() {
   return size_type(~0) / sizeof(_Tp);
 }
 
-// CHECK: cir.func @_Z8max_sizev() -> !u64i
+// CHECK: cir.func{{.*}} @_Z8max_sizev() -> !u64i
 // CHECK:   %0 = cir.alloca !u64i, !cir.ptr<!u64i>, ["__retval"] {alignment = 8 : i64}
 // CHECK:   %1 = cir.const #cir.int<0> : !s32i
 // CHECK:   %2 = cir.unary(not, %1) : !s32i, !s32i
@@ -137,7 +137,7 @@ void ref_arg(int &x) {
   x = 3;
 }
 
-// CHECK: cir.func @_Z7ref_argRi(%[[ARG:.*]]: !cir.ptr<!s32i> {{.*}})
+// CHECK: cir.func{{.*}} @_Z7ref_argRi(%[[ARG:.*]]: !cir.ptr<!s32i> {{.*}})
 // CHECK:   %[[X_REF_ADDR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["x", init, const] {alignment = 8 : i64}
 // CHECK:   %[[Y_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["y", init] {alignment = 4 : i64}
 // CHECK:   cir.store{{.*}} %[[ARG]], %[[X_REF_ADDR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
@@ -154,7 +154,7 @@ short &return_ref() {
   return gs;
 }
 
-// CHECK: cir.func @_Z10return_refv() -> !cir.ptr<!s16i>
+// CHECK: cir.func{{.*}} @_Z10return_refv() -> !cir.ptr<!s16i>
 // CHECK:   %[[RETVAL_ADDR:.*]] = cir.alloca !cir.ptr<!s16i>, !cir.ptr<!cir.ptr<!s16i>>, ["__retval"] {alignment = 8 : i64}
 // CHECK:   %[[GS_ADDR:.*]] = cir.get_global @gs : !cir.ptr<!s16i>
 // CHECK:   cir.store{{.*}} %[[GS_ADDR]], %[[RETVAL_ADDR]] : !cir.ptr<!s16i>, !cir.ptr<!cir.ptr<!s16i>>
@@ -165,7 +165,7 @@ void ref_local(short x) {
   short &y = x;
 }
 
-// CHECK: cir.func @_Z9ref_locals(%[[ARG:.*]]: !s16i {{.*}})
+// CHECK: cir.func{{.*}} @_Z9ref_locals(%[[ARG:.*]]: !s16i {{.*}})
 // CHECK:   %[[X_ADDR:.*]] = cir.alloca !s16i, !cir.ptr<!s16i>, ["x", init] {alignment = 2 : i64}
 // CHECK:   %[[Y_REF_ADDR:.*]] = cir.alloca !cir.ptr<!s16i>, !cir.ptr<!cir.ptr<!s16i>>, ["y", init, const] {alignment = 8 : i64}
 // CHECK:   cir.store{{.*}} %[[ARG]], %[[X_ADDR]] : !s16i, !cir.ptr<!s16i>
diff --git a/clang/test/CIR/CodeGen/binassign.c b/clang/test/CIR/CodeGen/binassign.c
index 4955c988ec09..541b50a664c0 100644
--- a/clang/test/CIR/CodeGen/binassign.c
+++ b/clang/test/CIR/CodeGen/binassign.c
@@ -17,7 +17,7 @@ void binary_assign(void) {
     i = 42;
 }
 
-// CIR-LABEL: cir.func @binary_assign() {
+// CIR-LABEL: cir.func{{.*}} @binary_assign() {
 // CIR:         %[[B:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b"]
 // CIR:         %[[C:.*]] = cir.alloca !s8i, !cir.ptr<!s8i>, ["c"]
 // CIR:         %[[F:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["f"]
diff --git a/clang/test/CIR/CodeGen/binop.cpp b/clang/test/CIR/CodeGen/binop.cpp
index c728f0d0c1bc..847e81755939 100644
--- a/clang/test/CIR/CodeGen/binop.cpp
+++ b/clang/test/CIR/CodeGen/binop.cpp
@@ -16,7 +16,7 @@ void b0(int a, int b) {
   x = x | b;
 }
 
-// CIR-LABEL: cir.func @_Z2b0ii(
+// CIR-LABEL: cir.func{{.*}} @_Z2b0ii(
 // CIR: %{{.+}} = cir.binop(mul, %{{.+}}, %{{.+}}) nsw : !s32i
 // CIR: %{{.+}} = cir.binop(div, %{{.+}}, %{{.+}}) : !s32i
 // CIR: %{{.+}} = cir.binop(rem, %{{.+}}, %{{.+}}) : !s32i
@@ -27,7 +27,7 @@ void b0(int a, int b) {
 // CIR: %{{.+}} = cir.binop(or, %{{.+}}, %{{.+}}) : !s32i
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z2b0ii(
+// LLVM-LABEL: define{{.*}} void @_Z2b0ii(
 // LLVM-SAME: i32 %[[A:.*]], i32 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i32
 // LLVM:         %[[B_ADDR:.*]] = alloca i32
@@ -77,7 +77,7 @@ void b0(int a, int b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z2b0ii(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} { 
+// OGCG-LABEL: define{{.*}} void @_Z2b0ii(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} { 
 // OGCG:         %[[A_ADDR:.*]] = alloca i32
 // OGCG:         %[[B_ADDR:.*]] = alloca i32
 // OGCG:         %[[X:.*]] = alloca i32
@@ -133,14 +133,14 @@ void testFloatingPointBinOps(float a, float b) {
   a - b;
 }
 
-// CIR-LABEL: cir.func @_Z23testFloatingPointBinOpsff(
+// CIR-LABEL: cir.func{{.*}} @_Z23testFloatingPointBinOpsff(
 // CIR: cir.binop(mul, %{{.+}}, %{{.+}}) : !cir.float
 // CIR: cir.binop(div, %{{.+}}, %{{.+}}) : !cir.float
 // CIR: cir.binop(add, %{{.+}}, %{{.+}}) : !cir.float
 // CIR: cir.binop(sub, %{{.+}}, %{{.+}}) : !cir.float
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z23testFloatingPointBinOpsff(
+// LLVM-LABEL: define{{.*}} void @_Z23testFloatingPointBinOpsff(
 // LLVM-SAME: float %[[A:.*]], float %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca float, i64 1
 // LLVM:         %[[B_ADDR:.*]] = alloca float, i64 1
@@ -165,7 +165,7 @@ void testFloatingPointBinOps(float a, float b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z23testFloatingPointBinOpsff(float {{.*}} %a, float {{.*}} %b)
+// OGCG-LABEL: define{{.*}} void @_Z23testFloatingPointBinOpsff(float {{.*}} %a, float {{.*}} %b)
 // OGCG:         %a.addr = alloca float
 // OGCG:         %b.addr = alloca float
 // OGCG:         store float %a, ptr %a.addr
@@ -194,7 +194,7 @@ void signed_shift(int a, int b) {
   x = a << b;
 }
 
-// CIR-LABEL: cir.func @_Z12signed_shiftii(
+// CIR-LABEL: cir.func{{.*}} @_Z12signed_shiftii(
 // CIR-SAME: %[[ARG0:.*]]: !s32i{{.*}}, %[[ARG1:.*]]: !s32i{{.*}})
 // CIR: %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
@@ -215,7 +215,7 @@ void signed_shift(int a, int b) {
 
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z12signed_shiftii
+// LLVM-LABEL: define{{.*}} void @_Z12signed_shiftii
 // LLVM-SAME: (i32 %[[A:.*]], i32 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i32
 // LLVM:         %[[B_ADDR:.*]] = alloca i32
@@ -235,7 +235,7 @@ void signed_shift(int a, int b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z12signed_shiftii
+// OGCG-LABEL: define{{.*}} void @_Z12signed_shiftii
 // OGCG-SAME: (i32 {{.*}} %[[A:.*]], i32 {{.*}} %[[B:.*]])
 // OGCG:         %[[A_ADDR:.*]] = alloca i32
 // OGCG:         %[[B_ADDR:.*]] = alloca i32
@@ -260,7 +260,7 @@ void unsigned_shift(unsigned a, unsigned b) {
   x = a << b;
 }
 
-// CIR-LABEL: cir.func @_Z14unsigned_shiftjj(
+// CIR-LABEL: cir.func{{.*}} @_Z14unsigned_shiftjj(
 // CIR-SAME: %[[ARG0:.*]]: !u32i{{.*}}, %[[ARG1:.*]]: !u32i{{.*}})
 // CIR: %[[A_PTR:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["b", init]
@@ -281,7 +281,7 @@ void unsigned_shift(unsigned a, unsigned b) {
 
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z14unsigned_shiftjj
+// LLVM-LABEL: define{{.*}} void @_Z14unsigned_shiftjj
 // LLVM-SAME: (i32 %[[A:.*]], i32 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i32
 // LLVM:         %[[B_ADDR:.*]] = alloca i32
@@ -301,7 +301,7 @@ void unsigned_shift(unsigned a, unsigned b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z14unsigned_shiftjj
+// OGCG-LABEL: define{{.*}} void @_Z14unsigned_shiftjj
 // OGCG-SAME: (i32 {{.*}} %[[A:.*]], i32 {{.*}} %[[B:.*]])
 // OGCG:         %[[A_ADDR:.*]] = alloca i32
 // OGCG:         %[[B_ADDR:.*]] = alloca i32
@@ -326,7 +326,7 @@ void zext_shift_example(int a, unsigned char b) {
   x = a << b;
 }
 
-// CIR-LABEL: cir.func @_Z18zext_shift_exampleih(
+// CIR-LABEL: cir.func{{.*}} @_Z18zext_shift_exampleih(
 // CIR-SAME: %[[ARG0:.*]]: !s32i{{.*}}, %[[ARG1:.*]]: !u8i{{.*}})
 // CIR: %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !u8i, !cir.ptr<!u8i>, ["b", init]
@@ -349,7 +349,7 @@ void zext_shift_example(int a, unsigned char b) {
 
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z18zext_shift_exampleih
+// LLVM-LABEL: define{{.*}} void @_Z18zext_shift_exampleih
 // LLVM-SAME: (i32 %[[A:.*]], i8 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i32
 // LLVM:         %[[B_ADDR:.*]] = alloca i8
@@ -371,7 +371,7 @@ void zext_shift_example(int a, unsigned char b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z18zext_shift_exampleih
+// OGCG-LABEL: define{{.*}} void @_Z18zext_shift_exampleih
 // OGCG-SAME: (i32 {{.*}} %[[A:.*]], i8 {{.*}} %[[B:.*]])
 // OGCG:         %[[A_ADDR:.*]] = alloca i32
 // OGCG:         %[[B_ADDR:.*]] = alloca i8
@@ -398,7 +398,7 @@ void sext_shift_example(int a, signed char b) {
   x = a << b;
 }
 
-// CIR-LABEL: cir.func @_Z18sext_shift_exampleia(
+// CIR-LABEL: cir.func{{.*}} @_Z18sext_shift_exampleia(
 // CIR-SAME: %[[ARG0:.*]]: !s32i{{.*}}, %[[ARG1:.*]]: !s8i{{.*}})
 // CIR: %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !s8i, !cir.ptr<!s8i>, ["b", init]
@@ -421,7 +421,7 @@ void sext_shift_example(int a, signed char b) {
 
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z18sext_shift_exampleia
+// LLVM-LABEL: define{{.*}} void @_Z18sext_shift_exampleia
 // LLVM-SAME: (i32 %[[A:.*]], i8 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i32
 // LLVM:         %[[B_ADDR:.*]] = alloca i8
@@ -443,7 +443,7 @@ void sext_shift_example(int a, signed char b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z18sext_shift_exampleia
+// OGCG-LABEL: define{{.*}} void @_Z18sext_shift_exampleia
 // OGCG-SAME: (i32 {{.*}} %[[A:.*]], i8 {{.*}} %[[B:.*]])
 // OGCG:         %[[A_ADDR:.*]] = alloca i32
 // OGCG:         %[[B_ADDR:.*]] = alloca i8
@@ -470,7 +470,7 @@ void long_shift_example(long long a, short b) {
   x = a << b;
 }
 
-// CIR-LABEL: cir.func @_Z18long_shift_examplexs(
+// CIR-LABEL: cir.func{{.*}} @_Z18long_shift_examplexs(
 // CIR-SAME: %[[ARG0:.*]]: !s64i{{.*}}, %[[ARG1:.*]]: !s16i{{.*}})
 // CIR: %[[A_PTR:.*]] = cir.alloca !s64i, !cir.ptr<!s64i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !s16i, !cir.ptr<!s16i>, ["b", init]
@@ -493,7 +493,7 @@ void long_shift_example(long long a, short b) {
 
 // CIR: cir.return
 
-// LLVM-LABEL: define void @_Z18long_shift_examplexs
+// LLVM-LABEL: define{{.*}} void @_Z18long_shift_examplexs
 // LLVM-SAME: (i64 %[[A:.*]], i16 %[[B:.*]])
 // LLVM:         %[[A_ADDR:.*]] = alloca i64
 // LLVM:         %[[B_ADDR:.*]] = alloca i16
@@ -517,7 +517,7 @@ void long_shift_example(long long a, short b) {
 
 // LLVM:         ret void
 
-// OGCG-LABEL: define dso_local void @_Z18long_shift_examplexs
+// OGCG-LABEL: define{{.*}} void @_Z18long_shift_examplexs
 // OGCG-SAME: (i64 {{.*}} %[[A:.*]], i16 {{.*}} %[[B:.*]])
 // OGCG:         %[[A_ADDR:.*]] = alloca i64
 // OGCG:         %[[B_ADDR:.*]] = alloca i16
@@ -546,7 +546,7 @@ void b1(bool a, bool b) {
   x = x || b;
 }
 
-// CIR-LABEL: cir.func @_Z2b1bb(
+// CIR-LABEL: cir.func{{.*}} @_Z2b1bb(
 // CIR-SAME: %[[ARG0:.*]]: !cir.bool {{.*}}, %[[ARG1:.*]]: !cir.bool {{.*}})
 // CIR: [[A:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["a", init]
 // CIR: [[B:%[0-9]+]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b", init]
@@ -574,7 +574,7 @@ void b1(bool a, bool b) {
 // CIR: cir.return
 
 
-// LLVM-LABEL: define void @_Z2b1bb(
+// LLVM-LABEL: define{{.*}} void @_Z2b1bb(
 // LLVM-SAME: i1 %[[ARG0:.+]], i1 %[[ARG1:.+]])
 // LLVM: %[[A_ADDR:.*]] = alloca i8
 // LLVM: %[[B_ADDR:.*]] = alloca i8
@@ -611,7 +611,7 @@ void b1(bool a, bool b) {
 // LLVM: store i8 %[[ZEXT_OR]], ptr %[[X]]
 // LLVM: ret void
 
-// OGCG-LABEL: define dso_local void @_Z2b1bb
+// OGCG-LABEL: define{{.*}} void @_Z2b1bb
 // OGCG-SAME: (i1 {{.*}} %[[ARG0:.+]], i1 {{.*}} %[[ARG1:.+]])
 // OGCG: [[ENTRY:.*]]:
 // OGCG: %[[A_ADDR:.*]] = alloca i8
@@ -650,7 +650,7 @@ void b3(int a, int b, int c, int d) {
   x = (a == b) || (c == d);
 }
 
-// CIR-LABEL: cir.func @_Z2b3iiii(
+// CIR-LABEL: cir.func{{.*}} @_Z2b3iiii(
 // CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}, %[[ARG1:.*]]: !s32i {{.*}}, %[[ARG2:.*]]: !s32i {{.*}}, %[[ARG3:.*]]: !s32i {{.*}})
 // CIR: [[A:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR: [[B:%[0-9]+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
@@ -690,7 +690,7 @@ void b3(int a, int b, int c, int d) {
 // CIR: cir.return
 
 
-// LLVM-LABEL: define void @_Z2b3iiii(
+// LLVM-LABEL: define{{.*}} void @_Z2b3iiii(
 // LLVM-SAME: i32 %[[ARG0:.+]], i32 %[[ARG1:.+]], i32 %[[ARG2:.+]], i32 %[[ARG3:.+]])
 // LLVM: %[[A_ADDR:.*]] = alloca i32, i64 1
 // LLVM: %[[B_ADDR:.*]] = alloca i32, i64 1
@@ -733,7 +733,7 @@ void b3(int a, int b, int c, int d) {
 // LLVM: store i8 %[[ZEXT_OR]], ptr %[[X]]
 // LLVM: ret void
 
-// OGCG-LABEL: define dso_local void @_Z2b3iiii(
+// OGCG-LABEL: define{{.*}} void @_Z2b3iiii(
 // OGCG-SAME: i32 {{.*}} %[[ARG0:.+]], i32 {{.*}} %[[ARG1:.+]], i32 {{.*}} %[[ARG2:.+]], i32 {{.*}} %[[ARG3:.+]])
 // OGCG: [[ENTRY:.*]]:
 // OGCG: %[[A_ADDR:.*]] = alloca i32
@@ -771,4 +771,4 @@ void b3(int a, int b, int c, int d) {
 // OGCG: %[[OR_PHI:.*]] = phi i1 [ true, %[[AND_MERGE]] ], [ %[[CMP4]], %[[OR_FALSE]] ]
 // OGCG: %[[ZEXT_OR:.*]] = zext i1 %[[OR_PHI]] to i8
 // OGCG: store i8 %[[ZEXT_OR]], ptr %[[X]]
-// OGCG: ret void
\ No newline at end of file
+// OGCG: ret void
diff --git a/clang/test/CIR/CodeGen/bitfield-union.c b/clang/test/CIR/CodeGen/bitfield-union.c
new file mode 100644
index 000000000000..b5d14540f488
--- /dev/null
+++ b/clang/test/CIR/CodeGen/bitfield-union.c
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+typedef union {
+  int x;
+  int y : 4;
+  int z : 8;
+} demo;
+
+// CIR:  !rec_demo = !cir.record<union "demo" {!s32i, !u8i, !u8i}>
+// LLVM: %union.demo = type { i32 }
+// OGCG: %union.demo = type { i32 }
+
+typedef union {
+  int x;
+  int y : 3;
+  int   : 0;
+  int z : 2;
+} zero_bit;
+
+// CIR:  !rec_zero_bit = !cir.record<union "zero_bit" {!s32i, !u8i, !u8i}>
+// LLVM: %union.zero_bit = type { i32 }
+// OGCG: %union.zero_bit = type { i32 }
+
+demo d;
+zero_bit z;
diff --git a/clang/test/CIR/CodeGen/bitfields.c b/clang/test/CIR/CodeGen/bitfields.c
index ff5c6bc1787b..6eb753c5cc3d 100644
--- a/clang/test/CIR/CodeGen/bitfields.c
+++ b/clang/test/CIR/CodeGen/bitfields.c
@@ -13,6 +13,7 @@ typedef struct {
 } A;
 
 // CIR-DAG:  !rec_A = !cir.record<struct "A" packed padded {!s8i, !s8i, !s8i, !u16i, !cir.array<!u8i x 3>}>
+// CIR-DAG:  #bfi_more_bits = #cir.bitfield_info<name = "more_bits", storageType = !u16i, size = 4, offset = 3, isSigned = false>
 // LLVM-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
 // OGCG-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
 
@@ -34,6 +35,7 @@ typedef struct {
   int e : 15;
   unsigned f; // type other than int above, not a bitfield
 } S;
+// CIR-DAG:  #bfi_c = #cir.bitfield_info<name = "c", storageType = !u64i, size = 17, offset = 32, isSigned = true>
 // CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
 // LLVM-DAG: %struct.S = type { i64, i16, i32 }
 // OGCG-DAG: %struct.S = type { i64, i16, i32 }
@@ -76,3 +78,59 @@ void def() {
   T t;
   U u;
 }
+
+int load_field(S* s) {
+  return s->c;
+}
+
+// CIR: cir.func {{.*@load_field}}
+// CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init]
+// CIR:   [[TMP1:%.*]] = cir.load{{.*}} [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+// CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u64i>) -> !s32i
+
+// LLVM: define dso_local i32 @load_field
+// LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+// LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+// LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+// LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+// LLVM:   [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+// LLVM:   [[TMP5:%.*]] = shl i64 [[TMP4]], 15
+// LLVM:   [[TMP6:%.*]] = ashr i64 [[TMP5]], 47
+// LLVM:   [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+
+// OGCG: define dso_local i32 @load_field
+// OGCG:  [[TMP0:%.*]] = alloca ptr, align 8
+// OGCG:  [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// OGCG:  [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+// OGCG:  [[TMP3:%.*]] = shl i64 [[TMP2]], 15
+// OGCG:  [[TMP4:%.*]] = ashr i64 [[TMP3]], 47
+// OGCG:  [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+
+unsigned int load_field_unsigned(A* s) {
+  return s->more_bits;
+}
+
+//CIR: cir.func dso_local @load_field_unsigned
+//CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_A>, !cir.ptr<!cir.ptr<!rec_A>>, ["s", init] {alignment = 8 : i64}
+//CIR:   [[TMP1:%.*]] = cir.load align(8) [[TMP0]] : !cir.ptr<!cir.ptr<!rec_A>>, !cir.ptr<!rec_A>
+//CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][3] {name = "more_bits"} : !cir.ptr<!rec_A> -> !cir.ptr<!u16i>
+//CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_more_bits, [[TMP2]] : !cir.ptr<!u16i>) -> !u32i
+
+//LLVM: define dso_local i32 @load_field_unsigned
+//LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+//LLVM:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//LLVM:   [[TMP2:%.*]] = getelementptr %struct.A, ptr [[TMP1]], i32 0, i32 3
+//LLVM:   [[TMP3:%.*]] = load i16, ptr [[TMP2]], align 2
+//LLVM:   [[TMP4:%.*]] = lshr i16 [[TMP3]], 3
+//LLVM:   [[TMP5:%.*]] = and i16 [[TMP4]], 15
+//LLVM:   [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
+
+//OGCG: define dso_local i32 @load_field_unsigned
+//OGCG:   [[TMP0:%.*]] = alloca ptr, align 8
+//OGCG:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//OGCG:   [[TMP2:%.*]] = getelementptr inbounds nuw %struct.A, ptr [[TMP1]], i32 0, i32 3
+//OGCG:   [[TMP3:%.*]] = load i16, ptr [[TMP2]], align 1
+//OGCG:   [[TMP4:%.*]] = lshr i16 [[TMP3]], 3
+//OGCG:   [[TMP5:%.*]] = and i16 [[TMP4]], 15
+//OGCG:   [[TMP6:%.*]] = zext i16 [[TMP5]] to i32
diff --git a/clang/test/CIR/CodeGen/bitfields.cpp b/clang/test/CIR/CodeGen/bitfields.cpp
index 762d24988474..a4d58b5cadce 100644
--- a/clang/test/CIR/CodeGen/bitfields.cpp
+++ b/clang/test/CIR/CodeGen/bitfields.cpp
@@ -14,6 +14,7 @@ typedef struct {
   unsigned f; // type other than int above, not a bitfield
 } S;
 // CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
+// CIR-DAG:  #bfi_c = #cir.bitfield_info<name = "c", storageType = !u64i, size = 17, offset = 32, isSigned = true>
 // LLVM-DAG: %struct.S = type { i64, i16, i32 }
 // OGCG-DAG: %struct.S = type { i64, i16, i32 }
 
@@ -30,3 +31,30 @@ void def() {
   S s;
   T t;
 }
+
+int load_field(S* s) {
+  return s->c;
+}
+// CIR: cir.func dso_local @_Z10load_field
+// CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init]
+// CIR:   [[TMP1:%.*]] = cir.load{{.*}} [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+// CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u64i>
+// CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u64i>) -> !s32i
+
+// LLVM: define dso_local i32 @_Z10load_fieldP1S
+// LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+// LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+// LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+// LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+// LLVM:   [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+// LLVM:   [[TMP5:%.*]] = shl i64 [[TMP4]], 15
+// LLVM:   [[TMP6:%.*]] = ashr i64 [[TMP5]], 47
+// LLVM:   [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+
+// OGCG: define dso_local noundef i32 @_Z10load_fieldP1S
+// OGCG:  [[TMP0:%.*]] = alloca ptr, align 8
+// OGCG:  [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+// OGCG:  [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
+// OGCG:  [[TMP3:%.*]] = shl i64 [[TMP2]], 15
+// OGCG:  [[TMP4:%.*]] = ashr i64 [[TMP3]], 47
+// OGCG:  [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
diff --git a/clang/test/CIR/CodeGen/bitfields_be.c b/clang/test/CIR/CodeGen/bitfields_be.c
index 149e9c9ac33f..e839bc2b9698 100644
--- a/clang/test/CIR/CodeGen/bitfields_be.c
+++ b/clang/test/CIR/CodeGen/bitfields_be.c
@@ -10,8 +10,35 @@ typedef struct {
     int b : 11;
     int c : 17;
 } S;
-S s;
 
 // CIR:  !rec_S = !cir.record<struct "S" {!u32i}>
 // LLVM: %struct.S = type { i32 }
 // OGCG: %struct.S = type { i32 }
+void def() {
+  S s;
+}
+int init(S* s) {
+  return s->c;
+}
+
+//CIR: cir.func dso_local @init
+//CIR:   [[TMP0:%.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["s", init] {alignment = 8 : i64}
+//CIR:   [[TMP1:%.*]] = cir.load align(8) [[TMP0]] : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+//CIR:   [[TMP2:%.*]] = cir.get_member [[TMP1]][0] {name = "c"} : !cir.ptr<!rec_S> -> !cir.ptr<!u32i>
+//CIR:   [[TMP3:%.*]] = cir.get_bitfield(#bfi_c, [[TMP2]] : !cir.ptr<!u32i>) -> !s32i
+
+//LLVM: define dso_local i32 @init(ptr %0) {
+//LLVM:   [[TMP0:%.*]] = alloca ptr, i64 1, align 8
+//LLVM:   [[TMP1:%.*]] = alloca i32, i64 1, align 4
+//LLVM:   [[TMP2:%.*]] = load ptr, ptr [[TMP0]], align 8
+//LLVM:   [[TMP3:%.*]] = getelementptr %struct.S, ptr [[TMP2]], i32 0, i32 0
+//LLVM:   [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+//LLVM:   [[TMP5:%.*]] = shl i32 [[TMP4]], 15
+//LLVM:   [[TMP6:%.*]] = ashr i32 [[TMP5]], 15
+
+//OGCG: define dso_local i32 @init
+//OGCG:   [[TMP0:%.*]] = alloca ptr, align 8
+//OGCG:   [[TMP1:%.*]] = load ptr, ptr [[TMP0]], align 8
+//OGCG:   [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
+//OGCG:   [[TMP3:%.*]] = shl i32 [[TMP2]], 15
+//OGCG:   [[TMP4:%.*]] = ashr i32 [[TMP3]], 15
diff --git a/clang/test/CIR/CodeGen/builtin_bit.cpp b/clang/test/CIR/CodeGen/builtin_bit.cpp
new file mode 100644
index 000000000000..ba56c91ce740
--- /dev/null
+++ b/clang/test/CIR/CodeGen/builtin_bit.cpp
@@ -0,0 +1,327 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int test_builtin_clrsb(int x) {
+  return __builtin_clrsb(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_clrsbi
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s32i) : !s32i
+
+// LLVM-LABEL: _Z18test_builtin_clrsbi
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i32 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i32 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i32 %[[X_NOT]], i32 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i32 @llvm.ctlz.i32(i32 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i32 %[[LZ]], 1
+
+// OGCG-LABEL: _Z18test_builtin_clrsbi
+// OGCG:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i32 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i32 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i32 %[[X_NOT]], i32 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i32 @llvm.ctlz.i32(i32 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i32 %[[LZ]], 1
+
+int test_builtin_clrsbl(long x) {
+  return __builtin_clrsbl(x);
+}
+
+// CIR-LABEL: _Z19test_builtin_clrsbll
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s64i) : !s64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !s64i), !s32i
+
+// LLVM-LABEL: _Z19test_builtin_clrsbll
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+// OGCG-LABEL: _Z19test_builtin_clrsbll
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+int test_builtin_clrsbll(long long x) {
+  return __builtin_clrsbll(x);
+}
+
+// CIR-LABEL: _Z20test_builtin_clrsbllx
+// CIR:         [[TMP:%.+]] = cir.bit.clrsb(%{{.+}} : !s64i) : !s64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !s64i), !s32i
+
+// LLVM-LABEL: _Z20test_builtin_clrsbllx
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// LLVM-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// LLVM-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// LLVM-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// LLVM-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+// OGCG-LABEL: _Z20test_builtin_clrsbllx
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[X_NEG:.+]] = icmp slt i64 %[[X]], 0
+// OGCG-NEXT:    %[[X_NOT:.+]] = xor i64 %[[X]], -1
+// OGCG-NEXT:    %[[P:.+]] = select i1 %[[X_NEG]], i64 %[[X_NOT]], i64 %[[X]]
+// OGCG-NEXT:    %[[LZ:.+]] = call i64 @llvm.ctlz.i64(i64 %[[P]], i1 false)
+// OGCG-NEXT:    %{{.+}} = sub i64 %[[LZ]], 1
+
+int test_builtin_ctzs(unsigned short x) {
+  return __builtin_ctzs(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzst
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u16i) poison_zero : !u16i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u16i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzst
+// LLVM:         %{{.+}} = call i16 @llvm.cttz.i16(i16 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzst
+// OGCG:         %{{.+}} = call i16 @llvm.cttz.i16(i16 %{{.+}}, i1 true)
+
+int test_builtin_ctz(unsigned x) {
+  return __builtin_ctz(x);
+}
+
+// CIR-LABEL: _Z16test_builtin_ctzj
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z16test_builtin_ctzj
+// LLVM:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z16test_builtin_ctzj
+// OGCG:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_ctzl(unsigned long x) {
+  return __builtin_ctzl(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzlm
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzlm
+// LLVM:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzlm
+// OGCG:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_ctzll(unsigned long long x) {
+  return __builtin_ctzll(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_ctzlly
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z18test_builtin_ctzlly
+// LLVM:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z18test_builtin_ctzlly
+// OGCG:         %{{.+}} = call i64 @llvm.cttz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_ctzg(unsigned x) {
+  return __builtin_ctzg(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_ctzgj
+// CIR:         [[TMP:%.+]] = cir.bit.ctz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_ctzgj
+// LLVM:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_ctzgj
+// OGCG:         %{{.+}} = call i32 @llvm.cttz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_clzs(unsigned short x) {
+  return __builtin_clzs(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzst
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u16i) poison_zero : !u16i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u16i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzst
+// LLVM:         %{{.+}} = call i16 @llvm.ctlz.i16(i16 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzst
+// OGCG:         %{{.+}} = call i16 @llvm.ctlz.i16(i16 %{{.+}}, i1 true)
+
+int test_builtin_clz(unsigned x) {
+  return __builtin_clz(x);
+}
+
+// CIR-LABEL: _Z16test_builtin_clzj
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z16test_builtin_clzj
+// LLVM:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z16test_builtin_clzj
+// OGCG:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_clzl(unsigned long x) {
+  return __builtin_clzl(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzlm
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzlm
+// LLVM:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzlm
+// OGCG:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_clzll(unsigned long long x) {
+  return __builtin_clzll(x);
+}
+
+// CIR-LABEL: _Z18test_builtin_clzlly
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u64i) poison_zero : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z18test_builtin_clzlly
+// LLVM:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z18test_builtin_clzlly
+// OGCG:         %{{.+}} = call i64 @llvm.ctlz.i64(i64 %{{.+}}, i1 true)
+
+int test_builtin_clzg(unsigned x) {
+  return __builtin_clzg(x);
+}
+
+// CIR-LABEL: _Z17test_builtin_clzgj
+// CIR:         [[TMP:%.+]] = cir.bit.clz(%{{.+}} : !u32i) poison_zero : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z17test_builtin_clzgj
+// LLVM:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+// OGCG-LABEL: _Z17test_builtin_clzgj
+// OGCG:         %{{.+}} = call i32 @llvm.ctlz.i32(i32 %{{.+}}, i1 true)
+
+int test_builtin_parity(unsigned x) {
+  return __builtin_parity(x);
+}
+
+// CIR-LABEL: _Z19test_builtin_parityj
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z19test_builtin_parityj
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i32 @llvm.ctpop.i32(i32 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i32 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z19test_builtin_parityj
+// OGCG:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i32 @llvm.ctpop.i32(i32 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i32 %[[POPCNT]], 1
+
+int test_builtin_parityl(unsigned long x) {
+  return __builtin_parityl(x);
+}
+
+// CIR-LABEL: _Z20test_builtin_paritylm
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z20test_builtin_paritylm
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z20test_builtin_paritylm
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+int test_builtin_parityll(unsigned long long x) {
+  return __builtin_parityll(x);
+}
+
+// CIR-LABEL: _Z21test_builtin_paritylly
+// CIR:         [[TMP:%.+]] = cir.bit.parity(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z21test_builtin_paritylly
+// LLVM:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// LLVM-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// LLVM-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+// OGCG-LABEL: _Z21test_builtin_paritylly
+// OGCG:         %[[X:.+]] = load i64, ptr %{{.+}}, align 8
+// OGCG-NEXT:    %[[POPCNT:.+]] = call i64 @llvm.ctpop.i64(i64 %[[X]])
+// OGCG-NEXT:    %{{.+}} = and i64 %[[POPCNT]], 1
+
+int test_builtin_popcount(unsigned x) {
+  return __builtin_popcount(x);
+}
+
+// CIR-LABEL: _Z21test_builtin_popcountj
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z21test_builtin_popcountj
+// LLVM:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+// OGCG-LABEL: _Z21test_builtin_popcountj
+// OGCG:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+int test_builtin_popcountl(unsigned long x) {
+  return __builtin_popcountl(x);
+}
+
+// CIR-LABEL: _Z22test_builtin_popcountlm
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z22test_builtin_popcountlm
+// LLVM:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+// OGCG-LABEL: _Z22test_builtin_popcountlm
+// OGCG:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+int test_builtin_popcountll(unsigned long long x) {
+  return __builtin_popcountll(x);
+}
+
+// CIR-LABEL: _Z23test_builtin_popcountlly
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u64i) : !u64i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u64i), !s32i
+
+// LLVM-LABEL: _Z23test_builtin_popcountlly
+// LLVM:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+// OGCG-LABEL: _Z23test_builtin_popcountlly
+// OGCG:         %{{.+}} = call i64 @llvm.ctpop.i64(i64 %{{.+}})
+
+int test_builtin_popcountg(unsigned x) {
+  return __builtin_popcountg(x);
+}
+
+// CIR-LABEL: _Z22test_builtin_popcountgj
+// CIR:         [[TMP:%.+]] = cir.bit.popcnt(%{{.+}} : !u32i) : !u32i
+// CIR:         {{%.+}} = cir.cast(integral, [[TMP]] : !u32i), !s32i
+
+// LLVM-LABEL: _Z22test_builtin_popcountgj
+// LLVM:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
+
+// OGCG-LABEL: _Z22test_builtin_popcountgj
+// OGCG:         %{{.+}} = call i32 @llvm.ctpop.i32(i32 %{{.+}})
diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGen/builtin_call.cpp
index 0a2226a2cc59..ad0e47804083 100644
--- a/clang/test/CIR/CodeGen/builtin_call.cpp
+++ b/clang/test/CIR/CodeGen/builtin_call.cpp
@@ -27,7 +27,7 @@ int is_constant_evaluated() {
   return __builtin_is_constant_evaluated();
 }
 
-// CIR: cir.func @_Z21is_constant_evaluatedv() -> !s32i
+// CIR: cir.func{{.*}} @_Z21is_constant_evaluatedv() -> !s32i
 // CIR: %[[ZERO:.+]] = cir.const #cir.int<0>
 
 // LLVM: define {{.*}}i32 @_Z21is_constant_evaluatedv()
@@ -45,7 +45,7 @@ long double constant_fp_builtin_ld() {
   return __builtin_fabsl(-0.1L);
 }
 
-// CIR: cir.func @_Z22constant_fp_builtin_ldv() -> !cir.long_double<!cir.f80>
+// CIR: cir.func{{.*}} @_Z22constant_fp_builtin_ldv() -> !cir.long_double<!cir.f80>
 // CIR: %[[PONE:.+]] = cir.const #cir.fp<1.000000e-01> : !cir.long_double<!cir.f80>
 
 // LLVM: define {{.*}}x86_fp80 @_Z22constant_fp_builtin_ldv()
@@ -63,7 +63,7 @@ float constant_fp_builtin_single() {
   return __builtin_fabsf(-0.1f);
 }
 
-// CIR: cir.func @_Z26constant_fp_builtin_singlev() -> !cir.float
+// CIR: cir.func{{.*}} @_Z26constant_fp_builtin_singlev() -> !cir.float
 // CIR: %[[PONE:.+]] = cir.const #cir.fp<1.000000e-01> : !cir.float
 
 // LLVM: define {{.*}}float @_Z26constant_fp_builtin_singlev()
@@ -82,16 +82,16 @@ void library_builtins() {
   __builtin_abort();
 }
 
-// CIR: cir.func @_Z16library_builtinsv() {
+// CIR: cir.func{{.*}} @_Z16library_builtinsv() {
 // CIR: %[[NULL:.+]] = cir.const #cir.ptr<null> : !cir.ptr<!s8i>
-// CIR: cir.call @printf(%[[NULL]]) : (!cir.ptr<!s8i>) -> !s32i
-// CIR: cir.call @abort() : () -> ()
+// CIR: cir.call @printf(%[[NULL]]) nothrow : (!cir.ptr<!s8i>) -> !s32i
+// CIR: cir.call @abort() nothrow : () -> ()
 
-// LLVM: define void @_Z16library_builtinsv()
+// LLVM: define{{.*}} void @_Z16library_builtinsv()
 // LLVM: call i32 (ptr, ...) @printf(ptr null)
 // LLVM: call void @abort()
 
-// OGCG: define dso_local void @_Z16library_builtinsv()
+// OGCG: define{{.*}} void @_Z16library_builtinsv()
 // OGCG: call i32 (ptr, ...) @printf(ptr noundef null)
 // OGCG: call void @abort()
 
@@ -99,14 +99,54 @@ void assume(bool arg) {
   __builtin_assume(arg);
 }
 
-// CIR: cir.func @_Z6assumeb
+// CIR: cir.func{{.*}} @_Z6assumeb
 // CIR:   cir.assume %{{.+}} : !cir.bool
 // CIR: }
 
-// LLVM: define void @_Z6assumeb
+// LLVM: define {{.*}}void @_Z6assumeb
 // LLVM:   call void @llvm.assume(i1 %{{.+}})
 // LLVM: }
 
 // OGCG: define {{.*}}void @_Z6assumeb
 // OGCG:   call void @llvm.assume(i1 %{{.+}})
 // OGCG: }
+
+void expect(int x, int y) {
+  __builtin_expect(x, y);
+}
+
+// CIR-LABEL: cir.func{{.*}} @_Z6expectii
+// CIR:         %[[X:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR-NEXT:    %[[X_LONG:.+]] = cir.cast(integral, %[[X]] : !s32i), !s64i
+// CIR-NEXT:    %[[Y:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR-NEXT:    %[[Y_LONG:.+]] = cir.cast(integral, %[[Y]] : !s32i), !s64i
+// CIR-NEXT:    %{{.+}} = cir.expect(%[[X_LONG]], %[[Y_LONG]]) : !s64i
+// CIR:       }
+
+// LLVM-LABEL: define{{.*}} void @_Z6expectii
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[X_LONG:.+]] = sext i32 %[[X]] to i64
+// LLVM-NEXT:    %[[Y:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[Y_LONG:.+]] = sext i32 %[[Y]] to i64
+// LLVM-NEXT:    %{{.+}} = call i64 @llvm.expect.i64(i64 %[[X_LONG]], i64 %[[Y_LONG]])
+// LLVM:       }
+
+void expect_prob(int x, int y) {
+  __builtin_expect_with_probability(x, y, 0.25);
+}
+
+// CIR-LABEL: cir.func{{.*}} @_Z11expect_probii
+// CIR:         %[[X:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR-NEXT:    %[[X_LONG:.+]] = cir.cast(integral, %[[X]] : !s32i), !s64i
+// CIR-NEXT:    %[[Y:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR-NEXT:    %[[Y_LONG:.+]] = cir.cast(integral, %[[Y]] : !s32i), !s64i
+// CIR-NEXT:    %{{.+}} = cir.expect(%[[X_LONG]], %[[Y_LONG]], 2.500000e-01) : !s64i
+// CIR:       }
+
+// LLVM:       define{{.*}} void @_Z11expect_probii
+// LLVM:         %[[X:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[X_LONG:.+]] = sext i32 %[[X]] to i64
+// LLVM-NEXT:    %[[Y:.+]] = load i32, ptr %{{.+}}, align 4
+// LLVM-NEXT:    %[[Y_LONG:.+]] = sext i32 %[[Y]] to i64
+// LLVM-NEXT:    %{{.+}} = call i64 @llvm.expect.with.probability.i64(i64 %[[X_LONG]], i64 %[[Y_LONG]], double 2.500000e-01)
+// LLVM:       }
diff --git a/clang/test/CIR/CodeGen/builtin_printf.cpp b/clang/test/CIR/CodeGen/builtin_printf.cpp
index 366e474c2b09..43128e44b19e 100644
--- a/clang/test/CIR/CodeGen/builtin_printf.cpp
+++ b/clang/test/CIR/CodeGen/builtin_printf.cpp
@@ -5,8 +5,8 @@
 // RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
 // RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
 
-// CIR: cir.global "private" cir_private dsolocal @".str" = #cir.const_array<"%s\00" : !cir.array<!s8i x 3>> : !cir.array<!s8i x 3> 
-// CIR: cir.global "private" cir_private dsolocal @".str.1" = #cir.const_array<"%s %d\0A\00" : !cir.array<!s8i x 7>> : !cir.array<!s8i x 7>
+// CIR: cir.global "private" cir_private dso_local @".str" = #cir.const_array<"%s\00" : !cir.array<!s8i x 3>> : !cir.array<!s8i x 3> 
+// CIR: cir.global "private" cir_private dso_local @".str.1" = #cir.const_array<"%s %d\0A\00" : !cir.array<!s8i x 7>> : !cir.array<!s8i x 7>
 // LLVM: @.str = private global [3 x i8] c"%s\00"
 // LLVM: @.str.1 = private global [7 x i8] c"%s %d\0A\00"
 // OGCG: @.str = private unnamed_addr constant [3 x i8] c"%s\00"
@@ -18,27 +18,27 @@ void func(char const * const str, int i) {
   __builtin_printf("%s %d\n", str, i);
 }
 
-// CIR: cir.func @printf(!cir.ptr<!s8i>, ...) -> !s32i
+// CIR: cir.func{{.*}} @printf(!cir.ptr<!s8i>, ...) -> !s32i
 
-// CIR: cir.func @_Z4funcPKci(%[[arg0:.+]]: !cir.ptr<!s8i>{{.*}}, %[[arg1:.+]]: !s32i{{.*}}) {
+// CIR: cir.func{{.*}} @_Z4funcPKci(%[[arg0:.+]]: !cir.ptr<!s8i>{{.*}}, %[[arg1:.+]]: !s32i{{.*}}) {
 // CIR:   %[[str_ptr:.+]] = cir.alloca !cir.ptr<!s8i>, !cir.ptr<!cir.ptr<!s8i>>, ["str", init, const]
 // CIR:   %[[i_ptr:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init]
 // CIR:   cir.store %[[arg0]], %[[str_ptr]] : !cir.ptr<!s8i>, !cir.ptr<!cir.ptr<!s8i>>
 // CIR:   cir.store %[[arg1]], %[[i_ptr]] : !s32i, !cir.ptr<!s32i>
 // CIR:   %[[null_ptr:.+]] = cir.const #cir.ptr<null> : !cir.ptr<!s8i>
-// CIR:   %[[printf_result1:.+]] = cir.call @printf(%[[null_ptr]]) : (!cir.ptr<!s8i>) -> !s32i
+// CIR:   %[[printf_result1:.+]] = cir.call @printf(%[[null_ptr]]) nothrow : (!cir.ptr<!s8i>) -> !s32i
 // CIR:   %[[str_fmt_global:.+]] = cir.get_global @".str" : !cir.ptr<!cir.array<!s8i x 3>>
 // CIR:   %[[str_fmt_ptr:.+]] = cir.cast(array_to_ptrdecay, %[[str_fmt_global]] : !cir.ptr<!cir.array<!s8i x 3>>), !cir.ptr<!s8i>
 // CIR:   %[[str_val:.+]] = cir.load{{.*}} %[[str_ptr]] : !cir.ptr<!cir.ptr<!s8i>>, !cir.ptr<!s8i>
-// CIR:   %[[printf_result2:.+]] = cir.call @printf(%[[str_fmt_ptr]], %[[str_val]]) : (!cir.ptr<!s8i>, !cir.ptr<!s8i>) -> !s32i
+// CIR:   %[[printf_result2:.+]] = cir.call @printf(%[[str_fmt_ptr]], %[[str_val]]) nothrow : (!cir.ptr<!s8i>, !cir.ptr<!s8i>) -> !s32i
 // CIR:   %[[full_fmt_global:.+]] = cir.get_global @".str.1" : !cir.ptr<!cir.array<!s8i x 7>>
 // CIR:   %[[full_fmt_ptr:.+]] = cir.cast(array_to_ptrdecay, %[[full_fmt_global]] : !cir.ptr<!cir.array<!s8i x 7>>), !cir.ptr<!s8i>
 // CIR:   %[[str_val2:.+]] = cir.load{{.*}} %[[str_ptr]] : !cir.ptr<!cir.ptr<!s8i>>, !cir.ptr<!s8i>
 // CIR:   %[[i_val:.+]] = cir.load{{.*}} %[[i_ptr]] : !cir.ptr<!s32i>, !s32i
-// CIR:   %[[printf_result3:.+]] = cir.call @printf(%[[full_fmt_ptr]], %[[str_val2]], %[[i_val]]) : (!cir.ptr<!s8i>, !cir.ptr<!s8i>, !s32i) -> !s32i
+// CIR:   %[[printf_result3:.+]] = cir.call @printf(%[[full_fmt_ptr]], %[[str_val2]], %[[i_val]]) nothrow : (!cir.ptr<!s8i>, !cir.ptr<!s8i>, !s32i) -> !s32i
 // CIR:   cir.return
 
-// LLVM: define void @_Z4funcPKci(ptr %[[arg0:.+]], i32 %[[arg1:.+]])
+// LLVM: define{{.*}} void @_Z4funcPKci(ptr %[[arg0:.+]], i32 %[[arg1:.+]])
 // LLVM:   %[[str_ptr:.+]] = alloca ptr
 // LLVM:   %[[i_ptr:.+]] = alloca i32
 // LLVM:   store ptr %[[arg0]], ptr %[[str_ptr]]{{.*}}
@@ -51,7 +51,7 @@ void func(char const * const str, int i) {
 // LLVM:   %[[printf_result3:.+]] = call i32 (ptr, ...) @printf(ptr @.str.1, ptr %[[str_val2]], i32 %[[i_val]])
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4funcPKci(ptr noundef %[[arg0:.+]], i32 noundef %[[arg1:.+]])
+// OGCG: define{{.*}} void @_Z4funcPKci(ptr noundef %[[arg0:.+]], i32 noundef %[[arg1:.+]])
 // OGCG:   %[[str_ptr:.+]] = alloca ptr
 // OGCG:   %[[i_ptr:.+]] = alloca i32
 // OGCG:   store ptr %[[arg0]], ptr %[[str_ptr]]{{.*}}
diff --git a/clang/test/CIR/CodeGen/call.c b/clang/test/CIR/CodeGen/call.c
index f6aa41df7439..83a66fca638c 100644
--- a/clang/test/CIR/CodeGen/call.c
+++ b/clang/test/CIR/CodeGen/call.c
@@ -16,15 +16,15 @@ void f2() {
   f1(s);
 }
 
-// CIR-LABEL: cir.func @f2()
+// CIR-LABEL: cir.func{{.*}} @f2()
 // CIR:         %[[S:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!rec_S>, !rec_S
 // CIR-NEXT:    cir.call @f1(%[[S]]) : (!rec_S) -> ()
 
-// LLVM-LABEL: define void @f2()
+// LLVM-LABEL: define{{.*}} void @f2()
 // LLVM:         %[[S:.+]] = load %struct.S, ptr %{{.+}}, align 4
 // LLVM-NEXT:    call void @f1(%struct.S %[[S]])
 
-// OGCG-LABEL: define dso_local void @f2()
+// OGCG-LABEL: define{{.*}} void @f2()
 // OGCG:         %[[S:.+]] = load i64, ptr %{{.+}}, align 4
 // OGCG-NEXT:    call void @f1(i64 %[[S]])
 
@@ -33,15 +33,15 @@ void f4() {
   struct S s = f3();
 }
 
-// CIR-LABEL: cir.func @f4() {
+// CIR-LABEL: cir.func{{.*}} @f4() {
 // CIR:         %[[S:.+]] = cir.call @f3() : () -> !rec_S
 // CIR-NEXT:    cir.store align(4) %[[S]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
 
-// LLVM-LABEL: define void @f4() {
+// LLVM-LABEL: define{{.*}} void @f4() {
 // LLVM:         %[[S:.+]] = call %struct.S (...) @f3()
 // LLVM-NEXT:    store %struct.S %[[S]], ptr %{{.+}}, align 4
 
-// OGCG-LABEL: define dso_local void @f4() #0 {
+// OGCG-LABEL: define{{.*}} void @f4() #0 {
 // OGCG:         %[[S:.+]] = call i64 (...) @f3()
 // OGCG-NEXT:    store i64 %[[S]], ptr %{{.+}}, align 4
 
@@ -57,15 +57,15 @@ void f7() {
   f5(b);
 }
 
-// CIR-LABEL: cir.func @f7()
+// CIR-LABEL: cir.func{{.*}} @f7()
 // CIR:         %[[B:.+]] = cir.load align(4) %{{.+}} : !cir.ptr<!rec_Big>, !rec_Big
 // CIR-NEXT:    cir.call @f5(%[[B]]) : (!rec_Big) -> ()
 
-// LLVM-LABEL: define void @f7() {
+// LLVM-LABEL: define{{.*}} void @f7() {
 // LLVM:         %[[B:.+]] = load %struct.Big, ptr %{{.+}}, align 4
 // LLVM-NEXT:    call void @f5(%struct.Big %[[B]])
 
-// OGCG-LABEL: define dso_local void @f7() #0 {
+// OGCG-LABEL: define{{.*}} void @f7() #0 {
 // OGCG:         %[[B:.+]] = alloca %struct.Big, align 8
 // OGCG-NEXT:    call void @f5(ptr noundef byval(%struct.Big) align 8 %[[B]])
 
@@ -73,15 +73,15 @@ void f8() {
   struct Big b = f6();
 }
 
-// CIR-LABEL: cir.func @f8()
+// CIR-LABEL: cir.func{{.*}} @f8()
 // CIR:         %[[B:.+]] = cir.call @f6() : () -> !rec_Big
 // CIR:         cir.store align(4) %[[B]], %{{.+}} : !rec_Big, !cir.ptr<!rec_Big>
 
-// LLVM-LABEL: define void @f8() {
+// LLVM-LABEL: define{{.*}} void @f8() {
 // LLVM:        %[[B:.+]] = call %struct.Big (...) @f6()
 // LLVM-NEXT:   store %struct.Big %[[B]], ptr %{{.+}}, align 4
 
-// OGCG-LABEL: define dso_local void @f8() #0 {
+// OGCG-LABEL: define{{.*}} void @f8() #0 {
 // OGCG:         %[[B:.+]] = alloca %struct.Big, align 4
 // OGCG-NEXT:    call void (ptr, ...) @f6(ptr dead_on_unwind writable sret(%struct.Big) align 4 %[[B]])
 
@@ -89,21 +89,21 @@ void f9() {
   f1(f3());
 }
 
-// CIR-LABEL: cir.func @f9()
+// CIR-LABEL: cir.func{{.*}} @f9()
 // CIR:         %[[SLOT:.+]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["agg.tmp0"] {alignment = 4 : i64}
 // CIR-NEXT:    %[[RET:.+]] = cir.call @f3() : () -> !rec_S
 // CIR-NEXT:    cir.store align(4) %[[RET]], %[[SLOT]] : !rec_S, !cir.ptr<!rec_S>
 // CIR-NEXT:    %[[ARG:.+]] = cir.load align(4) %[[SLOT]] : !cir.ptr<!rec_S>, !rec_S
 // CIR-NEXT:    cir.call @f1(%[[ARG]]) : (!rec_S) -> ()
 
-// LLVM-LABEL: define void @f9() {
+// LLVM-LABEL: define{{.*}} void @f9() {
 // LLVM:         %[[SLOT:.+]] = alloca %struct.S, i64 1, align 4
 // LLVM-NEXT:    %[[RET:.+]] = call %struct.S (...) @f3()
 // LLVM-NEXT:    store %struct.S %[[RET]], ptr %[[SLOT]], align 4
 // LLVM-NEXT:    %[[ARG:.+]] = load %struct.S, ptr %[[SLOT]], align 4
 // LLVM-NEXT:    call void @f1(%struct.S %[[ARG]])
 
-// OGCG-LABEL: define dso_local void @f9() #0 {
+// OGCG-LABEL: define{{.*}} void @f9() #0 {
 // OGCG:         %[[SLOT:.+]] = alloca %struct.S, align 4
 // OGCG-NEXT:    %[[RET:.+]] = call i64 (...) @f3()
 // OGCG-NEXT:    store i64 %[[RET]], ptr %[[SLOT]], align 4
@@ -116,17 +116,17 @@ int f12(void) {
   return f10(1) + f11(2);
 }
 
-// CIR-LABEL: cir.func @f12() -> !s32i
+// CIR-LABEL: cir.func{{.*}} @f12() -> !s32i
 // CIR:         %[[A:.+]] = cir.const #cir.int<1> : !s32i
 // CIR-NEXT:    %{{.+}} = cir.call @f10(%[[A]]) side_effect(pure) : (!s32i) -> !s32i
 // CIR-NEXT:    %[[B:.+]] = cir.const #cir.int<2> : !s32i
 // CIR-NEXT:    %{{.+}} = cir.call @f11(%[[B]]) side_effect(const) : (!s32i) -> !s32i
 
-// LLVM-LABEL: define i32 @f12()
+// LLVM-LABEL: define{{.*}} i32 @f12()
 // LLVM:         %{{.+}} = call i32 @f10(i32 1) #[[ATTR0:.+]]
 // LLVM-NEXT:    %{{.+}} = call i32 @f11(i32 2) #[[ATTR1:.+]]
 
-// OGCG-LABEL: define dso_local i32 @f12()
+// OGCG-LABEL: define{{.*}} i32 @f12()
 // OGCG:         %{{.+}} = call i32 @f10(i32 noundef 1) #[[ATTR0:.+]]
 // OGCG-NEXT:    %{{.+}} = call i32 @f11(i32 noundef 2) #[[ATTR1:.+]]
 
diff --git a/clang/test/CIR/CodeGen/call.cpp b/clang/test/CIR/CodeGen/call.cpp
index cc25afce1e5a..43e5d150a949 100644
--- a/clang/test/CIR/CodeGen/call.cpp
+++ b/clang/test/CIR/CodeGen/call.cpp
@@ -8,11 +8,11 @@ void f2() {
   f1();
 }
 
-// CIR-LABEL: cir.func @_Z2f1v
-// CIR-LABEL: cir.func @_Z2f2v
+// CIR-LABEL: cir.func{{.*}} @_Z2f1v
+// CIR-LABEL: cir.func{{.*}} @_Z2f2v
 // CIR:         cir.call @_Z2f1v() : () -> ()
 
-// LLVM-LABEL: define void @_Z2f2v() {
+// LLVM-LABEL: define{{.*}} void @_Z2f2v() {
 // LLVM:         call void @_Z2f1v()
 
 int f3() { return 2; }
@@ -21,11 +21,11 @@ int f4() {
   return x;
 }
 
-// CIR-LABEL: cir.func @_Z2f3v() -> !s32i
-// CIR-LABEL: cir.func @_Z2f4v() -> !s32i
+// CIR-LABEL: cir.func{{.*}} @_Z2f3v() -> !s32i
+// CIR-LABEL: cir.func{{.*}} @_Z2f4v() -> !s32i
 // CIR:         cir.call @_Z2f3v() : () -> !s32i
 
-// LLVM-LABEL: define i32 @_Z2f4v() {
+// LLVM-LABEL: define{{.*}} i32 @_Z2f4v() {
 // LLVM:         %{{.+}} = call i32 @_Z2f3v()
 
 int f5(int a, int *b, bool c);
@@ -34,26 +34,26 @@ int f6() {
   return f5(2, &b, false);
 }
 
-// CIR-LABEL: cir.func @_Z2f6v() -> !s32i
+// CIR-LABEL: cir.func{{.*}} @_Z2f6v() -> !s32i
 // CIR:         %[[#b:]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
 // CIR:         %[[#a:]] = cir.const #cir.int<2> : !s32i
 // CIR-NEXT:    %[[#c:]] = cir.const #false
 // CIR-NEXT:    %{{.+}} = cir.call @_Z2f5iPib(%[[#a]], %[[#b:]], %[[#c]]) : (!s32i, !cir.ptr<!s32i>, !cir.bool) -> !s32i
 
-// LLVM-LABEL: define i32 @_Z2f6v() {
+// LLVM-LABEL: define{{.*}} i32 @_Z2f6v() {
 // LLVM:         %{{.+}} = call i32 @_Z2f5iPib(i32 2, ptr %{{.+}}, i1 false)
 
 int f7(int (*ptr)(int, int)) {
   return ptr(1, 2);
 }
 
-// CIR-LABEL: cir.func @_Z2f7PFiiiE
+// CIR-LABEL: cir.func{{.*}} @_Z2f7PFiiiE
 // CIR:         %[[#ptr:]] = cir.load{{.*}} %{{.+}} : !cir.ptr<!cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>>, !cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>
 // CIR-NEXT:    %[[#a:]] = cir.const #cir.int<1> : !s32i
 // CIR-NEXT:    %[[#b:]] = cir.const #cir.int<2> : !s32i
 // CIR-NEXT:    %{{.+}} = cir.call %[[#ptr]](%[[#a]], %[[#b]]) : (!cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>, !s32i, !s32i) -> !s32i
 
-// LLVM-LABEL: define i32 @_Z2f7PFiiiE
+// LLVM-LABEL: define{{.*}} i32 @_Z2f7PFiiiE
 // LLVM:         %[[#ptr:]] = load ptr, ptr %{{.+}}
 // LLVM-NEXT:    %{{.+}} = call i32 %[[#ptr]](i32 1, i32 2)
 
@@ -63,11 +63,11 @@ void f9() {
   f8(1, 2, 3, 4);
 }
 
-// CIR-LABEL: cir.func @_Z2f9v()
+// CIR-LABEL: cir.func{{.*}} @_Z2f9v()
 // CIR:         cir.call @_Z2f8iz(%{{.+}}) : (!s32i) -> ()
 // CIR:         cir.call @_Z2f8iz(%{{.+}}, %{{.+}}, %{{.+}}, %{{.+}}) : (!s32i, !s32i, !s32i, !s32i) -> ()
 
-// LLVM-LABEL: define void @_Z2f9v()
+// LLVM-LABEL: define{{.*}} void @_Z2f9v()
 // LLVM:         call void (i32, ...) @_Z2f8iz(i32 1)
 // LLVM:         call void (i32, ...) @_Z2f8iz(i32 1, i32 2, i32 3, i32 4)
 
@@ -81,11 +81,11 @@ void f11() {
   S s = f10();
 }
 
-// CIR-LABEL: cir.func @_Z3f11v()
+// CIR-LABEL: cir.func{{.*}} @_Z3f11v()
 // CIR:         %[[#s:]] = cir.call @_Z3f10v() : () -> !rec_S
 // CIR-NEXT:    cir.store align(4) %[[#s]], %{{.+}} : !rec_S, !cir.ptr<!rec_S>
 
-// LLVM-LABEL: define void @_Z3f11v()
+// LLVM-LABEL: define{{.*}} void @_Z3f11v()
 // LLVM:         %[[#s:]] = call %struct.S @_Z3f10v()
 // LLVM-NEXT:    store %struct.S %[[#s]], ptr %{{.+}}, align 4
 
@@ -93,12 +93,27 @@ void f12() {
   f10();
 }
 
-// CIR-LABEL: cir.func @_Z3f12v()
+// CIR-LABEL: cir.func{{.*}} @_Z3f12v()
 // CIR:         %[[#slot:]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["agg.tmp0"]
 // CIR-NEXT:    %[[#ret:]] = cir.call @_Z3f10v() : () -> !rec_S
 // CIR-NEXT:    cir.store align(4) %[[#ret]], %[[#slot]] : !rec_S, !cir.ptr<!rec_S>
 
-// LLVM-LABEL: define void @_Z3f12v() {
+// LLVM-LABEL: define{{.*}} void @_Z3f12v() {
 // LLVM:         %[[#slot:]] = alloca %struct.S, i64 1, align 4
 // LLVM-NEXT:    %[[#ret:]] = call %struct.S @_Z3f10v()
 // LLVM-NEXT:    store %struct.S %[[#ret]], ptr %[[#slot]], align 4
+
+void f13() noexcept;
+void f14() {
+  f13();
+}
+
+// CIR-LABEL: cir.func{{.+}} @_Z3f14v()
+// CIR:         cir.call @_Z3f13v() nothrow : () -> ()
+// CIR:       }
+
+// LLVM-LABEL: define{{.+}} void @_Z3f14v()
+// LLVM:         call void @_Z3f13v() #[[LLVM_ATTR_0:.+]]
+// LLVM:       }
+
+// LLLVM: attributes #[[LLVM_ATTR_0]] = { nounwind }
diff --git a/clang/test/CIR/CodeGen/cast.cpp b/clang/test/CIR/CodeGen/cast.cpp
index 84f55242a611..caf6de7c7d48 100644
--- a/clang/test/CIR/CodeGen/cast.cpp
+++ b/clang/test/CIR/CodeGen/cast.cpp
@@ -7,7 +7,7 @@ unsigned char cxxstaticcast_0(unsigned int x) {
   return static_cast<unsigned char>(x);
 }
 
-// CIR: cir.func @_Z15cxxstaticcast_0j
+// CIR: cir.func{{.*}} @_Z15cxxstaticcast_0j
 // CIR:    %[[XPTR:[0-9]+]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["x", init] {alignment = 4 : i64}
 // CIR:    %[[RV:[0-9]+]] = cir.alloca !u8i, !cir.ptr<!u8i>, ["__retval"] {alignment = 1 : i64}
 // CIR:    cir.store %arg0, %[[XPTR]] : !u32i, !cir.ptr<!u32i>
@@ -18,7 +18,7 @@ unsigned char cxxstaticcast_0(unsigned int x) {
 // CIR:    cir.return %[[R]] : !u8i
 // CIR:  }
 
-// LLVM: define i8 @_Z15cxxstaticcast_0j(i32 %{{[0-9]+}})
+// LLVM: define{{.*}} i8 @_Z15cxxstaticcast_0j(i32 %{{[0-9]+}})
 // LLVM: %[[LOAD:[0-9]+]] = load i32, ptr %{{[0-9]+}}, align 4
 // LLVM: %[[TRUNC:[0-9]+]] = trunc i32 %[[LOAD]] to i8
 // LLVM: store i8 %[[TRUNC]], ptr %[[RV:[0-9]+]], align 1
@@ -26,8 +26,8 @@ unsigned char cxxstaticcast_0(unsigned int x) {
 // LLVM: ret i8 %[[R]]
 
 int cStyleCasts_0(unsigned x1, int x2, float x3, short x4, double x5) {
-// CIR: cir.func @_Z13cStyleCasts_0jifsd
-// LLVM: define i32 @_Z13cStyleCasts_0jifsd
+// CIR: cir.func{{.*}} @_Z13cStyleCasts_0jifsd
+// LLVM: define{{.*}} i32 @_Z13cStyleCasts_0jifsd
 
   char a = (char)x1; // truncate
   // CIR: %{{[0-9]+}} = cir.cast(integral, %{{[0-9]+}} : !u32i), !s8i
@@ -89,13 +89,13 @@ bool cptr(void *d) {
   return x;
 }
 
-// CIR: cir.func @_Z4cptrPv(%arg0: !cir.ptr<!void>
+// CIR: cir.func{{.*}} @_Z4cptrPv(%arg0: !cir.ptr<!void>
 // CIR:   %[[DPTR:[0-9]+]] = cir.alloca !cir.ptr<!void>, !cir.ptr<!cir.ptr<!void>>, ["d", init] {alignment = 8 : i64}
 
 // CIR:   %[[DVAL:[0-9]+]] = cir.load{{.*}} %[[DPTR]] : !cir.ptr<!cir.ptr<!void>>, !cir.ptr<!void>
 // CIR:   %{{[0-9]+}} = cir.cast(ptr_to_bool, %[[DVAL]] : !cir.ptr<!void>), !cir.bool
 
-// LLVM-LABEL: define i1 @_Z4cptrPv(ptr %0)
+// LLVM-LABEL: define{{.*}} i1 @_Z4cptrPv(ptr %0)
 // LLVM:         %[[ARG_STORAGE:.*]] = alloca ptr, i64 1
 // LLVM:         %[[RETVAL:.*]] = alloca i8, i64 1
 // LLVM:         %[[X_STORAGE:.*]] = alloca i8, i64 1
@@ -114,7 +114,7 @@ void should_not_cast() {
   (void) ib; // void cast
 }
 
-// CIR:     cir.func @_Z15should_not_castv
+// CIR:     cir.func{{.*}} @_Z15should_not_castv
 // CIR-NOT:   cir.cast
 // CIR:     cir.return
 
diff --git a/clang/test/CIR/CodeGen/class.cpp b/clang/test/CIR/CodeGen/class.cpp
index d7f3772c9582..43dde12df40f 100644
--- a/clang/test/CIR/CodeGen/class.cpp
+++ b/clang/test/CIR/CodeGen/class.cpp
@@ -51,7 +51,7 @@ class Derived : public Base {
 
 int use(Derived *d) { return d->b; }
 
-// CIR: cir.func @_Z3useP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
+// CIR: cir.func{{.*}} @_Z3useP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
 // CIR:  %[[D_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Derived>, !cir.ptr<!cir.ptr<!rec_Derived>>, ["d", init]
 // CIR:  cir.store %[[ARG0]], %[[D_ADDR]]
 // CIR:  %[[D_PTR:.*]] = cir.load align(8) %0
@@ -69,7 +69,7 @@ int use_base() {
   return d.a;
 }
 
-// CIR: cir.func @_Z8use_basev
+// CIR: cir.func{{.*}} @_Z8use_basev
 // CIR:   %[[D_ADDR:.*]] = cir.alloca !rec_Derived, !cir.ptr<!rec_Derived>, ["d"]
 // CIR:   %[[BASE_ADDR:.*]] cir.base_class_addr %[[D_ADDR]] : !cir.ptr<!rec_Derived> nonnull [0] -> !cir.ptr<!rec_Base>
 // CIR:   %[[D_A_ADDR:.*]] = cir.get_member %2[0] {name = "a"} : !cir.ptr<!rec_Base> -> !cir.ptr<!s32i>
@@ -87,7 +87,7 @@ int use_base_via_pointer(Derived *d) {
   return d->a;
 }
 
-// CIR: cir.func @_Z20use_base_via_pointerP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
+// CIR: cir.func{{.*}} @_Z20use_base_via_pointerP7Derived(%[[ARG0:.*]]: !cir.ptr<!rec_Derived>
 // CIR:   %[[D_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Derived>, !cir.ptr<!cir.ptr<!rec_Derived>>, ["d", init]
 // CIR:   cir.store %[[ARG0]], %[[D_ADDR]]
 // CIR:   %[[D:.*]] = cir.load align(8) %[[D_ADDR]]
diff --git a/clang/test/CIR/CodeGen/cmp.cpp b/clang/test/CIR/CodeGen/cmp.cpp
index 40529d92b2a0..75c8cda0c360 100644
--- a/clang/test/CIR/CodeGen/cmp.cpp
+++ b/clang/test/CIR/CodeGen/cmp.cpp
@@ -14,7 +14,7 @@ void c0(int a, int b) {
   x = a == b;
 }
 
-// CIR-LABEL: cir.func @_Z2c0ii(
+// CIR-LABEL: cir.func{{.*}} @_Z2c0ii(
 
 // CIR: %[[A_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
@@ -45,7 +45,7 @@ void c0(int a, int b) {
 // CIR: %[[B6:.*]] = cir.load{{.*}} %[[B_PTR]]
 // CIR: %{{.*}} = cir.cmp(eq, %[[A6]], %[[B6]]) : !s32i, !cir.bool
 
-// LLVM-LABEL: define void @_Z2c0ii(i32 %0, i32 %1) {
+// LLVM-LABEL: define{{.*}} void @_Z2c0ii(i32 %0, i32 %1) {
 // LLVM: %[[PTR1:.*]] = alloca i32, i64 1
 // LLVM: %[[PTR2:.*]] = alloca i32, i64 1
 // LLVM: %[[BOOL_PTR:.*]] = alloca i8, i64 1
@@ -88,7 +88,7 @@ void c0(int a, int b) {
 // LLVM: %[[ZEXT6:.*]] = zext i1 %[[CMP6]] to i8
 // LLVM: store i8 %[[ZEXT6]], ptr %[[BOOL_PTR]]
 
-// OGCG-LABEL: define dso_local void @_Z2c0ii(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} {
+// OGCG-LABEL: define{{.*}} void @_Z2c0ii(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} {
 // OGCG: %[[PTR1:.*]] = alloca i32
 // OGCG: %[[PTR2:.*]] = alloca i32
 // OGCG: %[[BOOL_PTR:.*]] = alloca i8
@@ -140,7 +140,7 @@ void c0_unsigned(unsigned int a, unsigned int b) {
   x = a == b;
 }
 
-// CIR-LABEL: cir.func @_Z11c0_unsignedjj(
+// CIR-LABEL: cir.func{{.*}} @_Z11c0_unsignedjj(
 
 // CIR: %[[U_A_PTR:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
 // CIR: %[[U_B_PTR:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["b", init]
@@ -170,7 +170,7 @@ void c0_unsigned(unsigned int a, unsigned int b) {
 // CIR: %[[UB6:.*]] = cir.load{{.*}} %[[U_B_PTR]]
 // CIR: %{{.*}} = cir.cmp(eq, %[[UA6]], %[[UB6]]) : !u32i, !cir.bool
 
-// LLVM-LABEL: define void @_Z11c0_unsignedjj(i32 %0, i32 %1) {
+// LLVM-LABEL: define{{.*}} void @_Z11c0_unsignedjj(i32 %0, i32 %1) {
 // LLVM: %[[U_PTR1:.*]] = alloca i32, i64 1
 // LLVM: %[[U_PTR2:.*]] = alloca i32, i64 1
 // LLVM: %[[U_BOOL_PTR:.*]] = alloca i8, i64 1
@@ -213,7 +213,7 @@ void c0_unsigned(unsigned int a, unsigned int b) {
 // LLVM: %[[UZEXT6:.*]] = zext i1 %[[UCMP6]] to i8
 // LLVM: store i8 %[[UZEXT6]], ptr %[[U_BOOL_PTR]]
 
-// OGCG-LABEL: define dso_local void @_Z11c0_unsignedjj(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} {
+// OGCG-LABEL: define{{.*}} void @_Z11c0_unsignedjj(i32 {{.*}} %a, i32 {{.*}} %b) {{.*}} {
 // OGCG: %[[U_PTR1:.*]] = alloca i32
 // OGCG: %[[U_PTR2:.*]] = alloca i32
 // OGCG: %[[U_BOOL_PTR:.*]] = alloca i8
@@ -265,7 +265,7 @@ void c0_float(float a, float b) {
   x = a == b;
 }
 
-// CIR-LABEL: cir.func @_Z8c0_floatff(%arg0: !cir.float{{.*}}, %arg1: !cir.float{{.*}}) {
+// CIR-LABEL: cir.func{{.*}} @_Z8c0_floatff(%arg0: !cir.float{{.*}}, %arg1: !cir.float{{.*}}) {
 // CIR: %[[A_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b", init]
 // CIR: %[[X_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init]
@@ -303,7 +303,7 @@ void c0_float(float a, float b) {
 // CIR: %[[CMP6:.*]] = cir.cmp(eq, %[[A6]], %[[B6]]) : !cir.float, !cir.bool
 // CIR: cir.store{{.*}} %[[CMP6]], %[[X_PTR]] : !cir.bool, !cir.ptr<!cir.bool>
 
-// LLVM-LABEL: define void @_Z8c0_floatff(float %0, float %1) {
+// LLVM-LABEL: define{{.*}} void @_Z8c0_floatff(float %0, float %1) {
 // LLVM: %[[A_PTR:.*]] = alloca float
 // LLVM: %[[B_PTR:.*]] = alloca float
 // LLVM: store float %0, ptr %[[A_PTR]]
@@ -320,7 +320,7 @@ void c0_float(float a, float b) {
 // LLVM: fcmp une float %{{.*}}, %{{.*}}
 // LLVM: fcmp oeq float %{{.*}}, %{{.*}}
 
-// OGCG-LABEL: define dso_local void @_Z8c0_floatff(float {{.*}} %a, float  {{.*}} %b)  {{.*}} {
+// OGCG-LABEL: define{{.*}} void @_Z8c0_floatff(float {{.*}} %a, float  {{.*}} %b)  {{.*}} {
 // OGCG: %[[A_PTR:.*]] = alloca float
 // OGCG: %[[B_PTR:.*]] = alloca float
 // OGCG: store float %a, ptr %[[A_PTR]]
@@ -346,7 +346,7 @@ void pointer_cmp(int *a, int *b) {
   x = a != b;
 }
 
-// CIR-LABEL: cir.func @_Z11pointer_cmpPiS_(%arg0: !cir.ptr<!s32i>{{.*}}, %arg1: !cir.ptr<!s32i>{{.*}}) {
+// CIR-LABEL: cir.func{{.*}} @_Z11pointer_cmpPiS_(%arg0: !cir.ptr<!s32i>{{.*}}, %arg1: !cir.ptr<!s32i>{{.*}}) {
 // CIR: %[[A_PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["b", init]
 
@@ -360,7 +360,7 @@ void pointer_cmp(int *a, int *b) {
 // CIR: cir.cmp(eq, {{.*}}, {{.*}}) : !cir.ptr<!s32i>, !cir.bool
 // CIR: cir.cmp(ne, {{.*}}, {{.*}}) : !cir.ptr<!s32i>, !cir.bool
 
-// LLVM-LABEL: define void @_Z11pointer_cmpPiS_(ptr %0, ptr %1) {
+// LLVM-LABEL: define{{.*}} void @_Z11pointer_cmpPiS_(ptr %0, ptr %1) {
 // LLVM: %[[A_PTR:.*]] = alloca ptr
 // LLVM: %[[B_PTR:.*]] = alloca ptr
 // LLVM: store ptr %0, ptr %[[A_PTR]]
@@ -376,7 +376,7 @@ void pointer_cmp(int *a, int *b) {
 // LLVM: icmp eq ptr %{{.*}}, %{{.*}}
 // LLVM: icmp ne ptr %{{.*}}, %{{.*}}
 
-// OGCG-LABEL: define dso_local void @_Z11pointer_cmpPiS_(ptr {{.*}} %a, ptr {{.*}} %b) {{.*}} {
+// OGCG-LABEL: define{{.*}} void @_Z11pointer_cmpPiS_(ptr {{.*}} %a, ptr {{.*}} %b) {{.*}} {
 // OGCG: %[[A_PTR:.*]] = alloca ptr
 // OGCG: %[[B_PTR:.*]] = alloca ptr
 // OGCG: store ptr %a, ptr %[[A_PTR]]
@@ -401,7 +401,7 @@ void bool_cmp(bool a, bool b) {
   x = a != b;
 }
 
-// CIR-LABEL: cir.func @_Z8bool_cmpbb(%arg0: !cir.bool{{.*}}, %arg1: !cir.bool{{.*}}) {
+// CIR-LABEL: cir.func{{.*}} @_Z8bool_cmpbb(%arg0: !cir.bool{{.*}}, %arg1: !cir.bool{{.*}}) {
 // CIR: %[[A_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["a", init]
 // CIR: %[[B_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b", init]
 // CIR: %[[X_PTR:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init]
@@ -419,7 +419,7 @@ void bool_cmp(bool a, bool b) {
 // CIR: cir.cmp(eq
 // CIR: cir.cmp(ne
 
-// LLVM-LABEL: define void @_Z8bool_cmpbb(i1 %0, i1 %1) {
+// LLVM-LABEL: define{{.*}} void @_Z8bool_cmpbb(i1 %0, i1 %1) {
 // LLVM: %[[A_PTR:.*]] = alloca i8
 // LLVM: %[[B_PTR:.*]] = alloca i8
 // LLVM: %[[X_PTR:.*]] = alloca i8
@@ -444,7 +444,7 @@ void bool_cmp(bool a, bool b) {
 // LLVM: icmp eq
 // LLVM: icmp ne
 
-// OGCG-LABEL: define dso_local void @_Z8bool_cmpbb(i1 {{.*}} %a, i1 {{.*}} %b) {{.*}} {
+// OGCG-LABEL: define{{.*}} void @_Z8bool_cmpbb(i1 {{.*}} %a, i1 {{.*}} %b) {{.*}} {
 // OGCG: %[[A_PTR:.*]] = alloca i8
 // OGCG: %[[B_PTR:.*]] = alloca i8
 // OGCG: %[[X_PTR:.*]] = alloca i8
diff --git a/clang/test/CIR/CodeGen/comma.c b/clang/test/CIR/CodeGen/comma.c
index d811f5a72bdd..a1479b85d3f0 100644
--- a/clang/test/CIR/CodeGen/comma.c
+++ b/clang/test/CIR/CodeGen/comma.c
@@ -16,7 +16,7 @@ void comma(void) {
     i = 100, 200;
 }
 
-// CIR-LABEL: cir.func @comma() {
+// CIR-LABEL: cir.func{{.*}} @comma() {
 // CIR:         %[[B:.*]] = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b"]
 // CIR:         %[[C:.*]] = cir.alloca !s8i, !cir.ptr<!s8i>, ["c"]
 // CIR:         %[[F:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["f"]
diff --git a/clang/test/CIR/CodeGen/complex-builtins.cpp b/clang/test/CIR/CodeGen/complex-builtins.cpp
new file mode 100644
index 000000000000..2372ab571e53
--- /dev/null
+++ b/clang/test/CIR/CodeGen/complex-builtins.cpp
@@ -0,0 +1,35 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void foo() {
+ int _Complex a;
+ int _Complex b;
+ int _Complex r = __builtin_choose_expr(true, a, b);
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR: %[[COMPLEX_R:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["r", init]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP_A]], %[[COMPLEX_R]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_R:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
+// LLVM: store { i32, i32 } %[[TMP_A]], ptr %[[COMPLEX_R]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_R:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[R_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_R]], i32 0, i32 0
+// OGCG: %[[R_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_R]], i32 0, i32 1
+// OGCG: store i32 %[[A_REAL]], ptr %[[R_REAL_PTR]], align 4
+// OGCG: store i32 %[[A_IMAG]], ptr %[[R_IMAG_PTR]], align 4
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index cfeed345b4f1..25b00788ff01 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
 // RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
 // RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
 
 int _Complex ci;
@@ -216,6 +216,53 @@ void foo9(double a, double b) {
 // OGCG: store double %[[TMP_A]], ptr %[[C_REAL_PTR]], align 8
 // OGCG: store double %[[TMP_B]], ptr %[[C_IMAG_PTR]], align 8
 
+void foo12() {
+  double _Complex c;
+  double imag = __imag__ c;
+}
+
+// CIR: %[[COMPLEX:.*]] = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["c"]
+// CIR: %[[INIT:.*]] = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["imag", init]
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX]] : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[IMAG:.*]] = cir.complex.imag %[[TMP]] : !cir.complex<!cir.double> -> !cir.double
+// CIR: cir.store{{.*}} %[[IMAG]], %[[INIT]] : !cir.double, !cir.ptr<!cir.double>
+
+// LLVM: %[[COMPLEX:.*]] = alloca { double, double }, i64 1, align 8
+// LLVM: %[[INIT:.*]] = alloca double, i64 1, align 8
+// LLVM: %[[TMP:.*]] = load { double, double }, ptr %[[COMPLEX]], align 8
+// LLVM: %[[IMAG:.*]] = extractvalue { double, double } %[[TMP]], 1
+// LLVM: store double %[[IMAG]], ptr %[[INIT]], align 8
+
+// OGCG: %[[COMPLEX:.*]] = alloca { double, double }, align 8
+// OGCG: %[[INIT:.*]] = alloca double, align 8
+// OGCG: %[[IMAG:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: %[[TMP:.*]] = load double, ptr %[[IMAG]], align 8
+// OGCG: store double %[[TMP]], ptr %[[INIT]], align 8
+
+void foo13() {
+  double _Complex c;
+  double real = __real__ c;
+}
+
+// CIR: %[[COMPLEX:.*]] = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["c"]
+// CIR: %[[INIT:.*]] = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["real", init]
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX]] : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[REAL:.*]] = cir.complex.real %[[TMP]] : !cir.complex<!cir.double> -> !cir.double
+// CIR: cir.store{{.*}} %[[REAL]], %[[INIT]] : !cir.double, !cir.ptr<!cir.double>
+
+// LLVM: %[[COMPLEX:.*]] = alloca { double, double }, i64 1, align 8
+// LLVM: %[[INIT:.*]] = alloca double, i64 1, align 8
+// LLVM: %[[TMP:.*]] = load { double, double }, ptr %[[COMPLEX]], align 8
+// LLVM: %[[REAL:.*]] = extractvalue { double, double } %[[TMP]], 0
+// LLVM: store double %[[REAL]], ptr %[[INIT]], align 8
+
+// OGCG: %[[COMPLEX:.*]] = alloca { double, double }, align 8
+// OGCG: %[[INIT:.*]] = alloca double, align 8
+// OGCG: %[[REAL:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[TMP:.*]] = load double, ptr %[[REAL]], align 8
+// OGCG: store double %[[TMP]], ptr %[[INIT]], align 8
+
+
 void foo14() {
   int _Complex c = 2i;
 }
@@ -256,3 +303,372 @@ void foo15() {
 // OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
 // OGCG: store i32 %[[A_REAL]], ptr %[[B_REAL_PTR]], align 4
 // OGCG: store i32 %[[A_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+int foo16(int _Complex a, int _Complex b) {
+  return __imag__ a + __imag__ b;
+}
+
+// CIR: %[[RET:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[COMPLEX_A]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[B_IMAG:.*]] = cir.complex.imag %[[COMPLEX_B]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[ADD:.*]] = cir.binop(add, %[[A_IMAG]], %[[B_IMAG]]) nsw : !s32i
+// CIR: cir.store %[[ADD]], %[[RET]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[RET:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 1
+// LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 1
+// LLVM: %[[ADD:.*]] = add nsw i32 %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: store i32 %[[ADD]], ptr %[[RET]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_IMAG:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[TMP_A:.*]] = load i32, ptr %[[A_IMAG]], align 4
+// OGCG: %[[B_IMAG:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[TMP_B:.*]] = load i32, ptr %[[B_IMAG]], align 4
+// OGCG: %[[ADD:.*]] = add nsw i32 %[[TMP_A]], %[[TMP_B]]
+// OGCG: ret i32 %[[ADD]]
+
+int foo17(int _Complex a, int _Complex b) {
+  return __real__ a + __real__ b;
+}
+
+// CIR: %[[RET:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[COMPLEX_A]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[B_REAL:.*]] = cir.complex.real %[[COMPLEX_B]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[ADD:.*]] = cir.binop(add, %[[A_REAL]], %[[B_REAL]]) nsw : !s32i
+// CIR: cir.store %[[ADD]], %[[RET]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[RET:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 0
+// LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 0
+// LLVM: %[[ADD:.*]] = add nsw i32 %[[A_REAL]], %[[B_REAL]]
+// LLVM: store i32 %[[ADD]], ptr %[[RET]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[TMP_A:.*]] = load i32, ptr %[[A_REAL]], align 4
+// OGCG: %[[B_REAL:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[TMP_B:.*]] = load i32, ptr %[[B_REAL]], align 4
+// OGCG: %[[ADD:.*]] = add nsw i32 %[[TMP_A]], %[[TMP_B]]
+// OGCG: ret i32 %[[ADD]]
+
+bool foo18(int _Complex a, int _Complex b) {
+  return a == b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[RESULT:.*]] = cir.cmp(eq, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!s32i>, !cir.bool
+
+// LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = icmp eq i32 %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = icmp eq i32 %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = and i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[CMP_REAL:.*]] = icmp eq i32 %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = icmp eq i32 %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = and i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+bool foo19(double _Complex a, double _Complex b) {
+  return a == b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[RESULT:.*]] = cir.cmp(eq, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!cir.double>, !cir.bool
+
+
+// LLVM: %[[COMPLEX_A:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[COMPLEX_B:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[A_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = fcmp oeq double %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = fcmp oeq double %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = and i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { double, double }, align 8
+// OGCG: %[[COMPLEX_B:.*]] = alloca { double, double }, align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load double, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load double, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load double, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load double, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[CMP_REAL:.*]] = fcmp oeq double %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = fcmp oeq double %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = and i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+
+bool foo20(int _Complex a, int _Complex b) {
+  return a != b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[RESULT:.*]] = cir.cmp(ne, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!s32i>, !cir.bool
+
+// LLVM: %[[COMPLEX_A:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[COMPLEX_B:.*]] = load { i32, i32 }, ptr {{.*}}, align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = icmp ne i32 %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = icmp ne i32 %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[CMP_REAL:.*]] = icmp ne i32 %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = icmp ne i32 %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+bool foo21(double _Complex a, double _Complex b) {
+  return a != b;
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[COMPLEX_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+// CIR: %[[RESULT:.*]] = cir.cmp(ne, %[[COMPLEX_A]], %[[COMPLEX_B]]) : !cir.complex<!cir.double>, !cir.bool
+
+// LLVM: %[[COMPLEX_A:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[COMPLEX_B:.*]] = load { double, double }, ptr {{.*}}, align 8
+// LLVM: %[[A_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_A]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { double, double } %[[COMPLEX_B]], 1
+// LLVM: %[[CMP_REAL:.*]] = fcmp une double %[[A_REAL]], %[[B_REAL]]
+// LLVM: %[[CMP_IMAG:.*]] = fcmp une double %[[A_IMAG]], %[[B_IMAG]]
+// LLVM: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { double, double }, align 8
+// OGCG: %[[COMPLEX_B:.*]] = alloca { double, double }, align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: store double {{.*}}, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: store double {{.*}}, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load double, ptr %[[A_REAL_PTR]], align 8
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load double, ptr %[[A_IMAG_PTR]], align 8
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load double, ptr %[[B_REAL_PTR]], align 8
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load double, ptr %[[B_IMAG_PTR]], align 8
+// OGCG: %[[CMP_REAL:.*]] = fcmp une double %[[A_REAL]], %[[B_REAL]]
+// OGCG: %[[CMP_IMAG:.*]] = fcmp une double %[[A_IMAG]], %[[B_IMAG]]
+// OGCG: %[[RESULT:.*]] = or i1 %[[CMP_REAL]], %[[CMP_IMAG]]
+
+void foo22(int _Complex a, int _Complex b) {
+  int _Complex c = (a, b);
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP_B]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: store { i32, i32 } %[[TMP_B]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[B_REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG: store i32 %[[B_IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+void foo23(int _Complex a, int _Complex b) {
+  float _Complex f;
+  int _Complex c = _Generic(a, int _Complex: b, default: f);
+}
+
+// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[COMPLEX_F:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["f"]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_F:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_F:.*]] = alloca { float, float }, align 4
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[RESULT_REAL_PT:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PT:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[B_REAL]], ptr %[[RESULT_REAL_PT]], align 4
+// OGCG: store i32 %[[B_IMAG]], ptr %[[RESULT_IMAG_PT]], align 4
+
+void foo24() {
+  int _Complex arr[2];
+  int _Complex r = arr[1];
+}
+
+// CIR: %[[ARR:.*]] = cir.alloca !cir.array<!cir.complex<!s32i> x 2>, !cir.ptr<!cir.array<!cir.complex<!s32i> x 2>>, ["arr"]
+// CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["r", init]
+// CIR: %[[IDX:.*]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[ARR_PTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!cir.complex<!s32i> x 2>>), !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[RESULT_VAL:.*]] = cir.ptr_stride(%[[ARR_PTR]] : !cir.ptr<!cir.complex<!s32i>>, %[[IDX]] : !s32i), !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[RESULT_VAL]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[RESULT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[ARR:.*]] = alloca [2 x { i32, i32 }], i64 1, align 16
+// LLVM: %[[RESULT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[ARR_PTR:.*]] = getelementptr { i32, i32 }, ptr %[[ARR]], i32 0
+// LLVM: %[[RESULT_VAL:.*]] = getelementptr { i32, i32 }, ptr %[[ARR_PTR]], i64 1
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[RESULT_VAL]], align 8
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[RESULT]], align 4
+
+// OGCG: %[[ARR:.*]] = alloca [2 x { i32, i32 }], align 16
+// OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[ELEM_PTR:.*]] = getelementptr inbounds [2 x { i32, i32 }], ptr %[[ARR]], i64 0, i64 1
+// OGCG: %[[ELEM_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[ELEM_PTR]], i32 0, i32 0
+// OGCG: %[[ELEM_REAL:.*]] = load i32, ptr %[[ELEM_REAL_PTR]]
+// OGCG: %[[ELEM_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[ELEM_PTR]], i32 0, i32 1
+// OGCG: %[[ELEM_IMAG:.*]] = load i32, ptr %[[ELEM_IMAG_PTR]]
+// OGCG: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 0
+// OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[RESULT]], i32 0, i32 1
+// OGCG: store i32 %[[ELEM_REAL]], ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG: store i32 %[[ELEM_IMAG]], ptr %[[RESULT_IMAG_PTR]], align 4
+
+template <double _Complex N> void template_foo() { double _Complex C = N; }
+
+void foo25() {
+  template_foo<__builtin_complex(1.0, 2.0)>();
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["C", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.fp<1.000000e+00> : !cir.double, #cir.fp<2.000000e+00> : !cir.double> : !cir.complex<!cir.double>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>
+
+// LLVM: %[[INIT:.*]] = alloca { double, double }, i64 1, align 8
+// LLVM: store { double, double } { double 1.000000e+00, double 2.000000e+00 }, ptr %[[INIT]], align 8
+
+// OGCG: %[[INIT:.*]] = alloca { double, double }, align 8
+// OGCG: %[[INIT_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[INIT]], i32 0, i32 0
+// OGCG: %[[INIT_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[INIT]], i32 0, i32 1
+// OGCG: store double 1.000000e+00, ptr %[[INIT_REAL_PTR]], align 8
+// OGCG: store double 2.000000e+00, ptr %[[INIT_IMAG_PTR]], align 8
+
+void foo26(int _Complex* a) {
+  int _Complex b = *a;
+}
+
+// CIR: %[[COMPLEX_A_PTR:.*]] = cir.alloca !cir.ptr<!cir.complex<!s32i>>, !cir.ptr<!cir.ptr<!cir.complex<!s32i>>>, ["a", init]
+// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b", init]
+// CIR: %[[COMPLEX_A:.*]] = cir.load deref {{.*}} %[[COMPLEX_A_PTR]] : !cir.ptr<!cir.ptr<!cir.complex<!s32i>>>, !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[TMP]], %[[COMPLEX_B]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[COMPLEX_A_PTR:.*]] = alloca ptr, i64 1, align 8
+// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[COMPLEX_A:.*]] = load ptr, ptr %[[COMPLEX_A_PTR]], align 8
+// LLVM: %[[TMP:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
+// LLVM: store { i32, i32 } %[[TMP]], ptr %[[COMPLEX_B]], align 4
+
+// OGCG: %[[COMPLEX_A_PTR:.*]] = alloca ptr, align 8
+// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[COMPLEX_A:.*]] = load ptr, ptr %[[COMPLEX_A_PTR]], align 8
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: store i32 %[[A_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store i32 %[[A_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo29() {
+  using IntComplex = int _Complex;
+  int _Complex a = IntComplex{};
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a", init]
+// CIR: %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<0> : !s32i, #cir.int<0> : !s32i> : !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[INIT:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: store { i32, i32 } zeroinitializer, ptr %[[INIT]], align 4
+
+// OGCG: %[[INIT:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[INIT_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 0
+// OGCG: %[[INIT_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[INIT]], i32 0, i32 1
+// OGCG: store i32 0, ptr %[[INIT_REAL_PTR]], align 4
+// OGCG: store i32 0, ptr %[[INIT_IMAG_PTR]], align 4
diff --git a/clang/test/CIR/CodeGen/compound_assign.cpp b/clang/test/CIR/CodeGen/compound_assign.cpp
index 60442bcdf912..04bf406d6dd2 100644
--- a/clang/test/CIR/CodeGen/compound_assign.cpp
+++ b/clang/test/CIR/CodeGen/compound_assign.cpp
@@ -20,7 +20,7 @@ int compound_assign(int b) {
   return x;
 }
 
-// CIR: cir.func @_Z15compound_assigni
+// CIR: cir.func{{.*}} @_Z15compound_assigni
 // CIR:   %[[MUL:.*]] = cir.binop(mul, %{{.*}}, %{{.*}}) nsw : !s32i
 // CIR:   cir.store{{.*}} %[[MUL]], %{{.*}} : !s32i, !cir.ptr<!s32i>
 // CIR:   %[[DIV:.*]] = cir.binop(div, %{{.*}}, %{{.*}}) : !s32i
diff --git a/clang/test/CIR/CodeGen/ctor-alias.cpp b/clang/test/CIR/CodeGen/ctor-alias.cpp
new file mode 100644
index 000000000000..a20e206c6d71
--- /dev/null
+++ b/clang/test/CIR/CodeGen/ctor-alias.cpp
@@ -0,0 +1,75 @@
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+struct B {
+  B();
+};
+B::B() {
+}
+
+// OGCG: @_ZN1BC1Ev = unnamed_addr alias void (ptr), ptr @_ZN1BC2Ev
+
+// CHECK: cir.func{{.*}} @_ZN1BC2Ev(%arg0: !cir.ptr<!rec_B>
+// CHECK:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_B>, !cir.ptr<!cir.ptr<!rec_B>>, ["this", init]
+// CHECK:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_B>>, !cir.ptr<!rec_B>
+
+// CHECK: cir.func{{.*}} private dso_local @_ZN1BC1Ev(!cir.ptr<!rec_B>) alias(@_ZN1BC2Ev)
+
+// LLVM: define{{.*}} @_ZN1BC2Ev(ptr %[[THIS_ARG:.*]])
+// LLVM:   %[[THIS_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]]
+// LLVM:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+
+// This should be an alias, like the similar OGCG alias above, but that's not
+// implemented yet.
+// LLVM: declare dso_local void @_ZN1BC1Ev(ptr)
+
+// OGCG: define{{.*}} @_ZN1BC2Ev(ptr{{.*}} %[[THIS_ARG:.*]])
+// OGCG:   %[[THIS_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]]
+// OGCG:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+
+// The constructor in this cases is handled by RAUW rather than aliasing.
+struct Struk {
+  Struk() {}
+};
+
+void baz() {
+  Struk s;
+}
+
+// CHECK:   cir.func{{.*}} @_ZN5StrukC2Ev(%arg0: !cir.ptr<!rec_Struk>
+// CHECK:     %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>, ["this", init]
+// CHECK:     cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>
+// CHECK:     %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_Struk>>, !cir.ptr<!rec_Struk>
+// CHECK:     cir.return
+
+// CHECK-NOT:   cir.func{{.*}} @_ZN5StrukC1Ev
+
+// CHECK:   cir.func{{.*}} @_Z3bazv()
+// CHECK:     %[[S_ADDR:.*]] = cir.alloca !rec_Struk, !cir.ptr<!rec_Struk>, ["s", init]
+// CHECK:     cir.call @_ZN5StrukC2Ev(%[[S_ADDR]]) : (!cir.ptr<!rec_Struk>) -> ()
+
+// LLVM: define linkonce_odr void @_ZN5StrukC2Ev(ptr %[[THIS_ARG]])
+// LLVM:   %[[THIS_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]]
+// LLVM:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+
+// LLVM: define{{.*}} void @_Z3bazv()
+// LLVM:   %[[S_ADDR:.*]] = alloca %struct.Struk
+// LLVM:   call void @_ZN5StrukC2Ev(ptr{{.*}} %[[S_ADDR]])
+
+// This function gets emitted before the constructor in OGCG.
+// OGCG: define{{.*}} void @_Z3bazv()
+// OGCG:   %[[S_ADDR:.*]] = alloca %struct.Struk
+// OGCG:   call void @_ZN5StrukC2Ev(ptr{{.*}} %[[S_ADDR]])
+
+// OGCG: define linkonce_odr void @_ZN5StrukC2Ev(ptr{{.*}} %[[THIS_ARG]])
+// OGCG:   %[[THIS_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[THIS_ARG]], ptr %[[THIS_ADDR]]
+// OGCG:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
diff --git a/clang/test/CIR/CodeGen/ctor.cpp b/clang/test/CIR/CodeGen/ctor.cpp
index 1a36eb0d9d3a..4c2877f8460d 100644
--- a/clang/test/CIR/CodeGen/ctor.cpp
+++ b/clang/test/CIR/CodeGen/ctor.cpp
@@ -16,20 +16,20 @@ void baz() {
 //       constructors here. The handling of constructor aliases is currently
 //       NYI, but when it is added this test should be updated to add a RUN
 //       line that passes '-mconstructor-aliases' to clang_cc1.
-// CHECK:   cir.func @_ZN5StrukC2Ev(%arg0: !cir.ptr<!rec_Struk>
+// CHECK:   cir.func{{.*}} @_ZN5StrukC2Ev(%arg0: !cir.ptr<!rec_Struk>
 // CHECK-NEXT:     %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>, ["this", init] {alignment = 8 : i64}
 // CHECK-NEXT:     cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>
 // CHECK-NEXT:     %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_Struk>>, !cir.ptr<!rec_Struk>
 // CHECK-NEXT:     cir.return
 
-// CHECK:   cir.func @_ZN5StrukC1Ev(%arg0: !cir.ptr<!rec_Struk>
+// CHECK:   cir.func{{.*}} @_ZN5StrukC1Ev(%arg0: !cir.ptr<!rec_Struk>
 // CHECK-NEXT:     %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>, ["this", init] {alignment = 8 : i64}
 // CHECK-NEXT:     cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_Struk>, !cir.ptr<!cir.ptr<!rec_Struk>>
 // CHECK-NEXT:     %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_Struk>>, !cir.ptr<!rec_Struk>
 // CHECK-NEXT:     cir.call @_ZN5StrukC2Ev(%[[THIS]]) : (!cir.ptr<!rec_Struk>) -> ()
 // CHECK-NEXT:     cir.return
 
-// CHECK:   cir.func @_Z3bazv()
+// CHECK:   cir.func{{.*}} @_Z3bazv()
 // CHECK-NEXT:     %[[S_ADDR:.*]] = cir.alloca !rec_Struk, !cir.ptr<!rec_Struk>, ["s", init] {alignment = 4 : i64}
 // CHECK-NEXT:     cir.call @_ZN5StrukC1Ev(%[[S_ADDR]]) : (!cir.ptr<!rec_Struk>) -> ()
 // CHECK-NEXT:     cir.return
@@ -45,9 +45,9 @@ void bar() {
 
 // When a variadic constructor is present, we call the C2 constructor directly.
 
-// CHECK-NOT: cir.func @_ZN13VariadicStrukC2Eiz
+// CHECK-NOT: cir.func{{.*}} @_ZN13VariadicStrukC2Eiz
 
-// CHECK:      cir.func @_ZN13VariadicStrukC1Eiz(%arg0: !cir.ptr<!rec_VariadicStruk>
+// CHECK:      cir.func{{.*}} @_ZN13VariadicStrukC1Eiz(%arg0: !cir.ptr<!rec_VariadicStruk>
 // CHECK-SAME:                                   %arg1: !s32i
 // CHECK-SAME:                                   ...) {
 // CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
@@ -60,7 +60,7 @@ void bar() {
 // CHECK-NEXT:   cir.store{{.*}} %[[N]], %[[A_ADDR]]
 // CHECK-NEXT:   cir.return
 
-// CHECK:  cir.func @_Z3barv
+// CHECK:  cir.func{{.*}} @_Z3barv
 // CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca !rec_VariadicStruk, !cir.ptr<!rec_VariadicStruk>, ["s", init]
 // CHECK-NEXT:    %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
 // CHECK-NEXT:    %[[TWO:.*]] = cir.const #cir.int<2> : !s32i
@@ -78,7 +78,7 @@ void bam() {
   DelegatingStruk s;
 }
 
-// CHECK:       cir.func @_ZN15DelegatingStrukC2Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK:       cir.func{{.*}} @_ZN15DelegatingStrukC2Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
 // CHECK-SAME:                                     %arg1: !s32i
 // CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
 // CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
@@ -90,7 +90,7 @@ void bam() {
 // CHECK-NEXT:   cir.store{{.*}} %[[N]], %[[A_ADDR]]
 // CHECK-NEXT:   cir.return
 
-// CHECK:       cir.func @_ZN15DelegatingStrukC1Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK:       cir.func{{.*}} @_ZN15DelegatingStrukC1Ei(%arg0: !cir.ptr<!rec_DelegatingStruk>
 // CHECK-SAME:                                     %arg1: !s32i
 // CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
 // CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
@@ -101,7 +101,7 @@ void bam() {
 // CHECK-NEXT:   cir.call @_ZN15DelegatingStrukC2Ei(%[[THIS]], %[[N]])
 // CHECK-NEXT:   cir.return
 
-// CHECK: cir.func @_ZN15DelegatingStrukC1Ev(%arg0: !cir.ptr<!rec_DelegatingStruk>
+// CHECK: cir.func{{.*}} @_ZN15DelegatingStrukC1Ev(%arg0: !cir.ptr<!rec_DelegatingStruk>
 // CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
 // CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
 // CHECK-NEXT:   %[[THIS:.*]] = cir.load{{.*}} %[[THIS_ADDR]]
@@ -109,7 +109,113 @@ void bam() {
 // CHECK-NEXT:   cir.call @_ZN15DelegatingStrukC1Ei(%[[THIS]], %[[ZERO]])
 // CHECK-NEXT:   cir.return
 
-// CHECK: cir.func @_Z3bamv
+// CHECK: cir.func{{.*}} @_Z3bamv
 // CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca {{.*}} ["s", init]
 // CHECK-NEXT:    cir.call @_ZN15DelegatingStrukC1Ev(%[[S_ADDR]])
 // CHECK-NEXT:    cir.return
+
+struct MemberInitStruk {
+  int a;
+  MemberInitStruk() : a(0) {}
+};
+
+void init_member() {
+  MemberInitStruk s;
+}
+
+// CHECK:      cir.func{{.*}} @_ZN15MemberInitStrukC2Ev(%arg0: !cir.ptr<!rec_MemberInitStruk>
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[A_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CHECK-NEXT:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CHECK-NEXT:   cir.store align(4) %[[ZERO]], %[[A_ADDR]]
+// CHECK-NEXT:   cir.return
+
+// CHECK:      cir.func{{.*}} @_ZN15MemberInitStrukC1Ev(%arg0: !cir.ptr<!rec_MemberInitStruk>
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.call @_ZN15MemberInitStrukC2Ev(%[[THIS]])
+// CHECK-NEXT:   cir.return
+
+// CHECK: cir.func{{.*}} @_Z11init_memberv
+// CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca {{.*}} ["s", init]
+// CHECK-NEXT:    cir.call @_ZN15MemberInitStrukC1Ev(%[[S_ADDR]])
+// CHECK-NEXT:    cir.return
+
+struct ParamMemberInitStruk {
+  int a;
+  ParamMemberInitStruk(int n) : a(n) {}
+};
+
+void init_param_member() {
+  ParamMemberInitStruk s(0);
+}
+
+// CHECK:      cir.func{{.*}} @_ZN20ParamMemberInitStrukC2Ei(%arg0: !cir.ptr<!rec_ParamMemberInitStruk>
+// CHECK-SAME:                                         %arg1: !s32i
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.store %arg1, %[[N_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[A_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = "a"}
+// CHECK-NEXT:   %[[N:.*]] = cir.load{{.*}} %[[N_ADDR]]
+// CHECK-NEXT:   cir.store{{.*}} %[[N]], %[[A_ADDR]]
+// CHECK-NEXT:   cir.return
+
+// CHECK:      cir.func{{.*}} @_ZN20ParamMemberInitStrukC1Ei(%arg0: !cir.ptr<!rec_ParamMemberInitStruk>
+// CHECK-SAME:                                         %arg1: !s32i
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   %[[N_ADDR:.*]] = cir.alloca {{.*}} ["n", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.store %arg1, %[[N_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[N:.*]] = cir.load{{.*}} %[[N_ADDR]]
+// CHECK-NEXT:   cir.call @_ZN20ParamMemberInitStrukC2Ei(%[[THIS]], %[[N]])
+// CHECK-NEXT:   cir.return
+
+// CHECK: cir.func{{.*}} @_Z17init_param_memberv
+// CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca {{.*}} ["s", init]
+// CHECK-NEXT:    %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT:    cir.call @_ZN20ParamMemberInitStrukC1Ei(%[[S_ADDR]], %[[ZERO]])
+// CHECK-NEXT:    cir.return
+
+struct UnionInitStruk {
+  union {
+    int a;
+    union {
+        float b;
+        double c;
+    };
+  };
+  UnionInitStruk() : c(0.0) {}
+};
+
+void init_union() {
+  UnionInitStruk s;
+}
+
+// CHECK:      cir.func{{.*}} @_ZN14UnionInitStrukC2Ev(%arg0: !cir.ptr<!rec_UnionInitStruk>
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[AU1_ADDR:.*]] = cir.get_member %[[THIS]][0] {name = ""}
+// CHECK-NEXT:   %[[AU2_ADDR:.*]] = cir.get_member %[[AU1_ADDR]][1] {name = ""}
+// CHECK-NEXT:   %[[C_ADDR:.*]] = cir.get_member %[[AU2_ADDR]][1] {name = "c"}
+// CHECK-NEXT:   %[[ZERO:.*]] = cir.const #cir.fp<0.000000e+00>
+// CHECK-NEXT:   cir.store{{.*}} %[[ZERO]], %[[C_ADDR]]
+// CHECK-NEXT:   cir.return
+
+// CHECK:      cir.func{{.*}} @_ZN14UnionInitStrukC1Ev(%arg0: !cir.ptr<!rec_UnionInitStruk>
+// CHECK-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca {{.*}} ["this", init]
+// CHECK-NEXT:   cir.store %arg0, %[[THIS_ADDR]]
+// CHECK-NEXT:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]]
+// CHECK-NEXT:   cir.call @_ZN14UnionInitStrukC2Ev
+// CHECK-NEXT:   cir.return
+
+// CHECK: cir.func{{.*}} @_Z10init_unionv
+// CHECK-NEXT:    %[[S_ADDR:.*]] = cir.alloca {{.*}} ["s", init]
+// CHECK-NEXT:    cir.call @_ZN14UnionInitStrukC1Ev(%[[S_ADDR]])
+// CHECK-NEXT:    cir.return
diff --git a/clang/test/CIR/CodeGen/deferred-defs.cpp b/clang/test/CIR/CodeGen/deferred-defs.cpp
index 4230a4ed3e24..0b9ff92fa2f4 100644
--- a/clang/test/CIR/CodeGen/deferred-defs.cpp
+++ b/clang/test/CIR/CodeGen/deferred-defs.cpp
@@ -18,5 +18,5 @@ void use() {
 }
 
 // CIR: cir.global external @locallyDefined = #cir.int<0> : !s32i
-// CIR: cir.global "private" internal dsolocal @_ZN12_GLOBAL__N_112usedInternalE = #cir.int<0> : !s32i
+// CIR: cir.global "private" internal dso_local @_ZN12_GLOBAL__N_112usedInternalE = #cir.int<0> : !s32i
 // CIR: cir.global "private" external @usedExternal : !s32i
diff --git a/clang/test/CIR/CodeGen/dso-local.c b/clang/test/CIR/CodeGen/dso-local.c
index 07c833d2fbc9..01c93cbd81ee 100644
--- a/clang/test/CIR/CodeGen/dso-local.c
+++ b/clang/test/CIR/CodeGen/dso-local.c
@@ -3,7 +3,9 @@
 
 // These are here so we find this test when grepping for missing features.
 // cir::MissingFeatures::opGlobalThreadLocal()
-// cir::MissingFeatures::opFuncDsoLocal()
+
+// Note: Unlike CIR doesn't set dso_local on function declarations. This is
+//       a difference from classic codege in the STATIC checks.
 
 /// Static relocation model defaults to -fdirect-access-external-data and sets
 /// dso_local on most global objects.
@@ -13,6 +15,9 @@
 // STATIC-NEXT: @import_var = external dso_local global i32
 // STATIC-NEXT: @weak_bar = extern_weak dso_local global i32
 // STATIC-NEXT: @bar = external dso_local global i32
+// STATIC-DAG: declare void @foo()
+// STATIC-DAG: define dso_local ptr @zed()
+// STATIC-DAG: declare void @import_func()
 
 /// If -fno-direct-access-external-data is set, drop dso_local from global variable
 /// declarations.
@@ -21,30 +26,45 @@
 // STATIC-INDIRECT-NEXT: @import_var = external global i32
 // STATIC-INDIRECT-NEXT: @weak_bar = extern_weak global i32
 // STATIC-INDIRECT-NEXT: @bar = external global i32
+// STATIC-INDIRECT-DAG:  declare void @import_func()
+// STATIC-INDIRECT-DAG:  define dso_local ptr @zed()
+// STATIC-INDIRECT-DAG:  declare void @foo()
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie %s -o - | FileCheck --check-prefix=PIE %s
 // PIE:      @baz = dso_local global i32 42
 // PIE-NEXT: @import_var = external global i32
 // PIE-NEXT: @weak_bar = extern_weak global i32
 // PIE-NEXT: @bar = external global i32
+// PIE-DAG: declare void @foo()
+// PIE-DAG: define dso_local ptr @zed()
+// PIE-DAG: declare void @import_func()
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie -fdirect-access-external-data %s -o - | FileCheck --check-prefix=PIE-DIRECT %s
 // PIE-DIRECT:      @baz = dso_local global i32 42
 // PIE-DIRECT-NEXT: @import_var = external dso_local global i32
 // PIE-DIRECT-NEXT: @weak_bar = extern_weak global i32
 // PIE-DIRECT-NEXT: @bar = external dso_local global i32
+// PIE-DIRECT-DAG: declare void @foo()
+// PIE-DIRECT-DAG: define dso_local ptr @zed()
+// PIE-DIRECT-DAG: declare void @import_func()
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -mrelocation-model static -fno-plt %s -o - | FileCheck --check-prefix=NOPLT %s
 // NOPLT:      @baz = dso_local global i32 42
 // NOPLT-NEXT: @import_var = external dso_local global i32
 // NOPLT-NEXT: @weak_bar = extern_weak dso_local global i32
 // NOPLT-NEXT: @bar = external dso_local global i32
+// NOPLT-DAG: declare void @foo()
+// NOPLT-DAG: define dso_local ptr @zed()
+// NOPLT-DAG: declare void @import_func()
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -fno-plt -pic-level 1 -pic-is-pie -fdirect-access-external-data %s -o - | FileCheck --check-prefix=PIE-DIRECT-NOPLT %s
 // PIE-DIRECT-NOPLT:      @baz = dso_local global i32 42
 // PIE-DIRECT-NOPLT-NEXT: @import_var = external dso_local global i32
 // PIE-DIRECT-NOPLT-NEXT: @weak_bar = extern_weak global i32
 // PIE-DIRECT-NOPLT-NEXT: @bar = external dso_local global i32
+// PIE-DIRECT-NOPLT-DAG: declare void @foo()
+// PIE-DIRECT-NOPLT-DAG: define dso_local ptr @zed()
+// PIE-DIRECT-NOPLT-DAG: declare void @import_func()
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 1 -pic-is-pie -fno-plt %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
 // RUN: %clang_cc1 -triple powerpc64le -fclangir -emit-llvm -mrelocation-model static %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
@@ -52,24 +72,34 @@
 // PIE-NO-PLT-NEXT: @import_var = external global i32
 // PIE-NO-PLT-NEXT: @weak_bar = extern_weak global i32
 // PIE-NO-PLT-NEXT: @bar = external global i32
+// PIE-NO-PLT-DAG:  declare void @import_func()
+// PIE-NO-PLT-DAG:  define dso_local ptr @zed()
+// PIE-NO-PLT-DAG:  declare void @foo()
 
 /// -fdirect-access-external-data is currently ignored for -fPIC.
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 2 %s -o - | FileCheck --check-prefix=SHARED %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm -pic-level 2 -fdirect-access-external-data %s -o - | FileCheck --check-prefix=SHARED %s
 // SHARED-DAG: @bar = external global i32
 // SHARED-DAG: @weak_bar = extern_weak global i32
+// SHARED-DAG: declare void @foo()
 // SHARED-DAG: @baz ={{.*}} global i32 42
+// SHARED-DAG: define{{.*}} ptr @zed()
 
 int baz = 42;
 __attribute__((dllimport)) extern int import_var;
 __attribute__((weak)) extern int weak_bar;
 extern int bar;
+__attribute__((dllimport)) void import_func(void);
 
 int *use_import(void) {
+  import_func();
   return &import_var;
 }
 
+void foo(void);
+
 int *zed(void) {
+  foo();
   if (baz)
     return &weak_bar;
   return &bar;
diff --git a/clang/test/CIR/CodeGen/forrange.cpp b/clang/test/CIR/CodeGen/forrange.cpp
index 45e146e9091d..485e9c331417 100644
--- a/clang/test/CIR/CodeGen/forrange.cpp
+++ b/clang/test/CIR/CodeGen/forrange.cpp
@@ -13,10 +13,10 @@ void for_range() {
     ;
 }
 
-// CIR: cir.func @_Z5beginR9Container(!cir.ptr<!rec_Container>) -> !cir.ptr<!rec_Element>
-// CIR: cir.func @_Z3endR9Container(!cir.ptr<!rec_Container>) -> !cir.ptr<!rec_Element
+// CIR: cir.func{{.*}} @_Z5beginR9Container(!cir.ptr<!rec_Container>) -> !cir.ptr<!rec_Element>
+// CIR: cir.func{{.*}} @_Z3endR9Container(!cir.ptr<!rec_Container>) -> !cir.ptr<!rec_Element
 
-// CIR: cir.func @_Z9for_rangev()
+// CIR: cir.func{{.*}} @_Z9for_rangev()
 // CIR:    %[[C_ADDR:.*]] = cir.alloca !rec_Container{{.*}} ["c"]
 // CIR:    cir.scope {
 // CIR:      %[[RANGE_ADDR:.*]] = cir.alloca !cir.ptr<!rec_Container>{{.*}} ["__range1", init, const]
@@ -59,7 +59,7 @@ void for_range2() {
     ;
 }
 
-// CIR: cir.func @_Z10for_range2v()
+// CIR: cir.func{{.*}} @_Z10for_range2v()
 // CIR:    %[[C_ADDR:.*]] = cir.alloca !rec_C2{{.*}} ["c"]
 // CIR:    cir.scope {
 // CIR:      %[[RANGE_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C2>{{.*}} ["__range1", init, const]
@@ -111,7 +111,7 @@ void for_range3() {
     ;
 }
 
-// CIR: cir.func @_Z10for_range3v()
+// CIR: cir.func{{.*}} @_Z10for_range3v()
 // CIR:    %[[C_ADDR:.*]] = cir.alloca !rec_C3{{.*}} ["c"]
 // CIR:    cir.scope {
 // CIR:      %[[RANGE_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C3>{{.*}} ["__range1", init, const]
diff --git a/clang/test/CIR/CodeGen/generic-selection.c b/clang/test/CIR/CodeGen/generic-selection.c
new file mode 100644
index 000000000000..c1659382a705
--- /dev/null
+++ b/clang/test/CIR/CodeGen/generic-selection.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void foo() {
+  int a;
+  int r = _Generic(a, double: 1, float: 2, int: 3, default: 4);
+}
+
+// CIR: %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a"]
+// CIR: %[[RES:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["r", init]
+// CIR: %[[RES_VAL:.*]] = cir.const #cir.int<3> : !s32i
+// CIR: cir.store{{.*}} %[[RES_VAL]], %[[RES]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[A:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RES:.*]] = alloca i32, i64 1, align 4
+// LLVM: store i32 3, ptr %[[RES]], align 4
+
+// OGCG: %[[A:.*]] = alloca i32, align 4
+// OGCG: %[[RES:.*]] = alloca i32, align 4
+// OGCG: store i32 3, ptr %[[RES]], align 4
diff --git a/clang/test/CIR/CodeGen/if.cpp b/clang/test/CIR/CodeGen/if.cpp
index c78ca103de63..daaec8a61484 100644
--- a/clang/test/CIR/CodeGen/if.cpp
+++ b/clang/test/CIR/CodeGen/if.cpp
@@ -14,7 +14,7 @@ int if0(bool a) {
 
 }
 
-// CIR: cir.func @_Z3if0b(%arg0: !cir.bool loc({{.*}})) -> !s32i
+// CIR: cir.func{{.*}} @_Z3if0b(%arg0: !cir.bool loc({{.*}})) -> !s32i
 // CIR: cir.scope {
 // CIR:   %4 = cir.load{{.*}} %0 : !cir.ptr<!cir.bool>, !cir.bool
 // CIR-NEXT: cir.if %4 {
@@ -26,7 +26,7 @@ int if0(bool a) {
 // CIR-NEXT:  }
 
 
-// LLVM: define i32 @_Z3if0b(i1 %0)
+// LLVM: define{{.*}} i32 @_Z3if0b(i1 %0)
 // LLVM:   br label %[[ENTRY:.*]]
 // LLVM: [[ENTRY]]:
 // LLVM:   %6 = load i8, ptr %2, align 1
@@ -43,7 +43,7 @@ int if0(bool a) {
 // LLVM:   %12 = load i32, ptr %3, align 4
 // LLVM:   ret i32 %12
 
-// OGCG: define dso_local noundef i32 @_Z3if0b(i1 noundef zeroext %a)
+// OGCG: define{{.*}} i32 @_Z3if0b(i1 noundef zeroext %a)
 // OGCG: entry:
 // OGCG:   %[[RETVAL:.*]] = alloca i32, align 4
 // OGCG:   %[[A_ADDR:.*]] = alloca i8, align 1
@@ -71,7 +71,7 @@ void if1(int a) {
   }
 }
 
-// CIR: cir.func @_Z3if1i(%arg0: !s32i loc({{.*}}))
+// CIR: cir.func{{.*}} @_Z3if1i(%arg0: !s32i loc({{.*}}))
 // CIR: cir.scope {
 // CIR:   %3 = cir.load{{.*}} %0 : !cir.ptr<!s32i>, !s32i
 // CIR:   %4 = cir.cast(int_to_bool, %3 : !s32i), !cir.bool
@@ -84,7 +84,7 @@ void if1(int a) {
 // CIR-NEXT:   }
 // CIR: }
 
-// LLVM: define void @_Z3if1i(i32 %0)
+// LLVM: define{{.*}} void @_Z3if1i(i32 %0)
 // LLVM: %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM: %[[X:.*]] = alloca i32, i64 1, align 4
 // LLVM: store i32 %0, ptr %[[A]], align 4
@@ -105,7 +105,7 @@ void if1(int a) {
 // LLVM: [[EXIT]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3if1i(i32 noundef %[[A:.*]])
+// OGCG: define{{.*}} void @_Z3if1i(i32 noundef %[[A:.*]])
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[X:.*]] = alloca i32, align 4
@@ -138,7 +138,7 @@ void if2(int a, bool b, bool c) {
   }
 }
 
-// CIR: cir.func @_Z3if2ibb(%arg0: !s32i loc({{.*}}), %arg1: !cir.bool loc({{.*}}), %arg2: !cir.bool loc({{.*}}))
+// CIR: cir.func{{.*}} @_Z3if2ibb(%arg0: !s32i loc({{.*}}), %arg1: !cir.bool loc({{.*}}), %arg2: !cir.bool loc({{.*}}))
 // CIR: cir.scope {
 // CIR:   %5 = cir.load{{.*}} %0 : !cir.ptr<!s32i>, !s32i
 // CIR:   %6 = cir.cast(int_to_bool, %5 : !s32i), !cir.bool
@@ -165,7 +165,7 @@ void if2(int a, bool b, bool c) {
 // CIR:   }
 // CIR: }
 
-// LLVM: define void @_Z3if2ibb(i32 %[[A:.*]], i1 %[[B:.*]], i1 %[[C:.*]])
+// LLVM: define{{.*}} void @_Z3if2ibb(i32 %[[A:.*]], i1 %[[B:.*]], i1 %[[C:.*]])
 // LLVM:   %[[VARA:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[VARB:.*]] = alloca i8, i64 1, align 1
 // LLVM:   %[[VARC:.*]] = alloca i8, i64 1, align 1
@@ -214,7 +214,7 @@ void if2(int a, bool b, bool c) {
 // LLVM: [[LABEL28]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3if2ibb(i32 noundef %[[A:.*]], i1 noundef zeroext %[[B:.*]], i1 noundef zeroext %[[C:.*]])
+// OGCG: define{{.*}} void @_Z3if2ibb(i32 noundef %[[A:.*]], i1 noundef zeroext %[[B:.*]], i1 noundef zeroext %[[C:.*]])
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[B_ADDR:.*]] = alloca i8, align 1
@@ -260,7 +260,7 @@ int if_init() {
   }
 }
 
-// CIR: cir.func @_Z7if_initv() -> !s32i
+// CIR: cir.func{{.*}} @_Z7if_initv() -> !s32i
 // CIR: %[[RETVAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>
 // CIR: cir.scope {
 // CIR:   %[[X:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>,
@@ -285,7 +285,7 @@ int if_init() {
 // CIR:   }
 // CIR: }
 
-// LLVM: define i32 @_Z7if_initv()
+// LLVM: define{{.*}} i32 @_Z7if_initv()
 // LLVM: %[[X:.*]] = alloca i32, i64 1, align 4
 // LLVM: %[[RETVAL:.*]] = alloca i32, i64 1, align 4
 // LLVM: store i32 42, ptr %[[X]], align 4
@@ -305,7 +305,7 @@ int if_init() {
 // LLVM:   %[[RETVAL_LOAD2:.*]] = load i32, ptr %[[RETVAL]], align 4
 // LLVM:   ret i32 %[[RETVAL_LOAD2]]
 
-// OGCG: define dso_local noundef i32 @_Z7if_initv()
+// OGCG: define{{.*}} i32 @_Z7if_initv()
 // OGCG: entry:
 // OGCG:   %[[RETVAL:.*]] = alloca i32, align 4
 // OGCG:   %[[X:.*]] = alloca i32, align 4
diff --git a/clang/test/CIR/CodeGen/inline-cxx-func.cpp b/clang/test/CIR/CodeGen/inline-cxx-func.cpp
index 31d0255f18df..d121daf81617 100644
--- a/clang/test/CIR/CodeGen/inline-cxx-func.cpp
+++ b/clang/test/CIR/CodeGen/inline-cxx-func.cpp
@@ -17,7 +17,7 @@ struct S {
 // LLVM: %struct.S = type { i32 }
 // OGCG: %struct.S = type { i32 }
 
-// CIR: cir.func @_ZN1S10InlineFuncEv(%arg0: !cir.ptr<!rec_S> {{.*}}) -> !s32i
+// CIR: cir.func{{.*}} @_ZN1S10InlineFuncEv(%arg0: !cir.ptr<!rec_S> {{.*}}) -> !s32i
 // CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["this", init]
 // CIR:   %[[RET_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
 // CIR:   cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>
@@ -46,7 +46,7 @@ void use() {
   s.InlineFunc();
 }
 
-// CIR: cir.func @_Z3usev()
+// CIR: cir.func{{.*}} @_Z3usev()
 // CIR:   %[[S_ADDR:.*]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["s"]
 // CIR:   %[[RET_VAL:.*]] = cir.call @_ZN1S10InlineFuncEv(%[[S_ADDR]]) : (!cir.ptr<!rec_S>) -> !s32i
 // CIR:   cir.return
diff --git a/clang/test/CIR/CodeGen/int-to-bool.cpp b/clang/test/CIR/CodeGen/int-to-bool.cpp
index 1dd15edbbf55..ad36af4552c2 100644
--- a/clang/test/CIR/CodeGen/int-to-bool.cpp
+++ b/clang/test/CIR/CodeGen/int-to-bool.cpp
@@ -9,14 +9,14 @@ bool f1(unsigned char c) {
   return c;
 }
 
-// CIR: cir.func @_Z2f1h
+// CIR: cir.func{{.*}} @_Z2f1h
 // CIR:   cir.cast(int_to_bool, %{{.*}} : !u8i), !cir.bool
 
 // Note: The full zext/store/load/trunc sequence is checked here to show what
 // CIR is being lowered to. There's no need to check it for every function since
 // the lowering is the same for all of them.
 
-// LLVM: define i1 @_Z2f1h
+// LLVM: define{{.*}} i1 @_Z2f1h
 // LLVM:   %[[CMP:.*]] = icmp ne i8 %4, 0
 // LLVM:   %[[ZEXT:.*]] = zext i1 %[[CMP]] to i8
 // LLVM:   store i8 %[[ZEXT]], ptr %{{.*}}
@@ -32,10 +32,10 @@ bool f2(short s) {
   return s;
 }
 
-// CIR: cir.func @_Z2f2s
+// CIR: cir.func{{.*}} @_Z2f2s
 // CIR:   cir.cast(int_to_bool, %{{.*}} : !s16i), !cir.bool
 
-// LLVM: define i1 @_Z2f2s
+// LLVM: define{{.*}} i1 @_Z2f2s
 // LLVM:   %[[CMP:.*]] = icmp ne i16 %4, 0
 // LLVM:   %[[ZEXT:.*]] = zext i1 %[[CMP]] to i8
 
@@ -47,10 +47,10 @@ bool f3(unsigned u) {
   return u;
 }
 
-// CIR: cir.func @_Z2f3j
+// CIR: cir.func{{.*}} @_Z2f3j
 // CIR:   cir.cast(int_to_bool, %{{.*}} : !u32i), !cir.bool
 
-// LLVM: define i1 @_Z2f3j
+// LLVM: define{{.*}} i1 @_Z2f3j
 // LLVM:   %[[CMP:.*]] = icmp ne i32 %4, 0
 // LLVM:   %[[ZEXT:.*]] = zext i1 %[[CMP]] to i8
 
@@ -62,10 +62,10 @@ bool f4(long l) {
   return l;
 }
 
-// CIR: cir.func @_Z2f4l
+// CIR: cir.func{{.*}} @_Z2f4l
 // CIR:   cir.cast(int_to_bool, %{{.*}} : !s64i), !cir.bool
 
-// LLVM: define i1 @_Z2f4l
+// LLVM: define{{.*}} i1 @_Z2f4l
 // LLVM:   %[[CMP:.*]] = icmp ne i64 %4, 0
 // LLVM:   %[[ZEXT:.*]] = zext i1 %[[CMP]] to i8
 
diff --git a/clang/test/CIR/CodeGen/linkage-spec.cpp b/clang/test/CIR/CodeGen/linkage-spec.cpp
index 01c4e3fbe181..eb6c7b0a546a 100644
--- a/clang/test/CIR/CodeGen/linkage-spec.cpp
+++ b/clang/test/CIR/CodeGen/linkage-spec.cpp
@@ -1,42 +1,42 @@
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o - 2>&1 | FileCheck %s
 
 extern "C" void TopLevelC(){}
-// CHECK: cir.func @TopLevelC() {
+// CHECK: cir.func{{.*}} @TopLevelC() {
 extern "C++" void TopLevelCpp(){}
-// CHECK: cir.func @_Z11TopLevelCppv() {
+// CHECK: cir.func{{.*}} @_Z11TopLevelCppv() {
 
 extern "C++" {
   void ExternCppEmpty(){}
-  // CHECK: cir.func @_Z14ExternCppEmptyv() {
+  // CHECK: cir.func{{.*}} @_Z14ExternCppEmptyv() {
   extern "C" void ExternCpp_C(){}
-  // CHECK: cir.func @ExternCpp_C() {
+  // CHECK: cir.func{{.*}} @ExternCpp_C() {
   extern "C++" void ExternCpp_Cpp(){}
-  // CHECK: cir.func @_Z13ExternCpp_Cppv() {
+  // CHECK: cir.func{{.*}} @_Z13ExternCpp_Cppv() {
 
   extern "C" {
   void ExternCpp_CEmpty(){}
-  // CHECK: cir.func @ExternCpp_CEmpty() {
+  // CHECK: cir.func{{.*}} @ExternCpp_CEmpty() {
   extern "C" void ExternCpp_C_C(){}
-  // CHECK: cir.func @ExternCpp_C_C() {
+  // CHECK: cir.func{{.*}} @ExternCpp_C_C() {
   extern "C++" void ExternCpp_C_Cpp(){}
-  // CHECK: cir.func @_Z15ExternCpp_C_Cppv() {
+  // CHECK: cir.func{{.*}} @_Z15ExternCpp_C_Cppv() {
   }
 }
 
 extern "C" {
   void ExternCEmpty(){}
-  // CHECK: cir.func @ExternCEmpty() {
+  // CHECK: cir.func{{.*}} @ExternCEmpty() {
   extern "C" void ExternC_C(){}
-  // CHECK: cir.func @ExternC_C() {
+  // CHECK: cir.func{{.*}} @ExternC_C() {
   extern "C++" void ExternC_Cpp(){}
-  // CHECK: cir.func @_Z11ExternC_Cppv() {
+  // CHECK: cir.func{{.*}} @_Z11ExternC_Cppv() {
   extern "C++" {
   void ExternC_CppEmpty(){}
-  // CHECK: cir.func @_Z16ExternC_CppEmptyv() {
+  // CHECK: cir.func{{.*}} @_Z16ExternC_CppEmptyv() {
   extern "C" void ExternC_Cpp_C(){}
-  // CHECK: cir.func @ExternC_Cpp_C() {
+  // CHECK: cir.func{{.*}} @ExternC_Cpp_C() {
   extern "C++" void ExternC_Cpp_Cpp(){}
-  // CHECK: cir.func @_Z15ExternC_Cpp_Cppv() {
+  // CHECK: cir.func{{.*}} @_Z15ExternC_Cpp_Cppv() {
   }
 }
 
diff --git a/clang/test/CIR/CodeGen/local-vars.cpp b/clang/test/CIR/CodeGen/local-vars.cpp
index 42d643364535..9385fdfa6560 100644
--- a/clang/test/CIR/CodeGen/local-vars.cpp
+++ b/clang/test/CIR/CodeGen/local-vars.cpp
@@ -22,7 +22,7 @@ void test() {
 }
 
 // CHECK: module
-// CHECK: cir.func @_Z4testv()
+// CHECK: cir.func{{.*}} @_Z4testv()
 // CHECK:    %[[I_PTR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CHECK:    %[[L_PTR:.*]] = cir.alloca !s64i, !cir.ptr<!s64i>, ["l", init] {alignment = 8 : i64}
 // CHECK:    %[[F_PTR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["f", init] {alignment = 4 : i64}
diff --git a/clang/test/CIR/CodeGen/loop.cpp b/clang/test/CIR/CodeGen/loop.cpp
index ba117c54b743..0eba0bbc97c1 100644
--- a/clang/test/CIR/CodeGen/loop.cpp
+++ b/clang/test/CIR/CodeGen/loop.cpp
@@ -10,7 +10,7 @@ void l0() {
   }
 }
 
-// CIR: cir.func @_Z2l0v
+// CIR: cir.func{{.*}} @_Z2l0v
 // CIR:   cir.scope {
 // CIR:     cir.for : cond {
 // CIR:       %[[TRUE:.*]] = cir.const #true
@@ -24,7 +24,7 @@ void l0() {
 // CIR:   cir.return
 // CIR: }
 
-// LLVM: define void @_Z2l0v()
+// LLVM: define{{.*}} void @_Z2l0v()
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
 // LLVM:   br label %[[LABEL2:.*]]
@@ -50,7 +50,7 @@ void l1() {
   }
 }
 
-// CIR:      cir.func @_Z2l1v
+// CIR:      cir.func{{.*}} @_Z2l1v
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:     %[[I:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CIR-NEXT:     %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
@@ -67,7 +67,7 @@ void l1() {
 // CIR-NEXT:   cir.return
 // CIR-NEXT: }
 
-// LLVM: define void @_Z2l1v()
+// LLVM: define{{.*}} void @_Z2l1v()
 // LLVM:   %[[I:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
@@ -98,7 +98,7 @@ void l2() {
   }
 }
 
-// CIR:      cir.func @_Z2l2v
+// CIR:      cir.func{{.*}} @_Z2l2v
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:     cir.for : cond {
 // CIR-NEXT:       %[[TRUE:.*]] = cir.const #true
@@ -117,7 +117,7 @@ void l2() {
 // CIR-NEXT:   cir.return
 // CIR-NEXT: }
 
-// LLVM: define void @_Z2l2v()
+// LLVM: define{{.*}} void @_Z2l2v()
 // LLVM:   %[[I:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
@@ -148,7 +148,7 @@ void l3() {
     int i = 0;
 }
 
-// CIR:      cir.func @_Z2l3v
+// CIR:      cir.func{{.*}} @_Z2l3v
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:     %[[I:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
 // CIR-NEXT:     cir.for : cond {
@@ -165,7 +165,7 @@ void l3() {
 // CIR-NEXT:   cir.return
 // CIR-NEXT: }
 
-// LLVM: define void @_Z2l3v()
+// LLVM: define{{.*}} void @_Z2l3v()
 // LLVM:   %[[I:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
@@ -196,7 +196,7 @@ void l4() {
     ;
 }
 
-// CIR: cir.func @_Z2l4v
+// CIR: cir.func{{.*}} @_Z2l4v
 // CIR:   %[[A_ADDR:.*]] = cir.alloca {{.*}} ["a"]
 // CIR:   cir.scope {
 // CIR:     %[[RANGE_ADDR:.*]] = cir.alloca {{.*}} ["__range1", init, const]
@@ -231,7 +231,7 @@ void l4() {
 // CIR:     }
 // CIR:   }
 
-// LLVM: define void @_Z2l4v() {
+// LLVM: define{{.*}} void @_Z2l4v() {
 // LLVM:   %[[RANGE_ADDR:.*]] = alloca ptr
 // LLVM:   %[[BEGIN_ADDR:.*]] = alloca ptr
 // LLVM:   %[[END_ADDR:.*]] = alloca ptr
@@ -305,7 +305,7 @@ void l5() {
   for (int arr[]{1,2,3,4}; auto x : arr) {} 
 }
 
-// CIR: cir.func @_Z2l5v
+// CIR: cir.func{{.*}} @_Z2l5v
 // CIR:   cir.scope {
 // CIR:     %[[ARR_ADDR:.*]] = cir.alloca {{.*}} ["arr", init]
 // CIR:     %[[RANGE_ADDR:.*]] = cir.alloca {{.*}} ["__range1", init, const]
@@ -355,7 +355,7 @@ void l5() {
 // CIR:     }
 // CIR:   }
 
-// LLVM: define void @_Z2l5v() {
+// LLVM: define{{.*}} void @_Z2l5v() {
 // LLVM:   %[[ARR_ADDR:.*]] = alloca [4 x i32]
 // LLVM:   %[[RANGE_ADDR:.*]] = alloca ptr
 // LLVM:   %[[BEGIN_ADDR:.*]] = alloca ptr
@@ -439,7 +439,7 @@ void test_do_while_false() {
   } while (0);
 }
 
-// CIR: cir.func @_Z19test_do_while_falsev()
+// CIR: cir.func{{.*}} @_Z19test_do_while_falsev()
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:     cir.do {
 // CIR-NEXT:       cir.yield
@@ -448,7 +448,7 @@ void test_do_while_false() {
 // CIR-NEXT:       %[[FALSE:.*]] = cir.cast(int_to_bool, %[[ZERO]] : !s32i), !cir.bool
 // CIR-NEXT:       cir.condition(%[[FALSE]])
 
-// LLVM: define void @_Z19test_do_while_falsev()
+// LLVM: define{{.*}} void @_Z19test_do_while_falsev()
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
 // LLVM:   br label %[[LABEL3:.*]]
@@ -475,7 +475,7 @@ void test_empty_while_true() {
   }
 }
 
-// CIR: cir.func @_Z21test_empty_while_truev()
+// CIR: cir.func{{.*}} @_Z21test_empty_while_truev()
 // CIR-NEXT:   cir.scope {
 // CIR-NEXT:     cir.while {
 // CIR-NEXT:       %[[TRUE:.*]] = cir.const #true
@@ -486,7 +486,7 @@ void test_empty_while_true() {
 // CIR-NEXT:       }
 // CIR-NEXT:       cir.yield
 
-// LLVM: define void @_Z21test_empty_while_truev()
+// LLVM: define{{.*}} void @_Z21test_empty_while_truev()
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
 // LLVM:   br label %[[LABEL2:.*]]
@@ -517,7 +517,7 @@ void unreachable_after_continue() {
   }
 }
 
-// CIR: cir.func @_Z26unreachable_after_continuev()
+// CIR: cir.func{{.*}} @_Z26unreachable_after_continuev()
 // CIR:   cir.scope {
 // CIR:     cir.for : cond {
 // CIR:       %[[TRUE:.*]] = cir.const #true
@@ -539,7 +539,7 @@ void unreachable_after_continue() {
 // CIR:   cir.return
 // CIR: }
 
-// LLVM: define void @_Z26unreachable_after_continuev()
+// LLVM: define{{.*}} void @_Z26unreachable_after_continuev()
 // LLVM:   %[[X:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
@@ -577,7 +577,7 @@ void unreachable_after_break() {
   }
 }
 
-// CIR: cir.func @_Z23unreachable_after_breakv()
+// CIR: cir.func{{.*}} @_Z23unreachable_after_breakv()
 // CIR:   cir.scope {
 // CIR:     cir.for : cond {
 // CIR:       %[[TRUE:.*]] = cir.const #true
@@ -599,7 +599,7 @@ void unreachable_after_break() {
 // CIR:   cir.return
 // CIR: }
 
-// LLVM: define void @_Z23unreachable_after_breakv()
+// LLVM: define{{.*}} void @_Z23unreachable_after_breakv()
 // LLVM:   %[[X:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[LABEL1:.*]]
 // LLVM: [[LABEL1]]:
diff --git a/clang/test/CIR/CodeGen/member-functions.cpp b/clang/test/CIR/CodeGen/member-functions.cpp
index c1d49ac4d8f3..8be2c7fc2edb 100644
--- a/clang/test/CIR/CodeGen/member-functions.cpp
+++ b/clang/test/CIR/CodeGen/member-functions.cpp
@@ -10,7 +10,7 @@ struct C {
 
 void C::f() {}
 
-// CIR: cir.func @_ZN1C1fEv(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C>
+// CIR: cir.func{{.*}} @_ZN1C1fEv(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C>
 // CIR:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init]
 // CIR:   cir.store %[[THIS_ARG]], %[[THIS_ADDR]] : !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>
 // CIR:   %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_C>>, !cir.ptr<!rec_C>
@@ -19,7 +19,7 @@ void C::f() {}
 
 void C::f2(int a, int b) {}
 
-// CIR:      cir.func @_ZN1C2f2Eii(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C> {{.*}}, %[[A_ARG:.*]]: !s32i {{.*}}, %[[B_ARG:.*]]: !s32i {{.*}}) {
+// CIR:      cir.func{{.*}} @_ZN1C2f2Eii(%[[THIS_ARG:.*]]: !cir.ptr<!rec_C> {{.*}}, %[[A_ARG:.*]]: !s32i {{.*}}, %[[B_ARG:.*]]: !s32i {{.*}}) {
 // CIR-NEXT:   %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_C>, !cir.ptr<!cir.ptr<!rec_C>>, ["this", init]
 // CIR-NEXT:   %[[A_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CIR-NEXT:   %[[B_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
@@ -36,7 +36,7 @@ void test1() {
   c.f2(1, 2);
 }
 
-// CIR: cir.func @_Z5test1v() {
+// CIR: cir.func{{.*}} @_Z5test1v() {
 // CIR-NEXT:   %[[C_ADDR:.*]] = cir.alloca !rec_C, !cir.ptr<!rec_C>, ["c"]
 // CIR-NEXT:   cir.call @_ZN1C1fEv(%[[C_ADDR]]) : (!cir.ptr<!rec_C>) -> ()
 // CIR-NEXT:   %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
diff --git a/clang/test/CIR/CodeGen/namespace.cpp b/clang/test/CIR/CodeGen/namespace.cpp
index 14490d505140..efae1f2f2f23 100644
--- a/clang/test/CIR/CodeGen/namespace.cpp
+++ b/clang/test/CIR/CodeGen/namespace.cpp
@@ -20,12 +20,12 @@ namespace test {
   }
 }
 
-// CHECK-DAG: cir.global "private" internal dsolocal @_ZN12_GLOBAL__N_12g1E = #cir.int<1> : !s32i
+// CHECK-DAG: cir.global "private" internal dso_local @_ZN12_GLOBAL__N_12g1E = #cir.int<1> : !s32i
 // CHECK-DAG: cir.global external @_ZN4test2g2E = #cir.int<2> : !s32i
 // CHECK-DAG: cir.global external @_ZN4test5test22g3E = #cir.int<3> : !s32i
-// CHECK-DAG: cir.func @_ZN12_GLOBAL__N_12f1Ev()
-// CHECK-DAG: cir.func @_ZN4test2f2Ev()
-// CHECK-DAG: cir.func @_ZN4test5test22f3Ev()
+// CHECK-DAG: cir.func{{.*}} @_ZN12_GLOBAL__N_12f1Ev()
+// CHECK-DAG: cir.func{{.*}} @_ZN4test2f2Ev()
+// CHECK-DAG: cir.func{{.*}} @_ZN4test5test22f3Ev()
 
 using namespace test;
 
@@ -38,7 +38,7 @@ int f4(void) {
 }
 
 // The namespace gets added during name mangling, so this is wrong but expected.
-// CHECK: cir.func @_Z2f4v()
+// CHECK: cir.func{{.*}} @_Z2f4v()
 // CHECK:   cir.call @_ZN12_GLOBAL__N_12f1Ev()
 // CHECK:   cir.call @_ZN4test2f2Ev()
 // CHECK:   cir.call @_ZN4test5test22f3Ev()
@@ -59,7 +59,7 @@ int f5() {
   return g3;
 }
 
-// CHECK: cir.func @_Z2f5v()
+// CHECK: cir.func{{.*}} @_Z2f5v()
 // CHECK:   cir.call @_ZN4test5test22f3Ev()
 // CHECK:   %[[G3_ADDR:.*]] = cir.get_global @_ZN4test5test22g3E : !cir.ptr<!s32i>
 // CHECK:   %[[G3_VAL:.*]] = cir.load{{.*}} %[[G3_ADDR]] : !cir.ptr<!s32i>, !s32i
@@ -76,7 +76,7 @@ int f6() {
   return s.a;
 }
 
-// CHECK: cir.func @_Z2f6v()
+// CHECK: cir.func{{.*}} @_Z2f6v()
 // CHECK:   cir.get_global @_ZN5test31sE : !cir.ptr<!rec_test33A3AS>
 // CHECK:   cir.get_member %{{.*}}[0] {name = "a"}
 
@@ -92,4 +92,4 @@ void f7() {
   shadow::shadowedFunc();
 }
 
-// CHECK: cir.func @_Z2f7v()
+// CHECK: cir.func{{.*}} @_Z2f7v()
diff --git a/clang/test/CIR/CodeGen/new.cpp b/clang/test/CIR/CodeGen/new.cpp
new file mode 100644
index 000000000000..4f88addc6116
--- /dev/null
+++ b/clang/test/CIR/CodeGen/new.cpp
@@ -0,0 +1,158 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+struct S {
+  int a;
+  int b;
+};
+
+void test_basic_new() {
+  S *ps = new S;
+  int *pn = new int;
+  double *pd = new double;
+}
+
+// CHECK: cir.func{{.*}} @_Z14test_basic_newv
+// CHECK:   %[[PS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["ps", init]
+// CHECK:   %[[PN_ADDR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["pn", init]
+// CHECK:   %[[PD_ADDR:.*]] = cir.alloca !cir.ptr<!cir.double>, !cir.ptr<!cir.ptr<!cir.double>>, ["pd", init]
+// CHECK:   %[[EIGHT:.*]] = cir.const #cir.int<8>
+// CHECK:   %[[NEW_S:.*]] = cir.call @_Znwm(%[[EIGHT]])
+// CHECK:   %[[NEW_S_PTR:.*]] = cir.cast(bitcast, %[[NEW_S]]
+// CHECK:   cir.call @_ZN1SC1Ev(%[[NEW_S_PTR]])
+// CHECK:   cir.store{{.*}} %[[NEW_S_PTR]], %[[PS_ADDR]]
+// CHECK:   %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK:   %[[NEW_INT:.*]] = cir.call @_Znwm(%[[FOUR]])
+// CHECK:   %[[NEW_INT_PTR:.*]] = cir.cast(bitcast, %[[NEW_INT]]
+// CHECK:   cir.store{{.*}} %[[NEW_INT_PTR]], %[[PN_ADDR]]
+// CHECK:   %[[EIGHT:.*]] = cir.const #cir.int<8>
+// CHECK:   %[[NEW_DOUBLE:.*]] = cir.call @_Znwm(%[[EIGHT]])
+// CHECK:   %[[NEW_DOUBLE_PTR:.*]] = cir.cast(bitcast, %[[NEW_DOUBLE]]
+// CHECK:   cir.store{{.*}} %[[NEW_DOUBLE_PTR]], %[[PD_ADDR]]
+// CHECK:   cir.return
+
+// LLVM: define{{.*}} void @_Z14test_basic_newv
+// LLVM:   %[[PS_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[PN_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[PD_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[NEW_S:.*]] = call{{.*}} ptr @_Znwm(i64 8)
+// LLVM:   call{{.*}} void @_ZN1SC1Ev(ptr %[[NEW_S]])
+// LLVM:   store ptr %[[NEW_S]], ptr %[[PS_ADDR]], align 8
+// LLVM:   %[[NEW_INT:.*]] = call{{.*}} ptr @_Znwm(i64 4)
+// LLVM:   store ptr %[[NEW_INT]], ptr %[[PN_ADDR]], align 8
+// LLVM:   %[[NEW_DOUBLE:.*]] = call{{.*}} ptr @_Znwm(i64 8)
+// LLVM:   store ptr %[[NEW_DOUBLE]], ptr %[[PD_ADDR]], align 8
+// LLVM:   ret void
+
+// NOTE: OGCG elides the constructor call here, but CIR does not.
+
+// OGCG: define{{.*}} void @_Z14test_basic_newv
+// OGCG:   %[[PS_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[PN_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[PD_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[NEW_S:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 8)
+// OGCG:   store ptr %[[NEW_S]], ptr %[[PS_ADDR]], align 8
+// OGCG:   %[[NEW_INT:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 4)
+// OGCG:   store ptr %[[NEW_INT]], ptr %[[PN_ADDR]], align 8
+// OGCG:   %[[NEW_DOUBLE:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 8)
+// OGCG:   store ptr %[[NEW_DOUBLE]], ptr %[[PD_ADDR]], align 8
+// OGCG:   ret void
+
+void test_new_with_init() {
+  int *pn = new int{2};
+  double *pd = new double{3.0};
+}
+
+// CHECK: cir.func{{.*}} @_Z18test_new_with_initv
+// CHECK:   %[[PN_ADDR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["pn", init]
+// CHECK:   %[[PD_ADDR:.*]] = cir.alloca !cir.ptr<!cir.double>, !cir.ptr<!cir.ptr<!cir.double>>, ["pd", init]
+// CHECK:   %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK:   %[[NEW_INT:.*]] = cir.call @_Znwm(%[[FOUR]])
+// CHECK:   %[[NEW_INT_PTR:.*]] = cir.cast(bitcast, %[[NEW_INT]]
+// CHECK:   %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK:   cir.store{{.*}} %[[TWO]], %[[NEW_INT_PTR]]
+// CHECK:   cir.store{{.*}} %[[NEW_INT_PTR]], %[[PN_ADDR]]
+// CHECK:   %[[EIGHT:.*]] = cir.const #cir.int<8>
+// CHECK:   %[[NEW_DOUBLE:.*]] = cir.call @_Znwm(%[[EIGHT]])
+// CHECK:   %[[NEW_DOUBLE_PTR:.*]] = cir.cast(bitcast, %[[NEW_DOUBLE]]
+// CHECK:   %[[THREE:.*]] = cir.const #cir.fp<3.000000e+00>
+// CHECK:   cir.store{{.*}} %[[THREE]], %[[NEW_DOUBLE_PTR]]
+// CHECK:   cir.store{{.*}} %[[NEW_DOUBLE_PTR]], %[[PD_ADDR]]
+// CHECK:   cir.return
+
+// LLVM: define{{.*}} void @_Z18test_new_with_initv
+// LLVM:   %[[PN_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[PD_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[NEW_INT:.*]] = call{{.*}} ptr @_Znwm(i64 4)
+// LLVM:   store i32 2, ptr %[[NEW_INT]], align 4
+// LLVM:   store ptr %[[NEW_INT]], ptr %[[PN_ADDR]], align 8
+// LLVM:   %[[NEW_DOUBLE:.*]] = call{{.*}} ptr @_Znwm(i64 8)
+// LLVM:   store double 3.000000e+00, ptr %[[NEW_DOUBLE]], align 8
+// LLVM:   store ptr %[[NEW_DOUBLE]], ptr %[[PD_ADDR]], align 8
+// LLVM:   ret void
+
+// OGCG: define{{.*}} void @_Z18test_new_with_initv
+// OGCG:   %[[PN_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[PD_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[NEW_INT:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 4)
+// OGCG:   store i32 2, ptr %[[NEW_INT]], align 4
+// OGCG:   store ptr %[[NEW_INT]], ptr %[[PN_ADDR]], align 8
+// OGCG:   %[[NEW_DOUBLE:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 8)
+// OGCG:   store double 3.000000e+00, ptr %[[NEW_DOUBLE]], align 8
+// OGCG:   store ptr %[[NEW_DOUBLE]], ptr %[[PD_ADDR]], align 8
+// OGCG:   ret void
+
+struct S2 {
+  S2();
+  S2(int, int);
+  int a;
+  int b;
+};
+
+void test_new_with_ctor() {
+  S2 *ps2 = new S2;
+  S2 *ps2_2 = new S2(1, 2);
+}
+
+// CHECK: cir.func{{.*}} @_Z18test_new_with_ctorv
+// CHECK:   %[[PS2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_S2>, !cir.ptr<!cir.ptr<!rec_S2>>, ["ps2", init]
+// CHECK:   %[[PS2_2_ADDR:.*]] = cir.alloca !cir.ptr<!rec_S2>, !cir.ptr<!cir.ptr<!rec_S2>>, ["ps2_2", init]
+// CHECK:   %[[EIGHT:.*]] = cir.const #cir.int<8>
+// CHECK:   %[[NEW_S2:.*]] = cir.call @_Znwm(%[[EIGHT]])
+// CHECK:   %[[NEW_S2_PTR:.*]] = cir.cast(bitcast, %[[NEW_S2]]
+// CHECK:   cir.call @_ZN2S2C1Ev(%[[NEW_S2_PTR]])
+// CHECK:   cir.store{{.*}} %[[NEW_S2_PTR]], %[[PS2_ADDR]]
+// CHECK:   %[[EIGHT:.*]] = cir.const #cir.int<8>
+// CHECK:   %[[NEW_S2_2:.*]] = cir.call @_Znwm(%[[EIGHT]])
+// CHECK:   %[[NEW_S2_2_PTR:.*]] = cir.cast(bitcast, %[[NEW_S2_2]]
+// CHECK:   %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK:   %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK:   cir.call @_ZN2S2C1Eii(%[[NEW_S2_2_PTR]], %[[ONE]], %[[TWO]])
+// CHECK:   cir.store{{.*}} %[[NEW_S2_2_PTR]], %[[PS2_2_ADDR]]
+// CHECK:   cir.return
+
+// LLVM: define{{.*}} void @_Z18test_new_with_ctorv
+// LLVM:   %[[PS2_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[PS2_2_ADDR:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[NEW_S2:.*]] = call{{.*}} ptr @_Znwm(i64 8)
+// LLVM:   call{{.*}} void @_ZN2S2C1Ev(ptr %[[NEW_S2]])
+// LLVM:   store ptr %[[NEW_S2]], ptr %[[PS2_ADDR]], align 8
+// LLVM:   %[[NEW_S2_2:.*]] = call{{.*}} ptr @_Znwm(i64 8)
+// LLVM:   call{{.*}} void @_ZN2S2C1Eii(ptr %[[NEW_S2_2]], i32 1, i32 2)
+// LLVM:   store ptr %[[NEW_S2_2]], ptr %[[PS2_2_ADDR]], align 8
+// LLVM:   ret void
+
+// OGCG: define{{.*}} void @_Z18test_new_with_ctorv
+// OGCG:   %[[PS2_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[PS2_2_ADDR:.*]] = alloca ptr, align 8
+// OGCG:   %[[NEW_S2:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 8)
+// OGCG:   call{{.*}} void @_ZN2S2C1Ev(ptr {{.*}} %[[NEW_S2]])
+// OGCG:   store ptr %[[NEW_S2]], ptr %[[PS2_ADDR]], align 8
+// OGCG:   %[[NEW_S2_2:.*]] = call{{.*}} ptr @_Znwm(i64 noundef 8)
+// OGCG:   call{{.*}} void @_ZN2S2C1Eii(ptr {{.*}} %[[NEW_S2_2]], i32 noundef 1, i32 noundef 2)
+// OGCG:   store ptr %[[NEW_S2_2]], ptr %[[PS2_2_ADDR]], align 8
+// OGCG:   ret void
diff --git a/clang/test/CIR/CodeGen/non-type-template-param.cpp b/clang/test/CIR/CodeGen/non-type-template-param.cpp
new file mode 100644
index 000000000000..8ef1d5785c70
--- /dev/null
+++ b/clang/test/CIR/CodeGen/non-type-template-param.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+template <const int N>
+void template_foo() {
+  int a = N + 5;
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
+// CIR: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
+// CIR: %[[CONST_2:.*]] = cir.const #cir.int<5> : !s32i
+// CIR: %[[ADD:.*]] = cir.binop(add, %[[CONST_1]], %[[CONST_2]]) nsw : !s32i
+// CIR: cir.store{{.*}} %[[ADD]], %[[INIT]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[INIT:.*]] = alloca i32, i64 1, align 4
+// LLVM: store i32 6, ptr %[[INIT]], align 4
+
+// OGCG: %[[INIT:.*]] = alloca i32, align 4
+// OGCG: store i32 6, ptr %[[INIT]], align 4
+
+void foo() {
+  template_foo<1>();
+}
diff --git a/clang/test/CIR/CodeGen/nullptr-init.cpp b/clang/test/CIR/CodeGen/nullptr-init.cpp
index 76965ce78469..091269d09c98 100644
--- a/clang/test/CIR/CodeGen/nullptr-init.cpp
+++ b/clang/test/CIR/CodeGen/nullptr-init.cpp
@@ -11,7 +11,7 @@ void t1() {
   int *p3 = (int*)0;
 }
 
-// CIR:      cir.func @_Z2t1v()
+// CIR:      cir.func{{.*}} @_Z2t1v()
 // CIR-NEXT:     %[[P1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p1", init] {alignment = 8 : i64}
 // CIR-NEXT:     %[[P2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p2", init] {alignment = 8 : i64}
 // CIR-NEXT:     %[[P3:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p3", init] {alignment = 8 : i64}
diff --git a/clang/test/CIR/CodeGen/opt-info-attr.cpp b/clang/test/CIR/CodeGen/opt-info-attr.cpp
new file mode 100644
index 000000000000..444286b8db8a
--- /dev/null
+++ b/clang/test/CIR/CodeGen/opt-info-attr.cpp
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O0
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O1 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O1
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O2 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O2
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -O3 -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-O3
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Os -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-Os
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Oz -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CHECK-Oz
+
+void f() {}
+
+// CHECK-O0: module attributes
+// CHECK-O0-NOT: cir.opt_info
+// CHECK-O1: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 1, size = 0>{{.+}}
+// CHECK-O2: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 0>{{.+}}
+// CHECK-O3: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 3, size = 0>{{.+}}
+// CHECK-Os: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 1>{{.+}}
+// CHECK-Oz: module attributes {{.+}}cir.opt_info = #cir.opt_info<level = 2, size = 2>{{.+}}
diff --git a/clang/test/CIR/CodeGen/pack-indexing.cpp b/clang/test/CIR/CodeGen/pack-indexing.cpp
new file mode 100644
index 000000000000..8b12b505164f
--- /dev/null
+++ b/clang/test/CIR/CodeGen/pack-indexing.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -fclangir -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++2c -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+int pack_indexing(auto... p) { return p...[0]; }
+
+// CIR: %[[P_0:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[P_1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[P_2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["p", init]
+// CIR: %[[RET_VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[RESULT:.*]] = cir.load{{.*}} %[[P_0]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.store %[[RESULT]], %[[RET_VAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET_VAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[P_0:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[P_1:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[P_2:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RET_VAL:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = load i32, ptr %[[P_0]], align 4
+// LLVM: store i32 %[[RESULT]], ptr %[[RET_VAL]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET_VAL]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG-DAG: %[[P_0:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[P_1:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[P_2:.*]] = alloca i32, align 4
+// OGCG-DAG: %[[RESULT:.*]] = load i32, ptr %[[P_0]], align 4
+// OGCG-DAG-NEXT: ret i32 %[[RESULT]]
+
+int foo() { return pack_indexing(1, 2, 3); }
+
+// CIR: %[[RET_VAL:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR: %[[RESULT:.*]] = cir.call @_Z13pack_indexingIJiiiEEiDpT_({{.*}}, {{.*}}, {{.*}}) : (!s32i, !s32i, !s32i) -> !s32i
+// CIR: cir.store %[[RESULT]], %[[RET_VAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[TMP:.*]] = cir.load %[[RET_VAL]] : !cir.ptr<!s32i>, !s32i
+// CIR: cir.return %[[TMP]] : !s32i
+
+// LLVM: %[[RET_VAL:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[RESULT:.*]] = call i32 @_Z13pack_indexingIJiiiEEiDpT_(i32 1, i32 2, i32 3)
+// LLVM: store i32 %[[RESULT]], ptr %[[RET_VAL]], align 4
+// LLVM: %[[TMP:.*]] = load i32, ptr %[[RET_VAL]], align 4
+// LLVM: ret i32 %[[TMP]]
+
+// OGCG-DAG: %[[CALL:.*]] = call noundef i32 @_Z13pack_indexingIJiiiEEiDpT_(i32 noundef 1, i32 noundef 2, i32 noundef 3)
+// OGCG-DAG-NEXT: ret i32 %[[RESULT]]
diff --git a/clang/test/CIR/CodeGen/static-vars.c b/clang/test/CIR/CodeGen/static-vars.c
index f45a41d9a00f..ee4db82c40ad 100644
--- a/clang/test/CIR/CodeGen/static-vars.c
+++ b/clang/test/CIR/CodeGen/static-vars.c
@@ -4,20 +4,20 @@
 void func1(void) {
   // Should lower default-initialized static vars.
   static int i;
-  // CHECK-DAG: cir.global "private" internal dsolocal @func1.i = #cir.int<0> : !s32i
+  // CHECK-DAG: cir.global "private" internal dso_local @func1.i = #cir.int<0> : !s32i
 
   // Should lower constant-initialized static vars.
   static int j = 1;
-  // CHECK-DAG: cir.global "private" internal dsolocal @func1.j = #cir.int<1> : !s32i
+  // CHECK-DAG: cir.global "private" internal dso_local @func1.j = #cir.int<1> : !s32i
 
   // Should properly shadow static vars in nested scopes.
   {
     static int j = 2;
-    // CHECK-DAG: cir.global "private" internal dsolocal @func1.j.1 = #cir.int<2> : !s32i
+    // CHECK-DAG: cir.global "private" internal dso_local @func1.j.1 = #cir.int<2> : !s32i
   }
   {
     static int j = 3;
-    // CHECK-DAG: cir.global "private" internal dsolocal @func1.j.2 = #cir.int<3> : !s32i
+    // CHECK-DAG: cir.global "private" internal dso_local @func1.j.2 = #cir.int<3> : !s32i
   }
 
   // Should lower basic static vars arithmetics.
@@ -31,7 +31,7 @@ void func1(void) {
 // Should shadow static vars on different functions.
 void func2(void) {
   static char i;
-  // CHECK-DAG: cir.global "private" internal dsolocal @func2.i = #cir.int<0> : !s8i
+  // CHECK-DAG: cir.global "private" internal dso_local @func2.i = #cir.int<0> : !s8i
   static float j;
-  // CHECK-DAG: cir.global "private" internal dsolocal @func2.j = #cir.fp<0.000000e+00> : !cir.float
+  // CHECK-DAG: cir.global "private" internal dso_local @func2.j = #cir.fp<0.000000e+00> : !cir.float
 }
diff --git a/clang/test/CIR/CodeGen/static-vars.cpp b/clang/test/CIR/CodeGen/static-vars.cpp
index 9b892c69a6fe..d949936f6bff 100644
--- a/clang/test/CIR/CodeGen/static-vars.cpp
+++ b/clang/test/CIR/CodeGen/static-vars.cpp
@@ -6,20 +6,20 @@
 void func1(void) {
   // Should lower default-initialized static vars.
   static int i;
-  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1i = #cir.int<0> : !s32i
+  // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func1vE1i = #cir.int<0> : !s32i
 
   // Should lower constant-initialized static vars.
   static int j = 1;
-  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j = #cir.int<1> : !s32i
+  // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func1vE1j = #cir.int<1> : !s32i
 
   // Should properly shadow static vars in nested scopes.
   {
     static int j = 2;
-    // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j_0 = #cir.int<2> : !s32i
+    // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func1vE1j_0 = #cir.int<2> : !s32i
   }
   {
     static int j = 3;
-    // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func1vE1j_1 = #cir.int<3> : !s32i
+    // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func1vE1j_1 = #cir.int<3> : !s32i
   }
 
   // Should lower basic static vars arithmetics.
@@ -33,9 +33,9 @@ void func1(void) {
 // Should shadow static vars on different functions.
 void func2(void) {
   static char i;
-  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func2vE1i = #cir.int<0> : !s8i
+  // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func2vE1i = #cir.int<0> : !s8i
   static float j;
-  // CHECK-DAG: cir.global "private" internal dsolocal @_ZZ5func2vE1j = #cir.fp<0.000000e+00> : !cir.float
+  // CHECK-DAG: cir.global "private" internal dso_local @_ZZ5func2vE1j = #cir.fp<0.000000e+00> : !cir.float
 }
 
 // CHECK-DAG: cir.global linkonce_odr comdat @_ZZ4testvE1c = #cir.int<0> : !s32i
diff --git a/clang/test/CIR/CodeGen/string-literals.c b/clang/test/CIR/CodeGen/string-literals.c
index 90ea21906f36..44fd191173c3 100644
--- a/clang/test/CIR/CodeGen/string-literals.c
+++ b/clang/test/CIR/CodeGen/string-literals.c
@@ -17,9 +17,9 @@ char g_exact[4] = "123";
 
 // CIR: cir.global external @g_exact = #cir.const_array<"123\00" : !cir.array<!s8i x 4>> : !cir.array<!s8i x 4>
 
-// CIR: cir.global "private" cir_private dsolocal @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
-// CIR: cir.global "private" cir_private dsolocal @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
-// CIR: cir.global "private" cir_private dsolocal @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
+// CIR: cir.global "private" cir_private dso_local @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
+// CIR: cir.global "private" cir_private dso_local @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
+// CIR: cir.global "private" cir_private dso_local @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
 
 // LLVM: @[[STR1_GLOBAL:.*]] = private global [2 x i8] c"1\00"
 // LLVM: @[[STR2_GLOBAL:.*]] = private global [1 x i8] zeroinitializer
@@ -33,10 +33,10 @@ char *f1() {
   return "1";
 }
 
-// CIR: cir.func @f1()
+// CIR: cir.func{{.*}} @f1()
 // CIR:   %[[STR:.*]] = cir.get_global @[[STR1_GLOBAL]] : !cir.ptr<!cir.array<!s8i x 2>>
 
-// LLVM: define ptr @f1()
+// LLVM: define{{.*}} ptr @f1()
 // LLVM:   store ptr @[[STR1_GLOBAL]], ptr {{.*}}
 
 // OGCG: define {{.*}}ptr @f1()
@@ -46,24 +46,24 @@ char *f2() {
   return "";
 }
 
-// CIR: cir.func @f2()
+// CIR: cir.func{{.*}} @f2()
 // CIR:   %[[STR2:.*]] = cir.get_global @[[STR2_GLOBAL]] : !cir.ptr<!cir.array<!s8i x 1>>
 
-// LLVM: define ptr @f2()
+// LLVM: define{{.*}} ptr @f2()
 // LLVM:   store ptr @[[STR2_GLOBAL]], ptr {{.*}}
 
-// OGCG: define {{.*}}ptr @f2()
+// OGCG: define{{.*}} ptr @f2()
 // OGCG:   ret ptr @[[STR2_GLOBAL]]
 
 char *f3() {
   return "\00";
 }
 
-// CIR: cir.func @f3()
+// CIR: cir.func{{.*}} @f3()
 // CIR:   %[[STR3:.*]] = cir.get_global @[[STR3_GLOBAL]] : !cir.ptr<!cir.array<!s8i x 2>>
 
-// LLVM: define ptr @f3()
+// LLVM: define{{.*}} ptr @f3()
 // LLVM:   store ptr @[[STR3_GLOBAL]], ptr {{.*}}
 
-// OGCG: define {{.*}}ptr @f3()
+// OGCG: define{{.*}} ptr @f3()
 // OGCG:   ret ptr @[[STR3_GLOBAL]]
diff --git a/clang/test/CIR/CodeGen/string-literals.cpp b/clang/test/CIR/CodeGen/string-literals.cpp
index c56eb7438732..081c4c2c5a4a 100644
--- a/clang/test/CIR/CodeGen/string-literals.cpp
+++ b/clang/test/CIR/CodeGen/string-literals.cpp
@@ -5,7 +5,7 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll
 // RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
 
-// CIR: cir.global "private" cir_private dsolocal @[[STR1_GLOBAL:.*]] = #cir.const_array<"abcd\00" : !cir.array<!s8i x 5>> : !cir.array<!s8i x 5>
+// CIR: cir.global "private" cir_private dso_local @[[STR1_GLOBAL:.*]] = #cir.const_array<"abcd\00" : !cir.array<!s8i x 5>> : !cir.array<!s8i x 5>
 
 // LLVM: @[[STR1_GLOBAL:.*]] = private global [5 x i8] c"abcd\00"
 
diff --git a/clang/test/CIR/CodeGen/struct.c b/clang/test/CIR/CodeGen/struct.c
index b722b64eeb58..aa7c4cf1c295 100644
--- a/clang/test/CIR/CodeGen/struct.c
+++ b/clang/test/CIR/CodeGen/struct.c
@@ -166,11 +166,11 @@ void f(void) {
   struct IncompleteS *p;
 }
 
-// CIR:      cir.func @f()
+// CIR:      cir.func{{.*}} @f()
 // CIR-NEXT:   cir.alloca !cir.ptr<!rec_IncompleteS>, !cir.ptr<!cir.ptr<!rec_IncompleteS>>, ["p"] {alignment = 8 : i64}
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @f()
+// LLVM:      define{{.*}} void @f()
 // LLVM-NEXT:   %[[P:.*]] = alloca ptr, i64 1, align 8
 // LLVM-NEXT:   ret void
 
@@ -183,11 +183,11 @@ void f2(void) {
   struct CompleteS s;
 }
 
-// CIR:      cir.func @f2()
+// CIR:      cir.func{{.*}} @f2()
 // CIR-NEXT:   cir.alloca !rec_CompleteS, !cir.ptr<!rec_CompleteS>, ["s"] {alignment = 4 : i64}
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @f2()
+// LLVM:      define{{.*}} void @f2()
 // LLVM-NEXT:   %[[S:.*]] = alloca %struct.CompleteS, i64 1, align 4
 // LLVM-NEXT:   ret void
 
@@ -201,7 +201,7 @@ char f3(int a) {
   return cs.b;
 }
 
-// CIR:      cir.func @f3(%[[ARG_A:.*]]: !s32i
+// CIR:      cir.func{{.*}} @f3(%[[ARG_A:.*]]: !s32i
 // CIR-NEXT:   %[[A_ADDR:.*]] = cir.alloca {{.*}} ["a", init] {alignment = 4 : i64}
 // CIR-NEXT:   %[[RETVAL_ADDR:.*]] = cir.alloca {{.*}} ["__retval"] {alignment = 1 : i64}
 // CIR-NEXT:   cir.store{{.*}} %[[ARG_A]], %[[A_ADDR]]
@@ -216,7 +216,7 @@ char f3(int a) {
 // CIR-NEXT:   %[[RETVAL:.*]] = cir.load{{.*}} %[[RETVAL_ADDR]]
 // CIR-NEXT:   cir.return %[[RETVAL]]
 
-// LLVM:      define i8 @f3(i32 %[[ARG_A:.*]])
+// LLVM:      define{{.*}} i8 @f3(i32 %[[ARG_A:.*]])
 // LLVM-NEXT:   %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[RETVAL_ADDR:.*]] = alloca i8, i64 1, align 1
 // LLVM-NEXT:   store i32 %[[ARG_A]], ptr %[[A_ADDR]], align 4
@@ -241,7 +241,7 @@ char f4(int a, struct CompleteS *p) {
   return p->b;
 }
 
-// CIR:      cir.func @f4(%[[ARG_A:.*]]: !s32i {{.*}}, %[[ARG_P:.*]]: !cir.ptr<!rec_CompleteS>
+// CIR:      cir.func{{.*}} @f4(%[[ARG_A:.*]]: !s32i {{.*}}, %[[ARG_P:.*]]: !cir.ptr<!rec_CompleteS>
 // CIR-NEXT:   %[[A_ADDR:.*]] = cir.alloca {{.*}} ["a", init] {alignment = 4 : i64}
 // CIR-NEXT:   %[[P_ADDR:.*]] = cir.alloca {{.*}} ["p", init] {alignment = 8 : i64}
 // CIR-NEXT:   %[[RETVAL_ADDR:.*]] = cir.alloca {{.*}} ["__retval"] {alignment = 1 : i64}
@@ -258,7 +258,7 @@ char f4(int a, struct CompleteS *p) {
 // CIR-NEXT:   %[[RETVAL:.*]] = cir.load{{.*}} %[[RETVAL_ADDR]]
 // CIR-NEXT:   cir.return %[[RETVAL]]
 
-// LLVM:      define i8 @f4(i32 %[[ARG_A:.*]], ptr %[[ARG_P:.*]])
+// LLVM:      define{{.*}} i8 @f4(i32 %[[ARG_A:.*]], ptr %[[ARG_P:.*]])
 // LLVM-NEXT:   %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[P_ADDR:.*]] = alloca ptr, i64 1, align 8
 // LLVM-NEXT:   %[[RETVAL_ADDR:.*]] = alloca i8, i64 1, align 1
@@ -294,7 +294,7 @@ void f5(struct NodeS* a) {
   a->next = 0;
 }
 
-// CIR: cir.func @f5
+// CIR: cir.func{{.*}} @f5
 // CIR:   %[[NEXT:.*]] = cir.get_member {{%.}}[0] {name = "next"} : !cir.ptr<!rec_NodeS> -> !cir.ptr<!cir.ptr<!rec_NodeS>>
 // CIR:   cir.store {{.*}}, %[[NEXT]]
 
@@ -312,7 +312,7 @@ void f6(struct CycleStart *start) {
   struct CycleStart *start2 = end->start;
 }
 
-// CIR: cir.func @f6
+// CIR: cir.func{{.*}} @f6
 // CIR:   %[[MIDDLE:.*]] = cir.get_member {{.*}}[0] {name = "middle"} : !cir.ptr<!rec_CycleStart> -> !cir.ptr<!cir.ptr<!rec_CycleMiddle>>
 // CIR:   %[[END:.*]] = cir.get_member %{{.*}}[0] {name = "end"} : !cir.ptr<!rec_CycleMiddle> -> !cir.ptr<!cir.ptr<!rec_CycleEnd>>
 // CIR:   %[[START2:.*]] = cir.get_member %{{.*}}[0] {name = "start"} : !cir.ptr<!rec_CycleEnd> -> !cir.ptr<!cir.ptr<!rec_CycleStart>>
diff --git a/clang/test/CIR/CodeGen/struct.cpp b/clang/test/CIR/CodeGen/struct.cpp
index c8406f811a46..ee6c4cab7341 100644
--- a/clang/test/CIR/CodeGen/struct.cpp
+++ b/clang/test/CIR/CodeGen/struct.cpp
@@ -27,11 +27,11 @@ void f(void) {
   IncompleteS *p;
 }
 
-// CIR:      cir.func @_Z1fv()
+// CIR:      cir.func{{.*}} @_Z1fv()
 // CIR-NEXT:   cir.alloca !cir.ptr<!rec_IncompleteS>, !cir.ptr<!cir.ptr<!rec_IncompleteS>>, ["p"]
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @_Z1fv()
+// LLVM:      define{{.*}} void @_Z1fv()
 // LLVM-NEXT:   %[[P:.*]] = alloca ptr, i64 1, align 8
 // LLVM-NEXT:   ret void
 
@@ -44,14 +44,14 @@ char f2(CompleteS &s) {
   return s.b;
 }
 
-// CIR: cir.func @_Z2f2R9CompleteS(%[[ARG_S:.*]]: !cir.ptr<!rec_CompleteS>{{.*}})
+// CIR: cir.func{{.*}} @_Z2f2R9CompleteS(%[[ARG_S:.*]]: !cir.ptr<!rec_CompleteS>{{.*}})
 // CIR:   %[[S_ADDR:.*]] = cir.alloca !cir.ptr<!rec_CompleteS>, !cir.ptr<!cir.ptr<!rec_CompleteS>>, ["s", init, const]
 // CIR:   cir.store %[[ARG_S]], %[[S_ADDR]]
 // CIR:   %[[S_REF:.*]] = cir.load{{.*}} %[[S_ADDR]]
 // CIR:   %[[S_ADDR2:.*]] = cir.get_member %[[S_REF]][1] {name = "b"}
 // CIR:   %[[S_B:.*]] = cir.load{{.*}} %[[S_ADDR2]]
 
-// LLVM: define i8 @_Z2f2R9CompleteS(ptr %[[ARG_S:.*]])
+// LLVM: define{{.*}} i8 @_Z2f2R9CompleteS(ptr %[[ARG_S:.*]])
 // LLVM:   %[[S_ADDR:.*]] = alloca ptr
 // LLVM:   store ptr %[[ARG_S]], ptr %[[S_ADDR]]
 // LLVM:   %[[S_REF:.*]] = load ptr, ptr %[[S_ADDR]], align 8
@@ -79,7 +79,7 @@ void f3() {
   o.i.n;
 }
 
-// CIR: cir.func @_Z2f3v()
+// CIR: cir.func{{.*}} @_Z2f3v()
 // CIR:   %[[O:.*]] = cir.alloca !rec_Outer, !cir.ptr<!rec_Outer>, ["o"]
 // CIR:   %[[O_I:.*]] = cir.get_member %[[O]][0] {name = "i"}
 // CIR:   %[[O_I_N:.*]] = cir.get_member %[[O_I]][0] {name = "n"}
diff --git a/clang/test/CIR/CodeGen/switch.cpp b/clang/test/CIR/CodeGen/switch.cpp
index 8786c2350c19..e13aa8f4f495 100644
--- a/clang/test/CIR/CodeGen/switch.cpp
+++ b/clang/test/CIR/CodeGen/switch.cpp
@@ -19,7 +19,7 @@ void sw1(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw1i
+// CIR: cir.func{{.*}} @_Z3sw1i
 // CIR: cir.switch (%[[COND:.*]] : !s32i) {
 // CIR-NEXT: cir.case(equal, [#cir.int<0> : !s32i]) {
 // CIR: cir.break
@@ -30,7 +30,7 @@ void sw1(int a) {
 // CIR: cir.alloca !s32i, !cir.ptr<!s32i>, ["yolo", init]
 // CIR: cir.break
 
-// LLVM: define void @_Z3sw1i
+// LLVM: define{{.*}} void @_Z3sw1i
 // LLVM:   store i32 1, ptr %[[B_ADDR:.*]], align 4
 // LLVM:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   br label %[[BB7:.*]]
@@ -60,7 +60,7 @@ void sw1(int a) {
 // LLVM: [[DEFAULT]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw1i
+// OGCG: define{{.*}} void @_Z3sw1i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[B:.*]] = alloca i32, align 4
@@ -97,7 +97,7 @@ void sw2(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw2i
+// CIR: cir.func{{.*}} @_Z3sw2i
 // CIR: cir.scope {
 // CIR-NEXT:   %[[YOLO:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["yolo", init]
 // CIR-NEXT:   %[[FOMO:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["fomo", init]
@@ -106,7 +106,7 @@ void sw2(int a) {
 // CIR-NEXT:     %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
 // CIR-NEXT:     cir.store{{.*}} %[[ZERO]], %[[FOMO]] : !s32i, !cir.ptr<!s32i>
 
-// LLVM: define void @_Z3sw2i
+// LLVM: define{{.*}} void @_Z3sw2i
 // LLVM:   store i32 2, ptr %[[YOLO_ADDR:.*]], align 4
 // LLVM:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   br label %[[SWITCH:.*]]
@@ -126,7 +126,7 @@ void sw2(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw2i
+// OGCG: define{{.*}} void @_Z3sw2i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[YOLO:.*]] = alloca i32, align 4
@@ -151,7 +151,7 @@ void sw3(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw3i
+// CIR: cir.func{{.*}} @_Z3sw3i
 // CIR: cir.scope {
 // CIR-NEXT:   %[[COND:.*]] = cir.load{{.*}} %[[A:.*]] : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.switch (%[[COND]] : !s32i) {
@@ -161,7 +161,7 @@ void sw3(int a) {
 // CIR-NEXT:   cir.yield
 // CIR-NEXT:   }
 
-// LLVM-LABEL: define void @_Z3sw3i
+// LLVM-LABEL: define{{.*}} void @_Z3sw3i
 // LLVM:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   br label %[[SWITCH:.*]]
 // LLVM: [[SWITCH]]:
@@ -174,7 +174,7 @@ void sw3(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw3i
+// OGCG: define{{.*}} void @_Z3sw3i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -195,7 +195,7 @@ int sw4(int a) {
   return 0;
 }
 
-// CIR: cir.func @_Z3sw4i
+// CIR: cir.func{{.*}} @_Z3sw4i
 // CIR:       cir.switch (%[[COND:.*]] : !s32i) {
 // CIR-NEXT:       cir.case(equal, [#cir.int<42> : !s32i]) {
 // CIR-NEXT:         cir.scope {
@@ -215,7 +215,7 @@ int sw4(int a) {
 // CIR-NEXT:       cir.yield
 // CIR-NEXT:  }
 
-// LLVM: define i32 @_Z3sw4i
+// LLVM: define{{.*}} i32 @_Z3sw4i
 // LLVM:   %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[RET_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[ENTRY:.*]]
@@ -241,7 +241,7 @@ int sw4(int a) {
 // LLVM:   %[[RET0:.*]] = load i32, ptr %[[RET_ADDR]], align 4
 // LLVM:   ret i32 %[[RET0]]
 
-// OGCG: define dso_local noundef i32 @_Z3sw4i
+// OGCG: define{{.*}} i32 @_Z3sw4i
 // OGCG: entry:
 // OGCG:   %[[RETVAL:.*]] = alloca i32, align 4
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
@@ -263,7 +263,7 @@ void sw5(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw5i
+// CIR: cir.func{{.*}} @_Z3sw5i
 // CIR: cir.switch (%[[A:.*]] : !s32i) {
 // CIR-NEXT:   cir.case(equal, [#cir.int<1> : !s32i]) {
 // CIR-NEXT:     cir.yield
@@ -271,7 +271,7 @@ void sw5(int a) {
 // CIR-NEXT:   cir.yield
 // CIR-NEXT:   }
 
-// LLVM-LABEL: define void @_Z3sw5i
+// LLVM-LABEL: define{{.*}} void @_Z3sw5i
 // LLVM:   %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   br label %[[ENTRY:.*]]
 // LLVM: [[ENTRY]]:
@@ -288,7 +288,7 @@ void sw5(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw5i
+// OGCG: define{{.*}} void @_Z3sw5i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -313,7 +313,7 @@ void sw6(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw6i
+// CIR: cir.func{{.*}} @_Z3sw6i
 // CIR: cir.switch (%[[A:.*]] : !s32i) {
 // CIR-NEXT: cir.case(equal, [#cir.int<0> : !s32i]) {
 // CIR-NEXT:     cir.yield
@@ -334,7 +334,7 @@ void sw6(int a) {
 // CIR-NEXT:     cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z3sw6i
+// LLVM: define{{.*}} void @_Z3sw6i
 // LLVM:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   br label %[[SWITCH:.*]]
 // LLVM: [[SWITCH]]:
@@ -371,7 +371,7 @@ void sw6(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw6i
+// OGCG: define{{.*}} void @_Z3sw6i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   store i32 %a, ptr %[[A_ADDR]], align 4
@@ -404,7 +404,7 @@ void sw7(int a) {
   }
 }
 
-// CIR: cir.func @_Z3sw7i
+// CIR: cir.func{{.*}} @_Z3sw7i
 // CIR: %[[X:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["x"]
 // CIR: cir.switch (%[[A:.*]] : !s32i)
 // CIR-NEXT: cir.case(equal, [#cir.int<0> : !s32i]) {
@@ -428,7 +428,7 @@ void sw7(int a) {
 // CIR-NEXT: cir.yield
 // CIR: }
 
-// LLVM: define void @_Z3sw7i
+// LLVM: define{{.*}} void @_Z3sw7i
 // LLVM:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   br label %[[SWITCH:.*]]
 // LLVM: [[SWITCH]]:
@@ -467,7 +467,7 @@ void sw7(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw7i
+// OGCG: define{{.*}} void @_Z3sw7i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -498,7 +498,7 @@ void sw8(int a) {
   }
 }
 
-// CIR:    cir.func @_Z3sw8i
+// CIR:    cir.func{{.*}} @_Z3sw8i
 // CIR:    cir.switch (%[[A:.*]] : !s32i)
 // CIR-NEXT: cir.case(equal, [#cir.int<3> : !s32i]) {
 // CIR-NEXT:   cir.break
@@ -510,7 +510,7 @@ void sw8(int a) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z3sw8i
+// LLVM: define{{.*}} void @_Z3sw8i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:  i32 3, label %[[CASE3:.*]]
 // LLVM-DAG:  i32 4, label %[[CASE4:.*]]
@@ -528,7 +528,7 @@ void sw8(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw8i
+// OGCG: define{{.*}} void @_Z3sw8i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -556,7 +556,7 @@ void sw9(int a) {
   }
 }
 
-// CIR:    cir.func @_Z3sw9i
+// CIR:    cir.func{{.*}} @_Z3sw9i
 // CIR:    cir.switch (%[[A:.*]] : !s32i)
 // CIR-NEXT: cir.case(equal, [#cir.int<3> : !s32i]) {
 // CIR-NEXT:   cir.break
@@ -568,7 +568,7 @@ void sw9(int a) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z3sw9i
+// LLVM: define{{.*}} void @_Z3sw9i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:     i32 3, label %[[CASE3:.*]]
 // LLVM-DAG:     i32 4, label %[[CASE4:.*]]
@@ -586,7 +586,7 @@ void sw9(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z3sw9i
+// OGCG: define{{.*}} void @_Z3sw9i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -615,7 +615,7 @@ void sw10(int a) {
   }
 }
 
-// CIR:    cir.func @_Z4sw10i
+// CIR:    cir.func{{.*}} @_Z4sw10i
 // CIR:    cir.switch (%[[A:.*]] : !s32i)
 // CIR-NEXT: cir.case(equal, [#cir.int<3> : !s32i]) {
 // CIR-NEXT:   cir.break
@@ -630,7 +630,7 @@ void sw10(int a) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z4sw10i
+// LLVM: define{{.*}} void @_Z4sw10i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:     i32 3, label %[[CASE_3:.*]]
 // LLVM-DAG:     i32 4, label %[[CASE_4:.*]]
@@ -653,7 +653,7 @@ void sw10(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw10i
+// OGCG: define{{.*}} void @_Z4sw10i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -687,7 +687,7 @@ void sw11(int a) {
   }
 }
 
-// CIR:    cir.func @_Z4sw11i
+// CIR:    cir.func{{.*}} @_Z4sw11i
 // CIR:    cir.switch (%[[A:.*]] : !s32i)
 // CIR-NEXT: cir.case(equal, [#cir.int<3> : !s32i]) {
 // CIR-NEXT:   cir.break
@@ -708,7 +708,7 @@ void sw11(int a) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z4sw11i
+// LLVM: define{{.*}} void @_Z4sw11i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:     i32 3, label %[[CASE_3:.*]]
 // LLVM-DAG:     i32 4, label %[[CASE_4:.*]]
@@ -741,7 +741,7 @@ void sw11(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw11i
+// OGCG: define{{.*}} void @_Z4sw11i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -772,7 +772,7 @@ void sw12(int a) {
   }
 }
 
-//      CIR: cir.func @_Z4sw12i
+//      CIR: cir.func{{.*}} @_Z4sw12i
 //      CIR:   cir.scope {
 //      CIR:     cir.switch
 // CIR-NEXT:     cir.case(equal, [#cir.int<3> : !s32i]) {
@@ -781,7 +781,7 @@ void sw12(int a) {
 // CIR-NEXT:       cir.break
 // CIR-NEXT:     }
 
-// LLVM: define void @_Z4sw12i
+// LLVM: define{{.*}} void @_Z4sw12i
 // LLVM:   switch i32 %[[COND:.*]], label %[[EXIT:.*]] [
 // LLVM-DAG:     i32 3, label %[[CASE_3:.*]]
 // LLVM:   ]
@@ -794,7 +794,7 @@ void sw12(int a) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw12i
+// OGCG: define{{.*}} void @_Z4sw12i
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[A_VAL:.*]] = load i32, ptr %[[A_ADDR]], align 4
@@ -816,7 +816,7 @@ void sw13(int a, int b) {
   }
 }
 
-//      CIR:  cir.func @_Z4sw13ii
+//      CIR:  cir.func{{.*}} @_Z4sw13ii
 //      CIR:    cir.scope {
 //      CIR:      cir.switch
 // CIR-NEXT:      cir.case(equal, [#cir.int<1> : !s32i]) {
@@ -832,7 +832,7 @@ void sw13(int a, int b) {
 //      CIR:    }
 //      CIR:    cir.return
 
-// LLVM: define void @_Z4sw13ii
+// LLVM: define{{.*}} void @_Z4sw13ii
 // LLVM:   switch i32 %[[COND:.*]], label %[[OUTER_EXIT:.*]] [
 // LLVM-DAG:     i32 1, label %[[CASE_A_1:.*]]
 // LLVM:   ]
@@ -858,7 +858,7 @@ void sw13(int a, int b) {
 // LLVM: [[EXIT]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw13ii
+// OGCG: define{{.*}} void @_Z4sw13ii
 // OGCG: entry:
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[B_ADDR:.*]] = alloca i32, align 4
@@ -890,7 +890,7 @@ void sw14(int x) {
   }
 }
 
-// CIR:      cir.func @_Z4sw14i
+// CIR:      cir.func{{.*}} @_Z4sw14i
 // CIR:      cir.switch
 // CIR-NEXT: cir.case(equal, [#cir.int<1> : !s32i]) {
 // CIR-NEXT:   cir.yield
@@ -908,7 +908,7 @@ void sw14(int x) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z4sw14i
+// LLVM: define{{.*}} void @_Z4sw14i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:     i32 1, label %[[CASE1:.*]]
 // LLVM-DAG:     i32 2, label %[[CASE2:.*]]
@@ -939,7 +939,7 @@ void sw14(int x) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw14i
+// OGCG: define{{.*}} void @_Z4sw14i
 // OGCG: entry:
 // OGCG:   %[[X_ADDR:.*]] = alloca i32, align 4
 // OGCG:   store i32 %x, ptr %[[X_ADDR]], align 4
@@ -977,7 +977,7 @@ void sw15(int x) {
   }
 }
 
-// CIR:      cir.func @_Z4sw15i
+// CIR:      cir.func{{.*}} @_Z4sw15i
 // CIR:      %[[Y:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["y"]
 // CIR:      cir.switch
 // CIR-NEXT: cir.case(equal, [#cir.int<1> : !s32i]) {
@@ -995,7 +995,7 @@ void sw15(int x) {
 // CIR-NEXT:   cir.break
 // CIR-NEXT: }
 
-// LLVM: define void @_Z4sw15i
+// LLVM: define{{.*}} void @_Z4sw15i
 // LLVM:   switch i32 %[[COND:.*]], label %[[DEFAULT:.*]] [
 // LLVM-DAG:     i32 1, label %[[CASE1:.*]]
 // LLVM-DAG:     i32 2, label %[[CASE2:.*]]
@@ -1019,7 +1019,7 @@ void sw15(int x) {
 // LLVM: [[RET]]:
 // LLVM:   ret void
 
-// OGCG: define dso_local void @_Z4sw15i
+// OGCG: define{{.*}} void @_Z4sw15i
 // OGCG: entry:
 // OGCG:   %[[X_ADDR:.*]] = alloca i32, align 4
 // OGCG:   %[[Y:.*]] = alloca i32, align 4
@@ -1081,7 +1081,7 @@ int nested_switch(int a) {
 // CIR:           cir.case(equal, [#cir.int<7> : !s32i]) {
 // CIR:           cir.return
 
-// LLVM: define i32 @_Z13nested_switchi
+// LLVM: define{{.*}} i32 @_Z13nested_switchi
 // LLVM:   %[[B_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A_ADDR:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[RES_ADDR:.*]] = alloca i32, i64 1, align 4
@@ -1136,7 +1136,7 @@ int nested_switch(int a) {
 // LLVM:   %[[RET0:.*]] = load i32, ptr %[[RES_ADDR]], align 4
 // LLVM:   ret i32 %[[RET0]]
 
-// OGCG: define dso_local noundef i32 @_Z13nested_switchi
+// OGCG: define{{.*}} i32 @_Z13nested_switchi
 // OGCG: entry:
 // OGCG:   %[[RETVAL:.*]] = alloca i32, align 4
 // OGCG:   %[[A_ADDR:.*]] = alloca i32, align 4
diff --git a/clang/test/CIR/CodeGen/switch_flat_op.cpp b/clang/test/CIR/CodeGen/switch_flat_op.cpp
index f917bd59f7ce..a3ea7e7a1554 100644
--- a/clang/test/CIR/CodeGen/switch_flat_op.cpp
+++ b/clang/test/CIR/CodeGen/switch_flat_op.cpp
@@ -18,7 +18,7 @@ void swf(int a) {
 
 }
 
-// BEFORE:  cir.func @_Z3swfi
+// BEFORE:  cir.func{{.*}} @_Z3swfi
 // BEFORE:   %[[VAR_B:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init] {alignment = 4 : i64}
 // BEFORE:   %[[CONST_3:.*]] = cir.const #cir.int<3> : !s32i
 // BEFORE:   cir.switch (%[[COND:.*]] : !s32i) {
@@ -44,7 +44,7 @@ void swf(int a) {
 // BEFORE: }
 // BEFORE: cir.return
 
-// AFTER: cir.func @_Z3swfi
+// AFTER: cir.func{{.*}} @_Z3swfi
 // AFTER:  %[[VAR_A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init] {alignment = 4 : i64}
 // AFTER:  cir.store{{.*}} %arg0, %[[VAR_A]] : !s32i, !cir.ptr<!s32i>
 // AFTER:  %[[VAR_B:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init] {alignment = 4 : i64}
diff --git a/clang/test/CIR/CodeGen/ternary.cpp b/clang/test/CIR/CodeGen/ternary.cpp
index 3b66f7ccdf54..781286a94cc2 100644
--- a/clang/test/CIR/CodeGen/ternary.cpp
+++ b/clang/test/CIR/CodeGen/ternary.cpp
@@ -9,7 +9,7 @@ int x(int y) {
   return y > 0 ? 3 : 5;
 }
 
-// CIR-LABEL: cir.func @_Z1xi(
+// CIR-LABEL: cir.func{{.*}} @_Z1xi(
 // CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}) -> !s32i {
 // CIR: [[Y:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["y", init] {alignment = 4 : i64}
 // CIR: [[RETVAL:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
@@ -24,7 +24,7 @@ int x(int y) {
 // CIR: [[RETVAL_VAL:%.+]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
 // CIR: cir.return [[RETVAL_VAL]] : !s32i
 
-// LLVM-LABEL: define i32 @_Z1xi(
+// LLVM-LABEL: define{{.*}} i32 @_Z1xi(
 // LLVM-SAME: i32 %[[ARG0:.+]])
 // LLVM: %[[Y:.*]] = alloca i32
 // LLVM: %[[RETVAL:.*]] = alloca i32
@@ -36,7 +36,7 @@ int x(int y) {
 // LLVM: %[[RESULT:.*]] = load i32, ptr %[[RETVAL]]
 // LLVM: ret i32 %[[RESULT]]
 
-// OGCG-LABEL: define dso_local noundef i32 @_Z1xi(
+// OGCG-LABEL: define{{.*}} i32 @_Z1xi(
 // OGCG-SAME: i32 {{.*}} %[[ARG0:.+]])
 // OGCG: %[[Y:.*]] = alloca i32
 // OGCG: store i32 %[[ARG0]], ptr %[[Y]]
@@ -51,7 +51,7 @@ int foo(int a, int b) {
   return 0;
 }
 
-// CIR-LABEL: cir.func @_Z3fooii(
+// CIR-LABEL: cir.func{{.*}} @_Z3fooii(
 // CIR-SAME: %[[ARG0:.*]]: !s32i {{.*}}, %[[ARG1:.*]]: !s32i {{.*}}) -> !s32i {
 // CIR: [[A:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init] {alignment = 4 : i64}
 // CIR: [[B:%.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init] {alignment = 4 : i64}
@@ -83,7 +83,7 @@ int foo(int a, int b) {
 // CIR: [[RETVAL_VAL2:%.+]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
 // CIR: cir.return [[RETVAL_VAL2]] : !s32i
 
-// LLVM-LABEL: define i32 @_Z3fooii(
+// LLVM-LABEL: define{{.*}} i32 @_Z3fooii(
 // LLVM-SAME: i32 %[[ARG0:.*]], i32 %[[ARG1:.*]])
 // LLVM: %[[A:.*]] = alloca i32
 // LLVM: %[[B:.*]] = alloca i32
@@ -116,7 +116,7 @@ int foo(int a, int b) {
 // LLVM: %[[RET2:.*]] = load i32, ptr %[[RETVAL]]
 // LLVM: ret i32 %[[RET2]]
 
-// OGCG-LABEL: define dso_local noundef i32 @_Z3fooii(
+// OGCG-LABEL: define{{.*}} i32 @_Z3fooii(
 // OGCG-SAME: i32 {{.*}} %[[ARG0:.*]], i32 {{.*}} %[[ARG1:.*]])
 // OGCG: %[[RETVAL:.*]] = alloca i32
 // OGCG: %[[A:.*]] = alloca i32
diff --git a/clang/test/CIR/CodeGen/typedef.c b/clang/test/CIR/CodeGen/typedef.c
index a87e6ffb1843..201df2e08ee2 100644
--- a/clang/test/CIR/CodeGen/typedef.c
+++ b/clang/test/CIR/CodeGen/typedef.c
@@ -10,12 +10,12 @@ void local_typedef(void) {
   Struct s;
 }
 
-// CIR:      cir.func @local_typedef()
+// CIR:      cir.func{{.*}} @local_typedef()
 // CIR:        cir.alloca !rec_Struct, !cir.ptr<!rec_Struct>, ["s"] {alignment = 4 : i64}
 // CIR:        cir.return
 
 // LLVM: %struct.Struct = type { i32 }
-// LLVM: define void @local_typedef()
+// LLVM: define{{.*}} void @local_typedef()
 // LLVM:   alloca %struct.Struct, i64 1, align 4
 // LLVM:   ret void
 
diff --git a/clang/test/CIR/CodeGen/unary.cpp b/clang/test/CIR/CodeGen/unary.cpp
index 0633cc3fd8e1..a7c946eaffd0 100644
--- a/clang/test/CIR/CodeGen/unary.cpp
+++ b/clang/test/CIR/CodeGen/unary.cpp
@@ -10,12 +10,12 @@ unsigned up0() {
   return +a;
 }
 
-// CHECK: cir.func @_Z3up0v() -> !u32i
+// CHECK: cir.func{{.*}} @_Z3up0v() -> !u32i
 // CHECK:   %[[A:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[OUTPUT:.*]] = cir.unary(plus, %[[INPUT]])
 
-// LLVM: define i32 @_Z3up0v()
+// LLVM: define{{.*}} i32 @_Z3up0v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -31,12 +31,12 @@ unsigned um0() {
   return -a;
 }
 
-// CHECK: cir.func @_Z3um0v() -> !u32i
+// CHECK: cir.func{{.*}} @_Z3um0v() -> !u32i
 // CHECK:   %[[A:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[OUTPUT:.*]] = cir.unary(minus, %[[INPUT]])
 
-// LLVM: define i32 @_Z3um0v()
+// LLVM: define{{.*}} i32 @_Z3um0v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -54,12 +54,12 @@ unsigned un0() {
   return ~a; // a ^ -1 , not
 }
 
-// CHECK: cir.func @_Z3un0v() -> !u32i
+// CHECK: cir.func{{.*}} @_Z3un0v() -> !u32i
 // CHECK:   %[[A:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init]
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[OUTPUT:.*]] = cir.unary(not, %[[INPUT]])
 
-// LLVM: define i32 @_Z3un0v()
+// LLVM: define{{.*}} i32 @_Z3un0v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -78,7 +78,7 @@ int inc0() {
   return a;
 }
 
-// CHECK: cir.func @_Z4inc0v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z4inc0v() -> !s32i
 // CHECK:   %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.int<1> : !s32i
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !s32i
@@ -87,7 +87,7 @@ int inc0() {
 // CHECK:   cir.store{{.*}} %[[INCREMENTED]], %[[A]]
 // CHECK:   %[[A_TO_OUTPUT:.*]] = cir.load{{.*}} %[[A]]
 
-// LLVM: define i32 @_Z4inc0v()
+// LLVM: define{{.*}} i32 @_Z4inc0v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -106,7 +106,7 @@ int dec0() {
   return a;
 }
 
-// CHECK: cir.func @_Z4dec0v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z4dec0v() -> !s32i
 // CHECK:   %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.int<1> : !s32i
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !s32i
@@ -115,7 +115,7 @@ int dec0() {
 // CHECK:   cir.store{{.*}} %[[DECREMENTED]], %[[A]]
 // CHECK:   %[[A_TO_OUTPUT:.*]] = cir.load{{.*}} %[[A]]
 
-// LLVM: define i32 @_Z4dec0v()
+// LLVM: define{{.*}} i32 @_Z4dec0v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -134,7 +134,7 @@ int inc1() {
   return a;
 }
 
-// CHECK: cir.func @_Z4inc1v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z4inc1v() -> !s32i
 // CHECK:   %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.int<1> : !s32i
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !s32i
@@ -143,7 +143,7 @@ int inc1() {
 // CHECK:   cir.store{{.*}} %[[INCREMENTED]], %[[A]]
 // CHECK:   %[[A_TO_OUTPUT:.*]] = cir.load{{.*}} %[[A]]
 
-// LLVM: define i32 @_Z4inc1v()
+// LLVM: define{{.*}} i32 @_Z4inc1v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -162,7 +162,7 @@ int dec1() {
   return a;
 }
 
-// CHECK: cir.func @_Z4dec1v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z4dec1v() -> !s32i
 // CHECK:   %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.int<1> : !s32i
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !s32i
@@ -171,7 +171,7 @@ int dec1() {
 // CHECK:   cir.store{{.*}} %[[DECREMENTED]], %[[A]]
 // CHECK:   %[[A_TO_OUTPUT:.*]] = cir.load{{.*}} %[[A]]
 
-// LLVM: define i32 @_Z4dec1v()
+// LLVM: define{{.*}} i32 @_Z4dec1v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   store i32 1, ptr %[[A]], align 4
@@ -191,7 +191,7 @@ int inc2() {
   return b;
 }
 
-// CHECK: cir.func @_Z4inc2v() -> !s32i
+// CHECK: cir.func{{.*}} @_Z4inc2v() -> !s32i
 // CHECK:   %[[A:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
 // CHECK:   %[[B:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.int<1> : !s32i
@@ -202,7 +202,7 @@ int inc2() {
 // CHECK:   cir.store{{.*}} %[[ATOB]], %[[B]]
 // CHECK:   %[[B_TO_OUTPUT:.*]] = cir.load{{.*}} %[[B]]
 
-// LLVM: define i32 @_Z4inc2v()
+// LLVM: define{{.*}} i32 @_Z4inc2v()
 // LLVM:   %[[RV:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca i32, i64 1, align 4
 // LLVM:   %[[B:.*]] = alloca i32, i64 1, align 4
@@ -228,12 +228,12 @@ float fpPlus() {
   return +a;
 }
 
-// CHECK: cir.func @_Z6fpPlusv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z6fpPlusv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[OUTPUT:.*]] = cir.unary(plus, %[[INPUT]])
 
-// LLVM: define float @_Z6fpPlusv()
+// LLVM: define{{.*}} float @_Z6fpPlusv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -249,12 +249,12 @@ float fpMinus() {
   return -a;
 }
 
-// CHECK: cir.func @_Z7fpMinusv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z7fpMinusv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[OUTPUT:.*]] = cir.unary(minus, %[[INPUT]])
 
-// LLVM: define float @_Z7fpMinusv()
+// LLVM: define{{.*}} float @_Z7fpMinusv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -272,14 +272,14 @@ float fpPreInc() {
   return ++a;
 }
 
-// CHECK: cir.func @_Z8fpPreIncv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z8fpPreIncv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.fp<1.000000e+00> : !cir.float
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !cir.float
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[INCREMENTED:.*]] = cir.unary(inc, %[[INPUT]])
 
-// LLVM: define float @_Z8fpPreIncv()
+// LLVM: define{{.*}} float @_Z8fpPreIncv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -297,14 +297,14 @@ float fpPreDec() {
   return --a;
 }
 
-// CHECK: cir.func @_Z8fpPreDecv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z8fpPreDecv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.fp<1.000000e+00> : !cir.float
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !cir.float
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[DECREMENTED:.*]] = cir.unary(dec, %[[INPUT]])
 
-// LLVM: define float @_Z8fpPreDecv()
+// LLVM: define{{.*}} float @_Z8fpPreDecv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -322,14 +322,14 @@ float fpPostInc() {
   return a++;
 }
 
-// CHECK: cir.func @_Z9fpPostIncv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z9fpPostIncv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.fp<1.000000e+00> : !cir.float
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !cir.float
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[INCREMENTED:.*]] = cir.unary(inc, %[[INPUT]])
 
-// LLVM: define float @_Z9fpPostIncv()
+// LLVM: define{{.*}} float @_Z9fpPostIncv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -347,14 +347,14 @@ float fpPostDec() {
   return a--;
 }
 
-// CHECK: cir.func @_Z9fpPostDecv() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z9fpPostDecv() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.fp<1.000000e+00> : !cir.float
 // CHECK:   cir.store{{.*}} %[[ATMP]], %[[A]] : !cir.float
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[A]]
 // CHECK:   %[[DECREMENTED:.*]] = cir.unary(dec, %[[INPUT]])
 
-// LLVM: define float @_Z9fpPostDecv()
+// LLVM: define{{.*}} float @_Z9fpPostDecv()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   store float 1.000000e+00, ptr %[[A]], align 4
@@ -374,7 +374,7 @@ float fpPostInc2() {
   return b;
 }
 
-// CHECK: cir.func @_Z10fpPostInc2v() -> !cir.float
+// CHECK: cir.func{{.*}} @_Z10fpPostInc2v() -> !cir.float
 // CHECK:   %[[A:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init]
 // CHECK:   %[[B:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b", init]
 // CHECK:   %[[ATMP:.*]] = cir.const #cir.fp<1.000000e+00> : !cir.float
@@ -385,7 +385,7 @@ float fpPostInc2() {
 // CHECK:   cir.store{{.*}} %[[ATOB]], %[[B]]
 // CHECK:   %[[B_TO_OUTPUT:.*]] = cir.load{{.*}} %[[B]]
 
-// LLVM: define float @_Z10fpPostInc2v()
+// LLVM: define{{.*}} float @_Z10fpPostInc2v()
 // LLVM:   %[[RV:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[A:.*]] = alloca float, i64 1, align 4
 // LLVM:   %[[B:.*]] = alloca float, i64 1, align 4
@@ -407,7 +407,7 @@ float fpPostInc2() {
 // OGCG:   %[[B_TO_OUTPUT:.*]] = load float, ptr %[[B]], align 4
 
 void chars(char c) {
-// CHECK: cir.func @_Z5charsc
+// CHECK: cir.func{{.*}} @_Z5charsc
 
   int c1 = +c;
   // CHECK: %[[PROMO:.*]] = cir.cast(integral, %{{.+}} : !s8i), !s32i
@@ -429,13 +429,13 @@ _Float16 fp16UPlus(_Float16 f) {
   return +f;
 }
 
-// CHECK: cir.func @_Z9fp16UPlusDF16_({{.*}}) -> !cir.f16
+// CHECK: cir.func{{.*}} @_Z9fp16UPlusDF16_({{.*}}) -> !cir.f16
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[F:.*]]
 // CHECK:   %[[PROMOTED:.*]] = cir.cast(floating, %[[INPUT]] : !cir.f16), !cir.float
 // CHECK:   %[[RESULT:.*]] = cir.unary(plus, %[[PROMOTED]])
 // CHECK:   %[[UNPROMOTED:.*]] = cir.cast(floating, %[[RESULT]] : !cir.float), !cir.f16
 
-// LLVM: define half @_Z9fp16UPlusDF16_({{.*}})
+// LLVM: define{{.*}} half @_Z9fp16UPlusDF16_({{.*}})
 // LLVM:   %[[F_LOAD:.*]] = load half, ptr %{{.*}}, align 2
 // LLVM:   %[[PROMOTED:.*]] = fpext half %[[F_LOAD]] to float
 // LLVM:   %[[UNPROMOTED:.*]] = fptrunc float %[[PROMOTED]] to half
@@ -449,13 +449,13 @@ _Float16 fp16UMinus(_Float16 f) {
   return -f;
 }
 
-// CHECK: cir.func @_Z10fp16UMinusDF16_({{.*}}) -> !cir.f16
+// CHECK: cir.func{{.*}} @_Z10fp16UMinusDF16_({{.*}}) -> !cir.f16
 // CHECK:   %[[INPUT:.*]] = cir.load{{.*}} %[[F:.*]]
 // CHECK:   %[[PROMOTED:.*]] = cir.cast(floating, %[[INPUT]] : !cir.f16), !cir.float
 // CHECK:   %[[RESULT:.*]] = cir.unary(minus, %[[PROMOTED]])
 // CHECK:   %[[UNPROMOTED:.*]] = cir.cast(floating, %[[RESULT]] : !cir.float), !cir.f16
 
-// LLVM: define half @_Z10fp16UMinusDF16_({{.*}})
+// LLVM: define{{.*}} half @_Z10fp16UMinusDF16_({{.*}})
 // LLVM:   %[[F_LOAD:.*]] = load half, ptr %{{.*}}, align 2
 // LLVM:   %[[PROMOTED:.*]] = fpext half %[[F_LOAD]] to float
 // LLVM:   %[[RESULT:.*]] = fneg float %[[PROMOTED]]
@@ -480,7 +480,7 @@ void test_logical_not() {
   b = !d;
 }
 
-// CHECK: cir.func @_Z16test_logical_notv()
+// CHECK: cir.func{{.*}} @_Z16test_logical_notv()
 // CHECK:   %[[A:.*]] = cir.load{{.*}} %[[A_ADDR:.*]] : !cir.ptr<!s32i>, !s32i
 // CHECK:   %[[A_BOOL:.*]] = cir.cast(int_to_bool, %[[A]] : !s32i), !cir.bool
 // CHECK:   %[[A_NOT:.*]] = cir.unary(not, %[[A_BOOL]]) : !cir.bool, !cir.bool
@@ -503,7 +503,7 @@ void test_logical_not() {
 // CHECK:   %[[D_NOT:.*]] = cir.unary(not, %[[D_BOOL]]) : !cir.bool, !cir.bool
 // CHECK:   cir.store{{.*}} %[[D_NOT]], %[[B_ADDR]] : !cir.bool, !cir.ptr<!cir.bool>
 
-// LLVM: define void @_Z16test_logical_notv()
+// LLVM: define{{.*}} void @_Z16test_logical_notv()
 // LLVM:   %[[A:.*]] = load i32, ptr %[[A_ADDR:.*]], align 4
 // LLVM:   %[[A_BOOL:.*]] = icmp ne i32 %[[A]], 0
 // LLVM:   %[[A_NOT:.*]] = xor i1 %[[A_BOOL]], true
diff --git a/clang/test/CIR/CodeGen/union.c b/clang/test/CIR/CodeGen/union.c
index d998b5add41a..23e862b24517 100644
--- a/clang/test/CIR/CodeGen/union.c
+++ b/clang/test/CIR/CodeGen/union.c
@@ -54,11 +54,11 @@ void f1(void) {
   union IncompleteU *p;
 }
 
-// CIR:      cir.func @f1()
+// CIR:      cir.func{{.*}} @f1()
 // CIR-NEXT:   cir.alloca !cir.ptr<!rec_IncompleteU>, !cir.ptr<!cir.ptr<!rec_IncompleteU>>, ["p"]
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @f1()
+// LLVM:      define{{.*}} void @f1()
 // LLVM-NEXT:   %[[P:.*]] = alloca ptr, i64 1, align 8
 // LLVM-NEXT:   ret void
 
@@ -73,7 +73,7 @@ int f2(void) {
   return u.n;
 }
 
-// CIR:      cir.func @f2() -> !s32i
+// CIR:      cir.func{{.*}} @f2() -> !s32i
 // CIR-NEXT:   %[[RETVAL_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CIR-NEXT:   %[[U:.*]] = cir.alloca !rec_U1, !cir.ptr<!rec_U1>, ["u"] {alignment = 4 : i64}
 // CIR-NEXT:   %[[I:.*]] = cir.const #cir.int<42> : !s32i
@@ -85,7 +85,7 @@ int f2(void) {
 // CIR-NEXT:   %[[RET:.*]] = cir.load{{.*}} %[[RETVAL_ADDR]] : !cir.ptr<!s32i>, !s32i
 // CIR-NEXT:   cir.return %[[RET]] : !s32i
 
-// LLVM:      define i32 @f2()
+// LLVM:      define{{.*}} i32 @f2()
 // LLVM-NEXT:   %[[RETVAL:.*]] = alloca i32, i64 1, align 4
 // LLVM-NEXT:   %[[U:.*]] = alloca %union.U1, i64 1, align 4
 // LLVM-NEXT:   store i32 42, ptr %[[U]], align 4
@@ -94,7 +94,7 @@ int f2(void) {
 // LLVM-NEXT:   %[[RET:.*]] = load i32, ptr %[[RETVAL]], align 4
 // LLVM-NEXT:   ret i32 %[[RET]]
 
-//      OGCG: define dso_local i32 @f2()
+//      OGCG: define{{.*}} i32 @f2()
 // OGCG-NEXT: entry:
 // OGCG-NEXT: %[[U:.*]] = alloca %union.U1, align 4
 // OGCG-NEXT: store i32 42, ptr %[[U]], align 4
@@ -112,7 +112,7 @@ void shouldGenerateUnionAccess(union U2 u) {
   u.d;
 }
 
-// CIR:      cir.func @shouldGenerateUnionAccess(%[[ARG:.*]]: !rec_U2
+// CIR:      cir.func{{.*}} @shouldGenerateUnionAccess(%[[ARG:.*]]: !rec_U2
 // CIR-NEXT:   %[[U:.*]] = cir.alloca !rec_U2, !cir.ptr<!rec_U2>, ["u", init] {alignment = 8 : i64}
 // CIR-NEXT:   cir.store{{.*}} %[[ARG]], %[[U]] : !rec_U2, !cir.ptr<!rec_U2>
 // CIR-NEXT:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
@@ -138,7 +138,7 @@ void shouldGenerateUnionAccess(union U2 u) {
 // CIR-NEXT:   %[[D_VAL:.*]] = cir.load{{.*}} %[[D_PTR2]] : !cir.ptr<!cir.double>, !cir.double
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @shouldGenerateUnionAccess(%union.U2 %[[ARG:.*]])
+// LLVM:      define{{.*}} void @shouldGenerateUnionAccess(%union.U2 %[[ARG:.*]])
 // LLVM-NEXT:   %[[U:.*]] = alloca %union.U2, i64 1, align 8
 // LLVM-NEXT:   store %union.U2 %[[ARG]], ptr %[[U]], align 8
 // LLVM-NEXT:   store i8 0, ptr %[[U]], align 8
@@ -151,7 +151,7 @@ void shouldGenerateUnionAccess(union U2 u) {
 // LLVM-NEXT:   %[[D_VAL:.*]] = load double, ptr %[[U]], align 8
 // LLVM-NEXT:   ret void
 
-// OGCG:      define dso_local void @shouldGenerateUnionAccess(i64 %[[ARG:.*]])
+// OGCG:      define{{.*}} void @shouldGenerateUnionAccess(i64 %[[ARG:.*]])
 // OGCG-NEXT: entry:
 // OGCG-NEXT:   %[[U:.*]] = alloca %union.U2, align 8
 // OGCG-NEXT:   %[[COERCE_DIVE:.*]] = getelementptr inbounds nuw %union.U2, ptr %[[U]], i32 0, i32 0
@@ -170,7 +170,7 @@ void f3(union U3 u) {
   u.c[2] = 0;
 }
 
-// CIR:      cir.func @f3(%[[ARG:.*]]: !rec_U3
+// CIR:      cir.func{{.*}} @f3(%[[ARG:.*]]: !rec_U3
 // CIR-NEXT:   %[[U:.*]] = cir.alloca !rec_U3, !cir.ptr<!rec_U3>, ["u", init] {alignment = 1 : i64}
 // CIR-NEXT:   cir.store{{.*}} %[[ARG]], %[[U]] : !rec_U3, !cir.ptr<!rec_U3>
 // CIR-NEXT:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
@@ -182,7 +182,7 @@ void f3(union U3 u) {
 // CIR-NEXT:   cir.store{{.*}} %[[ZERO_CHAR]], %[[ELEM_PTR]] : !s8i, !cir.ptr<!s8i>
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @f3(%union.U3 %[[ARG:.*]])
+// LLVM:      define{{.*}} void @f3(%union.U3 %[[ARG:.*]])
 // LLVM-NEXT:   %[[U:.*]] = alloca %union.U3, i64 1, align 1
 // LLVM-NEXT:   store %union.U3 %[[ARG]], ptr %[[U]], align 1
 // LLVM-NEXT:   %[[C_PTR:.*]] = getelementptr i8, ptr %[[U]], i32 0
@@ -190,7 +190,7 @@ void f3(union U3 u) {
 // LLVM-NEXT:   store i8 0, ptr %[[ELEM_PTR]], align 1
 // LLVM-NEXT:   ret void
 
-// OGCG:      define dso_local void @f3(i40 %[[ARG:.*]])
+// OGCG:      define{{.*}} void @f3(i40 %[[ARG:.*]])
 // OGCG-NEXT: entry:
 // OGCG-NEXT:   %[[U:.*]] = alloca %union.U3, align 1
 // OGCG-NEXT:   store i40 %[[ARG]], ptr %[[U]], align 1
@@ -202,7 +202,7 @@ void f5(union U4 u) {
   u.c[4] = 65;
 }
 
-// CIR:      cir.func @f5(%[[ARG:.*]]: !rec_U4
+// CIR:      cir.func{{.*}} @f5(%[[ARG:.*]]: !rec_U4
 // CIR-NEXT:   %[[U:.*]] = cir.alloca !rec_U4, !cir.ptr<!rec_U4>, ["u", init] {alignment = 4 : i64}
 // CIR-NEXT:   cir.store{{.*}} %[[ARG]], %[[U]] : !rec_U4, !cir.ptr<!rec_U4>
 // CIR-NEXT:   %[[CHAR_VAL:.*]] = cir.const #cir.int<65> : !s32i
@@ -214,7 +214,7 @@ void f5(union U4 u) {
 // CIR-NEXT:   cir.store{{.*}} %[[CHAR_CAST]], %[[ELEM_PTR]] : !s8i, !cir.ptr<!s8i>
 // CIR-NEXT:   cir.return
 
-// LLVM:      define void @f5(%union.U4 %[[ARG:.*]])
+// LLVM:      define{{.*}} void @f5(%union.U4 %[[ARG:.*]])
 // LLVM-NEXT:   %[[U:.*]] = alloca %union.U4, i64 1, align 4
 // LLVM-NEXT:   store %union.U4 %[[ARG]], ptr %[[U]], align 4
 // LLVM-NEXT:   %[[C_PTR:.*]] = getelementptr i8, ptr %[[U]], i32 0
@@ -222,7 +222,7 @@ void f5(union U4 u) {
 // LLVM-NEXT:   store i8 65, ptr %[[ELEM_PTR]], align 4
 // LLVM-NEXT:   ret void
 
-// OGCG:      define dso_local void @f5(i64 %[[ARG:.*]])
+// OGCG:      define{{.*}} void @f5(i64 %[[ARG:.*]])
 // OGCG-NEXT: entry:
 // OGCG-NEXT:   %[[U:.*]] = alloca %union.U4, align 4
 // OGCG-NEXT:   store i64 %[[ARG]], ptr %[[U]], align 4
diff --git a/clang/test/CIR/CodeGen/vector-ext.cpp b/clang/test/CIR/CodeGen/vector-ext.cpp
index fe4919ec0478..2ee42187a6e9 100644
--- a/clang/test/CIR/CodeGen/vector-ext.cpp
+++ b/clang/test/CIR/CodeGen/vector-ext.cpp
@@ -1161,3 +1161,18 @@ void foo20() {
 // OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
 // OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
 // OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+
+void foo21() {
+  vi4 a;
+  unsigned long size = __builtin_vectorelements(a);
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !u64i, !cir.ptr<!u64i>, ["size", init]
+// CIR: %[[SIZE:.*]] = cir.const #cir.int<4> : !u64i
+// CIR: cir.store align(8) %[[SIZE]], %[[INIT]] : !u64i, !cir.ptr<!u64i>
+
+// LLVM: %[[SIZE:.*]] = alloca i64, i64 1, align 8
+// LLVM: store i64 4, ptr %[[SIZE]], align 8
+
+// OGCG: %[[SIZE:.*]] = alloca i64, align 8
+// OGCG: store i64 4, ptr %[[SIZE]], align 8
diff --git a/clang/test/CIR/CodeGen/vector.cpp b/clang/test/CIR/CodeGen/vector.cpp
index d0c5b83cd5b0..18fa90bd2cb3 100644
--- a/clang/test/CIR/CodeGen/vector.cpp
+++ b/clang/test/CIR/CodeGen/vector.cpp
@@ -1203,3 +1203,18 @@ void foo23() {
 // OGCG: %[[TMP_A:.*]] = load <4 x i32>, ptr %[[VEC_A]], align 16
 // OGCG: %[[TMP_B:.*]] = load <4 x i32>, ptr %[[VEC_B]], align 16
 // OGCG: %[[SHUF:.*]] = shufflevector <4 x i32> %[[TMP_A]], <4 x i32> %[[TMP_B]], <4 x i32> <i32 poison, i32 1, i32 poison, i32 1>
+
+void foo24() {
+  vi4 a;
+  unsigned long size = __builtin_vectorelements(a);
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !u64i, !cir.ptr<!u64i>, ["size", init]
+// CIR: %[[SIZE:.*]] = cir.const #cir.int<4> : !u64i
+// CIR: cir.store align(8) %[[SIZE]], %[[INIT]] : !u64i, !cir.ptr<!u64i>
+
+// LLVM: %[[SIZE:.*]] = alloca i64, i64 1, align 8
+// LLVM: store i64 4, ptr %[[SIZE]], align 8
+
+// OGCG: %[[SIZE:.*]] = alloca i64, align 8
+// OGCG: store i64 4, ptr %[[SIZE]], align 8
diff --git a/clang/test/CIR/CodeGenOpenACC/cache.c b/clang/test/CIR/CodeGenOpenACC/cache.c
new file mode 100644
index 000000000000..76651c132f73
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/cache.c
@@ -0,0 +1,132 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_cache() {
+  // CHECK: cir.func{{.*}} @acc_cache() {
+
+  int iArr[10];
+  // CHECK-NEXT: %[[IARR:.*]] = cir.alloca !cir.array<!s32i x 10>, !cir.ptr<!cir.array<!s32i x 10>>, ["iArr"]
+  float fArr[10];
+  // CHECK-NEXT: %[[FARR:.*]] = cir.alloca !cir.array<!cir.float x 10>, !cir.ptr<!cir.array<!cir.float x 10>>, ["fArr"]
+
+#pragma acc cache(iArr[1], fArr[1:5])
+  // This does nothing, as it is not in a loop.
+
+#pragma acc parallel
+  {
+#pragma acc cache(iArr[1], fArr[1:5])
+  // This does nothing, as it is not in a loop.
+  }
+  // CHECK-NEXT: acc.parallel {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+    for(int j = 0; j < 5; ++j) {
+#pragma acc cache(iArr[1], fArr[1:5])
+    }
+  }
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+#pragma acc cache(iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+
+#pragma acc parallel loop
+  for(int i = 0; i < 5; ++i) {
+#pragma acc cache(iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: acc.parallel combined(loop) {
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop combined(parallel) cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop
+  for(int i = 0; i < 5; ++i) {
+    int localArr[5];
+    // The first term here isn't lowered, because it references data inside of the 'loop'.
+#pragma acc cache(localArr[i], iArr[1], fArr[1:5])
+  }
+  // CHECK-NEXT: acc.parallel combined(loop) {
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE1:.*]] = acc.cache varPtr(%[[IARR]] : !cir.ptr<!cir.array<!s32i x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 10>> {name = "iArr[1]", structured = false}
+  //
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[FIVE:.*]] = cir.const #cir.int<5>
+  // CHECK-NEXT: %[[FIVE_CAST:.*]] = builtin.unrealized_conversion_cast %[[FIVE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0 : i64
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1 : i64
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[FIVE_CAST]] : si32) stride(%[[ONE_CONST]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[CACHE2:.*]] = acc.cache varPtr(%[[FARR]] : !cir.ptr<!cir.array<!cir.float x 10>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 10>> {name = "fArr[1:5]", structured = false}
+  //
+  // CHECK-NEXT: acc.loop combined(parallel) cache(%[[CACHE1]], %[[CACHE2]] : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.array<!cir.float x 10>>) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } attributes {independent = [#acc.device_type<none>]}
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/combined-copy.c b/clang/test/CIR/CodeGenOpenACC/combined-copy.c
index 72471d4ec787..b4573e66f24a 100644
--- a/clang/test/CIR/CodeGenOpenACC/combined-copy.c
+++ b/clang/test/CIR/CodeGenOpenACC/combined-copy.c
@@ -2,7 +2,7 @@
 
 int global;
 void acc_compute(int parmVar) {
-  // CHECK: cir.func @acc_compute(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_compute(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
   int localVar1;
   short localVar2;
@@ -77,29 +77,29 @@ void acc_compute(int parmVar) {
   // these do nothing to the IR.
 #pragma acc parallel loop copy(alwaysin: localVar1) copy(alwaysout: localVar2) copy(always: localVar3)
   for(int i = 0; i < 5; ++i);
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
-  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) -> !cir.ptr<!s16i> {dataClause = #acc<data_clause acc_copy>, name = "localVar2"} loc
-  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copy>, name = "localVar3"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar1"} loc
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) -> !cir.ptr<!s16i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar2"} loc
+  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar3"} loc
   // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]], %[[COPYIN2]], %[[COPYIN3]] : !cir.ptr<!s32i>, !cir.ptr<!s16i>, !cir.ptr<!cir.float>) {
   // CHECK-NEXT: acc.loop combined(parallel) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) to varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copy>, name = "localVar3"} loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s16i>) to varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar2"} loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) to varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>,  name = "localVar3"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s16i>) to varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar2"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar1"} loc
 
 #pragma acc serial loop copy(always, alwaysin, alwaysout: localVar1)
   for(int i = 0; i < 5; ++i);
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"} loc
   // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
   // CHECK-NEXT: acc.loop combined(serial) {
   // CHECK: acc.yield
   // CHECK-NEXT: }
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"} loc
 
   short *localPointer;
   float localArray[5];
@@ -773,7 +773,7 @@ typedef struct StructTy {
 } Struct ;
 
 void acc_compute_members() {
-  // CHECK: cir.func @acc_compute_members()
+  // CHECK: cir.func{{.*}} @acc_compute_members()
   Struct localStruct;
   // CHECK-NEXT: %[[LOCALSTRUCT:.*]] = cir.alloca !rec_StructTy, !cir.ptr<!rec_StructTy>, ["localStruct"]
 
@@ -1082,7 +1082,7 @@ typedef struct OuterTy {
 } Outer;
 
 void copy_member_of_array_element_member() {
-  // CHECK: cir.func @copy_member_of_array_element_member() {
+  // CHECK: cir.func{{.*}} @copy_member_of_array_element_member() {
   Outer outer;
   // CHECK-NEXT: %[[OUTER:.*]] = cir.alloca !rec_OuterTy, !cir.ptr<!rec_OuterTy>, ["outer"]
 
@@ -1102,3 +1102,60 @@ void copy_member_of_array_element_member() {
   // CHECK-NEXT:  } loc
   // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[GETB]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "outer.inner[2].b"}
 }
+
+void modifier_list() {
+  // CHECK: cir.func{{.*}} @modifier_list() {
+  int localVar;
+  // CHECK-NEXT: %[[LOCALVAR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["localVar"]
+
+#pragma acc parallel loop copy(always:localVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar"}
+#pragma acc serial loop copy(alwaysin:localVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar"}
+#pragma acc kernels loop copy(alwaysout:localVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar"}
+#pragma acc parallel loop copy(capture:localVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "localVar"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "localVar"}
+#pragma acc serial loop copy(capture, always, alwaysin, alwaysout:localVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar"}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/combined-copyin-copyout-create.c b/clang/test/CIR/CodeGenOpenACC/combined-copyin-copyout-create.c
new file mode 100644
index 000000000000..d6179c012ee9
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/combined-copyin-copyout-create.c
@@ -0,0 +1,160 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_combined(int parmVar) {
+  // CHECK: cir.func{{.*}} @acc_combined(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+
+  int localVar1;
+  // CHECK-NEXT: %[[LV1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["localVar1"]
+  float localVar2;
+  // CHECK-NEXT: %[[LV2:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["localVar2"]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]]
+#pragma acc parallel loop copyin(parmVar) copyout(localVar1) create(localVar2)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "localVar2"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[COPYIN1]], %[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, name = "localVar2"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+
+#pragma acc serial loop copyin(parmVar, localVar1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+
+#pragma acc kernels loop copyout(parmVar, localVar1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar"}
+
+#pragma acc parallel loop create (parmVar, localVar2)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "localVar2"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "parmVar"}
+
+#pragma acc serial loop copyin(capture: parmVar) copyin(always: localVar1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc kernels loop copyout(capture: parmVar) copyout(always: localVar1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc parallel loop create(capture: parmVar)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc serial loop copyin(capture, always: parmVar, localVar1)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+
+#pragma acc kernels loop copyin(readonly, always, alwaysin, capture: parmVar, localVar1, localVar2)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.kernels combined(loop) dataOperands(%[[COPYIN1]], %[[COPYIN2]], %[[COPYIN3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(kernels) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+
+#pragma acc parallel loop copyout(zero, always, alwaysout, capture: parmVar, localVar1, localVar2)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE3:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.parallel combined(loop) dataOperands(%[[CREATE1]], %[[CREATE2]], %[[CREATE3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(parallel) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE3]] : !cir.ptr<!cir.float>) to varPtr(%[[LV2]] : !cir.ptr<!cir.float>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "parmVar"}
+
+#pragma acc serial loop create(zero, capture: parmVar, localVar1, localVar2)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE3:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.serial combined(loop) dataOperands(%[[CREATE1]], %[[CREATE2]], %[[CREATE3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(serial) {
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/combined.cpp b/clang/test/CIR/CodeGenOpenACC/combined.cpp
index 5b83a9cb9189..b8140335f7c2 100644
--- a/clang/test/CIR/CodeGenOpenACC/combined.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/combined.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 extern "C" void acc_combined(int N, int cond) {
-  // CHECK: cir.func @acc_combined(%[[ARG_N:.*]]: !s32i loc{{.*}}, %[[ARG_COND:.*]]: !s32i loc{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_combined(%[[ARG_N:.*]]: !s32i loc{{.*}}, %[[ARG_COND:.*]]: !s32i loc{{.*}}) {
   // CHECK-NEXT: %[[ALLOCA_N:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["N", init]
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG_N]], %[[ALLOCA_N]] : !s32i, !cir.ptr<!s32i>
@@ -1012,7 +1012,7 @@ extern "C" void acc_combined(int N, int cond) {
   // CHECK-NEXT: } loc
 }
 extern "C" void acc_combined_data_clauses(int *arg1, int *arg2) {
-  // CHECK: cir.func @acc_combined_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_combined_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
   // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
   // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
   // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-copy.c b/clang/test/CIR/CodeGenOpenACC/compute-copy.c
index 888bad29caa7..d7676d6d30c1 100644
--- a/clang/test/CIR/CodeGenOpenACC/compute-copy.c
+++ b/clang/test/CIR/CodeGenOpenACC/compute-copy.c
@@ -2,7 +2,7 @@
 
 int global;
 void acc_compute(int parmVar) {
-  // CHECK: cir.func @acc_compute(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_compute(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
   int localVar1;
   short localVar2;
@@ -69,23 +69,23 @@ void acc_compute(int parmVar) {
   // these do nothing to the IR.
 #pragma acc parallel copy(alwaysin: localVar1) copy(alwaysout: localVar2) copy(always: localVar3)
   ;
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
-  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) -> !cir.ptr<!s16i> {dataClause = #acc<data_clause acc_copy>, name = "localVar2"} loc
-  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copy>, name = "localVar3"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar1"} loc
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) -> !cir.ptr<!s16i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar2"} loc
+  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar3"} loc
   // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]], %[[COPYIN2]], %[[COPYIN3]] : !cir.ptr<!s32i>, !cir.ptr<!s16i>, !cir.ptr<!cir.float>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) to varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copy>, name = "localVar3"} loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s16i>) to varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar2"} loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) to varPtr(%[[LOCAL3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>,  name = "localVar3"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s16i>) to varPtr(%[[LOCAL2]] : !cir.ptr<!s16i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar2"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar1"} loc
 
 #pragma acc serial copy(always, alwaysin, alwaysout: localVar1)
   ;
-  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"} loc
   // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
   // CHECK-NEXT: acc.yield
   // CHECK-NEXT: } loc
-  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"} loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCAL1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"} loc
 
   short *localPointer;
   float localArray[5];
@@ -646,7 +646,7 @@ typedef struct StructTy {
 } Struct ;
 
 void acc_compute_members() {
-  // CHECK: cir.func @acc_compute_members()
+  // CHECK: cir.func{{.*}} @acc_compute_members()
   Struct localStruct;
   // CHECK-NEXT: %[[LOCALSTRUCT:.*]] = cir.alloca !rec_StructTy, !cir.ptr<!rec_StructTy>, ["localStruct"]
 
@@ -897,3 +897,46 @@ void acc_compute_members() {
   // CHECK-NEXT:  acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) bounds(%[[BOUNDS1]], %[[BOUNDS2]]) to varPtr(%[[GETPTRPTRMEMBER]] : !cir.ptr<!cir.ptr<!cir.ptr<!cir.double>>>) {dataClause = #acc<data_clause acc_copy>, name = "localStruct.ptrPtrMember[1:3][1:1]"}
 
 }
+
+void modifier_list() {
+  // CHECK: cir.func{{.*}} @modifier_list() {
+  int localVar;
+  // CHECK-NEXT: %[[LOCALVAR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["localVar"]
+
+#pragma acc parallel copy(always:localVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "localVar"}
+
+#pragma acc serial copy(alwaysin:localVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysin>, name = "localVar"}
+#pragma acc kernels copy(alwaysout:localVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier alwaysout>, name = "localVar"}
+#pragma acc parallel copy(capture:localVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "localVar"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "localVar"}
+#pragma acc serial copy(capture, always, alwaysin, alwaysout:localVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[LOCALVAR]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar"}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-copyin-copyout-create.c b/clang/test/CIR/CodeGenOpenACC/compute-copyin-copyout-create.c
new file mode 100644
index 000000000000..2180a3370939
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/compute-copyin-copyout-create.c
@@ -0,0 +1,128 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_compute(int parmVar) {
+  // CHECK: cir.func{{.*}} @acc_compute(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+
+  int localVar1;
+  // CHECK-NEXT: %[[LV1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["localVar1"]
+  float localVar2;
+  // CHECK-NEXT: %[[LV2:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["localVar2"]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]]
+
+#pragma acc parallel copyin(parmVar) copyout(localVar1) create(localVar2)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "localVar2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[COPYIN1]], %[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, name = "localVar2"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+
+#pragma acc serial copyin(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+
+#pragma acc kernels copyout(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar"}
+
+#pragma acc parallel create (parmVar, localVar2)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "localVar2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "parmVar"}
+
+#pragma acc serial copyin(capture: parmVar) copyin(always: localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc kernels copyout(capture: parmVar) copyout(always: localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc parallel create(capture: parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc serial copyin(capture, always: parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+
+#pragma acc kernels copyin(readonly, always, alwaysin, capture: parmVar, localVar1, localVar2)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[COPYIN3:.*]] = acc.copyin varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.kernels dataOperands(%[[COPYIN1]], %[[COPYIN2]], %[[COPYIN3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+
+#pragma acc parallel copyout(zero, always, alwaysout, capture: parmVar, localVar1, localVar2)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE3:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.parallel dataOperands(%[[CREATE1]], %[[CREATE2]], %[[CREATE3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE3]] : !cir.ptr<!cir.float>) to varPtr(%[[LV2]] : !cir.ptr<!cir.float>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always,zero,capture>, name = "parmVar"}
+
+#pragma acc serial create(zero, capture: parmVar, localVar1, localVar2)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: %[[CREATE3:.*]] = acc.create varPtr(%[[LV2]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.serial dataOperands(%[[CREATE1]], %[[CREATE2]], %[[CREATE3]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>, !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE3]] : !cir.ptr<!cir.float>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar2"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/data-copy-copyin-copyout-create.c b/clang/test/CIR/CodeGenOpenACC/data-copy-copyin-copyout-create.c
new file mode 100644
index 000000000000..068ca4a63e33
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/data-copy-copyin-copyout-create.c
@@ -0,0 +1,218 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_data(int parmVar) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  int localVar1;
+  // CHECK-NEXT: %[[LV1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["localVar1"]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]]
+
+#pragma acc data copy(parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+#pragma acc data copy(parmVar) copy(localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+#pragma acc data copy(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, name = "parmVar"}
+
+#pragma acc data copy(capture, always, alwaysin, alwaysout:  parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always,capture>, name = "parmVar"}
+
+#pragma acc data copy(capture: parmVar) copy(always, alwaysin, alwaysout:  parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier always>, name = "parmVar"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copy>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc data copyin(parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+#pragma acc data copyin(parmVar) copyin(localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+#pragma acc data copyin(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, name = "parmVar"}
+
+#pragma acc data copyin(capture, always, alwaysin, readonly:  parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly,capture>, name = "parmVar"}
+
+#pragma acc data copyin(capture: parmVar) copyin(readonly, always, alwaysin:  parmVar)
+  ;
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[COPYIN2:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier always,readonly>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[COPYIN1]], %[[COPYIN2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier always,readonly>, name = "parmVar"}
+  // CHECK-NEXT: acc.delete accPtr(%[[COPYIN1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_copyin>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc data copyout(parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar"}
+
+#pragma acc data copyout(parmVar) copyout(localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar"}
+
+#pragma acc data copyout(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[LV1]] : !cir.ptr<!s32i>) {name = "localVar1"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar"}
+
+#pragma acc data copyout(capture, zero, alwaysout:  parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout,capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout,capture>, name = "parmVar"}
+
+#pragma acc data copyout(capture: parmVar) copyout(always, alwaysout:  parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier always>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier always>, name = "parmVar"}
+  // CHECK-NEXT: acc.copyout accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc data create(parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "parmVar"}
+
+#pragma acc data create(parmVar) create(localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "parmVar"}
+
+#pragma acc data create(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, name = "parmVar"}
+
+
+#pragma acc data create(capture, zero:  parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero,capture>, name = "parmVar"}
+
+#pragma acc data create(capture: parmVar) create(zero:  parmVar)
+  ;
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+  // CHECK-NEXT: %[[CREATE2:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[CREATE1]], %[[CREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier zero>, name = "parmVar"}
+  // CHECK-NEXT: acc.delete accPtr(%[[CREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_create>, modifiers = #acc<data_clause_modifier capture>, name = "parmVar"}
+
+#pragma acc data no_create(parmVar)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: acc.data dataOperands(%[[NOCREATE1]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_no_create>, name = "parmVar"}
+
+#pragma acc data no_create(parmVar) no_create(localVar1)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_no_create>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_no_create>, name = "parmVar"}
+
+#pragma acc data no_create(parmVar, localVar1)
+  ;
+  // CHECK-NEXT: %[[NOCREATE1:.*]] = acc.nocreate varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar"}
+  // CHECK-NEXT: %[[NOCREATE2:.*]] = acc.nocreate varPtr(%[[LV1]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "localVar1"}
+  // CHECK-NEXT: acc.data dataOperands(%[[NOCREATE1]], %[[NOCREATE2]] : !cir.ptr<!s32i>, !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE2]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_no_create>, name = "localVar1"}
+  // CHECK-NEXT: acc.delete accPtr(%[[NOCREATE1]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_no_create>, name = "parmVar"}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/data.c b/clang/test/CIR/CodeGenOpenACC/data.c
index 948119f66e93..1f6a76ce1ea7 100644
--- a/clang/test/CIR/CodeGenOpenACC/data.c
+++ b/clang/test/CIR/CodeGenOpenACC/data.c
@@ -1,8 +1,11 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_data(int cond) {
-  // CHECK: cir.func @acc_data(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
+
+  int *ptr;
+  // CHECK-NEXT: %[[PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptr"]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 
 #pragma acc data default(none)
@@ -221,5 +224,50 @@ void acc_data(int cond) {
   // CHECK-NEXT: acc.terminator
   // CHECK-NEXT: attributes {defaultAttr = #acc<defaultvalue none>}
 
+#pragma acc data deviceptr(ptr)
+  {}
+  // CHECK-NEXT: %[[DEV_PTR:.*]] = acc.deviceptr varPtr(%[[PTR]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptr"}
+  // CHECK-NEXT: acc.data dataOperands(%[[DEV_PTR]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+#pragma acc data deviceptr(ptr) device_type(radeon) async
+  {}
+  // CHECK-NEXT: %[[DEV_PTR:.*]] = acc.deviceptr varPtr(%[[PTR]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptr"}
+  // CHECK-NEXT: acc.data async([#acc.device_type<radeon>]) dataOperands(%[[DEV_PTR]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+
+#pragma acc data present(cond)
+  {}
+  // CHECK-NEXT: %[[PRESENT:.*]] = acc.present varPtr(%[[COND]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "cond"}
+  // CHECK-NEXT: acc.data dataOperands(%[[PRESENT]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_present>, name = "cond"}
+
+#pragma acc data present(cond) device_type(radeon) async
+  {}
+  // CHECK-NEXT: %[[PRESENT:.*]] = acc.present varPtr(%[[COND]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) -> !cir.ptr<!s32i> {name = "cond"}
+  // CHECK-NEXT: acc.data async([#acc.device_type<radeon>]) dataOperands(%[[PRESENT]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.delete accPtr(%[[PRESENT]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_present>, name = "cond"}
+
+#pragma acc data attach(ptr)
+  {}
+  // CHECK-NEXT: %[[ATTACH:.*]] = acc.attach varPtr(%[[PTR]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptr"}
+  // CHECK-NEXT: acc.data dataOperands(%[[ATTACH]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_attach>, name = "ptr"}
+
+#pragma acc data attach(ptr) device_type(radeon) async
+  {}
+  // CHECK-NEXT: %[[ATTACH:.*]] = acc.attach varPtr(%[[PTR]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptr"}
+  // CHECK-NEXT: acc.data async([#acc.device_type<radeon>]) dataOperands(%[[ATTACH]] : !cir.ptr<!cir.ptr<!s32i>>) {
+  // CHECK-NEXT: acc.terminator
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.detach accPtr(%[[ATTACH]] : !cir.ptr<!cir.ptr<!s32i>>) async([#acc.device_type<radeon>]) {dataClause = #acc<data_clause acc_attach>, name = "ptr"}
+
   // CHECK-NEXT: cir.return
 }
diff --git a/clang/test/CIR/CodeGenOpenACC/enter-data.c b/clang/test/CIR/CodeGenOpenACC/enter-data.c
new file mode 100644
index 000000000000..1785fba1a105
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/enter-data.c
@@ -0,0 +1,125 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+void acc_data(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc enter data copyin(parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar) async
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data copyin(readonly, alwaysin: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async(parmVar) copyin(readonly, alwaysin: parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier readonly,alwaysin>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar) async
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data create(zero: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {modifiers = #acc<data_clause_modifier zero>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data attach(ptrParmVar)
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data attach(ptrParmVar) async
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data attach(ptrParmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ATTACH1:.*]] = acc.attach varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data async(%[[PARM_CAST]] : si32) dataOperands(%[[ATTACH1]] : !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc enter data if (parmVar == 1) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data async if (parmVar == 1) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) async dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data if (parmVar == 1) async(parmVar) copyin(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[COPYIN1:.*]] = acc.copyin varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data if(%[[CMP_CAST]]) async(%[[PARM_CAST]] : si32) dataOperands(%[[COPYIN1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait create(parmVar)
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(1) create(parmVar)
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait(%[[ONE_CAST]] : si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(parmVar, 1, 2) create(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait(%[[PARM_CAST]], %[[ONE_CAST]], %[[TWO_CAST]] : si32, si32, si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+#pragma acc enter data wait(devnum: parmVar: 1, 2) create(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[CREATE1:.*]] = acc.create varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.enter_data wait_devnum(%[[PARM_CAST]] : si32) wait(%[[ONE_CAST]], %[[TWO_CAST]] : si32, si32) dataOperands(%[[CREATE1]] : !cir.ptr<!s32i>)
+
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/exit-data.c b/clang/test/CIR/CodeGenOpenACC/exit-data.c
new file mode 100644
index 000000000000..ff987d20d5b6
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/exit-data.c
@@ -0,0 +1,134 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+void acc_data(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_data(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc exit data copyout(parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data copyout(zero, alwaysout: parmVar) async
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data async copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data finalize copyout(zero, alwaysout: parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data async(parmVar) copyout(zero, alwaysout: parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, modifiers = #acc<data_clause_modifier zero,alwaysout>,  name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {modifiers = #acc<data_clause_modifier zero,alwaysout>, name = "parmVar", structured = false}
+
+#pragma acc exit data delete(parmVar) finalize
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data delete(parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>)  async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) {name = "parmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar) async
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) async {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data detach(ptrParmVar) async(parmVar) finalize
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_detach>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) attributes {finalize}
+  // CHECK-NEXT: acc.detach accPtr(%[[GDP]] : !cir.ptr<!cir.ptr<!s32i>>) async(%[[PARM_CAST]] : si32) {name = "ptrParmVar", structured = false}
+
+#pragma acc exit data if (parmVar == 1) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data async if (parmVar == 1) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) async dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data if (parmVar == 1) async(parmVar) copyout(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_copyout>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data if(%[[CMP_CAST]]) async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.copyout accPtr(%[[GDP]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait delete(parmVar)
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(1) delete(parmVar)
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait(%[[ONE_CAST]] : si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(parmVar, 1, 2) delete(parmVar) finalize
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait(%[[PARM_CAST]], %[[ONE_CAST]], %[[TWO_CAST]] : si32, si32, si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>) attributes {finalize}
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc exit data wait(devnum: parmVar: 1, 2) delete(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: %[[GDP:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_delete>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.exit_data wait_devnum(%[[PARM_CAST]] : si32) wait(%[[ONE_CAST]], %[[TWO_CAST]] : si32, si32) dataOperands(%[[GDP]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.delete accPtr(%[[GDP]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/host_data.c b/clang/test/CIR/CodeGenOpenACC/host_data.c
index 4c3f7dd092a2..aeaf3d2f047b 100644
--- a/clang/test/CIR/CodeGenOpenACC/host_data.c
+++ b/clang/test/CIR/CodeGenOpenACC/host_data.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_host_data(int cond, int var1, int var2) {
-  // CHECK: cir.func @acc_host_data(%[[ARG_COND:.*]]: !s32i {{.*}}, %[[ARG_V1:.*]]: !s32i {{.*}}, %[[ARG_V2:.*]]: !s32i {{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_host_data(%[[ARG_COND:.*]]: !s32i {{.*}}, %[[ARG_V1:.*]]: !s32i {{.*}}, %[[ARG_V2:.*]]: !s32i {{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: %[[V1:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["var1", init]
   // CHECK-NEXT: %[[V2:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["var2", init]
diff --git a/clang/test/CIR/CodeGenOpenACC/init.c b/clang/test/CIR/CodeGenOpenACC/init.c
index 54f686dbe8eb..177e5a6ea211 100644
--- a/clang/test/CIR/CodeGenOpenACC/init.c
+++ b/clang/test/CIR/CodeGenOpenACC/init.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_init(int cond) {
-  // CHECK: cir.func @acc_init(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_init(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc init
diff --git a/clang/test/CIR/CodeGenOpenACC/kernels.c b/clang/test/CIR/CodeGenOpenACC/kernels.c
index d0c6f1134c8d..9b10b7489e81 100644
--- a/clang/test/CIR/CodeGenOpenACC/kernels.c
+++ b/clang/test/CIR/CodeGenOpenACC/kernels.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_kernels(int cond) {
-  // CHECK: cir.func @acc_kernels(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_kernels(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc kernels
@@ -418,7 +418,7 @@ void acc_kernels(int cond) {
 }
 
 void acc_kernels_data_clauses(int *arg1, int *arg2) {
-  // CHECK: cir.func @acc_kernels_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_kernels_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
   // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
   // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
   // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
diff --git a/clang/test/CIR/CodeGenOpenACC/loop.cpp b/clang/test/CIR/CodeGenOpenACC/loop.cpp
index c0bf11e35395..d8707ba78fb5 100644
--- a/clang/test/CIR/CodeGenOpenACC/loop.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/loop.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 extern "C" void acc_loop(int *A, int *B, int *C, int N) {
-  // CHECK: cir.func @acc_loop(%[[ARG_A:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_B:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_C:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_N:.*]]: !s32i loc{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_loop(%[[ARG_A:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_B:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_C:.*]]: !cir.ptr<!s32i> loc{{.*}}, %[[ARG_N:.*]]: !s32i loc{{.*}}) {
   // CHECK-NEXT: %[[ALLOCA_A:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["A", init]
   // CHECK-NEXT: %[[ALLOCA_B:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["B", init]
   // CHECK-NEXT: %[[ALLOCA_C:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["C", init]
diff --git a/clang/test/CIR/CodeGenOpenACC/parallel.c b/clang/test/CIR/CodeGenOpenACC/parallel.c
index 0127613233ec..5db174fb6549 100644
--- a/clang/test/CIR/CodeGenOpenACC/parallel.c
+++ b/clang/test/CIR/CodeGenOpenACC/parallel.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_parallel(int cond) {
-  // CHECK: cir.func @acc_parallel(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_parallel(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc parallel
@@ -445,7 +445,7 @@ void acc_parallel(int cond) {
 }
 
 void acc_parallel_data_clauses(int *arg1, int *arg2) {
-  // CHECK: cir.func @acc_parallel_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_parallel_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
   // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
   // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
   // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
diff --git a/clang/test/CIR/CodeGenOpenACC/serial.c b/clang/test/CIR/CodeGenOpenACC/serial.c
index 1c9695b34833..9e3359141838 100644
--- a/clang/test/CIR/CodeGenOpenACC/serial.c
+++ b/clang/test/CIR/CodeGenOpenACC/serial.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_serial(int cond) {
-  // CHECK: cir.func @acc_serial(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_serial(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc serial
@@ -268,7 +268,7 @@ void acc_serial(int cond) {
 }
 
 void acc_serial_data_clauses(int *arg1, int *arg2) {
-  // CHECK: cir.func @acc_serial_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_serial_data_clauses(%[[ARG1_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}, %[[ARG2_PARAM:.*]]: !cir.ptr<!s32i>{{.*}}) {
   // CHECK-NEXT: %[[ARG1:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg1", init]
   // CHECK-NEXT: %[[ARG2:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arg2", init]
   // CHECK-NEXT: cir.store %[[ARG1_PARAM]], %[[ARG1]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
diff --git a/clang/test/CIR/CodeGenOpenACC/set.c b/clang/test/CIR/CodeGenOpenACC/set.c
index ced581680c03..0b87f4260377 100644
--- a/clang/test/CIR/CodeGenOpenACC/set.c
+++ b/clang/test/CIR/CodeGenOpenACC/set.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_set(int cond) {
-  // CHECK: cir.func @acc_set(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_set(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 
diff --git a/clang/test/CIR/CodeGenOpenACC/shutdown.c b/clang/test/CIR/CodeGenOpenACC/shutdown.c
index e8ab6bd75d0e..52db382df217 100644
--- a/clang/test/CIR/CodeGenOpenACC/shutdown.c
+++ b/clang/test/CIR/CodeGenOpenACC/shutdown.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_shutdown(int cond) {
-  // CHECK: cir.func @acc_shutdown(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_shutdown(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 #pragma acc shutdown
diff --git a/clang/test/CIR/CodeGenOpenACC/update.c b/clang/test/CIR/CodeGenOpenACC/update.c
new file mode 100644
index 000000000000..2b29504e6ca2
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/update.c
@@ -0,0 +1,178 @@
+// RUN: %clang_cc1 -fopenacc -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir %s -o - | FileCheck %s
+
+void acc_update(int parmVar, int *ptrParmVar) {
+  // CHECK: cir.func{{.*}} @acc_update(%[[ARG:.*]]: !s32i{{.*}}, %[[PTRARG:.*]]: !cir.ptr<!s32i>{{.*}}) {
+  // CHECK-NEXT: %[[PARM:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["parmVar", init]
+  // CHECK-NEXT: %[[PTRPARM:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["ptrParmVar", init]
+  // CHECK-NEXT: cir.store %[[ARG]], %[[PARM]] : !s32i, !cir.ptr<!s32i>
+  // CHECK-NEXT: cir.store %[[PTRARG]], %[[PTRPARM]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+
+#pragma acc update device(parmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]] : !cir.ptr<!s32i>)
+
+#pragma acc update device(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc update device(parmVar) device(ptrParmVar)
+  // CHECK-NEXT: %[[UPD_DEV1:.*]] = acc.update_device varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[UPD_DEV1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+
+#pragma acc update host(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update host(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_host>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update host(parmVar) host(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_host>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_host>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar, ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) self(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[GDP2:.*]] = acc.getdeviceptr varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[GDP2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP2]] : !cir.ptr<!cir.ptr<!s32i>>) to varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) {dataClause = #acc<data_clause acc_update_self>, name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device(ptrParmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[UPD_DEV2:.*]] = acc.update_device varPtr(%[[PTRPARM]] : !cir.ptr<!cir.ptr<!s32i>>) -> !cir.ptr<!cir.ptr<!s32i>> {name = "ptrParmVar", structured = false}
+  // CHECK-NEXT: acc.update dataOperands(%[[GDP1]], %[[UPD_DEV2]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) if (parmVar == 1)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: acc.update if(%[[CMP_CAST]]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+#pragma acc update self(parmVar) if (parmVar == 1) if_present
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[CMP:.*]] = cir.cmp(eq, %[[PARM_LOAD]], %[[ONE_CONST]])
+  // CHECK-NEXT: %[[CMP_CAST:.*]] = builtin.unrealized_conversion_cast %[[CMP]]
+  // CHECK-NEXT: acc.update if(%[[CMP_CAST]]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>) attributes {ifPresent}
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update wait([#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32}) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) wait(parmVar) device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32}) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(parmVar)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(parmVar, 1, 2)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2>
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: acc.update wait({%[[PARM_CAST]] : si32, %[[ONE_CAST]] : si32, %[[TWO_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) wait(devnum:parmVar: 1, 2)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = cir.const #cir.int<1>
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE_CONST]]
+  // CHECK-NEXT: %[[TWO_CONST:.*]] = cir.const #cir.int<2>
+  // CHECK-NEXT: %[[TWO_CAST:.*]] = builtin.unrealized_conversion_cast %[[TWO_CONST]]
+  // CHECK-NEXT: acc.update wait({devnum: %[[PARM_CAST]] : si32, %[[ONE_CAST]] : si32, %[[TWO_CAST]] : si32} [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async device_type(nvidia)
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) async
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async([#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async([#acc.device_type<radeon>]) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) async(parmVar) device_type(nvidia)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+
+#pragma acc update self(parmVar) device_type(radeon) async(parmVar)
+  // CHECK-NEXT: %[[PARM_LOAD:.*]] = cir.load{{.*}} %[[PARM]]
+  // CHECK-NEXT: %[[PARM_CAST:.*]] = builtin.unrealized_conversion_cast %[[PARM_LOAD]]
+  // CHECK-NEXT: %[[GDP1:.*]] = acc.getdeviceptr varPtr(%[[PARM]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) -> !cir.ptr<!s32i> {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+  // CHECK-NEXT: acc.update async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) dataOperands(%[[GDP1]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.update_host accPtr(%[[GDP1]] : !cir.ptr<!s32i>) async(%[[PARM_CAST]] : si32 [#acc.device_type<radeon>]) to varPtr(%[[PARM]] : !cir.ptr<!s32i>) {dataClause = #acc<data_clause acc_update_self>, name = "parmVar", structured = false}
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/wait.c b/clang/test/CIR/CodeGenOpenACC/wait.c
index ec2ab6e9446c..aeda8b955a6d 100644
--- a/clang/test/CIR/CodeGenOpenACC/wait.c
+++ b/clang/test/CIR/CodeGenOpenACC/wait.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fopenacc -emit-cir -fclangir %s -o - | FileCheck %s
 
 void acc_wait(int cond) {
-  // CHECK: cir.func @acc_wait(%[[ARG:.*]]: !s32i{{.*}}) {
+  // CHECK: cir.func{{.*}} @acc_wait(%[[ARG:.*]]: !s32i{{.*}}) {
   // CHECK-NEXT: %[[COND:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["cond", init]
   // CHECK-NEXT: cir.store %[[ARG]], %[[COND]] : !s32i, !cir.ptr<!s32i>
 
diff --git a/clang/test/CIR/IR/array.cir b/clang/test/CIR/IR/array.cir
index 4930fc493c5a..bba536062d74 100644
--- a/clang/test/CIR/IR/array.cir
+++ b/clang/test/CIR/IR/array.cir
@@ -33,7 +33,7 @@ cir.func @func() {
   cir.return
 }
 
-// CHECK: cir.func @func() {
+// CHECK: cir.func{{.*}} @func() {
 // CHECK:   %0 = cir.alloca !cir.array<!s32i x 10>, !cir.ptr<!cir.array<!s32i x 10>>, ["l"] {alignment = 4 : i64}
 // CHECK:   cir.return
 // CHECK: }
@@ -44,7 +44,7 @@ cir.func @func2(%arg0: !cir.ptr<!s32i>) {
   cir.return
 }
 
-// CHECK: cir.func @func2(%arg0: !cir.ptr<!s32i>) {
+// CHECK: cir.func{{.*}} @func2(%arg0: !cir.ptr<!s32i>) {
 // CHECK:   %0 = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["p", init] {alignment = 8 : i64}
 // CHECK:   cir.store %arg0, %0 : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
 // CHECK:   cir.return
@@ -56,7 +56,7 @@ cir.func @func3(%arg0: !cir.ptr<!cir.array<!s32i x 10>>) {
   cir.return
 }
 
-// CHECK: cir.func @func3(%arg0: !cir.ptr<!cir.array<!s32i x 10>>) {
+// CHECK: cir.func{{.*}} @func3(%arg0: !cir.ptr<!cir.array<!s32i x 10>>) {
 // CHECK:   %0 = cir.alloca !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 10>>>, ["pp", init] {alignment = 8 : i64}
 // CHECK:   cir.store %arg0, %0 : !cir.ptr<!cir.array<!s32i x 10>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 10>>>
 // CHECK:   cir.return
diff --git a/clang/test/CIR/IR/binassign.cir b/clang/test/CIR/IR/binassign.cir
index 24ed95d3c29c..a25729635094 100644
--- a/clang/test/CIR/IR/binassign.cir
+++ b/clang/test/CIR/IR/binassign.cir
@@ -26,7 +26,7 @@ module {
 // CHECK: !s8i = !cir.int<s, 8>
 // CHECK: #true = #cir.bool<true> : !cir.bool
 // CHECK: module {
-// CHECK:   cir.func @binary_assign() {
+// CHECK:   cir.func{{.*}} @binary_assign() {
 // CHECK:     %0 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b"] {alignment = 1 : i64}
 // CHECK:     %1 = cir.alloca !s8i, !cir.ptr<!s8i>, ["c"] {alignment = 1 : i64}
 // CHECK:     %2 = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["f"] {alignment = 4 : i64}
diff --git a/clang/test/CIR/IR/call.cir b/clang/test/CIR/IR/call.cir
index 5f0916775479..9607df7202e0 100644
--- a/clang/test/CIR/IR/call.cir
+++ b/clang/test/CIR/IR/call.cir
@@ -4,7 +4,7 @@
 
 module {
 
-cir.func @f1()
+cir.func private @f1()
 
 cir.func @f2() {
   cir.call @f1() : () -> ()
@@ -13,26 +13,26 @@ cir.func @f2() {
   cir.return
 }
 
-// CHECK:      cir.func @f2() {
+// CHECK:      cir.func{{.*}} @f2() {
 // CHECK-NEXT:   cir.call @f1() : () -> ()
 // CHECK-NEXT:   cir.call @f1() side_effect(pure) : () -> ()
 // CHECK-NEXT:   cir.call @f1() side_effect(const) : () -> ()
 // CHECK-NEXT:   cir.return
 // CHECK-NEXT: }
 
-cir.func @f3() -> !s32i
+cir.func private @f3() -> !s32i
 
 cir.func @f4() -> !s32i {
   %0 = cir.call @f3() : () -> !s32i
   cir.return %0 : !s32i
 }
 
-// CHECK:      cir.func @f4() -> !s32i {
+// CHECK:      cir.func{{.*}} @f4() -> !s32i {
 // CHECK-NEXT:   %[[#x:]] = cir.call @f3() : () -> !s32i
 // CHECK-NEXT:   cir.return %[[#x]] : !s32i
 // CHECK-NEXT: }
 
-cir.func @f5(!s32i, !s32i) -> !s32i
+cir.func private @f5(!s32i, !s32i) -> !s32i
 cir.func @f6() -> !s32i {
   %0 = cir.const #cir.int<1> : !s32i
   %1 = cir.const #cir.int<2> : !s32i
@@ -40,7 +40,7 @@ cir.func @f6() -> !s32i {
   cir.return %2 : !s32i
 }
 
-// CHECK:      cir.func @f6() -> !s32i {
+// CHECK:      cir.func{{.*}} @f6() -> !s32i {
 // CHECK-NEXT:   %[[#a:]] = cir.const #cir.int<1> : !s32i
 // CHECK-NEXT:   %[[#b:]] = cir.const #cir.int<2> : !s32i
 // CHECK-NEXT:   %[[#c:]] = cir.call @f5(%[[#a]], %[[#b]]) : (!s32i, !s32i) -> !s32i
@@ -54,7 +54,7 @@ cir.func @f7(%arg0: !cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>) -> !s32i {
   cir.return %2 : !s32i
 }
 
-// CHECK:      cir.func @f7(%[[ptr:.+]]: !cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>) -> !s32i {
+// CHECK:      cir.func{{.*}} @f7(%[[ptr:.+]]: !cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>) -> !s32i {
 // CHECK-NEXT:   %[[#a:]] = cir.const #cir.int<1> : !s32i
 // CHECK-NEXT:   %[[#b:]] = cir.const #cir.int<2> : !s32i
 // CHECK-NEXT:   %[[#ret:]] = cir.call %[[ptr]](%[[#a]], %[[#b]]) : (!cir.ptr<!cir.func<(!s32i, !s32i) -> !s32i>>, !s32i, !s32i) -> !s32i
diff --git a/clang/test/CIR/IR/cast.cir b/clang/test/CIR/IR/cast.cir
index 4881db7fc271..a335887de7ec 100644
--- a/clang/test/CIR/IR/cast.cir
+++ b/clang/test/CIR/IR/cast.cir
@@ -15,9 +15,9 @@ module  {
   }
 }
 
-// CHECK: cir.func @yolo(%arg0: !s32i)
+// CHECK: cir.func{{.*}} @yolo(%arg0: !s32i)
 // CHECK: %0 = cir.cast(int_to_bool, %arg0 : !s32i), !cir.bool
 // CHECK: %1 = cir.const #cir.int<0> : !s32i
 
-// CHECK: cir.func @bitcast
+// CHECK: cir.func{{.*}} @bitcast
 // CHECK: %0 = cir.cast(bitcast, %arg0 : !cir.ptr<!s32i>), !cir.ptr<f32>
diff --git a/clang/test/CIR/IR/cmp.cir b/clang/test/CIR/IR/cmp.cir
index a049dc51f140..818527189af0 100644
--- a/clang/test/CIR/IR/cmp.cir
+++ b/clang/test/CIR/IR/cmp.cir
@@ -36,7 +36,7 @@ module {
     cir.return
   }
 
-  // CHECK: cir.func @c0(%arg0: !s32i, %arg1: !s32i) {
+  // CHECK: cir.func{{.*}} @c0(%arg0: !s32i, %arg1: !s32i) {
   // CHECK-NEXT:   %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %1 = cir.alloca !s32i, !cir.ptr<!s32i>, ["b", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %2 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init] {alignment = 1 : i64}
@@ -102,7 +102,7 @@ module {
     cir.return
   }
 
-  // CHECK: cir.func @c0_unsigned(%arg0: !u32i, %arg1: !u32i) {
+  // CHECK: cir.func{{.*}} @c0_unsigned(%arg0: !u32i, %arg1: !u32i) {
   // CHECK-NEXT:   %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %1 = cir.alloca !u32i, !cir.ptr<!u32i>, ["b", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %2 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init] {alignment = 1 : i64}
@@ -168,7 +168,7 @@ module {
     cir.return
   }
 
-  // CHECK: cir.func @c0_float(%arg0: !cir.float, %arg1: !cir.float) {
+  // CHECK: cir.func{{.*}} @c0_float(%arg0: !cir.float, %arg1: !cir.float) {
   // CHECK-NEXT:   %0 = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["a", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %1 = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %2 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init] {alignment = 1 : i64}
@@ -234,7 +234,7 @@ module {
     cir.return
   }
 
-  // CHECK: cir.func @pointer_cmp(%arg0: !cir.ptr<!s32i>, %arg1: !cir.ptr<!s32i>) {
+  // CHECK: cir.func{{.*}} @pointer_cmp(%arg0: !cir.ptr<!s32i>, %arg1: !cir.ptr<!s32i>) {
   // CHECK-NEXT:   %0 = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["a", init] {alignment = 8 : i64}
   // CHECK-NEXT:   %1 = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["b", init] {alignment = 8 : i64}
   // CHECK-NEXT:   %2 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init] {alignment = 1 : i64}
@@ -312,7 +312,7 @@ module {
     cir.return
   }
 
-  // CHECK: cir.func @bool_cmp(%arg0: !cir.bool, %arg1: !cir.bool) {
+  // CHECK: cir.func{{.*}} @bool_cmp(%arg0: !cir.bool, %arg1: !cir.bool) {
   // CHECK-NEXT:   %0 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["a", init] {alignment = 1 : i64}
   // CHECK-NEXT:   %1 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["b", init] {alignment = 1 : i64}
   // CHECK-NEXT:   %2 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["x", init] {alignment = 1 : i64}
diff --git a/clang/test/CIR/IR/func.cir b/clang/test/CIR/IR/func.cir
index 9ed44611e195..5636194d13ec 100644
--- a/clang/test/CIR/IR/func.cir
+++ b/clang/test/CIR/IR/func.cir
@@ -10,7 +10,7 @@ module {
 cir.func @empty() {
   cir.return
 }
-// CHECK: cir.func @empty() {
+// CHECK: cir.func{{.*}} @empty() {
 // CHECK:   cir.return
 // CHECK: }
 
@@ -18,7 +18,7 @@ cir.func @empty() {
 cir.func @voidret() {
   cir.return
 }
-// CHECK: cir.func @voidret() {
+// CHECK: cir.func{{.*}} @voidret() {
 // CHECK:   cir.return
 // CHECK: }
 
@@ -27,11 +27,15 @@ cir.func @intfunc() -> !s32i {
   %0 = cir.const #cir.int<42> : !s32i
   cir.return %0 : !s32i
 }
-// CHECK: cir.func @intfunc() -> !s32i {
+// CHECK: cir.func{{.*}} @intfunc() -> !s32i {
 // CHECK:   %[[VAL:.*]] = cir.const #cir.int<42> : !s32i
 // CHECK:   cir.return %[[VAL]] : !s32i
 // CHECK: }
 
+// Should print/parse function aliases.
+cir.func @a_empty() alias(@empty)
+// CHECK: cir.func @a_empty() alias(@empty)
+
 // int scopes() {
 //  {
 //    {
@@ -48,7 +52,7 @@ cir.func @scopes() -> !s32i {
   }
   cir.trap
 }
-// CHECK: cir.func @scopes() -> !s32i {
+// CHECK: cir.func{{.*}} @scopes() -> !s32i {
 // CHECK:   cir.scope {
 // CHECK:     cir.scope {
 // CHECK:       %[[VAL:.*]] = cir.const #cir.int<99> : !s32i
@@ -63,7 +67,7 @@ cir.func @longfunc() -> !s64i {
   %0 = cir.const #cir.int<42> : !s64i
   cir.return %0 : !s64i
 }
-// CHECK: cir.func @longfunc() -> !s64i
+// CHECK: cir.func{{.*}} @longfunc() -> !s64i
 // CHECK:   %0 = cir.const #cir.int<42> : !s64i
 // CHECK:   cir.return %0 : !s64i
 // CHECK: }
@@ -73,7 +77,7 @@ cir.func @unsignedfunc() -> !u32i {
   %0 = cir.const #cir.int<42> : !u32i
   cir.return %0 : !u32i
 }
-// CHECK: cir.func @unsignedfunc() -> !u32i
+// CHECK: cir.func{{.*}} @unsignedfunc() -> !u32i
 // CHECK:   %[[VAL:.*]] = cir.const #cir.int<42> : !u32i
 // CHECK:   cir.return %[[VAL]] : !u32i
 // CHECK: }
@@ -83,7 +87,7 @@ cir.func @ullfunc() -> !u64i {
   %0 = cir.const #cir.int<42> : !u64i
   cir.return %0 : !u64i
 }
-// CHECK: cir.func @ullfunc() -> !u64i
+// CHECK: cir.func{{.*}} @ullfunc() -> !u64i
 // CHECK:   %[[VAL:.*]] = cir.const #cir.int<42> : !u64i
 // CHECK:   cir.return %[[VAL:.*]] : !u64i
 // CHECK: }
diff --git a/clang/test/CIR/IR/invalid-call.cir b/clang/test/CIR/IR/invalid-call.cir
index 3ebb771ed72e..a9c7e38f73af 100644
--- a/clang/test/CIR/IR/invalid-call.cir
+++ b/clang/test/CIR/IR/invalid-call.cir
@@ -12,7 +12,7 @@ cir.func @f1() {
 
 !u32i = !cir.int<u, 32>
 
-cir.func @f2()
+cir.func private @f2()
 cir.func @f3() {
   // expected-error @below {{callee returns void but call has results}}
   %0 = cir.call @f2() : () -> !u32i
@@ -23,7 +23,7 @@ cir.func @f3() {
 
 !u32i = !cir.int<u, 32>
 
-cir.func @f4() -> !u32i
+cir.func private @f4() -> !u32i
 cir.func @f5() {
   // expected-error @below {{incorrect number of results for callee}}
   cir.call @f4() : () -> ()
@@ -35,7 +35,7 @@ cir.func @f5() {
 !s32i = !cir.int<s, 32>
 !u32i = !cir.int<u, 32>
 
-cir.func @f6() -> !u32i
+cir.func private @f6() -> !u32i
 cir.func @f7() {
   // expected-error @below {{result type mismatch}}
   %0 = cir.call @f6() : () -> !s32i
@@ -47,7 +47,7 @@ cir.func @f7() {
 !s32i = !cir.int<s, 32>
 !u32i = !cir.int<u, 32>
 
-cir.func @f8(!s32i, !s32i)
+cir.func private @f8(!s32i, !s32i)
 cir.func @f9() {
   %0 = cir.const #cir.int<1> : !s32i
   // expected-error @below {{incorrect number of operands for callee}}
@@ -60,7 +60,7 @@ cir.func @f9() {
 !s32i = !cir.int<s, 32>
 !u32i = !cir.int<u, 32>
 
-cir.func @f10(!s32i, !s32i)
+cir.func private @f10(!s32i, !s32i)
 cir.func @f11() {
   %0 = cir.const #cir.int<1> : !s32i
   %1 = cir.const #cir.int<2> : !u32i
@@ -73,7 +73,7 @@ cir.func @f11() {
 
 !s32i = !cir.int<s, 32>
 
-cir.func @f12(!s32i, !s32i, ...)
+cir.func private @f12(!s32i, !s32i, ...)
 cir.func @f13() {
   %0 = cir.const #cir.int<1> : !s32i
   // expected-error @below {{too few operands for callee}}
diff --git a/clang/test/CIR/IR/invalid-complex.cir b/clang/test/CIR/IR/invalid-complex.cir
index 8c6d89057932..2414809f7dbc 100644
--- a/clang/test/CIR/IR/invalid-complex.cir
+++ b/clang/test/CIR/IR/invalid-complex.cir
@@ -21,3 +21,27 @@ module {
 cir.global external @ci2 = #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s64i> : !cir.complex<!s32i>
 
 }
+
+// -----
+
+module {
+  cir.func @complex_real_invalid_result_type() -> !cir.double {
+    %0 = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["c"]
+    %2 = cir.load align(8) %0 : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+    // expected-error @below {{result type does not match operand type}}
+    %3 = cir.complex.real %2 : !cir.complex<!cir.double> -> !cir.float
+    cir.return
+  }
+}
+
+// -----
+
+module {
+  cir.func @complex_imag_invalid_result_type() -> !cir.double {
+    %0 = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["c"]
+    %2 = cir.load align(8) %0 : !cir.ptr<!cir.complex<!cir.double>>, !cir.complex<!cir.double>
+    // expected-error @below {{result type does not match operand type}}
+    %3 = cir.complex.imag %2 : !cir.complex<!cir.double> -> !cir.float
+    cir.return
+  }
+}
diff --git a/clang/test/CIR/IR/invalid-func.cir b/clang/test/CIR/IR/invalid-func.cir
new file mode 100644
index 000000000000..6c06079ed8fd
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-func.cir
@@ -0,0 +1,11 @@
+// RUN: cir-opt %s -verify-diagnostics
+
+module {
+  cir.func @l0() {
+    cir.return
+  }
+
+  cir.func @l1() alias(@l0) { // expected-error {{function alias shall not have a body}}
+    cir.return
+  }
+}
diff --git a/clang/test/CIR/IR/invalid-opt-info.cir b/clang/test/CIR/IR/invalid-opt-info.cir
new file mode 100644
index 000000000000..52d6afd2c697
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-opt-info.cir
@@ -0,0 +1,13 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+module attributes {
+  // expected-error @below {{optimization level must be between 0 and 3 inclusive}}
+  cir.opt_info = #cir.opt_info<level = 4, size = 0>
+} {}
+
+// -----
+
+module attributes {
+  // expected-error @below {{size optimization level must be between 0 and 2 inclusive}}
+  cir.opt_info = #cir.opt_info<level = 0, size = 3>
+} {}
diff --git a/clang/test/CIR/IR/invalid-vector-create-wrong-size.cir b/clang/test/CIR/IR/invalid-vector-create-wrong-size.cir
deleted file mode 100644
index 38ae74e83fb9..000000000000
--- a/clang/test/CIR/IR/invalid-vector-create-wrong-size.cir
+++ /dev/null
@@ -1,16 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-
-module  {
-  cir.func @foo() {
-    %1 = cir.const #cir.int<1> : !s32i
-    %2 = cir.const #cir.int<2> : !s32i
-    %3 = cir.const #cir.int<3> : !s32i
-    %4 = cir.const #cir.int<4> : !s32i
-
-    // expected-error @below {{operand count of 4 doesn't match vector type '!cir.vector<8 x !cir.int<s, 32>>' element count of 8}}
-    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<8 x !s32i>
-    cir.return
-  }
-}
diff --git a/clang/test/CIR/IR/invalid-vector-create-wrong-type.cir b/clang/test/CIR/IR/invalid-vector-create-wrong-type.cir
deleted file mode 100644
index ded0e6072b53..000000000000
--- a/clang/test/CIR/IR/invalid-vector-create-wrong-type.cir
+++ /dev/null
@@ -1,17 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-!s64i = !cir.int<s, 64>
-
-module  {
-  cir.func @foo() {
-    %1 = cir.const #cir.int<1> : !s32i
-    %2 = cir.const #cir.int<2> : !s32i
-    %3 = cir.const #cir.int<3> : !s32i
-    %4 = cir.const #cir.int<4> : !s64i
-
-    // expected-error @below {{operand type '!cir.int<s, 64>' doesn't match vector element type '!cir.int<s, 32>'}}
-    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s64i) : !cir.vector<4 x !s32i>
-    cir.return
-  }
-}
diff --git a/clang/test/CIR/IR/invalid-vector-shift-wrong-result-type.cir b/clang/test/CIR/IR/invalid-vector-shift-wrong-result-type.cir
deleted file mode 100644
index 5de9df9d572b..000000000000
--- a/clang/test/CIR/IR/invalid-vector-shift-wrong-result-type.cir
+++ /dev/null
@@ -1,17 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-
-module  {
-  cir.func @foo() {
-    %1 = cir.const #cir.int<1> : !s32i
-    %2 = cir.const #cir.int<2> : !s32i
-    %3 = cir.const #cir.int<3> : !s32i
-    %4 = cir.const #cir.int<4> : !s32i
-    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
-    %6 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
-    // expected-error @below {{the type of the result must be a vector if it is vector shift}}
-    %7 = cir.shift(left, %5 : !cir.vector<4 x !s32i>, %6 : !cir.vector<4 x !s32i>) -> !s32i
-    cir.return
-  }
-}
diff --git a/clang/test/CIR/IR/invalid-vector-shuffle-dyn-wrong-operands.cir b/clang/test/CIR/IR/invalid-vector-shuffle-dyn-wrong-operands.cir
deleted file mode 100644
index 2115e769e177..000000000000
--- a/clang/test/CIR/IR/invalid-vector-shuffle-dyn-wrong-operands.cir
+++ /dev/null
@@ -1,19 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-!s64i = !cir.int<s, 64>
-
-module  {
-  cir.func @foo() {
-    %1 = cir.const #cir.int<1> : !s32i
-    %2 = cir.const #cir.int<2> : !s32i
-    %3 = cir.const #cir.int<3> : !s32i
-    %4 = cir.const #cir.int<4> : !s32i
-    %vec = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
-    %indices = cir.vec.create(%1, %2 : !s32i, !s32i) : !cir.vector<2 x !s32i>
-
-    // expected-error @below {{the number of elements in '!cir.vector<4 x !cir.int<s, 32>>' and '!cir.vector<2 x !cir.int<s, 32>>' don't match}}
-    %new_vec = cir.vec.shuffle.dynamic %vec : !cir.vector<4 x !s32i>, %indices : !cir.vector<2 x !s32i>
-    cir.return
-  }
-}
diff --git a/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir b/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir
deleted file mode 100644
index 375b2d3dc563..000000000000
--- a/clang/test/CIR/IR/invalid-vector-shuffle-wrong-index.cir
+++ /dev/null
@@ -1,16 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-!s64i = !cir.int<s, 64>
-
-module  {
-  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
-    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
-    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
-
-    // expected-error @below {{index for __builtin_shufflevector must be less than the total number of vector elements}}
-    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<9> : !s64i, #cir.int<4> : !s64i,
-      #cir.int<1> : !s64i, #cir.int<5> : !s64i] : !cir.vector<4 x !s32i>
-    cir.return %new_vec : !cir.vector<4 x !s32i>
-  }
-}
diff --git a/clang/test/CIR/IR/invalid-vector-zero-size.cir b/clang/test/CIR/IR/invalid-vector-zero-size.cir
deleted file mode 100644
index ffe6c56a28aa..000000000000
--- a/clang/test/CIR/IR/invalid-vector-zero-size.cir
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: cir-opt %s -verify-diagnostics -split-input-file
-
-!s32i = !cir.int<s, 32>
-
-module  {
-
-// expected-error @below {{the number of vector elements must be non-zero}}
-cir.global external @vec_a = #cir.zero : !cir.vector<0 x !s32i>
-
-}
diff --git a/clang/test/CIR/IR/invalid-vector.cir b/clang/test/CIR/IR/invalid-vector.cir
index 98e1109df84f..679994925786 100644
--- a/clang/test/CIR/IR/invalid-vector.cir
+++ b/clang/test/CIR/IR/invalid-vector.cir
@@ -15,7 +15,7 @@ cir.global external @vec_b = #cir.zero : !cir.vector<4 x !cir.array<!s32i x 10>>
 !s64i = !cir.int<s, 64>
 
 module  {
-  cir.func @invalid_vector_shuffle() {
+  cir.func @vector_shuffle_invalid_element_type() {
     %1 = cir.const #cir.int<1> : !s32i
     %2 = cir.const #cir.int<2> : !s32i
     %3 = cir.const #cir.int<3> : !s32i
@@ -34,7 +34,7 @@ module  {
 !s64i = !cir.int<s, 64>
 
 module  {
-  cir.func @invalid_vector_shuffle() {
+  cir.func @vector_shuffle_different_number_of_elements() {
     %1 = cir.const #cir.int<1> : !s32i
     %2 = cir.const #cir.int<2> : !s32i
     %3 = cir.const #cir.int<3> : !s32i
@@ -46,3 +46,104 @@ module  {
     cir.return
   }
 }
+
+// -----
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  cir.func @vector_create_different_size() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s32i
+
+    // expected-error @below {{operand count of 4 doesn't match vector type '!cir.vector<8 x !cir.int<s, 32>>' element count of 8}}
+    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<8 x !s32i>
+    cir.return
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @vector_create_different_type_size() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s64i
+
+    // expected-error @below {{operand type '!cir.int<s, 64>' doesn't match vector element type '!cir.int<s, 32>'}}
+    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s64i) : !cir.vector<4 x !s32i>
+    cir.return
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  cir.func @vector_shift_invalid_result_type() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s32i
+    %5 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    %6 = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    // expected-error @below {{the type of the result must be a vector if it is vector shift}}
+    %7 = cir.shift(left, %5 : !cir.vector<4 x !s32i>, %6 : !cir.vector<4 x !s32i>) -> !s32i
+    cir.return
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @vector_shuffle_dynamic_different_number_of_elements() {
+    %1 = cir.const #cir.int<1> : !s32i
+    %2 = cir.const #cir.int<2> : !s32i
+    %3 = cir.const #cir.int<3> : !s32i
+    %4 = cir.const #cir.int<4> : !s32i
+    %vec = cir.vec.create(%1, %2, %3, %4 : !s32i, !s32i, !s32i, !s32i) : !cir.vector<4 x !s32i>
+    %indices = cir.vec.create(%1, %2 : !s32i, !s32i) : !cir.vector<2 x !s32i>
+
+    // expected-error @below {{the number of elements in '!cir.vector<4 x !cir.int<s, 32>>' and '!cir.vector<2 x !cir.int<s, 32>>' don't match}}
+    %new_vec = cir.vec.shuffle.dynamic %vec : !cir.vector<4 x !s32i>, %indices : !cir.vector<2 x !s32i>
+    cir.return
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!s64i = !cir.int<s, 64>
+
+module  {
+  cir.func @vector_shuffle_invalid_index_value() -> !cir.vector<4 x !s32i> {
+    %vec_1 = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<3> : !s32i, #cir.int<5> : !s32i, #cir.int<7> : !s32i]> : !cir.vector<4 x !s32i>
+    %vec_2 = cir.const #cir.const_vector<[#cir.int<2> : !s32i, #cir.int<4> : !s32i, #cir.int<6> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
+
+    // expected-error @below {{index for __builtin_shufflevector must be less than the total number of vector elements}}
+    %new_vec = cir.vec.shuffle(%vec_1, %vec_2 : !cir.vector<4 x !s32i>) [#cir.int<9> : !s64i, #cir.int<4> : !s64i,
+      #cir.int<1> : !s64i, #cir.int<5> : !s64i] : !cir.vector<4 x !s32i>
+    cir.return %new_vec : !cir.vector<4 x !s32i>
+  }
+}
+
+// -----
+
+!s32i = !cir.int<s, 32>
+
+module  {
+
+// expected-error @below {{the number of vector elements must be non-zero}}
+cir.global external @vec_a = #cir.zero : !cir.vector<0 x !s32i>
+
+}
diff --git a/clang/test/CIR/IR/ternary.cir b/clang/test/CIR/IR/ternary.cir
index 3827dc77726d..e419c7f5af40 100644
--- a/clang/test/CIR/IR/ternary.cir
+++ b/clang/test/CIR/IR/ternary.cir
@@ -16,7 +16,7 @@ module  {
 
 // CHECK: module  {
 
-// CHECK: cir.func @blue(%arg0: !cir.bool) -> !u32i {
+// CHECK: cir.func{{.*}} @blue(%arg0: !cir.bool) -> !u32i {
 // CHECK:   %0 = cir.ternary(%arg0, true {
 // CHECK:     %1 = cir.const #cir.int<0> : !u32i
 // CHECK:     cir.yield %1 : !u32i
diff --git a/clang/test/CIR/IR/unary.cir b/clang/test/CIR/IR/unary.cir
index f01121adc106..ba3bc20d574f 100644
--- a/clang/test/CIR/IR/unary.cir
+++ b/clang/test/CIR/IR/unary.cir
@@ -16,7 +16,7 @@ module {
     %6 = cir.unary(dec, %1) : !u32i, !u32i
     cir.return
   }
-// CHECK: cir.func @test_unary_unsigned() {
+// CHECK: cir.func{{.*}} @test_unary_unsigned() {
 // CHECK:   %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["a"] {alignment = 4 : i64}
 // CHECK:   %1 = cir.load %0 : !cir.ptr<!u32i>, !u32i
 // CHECK:   %2 = cir.unary(plus, %1) : !u32i, !u32i
@@ -37,7 +37,7 @@ module {
     %6 = cir.unary(dec, %1) nsw : !s32i, !s32i
     cir.return
   }
-// CHECK: cir.func @test_unary_signed() {
+// CHECK: cir.func{{.*}} @test_unary_signed() {
 // CHECK:   %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["a"] {alignment = 4 : i64}
 // CHECK:   %1 = cir.load %0 : !cir.ptr<!s32i>, !s32i
 // CHECK:   %2 = cir.unary(plus, %1) : !s32i, !s32i
diff --git a/clang/test/CIR/IR/vector.cir b/clang/test/CIR/IR/vector.cir
index f23f5de9692d..6d8e5beffd63 100644
--- a/clang/test/CIR/IR/vector.cir
+++ b/clang/test/CIR/IR/vector.cir
@@ -26,7 +26,7 @@ cir.func @vec_int_test() {
   cir.return
 }
 
-// CHECK: cir.func @vec_int_test() {
+// CHECK: cir.func{{.*}} @vec_int_test() {
 // CHECK:  %0 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
 // CHECK:  %1 = cir.alloca !cir.vector<3 x !s32i>, !cir.ptr<!cir.vector<3 x !s32i>>, ["b"]
 // CHECK:  %2 = cir.alloca !cir.vector<2 x !s32i>, !cir.ptr<!cir.vector<2 x !s32i>>, ["c"]
@@ -38,7 +38,7 @@ cir.func @vec_double_test() {
   cir.return
 }
 
-// CHECK: cir.func @vec_double_test() {
+// CHECK: cir.func{{.*}} @vec_double_test() {
 // CHECK:  %0 = cir.alloca !cir.vector<2 x !cir.double>, !cir.ptr<!cir.vector<2 x !cir.double>>, ["a"]
 // CHECK:  cir.return
 // CHECK: }
@@ -54,7 +54,7 @@ cir.func @local_vector_create_test() {
     cir.return
 }
 
-// CHECK: cir.func @local_vector_create_test() {
+// CHECK: cir.func{{.*}} @local_vector_create_test() {
 // CHECK:   %0 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CHECK:   %1 = cir.const #cir.int<1> : !s32i
 // CHECK:   %2 = cir.const #cir.int<2> : !s32i
@@ -81,7 +81,7 @@ cir.func @vector_extract_element_test() {
     cir.return
 }
 
-// CHECK: cir.func @vector_extract_element_test() {
+// CHECK: cir.func{{.*}} @vector_extract_element_test() {
 // CHECK:    %0 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["d", init]
 // CHECK:    %1 = cir.alloca !s32i, !cir.ptr<!s32i>, ["e", init]
 // CHECK:    %2 = cir.const #cir.int<1> : !s32i
@@ -116,7 +116,7 @@ cir.func @vector_insert_element_test() {
     cir.return
 }
 
-// CHECK: cir.func @vector_insert_element_test() {
+// CHECK: cir.func{{.*}} @vector_insert_element_test() {
 // CHECK:    %0 = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CHECK:    %1 = cir.const #cir.int<1> : !s32i
 // CHECK:    %2 = cir.const #cir.int<2> : !s32i
@@ -150,7 +150,7 @@ cir.func @vector_compare_test() {
     cir.return
 }
 
-// CHECK: cir.func @vector_compare_test() {
+// CHECK: cir.func{{.*}} @vector_compare_test() {
 // CHECK:    %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
 // CHECK:    %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
 // CHECK:    %[[INIT:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["o", init]
@@ -176,7 +176,7 @@ cir.func @vector_shuffle_dynamic_test() {
     cir.return
 }
 
-// CHECK: cir.func @vector_shuffle_dynamic_test() {
+// CHECK: cir.func{{.*}} @vector_shuffle_dynamic_test() {
 // CHECK:    %[[VEC_A:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a"]
 // CHECK:    %[[VEC_B:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["b"]
 // CHECK:    %[[RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["r", init]
@@ -204,7 +204,7 @@ cir.func @vector_splat_test() {
     cir.return
 }
 
-// CHECK: cir.func @vector_splat_test() {
+// CHECK: cir.func{{.*}} @vector_splat_test() {
 // CHECK-NEXT: %[[VEC:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["a", init]
 // CHECK-NEXT: %[[SHL_RES:.*]] = cir.alloca !cir.vector<4 x !s32i>, !cir.ptr<!cir.vector<4 x !s32i>>, ["shl", init]
 // CHECK-NEXT: %[[CONST_1:.*]] = cir.const #cir.int<1> : !s32i
diff --git a/clang/test/CIR/Lowering/array.cpp b/clang/test/CIR/Lowering/array.cpp
index 9bd3464dc314..438d41e2c2c2 100644
--- a/clang/test/CIR/Lowering/array.cpp
+++ b/clang/test/CIR/Lowering/array.cpp
@@ -41,7 +41,7 @@ void func() {
   int e = arr[0];
   int e2 = arr[1];
 }
-// CHECK: define void @_Z4funcv()
+// CHECK: define{{.*}} void @_Z4funcv()
 // CHECK-NEXT: %[[ARR_ALLOCA:.*]] = alloca [10 x i32], i64 1, align 16
 // CHECK-NEXT: %[[INIT:.*]] = alloca i32, i64 1, align 4
 // CHECK-NEXT: %[[INIT_2:.*]] = alloca i32, i64 1, align 4
@@ -57,7 +57,7 @@ void func() {
 void func2() {
   int arr[2] = {5};
 }
-// CHECK: define void @_Z5func2v()
+// CHECK: define{{.*}} void @_Z5func2v()
 // CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x i32], i64 1, align 4
 // CHECK:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
 // CHECK:  %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR_ALLOCA]], i32 0
@@ -72,7 +72,7 @@ void func2() {
 void func3() {
   int arr3[2] = {5, 6};
 }
-// CHECK: define void @_Z5func3v()
+// CHECK: define{{.*}} void @_Z5func3v()
 // CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x i32], i64 1, align 4
 // CHECK:  %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR_ALLOCA]], i32 0
 // CHECK:  store i32 5, ptr %[[ARR_PTR]], align 4
@@ -83,7 +83,7 @@ void func4() {
   int arr[2][1] = {{5}, {6}};
   int e = arr[1][0];
 }
-// CHECK: define void @_Z5func4v()
+// CHECK: define{{.*}} void @_Z5func4v()
 // CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
 // CHECK:  %[[INIT:.*]] = alloca i32, i64 1, align 4
 // CHECK:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_ALLOCA]], i32 0
@@ -102,7 +102,7 @@ void func4() {
 void func5() {
   int arr[2][1] = {{5}};
 }
-// CHECK: define void @_Z5func5v()
+// CHECK: define{{.*}} void @_Z5func5v()
 // CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
 // CHECK:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
 // CHECK:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_ALLOCA]], i32 0
@@ -119,7 +119,7 @@ void func6() {
   int x = 4;
   int arr[2] = { x, 5 };
 }
-// CHECK: define void @_Z5func6v()
+// CHECK: define{{.*}} void @_Z5func6v()
 // CHECK:  %[[VAR:.*]] = alloca i32, i64 1, align 4
 // CHECK:  %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
 // CHECK:  store i32 4, ptr %[[VAR]], align 4
@@ -132,7 +132,7 @@ void func6() {
 void func7() {
   int* arr[1] = {};
 }
-// CHECK: define void @_Z5func7v()
+// CHECK: define{{.*}} void @_Z5func7v()
 // CHECK:  %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
 // CHECK:  %[[ALLOCA:.*]] = alloca ptr, i64 1, align 8
 // CHECK:  %[[ELE_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
@@ -143,9 +143,9 @@ void func7() {
 // CHECK:  store ptr %[[ELE]], ptr %[[ALLOCA]], align 8
 
 void func8(int p[10]) {}
-// CHECK: define void @_Z5func8Pi(ptr {{%.*}})
+// CHECK: define{{.*}} void @_Z5func8Pi(ptr {{%.*}})
 // CHECK-NEXT: alloca ptr, i64 1, align 8
 
 void func9(int pp[10][5]) {}
-// CHECK: define void @_Z5func9PA5_i(ptr {{%.*}})
+// CHECK: define{{.*}} void @_Z5func9PA5_i(ptr {{%.*}})
 // CHECK-NEXT: alloca ptr, i64 1, align 8
diff --git a/clang/test/CIR/Transforms/canonicalize.cir b/clang/test/CIR/Transforms/canonicalize.cir
index 164d231db7bb..7ba163eb30bb 100644
--- a/clang/test/CIR/Transforms/canonicalize.cir
+++ b/clang/test/CIR/Transforms/canonicalize.cir
@@ -15,7 +15,7 @@ module {
   ^bb2:  // pred: ^bb1
     cir.return
   }
-  // CHECK:      cir.func @redundant_br() {
+  // CHECK:      cir.func{{.*}} @redundant_br() {
   // CHECK-NEXT:   %[[A:.*]] = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init] {alignment = 4 : i64}
   // CHECK-NEXT:   %[[FOUR:.*]] = cir.const #cir.int<4> : !u32i
   // CHECK-NEXT:   cir.store %[[FOUR]], %[[A]] : !u32i, !cir.ptr<!u32i>
@@ -27,7 +27,7 @@ module {
     }
     cir.return
   }
-  // CHECK:      cir.func @empty_scope() {
+  // CHECK:      cir.func{{.*}} @empty_scope() {
   // CHECK-NEXT:   cir.return
   // CHECK-NEXT: }
 
@@ -36,7 +36,7 @@ module {
     %1 = cir.unary(not, %0) : !cir.bool, !cir.bool
     cir.return %1 : !cir.bool
   }
-  // CHECK:      cir.func @unary_not(%arg0: !cir.bool) -> !cir.bool
+  // CHECK:      cir.func{{.*}} @unary_not(%arg0: !cir.bool) -> !cir.bool
   // CHECK-NEXT:   cir.return %arg0 : !cir.bool
 
   cir.func @cast1(%arg0: !cir.bool) -> !cir.bool {
@@ -44,7 +44,7 @@ module {
     %1 = cir.cast(int_to_bool, %0 : !s32i), !cir.bool
     cir.return %1 : !cir.bool
   }
-  // CHECK:      cir.func @cast1(%[[ARG0:.*]]: !cir.bool) -> !cir.bool
+  // CHECK:      cir.func{{.*}} @cast1(%[[ARG0:.*]]: !cir.bool) -> !cir.bool
   // CHECK-NEXT:   cir.return %[[ARG0]] : !cir.bool
 
   cir.func @cast2(%arg0: !s32i) -> !cir.bool {
@@ -54,7 +54,7 @@ module {
     %3 = cir.cast(int_to_bool, %2 : !s64i), !cir.bool
     cir.return %3 : !cir.bool
   }
-  // CHECK:      cir.func @cast2(%[[ARG0:.*]]: !s32i) -> !cir.bool
+  // CHECK:      cir.func{{.*}} @cast2(%[[ARG0:.*]]: !s32i) -> !cir.bool
   // CHECK-NEXT:   %[[CAST:.*]] = cir.cast(int_to_bool, %[[ARG0]] : !s32i), !cir.bool
   // CHECK-NEXT:   cir.return %[[CAST]] : !cir.bool
 
@@ -64,7 +64,7 @@ module {
     %2 = cir.cast(integral, %1 : !s32i), !s64i
     cir.return %2 : !s64i
   }
-  // CHECK:      cir.func @no_fold_cast(%[[ARG0:.*]]: !s32i) -> !s64i
+  // CHECK:      cir.func{{.*}} @no_fold_cast(%[[ARG0:.*]]: !s32i) -> !s64i
   // CHECK-NEXT:   %[[CAST:.*]] = cir.cast(int_to_bool, %[[ARG0]] : !s32i), !cir.bool
   // CHECK-NEXT:   %[[CAST2:.*]] = cir.cast(bool_to_int, %[[CAST]] : !cir.bool), !s32i
   // CHECK-NEXT:   %[[CAST3:.*]] = cir.cast(integral, %[[CAST2]] : !s32i), !s64i
diff --git a/clang/test/CIR/Transforms/complex-create-fold.cir b/clang/test/CIR/Transforms/complex-create-fold.cir
index 5d9d22112c8b..370acaecc222 100644
--- a/clang/test/CIR/Transforms/complex-create-fold.cir
+++ b/clang/test/CIR/Transforms/complex-create-fold.cir
@@ -16,7 +16,7 @@ module {
     cir.return %6 : !cir.complex<!s32i>
   }
 
-// CHECK: cir.func @fold_complex_create_test() -> !cir.complex<!s32i> {
+// CHECK: cir.func{{.*}} @fold_complex_create_test() -> !cir.complex<!s32i> {
 // CHECK:   %[[RET:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["__retval"]
 // CHECK:   %[[INIT:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
 // CHECK:   %[[COMPLEX:.*]] = cir.const #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
diff --git a/clang/test/CIR/Transforms/complex-imag-fold.cir b/clang/test/CIR/Transforms/complex-imag-fold.cir
new file mode 100644
index 000000000000..0d9a4e43142a
--- /dev/null
+++ b/clang/test/CIR/Transforms/complex-imag-fold.cir
@@ -0,0 +1,23 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+  cir.func @fold_complex_imag_test() -> !s32i {
+    %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+    %2 = cir.const #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+    %4 = cir.complex.imag %2 : !cir.complex<!s32i> -> !s32i
+    cir.store %4, %0 : !s32i, !cir.ptr<!s32i>
+    %5 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+    cir.return %5 : !s32i
+  }
+
+  // CHECK: cir.func @fold_complex_imag_test() -> !s32i {
+  // CHECK:   %[[RET:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+  // CHECK:   %[[IMAG:.*]] = cir.const #cir.int<2> : !s32i
+  // CHECK:   cir.store %[[IMAG]], %[[RET]] : !s32i, !cir.ptr<!s32i>
+  // CHECK:   %[[TMP:.]] = cir.load %[[RET]] : !cir.ptr<!s32i>, !s32i
+  // CHECK:   cir.return %[[TMP]] : !s32i
+  // CHECK: }
+
+}
diff --git a/clang/test/CIR/Transforms/complex-real-fold.cir b/clang/test/CIR/Transforms/complex-real-fold.cir
new file mode 100644
index 000000000000..1cab9be616af
--- /dev/null
+++ b/clang/test/CIR/Transforms/complex-real-fold.cir
@@ -0,0 +1,23 @@
+// RUN: cir-opt %s -cir-canonicalize -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module {
+  cir.func @fold_complex_real_test() -> !s32i {
+    %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+    %2 = cir.const #cir.const_complex<#cir.int<1> : !s32i, #cir.int<2> : !s32i> : !cir.complex<!s32i>
+    %4 = cir.complex.real %2 : !cir.complex<!s32i> -> !s32i
+    cir.store %4, %0 : !s32i, !cir.ptr<!s32i>
+    %5 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+    cir.return %5 : !s32i
+  }
+
+  // CHECK: cir.func @fold_complex_real_test() -> !s32i {
+  // CHECK:   %[[RET:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+  // CHECK:   %[[REAL:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK:   cir.store %[[REAL]], %[[RET]] : !s32i, !cir.ptr<!s32i>
+  // CHECK:   %[[TMP:.]] = cir.load %[[RET]] : !cir.ptr<!s32i>, !s32i
+  // CHECK:   cir.return %[[TMP]] : !s32i
+  // CHECK: }
+
+}
diff --git a/clang/test/CIR/Transforms/hoist-allocas.cir b/clang/test/CIR/Transforms/hoist-allocas.cir
index df7b9f48be9d..04724f3073e5 100644
--- a/clang/test/CIR/Transforms/hoist-allocas.cir
+++ b/clang/test/CIR/Transforms/hoist-allocas.cir
@@ -20,7 +20,7 @@ module {
     }
     cir.return
   }
-  // CHECK:      cir.func @l1
+  // CHECK:      cir.func{{.*}} @l1
   // CHECK-NEXT:   %[[I:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
   // CHECK-NEXT:   cir.scope {
   // CHECK-NEXT:     %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
@@ -55,7 +55,7 @@ module {
     }
     cir.return
   }
-  // CHECK:      cir.func @l2
+  // CHECK:      cir.func{{.*}} @l2
   // CHECK-NEXT:   %[[I:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
   // CHECK-NEXT:   cir.scope {
   // CHECK-NEXT:     cir.for : cond {
@@ -92,7 +92,7 @@ module {
     }
     cir.return
   }
-  // CHECK:      cir.func @l3
+  // CHECK:      cir.func{{.*}} @l3
   // CHECK-NEXT:   %[[I:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["i", init] {alignment = 4 : i64}
   // CHECK-NEXT:   cir.scope {
   // CHECK-NEXT:     cir.for : cond {
diff --git a/clang/test/CIR/Transforms/if.cir b/clang/test/CIR/Transforms/if.cir
index 03848bf8d063..3f817c793643 100644
--- a/clang/test/CIR/Transforms/if.cir
+++ b/clang/test/CIR/Transforms/if.cir
@@ -14,7 +14,7 @@ module {
     }
     cir.return %arg0 : !s32i
   }
-//      CHECK: cir.func @foo(%arg0: !s32i) -> !s32i {
+//      CHECK: cir.func{{.*}} @foo(%arg0: !s32i) -> !s32i {
 // CHECK-NEXT:   %0 = cir.cast(int_to_bool, %arg0 : !s32i), !cir.bool
 // CHECK-NEXT:   cir.brcond %0 ^bb1, ^bb2
 // CHECK-NEXT: ^bb1:  // pred: ^bb0
@@ -35,7 +35,7 @@ module {
     }
     cir.return %arg0 : !s32i
   }
-//      CHECK: cir.func @onlyIf(%arg0: !s32i) -> !s32i {
+//      CHECK: cir.func{{.*}} @onlyIf(%arg0: !s32i) -> !s32i {
 // CHECK-NEXT:   %0 = cir.cast(int_to_bool, %arg0 : !s32i), !cir.bool
 // CHECK-NEXT:   cir.brcond %0 ^bb1, ^bb2
 // CHECK-NEXT: ^bb1:  // pred: ^bb0
diff --git a/clang/test/CIR/Transforms/loop.cir b/clang/test/CIR/Transforms/loop.cir
index d02412d04915..9c76092bb8e7 100644
--- a/clang/test/CIR/Transforms/loop.cir
+++ b/clang/test/CIR/Transforms/loop.cir
@@ -16,7 +16,7 @@ module {
   }
 }
 
-// CHECK:  cir.func @testFor(%arg0: !cir.bool) {
+// CHECK:  cir.func{{.*}} @testFor(%arg0: !cir.bool) {
 // CHECK:    cir.br ^bb[[#COND:]]
 // CHECK:  ^bb[[#COND]]:
 // CHECK:    cir.brcond %arg0 ^bb[[#BODY:]], ^bb[[#EXIT:]]
@@ -38,7 +38,7 @@ module {
     cir.return
   }
 
-// CHECK:  cir.func @testWhile(%arg0: !cir.bool) {
+// CHECK:  cir.func{{.*}} @testWhile(%arg0: !cir.bool) {
 // CHECK:    cir.br ^bb[[#COND:]]
 // CHECK:  ^bb[[#COND]]:
 // CHECK:    cir.brcond %arg0 ^bb[[#BODY:]], ^bb[[#EXIT:]]
@@ -59,7 +59,7 @@ module {
     cir.return
   }
 
-// CHECK:  cir.func @testDoWhile(%arg0: !cir.bool) {
+// CHECK:  cir.func{{.*}} @testDoWhile(%arg0: !cir.bool) {
 // CHECK:    cir.br ^bb[[#BODY:]]
 // CHECK:  ^bb[[#COND]]:
 // CHECK:    cir.brcond %arg0 ^bb[[#BODY:]], ^bb[[#EXIT:]]
diff --git a/clang/test/CIR/Transforms/scope.cir b/clang/test/CIR/Transforms/scope.cir
index 2d14784c33f8..757428cd019a 100644
--- a/clang/test/CIR/Transforms/scope.cir
+++ b/clang/test/CIR/Transforms/scope.cir
@@ -11,7 +11,7 @@ module {
     }
     cir.return
   }
-// CHECK:  cir.func @foo() {
+// CHECK:  cir.func{{.*}} @foo() {
 // CHECK:    cir.br ^bb1
 // CHECK:  ^bb1:  // pred: ^bb0
 // CHECK:    %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["a", init] {alignment = 4 : i64}
@@ -28,7 +28,7 @@ module {
     }
     cir.return
   }
-// CHECK:  cir.func @empty_scope() {
+// CHECK:  cir.func{{.*}} @empty_scope() {
 // CHECK:    cir.return
 // CHECK:  }
 
@@ -44,7 +44,7 @@ module {
     cir.return %1 : !u32i
   }
 
-// CHECK:  cir.func @scope_with_return() -> !u32i {
+// CHECK:  cir.func{{.*}} @scope_with_return() -> !u32i {
 // CHECK:    %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:    cir.br ^bb1
 // CHECK:  ^bb1:  // pred: ^bb0
diff --git a/clang/test/CIR/Transforms/select.cir b/clang/test/CIR/Transforms/select.cir
index 29a5d1ed1dde..0ad5c4317883 100644
--- a/clang/test/CIR/Transforms/select.cir
+++ b/clang/test/CIR/Transforms/select.cir
@@ -10,7 +10,7 @@ module {
     cir.return %1 : !s32i
   }
 
-  //      CHECK: cir.func @fold_true(%[[ARG0:.+]]: !s32i, %[[ARG1:.+]]: !s32i) -> !s32i {
+  //      CHECK: cir.func{{.*}} @fold_true(%[[ARG0:.+]]: !s32i, %[[ARG1:.+]]: !s32i) -> !s32i {
   // CHECK-NEXT:   cir.return %[[ARG0]] : !s32i
   // CHECK-NEXT: }
 
@@ -20,7 +20,7 @@ module {
     cir.return %1 : !s32i
   }
 
-  //      CHECK: cir.func @fold_false(%[[ARG0:.+]]: !s32i, %[[ARG1:.+]]: !s32i) -> !s32i {
+  //      CHECK: cir.func{{.*}} @fold_false(%[[ARG0:.+]]: !s32i, %[[ARG1:.+]]: !s32i) -> !s32i {
   // CHECK-NEXT:   cir.return %[[ARG1]] : !s32i
   // CHECK-NEXT: }
 
@@ -30,7 +30,7 @@ module {
     cir.return %1 : !s32i
   }
 
-  //      CHECK: cir.func @fold_to_const(%{{.+}}: !cir.bool) -> !s32i {
+  //      CHECK: cir.func{{.*}} @fold_to_const(%{{.+}}: !cir.bool) -> !s32i {
   // CHECK-NEXT:   %[[#A:]] = cir.const #cir.int<42> : !s32i
   // CHECK-NEXT:   cir.return %[[#A]] : !s32i
   // CHECK-NEXT: }
@@ -42,7 +42,7 @@ module {
     cir.return %2 : !cir.bool
   }
 
-  //      CHECK: cir.func @simplify_1(%[[ARG0:.+]]: !cir.bool) -> !cir.bool {
+  //      CHECK: cir.func{{.*}} @simplify_1(%[[ARG0:.+]]: !cir.bool) -> !cir.bool {
   // CHECK-NEXT:   cir.return %[[ARG0]] : !cir.bool
   // CHECK-NEXT: }
 
@@ -53,7 +53,7 @@ module {
     cir.return %2 : !cir.bool
   }
 
-  //      CHECK: cir.func @simplify_2(%[[ARG0:.+]]: !cir.bool) -> !cir.bool {
+  //      CHECK: cir.func{{.*}} @simplify_2(%[[ARG0:.+]]: !cir.bool) -> !cir.bool {
   // CHECK-NEXT:   %[[#A:]] = cir.unary(not, %[[ARG0]]) : !cir.bool, !cir.bool
   // CHECK-NEXT:   cir.return %[[#A]] : !cir.bool
   // CHECK-NEXT: }
diff --git a/clang/test/CIR/Transforms/switch.cir b/clang/test/CIR/Transforms/switch.cir
index 00b462a6075c..a000d6b70fbc 100644
--- a/clang/test/CIR/Transforms/switch.cir
+++ b/clang/test/CIR/Transforms/switch.cir
@@ -17,7 +17,7 @@ module {
     }
     cir.return
   }
-// CHECK:  cir.func @shouldFlatSwitchWithDefault(%arg0: !s8i) {
+// CHECK:  cir.func{{.*}} @shouldFlatSwitchWithDefault(%arg0: !s8i) {
 // CHECK:    cir.switch.flat %arg0 : !s8i, ^bb[[#DEFAULT:]] [
 // CHECK:      1: ^bb[[#CASE1:]]
 // CHECK:    ]
@@ -38,7 +38,7 @@ module {
     }
     cir.return
   }
-// CHECK:  cir.func @shouldFlatSwitchWithoutDefault(%arg0: !s32i) {
+// CHECK:  cir.func{{.*}} @shouldFlatSwitchWithoutDefault(%arg0: !s32i) {
 // CHECK:    cir.switch.flat %arg0 : !s32i, ^bb[[#EXIT:]] [
 // CHECK:      1: ^bb[[#CASE1:]]
 // CHECK:    ]
@@ -58,7 +58,7 @@ module {
     }
     cir.return
   }
-// CHECK:  cir.func @shouldFlatSwitchWithImplicitFallthrough(%arg0: !s64i) {
+// CHECK:  cir.func{{.*}} @shouldFlatSwitchWithImplicitFallthrough(%arg0: !s64i) {
 // CHECK:    cir.switch.flat %arg0 : !s64i, ^bb[[#EXIT:]] [
 // CHECK:      1: ^bb[[#CASE1N2:]],
 // CHECK:      2: ^bb[[#CASE1N2]]
@@ -83,7 +83,7 @@ module {
       }
     cir.return
   }
-// CHECK:  cir.func @shouldFlatSwitchWithExplicitFallthrough(%arg0: !s64i) {
+// CHECK:  cir.func{{.*}} @shouldFlatSwitchWithExplicitFallthrough(%arg0: !s64i) {
 // CHECK:    cir.switch.flat %arg0 : !s64i, ^bb[[#EXIT:]] [
 // CHECK:      1: ^bb[[#CASE1:]],
 // CHECK:      2: ^bb[[#CASE2:]]
@@ -105,7 +105,7 @@ module {
       }
     cir.return
   }
-// CHECK:  cir.func @shouldFlatSwitchWithFallthroughToExit(%arg0: !s64i) {
+// CHECK:  cir.func{{.*}} @shouldFlatSwitchWithFallthroughToExit(%arg0: !s64i) {
 // CHECK:    cir.switch.flat %arg0 : !s64i, ^bb[[#EXIT:]] [
 // CHECK:      1: ^bb[[#CASE1:]]
 // CHECK:    ]
@@ -122,7 +122,7 @@ module {
     // CHECK-NOT: llvm.switch
     cir.return
   }
-// CHECK:      cir.func @shouldDropEmptySwitch(%arg0: !s64i)
+// CHECK:      cir.func{{.*}} @shouldDropEmptySwitch(%arg0: !s64i)
 // CHECK-NOT:    cir.switch.flat
 
 
@@ -143,7 +143,7 @@ module {
     cir.return
   }
 
-// CHECK:  cir.func @shouldFlatMultiBlockCase(%arg0: !s32i) {
+// CHECK:  cir.func{{.*}} @shouldFlatMultiBlockCase(%arg0: !s32i) {
 // CHECK:     %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init] {alignment = 4 : i64}
 // CHECK:     cir.store %arg0, %0 : !s32i, !cir.ptr<!s32i>
 // CHECK:     cir.br ^bb1
@@ -189,7 +189,7 @@ module {
     %4 = cir.load %2 : !cir.ptr<!s32i>, !s32i
     cir.return %4 : !s32i
   }
-// CHECK:  cir.func @shouldFlatNestedBreak(%arg0: !s32i, %arg1: !s32i) -> !s32i {
+// CHECK:  cir.func{{.*}} @shouldFlatNestedBreak(%arg0: !s32i, %arg1: !s32i) -> !s32i {
 // CHECK:    cir.switch.flat %[[COND:.*]] : !s32i, ^bb[[#DEFAULT_BB:]] [
 // CHECK:      0: ^bb[[#BB1:]]
 // CHECK:    ]
@@ -243,7 +243,7 @@ module {
     %5 = cir.load %1 : !cir.ptr<!s32i>, !s32i
     cir.return %5 : !s32i
   }
-//      CHECK:  cir.func @flatCaseRange(%arg0: !s32i) -> !s32i {
+//      CHECK:  cir.func{{.*}} @flatCaseRange(%arg0: !s32i) -> !s32i {
 //      CHECK:    cir.switch.flat %[[X:[0-9]+]] : !s32i, ^[[JUDGE_RANGE:bb[0-9]+]] [
 // CHECK-NEXT:      -100: ^[[CASE_EQUAL:bb[0-9]+]]
 // CHECK-NEXT:    ]
@@ -293,7 +293,7 @@ module {
     cir.return
   }
 
-// CHECK:  cir.func @_Z8bigRangei(%arg0: !s32i) {
+// CHECK:  cir.func{{.*}} @_Z8bigRangei(%arg0: !s32i) {
 // CHECK:    cir.switch.flat %[[COND:.*]] : !s32i, ^bb[[#RANGE_BR:]] [
 // CHECK:    ]
 // CHECK:  ^bb[[#NO_PRED_BB:]]:  // no predecessors
diff --git a/clang/test/CIR/Transforms/ternary-fold.cir b/clang/test/CIR/Transforms/ternary-fold.cir
index 1192a0ce2942..718906f5c6ee 100644
--- a/clang/test/CIR/Transforms/ternary-fold.cir
+++ b/clang/test/CIR/Transforms/ternary-fold.cir
@@ -14,7 +14,7 @@ module {
     cir.return %1 : !s32i
   }
 
-  //      CHECK: cir.func @fold_ternary(%{{.+}}: !s32i, %[[ARG:.+]]: !s32i) -> !s32i {
+  //      CHECK: cir.func{{.*}} @fold_ternary(%{{.+}}: !s32i, %[[ARG:.+]]: !s32i) -> !s32i {
   // CHECK-NEXT:   cir.return %[[ARG]] : !s32i
   // CHECK-NEXT: }
 
@@ -28,7 +28,7 @@ module {
     cir.return %0 : !s32i
   }
 
-  //      CHECK: cir.func @simplify_ternary(%[[ARG0:.+]]: !cir.bool, %[[ARG1:.+]]: !s32i) -> !s32i {
+  //      CHECK: cir.func{{.*}} @simplify_ternary(%[[ARG0:.+]]: !cir.bool, %[[ARG1:.+]]: !s32i) -> !s32i {
   // CHECK-NEXT:   %[[#A:]] = cir.const #cir.int<42> : !s32i
   // CHECK-NEXT:   %[[#B:]] = cir.select if %[[ARG0]] then %[[#A]] else %[[ARG1]] : (!cir.bool, !s32i, !s32i) -> !s32i
   // CHECK-NEXT:   cir.return %[[#B]] : !s32i
@@ -44,7 +44,7 @@ module {
     cir.return %0 : !s32i
   }
 
-  //      CHECK: cir.func @simplify_ternary_false_const(%[[ARG0:.+]]: !cir.bool, %[[ARG1:.+]]: !s32i) -> !s32i {
+  //      CHECK: cir.func{{.*}} @simplify_ternary_false_const(%[[ARG0:.+]]: !cir.bool, %[[ARG1:.+]]: !s32i) -> !s32i {
   // CHECK-NEXT:   %[[#A:]] = cir.const #cir.int<24> : !s32i
   // CHECK-NEXT:   %[[#B:]] = cir.select if %[[ARG0]] then %[[ARG1]] else %[[#A]] : (!cir.bool, !s32i, !s32i) -> !s32i
   // CHECK-NEXT:   cir.return %[[#B]] : !s32i
@@ -62,7 +62,7 @@ module {
     cir.return %1 : !s32i
   }
 
-  //      CHECK: cir.func @non_simplifiable_ternary(%[[ARG0:.+]]: !cir.bool) -> !s32i {
+  //      CHECK: cir.func{{.*}} @non_simplifiable_ternary(%[[ARG0:.+]]: !cir.bool) -> !s32i {
   // CHECK-NEXT:   %[[#A:]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["a", init]
   // CHECK-NEXT:   %[[#B:]] = cir.ternary(%[[ARG0]], true {
   // CHECK-NEXT:     %[[#C:]] = cir.const #cir.int<42> : !s32i
diff --git a/clang/test/CIR/Transforms/ternary.cir b/clang/test/CIR/Transforms/ternary.cir
index 67ef7f95a6b5..fffafa9ff8e4 100644
--- a/clang/test/CIR/Transforms/ternary.cir
+++ b/clang/test/CIR/Transforms/ternary.cir
@@ -22,7 +22,7 @@ module {
     cir.return %6 : !s32i
   }
 
-// CHECK: cir.func @foo(%arg0: !s32i) -> !s32i {
+// CHECK: cir.func{{.*}} @foo(%arg0: !s32i) -> !s32i {
 // CHECK:   %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["y", init] {alignment = 4 : i64}
 // CHECK:   %1 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   cir.store %arg0, %0 : !s32i, !cir.ptr<!s32i>
@@ -53,7 +53,7 @@ module {
     cir.return
   }
 
-// CHECK: cir.func @foo2(%arg0: !cir.bool) {
+// CHECK: cir.func{{.*}} @foo2(%arg0: !cir.bool) {
 // CHECK:   cir.brcond %arg0 ^bb1, ^bb2
 // CHECK: ^bb1:  // pred: ^bb0
 // CHECK:   cir.br ^bb3
diff --git a/clang/test/CIR/Transforms/vector-cmp-fold.cir b/clang/test/CIR/Transforms/vector-cmp-fold.cir
index b207fc08748e..f3486bd26fe1 100644
--- a/clang/test/CIR/Transforms/vector-cmp-fold.cir
+++ b/clang/test/CIR/Transforms/vector-cmp-fold.cir
@@ -10,7 +10,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -28,7 +28,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -46,7 +46,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -64,7 +64,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -82,7 +82,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -100,7 +100,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -120,7 +120,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -140,7 +140,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -160,7 +160,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -180,7 +180,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<1> : !s32i,
   // CHECK-SAME: #cir.int<1> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -200,7 +200,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -220,7 +220,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_cmp_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<0> : !s32i, #cir.int<0> : !s32i,
   // CHECK-SAME: #cir.int<0> : !s32i, #cir.int<0> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
diff --git a/clang/test/CIR/Transforms/vector-create-fold.cir b/clang/test/CIR/Transforms/vector-create-fold.cir
index fb8f66dc4deb..fb8c39e4dda4 100644
--- a/clang/test/CIR/Transforms/vector-create-fold.cir
+++ b/clang/test/CIR/Transforms/vector-create-fold.cir
@@ -12,7 +12,7 @@ module  {
     cir.return %vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK:  cir.func @fold_create_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK:  cir.func{{.*}} @fold_create_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[VEC:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
   // CHECK-SAME: #cir.int<3> : !s32i, #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[VEC]] : !cir.vector<4 x !s32i>
diff --git a/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir b/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir
index 46ab04502afe..6584df3d7050 100644
--- a/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir
+++ b/clang/test/CIR/Transforms/vector-shuffle-dynamic-fold.cir
@@ -11,7 +11,7 @@ module {
   }
 
   // Masking indices [8, 7, 6, 5] AND 3 = [0, 3, 2, 1]
-  // CHECK: cir.func @fold_shuffle_dynamic_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func{{.*}} @fold_shuffle_dynamic_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[NEW_VEC:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<4> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[NEW_VEC:.*]] : !cir.vector<4 x !s32i>
 
@@ -23,7 +23,7 @@ module {
   }
 
   // Masking indices [3, 2, 1, 0] AND 3 = [3, 2, 1, 0]
-  // CHECK: cir.func @fold_shuffle_dynamic_vector_op_test_2() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func{{.*}} @fold_shuffle_dynamic_vector_op_test_2() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[NEW_VEC:.*]] = cir.const #cir.const_vector<[#cir.int<4> : !s32i, #cir.int<3> : !s32i, #cir.int<2> : !s32i, #cir.int<1> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[NEW_VEC:.*]] : !cir.vector<4 x !s32i>
 }
diff --git a/clang/test/CIR/Transforms/vector-shuffle-fold.cir b/clang/test/CIR/Transforms/vector-shuffle-fold.cir
index 87d409728989..7aaddc051a75 100644
--- a/clang/test/CIR/Transforms/vector-shuffle-fold.cir
+++ b/clang/test/CIR/Transforms/vector-shuffle-fold.cir
@@ -12,7 +12,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func{{.*}} @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
   // CHECK-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
@@ -32,7 +32,7 @@ module  {
     cir.return %new_vec : !cir.vector<6 x !s32i>
   }
 
-  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<6 x !s32i> {
+  // CHECK: cir.func{{.*}} @fold_shuffle_vector_op_test() -> !cir.vector<6 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
   // CHECK-SAME: #cir.int<4> : !s32i, #cir.int<5> : !s32i, #cir.int<6> : !s32i]> : !cir.vector<6 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<6 x !s32i>
@@ -52,7 +52,7 @@ module  {
     cir.return %new_vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func{{.*}} @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK: cir.const #cir.const_vector<[#cir.undef : !s32i, #cir.int<2> : !s32i, #cir.int<3> : !s32i,
   // CHECK-SAME: #cir.int<4> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
diff --git a/clang/test/CIR/Transforms/vector-splat.cir b/clang/test/CIR/Transforms/vector-splat.cir
index e2274b8627b1..9b2b39d7f862 100644
--- a/clang/test/CIR/Transforms/vector-splat.cir
+++ b/clang/test/CIR/Transforms/vector-splat.cir
@@ -3,13 +3,13 @@
 !s32i = !cir.int<s, 32>
 
 module  {
-  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  cir.func @fold_splat_vector_op_test() -> !cir.vector<4 x !s32i> {
     %v = cir.const #cir.int<3> : !s32i
     %vec = cir.vec.splat %v : !s32i, !cir.vector<4 x !s32i>
     cir.return %vec : !cir.vector<4 x !s32i>
   }
 
-  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func @fold_splat_vector_op_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %0 = cir.const #cir.const_vector<[#cir.int<3> : !s32i, #cir.int<3> : !s32i,
   // CHECK-SAME: #cir.int<3> : !s32i, #cir.int<3> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %0 : !cir.vector<4 x !s32i>
diff --git a/clang/test/CIR/Transforms/vector-ternary-fold.cir b/clang/test/CIR/Transforms/vector-ternary-fold.cir
index f2e18576da74..3de614a39264 100644
--- a/clang/test/CIR/Transforms/vector-ternary-fold.cir
+++ b/clang/test/CIR/Transforms/vector-ternary-fold.cir
@@ -12,7 +12,7 @@ module {
   }
 
   // [1, 0, 1, 0] ? [1, 2, 3, 4] : [5, 6, 7, 8] Will be fold to [1, 6, 3, 8]
-  // CHECK: cir.func @vector_ternary_fold_test() -> !cir.vector<4 x !s32i> {
+  // CHECK: cir.func{{.*}} @vector_ternary_fold_test() -> !cir.vector<4 x !s32i> {
   // CHECK-NEXT: %[[RES:.*]] = cir.const #cir.const_vector<[#cir.int<1> : !s32i, #cir.int<6> : !s32i, #cir.int<3> : !s32i, #cir.int<8> : !s32i]> : !cir.vector<4 x !s32i>
   // CHECK-NEXT: cir.return %[[RES]] : !cir.vector<4 x !s32i>
 }
diff --git a/clang/test/CIR/func-linkage.cpp b/clang/test/CIR/func-linkage.cpp
new file mode 100644
index 000000000000..d43f7ed27306
--- /dev/null
+++ b/clang/test/CIR/func-linkage.cpp
@@ -0,0 +1,51 @@
+// Linkage types of global variables
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck %s -check-prefix=CIR --input-file %t.cir
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck %s -check-prefix=LLVM --input-file %t-cir.ll
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck %s -check-prefix=OGCG --input-file %t.ll
+
+void a() {}
+
+// CIR: cir.func dso_local @_Z1av()
+// LLVM: define dso_local void @_Z1av()
+// OGCG: define dso_local void @_Z1av()
+
+extern void b();
+// CIR: cir.func private @_Z1bv()
+// LLVM: declare void @_Z1bv()
+// OGCG: declare void @_Z1bv()
+
+static void c() {}
+// CIR: cir.func internal private dso_local @_ZL1cv()
+// LLVM: define internal void @_ZL1cv()
+// OGCG: define internal void @_ZL1cv()
+
+inline void d() {}
+// CIR: cir.func comdat linkonce_odr @_Z1dv()
+// LLVM: define linkonce_odr void @_Z1dv()
+// OGCG: define linkonce_odr void @_Z1dv(){{.*}} comdat
+
+namespace {
+  void e() {}
+}
+
+// CIR: cir.func internal private dso_local @_ZN12_GLOBAL__N_11eEv()
+// LLVM: define internal void @_ZN12_GLOBAL__N_11eEv()
+// OGCG: define internal void @_ZN12_GLOBAL__N_11eEv()
+
+void f();
+// CIR: cir.func private @_Z1fv()
+// LLVM: declare void @_Z1fv()
+// OGCG: declare void @_Z1fv()
+
+// Force the functions to be emitted
+void reference_funcs() {
+    a();
+    b();
+    c();
+    d();
+    e();
+    f();
+}
diff --git a/clang/test/CIR/func-simple.cpp b/clang/test/CIR/func-simple.cpp
index 45cf1746de71..c9cb5c559535 100644
--- a/clang/test/CIR/func-simple.cpp
+++ b/clang/test/CIR/func-simple.cpp
@@ -2,17 +2,17 @@
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o -  | FileCheck %s
 
 void empty() { }
-// CHECK: cir.func @_Z5emptyv() {
+// CHECK: cir.func{{.*}} @_Z5emptyv() {
 // CHECK:   cir.return
 // CHECK: }
 
 void voidret() { return; }
-// CHECK: cir.func @_Z7voidretv() {
+// CHECK: cir.func{{.*}} @_Z7voidretv() {
 // CHECK:   cir.return
 // CHECK: }
 
 int intfunc() { return 42; }
-// CHECK: cir.func @_Z7intfuncv() -> !s32i {
+// CHECK: cir.func{{.*}} @_Z7intfuncv() -> !s32i {
 // CHECK:   %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   %1 = cir.const #cir.int<42> : !s32i
 // CHECK:   cir.store %1, %0 : !s32i, !cir.ptr<!s32i>
@@ -27,7 +27,7 @@ int scopes() {
     }
   }
 }
-// CHECK: cir.func @_Z6scopesv() -> !s32i {
+// CHECK: cir.func{{.*}} @_Z6scopesv() -> !s32i {
 // CHECK:   %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   cir.scope {
 // CHECK:     cir.scope {
@@ -41,7 +41,7 @@ int scopes() {
 // CHECK: }
 
 long longfunc() { return 42l; }
-// CHECK: cir.func @_Z8longfuncv() -> !s64i
+// CHECK: cir.func{{.*}} @_Z8longfuncv() -> !s64i
 // CHECK:   %0 = cir.alloca !s64i, !cir.ptr<!s64i>, ["__retval"] {alignment = 8 : i64}
 // CHECK:   %1 = cir.const #cir.int<42> : !s64i
 // CHECK:   cir.store %1, %0 : !s64i, !cir.ptr<!s64i>
@@ -50,7 +50,7 @@ long longfunc() { return 42l; }
 // CHECK: }
 
 unsigned unsignedfunc() { return 42u; }
-// CHECK: cir.func @_Z12unsignedfuncv() -> !u32i
+// CHECK: cir.func{{.*}} @_Z12unsignedfuncv() -> !u32i
 // CHECK:   %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   %1 = cir.const #cir.int<42> : !u32i
 // CHECK:   cir.store %1, %0 : !u32i, !cir.ptr<!u32i>
@@ -59,7 +59,7 @@ unsigned unsignedfunc() { return 42u; }
 // CHECK: }
 
 unsigned long long ullfunc() { return 42ull; }
-// CHECK: cir.func @_Z7ullfuncv() -> !u64i
+// CHECK: cir.func{{.*}} @_Z7ullfuncv() -> !u64i
 // CHECK:   %0 = cir.alloca !u64i, !cir.ptr<!u64i>, ["__retval"] {alignment = 8 : i64}
 // CHECK:   %1 = cir.const #cir.int<42> : !u64i
 // CHECK:   cir.store %1, %0 : !u64i, !cir.ptr<!u64i>
@@ -68,7 +68,7 @@ unsigned long long ullfunc() { return 42ull; }
 // CHECK: }
 
 bool boolfunc() { return true; }
-// CHECK: cir.func @_Z8boolfuncv() -> !cir.bool {
+// CHECK: cir.func{{.*}} @_Z8boolfuncv() -> !cir.bool {
 // CHECK:   %0 = cir.alloca !cir.bool, !cir.ptr<!cir.bool>, ["__retval"] {alignment = 1 : i64}
 // CHECK:   %1 = cir.const #true
 // CHECK:   cir.store %1, %0 : !cir.bool, !cir.ptr<!cir.bool>
@@ -77,7 +77,7 @@ bool boolfunc() { return true; }
 // CHECK: }
 
 float floatfunc() { return 42.42f; }
-// CHECK: cir.func @_Z9floatfuncv() -> !cir.float {
+// CHECK: cir.func{{.*}} @_Z9floatfuncv() -> !cir.float {
 // CHECK:   %0 = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["__retval"] {alignment = 4 : i64}
 // CHECK:   %1 = cir.const #cir.fp<4.242
 // CHECK:   cir.store %1, %0 : !cir.float, !cir.ptr<!cir.float>
@@ -86,7 +86,7 @@ float floatfunc() { return 42.42f; }
 // CHECK: }
 
 double doublefunc() { return 42.42; }
-// CHECK: cir.func @_Z10doublefuncv() -> !cir.double {
+// CHECK: cir.func{{.*}} @_Z10doublefuncv() -> !cir.double {
 // CHECK:   %0 = cir.alloca !cir.double, !cir.ptr<!cir.double>, ["__retval"] {alignment = 8 : i64}
 // CHECK:   %1 = cir.const #cir.fp<4.242
 // CHECK:   cir.store %1, %0 : !cir.double, !cir.ptr<!cir.double>
diff --git a/clang/test/CIR/global-var-linkage.cpp b/clang/test/CIR/global-var-linkage.cpp
index 3a288b8990ad..c9ee8fb6a502 100644
--- a/clang/test/CIR/global-var-linkage.cpp
+++ b/clang/test/CIR/global-var-linkage.cpp
@@ -17,7 +17,7 @@ int aaaa;
 // OGCG: @dddd = weak_odr global i32 0, comdat
 
 static int bbbb;
-// CIR: cir.global "private" internal dsolocal @_ZL4bbbb
+// CIR: cir.global "private" internal dso_local @_ZL4bbbb
 // LLVM: @_ZL4bbbb = internal global i32 0
 // OGCG: @_ZL4bbbb = internal global i32 0
 
diff --git a/clang/test/CIR/mlprint.c b/clang/test/CIR/mlprint.c
index 755a6cb47855..1630bc1e3ce9 100644
--- a/clang/test/CIR/mlprint.c
+++ b/clang/test/CIR/mlprint.c
@@ -7,7 +7,7 @@ int foo(void) {
 }
 
 // CIR:  IR Dump After CIRCanonicalize (cir-canonicalize)
-// CIR:  cir.func @foo() -> !s32i
+// CIR:  cir.func{{.*}} @foo() -> !s32i
 // LLVM: IR Dump After cir::direct::ConvertCIRToLLVMPass (cir-flat-to-llvm)
 // LLVM: llvm.func @foo() -> i32
 // LLVM: IR Dump After
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index eaa0d65460d5..3255bbe05178 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -30,6 +30,26 @@ llvm_canonicalize_cmake_booleans(
   LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
   )
 
+# Run tests requiring Z3 headers only if LLVM was built with Z3
+# and the headers are available while building Clang -- the latter may
+# not be the case when building standalone against installed LLVM.
+set(TEST_WITH_Z3_MOCK 0)
+if(LLVM_WITH_Z3 AND Z3_FOUND AND CMAKE_SYSTEM_NAME MATCHES "Linux")
+  llvm_add_library(
+    MockZ3SolverCheck
+    MODULE Analysis/z3/Inputs/MockZ3_solver_check.cpp
+    DISABLE_LLVM_LINK_LLVM_DYLIB)
+
+  if(TARGET MockZ3SolverCheck)
+    list(APPEND CLANG_TEST_DEPS
+      MockZ3SolverCheck)
+    target_include_directories(
+      MockZ3SolverCheck
+      PRIVATE ${Z3_INCLUDE_DIR})
+    set(TEST_WITH_Z3_MOCK 1)
+  endif()
+endif()
+
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
@@ -94,6 +114,7 @@ if(CLANG_ENABLE_CIR)
   list(APPEND CLANG_TEST_DEPS
     cir-opt
     cir-translate
+    mlir-translate
     )
 endif()
 
@@ -124,10 +145,6 @@ if(LLVM_INCLUDE_SPIRV_TOOLS_TESTS)
     )
 endif()    
 
-set(CLANG_TEST_PARAMS
-  USE_Z3_SOLVER=0
-  )
-
 if( NOT CLANG_BUILT_STANDALONE )
   list(APPEND CLANG_TEST_DEPS
     llvm-config
@@ -195,13 +212,11 @@ set_target_properties(clang-test-depends PROPERTIES FOLDER "Clang/Tests")
 add_lit_testsuite(check-clang "Running the Clang regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   #LIT ${LLVM_LIT}
-  PARAMS ${CLANG_TEST_PARAMS}
   DEPENDS ${CLANG_TEST_DEPS}
   ARGS ${CLANG_TEST_EXTRA_ARGS}
   )
 
 add_lit_testsuites(CLANG ${CMAKE_CURRENT_SOURCE_DIR}
-  PARAMS ${CLANG_TEST_PARAMS}
   DEPENDS ${CLANG_TEST_DEPS}
   FOLDER "Clang tests/Suites"
 )
diff --git a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
index 497a9328c115..a8cd523b0837 100644
--- a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
+++ b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
@@ -102,11 +102,12 @@ namespace ns {
 }
 
 #elif TEST13
+// expected-no-diagnostics
 extern "C++" {
-  int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
+  int main();
 }
 
-extern "C++" int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
+extern "C++" int main();
 
 namespace ns1 {
   extern "C++" int main(); // ok
diff --git a/clang/test/CXX/class.access/class.access.dcl/p1.cpp b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
index fdb1373dd9b1..b7ee6c9a7c04 100644
--- a/clang/test/CXX/class.access/class.access.dcl/p1.cpp
+++ b/clang/test/CXX/class.access/class.access.dcl/p1.cpp
@@ -318,26 +318,23 @@ namespace test4 {
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
 #endif
 
-    C::foo; // legal in C++03
+    C::foo;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
 #else
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
-    // expected-error@-5 {{using declaration refers to its own class}}
 #endif
+    // expected-error@-6 {{using declaration refers to its own class}}
 
-    Subclass::foo; // legal in C++03
+    Subclass::foo;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
 #else
     // expected-error@-4 {{ISO C++11 does not allow access declarations; use using declarations instead}}
-    // expected-error@-5 {{using declaration refers into 'Subclass', which is not a base class of 'C'}}
 #endif
+    // expected-error@-6 {{using declaration refers into 'Subclass', which is not a base class of 'C'}}
 
     int bar();
-#if __cplusplus <= 199711L
-    //expected-note@-2 {{target of using declaration}}
-#endif
     C::bar;
 #if __cplusplus <= 199711L
     // expected-warning@-2 {{access declarations are deprecated; use using declarations instead}}
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
index 657695657e0f..bcc26bf4f435 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p3.cpp
@@ -30,17 +30,10 @@ class D2 : public B {
   using B::x;
   using C::g; // expected-error{{using declaration refers into 'C', which is not a base class of 'D2'}}
 
-  // These are valid in C++98 but not in C++11.
-  using D::f2;
-  using D::E2;
-  using D::e2;
-  using D::x2;
-#if __cplusplus >= 201103L
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-  // expected-error@-5 {{using declaration refers into 'D', which is not a base class of 'D2'}}
-#endif
+  using D::f2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::E2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::e2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
+  using D::x2; // expected-error {{using declaration refers into 'D', which is not a base class of 'D2'}}
 
   using B::EC;
   using B::EC::ec; // expected-warning {{a C++20 extension}} expected-warning 0-1 {{C++11}}
@@ -71,13 +64,7 @@ namespace test1 {
     using Base::bar; // expected-error {{no member named 'bar'}}
     using Unrelated::foo; // expected-error {{not a base class}}
 
-    // In C++98, it's hard to see that these are invalid, because indirect
-    // references to base class members are permitted.
-    using C::foo;
-    using Subclass::foo;
-#if __cplusplus >= 201103L
-    // expected-error@-3 {{refers to its own class}}
-    // expected-error@-3 {{not a base class}}
-#endif
+    using C::foo; // expected-error {{refers to its own class}}
+    using Subclass::foo; // expected-error {{not a base class}}
   };
 }
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
index 973cd77279f1..26b66350b86d 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p4.cpp
@@ -206,17 +206,9 @@ namespace test4 {
     using InnerNS::foo; // expected-error {{not a class}}
     using Base::bar; // expected-error {{no member named 'bar'}}
     using Unrelated::foo; // expected-error {{not a base class}}
-    using C::foo; // legal in C++03
-    using Subclass::foo; // legal in C++03
-#if __cplusplus >= 201103L
-    // expected-error@-3 {{refers to its own class}}
-    // expected-error@-3 {{refers into 'Subclass', which is not a base class}}
-#endif
-
+    using C::foo; // expected-error {{refers to its own class}}
+    using Subclass::foo; // expected-error {{refers into 'Subclass', which is not a base class}}
     int bar();
-#if __cplusplus < 201103L
-    // expected-note@-2 {{target of using declaration}}
-#endif
     using C::bar; // expected-error {{refers to its own class}}
   };
 }
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index 210f7ae71ec0..dc53ae712a0b 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -36,6 +36,15 @@ namespace cwg400 { // cwg400: 2.7
   // expected-error@-1 {{member 'a' found in multiple base classes of different types}}
   //   expected-note@#cwg400-A {{member type 'cwg400::A::a' found by ambiguous name lookup}}
   //   expected-note@#cwg400-B {{member type 'cwg400::B::a' found by ambiguous name lookup}}
+  struct F : A {};
+  struct G : A {
+    using G::A;
+    // expected-error@-1 {{using declaration refers to its own class}}
+    using G::a;
+    // expected-error@-1 {{using declaration refers to its own class}}
+    using F::a;
+    // expected-error@-1 {{using declaration refers into 'F', which is not a base class of 'G'}}
+  };
 } // namespace cwg400
 
 namespace cwg401 { // cwg401: 2.8
diff --git a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
index ac21e36daf87..abb42447d3e0 100644
--- a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
@@ -218,4 +218,26 @@ void regression() {
 
 }
 
+namespace GH146063 {
+template <typename>
+struct A {
+  static_assert([]() constexpr { return true; }());
+};
+
+void f1() {
+  if constexpr (false) {
+    A<int> a;
+  }
+}
+
+void f2() {
+  if constexpr (false) {
+    static_assert([]{});
+    // expected-warning@-1 {{address of lambda function pointer conversion operator will always evaluate to 'true'}}
+  }
+}
+
+}
+
+
 #endif
diff --git a/clang/test/ClangScanDeps/P1689.cppm b/clang/test/ClangScanDeps/P1689.cppm
index 4176a06ca3c3..99c2fabc4ee4 100644
--- a/clang/test/ClangScanDeps/P1689.cppm
+++ b/clang/test/ClangScanDeps/P1689.cppm
@@ -1,15 +1,14 @@
 // UNSUPPORTED: target={{.*}}-aix{{.*}}
 //
-// The slash direction in linux and windows are different.
-// UNSUPPORTED: system-windows
-//
 // RUN: rm -fr %t
 // RUN: mkdir -p %t
 // RUN: split-file %s %t
 //
 // RUN: sed "s|DIR|%/t|g" %t/P1689.json.in > %t/P1689.json
 // RUN: clang-scan-deps -compilation-database %t/P1689.json -format=p1689 | FileCheck %t/Checks.cpp -DPREFIX=%/t
-// RUN: clang-scan-deps --mode=preprocess-dependency-directives -compilation-database %t/P1689.json -format=p1689 | FileCheck %t/Checks.cpp -DPREFIX=%/t
+// RUN: clang-scan-deps --mode=preprocess-dependency-directives -compilation-database %t/P1689.json -format=p1689 \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/Checks.cpp -DPREFIX=%/t
 //
 // Check the separated dependency format. This is required by CMake for the case
 // that we have non-exist files in a fresh build and potentially out-of-date after that.
@@ -17,25 +16,32 @@
 // which is not so good. So here is the per file mode for P1689.
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/M.cppm -o %t/M.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/M.cppm -DPREFIX=%/t
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/Impl.cpp -o %t/Impl.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/Impl.cpp -DPREFIX=%/t
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/impl_part.cppm -o %t/impl_part.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/impl_part.cppm -DPREFIX=%/t
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/interface_part.cppm -o %t/interface_part.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/interface_part.cppm -DPREFIX=%/t
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/User.cpp -o %t/User.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/User.cpp -DPREFIX=%/t
 //
 // Check we can generate the make-style dependencies as expected.
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/impl_part.cppm -o %t/impl_part.o \
 // RUN:      -MT %t/impl_part.o.ddi -MD -MF %t/impl_part.dep
-// RUN:   cat %t/impl_part.dep | FileCheck %t/impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE
+// RUN: cat %t/impl_part.dep \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE
 //
 // Check that we can generate multiple make-style dependency information with compilation database.
 // RUN: cat %t/P1689.dep | FileCheck %t/Checks.cpp -DPREFIX=%/t --check-prefix=CHECK-MAKE
@@ -43,6 +49,7 @@
 // Check that we can mix the use of -format=p1689 and -fmodules.
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache -c %t/impl_part.cppm -o %t/impl_part.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
 // RUN:   | FileCheck %t/impl_part.cppm -DPREFIX=%/t
 //
 // Check the path in the make style dependencies are generated in relative path form
@@ -50,7 +57,9 @@
 // RUN: clang-scan-deps -format=p1689 \
 // RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t impl_part.cppm -o impl_part.o \
 // RUN:      -MT impl_part.o.ddi -MD -MF impl_part.dep
-// RUN:   cat impl_part.dep | FileCheck impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE-RELATIVE
+// RUN: cat impl_part.dep \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck impl_part.cppm -DPREFIX=%/t --check-prefix=CHECK-MAKE-RELATIVE
 
 
 //--- P1689.json.in
diff --git a/clang/test/ClangScanDeps/modules-full-named-modules.cppm b/clang/test/ClangScanDeps/modules-full-named-modules.cppm
new file mode 100644
index 000000000000..5967a8705c09
--- /dev/null
+++ b/clang/test/ClangScanDeps/modules-full-named-modules.cppm
@@ -0,0 +1,351 @@
+// This checks that clang-scan-deps properly outputs named module dependencies 
+// when using the the scanning output format 'experimental-full'.
+//
+// See PR #72304.
+// UNSUPPORTED: target={{.*}}-aix{{.*}}
+//
+// RUN: rm -fr %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// Check the separated dependency format.
+// RUN: sed "s|DIR|%/t|g" %t/compile_commands.json.in > %t/compile_commands.json
+// RUN: clang-scan-deps -format=experimental-full \
+// RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/M.cppm -o %t/M.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/M.cppm -DPREFIX=%/t
+// RUN: clang-scan-deps -format=experimental-full \
+// RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/Impl.cpp -o %t/Impl.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/Impl.cpp -DPREFIX=%/t
+// RUN: clang-scan-deps -format=experimental-full \
+// RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/impl_part.cppm -o %t/impl_part.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/impl_part.cppm -DPREFIX=%/t
+// RUN: clang-scan-deps -format=experimental-full \
+// RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/interface_part.cppm -o %t/interface_part.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/interface_part.cppm -DPREFIX=%/t
+// RUN: clang-scan-deps -format=experimental-full \
+// RUN:   -- %clang++ -std=c++20 -c -fprebuilt-module-path=%t %t/User.cpp -o %t/User.o \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/User.cpp -DPREFIX=%/t
+//
+// Check the combined dependency format.
+// RUN: clang-scan-deps -compilation-database %t/compile_commands.json -format=experimental-full \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/Checks.cpp -DPREFIX=%/t
+// RUN: clang-scan-deps --mode=preprocess-dependency-directives -compilation-database %t/compile_commands.json -format=experimental-full \
+// RUN:   | sed 's:\\\\\?:/:g' \
+// RUN:   | FileCheck %t/Checks.cpp -DPREFIX=%/t
+
+//--- compile_commands.json.in
+[
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/M.cppm -c -o DIR/M.o -MT DIR/M.o.ddi -MD -MF DIR/P1689.dep",
+  "file": "DIR/M.cppm",
+  "output": "DIR/M.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/Impl.cpp -c -o DIR/Impl.o -MT DIR/Impl.o.ddi -MD -MF DIR/P1689.dep",
+  "file": "DIR/Impl.cpp",
+  "output": "DIR/Impl.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/impl_part.cppm -c -o DIR/impl_part.o -MT DIR/impl_part.o.ddi -MD -MF DIR/P1689.dep",
+  "file": "DIR/impl_part.cppm",
+  "output": "DIR/impl_part.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/interface_part.cppm -c -o DIR/interface_part.o -MT DIR/interface_part.o.ddi -MD -MF DIR/P1689.dep",
+  "file": "DIR/interface_part.cppm",
+  "output": "DIR/interface_part.o"
+},
+{
+  "directory": "DIR",
+  "command": "clang++ -std=c++20 DIR/User.cpp -c -o DIR/User.o -MT DIR/User.o.ddi -MD -MF DIR/P1689.dep",
+  "file": "DIR/User.cpp",
+  "output": "DIR/User.o"
+}
+]
+
+//--- M.cppm
+export module M;
+export import :interface_part;
+import :impl_part;
+export void Hello();
+
+// CHECK:       {
+// CHECK:         "modules": []
+// CHECK:         "translation-units": [
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part",
+// CHECK-NEXT:              "M:impl_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/M-{{.*}}.pcm"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/M.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/M.o"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/M.cppm"
+// CHECK:               }
+// CHECK:             ]
+// CHECK:           }
+// CHECK:         ]
+// CHECK:       }
+
+//--- Impl.cpp
+module;
+#include "header.mock"
+module M;
+void Hello() {
+    std::cout << "Hello ";
+}
+
+// CHECK:       {
+// CHECK:         "modules": []
+// CHECK:         "translation-units": [
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK-NOT:             "named-module":
+// CHECK:                 "named-module-deps": [
+// CHECK-NEXT:              "M"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/Impl.o"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/Impl.cpp"
+// CHECK:               }
+// CHECK:             ]
+// CHECK:           }
+// CHECK:         ]
+// CHECK:       }
+
+//--- impl_part.cppm
+module;
+#include "header.mock"
+module M:impl_part;
+import :interface_part;
+
+std::string W = "World.";
+void World() {
+    std::cout << W << std::endl;
+}
+
+// CHECK:       {
+// CHECK:         "modules": [],
+// CHECK:         "translation-units": [
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:impl_part"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/impl_part-{{.*}}.pcm",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/impl_part.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:impl_part"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/impl_part.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/impl_part.cppm"
+// CHECK:               }
+// CHECK:             ]
+// CHECK:           }
+// CHECK:         ]
+// CHECK:       }
+
+//--- interface_part.cppm
+export module M:interface_part;
+export void World();
+
+// CHECK:       {
+// CHECK:         "modules": []
+// CHECK:         "translation-units": [
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:interface_part"
+// CHECK-NOT:             "named-module-deps": []
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/interface_part-{{.*}}.pcm",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/interface_part.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:interface_part"
+// CHECK-NOT:             "named-module-deps": []
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/interface_part.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/interface_part.cppm"
+// CHECK:               }
+// CHECK:             ]
+// CHECK:           }
+// CHECK:         ]
+// CHECK:       }
+
+//--- User.cpp
+import M;
+import third_party_module;
+int main() {
+    Hello();
+    World();
+    return 0;
+}
+
+// CHECK:       {
+// CHECK-NEXT:    "modules": []
+// CHECK-NEXT:    "translation-units": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "commands": [
+// CHECK-NEXT:          {
+// CHECK-NOT:             "named-module":
+// CHECK:                 "named-module-deps": [
+// CHECK-NEXT:              "M"
+// CHECK-NEXT:              "third_party_module"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/User.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/User.cpp
+// CHECK:               }
+// CHECK:             ]
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ]
+// CHECK:       }
+
+//--- header.mock
+  
+//--- Checks.cpp
+// CHECK:       {
+// CHECK:         "modules": []
+// CHECK:         "translation-units": [
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part",
+// CHECK-NEXT:              "M:impl_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/M-{{.*}}.pcm"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/M.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/M.o"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/M.cppm"
+// CHECK:               },
+// CHECK:               {
+// CHECK-NOT:             "named-module":
+// CHECK:                 "named-module-deps": [
+// CHECK-NEXT:              "M"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/Impl.o"
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/Impl.cpp"
+// CHECK:               }
+// CHECK:             ]
+// CHECK-NEXT:      },
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:impl_part"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/impl_part-{{.*}}.pcm",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/impl_part.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:impl_part"
+// CHECK-NEXT:            "named-module-deps": [
+// CHECK-NEXT:              "M:interface_part"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/impl_part.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/impl_part.cppm"
+// CHECK:               }
+// CHECK:             ]
+// CHECK-NEXT:      },
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:interface_part"
+// CHECK-NOT:             "named-module-deps": []
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "{{.*}}/interface_part-{{.*}}.pcm",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/interface_part.cppm"
+// CHECK:               },
+// CHECK-NEXT:          {
+// CHECK:                 "named-module": "M:interface_part"
+// CHECK-NOT:             "named-module-deps": []
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/interface_part.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/interface_part.cppm"
+// CHECK:               }
+// CHECK:             ]
+// CHECK-NEXT:      },
+// CHECK-NEXT:      {
+// CHECK:             "commands": [
+// CHECK-NEXT:          {
+// CHECK-NOT:             "named-module":
+// CHECK:                 "named-module-deps": [
+// CHECK-NEXT:              "M"
+// CHECK-NEXT:              "third_party_module"
+// CHECK-NEXT:             ]
+// CHECK:                 "command-line": [
+// CHECK:                   "-o",
+// CHECK-NEXT:              "[[PREFIX]]/User.o",
+// CHECK:                 ]
+// CHECK:                 "input-file": "[[PREFIX]]/User.cpp
+// CHECK:               } 
+// CHECK:             ]
+// CHECK:           }
+// CHECK:         ]
+// CHECK:       }
diff --git a/clang/test/ClangScanDeps/raw-strings.cpp b/clang/test/ClangScanDeps/raw-strings.cpp
new file mode 100644
index 000000000000..ff8a98664399
--- /dev/null
+++ b/clang/test/ClangScanDeps/raw-strings.cpp
@@ -0,0 +1,148 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json
+
+//--- cdb.json.in
+[{
+    "directory": "DIR",
+    "command": "clang -c DIR/tu.c -o DIR/tu.o -IDIR/include",
+    "file": "DIR/tu.c"
+}]
+//--- include/header.h
+//--- include/header2.h
+//--- include/header3.h
+//--- include/header4.h
+//--- include/header5.h
+//--- include/header6.h
+//--- include/header7.h
+//--- include/header8.h
+//--- include/header9.h
+//--- include/header10.h
+//--- include/header11.h
+//--- include/header12.h
+//--- include/header13.h
+//--- include/header14.h
+//--- tu.c
+#if 0
+R"x()x"
+#endif
+
+#include "header.h"
+
+#if 0
+R"y(";
+#endif
+#include "header2.h"
+
+#if 0
+//")y"
+#endif
+
+#if 0
+R"y(";
+R"z()y";
+#endif
+#include "header3.h"
+#if 0
+//")z"
+#endif
+
+#if 0
+R\
+"y(";
+R"z()y";
+#endif
+#include "header4.h"
+#if 0
+//")z"
+#endif
+
+// Test u8 prefix with escaped newline
+#if 0
+u8R\
+"prefix(test)prefix"
+#endif
+#include "header5.h"
+
+// Test u prefix with multiple escaped newlines
+#if 0
+uR\
+\
+"multi(test)multi"
+#endif
+#include "header6.h"
+
+// Test U prefix with escaped newline
+#if 0
+UR\
+"upper(test)upper"
+#endif
+#include "header7.h"
+
+// Test L prefix with escaped newline
+#if 0
+LR\
+"wide(test)wide"
+#endif
+#include "header8.h"
+
+// Test escaped newline with \r\n style
+#if 0
+R\
+"crlf(test)crlf"
+#endif
+#include "header9.h"
+
+// Test multiple escaped newlines in different positions
+#if 0
+u\
+8\
+R\
+"complex(test)complex"
+#endif
+#include "header10.h"
+
+// Test raw string that should NOT be treated as raw (no R prefix due to identifier continuation)
+#if 0
+identifierR"notraw(test)notraw"
+#endif
+#include "header11.h"
+
+// Test raw string with whitespace before escaped newline
+#if 0
+R \
+"whitespace(test)whitespace"
+#endif
+#include "header12.h"
+
+// Test nested raw strings in disabled code
+#if 0
+R"outer(
+    R"inner(content)inner"
+)outer"
+#endif
+#include "header13.h"
+
+// Test raw string with empty delimiter
+#if 0
+R\
+"(empty delimiter)";
+#endif
+#include "header14.h"
+
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -mode preprocess | FileCheck %s
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -mode preprocess-dependency-directives | FileCheck %s
+// CHECK: tu.c
+// CHECK-NEXT: header.h
+// CHECK-NEXT: header3.h
+// CHECK-NEXT: header4.h
+// CHECK-NEXT: header5.h
+// CHECK-NEXT: header6.h
+// CHECK-NEXT: header7.h
+// CHECK-NEXT: header8.h
+// CHECK-NEXT: header9.h
+// CHECK-NEXT: header10.h
+// CHECK-NEXT: header11.h
+// CHECK-NEXT: header12.h
+// CHECK-NEXT: header13.h
+// CHECK-NEXT: header14.h
diff --git a/clang/test/CodeCompletion/overrides.cpp b/clang/test/CodeCompletion/overrides.cpp
index 7f330904eede..fcafe03021f1 100644
--- a/clang/test/CodeCompletion/overrides.cpp
+++ b/clang/test/CodeCompletion/overrides.cpp
@@ -41,3 +41,17 @@ void func() {
 // Runs completion at empty line on line 37.
 // RUN: not %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-5):1 %s -o - | FileCheck -check-prefix=CHECK-CC4 %s
 // CHECK-CC4: COMPLETION: Pattern : void vfunc(bool param, int p) override{{$}}
+
+class NoexceptBase {
+ public:
+  virtual void method() noexcept;
+};
+
+class NoexceptDerived : public NoexceptBase {
+ public:
+  met;
+};
+
+// Runs completion at met^ in the body of NoexceptDerived.
+// RUN: not %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-4):6 %s -o - | FileCheck -check-prefix=CHECK-CC5 %s
+// CHECK-CC5: COMPLETION: Pattern : void method() noexcept override{{$}}
diff --git a/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp b/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
new file mode 100644
index 000000000000..55c16bb126fe
--- /dev/null
+++ b/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
@@ -0,0 +1,14 @@
+struct A {
+  void foo(this A self, int arg);
+};
+
+int main() {
+  A a {};
+  a.
+}
+// RUN: %clang_cc1 -cc1 -fsyntax-only -code-completion-at=%s:%(line-2):5 -std=c++23 %s | FileCheck %s
+// CHECK: COMPLETION: A : A::
+// CHECK-NEXT: COMPLETION: foo : [#void#]foo(<#int arg#>)
+// CHECK-NEXT: COMPLETION: operator= : [#A &#]operator=(<#const A &#>)
+// CHECK-NEXT: COMPLETION: operator= : [#A &#]operator=(<#A &&#>)
+// CHECK-NEXT: COMPLETION: ~A : [#void#]~A()
diff --git a/clang/test/CodeGen/AArch64/attr-fp8-function.c b/clang/test/CodeGen/AArch64/attr-fp8-function.c
new file mode 100644
index 000000000000..54bfd177bd80
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/attr-fp8-function.c
@@ -0,0 +1,37 @@
+// Check memory attribute for FP8 function
+
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +fp8 -target-feature +sve -target-feature +sme -target-feature +sme2 -target-feature +sme-f8f16 -target-feature +sme-f8f32  -target-feature +ssve-fp8fma -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+
+#include <arm_neon.h>
+#include <arm_sme.h>
+
+
+// SIMD
+mfloat8x16_t test_vcvtq_mf8_f16_fpm(float16x8_t vn, float16x8_t vm, fpm_t fpm) {
+  return vcvtq_mf8_f16_fpm(vn, vm, fpm);
+}
+
+// SVE
+svfloat16_t test_svcvtlt2_f16_mf8(svmfloat8_t zn, fpm_t fpm) __arm_streaming {
+  return svcvtlt2_f16_mf8_fpm(zn, fpm);
+}
+
+// CHECK: declare void @llvm.aarch64.set.fpmr(i64) [[ATTR3:#.*]]
+// CHECK: declare <vscale x 8 x half> @llvm.aarch64.sve.fp8.cvtlt2.nxv8f16(<vscale x 16 x i8>) [[ATTR4:#.*]]
+
+
+// SME
+svfloat32_t test_svmlalltt_lane_f32_mf8(svfloat32_t zda, svmfloat8_t zn, svmfloat8_t zm, fpm_t fpm) __arm_streaming {
+  return svmlalltt_lane_f32_mf8_fpm(zda, zn, zm, 7, fpm);
+}
+
+// CHECK: declare <vscale x 4 x float> @llvm.aarch64.sve.fp8.fmlalltt.lane.nxv4f32(<vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i8>, i32 immarg) [[ATTR4]]
+
+// CHECK: declare <16 x i8> @llvm.aarch64.neon.fp8.fcvtn.v16i8.v8f16(<8 x half>, <8 x half>) [[ATTR4]]
+
+// CHECK: attributes [[ATTR1:#.*]] = {{{.*}}} 
+// CHECK: attributes [[ATTR2:#.*]] = {{{.*}}}
+// CHECK: attributes [[ATTR3]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+// CHECK: attributes [[ATTR4]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: read) }
diff --git a/clang/test/CodeGen/AArch64/fmv-dependencies.c b/clang/test/CodeGen/AArch64/fmv-dependencies.c
index 7aaf143f2afc..a97c4e95cd03 100644
--- a/clang/test/CodeGen/AArch64/fmv-dependencies.c
+++ b/clang/test/CodeGen/AArch64/fmv-dependencies.c
@@ -188,7 +188,7 @@ int caller() {
 // CHECK: attributes #[[sve2]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+v8a"
 // CHECK: attributes #[[sve2_aes]] = { {{.*}} "target-features"="+aes,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve-aes,+sve2,+sve2-aes,+v8a"
 // CHECK: attributes #[[sve2_bitperm]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve-bitperm,+sve2,+sve2-bitperm,+v8a"
-// CHECK: attributes #[[sve2_sha3]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sha2,+sha3,+sve,+sve2,+sve2-sha3,+v8a"
+// CHECK: attributes #[[sve2_sha3]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sha2,+sha3,+sve,+sve-sha3,+sve2,+sve2-sha3,+v8a"
 // CHECK: attributes #[[sve2_sm4]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sm4,+sve,+sve2,+sve2-sm4,+v8a"
 // CHECK: attributes #[[wfxt]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a,+wfxt"
 // CHECK: attributes #[[cssc]] = { {{.*}} "target-features"="+cssc,+fp-armv8,+neon,+outline-atomics,+v8a"
diff --git a/clang/test/CodeGen/AArch64/neon-intrinsics.c b/clang/test/CodeGen/AArch64/neon-intrinsics.c
index 791f0a1a2940..630424561434 100644
--- a/clang/test/CodeGen/AArch64/neon-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/neon-intrinsics.c
@@ -13112,6 +13112,16 @@ float64x2_t test_vld1q_f64(float64_t const *a) {
   return vld1q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <16 x i8>, ptr [[A]], align 1
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+mfloat8x16_t test_vld1q_mf8(mfloat8_t const *a) {
+  return vld1q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13242,6 +13252,16 @@ float64x1_t test_vld1_f64(float64_t const *a) {
   return vld1_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[A]], align 1
+// CHECK-NEXT:    ret <8 x i8> [[TMP0]]
+//
+mfloat8x8_t test_vld1_mf8(mfloat8_t const *a) {
+  return vld1_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13546,6 +13566,20 @@ float64x2x2_t test_vld2q_f64(float64_t const *a) {
   return vld2q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_mf8(mfloat8_t const *a) {
+  return vld2q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x2_t @test_vld2q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13728,6 +13762,20 @@ float64x1x2_t test_vld2_f64(float64_t const *a) {
   return vld2_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_mf8(mfloat8_t const *a) {
+  return vld2_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x2_t @test_vld2_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -13932,6 +13980,22 @@ float64x2x3_t test_vld3q_f64(float64_t const *a) {
   return vld3q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_mf8(mfloat8_t const *a) {
+  return vld3q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x3_t @test_vld3q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14140,6 +14204,22 @@ float64x1x3_t test_vld3_f64(float64_t const *a) {
   return vld3_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_mf8(mfloat8_t const *a) {
+  return vld3_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x3_t @test_vld3_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14370,6 +14450,24 @@ float64x2x4_t test_vld4q_f64(float64_t const *a) {
   return vld4q_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_mf8(mfloat8_t const *a) {
+  return vld4q_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x4_t @test_vld4q_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14604,6 +14702,24 @@ float64x1x4_t test_vld4_f64(float64_t const *a) {
   return vld4_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_mf8(mfloat8_t const *a) {
+  return vld4_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x4_t @test_vld4_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14732,6 +14848,16 @@ void test_vst1q_s64(int64_t *a, int64x2_t b) {
   vst1q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    store <16 x i8> [[VAL]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8(mfloat8_t *a, mfloat8x16_t val) {
+  vst1q_mf8(a, val);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -14885,6 +15011,16 @@ void test_vst1_s64(int64_t *a, int64x1_t b) {
   vst1_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[VAL:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    store <8 x i8> [[VAL]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8(mfloat8_t *a, mfloat8x8_t val) {
+  vst1_mf8(a, val);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <4 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15067,6 +15203,18 @@ void test_vst2q_s64(int64_t *a, int64x2x2_t b) {
   vst2q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2q_mf8(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst2q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15269,6 +15417,18 @@ void test_vst2_s64(int64_t *a, int64x1x2_t b) {
   vst2_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2_mf8(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst2_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15493,6 +15653,19 @@ void test_vst3q_s64(int64_t *a, int64x2x3_t b) {
   vst3q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3q_mf8(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst3q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15731,6 +15904,19 @@ void test_vst3_s64(int64_t *a, int64x1x3_t b) {
   vst3_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3_mf8(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst3_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -15992,6 +16178,20 @@ void test_vst4q_s64(int64_t *a, int64x2x4_t b) {
   vst4q_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4q_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4q_mf8(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst4q_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4q_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16266,6 +16466,20 @@ void test_vst4_s64(int64_t *a, int64x1x4_t b) {
   vst4_s64(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4_mf8(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst4_mf8(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16398,6 +16612,20 @@ float64x2x2_t test_vld1q_f64_x2(float64_t const *a) {
   return vld1q_f64_x2(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld1q_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x2.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld1q_mf8_x2(mfloat8_t const *a) {
+  return vld1q_mf8_x2(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x2_t @test_vld1q_p64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16426,6 +16654,20 @@ float64x1x2_t test_vld1_f64_x2(float64_t const *a) {
   return vld1_f64_x2(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld1_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x2.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld1_mf8_x2(mfloat8_t const *a) {
+  return vld1_mf8_x2(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x2_t @test_vld1_p64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16456,6 +16698,22 @@ float64x2x3_t test_vld1q_f64_x3(float64_t const *a) {
   return vld1q_f64_x3(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld1q_mf8_x3(
+// CHECK-SAME: ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x3.v16i8.p0(ptr [[PTR]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld1q_mf8_x3(mfloat8_t const *ptr) {
+  return vld1q_mf8_x3(ptr);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x3_t @test_vld1q_p64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16488,6 +16746,23 @@ float64x1x3_t test_vld1_f64_x3(float64_t const *a) {
   return vld1_f64_x3(a);
 }
 
+
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld1_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x3.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld1_mf8_x3(mfloat8_t const *a) {
+  return vld1_mf8_x3(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x3_t @test_vld1_p64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16522,6 +16797,24 @@ float64x2x4_t test_vld1q_f64_x4(float64_t const *a) {
   return vld1q_f64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld1q_mf8_x4(
+// CHECK-SAME: ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld1x4.v16i8.p0(ptr [[PTR]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD1XN]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld1q_mf8_x4(mfloat8_t const *ptr) {
+  return vld1q_mf8_x4(ptr);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x4_t @test_vld1q_p64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16558,6 +16851,24 @@ float64x1x4_t test_vld1_f64_x4(float64_t const *a) {
   return vld1_f64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld1_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD1XN:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld1x4.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD1XN_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 0
+// CHECK-NEXT:    [[VLD1XN_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 1
+// CHECK-NEXT:    [[VLD1XN_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 2
+// CHECK-NEXT:    [[VLD1XN_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD1XN]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD1XN_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD1XN_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD1XN_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD1XN_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld1_mf8_x4(mfloat8_t const *a) {
+  return vld1_mf8_x4(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x4_t @test_vld1_p64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16576,6 +16887,18 @@ poly64x1x4_t test_vld1_p64_x4(poly64_t const *a) {
   return vld1_p64_x4(a);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x2.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x2(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst1q_mf8_x2(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16610,6 +16933,18 @@ void test_vst1q_p64_x2(poly64_t *a, poly64x2x2_t b) {
   vst1q_p64_x2(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x2(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x2.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x2(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst1_mf8_x2(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x2(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16646,6 +16981,19 @@ void test_vst1_p64_x2(poly64_t *a, poly64x1x2_t b) {
   vst1_p64_x2(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x3.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x3(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst1q_mf8_x3(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16687,6 +17035,19 @@ void test_vst1q_p64_x3(poly64_t *a, poly64x2x3_t b) {
   vst1q_p64_x3(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x3(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x3.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x3(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst1_mf8_x3(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x3(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16731,6 +17092,20 @@ void test_vst1_p64_x3(poly64_t *a, poly64x1x3_t b) {
   vst1_p64_x3(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x4.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_mf8_x4(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst1q_mf8_x4(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_f64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <2 x double>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -16779,6 +17154,20 @@ void test_vst1q_p64_x4(poly64_t *a, poly64x2x4_t b) {
   vst1q_p64_x4(a, b);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_mf8_x4(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st1x4.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst1_mf8_x4(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst1_mf8_x4(a, b);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_f64_x4(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <1 x double>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/neon-ldst-one.c b/clang/test/CodeGen/AArch64/neon-ldst-one.c
index 2cff007826ba..9a87e8d2ce6f 100644
--- a/clang/test/CodeGen/AArch64/neon-ldst-one.c
+++ b/clang/test/CodeGen/AArch64/neon-ldst-one.c
@@ -139,6 +139,18 @@ float64x2_t test_vld1q_dup_f64(float64_t  *a) {
   return vld1q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i32 0
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> [[TMP1]], <16 x i32> zeroinitializer
+// CHECK-NEXT:    ret <16 x i8> [[LANE]]
+//
+mfloat8x16_t test_vld1q_dup_mf8(mfloat8_t  *a) {
+  return vld1q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_dup_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -307,6 +319,18 @@ float64x1_t test_vld1_dup_f64(float64_t  *a) {
   return vld1_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i32 0
+// CHECK-NEXT:    [[LANE:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP1]], <8 x i32> zeroinitializer
+// CHECK-NEXT:    ret <8 x i8> [[LANE]]
+//
+mfloat8x8_t test_vld1_dup_mf8(mfloat8_t  *a) {
+  return vld1_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_dup_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -385,6 +409,20 @@ float64x2x2_t test_vld2q_dup_f64(float64_t  *a) {
   return vld2q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_dup_mf8(mfloat8_t  *a) {
+  return vld2q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x2_t @test_vld2q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -413,6 +451,20 @@ float64x1x2_t test_vld2_dup_f64(float64_t  *a) {
   return vld2_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD2:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 0
+// CHECK-NEXT:    [[VLD2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_dup_mf8(mfloat8_t  *a) {
+  return vld2_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x2_t @test_vld2_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -478,6 +530,22 @@ float64x2x3_t test_vld3q_dup_f64(float64_t  *a) {
   // [{{x[0-9]+|sp}}]
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_dup_mf8(mfloat8_t  *a) {
+  return vld3q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x3_t @test_vld3q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -512,6 +580,22 @@ float64x1x3_t test_vld3_dup_f64(float64_t  *a) {
   // [{{x[0-9]+|sp}}]
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD3:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD3_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 0
+// CHECK-NEXT:    [[VLD3_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 1
+// CHECK-NEXT:    [[VLD3_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_dup_mf8(mfloat8_t  *a) {
+  return vld3_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x3_t @test_vld3_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -583,6 +667,24 @@ float64x2x4_t test_vld4q_dup_f64(float64_t  *a) {
   return vld4q_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4r.v16i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_dup_mf8(mfloat8_t  *a) {
+  return vld4q_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x2x4_t @test_vld4q_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -619,6 +721,24 @@ float64x1x4_t test_vld4_dup_f64(float64_t  *a) {
   return vld4_dup_f64(a);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_dup_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VLD4:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4r.v8i8.p0(ptr [[A]])
+// CHECK-NEXT:    [[VLD4_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 0
+// CHECK-NEXT:    [[VLD4_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 1
+// CHECK-NEXT:    [[VLD4_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 2
+// CHECK-NEXT:    [[VLD4_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_dup_mf8(mfloat8_t  *a) {
+  return vld4_dup_mf8(a);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly64x1x4_t @test_vld4_dup_p64(
 // CHECK-SAME: ptr noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -779,6 +899,17 @@ float64x2_t test_vld1q_lane_f64(float64_t  *a, float64x2_t b) {
   return vld1q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <16 x i8> [[B]], i8 [[TMP0]], i32 15
+// CHECK-NEXT:    ret <16 x i8> [[VLD1_LANE]]
+//
+mfloat8x16_t test_vld1q_lane_mf8(mfloat8_t  *a, mfloat8x16_t b) {
+  return vld1q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local <16 x i8> @test_vld1q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -959,6 +1090,17 @@ float64x1_t test_vld1_lane_f64(float64_t  *a, float64x1_t b) {
   return vld1_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local <8 x i8> @test_vld1_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[A]], align 1
+// CHECK-NEXT:    [[VLD1_LANE:%.*]] = insertelement <8 x i8> [[B]], i8 [[TMP0]], i32 7
+// CHECK-NEXT:    ret <8 x i8> [[VLD1_LANE]]
+//
+mfloat8x8_t test_vld1_lane_mf8(mfloat8_t  *a, mfloat8x8_t b) {
+  return vld1_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local <8 x i8> @test_vld1_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1268,6 +1410,22 @@ float64x2x2_t test_vld2q_lane_f64(float64_t  *a, float64x2x2_t b) {
   return vld2q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vld2q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[VLD2_LANE:%.*]] = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld2lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2_LANE]], 0
+// CHECK-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLD2_LANE]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x16x2_t test_vld2q_lane_mf8(mfloat8_t  *a, mfloat8x16x2_t b) {
+  return vld2q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly16x8x2_t @test_vld2q_lane_p16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i16>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1528,6 +1686,22 @@ float64x1x2_t test_vld2_lane_f64(float64_t  *a, float64x1x2_t b) {
   return vld2_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x2_t @test_vld2_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[VLD2_LANE:%.*]] = call { <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld2lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD2_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2_LANE]], 0
+// CHECK-NEXT:    [[VLD2_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8> } [[VLD2_LANE]], 1
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T:%.*]] poison, <8 x i8> [[VLD2_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD2_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X2_T]] [[DOTFCA_0_1_INSERT]]
+//
+mfloat8x8x2_t test_vld2_lane_mf8(mfloat8_t  *a, mfloat8x8x2_t b) {
+  return vld2_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x2_t @test_vld2_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -1818,6 +1992,26 @@ float64x2x3_t test_vld3q_lane_f64(float64_t  *a, float64x2x3_t b) {
   return vld3q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x3_t @test_vld3q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[VLD3_LANE:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld3lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 0
+// CHECK-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 1
+// CHECK-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } [[VLD3_LANE]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T:%.*]] poison, <16 x i8> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x16x3_t test_vld3q_lane_mf8(mfloat8_t  *a, mfloat8x16x3_t b) {
+  return vld3q_lane_mf8(a, b, 15);
+}
+
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x3_t @test_vld3q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2162,6 +2356,25 @@ float64x1x3_t test_vld3_lane_f64(float64_t  *a, float64x1x3_t b) {
   return vld3_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x3_t @test_vld3_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[VLD3_LANE:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld3lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD3_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 0
+// CHECK-NEXT:    [[VLD3_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 1
+// CHECK-NEXT:    [[VLD3_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8> } [[VLD3_LANE]], 2
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T:%.*]] poison, <8 x i8> [[VLD3_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD3_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD3_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X3_T]] [[DOTFCA_0_2_INSERT]]
+//
+mfloat8x8x3_t test_vld3_lane_mf8(mfloat8_t  *a, mfloat8x8x3_t b) {
+  return vld3_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x3_t @test_vld3_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2557,6 +2770,28 @@ float64x2x4_t test_vld4q_lane_f64(float64_t  *a, float64x2x4_t b) {
   return vld4q_lane_f64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x16x4_t @test_vld4q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    [[VLD4_LANE:%.*]] = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.aarch64.neon.ld4lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 0
+// CHECK-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 1
+// CHECK-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 2
+// CHECK-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } [[VLD4_LANE]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T:%.*]] poison, <16 x i8> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_1_INSERT]], <16 x i8> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_2_INSERT]], <16 x i8> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X16X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x16x4_t test_vld4q_lane_mf8(mfloat8_t  *a, mfloat8x16x4_t b) {
+  return vld4q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x16x4_t @test_vld4q_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -2969,6 +3204,28 @@ float64x1x4_t test_vld4_lane_f64(float64_t  *a, float64x1x4_t b) {
   return vld4_lane_f64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local %struct.mfloat8x8x4_t @test_vld4_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    [[VLD4_LANE:%.*]] = call { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @llvm.aarch64.neon.ld4lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    [[VLD4_LANE_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 0
+// CHECK-NEXT:    [[VLD4_LANE_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 1
+// CHECK-NEXT:    [[VLD4_LANE_FCA_2_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 2
+// CHECK-NEXT:    [[VLD4_LANE_FCA_3_EXTRACT:%.*]] = extractvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } [[VLD4_LANE]], 3
+// CHECK-NEXT:    [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T:%.*]] poison, <8 x i8> [[VLD4_LANE_FCA_0_EXTRACT]], 0, 0
+// CHECK-NEXT:    [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_0_INSERT]], <8 x i8> [[VLD4_LANE_FCA_1_EXTRACT]], 0, 1
+// CHECK-NEXT:    [[DOTFCA_0_2_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_1_INSERT]], <8 x i8> [[VLD4_LANE_FCA_2_EXTRACT]], 0, 2
+// CHECK-NEXT:    [[DOTFCA_0_3_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_2_INSERT]], <8 x i8> [[VLD4_LANE_FCA_3_EXTRACT]], 0, 3
+// CHECK-NEXT:    ret [[STRUCT_MFLOAT8X8X4_T]] [[DOTFCA_0_3_INSERT]]
+//
+mfloat8x8x4_t test_vld4_lane_mf8(mfloat8_t  *a, mfloat8x8x4_t b) {
+  return vld4_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local %struct.poly8x8x4_t @test_vld4_lane_p8(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3151,6 +3408,17 @@ void test_vst1q_lane_s64(int64_t  *a, int64x2_t b) {
   vst1q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <16 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <16 x i8> [[B]], i32 15
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1q_lane_mf8(mfloat8_t *a, mfloat8x16_t b) {
+  vst1q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3330,6 +3598,17 @@ void test_vst1_lane_s64(int64_t  *a, int64x1_t b) {
   vst1_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst1_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], <8 x i8> [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = extractelement <8 x i8> [[B]], i32 7
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[A]], align 1
+// CHECK-NEXT:    ret void
+//
+void test_vst1_lane_mf8(mfloat8_t *a, mfloat8x8_t b) {
+  vst1_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst1_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], <4 x half> noundef [[B:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3530,6 +3809,18 @@ void test_vst2q_lane_s64(int64_t  *a, int64x2x2_t b) {
   vst2q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2q_lane_mf8(mfloat8_t *a, mfloat8x16x2_t b) {
+  vst2q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3748,6 +4039,18 @@ void test_vst2_lane_s64(int64_t  *a, int64x1x2_t b) {
   vst2_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst2_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [2 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st2lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst2_lane_mf8(mfloat8_t *a, mfloat8x8x2_t b) {
+  vst2_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst2_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [2 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -3988,6 +4291,19 @@ void test_vst3q_lane_s64(int64_t  *a, int64x2x3_t b) {
   vst3q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3q_lane_mf8(mfloat8_t *a, mfloat8x16x3_t b) {
+  vst3q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4245,6 +4561,19 @@ void test_vst3_lane_s64(int64_t  *a, int64x1x3_t b) {
   vst3_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst3_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [3 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [3 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st3lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst3_lane_mf8(mfloat8_t *a, mfloat8x8x3_t b) {
+  vst3_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst3_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [3 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4525,6 +4854,20 @@ void test_vst4q_lane_s64(int64_t  *a, int64x2x4_t b) {
   vst4q_lane_s64(a, b, 1);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4q_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <16 x i8>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <16 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4lane.v16i8.p0(<16 x i8> [[B_COERCE_FCA_0_EXTRACT]], <16 x i8> [[B_COERCE_FCA_1_EXTRACT]], <16 x i8> [[B_COERCE_FCA_2_EXTRACT]], <16 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 15, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4q_lane_mf8(mfloat8_t *a, mfloat8x16x4_t b) {
+  vst4q_lane_mf8(a, b, 15);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4q_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x half>] alignstack(16) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
@@ -4821,6 +5164,20 @@ void test_vst4_lane_s64(int64_t  *a, int64x1x4_t b) {
   vst4_lane_s64(a, b, 0);
 }
 
+// CHECK-LABEL: define dso_local void @test_vst4_lane_mf8(
+// CHECK-SAME: ptr noundef [[A:%.*]], [4 x <8 x i8>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 0
+// CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 1
+// CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 2
+// CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x <8 x i8>] [[B_COERCE]], 3
+// CHECK-NEXT:    call void @llvm.aarch64.neon.st4lane.v8i8.p0(<8 x i8> [[B_COERCE_FCA_0_EXTRACT]], <8 x i8> [[B_COERCE_FCA_1_EXTRACT]], <8 x i8> [[B_COERCE_FCA_2_EXTRACT]], <8 x i8> [[B_COERCE_FCA_3_EXTRACT]], i64 7, ptr [[A]])
+// CHECK-NEXT:    ret void
+//
+void test_vst4_lane_mf8(mfloat8_t *a, mfloat8x8x4_t b) {
+  vst4_lane_mf8(a, b, 7);
+}
+
 // CHECK-LABEL: define dso_local void @test_vst4_lane_f16(
 // CHECK-SAME: ptr noundef [[A:%.*]], [4 x <4 x half>] alignstack(8) [[B_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
index 2baba98cc505..e7520a504b12 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1ro.c
@@ -3,6 +3,7 @@
 // RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
 // RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +f64mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +f64mm -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
 
 // REQUIRES: aarch64-registered-target
 
@@ -15,6 +16,20 @@
 #define SVE_ACLE_FUNC(A1, A2, A3, A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svld1ro_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1ro_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svld1ro_mf8(svbool_t pg, mfloat8_t const *base) {
+  return SVE_ACLE_FUNC(svld1ro, _mf8, , )(pg, base);
+}
+
 // CHECK-LABEL: @test_svld1ro_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1ro.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
index 9784e1b6b800..0e7455d41327 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ld1rq.c
@@ -23,6 +23,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svld1rq_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svld1rq_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svld1rq_mf8(svbool_t pg, mfloat8_t const *base) MODE_ATTR
+{
+  return SVE_ACLE_FUNC(svld1rq,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svld1rq_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
index a647eb0469f9..ba4091660bfa 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldff1.c
@@ -14,6 +14,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svldff1_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svldff1_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svldff1_mf8(svbool_t pg, const mfloat8_t *base)
+{
+  return SVE_ACLE_FUNC(svldff1,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svldff1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
@@ -197,6 +212,23 @@ svfloat64_t test_svldff1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldff1,_f64,,)(pg, base);
 }
 
+// CHECK-LABEL: @test_svldff1_vnum_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svldff1_vnum_mf8u10__SVBool_tPKu6__mfp8l(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldff1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+svmfloat8_t test_svldff1_vnum_mf8(svbool_t pg, mfloat8_t const *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldff1_vnum,_mf8,,)(pg, base, vnum);
+}
+
 // CHECK-LABEL: @test_svldff1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
index 017f960d3706..8e738d839cd8 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_ldnf1.c
@@ -14,6 +14,21 @@
 #define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
 #endif
 
+// CHECK-LABEL: @test_svldnf1_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svldnf1_mf8u10__SVBool_tPKu6__mfp8(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP0]]
+//
+svmfloat8_t test_svldnf1_mf8(svbool_t pg, mfloat8_t const *base)
+{
+  return SVE_ACLE_FUNC(svldnf1,_mf8,,)(pg, base);
+}
+
 // CHECK-LABEL: @test_svldnf1_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[BASE:%.*]])
@@ -197,6 +212,23 @@ svfloat64_t test_svldnf1_f64(svbool_t pg, const float64_t *base)
   return SVE_ACLE_FUNC(svldnf1,_f64,,)(pg, base);
 }
 
+// CHECK-LABEL: @test_svldnf1_vnum_mf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z21test_svldnf1_vnum_mf8u10__SVBool_tPKu6__mfp8l(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.aarch64.sve.ldnf1.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], ptr [[TMP0]])
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i8> [[TMP1]]
+//
+svmfloat8_t test_svldnf1_vnum_mf8(svbool_t pg, mfloat8_t const *base, int64_t vnum)
+{
+  return SVE_ACLE_FUNC(svldnf1_vnum,_mf8,,)(pg, base, vnum);
+}
+
 // CHECK-LABEL: @test_svldnf1_vnum_s8(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP0:%.*]] = getelementptr <vscale x 16 x i8>, ptr [[BASE:%.*]], i64 [[VNUM:%.*]]
diff --git a/clang/test/CodeGen/LoongArch/bitint.c b/clang/test/CodeGen/LoongArch/bitint.c
new file mode 100644
index 000000000000..f346f569d0eb
--- /dev/null
+++ b/clang/test/CodeGen/LoongArch/bitint.c
@@ -0,0 +1,122 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple loongarch32 -fexperimental-max-bitint-width=1024 -emit-llvm %s -o - | FileCheck %s --check-prefix=LA32
+// RUN: %clang_cc1 -triple loongarch64 -fexperimental-max-bitint-width=1024 -emit-llvm %s -o - | FileCheck %s --check-prefix=LA64
+
+/// Testing the alignment and size of _BitInt(N) for different values of N.
+
+void pass_BitInt7(_BitInt(7));
+void pass_BitInt65(_BitInt(65));
+void pass_BitInt129(_BitInt(129));
+
+// LA32-LABEL: define dso_local void @example_BitInt(
+// LA32-SAME: ) #[[ATTR0:[0-9]+]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[L7:%.*]] = alloca i8, align 1
+// LA32-NEXT:    [[L65:%.*]] = alloca i128, align 16
+// LA32-NEXT:    [[L129:%.*]] = alloca [32 x i8], align 16
+// LA32-NEXT:    [[BYVAL_TEMP:%.*]] = alloca i128, align 16
+// LA32-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca [32 x i8], align 16
+// LA32-NEXT:    store i8 0, ptr [[L7]], align 1
+// LA32-NEXT:    store i128 0, ptr [[L65]], align 16
+// LA32-NEXT:    store i256 0, ptr [[L129]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i8, ptr [[L7]], align 1
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP0]] to i7
+// LA32-NEXT:    call void @pass_BitInt7(i7 noundef signext [[LOADEDV]])
+// LA32-NEXT:    [[TMP1:%.*]] = load i128, ptr [[L65]], align 16
+// LA32-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV1]] to i128
+// LA32-NEXT:    store i128 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
+// LA32-NEXT:    call void @pass_BitInt65(ptr noundef [[BYVAL_TEMP]])
+// LA32-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
+// LA32-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
+// LA32-NEXT:    [[STOREDV4:%.*]] = sext i129 [[LOADEDV2]] to i256
+// LA32-NEXT:    store i256 [[STOREDV4]], ptr [[BYVAL_TEMP3]], align 16
+// LA32-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP3]])
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local void @example_BitInt(
+// LA64-SAME: ) #[[ATTR0:[0-9]+]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    [[L7:%.*]] = alloca i8, align 1
+// LA64-NEXT:    [[L65:%.*]] = alloca i128, align 16
+// LA64-NEXT:    [[L129:%.*]] = alloca i256, align 16
+// LA64-NEXT:    [[BYVAL_TEMP:%.*]] = alloca i256, align 16
+// LA64-NEXT:    store i8 0, ptr [[L7]], align 1
+// LA64-NEXT:    store i128 0, ptr [[L65]], align 16
+// LA64-NEXT:    store i256 0, ptr [[L129]], align 16
+// LA64-NEXT:    [[TMP0:%.*]] = load i8, ptr [[L7]], align 1
+// LA64-NEXT:    [[LOADEDV:%.*]] = trunc i8 [[TMP0]] to i7
+// LA64-NEXT:    call void @pass_BitInt7(i7 noundef signext [[LOADEDV]])
+// LA64-NEXT:    [[TMP1:%.*]] = load i128, ptr [[L65]], align 16
+// LA64-NEXT:    [[LOADEDV1:%.*]] = trunc i128 [[TMP1]] to i65
+// LA64-NEXT:    call void @pass_BitInt65(i65 noundef [[LOADEDV1]])
+// LA64-NEXT:    [[TMP2:%.*]] = load i256, ptr [[L129]], align 16
+// LA64-NEXT:    [[LOADEDV2:%.*]] = trunc i256 [[TMP2]] to i129
+// LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV2]] to i256
+// LA64-NEXT:    store i256 [[STOREDV]], ptr [[BYVAL_TEMP]], align 16
+// LA64-NEXT:    call void @pass_BitInt129(ptr noundef [[BYVAL_TEMP]])
+// LA64-NEXT:    ret void
+//
+void example_BitInt(void) {
+    _BitInt(7) l7 = {0};
+    _BitInt(65) l65 = {0};
+    _BitInt(129) l129 = {0};
+    pass_BitInt7(l7);
+    pass_BitInt65(l65);
+    pass_BitInt129(l129);
+}
+
+// LA32-LABEL: define dso_local signext i7 @return_large_BitInt7(
+// LA32-SAME: ) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    ret i7 0
+//
+// LA64-LABEL: define dso_local signext i7 @return_large_BitInt7(
+// LA64-SAME: ) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    ret i7 0
+//
+_BitInt(7) return_large_BitInt7(void) { return 0; }
+// LA32-LABEL: define dso_local void @return_large_BitInt65(
+// LA32-SAME: ptr dead_on_unwind noalias writable sret(i128) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// LA32-NEXT:    store i128 0, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i128, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i128 [[TMP0]] to i65
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i65 [[LOADEDV]] to i128
+// LA32-NEXT:    store i128 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local i65 @return_large_BitInt65(
+// LA64-SAME: ) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    ret i65 0
+//
+_BitInt(65) return_large_BitInt65(void) { return 0; }
+// LA32-LABEL: define dso_local void @return_large_BitInt129(
+// LA32-SAME: ptr dead_on_unwind noalias writable sret([32 x i8]) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA32-NEXT:  [[ENTRY:.*:]]
+// LA32-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 4
+// LA32-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 4
+// LA32-NEXT:    store i256 0, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[TMP0:%.*]] = load i256, ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
+// LA32-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
+// LA32-NEXT:    store i256 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA32-NEXT:    ret void
+//
+// LA64-LABEL: define dso_local void @return_large_BitInt129(
+// LA64-SAME: ptr dead_on_unwind noalias writable sret(i256) align 16 [[AGG_RESULT:%.*]]) #[[ATTR0]] {
+// LA64-NEXT:  [[ENTRY:.*:]]
+// LA64-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr, align 8
+// LA64-NEXT:    store ptr [[AGG_RESULT]], ptr [[RESULT_PTR]], align 8
+// LA64-NEXT:    store i256 0, ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    [[TMP0:%.*]] = load i256, ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
+// LA64-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
+// LA64-NEXT:    store i256 [[STOREDV]], ptr [[AGG_RESULT]], align 16
+// LA64-NEXT:    ret void
+//
+_BitInt(129) return_large_BitInt129(void) { return 0; }
diff --git a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
index c5ecf0c929af..cab6182ac61c 100644
--- a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
+++ b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs-error.c
@@ -8,13 +8,4 @@ void test(void) {
   register float a1 asm ("$f32");
 // CHECK: :[[#@LINE+1]]:24: error: unknown register name '$foo' in asm
   register int a2 asm ("$foo");
-
-/// Names not prefixed with '$' are invalid.
-
-// CHECK: :[[#@LINE+1]]:26: error: unknown register name 'f0' in asm
-  register float a5 asm ("f0");
-// CHECK: :[[#@LINE+1]]:26: error: unknown register name 'fa0' in asm
-  register float a6 asm ("fa0");
-// CHECK: :[[#@LINE+1]]:15: error: unknown register name 'fcc0' in asm
-  asm ("" ::: "fcc0");
 }
diff --git a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
index e1015f6fc01d..80623c7fc356 100644
--- a/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
+++ b/clang/test/CodeGen/LoongArch/inline-asm-gcc-regs.c
@@ -4,7 +4,8 @@
 /// Check GCC register names and alias can be used in register variable definition.
 
 // CHECK-LABEL: @test_r0
-// CHECK: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r0}"(i32 undef)
 void test_r0() {
     register int a asm ("$r0");
     register int b asm ("r0");
@@ -13,7 +14,8 @@ void test_r0() {
 }
 
 // CHECK-LABEL: @test_r12
-// CHECK: call void asm sideeffect "", "{$r12}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r12}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r12}"(i32 undef)
 void test_r12() {
     register int a asm ("$r12");
     register int b asm ("r12");
@@ -22,7 +24,8 @@ void test_r12() {
 }
 
 // CHECK-LABEL: @test_r31
-// CHECK: call void asm sideeffect "", "{$r31}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r31}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r31}"(i32 undef)
 void test_r31() {
     register int a asm ("$r31");
     register int b asm ("r31");
@@ -31,7 +34,8 @@ void test_r31() {
 }
 
 // CHECK-LABEL: @test_zero
-// CHECK: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r0}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r0}"(i32 undef)
 void test_zero() {
     register int a asm ("$zero");
     register int b asm ("zero");
@@ -40,7 +44,8 @@ void test_zero() {
 }
 
 // CHECK-LABEL: @test_a0
-// CHECK: call void asm sideeffect "", "{$r4}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r4}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r4}"(i32 undef)
 void test_a0() {
     register int a asm ("$a0");
     register int b asm ("a0");
@@ -49,7 +54,8 @@ void test_a0() {
 }
 
 // CHECK-LABEL: @test_t1
-// CHECK: call void asm sideeffect "", "{$r13}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r13}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r13}"(i32 undef)
 void test_t1() {
     register int a asm ("$t1");
     register int b asm ("t1");
@@ -58,7 +64,8 @@ void test_t1() {
 }
 
 // CHECK-LABEL: @test_fp
-// CHECK: call void asm sideeffect "", "{$r22}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r22}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r22}"(i32 undef)
 void test_fp() {
     register int a asm ("$fp");
     register int b asm ("fp");
@@ -67,7 +74,8 @@ void test_fp() {
 }
 
 // CHECK-LABEL: @test_s2
-// CHECK: call void asm sideeffect "", "{$r25}"(i32 undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$r25}"(i32 undef)
+// CHECK-NOT: call void asm sideeffect "", "{$r25}"(i32 undef)
 void test_s2() {
     register int a asm ("$s2");
     register int b asm ("s2");
@@ -76,51 +84,73 @@ void test_s2() {
 }
 
 // CHECK-LABEL: @test_f0
-// CHECK: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f0}"(float undef)
 void test_f0() {
     register float a asm ("$f0");
+    register float b asm ("f0");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_f14
-// CHECK: call void asm sideeffect "", "{$f14}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f14}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f14}"(float undef)
 void test_f14() {
     register float a asm ("$f14");
+    register float b asm ("f14");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_f31
-// CHECK: call void asm sideeffect "", "{$f31}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f31}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f31}"(float undef)
 void test_f31() {
     register float a asm ("$f31");
+    register float b asm ("f31");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fa0
-// CHECK: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f0}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f0}"(float undef)
 void test_fa0() {
     register float a asm ("$fa0");
+    register float b asm ("fa0");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_ft1
-// CHECK: call void asm sideeffect "", "{$f9}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f9}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f9}"(float undef)
 void test_ft1() {
     register float a asm ("$ft1");
+    register float b asm ("ft1");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fs2
-// CHECK: call void asm sideeffect "", "{$f26}"(float undef)
+// CHECK-COUNT-2: call void asm sideeffect "", "{$f26}"(float undef)
+// CHECK-NOT: call void asm sideeffect "", "{$f26}"(float undef)
 void test_fs2() {
     register float a asm ("$fs2");
+    register float b asm ("fs2");
     asm ("" :: "f" (a));
+    asm ("" :: "f" (b));
 }
 
 // CHECK-LABEL: @test_fcc
-// CHECK: call void asm sideeffect "", "~{$fcc0}"()
-// CHECK: call void asm sideeffect "", "~{$fcc7}"()
+// CHECK-COUNT-2: call void asm sideeffect "", "~{$fcc0}"()
+// CHECK-NOT: call void asm sideeffect "", "~{$fcc0}"()
+// CHECK-COUNT-2: call void asm sideeffect "", "~{$fcc7}"()
+// CHECK-NOT: call void asm sideeffect "", "~{$fcc7}"()
 void test_fcc() {
     asm ("" ::: "$fcc0");
+    asm ("" ::: "fcc0");
     asm ("" ::: "$fcc7");
+    asm ("" ::: "fcc7");
 }
diff --git a/clang/test/CodeGen/PowerPC/builtins-bcd-transform.c b/clang/test/CodeGen/PowerPC/builtins-bcd-transform.c
new file mode 100644
index 000000000000..74a8500da6da
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/builtins-bcd-transform.c
@@ -0,0 +1,79 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// Testfile that verifies positive cases (0 or 1 only) for BCD builtins national2packed, packed2zoned and zoned2packed.
+// REQUIRES: powerpc-registered-target
+// RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_National2packed_imm1(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.national2packed(<16 x i8> [[A]], i32 1)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_National2packed_imm1(vector unsigned char a) {
+    return __builtin_ppc_national2packed (a,'\1');
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_National2packed_imm0(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.national2packed(<16 x i8> [[A]], i32 0)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_National2packed_imm0(vector unsigned char a) {
+    return __builtin_ppc_national2packed (a,'\0');
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_Packed2national(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.packed2national(<16 x i8> [[A]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_Packed2national(vector unsigned char a){
+    return __builtin_ppc_packed2national(a);
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_Packed2zoned_imm0(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.packed2zoned(<16 x i8> [[A]], i32 0)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_Packed2zoned_imm0(vector unsigned char a){
+    return __builtin_ppc_packed2zoned(a,'\0');
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_Packed2zoned_imm1(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.packed2zoned(<16 x i8> [[A]], i32 1)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_Packed2zoned_imm1(vector unsigned char a){
+    return __builtin_ppc_packed2zoned(a,'\1');
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_Zoned2packed_imm0(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.zoned2packed(<16 x i8> [[A]], i32 0)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_Zoned2packed_imm0(vector unsigned char a){
+    return __builtin_ppc_zoned2packed(a,'\0');
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @tBcd_Zoned2packed_imm1(
+// CHECK-SAME: <16 x i8> noundef [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.zoned2packed(<16 x i8> [[A]], i32 1)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+vector unsigned char tBcd_Zoned2packed_imm1(vector unsigned char a){
+    return __builtin_ppc_zoned2packed(a,'\1');
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c
new file mode 100644
index 000000000000..5c0606999ef0
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfncvtbf16s.c
@@ -0,0 +1,108 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4(vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf4(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2(vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf2(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1(vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m1(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2(vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m2(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4(vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m4(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_rm
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_rm(vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf4_rm(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_rm
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_rm(vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf2_rm(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_rm
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_rm(vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m1_rm(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_rm
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_rm(vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m2_rm(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_rm
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_rm(vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m4_rm(vs2, __RISCV_FRM_RNE, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
index 48e3a1b0fc29..ec0ad7d23ac8 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
index e435a31eb331..e0627b3e0f6c 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c
new file mode 100644
index 000000000000..b57704207d9d
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/non-overloaded/nds_vfwcvtsbf16.c
@@ -0,0 +1,58 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x float> @test_nds_vfwcvt_s_bf16_f32mf2
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv1f32.nxv1bf16.i64(<vscale x 1 x float> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
+//
+vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2(vbfloat16mf4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32mf2(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x float> @test_nds_vfwcvt_s_bf16_f32m1
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv2f32.nxv2bf16.i64(<vscale x 2 x float> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
+//
+vfloat32m1_t test_nds_vfwcvt_s_bf16_f32m1(vbfloat16mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m1(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x float> @test_nds_vfwcvt_s_bf16_f32m2
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv4f32.nxv4bf16.i64(<vscale x 4 x float> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+vfloat32m2_t test_nds_vfwcvt_s_bf16_f32m2(vbfloat16m1_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m2(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x float> @test_nds_vfwcvt_s_bf16_f32m4
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv8f32.nxv8bf16.i64(<vscale x 8 x float> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
+//
+vfloat32m4_t test_nds_vfwcvt_s_bf16_f32m4(vbfloat16m2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m4(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x float> @test_nds_vfwcvt_s_bf16_f32m8
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv16f32.nxv16bf16.i64(<vscale x 16 x float> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
+//
+vfloat32m8_t test_nds_vfwcvt_s_bf16_f32m8(vbfloat16m4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m8(vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c
new file mode 100644
index 000000000000..c54df913f687
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfncvtbf16s.c
@@ -0,0 +1,108 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4(vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2(vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1(vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2(vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4(vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_rm
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> poison, <vscale x 1 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_rm(vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_rm
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> poison, <vscale x 2 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_rm(vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_rm
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> poison, <vscale x 4 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_rm(vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_rm
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> poison, <vscale x 8 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_rm(vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_rm
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> poison, <vscale x 16 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_rm(vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16(vs2, __RISCV_FRM_RNE, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
index 46c6648bc7a8..686b476b0d59 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
index 62fe9b7bc10e..228d474f2c22 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c
new file mode 100644
index 000000000000..6194a7e82b94
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/non-policy/overloaded/nds_vfwcvtsbf16.c
@@ -0,0 +1,58 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x float> @test_nds_vfwcvt_s_bf16_f32mf2
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv1f32.nxv1bf16.i64(<vscale x 1 x float> poison, <vscale x 1 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
+//
+vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2(vbfloat16mf4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x float> @test_nds_vfwcvt_s_bf16_f32m1
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv2f32.nxv2bf16.i64(<vscale x 2 x float> poison, <vscale x 2 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
+//
+vfloat32m1_t test_nds_vfwcvt_s_bf16_f32m1(vbfloat16mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x float> @test_nds_vfwcvt_s_bf16_f32m2
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv4f32.nxv4bf16.i64(<vscale x 4 x float> poison, <vscale x 4 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+vfloat32m2_t test_nds_vfwcvt_s_bf16_f32m2(vbfloat16m1_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x float> @test_nds_vfwcvt_s_bf16_f32m4
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv8f32.nxv8bf16.i64(<vscale x 8 x float> poison, <vscale x 8 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
+//
+vfloat32m4_t test_nds_vfwcvt_s_bf16_f32m4(vbfloat16m2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s(vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x float> @test_nds_vfwcvt_s_bf16_f32m8
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv16f32.nxv16bf16.i64(<vscale x 16 x float> poison, <vscale x 16 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
+//
+vfloat32m8_t test_nds_vfwcvt_s_bf16_f32m8(vbfloat16m4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s(vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c
new file mode 100644
index 000000000000..ecd5c77911c6
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfncvtbf16s.c
@@ -0,0 +1,108 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_tu
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_tu(vbfloat16mf4_t vd, vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf4_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_tu
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_tu(vbfloat16mf2_t vd, vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf2_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_tu
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_tu(vbfloat16m1_t vd, vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m1_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_tu
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_tu(vbfloat16m2_t vd, vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m2_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_tu
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_tu(vbfloat16m4_t vd, vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m4_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_rm_tu
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_rm_tu(vbfloat16mf4_t vd, vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf4_rm_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_rm_tu
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_rm_tu(vbfloat16mf2_t vd, vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16mf2_rm_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_rm_tu
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_rm_tu(vbfloat16m1_t vd, vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m1_rm_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_rm_tu
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_rm_tu(vbfloat16m2_t vd, vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m2_rm_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_rm_tu
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_rm_tu(vbfloat16m4_t vd, vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_s_bf16m4_rm_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
index bda6a5292ae3..3c6143dc3260 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
index bfcc1e18bd35..f8852fba2f99 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c
new file mode 100644
index 000000000000..433be081bfdb
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/non-overloaded/nds_vfwcvtsbf16.c
@@ -0,0 +1,58 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x float> @test_nds_vfwcvt_s_bf16_f32mf2_tu
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv1f32.nxv1bf16.i64(<vscale x 1 x float> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
+//
+vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2_tu(vfloat32mf2_t vd, vbfloat16mf4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32mf2_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x float> @test_nds_vfwcvt_s_bf16_f32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv2f32.nxv2bf16.i64(<vscale x 2 x float> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
+//
+vfloat32m1_t test_nds_vfwcvt_s_bf16_f32m1_tu(vfloat32m1_t vd, vbfloat16mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m1_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x float> @test_nds_vfwcvt_s_bf16_f32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv4f32.nxv4bf16.i64(<vscale x 4 x float> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+vfloat32m2_t test_nds_vfwcvt_s_bf16_f32m2_tu(vfloat32m2_t vd, vbfloat16m1_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m2_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x float> @test_nds_vfwcvt_s_bf16_f32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv8f32.nxv8bf16.i64(<vscale x 8 x float> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
+//
+vfloat32m4_t test_nds_vfwcvt_s_bf16_f32m4_tu(vfloat32m4_t vd, vbfloat16m2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m4_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x float> @test_nds_vfwcvt_s_bf16_f32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv16f32.nxv16bf16.i64(<vscale x 16 x float> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
+//
+vfloat32m8_t test_nds_vfwcvt_s_bf16_f32m8_tu(vfloat32m8_t vd, vbfloat16m4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_bf16_f32m8_tu(vd, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dots.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dots.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dots.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dots.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotsu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotsu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotsu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotsu.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotu.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotu.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vd4dotu.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vd4dotu.c
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c
new file mode 100644
index 000000000000..fc6f2c96596b
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfncvtbf16s.c
@@ -0,0 +1,120 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_tu
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_tu(vbfloat16mf4_t vd,
+                                                 vfloat32mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_tu
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_tu(vbfloat16mf2_t vd,
+                                                 vfloat32m1_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_tu
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_tu(vbfloat16m1_t vd,
+                                               vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_tu
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_tu(vbfloat16m2_t vd,
+                                               vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_tu
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x float> [[VS2]], i64 7, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_tu(vbfloat16m4_t vd,
+                                               vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf4_rm_tu
+// CHECK-RV64-SAME: (<vscale x 1 x bfloat> [[VD:%.*]], <vscale x 1 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv1bf16.nxv1f32.i64(<vscale x 1 x bfloat> [[VD]], <vscale x 1 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x bfloat> [[TMP0]]
+//
+vbfloat16mf4_t test_nds_vfncvt_bf16_s_bf16mf4_rm_tu(vbfloat16mf4_t vd,
+                                                    vfloat32mf2_t vs2,
+                                                    size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x bfloat> @test_nds_vfncvt_bf16_s_bf16mf2_rm_tu
+// CHECK-RV64-SAME: (<vscale x 2 x bfloat> [[VD:%.*]], <vscale x 2 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv2bf16.nxv2f32.i64(<vscale x 2 x bfloat> [[VD]], <vscale x 2 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x bfloat> [[TMP0]]
+//
+vbfloat16mf2_t test_nds_vfncvt_bf16_s_bf16mf2_rm_tu(vbfloat16mf2_t vd,
+                                                    vfloat32m1_t vs2,
+                                                    size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x bfloat> @test_nds_vfncvt_bf16_s_bf16m1_rm_tu
+// CHECK-RV64-SAME: (<vscale x 4 x bfloat> [[VD:%.*]], <vscale x 4 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv4bf16.nxv4f32.i64(<vscale x 4 x bfloat> [[VD]], <vscale x 4 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x bfloat> [[TMP0]]
+//
+vbfloat16m1_t test_nds_vfncvt_bf16_s_bf16m1_rm_tu(vbfloat16m1_t vd,
+                                                  vfloat32m2_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x bfloat> @test_nds_vfncvt_bf16_s_bf16m2_rm_tu
+// CHECK-RV64-SAME: (<vscale x 8 x bfloat> [[VD:%.*]], <vscale x 8 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv8bf16.nxv8f32.i64(<vscale x 8 x bfloat> [[VD]], <vscale x 8 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x bfloat> [[TMP0]]
+//
+vbfloat16m2_t test_nds_vfncvt_bf16_s_bf16m2_rm_tu(vbfloat16m2_t vd,
+                                                  vfloat32m4_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x bfloat> @test_nds_vfncvt_bf16_s_bf16m4_rm_tu
+// CHECK-RV64-SAME: (<vscale x 16 x bfloat> [[VD:%.*]], <vscale x 16 x float> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x bfloat> @llvm.riscv.nds.vfncvt.bf16.s.nxv16bf16.nxv16f32.i64(<vscale x 16 x bfloat> [[VD]], <vscale x 16 x float> [[VS2]], i64 0, i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x bfloat> [[TMP0]]
+//
+vbfloat16m4_t test_nds_vfncvt_bf16_s_bf16m4_rm_tu(vbfloat16m4_t vd,
+                                                  vfloat32m8_t vs2, size_t vl) {
+  return __riscv_nds_vfncvt_bf16_tu(vd, vs2, __RISCV_FRM_RNE, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
index c7e6abc7a7e1..69e025709ec4 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadb.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadb.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
similarity index 99%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c
rename to clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
index 7e351e096821..a8270fa18cc7 100644
--- a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/nds_vfpmadt.c
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfpmadt.c
@@ -1,7 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
 // REQUIRES: riscv-registered-target
 // RUN: %clang_cc1 -triple riscv64 -target-feature +zve64x \
-// RUN:   -target-feature +zvfhmin \
 // RUN:   -target-feature +xandesvpackfph -disable-O0-optnone  \
 // RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
 // RUN:   FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c
new file mode 100644
index 000000000000..319c470f5c1c
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/andes-intrinsics/policy/overloaded/nds_vfwcvtsbf16.c
@@ -0,0 +1,63 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:   -target-feature +xandesvbfhcvt -disable-O0-optnone \
+// RUN:   -emit-llvm %s -o - | opt -S -passes=mem2reg | \
+// RUN:   FileCheck --check-prefix=CHECK-RV64 %s
+
+#include <andes_vector.h>
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 1 x float> @test_nds_vfwcvt_s_bf16_f32mf2_tu
+// CHECK-RV64-SAME: (<vscale x 1 x float> [[VD:%.*]], <vscale x 1 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 1 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv1f32.nxv1bf16.i64(<vscale x 1 x float> [[VD]], <vscale x 1 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 1 x float> [[TMP0]]
+//
+vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2_tu(vfloat32mf2_t vd,
+                                               vbfloat16mf4_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 2 x float> @test_nds_vfwcvt_s_bf16_f32m1_tu
+// CHECK-RV64-SAME: (<vscale x 2 x float> [[VD:%.*]], <vscale x 2 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 2 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv2f32.nxv2bf16.i64(<vscale x 2 x float> [[VD]], <vscale x 2 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 2 x float> [[TMP0]]
+//
+vfloat32m1_t test_nds_vfwcvt_s_bf16_f32m1_tu(vfloat32m1_t vd,
+                                             vbfloat16mf2_t vs2, size_t vl) {
+  return __riscv_nds_vfwcvt_s_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 4 x float> @test_nds_vfwcvt_s_bf16_f32m2_tu
+// CHECK-RV64-SAME: (<vscale x 4 x float> [[VD:%.*]], <vscale x 4 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 4 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv4f32.nxv4bf16.i64(<vscale x 4 x float> [[VD]], <vscale x 4 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 4 x float> [[TMP0]]
+//
+vfloat32m2_t test_nds_vfwcvt_s_bf16_f32m2_tu(vfloat32m2_t vd, vbfloat16m1_t vs2,
+                                             size_t vl) {
+  return __riscv_nds_vfwcvt_s_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 8 x float> @test_nds_vfwcvt_s_bf16_f32m4_tu
+// CHECK-RV64-SAME: (<vscale x 8 x float> [[VD:%.*]], <vscale x 8 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 8 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv8f32.nxv8bf16.i64(<vscale x 8 x float> [[VD]], <vscale x 8 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 8 x float> [[TMP0]]
+//
+vfloat32m4_t test_nds_vfwcvt_s_bf16_f32m4_tu(vfloat32m4_t vd, vbfloat16m2_t vs2,
+                                             size_t vl) {
+  return __riscv_nds_vfwcvt_s_tu(vd, vs2, vl);
+}
+
+// CHECK-RV64-LABEL: define dso_local <vscale x 16 x float> @test_nds_vfwcvt_s_bf16_f32m8_tu
+// CHECK-RV64-SAME: (<vscale x 16 x float> [[VD:%.*]], <vscale x 16 x bfloat> [[VS2:%.*]], i64 noundef [[VL:%.*]]) #[[ATTR0]] {
+// CHECK-RV64-NEXT:  entry:
+// CHECK-RV64-NEXT:    [[TMP0:%.*]] = call <vscale x 16 x float> @llvm.riscv.nds.vfwcvt.s.bf16.nxv16f32.nxv16bf16.i64(<vscale x 16 x float> [[VD]], <vscale x 16 x bfloat> [[VS2]], i64 [[VL]])
+// CHECK-RV64-NEXT:    ret <vscale x 16 x float> [[TMP0]]
+//
+vfloat32m8_t test_nds_vfwcvt_s_bf16_f32m8_tu(vfloat32m8_t vd, vbfloat16m4_t vs2,
+                                             size_t vl) {
+  return __riscv_nds_vfwcvt_s_tu(vd, vs2, vl);
+}
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/non-overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/non-policy/overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/non-policy/overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/non-overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/non-overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_x_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfnrclip_xu_f_qf_rm.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfwmacc_4x4x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfwmacc_4x4x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vfwmacc_4x4x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vfwmacc_4x4x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmacc_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmacc_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccsu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccsu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccu_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccu_4x8x4.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_2x8x2.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_2x8x2.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_2x8x2.c
diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c b/clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_4x8x4.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/rvv-intrinsics-autogenerated/policy/overloaded/sf_vqmaccus_4x8x4.c
rename to clang/test/CodeGen/RISCV/sifive-intrinsics/policy/overloaded/sf_vqmaccus_4x8x4.c
diff --git a/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll b/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll
new file mode 100644
index 000000000000..85bfc7f74dae
--- /dev/null
+++ b/clang/test/CodeGen/WebAssembly/wasm-exception-model-flag-parse-ir-input.ll
@@ -0,0 +1,17 @@
+; REQUIRES: webassembly-registered-target
+
+; Check all the options parse
+; RUN: %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=none %s | FileCheck %s
+; RUN: %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=wasm -mllvm -wasm-enable-eh %s | FileCheck %s
+
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=invalid %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=dwarf %s 2>&1 | FileCheck -check-prefix=ERR-BE %s
+; RUN: not %clang_cc1 -triple wasm32 -o - -emit-llvm -exception-model=sjlj %s 2>&1 | FileCheck -check-prefix=ERR-BE %s
+
+; CHECK-LABEL: define void @test(
+
+; ERR: error: invalid value 'invalid' in '-exception-model=invalid'
+; ERR-BE: fatal error: error in backend: -exception-model should be either 'none' or 'wasm'
+define void @test() {
+  ret void
+}
diff --git a/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll b/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll
new file mode 100644
index 000000000000..27fb696c9249
--- /dev/null
+++ b/clang/test/CodeGen/WebAssembly/wasm-invalid-exception-kinds.ll
@@ -0,0 +1,8 @@
+; RUN: not %clang_cc1 -triple wasm32 -exception-model=arst -S %s 2>&1 | FileCheck -check-prefix=INVALID-VALUE %s
+
+; Make sure invalid values are rejected for -exception-model when the
+; input is IR.
+
+; INVALID-VALUE: error: invalid value 'arst' in '-exception-model=arst'
+
+target triple = "wasm32"
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch-bad-file.c b/clang/test/CodeGen/X86/ms-secure-hotpatch-bad-file.c
new file mode 100644
index 000000000000..7c8c7d590060
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch-bad-file.c
@@ -0,0 +1,18 @@
+// REQUIRES: x86-registered-target
+
+// This verifies that we correctly handle a -fms-secure-hotpatch-functions-file argument that points
+// to a missing file.
+//
+// RUN: not %clang_cl -c --target=x86_64-windows-msvc -O2 /Z7 -fms-secure-hotpatch-functions-file=%S/this-file-is-intentionally-missing-do-not-create-it.txt /Fo%t.obj -- %s 2>&1 | FileCheck %s
+// CHECK: failed to open hotpatch functions file
+
+void this_might_have_side_effects();
+
+int __declspec(noinline) this_gets_hotpatched() {
+    this_might_have_side_effects();
+    return 42;
+}
+
+int __declspec(noinline) this_does_not_get_hotpatched() {
+    return this_gets_hotpatched() + 100;
+}
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch-cpp.cpp b/clang/test/CodeGen/X86/ms-secure-hotpatch-cpp.cpp
new file mode 100644
index 000000000000..24e1c2937baa
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch-cpp.cpp
@@ -0,0 +1,24 @@
+// REQUIRES: x86-registered-target
+
+// This verifies that hotpatch function attributes are correctly propagated when compiling directly to OBJ,
+// and that name mangling works as expected.
+//
+// RUN: %clang_cl -c --target=x86_64-windows-msvc -O2 /Z7 -fms-secure-hotpatch-functions-list=?this_gets_hotpatched@@YAHXZ /Fo%t.obj -- %s
+// RUN: llvm-readobj --codeview %t.obj | FileCheck %s
+
+void this_might_have_side_effects();
+
+int __declspec(noinline) this_gets_hotpatched() {
+    this_might_have_side_effects();
+    return 42;
+}
+
+// CHECK: Kind: S_HOTPATCHFUNC (0x1169)
+// CHECK-NEXT: Function: this_gets_hotpatched
+// CHECK-NEXT: Name: ?this_gets_hotpatched@@YAHXZ
+
+extern "C" int __declspec(noinline) this_does_not_get_hotpatched() {
+    return this_gets_hotpatched() + 100;
+}
+
+// CHECK-NOT: S_HOTPATCHFUNC
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch-eh.cpp b/clang/test/CodeGen/X86/ms-secure-hotpatch-eh.cpp
new file mode 100644
index 000000000000..66fbc3a950bb
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch-eh.cpp
@@ -0,0 +1,26 @@
+// REQUIRES: x86-registered-target
+
+// Global constant data such as exception handler tables should not be redirected by Windows Secure Hot-Patching
+//
+// RUN: %clang_cl -c --target=x86_64-windows-msvc /EHsc -O2 -fms-secure-hotpatch-functions-list=this_gets_hotpatched /Fo%t.obj /clang:-S /clang:-o- -- %s 2>& 1 | FileCheck %s
+
+class Foo {
+public:
+    int x;
+};
+
+void this_might_throw();
+
+extern "C" int this_gets_hotpatched(int k) {
+    int ret;
+    try {
+        this_might_throw();
+        ret = 1;
+    } catch (Foo& f) {
+        ret = 2;
+    }
+    return ret;
+}
+
+// We expect that RTTI data is not redirected.
+// CHECK-NOT: "__ref_??_R0?AVFoo@@@8"
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch-globals.c b/clang/test/CodeGen/X86/ms-secure-hotpatch-globals.c
new file mode 100644
index 000000000000..ff3a1a47288a
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch-globals.c
@@ -0,0 +1,135 @@
+// REQUIRES: x86-registered-target
+
+// This verifies that global variable redirection works correctly when using hotpatching.
+//
+// RUN: %clang_cl -c --target=x86_64-windows-msvc -O2 /Z7 \
+// RUN:   -fms-secure-hotpatch-functions-list=hp1,hp2,hp3,hp4,hp5_phi_ptr_mixed,hp_phi_ptr_both,hp_const_ptr_sub \
+// RUN:   /clang:-S /clang:-o- -- %s | FileCheck %s
+
+#ifdef __clang__
+#define NO_TAIL __attribute__((disable_tail_calls))
+#else
+#define NO_TAIL
+#endif
+
+extern int g_data[10];
+
+struct SomeData {
+    int x;
+    int y;
+};
+
+const struct SomeData g_this_is_const = { 100, 200 };
+
+struct HasPointers {
+    int* ptr;
+    int x;
+};
+
+extern struct HasPointers g_has_pointers;
+
+void take_data(const void* p);
+
+void do_side_effects();
+void do_other_side_effects();
+
+void hp1() NO_TAIL {
+    take_data(&g_data[5]);
+}
+
+// CHECK: hp1:
+// CHECK: mov rcx, qword ptr [rip + __ref_g_data]
+// CHECK: add rcx, 20
+// CHECK: call take_data
+// CHECK: .seh_endproc
+
+void hp2() NO_TAIL {
+    // We do not expect string literals to be redirected.
+    take_data("hello, world!");
+}
+
+// CHECK: hp2:
+// CHECK: lea rcx, [rip + "??_C@_0O@KJBLMJCB@hello?0?5world?$CB?$AA@"]
+// CHECK: call take_data
+// CHECK: .seh_endproc
+
+void hp3() NO_TAIL {
+    // We do not expect g_this_is_const to be redirected because it is const
+    // and contains no pointers.
+    take_data(&g_this_is_const);
+}
+
+// CHECK: hp3:
+// CHECK: lea rcx, [rip + g_this_is_const]
+// CHECK: call take_data
+// CHECK-NOT: __ref_g_this_is_const
+// CHECK: .seh_endproc
+
+void hp4() NO_TAIL {
+    take_data(&g_has_pointers);
+    // We expect &g_has_pointers to be redirected.
+}
+
+// CHECK: hp4:
+// CHECK: mov rcx, qword ptr [rip + __ref_g_has_pointers]
+// CHECK: call take_data
+// CHECK: .seh_endproc
+
+// This case checks that global variable redirection interacts correctly with PHI nodes.
+// The IR for this generates a "phi ptr g_has_pointers, g_this_is_const" node.
+// We expect g_has_pointers to be redirected, but not g_this_is_const.
+void hp5_phi_ptr_mixed(int x) NO_TAIL {
+    const void* y;
+    if (x) {
+        y = &g_has_pointers;
+        do_side_effects();
+    } else {
+        y = &g_this_is_const;
+        do_other_side_effects();
+    }
+    take_data(y);
+}
+
+// CHECK: hp5_phi_ptr_mixed
+// CHECK: .seh_endprologue
+// CHECK: test ecx, ecx
+// CHECK: mov rsi, qword ptr [rip + __ref_g_has_pointers]
+// CHECK: call do_side_effects
+// CHECK: jmp
+// CHECK: call do_other_side_effects
+// CHECK: lea rsi, [rip + g_this_is_const]
+// CHECK: mov rcx, rsi
+// CHECK: call take_data
+// CHECK: .seh_endproc
+
+// This case tests that global variable redirection interacts correctly with PHI nodes,
+// where two (all) operands of a given PHI node are globabl variables that redirect.
+void hp_phi_ptr_both(int x) NO_TAIL {
+    const void* y;
+    if (x) {
+        y = &g_has_pointers;
+        do_side_effects();
+    } else {
+        y = &g_data[5];
+        do_other_side_effects();
+    }
+    take_data(y);
+}
+
+// CHECK: hp_phi_ptr_both:
+// CHECK: .seh_endprologue
+// CHECK: test ecx, ecx
+// CHECK: mov rsi, qword ptr [rip + __ref_g_has_pointers]
+// CHECK: mov rsi, qword ptr [rip + __ref_g_data]
+// CHECK: take_data
+// CHECK: .seh_endproc
+
+// Test a constant expression which references global variable addresses.
+size_t hp_const_ptr_sub() NO_TAIL {
+    return (unsigned char*)&g_has_pointers - (unsigned char*)&g_data;
+}
+
+// CHECK: hp_const_ptr_sub:
+// CHECK: mov rax, qword ptr [rip + __ref_g_has_pointers]
+// CHECK: sub rax, qword ptr [rip + __ref_g_data]
+// CHECK: ret
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch-lto.c b/clang/test/CodeGen/X86/ms-secure-hotpatch-lto.c
new file mode 100644
index 000000000000..cbf19adb4739
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch-lto.c
@@ -0,0 +1,26 @@
+// REQUIRES: x86-registered-target
+
+// This verifies that hotpatch function attributes are correctly propagated through LLVM IR when compiling with LTO.
+//
+// RUN: %clang_cl -c --target=x86_64-windows-msvc -O2 /Z7 -fms-secure-hotpatch-functions-list=this_gets_hotpatched -flto /Fo%t.bc -- %s
+// RUN: llvm-dis %t.bc -o - | FileCheck %s
+//
+// CHECK-LABEL: define dso_local noundef i32 @this_gets_hotpatched()
+// CHECK-SAME: #0
+//
+// CHECK-LABEL: define dso_local noundef i32 @this_does_not_get_hotpatched()
+// CHECK-SAME: #1
+
+// CHECK: attributes #0
+// CHECK-SAME: "marked_for_windows_hot_patching"
+
+// CHECK: attributes #1
+// CHECK-NOT: "marked_for_windows_hot_patching"
+
+int __declspec(noinline) this_gets_hotpatched() {
+    return 42;
+}
+
+int __declspec(noinline) this_does_not_get_hotpatched() {
+    return this_gets_hotpatched() + 100;
+}
diff --git a/clang/test/CodeGen/X86/ms-secure-hotpatch.c b/clang/test/CodeGen/X86/ms-secure-hotpatch.c
new file mode 100644
index 000000000000..9bc8c2cf364b
--- /dev/null
+++ b/clang/test/CodeGen/X86/ms-secure-hotpatch.c
@@ -0,0 +1,23 @@
+// REQUIRES: x86-registered-target
+
+// This verifies that hotpatch function attributes are correctly propagated when compiling directly to OBJ.
+//
+// RUN: echo this_gets_hotpatched > %t.patch-functions.txt
+// RUN: %clang_cl -c --target=x86_64-windows-msvc -O2 /Z7 -fms-secure-hotpatch-functions-file=%t.patch-functions.txt /Fo%t.obj -- %s
+// RUN: llvm-readobj --codeview %t.obj | FileCheck %s
+
+void this_might_have_side_effects();
+
+int __declspec(noinline) this_gets_hotpatched() {
+    this_might_have_side_effects();
+    return 42;
+}
+
+// CHECK: Kind: S_HOTPATCHFUNC (0x1169)
+// CHECK-NEXT: Function: this_gets_hotpatched
+
+int __declspec(noinline) this_does_not_get_hotpatched() {
+    return this_gets_hotpatched() + 100;
+}
+
+// CHECK-NOT: S_HOTPATCHFUNC
diff --git a/clang/test/CodeGen/amdgpu-builtin-is-invocable.c b/clang/test/CodeGen/amdgpu-builtin-is-invocable.c
index 92755a787ec3..583e38319649 100644
--- a/clang/test/CodeGen/amdgpu-builtin-is-invocable.c
+++ b/clang/test/CodeGen/amdgpu-builtin-is-invocable.c
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --version 5
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCN-GFX900 %s
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx1010 -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCN-GFX1010 %s
 // RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCNSPIRV %s
@@ -11,6 +11,9 @@
 //       externally initialised bool global, and load from it to provide the
 //       condition to a br (abstract target)
 
+//.
+// AMDGCNSPIRV: @llvm.amdgcn.has.gfx10-insts = external addrspace(1) externally_initialized constant i1
+//.
 // AMDGCN-GFX900-LABEL: define dso_local void @foo(
 // AMDGCN-GFX900-SAME: ) #[[ATTR0:[0-9]+]] {
 // AMDGCN-GFX900-NEXT:  [[ENTRY:.*:]]
@@ -25,7 +28,7 @@
 // AMDGCNSPIRV-LABEL: define spir_func void @foo(
 // AMDGCNSPIRV-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
 // AMDGCNSPIRV-NEXT:  [[ENTRY:.*:]]
-// AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load i1, ptr addrspace(1) @llvm.amdgcn.has.gfx10-insts, align 1
+// AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load volatile i1, ptr addrspace(1) @llvm.amdgcn.has.gfx10-insts, align 1
 // AMDGCNSPIRV-NEXT:    [[TOBOOL:%.*]] = icmp ne i1 [[TMP0]], false
 // AMDGCNSPIRV-NEXT:    br i1 [[TOBOOL]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // AMDGCNSPIRV:       [[IF_THEN]]:
@@ -38,3 +41,24 @@ void foo() {
     if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_permlanex16))
         return __builtin_trap();
 }
+//.
+// AMDGCN-GFX900: attributes #[[ATTR0]] = { convergent noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx900" "target-features"="+16-bit-insts,+ci-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+//.
+// AMDGCN-GFX1010: attributes #[[ATTR0]] = { convergent noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1010" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dpp,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize32" }
+// AMDGCN-GFX1010: attributes #[[ATTR1:[0-9]+]] = { cold noreturn nounwind memory(inaccessiblemem: write) }
+//.
+// AMDGCNSPIRV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bf8-cvt-scale-insts,+bitop3-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot13-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+f16bf16-to-fp6bf6-cvt-scale-insts,+f32-to-f16bf16-cvt-sr-insts,+fp4-cvt-scale-insts,+fp6bf6-cvt-scale-insts,+fp8-conversion-insts,+fp8-cvt-scale-insts,+fp8-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+gfx940-insts,+gfx950-insts,+gws,+image-insts,+mai-insts,+permlane16-swap,+permlane32-swap,+prng-inst,+s-memrealtime,+s-memtime-inst,+vmem-to-lds-load-insts,+wavefrontsize32,+wavefrontsize64" }
+// AMDGCNSPIRV: attributes #[[ATTR1:[0-9]+]] = { cold noreturn nounwind memory(inaccessiblemem: write) }
+//.
+// AMDGCN-GFX900: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCN-GFX900: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCN-GFX900: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
+// AMDGCN-GFX1010: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCN-GFX1010: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCN-GFX1010: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
+// AMDGCNSPIRV: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCNSPIRV: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCNSPIRV: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/clang/test/CodeGen/amdgpu-builtin-processor-is.c b/clang/test/CodeGen/amdgpu-builtin-processor-is.c
index fda2bdfbc814..f3c24e66894c 100644
--- a/clang/test/CodeGen/amdgpu-builtin-processor-is.c
+++ b/clang/test/CodeGen/amdgpu-builtin-processor-is.c
@@ -1,4 +1,4 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --version 5
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCN-GFX900 %s
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx1010 -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCN-GFX1010 %s
 // RUN: %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm %s -o - | FileCheck --check-prefix=AMDGCNSPIRV %s
@@ -9,7 +9,9 @@
 //    2) for gfx1010 we emit an empty kernel (concrete target, does not match)
 //    3) for AMDGCNSPIRV we emit llvm.amdgcn.is.gfx900 as a bool global, and
 //       load from it to provide the condition a br (abstract target)
-
+//.
+// AMDGCNSPIRV: @llvm.amdgcn.is.gfx900 = external addrspace(1) externally_initialized constant i1
+//.
 // AMDGCN-GFX900-LABEL: define dso_local void @foo(
 // AMDGCN-GFX900-SAME: ) #[[ATTR0:[0-9]+]] {
 // AMDGCN-GFX900-NEXT:  [[ENTRY:.*:]]
@@ -24,7 +26,7 @@
 // AMDGCNSPIRV-LABEL: define spir_func void @foo(
 // AMDGCNSPIRV-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
 // AMDGCNSPIRV-NEXT:  [[ENTRY:.*:]]
-// AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load i1, ptr addrspace(1) @llvm.amdgcn.is.gfx900, align 1
+// AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load volatile i1, ptr addrspace(1) @llvm.amdgcn.is.gfx900, align 1
 // AMDGCNSPIRV-NEXT:    [[TOBOOL:%.*]] = icmp ne i1 [[TMP0]], false
 // AMDGCNSPIRV-NEXT:    br i1 [[TOBOOL]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
 // AMDGCNSPIRV:       [[IF_THEN]]:
@@ -37,3 +39,24 @@ void foo() {
     if (__builtin_amdgcn_processor_is("gfx900"))
         return __builtin_trap();
 }
+//.
+// AMDGCN-GFX900: attributes #[[ATTR0]] = { convergent noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx900" "target-features"="+16-bit-insts,+ci-insts,+dpp,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+// AMDGCN-GFX900: attributes #[[ATTR1:[0-9]+]] = { cold noreturn nounwind memory(inaccessiblemem: write) }
+//.
+// AMDGCN-GFX1010: attributes #[[ATTR0]] = { convergent noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx1010" "target-features"="+16-bit-insts,+ci-insts,+dl-insts,+dpp,+gfx10-insts,+gfx8-insts,+gfx9-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize32" }
+//.
+// AMDGCNSPIRV: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bf8-cvt-scale-insts,+bitop3-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot13-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+f16bf16-to-fp6bf6-cvt-scale-insts,+f32-to-f16bf16-cvt-sr-insts,+fp4-cvt-scale-insts,+fp6bf6-cvt-scale-insts,+fp8-conversion-insts,+fp8-cvt-scale-insts,+fp8-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+gfx940-insts,+gfx950-insts,+gws,+image-insts,+mai-insts,+permlane16-swap,+permlane32-swap,+prng-inst,+s-memrealtime,+s-memtime-inst,+vmem-to-lds-load-insts,+wavefrontsize32,+wavefrontsize64" }
+// AMDGCNSPIRV: attributes #[[ATTR1:[0-9]+]] = { cold noreturn nounwind memory(inaccessiblemem: write) }
+//.
+// AMDGCN-GFX900: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCN-GFX900: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCN-GFX900: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
+// AMDGCN-GFX1010: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCN-GFX1010: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCN-GFX1010: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
+// AMDGCNSPIRV: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 600}
+// AMDGCNSPIRV: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// AMDGCNSPIRV: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+//.
diff --git a/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp b/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp
index 9669b3469fd1..78f18d3a37b4 100644
--- a/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp
+++ b/clang/test/CodeGen/amdgpu-feature-builtins-invalid-use.cpp
@@ -1,43 +1,48 @@
 // RUN: not %clang_cc1 -triple amdgcn-amd-amdhsa -target-cpu gfx900 -emit-llvm %s -o - 2>&1 | FileCheck %s
 // RUN: not %clang_cc1 -triple spirv64-amd-amdhsa -emit-llvm %s -o - 2>&1 | FileCheck %s
 
-bool predicate(bool x) { return x; }
+bool predicate(bool x);
+void pass_by_value(__amdgpu_feature_predicate_t x);
 
-void invalid_uses(int* p, int x, bool (*pfn)(bool)) {
-    // CHECK: error: cannot initialize a variable of type 'bool' with an rvalue of type 'void'
+void invalid_uses(int *p, int x, const __amdgpu_feature_predicate_t &lv,
+                  __amdgpu_feature_predicate_t &&rv) {
+    // CHECK: error: 'a' has type __amdgpu_feature_predicate_t, which is not constructible
+    __amdgpu_feature_predicate_t a;
+    // CHECK: error: 'b' has type __amdgpu_feature_predicate_t, which is not constructible
+    __amdgpu_feature_predicate_t b = __builtin_amdgcn_processor_is("gfx906");
+    // CHECK: error: 'c' has type __amdgpu_feature_predicate_t, which is not constructible
+    __amdgpu_feature_predicate_t c = lv;
+    // CHECK: error: 'd' has type __amdgpu_feature_predicate_t, which is not constructible
+    __amdgpu_feature_predicate_t d = rv;
+    // CHECK: error: '__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
     bool invalid_use_in_init_0 = __builtin_amdgcn_processor_is("gfx906");
-    // CHECK: error: cannot initialize a variable of type 'const bool' with an rvalue of type 'void'
-    const bool invalid_use_in_init_1 = !__builtin_amdgcn_processor_is("gfx906");
-    // CHECK: error: cannot initialize a variable of type 'bool' with an rvalue of type 'void'
-    bool invalid_use_in_init_2 = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    // CHECK: error: cannot initialize a variable of type 'bool' with an rvalue of type 'void'
-    bool invalid_use_in_init_3 = !__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    // CHECK: error: variable has incomplete type 'const void'
-    const auto invalid_use_in_init_4 = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_wait_event_export_ready) || __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
-    // CHECK: error: variable has incomplete type 'const void'
-    const auto invalid_use_in_init_5 = __builtin_amdgcn_processor_is("gfx906") || __builtin_amdgcn_processor_is("gfx900");
-    // CHECK: error: variable has incomplete type 'const void'
-    const auto invalid_use_in_init_6 = __builtin_amdgcn_processor_is("gfx906") || __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep);
-    // CHECK: error: value of type 'void' is not contextually convertible to 'bool'
-    __builtin_amdgcn_processor_is("gfx1201")
-        ? __builtin_amdgcn_s_sleep_var(x) : __builtin_amdgcn_s_sleep(42);
-    // CHECK: error: no matching function for call to 'predicate'
+    // CHECK: error: 'x' has type __amdgpu_feature_predicate_t, which is not constructible
+    pass_by_value(__builtin_amdgcn_processor_is("gfx906"));
+    // CHECK: error: '__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var)' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
+    bool invalid_use_in_init_1 = __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var);
+    // CHECK: error: '__builtin_amdgcn_processor_is("gfx906")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
+    if (bool invalid_use_in_init_2 = __builtin_amdgcn_processor_is("gfx906")) return;
+    // CHECK: error: '__builtin_amdgcn_processor_is("gfx1200")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
     if (predicate(__builtin_amdgcn_processor_is("gfx1200"))) __builtin_amdgcn_s_sleep_var(x);
-    // CHECK: note: candidate function not viable: cannot convert argument of incomplete type 'void' to 'bool' for 1st argument
 }
 
 void invalid_invocations(int x, const char* str) {
     // CHECK: error: the argument to __builtin_amdgcn_processor_is must be a valid AMDGCN processor identifier; 'not_an_amdgcn_gfx_id' is not valid
+    // CHECK-DAG: note: valid AMDGCN processor identifiers are: {{.*}}gfx{{.*}}
     if (__builtin_amdgcn_processor_is("not_an_amdgcn_gfx_id")) return;
     // CHECK: error: the argument to __builtin_amdgcn_processor_is must be a string literal
     if (__builtin_amdgcn_processor_is(str)) return;
-
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; `'"__builtin_amdgcn_s_sleep_var"'` is not valid
+    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}__builtin_amdgcn_s_sleep_var{{.*}} is not valid
     if (__builtin_amdgcn_is_invocable("__builtin_amdgcn_s_sleep_var")) return;
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; `'str'` is not valid
+    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}str{{.*}} is not valid
     else if (__builtin_amdgcn_is_invocable(str)) return;
-    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; `'x'` is not valid
+    // CHECK: error: the argument to __builtin_amdgcn_is_invocable must be either a target agnostic builtin or an AMDGCN target specific builtin; {{.*}}x{{.*}} is not valid
     else if (__builtin_amdgcn_is_invocable(x)) return;
     // CHECK: error: use of undeclared identifier '__builtin_ia32_pause'
     else if (__builtin_amdgcn_is_invocable(__builtin_ia32_pause)) return;
 }
+
+bool return_needs_cast() {
+    // CHECK: error: '__builtin_amdgcn_processor_is("gfx900")' must be explicitly cast to 'bool'; however, please note that this is almost always an error and that it prevents the effective guarding of target dependent code, and thus should be avoided
+    return __builtin_amdgcn_processor_is("gfx900");
+}
diff --git a/clang/test/CodeGen/arm-neon-directed-rounding.c b/clang/test/CodeGen/arm-neon-directed-rounding.c
index be587ea8e697..08667314e37c 100644
--- a/clang/test/CodeGen/arm-neon-directed-rounding.c
+++ b/clang/test/CodeGen/arm-neon-directed-rounding.c
@@ -16,7 +16,7 @@
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDA_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDA_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrinta.v2f32(<2 x float> [[VRNDA_V_I]])
+// CHECK-A32-NEXT:    [[VRNDA_V1_I:%.*]] = call <2 x float> @llvm.round.v2f32(<2 x float> [[VRNDA_V_I]])
 // CHECK-A32-NEXT:    [[VRNDA_V2_I:%.*]] = bitcast <2 x float> [[VRNDA_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDA_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -41,7 +41,7 @@ float32x2_t test_vrnda_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDAQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrinta.v4f32(<4 x float> [[VRNDAQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <4 x float> @llvm.round.v4f32(<4 x float> [[VRNDAQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDAQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDAQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDAQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -66,7 +66,7 @@ float32x4_t test_vrndaq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDM_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDM_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintm.v2f32(<2 x float> [[VRNDM_V_I]])
+// CHECK-A32-NEXT:    [[VRNDM_V1_I:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[VRNDM_V_I]])
 // CHECK-A32-NEXT:    [[VRNDM_V2_I:%.*]] = bitcast <2 x float> [[VRNDM_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDM_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -91,7 +91,7 @@ float32x2_t test_vrndm_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDMQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintm.v4f32(<4 x float> [[VRNDMQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <4 x float> @llvm.floor.v4f32(<4 x float> [[VRNDMQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDMQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDMQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDMQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -116,7 +116,7 @@ float32x4_t test_vrndmq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDN_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintn.v2f32(<2 x float> [[VRNDN_V_I]])
+// CHECK-A32-NEXT:    [[VRNDN_V1_I:%.*]] = call <2 x float> @llvm.roundeven.v2f32(<2 x float> [[VRNDN_V_I]])
 // CHECK-A32-NEXT:    [[VRNDN_V2_I:%.*]] = bitcast <2 x float> [[VRNDN_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDN_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -141,7 +141,7 @@ float32x2_t test_vrndn_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDNQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintn.v4f32(<4 x float> [[VRNDNQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <4 x float> @llvm.roundeven.v4f32(<4 x float> [[VRNDNQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDNQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDNQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDNQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -166,7 +166,7 @@ float32x4_t test_vrndnq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDP_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDP_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintp.v2f32(<2 x float> [[VRNDP_V_I]])
+// CHECK-A32-NEXT:    [[VRNDP_V1_I:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[VRNDP_V_I]])
 // CHECK-A32-NEXT:    [[VRNDP_V2_I:%.*]] = bitcast <2 x float> [[VRNDP_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDP_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -191,7 +191,7 @@ float32x2_t test_vrndp_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDPQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintp.v4f32(<4 x float> [[VRNDPQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <4 x float> @llvm.ceil.v4f32(<4 x float> [[VRNDPQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDPQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDPQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDPQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -216,7 +216,7 @@ float32x4_t test_vrndpq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRNDX_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRNDX_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintx.v2f32(<2 x float> [[VRNDX_V_I]])
+// CHECK-A32-NEXT:    [[VRNDX_V1_I:%.*]] = call <2 x float> @llvm.rint.v2f32(<2 x float> [[VRNDX_V_I]])
 // CHECK-A32-NEXT:    [[VRNDX_V2_I:%.*]] = bitcast <2 x float> [[VRNDX_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDX_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -241,7 +241,7 @@ float32x2_t test_vrndx_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintx.v4f32(<4 x float> [[VRNDXQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <4 x float> @llvm.rint.v4f32(<4 x float> [[VRNDXQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDXQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDXQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDXQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -266,7 +266,7 @@ float32x4_t test_vrndxq_f32(float32x4_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
 // CHECK-A32-NEXT:    [[VRND_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x float>
-// CHECK-A32-NEXT:    [[VRND_V1_I:%.*]] = call <2 x float> @llvm.arm.neon.vrintz.v2f32(<2 x float> [[VRND_V_I]])
+// CHECK-A32-NEXT:    [[VRND_V1_I:%.*]] = call <2 x float> @llvm.trunc.v2f32(<2 x float> [[VRND_V_I]])
 // CHECK-A32-NEXT:    [[VRND_V2_I:%.*]] = bitcast <2 x float> [[VRND_V1_I]] to <8 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRND_V2_I]] to <2 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <2 x i32> [[TMP2]] to <2 x float>
@@ -291,7 +291,7 @@ float32x2_t test_vrnd_f32(float32x2_t a) {
 // CHECK-A32-NEXT:    [[TMP0:%.*]] = bitcast <4 x float> [[A]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
 // CHECK-A32-NEXT:    [[VRNDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <4 x float>
-// CHECK-A32-NEXT:    [[VRNDQ_V1_I:%.*]] = call <4 x float> @llvm.arm.neon.vrintz.v4f32(<4 x float> [[VRNDQ_V_I]])
+// CHECK-A32-NEXT:    [[VRNDQ_V1_I:%.*]] = call <4 x float> @llvm.trunc.v4f32(<4 x float> [[VRNDQ_V_I]])
 // CHECK-A32-NEXT:    [[VRNDQ_V2_I:%.*]] = bitcast <4 x float> [[VRNDQ_V1_I]] to <16 x i8>
 // CHECK-A32-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDQ_V2_I]] to <4 x i32>
 // CHECK-A32-NEXT:    [[TMP3:%.*]] = bitcast <4 x i32> [[TMP2]] to <4 x float>
@@ -310,24 +310,18 @@ float32x4_t test_vrndq_f32(float32x4_t a) {
   return vrndq_f32(a);
 }
 
-// CHECK-A32-LABEL: define dso_local float @test_vrndns_f32(
-// CHECK-A32-SAME: float noundef [[A:%.*]]) #[[ATTR0]] {
-// CHECK-A32-NEXT:  [[ENTRY:.*:]]
-// CHECK-A32-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.arm.neon.vrintn.f32(float [[A]])
-// CHECK-A32-NEXT:    ret float [[VRNDN_I]]
-//
-// CHECK-A64-LABEL: define dso_local float @test_vrndns_f32(
-// CHECK-A64-SAME: float noundef [[A:%.*]]) #[[ATTR0]] {
-// CHECK-A64-NEXT:  [[ENTRY:.*:]]
-// CHECK-A64-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.roundeven.f32(float [[A]])
-// CHECK-A64-NEXT:    ret float [[VRNDN_I]]
+// CHECK-LABEL: define dso_local float @test_vrndns_f32(
+// CHECK-SAME: float noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[VRNDN_I:%.*]] = call float @llvm.roundeven.f32(float [[A]])
+// CHECK-NEXT:    ret float [[VRNDN_I]]
 //
 float32_t test_vrndns_f32(float32_t a) {
   return vrndns_f32(a);
 }
 
 // CHECK-LABEL: define dso_local <2 x float> @test_vrndi_f32(
-// CHECK-SAME: <2 x float> noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: <2 x float> noundef [[A:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <2 x float> [[A]] to <2 x i32>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to <8 x i8>
diff --git a/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
index bd424172a186..98548749a04d 100644
--- a/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
+++ b/clang/test/CodeGen/arm-sve-vector-bits-vscale-range.c
@@ -1,22 +1,66 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 --check-prefix=CHECK-NOMAX
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=8
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=16
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=2 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=2
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=8 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=8
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=16 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE -D#VBITS=16
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2 -target-feature +sme -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-UNBOUNDED,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-UNBOUNDED,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-NOSTREAMING,CHECK-NOCOMPATIBLE
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=1 -mvscale-streaming-max=1 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=1
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=4 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-streaming-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NONE,CHECK-STREAMING-NOMAX,CHECK-NOCOMPATIBLE -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-min=2 -mvscale-streaming-min=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMAX,CHECK-STREAMING-NOMAX,CHECK-COMPATIBLE-NOMAX -D#VBITS=2 -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme -mvscale-max=2 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-NOMIN,CHECK-STREAMING-NOMIN,CHECK-COMPATIBLE-NOMIN -D#VBITS=2 -D#STREAMINGVBITS=4
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -target-feature +sme \
+// RUN:            -mvscale-min=2 -mvscale-streaming-min=4 -mvscale-max=2 -mvscale-streaming-max=4 -emit-llvm -o - %s | \
+// RUN: FileCheck %s --check-prefixes=CHECK,CHECK-MINMAX,CHECK-STREAMING,CHECK-COMPATIBLE -D#VBITS=2 -D#STREAMINGVBITS=4
+
 
 // CHECK-LABEL: @func() #0
-// CHECK: attributes #0 = { {{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}} }
-// CHECK-NOMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],0) {{.*}} }
-// CHECK-UNBOUNDED: attributes #0 = { {{.*}} vscale_range(1,0) {{.*}} }
+// CHECK-LABEL: @func2() #1
+// CHECK-LABEL: @func3() #2
+// CHECK-MINMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}} }
+// CHECK-NOMAX: attributes #0 = { {{.*}} vscale_range([[#VBITS]],16) {{.*}} }
+// CHECK-NOMIN: attributes #0 = { {{.*}} vscale_range(1,[[#VBITS]]) {{.*}} }
+// CHECK-UNBOUNDED: attributes #0 = { {{.*}} vscale_range(1,16) {{.*}} }
 // CHECK-NONE: attributes #0 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-STREAMING: attributes #1 = { {{.*}} vscale_range([[#STREAMINGVBITS]],[[#STREAMINGVBITS]])
+// CHECK-STREAMING-NOMAX: attributes #1 = { {{.*}} vscale_range([[#STREAMINGVBITS]],16)
+// CHECK-STREAMING-NOMIN: attributes #1 = { {{.*}} vscale_range(1,[[#STREAMINGVBITS]]) {{.*}} }
+// CHECK-NOSTREAMING: attributes #1 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-NOCOMPATIBLE: attributes #2 = { {{.*}} vscale_range(1,16) {{.*}} }
+// CHECK-COMPATIBLE: attributes #2 = { {{.*}} vscale_range([[#VBITS]],[[#STREAMINGVBITS]]) {{.*}} }
+// CHECK-COMPATIBLE-NOMAX: attributes #2 = { {{.*}} vscale_range([[#VBITS]],16) {{.*}} }
+// CHECK-COMPATIBLE-NOMIN: attributes #2 = { {{.*}} vscale_range(1,[[#STREAMINGVBITS]]) {{.*}} }
 void func(void) {}
+__arm_locally_streaming void func2(void) {}
+void func3(void) __arm_streaming_compatible {}
diff --git a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
index f85deeeca757..17d4eef1f763 100644
--- a/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
+++ b/clang/test/CodeGen/arm-v8.2a-neon-intrinsics.c
@@ -522,7 +522,7 @@ float16x8_t test_vrecpeq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRND_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRND_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintz.v4f16(<4 x half> [[VRND_V_I]])
+// CHECK-NEXT:    [[VRND_V1_I:%.*]] = call <4 x half> @llvm.trunc.v4f16(<4 x half> [[VRND_V_I]])
 // CHECK-NEXT:    [[VRND_V2_I:%.*]] = bitcast <4 x half> [[VRND_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRND_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -538,7 +538,7 @@ float16x4_t test_vrnd_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintz.v8f16(<8 x half> [[VRNDQ_V_I]])
+// CHECK-NEXT:    [[VRNDQ_V1_I:%.*]] = call <8 x half> @llvm.trunc.v8f16(<8 x half> [[VRNDQ_V_I]])
 // CHECK-NEXT:    [[VRNDQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -554,7 +554,7 @@ float16x8_t test_vrndq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDA_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDA_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrinta.v4f16(<4 x half> [[VRNDA_V_I]])
+// CHECK-NEXT:    [[VRNDA_V1_I:%.*]] = call <4 x half> @llvm.round.v4f16(<4 x half> [[VRNDA_V_I]])
 // CHECK-NEXT:    [[VRNDA_V2_I:%.*]] = bitcast <4 x half> [[VRNDA_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDA_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -570,7 +570,7 @@ float16x4_t test_vrnda_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDAQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrinta.v8f16(<8 x half> [[VRNDAQ_V_I]])
+// CHECK-NEXT:    [[VRNDAQ_V1_I:%.*]] = call <8 x half> @llvm.round.v8f16(<8 x half> [[VRNDAQ_V_I]])
 // CHECK-NEXT:    [[VRNDAQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDAQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDAQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -586,7 +586,7 @@ float16x8_t test_vrndaq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDM_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDM_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintm.v4f16(<4 x half> [[VRNDM_V_I]])
+// CHECK-NEXT:    [[VRNDM_V1_I:%.*]] = call <4 x half> @llvm.floor.v4f16(<4 x half> [[VRNDM_V_I]])
 // CHECK-NEXT:    [[VRNDM_V2_I:%.*]] = bitcast <4 x half> [[VRNDM_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDM_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -602,7 +602,7 @@ float16x4_t test_vrndm_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDMQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintm.v8f16(<8 x half> [[VRNDMQ_V_I]])
+// CHECK-NEXT:    [[VRNDMQ_V1_I:%.*]] = call <8 x half> @llvm.floor.v8f16(<8 x half> [[VRNDMQ_V_I]])
 // CHECK-NEXT:    [[VRNDMQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDMQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDMQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -618,7 +618,7 @@ float16x8_t test_vrndmq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDN_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDN_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintn.v4f16(<4 x half> [[VRNDN_V_I]])
+// CHECK-NEXT:    [[VRNDN_V1_I:%.*]] = call <4 x half> @llvm.roundeven.v4f16(<4 x half> [[VRNDN_V_I]])
 // CHECK-NEXT:    [[VRNDN_V2_I:%.*]] = bitcast <4 x half> [[VRNDN_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDN_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -634,7 +634,7 @@ float16x4_t test_vrndn_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDNQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintn.v8f16(<8 x half> [[VRNDNQ_V_I]])
+// CHECK-NEXT:    [[VRNDNQ_V1_I:%.*]] = call <8 x half> @llvm.roundeven.v8f16(<8 x half> [[VRNDNQ_V_I]])
 // CHECK-NEXT:    [[VRNDNQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDNQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDNQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -650,7 +650,7 @@ float16x8_t test_vrndnq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDP_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDP_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintp.v4f16(<4 x half> [[VRNDP_V_I]])
+// CHECK-NEXT:    [[VRNDP_V1_I:%.*]] = call <4 x half> @llvm.ceil.v4f16(<4 x half> [[VRNDP_V_I]])
 // CHECK-NEXT:    [[VRNDP_V2_I:%.*]] = bitcast <4 x half> [[VRNDP_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDP_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -666,7 +666,7 @@ float16x4_t test_vrndp_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDPQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintp.v8f16(<8 x half> [[VRNDPQ_V_I]])
+// CHECK-NEXT:    [[VRNDPQ_V1_I:%.*]] = call <8 x half> @llvm.ceil.v8f16(<8 x half> [[VRNDPQ_V_I]])
 // CHECK-NEXT:    [[VRNDPQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDPQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDPQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
@@ -682,7 +682,7 @@ float16x8_t test_vrndpq_f16(float16x8_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8>
 // CHECK-NEXT:    [[VRNDX_V_I:%.*]] = bitcast <8 x i8> [[TMP1]] to <4 x half>
-// CHECK-NEXT:    [[VRNDX_V1_I:%.*]] = call <4 x half> @llvm.arm.neon.vrintx.v4f16(<4 x half> [[VRNDX_V_I]])
+// CHECK-NEXT:    [[VRNDX_V1_I:%.*]] = call <4 x half> @llvm.rint.v4f16(<4 x half> [[VRNDX_V_I]])
 // CHECK-NEXT:    [[VRNDX_V2_I:%.*]] = bitcast <4 x half> [[VRNDX_V1_I]] to <8 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[VRNDX_V2_I]] to <4 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <4 x i16> [[TMP2]] to <4 x half>
@@ -698,7 +698,7 @@ float16x4_t test_vrndx_f16(float16x4_t a) {
 // CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
 // CHECK-NEXT:    [[VRNDXQ_V_I:%.*]] = bitcast <16 x i8> [[TMP1]] to <8 x half>
-// CHECK-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <8 x half> @llvm.arm.neon.vrintx.v8f16(<8 x half> [[VRNDXQ_V_I]])
+// CHECK-NEXT:    [[VRNDXQ_V1_I:%.*]] = call <8 x half> @llvm.rint.v8f16(<8 x half> [[VRNDXQ_V_I]])
 // CHECK-NEXT:    [[VRNDXQ_V2_I:%.*]] = bitcast <8 x half> [[VRNDXQ_V1_I]] to <16 x i8>
 // CHECK-NEXT:    [[TMP2:%.*]] = bitcast <16 x i8> [[VRNDXQ_V2_I]] to <8 x i16>
 // CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <8 x half>
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index e18977a4559b..51e0038b64cd 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -21,6 +21,36 @@ long test_InterlockedAdd_constant(long volatile *Addend) {
 // CHECK-MSVC: ret i32 %[[NEWVAL:[0-9]+]]
 // CHECK-LINUX: error: call to undeclared function '_InterlockedAdd'
 
+long test_InterlockedAdd_acq(long volatile *Addend, long Value) {
+  return _InterlockedAdd_acq(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i32 @test_InterlockedAdd_acq(ptr %Addend, i32 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i32 %2 acquire, align 4
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i32 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i32 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd_acq'
+
+long test_InterlockedAdd_nf(long volatile *Addend, long Value) {
+  return _InterlockedAdd_nf(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i32 @test_InterlockedAdd_nf(ptr %Addend, i32 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i32 %2 monotonic, align 4
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i32 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i32 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd_nf'
+
+long test_InterlockedAdd_rel(long volatile *Addend, long Value) {
+  return _InterlockedAdd_rel(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i32 @test_InterlockedAdd_rel(ptr %Addend, i32 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i32 %2 release, align 4
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i32 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i32 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd_rel'
+
 __int64 test_InterlockedAdd64(__int64 volatile *Addend, __int64 Value) {
   return _InterlockedAdd64(Addend, Value);
 }
@@ -35,6 +65,36 @@ __int64 test_InterlockedAdd64_constant(__int64 volatile *Addend) {
 // CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
 // CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64'
 
+__int64 test_InterlockedAdd64_acq(__int64 volatile *Addend, __int64 Value) {
+  return _InterlockedAdd64_acq(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i64 @test_InterlockedAdd64_acq(ptr %Addend, i64 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i64 %2 acquire, align 8
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i64 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64_acq'
+
+__int64 test_InterlockedAdd64_nf(__int64 volatile *Addend, __int64 Value) {
+  return _InterlockedAdd64_nf(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i64 @test_InterlockedAdd64_nf(ptr %Addend, i64 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i64 %2 monotonic, align 8
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i64 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64_nf'
+
+__int64 test_InterlockedAdd64_rel(__int64 volatile *Addend, __int64 Value) {
+  return _InterlockedAdd64_rel(Addend, Value);
+}
+
+// CHECK-LABEL: define {{.*}} i64 @test_InterlockedAdd64_rel(ptr %Addend, i64 %Value) {{.*}} {
+// CHECK-MSVC: %[[OLDVAL:[0-9]+]] = atomicrmw add ptr %1, i64 %2 release, align 8
+// CHECK-MSVC: %[[NEWVAL:[0-9]+]] = add i64 %[[OLDVAL:[0-9]+]], %2
+// CHECK-MSVC: ret i64 %[[NEWVAL:[0-9]+]]
+// CHECK-LINUX: error: call to undeclared function '_InterlockedAdd64_rel'
+
 void check_ReadWriteBarrier(void) {
   _ReadWriteBarrier();
 }
diff --git a/clang/test/CodeGen/arm64-microsoft-sys.c b/clang/test/CodeGen/arm64-microsoft-sys.c
new file mode 100644
index 000000000000..5913a02d933a
--- /dev/null
+++ b/clang/test/CodeGen/arm64-microsoft-sys.c
@@ -0,0 +1,65 @@
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple arm64-windows -fms-compatibility -S \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-ASM
+
+// RUN: %clang_cc1 -triple arm64-windows -fms-compatibility -emit-llvm \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-IR
+
+// RUN: %clang_cc1 -triple arm64-darwin -fms-compatibility -emit-llvm \
+// RUN: -o - %s | FileCheck %s -check-prefix CHECK-IR
+
+// From winnt.h
+// op0=1 encodings, use with __sys
+#define ARM64_SYSINSTR(op0, op1, crn, crm, op2) \
+        ( ((op1 & 7) << 11) | \
+          ((crn & 15) << 7) | \
+          ((crm & 15) << 3) | \
+          ((op2 & 7) << 0) )
+
+//
+// Sampling of instructions
+//
+#define ARM64_DC_CGDSW_EL1      ARM64_SYSINSTR(1,0, 7,10,6) // Clean of Data and Allocation Tags by Set/Way
+#define ARM64_IC_IALLU_EL1      ARM64_SYSINSTR(1,0, 7, 5,0) // Instruction Cache Invalidate All to PoU
+#define ARM64_AT_S1E2W          ARM64_SYSINSTR(1,4, 7, 8,1) // Translate Stage1, EL2, write
+#define ARM64_TLBI_VMALLE1      ARM64_SYSINSTR(1,0, 8, 7,0) // Invalidate stage 1 TLB [CP15_TLBIALL]
+#define ARM64_CFP_RCTX          ARM64_SYSINSTR(1,3, 7, 3,4) // Control Flow Prediction Restriction by Context
+
+// From intrin.h
+unsigned int __sys(int, __int64);
+
+void check__sys(__int64 v) {
+  __int64 ret;
+
+  __sys(ARM64_DC_CGDSW_EL1, v);
+// CHECK-ASM: msr     S1_0_C7_C10_6, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD2:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_IC_IALLU_EL1, v);
+// CHECK-ASM: msr     S1_0_C7_C5_0, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD3:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_AT_S1E2W, v);
+// CHECK-ASM: msr     S1_4_C7_C8_1, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD4:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_TLBI_VMALLE1, v);
+// CHECK-ASM: msr     S1_0_C8_C7_0, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD5:.*]], i64 %[[VAR]])
+
+  __sys(ARM64_CFP_RCTX, v);
+// CHECK-ASM: msr     S1_3_C7_C3_4, x8
+// CHECK-IR: %[[VAR:.*]] = load i64,
+// CHECK-IR-NEXT: call void @llvm.write_register.i64(metadata ![[MD6:.*]], i64 %[[VAR]])
+}
+
+// CHECK-IR: ![[MD2]] = !{!"1:0:7:10:6"}
+// CHECK-IR: ![[MD3]] = !{!"1:0:7:5:0"}
+// CHECK-IR: ![[MD4]] = !{!"1:4:7:8:1"}
+// CHECK-IR: ![[MD5]] = !{!"1:0:8:7:0"}
+// CHECK-IR: ![[MD6]] = !{!"1:3:7:3:4"}
diff --git a/clang/test/CodeGen/attr-target-mv.c b/clang/test/CodeGen/attr-target-mv.c
index 6911b55203b7..07f47d93cd29 100644
--- a/clang/test/CodeGen/attr-target-mv.c
+++ b/clang/test/CodeGen/attr-target-mv.c
@@ -201,6 +201,8 @@ void calls_pr50025c(void) { pr50025c(); }
 // ITANIUM: ret i32 25
 // ITANIUM: define{{.*}} i32 @foo.arch_clearwaterforest()
 // ITANIUM: ret i32 26
+// ITANIUM: define{{.*}} i32 @foo.arch_diamondrapids()
+// ITANIUM: ret i32 27
 // ITANIUM: define{{.*}} i32 @foo()
 // ITANIUM: ret i32 2
 // ITANIUM: define{{.*}} i32 @bar()
@@ -258,6 +260,8 @@ void calls_pr50025c(void) { pr50025c(); }
 // WINDOWS: ret i32 25
 // WINDOWS: define dso_local i32 @foo.arch_clearwaterforest()
 // WINDOWS: ret i32 26
+// WINDOWS: define dso_local i32 @foo.arch_diamondrapids()
+// WINDOWS: ret i32 27
 // WINDOWS: define dso_local i32 @foo()
 // WINDOWS: ret i32 2
 // WINDOWS: define dso_local i32 @bar()
diff --git a/clang/test/CodeGen/bitfield-access-pad.c b/clang/test/CodeGen/bitfield-access-pad.c
index 8608c5bd8be1..2044746118ad 100644
--- a/clang/test/CodeGen/bitfield-access-pad.c
+++ b/clang/test/CodeGen/bitfield-access-pad.c
@@ -18,7 +18,7 @@
 // RUN: %clang_cc1 -triple=hexagon-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
 
 // Big endian
-// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
+// RUN: %clang_cc1 -triple=m68k-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-M68K %s
 // RUN: %clang_cc1 -triple=mips-elf %s -emit-llvm -o /dev/null -fdump-record-layouts-simple | FileCheck --check-prefixes CHECK,LAYOUT-T %s
 
 // And now a few with -fno-bitfield-type-align. Precisely how this behaves is
@@ -45,6 +45,7 @@ struct P1 {
 // CHECK-LABEL: LLVMType:%struct.P1 =
 // LAYOUT-T-SAME: type { i8, i8, [2 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -60,6 +61,9 @@ struct P1 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:1
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:1
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -75,6 +79,7 @@ struct P2 {
 // CHECK-LABEL: LLVMType:%struct.P2 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -90,6 +95,9 @@ struct P2 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -105,6 +113,7 @@ struct P3 {
 // CHECK-LABEL: LLVMType:%struct.P3 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -120,6 +129,9 @@ struct P3 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -134,6 +146,7 @@ struct P4 {
 // CHECK-LABEL: LLVMType:%struct.P4 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -149,6 +162,9 @@ struct P4 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -162,6 +178,7 @@ struct P5 {
 // CHECK-LABEL: LLVMType:%struct.P5 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -177,6 +194,9 @@ struct P5 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -192,6 +212,7 @@ struct P6 {
 // CHECK-LABEL: LLVMType:%struct.P6 =
 // LAYOUT-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, [3 x i8], i8, [3 x i8] }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -207,6 +228,9 @@ struct P6 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -220,6 +244,7 @@ struct P7 {
 // CHECK-LABEL: LLVMType:%struct.P7 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -235,6 +260,9 @@ struct P7 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -250,6 +278,7 @@ struct __attribute__ ((aligned (2))) P7_align {
 // CHECK-LABEL: LLVMType:%struct.P7_align =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i8, i8 }
 // LAYOUT-STRICT-NT-SAME: type { i8, i8 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -265,6 +294,9 @@ struct __attribute__ ((aligned (2))) P7_align {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:8 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -278,6 +310,7 @@ struct P8 {
 // CHECK-LABEL: LLVMType:%struct.P8 =
 // LAYOUT-T-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-ARM64-T-SAME: type { i8, i8, i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8, i8, i8 }
 // LAYOUT-NT-SAME: type { i16 }
 // LAYOUT-STRICT-NT-SAME: type { i16 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -293,6 +326,9 @@ struct P8 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:2
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:2
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -306,6 +342,7 @@ struct P9 {
 // CHECK-LABEL: LLVMType:%struct.P9 =
 // LAYOUT-T-SAME: type { i8, i8, [2 x i8] }
 // LAYOUT-ARM64-T-SAME: type { i8, i8 }
+// LAYOUT-M68K-SAME: type { i8, i8 }
 // LAYOUT-NT-SAME: type { i16 }
 // LAYOUT-STRICT-NT-SAME: type { i16 }
 // LAYOUT-DWN32-SAME: type { i8, [3 x i8], i8, [3 x i8] }
@@ -321,6 +358,9 @@ struct P9 {
 
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:1
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:1
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:8 StorageOffset:4
@@ -335,6 +375,7 @@ struct __attribute__((aligned(4))) P10 {
 // CHECK-LABEL: LLVMType:%struct.P10 =
 // LAYOUT-T-SAME: type { i32 }
 // LAYOUT-ARM64-T-SAME: type { i32 }
+// LAYOUT-M68K-SAME: type { i32 }
 // LAYOUT-NT-SAME: type { i32 }
 // LAYOUT-STRICT-NT-SAME: type { i32 }
 // LAYOUT-DWN32-SAME: type { i32 }
@@ -354,6 +395,10 @@ struct __attribute__((aligned(4))) P10 {
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
@@ -369,6 +414,7 @@ struct __attribute__((aligned(4))) P11 {
 // CHECK-LABEL: LLVMType:%struct.P11 =
 // LAYOUT-T-SAME: type { i32 }
 // LAYOUT-ARM64-T-SAME: type { i32 }
+// LAYOUT-M68K-SAME: type { i32 }
 // LAYOUT-NT-SAME: type { i32 }
 // LAYOUT-STRICT-NT-SAME: type { i32 }
 // LAYOUT-DWN32-SAME: type { i32 }
@@ -388,6 +434,10 @@ struct __attribute__((aligned(4))) P11 {
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-ARM64-T-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:10 IsSigned:0 StorageSize:32 StorageOffset:0
+//
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
+// LAYOUT-M68K-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:10 IsSigned:0 StorageSize:32 StorageOffset:0
 
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:0 StorageSize:32 StorageOffset:0
diff --git a/clang/test/CodeGen/bittest-intrin.c b/clang/test/CodeGen/bittest-intrin.c
index 5641b04cf86e..bc5ec6773450 100644
--- a/clang/test/CodeGen/bittest-intrin.c
+++ b/clang/test/CodeGen/bittest-intrin.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -fms-extensions -triple x86_64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=X64
 // RUN: %clang_cc1 -fms-extensions -triple thumbv7-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
-// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
+// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM64 -check-prefix=ARM
 
 volatile unsigned char sink = 0;
 void test32(long *base, long idx) {
@@ -10,7 +10,6 @@ void test32(long *base, long idx) {
   sink = _bittestandset(base, idx);
   sink = _interlockedbittestandreset(base, idx);
   sink = _interlockedbittestandset(base, idx);
-  sink = _interlockedbittestandset(base, idx);
 }
 
 void test64(__int64 *base, __int64 idx) {
@@ -33,6 +32,17 @@ void test_arm(long *base, long idx) {
 }
 #endif
 
+#if defined(_M_ARM64)
+void test_arm64(__int64 *base, __int64 idx) {
+  sink = _interlockedbittestandreset64_acq(base, idx);
+  sink = _interlockedbittestandreset64_rel(base, idx);
+  sink = _interlockedbittestandreset64_nf(base, idx);
+  sink = _interlockedbittestandset64_acq(base, idx);
+  sink = _interlockedbittestandset64_rel(base, idx);
+  sink = _interlockedbittestandset64_nf(base, idx);
+}
+#endif
+
 // X64-LABEL: define dso_local void @test32(ptr noundef %base, i32 noundef %idx)
 // X64: call i8 asm sideeffect "btl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
 // X64: call i8 asm sideeffect "btcl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
@@ -117,13 +127,122 @@ void test_arm(long *base, long idx) {
 // ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
 // ARM: store volatile i8 %[[RES]], ptr @sink, align 1
 
+// ARM-LABEL: define dso_local {{.*}}void @test64(ptr noundef %base, i64 noundef %idx)
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
 
-// Just look for the atomicrmw instructions.
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = xor i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[NEWBYTE:[^ ]*]] = and i8 %[[BYTE]], %[[NOTMASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = or i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
 
 // ARM-LABEL: define dso_local {{.*}}void @test_arm(ptr noundef %base, i32 noundef %idx)
-// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
 // ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
 // ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
-// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
 // ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
 // ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+
+// ARM64-LABEL: define dso_local void @test_arm64(ptr noundef %base, i64 noundef %idx)
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
diff --git a/clang/test/CodeGen/builtins-overflow.c b/clang/test/CodeGen/builtins-overflow.c
index 7c524723f76e..0e04191b9e2a 100644
--- a/clang/test/CodeGen/builtins-overflow.c
+++ b/clang/test/CodeGen/builtins-overflow.c
@@ -604,3 +604,15 @@ long long test_mixed_sign_mul_overflow_extend_unsigned(int x, unsigned y) {
     return LongLongErrorCode;
   return result;
 }
+
+_BitInt(65) test_mixed_sign_mul_overflow_bitint(unsigned _BitInt(65) y, _BitInt(119) a) {
+// CHECK: call { i119, i1 } @llvm.umul.with.overflow.i119
+// CHECK: select i1 %{{.*}}, i119 %{{.*}}, i119 %{{.*}}
+// CHECK: trunc i119
+// CHECK: zext i65
+// CHECK: store
+  unsigned _BitInt(65) result;
+  if (__builtin_mul_overflow(a, y, &result))
+    return LongLongErrorCode;
+  return result;
+}
diff --git a/clang/test/CodeGen/debug-label-inline.c b/clang/test/CodeGen/debug-label-inline.c
index 972a32b5af32..9d92ffb5ef61 100644
--- a/clang/test/CodeGen/debug-label-inline.c
+++ b/clang/test/CodeGen/debug-label-inline.c
@@ -23,6 +23,6 @@ int f2(void) {
 
 // CHECK: distinct !DISubprogram(name: "f1", {{.*}}, retainedNodes: [[ELEMENTS:!.*]])
 // CHECK: [[ELEMENTS]] = !{{{.*}}, [[LABEL_METADATA]]}
-// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 8)
+// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 8, column: 1)
 // CHECK: [[INLINEDAT:!.*]] = distinct !DILocation(line: 18,
 // CHECK: [[LABEL_LOCATION]] = !DILocation(line: 8, {{.*}}, inlinedAt: [[INLINEDAT]])
diff --git a/clang/test/CodeGen/debug-label.c b/clang/test/CodeGen/debug-label.c
index 662b1a7dd206..308d664ab712 100644
--- a/clang/test/CodeGen/debug-label.c
+++ b/clang/test/CodeGen/debug-label.c
@@ -12,5 +12,5 @@ int f1(int a, int b) {
   return sum;
 }
 
-// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 9)
+// CHECK: [[LABEL_METADATA]] = !DILabel({{.*}}, name: "top", {{.*}}, line: 9, column: 1)
 // CHECK: [[LABEL_LOCATION]] = !DILocation(line: 9,
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index 14efd54e24ff..55d4c438eeda 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -27,8 +27,8 @@
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm64_32-apple-ios -target-abi darwinpcs -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=AARCH64DARWIN
 // RUN: %clang_cc1 -no-enable-noundef-analysis -triple arm -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=ARM
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch64 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA64
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch32 -O3 -disable-llvm-passes -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA32
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch64 -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA64
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple loongarch32 -O3 -disable-llvm-passes -fexperimental-max-bitint-width=1024 -emit-llvm -o - %s | FileCheck %s --check-prefixes=LA32
 
 // Make sure 128 and 64 bit versions are passed like integers.
 void ParamPassing(_BitInt(128) b, _BitInt(64) c) {}
@@ -158,8 +158,8 @@ void ParamPassing4(_BitInt(129) a) {}
 // PPC32-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // AARCH64DARWIN-NOT: define{{.*}} void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
 // ARM-NOT: define{{.*}} arm_aapcscc void @ParamPassing4(ptr byval(i129) align 8 %{{.+}})
-// LA64-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
-// LA32-NOT: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA64: define{{.*}} void @ParamPassing4(ptr %{{.+}})
+// LA32: define{{.*}} void @ParamPassing4(ptr %{{.+}})
 #endif
 
 _BitInt(63) ReturnPassing(void) { return 0; }
@@ -317,8 +317,8 @@ _BitInt(129) ReturnPassing5(void) { return 0; }
 // PPC32-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // AARCH64DARWIN-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 // ARM-NOT: define{{.*}} arm_aapcscc void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-// LA64-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
-// LA32-NOT: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
+// LA64: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
+// LA32: define{{.*}} void @ReturnPassing5(ptr dead_on_unwind noalias writable sret
 
 // SparcV9 is odd in that it has a return-size limit of 256, not 128 or 64
 // like other platforms, so test to make sure this behavior will still work.
diff --git a/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c b/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c
new file mode 100644
index 000000000000..0f1cdf951239
--- /dev/null
+++ b/clang/test/CodeGen/fallow-runtime-check-skip-hot-cutoff.c
@@ -0,0 +1,15 @@
+// RUN: %clang -fallow-runtime-check-skip-hot-cutoff=1.0 -S -emit-llvm %s -o - -O2 | FileCheck --check-prefix=ONE %s
+// RUN: %clang -fallow-runtime-check-skip-hot-cutoff=0.0 -S -emit-llvm %s -o - -O2 | FileCheck --check-prefix=ZERO %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=6.0 -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=SIX %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=-1.0 -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=MINUSONE %s
+// RUN: not %clang -fallow-runtime-check-skip-hot-cutoff=string -S -emit-llvm %s -o - -O2 2>&1 | FileCheck --check-prefix=STRING %s
+
+// ONE: ret i32 0
+// ZERO: ret i32 1
+// SIX: invalid value '6.0' in '-fallow-runtime-check-skip-hot-cutoff='
+// MINUSONE: invalid value '-1.0' in '-fallow-runtime-check-skip-hot-cutoff='
+// STRING: invalid value 'string' in '-fallow-runtime-check-skip-hot-cutoff='
+
+int main(int argc, char** argv) {
+    return __builtin_allow_runtime_check("foo");
+}
diff --git a/clang/test/CodeGen/logb_scalbn.c b/clang/test/CodeGen/logb_scalbn.c
index be5e68b5fd4b..52c52bcb292b 100644
--- a/clang/test/CodeGen/logb_scalbn.c
+++ b/clang/test/CodeGen/logb_scalbn.c
@@ -4,6 +4,11 @@
 // RUN: %clang -cc1 -triple amdgcn-amd-amdhsa -o - -ffp-exception-behavior=strict -emit-llvm %s | FileCheck %s -check-prefixes=STRICT
 // RUN: %clang -cc1 -triple amdgcn-amd-amdhsa -o - -ffp-exception-behavior=maytrap -emit-llvm %s | FileCheck %s -check-prefixes=MAYTRAP
 // RUN: %clang -cc1 -triple amdgcn-amd-amdhsa -o - -fmath-errno -emit-llvm %s | FileCheck %s -check-prefixes=ERRNO
+// RUN: %clang -cc1 -triple spirv64-amd-amdhsa -o - -emit-llvm %s | FileCheck %s -check-prefixes=AMDGCNSPIRV-DEFAULT
+// RUN: %clang -cc1 -triple spirv64-amd-amdhsa -o - -ffp-exception-behavior=ignore -emit-llvm %s | FileCheck %s -check-prefixes=AMDGCNSPIRV-IGNORE
+// RUN: %clang -cc1 -triple spirv64-amd-amdhsa -o - -ffp-exception-behavior=strict -emit-llvm %s | FileCheck %s -check-prefixes=AMDGCNSPIRV-STRICT
+// RUN: %clang -cc1 -triple spirv64-amd-amdhsa -o - -ffp-exception-behavior=maytrap -emit-llvm %s | FileCheck %s -check-prefixes=AMDGCNSPIRV-MAYTRAP
+// RUN: %clang -cc1 -triple spirv64-amd-amdhsa -o - -fmath-errno -emit-llvm %s | FileCheck %s -check-prefixes=AMDGCNSPIRV-ERRNO
 
 // DEFAULT-LABEL: define dso_local void @test_logbf(
 // DEFAULT-SAME: ) #[[ATTR0:[0-9]+]] {
@@ -78,6 +83,79 @@
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_logbf(
+// AMDGCNSPIRV-DEFAULT-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float 0x40301999A0000000)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = extractvalue { float, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP4:%.*]] = call addrspace(4) float @llvm.fabs.f32(float 0x40301999A0000000)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP5:%.*]] = fcmp one float [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP7:%.*]] = select i1 false, float 0xFFF0000000000000, float [[TMP6]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_logbf(
+// AMDGCNSPIRV-IGNORE-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float 0x40301999A0000000)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = extractvalue { float, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP4:%.*]] = call addrspace(4) float @llvm.fabs.f32(float 0x40301999A0000000)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP5:%.*]] = fcmp one float [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP7:%.*]] = select i1 false, float 0xFFF0000000000000, float [[TMP6]]
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_logbf(
+// AMDGCNSPIRV-STRICT-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float 0x40301999A0000000)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = extractvalue { float, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP4:%.*]] = call addrspace(4) float @llvm.fabs.f32(float 0x40301999A0000000)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP5:%.*]] = fcmp one float [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP7:%.*]] = select i1 false, float 0xFFF0000000000000, float [[TMP6]]
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_logbf(
+// AMDGCNSPIRV-MAYTRAP-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float 0x40301999A0000000)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = extractvalue { float, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to float
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP4:%.*]] = call addrspace(4) float @llvm.fabs.f32(float 0x40301999A0000000)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP5:%.*]] = fcmp one float [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP4]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP7:%.*]] = select i1 false, float 0xFFF0000000000000, float [[TMP6]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_logbf(
+// AMDGCNSPIRV-ERRNO-SAME: ) addrspace(4) #[[ATTR0:[0-9]+]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @logbf(float noundef 0x40301999A0000000) #[[ATTR2:[0-9]+]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_logbf() {
   float D1 = __builtin_logbf(16.1f);
 }
@@ -182,6 +260,107 @@ void test_logbf() {
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_logbf_var(
+// AMDGCNSPIRV-DEFAULT-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float [[TMP0]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = extractvalue { float, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to float
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP5:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP6:%.*]] = call addrspace(4) float @llvm.fabs.f32(float [[TMP5]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP7:%.*]] = fcmp one float [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP6]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP9:%.*]] = fcmp oeq float [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float 0xFFF0000000000000, float [[TMP8]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_logbf_var(
+// AMDGCNSPIRV-IGNORE-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float [[TMP0]])
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = extractvalue { float, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to float
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP5:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP6:%.*]] = call addrspace(4) float @llvm.fabs.f32(float [[TMP5]])
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP7:%.*]] = fcmp one float [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP6]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP9:%.*]] = fcmp oeq float [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float 0xFFF0000000000000, float [[TMP8]]
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_logbf_var(
+// AMDGCNSPIRV-STRICT-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float [[TMP0]])
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = extractvalue { float, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to float
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP5:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP6:%.*]] = call addrspace(4) float @llvm.fabs.f32(float [[TMP5]])
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP7:%.*]] = fcmp one float [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP6]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP9:%.*]] = fcmp oeq float [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float 0xFFF0000000000000, float [[TMP8]]
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_logbf_var(
+// AMDGCNSPIRV-MAYTRAP-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) { float, i32 } @llvm.frexp.f32.i32(float [[TMP0]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = extractvalue { float, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to float
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP5:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP6:%.*]] = call addrspace(4) float @llvm.fabs.f32(float [[TMP5]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP7:%.*]] = fcmp one float [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP6]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP9:%.*]] = fcmp oeq float [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float 0xFFF0000000000000, float [[TMP8]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_logbf_var(
+// AMDGCNSPIRV-ERRNO-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @logbf(float noundef [[TMP0]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_logbf_var(float a) {
   float D1 = __builtin_logbf(a);
 }
@@ -273,6 +452,79 @@ void test_logbf_var(float a) {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_logb(
+// AMDGCNSPIRV-DEFAULT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double 1.510000e+01)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = extractvalue { double, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to double
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP4:%.*]] = call addrspace(4) double @llvm.fabs.f64(double 1.510000e+01)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP5:%.*]] = fcmp one double [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP4]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP7:%.*]] = select i1 false, double 0xFFF0000000000000, double [[TMP6]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_logb(
+// AMDGCNSPIRV-IGNORE-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double 1.510000e+01)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = extractvalue { double, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to double
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP4:%.*]] = call addrspace(4) double @llvm.fabs.f64(double 1.510000e+01)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP5:%.*]] = fcmp one double [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP4]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP7:%.*]] = select i1 false, double 0xFFF0000000000000, double [[TMP6]]
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_logb(
+// AMDGCNSPIRV-STRICT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double 1.510000e+01)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = extractvalue { double, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to double
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP4:%.*]] = call addrspace(4) double @llvm.fabs.f64(double 1.510000e+01)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP5:%.*]] = fcmp one double [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP4]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP7:%.*]] = select i1 false, double 0xFFF0000000000000, double [[TMP6]]
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_logb(
+// AMDGCNSPIRV-MAYTRAP-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double 1.510000e+01)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = extractvalue { double, i32 } [[TMP0]], 1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = add nsw i32 [[TMP1]], -1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP3:%.*]] = sitofp i32 [[TMP2]] to double
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP4:%.*]] = call addrspace(4) double @llvm.fabs.f64(double 1.510000e+01)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP5:%.*]] = fcmp one double [[TMP4]], 0x7FF0000000000000
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP4]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP7:%.*]] = select i1 false, double 0xFFF0000000000000, double [[TMP6]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP7]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_logb(
+// AMDGCNSPIRV-ERRNO-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @logb(double noundef 1.510000e+01) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_logb() {
   double D1 = __builtin_logb(15.1);
 }
@@ -398,6 +650,107 @@ void test_logb() {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_logb_var(
+// AMDGCNSPIRV-DEFAULT-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double [[TMP0]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = extractvalue { double, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to double
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP6:%.*]] = call addrspace(4) double @llvm.fabs.f64(double [[TMP5]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP7:%.*]] = fcmp one double [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP6]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP9:%.*]] = fcmp oeq double [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[TMP8]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_logb_var(
+// AMDGCNSPIRV-IGNORE-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double [[TMP0]])
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = extractvalue { double, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to double
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP6:%.*]] = call addrspace(4) double @llvm.fabs.f64(double [[TMP5]])
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP7:%.*]] = fcmp one double [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP6]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP9:%.*]] = fcmp oeq double [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[TMP8]]
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_logb_var(
+// AMDGCNSPIRV-STRICT-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double [[TMP0]])
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = extractvalue { double, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to double
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP6:%.*]] = call addrspace(4) double @llvm.fabs.f64(double [[TMP5]])
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP7:%.*]] = fcmp one double [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP6]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP9:%.*]] = fcmp oeq double [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[TMP8]]
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_logb_var(
+// AMDGCNSPIRV-MAYTRAP-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) { double, i32 } @llvm.frexp.f64.i32(double [[TMP0]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = extractvalue { double, i32 } [[TMP1]], 1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP3:%.*]] = add nsw i32 [[TMP2]], -1
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP4:%.*]] = sitofp i32 [[TMP3]] to double
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP5:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP6:%.*]] = call addrspace(4) double @llvm.fabs.f64(double [[TMP5]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP7:%.*]] = fcmp one double [[TMP6]], 0x7FF0000000000000
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP6]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP9:%.*]] = fcmp oeq double [[TMP0]], 0.000000e+00
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double 0xFFF0000000000000, double [[TMP8]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP10]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_logb_var(
+// AMDGCNSPIRV-ERRNO-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @logb(double noundef [[TMP0]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_logb_var(double a) {
   double D1 = __builtin_logb(a);
 }
@@ -455,6 +808,51 @@ void test_logb_var(double a) {
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbnf(
+// AMDGCNSPIRV-DEFAULT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x4030B33340000000, i32 10)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbnf(
+// AMDGCNSPIRV-IGNORE-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x4030B33340000000, i32 10)
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbnf(
+// AMDGCNSPIRV-STRICT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x4030B33340000000, i32 10)
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbnf(
+// AMDGCNSPIRV-MAYTRAP-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x4030B33340000000, i32 10)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbnf(
+// AMDGCNSPIRV-ERRNO-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @scalbnf(float noundef 0x4030B33340000000, i32 noundef 10) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbnf() {
   float D1 = __builtin_scalbnf(16.7f, 10);
 }
@@ -535,6 +933,71 @@ void test_scalbnf() {
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbnf_var1(
+// AMDGCNSPIRV-DEFAULT-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 9)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbnf_var1(
+// AMDGCNSPIRV-IGNORE-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 9)
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbnf_var1(
+// AMDGCNSPIRV-STRICT-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 9)
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbnf_var1(
+// AMDGCNSPIRV-MAYTRAP-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 9)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbnf_var1(
+// AMDGCNSPIRV-ERRNO-SAME: float noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @scalbnf(float noundef [[TMP0]], i32 noundef 9) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbnf_var1(float a) {
   float D1 = __builtin_scalbnf(a, 9);
 }
@@ -615,6 +1078,71 @@ void test_scalbnf_var1(float a) {
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbnf_var2(
+// AMDGCNSPIRV-DEFAULT-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x402E666660000000, i32 [[TMP0]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbnf_var2(
+// AMDGCNSPIRV-IGNORE-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x402E666660000000, i32 [[TMP0]])
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbnf_var2(
+// AMDGCNSPIRV-STRICT-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x402E666660000000, i32 [[TMP0]])
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbnf_var2(
+// AMDGCNSPIRV-MAYTRAP-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float 0x402E666660000000, i32 [[TMP0]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbnf_var2(
+// AMDGCNSPIRV-ERRNO-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @scalbnf(float noundef 0x402E666660000000, i32 noundef [[TMP0]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbnf_var2(int b) {
   float D1 = __builtin_scalbnf(15.2f, b);
 }
@@ -719,6 +1247,91 @@ void test_scalbnf_var2(int b) {
 // ERRNO-NEXT:    store float [[CALL]], ptr [[D1_ASCAST]], align 4
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbnf_var3(
+// AMDGCNSPIRV-DEFAULT-SAME: float noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    store float [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbnf_var3(
+// AMDGCNSPIRV-IGNORE-SAME: float noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-IGNORE-NEXT:    store float [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbnf_var3(
+// AMDGCNSPIRV-STRICT-SAME: float noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-STRICT-NEXT:    store float [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbnf_var3(
+// AMDGCNSPIRV-MAYTRAP-SAME: float noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = call addrspace(4) float @llvm.ldexp.f32.i32(float [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store float [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbnf_var3(
+// AMDGCNSPIRV-ERRNO-SAME: float noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca float, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[A_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) float @scalbnf(float noundef [[TMP0]], i32 noundef [[TMP1]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store float [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbnf_var3(float a, int b) {
   float D1 = __builtin_scalbnf(a, b);
 }
@@ -776,6 +1389,51 @@ void test_scalbnf_var3(float a, int b) {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbn(
+// AMDGCNSPIRV-DEFAULT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.720000e+01, i32 10)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbn(
+// AMDGCNSPIRV-IGNORE-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.720000e+01, i32 10)
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbn(
+// AMDGCNSPIRV-STRICT-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.720000e+01, i32 10)
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbn(
+// AMDGCNSPIRV-MAYTRAP-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.720000e+01, i32 10)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP0]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbn(
+// AMDGCNSPIRV-ERRNO-SAME: ) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @scalbn(double noundef 1.720000e+01, i32 noundef 10) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbn() {
   double D1 = __builtin_scalbn(17.2, 10);
 }
@@ -856,6 +1514,71 @@ void test_scalbn() {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbn_var1(
+// AMDGCNSPIRV-DEFAULT-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 9)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbn_var1(
+// AMDGCNSPIRV-IGNORE-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 9)
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbn_var1(
+// AMDGCNSPIRV-STRICT-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 9)
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbn_var1(
+// AMDGCNSPIRV-MAYTRAP-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 9)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbn_var1(
+// AMDGCNSPIRV-ERRNO-SAME: double noundef [[A:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @scalbn(double noundef [[TMP0]], i32 noundef 9) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbn_var1(double a) {
   double D1 = __builtin_scalbn(a, 9);
 }
@@ -936,6 +1659,71 @@ void test_scalbn_var1(double a) {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbn_var2(
+// AMDGCNSPIRV-DEFAULT-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.540000e+01, i32 [[TMP0]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbn_var2(
+// AMDGCNSPIRV-IGNORE-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.540000e+01, i32 [[TMP0]])
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbn_var2(
+// AMDGCNSPIRV-STRICT-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.540000e+01, i32 [[TMP0]])
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbn_var2(
+// AMDGCNSPIRV-MAYTRAP-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double 1.540000e+01, i32 [[TMP0]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP1]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbn_var2(
+// AMDGCNSPIRV-ERRNO-SAME: i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @scalbn(double noundef 1.540000e+01, i32 noundef [[TMP0]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbn_var2(int b) {
   double D1 = __builtin_scalbn(15.4, b);
 }
@@ -1040,6 +1828,91 @@ void test_scalbn_var2(int b) {
 // ERRNO-NEXT:    store double [[CALL]], ptr [[D1_ASCAST]], align 8
 // ERRNO-NEXT:    ret void
 //
+// AMDGCNSPIRV-DEFAULT-LABEL: define spir_func void @test_scalbn_var3(
+// AMDGCNSPIRV-DEFAULT-SAME: double noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-DEFAULT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-DEFAULT-NEXT:    [[TMP2:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-DEFAULT-NEXT:    store double [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-DEFAULT-NEXT:    ret void
+//
+// AMDGCNSPIRV-IGNORE-LABEL: define spir_func void @test_scalbn_var3(
+// AMDGCNSPIRV-IGNORE-SAME: double noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-IGNORE-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-IGNORE-NEXT:    [[TMP2:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-IGNORE-NEXT:    store double [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-IGNORE-NEXT:    ret void
+//
+// AMDGCNSPIRV-STRICT-LABEL: define spir_func void @test_scalbn_var3(
+// AMDGCNSPIRV-STRICT-SAME: double noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-STRICT-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-STRICT-NEXT:    [[TMP2:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-STRICT-NEXT:    store double [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-STRICT-NEXT:    ret void
+//
+// AMDGCNSPIRV-MAYTRAP-LABEL: define spir_func void @test_scalbn_var3(
+// AMDGCNSPIRV-MAYTRAP-SAME: double noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-MAYTRAP-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-MAYTRAP-NEXT:    [[TMP2:%.*]] = call addrspace(4) double @llvm.ldexp.f64.i32(double [[TMP0]], i32 [[TMP1]])
+// AMDGCNSPIRV-MAYTRAP-NEXT:    store double [[TMP2]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-MAYTRAP-NEXT:    ret void
+//
+// AMDGCNSPIRV-ERRNO-LABEL: define spir_func void @test_scalbn_var3(
+// AMDGCNSPIRV-ERRNO-SAME: double noundef [[A:%.*]], i32 noundef [[B:%.*]]) addrspace(4) #[[ATTR0]] {
+// AMDGCNSPIRV-ERRNO-NEXT:  [[ENTRY:.*:]]
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR:%.*]] = alloca i32, align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1:%.*]] = alloca double, align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[B_ADDR_ASCAST:%.*]] = addrspacecast ptr [[B_ADDR]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    [[D1_ASCAST:%.*]] = addrspacecast ptr [[D1]] to ptr addrspace(4)
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    store i32 [[B]], ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(4) [[B_ADDR_ASCAST]], align 4
+// AMDGCNSPIRV-ERRNO-NEXT:    [[CALL:%.*]] = call spir_func addrspace(4) double @scalbn(double noundef [[TMP0]], i32 noundef [[TMP1]]) #[[ATTR2]]
+// AMDGCNSPIRV-ERRNO-NEXT:    store double [[CALL]], ptr addrspace(4) [[D1_ASCAST]], align 8
+// AMDGCNSPIRV-ERRNO-NEXT:    ret void
+//
 void test_scalbn_var3(double a, int b) {
   double D1 = __builtin_scalbn(a, b);
 }
diff --git a/clang/test/CodeGen/pragma-comment.c b/clang/test/CodeGen/pragma-comment.c
index a966840f7c26..aa3aba18b9b2 100644
--- a/clang/test/CodeGen/pragma-comment.c
+++ b/clang/test/CodeGen/pragma-comment.c
@@ -32,3 +32,6 @@
 // ELF: ![[space]] = !{!"with space"}
 // ELF-NOT: bar
 // ELF-NOT: foo
+// This following match prevents the clang version metadata from matching the forbidden 'foo' and 'bar' tokens.
+// This can happen if the clang version string contains a Git repo URL that includes one of those substrings.
+// ELF-LABEL: {{\!\".*clang version}}
diff --git a/clang/test/CodeGen/visibility.c b/clang/test/CodeGen/visibility.c
index ee760ec77879..947bdf3d377a 100644
--- a/clang/test/CodeGen/visibility.c
+++ b/clang/test/CodeGen/visibility.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 %s -triple i386-unknown-unknown -fvisibility=default -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-DEFAULT
-// RUN: %clang_cc1 %s -triple i386-unknown-unknown -fvisibility=protected -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-PROTECTED
-// RUN: %clang_cc1 %s -triple i386-unknown-unknown -fvisibility=hidden -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-HIDDEN
+// RUN: %clang_cc1 %s -triple i386-unknown-unknown -Wno-implicit-function-declaration -fvisibility=default -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-DEFAULT
+// RUN: %clang_cc1 %s -triple i386-unknown-unknown -Wno-implicit-function-declaration -fvisibility=protected -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-PROTECTED
+// RUN: %clang_cc1 %s -triple i386-unknown-unknown -Wno-implicit-function-declaration -fvisibility=hidden -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-HIDDEN
 
 // CHECK-DEFAULT: @g_def ={{.*}} global i32 0
 // CHECK-DEFAULT: @g_com ={{.*}} global i32 0
@@ -72,3 +72,27 @@ __private_extern__ int test4 = 10;
 // CHECK-HIDDEN-LABEL: define hidden void @test5()
 __attribute__((availability(macosx,introduced=10.5,deprecated=10.6)))
 __private_extern__ void test5(void) {}
+
+
+#pragma clang attribute push([[gnu::visibility("hidden")]], apply_to=function)
+
+// CHECK-DEFAULT-LABEL: define hidden void @func()
+// CHECK-PROTECTED-LABEL: define hidden void @func()
+// CHECK-HIDDEN-LABEL: define hidden void @func()
+void func(void) {}
+
+void call(void) {
+  implicit_function();
+  // CHECK-DEFAULT-LABEL: declare hidden i32 @implicit_function(...)
+  // CHECK-PROTECTED-LABEL: declare hidden i32 @implicit_function(...)
+  // CHECK-HIDDEN-LABEL: declare hidden i32 @implicit_function(...)
+}
+
+#pragma clang attribute pop
+
+void call2(void) {
+  implicit_function2();
+  // CHECK-DEFAULT-LABEL: declare i32 @implicit_function2(...)
+  // CHECK-PROTECTED-LABEL: declare i32 @implicit_function2(...)
+  // CHECK-HIDDEN-LABEL: declare i32 @implicit_function2(...)
+}
diff --git a/clang/test/CodeGenCUDA/bf16.cu b/clang/test/CodeGenCUDA/bf16.cu
index df56ec60c63a..12474381e718 100644
--- a/clang/test/CodeGenCUDA/bf16.cu
+++ b/clang/test/CodeGenCUDA/bf16.cu
@@ -37,11 +37,7 @@ __device__ __bf16 test_call( __bf16 in) {
 // CHECK:        ld.param.b16    %[[R:rs[0-9]+]], [_Z9test_callDF16b_param_0];
 // CHECK:        st.param.b16    [param0], %[[R]];
 // CHECK:        .param .align 2 .b8 retval0[2];
-// CHECK:        call.uni (retval0),
-// CHECK-NEXT:   _Z13external_funcDF16b,
-// CHECK-NEXT:   (
-// CHECK-NEXT:   param0
-// CHECK-NEXT    );
+// CHECK:        call.uni (retval0), _Z13external_funcDF16b, (param0);
 // CHECK:        ld.param.b16    %[[RET:rs[0-9]+]], [retval0];
   return external_func(in);
 // CHECK:        st.param.b16    [func_retval0], %[[RET]]
diff --git a/clang/test/CodeGenCXX/auto-var-init.cpp b/clang/test/CodeGenCXX/auto-var-init.cpp
index 94386e44573b..67bc5d417bce 100644
--- a/clang/test/CodeGenCXX/auto-var-init.cpp
+++ b/clang/test/CodeGenCXX/auto-var-init.cpp
@@ -1342,6 +1342,7 @@ TEST_UNINIT(base, base);
 // ZERO-O1-NOT: !annotation
 
 TEST_BRACES(base, base);
+// ZERO-LABEL: @test_base_braces()
 // CHECK-LABEL: @test_base_braces()
 // CHECK:       %braces = alloca %struct.base, align [[ALIGN:[0-9]*]]
 // CHECK-NEXT:  call void @llvm.memset{{.*}}(ptr align [[ALIGN]] %{{.*}}, i8 0, i64 8, i1 false)
diff --git a/clang/test/CodeGenCXX/bitfield-access-empty.cpp b/clang/test/CodeGenCXX/bitfield-access-empty.cpp
index 96047ce47299..a06f62b3eb05 100644
--- a/clang/test/CodeGenCXX/bitfield-access-empty.cpp
+++ b/clang/test/CodeGenCXX/bitfield-access-empty.cpp
@@ -1,4 +1,5 @@
 // Check if we can merge bitfields across empty members
+// XFAIL: *
 
 // Configs that have cheap unaligned access
 // Little Endian
diff --git a/clang/test/CodeGenCXX/builtin-invoke.cpp b/clang/test/CodeGenCXX/builtin-invoke.cpp
new file mode 100644
index 000000000000..af66dfd4dae3
--- /dev/null
+++ b/clang/test/CodeGenCXX/builtin-invoke.cpp
@@ -0,0 +1,61 @@
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void* memcpy(void*, const void*, decltype(sizeof(int)));
+void func();
+
+namespace std {
+  template <class T>
+  class reference_wrapper {
+    T* ptr;
+
+  public:
+    T& get() { return *ptr; }
+  };
+} // namespace std
+
+struct Callable {
+  void operator()() {}
+
+  void func();
+};
+
+extern "C" void call1() {
+  __builtin_invoke(func);
+  __builtin_invoke(Callable{});
+  __builtin_invoke(memcpy, nullptr, nullptr, 0);
+
+  // CHECK:      define dso_local void @call1
+  // CHECK-NEXT: entry:
+  // CHECK-NEXT:   %ref.tmp = alloca %struct.Callable, align 1
+  // CHECK-NEXT:   call void @_Z4funcv()
+  // CHECK-NEXT:   call void @_ZN8CallableclEv(ptr noundef nonnull align 1 dereferenceable(1) %ref.tmp)
+  // CHECK-NEXT:   call void @llvm.memcpy.p0.p0.i64(ptr align 1 null, ptr align 1 null, i64 0, i1 false)
+  // CHECK-NEXT:   ret void
+}
+
+extern "C" void call_memptr(std::reference_wrapper<Callable> wrapper) {
+  __builtin_invoke(&Callable::func, wrapper);
+
+  // CHECK:      define dso_local void @call_memptr
+  // CHECK-NEXT: entry:
+  // CHECK-NEXT:   %wrapper = alloca %"class.std::reference_wrapper", align 8
+  // CHECK-NEXT:   %coerce.dive = getelementptr inbounds nuw %"class.std::reference_wrapper", ptr %wrapper, i32 0, i32 0
+  // CHECK-NEXT:   store ptr %wrapper.coerce, ptr %coerce.dive, align 8
+  // CHECK-NEXT:   %call = call noundef nonnull align 1 dereferenceable(1) ptr @_ZNSt17reference_wrapperI8CallableE3getEv(ptr noundef nonnull align 8 dereferenceable(8) %wrapper)
+  // CHECK-NEXT:   %0 = getelementptr inbounds i8, ptr %call, i64 0
+  // CHECK-NEXT:   br i1 false, label %memptr.virtual, label %memptr.nonvirtual
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.virtual:
+  // CHECK-NEXT:   %vtable = load ptr, ptr %0, align 8
+  // CHECK-NEXT:   %1 = getelementptr i8, ptr %vtable, i64 sub (i64 ptrtoint (ptr @_ZN8Callable4funcEv to i64), i64 1), !nosanitize !2
+  // CHECK-NEXT:   %memptr.virtualfn = load ptr, ptr %1, align 8, !nosanitize !2
+  // CHECK-NEXT:   br label %memptr.end
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.nonvirtual:
+  // CHECK-NEXT:   br label %memptr.end
+  // CHECK-EMPTY:
+  // CHECK-NEXT: memptr.end:
+  // CHECK-NEXT:   %2 = phi ptr [ %memptr.virtualfn, %memptr.virtual ], [ @_ZN8Callable4funcEv, %memptr.nonvirtual ]
+  // CHECK-NEXT:   call void %2(ptr noundef nonnull align 1 dereferenceable(1) %0)
+  // CHECK-NEXT:   ret void
+}
diff --git a/clang/test/CodeGenCXX/builtins-eh-wasm.cpp b/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
index b0f763d3e54d..b3a6ffc92e11 100644
--- a/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
+++ b/clang/test/CodeGenCXX/builtins-eh-wasm.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -target-feature +reference-types -target-feature +exception-handling -target-feature +multivalue -exception-model=wasm -emit-llvm -o - %s | FileCheck %s
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -target-feature +reference-types -target-feature +exception-handling -target-feature +multivalue -mllvm -wasm-enable-eh -exception-model=wasm -emit-llvm -o - %s | FileCheck %s
 
 // Check if __builtin_wasm_throw and __builtin_wasm_rethrow are correctly
 // invoked when placed in try-catch.
diff --git a/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
new file mode 100644
index 000000000000..4bb51dcc4da5
--- /dev/null
+++ b/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-none-linux-gnu -emit-llvm -debug-info-kind=limited %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-windows-msvc -emit-llvm -debug-info-kind=limited %s -o - | FileCheck --check-prefix MSVC %s
+
+struct B {
+    virtual ~B() {}
+};
+
+struct A : virtual B {
+};
+
+A a;
+
+
+// CHECK-DAG: !{{[0-9]+}} = !DILocalVariable(name: "vtt", arg: 2, scope: ![[destructor:[0-9]+]], type: ![[vtttype:[0-9]+]], flags: DIFlagArtificial)
+// CHECK-DAG: ![[destructor]] = distinct !DISubprogram(name: "~A", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// CHECK-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// CHECK-DAG: [[types]] = !{null, !{{[0-9]+}}, ![[vtttype]]}
+
+// MSVC-DAG: ![[inttype:[0-9]+]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+// MSVC-DAG: ![[voidpointertype:[0-9]+]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+// MSVC-DAG: ![[destructor:[0-9]+]] = distinct !DISubprogram(name: "~A", linkageName: "??_GA@@UEAAPEAXI@Z", {{.*}}, type: ![[subroutinetype:[0-9]+]]
+// MSVC-DAG: !{{[0-9]+}} = !DILocalVariable(name: "should_call_delete", arg: 2, scope: ![[destructor]], type: ![[inttype]], flags: DIFlagArtificial)
+// MSVC-DAG: ![[subroutinetype]] = !DISubroutineType(types: ![[types:[0-9]+]])
+// MSVC-DAG: [[types]] = !{![[voidpointertype]], !{{[0-9]+}}, ![[inttype]]}
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
index fe4cab164249..806bc5b63ef0 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-member-pointers.cpp
@@ -985,3 +985,10 @@ namespace ContainerOf {
     return reinterpret_cast<Node*>(reinterpret_cast<char*>(list) - offset);
   }
 }
+
+namespace GH144081 {
+  struct A;
+  template<int A::*> void f() {}
+  template void f<nullptr>();
+  // CHECK-LABEL: define{{.*}} void @"??$f@$0A@@GH144081@@YAXXZ"
+} // namespace GH144081
diff --git a/clang/test/CodeGenCXX/visibility.cpp b/clang/test/CodeGenCXX/visibility.cpp
index 442e2a5aaa2b..3203d476c45f 100644
--- a/clang/test/CodeGenCXX/visibility.cpp
+++ b/clang/test/CodeGenCXX/visibility.cpp
@@ -218,7 +218,7 @@ namespace test27 {
 namespace Test1 {
   // CHECK-LABEL: define hidden void @_ZN5Test11fEv
   void HIDDEN f() { }
-  
+
 }
 
 namespace Test2 {
@@ -230,7 +230,7 @@ namespace Test2 {
   // CHECK-LABEL: define hidden void @_ZN5Test21A1fEv
   void A::f() { }
 }
- 
+
 namespace Test3 {
   struct HIDDEN A {
     struct B {
@@ -240,7 +240,7 @@ namespace Test3 {
 
   // B is a nested class where its parent class is hidden.
   // CHECK-LABEL: define hidden void @_ZN5Test31A1B1fEv
-  void A::B::f() { }  
+  void A::B::f() { }
 }
 
 namespace Test4 HIDDEN {
@@ -248,15 +248,15 @@ namespace Test4 HIDDEN {
 
   // Test4::g is in a hidden namespace.
   // CHECK-LABEL: define hidden void @_ZN5Test41gEv
-  void g() { } 
-  
+  void g() { }
+
   struct DEFAULT A {
     void f();
   };
-  
+
   // A has default visibility.
   // CHECK-LABEL: define void @_ZN5Test41A1fEv
-  void A::f() { } 
+  void A::f() { }
 }
 
 namespace Test5 {
@@ -266,7 +266,7 @@ namespace Test5 {
     // CHECK-LABEL: define hidden void @_ZN5Test52NS1fEv()
     void f() { }
   }
-  
+
   namespace NS {
     // g is in NS, but this NS decl is not hidden.
     // CHECK-LABEL: define void @_ZN5Test52NS1gEv
@@ -1536,3 +1536,27 @@ namespace test75 {
   // CHECK-LABEL: define linkonce_odr hidden void @_ZN6test751TIMNS_1AEFvvEE6InvokeIiEEvT_(
   // CHECK-HIDDEN-LABEL: define linkonce_odr hidden void @_ZN6test751TIMNS_1AEFvvEE6InvokeIiEEvT_(
 }
+
+#pragma clang attribute push([[gnu::visibility("hidden")]], apply_to=function)
+
+namespace pragma_test {
+  struct S {
+    S();
+  };
+
+  S::S() = default;
+  // CHECK-LABEL: define hidden void @_ZN11pragma_test1SC2Ev(
+  // CHECK-HIDDEN-LABEL: define hidden void @_ZN11pragma_test1SC2Ev(
+}
+
+#pragma clang attribute pop
+
+namespace no_pragma_test {
+  struct S {
+    S();
+  };
+
+  S::S() = default;
+  // CHECK-LABEL: define void @_ZN14no_pragma_test1SC2Ev(
+  // CHECK-HIDDEN-LABEL: define hidden void @_ZN14no_pragma_test1SC2Ev(
+}
diff --git a/clang/test/CodeGenCXX/wasm-eh.cpp b/clang/test/CodeGenCXX/wasm-eh.cpp
index faff764878f5..f243f37ecb43 100644
--- a/clang/test/CodeGenCXX/wasm-eh.cpp
+++ b/clang/test/CodeGenCXX/wasm-eh.cpp
@@ -393,9 +393,9 @@ void noexcept_throw() noexcept {
 // CHECK-NEXT:  call void @_ZSt9terminatev()
 
 
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-DEFAULT
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -Wwasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-ON
-// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -exception-model=wasm -target-feature +exception-handling -Wno-wasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-OFF
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-DEFAULT
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -Wwasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-ON
+// RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fms-extensions -fexceptions -fcxx-exceptions -mllvm -wasm-enable-eh -exception-model=wasm -target-feature +exception-handling -Wno-wasm-exception-spec -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=WARNING-OFF
 // RUN: %clang_cc1 %s -triple wasm32-unknown-unknown -fexceptions -fcxx-exceptions -emit-llvm -o - -std=c++11 2>&1 | FileCheck %s --check-prefix=EM-EH-WARNING
 
 // Wasm EH ignores dynamic exception specifications with types at the moment.
diff --git a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
index 55cc57fadf08..28d9a4b8e596 100644
--- a/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/AppendStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 struct MyStruct {
   float4 a;
@@ -21,7 +22,9 @@ struct MyStruct {
 // DXIL: %"class.hlsl::AppendStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type <{ <4 x float>, <2 x i32> }>
 // DXIL: %"class.hlsl::AppendStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// SPV: %"class.hlsl::AppendStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // DXIL: %"class.hlsl::AppendStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
+// SPV: %"class.hlsl::AppendStructuredBuffer.14" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 AppendStructuredBuffer<int16_t> BufI16;
 AppendStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
index 94fdd3cd9d0b..70fe8670c59a 100644
--- a/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/ConsumeStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=DXIL
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 struct MyStruct {
   float4 a;
@@ -21,7 +22,9 @@ struct MyStruct {
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.12" = type { target("dx.RawBuffer", %struct.MyStruct, 1, 0)
 // DXIL: %struct.MyStruct = type <{ <4 x float>, <2 x i32> }>
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.13" = type { target("dx.RawBuffer", i32, 1, 0)
+// SPV: %"class.hlsl::ConsumeStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // DXIL: %"class.hlsl::ConsumeStructuredBuffer.14" = type { target("dx.RawBuffer", <4 x i32>, 1, 0)
+// SPV: %"class.hlsl::ConsumeStructuredBuffer.14" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 ConsumeStructuredBuffer<int16_t> BufI16;
 ConsumeStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
index 0944ad59d5fb..5512a657bc5f 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-elementtype.hlsl
@@ -16,20 +16,20 @@
 // DXIL: %"class.hlsl::RWBuffer.11" = type { target("dx.TypedBuffer", <3 x float>, 1, 0, 0) }
 // DXIL: %"class.hlsl::RWBuffer.12" = type { target("dx.TypedBuffer", <4 x i32>, 1, 0, 1) }
 
-// SPIRV: %"class.hlsl::RWBuffer" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer" = type { target("spirv.SignedImage", i16, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.0" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.1" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.1" = type { target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.2" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.3" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.3" = type { target("spirv.SignedImage", i64, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.4" = type { target("spirv.Image", i64, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.5" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.6" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.7" = type { target("spirv.Image", double, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.8" = type { target("spirv.Image", i16, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.8" = type { target("spirv.SignedImage", i16, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.9" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.10" = type { target("spirv.Image", half, 5, 2, 0, 0, 2, 0) }
 // SPIRV: %"class.hlsl::RWBuffer.11" = type { target("spirv.Image", float, 5, 2, 0, 0, 2, 0) }
-// SPIRV: %"class.hlsl::RWBuffer.12" = type { target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) }
+// SPIRV: %"class.hlsl::RWBuffer.12" = type { target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) }
 
 RWBuffer<int16_t> BufI16;
 RWBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
index cf810ed909eb..63e3552b680b 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-subscript.hlsl
@@ -9,18 +9,18 @@ void main(unsigned GI : SV_GroupIndex) {
   // CHECK: define void @main()
 
   // DXC: %[[INPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[INPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[INPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}%[[INPTR]]
   // DXC: %[[OUTPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: store i32 %[[LOAD]], ptr {{.*}}%[[OUTPTR]]
   Out[GI] = In[GI];
 
   // DXC: %[[INPTR:.*]] = call ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[INPTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[INPTR:.*]] = call ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: %[[LOAD:.*]] = load i32, ptr {{.*}}%[[INPTR]]
   // DXC: %[[OUTPTR:.*]] = call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.TypedBuffer_i32_1_0_1t(target("dx.TypedBuffer", i32, 1, 0, 1) %{{.*}}, i32 %{{.*}})
-  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
+  // SPIRV: %[[OUTPTR:.*]] = call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %{{.*}}, i32 %{{.*}})
   // CHECK: store i32 %[[LOAD]], ptr {{.*}}%[[OUTPTR]]
   Out[GI] = In.Load(GI);
 }
diff --git a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
index e23859ee83d1..f7cdc5d4d39f 100644
--- a/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWStructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 // CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i16, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.0" = type { target("dx.RawBuffer", i16, 1, 0) }
@@ -14,7 +15,9 @@
 // CHECK: %"class.hlsl::RWStructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 1, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer.12" = type { target("dx.RawBuffer", i32, 1, 0) }
+// SPV: %"class.hlsl::RWStructuredBuffer.12" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 1)
 // CHECK: %"class.hlsl::RWStructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 1, 0) }
+// SPV: %"class.hlsl::RWStructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 1)
 
 RWStructuredBuffer<int16_t> BufI16;
 RWStructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
index 6c69b2b34bb2..0bb08065ddc2 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-elementtype.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.2-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=CHECK
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -finclude-default-header -fnative-half-type -emit-llvm -o - %s | FileCheck %s -check-prefixes=SPV
 
 // CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i16, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.0" = type { target("dx.RawBuffer", i16, 0, 0) }
@@ -14,7 +15,9 @@
 // CHECK: %"class.hlsl::StructuredBuffer.10" = type { target("dx.RawBuffer", <2 x half>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.11" = type { target("dx.RawBuffer", <3 x float>, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer.12" = type { target("dx.RawBuffer", i32, 0, 0) }
+// SPV: %"class.hlsl::StructuredBuffer.12" = type { target("spirv.VulkanBuffer", [0 x i32], 12, 0)
 // CHECK: %"class.hlsl::StructuredBuffer.13" = type { target("dx.RawBuffer", <4 x i32>, 0, 0) }
+// SPV: %"class.hlsl::StructuredBuffer.13" = type { target("spirv.VulkanBuffer", [0 x <4 x i32>], 12, 0)
 
 StructuredBuffer<int16_t> BufI16;
 StructuredBuffer<uint16_t> BufU16;
diff --git a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
index cbc1fa61eae2..15c54beb03d3 100644
--- a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
+++ b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
@@ -21,6 +21,9 @@ const unsigned short ushort_const = 10;
 [[vk::constant_id(6)]]
 const unsigned int uint_const = 12;
 
+[[vk::constant_id(6)]]
+const uint uint_const_2 = 12;
+
 [[vk::constant_id(7)]]
 const unsigned long long ulong_const = 25;
 
@@ -50,6 +53,7 @@ void main() {
     long long l = long_const;
     unsigned short us = ushort_const;
     unsigned int ui = uint_const;
+    uint ui2 = uint_const_2;
     unsigned long long ul = ulong_const;
     half h = half_const;
     float f = float_const;
@@ -63,6 +67,7 @@ void main() {
 // CHECK: @_ZL10long_const = internal addrspace(10) global i64 0, align 8
 // CHECK: @_ZL12ushort_const = internal addrspace(10) global i16 0, align 2
 // CHECK: @_ZL10uint_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL12uint_const_2 = internal addrspace(10) global i32 0, align 4
 // CHECK: @_ZL11ulong_const = internal addrspace(10) global i64 0, align 8
 // CHECK: @_ZL10half_const = internal addrspace(10) global float 0.000000e+00, align 4
 // CHECK: @_ZL11float_const = internal addrspace(10) global float 0.000000e+00, align 4
@@ -79,6 +84,7 @@ void main() {
 // CHECK-NEXT:    [[L:%.*]] = alloca i64, align 8
 // CHECK-NEXT:    [[US:%.*]] = alloca i16, align 2
 // CHECK-NEXT:    [[UI:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[UI2:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[UL:%.*]] = alloca i64, align 8
 // CHECK-NEXT:    [[H:%.*]] = alloca float, align 4
 // CHECK-NEXT:    [[F:%.*]] = alloca float, align 4
@@ -98,16 +104,18 @@ void main() {
 // CHECK-NEXT:    store i16 [[TMP5]], ptr [[US]], align 2
 // CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(10) @_ZL10uint_const, align 4
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[UI]], align 4
-// CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr addrspace(10) @_ZL11ulong_const, align 8
-// CHECK-NEXT:    store i64 [[TMP7]], ptr [[UL]], align 8
-// CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr addrspace(10) @_ZL10half_const, align 4
-// CHECK-NEXT:    store float [[TMP8]], ptr [[H]], align 4
-// CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr addrspace(10) @_ZL11float_const, align 4
-// CHECK-NEXT:    store float [[TMP9]], ptr [[F]], align 4
-// CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr addrspace(10) @_ZL12double_const, align 8
-// CHECK-NEXT:    store double [[TMP10]], ptr [[D]], align 8
-// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr addrspace(10) @_ZL10enum_const, align 4
-// CHECK-NEXT:    store i32 [[TMP11]], ptr [[E]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr addrspace(10) @_ZL12uint_const_2, align 4
+// CHECK-NEXT:    store i32 [[TMP7]], ptr [[UI2]], align 4
+// CHECK-NEXT:    [[TMP8:%.*]] = load i64, ptr addrspace(10) @_ZL11ulong_const, align 8
+// CHECK-NEXT:    store i64 [[TMP8]], ptr [[UL]], align 8
+// CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr addrspace(10) @_ZL10half_const, align 4
+// CHECK-NEXT:    store float [[TMP9]], ptr [[H]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr addrspace(10) @_ZL11float_const, align 4
+// CHECK-NEXT:    store float [[TMP10]], ptr [[F]], align 4
+// CHECK-NEXT:    [[TMP11:%.*]] = load double, ptr addrspace(10) @_ZL12double_const, align 8
+// CHECK-NEXT:    store double [[TMP11]], ptr [[D]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr addrspace(10) @_ZL10enum_const, align 4
+// CHECK-NEXT:    store i32 [[TMP12]], ptr [[E]], align 4
 // CHECK-NEXT:    ret void
 //
 // CHECK-LABEL: define internal spir_func void @__cxx_global_var_init(
@@ -169,12 +177,21 @@ void main() {
 // CHECK-SAME: ) #[[ATTR3]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantij(i32 6, i32 12)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL12uint_const_2, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.7(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
 // CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantiy(i32 7, i64 25)
 // CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL11ulong_const, align 8
 // CHECK-NEXT:    ret void
 //
 //
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.7(
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.8(
 // CHECK-SAME: ) #[[ATTR3]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
@@ -183,7 +200,7 @@ void main() {
 // CHECK-NEXT:    ret void
 //
 //
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.8(
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.9(
 // CHECK-SAME: ) #[[ATTR3]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
@@ -192,7 +209,7 @@ void main() {
 // CHECK-NEXT:    ret void
 //
 //
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.9(
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.10(
 // CHECK-SAME: ) #[[ATTR3]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
@@ -201,7 +218,7 @@ void main() {
 // CHECK-NEXT:    ret void
 //
 //
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.10(
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.11(
 // CHECK-SAME: ) #[[ATTR3]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index 730ed47f0b0c..dc7a83002b7f 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -108,7 +108,7 @@
 // GFX1153: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
-// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bitop3-insts,+ci-insts,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+wavefrontsize32"
+// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bitop3-insts,+ci-insts,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+transpose-load-f4f6-insts,+wavefrontsize32"
 
 // GFX1103-W64: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize64"
 
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl
new file mode 100644
index 000000000000..e3fe31ff7dd7
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-async-load-store-lds.cl
@@ -0,0 +1,24 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_atomic_async_barrier_arrive_b64(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) [[ADDR:%.*]])
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_ds_atomic_async_barrier_arrive_b64(local long* addr)
+{
+  __builtin_amdgcn_ds_atomic_async_barrier_arrive_b64(addr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_atomic_barrier_arrive_rtn_b64(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) [[ADDR:%.*]], i64 [[DATA:%.*]])
+// CHECK-GFX1250-NEXT:    store i64 [[TMP0]], ptr [[OUT:%.*]], align 8, !tbaa [[TBAA4:![0-9]+]]
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_ds_atomic_barrier_arrive_rtn_b64(local long* addr, long data, long *out)
+{
+  *out = __builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64(addr, data);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl
new file mode 100644
index 000000000000..1e3a88a41f90
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-load-tr.cl
@@ -0,0 +1,130 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -target-feature +wavefrontsize32 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+typedef int    v2i   __attribute__((ext_vector_type(2)));
+typedef int    v3i   __attribute__((ext_vector_type(3)));
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef short  v8s   __attribute__((ext_vector_type(8)));
+typedef half   v8h   __attribute__((ext_vector_type(8)));
+typedef __bf16 v8y  __attribute__((ext_vector_type(8)));
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr4_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_global_load_tr4_b64_v2i32(global v2i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr4_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr8_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_global_load_tr8_b64_v2i32(global v2i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr8_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr6_b96_v3i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3i test_amdgcn_global_load_tr6_b96_v3i32(global v3i* inptr)
+{
+  return __builtin_amdgcn_global_load_tr6_b96_v3i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8i16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x i16> [[TMP0]]
+//
+v8s test_amdgcn_global_load_tr16_b128_v8i16(global v8s* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8i16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8f16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x half> [[TMP0]]
+//
+v8h test_amdgcn_global_load_tr16_b128_v8f16(global v8h* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8f16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_global_load_tr16_b128_v8bf16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16(ptr addrspace(1) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+v8y test_amdgcn_global_load_tr16_b128_v8bf16(global v8y* inptr)
+{
+  return __builtin_amdgcn_global_load_tr16_b128_v8bf16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr4_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_ds_load_tr4_b64_v2i32(local v2i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr4_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr8_b64_v2i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <2 x i32> [[TMP0]]
+//
+v2i test_amdgcn_ds_load_tr8_b64_v2i32(local v2i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr8_b64_v2i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr6_b96_v3i32(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <3 x i32> [[TMP0]]
+//
+v3i test_amdgcn_ds_load_tr6_b96_v3i32(local v3i* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr6_b96_v3i32(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8i16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x i16> [[TMP0]]
+//
+v8s test_amdgcn_ds_load_tr16_b128_v8i16(local v8s* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8i16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8f16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x half> [[TMP0]]
+//
+v8h test_amdgcn_ds_load_tr16_b128_v8f16(local v8h* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8f16(inptr);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_ds_load_tr16_b128_v8bf16(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    [[TMP0:%.*]] = tail call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) [[INPTR:%.*]])
+// CHECK-GFX1250-NEXT:    ret <8 x bfloat> [[TMP0]]
+//
+v8y test_amdgcn_ds_load_tr16_b128_v8bf16(local v8y* inptr)
+{
+  return __builtin_amdgcn_ds_load_tr16_b128_v8bf16(inptr);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl
new file mode 100644
index 000000000000..49ffbf451716
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250-tensor-load-store.cl
@@ -0,0 +1,46 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -cl-std=CL2.0 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-GFX1250
+
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef int    v8i   __attribute__((ext_vector_type(8)));
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_load_to_lds(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], <4 x i32> [[SG2:%.*]], <4 x i32> [[SG3:%.*]], i32 0)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_load_to_lds(v4i sg0, v8i sg1, v4i sg2, v4i sg3)
+{
+  __builtin_amdgcn_tensor_load_to_lds(sg0, sg1, sg2, sg3, 0);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_load_to_lds_d2(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], i32 27)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_load_to_lds_d2(v4i sg0, v8i sg1)
+{
+  __builtin_amdgcn_tensor_load_to_lds_d2(sg0, sg1, 27);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_store_from_lds(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], <4 x i32> [[SG2:%.*]], <4 x i32> [[SG3:%.*]], i32 22)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_store_from_lds(v4i sg0, v8i sg1, v4i sg2, v4i sg3)
+{
+  __builtin_amdgcn_tensor_store_from_lds(sg0, sg1, sg2, sg3, 22);
+}
+
+// CHECK-GFX1250-LABEL: @test_amdgcn_tensor_store_from_lds_d2(
+// CHECK-GFX1250-NEXT:  entry:
+// CHECK-GFX1250-NEXT:    tail call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> [[SG0:%.*]], <8 x i32> [[SG1:%.*]], i32 0)
+// CHECK-GFX1250-NEXT:    ret void
+//
+void test_amdgcn_tensor_store_from_lds_d2(v4i sg0, v8i sg1)
+{
+  __builtin_amdgcn_tensor_store_from_lds_d2(sg0, sg1, 0);
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 3709b1ff52f3..4d4afedae365 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -cl-std=CL2.0 -O0 -triple amdgcn-unknown-unknown -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck %s
 // REQUIRES: amdgpu-registered-target
 
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+typedef half __attribute__((ext_vector_type(2))) half2;
+
 // CHECK-LABEL: @test_setprio_inc_wg(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    call void @llvm.amdgcn.s.setprio.inc.wg(i16 10)
@@ -10,3 +14,128 @@
 void test_setprio_inc_wg() {
   __builtin_amdgcn_s_setprio_inc_wg(10);
 }
+
+// CHECK-LABEL: @test_s_monitor_sleep(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.monitor.sleep(i16 10)
+// CHECK-NEXT:    ret void
+//
+void test_s_monitor_sleep() {
+  __builtin_amdgcn_s_monitor_sleep(10);
+}
+
+// CHECK-LABEL: @test_cvt_f16_fp8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP0]], i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store half [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP3]], i32 1)
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP5]], i64 1
+// CHECK-NEXT:    store half [[TMP4]], ptr addrspace(1) [[ARRAYIDX1]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP6]], i32 2)
+// CHECK-NEXT:    [[TMP8:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP8]], i64 2
+// CHECK-NEXT:    store half [[TMP7]], ptr addrspace(1) [[ARRAYIDX2]], align 2
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = call half @llvm.amdgcn.cvt.f16.fp8(i32 [[TMP9]], i32 3)
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP11]], i64 3
+// CHECK-NEXT:    store half [[TMP10]], ptr addrspace(1) [[ARRAYIDX3]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_cvt_f16_fp8(global half* out, int a)
+{
+  out[0] = __builtin_amdgcn_cvt_f16_fp8(a, 0);
+  out[1] = __builtin_amdgcn_cvt_f16_fp8(a, 1);
+  out[2] = __builtin_amdgcn_cvt_f16_fp8(a, 2);
+  out[3] = __builtin_amdgcn_cvt_f16_fp8(a, 3);
+}
+
+// CHECK-LABEL: @test_cvt_f16_bf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP0]], i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store half [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP3]], i32 1)
+// CHECK-NEXT:    [[TMP5:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP5]], i64 1
+// CHECK-NEXT:    store half [[TMP4]], ptr addrspace(1) [[ARRAYIDX1]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP6]], i32 2)
+// CHECK-NEXT:    [[TMP8:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP8]], i64 2
+// CHECK-NEXT:    store half [[TMP7]], ptr addrspace(1) [[ARRAYIDX2]], align 2
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = call half @llvm.amdgcn.cvt.f16.bf8(i32 [[TMP9]], i32 3)
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(1) [[TMP11]], i64 3
+// CHECK-NEXT:    store half [[TMP10]], ptr addrspace(1) [[ARRAYIDX3]], align 2
+// CHECK-NEXT:    ret void
+//
+void test_cvt_f16_bf8(global half* out, int a)
+{
+  out[0] = __builtin_amdgcn_cvt_f16_bf8(a, 0);
+  out[1] = __builtin_amdgcn_cvt_f16_bf8(a, 1);
+  out[2] = __builtin_amdgcn_cvt_f16_bf8(a, 2);
+  out[3] = __builtin_amdgcn_cvt_f16_bf8(a, 3);
+}
+
+// CHECK-LABEL: @test_cvt_pk_f16_fp8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store <2 x half> [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_cvt_pk_f16_fp8(global half2* out, short a)
+{
+  out[0] = __builtin_amdgcn_cvt_pk_f16_fp8(a);
+}
+
+// CHECK-LABEL: @test_cvt_pk_f16_bf8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[OUT_ADDR:%.*]] = alloca ptr addrspace(1), align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i16, align 2, addrspace(5)
+// CHECK-NEXT:    [[OUT_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[OUT_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr addrspace(1) [[OUT:%.*]], ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i16 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[A_ADDR_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.amdgcn.cvt.pk.f16.bf8(i16 [[TMP0]])
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(1) [[TMP2]], i64 0
+// CHECK-NEXT:    store <2 x half> [[TMP1]], ptr addrspace(1) [[ARRAYIDX]], align 4
+// CHECK-NEXT:    ret void
+//
+void test_cvt_pk_f16_bf8(global half2* out, short a)
+{
+  out[0] = __builtin_amdgcn_cvt_pk_f16_bf8(a);
+}
diff --git a/clang/test/DebugInfo/KeyInstructions/atomic.c b/clang/test/DebugInfo/KeyInstructions/atomic.c
new file mode 100644
index 000000000000..c25f4385843b
--- /dev/null
+++ b/clang/test/DebugInfo/KeyInstructions/atomic.c
@@ -0,0 +1,166 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c++ -std=c++17 %s -debug-info-kind=line-tables-only -emit-llvm -o - -gno-column-info \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions -x c %s -debug-info-kind=line-tables-only -emit-llvm -o - -gno-column-info \
+// RUN: | FileCheck %s --implicit-check-not atomGroup --implicit-check-not atomRank
+
+// Check that atomic handling code gets Key Instruction annotations.
+
+_Atomic(unsigned int) x;
+unsigned int y;
+void fun() {
+  unsigned int r1 = __c11_atomic_fetch_add(&x,- 1, __ATOMIC_RELAXED);
+// CHECK:      store i32 -1, ptr %.atomictmp, align 4,                !dbg [[LINE11_G2R1:!.*]]
+// CHECK-NEXT: %0 = load i32, ptr %.atomictmp, align 4,               !dbg [[LINE11:!.*]]
+// CHECK-NEXT: %1 = atomicrmw add ptr @x, i32 %0 monotonic, align 4,  !dbg [[LINE11_G2R2:!.*]]
+// CHECK-NEXT: store i32 %1, ptr %atomic-temp, align 4,               !dbg [[LINE11_G2R1]]
+// CHECK-NEXT: %2 = load i32, ptr %atomic-temp, align 4,              !dbg [[LINE11_G1R2:!.*]]
+// CHECK-NEXT: store i32 %2, ptr %r1, align 4,                        !dbg [[LINE11_G1R1:!.*]]
+
+  unsigned int r2 = __c11_atomic_load(&x, __ATOMIC_RELAXED);
+// CHECK-NEXT: %3 = load atomic i32, ptr @x monotonic, align 4,       !dbg [[LINE19_G4R2:!.*]]
+// CHECK-NEXT: store i32 %3, ptr %atomic-temp1, align 4,              !dbg [[LINE19_G4R1:!.*]]
+// CHECK-NEXT: %4 = load i32, ptr %atomic-temp1, align 4,             !dbg [[LINE19_G3R2:!.*]]
+// CHECK-NEXT: store i32 %4, ptr %r2, align 4,                        !dbg [[LINE19_G3R1:!.*]]
+
+  __c11_atomic_store(&x, 2, __ATOMIC_RELAXED);
+// CHECK-NEXT: store i32 2, ptr %.atomictmp2, align 4,                !dbg [[LINE25_G5R1:!.*]]
+// CHECK-NEXT: %5 = load i32, ptr %.atomictmp2, align 4,              !dbg [[LINE25_G5R2:!.*]]
+// CHECK-NEXT: store atomic i32 %5, ptr @x monotonic, align 4,        !dbg [[LINE25_G5R1:!.*]]
+
+  int r3 = __atomic_test_and_set(&x, __ATOMIC_RELAXED);
+// CHECK-NEXT: %6 = atomicrmw xchg ptr @x, i8 1 monotonic, align 4,   !dbg [[LINE30:!.*]]
+// CHECK-NEXT: %tobool = icmp ne i8 %6, 0,                            !dbg [[LINE30_G7R2:!.*]]
+// CHECK-NEXT: store i1 %tobool, ptr %atomic-temp3, align 1,          !dbg [[LINE30_G7R1:!.*]]
+// CHECK-NEXT: %7 = load i8, ptr %atomic-temp3, align 1,              !dbg [[LINE30_G6R4:!.*]]
+// CHECK-NEXT: %loadedv = trunc i8 %7 to i1,                          !dbg [[LINE30_G6R3:!.*]]
+// CHECK-NEXT: %conv = zext i1 %loadedv to i32,                       !dbg [[LINE30_G6R2:!.*]]
+// CHECK-NEXT: store i32 %conv, ptr %r3, align 4,                     !dbg [[LINE30_G6R1:!.*]]
+
+  __atomic_clear(&x, __ATOMIC_RELAXED);
+// CHECK-NEXT: store atomic i8 0, ptr @x monotonic, align 4,          !dbg [[LINE39_G8R1:!.*]]
+
+  int r4 = __c11_atomic_exchange(&x, 2,__ATOMIC_RELAXED);
+// CHECK-NEXT: store i32 2, ptr %.atomictmp4, align 4,                !dbg [[LINE42_G10R1:!.*]]
+// CHECK-NEXT: %8 = load i32, ptr %.atomictmp4, align 4,              !dbg [[LINE42:!.*]]
+// CHECK-NEXT: %9 = atomicrmw xchg ptr @x, i32 %8 monotonic, align 4, !dbg [[LINE42_G10R2:!.*]]
+// CHECK-NEXT: store i32 %9, ptr %atomic-temp5, align 4,              !dbg [[LINE42_G10R1:!.*]]
+// CHECK-NEXT: %10 = load i32, ptr %atomic-temp5, align 4,            !dbg [[LINE42_G9R2:!.*]]
+// CHECK-NEXT: store i32 %10, ptr %r4, align 4,                       !dbg [[LINE42_G9R1:!.*]]
+
+  int r5 = __atomic_compare_exchange(&y, &y, &y, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+// CHECK-NEXT: %11 = load i32, ptr @y, align 4,                       !dbg [[LINE50:!.*]]
+// CHECK-NEXT: %12 = load i32, ptr @y, align 4,                       !dbg [[LINE50]]
+// CHECK-NEXT: %13 = cmpxchg ptr @y, i32 %11, i32 %12 monotonic monotonic, align 4, !dbg [[LINE50]]
+// CHECK-NEXT: %14 = extractvalue { i32, i1 } %13, 0,                 !dbg [[LINE50_G12R2:!.*]]
+// CHECK-NEXT: %15 = extractvalue { i32, i1 } %13, 1,                 !dbg [[LINE50_G12R3:!.*]]
+// CHECK-NEXT: br i1 %15, label %cmpxchg.continue, label %cmpxchg.store_expected, !dbg [[LINE50]]
+// CHECK: cmpxchg.store_expected:
+// CHECK-NEXT: store i32 %14, ptr @y, align 4,                        !dbg [[LINE50_G12R1:!.*]]
+// CHECK-NEXT: br label %cmpxchg.continue,                            !dbg [[LINE50]]
+// CHECK: cmpxchg.continue:
+// CHECK-NEXT: %storedv = zext i1 %15 to i8,                          !dbg [[LINE50_G12R2]]
+// CHECK-NEXT: store i8 %storedv, ptr %cmpxchg.bool, align 1,         !dbg [[LINE50_G12R1]]
+// CHECK-NEXT: %16 = load i8, ptr %cmpxchg.bool, align 1,             !dbg [[LINE50_G11R4:!.*]]
+// CHECK-NEXT: %loadedv6 = trunc i8 %16 to i1,                        !dbg [[LINE50_G11R3:!.*]]
+// CHECK-NEXT: %conv7 = zext i1 %loadedv6 to i32,                     !dbg [[LINE50_G11R2:!.*]]
+// CHECK-NEXT: store i32 %conv7, ptr %r5, align 4,                    !dbg [[LINE50_G11R1:!.*]]
+
+  int r6 = __c11_atomic_compare_exchange_strong(&x, &y, 42, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+// CHECK-NEXT: store i32 42, ptr %.atomictmp8, align 4,               !dbg [[LINE68_G14R1:!.*]]
+// CHECK-NEXT: %17 = load i32, ptr @y, align 4,                       !dbg [[LINE68:!.*]]
+// CHECK-NEXT: %18 = load i32, ptr %.atomictmp8, align 4,             !dbg [[LINE68]]
+// CHECK-NEXT: %19 = cmpxchg ptr @x, i32 %17, i32 %18 monotonic monotonic, align 4, !dbg [[LINE68]]
+// CHECK-NEXT: %20 = extractvalue { i32, i1 } %19, 0,                 !dbg [[LINE68_G14R2:!.*]]
+// CHECK-NEXT: %21 = extractvalue { i32, i1 } %19, 1,                 !dbg [[LINE68_G14R3:!.*]]
+// CHECK-NEXT: br i1 %21, label %cmpxchg.continue11, label %cmpxchg.store_expected10, !dbg [[LINE68]]
+// CHECK: cmpxchg.store_expected10:
+// CHECK-NEXT: store i32 %20, ptr @y, align 4,                        !dbg [[LINE68_G14R1:!.*]]
+// CHECK-NEXT: br label %cmpxchg.continue11,                          !dbg [[LINE68]]
+// CHECK: cmpxchg.continue11:
+// CHECK-NEXT: %storedv12 = zext i1 %21 to i8,                        !dbg [[LINE68_G14R2]]
+// CHECK-NEXT: store i8 %storedv12, ptr %cmpxchg.bool9, align 1,      !dbg [[LINE68_G14R1:!.*]]
+// CHECK-NEXT: %22 = load i8, ptr %cmpxchg.bool9, align 1,            !dbg [[LINE68_G13R4:!.*]]
+// CHECK-NEXT: %loadedv13 = trunc i8 %22 to i1,                       !dbg [[LINE68_G13R3:!.*]]
+// CHECK-NEXT: %conv14 = zext i1 %loadedv13 to i32,                   !dbg [[LINE68_G13R2:!.*]]
+// CHECK-NEXT: store i32 %conv14, ptr %r6, align 4,                   !dbg [[LINE68_G13R1:!.*]]
+
+  int r7 = __c11_atomic_compare_exchange_weak(&x, &y, 43, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+// CHECK-NEXT: store i32 43, ptr %.atomictmp15, align 4,              !dbg [[LINE87_G16R1:!.*]]
+// CHECK-NEXT: %23 = load i32, ptr @y, align 4,                       !dbg [[LINE87:!.*]]
+// CHECK-NEXT: %24 = load i32, ptr %.atomictmp15, align 4,            !dbg [[LINE87]]
+// CHECK-NEXT: %25 = cmpxchg weak ptr @x, i32 %23, i32 %24 monotonic monotonic, align 4, !dbg [[LINE87]]
+// CHECK-NEXT: %26 = extractvalue { i32, i1 } %25, 0,                 !dbg [[LINE87_G16R2:!.*]]
+// CHECK-NEXT: %27 = extractvalue { i32, i1 } %25, 1,                 !dbg [[LINE87_G16R3:!.*]]
+// CHECK-NEXT: br i1 %27, label %cmpxchg.continue18, label %cmpxchg.store_expected17, !dbg [[LINE87]]
+// CHECK: cmpxchg.store_expected17:
+// CHECK-NEXT: store i32 %26, ptr @y, align 4,                        !dbg [[LINE87_G16R1]]
+// CHECK-NEXT: br label %cmpxchg.continue18,                          !dbg [[LINE87]]
+// CHECK: cmpxchg.continue18:
+// CHECK-NEXT: %storedv19 = zext i1 %27 to i8,                        !dbg [[LINE87_G16R2]]
+// CHECK-NEXT: store i8 %storedv19, ptr %cmpxchg.bool16, align 1,     !dbg [[LINE87_G16R1]]
+// CHECK-NEXT: %28 = load i8, ptr %cmpxchg.bool16, align 1,           !dbg [[LINE87_G15R4:!.*]]
+// CHECK-NEXT: %loadedv20 = trunc i8 %28 to i1,                       !dbg [[LINE87_G15R3:!.*]]
+// CHECK-NEXT: %conv21 = zext i1 %loadedv20 to i32,                   !dbg [[LINE87_G15R2:!.*]]
+// CHECK-NEXT: store i32 %conv21, ptr %r7, align 4,                   !dbg [[LINE87_G15R1:!.*]]
+
+// CHECK: ret{{.*}}, !dbg [[RET:!.*]]
+}
+
+// CHECK: [[LINE11_G2R1]] = !DILocation(line: 11, scope: ![[#]], atomGroup: 2, atomRank: 1)
+// CHECK: [[LINE11]]      = !DILocation(line: 11, scope: ![[#]])
+// CHECK: [[LINE11_G2R2]] = !DILocation(line: 11, scope: ![[#]], atomGroup: 2, atomRank: 2)
+// CHECK: [[LINE11_G1R2]] = !DILocation(line: 11, scope: ![[#]], atomGroup: 1, atomRank: 2)
+// CHECK: [[LINE11_G1R1]] = !DILocation(line: 11, scope: ![[#]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[LINE19_G4R2]] = !DILocation(line: 19, scope: ![[#]], atomGroup: 4, atomRank: 2)
+// CHECK: [[LINE19_G4R1]] = !DILocation(line: 19, scope: ![[#]], atomGroup: 4, atomRank: 1)
+// CHECK: [[LINE19_G3R2]] = !DILocation(line: 19, scope: ![[#]], atomGroup: 3, atomRank: 2)
+// CHECK: [[LINE19_G3R1]] = !DILocation(line: 19, scope: ![[#]], atomGroup: 3, atomRank: 1)
+
+// CHECK: [[LINE25_G5R1]] = !DILocation(line: 25, scope: ![[#]], atomGroup: 5, atomRank: 1)
+// CHECK: [[LINE25_G5R2]] = !DILocation(line: 25, scope: ![[#]], atomGroup: 5, atomRank: 2)
+
+// CHECK: [[LINE30]]      = !DILocation(line: 30, scope: ![[#]])
+// CHECK: [[LINE30_G7R2]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 7, atomRank: 2)
+// CHECK: [[LINE30_G7R1]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 7, atomRank: 1)
+// CHECK: [[LINE30_G6R4]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 6, atomRank: 4)
+// CHECK: [[LINE30_G6R3]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 6, atomRank: 3)
+// CHECK: [[LINE30_G6R2]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 6, atomRank: 2)
+// CHECK: [[LINE30_G6R1]] = !DILocation(line: 30, scope: ![[#]], atomGroup: 6, atomRank: 1)
+
+// CHECK: [[LINE39_G8R1]] = !DILocation(line: 39, scope: ![[#]], atomGroup: 8, atomRank: 1)
+
+// CHECK: [[LINE42_G10R1]] = !DILocation(line: 42, scope: ![[#]], atomGroup: 10, atomRank: 1)
+// CHECK: [[LINE42]]       = !DILocation(line: 42, scope: ![[#]])
+// CHECK: [[LINE42_G10R2]] = !DILocation(line: 42, scope: ![[#]], atomGroup: 10, atomRank: 2)
+// CHECK: [[LINE42_G9R2]]  = !DILocation(line: 42, scope: ![[#]], atomGroup: 9, atomRank: 2)
+// CHECK: [[LINE42_G9R1]]  = !DILocation(line: 42, scope: ![[#]], atomGroup: 9, atomRank: 1)
+
+// CHECK: [[LINE50]]       = !DILocation(line: 50, scope: ![[#]])
+// CHECK: [[LINE50_G12R2]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 12, atomRank: 2)
+// CHECK: [[LINE50_G12R3]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 12, atomRank: 3)
+// CHECK: [[LINE50_G12R1]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 12, atomRank: 1)
+// CHECK: [[LINE50_G11R4]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 11, atomRank: 4)
+// CHECK: [[LINE50_G11R3]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 11, atomRank: 3)
+// CHECK: [[LINE50_G11R2]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 11, atomRank: 2)
+// CHECK: [[LINE50_G11R1]] = !DILocation(line: 50, scope: ![[#]], atomGroup: 11, atomRank: 1)
+
+// CHECK: [[LINE68_G14R1]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 14, atomRank: 1)
+// CHECK: [[LINE68]]       = !DILocation(line: 68, scope: ![[#]])
+// CHECK: [[LINE68_G14R2]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 14, atomRank: 2)
+// CHECK: [[LINE68_G14R3]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 14, atomRank: 3)
+// CHECK: [[LINE68_G13R4]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 13, atomRank: 4)
+// CHECK: [[LINE68_G13R3]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 13, atomRank: 3)
+// CHECK: [[LINE68_G13R2]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 13, atomRank: 2)
+// CHECK: [[LINE68_G13R1]] = !DILocation(line: 68, scope: ![[#]], atomGroup: 13, atomRank: 1)
+
+// CHECK: [[LINE87_G16R1]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 16, atomRank: 1)
+// CHECK: [[LINE87]]       = !DILocation(line: 87, scope: ![[#]])
+// CHECK: [[LINE87_G16R2]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 16, atomRank: 2)
+// CHECK: [[LINE87_G16R3]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 16, atomRank: 3)
+// CHECK: [[LINE87_G15R4]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 15, atomRank: 4)
+// CHECK: [[LINE87_G15R3]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 15, atomRank: 3)
+// CHECK: [[LINE87_G15R2]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 15, atomRank: 2)
+// CHECK: [[LINE87_G15R1]] = !DILocation(line: 87, scope: ![[#]], atomGroup: 15, atomRank: 1)
+
+// CHECK: [[RET]] = !DILocation({{.*}}, atomGroup: 17, atomRank: 1)
diff --git a/clang/test/DebugInfo/KeyInstructions/flag.cpp b/clang/test/DebugInfo/KeyInstructions/flag.cpp
index 93503dd4bdb4..813f7e908011 100644
--- a/clang/test/DebugInfo/KeyInstructions/flag.cpp
+++ b/clang/test/DebugInfo/KeyInstructions/flag.cpp
@@ -3,13 +3,19 @@
 //// Default: Off.
 // RUN: %clang -### -target x86_64 -c -gdwarf %s 2>&1 | FileCheck %s --check-prefixes=NO-KEY-INSTRUCTIONS
 
-//// Help hidden.
-// RUN %clang --help | FileCheck %s --check-prefix=HELP
-// HELP-NOT: key-instructions
-
 // KEY-INSTRUCTIONS: "-gkey-instructions"
-// KEY-INSTRUCTIONS: "-mllvm" "-dwarf-use-key-instructions"
-
 // NO-KEY-INSTRUCTIONS-NOT: key-instructions
 
-//// TODO: Add smoke test once some functionality has been added.
+//// Help hidden: flag should not be visible.
+// RUN: %clang --help | FileCheck %s --check-prefix=HELP
+// HELP-NOT: key-instructions
+
+// Smoke test: check for Key Instructions keywords in the IR.
+void f() {}
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -debug-info-kind=line-tables-only -emit-llvm -o - | FileCheck %s --check-prefix=SMOKETEST-OFF
+// SMOKETEST-OFF-NOT: keyInstructions:
+// SMOKETEST-OFF-NOT: atomGroup
+
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -gkey-instructions -debug-info-kind=line-tables-only -emit-llvm -o - | FileCheck %s --check-prefix=SMOKETEST-ON
+// SMOKETEST-ON: keyInstructions: true
+// SMOKETEST-ON: atomGroup: 1
diff --git a/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp
new file mode 100644
index 000000000000..c94fc588bf13
--- /dev/null
+++ b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable-2.cpp
@@ -0,0 +1,59 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions %s -gno-column-info -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// g::h and i can be memcpy'd, check the assignment gets Key Instructions metadata.
+
+struct e {
+  e(e &);
+  e& operator=(const e&);
+};
+
+struct g {
+  e f;
+  int h;
+  int i;
+};
+
+// Copy assignment operator.
+// CHECK: define{{.*}}ptr @_ZN1gaSERKS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %call = call {{.*}}ptr @_ZN1eaSERKS_(ptr {{.*}}%this1, ptr {{.*}}%1)
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: call void @llvm.memcpy{{.*}}(ptr align 4 %h, ptr align 4 %h2, i64 8, i1 false), !dbg [[S1_G1R1:!.*]]
+// CHECK-NEXT: ret ptr %this1, !dbg
+
+// Copy ctor.
+// CHECK: define{{.*}}void @_ZN1gC2ERS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: call void @_ZN1eC1ERS_
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: call void @llvm.memcpy{{.*}}(ptr align 4 %h, ptr align 4 %h2, i64 8, i1 false), !dbg [[S2_G1R1:!.*]]
+// CHECK-NEXT: ret void, !dbg
+
+// CHECK: [[S1:!.*]] = distinct !DISubprogram(name: "operator=",
+// CHECK: [[S1_G1R1]] = !DILocation(line: 11, scope: [[S1]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[S2:!.*]] = distinct !DISubprogram(name: "g",
+// CHECK: [[S2_G1R1]] = !DILocation(line: 11, scope: [[S2]], atomGroup: 1, atomRank: 1)
+
+[[gnu::nodebug]]
+void fun(g *x) {
+  g y = g(*x);
+  y = *x;
+}
diff --git a/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp
new file mode 100644
index 000000000000..cd3807735fa3
--- /dev/null
+++ b/clang/test/DebugInfo/KeyInstructions/init-member-memcopyable.cpp
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -gkey-instructions %s -gno-column-info -debug-info-kind=line-tables-only -emit-llvm -o - \
+// RUN: | FileCheck %s
+
+// g::h can be memcpy'd (in this case emitted as load/stored), check the
+// assignment gets Key Instructions metadata.
+
+struct e {
+  e(e&);
+  e& operator=(const e&);
+};
+
+struct g {
+  e f;
+  int h;
+};
+
+// Copy assignment operator.
+// CHECK: define{{.*}}ptr @_ZN1gaSERKS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %call = call {{.*}}ptr @_ZN1eaSERKS_(ptr {{.*}}%this1, ptr {{.*}}%1)
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: %3 = load i32, ptr %h, align 4, !dbg [[S1_G1R2:!.*]]
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: store i32 %3, ptr %h2, align 4, !dbg [[S1_G1R1:!.*]]
+// CHECK-NEXT: ret ptr %this1, !dbg
+
+// Copy ctor.
+// CHECK: define{{.*}}void @_ZN1gC2ERS_
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %this.addr = alloca ptr, align 8
+// CHECK-NEXT: %.addr = alloca ptr, align 8
+// CHECK-NEXT: store ptr %this, ptr %this.addr, align 8
+// CHECK-NEXT: store ptr %0, ptr %.addr, align 8
+// CHECK-NEXT: %this1 = load ptr, ptr %this.addr, align 8
+// CHECK-NEXT: %1 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: call void @_ZN1eC1ERS_
+// CHECK-NEXT: %h = getelementptr inbounds nuw %struct.g, ptr %this1, i32 0, i32 1
+// CHECK-NEXT: %2 = load ptr, ptr %.addr, align 8
+// CHECK-NEXT: %h2 = getelementptr inbounds nuw %struct.g, ptr %2, i32 0, i32 1
+// CHECK-NEXT: %3 = load i32, ptr %h2, align 4, !dbg [[S2_G1R2:!.*]]
+// CHECK-NEXT: store i32 %3, ptr %h, align 4, !dbg [[S2_G1R1:!.*]]
+// CHECK-NEXT: ret void, !dbg
+
+// CHECK: [[S1:!.*]] = distinct !DISubprogram(name: "operator=",
+// CHECK: [[S1_G1R2]] = !DILocation(line: 12, scope: [[S1]], atomGroup: 1, atomRank: 2)
+// CHECK: [[S1_G1R1]] = !DILocation(line: 12, scope: [[S1]], atomGroup: 1, atomRank: 1)
+
+// CHECK: [[S2:!.*]] = distinct !DISubprogram(name: "g",
+// CHECK: [[S2_G1R2]] = !DILocation(line: 12, scope: [[S2]], atomGroup: 1, atomRank: 2)
+// CHECK: [[S2_G1R1]] = !DILocation(line: 12, scope: [[S2]], atomGroup: 1, atomRank: 1)
+
+void fun(g *x) {
+  g y = g(*x);
+  y = *x;
+}
diff --git a/clang/test/Driver/HLSL/dxc_fvk_layout.hlsl b/clang/test/Driver/HLSL/dxc_fvk_layout.hlsl
new file mode 100644
index 000000000000..a796fbb8e052
--- /dev/null
+++ b/clang/test/Driver/HLSL/dxc_fvk_layout.hlsl
@@ -0,0 +1,8 @@
+// No errors. Otherwise nothing observable.
+// RUN: %clang_dxc -fvk-use-dx-layout -spirv -Tlib_6_7 -### %s
+
+// RUN: not %clang_dxc -fvk-use-scalar-layout -spirv -Tlib_6_7 -### %s 2>&1 | FileCheck %s -check-prefix=SCALAR
+// SCALAR: error: the clang compiler does not support '-fvk-use-scalar-layout'
+
+// RUN: not %clang_dxc -fvk-use-gl-layout -spirv -Tlib_6_7 -### %s 2>&1 | FileCheck %s -check-prefix=GL
+// GL: error: the clang compiler does not support '-fvk-use-gl-layout' 
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/a.txt b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/a.txt
rename to clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/dir/b.txt
rename to clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/aarch64-gnutools.c b/clang/test/Driver/aarch64-gnutools.c
new file mode 100644
index 000000000000..0214639ed380
--- /dev/null
+++ b/clang/test/Driver/aarch64-gnutools.c
@@ -0,0 +1,4 @@
+// RUN: %clang --target=aarch64-none-elf  --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/aarch64-implied-sve-features.c b/clang/test/Driver/aarch64-implied-sve-features.c
index ecc1e9500b66..18c39974a5c1 100644
--- a/clang/test/Driver/aarch64-implied-sve-features.c
+++ b/clang/test/Driver/aarch64-implied-sve-features.c
@@ -46,13 +46,13 @@
 // SVE2-AES-REVERT: "-target-feature" "+sve" "-target-feature" "-sve-aes" "-target-feature" "+sve2" "-target-feature" "-sve2-aes"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sha3+nosve2-sha3 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SHA3-REVERT
-// SVE2-SHA3-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-sha3"
+// SVE2-SHA3-REVERT: "-target-feature" "+sve" "-target-feature" "-sve-sha3" "-target-feature" "+sve2" "-target-feature" "-sve2-sha3"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sm4+nosve2-sm4 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SM4-REVERT
 // SVE2-SM4-REVERT: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "-sve2-sm4"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve2-sha3 %s -### 2>&1 | FileCheck %s --check-prefix=SVE2-SHA3
-// SVE2-SHA3: "-target-feature" "+sve" "-target-feature" "+sve2" "-target-feature" "+sve2-sha3"
+// SVE2-SHA3: "-target-feature" "+sve" "-target-feature" "+sve-sha3" "-target-feature" "+sve2" "-target-feature" "+sve2-sha3"
 
 // RUN: %clang --target=aarch64-linux-gnu -march=armv8-a+sve-aes %s -### 2>&1 | FileCheck %s --check-prefix=SVE-AES
 // SVE-AES: "-target-feature" "+aes"{{.*}} "-target-feature" "+sve-aes"
diff --git a/clang/test/Driver/aarch64-sve-vector-bits.c b/clang/test/Driver/aarch64-sve-vector-bits.c
index 535b0f157019..f54cd94764bf 100644
--- a/clang/test/Driver/aarch64-sve-vector-bits.c
+++ b/clang/test/Driver/aarch64-sve-vector-bits.c
@@ -24,6 +24,8 @@
 // RUN:  -msve-vector-bits=2048+ 2>&1 | FileCheck --check-prefix=CHECK-2048P %s
 // RUN: %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve \
 // RUN:  -msve-vector-bits=scalable 2>&1 | FileCheck --check-prefix=CHECK-SCALABLE %s
+// RUN: %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve+sme \
+// RUN:  -msve-streaming-vector-bits=128 2>&1 | FileCheck --check-prefix=STREAMING-128 %s
 
 // CHECK-128: "-mvscale-max=1" "-mvscale-min=1"
 // CHECK-256: "-mvscale-max=2" "-mvscale-min=2"
@@ -44,6 +46,8 @@
 // CHECK-SCALABLE-NOT: "-mvscale-min=
 // CHECK-SCALABLE-NOT: "-mvscale-max=
 
+// STREAMING-128: "-mvscale-streaming-max=1" "-mvscale-streaming-min=1"
+
 // Error out if an unsupported value is passed to -msve-vector-bits.
 // -----------------------------------------------------------------------------
 // RUN: not %clang -c %s -### --target=aarch64-none-linux-gnu -march=armv8-a+sve \
diff --git a/clang/test/Driver/aarch64-toolchain-extra.c b/clang/test/Driver/aarch64-toolchain-extra.c
new file mode 100644
index 000000000000..4945a622969c
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain-extra.c
@@ -0,0 +1,35 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in aarch64-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/aarch64-nogcc/bin
+// RUN: ln -s %clang %t/aarch64-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf %t/aarch64-nogcc/aarch64-none-elf
+// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld %t/aarch64-nogcc/bin/aarch64-none-elf-ld
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/aarch64-nogcc/invalid \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOGCC %s
+
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/aarch64-nogcc/bin/../aarch64-none-elf \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOGCC %s
+
+// C-AARCH64-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/{{.*}}/aarch64-none-elf/lib/crtbegin.o"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL-NOGCC: "{{.*}}/aarch64-nogcc/{{.*}}/aarch64-none-elf/lib/crtend.o"
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
new file mode 100644
index 000000000000..cfad4b8eb682
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain.c
@@ -0,0 +1,159 @@
+// UNSUPPORTED: system-windows
+
+// Test interaction with -fuse-ld=lld
+// RUN: %clang -### %s -fuse-ld=lld -B%S/Inputs/lld \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=LLD-AARCH64-BAREMETAL %s
+
+// LLD-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// LLD-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// LLD-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/lld/ld.lld"
+// LLD-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// LLD-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// LLD-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// LLD-AARCH64-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// LLD-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL %s
+
+// C-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// C-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// C-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// C-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// C-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "aarch64linux" "-EL"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL %s
+
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "--sysroot={{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../bin/aarch64-none-elf-ld"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-Bstatic" "-m" "aarch64linux" "-EL"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib/crt0.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/lib"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=compiler-rt --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-COMPILER-RT %s
+
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}crt0.o"
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtbegin.o"
+// AARCH64-BAREMETAL-COMPILER-RT: "--start-group" "{{.*}}libclang_rt.builtins{{.*}}.a" "-lc" "-lgloss" "--end-group"
+// AARCH64-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=platform --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-UNWINDLIB %s
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=compiler-rt --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=AARCH64-BAREMETAL-UNWINDLIB %s
+
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}crt0.o"
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
+// AARCH64-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins{{.*}}.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
diff --git a/clang/test/Driver/amdgpu-openmp-sanitize-options.c b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
index f6a8a7dc57cc..985eca169280 100644
--- a/clang/test/Driver/amdgpu-openmp-sanitize-options.c
+++ b/clang/test/Driver/amdgpu-openmp-sanitize-options.c
@@ -54,11 +54,11 @@
 // NOXNACK: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 // XNACKNEG: warning: ignoring '-fsanitize=address' option for offload arch 'gfx908:xnack-' as it is not currently supported there. Use it with an offload arch containing 'xnack+' instead
 
-// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
+// HOSTSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "c".*}}
 
 // GPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-mlink-bitcode-file" "[^"]*asanrtl.bc".* "-mlink-bitcode-file" "[^"]*ockl.bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-fsanitize=address".* "-x" "c".*}}
 // NOGPUSAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu".* "-emit-llvm-bc".* "-target-cpu" "(gfx908|gfx900)".* "-fopenmp".* "-x" "c".*}}
 
 // SAN: {{"[^"]*clang-offload-packager[^"]*" "-o".* "--image=file=.*.bc,triple=amdgcn-amd-amdhsa,arch=gfx908(:xnack\-|:xnack\+)?,kind=openmp(,feature=(\-xnack|\+xnack))?"}}
-// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "-fopenmp-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
+// SAN: {{"[^"]*clang[^"]*" "-cc1" "-triple" "x86_64-unknown-linux-gnu".* "-fopenmp".* "-fsanitize=address".* "--offload-targets=amdgcn-amd-amdhsa".* "-x" "ir".*}}
 // SAN: {{"[^"]*clang-linker-wrapper[^"]*".* "--host-triple=x86_64-unknown-linux-gnu".* "--linker-path=[^"]*".* "--whole-archive" "[^"]*(libclang_rt.asan_static.a|libclang_rt.asan_static-x86_64.a)".* "--whole-archive" "[^"]*(libclang_rt.asan.a|libclang_rt.asan-x86_64.a)".*}}
diff --git a/clang/test/Driver/arm-gnutools.c b/clang/test/Driver/arm-gnutools.c
new file mode 100644
index 000000000000..6e107f19dabc
--- /dev/null
+++ b/clang/test/Driver/arm-gnutools.c
@@ -0,0 +1,6 @@
+// check that gnu assembler is invoked with arm baremetal as well
+
+// RUN: %clang --target=armv6m-none-eabi  --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/arm-toolchain-extra.c b/clang/test/Driver/arm-toolchain-extra.c
new file mode 100644
index 000000000000..43cca6112176
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain-extra.c
@@ -0,0 +1,36 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in arm-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/arm-nogcc/bin
+// RUN: ln -s %clang %t/arm-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi %t/arm-nogcc/armv6m-none-eabi
+// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld %t/arm-nogcc/bin/armv6m-none-eabi-ld
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/arm-nogcc/invalid \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/arm-nogcc/bin/../armv6m-none-eabi \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/include"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/{{.*}}/armv6m-none-eabi/lib/crtbegin.o"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL-NOGCC: "{{.*}}/arm-nogcc/{{.*}}/armv6m-none-eabi/lib/crtend.o"
+
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
new file mode 100644
index 000000000000..c367594b0a75
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain.c
@@ -0,0 +1,160 @@
+// UNSUPPORTED: system-windows
+
+// RUN: %clang -### %s -fuse-ld=lld -B%S/Inputs/lld \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=LLD-ARM-BAREMETAL %s
+
+// LLD-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// LLD-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// LLD-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/lld/ld.lld"
+// LLD-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// LLD-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// LLD-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// LLD-ARM-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// LLD-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL %s
+
+// C-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// C-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// C-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// C-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// C-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL-NOSYSROOT %s
+
+// C-ARM-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// C-ARM-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// C-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// C-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}.o" "--start-group" "-lgcc" "--as-needed" "-lgcc_s" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// C-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL %s
+
+// CXX-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}.o" "-lstdc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-LIBCXX: "--sysroot={{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-LIBCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../bin/armv6m-none-eabi-ld"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib/crt0.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-L{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/lib"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}.o" "-lc++" "-lm" "--start-group" "-lgcc_s" "-lgcc" "-lc" "-lgloss" "--end-group"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=compiler-rt --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-COMPILER-RT %s
+
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}crt0.o"
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtbegin.o"
+// ARM-BAREMETAL-COMPILER-RT: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
+// ARM-BAREMETAL-COMPILER-RT: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=platform --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-UNWINDLIB %s
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=compiler-rt --unwindlib=libunwind \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=ARM-BAREMETAL-UNWINDLIB %s
+
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}crt0.o"
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
+// ARM-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
+// ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
diff --git a/clang/test/Driver/autocomplete.c b/clang/test/Driver/autocomplete.c
index 8cc604dbff87..4983b7149683 100644
--- a/clang/test/Driver/autocomplete.c
+++ b/clang/test/Driver/autocomplete.c
@@ -111,6 +111,7 @@
 // RUN: %clang --autocomplete=-Wma | FileCheck %s -check-prefix=WARNING
 // WARNING: -Wmacro-redefined
 // WARNING-NEXT: -Wmain
+// WARNING-NEXT: -Wmain-attached-to-named-module
 // WARNING-NEXT: -Wmain-return-type
 // WARNING-NEXT: -Wmalformed-warning-check
 // WARNING-NEXT: -Wmany-braces-around-scalar-init
diff --git a/clang/test/Driver/baremetal-multilib.yaml b/clang/test/Driver/baremetal-multilib.yaml
index 853a4e9e36e4..1a80c3b4ccfc 100644
--- a/clang/test/Driver/baremetal-multilib.yaml
+++ b/clang/test/Driver/baremetal-multilib.yaml
@@ -8,8 +8,9 @@
 # CHECK-SAME: "-internal-isystem" "[[SYSROOT:[^"]*]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/include/c++/v1"
 # CHECK-SAME: "-internal-isystem" "[[SYSROOT]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/include"
 # CHECK-SAME: "-x" "c++" "{{.*}}baremetal-multilib.yaml"
-# CHECK-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+# CHECK-NEXT: ld{{(.exe)?}}" "-Bstatic"
 # CHECK-SAME: "-L[[SYSROOT]]/bin/../lib/clang-runtimes/arm-none-eabi/thumb/v8-m.main/fp/lib"
+# CHECK-SAME: "{{.*}}.o"
 # CHECK-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 # CHECK-SAME: "-lc"
 # CHECK-SAME: "-o" "{{.*}}.tmp.out"
diff --git a/clang/test/Driver/baremetal-sysroot.cpp b/clang/test/Driver/baremetal-sysroot.cpp
index 5d5b336a01b0..47f0616df850 100644
--- a/clang/test/Driver/baremetal-sysroot.cpp
+++ b/clang/test/Driver/baremetal-sysroot.cpp
@@ -9,15 +9,17 @@
 // RUN: mkdir -p %T/baremetal_default_sysroot/lib/clang-runtimes/armv6m-none-eabi
 // RUN: ln -s %clang %T/baremetal_default_sysroot/bin/clang
 
-// RUN: %T/baremetal_default_sysroot/bin/clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
+// RUN: %T/baremetal_default_sysroot/bin/clang -no-canonical-prefixes %s -### -o %t.out 2>&1 \
 // RUN:     -target armv6m-none-eabi --sysroot= \
 // RUN:   | FileCheck --check-prefix=CHECK-V6M-C %s
 // CHECK-V6M-C: "{{.*}}clang{{.*}}" "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
 // CHECK-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include"
 // CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal-sysroot.cpp"
-// CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "-Bstatic"
+// CHECK-V6M-C-SAME: "crt0.o"
 // CHECK-V6M-C-SAME: "-L{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}lib"
+// CHECK-V6M-C-SAME: "{{.*}}.o"
 // CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-C-SAME: "-lc"
-// CHECK-V6M-C-SAME: "-o" "{{.*}}.o"
+// CHECK-V6M-C-SAME: "-o" "{{.*}}.tmp.out"
diff --git a/clang/test/Driver/baremetal-undefined-symbols.c b/clang/test/Driver/baremetal-undefined-symbols.c
new file mode 100644
index 000000000000..6ed955b1b749
--- /dev/null
+++ b/clang/test/Driver/baremetal-undefined-symbols.c
@@ -0,0 +1,10 @@
+// Check the arguments are correctly passed
+
+// Make sure -T is the last with gcc-toolchain option
+// RUN: %clang -### --target=aarch64-none-elf --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// RUN: %clang -### --target=armv6m-none-eabi --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// RUN: %clang -### --target=riscv32 --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHECK-LD %s
+// CHECK-LD: {{.*}} "-T" "a.lds" "-u" "foo" {{.*}} "--defsym=FOO=10"
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index a80aa9b43711..4dc320191317 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -15,11 +15,14 @@
 // CHECK-V6M-C-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-C-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-V6M-C-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-C-NEXT: ld{{(.exe)?}}"
+// CHECK-V6M-C-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-C-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-C-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-C-SAME: "-T" "semihosted.lds" "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-V6M-C-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
-// CHECK-V6M-C-SAME: "{{[^"]*}}libclang_rt.builtins.a"
+// CHECK-V6M-C-SAME: "{{.*}}.o"
+// CHECK-V6M-C-SAME: {{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-C-SAME: "-lc"
 // CHECK-V6M-C-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
 
@@ -39,9 +42,11 @@
 // CHECK-V6M-TREE-SAME: {{^}} "-internal-isystem" "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-TREE-SAME: "-internal-isystem" "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}include{{[/\\]+}}armv6m-unknown-none-eabi"
 // CHECK-V6M-TREE-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-V6M-TREE-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-TREE-NEXT: ld{{(.exe)?}}"
+// CHECK-V6M-TREE-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-TREE-SAME: "[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi{{[/\\]+}}crt0.o"
 // CHECK-V6M-TREE-SAME: "-L[[INSTALLED_DIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}armv6m-unknown-none-eabi"
+// CHECK-V6M-TREE-SAME "{{.*}}.o"
 // CHECK-V6M-TREE-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-TREE-SAME: "-lc"
 // CHECK-V6M-TREE-SAME: "--target2=rel" "-o" "{{.*}}.tmp.out"
@@ -53,19 +58,25 @@
 // CHECK-ARMV7M-PER-TARGET: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-ARMV7M-PER-TARGET: "-isysroot" "[[SYSROOT:[^"]*]]"
 // CHECK-ARMV7M-PER-TARGET: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-ARMV7M-PER-TARGET: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-ARMV7M-PER-TARGET: ld{{(.exe)?}}"
+// CHECK-ARMV7M-PER-TARGET: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7M-PER-TARGET: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-ARMV7M-PER_TARGET: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-ARMV7M-PER-TARGET: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
 // CHECK-ARMV7M-PER-TARGET: "-L[[RESOURCE_DIR:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}armv7m-vendor-none-eabi
+// CHECK-ARMV7M-PER-TARGET: "{{.*}}.o"
 // CHECK-ARMV7M-PER-TARGET: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-ARMV7M-PER-TARGET: "-lc"
 
 // RUN: %clangxx %s -### --target=armv6m-none-eabi 2>&1 \
 // RUN:     --sysroot=%S/Inputs/baremetal_arm | FileCheck --check-prefix=CHECK-V6M-DEFAULTCXX %s
 // CHECK-V6M-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-V6M-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-V6M-DEFAULTCXX: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-DEFAULTCXX: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-DEFAULTCXX-SAME: "[[SYSROOT:[^"]+]]{{[/\\]+}}lib{{[/\\]+}}crt0.o"
 // CHECK-V6M-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-DEFAULTCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-DEFAULTCXX-SAME: "-lc++"
 // CHECK-V6M-DEFAULTCXX-SAME: "-lm"
 // CHECK-V6M-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
@@ -77,8 +88,11 @@
 // CHECK-V6M-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-V6M-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-V6M-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-V6M-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-LIBCXX: ld{{(.exe)?}}"
+// CHECK-V6M-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-LIBCXX-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-LIBCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-LIBCXX-SAME: "-lc++"
 // CHECK-V6M-LIBCXX-SAME: "-lm"
 // CHECK-V6M-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
@@ -92,8 +106,11 @@
 // CHECK-V6M-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-V6M-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}6.0.0"
-// CHECK-V6M-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-V6M-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-LIBSTDCXX-SAME: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
+// CHECK-V6M-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-V6M-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-V6M-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-LIBSTDCXX-SAME: "-lc"
@@ -104,7 +121,9 @@
 // RUN:     -nodefaultlibs \
 // RUN:   | FileCheck --check-prefix=CHECK-V6M-NDL %s
 // CHECK-V6M-NDL: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-V6M-NDL: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-V6M-NDL: ld{{(.exe)?}}"
+// CHECK-V6M-NDL: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-V6M-NDL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-V6M-NDL-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm{{[/\\]+}}lib"
 
 // RUN: rm -rf %T/baremetal_cxx_sysroot
@@ -119,6 +138,7 @@
 // CHECK-V6M-LIBCXX-USR-SAME: "-internal-isystem" "{{[^"]+}}baremetal_cxx_sysroot{{[/\\]+}}usr{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-V6M-LIBCXX-USR: "{{[^"]*}}-Bstatic"
 // CHECK-V6M-LIBCXX-USR-SAME: "-L{{[^"]*}}{{[/\\]+}}baremetal_cxx_sysroot{{[/\\]+}}lib"
+// CHECK-V6M-LIBCXX-USR: "{{.*}}.o"
 // CHECK-V6M-LIBCXX-USR-SAME: "-lc++" "-lm"
 // CHECK-V6M-LIBCXX-USR-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-V6M-LIBCXX-USR-SAME: "-lc"
@@ -149,7 +169,9 @@
 
 // RUN: %clang -### %s --target=armebv7-none-eabi --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EB %s
-// CHECK-ARMV7EB: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "--be8" "-EB"
+// CHECK-ARMV7EB: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ARMV7EB: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7EB: "-Bstatic" "-m" "armelfb_linux_eabi" "--be8" "-EB"
 
 // RUN: %clang -### %s --target=armv7-none-eabi -mbig-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EB %s
@@ -159,7 +181,9 @@
 
 // RUN: %clang -### %s --target=armv7-none-eabi --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-ARMV7EL %s
-// CHECK-ARMV7EL: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-ARMV7EL: "{{.*}}ld{{(.exe)?}}"
+// CHECK-ARMV7EL: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-ARMV7EL: "-Bstatic" "-m" "armelf_linux_eabi" "-EL"
 // CHECK-ARMV7EL-NOT: "--be8"
 
 // RUN: %clang -### %s --target=armebv7-none-eabi -mlittle-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -170,7 +194,9 @@
 
 // RUN: %clang -### %s --target=aarch64_be-none-elf --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64BE %s
-// CHECK-AARCH64BE: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EB"
+// CHECK-AARCH64BE: "{{.*}}ld{{(.exe)?}}"
+// CHECK-AARCH64BE: sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-AARCH64BE: "-Bstatic" "-m" "aarch64linuxb" "-EB"
 // CHECK-AARCH64BE-NOT: "--be8"
 
 // RUN: %clang -### %s --target=aarch64-none-elf -mbig-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -181,7 +207,9 @@
 
 // RUN: %clang -### %s --target=aarch64-none-elf --sysroot=%S/Inputs/baremetal_arm 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-AARCH64LE %s
-// CHECK-AARCH64LE: "{{.*}}ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic" "-EL"
+// CHECK-AARCH64LE: "{{.*}}ld{{(.exe)?}}"
+// CHECK-AARCH64LE: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}baremetal_arm"
+// CHECK-AARCH64LE: "-Bstatic" "-m" "aarch64linux" "-EL"
 // CHECK-AARCH64LE-NOT: "--be8"
 
 // RUN: %clang -### %s --target=aarch64_be-none-elf -mlittle-endian --sysroot=%S/Inputs/baremetal_arm 2>&1 \
@@ -196,6 +224,22 @@
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv32-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV32-NO-HOST-INC %s
+// CHECK-RISCV32-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV32-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv64-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV64-NO-HOST-INC %s
+// CHECK-RISCV64-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV64-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
 // RUN: %clang %s -### --target=riscv64-unknown-elf -o %t.out -L some/directory/user/asked/for \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64 %s
@@ -205,9 +249,12 @@
 // CHECK-RV64-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-RV64-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV64-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-NEXT: ld{{(.exe)?}}"
+// CHECK-RV64-SAME: "--sysroot={{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf"
+// CHECK-RV64-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV64-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
+// CHECK-RV64-SAME:"{{.*}}.o"
 // CHECK-RV64-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-SAME: "-lc"
 // CHECK-RV64-SAME: "-X" "-o" "{{.*}}.tmp.out"
@@ -216,8 +263,11 @@
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-DEFAULTCXX %s
 // CHECK-RV64-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV64-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-RV64-DEFAULTCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-DEFAULTCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-DEFAULTCXX-SAME:"{{.*}}.o"
 // CHECK-RV64-DEFAULTCXX-SAME: "-lc++" "-lm"
 // CHECK-RV64-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-DEFAULTCXX-SAME: "-lc"
@@ -230,8 +280,11 @@
 // CHECK-RV64-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV64-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-RV64-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RV64-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-LIBCXX: ld{{(.exe)?}}"
+// CHECK-RV64-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-LIBCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-LIBCXX-SAME:"{{.*}}.o"
 // CHECK-RV64-LIBCXX-SAME: "-lc++" "-lm"
 // CHECK-RV64-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-LIBCXX-SAME: "-lc"
@@ -244,8 +297,11 @@
 // CHECK-RV64-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV64-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
-// CHECK-RV64-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-RV64-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-LIBSTDCXX-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV64-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-RV64-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-RV64-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV64-LIBSTDCXX-SAME: "-lc"
@@ -261,9 +317,12 @@
 // CHECK-RV32-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}include"
 // CHECK-RV32-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32-SAME: "--sysroot={{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf"
+// CHECK-RV32-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-SAME: "-Lsome{{[/\\]+}}directory{{[/\\]+}}user{{[/\\]+}}asked{{[/\\]+}}for"
 // CHECK-RV32-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}lib"
+// CHECK-RV32-SAME: "{{.*}}.o"
 // CHECK-RV32-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-SAME: "-lc"
 // CHECK-RV32-SAME: "-X" "-o" "a.out"
@@ -272,8 +331,11 @@
 // RUN:     --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf \
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-DEFAULTCXX %s
 // CHECK-RV32-DEFAULTCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV32-DEFAULTCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-DEFAULTCXX: ld{{(.exe)?}}"
+// CHECK-RV32-DEFAULTCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-DEFAULTCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-DEFAULTCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-DEFAULTCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-DEFAULTCXX-SAME: "-lc++" "-lm"
 // CHECK-RV32-DEFAULTCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-DEFAULTCXX-SAME: "-lc"
@@ -286,8 +348,11 @@
 // CHECK-RV32-LIBCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV32-LIBCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}{{[^v].*}}"
 // CHECK-RV32-LIBCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RV32-LIBCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-LIBCXX: ld{{(.exe)?}}"
+// CHECK-RV32-LIBCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-LIBCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-LIBCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-LIBCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-LIBCXX-SAME: "-lc++" "-lm"
 // CHECK-RV32-LIBCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-RV32-LIBCXX-SAME: "-X" "-o" "a.out"
@@ -299,11 +364,15 @@
 // CHECK-RV32-LIBSTDCXX: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-RV32-LIBSTDCXX-NOT: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32-LIBSTDCXX-SAME: "-internal-isystem" "{{[^"]+}}{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}8.0.1"
-// CHECK-RV32-LIBSTDCXX: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32-LIBSTDCXX: ld{{(.exe)?}}"
+// CHECK-RV32-LIBSTDCXX-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32-LIBSTDCXX-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32-LIBSTDCXX-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf{{[/\\]+}}lib"
+// CHECK-RV32-LIBSTDCXX-SAME: "{{.*}}.o"
 // CHECK-RV32-LIBSTDCXX-SAME: "-lstdc++" "-lm"
 // CHECK-RV32-LIBSTDCXX-SAME: "{{[^"]*}}libclang_rt.builtins.a"
-// CHECK-RV32-LIBSTDCXX-SAME: "-lc" "-X" "-o" "a.out"
+// CHECK-RV32-LIBSTDCXX-SAME: "-lc"
+// CHECK-RV32-LIBSTDCXX-SAME: "-X" "-o" "a.out"
 
 // RUN: %clang %s -### 2>&1 --target=riscv64-unknown-elf \
 // RUN:     -nostdlibinc -nobuiltininc \
@@ -320,7 +389,9 @@
 // RUN:     -nodefaultlibs \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-NDL %s
 // CHECK-RV64-NDL: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
-// CHECK-RV64-NDL: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64-NDL: ld{{(.exe)?}}"
+// CHECK-RV64-NDL-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64-NDL-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64-NDL-SAME: "-L{{[^"]*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv64-unknown-elf \
@@ -339,7 +410,9 @@
 // CHECK-RV64FD-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV64FD-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}include"
 // CHECK-RV64FD-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV64FD-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV64FD-NEXT: ld{{(.exe)?}}"
+// CHECK-RV64FD-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv64_tree{{[/\\]+}}riscv64-unknown-elf"
+// CHECK-RV64FD-SAME: "-Bstatic" "-m" "elf64lriscv" "-X"
 // CHECK-RV64FD-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv64imafdc{{[/\\]+}}lp64d{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -358,7 +431,9 @@
 // CHECK-RV32I-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32I-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32I-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32I-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32I-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32I-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32I-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32I-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32i{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -377,7 +452,9 @@
 // CHECK-RV32IM-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IM-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32IM-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IM-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IM-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IM-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IM-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IM-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32im{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf \
@@ -391,7 +468,9 @@
 // CHECK-RV32IAC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IAC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}include"
 // CHECK-RV32IAC-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IAC-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IAC-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IAC-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IAC-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IAC-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32iac{{[/\\]+}}ilp32{{[/\\]+}}lib"
 
 // RUN: %clang %s -### 2>&1 --target=riscv32-unknown-elf -march=rv32imafc -mabi=ilp32f \
@@ -412,7 +491,9 @@
 // CHECK-RV32IMAFC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-RV32IMAFC-SAME: "-internal-isystem" "[[SYSROOT]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}include"
 // CHECK-RV32IMAFC-SAME: "-x" "c++" "{{.*}}baremetal.cpp"
-// CHECK-RV32IMAFC-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-RV32IMAFC-NEXT: ld{{(.exe)?}}"
+// CHECK-RV32IMAFC-SAME: "--sysroot={{.*}}{{[/\\]+}}Inputs{{[/\\]+}}basic_riscv32_tree{{[/\\]+}}riscv32-unknown-elf"
+// CHECK-RV32IMAFC-SAME: "-Bstatic" "-m" "elf32lriscv" "-X"
 // CHECK-RV32IMAFC-SAME: "-L[[SYSROOT:[^"]+]]{{[/\\]+}}rv32imafc{{[/\\]+}}ilp32f{{[/\\]+}}lib"
 
 // RUN: %clang -no-canonical-prefixes %s -### --target=powerpc-unknown-eabi 2>&1 \
@@ -423,8 +504,9 @@
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPCEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPCEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPCEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPCEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPCEABI-SAME:"{{.*}}.o"
 // CHECK-PPCEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPCEABI-SAME: "-lc"
 // CHECK-PPCEABI-SAME: "-o" "a.out"
@@ -437,8 +519,9 @@
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPC64EABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPC64EABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPC64EABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPC64EABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPC64EABI-SAME:"{{.*}}.o"
 // CHECK-PPC64EABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPC64EABI-SAME: "-lc"
 // CHECK-PPC64EABI-SAME: "-o" "a.out"
@@ -451,8 +534,9 @@
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPCLEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPCLEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPCLEEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPCLEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPCLEEABI-SAME:"{{.*}}.o"
 // CHECK-PPCLEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPCLEEABI-SAME: "-lc"
 // CHECK-PPCLEEABI-SAME: "-o" "a.out"
@@ -465,8 +549,9 @@
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-PPC64LEEABI-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-// CHECK-PPC64LEEABI-NEXT: ld{{(.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-PPC64LEEABI-NEXT: ld{{(.exe)?}}" "-Bstatic"
 // CHECK-PPC64LEEABI-SAME: "-L[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}lib"
+// CHECK-PPC64LEEABI-SAME:"{{.*}}.o"
 // CHECK-PPC64LEEABI-SAME: "{{[^"]*}}libclang_rt.builtins.a"
 // CHECK-PPC64LEEABI-SAME: "-lc"
 // CHECK-PPC64LEEABI-SAME: "-o" "a.out"
diff --git a/clang/test/Driver/check-no-multlib-warning.c b/clang/test/Driver/check-no-multlib-warning.c
new file mode 100644
index 000000000000..9a0d7cee450a
--- /dev/null
+++ b/clang/test/Driver/check-no-multlib-warning.c
@@ -0,0 +1,10 @@
+// UNSUPPORTED: system-windows
+
+
+// RUN: %clang --target=armv6m-none-eabi --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=aarch64-none-elf --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=riscv32-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv32_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+// RUN: %clang --target=riscv64-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv64_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+
+// CHECK: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
+// NOCHECK-NOT: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
diff --git a/clang/test/Driver/cl-include.c b/clang/test/Driver/cl-include.c
index ca9e7db1e6f0..d041698c7fd3 100644
--- a/clang/test/Driver/cl-include.c
+++ b/clang/test/Driver/cl-include.c
@@ -41,3 +41,8 @@
 // EXTERNAL_ENV: "-internal-isystem" "/dir1"
 // EXTERNAL_ENV: "-internal-isystem" "/dir2"
 // EXTERNAL_ENV: "-internal-isystem" "/dir3"
+
+// /winsdkversion suppresses %INCLUDE% and %EXTERNAL_INCLUDE%
+// RUN:env INCLUDE=/my/system/inc env EXTERNAL_INCLUDE=/my/system/inc2 %clang_cl /winsdkversion 99.99.9999.9 -### -- %s 2>&1 | FileCheck %s --check-prefix=SDKVERSION
+// SDKVERSION-NOT: "-internal-isystem" "/my/system/inc"
+// SDKVERSION-NOT: "-internal-isystem" "/my/system/inc2"
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index eb079895a0a8..57e16e8795a2 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -362,6 +362,12 @@
 // RUN: %clang_cl -c -### /std:c11 -- %s 2>&1 | FileCheck -check-prefix CHECK-C11 %s
 // CHECK-C11: -std=c11
 
+// RUN: %clang_cl -c -### /std:c17 -- %s 2>&1 | FileCheck -check-prefix CHECK-C17 %s
+// CHECK-C17: -std=c17
+
+// RUN: %clang_cl -c -### /std:clatest -- %s 2>&1 | FileCheck -check-prefix CHECK-CLATEST %s
+// CHECK-CLATEST: -std=c23
+
 // For some warning ids, we can map from MSVC warning to Clang warning.
 // RUN: %clang_cl -wd4005 -wd4100 -wd4910 -wd4996 -wd12345678 -### -- %s 2>&1 | FileCheck -check-prefix=Wno %s
 // Wno: "-cc1"
diff --git a/clang/test/Driver/dxc_fspv_extension.hlsl b/clang/test/Driver/dxc_fspv_extension.hlsl
index 0a9d321b8d95..f09f21e8a72c 100644
--- a/clang/test/Driver/dxc_fspv_extension.hlsl
+++ b/clang/test/Driver/dxc_fspv_extension.hlsl
@@ -10,10 +10,15 @@
 // RUN: %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=SPV_TEST1 -fspv-extension=SPV_TEST2 2>&1 | FileCheck %s -check-prefix=TEST2
 // TEST2: "-spirv-ext=+SPV_TEST1,+SPV_TEST2"
 
+// Merge KHR with other extensions.
+// RUN: %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=SPV_TEST1 -fspv-extension=KHR -fspv-extension=SPV_TEST2 2>&1 | FileCheck %s -check-prefix=TEST3
+// TEST3: "-spirv-ext=+SPV_TEST1,KHR,+SPV_TEST2"
+
 // Check for the error message if the extension name is not properly formed.
-// RUN: not %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=TEST1 -fspv-extension=SPV_GOOD -fspv-extension=TEST2 2>&1 | FileCheck %s -check-prefix=FAIL
-// FAIL: invalid value 'TEST1' in '-fspv_extension'
-// FAIL: invalid value 'TEST2' in '-fspv_extension'
+// RUN: not %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=KHR_BAD -fspv-extension=TEST1 -fspv-extension=SPV_GOOD -fspv-extension=TEST2 2>&1 | FileCheck %s -check-prefix=FAIL
+// FAIL: invalid value 'KHR_BAD' in '-fspv-extension'
+// FAIL: invalid value 'TEST1' in '-fspv-extension'
+// FAIL: invalid value 'TEST2' in '-fspv-extension'
 
 // If targeting DXIL, the `-spirv-ext` should not be passed to the backend.
 // RUN: %clang_dxc -Tlib_6_7 -### %s 2>&1 | FileCheck %s -check-prefix=DXIL
diff --git a/clang/test/Driver/dxc_hlsl-rootsig-ver.hlsl b/clang/test/Driver/dxc_hlsl-rootsig-ver.hlsl
new file mode 100644
index 000000000000..c6aa1e94096c
--- /dev/null
+++ b/clang/test/Driver/dxc_hlsl-rootsig-ver.hlsl
@@ -0,0 +1,16 @@
+// RUN: %clang_dxc -T cs_6_0 -fcgl %s | FileCheck %s --check-prefix=CHECK-V1_1
+
+// RUN: %clang_dxc -T cs_6_0 -fcgl -force-rootsig-ver rootsig_1_0 %s | FileCheck %s --check-prefix=CHECK-V1_0
+// RUN: %clang_dxc -T cs_6_0 -fcgl -force-rootsig-ver rootsig_1_1 %s | FileCheck %s --check-prefix=CHECK-V1_1
+
+// Test to demonstrate that we can specify the root-signature versions
+
+// CHECK: !dx.rootsignatures = !{![[#EMPTY_ENTRY:]]}
+// CHECK: ![[#EMPTY_ENTRY]] = !{ptr @EmptyEntry, ![[#EMPTY:]],
+// CHECK-V1_0: i32 1}
+// CHECK-V1_1: i32 2}
+// CHECK: ![[#EMPTY]] = !{}
+
+[shader("compute"), RootSignature("")]
+[numthreads(1,1,1)]
+void EmptyEntry() {}
diff --git a/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c b/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c
new file mode 100644
index 000000000000..b186e6e74f16
--- /dev/null
+++ b/clang/test/Driver/fallow-runtime-check-skip-hot-cutoff.c
@@ -0,0 +1,6 @@
+// RUN: %clang -### -fallow-runtime-check-skip-hot-cutoff=1.0 %s 2>&1 | FileCheck %s
+// CHECK: -fallow-runtime-check-skip-hot-cutoff=1.0
+
+int main(int argc, char** argv) {
+    return __builtin_allow_runtime_check("foo");
+}
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index 1f696aba8d08..fbe1fd72c84c 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -1284,18 +1284,26 @@
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff=undefined=xyzzy %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF8
 // CHECK-SKIP-HOT-CUTOFF8: unsupported argument 'undefined=xyzzy' to option '-fsanitize-skip-hot-cutoff='
 
-// Invalid: -fno-sanitize-top without parameters
+// Invalid: -fsanitize-skip-hot-cutoff without parameters
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF9
 // CHECK-SKIP-HOT-CUTOFF9: unknown argument: '-fsanitize-skip-hot-cutoff'
 
-// Invalid: -fno-sanitize-top=undefined without cutoff
+// Invalid: -fsanitize-skip-hot-cutoff=undefined without cutoff
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff=undefined %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF10
 // CHECK-SKIP-HOT-CUTOFF10: unsupported argument 'undefined' to option '-fsanitize-skip-hot-cutoff='
 
-// Invalid: -fno-sanitize-top=undefined= without cutoff
+// Invalid: -fsanitize-skip-hot-cutoff=undefined= without cutoff
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff=undefined= %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF11
 // CHECK-SKIP-HOT-CUTOFF11: unsupported argument 'undefined=' to option '-fsanitize-skip-hot-cutoff='
 
-// No-op: -fno-sanitize-top= without parameters is unusual but valid
+// No-op: -fsanitize-skip-hot-cutoff= without parameters is unusual but valid
 // RUN: %clang -Werror --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff= %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF12
 // CHECK-SKIP-HOT-CUTOFF12-NOT: "-fsanitize-skip-hot-cutoff"
+
+// Invalid: out of range cutoff
+// RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff=undefined=1.1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF13
+// CHECK-SKIP-HOT-CUTOFF13: unsupported argument 'undefined=1.1' to option '-fsanitize-skip-hot-cutoff='
+
+// Invalid: out of range cutoff
+// RUN: not %clang --target=x86_64-linux-gnu -fsanitize-skip-hot-cutoff=undefined=-0.1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-SKIP-HOT-CUTOFF14
+// CHECK-SKIP-HOT-CUTOFF14: unsupported argument 'undefined=-0.1' to option '-fsanitize-skip-hot-cutoff='
diff --git a/clang/test/Driver/gcc-toolchain.cpp b/clang/test/Driver/gcc-toolchain.cpp
index 6d4ad417cfec..a14e8d00af1e 100644
--- a/clang/test/Driver/gcc-toolchain.cpp
+++ b/clang/test/Driver/gcc-toolchain.cpp
@@ -6,11 +6,6 @@
 // RUN:   --gcc-toolchain=%S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc --unwindlib=libgcc -no-pie 2>&1 | \
 // RUN:   FileCheck %s
 //
-// Additionally check that the legacy spelling of the flag works.
-// RUN: %clangxx %s -### --target=x86_64-linux-gnu --sysroot= \
-// RUN:   --gcc-toolchain=%S/Inputs/ubuntu_14.04_multiarch_tree/usr -stdlib=libstdc++ --rtlib=libgcc --unwindlib=libgcc -no-pie 2>&1 | \
-// RUN:   FileCheck %s
-//
 // Test for header search toolchain detection.
 // CHECK: "-internal-isystem"
 // CHECK: "[[TOOLCHAIN:[^"]+]]/usr/lib/gcc/x86_64-linux-gnu/4.8/../../../../include/c++/4.8"
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index a07dca363856..4fb5571b838f 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -118,7 +118,7 @@
 // RUN: not %clang --target=x86_64-unknown-linux-gnu -nogpuinc -nogpulib \
 // RUN:   --offload-arch=gfx906 -fopenmp=libomp -fopenmp-targets=amdgcn %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=OMPTGT %s
-// OMPTGT: unsupported option '-fopenmp-targets=' for language mode 'HIP'
+// OMPTGT: unsupported option '--offload-targets=' for language mode 'HIP'
 
 // Check -Xoffload-linker option is passed to lld.
 
diff --git a/clang/test/Driver/ir-exception-model.c b/clang/test/Driver/ir-exception-model.c
new file mode 100644
index 000000000000..9e8f998de0d6
--- /dev/null
+++ b/clang/test/Driver/ir-exception-model.c
@@ -0,0 +1,14 @@
+// RUN: %clang -### -target wasm32-unknown-unknown -fwasm-exceptions -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=wasm -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=dwarf -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=DWARF %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=sjlj -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=SJLJ %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=wineh -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=WINEH %s
+// RUN: %clang -### -target wasm32-unknown-unknown -Xclang -exception-model=arst -c -S -o - %S/Inputs/file.ll 2>&1 | FileCheck -check-prefix=INVALID %s
+
+// Check that -fwasm-exceptions propagates -exception-model to cc1
+
+// CHECK: "-exception-model=wasm"
+// DWARF: "-exception-model=dwarf"
+// SJLJ: "-exception-model=sjlj"
+// WINEH: "-exception-model=wineh"
+// INVALID: "-exception-model=arst"
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index fb62ded7cf54..a924a934b6d4 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -311,7 +311,7 @@
 // DRIVER_EMBEDDING: -fembed-offload-object={{.*}}.out
 
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -ccc-print-bindings -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
-// RUN:     --offload-host-only -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-HOST-ONLY
+// RUN:     -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_52 --offload-host-only -nogpulib %s 2>&1 | FileCheck %s --check-prefix=CHECK-HOST-ONLY
 // CHECK-HOST-ONLY: "x86_64-unknown-linux-gnu" - "clang", inputs: ["[[INPUT:.*]]"], output: "[[OUTPUT:.*]]"
 // CHECK-HOST-ONLY: "x86_64-unknown-linux-gnu" - "Offload::Linker", inputs: ["[[OUTPUT]]"], output: "a.out"
 
diff --git a/clang/test/Driver/pgo-sample-use-profi.c b/clang/test/Driver/pgo-sample-use-profi.c
index 0370d947ce59..aef00724f868 100644
--- a/clang/test/Driver/pgo-sample-use-profi.c
+++ b/clang/test/Driver/pgo-sample-use-profi.c
@@ -1,4 +1,23 @@
-/// Test if profi flat is enabled in frontend as user-facing feature.
+/// Test if profi flag is enabled/disabled correctly based on user-specified configuration.
+/// Ensure that profi flag is disabled by default
+
+// RUN: %clang --target=x86_64 -c -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
 // RUN: %clang --target=x86_64 -c -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+// RUN: %clang --target=x86_64 -c -fno-sample-profile-use-profi -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+
+// RUN: %clang --target=AArch64 -c -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+// RUN: %clang --target=AArch64 -c -fno-sample-profile-use-profi -fsample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s
+
+// Cases where profi flag is explicitly disabled:
+// RUN: %clang --target=x86_64 -c -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=x86_64 -c -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=x86_64 -c -fsample-profile-use-profi -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+
+// RUN: %clang --target=AArch64 -c -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+// RUN: %clang --target=AArch64 -c -fsample-profile-use-profi -fno-sample-profile-use-profi -fprofile-sample-use=/dev/null -### %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NO-PROFI
+
 
 // CHECK: "-mllvm" "-sample-profile-use-profi"
+// CHECK-NO-PROFI-NOT: "-sample-profile-use-profi"
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
index 29e9682d5870..8576edff9bcb 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-fujitsu-monaka.c
@@ -70,7 +70,7 @@
 // CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
-// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SHA3 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
 // CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c b/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c
new file mode 100644
index 000000000000..589f7e3e5ee4
--- /dev/null
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-gb10.c
@@ -0,0 +1,69 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang --target=aarch64 --print-enabled-extensions -mcpu=gb10 | FileCheck --strict-whitespace --implicit-check-not=FEAT_ %s
+
+// CHECK: Extensions enabled for the given AArch64 target
+// CHECK-EMPTY:
+// CHECK-NEXT:     Architecture Feature(s)                                Description
+// CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
+// CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
+// CHECK-NEXT:     FEAT_AMUv1p1                                           Enable Armv8.6-A Activity Monitors Virtualization support
+// CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
+// CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
+// CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
+// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
+// CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
+// CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
+// CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
+// CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
+// CHECK-NEXT:     FEAT_DPB2                                              Enable Armv8.5-A Cache Clean to Point of Deep Persistence
+// CHECK-NEXT:     FEAT_DotProd                                           Enable dot product support
+// CHECK-NEXT:     FEAT_ECV                                               Enable enhanced counter virtualization extension
+// CHECK-NEXT:     FEAT_ETE                                               Enable Embedded Trace Extension
+// CHECK-NEXT:     FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
+// CHECK-NEXT:     FEAT_FGT                                               Enable fine grained virtualization traps extension
+// CHECK-NEXT:     FEAT_FHM                                               Enable FP16 FML instructions
+// CHECK-NEXT:     FEAT_FP                                                Enable Armv8.0-A Floating Point Extensions
+// CHECK-NEXT:     FEAT_FP16                                              Enable half-precision floating-point data processing
+// CHECK-NEXT:     FEAT_FPAC                                              Enable Armv8.3-A Pointer Authentication Faulting enhancement
+// CHECK-NEXT:     FEAT_FRINTTS                                           Enable FRInt[32|64][Z|X] instructions that round a floating-point number to an integer (in FP format) forcing it to fit into a 32- or 64-bit int
+// CHECK-NEXT:     FEAT_FlagM                                             Enable Armv8.4-A Flag Manipulation instructions
+// CHECK-NEXT:     FEAT_FlagM2                                            Enable alternative NZCV format for floating point comparisons
+// CHECK-NEXT:     FEAT_HCX                                               Enable Armv8.7-A HCRX_EL2 system register
+// CHECK-NEXT:     FEAT_I8MM                                              Enable Matrix Multiply Int8 Extension
+// CHECK-NEXT:     FEAT_JSCVT                                             Enable Armv8.3-A JavaScript FP conversion instructions
+// CHECK-NEXT:     FEAT_LOR                                               Enable Armv8.1-A Limited Ordering Regions extension
+// CHECK-NEXT:     FEAT_LRCPC                                             Enable support for RCPC extension
+// CHECK-NEXT:     FEAT_LRCPC2                                            Enable Armv8.4-A RCPC instructions with Immediate Offsets
+// CHECK-NEXT:     FEAT_LSE                                               Enable Armv8.1-A Large System Extension (LSE) atomic instructions
+// CHECK-NEXT:     FEAT_LSE2                                              Enable Armv8.4-A Large System Extension 2 (LSE2) atomicity rules
+// CHECK-NEXT:     FEAT_MPAM                                              Enable Armv8.4-A Memory system Partitioning and Monitoring extension
+// CHECK-NEXT:     FEAT_MTE, FEAT_MTE2                                    Enable Memory Tagging Extension
+// CHECK-NEXT:     FEAT_NV, FEAT_NV2                                      Enable Armv8.4-A Nested Virtualization Enchancement
+// CHECK-NEXT:     FEAT_PAN                                               Enable Armv8.1-A Privileged Access-Never extension
+// CHECK-NEXT:     FEAT_PAN2                                              Enable Armv8.2-A PAN s1e1R and s1e1W Variants
+// CHECK-NEXT:     FEAT_PAuth                                             Enable Armv8.3-A Pointer Authentication extension
+// CHECK-NEXT:     FEAT_PMUv3                                             Enable Armv8.0-A PMUv3 Performance Monitors extension
+// CHECK-NEXT:     FEAT_RAS, FEAT_RASv1p1                                 Enable Armv8.0-A Reliability, Availability and Serviceability Extensions
+// CHECK-NEXT:     FEAT_RDM                                               Enable Armv8.1-A Rounding Double Multiply Add/Subtract instructions
+// CHECK-NEXT:     FEAT_SB                                                Enable Armv8.5-A Speculation Barrier
+// CHECK-NEXT:     FEAT_SEL2                                              Enable Armv8.4-A Secure Exception Level 2 extension
+// CHECK-NEXT:     FEAT_SHA1, FEAT_SHA256                                 Enable SHA1 and SHA256 support
+// CHECK-NEXT:     FEAT_SHA3, FEAT_SHA512                                 Enable SHA512 and SHA3 support
+// CHECK-NEXT:     FEAT_SM4, FEAT_SM3                                     Enable SM3 and SM4 support
+// CHECK-NEXT:     FEAT_SPE                                               Enable Statistical Profiling extension
+// CHECK-NEXT:     FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
+// CHECK-NEXT:     FEAT_SPEv1p2                                           Enable extra register in the Statistical Profiling Extension
+// CHECK-NEXT:     FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
+// CHECK-NEXT:     FEAT_SVE                                               Enable Scalable Vector Extension (SVE) instructions
+// CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
+// CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
+// CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
+// CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
+// CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
+// CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
+// CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
+// CHECK-NEXT:     FEAT_UAO                                               Enable Armv8.2-A UAO PState
+// CHECK-NEXT:     FEAT_VHE                                               Enable Armv8.1-A Virtual Host extension
+// CHECK-NEXT:     FEAT_WFxT                                              Enable Armv8.7-A WFET and WFIT instruction
+// CHECK-NEXT:     FEAT_XS                                                Enable Armv8.7-A limited-TLB-maintenance instruction
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-grace.c b/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
index 739d86f1fae0..bb24dfbbc070 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-grace.c
@@ -54,7 +54,7 @@
 // CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
-// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SHA3 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
 // CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c b/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
index 64106e583d5f..0cfb9e1fb981 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-olympus.c
@@ -70,7 +70,7 @@
 // CHECK-NEXT:     FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
 // CHECK-NEXT:     FEAT_SVE_AES, FEAT_SVE_PMULL128                        Enable SVE AES and quadword SVE polynomial multiply instructions
 // CHECK-NEXT:     FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
-// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SHA3 SVE2 instructions
+// CHECK-NEXT:     FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
 // CHECK-NEXT:     FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRBE                                              Enable Trace Buffer Extension
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
index cfb4d0ed58d1..5d616c48247d 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-a25.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-a25 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-a25 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
index 3c3c554dffc5..b5885f8e5200 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-a45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-a45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-a45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
index 70100a0a8df1..a8b909cf9fdd 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax25.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax25 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax25 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
index d2b1a32e321e..45571a171363 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
index bab35b117f57..1917d5332fee 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-ax45mpv.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-ax45mpv --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-ax45mpv --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
index 1a2c30bfc7a2..18cd4b7bbcdc 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-n45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv32 -mcpu=andes-n45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv32 -mcpu=andes-n45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c b/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
index 50c38da3bd03..ca6a13b3d983 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-andes-nx45.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=andes-nx45 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=andes-nx45 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c b/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
index f76e5daec672..e50768a62e6e 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-sifive-p870.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=sifive-p870 --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=sifive-p870 --print-enabled-extensions | FileCheck %s
 
 // CHECK: Extensions enabled for the given RISC-V target
 // CHECK-EMPTY:
diff --git a/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c b/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
index 3dc4d6a6b4fa..aa038a18693e 100644
--- a/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
+++ b/clang/test/Driver/print-enabled-extensions/riscv-sifive-x390.c
@@ -1,5 +1,5 @@
-// RUN: %clang --target=riscv64 -mcpu=sifive-x390 -menable-experimental-extensions --print-enabled-extensions | FileCheck %s
 // REQUIRES: riscv-registered-target
+// RUN: %clang --target=riscv64 -mcpu=sifive-x390 -menable-experimental-extensions --print-enabled-extensions | FileCheck %s
 
 // CHECK:     Name                 Version   Description
 // CHECK-NEXT:     i                    2.1       'I' (Base Integer Instruction Set)
diff --git a/clang/test/Driver/print-libgcc-file-name-clangrt.c b/clang/test/Driver/print-libgcc-file-name-clangrt.c
index a902eedc8520..1cb28d43748e 100644
--- a/clang/test/Driver/print-libgcc-file-name-clangrt.c
+++ b/clang/test/Driver/print-libgcc-file-name-clangrt.c
@@ -63,3 +63,12 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARM-BAREMETAL-PER-TARGET %s
 // CHECK-CLANGRT-ARM-BAREMETAL-PER-TARGET: libclang_rt.builtins.a
+
+// RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
+// RUN:     --target=riscv32-unknown-elf \
+// RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     --gcc-toolchain=%S/Inputs/basic_riscv32_tree 2>&1 \
+// RUN:    | FileCheck --check-prefix=CHECK-CLANGRT-RISCV-BAREMETAL %s
+// CHECK-CLANGRT-RISCV-BAREMETAL-NOT: baremetal{{/|\\}}libclang_rt.builtins-riscv32.a
+// CHECK-CLANGRT-RISCV-BAREMETAL: libclang_rt.builtins.a
diff --git a/clang/test/Driver/print-supported-extensions-aarch64.c b/clang/test/Driver/print-supported-extensions-aarch64.c
index 539c1937a971..af4db9944c2d 100644
--- a/clang/test/Driver/print-supported-extensions-aarch64.c
+++ b/clang/test/Driver/print-supported-extensions-aarch64.c
@@ -92,10 +92,11 @@
 // CHECK-NEXT:     sve-bfscale         FEAT_SVE_BFSCALE                                       Enable Armv9.6-A SVE BFloat16 scaling instructions
 // CHECK-NEXT:     sve-bitperm         FEAT_SVE_BitPerm                                       Enable bit permutation SVE2 instructions
 // CHECK-NEXT:     sve-f16f32mm        FEAT_SVE_F16F32MM                                      Enable Armv9.6-A FP16 to FP32 Matrix Multiply
+// CHECK-NEXT:     sve-sha3            FEAT_SVE_SHA3                                          Enable SVE SHA3 instructions
 // CHECK-NEXT:     sve2                FEAT_SVE2                                              Enable Scalable Vector Extension 2 (SVE2) instructions
 // CHECK-NEXT:     sve2-aes                                                                   Shorthand for +sve2+sve-aes
 // CHECK-NEXT:     sve2-bitperm                                                               Shorthand for +sve2+sve-bitperm
-// CHECK-NEXT:     sve2-sha3           FEAT_SVE_SHA3                                          Enable SHA3 SVE2 instructions
+// CHECK-NEXT:     sve2-sha3                                                                  Shorthand for +sve2+sve-sha3
 // CHECK-NEXT:     sve2-sm4            FEAT_SVE_SM4                                           Enable SM4 SVE2 instructions
 // CHECK-NEXT:     sve2p1              FEAT_SVE2p1                                            Enable Scalable Vector Extension 2.1 instructions
 // CHECK-NEXT:     sve2p2              FEAT_SVE2p2                                            Enable Armv9.6-A Scalable Vector Extension 2.2 instructions
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 5008c2b7f789..b87f0cb568e0 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -162,6 +162,7 @@
 // CHECK-NEXT:     xandesvbfhcvt        5.0       'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension)
 // CHECK-NEXT:     xandesvdot           5.0       'XAndesVDot' (Andes Vector Dot Product Extension)
 // CHECK-NEXT:     xandesvpackfph       5.0       'XAndesVPackFPH' (Andes Vector Packed FP16 Extension)
+// CHECK-NEXT:     xandesvsintload      5.0       'XAndesVSIntLoad' (Andes Vector INT4 Load Extension)
 // CHECK-NEXT:     xcvalu               1.0       'XCValu' (CORE-V ALU Operations)
 // CHECK-NEXT:     xcvbi                1.0       'XCVbi' (CORE-V Immediate Branching)
 // CHECK-NEXT:     xcvbitmanip          1.0       'XCVbitmanip' (CORE-V Bit Manipulation)
@@ -169,6 +170,7 @@
 // CHECK-NEXT:     xcvmac               1.0       'XCVmac' (CORE-V Multiply-Accumulate)
 // CHECK-NEXT:     xcvmem               1.0       'XCVmem' (CORE-V Post-incrementing Load & Store)
 // CHECK-NEXT:     xcvsimd              1.0       'XCVsimd' (CORE-V SIMD ALU)
+// CHECK-NEXT:     xmipscbop            1.0       'XMIPSCBOP' (MIPS Software Prefetch)
 // CHECK-NEXT:     xmipscmov            1.0       'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov))
 // CHECK-NEXT:     xmipslsp             1.0       'XMIPSLSP' (MIPS optimization for hardware load-store bonding)
 // CHECK-NEXT:     xsfcease             1.0       'XSfcease' (SiFive sf.cease Instruction)
@@ -184,7 +186,7 @@
 // CHECK-NEXT:     xsfmmbase            0.6       'XSfmmbase' (All non arithmetic instructions for all TEWs and sf.vtzero)
 // CHECK-NEXT:     xsfvcp               1.0       'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions)
 // CHECK-NEXT:     xsfvfnrclipxfqf      1.0       'XSfvfnrclipxfqf' (SiFive FP32-to-int8 Ranged Clip Instructions)
-// CHECK-NEXT:     xsfvfwmaccqqq        1.0       'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))
+// CHECK-NEXT:     xsfvfwmaccqqq        1.0       'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4))
 // CHECK-NEXT:     xsfvqmaccdod         1.0       'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2))
 // CHECK-NEXT:     xsfvqmaccqoq         1.0       'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
 // CHECK-NEXT:     xsifivecdiscarddlone 1.0       'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction)
diff --git a/clang/test/Driver/ps5-linker.c b/clang/test/Driver/ps5-linker.c
index 777826aade7f..753085d2e1b0 100644
--- a/clang/test/Driver/ps5-linker.c
+++ b/clang/test/Driver/ps5-linker.c
@@ -146,9 +146,9 @@
 // RUN: %clang --target=x86_64-sie-ps5 %s -nostdlib++ -static -### 2>&1 | FileCheck --check-prefixes=CHECK-LD,CHECK-STATIC-CRT,CHECK-NO-LIBCPP,CHECK-STATIC-LIBC,CHECK-STATIC-CORE-LIBS %s
 
 // CHECK-LD: {{ld(\.exe)?}}"
-// CHECK-MAIN-CRT-SAME: "crt1.o" "crti.o" "crtbegin.o"
-// CHECK-SHARED-CRT-SAME: "crti.o" "crtbeginS.o"
-// CHECK-STATIC-CRT-SAME: "crt1.o" "crti.o" "crtbeginT.o"
+// CHECK-MAIN-CRT-SAME: "-l:crt1.o" "-l:crti.o" "-l:crtbegin.o"
+// CHECK-SHARED-CRT-SAME: "-l:crti.o" "-l:crtbeginS.o"
+// CHECK-STATIC-CRT-SAME: "-l:crt1.o" "-l:crti.o" "-l:crtbeginT.o"
 
 // CHECK-NO-LIBC-NOT: "-lc{{(_stub_weak)?}}"
 // CHECK-NO-LIBCPP-NOT: "-l{{c_stub_weak|stdc\+\+}}"
@@ -161,11 +161,11 @@
 
 // CHECK-PTHREAD-SAME: "-lpthread"
 
-// CHECK-MAIN-CRT-SAME: "crtend.o" "crtn.o"
-// CHECK-SHARED-CRT-SAME: "crtendS.o" "crtn.o"
-// CHECK-STATIC-CRT-SAME: "crtend.o" "crtn.o"
+// CHECK-MAIN-CRT-SAME: "-l:crtend.o" "-l:crtn.o"
+// CHECK-SHARED-CRT-SAME: "-l:crtendS.o" "-l:crtn.o"
+// CHECK-STATIC-CRT-SAME: "-l:crtend.o" "-l:crtn.o"
 
-// CHECK-NO-CRT-NOT: "crt{{[^"]*}}.o"
+// CHECK-NO-CRT-NOT: crt{{[^"]*}}.o"
 // CHECK-NO-LIBS-NOT: "-l{{[^"]*}}"
 
 // Test the driver's control over the -fcrash-diagnostics-dir behavior with linker flags.
@@ -186,9 +186,9 @@
 
 // Test implicit library search paths are supplied to the linker, after any
 // search paths specified by the user. <sdk-root>/target/lib is implicitly
-// added if it exists. CRT objects are found there. "." is always implicitly
-// added to library search paths. This is long-standing behavior, unique to
-// PlayStation toolchains.
+// added if it exists. CRT objects are found there if not on user search paths.
+// "." is always implicitly added to library search paths. These are
+// long-standing and entrenched behaviors, unique to PlayStation toolchains.
 
 // RUN: rm -rf %t.dir && mkdir %t.dir
 // RUN: env SCE_PROSPERO_SDK_DIR=%t.dir %clang --target=x64_64-sie-ps5 %s -### -Luser 2>&1 | FileCheck --check-prefixes=CHECK-NO-TARGETLIB %s
@@ -200,7 +200,6 @@
 // CHECK-NO-TARGETLIB-SAME: "-L."
 
 // RUN: mkdir -p %t.dir/myroot/target/lib
-// RUN: touch %t.dir/myroot/target/lib/crti.o
 // RUN: env SCE_PROSPERO_SDK_DIR=%t.dir/myroot %clang --target=x64_64-sie-ps5 %s -### -Luser 2>&1 | FileCheck --check-prefixes=CHECK-TARGETLIB %s
 // RUN: %clang --target=x64_64-sie-ps5 %s -### -Luser --sysroot=%t.dir/myroot 2>&1 | FileCheck --check-prefixes=CHECK-TARGETLIB %s
 
@@ -208,4 +207,3 @@
 // CHECK-TARGETLIB-SAME: "-Luser"
 // CHECK-TARGETLIB-SAME: "-L{{.*}}myroot{{/|\\\\}}target{{/|\\\\}}lib"
 // CHECK-TARGETLIB-SAME: "-L."
-// CHECK-TARGETLIB-SAME: "{{.*}}myroot{{/|\\\\}}target{{/|\\\\}}lib{{/|\\\\}}crti.o"
diff --git a/clang/test/Driver/riscv-args.c b/clang/test/Driver/riscv-args.c
deleted file mode 100644
index cab08e5b0f81..000000000000
--- a/clang/test/Driver/riscv-args.c
+++ /dev/null
@@ -1,6 +0,0 @@
-// Check the arguments are correctly passed
-
-// Make sure -T is the last with gcc-toolchain option
-// RUN: %clang -### --target=riscv32 --gcc-toolchain= -Xlinker --defsym=FOO=10 -T a.lds -u foo %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHECK-LD %s
-// CHECK-LD: {{.*}} "--defsym=FOO=10" {{.*}} "-u" "foo" {{.*}} "-T" "a.lds"
diff --git a/clang/test/Driver/riscv-cpus.c b/clang/test/Driver/riscv-cpus.c
index 3736e76ed06d..d32d1c1a8183 100644
--- a/clang/test/Driver/riscv-cpus.c
+++ b/clang/test/Driver/riscv-cpus.c
@@ -186,6 +186,9 @@
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zalrsc"
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zba"
 // MCPU-MIPS-P8700-SAME: "-target-feature" "+zbb"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipscbop"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipscmov"
+// MCPU-MIPS-P8700-SAME: "-target-feature" "+xmipslsp"
 
 // RUN: %clang --target=riscv32 -### -c %s 2>&1 -mtune=syntacore-scr1-base | FileCheck -check-prefix=MTUNE-SYNTACORE-SCR1-BASE %s
 // MTUNE-SYNTACORE-SCR1-BASE: "-tune-cpu" "syntacore-scr1-base"
diff --git a/clang/test/Driver/riscv32-toolchain-extra.c b/clang/test/Driver/riscv32-toolchain-extra.c
index cbb3c23ebb34..420f7b520360 100644
--- a/clang/test/Driver/riscv32-toolchain-extra.c
+++ b/clang/test/Driver/riscv32-toolchain-extra.c
@@ -18,12 +18,12 @@
 // RUN: ln -s %S/Inputs/basic_riscv32_nogcc_tree/riscv32-unknown-elf %t/riscv32-nogcc/riscv32-unknown-elf
 // RUN: %t/riscv32-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%t/riscv32-nogcc/invalid \
-// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // RUN: %t/riscv32-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --sysroot=%t/riscv32-nogcc/bin/../riscv32-unknown-elf \
-// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv32-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // C-RV32-BAREMETAL-ILP32-NOGCC: "-internal-isystem" "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/include"
@@ -31,6 +31,5 @@
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib/crt0.o"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV32-BAREMETAL-ILP32-NOGCC: "--start-group" "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv32-toolchain.c b/clang/test/Driver/riscv32-toolchain.c
index 50db13f8294c..8cf20aa592a3 100644
--- a/clang/test/Driver/riscv32-toolchain.c
+++ b/clang/test/Driver/riscv32-toolchain.c
@@ -28,7 +28,7 @@
 // C-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // C-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // C-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-BAREMETAL-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clang -### %s -fuse-ld= \
@@ -42,7 +42,7 @@
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/../../../../riscv32-unknown-elf/lib"
-// C-RV32-BAREMETAL-NOSYSROOT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-BAREMETAL-NOSYSROOT-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clangxx -### %s -fuse-ld= \
@@ -58,7 +58,7 @@
 // CXX-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // CXX-RV32-BAREMETAL-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/riscv32-unknown-elf/lib"
-// CXX-RV32-BAREMETAL-ILP32: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV32-BAREMETAL-ILP32: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: %clangxx -### %s -fuse-ld= \
@@ -73,7 +73,7 @@
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-L{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/../../../../riscv32-unknown-elf/lib"
-// CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie \
@@ -117,7 +117,7 @@
 // C-RV32I-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtbegin.o"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32I-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32I-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32I-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -132,7 +132,7 @@
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32im/ilp32/crtbegin.o"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IM-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IM-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IM-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32im/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -147,7 +147,7 @@
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32iac/ilp32/crtbegin.o"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32iac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -162,7 +162,7 @@
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtbegin.o"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IMAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IMAC-BAREMETAL-MULTI-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IMAC-BAREMETAL-MULTI-ILP32: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imac/ilp32/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -177,7 +177,7 @@
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imafc/ilp32f/crtbegin.o"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32IMAFC-BAREMETAL-MULTI-ILP32F: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv32imafc/ilp32f/crtend.o"
 
 // Check that --rtlib can be used to override the used runtime library
@@ -187,7 +187,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV32-RTLIB-LIBGCC-ILP32 %s
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crt0.o"
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crtbegin.o"
-// C-RV32-RTLIB-LIBGCC-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV32-RTLIB-LIBGCC-ILP32: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV32-RTLIB-LIBGCC-ILP32: "{{.*}}crtend.o"
 
 // RUN: %clang -### %s \
@@ -196,7 +196,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV32-RTLIB-COMPILERRT-ILP32 %s
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}crt0.o"
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtbegin.o"
-// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv32 \
@@ -215,7 +215,7 @@
 
 // RUN: %clang --target=riscv32 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
@@ -223,7 +223,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-NORELAX %s
 // CHECK-RV32-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
diff --git a/clang/test/Driver/riscv64-toolchain-extra.c b/clang/test/Driver/riscv64-toolchain-extra.c
index 99460a3030d3..87bcdeb651a1 100644
--- a/clang/test/Driver/riscv64-toolchain-extra.c
+++ b/clang/test/Driver/riscv64-toolchain-extra.c
@@ -18,12 +18,12 @@
 // RUN: ln -s %S/Inputs/basic_riscv64_nogcc_tree/riscv64-unknown-elf %t/riscv64-nogcc/riscv64-unknown-elf
 // RUN: %t/riscv64-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%t/riscv64-nogcc/invalid \
-// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // RUN: %t/riscv64-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --sysroot=%t/riscv64-nogcc/bin/../riscv64-unknown-elf \
-// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
+// RUN:    --target=riscv64-unknown-elf --rtlib=platform -fuse-ld=ld 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // C-RV64-BAREMETAL-LP64-NOGCC: "-internal-isystem" "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/include"
@@ -31,6 +31,5 @@
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib/crt0.o"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-LP64-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV64-BAREMETAL-LP64-NOGCC: "--start-group" "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv64-toolchain.c b/clang/test/Driver/riscv64-toolchain.c
index 06d5f50f3f40..1550f46af8c9 100644
--- a/clang/test/Driver/riscv64-toolchain.c
+++ b/clang/test/Driver/riscv64-toolchain.c
@@ -28,7 +28,7 @@
 // C-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // C-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // C-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-BAREMETAL-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld= \
@@ -42,7 +42,7 @@
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/../../../../riscv64-unknown-elf/lib"
-// C-RV64-BAREMETAL-NOSYSROOT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-BAREMETAL-NOSYSROOT-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clangxx -### %s -fuse-ld= \
@@ -58,7 +58,7 @@
 // CXX-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // CXX-RV64-BAREMETAL-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/riscv64-unknown-elf/lib"
-// CXX-RV64-BAREMETAL-LP64: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV64-BAREMETAL-LP64: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clangxx -### %s -fuse-ld= \
@@ -73,7 +73,7 @@
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtbegin.o"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-L{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/../../../../riscv64-unknown-elf/lib"
-// CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-lstdc++" "-lm" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-lstdc++" "-lm" "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld= -no-pie \
@@ -118,7 +118,7 @@
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imac/lp64/crtbegin.o"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV64IMAC-BAREMETAL-MULTI-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64IMAC-BAREMETAL-MULTI-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64IMAC-BAREMETAL-MULTI-LP64: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imac/lp64/crtend.o"
 
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld \
@@ -133,7 +133,7 @@
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imafdc/lp64d/crtbegin.o"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "-L{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/../../../../riscv64-unknown-elf/lib"
-// C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64IMAFDC-BAREMETAL-MULTI-ILP64D: "{{.*}}/Inputs/multilib_riscv_elf_sdk/lib/gcc/riscv64-unknown-elf/8.2.0/rv64imafdc/lp64d/crtend.o"
 
 // Check that --rtlib can be used to override the used runtime library
@@ -143,7 +143,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV64-RTLIB-LIBGCC-LP64 %s
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crt0.o"
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crtbegin.o"
-// C-RV64-RTLIB-LIBGCC-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
+// C-RV64-RTLIB-LIBGCC-LP64: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
 // C-RV64-RTLIB-LIBGCC-LP64: "{{.*}}crtend.o"
 
 // RUN: %clang -### %s \
@@ -152,7 +152,7 @@
 // RUN:   | FileCheck -check-prefix=C-RV64-RTLIB-COMPILERRT-LP64 %s
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}crt0.o"
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtbegin.o"
-// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "{{.*}}libclang_rt.builtins.a" "-lc" "-lgloss" "--end-group"
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv64 \
@@ -171,7 +171,7 @@
 
 // RUN: %clang --target=riscv64 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
@@ -179,7 +179,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-NORELAX %s
 // CHECK-RV64-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (RISCVToolchain.cpp).
+// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 6b245c6300b3..f804202a91b7 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -1033,7 +1033,7 @@
 // RUN:     --target=riscv32-unknown-elf -fuse-ld=ld \
 // RUN:   | %{filecheck} --check-prefix=CHECK-SHADOWCALLSTACK-ELF-RISCV32
 // CHECK-SHADOWCALLSTACK-ELF-RISCV32-NOT: error:
-// CHECK-SHADOWCALLSTACK-ELF-RISCV32: "{{(.*[^-.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
+// CHECK-SHADOWCALLSTACK-ELF-RISCV32: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 
 // RUN: %clang -fsanitize=shadow-call-stack -### %s 2>&1 \
 // RUN:     --target=riscv64-unknown-linux -fuse-ld=ld \
diff --git a/clang/test/ExtractAPI/constructor_destructor.cpp b/clang/test/ExtractAPI/constructor_destructor.cpp
index 27112c95ac45..2f2150a6d0da 100644
--- a/clang/test/ExtractAPI/constructor_destructor.cpp
+++ b/clang/test/ExtractAPI/constructor_destructor.cpp
@@ -213,7 +213,7 @@ class Foo {
         "subHeading": [
           {
             "kind": "identifier",
-            "spelling": "Foo"
+            "spelling": "~Foo"
           }
         ],
         "title": "~Foo"
diff --git a/clang/test/ExtractAPI/global_record.c b/clang/test/ExtractAPI/global_record.c
index a08d51d21f95..287fa24c4c64 100644
--- a/clang/test/ExtractAPI/global_record.c
+++ b/clang/test/ExtractAPI/global_record.c
@@ -185,7 +185,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -341,7 +341,7 @@ char unavailable __attribute__((unavailable));
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/global_record_multifile.c b/clang/test/ExtractAPI/global_record_multifile.c
index ffdfbcb7eb80..b98cd27b1601 100644
--- a/clang/test/ExtractAPI/global_record_multifile.c
+++ b/clang/test/ExtractAPI/global_record_multifile.c
@@ -187,7 +187,7 @@ char unavailable __attribute__((unavailable));
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -343,7 +343,7 @@ char unavailable __attribute__((unavailable));
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/macro_undefined.c b/clang/test/ExtractAPI/macro_undefined.c
index ec60f95d3d6c..7bb50af380c2 100644
--- a/clang/test/ExtractAPI/macro_undefined.c
+++ b/clang/test/ExtractAPI/macro_undefined.c
@@ -148,7 +148,7 @@ FUNC_GEN(bar, const int *, unsigned);
         },
         {
           "kind": "text",
-          "spelling": " * "
+          "spelling": " *"
         },
         {
           "kind": "internalParam",
@@ -195,7 +195,7 @@ FUNC_GEN(bar, const int *, unsigned);
               },
               {
                 "kind": "text",
-                "spelling": " * "
+                "spelling": " *"
               },
               {
                 "kind": "internalParam",
diff --git a/clang/test/ExtractAPI/pointers.c b/clang/test/ExtractAPI/pointers.c
new file mode 100644
index 000000000000..d7baf541ec03
--- /dev/null
+++ b/clang/test/ExtractAPI/pointers.c
@@ -0,0 +1,388 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \
+// RUN: %t/reference.output.json.in >> %t/reference.output.json
+// RUN: %clang -extract-api --pretty-sgf --product-name=Pointers -target arm64-apple-macosx \
+// RUN: %t/input.h -o %t/output.json | FileCheck -allow-empty %s
+
+// Generator version is not consistent across test runs, normalize it.
+// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \
+// RUN: %t/output.json >> %t/output-normalized.json
+// RUN: diff %t/reference.output.json %t/output-normalized.json
+
+// CHECK-NOT: error:
+// CHECK-NOT: warning:
+
+//--- input.h
+void foo(int *a);
+void bar(int **a);
+void *baz();
+void **qux();
+
+//--- reference.output.json.in
+{
+  "metadata": {
+    "formatVersion": {
+      "major": 0,
+      "minor": 5,
+      "patch": 3
+    },
+    "generator": "?"
+  },
+  "module": {
+    "name": "Pointers",
+    "platform": {
+      "architecture": "arm64",
+      "operatingSystem": {
+        "minimumVersion": {
+          "major": 11,
+          "minor": 0,
+          "patch": 0
+        },
+        "name": "macosx"
+      },
+      "vendor": "apple"
+    }
+  },
+  "relationships": [],
+  "symbols": [
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "foo"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "a"
+        },
+        {
+          "kind": "text",
+          "spelling": ");"
+        }
+      ],
+      "functionSignature": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " *"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "a"
+              }
+            ],
+            "name": "a"
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@foo"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 5,
+          "line": 0
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "foo"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "foo"
+          }
+        ],
+        "title": "foo"
+      },
+      "pathComponents": [
+        "foo"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " "
+        },
+        {
+          "kind": "identifier",
+          "spelling": "bar"
+        },
+        {
+          "kind": "text",
+          "spelling": "("
+        },
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:I",
+          "spelling": "int"
+        },
+        {
+          "kind": "text",
+          "spelling": " **"
+        },
+        {
+          "kind": "internalParam",
+          "spelling": "a"
+        },
+        {
+          "kind": "text",
+          "spelling": ");"
+        }
+      ],
+      "functionSignature": {
+        "parameters": [
+          {
+            "declarationFragments": [
+              {
+                "kind": "typeIdentifier",
+                "preciseIdentifier": "c:I",
+                "spelling": "int"
+              },
+              {
+                "kind": "text",
+                "spelling": " **"
+              },
+              {
+                "kind": "internalParam",
+                "spelling": "a"
+              }
+            ],
+            "name": "a"
+          }
+        ],
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@bar"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 5,
+          "line": 1
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "bar"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "bar"
+          }
+        ],
+        "title": "bar"
+      },
+      "pathComponents": [
+        "bar"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " *"
+        },
+        {
+          "kind": "identifier",
+          "spelling": "baz"
+        },
+        {
+          "kind": "text",
+          "spelling": "();"
+        }
+      ],
+      "functionSignature": {
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          },
+          {
+            "kind": "text",
+            "spelling": " *"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@baz"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 6,
+          "line": 2
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "baz"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "baz"
+          }
+        ],
+        "title": "baz"
+      },
+      "pathComponents": [
+        "baz"
+      ]
+    },
+    {
+      "accessLevel": "public",
+      "declarationFragments": [
+        {
+          "kind": "typeIdentifier",
+          "preciseIdentifier": "c:v",
+          "spelling": "void"
+        },
+        {
+          "kind": "text",
+          "spelling": " **"
+        },
+        {
+          "kind": "identifier",
+          "spelling": "qux"
+        },
+        {
+          "kind": "text",
+          "spelling": "();"
+        }
+      ],
+      "functionSignature": {
+        "returns": [
+          {
+            "kind": "typeIdentifier",
+            "preciseIdentifier": "c:v",
+            "spelling": "void"
+          },
+          {
+            "kind": "text",
+            "spelling": " **"
+          }
+        ]
+      },
+      "identifier": {
+        "interfaceLanguage": "c",
+        "precise": "c:@F@qux"
+      },
+      "kind": {
+        "displayName": "Function",
+        "identifier": "c.func"
+      },
+      "location": {
+        "position": {
+          "character": 7,
+          "line": 3
+        },
+        "uri": "file://INPUT_DIR/input.h"
+      },
+      "names": {
+        "navigator": [
+          {
+            "kind": "identifier",
+            "spelling": "qux"
+          }
+        ],
+        "subHeading": [
+          {
+            "kind": "identifier",
+            "spelling": "qux"
+          }
+        ],
+        "title": "qux"
+      },
+      "pathComponents": [
+        "qux"
+      ]
+    }
+  ]
+}
diff --git a/clang/test/Index/inline-assembly.c b/clang/test/Index/inline-assembly.c
new file mode 100644
index 000000000000..64a7ce03852c
--- /dev/null
+++ b/clang/test/Index/inline-assembly.c
@@ -0,0 +1,46 @@
+static void inline_assembly_template_regardless_of_target_machine() {
+    int tmp;
+    asm volatile (
+        "nop\n"
+        "a_value %w[v]\n"
+        "o_value %w[o]"
+        : [v] "=&r" (tmp)
+        : [o] "r" (tmp)
+        : "cc", "memory"
+    );
+}
+
+// RUN: c-index-test -test-inline-assembly %s 2>&1 | FileCheck %s
+// CHECK: ===ASM TEMPLATE===
+// CHECK: nop
+// CHECK: a_value ${0:w}
+// CHECK: o_value ${1:w}
+// CHECK: ===ASM TEMPLATE END===
+// CHECK: volatile: true
+// CHECK: Output #0 Constraint (=&r): DeclRefExpr=tmp:2:9
+// CHECK: Input #0 Constraint (r): UnexposedExpr=tmp:2:9
+// CHECK: Clobber #0: cc
+// CHECK: Clobber #1: memory
+// CHECK: ===ASM END===
+
+static void inline_assembly_valid_x86_example() {
+    int tmp;
+    asm (
+        "nop\n"
+        "mov %w[o], %w[v]"
+        : [v] "=&r" (tmp)
+        : [o] "r" (tmp)
+        : "cc", "memory"
+    );
+}
+
+// CHECK: ===ASM TEMPLATE===
+// CHECK: nop
+// CHECK: mov ${1:w}, ${0:w}
+// CHECK: ===ASM TEMPLATE END===
+// CHECK: volatile: false
+// CHECK: Output #0 Constraint (=&r): DeclRefExpr=tmp:27:9
+// CHECK: Input #0 Constraint (r): UnexposedExpr=tmp:27:9
+// CHECK: Clobber #0: cc
+// CHECK: Clobber #1: memory
+// CHECK: ===ASM END===
diff --git a/clang/test/Interpreter/out-of-process.cpp b/clang/test/Interpreter/out-of-process.cpp
new file mode 100644
index 000000000000..6922ca6e8205
--- /dev/null
+++ b/clang/test/Interpreter/out-of-process.cpp
@@ -0,0 +1,88 @@
+// REQUIRES: host-supports-jit,  host-supports-out-of-process-jit, x86_64-linux
+
+// RUN: cat %s | clang-repl -oop-executor -orc-runtime | FileCheck %s
+
+extern "C" int printf(const char *, ...);
+
+int intVar = 0;
+double doubleVar = 3.14;
+%undo
+double doubleVar = 2.71;
+
+auto r1 = printf("intVar = %d\n", intVar);
+// CHECK: intVar = 0
+auto r2 = printf("doubleVar = %.2f\n", doubleVar);
+// CHECK: doubleVar = 2.71
+
+// Test redefinition with inline and static functions.
+int add(int a, int b, int c) { return a + b + c; }
+%undo  // Revert to the initial version of add
+inline int add(int a, int b) { return a + b; }
+
+auto r3 = printf("add(1, 2) = %d\n", add(1, 2));
+// CHECK-NEXT: add(1, 2) = 3
+
+// Test inline and lambda functions with variations.
+inline int square(int x) { return x * x; }
+auto lambdaSquare = [](int x) { return x * x; };
+auto lambdaMult = [](int a, int b) { return a * b; };
+
+auto r4 = printf("square(4) = %d\n", square(4));
+// CHECK-NEXT: square(4) = 16
+auto lambda_r1 = printf("lambdaSquare(5) = %d\n", lambdaSquare(5));
+// CHECK-NEXT: lambdaSquare(5) = 25
+auto lambda_r2 = printf("lambdaMult(2, 3) = %d\n", lambdaMult(2, 3));
+// CHECK-NEXT: lambdaMult(2, 3) = 6
+
+%undo  // Undo previous lambda assignments
+auto lambda_r3 = lambdaMult(3, 4);  // Should fail or revert to the original lambda
+
+// Test weak and strong symbol linkage.
+int __attribute__((weak)) weakFunc() { return 42; }
+int strongFunc() { return 100; }
+%undo  // Revert the weak function
+
+auto r5 = printf("weakFunc() = %d\n", weakFunc());
+// CHECK: weakFunc() = 42
+auto r6 = printf("strongFunc() = %d\n", strongFunc());
+// CHECK-NEXT: strongFunc() = 100
+
+// Weak variable linkage with different types.
+int varA = 20;
+static __typeof(varA) weakVarA __attribute__((__weakref__("varA")));
+char charVar = 'c';
+static __typeof(charVar) weakCharVar __attribute__((__weakref__("charVar")));
+auto r7 = printf("weakVarA = %d\n", weakVarA);
+// CHECK: weakVarA = 20
+auto r8 = printf("weakCharVar = %c\n", weakCharVar);
+// CHECK-NEXT: weakCharVar = c
+
+// Test complex lambdas with captures.
+int captureVar = 5;
+auto captureLambda = [](int x) { return x + captureVar; };
+int result1 = captureLambda(10);
+%undo  // Undo capture lambda
+
+auto r9 = printf("captureLambda(10) = %d\n", result1);
+// CHECK: captureLambda(10) = 15
+
+// Multiline statement test with arithmetic operations.
+int sum = \
+  5 + \
+  10;
+int prod = sum * 2;
+auto r10 = printf("sum = %d, prod = %d\n", sum, prod);
+// CHECK: sum = 15, prod = 30
+
+// Test multiline functions and macro behavior.
+#define MULTIPLY(a, b) ((a) * (b))
+
+int complexFunc(int x) \
+{ \
+  return MULTIPLY(x, 2) + x; \
+}
+
+auto r11 = printf("complexFunc(5) = %d\n", complexFunc(5));
+// CHECK: complexFunc(5) = 15
+
+%quit
\ No newline at end of file
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 41d00dae3f69..05693538252a 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -212,6 +212,7 @@
 // CHECK-NEXT: VTablePointerAuthentication (SubjectMatchRule_record)
 // CHECK-NEXT: VecReturn (SubjectMatchRule_record)
 // CHECK-NEXT: VecTypeHint (SubjectMatchRule_function)
+// CHECK-NEXT: Visibility ()
 // CHECK-NEXT: WarnUnused (SubjectMatchRule_record)
 // CHECK-NEXT: WarnUnusedResult (SubjectMatchRule_objc_method, SubjectMatchRule_enum, SubjectMatchRule_record, SubjectMatchRule_hasType_functionType, SubjectMatchRule_type_alias)
 // CHECK-NEXT: Weak (SubjectMatchRule_variable, SubjectMatchRule_function, SubjectMatchRule_record)
diff --git a/clang/test/Misc/target-invalid-cpu-note/aarch64.c b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
index 0a5c485e896b..0346ab2bb6b1 100644
--- a/clang/test/Misc/target-invalid-cpu-note/aarch64.c
+++ b/clang/test/Misc/target-invalid-cpu-note/aarch64.c
@@ -75,6 +75,7 @@
 // CHECK-SAME: {{^}}, exynos-m5
 // CHECK-SAME: {{^}}, falkor
 // CHECK-SAME: {{^}}, fujitsu-monaka
+// CHECK-SAME: {{^}}, gb10
 // CHECK-SAME: {{^}}, generic
 // CHECK-SAME: {{^}}, grace
 // CHECK-SAME: {{^}}, kryo
diff --git a/clang/test/Modules/ModulesLocalNamespace.cppm b/clang/test/Modules/ModulesLocalNamespace.cppm
new file mode 100644
index 000000000000..9e0559780975
--- /dev/null
+++ b/clang/test/Modules/ModulesLocalNamespace.cppm
@@ -0,0 +1,47 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/m-a.cppm -emit-module-interface -o %t/m-a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-module-interface -o %t/m-b.pcm \
+// RUN:     -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 %t/b.cpp -fsyntax-only -verify -fprebuilt-module-path=%t
+
+//--- a.cppm
+export module a;
+namespace aa {
+
+}
+
+//--- m-a.cppm
+export module m:a;
+namespace aa {
+struct A {};
+}
+
+//--- b.cppm
+module m:b;
+import :a;
+
+namespace bb {
+struct B {
+    void func(aa::A);
+};
+}
+
+//--- b.cpp
+// expected-no-diagnostics
+module m:b.impl;
+import a;
+import :b;
+
+namespace bb {
+using namespace aa;
+
+void B::func(A) {} 
+
+}
+
+
+
diff --git a/clang/test/Modules/mingw-exceptions.cppm b/clang/test/Modules/mingw-exceptions.cppm
new file mode 100644
index 000000000000..db7aa2ce90a9
--- /dev/null
+++ b/clang/test/Modules/mingw-exceptions.cppm
@@ -0,0 +1,5 @@
+// REQUIRES: x86-registered-target
+// RUN: %clang -target x86_64-windows-gnu -x c++-module -std=gnu++23 -c -o /dev/null -Xclang -disable-llvm-passes %s
+
+// Make sure the command succeeds and doesn't break on the -exception-model flag in cc1.
+export module empty;
diff --git a/clang/test/Modules/no-transitive-decl-change-4.cppm b/clang/test/Modules/no-transitive-decl-change-4.cppm
new file mode 100644
index 000000000000..944878be8df6
--- /dev/null
+++ b/clang/test/Modules/no-transitive-decl-change-4.cppm
@@ -0,0 +1,81 @@
+// Test that adding a new unused decl within reduced BMI may not produce a transitive change.
+//
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/T.cppm -o %t/T.pcm   \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/T1.cppm -o %t/T1.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/T2.cppm -o %t/T2.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/T3.cppm -o %t/T3.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/T4.cppm -o %t/T4.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/A.cppm -o %t/A.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/AWrapper.cppm -o %t/AWrapper.pcm \
+// RUN:      -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/B.cppm -o %t/B.pcm \
+// RUN:     -fprebuilt-module-path=%t
+//
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/A.v1.cppm -o %t/A.v1.pcm \
+// RUN:     -fmodule-file=T=%t/T.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/AWrapper.cppm -o %t/AWrapper.v1.pcm \
+// RUN:      -fprebuilt-module-path=%t -fmodule-file=A=%t/A.v1.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/B.cppm -o %t/B.v1.pcm \
+// RUN:     -fprebuilt-module-path=%t -fmodule-file=AWrapper=%t/AWrapper.v1.pcm -fmodule-file=A=%t/A.v1.pcm
+//
+// RUN: not diff %t/B.pcm %t/B.v1.pcm &> /dev/null
+
+//--- T.cppm
+export module T;
+export template <class T>
+struct Templ {};
+
+//--- T1.cppm
+export module T1;
+import T;
+export using Tunsigned = Templ<unsigned>;
+
+//--- T2.cppm
+export module T2;
+import T;
+export using Tfloat = Templ<float>;
+
+//--- T3.cppm
+export module T3;
+import T;
+export using Tlong = Templ<long long>;
+
+//--- T4.cppm
+export module T4;
+import T;
+export using Tshort = Templ<short>;
+
+//--- A.cppm
+export module A;
+import T;
+export using Tint = Templ<int>;
+
+//--- A.v1.cppm
+export module A;
+import T;
+export using Tint = Templ<int>;
+void __unused__() {}
+
+//--- AWrapper.cppm
+export module AWrapper;
+import A;
+
+//--- B.cppm
+export module B;
+import AWrapper;
+import T;
+import T1;
+import T2;
+import T3;
+import T4;
+
+export using Tdouble = Templ<double>;
diff --git a/clang/test/Modules/odr_hash.cpp b/clang/test/Modules/odr_hash.cpp
index da24b1fe8729..8ef53e32f2e9 100644
--- a/clang/test/Modules/odr_hash.cpp
+++ b/clang/test/Modules/odr_hash.cpp
@@ -5164,6 +5164,29 @@ namespace A {
 A::X x;
 #endif
 
+namespace TemplateDecltypeOperator {
+
+#if defined(FIRST) || defined(SECOND)
+template <class T6>
+T6 func();
+#endif
+
+#if defined(SECOND)
+template <class UnrelatedT>
+using UnrelatedAlias = decltype(func<UnrelatedT>())();
+#endif
+
+#if defined(FIRST) || defined(SECOND)
+class A {
+  template <class T6>
+  operator decltype(func<T6>()) () {}
+};
+#else
+A a;
+#endif
+
+}
+
 // Keep macros contained to one file.
 #ifdef FIRST
 #undef FIRST
diff --git a/clang/test/Modules/pr118137.cppm b/clang/test/Modules/pr118137.cppm
new file mode 100644
index 000000000000..38e35399b05c
--- /dev/null
+++ b/clang/test/Modules/pr118137.cppm
@@ -0,0 +1,24 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++23 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++23 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++23 %t/a.cppm -emit-llvm -o -
+
+//--- a.h
+typedef int nghttp2_session_callbacks;
+
+//--- a.cppm
+module;
+#include "a.h"
+export module g;
+template <typename, typename T>
+concept Deleter = requires(T ptr) { ptr; };
+template <typename T, Deleter<T>> struct Handle {
+  void GetRaw(this auto);
+};
+struct SessionCallbacksDeleter
+    : Handle<nghttp2_session_callbacks, SessionCallbacksDeleter> {
+} Server_callbacks;
+void Server() { Server_callbacks.GetRaw(); }
diff --git a/clang/test/Modules/pr145975.cppm b/clang/test/Modules/pr145975.cppm
new file mode 100644
index 000000000000..792c4ad811a1
--- /dev/null
+++ b/clang/test/Modules/pr145975.cppm
@@ -0,0 +1,58 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-module-interface -o %t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm \
+// RUN:     -fmodule-file=a:b=%t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-module-interface -o %t/c.pcm \
+// RUN:     -fmodule-file=a:b=%t/b.pcm -fmodule-file=a=%t/a.pcm
+
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm \
+// RUN:     -fmodule-file=a:b=%t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/c.cppm -emit-reduced-module-interface -o %t/c.pcm \
+// RUN:     -fmodule-file=a:b=%t/b.pcm -fmodule-file=a=%t/a.pcm
+
+
+//--- b.cppm
+export module a:b;
+class Polymorphic {
+public:
+    virtual ~Polymorphic() = default;
+};
+
+//--- a.h
+namespace std {
+using X = int;
+}
+
+namespace a {
+    using std::X;
+}
+
+//--- a.cppm
+module;
+#include "a.h"
+export module a;
+import :b;
+std::X var;
+namespace std {
+    export using std::X;
+}
+namespace a {
+    export using std::X;
+}
+
+//--- c.cppm
+export module c;
+import a;
+namespace a {
+    export using std::X;
+}
+
+namespace a {
+    X test() {
+        return X{};
+    }
+}
diff --git a/clang/test/Modules/template-declare.cppm b/clang/test/Modules/template-declare.cppm
new file mode 100644
index 000000000000..01a7cca10e4e
--- /dev/null
+++ b/clang/test/Modules/template-declare.cppm
@@ -0,0 +1,39 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -dwarf-version=4 -debug-info-kind=constructor \
+// RUN:     -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -dwarf-version=4 -debug-info-kind=constructor \
+// RUN:     -emit-module-interface -o %t/b.pcm -fmodule-file=a=%t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cpp -dwarf-version=4 -debug-info-kind=constructor \
+// RUN:     -emit-llvm -o - -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm | FileCheck %t/b.cpp 
+
+//--- a.cppm
+export module a;
+export template <class T>
+class a {
+private:
+    T *data;
+
+public:
+    virtual T* getData();
+};
+
+extern template class a<char>;
+
+//--- b.cppm
+export module b;
+import a;
+export struct b {
+    a<char> v;
+};
+
+//--- b.cpp
+module b;
+extern "C" void func() {
+    b();
+}
+
+// It is fine enough to check that we won't crash.
+// CHECK: define {{.*}}void @func()
diff --git a/clang/test/Modules/var-init-side-effects-modulemap.cpp b/clang/test/Modules/var-init-side-effects-modulemap.cpp
new file mode 100644
index 000000000000..750a6f173140
--- /dev/null
+++ b/clang/test/Modules/var-init-side-effects-modulemap.cpp
@@ -0,0 +1,51 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -fsyntax-only -fmodules -fmodules-cache-path=%t -fmodule-map-file=%t/module.modulemap  %t/test.cppm -I%t
+//
+
+//--- test.cppm
+#pragma clang module import Baz
+
+//--- Foo.h
+#pragma once
+class foo {
+  char dummy = 1;
+
+public:
+  static foo var;
+
+};
+
+inline foo foo::var;
+
+//--- Bar.h
+#pragma once
+#include <Foo.h>
+
+void bar() {
+  (void) foo::var;
+}
+
+//--- Baz.h
+#pragma once
+#include <Foo.h>
+
+void baz() {
+  (void) foo::var;
+}
+
+#include <Bar.h>
+
+//--- module.modulemap
+module Foo {
+  header "Foo.h"
+}
+module Bar {
+  header "Bar.h"
+}
+module Baz {
+  header "Baz.h"
+}
+
diff --git a/clang/test/OpenMP/declare_mapper_messages.c b/clang/test/OpenMP/declare_mapper_messages.c
index 223868922731..4631016698c7 100644
--- a/clang/test/OpenMP/declare_mapper_messages.c
+++ b/clang/test/OpenMP/declare_mapper_messages.c
@@ -1,12 +1,12 @@
-// RUN: %clang_cc1 -verify=omp50,expected -fopenmp -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
-// RUN: %clang_cc1 -verify=omp51,expected -fopenmp -ferror-limit 100 %s
-// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
-// RUN: %clang_cc1 -verify=expected,omp60 -fopenmp -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
+// RUN: %clang_cc1 -verify=omp50,omp5x,expected -fopenmp -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
+// RUN: %clang_cc1 -verify=omp51,omp5x,expected -fopenmp -ferror-limit 100 %s
+// RUN: %clang_cc1 -verify=expected,omp52,omp5x -fopenmp -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
+// RUN: %clang_cc1 -verify=expected,omp60,omp60-maybe-simd -fopenmp -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
 
-// RUN: %clang_cc1 -verify=omp50,expected -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
-// RUN: %clang_cc1 -verify=omp51-simd,expected -fopenmp-simd -ferror-limit 100 %s
-// RUN: %clang_cc1 -verify=expected,omp52 -fopenmp-simd -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
-// RUN: %clang_cc1 -verify=expected,omp60-simd -fopenmp-simd -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
+// RUN: %clang_cc1 -verify=omp50,omp5x,expected -fopenmp-simd -fopenmp-version=50 -ferror-limit 100 -DOMP50 %s
+// RUN: %clang_cc1 -verify=omp51-simd,omp5x,expected -fopenmp-simd -ferror-limit 100 %s
+// RUN: %clang_cc1 -verify=expected,omp52,omp5x -fopenmp-simd -fopenmp-version=52 -ferror-limit 100 -DOMP52 %s
+// RUN: %clang_cc1 -verify=expected,omp60-simd,omp60-maybe-simd -fopenmp-simd -fopenmp-version=60 -ferror-limit 100 -DOMP60 %s
 
 int temp; // expected-note {{'temp' declared here}}
 
@@ -16,17 +16,38 @@ struct vec {                                                            // expec
   double *data;
 };
 
-#pragma omp declare mapper                                              // expected-error {{expected '(' after 'declare mapper'}}
-#pragma omp declare mapper {                                            // expected-error {{expected '(' after 'declare mapper'}}
-#pragma omp declare mapper(                                             // expected-error {{expected a type}} expected-error {{expected declarator on 'omp declare mapper' directive}}
-#pragma omp declare mapper(#                                            // expected-error {{expected a type}} expected-error {{expected declarator on 'omp declare mapper' directive}}
-#pragma omp declare mapper(struct v                                     // expected-error {{expected declarator on 'omp declare mapper' directive}}
-#pragma omp declare mapper(struct vec                                   // expected-error {{expected declarator on 'omp declare mapper' directive}}
-#pragma omp declare mapper(S v                                          // expected-error {{unknown type name 'S'}}
-#pragma omp declare mapper(struct vec v                                 // expected-error {{expected ')'}} expected-note {{to match this '('}}
-#pragma omp declare mapper(aa:struct vec v)                             // expected-error {{expected at least one clause on '#pragma omp declare mapper' directive}}
-#pragma omp declare mapper(bb:struct vec v) private(v)                  // expected-error {{expected at least one clause on '#pragma omp declare mapper' directive}} // expected-error {{unexpected OpenMP clause 'private' in directive '#pragma omp declare mapper'}}
-#pragma omp declare mapper(cc:struct vec v) map(v) (                    // expected-warning {{extra tokens at the end of '#pragma omp declare mapper' are ignored}}
+// omp60-maybe-simd-error@+2 {{expected '(' after 'declare_mapper'}}
+// omp5x-error@+1 {{expected '(' after 'declare mapper'}}
+#pragma omp declare mapper
+// omp60-maybe-simd-error@+2 {{expected '(' after 'declare_mapper'}}
+// omp5x-error@+1 {{expected '(' after 'declare mapper'}}
+#pragma omp declare mapper {
+// expected-error@+2 {{expected a type}}
+// expected-error@+1 {{expected declarator on 'omp declare mapper' directive}}
+#pragma omp declare mapper(
+// expected-error@+2 {{expected a type}}
+// expected-error@+1 {{expected declarator on 'omp declare mapper' directive}}
+#pragma omp declare mapper(#
+// expected-error@+1 {{expected declarator on 'omp declare mapper' directive}}
+#pragma omp declare mapper(struct v
+// expected-error@+1 {{expected declarator on 'omp declare mapper' directive}}
+#pragma omp declare mapper(struct vec
+// expected-error@+1 {{unknown type name 'S'}}
+#pragma omp declare mapper(S v
+// expected-error@+2 {{expected ')'}}
+// expected-note@+1 {{to match this '('}}
+#pragma omp declare mapper(struct vec v
+// omp60-maybe-simd-error@+2 {{expected at least one clause on '#pragma omp declare_mapper' directive}}
+// omp5x-error@+1 {{expected at least one clause on '#pragma omp declare mapper' directive}}
+#pragma omp declare mapper(aa:struct vec v)
+// omp60-maybe-simd-error@+4 {{expected at least one clause on '#pragma omp declare_mapper' directive}}
+// omp60-maybe-simd-error@+3 {{unexpected OpenMP clause 'private' in directive '#pragma omp declare_mapper'}}
+// omp5x-error@+2 {{expected at least one clause on '#pragma omp declare mapper' directive}}
+// omp5x-error@+1 {{unexpected OpenMP clause 'private' in directive '#pragma omp declare mapper'}}
+#pragma omp declare mapper(bb:struct vec v) private(v)
+// omp60-maybe-simd-warning@+2 {{extra tokens at the end of '#pragma omp declare_mapper' are ignored}}
+// omp5x-warning@+1 {{extra tokens at the end of '#pragma omp declare mapper' are ignored}}
+#pragma omp declare mapper(cc:struct vec v) map(v) (
 
 #pragma omp declare mapper(++: struct vec v) map(v.len)                 // expected-error {{illegal OpenMP user-defined mapper identifier}}
 #pragma omp declare mapper(id1: struct vec v) map(v.len, temp)          // expected-error {{only variable 'v' is allowed in map clauses of this 'omp declare mapper' directive}}
@@ -58,7 +79,9 @@ int fun(int arg) {
 #pragma omp declare mapper(id: struct vec v) map(v.len)                 // expected-note {{previous definition is here}}
 #pragma omp declare mapper(id: struct vec v) map(v.len)                 // expected-error {{redefinition of user-defined mapper for type 'struct vec' with name 'id'}}
     {
-#pragma omp declare mapper(id: struct vec v) map(v.len) allocate(v)   // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp declare mapper'}}
+// omp60-maybe-simd-error@+2 {{unexpected OpenMP clause 'allocate' in directive '#pragma omp declare_mapper'}}
+// omp5x-error@+1 {{unexpected OpenMP clause 'allocate' in directive '#pragma omp declare mapper'}}
+#pragma omp declare mapper(id: struct vec v) map(v.len) allocate(v)
       struct vec vv, v1;
       struct vec arr[10];
       double d;
diff --git a/clang/test/OpenMP/declare_target_messages.cpp b/clang/test/OpenMP/declare_target_messages.cpp
index 3c0e766cf72c..4aa4d686eaaf 100644
--- a/clang/test/OpenMP/declare_target_messages.cpp
+++ b/clang/test/OpenMP/declare_target_messages.cpp
@@ -11,14 +11,14 @@
 
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp45,omp45-to-51,omp45-to-51-var,omp45-to-51-clause,omp45-to-51-clause  %{openmp45} %{limit} -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51  %{openmp50} %{target_mac} %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var  %{openmp60} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var  %{openmp60} %{target_mac} %{limit} -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51,dev5  %{openmp50} -fopenmp-is-target-device %{target_mac} %{aux_triple} %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -fopenmp-is-target-device %{target_mac} %{aux_triple} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -fopenmp-is-target-device %{target_mac} %{aux_triple} %{limit} -o - %s
 
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51 %{openmp50_simd} %{target_mac} %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} %{target_mac} %{limit} -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host5,host-5-and-51,no-host5-and-51 %{openmp50_simd} -fopenmp-is-target-device %{target_mac} %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} -fopenmp-is-target-device %{target_mac} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp60_simd} -fopenmp-is-target-device %{target_mac} %{limit} -o - %s
 
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp45,omp45-to-51,omp45-to-51-var,omp45-to-51-clause -fopenmp-version=45 -fopenmp-simd %{limit} -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -o - %s
@@ -26,12 +26,12 @@
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -I%S/Inputs -DTESTENDINC=1 -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp-simd %{limit} -o - %s
 
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp52} -DVERBOSE_MODE=1 %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -DVERBOSE_MODE=1 %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp52} -DVERBOSE_MODE=1 %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} -DVERBOSE_MODE=1 %{limit} -o - %s
 
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp5,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 %{openmp50} %{limit} -o - %s
 // RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp51,ompvar,omp45-to-51,omp5-and-51,omp5-or-later,omp5-or-later-var,omp45-to-51-var,omp45-to-51-clause,host-5-and-51,no-host5-and-51 -fopenmp %{limit} -o - %s
-// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp52,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} %{limit} -o - %s
+// RUN: %clang_cc1 %{common_opts_mac} -verify=expected,omp60,omp52-or-later,ompvar,omp5-or-later,omp5-or-later-var %{openmp60} %{limit} -o - %s
 
 #pragma omp begin declare target
 static int gg;
@@ -39,7 +39,9 @@ static int gg;
 int recursive = recursive ^ 3 + gg;
 #pragma omp end declare target
 
-// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp60-error@+3 {{unexpected OpenMP directive '#pragma omp end declare_target'}}
+// omp52-error@+2 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp45-to-51-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
 #pragma omp end declare target 
 
 // ompvar-error@+1 {{variable captured in declare target region must appear in a to clause}}
@@ -47,14 +49,20 @@ int a, b, z;
 // expected-note@+1 {{defined as threadprivate or thread local}}
 __thread int t;
 
-// expected-error@+1 {{expected '(' after 'declare target'}}
+// omp60-error@+3 {{expected '(' after 'declare_target'}}
+// omp52-error@+2 {{expected '(' after 'declare target'}}
+// omp45-to-51-error@+1 {{expected '(' after 'declare target'}}
 #pragma omp declare target . 
 
 #pragma omp declare target
 void f();
-// expected-warning@+1 {{extra tokens at the end of '#pragma omp end declare target' are ignored}}
+// omp60-warning@+3 {{extra tokens at the end of '#pragma omp end declare_target' are ignored}}
+// omp52-warning@+2 {{extra tokens at the end of '#pragma omp end declare target' are ignored}}
+// omp45-to-51-warning@+1 {{extra tokens at the end of '#pragma omp end declare target' are ignored}}
 #pragma omp end declare target shared(a) 
 
+// omp60-error@+10 {{unexpected 'map' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp60-error@+9 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp52-error@+8 {{unexpected 'map' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
 // omp52-error@+7 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp51-error@+6 {{unexpected 'map' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}} 
@@ -65,6 +73,8 @@ void f();
 // omp45-error@+1 {{expected at least one 'to' or 'link' clause}} 
 #pragma omp declare target map(a)
 
+// omp60-error@+5 {{unexpected 'to' clause, use 'enter' instead}}
+// omp60-error@+4 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
 // omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp45-to-51-error@+1 {{use of undeclared identifier 'foo1'}}
@@ -73,6 +83,8 @@ void f();
 // expected-error@+1 {{use of undeclared identifier 'foo2'}}
 #pragma omp declare target link(foo2) 
 
+// omp60-error@+6 {{unexpected 'to' clause, use 'enter' instead}}
+// omp60-error@+5 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
 // omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // dev5-note@+2 {{marked as 'device_type(host)' here}}
@@ -80,8 +92,8 @@ void f();
 #pragma omp declare target to(f) device_type(host)
 
 void q();
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp5-and-51-warning@+2 {{more than one 'device_type' clause is specified}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(q) device_type(any) device_type(any) device_type(host) 
@@ -121,7 +133,7 @@ void c();
 // expected-note@+1 {{'func' defined here}}
 void func() {} 
 
-// omp52-error@+5 {{unexpected 'allocate' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
+// omp52-or-later-error@+5 {{unexpected 'allocate' clause, only 'enter', 'link', 'device_type' or 'indirect' clauses expected}}
 // omp51-error@+4 {{unexpected 'allocate' clause, only 'to', 'link', 'device_type' or 'indirect' clauses expected}}
 // omp5-error@+3 {{unexpected 'allocate' clause, only 'to', 'link' or 'device_type' clauses expected}}
 // expected-error@+2 {{function name is not allowed in 'link' clause}}
@@ -171,7 +183,9 @@ void t2() {
   void abc();
 #pragma omp end declare target
 void cba();
-// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp60-error@+3 {{unexpected OpenMP directive '#pragma omp end declare_target'}}
+// omp52-error@+2 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp45-to-51-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
 #pragma omp end declare target 
 
 #pragma omp declare target
@@ -234,7 +248,9 @@ void foo1() {
 
 #pragma omp end declare target
 #pragma omp end declare target
-// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp60-error@+3 {{unexpected OpenMP directive '#pragma omp end declare_target'}}
+// omp52-error@+2 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp45-to-51-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
 #pragma omp end declare target 
 
 int C::method() {
@@ -255,18 +271,22 @@ int *y;
 int **w = &y;
 int main (int argc, char **argv) {
   int a = 2;
-// expected-error@+1 {{unexpected OpenMP directive '#pragma omp declare target'}}
+// omp60-error@+3 {{unexpected OpenMP directive '#pragma omp declare_target'}}
+// omp52-error@+2 {{unexpected OpenMP directive '#pragma omp declare target'}}
+// omp45-to-51-error@+1 {{unexpected OpenMP directive '#pragma omp declare target'}}
 #pragma omp declare target 
   int v;
-// expected-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp60-error@+3 {{unexpected OpenMP directive '#pragma omp end declare_target'}}
+// omp52-error@+2 {{unexpected OpenMP directive '#pragma omp end declare target'}}
+// omp45-to-51-error@+1 {{unexpected OpenMP directive '#pragma omp end declare target'}}
 #pragma omp end declare target 
   foo(v);
 
-  // omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
-  // omp52-error@+1 {{unexpected 'to' clause, use 'enter' instead}}
+  // omp52-or-later-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+  // omp52-or-later-error@+1 {{unexpected 'to' clause, use 'enter' instead}}
 #pragma omp declare target to(foo3) link(w)
-  // omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
-  // omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+  // omp52-or-later-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+  // omp52-or-later-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
   // omp45-to-51-var-error@+1 {{local variable 'a' should not be used in 'declare target' directive}}
 #pragma omp declare target to(a) 
   return (0);
@@ -283,48 +303,48 @@ namespace {
 
 // expected-error@+1 {{'x' appears multiple times in clauses on the same declare target directive}}
 #pragma omp declare target (x, x) 
-// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp45-to-51-clause-error@+1 {{'x' appears multiple times in clauses on the same declare target directive}}
 #pragma omp declare target to(x) to(x)
 // expected-error@+1 {{'x' must not appear in both clauses 'to' and 'link'}}
 #pragma omp declare target link(x) 
 
 void bazz() {}
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // host5-note@+2 3 {{marked as 'device_type(nohost)' here}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}} 
 #pragma omp declare target to(bazz) device_type(nohost)
 void bazzz() {bazz();}
-// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(bazzz) device_type(nohost) 
 // host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
 void any() {bazz();} 
 // host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
 void host1() {bazz();}
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // dev5-note@+2 3 {{marked as 'device_type(host)' here}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(host1) device_type(host)
 //host5-error@+1 {{function with 'device_type(nohost)' is not available on host}}
 void host2() {bazz();}
-// omp52-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 #pragma omp declare target to(host2) 
 // dev5-error@+1 {{function with 'device_type(host)' is not available on device}}
 void device() {host1();}
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // host5-note@+2 2 {{marked as 'device_type(nohost)' here}} 
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(device) device_type(nohost)
 void host3() {host1();} // dev5-error {{function with 'device_type(host)' is not available on device}}
-// omp52-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+2 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+1 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 #pragma omp declare target to(host3)
 
 #pragma omp declare target
@@ -343,17 +363,17 @@ void any7() {device();}
 void any8() {any2();}
 
 int MultiDevTy;
-// omp52-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+3 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+2 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(MultiDevTy) device_type(any)
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // host-5-and-51-error@+2 {{'device_type(host)' does not match previously specified 'device_type(any)' for the same declaration}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(MultiDevTy) device_type(host)
-// omp52-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
-// omp52-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
+// omp52-or-later-error@+4 {{unexpected 'to' clause, use 'enter' instead}}
+// omp52-or-later-error@+3 {{expected at least one 'enter', 'link' or 'indirect' clause}}
 // no-host5-and-51-error@+2 {{'device_type(nohost)' does not match previously specified 'device_type(any)' for the same declaration}}
 // omp45-error@+1 {{unexpected 'device_type' clause, only 'to' or 'link' clauses expected}}
 #pragma omp declare target to(MultiDevTy) device_type(nohost)
@@ -414,6 +434,8 @@ target *S1 = &S;
 // expected-warning@+1 {{expected '#pragma omp end declare target' at end of file to match '#pragma omp declare target'}}
 #pragma omp declare target
 #else
-// expected-warning@+1 {{expected '#pragma omp end declare target' at end of file to match '#pragma omp begin declare target'}}
+// omp60-warning@+3 {{expected '#pragma omp end declare target' at end of file to match '#pragma omp begin declare_target'}}
+// omp52-warning@+2 {{expected '#pragma omp end declare target' at end of file to match '#pragma omp begin declare target'}}
+// omp45-to-51-warning@+1 {{expected '#pragma omp end declare target' at end of file to match '#pragma omp begin declare target'}}
 #pragma omp begin declare target
 #endif
diff --git a/clang/test/OpenMP/declare_variant_clauses_messages.cpp b/clang/test/OpenMP/declare_variant_clauses_messages.cpp
index aadded7699ea..bca91481220f 100644
--- a/clang/test/OpenMP/declare_variant_clauses_messages.cpp
+++ b/clang/test/OpenMP/declare_variant_clauses_messages.cpp
@@ -152,7 +152,7 @@ void vararg_bar2(const char *fmt) { return; }
 #pragma omp declare variant(foo_v1) match(construct={dispatch}) \
                                     append_args(foobar(target))
 
-// expected-error@+2 {{directive '#pragma omp declare variant' cannot contain more than one 'append_args' clause}}
+// expected-error@+2 {{directive '#pragma omp declare_variant' cannot contain more than one 'append_args' clause}}
 #pragma omp declare variant(foo_v1) match(construct={dispatch}) \
                                     append_args(interop(target)) \
                                     append_args(interop(targetsync))
diff --git a/clang/test/OpenMP/openmp-6-future-spellings.cpp b/clang/test/OpenMP/openmp-6-future-spellings.cpp
new file mode 100644
index 000000000000..642ed3502d47
--- /dev/null
+++ b/clang/test/OpenMP/openmp-6-future-spellings.cpp
@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=52 -ferror-limit 100 -o - %s
+
+// expected-warning@+1 {{directive spelling 'begin declare_target' is introduced in a later OpenMP version}}
+#pragma omp begin declare_target
+void f0();
+// expected-warning@+1 {{directive spelling 'end declare_target' is introduced in a later OpenMP version}}
+#pragma omp end declare_target
+
+// expected-warning@+1 {{directive spelling 'begin declare_variant' is introduced in a later OpenMP version}}
+#pragma omp begin declare_variant match(user={condition(true)})
+void f1();
+// expected-warning@+1 {{directive spelling 'end declare_variant' is introduced in a later OpenMP version}}
+#pragma omp end declare_variant
+
+int x;
+// expected-warning@+1 {{directive spelling 'declare_target' is introduced in a later OpenMP version}}
+#pragma omp declare_target(x)
+
+struct A {
+  int x, y;
+};
+// expected-warning@+1 {{directive spelling 'declare_mapper' is introduced in a later OpenMP version}}
+#pragma omp declare_mapper(mymapper: A a) map(tofrom:a.x, a.y)
+A add(A, A);
+// expected-warning@+1 {{directive spelling 'declare_reduction' is introduced in a later OpenMP version}}
+#pragma omp declare_reduction(+: A: omp_out = add(omp_in, omp_out))
+
+// expected-warning@+1 {{directive spelling 'declare_simd' is introduced in a later OpenMP version}}
+#pragma omp declare_simd
+void f2();
+
+void g3();
+// expected-warning@+1 {{directive spelling 'declare_variant' is introduced in a later OpenMP version}}
+#pragma omp declare_variant(g3) match(user={condition(true)})
+void f3() {}
+
+void fred() {
+  #pragma omp parallel
+  {
+    // expected-warning@+1 {{directive spelling 'cancellation_point' is introduced in a later OpenMP version}}
+    #pragma omp cancellation_point parallel
+  }
+
+  // expected-warning@+1 {{directive spelling 'target_data' is introduced in a later OpenMP version}}
+  #pragma omp target_data map(tofrom: x)
+  {}
+
+  // expected-warning@+1 {{directive spelling 'target_enter_data' is introduced in a later OpenMP version}}
+  #pragma omp target_enter_data map(to: x)
+  // expected-warning@+1 {{directive spelling 'target_exit_data' is introduced in a later OpenMP version}}
+  #pragma omp target_exit_data map(from: x)
+  // expected-warning@+1 {{directive spelling 'target_update' is introduced in a later OpenMP version}}
+  #pragma omp target_update from(x)
+}
+
diff --git a/clang/test/OpenMP/parallel_ast_print.cpp b/clang/test/OpenMP/parallel_ast_print.cpp
index 83afedcb740d..948baaff30d8 100644
--- a/clang/test/OpenMP/parallel_ast_print.cpp
+++ b/clang/test/OpenMP/parallel_ast_print.cpp
@@ -13,6 +13,14 @@
 // RUN: %clang_cc1 -DOMP51 -verify -Wno-vla -fopenmp-simd -ast-print %s | FileCheck -check-prefixes=CHECK,OMP51 %s
 // RUN: %clang_cc1 -DOMP51 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -DOMP51 -fopenmp-simd -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck -check-prefixes=CHECK,OMP51 %s
+
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP60 %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck -check-prefixes=CHECK,OMP60 %s
+
+// RUN: %clang_cc1 -DOMP60 -verify -Wno-vla -fopenmp-simd -fopenmp-version=60 -ast-print %s | FileCheck -check-prefixes=CHECK,OMP60 %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -DOMP60 -fopenmp-simd -fopenmp-version=60 -std=c++11 -include-pch %t -verify -Wno-vla %s -ast-print | FileCheck -check-prefixes=CHECK,OMP60 %s
 // expected-no-diagnostics
 
 #ifndef HEADER
@@ -164,8 +172,16 @@ T tmain(T argc, T *argv) {
 #pragma omp parallel default(none), private(argc,b) firstprivate(argv) shared (d) if (parallel:argc > 0) num_threads(C) copyin(S<T>::TS, thrp) proc_bind(primary) reduction(+:c, arr1[argc]) reduction(max:e, arr[:C][0:10])
   foo();
 #endif
+#ifdef OMP60
+#pragma omp parallel default(none), private(argc,b) firstprivate(argv) shared (d) if (parallel:argc > 0) num_threads(strict: C) copyin(S<T>::TS, thrp) proc_bind(primary) reduction(+:c, arr1[argc]) reduction(max:e, arr[:C][0:10])
+  foo();
+#endif
 #pragma omp parallel if (C) num_threads(s) proc_bind(close) reduction(^:e, f, arr[0:C][:argc]) reduction(default, && : g) reduction(task,+:argc)
   foo();
+#ifdef OMP60
+#pragma omp parallel if (C) num_threads(strict: s) proc_bind(close) reduction(^:e, f, arr[0:C][:argc]) reduction(default, && : g) reduction(task,+:argc)
+  foo();
+#endif
   return 0;
 }
 
@@ -180,8 +196,12 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo()
 // OMP51-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(C) copyin(S<T>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:C][0:10])
 // OMP51-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(strict: C) copyin(S<T>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:C][0:10])
+// OMP60-NEXT: foo()
 // CHECK-NEXT: #pragma omp parallel if(C) num_threads(s) proc_bind(close) reduction(^: e,f,arr[0:C][:argc]) reduction(default, &&: g) reduction(task, +: argc)
 // CHECK-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel if(C) num_threads(strict: s) proc_bind(close) reduction(^: e,f,arr[0:C][:argc]) reduction(default, &&: g) reduction(task, +: argc)
+// OMP60-NEXT: foo()
 // CHECK: template<> int tmain<int, 5>(int argc, int *argv) {
 // CHECK-NEXT: int b = argc, c, d, e, f, g;
 // CHECK-NEXT: static int a;
@@ -193,8 +213,12 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo()
 // OMP51-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(5) copyin(S<int>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:5][0:10])
 // OMP51-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(strict: 5) copyin(S<int>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:5][0:10])
+// OMP60-NEXT: foo()
 // CHECK-NEXT: #pragma omp parallel if(5) num_threads(s) proc_bind(close) reduction(^: e,f,arr[0:5][:argc]) reduction(default, &&: g) reduction(task, +: argc)
 // CHECK-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel if(5) num_threads(strict: s) proc_bind(close) reduction(^: e,f,arr[0:5][:argc]) reduction(default, &&: g) reduction(task, +: argc)
+// OMP60-NEXT: foo()
 // CHECK: template<> long tmain<long, 1>(long argc, long *argv) {
 // CHECK-NEXT: long b = argc, c, d, e, f, g;
 // CHECK-NEXT: static long a;
@@ -206,8 +230,12 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: foo()
 // OMP51-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(1) copyin(S<long>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:1][0:10])
 // OMP51-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) shared(d) if(parallel: argc > 0) num_threads(strict: 1) copyin(S<long>::TS,thrp) proc_bind(primary) reduction(+: c,arr1[argc]) reduction(max: e,arr[:1][0:10])
+// OMP60-NEXT: foo()
 // CHECK-NEXT: #pragma omp parallel if(1) num_threads(s) proc_bind(close) reduction(^: e,f,arr[0:1][:argc]) reduction(default, &&: g) reduction(task, +: argc)
 // CHECK-NEXT: foo()
+// OMP60-NEXT: #pragma omp parallel if(1) num_threads(strict: s) proc_bind(close) reduction(^: e,f,arr[0:1][:argc]) reduction(default, &&: g) reduction(task, +: argc)
+// OMP60-NEXT: foo()
 
 enum Enum { };
 
@@ -227,10 +255,22 @@ int main (int argc, char **argv) {
 // CHECK-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) if(parallel: argc > 0) num_threads(ee) copyin(a) proc_bind(spread) reduction(|: c,d,arr1[argc]) reduction(*: e,arr[:10][0:argc]) allocate(e)
   foo();
 // CHECK-NEXT: foo();
+#ifdef OMP60
+#pragma omp parallel default(none), private(argc,b) firstprivate(argv) if (parallel: argc > 0) num_threads(strict: ee) copyin(a) proc_bind(spread) reduction(| : c, d, arr1[argc]) reduction(* : e, arr[:10][0:argc]) allocate(e)
+// OMP60-NEXT: #pragma omp parallel default(none) private(argc,b) firstprivate(argv) if(parallel: argc > 0) num_threads(strict: ee) copyin(a) proc_bind(spread) reduction(|: c,d,arr1[argc]) reduction(*: e,arr[:10][0:argc]) allocate(e)
+  foo();
+// OMP60-NEXT: foo();
+#endif
+#pragma omp parallel allocate(e) if (b) num_threads(c) proc_bind(close) reduction(^:e, f) reduction(&& : g, arr[0:argc][:10])
 // CHECK-NEXT: #pragma omp parallel allocate(e) if(b) num_threads(c) proc_bind(close) reduction(^: e,f) reduction(&&: g,arr[0:argc][:10])
+  foo();
 // CHECK-NEXT: foo()
-#pragma omp parallel allocate(e) if (b) num_threads(c) proc_bind(close) reduction(^:e, f) reduction(&& : g, arr[0:argc][:10])
+#ifdef OMP60
+#pragma omp parallel allocate(e) if (b) num_threads(strict: c) proc_bind(close) reduction(^:e, f) reduction(&& : g, arr[0:argc][:10])
+// OMP60-NEXT: #pragma omp parallel allocate(e) if(b) num_threads(strict: c) proc_bind(close) reduction(^: e,f) reduction(&&: g,arr[0:argc][:10])
   foo();
+// OMP60-NEXT: foo()
+#endif
   return tmain<int, 5>(b, &b) + tmain<long, 1>(x, &x);
 }
 
diff --git a/clang/test/OpenMP/parallel_num_threads_messages.cpp b/clang/test/OpenMP/parallel_num_threads_messages.cpp
index 0e50c7e1eb2a..02c1cb6e69d9 100644
--- a/clang/test/OpenMP/parallel_num_threads_messages.cpp
+++ b/clang/test/OpenMP/parallel_num_threads_messages.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wuninitialized
-
 // RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wuninitialized
 
+// RUN: %clang_cc1 -DOMP60 -verify=expected,omp60 -fopenmp -fopenmp-version=60 -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -DOMP60 -verify=expected,omp60 -fopenmp-simd -fopenmp-version=60 -ferror-limit 100 %s -Wuninitialized
+
 void foo() {
 }
 
@@ -9,11 +11,11 @@ bool foobool(int argc) {
   return argc;
 }
 
-struct S1; // expected-note {{declared here}}
+struct S1; // expected-note {{declared here}} omp60-note {{declared here}}
 
 #define redef_num_threads(a, b) num_threads(a)
 
-template <class T, typename S, int N> // expected-note {{declared here}}
+template <class T, typename S, int N> // expected-note {{declared here}} omp60-note {{declared here}}
 T tmain(T argc, S **argv) {
   T z;
   #pragma omp parallel num_threads // expected-error {{expected '(' after 'num_threads'}}
@@ -28,6 +30,39 @@ T tmain(T argc, S **argv) {
   #pragma omp parallel num_threads (argc+z)
   #pragma omp parallel num_threads (N) // expected-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
   #pragma omp parallel redef_num_threads (argc, argc)
+
+#ifdef OMP60
+  // Valid uses of strict modifier
+  #pragma omp parallel num_threads(strict: 4)
+  #pragma omp parallel num_threads(strict: argc+z)
+
+  // Invalid: missing expression after strict:
+  #pragma omp parallel num_threads(strict: ) // omp60-error {{expected expression}}
+  #pragma omp parallel num_threads(strict:) // omp60-error {{expected expression}}
+  #pragma omp parallel num_threads(strict: // omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+
+  // Invalid: unknown/missing modifier
+  #pragma omp parallel num_threads(foo: 4) // omp60-error {{expected 'strict' in OpenMP clause 'num_threads'}}
+  #pragma omp parallel num_threads(: 4) // omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+  #pragma omp parallel num_threads(:)// omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+
+  // Invalid: missing colon after modifier
+  #pragma omp parallel num_threads(strict 4) // omp60-error {{missing ':' after strict modifier}}
+
+  // Invalid: negative, zero, or non-integral
+  #pragma omp parallel num_threads(strict: -1) // omp60-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
+  #pragma omp parallel num_threads(strict: 0) // omp60-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
+  #pragma omp parallel num_threads(strict: (argc > 0) ? argv[1] : argv[2]) // omp60-error 2 {{expression must have integral or unscoped enumeration type, not 'char *'}}
+  #pragma omp parallel num_threads(strict: S) // omp60-error {{'S' does not refer to a value}}
+  #pragma omp parallel num_threads(strict: argv[1]=2) // omp60-error {{expected ')'}} omp60-note {{to match this '('}} omp60-error 2 {{expression must have integral or unscoped enumeration type, not 'char *'}}
+  #pragma omp parallel num_threads(strict: N) // omp60-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
+
+  // Invalid: multiple strict modifiers or mixed with non-strict
+  #pragma omp parallel num_threads(strict: 4, strict: 5) // omp60-error {{expected ')'}} expected-note {{to match this '('}}
+  #pragma omp parallel num_threads(strict: 4), num_threads(5) // omp60-error {{directive '#pragma omp parallel' cannot contain more than one 'num_threads' clause}}
+  #pragma omp parallel num_threads(4), num_threads(strict: 5)  // omp60-error {{directive '#pragma omp parallel' cannot contain more than one 'num_threads' clause}}
+#endif // OMP60
+
   foo();
 
   return argc;
@@ -46,6 +81,38 @@ int z;
   #pragma omp parallel num_threads (argv[1]=2) // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expression must have integral or unscoped enumeration type, not 'char *'}}
   #pragma omp parallel num_threads (num_threads(tmain<int, char, -1>(argc, argv) // expected-error 2 {{expected ')'}} expected-note 2 {{to match this '('}} expected-note {{in instantiation of function template specialization 'tmain<int, char, -1>' requested here}}
   #pragma omp parallel redef_num_threads (argc, argc)
+
+#ifdef OMP60
+  // Valid uses of strict modifier
+  #pragma omp parallel num_threads(strict: 4)
+  #pragma omp parallel num_threads(strict: argc+z)
+
+  // Invalid: missing expression after strict:
+  #pragma omp parallel num_threads(strict: ) // omp60-error {{expected expression}}
+  #pragma omp parallel num_threads(strict:) // omp60-error {{expected expression}}
+  #pragma omp parallel num_threads(strict: // omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+
+  // Invalid: unknown/missing modifier
+  #pragma omp parallel num_threads(foo: 4) // omp60-error {{expected 'strict' in OpenMP clause 'num_threads'}}
+  #pragma omp parallel num_threads(: 4) // omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+  #pragma omp parallel num_threads(:) // omp60-error {{expected expression}} omp60-error {{expected ')'}} omp60-note {{to match this '('}}
+
+  // Invalid: missing colon after modifier
+  #pragma omp parallel num_threads(strict 4) // omp60-error {{missing ':' after strict modifier}}
+
+  // Invalid: negative, zero, or non-integral
+  #pragma omp parallel num_threads(strict: -1) // omp60-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
+  #pragma omp parallel num_threads(strict: 0) // omp60-error {{argument to 'num_threads' clause must be a strictly positive integer value}}
+  #pragma omp parallel num_threads(strict: (argc > 0) ? argv[1] : argv[2]) // omp60-error {{expression must have integral or unscoped enumeration type, not 'char *'}}
+  #pragma omp parallel num_threads(strict: S1) // omp60-error {{'S1' does not refer to a value}}
+  #pragma omp parallel num_threads(strict: argv[1]=2) // omp60-error {{expected ')'}} omp60-note {{to match this '('}} omp60-error {{expression must have integral or unscoped enumeration type, not 'char *'}}
+
+  // Invalid: multiple strict modifiers or mixed with non-strict
+  #pragma omp parallel num_threads(strict: 4, strict: 5) // omp60-error {{expected ')'}} expected-note {{to match this '('}}
+  #pragma omp parallel num_threads(strict: 4), num_threads(5) // omp60-error {{directive '#pragma omp parallel' cannot contain more than one 'num_threads' clause}}
+  #pragma omp parallel num_threads(4), num_threads(strict: 5) // omp60-error {{directive '#pragma omp parallel' cannot contain more than one 'num_threads' clause}}
+#endif // OMP60
+
   foo();
 
   return tmain<int, char, 3>(argc, argv); // expected-note {{in instantiation of function template specialization 'tmain<int, char, 3>' requested here}}
diff --git a/clang/test/OpenMP/target_data_ast_print.cpp b/clang/test/OpenMP/target_data_ast_print.cpp
index a41c7f1a0da5..3f939549dcb5 100644
--- a/clang/test/OpenMP/target_data_ast_print.cpp
+++ b/clang/test/OpenMP/target_data_ast_print.cpp
@@ -102,7 +102,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: T i, j, b, c, d, e, x[20];
 // CHECK-NEXT: #pragma omp target data map(to: c){{$}}
 // CHECK-NEXT: i = argc;
-// CHECK-NEXT: #pragma omp target data map(to: c) if(target data: j > 0)
+// CHECK-NEXT: #pragma omp target data map(to: c) if(target{{[ _]}}data: j > 0)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(to: c) if(b)
 // CHECK-NEXT: foo();
@@ -140,7 +140,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: int i, j, b, c, d, e, x[20];
 // CHECK-NEXT: #pragma omp target data map(to: c)
 // CHECK-NEXT: i = argc;
-// CHECK-NEXT: #pragma omp target data map(to: c) if(target data: j > 0)
+// CHECK-NEXT: #pragma omp target data map(to: c) if(target{{[ _]}}data: j > 0)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(to: c) if(b)
 // CHECK-NEXT: foo();
@@ -178,7 +178,7 @@ T tmain(T argc, T *argv) {
 // CHECK-NEXT: char i, j, b, c, d, e, x[20];
 // CHECK-NEXT: #pragma omp target data map(to: c)
 // CHECK-NEXT: i = argc;
-// CHECK-NEXT: #pragma omp target data map(to: c) if(target data: j > 0)
+// CHECK-NEXT: #pragma omp target data map(to: c) if(target{{[ _]}}data: j > 0)
 // CHECK-NEXT: foo();
 // CHECK-NEXT: #pragma omp target data map(to: c) if(b)
 // CHECK-NEXT: foo();
@@ -225,7 +225,7 @@ int main (int argc, char **argv) {
   a=2;
 // CHECK-NEXT: a = 2;
 #pragma omp target data map(to: c) if (target data: b)
-// CHECK: #pragma omp target data map(to: c) if(target data: b)
+// CHECK: #pragma omp target data map(to: c) if(target{{[ _]}}data: b)
   foo();
 // CHECK-NEXT: foo();
 
diff --git a/clang/test/OpenMP/target_map_messages.cpp b/clang/test/OpenMP/target_map_messages.cpp
index 911031d5412a..4a026584a47c 100644
--- a/clang/test/OpenMP/target_map_messages.cpp
+++ b/clang/test/OpenMP/target_map_messages.cpp
@@ -1,35 +1,35 @@
 // -fopenmp, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,ge52,lt60,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge52,ge60,omp,ge60-omp,omp60 -fopenmp -fno-openmp-extensions -fopenmp-version=60 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,omp,lt51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,ge51,lt60,omp,ge51-omp -fopenmp -fno-openmp-extensions -fopenmp-version=51 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,ge51,ge52,lt60,omp,ge52-omp,omp52 -fopenmp -fno-openmp-extensions -fopenmp-version=52 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,ge60,omp,ge60-omp,omp60 -fopenmp -fno-openmp-extensions -fopenmp-version=60 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd, -fno-openmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,omp,lt51-omp -fopenmp-simd -fno-openmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,ge51,lt60,omp,ge51-omp -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fno-openmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // -fopenmp -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp -Wuninitialized -Wno-vla
 
 // -fopenmp-simd -fopenmp-extensions
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=40 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -verify=expected,lt50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=45 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
-// RUN: %clang_cc1 -verify=expected,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,lt51,lt60,ompx,lt51-ompx -fopenmp-simd -fopenmp-extensions -fopenmp-version=50 -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
+// RUN: %clang_cc1 -verify=expected,omp5x,ge50,ge51,lt60,ompx,ge51-ompx -fopenmp-simd -fopenmp-extensions -ferror-limit 300 %s -Wno-openmp-target -Wuninitialized -Wno-vla
 // RUN: %clang_cc1 -DCCODE -verify -fopenmp-simd -fopenmp-extensions -ferror-limit 300 -x c %s -Wno-openmp-mapping -Wuninitialized -Wno-vla
 
 // Check
@@ -87,7 +87,7 @@ struct SA {
     {}
     #pragma omp target map(arg[2:2],a,d) // expected-error {{subscripted value is not an array or pointer}}
     {}
-    #pragma omp target map(arg,a*2) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
+    #pragma omp target map(arg,a*2) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}} ge60-error {{expected addressable lvalue in 'map' clause}}
     {}
     #pragma omp target map(arg,(c+1)[2]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}}
     {}
@@ -170,27 +170,32 @@ struct SA {
     // expected-error@+1 {{use of undeclared identifier 'present'}}
     #pragma omp target map(present)
     {}
+    // ge60-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f)
     {}
+    // ge60-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[1:2],f)
     {}
+    // ge60-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c,f[1:2])
     {}
+    // ge60-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, tofrom: c[:],f)
     {}
+    // ge60-error@+5 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+4 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+3 {{section length is unspecified and cannot be inferred because subscripted value is not an array}}
     // ge51-omp-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
@@ -211,12 +216,14 @@ struct SA {
     // lt51-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(present, present, tofrom: a)
     {}
+    // ge60-error@+5 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge52-error@+4 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // ompx-error@+3 {{same map type modifier has been specified more than once}}
     // ge51-omp-error@+2 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-omp-error@+1 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(ompx_hold, ompx_hold, tofrom: a)
     {}
+    // ge60-error@+10 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator', 'self}}
     // ge60-error@+9 {{same map type modifier has been specified more than once}}
     // ge52-error@+8 2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present', 'iterator'}}
     // expected-error@+7 2 {{same map type modifier has been specified more than once}}
@@ -247,10 +254,11 @@ struct SA {
     // lt60-error@+1 {{missing map type}}
     #pragma omp target map( , f, : a)
     {}
-    #pragma omp target map(always close: a)   // lt60-error {{missing map type}} ge52-error{{missing ',' after map type modifier}}
+    #pragma omp target map(always close: a)   // lt60-error {{missing map type}} ge52-error{{missing ',' after map type modifier}} ge60-error {{missing ',' after map type modifier}}
     {}
-    #pragma omp target map(always close bf: a)   // ge52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    #pragma omp target map(always close bf: a)   // ge52-error 2 {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}} ge60-error 2 {{missing ',' after map type modifier}}
     {}
+    // ge60-error@+5 {{missing ',' after map type modifier}}
     // ge52-error@+4 {{missing ',' after map type modifier}}
     // ge51-error@+3 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper', 'present'}}
     // lt51-error@+2 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
@@ -263,6 +271,7 @@ struct SA {
     // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(tofrom from: a)
     {}
+    // ge60-error@+6 {{missing ',' after map type modifier}}
     // ge60-note@+5 {{map type 'to' is previous specified here}}
     // ge60-error@+4 {{map type is already specified}}
     // ge52-error@+3 {{missing ',' after map type modifier}}
@@ -270,7 +279,7 @@ struct SA {
     // lt51-error@+1 {{incorrect map type modifier, expected one of: 'always', 'close', 'mapper'}}
     #pragma omp target map(to always from: a)
     {}
-    #pragma omp target map(close bf: a)   // ge52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}}
+    #pragma omp target map(close bf: a)   // ge52-error {{missing ',' after map type modifier}} expected-error {{incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'}} ge60-error {{missing ',' after map type modifier}}
     {}
     #pragma omp target map(([b[I]][bf])f)  // lt50-error {{expected ',' or ']' in lambda capture list}} lt50-error {{expected ')'}} lt50-note {{to match this '('}}
     {}
@@ -476,7 +485,7 @@ void SAclient(int arg) {
   {}
 #pragma omp target map(r.S.Arr[:12])
   {}
-#pragma omp target map(r.S.foo() [:12]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
+#pragma omp target map(r.S.foo() [:12]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}} ge60-error {{expected addressable lvalue in 'map' clause}}
   {}
 #pragma omp target map(r.C, r.D)
   {}
@@ -673,7 +682,7 @@ T tmain(T argc) {
   foo();
 #pragma omp target map(T) // expected-error {{'T' does not refer to a value}}
   foo();
-#pragma omp target map(I) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}}
+#pragma omp target map(I) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}} ge60-error 2 {{expected addressable lvalue in 'map' clause}}
   foo();
 #pragma omp target map(S2::S2s)
   foo();
@@ -690,7 +699,7 @@ T tmain(T argc) {
 #pragma omp target map(to, x)
   foo();
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? x : y) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}}
+#pragma omp target data map(tofrom: argc > 0 ? x : y) // lt50-error 2 {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error 2 {{expected addressable lvalue in 'map' clause}} ge60-error 2 {{expected addressable lvalue in 'map' clause}}
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
 #pragma omp target data map(a, b, c, d, f) // expected-error {{incomplete type 'S1' where a complete type is required}} warn-warning 2 {{type 'const S2' is not trivially copyable and not guaranteed to be mapped correctly}} warn-warning 2 {{type 'const S3' is not trivially copyable and not guaranteed to be mapped correctly}}
@@ -809,7 +818,11 @@ int main(int argc, char **argv) {
   SC1 s;
   SC1 *p;
   int Arr[10];
-#pragma omp target data map // expected-error {{expected '(' after 'map'}} lt50-error {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}} ge50-error {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
+// expected-error@+4 {{expected '(' after 'map'}}
+// lt50-error@+3 {{expected at least one 'map' or 'use_device_ptr' clause for '#pragma omp target data'}}
+// omp5x-error@+2 {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target data'}}
+// ge60-error@+1 {{expected at least one 'map', 'use_device_ptr', or 'use_device_addr' clause for '#pragma omp target_data'}}
+#pragma omp target data map
 #pragma omp target data map( // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected expression}}
 #pragma omp target data map() // expected-error {{expected expression}}
 #pragma omp target data map(alloc) // expected-error {{use of undeclared identifier 'alloc'}}
@@ -828,7 +841,7 @@ int main(int argc, char **argv) {
 #pragma omp target map(to, x)
   foo();
 #pragma omp target data map(to x) // expected-error {{expected ',' or ')' in 'map' clause}}
-#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}}
+#pragma omp target data map(tofrom: argc > 0 ? argv[1] : argv[2]) // lt50-error {{expected expression containing only member accesses and/or array sections based on named variables}} ge50-error {{expected addressable lvalue in 'map' clause}} ge60-error {{expected addressable lvalue in 'map' clause}}
 #pragma omp target data map(argc)
 #pragma omp target data map(S1) // expected-error {{'S1' does not refer to a value}}
 #pragma omp target data map(a, b, c, d, f) // expected-error {{incomplete type 'S1' where a complete type is required}} warn-warning {{type 'const S2' is not trivially copyable and not guaranteed to be mapped correctly}} warn-warning {{type 'const S3' is not trivially copyable and not guaranteed to be mapped correctly}}
diff --git a/clang/test/Options/Gis.hlsl b/clang/test/Options/Gis.hlsl
new file mode 100644
index 000000000000..a124b7807eaa
--- /dev/null
+++ b/clang/test/Options/Gis.hlsl
@@ -0,0 +1,13 @@
+// RUN: %clang_dxc -T lib_6_4 -Gis %s 2>&1 -### | FileCheck -check-prefix=Gis %s
+// RUN: %clang_dxc -T lib_6_4 %s 2>&1 -### | FileCheck -check-prefix=NO_Gis %s
+
+// Gis: "-ffp-contract=off" "-frounding-math" "-ffp-exception-behavior=strict" "-complex-range=full"
+// assert expected floating point options are present
+// NO_Gis-NOT: "-ffp-contract=off" 
+// NO_Gis-NOT: "-frounding-math" 
+// NO_Gis-NOT: "-ffp-exception-behavior=strict" 
+// NO_Gis-NOT: "-complex-range=full"
+float4 main(float4 a : A) : SV_TARGET
+{
+  return -a.yxxx;
+}
diff --git a/clang/test/Parser/c23-typeof.m b/clang/test/Parser/c23-typeof.m
new file mode 100644
index 000000000000..e6a8dc918dbb
--- /dev/null
+++ b/clang/test/Parser/c23-typeof.m
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -verify -std=c23 -fblocks -Wno-unused %s
+// expected-no-diagnostics
+
+void f() {
+  ^ typeof((void)0) {}; // Ok
+  ^ typeof(void) {};    // Ok
+}
diff --git a/clang/test/Parser/c2x-typeof.c b/clang/test/Parser/c2x-typeof.c
index 9c836dfa6d82..245c127d71e0 100644
--- a/clang/test/Parser/c2x-typeof.c
+++ b/clang/test/Parser/c2x-typeof.c
@@ -42,3 +42,13 @@ _Static_assert(__builtin_offsetof(typeof(s), i) == 0);
 _Static_assert(__builtin_offsetof(typeof_unqual(struct S), i) == 0);
 _Static_assert(__builtin_offsetof(typeof_unqual(s), i) == 0);
 
+// Show that typeof and typeof_unqual can be used in the underlying type of an
+// enumeration even when given the type form. Note, this can look like a
+// compound literal expression, which caused GH146351.
+enum E3 : typeof(int) { ThirdZero }; // (int) {}; is not a compound literal!
+enum E4 : typeof_unqual(int) { FourthZero }; // Same here
+
+// Ensure that this invalid construct is diagnosed instead of being treated
+// as typeof((int){ 0 }).
+typeof(int) { 0 } x; // expected-error {{a type specifier is required for all declarations}} \
+                        expected-error {{expected identifier or '('}}
diff --git a/clang/test/ParserOpenACC/parse-cache-construct.c b/clang/test/ParserOpenACC/parse-cache-construct.c
index d52a5e4ea344..76d6b32a574e 100644
--- a/clang/test/ParserOpenACC/parse-cache-construct.c
+++ b/clang/test/ParserOpenACC/parse-cache-construct.c
@@ -46,10 +46,11 @@ void func() {
     #pragma acc cache(invalid
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected ')'}}
     // expected-note@+2{{to match this '('}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(ArrayPtr
   }
 
@@ -58,11 +59,13 @@ void func() {
     #pragma acc cache(invalid)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+5{{expected expression}}
     // expected-error@+4{{expected ']'}}
@@ -72,6 +75,7 @@ void func() {
     #pragma acc cache(ArrayPtr[
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
@@ -79,6 +83,7 @@ void func() {
     #pragma acc cache(ArrayPtr[, 5)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
@@ -86,10 +91,12 @@ void func() {
     #pragma acc cache(Array[)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Array[*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+5{{expected expression}}
     // expected-error@+4{{expected ']'}}
@@ -99,49 +106,59 @@ void func() {
     #pragma acc cache(Array[*readonly:
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{invalid tag 'devnum' on 'cache' directive}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(devnum:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{invalid tag 'invalid' on 'cache' directive}}
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(invalid:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:ArrayPtr)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:1])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+    // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:ArrayPtr[5:*readonly], Array)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly], Array[*readonly:3])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-    #pragma acc cache(readonly:ArrayPtr[5 + i:*readonly], Array[*readonly + i:3])
+    #pragma acc cache(readonly:ArrayPtr[5 + 0:*readonly], Array[*readonly + 0:3])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ')'}}
@@ -149,26 +166,31 @@ void func() {
     #pragma acc cache(readonly:ArrayPtr[5:*readonly],
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{expected expression}}
     #pragma acc cache(readonly:ArrayPtr[5:*readonly],)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:ArrayPtr[5,6:*readonly])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:ArrayPtr[5:3, *readonly], ArrayPtr[0])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
     #pragma acc cache(readonly:s.foo)
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
     #pragma acc cache(readonly:s.Array[1,2])
diff --git a/clang/test/ParserOpenACC/parse-cache-construct.cpp b/clang/test/ParserOpenACC/parse-cache-construct.cpp
index 948f2e30f149..fe794707bb7d 100644
--- a/clang/test/ParserOpenACC/parse-cache-construct.cpp
+++ b/clang/test/ParserOpenACC/parse-cache-construct.cpp
@@ -8,28 +8,34 @@ char *getArrayPtr();
 template<typename T, int I>
 void func() {
   char *ArrayPtr = getArrayPtr();
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-warning@+1{{left operand of comma operator has no effect}}
-    #pragma acc cache(ArrayPtr[T::value + I:I + 3], T::array[(i + T::value, 2): 4])
+    #pragma acc cache(ArrayPtr[T::value + I:I + 3], T::array[(T::value, 2): 2])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(NS::NSArray[NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(NS::NSArray[NS::NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSArray'}}
     #pragma acc cache(NSArray[NS::NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NSInt : NS::NSInt])
   }
 
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{use of undeclared identifier 'NSInt'}}
     #pragma acc cache(NS::NSArray[NS::NSInt : NSInt])
@@ -53,43 +59,53 @@ struct HasMembersArray {
 void use() {
 
   Members s;
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(s.array[s.value])
   }
   HasMembersArray Arrs;
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Arrs.MemArr[3].array[4])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     #pragma acc cache(Arrs.MemArr[3].array[1:4])
   }
+#pragma acc loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[2:1].array[1:4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[2:1].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+2{{expected ']'}}
     // expected-note@+1{{to match this '['}}
     #pragma acc cache(Arrs.MemArr[3:4:].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[:].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{expected unqualified-id}}
     #pragma acc cache(Arrs.MemArr[::].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+3{{expected expression}}
     // expected-error@+2{{expected ']'}}
     // expected-note@+1{{to match this '['}}
     #pragma acc cache(Arrs.MemArr[: :].array[4])
   }
+#pragma acc parallel loop
   for (int i = 0; i < 10; ++i) {
     // expected-error@+1{{OpenACC sub-array is not allowed here}}
     #pragma acc cache(Arrs.MemArr[3:].array[4])
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index fd83e4b689a2..d7201207db57 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -316,7 +316,7 @@
 
 // ================== Check whether -mtune accepts mixed-case features.
 // RUN: %clang -target aarch64 -mtune=CYCLONE -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MTUNE-CYCLONE %s
-// CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a"
+// CHECK-MTUNE-CYCLONE: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a"
 
 // RUN: %clang -target aarch64 -mcpu=apple-a7 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
 // RUN: %clang -target aarch64 -mcpu=apple-a8 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-APPLE-A7 %s
@@ -342,12 +342,12 @@
 // RUN: %clang -target aarch64 -mcpu=thunderx2t99 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-THUNDERX2T99 %s
 // RUN: %clang -target aarch64 -mcpu=a64fx -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-A64FX %s
 // RUN: %clang -target aarch64 -mcpu=carmel -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MCPU-CARMEL %s
-// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
-// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}}"-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A7: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A10: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+lor" "-target-feature" "+neon" "-target-feature" "+pan" "-target-feature" "+perfmon" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+vh"
+// CHECK-MCPU-APPLE-A11: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"{{.*}} "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-MCPU-APPLE-A12: "-cc1"{{.*}} "-triple" "aarch64"{{.*}} "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
 // CHECK-MCPU-A34: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
-// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
+// CHECK-MCPU-APPLE-A13: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2" "-target-feature" "+sha3"
 // CHECK-MCPU-A35: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
 // CHECK-MCPU-A53: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
 // CHECK-MCPU-A57: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+perfmon" "-target-feature" "+sha2"
@@ -362,10 +362,10 @@
 // CHECK-MCPU-CARMEL: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8.2a" "-target-feature" "+aes" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+sha2"
 
 // RUN: %clang -target x86_64-apple-macosx -arch arm64 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64 %s
-// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+ccpp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
+// CHECK-ARCH-ARM64: "-target-cpu" "apple-m1" "-target-feature" "+v8.4a" "-target-feature" "+aes" "-target-feature" "+altnzcv" "-target-feature" "+ccdp" "-target-feature" "+ccpp" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+flagm" "-target-feature" "+fp-armv8" "-target-feature" "+fp16fml" "-target-feature" "+fptoint" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+predres" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sb" "-target-feature" "+sha2" "-target-feature" "+sha3" "-target-feature" "+specrestrict" "-target-feature" "+ssbs"
 
 // RUN: %clang -target x86_64-apple-macosx -arch arm64_32 -### -c %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH-ARM64_32 %s
-// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+zcm" "-target-feature" "+zcz" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
+// CHECK-ARCH-ARM64_32: "-target-cpu" "apple-s4" "-target-feature" "+v8.3a" "-target-feature" "+aes" "-target-feature" "+complxnum" "-target-feature" "+crc" "-target-feature" "+fp-armv8" "-target-feature" "+fullfp16" "-target-feature" "+jsconv" "-target-feature" "+lse" "-target-feature" "+neon" "-target-feature" "+pauth" "-target-feature" "+perfmon" "-target-feature" "+ras" "-target-feature" "+rcpc" "-target-feature" "+rdm" "-target-feature" "+sha2"
 
 // RUN: %clang -target aarch64 -march=armv8-a+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
 // RUN: %clang -target aarch64 -march=armv8-a+nofp+nosimd+nocrc+nocrypto+fp+simd+crc+crypto -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-MARCH-1 %s
diff --git a/clang/test/Preprocessor/arm-target-features.c b/clang/test/Preprocessor/arm-target-features.c
index 0f9063a27371..fc37bebcb833 100644
--- a/clang/test/Preprocessor/arm-target-features.c
+++ b/clang/test/Preprocessor/arm-target-features.c
@@ -1049,3 +1049,8 @@
 // CHECK-NOSIMD-NOT: #define __ARM_NEON 1
 // CHECK-NOSIMD-NOT: #define __ARM_NEON_FP 0x6
 // CHECK-NOSIMD-NOT: #define __ARM_NEON__ 1
+
+// Check that disabling SVE, as set by armv9-a, does not define __ARM_FEATURE_SVE* macros.
+// RUN:  %clang -target aarch64 -march=armv9-a+nosve -x c -E -dM -o - %s | FileCheck -check-prefix=CHECK-NOSVE %s
+// CHECK-NOSVE-NOT: #define __ARM_FEATURE_SVE 1
+// CHECK-NOSVE-NOT: #define __ARM_FEATURE_SVE2 1
diff --git a/clang/test/Preprocessor/macro_paste_identifier_ucn.c b/clang/test/Preprocessor/macro_paste_identifier_ucn.c
new file mode 100644
index 000000000000..c9eb8190edfe
--- /dev/null
+++ b/clang/test/Preprocessor/macro_paste_identifier_ucn.c
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -fms-extensions %s -verify
+// RUN: %clang_cc1 -E -fms-extensions %s | FileCheck %s
+// expected-no-diagnostics
+
+#define CAT(a,b) a##b
+
+char foo\u00b5;
+char*p = &CAT(foo, \u00b5);
+// CHECK: char fooµ;
+// CHECK-NEXT: char*p = &fooµ;
diff --git a/clang/test/Preprocessor/riscv-target-features-andes.c b/clang/test/Preprocessor/riscv-target-features-andes.c
index c66d4427b5cf..083deb02a267 100644
--- a/clang/test/Preprocessor/riscv-target-features-andes.c
+++ b/clang/test/Preprocessor/riscv-target-features-andes.c
@@ -4,6 +4,8 @@
 // RUN:   -o - | FileCheck %s
 
 // CHECK-NOT: __riscv_xandesperf {{.*$}}
+// CHECK-NOT: __riscv_xandesvbfhcvt {{.*$}}
+// CHECK-NOT: __riscv_xandesvsintload {{.*$}}
 // CHECK-NOT: __riscv_xandesvpackfph {{.*$}}
 // CHECK-NOT: __riscv_xandesvdot {{.*$}}
 
@@ -23,6 +25,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVBFHCVT %s
 // CHECK-XANDESVBFHCVT: __riscv_xandesvbfhcvt  5000000{{$}}
 
+// RUN: %clang --target=riscv32 \
+// RUN:   -march=rv32i_xandesvsintload -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVSINTLOAD %s
+// RUN: %clang --target=riscv64 \
+// RUN:   -march=rv64i_xandesvsintload -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVSINTLOAD %s
+// CHECK-XANDESVSINTLOAD: __riscv_xandesvsintload  5000000{{$}}
+
 // RUN: %clang --target=riscv32 \
 // RUN:   -march=rv32i_xandesvpackfph -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-XANDESVPACKFPH %s
diff --git a/clang/test/Sema/Float16.c b/clang/test/Sema/Float16.c
index 9269a0ded4eb..2830213b6099 100644
--- a/clang/test/Sema/Float16.c
+++ b/clang/test/Sema/Float16.c
@@ -6,6 +6,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -triple aarch64-linux-gnu %s -DHAVE
 // RUN: %clang_cc1 -fsyntax-only -verify -triple riscv32 %s -DHAVE
 // RUN: %clang_cc1 -fsyntax-only -verify -triple riscv64 %s -DHAVE
+// RUN: %clang_cc1 -fsyntax-only -verify -triple s390x-ibm-zos %s
 
 #ifndef HAVE
 // expected-error@+2{{_Float16 is not supported on this target}}
diff --git a/clang/test/Sema/attr-arm-sve-vector-bits.c b/clang/test/Sema/attr-arm-sve-vector-bits.c
index d34b13a74df6..447addb4d5d3 100644
--- a/clang/test/Sema/attr-arm-sve-vector-bits.c
+++ b/clang/test/Sema/attr-arm-sve-vector-bits.c
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=1 -mvscale-max=1 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=2 -mvscale-max=2 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=4 -mvscale-max=4 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=8 -mvscale-max=8 %s
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -ffreestanding -fsyntax-only -verify -mvscale-min=16 -mvscale-max=16 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=1 -mvscale-max=1 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=2 -mvscale-max=2 -mvscale-streaming-min=2 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected -mvscale-min=4 -mvscale-max=4 -mvscale-streaming-min=4 -mvscale-streaming-max=4 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=8 -mvscale-max=8 -mvscale-streaming-min=4 -mvscale-streaming-max=8 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +bf16 -target-feature +sme -ffreestanding -fsyntax-only -verify=expected,streamingdifferent -mvscale-min=16 -mvscale-max=16 %s
 
 #include <stdint.h>
 
@@ -382,3 +382,24 @@ TEST_INT_OPS(fixed_uint64_t)
 TEST_OPS(fixed_float16_t)
 TEST_OPS(fixed_float32_t)
 TEST_OPS(fixed_float64_t)
+
+// --------------------------------------------------------------------------//
+// Streaming
+__arm_locally_streaming void locally_streaming() {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming function}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming function}}
+  void* t3 = extern_int8_ptr;
+}
+void streaming(void) __arm_streaming {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming function}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming function}}
+  void* t3 = extern_int8_ptr;
+}
+void streaming_compatible(void) __arm_streaming_compatible {
+  svint8_t t1 = extern_int8; // streamingdifferent-error {{cannot be used in a streaming-compatible function}} \
+                             // streamingdifferent-error {{initializing}}
+  svbool_t t2 = extern_bool; // streamingdifferent-error {{cannot be used in a streaming-compatible function}} \
+                             // streamingdifferent-error {{initializing}}
+  void* t3 = extern_int8_ptr;
+}
+__arm_locally_streaming void locally_streaming_arg(fixed_int8_t x) {} // streamingdifferent-error {{cannot be used in a streaming function}}
diff --git a/clang/test/CodeGen/attr-cpuspecific-cpus.c b/clang/test/Sema/attr-cpuspecific-cpus.c
similarity index 97%
rename from clang/test/CodeGen/attr-cpuspecific-cpus.c
rename to clang/test/Sema/attr-cpuspecific-cpus.c
index 1dd095ec9e19..48543ac30da8 100644
--- a/clang/test/CodeGen/attr-cpuspecific-cpus.c
+++ b/clang/test/Sema/attr-cpuspecific-cpus.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -verify -triple x86_64-linux-gnu -emit-llvm -o - %s
-// RUN: %clang_cc1 -verify -triple x86_64-windows-pc -fms-compatibility -emit-llvm -o - %s
+// RUN: %clang_cc1 -verify -triple x86_64-linux-gnu -fsyntax-only %s
+// RUN: %clang_cc1 -verify -triple x86_64-windows-pc -fms-compatibility -fsyntax-only  %s
 
 // expected-no-diagnostics
 
diff --git a/clang/test/Sema/attr-nonstring_safe.c b/clang/test/Sema/attr-nonstring_safe.c
new file mode 100644
index 000000000000..708308e38cab
--- /dev/null
+++ b/clang/test/Sema/attr-nonstring_safe.c
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -fsyntax-only -verify=compat,expected -Wunterminated-string-initialization %s -x c
+// RUN: %clang_cc1 -fsyntax-only -verify=cxx -Wunterminated-string-initialization %s -x c++
+// REQUIRES: !system-linux
+
+#ifndef __cplusplus
+typedef __CHAR16_TYPE__ char16_t;
+typedef __CHAR32_TYPE__ char32_t;
+typedef __WCHAR_TYPE__  wchar_t;
+#endif
+
+// C++ is stricter so the following cases should be warned about. In
+// C, the following examples are fine.
+
+char foo3[3] = "fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}}
+char foo1[1] = "\0";   // cxx-error {{initializer-string for char array is too long, array size is 1 but initializer has size 2 (including the null terminating character)}}
+
+struct S {
+  char buf[3];
+  char fub[3];
+} s = { "ba\0", "bo\0" }; // cxx-error 2{{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}}
+
+#pragma clang diagnostic push
+#pragma clang diagnostic warning "-Wc++-compat"
+// Test different encodings:
+signed char scfoo[3] = "fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			          compat-warning {{initializer-string for character array is too long for C++, array size is 3 but initializer has size 4 (including the null terminating character)}}
+unsigned char ucfoo[3] = "fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+				    compat-warning {{initializer-string for character array is too long for C++, array size is 3 but initializer has size 4 (including the null terminating character)}}
+wchar_t wcfoo[3] = L"fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			       compat-warning {{initializer-string for character array is too long for C++, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			       compat-warning {{identifier 'wchar_t' conflicts with a C++ keyword}}
+char16_t c16foo[3] = u"fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			         compat-warning {{initializer-string for character array is too long for C++, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			         compat-warning {{identifier 'char16_t' conflicts with a C++ keyword}}
+char32_t c32foo[3] = U"fo\0"; // cxx-error {{initializer-string for char array is too long, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			         compat-warning {{initializer-string for character array is too long for C++, array size is 3 but initializer has size 4 (including the null terminating character)}} \
+			         compat-warning {{identifier 'char32_t' conflicts with a C++ keyword}}
+#pragma clang diagnostic pop
+
+// Test list initializer:
+signed char scfoo_lst[3] = {'f', 'o', '\0'};
+unsigned char ucfoo_lst[3] = {'f', 'o', '\0'};
+wchar_t wcfoo_lst[3] = {L'f', L'o', L'\0'};
+char16_t c16foo_lst[3] = {u'f', u'o', u'\0'};
+char32_t c32foo_lst[3] = {U'f', U'o', U'\0'};
+
+// Declaring an array of size 0 is invalid by C standard but compilers
+// may allow it:
+char a[0] = ""; // expected-warning {{initializer-string for character array is too long, array size is 0 but initializer has size 1 (including the null terminating character); did you mean to use the 'nonstring' attribute?}} \
+		   cxx-error {{initializer-string for char array is too long, array size is 0 but initializer has size 1 (including the null terminating character)}}
+char b[1] = ""; // no warn
diff --git a/clang/test/Sema/attr-visibility.c b/clang/test/Sema/attr-visibility.c
index 4acca7a7f69a..0497e9760c44 100644
--- a/clang/test/Sema/attr-visibility.c
+++ b/clang/test/Sema/attr-visibility.c
@@ -25,3 +25,9 @@ typedef int __attribute__((visibility("default"))) bar; // expected-warning {{'v
 int x __attribute__((type_visibility("default"))); // expected-error {{'type_visibility' attribute only applies to types and namespaces}}
 
 int PR17105 __attribute__((visibility(hidden))); // expected-error {{'visibility' attribute requires a string}}
+
+#pragma clang attribute push([[gnu::visibility("default")]], apply_to=function)
+
+void func(void) {}
+
+#pragma clang attribute pop
diff --git a/clang/test/Sema/builtins-bcd-transform.c b/clang/test/Sema/builtins-bcd-transform.c
new file mode 100644
index 000000000000..103a6be6452b
--- /dev/null
+++ b/clang/test/Sema/builtins-bcd-transform.c
@@ -0,0 +1,30 @@
+// Testfile to verify the semantics and the error handling for BCD builtins national2packed, packed2zoned and zoned2packed.
+// REQUIRES: powerpc-registered-target
+// RUN: %clang_cc1 -target-feature +altivec -triple powerpc64-unknown-unknown -fsyntax-only -verify %s
+// RUN: %clang_cc1 -target-feature +altivec -triple powerpc64le-unknown-unknown -fsyntax-only -verify %s
+// RUN: %clang_cc1 -target-feature +altivec -triple powerpc-unknown-unknown -fsyntax-only -verify %s
+
+#include <altivec.h>
+vector unsigned char test_national2packed(void)
+{
+  vector unsigned char a = {1,2,3,4};
+  vector unsigned char res_a = __builtin_ppc_national2packed(a, 2);  // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  vector unsigned char res_b = __builtin_ppc_national2packed(a, -1); // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  return __builtin_ppc_national2packed(a, 0);
+}
+
+vector unsigned char test_packed2zoned(void)
+{
+  vector unsigned char a = {1,2,3,4};
+  vector unsigned char res_a = __builtin_ppc_packed2zoned(a,2);  // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  vector unsigned char res_b = __builtin_ppc_packed2zoned(a, -1); // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  return __builtin_ppc_packed2zoned(a,1);
+}
+
+vector unsigned char test_zoned2packed(void)
+{
+  vector unsigned char a = {1,2,3,4};
+  vector unsigned char res_a = __builtin_ppc_zoned2packed(a,2);  // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  vector unsigned char res_b = __builtin_ppc_zoned2packed(a, -1); // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  return __builtin_ppc_zoned2packed(a,0);
+}
\ No newline at end of file
diff --git a/clang/test/Sema/builtins-microsoft-arm64.c b/clang/test/Sema/builtins-microsoft-arm64.c
index 322cf7542f43..a915a370cde1 100644
--- a/clang/test/Sema/builtins-microsoft-arm64.c
+++ b/clang/test/Sema/builtins-microsoft-arm64.c
@@ -24,3 +24,12 @@ void check_ReadWriteStatusReg(int v) {
   _ReadStatusReg(x); // expected-error {{argument to '_ReadStatusReg' must be a constant integer}}
   _WriteStatusReg(x, v); // expected-error {{argument to '_WriteStatusReg' must be a constant integer}}
 }
+
+void check__sys(int v) {
+  int x;
+  __sys(x, v); // expected-error {{argument to '__sys' must be a constant integer}}
+}
+
+unsigned int check__sys_retval() {
+  return __sys(0, 1); // builtin has superfluous return value for MSVC compatibility
+}
diff --git a/clang/test/Sema/deep_recursion.c b/clang/test/Sema/deep_recursion.c
new file mode 100644
index 000000000000..866ad8f45c91
--- /dev/null
+++ b/clang/test/Sema/deep_recursion.c
@@ -0,0 +1,209 @@
+// RUN: %clang_cc1 -verify -Wno-unused -I %S/Inputs %s
+// expected-no-diagnostics
+void func(void) {
+  (void)0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,\
+  5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,\
+  3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,\
+  1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,\
+  9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,\
+  7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0,1,2,3,4;
+}
diff --git a/clang/test/Sema/implicit-int-conversion-on-int.c b/clang/test/Sema/implicit-int-conversion-on-int.c
index f555893d9e17..3b1d81cd6a9a 100644
--- a/clang/test/Sema/implicit-int-conversion-on-int.c
+++ b/clang/test/Sema/implicit-int-conversion-on-int.c
@@ -48,3 +48,11 @@ unsigned long long test_ull(unsigned long long x) {
 unsigned _BitInt(16) test_unsigned_bit_int(unsigned _BitInt(16) x) {
   return -x;
 }
+
+unsigned test_shift_minus(int i) {
+  return -(1 << i);
+}
+
+unsigned test_shift_not(int i) {
+  return ~(1 << i);
+}
diff --git a/clang/test/Sema/offsetof-64.c b/clang/test/Sema/offsetof-64.c
index 8ffc3af98588..692698fe39e0 100644
--- a/clang/test/Sema/offsetof-64.c
+++ b/clang/test/Sema/offsetof-64.c
@@ -2,7 +2,7 @@
 
 // PR15216
 // Don't crash when taking computing the offset of structs with large arrays.
-const unsigned long Size = (1l << 60);
+const unsigned long Size = (1l << 58);
 
 struct Chunk1 {
   char padding[Size]; // expected-warning {{folded to constant}}
@@ -10,7 +10,7 @@ struct Chunk1 {
   char data;
 };
 
-int test1 = __builtin_offsetof(struct Chunk1, data);
+unsigned long test1 = __builtin_offsetof(struct Chunk1, data);
 
 struct Chunk2 {
   char padding[Size][Size][Size];  // expected-error {{array is too large}}
diff --git a/clang/test/Sema/rvv-andes-required-features-invalid.c b/clang/test/Sema/rvv-andes-required-features-invalid.c
new file mode 100644
index 000000000000..cd59322225e0
--- /dev/null
+++ b/clang/test/Sema/rvv-andes-required-features-invalid.c
@@ -0,0 +1,20 @@
+// REQUIRES: riscv-registered-target
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v %s -fsyntax-only -verify
+
+#include <andes_vector.h>
+
+vfloat32mf2_t test_nds_vfwcvt_s_bf16_f32mf2(vbfloat16mf4_t vs2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vbfloat16mf4_t' (aka '__rvv_bfloat16mf4_t') requires the 'zvfbfmin or xandesvbfhcvt' extension}}
+  return __riscv_nds_vfwcvt_s_bf16_f32mf2(vs2, vl); // expected-error {{RISC-V type '__rvv_bfloat16mf4_t' requires the 'zvfbfmin or xandesvbfhcvt' extension}}
+  // expected-error@-1 {{RISC-V type 'vbfloat16mf4_t' (aka '__rvv_bfloat16mf4_t') requires the 'zvfbfmin or xandesvbfhcvt' extension}}
+  // expected-error@-2 {{builtin requires at least one of the following extensions: xandesvbfhcvt}}
+}
+
+vfloat16mf4_t test_nds_vfpmadb_vf_f16mf4(vfloat16mf4_t op1, float op2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-2 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  return __riscv_nds_vfpmadb_vf_f16mf4(op1, op2, vl); // expected-error {{RISC-V type '__rvv_float16mf4_t' requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-1 {{RISC-V type '__rvv_float16mf4_t' requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-2 {{RISC-V type 'vfloat16mf4_t' (aka '__rvv_float16mf4_t') requires the 'zvfh, zvfhmin or xandesvpackfph' extension}}
+  // expected-error@-3 {{builtin requires at least one of the following extensions: xandesvpackfph}}
+}
diff --git a/clang/test/Sema/rvv-required-features-invalid.c b/clang/test/Sema/rvv-required-features-invalid.c
index c96d0e158062..1e08e4e9e4b6 100644
--- a/clang/test/Sema/rvv-required-features-invalid.c
+++ b/clang/test/Sema/rvv-required-features-invalid.c
@@ -48,3 +48,10 @@ void test_zvfbfwma_vfwmaccbf16(vfloat32m4_t vd, __bf16 vs1, vbfloat16m2_t vs2, s
   // expected-error@-1 {{RISC-V type '__rvv_bfloat16m2_t' requires the 'zvfbfmin' extension}}
   // expected-error@-2 {{builtin requires at least one of the following extensions: zvfbfwma}}
 }
+
+void test_zvfbfmin_vfwcvtbf16(vbfloat16m2_t vs2, size_t vl) {
+  // expected-error@-1 {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  __riscv_vfwcvtbf16_f_f_v_f32m4(vs2, vl);; // expected-error {{RISC-V type 'vbfloat16m2_t' (aka '__rvv_bfloat16m2_t') requires the 'zvfbfmin' extension}}
+  // expected-error@-1 {{RISC-V type '__rvv_bfloat16m2_t' requires the 'zvfbfmin' extension}}
+  // expected-error@-2 {{builtin requires at least one of the following extensions: zvfbfmin}}
+}
diff --git a/clang/test/Sema/warn-documentation.cpp b/clang/test/Sema/warn-documentation.cpp
index 4839bdff6179..0d1faa1b562f 100644
--- a/clang/test/Sema/warn-documentation.cpp
+++ b/clang/test/Sema/warn-documentation.cpp
@@ -1524,3 +1524,15 @@ F &f = FF; ///< \return none
 // expected-warning@-1 {{'\return' command used in a comment that is not attached to a function or method declaration}}
 
 } // namespace PR42844
+
+#if __cplusplus >= 201402L
+namespace GH144775 {
+/// @brief primary template
+/// @tparam T type
+template <class T, class = void> constexpr auto var = T{};
+
+/// @brief variable template partial specialization
+/// @tparam T type
+template <typename T> constexpr auto var<T> = T{};
+} // namespace GH144775
+#endif
diff --git a/clang/test/Sema/warn-format-overflow-truncation.c b/clang/test/Sema/warn-format-overflow-truncation.c
index c64a1ed8aaa0..eb1feaae834e 100644
--- a/clang/test/Sema/warn-format-overflow-truncation.c
+++ b/clang/test/Sema/warn-format-overflow-truncation.c
@@ -43,6 +43,13 @@ void call_snprintf(double d, int n, int *ptr) {
   __builtin_snprintf(node_name, sizeof(node_name), "%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
   __builtin_snprintf(node_name, sizeof(node_name), "12345%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 12}}
   __builtin_snprintf(node_name, sizeof(node_name), "123456%pOFn", ptr); // nonkprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 13}}
+  __builtin_snprintf(buf, 6, "%5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%+5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "% 5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%   5.1f", 9.f);
+  __builtin_snprintf(buf, 6, "%+6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
+  __builtin_snprintf(buf, 6, "% 6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
+  __builtin_snprintf(buf, 6, "%   6.1f", 9.f); // kprintf-warning {{'snprintf' will always be truncated; specified size is 6, but format string expands to at least 7}}
 }
 
 void call_vsnprintf(void) {
@@ -153,4 +160,11 @@ void call_sprintf(void) {
   sprintf(buf, "%+.3f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
   sprintf(buf, "%.0e", 9.f);
   sprintf(buf, "5%.1e", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 8}}
+  sprintf(buf, "%5.1f", 9.f);
+  sprintf(buf, "%+5.1f", 9.f);
+  sprintf(buf, "% 5.1f", 9.f);
+  sprintf(buf, "%   5.1f", 9.f);
+  sprintf(buf, "%+6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
+  sprintf(buf, "% 6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
+  sprintf(buf, "%   6.1f", 9.f); // kprintf-warning {{'sprintf' will always overflow; destination buffer has size 6, but format string expands to at least 7}}
 }
diff --git a/clang/test/SemaCXX/Float16.cpp b/clang/test/SemaCXX/Float16.cpp
index 61b02a50687b..9646a8d0c317 100644
--- a/clang/test/SemaCXX/Float16.cpp
+++ b/clang/test/SemaCXX/Float16.cpp
@@ -4,7 +4,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -triple spir-unknown-unknown %s -DHAVE
 // RUN: %clang_cc1 -fsyntax-only -verify -triple armv7a-linux-gnu %s -DHAVE
 // RUN: %clang_cc1 -fsyntax-only -verify -triple aarch64-linux-gnu %s -DHAVE
-
+// RUN: %clang_cc1 -fsyntax-only -verify -triple s390x-ibm-zos %s
 #ifdef HAVE
 // expected-no-diagnostics
 #endif // HAVE
diff --git a/clang/test/SemaCXX/P1979.cpp b/clang/test/SemaCXX/P1979.cpp
new file mode 100644
index 000000000000..65c4b839632c
--- /dev/null
+++ b/clang/test/SemaCXX/P1979.cpp
@@ -0,0 +1,38 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/A.cpp -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface %t/myV.cpp -o %t/myV.pcm
+// RUN: %clang_cc1 -std=c++20 -verify -emit-module-interface -fmodule-file=V=%t/myV.pcm %t/partition.cpp -o %t/partition.pcm
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -verify -fmodule-file=V=%t/myV.pcm -fmodule-file=A=%t/A.pcm -fmodule-file=A:partition=%t/partition.pcm %t/interface.cpp
+
+//--- A.cpp
+// expected-no-diagnostics
+export module A;
+
+//--- myV.cpp
+// expected-no-diagnostics
+export module V;
+
+export struct myV{};
+
+//--- uses_vector.h
+// expected-no-diagnostics
+#ifndef x
+#define x
+
+import V;
+#endif
+
+//--- partition.cpp
+// expected-no-diagnostics
+module;
+#include "uses_vector.h" // textually expands to import V;
+module A:partition;
+
+//--- interface.cpp
+module A;
+import :partition;
+myV V; // expected-error {{declaration of 'myV' must be imported from module 'V' before it is required}}
+       // expected-note@myV.cpp:4 {{declaration here is not visible}}
diff --git a/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp b/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp
new file mode 100644
index 000000000000..1520e2b572ce
--- /dev/null
+++ b/clang/test/SemaCXX/aarch64-streaming-sve-vector-conversions.cpp
@@ -0,0 +1,69 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme -mvscale-min=1 -mvscale-max=1 -mvscale-streaming-min=2 -mvscale-streaming-max=2 -flax-vector-conversions=integer -ffreestanding -fsyntax-only -verify %s
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+#define SVE_BITS 128
+#define SVE_FIXED_ATTR __attribute__((arm_sve_vector_bits(SVE_BITS)))
+#define GNU_FIXED_ATTR __attribute__((vector_size(SVE_BITS / 8)))
+#define GNU_BOOL_FIXED_ATTR __attribute__((vector_size(SVE_BITS / 64)))
+#define STREAMING_BITS 256
+#define GNU_FIXED_STREAMING_ATTR __attribute__((vector_size(STREAMING_BITS / 8)))
+#define GNU_BOOL_FIXED_STREAMING_ATTR __attribute__((vector_size(STREAMING_BITS / 64)))
+
+typedef svfloat32_t sve_fixed_float32_t SVE_FIXED_ATTR;
+typedef svint32_t sve_fixed_int32_t SVE_FIXED_ATTR;
+typedef svbool_t sve_fixed_bool_t SVE_FIXED_ATTR;
+typedef float gnu_fixed_float32_t GNU_FIXED_ATTR;
+typedef int gnu_fixed_int32_t GNU_FIXED_ATTR;
+typedef int8_t gnu_fixed_bool_t GNU_BOOL_FIXED_ATTR;
+
+typedef float gnu_fixed_float32_t_streaming GNU_FIXED_STREAMING_ATTR;
+typedef int gnu_fixed_int32_t_streaming GNU_FIXED_STREAMING_ATTR;
+typedef int8_t gnu_fixed_bool_t_streaming GNU_BOOL_FIXED_STREAMING_ATTR;
+
+void sve_fixed() {
+  gnu_fixed_int32_t fi;
+  gnu_fixed_float32_t_streaming fi_wrong;
+  gnu_fixed_float32_t ff;
+  gnu_fixed_float32_t_streaming ff_wrong;
+  gnu_fixed_bool_t fb;
+  gnu_fixed_bool_t_streaming fb_wrong;
+  *(volatile svint32_t*)0 = fi;
+  *(volatile svint32_t*)0 = fi_wrong; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff;
+  *(volatile svfloat32_t*)0 = ff_wrong; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb;
+  *(volatile svbool_t*)0 = fb_wrong; // expected-error {{incompatible}}
+}
+
+__arm_locally_streaming void streaming_fixed() {
+  gnu_fixed_int32_t_streaming fi;
+  gnu_fixed_float32_t fi_wrong;
+  gnu_fixed_float32_t_streaming ff;
+  gnu_fixed_float32_t ff_wrong;
+  gnu_fixed_bool_t_streaming fb;
+  gnu_fixed_bool_t fb_wrong;
+  *(volatile svint32_t*)0 = fi;
+  *(volatile svint32_t*)0 = fi_wrong; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff;
+  *(volatile svfloat32_t*)0 = ff_wrong; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb;
+  *(volatile svbool_t*)0 = fb_wrong; // expected-error {{incompatible}}
+}
+
+void streaming_compatible() __arm_streaming_compatible {
+  gnu_fixed_int32_t fi_ns;
+  gnu_fixed_float32_t_streaming fi_s;
+  gnu_fixed_float32_t ff_ns;
+  gnu_fixed_float32_t_streaming ff_s;
+  gnu_fixed_bool_t fb_ns;
+  gnu_fixed_bool_t_streaming fb_s;
+  *(volatile svint32_t*)0 = fi_ns; // expected-error {{incompatible}}
+  *(volatile svint32_t*)0 = fi_s; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff_ns; // expected-error {{incompatible}}
+  *(volatile svfloat32_t*)0 = ff_s; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb_ns; // expected-error {{incompatible}}
+  *(volatile svbool_t*)0 = fb_s; // expected-error {{incompatible}}
+}
+
diff --git a/clang/test/SemaCXX/builtin-invoke.cpp b/clang/test/SemaCXX/builtin-invoke.cpp
new file mode 100644
index 000000000000..d7694d619c69
--- /dev/null
+++ b/clang/test/SemaCXX/builtin-invoke.cpp
@@ -0,0 +1,240 @@
+// RUN: %clang_cc1 -verify -fsyntax-only %s -std=c++23
+
+void func() { // expected-note {{'func' declared here}}
+  __builtin_invoke(); // expected-error {{too few arguments to function call, expected at least 1, have 0}}
+}
+
+void nfunc() noexcept {}
+
+struct S {};
+void argfunc(int, S) {} // expected-note {{'argfunc' declared here}}
+
+struct Callable {
+  void operator()() {}
+
+  void func() {}
+
+  int var;
+};
+
+void* malloc(decltype(sizeof(int)));
+
+template <class T>
+struct pointer_wrapper {
+  T* v;
+
+  T& operator*() {
+    return *v;
+  }
+};
+
+namespace std {
+  template <class T>
+  class reference_wrapper {
+    T* ptr;
+
+  public:
+    constexpr reference_wrapper(T& ref) : ptr(&ref) {}
+
+    constexpr T& get() { return *ptr; }
+  };
+
+  template <class T>
+  constexpr reference_wrapper<T> ref(T& v) {
+    return reference_wrapper<T>(v);
+  }
+} // namespace std
+
+struct InvalidSpecialization1 {
+  void func() {}
+
+  int var;
+};
+
+template <>
+class std::reference_wrapper<InvalidSpecialization1> {
+public:
+  reference_wrapper(InvalidSpecialization1&) {}
+};
+
+struct InvalidSpecialization2 {
+  void func() {}
+
+  int var;
+};
+
+template <>
+class std::reference_wrapper<InvalidSpecialization2> {
+public:
+  reference_wrapper(InvalidSpecialization2&) {}
+
+private:
+  InvalidSpecialization2& get(); // expected-note 2 {{declared private here}}
+};
+
+struct ExplicitObjectParam {
+  void func(this const ExplicitObjectParam& self) {}
+};
+
+struct Incomplete; // expected-note 2 {{forward declaration}}
+struct Incomplete2;
+
+void incomplete_by_val_test(Incomplete);
+
+void incomplete_test(Incomplete& incomplete) {
+  __builtin_invoke((int (Incomplete2::*)){}, incomplete); // expected-error {{incomplete type 'Incomplete' used in type trait expression}} \
+                                                             expected-error {{indirection requires pointer operand ('Incomplete' invalid)}}
+  __builtin_invoke(incomplete_test, incomplete);
+  __builtin_invoke(incomplete_by_val_test, incomplete); // expected-error {{argument type 'Incomplete' is incomplete}}
+}
+
+void call() {
+  __builtin_invoke(func);
+  __builtin_invoke(nfunc);
+  static_assert(!noexcept(__builtin_invoke(func)));
+  static_assert(noexcept(__builtin_invoke(nfunc)));
+  __builtin_invoke(func, 1); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(argfunc, 1); // expected-error {{too few arguments to function call, expected 2, have 1}}
+  __builtin_invoke(Callable{});
+  __builtin_invoke(malloc, 0);
+  __builtin_invoke(__builtin_malloc, 0); // expected-error {{builtin functions must be directly called}}
+
+  // Variadic function
+  void variadic_func(int, ...); // expected-note {{declared here}}
+
+  __builtin_invoke(variadic_func); // expected-error {{too few arguments to function call, expected at least 1, have 0}}
+  __builtin_invoke(variadic_func, 1);
+  __builtin_invoke(variadic_func, 1, 2, 3);
+
+  // static member function
+  struct StaticMember {
+    static void func(int);
+  };
+
+  __builtin_invoke(StaticMember::func, 1);
+  StaticMember sm;
+  __builtin_invoke(sm.func, 1);
+
+  // lambda
+  __builtin_invoke([] {});
+  __builtin_invoke([](int) {}, 1);
+
+  // Member function pointer
+  __builtin_invoke(&Callable::func); // expected-error {{too few arguments to function call, expected at least 2, have 1}}
+  __builtin_invoke(&Callable::func, 1); // expected-error {{indirection requires pointer operand ('int' invalid)}}
+  __builtin_invoke(&Callable::func, Callable{});
+  __builtin_invoke(&Callable::func, Callable{}, 1); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(&ExplicitObjectParam::func, ExplicitObjectParam{});
+
+  Callable c;
+  __builtin_invoke(&Callable::func, &c);
+  __builtin_invoke(&Callable::func, std::ref(c));
+  __builtin_invoke(&Callable::func, &c);
+  __builtin_invoke(&Callable::func, &c, 2); // expected-error {{too many arguments to function call, expected 0, have 1}}
+  __builtin_invoke(&Callable::func, pointer_wrapper<Callable>{&c});
+  __builtin_invoke(&Callable::func, pointer_wrapper<Callable>{&c}, 2); // expected-error {{too many arguments to function call, expected 0, have 1}}
+
+  InvalidSpecialization1 is1;
+  InvalidSpecialization2 is2;
+  __builtin_invoke(&InvalidSpecialization1::func, std::ref(is1)); // expected-error {{no member named 'get' in 'std::reference_wrapper<InvalidSpecialization1>'}}
+  __builtin_invoke(&InvalidSpecialization2::func, std::ref(is2)); // expected-error {{'get' is a private member of 'std::reference_wrapper<InvalidSpecialization2>'}}
+
+  // Member data pointer
+  __builtin_invoke(&Callable::var); // expected-error {{too few arguments to function call, expected at least 2, have 1}}
+  __builtin_invoke(&Callable::var, 1); // expected-error {{indirection requires pointer operand ('int' invalid)}}
+  (void)__builtin_invoke(&Callable::var, Callable{});
+  __builtin_invoke(&Callable::var, Callable{}, 1); // expected-error {{too many arguments to function call, expected 2, have 3}}
+
+  (void)__builtin_invoke(&Callable::var, &c);
+  (void)__builtin_invoke(&Callable::var, std::ref(c));
+  (void)__builtin_invoke(&Callable::var, &c);
+  __builtin_invoke(&Callable::var, &c, 2); // expected-error {{too many arguments to function call, expected 2, have 3}}
+  (void)__builtin_invoke(&Callable::var, pointer_wrapper<Callable>{&c});
+  __builtin_invoke(&Callable::var, pointer_wrapper<Callable>{&c}, 2); // expected-error {{too many arguments to function call, expected 2, have 3}}
+
+  __builtin_invoke(&InvalidSpecialization1::var, std::ref(is1)); // expected-error {{no member named 'get' in 'std::reference_wrapper<InvalidSpecialization1>'}}
+  (void)__builtin_invoke(&InvalidSpecialization2::var, std::ref(is2)); // expected-error {{'get' is a private member of 'std::reference_wrapper<InvalidSpecialization2>'}}
+}
+
+[[nodiscard]] int diagnose_discard();
+int no_diagnose_discard();
+
+namespace std {
+  template <class... Args>
+  auto invoke(Args&&... args) -> decltype(__builtin_invoke(args...));
+} // namespace std
+
+template <class... Args>
+concept invocable = requires(Args... args) { __builtin_invoke(args...); };
+
+static_assert(!invocable<std::reference_wrapper<InvalidSpecialization1>>);
+static_assert(!invocable<std::reference_wrapper<InvalidSpecialization2>>);
+
+void test3() {
+  __builtin_invoke(diagnose_discard); // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+  __builtin_invoke(no_diagnose_discard);
+}
+
+template <class T>
+auto test(T v) {
+  return __builtin_invoke(v);
+}
+
+auto call2() {
+  test(call);
+}
+
+template <class ClassT, class FuncT>
+void func(ClassT& c, FuncT&& func) {
+  __builtin_invoke(func, c, 1, 2, 3); // expected-error {{too many arguments to function call, expected 0, have 3}}
+}
+
+struct DependentTest {
+  void func(int, int, int);
+  void bad_func();
+};
+
+void call3() {
+  DependentTest d;
+  func(d, &DependentTest::func);
+  func(d, &DependentTest::bad_func); // expected-note {{requested here}}
+}
+
+constexpr int constexpr_func() {
+  return 42;
+}
+
+struct ConstexprTestStruct {
+  int i;
+  constexpr int func() {
+    return 55;
+  }
+};
+
+// Make sure that constant evaluation works
+static_assert([]() {
+
+  ConstexprTestStruct s;
+  if (__builtin_invoke(&ConstexprTestStruct::func, s) != 55) // [func.requires]/p1.1
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::func, std::ref(s)) != 55) // [func.requires]/p1.2
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::func, &s) != 55) // [func.requires]/p1.3
+    return false;
+
+  s.i = 22;
+  if (__builtin_invoke(&ConstexprTestStruct::i, s) != 22) // [func.requires]/p1.4
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::i, std::ref(s)) != 22) // [func.requires]/p1.5
+    return false;
+  if (__builtin_invoke(&ConstexprTestStruct::i, &s) != 22) // [func.requires]/p1.6
+    return false;
+
+  // [func.requires]/p1.7
+  if (__builtin_invoke(constexpr_func) != 42)
+    return false;
+  if (__builtin_invoke([] { return 34; }) != 34)
+    return false;
+
+  return true;
+}());
diff --git a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
index ed8cbbbfe706..c5d542717039 100644
--- a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
+++ b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
@@ -242,3 +242,67 @@ void f() {
 }
 
 }
+
+namespace GH134820 {
+struct S {
+    char* c = new char;
+    constexpr ~S() {
+        delete c;
+    }
+    int i = 0;
+};
+
+int f() {
+    if constexpr((S{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(S s; (S{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(S s; (s, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(constexpr int _ = S{}.i; true) {
+        return 1;
+    }
+    return 0;
+}
+
+template <typename T>
+int f2() {
+    if constexpr((T{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(T s; (T{}, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(T s; (s, true)) { // expected-warning{{left operand of comma operator has no effect}}
+        return 1;
+    }
+    if constexpr(constexpr int _ = T{}.i; true) {
+        return 1;
+    }
+    return 0;
+}
+
+void test() {
+  f2<S>(); // expected-note {{in instantiation}}
+}
+
+}
+namespace GH120197{
+struct NonTrivialDtor {
+  NonTrivialDtor() = default;
+  NonTrivialDtor(const NonTrivialDtor&) = default;
+  NonTrivialDtor(NonTrivialDtor&&) = default;
+  NonTrivialDtor& operator=(const NonTrivialDtor&)  = default;
+  NonTrivialDtor& operator=(NonTrivialDtor&&) = default;
+  constexpr ~NonTrivialDtor() noexcept {}
+};
+
+static_assert(((void)NonTrivialDtor{}, true)); // passes
+
+void f() {
+  if constexpr ((void)NonTrivialDtor{}, true) {}
+}
+}
diff --git a/clang/test/SemaCXX/cxx2c-binding-pack.cpp b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
index 0f10dad3937b..a8c1386ea5c1 100644
--- a/clang/test/SemaCXX/cxx2c-binding-pack.cpp
+++ b/clang/test/SemaCXX/cxx2c-binding-pack.cpp
@@ -1,10 +1,11 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++26 %s -verify
+// RUN: %clang_cc1 -fsyntax-only -std=c++26 %s -verify -fexperimental-new-constant-interpreter
 
 template <typename T>
 struct type_ { };
 
 template <typename ...T>
-auto sum(T... t) { return (t + ...); }
+constexpr auto sum(T... t) { return (t + ...); }
 
 struct my_struct {
   int a;
@@ -17,7 +18,7 @@ struct fake_tuple {
   int arr[4] = {1, 2, 3, 6};
 
   template <unsigned i>
-  int get() {
+  constexpr int& get() {
     return arr[i];
   }
 };
@@ -233,3 +234,28 @@ void g() {
 }
 
 }
+
+namespace constant_interpreter {
+using Arr = int[2];
+struct Triple { int x, y, z = 3; };
+
+constexpr int ref_to_same_obj(auto&& arg) {
+  auto& [...xs] = arg;
+  auto& [...ys] = arg;
+  (..., (xs += 2));
+  return sum(ys...);
+}
+static_assert(ref_to_same_obj(Arr{}) == 4);
+static_assert(ref_to_same_obj(fake_tuple{}) == 20);
+static_assert(ref_to_same_obj(Triple{}) == 9);
+
+constexpr int copy_obj(auto&& arg) {
+  auto& [...xs] = arg;
+  auto [...ys] = arg;
+  (..., (xs += 2));
+  return sum(ys...);
+}
+static_assert(copy_obj(Arr{}) == 0);
+static_assert(copy_obj(fake_tuple{}) == 12);
+static_assert(copy_obj(Triple{}) == 3);
+}
diff --git a/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp b/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
index 7152a5937d9b..6f4003f52593 100644
--- a/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
+++ b/clang/test/SemaCXX/cxx2c-trivially-relocatable.cpp
@@ -410,3 +410,39 @@ C& C::operator=(const C&) = default;
 static_assert (!__builtin_is_cpp_trivially_relocatable(C));
 static_assert (!__builtin_is_replaceable(C));
 }
+
+namespace GH144232 {
+
+struct E trivially_relocatable_if_eligible replaceable_if_eligible {
+  E (E &&);
+  E &operator= (E &&) = default;
+};
+
+struct F trivially_relocatable_if_eligible replaceable_if_eligible {
+  F (F &&) = default;
+  F &operator= (F &&);
+};
+
+struct G trivially_relocatable_if_eligible replaceable_if_eligible { G (G const &) = default; };
+
+struct I trivially_relocatable_if_eligible replaceable_if_eligible { I &operator= (const I &) = default; };
+
+struct J trivially_relocatable_if_eligible replaceable_if_eligible { J (J const &); };
+struct K trivially_relocatable_if_eligible replaceable_if_eligible { K (K const &); };
+
+
+
+static_assert (__builtin_is_replaceable (E));
+static_assert (__builtin_is_cpp_trivially_relocatable(E));
+static_assert (__builtin_is_replaceable (F));
+static_assert (__builtin_is_cpp_trivially_relocatable(F));
+static_assert (__builtin_is_replaceable (G));
+static_assert (__builtin_is_cpp_trivially_relocatable(G));
+static_assert (__builtin_is_replaceable (I));
+static_assert (__builtin_is_cpp_trivially_relocatable(I));
+static_assert (__builtin_is_replaceable (J));
+static_assert (__builtin_is_cpp_trivially_relocatable(J));
+static_assert (__builtin_is_replaceable (K));
+static_assert (__builtin_is_cpp_trivially_relocatable(K));
+
+}
diff --git a/clang/test/SemaCXX/modules.cppm b/clang/test/SemaCXX/modules.cppm
index 5d0d6da44a2e..4ef4146df4d9 100644
--- a/clang/test/SemaCXX/modules.cppm
+++ b/clang/test/SemaCXX/modules.cppm
@@ -41,6 +41,8 @@ struct S {
   export static int n; // expected-error {{expected member name or ';'}}
 };
 
+int main() {} // expected-warning {{'main' never has module linkage}}
+
 // FIXME: Exports of declarations without external linkage are disallowed.
 // Exports of declarations with non-external-linkage types are disallowed.
 
@@ -68,6 +70,8 @@ int n;
 //--- test3.cpp
 export module bar;
 
+extern "C++" int main() {}
+
 static int m;
 
 int n;
diff --git a/clang/test/SemaCXX/nested-name-spec.cpp b/clang/test/SemaCXX/nested-name-spec.cpp
index abeaba9d8dde..fedbb3070070 100644
--- a/clang/test/SemaCXX/nested-name-spec.cpp
+++ b/clang/test/SemaCXX/nested-name-spec.cpp
@@ -1,4 +1,11 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++98 -verify -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++98 -verify=expected,cxx98,cxx98-11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify=expected,since-cxx11,cxx98-11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++14 -verify=expected,since-cxx11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++17 -verify=expected,since-cxx11 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++20 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++23 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++26 -verify=expected,since-cxx11,since-cxx20 -fblocks %s
+
 namespace A {
   struct C {
     static int cx;
@@ -118,7 +125,7 @@ namespace E {
     };
 
     int f() {
-      return E::X; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+      return E::X; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
     }
   }
 }
@@ -170,8 +177,9 @@ void ::global_func2(int) { } // expected-warning{{extra qualification on member
 
 void N::f() { } // okay
 
-struct Y;  // expected-note{{forward declaration of 'Y'}}
-Y::foo y; // expected-error{{incomplete type 'Y' named in nested name specifier}}
+// FIXME (GH147000): duplicate diagnostics
+struct Y;  // expected-note{{forward declaration of 'Y'}} since-cxx20-note{{forward declaration of 'Y'}}
+Y::foo y; // expected-error{{incomplete type 'Y' named in nested name specifier}} since-cxx20-error{{incomplete type 'Y' named in nested name specifier}}
 
 namespace PR25156 {
 struct Y;  // expected-note{{forward declaration of 'PR25156::Y'}}
@@ -189,7 +197,9 @@ bool (foo_S::value);
 
 
 namespace somens {
-  struct a { }; // expected-note{{candidate constructor (the implicit copy constructor)}}
+  struct a { };
+  // expected-note@-1 {{candidate constructor (the implicit copy constructor)}}
+  // since-cxx11-note@-2 {{candidate constructor (the implicit move constructor)}}
 }
 
 template <typename T>
@@ -432,20 +442,20 @@ namespace PR16951 {
     };
   }
 
-  int x1 = ns::an_enumeration::ENUMERATOR; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+  int x1 = ns::an_enumeration::ENUMERATOR; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
 
-  int x2 = ns::an_enumeration::ENUMERATOR::vvv; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x2 = ns::an_enumeration::ENUMERATOR::vvv; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
                                                 // expected-error{{'ENUMERATOR' is not a class, namespace, or enumeration}} \
 
-  int x3 = ns::an_enumeration::X; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x3 = ns::an_enumeration::X; // cxx98-warning {{use of enumeration in a nested name specifier is a C++11 extension}} \
                                   // expected-error{{no member named 'X'}}
 
   enum enumerator_2 {
     ENUMERATOR_2
   };
 
-  int x4 = enumerator_2::ENUMERATOR_2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
-  int x5 = enumerator_2::X2; // expected-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
+  int x4 = enumerator_2::ENUMERATOR_2; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}}
+  int x5 = enumerator_2::X2; // cxx98-warning{{use of enumeration in a nested name specifier is a C++11 extension}} \
                              // expected-error{{no member named 'X2' in 'PR16951::enumerator_2'}} \
                              // expected-error{{cannot initialize a variable of type 'int' with an lvalue of type 'int (*)()'}}
 
@@ -487,7 +497,7 @@ struct x; // expected-note {{template is declared here}}
 
 template <typename T>
 int issue55962 = x::a; // expected-error {{use of class template 'x' requires template arguments}} \
-                       // expected-warning {{variable templates are a C++14 extension}}
+                       // cxx98-11-warning {{variable templates are a C++14 extension}}
 
 namespace ForwardDeclared {
   typedef class A B;
@@ -496,3 +506,11 @@ namespace ForwardDeclared {
     void F(B::C);
   };
 }
+
+namespace GH63254 {
+template <typename...> struct V {}; // cxx98-warning {{variadic templates are a C++11 extension}}
+struct S : V<> {
+  using S::V; // expected-error {{using declaration refers to its own class}}
+  V<> v; // no crash
+};
+}
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
index 329b611110c1..cf33ac283ab4 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
@@ -20,6 +20,28 @@ struct is_trivially_copyable {
 
 template <typename T>
 constexpr bool is_trivially_copyable_v = __is_trivially_copyable(T);
+
+template <typename T, typename U>
+struct is_assignable {
+    static constexpr bool value = __is_assignable(T, U);
+};
+
+template <typename T, typename U>
+constexpr bool is_assignable_v = __is_assignable(T, U);
+
+template <typename T>
+struct is_empty {
+    static constexpr bool value = __is_empty(T);
+};
+template <typename T>
+constexpr bool is_empty_v = __is_empty(T);
+
+template <typename T>
+struct is_standard_layout {
+static constexpr bool value = __is_standard_layout(T);
+};
+template <typename T>
+constexpr bool is_standard_layout_v = __is_standard_layout(T);
 #endif
 
 #ifdef STD2
@@ -44,6 +66,37 @@ using is_trivially_copyable  = __details_is_trivially_copyable<T>;
 
 template <typename T>
 constexpr bool is_trivially_copyable_v = __is_trivially_copyable(T);
+
+template <typename T, typename U>
+struct __details_is_assignable {
+    static constexpr bool value = __is_assignable(T, U);
+};
+
+template <typename T, typename U>
+using is_assignable = __details_is_assignable<T, U>;
+
+template <typename T, typename U>
+constexpr bool is_assignable_v = __is_assignable(T, U);
+
+template <typename T>
+struct __details_is_empty {
+    static constexpr bool value = __is_empty(T);
+};
+template <typename T>
+using is_empty  = __details_is_empty<T>;
+template <typename T>
+constexpr bool is_empty_v = __is_empty(T);
+
+template <typename T>
+struct __details_is_standard_layout {
+static constexpr bool value = __is_standard_layout(T);
+
+
+};
+template <typename T>
+using is_standard_layout = __details_is_standard_layout<T>;
+template <typename T>
+constexpr bool is_standard_layout_v = __is_standard_layout(T);
 #endif
 
 
@@ -73,6 +126,29 @@ using is_trivially_copyable  = __details_is_trivially_copyable<T>;
 
 template <typename T>
 constexpr bool is_trivially_copyable_v = is_trivially_copyable<T>::value;
+
+template <typename T, typename U>
+struct __details_is_assignable : bool_constant<__is_assignable(T, U)> {};
+
+template <typename T, typename U>
+using is_assignable  = __details_is_assignable<T, U>;
+
+template <typename T, typename U>
+constexpr bool is_assignable_v = is_assignable<T, U>::value;
+
+template <typename T>
+struct __details_is_empty : bool_constant<__is_empty(T)> {};
+template <typename T>
+using is_empty  = __details_is_empty<T>;
+template <typename T>
+constexpr bool is_empty_v = is_empty<T>::value;
+
+template <typename T>
+struct __details_is_standard_layout : bool_constant<__is_standard_layout(T)> {};
+template <typename T>
+using is_standard_layout = __details_is_standard_layout<T>;
+template <typename T>
+constexpr bool is_standard_layout_v = is_standard_layout<T>::value;
 #endif
 
 }
@@ -100,6 +176,41 @@ static_assert(std::is_trivially_copyable_v<int&>);
 // expected-note@-1 {{because it is a reference type}}
 
 
+ // Direct tests
+ static_assert(std::is_standard_layout<int>::value);
+ static_assert(std::is_standard_layout_v<int>);
+
+ static_assert(std::is_standard_layout<int&>::value);
+ // expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_standard_layout<int &>::value'}} \
+ // expected-note@-1 {{'int &' is not standard-layout}} \
+ // expected-note@-1 {{because it is a reference type}}
+
+ static_assert(std::is_standard_layout_v<int&>);
+ // expected-error@-1 {{static assertion failed due to requirement 'std::is_standard_layout_v<int &>'}} \
+ // expected-note@-1 {{'int &' is not standard-layout}} \
+ // expected-note@-1 {{because it is a reference type}}
+
+static_assert(!std::is_empty<int>::value);
+
+static_assert(std::is_empty<int&>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_empty<int &>::value'}} \
+// expected-note@-1 {{'int &' is not empty}} \
+// expected-note@-1 {{because it is a reference type}}
+static_assert(std::is_empty_v<int&>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_empty_v<int &>'}} \
+// expected-note@-1 {{'int &' is not empty}} \
+// expected-note@-1 {{because it is a reference type}}
+
+
+static_assert(std::is_assignable<int&, int>::value);
+
+static_assert(std::is_assignable<int&, void>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_assignable<int &, void>::value'}} \
+// expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
+static_assert(std::is_assignable_v<int&, void>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_assignable_v<int &, void>'}} \
+// expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
+
 namespace test_namespace {
     using namespace std;
     static_assert(is_trivially_relocatable<int&>::value);
@@ -119,6 +230,32 @@ namespace test_namespace {
     // expected-error@-1 {{static assertion failed due to requirement 'is_trivially_copyable_v<int &>'}} \
     // expected-note@-1 {{'int &' is not trivially copyable}} \
     // expected-note@-1 {{because it is a reference type}}
+
+    static_assert(is_standard_layout<int&>::value);
+     // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_standard_layout<int &>::value'}} \
+     // expected-note@-1 {{'int &' is not standard-layout}} \
+     // expected-note@-1 {{because it is a reference type}}
+
+     static_assert(is_standard_layout_v<int&>);
+     // expected-error@-1 {{static assertion failed due to requirement 'is_standard_layout_v<int &>'}} \
+     // expected-note@-1 {{'int &' is not standard-layout}} \
+     // expected-note@-1 {{because it is a reference type}}
+
+    static_assert(is_assignable<int&, void>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_assignable<int &, void>::value'}} \
+    // expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
+    static_assert(is_assignable_v<int&, void>);
+    // expected-error@-1 {{static assertion failed due to requirement 'is_assignable_v<int &, void>'}} \
+    // expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
+
+    static_assert(is_empty<int&>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_empty<int &>::value'}} \
+    // expected-note@-1 {{'int &' is not empty}} \
+    // expected-note@-1 {{because it is a reference type}} 
+    static_assert(is_empty_v<int&>);
+    // expected-error@-1 {{static assertion failed due to requirement 'is_empty_v<int &>'}} \
+    // expected-note@-1 {{'int &' is not empty}} \
+    // expected-note@-1 {{because it is a reference type}}
 }
 
 
@@ -139,6 +276,14 @@ concept C2 = std::is_trivially_copyable_v<T>; // #concept4
 
 template <C2 T> void g2();  // #cand4
 
+template <typename T, typename U>
+requires std::is_assignable<T, U>::value void f4();  // #cand7
+
+template <typename T, typename U>
+concept C4 = std::is_assignable_v<T, U>; // #concept8
+
+template <C4<void> T> void g4();  // #cand8
+
 void test() {
     f<int&>();
     // expected-error@-1 {{no matching function for call to 'f'}} \
@@ -169,6 +314,19 @@ void test() {
     // expected-note@#concept4 {{because 'std::is_trivially_copyable_v<int &>' evaluated to false}} \
     // expected-note@#concept4 {{'int &' is not trivially copyable}} \
     // expected-note@#concept4 {{because it is a reference type}}
+
+    f4<int&, void>();
+    // expected-error@-1 {{no matching function for call to 'f4'}} \
+    // expected-note@#cand7 {{candidate template ignored: constraints not satisfied [with T = int &, U = void]}} \
+    // expected-note-re@#cand7 {{because '{{.*}}is_assignable<int &, void>::value' evaluated to false}} \
+    // expected-error@#cand7 {{assigning to 'int' from incompatible type 'void'}}
+
+    g4<int&>();
+    // expected-error@-1 {{no matching function for call to 'g4'}} \
+    // expected-note@#cand8 {{candidate template ignored: constraints not satisfied [with T = int &]}} \
+    // expected-note@#cand8 {{because 'C4<int &, void>' evaluated to false}} \
+    // expected-note@#concept8 {{because 'std::is_assignable_v<int &, void>' evaluated to false}} \
+    // expected-error@#concept8 {{assigning to 'int' from incompatible type 'void'}}
 }
 }
 
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
index 5210354a66d4..cc923d206ab3 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
@@ -488,3 +488,282 @@ static_assert(__is_trivially_copyable(S12));
 // expected-note@-1 {{'S12' is not trivially copyable}} \
 // expected-note@#tc-S12 {{'S12' defined here}}
 }
+
+namespace assignable {
+struct S1;
+static_assert(__is_assignable(S1&, const S1&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(assignable::S1 &, const assignable::S1 &)'}} \
+// expected-error@-1 {{no viable overloaded '='}} \
+// expected-note@-1 {{type 'S1' is incomplete}}
+
+static_assert(__is_assignable(void, int));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(void, int)'}} \
+// expected-error@-1 {{expression is not assignable}}
+
+static_assert(__is_assignable(int, int));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(int, int)'}} \
+// expected-error@-1 {{expression is not assignable}}
+
+static_assert(__is_assignable(int*, int));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(int *, int)'}} \
+// expected-error@-1 {{expression is not assignable}}
+
+static_assert(__is_assignable(int[], int));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(int[], int)'}} \
+// expected-error@-1 {{expression is not assignable}}
+
+static_assert(__is_assignable(int&, void));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(int &, void)'}} \
+// expected-error@-1 {{assigning to 'int' from incompatible type 'void'}}
+
+static_assert(__is_assignable(int*&, float*));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(int *&, float *)'}} \
+// expected-error@-1 {{incompatible pointer types assigning to 'int *' from 'float *'}}
+
+static_assert(__is_assignable(const int&, int));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(const int &, int)'}} \
+// expected-error@-1 {{read-only variable is not assignable}}
+
+struct S2 {}; // #a-S2
+static_assert(__is_assignable(const S2, S2));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(const assignable::S2, assignable::S2)'}} \
+// expected-error@-1 {{no viable overloaded '='}} \
+// expected-note@#a-S2 {{candidate function (the implicit copy assignment operator) not viable: 'this' argument has type 'const S2', but method is not marked const}} \
+// expected-note@#a-S2 {{candidate function (the implicit move assignment operator) not viable: 'this' argument has type 'const S2', but method is not marked const}} \
+// expected-note@#a-S2 {{'S2' defined here}}
+
+struct S3 { // #a-S3
+    S3& operator=(const S3&) = delete; // #aca-S3
+    S3& operator=(S3&&) = delete;  // #ama-S3
+};
+static_assert(__is_assignable(S3, const S3&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(assignable::S3, const assignable::S3 &)'}} \
+// expected-error@-1 {{overload resolution selected deleted operator '='}} \
+// expected-note@#aca-S3 {{candidate function has been explicitly deleted}} \
+// expected-note@#ama-S3 {{candidate function not viable: 1st argument ('const S3') would lose const qualifier}} \
+// expected-note@#a-S3 {{'S3' defined here}}
+static_assert(__is_assignable(S3, S3&&));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(assignable::S3, assignable::S3 &&)'}} \
+// expected-error@-1 {{overload resolution selected deleted operator '='}} \
+// expected-note@#aca-S3 {{candidate function has been explicitly deleted}} \
+// expected-note@#ama-S3 {{candidate function has been explicitly deleted}} \
+// expected-note@#a-S3 {{'S3' defined here}}
+
+class C1 { // #a-C1
+    C1& operator=(const C1&) = default;
+    C1& operator=(C1&&) = default; // #ama-C1
+};
+static_assert(__is_assignable(C1, C1));
+// expected-error@-1 {{static assertion failed due to requirement '__is_assignable(assignable::C1, assignable::C1)'}} \
+// expected-error@-1 {{'operator=' is a private member of 'assignable::C1'}} \
+// expected-note@#ama-C1 {{implicitly declared private here}} \
+// expected-note@#a-C1 {{'C1' defined here}}
+}
+
+namespace is_empty_tests {
+    // Non-static data member.
+    struct A { int x; }; // #e-A
+    static_assert(__is_empty(A));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::A)'}} \
+    // expected-note@-1 {{'A' is not empty}} \
+    // expected-note@-1 {{because it has a non-static data member 'x' of type 'int'}} \
+    // expected-note@#e-A {{'A' defined here}}
+
+    // Reference member.
+    struct R {int &r; }; // #e-R
+    static_assert(__is_empty(R));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::R)'}} \
+    // expected-note@-1 {{'R' is not empty}} \
+    // expected-note@-1 {{because it has a non-static data member 'r' of type 'int &'}} \
+    // expected-note@#e-R {{'R' defined here}}
+
+    // Virtual function.
+    struct VirtualFunc {virtual void f(); }; // #e-VirtualFunc
+    static_assert(__is_empty(VirtualFunc));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::VirtualFunc)'}} \
+    // expected-note@-1 {{'VirtualFunc' is not empty}} \
+    // expected-note@-1 {{because it has a virtual function 'f'}} \
+    // expected-note@#e-VirtualFunc {{'VirtualFunc' defined here}}
+
+    // Virtual base class.
+    struct EB {};
+    struct VB: virtual EB {}; // #e-VB
+    static_assert(__is_empty(VB));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::VB)'}} \
+    // expected-note@-1 {{'VB' is not empty}} \
+    // expected-note@-1 {{because it has a virtual base 'EB'}} \
+    // expected-note@#e-VB {{'VB' defined here}}
+
+    // Non-empty base class.
+    struct Base { int b; }; // #e-Base
+    struct Derived : Base {}; // #e-Derived
+    static_assert(__is_empty(Derived));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::Derived)'}} \
+    // expected-note@-1 {{'Derived' is not empty}} \
+    // expected-note@-1 {{because it has a base class 'Base' that is not empty}} \
+    // expected-note@#e-Derived {{'Derived' defined here}} 
+
+    // Combination of the above.
+    struct Multi : Base, virtual EB { // #e-Multi
+        int z;
+        virtual void g();
+    };
+    static_assert(__is_empty(Multi));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::Multi)'}} \
+    // expected-note@-1 {{'Multi' is not empty}} \
+    // expected-note@-1 {{because it has a non-static data member 'z' of type 'int'}} \
+    // expected-note@-1 {{because it has a virtual function 'g'}} \
+    // expected-note@-1 {{because it has a base class 'Base' that is not empty}} \
+    // expected-note@-1 {{because it has a virtual base 'EB'}} \
+    // expected-note@#e-Multi {{'Multi' defined here}}
+
+    // Zero-width bit-field.
+    struct BitField { int : 0; }; // #e-BitField
+    static_assert(__is_empty(BitField)); // no diagnostics  
+
+    // Dependent bit-field width. 
+    template <int N>
+    struct DependentBitField { int : N; }; // #e-DependentBitField
+
+    static_assert(__is_empty(DependentBitField<0>)); // no diagnostics
+
+    static_assert(__is_empty(DependentBitField<2>)); 
+    // expected-error@-1 {{static assertion failed due to requirement '__is_empty(is_empty_tests::DependentBitField<2>)'}} \
+    // expected-note@-1 {{'DependentBitField<2>' is not empty}} \
+    // expected-note@-1 {{because it field '' is a non-zero-length bit-field}} \
+    // expected-note@#e-DependentBitField {{'DependentBitField<2>' defined here}}
+
+}
+
+namespace standard_layout_tests {
+struct WithVirtual { // #sl-Virtual
+    virtual void foo(); // #sl-Virtual-Foo
+};
+static_assert(__is_standard_layout(WithVirtual));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::WithVirtual)'}} \
+// expected-note@-1 {{'WithVirtual' is not standard-layout}} \
+// expected-note@-1 {{because it has a virtual function 'foo'}} \
+// expected-note@#sl-Virtual-Foo {{'foo' defined here}} \
+// expected-note@#sl-Virtual {{'WithVirtual' defined here}}
+
+struct MixedAccess { // #sl-Mixed
+public:
+    int a; // #sl-MixedF1
+private:
+    int b; // #sl-MixedF2
+};
+static_assert(__is_standard_layout(MixedAccess));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::MixedAccess)'}} \
+// expected-note@-1 {{'MixedAccess' is not standard-layout}} \
+// expected-note@-1 {{because it has mixed access specifiers}} \
+// expected-note@#sl-MixedF1 {{'a' defined here}}
+// expected-note@#sl-MixedF2 {{field 'b' has a different access specifier than field 'a'}}
+// expected-note@#sl-Mixed {{'MixedAccess' defined here}}
+
+struct VirtualBase { virtual ~VirtualBase(); };               // #sl-VirtualBase
+struct VB : virtual VirtualBase {};                            // #sl-VB
+static_assert(__is_standard_layout(VB));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::VB)'}} \
+// expected-note@-1 {{'VB' is not standard-layout}} \
+// expected-note@-1 {{because it has a virtual base 'VirtualBase'}} \
+// expected-note@-1 {{because it has a non-standard-layout base 'VirtualBase'}} \
+// expected-note@-1 {{because it has a virtual function '~VB'}} \
+// expected-note@#sl-VB {{'VB' defined here}}
+// expected-note@#sl-VB {{'~VB' defined here}}
+
+union U {      // #sl-U
+public:
+    int x; // #sl-UF1
+private:
+    int y; // #sl-UF2
+};                                                       
+static_assert(__is_standard_layout(U));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::U)'}} \
+// expected-note@-1 {{'U' is not standard-layout}} \
+// expected-note@-1 {{because it has mixed access specifiers}}
+// expected-note@#sl-UF1 {{'x' defined here}}
+// expected-note@#sl-UF2 {{field 'y' has a different access specifier than field 'x'}}
+// expected-note@#sl-U {{'U' defined here}}
+
+// Single base class is OK
+struct BaseClass{ int a; };                                   // #sl-BaseClass
+struct DerivedOK : BaseClass {};                                // #sl-DerivedOK
+static_assert(__is_standard_layout(DerivedOK));    
+
+// Primitive types should be standard layout
+static_assert(__is_standard_layout(int));                     // #sl-Int
+static_assert(__is_standard_layout(float));                   // #sl-Float
+
+// Multi-level inheritance: Non-standard layout
+struct Base1 { int a; };                                      // #sl-Base1
+struct Base2 { int b; };                                      // #sl-Base2
+struct DerivedClass : Base1, Base2 {};                        // #sl-DerivedClass
+static_assert(__is_standard_layout(DerivedClass));               
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::DerivedClass)'}} \
+// expected-note@-1 {{'DerivedClass' is not standard-layout}} \
+// expected-note@-1 {{because it has multiple base classes with data members}} \
+// expected-note@#sl-DerivedClass {{'DerivedClass' defined here}} 
+
+// Inheritance hierarchy with multiple classes having data members
+struct BaseA { int a; };                                      // #sl-BaseA
+struct BaseB : BaseA {};                                      // inherits BaseA, has no new members
+struct BaseC: BaseB { int c; };                               // #sl-BaseC
+static_assert(__is_standard_layout(BaseC));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::BaseC)'}} \
+// expected-note@-1 {{'BaseC' is not standard-layout}} \
+// expected-note@-1 {{because it has an indirect base 'BaseA' with data members}} \
+// expected-note@#sl-BaseC {{'BaseC' defined here}} \
+// Multiple direct base classes with no data members --> standard layout
+struct BaseX {};                                              // #sl-BaseX
+struct BaseY {};                                              // #sl-BaseY
+struct MultiBase : BaseX, BaseY {};                          // #sl-MultiBase
+static_assert(__is_standard_layout(MultiBase));
+
+struct A {
+  int x;
+};
+
+struct B : A {
+};
+// Indirect base with data members
+struct C : B { int y; }; // #sl-C
+static_assert(__is_standard_layout(C));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::C)'}} \
+// expected-note@-1 {{'C' is not standard-layout}} \
+// expected-note@-1 {{because it has an indirect base 'A' with data members}} \
+// expected-note@#sl-C {{'C' defined here}}
+
+struct D {
+    union { int a; float b; };
+  }; // #sl-D
+static_assert(__is_standard_layout(D)); // no diagnostics
+
+// E inherits D but adds a new member
+struct E : D { int x; }; // #sl-E
+static_assert(__is_standard_layout(E));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::E)'}} \
+// expected-note@-1 {{'E' is not standard-layout}} \
+// expected-note@-1 {{because it has an indirect base 'D' with data members}} \
+// expected-note@#sl-E {{'E' defined here}}
+
+// F inherits D but only an unnamed bitfield
+// This should still fail because F ends up with a 
+// base class with a data member and its own unnamed bitfield
+// which is not allowed in standard layout
+struct F : D { int : 0; }; // #sl-F
+static_assert(__is_standard_layout(F));
+// expected-error@-1 {{static assertion failed due to requirement '__is_standard_layout(standard_layout_tests::F)'}} \
+// expected-note@-1 {{'F' is not standard-layout}} \
+// expected-note@#sl-F {{'F' defined here}}
+
+struct Empty {};
+struct G { Empty a, b; }; // #sl-G
+static_assert(__is_standard_layout(G)); // no diagnostics
+
+struct H { Empty a; int x; }; // #sl-H
+static_assert(__is_standard_layout(H)); // no diagnostics
+
+ struct I { Empty a; int : 0; int x; }; // #sl-I
+static_assert(__is_standard_layout(I)); // no diagnostics
+}
+
diff --git a/clang/test/SemaCXX/unique_object_duplication.h b/clang/test/SemaCXX/unique_object_duplication.h
index bd0ee6bd14d6..a5da767b7947 100644
--- a/clang/test/SemaCXX/unique_object_duplication.h
+++ b/clang/test/SemaCXX/unique_object_duplication.h
@@ -173,10 +173,6 @@ namespace GlobalTest {
 
     // Always visible
     VISIBLE static inline float allowedStaticMember2 = 0.0;
-
-    // Tests here are sparse because the AddrTest case below will define plenty
-    // more, which aren't problematic to define (because they're immutable), but
-    // may still cause problems if their address is taken.
   };
 
   inline float Test::disallowedStaticMember2 = 2.3; // hidden-warning {{'disallowedStaticMember2' may be duplicated when built into a shared library: it is mutable, with external linkage and hidden visibility}}
@@ -211,3 +207,44 @@ inline int allowedTemplate2 = 0;
 template int allowedTemplate2<int>;
 
 } // namespace TemplateTest
+
+/******************************************************************************
+ * Case four: Nested classes
+ ******************************************************************************/
+
+namespace NestedClassTest {
+// Unlike visibility, declexport annotations do not propagate down to nested
+// classes. Hence on windows, we only avoid duplication of class members if
+// their immediate containing class is annotated. On posix, we get avoid
+// duplication if any containing class is annotated.
+
+class VISIBLE Outer {
+  // Visible because their class is exported
+  inline static int allowedOuterMember = 0;
+  int* allowedOuterFunction() {
+    static int allowed = 0;
+    return &allowed;
+  }
+
+  // No annotation, and visibility is only propagated on posix.
+  class HiddenOnWindows {
+    inline static int disallowedInnerMember = 0; // windows-warning {{'disallowedInnerMember' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+
+
+    int* disallowedInnerFunction() {
+      static int disallowed = 0; // windows-warning {{'disallowed' may be duplicated when built into a shared library: it is mutable, with external linkage and no import/export annotation}}
+      return &disallowed;
+    }
+  };
+
+  class VISIBLE AlwaysVisible {
+    inline static int allowedInnerMember = 0;
+
+    int* allowedInnerFunction() {
+      static int allowed = 0;
+      return &allowed;
+    }
+  };
+};
+
+}
\ No newline at end of file
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index e287f00a5453..1e7855517207 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -1,5 +1,7 @@
 // RUN: %clang_cc1 -std=c++20  -Wno-all  -Wunsafe-buffer-usage-in-container -verify %s
 
+typedef unsigned int size_t;
+
 namespace std {
   template <class T> class span {
   public:
@@ -16,6 +18,9 @@ namespace std {
 
     template<class R>
     constexpr span(R && range){};
+
+    T* begin() noexcept;
+    T* end() noexcept;
   };
 
 
@@ -27,6 +32,37 @@ namespace std {
     return &__x;
   }
 
+  template <typename T, size_t N>
+  struct array {
+    T* begin() noexcept;
+    const T* begin() const noexcept;
+    T* end() noexcept;
+    const T* end() const noexcept;
+    size_t size() const noexcept;
+    T * data() const noexcept;
+    T& operator[](size_t n);
+  };
+
+  template<class T>
+  class initializer_list {
+  public:
+    size_t size() const noexcept;
+    const T* begin() const noexcept;
+    const T* end() const noexcept;
+    T * data() const noexcept;
+  };
+
+  template<typename T>
+  struct basic_string {
+    T *c_str() const noexcept;
+    T *data()  const noexcept;
+    unsigned size();
+    const T* begin() const noexcept;
+    const T* end() const noexcept;
+  };
+
+  typedef basic_string<char> string;
+  typedef basic_string<wchar_t> wstring;
 }
 
 namespace irrelevant_constructors {
@@ -232,3 +268,27 @@ struct HoldsStdSpanAndNotInitializedInCtor {
       : Ptr(P), Size(S)
   {}
 };
+
+namespace test_begin_end {
+  struct Object {
+    int * begin();
+    int * end();
+  };
+  void safe_cases(std::span<int> Sp, std::array<int, 10> Arr, std::string Str, std::initializer_list<Object> Il) {
+    std::span<int>{Sp.begin(), Sp.end()};
+    std::span<int>{Arr.begin(), Arr.end()};
+    std::span<char>{Str.begin(), Str.end()};
+    std::span<Object>{Il.begin(), Il.end()};
+  }
+
+  void unsafe_cases(std::span<int> Sp, std::array<int, 10> Arr, std::string Str, std::initializer_list<Object> Il,
+		    Object Obj) {
+    std::span<int>{Obj.begin(), Obj.end()}; // expected-warning {{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<int>{Sp.end(), Sp.begin()};   // expected-warning {{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+    std::span<int>{Sp.begin(), Arr.end()};   // expected-warning {{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  }
+
+  void unsupport_cases(std::array<Object, 10> Arr) {
+    std::span<int>{Arr[0].begin(), Arr[0].end()}; // expected-warning {{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
+  }
+}
diff --git a/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
new file mode 100644
index 000000000000..7548ba8904a7
--- /dev/null
+++ b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -Wmissing-noreturn -verify %s
+
+namespace std {
+  class string {
+  public:
+    string(const char*);
+  };
+  class runtime_error {
+  public:
+    runtime_error(const string&);
+  };
+}
+
+// This function always throws. Suggest [[noreturn]].
+void throwError(const std::string& msg) { // expected-warning {{function 'throwError' could be declared with attribute 'noreturn'}}
+  throw std::runtime_error(msg);
+}
+
+// The non-void caller should not warn about missing return.
+int ensureZero(int i) {
+  if (i == 0) return 0;
+  throwError("ERROR"); // no-warning
+}
diff --git a/clang/test/SemaCXX/wreturn-always-throws.cpp b/clang/test/SemaCXX/wreturn-always-throws.cpp
new file mode 100644
index 000000000000..addcadd1183d
--- /dev/null
+++ b/clang/test/SemaCXX/wreturn-always-throws.cpp
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -verify %s
+// expected-no-diagnostics
+
+namespace std {
+  class string {
+  public:
+    string(const char*); // constructor for runtime_error
+  };
+  class runtime_error {
+  public:
+    runtime_error(const string &); 
+  };
+}
+
+// Non-template version.
+
+void throwError(const std::string& msg) {
+  throw std::runtime_error(msg);
+}
+
+int ensureZero(const int i) {
+  if (i == 0) return 0;
+  throwError("ERROR"); // no-warning
+}
+
+int alwaysThrows() {
+  throw std::runtime_error("This function always throws"); // no-warning
+}
+
+// Template version.
+
+template<typename T> 
+void throwErrorTemplate(const T& msg) {
+  throw msg;
+}
+
+template <typename T>
+int ensureZeroTemplate(T i) {
+  if (i == 0) return 0;
+  throwErrorTemplate("ERROR"); // no-warning
+}
+
+void testTemplates() {
+  throwErrorTemplate("ERROR");
+  (void)ensureZeroTemplate(42);
+}
diff --git a/clang/test/SemaHIP/amdgpu-feature-builtins-cant-jump.hip b/clang/test/SemaHIP/amdgpu-feature-builtins-cant-jump.hip
new file mode 100644
index 000000000000..a7f1abcdcd8f
--- /dev/null
+++ b/clang/test/SemaHIP/amdgpu-feature-builtins-cant-jump.hip
@@ -0,0 +1,62 @@
+// REQUIRES: amdgpu-registered-target
+// REQUIRES: spirv-registered-target
+// RUN: %clang_cc1 -fsyntax-only -verify -triple amdgcn -target-cpu gfx900 -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple amdgcn -target-cpu gfx1201 -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple spirv64-amd-amdhsa -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple x86_64 -aux-triple amdgcn -Wno-unused-value %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple x86_64 -aux-triple spirv64-amd-amdhsa -Wno-unused-value %s
+
+#define __device__ __attribute__((device))
+#define __global__ __attribute__((global))
+
+__device__ void f(int *ptr, int size, bool f) {
+    int i = 0;
+    if (f)
+        goto label; // expected-error {{cannot jump from this goto statement to its label}}
+
+    if (__builtin_amdgcn_processor_is("gfx900")) { // expected-note {{jump enters statement controlled by AMDGPU feature predicate}}
+        for (i = 0; i < size; ++i) {
+            label:
+            ptr[i] = i;
+        }
+    }
+}
+
+__device__ void g(int *ptr, int size, bool f) {
+    int i = 0;
+    if (f)
+        goto label; // expected-error {{cannot jump from this goto statement to its label}}
+
+    if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var)) { // expected-note {{jump enters statement controlled by AMDGPU feature predicate}}
+        for (i = 0; i < size; ++i) {
+            label:
+            ptr[i] = i;
+        }
+    }
+}
+
+__global__ void h(int *ptr, int size, bool f) {
+    int i = 0;
+    if (f)
+        goto label; // expected-error {{cannot jump from this goto statement to its label}}
+
+    if (__builtin_amdgcn_processor_is("gfx900")) { // expected-note {{jump enters statement controlled by AMDGPU feature predicate}}
+        for (i = 0; i < size; ++i) {
+            label:
+            ptr[i] = i;
+        }
+    }
+}
+
+__global__ void i(int *ptr, int size, bool f) {
+    int i = 0;
+    if (f)
+        goto label; // expected-error {{cannot jump from this goto statement to its label}}
+
+    if (__builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep_var)) { // expected-note {{jump enters statement controlled by AMDGPU feature predicate}}
+        for (i = 0; i < size; ++i) {
+            label:
+            ptr[i] = i;
+        }
+    }
+}
diff --git a/clang/test/SemaHIP/amdgpu-feature-builtins-return-type-deduction.hip b/clang/test/SemaHIP/amdgpu-feature-builtins-return-type-deduction.hip
new file mode 100644
index 000000000000..27bbb3f2f3d0
--- /dev/null
+++ b/clang/test/SemaHIP/amdgpu-feature-builtins-return-type-deduction.hip
@@ -0,0 +1,31 @@
+// REQUIRES: amdgpu-registered-target
+// REQUIRES: spirv-registered-target
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx900 -ast-dump -ast-dump-decl-types %s | FileCheck %s --strict-whitespace
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1201 -ast-dump -ast-dump-decl-types %s | FileCheck %s --strict-whitespace
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -ast-dump -ast-dump-decl-types %s | FileCheck %s --strict-whitespace
+// RUN: %clang_cc1 -triple x86_64 -aux-triple amdgcn -ast-dump -ast-dump-decl-types %s | FileCheck %s --strict-whitespace
+// RUN: %clang_cc1 -triple x86_64 -aux-triple spirv64-amd-amdhsa -ast-dump -ast-dump-decl-types %s | FileCheck %s --strict-whitespace
+
+__attribute__((device)) auto foo() {
+  return __builtin_amdgcn_processor_is("gfx900");
+}
+
+__attribute__((device)) decltype(auto) bar() {
+  return __builtin_amdgcn_is_invocable(__builtin_amdgcn_s_sleep);
+}
+
+// CHECK: |-TypedefDecl {{.*}} implicit __amdgpu_feature_predicate_t '__amdgpu_feature_predicate_t'
+// CHECK-NEXT: | `-BuiltinType {{.*}} '__amdgpu_feature_predicate_t'
+// CHECK-DAG: |-FunctionDecl {{.*}} foo '__amdgpu_feature_predicate_t ()'
+// CHECK-NEXT: |-CompoundStmt {{.*}}
+// CHECK-NEXT: | `-ReturnStmt {{.*}}
+// CHECK-NEXT: |   `-CallExpr {{.*}} '__amdgpu_feature_predicate_t'
+// CHECK-NEXT: |     |-ImplicitCastExpr {{.*}} '__amdgpu_feature_predicate_t (*)(const char *) noexcept'
+// CHECK-NEXT: |     | `-DeclRefExpr {{.*}} Function {{.*}} '__builtin_amdgcn_processor_is' '__amdgpu_feature_predicate_t (const char *) noexcept'
+// CHECK-NEXT: |     `-StringLiteral {{.*}} "gfx900"
+// CHECK-DAG: |-FunctionDecl {{.*}} bar '__amdgpu_feature_predicate_t ()'
+// CHECK-NEXT: |-CompoundStmt {{.*}}
+// CHECK-NEXT: | `-ReturnStmt {{.*}}
+// CHECK-NEXT: |   `-CallExpr {{.*}} '__amdgpu_feature_predicate_t'
+// CHECK-NEXT: |     |-ImplicitCastExpr {{.*}} '__amdgpu_feature_predicate_t (*)() noexcept' <BuiltinFnToFnPtr>
+// CHECK-NEXT: |     | `-DeclRefExpr {{.*}} Function {{.*}} '__builtin_amdgcn_is_invocable' '__amdgpu_feature_predicate_t () noexcept'
diff --git a/clang/test/SemaHLSL/RootSignature-err.hlsl b/clang/test/SemaHLSL/RootSignature-err.hlsl
index f544247f4db2..aec8a15f8ed7 100644
--- a/clang/test/SemaHLSL/RootSignature-err.hlsl
+++ b/clang/test/SemaHLSL/RootSignature-err.hlsl
@@ -18,3 +18,7 @@ void bad_root_signature_3() {}
 
 [RootSignature("DescriptorTable(), invalid")] // expected-error {{expected end of stream to denote end of parameters, or, another valid parameter of RootSignature}}
 void bad_root_signature_4() {}
+
+// expected-error@+1 {{expected ')' to denote end of parameters, or, another valid parameter of RootConstants}}
+[RootSignature("RootConstants(b0, num32BitConstants = 1, invalid)")]
+void bad_root_signature_5() {}
diff --git a/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl b/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
index e5152e72d480..4b3579d51818 100644
--- a/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
+++ b/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
@@ -19,3 +19,46 @@ void bad_root_signature_3() {}
 // expected-error@+1 {{resource ranges u[0;0] and u[0;0] overlap within space = 0 and visibility = Pixel}}
 [RootSignature("UAV(u0, visibility = SHADER_VISIBILITY_PIXEL), UAV(u0, visibility = SHADER_VISIBILITY_ALL)")]
 void bad_root_signature_4() {}
+
+// expected-error@+1 {{resource ranges b[0;0] and b[0;0] overlap within space = 0 and visibility = All}}
+[RootSignature("RootConstants(num32BitConstants=4, b0), RootConstants(num32BitConstants=2, b0)")]
+void bad_root_signature_5() {}
+
+// expected-error@+1 {{resource ranges s[3;3] and s[3;3] overlap within space = 0 and visibility = All}}
+[RootSignature("StaticSampler(s3), StaticSampler(s3)")]
+void bad_root_signature_6() {}
+
+// expected-error@+1 {{resource ranges t[2;5] and t[0;3] overlap within space = 0 and visibility = All}}
+[RootSignature("DescriptorTable(SRV(t0, numDescriptors=4), SRV(t2, numDescriptors=4))")]
+void bad_root_signature_7() {}
+
+// expected-error@+1 {{resource ranges u[2;5] and u[0;unbounded) overlap within space = 0 and visibility = Hull}}
+[RootSignature("DescriptorTable(UAV(u0, numDescriptors=unbounded), visibility = SHADER_VISIBILITY_HULL), DescriptorTable(UAV(u2, numDescriptors=4))")]
+void bad_root_signature_8() {}
+
+// expected-error@+1 {{resource ranges b[0;2] and b[2;2] overlap within space = 0 and visibility = All}}
+[RootSignature("RootConstants(num32BitConstants=4, b2), DescriptorTable(CBV(b0, numDescriptors=3))")]
+void bad_root_signature_9() {}
+
+// expected-error@+1 {{resource ranges s[4;unbounded) and s[17;17] overlap within space = 0 and visibility = All}}
+[RootSignature("StaticSampler(s17), DescriptorTable(Sampler(s0, numDescriptors=3),Sampler(s4, numDescriptors=unbounded))")]
+void bad_root_signature_10() {}
+
+// expected-error@+1 {{resource ranges b[45;45] and b[4;unbounded) overlap within space = 0 and visibility = Geometry}}
+[RootSignature("DescriptorTable(CBV(b4, numDescriptors=unbounded)), CBV(b45, visibility = SHADER_VISIBILITY_GEOMETRY)")]
+void bad_root_signature_11() {}
+
+#define ReportFirstOverlap \
+ "DescriptorTable( " \
+ "  CBV(b4, numDescriptors = 4), " \
+ "  CBV(b1, numDescriptors = 2), " \
+ "  CBV(b0, numDescriptors = 8), " \
+ ")"
+
+// expected-error@+1 {{resource ranges b[0;7] and b[1;2] overlap within space = 0 and visibility = All}}
+[RootSignature(ReportFirstOverlap)]
+void bad_root_signature_12() {}
+
+// expected-error@+1 {{resource ranges s[2;2] and s[2;2] overlap within space = 0 and visibility = Vertex}}
+[RootSignature("StaticSampler(s2, visibility=SHADER_VISIBILITY_ALL), DescriptorTable(Sampler(s2), visibility=SHADER_VISIBILITY_VERTEX)")]
+void valid_root_signature_13() {}
diff --git a/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl b/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl
index 5778fb2ae4eb..09a1110b0fbc 100644
--- a/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl
+++ b/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl
@@ -13,3 +13,12 @@ void valid_root_signature_2() {}
 
 [RootSignature("CBV(b0), SRV(t0)")]
 void valid_root_signature_3() {}
+
+[RootSignature("RootConstants(num32BitConstants=4, b0, space=0), DescriptorTable(CBV(b0, space=1))")]
+void valid_root_signature_4() {}
+
+[RootSignature("StaticSampler(s2, visibility=SHADER_VISIBILITY_PIXEL), DescriptorTable(Sampler(s2), visibility=SHADER_VISIBILITY_VERTEX)")]
+void valid_root_signature_5() {}
+
+[RootSignature("DescriptorTable(SRV(t5), UAV(u5, numDescriptors=2))")]
+void valid_root_signature_6() {}
diff --git a/clang/test/SemaObjCXX/arc-0x.mm b/clang/test/SemaObjCXX/arc-0x.mm
index ac788686a737..bcaa5da6b928 100644
--- a/clang/test/SemaObjCXX/arc-0x.mm
+++ b/clang/test/SemaObjCXX/arc-0x.mm
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only -fobjc-arc -fobjc-runtime-has-weak -fobjc-weak -verify -fblocks -fobjc-exceptions %s
+// RUN: %clang_cc1 -std=c++11 -fobjc-arc -fobjc-runtime-has-weak -fobjc-weak -verify -fblocks -fobjc-exceptions -emit-llvm -o - %s
 
 // "Move" semantics, trivial version.
 void move_it(__strong id &&from) {
@@ -14,6 +14,11 @@ @interface A
 // don't warn about this
 extern "C" A* MakeA();
 
+void test_nonexistent_method(A *a) {
+  // This used to crash in codegen.
+  auto a1 = [a foo]; // expected-error {{no visible @interface for 'A' declares the selector 'foo'}}
+}
+
 // Ensure that deduction works with lifetime qualifiers.
 void deduction(id obj) {
   auto a = [[A alloc] init];
diff --git a/clang/test/SemaObjCXX/attr-swift_name-cxx.mm b/clang/test/SemaObjCXX/attr-swift_name-cxx.mm
index 658dbfff185c..9e83c63ecf2f 100644
--- a/clang/test/SemaObjCXX/attr-swift_name-cxx.mm
+++ b/clang/test/SemaObjCXX/attr-swift_name-cxx.mm
@@ -1,7 +1,43 @@
 // RUN: %clang_cc1 -verify -fsyntax-only -fobjc-arc -fblocks %s
 
+#define SWIFT_NAME(name) __attribute__((swift_name(name)))
 #define SWIFT_ASYNC_NAME(name) __attribute__((__swift_async_name__(name)))
 
+namespace MyNS {
+struct NestedStruct {};
+}
+
+void nestedStruct_method(MyNS::NestedStruct) SWIFT_NAME("MyNS.NestedStruct.method(self:)");
+void nestedStruct_methodConstRef(const MyNS::NestedStruct&) SWIFT_NAME("MyNS.NestedStruct.methodConstRef(self:)");
+void nestedStruct_invalidContext1(MyNS::NestedStruct) SWIFT_NAME(".MyNS.NestedStruct.invalidContext1(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+void nestedStruct_invalidContext2(MyNS::NestedStruct) SWIFT_NAME("MyNS::NestedStruct.invalidContext2(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+void nestedStruct_invalidContext3(MyNS::NestedStruct) SWIFT_NAME("::MyNS::NestedStruct.invalidContext3(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+void nestedStruct_invalidContext4(MyNS::NestedStruct) SWIFT_NAME("MyNS..NestedStruct.invalidContext4(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+void nestedStruct_invalidContext5(MyNS::NestedStruct) SWIFT_NAME("MyNS.NestedStruct.invalidContext5.(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the base name}}
+void nestedStruct_invalidContext6(MyNS::NestedStruct) SWIFT_NAME("MyNS.NestedStruct::invalidContext6(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the base name}}
+
+namespace MyNS {
+namespace MyDeepNS {
+struct DeepNestedStruct {};
+}
+}
+
+void deepNestedStruct_method(MyNS::MyDeepNS::DeepNestedStruct) SWIFT_NAME("MyNS.MyDeepNS.DeepNestedStruct.method(self:)");
+void deepNestedStruct_methodConstRef(const MyNS::MyDeepNS::DeepNestedStruct&) SWIFT_NAME("MyNS.MyDeepNS.DeepNestedStruct.methodConstRef(self:)");
+void deepNestedStruct_invalidContext(const MyNS::MyDeepNS::DeepNestedStruct&) SWIFT_NAME("MyNS::MyDeepNS::DeepNestedStruct.methodConstRef(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+
+typedef MyNS::MyDeepNS::DeepNestedStruct DeepNestedStructTypedef;
+
+void deepNestedStructTypedef_method(DeepNestedStructTypedef) SWIFT_NAME("DeepNestedStructTypedef.method(self:)");
+void deepNestedStructTypedef_methodQualName(MyNS::MyDeepNS::DeepNestedStruct) SWIFT_NAME("DeepNestedStructTypedef.method(self:)");
+
+struct TopLevelStruct {
+  struct StructInStruct {};
+};
+
+void structInStruct_method(TopLevelStruct::StructInStruct) SWIFT_NAME("TopLevelStruct.StructInStruct.method(self:)");
+void structInStruct_invalidContext(TopLevelStruct::StructInStruct) SWIFT_NAME("TopLevelStruct::StructInStruct.method(self:)"); // expected-warning {{'swift_name' attribute has invalid identifier for the context name}}
+
 typedef int (^CallbackTy)(void);
 
 class CXXClass {
diff --git a/clang/test/SemaOpenACC/cache-construct-ast.cpp b/clang/test/SemaOpenACC/cache-construct-ast.cpp
index 44c80beed77a..fd8cd049840c 100644
--- a/clang/test/SemaOpenACC/cache-construct-ast.cpp
+++ b/clang/test/SemaOpenACC/cache-construct-ast.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -fopenacc -ast-dump | FileCheck %s
+// RUN: %clang_cc1 %s -fopenacc -Wno-openacc-cache-var-inside-loop -ast-dump | FileCheck %s
 
 // Test this with PCH.
 // RUN: %clang_cc1 %s -fopenacc -emit-pch -o %t %s
@@ -14,6 +14,21 @@ void use() {
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'int[5]'
 
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
@@ -33,6 +48,18 @@ _Pragma("acc cache(Array[1])")
   // CHECK-NEXT: DeclRefExpr{{.*}}'Array' 'int[5]'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+
+  // No vars here, since they are bad references.
+#pragma acc cache (Array[i])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  }
+
+  // No vars here, since they are bad references.
+#pragma acc cache(Array[1:2])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+
+  return;
+  // CHECK-NEXT: ReturnStmt
 }
 
 struct S {
@@ -42,6 +69,21 @@ struct S {
   void StructUse() {
     // CHECK: CXXMethodDecl{{.*}}StructUse 'void ()'
     // CHECK-NEXT: CompoundStmt
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
     // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
     // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
@@ -79,6 +121,7 @@ struct S {
     // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
     // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
   }
+  }
 };
 
 template<typename T>
@@ -89,6 +132,21 @@ void templ_use() {
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'T[5]'
 
+#pragma acc loop
+  // CHECK-NEXT: OpenACCLoopConstruct
+  for (int i = 0; i < 5; ++i) {
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
 #pragma acc cache(Array[1])
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'T' lvalue
@@ -101,6 +159,17 @@ void templ_use() {
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
 
+    // No vars here, since they are bad references.
+#pragma acc cache(Array[i])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  }
+  // No vars here, since they are bad references.
+#pragma acc cache(Array[1])
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+
+  return;
+  // CHECK-NEXT: ReturnStmt
+
   // Instantiation:
   // CHECK: FunctionDecl{{.*}} templ_use 'void ()' implicit_instantiation
   // CHECK-NEXT: TemplateArgument type 'int'
@@ -108,6 +177,21 @@ void templ_use() {
   // CHECK-NEXT: CompoundStmt
   // CHECK-NEXT: DeclStmt
   // CHECK-NEXT: VarDecl{{.*}}Array 'int[5]'
+
+  // CHECK-NEXT: OpenACCLoopConstruct
+  // CHECK-NEXT: ForStmt
+  // CHECK-NEXT: DeclStmt
+  // CHECK-NEXT: VarDecl{{.*}}i 'int'
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: <<<NULL>>>
+  // CHECK-NEXT: BinaryOperator
+  // CHECK-NEXT: ImplicitCastExpr
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: IntegerLiteral
+  // CHECK-NEXT: UnaryOperator
+  // CHECK-NEXT: DeclRefExpr
+  // CHECK-NEXT: CompoundStmt
+  //
   // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'int' lvalue
   // CHECK-NEXT: ImplicitCastExpr{{.*}}'int *' <ArrayToPointerDecay>
@@ -119,6 +203,11 @@ void templ_use() {
   // CHECK-NEXT: DeclRefExpr{{.*}}'Array' 'int[5]'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 1
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 2
+
+  // No vars here, since they are bad references.
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  // CHECK-NEXT: OpenACCCacheConstruct{{.*}} cache
+  // CHECK-NEXT: ReturnStmt
 }
 
 void foo() {
diff --git a/clang/test/SemaOpenACC/cache-construct.cpp b/clang/test/SemaOpenACC/cache-construct.cpp
index 0a32754c8271..3196cd010cea 100644
--- a/clang/test/SemaOpenACC/cache-construct.cpp
+++ b/clang/test/SemaOpenACC/cache-construct.cpp
@@ -5,13 +5,16 @@ void use() {
   int Array[5];
   int NotArray;
 
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
+  }
 }
 
 struct S {
@@ -20,16 +23,19 @@ struct S {
   int Array2D[5][5];
 
   void use() {
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 #pragma acc cache(Array2D[1][1])
 #pragma acc cache(Array2D[1][1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
   }
+  }
 };
 
 template<typename T>
@@ -37,13 +43,16 @@ void templ_use() {
   T Array[5];
   T NotArray;
 
+#pragma acc loop
+  for (int i = 0; i < 5;++i) {
 #pragma acc cache(Array[1])
 #pragma acc cache(Array[1:2])
 
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(Array)
-  // expected-error@+1{{OpenACC variable in cache directive is not a valid sub-array or array element}}
+  // expected-error@+1{{OpenACC variable in 'cache' directive is not a valid sub-array or array element}}
 #pragma acc cache(NotArray)
+  }
 }
 
 void foo() {
diff --git a/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp b/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp
new file mode 100644
index 000000000000..bc24263c669d
--- /dev/null
+++ b/clang/test/SemaOpenACC/cache-warn-invalid-varloc.cpp
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+void foo() {
+  int Array[5];
+  // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+  #pragma acc cache(readonly:Array[1], Array[1:2])
+}
+
+
+void foo2() {
+#pragma acc loop
+  for(int i = 0; i < 5; ++i) {
+    int Array[5];
+    // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+    #pragma acc cache(readonly:Array[1], Array[1:2])
+  }
+}
+
+
+template<typename T>
+void foo3() {
+  T Array[5];
+  // expected-warning@+1 2{{OpenACC variable in 'cache' directive was not declared outside of the associated 'loop' directive; directive has no effect}}
+  #pragma acc cache(readonly:Array[1], Array[1:2])
+}
+
+void Inst() {
+  foo3<int>();
+}
diff --git a/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp b/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp
index ec4451e9df7d..f865f826757e 100644
--- a/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-copyout-ast.cpp
@@ -14,7 +14,7 @@ void NormalUses(float *PointerParam) {
   // CHECK: ParmVarDecl
   // CHECK-NEXT: CompoundStmt
 
-#pragma acc parallel loop copyout(GlobalArray) pcopyout(zero:PointerParam[Global]) present_or_copyout(always, alwaysin: Global)
+#pragma acc parallel loop copyout(GlobalArray) pcopyout(zero:PointerParam[Global]) present_or_copyout(always, alwaysout: Global)
   for (unsigned i = 0; i < 5; ++i);
   // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
   // CHECK-NEXT: copyout clause
@@ -25,7 +25,7 @@ void NormalUses(float *PointerParam) {
   // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
-  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
   // CHECK-NEXT: For
   // CHECK: NullStmt
@@ -42,10 +42,10 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
   // CHECK-NEXT: CompoundStmt
 
-#pragma acc parallel loop copyout(always, alwaysin: t) pcopyout(zero: NTTP, u) present_or_copyout(u[0:t])
+#pragma acc parallel loop copyout(always, alwaysout: t) pcopyout(zero: NTTP, u) present_or_copyout(u[0:t])
   for (unsigned i = 0; i < 5; ++i);
   // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
-  // CHECK-NEXT: copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: DeclRefExpr{{.*}}'T' lvalue ParmVar{{.*}} 't' 'T'
   // CHECK-NEXT: pcopyout clause modifiers: zero
   // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
@@ -73,7 +73,7 @@ void TemplUses(T t, U u) {
 
 // #pragma acc parallel copyout(t) pcopyout(zero: NTTP, u) present_or_copyout(u[0:t])
   // CHECK-NEXT: OpenACCCombinedConstruct{{.*}} parallel loop
-  // CHECK-NEXT: copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
   // CHECK-NEXT: pcopyout clause modifiers: zero
   // CHECK-NEXT: SubstNonTypeTemplateParmExpr{{.*}}'const unsigned int' lvalue
diff --git a/clang/test/SemaOpenACC/combined-construct-copyout-clause.c b/clang/test/SemaOpenACC/combined-construct-copyout-clause.c
index 2ca70736d30e..a1774e070a7b 100644
--- a/clang/test/SemaOpenACC/combined-construct-copyout-clause.c
+++ b/clang/test/SemaOpenACC/combined-construct-copyout-clause.c
@@ -78,19 +78,19 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 }
 void ModList() {
   int V1;
-  // expected-error@+2{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc parallel loop copyout(always, alwaysin, alwaysout, zero, readonly: V1)
   for(int i = 0; i < 6;++i);
-  // expected-error@+1{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
-#pragma acc serial loop copyout(alwaysout: V1)
+  // expected-error@+1{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
+#pragma acc serial loop copyout(alwaysin: V1)
   for(int i = 0; i < 6;++i);
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc kernels loop copyout(readonly: V1)
   for(int i = 0; i < 6;++i);
 #pragma acc parallel loop copyout(capture:V1)
   for(int i = 5; i < 10;++i);
-#pragma acc parallel loop copyout(always, alwaysin, zero, capture: V1)
+#pragma acc parallel loop copyout(always, alwaysout, zero, capture: V1)
   for(int i = 5; i < 10;++i);
 }
 
diff --git a/clang/test/SemaOpenACC/compute-construct-copyout-clause.c b/clang/test/SemaOpenACC/compute-construct-copyout-clause.c
index 34f8d5aa60e0..c0ab6c3ea658 100644
--- a/clang/test/SemaOpenACC/compute-construct-copyout-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-copyout-clause.c
@@ -78,18 +78,18 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 }
 void ModList() {
   int V1;
-  // expected-error@+2{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc parallel copyout(always, alwaysin, alwaysout, zero, readonly: V1)
   for(int i = 0; i < 6;++i);
-  // expected-error@+1{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
-#pragma acc serial copyout(alwaysout: V1)
+  // expected-error@+1{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
+#pragma acc serial copyout(alwaysin: V1)
   for(int i = 0; i < 6;++i);
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc kernels copyout(readonly: V1)
   for(int i = 0; i < 6;++i);
 #pragma acc parallel copyout(capture:V1)
   for(int i = 5; i < 10;++i);
-#pragma acc parallel copyout(always, alwaysin, zero, capture: V1)
+#pragma acc parallel copyout(always, alwaysout, zero, capture: V1)
   for(int i = 5; i < 10;++i);
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp b/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
index df01c5934966..e3c41849eba8 100644
--- a/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
@@ -133,7 +133,7 @@ void NormalUses(float *PointerParam) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
 
-#pragma acc parallel copyout(GlobalArray) pcopyout(zero:PointerParam[Global]) present_or_copyout(always, alwaysin: Global)
+#pragma acc parallel copyout(GlobalArray) pcopyout(zero:PointerParam[Global]) present_or_copyout(always, alwaysout: Global)
   while(true);
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}} parallel
   // CHECK-NEXT: copyout clause
@@ -144,7 +144,7 @@ void NormalUses(float *PointerParam) {
   // CHECK-NEXT: DeclRefExpr{{.*}}'float *' lvalue ParmVar{{.*}}'PointerParam' 'float *'
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
-  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue Var{{.*}}'Global' 'int'
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
@@ -363,7 +363,7 @@ void TemplUses(T t, U u, T*PointerParam) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
 
-#pragma acc parallel copyout(t) pcopyout(zero:NTTP, u) present_or_copyout(always, alwaysin: u[0:t])
+#pragma acc parallel copyout(t) pcopyout(zero:NTTP, u) present_or_copyout(always, alwaysout: u[0:t])
   while(true);
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}} parallel
   // CHECK-NEXT: copyout clause
@@ -371,7 +371,7 @@ void TemplUses(T t, U u, T*PointerParam) {
   // CHECK-NEXT: pcopyout clause modifiers: zero
   // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
   // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
-  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: ArraySectionExpr
   // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
@@ -569,7 +569,7 @@ void TemplUses(T t, U u, T*PointerParam) {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
 
-// #pragma acc parallel copyout(t) pcopyout(zero:NTTP, u) present_or_copyout(always, alwaysin: u[0:t])
+// #pragma acc parallel copyout(t) pcopyout(zero:NTTP, u) present_or_copyout(always, alwaysout: u[0:t])
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}} parallel
   // CHECK-NEXT: copyout clause
   // CHECK-NEXT: DeclRefExpr{{.*}}'int' lvalue ParmVar{{.*}} 't' 'int'
@@ -578,7 +578,7 @@ void TemplUses(T t, U u, T*PointerParam) {
   // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
   // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
   // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
-  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: always, alwaysout
   // CHECK-NEXT: ArraySectionExpr
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
   // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
diff --git a/clang/test/SemaOpenACC/data-construct-copyout-ast.cpp b/clang/test/SemaOpenACC/data-construct-copyout-ast.cpp
index 069ced7de83d..1be6854ac996 100644
--- a/clang/test/SemaOpenACC/data-construct-copyout-ast.cpp
+++ b/clang/test/SemaOpenACC/data-construct-copyout-ast.cpp
@@ -54,7 +54,7 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: ParmVarDecl{{.*}} referenced u 'U'
   // CHECK-NEXT: CompoundStmt
 
-#pragma acc data copyout(t) pcopyout(zero: NTTP, u) present_or_copyout(alwaysin: u[0:t])
+#pragma acc data copyout(t) pcopyout(zero: NTTP, u) present_or_copyout(alwaysout: u[0:t])
   ;
   // CHECK-NEXT: OpenACCDataConstruct{{.*}} data
   // CHECK-NEXT: copyout clause
@@ -62,7 +62,7 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: pcopyout clause modifiers: zero
   // CHECK-NEXT: DeclRefExpr{{.*}}'auto' lvalue NonTypeTemplateParm{{.*}} 'NTTP' 'auto &'
   // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
-  // CHECK-NEXT: present_or_copyout clause modifiers: alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: alwaysout
   // CHECK-NEXT: ArraySectionExpr
   // CHECK-NEXT: DeclRefExpr{{.*}}'U' lvalue ParmVar{{.*}} 'u' 'U'
   // CHECK-NEXT: IntegerLiteral{{.*}} 'int' 0
@@ -103,7 +103,7 @@ void TemplUses(T t, U u) {
   // CHECK-NEXT: NonTypeTemplateParmDecl{{.*}} referenced 'auto &' depth 0 index 0 NTTP
   // CHECK-NEXT: DeclRefExpr{{.*}}'const unsigned int' lvalue Var{{.*}} 'CEVar' 'const unsigned int'
   // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
-  // CHECK-NEXT: present_or_copyout clause modifiers: alwaysin
+  // CHECK-NEXT: present_or_copyout clause modifiers: alwaysout
   // CHECK-NEXT: ArraySectionExpr
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int *' <LValueToRValue>
   // CHECK-NEXT: DeclRefExpr{{.*}}'int *' lvalue ParmVar{{.*}} 'u' 'int *'
diff --git a/clang/test/SemaOpenACC/data-construct-copyout-clause.c b/clang/test/SemaOpenACC/data-construct-copyout-clause.c
index deaf48fc7759..79a2f7f4d263 100644
--- a/clang/test/SemaOpenACC/data-construct-copyout-clause.c
+++ b/clang/test/SemaOpenACC/data-construct-copyout-clause.c
@@ -74,20 +74,22 @@ void uses(int IntParam, short *PointerParam, float ArrayParam[5], Complete Compo
 
 void ModList() {
   int V1;
-  // expected-error@+2{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc data copyout(always, alwaysin, alwaysout, zero, readonly: V1)
-  // expected-error@+1{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+1{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
+#pragma acc data copyout(alwaysin: V1)
 #pragma acc data copyout(alwaysout: V1)
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc data copyout(readonly: V1)
 #pragma acc data copyout(capture: V1)
-#pragma acc data copyout(always, alwaysin, zero, capture: V1)
+#pragma acc data copyout(always, alwaysout, zero, capture: V1)
 
-  // expected-error@+2{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc exit data copyout(always, alwaysin, alwaysout, zero, readonly: V1)
-  // expected-error@+1{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+1{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
+#pragma acc exit data copyout(alwaysin: V1)
 #pragma acc exit data copyout(alwaysout: V1)
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc exit data copyout(readonly: V1)
diff --git a/clang/test/SemaOpenACC/declare-construct.cpp b/clang/test/SemaOpenACC/declare-construct.cpp
index ff8a2ec64c31..6f21aedc39b4 100644
--- a/clang/test/SemaOpenACC/declare-construct.cpp
+++ b/clang/test/SemaOpenACC/declare-construct.cpp
@@ -331,17 +331,18 @@ void ModList() {
   // expected-error@+1{{OpenACC 'capture' modifier not valid on 'copyin' clause}}
 #pragma acc declare copyin(always, alwaysin, readonly, capture: V8)
 
-  // expected-error@+2{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc declare copyout(always, alwaysin, alwaysout, zero, readonly: V9)
-  // expected-error@+1{{OpenACC 'alwaysout' modifier not valid on 'copyout' clause}}
-#pragma acc declare copyout(alwaysout: V10)
+  // expected-error@+1{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
+#pragma acc declare copyout(alwaysin: V10)
   // expected-error@+1{{OpenACC 'readonly' modifier not valid on 'copyout' clause}}
 #pragma acc declare copyout(readonly: V11)
   // expected-error@+1{{OpenACC 'capture' modifier not valid on 'copyout' clause}}
 #pragma acc declare copyout(capture: V11B)
+  // expected-error@+2{{OpenACC 'alwaysin' modifier not valid on 'copyout' clause}}
   // expected-error@+1{{OpenACC 'capture' modifier not valid on 'copyout' clause}}
-#pragma acc declare copyout(always, alwaysin, zero, capture: V12)
+#pragma acc declare copyout(always, alwaysin, alwaysout, zero, capture: V12)
 
   // expected-error@+5{{OpenACC 'always' modifier not valid on 'create' clause}}
   // expected-error@+4{{OpenACC 'alwaysin' modifier not valid on 'create' clause}}
diff --git a/clang/test/SemaOpenCL/amd-media-ops.cl b/clang/test/SemaOpenCL/amd-media-ops.cl
new file mode 100644
index 000000000000..9b9b9f54ecf8
--- /dev/null
+++ b/clang/test/SemaOpenCL/amd-media-ops.cl
@@ -0,0 +1,103 @@
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops,+cl_amd_media_ops2 -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops,+cl_amd_media_ops2 -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+// expected-no-diagnostics
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops2 -verify=ops  -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops  -verify=ops2 -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops2 -verify=ops  -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+// RUN: %clang_cc1 %s -triple amdgcn-unknown-unknown -cl-ext=-all,+cl_amd_media_ops  -verify=ops2 -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -finclude-default-header -fdeclare-opencl-builtins
+
+#define TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _## type (type a) { \
+        return builtin(a); \
+    }
+
+#define TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _## type (type a, type b) { \
+        return builtin(a, b); \
+    }
+
+#define TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    ret test_ ## builtin ## _ ## ret ## _ ## type (type a, type b, type c) { \
+        return builtin(a, b, c); \
+    }
+
+#define TEST_1ARG_BUILTIN(builtin, ret, type) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_1ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_2ARG_BUILTIN(builtin, ret, type) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_2ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_3ARG_BUILTIN(builtin, ret, type) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret, type) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 2, type ## 2) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 3, type ## 3) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 4, type ## 4) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 8, type ## 8) \
+    TEST_3ARG_BUILTIN_WITH_TYPE(builtin, ret ## 16, type ## 16)
+
+#define TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret, type_a, type_b, type_c) \
+    ret test_ ## builtin ## _ ## ret ## _ ## type_a ## _ ## type_b ## _ ## type_c (type_a a, type_b b, type_c c) { \
+        return builtin(a, b, c); \
+    }
+
+#define TEST_3ARG_WITH_TYPES_BUILTIN(builtin, ret, type_a, type_b, type_c) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret, type_a, type_b, type_c) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 2, type_a ## 2, type_b ## 2, type_c ## 2) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 3, type_a ## 3, type_b ## 3, type_c ## 3) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 4, type_a ## 4, type_b ## 4, type_c ## 4) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 8, type_a ## 8, type_b ## 8, type_c ## 8) \
+    TEST_3ARG_BUILTIN_WITH_TYPES(builtin, ret ## 16, type_a ## 16, type_b ## 16, type_c ## 16)
+
+TEST_3ARG_BUILTIN(amd_bitalign, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_bytealign, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_lerp, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+
+uint test_amd_pack(float4 a) {
+    return amd_pack(a); // ops-error{{use of undeclared identifier}}
+}
+
+uint test_amd_sad4(uint4 a, uint4 b, uint c) {
+    return amd_sad4(a, b, c); // ops-error{{use of undeclared identifier}}
+}
+
+TEST_3ARG_BUILTIN(amd_sadhi, uint, uint)// ops-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sad, uint, uint) // ops-error 6 {{use of undeclared identifier}}
+
+
+TEST_1ARG_BUILTIN(amd_unpack0, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack1, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack2, float, uint) // ops-error 6 {{use of undeclared identifier}}
+TEST_1ARG_BUILTIN(amd_unpack3, float, uint) // ops-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_bfe, int, int, uint, uint) // ops2-error 6 {{use of undeclared identifier}} #amd_bfe0
+TEST_3ARG_BUILTIN(amd_bfe, uint, uint) // ops2-error 6 {{use of undeclared identifier}} #amd_bfe1
+TEST_2ARG_BUILTIN(amd_bfm, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_max3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_max3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_max3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_median3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_median3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_median3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_min3, float, float) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_min3, int, int) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_min3, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_mqsad, ulong, ulong, uint, ulong) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_WITH_TYPES_BUILTIN(amd_qsad, ulong, ulong, uint, ulong) // ops2-error 6 {{use of undeclared identifier}}
+
+TEST_3ARG_BUILTIN(amd_msad, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sadd, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
+TEST_3ARG_BUILTIN(amd_sadw, uint, uint) // ops2-error 6 {{use of undeclared identifier}}
diff --git a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
index b69fcb5f445b..3ba0d50e7903 100644
--- a/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
+++ b/clang/test/SemaOpenCL/builtins-amdgcn-error-gfx1250-param.cl
@@ -1,6 +1,29 @@
 // REQUIRES: amdgpu-registered-target
 // RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx1250 -verify -S -o - %s
 
+typedef int    v4i   __attribute__((ext_vector_type(4)));
+typedef int    v8i   __attribute__((ext_vector_type(8)));
+
 void test_setprio_inc_wg(short a) {
   __builtin_amdgcn_s_setprio_inc_wg(a); // expected-error {{'__builtin_amdgcn_s_setprio_inc_wg' must be a constant integer}}
 }
+
+void test_s_monitor_sleep(short a) {
+  __builtin_amdgcn_s_monitor_sleep(a); // expected-error {{'__builtin_amdgcn_s_monitor_sleep' must be a constant integer}}
+}
+
+void test__builtin_amdgcn_cvt_f16_fp8(int a, int b) {
+  __builtin_amdgcn_cvt_f16_fp8(a, b); // expected-error {{'__builtin_amdgcn_cvt_f16_fp8' must be a constant integer}}
+}
+
+void test__builtin_amdgcn_cvt_f16_bf8(int a, int b) {
+  __builtin_amdgcn_cvt_f16_bf8(a, b); // expected-error {{'__builtin_amdgcn_cvt_f16_bf8' must be a constant integer}}
+}
+
+void test_amdgcn_tensor_load_store(v4i sg0, v8i sg1, v4i sg2, v4i sg3, int cpol)
+{
+  __builtin_amdgcn_tensor_load_to_lds(sg0, sg1, sg2, sg3, cpol); // expected-error {{'__builtin_amdgcn_tensor_load_to_lds' must be a constant integer}}
+  __builtin_amdgcn_tensor_load_to_lds_d2(sg0, sg1, cpol); // expected-error {{'__builtin_amdgcn_tensor_load_to_lds_d2' must be a constant integer}}
+  __builtin_amdgcn_tensor_store_from_lds(sg0, sg1, sg2, sg3, cpol); // expected-error {{'__builtin_amdgcn_tensor_store_from_lds' must be a constant integer}}
+  __builtin_amdgcn_tensor_store_from_lds_d2(sg0, sg1, cpol); // expected-error {{'__builtin_amdgcn_tensor_store_from_lds_d2' must be a constant integer}}
+}
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index cde5f3c68e49..e433ea4c09c1 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -117,6 +117,29 @@
     config.available_features.add("examples")
 
 
+def have_host_out_of_process_jit_feature_support():
+    clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
+
+    if not clang_repl_exe:
+        return False
+
+    testcode = b"\n".join([b"int i = 0;", b"%quit"])
+
+    try:
+        clang_repl_cmd = subprocess.run(
+            [clang_repl_exe, "-orc-runtime", "-oop-executor"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            input=testcode,
+        )
+    except OSError:
+        return False
+
+    if clang_repl_cmd.returncode == 0:
+        return True
+
+    return False
+
 def have_host_jit_feature_support(feature_name):
     clang_repl_exe = lit.util.which("clang-repl", config.clang_tools_dir)
 
@@ -169,12 +192,17 @@ def have_host_clang_repl_cuda():
     if have_host_clang_repl_cuda():
         config.available_features.add('host-supports-cuda')
 
+    if have_host_out_of_process_jit_feature_support():
+        config.available_features.add("host-supports-out-of-process-jit")
+
 if config.clang_staticanalyzer:
     config.available_features.add("staticanalyzer")
     tools.append("clang-check")
 
     if config.clang_staticanalyzer_z3:
         config.available_features.add("z3")
+        if config.clang_staticanalyzer_z3_mock:
+            config.available_features.add("z3-mock")
     else:
         config.available_features.add("no-z3")
 
@@ -220,9 +248,6 @@ def have_host_clang_repl_cuda():
     )
 )
 
-config.substitutions.append(("%host_cc", config.host_cc))
-config.substitutions.append(("%host_cxx", config.host_cxx))
-
 # Determine whether the test target is compatible with execution on the host.
 if "aarch64" in config.host_arch:
     config.available_features.add("aarch64-host")
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 4e7d44b7122b..30cc915dad40 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -17,8 +17,6 @@ config.clang_tools_dir = lit_config.substitute(path(r"@CURRENT_TOOLS_DIR@"))
 config.clang_lib_dir = path(r"@CMAKE_LIBRARY_OUTPUT_DIRECTORY@")
 config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
-config.host_cc = "@CMAKE_C_COMPILER@"
-config.host_cxx = "@CMAKE_CXX_COMPILER@"
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
 config.have_zstd = @LLVM_ENABLE_ZSTD@
@@ -27,6 +25,7 @@ config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 config.clang_staticanalyzer_z3 = @LLVM_WITH_Z3@
+config.clang_staticanalyzer_z3_mock = @TEST_WITH_Z3_MOCK@
 config.clang_enable_cir = @CLANG_ENABLE_CIR@
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
@@ -39,7 +38,6 @@ config.reverse_iteration = @LLVM_ENABLE_REVERSE_ITERATION@
 config.host_arch = "@HOST_ARCH@"
 config.perl_executable = "@PERL_EXECUTABLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
-config.use_z3_solver = lit_config.params.get('USE_Z3_SOLVER', "@USE_Z3_SOLVER@")
 config.has_plugins = @CLANG_PLUGIN_SUPPORT@
 config.clang_vendor_uti = "@CLANG_VENDOR_UTI@"
 config.llvm_external_lit = path(r"@LLVM_EXTERNAL_LIT@")
diff --git a/clang/tools/c-index-test/c-index-test.c b/clang/tools/c-index-test/c-index-test.c
index 4a887cd0c1e2..cb3245756a39 100644
--- a/clang/tools/c-index-test/c-index-test.c
+++ b/clang/tools/c-index-test/c-index-test.c
@@ -1988,6 +1988,51 @@ static enum CXChildVisitResult PrintDeclAttributes(CXCursor cursor, CXCursor p,
   return CXChildVisit_Continue;
 }
 
+/******************************************************************************/
+/* Inline assembly cursor testing                                             */
+/******************************************************************************/
+
+static enum CXChildVisitResult
+PrintGCCInlineAssembly(CXCursor cursor, CXCursor p, CXClientData d) {
+  CXString Constraint, Template, Clobber;
+  CXCursor Expr;
+  unsigned hasGoto, i, e;
+  if (clang_getCursorKind(cursor) != CXCursor_AsmStmt)
+    return CXChildVisit_Recurse;
+
+  hasGoto = clang_Cursor_isGCCAssemblyHasGoto(cursor);
+  printf("===ASM TEMPLATE%s===\n", hasGoto ? " (WITH GOTO)" : "");
+  Template = clang_Cursor_getGCCAssemblyTemplate(cursor);
+  printf("%s", clang_getCString(Template));
+  clang_disposeString(Template);
+  printf("\n===ASM TEMPLATE END===\n");
+
+  printf("volatile: %s\n",
+         clang_Cursor_isGCCAssemblyVolatile(cursor) ? "true" : "false");
+
+  for (i = 0, e = clang_Cursor_getGCCAssemblyNumOutputs(cursor); i < e; ++i) {
+    clang_Cursor_getGCCAssemblyOutput(cursor, i, &Constraint, &Expr);
+    printf("Output #%d Constraint (%s): ", i, clang_getCString(Constraint));
+    PrintCursor(Expr, NULL);
+    printf("\n");
+    clang_disposeString(Constraint);
+  }
+  for (i = 0, e = clang_Cursor_getGCCAssemblyNumInputs(cursor); i < e; ++i) {
+    clang_Cursor_getGCCAssemblyInput(cursor, i, &Constraint, &Expr);
+    printf("Input #%d Constraint (%s): ", i, clang_getCString(Constraint));
+    PrintCursor(Expr, NULL);
+    printf("\n");
+    clang_disposeString(Constraint);
+  }
+  for (i = 0, e = clang_Cursor_getGCCAssemblyNumClobbers(cursor); i < e; ++i) {
+    Clobber = clang_Cursor_getGCCAssemblyClobber(cursor, i);
+    printf("Clobber #%d: %s\n", i, clang_getCString(Clobber));
+    clang_disposeString(Clobber);
+  }
+  printf("===ASM END===\n");
+  return CXChildVisit_Recurse;
+}
+
 /******************************************************************************/
 /* Target information testing.                                                */
 /******************************************************************************/
@@ -5010,6 +5055,7 @@ static void print_usage(void) {
     "       c-index-test -test-annotate-tokens=<range> {<args>}*\n"
     "       c-index-test -test-inclusion-stack-source {<args>}*\n"
     "       c-index-test -test-inclusion-stack-tu <AST file>\n");
+  fprintf(stderr, "       c-index-test -test-inline-assembly <AST file>\n");
   fprintf(stderr,
     "       c-index-test -test-print-linkage-source {<args>}*\n"
     "       c-index-test -test-print-visibility {<args>}*\n"
@@ -5167,6 +5213,10 @@ int cindextest_main(int argc, const char **argv) {
   else if (argc > 2 && strstr(argv[1], "-single-symbol-sgf-for=") == argv[1])
     return perform_test_single_symbol_sgf(argv[1], argc - 2, argv + 2);
 
+  if (argc > 2 && strstr(argv[1], "-test-inline-assembly") == argv[1])
+    return perform_test_load_source(argc - 2, argv + 2, "all",
+                                    PrintGCCInlineAssembly, NULL);
+
   print_usage();
   return 1;
 }
diff --git a/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt b/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
index 9962f9850f54..49a95a02d044 100644
--- a/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
+++ b/clang/tools/clang-fuzzer/handle-llvm/CMakeLists.txt
@@ -24,4 +24,5 @@ add_clang_library(clangHandleLLVM
 
   DEPENDS
   intrinsics_gen
+  vt_gen
   )
diff --git a/clang/tools/clang-installapi/ClangInstallAPI.cpp b/clang/tools/clang-installapi/ClangInstallAPI.cpp
index f68236abaa6c..60e9fc4354be 100644
--- a/clang/tools/clang-installapi/ClangInstallAPI.cpp
+++ b/clang/tools/clang-installapi/ClangInstallAPI.cpp
@@ -102,8 +102,8 @@ static bool run(ArrayRef<const char *> Args, const char *ProgName) {
 
   if (!Opts.DriverOpts.DylibToVerify.empty()) {
     TargetList Targets;
-    llvm::for_each(Opts.DriverOpts.Targets,
-                   [&](const auto &T) { Targets.push_back(T.first); });
+    for (const auto &T : Opts.DriverOpts.Targets)
+      Targets.push_back(T.first);
     if (!Ctx.Verifier->verifyBinaryAttrs(Targets, Ctx.BA, Ctx.Reexports,
                                          Opts.LinkerOpts.AllowableClients,
                                          Opts.LinkerOpts.RPaths, Ctx.FT))
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 52b48a34d361..64324a3f8b01 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -100,8 +100,8 @@ getArgListFromJSON(const StringRef Input, llvm::opt::OptTable *Table,
   }
 
   std::vector<const char *> CArgs(Storage.size());
-  llvm::for_each(Storage,
-                 [&CArgs](StringRef Str) { CArgs.emplace_back(Str.data()); });
+  for (StringRef Str : Storage)
+    CArgs.emplace_back(Str.data());
 
   unsigned MissingArgIndex, MissingArgCount;
   return Table->ParseArgs(CArgs, MissingArgIndex, MissingArgCount);
@@ -730,8 +730,8 @@ Options::Options(DiagnosticsEngine &Diag, FileManager *FM,
   // After all InstallAPI necessary arguments have been collected. Go back and
   // assign values that were unknown before the clang driver opt table was used.
   ArchitectureSet AllArchs;
-  llvm::for_each(DriverOpts.Targets,
-                 [&AllArchs](const auto &T) { AllArchs.set(T.first.Arch); });
+  for (const auto &T : DriverOpts.Targets)
+    AllArchs.set(T.first.Arch);
   auto assignDefaultLibAttrs = [&AllArchs](LibAttrs &Attrs) {
     for (auto &[_, Archs] : Attrs.get())
       if (Archs.empty())
@@ -794,8 +794,8 @@ std::pair<LibAttrs, ReexportedInterfaces> Options::getReexportedLibraries() {
   };
 
   PlatformSet Platforms;
-  llvm::for_each(DriverOpts.Targets,
-                 [&](const auto &T) { Platforms.insert(T.first.Platform); });
+  for (const auto &T : DriverOpts.Targets)
+    Platforms.insert(T.first.Platform);
   // Populate search paths by looking at user paths before system ones.
   PathSeq FwkSearchPaths(FEOpts.FwkPaths.begin(), FEOpts.FwkPaths.end());
   for (const PlatformType P : Platforms) {
diff --git a/clang/tools/clang-repl/CMakeLists.txt b/clang/tools/clang-repl/CMakeLists.txt
index f9a911b0ae8e..68d86dd98cac 100644
--- a/clang/tools/clang-repl/CMakeLists.txt
+++ b/clang/tools/clang-repl/CMakeLists.txt
@@ -4,7 +4,9 @@ set( LLVM_LINK_COMPONENTS
   LineEditor
   Option
   OrcJIT
+  OrcShared
   Support
+  TargetParser
   )
 
 add_clang_tool(clang-repl
diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp
index 7af8e4f25d99..54b27782d3db 100644
--- a/clang/tools/clang-repl/ClangRepl.cpp
+++ b/clang/tools/clang-repl/ClangRepl.cpp
@@ -10,7 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Interpreter/RemoteJITUtils.h"
+
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/Version.h"
+#include "clang/Config/config.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Interpreter/CodeCompletion.h"
@@ -24,8 +28,11 @@
 #include "llvm/Support/ManagedStatic.h" // llvm_shutdown
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/TargetParser/Host.h"
 #include <optional>
 
+#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h"
+
 // Disable LSan for this test.
 // FIXME: Re-enable once we can assume GCC 13.2 or higher.
 // https://llvm.org/github.com/llvm/llvm-project/issues/67586.
@@ -34,10 +41,36 @@
 LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; }
 #endif
 
+#define DEBUG_TYPE "clang-repl"
+
 static llvm::cl::opt<bool> CudaEnabled("cuda", llvm::cl::Hidden);
 static llvm::cl::opt<std::string> CudaPath("cuda-path", llvm::cl::Hidden);
 static llvm::cl::opt<std::string> OffloadArch("offload-arch", llvm::cl::Hidden);
-
+static llvm::cl::OptionCategory OOPCategory("Out-of-process Execution Options");
+static llvm::cl::opt<std::string> SlabAllocateSizeString(
+    "slab-allocate",
+    llvm::cl::desc("Allocate from a slab of the given size "
+                   "(allowable suffixes: Kb, Mb, Gb. default = "
+                   "Kb)"),
+    llvm::cl::init(""), llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<std::string>
+    OOPExecutor("oop-executor",
+                llvm::cl::desc("Launch an out-of-process executor to run code"),
+                llvm::cl::init(""), llvm::cl::ValueOptional,
+                llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<std::string> OOPExecutorConnect(
+    "oop-executor-connect",
+    llvm::cl::desc(
+        "Connect to an out-of-process executor through a TCP socket"),
+    llvm::cl::value_desc("<hostname>:<port>"));
+static llvm::cl::opt<std::string>
+    OrcRuntimePath("orc-runtime", llvm::cl::desc("Path to the ORC runtime"),
+                   llvm::cl::init(""), llvm::cl::ValueOptional,
+                   llvm::cl::cat(OOPCategory));
+static llvm::cl::opt<bool> UseSharedMemory(
+    "use-shared-memory",
+    llvm::cl::desc("Use shared memory to transfer generated code and data"),
+    llvm::cl::init(false), llvm::cl::cat(OOPCategory));
 static llvm::cl::list<std::string>
     ClangArgs("Xcc",
               llvm::cl::desc("Argument to pass to the CompilerInvocation"),
@@ -47,6 +80,72 @@ static llvm::cl::opt<bool> OptHostSupportsJit("host-supports-jit",
 static llvm::cl::list<std::string> OptInputs(llvm::cl::Positional,
                                              llvm::cl::desc("[code to run]"));
 
+static llvm::Error sanitizeOopArguments(const char *ArgV0) {
+  // Only one of -oop-executor and -oop-executor-connect can be used.
+  if (!!OOPExecutor.getNumOccurrences() &&
+      !!OOPExecutorConnect.getNumOccurrences())
+    return llvm::make_error<llvm::StringError>(
+        "Only one of -" + OOPExecutor.ArgStr + " and -" +
+            OOPExecutorConnect.ArgStr + " can be specified",
+        llvm::inconvertibleErrorCode());
+
+  llvm::Triple SystemTriple(llvm::sys::getProcessTriple());
+  // TODO: Remove once out-of-process execution support is implemented for
+  // non-Unix platforms.
+  if ((!SystemTriple.isOSBinFormatELF() &&
+       !SystemTriple.isOSBinFormatMachO()) &&
+      (OOPExecutor.getNumOccurrences() ||
+       OOPExecutorConnect.getNumOccurrences()))
+    return llvm::make_error<llvm::StringError>(
+        "Out-of-process execution is only supported on Unix platforms",
+        llvm::inconvertibleErrorCode());
+
+  // If -slab-allocate is passed, check that we're not trying to use it in
+  // -oop-executor or -oop-executor-connect mode.
+  //
+  // FIXME: Remove once we enable remote slab allocation.
+  if (SlabAllocateSizeString != "") {
+    if (OOPExecutor.getNumOccurrences() ||
+        OOPExecutorConnect.getNumOccurrences())
+      return llvm::make_error<llvm::StringError>(
+          "-slab-allocate cannot be used with -oop-executor or "
+          "-oop-executor-connect",
+          llvm::inconvertibleErrorCode());
+  }
+
+  // Out-of-process executors require the ORC runtime.
+  if (OrcRuntimePath.empty() && (OOPExecutor.getNumOccurrences() ||
+                                 OOPExecutorConnect.getNumOccurrences())) {
+    llvm::SmallString<256> BasePath(llvm::sys::fs::getMainExecutable(
+        ArgV0, reinterpret_cast<void *>(&sanitizeOopArguments)));
+    llvm::sys::path::remove_filename(BasePath); // Remove clang-repl filename.
+    llvm::sys::path::remove_filename(BasePath); // Remove ./bin directory.
+    llvm::sys::path::append(BasePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang",
+                            CLANG_VERSION_MAJOR_STRING);
+    if (SystemTriple.isOSBinFormatELF())
+      OrcRuntimePath =
+          BasePath.str().str() + "/lib/x86_64-unknown-linux-gnu/liborc_rt.a";
+    else if (SystemTriple.isOSBinFormatMachO())
+      OrcRuntimePath = BasePath.str().str() + "/lib/darwin/liborc_rt_osx.a";
+    else
+      return llvm::make_error<llvm::StringError>(
+          "Out-of-process execution is not supported on non-unix platforms",
+          llvm::inconvertibleErrorCode());
+  }
+
+  // If -oop-executor was used but no value was specified then use a sensible
+  // default.
+  if (!!OOPExecutor.getNumOccurrences() && OOPExecutor.empty()) {
+    llvm::SmallString<256> OOPExecutorPath(llvm::sys::fs::getMainExecutable(
+        ArgV0, reinterpret_cast<void *>(&sanitizeOopArguments)));
+    llvm::sys::path::remove_filename(OOPExecutorPath);
+    llvm::sys::path::append(OOPExecutorPath, "llvm-jitlink-executor");
+    OOPExecutor = OOPExecutorPath.str().str();
+  }
+
+  return llvm::Error::success();
+}
+
 static void LLVMErrorHandler(void *UserData, const char *Message,
                              bool GenCrashDiag) {
   auto &Diags = *static_cast<clang::DiagnosticsEngine *>(UserData);
@@ -183,6 +282,25 @@ int main(int argc, const char **argv) {
     DeviceCI = ExitOnErr(CB.CreateCudaDevice());
   }
 
+  ExitOnErr(sanitizeOopArguments(argv[0]));
+
+  std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC;
+  if (OOPExecutor.getNumOccurrences()) {
+    // Launch an out-of-process executor locally in a child process.
+    EPC = ExitOnErr(
+        launchExecutor(OOPExecutor, UseSharedMemory, SlabAllocateSizeString));
+  } else if (OOPExecutorConnect.getNumOccurrences()) {
+    EPC = ExitOnErr(connectTCPSocket(OOPExecutorConnect, UseSharedMemory,
+                                     SlabAllocateSizeString));
+  }
+
+  std::unique_ptr<llvm::orc::LLJITBuilder> JB;
+  if (EPC) {
+    CB.SetTargetTriple(EPC->getTargetTriple().getTriple());
+    JB = ExitOnErr(
+        clang::Interpreter::createLLJITBuilder(std::move(EPC), OrcRuntimePath));
+  }
+
   // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It
   // can replace the boilerplate code for creation of the compiler instance.
   std::unique_ptr<clang::CompilerInstance> CI;
@@ -214,6 +332,9 @@ int main(int argc, const char **argv) {
       auto CudaRuntimeLibPath = CudaPath + "/lib/libcudart.so";
       ExitOnErr(Interp->LoadDynamicLibrary(CudaRuntimeLibPath.c_str()));
     }
+  } else if (JB) {
+    Interp =
+        ExitOnErr(clang::Interpreter::create(std::move(CI), std::move(JB)));
   } else
     Interp = ExitOnErr(clang::Interpreter::create(std::move(CI)));
 
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index ce0770a51d65..8b590bd57e1a 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -85,7 +85,6 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make;
 static ScanningOptimizations OptimizeArgs;
 static std::string ModuleFilesDir;
 static bool EagerLoadModules;
-static bool CacheNegativeStats = true;
 static unsigned NumThreads = 0;
 static std::string CompilationDB;
 static std::optional<std::string> ModuleName;
@@ -192,8 +191,6 @@ static void ParseArgs(int argc, char **argv) {
 
   EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
 
-  CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats);
-
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
     StringRef S{A->getValue()};
     if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -396,7 +393,9 @@ class FullDeps {
     ID.FileName = std::string(Input);
     ID.ContextHash = std::move(TUDeps.ID.ContextHash);
     ID.FileDeps = std::move(TUDeps.FileDeps);
-    ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps);
+    ID.NamedModule = std::move(TUDeps.ID.ModuleName);
+    ID.NamedModuleDeps = std::move(TUDeps.NamedModuleDeps);
+    ID.ClangModuleDeps = std::move(TUDeps.ClangModuleDeps);
     ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);
     ID.Commands = std::move(TUDeps.Commands);
 
@@ -511,8 +510,15 @@ class FullDeps {
                   JOS.object([&] {
                     JOS.attribute("clang-context-hash",
                                   StringRef(I.ContextHash));
+                    if (!I.NamedModule.empty())
+                      JOS.attribute("named-module", (I.NamedModule));
+                    if (!I.NamedModuleDeps.empty())
+                      JOS.attributeArray("named-module-deps", [&] {
+                        for (const auto &Dep : I.NamedModuleDeps)
+                          JOS.value(Dep);
+                      });
                     JOS.attributeArray("clang-module-deps",
-                                       toJSONSorted(JOS, I.ModuleDeps));
+                                       toJSONSorted(JOS, I.ClangModuleDeps));
                     JOS.attributeArray("command-line",
                                        toJSONStrings(JOS, Cmd.Arguments));
                     JOS.attribute("executable", StringRef(Cmd.Executable));
@@ -524,8 +530,15 @@ class FullDeps {
               } else {
                 JOS.object([&] {
                   JOS.attribute("clang-context-hash", StringRef(I.ContextHash));
+                  if (!I.NamedModule.empty())
+                    JOS.attribute("named-module", (I.NamedModule));
+                  if (!I.NamedModuleDeps.empty())
+                    JOS.attributeArray("named-module-deps", [&] {
+                      for (const auto &Dep : I.NamedModuleDeps)
+                        JOS.value(Dep);
+                    });
                   JOS.attributeArray("clang-module-deps",
-                                     toJSONSorted(JOS, I.ModuleDeps));
+                                     toJSONSorted(JOS, I.ClangModuleDeps));
                   JOS.attributeArray("command-line",
                                      toJSONStrings(JOS, I.DriverCommandLine));
                   JOS.attribute("executable", "clang");
@@ -580,7 +593,9 @@ class FullDeps {
     std::string FileName;
     std::string ContextHash;
     std::vector<std::string> FileDeps;
-    std::vector<ModuleID> ModuleDeps;
+    std::string NamedModule;
+    std::vector<std::string> NamedModuleDeps;
+    std::vector<ModuleID> ClangModuleDeps;
     std::vector<std::string> DriverCommandLine;
     std::vector<Command> Commands;
   };
@@ -1083,9 +1098,8 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
     });
   };
 
-  DependencyScanningService Service(
-      ScanMode, Format, OptimizeArgs, EagerLoadModules, /*TraceVFS=*/Verbose,
-      llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats);
+  DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
+                                    EagerLoadModules, /*TraceVFS=*/Verbose);
 
   llvm::Timer T;
   T.startTimer();
diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td
index 582ae60851e1..9cccbb3aaf0c 100644
--- a/clang/tools/clang-scan-deps/Opts.td
+++ b/clang/tools/clang-scan-deps/Opts.td
@@ -22,7 +22,6 @@ defm module_files_dir : Eq<"module-files-dir",
 
 def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">;
 def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">;
-def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat failures">;
 
 def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">;
 
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 3068621d9c00..9089984fa4a5 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -270,10 +270,10 @@ bool CursorVisitor::visitFileRegion() {
   ASTUnit *Unit = cxtu::getASTUnit(TU);
   SourceManager &SM = Unit->getSourceManager();
 
-  std::pair<FileID, unsigned> Begin = SM.getDecomposedLoc(
-                                  SM.getFileLoc(RegionOfInterest.getBegin())),
-                              End = SM.getDecomposedLoc(
-                                  SM.getFileLoc(RegionOfInterest.getEnd()));
+  FileIDAndOffset Begin = SM.getDecomposedLoc(
+                      SM.getFileLoc(RegionOfInterest.getBegin())),
+                  End = SM.getDecomposedLoc(
+                      SM.getFileLoc(RegionOfInterest.getEnd()));
 
   if (End.first != Begin.first) {
     // If the end does not reside in the same file, try to recover by
@@ -7636,7 +7636,7 @@ CXString clang_getTokenSpelling(CXTranslationUnit TU, CXToken CXTok) {
     return cxstring::createEmpty();
 
   SourceLocation Loc = SourceLocation::getFromRawEncoding(CXTok.int_data[1]);
-  std::pair<FileID, unsigned> LocInfo =
+  FileIDAndOffset LocInfo =
       CXXUnit->getSourceManager().getDecomposedSpellingLoc(Loc);
   bool Invalid = false;
   StringRef Buffer =
@@ -7680,9 +7680,9 @@ CXSourceRange clang_getTokenExtent(CXTranslationUnit TU, CXToken CXTok) {
 static void getTokens(ASTUnit *CXXUnit, SourceRange Range,
                       SmallVectorImpl<CXToken> &CXTokens) {
   SourceManager &SourceMgr = CXXUnit->getSourceManager();
-  std::pair<FileID, unsigned> BeginLocInfo =
+  FileIDAndOffset BeginLocInfo =
       SourceMgr.getDecomposedSpellingLoc(Range.getBegin());
-  std::pair<FileID, unsigned> EndLocInfo =
+  FileIDAndOffset EndLocInfo =
       SourceMgr.getDecomposedSpellingLoc(Range.getEnd());
 
   // Cannot tokenize across files.
@@ -7761,7 +7761,7 @@ CXToken *clang_getToken(CXTranslationUnit TU, CXSourceLocation Location) {
   if (Begin.isInvalid())
     return nullptr;
   SourceManager &SM = CXXUnit->getSourceManager();
-  std::pair<FileID, unsigned> DecomposedEnd = SM.getDecomposedLoc(Begin);
+  FileIDAndOffset DecomposedEnd = SM.getDecomposedLoc(Begin);
   DecomposedEnd.second +=
       Lexer::MeasureTokenLength(Begin, SM, CXXUnit->getLangOpts());
 
@@ -8410,9 +8410,9 @@ static void annotatePreprocessorTokens(CXTranslationUnit TU,
 
   Preprocessor &PP = CXXUnit->getPreprocessor();
   SourceManager &SourceMgr = CXXUnit->getSourceManager();
-  std::pair<FileID, unsigned> BeginLocInfo =
+  FileIDAndOffset BeginLocInfo =
       SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getBegin());
-  std::pair<FileID, unsigned> EndLocInfo =
+  FileIDAndOffset EndLocInfo =
       SourceMgr.getDecomposedSpellingLoc(RegionOfInterest.getEnd());
 
   if (BeginLocInfo.first != EndLocInfo.first)
@@ -8648,6 +8648,100 @@ void clang_annotateTokens(CXTranslationUnit TU, CXToken *Tokens,
   }
 }
 
+//===----------------------------------------------------------------------===//
+// Operations for querying information of a GCC inline assembly block under a
+// cursor.
+//===----------------------------------------------------------------------===//
+CXString clang_Cursor_getGCCAssemblyTemplate(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return cxstring::createEmpty();
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor))) {
+    ASTContext const &C = getCursorContext(Cursor);
+    std::string AsmTemplate = S->generateAsmString(C);
+    return cxstring::createDup(AsmTemplate);
+  }
+  return cxstring::createEmpty();
+}
+
+unsigned clang_Cursor_isGCCAssemblyHasGoto(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor)))
+    return S->isAsmGoto();
+  return 0;
+}
+
+unsigned clang_Cursor_getGCCAssemblyNumOutputs(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor)))
+    return S->getNumOutputs();
+  return 0;
+}
+
+unsigned clang_Cursor_getGCCAssemblyNumInputs(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor)))
+    return S->getNumInputs();
+  return 0;
+}
+
+unsigned clang_Cursor_getGCCAssemblyInput(CXCursor Cursor, unsigned Index,
+                                          CXString *Constraint,
+                                          CXCursor *ExprCursor) {
+  if (!clang_isStatement(Cursor.kind) || !Constraint || !ExprCursor)
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor));
+      S && Index < S->getNumInputs()) {
+    *Constraint = cxstring::createDup(S->getInputConstraint(Index));
+    *ExprCursor = MakeCXCursor(S->getInputExpr(Index), getCursorDecl(Cursor),
+                               cxcursor::getCursorTU(Cursor));
+    return 1;
+  }
+  return 0;
+}
+
+unsigned clang_Cursor_getGCCAssemblyOutput(CXCursor Cursor, unsigned Index,
+                                           CXString *Constraint,
+                                           CXCursor *ExprCursor) {
+  if (!clang_isStatement(Cursor.kind) || !Constraint || !ExprCursor)
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor));
+      S && Index < S->getNumOutputs()) {
+    *Constraint = cxstring::createDup(S->getOutputConstraint(Index));
+    *ExprCursor = MakeCXCursor(S->getOutputExpr(Index), getCursorDecl(Cursor),
+                               cxcursor::getCursorTU(Cursor));
+    return 1;
+  }
+  return 0;
+}
+
+unsigned clang_Cursor_getGCCAssemblyNumClobbers(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor)))
+    return S->getNumClobbers();
+  return 0;
+}
+
+CXString clang_Cursor_getGCCAssemblyClobber(CXCursor Cursor, unsigned Index) {
+  if (!clang_isStatement(Cursor.kind))
+    return cxstring::createEmpty();
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor));
+      S && Index < S->getNumClobbers())
+    return cxstring::createDup(S->getClobber(Index));
+  return cxstring::createEmpty();
+}
+
+unsigned clang_Cursor_isGCCAssemblyVolatile(CXCursor Cursor) {
+  if (!clang_isStatement(Cursor.kind))
+    return 0;
+  if (auto const *S = dyn_cast_or_null<GCCAsmStmt>(getCursorStmt(Cursor)))
+    return S->isVolatile();
+  return 0;
+}
+
 //===----------------------------------------------------------------------===//
 // Operations for querying linkage of a cursor.
 //===----------------------------------------------------------------------===//
diff --git a/clang/tools/libclang/CIndexHigh.cpp b/clang/tools/libclang/CIndexHigh.cpp
index c055ff91da55..1f6d3b9dcfe3 100644
--- a/clang/tools/libclang/CIndexHigh.cpp
+++ b/clang/tools/libclang/CIndexHigh.cpp
@@ -193,7 +193,7 @@ static enum CXChildVisitResult findFileIdRefVisit(CXCursor cursor,
     }
 
     // We are looking for identifiers in a specific file.
-    std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+    FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
     if (LocInfo.first != data->FID)
       return CXChildVisit_Recurse;
 
@@ -288,7 +288,7 @@ static enum CXChildVisitResult findFileMacroRefVisit(CXCursor cursor,
   }
 
   // We are looking for identifiers in a specific file.
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
   if (SM.getFileEntryForID(LocInfo.first) != data->File)
     return CXChildVisit_Continue;
 
@@ -360,7 +360,7 @@ struct FindFileIncludesVisitor {
     SourceManager &SM = Ctx.getSourceManager();
 
     // We are looking for includes in a specific file.
-    std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+    FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
     if (SM.getFileEntryForID(LocInfo.first) != File)
       return CXChildVisit_Continue;
 
diff --git a/clang/tools/libclang/CXIndexDataConsumer.cpp b/clang/tools/libclang/CXIndexDataConsumer.cpp
index 2b2e70d60d1d..f0d92e8c4012 100644
--- a/clang/tools/libclang/CXIndexDataConsumer.cpp
+++ b/clang/tools/libclang/CXIndexDataConsumer.cpp
@@ -1011,8 +1011,8 @@ bool CXIndexDataConsumer::markEntityOccurrenceInFile(const NamedDecl *D,
 
   SourceManager &SM = Ctx->getSourceManager();
   D = getEntityDecl(D);
-  
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SM.getFileLoc(Loc));
+
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(SM.getFileLoc(Loc));
   FileID FID = LocInfo.first;
   if (FID.isInvalid())
     return true;
@@ -1093,7 +1093,7 @@ void CXIndexDataConsumer::translateLoc(SourceLocation Loc,
   SourceManager &SM = Ctx->getSourceManager();
   Loc = SM.getFileLoc(Loc);
 
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc);
   FileID FID = LocInfo.first;
   unsigned FileOffset = LocInfo.second;
 
diff --git a/clang/tools/libclang/CXSourceLocation.cpp b/clang/tools/libclang/CXSourceLocation.cpp
index cd41e9f0d640..4c16e5de4498 100644
--- a/clang/tools/libclang/CXSourceLocation.cpp
+++ b/clang/tools/libclang/CXSourceLocation.cpp
@@ -332,7 +332,7 @@ void clang_getSpellingLocation(CXSourceLocation location,
   const SourceManager &SM =
   *static_cast<const SourceManager*>(location.ptr_data[0]);
   SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellLoc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(SpellLoc);
   FileID FID = LocInfo.first;
   unsigned FileOffset = LocInfo.second;
   
@@ -368,7 +368,7 @@ void clang_getFileLocation(CXSourceLocation location,
   const SourceManager &SM =
   *static_cast<const SourceManager*>(location.ptr_data[0]);
   SourceLocation FileLoc = SM.getFileLoc(Loc);
-  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(FileLoc);
+  FileIDAndOffset LocInfo = SM.getDecomposedLoc(FileLoc);
   FileID FID = LocInfo.first;
   unsigned FileOffset = LocInfo.second;
 
diff --git a/clang/tools/libclang/Indexing.cpp b/clang/tools/libclang/Indexing.cpp
index 3cd49bf90235..32a7147af338 100644
--- a/clang/tools/libclang/Indexing.cpp
+++ b/clang/tools/libclang/Indexing.cpp
@@ -389,9 +389,7 @@ class IndexingFrontendAction : public ASTFrontendAction {
     if (SM.isInSystemHeader(Loc))
       return true; // always skip bodies from system headers.
 
-    FileID FID;
-    unsigned Offset;
-    std::tie(FID, Offset) = SM.getDecomposedLoc(Loc);
+    auto [FID, Offset] = SM.getDecomposedLoc(Loc);
     // Don't skip bodies from main files; this may be revisited.
     if (SM.getMainFileID() == FID)
       return false;
diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map
index f08d13c3da9e..d140a71e771a 100644
--- a/clang/tools/libclang/libclang.map
+++ b/clang/tools/libclang/libclang.map
@@ -441,6 +441,15 @@ LLVM_20 {
 LLVM_21 {
   global:
     clang_getFullyQualifiedName;
+    clang_Cursor_getGCCAssemblyTemplate;
+    clang_Cursor_isGCCAssemblyHasGoto;
+    clang_Cursor_getGCCAssemblyNumOutputs;
+    clang_Cursor_getGCCAssemblyNumInputs;
+    clang_Cursor_getGCCAssemblyInput;
+    clang_Cursor_getGCCAssemblyOutput;
+    clang_Cursor_getGCCAssemblyNumClobbers;
+    clang_Cursor_getGCCAssemblyClobber;
+    clang_Cursor_isGCCAssemblyVolatile;
 };
 
 # Example of how to add a new symbol version entry.  If you do add a new symbol
diff --git a/clang/unittests/AST/ASTTraverserTest.cpp b/clang/unittests/AST/ASTTraverserTest.cpp
index 8b6e3e90c0ea..988e81d8e51d 100644
--- a/clang/unittests/AST/ASTTraverserTest.cpp
+++ b/clang/unittests/AST/ASTTraverserTest.cpp
@@ -28,6 +28,10 @@ class NodeTreePrinter : public TextTreeStructure {
       : TextTreeStructure(OS, /* showColors */ false), OS(OS) {}
 
   void Visit(const Decl *D) {
+    if (!D) {
+      OS << "<<<NULL>>>";
+      return;
+    }
     OS << D->getDeclKindName() << "Decl";
     if (auto *ND = dyn_cast<NamedDecl>(D)) {
       OS << " '" << ND->getDeclName() << "'";
@@ -1932,4 +1936,75 @@ CXXRewrittenBinaryOperator
   }
 }
 
+TEST(Traverse, CatchStatements) {
+
+  auto AST = buildASTFromCode(R"cpp(
+void test()
+{
+  try
+  {
+    int a;
+  }
+  catch (...)
+  {
+    int b;
+  }
+
+  try
+  {
+    int a;
+  }
+  catch (const int&)
+  {
+    int b;
+  }
+}
+)cpp");
+
+  auto BN =
+      ast_matchers::match(cxxCatchStmt().bind("catch"), AST->getASTContext());
+  EXPECT_EQ(BN.size(), 2u);
+  const auto *catchWithoutDecl = BN[0].getNodeAs<Stmt>("catch");
+
+  llvm::StringRef Expected = R"cpp(
+CXXCatchStmt
+|-<<<NULL>>>
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_AsIs, catchWithoutDecl), Expected);
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-<<<NULL>>>
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource, catchWithoutDecl),
+            Expected);
+
+  const auto *catchWithDecl = BN[1].getNodeAs<Stmt>("catch");
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-VarDecl ''
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_AsIs, catchWithDecl), Expected);
+
+  Expected = R"cpp(
+CXXCatchStmt
+|-VarDecl ''
+`-CompoundStmt
+  `-DeclStmt
+    `-VarDecl 'b'
+)cpp";
+  EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource, catchWithDecl),
+            Expected);
+}
+
 } // namespace clang
diff --git a/clang/unittests/AllClangUnitTests.cpp b/clang/unittests/AllClangUnitTests.cpp
new file mode 100644
index 000000000000..6f33230a617c
--- /dev/null
+++ b/clang/unittests/AllClangUnitTests.cpp
@@ -0,0 +1,28 @@
+//===- clang/unittests/AllClangUnitTests.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/TargetSelect.h"
+
+namespace {
+struct RegisterAllLLVMTargets {
+  RegisterAllLLVMTargets();
+} gv;
+} // namespace
+
+// This dynamic initializer initializes all layers (TargetInfo, MC, CodeGen,
+// AsmPrinter, etc) of all LLVM targets. This matches what cc1_main does on
+// startup, and prevents tests from initializing some of the Target layers,
+// which can interfere with tests that assume that lower target layers are
+// registered if the TargetInfo is registered.
+RegisterAllLLVMTargets::RegisterAllLLVMTargets() {
+  llvm::InitializeAllTargetInfos();
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllAsmPrinters();
+  llvm::InitializeAllAsmParsers();
+}
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index aef28f914b64..54c781a35c20 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -117,6 +117,7 @@ get_property(LINK_LIBS GLOBAL PROPERTY CLANG_UNITTEST_LINK_LIBS)
 get_property(LLVM_COMPONENTS GLOBAL PROPERTY CLANG_UNITTEST_LLVM_COMPONENTS)
 add_distinct_clang_unittest(AllClangUnitTests
   ${SRCS}
+  AllClangUnitTests.cpp
   CLANG_LIBS
   ${CLANG_LIBS}
   LINK_LIBS
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index c0633ba3c29b..944e7c3fb152 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -25768,6 +25768,21 @@ TEST_F(FormatTest, OperatorPassedAsAFunctionPtr) {
   verifyFormat("foo(operator, , -42);", Style);
 }
 
+TEST_F(FormatTest, LineSpliceWithTrailingWhitespace) {
+  auto Style = getLLVMStyle();
+  Style.AlignEscapedNewlines = FormatStyle::ENAS_DontAlign;
+  Style.UseTab = FormatStyle::UT_Never;
+
+  verifyFormat("int i;", "  \\  \n"
+                         "  int i;");
+  verifyFormat("#define FOO(args) \\\n"
+               "  struct a {};",
+               "#define FOO( args )   \\   \n"
+               "struct a{\\\t\t\t\n"
+               "  };",
+               Style);
+}
+
 TEST_F(FormatTest, WhitespaceSensitiveMacros) {
   FormatStyle Style = getLLVMStyle();
   Style.WhitespaceSensitiveMacros.push_back("FOO");
@@ -27938,13 +27953,6 @@ TEST_F(FormatTest, AlignArrayOfStructuresLeftAlignmentNonSquare) {
                "};",
                Style);
   verifyFormat("void foo() {\n"
-               "  auto thing = test{\n"
-               "      {\n"
-               "       {13}, {something}, // A\n"
-               "      }\n"
-               "  };\n"
-               "}",
-               "void foo() {\n"
                "  auto thing = test{\n"
                "      {\n"
                "       {13},\n"
@@ -28002,13 +28010,6 @@ TEST_F(FormatTest, AlignArrayOfStructuresRightAlignmentNonSquare) {
                "};",
                Style);
   verifyFormat("void foo() {\n"
-               "  auto thing = test{\n"
-               "      {\n"
-               "       {13}, {something}, // A\n"
-               "      }\n"
-               "  };\n"
-               "}",
-               "void foo() {\n"
                "  auto thing = test{\n"
                "      {\n"
                "       {13},\n"
diff --git a/clang/unittests/Format/FormatTestBase.h b/clang/unittests/Format/FormatTestBase.h
index 9d9472964fd3..5a78499228f1 100644
--- a/clang/unittests/Format/FormatTestBase.h
+++ b/clang/unittests/Format/FormatTestBase.h
@@ -137,7 +137,7 @@ class FormatTestBase : public testing::Test {
     _verifyFormat(File, Line, Code, Code, Style);
   }
 
-  /// \brief Verify that clang-format does not crash on the given input.
+  /// Verify that clang-format does not crash on the given input.
   void verifyNoCrash(StringRef Code,
                      const std::optional<FormatStyle> &Style = {}) {
     format(Code, Style, SC_DoNotCheck);
diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp
index 5eefd767706a..a16fbffb7627 100644
--- a/clang/unittests/Format/FormatTestComments.cpp
+++ b/clang/unittests/Format/FormatTestComments.cpp
@@ -2486,7 +2486,7 @@ TEST_F(FormatTestComments, BlockComments) {
   EXPECT_EQ("/*\n"
             "**\n"
             "* aaaaaa\n"
-            "*aaaaaa\n"
+            "* aaaaaa\n"
             "*/",
             format("/*\n"
                    "**\n"
diff --git a/clang/unittests/Format/FormatTestSelective.cpp b/clang/unittests/Format/FormatTestSelective.cpp
index 624684c7a079..0b7ac21fd33d 100644
--- a/clang/unittests/Format/FormatTestSelective.cpp
+++ b/clang/unittests/Format/FormatTestSelective.cpp
@@ -41,6 +41,11 @@ TEST_F(FormatTestSelective, RemovesTrailingWhitespaceOfFormattedLine) {
   EXPECT_EQ("int a;", format("int a;         ", 0, 0));
   EXPECT_EQ("int a;\n", format("int a;  \n   \n   \n ", 0, 0));
   EXPECT_EQ("int a;\nint b;    ", format("int a;  \nint b;    ", 0, 0));
+
+  EXPECT_EQ("void f() {}", format("void f() {\n"
+                                  "  \n"
+                                  "}",
+                                  11, 0));
 }
 
 TEST_F(FormatTestSelective, FormatsCorrectRegionForLeadingWhitespace) {
diff --git a/clang/unittests/Format/QualifierFixerTest.cpp b/clang/unittests/Format/QualifierFixerTest.cpp
index 3eae39f267c3..f42f2e307f71 100644
--- a/clang/unittests/Format/QualifierFixerTest.cpp
+++ b/clang/unittests/Format/QualifierFixerTest.cpp
@@ -1122,14 +1122,17 @@ TEST_F(QualifierFixerTest, IsQualifierType) {
 }
 
 TEST_F(QualifierFixerTest, IsMacro) {
-
   auto Tokens = annotate("INT INTPR Foo int");
   ASSERT_EQ(Tokens.size(), 5u) << Tokens;
-
   EXPECT_TRUE(isPossibleMacro(Tokens[0]));
   EXPECT_TRUE(isPossibleMacro(Tokens[1]));
   EXPECT_FALSE(isPossibleMacro(Tokens[2]));
   EXPECT_FALSE(isPossibleMacro(Tokens[3]));
+
+  Tokens = annotate("FOO::BAR");
+  ASSERT_EQ(Tokens.size(), 4u) << Tokens;
+  EXPECT_FALSE(isPossibleMacro(Tokens[0]));
+  EXPECT_FALSE(isPossibleMacro(Tokens[2]));
 }
 
 TEST_F(QualifierFixerTest, OverlappingQualifier) {
diff --git a/clang/unittests/Index/IndexTests.cpp b/clang/unittests/Index/IndexTests.cpp
index 8e9a1c6bf882..05ce448237fd 100644
--- a/clang/unittests/Index/IndexTests.cpp
+++ b/clang/unittests/Index/IndexTests.cpp
@@ -35,9 +35,7 @@ struct Position {
 
   static Position fromSourceLocation(SourceLocation Loc,
                                      const SourceManager &SM) {
-    FileID FID;
-    unsigned Offset;
-    std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc);
+    auto [FID, Offset] = SM.getDecomposedSpellingLoc(Loc);
     Position P;
     P.Line = SM.getLineNumber(FID, Offset);
     P.Column = SM.getColumnNumber(FID, Offset);
diff --git a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
index 61f74929c1e9..d2ef27155df9 100644
--- a/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesScannerTest.cpp
@@ -880,6 +880,37 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
   EXPECT_EQ(pp_eof, Directives[22].Kind);
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest,
+     TestFixedBugThatReportUnterminatedDirectiveFalsely) {
+  SmallVector<char, 512> Out;
+  SmallVector<dependency_directives_scan::Token, 16> Tokens;
+  SmallVector<Directive, 16> Directives;
+
+  StringRef Input = "#ifndef __TEST \n"
+                    "#define __TEST \n"
+                    "#if defined(__TEST_DUMMY) \n"
+                    "#if defined(__TEST_DUMMY2) \n"
+                    "#pragma GCC warning        \\  \n"
+                    "\"hello!\"\n"
+                    "#else\n"
+                    "#pragma GCC error          \\  \n"
+                    "\"world!\" \n"
+                    "#endif // defined(__TEST_DUMMY2) \n"
+                    "#endif  // defined(__TEST_DUMMY) \n"
+                    "#endif // #ifndef __TEST \n";
+  ASSERT_FALSE( // False on no error:
+      minimizeSourceToDependencyDirectives(Input, Out, Tokens, Directives));
+  ASSERT_TRUE(Directives.size() == 8);
+  EXPECT_EQ(pp_ifndef, Directives[0].Kind);
+  EXPECT_EQ(pp_define, Directives[1].Kind);
+  EXPECT_EQ(pp_if, Directives[2].Kind);
+  EXPECT_EQ(pp_if, Directives[3].Kind);
+  EXPECT_EQ(pp_endif, Directives[4].Kind);
+  EXPECT_EQ(pp_endif, Directives[5].Kind);
+  EXPECT_EQ(pp_endif, Directives[6].Kind);
+  EXPECT_EQ(pp_eof, Directives[7].Kind);
+}
+
 TEST(MinimizeSourceToDependencyDirectivesTest, PoundWarningAndError) {
   SmallVector<char, 128> Out;
 
diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp
index 33c8abbec35a..86df872f6b7d 100644
--- a/clang/unittests/Lex/LexerTest.cpp
+++ b/clang/unittests/Lex/LexerTest.cpp
@@ -49,8 +49,7 @@ class LexerTest : public ::testing::Test {
   }
 
   std::unique_ptr<Preprocessor> CreatePP(StringRef Source,
-                                         TrivialModuleLoader &ModLoader,
-                                         StringRef PreDefines = {}) {
+                                         TrivialModuleLoader &ModLoader) {
     std::unique_ptr<llvm::MemoryBuffer> Buf =
         llvm::MemoryBuffer::getMemBuffer(Source);
     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
@@ -63,7 +62,7 @@ class LexerTest : public ::testing::Test {
         /*IILookup =*/nullptr,
         /*OwnsHeaderSearch =*/false);
     if (!PreDefines.empty())
-      PP->setPredefines(PreDefines.str());
+      PP->setPredefines(PreDefines);
     PP->Initialize(*Target);
     PP->EnterMainSourceFile();
     return PP;
@@ -111,6 +110,7 @@ class LexerTest : public ::testing::Test {
   std::shared_ptr<TargetOptions> TargetOpts;
   IntrusiveRefCntPtr<TargetInfo> Target;
   std::unique_ptr<Preprocessor> PP;
+  std::string PreDefines;
 };
 
 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
@@ -522,15 +522,14 @@ TEST_F(LexerTest, GetBeginningOfTokenWithEscapedNewLine) {
   std::vector<Token> LexedTokens = CheckLex(TextToLex, ExpectedTokens);
 
   for (const Token &Tok : LexedTokens) {
-    std::pair<FileID, unsigned> OriginalLocation =
+    FileIDAndOffset OriginalLocation =
         SourceMgr.getDecomposedLoc(Tok.getLocation());
     for (unsigned Offset = 0; Offset < IdentifierLength; ++Offset) {
       SourceLocation LookupLocation =
           Tok.getLocation().getLocWithOffset(Offset);
 
-      std::pair<FileID, unsigned> FoundLocation =
-          SourceMgr.getDecomposedExpansionLoc(
-              Lexer::GetBeginningOfToken(LookupLocation, SourceMgr, LangOpts));
+      FileIDAndOffset FoundLocation = SourceMgr.getDecomposedExpansionLoc(
+          Lexer::GetBeginningOfToken(LookupLocation, SourceMgr, LangOpts));
 
       // Check that location returned by the GetBeginningOfToken
       // is the same as original token location reported by Lexer.
@@ -773,6 +772,7 @@ TEST(LexerPreambleTest, PreambleBounds) {
 }
 
 TEST_F(LexerTest, CheckFirstPPToken) {
+  LangOpts.CPlusPlusModules = true;
   {
     TrivialModuleLoader ModLoader;
     auto PP = CreatePP("// This is a comment\n"
@@ -781,9 +781,8 @@ TEST_F(LexerTest, CheckFirstPPToken) {
     Token Tok;
     PP->Lex(Tok);
     EXPECT_TRUE(Tok.is(tok::kw_int));
-    EXPECT_TRUE(PP->hasSeenMainFileFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().isFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().is(tok::kw_int));
+    EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc().isValid());
+    EXPECT_EQ(PP->getMainFileFirstPPTokenLoc(), Tok.getLocation());
   }
   {
     TrivialModuleLoader ModLoader;
@@ -794,24 +793,28 @@ TEST_F(LexerTest, CheckFirstPPToken) {
     Token Tok;
     PP->Lex(Tok);
     EXPECT_TRUE(Tok.is(tok::kw_int));
-    EXPECT_TRUE(PP->hasSeenMainFileFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().isFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().is(tok::hash));
+    EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,
+                                    PP->getSourceManager(), PP->getLangOpts(),
+                                    /*IgnoreWhiteSpace=*/false));
+    EXPECT_TRUE(Tok.isFirstPPToken());
+    EXPECT_TRUE(Tok.is(tok::hash));
   }
 
   {
+    PreDefines = "#define FOO int\n";
     TrivialModuleLoader ModLoader;
     auto PP = CreatePP("// This is a comment\n"
                        "FOO a;",
-                       ModLoader, "#define FOO int\n");
+                       ModLoader);
     Token Tok;
     PP->Lex(Tok);
     EXPECT_TRUE(Tok.is(tok::kw_int));
-    EXPECT_TRUE(PP->hasSeenMainFileFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().isFirstPPToken());
-    EXPECT_TRUE(PP->getMainFileFirstPPToken().is(tok::identifier));
-    EXPECT_TRUE(
-        PP->getMainFileFirstPPToken().getIdentifierInfo()->isStr("FOO"));
+    EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,
+                                    PP->getSourceManager(), PP->getLangOpts(),
+                                    /*IgnoreWhiteSpace=*/false));
+    EXPECT_TRUE(Tok.isFirstPPToken());
+    EXPECT_TRUE(Tok.is(tok::raw_identifier));
+    EXPECT_TRUE(Tok.getRawIdentifier() == "FOO");
   }
 }
 } // anonymous namespace
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 1e46ee35d5d4..871f12ef3cce 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -29,6 +29,8 @@ using namespace llvm::hlsl::rootsig;
 
 namespace {
 
+using llvm::dxbc::RootSignatureVersion;
+
 // Diagnostic helper for helper tests
 class ExpectedDiagConsumer : public DiagnosticConsumer {
   virtual void anchor() {}
@@ -115,7 +117,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -127,6 +130,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseEmptyTest) {
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
+  using llvm::dxbc::DescriptorRangeFlags;
   const llvm::StringLiteral Source = R"cc(
     DescriptorTable(
       CBV(b0),
@@ -148,7 +152,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -205,20 +210,27 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Space, 0u);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Offset,
             DescriptorTableOffsetAppend);
+  auto ValidDescriptorRangeFlags =
+      DescriptorRangeFlags::DescriptorsVolatile |
+      DescriptorRangeFlags::DataVolatile |
+      DescriptorRangeFlags::DataStaticWhileSetAtExecute |
+      DescriptorRangeFlags::DataStatic |
+      DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks;
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
-            DescriptorRangeFlags::ValidFlags);
+            ValidDescriptorRangeFlags);
 
   Elem = Elements[4];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, (uint32_t)4);
   ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility,
-            ShaderVisibility::Pixel);
+            llvm::dxbc::ShaderVisibility::Pixel);
 
   // Empty Descriptor Table
   Elem = Elements[5];
   ASSERT_TRUE(std::holds_alternative<DescriptorTable>(Elem));
   ASSERT_EQ(std::get<DescriptorTable>(Elem).NumClauses, 0u);
-  ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<DescriptorTable>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -244,7 +256,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -258,19 +271,25 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_TRUE(std::holds_alternative<StaticSampler>(Elem));
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter, SamplerFilter::Anisotropic);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Wrap);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Wrap);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Filter,
+            llvm::dxbc::SamplerFilter::Anisotropic);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU,
+            llvm::dxbc::TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV,
+            llvm::dxbc::TextureAddressMode::Wrap);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW,
+            llvm::dxbc::TextureAddressMode::Wrap);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MipLODBias, 0.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).MaxAnisotropy, 16u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc, ComparisonFunc::LessEqual);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc,
+            llvm::dxbc::ComparisonFunc::LessEqual);
   ASSERT_EQ(std::get<StaticSampler>(Elem).BorderColor,
-            StaticBorderColor::OpaqueWhite);
+            llvm::dxbc::StaticBorderColor::OpaqueWhite);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MinLOD, 0.f);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MaxLOD, 3.402823466e+38f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   // Check values can be set as expected
   Elem = Elements[1];
@@ -278,19 +297,24 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseStaticSamplerTest) {
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.ViewType, RegisterType::SReg);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Filter,
-            SamplerFilter::MaximumMinPointMagLinearMipPoint);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU, TextureAddressMode::Mirror);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV, TextureAddressMode::Border);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW, TextureAddressMode::Clamp);
+            llvm::dxbc::SamplerFilter::MaximumMinPointMagLinearMipPoint);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressU,
+            llvm::dxbc::TextureAddressMode::Mirror);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressV,
+            llvm::dxbc::TextureAddressMode::Border);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).AddressW,
+            llvm::dxbc::TextureAddressMode::Clamp);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MipLODBias, 230.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).MaxAnisotropy, 3u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc, ComparisonFunc::NotEqual);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).CompFunc,
+            llvm::dxbc::ComparisonFunc::NotEqual);
   ASSERT_EQ(std::get<StaticSampler>(Elem).BorderColor,
-            StaticBorderColor::OpaqueBlackUint);
+            llvm::dxbc::StaticBorderColor::OpaqueBlackUint);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MinLOD, 4.2f);
   ASSERT_FLOAT_EQ(std::get<StaticSampler>(Elem).MaxLOD, 9000.f);
   ASSERT_EQ(std::get<StaticSampler>(Elem).Space, 4u);
-  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility, ShaderVisibility::Domain);
+  ASSERT_EQ(std::get<StaticSampler>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Domain);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -318,7 +342,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseFloatsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -393,7 +418,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidSamplerFlagsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -403,8 +429,9 @@ TEST_F(ParseHLSLRootSignatureTest, ValidSamplerFlagsTest) {
   RootElement Elem = Elements[0];
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
-            DescriptorRangeFlags::ValidSamplerFlags);
+  auto ValidSamplerFlags =
+      llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile;
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidSamplerFlags);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
@@ -423,7 +450,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -438,7 +466,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootConstants>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
 
   Elem = Elements[1];
   ASSERT_TRUE(std::holds_alternative<RootConstants>(Elem));
@@ -446,12 +475,14 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootConsantsTest) {
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootConstants>(Elem).Reg.Number, 42u);
   ASSERT_EQ(std::get<RootConstants>(Elem).Space, 3u);
-  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility, ShaderVisibility::Hull);
+  ASSERT_EQ(std::get<RootConstants>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Hull);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
+  using llvm::dxbc::RootFlags;
   const llvm::StringLiteral Source = R"cc(
     RootFlags(),
     RootFlags(0),
@@ -477,7 +508,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -496,12 +528,25 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootFlagsTest) {
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<RootFlags>(Elem));
-  ASSERT_EQ(std::get<RootFlags>(Elem), RootFlags::ValidFlags);
+  auto ValidRootFlags = RootFlags::AllowInputAssemblerInputLayout |
+                        RootFlags::DenyVertexShaderRootAccess |
+                        RootFlags::DenyHullShaderRootAccess |
+                        RootFlags::DenyDomainShaderRootAccess |
+                        RootFlags::DenyGeometryShaderRootAccess |
+                        RootFlags::DenyPixelShaderRootAccess |
+                        RootFlags::AllowStreamOutput |
+                        RootFlags::LocalRootSignature |
+                        RootFlags::DenyAmplificationShaderRootAccess |
+                        RootFlags::DenyMeshShaderRootAccess |
+                        RootFlags::CBVSRVUAVHeapDirectlyIndexed |
+                        RootFlags::SamplerHeapDirectlyIndexed;
+  ASSERT_EQ(std::get<RootFlags>(Elem), ValidRootFlags);
 
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
 TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
+  using llvm::dxbc::RootDescriptorFlags;
   const llvm::StringLiteral Source = R"cc(
     CBV(b0),
     SRV(space = 4, t42, visibility = SHADER_VISIBILITY_GEOMETRY,
@@ -517,7 +562,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
@@ -532,7 +578,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             RootDescriptorFlags::DataStaticWhileSetAtExecute);
 
@@ -543,9 +590,12 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 42u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 4u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
-            ShaderVisibility::Geometry);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
-            RootDescriptorFlags::ValidFlags);
+            llvm::dxbc::ShaderVisibility::Geometry);
+  auto ValidRootDescriptorFlags =
+      RootDescriptorFlags::DataVolatile |
+      RootDescriptorFlags::DataStaticWhileSetAtExecute |
+      RootDescriptorFlags::DataStatic;
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, ValidRootDescriptorFlags);
 
   Elem = Elements[2];
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
@@ -553,7 +603,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::UReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 34893247u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::Hull);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::Hull);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             RootDescriptorFlags::DataVolatile);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
@@ -564,7 +615,8 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility, ShaderVisibility::All);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Visibility,
+            llvm::dxbc::ShaderVisibility::All);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, RootDescriptorFlags::None);
 
   ASSERT_TRUE(Consumer->isSatisfied());
@@ -585,13 +637,160 @@ TEST_F(ParseHLSLRootSignatureTest, ValidTrailingCommaTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
+
+  // Test no diagnostics produced
+  Consumer->setNoDiag();
+
+  ASSERT_FALSE(Parser.parse());
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, ValidVersion10Test) {
+  // This test checks that the default values are set correctly
+  // when parsing with root signature version 1.0
+  const llvm::StringLiteral Source = R"cc(
+    CBV(b0),
+    SRV(t0),
+    UAV(u0),
+    DescriptorTable(
+      CBV(b1),
+      SRV(t1),
+      UAV(u1),
+      Sampler(s1),
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_0, Elements, Lexer,
+                                   *PP);
+
+  // Test no diagnostics produced
+  Consumer->setNoDiag();
+
+  ASSERT_FALSE(Parser.parse());
+
+  auto DefRootDescriptorFlag = llvm::dxbc::RootDescriptorFlags::DataVolatile;
+  RootElement Elem = Elements[0];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  Elem = Elements[1];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  Elem = Elements[2];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
+
+  auto ValidNonSamplerFlags =
+      llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile |
+      llvm::dxbc::DescriptorRangeFlags::DataVolatile;
+  Elem = Elements[3];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[4];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[5];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
+
+  Elem = Elements[6];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile);
+
+  ASSERT_TRUE(Consumer->isSatisfied());
+}
+
+TEST_F(ParseHLSLRootSignatureTest, ValidVersion11Test) {
+  // This test checks that the default values are set correctly
+  // when parsing with root signature version 1.1
+  const llvm::StringLiteral Source = R"cc(
+    CBV(b0),
+    SRV(t0),
+    UAV(u0),
+    DescriptorTable(
+      CBV(b1),
+      SRV(t1),
+      UAV(u1),
+      Sampler(s1),
+    )
+  )cc";
+
+  TrivialModuleLoader ModLoader;
+  auto PP = createPP(Source, ModLoader);
+  auto TokLoc = SourceLocation();
+
+  hlsl::RootSignatureLexer Lexer(Source, TokLoc);
+  SmallVector<RootElement> Elements;
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test no diagnostics produced
   Consumer->setNoDiag();
 
   ASSERT_FALSE(Parser.parse());
 
+  RootElement Elem = Elements[0];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[1];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[2];
+  ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
+            llvm::dxbc::RootDescriptorFlags::DataVolatile);
+
+  Elem = Elements[3];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[4];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
+
+  Elem = Elements[5];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::DataVolatile);
+
+  Elem = Elements[6];
+  ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
+            llvm::dxbc::DescriptorRangeFlags::None);
+
   ASSERT_TRUE(Consumer->isSatisfied());
 }
 
@@ -609,7 +808,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedTokenTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_unexpected_end_of_params);
@@ -629,7 +829,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseInvalidTokenTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced - invalid token
   Consumer->setExpected(diag::err_hlsl_unexpected_end_of_params);
@@ -649,7 +850,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseUnexpectedEndOfStreamTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced - end of stream
   Consumer->setExpected(diag::err_expected_after);
@@ -674,7 +876,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingDTParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -696,7 +899,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingRDParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -718,7 +922,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidMissingRCParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_missing_param);
@@ -742,7 +947,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryDTParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -764,7 +970,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedMandatoryRCParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -788,7 +995,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalDTParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -814,7 +1022,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidRepeatedOptionalRCParameterTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_repeat_param);
@@ -837,7 +1046,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedNumberTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -859,7 +1069,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidParseOverflowedNegativeNumberTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -880,7 +1091,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedFloatTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -901,7 +1113,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexNegOverflowedFloatTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -922,7 +1135,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexOverflowedDoubleTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_overflow);
@@ -943,7 +1157,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidLexUnderflowFloatTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_number_literal_underflow);
@@ -967,7 +1182,8 @@ TEST_F(ParseHLSLRootSignatureTest, InvalidNonZeroFlagsTest) {
 
   hlsl::RootSignatureLexer Lexer(Source, TokLoc);
   SmallVector<RootElement> Elements;
-  hlsl::RootSignatureParser Parser(Elements, Lexer, *PP);
+  hlsl::RootSignatureParser Parser(RootSignatureVersion::V1_1, Elements, Lexer,
+                                   *PP);
 
   // Test correct diagnostic produced
   Consumer->setExpected(diag::err_hlsl_rootsig_non_zero_flag);
diff --git a/clang/unittests/Sema/HeuristicResolverTest.cpp b/clang/unittests/Sema/HeuristicResolverTest.cpp
index 3ed6bba790be..ee434f7a1d43 100644
--- a/clang/unittests/Sema/HeuristicResolverTest.cpp
+++ b/clang/unittests/Sema/HeuristicResolverTest.cpp
@@ -766,5 +766,85 @@ TEST(HeuristicResolver, UsingValueDecl) {
                    cxxMethodDecl(hasName("waldo")).bind("output"));
 }
 
+// `arg` is a ParamVarDecl*, `Expected` is a string
+MATCHER_P(ParamNameMatcher, Expected, "paramNameMatcher") {
+  EXPECT_TRUE(arg);
+  if (IdentifierInfo *Ident = arg->getDeclName().getAsIdentifierInfo()) {
+    return Ident->getName() == Expected;
+  }
+  return false;
+}
+
+// Helper function for testing HeuristicResolver::getProtoTypeLoc.
+// Takes a matcher that selects a callee expression bound to the ID "input",
+// calls getProtoTypeLoc() on it, and checks that the call found a
+// FunctionProtoTypeLoc encoding the given parameter names.
+template <typename InputMatcher, typename... ParameterNames>
+void expectParameterNames(ASTContext &Ctx, const InputMatcher &IM,
+                          ParameterNames... ExpectedParameterNames) {
+  auto InputMatches = match(IM, Ctx);
+  ASSERT_EQ(1u, InputMatches.size());
+  const auto *Input = InputMatches[0].template getNodeAs<Expr>("input");
+  ASSERT_TRUE(Input);
+
+  HeuristicResolver H(Ctx);
+  auto Loc = H.getFunctionProtoTypeLoc(Input);
+  ASSERT_TRUE(Loc);
+  EXPECT_THAT(Loc.getParams(),
+              ElementsAre(ParamNameMatcher(ExpectedParameterNames)...));
+}
+
+TEST(HeuristicResolver, ProtoTypeLoc) {
+  std::string Code = R"cpp(
+    void (*f1)(int param1);
+    void (__stdcall *f2)(int param2);
+    using f3_t = void(*)(int param3);
+    f3_t f3;
+    using f4_t = void(__stdcall *)(int param4);
+    f4_t f4;
+    struct S {
+      void (*f5)(int param5);
+      using f6_t = void(*)(int param6);
+      f6_t f6;
+    };
+    void bar() {
+      f1(42);
+      f2(42);
+      f3(42);
+      f4(42);
+      S s;
+      s.f5(42);
+      s.f6(42);
+    }
+  )cpp";
+  auto TU = tooling::buildASTFromCodeWithArgs(Code, {"-std=c++20"});
+  auto &Ctx = TU->getASTContext();
+  auto checkFreeFunction = [&](llvm::StringRef FunctionName,
+                               llvm::StringRef ParamName) {
+    expectParameterNames(
+        Ctx,
+        callExpr(
+            callee(implicitCastExpr(hasSourceExpression(declRefExpr(
+                                        to(namedDecl(hasName(FunctionName))))))
+                       .bind("input"))),
+        ParamName);
+  };
+  checkFreeFunction("f1", "param1");
+  checkFreeFunction("f2", "param2");
+  checkFreeFunction("f3", "param3");
+  checkFreeFunction("f4", "param4");
+  auto checkMemberFunction = [&](llvm::StringRef MemberName,
+                                 llvm::StringRef ParamName) {
+    expectParameterNames(
+        Ctx,
+        callExpr(callee(implicitCastExpr(hasSourceExpression(memberExpr(
+                                             member(hasName(MemberName)))))
+                            .bind("input"))),
+        ParamName);
+  };
+  checkMemberFunction("f5", "param5");
+  checkMemberFunction("f6", "param6");
+}
+
 } // namespace
 } // namespace clang
diff --git a/clang/unittests/Serialization/SourceLocationEncodingTest.cpp b/clang/unittests/Serialization/SourceLocationEncodingTest.cpp
index c80a8fd0e52b..18fedd4de397 100644
--- a/clang/unittests/Serialization/SourceLocationEncodingTest.cpp
+++ b/clang/unittests/Serialization/SourceLocationEncodingTest.cpp
@@ -16,7 +16,6 @@ using namespace llvm;
 using namespace clang;
 
 namespace {
-using LocSeq = SourceLocationSequence;
 
 // Convert a single source location into encoded form and back.
 // If ExpectedEncoded is provided, verify the encoded value too.
@@ -34,37 +33,9 @@ void roundTrip(SourceLocation::UIntTy Loc,
   ASSERT_EQ(DecodedEncoded, Loc) << "Decoding " << ActualEncoded;
 }
 
-// As above, but use sequence encoding for a series of locations.
-void roundTrip(std::vector<SourceLocation::UIntTy> Locs,
-               std::vector<uint64_t> ExpectedEncoded = {}) {
-  std::vector<uint64_t> ActualEncoded;
-  {
-    LocSeq::State Seq;
-    for (auto L : Locs)
-      ActualEncoded.push_back(SourceLocationEncoding::encode(
-          SourceLocation::getFromRawEncoding(L), /*BaseOffset=*/0,
-          /*BaseModuleFileIndex=*/0, Seq));
-    if (!ExpectedEncoded.empty()) {
-      ASSERT_EQ(ActualEncoded, ExpectedEncoded)
-          << "Encoding " << testing::PrintToString(Locs);
-    }
-  }
-  std::vector<SourceLocation::UIntTy> DecodedEncoded;
-  {
-    LocSeq::State Seq;
-    for (auto L : ActualEncoded) {
-      SourceLocation Loc = SourceLocationEncoding::decode(L, Seq).first;
-      DecodedEncoded.push_back(Loc.getRawEncoding());
-    }
-    ASSERT_EQ(DecodedEncoded, Locs)
-        << "Decoding " << testing::PrintToString(ActualEncoded);
-  }
-}
-
 constexpr SourceLocation::UIntTy MacroBit =
     1 << (sizeof(SourceLocation::UIntTy) * CHAR_BIT - 1);
 constexpr SourceLocation::UIntTy Big = MacroBit >> 1;
-constexpr SourceLocation::UIntTy Biggest = -1;
 
 TEST(SourceLocationEncoding, Individual) {
   roundTrip(1, 2);
@@ -77,33 +48,4 @@ TEST(SourceLocationEncoding, Individual) {
   roundTrip(MacroBit | (Big + 1));
 }
 
-TEST(SourceLocationEncoding, Sequence) {
-  roundTrip({1, 2, 3, 3, 2, 1},
-            {2, // 1
-             5, // +2 (+1 of non-raw)
-             5, // +2
-             1, // +0
-             4, // -2
-             4} // -2
-  );
-  roundTrip({100, 0, 100},
-            {200, // 100
-             0,   // 0
-             1}   // +0
-  );
-
-  roundTrip({1, Big}, {2, ((Big - 1) << 2) + 1});
-  roundTrip({2, MacroBit | Big}, {4, ((Big - 1) << 2) - 1});
-
-  roundTrip({3, MacroBit | 5, MacroBit | 4, 3},
-            {6,  // 3
-             11, // +5 (+2 of non-raw + set macro bit)
-             4,  // -2
-             6}  // -3 (-2 of non-raw, clear macro bit)
-  );
-
-  roundTrip(
-      {123 | MacroBit, 1, 9, Biggest, Big, Big + 1, 0, MacroBit | Big, 0});
-}
-
 } // namespace
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index d194b2877ad8..1cdb5397aa57 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -299,8 +299,7 @@ TEST(DependencyScanner, ScanDepsWithModuleLookup) {
       ScanTool.getDependencyFile(CommandLine, CWD).moveInto(DepFile),
       llvm::Failed());
 
-  EXPECT_TRUE(llvm::find(InterceptFS->StatPaths, OtherPath) ==
-              InterceptFS->StatPaths.end());
+  EXPECT_TRUE(!llvm::is_contained(InterceptFS->StatPaths, OtherPath));
   EXPECT_EQ(InterceptFS->ReadFiles, std::vector<std::string>{"test.m"});
 }
 
@@ -384,55 +383,3 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) {
     EXPECT_TRUE(DiagConsumer.Finished);
   }
 }
-
-TEST(DependencyScanner, NoNegativeCache) {
-  StringRef CWD = "/root";
-
-  auto VFS = new llvm::vfs::InMemoryFileSystem();
-  VFS->setCurrentWorkingDirectory(CWD);
-  auto Sept = llvm::sys::path::get_separator();
-  std::string HeaderPath =
-      std::string(llvm::formatv("{0}root{0}header.h", Sept));
-  std::string Test0Path =
-      std::string(llvm::formatv("{0}root{0}test0.cpp", Sept));
-  std::string Test1Path =
-      std::string(llvm::formatv("{0}root{0}test1.cpp", Sept));
-
-  VFS->addFile(Test0Path, 0,
-               llvm::MemoryBuffer::getMemBuffer(
-                   "#if __has_include(\"header.h\")\n#endif"));
-  VFS->addFile(Test1Path, 0,
-               llvm::MemoryBuffer::getMemBuffer("#include \"header.h\""));
-
-  DependencyScanningService Service(
-      ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make,
-      ScanningOptimizations::All, false, false,
-      llvm::sys::toTimeT(std::chrono::system_clock::now()), false);
-  DependencyScanningTool ScanTool(Service, VFS);
-
-  std::vector<std::string> CommandLine0 = {"clang",
-                                           "-target",
-                                           "x86_64-apple-macosx10.7",
-                                           "-c",
-                                           "test0.cpp",
-                                           "-o"
-                                           "test0.cpp.o"};
-  std::vector<std::string> CommandLine1 = {"clang",
-                                           "-target",
-                                           "x86_64-apple-macosx10.7",
-                                           "-c",
-                                           "test1.cpp",
-                                           "-o"
-                                           "test1.cpp.o"};
-
-  std::string Result;
-  ASSERT_THAT_ERROR(
-      ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result),
-      llvm::Succeeded());
-
-  VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
-
-  ASSERT_THAT_ERROR(
-      ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result),
-      llvm::Succeeded());
-}
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index a68ea72d3816..b461d9109271 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
-#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "gtest/gtest.h"
@@ -20,10 +19,9 @@ TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
-  DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
 
   DepFS.status("/foo.c");
   EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
@@ -47,10 +45,9 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
-  DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
 
   {
     llvm::SmallString<128> Result;
@@ -83,9 +80,8 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
   InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
   InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
 
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
 
   // Success.
   {
@@ -137,9 +133,8 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) {
   InMemoryFS->setCurrentWorkingDirectory("/");
   InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
   InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
 
   DepFS.status("/foo");
   DepFS.status("/foo");
@@ -161,9 +156,8 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
 
   DepFS.status("/dir");
   DepFS.status("/dir");
@@ -189,23 +183,53 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
   auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
   InMemoryFS->setCurrentWorkingDirectory("/");
 
-  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
-                                    ScanningOutputFormat::Make);
-  DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
 
   bool Path1Exists = DepFS.exists("/path1.suffix");
-  EXPECT_EQ(Path1Exists, false);
+  ASSERT_EQ(Path1Exists, false);
 
   // Adding a file that has been stat-ed,
   InMemoryFS->addFile("/path1.suffix", 0, llvm::MemoryBuffer::getMemBuffer(""));
   Path1Exists = DepFS.exists("/path1.suffix");
   // Due to caching in SharedCache, path1 should not exist in
   // DepFS's eyes.
-  EXPECT_EQ(Path1Exists, false);
+  ASSERT_EQ(Path1Exists, false);
 
-  std::vector<llvm::StringRef> InvalidPaths =
-      Service.getSharedCache().getInvalidNegativeStatCachedPaths(*InMemoryFS);
+  auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS);
 
-  EXPECT_EQ(InvalidPaths.size(), 1u);
-  ASSERT_STREQ("/path1.suffix", InvalidPaths[0].str().c_str());
+  EXPECT_EQ(InvalidEntries.size(), 1u);
+  ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path);
+}
+
+TEST(DependencyScanningFilesystem, DiagnoseCachedFileSizeChange) {
+  auto InMemoryFS1 = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  auto InMemoryFS2 = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  InMemoryFS1->setCurrentWorkingDirectory("/");
+  InMemoryFS2->setCurrentWorkingDirectory("/");
+
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS1);
+
+  InMemoryFS1->addFile("/path1.suffix", 0,
+                       llvm::MemoryBuffer::getMemBuffer(""));
+  bool Path1Exists = DepFS.exists("/path1.suffix");
+  ASSERT_EQ(Path1Exists, true);
+
+  // Add a file to a new FS that has the same path but different content.
+  InMemoryFS2->addFile("/path1.suffix", 1,
+                       llvm::MemoryBuffer::getMemBuffer("        "));
+
+  // Check against the new file system. InMemoryFS2 could be the underlying
+  // physical system in the real world.
+  auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS2);
+
+  ASSERT_EQ(InvalidEntries.size(), 1u);
+  ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path);
+  auto SizeInfo = std::get_if<
+      DependencyScanningFilesystemSharedCache::OutOfDateEntry::SizeChangedInfo>(
+      &InvalidEntries[0].Info);
+  ASSERT_TRUE(SizeInfo);
+  ASSERT_EQ(SizeInfo->CachedSize, 0u);
+  ASSERT_EQ(SizeInfo->ActualSize, 8u);
 }
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index dfeb6b1b1ec1..d63e79a5f515 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -4904,9 +4904,8 @@ void EmitClangAttrParsedAttrImpl(const RecordKeeper &Records, raw_ostream &OS) {
   }
 
   OS << "static const ParsedAttrInfo *AttrInfoMap[] = {\n";
-  for (auto I = Attrs.begin(), E = Attrs.end(); I != E; ++I) {
-    OS << "&ParsedAttrInfo" << I->first << "::Instance,\n";
-  }
+  for (const auto &Attr : Attrs)
+    OS << "&ParsedAttrInfo" << Attr.first << "::Instance,\n";
   OS << "};\n\n";
 
   // Generate function for handling attributes with delayed arguments
@@ -5261,10 +5260,9 @@ GetAttributeHeadingAndSpellings(const Record &Documentation,
       Heading = Spellings.begin()->name();
     else {
       std::set<std::string> Uniques;
-      for (auto I = Spellings.begin(), E = Spellings.end();
-           I != E; ++I) {
+      for (const FlattenedSpelling &FS : Spellings) {
         std::string Spelling =
-            NormalizeNameForSpellingComparison(I->name()).str();
+            NormalizeNameForSpellingComparison(FS.name()).str();
         Uniques.insert(Spelling);
       }
       // If the semantic map has only one spelling, that is sufficient for our
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index b28cb2c09ac5..17078e2bc150 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -1657,8 +1657,7 @@ void clang::EmitClangDiagsEnums(const RecordKeeper &Records, raw_ostream &OS,
 
       llvm::SmallVector<std::string> EnumeratorNames;
       for (auto &Enumerator : Enumeration.second) {
-        if (llvm::find(EnumeratorNames, Enumerator.second) !=
-            EnumeratorNames.end())
+        if (llvm::is_contained(EnumeratorNames, Enumerator.second))
           PrintError(&R,
                      "Duplicate enumerator name '" + Enumerator.second + "'");
         EnumeratorNames.push_back(Enumerator.second);
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index a70e65e35d5e..831f79f7cf17 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -317,7 +317,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Attaching main to the global module</td>
   <td><a href="https://wg21.link/P3618">P3618R0</a> (<a href="#dr">DR</a>)</td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 21</td>
  </tr>
  <tr>
   <td>Expansion Statements</td>
@@ -923,7 +923,7 @@ <h2 id="cxx20">C++20 implementation status</h2>
       </tr>
       <tr> <!-- from Belfast -->
         <td><a href="https://wg21.link/p1979r0">P1979R0</a></td>
-        <td class="none" align="center">No</td>
+        <td class="full" align="center">Clang 17</td>
       </tr>
       <tr> <!-- from Prague -->
         <td><a href="https://wg21.link/p1779r3">P1779R3</a></td>
diff --git a/cmake/Modules/GetClangResourceDir.cmake b/cmake/Modules/GetClangResourceDir.cmake
index def5bd60424e..518220188f96 100644
--- a/cmake/Modules/GetClangResourceDir.cmake
+++ b/cmake/Modules/GetClangResourceDir.cmake
@@ -10,7 +10,8 @@ function(get_clang_resource_dir out_var)
   cmake_parse_arguments(ARG "" "PREFIX;SUBDIR" "" ${ARGN})
 
   if(DEFINED CLANG_RESOURCE_DIR AND NOT CLANG_RESOURCE_DIR STREQUAL "")
-    set(ret_dir bin/${CLANG_RESOURCE_DIR})
+    set(ret_dir bin)
+    cmake_path(APPEND ret_dir ${CLANG_RESOURCE_DIR})
   else()
     if (NOT CLANG_VERSION_MAJOR)
       string(REGEX MATCH "^[0-9]+" CLANG_VERSION_MAJOR ${PACKAGE_VERSION})
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index 86e19e08270d..45a2f5c0a61f 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -582,6 +582,24 @@ macro(add_compiler_rt_script name)
     DESTINATION ${COMPILER_RT_INSTALL_BINARY_DIR})
 endmacro(add_compiler_rt_script src name)
 
+
+macro(add_compiler_rt_cfg target_name file_name component arch)
+  set(src_file "${CMAKE_CURRENT_SOURCE_DIR}/${file_name}")
+  get_compiler_rt_output_dir(${arch} output_dir)
+  set(dst_file "${output_dir}/${file_name}")
+  add_custom_command(OUTPUT ${dst_file}
+    DEPENDS ${src_file}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${src_file} ${dst_file}
+    COMMENT "Copying ${file_name}...")
+  add_custom_target(${target_name} DEPENDS ${dst_file})
+  install(FILES ${file_name}
+    DESTINATION ${COMPILER_RT_INSTALL_LIBRARY_DIR}
+    COMPONENT ${component})
+  add_dependencies(${component} ${target_name})
+
+  set_target_properties(${target_name} PROPERTIES FOLDER "Compiler-RT Misc")
+endmacro()
+
 # Builds custom version of libc++ and installs it in <prefix>.
 # Can be used to build sanitized versions of libc++ for running unit tests.
 # add_custom_libcxx(<name> <prefix>
diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index e2f39f224df9..97cc5c85703e 100755
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -281,6 +281,8 @@ else()
       PARENT_TARGET asan)
   endif()
 
+  # On AIX, we only need the static libraries.
+  if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
   foreach(arch ${ASAN_SUPPORTED_ARCH})
     if (COMPILER_RT_HAS_VERSION_SCRIPT)
       if(WIN32)
@@ -382,10 +384,21 @@ else()
       endif()
     endif()
   endforeach()
+  endif()
 endif()
 
 add_compiler_rt_resource_file(asan_ignorelist asan_ignorelist.txt asan)
 
+# On AIX, static sanitizer libraries are not added to the DSO, so we need to put 
+# asan.link_with_main_exec.txt and asan_cxx.link_with_main_exec.txt to the build
+# and install dir for use in resolving undefined sanitizer symbols at runtime.
+if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  foreach(arch ${ASAN_SUPPORTED_ARCH})
+    add_compiler_rt_cfg(asan_symbols_${arch} asan.link_with_main_exec.txt asan ${arch})
+    add_compiler_rt_cfg(asan_cxx_symbols_${arch} asan_cxx.link_with_main_exec.txt asan ${arch})
+  endforeach()
+endif()
+
 add_subdirectory(scripts)
 
 if(COMPILER_RT_INCLUDE_TESTS)
diff --git a/compiler-rt/lib/asan/asan.link_with_main_exec.txt b/compiler-rt/lib/asan/asan.link_with_main_exec.txt
new file mode 100644
index 000000000000..5efc48c26236
--- /dev/null
+++ b/compiler-rt/lib/asan/asan.link_with_main_exec.txt
@@ -0,0 +1,115 @@
+#! .
+__asan_report_load_n
+__asan_loadN
+__asan_report_load1
+__asan_load1
+__asan_report_load2
+__asan_load2
+__asan_report_load4
+__asan_load4
+__asan_report_load8
+__asan_load8
+__asan_report_load16
+__asan_load16
+__asan_report_store_n
+__asan_storeN
+__asan_report_store1
+__asan_store1
+__asan_report_store2
+__asan_store2
+__asan_report_store4
+__asan_store4
+__asan_report_store8
+__asan_store8
+__asan_report_store16
+__asan_store16
+__asan_report_exp_load_n
+__asan_exp_loadN
+__asan_report_exp_load1
+__asan_exp_load1
+__asan_report_exp_load2
+__asan_exp_load2
+__asan_report_exp_load4
+__asan_exp_load4
+__asan_report_exp_load8
+__asan_exp_load8
+__asan_report_exp_load16
+__asan_exp_load16
+__asan_report_exp_store_n
+__asan_exp_storeN
+__asan_report_exp_store1
+__asan_exp_store1
+__asan_report_exp_store2
+__asan_exp_store2
+__asan_report_exp_store4
+__asan_exp_store4
+__asan_report_exp_store8
+__asan_exp_store8
+__asan_report_exp_store16
+__asan_exp_store16
+__asan_memmove
+__asan_memcpy
+__asan_memset
+__asan_handle_no_return
+__sanitizer_ptr_cmp
+__sanitizer_ptr_sub
+__asan_before_dynamic_init
+__asan_after_dynamic_init
+__asan_register_globals
+__asan_unregister_globals
+__asan_register_image_globals
+__asan_unregister_image_globals
+__asan_register_elf_globals
+__asan_unregister_elf_globals
+__asan_init
+__asan_version_mismatch_check_v8
+__asan_stack_malloc_0
+__asan_stack_malloc_1
+__asan_stack_malloc_2
+__asan_stack_malloc_3
+__asan_stack_malloc_4
+__asan_stack_malloc_5
+__asan_stack_malloc_6
+__asan_stack_malloc_7
+__asan_stack_malloc_8
+__asan_stack_malloc_9
+__asan_stack_malloc_10
+__asan_stack_malloc_always_0
+__asan_stack_malloc_always_1
+__asan_stack_malloc_always_2
+__asan_stack_malloc_always_3
+__asan_stack_malloc_always_4
+__asan_stack_malloc_always_5
+__asan_stack_malloc_always_6
+__asan_stack_malloc_always_7
+__asan_stack_malloc_always_8
+__asan_stack_malloc_always_9
+__asan_stack_malloc_always_10
+__asan_stack_free_0
+__asan_stack_free_1
+__asan_stack_free_2
+__asan_stack_free_3
+__asan_stack_free_4
+__asan_stack_free_5
+__asan_stack_free_6
+__asan_stack_free_7
+__asan_stack_free_8
+__asan_stack_free_9
+__asan_stack_free_10
+__asan_set_shadow_00
+__asan_set_shadow_01
+__asan_set_shadow_02
+__asan_set_shadow_03
+__asan_set_shadow_04
+__asan_set_shadow_05
+__asan_set_shadow_06
+__asan_set_shadow_07
+__asan_set_shadow_f1
+__asan_set_shadow_f2
+__asan_set_shadow_f3
+__asan_set_shadow_f5
+__asan_set_shadow_f8
+__asan_poison_stack_memory
+__asan_unpoison_stack_memory
+__asan_option_detect_stack_use_after_return
+__asan_shadow_memory_dynamic_address
diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index 371205250794..8d7b3abe75c0 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -428,10 +428,15 @@ struct Allocator {
     PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic);
   }
 
-  void ReInitialize(const AllocatorOptions &options) {
+  // Apply provided AllocatorOptions to an Allocator
+  void ApplyOptions(const AllocatorOptions &options) {
     SetAllocatorMayReturnNull(options.may_return_null);
     allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms);
     SharedInitCode(options);
+  }
+
+  void ReInitialize(const AllocatorOptions &options) {
+    ApplyOptions(options);
 
     // Poison all existing allocation's redzones.
     if (CanPoisonMemory()) {
@@ -982,6 +987,11 @@ void ReInitializeAllocator(const AllocatorOptions &options) {
   instance.ReInitialize(options);
 }
 
+// Apply provided AllocatorOptions to an Allocator
+void ApplyAllocatorOptions(const AllocatorOptions &options) {
+  instance.ApplyOptions(options);
+}
+
 void GetAllocatorOptions(AllocatorOptions *options) {
   instance.GetOptions(options);
 }
diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index 92f80de7144a..cde6951e20af 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -47,6 +47,7 @@ struct AllocatorOptions {
 void InitializeAllocator(const AllocatorOptions &options);
 void ReInitializeAllocator(const AllocatorOptions &options);
 void GetAllocatorOptions(AllocatorOptions *options);
+void ApplyAllocatorOptions(const AllocatorOptions &options);
 
 class AsanChunkView {
  public:
diff --git a/compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt b/compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt
new file mode 100644
index 000000000000..7387f8173e85
--- /dev/null
+++ b/compiler-rt/lib/asan/asan_cxx.link_with_main_exec.txt
@@ -0,0 +1,21 @@
+#! .
+_ZdaPv
+_ZdaPvRKSt9nothrow_t
+_ZdaPvSt11align_val_t
+_ZdaPvSt11align_val_tRKSt9nothrow_t
+_ZdaPvm
+_ZdaPvmSt11align_val_t
+_ZdlPv
+_ZdlPvRKSt9nothrow_t
+_ZdlPvSt11align_val_t
+_ZdlPvSt11align_val_tRKSt9nothrow_t
+_ZdlPvm
+_ZdlPvmSt11align_val_t
+_Znam
+_ZnamRKSt9nothrow_t
+_ZnamSt11align_val_t
+_ZnamSt11align_val_tRKSt9nothrow_t
+_Znwm
+_ZnwmRKSt9nothrow_t
+_ZnwmSt11align_val_t
+_ZnwmSt11align_val_tRKSt9nothrow_t
diff --git a/compiler-rt/lib/asan/asan_descriptions.cpp b/compiler-rt/lib/asan/asan_descriptions.cpp
index 6dd9b96a8d34..96a1467d8e5c 100644
--- a/compiler-rt/lib/asan/asan_descriptions.cpp
+++ b/compiler-rt/lib/asan/asan_descriptions.cpp
@@ -211,10 +211,10 @@ bool GetStackAddressInformation(uptr addr, uptr access_size,
   descr->frame_pc = access.frame_pc;
   descr->frame_descr = access.frame_descr;
 
-#if SANITIZER_PPC64V1
-  // On PowerPC64 ELFv1, the address of a function actually points to a
-  // three-doubleword data structure with the first field containing
-  // the address of the function's code.
+#if SANITIZER_PPC64V1 || SANITIZER_AIX
+  // On PowerPC64 ELFv1 or AIX, the address of a function actually points to a
+  // three-doubleword (or three-word for 32-bit AIX) data structure with
+  // the first field containing the address of the function's code.
   descr->frame_pc = *reinterpret_cast<uptr *>(descr->frame_pc);
 #endif
   descr->frame_pc += 16;
@@ -445,6 +445,16 @@ AddressDescription::AddressDescription(uptr addr, uptr access_size,
     data.kind = kAddressKindShadow;
     return;
   }
+
+  // Check global first. On AIX, some global data defined in shared libraries
+  // are put to the STACK region for unknown reasons. Check global first can
+  // workaround this issue.
+  // TODO: Look into whether there's a different solution to this problem.
+  if (GetGlobalAddressInformation(addr, access_size, &data.global)) {
+    data.kind = kAddressKindGlobal;
+    return;
+  }
+
   if (GetHeapAddressInformation(addr, access_size, &data.heap)) {
     data.kind = kAddressKindHeap;
     return;
@@ -462,10 +472,6 @@ AddressDescription::AddressDescription(uptr addr, uptr access_size,
     return;
   }
 
-  if (GetGlobalAddressInformation(addr, access_size, &data.global)) {
-    data.kind = kAddressKindGlobal;
-    return;
-  }
   data.kind = kAddressKindWild;
   data.wild.addr = addr;
   data.wild.access_size = access_size;
diff --git a/compiler-rt/lib/asan/asan_flags.cpp b/compiler-rt/lib/asan/asan_flags.cpp
index 9cfb70bd00c7..190a89345dd1 100644
--- a/compiler-rt/lib/asan/asan_flags.cpp
+++ b/compiler-rt/lib/asan/asan_flags.cpp
@@ -144,6 +144,7 @@ static void InitializeDefaultFlags() {
   DisplayHelpMessages(&asan_parser);
 }
 
+// Validate flags and report incompatible configurations
 static void ProcessFlags() {
   Flags *f = flags();
 
@@ -217,11 +218,12 @@ void InitializeFlags() {
   ProcessFlags();
 
 #if SANITIZER_WINDOWS
-  // On Windows, weak symbols are emulated by having the user program
-  // register which weak functions are defined.
-  // The ASAN DLL will initialize flags prior to user module initialization,
-  // so __asan_default_options will not point to the user definition yet.
-  // We still want to ensure we capture when options are passed via
+  // On Windows, weak symbols (such as the `__asan_default_options` function)
+  // are emulated by having the user program register which weak functions are
+  // defined. The ASAN DLL will initialize flags prior to user module
+  // initialization, so __asan_default_options will not point to the user
+  // definition yet. We still want to ensure we capture when options are passed
+  // via
   // __asan_default_options, so we add a callback to be run
   // when it is registered with the runtime.
 
@@ -232,21 +234,13 @@ void InitializeFlags() {
   // __sanitizer_register_weak_function.
   AddRegisterWeakFunctionCallback(
       reinterpret_cast<uptr>(__asan_default_options), []() {
-        FlagParser asan_parser;
-
-        RegisterAsanFlags(&asan_parser, flags());
-        RegisterCommonFlags(&asan_parser);
-        asan_parser.ParseString(__asan_default_options());
-
-        DisplayHelpMessages(&asan_parser);
+        // We call `InitializeDefaultFlags` again, instead of just parsing
+        // `__asan_default_options` directly, to ensure that flags set through
+        // `ASAN_OPTS` take precedence over those set through
+        // `__asan_default_options`.
+        InitializeDefaultFlags();
         ProcessFlags();
-
-        // TODO: Update other globals and data structures that may need to change
-        // after initialization due to new flags potentially being set changing after
-        // `__asan_default_options` is registered.
-        // See GH issue 'https://github.com/llvm/llvm-project/issues/117925' for
-        // details.
-        SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
+        ApplyFlags();
       });
 
 #  if CAN_SANITIZE_UB
diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 8205e19fdab7..35c887eaf851 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -61,6 +61,7 @@ using __sanitizer::StackTrace;
 
 void AsanInitFromRtl();
 bool TryAsanInitFromRtl();
+void ApplyFlags();
 
 // asan_win.cpp
 void InitializePlatformExceptionHandlers();
diff --git a/compiler-rt/lib/asan/asan_new_delete.cpp b/compiler-rt/lib/asan/asan_new_delete.cpp
index b5b1ced8ac5e..99c7c9938dfb 100644
--- a/compiler-rt/lib/asan/asan_new_delete.cpp
+++ b/compiler-rt/lib/asan/asan_new_delete.cpp
@@ -60,18 +60,42 @@ enum class align_val_t: size_t {};
 // TODO(alekseyshl): throw std::bad_alloc instead of dying on OOM.
 // For local pool allocation, align to SHADOW_GRANULARITY to match asan
 // allocator behavior.
-#define OPERATOR_NEW_BODY(type, nothrow)            \
-  GET_STACK_TRACE_MALLOC;                           \
-  void *res = asan_memalign(0, size, &stack, type); \
-  if (!nothrow && UNLIKELY(!res))                   \
-    ReportOutOfMemory(size, &stack);                \
-  return res;
-#define OPERATOR_NEW_BODY_ALIGN(type, nothrow)                \
-  GET_STACK_TRACE_MALLOC;                                     \
-  void *res = asan_memalign((uptr)align, size, &stack, type); \
-  if (!nothrow && UNLIKELY(!res))                             \
-    ReportOutOfMemory(size, &stack);                          \
-  return res;
+#define OPERATOR_NEW_BODY                               \
+  GET_STACK_TRACE_MALLOC;                               \
+  void *res = asan_memalign(0, size, &stack, FROM_NEW); \
+  if (UNLIKELY(!res))                                   \
+    ReportOutOfMemory(size, &stack);                    \
+  return res
+#define OPERATOR_NEW_BODY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;         \
+  return asan_memalign(0, size, &stack, FROM_NEW)
+#define OPERATOR_NEW_BODY_ARRAY                            \
+  GET_STACK_TRACE_MALLOC;                                  \
+  void *res = asan_memalign(0, size, &stack, FROM_NEW_BR); \
+  if (UNLIKELY(!res))                                      \
+    ReportOutOfMemory(size, &stack);                       \
+  return res
+#define OPERATOR_NEW_BODY_ARRAY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;               \
+  return asan_memalign(0, size, &stack, FROM_NEW_BR)
+#define OPERATOR_NEW_BODY_ALIGN                                   \
+  GET_STACK_TRACE_MALLOC;                                         \
+  void *res = asan_memalign((uptr)align, size, &stack, FROM_NEW); \
+  if (UNLIKELY(!res))                                             \
+    ReportOutOfMemory(size, &stack);                              \
+  return res
+#define OPERATOR_NEW_BODY_ALIGN_NOTHROW \
+  GET_STACK_TRACE_MALLOC;               \
+  return asan_memalign((uptr)align, size, &stack, FROM_NEW)
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY                                \
+  GET_STACK_TRACE_MALLOC;                                            \
+  void *res = asan_memalign((uptr)align, size, &stack, FROM_NEW_BR); \
+  if (UNLIKELY(!res))                                                \
+    ReportOutOfMemory(size, &stack);                                 \
+  return res
+#define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \
+  GET_STACK_TRACE_MALLOC;                     \
+  return asan_memalign((uptr)align, size, &stack, FROM_NEW_BR)
 
 // On OS X it's not enough to just provide our own 'operator new' and
 // 'operator delete' implementations, because they're going to be in the
@@ -82,106 +106,128 @@ enum class align_val_t: size_t {};
 // OS X we need to intercept them using their mangled names.
 #if !SANITIZER_APPLE
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size)
-{ OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/); }
+void *operator new(size_t size) { OPERATOR_NEW_BODY; }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/); }
+void *operator new[](size_t size) { OPERATOR_NEW_BODY_ARRAY; }
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/); }
+void *operator new(size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/); }
+void *operator new[](size_t size, std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ARRAY_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, false /*nothrow*/); }
+void *operator new(size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::align_val_t align)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, false /*nothrow*/); }
+void *operator new[](size_t size, std::align_val_t align) {
+  OPERATOR_NEW_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new(size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW, true /*nothrow*/); }
+void *operator new(size_t size, std::align_val_t align,
+                   std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN_NOTHROW;
+}
 CXX_OPERATOR_ATTRIBUTE
-void *operator new[](size_t size, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_NEW_BODY_ALIGN(FROM_NEW_BR, true /*nothrow*/); }
+void *operator new[](size_t size, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW;
+}
 
 #else  // SANITIZER_APPLE
-INTERCEPTOR(void *, _Znwm, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW, false /*nothrow*/);
-}
-INTERCEPTOR(void *, _Znam, size_t size) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR, false /*nothrow*/);
-}
+INTERCEPTOR(void *, _Znwm, size_t size) { OPERATOR_NEW_BODY; }
+INTERCEPTOR(void *, _Znam, size_t size) { OPERATOR_NEW_BODY_ARRAY; }
 INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW, true /*nothrow*/);
+  OPERATOR_NEW_BODY_NOTHROW;
 }
 INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(FROM_NEW_BR, true /*nothrow*/);
+  OPERATOR_NEW_BODY_ARRAY_NOTHROW;
 }
 #endif  // !SANITIZER_APPLE
 
-#define OPERATOR_DELETE_BODY(type) \
+#define OPERATOR_DELETE_BODY \
+  GET_STACK_TRACE_FREE;      \
+  asan_delete(ptr, 0, 0, &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_ARRAY \
   GET_STACK_TRACE_FREE;            \
-  asan_delete(ptr, 0, 0, &stack, type);
-
-#define OPERATOR_DELETE_BODY_SIZE(type) \
-  GET_STACK_TRACE_FREE;                 \
-  asan_delete(ptr, size, 0, &stack, type);
-
-#define OPERATOR_DELETE_BODY_ALIGN(type) \
+  asan_delete(ptr, 0, 0, &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_ALIGN \
+  GET_STACK_TRACE_FREE;            \
+  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_ALIGN_ARRAY \
   GET_STACK_TRACE_FREE;                  \
-  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type);
-
-#define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \
+  asan_delete(ptr, 0, static_cast<uptr>(align), &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_SIZE \
+  GET_STACK_TRACE_FREE;           \
+  asan_delete(ptr, size, 0, &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_SIZE_ARRAY \
+  GET_STACK_TRACE_FREE;                 \
+  asan_delete(ptr, size, 0, &stack, FROM_NEW_BR)
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN \
+  GET_STACK_TRACE_FREE;                 \
+  asan_delete(ptr, size, static_cast<uptr>(align), &stack, FROM_NEW)
+#define OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY \
   GET_STACK_TRACE_FREE;                       \
-  asan_delete(ptr, size, static_cast<uptr>(align), &stack, type);
+  asan_delete(ptr, size, static_cast<uptr>(align), &stack, FROM_NEW_BR)
 
 #if !SANITIZER_APPLE
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
+void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY_ARRAY; }
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
+void operator delete(void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW); }
+void operator delete(void *ptr, size_t size) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE(FROM_NEW_BR); }
+void operator delete[](void *ptr, size_t size) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, std::align_val_t align,
+                     std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, std::align_val_t align, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, std::align_val_t align,
+                       std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ALIGN_ARRAY;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW); }
+void operator delete(void *ptr, size_t size, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ALIGN;
+}
 CXX_OPERATOR_ATTRIBUTE
-void operator delete[](void *ptr, size_t size, std::align_val_t align) NOEXCEPT
-{ OPERATOR_DELETE_BODY_SIZE_ALIGN(FROM_NEW_BR); }
+void operator delete[](void *ptr, size_t size,
+                       std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY;
+}
 
 #else  // SANITIZER_APPLE
-INTERCEPTOR(void, _ZdlPv, void *ptr)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
-INTERCEPTOR(void, _ZdaPv, void *ptr)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
-INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW); }
-INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
-{ OPERATOR_DELETE_BODY(FROM_NEW_BR); }
+INTERCEPTOR(void, _ZdlPv, void *ptr) { OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPv, void *ptr) { OPERATOR_DELETE_BODY_ARRAY; }
+INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY_ARRAY;
+}
 #endif  // !SANITIZER_APPLE
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index 833873a02129..a5cab1e70ef9 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -429,6 +429,39 @@ void PrintAddressSpaceLayout() {
           kHighShadowBeg > kMidMemEnd);
 }
 
+// Apply most options specified either through the ASAN_OPTIONS
+// environment variable, or through the `__asan_default_options` user function.
+//
+// This function may be called multiple times, once per weak reference callback
+// on Windows, so it needs to be idempotent.
+//
+// Context:
+// For maximum compatibility on Windows, it is necessary for ASan options to be
+// configured/registered/applied inside this method (instead of in
+// ASanInitInternal, for example). That's because, on Windows, the user-provided
+// definition for `__asan_default_opts` may not be bound when `ASanInitInternal`
+// is invoked (it is bound later).
+//
+// To work around the late binding on windows, `ApplyOptions` will be called,
+// again, after binding to the user-provided `__asan_default_opts` function.
+// Therefore, any flags not configured here are not guaranteed to be
+// configurable through `__asan_default_opts` on Windows.
+//
+//
+// For more details on this issue, see:
+// https://github.com/llvm/llvm-project/issues/117925
+void ApplyFlags() {
+  SetCanPoisonMemory(flags()->poison_heap);
+  SetMallocContextSize(common_flags()->malloc_context_size);
+
+  __asan_option_detect_stack_use_after_return =
+      flags()->detect_stack_use_after_return;
+
+  AllocatorOptions allocator_options;
+  allocator_options.SetFrom(flags(), common_flags());
+  ApplyAllocatorOptions(allocator_options);
+}
+
 static bool AsanInitInternal() {
   if (LIKELY(AsanInited()))
     return true;
@@ -436,8 +469,9 @@ static bool AsanInitInternal() {
 
   CacheBinaryName();
 
-  // Initialize flags. This must be done early, because most of the
-  // initialization steps look at flags().
+  // Initialize flags. On Windows it also also register weak function callbacks.
+  // This must be done early, because most of the initialization steps look at
+  // flags().
   InitializeFlags();
 
   WaitForDebugger(flags()->sleep_before_init, "before init");
@@ -455,9 +489,6 @@ static bool AsanInitInternal() {
   AsanCheckDynamicRTPrereqs();
   AvoidCVE_2016_2143();
 
-  SetCanPoisonMemory(flags()->poison_heap);
-  SetMallocContextSize(common_flags()->malloc_context_size);
-
   InitializePlatformExceptionHandlers();
 
   InitializeHighMemEnd();
@@ -468,10 +499,6 @@ static bool AsanInitInternal() {
   SetPrintfAndReportCallback(AppendToErrorMessageBuffer);
 
   __sanitizer_set_report_path(common_flags()->log_path);
-
-  __asan_option_detect_stack_use_after_return =
-      flags()->detect_stack_use_after_return;
-
   __sanitizer::InitializePlatformEarly();
 
   // Setup internal allocator callback.
@@ -511,6 +538,13 @@ static bool AsanInitInternal() {
   allocator_options.SetFrom(flags(), common_flags());
   InitializeAllocator(allocator_options);
 
+  // Apply ASan flags.
+  // NOTE: In order for options specified through `__asan_default_options` to be
+  // honored on Windows, it is necessary for those options to be configured
+  // inside the `ApplyOptions` method. See the function-level comment for
+  // `ApplyFlags` for more details.
+  ApplyFlags();
+
   if (SANITIZER_START_BACKGROUND_THREAD_IN_ASAN_INTERNAL)
     MaybeStartBackgroudThread();
 
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
index ba1eab1a90f4..f485404758b5 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cpp
@@ -80,8 +80,8 @@ static void *MallocStress(void *NumOfItrPtr) {
         case 2: size += 4096; break;
       }
       size_t alignment = 1 << (my_rand_r(&seed) % 10 + 1);
-      char *ptr = (char*)__asan::asan_memalign(alignment, size,
-                                               &stack2, __asan::FROM_MALLOC);
+      char *ptr = (char *)__asan::asan_memalign(alignment, size, &stack2,
+                                                __asan::FROM_MALLOC);
       EXPECT_EQ(size, __asan::asan_malloc_usable_size(ptr, 0, 0));
       vec.push_back(ptr);
       ptr[0] = 0;
diff --git a/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S b/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
index 19311396b201..0271afe7844f 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_cdcmp.S
@@ -128,7 +128,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple)
         msr APSR_nzcvq, ip
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop {r0-r3, r12, lr}
-        bxaut r12, lr, sp
+        PAC_RETURN
 #else
         pop {r0-r3}
         POP_PC()
diff --git a/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S b/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
index ca382968cf5d..5cb7d16042f4 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_cfcmp.S
@@ -128,7 +128,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple)
         msr APSR_nzcvq, ip
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop {r0-r3, r12, lr}
-        bxaut r12, lr, sp
+        PAC_RETURN
 #else
         pop {r0-r3}
         POP_PC()
diff --git a/compiler-rt/lib/builtins/arm/aeabi_dcmp.S b/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
index 4d1cc77d9a28..2ed13789d2f3 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_dcmp.S
@@ -29,7 +29,7 @@
 #  define PROLOGUE PACBTI_LANDING        SEPARATOR \
                    push      { r12, lr }
 #  define EPILOGUE pop       { r12, lr } SEPARATOR \
-                   bxaut     r12, lr, sp
+                   PAC_RETURN
 #elif defined(__ARM_FEATURE_BTI_DEFAULT)
 #  define PROLOGUE PACBTI_LANDING        SEPARATOR \
                    push      { r4, lr }
diff --git a/compiler-rt/lib/builtins/arm/aeabi_fcmp.S b/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
index 190a111d3447..0e3d399a552b 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_fcmp.S
@@ -29,7 +29,7 @@
 #  define PROLOGUE PACBTI_LANDING        SEPARATOR \
                    push      { r12, lr }
 #  define EPILOGUE pop       { r12, lr } SEPARATOR \
-                   bxaut     r12, lr, sp
+                   PAC_RETURN
 #elif defined(__ARM_FEATURE_BTI_DEFAULT)
 #  define PROLOGUE PACBTI_LANDING        SEPARATOR \
                    push      { r4, lr }
diff --git a/compiler-rt/lib/builtins/arm/aeabi_idivmod.S b/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
index 4b82c946b1eb..d95b1db0b459 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_idivmod.S
@@ -49,7 +49,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod)
         add     sp, sp, #4
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop     { r12, lr }
-        bxaut   r12, lr, sp
+        PAC_RETURN
 #else
         pop     { pc }
 #endif
diff --git a/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S b/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
index c11ead68ff8f..4fbfe3f1d265 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_ldivmod.S
@@ -45,7 +45,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_ldivmod)
         add     sp, sp, #16
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop     {r6, r12, lr}
-        bxaut   r12, lr, sp
+        PAC_RETURN
 #else
         pop     {r6, pc}
 #endif
diff --git a/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S b/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
index db2eecb3234f..f77beab77411 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_uidivmod.S
@@ -56,7 +56,7 @@ LOCAL_LABEL(case_denom_larger):
         add     sp, sp, #4
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop     { r12, lr }
-        bxaut   r12, lr, sp
+        PAC_RETURN
 #else
         pop     { pc }
 #endif
diff --git a/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S b/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
index 66da6a39ef5d..e019ba721b75 100644
--- a/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
+++ b/compiler-rt/lib/builtins/arm/aeabi_uldivmod.S
@@ -45,7 +45,7 @@ DEFINE_COMPILERRT_FUNCTION(__aeabi_uldivmod)
         add	sp, sp, #16
 #if defined(__ARM_FEATURE_PAC_DEFAULT)
         pop     {r6, r12, lr}
-        bxaut   r12, lr, sp
+        PAC_RETURN
 #else
         pop     {r6, pc}
 #endif
diff --git a/compiler-rt/lib/builtins/assembly.h b/compiler-rt/lib/builtins/assembly.h
index 714bcd4d5c1f..89372f18c84b 100644
--- a/compiler-rt/lib/builtins/assembly.h
+++ b/compiler-rt/lib/builtins/assembly.h
@@ -205,6 +205,12 @@
 #define PACBTI_LANDING
 #endif
 
+#if defined(__ARM_FEATURE_PAUTH)
+#define PAC_RETURN bxaut r12, lr, sp
+#else
+#define PAC_RETURN aut r12, lr, sp SEPARATOR bx lr
+#endif
+
 #else // !defined(__arm)
 #define DECLARE_FUNC_ENCODING
 #define DEFINE_CODE_STATE
diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp
index a3c0c2e485af..67879e37fad5 100644
--- a/compiler-rt/lib/msan/msan.cpp
+++ b/compiler-rt/lib/msan/msan.cpp
@@ -352,16 +352,38 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
 
 using namespace __msan;
 
-#define MSAN_MAYBE_WARNING(type, size)              \
-  void __msan_maybe_warning_##size(type s, u32 o) { \
-    GET_CALLER_PC_BP;                               \
-    if (UNLIKELY(s)) {                              \
-      PrintWarningWithOrigin(pc, bp, o);            \
-      if (__msan::flags()->halt_on_error) {         \
-        Printf("Exiting\n");                        \
-        Die();                                      \
-      }                                             \
-    }                                               \
+// N.B. Only [shadow, shadow+size) is defined. shadow is *not* a pointer into
+// an MSan shadow region.
+static void print_shadow_value(void *shadow, u64 size) {
+  Printf("Shadow value (%llu byte%s):", size, size == 1 ? "" : "s");
+  for (unsigned int i = 0; i < size; i++) {
+    if (i % 4 == 0)
+      Printf(" ");
+
+    unsigned char x = ((unsigned char *)shadow)[i];
+    Printf("%x%x", x >> 4, x & 0xf);
+  }
+  Printf("\n");
+  Printf(
+      "Caveat: the shadow value does not necessarily directly correspond to a "
+      "single user variable. The correspondence is stronger, but not always "
+      "perfect, when origin tracking is enabled.\n");
+  Printf("\n");
+}
+
+#define MSAN_MAYBE_WARNING(type, size)               \
+  void __msan_maybe_warning_##size(type s, u32 o) {  \
+    GET_CALLER_PC_BP;                                \
+                                                     \
+    if (UNLIKELY(s)) {                               \
+      if (Verbosity() >= 1)                          \
+        print_shadow_value((void *)(&s), sizeof(s)); \
+      PrintWarningWithOrigin(pc, bp, o);             \
+      if (__msan::flags()->halt_on_error) {          \
+        Printf("Exiting\n");                         \
+        Die();                                       \
+      }                                              \
+    }                                                \
   }
 
 MSAN_MAYBE_WARNING(u8, 1)
@@ -369,6 +391,30 @@ MSAN_MAYBE_WARNING(u16, 2)
 MSAN_MAYBE_WARNING(u32, 4)
 MSAN_MAYBE_WARNING(u64, 8)
 
+// N.B. Only [shadow, shadow+size) is defined. shadow is *not* a pointer into
+// an MSan shadow region.
+void __msan_maybe_warning_N(void *shadow, u64 size, u32 o) {
+  GET_CALLER_PC_BP;
+
+  bool allZero = true;
+  for (unsigned int i = 0; i < size; i++) {
+    if (((char *)shadow)[i]) {
+      allZero = false;
+      break;
+    }
+  }
+
+  if (UNLIKELY(!allZero)) {
+    if (Verbosity() >= 1)
+      print_shadow_value(shadow, size);
+    PrintWarningWithOrigin(pc, bp, o);
+    if (__msan::flags()->halt_on_error) {
+      Printf("Exiting\n");
+      Die();
+    }
+  }
+}
+
 #define MSAN_MAYBE_STORE_ORIGIN(type, size)                       \
   void __msan_maybe_store_origin_##size(type s, void *p, u32 o) { \
     if (UNLIKELY(s)) {                                            \
diff --git a/compiler-rt/lib/msan/msan_interface_internal.h b/compiler-rt/lib/msan/msan_interface_internal.h
index c2eead13c20c..75425b98166a 100644
--- a/compiler-rt/lib/msan/msan_interface_internal.h
+++ b/compiler-rt/lib/msan/msan_interface_internal.h
@@ -60,6 +60,8 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __msan_maybe_warning_4(u32 s, u32 o);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __msan_maybe_warning_8(u64 s, u32 o);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __msan_maybe_warning_N(void *shadow, u64 size, u32 o);
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __msan_maybe_store_origin_1(u8 s, void *p, u32 o);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index a4bc3d6cff91..14b25a8995da 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -22,6 +22,7 @@
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_posix.h"
@@ -747,6 +748,41 @@ TSAN_INTERCEPTOR(void, free, void *p) {
   user_free(thr, pc, p);
 }
 
+#  if SANITIZER_INTERCEPT_FREE_SIZED
+TSAN_INTERCEPTOR(void, free_sized, void *p, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free_sized, p, size);
+  user_free(thr, pc, p);
+}
+#    define TSAN_MAYBE_INTERCEPT_FREE_SIZED INTERCEPT_FUNCTION(free_sized)
+#  else
+#    define TSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  endif
+
+#  if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+TSAN_INTERCEPTOR(void, free_aligned_sized, void *p, uptr alignment, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (in_symbolizer())
+    return InternalFree(p);
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  invoke_free_hook(p);
+  SCOPED_INTERCEPTOR_RAW(free_aligned_sized, p, alignment, size);
+  user_free(thr, pc, p);
+}
+#    define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED \
+      INTERCEPT_FUNCTION(free_aligned_sized)
+#  else
+#    define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
+#  endif
+
 TSAN_INTERCEPTOR(void, cfree, void *p) {
   if (UNLIKELY(!p))
     return;
@@ -763,6 +799,9 @@ TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
   return user_alloc_usable_size(p);
 }
+#else
+#  define TSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  define TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
 #endif
 
 TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) {
@@ -2963,6 +3002,8 @@ void InitializeInterceptors() {
   TSAN_INTERCEPT(realloc);
   TSAN_INTERCEPT(reallocarray);
   TSAN_INTERCEPT(free);
+  TSAN_MAYBE_INTERCEPT_FREE_SIZED;
+  TSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED;
   TSAN_INTERCEPT(cfree);
   TSAN_INTERCEPT(munmap);
   TSAN_MAYBE_INTERCEPT_MEMALIGN;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
index 7c15a1638826..cb4d767d903d 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_java.cpp
@@ -122,7 +122,6 @@ void __tsan_java_move(jptr src, jptr dst, jptr size) {
   DCHECK_GE(dst, jctx->heap_begin);
   DCHECK_LE(dst + size, jctx->heap_begin + jctx->heap_size);
   DCHECK_NE(dst, src);
-  DCHECK_NE(size, 0);
 
   // Assuming it's not running concurrently with threads that do
   // memory accesses and mutex operations (stop-the-world phase).
diff --git a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
index e973be963e57..9d097806fa9b 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_malloc_mac.cpp
@@ -73,15 +73,19 @@ using namespace __tsan;
   invoke_free_hook(ptr);                                     \
   SCOPED_INTERCEPTOR_RAW(free, ptr);                         \
   user_free(thr, pc, ptr)
-#define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
-#define COMMON_MALLOC_FILL_STATS(zone, stats)
-#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
-  (void)zone_name; \
-  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
-#define COMMON_MALLOC_NAMESPACE __tsan
-#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
-#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
+#  define COMMON_MALLOC_FREE_SIZED(ptr, size) COMMON_MALLOC_FREE(ptr)
+#  define COMMON_MALLOC_FREE_ALIGNED_SIZED(ptr, alignment, size) \
+    COMMON_MALLOC_FREE(ptr)
+#  define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr);
+#  define COMMON_MALLOC_FILL_STATS(zone, stats)
+#  define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name)    \
+    (void)zone_name;                                                        \
+    Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", \
+           ptr);
+#  define COMMON_MALLOC_NAMESPACE __tsan
+#  define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
+#  define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
 
-#include "sanitizer_common/sanitizer_malloc_mac.inc"
+#  include "sanitizer_common/sanitizer_malloc_mac.inc"
 
 #endif
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index 981f37b89e78..0d7247a56a4c 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -566,17 +566,32 @@ static bool IsValidMmapRange(uptr addr, uptr size) {
   return false;
 }
 
-void UnmapShadow(ThreadState *thr, uptr addr, uptr size) {
+void UnmapShadow(ThreadState* thr, uptr addr, uptr size) {
   if (size == 0 || !IsValidMmapRange(addr, size))
     return;
-  DontNeedShadowFor(addr, size);
+  // unmap shadow is related to semantic of mmap/munmap, so we
+  // should clear the whole shadow range, including the tail shadow
+  // while addr + size % kShadowCell != 0.
+  uptr rounded_size_shadow = RoundUp(addr + size, kShadowCell) - addr;
+  DontNeedShadowFor(addr, rounded_size_shadow);
   ScopedGlobalProcessor sgp;
   SlotLocker locker(thr, true);
-  ctx->metamap.ResetRange(thr->proc(), addr, size, true);
+  uptr rounded_size_meta = RoundUp(addr + size, kMetaShadowCell) - addr;
+  ctx->metamap.ResetRange(thr->proc(), addr, rounded_size_meta, true);
 }
 #endif
 
 void MapShadow(uptr addr, uptr size) {
+  // Although named MapShadow, this function's semantic is unrelated to
+  // UnmapShadow. This function currently only used for Go's lazy allocation
+  // of shadow, whose targets are program section (e.g., bss, data, etc.).
+  // Therefore, we can guarantee that the addr and size align to kShadowCell
+  // and kMetaShadowCell by the following assertions.
+  DCHECK_EQ(addr % kShadowCell, 0);
+  DCHECK_EQ(size % kShadowCell, 0);
+  DCHECK_EQ(addr % kMetaShadowCell, 0);
+  DCHECK_EQ(size % kMetaShadowCell, 0);
+
   // Ensure thead registry lock held, so as to synchronize
   // with DoReset, which also access the mapped_shadow_* ctxt fields.
   ThreadRegistryLock lock0(&ctx->thread_registry);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
index bd8deefefa1b..487fa490636e 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
@@ -688,16 +688,18 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
     DCHECK(IsShadowMem(shadow_mem));
   }
 
-  RawShadow* shadow_mem_end = reinterpret_cast<RawShadow*>(
-      reinterpret_cast<uptr>(shadow_mem) + size * kShadowMultiplier - 1);
-  if (!IsShadowMem(shadow_mem_end)) {
-    Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end,
+  uptr rounded_size =
+      (RoundUpTo(addr + size, kShadowCell) - RoundDownTo(addr, kShadowCell));
+  RawShadow* shadow_mem_end =
+      shadow_mem + rounded_size / kShadowCell * kShadowCnt;
+  if (!IsShadowMem(shadow_mem_end - 1)) {
+    Printf("Bad shadow end addr: %p (%p)\n", shadow_mem_end - 1,
            (void*)(addr + size - 1));
     Printf(
-        "Shadow start addr (ok): %p (%p); size: 0x%zx; kShadowMultiplier: "
-        "%zx\n",
-        shadow_mem, (void*)addr, size, kShadowMultiplier);
-    DCHECK(IsShadowMem(shadow_mem_end));
+        "Shadow start addr (ok): %p (%p); size: 0x%zx; rounded_size: 0x%zx; "
+        "kShadowMultiplier: %zx\n",
+        shadow_mem, (void*)addr, size, rounded_size, kShadowMultiplier);
+    DCHECK(IsShadowMem(shadow_mem_end - 1));
   }
 #endif
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
index 97335bc8ecf7..be5829bc823d 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_sync.cpp
@@ -246,6 +246,20 @@ void MetaMap::MoveMemory(uptr src, uptr dst, uptr sz) {
   // there are no concurrent accesses to the regions (e.g. stop-the-world).
   CHECK_NE(src, dst);
   CHECK_NE(sz, 0);
+
+  // The current MoveMemory implementation behaves incorrectly when src, dst,
+  // and sz are not aligned to kMetaShadowCell.
+  // For example, with kMetaShadowCell == 8:
+  // - src = 4: unexpectedly clears the metadata for the range [0, 4).
+  // - src = 16, dst = 4, size = 8: A sync variable for addr = 20, which should
+  //   be moved to the metadata for address 8, is incorrectly moved to the
+  //   metadata for address 0 instead.
+  // - src = 0, sz = 4: fails to move the tail metadata.
+  // Therefore, the following assertions is needed.
+  DCHECK_EQ(src % kMetaShadowCell, 0);
+  DCHECK_EQ(dst % kMetaShadowCell, 0);
+  DCHECK_EQ(sz % kMetaShadowCell, 0);
+
   uptr diff = dst - src;
   u32 *src_meta, *dst_meta, *src_meta_end;
   uptr inc;
diff --git a/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp b/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp
new file mode 100644
index 000000000000..c7d62f15c3c3
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/alloc_dealloc_mismatch.cpp
@@ -0,0 +1,29 @@
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-MISMATCH
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SUCCESS
+
+// RUN: %clangxx_asan -O0 %s -o %t -DUSER_FUNCTION
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-MISMATCH
+
+// It is expected that ASAN_OPTS will override the value set through the user function.
+// RUN: %env_asan_opts=alloc_dealloc_mismatch=0 %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SUCCESS
+
+#if USER_FUNCTION
+// It's important to test the `alloc_dealloc_mismatch` flag set through the user function because, on Windows,
+// flags configured through the user-defined function `__asan_default_options` are not always be honored.
+// See: https://github.com/llvm/llvm-project/issues/117925
+extern "C" __declspec(dllexport) extern const char *__asan_default_options() {
+  return "alloc_dealloc_mismatch=1";
+}
+#endif
+
+#include <cstdio>
+#include <cstdlib>
+
+// Tests the `alloc_dealloc_mismatch` flag set both via user function and through the environment variable.
+int main() {
+  // In the 'CHECK-MISMATCH' case, we simply check that the AddressSanitizer reports an error.
+  delete (new int[10]); // CHECK-MISMATCH: AddressSanitizer:
+  printf("Success");    // CHECK-SUCCESS: Success
+  return 0;
+}
\ No newline at end of file
diff --git a/compiler-rt/test/fuzzer/uncaught-exception.test b/compiler-rt/test/fuzzer/uncaught-exception.test
index d1b98cfb7c74..264bb5dc3a91 100644
--- a/compiler-rt/test/fuzzer/uncaught-exception.test
+++ b/compiler-rt/test/fuzzer/uncaught-exception.test
@@ -6,8 +6,6 @@ RUN: %cpp_compiler %S/UncaughtException.cpp -o %t-UncaughtException
 
 # Clang will fail the test with 'deadly signal', but other compilers may fail with different error messages.
 # For example, msvc fails with 'uncaught C++ exception'. So the error we check depends on the compiler target.
-RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s --check-prefixes=CHECK-CRASH,%if target={{.*-windows-msvc.*}} %{CHECK-MSVC%} %else %{CHECK-ERROR%}
+RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s
 
-CHECK-ERROR: ERROR: libFuzzer: deadly signal
-CHECK-MSVC: ERROR: libFuzzer: uncaught C++ exception
-CHECK-CRASH: Test unit written to ./crash
+CHECK: ERROR: libFuzzer: {{deadly signal|uncaught C\+\+ exception}}
diff --git a/compiler-rt/test/msan/msan_print_shadow_on_outlined_check.cpp b/compiler-rt/test/msan/msan_print_shadow_on_outlined_check.cpp
new file mode 100644
index 000000000000..74de0c322f60
--- /dev/null
+++ b/compiler-rt/test/msan/msan_print_shadow_on_outlined_check.cpp
@@ -0,0 +1,42 @@
+// RUN: %clangxx_msan -fsanitize-recover=memory -mllvm -msan-instrumentation-with-call-threshold=0 -g %s -o %t \
+// RUN:   && not env MSAN_OPTIONS=verbosity=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: x86_64-target-arch
+// 'long double' implementation varies between platforms.
+
+#include <ctype.h>
+#include <stdio.h>
+
+#include <sanitizer/msan_interface.h>
+
+int main(int argc, char *argv[]) {
+  long double a;
+  printf("a: %Lf\n", a);
+  // CHECK: Shadow value (16 bytes): ffffffff ffffffff ffff0000 00000000
+
+  unsigned long long b;
+  printf("b: %llu\n", b);
+  // CHECK: Shadow value (8 bytes): ffffffff ffffffff
+
+  char *p = (char *)(&b);
+  p[2] = 36;
+  printf("b: %lld\n", b);
+  // CHECK: Shadow value (8 bytes): ffff00ff ffffffff
+
+  b = b << 8;
+  printf("b: %lld\n", b);
+  __msan_print_shadow(&b, sizeof(b));
+  // CHECK: Shadow value (8 bytes): 00ffff00 ffffffff
+
+  unsigned int c;
+  printf("c: %u\n", c);
+  // CHECK: Shadow value (4 bytes): ffffffff
+
+  // Converted to boolean
+  if (c) {
+    // CHECK: Shadow value (1 byte): 01
+    printf("Hello\n");
+  }
+
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
index 7710c6236819..dc11e7fa5e9b 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
index 9eac562fecb0..7a993d481bc5 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
index 57a01f3852b0..ddc260fa6164 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_env.py
@@ -5,13 +5,13 @@
 
 idx = 1
 for arg in sys.argv[1:]:
-  if not "=" in arg:
-    break
-  idx += 1
-  (argname, argval) = arg.split("=")
-  os.environ["SIMCTL_CHILD_" + argname] = argval
+    if not "=" in arg:
+        break
+    idx += 1
+    (argname, argval) = arg.split("=", maxsplit=1)
+    os.environ["SIMCTL_CHILD_" + argname] = argval
 
 exitcode = subprocess.call(sys.argv[idx:])
 if exitcode > 125:
-  exitcode = 126
+    exitcode = 126
 sys.exit(exitcode)
diff --git a/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py b/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
index 5e977ea5ed90..ec394067dc7a 100755
--- a/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
+++ b/compiler-rt/test/sanitizer_common/ios_commands/iossim_run.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import glob, os, pipes, sys, subprocess
+import glob, os, shlex, sys, subprocess
 
 
 device_id = os.environ.get("SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER")
@@ -21,8 +21,11 @@
     "ASAN_ACTIVATION_OPTIONS",
     "MallocNanoZone",
 ]:
-    if e in os.environ:
-        os.environ["SIMCTL_CHILD_" + e] = os.environ[e]
+    simctl_version = "SIMCTL_CHILD_" + e
+    # iossim_env.py might have already set these using arguments it was given
+    # (and that we can't see from inside this script). Don't overwrite them!
+    if e in os.environ and simctl_version not in os.environ:
+        os.environ[simctl_version] = os.environ[e]
 
 find_atos_cmd = "xcrun -sdk iphonesimulator -f atos"
 atos_path = (
@@ -49,8 +52,7 @@
             # Don't quote glob pattern
             rm_args.append(arg)
         else:
-            # FIXME(dliew): pipes.quote() is deprecated
-            rm_args.append(pipes.quote(arg))
+            rm_args.append(shlex.quote(arg))
     rm_cmd_line = ["/bin/rm"] + rm_args
     rm_cmd_line_str = " ".join(rm_cmd_line)
     # We use `shell=True` so that any wildcard globs get expanded by the shell.
diff --git a/compiler-rt/test/sanitizer_common/lit.common.cfg.py b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
index c3c1336bacd5..88d3ea9bc5ad 100644
--- a/compiler-rt/test/sanitizer_common/lit.common.cfg.py
+++ b/compiler-rt/test/sanitizer_common/lit.common.cfg.py
@@ -87,7 +87,7 @@ def build_invocation(compile_flags):
 config.substitutions.append(("%tool_name", config.tool_name))
 config.substitutions.append(("%tool_options", tool_options))
 config.substitutions.append(
-    ("%env_tool_opts=", "env " + tool_options + "=" + default_tool_options_str)
+    ("%env_tool_opts=", "%env " + tool_options + "=" + default_tool_options_str)
 )
 
 config.suffixes = [".c", ".cpp"]
diff --git a/compiler-rt/test/tsan/java_heap_init2.cpp b/compiler-rt/test/tsan/java_heap_init2.cpp
new file mode 100644
index 000000000000..3317a5b3c2f4
--- /dev/null
+++ b/compiler-rt/test/tsan/java_heap_init2.cpp
@@ -0,0 +1,33 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+
+#include "java.h"
+#include <errno.h>
+#include <sys/mman.h>
+
+int main() {
+  // Test a non-regular kHeapSize
+  // Previously __tsan_java_init failed because it encountered non-zero meta
+  // shadow for the destination.
+  size_t const kPageSize = sysconf(_SC_PAGESIZE);
+  int const kSize = kPageSize - 1;
+  jptr jheap2 = (jptr)mmap(0, kSize, PROT_READ | PROT_WRITE,
+                           MAP_ANON | MAP_PRIVATE, -1, 0);
+  if (jheap2 == (jptr)MAP_FAILED)
+    return printf("mmap failed with %d\n", errno);
+  __atomic_store_n((int *)(jheap2 + kSize - 3), 1, __ATOMIC_RELEASE);
+  // Due to the previous incorrect meta-end calculation, the following munmap
+  // did not clear the tail meta shadow.
+  munmap((void *)jheap2, kSize);
+  int const kHeapSize2 = kSize + 1;
+  jheap2 = (jptr)mmap((void *)jheap2, kHeapSize2, PROT_READ | PROT_WRITE,
+                      MAP_ANON | MAP_PRIVATE, -1, 0);
+  if (jheap2 == (jptr)MAP_FAILED)
+    return printf("second mmap failed with %d\n", errno);
+  __tsan_java_init(jheap2, kHeapSize2);
+  __tsan_java_move(jheap2, jheap2 + kHeapSize2 - 8, 8);
+  fprintf(stderr, "DONE\n");
+  return __tsan_java_fini();
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer: data race
+// CHECK: DONE
diff --git a/compiler-rt/test/tsan/munmap_clear_shadow.c b/compiler-rt/test/tsan/munmap_clear_shadow.c
new file mode 100644
index 000000000000..1fbc77ec8fd1
--- /dev/null
+++ b/compiler-rt/test/tsan/munmap_clear_shadow.c
@@ -0,0 +1,63 @@
+// RUN: %clang_tsan %s -o %t && %run %t | FileCheck %s
+
+// In these systems, the behavior of ReleaseMemoryPagesToOS is madvise(beg, end, MADV_FREE),
+// which tags the relevant pages as 'FREE' and does not release them immediately.
+// Therefore, we cannot assume that __tsan_read1 will not race with the shadow cleared.
+// UNSUPPORTED: darwin,target={{.*(freebsd|netbsd|solaris|haiku).*}}
+
+#include "test.h"
+#include <assert.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+void __tsan_read1(void *addr);
+
+struct thread_params {
+  char *buf;
+  unsigned int size;
+};
+
+static void *thread_func(void *arg) {
+  struct thread_params *p = (struct thread_params *)arg;
+  // Access 1
+  p->buf[0] = 0x42;
+  p->buf[p->size - 1] = 0x42;
+  barrier_wait(&barrier);
+  return 0;
+}
+
+int main() {
+  const unsigned int kPageSize = sysconf(_SC_PAGESIZE);
+  // The relevant shadow memory size should be exactly multiple of kPageSize,
+  // even if Size = kPageSize - 1.
+  const unsigned int Size = kPageSize - 1;
+
+  barrier_init(&barrier, 2);
+  char *buf = (char *)mmap(NULL, Size, PROT_READ | PROT_WRITE,
+                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  assert(buf != MAP_FAILED);
+  assert(((uintptr_t)buf % kPageSize) == 0);
+
+  pthread_t t;
+  struct thread_params p = {buf, Size};
+  pthread_create(&t, 0, thread_func, &p);
+
+  barrier_wait(&barrier);
+  // Should clear all the shadow memory related to the mmaped memory.
+  munmap(buf, Size);
+
+  // If the shadow memory is cleared completely, the following reads should not
+  // cause races and behave the same. However, previously, __tsan_read1(&buf[0])
+  // would not report a race, while __tsan_read1(&buf[Size - 1]) did.
+  // CHECK-NOT: WARNING: ThreadSanitizer: data race
+  __tsan_read1(&buf[0]);        // Access 2
+  __tsan_read1(&buf[Size - 1]); // Access 2
+  pthread_join(t, 0);
+
+  puts("DONE");
+
+  return 0;
+}
diff --git a/cross-project-tests/CMakeLists.txt b/cross-project-tests/CMakeLists.txt
index 7f2fee48fda7..b4b1f4762607 100644
--- a/cross-project-tests/CMakeLists.txt
+++ b/cross-project-tests/CMakeLists.txt
@@ -94,6 +94,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}
diff --git a/cross-project-tests/dtlto/README.md b/cross-project-tests/dtlto/README.md
new file mode 100644
index 000000000000..12f9aa19b0d9
--- /dev/null
+++ b/cross-project-tests/dtlto/README.md
@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.
\ No newline at end of file
diff --git a/cross-project-tests/dtlto/ld-dtlto.c b/cross-project-tests/dtlto/ld-dtlto.c
new file mode 100644
index 000000000000..3ee962346bd4
--- /dev/null
+++ b/cross-project-tests/dtlto/ld-dtlto.c
@@ -0,0 +1,41 @@
+// REQUIRES: ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | sort | FileCheck %s
+
+/// No files are expected before.
+// CHECK-NOT: {{.}}
+
+/// Linked ELF.
+// CHECK: {{^}}a.out{{$}}
+
+/// Produced by the bitcode compilation.
+// CHECK-NEXT: {{^}}dtlto.o{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-NEXT: {{^}}dtlto.s{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
+
+/// No files are expected after.
+// CHECK-NOT: {{.}}
+
+int _start() { return 0; }
diff --git a/cross-project-tests/dtlto/lit.local.cfg b/cross-project-tests/dtlto/lit.local.cfg
new file mode 100644
index 000000000000..38eed282704c
--- /dev/null
+++ b/cross-project-tests/dtlto/lit.local.cfg
@@ -0,0 +1,5 @@
+if any(
+    f not in config.available_features
+    for f in ("clang", "x86-registered-target")
+):
+    config.unsupported = True
diff --git a/flang-rt/lib/runtime/descriptor-io.cpp b/flang-rt/lib/runtime/descriptor-io.cpp
index ce080adbc916..536d93e823f7 100644
--- a/flang-rt/lib/runtime/descriptor-io.cpp
+++ b/flang-rt/lib/runtime/descriptor-io.cpp
@@ -451,39 +451,42 @@ RT_API_ATTRS int DescriptorIoTicket<DIR>::Begin(WorkQueue &workQueue) {
     if (const typeInfo::DerivedType *type{
             addendum ? addendum->derivedType() : nullptr}) {
       // derived type unformatted I/O
-      if (table_) {
-        if (const auto *definedIo{table_->Find(*type,
-                DIR == Direction::Input
-                    ? common::DefinedIo::ReadUnformatted
-                    : common::DefinedIo::WriteUnformatted)}) {
-          if (definedIo->subroutine) {
-            typeInfo::SpecialBinding special{DIR == Direction::Input
-                    ? typeInfo::SpecialBinding::Which::ReadUnformatted
-                    : typeInfo::SpecialBinding::Which::WriteUnformatted,
-                definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
-                false};
-            if (DefinedUnformattedIo(io_, instance_, *type, special)) {
-              anyIoTookPlace_ = true;
-              return StatOk;
+      if (DIR == Direction::Input || !io_.get_if<InquireIOLengthState>()) {
+        if (table_) {
+          if (const auto *definedIo{table_->Find(*type,
+                  DIR == Direction::Input
+                      ? common::DefinedIo::ReadUnformatted
+                      : common::DefinedIo::WriteUnformatted)}) {
+            if (definedIo->subroutine) {
+              typeInfo::SpecialBinding special{DIR == Direction::Input
+                      ? typeInfo::SpecialBinding::Which::ReadUnformatted
+                      : typeInfo::SpecialBinding::Which::WriteUnformatted,
+                  definedIo->subroutine, definedIo->isDtvArgPolymorphic, false,
+                  false};
+              if (DefinedUnformattedIo(io_, instance_, *type, special)) {
+                anyIoTookPlace_ = true;
+                return StatOk;
+              }
+            } else {
+              int status{workQueue.BeginDerivedIo<DIR>(
+                  io_, instance_, *type, table_, anyIoTookPlace_)};
+              return status == StatContinue ? StatOk : status; // done here
             }
-          } else {
-            int status{workQueue.BeginDerivedIo<DIR>(
-                io_, instance_, *type, table_, anyIoTookPlace_)};
-            return status == StatContinue ? StatOk : status; // done here
           }
         }
-      }
-      if (const typeInfo::SpecialBinding *special{
-              type->FindSpecialBinding(DIR == Direction::Input
-                      ? typeInfo::SpecialBinding::Which::ReadUnformatted
-                      : typeInfo::SpecialBinding::Which::WriteUnformatted)}) {
-        if (!table_ || !table_->ignoreNonTbpEntries || special->IsTypeBound()) {
-          // defined derived type unformatted I/O
-          if (DefinedUnformattedIo(io_, instance_, *type, *special)) {
-            anyIoTookPlace_ = true;
-            return StatOk;
-          } else {
-            return IostatEnd;
+        if (const typeInfo::SpecialBinding *special{
+                type->FindSpecialBinding(DIR == Direction::Input
+                        ? typeInfo::SpecialBinding::Which::ReadUnformatted
+                        : typeInfo::SpecialBinding::Which::WriteUnformatted)}) {
+          if (!table_ || !table_->ignoreNonTbpEntries ||
+              special->IsTypeBound()) {
+            // defined derived type unformatted I/O
+            if (DefinedUnformattedIo(io_, instance_, *type, *special)) {
+              anyIoTookPlace_ = true;
+              return StatOk;
+            } else {
+              return IostatEnd;
+            }
           }
         }
       }
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 068d134671db..0bfada476348 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -504,9 +504,10 @@ add_subdirectory(include)
 add_subdirectory(lib)
 add_subdirectory(cmake/modules)
 
+option(FLANG_INCLUDE_TOOLS "Generate build targets for the Flang tools." ON)
 option(FLANG_BUILD_TOOLS
   "Build the Flang tools. If OFF, just generate build targets." ON)
-if (FLANG_BUILD_TOOLS)
+if (FLANG_INCLUDE_TOOLS)
   add_subdirectory(tools)
 endif()
 
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 871749934810..37a49f12f917 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -868,6 +868,20 @@ print *, [(j,j=1,10)]
   the elements for each component before proceeding to the next component.
   A program using defined assignment might be able to detect the difference.
 
+* The standard forbids instances of derived types with defined unformatted
+  WRITE subroutines from appearing in the I/O list of an `INQUIRE(IOLENGTH=...)`
+  statement.  It then also says that these defined I/O procedures should be
+  ignored for that statement.  So we allow them to appear (like most
+  compilers) and don't use any defined unformatted WRITE that might have been
+  defined.
+
+* Forward references to target objects are allowed to appear
+  in the initializers of data pointer declarationss.
+  Forward references to target objects are not accepted in the default
+  initializers of derived type component declarations, however,
+  since these default values need to be available to process incomplete
+  structure constructors.
+
 ## De Facto Standard Features
 
 * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the
diff --git a/flang/docs/OpenMP-declare-target.md b/flang/docs/OpenMP-declare-target.md
index 45062469007b..478db2d004c8 100644
--- a/flang/docs/OpenMP-declare-target.md
+++ b/flang/docs/OpenMP-declare-target.md
@@ -7,7 +7,15 @@
 
 -->
 
-# Introduction to Declare Target
+# OpenMP Declare Target
+
+```{contents}
+---
+local:
+---
+```
+
+## Introduction to Declare Target
 
 In OpenMP `declare target` is a directive that can be applied to a function or
 variable (primarily global) to notate to the compiler that it should be 
@@ -55,7 +63,7 @@ in this case, leave off the `declare target` from the function and it
 would be implicitly marked `declare target any` (for both host and device), 
 as it's been utilised within a target region.
 
-# Declare Target as represented in the OpenMP Dialect
+## Declare Target as represented in the OpenMP Dialect
 
 In the OpenMP Dialect `declare target` is not represented by a specific 
 `operation`. Instead, it's an OpenMP dialect specific `attribute` that can be 
@@ -89,7 +97,7 @@ llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(Op.getOperation());
 declareTargetGlobal.isDeclareTarget();
 ```
 
-# Declare Target Fortran OpenMP Lowering
+## Declare Target Fortran OpenMP Lowering
 
 The initial lowering of `declare target` to MLIR for both use-cases is done
 inside of the usual OpenMP lowering in flang/lib/Lower/OpenMP.cpp. However, 
@@ -145,7 +153,7 @@ to the way `threadprivate` handles these cases, we raise and initialize the
 variable as an internal `GlobalOp` and apply the attribute. This occurs in the
 flang/lib/Lower/OpenMP.cpp function `genDeclareTargetIntGlobal`.
 
-# Declare Target Transformation Passes for Flang
+## Declare Target Transformation Passes for Flang
 
 There are currently two passes within Flang that are related to the processing 
 of `declare target`:
@@ -173,7 +181,7 @@ While this infrastructure could be generally applicable to more than just Flang,
 it is only utilised in the Flang frontend, so it resides there rather than in 
 the OpenMP dialect codebase. 
 
-# Declare Target OpenMP Dialect To LLVM-IR Lowering
+## Declare Target OpenMP Dialect To LLVM-IR Lowering
 
 The OpenMP dialect lowering of `declare target` is done through the 
 `amendOperation` flow, as it's not an `operation` but rather an 
@@ -228,7 +236,7 @@ place. It may be possible to store this information in the OMPIRBuilder and
 then perform this cleanup process on finalization, but this is open for 
 discussion and implementation still.
 
-# Current Support
+## Current Support
 
 For the moment, `declare target` should work for:
 * Marking functions/subroutines and function/subroutine interfaces for 
diff --git a/flang/docs/OpenMP-descriptor-management.md b/flang/docs/OpenMP-descriptor-management.md
index cbc27f2a0fcb..d0764c2df464 100644
--- a/flang/docs/OpenMP-descriptor-management.md
+++ b/flang/docs/OpenMP-descriptor-management.md
@@ -6,8 +6,15 @@
 
 -->
 
-# OpenMP dialect: Fortran descriptor type mapping for offload
+# Fortran descriptor type mapping for OpenMP offload
+```{contents}
+---
+local:
+---
+```
+
 
+## Details
 The initial method for mapping Fortran types tied to descriptors for OpenMP offloading is to treat these types 
 as a special case of OpenMP record type (C/C++ structure/class, Fortran derived type etc.) mapping as far as the 
 runtime is concerned. Where the box (descriptor information) is the holding container and the underlying 
@@ -103,7 +110,7 @@ it is a little rigid in how the descriptor mappings are handled as there is no s
 to specialise the mappings for possible edge cases without polluting the dialect or lowering with further
 knowledge of Fortran and the FIR dialect.
 
-# OpenMP dialect differences from OpenACC dialect
+## Differences from OpenACC
 
 The descriptor mapping for OpenMP currently works differently to the planned direction for OpenACC, however, 
 it is possible and would likely be ideal to align the method with OpenACC in the future. 
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 36be369595ff..b2dbbcb5630f 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -34,6 +34,8 @@ page](https://llvm.org/releases/).
 
 * -floop-interchange is now recognized by flang.
 * -floop-interchange is enabled by default at -O2 and above.
+* -fveclib=libmvec is supported for AArch64 (same as Flang/x86 and
+  Clang/AArch64) (requires GLIBC 2.40 or newer)
 
 ## Windows Support
 
@@ -54,6 +56,11 @@ page](https://llvm.org/releases/).
    now be emitted into Clang's per-target resource directory
    (next to libclang_rt.*.*) where it is also found by Flang's driver.
 
+  * Flang on AArch64 now always depends on compiler-rt to provide the
+    `__trampoline_setup` function. This dependency will be automatically added
+    in in-tree builds with the AArch64 target, but compiler-rt will need to be
+    manually added to LLVM builds when building flang out-of-tree.
+
 ## New Issues Found
 
 
diff --git a/flang/include/flang/Evaluate/complex.h b/flang/include/flang/Evaluate/complex.h
index 2dcd28b59968..720ccaf512df 100644
--- a/flang/include/flang/Evaluate/complex.h
+++ b/flang/include/flang/Evaluate/complex.h
@@ -45,7 +45,7 @@ template <typename REAL_TYPE> class Complex {
         im_.Compare(that.im_) == Relation::Equal;
   }
 
-  constexpr bool IsZero() const { return re_.IsZero() || im_.IsZero(); }
+  constexpr bool IsZero() const { return re_.IsZero() && im_.IsZero(); }
 
   constexpr bool IsInfinite() const {
     return re_.IsInfinite() || im_.IsInfinite();
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index e04621f71f9a..cad1b634f892 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1510,6 +1510,16 @@ Operator OperationCode(
   }
 }
 
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Extremum<T>, Ts...> &op) {
+  if (op.derived().ordering == evaluate::Ordering::Greater) {
+    return Operator::Max;
+  } else {
+    return Operator::Min;
+  }
+}
+
 template <typename T> Operator OperationCode(const evaluate::Constant<T> &x) {
   return Operator::Constant;
 }
@@ -1575,6 +1585,7 @@ bool IsExtensibleType(const DerivedTypeSpec *);
 bool IsSequenceOrBindCType(const DerivedTypeSpec *);
 bool IsBuiltinDerivedType(const DerivedTypeSpec *derived, const char *name);
 bool IsBuiltinCPtr(const Symbol &);
+bool IsFromBuiltinModule(const Symbol &);
 bool IsEventType(const DerivedTypeSpec *);
 bool IsLockType(const DerivedTypeSpec *);
 bool IsNotifyType(const DerivedTypeSpec *);
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 8ae68e143cd2..ef956a36917f 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -15,6 +15,7 @@
 
 #include "flang/Lower/LoweringOptions.h"
 #include "flang/Lower/PFTDefs.h"
+#include "flang/Lower/Support/Utils.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Dialect/FIRAttr.h"
 #include "flang/Semantics/symbol.h"
@@ -23,9 +24,11 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
 
 namespace mlir {
 class SymbolTable;
+class StateStack;
 }
 
 namespace fir {
@@ -266,7 +269,7 @@ class AbstractConverter {
   /// Generate the type from a category and kind and length parameters.
   virtual mlir::Type
   genType(Fortran::common::TypeCategory tc, int kind,
-          llvm::ArrayRef<std::int64_t> lenParameters = std::nullopt) = 0;
+          llvm::ArrayRef<std::int64_t> lenParameters = {}) = 0;
   /// Generate the type from a DerivedTypeSpec.
   virtual mlir::Type genType(const Fortran::semantics::DerivedTypeSpec &) = 0;
   /// Generate the type from a Variable
@@ -361,6 +364,8 @@ class AbstractConverter {
   /// functions in order to be in sync).
   virtual mlir::SymbolTable *getMLIRSymbolTable() = 0;
 
+  virtual mlir::StateStack &getStateStack() = 0;
+
 private:
   /// Options controlling lowering behavior.
   const Fortran::lower::LoweringOptions &loweringOptions;
diff --git a/flang/include/flang/Lower/LoweringOptions.def b/flang/include/flang/Lower/LoweringOptions.def
index d97abf4d864b..3263ab129d07 100644
--- a/flang/include/flang/Lower/LoweringOptions.def
+++ b/flang/include/flang/Lower/LoweringOptions.def
@@ -66,5 +66,9 @@ ENUM_LOWERINGOPT(RepackArraysWhole, unsigned, 1, 0)
 /// If true, CUDA Fortran runtime check is inserted.
 ENUM_LOWERINGOPT(CUDARuntimeCheck, unsigned, 1, 0)
 
+/// If true, do not generate definition for runtime type info global objects of
+/// derived types defined in other compilation units.
+ENUM_LOWERINGOPT(SkipExternalRttiDefinition, unsigned, 1, 0)
+
 #undef LOWERINGOPT
 #undef ENUM_LOWERINGOPT
diff --git a/flang/include/flang/Lower/SymbolMap.h b/flang/include/flang/Lower/SymbolMap.h
index c03f9afd4080..e52d1075e2b9 100644
--- a/flang/include/flang/Lower/SymbolMap.h
+++ b/flang/include/flang/Lower/SymbolMap.h
@@ -128,7 +128,7 @@ struct SymbolBox : public fir::details::matcher<SymbolBox> {
                                        const SymbolBox &symBox);
 
   /// Dump the map. For debugging.
-  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
+  LLVM_DUMP_METHOD void dump() const;
 
 private:
   VT box;
@@ -293,7 +293,7 @@ class SymMap {
                                        const SymMap &symMap);
 
   /// Dump the map. For debugging.
-  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << '\n'; }
+  LLVM_DUMP_METHOD void dump() const;
 
   void addVariableDefinition(semantics::SymbolRef symRef,
                              fir::FortranVariableOpInterface definingOp,
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index 9986f37d7d13..f2c363e09e9e 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -537,14 +537,14 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
   /// Create an IfOp with no "else" region, and no result values.
   /// Usage: genIfThen(loc, cdt).genThen(lambda).end();
   IfBuilder genIfThen(mlir::Location loc, mlir::Value cdt) {
-    auto op = create<fir::IfOp>(loc, std::nullopt, cdt, false);
+    auto op = create<fir::IfOp>(loc, mlir::TypeRange(), cdt, false);
     return IfBuilder(op, *this);
   }
 
   /// Create an IfOp with an "else" region, and no result values.
   /// Usage: genIfThenElse(loc, cdt).genThen(lambda).genElse(lambda).end();
   IfBuilder genIfThenElse(mlir::Location loc, mlir::Value cdt) {
-    auto op = create<fir::IfOp>(loc, std::nullopt, cdt, true);
+    auto op = create<fir::IfOp>(loc, mlir::TypeRange(), cdt, true);
     return IfBuilder(op, *this);
   }
 
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 17052113859e..d38c5b6d09a8 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -442,6 +442,9 @@ struct IntrinsicLibrary {
                                  llvm::ArrayRef<fir::ExtendedValue>);
   fir::ExtendedValue genTranspose(mlir::Type,
                                   llvm::ArrayRef<fir::ExtendedValue>);
+  mlir::Value genThisGrid(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genThisThreadBlock(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  mlir::Value genThisWarp(mlir::Type, llvm::ArrayRef<mlir::Value>);
   void genThreadFence(llvm::ArrayRef<fir::ExtendedValue>);
   void genThreadFenceBlock(llvm::ArrayRef<fir::ExtendedValue>);
   void genThreadFenceSystem(llvm::ArrayRef<fir::ExtendedValue>);
@@ -750,7 +753,7 @@ static inline mlir::FunctionType genFuncType(mlir::MLIRContext *context,
   }
 
   if (TyR::ty == ParamTypeId::Void)
-    return mlir::FunctionType::get(context, argTypes, std::nullopt);
+    return mlir::FunctionType::get(context, argTypes, {});
 
   auto resType = getTypeHelper(context, builder, TyR::ty, TyR::kind);
   return mlir::FunctionType::get(context, argTypes, {resType});
diff --git a/flang/include/flang/Optimizer/CodeGen/CodeGen.h b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
index 0398d0f248e0..93f07d8d5d4d 100644
--- a/flang/include/flang/Optimizer/CodeGen/CodeGen.h
+++ b/flang/include/flang/Optimizer/CodeGen/CodeGen.h
@@ -39,6 +39,9 @@ struct FIRToLLVMPassOptions {
   // that such programs would crash at runtime if the derived type descriptors
   // are required by the runtime, so this is only an option to help debugging.
   bool ignoreMissingTypeDescriptors = false;
+  // Similar to ignoreMissingTypeDescriptors, but generate external declaration
+  // for the missing type descriptor globals instead.
+  bool skipExternalRttiDefinition = false;
 
   // Generate TBAA information for FIR types and memory accessing operations.
   bool applyTBAA = false;
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 8ac847dd7dd0..b96c0360316a 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2057,8 +2057,8 @@ def fir_SliceOp : fir_Op<"slice", [NoMemoryEffect, AttrSizedOperandSegments]> {
 
   let builders = [
     OpBuilder<(ins "mlir::ValueRange":$triples,
-      CArg<"mlir::ValueRange", "std::nullopt">:$fields,
-      CArg<"mlir::ValueRange", "std::nullopt">:$substr)>
+      CArg<"mlir::ValueRange", "{}">:$fields,
+      CArg<"mlir::ValueRange", "{}">:$substr)>
   ];
 
   let hasVerifier = 1;
@@ -2253,8 +2253,8 @@ def fir_DoLoopOp : region_Op<"do_loop", [AttrSizedOperandSegments,
     OpBuilder<(ins "mlir::Value":$lowerBound, "mlir::Value":$upperBound,
       "mlir::Value":$step, CArg<"bool", "false">:$unordered,
       CArg<"bool", "false">:$finalCountValue,
-      CArg<"mlir::ValueRange", "std::nullopt">:$iterArgs,
-      CArg<"mlir::ValueRange", "std::nullopt">:$reduceOperands,
+      CArg<"mlir::ValueRange", "{}">:$iterArgs,
+      CArg<"mlir::ValueRange", "{}">:$reduceOperands,
       CArg<"llvm::ArrayRef<mlir::Attribute>", "{}">:$reduceAttrs,
       CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>
   ];
@@ -2441,7 +2441,7 @@ def fir_IterWhileOp : region_Op<"iterate_while",
     OpBuilder<(ins "mlir::Value":$lowerBound, "mlir::Value":$upperBound,
       "mlir::Value":$step, "mlir::Value":$iterate,
       CArg<"bool", "false">:$finalCountValue,
-      CArg<"mlir::ValueRange", "std::nullopt">:$iterArgs,
+      CArg<"mlir::ValueRange", "{}">:$iterArgs,
       CArg<"llvm::ArrayRef<mlir::NamedAttribute>", "{}">:$attributes)>
   ];
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 6fad77dffd9b..0ead54df3ca9 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -330,7 +330,8 @@ def fir_RecordType : FIR_Type<"Record", "type"> {
 
   let extraClassDeclaration = [{
     using TypePair = std::pair<std::string, mlir::Type>;
-    using TypeList = std::vector<TypePair>;
+    using TypeList = llvm::ArrayRef<TypePair>;
+    using TypeVector = llvm::SmallVector<TypePair>;
     TypeList getTypeList() const;
     TypeList getLenParamList() const;
 
diff --git a/flang/include/flang/Optimizer/Dialect/Support/KindMapping.h b/flang/include/flang/Optimizer/Dialect/Support/KindMapping.h
index 083526fcbac9..28e101a60d1f 100644
--- a/flang/include/flang/Optimizer/Dialect/Support/KindMapping.h
+++ b/flang/include/flang/Optimizer/Dialect/Support/KindMapping.h
@@ -66,7 +66,7 @@ class KindMapping {
   /// of 6 KindTy must be passed. The kinds must be the given in the following
   /// order: CHARACTER, COMPLEX, DOUBLE PRECISION, INTEGER, LOGICAL, and REAL.
   explicit KindMapping(mlir::MLIRContext *context, llvm::StringRef map,
-                       llvm::ArrayRef<KindTy> defs = std::nullopt);
+                       llvm::ArrayRef<KindTy> defs = {});
   explicit KindMapping(mlir::MLIRContext *context, llvm::StringRef map,
                        llvm::StringRef defs)
       : KindMapping{context, map, toDefaultKinds(defs)} {}
diff --git a/flang/include/flang/Optimizer/Passes/CommandLineOpts.h b/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
index 1cfaf285e75e..76ac9d0622d2 100644
--- a/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
+++ b/flang/include/flang/Optimizer/Passes/CommandLineOpts.h
@@ -32,6 +32,19 @@ extern llvm::cl::opt<std::size_t> arrayStackAllocationThreshold;
 /// generated by the frontend.
 extern llvm::cl::opt<bool> ignoreMissingTypeDescriptors;
 
+/// Shared option in tools to only generate rtti static object definitions for
+/// derived types defined in the current compilation unit. Derived type
+/// descriptor object for types defined in other objects will only be declared
+/// as external. This also changes the linkage of rtti objects defined in the
+/// current compilation unit from linkonce_odr to external so that unused rtti
+/// objects are retained and can be accessed from other compilation units. This
+/// is an experimental option to explore compilation speed improvements and is
+/// an ABI breaking change because of the linkage change.
+/// It will also require linking against module file objects of modules defining
+/// only types (even for trivial types without type bound procedures, which
+/// differs from most compilers).
+extern llvm::cl::opt<bool> skipExternalRttiDefinition;
+
 /// Default optimization level used to create Flang pass pipeline is O0.
 extern llvm::OptimizationLevel defaultOptLevel;
 
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index ec73af6ec72e..83c936b7dcad 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -35,32 +35,6 @@ inline std::int64_t toInt(mlir::arith::ConstantOp cop) {
       .getSExtValue();
 }
 
-// Reconstruct binding tables for dynamic dispatch.
-using BindingTable = llvm::DenseMap<llvm::StringRef, unsigned>;
-using BindingTables = llvm::DenseMap<llvm::StringRef, BindingTable>;
-
-inline void buildBindingTables(BindingTables &bindingTables,
-                               mlir::ModuleOp mod) {
-
-  // The binding tables are defined in FIR after lowering inside fir.type_info
-  // operations. Go through each binding tables and store the procedure name and
-  // binding index for later use by the fir.dispatch conversion pattern.
-  for (auto typeInfo : mod.getOps<fir::TypeInfoOp>()) {
-    unsigned bindingIdx = 0;
-    BindingTable bindings;
-    if (typeInfo.getDispatchTable().empty()) {
-      bindingTables[typeInfo.getSymName()] = bindings;
-      continue;
-    }
-    for (auto dtEntry :
-         typeInfo.getDispatchTable().front().getOps<fir::DTEntryOp>()) {
-      bindings[dtEntry.getMethod()] = bindingIdx;
-      ++bindingIdx;
-    }
-    bindingTables[typeInfo.getSymName()] = bindings;
-  }
-}
-
 // Translate front-end KINDs for use in the IR and code gen.
 inline std::vector<fir::KindTy>
 fromDefaultKinds(const Fortran::common::IntrinsicTypeDefaultKinds &defKinds) {
diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index e19b16c23b82..7371a46ad36c 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -292,7 +292,8 @@ class Message : public common::ReferenceCounted<Message> {
   std::optional<ProvenanceRange> GetProvenanceRange(
       const AllCookedSources &) const;
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
-      bool echoSourceLine = true) const;
+      bool echoSourceLine = true,
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
 
   // If this Message or any of its attachments locates itself via a CharBlock,
   // replace its location with the corresponding ProvenanceRange.
@@ -352,7 +353,8 @@ class Messages {
   void Copy(const Messages &);
   void ResolveProvenances(const AllCookedSources &);
   void Emit(llvm::raw_ostream &, const AllCookedSources &,
-      bool echoSourceLines = true) const;
+      bool echoSourceLines = true,
+      const common::LanguageFeatureControl *hintFlags = nullptr) const;
   void AttachTo(Message &, std::optional<Severity> = std::nullopt);
   bool AnyFatalError() const;
 
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 61f97b855b0e..7e752eeb4dfe 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -267,6 +267,7 @@ struct AccEndCombinedDirective;
 struct OpenACCDeclarativeConstruct;
 struct OpenACCRoutineConstruct;
 struct OpenMPConstruct;
+struct OpenMPLoopConstruct;
 struct OpenMPDeclarativeConstruct;
 struct OmpEndLoopDirective;
 struct OmpMemoryOrderClause;
@@ -5021,11 +5022,13 @@ struct OpenMPBlockConstruct {
 };
 
 // OpenMP directives enclosing do loop
+using NestedConstruct =
+    std::variant<DoConstruct, common::Indirection<OpenMPLoopConstruct>>;
 struct OpenMPLoopConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPLoopConstruct);
   OpenMPLoopConstruct(OmpBeginLoopDirective &&a)
       : t({std::move(a), std::nullopt, std::nullopt}) {}
-  std::tuple<OmpBeginLoopDirective, std::optional<DoConstruct>,
+  std::tuple<OmpBeginLoopDirective, std::optional<NestedConstruct>,
       std::optional<OmpEndLoopDirective>>
       t;
 };
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
index e90d3ae8baf1..6c5a061d1c1a 100644
--- a/flang/include/flang/Semantics/runtime-type-info.h
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -38,6 +38,10 @@ RuntimeDerivedTypeTables BuildRuntimeDerivedTypeTables(SemanticsContext &);
 /// to describe other derived types at runtime in flang descriptor.
 constexpr char typeInfoBuiltinModule[]{"__fortran_type_info"};
 
+/// Name of the builtin derived type in __fortran_type_inf that is used for
+/// derived type descriptors.
+constexpr char typeDescriptorTypeName[]{"derivedtype"};
+
 /// Name of the bindings descriptor component in the DerivedType type of the
 /// __Fortran_type_info module
 constexpr char bindingDescCompName[]{"binding"};
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 04fa26d72a10..2404e87a16ad 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -157,14 +157,13 @@ class LanguageFeatureControl {
 private:
   // Map from Cli syntax of language features and usage warnings to their enum
   // values.
-  std::unordered_map<std::string, std::variant<LanguageFeature, UsageWarning>>
-      cliOptions_;
+  std::unordered_map<std::string, LanguageFeatureOrWarning> cliOptions_;
   // These two arrays map the enum values to their cannonical Cli spellings.
   // Since each of the CanonicalSpelling is a string in the domain of the map
   // above we just use a view of the string instead of another copy.
-  std::array<std::string_view, LanguageFeature_enumSize>
+  std::array<std::string, LanguageFeature_enumSize>
       languageFeatureCliCanonicalSpelling_;
-  std::array<std::string_view, UsageWarning_enumSize>
+  std::array<std::string, UsageWarning_enumSize>
       usageWarningCliCanonicalSpelling_;
   LanguageFeatures disable_;
   LanguageFeatures warnLanguage_;
diff --git a/flang/include/flang/Support/OpenMP-features.h b/flang/include/flang/Support/OpenMP-features.h
index 349cd19c1224..5e722930ae1b 100644
--- a/flang/include/flang/Support/OpenMP-features.h
+++ b/flang/include/flang/Support/OpenMP-features.h
@@ -15,16 +15,8 @@ namespace Fortran::common {
 template <typename FortranPredefinitions>
 void setOpenMPMacro(int version, FortranPredefinitions &predefinitions) {
   switch (version) {
-  case 20:
-    predefinitions.emplace_back("_OPENMP", "200011");
-    break;
-  case 25:
-    predefinitions.emplace_back("_OPENMP", "200505");
-    break;
-  case 30:
-    predefinitions.emplace_back("_OPENMP", "200805");
-    break;
   case 31:
+  default:
     predefinitions.emplace_back("_OPENMP", "201107");
     break;
   case 40:
@@ -45,10 +37,6 @@ void setOpenMPMacro(int version, FortranPredefinitions &predefinitions) {
   case 60:
     predefinitions.emplace_back("_OPENMP", "202411");
     break;
-  case 11:
-  default:
-    predefinitions.emplace_back("_OPENMP", "199911");
-    break;
   }
 }
 } // namespace Fortran::common
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index b0f39e63d094..52e954d1cffb 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -1102,6 +1102,8 @@ template <typename T> Expr<T> Folder<T>::TRANSFER(FunctionRef<T> &&funcRef) {
   }
 }
 
+// TODO: Once the backend supports character extremums we could support
+// min/max with non-optional arguments to trees of extremum operations.
 template <typename T>
 Expr<T> FoldMINorMAX(
     FoldingContext &context, FunctionRef<T> &&funcRef, Ordering order) {
@@ -1109,42 +1111,76 @@ Expr<T> FoldMINorMAX(
       T::category == TypeCategory::Unsigned ||
       T::category == TypeCategory::Real ||
       T::category == TypeCategory::Character);
+
+  // Lots of constraints:
+  // - We want Extremum<T> generated by semantics to compare equal to
+  //   Extremum<T> written out to module files as max or min calls.
+  // - Users can also write min/max calls that must also compare equal
+  //   to min/max calls that wind up being written to module files.
+  // - Extremeum<T> is binary and can't currently handle processing
+  //   optional arguments that may show up in 3rd + argument.
+  // - The code below only accepts more than 2 arguments if all the
+  //   arguments are constant (and hence known to be present).
+  // - ConvertExprToHLFIR can't currently handle Extremum<Character>
+  // - Semantics doesn't currently generate Extremum<Character>
+  // The original code did the folding of arguments and the overall extremum
+  // operation in a single pass. This was shorter code-wise, but took me
+  // a while to tease out all the logic and was doing redundant work.
+  // So I split it into two passes:
+  // 1) fold the arguments and check if they are constant,
+  // 2) Decide if we:
+  //    - can constant-fold the min/max operation, or
+  //    - need to generate an extremum anyway,
+  //    and do it if so.
+  //    Otherwise, return the original call.
   auto &args{funcRef.arguments()};
-  bool ok{true};
-  std::optional<Expr<T>> result;
-  Folder<T> folder{context};
-  for (std::optional<ActualArgument> &arg : args) {
-    // Call Folding on all arguments to make operand promotion explicit.
-    if (!folder.Folding(arg)) {
-      // TODO: Lowering can't handle having every FunctionRef for max and min
-      // being converted into Extremum<T>.  That needs fixing.  Until that
-      // is corrected, however, it is important that max and min references
-      // in module files be converted into Extremum<T> even when not constant;
-      // the Extremum<SubscriptInteger> operations created to normalize the
-      // values of array bounds are formatted as max operations in the
-      // declarations in modules, and need to be read back in as such in
-      // order for expression comparison to not produce false inequalities
-      // when checking function results for procedure interface compatibility.
-      if (!context.moduleFileName()) {
-        ok = false;
+  std::size_t nargs{args.size()};
+  bool allArgsConstant{true};
+  bool extremumAnyway{nargs == 2 && T::category != TypeCategory::Character};
+  // 1a)Fold the first two arguments.
+  {
+    Folder<T> folder{context, /*forOptionalArgument=*/false};
+    if (!folder.Folding(args[0])) {
+      allArgsConstant = false;
+    }
+    if (!folder.Folding(args[1])) {
+      allArgsConstant = false;
+    }
+  }
+  // 1b) Fold any optional arguments.
+  if (nargs > 2) {
+    Folder<T> folder{context, /*forOptionalArgument=*/true};
+    for (std::size_t i{2}; i < nargs; ++i) {
+      if (args[i]) {
+        if (!folder.Folding(args[i])) {
+          allArgsConstant = false;
+        }
       }
     }
-    Expr<SomeType> *argExpr{arg ? arg->UnwrapExpr() : nullptr};
-    if (argExpr) {
-      *argExpr = Fold(context, std::move(*argExpr));
-    }
-    if (Expr<T> * tExpr{UnwrapExpr<Expr<T>>(argExpr)}) {
-      if (result) {
-        result = FoldOperation(
-            context, Extremum<T>{order, std::move(*result), Expr<T>{*tExpr}});
-      } else {
-        result = Expr<T>{*tExpr};
+  }
+  // 2) If we can fold the result or the call to min/max may compare equal to
+  // an extremum generated by semantics go ahead and convert to an extremum,
+  // and try to fold the result.
+  if (allArgsConstant || extremumAnyway) {
+    // Folding updates the argument expressions in place, no need to call
+    // Fold() on each argument again.
+    if (const auto *resultp{UnwrapExpr<Expr<T>>(args[0])}) {
+      Expr<T> result{*resultp};
+      for (std::size_t i{1}; i < nargs; ++i) {
+        if (const auto *tExpr{UnwrapExpr<Expr<T>>(args[i])}) {
+          result = FoldOperation(
+              context, Extremum<T>{order, std::move(result), *tExpr});
+        } else {
+          // This should never happen, but here is a value to return.
+          return Expr<T>{std::move(funcRef)};
+        }
       }
-    } else {
-      ok = false;
+      return result;
     }
   }
-  return ok && result ? std::move(*result) : Expr<T>{std::move(funcRef)};
+  // If we decided to not generate an extremum just return the original call,
+  // with the arguments folded.
+  return Expr<T>{std::move(funcRef)};
 }
 
 // For AMAX0, AMIN0, AMAX1, AMIN1, DMAX1, DMIN1, MAX0, MIN0, MAX1, and MIN1
@@ -2156,7 +2192,14 @@ Expr<T> FoldOperation(FoldingContext &context, Power<T> &&x) {
       }
       return Expr<T>{Constant<T>{power.power}};
     } else {
-      if (auto callable{GetHostRuntimeWrapper<T, T, T>("pow")}) {
+      if (folded->first.IsZero()) {
+        if (folded->second.IsZero()) {
+          context.messages().Say(common::UsageWarning::FoldingException,
+              "REAL/COMPLEX 0**0 is not defined"_warn_en_US);
+        } else {
+          return Expr<T>(Constant<T>{folded->first}); // 0. ** nonzero -> 0.
+        }
+      } else if (auto callable{GetHostRuntimeWrapper<T, T, T>("pow")}) {
         return Expr<T>{
             Constant<T>{(*callable)(context, folded->first, folded->second)}};
       } else if (context.languageFeatures().ShouldWarn(
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 68838564f87b..fcacdb93d662 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -2334,6 +2334,11 @@ bool IsBuiltinCPtr(const Symbol &symbol) {
   return false;
 }
 
+bool IsFromBuiltinModule(const Symbol &symbol) {
+  const Scope &scope{symbol.GetUltimate().owner()};
+  return IsSameModule(&scope, scope.context().GetBuiltinsScope());
+}
+
 bool IsIsoCType(const DerivedTypeSpec *derived) {
   return IsBuiltinDerivedType(derived, "c_ptr") ||
       IsBuiltinDerivedType(derived, "c_funptr");
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index 5b5f3c2cd0cf..99dc8b1e5c67 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -299,13 +299,18 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &,
 
 // F2023 7.5.3.2
 static bool AreSameComponent(const semantics::Symbol &x,
-    const semantics::Symbol &y, bool ignoreSequence,
+    const semantics::Symbol &y, bool ignoreSequence, bool sameModuleName,
     SetOfDerivedTypePairs &inProgress) {
   if (x.attrs() != y.attrs()) {
     return false;
   }
-  if (x.attrs().test(semantics::Attr::PRIVATE)) {
-    return false;
+  if (x.attrs().test(semantics::Attr::PRIVATE) ||
+      y.attrs().test(semantics::Attr::PRIVATE)) {
+    if (!sameModuleName ||
+        x.attrs().test(semantics::Attr::PRIVATE) !=
+            y.attrs().test(semantics::Attr::PRIVATE)) {
+      return false;
+    }
   }
   if (x.size() && y.size()) {
     if (x.offset() != y.offset() || x.size() != y.size()) {
@@ -482,9 +487,20 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
           ySymbol.attrs().test(semantics::Attr::BIND_C)) {
     return false;
   }
-  if (!ignoreSequence && !(xDetails.sequence() && yDetails.sequence()) &&
-      !(xSymbol.attrs().test(semantics::Attr::BIND_C) &&
-          ySymbol.attrs().test(semantics::Attr::BIND_C))) {
+  bool sameModuleName{false};
+  const semantics::Scope &xOwner{xSymbol.owner()};
+  const semantics::Scope &yOwner{ySymbol.owner()};
+  if (xOwner.IsModule() && yOwner.IsModule()) {
+    if (auto xModuleName{xOwner.GetName()}) {
+      if (auto yModuleName{yOwner.GetName()}) {
+        if (*xModuleName == *yModuleName) {
+          sameModuleName = true;
+        }
+      }
+    }
+  }
+  if (!sameModuleName && !ignoreSequence && !xDetails.sequence() &&
+      !xSymbol.attrs().test(semantics::Attr::BIND_C)) {
     // PGI does not enforce this requirement; all other Fortran
     // compilers do with a hard error when violations are caught.
     return false;
@@ -502,9 +518,10 @@ static bool AreSameDerivedType(const semantics::DerivedTypeSpec &x,
     const auto xLookup{xSymbol.scope()->find(*xComponentName)};
     const auto yLookup{ySymbol.scope()->find(*yComponentName)};
     if (xLookup == xSymbol.scope()->end() ||
-        yLookup == ySymbol.scope()->end() ||
-        !AreSameComponent(
-            *xLookup->second, *yLookup->second, ignoreSequence, inProgress)) {
+        yLookup == ySymbol.scope()->end()) {
+      return false;
+    } else if (!AreSameComponent(*xLookup->second, *yLookup->second,
+                   ignoreSequence, sameModuleName, inProgress)) {
       return false;
     }
   }
@@ -576,17 +593,15 @@ static bool AreCompatibleTypes(const DynamicType &x, const DynamicType &y,
     const auto yLen{y.knownLength()};
     return x.kind() == y.kind() &&
         (ignoreLengths || !xLen || !yLen || *xLen == *yLen);
-  } else if (x.category() != TypeCategory::Derived) {
-    if (x.IsTypelessIntrinsicArgument()) {
-      return y.IsTypelessIntrinsicArgument();
-    } else {
-      return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
-    }
-  } else {
+  } else if (x.category() == TypeCategory::Derived) {
     const auto *xdt{GetDerivedTypeSpec(x)};
     const auto *ydt{GetDerivedTypeSpec(y)};
     return AreCompatibleDerivedTypes(
         xdt, ydt, x.IsPolymorphic(), ignoreTypeParameterValues, false);
+  } else if (x.IsTypelessIntrinsicArgument()) {
+    return y.IsTypelessIntrinsicArgument();
+  } else {
+    return !y.IsTypelessIntrinsicArgument() && x.kind() == y.kind();
   }
 }
 
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 302bd20be1b2..3a331d52375a 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -14,6 +14,7 @@
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Frontend/TargetOptions.h"
+#include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Support/Fortran-features.h"
 #include "flang/Support/OpenMP-features.h"
@@ -26,6 +27,7 @@
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/OptionUtils.h"
 #include "clang/Driver/Options.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Frontend/Debug/Options.h"
@@ -44,6 +46,7 @@
 #include <cstdlib>
 #include <memory>
 #include <optional>
+#include <sstream>
 
 using namespace Fortran::frontend;
 
@@ -1152,15 +1155,49 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
   unsigned numErrorsBefore = diags.getNumErrors();
   llvm::Triple t(res.getTargetOpts().triple);
 
+  constexpr unsigned newestFullySupported = 52;
   // By default OpenMP is set to 5.2 version
-  res.getLangOpts().OpenMPVersion = 52;
+  res.getLangOpts().OpenMPVersion = newestFullySupported;
   res.getFrontendOpts().features.Enable(
       Fortran::common::LanguageFeature::OpenMP);
-  if (int Version = getLastArgIntValue(
-          args, clang::driver::options::OPT_fopenmp_version_EQ,
-          res.getLangOpts().OpenMPVersion, diags)) {
-    res.getLangOpts().OpenMPVersion = Version;
+  if (auto *arg =
+          args.getLastArg(clang::driver::options::OPT_fopenmp_version_EQ)) {
+    llvm::ArrayRef<unsigned> ompVersions = llvm::omp::getOpenMPVersions();
+    unsigned oldVersions[] = {11, 20, 25, 30};
+    unsigned version = 0;
+
+    auto reportBadVersion = [&](llvm::StringRef value) {
+      const unsigned diagID =
+          diags.getCustomDiagID(clang::DiagnosticsEngine::Error,
+                                "'%0' is not a valid OpenMP version in '%1', "
+                                "valid versions are %2");
+      std::string buffer;
+      llvm::raw_string_ostream versions(buffer);
+      llvm::interleaveComma(ompVersions, versions);
+
+      diags.Report(diagID) << value << arg->getAsString(args) << versions.str();
+    };
+
+    llvm::StringRef value = arg->getValue();
+    if (!value.getAsInteger(/*radix=*/10, version)) {
+      if (llvm::is_contained(ompVersions, version)) {
+        res.getLangOpts().OpenMPVersion = version;
+
+      } else if (llvm::is_contained(oldVersions, version)) {
+        const unsigned diagID =
+            diags.getCustomDiagID(clang::DiagnosticsEngine::Warning,
+                                  "OpenMP version %0 is no longer supported, "
+                                  "assuming version %1");
+        std::string assumed = std::to_string(res.getLangOpts().OpenMPVersion);
+        diags.Report(diagID) << value << assumed;
+      } else {
+        reportBadVersion(value);
+      }
+    } else {
+      reportBadVersion(value);
+    }
   }
+
   if (args.hasArg(clang::driver::options::OPT_fopenmp_force_usm)) {
     res.getLangOpts().OpenMPForceUSM = 1;
   }
@@ -1225,7 +1262,7 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 
   // Get the OpenMP target triples if any.
   if (auto *arg =
-          args.getLastArg(clang::driver::options::OPT_fopenmp_targets_EQ)) {
+          args.getLastArg(clang::driver::options::OPT_offload_targets_EQ)) {
     enum ArchPtrSize { Arch16Bit, Arch32Bit, Arch64Bit };
     auto getArchPtrSize = [](const llvm::Triple &triple) {
       if (triple.isArch16Bit())
@@ -1771,6 +1808,7 @@ void CompilerInvocation::setLoweringOptions() {
   // Lower TRANSPOSE as a runtime call under -O0.
   loweringOpts.setOptimizeTranspose(codegenOpts.OptimizationLevel > 0);
   loweringOpts.setUnderscoring(codegenOpts.Underscoring);
+  loweringOpts.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
 
   const Fortran::common::LangOptions &langOptions = getLangOpts();
   loweringOpts.setIntegerWrapAround(langOptions.getSignedOverflowBehavior() ==
diff --git a/flang/lib/Frontend/FrontendAction.cpp b/flang/lib/Frontend/FrontendAction.cpp
index d178fd6a9395..5d788fe5619a 100644
--- a/flang/lib/Frontend/FrontendAction.cpp
+++ b/flang/lib/Frontend/FrontendAction.cpp
@@ -171,7 +171,10 @@ bool FrontendAction::runParse(bool emitMessages) {
   if (emitMessages) {
     // Report any non-fatal diagnostics from getParsing now rather than
     // combining them with messages from semantics.
-    ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources());
+    const common::LanguageFeatureControl &features{
+        ci.getInvocation().getFortranOpts().features};
+    ci.getParsing().messages().Emit(llvm::errs(), ci.getAllCookedSources(),
+                                    /*echoSourceLine=*/true, &features);
   }
   return true;
 }
@@ -223,6 +226,8 @@ bool FrontendAction::generateRtTypeTables() {
 
 template <unsigned N>
 bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
+  const common::LanguageFeatureControl &features{
+      instance->getInvocation().getFortranOpts().features};
   if (!instance->getParsing().messages().empty() &&
       (instance->getInvocation().getWarnAsErr() ||
        instance->getParsing().messages().AnyFatalError())) {
@@ -230,7 +235,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(llvm::errs(),
-                                           instance->getAllCookedSources());
+                                           instance->getAllCookedSources(),
+                                           /*echoSourceLines=*/true, &features);
     return true;
   }
   if (instance->getParsing().parseTree().has_value() &&
@@ -240,7 +246,8 @@ bool FrontendAction::reportFatalErrors(const char (&message)[N]) {
         clang::DiagnosticsEngine::Error, message);
     instance->getDiagnostics().Report(diagID) << getCurrentFileOrBufferName();
     instance->getParsing().messages().Emit(llvm::errs(),
-                                           instance->getAllCookedSources());
+                                           instance->getAllCookedSources(),
+                                           /*echoSourceLine=*/true, &features);
     instance->getParsing().EmitMessage(
         llvm::errs(), instance->getParsing().finalRestingPlace(),
         "parser FAIL (final position)", "error: ", llvm::raw_ostream::RED);
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index d7371d0d9948..42cd6d9f4afb 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -74,10 +74,6 @@
 #include <memory>
 #include <system_error>
 
-namespace llvm {
-extern cl::opt<bool> PrintPipelinePasses;
-} // namespace llvm
-
 using namespace Fortran::frontend;
 
 constexpr llvm::StringLiteral timingIdParse = "Parse";
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index ecc1aca56635..994dfd010ec9 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -1096,19 +1096,19 @@ createMutableProperties(Fortran::lower::AbstractConverter &converter,
     baseAddrTy = boxType.getEleTy();
   // Allocate and set a variable to hold the address.
   // It will be set to null in setUnallocatedStatus.
-  mutableProperties.addr = builder.allocateLocal(
-      loc, baseAddrTy, name + ".addr", "",
-      /*shape=*/std::nullopt, /*typeparams=*/std::nullopt);
+  mutableProperties.addr =
+      builder.allocateLocal(loc, baseAddrTy, name + ".addr", "",
+                            /*shape=*/{}, /*typeparams=*/{});
   // Allocate variables to hold lower bounds and extents.
   int rank = sym.Rank();
   mlir::Type idxTy = builder.getIndexType();
   for (decltype(rank) i = 0; i < rank; ++i) {
-    mlir::Value lboundVar = builder.allocateLocal(
-        loc, idxTy, name + ".lb" + std::to_string(i), "",
-        /*shape=*/std::nullopt, /*typeparams=*/std::nullopt);
-    mlir::Value extentVar = builder.allocateLocal(
-        loc, idxTy, name + ".ext" + std::to_string(i), "",
-        /*shape=*/std::nullopt, /*typeparams=*/std::nullopt);
+    mlir::Value lboundVar =
+        builder.allocateLocal(loc, idxTy, name + ".lb" + std::to_string(i), "",
+                              /*shape=*/{}, /*typeparams=*/{});
+    mlir::Value extentVar =
+        builder.allocateLocal(loc, idxTy, name + ".ext" + std::to_string(i), "",
+                              /*shape=*/{}, /*typeparams=*/{});
     mutableProperties.lbounds.emplace_back(lboundVar);
     mutableProperties.extents.emplace_back(extentVar);
   }
@@ -1123,10 +1123,9 @@ createMutableProperties(Fortran::lower::AbstractConverter &converter,
     if (record.getNumLenParams() != 0)
       TODO(loc, "deferred length type parameters.");
   if (fir::isa_char(eleTy) && nonDeferredParams.empty()) {
-    mlir::Value lenVar =
-        builder.allocateLocal(loc, builder.getCharacterLengthType(),
-                              name + ".len", "", /*shape=*/std::nullopt,
-                              /*typeparams=*/std::nullopt);
+    mlir::Value lenVar = builder.allocateLocal(
+        loc, builder.getCharacterLengthType(), name + ".len", "", /*shape=*/{},
+        /*typeparams=*/{});
     mutableProperties.deferredParams.emplace_back(lenVar);
   }
   return mutableProperties;
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 2f2845ee683a..1934d660721c 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -70,6 +70,7 @@
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Parser/Parser.h"
+#include "mlir/Support/StateStack.h"
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSet.h"
@@ -262,6 +263,7 @@ class TypeInfoConverter {
   }
 
   void createTypeInfo(Fortran::lower::AbstractConverter &converter) {
+    createTypeInfoForTypeDescriptorBuiltinType(converter);
     while (!registeredTypeInfoA.empty()) {
       currentTypeInfoStack = &registeredTypeInfoB;
       for (const TypeInfo &info : registeredTypeInfoA)
@@ -277,10 +279,22 @@ class TypeInfoConverter {
 private:
   void createTypeInfoOpAndGlobal(Fortran::lower::AbstractConverter &converter,
                                  const TypeInfo &info) {
-    Fortran::lower::createRuntimeTypeInfoGlobal(converter, info.symbol.get());
+    if (!converter.getLoweringOptions().getSkipExternalRttiDefinition())
+      Fortran::lower::createRuntimeTypeInfoGlobal(converter, info.symbol.get());
     createTypeInfoOp(converter, info);
   }
 
+  void createTypeInfoForTypeDescriptorBuiltinType(
+      Fortran::lower::AbstractConverter &converter) {
+    if (registeredTypeInfoA.empty())
+      return;
+    auto builtinTypeInfoType = llvm::cast<fir::RecordType>(
+        converter.genType(registeredTypeInfoA[0].symbol.get()));
+    converter.getFirOpBuilder().createTypeInfoOp(
+        registeredTypeInfoA[0].loc, builtinTypeInfoType,
+        /*parentType=*/fir::RecordType{});
+  }
+
   void createTypeInfoOp(Fortran::lower::AbstractConverter &converter,
                         const TypeInfo &info) {
     fir::RecordType parentType{};
@@ -705,8 +719,7 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   }
   mlir::Type genType(Fortran::common::TypeCategory tc) override final {
     return Fortran::lower::getFIRType(
-        &getMLIRContext(), tc, bridge.getDefaultKinds().GetDefaultKind(tc),
-        std::nullopt);
+        &getMLIRContext(), tc, bridge.getDefaultKinds().GetDefaultKind(tc), {});
   }
 
   Fortran::lower::TypeConstructionStack &
@@ -1242,6 +1255,8 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
   mlir::SymbolTable *getMLIRSymbolTable() override { return &mlirSymbolTable; }
 
+  mlir::StateStack &getStateStack() override { return stateStack; }
+
   /// Add the symbol to the local map and return `true`. If the symbol is
   /// already in the map and \p forced is `false`, the map is not updated.
   /// Instead the value `false` is returned.
@@ -6563,6 +6578,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   /// attribute since mlirSymbolTable must pro-actively be maintained when
   /// new Symbol operations are created.
   mlir::SymbolTable mlirSymbolTable;
+
+  /// Used to store context while recursing into regions during lowering.
+  mlir::StateStack stateStack;
 };
 
 } // namespace
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 676c26dbcdbe..72431a9cfacc 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1338,15 +1338,13 @@ class Fortran::lower::CallInterfaceImpl {
           getConverter().getFoldingContext(), toEvExpr(*expr)));
     return std::nullopt;
   }
-  void addFirOperand(
-      mlir::Type type, int entityPosition, Property p,
-      llvm::ArrayRef<mlir::NamedAttribute> attributes = std::nullopt) {
+  void addFirOperand(mlir::Type type, int entityPosition, Property p,
+                     llvm::ArrayRef<mlir::NamedAttribute> attributes = {}) {
     interface.inputs.emplace_back(
         FirPlaceHolder{type, entityPosition, p, attributes});
   }
-  void
-  addFirResult(mlir::Type type, int entityPosition, Property p,
-               llvm::ArrayRef<mlir::NamedAttribute> attributes = std::nullopt) {
+  void addFirResult(mlir::Type type, int entityPosition, Property p,
+                    llvm::ArrayRef<mlir::NamedAttribute> attributes = {}) {
     interface.outputs.emplace_back(
         FirPlaceHolder{type, entityPosition, p, attributes});
   }
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 864499e6c343..6ed15df0de75 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -20,6 +20,7 @@
 #include "flang/Lower/StatementContext.h"
 #include "flang/Lower/SymbolMap.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
+#include "flang/Optimizer/Builder/CUFCommon.h"
 #include "flang/Optimizer/Builder/Character.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
@@ -545,7 +546,8 @@ Fortran::lower::genCallOpAndResult(
       caller.getProcedureAttrs(builder.getContext());
 
   if (converter.getLoweringOptions().getCUDARuntimeCheck()) {
-    if (caller.getCallDescription().chevrons().empty()) {
+    if (caller.getCallDescription().chevrons().empty() &&
+        !cuf::isCUDADeviceContext(builder.getRegion())) {
       for (auto [oper, arg] :
            llvm::zip(operands, caller.getPassedArguments())) {
         if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(oper.getType())) {
diff --git a/flang/lib/Lower/ConvertConstant.cpp b/flang/lib/Lower/ConvertConstant.cpp
index 38f83f1deceb..1051d50ce8a9 100644
--- a/flang/lib/Lower/ConvertConstant.cpp
+++ b/flang/lib/Lower/ConvertConstant.cpp
@@ -150,8 +150,8 @@ class DenseGlobalBuilder {
     auto attrTc = TC == Fortran::common::TypeCategory::Logical
                       ? Fortran::common::TypeCategory::Integer
                       : TC;
-    attributeElementType = Fortran::lower::getFIRType(
-        builder.getContext(), attrTc, KIND, std::nullopt);
+    attributeElementType =
+        Fortran::lower::getFIRType(builder.getContext(), attrTc, KIND, {});
     for (auto element : constant.values())
       attributes.push_back(
           convertToAttribute<TC, KIND>(builder, element, attributeElementType));
@@ -230,8 +230,7 @@ static mlir::Value genScalarLit(
                 TC == Fortran::common::TypeCategory::Unsigned) {
     // MLIR requires constants to be signless
     mlir::Type ty = Fortran::lower::getFIRType(
-        builder.getContext(), Fortran::common::TypeCategory::Integer, KIND,
-        std::nullopt);
+        builder.getContext(), Fortran::common::TypeCategory::Integer, KIND, {});
     if (KIND == 16) {
       auto bigInt = llvm::APInt(ty.getIntOrFloatBitWidth(),
                                 TC == Fortran::common::TypeCategory::Unsigned
diff --git a/flang/lib/Lower/ConvertExprToHLFIR.cpp b/flang/lib/Lower/ConvertExprToHLFIR.cpp
index 808928bc97ad..df8dfbc72c03 100644
--- a/flang/lib/Lower/ConvertExprToHLFIR.cpp
+++ b/flang/lib/Lower/ConvertExprToHLFIR.cpp
@@ -1065,7 +1065,7 @@ struct BinaryOp<Fortran::evaluate::Divide<
                                          hlfir::Entity lhs, hlfir::Entity rhs) {
     mlir::Type ty = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Complex, KIND,
-        /*params=*/std::nullopt);
+        /*params=*/{});
     return hlfir::EntityWithAttributes{
         fir::genDivC(builder, loc, ty, lhs, rhs)};
   }
@@ -1078,7 +1078,7 @@ struct BinaryOp<Fortran::evaluate::Power<Fortran::evaluate::Type<TC, KIND>>> {
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs, hlfir::Entity rhs) {
     mlir::Type ty = Fortran::lower::getFIRType(builder.getContext(), TC, KIND,
-                                               /*params=*/std::nullopt);
+                                               /*params=*/{});
     return hlfir::EntityWithAttributes{fir::genPow(builder, loc, ty, lhs, rhs)};
   }
 };
@@ -1092,7 +1092,7 @@ struct BinaryOp<
                                          fir::FirOpBuilder &builder, const Op &,
                                          hlfir::Entity lhs, hlfir::Entity rhs) {
     mlir::Type ty = Fortran::lower::getFIRType(builder.getContext(), TC, KIND,
-                                               /*params=*/std::nullopt);
+                                               /*params=*/{});
     return hlfir::EntityWithAttributes{fir::genPow(builder, loc, ty, lhs, rhs)};
   }
 };
@@ -1416,7 +1416,7 @@ struct UnaryOp<Fortran::evaluate::Negate<
     // Like LLVM, integer negation is the binary op "0 - value"
     mlir::Type type = Fortran::lower::getFIRType(
         builder.getContext(), Fortran::common::TypeCategory::Integer, KIND,
-        /*params=*/std::nullopt);
+        /*params=*/{});
     mlir::Value zero = builder.createIntegerConstant(loc, type, 0);
     return hlfir::EntityWithAttributes{
         builder.create<mlir::arith::SubIOp>(loc, zero, lhs)};
@@ -1517,7 +1517,7 @@ struct UnaryOp<
       return hlfir::convertCharacterKind(loc, builder, lhs, KIND);
     }
     mlir::Type type = Fortran::lower::getFIRType(builder.getContext(), TC1,
-                                                 KIND, /*params=*/std::nullopt);
+                                                 KIND, /*params=*/{});
     mlir::Value res = builder.convertWithSemantics(loc, type, lhs);
     return hlfir::EntityWithAttributes{res};
   }
@@ -1661,7 +1661,7 @@ class HlfirBuilder {
     } else {
       elementType =
           Fortran::lower::getFIRType(builder.getContext(), R::category, R::kind,
-                                     /*params=*/std::nullopt);
+                                     /*params=*/{});
     }
     mlir::Value shape = hlfir::genShape(loc, builder, left);
     auto genKernel = [&op, &left, &unaryOp](
@@ -1699,7 +1699,7 @@ class HlfirBuilder {
     // Elemental expression.
     mlir::Type elementType =
         Fortran::lower::getFIRType(builder.getContext(), R::category, R::kind,
-                                   /*params=*/std::nullopt);
+                                   /*params=*/{});
     // TODO: "merge" shape, get cst shape from front-end if possible.
     mlir::Value shape;
     if (left.isArray()) {
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index a28596bfd009..7ab3c43016bd 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -647,13 +647,19 @@ fir::GlobalOp Fortran::lower::defineGlobal(
 
 /// Return linkage attribute for \p var.
 static mlir::StringAttr
-getLinkageAttribute(fir::FirOpBuilder &builder,
+getLinkageAttribute(Fortran::lower::AbstractConverter &converter,
                     const Fortran::lower::pft::Variable &var) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   // Runtime type info for a same derived type is identical in each compilation
   // unit. It desired to avoid having to link against module that only define a
   // type. Therefore the runtime type info is generated everywhere it is needed
-  // with `linkonce_odr` LLVM linkage.
-  if (var.isRuntimeTypeInfoData())
+  // with `linkonce_odr` LLVM linkage (unless the skipExternalRttiDefinition
+  // option is set, in which case one will need to link against objects of
+  // modules defining types). Builtin objects rtti is always generated because
+  // the builtin module is currently not compiled or part of the runtime.
+  if (var.isRuntimeTypeInfoData() &&
+      (!converter.getLoweringOptions().getSkipExternalRttiDefinition() ||
+       Fortran::semantics::IsFromBuiltinModule(var.getSymbol())))
     return builder.createLinkOnceODRLinkage();
   if (var.isModuleOrSubmoduleVariable())
     return {}; // external linkage
@@ -673,7 +679,7 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   std::string globalName = converter.mangleName(sym);
   mlir::Location loc = genLocation(converter, sym);
-  mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   fir::GlobalOp global;
   if (var.isModuleOrSubmoduleVariable()) {
     // A non-intrinsic module global is defined when lowering the module.
@@ -687,8 +693,13 @@ static void instantiateGlobal(Fortran::lower::AbstractConverter &converter,
   }
   auto addrOf = builder.create<fir::AddrOfOp>(loc, global.resultType(),
                                               global.getSymbol());
+  // The type of the global cannot be trusted to be the same as the one
+  // of the variable as some existing programs map common blocks to
+  // BIND(C) module variables (e.g. mpi_argv_null in MPI and MPI_F08).
+  mlir::Type varAddrType = fir::ReferenceType::get(converter.genType(sym));
+  mlir::Value cast = builder.createConvert(loc, varAddrType, addrOf);
   Fortran::lower::StatementContext stmtCtx;
-  mapSymbolAttributes(converter, var, symMap, stmtCtx, addrOf);
+  mapSymbolAttributes(converter, var, symMap, stmtCtx, cast);
 }
 
 //===----------------------------------------------------------------===//
@@ -1260,7 +1271,7 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
   if (var.isGlobal()) {
     fir::GlobalOp global;
     auto &aggregate = var.getAggregateStore();
-    mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+    mlir::StringAttr linkage = getLinkageAttribute(converter, var);
     if (var.isModuleOrSubmoduleVariable()) {
       // A module global was or will be defined when lowering the module. Emit
       // only a declaration if the global does not exist at that point.
@@ -1284,9 +1295,8 @@ instantiateAggregateStore(Fortran::lower::AbstractConverter &converter,
   auto size = std::get<1>(var.getInterval());
   fir::SequenceType::Shape shape(1, size);
   auto seqTy = fir::SequenceType::get(shape, i8Ty);
-  mlir::Value local =
-      builder.allocateLocal(loc, seqTy, aggName, "", std::nullopt, std::nullopt,
-                            /*target=*/false);
+  mlir::Value local = builder.allocateLocal(loc, seqTy, aggName, "", {}, {},
+                                            /*target=*/false);
   insertAggregateStore(storeMap, var, local);
 }
 
@@ -1834,8 +1844,8 @@ static void genDeclareSymbol(Fortran::lower::AbstractConverter &converter,
                              Fortran::lower::SymMap &symMap,
                              const Fortran::semantics::Symbol &sym,
                              mlir::Value base, mlir::Value len = {},
-                             llvm::ArrayRef<mlir::Value> shape = std::nullopt,
-                             llvm::ArrayRef<mlir::Value> lbounds = std::nullopt,
+                             llvm::ArrayRef<mlir::Value> shape = {},
+                             llvm::ArrayRef<mlir::Value> lbounds = {},
                              bool force = false) {
   // In HLFIR, procedure dummy symbols are not added with an hlfir.declare
   // because they are "values", and hlfir.declare is intended for variables. It
@@ -2003,8 +2013,8 @@ genAllocatableOrPointerDeclare(Fortran::lower::AbstractConverter &converter,
     explictLength = box.nonDeferredLenParams()[0];
   }
   genDeclareSymbol(converter, symMap, sym, base, explictLength,
-                   /*shape=*/std::nullopt,
-                   /*lbounds=*/std::nullopt, force);
+                   /*shape=*/{},
+                   /*lbounds=*/{}, force);
 }
 
 /// Map a procedure pointer
@@ -2013,8 +2023,8 @@ static void genProcPointer(Fortran::lower::AbstractConverter &converter,
                            const Fortran::semantics::Symbol &sym,
                            mlir::Value addr, bool force = false) {
   genDeclareSymbol(converter, symMap, sym, addr, mlir::Value{},
-                   /*shape=*/std::nullopt,
-                   /*lbounds=*/std::nullopt, force);
+                   /*shape=*/{},
+                   /*lbounds=*/{}, force);
 }
 
 /// Map a symbol represented with a runtime descriptor to its FIR fir.box and
@@ -2466,8 +2476,7 @@ void Fortran::lower::defineModuleVariable(
     AbstractConverter &converter, const Fortran::lower::pft::Variable &var) {
   // Use empty linkage for module variables, which makes them available
   // for use in another unit.
-  mlir::StringAttr linkage =
-      getLinkageAttribute(converter.getFirOpBuilder(), var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   if (!var.isGlobal())
     fir::emitFatalError(converter.getCurrentLocation(),
                         "attempting to lower module variable as local");
@@ -2602,10 +2611,9 @@ void Fortran::lower::createIntrinsicModuleGlobal(
 void Fortran::lower::createRuntimeTypeInfoGlobal(
     Fortran::lower::AbstractConverter &converter,
     const Fortran::semantics::Symbol &typeInfoSym) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   std::string globalName = converter.mangleName(typeInfoSym);
   auto var = Fortran::lower::pft::Variable(typeInfoSym, /*global=*/true);
-  mlir::StringAttr linkage = getLinkageAttribute(builder, var);
+  mlir::StringAttr linkage = getLinkageAttribute(converter, var);
   defineGlobal(converter, var, globalName, linkage);
 }
 
diff --git a/flang/lib/Lower/HostAssociations.cpp b/flang/lib/Lower/HostAssociations.cpp
index e58dc827652e..6a44be65a6cd 100644
--- a/flang/lib/Lower/HostAssociations.cpp
+++ b/flang/lib/Lower/HostAssociations.cpp
@@ -449,7 +449,7 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
     }
 
     if (canReadCapturedBoxValue(converter, sym)) {
-      fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/std::nullopt);
+      fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/{});
       bindCapturedSymbol(sym,
                          fir::factory::readBoxValue(builder, loc, boxValue),
                          converter, args.symMap);
@@ -470,7 +470,7 @@ class CapturedArrays : public CapturedSymbols<CapturedArrays> {
         box = builder.create<mlir::arith::SelectOp>(loc, isPresent, box,
                                                     absentBox);
       }
-      fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/std::nullopt);
+      fir::BoxValue boxValue(box, lbounds, /*explicitParams=*/{});
       bindCapturedSymbol(sym, boxValue, converter, args.symMap);
     }
   }
diff --git a/flang/lib/Lower/Mangler.cpp b/flang/lib/Lower/Mangler.cpp
index 1165417ef89a..1333e3fe349d 100644
--- a/flang/lib/Lower/Mangler.cpp
+++ b/flang/lib/Lower/Mangler.cpp
@@ -119,8 +119,7 @@ std::string Fortran::lower::mangle::mangleName(
             // Mangle external procedure without any scope prefix.
             if (!keepExternalInScope &&
                 Fortran::semantics::IsExternal(ultimateSymbol))
-              return fir::NameUniquer::doProcedure(std::nullopt, std::nullopt,
-                                                   symbolName);
+              return fir::NameUniquer::doProcedure({}, {}, symbolName);
             // A separate module procedure must be mangled according to its
             // declaration scope, not its definition scope.
             const Fortran::semantics::Symbol *interface = &ultimateSymbol;
@@ -142,8 +141,7 @@ std::string Fortran::lower::mangle::mangleName(
             }
             // Otherwise, this is an external procedure, with or without an
             // explicit EXTERNAL attribute. Mangle it without any prefix.
-            return fir::NameUniquer::doProcedure(std::nullopt, std::nullopt,
-                                                 symbolName);
+            return fir::NameUniquer::doProcedure({}, {}, symbolName);
           },
           [&](const Fortran::semantics::ObjectEntityDetails &) {
             return mangleObject();
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 3ef3330cba2d..e56d7f7ed9b6 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -4561,7 +4561,8 @@ void createOpenACCRoutineConstruct(
     llvm::SmallVector<mlir::Attribute> &vectorDeviceTypes) {
 
   for (auto routineOp : mod.getOps<mlir::acc::RoutineOp>()) {
-    if (routineOp.getFuncName().str().compare(funcName) == 0) {
+    if (routineOp.getFuncName().getLeafReference().str().compare(funcName) ==
+        0) {
       // If the routine is already specified with the same clauses, just skip
       // the operation creation.
       if (compareDeviceTypeInfo(routineOp, bindNames, bindNameDeviceTypes,
@@ -4579,7 +4580,9 @@ void createOpenACCRoutineConstruct(
   mlir::OpBuilder modBuilder(mod.getBodyRegion());
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   modBuilder.create<mlir::acc::RoutineOp>(
-      loc, routineOpStr, funcName, getArrayAttrOrNull(builder, bindNames),
+      loc, routineOpStr,
+      mlir::SymbolRefAttr::get(builder.getContext(), funcName),
+      getArrayAttrOrNull(builder, bindNames),
       getArrayAttrOrNull(builder, bindNameDeviceTypes),
       getArrayAttrOrNull(builder, workerDeviceTypes),
       getArrayAttrOrNull(builder, vectorDeviceTypes),
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index e641a48a0335..307b10b0c261 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -11,6 +11,8 @@
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -41,6 +43,179 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
+namespace {
+// An example of a type that can be used to get the return value from
+// the visitor:
+//   visitor(type_identity<Xyz>) -> result_type
+using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
+
+struct GetProc
+    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
+                                false> {
+  using Result = const evaluate::ProcedureDesignator *;
+  using Base = evaluate::Traverse<GetProc, Result, false>;
+  GetProc() : Base(*this) {}
+
+  using Base::operator();
+
+  static Result Default() { return nullptr; }
+
+  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
+  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
+};
+
+struct WithType {
+  WithType(const evaluate::DynamicType &t) : type(t) {
+    assert(type.category() != common::TypeCategory::Derived &&
+           "Type cannot be a derived type");
+  }
+
+  template <typename VisitorTy> //
+  auto visit(VisitorTy &&visitor) const
+      -> std::invoke_result_t<VisitorTy, SomeArgType> {
+    switch (type.category()) {
+    case common::TypeCategory::Integer:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Unsigned:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Real:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Complex:
+      switch (type.kind()) {
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
+      case 3:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
+      case 10:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
+      case 16:
+        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
+      }
+      break;
+    case common::TypeCategory::Logical:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
+      case 8:
+        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
+      }
+      break;
+    case common::TypeCategory::Character:
+      switch (type.kind()) {
+      case 1:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
+      case 2:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
+      case 4:
+        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
+      }
+      break;
+    case common::TypeCategory::Derived:
+      (void)Derived;
+      break;
+    }
+    llvm_unreachable("Unhandled type");
+  }
+
+  const evaluate::DynamicType &type;
+
+private:
+  // Shorter names.
+  static constexpr auto Character = common::TypeCategory::Character;
+  static constexpr auto Complex = common::TypeCategory::Complex;
+  static constexpr auto Derived = common::TypeCategory::Derived;
+  static constexpr auto Integer = common::TypeCategory::Integer;
+  static constexpr auto Logical = common::TypeCategory::Logical;
+  static constexpr auto Real = common::TypeCategory::Real;
+  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
+};
+
+template <typename T, typename U = std::remove_const_t<T>>
+U AsRvalue(T &t) {
+  U copy{t};
+  return std::move(copy);
+}
+
+template <typename T>
+T &&AsRvalue(T &&t) {
+  return std::move(t);
+}
+
+struct ArgumentReplacer
+    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
+  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
+  using Result = bool;
+
+  Result Default() const { return false; }
+
+  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
+      : Base(*this), args_(std::move(newArgs)) {}
+
+  using Base::operator();
+
+  template <typename T>
+  Result operator()(const evaluate::FunctionRef<T> &x) {
+    assert(!done_);
+    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
+    mut.arguments() = args_;
+    done_ = true;
+    return true;
+  }
+
+  Result Combine(Result &&a, Result &&b) { return a || b; }
+
+private:
+  bool done_{false};
+  evaluate::ActualArguments &&args_;
+};
+} // namespace
+
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -237,6 +412,85 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
+static bool replaceArgs(semantics::SomeExpr &expr,
+                        evaluate::ActualArguments &&newArgs) {
+  return ArgumentReplacer(std::move(newArgs))(expr);
+}
+
+static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
+                                    const evaluate::ProcedureDesignator &proc,
+                                    const evaluate::ActualArguments &args) {
+  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
+    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
+    return evaluate::AsGenericExpr(
+        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
+  });
+}
+
+static const evaluate::ProcedureDesignator &
+getProcedureDesignator(const semantics::SomeExpr &call) {
+  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
+  assert(proc && "Call has no procedure designator");
+  return *proc;
+}
+
+static semantics::SomeExpr //
+genReducedMinMax(const semantics::SomeExpr &orig,
+                 const semantics::SomeExpr *atomArg,
+                 const std::vector<semantics::SomeExpr> &args) {
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be atomArg.
+  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
+  // call = min/max(a_t, tmp).
+  // Return "call".
+
+  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
+  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
+  // argument to a non-optional position. This could happen if a_t is at
+  // position 0 or 1.
+  if (args.size() <= 2)
+    return orig;
+
+  evaluate::ActualArguments nonAtoms;
+
+  auto AsActual = [](const semantics::SomeExpr &x) {
+    semantics::SomeExpr copy = x;
+    return evaluate::ActualArgument(std::move(copy));
+  };
+  // Semantic checks guarantee that the "atom" shows exactly once in the
+  // argument list (with potential conversions around it).
+  // For the first two (non-optional) arguments, if "atom" is among them,
+  // replace it with another occurrence of the other non-optional argument.
+  if (atomArg == &args[0]) {
+    // (atom, x, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[1]));
+    nonAtoms.push_back(AsActual(args[1]));
+  } else if (atomArg == &args[1]) {
+    // (x, atom, y...) -> (x, x, y...)
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[0]));
+  } else {
+    // (x, y, z...) -> unchanged
+    nonAtoms.push_back(AsActual(args[0]));
+    nonAtoms.push_back(AsActual(args[1]));
+  }
+
+  // The rest of arguments are optional, so we can just skip "atom".
+  for (size_t i = 2, e = args.size(); i != e; ++i) {
+    if (atomArg != &args[i])
+      nonAtoms.push_back(AsActual(args[i]));
+  }
+
+  // The type of the intermediate min/max is the same as the type of its
+  // arguments, which may be different from the type of the original
+  // expression. The original expression may have additional coverts.
+  auto tmp =
+      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
+  semantics::SomeExpr call = orig;
+  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
+  return call;
+}
+
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -350,10 +604,29 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
-  std::vector<semantics::SomeExpr> args =
-      evaluate::GetTopLevelOperation(input).second;
+  semantics::SomeExpr rhs = assign.rhs;
+  semantics::SomeExpr input = *evaluate::GetConvertInput(rhs);
+  auto [opcode, args] = evaluate::GetTopLevelOperation(input);
   assert(!args.empty() && "Update operation without arguments");
+
+  // Pass args as an argument to avoid capturing a structured binding.
+  const semantics::SomeExpr *atomArg = [&](auto &args) {
+    for (const semantics::SomeExpr &e : args) {
+      if (evaluate::IsSameOrConvertOf(e, atom))
+        return &e;
+    }
+    llvm_unreachable("Atomic variable not in argument list");
+  }(args);
+
+  if (opcode == evaluate::operation::Operator::Min ||
+      opcode == evaluate::operation::Operator::Max) {
+    // Min and max operations are expanded inline, so reduce them to
+    // operations with exactly two (non-optional) arguments.
+    rhs = genReducedMinMax(rhs, atomArg, args);
+    input = *evaluate::GetConvertInput(rhs);
+    std::tie(opcode, args) = evaluate::GetTopLevelOperation(input);
+    atomArg = nullptr; // No longer valid.
+  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
@@ -372,7 +645,7 @@ genAtomicUpdate(lower::AbstractConverter &converter,
 
   converter.overrideExprValues(&overrides);
   mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+      fir::getBase(converter.genExprValue(rhs, stmtCtx, &loc));
   mlir::Value converted = builder.createConvert(loc, atomType, updated);
   builder.create<mlir::omp::YieldOp>(loc, converted);
   converter.resetExprOverrides();
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 657fa1485e0f..e08c6143edae 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -625,7 +625,7 @@ void DataSharingProcessor::privatizeSymbol(
         result.getDeallocRegion(),
         emitCopyRegion ? omp::DeclOperationKind::FirstPrivate
                        : omp::DeclOperationKind::Private,
-        symToPrivatize);
+        symToPrivatize, cannotHaveNonDefaultLowerBounds);
     // TODO: currently there are false positives from dead uses of the mold
     // arg
     if (result.initReadsFromMold())
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 5544d85128ed..1ccf4fb87477 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -39,6 +39,7 @@
 #include "flang/Support/OpenMP-utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Support/StateStack.h"
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -198,9 +199,41 @@ class HostEvalInfo {
 /// the handling of the outer region by keeping a stack of information
 /// structures, but it will probably still require some further work to support
 /// reverse offloading.
-static llvm::SmallVector<HostEvalInfo, 0> hostEvalInfo;
-static llvm::SmallVector<const parser::OpenMPSectionsConstruct *, 0>
-    sectionsStack;
+class HostEvalInfoStackFrame
+    : public mlir::StateStackFrameBase<HostEvalInfoStackFrame> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(HostEvalInfoStackFrame)
+
+  HostEvalInfo info;
+};
+
+static HostEvalInfo *
+getHostEvalInfoStackTop(lower::AbstractConverter &converter) {
+  HostEvalInfoStackFrame *frame =
+      converter.getStateStack().getStackTop<HostEvalInfoStackFrame>();
+  return frame ? &frame->info : nullptr;
+}
+
+/// Stack frame for storing the OpenMPSectionsConstruct currently being
+/// processed so that it can be referred to when lowering the construct.
+class SectionsConstructStackFrame
+    : public mlir::StateStackFrameBase<SectionsConstructStackFrame> {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SectionsConstructStackFrame)
+
+  explicit SectionsConstructStackFrame(
+      const parser::OpenMPSectionsConstruct &sectionsConstruct)
+      : sectionsConstruct{sectionsConstruct} {}
+
+  const parser::OpenMPSectionsConstruct &sectionsConstruct;
+};
+
+static const parser::OpenMPSectionsConstruct *
+getSectionsConstructStackTop(lower::AbstractConverter &converter) {
+  SectionsConstructStackFrame *frame =
+      converter.getStateStack().getStackTop<SectionsConstructStackFrame>();
+  return frame ? &frame->sectionsConstruct : nullptr;
+}
 
 /// Bind symbols to their corresponding entry block arguments.
 ///
@@ -535,31 +568,32 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     if (!ompEval)
       return;
 
-    HostEvalInfo &hostInfo = hostEvalInfo.back();
+    HostEvalInfo *hostInfo = getHostEvalInfoStackTop(converter);
+    assert(hostInfo && "expected HOST_EVAL info structure");
 
     switch (extractOmpDirective(*ompEval)) {
     case OMPD_teams_distribute_parallel_do:
     case OMPD_teams_distribute_parallel_do_simd:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      cp.processThreadLimit(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_target_teams_distribute_parallel_do:
     case OMPD_target_teams_distribute_parallel_do_simd:
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      cp.processNumTeams(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_distribute_parallel_do:
     case OMPD_distribute_parallel_do_simd:
-      cp.processNumThreads(stmtCtx, hostInfo.ops);
+      cp.processNumThreads(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_distribute:
     case OMPD_distribute_simd:
-      cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
+      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
       break;
 
     case OMPD_teams:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      cp.processThreadLimit(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_target_teams:
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      cp.processNumTeams(stmtCtx, hostInfo->ops);
       processSingleNestedIf([](Directive nestedDir) {
         return topDistributeSet.test(nestedDir) || topLoopSet.test(nestedDir);
       });
@@ -567,22 +601,22 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
 
     case OMPD_teams_distribute:
     case OMPD_teams_distribute_simd:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      cp.processThreadLimit(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_target_teams_distribute:
     case OMPD_target_teams_distribute_simd:
-      cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
+      cp.processNumTeams(stmtCtx, hostInfo->ops);
       break;
 
     case OMPD_teams_loop:
-      cp.processThreadLimit(stmtCtx, hostInfo.ops);
+      cp.processThreadLimit(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_target_teams_loop:
-      cp.processNumTeams(stmtCtx, hostInfo.ops);
+      cp.processNumTeams(stmtCtx, hostInfo->ops);
       [[fallthrough]];
     case OMPD_loop:
-      cp.processCollapse(loc, eval, hostInfo.ops, hostInfo.iv);
+      cp.processCollapse(loc, eval, hostInfo->ops, hostInfo->iv);
       break;
 
     // Standalone 'target' case.
@@ -596,8 +630,6 @@ static void processHostEvalClauses(lower::AbstractConverter &converter,
     }
   };
 
-  assert(!hostEvalInfo.empty() && "expected HOST_EVAL info structure");
-
   const auto *ompEval = eval.getIf<parser::OpenMPConstruct>();
   assert(ompEval &&
          llvm::omp::allTargetSet.test(extractOmpDirective(*ompEval)) &&
@@ -1459,8 +1491,9 @@ static void genBodyOfTargetOp(
     dsp.processStep2();
 
   bindEntryBlockArgs(converter, targetOp, args);
-  if (!hostEvalInfo.empty())
-    hostEvalInfo.back().bindOperands(argIface.getHostEvalBlockArgs());
+  if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter))
+    hostEvalInfo->bindOperands(argIface.getHostEvalBlockArgs());
+
   // Check if cloning the bounds introduced any dependency on the outer region.
   // If so, then either clone them as well if they are MemoryEffectFree, or else
   // copy them to a new temporary and add them to the map and block_argument
@@ -1601,7 +1634,8 @@ genLoopNestClauses(lower::AbstractConverter &converter,
                    llvm::SmallVectorImpl<const semantics::Symbol *> &iv) {
   ClauseProcessor cp(converter, semaCtx, clauses);
 
-  if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps, iv))
+  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
+  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps, iv))
     cp.processCollapse(loc, eval, clauseOps, iv);
 
   clauseOps.loopInclusive = converter.getFirOpBuilder().getUnitAttr();
@@ -1646,7 +1680,8 @@ static void genParallelClauses(
   cp.processAllocate(clauseOps);
   cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
 
-  if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps))
+  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
+  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps))
     cp.processNumThreads(stmtCtx, clauseOps);
 
   cp.processProcBind(clauseOps);
@@ -1716,10 +1751,10 @@ static void genTargetClauses(
   cp.processDepend(symTable, stmtCtx, clauseOps);
   cp.processDevice(stmtCtx, clauseOps);
   cp.processHasDeviceAddr(stmtCtx, clauseOps, hasDeviceAddrSyms);
-  if (!hostEvalInfo.empty()) {
+  if (HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter)) {
     // Only process host_eval if compiling for the host device.
     processHostEvalClauses(converter, semaCtx, stmtCtx, eval, loc);
-    hostEvalInfo.back().collectValues(clauseOps.hostEvalVars);
+    hostEvalInfo->collectValues(clauseOps.hostEvalVars);
   }
   cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
   cp.processIsDevicePtr(clauseOps, isDevicePtrSyms);
@@ -1859,7 +1894,8 @@ static void genTeamsClauses(
   cp.processAllocate(clauseOps);
   cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
 
-  if (hostEvalInfo.empty() || !hostEvalInfo.back().apply(clauseOps)) {
+  HostEvalInfo *hostEvalInfo = getHostEvalInfoStackTop(converter);
+  if (!hostEvalInfo || !hostEvalInfo->apply(clauseOps)) {
     cp.processNumTeams(stmtCtx, clauseOps);
     cp.processThreadLimit(stmtCtx, clauseOps);
   }
@@ -2112,19 +2148,18 @@ genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       converter.getCurrentLocation(), clauseOps);
 }
 
-/// This breaks the normal prototype of the gen*Op functions: adding the
-/// sectionBlocks argument so that the enclosed section constructs can be
-/// lowered here with correct reduction symbol remapping.
 static mlir::omp::SectionsOp
 genSectionsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
               semantics::SemanticsContext &semaCtx,
               lower::pft::Evaluation &eval, mlir::Location loc,
               const ConstructQueue &queue,
               ConstructQueue::const_iterator item) {
-  assert(!sectionsStack.empty());
+  const parser::OpenMPSectionsConstruct *sectionsConstruct =
+      getSectionsConstructStackTop(converter);
+  assert(sectionsConstruct && "Missing additional parsing information");
+
   const auto &sectionBlocks =
-      std::get<parser::OmpSectionBlocks>(sectionsStack.back()->t);
-  sectionsStack.pop_back();
+      std::get<parser::OmpSectionBlocks>(sectionsConstruct->t);
   mlir::omp::SectionsOperands clauseOps;
   llvm::SmallVector<const semantics::Symbol *> reductionSyms;
   genSectionsClauses(converter, semaCtx, item->clauses, loc, clauseOps,
@@ -2280,7 +2315,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   // Introduce a new host_eval information structure for this target region.
   if (!isTargetDevice)
-    hostEvalInfo.emplace_back();
+    converter.getStateStack().stackPush<HostEvalInfoStackFrame>();
 
   mlir::omp::TargetOperands clauseOps;
   DefaultMapsTy defaultMaps;
@@ -2306,6 +2341,10 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
     if (dsp.getAllSymbolsToPrivatize().contains(&sym))
       return;
 
+    // Skip parameters/constants as they do not need to be mapped.
+    if (semantics::IsNamedConstant(sym))
+      return;
+
     // These symbols are mapped individually in processHasDeviceAddr.
     if (llvm::is_contained(hasDeviceAddrSyms, &sym))
       return;
@@ -2407,7 +2446,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
   // Remove the host_eval information structure created for this target region.
   if (!isTargetDevice)
-    hostEvalInfo.pop_back();
+    converter.getStateStack().stackPop();
   return targetOp;
 }
 
@@ -3653,6 +3692,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
 
+  auto &optLoopCons =
+      std::get<std::optional<parser::NestedConstruct>>(loopConstruct.t);
+  if (optLoopCons.has_value()) {
+    if (auto *ompNestedLoopCons{
+            std::get_if<common::Indirection<parser::OpenMPLoopConstruct>>(
+                &*optLoopCons)}) {
+      genOMP(converter, symTable, semaCtx, eval, ompNestedLoopCons->value());
+    }
+  }
+
   llvm::omp::Directive directive =
       std::get<parser::OmpLoopDirective>(beginLoopDirective.t).v;
   const parser::CharBlock &source =
@@ -3693,7 +3742,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
       buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx,
                           eval, source, directive, clauses)};
 
-  sectionsStack.push_back(&sectionsConstruct);
+  mlir::SaveStateStack<SectionsConstructStackFrame> saveStateStack{
+      converter.getStateStack(), sectionsConstruct};
   genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue,
                  queue.begin());
 }
diff --git a/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp b/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
index cc5c6448401b..2d945d535d1a 100644
--- a/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/OpenMP/PrivateReductionUtils.cpp
@@ -123,16 +123,15 @@ static void createCleanupRegion(Fortran::lower::AbstractConverter &converter,
   typeError();
 }
 
-fir::ShapeShiftOp
-Fortran::lower::omp::getShapeShift(fir::FirOpBuilder &builder,
-                                   mlir::Location loc, mlir::Value box,
-                                   bool useDefaultLowerBounds) {
+fir::ShapeShiftOp Fortran::lower::omp::getShapeShift(
+    fir::FirOpBuilder &builder, mlir::Location loc, mlir::Value box,
+    bool cannotHaveNonDefaultLowerBounds, bool useDefaultLowerBounds) {
   fir::SequenceType sequenceType = mlir::cast<fir::SequenceType>(
       hlfir::getFortranElementOrSequenceType(box.getType()));
   const unsigned rank = sequenceType.getDimension();
+
   llvm::SmallVector<mlir::Value> lbAndExtents;
   lbAndExtents.reserve(rank * 2);
-
   mlir::Type idxTy = builder.getIndexType();
 
   mlir::Value oneVal;
@@ -142,7 +141,8 @@ Fortran::lower::omp::getShapeShift(fir::FirOpBuilder &builder,
     return oneVal;
   };
 
-  if ((useDefaultLowerBounds) && !sequenceType.hasDynamicExtents()) {
+  if ((cannotHaveNonDefaultLowerBounds || useDefaultLowerBounds) &&
+      !sequenceType.hasDynamicExtents()) {
     // We don't need fir::BoxDimsOp if all of the extents are statically known
     // and we can assume default lower bounds. This helps avoids reads from the
     // mold arg.
@@ -273,12 +273,13 @@ class PopulateInitAndCleanupRegionsHelper {
       mlir::Type argType, mlir::Value scalarInitValue,
       mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
       mlir::Block *initBlock, mlir::Region &cleanupRegion,
-      DeclOperationKind kind, const Fortran::semantics::Symbol *sym)
+      DeclOperationKind kind, const Fortran::semantics::Symbol *sym,
+      bool cannotHaveLowerBounds)
       : converter{converter}, builder{converter.getFirOpBuilder()}, loc{loc},
         argType{argType}, scalarInitValue{scalarInitValue},
         allocatedPrivVarArg{allocatedPrivVarArg}, moldArg{moldArg},
         initBlock{initBlock}, cleanupRegion{cleanupRegion}, kind{kind},
-        sym{sym} {
+        sym{sym}, cannotHaveNonDefaultLowerBounds{cannotHaveLowerBounds} {
     valType = fir::unwrapRefType(argType);
   }
 
@@ -320,6 +321,10 @@ class PopulateInitAndCleanupRegionsHelper {
   /// Any length parameters which have been fetched for the type
   mlir::SmallVector<mlir::Value> lenParams;
 
+  /// If the source variable being privatized definitely can't have non-default
+  /// lower bounds then we don't need to generate code to read them.
+  bool cannotHaveNonDefaultLowerBounds;
+
   void createYield(mlir::Value ret) {
     builder.create<mlir::omp::YieldOp>(loc, ret);
   }
@@ -457,7 +462,8 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
   // Special case for (possibly allocatable) arrays of polymorphic types
   // e.g. !fir.class<!fir.heap<!fir.array<?x!fir.type<>>>>
   if (source.isPolymorphic()) {
-    fir::ShapeShiftOp shape = getShapeShift(builder, loc, source);
+    fir::ShapeShiftOp shape =
+        getShapeShift(builder, loc, source, cannotHaveNonDefaultLowerBounds);
     mlir::Type arrayType = source.getElementOrSequenceType();
     mlir::Value allocatedArray = builder.create<fir::AllocMemOp>(
         loc, arrayType, /*typeparams=*/mlir::ValueRange{}, shape.getExtents());
@@ -505,8 +511,8 @@ void PopulateInitAndCleanupRegionsHelper::initAndCleanupBoxedArray(
   // Put the temporary inside of a box:
   // hlfir::genVariableBox doesn't handle non-default lower bounds
   mlir::Value box;
-  fir::ShapeShiftOp shapeShift =
-      getShapeShift(builder, loc, getLoadedMoldArg());
+  fir::ShapeShiftOp shapeShift = getShapeShift(builder, loc, getLoadedMoldArg(),
+                                               cannotHaveNonDefaultLowerBounds);
   mlir::Type boxType = getLoadedMoldArg().getType();
   if (mlir::isa<fir::BaseBoxType>(temp.getType()))
     // the box created by the declare form createTempFromMold is missing
@@ -641,10 +647,10 @@ void Fortran::lower::omp::populateByRefInitAndCleanupRegions(
     mlir::Type argType, mlir::Value scalarInitValue, mlir::Block *initBlock,
     mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
     mlir::Region &cleanupRegion, DeclOperationKind kind,
-    const Fortran::semantics::Symbol *sym) {
+    const Fortran::semantics::Symbol *sym, bool cannotHaveLowerBounds) {
   PopulateInitAndCleanupRegionsHelper helper(
       converter, loc, argType, scalarInitValue, allocatedPrivVarArg, moldArg,
-      initBlock, cleanupRegion, kind, sym);
+      initBlock, cleanupRegion, kind, sym, cannotHaveLowerBounds);
   helper.populateByRefInitAndCleanupRegions();
 
   // Often we load moldArg to check something (e.g. length parameters, shape)
diff --git a/flang/lib/Lower/OpenMP/PrivateReductionUtils.h b/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
index 8652c098367e..9f8c9aee4d8e 100644
--- a/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
+++ b/flang/lib/Lower/OpenMP/PrivateReductionUtils.h
@@ -55,7 +55,8 @@ void populateByRefInitAndCleanupRegions(
     mlir::Value scalarInitValue, mlir::Block *initBlock,
     mlir::Value allocatedPrivVarArg, mlir::Value moldArg,
     mlir::Region &cleanupRegion, DeclOperationKind kind,
-    const Fortran::semantics::Symbol *sym = nullptr);
+    const Fortran::semantics::Symbol *sym = nullptr,
+    bool cannotHaveNonDefaultLowerBounds = false);
 
 /// Generate a fir::ShapeShift op describing the provided boxed array.
 /// `cannotHaveNonDefaultLowerBounds` should be set if `box` is known to have
@@ -65,6 +66,7 @@ void populateByRefInitAndCleanupRegions(
 /// elements without having to adjust each index.
 fir::ShapeShiftOp getShapeShift(fir::FirOpBuilder &builder, mlir::Location loc,
                                 mlir::Value box,
+                                bool cannotHaveNonDefaultLowerBounds = false,
                                 bool useDefaultLowerBounds = false);
 
 } // namespace omp
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index fb7a33bc4aa0..10324272e0d6 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -149,13 +149,17 @@ std::string ReductionProcessor::getReductionName(
     reductionName = "multiply_reduction";
     break;
   case omp::clause::DefinedOperator::IntrinsicOperator::AND:
-    return "and_reduction";
+    reductionName = "and_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::EQV:
-    return "eqv_reduction";
+    reductionName = "eqv_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::OR:
-    return "or_reduction";
+    reductionName = "or_reduction";
+    break;
   case omp::clause::DefinedOperator::IntrinsicOperator::NEQV:
-    return "neqv_reduction";
+    reductionName = "neqv_reduction";
+    break;
   default:
     reductionName = "other_reduction";
     break;
@@ -371,8 +375,10 @@ static void genBoxCombiner(fir::FirOpBuilder &builder, mlir::Location loc,
 
   // Get ShapeShift with default lower bounds. This makes it possible to use
   // unmodified LoopNest's indices with ArrayCoorOp.
-  fir::ShapeShiftOp shapeShift = getShapeShift(builder, loc, lhs,
-                                               /*useDefaultLowerBounds=*/true);
+  fir::ShapeShiftOp shapeShift =
+      getShapeShift(builder, loc, lhs,
+                    /*cannotHaveNonDefaultLowerBounds=*/false,
+                    /*useDefaultLowerBounds=*/true);
 
   // Iterate over array elements, applying the equivalent scalar reduction:
 
@@ -671,9 +677,6 @@ void ReductionProcessor::processReductionArguments(
     const auto &kindMap = firOpBuilder.getKindMap();
     std::string reductionName;
     ReductionIdentifier redId;
-    mlir::Type redNameTy = redType;
-    if (mlir::isa<fir::LogicalType>(redType.getEleTy()))
-      redNameTy = builder.getI1Type();
 
     if (const auto &redDefinedOp =
             std::get_if<omp::clause::DefinedOperator>(&redOperator.u)) {
@@ -695,8 +698,7 @@ void ReductionProcessor::processReductionArguments(
         break;
       }
 
-      reductionName =
-          getReductionName(intrinsicOp, kindMap, redNameTy, isByRef);
+      reductionName = getReductionName(intrinsicOp, kindMap, redType, isByRef);
     } else if (const auto *reductionIntrinsic =
                    std::get_if<omp::clause::ProcedureDesignator>(
                        &redOperator.u)) {
@@ -707,7 +709,7 @@ void ReductionProcessor::processReductionArguments(
       redId = getReductionType(*reductionIntrinsic);
       reductionName =
           getReductionName(getRealName(*reductionIntrinsic).ToString(), kindMap,
-                           redNameTy, isByRef);
+                           redType, isByRef);
     } else {
       TODO(currentLocation, "Unexpected reduction type");
     }
diff --git a/flang/lib/Lower/SymbolMap.cpp b/flang/lib/Lower/SymbolMap.cpp
index ce078a7fbde6..b929dfbd5aec 100644
--- a/flang/lib/Lower/SymbolMap.cpp
+++ b/flang/lib/Lower/SymbolMap.cpp
@@ -82,6 +82,10 @@ Fortran::lower::SymMap::lookupImpliedDo(Fortran::lower::SymMap::AcDoVar var) {
   return {};
 }
 
+void Fortran::lower::SymbolBox::dump() const { llvm::errs() << *this << '\n'; }
+
+void Fortran::lower::SymMap::dump() const { llvm::errs() << *this << '\n'; }
+
 llvm::raw_ostream &
 Fortran::lower::operator<<(llvm::raw_ostream &os,
                            const Fortran::lower::SymbolBox &symBox) {
diff --git a/flang/lib/Optimizer/Builder/Character.cpp b/flang/lib/Optimizer/Builder/Character.cpp
index cd7ffd27117d..61428ac490a4 100644
--- a/flang/lib/Optimizer/Builder/Character.cpp
+++ b/flang/lib/Optimizer/Builder/Character.cpp
@@ -380,7 +380,7 @@ fir::factory::CharacterExprHelper::createCharacterTemp(mlir::Type type,
   if (typeLen == fir::CharacterType::unknownLen())
     lenParams.push_back(len);
   auto ref = builder.allocateLocal(loc, charTy, "", ".chrtmp",
-                                   /*shape=*/std::nullopt, lenParams);
+                                   /*shape=*/{}, lenParams);
   return {ref, len};
 }
 
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 178b6770d6b5..d32c1fde59f2 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -932,6 +932,9 @@ static constexpr IntrinsicHandler handlers[]{
      {{{"count", asAddr}, {"count_rate", asAddr}, {"count_max", asAddr}}},
      /*isElemental=*/false},
     {"tand", &I::genTand},
+    {"this_grid", &I::genThisGrid, {}, /*isElemental=*/false},
+    {"this_thread_block", &I::genThisThreadBlock, {}, /*isElemental=*/false},
+    {"this_warp", &I::genThisWarp, {}, /*isElemental=*/false},
     {"threadfence", &I::genThreadFence, {}, /*isElemental=*/false},
     {"threadfence_block", &I::genThreadFenceBlock, {}, /*isElemental=*/false},
     {"threadfence_system", &I::genThreadFenceSystem, {}, /*isElemental=*/false},
@@ -2645,16 +2648,18 @@ mlir::Value IntrinsicLibrary::genAbs(mlir::Type resultType,
 // ACOSD
 mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ACOSD to ACOS * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("acos", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("acos", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ADJUSTL & ADJUSTR
@@ -2796,16 +2801,18 @@ IntrinsicLibrary::genAny(mlir::Type resultType,
 // ASIND
 mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
                                        llvm::ArrayRef<mlir::Value> args) {
+  // maps ASIND to ASIN * 180 / pi
   assert(args.size() == 1);
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
+  mlir::Value result =
+      getRuntimeCallGenerator("asin", ftype)(builder, loc, {args[0]});
   llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
   mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), pi / llvm::APFloat(180.0));
+      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
   mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
-  mlir::Value arg = builder.create<mlir::arith::MulFOp>(loc, args[0], factor);
-  return getRuntimeCallGenerator("asin", ftype)(builder, loc, {arg});
+  return builder.create<mlir::arith::MulFOp>(loc, result, factor);
 }
 
 // ATAND, ATAN2D
@@ -5147,9 +5154,9 @@ void IntrinsicLibrary::genIeeeGetOrSetModesOrStatus(
         isModes ? fir::runtime::genGetModesTypeSize(builder, loc)
                 : fir::runtime::genGetStatusTypeSize(builder, loc);
     byteSize = builder.createConvert(loc, builder.getIndexType(), byteSize);
-    addr =
-        builder.create<fir::AllocMemOp>(loc, extractSequenceType(heapTy),
-                                        /*typeparams=*/std::nullopt, byteSize);
+    addr = builder.create<fir::AllocMemOp>(loc, extractSequenceType(heapTy),
+                                           /*typeparams=*/mlir::ValueRange(),
+                                           byteSize);
     mlir::Value shape = builder.create<fir::ShapeOp>(loc, byteSize);
     builder.create<fir::StoreOp>(
         loc, builder.create<fir::EmboxOp>(loc, fieldTy, addr, shape), fieldRef);
@@ -8109,6 +8116,183 @@ mlir::Value IntrinsicLibrary::genTand(mlir::Type resultType,
   return getRuntimeCallGenerator("tan", ftype)(builder, loc, {arg});
 }
 
+// THIS_GRID
+mlir::Value IntrinsicLibrary::genThisGrid(mlir::Type resultType,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0);
+  auto recTy = mlir::cast<fir::RecordType>(resultType);
+  assert(recTy && "RecordType expepected");
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Type i32Ty = builder.getI32Type();
+
+  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value threadIdY = builder.create<mlir::NVVM::ThreadIdYOp>(loc, i32Ty);
+  mlir::Value threadIdZ = builder.create<mlir::NVVM::ThreadIdZOp>(loc, i32Ty);
+
+  mlir::Value blockIdX = builder.create<mlir::NVVM::BlockIdXOp>(loc, i32Ty);
+  mlir::Value blockIdY = builder.create<mlir::NVVM::BlockIdYOp>(loc, i32Ty);
+  mlir::Value blockIdZ = builder.create<mlir::NVVM::BlockIdZOp>(loc, i32Ty);
+
+  mlir::Value blockDimX = builder.create<mlir::NVVM::BlockDimXOp>(loc, i32Ty);
+  mlir::Value blockDimY = builder.create<mlir::NVVM::BlockDimYOp>(loc, i32Ty);
+  mlir::Value blockDimZ = builder.create<mlir::NVVM::BlockDimZOp>(loc, i32Ty);
+  mlir::Value gridDimX = builder.create<mlir::NVVM::GridDimXOp>(loc, i32Ty);
+  mlir::Value gridDimY = builder.create<mlir::NVVM::GridDimYOp>(loc, i32Ty);
+  mlir::Value gridDimZ = builder.create<mlir::NVVM::GridDimZOp>(loc, i32Ty);
+
+  // this_grid.size = ((blockDim.z * gridDim.z) * (blockDim.y * gridDim.y)) *
+  // (blockDim.x * gridDim.x);
+  mlir::Value resZ =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimZ, gridDimZ);
+  mlir::Value resY =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimY, gridDimY);
+  mlir::Value resX =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimX, gridDimX);
+  mlir::Value resZY = builder.create<mlir::arith::MulIOp>(loc, resZ, resY);
+  mlir::Value size = builder.create<mlir::arith::MulIOp>(loc, resZY, resX);
+
+  // tmp = ((blockIdx.z * gridDim.y * gridDim.x) + (blockIdx.y * gridDim.x)) +
+  //   blockIdx.x;
+  // this_group.rank = tmp * ((blockDim.x * blockDim.y) * blockDim.z) +
+  //   ((threadIdx.z * blockDim.y) * blockDim.x) +
+  //   (threadIdx.y * blockDim.x) + threadIdx.x + 1;
+  mlir::Value r1 = builder.create<mlir::arith::MulIOp>(loc, blockIdZ, gridDimY);
+  mlir::Value r2 = builder.create<mlir::arith::MulIOp>(loc, r1, gridDimX);
+  mlir::Value r3 = builder.create<mlir::arith::MulIOp>(loc, blockIdY, gridDimX);
+  mlir::Value r2r3 = builder.create<mlir::arith::AddIOp>(loc, r2, r3);
+  mlir::Value tmp = builder.create<mlir::arith::AddIOp>(loc, r2r3, blockIdX);
+
+  mlir::Value bXbY =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimX, blockDimY);
+  mlir::Value bXbYbZ =
+      builder.create<mlir::arith::MulIOp>(loc, bXbY, blockDimZ);
+  mlir::Value tZbY =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdZ, blockDimY);
+  mlir::Value tZbYbX =
+      builder.create<mlir::arith::MulIOp>(loc, tZbY, blockDimX);
+  mlir::Value tYbX =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdY, blockDimX);
+  mlir::Value rank = builder.create<mlir::arith::MulIOp>(loc, tmp, bXbYbZ);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, tZbYbX);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, tYbX);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, threadIdX);
+  mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, one);
+
+  auto sizeFieldName = recTy.getTypeList()[1].first;
+  mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
+  mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
+  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, sizeFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+
+  auto rankFieldName = recTy.getTypeList()[2].first;
+  mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
+  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, rankFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  return res;
+}
+
+// THIS_THREAD_BLOCK
+mlir::Value
+IntrinsicLibrary::genThisThreadBlock(mlir::Type resultType,
+                                     llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0);
+  auto recTy = mlir::cast<fir::RecordType>(resultType);
+  assert(recTy && "RecordType expepected");
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Type i32Ty = builder.getI32Type();
+
+  // this_thread_block%size = blockDim.z * blockDim.y * blockDim.x;
+  mlir::Value blockDimX = builder.create<mlir::NVVM::BlockDimXOp>(loc, i32Ty);
+  mlir::Value blockDimY = builder.create<mlir::NVVM::BlockDimYOp>(loc, i32Ty);
+  mlir::Value blockDimZ = builder.create<mlir::NVVM::BlockDimZOp>(loc, i32Ty);
+  mlir::Value size =
+      builder.create<mlir::arith::MulIOp>(loc, blockDimZ, blockDimY);
+  size = builder.create<mlir::arith::MulIOp>(loc, size, blockDimX);
+
+  // this_thread_block%rank = ((threadIdx.z * blockDim.y) * blockDim.x) +
+  //   (threadIdx.y * blockDim.x) + threadIdx.x + 1;
+  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value threadIdY = builder.create<mlir::NVVM::ThreadIdYOp>(loc, i32Ty);
+  mlir::Value threadIdZ = builder.create<mlir::NVVM::ThreadIdZOp>(loc, i32Ty);
+  mlir::Value r1 =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdZ, blockDimY);
+  mlir::Value r2 = builder.create<mlir::arith::MulIOp>(loc, r1, blockDimX);
+  mlir::Value r3 =
+      builder.create<mlir::arith::MulIOp>(loc, threadIdY, blockDimX);
+  mlir::Value r2r3 = builder.create<mlir::arith::AddIOp>(loc, r2, r3);
+  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, r2r3, threadIdX);
+  mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
+  rank = builder.create<mlir::arith::AddIOp>(loc, rank, one);
+
+  auto sizeFieldName = recTy.getTypeList()[1].first;
+  mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
+  mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
+  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, sizeFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+
+  auto rankFieldName = recTy.getTypeList()[2].first;
+  mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
+  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, rankFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  return res;
+}
+
+// THIS_WARP
+mlir::Value IntrinsicLibrary::genThisWarp(mlir::Type resultType,
+                                          llvm::ArrayRef<mlir::Value> args) {
+  assert(args.size() == 0);
+  auto recTy = mlir::cast<fir::RecordType>(resultType);
+  assert(recTy && "RecordType expepected");
+  mlir::Value res = builder.create<fir::AllocaOp>(loc, resultType);
+  mlir::Type i32Ty = builder.getI32Type();
+
+  // coalesced_group%size = 32
+  mlir::Value size = builder.createIntegerConstant(loc, i32Ty, 32);
+  auto sizeFieldName = recTy.getTypeList()[1].first;
+  mlir::Type sizeFieldTy = recTy.getTypeList()[1].second;
+  mlir::Type fieldIndexType = fir::FieldType::get(resultType.getContext());
+  mlir::Value sizeFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, sizeFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value sizeCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(sizeFieldTy), res, sizeFieldIndex);
+  builder.create<fir::StoreOp>(loc, size, sizeCoord);
+
+  // coalesced_group%rank = threadIdx.x & 31 + 1
+  mlir::Value threadIdX = builder.create<mlir::NVVM::ThreadIdXOp>(loc, i32Ty);
+  mlir::Value mask = builder.createIntegerConstant(loc, i32Ty, 31);
+  mlir::Value one = builder.createIntegerConstant(loc, i32Ty, 1);
+  mlir::Value masked =
+      builder.create<mlir::arith::AndIOp>(loc, threadIdX, mask);
+  mlir::Value rank = builder.create<mlir::arith::AddIOp>(loc, masked, one);
+  auto rankFieldName = recTy.getTypeList()[2].first;
+  mlir::Type rankFieldTy = recTy.getTypeList()[2].second;
+  mlir::Value rankFieldIndex = builder.create<fir::FieldIndexOp>(
+      loc, fieldIndexType, rankFieldName, recTy,
+      /*typeParams=*/mlir::ValueRange{});
+  mlir::Value rankCoord = builder.create<fir::CoordinateOp>(
+      loc, builder.getRefType(rankFieldTy), res, rankFieldIndex);
+  builder.create<fir::StoreOp>(loc, rank, rankCoord);
+  return res;
+}
+
 // TRAILZ
 mlir::Value IntrinsicLibrary::genTrailz(mlir::Type resultType,
                                         llvm::ArrayRef<mlir::Value> args) {
diff --git a/flang/lib/Optimizer/Builder/MutableBox.cpp b/flang/lib/Optimizer/Builder/MutableBox.cpp
index f20b5e37dffd..d944a4c98473 100644
--- a/flang/lib/Optimizer/Builder/MutableBox.cpp
+++ b/flang/lib/Optimizer/Builder/MutableBox.cpp
@@ -362,7 +362,7 @@ mlir::Value fir::factory::createUnallocatedBox(
     auto zero = builder.createIntegerConstant(loc, builder.getIndexType(), 0);
     llvm::SmallVector<mlir::Value> extents(seqTy.getDimension(), zero);
     shape = builder.createShape(
-        loc, fir::ArrayBoxValue{nullAddr, extents, /*lbounds=*/std::nullopt});
+        loc, fir::ArrayBoxValue{nullAddr, extents, /*lbounds=*/{}});
   }
   // Provide dummy length parameters if they are dynamic. If a length parameter
   // is deferred. It is set to zero here and will be set on allocation.
diff --git a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
index fcc91752552c..0094ce892d61 100644
--- a/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp
@@ -2888,8 +2888,7 @@ void PPCIntrinsicLibrary::genVecStore(llvm::ArrayRef<fir::ExtendedValue> args) {
     llvm_unreachable("invalid vector operation for generator");
   }
 
-  auto funcType{
-      mlir::FunctionType::get(context, {stTy, addr.getType()}, std::nullopt)};
+  auto funcType{mlir::FunctionType::get(context, {stTy, addr.getType()}, {})};
   mlir::func::FuncOp funcOp = builder.createFunction(loc, fname, funcType);
 
   llvm::SmallVector<mlir::Value, 4> biArgs;
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index b6269c11265b..09038f6e4d3a 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -1332,6 +1332,56 @@ genCUFAllocDescriptor(mlir::Location loc,
       .getResult();
 }
 
+/// Get the address of the type descriptor global variable that was created by
+/// lowering for derived type \p recType.
+template <typename ModOpTy>
+static mlir::Value
+getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
+                  mlir::Location loc, fir::RecordType recType,
+                  const fir::FIRToLLVMPassOptions &options) {
+  std::string name =
+      options.typeDescriptorsRenamedForAssembly
+          ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
+          : fir::NameUniquer::getTypeDescriptorName(recType.getName());
+  mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
+  mlir::DataLayout dataLayout(mod);
+  if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name))
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, fir::factory::getGlobalAddressSpace(&dataLayout),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
+  // The global may have already been translated to LLVM.
+  if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name))
+    return replaceWithAddrOfOrASCast(
+        rewriter, loc, global.getAddrSpace(),
+        fir::factory::getProgramAddressSpace(&dataLayout), global.getSymName(),
+        llvmPtrTy);
+  // Type info derived types do not have type descriptors since they are the
+  // types defining type descriptors.
+  if (options.ignoreMissingTypeDescriptors ||
+      fir::NameUniquer::belongsToModule(
+          name, Fortran::semantics::typeInfoBuiltinModule))
+    return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
+
+  if (!options.skipExternalRttiDefinition)
+    fir::emitFatalError(loc,
+                        "runtime derived type info descriptor was not "
+                        "generated and skipExternalRttiDefinition and "
+                        "ignoreMissingTypeDescriptors options are not set");
+
+  // Rtti for a derived type defined in another compilation unit and for which
+  // rtti was not defined in lowering because of the skipExternalRttiDefinition
+  // option. Generate the object declaration now.
+  auto insertPt = rewriter.saveInsertionPoint();
+  rewriter.setInsertionPoint(mod.getBody(), mod.getBody()->end());
+  mlir::LLVM::GlobalOp global = rewriter.create<mlir::LLVM::GlobalOp>(
+      loc, llvmPtrTy, /*constant=*/true, mlir::LLVM::Linkage::External, name,
+      mlir::Attribute());
+  rewriter.restoreInsertionPoint(insertPt);
+  return rewriter.create<mlir::LLVM::AddressOfOp>(loc, llvmPtrTy,
+                                                  global.getSymName());
+}
+
 /// Common base class for embox to descriptor conversion.
 template <typename OP>
 struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
@@ -1444,41 +1494,6 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
                        stride);
   }
 
-  /// Get the address of the type descriptor global variable that was created by
-  /// lowering for derived type \p recType.
-  template <typename ModOpTy>
-  mlir::Value
-  getTypeDescriptor(ModOpTy mod, mlir::ConversionPatternRewriter &rewriter,
-                    mlir::Location loc, fir::RecordType recType) const {
-    std::string name =
-        this->options.typeDescriptorsRenamedForAssembly
-            ? fir::NameUniquer::getTypeDescriptorAssemblyName(recType.getName())
-            : fir::NameUniquer::getTypeDescriptorName(recType.getName());
-    mlir::Type llvmPtrTy = ::getLlvmPtrType(mod.getContext());
-    mlir::DataLayout dataLayout(mod);
-    if (auto global = mod.template lookupSymbol<fir::GlobalOp>(name)) {
-      return replaceWithAddrOfOrASCast(
-          rewriter, loc, fir::factory::getGlobalAddressSpace(&dataLayout),
-          fir::factory::getProgramAddressSpace(&dataLayout),
-          global.getSymName(), llvmPtrTy);
-    }
-    if (auto global = mod.template lookupSymbol<mlir::LLVM::GlobalOp>(name)) {
-      // The global may have already been translated to LLVM.
-      return replaceWithAddrOfOrASCast(
-          rewriter, loc, global.getAddrSpace(),
-          fir::factory::getProgramAddressSpace(&dataLayout),
-          global.getSymName(), llvmPtrTy);
-    }
-    // Type info derived types do not have type descriptors since they are the
-    // types defining type descriptors.
-    if (!this->options.ignoreMissingTypeDescriptors &&
-        !fir::NameUniquer::belongsToModule(
-            name, Fortran::semantics::typeInfoBuiltinModule))
-      fir::emitFatalError(
-          loc, "runtime derived type info descriptor was not generated");
-    return rewriter.create<mlir::LLVM::ZeroOp>(loc, llvmPtrTy);
-  }
-
   template <typename ModOpTy>
   mlir::Value populateDescriptor(mlir::Location loc, ModOpTy mod,
                                  fir::BaseBoxType boxTy, mlir::Type inputType,
@@ -1543,7 +1558,8 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
           mlir::Type innerType = fir::unwrapInnerType(inputType);
           if (innerType && mlir::isa<fir::RecordType>(innerType)) {
             auto recTy = mlir::dyn_cast<fir::RecordType>(innerType);
-            typeDesc = getTypeDescriptor(mod, rewriter, loc, recTy);
+            typeDesc =
+                getTypeDescriptor(mod, rewriter, loc, recTy, this->options);
           } else {
             // Unlimited polymorphic type descriptor with no record type. Set
             // type descriptor address to a clean state.
@@ -1551,8 +1567,8 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
                 loc, ::getLlvmPtrType(mod.getContext()));
           }
         } else {
-          typeDesc = getTypeDescriptor(mod, rewriter, loc,
-                                       fir::unwrapIfDerived(boxTy));
+          typeDesc = getTypeDescriptor(
+              mod, rewriter, loc, fir::unwrapIfDerived(boxTy), this->options);
         }
       }
       if (typeDesc)
@@ -3064,26 +3080,10 @@ struct TypeDescOpConversion : public fir::FIROpConversion<fir::TypeDescOp> {
     assert(mlir::isa<fir::RecordType>(inTy) && "expecting fir.type");
     auto recordType = mlir::dyn_cast<fir::RecordType>(inTy);
     auto module = typeDescOp.getOperation()->getParentOfType<mlir::ModuleOp>();
-    std::string typeDescName =
-        this->options.typeDescriptorsRenamedForAssembly
-            ? fir::NameUniquer::getTypeDescriptorAssemblyName(
-                  recordType.getName())
-            : fir::NameUniquer::getTypeDescriptorName(recordType.getName());
-    auto llvmPtrTy = ::getLlvmPtrType(typeDescOp.getContext());
-    if (auto global = module.lookupSymbol<mlir::LLVM::GlobalOp>(typeDescName)) {
-      replaceWithAddrOfOrASCast(rewriter, typeDescOp->getLoc(),
-                                global.getAddrSpace(),
-                                getProgramAddressSpace(rewriter),
-                                global.getSymName(), llvmPtrTy, typeDescOp);
-      return mlir::success();
-    } else if (auto global = module.lookupSymbol<fir::GlobalOp>(typeDescName)) {
-      replaceWithAddrOfOrASCast(rewriter, typeDescOp->getLoc(),
-                                getGlobalAddressSpace(rewriter),
-                                getProgramAddressSpace(rewriter),
-                                global.getSymName(), llvmPtrTy, typeDescOp);
-      return mlir::success();
-    }
-    return mlir::failure();
+    mlir::Value typeDesc = getTypeDescriptor(
+        module, rewriter, typeDescOp.getLoc(), recordType, this->options);
+    rewriter.replaceOp(typeDescOp, typeDesc);
+    return mlir::success();
   }
 };
 
diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
index a352e00649c2..d56458a26ce2 100644
--- a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
+++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Optimizer/CodeGen/FIROpPatterns.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "llvm/Support/Debug.h"
 
@@ -376,14 +377,8 @@ unsigned ConvertFIRToLLVMPattern::getGlobalAddressSpace(
   mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp();
   assert(parentOp != nullptr &&
          "expected insertion block to have parent operation");
-  auto module = mlir::isa<mlir::ModuleOp>(parentOp)
-                    ? mlir::cast<mlir::ModuleOp>(parentOp)
-                    : parentOp->getParentOfType<mlir::ModuleOp>();
-  if (module)
-    if (mlir::Attribute addrSpace =
-            mlir::DataLayout(module).getGlobalMemorySpace())
-      return llvm::cast<mlir::IntegerAttr>(addrSpace).getUInt();
-  return defaultAddressSpace;
+  auto dataLayout = mlir::DataLayout::closest(parentOp);
+  return fir::factory::getGlobalAddressSpace(&dataLayout);
 }
 
 } // namespace fir
diff --git a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
index 1a1d3a8cfb87..22835605ba9d 100644
--- a/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
+++ b/flang/lib/Optimizer/CodeGen/TypeConverter.cpp
@@ -128,8 +128,8 @@ LLVMTypeConverter::LLVMTypeConverter(mlir::ModuleOp module, bool applyTBAA,
                                                   /*isPacked=*/false);
   });
   addConversion([&](mlir::NoneType none) {
-    return mlir::LLVM::LLVMStructType::getLiteral(
-        none.getContext(), std::nullopt, /*isPacked=*/false);
+    return mlir::LLVM::LLVMStructType::getLiteral(none.getContext(), {},
+                                                  /*isPacked=*/false);
   });
   addConversion([&](fir::DummyScopeType dscope) {
     // DummyScopeType values must not have any uses after PreCGRewrite.
diff --git a/flang/lib/Optimizer/Dialect/FIRType.cpp b/flang/lib/Optimizer/Dialect/FIRType.cpp
index 78571f1f4bc2..2ff1d6d945ba 100644
--- a/flang/lib/Optimizer/Dialect/FIRType.cpp
+++ b/flang/lib/Optimizer/Dialect/FIRType.cpp
@@ -261,6 +261,8 @@ mlir::Type dyn_cast_ptrOrBoxEleTy(mlir::Type t) {
 }
 
 static bool hasDynamicSize(fir::RecordType recTy) {
+  if (recTy.getLenParamList().empty())
+    return false;
   for (auto field : recTy.getTypeList()) {
     if (auto arr = mlir::dyn_cast<fir::SequenceType>(field.second)) {
       if (sequenceWithNonConstantShape(arr))
@@ -1006,7 +1008,7 @@ mlir::Type fir::RecordType::parse(mlir::AsmParser &parser) {
     return {};
   RecordType result = RecordType::get(parser.getContext(), name);
 
-  RecordType::TypeList lenParamList;
+  RecordType::TypeVector lenParamList;
   if (!parser.parseOptionalLParen()) {
     while (true) {
       llvm::StringRef lenparam;
@@ -1024,7 +1026,7 @@ mlir::Type fir::RecordType::parse(mlir::AsmParser &parser) {
       return {};
   }
 
-  RecordType::TypeList typeList;
+  RecordType::TypeVector typeList;
   if (!parser.parseOptionalLess()) {
     result.pack(true);
   }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index f7efaa736a27..33f687db08f9 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -326,11 +326,14 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
       auto genHlfirBox = [&]() -> mlir::Value {
         if (auto baseBoxType =
                 mlir::dyn_cast<fir::BaseBoxType>(firBase.getType())) {
-          // Rebox so that lower bounds are correct.
+          // Rebox so that lower bounds and attributes are correct.
           if (baseBoxType.isAssumedRank())
             return builder.create<fir::ReboxAssumedRankOp>(
                 loc, hlfirBaseType, firBase,
                 fir::LowerBoundModifierAttribute::SetToOnes);
+          if (!fir::extractSequenceType(baseBoxType.getEleTy()) &&
+              baseBoxType == hlfirBaseType)
+            return firBase;
           return builder.create<fir::ReboxOp>(loc, hlfirBaseType, firBase,
                                               declareOp.getShape(),
                                               /*slice=*/mlir::Value{});
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index 91df8672c20d..54892ef99bf5 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/HLFIR/HLFIROps.h"
 #include "flang/Optimizer/HLFIR/Passes.h"
 #include "flang/Optimizer/OpenMP/Passes.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/Dominance.h"
@@ -786,13 +787,55 @@ llvm::LogicalResult BroadcastAssignBufferization::matchAndRewrite(
   mlir::Value shape = hlfir::genShape(loc, builder, lhs);
   llvm::SmallVector<mlir::Value> extents =
       hlfir::getIndexExtents(loc, builder, shape);
-  hlfir::LoopNest loopNest =
-      hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
-                         flangomp::shouldUseWorkshareLowering(assign));
-  builder.setInsertionPointToStart(loopNest.body);
-  auto arrayElement =
-      hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
-  builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+
+  if (lhs.isSimplyContiguous() && extents.size() > 1) {
+    // Flatten the array to use a single assign loop, that can be better
+    // optimized.
+    mlir::Value n = extents[0];
+    for (size_t i = 1; i < extents.size(); ++i)
+      n = builder.create<mlir::arith::MulIOp>(loc, n, extents[i]);
+    llvm::SmallVector<mlir::Value> flatExtents = {n};
+
+    mlir::Type flatArrayType;
+    mlir::Value flatArray = lhs.getBase();
+    if (mlir::isa<fir::BoxType>(lhs.getType())) {
+      shape = builder.genShape(loc, flatExtents);
+      flatArrayType = fir::BoxType::get(fir::SequenceType::get(eleTy, 1));
+      flatArray = builder.create<fir::ReboxOp>(loc, flatArrayType, flatArray,
+                                               shape, /*slice=*/mlir::Value{});
+    } else {
+      // Array references must have fixed shape, when used in assignments.
+      int64_t flatExtent = 1;
+      for (const mlir::Value &extent : extents) {
+        mlir::Operation *op = extent.getDefiningOp();
+        assert(op && "no defining operation for constant array extent");
+        flatExtent *= fir::toInt(mlir::cast<mlir::arith::ConstantOp>(*op));
+      }
+
+      flatArrayType =
+          fir::ReferenceType::get(fir::SequenceType::get({flatExtent}, eleTy));
+      flatArray = builder.createConvert(loc, flatArrayType, flatArray);
+    }
+
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, flatExtents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+
+    mlir::Value arrayElement =
+        builder.create<hlfir::DesignateOp>(loc, fir::ReferenceType::get(eleTy),
+                                           flatArray, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  } else {
+    hlfir::LoopNest loopNest =
+        hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
+                           flangomp::shouldUseWorkshareLowering(assign));
+    builder.setInsertionPointToStart(loopNest.body);
+    auto arrayElement =
+        hlfir::getElementAt(loc, builder, lhs, loopNest.oneBasedIndices);
+    builder.create<hlfir::AssignOp>(loc, rhs, arrayElement);
+  }
+
   rewriter.eraseOp(assign);
   return mlir::success();
 }
diff --git a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
index 673d7e86c7ba..317a41a2129c 100644
--- a/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/FIROpenACCTypeInterfaces.cpp
@@ -306,6 +306,10 @@ static bool isArrayLike(mlir::Type type) {
 }
 
 static bool isCompositeLike(mlir::Type type) {
+  // class(*) is not a composite type since it does not have a determined type.
+  if (fir::isUnlimitedPolymorphicType(type))
+    return false;
+
   return mlir::isa<fir::RecordType, fir::ClassType, mlir::TupleType>(type);
 }
 
@@ -320,8 +324,18 @@ template <>
 mlir::acc::VariableTypeCategory
 OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
                                                         mlir::Value var) const {
+  // Class-type does not behave like a normal box because it does not hold an
+  // element type. Thus special handle it here.
+  if (mlir::isa<fir::ClassType>(type)) {
+    // class(*) is not a composite type since it does not have a determined
+    // type.
+    if (fir::isUnlimitedPolymorphicType(type))
+      return mlir::acc::VariableTypeCategory::uncategorized;
+    return mlir::acc::VariableTypeCategory::composite;
+  }
 
   mlir::Type eleTy = fir::dyn_cast_ptrOrBoxEleTy(type);
+  assert(eleTy && "expect to be able to unwrap the element type");
 
   // If the type enclosed by the box is a mappable type, then have it
   // provide the type category.
@@ -346,7 +360,7 @@ OpenACCMappableModel<fir::BaseBoxType>::getTypeCategory(mlir::Type type,
   return mlir::acc::VariableTypeCategory::nonscalar;
 }
 
-static mlir::TypedValue<mlir::acc::PointerLikeType>
+static mlir::Value
 getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
   // If there is no defining op - the unwrapped reference is the base one.
   mlir::Operation *op = varPtr.getDefiningOp();
@@ -372,7 +386,7 @@ getBaseRef(mlir::TypedValue<mlir::acc::PointerLikeType> varPtr) {
           })
           .Default([&](mlir::Operation *) { return varPtr; });
 
-  return mlir::cast<mlir::TypedValue<mlir::acc::PointerLikeType>>(baseRef);
+  return baseRef;
 }
 
 static mlir::acc::VariableTypeCategory
@@ -384,10 +398,17 @@ categorizePointee(mlir::Type pointer,
   // value would both be represented as !fir.ref<f32>. We do not want to treat
   // such a reference as a scalar. Thus unwrap interior pointer calculations.
   auto baseRef = getBaseRef(varPtr);
-  mlir::Type eleTy = baseRef.getType().getElementType();
 
-  if (auto mappableTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
-    return mappableTy.getTypeCategory(varPtr);
+  if (auto mappableTy =
+          mlir::dyn_cast<mlir::acc::MappableType>(baseRef.getType()))
+    return mappableTy.getTypeCategory(baseRef);
+
+  // It must be a pointer-like type since it is not a MappableType.
+  auto ptrLikeTy = mlir::cast<mlir::acc::PointerLikeType>(baseRef.getType());
+  mlir::Type eleTy = ptrLikeTy.getElementType();
+
+  if (auto mappableEleTy = mlir::dyn_cast<mlir::acc::MappableType>(eleTy))
+    return mappableEleTy.getTypeCategory(varPtr);
 
   if (isScalarLike(eleTy))
     return mlir::acc::VariableTypeCategory::scalar;
@@ -397,8 +418,12 @@ categorizePointee(mlir::Type pointer,
     return mlir::acc::VariableTypeCategory::composite;
   if (mlir::isa<fir::CharacterType, mlir::FunctionType>(eleTy))
     return mlir::acc::VariableTypeCategory::nonscalar;
+  // Assumed-type (type(*))does not have a determined type that can be
+  // categorized.
+  if (mlir::isa<mlir::NoneType>(eleTy))
+    return mlir::acc::VariableTypeCategory::uncategorized;
   // "pointers" - in the sense of raw address point-of-view, are considered
-  // scalars. However
+  // scalars.
   if (mlir::isa<fir::LLVMPointerType>(eleTy))
     return mlir::acc::VariableTypeCategory::scalar;
 
diff --git a/flang/lib/Optimizer/Passes/CommandLineOpts.cpp b/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
index f95a280883cb..014237542f24 100644
--- a/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
+++ b/flang/lib/Optimizer/Passes/CommandLineOpts.cpp
@@ -39,6 +39,12 @@ cl::opt<bool> ignoreMissingTypeDescriptors(
              "translating FIR to LLVM"),
     cl::init(false), cl::Hidden);
 
+cl::opt<bool> skipExternalRttiDefinition(
+    "skip-external-rtti-definition", llvm::cl::init(false),
+    llvm::cl::desc("do not define rtti static objects for types belonging to "
+                   "other compilation units"),
+    cl::Hidden);
+
 OptimizationLevel defaultOptLevel{OptimizationLevel::O0};
 
 codegenoptions::DebugInfoKind noDebugInfo{codegenoptions::NoDebugInfo};
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index 168eddd909c0..bf8eff7fe12f 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -108,6 +108,7 @@ void addFIRToLLVMPass(mlir::PassManager &pm,
                       const MLIRToLLVMPassPipelineConfig &config) {
   fir::FIRToLLVMPassOptions options;
   options.ignoreMissingTypeDescriptors = ignoreMissingTypeDescriptors;
+  options.skipExternalRttiDefinition = skipExternalRttiDefinition;
   options.applyTBAA = config.AliasAnalysis;
   options.forceUnifiedTBAATree = useOldAliasTags;
   options.typeDescriptorsRenamedForAssembly =
diff --git a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
index 5cfbdc33285f..4e788f75d9d7 100644
--- a/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
+++ b/flang/lib/Optimizer/Transforms/AddAliasTags.cpp
@@ -73,7 +73,12 @@ class PassState {
   }
 
   void processFunctionScopes(mlir::func::FuncOp func);
-  fir::DummyScopeOp getDeclarationScope(fir::DeclareOp declareOp);
+  // For the given fir.declare returns the dominating fir.dummy_scope
+  // operation.
+  fir::DummyScopeOp getDeclarationScope(fir::DeclareOp declareOp) const;
+  // For the given fir.declare returns the outermost fir.dummy_scope
+  // in the current function.
+  fir::DummyScopeOp getOutermostScope(fir::DeclareOp declareOp) const;
 
 private:
   mlir::DominanceInfo &domInfo;
@@ -119,9 +124,8 @@ void PassState::processFunctionScopes(mlir::func::FuncOp func) {
   }
 }
 
-// For the given fir.declare returns the dominating fir.dummy_scope
-// operation.
-fir::DummyScopeOp PassState::getDeclarationScope(fir::DeclareOp declareOp) {
+fir::DummyScopeOp
+PassState::getDeclarationScope(fir::DeclareOp declareOp) const {
   auto func = declareOp->getParentOfType<mlir::func::FuncOp>();
   assert(func && "fir.declare does not have parent func.func");
   auto &scopeOps = sortedScopeOperations.at(func);
@@ -132,6 +136,15 @@ fir::DummyScopeOp PassState::getDeclarationScope(fir::DeclareOp declareOp) {
   return nullptr;
 }
 
+fir::DummyScopeOp PassState::getOutermostScope(fir::DeclareOp declareOp) const {
+  auto func = declareOp->getParentOfType<mlir::func::FuncOp>();
+  assert(func && "fir.declare does not have parent func.func");
+  auto &scopeOps = sortedScopeOperations.at(func);
+  if (!scopeOps.empty())
+    return scopeOps[0];
+  return nullptr;
+}
+
 class AddAliasTagsPass : public fir::impl::AddAliasTagsBase<AddAliasTagsPass> {
 public:
   void runOnOperation() override;
@@ -279,6 +292,15 @@ void AddAliasTagsPass::runOnAliasInterface(fir::FirAliasTagOpInterface op,
     else
       unknownAllocOp = true;
 
+    if (auto declOp = source.origin.instantiationPoint) {
+      // Use the outermost scope for local allocations,
+      // because using the innermost scope may result
+      // in incorrect TBAA, when calls are inlined in MLIR.
+      auto declareOp = mlir::dyn_cast<fir::DeclareOp>(declOp);
+      assert(declareOp && "Instantiation point must be fir.declare");
+      scopeOp = state.getOutermostScope(declareOp);
+    }
+
     if (unknownAllocOp) {
       LLVM_DEBUG(llvm::dbgs().indent(2)
                  << "WARN: unknown defining op for SourceKind::Allocate " << *op
diff --git a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 8fa2f38818c0..6eb914e67fd5 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -35,6 +35,7 @@
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -104,6 +105,37 @@ bool debugInfoIsAlreadySet(mlir::Location loc) {
   return false;
 }
 
+// Generates the name for the artificial DISubprogram that we are going to
+// generate for omp::TargetOp. Its logic is borrowed from
+// getTargetEntryUniqueInfo and
+// TargetRegionEntryInfo::getTargetRegionEntryFnName to generate the same name.
+// But even if there was a slight mismatch, it is not a problem because this
+// name is artificial and not important to debug experience.
+mlir::StringAttr getTargetFunctionName(mlir::MLIRContext *context,
+                                       mlir::Location Loc,
+                                       llvm::StringRef parentName) {
+  auto fileLoc = Loc->findInstanceOf<mlir::FileLineColLoc>();
+
+  assert(fileLoc && "No file found from location");
+  llvm::StringRef fileName = fileLoc.getFilename().getValue();
+
+  llvm::sys::fs::UniqueID id;
+  uint64_t line = fileLoc.getLine();
+  size_t fileId;
+  size_t deviceId;
+  if (auto ec = llvm::sys::fs::getUniqueID(fileName, id)) {
+    fileId = llvm::hash_value(fileName.str());
+    deviceId = 0xdeadf17e;
+  } else {
+    fileId = id.getFile();
+    deviceId = id.getDevice();
+  }
+  return mlir::StringAttr::get(
+      context,
+      std::string(llvm::formatv("__omp_offloading_{0:x-}_{1:x-}_{2}_l{3}",
+                                deviceId, fileId, parentName, line)));
+}
+
 } // namespace
 
 bool AddDebugInfoPass::createCommonBlockGlobal(
@@ -446,6 +478,79 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
                                   line - 1, false);
   }
 
+  auto addTargetOpDISP = [&](bool lineTableOnly,
+                             llvm::ArrayRef<mlir::LLVM::DINodeAttr> entities) {
+    // When we process the DeclareOp inside the OpenMP target region, all the
+    // variables get the DISubprogram of the parent function of the target op as
+    // the scope. In the codegen (to llvm ir), OpenMP target op results in the
+    // creation of a separate function. As the variables in the debug info have
+    // the DISubprogram of the parent function as the scope, the variables
+    // need to be updated at codegen time to avoid verification failures.
+
+    // This updating after the fact becomes more and more difficult when types
+    // are dependent on local variables like in the case of variable size arrays
+    // or string. We not only have to generate new variables but also new types.
+    // We can avoid this problem by generating a DISubprogramAttr here for the
+    // target op and make sure that all the variables inside the target region
+    // get the correct scope in the first place.
+    funcOp.walk([&](mlir::omp::TargetOp targetOp) {
+      unsigned line = getLineFromLoc(targetOp.getLoc());
+      mlir::StringAttr name =
+          getTargetFunctionName(context, targetOp.getLoc(), funcOp.getName());
+      mlir::LLVM::DISubprogramFlags flags =
+          mlir::LLVM::DISubprogramFlags::Definition |
+          mlir::LLVM::DISubprogramFlags::LocalToUnit;
+      if (isOptimized)
+        flags = flags | mlir::LLVM::DISubprogramFlags::Optimized;
+
+      mlir::DistinctAttr id =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      llvm::SmallVector<mlir::LLVM::DITypeAttr> types;
+      types.push_back(mlir::LLVM::DINullTypeAttr::get(context));
+      for (auto arg : targetOp.getRegion().getArguments()) {
+        auto tyAttr = typeGen.convertType(fir::unwrapRefType(arg.getType()),
+                                          fileAttr, cuAttr, /*declOp=*/nullptr);
+        types.push_back(tyAttr);
+      }
+      CC = llvm::dwarf::getCallingConvention("DW_CC_normal");
+      mlir::LLVM::DISubroutineTypeAttr spTy =
+          mlir::LLVM::DISubroutineTypeAttr::get(context, CC, types);
+      if (lineTableOnly) {
+        auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+            context, id, compilationUnit, Scope, name, name, funcFileAttr, line,
+            line, flags, spTy, /*retainedNodes=*/{}, /*annotations=*/{});
+        targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+        return;
+      }
+      mlir::DistinctAttr recId =
+          mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      auto spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/true, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, /*retainedNodes=*/{},
+          /*annotations=*/{});
+
+      // Make sure that information about the imported modules is copied in the
+      // new function.
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> opEntities;
+      for (mlir::LLVM::DINodeAttr N : entities) {
+        if (auto entity = mlir::dyn_cast<mlir::LLVM::DIImportedEntityAttr>(N)) {
+          auto importedEntity = mlir::LLVM::DIImportedEntityAttr::get(
+              context, llvm::dwarf::DW_TAG_imported_module, spAttr,
+              entity.getEntity(), fileAttr, /*line=*/1, /*name=*/nullptr,
+              /*elements*/ {});
+          opEntities.push_back(importedEntity);
+        }
+      }
+
+      id = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+      spAttr = mlir::LLVM::DISubprogramAttr::get(
+          context, recId, /*isRecSelf=*/false, id, compilationUnit, Scope, name,
+          name, funcFileAttr, line, line, flags, spTy, opEntities,
+          /*annotations=*/{});
+      targetOp->setLoc(builder.getFusedLoc({targetOp.getLoc()}, spAttr));
+    });
+  };
+
   // Don't process variables if user asked for line tables only.
   if (debugLevel == mlir::LLVM::DIEmissionKind::LineTablesOnly) {
     auto spAttr = mlir::LLVM::DISubprogramAttr::get(
@@ -453,6 +558,7 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
         line, line, subprogramFlags, subTypeAttr, /*retainedNodes=*/{},
         /*annotations=*/{});
     funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+    addTargetOpDISP(/*lineTableOnly=*/true, /*entities=*/{});
     return;
   }
 
@@ -510,9 +616,18 @@ void AddDebugInfoPass::handleFuncOp(mlir::func::FuncOp funcOp,
       funcName, fullName, funcFileAttr, line, line, subprogramFlags,
       subTypeAttr, entities, /*annotations=*/{});
   funcOp->setLoc(builder.getFusedLoc({l}, spAttr));
+  addTargetOpDISP(/*lineTableOnly=*/false, entities);
 
   funcOp.walk([&](fir::cg::XDeclareOp declOp) {
-    handleDeclareOp(declOp, fileAttr, spAttr, typeGen, symbolTable);
+    mlir::LLVM::DISubprogramAttr spTy = spAttr;
+    if (auto tOp = declOp->getParentOfType<mlir::omp::TargetOp>()) {
+      if (auto fusedLoc = llvm::dyn_cast<mlir::FusedLoc>(tOp.getLoc())) {
+        if (auto sp = llvm::dyn_cast<mlir::LLVM::DISubprogramAttr>(
+                fusedLoc.getMetadata()))
+          spTy = sp;
+      }
+    }
+    handleDeclareOp(declOp, fileAttr, spTy, typeGen, symbolTable);
   });
   // commonBlockMap ensures that we don't create multiple DICommonBlockAttr of
   // the same name in one function. But it is ok (rather required) to create
diff --git a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
index 328e2374115b..bfb0daeacb8c 100644
--- a/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFDeviceGlobal.cpp
@@ -113,8 +113,16 @@ class CUFDeviceGlobal : public fir::impl::CUFDeviceGlobalBase<CUFDeviceGlobal> {
       return signalPassFailure();
     mlir::SymbolTable gpuSymTable(gpuMod);
     for (auto globalOp : mod.getOps<fir::GlobalOp>()) {
-      if (cuf::isRegisteredDeviceGlobal(globalOp))
+      if (cuf::isRegisteredDeviceGlobal(globalOp)) {
         candidates.insert(globalOp);
+      } else if (globalOp.getConstant() &&
+                 mlir::isa<fir::SequenceType>(
+                     fir::unwrapRefType(globalOp.resultType()))) {
+        mlir::Attribute initAttr =
+            globalOp.getInitVal().value_or(mlir::Attribute());
+        if (initAttr && mlir::dyn_cast<mlir::DenseElementsAttr>(initAttr))
+          candidates.insert(globalOp);
+      }
     }
     for (auto globalOp : candidates) {
       auto globalName{globalOp.getSymbol().getValue()};
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index cdd30dce183d..abad500d3f65 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -48,8 +48,7 @@ DebugTypeGenerator::DebugTypeGenerator(mlir::ModuleOp m,
                                        mlir::SymbolTable *symbolTable_,
                                        const mlir::DataLayout &dl)
     : module(m), symbolTable(symbolTable_), dataLayout{&dl},
-      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl),
-      derivedTypeDepth(0) {
+      kindMapping(getKindMapping(m)), llvmTypeConverter(m, false, false, dl) {
   LLVM_DEBUG(llvm::dbgs() << "DITypeAttr generator\n");
 
   mlir::MLIRContext *context = module.getContext();
@@ -272,31 +271,127 @@ DebugTypeGenerator::getFieldSizeAndAlign(mlir::Type fieldTy) {
   return std::pair{byteSize, byteAlign};
 }
 
+mlir::LLVM::DITypeAttr DerivedTypeCache::lookup(mlir::Type type) {
+  auto iter = typeCache.find(type);
+  if (iter != typeCache.end()) {
+    if (iter->second.first) {
+      componentActiveRecursionLevels = iter->second.second;
+    }
+    return iter->second.first;
+  }
+  return nullptr;
+}
+
+DerivedTypeCache::ActiveLevels
+DerivedTypeCache::startTranslating(mlir::Type type,
+                                   mlir::LLVM::DITypeAttr placeHolder) {
+  derivedTypeDepth++;
+  if (!placeHolder)
+    return {};
+  typeCache[type] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      placeHolder, {derivedTypeDepth});
+  return {};
+}
+
+void DerivedTypeCache::preComponentVisitUpdate() {
+  componentActiveRecursionLevels.clear();
+}
+
+void DerivedTypeCache::postComponentVisitUpdate(
+    ActiveLevels &activeRecursionLevels) {
+  if (componentActiveRecursionLevels.empty())
+    return;
+  ActiveLevels oldLevels;
+  oldLevels.swap(activeRecursionLevels);
+  std::set_union(componentActiveRecursionLevels.begin(),
+                 componentActiveRecursionLevels.end(), oldLevels.begin(),
+                 oldLevels.end(), std::back_inserter(activeRecursionLevels));
+}
+
+void DerivedTypeCache::finalize(mlir::Type ty, mlir::LLVM::DITypeAttr attr,
+                                ActiveLevels &&activeRecursionLevels) {
+  // If there is no nested recursion or if this type does not point to any type
+  // nodes above it, it is safe to cache it indefinitely (it can be used in any
+  // contexts).
+  if (activeRecursionLevels.empty() ||
+      (activeRecursionLevels[0] == derivedTypeDepth)) {
+    typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(attr, {});
+    componentActiveRecursionLevels.clear();
+    cleanUpCache(derivedTypeDepth);
+    --derivedTypeDepth;
+    return;
+  }
+  // Trim any recursion below the current type.
+  if (activeRecursionLevels.back() >= derivedTypeDepth) {
+    auto last = llvm::find_if(activeRecursionLevels, [&](std::int32_t depth) {
+      return depth >= derivedTypeDepth;
+    });
+    if (last != activeRecursionLevels.end()) {
+      activeRecursionLevels.erase(last, activeRecursionLevels.end());
+    }
+  }
+  componentActiveRecursionLevels = std::move(activeRecursionLevels);
+  typeCache[ty] = std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>(
+      attr, componentActiveRecursionLevels);
+  cleanUpCache(derivedTypeDepth);
+  if (!componentActiveRecursionLevels.empty())
+    insertCacheCleanUp(ty, componentActiveRecursionLevels.back());
+  --derivedTypeDepth;
+}
+
+void DerivedTypeCache::insertCacheCleanUp(mlir::Type type, int32_t depth) {
+  auto iter = llvm::find_if(cacheCleanupList,
+                            [&](const auto &x) { return x.second >= depth; });
+  if (iter == cacheCleanupList.end()) {
+    cacheCleanupList.emplace_back(
+        std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+    return;
+  }
+  if (iter->second == depth) {
+    iter->first.push_back(type);
+    return;
+  }
+  cacheCleanupList.insert(
+      iter, std::pair<llvm::SmallVector<mlir::Type>, int32_t>({type}, depth));
+}
+
+void DerivedTypeCache::cleanUpCache(int32_t depth) {
+  if (cacheCleanupList.empty())
+    return;
+  // cleanups are done in the post actions when visiting a derived type
+  // tree. So if there is a clean-up for the current depth, it has to be
+  // the last one (deeper ones must have been done already).
+  if (cacheCleanupList.back().second == depth) {
+    for (mlir::Type type : cacheCleanupList.back().first)
+      typeCache[type].first = nullptr;
+    cacheCleanupList.pop_back_n(1);
+  }
+}
+
 mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     fir::RecordType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
-  // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
 
-  bool canCacheThisType = true;
-  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
   mlir::MLIRContext *context = module.getContext();
-  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto [nameKind, sourceName] = fir::NameUniquer::deconstruct(Ty.getName());
+  if (nameKind != fir::NameUniquer::NameKind::DERIVED_TYPE)
+    return genPlaceholderType(context);
+
+  llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   // Generate a place holder TypeAttr which will be used if a member
   // references the parent type.
-  auto comAttr = mlir::LLVM::DICompositeTypeAttr::get(
+  auto recId = mlir::DistinctAttr::create(mlir::UnitAttr::get(context));
+  auto placeHolder = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/true, llvm::dwarf::DW_TAG_structure_type,
       mlir::StringAttr::get(context, ""), fileAttr, /*line=*/0, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = comAttr;
-
-  auto result = fir::NameUniquer::deconstruct(Ty.getName());
-  if (result.first != fir::NameUniquer::NameKind::DERIVED_TYPE)
-    return genPlaceholderType(context);
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty, placeHolder);
 
   fir::TypeInfoOp tiOp = symbolTable->lookup<fir::TypeInfoOp>(Ty.getName());
   unsigned line = (tiOp) ? getLineFromLoc(tiOp.getLoc()) : 1;
@@ -305,6 +400,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
   mlir::IntegerType intTy = mlir::IntegerType::get(context, 64);
   std::uint64_t offset = 0;
   for (auto [fieldName, fieldTy] : Ty.getTypeList()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     std::optional<llvm::ArrayRef<int64_t>> lowerBounds =
         fir::getComponentLowerBoundsIfNonDefault(Ty, fieldName, module,
@@ -317,7 +413,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
     mlir::LLVM::DITypeAttr elemTy;
     if (lowerBounds && seqTy &&
         lowerBounds->size() == seqTy.getShape().size()) {
-      llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
+      llvm::SmallVector<mlir::LLVM::DINodeAttr> arrayElements;
       for (auto [bound, dim] :
            llvm::zip_equal(*lowerBounds, seqTy.getShape())) {
         auto countAttr = mlir::IntegerAttr::get(intTy, llvm::APInt(64, dim));
@@ -325,14 +421,14 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         auto subrangeTy = mlir::LLVM::DISubrangeAttr::get(
             context, countAttr, lowerAttr, /*upperBound=*/nullptr,
             /*stride=*/nullptr);
-        elements.push_back(subrangeTy);
+        arrayElements.push_back(subrangeTy);
       }
       elemTy = mlir::LLVM::DICompositeTypeAttr::get(
           context, llvm::dwarf::DW_TAG_array_type, /*name=*/nullptr,
           /*file=*/nullptr, /*line=*/0, /*scope=*/nullptr,
           convertType(seqTy.getEleTy(), fileAttr, scope, declOp),
           mlir::LLVM::DIFlags::Zero, /*sizeInBits=*/0, /*alignInBits=*/0,
-          elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
+          arrayElements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
           /*allocated=*/nullptr, /*associated=*/nullptr);
     } else
       elemTy = convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -344,80 +440,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertRecordType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
-
-    // Currently, the handling of recursive debug type in mlir has some
-    // limitations that were discussed at the end of the thread for following
-    // PR.
-    // https://github.com/llvm/llvm-project/pull/106571
-    //
-    // Problem could be explained with the following example code:
-    //  type t2
-    //   type(t1), pointer :: p1
-    // end type
-    // type t1
-    //   type(t2), pointer :: p2
-    // end type
-    // In the description below, type_self means a temporary type that is
-    // generated
-    // as a place holder while the members of that type are being processed.
-    //
-    // If we process t1 first then we will have the following structure after
-    // it has been processed.
-    // t1 -> t2 -> t1_self
-    // This is because when we started processing t2, we did not have the
-    // complete t1 but its place holder t1_self.
-    // Now if some entity requires t2, we will already have that in cache and
-    // will return it. But this t2 refers to t1_self and not to t1. In mlir
-    // handling, only those types are allowed to have _self reference which are
-    // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
-    // because the t1_self reference can be resolved by the outer t1. But
-    // standalone t2 is not because there will be no way to resolve it. Until
-    // this is fixed in mlir, we avoid caching such types. Please see
-    // DebugTranslation::translateRecursive for details on how mlir handles
-    // recursive types.
-    // The code below checks for situation where it will be unsafe to cache
-    // a type to avoid this problem. We do that in 2 situations.
-    // 1. If a member is record type, then its type would have been processed
-    // before reaching here. If it is not in the cache, it means that it was
-    // found to be unsafe to cache. So any type containing it will also not
-    // be cached
-    // 2. The type of the member is found in the cache but it is a place holder.
-    // In this case, its recID should match the recID of the type we are
-    // processing. This helps us to cache the following type.
-    // type t
-    //  type(t), allocatable :: p
-    // end type
-    mlir::Type baseTy = getDerivedType(fieldTy);
-    if (auto recTy = mlir::dyn_cast<fir::RecordType>(baseTy)) {
-      auto iter = typeCache.find(recTy);
-      if (iter == typeCache.end())
-        canCacheThisType = false;
-      else {
-        if (auto tyAttr =
-                mlir::dyn_cast<mlir::LLVM::DICompositeTypeAttr>(iter->second)) {
-          if (tyAttr.getIsRecSelf() && tyAttr.getRecId() != recId)
-            canCacheThisType = false;
-        }
-      }
-    }
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto finalAttr = mlir::LLVM::DICompositeTypeAttr::get(
       context, recId, /*isRecSelf=*/false, llvm::dwarf::DW_TAG_structure_type,
-      mlir::StringAttr::get(context, result.second.name), fileAttr, line, scope,
+      mlir::StringAttr::get(context, sourceName.name), fileAttr, line, scope,
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
 
-  // derivedTypeDepth == 1 means that it is a top level type which is safe to
-  // cache.
-  if (canCacheThisType || derivedTypeDepth == 1) {
-    typeCache[Ty] = finalAttr;
-  } else {
-    auto iter = typeCache.find(Ty);
-    if (iter != typeCache.end())
-      typeCache.erase(iter);
-  }
+  derivedTypeCache.finalize(Ty, finalAttr, std::move(nestedRecursions));
+
   return finalAttr;
 }
 
@@ -425,15 +459,18 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
     mlir::TupleType Ty, mlir::LLVM::DIFileAttr fileAttr,
     mlir::LLVM::DIScopeAttr scope, fir::cg::XDeclareOp declOp) {
   // Check if this type has already been converted.
-  auto iter = typeCache.find(Ty);
-  if (iter != typeCache.end())
-    return iter->second;
+  if (mlir::LLVM::DITypeAttr attr = derivedTypeCache.lookup(Ty))
+    return attr;
+
+  DerivedTypeCache::ActiveLevels nestedRecursions =
+      derivedTypeCache.startTranslating(Ty);
 
   llvm::SmallVector<mlir::LLVM::DINodeAttr> elements;
   mlir::MLIRContext *context = module.getContext();
 
   std::uint64_t offset = 0;
   for (auto fieldTy : Ty.getTypes()) {
+    derivedTypeCache.preComponentVisitUpdate();
     auto [byteSize, byteAlign] = getFieldSizeAndAlign(fieldTy);
     mlir::LLVM::DITypeAttr elemTy =
         convertType(fieldTy, fileAttr, scope, /*declOp=*/nullptr);
@@ -445,6 +482,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
         /*extra data=*/nullptr);
     elements.push_back(tyAttr);
     offset += llvm::alignTo(byteSize, byteAlign);
+    derivedTypeCache.postComponentVisitUpdate(nestedRecursions);
   }
 
   auto typeAttr = mlir::LLVM::DICompositeTypeAttr::get(
@@ -453,7 +491,7 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertTupleType(
       /*baseType=*/nullptr, mlir::LLVM::DIFlags::Zero, offset * 8,
       /*alignInBits=*/0, elements, /*dataLocation=*/nullptr, /*rank=*/nullptr,
       /*allocated=*/nullptr, /*associated=*/nullptr);
-  typeCache[Ty] = typeAttr;
+  derivedTypeCache.finalize(Ty, typeAttr, std::move(nestedRecursions));
   return typeAttr;
 }
 
@@ -667,27 +705,7 @@ DebugTypeGenerator::convertType(mlir::Type Ty, mlir::LLVM::DIFileAttr fileAttr,
     return convertCharacterType(charTy, fileAttr, scope, declOp,
                                 /*hasDescriptor=*/false);
   } else if (auto recTy = mlir::dyn_cast_if_present<fir::RecordType>(Ty)) {
-    // For nested derived types like shown below, the call sequence of the
-    // convertRecordType will look something like as follows:
-    // convertRecordType (t1)
-    //  convertRecordType (t2)
-    //    convertRecordType (t3)
-    // We need to recognize when we are processing the top level type like t1
-    // to make caching decision. The variable `derivedTypeDepth` is used for
-    // this purpose and maintains the current depth of derived type processing.
-    //  type t1
-    //   type(t2), pointer :: p1
-    // end type
-    // type t2
-    //   type(t3), pointer :: p2
-    // end type
-    // type t2
-    //   integer a
-    // end type
-    derivedTypeDepth++;
-    auto result = convertRecordType(recTy, fileAttr, scope, declOp);
-    derivedTypeDepth--;
-    return result;
+    return convertRecordType(recTy, fileAttr, scope, declOp);
   } else if (auto tupleTy = mlir::dyn_cast_if_present<mlir::TupleType>(Ty)) {
     return convertTupleType(tupleTy, fileAttr, scope, declOp);
   } else if (auto refTy = mlir::dyn_cast_if_present<fir::ReferenceType>(Ty)) {
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
index 93b9ac2d90fd..854e4397ca32 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.h
@@ -23,6 +23,77 @@
 
 namespace fir {
 
+/// Special cache to deal with the fact that mlir::LLVM::DITypeAttr for
+/// derived types may only be valid in specific nesting contexts in presence
+/// of derived type recursion and cannot be cached for the whole compilation.
+/// It is however still desirable to cache such mlir::LLVM::DITypeAttr as
+/// long as possible to avoid catastrophic compilation slow downs in very
+/// complex derived types where an intermediate type in a derived type cycle may
+/// indirectly appear hundreds of times under the top type of the derived type
+/// cycle. More details in the comment below.
+class DerivedTypeCache {
+public:
+  // Currently, the handling of recursive debug type in mlir has some
+  // limitations that were discussed at the end of the thread for following
+  // PR.
+  // https://github.com/llvm/llvm-project/pull/106571
+  //
+  // Problem could be explained with the following example code:
+  //  type t2
+  //   type(t1), pointer :: p1
+  // end type
+  // type t1
+  //   type(t2), pointer :: p2
+  // end type
+  // In the description below, type_self means a temporary type that is
+  // generated
+  // as a place holder while the members of that type are being processed.
+  //
+  // If we process t1 first then we will have the following structure after
+  // it has been processed.
+  // t1 -> t2 -> t1_self
+  // This is because when we started processing t2, we did not have the
+  // complete t1 but its place holder t1_self.
+  // Now if some entity requires t2, we will already have that in cache and
+  // will return it. But this t2 refers to t1_self and not to t1. In mlir
+  // handling, only those types are allowed to have _self reference which are
+  // wrapped by entity whose reference it is. So t1 -> t2 -> t1_self is ok
+  // because the t1_self reference can be resolved by the outer t1. But
+  // standalone t2 is not because there will be no way to resolve it. Until
+  // this is fixed in mlir, we avoid caching such types. Please see
+  // DebugTranslation::translateRecursive for details on how mlir handles
+  // recursive types.
+  using ActiveLevels = llvm::SmallVector<int32_t, 1>;
+  mlir::LLVM::DITypeAttr lookup(mlir::Type);
+  ActiveLevels startTranslating(mlir::Type,
+                                mlir::LLVM::DITypeAttr placeHolder = nullptr);
+  void finalize(mlir::Type, mlir::LLVM::DITypeAttr, ActiveLevels &&);
+  void preComponentVisitUpdate();
+  void postComponentVisitUpdate(ActiveLevels &);
+
+private:
+  void insertCacheCleanUp(mlir::Type type, int32_t depth);
+  void cleanUpCache(int32_t depth);
+  // Current depth inside a top level derived type being converted.
+  int32_t derivedTypeDepth = 0;
+  // Cache for already translated derived types with the minimum depth where
+  // this cache entry is valid. Zero means the translation is always valid, "i"
+  // means the type depends its derived type tree parent node at depth "i". Such
+  // types should be cleaned-up from the cache in the post visit of node "i".
+  // Note that any new metadata created for a type with a component in the cache
+  // with validity of "i" shall not be added to the cache with a validity
+  // smaller than "i".
+  llvm::DenseMap<mlir::Type, std::pair<mlir::LLVM::DITypeAttr, ActiveLevels>>
+      typeCache;
+  // List of parent nodes that are being recursively referred to in the
+  // component type that has just been computed.
+  ActiveLevels componentActiveRecursionLevels;
+  // Helper list that maintains the list of nodes that must be deleted from the
+  // cache when going back past listed parent depths.
+  llvm::SmallVector<std::pair<llvm::SmallVector<mlir::Type>, int32_t>>
+      cacheCleanupList;
+};
+
 /// This converts FIR/mlir type to DITypeAttr.
 class DebugTypeGenerator {
 public:
@@ -91,8 +162,7 @@ class DebugTypeGenerator {
   std::uint64_t lenOffset;
   std::uint64_t rankOffset;
   std::uint64_t rankSize;
-  int32_t derivedTypeDepth;
-  llvm::DenseMap<mlir::Type, mlir::LLVM::DITypeAttr> typeCache;
+  DerivedTypeCache derivedTypeCache;
 };
 
 } // namespace fir
diff --git a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
index f9a4c4d0283c..57eae1ff052a 100644
--- a/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
+++ b/flang/lib/Optimizer/Transforms/PolymorphicOpConversion.cpp
@@ -16,7 +16,6 @@
 #include "flang/Optimizer/Dialect/Support/KindMapping.h"
 #include "flang/Optimizer/Support/InternalNames.h"
 #include "flang/Optimizer/Support/TypeCode.h"
-#include "flang/Optimizer/Support/Utils.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "flang/Runtime/derived-api.h"
 #include "flang/Semantics/runtime-type-info.h"
@@ -38,6 +37,45 @@ namespace fir {
 using namespace fir;
 using namespace mlir;
 
+// Reconstruct binding tables for dynamic dispatch.
+using BindingTable = llvm::DenseMap<llvm::StringRef, unsigned>;
+using BindingTables = llvm::DenseMap<llvm::StringRef, BindingTable>;
+
+static std::string getTypeDescriptorTypeName() {
+  llvm::SmallVector<llvm::StringRef, 1> modules = {
+      Fortran::semantics::typeInfoBuiltinModule};
+  return fir::NameUniquer::doType(modules, /*proc=*/{}, /*blockId=*/0,
+                                  Fortran::semantics::typeDescriptorTypeName,
+                                  /*kinds=*/{});
+}
+
+static std::optional<mlir::Type>
+buildBindingTables(BindingTables &bindingTables, mlir::ModuleOp mod) {
+
+  std::optional<mlir::Type> typeDescriptorType;
+  std::string typeDescriptorTypeName = getTypeDescriptorTypeName();
+  // The binding tables are defined in FIR after lowering inside fir.type_info
+  // operations. Go through each binding tables and store the procedure name and
+  // binding index for later use by the fir.dispatch conversion pattern.
+  for (auto typeInfo : mod.getOps<fir::TypeInfoOp>()) {
+    if (!typeDescriptorType && typeInfo.getSymName() == typeDescriptorTypeName)
+      typeDescriptorType = typeInfo.getType();
+    unsigned bindingIdx = 0;
+    BindingTable bindings;
+    if (typeInfo.getDispatchTable().empty()) {
+      bindingTables[typeInfo.getSymName()] = bindings;
+      continue;
+    }
+    for (auto dtEntry :
+         typeInfo.getDispatchTable().front().getOps<fir::DTEntryOp>()) {
+      bindings[dtEntry.getMethod()] = bindingIdx;
+      ++bindingIdx;
+    }
+    bindingTables[typeInfo.getSymName()] = bindings;
+  }
+  return typeDescriptorType;
+}
+
 namespace {
 
 /// SelectTypeOp converted to an if-then-else chain
@@ -77,9 +115,10 @@ class SelectTypeConv : public OpConversionPattern<fir::SelectTypeOp> {
 struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
   using OpConversionPattern<fir::DispatchOp>::OpConversionPattern;
 
-  DispatchOpConv(mlir::MLIRContext *ctx, const BindingTables &bindingTables)
+  DispatchOpConv(mlir::MLIRContext *ctx, const BindingTables &bindingTables,
+                 std::optional<mlir::Type> typeDescriptorType)
       : mlir::OpConversionPattern<fir::DispatchOp>(ctx),
-        bindingTables(bindingTables) {}
+        bindingTables(bindingTables), typeDescriptorType{typeDescriptorType} {}
 
   llvm::LogicalResult
   matchAndRewrite(fir::DispatchOp dispatch, OpAdaptor adaptor,
@@ -111,13 +150,11 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
 
     mlir::Value passedObject = dispatch.getObject();
 
-    auto module = dispatch.getOperation()->getParentOfType<mlir::ModuleOp>();
-    Type typeDescTy;
-    std::string typeDescName =
-        NameUniquer::getTypeDescriptorName(recordType.getName());
-    if (auto global = module.lookupSymbol<fir::GlobalOp>(typeDescName)) {
-      typeDescTy = global.getType();
-    }
+    if (!typeDescriptorType)
+      return emitError(loc) << "cannot find " << getTypeDescriptorTypeName()
+                            << " fir.type_info that is required to get the "
+                               "related builtin type and lower fir.dispatch";
+    mlir::Type typeDescTy = *typeDescriptorType;
 
     // clang-format off
     // Before:
@@ -213,6 +250,7 @@ struct DispatchOpConv : public OpConversionPattern<fir::DispatchOp> {
 
 private:
   BindingTables bindingTables;
+  std::optional<mlir::Type> typeDescriptorType;
 };
 
 /// Convert FIR structured control flow ops to CFG ops.
@@ -229,10 +267,11 @@ class PolymorphicOpConversion
     mlir::RewritePatternSet patterns(context);
 
     BindingTables bindingTables;
-    buildBindingTables(bindingTables, mod);
+    std::optional<mlir::Type> typeDescriptorType =
+        buildBindingTables(bindingTables, mod);
 
     patterns.insert<SelectTypeConv>(context);
-    patterns.insert<DispatchOpConv>(context, bindingTables);
+    patterns.insert<DispatchOpConv>(context, bindingTables, typeDescriptorType);
     mlir::ConversionTarget target(*context);
     target.addLegalDialect<mlir::affine::AffineDialect,
                            mlir::cf::ControlFlowDialect, FIROpsDialect,
@@ -379,16 +418,11 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
   } else if (auto a = mlir::dyn_cast<fir::SubclassAttr>(attr)) {
     // Retrieve the type descriptor from the type guard statement record type.
     assert(mlir::isa<fir::RecordType>(a.getType()) && "expect fir.record type");
-    fir::RecordType recTy = mlir::dyn_cast<fir::RecordType>(a.getType());
-    std::string typeDescName =
-        fir::NameUniquer::getTypeDescriptorName(recTy.getName());
-    auto typeDescGlobal = mod.lookupSymbol<fir::GlobalOp>(typeDescName);
-    auto typeDescAddr = rewriter.create<fir::AddrOfOp>(
-        loc, fir::ReferenceType::get(typeDescGlobal.getType()),
-        typeDescGlobal.getSymbol());
-    mlir::Type typeDescTy = ReferenceType::get(rewriter.getNoneType());
+    mlir::Value typeDescAddr =
+        rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(a.getType()));
+    mlir::Type refNoneType = ReferenceType::get(rewriter.getNoneType());
     mlir::Value typeDesc =
-        rewriter.create<ConvertOp>(loc, typeDescTy, typeDescAddr);
+        rewriter.create<ConvertOp>(loc, refNoneType, typeDescAddr);
 
     // Prepare the selector descriptor for the runtime call.
     mlir::Type descNoneTy = fir::BoxType::get(rewriter.getNoneType());
@@ -406,7 +440,7 @@ llvm::LogicalResult SelectTypeConv::genTypeLadderStep(
                                mlir::UnitAttr::get(rewriter.getContext()));
       callee =
           fir::createFuncOp(rewriter.getUnknownLoc(), mod, fctName,
-                            rewriter.getFunctionType({descNoneTy, typeDescTy},
+                            rewriter.getFunctionType({descNoneTy, refNoneType},
                                                      rewriter.getI1Type()),
                             {runtimeAttr});
     }
@@ -435,20 +469,11 @@ SelectTypeConv::genTypeDescCompare(mlir::Location loc, mlir::Value selector,
                                    mlir::Type ty, mlir::ModuleOp mod,
                                    mlir::PatternRewriter &rewriter) const {
   assert(mlir::isa<fir::RecordType>(ty) && "expect fir.record type");
-  fir::RecordType recTy = mlir::dyn_cast<fir::RecordType>(ty);
-  std::string typeDescName =
-      fir::NameUniquer::getTypeDescriptorName(recTy.getName());
-  auto typeDescGlobal = mod.lookupSymbol<fir::GlobalOp>(typeDescName);
-  if (!typeDescGlobal)
-    return {};
-  auto typeDescAddr = rewriter.create<fir::AddrOfOp>(
-      loc, fir::ReferenceType::get(typeDescGlobal.getType()),
-      typeDescGlobal.getSymbol());
+  mlir::Value typeDescAddr =
+      rewriter.create<fir::TypeDescOp>(loc, mlir::TypeAttr::get(ty));
+  mlir::Value selectorTdescAddr = rewriter.create<fir::BoxTypeDescOp>(
+      loc, typeDescAddr.getType(), selector);
   auto intPtrTy = rewriter.getIndexType();
-  mlir::Type tdescType =
-      fir::TypeDescType::get(mlir::NoneType::get(rewriter.getContext()));
-  mlir::Value selectorTdescAddr =
-      rewriter.create<fir::BoxTypeDescOp>(loc, tdescType, selector);
   auto typeDescInt =
       rewriter.create<fir::ConvertOp>(loc, intPtrTy, typeDescAddr);
   auto selectorTdescInt =
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index 799998c54b53..f2237da3857a 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -273,14 +273,36 @@ static llvm::raw_ostream::Colors PrefixColor(Severity severity) {
   return llvm::raw_ostream::SAVEDCOLOR;
 }
 
+static std::string HintLanguageControlFlag(
+    const common::LanguageFeatureControl *hintFlagPtr,
+    std::optional<common::LanguageFeature> feature,
+    std::optional<common::UsageWarning> warning) {
+  if (hintFlagPtr) {
+    std::string flag;
+    if (warning) {
+      flag = hintFlagPtr->getDefaultCliSpelling(*warning);
+    } else if (feature) {
+      flag = hintFlagPtr->getDefaultCliSpelling(*feature);
+    }
+    if (!flag.empty()) {
+      return " [-W" + flag + "]";
+    }
+  }
+  return "";
+}
+
 static constexpr int MAX_CONTEXTS_EMITTED{2};
 static constexpr bool OMIT_SHARED_CONTEXTS{true};
 
 void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLine) const {
+    bool echoSourceLine,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::optional<ProvenanceRange> provenanceRange{GetProvenanceRange(allCooked)};
   const AllSources &sources{allCooked.allSources()};
-  sources.EmitMessage(o, provenanceRange, ToString(), Prefix(severity()),
+  const std::string text{ToString()};
+  const std::string hint{
+      HintLanguageControlFlag(hintFlagPtr, languageFeature_, usageWarning_)};
+  sources.EmitMessage(o, provenanceRange, text + hint, Prefix(severity()),
       PrefixColor(severity()), echoSourceLine);
   // Refers to whether the attachment in the loop below is a context, but can't
   // be declared inside the loop because the previous iteration's
@@ -430,7 +452,8 @@ void Messages::ResolveProvenances(const AllCookedSources &allCooked) {
 }
 
 void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
-    bool echoSourceLines) const {
+    bool echoSourceLines,
+    const common::LanguageFeatureControl *hintFlagPtr) const {
   std::vector<const Message *> sorted;
   for (const auto &msg : messages_) {
     sorted.push_back(&msg);
@@ -443,7 +466,7 @@ void Messages::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
       // Don't emit two identical messages for the same location
       continue;
     }
-    msg->Emit(o, allCooked, echoSourceLines);
+    msg->Emit(o, allCooked, echoSourceLines, hintFlagPtr);
     lastMsg = msg;
   }
 }
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index c55642d96950..30b5d4f9513a 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -1553,7 +1553,7 @@ static OmpMapperSpecifier ConstructOmpMapperSpecifier(
         std::get<Name>(derived->t).ToString() + llvm::omp::OmpDefaultMapperName,
         std::move(typeSpec), std::move(varName)};
   }
-  return OmpMapperSpecifier{std::string("omp.default.mapper"),
+  return OmpMapperSpecifier{std::string(llvm::omp::OmpDefaultMapperName),
       std::move(typeSpec), std::move(varName)};
 }
 
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
index ef24c704db88..8ef9810463d5 100644
--- a/flang/lib/Parser/preprocessor.cpp
+++ b/flang/lib/Parser/preprocessor.cpp
@@ -1276,16 +1276,20 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
       left = right >= 64 ? 0 : left >> right;
       break;
     case BITAND:
-    case AND:
       left = left & right;
       break;
     case BITXOR:
       left = left ^ right;
       break;
     case BITOR:
-    case OR:
       left = left | right;
       break;
+    case AND:
+      left = left && right;
+      break;
+    case OR:
+      left = left || right;
+      break;
     case LT:
       left = -(left < right);
       break;
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
index 9aef0c9981e3..ec894ab8513d 100644
--- a/flang/lib/Parser/prescan.cpp
+++ b/flang/lib/Parser/prescan.cpp
@@ -593,13 +593,13 @@ bool Prescanner::SkipToNextSignificantCharacter() {
     return false;
   } else {
     auto anyContinuationLine{false};
-    bool mightNeedSpace{false};
+    bool atNewline{false};
     if (MustSkipToEndOfLine()) {
       SkipToEndOfLine();
     } else {
-      mightNeedSpace = *at_ == '\n';
+      atNewline = *at_ == '\n';
     }
-    for (; Continuation(mightNeedSpace); mightNeedSpace = false) {
+    for (; Continuation(atNewline); atNewline = false) {
       anyContinuationLine = true;
       ++continuationLines_;
       if (MustSkipToEndOfLine()) {
@@ -641,7 +641,7 @@ void Prescanner::SkipSpaces() {
   while (IsSpaceOrTab(at_)) {
     NextChar();
   }
-  insertASpace_ = false;
+  brokenToken_ = false;
 }
 
 const char *Prescanner::SkipWhiteSpace(const char *p) {
@@ -745,10 +745,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       }
     }
   }
-  if (insertASpace_) {
-    tokens.PutNextTokenChar(' ', spaceProvenance_);
-    insertASpace_ = false;
-  }
+  brokenToken_ = false;
   if (*at_ == '\n') {
     return false;
   }
@@ -808,7 +805,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     bool anyDefined{false};
     bool hadContinuation{false};
     // Subtlety: When an identifier is split across continuation lines,
-    // its parts are kept as distinct pp-tokens if that macro replacement
+    // its parts are kept as distinct pp-tokens if macro replacement
     // should operate on them independently.  This trick accommodates the
     // historic practice of using line continuation for token pasting after
     // replacement.
@@ -822,6 +819,9 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       ++at_, ++column_;
       hadContinuation = SkipToNextSignificantCharacter();
       if (hadContinuation && IsLegalIdentifierStart(*at_)) {
+        if (brokenToken_) {
+          break;
+        }
         // Continued identifier
         tokens.CloseToken();
         ++parts;
@@ -915,12 +915,21 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
     int startColumn{column_};
     TokenSequence possible;
     EmitCharAndAdvance(possible, *at_);
+    if (InFixedFormSource()) {
+      SkipSpaces();
+    }
     if (*at_ == '+' || *at_ == '-') {
       EmitCharAndAdvance(possible, *at_);
+      if (InFixedFormSource()) {
+        SkipSpaces();
+      }
     }
     if (IsDecimalDigit(*at_)) { // it's an exponent; scan it
       while (IsDecimalDigit(*at_)) {
         EmitCharAndAdvance(possible, *at_);
+        if (InFixedFormSource()) {
+          SkipSpaces();
+        }
       }
       possible.CloseToken();
       tokens.AppendRange(possible, 0); // appends to current token
@@ -940,13 +949,22 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
   TokenSequence withUnderscore, separate;
   EmitChar(withUnderscore, '_');
   EmitCharAndAdvance(separate, '_');
+  if (InFixedFormSource()) {
+    SkipSpaces();
+  }
   if (IsLegalInIdentifier(*at_)) {
     separate.CloseToken();
     EmitChar(withUnderscore, *at_);
     EmitCharAndAdvance(separate, *at_);
+    if (InFixedFormSource()) {
+      SkipSpaces();
+    }
     while (IsLegalInIdentifier(*at_)) {
       EmitChar(withUnderscore, *at_);
       EmitCharAndAdvance(separate, *at_);
+      if (InFixedFormSource()) {
+        SkipSpaces();
+      }
     }
   }
   withUnderscore.CloseToken();
@@ -1330,7 +1348,7 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
   return false;
 }
 
-const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
+const char *Prescanner::FixedFormContinuationLine(bool atNewline) {
   if (IsAtEnd()) {
     return nullptr;
   }
@@ -1363,8 +1381,8 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
       }
       const char *col6{nextLine_ + 5};
       if (*col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        if (mightNeedSpace && !IsSpace(nextLine_ + 6)) {
-          insertASpace_ = true;
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
         }
         return nextLine_ + 6;
       }
@@ -1377,7 +1395,9 @@ const char *Prescanner::FixedFormContinuationLine(bool mightNeedSpace) {
         nextLine_[4] == ' ' && IsCompilerDirectiveSentinel(&nextLine_[1], 1)) {
       if (const char *col6{nextLine_ + 5};
           *col6 != '\n' && *col6 != '0' && !IsSpaceOrTab(col6)) {
-        insertASpace_ |= mightNeedSpace && !IsSpace(nextLine_ + 6);
+        if (atNewline && !IsSpace(nextLine_ + 6)) {
+          brokenToken_ = true;
+        }
         return nextLine_ + 6;
       } else {
         return nullptr;
@@ -1446,7 +1466,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     p = SkipWhiteSpace(p);
     if (*p == '&') {
       if (!ampersand) {
-        insertASpace_ = true;
+        brokenToken_ = true;
       }
       return p + 1;
     } else if (ampersand) {
@@ -1476,7 +1496,7 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
     } else if (p > lineStart && IsSpaceOrTab(p - 1)) {
       --p;
     } else {
-      insertASpace_ = true;
+      brokenToken_ = true;
     }
     return p;
   } else {
@@ -1484,14 +1504,14 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) {
   }
 }
 
-bool Prescanner::FixedFormContinuation(bool mightNeedSpace) {
+bool Prescanner::FixedFormContinuation(bool atNewline) {
   // N.B. We accept '&' as a continuation indicator in fixed form, too,
   // but not in a character literal.
   if (*at_ == '&' && inCharLiteral_) {
     return false;
   }
   do {
-    if (const char *cont{FixedFormContinuationLine(mightNeedSpace)}) {
+    if (const char *cont{FixedFormContinuationLine(atNewline)}) {
       BeginSourceLine(cont);
       column_ = 7;
       NextLine();
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
index ec4c53cf3e0f..f650d548e6ef 100644
--- a/flang/lib/Parser/prescan.h
+++ b/flang/lib/Parser/prescan.h
@@ -203,10 +203,10 @@ class Prescanner {
   std::optional<std::size_t> IsIncludeLine(const char *) const;
   void FortranInclude(const char *quote);
   const char *IsPreprocessorDirectiveLine(const char *) const;
-  const char *FixedFormContinuationLine(bool mightNeedSpace);
+  const char *FixedFormContinuationLine(bool atNewline);
   const char *FreeFormContinuationLine(bool ampersand);
   bool IsImplicitContinuation() const;
-  bool FixedFormContinuation(bool mightNeedSpace);
+  bool FixedFormContinuation(bool atNewline);
   bool FreeFormContinuation();
   bool Continuation(bool mightNeedFixedFormSpace);
   std::optional<LineClassification> IsFixedFormCompilerDirectiveLine(
@@ -256,10 +256,15 @@ class Prescanner {
   bool continuationInCharLiteral_{false};
   bool inPreprocessorDirective_{false};
 
-  // In some edge cases of compiler directive continuation lines, it
-  // is necessary to treat the line break as a space character by
-  // setting this flag, which is cleared by EmitChar().
-  bool insertASpace_{false};
+  // True after processing a continuation that can't be allowed
+  // to appear in the middle of an identifier token, but is fixed form,
+  // or is free form and doesn't have a space character handy to use as
+  // a separator when:
+  // a) (standard) doesn't begin with a leading '&' on the continuation
+  //     line, but has a non-blank in column 1, or
+  // b) (extension) does have a leading '&', but didn't have one
+  //    on the continued line.
+  bool brokenToken_{false};
 
   // When a free form continuation marker (&) appears at the end of a line
   // before a INCLUDE or #include, we delete it and omit the newline, so
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index 40a074eaf0a4..9deb513e4f64 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -30,7 +30,8 @@ void TokenSequence::clear() {
 
 void TokenSequence::pop_back() {
   CHECK(!start_.empty());
-  CHECK(nextStart_ > start_.back());
+  // If the last token is empty then `nextStart_ == start_.back()`.
+  CHECK(nextStart_ >= start_.back());
   std::size_t bytes{nextStart_ - start_.back()};
   nextStart_ = start_.back();
   start_.pop_back();
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index ed0f227fd5b9..dcd1ac165adc 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2094,7 +2094,7 @@ class UnparseVisitor {
   }
   void Unparse(const OmpMapperSpecifier &x) {
     const auto &mapperName{std::get<std::string>(x.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }
@@ -2740,7 +2740,7 @@ class UnparseVisitor {
     Word("!$OMP DECLARE MAPPER (");
     const auto &spec{std::get<OmpMapperSpecifier>(z.t)};
     const auto &mapperName{std::get<std::string>(spec.t)};
-    if (mapperName.find("omp.default.mapper") == std::string::npos) {
+    if (mapperName.find(llvm::omp::OmpDefaultMapperName) == std::string::npos) {
       Walk(mapperName);
       Put(":");
     }
@@ -2926,7 +2926,8 @@ class UnparseVisitor {
     Walk(std::get<OmpBeginLoopDirective>(x.t));
     Put("\n");
     EndOpenMP();
-    Walk(std::get<std::optional<DoConstruct>>(x.t));
+    Walk(std::get<std::optional<std::variant<DoConstruct,
+            common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t));
     Walk(std::get<std::optional<OmpEndLoopDirective>>(x.t));
   }
   void Unparse(const BasedPointer &x) {
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index c0fda3631c01..109bc2dbb856 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -20,6 +20,9 @@ add_flang_library(FortranSemantics
   check-io.cpp
   check-namelist.cpp
   check-nullify.cpp
+  check-omp-atomic.cpp
+  check-omp-loop.cpp
+  check-omp-metadirective.cpp
   check-omp-structure.cpp
   check-purity.cpp
   check-return.cpp
@@ -34,12 +37,13 @@ add_flang_library(FortranSemantics
   mod-file.cpp
   openmp-dsa.cpp
   openmp-modifiers.cpp
+  openmp-utils.cpp
   pointer-assignment.cpp
   program-tree.cpp
-  resolve-labels.cpp
   resolve-directives.cpp
-  resolve-names-utils.cpp
+  resolve-labels.cpp
   resolve-names.cpp
+  resolve-names-utils.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/canonicalize-omp.cpp b/flang/lib/Semantics/canonicalize-omp.cpp
index 5164f1dc6faa..1edcb376596b 100644
--- a/flang/lib/Semantics/canonicalize-omp.cpp
+++ b/flang/lib/Semantics/canonicalize-omp.cpp
@@ -8,6 +8,7 @@
 
 #include "canonicalize-omp.h"
 #include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
 
 // After Loop Canonicalization, rewrite OpenMP parse tree to make OpenMP
 // Constructs more structured which provide explicit scopes for later
@@ -125,6 +126,16 @@ class CanonicalizationOfOmp {
     parser::Block::iterator nextIt;
     auto &beginDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
     auto &dir{std::get<parser::OmpLoopDirective>(beginDir.t)};
+    auto missingDoConstruct = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
+    auto tileUnrollError = [](auto &dir, auto &messages) {
+      messages.Say(dir.source,
+          "If a loop construct has been fully unrolled, it cannot then be tiled"_err_en_US,
+          parser::ToUpperCaseLetters(dir.source.ToString()));
+    };
 
     nextIt = it;
     while (++nextIt != block.end()) {
@@ -135,31 +146,95 @@ class CanonicalizationOfOmp {
       if (auto *doCons{GetConstructIf<parser::DoConstruct>(*nextIt)}) {
         if (doCons->GetLoopControl()) {
           // move DoConstruct
-          std::get<std::optional<parser::DoConstruct>>(x.t) =
+          std::get<std::optional<std::variant<parser::DoConstruct,
+              common::Indirection<parser::OpenMPLoopConstruct>>>>(x.t) =
               std::move(*doCons);
           nextIt = block.erase(nextIt);
           // try to match OmpEndLoopDirective
-          if (nextIt != block.end()) {
-            if (auto *endDir{
-                    GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
-              std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
-                  std::move(*endDir);
-              block.erase(nextIt);
-            }
+          if (auto *endDir{
+                  GetConstructIf<parser::OmpEndLoopDirective>(*nextIt)}) {
+            std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                std::move(*endDir);
+            nextIt = block.erase(nextIt);
           }
         } else {
           messages_.Say(dir.source,
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+      } else if (auto *ompLoopCons{
+                     GetOmpIf<parser::OpenMPLoopConstruct>(*nextIt)}) {
+        // We should allow UNROLL and TILE constructs to be inserted between an
+        // OpenMP Loop Construct and the DO loop itself
+        auto &nestedBeginDirective =
+            std::get<parser::OmpBeginLoopDirective>(ompLoopCons->t);
+        auto &nestedBeginLoopDirective =
+            std::get<parser::OmpLoopDirective>(nestedBeginDirective.t);
+        if ((nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll ||
+                nestedBeginLoopDirective.v ==
+                    llvm::omp::Directive::OMPD_tile) &&
+            !(nestedBeginLoopDirective.v == llvm::omp::Directive::OMPD_unroll &&
+                dir.v == llvm::omp::Directive::OMPD_tile)) {
+          // iterate through the remaining block items to find the end directive
+          // for the unroll/tile directive.
+          parser::Block::iterator endIt;
+          endIt = nextIt;
+          while (endIt != block.end()) {
+            if (auto *endDir{
+                    GetConstructIf<parser::OmpEndLoopDirective>(*endIt)}) {
+              auto &endLoopDirective =
+                  std::get<parser::OmpLoopDirective>(endDir->t);
+              if (endLoopDirective.v == dir.v) {
+                std::get<std::optional<parser::OmpEndLoopDirective>>(x.t) =
+                    std::move(*endDir);
+                endIt = block.erase(endIt);
+                continue;
+              }
+            }
+            ++endIt;
+          }
+          RewriteOpenMPLoopConstruct(*ompLoopCons, block, nextIt);
+          auto &ompLoop = std::get<std::optional<parser::NestedConstruct>>(x.t);
+          ompLoop =
+              std::optional<parser::NestedConstruct>{parser::NestedConstruct{
+                  common::Indirection{std::move(*ompLoopCons)}}};
+          nextIt = block.erase(nextIt);
+        } else if (nestedBeginLoopDirective.v ==
+                llvm::omp::Directive::OMPD_unroll &&
+            dir.v == llvm::omp::Directive::OMPD_tile) {
+          // if a loop has been unrolled, the user can not then tile that loop
+          // as it has been unrolled
+          parser::OmpClauseList &unrollClauseList{
+              std::get<parser::OmpClauseList>(nestedBeginDirective.t)};
+          if (unrollClauseList.v.empty()) {
+            // if the clause list is empty for an unroll construct, we assume
+            // the loop is being fully unrolled
+            tileUnrollError(dir, messages_);
+          } else {
+            // parse the clauses for the unroll directive to find the full
+            // clause
+            for (auto clause{unrollClauseList.v.begin()};
+                clause != unrollClauseList.v.end(); ++clause) {
+              if (clause->Id() == llvm::omp::OMPC_full) {
+                tileUnrollError(dir, messages_);
+              }
+            }
+          }
+        } else {
+          messages_.Say(nestedBeginLoopDirective.source,
+              "Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs"_err_en_US,
+              parser::ToUpperCaseLetters(
+                  nestedBeginLoopDirective.source.ToString()));
+        }
       } else {
-        messages_.Say(dir.source,
-            "A DO loop must follow the %s directive"_err_en_US,
-            parser::ToUpperCaseLetters(dir.source.ToString()));
+        missingDoConstruct(dir, messages_);
       }
       // If we get here, we either found a loop, or issued an error message.
       return;
     }
+    if (nextIt == block.end()) {
+      missingDoConstruct(dir, messages_);
+    }
   }
 
   void RewriteOmpAllocations(parser::ExecutionPart &body) {
diff --git a/flang/lib/Semantics/check-directive-structure.h b/flang/lib/Semantics/check-directive-structure.h
index 4a4893fe805a..b1bf3e550aeb 100644
--- a/flang/lib/Semantics/check-directive-structure.h
+++ b/flang/lib/Semantics/check-directive-structure.h
@@ -76,6 +76,8 @@ template <typename D> class NoBranchingEnforce {
         case llvm::omp::Directive::OMPD_distribute_parallel_for:
         case llvm::omp::Directive::OMPD_distribute_simd:
         case llvm::omp::Directive::OMPD_distribute_parallel_for_simd:
+        case llvm::omp::Directive::OMPD_target_teams_distribute:
+        case llvm::omp::Directive::OMPD_target_teams_distribute_simd:
         case llvm::omp::Directive::OMPD_target_teams_distribute_parallel_do:
         case llvm::omp::Directive::
             OMPD_target_teams_distribute_parallel_do_simd:
diff --git a/flang/lib/Semantics/check-io.cpp b/flang/lib/Semantics/check-io.cpp
index 10b32d9af0f8..a1ff4b922268 100644
--- a/flang/lib/Semantics/check-io.cpp
+++ b/flang/lib/Semantics/check-io.cpp
@@ -478,6 +478,8 @@ void IoChecker::Enter(const parser::InquireSpec::LogVar &spec) {
     specKind = IoSpecKind::Pending;
     break;
   }
+  CheckForDefinableVariable(std::get<parser::ScalarLogicalVariable>(spec.t),
+      parser::ToUpperCaseLetters(common::EnumToString(specKind)));
   SetSpecifier(specKind);
 }
 
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
new file mode 100644
index 000000000000..047c60469346
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -0,0 +1,1295 @@
+//===-- lib/Semantics/check-omp-atomic.cpp --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks related to the ATOMIC construct.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <cassert>
+#include <list>
+#include <optional>
+#include <string_view>
+#include <tuple>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+namespace operation = Fortran::evaluate::operation;
+
+template <typename T, typename U>
+static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
+  return !(e == f);
+}
+
+// There is no consistent way to get the source of a given ActionStmt, so
+// extract the source information from Statement<ActionStmt> when we can,
+// and keep it around for error reporting in further analyses.
+struct SourcedActionStmt {
+  const parser::ActionStmt *stmt{nullptr};
+  parser::CharBlock source;
+
+  operator bool() const { return stmt != nullptr; }
+};
+
+struct AnalyzedCondStmt {
+  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
+  parser::CharBlock source;
+  SourcedActionStmt ift, iff;
+};
+
+static SourcedActionStmt GetActionStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return SourcedActionStmt{};
+  }
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    using ActionStmt = parser::Statement<parser::ActionStmt>;
+    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
+      return SourcedActionStmt{&stmt->statement, stmt->source};
+    }
+  }
+  return SourcedActionStmt{};
+}
+
+static SourcedActionStmt GetActionStmt(const parser::Block &block) {
+  if (block.size() == 1) {
+    return GetActionStmt(&block.front());
+  }
+  return SourcedActionStmt{};
+}
+
+// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
+// is that the ActionStmt will be either an assignment or a pointer-assignment,
+// otherwise return std::nullopt.
+// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
+// the "typedAssignment" is unset. This can happen if there are semantic errors
+// in the purported assignment.
+static std::optional<evaluate::Assignment> GetEvaluateAssignment(
+    const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
+
+  return common::visit(
+      [](auto &&s) -> std::optional<evaluate::Assignment> {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>) {
+          const TypedAssignment &typed{s.value().typedAssignment};
+          // ForwardOwningPointer                 typedAssignment
+          // `- GenericAssignmentWrapper          ^.get()
+          //    `- std::optional<Assignment>      ^->v
+          return typed.get()->v;
+        } else {
+          return std::nullopt;
+        }
+      },
+      x->u);
+}
+
+static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
+    const parser::ExecutionPartConstruct *x) {
+  if (x == nullptr) {
+    return std::nullopt;
+  }
+
+  // Extract the evaluate::Expr from ScalarLogicalExpr.
+  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
+    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
+    const parser::Expr &expr{logical.thing.thing.value()};
+    return GetEvaluateExpr(expr);
+  }};
+
+  // Recognize either
+  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
+  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
+
+  if (auto &&action{GetActionStmt(x)}) {
+    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
+            &action.stmt->u)}) {
+      const parser::IfStmt &s{ifs->value()};
+      auto &&maybeCond{
+          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
+      auto &thenStmt{
+          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
+      if (maybeCond) {
+        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
+            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
+            SourcedActionStmt{}};
+      }
+    }
+    return std::nullopt;
+  }
+
+  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
+    if (auto *ifc{
+            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
+      using ElseBlock = parser::IfConstruct::ElseBlock;
+      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
+      const parser::IfConstruct &s{ifc->value()};
+
+      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
+        // Not expecting any else-if statements.
+        return std::nullopt;
+      }
+      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
+      auto &&maybeCond{getFromLogical(
+          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
+      if (!maybeCond) {
+        return std::nullopt;
+      }
+
+      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)),
+            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
+        if (result.ift.stmt && result.iff.stmt) {
+          return result;
+        }
+      } else {
+        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
+            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
+        if (result.ift.stmt) {
+          return result;
+        }
+      }
+    }
+    return std::nullopt;
+  }
+
+  return std::nullopt;
+}
+
+static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
+    parser::CharBlock source) {
+  // Find => in the range, if not found, find = that is not a part of
+  // <=, >=, ==, or /=.
+  auto trim{[](std::string_view v) {
+    const char *begin{v.data()};
+    const char *end{begin + v.size()};
+    while (*begin == ' ' && begin != end) {
+      ++begin;
+    }
+    while (begin != end && end[-1] == ' ') {
+      --end;
+    }
+    assert(begin != end && "Source should not be empty");
+    return parser::CharBlock(begin, end - begin);
+  }};
+
+  std::string_view sv(source.begin(), source.size());
+
+  if (auto where{sv.find("=>")}; where != sv.npos) {
+    std::string_view lhs(sv.data(), where);
+    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
+    return std::make_pair(trim(lhs), trim(rhs));
+  }
+
+  // Go backwards, since all the exclusions above end with a '='.
+  for (size_t next{source.size()}; next > 1; --next) {
+    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
+      std::string_view lhs(sv.data(), next - 1);
+      std::string_view rhs(sv.data() + next, sv.size() - next);
+      return std::make_pair(trim(lhs), trim(rhs));
+    }
+  }
+  llvm_unreachable("Could not find assignment operator");
+}
+
+static bool IsCheckForAssociated(const SomeExpr &cond) {
+  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
+}
+
+static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
+  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
+  // for example.
+  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
+}
+
+static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
+  if (value) {
+    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
+        evaluate::GenericExprWrapper::Deleter);
+  }
+}
+
+static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
+    std::optional<evaluate::Assignment> value) {
+  if (value) {
+    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
+        evaluate::GenericAssignmentWrapper::Deleter);
+  }
+}
+
+static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
+    int what,
+    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+  parser::OpenMPAtomicConstruct::Analysis::Op operation;
+  operation.what = what;
+  SetAssignment(operation.assign, maybeAssign);
+  return operation;
+}
+
+static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
+    const SomeExpr &atom, const MaybeExpr &cond,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
+    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
+  // Defined in flang/include/flang/Parser/parse-tree.h
+  //
+  // struct Analysis {
+  //   struct Kind {
+  //     static constexpr int None = 0;
+  //     static constexpr int Read = 1;
+  //     static constexpr int Write = 2;
+  //     static constexpr int Update = Read | Write;
+  //     static constexpr int Action = 3; // Bits containing N, R, W, U
+  //     static constexpr int IfTrue = 4;
+  //     static constexpr int IfFalse = 8;
+  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
+  //   };
+  //   struct Op {
+  //     int what;
+  //     TypedAssignment assign;
+  //   };
+  //   TypedExpr atom, cond;
+  //   Op op0, op1;
+  // };
+
+  parser::OpenMPAtomicConstruct::Analysis an;
+  SetExpr(an.atom, atom);
+  SetExpr(an.cond, cond);
+  an.op0 = std::move(op0);
+  an.op1 = std::move(op1);
+  return an;
+}
+
+/// Check if `expr` satisfies the following conditions for x and v:
+///
+/// [6.0:189:10-12]
+/// - x and v (as applicable) are either scalar variables or
+///   function references with scalar data pointer result of non-character
+///   intrinsic type or variables that are non-polymorphic scalar pointers
+///   and any length type parameter must be constant.
+void OmpStructureChecker::CheckAtomicType(
+    SymbolRef sym, parser::CharBlock source, std::string_view name) {
+  const DeclTypeSpec *typeSpec{sym->GetType()};
+  if (!typeSpec) {
+    return;
+  }
+
+  if (!IsPointer(sym)) {
+    using Category = DeclTypeSpec::Category;
+    Category cat{typeSpec->category()};
+    if (cat == Category::Character) {
+      context_.Say(source,
+          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
+    } else if (cat != Category::Numeric && cat != Category::Logical) {
+      context_.Say(source,
+          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
+    }
+    return;
+  }
+
+  // Variable is a pointer.
+  if (typeSpec->IsPolymorphic()) {
+    context_.Say(source,
+        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
+        name);
+    return;
+  }
+
+  // Go over all length parameters, if any, and check if they are
+  // explicit.
+  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
+    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
+          // "entry" is a map entry
+          return entry.second.isLen() && !entry.second.isExplicit();
+        })) {
+      context_.Say(source,
+          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
+          name);
+    }
+  }
+}
+
+void OmpStructureChecker::CheckAtomicVariable(
+    const SomeExpr &atom, parser::CharBlock source) {
+  if (atom.Rank() != 0) {
+    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
+        atom.AsFortran());
+  }
+
+  std::vector<SomeExpr> dsgs{GetAllDesignators(atom)};
+  assert(dsgs.size() == 1 && "Should have a single top-level designator");
+  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
+
+  CheckAtomicType(syms.back(), source, atom.AsFortran());
+
+  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
+    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
+        atom.AsFortran());
+  }
+}
+
+void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
+    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
+    parser::CharBlock source) {
+  if (auto *expr{HasStorageOverlap(base, exprs)}) {
+    context_.Say(source,
+        "Within atomic operation %s and %s access the same storage"_warn_en_US,
+        base.AsFortran(), expr->AsFortran());
+  }
+}
+
+void OmpStructureChecker::ErrorShouldBeVariable(
+    const MaybeExpr &expr, parser::CharBlock source) {
+  if (expr) {
+    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
+        expr->AsFortran());
+  } else {
+    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
+  }
+}
+
+std::pair<const parser::ExecutionPartConstruct *,
+    const parser::ExecutionPartConstruct *>
+OmpStructureChecker::CheckUpdateCapture(
+    const parser::ExecutionPartConstruct *ec1,
+    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
+  // Decide which statement is the atomic update and which is the capture.
+  //
+  // The two allowed cases are:
+  //   x = ...      atomic-var = ...
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  // or
+  //   ... = x      capture-var = atomic-var (with optional converts)
+  //   x = ...      atomic-var = ...
+  //
+  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
+  // (which makes more sense, as it captures the original value of the atomic
+  // variable).
+  //
+  // If the two statements don't fit these criteria, return a pair of default-
+  // constructed values.
+  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
+      const parser::ExecutionPartConstruct *>;
+
+  SourcedActionStmt act1{GetActionStmt(ec1)};
+  SourcedActionStmt act2{GetActionStmt(ec2)};
+  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
+  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
+  if (!maybeAssign1 || !maybeAssign2) {
+    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
+      context_.Say(source,
+          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
+    }
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  auto as1{*maybeAssign1}, as2{*maybeAssign2};
+
+  auto isUpdateCapture{
+      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
+        return IsSameOrConvertOf(c.rhs, u.lhs);
+      }};
+
+  // Do some checks that narrow down the possible choices for the update
+  // and the capture statements. This will help to emit better diagnostics.
+  // 1. An assignment could be an update (cbu) if the left-hand side is a
+  //    subexpression of the right-hand side.
+  // 2. An assignment could be a capture (cbc) if the right-hand side is
+  //    a variable (or a function ref), with potential type conversions.
+  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
+  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
+  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
+  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
+
+  // We want to diagnose cases where both assignments cannot be an update,
+  // or both cannot be a capture, as well as cases where either assignment
+  // cannot be any of these two.
+  //
+  // If we organize these boolean values into a matrix
+  //   |cbu1 cbu2|
+  //   |cbc1 cbc2|
+  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
+  // row or column, including the case where everything is zero. All these
+  // cases correspond to the determinant of the matrix being 0, which suggests
+  // that checking the det may be a convenient diagnostic check. There is only
+  // one additional case where the det is 0, which is when the matrix is all 1
+  // ("true"). The "all true" case represents the situation where both
+  // assignments could be an update as well as a capture. On the other hand,
+  // whenever det != 0, the roles of the update and the capture can be
+  // unambiguously assigned to as1 and as2 [1].
+  //
+  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
+  // det = 0, leaving 6 cases where det != 0:
+  //   0 1   0 1   1 0   1 0   1 1   1 1
+  //   1 0   1 1   0 1   1 1   0 1   1 0
+  // In each case the classification is unambiguous.
+
+  //     |cbu1 cbu2|
+  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
+  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
+
+  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
+                                  const std::string &expr) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
+        expr);
+  }};
+
+  auto errorNeitherWorks{[&]() {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
+  }};
+
+  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
+    // If det != 0, then the checks unambiguously suggest a specific
+    // categorization.
+    // If det == 0, then this function should be called only if the
+    // checks haven't ruled out any possibility, i.e. when both assigments
+    // could still be either updates or captures.
+    if (det > 0) {
+      // as1 is update, as2 is capture
+      if (isUpdateCapture(as1, as2)) {
+        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
+      } else {
+        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else if (det < 0) {
+      // as2 is update, as1 is capture
+      if (isUpdateCapture(as2, as1)) {
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      } else {
+        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
+        return std::make_pair(nullptr, nullptr);
+      }
+    } else {
+      bool updateFirst{isUpdateCapture(as1, as2)};
+      bool captureFirst{isUpdateCapture(as2, as1)};
+      if (updateFirst && captureFirst) {
+        // If both assignment could be the update and both could be the
+        // capture, emit a warning about the ambiguity.
+        context_.Say(act1.source,
+            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
+        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
+      }
+      if (updateFirst != captureFirst) {
+        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
+        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
+        return std::make_pair(upd, cap);
+      }
+      assert(!updateFirst && !captureFirst);
+      errorNeitherWorks();
+      return std::make_pair(nullptr, nullptr);
+    }
+  }};
+
+  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
+    return makeSelectionFromDet(det);
+  }
+  assert(det == 0 && "Prior checks should have covered det != 0");
+
+  // If neither of the statements is an RMW update, it could still be a
+  // "write" update. Pretty much any assignment can be a write update, so
+  // recompute det with cbu1 = cbu2 = true.
+  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
+    return makeSelectionFromDet(writeDet);
+  }
+
+  // It's only errors from here on.
+
+  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
+    errorNeitherWorks();
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // The remaining cases are that
+  // - no candidate for update, or for capture,
+  // - one of the assigments cannot be anything.
+
+  if (!cbu1 && !cbu2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  } else if (!cbc1 && !cbc2) {
+    context_.Say(source,
+        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
+    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
+    context_.Say(src,
+        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
+    return std::make_pair(nullptr, nullptr);
+  }
+
+  // All cases should have been covered.
+  llvm_unreachable("Unchecked condition");
+}
+
+void OmpStructureChecker::CheckAtomicCaptureAssignment(
+    const evaluate::Assignment &capture, const SomeExpr &atom,
+    parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &cap{capture.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, rsrc);
+    // This part should have been checked prior to calling this function.
+    assert(*GetConvertInput(capture.rhs) == atom &&
+        "This cannot be a capture assignment");
+    CheckStorageOverlap(atom, {cap}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicReadAssignment(
+    const evaluate::Assignment &read, parser::CharBlock source) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+
+  if (auto maybe{GetConvertInput(read.rhs)}) {
+    const SomeExpr &atom{*maybe};
+
+    if (!IsVarOrFunctionRef(atom)) {
+      ErrorShouldBeVariable(atom, rsrc);
+    } else {
+      CheckAtomicVariable(atom, rsrc);
+      CheckStorageOverlap(atom, {read.lhs}, source);
+    }
+  } else {
+    ErrorShouldBeVariable(read.rhs, rsrc);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWriteAssignment(
+    const evaluate::Assignment &write, parser::CharBlock source) {
+  // [6.0:190:13-15]
+  // A write structured block is write-statement, a write statement that has
+  // one of the following forms:
+  //   x = expr
+  //   x => expr
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{write.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+  } else {
+    CheckAtomicVariable(atom, lsrc);
+    CheckStorageOverlap(atom, {write.rhs}, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateAssignment(
+    const evaluate::Assignment &update, parser::CharBlock source) {
+  // [6.0:191:1-7]
+  // An update structured block is update-statement, an update statement
+  // that has one of the following forms:
+  //   x = x operator expr
+  //   x = expr operator x
+  //   x = intrinsic-procedure-name (x)
+  //   x = intrinsic-procedure-name (x, expr-list)
+  //   x = intrinsic-procedure-name (expr-list, x)
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  const SomeExpr &atom{update.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, rsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, lsrc);
+
+  std::pair<operation::Operator, std::vector<SomeExpr>> top{
+      operation::Operator::Unknown, {}};
+  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+    top = GetTopLevelOperation(*maybeInput);
+  }
+  switch (top.first) {
+  case operation::Operator::Add:
+  case operation::Operator::Sub:
+  case operation::Operator::Mul:
+  case operation::Operator::Div:
+  case operation::Operator::And:
+  case operation::Operator::Or:
+  case operation::Operator::Eqv:
+  case operation::Operator::Neqv:
+  case operation::Operator::Min:
+  case operation::Operator::Max:
+  case operation::Operator::Identity:
+    break;
+  case operation::Operator::Call:
+    context_.Say(source,
+        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Convert:
+    context_.Say(source,
+        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Intrinsic:
+    context_.Say(source,
+        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  case operation::Operator::Constant:
+  case operation::Operator::Unknown:
+    context_.Say(
+        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    return;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(source,
+        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+        operation::ToString(top.first));
+    return;
+  }
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
+  std::vector<SomeExpr> nonAtom;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
+      } else {
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
+      }
+    }
+    return count;
+  }()};
+
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+          atom.AsFortran());
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
+      context_.Say(rsrc,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    }
+  }
+
+  if (!hasError) {
+    CheckStorageOverlap(atom, nonAtom, source);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
+    const SomeExpr &cond, parser::CharBlock condSource,
+    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
+  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
+  const SomeExpr &atom{assign.lhs};
+
+  if (!IsVarOrFunctionRef(atom)) {
+    ErrorShouldBeVariable(atom, arsrc);
+    // Skip other checks.
+    return;
+  }
+
+  CheckAtomicVariable(atom, alsrc);
+
+  auto top{GetTopLevelOperation(cond)};
+  // Missing arguments to operations would have been diagnosed by now.
+
+  switch (top.first) {
+  case operation::Operator::Associated:
+    if (atom != top.second.front()) {
+      context_.Say(assignSource,
+          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
+    }
+    break;
+  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
+  case operation::Operator::Eq:
+  case operation::Operator::Eqv:
+  // x ordop expr | expr ordop x
+  case operation::Operator::Lt:
+  case operation::Operator::Gt: {
+    const SomeExpr &arg0{top.second[0]};
+    const SomeExpr &arg1{top.second[1]};
+    if (IsSameOrConvertOf(arg0, atom)) {
+      CheckStorageOverlap(atom, {arg1}, condSource);
+    } else if (IsSameOrConvertOf(arg1, atom)) {
+      CheckStorageOverlap(atom, {arg0}, condSource);
+    } else {
+      assert(top.first != operation::Operator::Identity &&
+          "Handle this separately");
+      context_.Say(assignSource,
+          "An argument of the %s operator should be the target of the assignment"_err_en_US,
+          operation::ToString(top.first));
+    }
+    break;
+  }
+  case operation::Operator::Identity:
+  case operation::Operator::True:
+  case operation::Operator::False:
+    break;
+  default:
+    assert(
+        top.first != operation::Operator::Identity && "Handle this separately");
+    context_.Say(condSource,
+        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
+        operation::ToString(top.first));
+    break;
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
+    const AnalyzedCondStmt &update, parser::CharBlock source) {
+  // The condition/statements must be:
+  // - cond: x equalop e      ift: x =  d     iff: -
+  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
+  // - cond: associated(x)    ift: x => expr  iff: -
+  // - cond: associated(x, e) ift: x => expr  iff: -
+
+  // The if-true statement must be present, and must be an assignment.
+  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
+  if (!maybeAssign) {
+    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
+    } else {
+      context_.Say(
+          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
+    }
+    return;
+  }
+  const evaluate::Assignment assign{*maybeAssign};
+  const SomeExpr &atom{assign.lhs};
+
+  CheckAtomicConditionalUpdateAssignment(
+      update.cond, update.source, assign, update.ift.source);
+
+  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
+
+  if (update.iff) {
+    context_.Say(update.iff.source,
+        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdateOnly(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeUpdate->lhs};
+      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdate(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // Allowable forms are (single-statement):
+  // - if ...
+  // - x = (... ? ... : x)
+  // and two-statement:
+  // - r = cond ; if (r) ...
+
+  const parser::ExecutionPartConstruct *ust{nullptr}; // update
+  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
+
+  if (body.size() == 1) {
+    ust = &body.front();
+  } else if (body.size() == 2) {
+    cst = &body.front();
+    ust = &body.back();
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
+    return;
+  }
+
+  // Flang doesn't support conditional-expr yet, so all update statements
+  // are if-statements.
+
+  // IfStmt:        if (...) ...
+  // IfConstruct:   if (...) then ... endif
+  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
+  if (!maybeUpdate) {
+    context_.Say(source,
+        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
+    return;
+  }
+
+  AnalyzedCondStmt &update{*maybeUpdate};
+
+  if (SourcedActionStmt action{GetActionStmt(cst)}) {
+    // The "condition" statement must be `r = cond`.
+    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
+      if (maybeCond->lhs != update.cond) {
+        context_.Say(update.source,
+            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
+            maybeCond->lhs.AsFortran());
+      } else {
+        // If it's "r = ...; if (r) ..." then put the original condition
+        // in `update`.
+        update.cond = maybeCond->rhs;
+      }
+    } else {
+      context_.Say(action.source,
+          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
+    }
+  }
+
+  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
+
+  CheckAtomicConditionalUpdateStmt(update, source);
+  if (IsCheckForAssociated(update.cond)) {
+    if (!IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
+    }
+  } else {
+    if (IsPointerAssignment(assign)) {
+      context_.Say(source,
+          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
+    }
+  }
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
+      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
+      MakeAtomicAnalysisOp(Analysis::None));
+}
+
+void OmpStructureChecker::CheckAtomicUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  if (body.size() != 2) {
+    context_.Say(source,
+        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
+    return;
+  }
+
+  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
+  if (!uec || !cec) {
+    // Diagnostics already emitted.
+    return;
+  }
+  SourcedActionStmt uact{GetActionStmt(uec)};
+  SourcedActionStmt cact{GetActionStmt(cec)};
+  // The "dereferences" of std::optional are guaranteed to be valid after
+  // CheckUpdateCapture.
+  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
+  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
+
+  const SomeExpr &atom{update.lhs};
+
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  int action;
+
+  if (IsMaybeAtomicWrite(update)) {
+    action = Analysis::Write;
+    CheckAtomicWriteAssignment(update, uact.source);
+  } else {
+    action = Analysis::Update;
+    CheckAtomicUpdateAssignment(update, uact.source);
+  }
+  CheckAtomicCaptureAssignment(capture, atom, cact.source);
+
+  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+    context_.Say(cact.source,
+        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+    return;
+  }
+
+  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(action, update),
+        MakeAtomicAnalysisOp(Analysis::Read, capture));
+  } else {
+    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+        MakeAtomicAnalysisOp(Analysis::Read, capture),
+        MakeAtomicAnalysisOp(action, update));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
+    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
+    parser::CharBlock source) {
+  // There are two different variants of this:
+  // (1) conditional-update and capture separately:
+  //     This form only allows single-statement updates, i.e. the update
+  //     form "r = cond; if (r) ..." is not allowed.
+  // (2) conditional-update combined with capture in a single statement:
+  //     This form does allow the condition to be calculated separately,
+  //     i.e. "r = cond; if (r) ...".
+  // Regardless of what form it is, the actual update assignment is a
+  // proper write, i.e. "x = d", where d does not depend on x.
+
+  AnalyzedCondStmt update;
+  SourcedActionStmt capture;
+  bool captureAlways{true}, captureFirst{true};
+
+  auto extractCapture{[&]() {
+    capture = update.iff;
+    captureAlways = false;
+    update.iff = SourcedActionStmt{};
+  }};
+
+  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
+    // The non-update statement is either "r = cond" or the capture.
+    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
+      if (update.cond == maybeAssign->lhs) {
+        // If this is "r = cond; if (r) ...", then update the condition.
+        update.cond = maybeAssign->rhs;
+        update.source = action.source;
+        // In this form, the update and the capture are combined into
+        // an IF-THEN-ELSE statement.
+        extractCapture();
+      } else {
+        // Assume this is the capture-statement.
+        capture = action;
+      }
+    }
+  }};
+
+  if (body.size() == 2) {
+    // This could be
+    // - capture; conditional-update (in any order), or
+    // - r = cond; if (r) capture-update
+    const parser::ExecutionPartConstruct *st1{&body.front()};
+    const parser::ExecutionPartConstruct *st2{&body.back()};
+    // In either case, the conditional statement can be analyzed by
+    // AnalyzeConditionalStmt, whereas the other statement cannot.
+    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
+      update = *maybeUpdate1;
+      classifyNonUpdate(GetActionStmt(st2));
+      captureFirst = false;
+    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
+      update = *maybeUpdate2;
+      classifyNonUpdate(GetActionStmt(st1));
+    } else {
+      // None of the statements are conditional, this rules out the
+      // "r = cond; if (r) ..." and the "capture + conditional-update"
+      // variants. This could still be capture + write (which is classified
+      // as conditional-update-capture in the spec).
+      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
+      if (!uec || !cec) {
+        // Diagnostics already emitted.
+        return;
+      }
+      SourcedActionStmt uact{GetActionStmt(uec)};
+      SourcedActionStmt cact{GetActionStmt(cec)};
+      update.ift = uact;
+      capture = cact;
+      if (uec == st1) {
+        captureFirst = false;
+      }
+    }
+  } else if (body.size() == 1) {
+    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
+      update = *maybeUpdate;
+      // This is the form with update and capture combined into an IF-THEN-ELSE
+      // statement. The capture-statement is always the ELSE branch.
+      extractCapture();
+    } else {
+      goto invalid;
+    }
+  } else {
+    context_.Say(source,
+        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
+    return;
+  invalid:
+    context_.Say(source,
+        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
+    return;
+  }
+
+  // The update must have a form `x = d` or `x => d`.
+  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
+    const SomeExpr &atom{maybeWrite->lhs};
+    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
+    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
+      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
+
+      if (IsPointerAssignment(*maybeWrite) !=
+          IsPointerAssignment(*maybeCapture)) {
+        context_.Say(capture.source,
+            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
+        return;
+      }
+    } else {
+      if (!IsAssignment(capture.stmt)) {
+        context_.Say(capture.source,
+            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
+      }
+      return;
+    }
+  } else {
+    if (!IsAssignment(update.ift.stmt)) {
+      context_.Say(update.ift.source,
+          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
+    }
+    return;
+  }
+
+  // update.iff should be empty here, the capture statement should be
+  // stored in "capture".
+
+  // Fill out the analysis in the AST node.
+  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+  bool condUnused{std::visit(
+      [](auto &&s) {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
+          return true;
+        } else {
+          return false;
+        }
+      },
+      update.cond.u)};
+
+  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
+  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
+
+  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
+  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+
+  if (captureFirst) {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+  } else {
+    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
+        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
+        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+  }
+}
+
+void OmpStructureChecker::CheckAtomicRead(
+    const parser::OpenMPAtomicConstruct &x) {
+  // [6.0:190:5-7]
+  // A read structured block is read-statement, a read statement that has one
+  // of the following forms:
+  //   v = x
+  //   v => x
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Read cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
+      CheckAtomicReadAssignment(*maybeRead, action.source);
+
+      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
+        const SomeExpr &atom{*maybe};
+        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
+            MakeAtomicAnalysisOp(Analysis::None));
+      }
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC READ operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicWrite(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &block{std::get<parser::Block>(x.t)};
+
+  // Write cannot be conditional or have a capture statement.
+  if (x.IsCompare() || x.IsCapture()) {
+    context_.Say(dirSpec.source,
+        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
+    return;
+  }
+
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (body.size() == 1) {
+    SourcedActionStmt action{GetActionStmt(&body.front())};
+    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
+      const SomeExpr &atom{maybeWrite->lhs};
+      CheckAtomicWriteAssignment(*maybeWrite, action.source);
+
+      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
+      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
+          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
+          MakeAtomicAnalysisOp(Analysis::None));
+    } else if (!IsAssignment(action.stmt)) {
+      context_.Say(
+          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
+    }
+  } else {
+    context_.Say(x.source,
+        "ATOMIC WRITE operation should have a single statement"_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckAtomicUpdate(
+    const parser::OpenMPAtomicConstruct &x) {
+  auto &block{std::get<parser::Block>(x.t)};
+
+  bool isConditional{x.IsCompare()};
+  bool isCapture{x.IsCapture()};
+  const parser::Block &body{GetInnermostExecPart(block)};
+
+  if (isConditional && isCapture) {
+    CheckAtomicConditionalUpdateCapture(x, body, x.source);
+  } else if (isConditional) {
+    CheckAtomicConditionalUpdate(x, body, x.source);
+  } else if (isCapture) {
+    CheckAtomicUpdateCapture(x, body, x.source);
+  } else { // update-only
+    CheckAtomicUpdateOnly(x, body, x.source);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
+  // All of the following groups have the "exclusive" property, i.e. at
+  // most one clause from each group is allowed.
+  // The exclusivity-checking code should eventually be unified for all
+  // clauses, with clause groups defined in OMP.td.
+  std::array atomic{llvm::omp::Clause::OMPC_read,
+      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
+  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
+      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
+      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
+
+  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
+                          std::string_view name,
+                          const parser::OmpClauseList &clauses) {
+    const parser::OmpClause *present{nullptr};
+    for (const parser::OmpClause &clause : clauses.v) {
+      llvm::omp::Clause id{clause.Id()};
+      if (!llvm::is_contained(group, id)) {
+        continue;
+      }
+      if (present == nullptr) {
+        present = &clause;
+        continue;
+      } else if (id == present->Id()) {
+        // Ignore repetitions of the same clause, those will be diagnosed
+        // separately.
+        continue;
+      }
+      parser::MessageFormattedText txt(
+          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
+          name.data());
+      parser::Message message(clause.source, txt);
+      message.Attach(present->source,
+          "Previous clause from this group provided here"_en_US);
+      context_.Say(std::move(message));
+      return;
+    }
+  }};
+
+  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
+  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
+  llvm::omp::Clause kind{x.GetKind()};
+
+  checkExclusive(atomic, "atomic", dirSpec.Clauses());
+  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
+
+  switch (kind) {
+  case llvm::omp::Clause::OMPC_read:
+    CheckAtomicRead(x);
+    break;
+  case llvm::omp::Clause::OMPC_write:
+    CheckAtomicWrite(x);
+    break;
+  case llvm::omp::Clause::OMPC_update:
+    CheckAtomicUpdate(x);
+    break;
+  default:
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
new file mode 100644
index 000000000000..b82e2f7342d8
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -0,0 +1,671 @@
+//===-- lib/Semantics/check-omp-loop.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for constructs and clauses related to loops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree-visitor.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
+#include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <variant>
+
+namespace {
+using namespace Fortran;
+
+class AssociatedLoopChecker {
+public:
+  AssociatedLoopChecker(
+      semantics::SemanticsContext &context, std::int64_t level)
+      : context_{context}, level_{level} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::DoConstruct &dc) {
+    level_--;
+    const auto &doStmt{
+        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
+    const auto &constructName{
+        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
+    if (constructName) {
+      constructNamesAndLevels_.emplace(
+          constructName.value().ToString(), level_);
+    }
+    if (level_ >= 0) {
+      if (dc.IsDoWhile()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
+      }
+      if (!dc.GetLoopControl()) {
+        context_.Say(doStmt.source,
+            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  void Post(const parser::DoConstruct &dc) { level_++; }
+
+  bool Pre(const parser::CycleStmt &cyclestmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (cyclestmt.v) {
+      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second > 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ > 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::ExitStmt &exitStmt) {
+    std::map<std::string, std::int64_t>::iterator it;
+    bool err{false};
+    if (exitStmt.v) {
+      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
+      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
+    } else { // If there is no label then use the level of the last enclosing DO
+      err = level_ >= 0;
+    }
+    if (err) {
+      context_.Say(*source_,
+          "EXIT statement terminates associated loop of an OpenMP DO "
+          "construct"_err_en_US);
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
+    source_ = &actionstmt.source;
+    return true;
+  }
+
+private:
+  semantics::SemanticsContext &context_;
+  const parser::CharBlock *source_;
+  std::int64_t level_;
+  std::map<std::string, std::int64_t> constructNamesAndLevels_;
+};
+} // namespace
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::HasInvalidDistributeNesting(
+    const parser::OpenMPLoopConstruct &x) {
+  bool violation{false};
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
+    // `distribute` region has to be nested
+    if (!CurrentDirectiveIsNested()) {
+      violation = true;
+    } else {
+      // `distribute` region has to be strictly nested inside `teams`
+      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+        violation = true;
+      }
+    }
+  }
+  if (violation) {
+    context_.Say(beginDir.source,
+        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
+        "region."_err_en_US);
+  }
+}
+void OmpStructureChecker::HasInvalidLoopBinding(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
+    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+    for (const auto &clause : clauseList.v) {
+      if (const auto *bindClause{
+              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
+        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
+          context_.Say(beginDir.source, msg);
+        }
+      }
+    }
+  };
+
+  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
+      CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
+        "strictly nested inside a `TEAMS` region."_err_en_US);
+  }
+
+  if (OmpDirectiveSet{
+          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
+          .test(beginDir.v)) {
+    teamsBindingChecker(
+        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
+        "combined with a `TEAMS` construct."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
+  // Check the following:
+  //  The only OpenMP constructs that can be encountered during execution of
+  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
+  // construct and the `ordered` construct with the `simd` clause.
+
+  // Check if the parent context has the SIMD clause
+  // Please note that we use GetContext() instead of GetContextParent()
+  // because PushContextAndClauseSets() has not been called on the
+  // current context yet.
+  // TODO: Check for declare simd regions.
+  bool eligibleSIMD{false};
+  common::visit(
+      common::visitors{
+          // Allow `!$OMP ORDERED SIMD`
+          [&](const parser::OpenMPBlockConstruct &c) {
+            const auto &beginBlockDir{
+                std::get<parser::OmpBeginBlockDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
+            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
+              const auto &clauses{
+                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
+              for (const auto &clause : clauses.v) {
+                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
+                  eligibleSIMD = true;
+                  break;
+                }
+              }
+            }
+          },
+          [&](const parser::OpenMPStandaloneConstruct &c) {
+            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                    &c.u)}) {
+              llvm::omp::Directive dirId{ssc->v.DirId()};
+              if (dirId == llvm::omp::Directive::OMPD_ordered) {
+                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
+                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
+                    eligibleSIMD = true;
+                    break;
+                  }
+                }
+              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
+                eligibleSIMD = true;
+              }
+            }
+          },
+          // Allowing SIMD and loop construct
+          [&](const parser::OpenMPLoopConstruct &c) {
+            const auto &beginLoopDir{
+                std::get<parser::OmpBeginLoopDirective>(c.t)};
+            const auto &beginDir{
+                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
+                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
+              eligibleSIMD = true;
+            }
+          },
+          [&](const parser::OpenMPAtomicConstruct &c) {
+            // Allow `!$OMP ATOMIC`
+            eligibleSIMD = true;
+          },
+          [&](const auto &c) {},
+      },
+      c.u);
+  if (!eligibleSIMD) {
+    context_.Say(parser::FindSourceLocation(c),
+        "The only OpenMP constructs that can be encountered during execution "
+        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
+        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
+        "construct with the `SIMD` clause."_err_en_US);
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
+  loopStack_.push_back(&x);
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
+
+  PushContextAndClauseSets(beginDir.source, beginDir.v);
+
+  // check matching, End directive is optional
+  if (const auto &endLoopDir{
+          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
+    const auto &endDir{
+        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
+
+    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
+
+    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
+  }
+
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    EnterDirectiveNest(SIMDNest);
+  }
+
+  // Combined target loop constructs are target device constructs. Keep track of
+  // whether any such construct has been visited to later check that REQUIRES
+  // directives for target-related options don't appear after them.
+  if (llvm::omp::allTargetSet.test(beginDir.v)) {
+    deviceConstructFound_ = true;
+  }
+
+  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
+    // 2.7.1 do-clause -> private-clause |
+    //                    firstprivate-clause |
+    //                    lastprivate-clause |
+    //                    linear-clause |
+    //                    reduction-clause |
+    //                    schedule-clause |
+    //                    collapse-clause |
+    //                    ordered-clause
+
+    // nesting check
+    HasInvalidWorksharingNesting(
+        beginDir.source, llvm::omp::nestedWorkshareErrSet);
+  }
+  SetLoopInfo(x);
+
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &doConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
+      CheckNoBranching(doBlock, beginDir.v, beginDir.source);
+    }
+  }
+  CheckLoopItrVariableIsInt(x);
+  CheckAssociatedLoopConstraints(x);
+  HasInvalidDistributeNesting(x);
+  HasInvalidLoopBinding(x);
+  if (CurrentDirectiveIsNested() &&
+      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
+    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
+  }
+  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
+      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
+    CheckDistLinear(x);
+  }
+}
+
+const parser::Name OmpStructureChecker::GetLoopIndex(
+    const parser::DoConstruct *x) {
+  using Bounds = parser::LoopControl::Bounds;
+  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
+}
+
+void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      const parser::DoConstruct *loop{&*loopConstruct};
+      if (loop && loop->IsDoNormal()) {
+        const parser::Name &itrVal{GetLoopIndex(loop)};
+        SetLoopIv(itrVal.symbol);
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::CheckLoopItrVariableIsInt(
+    const parser::OpenMPLoopConstruct &x) {
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &loopConstruct{
+            std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+
+      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+        if (loop->IsDoNormal()) {
+          const parser::Name &itrVal{GetLoopIndex(loop)};
+          if (itrVal.symbol) {
+            const auto *type{itrVal.symbol->GetType()};
+            if (!type->IsNumeric(TypeCategory::Integer)) {
+              context_.Say(itrVal.source,
+                  "The DO loop iteration"
+                  " variable must be of the type integer."_err_en_US,
+                  itrVal.ToString());
+            }
+          }
+        }
+        // Get the next DoConstruct if block is not empty.
+        const auto &block{std::get<parser::Block>(loop->t)};
+        const auto it{block.begin()};
+        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                 : nullptr;
+      }
+    }
+  }
+}
+
+std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
+    const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+  std::int64_t orderedCollapseLevel{1};
+  std::int64_t orderedLevel{1};
+  std::int64_t collapseLevel{1};
+
+  for (const auto &clause : clauseList.v) {
+    if (const auto *collapseClause{
+            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
+      if (const auto v{GetIntValue(collapseClause->v)}) {
+        collapseLevel = *v;
+      }
+    }
+    if (const auto *orderedClause{
+            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
+      if (const auto v{GetIntValue(orderedClause->v)}) {
+        orderedLevel = *v;
+      }
+    }
+  }
+  if (orderedLevel >= collapseLevel) {
+    orderedCollapseLevel = orderedLevel;
+  } else {
+    orderedCollapseLevel = collapseLevel;
+  }
+  return orderedCollapseLevel;
+}
+
+void OmpStructureChecker::CheckAssociatedLoopConstraints(
+    const parser::OpenMPLoopConstruct &x) {
+  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
+  AssociatedLoopChecker checker{context_, ordCollapseLevel};
+  parser::Walk(x, checker);
+}
+
+void OmpStructureChecker::CheckDistLinear(
+    const parser::OpenMPLoopConstruct &x) {
+
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  SymbolSourceMap indexVars;
+
+  // Collect symbols of all the variables from linear clauses
+  for (auto &clause : clauses.v) {
+    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
+      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
+      GetSymbolsInObjectList(objects, indexVars);
+    }
+  }
+
+  if (!indexVars.empty()) {
+    // Get collapse level, if given, to find which loops are "associated."
+    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
+    // Include the top loop if no collapse is specified
+    if (collapseVal == 0) {
+      collapseVal = 1;
+    }
+
+    // Match the loop index variables with the collected symbols from linear
+    // clauses.
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &loopConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
+          if (loop->IsDoNormal()) {
+            const parser::Name &itrVal{GetLoopIndex(loop)};
+            if (itrVal.symbol) {
+              // Remove the symbol from the collected set
+              indexVars.erase(&itrVal.symbol->GetUltimate());
+            }
+            collapseVal--;
+            if (collapseVal == 0) {
+              break;
+            }
+          }
+          // Get the next DoConstruct if block is not empty.
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
+                                   : nullptr;
+        }
+      }
+    }
+
+    // Show error for the remaining variables
+    for (auto &[symbol, source] : indexVars) {
+      const Symbol &root{GetAssociationRoot(*symbol)};
+      context_.Say(source,
+          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
+          root.name());
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
+  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
+  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
+
+  // A few semantic checks for InScan reduction are performed below as SCAN
+  // constructs inside LOOP may add the relevant information. Scan reduction is
+  // supported only in loop constructs, so same checks are not applicable to
+  // other directives.
+  using ReductionModifier = parser::OmpReductionModifier;
+  for (const auto &clause : clauseList.v) {
+    if (const auto *reductionClause{
+            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
+      auto &modifiers{OmpGetModifiers(reductionClause->v)};
+      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
+      if (maybeModifier &&
+          maybeModifier->v == ReductionModifier::Value::Inscan) {
+        const auto &objectList{
+            std::get<parser::OmpObjectList>(reductionClause->v.t)};
+        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
+          if (auto &symbol = name->symbol) {
+            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
+                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
+              context_.Say(name->source,
+                  "List item %s must appear in EXCLUSIVE or "
+                  "INCLUSIVE clause of an "
+                  "enclosed SCAN directive"_err_en_US,
+                  name->ToString());
+            }
+          }
+        };
+        for (const auto &ompObj : objectList.v) {
+          common::visit(
+              common::visitors{
+                  [&](const parser::Designator &designator) {
+                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
+                            designator)}) {
+                      checkReductionSymbolInScan(name);
+                    }
+                  },
+                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
+              },
+              ompObj.u);
+        }
+      }
+    }
+  }
+  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
+    ExitDirectiveNest(SIMDNest);
+  }
+  dirContext_.pop_back();
+
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top{loopStack_.back()};
+  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
+  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+}
+
+void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
+  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  // 2.7.1 end-do -> END DO [nowait-clause]
+  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
+  case llvm::omp::Directive::OMPD_do:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
+    break;
+  case llvm::omp::Directive::OMPD_do_simd:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
+    break;
+  default:
+    // no clauses are allowed
+    break;
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
+    dirContext_.pop_back();
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
+  unsigned version{context_.langOptions().OpenMPVersion};
+  llvm::omp::Directive dir{GetContext().directive};
+  parser::CharBlock clauseSource{GetContext().clauseSource};
+  const parser::OmpLinearModifier *linearMod{nullptr};
+
+  SymbolSourceMap symbols;
+  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
+  CheckCrayPointee(objects, "LINEAR", false);
+  GetSymbolsInObjectList(objects, symbols);
+
+  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
+    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      context_.Say(source,
+          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
+          symbol->name(), desc.name.str());
+    }
+  }};
+
+  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
+    auto &modifiers{OmpGetModifiers(x.v)};
+    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
+    if (linearMod) {
+      // 2.7 Loop Construct Restriction
+      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
+        context_.Say(clauseSource,
+            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+        return;
+      }
+
+      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
+      for (auto &[symbol, source] : symbols) {
+        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
+          CheckIntegerNoRef(symbol, source);
+        } else {
+          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
+              !IsPolymorphic(*symbol)) {
+            context_.Say(source,
+                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
+                symbol->name(), desc.name.str());
+          }
+        }
+        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
+            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
+          if (!IsDummy(*symbol) || IsValue(*symbol)) {
+            context_.Say(source,
+                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
+                desc.name.str(), symbol->name());
+          }
+        }
+      } // for (symbol, source)
+
+      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
+        context_.Say(OmpGetModifierSource(modifiers, linearMod),
+            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
+            ThisVersion(version));
+      }
+    }
+  }
+
+  // OpenMP 5.2: Ordered clause restriction
+  if (const auto *clause{
+          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
+    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
+    if (orderedClause.v) {
+      return;
+    }
+  }
+
+  // OpenMP 5.2: Linear clause Restrictions
+  for (auto &[symbol, source] : symbols) {
+    if (!linearMod) {
+      // Already checked this with the modifier present.
+      CheckIntegerNoRef(symbol, source);
+    }
+    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
+      context_.Say(source,
+          "The list item `%s` must be a dummy argument"_err_en_US,
+          symbol->name());
+    }
+    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
+      context_.Say(source,
+          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
+          symbol->name());
+    }
+    if (FindCommonBlockContaining(*symbol)) {
+      context_.Say(source,
+          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
+          symbol->name());
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
+  Base::Enter(x);
+  loopStack_.push_back(&x);
+}
+
+void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
+  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
+#ifndef NDEBUG
+  const LoopConstruct &top = loopStack_.back();
+  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
+  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
+#endif
+  loopStack_.pop_back();
+  Base::Leave(x);
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
new file mode 100644
index 000000000000..03487da64f1b
--- /dev/null
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -0,0 +1,548 @@
+//===-- lib/Semantics/check-omp-metadirective.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Semantic checks for METADIRECTIVE and related constructs/clauses.
+//
+//===----------------------------------------------------------------------===//
+
+#include "check-omp-structure.h"
+
+#include "openmp-utils.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <variant>
+
+namespace Fortran::semantics {
+
+using namespace Fortran::semantics::omp;
+
+void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
+  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
+  OmpVerifyModifiers(
+      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
+}
+
+void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
+  EnterDirectiveNest(ContextSelectorNest);
+
+  using SetName = parser::OmpTraitSetSelectorName;
+  std::map<SetName::Value, const SetName *> visited;
+
+  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
+    auto &name{std::get<SetName>(traitSet.t)};
+    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait set name %s in a context specifier"_err_en_US,
+          showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait set %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+    CheckTraitSetSelector(traitSet);
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
+  ExitDirectiveNest(ContextSelectorNest);
+}
+
+const std::list<parser::OmpTraitProperty> &
+OmpStructureChecker::GetTraitPropertyList(
+    const parser::OmpTraitSelector &trait) {
+  static const std::list<parser::OmpTraitProperty> empty{};
+  auto &[_, maybeProps]{trait.t};
+  if (maybeProps) {
+    using PropertyList = std::list<parser::OmpTraitProperty>;
+    return std::get<PropertyList>(maybeProps->t);
+  } else {
+    return empty;
+  }
+}
+
+std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
+    const parser::OmpTraitProperty &property) {
+  using MaybeClause = std::optional<llvm::omp::Clause>;
+
+  // The parser for OmpClause will only succeed if the clause was
+  // given with all required arguments.
+  // If this is a string or complex extension with a clause name,
+  // treat it as a clause and let the trait checker deal with it.
+
+  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
+    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
+    if (id != llvm::omp::Clause::OMPC_unknown) {
+      return id;
+    } else {
+      return std::nullopt;
+    }
+  }};
+
+  return common::visit( //
+      common::visitors{
+          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
+            return getClauseFromString(x.v);
+          },
+          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
+            return x.value().Id();
+          },
+          [&](const parser::ScalarExpr &x) -> MaybeClause {
+            return std::nullopt;
+          },
+          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
+            using ExtProperty = parser::OmpTraitPropertyExtension;
+            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
+              return getClauseFromString(name->v);
+            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
+              return getClauseFromString(
+                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
+            }
+            return std::nullopt;
+          },
+      },
+      property.u);
+}
+
+void OmpStructureChecker::CheckTraitSelectorList(
+    const std::list<parser::OmpTraitSelector> &traits) {
+  // [6.0:322:20]
+  // Each trait-selector-name may only be specified once in a trait selector
+  // set.
+
+  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
+  using TraitName = parser::OmpTraitSelectorName;
+  using BareName = decltype(TraitName::u);
+  std::map<BareName, const TraitName *> visited;
+
+  for (const parser::OmpTraitSelector &trait : traits) {
+    auto &name{std::get<TraitName>(trait.t)};
+
+    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
+    if (!unique) {
+      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
+      parser::MessageFormattedText txt(
+          "Repeated trait name %s in a trait set"_err_en_US, showName);
+      parser::Message message(name.source, txt);
+      message.Attach(prev->second->source,
+          "Previous trait %s provided here"_en_US, showName);
+      context_.Say(std::move(message));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSetSelector(
+    const parser::OmpTraitSetSelector &traitSet) {
+
+  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
+  //
+  // Construct      |     Simd, directive-name |      Yes |       No |    No |
+  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
+  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
+  //                |      Extension, Requires |          |          |       |
+  //                |                   Vendor |          |          |       |
+  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
+  //                |                Kind, Uid |          |          |       |
+  // User           |                Condition |       No |       No |   Yes |
+
+  struct TraitSetConfig {
+    std::set<parser::OmpTraitSelectorName::Value> allowed;
+    bool allowsDirectiveTraits;
+    bool allowsExtensionTraits;
+    bool allowsScore;
+  };
+
+  using SName = parser::OmpTraitSetSelectorName::Value;
+  using TName = parser::OmpTraitSelectorName::Value;
+
+  static const std::map<SName, TraitSetConfig> configs{
+      {SName::Construct, //
+          {{TName::Simd}, true, false, false}},
+      {SName::Device, //
+          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
+      {SName::Implementation, //
+          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
+               TName::Vendor},
+              false, true, true}},
+      {SName::Target_Device, //
+          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
+               TName::Uid},
+              false, true, false}},
+      {SName::User, //
+          {{TName::Condition}, false, false, true}},
+  };
+
+  auto checkTraitSet{[&](const TraitSetConfig &config) {
+    auto &[setName, traits]{traitSet.t};
+    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
+
+    // Check if there are any duplicate traits.
+    CheckTraitSelectorList(traits);
+
+    for (const parser::OmpTraitSelector &trait : traits) {
+      // Don't use structured bindings here, because they cannot be captured
+      // before C++20.
+      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
+      auto &maybeProps =
+          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
+              trait.t);
+
+      // Check allowed traits
+      common::visit( //
+          common::visitors{
+              [&](parser::OmpTraitSelectorName::Value v) {
+                if (!config.allowed.count(v)) {
+                  context_.Say(traitName.source,
+                      "%s is not a valid trait for %s trait set"_err_en_US,
+                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
+                }
+              },
+              [&](llvm::omp::Directive) {
+                if (!config.allowsDirectiveTraits) {
+                  context_.Say(traitName.source,
+                      "Directive name is not a valid trait for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+              [&](const std::string &) {
+                if (!config.allowsExtensionTraits) {
+                  context_.Say(traitName.source,
+                      "Extension traits are not valid for %s trait set"_err_en_US,
+                      usn);
+                }
+              },
+          },
+          traitName.u);
+
+      // Check score
+      if (maybeProps) {
+        auto &[maybeScore, _]{maybeProps->t};
+        if (maybeScore) {
+          CheckTraitScore(*maybeScore);
+        }
+      }
+
+      // Check the properties of the individual traits
+      CheckTraitSelector(traitSet, trait);
+    }
+  }};
+
+  checkTraitSet(
+      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
+}
+
+void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
+  // [6.0:322:23]
+  // A score-expression must be a non-negative constant integer expression.
+  if (auto value{GetIntValue(score)}; !value || value < 0) {
+    context_.Say(score.source,
+        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
+  }
+}
+
+bool OmpStructureChecker::VerifyTraitPropertyLists(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  using PropertyList = std::list<parser::OmpTraitProperty>;
+  auto &[traitName, maybeProps]{trait.t};
+
+  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
+                             const std::string &message) {
+    bool foundInvalid{false};
+    for (const parser::OmpTraitProperty &prop : properties) {
+      if (!isValid(prop)) {
+        if (foundInvalid) {
+          context_.Say(
+              prop.source, "More invalid properties are present"_err_en_US);
+          break;
+        }
+        context_.Say(prop.source, "%s"_err_en_US, message);
+        foundInvalid = true;
+      }
+    }
+    return !foundInvalid;
+  }};
+
+  bool invalid{false};
+
+  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
+    // Directive-name traits don't have properties.
+    if (maybeProps) {
+      context_.Say(trait.source,
+          "Directive-name traits cannot have properties"_err_en_US);
+      invalid = true;
+    }
+  }
+  // Ignore properties on extension traits.
+
+  // See `TraitSelectorParser` in openmp-parser.cpp
+  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
+    switch (*v) {
+    // name-list properties
+    case parser::OmpTraitSelectorName::Value::Arch:
+    case parser::OmpTraitSelectorName::Value::Extension:
+    case parser::OmpTraitSelectorName::Value::Isa:
+    case parser::OmpTraitSelectorName::Value::Kind:
+    case parser::OmpTraitSelectorName::Value::Uid:
+    case parser::OmpTraitSelectorName::Value::Vendor:
+      if (maybeProps) {
+        auto isName{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isName, "Trait property should be a name");
+      }
+      break;
+    // clause-list
+    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
+    case parser::OmpTraitSelectorName::Value::Requires:
+    case parser::OmpTraitSelectorName::Value::Simd:
+      if (maybeProps) {
+        auto isClause{[&](const parser::OmpTraitProperty &prop) {
+          return GetClauseFromProperty(prop).has_value();
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isClause, "Trait property should be a clause");
+      }
+      break;
+    // expr-list
+    case parser::OmpTraitSelectorName::Value::Condition:
+    case parser::OmpTraitSelectorName::Value::Device_Num:
+      if (maybeProps) {
+        auto isExpr{[](const parser::OmpTraitProperty &prop) {
+          return std::holds_alternative<parser::ScalarExpr>(prop.u);
+        }};
+        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
+            isExpr, "Trait property should be a scalar expression");
+      }
+      break;
+    } // switch
+  }
+
+  return !invalid;
+}
+
+void OmpStructureChecker::CheckTraitSelector(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  using TraitName = parser::OmpTraitSelectorName;
+  auto &[traitName, maybeProps]{trait.t};
+
+  // Only do the detailed checks if the property lists are valid.
+  if (VerifyTraitPropertyLists(traitSet, trait)) {
+    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
+        std::holds_alternative<std::string>(traitName.u)) {
+      // No properties here: directives don't have properties, and
+      // we don't implement any extension traits now.
+      return;
+    }
+
+    // Specific traits we want to check.
+    // Limitations:
+    // (1) The properties for these traits are defined in "Additional
+    // Definitions for the OpenMP API Specification". It's not clear how
+    // to define them in a portable way, and how to verify their validity,
+    // especially if they get replaced by their integer values (in case
+    // they are defined as enums).
+    // (2) These are entirely implementation-defined, and at the moment
+    // there is no known schema to validate these values.
+    auto v{std::get<TraitName::Value>(traitName.u)};
+    switch (v) {
+    case TraitName::Value::Arch:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Atomic_Default_Mem_Order:
+      CheckTraitADMO(traitSet, trait);
+      break;
+    case TraitName::Value::Condition:
+      CheckTraitCondition(traitSet, trait);
+      break;
+    case TraitName::Value::Device_Num:
+      CheckTraitDeviceNum(traitSet, trait);
+      break;
+    case TraitName::Value::Extension:
+      // Ignore
+      break;
+    case TraitName::Value::Isa:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Kind:
+      // Unchecked, TBD(1)
+      break;
+    case TraitName::Value::Requires:
+      CheckTraitRequires(traitSet, trait);
+      break;
+    case TraitName::Value::Simd:
+      CheckTraitSimd(traitSet, trait);
+      break;
+    case TraitName::Value::Uid:
+      // Unchecked, TBD(2)
+      break;
+    case TraitName::Value::Vendor:
+      // Unchecked, TBD(1)
+      break;
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitADMO(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single clause property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto clauseId{*GetClauseFromProperty(property)};
+    // Check that the clause belongs to the memory-order clause-set.
+    // Clause sets will hopefully be autogenerated at some point.
+    switch (clauseId) {
+    case llvm::omp::Clause::OMPC_acq_rel:
+    case llvm::omp::Clause::OMPC_acquire:
+    case llvm::omp::Clause::OMPC_relaxed:
+    case llvm::omp::Clause::OMPC_release:
+    case llvm::omp::Clause::OMPC_seq_cst:
+      break;
+    default:
+      context_.Say(property.source,
+          "%s trait requires a clause from the memory-order clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitCondition(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  } else {
+    const parser::OmpTraitProperty &property{properties.front()};
+    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
+
+    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
+    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
+      context_.Say(property.source,
+          "%s trait requires a single LOGICAL expression"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitDeviceNum(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  if (properties.size() != 1) {
+    context_.Say(trait.source,
+        "%s trait requires a single expression property"_err_en_US,
+        parser::ToUpperCaseLetters(traitName.ToString()));
+  }
+  // No other checks at the moment.
+}
+
+void OmpStructureChecker::CheckTraitRequires(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_requires, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause from the requirement clause set"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::CheckTraitSimd(
+    const parser::OmpTraitSetSelector &traitSet,
+    const parser::OmpTraitSelector &trait) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
+  auto &properties{GetTraitPropertyList(trait)};
+
+  for (const parser::OmpTraitProperty &property : properties) {
+    auto clauseId{*GetClauseFromProperty(property)};
+    if (!llvm::omp::isAllowedClauseForDirective(
+            llvm::omp::OMPD_declare_simd, clauseId, version)) {
+      context_.Say(property.source,
+          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(traitName.ToString()),
+          parser::ToUpperCaseLetters(
+              getDirectiveName(llvm::omp::OMPD_declare_simd)));
+    }
+
+    using ClauseProperty = common::Indirection<parser::OmpClause>;
+    if (!std::holds_alternative<ClauseProperty>(property.u)) {
+      context_.Say(property.source,
+          "Invalid clause specification for %s"_err_en_US,
+          parser::ToUpperCaseLetters(getClauseName(clauseId)));
+    }
+  }
+}
+
+void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
+  EnterDirectiveNest(MetadirectiveNest);
+  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
+}
+
+void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
+  ExitDirectiveNest(MetadirectiveNest);
+  dirContext_.pop_back();
+}
+
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index cac1b18a0a31..71c38aa2b300 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -7,27 +7,56 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
+
+#include "check-directive-structure.h"
 #include "definable.h"
+#include "openmp-utils.h"
 #include "resolve-names-utils.h"
-#include "flang/Evaluate/check-expression.h"
-#include "flang/Evaluate/expression.h"
+
+#include "flang/Common/idioms.h"
+#include "flang/Common/indirection.h"
+#include "flang/Common/visit.h"
 #include "flang/Evaluate/shape.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/characters.h"
+#include "flang/Parser/message.h"
+#include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran-features.h"
+
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+#include <list>
+#include <map>
+#include <optional>
+#include <set>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
 #include <variant>
 
 namespace Fortran::semantics {
 
-template <typename T, typename U>
-static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
-  return !(e == f);
-}
+using namespace Fortran::semantics::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -53,66 +82,6 @@ static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
     CheckAllowedClause(llvm::omp::Y); \
   }
 
-std::string ThisVersion(unsigned version) {
-  std::string tv{
-      std::to_string(version / 10) + "." + std::to_string(version % 10)};
-  return "OpenMP v" + tv;
-}
-
-std::string TryVersion(unsigned version) {
-  return "try -fopenmp-version=" + std::to_string(version);
-}
-
-static const parser::Designator *GetDesignatorFromObj(
-    const parser::OmpObject &object) {
-  return std::get_if<parser::Designator>(&object.u);
-}
-
-static const parser::DataRef *GetDataRefFromObj(
-    const parser::OmpObject &object) {
-  if (auto *desg{GetDesignatorFromObj(object)}) {
-    return std::get_if<parser::DataRef>(&desg->u);
-  }
-  return nullptr;
-}
-
-static const parser::ArrayElement *GetArrayElementFromObj(
-    const parser::OmpObject &object) {
-  if (auto *dataRef{GetDataRefFromObj(object)}) {
-    using ElementIndirection = common::Indirection<parser::ArrayElement>;
-    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
-      return &ind->value();
-    }
-  }
-  return nullptr;
-}
-
-static bool IsVarOrFunctionRef(const MaybeExpr &expr) {
-  if (expr) {
-    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
-        evaluate::IsVariable(*expr);
-  } else {
-    return false;
-  }
-}
-
-static std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
-  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
-  // ForwardOwningPointer           typedExpr
-  // `- GenericExprWrapper          ^.get()
-  //    `- std::optional<Expr>      ^->v
-  return typedExpr.get()->v;
-}
-
-static std::optional<evaluate::DynamicType> GetDynamicType(
-    const parser::Expr &parserExpr) {
-  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
-    return maybeExpr->GetType();
-  } else {
-    return std::nullopt;
-  }
-}
-
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
 // statements and the expressions enclosed in an OpenMP Workshare construct
 class OmpWorkshareBlockChecker {
@@ -172,85 +141,6 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
-class AssociatedLoopChecker {
-public:
-  AssociatedLoopChecker(SemanticsContext &context, std::int64_t level)
-      : context_{context}, level_{level} {}
-
-  template <typename T> bool Pre(const T &) { return true; }
-  template <typename T> void Post(const T &) {}
-
-  bool Pre(const parser::DoConstruct &dc) {
-    level_--;
-    const auto &doStmt{
-        std::get<parser::Statement<parser::NonLabelDoStmt>>(dc.t)};
-    const auto &constructName{
-        std::get<std::optional<parser::Name>>(doStmt.statement.t)};
-    if (constructName) {
-      constructNamesAndLevels_.emplace(
-          constructName.value().ToString(), level_);
-    }
-    if (level_ >= 0) {
-      if (dc.IsDoWhile()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO WHILE."_err_en_US);
-      }
-      if (!dc.GetLoopControl()) {
-        context_.Say(doStmt.source,
-            "The associated loop of a loop-associated directive cannot be a DO without control."_err_en_US);
-      }
-    }
-    return true;
-  }
-
-  void Post(const parser::DoConstruct &dc) { level_++; }
-
-  bool Pre(const parser::CycleStmt &cyclestmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (cyclestmt.v) {
-      it = constructNamesAndLevels_.find(cyclestmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second > 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ > 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "CYCLE statement to non-innermost associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::ExitStmt &exitStmt) {
-    std::map<std::string, std::int64_t>::iterator it;
-    bool err{false};
-    if (exitStmt.v) {
-      it = constructNamesAndLevels_.find(exitStmt.v->source.ToString());
-      err = (it != constructNamesAndLevels_.end() && it->second >= 0);
-    } else { // If there is no label then use the level of the last enclosing DO
-      err = level_ >= 0;
-    }
-    if (err) {
-      context_.Say(*source_,
-          "EXIT statement terminates associated loop of an OpenMP DO "
-          "construct"_err_en_US);
-    }
-    return true;
-  }
-
-  bool Pre(const parser::Statement<parser::ActionStmt> &actionstmt) {
-    source_ = &actionstmt.source;
-    return true;
-  }
-
-private:
-  SemanticsContext &context_;
-  const parser::CharBlock *source_;
-  std::int64_t level_;
-  std::map<std::string, std::int64_t> constructNamesAndLevels_;
-};
-
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -318,18 +208,6 @@ bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
   return CheckAllowed(clause);
 }
 
-bool OmpStructureChecker::IsCommonBlock(const Symbol &sym) {
-  return sym.detailsIf<CommonBlockDetails>() != nullptr;
-}
-
-bool OmpStructureChecker::IsVariableListItem(const Symbol &sym) {
-  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
-}
-
-bool OmpStructureChecker::IsExtendedListItem(const Symbol &sym) {
-  return IsVariableListItem(sym) || sym.IsSubprogram();
-}
-
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -371,60 +249,6 @@ bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   return false;
 }
 
-namespace {
-struct ContiguousHelper {
-  ContiguousHelper(SemanticsContext &context)
-      : fctx_(context.foldingContext()) {}
-
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
-    return Visit(x.value());
-  }
-  template <typename Contained>
-  std::optional<bool> Visit(const common::Reference<Contained> &x) {
-    return Visit(x.get());
-  }
-  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
-    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
-  }
-  template <typename T>
-  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
-    return common::visit(
-        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
-  }
-  template <typename T> std::optional<bool> Visit(const T &) {
-    // Everything else.
-    return std::nullopt;
-  }
-
-private:
-  evaluate::FoldingContext &fctx_;
-};
-} // namespace
-
-// Return values:
-// - std::optional<bool>{true} if the object is known to be contiguous
-// - std::optional<bool>{false} if the object is known not to be contiguous
-// - std::nullopt if the object contiguity cannot be determined
-std::optional<bool> OmpStructureChecker::IsContiguous(
-    const parser::OmpObject &object) {
-  return common::visit( //
-      common::visitors{
-          [&](const parser::Name &x) {
-            // Any member of a common block must be contiguous.
-            return std::optional<bool>{true};
-          },
-          [&](const parser::Designator &x) {
-            evaluate::ExpressionAnalyzer ea{context_};
-            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
-              return ContiguousHelper{context_}.Visit(*maybeExpr);
-            }
-            return std::optional<bool>{};
-          },
-      },
-      object.u);
-}
-
 void OmpStructureChecker::CheckVariableListItem(
     const SymbolSourceMap &symbols) {
   for (auto &[symbol, source] : symbols) {
@@ -522,62 +346,6 @@ bool OmpStructureChecker::HasInvalidWorksharingNesting(
   return false;
 }
 
-void OmpStructureChecker::HasInvalidDistributeNesting(
-    const parser::OpenMPLoopConstruct &x) {
-  bool violation{false};
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-  if (llvm::omp::topDistributeSet.test(beginDir.v)) {
-    // `distribute` region has to be nested
-    if (!CurrentDirectiveIsNested()) {
-      violation = true;
-    } else {
-      // `distribute` region has to be strictly nested inside `teams`
-      if (!llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-        violation = true;
-      }
-    }
-  }
-  if (violation) {
-    context_.Say(beginDir.source,
-        "`DISTRIBUTE` region has to be strictly nested inside `TEAMS` "
-        "region."_err_en_US);
-  }
-}
-void OmpStructureChecker::HasInvalidLoopBinding(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  auto teamsBindingChecker = [&](parser::MessageFixedText msg) {
-    const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-    for (const auto &clause : clauseList.v) {
-      if (const auto *bindClause{
-              std::get_if<parser::OmpClause::Bind>(&clause.u)}) {
-        if (bindClause->v.v != parser::OmpBindClause::Binding::Teams) {
-          context_.Say(beginDir.source, msg);
-        }
-      }
-    }
-  };
-
-  if (llvm::omp::Directive::OMPD_loop == beginDir.v &&
-      CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` region is "
-        "strictly nested inside a `TEAMS` region."_err_en_US);
-  }
-
-  if (OmpDirectiveSet{
-          llvm::omp::OMPD_teams_loop, llvm::omp::OMPD_target_teams_loop}
-          .test(beginDir.v)) {
-    teamsBindingChecker(
-        "`BIND(TEAMS)` must be specified since the `LOOP` directive is "
-        "combined with a `TEAMS` construct."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::HasInvalidTeamsNesting(
     const llvm::omp::Directive &dir, const parser::CharBlock &source) {
   if (!llvm::omp::nestedTeamsAllowedSet.test(dir)) {
@@ -668,16 +436,6 @@ void OmpStructureChecker::Leave(const parser::OmpDirectiveSpecification &) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpMetadirectiveDirective &x) {
-  EnterDirectiveNest(MetadirectiveNest);
-  PushContextAndClauseSets(x.source, llvm::omp::Directive::OMPD_metadirective);
-}
-
-void OmpStructureChecker::Leave(const parser::OmpMetadirectiveDirective &) {
-  ExitDirectiveNest(MetadirectiveNest);
-  dirContext_.pop_back();
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPConstruct &x) {
   // Simd Construct with Ordered Construct Nesting check
   // We cannot use CurrentDirectiveIsNested() here because
@@ -717,85 +475,6 @@ void OmpStructureChecker::AddEndDirectiveClauses(
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OpenMPLoopConstruct &x) {
-  loopStack_.push_back(&x);
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &beginDir{std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-
-  PushContextAndClauseSets(beginDir.source, beginDir.v);
-
-  // check matching, End directive is optional
-  if (const auto &endLoopDir{
-          std::get<std::optional<parser::OmpEndLoopDirective>>(x.t)}) {
-    const auto &endDir{
-        std::get<parser::OmpLoopDirective>(endLoopDir.value().t)};
-
-    CheckMatching<parser::OmpLoopDirective>(beginDir, endDir);
-
-    AddEndDirectiveClauses(std::get<parser::OmpClauseList>(endLoopDir->t));
-  }
-
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    EnterDirectiveNest(SIMDNest);
-  }
-
-  // Combined target loop constructs are target device constructs. Keep track of
-  // whether any such construct has been visited to later check that REQUIRES
-  // directives for target-related options don't appear after them.
-  if (llvm::omp::allTargetSet.test(beginDir.v)) {
-    deviceConstructFound_ = true;
-  }
-
-  if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    // 2.7.1 do-clause -> private-clause |
-    //                    firstprivate-clause |
-    //                    lastprivate-clause |
-    //                    linear-clause |
-    //                    reduction-clause |
-    //                    schedule-clause |
-    //                    collapse-clause |
-    //                    ordered-clause
-
-    // nesting check
-    HasInvalidWorksharingNesting(
-        beginDir.source, llvm::omp::nestedWorkshareErrSet);
-  }
-  SetLoopInfo(x);
-
-  if (const auto &doConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const auto &doBlock{std::get<parser::Block>(doConstruct->t)};
-    CheckNoBranching(doBlock, beginDir.v, beginDir.source);
-  }
-  CheckLoopItrVariableIsInt(x);
-  CheckAssociatedLoopConstraints(x);
-  HasInvalidDistributeNesting(x);
-  HasInvalidLoopBinding(x);
-  if (CurrentDirectiveIsNested() &&
-      llvm::omp::bottomTeamsSet.test(GetContextParent().directive)) {
-    HasInvalidTeamsNesting(beginDir.v, beginDir.source);
-  }
-  if ((beginDir.v == llvm::omp::Directive::OMPD_distribute_parallel_do_simd) ||
-      (beginDir.v == llvm::omp::Directive::OMPD_distribute_simd)) {
-    CheckDistLinear(x);
-  }
-}
-const parser::Name OmpStructureChecker::GetLoopIndex(
-    const parser::DoConstruct *x) {
-  using Bounds = parser::LoopControl::Bounds;
-  return std::get<Bounds>(x->GetLoopControl()->u).name.thing;
-}
-void OmpStructureChecker::SetLoopInfo(const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-    const parser::DoConstruct *loop{&*loopConstruct};
-    if (loop && loop->IsDoNormal()) {
-      const parser::Name &itrVal{GetLoopIndex(loop)};
-      SetLoopIv(itrVal.symbol);
-    }
-  }
-}
-
 void OmpStructureChecker::CheckIteratorRange(
     const parser::OmpIteratorSpecifier &x) {
   // Check:
@@ -855,108 +534,6 @@ void OmpStructureChecker::CheckIteratorModifier(const parser::OmpIterator &x) {
   }
 }
 
-void OmpStructureChecker::CheckLoopItrVariableIsInt(
-    const parser::OpenMPLoopConstruct &x) {
-  if (const auto &loopConstruct{
-          std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-
-    for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-      if (loop->IsDoNormal()) {
-        const parser::Name &itrVal{GetLoopIndex(loop)};
-        if (itrVal.symbol) {
-          const auto *type{itrVal.symbol->GetType()};
-          if (!type->IsNumeric(TypeCategory::Integer)) {
-            context_.Say(itrVal.source,
-                "The DO loop iteration"
-                " variable must be of the type integer."_err_en_US,
-                itrVal.ToString());
-          }
-        }
-      }
-      // Get the next DoConstruct if block is not empty.
-      const auto &block{std::get<parser::Block>(loop->t)};
-      const auto it{block.begin()};
-      loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                               : nullptr;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckSIMDNest(const parser::OpenMPConstruct &c) {
-  // Check the following:
-  //  The only OpenMP constructs that can be encountered during execution of
-  // a simd region are the `atomic` construct, the `loop` construct, the `simd`
-  // construct and the `ordered` construct with the `simd` clause.
-
-  // Check if the parent context has the SIMD clause
-  // Please note that we use GetContext() instead of GetContextParent()
-  // because PushContextAndClauseSets() has not been called on the
-  // current context yet.
-  // TODO: Check for declare simd regions.
-  bool eligibleSIMD{false};
-  common::visit(
-      common::visitors{
-          // Allow `!$OMP ORDERED SIMD`
-          [&](const parser::OpenMPBlockConstruct &c) {
-            const auto &beginBlockDir{
-                std::get<parser::OmpBeginBlockDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpBlockDirective>(beginBlockDir.t)};
-            if (beginDir.v == llvm::omp::Directive::OMPD_ordered) {
-              const auto &clauses{
-                  std::get<parser::OmpClauseList>(beginBlockDir.t)};
-              for (const auto &clause : clauses.v) {
-                if (std::get_if<parser::OmpClause::Simd>(&clause.u)) {
-                  eligibleSIMD = true;
-                  break;
-                }
-              }
-            }
-          },
-          [&](const parser::OpenMPStandaloneConstruct &c) {
-            if (auto *ssc{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
-                    &c.u)}) {
-              llvm::omp::Directive dirId{ssc->v.DirId()};
-              if (dirId == llvm::omp::Directive::OMPD_ordered) {
-                for (const parser::OmpClause &x : ssc->v.Clauses().v) {
-                  if (x.Id() == llvm::omp::Clause::OMPC_simd) {
-                    eligibleSIMD = true;
-                    break;
-                  }
-                }
-              } else if (dirId == llvm::omp::Directive::OMPD_scan) {
-                eligibleSIMD = true;
-              }
-            }
-          },
-          // Allowing SIMD and loop construct
-          [&](const parser::OpenMPLoopConstruct &c) {
-            const auto &beginLoopDir{
-                std::get<parser::OmpBeginLoopDirective>(c.t)};
-            const auto &beginDir{
-                std::get<parser::OmpLoopDirective>(beginLoopDir.t)};
-            if ((beginDir.v == llvm::omp::Directive::OMPD_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_do_simd) ||
-                (beginDir.v == llvm::omp::Directive::OMPD_loop)) {
-              eligibleSIMD = true;
-            }
-          },
-          [&](const parser::OpenMPAtomicConstruct &c) {
-            // Allow `!$OMP ATOMIC`
-            eligibleSIMD = true;
-          },
-          [&](const auto &c) {},
-      },
-      c.u);
-  if (!eligibleSIMD) {
-    context_.Say(parser::FindSourceLocation(c),
-        "The only OpenMP constructs that can be encountered during execution "
-        "of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, "
-        "the `SIMD` construct, the `SCAN` construct and the `ORDERED` "
-        "construct with the `SIMD` clause."_err_en_US);
-  }
-}
-
 void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   // 2.12.5 Target Construct Restriction
   bool eligibleTarget{true};
@@ -1014,187 +591,6 @@ void OmpStructureChecker::CheckTargetNest(const parser::OpenMPConstruct &c) {
   }
 }
 
-std::int64_t OmpStructureChecker::GetOrdCollapseLevel(
-    const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-  std::int64_t orderedCollapseLevel{1};
-  std::int64_t orderedLevel{1};
-  std::int64_t collapseLevel{1};
-
-  for (const auto &clause : clauseList.v) {
-    if (const auto *collapseClause{
-            std::get_if<parser::OmpClause::Collapse>(&clause.u)}) {
-      if (const auto v{GetIntValue(collapseClause->v)}) {
-        collapseLevel = *v;
-      }
-    }
-    if (const auto *orderedClause{
-            std::get_if<parser::OmpClause::Ordered>(&clause.u)}) {
-      if (const auto v{GetIntValue(orderedClause->v)}) {
-        orderedLevel = *v;
-      }
-    }
-  }
-  if (orderedLevel >= collapseLevel) {
-    orderedCollapseLevel = orderedLevel;
-  } else {
-    orderedCollapseLevel = collapseLevel;
-  }
-  return orderedCollapseLevel;
-}
-
-void OmpStructureChecker::CheckAssociatedLoopConstraints(
-    const parser::OpenMPLoopConstruct &x) {
-  std::int64_t ordCollapseLevel{GetOrdCollapseLevel(x)};
-  AssociatedLoopChecker checker{context_, ordCollapseLevel};
-  parser::Walk(x, checker);
-}
-
-void OmpStructureChecker::CheckDistLinear(
-    const parser::OpenMPLoopConstruct &x) {
-
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauses{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  SymbolSourceMap indexVars;
-
-  // Collect symbols of all the variables from linear clauses
-  for (auto &clause : clauses.v) {
-    if (auto *linearClause{std::get_if<parser::OmpClause::Linear>(&clause.u)}) {
-      auto &objects{std::get<parser::OmpObjectList>(linearClause->v.t)};
-      GetSymbolsInObjectList(objects, indexVars);
-    }
-  }
-
-  if (!indexVars.empty()) {
-    // Get collapse level, if given, to find which loops are "associated."
-    std::int64_t collapseVal{GetOrdCollapseLevel(x)};
-    // Include the top loop if no collapse is specified
-    if (collapseVal == 0) {
-      collapseVal = 1;
-    }
-
-    // Match the loop index variables with the collected symbols from linear
-    // clauses.
-    if (const auto &loopConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      for (const parser::DoConstruct *loop{&*loopConstruct}; loop;) {
-        if (loop->IsDoNormal()) {
-          const parser::Name &itrVal{GetLoopIndex(loop)};
-          if (itrVal.symbol) {
-            // Remove the symbol from the collected set
-            indexVars.erase(&itrVal.symbol->GetUltimate());
-          }
-          collapseVal--;
-          if (collapseVal == 0) {
-            break;
-          }
-        }
-        // Get the next DoConstruct if block is not empty.
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? parser::Unwrap<parser::DoConstruct>(*it)
-                                 : nullptr;
-      }
-    }
-
-    // Show error for the remaining variables
-    for (auto &[symbol, source] : indexVars) {
-      const Symbol &root{GetAssociationRoot(*symbol)};
-      context_.Say(source,
-          "Variable '%s' not allowed in LINEAR clause, only loop iterator can be specified in LINEAR clause of a construct combined with DISTRIBUTE"_err_en_US,
-          root.name());
-    }
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OpenMPLoopConstruct &x) {
-  const auto &beginLoopDir{std::get<parser::OmpBeginLoopDirective>(x.t)};
-  const auto &clauseList{std::get<parser::OmpClauseList>(beginLoopDir.t)};
-
-  // A few semantic checks for InScan reduction are performed below as SCAN
-  // constructs inside LOOP may add the relevant information. Scan reduction is
-  // supported only in loop constructs, so same checks are not applicable to
-  // other directives.
-  using ReductionModifier = parser::OmpReductionModifier;
-  for (const auto &clause : clauseList.v) {
-    if (const auto *reductionClause{
-            std::get_if<parser::OmpClause::Reduction>(&clause.u)}) {
-      auto &modifiers{OmpGetModifiers(reductionClause->v)};
-      auto *maybeModifier{OmpGetUniqueModifier<ReductionModifier>(modifiers)};
-      if (maybeModifier &&
-          maybeModifier->v == ReductionModifier::Value::Inscan) {
-        const auto &objectList{
-            std::get<parser::OmpObjectList>(reductionClause->v.t)};
-        auto checkReductionSymbolInScan = [&](const parser::Name *name) {
-          if (auto &symbol = name->symbol) {
-            if (!symbol->test(Symbol::Flag::OmpInclusiveScan) &&
-                !symbol->test(Symbol::Flag::OmpExclusiveScan)) {
-              context_.Say(name->source,
-                  "List item %s must appear in EXCLUSIVE or "
-                  "INCLUSIVE clause of an "
-                  "enclosed SCAN directive"_err_en_US,
-                  name->ToString());
-            }
-          }
-        };
-        for (const auto &ompObj : objectList.v) {
-          common::visit(
-              common::visitors{
-                  [&](const parser::Designator &designator) {
-                    if (const auto *name{semantics::getDesignatorNameIfDataRef(
-                            designator)}) {
-                      checkReductionSymbolInScan(name);
-                    }
-                  },
-                  [&](const auto &name) { checkReductionSymbolInScan(&name); },
-              },
-              ompObj.u);
-        }
-      }
-    }
-  }
-  if (llvm::omp::allSimdSet.test(GetContext().directive)) {
-    ExitDirectiveNest(SIMDNest);
-  }
-  dirContext_.pop_back();
-
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top{loopStack_.back()};
-  auto *loopc{std::get_if<const parser::OpenMPLoopConstruct *>(&top)};
-  assert(loopc != nullptr && *loopc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndLoopDirective &x) {
-  const auto &dir{std::get<parser::OmpLoopDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  // 2.7.1 end-do -> END DO [nowait-clause]
-  // 2.8.3 end-do-simd -> END DO SIMD [nowait-clause]
-  case llvm::omp::Directive::OMPD_do:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_do);
-    break;
-  case llvm::omp::Directive::OMPD_do_simd:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_do_simd);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpEndLoopDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_do) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_do_simd)) {
-    dirContext_.pop_back();
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
   const auto &beginBlockDir{std::get<parser::OmpBeginBlockDirective>(x.t)};
   const auto &endBlockDir{std::get<parser::OmpEndBlockDirective>(x.t)};
@@ -2335,7 +1731,7 @@ void OmpStructureChecker::Enter(const parser::OpenMPFlushConstruct &x) {
 void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &x) {
   auto &flushList{std::get<std::optional<parser::OmpArgumentList>>(x.v.t)};
 
-  auto isVariableListItemOrCommonBlock{[this](const Symbol &sym) {
+  auto isVariableListItemOrCommonBlock{[](const Symbol &sym) {
     return IsVariableListItem(sym) ||
         sym.detailsIf<semantics::CommonBlockDetails>();
   }};
@@ -2646,1488 +2042,46 @@ void OmpStructureChecker::CheckCancellationNest(
           parser::ToUpperCaseLetters(typeName.str()));
       break;
     default:
-      // This is diagnosed later.
-      return;
-    }
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
-  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
-  ResetPartialContext(dir.source);
-  switch (dir.v) {
-  case llvm::omp::Directive::OMPD_scope:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
-    break;
-  // 2.7.3 end-single-clause -> copyprivate-clause |
-  //                            nowait-clause
-  case llvm::omp::Directive::OMPD_single:
-    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
-    break;
-  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
-  case llvm::omp::Directive::OMPD_workshare:
-    PushContextAndClauseSets(
-        dir.source, llvm::omp::Directive::OMPD_end_workshare);
-    break;
-  default:
-    // no clauses are allowed
-    break;
-  }
-}
-
-// TODO: Verify the popping of dirContext requirement after nowait
-// implementation, as there is an implicit barrier at the end of the worksharing
-// constructs unless a nowait clause is specified. Only OMPD_end_single and
-// end_workshareare popped as they are pushed while entering the
-// EndBlockDirective.
-void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
-  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
-      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
-    dirContext_.pop_back();
-  }
-}
-
-/// parser::Block is a list of executable constructs, parser::BlockConstruct
-/// is Fortran's BLOCK/ENDBLOCK construct.
-/// Strip the outermost BlockConstructs, return the reference to the Block
-/// in the executable part of the innermost of the stripped constructs.
-/// Specifically, if the given `block` has a single entry (it's a list), and
-/// the entry is a BlockConstruct, get the Block contained within. Repeat
-/// this step as many times as possible.
-static const parser::Block &GetInnermostExecPart(const parser::Block &block) {
-  const parser::Block *iter{&block};
-  while (iter->size() == 1) {
-    const parser::ExecutionPartConstruct &ep{iter->front()};
-    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
-      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
-      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
-        iter = &std::get<parser::Block>(bc->value().t);
-        continue;
-      }
-    }
-    break;
-  }
-  return *iter;
-}
-
-// There is no consistent way to get the source of a given ActionStmt, so
-// extract the source information from Statement<ActionStmt> when we can,
-// and keep it around for error reporting in further analyses.
-struct SourcedActionStmt {
-  const parser::ActionStmt *stmt{nullptr};
-  parser::CharBlock source;
-
-  operator bool() const { return stmt != nullptr; }
-};
-
-struct AnalyzedCondStmt {
-  SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
-  parser::CharBlock source;
-  SourcedActionStmt ift, iff;
-};
-
-static SourcedActionStmt GetActionStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return SourcedActionStmt{};
-  }
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    using ActionStmt = parser::Statement<parser::ActionStmt>;
-    if (auto *stmt{std::get_if<ActionStmt>(&exec->u)}) {
-      return SourcedActionStmt{&stmt->statement, stmt->source};
-    }
-  }
-  return SourcedActionStmt{};
-}
-
-static SourcedActionStmt GetActionStmt(const parser::Block &block) {
-  if (block.size() == 1) {
-    return GetActionStmt(&block.front());
-  }
-  return SourcedActionStmt{};
-}
-
-// Compute the `evaluate::Assignment` from parser::ActionStmt. The assumption
-// is that the ActionStmt will be either an assignment or a pointer-assignment,
-// otherwise return std::nullopt.
-// Note: This function can return std::nullopt on [Pointer]AssignmentStmt where
-// the "typedAssignment" is unset. This can happen if there are semantic errors
-// in the purported assignment.
-static std::optional<evaluate::Assignment> GetEvaluateAssignment(
-    const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-  using TypedAssignment = parser::AssignmentStmt::TypedAssignment;
-
-  return common::visit(
-      [](auto &&s) -> std::optional<evaluate::Assignment> {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>) {
-          const TypedAssignment &typed{s.value().typedAssignment};
-          // ForwardOwningPointer                 typedAssignment
-          // `- GenericAssignmentWrapper          ^.get()
-          //    `- std::optional<Assignment>      ^->v
-          return typed.get()->v;
-        } else {
-          return std::nullopt;
-        }
-      },
-      x->u);
-}
-
-// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
-// to separate cases where the source has something that looks like an
-// assignment, but is semantically wrong (diagnosed by general semantic
-// checks), and where the source has some other statement (which we want
-// to report as "should be an assignment").
-static bool IsAssignment(const parser::ActionStmt *x) {
-  if (x == nullptr) {
-    return false;
-  }
-
-  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
-  using PointerAssignmentStmt =
-      common::Indirection<parser::PointerAssignmentStmt>;
-
-  return common::visit(
-      [](auto &&s) -> bool {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        return std::is_same_v<BareS, AssignmentStmt> ||
-            std::is_same_v<BareS, PointerAssignmentStmt>;
-      },
-      x->u);
-}
-
-static std::optional<AnalyzedCondStmt> AnalyzeConditionalStmt(
-    const parser::ExecutionPartConstruct *x) {
-  if (x == nullptr) {
-    return std::nullopt;
-  }
-
-  // Extract the evaluate::Expr from ScalarLogicalExpr.
-  auto getFromLogical{[](const parser::ScalarLogicalExpr &logical) {
-    // ScalarLogicalExpr is Scalar<Logical<common::Indirection<Expr>>>
-    const parser::Expr &expr{logical.thing.thing.value()};
-    return GetEvaluateExpr(expr);
-  }};
-
-  // Recognize either
-  // ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> IfStmt, or
-  // ExecutionPartConstruct -> ExecutableConstruct -> IfConstruct.
-
-  if (auto &&action{GetActionStmt(x)}) {
-    if (auto *ifs{std::get_if<common::Indirection<parser::IfStmt>>(
-            &action.stmt->u)}) {
-      const parser::IfStmt &s{ifs->value()};
-      auto &&maybeCond{
-          getFromLogical(std::get<parser::ScalarLogicalExpr>(s.t))};
-      auto &thenStmt{
-          std::get<parser::UnlabeledStatement<parser::ActionStmt>>(s.t)};
-      if (maybeCond) {
-        return AnalyzedCondStmt{std::move(*maybeCond), action.source,
-            SourcedActionStmt{&thenStmt.statement, thenStmt.source},
-            SourcedActionStmt{}};
-      }
-    }
-    return std::nullopt;
-  }
-
-  if (auto *exec{std::get_if<parser::ExecutableConstruct>(&x->u)}) {
-    if (auto *ifc{
-            std::get_if<common::Indirection<parser::IfConstruct>>(&exec->u)}) {
-      using ElseBlock = parser::IfConstruct::ElseBlock;
-      using ElseIfBlock = parser::IfConstruct::ElseIfBlock;
-      const parser::IfConstruct &s{ifc->value()};
-
-      if (!std::get<std::list<ElseIfBlock>>(s.t).empty()) {
-        // Not expecting any else-if statements.
-        return std::nullopt;
-      }
-      auto &stmt{std::get<parser::Statement<parser::IfThenStmt>>(s.t)};
-      auto &&maybeCond{getFromLogical(
-          std::get<parser::ScalarLogicalExpr>(stmt.statement.t))};
-      if (!maybeCond) {
-        return std::nullopt;
-      }
-
-      if (auto &maybeElse{std::get<std::optional<ElseBlock>>(s.t)}) {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)),
-            GetActionStmt(std::get<parser::Block>(maybeElse->t))};
-        if (result.ift.stmt && result.iff.stmt) {
-          return result;
-        }
-      } else {
-        AnalyzedCondStmt result{std::move(*maybeCond), stmt.source,
-            GetActionStmt(std::get<parser::Block>(s.t)), SourcedActionStmt{}};
-        if (result.ift.stmt) {
-          return result;
-        }
-      }
-    }
-    return std::nullopt;
-  }
-
-  return std::nullopt;
-}
-
-static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
-    parser::CharBlock source) {
-  // Find => in the range, if not found, find = that is not a part of
-  // <=, >=, ==, or /=.
-  auto trim{[](std::string_view v) {
-    const char *begin{v.data()};
-    const char *end{begin + v.size()};
-    while (*begin == ' ' && begin != end) {
-      ++begin;
-    }
-    while (begin != end && end[-1] == ' ') {
-      --end;
-    }
-    assert(begin != end && "Source should not be empty");
-    return parser::CharBlock(begin, end - begin);
-  }};
-
-  std::string_view sv(source.begin(), source.size());
-
-  if (auto where{sv.find("=>")}; where != sv.npos) {
-    std::string_view lhs(sv.data(), where);
-    std::string_view rhs(sv.data() + where + 2, sv.size() - where - 2);
-    return std::make_pair(trim(lhs), trim(rhs));
-  }
-
-  // Go backwards, since all the exclusions above end with a '='.
-  for (size_t next{source.size()}; next > 1; --next) {
-    if (sv[next - 1] == '=' && !llvm::is_contained("<>=/", sv[next - 2])) {
-      std::string_view lhs(sv.data(), next - 1);
-      std::string_view rhs(sv.data() + next, sv.size() - next);
-      return std::make_pair(trim(lhs), trim(rhs));
-    }
-  }
-  llvm_unreachable("Could not find assignment operator");
-}
-
-namespace atomic {
-
-struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
-                                 std::vector<SomeExpr>, false> {
-  using Result = std::vector<SomeExpr>;
-  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
-  DesignatorCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    // Once in a designator, don't traverse it any further (i.e. only
-    // collect top-level designators).
-    auto copy{x};
-    return Result{AsGenericExpr(std::move(copy))};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (moveAppend(v, std::move(results)), ...);
-    return v;
-  }
-};
-
-struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
-  using Base = evaluate::AnyTraverse<VariableFinder>;
-  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
-
-  using Base::operator();
-
-  template <typename T>
-  bool operator()(const evaluate::Designator<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-  template <typename T>
-  bool operator()(const evaluate::FunctionRef<T> &x) const {
-    auto copy{x};
-    return evaluate::AsGenericExpr(std::move(copy)) == var;
-  }
-
-private:
-  const SomeExpr &var;
-};
-} // namespace atomic
-
-static bool IsPointerAssignment(const evaluate::Assignment &x) {
-  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
-      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
-}
-
-namespace operation = Fortran::evaluate::operation;
-
-static bool IsCheckForAssociated(const SomeExpr &cond) {
-  return GetTopLevelOperation(cond).first == operation::Operator::Associated;
-}
-
-static bool HasCommonDesignatorSymbols(
-    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
-  // Compare the designators used in "other" with the designators whose
-  // symbols are given in baseSyms.
-  // This is a part of the check if these two expressions can access the same
-  // storage: if the designators used in them are different enough, then they
-  // will be assumed not to access the same memory.
-  //
-  // Consider an (array element) expression x%y(w%z), the corresponding symbol
-  // vector will be {x, y, w, z} (i.e. the symbols for these names).
-  // Check whether this exact sequence appears anywhere in any the symbol
-  // vector for "other". This will be true for x(y) and x(y+1), so this is
-  // not a sufficient condition, but can be used to eliminate candidates
-  // before doing more exhaustive checks.
-  //
-  // If any of the symbols in this sequence are function names, assume that
-  // there is no storage overlap, mostly because it would be impossible in
-  // general to determine what storage the function will access.
-  // Note: if f is pure, then two calls to f will access the same storage
-  // when called with the same arguments. This check is not done yet.
-
-  if (llvm::any_of(
-          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
-    // If there is a function symbol in the chain then we can't infer much
-    // about the accessed storage.
-    return false;
-  }
-
-  auto isSubsequence{// Is u a subsequence of v.
-      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
-        size_t us{u.size()}, vs{v.size()};
-        if (us > vs) {
-          return false;
-        }
-        for (size_t off{0}; off != vs - us + 1; ++off) {
-          bool same{true};
-          for (size_t i{0}; i != us; ++i) {
-            if (u[i] != v[off + i]) {
-              same = false;
-              break;
-            }
-          }
-          if (same) {
-            return true;
-          }
-        }
-        return false;
-      }};
-
-  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
-  return isSubsequence(baseSyms, otherSyms);
-}
-
-static bool HasCommonTopLevelDesignators(
-    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
-  // Compare designators directly as expressions. This will ensure
-  // that x(y) and x(y+1) are not flagged as overlapping, whereas
-  // the symbol vectors for both of these would be identical.
-  std::vector<SomeExpr> otherDsgs{atomic::DesignatorCollector{}(other)};
-
-  for (auto &s : baseDsgs) {
-    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
-      return true;
-    }
-  }
-  return false;
-}
-
-static const SomeExpr *HasStorageOverlap(
-    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
-  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
-  std::vector<SomeExpr> baseDsgs{atomic::DesignatorCollector{}(base)};
-
-  for (const SomeExpr &expr : exprs) {
-    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
-      continue;
-    }
-    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
-      return &expr;
-    }
-  }
-  return nullptr;
-}
-
-static bool IsMaybeAtomicWrite(const evaluate::Assignment &assign) {
-  // This ignores function calls, so it will accept "f(x) = f(x) + 1"
-  // for example.
-  return HasStorageOverlap(assign.lhs, assign.rhs) == nullptr;
-}
-
-static bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
-  return atomic::VariableFinder{sub}(super);
-}
-
-static void SetExpr(parser::TypedExpr &expr, MaybeExpr value) {
-  if (value) {
-    expr.Reset(new evaluate::GenericExprWrapper(std::move(value)),
-        evaluate::GenericExprWrapper::Deleter);
-  }
-}
-
-static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
-    std::optional<evaluate::Assignment> value) {
-  if (value) {
-    assign.Reset(new evaluate::GenericAssignmentWrapper(std::move(value)),
-        evaluate::GenericAssignmentWrapper::Deleter);
-  }
-}
-
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
-
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
-
-void OmpStructureChecker::CheckStorageOverlap(const SomeExpr &base,
-    llvm::ArrayRef<evaluate::Expr<evaluate::SomeType>> exprs,
-    parser::CharBlock source) {
-  if (auto *expr{HasStorageOverlap(base, exprs)}) {
-    context_.Say(source,
-        "Within atomic operation %s and %s access the same storage"_warn_en_US,
-        base.AsFortran(), expr->AsFortran());
-  }
-}
-
-void OmpStructureChecker::ErrorShouldBeVariable(
-    const MaybeExpr &expr, parser::CharBlock source) {
-  if (expr) {
-    context_.Say(source, "Atomic expression %s should be a variable"_err_en_US,
-        expr->AsFortran());
-  } else {
-    context_.Say(source, "Atomic expression should be a variable"_err_en_US);
-  }
-}
-
-/// Check if `expr` satisfies the following conditions for x and v:
-///
-/// [6.0:189:10-12]
-/// - x and v (as applicable) are either scalar variables or
-///   function references with scalar data pointer result of non-character
-///   intrinsic type or variables that are non-polymorphic scalar pointers
-///   and any length type parameter must be constant.
-void OmpStructureChecker::CheckAtomicType(
-    SymbolRef sym, parser::CharBlock source, std::string_view name) {
-  const DeclTypeSpec *typeSpec{sym->GetType()};
-  if (!typeSpec) {
-    return;
-  }
-
-  if (!IsPointer(sym)) {
-    using Category = DeclTypeSpec::Category;
-    Category cat{typeSpec->category()};
-    if (cat == Category::Character) {
-      context_.Say(source,
-          "Atomic variable %s cannot have CHARACTER type"_err_en_US, name);
-    } else if (cat != Category::Numeric && cat != Category::Logical) {
-      context_.Say(source,
-          "Atomic variable %s should have an intrinsic type"_err_en_US, name);
-    }
-    return;
-  }
-
-  // Variable is a pointer.
-  if (typeSpec->IsPolymorphic()) {
-    context_.Say(source,
-        "Atomic variable %s cannot be a pointer to a polymorphic type"_err_en_US,
-        name);
-    return;
-  }
-
-  // Go over all length parameters, if any, and check if they are
-  // explicit.
-  if (const DerivedTypeSpec *derived{typeSpec->AsDerived()}) {
-    if (llvm::any_of(derived->parameters(), [](auto &&entry) {
-          // "entry" is a map entry
-          return entry.second.isLen() && !entry.second.isExplicit();
-        })) {
-      context_.Say(source,
-          "Atomic variable %s is a pointer to a type with non-constant length parameter"_err_en_US,
-          name);
-    }
-  }
-}
-
-void OmpStructureChecker::CheckAtomicVariable(
-    const SomeExpr &atom, parser::CharBlock source) {
-  if (atom.Rank() != 0) {
-    context_.Say(source, "Atomic variable %s should be a scalar"_err_en_US,
-        atom.AsFortran());
-  }
-
-  std::vector<SomeExpr> dsgs{atomic::DesignatorCollector{}(atom)};
-  assert(dsgs.size() == 1 && "Should have a single top-level designator");
-  evaluate::SymbolVector syms{evaluate::GetSymbolVector(dsgs.front())};
-
-  CheckAtomicType(syms.back(), source, atom.AsFortran());
-
-  if (IsAllocatable(syms.back()) && !IsArrayElement(atom)) {
-    context_.Say(source, "Atomic variable %s cannot be ALLOCATABLE"_err_en_US,
-        atom.AsFortran());
-  }
-}
-
-std::pair<const parser::ExecutionPartConstruct *,
-    const parser::ExecutionPartConstruct *>
-OmpStructureChecker::CheckUpdateCapture(
-    const parser::ExecutionPartConstruct *ec1,
-    const parser::ExecutionPartConstruct *ec2, parser::CharBlock source) {
-  // Decide which statement is the atomic update and which is the capture.
-  //
-  // The two allowed cases are:
-  //   x = ...      atomic-var = ...
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  // or
-  //   ... = x      capture-var = atomic-var (with optional converts)
-  //   x = ...      atomic-var = ...
-  //
-  // The case of 'a = b; b = a' is ambiguous, so pick the first one as capture
-  // (which makes more sense, as it captures the original value of the atomic
-  // variable).
-  //
-  // If the two statements don't fit these criteria, return a pair of default-
-  // constructed values.
-  using ReturnTy = std::pair<const parser::ExecutionPartConstruct *,
-      const parser::ExecutionPartConstruct *>;
-
-  SourcedActionStmt act1{GetActionStmt(ec1)};
-  SourcedActionStmt act2{GetActionStmt(ec2)};
-  auto maybeAssign1{GetEvaluateAssignment(act1.stmt)};
-  auto maybeAssign2{GetEvaluateAssignment(act2.stmt)};
-  if (!maybeAssign1 || !maybeAssign2) {
-    if (!IsAssignment(act1.stmt) || !IsAssignment(act2.stmt)) {
-      context_.Say(source,
-          "ATOMIC UPDATE operation with CAPTURE should contain two assignments"_err_en_US);
-    }
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  auto as1{*maybeAssign1}, as2{*maybeAssign2};
-
-  auto isUpdateCapture{
-      [](const evaluate::Assignment &u, const evaluate::Assignment &c) {
-        return IsSameOrConvertOf(c.rhs, u.lhs);
-      }};
-
-  // Do some checks that narrow down the possible choices for the update
-  // and the capture statements. This will help to emit better diagnostics.
-  // 1. An assignment could be an update (cbu) if the left-hand side is a
-  //    subexpression of the right-hand side.
-  // 2. An assignment could be a capture (cbc) if the right-hand side is
-  //    a variable (or a function ref), with potential type conversions.
-  bool cbu1{IsSubexpressionOf(as1.lhs, as1.rhs)}; // Can as1 be an update?
-  bool cbu2{IsSubexpressionOf(as2.lhs, as2.rhs)}; // Can as2 be an update?
-  bool cbc1{IsVarOrFunctionRef(GetConvertInput(as1.rhs))}; // Can 1 be capture?
-  bool cbc2{IsVarOrFunctionRef(GetConvertInput(as2.rhs))}; // Can 2 be capture?
-
-  // We want to diagnose cases where both assignments cannot be an update,
-  // or both cannot be a capture, as well as cases where either assignment
-  // cannot be any of these two.
-  //
-  // If we organize these boolean values into a matrix
-  //   |cbu1 cbu2|
-  //   |cbc1 cbc2|
-  // then we want to diagnose cases where the matrix has a zero (i.e. "false")
-  // row or column, including the case where everything is zero. All these
-  // cases correspond to the determinant of the matrix being 0, which suggests
-  // that checking the det may be a convenient diagnostic check. There is only
-  // one additional case where the det is 0, which is when the matrix is all 1
-  // ("true"). The "all true" case represents the situation where both
-  // assignments could be an update as well as a capture. On the other hand,
-  // whenever det != 0, the roles of the update and the capture can be
-  // unambiguously assigned to as1 and as2 [1].
-  //
-  // [1] This can be easily verified by hand: there are 10 2x2 matrices with
-  // det = 0, leaving 6 cases where det != 0:
-  //   0 1   0 1   1 0   1 0   1 1   1 1
-  //   1 0   1 1   0 1   1 1   0 1   1 0
-  // In each case the classification is unambiguous.
-
-  //     |cbu1 cbu2|
-  // det |cbc1 cbc2| = cbu1*cbc2 - cbu2*cbc1
-  int det{int(cbu1) * int(cbc2) - int(cbu2) * int(cbc1)};
-
-  auto errorCaptureShouldRead{[&](const parser::CharBlock &source,
-                                  const std::string &expr) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE the right-hand side of the capture assignment should read %s"_err_en_US,
-        expr);
-  }};
-
-  auto errorNeitherWorks{[&]() {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update or the capture"_err_en_US);
-  }};
-
-  auto makeSelectionFromDet{[&](int det) -> ReturnTy {
-    // If det != 0, then the checks unambiguously suggest a specific
-    // categorization.
-    // If det == 0, then this function should be called only if the
-    // checks haven't ruled out any possibility, i.e. when both assigments
-    // could still be either updates or captures.
-    if (det > 0) {
-      // as1 is update, as2 is capture
-      if (isUpdateCapture(as1, as2)) {
-        return std::make_pair(/*Update=*/ec1, /*Capture=*/ec2);
-      } else {
-        errorCaptureShouldRead(act2.source, as1.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else if (det < 0) {
-      // as2 is update, as1 is capture
-      if (isUpdateCapture(as2, as1)) {
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      } else {
-        errorCaptureShouldRead(act1.source, as2.lhs.AsFortran());
-        return std::make_pair(nullptr, nullptr);
-      }
-    } else {
-      bool updateFirst{isUpdateCapture(as1, as2)};
-      bool captureFirst{isUpdateCapture(as2, as1)};
-      if (updateFirst && captureFirst) {
-        // If both assignment could be the update and both could be the
-        // capture, emit a warning about the ambiguity.
-        context_.Say(act1.source,
-            "In ATOMIC UPDATE operation with CAPTURE either statement could be the update and the capture, assuming the first one is the capture statement"_warn_en_US);
-        return std::make_pair(/*Update=*/ec2, /*Capture=*/ec1);
-      }
-      if (updateFirst != captureFirst) {
-        const parser::ExecutionPartConstruct *upd{updateFirst ? ec1 : ec2};
-        const parser::ExecutionPartConstruct *cap{captureFirst ? ec1 : ec2};
-        return std::make_pair(upd, cap);
-      }
-      assert(!updateFirst && !captureFirst);
-      errorNeitherWorks();
-      return std::make_pair(nullptr, nullptr);
-    }
-  }};
-
-  if (det != 0 || (cbu1 && cbu2 && cbc1 && cbc2)) {
-    return makeSelectionFromDet(det);
-  }
-  assert(det == 0 && "Prior checks should have covered det != 0");
-
-  // If neither of the statements is an RMW update, it could still be a
-  // "write" update. Pretty much any assignment can be a write update, so
-  // recompute det with cbu1 = cbu2 = true.
-  if (int writeDet{int(cbc2) - int(cbc1)}; writeDet || (cbc1 && cbc2)) {
-    return makeSelectionFromDet(writeDet);
-  }
-
-  // It's only errors from here on.
-
-  if (!cbu1 && !cbu2 && !cbc1 && !cbc2) {
-    errorNeitherWorks();
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // The remaining cases are that
-  // - no candidate for update, or for capture,
-  // - one of the assigments cannot be anything.
-
-  if (!cbu1 && !cbu2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the update"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  } else if (!cbc1 && !cbc2) {
-    context_.Say(source,
-        "In ATOMIC UPDATE operation with CAPTURE neither statement could be the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  if ((!cbu1 && !cbc1) || (!cbu2 && !cbc2)) {
-    auto &src = (!cbu1 && !cbc1) ? act1.source : act2.source;
-    context_.Say(src,
-        "In ATOMIC UPDATE operation with CAPTURE the statement could be neither the update nor the capture"_err_en_US);
-    return std::make_pair(nullptr, nullptr);
-  }
-
-  // All cases should have been covered.
-  llvm_unreachable("Unchecked condition");
-}
-
-void OmpStructureChecker::CheckAtomicCaptureAssignment(
-    const evaluate::Assignment &capture, const SomeExpr &atom,
-    parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &cap{capture.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, rsrc);
-    // This part should have been checked prior to calling this function.
-    assert(*GetConvertInput(capture.rhs) == atom &&
-        "This cannot be a capture assignment");
-    CheckStorageOverlap(atom, {cap}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicReadAssignment(
-    const evaluate::Assignment &read, parser::CharBlock source) {
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-
-  if (auto maybe{GetConvertInput(read.rhs)}) {
-    const SomeExpr &atom{*maybe};
-
-    if (!IsVarOrFunctionRef(atom)) {
-      ErrorShouldBeVariable(atom, rsrc);
-    } else {
-      CheckAtomicVariable(atom, rsrc);
-      CheckStorageOverlap(atom, {read.lhs}, source);
-    }
-  } else {
-    ErrorShouldBeVariable(read.rhs, rsrc);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicWriteAssignment(
-    const evaluate::Assignment &write, parser::CharBlock source) {
-  // [6.0:190:13-15]
-  // A write structured block is write-statement, a write statement that has
-  // one of the following forms:
-  //   x = expr
-  //   x => expr
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{write.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-  } else {
-    CheckAtomicVariable(atom, lsrc);
-    CheckStorageOverlap(atom, {write.rhs}, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
-    const evaluate::Assignment &update, parser::CharBlock source) {
-  // [6.0:191:1-7]
-  // An update structured block is update-statement, an update statement
-  // that has one of the following forms:
-  //   x = x operator expr
-  //   x = expr operator x
-  //   x = intrinsic-procedure-name (x)
-  //   x = intrinsic-procedure-name (x, expr-list)
-  //   x = intrinsic-procedure-name (expr-list, x)
-  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
-  const SomeExpr &atom{update.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, rsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, lsrc);
-
-  std::pair<operation::Operator, std::vector<SomeExpr>> top{
-      operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
-    top = GetTopLevelOperation(*maybeInput);
-  }
-  switch (top.first) {
-  case operation::Operator::Add:
-  case operation::Operator::Sub:
-  case operation::Operator::Mul:
-  case operation::Operator::Div:
-  case operation::Operator::And:
-  case operation::Operator::Or:
-  case operation::Operator::Eqv:
-  case operation::Operator::Neqv:
-  case operation::Operator::Min:
-  case operation::Operator::Max:
-  case operation::Operator::Identity:
-    break;
-  case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  case operation::Operator::Constant:
-  case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
-  }
-  // Check how many times `atom` occurs as an argument, if it's a subexpression
-  // of an argument, and collect the non-atom arguments.
-  std::vector<SomeExpr> nonAtom;
-  MaybeExpr subExpr;
-  auto atomCount{[&]() {
-    int count{0};
-    for (const SomeExpr &arg : top.second) {
-      if (IsSameOrConvertOf(arg, atom)) {
-        ++count;
-      } else {
-        if (!subExpr && IsSubexpressionOf(atom, arg)) {
-          subExpr = arg;
-        }
-        nonAtom.push_back(arg);
-      }
-    }
-    return count;
-  }()};
-
-  bool hasError{false};
-  if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
-    hasError = true;
-  }
-  if (top.first == operation::Operator::Identity) {
-    // This is "x = y".
-    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
-      hasError = true;
-    }
-  } else {
-    if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
-      hasError = true;
-    }
-  }
-
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
-    const SomeExpr &cond, parser::CharBlock condSource,
-    const evaluate::Assignment &assign, parser::CharBlock assignSource) {
-  auto [alsrc, arsrc]{SplitAssignmentSource(assignSource)};
-  const SomeExpr &atom{assign.lhs};
-
-  if (!IsVarOrFunctionRef(atom)) {
-    ErrorShouldBeVariable(atom, arsrc);
-    // Skip other checks.
-    return;
-  }
-
-  CheckAtomicVariable(atom, alsrc);
-
-  auto top{GetTopLevelOperation(cond)};
-  // Missing arguments to operations would have been diagnosed by now.
-
-  switch (top.first) {
-  case operation::Operator::Associated:
-    if (atom != top.second.front()) {
-      context_.Say(assignSource,
-          "The pointer argument to ASSOCIATED must be same as the target of the assignment"_err_en_US);
-    }
-    break;
-  // x equalop e | e equalop x  (allowing "e equalop x" is an extension)
-  case operation::Operator::Eq:
-  case operation::Operator::Eqv:
-  // x ordop expr | expr ordop x
-  case operation::Operator::Lt:
-  case operation::Operator::Gt: {
-    const SomeExpr &arg0{top.second[0]};
-    const SomeExpr &arg1{top.second[1]};
-    if (IsSameOrConvertOf(arg0, atom)) {
-      CheckStorageOverlap(atom, {arg1}, condSource);
-    } else if (IsSameOrConvertOf(arg1, atom)) {
-      CheckStorageOverlap(atom, {arg0}, condSource);
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
-      context_.Say(assignSource,
-          "An argument of the %s operator should be the target of the assignment"_err_en_US,
-          operation::ToString(top.first));
-    }
-    break;
-  }
-  case operation::Operator::Identity:
-  case operation::Operator::True:
-  case operation::Operator::False:
-    break;
-  default:
-    assert(
-        top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(condSource,
-        "The %s operator is not a valid condition for ATOMIC operation"_err_en_US,
-        operation::ToString(top.first));
-    break;
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateStmt(
-    const AnalyzedCondStmt &update, parser::CharBlock source) {
-  // The condition/statements must be:
-  // - cond: x equalop e      ift: x =  d     iff: -
-  // - cond: x ordop expr     ift: x =  expr  iff: -  (+ commute ordop)
-  // - cond: associated(x)    ift: x => expr  iff: -
-  // - cond: associated(x, e) ift: x => expr  iff: -
-
-  // The if-true statement must be present, and must be an assignment.
-  auto maybeAssign{GetEvaluateAssignment(update.ift.stmt)};
-  if (!maybeAssign) {
-    if (update.ift.stmt && !IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE the update statement should be an assignment"_err_en_US);
-    } else {
-      context_.Say(
-          source, "Invalid body of ATOMIC UPDATE COMPARE operation"_err_en_US);
-    }
-    return;
-  }
-  const evaluate::Assignment assign{*maybeAssign};
-  const SomeExpr &atom{assign.lhs};
-
-  CheckAtomicConditionalUpdateAssignment(
-      update.cond, update.source, assign, update.ift.source);
-
-  CheckStorageOverlap(atom, {assign.rhs}, update.ift.source);
-
-  if (update.iff) {
-    context_.Say(update.iff.source,
-        "In ATOMIC UPDATE COMPARE the update statement should not have an ELSE branch"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdateOnly(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC UPDATE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdate(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // Allowable forms are (single-statement):
-  // - if ...
-  // - x = (... ? ... : x)
-  // and two-statement:
-  // - r = cond ; if (r) ...
-
-  const parser::ExecutionPartConstruct *ust{nullptr}; // update
-  const parser::ExecutionPartConstruct *cst{nullptr}; // condition
-
-  if (body.size() == 1) {
-    ust = &body.front();
-  } else if (body.size() == 2) {
-    cst = &body.front();
-    ust = &body.back();
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE operation should contain one or two statements"_err_en_US);
-    return;
-  }
-
-  // Flang doesn't support conditional-expr yet, so all update statements
-  // are if-statements.
-
-  // IfStmt:        if (...) ...
-  // IfConstruct:   if (...) then ... endif
-  auto maybeUpdate{AnalyzeConditionalStmt(ust)};
-  if (!maybeUpdate) {
-    context_.Say(source,
-        "In ATOMIC UPDATE COMPARE the update statement should be a conditional statement"_err_en_US);
-    return;
-  }
-
-  AnalyzedCondStmt &update{*maybeUpdate};
-
-  if (SourcedActionStmt action{GetActionStmt(cst)}) {
-    // The "condition" statement must be `r = cond`.
-    if (auto maybeCond{GetEvaluateAssignment(action.stmt)}) {
-      if (maybeCond->lhs != update.cond) {
-        context_.Say(update.source,
-            "In ATOMIC UPDATE COMPARE the conditional statement must use %s as the condition"_err_en_US,
-            maybeCond->lhs.AsFortran());
-      } else {
-        // If it's "r = ...; if (r) ..." then put the original condition
-        // in `update`.
-        update.cond = maybeCond->rhs;
-      }
-    } else {
-      context_.Say(action.source,
-          "In ATOMIC UPDATE COMPARE with two statements the first statement should compute the condition"_err_en_US);
-    }
-  }
-
-  evaluate::Assignment assign{*GetEvaluateAssignment(update.ift.stmt)};
-
-  CheckAtomicConditionalUpdateStmt(update, source);
-  if (IsCheckForAssociated(update.cond)) {
-    if (!IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment should be a pointer-assignment when the condition is ASSOCIATED"_err_en_US);
-    }
-  } else {
-    if (IsPointerAssignment(assign)) {
-      context_.Say(source,
-          "The assignment cannot be a pointer-assignment except when the condition is ASSOCIATED"_err_en_US);
-    }
-  }
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
-}
-
-void OmpStructureChecker::CheckAtomicUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  if (body.size() != 2) {
-    context_.Say(source,
-        "ATOMIC UPDATE operation with CAPTURE should contain two statements"_err_en_US);
-    return;
-  }
-
-  auto [uec, cec]{CheckUpdateCapture(&body.front(), &body.back(), source)};
-  if (!uec || !cec) {
-    // Diagnostics already emitted.
-    return;
-  }
-  SourcedActionStmt uact{GetActionStmt(uec)};
-  SourcedActionStmt cact{GetActionStmt(cec)};
-  // The "dereferences" of std::optional are guaranteed to be valid after
-  // CheckUpdateCapture.
-  evaluate::Assignment update{*GetEvaluateAssignment(uact.stmt)};
-  evaluate::Assignment capture{*GetEvaluateAssignment(cact.stmt)};
-
-  const SomeExpr &atom{update.lhs};
-
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  int action;
-
-  if (IsMaybeAtomicWrite(update)) {
-    action = Analysis::Write;
-    CheckAtomicWriteAssignment(update, uact.source);
-  } else {
-    action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
-  }
-  CheckAtomicCaptureAssignment(capture, atom, cact.source);
-
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
-    context_.Say(cact.source,
-        "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-    return;
-  }
-
-  if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
-  } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
-    const parser::OpenMPAtomicConstruct &x, const parser::Block &body,
-    parser::CharBlock source) {
-  // There are two different variants of this:
-  // (1) conditional-update and capture separately:
-  //     This form only allows single-statement updates, i.e. the update
-  //     form "r = cond; if (r) ..." is not allowed.
-  // (2) conditional-update combined with capture in a single statement:
-  //     This form does allow the condition to be calculated separately,
-  //     i.e. "r = cond; if (r) ...".
-  // Regardless of what form it is, the actual update assignment is a
-  // proper write, i.e. "x = d", where d does not depend on x.
-
-  AnalyzedCondStmt update;
-  SourcedActionStmt capture;
-  bool captureAlways{true}, captureFirst{true};
-
-  auto extractCapture{[&]() {
-    capture = update.iff;
-    captureAlways = false;
-    update.iff = SourcedActionStmt{};
-  }};
-
-  auto classifyNonUpdate{[&](const SourcedActionStmt &action) {
-    // The non-update statement is either "r = cond" or the capture.
-    if (auto maybeAssign{GetEvaluateAssignment(action.stmt)}) {
-      if (update.cond == maybeAssign->lhs) {
-        // If this is "r = cond; if (r) ...", then update the condition.
-        update.cond = maybeAssign->rhs;
-        update.source = action.source;
-        // In this form, the update and the capture are combined into
-        // an IF-THEN-ELSE statement.
-        extractCapture();
-      } else {
-        // Assume this is the capture-statement.
-        capture = action;
-      }
-    }
-  }};
-
-  if (body.size() == 2) {
-    // This could be
-    // - capture; conditional-update (in any order), or
-    // - r = cond; if (r) capture-update
-    const parser::ExecutionPartConstruct *st1{&body.front()};
-    const parser::ExecutionPartConstruct *st2{&body.back()};
-    // In either case, the conditional statement can be analyzed by
-    // AnalyzeConditionalStmt, whereas the other statement cannot.
-    if (auto maybeUpdate1{AnalyzeConditionalStmt(st1)}) {
-      update = *maybeUpdate1;
-      classifyNonUpdate(GetActionStmt(st2));
-      captureFirst = false;
-    } else if (auto maybeUpdate2{AnalyzeConditionalStmt(st2)}) {
-      update = *maybeUpdate2;
-      classifyNonUpdate(GetActionStmt(st1));
-    } else {
-      // None of the statements are conditional, this rules out the
-      // "r = cond; if (r) ..." and the "capture + conditional-update"
-      // variants. This could still be capture + write (which is classified
-      // as conditional-update-capture in the spec).
-      auto [uec, cec]{CheckUpdateCapture(st1, st2, source)};
-      if (!uec || !cec) {
-        // Diagnostics already emitted.
-        return;
-      }
-      SourcedActionStmt uact{GetActionStmt(uec)};
-      SourcedActionStmt cact{GetActionStmt(cec)};
-      update.ift = uact;
-      capture = cact;
-      if (uec == st1) {
-        captureFirst = false;
-      }
-    }
-  } else if (body.size() == 1) {
-    if (auto maybeUpdate{AnalyzeConditionalStmt(&body.front())}) {
-      update = *maybeUpdate;
-      // This is the form with update and capture combined into an IF-THEN-ELSE
-      // statement. The capture-statement is always the ELSE branch.
-      extractCapture();
-    } else {
-      goto invalid;
-    }
-  } else {
-    context_.Say(source,
-        "ATOMIC UPDATE COMPARE CAPTURE operation should contain one or two statements"_err_en_US);
-    return;
-  invalid:
-    context_.Say(source,
-        "Invalid body of ATOMIC UPDATE COMPARE CAPTURE operation"_err_en_US);
-    return;
-  }
-
-  // The update must have a form `x = d` or `x => d`.
-  if (auto maybeWrite{GetEvaluateAssignment(update.ift.stmt)}) {
-    const SomeExpr &atom{maybeWrite->lhs};
-    CheckAtomicWriteAssignment(*maybeWrite, update.ift.source);
-    if (auto maybeCapture{GetEvaluateAssignment(capture.stmt)}) {
-      CheckAtomicCaptureAssignment(*maybeCapture, atom, capture.source);
-
-      if (IsPointerAssignment(*maybeWrite) !=
-          IsPointerAssignment(*maybeCapture)) {
-        context_.Say(capture.source,
-            "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
-        return;
-      }
-    } else {
-      if (!IsAssignment(capture.stmt)) {
-        context_.Say(capture.source,
-            "In ATOMIC UPDATE COMPARE CAPTURE the capture statement should be an assignment"_err_en_US);
-      }
-      return;
-    }
-  } else {
-    if (!IsAssignment(update.ift.stmt)) {
-      context_.Say(update.ift.source,
-          "In ATOMIC UPDATE COMPARE CAPTURE the update statement should be an assignment"_err_en_US);
-    }
-    return;
-  }
-
-  // update.iff should be empty here, the capture statement should be
-  // stored in "capture".
-
-  // Fill out the analysis in the AST node.
-  using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  bool condUnused{std::visit(
-      [](auto &&s) {
-        using BareS = llvm::remove_cvref_t<decltype(s)>;
-        if constexpr (std::is_same_v<BareS, evaluate::NullPointer>) {
-          return true;
-        } else {
-          return false;
-        }
-      },
-      update.cond.u)};
-
-  int updateWhen{!condUnused ? Analysis::IfTrue : 0};
-  int captureWhen{!captureAlways ? Analysis::IfFalse : 0};
-
-  evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
-  evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
-
-  if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
-  } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
-  }
-}
-
-void OmpStructureChecker::CheckAtomicRead(
-    const parser::OpenMPAtomicConstruct &x) {
-  // [6.0:190:5-7]
-  // A read structured block is read-statement, a read statement that has one
-  // of the following forms:
-  //   v = x
-  //   v => x
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Read cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC READ cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeRead{GetEvaluateAssignment(action.stmt)}) {
-      CheckAtomicReadAssignment(*maybeRead, action.source);
-
-      if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
-        const SomeExpr &atom{*maybe};
-        using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
-      }
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC READ operation should be an assignment"_err_en_US);
+      // This is diagnosed later.
+      return;
     }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC READ operation should have a single statement"_err_en_US);
   }
 }
 
-void OmpStructureChecker::CheckAtomicWrite(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &block{std::get<parser::Block>(x.t)};
-
-  // Write cannot be conditional or have a capture statement.
-  if (x.IsCompare() || x.IsCapture()) {
-    context_.Say(dirSpec.source,
-        "ATOMIC WRITE cannot have COMPARE or CAPTURE clauses"_err_en_US);
-    return;
-  }
-
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (body.size() == 1) {
-    SourcedActionStmt action{GetActionStmt(&body.front())};
-    if (auto maybeWrite{GetEvaluateAssignment(action.stmt)}) {
-      const SomeExpr &atom{maybeWrite->lhs};
-      CheckAtomicWriteAssignment(*maybeWrite, action.source);
-
-      using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
-    } else if (!IsAssignment(action.stmt)) {
-      context_.Say(
-          x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
-    }
-  } else {
-    context_.Say(x.source,
-        "ATOMIC WRITE operation should have a single statement"_err_en_US);
-  }
-}
-
-void OmpStructureChecker::CheckAtomicUpdate(
-    const parser::OpenMPAtomicConstruct &x) {
-  auto &block{std::get<parser::Block>(x.t)};
-
-  bool isConditional{x.IsCompare()};
-  bool isCapture{x.IsCapture()};
-  const parser::Block &body{GetInnermostExecPart(block)};
-
-  if (isConditional && isCapture) {
-    CheckAtomicConditionalUpdateCapture(x, body, x.source);
-  } else if (isConditional) {
-    CheckAtomicConditionalUpdate(x, body, x.source);
-  } else if (isCapture) {
-    CheckAtomicUpdateCapture(x, body, x.source);
-  } else { // update-only
-    CheckAtomicUpdateOnly(x, body, x.source);
-  }
-}
-
-void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
-  if (visitedAtomicSource_.empty())
-    visitedAtomicSource_ = x.source;
-
-  // All of the following groups have the "exclusive" property, i.e. at
-  // most one clause from each group is allowed.
-  // The exclusivity-checking code should eventually be unified for all
-  // clauses, with clause groups defined in OMP.td.
-  std::array atomic{llvm::omp::Clause::OMPC_read,
-      llvm::omp::Clause::OMPC_update, llvm::omp::Clause::OMPC_write};
-  std::array memoryOrder{llvm::omp::Clause::OMPC_acq_rel,
-      llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_relaxed,
-      llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_seq_cst};
-
-  auto checkExclusive{[&](llvm::ArrayRef<llvm::omp::Clause> group,
-                          std::string_view name,
-                          const parser::OmpClauseList &clauses) {
-    const parser::OmpClause *present{nullptr};
-    for (const parser::OmpClause &clause : clauses.v) {
-      llvm::omp::Clause id{clause.Id()};
-      if (!llvm::is_contained(group, id)) {
-        continue;
-      }
-      if (present == nullptr) {
-        present = &clause;
-        continue;
-      } else if (id == present->Id()) {
-        // Ignore repetitions of the same clause, those will be diagnosed
-        // separately.
-        continue;
-      }
-      parser::MessageFormattedText txt(
-          "At most one clause from the '%s' group is allowed on ATOMIC construct"_err_en_US,
-          name.data());
-      parser::Message message(clause.source, txt);
-      message.Attach(present->source,
-          "Previous clause from this group provided here"_en_US);
-      context_.Say(std::move(message));
-      return;
-    }
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &dir{std::get<parser::OmpDirectiveName>(dirSpec.t)};
-  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_atomic);
-  llvm::omp::Clause kind{x.GetKind()};
-
-  checkExclusive(atomic, "atomic", dirSpec.Clauses());
-  checkExclusive(memoryOrder, "memory-order", dirSpec.Clauses());
-
-  switch (kind) {
-  case llvm::omp::Clause::OMPC_read:
-    CheckAtomicRead(x);
+void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
+  const auto &dir{std::get<parser::OmpBlockDirective>(x.t)};
+  ResetPartialContext(dir.source);
+  switch (dir.v) {
+  case llvm::omp::Directive::OMPD_scope:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_scope);
     break;
-  case llvm::omp::Clause::OMPC_write:
-    CheckAtomicWrite(x);
+  // 2.7.3 end-single-clause -> copyprivate-clause |
+  //                            nowait-clause
+  case llvm::omp::Directive::OMPD_single:
+    PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_end_single);
     break;
-  case llvm::omp::Clause::OMPC_update:
-    CheckAtomicUpdate(x);
+  // 2.7.4 end-workshare -> END WORKSHARE [nowait-clause]
+  case llvm::omp::Directive::OMPD_workshare:
+    PushContextAndClauseSets(
+        dir.source, llvm::omp::Directive::OMPD_end_workshare);
     break;
   default:
+    // no clauses are allowed
     break;
   }
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
-  dirContext_.pop_back();
+// TODO: Verify the popping of dirContext requirement after nowait
+// implementation, as there is an implicit barrier at the end of the worksharing
+// constructs unless a nowait clause is specified. Only OMPD_end_single and
+// end_workshareare popped as they are pushed while entering the
+// EndBlockDirective.
+void OmpStructureChecker::Leave(const parser::OmpEndBlockDirective &x) {
+  if ((GetContext().directive == llvm::omp::Directive::OMPD_end_scope) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_single) ||
+      (GetContext().directive == llvm::omp::Directive::OMPD_end_workshare)) {
+    dirContext_.pop_back();
+  }
 }
 
 // Clauses
@@ -5195,102 +3149,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
-  unsigned version{context_.langOptions().OpenMPVersion};
-  llvm::omp::Directive dir{GetContext().directive};
-  parser::CharBlock clauseSource{GetContext().clauseSource};
-  const parser::OmpLinearModifier *linearMod{nullptr};
-
-  SymbolSourceMap symbols;
-  auto &objects{std::get<parser::OmpObjectList>(x.v.t)};
-  CheckCrayPointee(objects, "LINEAR", false);
-  GetSymbolsInObjectList(objects, symbols);
-
-  auto CheckIntegerNoRef{[&](const Symbol *symbol, parser::CharBlock source) {
-    if (!symbol->GetType()->IsNumeric(TypeCategory::Integer)) {
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      context_.Say(source,
-          "The list item '%s' specified without the REF '%s' must be of INTEGER type"_err_en_US,
-          symbol->name(), desc.name.str());
-    }
-  }};
-
-  if (OmpVerifyModifiers(x.v, llvm::omp::OMPC_linear, clauseSource, context_)) {
-    auto &modifiers{OmpGetModifiers(x.v)};
-    linearMod = OmpGetUniqueModifier<parser::OmpLinearModifier>(modifiers);
-    if (linearMod) {
-      // 2.7 Loop Construct Restriction
-      if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(dir)) {
-        context_.Say(clauseSource,
-            "A modifier may not be specified in a LINEAR clause on the %s directive"_err_en_US,
-            ContextDirectiveAsFortran());
-        return;
-      }
-
-      auto &desc{OmpGetDescriptor<parser::OmpLinearModifier>()};
-      for (auto &[symbol, source] : symbols) {
-        if (linearMod->v != parser::OmpLinearModifier::Value::Ref) {
-          CheckIntegerNoRef(symbol, source);
-        } else {
-          if (!IsAllocatable(*symbol) && !IsAssumedShape(*symbol) &&
-              !IsPolymorphic(*symbol)) {
-            context_.Say(source,
-                "The list item `%s` specified with the REF '%s' must be polymorphic variable, assumed-shape array, or a variable with the `ALLOCATABLE` attribute"_err_en_US,
-                symbol->name(), desc.name.str());
-          }
-        }
-        if (linearMod->v == parser::OmpLinearModifier::Value::Ref ||
-            linearMod->v == parser::OmpLinearModifier::Value::Uval) {
-          if (!IsDummy(*symbol) || IsValue(*symbol)) {
-            context_.Say(source,
-                "If the `%s` is REF or UVAL, the list item '%s' must be a dummy argument without the VALUE attribute"_err_en_US,
-                desc.name.str(), symbol->name());
-          }
-        }
-      } // for (symbol, source)
-
-      if (version >= 52 && !std::get</*PostModified=*/bool>(x.v.t)) {
-        context_.Say(OmpGetModifierSource(modifiers, linearMod),
-            "The 'modifier(<list>)' syntax is deprecated in %s, use '<list> : modifier' instead"_warn_en_US,
-            ThisVersion(version));
-      }
-    }
-  }
-
-  // OpenMP 5.2: Ordered clause restriction
-  if (const auto *clause{
-          FindClause(GetContext(), llvm::omp::Clause::OMPC_ordered)}) {
-    const auto &orderedClause{std::get<parser::OmpClause::Ordered>(clause->u)};
-    if (orderedClause.v) {
-      return;
-    }
-  }
-
-  // OpenMP 5.2: Linear clause Restrictions
-  for (auto &[symbol, source] : symbols) {
-    if (!linearMod) {
-      // Already checked this with the modifier present.
-      CheckIntegerNoRef(symbol, source);
-    }
-    if (dir == llvm::omp::Directive::OMPD_declare_simd && !IsDummy(*symbol)) {
-      context_.Say(source,
-          "The list item `%s` must be a dummy argument"_err_en_US,
-          symbol->name());
-    }
-    if (IsPointer(*symbol) || symbol->test(Symbol::Flag::CrayPointer)) {
-      context_.Say(source,
-          "The list item `%s` in a LINEAR clause must not be Cray Pointer or a variable with POINTER attribute"_err_en_US,
-          symbol->name());
-    }
-    if (FindCommonBlockContaining(*symbol)) {
-      context_.Say(source,
-          "'%s' is a common block name and must not appear in an LINEAR clause"_err_en_US,
-          symbol->name());
-    }
-  }
-}
-
 void OmpStructureChecker::Enter(const parser::OmpClause::Detach &x) {
   unsigned version{context_.langOptions().OpenMPVersion};
   if (version >= 52) {
@@ -5999,503 +3857,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::OmpxBare &x) {
   }
 }
 
-void OmpStructureChecker::Enter(const parser::OmpClause::When &x) {
-  CheckAllowedClause(llvm::omp::Clause::OMPC_when);
-  OmpVerifyModifiers(
-      x.v, llvm::omp::OMPC_when, GetContext().clauseSource, context_);
-}
-
-void OmpStructureChecker::Enter(const parser::OmpContextSelector &ctx) {
-  EnterDirectiveNest(ContextSelectorNest);
-
-  using SetName = parser::OmpTraitSetSelectorName;
-  std::map<SetName::Value, const SetName *> visited;
-
-  for (const parser::OmpTraitSetSelector &traitSet : ctx.v) {
-    auto &name{std::get<SetName>(traitSet.t)};
-    auto [prev, unique]{visited.insert(std::make_pair(name.v, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait set name %s in a context specifier"_err_en_US,
-          showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait set %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-    CheckTraitSetSelector(traitSet);
-  }
-}
-
-void OmpStructureChecker::Leave(const parser::OmpContextSelector &) {
-  ExitDirectiveNest(ContextSelectorNest);
-}
-
-const std::list<parser::OmpTraitProperty> &
-OmpStructureChecker::GetTraitPropertyList(
-    const parser::OmpTraitSelector &trait) {
-  static const std::list<parser::OmpTraitProperty> empty{};
-  auto &[_, maybeProps]{trait.t};
-  if (maybeProps) {
-    using PropertyList = std::list<parser::OmpTraitProperty>;
-    return std::get<PropertyList>(maybeProps->t);
-  } else {
-    return empty;
-  }
-}
-
-std::optional<llvm::omp::Clause> OmpStructureChecker::GetClauseFromProperty(
-    const parser::OmpTraitProperty &property) {
-  using MaybeClause = std::optional<llvm::omp::Clause>;
-
-  // The parser for OmpClause will only succeed if the clause was
-  // given with all required arguments.
-  // If this is a string or complex extension with a clause name,
-  // treat it as a clause and let the trait checker deal with it.
-
-  auto getClauseFromString{[&](const std::string &s) -> MaybeClause {
-    auto id{llvm::omp::getOpenMPClauseKind(parser::ToLowerCaseLetters(s))};
-    if (id != llvm::omp::Clause::OMPC_unknown) {
-      return id;
-    } else {
-      return std::nullopt;
-    }
-  }};
-
-  return common::visit( //
-      common::visitors{
-          [&](const parser::OmpTraitPropertyName &x) -> MaybeClause {
-            return getClauseFromString(x.v);
-          },
-          [&](const common::Indirection<parser::OmpClause> &x) -> MaybeClause {
-            return x.value().Id();
-          },
-          [&](const parser::ScalarExpr &x) -> MaybeClause {
-            return std::nullopt;
-          },
-          [&](const parser::OmpTraitPropertyExtension &x) -> MaybeClause {
-            using ExtProperty = parser::OmpTraitPropertyExtension;
-            if (auto *name{std::get_if<parser::OmpTraitPropertyName>(&x.u)}) {
-              return getClauseFromString(name->v);
-            } else if (auto *cpx{std::get_if<ExtProperty::Complex>(&x.u)}) {
-              return getClauseFromString(
-                  std::get<parser::OmpTraitPropertyName>(cpx->t).v);
-            }
-            return std::nullopt;
-          },
-      },
-      property.u);
-}
-
-void OmpStructureChecker::CheckTraitSelectorList(
-    const std::list<parser::OmpTraitSelector> &traits) {
-  // [6.0:322:20]
-  // Each trait-selector-name may only be specified once in a trait selector
-  // set.
-
-  // Cannot store OmpTraitSelectorName directly, because it's not copyable.
-  using TraitName = parser::OmpTraitSelectorName;
-  using BareName = decltype(TraitName::u);
-  std::map<BareName, const TraitName *> visited;
-
-  for (const parser::OmpTraitSelector &trait : traits) {
-    auto &name{std::get<TraitName>(trait.t)};
-
-    auto [prev, unique]{visited.insert(std::make_pair(name.u, &name))};
-    if (!unique) {
-      std::string showName{parser::ToUpperCaseLetters(name.ToString())};
-      parser::MessageFormattedText txt(
-          "Repeated trait name %s in a trait set"_err_en_US, showName);
-      parser::Message message(name.source, txt);
-      message.Attach(prev->second->source,
-          "Previous trait %s provided here"_en_US, showName);
-      context_.Say(std::move(message));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSetSelector(
-    const parser::OmpTraitSetSelector &traitSet) {
-
-  // Trait Set      |           Allowed traits | D-traits | X-traits | Score |
-  //
-  // Construct      |     Simd, directive-name |      Yes |       No |    No |
-  // Device         |          Arch, Isa, Kind |       No |      Yes |    No |
-  // Implementation | Atomic_Default_Mem_Order |       No |      Yes |   Yes |
-  //                |      Extension, Requires |          |          |       |
-  //                |                   Vendor |          |          |       |
-  // Target_Device  |    Arch, Device_Num, Isa |       No |      Yes |    No |
-  //                |                Kind, Uid |          |          |       |
-  // User           |                Condition |       No |       No |   Yes |
-
-  struct TraitSetConfig {
-    std::set<parser::OmpTraitSelectorName::Value> allowed;
-    bool allowsDirectiveTraits;
-    bool allowsExtensionTraits;
-    bool allowsScore;
-  };
-
-  using SName = parser::OmpTraitSetSelectorName::Value;
-  using TName = parser::OmpTraitSelectorName::Value;
-
-  static const std::map<SName, TraitSetConfig> configs{
-      {SName::Construct, //
-          {{TName::Simd}, true, false, false}},
-      {SName::Device, //
-          {{TName::Arch, TName::Isa, TName::Kind}, false, true, false}},
-      {SName::Implementation, //
-          {{TName::Atomic_Default_Mem_Order, TName::Extension, TName::Requires,
-               TName::Vendor},
-              false, true, true}},
-      {SName::Target_Device, //
-          {{TName::Arch, TName::Device_Num, TName::Isa, TName::Kind,
-               TName::Uid},
-              false, true, false}},
-      {SName::User, //
-          {{TName::Condition}, false, false, true}},
-  };
-
-  auto checkTraitSet{[&](const TraitSetConfig &config) {
-    auto &[setName, traits]{traitSet.t};
-    auto usn{parser::ToUpperCaseLetters(setName.ToString())};
-
-    // Check if there are any duplicate traits.
-    CheckTraitSelectorList(traits);
-
-    for (const parser::OmpTraitSelector &trait : traits) {
-      // Don't use structured bindings here, because they cannot be captured
-      // before C++20.
-      auto &traitName = std::get<parser::OmpTraitSelectorName>(trait.t);
-      auto &maybeProps =
-          std::get<std::optional<parser::OmpTraitSelector::Properties>>(
-              trait.t);
-
-      // Check allowed traits
-      common::visit( //
-          common::visitors{
-              [&](parser::OmpTraitSelectorName::Value v) {
-                if (!config.allowed.count(v)) {
-                  context_.Say(traitName.source,
-                      "%s is not a valid trait for %s trait set"_err_en_US,
-                      parser::ToUpperCaseLetters(traitName.ToString()), usn);
-                }
-              },
-              [&](llvm::omp::Directive) {
-                if (!config.allowsDirectiveTraits) {
-                  context_.Say(traitName.source,
-                      "Directive name is not a valid trait for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-              [&](const std::string &) {
-                if (!config.allowsExtensionTraits) {
-                  context_.Say(traitName.source,
-                      "Extension traits are not valid for %s trait set"_err_en_US,
-                      usn);
-                }
-              },
-          },
-          traitName.u);
-
-      // Check score
-      if (maybeProps) {
-        auto &[maybeScore, _]{maybeProps->t};
-        if (maybeScore) {
-          CheckTraitScore(*maybeScore);
-        }
-      }
-
-      // Check the properties of the individual traits
-      CheckTraitSelector(traitSet, trait);
-    }
-  }};
-
-  checkTraitSet(
-      configs.at(std::get<parser::OmpTraitSetSelectorName>(traitSet.t).v));
-}
-
-void OmpStructureChecker::CheckTraitScore(const parser::OmpTraitScore &score) {
-  // [6.0:322:23]
-  // A score-expression must be a non-negative constant integer expression.
-  if (auto value{GetIntValue(score)}; !value || value < 0) {
-    context_.Say(score.source,
-        "SCORE expression must be a non-negative constant integer expression"_err_en_US);
-  }
-}
-
-bool OmpStructureChecker::VerifyTraitPropertyLists(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  using PropertyList = std::list<parser::OmpTraitProperty>;
-  auto &[traitName, maybeProps]{trait.t};
-
-  auto checkPropertyList{[&](const PropertyList &properties, auto isValid,
-                             const std::string &message) {
-    bool foundInvalid{false};
-    for (const parser::OmpTraitProperty &prop : properties) {
-      if (!isValid(prop)) {
-        if (foundInvalid) {
-          context_.Say(
-              prop.source, "More invalid properties are present"_err_en_US);
-          break;
-        }
-        context_.Say(prop.source, "%s"_err_en_US, message);
-        foundInvalid = true;
-      }
-    }
-    return !foundInvalid;
-  }};
-
-  bool invalid{false};
-
-  if (std::holds_alternative<llvm::omp::Directive>(traitName.u)) {
-    // Directive-name traits don't have properties.
-    if (maybeProps) {
-      context_.Say(trait.source,
-          "Directive-name traits cannot have properties"_err_en_US);
-      invalid = true;
-    }
-  }
-  // Ignore properties on extension traits.
-
-  // See `TraitSelectorParser` in openmp-parser.cpp
-  if (auto *v{std::get_if<TraitName::Value>(&traitName.u)}) {
-    switch (*v) {
-    // name-list properties
-    case parser::OmpTraitSelectorName::Value::Arch:
-    case parser::OmpTraitSelectorName::Value::Extension:
-    case parser::OmpTraitSelectorName::Value::Isa:
-    case parser::OmpTraitSelectorName::Value::Kind:
-    case parser::OmpTraitSelectorName::Value::Uid:
-    case parser::OmpTraitSelectorName::Value::Vendor:
-      if (maybeProps) {
-        auto isName{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::OmpTraitPropertyName>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isName, "Trait property should be a name");
-      }
-      break;
-    // clause-list
-    case parser::OmpTraitSelectorName::Value::Atomic_Default_Mem_Order:
-    case parser::OmpTraitSelectorName::Value::Requires:
-    case parser::OmpTraitSelectorName::Value::Simd:
-      if (maybeProps) {
-        auto isClause{[&](const parser::OmpTraitProperty &prop) {
-          return GetClauseFromProperty(prop).has_value();
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isClause, "Trait property should be a clause");
-      }
-      break;
-    // expr-list
-    case parser::OmpTraitSelectorName::Value::Condition:
-    case parser::OmpTraitSelectorName::Value::Device_Num:
-      if (maybeProps) {
-        auto isExpr{[](const parser::OmpTraitProperty &prop) {
-          return std::holds_alternative<parser::ScalarExpr>(prop.u);
-        }};
-        invalid = !checkPropertyList(std::get<PropertyList>(maybeProps->t),
-            isExpr, "Trait property should be a scalar expression");
-      }
-      break;
-    } // switch
-  }
-
-  return !invalid;
-}
-
-void OmpStructureChecker::CheckTraitSelector(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  using TraitName = parser::OmpTraitSelectorName;
-  auto &[traitName, maybeProps]{trait.t};
-
-  // Only do the detailed checks if the property lists are valid.
-  if (VerifyTraitPropertyLists(traitSet, trait)) {
-    if (std::holds_alternative<llvm::omp::Directive>(traitName.u) ||
-        std::holds_alternative<std::string>(traitName.u)) {
-      // No properties here: directives don't have properties, and
-      // we don't implement any extension traits now.
-      return;
-    }
-
-    // Specific traits we want to check.
-    // Limitations:
-    // (1) The properties for these traits are defined in "Additional
-    // Definitions for the OpenMP API Specification". It's not clear how
-    // to define them in a portable way, and how to verify their validity,
-    // especially if they get replaced by their integer values (in case
-    // they are defined as enums).
-    // (2) These are entirely implementation-defined, and at the moment
-    // there is no known schema to validate these values.
-    auto v{std::get<TraitName::Value>(traitName.u)};
-    switch (v) {
-    case TraitName::Value::Arch:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Atomic_Default_Mem_Order:
-      CheckTraitADMO(traitSet, trait);
-      break;
-    case TraitName::Value::Condition:
-      CheckTraitCondition(traitSet, trait);
-      break;
-    case TraitName::Value::Device_Num:
-      CheckTraitDeviceNum(traitSet, trait);
-      break;
-    case TraitName::Value::Extension:
-      // Ignore
-      break;
-    case TraitName::Value::Isa:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Kind:
-      // Unchecked, TBD(1)
-      break;
-    case TraitName::Value::Requires:
-      CheckTraitRequires(traitSet, trait);
-      break;
-    case TraitName::Value::Simd:
-      CheckTraitSimd(traitSet, trait);
-      break;
-    case TraitName::Value::Uid:
-      // Unchecked, TBD(2)
-      break;
-    case TraitName::Value::Vendor:
-      // Unchecked, TBD(1)
-      break;
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitADMO(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single clause property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto clauseId{*GetClauseFromProperty(property)};
-    // Check that the clause belongs to the memory-order clause-set.
-    // Clause sets will hopefully be autogenerated at some point.
-    switch (clauseId) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-    case llvm::omp::Clause::OMPC_acquire:
-    case llvm::omp::Clause::OMPC_relaxed:
-    case llvm::omp::Clause::OMPC_release:
-    case llvm::omp::Clause::OMPC_seq_cst:
-      break;
-    default:
-      context_.Say(property.source,
-          "%s trait requires a clause from the memory-order clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitCondition(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  } else {
-    const parser::OmpTraitProperty &property{properties.front()};
-    auto &scalarExpr{std::get<parser::ScalarExpr>(property.u)};
-
-    auto maybeType{GetDynamicType(scalarExpr.thing.value())};
-    if (!maybeType || maybeType->category() != TypeCategory::Logical) {
-      context_.Say(property.source,
-          "%s trait requires a single LOGICAL expression"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitDeviceNum(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  if (properties.size() != 1) {
-    context_.Say(trait.source,
-        "%s trait requires a single expression property"_err_en_US,
-        parser::ToUpperCaseLetters(traitName.ToString()));
-  }
-  // No other checks at the moment.
-}
-
-void OmpStructureChecker::CheckTraitRequires(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_requires, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause from the requirement clause set"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
-void OmpStructureChecker::CheckTraitSimd(
-    const parser::OmpTraitSetSelector &traitSet,
-    const parser::OmpTraitSelector &trait) {
-  unsigned version{context_.langOptions().OpenMPVersion};
-  auto &traitName{std::get<parser::OmpTraitSelectorName>(trait.t)};
-  auto &properties{GetTraitPropertyList(trait)};
-
-  for (const parser::OmpTraitProperty &property : properties) {
-    auto clauseId{*GetClauseFromProperty(property)};
-    if (!llvm::omp::isAllowedClauseForDirective(
-            llvm::omp::OMPD_declare_simd, clauseId, version)) {
-      context_.Say(property.source,
-          "%s trait requires a clause that is allowed on the %s directive"_err_en_US,
-          parser::ToUpperCaseLetters(traitName.ToString()),
-          parser::ToUpperCaseLetters(
-              getDirectiveName(llvm::omp::OMPD_declare_simd)));
-    }
-
-    using ClauseProperty = common::Indirection<parser::OmpClause>;
-    if (!std::holds_alternative<ClauseProperty>(property.u)) {
-      context_.Say(property.source,
-          "Invalid clause specification for %s"_err_en_US,
-          parser::ToUpperCaseLetters(getClauseName(clauseId)));
-    }
-  }
-}
-
 llvm::StringRef OmpStructureChecker::getClauseName(llvm::omp::Clause clause) {
   return llvm::omp::getOpenMPClauseName(clause);
 }
@@ -6506,39 +3867,6 @@ llvm::StringRef OmpStructureChecker::getDirectiveName(
   return llvm::omp::getOpenMPDirectiveName(directive, version);
 }
 
-const Symbol *OmpStructureChecker::GetObjectSymbol(
-    const parser::OmpObject &object) {
-  // Some symbols may be missing if the resolution failed, e.g. when an
-  // undeclared name is used with implicit none.
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    auto &last{GetLastName(*desg)};
-    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
-  }
-  return nullptr;
-}
-
-const Symbol *OmpStructureChecker::GetArgumentSymbol(
-    const parser::OmpArgument &argument) {
-  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
-    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
-      return GetObjectSymbol(*object);
-    }
-  }
-  return nullptr;
-}
-
-std::optional<parser::CharBlock> OmpStructureChecker::GetObjectSource(
-    const parser::OmpObject &object) {
-  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
-    return name->source;
-  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
-    return GetLastName(*desg).source;
-  }
-  return std::nullopt;
-}
-
 void OmpStructureChecker::CheckDependList(const parser::DataRef &d) {
   common::visit(
       common::visitors{
@@ -6848,7 +4176,7 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
 }
 
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
-  if (auto contig{IsContiguous(object)}; contig && !*contig) {
+  if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
     assert(name && "Expecting name component");
     context_.Say(name->source,
@@ -6951,22 +4279,6 @@ void OmpStructureChecker::Enter(const parser::OmpClause::SelfMaps &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_self_maps);
 }
 
-void OmpStructureChecker::Enter(const parser::DoConstruct &x) {
-  Base::Enter(x);
-  loopStack_.push_back(&x);
-}
-
-void OmpStructureChecker::Leave(const parser::DoConstruct &x) {
-  assert(!loopStack_.empty() && "Expecting non-empty loop stack");
-#ifndef NDEBUG
-  const LoopConstruct &top = loopStack_.back();
-  auto *doc{std::get_if<const parser::DoConstruct *>(&top)};
-  assert(doc != nullptr && *doc == &x && "Mismatched loop constructs");
-#endif
-  loopStack_.pop_back();
-  Base::Leave(x);
-}
-
 void OmpStructureChecker::Enter(const parser::OpenMPInteropConstruct &x) {
   bool isDependClauseOccured{false};
   int targetCount{0}, targetSyncCount{0};
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index beb6e0528e81..2a3853335fd1 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -162,10 +162,6 @@ class OmpStructureChecker
 
 private:
   bool CheckAllowedClause(llvmOmpClause clause);
-  bool IsVariableListItem(const Symbol &sym);
-  bool IsExtendedListItem(const Symbol &sym);
-  bool IsCommonBlock(const Symbol &sym);
-  std::optional<bool> IsContiguous(const parser::OmpObject &object);
   void CheckVariableListItem(const SymbolSourceMap &symbols);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
@@ -215,10 +211,6 @@ class OmpStructureChecker
       typename IterTy = decltype(std::declval<RangeTy>().begin())>
   std::optional<IterTy> FindDuplicate(RangeTy &&);
 
-  const Symbol *GetObjectSymbol(const parser::OmpObject &object);
-  const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
-  std::optional<parser::CharBlock> GetObjectSource(
-      const parser::OmpObject &object);
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
new file mode 100644
index 000000000000..fd9596a09cd5
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -0,0 +1,393 @@
+//===-- lib/Semantics/openmp-utils.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#include "openmp-utils.h"
+
+#include "flang/Common/indirection.h"
+#include "flang/Common/reference.h"
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/traverse.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Evaluate/variable.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/expression.h"
+#include "flang/Semantics/semantics.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <optional>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace Fortran::semantics::omp {
+
+std::string ThisVersion(unsigned version) {
+  std::string tv{
+      std::to_string(version / 10) + "." + std::to_string(version % 10)};
+  return "OpenMP v" + tv;
+}
+
+std::string TryVersion(unsigned version) {
+  return "try -fopenmp-version=" + std::to_string(version);
+}
+
+const parser::Designator *GetDesignatorFromObj(
+    const parser::OmpObject &object) {
+  return std::get_if<parser::Designator>(&object.u);
+}
+
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object) {
+  if (auto *desg{GetDesignatorFromObj(object)}) {
+    return std::get_if<parser::DataRef>(&desg->u);
+  }
+  return nullptr;
+}
+
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object) {
+  if (auto *dataRef{GetDataRefFromObj(object)}) {
+    using ElementIndirection = common::Indirection<parser::ArrayElement>;
+    if (auto *ind{std::get_if<ElementIndirection>(&dataRef->u)}) {
+      return &ind->value();
+    }
+  }
+  return nullptr;
+}
+
+const Symbol *GetObjectSymbol(const parser::OmpObject &object) {
+  // Some symbols may be missing if the resolution failed, e.g. when an
+  // undeclared name is used with implicit none.
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->symbol ? &name->symbol->GetUltimate() : nullptr;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    auto &last{GetLastName(*desg)};
+    return last.symbol ? &GetLastName(*desg).symbol->GetUltimate() : nullptr;
+  }
+  return nullptr;
+}
+
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument) {
+  if (auto *locator{std::get_if<parser::OmpLocator>(&argument.u)}) {
+    if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+      return GetObjectSymbol(*object);
+    }
+  }
+  return nullptr;
+}
+
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object) {
+  if (auto *name{std::get_if<parser::Name>(&object.u)}) {
+    return name->source;
+  } else if (auto *desg{std::get_if<parser::Designator>(&object.u)}) {
+    return GetLastName(*desg).source;
+  }
+  return std::nullopt;
+}
+
+bool IsCommonBlock(const Symbol &sym) {
+  return sym.detailsIf<CommonBlockDetails>() != nullptr;
+}
+
+bool IsVariableListItem(const Symbol &sym) {
+  return evaluate::IsVariable(sym) || sym.attrs().test(Attr::POINTER);
+}
+
+bool IsExtendedListItem(const Symbol &sym) {
+  return IsVariableListItem(sym) || sym.IsSubprogram();
+}
+
+bool IsVarOrFunctionRef(const MaybeExpr &expr) {
+  if (expr) {
+    return evaluate::UnwrapProcedureRef(*expr) != nullptr ||
+        evaluate::IsVariable(*expr);
+  } else {
+    return false;
+  }
+}
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr) {
+  const parser::TypedExpr &typedExpr{parserExpr.typedExpr};
+  // ForwardOwningPointer           typedExpr
+  // `- GenericExprWrapper          ^.get()
+  //    `- std::optional<Expr>      ^->v
+  return typedExpr.get()->v;
+}
+
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr) {
+  if (auto maybeExpr{GetEvaluateExpr(parserExpr)}) {
+    return maybeExpr->GetType();
+  } else {
+    return std::nullopt;
+  }
+}
+
+namespace {
+struct ContiguousHelper {
+  ContiguousHelper(SemanticsContext &context)
+      : fctx_(context.foldingContext()) {}
+
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Indirection<Contained> &x) {
+    return Visit(x.value());
+  }
+  template <typename Contained>
+  std::optional<bool> Visit(const common::Reference<Contained> &x) {
+    return Visit(x.get());
+  }
+  template <typename T> std::optional<bool> Visit(const evaluate::Expr<T> &x) {
+    return common::visit([&](auto &&s) { return Visit(s); }, x.u);
+  }
+  template <typename T>
+  std::optional<bool> Visit(const evaluate::Designator<T> &x) {
+    return common::visit(
+        [this](auto &&s) { return evaluate::IsContiguous(s, fctx_); }, x.u);
+  }
+  template <typename T> std::optional<bool> Visit(const T &) {
+    // Everything else.
+    return std::nullopt;
+  }
+
+private:
+  evaluate::FoldingContext &fctx_;
+};
+} // namespace
+
+// Return values:
+// - std::optional<bool>{true} if the object is known to be contiguous
+// - std::optional<bool>{false} if the object is known not to be contiguous
+// - std::nullopt if the object contiguity cannot be determined
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object) {
+  return common::visit( //
+      common::visitors{
+          [&](const parser::Name &x) {
+            // Any member of a common block must be contiguous.
+            return std::optional<bool>{true};
+          },
+          [&](const parser::Designator &x) {
+            evaluate::ExpressionAnalyzer ea{semaCtx};
+            if (MaybeExpr maybeExpr{ea.Analyze(x)}) {
+              return ContiguousHelper{semaCtx}.Visit(*maybeExpr);
+            }
+            return std::optional<bool>{};
+          },
+      },
+      object.u);
+}
+
+struct DesignatorCollector : public evaluate::Traverse<DesignatorCollector,
+                                 std::vector<SomeExpr>, false> {
+  using Result = std::vector<SomeExpr>;
+  using Base = evaluate::Traverse<DesignatorCollector, Result, false>;
+  DesignatorCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> //
+  Result operator()(const evaluate::Designator<T> &x) const {
+    // Once in a designator, don't traverse it any further (i.e. only
+    // collect top-level designators).
+    auto copy{x};
+    return Result{AsGenericExpr(std::move(copy))};
+  }
+
+  template <typename... Rs> //
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (moveAppend(v, std::move(results)), ...);
+    return v;
+  }
+};
+
+struct VariableFinder : public evaluate::AnyTraverse<VariableFinder> {
+  using Base = evaluate::AnyTraverse<VariableFinder>;
+  VariableFinder(const SomeExpr &v) : Base(*this), var(v) {}
+
+  using Base::operator();
+
+  template <typename T>
+  bool operator()(const evaluate::Designator<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+  template <typename T>
+  bool operator()(const evaluate::FunctionRef<T> &x) const {
+    auto copy{x};
+    return evaluate::AsGenericExpr(std::move(copy)) == var;
+  }
+
+private:
+  const SomeExpr &var;
+};
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr) {
+  return DesignatorCollector{}(expr);
+}
+
+static bool HasCommonDesignatorSymbols(
+    const evaluate::SymbolVector &baseSyms, const SomeExpr &other) {
+  // Compare the designators used in "other" with the designators whose
+  // symbols are given in baseSyms.
+  // This is a part of the check if these two expressions can access the same
+  // storage: if the designators used in them are different enough, then they
+  // will be assumed not to access the same memory.
+  //
+  // Consider an (array element) expression x%y(w%z), the corresponding symbol
+  // vector will be {x, y, w, z} (i.e. the symbols for these names).
+  // Check whether this exact sequence appears anywhere in any the symbol
+  // vector for "other". This will be true for x(y) and x(y+1), so this is
+  // not a sufficient condition, but can be used to eliminate candidates
+  // before doing more exhaustive checks.
+  //
+  // If any of the symbols in this sequence are function names, assume that
+  // there is no storage overlap, mostly because it would be impossible in
+  // general to determine what storage the function will access.
+  // Note: if f is pure, then two calls to f will access the same storage
+  // when called with the same arguments. This check is not done yet.
+
+  if (llvm::any_of(
+          baseSyms, [](const SymbolRef &s) { return s->IsSubprogram(); })) {
+    // If there is a function symbol in the chain then we can't infer much
+    // about the accessed storage.
+    return false;
+  }
+
+  auto isSubsequence{// Is u a subsequence of v.
+      [](const evaluate::SymbolVector &u, const evaluate::SymbolVector &v) {
+        size_t us{u.size()}, vs{v.size()};
+        if (us > vs) {
+          return false;
+        }
+        for (size_t off{0}; off != vs - us + 1; ++off) {
+          bool same{true};
+          for (size_t i{0}; i != us; ++i) {
+            if (u[i] != v[off + i]) {
+              same = false;
+              break;
+            }
+          }
+          if (same) {
+            return true;
+          }
+        }
+        return false;
+      }};
+
+  evaluate::SymbolVector otherSyms{evaluate::GetSymbolVector(other)};
+  return isSubsequence(baseSyms, otherSyms);
+}
+
+static bool HasCommonTopLevelDesignators(
+    const std::vector<SomeExpr> &baseDsgs, const SomeExpr &other) {
+  // Compare designators directly as expressions. This will ensure
+  // that x(y) and x(y+1) are not flagged as overlapping, whereas
+  // the symbol vectors for both of these would be identical.
+  std::vector<SomeExpr> otherDsgs{GetAllDesignators(other)};
+
+  for (auto &s : baseDsgs) {
+    if (llvm::any_of(otherDsgs, [&](auto &&t) { return s == t; })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs) {
+  evaluate::SymbolVector baseSyms{evaluate::GetSymbolVector(base)};
+  std::vector<SomeExpr> baseDsgs{GetAllDesignators(base)};
+
+  for (const SomeExpr &expr : exprs) {
+    if (!HasCommonDesignatorSymbols(baseSyms, expr)) {
+      continue;
+    }
+    if (HasCommonTopLevelDesignators(baseDsgs, expr)) {
+      return &expr;
+    }
+  }
+  return nullptr;
+}
+
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super) {
+  return VariableFinder{sub}(super);
+}
+
+// Check if the ActionStmt is actually a [Pointer]AssignmentStmt. This is
+// to separate cases where the source has something that looks like an
+// assignment, but is semantically wrong (diagnosed by general semantic
+// checks), and where the source has some other statement (which we want
+// to report as "should be an assignment").
+bool IsAssignment(const parser::ActionStmt *x) {
+  if (x == nullptr) {
+    return false;
+  }
+
+  using AssignmentStmt = common::Indirection<parser::AssignmentStmt>;
+  using PointerAssignmentStmt =
+      common::Indirection<parser::PointerAssignmentStmt>;
+
+  return common::visit(
+      [](auto &&s) -> bool {
+        using BareS = llvm::remove_cvref_t<decltype(s)>;
+        return std::is_same_v<BareS, AssignmentStmt> ||
+            std::is_same_v<BareS, PointerAssignmentStmt>;
+      },
+      x->u);
+}
+
+bool IsPointerAssignment(const evaluate::Assignment &x) {
+  return std::holds_alternative<evaluate::Assignment::BoundsSpec>(x.u) ||
+      std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
+}
+
+/// parser::Block is a list of executable constructs, parser::BlockConstruct
+/// is Fortran's BLOCK/ENDBLOCK construct.
+/// Strip the outermost BlockConstructs, return the reference to the Block
+/// in the executable part of the innermost of the stripped constructs.
+/// Specifically, if the given `block` has a single entry (it's a list), and
+/// the entry is a BlockConstruct, get the Block contained within. Repeat
+/// this step as many times as possible.
+const parser::Block &GetInnermostExecPart(const parser::Block &block) {
+  const parser::Block *iter{&block};
+  while (iter->size() == 1) {
+    const parser::ExecutionPartConstruct &ep{iter->front()};
+    if (auto *exec{std::get_if<parser::ExecutableConstruct>(&ep.u)}) {
+      using BlockConstruct = common::Indirection<parser::BlockConstruct>;
+      if (auto *bc{std::get_if<BlockConstruct>(&exec->u)}) {
+        iter = &std::get<parser::Block>(bc->value().t);
+        continue;
+      }
+    }
+    break;
+  }
+  return *iter;
+}
+
+} // namespace Fortran::semantics::omp
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/lib/Semantics/openmp-utils.h
new file mode 100644
index 000000000000..dbb056521535
--- /dev/null
+++ b/flang/lib/Semantics/openmp-utils.h
@@ -0,0 +1,66 @@
+//===-- lib/Semantics/openmp-utils.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common utilities used in OpenMP semantic checks.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_SEMANTICS_OPENMP_UTILS_H
+#define FORTRAN_SEMANTICS_OPENMP_UTILS_H
+
+#include "flang/Evaluate/type.h"
+#include "flang/Parser/char-block.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/tools.h"
+
+#include "llvm/ADT/ArrayRef.h"
+
+#include <optional>
+#include <string>
+
+namespace Fortran::semantics {
+class SemanticsContext;
+class Symbol;
+
+// Add this namespace to avoid potential conflicts
+namespace omp {
+std::string ThisVersion(unsigned version);
+std::string TryVersion(unsigned version);
+
+const parser::Designator *GetDesignatorFromObj(const parser::OmpObject &object);
+const parser::DataRef *GetDataRefFromObj(const parser::OmpObject &object);
+const parser::ArrayElement *GetArrayElementFromObj(
+    const parser::OmpObject &object);
+const Symbol *GetObjectSymbol(const parser::OmpObject &object);
+const Symbol *GetArgumentSymbol(const parser::OmpArgument &argument);
+std::optional<parser::CharBlock> GetObjectSource(
+    const parser::OmpObject &object);
+
+bool IsCommonBlock(const Symbol &sym);
+bool IsExtendedListItem(const Symbol &sym);
+bool IsVariableListItem(const Symbol &sym);
+bool IsVarOrFunctionRef(const MaybeExpr &expr);
+
+std::optional<SomeExpr> GetEvaluateExpr(const parser::Expr &parserExpr);
+std::optional<evaluate::DynamicType> GetDynamicType(
+    const parser::Expr &parserExpr);
+
+std::optional<bool> IsContiguous(
+    SemanticsContext &semaCtx, const parser::OmpObject &object);
+
+std::vector<SomeExpr> GetAllDesignators(const SomeExpr &expr);
+const SomeExpr *HasStorageOverlap(
+    const SomeExpr &base, llvm::ArrayRef<SomeExpr> exprs);
+bool IsSubexpressionOf(const SomeExpr &sub, const SomeExpr &super);
+bool IsAssignment(const parser::ActionStmt *x);
+bool IsPointerAssignment(const evaluate::Assignment &x);
+const parser::Block &GetInnermostExecPart(const parser::Block &block);
+} // namespace omp
+} // namespace Fortran::semantics
+
+#endif // FORTRAN_SEMANTICS_OPENMP_UTILS_H
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 885c02e6ec74..299bb6ff876e 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -495,6 +495,12 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
   bool Pre(const parser::OpenMPAllocatorsConstruct &);
   void Post(const parser::OpenMPAllocatorsConstruct &);
 
+  bool Pre(const parser::OmpDeclareVariantDirective &x) {
+    PushContext(x.source, llvm::omp::Directive::OMPD_declare_variant);
+    return true;
+  }
+  void Post(const parser::OmpDeclareVariantDirective &) { PopContext(); };
+
   void Post(const parser::OmpObjectList &x) {
     // The objects from OMP clauses should have already been resolved,
     // except common blocks (the ResolveNamesVisitor does not visit
@@ -1796,10 +1802,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
   if (beginDir.v == llvm::omp::Directive::OMPD_do) {
-    if (const auto &doConstruct{
-            std::get<std::optional<parser::DoConstruct>>(x.t)}) {
-      if (doConstruct.value().IsDoWhile()) {
-        return true;
+    auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+    if (optLoopCons.has_value()) {
+      if (const auto &doConstruct{
+              std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+        if (doConstruct->IsDoWhile()) {
+          return true;
+        }
       }
     }
   }
@@ -1962,48 +1971,69 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
   bool hasCollapseClause{
       clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
 
-  const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
-  if (outer.has_value()) {
-    for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
-      if (loop->IsDoConcurrent()) {
-        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
-        // there isn't a COLLAPSE clause
-        if (isLoopConstruct) {
-          if (hasCollapseClause) {
-            // hasCollapseClause implies clause != nullptr
-            context_.Say(clause->source,
-                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+  auto &optLoopCons = std::get<std::optional<parser::NestedConstruct>>(x.t);
+  if (optLoopCons.has_value()) {
+    if (const auto &outer{std::get_if<parser::DoConstruct>(&*optLoopCons)}) {
+      for (const parser::DoConstruct *loop{&*outer}; loop && level > 0;
+          --level) {
+        if (loop->IsDoConcurrent()) {
+          // DO CONCURRENT is explicitly allowed for the LOOP construct so long
+          // as there isn't a COLLAPSE clause
+          if (isLoopConstruct) {
+            if (hasCollapseClause) {
+              // hasCollapseClause implies clause != nullptr
+              context_.Say(clause->source,
+                  "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+            }
+          } else {
+            auto &stmt =
+                std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+            context_.Say(stmt.source,
+                "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
           }
-        } else {
-          auto &stmt =
-              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
-          context_.Say(stmt.source,
-              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
-        }
-      }
-      // go through all the nested do-loops and resolve index variables
-      const parser::Name *iv{GetLoopIndex(*loop)};
-      if (iv) {
-        if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
-          SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
-          iv->symbol = symbol; // adjust the symbol within region
-          AddToContextObjectWithDSA(*symbol, ivDSA);
         }
+        // go through all the nested do-loops and resolve index variables
+        const parser::Name *iv{GetLoopIndex(*loop)};
+        if (iv) {
+          if (auto *symbol{ResolveOmp(*iv, ivDSA, currScope())}) {
+            SetSymbolDSA(*symbol, {Symbol::Flag::OmpPreDetermined, ivDSA});
+            iv->symbol = symbol; // adjust the symbol within region
+            AddToContextObjectWithDSA(*symbol, ivDSA);
+          }
 
-        const auto &block{std::get<parser::Block>(loop->t)};
-        const auto it{block.begin()};
-        loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+          const auto &block{std::get<parser::Block>(loop->t)};
+          const auto it{block.begin()};
+          loop = it != block.end() ? GetDoConstructIf(*it) : nullptr;
+        }
+      }
+      CheckAssocLoopLevel(level, GetAssociatedClause());
+    } else if (const auto &loop{std::get_if<
+                   common::Indirection<parser::OpenMPLoopConstruct>>(
+                   &*optLoopCons)}) {
+      auto &beginDirective =
+          std::get<parser::OmpBeginLoopDirective>(loop->value().t);
+      auto &beginLoopDirective =
+          std::get<parser::OmpLoopDirective>(beginDirective.t);
+      if (beginLoopDirective.v != llvm::omp::Directive::OMPD_unroll &&
+          beginLoopDirective.v != llvm::omp::Directive::OMPD_tile) {
+        context_.Say(GetContext().directiveSource,
+            "Only UNROLL or TILE constructs are allowed between an OpenMP Loop Construct and a DO construct"_err_en_US,
+            parser::ToUpperCaseLetters(llvm::omp::getOpenMPDirectiveName(
+                GetContext().directive, version)
+                    .str()));
+      } else {
+        PrivatizeAssociatedLoopIndexAndCheckLoopLevel(loop->value());
       }
+    } else {
+      context_.Say(GetContext().directiveSource,
+          "A DO loop must follow the %s directive"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
+                  .str()));
     }
-    CheckAssocLoopLevel(level, GetAssociatedClause());
-  } else {
-    context_.Say(GetContext().directiveSource,
-        "A DO loop must follow the %s directive"_err_en_US,
-        parser::ToUpperCaseLetters(
-            llvm::omp::getOpenMPDirectiveName(GetContext().directive, version)
-                .str()));
   }
 }
+
 void OmpAttributeVisitor::CheckAssocLoopLevel(
     std::int64_t level, const parser::OmpClause *clause) {
   if (clause && level != 0) {
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 9e465f8ff3e1..d0336c9cb661 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -5383,7 +5383,7 @@ void DeclarationVisitor::Post(const parser::EntityDecl &x) {
     ConvertToObjectEntity(symbol) || ConvertToProcEntity(symbol);
     symbol.set(
         Symbol::Flag::EntryDummyArgument, false); // forestall excessive errors
-    Initialization(name, *init, false);
+    Initialization(name, *init, /*inComponentDecl=*/false);
   } else if (attrs.test(Attr::PARAMETER)) { // C882, C883
     Say(name, "Missing initialization for parameter '%s'"_err_en_US);
   }
@@ -6398,7 +6398,7 @@ void DeclarationVisitor::Post(const parser::ComponentDecl &x) {
     SetCUDADataAttr(name.source, symbol, cudaDataAttr());
     if (symbol.has<ObjectEntityDetails>()) {
       if (auto &init{std::get<std::optional<parser::Initialization>>(x.t)}) {
-        Initialization(name, *init, true);
+        Initialization(name, *init, /*inComponentDecl=*/true);
       }
     }
     currScope().symbol()->get<DerivedTypeDetails>().add_component(symbol);
@@ -7278,6 +7278,14 @@ bool DeclarationVisitor::PassesLocalityChecks(
           specName);
       return false;
     }
+    if (const DerivedTypeSpec *derived{type->AsDerived()}) { // F'2023 C1130
+      if (auto bad{FindAllocatableUltimateComponent(*derived)}) {
+        SayWithDecl(name, symbol,
+            "Derived type variable '%s' with ultimate ALLOCATABLE component '%s' not allowed in a %s locality-spec"_err_en_US,
+            bad.BuildResultDesignatorName(), specName);
+        return false;
+      }
+    }
   }
   if (symbol.attrs().test(Attr::ASYNCHRONOUS) && isReduce) { // F'2023 C1131
     SayWithDecl(name, symbol,
@@ -8933,9 +8941,13 @@ void DeclarationVisitor::Initialization(const parser::Name &name,
             // Defer analysis to the end of the specification part
             // so that forward references and attribute checks like SAVE
             // work better.
-            auto restorer{common::ScopedSet(deferImplicitTyping_, true)};
-            Walk(target);
-            ultimate.set(Symbol::Flag::InDataStmt);
+            if (inComponentDecl) {
+              PointerInitialization(name, target);
+            } else {
+              auto restorer{common::ScopedSet(deferImplicitTyping_, true)};
+              Walk(target);
+              ultimate.set(Symbol::Flag::InDataStmt);
+            }
           },
           [&](const std::list<Indirection<parser::DataStmtValue>> &values) {
             // Handled later in data-to-inits conversion
@@ -10355,11 +10367,6 @@ class DeferredCheckVisitor {
         std::get<std::optional<parser::Initialization>>(decl.t));
     return false;
   }
-  bool Pre(const parser::ComponentDecl &decl) {
-    Init(std::get<parser::Name>(decl.t),
-        std::get<std::optional<parser::Initialization>>(decl.t));
-    return false;
-  }
   bool Pre(const parser::ProcDecl &decl) {
     if (const auto &init{
             std::get<std::optional<parser::ProcPointerInit>>(decl.t)}) {
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index e07054f8ec56..ed41c6fb1689 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -655,8 +655,10 @@ bool Semantics::Perform() {
 void Semantics::EmitMessages(llvm::raw_ostream &os) {
   // Resolve the CharBlock locations of the Messages to ProvenanceRanges
   // so messages from parsing and semantics are intermixed in source order.
+  const common::LanguageFeatureControl &features{context_.languageFeatures()};
   context_.messages().ResolveProvenances(context_.allCookedSources());
-  context_.messages().Emit(os, context_.allCookedSources());
+  context_.messages().Emit(
+      os, context_.allCookedSources(), /*echoSourceLine=*/true, &features);
 }
 
 void SemanticsContext::DumpSymbols(llvm::raw_ostream &os) {
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index a65a737f1544..7e13c521c40c 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -59,7 +59,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
     std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
     cliOptions_.insert({cliOption, {feature}});
     languageFeatureCliCanonicalSpelling_[EnumToInt(feature)] =
-        std::string_view{cliOption};
+        std::move(cliOption);
   });
 
   ForEachUsageWarning([&](auto warning) {
@@ -67,7 +67,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
     std::string cliOption{details::CamelCaseToLowerCaseHyphenated(name)};
     cliOptions_.insert({cliOption, {warning}});
     usageWarningCliCanonicalSpelling_[EnumToInt(warning)] =
-        std::string_view{cliOption};
+        std::move(cliOption);
   });
 
   // These features must be explicitly enabled by command line options.
@@ -175,18 +175,16 @@ bool LanguageFeatureControl::EnableWarning(std::string_view input) {
 
 void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
     LanguageFeature f, std::string input) {
-  std::string_view &old{languageFeatureCliCanonicalSpelling_[EnumToInt(f)]};
-  cliOptions_.erase(std::string{old});
-  languageFeatureCliCanonicalSpelling_[EnumToInt(f)] = input;
+  cliOptions_.erase(languageFeatureCliCanonicalSpelling_[EnumToInt(f)]);
   cliOptions_.insert({input, {f}});
+  languageFeatureCliCanonicalSpelling_[EnumToInt(f)] = std::move(input);
 }
 
 void LanguageFeatureControl::ReplaceCliCanonicalSpelling(
     UsageWarning w, std::string input) {
-  std::string_view &old{usageWarningCliCanonicalSpelling_[EnumToInt(w)]};
-  cliOptions_.erase(std::string{old});
-  usageWarningCliCanonicalSpelling_[EnumToInt(w)] = input;
+  cliOptions_.erase(usageWarningCliCanonicalSpelling_[EnumToInt(w)]);
   cliOptions_.insert({input, {w}});
+  usageWarningCliCanonicalSpelling_[EnumToInt(w)] = std::move(input);
 }
 
 std::vector<const char *> LanguageFeatureControl::GetNames(
diff --git a/flang/module/cooperative_groups.f90 b/flang/module/cooperative_groups.f90
new file mode 100644
index 000000000000..b8875f72f807
--- /dev/null
+++ b/flang/module/cooperative_groups.f90
@@ -0,0 +1,56 @@
+!===-- module/cooperative_groups.f90 ---------------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! CUDA Fortran cooperative groups
+
+module cooperative_groups
+
+use, intrinsic :: __fortran_builtins, only: c_devptr => __builtin_c_devptr
+
+implicit none
+
+type :: grid_group
+  type(c_devptr), private :: handle
+  integer(4) :: size
+  integer(4) :: rank
+end type grid_group
+
+type :: coalesced_group
+  type(c_devptr), private :: handle
+  integer(4) :: size
+  integer(4) :: rank
+end type coalesced_group
+
+type :: thread_group
+  type(c_devptr), private :: handle
+  integer(4) :: size
+  integer(4) :: rank
+end type thread_group
+
+interface
+  attributes(device) function this_grid()
+    import
+    type(grid_group) :: this_grid
+  end function
+end interface
+
+interface
+  attributes(device) function this_thread_block()
+    import
+    type(thread_group) :: this_thread_block
+  end function
+end interface
+
+interface this_warp
+  attributes(device) function this_warp()
+    import
+    type(coalesced_group) :: this_warp
+  end function
+end interface
+
+end module
diff --git a/flang/test/Driver/bbc-openmp-version-macro.f90 b/flang/test/Driver/bbc-openmp-version-macro.f90
index 31eb1289b2e9..83e85c9fba94 100644
--- a/flang/test/Driver/bbc-openmp-version-macro.f90
+++ b/flang/test/Driver/bbc-openmp-version-macro.f90
@@ -1,29 +1,22 @@
 ! Test predefined _OPENMP macro which denotes OpenMP version
 
 ! RUN: bbc -fopenmp -o -  %s | FileCheck %s --check-prefix=DEFAULT-OPENMP-VERSION
-! RUN: bbc -fopenmp -fopenmp-version=11 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-11
-! RUN: bbc -fopenmp -fopenmp-version=11 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-11
-! RUN: bbc -fopenmp -fopenmp-version=20 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-20
-! RUN: bbc -fopenmp -fopenmp-version=25 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-25
-! RUN: bbc -fopenmp -fopenmp-version=30 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-30
 ! RUN: bbc -fopenmp -fopenmp-version=31 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-31
 ! RUN: bbc -fopenmp -fopenmp-version=40 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-40
 ! RUN: bbc -fopenmp -fopenmp-version=45 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-45
 ! RUN: bbc -fopenmp -fopenmp-version=50 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-50
 ! RUN: bbc -fopenmp -fopenmp-version=51 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-51
 ! RUN: bbc -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-52
+! RUN: bbc -fopenmp -fopenmp-version=60 -o - %s | FileCheck %s --check-prefix=OPENMP-VERSION-60
 
 ! DEFAULT-OPENMP-VERSION: {{.*}} = arith.constant 202111 : i32
-! OPENMP-VERSION-11: {{.*}} = arith.constant 199911 : i32
-! OPENMP-VERSION-20: {{.*}} = arith.constant 200011 : i32
-! OPENMP-VERSION-25: {{.*}} = arith.constant 200505 : i32
-! OPENMP-VERSION-30: {{.*}} = arith.constant 200805 : i32
 ! OPENMP-VERSION-31: {{.*}} = arith.constant 201107 : i32
 ! OPENMP-VERSION-40: {{.*}} = arith.constant 201307 : i32
 ! OPENMP-VERSION-45: {{.*}} = arith.constant 201511 : i32
 ! OPENMP-VERSION-50: {{.*}} = arith.constant 201811 : i32
 ! OPENMP-VERSION-51: {{.*}} = arith.constant 202011 : i32
 ! OPENMP-VERSION-52: {{.*}} = arith.constant 202111 : i32
+! OPENMP-VERSION-60: {{.*}} = arith.constant 202411 : i32
 
 #if _OPENMP
   integer :: var1 = _OPENMP
diff --git a/flang/test/Driver/flang-openmp-version-macro.f90 b/flang/test/Driver/flang-openmp-version-macro.f90
index eacff0ab4626..c273479dbb7d 100644
--- a/flang/test/Driver/flang-openmp-version-macro.f90
+++ b/flang/test/Driver/flang-openmp-version-macro.f90
@@ -1,10 +1,6 @@
 ! Test predefined _OPENMP macro which denotes OpenMP version
 
 ! RUN: %flang_fc1 -fopenmp -cpp -E %s | FileCheck %s --check-prefix=DEFAULT-OPENMP-VERSION
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=11 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-11
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=20 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-20
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=25 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-25
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=30 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-30
 ! RUN: %flang_fc1 -fopenmp -fopenmp-version=31 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-31
 ! RUN: %flang_fc1 -fopenmp -fopenmp-version=40 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-40
 ! RUN: %flang_fc1 -fopenmp -fopenmp-version=45 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-45
@@ -14,10 +10,6 @@
 ! RUN: %flang_fc1 -fopenmp -fopenmp-version=60 -cpp -E %s | FileCheck %s --check-prefix=OPENMP-VERSION-60
 
 ! DEFAULT-OPENMP-VERSION: integer :: var1 = 202111
-! OPENMP-VERSION-11: integer :: var1 = 199911
-! OPENMP-VERSION-20: integer :: var1 = 200011
-! OPENMP-VERSION-25: integer :: var1 = 200505
-! OPENMP-VERSION-30: integer :: var1 = 200805
 ! OPENMP-VERSION-31: integer :: var1 = 201107
 ! OPENMP-VERSION-40: integer :: var1 = 201307
 ! OPENMP-VERSION-45: integer :: var1 = 201511
diff --git a/flang/test/Driver/fopenmp-version.F90 b/flang/test/Driver/fopenmp-version.F90
new file mode 100644
index 000000000000..7f1396dbc918
--- /dev/null
+++ b/flang/test/Driver/fopenmp-version.F90
@@ -0,0 +1,25 @@
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=31 %s | FileCheck --check-prefix=V31 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=40 %s | FileCheck --check-prefix=V40 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=45 %s | FileCheck --check-prefix=V45 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix=V50 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=51 %s | FileCheck --check-prefix=V51 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix=V52 %s
+!RUN: %flang -dM -E -o - -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix=V60 %s
+
+!V31: #define _OPENMP 201107
+!V40: #define _OPENMP 201307
+!V45: #define _OPENMP 201511
+!V50: #define _OPENMP 201811
+!V51: #define _OPENMP 202011
+!V52: #define _OPENMP 202111
+!V60: #define _OPENMP 202411
+
+
+!RUN: %flang -c -fopenmp -fopenmp-version=25 %s 2>&1 | FileCheck --check-prefix=WARN-ASSUMED %s
+
+!WARN-ASSUMED: warning: OpenMP version 25 is no longer supported, assuming version 52
+
+
+!RUN: not %flang -c -fopenmp -fopenmp-version=29 %s 2>&1 | FileCheck --check-prefix=ERR-BAD %s
+
+!ERR-BAD: error: '29' is not a valid OpenMP version in '-fopenmp-version=29', valid versions are 31, 40, 45, 50, 51, 52, 60
diff --git a/flang/test/Driver/fveclib-codegen.f90 b/flang/test/Driver/fveclib-codegen.f90
index 4cbb1e284f18..88b31da40167 100644
--- a/flang/test/Driver/fveclib-codegen.f90
+++ b/flang/test/Driver/fveclib-codegen.f90
@@ -1,6 +1,8 @@
 ! test that -fveclib= is passed to the backend
 ! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=SLEEF -o - %s | FileCheck %s --check-prefix=SLEEF %}
 ! RUN: %if x86-registered-target %{ %flang -S -Ofast -target x86_64-unknown-linux-gnu -fveclib=libmvec -o - %s | FileCheck %s %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+nosve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-NEON %}
+! RUN: %if aarch64-registered-target %{ %flang -S -Ofast -target aarch64-unknown-linux-gnu -fveclib=libmvec -march=armv8.2-a+sve -o - %s | FileCheck %s --check-prefix=LIBMVEC-AARCH64-SVE %}
 ! RUN: %if x86-registered-target %{ %flang -S -O3 -ffast-math -target x86_64-unknown-linux-gnu -fveclib=AMDLIBM -o - %s | FileCheck %s --check-prefix=AMDLIBM %}
 ! RUN: %flang -S -Ofast -fveclib=NoLibrary -o - %s | FileCheck %s --check-prefix=NOLIB
 
@@ -11,6 +13,8 @@ subroutine sb(a, b)
 ! check that we used a vectorized call to powf()
 ! CHECK: _ZGVbN4vv_powf
 ! SLEEF: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-NEON: _ZGVnN4vv_powf
+! LIBMVEC-AARCH64-SVE: _ZGVsMxvv_powf
 ! AMDLIBM: amd_vrs4_powf
 ! NOLIB: powf
     a(i) = a(i) ** b(i)
diff --git a/flang/test/Driver/fveclib.f90 b/flang/test/Driver/fveclib.f90
index 431a4bfc0252..d21e85e486f8 100644
--- a/flang/test/Driver/fveclib.f90
+++ b/flang/test/Driver/fveclib.f90
@@ -1,6 +1,7 @@
 ! RUN: %flang -### -c -fveclib=none %s 2>&1 | FileCheck -check-prefix CHECK-NOLIB %s
 ! RUN: %flang -### -c -fveclib=Accelerate %s 2>&1 | FileCheck -check-prefix CHECK-ACCELERATE %s
 ! RUN: %flang -### -c --target=x86_64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
+! RUN: %flang -### -c --target=aarch64-unknown-linux-gnu -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-libmvec %s
 ! RUN: %flang -### -c -fveclib=MASSV %s 2>&1 | FileCheck -check-prefix CHECK-MASSV %s
 ! RUN: %flang -### -c -fveclib=Darwin_libsystem_m %s 2>&1 | FileCheck -check-prefix CHECK-DARWIN_LIBSYSTEM_M %s
 ! RUN: %flang -### -c --target=aarch64-none-none -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-SLEEF %s
@@ -23,7 +24,7 @@
 
 ! RUN: not %flang --target=x86-none-none -c -fveclib=SLEEF %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=x86-none-none -c -fveclib=ArmPL %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
-! RUN: not %flang --target=aarch64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
+! RUN: not %flang --target=riscv64-none-none -c -fveclib=libmvec %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=SVML %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! RUN: not %flang --target=aarch64-none-none -c -fveclib=AMDLIBM %s 2>&1 | FileCheck -check-prefix CHECK-ERROR %s
 ! CHECK-ERROR: unsupported option {{.*}} for target
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 1b1231517d17..fff41e09e8d2 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -225,8 +225,8 @@
 ! RUN: | FileCheck %s --check-prefix=OFFLOAD-TARGETS
 
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
 ! OFFLOAD-TARGETS-NEXT: "{{[^"]*}}flang" "-fc1" "-triple" "amdgcn-amd-amdhsa"
-! OFFLOAD-TARGETS-NOT: -fopenmp-targets
+! OFFLOAD-TARGETS-NOT: --offload-targets
 ! OFFLOAD-TARGETS: "{{[^"]*}}flang" "-fc1" "-triple" "x86_64-unknown-linux-gnu"
-! OFFLOAD-TARGETS-SAME: "-fopenmp-targets=amdgcn-amd-amdhsa"
+! OFFLOAD-TARGETS-SAME: "--offload-targets=amdgcn-amd-amdhsa"
diff --git a/flang/test/Driver/tco-emit-final-mlir.fir b/flang/test/Driver/tco-emit-final-mlir.fir
new file mode 100644
index 000000000000..75f8f153127a
--- /dev/null
+++ b/flang/test/Driver/tco-emit-final-mlir.fir
@@ -0,0 +1,19 @@
+// Test tco's -emit-final-mlir option.
+
+// RUN: tco -emit-final-mlir %s 2>&1 | FileCheck %s
+
+// Check that the FIR file is translated into the LLVM IR
+// MLIR dialect, but not into LLVM IR itself.
+
+// CHECK-NOT: func.func
+// CHECK-LABEL: llvm.func @_QPfoo()
+// CHECK-NOT: fir.alloca
+// CHECK: %[[VAL_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+// CHECK: %[[VAL_1:.*]] = llvm.alloca %[[VAL_0]] x i32 : (i64) -> !llvm.ptr
+// CHECK: llvm.return
+// CHECK-NOT: func.func
+
+func.func @_QPfoo() {
+  %1 = fir.alloca i32
+  return
+}
diff --git a/flang/test/Evaluate/fold-dim.f90 b/flang/test/Evaluate/fold-dim.f90
index 40163c3f5a5e..8e1b6ff0d1d8 100644
--- a/flang/test/Evaluate/fold-dim.f90
+++ b/flang/test/Evaluate/fold-dim.f90
@@ -11,7 +11,7 @@ module m
   logical, parameter :: test_a3 = dim(2., 1.) == 1.
   logical, parameter :: test_a4 = dim(2., -1.) == 3.
   logical, parameter :: test_a5 = dim(-1., 2.) == 0.
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real, parameter :: nan = 0./0.
   logical, parameter :: test_a6 = dim(nan, 1.) /= dim(nan, 1.)
 end module
diff --git a/flang/test/Evaluate/fold-nearest.f90 b/flang/test/Evaluate/fold-nearest.f90
index 48b9ef37840e..af5750462f6b 100644
--- a/flang/test/Evaluate/fold-nearest.f90
+++ b/flang/test/Evaluate/fold-nearest.f90
@@ -19,16 +19,16 @@ module m1
   real, parameter :: negZero = sign(0., -1.)
   logical, parameter :: test_12 = nearest(negZero, 1.) == minSubnormal
   logical, parameter :: test_13 = nearest(negZero, -1.) == -minSubnormal
-  !WARN: warning: NEAREST: S argument is zero
+  !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
   logical, parameter :: test_14 = nearest(0., negZero) == -minSubnormal
-  !WARN: warning: NEAREST: S argument is zero
+  !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
   logical, parameter :: test_15 = nearest(negZero, 0.) == minSubnormal
   logical, parameter :: test_16 = nearest(tiny(1.),-1.) == 1.1754942E-38
   logical, parameter :: test_17 = nearest(tiny(1.),1.) == 1.1754945E-38
  contains
   subroutine subr(a)
     real, intent(in) :: a
-    !WARN: warning: NEAREST: S argument is zero
+    !WARN: warning: NEAREST: S argument is zero [-Wfolding-value-checks]
     print *, nearest(a, 0.)
   end
 end module
@@ -42,7 +42,7 @@ module m2
   logical, parameter :: test_2 = ieee_next_after(minSubnormal, -1.) == 0
   logical, parameter :: test_3 = ieee_next_after(1., 2.) == 1.0000001
   logical, parameter :: test_4 = ieee_next_after(1.0000001, -1.) == 1
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real, parameter :: inf = 1. / 0.
   logical, parameter :: test_5 = ieee_next_after(inf, inf) == inf
   logical, parameter :: test_6 = ieee_next_after(inf, -inf) == h
@@ -54,12 +54,12 @@ module m2
   logical, parameter :: test_11 = ieee_next_after(1.9999999999999999999_10, 3.) == 2._10
 #endif
   logical, parameter :: test_12 = ieee_next_after(1., 1.) == 1.
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real, parameter :: nan = 0. / 0.
-  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered
+  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered [-Wfolding-value-checks]
   real, parameter :: x13 = ieee_next_after(nan, nan)
   logical, parameter :: test_13 = .not. (x13 == x13)
-  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered
+  !WARN: warning: IEEE_NEXT_AFTER intrinsic folding: arguments are unordered [-Wfolding-value-checks]
   real, parameter :: x14 = ieee_next_after(nan, 0.)
   logical, parameter :: test_14 = .not. (x14 == x14)
 end module
@@ -72,7 +72,7 @@ module m3
   logical, parameter :: test_2 = ieee_next_down(0.d0) == -minSubnormal
   logical, parameter :: test_3 = ieee_next_up(1.d0) == 1.0000000000000002d0
   logical, parameter :: test_4 = ieee_next_down(1.0000000000000002d0) == 1.d0
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(kind(0.d0)), parameter :: inf = 1.d0 / 0.d0
   logical, parameter :: test_5 = ieee_next_up(huge(0.d0)) == inf
   logical, parameter :: test_6 = ieee_next_down(-huge(0.d0)) == -inf
@@ -82,12 +82,12 @@ module m3
   logical, parameter :: test_10 = ieee_next_down(-inf) == -inf
   logical, parameter :: test_11 = ieee_next_up(1.9999999999999997d0) == 2.d0
   logical, parameter :: test_12 = ieee_next_down(2.d0) == 1.9999999999999997d0
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(kind(0.d0)), parameter :: nan = 0.d0 / 0.d0
-  !WARN: warning: IEEE_NEXT_UP intrinsic folding: argument is NaN
+  !WARN: warning: IEEE_NEXT_UP intrinsic folding: argument is NaN [-Wfolding-exception]
   real(kind(0.d0)), parameter :: x13 = ieee_next_up(nan)
   logical, parameter :: test_13 = .not. (x13 == x13)
-  !WARN: warning: IEEE_NEXT_DOWN intrinsic folding: argument is NaN
+  !WARN: warning: IEEE_NEXT_DOWN intrinsic folding: argument is NaN [-Wfolding-exception]
   real(kind(0.d0)), parameter :: x14 = ieee_next_down(nan)
   logical, parameter :: test_14 = .not. (x14 == x14)
 end module
diff --git a/flang/test/Evaluate/fold-out_of_range.f90 b/flang/test/Evaluate/fold-out_of_range.f90
index 6360eee5322b..a2f453acb4cb 100644
--- a/flang/test/Evaluate/fold-out_of_range.f90
+++ b/flang/test/Evaluate/fold-out_of_range.f90
@@ -9,23 +9,23 @@ module m
   integer(4),  parameter :: i4v(*)  = [ -huge(1_4)  - 1_4,  huge(1_4) ]
   integer(8),  parameter :: i8v(*)  = [ -huge(1_8)  - 1_8,  huge(1_8) ]
   integer(16), parameter :: i16v(*) = [ -huge(1_16) - 1_16, huge(1_16) ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(2),  parameter :: r2v(*)  = [ -huge(1._2),  huge(1._2),  1._2/0._2,   0._2/0._2 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(3),  parameter :: r3v(*)  = [ -huge(1._3),  huge(1._3),  1._3/0._3,   0._3/0._3 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4),  parameter :: r4v(*)  = [ -huge(1._4),  huge(1._4),  1._4/0._4,   0._4/0._4 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(8),  parameter :: r8v(*)  = [ -huge(1._8),  huge(1._8),  1._8/0._8,   0._8/0._8 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(10), parameter :: r10v(*) = [ -huge(1._10), huge(1._10), 1._10/0._10, 0._10/0._10 ]
-  !WARN: warning: division by zero
-  !WARN: warning: invalid argument on division
+  !WARN: warning: division by zero [-Wfolding-exception]
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(16), parameter :: r16v(*) = [ -huge(1._16), huge(1._16), 1._16/0._16, 0._16/0._16 ]
   logical, parameter :: finites(*) = [ .true., .true., .false., .false. ]
 
@@ -93,7 +93,7 @@ module m
   logical, parameter :: test_r2r10  = .not. any(out_of_range(r2v,  1._10))
   logical, parameter :: test_r2r16  = .not. any(out_of_range(r2v,  1._16))
   logical, parameter :: test_r3r2   =       all(out_of_range(r3v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(3) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(3) conversion [-Wfolding-exception]
   logical, parameter :: test_r3r2b  = .not. any(out_of_range(real(r2v, 3),  1._2))
   logical, parameter :: test_r3r3   = .not. any(out_of_range(r3v,  1._3))
   logical, parameter :: test_r3r4   = .not. any(out_of_range(r3v,  1._4))
@@ -101,55 +101,55 @@ module m
   logical, parameter :: test_r3r10  = .not. any(out_of_range(r3v,  1._10))
   logical, parameter :: test_r3r16  = .not. any(out_of_range(r3v,  1._16))
   logical, parameter :: test_r4r2   =       all(out_of_range(r4v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(4) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(4) conversion [-Wfolding-exception]
   logical, parameter :: test_r4r2b  = .not. any(out_of_range(real(r2v, 4),  1._2))
   logical, parameter :: test_r4r3   =       all(out_of_range(r4v,  1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(4) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(4) conversion [-Wfolding-exception]
   logical, parameter :: test_r4r3b  = .not. any(out_of_range(real(r3v, 4),  1._3))
   logical, parameter :: test_r4r4   = .not. any(out_of_range(r4v,  1._4))
   logical, parameter :: test_r4r8   = .not. any(out_of_range(r4v,  1._8))
   logical, parameter :: test_r4r10  = .not. any(out_of_range(r4v,  1._10))
   logical, parameter :: test_r4r16  = .not. any(out_of_range(r4v,  1._16))
   logical, parameter :: test_r8r2   =       all(out_of_range(r8v,  1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r2b  = .not. any(out_of_range(real(r2v, 8),  1._2))
   logical, parameter :: test_r8r3   =       all(out_of_range(r8v,  1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r3b  = .not. any(out_of_range(real(r3v, 8),  1._3))
   logical, parameter :: test_r8r4   =       all(out_of_range(r8v,  1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(8) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(8) conversion [-Wfolding-exception]
   logical, parameter :: test_r8r4b  = .not. any(out_of_range(real(r4v, 8),  1._4))
   logical, parameter :: test_r8r8   = .not. any(out_of_range(r8v,  1._8))
   logical, parameter :: test_r8r10  = .not. any(out_of_range(r8v,  1._10))
   logical, parameter :: test_r8r16  = .not. any(out_of_range(r8v,  1._16))
   logical, parameter :: test_r10r2  =       all(out_of_range(r10v, 1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r2b = .not. any(out_of_range(real(r2v, 10),  1._2))
   logical, parameter :: test_r10r3  =       all(out_of_range(r10v, 1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r3b = .not. any(out_of_range(real(r3v, 10),  1._3))
   logical, parameter :: test_r10r4  =       all(out_of_range(r10v, 1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r4b = .not. any(out_of_range(real(r4v, 10),  1._4))
   logical, parameter :: test_r10r8  =       all(out_of_range(r10v, 1._8) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(8) to REAL(10) conversion
+  !WARN: warning: invalid argument on REAL(8) to REAL(10) conversion [-Wfolding-exception]
   logical, parameter :: test_r10r8b = .not. any(out_of_range(real(r8v, 10),  1._8))
   logical, parameter :: test_r10r10 = .not. any(out_of_range(r10v, 1._10))
   logical, parameter :: test_r10r16 = .not. any(out_of_range(r10v, 1._16))
   logical, parameter :: test_r16r2  =       all(out_of_range(r16v, 1._2) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(2) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(2) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r2b = .not. any(out_of_range(real(r2v, 16), 1._2))
   logical, parameter :: test_r16r3  =       all(out_of_range(r16v, 1._3) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(3) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(3) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r3b = .not. any(out_of_range(real(r3v, 16), 1._3))
   logical, parameter :: test_r16r4  =       all(out_of_range(r16v, 1._4) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(4) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(4) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r4b = .not. any(out_of_range(real(r4v, 16), 1._4))
   logical, parameter :: test_r16r8  =       all(out_of_range(r16v, 1._8) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(8) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(8) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r8b = .not. any(out_of_range(real(r8v, 16), 1._8))
   logical, parameter :: test_r16r10 =       all(out_of_range(r16v, 1._10) .eqv. finites)
-  !WARN: warning: invalid argument on REAL(10) to REAL(16) conversion
+  !WARN: warning: invalid argument on REAL(10) to REAL(16) conversion [-Wfolding-exception]
   logical, parameter :: test_r16r10b= .not. any(out_of_range(real(r10v, 16), 1._10))
   logical, parameter :: test_r16r16 = .not. any(out_of_range(r16v, 1._16))
 
@@ -223,29 +223,29 @@ module m
   logical, parameter :: test_r2i2ur   = all(out_of_range(real(i2v,  kind=2)+.5_2, 1_2,  .true.)  .eqv. [.false., .true.])
   logical, parameter :: test_r2i2d    = all(out_of_range(real(i2v,  kind=2)-.5_2, 1_2,  .false.) .eqv. [.false., .true.])
   logical, parameter :: test_r2i2dr   = all(out_of_range(real(i2v,  kind=2)-.5_2, 1_2,  .true.)  .eqv. [.false., .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4u    = all(out_of_range(real(i4v,  kind=2)+.5_2, 1_4,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4ur   = all(out_of_range(real(i4v,  kind=2)+.5_2, 1_4,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4d    = all(out_of_range(real(i4v,  kind=2)-.5_2, 1_4,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(4) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i4dr   = all(out_of_range(real(i4v,  kind=2)-.5_2, 1_4,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8u    = all(out_of_range(real(i8v,  kind=2)+.5_2, 1_8,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8ur   = all(out_of_range(real(i8v,  kind=2)+.5_2, 1_8,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8d    = all(out_of_range(real(i8v,  kind=2)-.5_2, 1_8,  .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(8) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i8dr   = all(out_of_range(real(i8v,  kind=2)-.5_2, 1_8,  .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16u   = all(out_of_range(real(i16v, kind=2)+.5_2, 1_16, .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16ur  = all(out_of_range(real(i16v, kind=2)+.5_2, 1_16, .true.)  .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16d   = all(out_of_range(real(i16v, kind=2)-.5_2, 1_16, .false.) .eqv. [.true.,  .true.])
-  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion
+  !WARN: warning: overflow on INTEGER(16) to REAL(2) conversion [-Wfolding-exception]
   logical, parameter :: test_r2i16dr  = all(out_of_range(real(i16v, kind=2)-.5_2, 1_16, .true.)  .eqv. [.true.,  .true.])
 
   logical, parameter :: test_r3i1u    = all(out_of_range(real(i1v,  kind=3)+.5_3, 1_1,  .false.) .eqv. [.false., .false.])
@@ -357,7 +357,7 @@ module m
   subroutine s(x, r)
     real(8), intent(in) :: x
     logical, intent(in), optional :: r
-    !WARN: warning: ROUND= argument to OUT_OF_RANGE() is an optional dummy argument that must be present at execution
+    !WARN: warning: ROUND= argument to OUT_OF_RANGE() is an optional dummy argument that must be present at execution [-Woptional-must-be-present]
     print *, out_of_range(x, 1, round=r)
   end
 end
diff --git a/flang/test/Evaluate/fold-unsigned.f90 b/flang/test/Evaluate/fold-unsigned.f90
index 719bdcc1a40b..d53b7c7be0d5 100644
--- a/flang/test/Evaluate/fold-unsigned.f90
+++ b/flang/test/Evaluate/fold-unsigned.f90
@@ -24,7 +24,7 @@ module m
 
   logical, parameter :: test_cus0    = int(0u,1) == 0
   logical, parameter :: test_cus0_k  = kind(int(0u,1)) == 1
-  !WARN: warning: conversion of 255_U1 to INTEGER(1) overflowed; result is -1
+  !WARN: warning: conversion of 255_U1 to INTEGER(1) overflowed; result is -1 [-Wfolding-exception]
   logical, parameter :: test_cus255  = int(255u_1,1) == -1
   logical, parameter :: test_cur255  = real(255u) == 255.
 
diff --git a/flang/test/Evaluate/folding03.f90 b/flang/test/Evaluate/folding03.f90
index 827bde86757c..5b7ddd3c6c23 100644
--- a/flang/test/Evaluate/folding03.f90
+++ b/flang/test/Evaluate/folding03.f90
@@ -15,32 +15,32 @@ module integer_tests
   ! Integer division by zero are not tested here because they are handled as fatal
   ! errors in constants.
 
-  !WARN: warning: INTEGER(4) negation overflowed
+  !WARN: warning: INTEGER(4) negation overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_unary_minus1 = (-i4_nmax).EQ.i4_nmax
   logical, parameter :: test_no_overflow_unary_minus1 = (-i4_pmax).EQ.(i4_nmax+1_4)
   logical, parameter :: test_no_overflow_unary_plus1 = (+i4_pmax).EQ.i4_pmax
   logical, parameter :: test_no_overflow_unary_plus2 = (+i4_nmax).EQ.i4_nmax
 
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add1 = (i4_pmax+1_4).EQ.i4_nmax
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add2 = (i4_nmax + (-1_4)).EQ.i4_pmax
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add3 = (i4_pmax + i4_pmax).EQ.(-2_4)
-  !WARN: warning: INTEGER(4) addition overflowed
+  !WARN: warning: INTEGER(4) addition overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_add4 = (i4_nmax + i4_nmax).EQ.(0_4)
   logical, parameter :: test_no_overflow_add1 = (i4_pmax + 0_4).EQ.i4_pmax
   logical, parameter :: test_no_overflow_add2 = (i4_nmax + (-0_4)).EQ.i4_nmax
   logical, parameter :: test_no_overflow_add3 = (i4_pmax + i4_nmax).EQ.(-1_4)
   logical, parameter :: test_no_overflow_add4 = (i4_nmax + i4_pmax).EQ.(-1_4)
 
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub1 = (i4_nmax - 1_4).EQ.i4_pmax
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub2 = (i4_pmax - (-1_4)).EQ.i4_nmax
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub3 = (i4_nmax - i4_pmax).EQ.(1_4)
-  !WARN: warning: INTEGER(4) subtraction overflowed
+  !WARN: warning: INTEGER(4) subtraction overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_sub4 = (i4_pmax - i4_nmax).EQ.(-1_4)
   logical, parameter :: test_no_overflow_sub1 = (i4_nmax - 0_4).EQ.i4_nmax
   logical, parameter :: test_no_overflow_sub2 = (i4_pmax - (-0_4)).EQ.i4_pmax
@@ -48,23 +48,23 @@ module integer_tests
   logical, parameter :: test_no_overflow_sub4 = (i4_pmax - i4_pmax).EQ.0_4
 
 
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult1 = (i4_pmax*2_4).EQ.(-2_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult2 = (i4_nmax*2_4).EQ.(0_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult3 = (i4_nmax*i4_nmax).EQ.(0_4)
-  !WARN: warning: INTEGER(4) multiplication overflowed
+  !WARN: warning: INTEGER(4) multiplication overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_mult4 = (i4_pmax*i4_pmax).EQ.(1_4)
 
-  !WARN: warning: INTEGER(4) division overflowed
+  !WARN: warning: INTEGER(4) division overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_div1 = (i4_nmax/(-1_4)).EQ.(i4_nmax)
   logical, parameter :: test_no_overflow_div1 = (i4_nmax/(-2_4)).EQ.(1_4 + i4_pmax/2_4)
   logical, parameter :: test_no_overflow_div2 = (i4_nmax/i4_nmax).EQ.(1_4)
 
-  !WARN: warning: INTEGER(4) power overflowed
+  !WARN: warning: INTEGER(4) power overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_pow1 = (i4_pmax**2_4).EQ.(1_4)
-  !WARN: warning: INTEGER(4) power overflowed
+  !WARN: warning: INTEGER(4) power overflowed [-Wfolding-exception]
   logical, parameter :: test_overflow_pow3 = (i4_nmax**2_4).EQ.(0_4)
   logical, parameter :: test_no_overflow_pow1 = ((-1_4)**i4_nmax).EQ.(1_4)
   logical, parameter :: test_no_overflow_pow2 = ((-1_4)**i4_pmax).EQ.(-1_4)
@@ -76,12 +76,12 @@ module real_tests
 
   real(4), parameter :: r4_pmax = 3.4028235E38
   real(4), parameter :: r4_nmax = -3.4028235E38
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan = 0._4/0._4
   TEST_ISNAN(r4_nan)
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_pinf = 1._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_ninf = -1._4/0._4
 
   logical, parameter :: test_r4_nan_parentheses1 = .NOT.(((r4_nan)).EQ.r4_nan)
@@ -106,13 +106,13 @@ module real_tests
   real(4), parameter :: r4_nan_plus = (+r4_nan)
   TEST_ISNAN(r4_nan_plus)
 
-  !WARN: warning: overflow on addition
+  !WARN: warning: overflow on addition [-Wfolding-exception]
   logical, parameter :: test_inf_r4_add9 = (r4_pmax + r4_pmax).eq.(r4_pinf)
-  !WARN: warning: overflow on addition
+  !WARN: warning: overflow on addition [-Wfolding-exception]
   logical, parameter :: test_inf_r4_add10 = (r4_nmax + r4_nmax).eq.(r4_ninf)
-  !WARN: warning: overflow on subtraction
+  !WARN: warning: overflow on subtraction [-Wfolding-exception]
   logical, parameter :: test_inf_r4_sub9 = (r4_pmax - r4_nmax).eq.(r4_pinf)
-  !WARN: warning: overflow on subtraction
+  !WARN: warning: overflow on subtraction [-Wfolding-exception]
   logical, parameter :: test_inf_r4_sub10 = (r4_nmax - r4_pmax).eq.(r4_ninf)
 
   ! No warnings expected below (inf propagation).
@@ -125,16 +125,16 @@ module real_tests
   logical, parameter :: test_inf_r4_add7 = (r4_ninf + 0._4).EQ.(r4_ninf)
   logical, parameter :: test_inf_r4_add8 = (r4_pinf + 0._4).EQ.(r4_pinf)
 
-  !WARN: warning: invalid argument on subtraction
+  !WARN: warning: invalid argument on subtraction [-Wfolding-exception]
   real(4), parameter :: r4_nan_sub1 = r4_pinf - r4_pinf
   TEST_ISNAN(r4_nan_sub1)
-  !WARN: warning: invalid argument on subtraction
+  !WARN: warning: invalid argument on subtraction [-Wfolding-exception]
   real(4), parameter :: r4_nan_sub2 = r4_ninf - r4_ninf
   TEST_ISNAN(r4_nan_sub2)
-  !WARN: warning: invalid argument on addition
+  !WARN: warning: invalid argument on addition [-Wfolding-exception]
   real(4), parameter :: r4_nan_add1 = r4_ninf + r4_pinf
   TEST_ISNAN(r4_nan_add1)
-  !WARN: warning: invalid argument on addition
+  !WARN: warning: invalid argument on addition [-Wfolding-exception]
   real(4), parameter :: r4_nan_add2 = r4_pinf + r4_ninf
   TEST_ISNAN(r4_nan_add2)
 
@@ -156,13 +156,13 @@ module real_tests
   real(4), parameter :: r4_nan_add6 = r4_nan + r4_nan
   TEST_ISNAN(r4_nan_add6)
 
-  !WARN: warning: overflow on multiplication
+  !WARN: warning: overflow on multiplication [-Wfolding-exception]
   logical, parameter :: test_inf_r4_mult1 = (1.5_4*r4_pmax).eq.(r4_pinf)
-  !WARN: warning: overflow on multiplication
+  !WARN: warning: overflow on multiplication [-Wfolding-exception]
   logical, parameter :: test_inf_r4_mult2 = (1.5_4*r4_nmax).eq.(r4_ninf)
-  !WARN: warning: overflow on division
+  !WARN: warning: overflow on division [-Wfolding-exception]
   logical, parameter :: test_inf_r4_div1 = (r4_nmax/(-0.5_4)).eq.(r4_pinf)
-  !WARN: warning: overflow on division
+  !WARN: warning: overflow on division [-Wfolding-exception]
   logical, parameter :: test_inf_r4_div2 = (r4_pmax/(-0.5_4)).eq.(r4_ninf)
 
   ! No warnings expected below (inf propagation).
@@ -179,25 +179,25 @@ module real_tests
   logical, parameter :: test_inf_r4_div9 = (r4_nmax/r4_pinf).EQ.(0.)
   logical, parameter :: test_inf_r4_div10 = (r4_nmax/r4_ninf).EQ.(0.)
 
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div1 = 0._4/0._4
   TEST_ISNAN(r4_nan_div1)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div2 = r4_ninf/r4_ninf
   TEST_ISNAN(r4_nan_div2)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div3 = r4_ninf/r4_pinf
   TEST_ISNAN(r4_nan_div3)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div4 = r4_pinf/r4_ninf
   TEST_ISNAN(r4_nan_div4)
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan_div5 = r4_pinf/r4_pinf
   TEST_ISNAN(r4_nan_div5)
-  !WARN: warning: invalid argument on multiplication
+  !WARN: warning: invalid argument on multiplication [-Wfolding-exception]
   real(4), parameter :: r4_nan_mult1 = r4_pinf*0._4
   TEST_ISNAN(r4_nan_mult1)
-  !WARN: warning: invalid argument on multiplication
+  !WARN: warning: invalid argument on multiplication [-Wfolding-exception]
   real(4), parameter :: r4_nan_mult2 = 0._4*r4_ninf
   TEST_ISNAN(r4_nan_mult2)
 
diff --git a/flang/test/Evaluate/folding04.f90 b/flang/test/Evaluate/folding04.f90
index 08c5c6eee9b1..027db20f608b 100644
--- a/flang/test/Evaluate/folding04.f90
+++ b/flang/test/Evaluate/folding04.f90
@@ -10,11 +10,11 @@ module real_tests
 
   real(4), parameter :: r4_pmax = 3.4028235E38
   real(4), parameter :: r4_nmax = -3.4028235E38
-  !WARN: warning: invalid argument on division
+  !WARN: warning: invalid argument on division [-Wfolding-exception]
   real(4), parameter :: r4_nan = 0._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_pinf = 1._4/0._4
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(4), parameter :: r4_ninf = -1._4/0._4
 
   !WARN: warning: argument is out of range [-1., 1.]
@@ -37,7 +37,7 @@ module real_tests
   TEST_ISNAN(nan_r8_dasin1)
   !WARN: warning: complex argument must be different from zero
   complex(4), parameter :: c4_clog1 = clog((0., 0.))
-  !WARN: warning: MOD: P argument is zero
+  !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
   real(4), parameter :: nan_r4_mod = mod(3.5, 0.)
   TEST_ISNAN(nan_r4_mod)
   real(4), parameter :: ok_r4_gamma = gamma(-1.1)
@@ -53,17 +53,17 @@ module real_tests
   !WARN: warning: 'x' and 'y' arguments must not be both zero
   real(4), parameter :: r4_atan2 = atan2(0., 0.)
 
-  !WARN: warning: overflow on evaluation of intrinsic function or operation
+  !WARN: warning: overflow on evaluation of intrinsic function or operation [-Wfolding-exception]
   logical, parameter :: test_exp_overflow = exp(256._4).EQ.r4_pinf
  contains
   subroutine s1(a,j)
-    !WARN: warning: MOD: P argument is zero
+    !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, mod(a, 0.)
-    !WARN: warning: MODULO: P argument is zero
+    !WARN: warning: MODULO: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, modulo(a, 0.)
-    !WARN: warning: MOD: P argument is zero
+    !WARN: warning: MOD: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, mod(j, 0.)
-    !WARN: warning: MODULO: P argument is zero
+    !WARN: warning: MODULO: P argument is zero [-Wfolding-avoids-runtime-crash]
     print *, modulo(j, 0.)
   end
 end module
@@ -86,13 +86,13 @@ module specific_extremums
   ! specified for f18 (converting the result).
   integer(8), parameter :: max_i32_8 = 2_8**31-1
   integer, parameter :: expected_min0 = int(min(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed
+  !WARN: portability: Argument types do not match specific intrinsic 'min0' requirements; using 'min' generic instead and converting the result to INTEGER(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
   integer, parameter :: result_min0 =  min0(max_i32_8, 2_8*max_i32_8)
   ! result_min0 would be -2  if arguments were converted to default integer.
   logical, parameter :: test_min0 = expected_min0 .EQ. result_min0
 
   real, parameter :: expected_amax0 = real(max(max_i32_8, 2_8*max_i32_8), 4)
-  !WARN: portability: Argument types do not match specific intrinsic 'amax0' requirements; using 'max' generic instead and converting the result to REAL(4) if needed
+  !WARN: portability: Argument types do not match specific intrinsic 'amax0' requirements; using 'max' generic instead and converting the result to REAL(4) if needed [-Wuse-generic-intrinsic-when-specific-doesnt-match]
   real, parameter :: result_amax0 = amax0(max_i32_8, 2_8*max_i32_8)
   ! result_amax0 would be 2.1474836E+09 if arguments were converted to default integer first.
   logical, parameter :: test_amax0 = expected_amax0 .EQ. result_amax0
diff --git a/flang/test/Evaluate/folding05.f90 b/flang/test/Evaluate/folding05.f90
index eb762775e641679cac704694f6d4aa6fb191083f..aed1906e6a640e9c9e71a800179b2b5d4bdbb688 100644
GIT binary patch
delta 312
zcmewudM12>4wIxpv~GA>eojheUb=2sVoqtQZgNIya&~d-WJg~4$>*5ZCkHXb;#al#
z9Mf+`0!m)6#4(E@+m%|8oLW$lnV%Os`5l+^<U|e)JSJ>pSKhpaLmZzaASKT@IT&$U
X;)BP6KrWTZaonfytJv(rvs4ZM6`5}~

delta 79
zcmX>T{xNie4%6g^Olp&}nPN9TWRe1LlqL(Xuxz$s4QHObgT;}_P!lL7w^@->h7rst
X;o@bSJe}JI#F;F`a{(gDyIKwaQ{oz{

diff --git a/flang/test/Evaluate/folding06.f90 b/flang/test/Evaluate/folding06.f90
index da434fb6d486..d4f11b0c30f1 100644
--- a/flang/test/Evaluate/folding06.f90
+++ b/flang/test/Evaluate/folding06.f90
@@ -7,12 +7,12 @@ module m
   integer, pointer :: int_pointer
   integer, allocatable :: int_allocatable
   logical, parameter :: test_Assoc1 = .not.(associated(null()))
-  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
   !WARN: because: 'NULL()' is a null pointer
   logical, parameter :: test_Assoc2 = .not.(associated(null(), null()))
   logical, parameter :: test_Assoc3 = .not.(associated(null(int_pointer)))
   logical, parameter :: test_Alloc1 = .not.(allocated(null(int_allocatable)))
-  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+  !WARN: portability: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
   !WARN: because: 'NULL()' is a null pointer
   logical, parameter :: test_Assoc5 = .not. associated(null(), null(int_pointer))
 
diff --git a/flang/test/Evaluate/folding14.f90 b/flang/test/Evaluate/folding14.f90
index 2cf1c32f6cb3..4ff9e107d161 100644
--- a/flang/test/Evaluate/folding14.f90
+++ b/flang/test/Evaluate/folding14.f90
@@ -4,9 +4,9 @@ module m1
   logical, parameter :: results(*) = isnan([ &
     0., &
     -0., &
-!WARN: warning: division by zero
+!WARN: warning: division by zero [-Wfolding-exception]
     1./0., &
-!WARN: warning: invalid argument on division
+!WARN: warning: invalid argument on division [-Wfolding-exception]
     0./0., &
     real(z'7ff80001',kind=4), &
     real(z'fff80001',kind=4), &
diff --git a/flang/test/Evaluate/folding28.f90 b/flang/test/Evaluate/folding28.f90
index 642919de7414..3a795540812a 100644
--- a/flang/test/Evaluate/folding28.f90
+++ b/flang/test/Evaluate/folding28.f90
@@ -15,7 +15,7 @@ module m
   ! -0 (sqrt is -0)
   real(8), parameter :: n08 = z'8000000000000000'
   real(8), parameter :: sqrt_n08 = sqrt(n08)
-  !WARN: warning: division by zero
+  !WARN: warning: division by zero [-Wfolding-exception]
   real(8), parameter :: inf_n08 = 1.0_8 / sqrt_n08, inf_n08z = z'fff0000000000000'
   logical, parameter :: test_n08 = inf_n08 == inf_n08z
   ! min normal
diff --git a/flang/test/Fir/CUDA/cuda-device-global.f90 b/flang/test/Fir/CUDA/cuda-device-global.f90
index 8cac643b27c3..4c634513745f 100644
--- a/flang/test/Fir/CUDA/cuda-device-global.f90
+++ b/flang/test/Fir/CUDA/cuda-device-global.f90
@@ -11,3 +11,16 @@ module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.conta
 
 // CHECK: gpu.module @cuda_device_mo
 // CHECK-NEXT: fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda<device>} : !fir.array<5xi32>
+
+// -----
+
+module attributes {fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module} {
+  fir.global @_QMm1ECb(dense<[90, 100, 110]> : tensor<3xi32>) constant : !fir.array<3xi32>
+  fir.global @_QMm2ECc(dense<[100, 200, 300]> : tensor<3xi32>) constant : !fir.array<3xi32>
+}
+
+// CHECK: fir.global @_QMm1ECb
+// CHECK: fir.global @_QMm2ECc
+// CHECK: gpu.module @cuda_device_mod
+// CHECK-DAG: fir.global @_QMm2ECc
+// CHECK-DAG: fir.global @_QMm1ECb
diff --git a/flang/test/Fir/OpenACC/openacc-type-categories-class.f90 b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
new file mode 100644
index 000000000000..58025bfa556a
--- /dev/null
+++ b/flang/test/Fir/OpenACC/openacc-type-categories-class.f90
@@ -0,0 +1,46 @@
+! RUN: bbc -fopenacc -emit-hlfir %s -o - | fir-opt -pass-pipeline='builtin.module(test-fir-openacc-interfaces)' --mlir-disable-threading 2>&1 | FileCheck %s
+
+module mm
+  type, public :: polyty
+    real :: field
+  end type
+contains
+  subroutine init(this)
+    class(polyty), intent(inout) :: this
+    !$acc enter data copyin(this, this%field)
+  end subroutine
+  subroutine init_assumed_type(var)
+    type(*), intent(inout) :: var
+    !$acc enter data copyin(var)
+  end subroutine
+  subroutine init_unlimited(this)
+    class(*), intent(inout) :: this
+    !$acc enter data copyin(this)
+    select type(this)
+    type is(real)
+      !$acc enter data copyin(this)
+    class is(polyty)
+      !$acc enter data copyin(this, this%field)
+    end select
+  end subroutine
+end module
+
+! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "this", structured = false}
+! CHECK: Mappable: !fir.class<!fir.type<_QMmmTpolyty{field:f32}>>
+! CHECK: Type category: composite
+! CHECK: Visiting: {{.*}} acc.copyin {{.*}} {name = "this%field", structured = false}
+! CHECK: Pointer-like: !fir.ref<f32>
+! CHECK: Type category: composite
+
+! For unlimited polymorphic entities and assumed types - they effectively have
+! no declared type. Thus the type categorizer cannot categorize it.
+! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "var", structured = false}
+! CHECK: Pointer-like: !fir.ref<none>
+! CHECK: Type category: uncategorized
+! CHECK: Visiting: {{.*}} = acc.copyin {{.*}} {name = "this", structured = false}
+! CHECK: Mappable: !fir.class<none>
+! CHECK: Type category: uncategorized
+
+! TODO: After using select type - the appropriate type category should be
+! possible. Add the rest of the test once OpenACC lowering correctly handles
+! unlimited polymorhic.
diff --git a/flang/test/Fir/convert-nontemporal-to-llvm.fir b/flang/test/Fir/convert-nontemporal-to-llvm.fir
index 6200ef1c621d..e0a836ab0dad 100644
--- a/flang/test/Fir/convert-nontemporal-to-llvm.fir
+++ b/flang/test/Fir/convert-nontemporal-to-llvm.fir
@@ -14,7 +14,7 @@
 // CHECK:    %[[VAL5:.*]] = llvm.alloca %[[CONST_VAL4]] x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr
 // CHECK:    %[[CONST_VAL5:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[VAL6:.*]] = llvm.load %[[VAL1]] : !llvm.ptr -> i32
-// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) private(@_QFtestEi_private_i32 %[[VAL2]] -> %arg0 : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[VAL5]], %[[VAL3]] : !llvm.ptr, !llvm.ptr) {
 // CHECK:      omp.loop_nest (%{{.*}}) : i32 = (%[[CONST_VAL5]]) to (%[[VAL6]]) inclusive step (%[[CONST_VAL5]]) {
 // CHECK:        llvm.store %{{.*}}, %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[VAL8:.*]] = llvm.load %[[VAL5]] {nontemporal} : !llvm.ptr -> i32
@@ -33,9 +33,9 @@
     %3 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFtestEi"}
     %4 = fir.alloca i32 {bindc_name = "n", uniq_name = "_QFtestEn"}
     %5 = fir.load %4 : !fir.ref<i32>
-    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFtestEi_private_i32 %3 -> %arg0 : !fir.ref<i32>) {
+    omp.simd nontemporal(%0, %2 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg1) : i32 = (%c1_i32) to (%5) inclusive step (%c1_i32) {
-        fir.store %arg1 to %arg0 : !fir.ref<i32>
+        fir.store %arg1 to %3 : !fir.ref<i32>
         %6 = fir.load %0 {nontemporal}: !fir.ref<i32>
         %7 = fir.load %1 : !fir.ref<i32>
         %8 = arith.addi %6, %7 : i32
@@ -52,12 +52,12 @@
 // CHECK:    %[[IDX_VAL:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:    %[[CONST_VAL1:.*]] = llvm.mlir.constant(0 : index) : i64
 // CHECK:    %[[END_IDX:.*]] = llvm.mlir.constant(100 : i32) : i32
-// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %[[ALLOCA2]] -> %[[ARG2:.*]] : !llvm.ptr) {
+// CHECK:    omp.simd nontemporal(%[[ARG0:.*]] : !llvm.ptr) {
 // CHECK:      omp.loop_nest (%[[ARG3:.*]]) : i32 = (%[[IDX_VAL]]) to (%[[END_IDX]]) inclusive step (%[[IDX_VAL]]) {
-// CHECK:        llvm.store %[[ARG3]], %[[ARG2]] : i32, !llvm.ptr
+// CHECK:        llvm.store %[[ARG3]], %{{.*}} : i32, !llvm.ptr
 // CHECK:        %[[CONST_VAL2:.*]] = llvm.mlir.constant(48 : i32) : i32
 // CHECK:        "llvm.intr.memcpy"(%[[ALLOCA1:.*]], %[[ARG0]], %[[CONST_VAL2]]) <{isVolatile = false}> : (!llvm.ptr, !llvm.ptr, i32) -> ()
-// CHECK:        %[[VAL1:.*]] = llvm.load %[[ARG2]] : !llvm.ptr -> i32
+// CHECK:        %[[VAL1:.*]] = llvm.load %{{.*}} : !llvm.ptr -> i32
 // CHECK:        %[[VAL2:.*]] = llvm.sext %[[VAL1]] : i32 to i64
 // CHECK:        %[[VAL3:.*]] = llvm.getelementptr %[[ALLOCA1]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)>
 // CHECK:        %[[VAL4:.*]] = llvm.load %[[VAL3]] : !llvm.ptr -> !llvm.ptr
@@ -91,11 +91,11 @@
     %1 = fir.allocmem !fir.array<?xi32>, %c100 {fir.must_be_heap = true, uniq_name = "_QFsimd_nontemporal_allocatableEx.alloc"}
     %2 = fircg.ext_embox %1(%c100) : (!fir.heap<!fir.array<?xi32>>, index) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %2 to %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %0 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        fir.store %arg3 to %arg2 : !fir.ref<i32>
+        fir.store %arg3 to %0 : !fir.ref<i32>
         %7 = fir.load %arg0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-        %8 = fir.load %arg2 : !fir.ref<i32>
+        %8 = fir.load %0 : !fir.ref<i32>
         %9 = fir.convert %8 : (i32) -> i64
         %10 = fir.box_addr %7 : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
         %11:3 = fir.box_dims %7, %c0 : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index 6d8a8bb606b9..0e2bfe48a807 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -1817,8 +1817,8 @@ func.func private @custom_typeP.field_1.offset() -> i32
 func.func private @custom_typeP.field_2.offset() -> i32
 
 func.func @field_index_dynamic_size() -> () {
-  %1 = fir.field_index field_1, !fir.type<custom_type{field_1:i32, field_2:!fir.array<?xf32>}>
-  %2 = fir.field_index field_2, !fir.type<custom_type{field_1:i32, field_2:!fir.array<?xf32>}>
+  %1 = fir.field_index field_1, !fir.type<custom_type(l:i32){field_1:i32, field_2:!fir.array<?xf32>}>
+  %2 = fir.field_index field_2, !fir.type<custom_type(l:i32){field_1:i32, field_2:!fir.array<?xf32>}>
   return
 }
 
diff --git a/flang/test/Fir/simd-nontemporal.fir b/flang/test/Fir/simd-nontemporal.fir
index 31051ff52f9b..a58bf01a6098 100644
--- a/flang/test/Fir/simd-nontemporal.fir
+++ b/flang/test/Fir/simd-nontemporal.fir
@@ -17,11 +17,11 @@ func.func @_QPsimd_with_nontemporal_clause(%arg0: !fir.ref<i32> {fir.bindc_name
     %8 = fir.declare %7 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
     %9 = fir.declare %arg0 dummy_scope %0 {uniq_name = "_QFsimd_with_nontemporal_clauseEn"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32>
     %10 = fir.load %9 : !fir.ref<i32>
-    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    // CHECK: omp.simd nontemporal(%[[A_DECL]], %[[C_DECL]] : !fir.ref<i32>, !fir.ref<i32>) {
     // CHECK-NEXT: omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) private(@_QFsimd_with_nontemporal_clauseEi_private_i32 %8 -> %arg1 : !fir.ref<i32>) {
+    omp.simd nontemporal(%2, %6 : !fir.ref<i32>, !fir.ref<i32>) {
       omp.loop_nest (%arg2) : i32 = (%c1_i32) to (%10) inclusive step (%c1_i32) {
-        %11 = fir.declare %arg1 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %11 = fir.declare %8 {uniq_name = "_QFsimd_with_nontemporal_clauseEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg2 to %11 : !fir.ref<i32>
         // CHECK:  %[[LOAD:.*]] = fir.load %[[A_DECL]] {nontemporal} : !fir.ref<i32>
         %12 = fir.load %2 : !fir.ref<i32>
@@ -56,11 +56,11 @@ func.func @_QPsimd_nontemporal_allocatable(%arg0: !fir.ref<!fir.box<!fir.heap<!f
     %9 = fir.shape %7 : (index) -> !fir.shape<1>
     %10 = fir.embox %8(%9) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
     fir.store %10 to %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    // CHECK:  omp.simd nontemporal(%[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
     // CHECK:   omp.loop_nest (%{{.*}}) : i32 = (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}}) {
-    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) private(@_QFsimd_nontemporal_allocatableEi_private_i32 %2 -> %arg2 : !fir.ref<i32>) {
+    omp.simd nontemporal(%3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) {
       omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100_i32) inclusive step (%c1_i32) {
-        %16 = fir.declare %arg2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
+        %16 = fir.declare %2 {uniq_name = "_QFsimd_nontemporal_allocatableEi"} : (!fir.ref<i32>) -> !fir.ref<i32>
         fir.store %arg3 to %16 : !fir.ref<i32>
         // CHECK:  %[[VAL1:.*]] = fir.load %[[X_DECL]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
         %17 = fir.load %3 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
diff --git a/flang/test/HLFIR/declare-codegen.fir b/flang/test/HLFIR/declare-codegen.fir
index bd0d61a2559d..a4edb630c4ad 100644
--- a/flang/test/HLFIR/declare-codegen.fir
+++ b/flang/test/HLFIR/declare-codegen.fir
@@ -219,3 +219,21 @@ func.func @assumed_rank_declare(%arg0: !fir.box<!fir.array<*:f32>>) {
 // CHECK-SAME: %[[VAL_0:.*]]: !fir.box<!fir.array<*:f32>>) {
 // CHECK:    %[[VAL_1:.*]] = fir.declare %[[VAL_0]] {uniq_name = "x"} : (!fir.box<!fir.array<*:f32>>) -> !fir.box<!fir.array<*:f32>>
 // CHECK:    %[[VAL_2:.*]] = fir.rebox_assumed_rank %[[VAL_1]] lbs ones : (!fir.box<!fir.array<*:f32>>) -> !fir.box<!fir.array<*:f32>>
+
+func.func @no_useless_rebox(%arg0: !fir.class<!fir.type<sometype{i:i32}>>) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "x"} : (!fir.class<!fir.type<sometype{i:i32}>>) -> (!fir.class<!fir.type<sometype{i:i32}>>, !fir.class<!fir.type<sometype{i:i32}>>)
+  fir.call @takes_class(%0#0) : (!fir.class<!fir.type<sometype{i:i32}>>) -> ()
+  return
+}
+// CHECK-LABEL: @no_useless_rebox
+// CHECK-NOT: fir.rebox
+// CHECK: return
+
+func.func @rebox_scalar_attrs(%arg0: !fir.class<!fir.ptr<!fir.type<sometype{i:i32}>>>) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "x"} : (!fir.class<!fir.ptr<!fir.type<sometype{i:i32}>>>) -> (!fir.class<!fir.type<sometype{i:i32}>>, !fir.class<!fir.type<sometype{i:i32}>>)
+  fir.call @takes_class(%0#0) : (!fir.class<!fir.type<sometype{i:i32}>>) -> ()
+  return
+}
+// CHECK-LABEL: @rebox_scalar_attrs
+// CHECK: fir.rebox %{{.*}} : (!fir.class<!fir.ptr<!fir.type<sometype{i:i32}>>>) -> !fir.class<!fir.type<sometype{i:i32}>>
+// CHECK: return
diff --git a/flang/test/HLFIR/opt-scalar-assign.fir b/flang/test/HLFIR/opt-scalar-assign.fir
index 02ab02945b04..74cdcd9622ad 100644
--- a/flang/test/HLFIR/opt-scalar-assign.fir
+++ b/flang/test/HLFIR/opt-scalar-assign.fir
@@ -12,18 +12,18 @@ func.func @_QPtest1() {
   return
 }
 // CHECK-LABEL:   func.func @_QPtest1() {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_2:.*]] = arith.constant 11 : index
-// CHECK:           %[[VAL_3:.*]] = arith.constant 13 : index
-// CHECK:           %[[VAL_4:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
-// CHECK:           %[[VAL_5:.*]] = fir.shape %[[VAL_2]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
-// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%[[VAL_5]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
-// CHECK:           fir.do_loop %[[VAL_7:.*]] = %[[VAL_0]] to %[[VAL_3]] step %[[VAL_0]] unordered {
-// CHECK:             fir.do_loop %[[VAL_8:.*]] = %[[VAL_0]] to %[[VAL_2]] step %[[VAL_0]] unordered {
-// CHECK:               %[[VAL_9:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_8]], %[[VAL_7]])  : (!fir.ref<!fir.array<11x13xf32>>, index, index) -> !fir.ref<f32>
-// CHECK:               hlfir.assign %[[VAL_1]] to %[[VAL_9]] : f32, !fir.ref<f32>
-// CHECK:             }
+// CHECK:           %[[VAL_0:.*]] = arith.constant 143 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 11 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 13 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.array<11x13xf32> {bindc_name = "x", uniq_name = "_QFtest1Ex"}
+// CHECK:           %[[VAL_6:.*]] = fir.shape %[[VAL_3]], %[[VAL_4]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_5]](%[[VAL_6]]) {uniq_name = "_QFtest1Ex"} : (!fir.ref<!fir.array<11x13xf32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<11x13xf32>>, !fir.ref<!fir.array<11x13xf32>>)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.array<11x13xf32>>) -> !fir.ref<!fir.array<143xf32>>
+// CHECK:           fir.do_loop %[[VAL_9:.*]] = %[[VAL_1]] to %[[VAL_0]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_10:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_9]]) : (!fir.ref<!fir.array<143xf32>>, index) -> !fir.ref<f32>
+// CHECK:             hlfir.assign %[[VAL_2]] to %[[VAL_10]] : f32, !fir.ref<f32>
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
@@ -129,3 +129,29 @@ func.func @_QPtest5(%arg0: !fir.ref<!fir.array<77xcomplex<f32>>> {fir.bindc_name
 // CHECK:           }
 // CHECK:           return
 // CHECK:         }
+
+func.func @_QPtest6(%arg0: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+  %c0_i32 = arith.constant 0 : i32
+  %0:2 = hlfir.declare %arg0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+  hlfir.assign %c0_i32 to %0#0 realloc : i32, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+  return
+}
+
+// CHECK-LABEL:   func.func @_QPtest6(
+// CHECK-SAME:                        %[[VAL_0:.*]]: !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>> {fir.bindc_name = "x"}) {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFtest6Ex"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>)
+// CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[VAL_6:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_2]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]], %[[VAL_1]] : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, index) -> (index, index, index)
+// CHECK:           %[[VAL_8:.*]] = arith.muli %[[VAL_6]]#1, %[[VAL_7]]#1 : index
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_8]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.rebox %[[VAL_5]](%[[VAL_9]]) : (!fir.box<!fir.heap<!fir.array<?x?xi32>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           fir.do_loop %[[VAL_11:.*]] = %[[VAL_1]] to %[[VAL_8]] step %[[VAL_1]] unordered {
+// CHECK:             %[[VAL_12:.*]] = hlfir.designate %[[VAL_10]] (%[[VAL_11]]) : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK:             hlfir.assign %[[VAL_3]] to %[[VAL_12]] : i32, !fir.ref<i32>
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
diff --git a/flang/test/Integration/OpenMP/map-types-and-sizes.f90 b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
index 5013c0fdc404..7fa0f242b2ca 100644
--- a/flang/test/Integration/OpenMP/map-types-and-sizes.f90
+++ b/flang/test/Integration/OpenMP/map-types-and-sizes.f90
@@ -6,7 +6,7 @@
 ! added to this directory and sub-directories.
 !===----------------------------------------------------------------------===!
 
-!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-llvm -fopenmp -fopenmp-version=51 -fopenmp-targets=amdgcn-amd-amdhsa %s -o - | FileCheck %s
 
 !===============================================================================
 ! Check MapTypes for target implicit captures
@@ -494,15 +494,18 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_single_member_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_single_memberTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 1
+!CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-!CHECK: store ptr %[[MEMBER_ACCESS]], ptr %[[OFFLOAD_PTR_ARR]], align 8
+!CHECK: store ptr %[[MEMBER_ACCESS_2]], ptr %[[OFFLOAD_PTR_ARR]], align 8
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_multiple_nested_members_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 0
-!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 1
+!CHECK: %[[MEMBER_ACCESS_0:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 0
+!CHECK: %[[MEMBER_ACCESS_2:.*]] = getelementptr %_QFmaptype_derived_explicit_multiple_nested_membersTnested, ptr %[[MEMBER_ACCESS_0]], i32 0, i32 1
 !CHECK: %[[ARR_END_OFF:.*]] = getelementptr float, ptr %[[MEMBER_ACCESS_2]], i64 1
 !CHECK: %[[ARR_END:.*]] = ptrtoint ptr %[[ARR_END_OFF]] to i64
 !CHECK: %[[FIRST_MEMBER:.*]] = ptrtoint ptr %[[MEMBER_ACCESS_1]] to i64
@@ -525,8 +528,9 @@ end subroutine mapType_common_block_members
 
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_derived_explicit_nested_member_with_bounds_{{.*}}
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, i64 1, align 8
-!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2, i32 2
-!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS]], i64 0, i64 1
+!CHECK: %[[MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTscalar_and_array, ptr %[[ALLOCA]], i32 0, i32 2
+!CHECK: %[[MEMBER_ACCESS_1:.*]] = getelementptr %_QFmaptype_derived_explicit_nested_member_with_boundsTnested, ptr %[[MEMBER_ACCESS]], i32 0, i32 2
+!CHECK: %[[ARR_OFF:.*]] = getelementptr inbounds [10 x i32], ptr %[[MEMBER_ACCESS_1]], i64 0, i64 1
 !CHECK: %[[BASE_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
 !CHECK: store ptr %[[ALLOCA]], ptr %[[BASE_PTR_ARR]], align 8
 !CHECK: %[[OFFLOAD_PTR_ARR:.*]] = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
@@ -574,19 +578,27 @@ end subroutine mapType_common_block_members
 !CHECK: %[[OFFLOAD_SIZE_ARR:.*]] = getelementptr inbounds [4 x i64], ptr %.offload_sizes, i32 0, i32 3
 !CHECK: store i64 %[[DTYPE_SEL]], ptr %[[OFFLOAD_SIZE_ARR]], align 8
 
+
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_alloca_derived_type_{{.*}}
 !CHECK: %{{.*}} = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, align 8
 !CHECK: %[[DTYPE_DESC_ALLOCA_3:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, i64 1, align 8
-!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
-!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
-!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
+!CHECK: %{{.*}} = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %{{.*}}, i32 0, i32 0
+!CHECK: %{{.*}} = load ptr, ptr %{{.*}}, align 8
+!CHECK: %{{.*}} = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %{{.*}}, i32 0, i32 4
 !CHECK: %[[ACCESS_DESC_MEMBER_UB:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ARRAY_MEMBER_DESC_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[LOAD_DESC_MEMBER_UB:.*]] = load i64, ptr %[[ACCESS_DESC_MEMBER_UB]], align 8
 !CHECK: %[[OFFSET_MEMBER_UB:.*]] = sub i64 %[[LOAD_DESC_MEMBER_UB]], 1
+!CHECK: %[[DTYPE_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA_2]], i32 0, i32 0
+!CHECK: %[[DTYPE_BASE_ADDR_LOAD:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS]], align 8
+!CHECK: %[[DTYPE_ALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD]], i32 0, i32 4
 !CHECK: %[[DTYPE_ALLOCA_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[DTYPE_ALLOCA_MEMBER_ACCESS]], i32 0, i32 0
+
 !CHECK: %[[DTYPE_BASE_ADDR_ACCESS_2:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, ptr, [1 x i64] }, ptr %[[DTYPE_DESC_ALLOCA]], i32 0, i32 0
 !CHECK: %[[DTYPE_BASE_ADDR_LOAD_2:.*]] = load ptr, ptr %[[DTYPE_BASE_ADDR_ACCESS_2]], align 8
 !CHECK: %[[DTYPE_NONALLOCA_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_alloca_derived_typeTone_layer, ptr %[[DTYPE_BASE_ADDR_LOAD_2]], i32 0, i32 5
@@ -715,11 +727,11 @@ end subroutine mapType_common_block_members
 !CHECK-LABEL: define {{.*}} @{{.*}}maptype_nested_derived_type_alloca{{.*}}
 !CHECK: %[[ALLOCATABLE_MEMBER_ALLOCA:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, align 8
 !CHECK: %[[ALLOCA:.*]] = alloca %_QFmaptype_nested_derived_type_allocaTtop_layer, i64 1, align 8
+!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
+!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[ALLOCATABLE_MEMBER_BASE_ADDR:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[ALLOCATABLE_MEMBER_ALLOCA]], i32 0, i32 7, i64 0, i32 1
 !CHECK: %[[ALLOCATABLE_MEMBER_ADDR_LOAD:.*]] = load i64, ptr %[[ALLOCATABLE_MEMBER_BASE_ADDR]], align 8
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_1:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_ADDR_LOAD]], 1
-!CHECK: %[[NESTED_DTYPE_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTtop_layer, ptr %[[ALLOCA]], i32 0, i32 6
-!CHECK: %[[NESTED_MEMBER_ACCESS:.*]] = getelementptr %_QFmaptype_nested_derived_type_allocaTmiddle_layer, ptr %[[NESTED_DTYPE_MEMBER_ACCESS]], i32 0, i32 2
 !CHECK: %[[NESTED_MEMBER_BASE_ADDR_ACCESS:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_2:.*]] = sub i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_1]], 0
 !CHECK: %[[ALLOCATABLE_MEMBER_SIZE_CALC_3:.*]] = add i64 %[[ALLOCATABLE_MEMBER_SIZE_CALC_2]], 1
diff --git a/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90 b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
new file mode 100644
index 000000000000..249d403057b6
--- /dev/null
+++ b/flang/test/Integration/amdgpu-target-desc-cast-to-global-addrspace.f90
@@ -0,0 +1,18 @@
+!REQUIRES: amdgpu-registered-target
+
+!RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -target-cpu gfx908 %s -o - | FileCheck %s
+
+subroutine maintest
+  implicit none
+
+  type r1_t
+  end type r1_t
+
+  type(r1_t), pointer :: A
+end subroutine
+
+! CHECK: @[[TYPE_DESC:.*XdtXr1_t]] = linkonce_odr addrspace(1) constant %_QM__fortran_type_infoTderivedtype
+
+! CHECK: define void @maintest_() {{.*}} {
+! CHECK:   store { {{.*}} } { {{.*}}, ptr addrspacecast (ptr addrspace(1) @[[TYPE_DESC]] to ptr), {{.*}} }, {{.*}}
+! CHECK: }
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
new file mode 100644
index 000000000000..72b7a78bef51
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-complex.f90
@@ -0,0 +1,116 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Test that debug metadata for derived types is not duplicated more than needed
+! when emitting debug info with non trivial cycles.
+
+
+! In the type graph below, G has a back edge to B, and F to D.
+! This causes C to be in the middle of B cycle, and E to be
+! both in B and D cycles.
+! C and E are used in several contexts, under B, under D, and outside
+! of it to test how metadata is generated for them.
+!
+! Without "local caching" of C and E when generating mlir::LLVM::DITypeAttr
+! for such derived types, many duplicate llvm metadata for the derived types
+! would be emitted, while with the right duplication of mlir::LLVM::DITypeAttr,
+! a lot more duplicate llvm IR metadata ends up emitted (19 DICompositeType
+! vs 71 before the patch that added this test).
+!
+!
+!  A -> B -> C -> D -> E -> F -> G -> B
+!  |    |         |         |
+!  |    |         |         | -> D
+!  |    |         |
+!  |    |         | -> H -> E
+!  |    |
+!  |    | -> I -> E
+!  |         | -> C
+!  |
+!  | -> C
+!  | -> E
+
+subroutine type_cycles_caching()
+  type g
+    type(b), pointer :: c_b
+  end type
+  type f
+    type(g) :: c_b
+    type(d), pointer :: c_d
+  end type
+  type e
+    type(f) :: c_f
+  end type
+    type h
+      ! Can reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+    end type
+  type d
+    type(e) :: c_e
+    type(h) :: c_h
+  end type
+  type c
+    type(d) :: c_d
+  end type
+    type i
+      ! Cannot reuse metadata of type(e) under 'd'.
+      type(e) :: c_e
+      ! Can reuse metadata of type(c) under 'b'.
+      type(c) :: c_c
+    end type
+  type b
+    type(c) :: c_c
+    type(i) :: c_i
+  end type
+  type a
+    type(b) :: c_b
+    ! Cannot reuse metadata of type(c) under 'b'
+    type(c) :: c_c
+    ! Cannot reuse metadata of type(e) under 'd', nor the one under 'b'
+    type(e) :: c_e
+  end type
+  type(a) :: xa
+  ! Can reuse metadata of type(c) for xa%c_c
+  type(c) :: xc
+  ! Can reuse metadata of type(c) for xa%c_e
+  type(e) :: xe
+  call bar(xa, xc, xe)
+end subroutine
+
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "a",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "b",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "i",
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "c"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "d"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "g"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "h"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "e"
+! CHECK-NOT: distinct !DICompositeType
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "f"
+! CHECK-NOT: distinct !DICompositeType
diff --git a/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90 b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
new file mode 100644
index 000000000000..c1f49d2f4a74
--- /dev/null
+++ b/flang/test/Integration/debug-cyclic-derived-type-caching-simple.f90
@@ -0,0 +1,39 @@
+! RUN: %flang_fc1 -emit-llvm -debug-info-kind=standalone %s -o - | FileCheck  %s
+
+! Simple test that checks that metadata for `t0` is only duplicated once.
+
+! The difficulty is that at the mlir::LLVM::DITypeAttr, because of the
+! lack of MLIR attribute true recursion, the mlir::LLVM::DITypeAttr for
+! `t0` inside `t1` is special because it is only valid when found
+! in an mlir::LLVM::DITypeAttr tree under `base` (it will point to `base`
+! via an integer id that is only meanigfu;l when a node with such id has
+! been traversed).
+! A different node has to be created for `t0` usage in `x0` (will
+! point to the actual mlir::LLVM::DITypeAttr for `base` instead of
+! an integer id since the cycle is already "taken care of" in `base`
+! definition).
+! However, the same special `t0` node can be used for both `x1_1` and
+! `x1_2` components because they are both under `base` in the
+! mlir::LLVM::DITypeAttr tree definition. This used to not be the case,
+! leading to a lot of duplicate mlir::LLVM::DITypeAttr and actual LLVM IR
+! metadata, causing noticeable compilation slowdowns in apps with non trivial
+! derived types.
+
+subroutine duplicate_cycle_branch()
+  type t0
+    type(base), pointer :: x
+  end type
+  type t1
+    type(t0) :: x1_1
+    type(t0) :: x1_2
+  end type
+  type base
+    type(t1) :: x_base
+  end type
+  type(base) :: x
+  type(t0) :: x0
+  call bar(x, x0)
+end subroutine
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
+! CHECK-NOT: distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t0",
diff --git a/flang/test/Integration/skip-external-rtti-definition.F90 b/flang/test/Integration/skip-external-rtti-definition.F90
new file mode 100644
index 000000000000..35a5d559efa5
--- /dev/null
+++ b/flang/test/Integration/skip-external-rtti-definition.F90
@@ -0,0 +1,47 @@
+! Test -skip-external-rtti-definition option
+
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -fsyntax-only -DSTEP=1 -J%t %s
+!RUN: %flang_fc1 -emit-llvm -J%t  %s -o - | FileCheck %s -check-prefix=LINKONCE
+!RUN: %flang_fc1 -emit-llvm -J%t -mllvm -skip-external-rtti-definition %s -o - | FileCheck %s -check-prefix=EXTERNAL
+
+#if STEP == 1
+module module_external_type_definition
+ type t1
+ end type
+end module
+#else
+
+module module_same_unit_type_definition
+ type t2
+ end type
+end module
+
+subroutine test
+  use module_external_type_definition
+  use module_same_unit_type_definition
+  interface
+  subroutine needs_descriptor(x)
+    class(*) :: x
+  end subroutine
+  end interface
+  type(t1) :: x1
+  type(t2) :: x2
+  call needs_descriptor(x1)
+  call needs_descriptor(x2)
+end subroutine
+
+#endif
+
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXnXt1 = linkonce_odr constant [2 x i8] c"t1"
+! LINKONCE-DAG: @_QMmodule_external_type_definitionEXdtXt1 = linkonce_odr constant {{.*}} {
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXnXt2 = linkonce_odr constant [2 x i8] c"t2"
+! LINKONCE-DAG: @_QMmodule_same_unit_type_definitionEXdtXt2 = linkonce_odr constant {{.*}} {
+
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_same_unit_type_definitionEXnXt2 = constant [2 x i8] c"t2"
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_same_unit_type_definitionEXdtXt2 = constant {{.*}} {
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
+! EXTERNAL: @_QMmodule_external_type_definitionEXdtXt1 =  external constant ptr
+! EXTERNAL-NOT: @_QMmodule_external_type_definitionEXnXt1
diff --git a/flang/test/Lower/CUDA/cuda-cooperative.cuf b/flang/test/Lower/CUDA/cuda-cooperative.cuf
new file mode 100644
index 000000000000..657a87c0b5a0
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-cooperative.cuf
@@ -0,0 +1,99 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+! Test CUDA Fortran procedures available in cooperative_groups module.
+
+attributes(grid_global) subroutine g1()
+  use cooperative_groups
+  type(grid_group) :: gg
+  gg = this_grid()
+end subroutine
+
+! CHECK: %{{.*}} = fir.alloca !fir.type<_QMcooperative_groupsTgrid_group{_QMcooperative_groupsTgrid_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[RES:.*]] = fir.alloca !fir.type<_QMcooperative_groupsTgrid_group{_QMcooperative_groupsTgrid_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[THREAD_ID_X:.*]] = nvvm.read.ptx.sreg.tid.x : i32
+! CHECK: %[[THREAD_ID_Y:.*]] = nvvm.read.ptx.sreg.tid.y : i32
+! CHECK: %[[THREAD_ID_Z:.*]] = nvvm.read.ptx.sreg.tid.z : i32
+! CHECK: %[[BLOCK_ID_X:.*]] = nvvm.read.ptx.sreg.ctaid.x : i32
+! CHECK: %[[BLOCK_ID_Y:.*]] = nvvm.read.ptx.sreg.ctaid.y : i32
+! CHECK: %[[BLOCK_ID_Z:.*]] = nvvm.read.ptx.sreg.ctaid.z : i32
+! CHECK: %[[BLOCK_DIM_X:.*]] = nvvm.read.ptx.sreg.ntid.x : i32
+! CHECK: %[[BLOCK_DIM_Y:.*]] = nvvm.read.ptx.sreg.ntid.y : i32
+! CHECK: %[[BLOCK_DIM_Z:.*]] = nvvm.read.ptx.sreg.ntid.z : i32
+! CHECK: %[[GRID_DIM_X:.*]] = nvvm.read.ptx.sreg.nctaid.x : i32
+! CHECK: %[[GRID_DIM_Y:.*]] = nvvm.read.ptx.sreg.nctaid.y : i32
+! CHECK: %[[GRID_DIM_Z:.*]] = nvvm.read.ptx.sreg.nctaid.z : i32
+
+! CHECK: %[[R1:.*]] = arith.muli %[[BLOCK_DIM_Z]], %[[GRID_DIM_Z]] : i32
+! CHECK: %[[R2:.*]] = arith.muli %[[BLOCK_DIM_Y]], %[[GRID_DIM_Y]] : i32
+! CHECK: %[[R3:.*]] = arith.muli %[[BLOCK_DIM_X]], %[[GRID_DIM_X]] : i32
+! CHECK: %[[R4:.*]] = arith.muli %[[R1]], %[[R2]] : i32
+! CHECK: %[[SIZE:.*]] = arith.muli %[[R4]], %[[R3]] : i32
+
+! CHECK: %[[R1:.*]] = arith.muli %[[BLOCK_ID_Z]], %[[GRID_DIM_Y]] : i32
+! CHECK: %[[R2:.*]] = arith.muli %[[R1]], %[[GRID_DIM_X]] : i32
+! CHECK: %[[R3:.*]] = arith.muli %[[BLOCK_ID_Y]], %[[GRID_DIM_X]] : i32
+! CHECK: %[[R4:.*]] = arith.addi %[[R2]], %[[R3]] : i32
+! CHECK: %[[TMP:.*]] = arith.addi %[[R4]], %[[BLOCK_ID_X]] : i32
+
+! CHECK: %[[R1:.*]] = arith.muli %[[BLOCK_DIM_X]], %[[BLOCK_DIM_Y]] : i32
+! CHECK: %[[R2:.*]] = arith.muli %[[R1]], %[[BLOCK_DIM_Z]] : i32
+! CHECK: %[[R3:.*]] = arith.muli %[[THREAD_ID_Z]], %[[BLOCK_DIM_Y]] : i32
+! CHECK: %[[R4:.*]] = arith.muli %[[R3]], %[[BLOCK_DIM_X]] : i32
+! CHECK: %[[R5:.*]] = arith.muli %[[THREAD_ID_Y]], %[[BLOCK_DIM_X]] : i32
+! CHECK: %[[RES0:.*]] = arith.muli %[[TMP]], %[[R2]] : i32
+! CHECK: %[[RES1:.*]] = arith.addi %[[RES0]], %[[R4]] : i32
+! CHECK: %[[RES2:.*]] = arith.addi %[[RES1]], %[[R5]] : i32
+! CHECK: %[[RES3:.*]] = arith.addi %[[RES2]], %[[THREAD_ID_X]] : i32
+! CHECK: %[[ONE:.*]] = arith.constant 1 : i32
+! CHECK: %[[RANK:.*]] = arith.addi %[[RES3]], %[[ONE]] : i32
+! CHECK: %[[COORD_SIZE:.*]] = fir.coordinate_of %[[RES]], size : (!fir.ref<!fir.type<_QMcooperative_groupsTgrid_group{_QMcooperative_groupsTgrid_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[SIZE]] to %[[COORD_SIZE]] : !fir.ref<i32>
+! CHECK: %[[COORD_RANK:.*]] = fir.coordinate_of %[[RES]], rank : (!fir.ref<!fir.type<_QMcooperative_groupsTgrid_group{_QMcooperative_groupsTgrid_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[RANK]] to %[[COORD_RANK]] : !fir.ref<i32>
+
+attributes(grid_global) subroutine w1()
+  use cooperative_groups
+  type(coalesced_group) :: gg
+  gg = this_warp()
+end subroutine
+
+! CHECK: %[[WARPSIZE:.*]] = fir.alloca i32 {bindc_name = "__builtin_warpsize", uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"}
+! CHECK: %[[WARPSIZE_DECL:.*]]:2 = hlfir.declare %[[WARPSIZE]] {uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK: %[[COALESCED_GROUP:.*]] = fir.alloca !fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[C32:.*]] = arith.constant 32 : i32
+! CHECK: %[[SIZE_COORD:.*]] = fir.coordinate_of %[[COALESCED_GROUP]], size : (!fir.ref<!fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[C32]] to %[[SIZE_COORD]] : !fir.ref<i32>
+
+! CHECK: %[[THREAD_ID:.*]] = nvvm.read.ptx.sreg.tid.x : i32
+! CHECK: %[[C31:.*]] = arith.constant 31 : i32
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[AND:.*]] = arith.andi %[[THREAD_ID]], %[[C31]] : i32
+! CHECK: %[[RANK:.*]] = arith.addi %[[AND]], %[[C1]] : i32
+! CHECK: %[[RANK_COORD:.*]] = fir.coordinate_of %{{.*}}, rank : (!fir.ref<!fir.type<_QMcooperative_groupsTcoalesced_group{_QMcooperative_groupsTcoalesced_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+
+attributes(grid_global) subroutine t1()
+  use cooperative_groups
+  type(thread_group) :: gg
+  gg = this_thread_block()
+end subroutine
+! CHECK: %{{.*}} = fir.alloca !fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[THREAD_GROUP:.*]] = fir.alloca !fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>
+! CHECK: %[[NTID_X:.*]] = nvvm.read.ptx.sreg.ntid.x : i32
+! CHECK: %[[NTID_Y:.*]] = nvvm.read.ptx.sreg.ntid.y : i32
+! CHECK: %[[NTID_Z:.*]] = nvvm.read.ptx.sreg.ntid.z : i32
+! CHECK: %[[SIZE_ZY:.*]] = arith.muli %[[NTID_Z]], %[[NTID_Y]] : i32
+! CHECK: %[[SIZE:.*]] = arith.muli %[[SIZE_ZY]], %[[NTID_X]] : i32
+! CHECK: %[[TID_X:.*]] = nvvm.read.ptx.sreg.tid.x : i32
+! CHECK: %[[TID_Y:.*]] = nvvm.read.ptx.sreg.tid.y : i32
+! CHECK: %[[TID_Z:.*]] = nvvm.read.ptx.sreg.tid.z : i32
+! CHECK: %[[RANK_ZY:.*]] = arith.muli %[[TID_Z]], %[[NTID_Y]] : i32
+! CHECK: %[[RANK_ZYX:.*]] = arith.muli %[[RANK_ZY]], %[[NTID_X]] : i32
+! CHECK: %[[RANK_YX:.*]] = arith.muli %[[TID_Y]], %[[NTID_X]] : i32
+! CHECK: %[[RANK_SUM1:.*]] = arith.addi %[[RANK_ZYX]], %[[RANK_YX]] : i32
+! CHECK: %[[RANK_SUM2:.*]] = arith.addi %[[RANK_SUM1]], %[[TID_X]] : i32
+! CHECK: %[[C1:.*]] = arith.constant 1 : i32
+! CHECK: %[[RANK:.*]] = arith.addi %[[RANK_SUM2]], %[[C1]] : i32
+! CHECK: %[[SIZE_COORD:.*]] = fir.coordinate_of %[[THREAD_GROUP]], size : (!fir.ref<!fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[SIZE]] to %[[SIZE_COORD]] : !fir.ref<i32>
+! CHECK: %[[RANK_COORD:.*]] = fir.coordinate_of %[[THREAD_GROUP]], rank : (!fir.ref<!fir.type<_QMcooperative_groupsTthread_group{_QMcooperative_groupsTthread_group.handle:!fir.type<_QM__fortran_builtinsT__builtin_c_devptr{cptr:!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>}>,size:i32,rank:i32}>>) -> !fir.ref<i32>
+! CHECK: fir.store %[[RANK]] to %[[RANK_COORD]] : !fir.ref<i32>
diff --git a/flang/test/Lower/CUDA/cuda-runtime-check.cuf b/flang/test/Lower/CUDA/cuda-runtime-check.cuf
index f26d372769ca..1aa95ec0ff40 100644
--- a/flang/test/Lower/CUDA/cuda-runtime-check.cuf
+++ b/flang/test/Lower/CUDA/cuda-runtime-check.cuf
@@ -1,22 +1,36 @@
 ! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
 
-! Check insertion of runtime checks
+! Check insertion of runtime checks in host side only
+
+module section_test
+
+contains
 
-interface
   subroutine foo(a)
     real, device, dimension(:,:) :: a
   end subroutine
-end interface
 
-  real, device, allocatable, dimension(:,:) :: a
-  allocate(a(10,10))
-  call foo(a(1:10,1:10:2))
-end
+  subroutine test_host()
+    real, device, allocatable, dimension(:,:) :: a
+    allocate(a(10,10))
+    call foo(a(1:10,1:10:2))
+  end subroutine
+
+  attributes(device) subroutine zoo(a)
+    real, device, dimension(:,:) :: a
+  end subroutine
 
-subroutine foo(a)
-  real, device, dimension(:,:) :: a
-end subroutine
+  attributes(global) subroutine test_device()
+    real, device, allocatable, dimension(:,:) :: a
+    allocate(a(10,10))
+    call zoo(a(1:10,1:10:2))
+  end subroutine
+end module
 
-! CHECK-LABEL: func.func @_QQmain()
+! CHECK-LABEL: func.func @_QMsection_testPtest_host()
 ! CHECK: fir.call @_FortranACUFDescriptorCheckSection
-! CHECK: fir.call @_QPfoo
+! CHECK: fir.call @_QMsection_testPfoo
+
+! CHECK-LABEL: func.func @_QMsection_testPtest_device()
+! CHECK-NOT: fir.call @_FortranACUFDescriptorCheckSection
+! CHECK: fir.call @_QMsection_testPzoo
diff --git a/flang/test/Lower/HLFIR/custom-intrinsic.f90 b/flang/test/Lower/HLFIR/custom-intrinsic.f90
index 161a2ab75b7c..5ec6e0a17e9a 100644
--- a/flang/test/Lower/HLFIR/custom-intrinsic.f90
+++ b/flang/test/Lower/HLFIR/custom-intrinsic.f90
@@ -115,10 +115,10 @@ function max_array(a, b)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) {uniq_name = "_QFmax_arrayEmax_array"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_12:.*]] = hlfir.elemental %[[VAL_3]] unordered : (!fir.shape<1>) -> !hlfir.expr<42xi32> {
 ! CHECK:           ^bb0(%[[VAL_13:.*]]: index):
-! CHECK:             %[[VAL_14:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_14:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_16:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
 ! CHECK:             %[[VAL_18:.*]] = arith.cmpi sgt, %[[VAL_15]], %[[VAL_17]] : i32
 ! CHECK:             %[[VAL_19:.*]] = arith.select %[[VAL_18]], %[[VAL_15]], %[[VAL_17]] : i32
 ! CHECK:             hlfir.yield_element %[[VAL_19]] : i32
@@ -288,10 +288,10 @@ function min_array(a, b)
 ! CHECK:           %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_10]]) {uniq_name = "_QFmin_arrayEmin_array"} : (!fir.ref<!fir.array<42xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<42xi32>>, !fir.ref<!fir.array<42xi32>>)
 ! CHECK:           %[[VAL_12:.*]] = hlfir.elemental %[[VAL_3]] unordered : (!fir.shape<1>) -> !hlfir.expr<42xi32> {
 ! CHECK:           ^bb0(%[[VAL_13:.*]]: index):
-! CHECK:             %[[VAL_14:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
-! CHECK:             %[[VAL_16:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
-! CHECK:             %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_14:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_15:.*]] = fir.load %[[VAL_14]] : !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_16:.*]] = hlfir.designate %[[VAL_7]]#0 (%[[VAL_13]])  : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:             %[[VAL_17:.*]] = fir.load %[[VAL_16]] : !fir.ref<i32>
 ! CHECK:             %[[VAL_18:.*]] = arith.cmpi slt, %[[VAL_15]], %[[VAL_17]] : i32
 ! CHECK:             %[[VAL_19:.*]] = arith.select %[[VAL_18]], %[[VAL_15]], %[[VAL_17]] : i32
 ! CHECK:             hlfir.yield_element %[[VAL_19]] : i32
diff --git a/flang/test/Lower/Intrinsics/acosd.f90 b/flang/test/Lower/Intrinsics/acosd.f90
index 3f558d66c35a..7dfa28fd6494 100644
--- a/flang/test/Lower/Intrinsics/acosd.f90
+++ b/flang/test/Lower/Intrinsics/acosd.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @acosf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @acos(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
diff --git a/flang/test/Lower/Intrinsics/asind.f90 b/flang/test/Lower/Intrinsics/asind.f90
index e03f2e559855..564fa955fa96 100644
--- a/flang/test/Lower/Intrinsics/asind.f90
+++ b/flang/test/Lower/Intrinsics/asind.f90
@@ -1,5 +1,3 @@
-! RUN: bbc -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
-! RUN: bbc --math-runtime=precise -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -8,10 +6,10 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 0.017453292519943295 : f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f32
 ! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f32
-! CHECK-PRECISE: %{{.*}} = fir.call @asinf(%[[arg]]) fastmath<contract> : (f32) -> f32
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
   real(8) :: x, test_real8
@@ -19,6 +17,6 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[factor:.*]] = arith.constant 0.017453292519943295 : f64
-! CHECK: %[[arg:.*]] = arith.mulf %{{[A-Za-z0-9._]+}}, %[[factor]] fastmath<contract> : f64
-! CHECK-PRECISE: %{{.*}} = fir.call @asin(%[[arg]]) fastmath<contract> : (f64) -> f64
+! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
diff --git a/flang/test/Lower/Intrinsics/ieee_class_queries.f90 b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
index aae8071b6a8b..8d20ff8f3178 100644
--- a/flang/test/Lower/Intrinsics/ieee_class_queries.f90
+++ b/flang/test/Lower/Intrinsics/ieee_class_queries.f90
@@ -1,5 +1,5 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir -o - %s | FileCheck %s
 
   ! CHECK-LABEL: func @_QQmain
diff --git a/flang/test/Lower/Intrinsics/product.f90 b/flang/test/Lower/Intrinsics/product.f90
index f5f7cff106ce..df7c1e4ce7ea 100644
--- a/flang/test/Lower/Intrinsics/product.f90
+++ b/flang/test/Lower/Intrinsics/product.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPproduct_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32
diff --git a/flang/test/Lower/Intrinsics/sum.f90 b/flang/test/Lower/Intrinsics/sum.f90
index d36b2b126388..3167617b6045 100644
--- a/flang/test/Lower/Intrinsics/sum.f90
+++ b/flang/test/Lower/Intrinsics/sum.f90
@@ -1,5 +1,5 @@
-! REQUIRES: x86_64-registered-target
-! RUN: bbc --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu --use-desc-for-alloc=false -emit-fir -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: func @_QPsum_test(
 ! CHECK-SAME: %[[arg0:.*]]: !fir.box<!fir.array<?xi32>>{{.*}}) -> i32 {
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 3f840acefa6e..f88bbea6fca8 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -107,8 +107,6 @@ program OmpAtomicUpdate
 !CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_Y_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK: ^bb0(%[[ARG:.*]]: i32):
 !CHECK:  {{.*}} = arith.cmpi sgt, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.select {{.*}}, %[[ARG]], {{.*}} : i32
-!CHECK:  {{.*}} = arith.cmpi sgt, {{.*}}
 !CHECK:  %[[TEMP:.*]] = arith.select {{.*}} : i32
 !CHECK:  omp.yield(%[[TEMP]] : i32)
 !CHECK: }
@@ -177,13 +175,9 @@ program OmpAtomicUpdate
 !CHECK:   %[[VAL_Z_LOADED:.*]] = fir.load %[[VAL_Z_DECLARE]]#0 : !fir.ref<i32>
 !CHECK:   omp.atomic.update %[[VAL_W_DECLARE]]#0 : !fir.ref<i32> {
 !CHECK:   ^bb0(%[[ARG_W:.*]]: i32):
-!CHECK:     %[[WX_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WX_MIN:.*]] = arith.select %[[WX_CMP]], %[[ARG_W]], %[[VAL_X_LOADED]] : i32
-!CHECK:     %[[WXY_CMP:.*]] = arith.cmpi sgt, %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXY_MIN:.*]] = arith.select %[[WXY_CMP]], %[[WX_MIN]], %[[VAL_Y_LOADED]] : i32
-!CHECK:     %[[WXYZ_CMP:.*]] = arith.cmpi sgt, %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     %[[WXYZ_MIN:.*]] = arith.select %[[WXYZ_CMP]], %[[WXY_MIN]], %[[VAL_Z_LOADED]] : i32
-!CHECK:     omp.yield(%[[WXYZ_MIN]] : i32)
+!CHECK:     %[[W_CMP:.*]] = arith.cmpi sgt, %[[ARG_W]], {{.*}} : i32
+!CHECK:     %[[WXYZ_MAX:.*]] = arith.select %[[W_CMP]], %[[ARG_W]], {{.*}} : i32
+!CHECK:     omp.yield(%[[WXYZ_MAX]] : i32)
 !CHECK:   }
   !$omp atomic update
     w = max(w,x,y,z)
diff --git a/flang/test/Lower/OpenMP/minmax-optional-parameters.f90 b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
new file mode 100644
index 000000000000..418a3cad8cda
--- /dev/null
+++ b/flang/test/Lower/OpenMP/minmax-optional-parameters.f90
@@ -0,0 +1,68 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+! Check that the presence tests are done outside of the atomic update
+! construct.
+
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<f32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<f32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpf ogt, %[[V4]], %[[V5]] fastmath<contract> : f32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : f32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (f32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<f32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpf ogt, %[[V8]], %[[V10]] fastmath<contract> : f32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : f32
+!CHECK:   fir.result %[[V12]] : f32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : f32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: f32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpf ogt, %[[ARG]], %[[V9]] fastmath<contract> : f32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : f32
+!CHECK:   omp.yield(%[[V11]] : f32)
+!CHECK: }
+
+subroutine f00(a, x, y)
+  real :: a
+  real, optional :: x, y
+  !$omp atomic update
+  a = max(x, a, y)
+end
+
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[VAL_A:[0-9]+]]:2 = hlfir.declare %arg0 dummy_scope %0
+!CHECK: %[[VAL_X:[0-9]+]]:2 = hlfir.declare %arg1 dummy_scope %0
+!CHECK: %[[VAL_Y:[0-9]+]]:2 = hlfir.declare %arg2 dummy_scope %0
+!CHECK: %[[V4:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V5:[0-9]+]] = fir.load %[[VAL_X]]#0 : !fir.ref<i32>
+!CHECK: %[[V6:[0-9]+]] = fir.is_present %[[VAL_Y]]#0 : (!fir.ref<i32>) -> i1
+!CHECK: %[[V7:[0-9]+]] = arith.cmpi slt, %[[V4]], %[[V5]] : i32
+!CHECK: %[[V8:[0-9]+]] = arith.select %[[V7]], %[[V4]], %[[V5]] : i32
+!CHECK: %[[V9:[0-9]+]] = fir.if %[[V6]] -> (i32) {
+!CHECK:   %[[V10:[0-9]+]] = fir.load %[[VAL_Y]]#0 : !fir.ref<i32>
+!CHECK:   %[[V11:[0-9]+]] = arith.cmpi slt, %[[V8]], %[[V10]] : i32
+!CHECK:   %[[V12:[0-9]+]] = arith.select %[[V11]], %[[V8]], %[[V10]] : i32
+!CHECK:   fir.result %[[V12]] : i32
+!CHECK: } else {
+!CHECK:   fir.result %[[V8]] : i32
+!CHECK: }
+!CHECK: omp.atomic.update memory_order(relaxed) %[[VAL_A]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:[a-z0-9]+]]: i32):
+!CHECK:   %[[V10:[0-9]+]] = arith.cmpi slt, %[[ARG]], %[[V9]] : i32
+!CHECK:   %[[V11:[0-9]+]] = arith.select %[[V10]], %[[ARG]], %[[V9]] : i32
+!CHECK:   omp.yield(%[[V11]] : i32)
+!CHECK: }
+
+subroutine f01(a, x, y)
+  integer :: a
+  integer, optional :: x, y
+  !$omp atomic update
+  a = min(x, a, y)
+end
+
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
new file mode 100644
index 000000000000..a76e7e52100d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct01.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for tile OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end do
+end subroutine
+
+!CHECK: not yet implemented: Unhandled loop directive (tile)
diff --git a/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90 b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
new file mode 100644
index 000000000000..33b7c5a91761
--- /dev/null
+++ b/flang/test/Lower/OpenMP/nested-loop-transformation-construct02.f90
@@ -0,0 +1,20 @@
+! Test to ensure TODO message is emitted for unroll OpenMP 5.1 Directives when they are nested.
+
+!RUN: not %flang -fopenmp -fopenmp-version=51 %s 2>&1 | FileCheck %s
+
+program loop_transformation_construct
+  implicit none
+  integer, parameter :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do x = 1, I
+    y(x) = y(x) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end program loop_transformation_construct
+
+!CHECK: not yet implemented: Unhandled loop directive (unroll)
diff --git a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90 b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
index 017b2a3f2edd..2bb1036c4a7f 100644
--- a/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
+++ b/flang/test/Lower/OpenMP/parallel-firstprivate-clause-scalar.f90
@@ -1,8 +1,8 @@
 ! This test checks lowering of `FIRSTPRIVATE` clause for scalar types.
 
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! REQUIRES: shell
-! RUN: bbc -fopenmp -emit-hlfir %s -o - \
+! RUN: bbc -target x86_64-unknown-linux-gnu -fopenmp -emit-hlfir %s -o - \
 ! RUN: | FileCheck %s --check-prefixes=CHECK%if target=x86_64{{.*}} %{,CHECK-KIND10%}%if flang-supports-f128-math %{,CHECK-KIND16%}
 
 !CHECK:  omp.private {type = firstprivate} @[[ARG2_LOGICAL_PRIVATIZER:_QFfirstprivate_logicalEarg2_firstprivate_l8]] : !fir.logical<1>
@@ -29,7 +29,7 @@
 !CHECK:   omp.parallel private(@[[ARG1_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG1_PVT:.*]], @[[ARG2_COMPLEX_PRIVATIZER]] %{{.*}}#0 -> %[[ARG2_PVT:.*]] : {{.*}}) {
 !CHECK:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_complexEarg1"} : (!fir.ref<complex<f32>>) -> (!fir.ref<complex<f32>>, !fir.ref<complex<f32>>)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_complexEarg2"} : (!fir.ref<complex<f64>>) -> (!fir.ref<complex<f64>>, !fir.ref<complex<f64>>)
-!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
+!CHECK:     fir.call @_QPfoo(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0) {{.*}}: (!fir.ref<complex<f32>>, !fir.ref<complex<f64>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -57,8 +57,8 @@ subroutine firstprivate_complex(arg1, arg2)
 !CHECK:    %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_integerEarg4"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:    %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_integerEarg5"} : (!fir.ref<i64>) -> (!fir.ref<i64>, !fir.ref<i64>)
 !CHECK:    %[[ARG6_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG6_PVT]] {uniq_name = "_QFfirstprivate_integerEarg6"} : (!fir.ref<i128>) -> (!fir.ref<i128>, !fir.ref<i128>)
-!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1,
-!%[[ARG5_PVT_DECL]]#1, %[[ARG6_PVT_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
+!CHECK:    fir.call @_QPbar(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0,
+!%[[ARG5_PVT_DECL]]#0, %[[ARG6_PVT_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i8>, !fir.ref<i16>, !fir.ref<i32>, !fir.ref<i64>, !fir.ref<i128>) -> ()
 !CHECK:    omp.terminator
 !CHECK:  }
 
@@ -88,7 +88,7 @@ subroutine firstprivate_integer(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg3"} : (!fir.ref<!fir.logical<2>>) -> (!fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<2>>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg4"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
 !CHECK:     %[[ARG5_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG5_PVT]] {uniq_name = "_QFfirstprivate_logicalEarg5"} : (!fir.ref<!fir.logical<8>>) -> (!fir.ref<!fir.logical<8>>, !fir.ref<!fir.logical<8>>)
-!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1, %[[ARG5_PVT_DECL]]#1) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
+!CHECK:     fir.call @_QPbaz(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0, %[[ARG5_PVT_DECL]]#0) {{.*}}: (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<1>>, !fir.ref<!fir.logical<2>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<8>>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 
@@ -116,7 +116,7 @@ subroutine firstprivate_logical(arg1, arg2, arg3, arg4, arg5)
 !CHECK:     %[[ARG2_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG2_PVT]] {uniq_name = "_QFfirstprivate_realEarg2"} : (!fir.ref<f16>) -> (!fir.ref<f16>, !fir.ref<f16>)
 !CHECK:     %[[ARG3_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG3_PVT]] {uniq_name = "_QFfirstprivate_realEarg3"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
 !CHECK:     %[[ARG4_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG4_PVT]] {uniq_name = "_QFfirstprivate_realEarg4"} : (!fir.ref<f64>) -> (!fir.ref<f64>, !fir.ref<f64>)
-!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#1, %[[ARG2_PVT_DECL]]#1, %[[ARG3_PVT_DECL]]#1, %[[ARG4_PVT_DECL]]#1) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
+!CHECK:     fir.call @_QPqux(%[[ARG1_PVT_DECL]]#0, %[[ARG2_PVT_DECL]]#0, %[[ARG3_PVT_DECL]]#0, %[[ARG4_PVT_DECL]]#0) {{.*}}: (!fir.ref<f32>, !fir.ref<f16>, !fir.ref<f32>, !fir.ref<f64>) -> ()
 !CHECK:     omp.terminator
 !CHECK:   }
 subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
@@ -136,7 +136,7 @@ subroutine firstprivate_real(arg1, arg2, arg3, arg4, arg5, arg6)
 !CHECK-KIND10:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>, !fir.dscope) -> (!fir.ref<f80>, !fir.ref<f80>)
 !CHECK-KIND10:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND10:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real10Earg1"} : (!fir.ref<f80>) -> (!fir.ref<f80>, !fir.ref<f80>)
-!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f80>) -> ()
+!CHECK-KIND10:     fir.call @_QPqux10(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f80>) -> ()
 !CHECK-KIND10:     omp.terminator
 !CHECK-KIND10:   }
 subroutine firstprivate_real10(arg1)
@@ -152,7 +152,7 @@ subroutine firstprivate_real10(arg1)
 !CHECK-KIND16:   %[[ARG1_DECL:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %{{[0-9]+}} {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>, !fir.dscope) -> (!fir.ref<f128>, !fir.ref<f128>)
 !CHECK-KIND16:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[ARG1_PVT:.*]] : {{.*}}) {
 !CHECK-KIND16:     %[[ARG1_PVT_DECL:.*]]:2 = hlfir.declare %[[ARG1_PVT]] {uniq_name = "_QFfirstprivate_real16Earg1"} : (!fir.ref<f128>) -> (!fir.ref<f128>, !fir.ref<f128>)
-!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#1) {{.*}} : (!fir.ref<f128>) -> ()
+!CHECK-KIND16:     fir.call @_QPqux16(%[[ARG1_PVT_DECL]]#0) {{.*}} : (!fir.ref<f128>) -> ()
 !CHECK-KIND16:     omp.terminator
 !CHECK-KIND16:   }
 subroutine firstprivate_real16(arg1)
@@ -171,7 +171,7 @@ subroutine firstprivate_real16(arg1)
 !CHECK:  omp.parallel private({{.*firstprivate.*}} {{.*}}#0 -> %[[A_PRIV_ADDR:.*]], {{.*firstprivate.*}} {{.*}}#0 -> %[[B_PRIV_ADDR:.*]] : {{.*}}) {
 !CHECK:             %[[A_PRIV_DECL:.*]]:2 = hlfir.declare %[[A_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 !CHECK:             %[[B_PRIV_DECL:.*]]:2 = hlfir.declare %[[B_PRIV_ADDR]] {uniq_name = "_QFmultiple_firstprivateEb"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#1, %[[B_PRIV_DECL]]#1) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
+!CHECK:             fir.call @_QPquux(%[[A_PRIV_DECL]]#0, %[[B_PRIV_DECL]]#0) {{.*}}: (!fir.ref<i32>, !fir.ref<i32>) -> ()
 !CHECK:             omp.terminator
 !CHECK:           }
 !CHECK:           return
diff --git a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90 b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
index 1aa785faa172..01261a17992d 100644
--- a/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
+++ b/flang/test/Lower/OpenMP/reduction-array-intrinsic.f90
@@ -83,10 +83,10 @@ subroutine max_array_reduction(l, r)
 ! CHECK:               %[[VAL_16:.*]] = arith.constant 1 : index
 ! CHECK:               %[[VAL_17:.*]] = arith.subi %[[VAL_15]]#0, %[[VAL_16]] : index
 ! CHECK:               %[[VAL_18:.*]] = arith.addi %[[VAL_13]], %[[VAL_17]] : index
-! CHECK:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_18]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-! CHECK:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<i32>
-! CHECK:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
-! CHECK:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_19:.*]] = hlfir.designate %[[VAL_8]] (%[[VAL_18]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_20:.*]] = fir.load %[[VAL_19]] : !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_21:.*]] = hlfir.designate %[[VAL_4]]#0 (%[[VAL_13]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_22:.*]] = fir.load %[[VAL_21]] : !fir.ref<i32>
 ! CHECK:               %[[VAL_23:.*]] = arith.cmpi sgt, %[[VAL_20]], %[[VAL_22]] : i32
 ! CHECK:               %[[VAL_24:.*]] = arith.select %[[VAL_23]], %[[VAL_20]], %[[VAL_22]] : i32
 ! CHECK:               hlfir.yield_element %[[VAL_24]] : i32
diff --git a/flang/test/Lower/OpenMP/target-data-if-false.f90 b/flang/test/Lower/OpenMP/target-data-if-false.f90
new file mode 100644
index 000000000000..ac0f675ddc80
--- /dev/null
+++ b/flang/test/Lower/OpenMP/target-data-if-false.f90
@@ -0,0 +1,25 @@
+!RUN: %flang_fc1 -emit-llvm -fopenmp %openmp_flags -fopenmp-version=52 %s -o - | FileCheck %s
+
+!Check that this doesn't crash.
+
+!CHECK-LABEL: define void @f00_()
+!CHECK: call i1 @_FortranAioOutputDerivedType
+
+subroutine f00
+  use iso_c_binding
+  type(c_ptr) :: x
+
+!$omp target data use_device_ptr(x) if(.false.)
+  print *, x
+!$omp end target data
+end
+
+!CHECK-LABEL: define void @f01_()
+!CHECK: call i1 @_FortranAioOutputInteger32
+subroutine f01
+  integer :: x
+
+!$omp target data use_device_addr(x) if(.false.)
+  print *, x
+!$omp end target data
+end
diff --git a/flang/test/Lower/OpenMP/target-data-skip-mapper-calls.f90 b/flang/test/Lower/OpenMP/target-data-skip-mapper-calls.f90
new file mode 100644
index 000000000000..f1a150d5dfab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/target-data-skip-mapper-calls.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=NORT
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=LLVM
+
+!Make sure that there are no calls to the mapper.
+!NORT-NOT: call{{.*}}__tgt_target_data_begin_mapper
+!NORT-NOT: call{{.*}}__tgt_target_data_end_mapper
+
+!Make sure we generate the body
+!LLVM: define internal void @_QFPf(ptr %[[A0:[0-9]+]], ptr %[[A1:[0-9]+]])
+!LLVM:   %[[V0:[0-9]+]] = load i32, ptr %[[A0]], align 4
+!LLVM:   %[[V1:[0-9]+]] = load i32, ptr %[[A1]], align 4
+!LLVM:   %[[V2:[0-9]+]] = add i32 %[[V0]], %[[V1]]
+!LLVM:   store i32 %[[V2]], ptr %[[A0]], align 4
+!LLVM:   ret void
+!LLVM: }
+
+
+program test
+
+call f(1, 2)
+
+contains
+
+subroutine f(x, y)
+  integer :: x, y
+  !$omp target data map(tofrom: x, y)
+  x = x + y
+  !$omp end target data
+end subroutine
+end
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
index 5b4c5e65ffcc..58b68e5ec4cf 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-allocatable-array-minmax.f90
@@ -228,11 +228,11 @@ program reduce15
 ! CHECK:                 %[[VAL_56:.*]]:2 = hlfir.declare %[[VAL_55]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_62:.*]]:2 = hlfir.declare %[[VAL_60]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEmaxes"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 hlfir.assign %[[VAL_61]] to %[[VAL_56]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_63:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:                 %[[VAL_64:.*]] = arith.constant 0 : index
-! CHECK:                 %[[VAL_65:.*]]:3 = fir.box_dims %[[VAL_63]], %[[VAL_64]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-! CHECK:                 %[[VAL_66:.*]] = fir.shape %[[VAL_65]]#1 : (index) -> !fir.shape<1>
-! CHECK:                 %[[VAL_67:.*]] = fir.load %[[VAL_62]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK-DAG:             %[[VAL_63:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK-DAG:             %[[VAL_64:.*]] = arith.constant 0 : index
+! CHECK-DAG:             %[[VAL_65:.*]]:3 = fir.box_dims %[[VAL_63]], %[[VAL_64]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK-DAG:             %[[VAL_66:.*]] = fir.shape %[[VAL_65]]#1 : (index) -> !fir.shape<1>
+! CHECK-DAG:             %[[VAL_67:.*]] = fir.load %[[VAL_62]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
 ! CHECK:                 %[[VAL_68:.*]] = hlfir.elemental %[[VAL_66]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
 ! CHECK:                 ^bb0(%[[VAL_69:.*]]: index):
 ! CHECK:                   %[[VAL_70:.*]] = arith.constant 0 : index
@@ -240,15 +240,15 @@ program reduce15
 ! CHECK:                   %[[VAL_72:.*]] = arith.constant 1 : index
 ! CHECK:                   %[[VAL_73:.*]] = arith.subi %[[VAL_71]]#0, %[[VAL_72]] : index
 ! CHECK:                   %[[VAL_74:.*]] = arith.addi %[[VAL_69]], %[[VAL_73]] : index
-! CHECK:                   %[[VAL_75:.*]] = hlfir.designate %[[VAL_63]] (%[[VAL_74]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-! CHECK:                   %[[VAL_76:.*]] = fir.load %[[VAL_75]] : !fir.ref<i32>
-! CHECK:                   %[[VAL_77:.*]] = arith.constant 0 : index
-! CHECK:                   %[[VAL_78:.*]]:3 = fir.box_dims %[[VAL_67]], %[[VAL_77]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-! CHECK:                   %[[VAL_79:.*]] = arith.constant 1 : index
-! CHECK:                   %[[VAL_80:.*]] = arith.subi %[[VAL_78]]#0, %[[VAL_79]] : index
-! CHECK:                   %[[VAL_81:.*]] = arith.addi %[[VAL_69]], %[[VAL_80]] : index
-! CHECK:                   %[[VAL_82:.*]] = hlfir.designate %[[VAL_67]] (%[[VAL_81]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-! CHECK:                   %[[VAL_83:.*]] = fir.load %[[VAL_82]] : !fir.ref<i32>
+! CHECK-DAG:                   %[[VAL_75:.*]] = hlfir.designate %[[VAL_63]] (%[[VAL_74]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+! CHECK-DAG:                   %[[VAL_76:.*]] = fir.load %[[VAL_75]] : !fir.ref<i32>
+! CHECK-DAG:                   %[[VAL_77:.*]] = arith.constant 0 : index
+! CHECK-DAG:                   %[[VAL_78:.*]]:3 = fir.box_dims %[[VAL_67]], %[[VAL_77]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK-DAG:                   %[[VAL_79:.*]] = arith.constant 1 : index
+! CHECK-DAG:                   %[[VAL_80:.*]] = arith.subi %[[VAL_78]]#0, %[[VAL_79]] : index
+! CHECK-DAG:                   %[[VAL_81:.*]] = arith.addi %[[VAL_69]], %[[VAL_80]] : index
+! CHECK-DAG:                   %[[VAL_82:.*]] = hlfir.designate %[[VAL_67]] (%[[VAL_81]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+! CHECK-DAG:                   %[[VAL_83:.*]] = fir.load %[[VAL_82]] : !fir.ref<i32>
 ! CHECK:                   %[[VAL_84:.*]] = arith.cmpi sgt, %[[VAL_76]], %[[VAL_83]] : i32
 ! CHECK:                   %[[VAL_85:.*]] = arith.select %[[VAL_84]], %[[VAL_76]], %[[VAL_83]] : i32
 ! CHECK:                   hlfir.yield_element %[[VAL_85]] : i32
@@ -269,27 +269,27 @@ program reduce15
 ! CHECK:                 %[[VAL_88:.*]]:2 = hlfir.declare %[[VAL_87]] {uniq_name = "_QFEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_94:.*]]:2 = hlfir.declare %[[VAL_92]] {fortran_attrs = {{.*}}<allocatable>, uniq_name = "_QFEmins"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
 ! CHECK:                 hlfir.assign %[[VAL_93]] to %[[VAL_88]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_95:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:                 %[[VAL_96:.*]] = arith.constant 0 : index
-! CHECK:                 %[[VAL_97:.*]]:3 = fir.box_dims %[[VAL_95]], %[[VAL_96]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-! CHECK:                 %[[VAL_98:.*]] = fir.shape %[[VAL_97]]#1 : (index) -> !fir.shape<1>
-! CHECK:                 %[[VAL_99:.*]] = fir.load %[[VAL_94]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK:                 %[[VAL_100:.*]] = hlfir.elemental %[[VAL_98]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+! CHECK-DAG:                 %[[VAL_95:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK-DAG:                 %[[VAL_96:.*]] = arith.constant 0 : index
+! CHECK-DAG:                 %[[VAL_97:.*]]:3 = fir.box_dims %[[VAL_95]], %[[VAL_96]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK-DAG:                 %[[VAL_98:.*]] = fir.shape %[[VAL_97]]#1 : (index) -> !fir.shape<1>
+! CHECK-DAG:                 %[[VAL_99:.*]] = fir.load %[[VAL_94]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK-DAG:                 %[[VAL_100:.*]] = hlfir.elemental %[[VAL_98]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
 ! CHECK:                 ^bb0(%[[VAL_101:.*]]: index):
 ! CHECK:                   %[[VAL_102:.*]] = arith.constant 0 : index
 ! CHECK:                   %[[VAL_103:.*]]:3 = fir.box_dims %[[VAL_95]], %[[VAL_102]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
 ! CHECK:                   %[[VAL_104:.*]] = arith.constant 1 : index
 ! CHECK:                   %[[VAL_105:.*]] = arith.subi %[[VAL_103]]#0, %[[VAL_104]] : index
 ! CHECK:                   %[[VAL_106:.*]] = arith.addi %[[VAL_101]], %[[VAL_105]] : index
-! CHECK:                   %[[VAL_107:.*]] = hlfir.designate %[[VAL_95]] (%[[VAL_106]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-! CHECK:                   %[[VAL_108:.*]] = fir.load %[[VAL_107]] : !fir.ref<i32>
-! CHECK:                   %[[VAL_109:.*]] = arith.constant 0 : index
-! CHECK:                   %[[VAL_110:.*]]:3 = fir.box_dims %[[VAL_99]], %[[VAL_109]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
-! CHECK:                   %[[VAL_111:.*]] = arith.constant 1 : index
-! CHECK:                   %[[VAL_112:.*]] = arith.subi %[[VAL_110]]#0, %[[VAL_111]] : index
-! CHECK:                   %[[VAL_113:.*]] = arith.addi %[[VAL_101]], %[[VAL_112]] : index
-! CHECK:                   %[[VAL_114:.*]] = hlfir.designate %[[VAL_99]] (%[[VAL_113]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
-! CHECK:                   %[[VAL_115:.*]] = fir.load %[[VAL_114]] : !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_107:.*]] = hlfir.designate %[[VAL_95]] (%[[VAL_106]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_108:.*]] = fir.load %[[VAL_107]] : !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_109:.*]] = arith.constant 0 : index
+! CHECK-DAG:               %[[VAL_110:.*]]:3 = fir.box_dims %[[VAL_99]], %[[VAL_109]] : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> (index, index, index)
+! CHECK-DAG:               %[[VAL_111:.*]] = arith.constant 1 : index
+! CHECK-DAG:               %[[VAL_112:.*]] = arith.subi %[[VAL_110]]#0, %[[VAL_111]] : index
+! CHECK-DAG:               %[[VAL_113:.*]] = arith.addi %[[VAL_101]], %[[VAL_112]] : index
+! CHECK-DAG:               %[[VAL_114:.*]] = hlfir.designate %[[VAL_99]] (%[[VAL_113]])  : (!fir.box<!fir.heap<!fir.array<?xi32>>>, index) -> !fir.ref<i32>
+! CHECK-DAG:               %[[VAL_115:.*]] = fir.load %[[VAL_114]] : !fir.ref<i32>
 ! CHECK:                   %[[VAL_116:.*]] = arith.cmpi slt, %[[VAL_108]], %[[VAL_115]] : i32
 ! CHECK:                   %[[VAL_117:.*]] = arith.select %[[VAL_116]], %[[VAL_108]], %[[VAL_115]] : i32
 ! CHECK:                   hlfir.yield_element %[[VAL_117]] : i32
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
index 18e252a787d7..cf5c1b127c34 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -147,7 +147,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @and_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @and_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @and_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
index 7f3e3922ab29..eff97aa5d0db 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-and.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @and_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @and_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ end subroutine simple_reduction
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -139,7 +139,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@and_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @and_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @and_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
index e7654edffec8..bbe4bf04b682 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -89,7 +89,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -146,7 +146,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @eqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @eqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @eqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
index dffd6545b95b..304c7eefd58b 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-eqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @eqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @eqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant true
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@eqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @eqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @eqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
new file mode 100644
index 000000000000..f7fd2a52060e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-kinds.f90
@@ -0,0 +1,24 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s | FileCheck %s
+
+! Check that logical reductions of different kinds do not end up using the same
+! reduction declaration
+
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l64 : !fir.logical<8> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
+
+subroutine test(a4, a8, sz)
+  integer :: sz
+  logical(4), dimension(sz) :: a4
+  logical(8), dimension(sz) :: a8
+
+  logical(4) :: res4 = .false.
+  logical(8) :: res8 = .false.
+  integer i
+
+! CHECK: omp.wsloop private({{.*}}) reduction(@or_reduction_l32 {{.*}}, @or_reduction_l64 {{.*}}) {
+  !$omp do reduction(.or.:res4, res8)
+  do i = 1,sz
+    res4 = res4 .or. a4(i)
+    res8 = res8 .or. a8(i)
+  enddo
+end subroutine
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
index 366dd67591cf..8c869952547c 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -42,7 +42,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -90,7 +90,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -149,7 +149,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @neqv_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @neqv_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @neqv_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
index 66f624ff493f..e0901bec6bc9 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-neqv.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @neqv_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @neqv_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -82,7 +82,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -141,7 +141,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@neqv_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @neqv_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @neqv_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
index a867d0a550bc..bb09e2563e37 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or-byref.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.ref<!fir.logical<4>>
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_byref_l32 : !fir.ref<!fir.logical<4>>
 ! CHECK-SAME:    alloc {
 ! CHECK:           %[[REF:.*]] = fir.alloca !fir.logical<4>
 ! CHECK:           omp.yield(%[[REF]] : !fir.ref<!fir.logical<4>>)
@@ -41,7 +41,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -88,7 +88,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -145,7 +145,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(byref @or_reduction_byref_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], byref @or_reduction_byref_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], byref @or_reduction_byref_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
index 592d4fa516bd..5c9bcbf584d4 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-logical-or.f90
@@ -3,7 +3,7 @@
 
 ! NOTE: Assertions have been autogenerated by utils/generate-test-checks.py
 
-! CHECK-LABEL:   omp.declare_reduction @or_reduction : !fir.logical<4> init {
+! CHECK-LABEL:   omp.declare_reduction @or_reduction_l32 : !fir.logical<4> init {
 ! CHECK:         ^bb0(%[[VAL_0:.*]]: !fir.logical<4>):
 ! CHECK:           %[[VAL_1:.*]] = arith.constant false
 ! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]] : (i1) -> !fir.logical<4>
@@ -34,7 +34,7 @@
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reductionEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reductionEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -81,7 +81,7 @@ subroutine simple_reduction(y)
 ! CHECK:             %[[VAL_12:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_13:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_14:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_10:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_4]]#0 -> %[[VAL_15:.*]] : !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_16:.*]]) : i32 = (%[[VAL_12]]) to (%[[VAL_13]]) inclusive step (%[[VAL_14]]) {
 ! CHECK:                 %[[VAL_11:.*]]:2 = hlfir.declare %[[VAL_10]] {uniq_name = "_QFsimple_reduction_switch_orderEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_15]] {uniq_name = "_QFsimple_reduction_switch_orderEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
@@ -138,7 +138,7 @@ subroutine simple_reduction_switch_order(y)
 ! CHECK:             %[[VAL_20:.*]] = arith.constant 1 : i32
 ! CHECK:             %[[VAL_21:.*]] = arith.constant 100 : i32
 ! CHECK:             %[[VAL_22:.*]] = arith.constant 1 : i32
-! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
+! CHECK:             omp.wsloop private(@{{.*}} %{{.*}}#0 -> %[[VAL_18:.*]] : !fir.ref<i32>) reduction(@or_reduction_l32 %[[VAL_7]]#0 -> %[[VAL_23:.*]], @or_reduction_l32 %[[VAL_9]]#0 -> %[[VAL_24:.*]], @or_reduction_l32 %[[VAL_11]]#0 -> %[[VAL_25:.*]] : !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>) {
 ! CHECK-NEXT:          omp.loop_nest (%[[VAL_26:.*]]) : i32 = (%[[VAL_20]]) to (%[[VAL_21]]) inclusive step (%[[VAL_22]]) {
 ! CHECK:                 %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_18]] {uniq_name = "_QFmultiple_reductionsEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_23]] {uniq_name = "_QFmultiple_reductionsEx"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
index d27804fb5606..69219331ab3a 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max-byref.f90
@@ -53,11 +53,11 @@
 ! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 hlfir.assign %[[VAL_13]] to %[[VAL_8]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
-! CHECK:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
-! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+! CHECK-DAG:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
 ! CHECK:                 %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
index 5eddd8655d9a..83582d279fd3 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-max.f90
@@ -43,11 +43,11 @@
 ! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_max_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_max_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 hlfir.assign %[[VAL_13]] to %[[VAL_8]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
-! CHECK:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
-! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+! CHECK-DAG:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
 ! CHECK:                 %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
index e1b5c3d6eaae..f691d57e276d 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min-byref.f90
@@ -53,11 +53,11 @@
 ! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 hlfir.assign %[[VAL_13]] to %[[VAL_8]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
-! CHECK:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
-! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+! CHECK-DAG:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
 ! CHECK:                 %[[VAL_20:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90 b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
index 326447d6fd5c..3ee2ecc50e19 100644
--- a/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
+++ b/flang/test/Lower/OpenMP/wsloop-reduction-min.f90
@@ -43,11 +43,11 @@
 ! CHECK:                 %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] {uniq_name = "_QFreduction_min_intEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_12]] {uniq_name = "_QFreduction_min_intEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
 ! CHECK:                 hlfir.assign %[[VAL_13]] to %[[VAL_8]]#0 : i32, !fir.ref<i32>
-! CHECK:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
-! CHECK:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
-! CHECK:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
-! CHECK:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+! CHECK-DAG:                 %[[VAL_17:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_16]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_18:.*]] = fir.load %[[VAL_14]]#0 : !fir.ref<i32>
+! CHECK-DAG:                 %[[VAL_19:.*]] = fir.load %[[VAL_17]] : !fir.ref<i32>
 ! CHECK:                 %[[VAL_20:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_18]], %[[VAL_19]] : i32
 ! CHECK:                 hlfir.assign %[[VAL_21]] to %[[VAL_14]]#0 : i32, !fir.ref<i32>
diff --git a/flang/test/Lower/complex-operations.f90 b/flang/test/Lower/complex-operations.f90
index fdd13719c5b7..38954626e77e 100644
--- a/flang/test/Lower/complex-operations.f90
+++ b/flang/test/Lower/complex-operations.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: flang-supports-f128-math
-! REQUIRES: x86_64-registered-target
-! RUN: bbc -hlfir=false %s -o - | FileCheck %s
+! REQUIRES: x86-registered-target
+! RUN: bbc -target x86_64-unknown-linux-gnu -hlfir=false %s -o - | FileCheck %s
 
 ! CHECK-LABEL: @_QPadd_test
 subroutine add_test(a,b,c)
diff --git a/flang/test/Lower/math-lowering/aint.f90 b/flang/test/Lower/math-lowering/aint.f90
index 6c7809f8ea1a..b36fb421d7c8 100644
--- a/flang/test/Lower/math-lowering/aint.f90
+++ b/flang/test/Lower/math-lowering/aint.f90
@@ -1,10 +1,10 @@
-! REQUIRES: x86_64-registered-target
+! REQUIRES: x86-registered-target
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=fast | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=fast %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=relaxed | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=relaxed %s -o - | FileCheck --check-prefixes=ALL %s
 ! RUN: bbc -target x86_64-unknown-linux-gnu -emit-fir %s -o - --math-runtime=precise | FileCheck --check-prefixes=ALL %s
-! RUN: %flang_fc1 -target x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
+! RUN: %flang_fc1 -triple x86_64-unknown-linux-gnu -emit-fir -mllvm -math-runtime=precise %s -o - | FileCheck --check-prefixes=ALL %s
 
 function test_real4(x)
   real :: x, test_real4
diff --git a/flang/test/Lower/select-type-2.f90 b/flang/test/Lower/select-type-2.f90
index 7b4cf96069e5..59485c176c76 100644
--- a/flang/test/Lower/select-type-2.f90
+++ b/flang/test/Lower/select-type-2.f90
@@ -30,8 +30,8 @@ subroutine select_type1(a)
 
 ! CHECK-LABEL: func.func @_QMselect_type_2Pselect_type1(
 ! CHECK-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CHECK:      %[[TDESC_P3_ADDR:.*]] = fir.address_of(@_QMselect_type_2E.dt.p3) : !fir.ref<!fir.type<{{.*}}>>
-! CHECK:      %[[TDESC_P3_CONV:.*]] = fir.convert %[[TDESC_P3_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CHECK:      %[[TDESC_P3_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_2Tp3
+! CHECK:      %[[TDESC_P3_CONV:.*]] = fir.convert %[[TDESC_P3_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CHECK:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CHECK:      %[[CLASS_IS_CMP:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P3_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CHECK:      cf.cond_br %[[CLASS_IS_CMP]], ^[[CLASS_IS_P3_BLK:.*]], ^[[NOT_CLASS_IS_P3_BLK:.*]]
@@ -40,8 +40,8 @@ subroutine select_type1(a)
 ! CHECK:    ^bb[[CLASS_IS_P1:[0-9]]]:
 ! CHECK:      cf.br ^bb[[END_SELECT_BLK:[0-9]]]
 ! CHECK:    ^[[NOT_CLASS_IS_P3_BLK]]:
-! CHECK:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_2E.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CHECK:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CHECK:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_2Tp1
+! CHECK:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CHECK:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_2Tp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CHECK:      %[[CLASS_IS_CMP:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CHECK:      cf.cond_br %[[CLASS_IS_CMP]], ^bb[[CLASS_IS_P1]], ^bb[[NOT_CLASS_IS_P1]]
diff --git a/flang/test/Lower/select-type.f90 b/flang/test/Lower/select-type.f90
index 64dd639731ab..e2ca87ad447c 100644
--- a/flang/test/Lower/select-type.f90
+++ b/flang/test/Lower/select-type.f90
@@ -68,15 +68,15 @@ subroutine select_type1(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type1(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:    ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p2) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp2{{.*}}
+! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.tdesc<{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P2_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^bb[[CLASS_IS_P2_BLK:.*]], ^[[NOT_CLASS_IS_P2_BLK:.*]]
@@ -87,8 +87,8 @@ subroutine select_type1(a)
 ! CFG:    ^bb[[CLASS_IS_P1_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[END_SELECT_BLK:.*]]
 ! CFG:    ^[[NOT_CLASS_IS_P2_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc<{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^bb[[CLASS_IS_P1_BLK]], ^bb[[NOT_CLASS_IS_P1_BLK]]
@@ -126,15 +126,15 @@ subroutine select_type2()
 ! CFG:     %[[GET_CLASS:.*]] = fir.call @_QMselect_type_lower_testPget_class() {{.*}} : () -> !fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>
 ! CFG:     fir.save_result %[[GET_CLASS]] to %[[CLASS_ALLOCA]] : !fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>, !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>
 ! CFG:     %[[LOAD_CLASS:.*]] = fir.load %[[CLASS_ALLOCA]] : !fir.ref<!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>
-! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:     %[[CLASS_TDESC:.*]] = fir.box_tdesc %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.tdesc<none>
-! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:     %[[BOX_TDESC_CONV:.*]] = fir.convert %[[CLASS_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:     %[[CLASS_TDESC:.*]] = fir.box_tdesc %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.tdesc<{{.*}}>
+! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:     %[[BOX_TDESC_CONV:.*]] = fir.convert %[[CLASS_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:     %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:     cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:   ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:     %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:     %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:     %[[BOX_NONE:.*]] = fir.convert %[[LOAD_CLASS]] : (!fir.class<!fir.ptr<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>) -> !fir.box<none>
 ! CFG:     %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:     cf.cond_br %[[CLASS_IS]], ^[[CLASS_IS_BLK:.*]], ^[[NOT_CLASS_IS_BLK:.*]]
@@ -176,15 +176,15 @@ subroutine select_type3(a)
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type3(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.ptr<!fir.array<?x!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>>>> {fir.bindc_name = "a"}) {
 ! CFG:      %[[SELECTOR:.*]] = fir.embox %{{.*}} source_box %{{.*}} : (!fir.ref<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>, !fir.class<{{.*}}>) -> !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[SELECTOR_TDESC:.*]] = fir.box_tdesc %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[TDESC_CONV:.*]] = fir.convert %[[SELECTOR_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[SELECTOR_TDESC:.*]] = fir.box_tdesc %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[TDESC_CONV:.*]] = fir.convert %[[SELECTOR_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P1_CONV]], %[[TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[TYPE_IS_P1_BLK:.*]], ^[[NOT_TYPE_IS_P1_BLK:.*]]
 ! CFG:    ^[[NOT_TYPE_IS_P1_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[SELECTOR]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^[[CLASS_IS_BLK:.*]], ^[[NOT_CLASS_IS:.*]]
@@ -222,25 +222,25 @@ subroutine select_type4(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type4(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P3_8_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p3.8) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P3_8_CONV:.*]] = fir.convert %[[TDESC_P3_8_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P3_8_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp3K8
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P3_8_CONV:.*]] = fir.convert %[[TDESC_P3_8_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P3_8_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[P3_8_BLK:.*]], ^[[NOT_P3_8_BLK:.*]]
 ! CFG:    ^[[NOT_P3_8_BLK]]:
-! CFG:      %[[TDESC_P3_4_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p3.4) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<none>
-! CFG:      %[[TDESC_P3_4_CONV:.*]] = fir.convert %[[TDESC_P3_4_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> index
-! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<none>) -> index
+! CFG:      %[[TDESC_P3_4_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp3K4
+! CFG:      %[[BOX_TDESC:.*]] = fir.box_tdesc %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.tdesc<{{.*}}>
+! CFG:      %[[TDESC_P3_4_CONV:.*]] = fir.convert %[[TDESC_P3_4_ADDR]] : (!fir.tdesc{{.*}}>) -> index
+! CFG:      %[[BOX_TDESC_CONV:.*]] = fir.convert %[[BOX_TDESC]] : (!fir.tdesc<{{.*}}>) -> index
 ! CFG:      %[[TDESC_CMP:.*]] = arith.cmpi eq, %[[TDESC_P3_4_CONV]], %[[BOX_TDESC_CONV]] : index
 ! CFG:      cf.cond_br %[[TDESC_CMP]], ^[[P3_4_BLK:.*]], ^[[NOT_P3_4_BLK:.*]]
 ! CFG:    ^[[P3_8_BLK]]:
 ! CFG:      _FortranAioOutputAscii
 ! CFG:      cf.br ^bb[[EXIT_SELECT_BLK:[0-9]]]
 ! CFG:    ^[[NOT_P3_4_BLK]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS]], ^[[P1_BLK:.*]], ^[[NOT_P1_BLK:.*]]
@@ -409,8 +409,8 @@ subroutine select_type7(a)
 
 ! CFG-LABEL: func.func @_QMselect_type_lower_testPselect_type7(
 ! CFG-SAME: %[[ARG0:.*]]: !fir.class<none> {fir.bindc_name = "a"}) {
-! CFG:      %[[TDESC_P4_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p4) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P4_CONV:.*]] = fir.convert %[[TDESC_P4_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P4_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp4
+! CFG:      %[[TDESC_P4_CONV:.*]] = fir.convert %[[TDESC_P4_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P4:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P4_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P4]], ^[[CLASS_IS_P4_BLK:.*]], ^[[CLASS_NOT_P4_BLK:.*]]
@@ -419,16 +419,16 @@ subroutine select_type7(a)
 ! CFG:    ^bb[[CLASS_IS_P1_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[EXIT_SELECT_BLK:.*]]
 ! CFG:    ^bb[[CLASS_NOT_P2_BLK:[0-9]]]:
-! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p1) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P1_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp1{a:i32,b:i32}>
+! CFG:      %[[TDESC_P1_CONV:.*]] = fir.convert %[[TDESC_P1_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P1:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P1_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P1]], ^bb[[CLASS_IS_P1_BLK]], ^bb[[CLASS_NOT_P1_BLK]]
 ! CFG:    ^bb[[CLASS_IS_P2_BLK:[0-9]]]:
 ! CFG:      cf.br ^[[EXIT_SELECT_BLK]]
 ! CFG:    ^[[CLASS_NOT_P4_BLK]]:
-! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.address_of(@_QMselect_type_lower_testE.dt.p2) : !fir.ref<!fir.type<{{.*}}>>
-! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.ref<!fir.type<{{.*}}>>) -> !fir.ref<none>
+! CFG:      %[[TDESC_P2_ADDR:.*]] = fir.type_desc !fir.type<_QMselect_type_lower_testTp2
+! CFG:      %[[TDESC_P2_CONV:.*]] = fir.convert %[[TDESC_P2_ADDR]] : (!fir.tdesc{{.*}}>) -> !fir.ref<none>
 ! CFG:      %[[BOX_NONE:.*]] = fir.convert %[[ARG0]] : (!fir.class<none>) -> !fir.box<none>
 ! CFG:      %[[CLASS_IS_P2:.*]] = fir.call @_FortranAClassIs(%[[BOX_NONE]], %[[TDESC_P2_CONV]]) : (!fir.box<none>, !fir.ref<none>) -> i1
 ! CFG:      cf.cond_br %[[CLASS_IS_P2]], ^bb[[CLASS_IS_P2_BLK]], ^bb[[CLASS_NOT_P2_BLK]]
diff --git a/flang/test/Lower/variable-common-viewed-as-module-var.f90 b/flang/test/Lower/variable-common-viewed-as-module-var.f90
new file mode 100644
index 000000000000..e303df6d91a9
--- /dev/null
+++ b/flang/test/Lower/variable-common-viewed-as-module-var.f90
@@ -0,0 +1,37 @@
+! RUN: bbc -emit-hlfir %s -o - | FileCheck %s
+
+! Test non standard definition of a common block as a BIND(C) variable.
+! This happens when MPI and MPI_F08 are used inside the same compilation
+! unit because MPI uses common blocks while MPI_F08 uses BIND(C) variables
+! to refer to the same objects (e.g. mpi_argv_null).
+
+module m_common_var
+ character(1) :: var
+ common /var_storage/var
+end module
+
+module m_bindc_var
+  character(1), bind(c, name="var_storage_") :: var
+end module
+
+subroutine s1()
+  use m_common_var, only : var
+  var = "a"
+end subroutine
+
+subroutine s2()
+  use m_bindc_var, only : var
+  print *, var
+end subroutine
+
+  call s1()
+  call s2()
+end
+
+! CHECK: fir.global common @var_storage_(dense<0> : vector<1xi8>) {alignment = 1 : i64} : !fir.array<1xi8>
+
+! CHECK-LABEL: func.func @_QPs1
+! CHECK: hlfir.declare %{{.*}} typeparams %c1 {uniq_name = "_QMm_common_varEvar"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
+
+! CHECK-LABEL: func.func @_QPs2
+! CHECK: hlfir.declare %{{.*}} typeparams %c1 {fortran_attrs = #fir.var_attrs<bind_c>, uniq_name = "var_storage_"} : (!fir.ref<!fir.char<1>>, index) -> (!fir.ref<!fir.char<1>>, !fir.ref<!fir.char<1>>)
diff --git a/flang/test/Parser/OpenMP/declare-variant.f90 b/flang/test/Parser/OpenMP/declare-variant.f90
index 1b97733ea952..3366b143e62e 100644
--- a/flang/test/Parser/OpenMP/declare-variant.f90
+++ b/flang/test/Parser/OpenMP/declare-variant.f90
@@ -102,3 +102,30 @@ subroutine vsub(v1, v2)
     type(c_ptr), value :: v1, v2
   end
 end subroutine
+
+subroutine f
+  real :: x, y
+  y = 2
+  !omp simd
+  call f2(x, y)
+  !omp end simd 
+contains
+  subroutine f1 (x, y)
+    real :: x, y
+  end
+
+  subroutine f2 (x, y)
+    real :: x, y
+    !$omp declare variant (f1) match (construct={simd(uniform(y))})
+  end
+end subroutine
+!CHECK: !$OMP DECLARE VARIANT (f1) MATCH(CONSTRUCT={SIMD(UNIFORM(y))})
+!PARSE-TREE: | | | | DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OmpDeclareVariantDirective
+!PARSE-TREE-NEXT: | | | | | Verbatim
+!PARSE-TREE-NEXT: | | | | | Name = 'f1'
+!PARSE-TREE-NEXT: | | | | | OmpClauseList -> OmpClause -> Match -> OmpMatchClause -> OmpContextSelectorSpecification -> OmpTraitSetSelector
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSetSelectorName -> Value = Construct
+!PARSE-TREE-NEXT: | | | | | | OmpTraitSelector
+!PARSE-TREE-NEXT: | | | | | | | OmpTraitSelectorName -> Value = Simd
+!PARSE-TREE-NEXT: | | | | | | | Properties
+!PARSE-TREE-NEXT: | | | | | | | | OmpTraitProperty -> OmpClause -> Uniform -> Name = 'y'
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct01.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 000000000000..baffc2f6e2f1
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,74 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with 1 nested loop.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/OpenMP/loop-transformation-construct02.f90 b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
new file mode 100644
index 000000000000..b50e7183841c
--- /dev/null
+++ b/flang/test/Parser/OpenMP/loop-transformation-construct02.f90
@@ -0,0 +1,85 @@
+! Test the Parse Tree to ensure the OpenMP Loop Transformation Constructs nest correctly with multiple nested loops.
+
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-PARSE
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck %s --check-prefix=CHECK-UNPARSE
+
+subroutine loop_transformation_construct
+  implicit none
+  integer :: I = 10
+  integer :: x
+  integer :: y(I)
+
+  !$omp do
+  !$omp unroll
+  !$omp tile
+  do i = 1, I
+    y(i) = y(i) * 5
+  end do
+  !$omp end tile
+  !$omp end unroll
+  !$omp end do
+end subroutine
+
+!CHECK-PARSE: | ExecutionPart -> Block
+!CHECK-PARSE-NEXT: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OpenMPLoopConstruct
+!CHECK-PARSE-NEXT: | | | | | OmpBeginLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | | DoConstruct
+!CHECK-PARSE-NEXT: | | | | | | NonLabelDoStmt
+!CHECK-PARSE-NEXT: | | | | | | | LoopControl -> LoopBounds
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = '1_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | LiteralConstant -> IntLiteralConstant = '1'
+!CHECK-PARSE-NEXT: | | | | | | | | Scalar -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | Block
+!CHECK-PARSE-NEXT: | | | | | | | ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y(int(i,kind=8))=5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | Variable = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | Expr = '5_4*y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | Multiply
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = 'y(int(i,kind=8))'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | Designator -> DataRef -> ArrayElement
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | DataRef -> Name = 'y'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | SectionSubscript -> Integer -> Expr = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | | | Designator -> DataRef -> Name = 'i'
+!CHECK-PARSE-NEXT: | | | | | | | | | | Expr = '5_4'
+!CHECK-PARSE-NEXT: | | | | | | | | | | | LiteralConstant -> IntLiteralConstant = '5'
+!CHECK-PARSE-NEXT: | | | | | | EndDoStmt ->
+!CHECK-PARSE-NEXT: | | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | | OmpLoopDirective -> llvm::omp::Directive = tile
+!CHECK-PARSE-NEXT: | | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | | OmpLoopDirective -> llvm::omp::Directive = unroll
+!CHECK-PARSE-NEXT: | | | | | OmpClauseList ->
+!CHECK-PARSE-NEXT: | | | OmpEndLoopDirective
+!CHECK-PARSE-NEXT: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+!CHECK-PARSE-NEXT: | | | | OmpClauseList ->
+
+!CHECK-UNPARSE: SUBROUTINE loop_transformation_construct
+!CHECK-UNPARSE-NEXT:  IMPLICIT NONE
+!CHECK-UNPARSE-NEXT:  INTEGER :: i = 10_4
+!CHECK-UNPARSE-NEXT:  INTEGER x
+!CHECK-UNPARSE-NEXT:  INTEGER y(i)
+!CHECK-UNPARSE-NEXT: !$OMP DO
+!CHECK-UNPARSE-NEXT: !$OMP UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP TILE
+!CHECK-UNPARSE-NEXT:  DO i=1_4,i
+!CHECK-UNPARSE-NEXT:    y(int(i,kind=8))=5_4*y(int(i,kind=8))
+!CHECK-UNPARSE-NEXT:  END DO
+!CHECK-UNPARSE-NEXT: !$OMP END TILE
+!CHECK-UNPARSE-NEXT: !$OMP END UNROLL
+!CHECK-UNPARSE-NEXT: !$OMP END DO
+!CHECK-UNPARSE-NEXT: END SUBROUTINE
diff --git a/flang/test/Parser/issue-146362.1.f90 b/flang/test/Parser/issue-146362.1.f90
new file mode 100644
index 000000000000..a37fd4b182a7
--- /dev/null
+++ b/flang/test/Parser/issue-146362.1.f90
@@ -0,0 +1,5 @@
+! RUN: %flang_fc1 -fsyntax-only -cpp %s 2>&1
+#define UNITY(k) 1_ ## k
+PROGRAM REPRODUCER
+WRITE(*,*) UNITY(4)
+END PROGRAM REPRODUCER
\ No newline at end of file
diff --git a/flang/test/Parser/issue-146362.2.f90 b/flang/test/Parser/issue-146362.2.f90
new file mode 100644
index 000000000000..d9ad8c7d547f
--- /dev/null
+++ b/flang/test/Parser/issue-146362.2.f90
@@ -0,0 +1,18 @@
+!RUN: %flang_fc1 -cpp -fdebug-unparse %s | FileCheck %s
+PROGRAM P
+#if (1 && 2)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+#if ((1 || 2) != 3)
+  !CHECK: TRUE
+  WRITE(*,*) 'TRUE'
+#else
+  !CHECK-NOT: FALSE
+  WRITE(*,*) 'FALSE'
+#endif
+END PROGRAM
+
diff --git a/flang/test/Preprocessing/bug1077.F90 b/flang/test/Preprocessing/bug1077.F90
new file mode 100644
index 000000000000..dd7391813a35
--- /dev/null
+++ b/flang/test/Preprocessing/bug1077.F90
@@ -0,0 +1,7 @@
+!RUN: %flang -E %s 2>&1 | FileCheck %s
+!CHECK: print *,((1)+(2)),4
+#define foo(x,y) ((x)+(y))
+print *,&
+foo(1,2)&
+,4
+end
diff --git a/flang/test/Preprocessing/bug518.F b/flang/test/Preprocessing/bug518.F
index 0b680dd5751b..beb33564e423 100644
--- a/flang/test/Preprocessing/bug518.F
+++ b/flang/test/Preprocessing/bug518.F
@@ -1,5 +1,5 @@
 ! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
 ! CHECK: k=1_4
                         k=                                            1_99999999
-     &4
+     & 4
       end
diff --git a/flang/test/Preprocessing/pp111.F90 b/flang/test/Preprocessing/pp111.F90
index 4da45ef35f5c..bbf8709c3ab1 100644
--- a/flang/test/Preprocessing/pp111.F90
+++ b/flang/test/Preprocessing/pp111.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! FLM call name split across continuation, no leading &, with & ! comment
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp112.F90 b/flang/test/Preprocessing/pp112.F90
index 16705527f68c..a5244410f31a 100644
--- a/flang/test/Preprocessing/pp112.F90
+++ b/flang/test/Preprocessing/pp112.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = IFLM(666)
 ! ditto, but without & ! comment
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp115.F90 b/flang/test/Preprocessing/pp115.F90
index 4e4c621110ed..eea42c53b936 100644
--- a/flang/test/Preprocessing/pp115.F90
+++ b/flang/test/Preprocessing/pp115.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! ditto, with & ! comment, no leading &
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Preprocessing/pp116.F90 b/flang/test/Preprocessing/pp116.F90
index e35a13cbf648..39edf95763ea 100644
--- a/flang/test/Preprocessing/pp116.F90
+++ b/flang/test/Preprocessing/pp116.F90
@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: res = IFLM (666)
+! CHECK: res = ((666)+111)
 ! FLM call split between name and (, no leading &
       integer function IFLM(x)
         integer :: x
diff --git a/flang/test/Semantics/OpenACC/acc-branch.f90 b/flang/test/Semantics/OpenACC/acc-branch.f90
index b3692d016589..a2d7b583f679 100644
--- a/flang/test/Semantics/OpenACC/acc-branch.f90
+++ b/flang/test/Semantics/OpenACC/acc-branch.f90
@@ -74,7 +74,7 @@ subroutine openacc_clause_validity
   ! Exit branches out of parallel construct, attached to an OpenACC parallel construct.
   thisblk: BLOCK
     fortname: if (.true.) then
-      !PORTABILITY: The construct name 'name1' should be distinct at the subprogram level
+      !PORTABILITY: The construct name 'name1' should be distinct at the subprogram level [-Wbenign-name-clash]
       name1: do k = 1, N
         !$acc parallel
         !ERROR: EXIT to construct 'fortname' outside of PARALLEL construct is not allowed
diff --git a/flang/test/Semantics/OpenACC/acc-data.f90 b/flang/test/Semantics/OpenACC/acc-data.f90
index 473b91d555ce..98582f817489 100644
--- a/flang/test/Semantics/OpenACC/acc-data.f90
+++ b/flang/test/Semantics/OpenACC/acc-data.f90
@@ -132,7 +132,7 @@ program openacc_data_validity
   !ERROR: At least one of COPYOUT, DELETE, DETACH clause must appear on the EXIT DATA directive
   !$acc exit data
 
-  !PORTABILITY: At least one of ATTACH, COPY, COPYIN, COPYOUT, CREATE, DEFAULT, DEVICEPTR, NO_CREATE, PRESENT clause should appear on the DATA directive
+  !PORTABILITY: At least one of ATTACH, COPY, COPYIN, COPYOUT, CREATE, DEFAULT, DEVICEPTR, NO_CREATE, PRESENT clause should appear on the DATA directive [-Wportability]
   !$acc data
   !$acc end data
 
diff --git a/flang/test/Semantics/OpenACC/acc-declare-validity.f90 b/flang/test/Semantics/OpenACC/acc-declare-validity.f90
index 7cdc69436704..5ee61fac0660 100644
--- a/flang/test/Semantics/OpenACC/acc-declare-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-declare-validity.f90
@@ -14,7 +14,7 @@ module openacc_declare_validity
 
   !$acc declare create(aa, bb)
 
-  !WARNING: 'aa' in the CREATE clause is already present in the same clause in this module
+  !WARNING: 'aa' in the CREATE clause is already present in the same clause in this module [-Wopen-acc-usage]
   !$acc declare create(aa)
 
   !$acc declare link(ab)
diff --git a/flang/test/Semantics/OpenACC/acc-serial.f90 b/flang/test/Semantics/OpenACC/acc-serial.f90
index 1f22003ed6b1..f3b81c992e6e 100644
--- a/flang/test/Semantics/OpenACC/acc-serial.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial.f90
@@ -84,15 +84,15 @@ program openacc_serial_validity
   !$acc serial wait(wait1) wait(wait2)
   !$acc end serial
 
-  !PORTABILITY: NUM_GANGS clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: NUM_GANGS clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial num_gangs(8)
   !$acc end serial
 
-  !PORTABILITY: NUM_WORKERS clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: NUM_WORKERS clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial num_workers(8)
   !$acc end serial
 
-  !PORTABILITY: VECTOR_LENGTH clause is not allowed on the SERIAL directive and will be ignored
+  !PORTABILITY: VECTOR_LENGTH clause is not allowed on the SERIAL directive and will be ignored [-Wportability]
   !$acc serial vector_length(128)
   !$acc end serial
 
diff --git a/flang/test/Semantics/OpenMP/allocate-align01.f90 b/flang/test/Semantics/OpenMP/allocate-align01.f90
index bc17d7047bbb..508efa82f12a 100644
--- a/flang/test/Semantics/OpenMP/allocate-align01.f90
+++ b/flang/test/Semantics/OpenMP/allocate-align01.f90
@@ -13,7 +13,7 @@ program allocate_align_tree
     z = 3
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(j) align(xx)
-    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead. [-Wopen-mp-usage]
     !ERROR: The alignment value should be a constant positive integer
 !$omp allocate(xarray) align(-32) allocator(omp_large_cap_mem_alloc)
     allocate(j(z), xarray(t))
diff --git a/flang/test/Semantics/OpenMP/allocate01.f90 b/flang/test/Semantics/OpenMP/allocate01.f90
index b205b2c79d65..e0b084ff0030 100644
--- a/flang/test/Semantics/OpenMP/allocate01.f90
+++ b/flang/test/Semantics/OpenMP/allocate01.f90
@@ -19,7 +19,7 @@ subroutine sema()
     !$omp allocate(y)
         print *, a
 
-    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
+    !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead. [-Wopen-mp-usage]
     !ERROR: List items must be declared in the same scoping unit in which the ALLOCATE directive appears
     !$omp allocate(x) allocator(omp_default_mem_alloc)
       allocate ( x(a), darray(a, b) )
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 6989a183e83e..bc1371847b79 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -483,14 +483,14 @@
 ! 2.13.1 master
 
   !$omp parallel
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !$omp master
   a=3.14
   !$omp end master
   !$omp end parallel
 
   !$omp parallel
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !ERROR: NUM_THREADS clause is not allowed on the MASTER directive
   !$omp master num_threads(4)
   a=3.14
diff --git a/flang/test/Semantics/OpenMP/combined-constructs.f90 b/flang/test/Semantics/OpenMP/combined-constructs.f90
index 9fa1e7515dc0..ed7125537a1f 100644
--- a/flang/test/Semantics/OpenMP/combined-constructs.f90
+++ b/flang/test/Semantics/OpenMP/combined-constructs.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=11
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 
 program main
   implicit none
@@ -33,7 +33,6 @@ program main
   enddo
   !$omp end target parallel
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target parallel defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
@@ -80,7 +79,6 @@ program main
   enddo
   !$omp end target parallel do
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target parallel do defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
@@ -140,7 +138,6 @@ program main
   enddo
   !$omp end target teams
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target teams defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
@@ -240,7 +237,6 @@ program main
   enddo
   !$omp end target teams distribute
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
@@ -333,7 +329,6 @@ program main
   enddo
   !$omp end target teams distribute parallel do
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute parallel do defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
@@ -433,7 +428,6 @@ program main
   enddo
   !$omp end target teams distribute parallel do simd
 
-  !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute parallel do simd defaultmap(tofrom)
   do i = 1, N
      a(i) = 3.14
diff --git a/flang/test/Semantics/OpenMP/copying.f90 b/flang/test/Semantics/OpenMP/copying.f90
index 63fb39a0f26e..d79027361f2a 100644
--- a/flang/test/Semantics/OpenMP/copying.f90
+++ b/flang/test/Semantics/OpenMP/copying.f90
@@ -10,7 +10,7 @@
 subroutine firstprivate()
   class(*), allocatable, save :: x
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in FIRSTPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in FIRSTPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp parallel firstprivate(x)
     call sub()
   !$omp end parallel
@@ -20,7 +20,7 @@ subroutine firstprivate()
 subroutine lastprivate()
   class(*), allocatable, save :: x
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in LASTPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in LASTPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp do lastprivate(x)
   do i = 1, 10
     call sub()
@@ -33,7 +33,7 @@ subroutine copyin()
   class(*), allocatable, save :: x
   !$omp threadprivate(x)
 
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYIN clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYIN clause, the behavior is unspecified [-Wportability]
   !$omp parallel copyin(x)
     call sub()
   !$omp end parallel
@@ -46,7 +46,7 @@ subroutine copyprivate()
 
   !$omp single
     call sub()
-  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYPRIVATE clause, the behavior is unspecified
+  !PORTABILITY: If a polymorphic variable with allocatable attribute 'x' is in COPYPRIVATE clause, the behavior is unspecified [-Wportability]
   !$omp end single copyprivate(x)
 
 end
diff --git a/flang/test/Semantics/OpenMP/declarative-directive01.f90 b/flang/test/Semantics/OpenMP/declarative-directive01.f90
index e8bf605565fa..9e6e6021ffab 100644
--- a/flang/test/Semantics/OpenMP/declarative-directive01.f90
+++ b/flang/test/Semantics/OpenMP/declarative-directive01.f90
@@ -56,10 +56,10 @@ module m2
 contains
   subroutine foo
     !$omp declare target
-    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive
-    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive
+    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive [-Wopen-mp-usage]
+    !WARNING: The entity with PARAMETER attribute is used in a DECLARE TARGET directive [-Wopen-mp-usage]
     !$omp declare target (foo, N, M)
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
@@ -68,7 +68,7 @@ subroutine foo
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target enter(Q, S) link(R)
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !ERROR: MAP clause is not allowed on the DECLARE TARGET directive
     !$omp declare target to(Q) map(from:Q)
diff --git a/flang/test/Semantics/OpenMP/declare-target01.f90 b/flang/test/Semantics/OpenMP/declare-target01.f90
index 0651d3b5d89c..39185b171b6b 100644
--- a/flang/test/Semantics/OpenMP/declare-target01.f90
+++ b/flang/test/Semantics/OpenMP/declare-target01.f90
@@ -51,84 +51,84 @@ module declare_target01
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target (y%KIND)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var)
 
   !$omp declare target enter (my_var)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var) device_type(host)
   
   !$omp declare target enter (my_var) device_type(host)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%t_i)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%t_i)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%t_arr)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%t_arr)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%kind_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%kind_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (my_var%len_param)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (my_var%len_param)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr)
 
   !$omp declare target enter (arr)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr) device_type(nohost)
 
   !$omp declare target enter (arr) device_type(nohost)
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr(1))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (arr(1))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr(1:2))
 
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (arr(1:2))
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (x%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (x%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (w%LEN)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
   !$omp declare target enter (w%LEN)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (y%KIND)
 
   !ERROR: A type parameter inquiry cannot appear on the DECLARE TARGET directive
diff --git a/flang/test/Semantics/OpenMP/declare-target02.f90 b/flang/test/Semantics/OpenMP/declare-target02.f90
index 0f12180587f8..5146ed8c89f0 100644
--- a/flang/test/Semantics/OpenMP/declare-target02.f90
+++ b/flang/test/Semantics/OpenMP/declare-target02.f90
@@ -16,17 +16,17 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a1)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr1_to)
 
   !$omp declare target enter (arr1_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (blk1_to)
 
   !$omp declare target enter (blk1_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a1_to)
 
@@ -44,7 +44,7 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target (eq_a)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target to (eq_a)
 
@@ -57,7 +57,7 @@ program declare_target02
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target (eq_c)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot appear in an EQUIVALENCE statement
   !$omp declare target to (eq_c)
 
@@ -87,26 +87,26 @@ subroutine func()
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target (a3)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target to (arr2_to)
 
     !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
     !$omp declare target enter (arr2_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !$omp declare target to (arr3_to)
 
     !$omp declare target enter (arr3_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target to (a2_to)
 
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target enter (a2_to)
 
-    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+    !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
     !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
     !$omp declare target to (a3_to)
 
@@ -137,16 +137,16 @@ module mod4
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a4)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (arr4_to)
 
   !$omp declare target enter (arr4_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to (blk4_to)
   !$omp declare target enter (blk4_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a4_to)
 
@@ -174,21 +174,21 @@ subroutine func5()
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target (a5)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target to (arr5_to)
   
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target enter (arr5_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target to (blk5_to)
 
   !ERROR: A variable that appears in a DECLARE TARGET directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly
   !$omp declare target enter (blk5_to)
 
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !ERROR: A variable in a DECLARE TARGET directive cannot be an element of a common block
   !$omp declare target to (a5_to)
 
diff --git a/flang/test/Semantics/OpenMP/declare-target03.f90 b/flang/test/Semantics/OpenMP/declare-target03.f90
index bb1ed90e390f..64a299d78224 100644
--- a/flang/test/Semantics/OpenMP/declare-target03.f90
+++ b/flang/test/Semantics/OpenMP/declare-target03.f90
@@ -16,7 +16,7 @@ program main
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   !ERROR: The module name or main program name cannot be in a DECLARE TARGET directive
   !$omp declare target (main)
 end
diff --git a/flang/test/Semantics/OpenMP/declare-target06.f90 b/flang/test/Semantics/OpenMP/declare-target06.f90
index 7df0a7312309..08fe531af695 100644
--- a/flang/test/Semantics/OpenMP/declare-target06.f90
+++ b/flang/test/Semantics/OpenMP/declare-target06.f90
@@ -12,7 +12,7 @@ module test_0
 !ERROR: No explicit type declared for 'no_implicit_materialization_2'
 !$omp declare target link(no_implicit_materialization_2)
 
-!WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+!WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
 !ERROR: No explicit type declared for 'no_implicit_materialization_3'
 !$omp declare target to(no_implicit_materialization_3)
 
diff --git a/flang/test/Semantics/OpenMP/deprecation.f90 b/flang/test/Semantics/OpenMP/deprecation.f90
index df15c3bcc0b1..bc1c96d595e7 100644
--- a/flang/test/Semantics/OpenMP/deprecation.f90
+++ b/flang/test/Semantics/OpenMP/deprecation.f90
@@ -4,7 +4,7 @@
 
 subroutine test_master()
   integer :: c = 1
-!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
+!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead. [-Wopen-mp-usage]
   !$omp master 
   c = c + 1
   !$omp end master
@@ -12,7 +12,7 @@ subroutine test_master()
 
 subroutine test_parallel_master
   integer :: c = 2
-!WARNING: OpenMP directive PARALLEL MASTER has been deprecated, please use PARALLEL MASKED instead.
+!WARNING: OpenMP directive PARALLEL MASTER has been deprecated, please use PARALLEL MASKED instead. [-Wopen-mp-usage]
   !$omp parallel master
   c = c + 2
   !$omp end parallel master
@@ -20,7 +20,7 @@ subroutine test_parallel_master
 
 subroutine test_master_taskloop_simd()
   integer :: i, j = 1
-!WARNING: OpenMP directive MASTER TASKLOOP SIMD has been deprecated, please use MASKED TASKLOOP SIMD instead.
+!WARNING: OpenMP directive MASTER TASKLOOP SIMD has been deprecated, please use MASKED TASKLOOP SIMD instead. [-Wopen-mp-usage]
   !$omp master taskloop simd 
   do i=1,10
    j = j + 1
@@ -30,7 +30,7 @@ subroutine test_master_taskloop_simd()
 
 subroutine test_master_taskloop
   integer :: i, j = 1
-!WARNING: OpenMP directive MASTER TASKLOOP has been deprecated, please use MASKED TASKLOOP instead.
+!WARNING: OpenMP directive MASTER TASKLOOP has been deprecated, please use MASKED TASKLOOP instead. [-Wopen-mp-usage]
   !$omp master taskloop
   do i=1,10
    j = j + 1
@@ -40,7 +40,7 @@ subroutine test_master_taskloop
 
 subroutine test_parallel_master_taskloop_simd
   integer :: i, j = 1
-!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP SIMD has been deprecated, please use PARALLEL_MASKED TASKLOOP SIMD instead.
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP SIMD has been deprecated, please use PARALLEL_MASKED TASKLOOP SIMD instead. [-Wopen-mp-usage]
   !$omp parallel master taskloop simd 
   do i=1,10
    j = j + 1
@@ -50,7 +50,7 @@ subroutine test_parallel_master_taskloop_simd
 
 subroutine test_parallel_master_taskloop
   integer :: i, j = 1
-!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP has been deprecated, please use PARALLEL MASKED TASKLOOP instead.
+!WARNING: OpenMP directive PARALLEL MASTER TASKLOOP has been deprecated, please use PARALLEL MASKED TASKLOOP instead. [-Wopen-mp-usage]
   !$omp parallel master taskloop
   do i=1,10
    j = j + 1
diff --git a/flang/test/Semantics/OpenMP/do05-positivecase.f90 b/flang/test/Semantics/OpenMP/do05-positivecase.f90
index 8481cb2fc2ca..eda04610535c 100644
--- a/flang/test/Semantics/OpenMP/do05-positivecase.f90
+++ b/flang/test/Semantics/OpenMP/do05-positivecase.f90
@@ -57,4 +57,18 @@ program omp_do
     if(i<10) cycle
   end do
   !$omp end target teams distribute parallel do simd
+
+  !$omp target teams distribute 
+  !DEF: /omp_do/OtherConstruct6/i (OmpPrivate, OmpPreDetermined) HostAssoc INTEGER(4)
+  do i=1,100
+    !REF: /omp_do/OtherConstruct6/i
+    if(i < 5) cycle
+  end do
+
+  !$omp target teams distribute simd
+  !DEF: /omp_do/OtherConstruct7/i (OmpLinear, OmpPreDetermined) HostAssoc INTEGER(4)
+  do i=1,100
+    !REF: /omp_do/OtherConstruct7/i
+    if(i < 5) cycle
+  end do
 end program omp_do
diff --git a/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90 b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
new file mode 100644
index 000000000000..f718efc32aab
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/loop-transformation-construct01.f90
@@ -0,0 +1,100 @@
+! Testing the Semantics of nested Loop Transformation Constructs
+
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
+
+subroutine loop_transformation_construct1
+  implicit none
+
+  !$omp do
+  !ERROR: A DO loop must follow the UNROLL directive
+  !$omp unroll
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !$omp end tile
+  !$omp end do
+  !ERROR: The END TILE directive must follow the DO loop associated with the loop construct
+  !$omp end tile
+end subroutine
+
+subroutine loop_transformation_construct2
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: Only Loop Transformation Constructs or Loop Nests can be nested within Loop Constructs
+  !$omp parallel do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct3
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+  !ERROR: A DO loop must follow the TILE directive
+  !$omp tile
+end subroutine
+
+subroutine loop_transformation_construct4
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll full
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct5
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !ERROR: If a loop construct has been fully unrolled, it cannot then be tiled
+  !$omp tile
+  !$omp unroll
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
+
+subroutine loop_transformation_construct6
+  implicit none
+  integer :: i = 5
+  integer :: y
+  integer :: v(i)
+
+  !$omp do
+  !$omp tile
+  !$omp unroll partial(2)
+  do x = 1, i
+    v(x) = x(x) * 2
+  end do
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/nested-target.f90 b/flang/test/Semantics/OpenMP/nested-target.f90
index 6a56a84f4f57..a82b6c01b196 100644
--- a/flang/test/Semantics/OpenMP/nested-target.f90
+++ b/flang/test/Semantics/OpenMP/nested-target.f90
@@ -10,7 +10,7 @@ program main
   real, allocatable :: B(:)
 
   !$omp target
-  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target update from(arrayA) to(arrayB)
   do i = 1, 512
     arrayA(i) = arrayB(i)
@@ -20,7 +20,7 @@ program main
   !$omp parallel
   !$omp target
   !$omp parallel
-  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET UPDATE directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target update from(arrayA) to(arrayB)
   do i = 1, 512
     arrayA(i) = arrayB(i)
@@ -30,7 +30,7 @@ program main
   !$omp end parallel
 
   !$omp target
-  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target data map(to: a)
   do i = 1, N
     a = 3.14
@@ -40,12 +40,12 @@ program main
 
   allocate(B(N))
   !$omp target
-  !PORTABILITY: If TARGET ENTER DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET ENTER DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target enter data map(alloc:B)
   !$omp end target
 
   !$omp target
-  !PORTABILITY: If TARGET EXIT DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET EXIT DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !$omp target exit data map(delete:B)
   !$omp end target
   deallocate(B)
@@ -53,7 +53,7 @@ program main
   n1 = 10
   n2 = 10
   !$omp target teams map(to:a)
-  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET DATA directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !ERROR: Only `DISTRIBUTE`, `PARALLEL`, or `LOOP` regions are allowed to be strictly nested inside `TEAMS` region.
   !$omp target data map(n1,n2)
   do i=1, n1
@@ -65,7 +65,7 @@ program main
   !$omp end target teams
 
   !$omp target teams map(to:a) map(from:n1,n2)
-  !PORTABILITY: If TARGET TEAMS DISTRIBUTE PARALLEL DO directive is nested inside TARGET region, the behaviour is unspecified
+  !PORTABILITY: If TARGET TEAMS DISTRIBUTE PARALLEL DO directive is nested inside TARGET region, the behaviour is unspecified [-Wopen-mp-usage]
   !ERROR: Only `DISTRIBUTE`, `PARALLEL`, or `LOOP` regions are allowed to be strictly nested inside `TEAMS` region.
   !$omp target teams distribute parallel do
   do i=1, n1
diff --git a/flang/test/Semantics/OpenMP/requires04.f90 b/flang/test/Semantics/OpenMP/requires04.f90
index 1fbb3aa6219b..6eb0fb7f046a 100644
--- a/flang/test/Semantics/OpenMP/requires04.f90
+++ b/flang/test/Semantics/OpenMP/requires04.f90
@@ -6,7 +6,7 @@
 
 subroutine f
   integer, save :: x
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to(x) device_type(nohost)
   !$omp declare target enter(x) device_type(nohost)
 end subroutine f
diff --git a/flang/test/Semantics/OpenMP/requires05.f90 b/flang/test/Semantics/OpenMP/requires05.f90
index f410f0104d89..0e2352ead059 100644
--- a/flang/test/Semantics/OpenMP/requires05.f90
+++ b/flang/test/Semantics/OpenMP/requires05.f90
@@ -5,7 +5,7 @@
 ! device constructs, such as declare target with 'to' clause and no device_type.
 
 subroutine f
-  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead.
+  !WARNING: The usage of TO clause on DECLARE TARGET directive has been deprecated. Use ENTER clause instead. [-Wopen-mp-usage]
   !$omp declare target to(f)
   !$omp declare target enter(f)
 end subroutine f
diff --git a/flang/test/Semantics/OpenMP/single03.f90 b/flang/test/Semantics/OpenMP/single03.f90
index dc2c2fd27eb0..789fcfdd8808 100644
--- a/flang/test/Semantics/OpenMP/single03.f90
+++ b/flang/test/Semantics/OpenMP/single03.f90
@@ -39,7 +39,7 @@ subroutine omp_single
         !$omp single private(j) copyprivate(j)
             print *, "omp single", j
         !ERROR: COPYPRIVATE variable 'j' may not appear on a PRIVATE or FIRSTPRIVATE clause on a SINGLE construct
-        !WARNING: The COPYPRIVATE clause with 'j' is already used on the SINGLE directive
+        !WARNING: The COPYPRIVATE clause with 'j' is already used on the SINGLE directive [-Wopen-mp-usage]
         !$omp end single copyprivate(j)
 
         !$omp single nowait
diff --git a/flang/test/Semantics/OpenMP/single04.f90 b/flang/test/Semantics/OpenMP/single04.f90
index 9505745c600e..eabfe54b8ee3 100644
--- a/flang/test/Semantics/OpenMP/single04.f90
+++ b/flang/test/Semantics/OpenMP/single04.f90
@@ -52,20 +52,20 @@ program single
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x) nowait
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x) nowait
 
     !$omp single copyprivate(x)
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp end single copyprivate(x) nowait
 
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x, y) nowait
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: 'z' appears in more than one COPYPRIVATE clause on the END SINGLE directive
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x, z) copyprivate(z) nowait
@@ -73,9 +73,9 @@ program single
     !ERROR: NOWAIT clause must not be used with COPYPRIVATE clause on the SINGLE directive
     !$omp single copyprivate(x) nowait copyprivate(y) copyprivate(z)
         print *, x
-    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive
-    !WARNING: The COPYPRIVATE clause with 'y' is already used on the SINGLE directive
-    !WARNING: The COPYPRIVATE clause with 'z' is already used on the SINGLE directive
+    !WARNING: The COPYPRIVATE clause with 'x' is already used on the SINGLE directive [-Wopen-mp-usage]
+    !WARNING: The COPYPRIVATE clause with 'y' is already used on the SINGLE directive [-Wopen-mp-usage]
+    !WARNING: The COPYPRIVATE clause with 'z' is already used on the SINGLE directive [-Wopen-mp-usage]
     !ERROR: At most one NOWAIT clause can appear on the SINGLE directive
     !$omp end single copyprivate(x, y, z) nowait
 end program
diff --git a/flang/test/Semantics/OpenMP/target01.f90 b/flang/test/Semantics/OpenMP/target01.f90
index 545cc8a7b69f..f6e7c6a67d41 100644
--- a/flang/test/Semantics/OpenMP/target01.f90
+++ b/flang/test/Semantics/OpenMP/target01.f90
@@ -39,19 +39,19 @@ subroutine bar(b1, b2, b3)
   type(c_ptr), pointer :: b2
   type(c_ptr), value :: b3
 
-  !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(c)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b1)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b2)
     y = y + 1
   !$omp end target
-  !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later.
+  !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. [-Wopen-mp-usage]
   !$omp target is_device_ptr(b3)
     y = y + 1
   !$omp end target
diff --git a/flang/test/Semantics/OpenMP/threadprivate03.f90 b/flang/test/Semantics/OpenMP/threadprivate03.f90
index b466a8e05e9c..81e26ee327a9 100644
--- a/flang/test/Semantics/OpenMP/threadprivate03.f90
+++ b/flang/test/Semantics/OpenMP/threadprivate03.f90
@@ -13,7 +13,7 @@ program main
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(mod1)
 
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   !ERROR: The module name or main program name cannot be in a THREADPRIVATE directive
   !$omp threadprivate(main)
 
diff --git a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
index 5a5437e61845..ed0a1d238c96 100644
--- a/flang/test/Semantics/OpenMP/use_device_ptr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
@@ -27,7 +27,7 @@ subroutine omp_target_data
       a = arrayB
    !$omp end target data
 
-   !WARNING: Use of non-C_PTR type 'a' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead
+   !WARNING: Use of non-C_PTR type 'a' in USE_DEVICE_PTR is deprecated, use USE_DEVICE_ADDR instead [-Wopen-mp-usage]
    !$omp target data map(tofrom: a) use_device_ptr(a)
       a = 2
    !$omp end target data
diff --git a/flang/test/Semantics/allocate09.f90 b/flang/test/Semantics/allocate09.f90
index c6b3b58773b0..15ede202d55f 100644
--- a/flang/test/Semantics/allocate09.f90
+++ b/flang/test/Semantics/allocate09.f90
@@ -85,9 +85,9 @@ subroutine C946(param_ca_4_assumed, param_ta_4_assumed, param_ca_4_deferred)
 
   allocate(deferredChar, source="abcd")
   allocate(deferredChar, mold=deferredChar)
-  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD
+  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD [-Wallocate-to-other-length]
   allocate(char2, source="a")
-  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD
+  !PORTABILITY: Character length of allocatable object in ALLOCATE should be the same as the SOURCE or MOLD [-Wallocate-to-other-length]
   allocate(char2, source="abc")
   allocate(char2, mold=deferredChar)
 
diff --git a/flang/test/Semantics/argshape01.f90 b/flang/test/Semantics/argshape01.f90
index 19cca1ca4620..a1766c216bdb 100644
--- a/flang/test/Semantics/argshape01.f90
+++ b/flang/test/Semantics/argshape01.f90
@@ -113,7 +113,7 @@ program main
   !ERROR: Actual procedure argument has interface incompatible with dummy argument 's=': incompatible dummy argument #1: incompatible dummy data object shapes
   call s8c(s7)
   call s8c(s8)
-  !WARNING: Actual procedure argument has possible interface incompatibility with dummy argument 's=': possibly incompatible dummy argument #1: distinct dummy data object shapes
+  !WARNING: Actual procedure argument has possible interface incompatibility with dummy argument 's=': possibly incompatible dummy argument #1: distinct dummy data object shapes [-Wproc-dummy-arg-shapes]
   call s8c(s8b)
   call s9c(s9)
   call s9c(s9b)
@@ -154,7 +154,7 @@ program main
   !ERROR: Procedure pointer 'ps7' associated with incompatible procedure designator 's8': incompatible dummy argument #1: incompatible dummy data object shapes
   ps7 => s8
   ps8 => s8
-  !WARNING: pointer 'ps8' and s8b may not be completely compatible procedures: possibly incompatible dummy argument #1: distinct dummy data object shapes
+  !WARNING: pointer 'ps8' and s8b may not be completely compatible procedures: possibly incompatible dummy argument #1: distinct dummy data object shapes [-Wproc-dummy-arg-shapes]
   ps8 => s8b
   !ERROR: Procedure pointer 'ps8' associated with incompatible procedure designator 's6': incompatible dummy argument #1: incompatible dummy data object shapes
   ps8 => s6
diff --git a/flang/test/Semantics/assign02.f90 b/flang/test/Semantics/assign02.f90
index 9fa672025bfe..f998197aeab5 100644
--- a/flang/test/Semantics/assign02.f90
+++ b/flang/test/Semantics/assign02.f90
@@ -85,7 +85,7 @@ subroutine s5
     real, pointer, volatile :: q
     p => x
     !ERROR: Pointer must be VOLATILE when target is a VOLATILE coarray
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p => y
     !ERROR: Pointer may not be VOLATILE when target is a non-VOLATILE coarray
     q => x
@@ -175,7 +175,7 @@ subroutine s12
     real, volatile, target :: x
     real, pointer :: p
     real, pointer, volatile :: q
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p => x
     q => x
   end
@@ -188,11 +188,11 @@ subroutine s13
     real, pointer, volatile :: q1
     type(t2), pointer, volatile :: q2
     type(t3), pointer, volatile :: q3
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p1 => y%t3Field%t2Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p2 => y%t3Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3 => y
     !OK:
     q1 => y%t3Field%t2Field
@@ -200,9 +200,9 @@ subroutine s13
     q2 => y%t3Field
     !OK:
     q3 => y
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3%t3FieldPtr => y%t3Field
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     p3%t3FieldPtr%t2FieldPtr => y%t3Field%t2Field
     !OK
     q3%t3FieldPtr => y%t3Field
diff --git a/flang/test/Semantics/assign09.f90 b/flang/test/Semantics/assign09.f90
index b29e67a473e1..7c95e3a64ea5 100644
--- a/flang/test/Semantics/assign09.f90
+++ b/flang/test/Semantics/assign09.f90
@@ -49,9 +49,9 @@ elemental real function userElemental(a)
   realToRealProcPtr => noInterfaceExternal ! ok
   intToRealProcPtr => noInterfaceExternal !ok
   call sub1(noInterfaceExternal) ! ok
-  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface
+  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface [-Wimplicit-interface-actual]
   call sub2(noInterfaceExternal)
-  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface
+  !WARNING: Actual procedure argument has an implicit interface which is not known to be compatible with dummy argument 'p=' which has an explicit interface [-Wimplicit-interface-actual]
   call sub3(noInterfaceExternal)
 
   !ERROR: Procedure pointer 'nointerfaceprocptr' with implicit interface may not be associated with procedure designator 'userelemental' with explicit interface that cannot be called via an implicit interface
diff --git a/flang/test/Semantics/associate01.f90 b/flang/test/Semantics/associate01.f90
index deafea695e84..b51379bab3c3 100644
--- a/flang/test/Semantics/associate01.f90
+++ b/flang/test/Semantics/associate01.f90
@@ -13,7 +13,7 @@ module m1
   function iptr(n)
     integer, intent(in), target :: n
     integer, pointer :: iptr
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'n' is an INTENT(IN) dummy argument
     iptr => n
   end function
diff --git a/flang/test/Semantics/associated.f90 b/flang/test/Semantics/associated.f90
index 3f3f5488ad9f..7cb6c240db22 100644
--- a/flang/test/Semantics/associated.f90
+++ b/flang/test/Semantics/associated.f90
@@ -123,15 +123,15 @@ subroutine test(assumedRank)
     lVar = associated(null(intAllocVar))
     lVar = associated(null()) !OK
     lVar = associated(null(intPointerVar1)) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
     !BECAUSE: 'NULL()' is a null pointer
     lVar = associated(null(), null()) !OK
     lVar = associated(intPointerVar1, null(intPointerVar2)) !OK
     lVar = associated(intPointerVar1, null()) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a valid left-hand side of a pointer assignment statement [-Wportability]
     !BECAUSE: 'NULL()' is a null pointer
     lVar = associated(null(), null(intPointerVar1)) !OK
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(null(intPointerVar1), null()) !OK
     !ERROR: POINTER= argument of ASSOCIATED() must be a pointer
     lVar = associated(intVar)
@@ -180,18 +180,18 @@ subroutine test(assumedRank)
 
     ! Functions (other than NULL) returning pointers
     lVar = associated(objPtrFunc(targetIntVar1)) ! ok
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), targetIntVar1) ! ok
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), objPtrFunc(targetIntVar1)) ! ok
     lVar = associated(procPtrFunc()) ! ok
     lVar = associated(procPtrFunc(), intFunc) ! ok
     lVar = associated(procPtrFunc(), procPtrFunc()) ! ok
     !ERROR: POINTER= argument 'objptrfunc(targetintvar1)' is an object pointer but the TARGET= argument 'intfunc' is not a variable
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), intFunc)
     !ERROR: POINTER= argument 'objptrfunc(targetintvar1)' is an object pointer but the TARGET= argument 'procptrfunc()' is not a variable
-    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer
+    !PORTABILITY: POINTER= argument of ASSOCIATED() is required by some other compilers to be a pointer [-Wportability]
     lVar = associated(objPtrFunc(targetIntVar1), procPtrFunc())
     !ERROR: POINTER= argument 'procptrfunc()' is a procedure pointer but the TARGET= argument 'objptrfunc(targetintvar1)' is not a procedure or procedure pointer
     lVar = associated(procPtrFunc(), objPtrFunc(targetIntVar1))
diff --git a/flang/test/Semantics/bind-c02.f90 b/flang/test/Semantics/bind-c02.f90
index 416d071542fe..66e1f0b816b9 100644
--- a/flang/test/Semantics/bind-c02.f90
+++ b/flang/test/Semantics/bind-c02.f90
@@ -19,8 +19,8 @@ subroutine proc()
   !ERROR: Only variable and named common block can be in BIND statement
   bind(c) :: sub
 
-  !PORTABILITY: Global name 'm' conflicts with a module
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Global name 'm' conflicts with a module [-Wbenign-name-clash]
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   bind(c) :: m ! no error for implicit type variable
 
   type my_type
diff --git a/flang/test/Semantics/bind-c04.f90 b/flang/test/Semantics/bind-c04.f90
index 27119e375ce0..86a925f0090e 100644
--- a/flang/test/Semantics/bind-c04.f90
+++ b/flang/test/Semantics/bind-c04.f90
@@ -37,7 +37,7 @@ subroutine aproc2() bind(c) ! ok
 
   procedure(proc), bind(c) :: y
 
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   !ERROR: A procedure pointer may not have a BIND attribute with a name
   procedure(proc), bind(c, name="pc8"), bind(c), pointer :: pc8
 
diff --git a/flang/test/Semantics/bind-c06.f90 b/flang/test/Semantics/bind-c06.f90
index ff78a4743dee..608d193ee04d 100644
--- a/flang/test/Semantics/bind-c06.f90
+++ b/flang/test/Semantics/bind-c06.f90
@@ -40,7 +40,7 @@ program main
     procedure, nopass :: b => s
   end type
 
-  ! WARNING: A derived type with the BIND attribute should not be empty
+  ! WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c) :: t5
   end type
 
@@ -71,7 +71,7 @@ program main
   end type
 
   type, bind(c) :: t10
-    !WARNING: A CHARACTER component of an interoperable type should have length 1
+    !WARNING: A CHARACTER component of an interoperable type should have length 1 [-Wbind-c-char-length]
     character(len=2) x
   end type
   type, bind(c) :: t11
@@ -79,7 +79,7 @@ program main
     character(kind=2) x
   end type
   type, bind(c) :: t12
-    !PORTABILITY: A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL
+    !PORTABILITY: A LOGICAL component of an interoperable type should have the interoperable KIND=C_BOOL [-Wlogical-vs-c-bool]
     logical(kind=8) x
   end type
   type, bind(c) :: t13
diff --git a/flang/test/Semantics/bind-c11.f90 b/flang/test/Semantics/bind-c11.f90
index 54021f68b3d8..761a113fe7f7 100644
--- a/flang/test/Semantics/bind-c11.f90
+++ b/flang/test/Semantics/bind-c11.f90
@@ -8,7 +8,7 @@ module m
   real, allocatable, bind(c) :: x3(:)
  contains
   subroutine s1(x) bind(c)
-    !PORTABILITY: A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL
+    !PORTABILITY: A BIND(C) LOGICAL dummy argument should have the interoperable KIND=C_BOOL [-Wlogical-vs-c-bool]
     logical(2), intent(in), value :: x
   end
   subroutine s2(x) bind(c)
diff --git a/flang/test/Semantics/bind-c13.f90 b/flang/test/Semantics/bind-c13.f90
index 81815c1a95ef..867d63fc0e30 100644
--- a/flang/test/Semantics/bind-c13.f90
+++ b/flang/test/Semantics/bind-c13.f90
@@ -5,8 +5,8 @@ subroutine interop(ptr,ashape,arank,eshape,asize) bind(c)
   real, pointer, contiguous :: ptr(:)
   real, contiguous :: ashape(:) ! ok
   real, contiguous :: arank(..) ! ok
-  !PORTABILITY: CONTIGUOUS entity 'eshape' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'eshape' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: eshape(10)
-  !PORTABILITY: CONTIGUOUS entity 'asize' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'asize' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: asize(*)
 end
diff --git a/flang/test/Semantics/bind-c17.f90 b/flang/test/Semantics/bind-c17.f90
index 8e0ecde67a0a..34d21865d4e4 100644
--- a/flang/test/Semantics/bind-c17.f90
+++ b/flang/test/Semantics/bind-c17.f90
@@ -4,7 +4,7 @@ module m
   end type
  contains
   subroutine sub(x) bind(c)
-    !PORTABILITY: The derived type of this interoperable object should be BIND(C)
+    !PORTABILITY: The derived type of this interoperable object should be BIND(C) [-Wportability]
     type(a), pointer, intent(in) :: x
   end
 end
diff --git a/flang/test/Semantics/bindings01.f90 b/flang/test/Semantics/bindings01.f90
index dc44db09c4a6..3283ac30eb07 100644
--- a/flang/test/Semantics/bindings01.f90
+++ b/flang/test/Semantics/bindings01.f90
@@ -4,7 +4,7 @@
 
 module m
   !ERROR: An ABSTRACT derived type must be extensible
-  !PORTABILITY: A derived type with the BIND attribute should not be empty
+  !PORTABILITY: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, abstract, bind(c) :: badAbstract1
   end type
   !ERROR: An ABSTRACT derived type must be extensible
@@ -45,7 +45,7 @@ module m
   end type
   type, extends(intermediate) :: concrete2  ! ensure no false missing binding error
   end type
-  !WARNING: A derived type with the BIND attribute should not be empty
+  !WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c) :: inextensible1
   end type
   !ERROR: The parent type is not extensible
@@ -226,7 +226,7 @@ subroutine mysubr
   end subroutine
   subroutine test
     type(t) a(2)
-    !PORTABILITY: Base of NOPASS type-bound procedure reference should be scalar
+    !PORTABILITY: Base of NOPASS type-bound procedure reference should be scalar [-Wnopass-scalar-base]
     call a%tbp
     !ERROR: Base of procedure component reference must be scalar
     call a%pp
diff --git a/flang/test/Semantics/bindings03.f90 b/flang/test/Semantics/bindings03.f90
index baa8432a2701..b03caf0ac452 100644
--- a/flang/test/Semantics/bindings03.f90
+++ b/flang/test/Semantics/bindings03.f90
@@ -15,9 +15,9 @@ program test
   use m
   procedure(sub), pointer :: p
   type(t) x
-  !PORTABILITY: Procedure binding 'sub' used as target of a pointer assignment
+  !PORTABILITY: Procedure binding 'sub' used as target of a pointer assignment [-Wbinding-as-procedure]
   p => x%sub
-  !PORTABILITY: Procedure binding 'sub' passed as an actual argument
+  !PORTABILITY: Procedure binding 'sub' passed as an actual argument [-Wbinding-as-procedure]
   call sub2(x%sub)
  contains
   subroutine sub2(s)
diff --git a/flang/test/Semantics/block-data01.f90 b/flang/test/Semantics/block-data01.f90
index aa4ede787c17..c75c63afcf8c 100644
--- a/flang/test/Semantics/block-data01.f90
+++ b/flang/test/Semantics/block-data01.f90
@@ -7,7 +7,7 @@ block data foo
   !ERROR: An initialized variable in BLOCK DATA must be in a COMMON block
   integer :: notInCommon = 1
   integer :: uninitialized ! ok
-  !PORTABILITY: Procedure pointer 'q' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'q' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: 'q' may not appear in a BLOCK DATA subprogram
   procedure(sin), pointer :: q => cos
   !ERROR: 'p' may not be a procedure as it is in a COMMON block
diff --git a/flang/test/Semantics/boz-literal-constants.f90 b/flang/test/Semantics/boz-literal-constants.f90
index ee6919cb9ecd..4d957d13f3f6 100644
--- a/flang/test/Semantics/boz-literal-constants.f90
+++ b/flang/test/Semantics/boz-literal-constants.f90
@@ -56,7 +56,7 @@ subroutine explicit(n, x, c)
   res = CMPLX (realpart, img, 4)
   res = CMPLX (B"0101", B"1111", 4)
 
-  !WARNING: underflow on REAL(8) to REAL(4) conversion
+  !WARNING: underflow on REAL(8) to REAL(4) conversion [-Wfolding-exception]
   dbl = DBLE(B"1111")
   dbl = DBLE(realpart)
 
diff --git a/flang/test/Semantics/bug1046.f90 b/flang/test/Semantics/bug1046.f90
new file mode 100644
index 000000000000..dc5c47062138
--- /dev/null
+++ b/flang/test/Semantics/bug1046.f90
@@ -0,0 +1,15 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic -Werror
+!WARNING: INTEGER(4) 0**0 is not defined [-Wfolding-exception]
+print *, 0**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, 0**0.
+!WARNING: invalid argument on power with INTEGER exponent [-Wfolding-exception]
+print *, 0.0**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, 0.0**0.
+!WARNING: invalid argument on power with INTEGER exponent [-Wfolding-exception]
+print *, (0.0, 0.0)**0
+!WARNING: REAL/COMPLEX 0**0 is not defined [-Wfolding-exception]
+print *, (0.0, 0.0)**0.
+print *, (0.0, 0.0)**2.5
+end
diff --git a/flang/test/Semantics/bug1056.f90 b/flang/test/Semantics/bug1056.f90
new file mode 100644
index 000000000000..b32270dab8f7
--- /dev/null
+++ b/flang/test/Semantics/bug1056.f90
@@ -0,0 +1,13 @@
+! RUN: %flang_fc1 -fdebug-unparse %s | FileCheck %s
+program bug
+  integer, target :: ita(2) = [1,2], itb(2) = [3,4], itc(2) = [5,6]
+  type t1
+    integer, pointer :: p1(:) => ita, p2(:) => itb
+  end type
+  type t2
+    !CHECK: TYPE(t1) :: comp = t1(p1=itc,p2=itb)
+    type(t1) :: comp = t1(itc)
+  end type
+  integer, pointer :: p3(:) => itd
+  integer, target :: itd(2) = [7,8]
+end
diff --git a/flang/test/Semantics/bug1092.F90 b/flang/test/Semantics/bug1092.F90
new file mode 100644
index 000000000000..ef108733fd78
--- /dev/null
+++ b/flang/test/Semantics/bug1092.F90
@@ -0,0 +1,26 @@
+!RUN: rm -rf %t && mkdir -p %t
+!RUN: %flang_fc1 -DWHICH=1 -fsyntax-only -J%t %s
+!RUN: %flang_fc1 -DWHICH=2 -fsyntax-only -fhermetic-module-files -I%t -J%t %s
+!RUN: %flang_fc1 -fsyntax-only -I%t %s 2>&1 | FileCheck --allow-empty %s
+!CHECK-NOT: error:
+
+#if WHICH == 1
+module bug1092a
+  type t
+  end type
+ contains
+  subroutine subr(x)
+    type(t) x
+  end
+end
+#elif WHICH == 2
+module bug1092b
+  use bug1092a, only: subr
+end
+#else
+use bug1092a, only: t
+use bug1092b, only: subr
+type(t) x
+call subr(x)
+end
+#endif
diff --git a/flang/test/Semantics/bug144453.f90 b/flang/test/Semantics/bug144453.f90
new file mode 100644
index 000000000000..85c0892bffb5
--- /dev/null
+++ b/flang/test/Semantics/bug144453.f90
@@ -0,0 +1,10 @@
+!RUN: %python %S/test_errors.py %s %flang_fc1
+function func(immutable)
+  logical func
+  logical, intent(in) :: immutable
+  !No warning about an undefined function result should appear
+  INQUIRE(file="/usr/local/games/adventure", EXIST=func)
+  !ERROR: EXIST variable 'immutable' is not definable
+  !BECAUSE: 'immutable' is an INTENT(IN) dummy argument
+  INQUIRE(file="/usr/local/games/adventure", EXIST=immutable)
+end
diff --git a/flang/test/Semantics/c7108.f90 b/flang/test/Semantics/c7108.f90
index c23a0abe3ee0..0b7148f4d645 100644
--- a/flang/test/Semantics/c7108.f90
+++ b/flang/test/Semantics/c7108.f90
@@ -33,9 +33,9 @@ program p
   use m
   type(foo) x
   x = foo(); print *, x       ! ok, not ambiguous
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar0') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar0') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   x = foo(2); print *, x      ! ambigous
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar2') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar2') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   x = foo(3.); print *, x     ! ambiguous due to data conversion
   x = foo(.true.); print *, x ! ok, not ambigous
 end
diff --git a/flang/test/Semantics/c_f_pointer.f90 b/flang/test/Semantics/c_f_pointer.f90
index de5673f8dc1d..e2d00536cdac 100644
--- a/flang/test/Semantics/c_f_pointer.f90
+++ b/flang/test/Semantics/c_f_pointer.f90
@@ -46,12 +46,12 @@ program test
   call c_f_pointer(scalarC, multiDimIntF, shape=[1_8])
   !ERROR: SHAPE= argument to C_F_POINTER() must be a rank-one array.
   call c_f_pointer(scalarC, multiDimIntF, shape=rankTwoArray)
-  !WARNING: FPTR= argument to C_F_POINTER() should not be unlimited polymorphic
+  !WARNING: FPTR= argument to C_F_POINTER() should not be unlimited polymorphic [-Winteroperability]
   call c_f_pointer(scalarC, unlimited)
-  !PORTABILITY: FPTR= argument to C_F_POINTER() should not have a derived type that is not BIND(C)
+  !PORTABILITY: FPTR= argument to C_F_POINTER() should not have a derived type that is not BIND(C) [-Wportability]
   call c_f_pointer(scalarC, notBindC)
-  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable character length CHARACTER(KIND=1,LEN=2_8)
+  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable character length CHARACTER(KIND=1,LEN=2_8) [-Wcharacter-interoperability]
   call c_f_pointer(scalarC, c2ptr)
-  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8)
+  !WARNING: FPTR= argument to C_F_POINTER() should not have the non-interoperable intrinsic type or kind CHARACTER(KIND=4,LEN=1_8) [-Winteroperability]
   call c_f_pointer(scalarC, unicodePtr)
 end program
diff --git a/flang/test/Semantics/c_loc01.f90 b/flang/test/Semantics/c_loc01.f90
index a515a7a64f02..da8a0e5bdd9e 100644
--- a/flang/test/Semantics/c_loc01.f90
+++ b/flang/test/Semantics/c_loc01.f90
@@ -14,7 +14,7 @@ subroutine test(assumedType, poly, nclen, n)
     type(c_ptr) cp
     type(c_funptr) cfp
     real notATarget
-    !PORTABILITY: Procedure pointer 'pptr' should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: Procedure pointer 'pptr' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(sin), pointer :: pptr
     real, target :: arr(3)
     type(hasLen(1)), target :: clen
@@ -41,9 +41,9 @@ subroutine test(assumedType, poly, nclen, n)
     cp = c_loc(nclen)
     !ERROR: C_LOC() argument may not be zero-length character
     cp = c_loc(ch(2:1))
-    !WARNING: C_LOC() argument has non-interoperable character length
+    !WARNING: C_LOC() argument has non-interoperable character length [-Wcharacter-interoperability]
     cp = c_loc(ch)
-    !WARNING: C_LOC() argument has non-interoperable intrinsic type or kind
+    !WARNING: C_LOC() argument has non-interoperable intrinsic type or kind [-Winteroperability]
     cp = c_loc(unicode)
     cp = c_loc(ch(1:1)) ! ok
     cp = c_loc(deferred) ! ok
diff --git a/flang/test/Semantics/call01.f90 b/flang/test/Semantics/call01.f90
index 67d14c380067..6849c5f6500b 100644
--- a/flang/test/Semantics/call01.f90
+++ b/flang/test/Semantics/call01.f90
@@ -119,16 +119,16 @@ end function nested
 end function
 
 subroutine s01(f1, f2, fp1, fp2, fp3)
-  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
   character*(*) :: f1, f3, fp1
   external :: f1, f3
   pointer :: fp1, fp3
-  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+  !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
   procedure(character*(*)), pointer :: fp2
   interface
     character*(*) function f2()
     end function
-    !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+    !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
     character*(*) function fp3()
     end function
     !ERROR: A function interface may not declare an assumed-length CHARACTER(*) result
diff --git a/flang/test/Semantics/call02.f90 b/flang/test/Semantics/call02.f90
index 0ec5530f9808..18cc0ecdfd85 100644
--- a/flang/test/Semantics/call02.f90
+++ b/flang/test/Semantics/call02.f90
@@ -8,7 +8,7 @@ elemental real function elem(x)
       real, intent(in), value :: x
     end function
     subroutine subr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin) :: dummy
     end subroutine
     subroutine badsubr(dummy)
@@ -17,11 +17,11 @@ subroutine badsubr(dummy)
       procedure(elem) :: dummy
     end subroutine
     subroutine optionalsubr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin), optional :: dummy
     end subroutine
     subroutine ptrsubr(dummy)
-      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+      !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
       procedure(sin), pointer, intent(in) :: dummy
     end subroutine
   end interface
diff --git a/flang/test/Semantics/call03.f90 b/flang/test/Semantics/call03.f90
index 59513557324e..1721b5998686 100644
--- a/flang/test/Semantics/call03.f90
+++ b/flang/test/Semantics/call03.f90
@@ -163,9 +163,9 @@ subroutine test06 ! 15.5.2.4(4)
     type(pdtWithDefault(3)) :: defaultVar3
     type(pdtWithDefault(4)) :: defaultVar4
     character :: ch1
-    !ERROR: Actual argument variable length '1' is less than expected length '2'
+    !ERROR: Actual argument variable length '1' is less than expected length '2' [-Wshort-character-actual]
     call ch2(ch1)
-    !WARNING: Actual argument expression length '0' is less than expected length '2'
+    !WARNING: Actual argument expression length '0' is less than expected length '2' [-Wshort-character-actual]
     call ch2("")
     call pdtdefault(vardefault)
     !ERROR: Actual argument type 'pdt(n=3_4)' is not compatible with dummy argument type 'pdt'
@@ -300,10 +300,10 @@ subroutine test12 ! 15.5.2.4(21)
     !ERROR: Actual argument associated with INTENT(IN OUT) dummy argument 'x=' is not definable
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call intentinout_arr(a(j))
-    !WARNING: Actual argument associated with ASYNCHRONOUS dummy argument 'x=' is not definable
+    !WARNING: Actual argument associated with ASYNCHRONOUS dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call asynchronous_arr(a(j))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'x=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'x=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: Variable 'a(int(j,kind=8))' has a vector subscript
     call volatile_arr(a(j))
   end subroutine
@@ -386,9 +386,9 @@ subroutine test16() ! C1540
     call contiguous(a) ! ok
     call pointer(a) ! ok
     call pointer(b) ! ok
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     call pointer(c) ! ok
-    !ERROR: VOLATILE target associated with non-VOLATILE pointer
+    !ERROR: VOLATILE target associated with non-VOLATILE pointer [-Wnon-volatile-pointer-to-volatile]
     call pointer(d) ! ok
     call valueassumedsize(a) ! ok
     call valueassumedsize(b) ! ok
diff --git a/flang/test/Semantics/call05.f90 b/flang/test/Semantics/call05.f90
index a06fe4f196c8..b9b463a44979 100644
--- a/flang/test/Semantics/call05.f90
+++ b/flang/test/Semantics/call05.f90
@@ -73,9 +73,9 @@ subroutine test
     call sma(ma) ! ok
     call spp(pp) ! ok
     call spa(pa) ! ok
-    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so [-Wpolymorphic-actual-allocatable-or-pointer-to-monomorphic-dummy]
     call smp(pp)
-    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE actual argument is polymorphic, the corresponding dummy argument should also be so [-Wpolymorphic-actual-allocatable-or-pointer-to-monomorphic-dummy]
     call sma(pa)
     !ERROR: If a POINTER or ALLOCATABLE dummy or actual argument is polymorphic, both must be so
     call spp(mp)
diff --git a/flang/test/Semantics/call07.f90 b/flang/test/Semantics/call07.f90
index 92f2bdba882d..7e29fb74dd61 100644
--- a/flang/test/Semantics/call07.f90
+++ b/flang/test/Semantics/call07.f90
@@ -19,17 +19,17 @@ subroutine s04(p)
   end subroutine
 
   subroutine test
-    !PORTABILITY: CONTIGUOUS entity 'a01' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'a01' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real, pointer, contiguous :: a01 ! C830
     real, pointer :: a02(:)
     real, target :: a03(10)
     real :: a04(10) ! not TARGET
-    !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real, contiguous :: scalar
     call s01(a03) ! ok
     !ERROR: CONTIGUOUS pointer dummy argument may not be associated with non-CONTIGUOUS pointer actual argument
     call s01(a02)
-    !WARNING: Target of CONTIGUOUS pointer association is not known to be contiguous
+    !WARNING: Target of CONTIGUOUS pointer association is not known to be contiguous [-Wpointer-to-possible-noncontiguous]
     call s01(a02(:))
     !ERROR: CONTIGUOUS pointer may not be associated with a discontiguous target
     call s01(a03(::2))
diff --git a/flang/test/Semantics/call09.f90 b/flang/test/Semantics/call09.f90
index 58b2382f600e..e7ff72f74de0 100644
--- a/flang/test/Semantics/call09.f90
+++ b/flang/test/Semantics/call09.f90
@@ -37,7 +37,7 @@ subroutine s05(p)
   end subroutine
 
   subroutine selemental1(p)
-    !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(cos) :: p ! ok
   end subroutine
 
diff --git a/flang/test/Semantics/call10.f90 b/flang/test/Semantics/call10.f90
index 47976ca9630c..a9b760cc692e 100644
--- a/flang/test/Semantics/call10.f90
+++ b/flang/test/Semantics/call10.f90
@@ -62,7 +62,7 @@ pure real function f03a(a)
     real, pointer :: a ! ok
   end function
   pure real function f04(a) ! C1583
-    !WARNING: non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE
+    !WARNING: non-POINTER dummy argument of pure function should be INTENT(IN) or VALUE [-Wrelaxed-pure-dummy]
     real, intent(out) :: a
   end function
   pure real function f04a(a)
diff --git a/flang/test/Semantics/call14.f90 b/flang/test/Semantics/call14.f90
index fba11d35790b..ad67caa41b4e 100644
--- a/flang/test/Semantics/call14.f90
+++ b/flang/test/Semantics/call14.f90
@@ -9,7 +9,7 @@ module m
   !ERROR: VALUE attribute may apply only to a dummy data object
   subroutine C863(notData,assumedSize,coarray,coarrayComponent,assumedRank,assumedLen)
     external :: notData
-    !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+    !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
     real, value :: notADummy
     value :: notData
     !ERROR: VALUE attribute may not apply to an assumed-size array
@@ -20,7 +20,7 @@ subroutine C863(notData,assumedSize,coarray,coarrayComponent,assumedRank,assumed
     type(hasCoarray), value :: coarrayComponent
     !ERROR: VALUE attribute may not apply to an assumed-rank array
     real, value :: assumedRank(..)
-    !PORTABILITY: VALUE attribute on assumed-length CHARACTER may not be portable
+    !PORTABILITY: VALUE attribute on assumed-length CHARACTER may not be portable [-Wportability]
     character(*), value :: assumedLen
   end subroutine
   subroutine C864(allocatable, inout, out, pointer, volatile)
diff --git a/flang/test/Semantics/call24.f90 b/flang/test/Semantics/call24.f90
index c1053db93648..bdcf80a74b84 100644
--- a/flang/test/Semantics/call24.f90
+++ b/flang/test/Semantics/call24.f90
@@ -38,7 +38,7 @@ subroutine test()
 
   !ERROR: References to the procedure 'bar' require an explicit interface
   !BECAUSE: a dummy procedure is optional or a pointer
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Actual argument associated with procedure pointer dummy argument 'a_pointer=' is not a procedure pointer
   call bar(sin)
 
diff --git a/flang/test/Semantics/call27.f90 b/flang/test/Semantics/call27.f90
index f401eb254fde..2559482cfe2b 100644
--- a/flang/test/Semantics/call27.f90
+++ b/flang/test/Semantics/call27.f90
@@ -4,9 +4,9 @@ program test
   real, allocatable :: a
   !ERROR: NULL() actual argument 'NULL()' may not be associated with allocatable dummy argument dummy argument 'a=' that is INTENT(OUT) or INTENT(IN OUT)
   call foo0(null())
-  !WARNING: NULL() actual argument 'NULL()' should not be associated with allocatable dummy argument dummy argument 'a=' without INTENT(IN)
+  !WARNING: NULL() actual argument 'NULL()' should not be associated with allocatable dummy argument dummy argument 'a=' without INTENT(IN) [-Wnull-actual-for-default-intent-allocatable]
   call foo1(null())
-  !PORTABILITY: Allocatable dummy argument 'a=' is associated with NULL()
+  !PORTABILITY: Allocatable dummy argument 'a=' is associated with NULL() [-Wnull-actual-for-allocatable]
   call foo2(null())
   call foo3(null()) ! ok
   !ERROR: Actual argument associated with INTENT(IN OUT) dummy argument 'a=' is not definable
diff --git a/flang/test/Semantics/call30.f90 b/flang/test/Semantics/call30.f90
index 7aec5903606e..1570eb02b675 100644
--- a/flang/test/Semantics/call30.f90
+++ b/flang/test/Semantics/call30.f90
@@ -23,49 +23,49 @@ subroutine vol_dum_int_arr(my_int_arr)
   end subroutine vol_dum_int_arr
 
   subroutine test_all_subprograms()
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '6_4' is not a variable or pointer
     call vol_dum_int(6)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '18_4' is not a variable or pointer
     call vol_dum_int(6+12)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '72_4' is not a variable or pointer
     call vol_dum_int(6*12)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '-3_4' is not a variable or pointer
     call vol_dum_int(-6/2)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.1415927410125732421875_4' is not a variable or pointer
     call vol_dum_real(3.141592653)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.1415927410125732421875_4' is not a variable or pointer
     call vol_dum_real(3.141592653 + (-10.6e-11))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '3.3300884272335906644002534449100494384765625e-10_4' is not a variable or pointer
     call vol_dum_real(3.141592653 * 10.6e-11)
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_real=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '-2.9637666816e10_4' is not a variable or pointer
     call vol_dum_real(3.141592653 / (-10.6e-11))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(1._4,3.2000000476837158203125_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(-1._4,6.340000152587890625_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) + (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(-1.2048000335693359375e1_4,-3.2599999904632568359375_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) * (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_complex=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '(5.80680549144744873046875e-1_4,-6.8833148479461669921875e-1_4)' is not a variable or pointer
     call vol_dum_complex((1., 3.2) / (-2., 3.14))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '[INTEGER(4)::1_4,2_4,3_4,4_4]' is not a variable or pointer
     call vol_dum_int_arr((/ 1, 2, 3, 4 /))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: 'reshape([INTEGER(4)::1_4,2_4,3_4,4_4],shape=[2,2])' is not a variable or pointer
     call vol_dum_int_arr(reshape((/ 1, 2, 3, 4 /), (/ 2, 2/)))
-    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable
+    !WARNING: Actual argument associated with VOLATILE dummy argument 'my_int_arr=' is not definable [-Wundefinable-asynchronous-or-volatile-actual]
     !BECAUSE: '[INTEGER(4)::1_4,2_4,3_4,4_4]' is not a variable or pointer
     call vol_dum_int_arr((/ 1, 2, 3, 4 /))
   end subroutine test_all_subprograms
diff --git a/flang/test/Semantics/call31.f90 b/flang/test/Semantics/call31.f90
index d9fd2cc60ec3..71fecd536345 100644
--- a/flang/test/Semantics/call31.f90
+++ b/flang/test/Semantics/call31.f90
@@ -4,7 +4,7 @@
       module m
        contains
         subroutine subr(parg)
-          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
           procedure(character(*)), pointer :: parg
           !ERROR: An assumed (*) type parameter may be used only for a (non-statement function) dummy argument, associate name, character named constant, or external function result
           procedure(character(*)), pointer :: plocal
@@ -14,7 +14,7 @@ subroutine subr(parg)
         end subroutine
 
         subroutine subr_1(parg_1)
-          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type
+          !PORTABILITY: A dummy procedure pointer should not have assumed-length CHARACTER(*) result type [-Wportability]
           procedure(character(*)), pointer :: parg_1
           print *, parg_1()
         end subroutine
diff --git a/flang/test/Semantics/call33.f90 b/flang/test/Semantics/call33.f90
index 285c4be98a9d..05b7dfb8d8c4 100644
--- a/flang/test/Semantics/call33.f90
+++ b/flang/test/Semantics/call33.f90
@@ -29,7 +29,7 @@ program test
   character(4) long, longarr(1)
   character(4), allocatable :: longalloc
   character(4), pointer :: longptr
-  !WARNING: Actual argument variable length '2' is less than expected length '3'
+  !WARNING: Actual argument variable length '2' is less than expected length '3' [-Wshort-character-actual]
   call s1(short)
   !ERROR: Actual argument array has fewer characters (2) than dummy argument 'x=' array (3)
   call s2(shortarr)
diff --git a/flang/test/Semantics/call34.f90 b/flang/test/Semantics/call34.f90
index 325a267309d4..a21536c6935f 100644
--- a/flang/test/Semantics/call34.f90
+++ b/flang/test/Semantics/call34.f90
@@ -11,11 +11,11 @@ program test
   real, target :: a(1)
   real :: b(1)
   call foo(a) ! ok
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call must not be used afterwards, as 'b' is not a target
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call must not be used afterwards, as 'b' is not a target [-Wnon-target-passed-to-target]
   call foo(b)
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of '(a)' afterwards
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of '(a)' afterwards [-Wnon-target-passed-to-target]
   call foo((a))
-  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of 'a([INTEGER(8)::1_8])' afterwards
+  !WARNING: Any pointer associated with TARGET dummy argument 'a=' during this call will not be associated with the value of 'a([INTEGER(8)::1_8])' afterwards [-Wnon-target-passed-to-target]
   call foo(a([1]))
   !ERROR: Scalar actual argument may not be associated with assumed-shape dummy argument 'a='
   call foo(a(1))
diff --git a/flang/test/Semantics/call35.f90 b/flang/test/Semantics/call35.f90
index ff819481226d..d500a3aef127 100644
--- a/flang/test/Semantics/call35.f90
+++ b/flang/test/Semantics/call35.f90
@@ -5,14 +5,14 @@ subroutine s1
 end
 
 subroutine s2
-  !WARNING: Reference to the procedure 'ext' has an implicit interface that is distinct from another reference: distinct numbers of dummy arguments
+  !WARNING: Reference to the procedure 'ext' has an implicit interface that is distinct from another reference: distinct numbers of dummy arguments [-Wincompatible-implicit-interfaces]
   call ext(1.)
   call myerror('abcd') ! don't warn about distinct lengths
 end
 
 subroutine s3
   interface
-    !WARNING: The global subprogram 'ext' is not compatible with its local procedure declaration (incompatible procedure attributes: ImplicitInterface)
+    !WARNING: The global subprogram 'ext' is not compatible with its local procedure declaration (incompatible procedure attributes: ImplicitInterface) [-Wexternal-interface-mismatch]
     subroutine ext(n)
       integer n
     end
diff --git a/flang/test/Semantics/call36.f90 b/flang/test/Semantics/call36.f90
index 779aafdd2340..387758408a84 100644
--- a/flang/test/Semantics/call36.f90
+++ b/flang/test/Semantics/call36.f90
@@ -15,7 +15,7 @@ subroutine test
     call intentInUnlimited(scalar)
     !ERROR: Actual argument associated with POINTER dummy argument 'x=' must also be POINTER unless INTENT(IN)
     call intentInOutUnlimited(scalar)
-    !PORTABILITY: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both should be so
+    !PORTABILITY: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both should be so [-Wrelaxed-intent-in-checking]
     call intentInUnlimited(arrayptr)
     !ERROR: If a POINTER or ALLOCATABLE dummy or actual argument is unlimited polymorphic, both must be so
     call intentInOutUnlimited(arrayptr)
diff --git a/flang/test/Semantics/call37.f90 b/flang/test/Semantics/call37.f90
index 6018b8697ad5..a536f62a1a7a 100644
--- a/flang/test/Semantics/call37.f90
+++ b/flang/test/Semantics/call37.f90
@@ -28,15 +28,15 @@ subroutine exprLen(s)
 
 module m1
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: CHARACTER(KIND=1,LEN=1_8) vs CHARACTER(KIND=1,LEN=2_8))
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: CHARACTER(KIND=1,LEN=1_8) vs CHARACTER(KIND=1,LEN=2_8)) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       character(len=2) s
     end
-    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine assumedLen(s)
       character(len=2) s
     end
-    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments)
+    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments) [-Wexternal-interface-mismatch]
     subroutine exprLen(s)
       character(len=2) s
     end
@@ -45,11 +45,11 @@ subroutine exprLen(s)
 
 module m2
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       character(len=*) s
     end
-    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'exprlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine exprLen(s)
       character(len=*) s
     end
@@ -58,12 +58,12 @@ subroutine exprLen(s)
 
 module m3
   interface
-    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments)
+    !WARNING: The global subprogram 'constlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: constant-length vs non-constant-length character dummy arguments) [-Wexternal-interface-mismatch]
     subroutine constLen(s)
       common n
       character(len=n) s
     end
-    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character)
+    !WARNING: The global subprogram 'assumedlen' is not compatible with its local procedure declaration (incompatible dummy argument #1: assumed-length character vs explicit-length character) [-Wexternal-interface-mismatch]
     subroutine assumedLen(s)
       common n
       character(len=n) s
diff --git a/flang/test/Semantics/call38.f90 b/flang/test/Semantics/call38.f90
index b1a35973e35f..4a0069977956 100644
--- a/flang/test/Semantics/call38.f90
+++ b/flang/test/Semantics/call38.f90
@@ -516,9 +516,9 @@ subroutine test
     !ERROR: Actual argument associated with coarray dummy argument 'a=' must be a coarray
     call coarray0(matrix11(1,1))
 
-    !WARNING: Actual argument variable length '1' is less than expected length '2'
+    !WARNING: Actual argument variable length '1' is less than expected length '2' [-Wshort-character-actual]
     call scalar(scalar0(1:1))
-    !WARNING: Actual argument expression length '1' is less than expected length '2'
+    !WARNING: Actual argument expression length '1' is less than expected length '2' [-Wshort-character-actual]
     call scalar('a')
   end
 end
diff --git a/flang/test/Semantics/call41.f90 b/flang/test/Semantics/call41.f90
index a4c7514d99ba..c26ef2899576 100644
--- a/flang/test/Semantics/call41.f90
+++ b/flang/test/Semantics/call41.f90
@@ -5,7 +5,7 @@ subroutine unlimited(x)
     class(*), intent(in) :: x
   end
   subroutine test
-    !PORTABILITY: passing Hollerith to unlimited polymorphic as if it were CHARACTER
+    !PORTABILITY: passing Hollerith to unlimited polymorphic as if it were CHARACTER [-Whollerith-polymorphic]
     call unlimited(6HHERMAN)
     call unlimited('abc') ! ok
   end
diff --git a/flang/test/Semantics/call42.f90 b/flang/test/Semantics/call42.f90
index 2d5303b58cb0..c264f8e842c5 100644
--- a/flang/test/Semantics/call42.f90
+++ b/flang/test/Semantics/call42.f90
@@ -22,17 +22,17 @@ recursive subroutine typeOutAssumedRank(a,b,c,d)
     type(hasAlloc), intent(out) :: b(..)
     type(hasInit), intent(out) :: c(..)
     type(hasFinal), intent(out) :: d(..)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call typeOutAssumedRank(a, b, c, d)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call classOutAssumedRank(a, b, c, d)
-    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-rank actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-rank actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
@@ -95,17 +95,17 @@ subroutine typeAssumedSize(a,b,c,d)
     type(hasAlloc) b(*)
     type(hasInit) c(*)
     type(hasFinal) d(*)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call typeOutAssumedRank(a,b,c,d)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     call classOutAssumedRank(a,b,c,d)
-    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument
+    !PORTABILITY: Assumed-size actual argument should not be associated with INTENT(OUT) assumed-rank dummy argument [-Wportability]
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
     !ERROR: Assumed-size actual argument may not be associated with INTENT(OUT) assumed-rank dummy argument requiring finalization, destruction, or initialization
diff --git a/flang/test/Semantics/call43.f90 b/flang/test/Semantics/call43.f90
index d8cc543a4838..938ecb9a2a7d 100644
--- a/flang/test/Semantics/call43.f90
+++ b/flang/test/Semantics/call43.f90
@@ -5,11 +5,11 @@ subroutine from(a, b, c, d)
   call to(a)
   call to(a(1)) ! ok
   call to(b) ! ok, passed via temp
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(b(1))
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(c)
-  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes
+  !WARNING: Reference to the procedure 'to' has an implicit interface that is distinct from another reference: incompatible dummy argument #1: incompatible dummy data object shapes [-Wincompatible-implicit-interfaces]
   call to(1.)
   call to([1., 2.]) ! ok
   call to(d) ! ok
diff --git a/flang/test/Semantics/call44.f90 b/flang/test/Semantics/call44.f90
index f7c4c9093b43..6e52aa9de55f 100644
--- a/flang/test/Semantics/call44.f90
+++ b/flang/test/Semantics/call44.f90
@@ -2,10 +2,10 @@
 subroutine assumedshape(normal, contig)
   real normal(:)
   real, contiguous :: contig(:)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Element of assumed-shape array may not be associated with a dummy argument 'assumedsize=' array
   call seqAssociate(normal(1))
-  !PORTABILITY: Element of contiguous assumed-shape array is accepted for storage sequence association
+  !PORTABILITY: Element of contiguous assumed-shape array is accepted for storage sequence association [-Wcontiguous-ok-for-seq-association]
   call seqAssociate(contig(1))
 end
 subroutine seqAssociate(assumedSize)
diff --git a/flang/test/Semantics/case01.f90 b/flang/test/Semantics/case01.f90
index 15bc30498ab5..c9631d299e49 100644
--- a/flang/test/Semantics/case01.f90
+++ b/flang/test/Semantics/case01.f90
@@ -129,7 +129,7 @@ program selectCaseProg
   end select
 
   select case (grade2)
-     !WARNING: CASE has lower bound greater than upper bound
+     !WARNING: CASE has lower bound greater than upper bound [-Wempty-case]
      case (51:50)
      case (100:)
      case (:30)
@@ -183,13 +183,13 @@ subroutine test_overflow
   integer :: j
   select case(1_1)
   case (127)
-  !WARNING: CASE value (128_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (128_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (128)
-  !WARNING: CASE value (129_4) overflows type (INTEGER(1)) of SELECT CASE expression
-  !WARNING: CASE value (130_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (129_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
+  !WARNING: CASE value (130_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (129:130)
-  !WARNING: CASE value (-130_4) overflows type (INTEGER(1)) of SELECT CASE expression
-  !WARNING: CASE value (-129_4) overflows type (INTEGER(1)) of SELECT CASE expression
+  !WARNING: CASE value (-130_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
+  !WARNING: CASE value (-129_4) overflows type (INTEGER(1)) of SELECT CASE expression [-Wcase-overflow]
   case (-130:-129)
   case (-128)
   !ERROR: Must be a scalar value, but is a rank-1 array
diff --git a/flang/test/Semantics/common-blocks.f90 b/flang/test/Semantics/common-blocks.f90
index 65f17f6d3fe9..816a9039dd49 100644
--- a/flang/test/Semantics/common-blocks.f90
+++ b/flang/test/Semantics/common-blocks.f90
@@ -7,7 +7,7 @@ subroutine init_1
   common x, y
   common /a/ xa, ya
   common /b/ xb, yb
-  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard
+  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard [-Wdata-stmt-extensions]
   data x /42./, xa /42./, yb/42./
 end subroutine
 
@@ -18,7 +18,7 @@ subroutine init_conflict
   common /a/ xa, ya
   common /b/ xb, yb
   equivalence (yb, yb_eq)
-  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard
+  !WARNING: Blank COMMON object 'x' in a DATA statement is not standard [-Wdata-stmt-extensions]
   !ERROR: Multiple initialization of COMMON block /b/
   data x /66./, xa /66./, yb_eq /66./
 end subroutine
diff --git a/flang/test/Semantics/contiguous01.f90 b/flang/test/Semantics/contiguous01.f90
index 89382775261b..eca5b4c69a60 100644
--- a/flang/test/Semantics/contiguous01.f90
+++ b/flang/test/Semantics/contiguous01.f90
@@ -5,30 +5,30 @@ module m0
 end
 module m
   use m0
-  !WARNING: Use-associated 'p1' already has 'CONTIGUOUS' attribute
+  !WARNING: Use-associated 'p1' already has 'CONTIGUOUS' attribute [-Wredundant-attribute]
   contiguous p1
   !ERROR: Cannot change CONTIGUOUS attribute on use-associated 'p2'
   contiguous p2
-  !PORTABILITY: CONTIGUOUS entity 'x' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'x' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous :: x
-  !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'scalar' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous, pointer :: scalar
-  !PORTABILITY: CONTIGUOUS entity 'allocatable' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'allocatable' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   real, contiguous, allocatable :: allocatable
  contains
-  !PORTABILITY: CONTIGUOUS entity 'func' should be an array pointer, assumed-shape, or assumed-rank
+  !PORTABILITY: CONTIGUOUS entity 'func' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
   function func(ashape,arank) result(r)
     real, contiguous :: ashape(:) ! ok
     real, contiguous :: arank(..) ! ok
-    !PORTABILITY: CONTIGUOUS entity 'r' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'r' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real :: r(10)
-    !PORTABILITY: CONTIGUOUS entity 'r2' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'r2' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     real :: r2(10)
     contiguous func
     contiguous r
     contiguous e
     contiguous r2
-    !PORTABILITY: CONTIGUOUS entity 'e' should be an array pointer, assumed-shape, or assumed-rank
+    !PORTABILITY: CONTIGUOUS entity 'e' should be an array pointer, assumed-shape, or assumed-rank [-Wredundant-contiguous]
     entry e() result(r2)
     r2 = 0
   end
diff --git a/flang/test/Semantics/cuf01.cuf b/flang/test/Semantics/cuf01.cuf
index 574d345eae04..fc405fcd3e36 100644
--- a/flang/test/Semantics/cuf01.cuf
+++ b/flang/test/Semantics/cuf01.cuf
@@ -2,15 +2,15 @@
 ! Test conflicting CUDA subprogram attributes
 module m1
  contains
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host,host) subroutine ok1; end
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host) attributes(host) subroutine ok2; end
   attributes(host,device) subroutine ok3; end
   attributes(device,host) subroutine ok4; end
-  !WARNING: ATTRIBUTES(Host) appears more than once
+  !WARNING: ATTRIBUTES(Host) appears more than once [-Wredundant-attribute]
   attributes(host,device,host) subroutine ok5; end
-  !WARNING: ATTRIBUTES(Device) appears more than once
+  !WARNING: ATTRIBUTES(Device) appears more than once [-Wredundant-attribute]
   attributes(device,host,device) subroutine ok6; end
   !ERROR: ATTRIBUTES(Global) conflicts with earlier ATTRIBUTES(Host)
   attributes(host,global) subroutine conflict1; end
diff --git a/flang/test/Semantics/cuf03.cuf b/flang/test/Semantics/cuf03.cuf
index fe9dd5b3ecf0..d2a189f41929 100644
--- a/flang/test/Semantics/cuf03.cuf
+++ b/flang/test/Semantics/cuf03.cuf
@@ -38,14 +38,14 @@ module m
   !ERROR: Object 'mmp' with ATTRIBUTES(MANAGED) must also be allocatable, automatic, explicit shape, or a dummy argument
   real, managed, pointer :: mmp(:)
   real, managed, target :: mmt
-  !WARNING: Object 'mp' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mp' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned :: mp
-  !WARNING: Object 'mpi' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mpi' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned :: mpi = 1.
   real, pinned, allocatable :: mpl ! ok
-  !ERROR: Object 'mpp' with ATTRIBUTES(PINNED) may not be a pointer
+  !ERROR: Object 'mpp' with ATTRIBUTES(PINNED) may not be a pointer [-Wcuda-usage]
   real, pinned, pointer :: mpp
-  !WARNING: Object 'mpt' with ATTRIBUTES(PINNED) should also be allocatable
+  !WARNING: Object 'mpt' with ATTRIBUTES(PINNED) should also be allocatable [-Wcuda-usage]
   real, pinned, target :: mpt ! ok
   !ERROR: ATTRIBUTES(TEXTURE) is obsolete and no longer supported
   real, texture, pointer :: mt
@@ -67,7 +67,7 @@ module m
     integer, intent(in) :: n
     real, device :: da(*) ! ok
     real, managed :: ma(n) ! ok
-    !WARNING: Pointer 'dp' may not be associated in a device subprogram
+    !WARNING: Pointer 'dp' may not be associated in a device subprogram [-Wcuda-usage]
     real, device, pointer :: dp
     real, constant :: rc ! ok
     real, shared :: rs ! ok
diff --git a/flang/test/Semantics/cuf04.cuf b/flang/test/Semantics/cuf04.cuf
index 2e2faa90b490..3c9ee7388ab2 100644
--- a/flang/test/Semantics/cuf04.cuf
+++ b/flang/test/Semantics/cuf04.cuf
@@ -4,9 +4,9 @@ module m
  contains
   attributes(device) subroutine devsubr(n)
     integer, intent(in) :: n
-    !WARNING: 'x1' should not have the SAVE attribute or initialization in a device subprogram
+    !WARNING: 'x1' should not have the SAVE attribute or initialization in a device subprogram [-Wcuda-usage]
     real, save :: x1
-    !WARNING: 'x2' should not have the SAVE attribute or initialization in a device subprogram
+    !WARNING: 'x2' should not have the SAVE attribute or initialization in a device subprogram [-Wcuda-usage]
     real :: x2 = 1.
     !ERROR: Device subprogram 'devsubr' cannot call itself
     if (n > 0) call devsubr(n-1)
diff --git a/flang/test/Semantics/cuf09.cuf b/flang/test/Semantics/cuf09.cuf
index 1e23819f9afe..9178b0a63adb 100644
--- a/flang/test/Semantics/cuf09.cuf
+++ b/flang/test/Semantics/cuf09.cuf
@@ -15,7 +15,7 @@ module m
     print*,'from device'
     print '(f10.5)', (x(ivar), ivar = 1, 10)
     write(*,*), "Hello world from device!"
-    !WARNING: I/O statement might not be supported on device
+    !WARNING: I/O statement might not be supported on device [-Wcuda-usage]
     write(12,'(10F4.1)'), x
   end
   attributes(global) subroutine devsub3(n)
diff --git a/flang/test/Semantics/data06.f90 b/flang/test/Semantics/data06.f90
index 08b470002843..7ca514ec923f 100644
--- a/flang/test/Semantics/data06.f90
+++ b/flang/test/Semantics/data06.f90
@@ -41,7 +41,7 @@ real function rfunc(x)
   data rp/rfunc/
   procedure(rfunc), pointer :: rpp
   real, target :: rt
-  !WARNING: Procedure pointer 'rpp' in a DATA statement is not standard
+  !WARNING: Procedure pointer 'rpp' in a DATA statement is not standard [-Wdata-stmt-extensions]
   !ERROR: Data object 'rt' may not be used to initialize 'rpp', which is a procedure pointer
   data rpp/rt/
   !ERROR: Initializer for 'rt' must not be a pointer
@@ -49,7 +49,7 @@ real function rfunc(x)
   !ERROR: Initializer for 'rt' must not be a procedure
   data rt/rfunc/
   integer :: jx, jy
-  !WARNING: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER
+  !WARNING: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER [-Wdata-stmt-extensions]
   data jx/'abc'/
   !ERROR: DATA statement value could not be converted to the type 'INTEGER(4)' of the object 'jx'
   data jx/t1()/
diff --git a/flang/test/Semantics/data17.f90 b/flang/test/Semantics/data17.f90
index 86f1c4c6c126..2f4e5fb02e6a 100644
--- a/flang/test/Semantics/data17.f90
+++ b/flang/test/Semantics/data17.f90
@@ -1,8 +1,8 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 character(4) a, b, c, d, e, f
-!WARNING: DATA statement value '"abcde"' for 'a' has the wrong length
+!WARNING: DATA statement value '"abcde"' for 'a' has the wrong length [-Wdata-length]
 data a(1:4)/'abcde'/
-!WARNING: DATA statement value '"abc"' for 'b' has the wrong length
+!WARNING: DATA statement value '"abc"' for 'b' has the wrong length [-Wdata-length]
 data b(1:4)/'abc'/
 data c/'abcde'/ ! not a substring, conforms
 data d/'abc'/ ! not a substring, conforms
diff --git a/flang/test/Semantics/declarations04.f90 b/flang/test/Semantics/declarations04.f90
index f0ca568e7a97..5585b08904dd 100644
--- a/flang/test/Semantics/declarations04.f90
+++ b/flang/test/Semantics/declarations04.f90
@@ -20,7 +20,7 @@ subroutine bar() bind(c,name="ext3")
 end
 
 block data ext3
-  !PORTABILITY: Global name 'ext4' conflicts with a module
+  !PORTABILITY: Global name 'ext4' conflicts with a module [-Wbenign-name-clash]
   common /ext4/ x
 end
 
diff --git a/flang/test/Semantics/declarations05.f90 b/flang/test/Semantics/declarations05.f90
index b1e3d3c77316..06191e165bce 100644
--- a/flang/test/Semantics/declarations05.f90
+++ b/flang/test/Semantics/declarations05.f90
@@ -27,7 +27,7 @@ pure subroutine test
     !ERROR: 'x1' may not be a local variable in a pure subprogram
     !BECAUSE: 'x1' has an impure FINAL procedure 'final'
     type(t1) x1
-    !WARNING: 'x1a' of derived type 't1' does not have a FINAL subroutine for its rank (1)
+    !WARNING: 'x1a' of derived type 't1' does not have a FINAL subroutine for its rank (1) [-Wfinal]
     type(t1), allocatable :: x1a(:)
     type(t1), parameter :: namedConst = t1() ! ok
     !ERROR: 'x2' may not be a local variable in a pure subprogram
diff --git a/flang/test/Semantics/declarations07.f90 b/flang/test/Semantics/declarations07.f90
index 8c95c163b043..75e5bcf22baf 100644
--- a/flang/test/Semantics/declarations07.f90
+++ b/flang/test/Semantics/declarations07.f90
@@ -11,8 +11,8 @@ pure integer function mykind(x)
   real a, b
   integer, parameter :: ak = kind(a)
   integer, parameter :: br = rank(b)
-  !WARNING: 'a' appeared earlier as a scalar actual argument to a specification function
+  !WARNING: 'a' appeared earlier as a scalar actual argument to a specification function [-Wprevious-scalar-use]
   dimension a(1)
-  !WARNING: 'b' appeared earlier as a scalar actual argument to a specification function
+  !WARNING: 'b' appeared earlier as a scalar actual argument to a specification function [-Wprevious-scalar-use]
   dimension b(1)
 end
diff --git a/flang/test/Semantics/deferred01.f90 b/flang/test/Semantics/deferred01.f90
index ce406a72b8fa..8117b91eef43 100644
--- a/flang/test/Semantics/deferred01.f90
+++ b/flang/test/Semantics/deferred01.f90
@@ -19,7 +19,7 @@ module m2
   use m1
   type, extends(absBase) :: ext
    contains
-    !WARNING: Override of PRIVATE DEFERRED 'deferredtbp' should appear in its module
+    !WARNING: Override of PRIVATE DEFERRED 'deferredtbp' should appear in its module [-Winaccessible-deferred-override]
     procedure :: deferredTbp => implTbp
   end type
  contains
diff --git a/flang/test/Semantics/definable02.f90 b/flang/test/Semantics/definable02.f90
index 666fee91a97d..559efb0dbb00 100644
--- a/flang/test/Semantics/definable02.f90
+++ b/flang/test/Semantics/definable02.f90
@@ -36,9 +36,9 @@ program test
   x1(:) = [t1()] ! ok
   x2(:) = [t2()] ! ok
   x3(:) = [t3()] ! ok
-  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f1'
+  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f1' [-Wvector-subscript-finalization]
   x1([1]) = [t1()]
-  !PORTABILITY: Variable 'x2([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f2'
+  !PORTABILITY: Variable 'x2([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'f2' [-Wvector-subscript-finalization]
   x2([1]) = [t2()]
   x3([1]) = [t3()] ! ok
 end
diff --git a/flang/test/Semantics/dim01.f90 b/flang/test/Semantics/dim01.f90
index aba9a15b95a4..c00cd39ecd09 100644
--- a/flang/test/Semantics/dim01.f90
+++ b/flang/test/Semantics/dim01.f90
@@ -16,21 +16,21 @@ function f0b(a)
   function f1(a,d)
     real, intent(in) :: a(:)
     integer, optional, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f1 = sum(a,dim=d)
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f1 = norm2(a,dim=d)
   end function
   function f2(a,d)
     real, intent(in) :: a(:)
     integer, pointer, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f2 = sum(a,dim=d)
   end function
   function f3(a,d)
     real, intent(in) :: a(:)
     integer, allocatable, intent(in) :: d
-    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time
+    !PORTABILITY: The actual argument for DIM= is optional, pointer, or allocatable, and it is assumed to be present and equal to 1 at execution time [-Woptional-must-be-present]
     f3 = sum(a,dim=d)
   end function
   function f10a(a)
@@ -49,23 +49,23 @@ function f11(a,d)
     real, intent(in) :: a(:,:)
     integer, optional, intent(in) :: d
     real, allocatable :: f11(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f11 = sum(a,dim=d)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f11 = norm2(a,dim=d)
   end function
   function f12(a,d)
     real, intent(in) :: a(:,:)
     integer, pointer, intent(in) :: d
     real, allocatable :: f12(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f12 = sum(a,dim=d)
   end function
   function f13(a,d)
     real, intent(in) :: a(:,:)
     integer, allocatable, intent(in) :: d
     real, allocatable :: f13(:)
-    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning
+    !WARNING: The actual argument for DIM= is optional, pointer, or allocatable, and may not be absent during execution; parenthesize to silence this warning [-Woptional-must-be-present]
     f13 = sum(a,dim=d)
   end function
 end module
diff --git a/flang/test/Semantics/dosemantics02.f90 b/flang/test/Semantics/dosemantics02.f90
index d6075a8a3f8f..2e7713db28ef 100644
--- a/flang/test/Semantics/dosemantics02.f90
+++ b/flang/test/Semantics/dosemantics02.f90
@@ -23,12 +23,12 @@ SUBROUTINE s1()
   INTEGER, PARAMETER :: constInt = 0
 
   ! Warn on this one for backwards compatibility
-  !WARNING: DO step expression should not be zero
+  !WARNING: DO step expression should not be zero [-Wzero-do-step]
   DO 10 I = 1, 10, 0
   10 CONTINUE
 
   ! Warn on this one for backwards compatibility
-  !WARNING: DO step expression should not be zero
+  !WARNING: DO step expression should not be zero [-Wzero-do-step]
   DO 20 I = 1, 10, 5 - 5
   20 CONTINUE
 
diff --git a/flang/test/Semantics/dosemantics03.f90 b/flang/test/Semantics/dosemantics03.f90
index a36c86b202c4..a2ec1135d7b1 100644
--- a/flang/test/Semantics/dosemantics03.f90
+++ b/flang/test/Semantics/dosemantics03.f90
@@ -50,13 +50,13 @@ END FUNCTION ifunc
   END DO
 
 ! REAL DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO rvar = 1, 10, 3
     PRINT *, "rvar is: ", rvar
   END DO
 
 ! DOUBLE PRECISISON DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO dvar = 1, 10, 3
     PRINT *, "dvar is: ", dvar
   END DO
@@ -69,14 +69,14 @@ END FUNCTION ifunc
 
 ! Pointer to REAL DO variable
   ALLOCATE(prvar)
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO prvar = 1, 10, 3
     PRINT *, "prvar is: ", prvar
   END DO
 
 ! Pointer to DOUBLE PRECISION DO variable
   ALLOCATE(pdvar)
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO pdvar = 1, 10, 3
     PRINT *, "pdvar is: ", pdvar
   END DO
@@ -148,26 +148,26 @@ END FUNCTION ifunc
   END DO
 
 ! Shared association REAL DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO realvarshare = 1, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Shared association DOUBLE PRECISION DO variable
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO dpvarshare = 1, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Initial expressions
 ! REAL initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = rvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = dvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -178,13 +178,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = prvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION initial expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = pdvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -221,13 +221,13 @@ END FUNCTION ifunc
 
 ! Final expression
 ! REAL final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, rvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, dvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -238,13 +238,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, prvar, 3
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION final expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = pdvar, 10, 3
     PRINT *, "ivar is: ", ivar
   END DO
@@ -263,13 +263,13 @@ END FUNCTION ifunc
 
 ! Step expression
 ! REAL step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, rvar
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! DOUBLE PRECISION step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, dvar
     PRINT *, "ivar is: ", ivar
   END DO
@@ -280,13 +280,13 @@ END FUNCTION ifunc
   END DO
 
 ! Pointer to REAL step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, prvar
     PRINT *, "ivar is: ", ivar
   END DO
 
 ! Pointer to DOUBLE PRECISION step expression
-!PORTABILITY: DO controls should be INTEGER
+!PORTABILITY: DO controls should be INTEGER [-Wreal-do-controls]
   DO ivar = 1, 10, pdvar
     PRINT *, "ivar is: ", ivar
   END DO
diff --git a/flang/test/Semantics/dosemantics12.f90 b/flang/test/Semantics/dosemantics12.f90
index 1757ade4b7c8..3416142dd1b6 100644
--- a/flang/test/Semantics/dosemantics12.f90
+++ b/flang/test/Semantics/dosemantics12.f90
@@ -316,7 +316,7 @@ subroutine s9()
   ! Technically non-conformant (F'2023 19.4 p8)
   do concurrent (ivar = 1:10)
     print *, "hello"
-    !PORTABILITY: Index variable 'ivar' should not also be an index in an enclosing FORALL or DO CONCURRENT
+    !PORTABILITY: Index variable 'ivar' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
     do concurrent (ivar = 1:10)
       print *, "hello"
     end do
@@ -392,7 +392,7 @@ subroutine s12()
 
   call intentInOutSub(jvar, ivar)
   do ivar = 1,10
-    !WARNING: Possible redefinition of DO variable 'ivar'
+    !WARNING: Possible redefinition of DO variable 'ivar' [-Windex-var-redefinition]
     call intentInOutSub(jvar, ivar)
   end do
 
@@ -437,7 +437,7 @@ subroutine s13()
   end do
 
   do ivar = 1, 10
-    !WARNING: Possible redefinition of DO variable 'ivar'
+    !WARNING: Possible redefinition of DO variable 'ivar' [-Windex-var-redefinition]
     jvar = intentInOutFunc(ivar)
   end do
 
diff --git a/flang/test/Semantics/expr-errors05.f90 b/flang/test/Semantics/expr-errors05.f90
index 0328165a7921..43133c836f3b 100644
--- a/flang/test/Semantics/expr-errors05.f90
+++ b/flang/test/Semantics/expr-errors05.f90
@@ -1,16 +1,16 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
-!PORTABILITY: nonstandard usage: generalized COMPLEX constructor
-!PORTABILITY: Real part of complex constructor is not scalar
+!PORTABILITY: nonstandard usage: generalized COMPLEX constructor [-Wcomplex-constructor]
+!PORTABILITY: Real part of complex constructor is not scalar [-Wcomplex-constructor]
 complex, parameter :: z1(*) = ([1.,2.], 3.)
-!PORTABILITY: nonstandard usage: generalized COMPLEX constructor
-!PORTABILITY: Imaginary part of complex constructor is not scalar
+!PORTABILITY: nonstandard usage: generalized COMPLEX constructor [-Wcomplex-constructor]
+!PORTABILITY: Imaginary part of complex constructor is not scalar [-Wcomplex-constructor]
 complex, parameter :: z2(*) = (4., [5.,6.])
 real, parameter :: aa(*) = [7.,8.]
-!PORTABILITY: Real part of complex literal constant is not scalar
+!PORTABILITY: Real part of complex literal constant is not scalar [-Wcomplex-constructor]
 complex, parameter :: z3(*) = (aa, 9.)
-!PORTABILITY: Imaginary part of complex literal constant is not scalar
+!PORTABILITY: Imaginary part of complex literal constant is not scalar [-Wcomplex-constructor]
 complex, parameter :: z4(*) = (10., aa)
 !We need a nonzero exit status to make test_errors.py look at messages :-(
-!WARNING: division by zero
+!WARNING: division by zero [-Wfolding-exception]
 real, parameter :: xxx = 1./0.
 end
diff --git a/flang/test/Semantics/expr-errors06.f90 b/flang/test/Semantics/expr-errors06.f90
index 7d4283d35cdc..c36606069155 100644
--- a/flang/test/Semantics/expr-errors06.f90
+++ b/flang/test/Semantics/expr-errors06.f90
@@ -43,6 +43,6 @@ subroutine subr(da)
   print *, empty(1:0,1) ! ok
   print *, empty(:,1) ! ok
   print *, empty(i:j,k) ! ok
-  !WARNING: Empty array dimension 1 should not be subscripted as an element or non-empty array section
+  !WARNING: Empty array dimension 1 should not be subscripted as an element or non-empty array section [-Wsubscripted-empty-array]
   print *, empty(i,1)
 end
diff --git a/flang/test/Semantics/final03.f90 b/flang/test/Semantics/final03.f90
index 3c402540152b..15335b2eeadc 100644
--- a/flang/test/Semantics/final03.f90
+++ b/flang/test/Semantics/final03.f90
@@ -20,7 +20,7 @@ program test
   type(pdt(1)) x1(1)
   type(pdt(2)) x2(1)
   type(pdt(3)) x3(1)
-  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'finalarr'
+  !PORTABILITY: Variable 'x1([INTEGER(8)::1_8])' has a vector subscript and will be finalized by non-elemental subroutine 'finalarr' [-Wvector-subscript-finalization]
   x1([1]) = pdt(1)()
   x2([1]) = pdt(2)() ! ok, doesn't match either
   x3([1]) = pdt(3)() ! ok, calls finalElem
diff --git a/flang/test/Semantics/forall01.f90 b/flang/test/Semantics/forall01.f90
index 72ad9ecd3947..d95328709ef4 100644
--- a/flang/test/Semantics/forall01.f90
+++ b/flang/test/Semantics/forall01.f90
@@ -10,7 +10,7 @@ subroutine forall1
     a(i) = i
   end forall
   forall (j=1:8)
-    !PORTABILITY: Index variable 'j' should not also be an index in an enclosing FORALL or DO CONCURRENT
+    !PORTABILITY: Index variable 'j' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
     forall (j=1:9)
     end forall
   end forall
@@ -41,7 +41,7 @@ subroutine forall3
   forall(i=1:10)
     forall(j=1:10)
       !ERROR: Cannot redefine FORALL variable 'i'
-      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment [-Wunused-forall-index]
       i = 1
     end forall
   end forall
@@ -80,20 +80,20 @@ subroutine forall5
     x(i) = y(i)
   end forall
   forall(i=1:10)
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     x = y
     forall(j=1:10)
-      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'j' not used on left-hand side of assignment [-Wunused-forall-index]
       x(i) = y(i)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       x(j) = y(j)
     endforall
   endforall
   do concurrent(i=1:10)
     x = y
     !Odd rule from F'2023 19.4 p8
-    !PORTABILITY: Index variable 'i' should not also be an index in an enclosing FORALL or DO CONCURRENT
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !PORTABILITY: Index variable 'i' should not also be an index in an enclosing FORALL or DO CONCURRENT [-Wodd-index-variable-restrictions]
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     forall(i=1:10) x = y
   end do
 end
@@ -106,7 +106,7 @@ subroutine forall6
   real, target :: b(10)
   forall(i=1:10)
     a(i)%p => b(i)
-    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+    !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
     a(1)%p => b(i)
   end forall
 end
@@ -116,20 +116,20 @@ subroutine forall7(x)
   real :: a(10)
   class(*) :: x
   associate (j => iarr(1))
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
   end associate
   associate (j => iarr(1) + 1)
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
   end associate
   select type (j => x)
   type is (integer)
-    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'j' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     forall (j=1:size(a))
       a(j) = a(j) + 1
     end forall
diff --git a/flang/test/Semantics/forall02.f90 b/flang/test/Semantics/forall02.f90
index c4f4311a175a..17e61769dc37 100644
--- a/flang/test/Semantics/forall02.f90
+++ b/flang/test/Semantics/forall02.f90
@@ -35,7 +35,7 @@ subroutine s1()
 
     ! Error to invoke an IMPURE FINAL procedure in a FORALL
     forall (i = 1:10)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresub' is referenced by finalization in a FORALL
       ifvar = ifvar1
     end forall
@@ -46,7 +46,7 @@ subroutine s1()
     end forall
 
     forall (i = 1:5)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresubrank1' is referenced by finalization in a FORALL
       ifArr1 = if0
     end forall
@@ -57,7 +57,7 @@ subroutine s1()
     end forall
 
     forall (i = 1:5)
-      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment
+      !WARNING: FORALL index variable 'i' not used on left-hand side of assignment [-Wunused-forall-index]
       !ERROR: Impure procedure 'impuresubrank2' is referenced by finalization in a FORALL
       ifArr2(:,:) = if0
     end forall
diff --git a/flang/test/Semantics/function-result-extent-max.f90 b/flang/test/Semantics/function-result-extent-max.f90
new file mode 100644
index 000000000000..932f3ef0b3ce
--- /dev/null
+++ b/flang/test/Semantics/function-result-extent-max.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -cpp -DFILE1 -DFILE2 %s | FileCheck %s --allow-empty
+!RUN: %flang_fc1 -cpp -DFILE1 %s | FileCheck %s --allow-empty && %flang_fc1 -cpp -DFILE2 %s | FileCheck %s --allow-empty
+
+!CHECK-NOT: error
+!CHECK-NOT: warning
+
+#ifdef FILE1
+module function_with_max_result_extent_module
+    implicit none
+    public :: function_with_max_result_extent
+  interface function_with_max_result_extent
+    pure module function function_with_max_result_extent(n) result(res)
+      integer, intent(in) :: n
+      integer :: res(max(n, 0))
+    end function function_with_max_result_extent
+  end interface function_with_max_result_extent
+end module function_with_max_result_extent_module
+#endif
+
+#ifdef FILE2
+submodule (function_with_max_result_extent_module) function_with_max_result_extent_submodule
+  implicit none
+contains
+  pure module function function_with_max_result_extent(n) result(res)
+      integer, intent(in) :: n
+      integer :: res(max(n, 0))
+      res(:) = 0
+    end function function_with_max_result_extent
+end submodule function_with_max_result_extent_submodule
+#endif
diff --git a/flang/test/Semantics/generic03.f90 b/flang/test/Semantics/generic03.f90
index 829780f52174..c5a2156c365d 100644
--- a/flang/test/Semantics/generic03.f90
+++ b/flang/test/Semantics/generic03.f90
@@ -15,7 +15,7 @@ integer function f1(x, j)
 
 program test
   use m1
-  !WARNING: Generic interface 'g1' has both a function and a subroutine
+  !WARNING: Generic interface 'g1' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface g1
     subroutine s1(x, a)
       import t1
diff --git a/flang/test/Semantics/generic06.f90 b/flang/test/Semantics/generic06.f90
index b47d442f3549..30dc05fc09f9 100644
--- a/flang/test/Semantics/generic06.f90
+++ b/flang/test/Semantics/generic06.f90
@@ -1,11 +1,11 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 module m
-  !PORTABILITY: Specific procedure 'sin' of generic interface 'yintercept' should not be INTRINSIC
+  !PORTABILITY: Specific procedure 'sin' of generic interface 'yintercept' should not be INTRINSIC [-Wintrinsic-as-specific]
   intrinsic sin
   interface yIntercept
     procedure sin
   end interface
-  !PORTABILITY: Specific procedure 'cos' of generic interface 'xintercept' should not be INTRINSIC
+  !PORTABILITY: Specific procedure 'cos' of generic interface 'xintercept' should not be INTRINSIC [-Wintrinsic-as-specific]
   intrinsic cos
   generic :: xIntercept => cos
 end module
diff --git a/flang/test/Semantics/global01.f90 b/flang/test/Semantics/global01.f90
index ab3edf9ef805..ee7b170f577b 100644
--- a/flang/test/Semantics/global01.f90
+++ b/flang/test/Semantics/global01.f90
@@ -30,7 +30,7 @@ recursive function global6()
 
 program test
   interface
-    !WARNING: The global subprogram 'global1' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: INTEGER(4) vs REAL(4))
+    !WARNING: The global subprogram 'global1' is not compatible with its local procedure declaration (incompatible dummy argument #1: incompatible dummy data object types: INTEGER(4) vs REAL(4)) [-Wexternal-interface-mismatch]
     subroutine global1(x)
       real, intent(in) :: x
     end subroutine
@@ -43,7 +43,7 @@ subroutine global3(x) bind(c,name="abc")
     subroutine global4(x) ! not PURE, but that's ok
       integer, intent(in) :: x
     end subroutine
-    !WARNING: The global subprogram 'global5' is not compatible with its local procedure declaration (incompatible procedure attributes: Pure)
+    !WARNING: The global subprogram 'global5' is not compatible with its local procedure declaration (incompatible procedure attributes: Pure) [-Wexternal-interface-mismatch]
     pure subroutine global5(x)
       integer, intent(in) :: x
     end subroutine
diff --git a/flang/test/Semantics/ichar01.f90 b/flang/test/Semantics/ichar01.f90
index 5eb25a97e6d9..d3609652165b 100644
--- a/flang/test/Semantics/ichar01.f90
+++ b/flang/test/Semantics/ichar01.f90
@@ -5,9 +5,9 @@
 print *, iachar('')
 print *, ichar('a')
 print *, iachar('a')
-!PORTABILITY: Character in intrinsic function ichar should have length one
+!PORTABILITY: Character in intrinsic function ichar should have length one [-Wportability]
 print *, ichar('ab')
-!PORTABILITY: Character in intrinsic function iachar should have length one
+!PORTABILITY: Character in intrinsic function iachar should have length one [-Wportability]
 print *, iachar('ab')
 end
 
diff --git a/flang/test/Semantics/ignore_tkr01.f90 b/flang/test/Semantics/ignore_tkr01.f90
index 2af4974b1c03..d069c97331bf 100644
--- a/flang/test/Semantics/ignore_tkr01.f90
+++ b/flang/test/Semantics/ignore_tkr01.f90
@@ -53,13 +53,13 @@ subroutine t8(x)
 
     subroutine t9(x)
 !dir$ ignore_tkr x
-!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer
+!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer [-Wignore-tkr-usage]
       real, intent(in), allocatable :: x
     end
 
     subroutine t10(x)
 !dir$ ignore_tkr x
-!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer
+!WARNING: !DIR$ IGNORE_TKR should not apply to an allocatable or pointer [-Wignore-tkr-usage]
       real, intent(in), pointer :: x
     end
 
@@ -88,7 +88,7 @@ elemental subroutine t13(x)
 
     subroutine t14(x)
 !dir$ ignore_tkr(r) x
-!WARNING: !DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor
+!WARNING: !DIR$ IGNORE_TKR(R) should not apply to a dummy argument passed via descriptor [-Wignore-tkr-usage]
       real x(:)
     end
 
@@ -145,7 +145,7 @@ subroutine t20(x)
 
   subroutine t22(x)
 !dir$ ignore_tkr(r) x
-!WARNING: !DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array
+!WARNING: !DIR$ IGNORE_TKR(R) is not meaningful for an assumed-rank array [-Wignore-tkr-usage]
     real x(..)
   end
 
@@ -198,7 +198,7 @@ program test
   !ERROR: Actual argument type 'INTEGER(4)' is not compatible with dummy argument type 'REAL(4)'
   call t3(1)
   call t3(dx)
-  !ERROR: passing Hollerith or character literal as if it were BOZ
+  !ERROR: passing Hollerith or character literal as if it were BOZ [-Whollerith-or-character-as-boz]
   call t3('a')
   !ERROR: Actual argument type 'COMPLEX(4)' is not compatible with dummy argument type 'REAL(4)'
   call t3((1.,2.))
diff --git a/flang/test/Semantics/int-literals.f90 b/flang/test/Semantics/int-literals.f90
index 6d4a63e4bba7..7a386e0103b4 100644
--- a/flang/test/Semantics/int-literals.f90
+++ b/flang/test/Semantics/int-literals.f90
@@ -30,9 +30,9 @@
 complex, parameter :: okz4a = (+2147483648_8, 0)
 
 complex, parameter :: okj4d = 2147483647, okz4d = (+2147483647, -2147483648)
-!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8)
+!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8) [-Wbig-int-literals]
 complex, parameter :: badj4dext = 2147483648
-!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8)
+!WARNING: Integer literal is too large for default INTEGER(KIND=4); assuming INTEGER(KIND=8) [-Wbig-int-literals]
 complex, parameter :: badz4dext = (+2147483648, 0)
 
 complex, parameter :: okj8 = 9223372036854775807_8, okz8 = (+9223372036854775807_8, -9223372036854775808_8)
diff --git a/flang/test/Semantics/intrinsics02.f90 b/flang/test/Semantics/intrinsics02.f90
index 0b1f7c13a156..615045d27581 100644
--- a/flang/test/Semantics/intrinsics02.f90
+++ b/flang/test/Semantics/intrinsics02.f90
@@ -17,7 +17,7 @@ pure real function mycos(x)
 end
 subroutine sameIntrinsic1
   use explicit
-  !WARNING: Use-associated 'cos' already has 'INTRINSIC' attribute
+  !WARNING: Use-associated 'cos' already has 'INTRINSIC' attribute [-Wredundant-attribute]
   intrinsic cos
   real :: one = cos(0.)
 end
diff --git a/flang/test/Semantics/kinds05b.f90 b/flang/test/Semantics/kinds05b.f90
index 3927829bbaab..3f2d4605adbd 100644
--- a/flang/test/Semantics/kinds05b.f90
+++ b/flang/test/Semantics/kinds05b.f90
@@ -8,8 +8,8 @@ subroutine s
   real :: realvar3 = 4.0_8
   real :: realvar4 = 4.0E6_4
   real :: realvar5 = 4.0E6_8
-  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
+  !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard [-Wexponent-matching-kind-param]
   real :: realvar6 = 4.0D6_8
-  !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'd'
+  !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'd' [-Wexponent-matching-kind-param]
   real :: realvar7 = 4.0D6_4
 end subroutine s
diff --git a/flang/test/Semantics/label18.f90 b/flang/test/Semantics/label18.f90
index a9f3e50237f9..152a189876b8 100644
--- a/flang/test/Semantics/label18.f90
+++ b/flang/test/Semantics/label18.f90
@@ -9,10 +9,10 @@ program main
 1 end if
   if (.true.) then
     do j = 1, 2
-      !WARNING: Label '1' is in a construct that should not be used as a branch target here
+      !WARNING: Label '1' is in a construct that should not be used as a branch target here [-Wbranch-into-construct]
       goto 1
     end do
   end if
-  !WARNING: Label '1' is in a construct that should not be used as a branch target here
+  !WARNING: Label '1' is in a construct that should not be used as a branch target here [-Wbranch-into-construct]
   goto 1
 end
diff --git a/flang/test/Semantics/local-vs-global.f90 b/flang/test/Semantics/local-vs-global.f90
index 3f7e9339639c..21a3009ad7f2 100644
--- a/flang/test/Semantics/local-vs-global.f90
+++ b/flang/test/Semantics/local-vs-global.f90
@@ -46,24 +46,24 @@ function implicit_func_before_2(a)
 
 program test
   external justfine ! OK to name a BLOCK DATA if not called
-  !WARNING: The global entity 'module_before_1' corresponding to the local procedure 'module_before_1' is not a callable subprogram
+  !WARNING: The global entity 'module_before_1' corresponding to the local procedure 'module_before_1' is not a callable subprogram [-Wexternal-name-conflict]
   external module_before_1
-  !WARNING: The global entity 'block_data_before_1' corresponding to the local procedure 'block_data_before_1' is not a callable subprogram
+  !WARNING: The global entity 'block_data_before_1' corresponding to the local procedure 'block_data_before_1' is not a callable subprogram [-Wexternal-name-conflict]
   external block_data_before_1
-  !WARNING: The global subprogram 'explicit_before_1' should not be referenced via the implicit interface 'explicit_before_1'
+  !WARNING: The global subprogram 'explicit_before_1' should not be referenced via the implicit interface 'explicit_before_1' [-Wexternal-interface-mismatch]
   external explicit_before_1
   external implicit_before_1
-  !WARNING: The global subprogram 'explicit_func_before_1' should not be referenced via the implicit interface 'explicit_func_before_1'
+  !WARNING: The global subprogram 'explicit_func_before_1' should not be referenced via the implicit interface 'explicit_func_before_1' [-Wexternal-interface-mismatch]
   external explicit_func_before_1
   external implicit_func_before_1
-  !WARNING: The global entity 'module_after_1' corresponding to the local procedure 'module_after_1' is not a callable subprogram
+  !WARNING: The global entity 'module_after_1' corresponding to the local procedure 'module_after_1' is not a callable subprogram [-Wexternal-name-conflict]
   external module_after_1
-  !WARNING: The global entity 'block_data_after_1' corresponding to the local procedure 'block_data_after_1' is not a callable subprogram
+  !WARNING: The global entity 'block_data_after_1' corresponding to the local procedure 'block_data_after_1' is not a callable subprogram [-Wexternal-name-conflict]
   external block_data_after_1
-  !WARNING: The global subprogram 'explicit_after_1' should not be referenced via the implicit interface 'explicit_after_1'
+  !WARNING: The global subprogram 'explicit_after_1' should not be referenced via the implicit interface 'explicit_after_1' [-Wexternal-interface-mismatch]
   external explicit_after_1
   external implicit_after_1
-  !WARNING: The global subprogram 'explicit_func_after_1' should not be referenced via the implicit interface 'explicit_func_after_1'
+  !WARNING: The global subprogram 'explicit_func_after_1' should not be referenced via the implicit interface 'explicit_func_after_1' [-Wexternal-interface-mismatch]
   external explicit_func_after_1
   external implicit_func_after_1
   call module_before_1
@@ -76,20 +76,20 @@ program test
   !ERROR: References to the procedure 'explicit_before_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   call explicit_before_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_before_1
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_before_2
   print *, explicit_func_before_1(1.)
   !ERROR: References to the procedure 'explicit_func_before_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   print *, explicit_func_before_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_before_1()
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_before_2()
   call module_after_1
@@ -100,20 +100,20 @@ program test
   !ERROR: References to the procedure 'explicit_after_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   call explicit_after_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_after_1
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   call implicit_after_2
   print *, explicit_func_after_1(1.)
   !ERROR: References to the procedure 'explicit_func_after_2' require an explicit interface
   !BECAUSE: a dummy argument has the allocatable, asynchronous, optional, pointer, target, value, or volatile attribute
   print *, explicit_func_after_2(1.)
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_after_1()
-  !WARNING: If the procedure's interface were explicit, this reference would be in error
+  !WARNING: If the procedure's interface were explicit, this reference would be in error [-Wknown-bad-implicit-interface]
   !BECAUSE: Dummy argument 'a=' (#1) is not OPTIONAL and is not associated with an actual argument in this procedure reference
   print *, implicit_func_after_2()
 end program
diff --git a/flang/test/Semantics/long-name.f90 b/flang/test/Semantics/long-name.f90
index c406e366f76d..44899b13edd5 100644
--- a/flang/test/Semantics/long-name.f90
+++ b/flang/test/Semantics/long-name.f90
@@ -1,14 +1,14 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror -pedantic
 
-!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63
+!PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
 program aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg1
 
-  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63
+  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
   integer :: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg2
 
   integer :: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg
 
-  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3 has length 64, which is greater than the maximum name length 63
+  !PORTABILITY: aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3 has length 64, which is greater than the maximum name length 63 [-Wlong-names]
   call aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffggg3
 
 end
diff --git a/flang/test/Semantics/modfile43.f90 b/flang/test/Semantics/modfile43.f90
index 1a50e94698f6..ed66dff27fb0 100644
--- a/flang/test/Semantics/modfile43.f90
+++ b/flang/test/Semantics/modfile43.f90
@@ -5,7 +5,7 @@ module iso_fortran_env
 end module
 module m1
   use, intrinsic :: iso_fortran_env, only: int32
-  !PORTABILITY: Should not USE the non-intrinsic module 'iso_fortran_env' in the same scope as a USE of the intrinsic module
+  !PORTABILITY: Should not USE the non-intrinsic module 'iso_fortran_env' in the same scope as a USE of the intrinsic module [-Wmisc-use-extensions]
   use, non_intrinsic :: iso_fortran_env, only: user_defined_123
 end module
 module m2
diff --git a/flang/test/Semantics/null-init.f90 b/flang/test/Semantics/null-init.f90
index ad3f91679a42..d01ad75a75a1 100644
--- a/flang/test/Semantics/null-init.f90
+++ b/flang/test/Semantics/null-init.f90
@@ -37,7 +37,7 @@ module m6
 
 module m7
   interface
-    !WARNING: The external interface 'null' is not compatible with an earlier definition (incompatible procedure attributes: ImplicitInterface)
+    !WARNING: The external interface 'null' is not compatible with an earlier definition (incompatible procedure attributes: ImplicitInterface) [-Wexternal-interface-mismatch]
     function null() result(p)
       integer, pointer :: p
     end function
diff --git a/flang/test/Semantics/null01.f90 b/flang/test/Semantics/null01.f90
index b21ee91a0be0..64c988129730 100644
--- a/flang/test/Semantics/null01.f90
+++ b/flang/test/Semantics/null01.f90
@@ -103,9 +103,9 @@ function f3()
   dt3x = dt3(pps1=null(mold=dt2x%pps0))
   dt3x = dt3(pps1=null(mold=dt3x%pps1))
   dt4x = dt4(null()) ! ok
-  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0'
+  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0' [-Wnull-mold-allocatable-component-value]
   dt4x = dt4(null(rp0))
-  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0'
+  !PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'ra0' [-Wnull-mold-allocatable-component-value]
   !ERROR: Rank-1 array value is not compatible with scalar component 'ra0'
   dt4x = dt4(null(rp1))
   !ERROR: A NULL procedure pointer may not be used as the value for component 'ra0'
@@ -133,7 +133,7 @@ function f3()
   print *, same_type_as(null(dt5p), null(dt4p)) ! ok
   !ERROR: A NULL() pointer is not allowed for 'source=' intrinsic argument
   print *, transfer(null(rp0),ip0)
-  !WARNING: Source of TRANSFER contains allocatable or pointer component %ra0
+  !WARNING: Source of TRANSFER contains allocatable or pointer component %ra0 [-Wpointer-component-transfer-arg]
   print *, transfer(dt4(null()),[0])
   !ERROR: NULL() may not be used as an expression in this context
   select case(null(ip0))
diff --git a/flang/test/Semantics/pointer01.f90 b/flang/test/Semantics/pointer01.f90
index cb860f3a3f43..eaa2426dd77e 100644
--- a/flang/test/Semantics/pointer01.f90
+++ b/flang/test/Semantics/pointer01.f90
@@ -7,7 +7,7 @@ subroutine msubr
 end module
 program main
   use m
-  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'main' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   pointer main
   !ERROR: Cannot change POINTER attribute on use-associated 'mobj'
   pointer mobj
diff --git a/flang/test/Semantics/procinterface02.f90 b/flang/test/Semantics/procinterface02.f90
index 8b1becbe081e..6aa5dc13bd9a 100644
--- a/flang/test/Semantics/procinterface02.f90
+++ b/flang/test/Semantics/procinterface02.f90
@@ -12,7 +12,7 @@ real function foo_nonelemental(x)
     end function
   end interface
   real :: A(:), B(:)
-  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sqrt), pointer :: P
   !ERROR: Rank of dummy argument is 0, but actual argument has rank 1
   A = P(B)
diff --git a/flang/test/Semantics/procinterface04.f90 b/flang/test/Semantics/procinterface04.f90
index f59e53f90dab..fb48c4b17b41 100644
--- a/flang/test/Semantics/procinterface04.f90
+++ b/flang/test/Semantics/procinterface04.f90
@@ -9,11 +9,11 @@ pure real function nonelemental(x)
       real, intent(in) :: x
     end
   end interface
-  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: A dummy procedure should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin) :: dp1
   !ERROR: A dummy procedure may not be ELEMENTAL
   procedure(elemental) :: dp2
-  !PORTABILITY: Procedure pointer 'pp1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'pp1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: pp1
   !ERROR: Procedure pointer 'pp2' may not be ELEMENTAL
   procedure(elemental), pointer :: pp2
diff --git a/flang/test/Semantics/resolve05.f90 b/flang/test/Semantics/resolve05.f90
index 4e50feb64e4c..0c9877af9b4e 100644
--- a/flang/test/Semantics/resolve05.f90
+++ b/flang/test/Semantics/resolve05.f90
@@ -1,18 +1,18 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -pedantic
 program p
-  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program
+  !PORTABILITY: Name 'p' declared in a main program should not have the same name as the main program [-Wbenign-name-clash]
   integer :: p
 end
 module m
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   integer :: m
 end
 submodule(m) sm
-  !PORTABILITY: Name 'sm' declared in a submodule should not have the same name as the submodule
+  !PORTABILITY: Name 'sm' declared in a submodule should not have the same name as the submodule [-Wbenign-name-clash]
   integer :: sm
 end
 block data bd
-  !PORTABILITY: Name 'bd' declared in a BLOCK DATA subprogram should not have the same name as the BLOCK DATA subprogram
+  !PORTABILITY: Name 'bd' declared in a BLOCK DATA subprogram should not have the same name as the BLOCK DATA subprogram [-Wbenign-name-clash]
   type bd
   end type
 end
diff --git a/flang/test/Semantics/resolve108.f90 b/flang/test/Semantics/resolve108.f90
index 644b8fba8689..4ad53da5ddeb 100644
--- a/flang/test/Semantics/resolve108.f90
+++ b/flang/test/Semantics/resolve108.f90
@@ -44,7 +44,7 @@ subroutine s2
   use :: m2, only: foo
   !If we got the type of foo right, this declaration will fail
   !due to an attempted division by zero.
-  !WARNING: INTEGER(4) division by zero
+  !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
   !ERROR: Must be a constant value
   integer, parameter :: test = 1 / (kind(foo(1)) - kind(1.d0))
 end subroutine
@@ -54,7 +54,7 @@ module m3
  contains
   real(kind=kind(x)) function foo(x)
     real(kind=kind(1.0d0)) x
-    !WARNING: INTEGER(4) division by zero
+    !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
     !ERROR: Must be a constant value
     integer, parameter :: test = 1 / (kind(foo) - kind(1.d0))
     foo = n
diff --git a/flang/test/Semantics/resolve11.f90 b/flang/test/Semantics/resolve11.f90
index 9ae4f52c4fd5..0c457f2c9f3f 100644
--- a/flang/test/Semantics/resolve11.f90
+++ b/flang/test/Semantics/resolve11.f90
@@ -4,7 +4,7 @@ module m
   integer, private :: j
   !ERROR: The accessibility of 'i' has already been specified as PUBLIC
   private i
-  !WARNING: The accessibility of 'j' has already been specified as PRIVATE
+  !WARNING: The accessibility of 'j' has already been specified as PRIVATE [-Wredundant-attribute]
   private j
 end
 
@@ -66,7 +66,7 @@ subroutine s4
   !ERROR: 'fun' is PRIVATE in 'm4'
   use m4, only: foo, fun
   type(foo) x ! ok
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'fun') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'fun') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   print *, foo()
 end
 
diff --git a/flang/test/Semantics/resolve114.f90 b/flang/test/Semantics/resolve114.f90
index 6204e5fc2f3f..05fb7aa5e479 100644
--- a/flang/test/Semantics/resolve114.f90
+++ b/flang/test/Semantics/resolve114.f90
@@ -34,9 +34,9 @@ end module m2
 subroutine s2a
   use m1
   use m2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p1 => sin
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iabs), pointer :: p2 => iabs
   procedure(ext1), pointer :: p3 => ext1
   procedure(ext2), pointer :: p4 => ext2
@@ -46,9 +46,9 @@ subroutine s2b
   use m1, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m2, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface2), pointer :: p2 => x2
   procedure(iface3), pointer :: p3 => x3
   procedure(iface4), pointer :: p4 => x4
@@ -60,9 +60,9 @@ module m3
 end module
 subroutine s3
   use m3
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p1 => sin
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iabs), pointer :: p2 => iabs
   procedure(ext1), pointer :: p3 => ext1
   procedure(ext2), pointer :: p4 => ext2
@@ -75,9 +75,9 @@ module m4
 subroutine s4
   use m4
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(iface2), pointer :: p2 => x2
   procedure(iface3), pointer :: p3 => x3
   procedure(iface4), pointer :: p4 => x4
@@ -87,10 +87,10 @@ subroutine s5
   use m1, only: x1 => sin, x2 => iabs, x3 => ext1, x4 => ext2
   use m2, only: x1 => tan, x2 => idim, x3 => ext2, x4 => ext1
   use m1, only: iface1 => sin, iface2 => iabs, iface3 => ext1, iface4 => ext2
-  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p1' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: Reference to 'x1' is ambiguous
   procedure(iface1), pointer :: p1 => x1
-  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p2' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   !ERROR: Reference to 'x2' is ambiguous
   procedure(iface2), pointer :: p2 => x2
   !ERROR: Reference to 'x3' is ambiguous
diff --git a/flang/test/Semantics/resolve118.f90 b/flang/test/Semantics/resolve118.f90
index 024b67b5a471..59199dfd6311 100644
--- a/flang/test/Semantics/resolve118.f90
+++ b/flang/test/Semantics/resolve118.f90
@@ -17,7 +17,7 @@ module m3
   interface
     subroutine s1(x)
       use m1
-      !PORTABILITY: The same 't' is already present in this scope
+      !PORTABILITY: The same 't' is already present in this scope [-Wbenign-name-clash]
       import t
       type(t) x
     end
diff --git a/flang/test/Semantics/resolve17.f90 b/flang/test/Semantics/resolve17.f90
index 6a6e355abe0b..173fb9c6c5b7 100644
--- a/flang/test/Semantics/resolve17.f90
+++ b/flang/test/Semantics/resolve17.f90
@@ -11,7 +11,7 @@ module m2
   interface s
   end interface
 contains
-  !WARNING: 's' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic
+  !WARNING: 's' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic [-Whomonymous-specific]
   subroutine s
   end subroutine
 end module
@@ -290,7 +290,7 @@ module m14d
  contains
   subroutine test
     real :: y
-    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name
+    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
     y = foo(1.0)
     x = foo(2)
   end subroutine
@@ -302,7 +302,7 @@ module m14e
  contains
   subroutine test
     real :: y
-    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name
+    !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'bar') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
     y = foo(1.0)
     x = foo(2)
   end subroutine
diff --git a/flang/test/Semantics/resolve18.f90 b/flang/test/Semantics/resolve18.f90
index 547db5e85714..65c39931cc7c 100644
--- a/flang/test/Semantics/resolve18.f90
+++ b/flang/test/Semantics/resolve18.f90
@@ -11,7 +11,7 @@ subroutine foo(x)
 module m2
   use m1
   implicit none
-  !WARNING: 'foo' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic
+  !WARNING: 'foo' should not be the name of both a generic interface and a procedure unless it is a specific procedure of the generic [-Whomonymous-specific]
   interface foo
     module procedure s
   end interface
@@ -22,14 +22,14 @@ subroutine s(i)
 end module
 
 subroutine foo
-  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name
+  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name [-Wuse-association-into-same-name-subprogram]
   use m1
   !ERROR: Reference to 'foo' is ambiguous
   call foo
 end
 
 subroutine bar
-  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name
+  !PORTABILITY: 'foo' is use-associated into a subprogram of the same name [-Wuse-association-into-same-name-subprogram]
   use m1, bar => foo
   !ERROR: Reference to 'bar' is ambiguous
   call bar
@@ -348,7 +348,7 @@ subroutine s_21_23
   use m21
   use m23
   type(foo) x ! Intel and NAG error
-  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'f1') is ambiguous with a structure constructor of the same name
+  !PORTABILITY: Reference to generic function 'foo' (resolving to specific 'f1') is ambiguous with a structure constructor of the same name [-Wambiguous-structure-constructor]
   print *, foo(1.) ! Intel error
   print *, foo(1.,2.,3.) ! Intel error
   call ext(foo) ! GNU and Intel error
diff --git a/flang/test/Semantics/resolve20.f90 b/flang/test/Semantics/resolve20.f90
index 359bc0c1f9a3..8b8d19020668 100644
--- a/flang/test/Semantics/resolve20.f90
+++ b/flang/test/Semantics/resolve20.f90
@@ -38,7 +38,7 @@ subroutine forward
   type :: bad3
   end type
 
-  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module
+  !PORTABILITY: Name 'm' declared in a module should not have the same name as the module [-Wbenign-name-clash]
   type :: m
   end type m
 
@@ -49,7 +49,7 @@ subroutine forward
   external :: a, b, c, d
   !ERROR: EXTERNAL attribute not allowed on 'm'
   external :: m
-  !WARNING: EXTERNAL attribute was already specified on 'foo'
+  !WARNING: EXTERNAL attribute was already specified on 'foo' [-Wredundant-attribute]
   external :: foo
   !ERROR: EXTERNAL attribute not allowed on 'bar'
   external :: bar
diff --git a/flang/test/Semantics/resolve24.f90 b/flang/test/Semantics/resolve24.f90
index 72d6719665bb..664fe0faeedc 100644
--- a/flang/test/Semantics/resolve24.f90
+++ b/flang/test/Semantics/resolve24.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 subroutine test1
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     subroutine s1(x)
     end subroutine
@@ -12,7 +12,7 @@ function f()
 end subroutine
 
 subroutine test2
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     function t2f1(x)
     end function
@@ -24,7 +24,7 @@ function t2f2(x, y)
 end subroutine
 
 module test3
-  !WARNING: Generic interface 'foo' has both a function and a subroutine
+  !WARNING: Generic interface 'foo' has both a function and a subroutine [-Wsubroutine-and-function-specifics]
   interface foo
     module procedure s
     module procedure f
@@ -39,7 +39,7 @@ function f()
 subroutine test4
   type foo
   end type
-  !WARNING: Generic interface 'foo' should only contain functions due to derived type with same name
+  !WARNING: Generic interface 'foo' should only contain functions due to derived type with same name [-Wsubroutine-and-function-specifics]
   interface foo
     subroutine s()
     end subroutine
diff --git a/flang/test/Semantics/resolve30.f90 b/flang/test/Semantics/resolve30.f90
index 32108e89cdef..28613dfe4ffd 100644
--- a/flang/test/Semantics/resolve30.f90
+++ b/flang/test/Semantics/resolve30.f90
@@ -41,6 +41,6 @@ subroutine s4
 subroutine s5
   implicit none
   data x/1./
-  !PORTABILITY: 'x' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE)
+  !PORTABILITY: 'x' appeared in a DATA statement before its type was declared under IMPLICIT NONE(TYPE) [-Wforward-ref-implicit-none-data]
   real x
 end
diff --git a/flang/test/Semantics/resolve31.f90 b/flang/test/Semantics/resolve31.f90
index 6bf8e877a515..eabd87776904 100644
--- a/flang/test/Semantics/resolve31.f90
+++ b/flang/test/Semantics/resolve31.f90
@@ -49,9 +49,9 @@ module m4
   type :: t1
     private
     sequence
-    !WARNING: PRIVATE should not appear more than once in derived type components
+    !WARNING: PRIVATE should not appear more than once in derived type components [-Wredundant-attribute]
     private
-    !WARNING: SEQUENCE should not appear more than once in derived type components
+    !WARNING: SEQUENCE should not appear more than once in derived type components [-Wredundant-attribute]
     sequence
     real :: t1Field
   end type
@@ -68,7 +68,7 @@ module m4
   !ERROR: A sequence type may not have a CONTAINS statement
   contains
   end type
-  !WARNING: A sequence type should have at least one component
+  !WARNING: A sequence type should have at least one component [-Wempty-sequence-type]
   type :: emptyType
     sequence
   end type emptyType
@@ -85,7 +85,7 @@ module m4
     class(*), allocatable :: typeStarField
     !ERROR: A sequence type data component must either be of an intrinsic type or a derived sequence type
     type(plainType) :: testField1
-    !WARNING: A sequence type data component that is a pointer to a non-sequence type is not standard
+    !WARNING: A sequence type data component that is a pointer to a non-sequence type is not standard [-Wpointer-in-seq-type]
     type(plainType), pointer :: testField1p
     type(sequenceType) :: testField2
     procedure(real), pointer, nopass :: procField
diff --git a/flang/test/Semantics/resolve35.f90 b/flang/test/Semantics/resolve35.f90
index 2947b225978d..f3f8409161df 100644
--- a/flang/test/Semantics/resolve35.f90
+++ b/flang/test/Semantics/resolve35.f90
@@ -21,14 +21,14 @@ subroutine s3
   real :: a(10,10), b(10,10)
   type y; end type
   integer(8) :: x
-  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   forall(x=1:10, y=1:10)
     !ERROR: Must have INTEGER type, but is REAL(4)
     !ERROR: Must have INTEGER type, but is REAL(4)
     a(x, y) = b(x, y)
   end forall
-  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 'y' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
@@ -51,7 +51,7 @@ subroutine s4
     !ERROR: Must have INTEGER type, but is REAL(4)
     a(y) = b(y)
   end forall
-  !PORTABILITY: Index variable 'i' should be scalar in the enclosing scope
+  !PORTABILITY: Index variable 'i' should be scalar in the enclosing scope [-Wodd-index-variable-restrictions]
   forall(i=1:10)
     a(i) = b(i)
   end forall
@@ -61,7 +61,7 @@ subroutine s6
   integer, parameter :: n = 4
   real, dimension(n) :: x
   data(x(i), i=1, n) / n * 0.0 /
-  !PORTABILITY: Index variable 't' should be a scalar object or common block if it is present in the enclosing scope
+  !PORTABILITY: Index variable 't' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
   !ERROR: Must have INTEGER type, but is REAL(4)
   !ERROR: Must have INTEGER type, but is REAL(4)
   forall(t=1:n) x(t) = 0.0
@@ -86,7 +86,7 @@ subroutine s7
   do concurrent(integer::i=1:5) local(j, i) &
       !ERROR: 'j' is already declared in this scoping unit
       local_init(k, j) &
-      !WARNING: Variable 'a' with SHARED locality implicitly declared
+      !WARNING: Variable 'a' with SHARED locality implicitly declared [-Wimplicit-shared]
       shared(a)
     a = j + 1
   end do
diff --git a/flang/test/Semantics/resolve37.f90 b/flang/test/Semantics/resolve37.f90
index f8229f1f974a..d35174e62d64 100644
--- a/flang/test/Semantics/resolve37.f90
+++ b/flang/test/Semantics/resolve37.f90
@@ -25,25 +25,25 @@
 character(len=l) :: v
 !ERROR: Value of named constant 'o' (o) cannot be computed as a constant value
 real, parameter ::  o = o
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer, parameter ::  p = 0/0
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
-!WARNING: INTEGER(4) division by zero
-!WARNING: INTEGER(4) division by zero
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 integer, parameter ::  q = 1+2*(1/0)
 integer not_constant
 !ERROR: Must be a constant value
 integer, parameter :: s1 = not_constant/2
 !ERROR: Must be a constant value
 integer, parameter :: s2 = 3/not_constant
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer(kind=2/0) r
 integer, parameter :: sok(*)=[1,2]/[1,2]
-!WARNING: INTEGER(4) division by zero
+!WARNING: INTEGER(4) division by zero [-Wfolding-exception]
 !ERROR: Must be a constant value
 integer, parameter :: snok(*)=[1,2]/[1,0]
 end
diff --git a/flang/test/Semantics/resolve45.f90 b/flang/test/Semantics/resolve45.f90
index 7f6d8d6ae50b..18a382a788b6 100644
--- a/flang/test/Semantics/resolve45.f90
+++ b/flang/test/Semantics/resolve45.f90
@@ -36,13 +36,13 @@ subroutine s3(x)
 end
 
 subroutine s4
-  !WARNING: Explicit SAVE of 'z' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'z' is redundant due to global SAVE statement [-Wredundant-attribute]
   save z
   save
   procedure(integer), pointer :: x
-  !WARNING: Explicit SAVE of 'x' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'x' is redundant due to global SAVE statement [-Wredundant-attribute]
   save :: x
-  !WARNING: Explicit SAVE of 'y' is redundant due to global SAVE statement
+  !WARNING: Explicit SAVE of 'y' is redundant due to global SAVE statement [-Wredundant-attribute]
   integer, save :: y
 end
 
diff --git a/flang/test/Semantics/resolve46.f90 b/flang/test/Semantics/resolve46.f90
index 0acc20b19f0b..d473226edaab 100644
--- a/flang/test/Semantics/resolve46.f90
+++ b/flang/test/Semantics/resolve46.f90
@@ -20,7 +20,7 @@ logical function chrcmp(a,b)
      end function chrcmp
   end interface
 
-  !PORTABILITY: Procedure pointer 'p' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 'p' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(sin), pointer :: p => cos
   !ERROR: Intrinsic procedure 'amin0' is not an unrestricted specific intrinsic permitted for use as the definition of the interface to procedure pointer 'q'
   procedure(amin0), pointer :: q
@@ -29,7 +29,7 @@ end function chrcmp
   !ERROR: Intrinsic procedure 'llt' is not an unrestricted specific intrinsic permitted for use as the initializer for procedure pointer 's'
   procedure(chrcmp), pointer :: s => llt
   !ERROR: Intrinsic procedure 'bessel_j0' is not an unrestricted specific intrinsic permitted for use as the initializer for procedure pointer 't'
-  !PORTABILITY: Procedure pointer 't' should not have an ELEMENTAL intrinsic as its interface
+  !PORTABILITY: Procedure pointer 't' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
   procedure(cos), pointer :: t => bessel_j0
   procedure(chrcmp), pointer :: u
   p => alog ! valid use of an unrestricted specific intrinsic
diff --git a/flang/test/Semantics/resolve55.f90 b/flang/test/Semantics/resolve55.f90
index 5f7a3044e834..908fda8a83e1 100644
--- a/flang/test/Semantics/resolve55.f90
+++ b/flang/test/Semantics/resolve55.f90
@@ -94,3 +94,23 @@ subroutine s8(arg)
   do concurrent(i=1:5) local(arg)
   end do
 end subroutine s8
+
+subroutine s9()
+  type l3
+    integer, allocatable :: a
+  end type
+  type l2
+    type(l3) :: l2_3
+  end type
+  type l1
+    type(l2) :: l1_2
+  end type
+  type(l1) :: v
+  integer sum
+
+  sum = 0
+!ERROR: Derived type variable 'v' with ultimate ALLOCATABLE component '%l1_2%l2_3%a' not allowed in a LOCAL_INIT locality-spec
+  do concurrent (i = 1:10) local_init(v)
+    sum = sum + i
+  end do
+end subroutine s9
diff --git a/flang/test/Semantics/resolve58.f90 b/flang/test/Semantics/resolve58.f90
index 7686de489840..ff9d6c358c69 100644
--- a/flang/test/Semantics/resolve58.f90
+++ b/flang/test/Semantics/resolve58.f90
@@ -69,12 +69,12 @@ subroutine s6()
 
   !ERROR: Implied-shape array 'local1' must be a named constant or a dummy argument
   real, dimension (*) :: local1
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   real, intent(in) :: local2
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   procedure(), intent(in) :: p1
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   real, optional :: local3
-  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute
+  !WARNING: Only a dummy argument should have an INTENT, VALUE, or OPTIONAL attribute [-Wignore-irrelevant-attributes]
   procedure(), optional :: p2
 end subroutine
diff --git a/flang/test/Semantics/resolve59.f90 b/flang/test/Semantics/resolve59.f90
index 3bdcf67aa958..a598fbe1849f 100644
--- a/flang/test/Semantics/resolve59.f90
+++ b/flang/test/Semantics/resolve59.f90
@@ -59,10 +59,10 @@ real function rfunc(x)
     x = acos(f5)
   end function
   ! Sanity test: f18 handles C1560 violation by ignoring RESULT
-  !WARNING: The function name should not appear in RESULT; references to 'f6' inside the function will be considered as references to the result only
+  !WARNING: The function name should not appear in RESULT; references to 'f6' inside the function will be considered as references to the result only [-Whomonymous-result]
   function f6() result(f6)
   end function
-  !WARNING: The function name should not appear in RESULT; references to 'f7' inside the function will be considered as references to the result only
+  !WARNING: The function name should not appear in RESULT; references to 'f7' inside the function will be considered as references to the result only [-Whomonymous-result]
   function f7() result(f7)
     real :: x, f7
     !ERROR: Recursive call to 'f7' requires a distinct RESULT in its declaration
@@ -114,7 +114,7 @@ function f4() result(r)
   end function
   function f5(x) result(r)
     real :: x
-    !PORTABILITY: Procedure pointer 'r' should not have an ELEMENTAL intrinsic as its interface
+    !PORTABILITY: Procedure pointer 'r' should not have an ELEMENTAL intrinsic as its interface [-Wportability]
     procedure(acos), pointer :: r
     r => acos
     !ERROR: Actual argument for 'x=' may not be a procedure
diff --git a/flang/test/Semantics/resolve60.f90 b/flang/test/Semantics/resolve60.f90
index ff988d54a821..0192f830dad6 100644
--- a/flang/test/Semantics/resolve60.f90
+++ b/flang/test/Semantics/resolve60.f90
@@ -32,7 +32,7 @@
 
   enum, bind(C)
     !ERROR: Enumerator value could not be computed from the given expression
-    !WARNING: INTEGER(4) division by zero
+    !WARNING: INTEGER(4) division by zero [-Wfolding-exception]
     !ERROR: Must be a constant value
     enumerator :: wrong = 0/0
   end enum
diff --git a/flang/test/Semantics/resolve61.f90 b/flang/test/Semantics/resolve61.f90
index 2a1f584ffaf0..ffee32243af6 100644
--- a/flang/test/Semantics/resolve61.f90
+++ b/flang/test/Semantics/resolve61.f90
@@ -114,7 +114,7 @@ subroutine p12
   type(t2) :: x2
   type(t3) :: x3
   pointer(a, x1)
-  !WARNING: Type of Cray pointee 'x2' is a derived type that is neither SEQUENCE nor BIND(C)
+  !WARNING: Type of Cray pointee 'x2' is a derived type that is neither SEQUENCE nor BIND(C) [-Wnon-sequence-cray-pointee]
   pointer(b, x2)
   pointer(c, x3)
 end
diff --git a/flang/test/Semantics/resolve65.f90 b/flang/test/Semantics/resolve65.f90
index b2815c4ed1c7..1f985125a152 100644
--- a/flang/test/Semantics/resolve65.f90
+++ b/flang/test/Semantics/resolve65.f90
@@ -15,8 +15,8 @@ module m1
     procedure :: assign_t2
     procedure :: assign_t3
     !ERROR: Defined assignment subroutine 'assign_t2' must have two dummy arguments
-    !WARNING: In defined assignment subroutine 'assign_t3', second dummy argument 'y' should have INTENT(IN) or VALUE attribute
-    !WARNING: In defined assignment subroutine 'assign_t4', first dummy argument 'x' should have INTENT(OUT) or INTENT(INOUT)
+    !WARNING: In defined assignment subroutine 'assign_t3', second dummy argument 'y' should have INTENT(IN) or VALUE attribute [-Wdefined-operator-args]
+    !WARNING: In defined assignment subroutine 'assign_t4', first dummy argument 'x' should have INTENT(OUT) or INTENT(INOUT) [-Wdefined-operator-args]
     !ERROR: In defined assignment subroutine 'assign_t5', first dummy argument 'x' may not have INTENT(IN)
     !ERROR: In defined assignment subroutine 'assign_t6', second dummy argument 'y' may not have INTENT(OUT)
     generic :: assignment(=) => assign_t, assign_t2, assign_t3, assign_t4, assign_t5, assign_t6
diff --git a/flang/test/Semantics/resolve67.f90 b/flang/test/Semantics/resolve67.f90
index 677eef21d0a8..7dd0f2adc9d7 100644
--- a/flang/test/Semantics/resolve67.f90
+++ b/flang/test/Semantics/resolve67.f90
@@ -41,7 +41,7 @@ character(*) function divide(x, y)
     end
   end interface
   interface operator(<)
-    !WARNING: In OPERATOR(<) function 'lt1', dummy argument 'x' should have INTENT(IN) or VALUE attribute
+    !WARNING: In OPERATOR(<) function 'lt1', dummy argument 'x' should have INTENT(IN) or VALUE attribute [-Wdefined-operator-args]
     !ERROR: In OPERATOR(<) function 'lt1', dummy argument 'y' may not be OPTIONAL
     logical function lt1(x, y)
       logical :: x
@@ -90,7 +90,7 @@ real function plus(x)
     end
   end interface
   interface operator(.not.)
-    !WARNING: The external interface 'not1' is not compatible with an earlier definition (distinct numbers of dummy arguments)
+    !WARNING: The external interface 'not1' is not compatible with an earlier definition (distinct numbers of dummy arguments) [-Wexternal-interface-mismatch]
     real function not1(x)
       real, value :: x
     end
diff --git a/flang/test/Semantics/resolve69.f90 b/flang/test/Semantics/resolve69.f90
index 5acfd30604fe..5af14b48a07e 100644
--- a/flang/test/Semantics/resolve69.f90
+++ b/flang/test/Semantics/resolve69.f90
@@ -16,7 +16,7 @@ subroutine s1()
   !
   integer, parameter :: constVal = 1
   integer :: nonConstVal = 1
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   character(nonConstVal) :: colonString1
   character(len=20, kind=constVal + 1) :: constKindString
   character(len=:, kind=constVal + 1), pointer :: constKindString1
@@ -53,13 +53,13 @@ function foo3()
 
   type (derived(constVal, 3)) :: constDerivedKind
 !ERROR: Value of KIND type parameter 'typekind' must be constant
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   type (derived(nonConstVal, 3)) :: nonConstDerivedKind
 
   !OK because all type-params are constants
   type (derived(3, constVal)) :: constDerivedLen
 
-!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved)
+!PORTABILITY: specification expression refers to local object 'nonconstval' (initialized and saved) [-Wsaved-local-in-spec-expr]
   type (derived(3, nonConstVal)) :: nonConstDerivedLen
 !ERROR: 'colonderivedlen' has a type derived(typekind=3_4,typelen=:) with a deferred type parameter but is neither an allocatable nor an object pointer
   type (derived(3, :)) :: colonDerivedLen
diff --git a/flang/test/Semantics/resolve77.f90 b/flang/test/Semantics/resolve77.f90
index 0133fac3bfbc..f58f2637c453 100644
--- a/flang/test/Semantics/resolve77.f90
+++ b/flang/test/Semantics/resolve77.f90
@@ -60,13 +60,13 @@ pure integer function if2(n)
 block data
   common /blk2/ n
   data n/100/
-  !PORTABILITY: specification expression refers to local object 'n' (initialized and saved)
+  !PORTABILITY: specification expression refers to local object 'n' (initialized and saved) [-Wsaved-local-in-spec-expr]
   !ERROR: Automatic data object 'a' may not appear in a BLOCK DATA subprogram
   real a(n)
 end
 
 program main
   common /blk2/ n
-  !PORTABILITY: Automatic data object 'a' should not appear in the specification part of a main program
+  !PORTABILITY: Automatic data object 'a' should not appear in the specification part of a main program [-Wautomatic-in-main-program]
   real a(n)
 end
diff --git a/flang/test/Semantics/resolve78.f90 b/flang/test/Semantics/resolve78.f90
index 8c25f32e80cd..c4163fe37031 100644
--- a/flang/test/Semantics/resolve78.f90
+++ b/flang/test/Semantics/resolve78.f90
@@ -11,21 +11,21 @@ module m
 !      POINTER
 
   type :: derived
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     real, public, allocatable, public :: field1
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     real, private, allocatable, private :: field2
     !ERROR: Attributes 'PUBLIC' and 'PRIVATE' conflict with each other
     real, public, allocatable, private :: field3
-    !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once
+    !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once [-Wredundant-attribute]
     real, allocatable, public, allocatable :: field4
     !ERROR: Attribute 'CODIMENSION' cannot be used more than once
     real, public, codimension[:], allocatable, codimension[:] :: field5
-    !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once
+    !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once [-Wredundant-attribute]
     real, public, contiguous, pointer, contiguous, dimension(:) :: field6
     !ERROR: Attribute 'DIMENSION' cannot be used more than once
     real, dimension(5), public, dimension(5) :: field7
-    !WARNING: Attribute 'POINTER' cannot be used more than once
+    !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
     real, pointer, public, pointer :: field8
   end type derived
 
diff --git a/flang/test/Semantics/resolve79.f90 b/flang/test/Semantics/resolve79.f90
index 037e107dc587..87bd55a3bcfb 100644
--- a/flang/test/Semantics/resolve79.f90
+++ b/flang/test/Semantics/resolve79.f90
@@ -12,17 +12,17 @@ module m
 !    PUBLIC, PRIVATE, NOPASS, PASS, POINTER
 
   type :: procComponentType
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     procedure(publicProc), public, pointer, public :: publicField
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     procedure(privateProc), private, pointer, private :: privateField
-    !WARNING: Attribute 'NOPASS' cannot be used more than once
+    !WARNING: Attribute 'NOPASS' cannot be used more than once [-Wredundant-attribute]
     procedure(nopassProc), nopass, pointer, nopass :: noPassField
-    !WARNING: Attribute 'PASS' cannot be used more than once
+    !WARNING: Attribute 'PASS' cannot be used more than once [-Wredundant-attribute]
     procedure(passProc), pass, pointer, pass :: passField
     !ERROR: Attributes 'PASS' and 'NOPASS' conflict with each other
     procedure(passNopassProc), pass, pointer, nopass :: passNopassField
-    !WARNING: Attribute 'POINTER' cannot be used more than once
+    !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
     procedure(pointerProc), pointer, public, pointer :: pointerField
     !ERROR: Procedure component 'nonpointerfield' must have POINTER attribute
     procedure(publicProc), public :: nonpointerField
diff --git a/flang/test/Semantics/resolve80.f90 b/flang/test/Semantics/resolve80.f90
index e787a4ba7f86..7df6a6c57d69 100644
--- a/flang/test/Semantics/resolve80.f90
+++ b/flang/test/Semantics/resolve80.f90
@@ -13,17 +13,17 @@ module m
 !
   type, abstract :: boundProcType
    contains
-    !WARNING: Attribute 'PUBLIC' cannot be used more than once
+    !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
     procedure(subPublic), public, deferred, public :: publicBinding
-    !WARNING: Attribute 'PRIVATE' cannot be used more than once
+    !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
     procedure(subPrivate), private, deferred, private :: privateBinding
-    !WARNING: Attribute 'DEFERRED' cannot be used more than once
+    !WARNING: Attribute 'DEFERRED' cannot be used more than once [-Wredundant-attribute]
     procedure(subDeferred), deferred, public, deferred :: deferredBinding
-    !WARNING: Attribute 'NON_OVERRIDABLE' cannot be used more than once
+    !WARNING: Attribute 'NON_OVERRIDABLE' cannot be used more than once [-Wredundant-attribute]
     procedure, non_overridable, public, non_overridable :: subNon_overridable;
-    !WARNING: Attribute 'NOPASS' cannot be used more than once
+    !WARNING: Attribute 'NOPASS' cannot be used more than once [-Wredundant-attribute]
     procedure(subNopass), nopass, deferred, nopass :: nopassBinding
-    !WARNING: Attribute 'PASS' cannot be used more than once
+    !WARNING: Attribute 'PASS' cannot be used more than once [-Wredundant-attribute]
     procedure(subPass), pass, deferred, pass :: passBinding
     !ERROR: Attributes 'PASS' and 'NOPASS' conflict with each other
     procedure(subPassNopass), pass, deferred, nopass :: passNopassBinding  ! C781
diff --git a/flang/test/Semantics/resolve81.f90 b/flang/test/Semantics/resolve81.f90
index db5b19f1155e..23651cbb257e 100644
--- a/flang/test/Semantics/resolve81.f90
+++ b/flang/test/Semantics/resolve81.f90
@@ -11,52 +11,52 @@
 !    VOLATILE
 module m
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   real, public, allocatable, public :: publicVar
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   real, private, allocatable, private :: privateVar
-  !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once
+  !WARNING: Attribute 'ALLOCATABLE' cannot be used more than once [-Wredundant-attribute]
   real, allocatable, allocatable :: allocVar
-  !WARNING: Attribute 'ASYNCHRONOUS' cannot be used more than once
+  !WARNING: Attribute 'ASYNCHRONOUS' cannot be used more than once [-Wredundant-attribute]
   real, asynchronous, public, asynchronous :: asynchVar
   !ERROR: Attribute 'CODIMENSION' cannot be used more than once
   real, codimension[*], codimension[*] :: codimensionVar
-  !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once
+  !WARNING: Attribute 'CONTIGUOUS' cannot be used more than once [-Wredundant-attribute]
   real, contiguous, pointer, contiguous :: contigVar(:)
   !ERROR: Attribute 'DIMENSION' cannot be used more than once
   real, dimension(5), dimension(5) :: arrayVar
-  !WARNING: Attribute 'EXTERNAL' cannot be used more than once
+  !WARNING: Attribute 'EXTERNAL' cannot be used more than once [-Wredundant-attribute]
   real, external, external :: externFunc
-  !WARNING: Attribute 'INTRINSIC' cannot be used more than once
+  !WARNING: Attribute 'INTRINSIC' cannot be used more than once [-Wredundant-attribute]
   !ERROR: 'cos' may not have both the BIND(C) and INTRINSIC attributes
   !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   real, intrinsic, bind(c), intrinsic :: cos
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   integer, bind(c), volatile, bind(c) :: bindVar
-  !WARNING: Attribute 'PARAMETER' cannot be used more than once
+  !WARNING: Attribute 'PARAMETER' cannot be used more than once [-Wredundant-attribute]
   real, parameter, parameter :: realConst = 4.3
-  !WARNING: Attribute 'POINTER' cannot be used more than once
+  !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
   real, pointer, pointer :: realPtr
-  !WARNING: Attribute 'PROTECTED' cannot be used more than once
+  !WARNING: Attribute 'PROTECTED' cannot be used more than once [-Wredundant-attribute]
   real, protected, protected :: realProt
-  !WARNING: Attribute 'SAVE' cannot be used more than once
+  !WARNING: Attribute 'SAVE' cannot be used more than once [-Wredundant-attribute]
   real, save, save :: saveVar
-  !WARNING: Attribute 'TARGET' cannot be used more than once
+  !WARNING: Attribute 'TARGET' cannot be used more than once [-Wredundant-attribute]
   real, target, target :: targetVar
-  !WARNING: Attribute 'VOLATILE' cannot be used more than once
+  !WARNING: Attribute 'VOLATILE' cannot be used more than once [-Wredundant-attribute]
   real, volatile, volatile :: volatileVar
 
 contains
     subroutine testTypeDecl(arg1, arg2, arg3, arg4, arg5, arg6)
-      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once [-Wredundant-attribute]
       real, intent(in), intent(in) :: arg1
-      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once [-Wredundant-attribute]
       real, intent(out), intent(out) :: arg2
-      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once [-Wredundant-attribute]
       real, intent(inout), intent(inout) :: arg3
-      !WARNING: Attribute 'OPTIONAL' cannot be used more than once
+      !WARNING: Attribute 'OPTIONAL' cannot be used more than once [-Wredundant-attribute]
       integer, optional, intent(in), optional :: arg4
-      !WARNING: Attribute 'VALUE' cannot be used more than once
+      !WARNING: Attribute 'VALUE' cannot be used more than once [-Wredundant-attribute]
       integer, value, intent(in), value :: arg5
       !ERROR: Attributes 'INTENT(IN)' and 'INTENT(INOUT)' conflict with each other
       integer, intent(in), pointer, intent(inout) :: arg6
diff --git a/flang/test/Semantics/resolve82.f90 b/flang/test/Semantics/resolve82.f90
index 989ce1d837c7..c06edd6bb5d9 100644
--- a/flang/test/Semantics/resolve82.f90
+++ b/flang/test/Semantics/resolve82.f90
@@ -14,14 +14,14 @@ real function procFunc()
     end function procFunc
   end interface
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), public, pointer, public :: proc1
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), private, pointer, private :: proc2
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
   !ERROR: An interface name with the BIND attribute must appear if the BIND attribute appears in a procedure declaration
   procedure(procFunc), bind(c), pointer, bind(c) :: proc3
-  !WARNING: Attribute 'PROTECTED' cannot be used more than once
+  !WARNING: Attribute 'PROTECTED' cannot be used more than once [-Wredundant-attribute]
   procedure(procFunc), protected, pointer, protected :: proc4
   !ERROR: A PROTECTED entity must be a variable or pointer
   external extsub
@@ -39,21 +39,21 @@ end function procFunc
 contains
 
     subroutine testProcDecl(arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11)
-      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(IN)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(in), pointer, intent(in) :: arg4
-      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(OUT)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(out), pointer, intent(out) :: arg5
-      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once
+      !WARNING: Attribute 'INTENT(INOUT)' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), intent(inout), pointer, intent(inout) :: arg6
       !ERROR: Attributes 'INTENT(INOUT)' and 'INTENT(OUT)' conflict with each other
       procedure(procFunc), intent(inout), pointer, intent(out) :: arg7
       !ERROR: Attributes 'INTENT(INOUT)' and 'INTENT(OUT)' conflict with each other
       procedure(procFunc), intent(out), pointer, intent(inout) :: arg8
-      !WARNING: Attribute 'OPTIONAL' cannot be used more than once
+      !WARNING: Attribute 'OPTIONAL' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), optional, pointer, optional :: arg9
-      !WARNING: Attribute 'POINTER' cannot be used more than once
+      !WARNING: Attribute 'POINTER' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), pointer, optional, pointer :: arg10
-      !WARNING: Attribute 'SAVE' cannot be used more than once
+      !WARNING: Attribute 'SAVE' cannot be used more than once [-Wredundant-attribute]
       procedure(procFunc), save, pointer, save :: localProc
       !ERROR: A PROTECTED entity must be in the specification part of a module
       real x
diff --git a/flang/test/Semantics/resolve83.f90 b/flang/test/Semantics/resolve83.f90
index e9d53dd6bd81..a60a9980293f 100644
--- a/flang/test/Semantics/resolve83.f90
+++ b/flang/test/Semantics/resolve83.f90
@@ -3,7 +3,7 @@ module m
 
   ! For C1543
   interface intFace
-    !WARNING: Attribute 'MODULE' cannot be used more than once
+    !WARNING: Attribute 'MODULE' cannot be used more than once [-Wredundant-attribute]
     module pure module real function moduleFunc()
     end function moduleFunc
   end interface
@@ -27,17 +27,17 @@ end function moduleFunc
     real pure real function realFunc()
     end function realFunc
 
-    !WARNING: Attribute 'ELEMENTAL' cannot be used more than once
+    !WARNING: Attribute 'ELEMENTAL' cannot be used more than once [-Wredundant-attribute]
     elemental real elemental function elementalFunc(x)
       real, value :: x
       elementalFunc = x
     end function elementalFunc
 
-    !WARNING: Attribute 'IMPURE' cannot be used more than once
+    !WARNING: Attribute 'IMPURE' cannot be used more than once [-Wredundant-attribute]
     impure real impure function impureFunc()
     end function impureFunc
 
-    !WARNING: Attribute 'PURE' cannot be used more than once
+    !WARNING: Attribute 'PURE' cannot be used more than once [-Wredundant-attribute]
     pure real pure function pureFunc()
     end function pureFunc
 
@@ -45,11 +45,11 @@ end function pureFunc
     impure real pure function impurePureFunc()
     end function impurePureFunc
 
-    !WARNING: Attribute 'RECURSIVE' cannot be used more than once
+    !WARNING: Attribute 'RECURSIVE' cannot be used more than once [-Wredundant-attribute]
     recursive real recursive function recursiveFunc()
     end function recursiveFunc
 
-    !WARNING: Attribute 'NON_RECURSIVE' cannot be used more than once
+    !WARNING: Attribute 'NON_RECURSIVE' cannot be used more than once [-Wredundant-attribute]
     non_recursive real non_recursive function non_recursiveFunc()
     end function non_recursiveFunc
 
diff --git a/flang/test/Semantics/resolve85.f90 b/flang/test/Semantics/resolve85.f90
index 9b9358ecf477..64b8081dd457 100644
--- a/flang/test/Semantics/resolve85.f90
+++ b/flang/test/Semantics/resolve85.f90
@@ -7,15 +7,15 @@ module m
 !        TYPE [[, type-attr-spec-list] ::] type-name [( type-param-name-list )]
 !  type-attr-spec values are:
 !    ABSTRACT, PUBLIC, PRIVATE, BIND(C), EXTENDS(parent-type-name)
-  !WARNING: Attribute 'ABSTRACT' cannot be used more than once
+  !WARNING: Attribute 'ABSTRACT' cannot be used more than once [-Wredundant-attribute]
   type, abstract, public, abstract :: derived1
   end type derived1
 
-  !WARNING: Attribute 'PUBLIC' cannot be used more than once
+  !WARNING: Attribute 'PUBLIC' cannot be used more than once [-Wredundant-attribute]
   type, public, abstract, public :: derived2
   end type derived2
 
-  !WARNING: Attribute 'PRIVATE' cannot be used more than once
+  !WARNING: Attribute 'PRIVATE' cannot be used more than once [-Wredundant-attribute]
   type, private, abstract, private :: derived3
   end type derived3
 
@@ -23,8 +23,8 @@ module m
   type, public, abstract, private :: derived4
   end type derived4
 
-  !WARNING: Attribute 'BIND(C)' cannot be used more than once
-  !WARNING: A derived type with the BIND attribute should not be empty
+  !WARNING: Attribute 'BIND(C)' cannot be used more than once [-Wredundant-attribute]
+  !WARNING: A derived type with the BIND attribute should not be empty [-Wempty-bind-c-derived-type]
   type, bind(c), public, bind(c) :: derived5
   end type derived5
 
diff --git a/flang/test/Semantics/resolve90.f90 b/flang/test/Semantics/resolve90.f90
index baa108f4d083..7d2b5e9d333f 100644
--- a/flang/test/Semantics/resolve90.f90
+++ b/flang/test/Semantics/resolve90.f90
@@ -12,7 +12,7 @@ subroutine s()
     !ERROR: 'pointerallocatablefield' may not have both the POINTER and ALLOCATABLE attributes
     real, pointer, allocatable :: pointerAllocatableField
     real, dimension(:), contiguous, pointer :: goodContigField
-    !PORTABILITY: CONTIGUOUS component 'badcontigfield' should be an array with the POINTER attribute
+    !PORTABILITY: CONTIGUOUS component 'badcontigfield' should be an array with the POINTER attribute [-Wredundant-contiguous]
     real, dimension(:), contiguous, allocatable :: badContigField
     character :: charField * 3
     !ERROR: A length specifier cannot be used to declare the non-character entity 'realfield'
diff --git a/flang/test/Semantics/resolve99.f90 b/flang/test/Semantics/resolve99.f90
index e56022b61bfd..796c354af134 100644
--- a/flang/test/Semantics/resolve99.f90
+++ b/flang/test/Semantics/resolve99.f90
@@ -31,7 +31,7 @@ subroutine constructAssoc()
     integer, dimension(4) :: table
     integer :: localVar
     associate (assocVar => localVar)
-      !PORTABILITY: Index variable 'assocvar' should be a scalar object or common block if it is present in the enclosing scope
+      !PORTABILITY: Index variable 'assocvar' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
       FORALL (assocVar=1:4) table(assocVar) = 343
     end associate
   end subroutine constructAssoc
@@ -44,7 +44,7 @@ end subroutine commonSub
 
   subroutine mismatch()
     integer, dimension(4) :: table
-    !PORTABILITY: Index variable 'typename' should be a scalar object or common block if it is present in the enclosing scope
+    !PORTABILITY: Index variable 'typename' should be a scalar object or common block if it is present in the enclosing scope [-Wodd-index-variable-restrictions]
     !ERROR: Must have INTEGER type, but is REAL(4)
     !ERROR: Must have INTEGER type, but is REAL(4)
     FORALL (typeName=1:4) table(typeName) = 343
diff --git a/flang/test/Semantics/separate-mp02.f90 b/flang/test/Semantics/separate-mp02.f90
index cb1e2687bad7..7d8ea6e0c393 100644
--- a/flang/test/Semantics/separate-mp02.f90
+++ b/flang/test/Semantics/separate-mp02.f90
@@ -368,7 +368,7 @@ module subroutine s(x)
 end
 submodule(m11) sm11
  contains
-  !WARNING: Dummy procedure 'x' does not exactly match the corresponding argument in the interface body
+  !WARNING: Dummy procedure 'x' does not exactly match the corresponding argument in the interface body [-Wmismatching-dummy-procedure]
   module subroutine s(x)
     call x ! no error
   end
diff --git a/flang/test/Semantics/spec-expr.f90 b/flang/test/Semantics/spec-expr.f90
index 28ebea1109f1..4d79f2ca188f 100644
--- a/flang/test/Semantics/spec-expr.f90
+++ b/flang/test/Semantics/spec-expr.f90
@@ -117,7 +117,7 @@ subroutine s7biii(x, y)
   integer :: local = 5
   ! OK, since "localConst" is a constant
   real, dimension(localConst) :: realArray1
-  !PORTABILITY: specification expression refers to local object 'local' (initialized and saved)
+  !PORTABILITY: specification expression refers to local object 'local' (initialized and saved) [-Wsaved-local-in-spec-expr]
   real, dimension(local) :: realArray2
   real, dimension(size(realArray1)) :: realArray3 ! ok
   real, dimension(size(x)) :: realArray4 ! ok
diff --git a/flang/test/Semantics/stmt-func01.f90 b/flang/test/Semantics/stmt-func01.f90
index a87b0d7af52b..d8ef9af25b38 100644
--- a/flang/test/Semantics/stmt-func01.f90
+++ b/flang/test/Semantics/stmt-func01.f90
@@ -10,14 +10,14 @@ program main
     pure integer function ifunc()
     end function
   end interface
-  !PORTABILITY: Automatic data object 'x1' should not appear in the specification part of a main program
+  !PORTABILITY: Automatic data object 'x1' should not appear in the specification part of a main program [-Wautomatic-in-main-program]
   type(t1(k=4,l=ifunc())) x1
-  !PORTABILITY: Statement function 'sf1' should not contain an array constructor
+  !PORTABILITY: Statement function 'sf1' should not contain an array constructor [-Wstatement-function-extensions]
   sf1(n) = sum([(j,j=1,n)])
   type(t1) sf2
-  !PORTABILITY: Statement function 'sf2' should not contain a structure constructor
+  !PORTABILITY: Statement function 'sf2' should not contain a structure constructor [-Wstatement-function-extensions]
   sf2(n) = t1(n)
-  !PORTABILITY: Statement function 'sf3' should not contain a type parameter inquiry
+  !PORTABILITY: Statement function 'sf3' should not contain a type parameter inquiry [-Wstatement-function-extensions]
   sf3(n) = x1%l
   !ERROR: Recursive call to statement function 'sf4' is not allowed
   sf4(n) = sf4(n)
@@ -26,10 +26,10 @@ pure integer function ifunc()
   real sf7
   !ERROR: Statement function 'sf6' may not reference another statement function 'sf7' that is defined later
   sf6(n) = sf7(n)
-  !PORTABILITY: Statement function 'sf7' should not reference function 'explicit' that requires an explicit interface
+  !PORTABILITY: Statement function 'sf7' should not reference function 'explicit' that requires an explicit interface [-Wstatement-function-extensions]
   sf7(n) = explicit(n)
   real :: a(3) = [1., 2., 3.]
-  !PORTABILITY: Statement function 'sf8' should not pass an array argument that is not a whole array
+  !PORTABILITY: Statement function 'sf8' should not pass an array argument that is not a whole array [-Wstatement-function-extensions]
   sf8(n) = sum(a(1:2))
   sf8a(n) = sum(a) ! ok
   integer :: sf9
@@ -56,7 +56,7 @@ pure function arr()
     arr = [1., 2.]
   end function
   subroutine foo
-    !PORTABILITY: An implicitly typed statement function should not appear when the same symbol is available in its host scope
+    !PORTABILITY: An implicitly typed statement function should not appear when the same symbol is available in its host scope [-Wstatement-function-extensions]
     sf14(x) = 2.*x
   end subroutine
 end
@@ -94,7 +94,7 @@ subroutine s4
 subroutine s5
   !ERROR: Invalid specification expression: reference to impure function 'k'
   real x(k())
-  !WARNING: Name 'k' from host scope should have a type declaration before its local statement function definition
+  !WARNING: Name 'k' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
   !ERROR: 'k' is already declared in this scoping unit
   k() = 0.0
 end
diff --git a/flang/test/Semantics/stmt-func02.f90 b/flang/test/Semantics/stmt-func02.f90
index bfed280ded58..10166a0abf7b 100644
--- a/flang/test/Semantics/stmt-func02.f90
+++ b/flang/test/Semantics/stmt-func02.f90
@@ -24,11 +24,11 @@ subroutine test1
     print *, x
   end
   subroutine test2
-    !PORTABILITY: Name 'rf' from host scope should have a type declaration before its local statement function definition
+    !PORTABILITY: Name 'rf' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     rf(x) = 1.
   end
   subroutine test2b
-    !PORTABILITY: Name 'rf2' from host scope should have a type declaration before its local statement function definition
+    !PORTABILITY: Name 'rf2' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     rf2(x) = 1.
   end
   subroutine test3
@@ -43,7 +43,7 @@ function f()
     f() = 1. ! statement function of same name as function
   end
   function g() result(r)
-    !WARNING: Name 'g' from host scope should have a type declaration before its local statement function definition
+    !WARNING: Name 'g' from host scope should have a type declaration before its local statement function definition [-Wstatement-function-extensions]
     !ERROR: 'g' is already declared in this scoping unit
     g() = 1. ! statement function of same name as function
   end
diff --git a/flang/test/Semantics/structconst03.f90 b/flang/test/Semantics/structconst03.f90
index ecd31723b12b..006f21e5b29f 100644
--- a/flang/test/Semantics/structconst03.f90
+++ b/flang/test/Semantics/structconst03.f90
@@ -183,7 +183,7 @@ impure real function ipf1(dummy1, dummy2, dummy3, dummy4)
     x1 = t1(0)(usedfrom1)
     x1 = t1(0)(modulevar1)
     x1 = t1(0)(commonvar1)
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'dummy1' is an INTENT(IN) dummy argument
     x1 = t1(0)(dummy1)
     x1 = t1(0)(dummy2)
diff --git a/flang/test/Semantics/structconst04.f90 b/flang/test/Semantics/structconst04.f90
index abddf6001726..b44f87aeb13f 100644
--- a/flang/test/Semantics/structconst04.f90
+++ b/flang/test/Semantics/structconst04.f90
@@ -177,7 +177,7 @@ impure real function ipf1(dummy1, dummy2, dummy3, dummy4)
     x1 = t1(usedfrom1)
     x1 = t1(modulevar1)
     x1 = t1(commonvar1)
-    !WARNING: Pointer target is not a definable variable
+    !WARNING: Pointer target is not a definable variable [-Wpointer-to-undefinable]
     !BECAUSE: 'dummy1' is an INTENT(IN) dummy argument
     x1 = t1(dummy1)
     x1 = t1(dummy2)
diff --git a/flang/test/Semantics/structconst08.f90 b/flang/test/Semantics/structconst08.f90
index 149c898dda9d..78e44a49735d 100644
--- a/flang/test/Semantics/structconst08.f90
+++ b/flang/test/Semantics/structconst08.f90
@@ -25,7 +25,7 @@ subroutine test1()
 !ERROR: Must be a constant value
     type(parent1) :: tp2 = parent1(j)
     type(parent1) :: tp3 = parent1(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(parent1) :: tp4 = parent1(null(ipp))
 
 !ERROR: Must be a constant value
@@ -33,7 +33,7 @@ subroutine test1()
 !ERROR: Must be a constant value
     type(parent2) :: tp6 = parent2(arr)
     type(parent2) :: tp7 = parent2(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(parent2) :: tp8 = parent2(null(rpp))
   end subroutine test1
 
@@ -47,20 +47,20 @@ subroutine test2()
     tp1 = parent1(3)
     tp1 = parent1(j)
     tp1 = parent1(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     tp1 = parent1(null(ipp))
 
     tp2 = parent2([1.1,2.1,3.1])
     tp2 = parent2(arr)
     tp2 = parent2(null())
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     tp2 = parent2(null(rpp))
   end subroutine test2
 
   subroutine test3()
     real, pointer :: pp(:)
     type(child) :: tc1 = child(5, parent2(null()))
-!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa'
+!PORTABILITY: NULL() with arguments is not standard conforming as the value for allocatable component 'pa' [-Wnull-mold-allocatable-component-value]
     type(child) :: tc10 = child(5, parent2(null(pp)))
 !ERROR: Must be a constant value
     type(child) :: tc3 = child(5, parent2([1.1,1.2]))
diff --git a/flang/test/Semantics/structconst10.f90 b/flang/test/Semantics/structconst10.f90
index 582f8fc15704..e6a22a9bfac5 100644
--- a/flang/test/Semantics/structconst10.f90
+++ b/flang/test/Semantics/structconst10.f90
@@ -14,12 +14,12 @@ end module m1
 program test
   use m1
   type(a3) v
-  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a2(x1=18,x2=6),x3=6)
-  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a1(x1=18),x2=6,x3=6)
-  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous
-  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous
+  !PORTABILITY: Whole parent component 'a2' in structure constructor should not be anonymous [-Wanonymous-parents]
+  !PORTABILITY: Whole parent component 'a1' in structure constructor should not be anonymous [-Wanonymous-parents]
   v=a3(a2(a1(x1=18),x2=6),x3=6)
   v=a3(a2=a2(a1=a1(x1=18),x2=6),x3=6) ! ok
 end
diff --git a/flang/test/Semantics/symbol15.f90 b/flang/test/Semantics/symbol15.f90
index 97dc50a23845..df10942e6af2 100644
--- a/flang/test/Semantics/symbol15.f90
+++ b/flang/test/Semantics/symbol15.f90
@@ -43,6 +43,9 @@ subroutine iface
  !DEF: /m/pp6 EXTERNAL, POINTER, PUBLIC (Subroutine) ProcEntity
  !DEF: /m/modproc1 PUBLIC (Subroutine) Subprogram
  procedure(iface), pointer :: pp6 => modproc1
+ !DEF: /m/xx PUBLIC, TARGET ObjectEntity REAL(4)
+ !DEF: /m/yy PUBLIC, TARGET ObjectEntity REAL(4)
+ real, target :: xx, yy(2)
  !DEF: /m/t1 PUBLIC DerivedType
  type :: t1
   !DEF: /m/t1/opc1 POINTER ObjectEntity REAL(4)
@@ -51,11 +54,11 @@ subroutine iface
   !REF: /m/null
   real, pointer :: opc2 => null()
   !DEF: /m/t1/opc3 POINTER ObjectEntity REAL(4)
-  !REF: /m/x
-  real, pointer :: opc3 => x
+  !REF: /m/xx
+  real, pointer :: opc3 => xx
   !DEF: /m/t1/opc4 POINTER ObjectEntity REAL(4)
-  !REF: /m/y
-  real, pointer :: opc4 => y(1)
+  !REF: /m/yy
+  real, pointer :: opc4 => yy(1)
   !REF: /m/iface
   !DEF: /m/t1/ppc1 NOPASS, POINTER (Subroutine) ProcEntity
   procedure(iface), nopass, pointer :: ppc1
@@ -101,12 +104,12 @@ subroutine iface
   !REF: /m/null
   real, pointer :: opc2 => null()
   !DEF: /m/pdt1/opc3 POINTER ObjectEntity REAL(4)
-  !REF: /m/x
-  real, pointer :: opc3 => x
+  !REF: /m/xx
+  real, pointer :: opc3 => xx
   !DEF: /m/pdt1/opc4 POINTER ObjectEntity REAL(4)
-  !REF: /m/y
+  !REF: /m/yy
   !REF: /m/pdt1/k
-  real, pointer :: opc4 => y(k)
+  real, pointer :: opc4 => yy(k)
   !REF: /m/iface
   !DEF: /m/pdt1/ppc1 NOPASS, POINTER (Subroutine) ProcEntity
   procedure(iface), nopass, pointer :: ppc1
diff --git a/flang/test/Semantics/transfer01.f90 b/flang/test/Semantics/transfer01.f90
index 26f4f1b3eb62..d58cf1a4408e 100644
--- a/flang/test/Semantics/transfer01.f90
+++ b/flang/test/Semantics/transfer01.f90
@@ -20,17 +20,17 @@ subroutine subr(o)
   print *, transfer(1., empty2) ! ok
   !ERROR: Element size of MOLD= array may not be zero when SOURCE= is not empty
   print *, transfer(1., empty3)
-  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty
+  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty [-Wvoid-mold]
   print *, transfer(source, empty1)
   print *, transfer(source, empty2) ! ok
-  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty
+  !WARNING: Element size of MOLD= array may not be zero unless SOURCE= is empty [-Wvoid-mold]
   print *, transfer(source, empty3)
   !ERROR: SIZE= argument may not be the optional dummy argument 'o'
   print *, transfer(1., empty2, size=o)
-  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning
+  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning [-Wtransfer-size-presence]
   print *, transfer(1., empty2, size=ia)
-  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning
+  !WARNING: SIZE= argument that is allocatable or pointer must be present at execution; parenthesize to silence this warning [-Wtransfer-size-presence]
   print *, transfer(1., empty2, size=ip)
-  !WARNING: Source of TRANSFER contains allocatable or pointer component %allocatable
+  !WARNING: Source of TRANSFER contains allocatable or pointer component %allocatable [-Wpointer-component-transfer-arg]
   print *, transfer(hasDesc, 1)
 end
diff --git a/flang/test/Semantics/undef-result01.f90 b/flang/test/Semantics/undef-result01.f90
index e1ae58dae7c0..85cd181cd441 100644
--- a/flang/test/Semantics/undef-result01.f90
+++ b/flang/test/Semantics/undef-result01.f90
@@ -1,6 +1,6 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function basic()
 end
 
@@ -29,7 +29,7 @@ subroutine intentInPtr(p)
   end
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function notDefdByCall()
   call intentin(notDefdByCall)
  contains
@@ -38,7 +38,7 @@ subroutine intentin(n)
   end
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 function basicAlloc()
   real, allocatable :: basicAlloc
   allocate(basicAlloc)
@@ -134,7 +134,7 @@ integer function defdBySize()
   inquire(6,status=defdByInquire)
 end
 
-!WARNING: Function result is never defined
+!WARNING: Function result is never defined [-Wundefined-function-result]
 character(20) function notDefdByInquire()
   inquire(file=notDefdByInquire)
 end
diff --git a/flang/test/Semantics/windows.f90 b/flang/test/Semantics/windows.f90
index 705fb1b81689..799a34e6b89c 100644
--- a/flang/test/Semantics/windows.f90
+++ b/flang/test/Semantics/windows.f90
@@ -1,4 +1,4 @@
-! RUN: %if x86_64-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
+! RUN: %if x86-registered-target %{ %python %S/test_errors.py %s %flang --target=x86_64-pc-windows-msvc -Werror %}
 ! RUN: %if aarch64-registered-target %{ %python %S/test_errors.py %s %flang --target=aarch64-pc-windows-msvc -Werror %}
 
 subroutine uid
diff --git a/flang/test/Transforms/debug-omp-target-op-1.fir b/flang/test/Transforms/debug-omp-target-op-1.fir
new file mode 100644
index 000000000000..6b895b732c42
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-1.fir
@@ -0,0 +1,40 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info="debug-level=LineTablesOnly" --mlir-print-debuginfo %s | FileCheck %s --check-prefix=LINETABLE
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @_QQmain() attributes {fir.bindc_name = "test"} {
+    %c13_i32 = arith.constant 13 : i32
+    %c12_i32 = arith.constant 12 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c1_i32 = arith.constant 1 : i32
+    %c5_i32 = arith.constant 5 : i32
+    %0 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFEx"} loc(#loc1)
+    %1 = fircg.ext_declare %0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc1)
+    %2 = fir.alloca i32 {bindc_name = "y", uniq_name = "_QFEy"} loc(#loc2)
+    %3 = fircg.ext_declare %2 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc2)
+    %4 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "x"}
+    %5 = omp.map.info var_ptr(%3 : !fir.ref<i32>, i32) map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "y"}
+    omp.target map_entries(%4 -> %arg0, %5 -> %arg1 : !fir.ref<i32>, !fir.ref<i32>) {
+      %16 = fircg.ext_declare %arg0 {uniq_name = "_QFEx"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc3)
+      %17 = fircg.ext_declare %arg1 {uniq_name = "_QFEy"} : (!fir.ref<i32>) -> !fir.ref<i32> loc(#loc4)
+      omp.terminator
+    } loc(#loc5)
+    return
+  }
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// CHECK: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "x"{{.*}}line = 1, type = #[[TY:.*]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "y"{{.*}}line = 3, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "x"{{.*}}line = 7, type = #[[TY]]>
+// CHECK: #llvm.di_local_variable<scope = #[[SP1]], name = "y"{{.*}}line = 8, type = #[[TY]]>
+
+// LINETABLE: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "test"{{.*}}>
+// LINETABLE: #[[SP1:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_QQmain_l6"{{.*}}line = 6{{.*}}subprogramFlags = "LocalToUnit|Definition"{{.*}}>
+// LINETABLE-NOT: #llvm.di_local_variable
diff --git a/flang/test/Transforms/debug-omp-target-op-2.fir b/flang/test/Transforms/debug-omp-target-op-2.fir
new file mode 100644
index 000000000000..15dcf2389b21
--- /dev/null
+++ b/flang/test/Transforms/debug-omp-target-op-2.fir
@@ -0,0 +1,53 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @fn_(%arg0: !fir.ref<!fir.array<?x?xi32>> {fir.bindc_name = "b"}, %arg1: !fir.ref<i32> {fir.bindc_name = "c"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+    %c1 = arith.constant 1 : index
+    %c0 = arith.constant 0 : index
+    %0 = fir.alloca i32
+    %1 = fir.alloca i32
+    %2 = fir.undefined !fir.dscope
+    %3 = fircg.ext_declare %arg1 dummy_scope %2 {uniq_name = "_QFfnEc"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc2)
+    %4 = fircg.ext_declare %arg2 dummy_scope %2 {uniq_name = "_QFfnEd"} : (!fir.ref<i32>, !fir.dscope) -> !fir.ref<i32> loc(#loc3)
+    %5 = fir.load %3 : !fir.ref<i32>
+    %6 = fir.convert %5 : (i32) -> index
+    %9 = fir.load %4 : !fir.ref<i32>
+    %10 = fir.convert %9 : (i32) -> index
+    %15 = fircg.ext_declare %arg0(%6, %10) dummy_scope %2 {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index, !fir.dscope) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc4)
+    %16 = fircg.ext_embox %15(%6, %10) : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.box<!fir.array<?x?xi32>>
+    %17:3 = fir.box_dims %16, %c0 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %18 = arith.subi %17#1, %c1 : index
+    %19 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%18 : index) extent(%17#1 : index) stride(%17#2 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %20 = arith.muli %17#2, %17#1 : index
+    %21:3 = fir.box_dims %16, %c1 : (!fir.box<!fir.array<?x?xi32>>, index) -> (index, index, index)
+    %22 = arith.subi %21#1, %c1 : index
+    %23 = omp.map.bounds lower_bound(%c0 : index) upper_bound(%22 : index) extent(%21#1 : index) stride(%20 : index) start_idx(%c1 : index) {stride_in_bytes = true}
+    %24 = omp.map.info var_ptr(%15 : !fir.ref<!fir.array<?x?xi32>>, i32) map_clauses(tofrom) capture(ByRef) bounds(%19, %23) -> !fir.ref<!fir.array<?x?xi32>> {name = "b"}
+    %25 = omp.map.info var_ptr(%1 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    %26 = omp.map.info var_ptr(%0 : !fir.ref<i32>, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !fir.ref<i32> {name = ""}
+    omp.target map_entries(%24 -> %arg3, %25 -> %arg4, %26 -> %arg5 : !fir.ref<!fir.array<?x?xi32>>, !fir.ref<i32>, !fir.ref<i32>) {
+      %27 = fir.load %arg5 : !fir.ref<i32>
+      %28 = fir.load %arg4 : !fir.ref<i32>
+      %29 = fir.convert %27 : (i32) -> index
+      %31 = fir.convert %28 : (i32) -> index
+      %37 = fircg.ext_declare %arg3(%29, %31) {uniq_name = "_QFfnEb"} : (!fir.ref<!fir.array<?x?xi32>>, index, index) -> !fir.ref<!fir.array<?x?xi32>> loc(#loc5)
+      omp.terminator
+    } loc(#loc6)
+    return
+  } loc(#loc7)
+}
+#loc1 = loc("test.f90":1:1)
+#loc2 = loc("test.f90":3:1)
+#loc3 = loc("test.f90":7:1)
+#loc4 = loc("test.f90":8:1)
+#loc5 = loc("test.f90":6:1)
+#loc6 = loc("test.f90":16:1)
+#loc7 = loc("test.f90":26:1)
+
+
+// Test that variable size arrays inside target regions get their own
+// compiler generated variables for size.
+
+// CHECK: #[[SP:.*]] = #llvm.di_subprogram<{{.*}}name = "__omp_offloading_{{.*}}_fn__l16"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb1"{{.*}}>
+// CHECK: #llvm.di_local_variable<scope = #[[SP]], name = "._QFfnEb2"{{.*}}>
diff --git a/flang/test/Transforms/tbaa-for-local-vars.fir b/flang/test/Transforms/tbaa-for-local-vars.fir
new file mode 100644
index 000000000000..82058ffef290
--- /dev/null
+++ b/flang/test/Transforms/tbaa-for-local-vars.fir
@@ -0,0 +1,92 @@
+// Test TBAA tags assignment for access off the fir.declare
+// placed in the middle of the routine (e.g. a temporary).
+// Original example:
+// module m
+//   type t
+//      real :: x
+//   end type t
+// contains
+//   subroutine bar(this)
+//     class(t), intent(out) :: this
+//     this%x = 1.0
+//   end subroutine bar
+//   function foo() result(res)
+//     type(t) :: res
+//     call bar(res)
+//   end function foo
+//   subroutine test(arg)
+//     type(t) :: var
+//     var = foo()
+//     arg = var%x
+//   end subroutine test
+// end module m
+//
+// The calls were manually inlined in FIR with fir.save_result's
+// destination operand being used instead of the temporary
+// alloca of the result inside foo. Runtime calls were removed
+// to reduce the test.
+// The temporary function result fir.declare is then dominated
+// by fir.dummy_scope of the foo function. If we use this scope
+// to assign the TBAA local-alloc tag to the accesses of
+// the temporary, then it won't conflict with the TBAA dummy tag
+// assigned to the accesses of this argument of bar.
+// That would be incorrect, because they access the same memory.
+// Check that the local-alloc tag is placed into the outermost
+// scope's TBAA tree.
+// RUN: fir-opt --fir-add-alias-tags %s | FileCheck %s
+
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QMmPtest - Scope 2">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QMmPtest">
+// CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
+// CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
+// CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
+// CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QMmFbarEthis", members = {<#[[$ATTR_6]], 0>}>
+// CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$ATTR_7]], 0>}>
+// CHECK: #[[$ATTR_12:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_9]], access_type = #[[$ATTR_9]], offset = 0>
+// CHECK: #[[$ATTR_13:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_10]], access_type = #[[$ATTR_10]], offset = 0>
+
+// CHECK-LABEL:   func.func @_QMmPtest(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<f32> {fir.bindc_name = "arg"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = ".result"}
+// CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_3:.*]] = fir.declare %[[ARG0]] dummy_scope %[[VAL_2]] {uniq_name = "_QMmFtestEarg"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]] = fir.declare %[[VAL_1]] {uniq_name = "_QMmFfooEres"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_8:.*]] = fir.embox %[[VAL_7]] : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.box<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.type<_QMmTt{x:f32}>>) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_10:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_11:.*]] = fir.declare %[[VAL_9]] dummy_scope %[[VAL_10]] {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QMmFbarEthis"} : (!fir.class<!fir.type<_QMmTt{x:f32}>>, !fir.dscope) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_12:.*]] = fir.coordinate_of %[[VAL_11]], x : (!fir.class<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+// CHECK:           fir.store %[[VAL_0]] to %[[VAL_12]] {tbaa = [#[[$ATTR_12]]]} : !fir.ref<f32>
+// CHECK:           %[[VAL_13:.*]] = fir.declare %[[VAL_1]] {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+// CHECK:           %[[VAL_14:.*]] = fir.coordinate_of %[[VAL_13]], x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+// CHECK:           %[[VAL_16:.*]] = fir.load %[[VAL_14]] {tbaa = [#[[$ATTR_13]]]} : !fir.ref<f32>
+func.func @_QMmPtest(%arg0: !fir.ref<f32> {fir.bindc_name = "arg"}) {
+  %cst = arith.constant 1.000000e+00 : f32
+  %0 = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = ".result"}
+  %1 = fir.dummy_scope : !fir.dscope
+  %2 = fir.declare %arg0 dummy_scope %1 {uniq_name = "_QMmFtestEarg"} : (!fir.ref<f32>, !fir.dscope) -> !fir.ref<f32>
+  %3 = fir.alloca !fir.type<_QMmTt{x:f32}> {bindc_name = "var", uniq_name = "_QMmFtestEvar"}
+  %4 = fir.declare %3 {uniq_name = "_QMmFtestEvar"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %5 = fir.dummy_scope : !fir.dscope
+  %6 = fir.declare %0 {uniq_name = "_QMmFfooEres"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %7 = fir.embox %6 : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.box<!fir.type<_QMmTt{x:f32}>>
+  %8 = fir.convert %7 : (!fir.box<!fir.type<_QMmTt{x:f32}>>) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+  %9 = fir.dummy_scope : !fir.dscope
+  %10 = fir.declare %8 dummy_scope %9 {fortran_attrs = #fir.var_attrs<intent_out>, uniq_name = "_QMmFbarEthis"} : (!fir.class<!fir.type<_QMmTt{x:f32}>>, !fir.dscope) -> !fir.class<!fir.type<_QMmTt{x:f32}>>
+  %14 = fir.coordinate_of %10, x : (!fir.class<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  fir.store %cst to %14 : !fir.ref<f32>
+  %15 = fir.declare %0 {uniq_name = ".tmp.func_result"} : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<!fir.type<_QMmTt{x:f32}>>
+  %16 = fir.coordinate_of %15, x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  %17 = fir.coordinate_of %4, x : (!fir.ref<!fir.type<_QMmTt{x:f32}>>) -> !fir.ref<f32>
+  %18 = fir.load %16 : !fir.ref<f32>
+  fir.store %18 to %17 : !fir.ref<f32>
+  %19 = fir.load %17 : !fir.ref<f32>
+  fir.store %19 to %2 : !fir.ref<f32>
+  return
+}
+
diff --git a/flang/test/Transforms/tbaa-with-dummy-scope2.fir b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
index 249471de458d..fd711a4d70eb 100644
--- a/flang/test/Transforms/tbaa-with-dummy-scope2.fir
+++ b/flang/test/Transforms/tbaa-with-dummy-scope2.fir
@@ -84,18 +84,17 @@ func.func @_QPtest2() attributes {noinline} {
   fir.store %c2_i32 to %2 : !fir.ref<i32>
   return
 }
-// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2 - Scope 1">
-// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2">
+// CHECK: #[[$ATTR_0:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2">
+// CHECK: #[[$ATTR_1:.+]] = #llvm.tbaa_root<id = "Flang function root _QPtest2 - Scope 1">
 // CHECK: #[[$ATTR_2:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_3:.+]] = #llvm.tbaa_type_desc<id = "any access", members = {<#[[$ATTR_1]], 0>}>
 // CHECK: #[[$ATTR_4:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_2]], 0>}>
 // CHECK: #[[$ATTR_5:.+]] = #llvm.tbaa_type_desc<id = "any data access", members = {<#[[$ATTR_3]], 0>}>
 // CHECK: #[[$TARGETDATA_0:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_4]], 0>}>
-// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_4]], 0>}>
-// CHECK: #[[$TARGETDATA_1:.+]] = #llvm.tbaa_type_desc<id = "target data", members = {<#[[$ATTR_5]], 0>}>
+// CHECK: #[[$ATTR_6:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data", members = {<#[[$ATTR_5]], 0>}>
 // CHECK: #[[$LOCAL_ATTR_0:.+]] = #llvm.tbaa_type_desc<id = "allocated data", members = {<#[[$TARGETDATA_0]], 0>}>
 // CHECK: #[[$ATTR_8:.+]] = #llvm.tbaa_type_desc<id = "dummy arg data/_QFtest2FinnerEx", members = {<#[[$ATTR_6]], 0>}>
-// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA_1]], 0>}>
+// CHECK: #[[$ATTR_7:.+]] = #llvm.tbaa_type_desc<id = "global data", members = {<#[[$TARGETDATA_0]], 0>}>
 // CHECK: #[[$ATTR_10:.+]] = #llvm.tbaa_tag<base_type = #[[$ATTR_8]], access_type = #[[$ATTR_8]], offset = 0>
 // CHECK: #[[$LOCAL_ATTR_1:.+]] = #llvm.tbaa_type_desc<id = "allocated data/_QFtest2FinnerEy", members = {<#[[$LOCAL_ATTR_0]], 0>}>
 // CHECK: #[[$ATTR_9:.+]] = #llvm.tbaa_type_desc<id = "global data/_QMmEglob", members = {<#[[$ATTR_7]], 0>}>
diff --git a/flang/test/lib/OpenACC/CMakeLists.txt b/flang/test/lib/OpenACC/CMakeLists.txt
index 1c0ac748f85e..d8d0109ed584 100644
--- a/flang/test/lib/OpenACC/CMakeLists.txt
+++ b/flang/test/lib/OpenACC/CMakeLists.txt
@@ -2,22 +2,26 @@ add_flang_library(FIRTestOpenACCInterfaces
   TestOpenACCInterfaces.cpp
 
   DEPENDS
+  HLFIRDialect
   FIRDialect
   FIROpenACCSupport
   FIRSupport
 
   LINK_LIBS
+  HLFIRDialect
   FIRDialect
   FIROpenACCSupport
   FIRSupport
 
   MLIR_DEPS
+  MLIRDLTIDialect
   MLIRIR
   MLIROpenACCDialect
   MLIRPass
   MLIRSupport
 
   MLIR_LIBS
+  MLIRDLTIDialect
   MLIRIR
   MLIROpenACCDialect
   MLIRPass
diff --git a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
index 90aabd7d40d4..e72b96fe7cd1 100644
--- a/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
+++ b/flang/test/lib/OpenACC/TestOpenACCInterfaces.cpp
@@ -6,11 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
+#include "flang/Optimizer/Dialect/FIRDialect.h"
+#include "flang/Optimizer/HLFIR/HLFIRDialect.h"
 #include "flang/Optimizer/Support/DataLayout.h"
 
 using namespace mlir;
@@ -25,6 +29,11 @@ struct TestFIROpenACCInterfaces
   StringRef getDescription() const final {
     return "Test FIR implementation of the OpenACC interfaces.";
   }
+  void getDependentDialects(::mlir::DialectRegistry &registry) const override {
+    registry.insert<fir::FIROpsDialect, hlfir::hlfirDialect,
+        mlir::arith::ArithDialect, mlir::acc::OpenACCDialect,
+        mlir::DLTIDialect>();
+  }
   void runOnOperation() override {
     mlir::ModuleOp mod = getOperation();
     auto datalayout =
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index 10d8979dd1f4..5e6bd6609a33 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -441,6 +441,7 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
   loweringOptions.setStackRepackArrays(stackRepackArrays);
   loweringOptions.setRepackArrays(repackArrays);
   loweringOptions.setRepackArraysWhole(repackArraysWhole);
+  loweringOptions.setSkipExternalRttiDefinition(skipExternalRttiDefinition);
   if (enableCUDA)
     loweringOptions.setCUDARuntimeCheck(true);
   std::vector<Fortran::lower::EnvironmentDefault> envDefaults = {};
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 092eff5cc771..281c5adbfc87 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -22,6 +22,7 @@ set(MODULES_WITHOUT_IMPLEMENTATION
   "mma"
   "__cuda_builtins"
   "__cuda_device"
+  "cooperative_groups"
   "cudadevice"
   "ieee_arithmetic"
   "ieee_exceptions"
@@ -70,12 +71,17 @@ if (NOT CMAKE_CROSSCOMPILING)
     elseif(${filename} STREQUAL "__ppc_intrinsics" OR
            ${filename} STREQUAL "mma")
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod)
-    elseif(${filename} STREQUAL "__cuda_device")
+    elseif(${filename} STREQUAL "__cuda_device" OR
+           ${filename} STREQUAL "cudadevice" OR
+           ${filename} STREQUAL "cooperative_groups")
       set(opts -fc1 -xcuda)
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
-    elseif(${filename} STREQUAL "cudadevice")
-      set(opts -fc1 -xcuda)
-      set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
+      if(${filename} STREQUAL "__cuda_device")
+        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_builtins.mod)
+      elseif(${filename} STREQUAL "cudadevice")
+        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__cuda_device.mod)
+      elseif(${filename} STREQUAL "cooperative_groups")
+        set(depends ${FLANG_INTRINSIC_MODULES_DIR}/cudadevice.mod)
+      endif()
     else()
       set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod)
       if(${filename} STREQUAL "iso_fortran_env")
diff --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index eaf4bae08845..806cd3148d09 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -70,6 +70,11 @@ static cl::opt<bool> codeGenLLVM(
     cl::desc("Run only CodeGen passes and translate FIR to LLVM IR"),
     cl::init(false));
 
+static cl::opt<bool> emitFinalMLIR(
+    "emit-final-mlir",
+    cl::desc("Only translate FIR to MLIR, do not lower to LLVM IR"),
+    cl::init(false));
+
 #include "flang/Optimizer/Passes/CommandLineOpts.h"
 #include "flang/Optimizer/Passes/Pipelines.h"
 
@@ -149,13 +154,15 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
       fir::registerDefaultInlinerPass(config);
       fir::createMLIRToLLVMPassPipeline(pm, config);
     }
-    fir::addLLVMDialectToLLVMPass(pm, out.os());
+    if (!emitFinalMLIR)
+      fir::addLLVMDialectToLLVMPass(pm, out.os());
   }
 
   // run the pass manager
   if (mlir::succeeded(pm.run(*owningRef))) {
     // passes ran successfully, so keep the output
-    if ((emitFir || passPipeline.hasAnyOccurrences()) && !codeGenLLVM)
+    if ((emitFir || passPipeline.hasAnyOccurrences() || emitFinalMLIR) &&
+        !codeGenLLVM)
       printModule(*owningRef, out.os());
     out.keep();
     return mlir::success();
diff --git a/flang/unittests/Common/FortranFeaturesTest.cpp b/flang/unittests/Common/FortranFeaturesTest.cpp
index afd677f69a16..9408da0361e1 100644
--- a/flang/unittests/Common/FortranFeaturesTest.cpp
+++ b/flang/unittests/Common/FortranFeaturesTest.cpp
@@ -558,4 +558,12 @@ TEST(FortranFeaturesTest, CamelCaseToLowerCaseHyphenated) {
       "non-volatile-pointer-to-volatile");
 }
 
+TEST(FortranFeaturesTest, HintLanguageControlFlag) {
+  LanguageFeatureControl control{};
+  EXPECT_EQ(control.getDefaultCliSpelling(LanguageFeature::BenignNameClash),
+      "benign-name-clash");
+  EXPECT_EQ(
+      control.getDefaultCliSpelling(UsageWarning::Portability), "portability");
+}
+
 } // namespace Fortran::common::details
diff --git a/libc/config/gpu/amdgpu/entrypoints.txt b/libc/config/gpu/amdgpu/entrypoints.txt
index f41ebdc59f6b..463727b9bb32 100644
--- a/libc/config/gpu/amdgpu/entrypoints.txt
+++ b/libc/config/gpu/amdgpu/entrypoints.txt
@@ -511,14 +511,25 @@ set(TARGET_LIBM_ENTRYPOINTS
 if(LIBC_TYPES_HAS_FLOAT16)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float16 entrypoints
+    libc.src.math.acosf16
+    libc.src.math.acoshf16
+    libc.src.math.acospif16
+    libc.src.math.asinf16
+    libc.src.math.asinhf16
+    libc.src.math.atanf16
+    libc.src.math.atanhf16
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
+    libc.src.math.cosf16
     libc.src.math.coshf16
+    libc.src.math.cospif16
     libc.src.math.exp10f16
     libc.src.math.exp10m1f16
     libc.src.math.exp2f16
+    # libc.src.math.exp2m1f16
     libc.src.math.expf16
+    libc.src.math.expm1f16
     libc.src.math.f16add
     libc.src.math.f16addf
     libc.src.math.f16addl
@@ -556,7 +567,10 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fromfpf16
     libc.src.math.fromfpxf16
     libc.src.math.getpayloadf16
+    libc.src.math.hypotf16
     libc.src.math.ilogbf16
+    libc.src.math.iscanonicalf16
+    libc.src.math.issignalingf16
     libc.src.math.ldexpf16
     libc.src.math.llogbf16
     libc.src.math.llrintf16
@@ -583,9 +597,13 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.scalbnf16
     libc.src.math.setpayloadf16
     libc.src.math.setpayloadsigf16
+    libc.src.math.sinf16
     libc.src.math.sinhf16
+    libc.src.math.sinpif16
     libc.src.math.sqrtf16
+    libc.src.math.tanf16
     libc.src.math.tanhf16
+    libc.src.math.tanpif16
     libc.src.math.totalorderf16
     libc.src.math.totalordermagf16
     libc.src.math.truncf16
diff --git a/libc/config/gpu/nvptx/entrypoints.txt b/libc/config/gpu/nvptx/entrypoints.txt
index 2b8c29fda3dc..13b77172dcf5 100644
--- a/libc/config/gpu/nvptx/entrypoints.txt
+++ b/libc/config/gpu/nvptx/entrypoints.txt
@@ -513,14 +513,25 @@ set(TARGET_LIBM_ENTRYPOINTS
 if(LIBC_TYPES_HAS_FLOAT16)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float16 entrypoints
+    libc.src.math.acosf16
+    libc.src.math.acoshf16
+    libc.src.math.acospif16
+    libc.src.math.asinf16
+    libc.src.math.asinhf16
+    libc.src.math.atanf16
+    libc.src.math.atanhf16
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
+    libc.src.math.cosf16
     libc.src.math.coshf16
+    libc.src.math.cospif16
     libc.src.math.exp10f16
     libc.src.math.exp10m1f16
     libc.src.math.exp2f16
+    libc.src.math.exp2m1f16
     libc.src.math.expf16
+    libc.src.math.expm1f16
     libc.src.math.f16add
     libc.src.math.f16addf
     libc.src.math.f16addl
@@ -558,7 +569,10 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.fromfpf16
     libc.src.math.fromfpxf16
     libc.src.math.getpayloadf16
+    libc.src.math.hypotf16
     libc.src.math.ilogbf16
+    libc.src.math.iscanonicalf16
+    libc.src.math.issignalingf16
     libc.src.math.ldexpf16
     libc.src.math.llogbf16
     libc.src.math.llrintf16
@@ -585,9 +599,13 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.scalbnf16
     libc.src.math.setpayloadf16
     libc.src.math.setpayloadsigf16
+    libc.src.math.sinf16
     libc.src.math.sinhf16
+    libc.src.math.sinpif16
     libc.src.math.sqrtf16
+    libc.src.math.tanf16
     libc.src.math.tanhf16
+    libc.src.math.tanpif16
     libc.src.math.totalorderf16
     libc.src.math.totalordermagf16
     libc.src.math.truncf16
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 1161ae260be2..fa62f4d290c3 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -201,6 +201,8 @@ if(LLVM_LIBC_FULL_BUILD)
     # setjmp.h entrypoints
     libc.src.setjmp.longjmp
     libc.src.setjmp.setjmp
+    libc.src.setjmp.siglongjmp
+    libc.src.setjmp.sigsetjmp
   )
 endif()
 
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index c8a6d6e648af..59c248871f83 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -366,6 +366,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     # wchar.h entrypoints
     libc.src.wchar.btowc
     libc.src.wchar.wcslen
+    libc.src.wchar.wcsnlen
     libc.src.wchar.wctob
     libc.src.wchar.wmemmove
     libc.src.wchar.wmemset
@@ -375,6 +376,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wcspbrk
     libc.src.wchar.wcsrchr
     libc.src.wchar.wcsspn
+    libc.src.wchar.wcscspn
     libc.src.wchar.wmemcmp
     libc.src.wchar.wmempcpy
     libc.src.wchar.wmemcpy
@@ -386,6 +388,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.wchar.wmemchr
     libc.src.wchar.wcpcpy
     libc.src.wchar.wcpncpy
+    libc.src.wchar.wcstok
 
     # sys/uio.h entrypoints
     libc.src.sys.uio.writev
@@ -1251,6 +1254,7 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.wchar.mbrtowc
     libc.src.wchar.mbtowc
     libc.src.wchar.wcrtomb
+    libc.src.wchar.wctomb
   )
 endif()
 
diff --git a/libc/include/llvm-libc-types/jmp_buf.h b/libc/include/llvm-libc-types/jmp_buf.h
index 2fec35dbc916..8a7d2839e21d 100644
--- a/libc/include/llvm-libc-types/jmp_buf.h
+++ b/libc/include/llvm-libc-types/jmp_buf.h
@@ -13,7 +13,7 @@
 // Issue: https://github.com/llvm/llvm-project/issues/136358
 #if defined(__linux__)
 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) ||        \
-    defined(__riscv)
+    defined(__arm__) || defined(__riscv)
 #define __LIBC_HAS_SIGJMP_BUF
 #endif
 #endif
diff --git a/libc/include/wchar.yaml b/libc/include/wchar.yaml
index 98cb3bdaf0ac..02e1ba331b7c 100644
--- a/libc/include/wchar.yaml
+++ b/libc/include/wchar.yaml
@@ -17,6 +17,13 @@ functions:
     return_type: size_t
     arguments:
       - type: const wchar_t *
+  - name: wcsnlen
+    standards:
+      - stdc
+    return_type: size_t
+    arguments:
+      - type: const wchar_t *
+      - type: size_t
   - name: wctob
     standards:
       - stdc
@@ -97,6 +104,13 @@ functions:
     arguments:
       - type: const wchar_t *
       - type: const wchar_t *
+  - name: wcscspn
+    standards:
+      - stdc
+    return_type: size_t
+    arguments:
+      - type: const wchar_t *
+      - type: const wchar_t *
   - name: wmemcmp
     standards:
       - stdc
@@ -175,6 +189,13 @@ functions:
       - type: char *__restrict
       - type: wchar_t
       - type: mbstate_t *__restrict
+  - name: wctomb
+    standards:
+      - stdc
+    return_type: int
+    arguments:
+      - type: char *
+      - type: wchar_t
   - name: wcscpy
     standards:
       - stdc
@@ -182,6 +203,14 @@ functions:
     arguments:
       - type: wchar_t *__restrict
       - type: const wchar_t *__restrict
+  - name: wcstok
+    standards:
+      - stdc
+    return_type: wchar_t *
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const wchar_t *__restrict
+      - type: wchar_t** __restrict
   - name: wcpcpy
     standards:
       - stdc
diff --git a/libc/src/__support/CPP/new.h b/libc/src/__support/CPP/new.h
index 8694d9c47550..fe36de29468a 100644
--- a/libc/src/__support/CPP/new.h
+++ b/libc/src/__support/CPP/new.h
@@ -29,6 +29,14 @@ enum class align_val_t : size_t {};
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace cpp {
+template <class T> [[nodiscard]] constexpr T *launder(T *p) {
+  static_assert(__has_builtin(__builtin_launder),
+                "cpp::launder requires __builtin_launder");
+  return __builtin_launder(p);
+}
+} // namespace cpp
+
 class AllocChecker {
   bool success = false;
 
diff --git a/libc/src/__support/GPU/allocator.cpp b/libc/src/__support/GPU/allocator.cpp
index 66ab155e5c29..7923fbb2c1c2 100644
--- a/libc/src/__support/GPU/allocator.cpp
+++ b/libc/src/__support/GPU/allocator.cpp
@@ -22,6 +22,7 @@
 #include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/threads/sleep.h"
+#include "src/string/memory_utils/inline_memcpy.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -137,6 +138,11 @@ void uniform_memset(uint32_t *s, uint32_t c, uint32_t n, uint64_t uniform) {
     s[i] = c;
 }
 
+// Indicates that the provided value is a power of two.
+static inline constexpr bool is_pow2(uint64_t x) {
+  return x && (x & (x - 1)) == 0;
+}
+
 } // namespace impl
 
 /// A slab allocator used to hand out identically sized slabs of memory.
@@ -183,7 +189,9 @@ struct Slab {
 
   // Get the number of bytes needed to contain the bitfield bits.
   constexpr static uint32_t bitfield_bytes(uint32_t chunk_size) {
-    return ((num_chunks(chunk_size) + BITS_IN_WORD - 1) / BITS_IN_WORD) * 8;
+    return __builtin_align_up(
+        ((num_chunks(chunk_size) + BITS_IN_WORD - 1) / BITS_IN_WORD) * 8,
+        MIN_ALIGNMENT + 1);
   }
 
   // The actual amount of memory available excluding the bitfield and metadata.
@@ -540,15 +548,58 @@ void deallocate(void *ptr) {
     return;
 
   // All non-slab allocations will be aligned on a 2MiB boundary.
-  if ((reinterpret_cast<uintptr_t>(ptr) & SLAB_ALIGNMENT) == 0)
+  if (__builtin_is_aligned(ptr, SLAB_ALIGNMENT + 1))
     return impl::rpc_free(ptr);
 
   // The original slab pointer is the 2MiB boundary using the given pointer.
-  Slab *slab = reinterpret_cast<Slab *>(
-      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT));
+  Slab *slab = cpp::launder(reinterpret_cast<Slab *>(
+      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT)));
   slab->deallocate(ptr);
   release_slab(slab);
 }
 
+void *reallocate(void *ptr, uint64_t size) {
+  if (ptr == nullptr)
+    return gpu::allocate(size);
+
+  // Non-slab allocations are considered foreign pointers so we fail.
+  if (__builtin_is_aligned(ptr, SLAB_ALIGNMENT + 1))
+    return nullptr;
+
+  // The original slab pointer is the 2MiB boundary using the given pointer.
+  Slab *slab = cpp::launder(reinterpret_cast<Slab *>(
+      (reinterpret_cast<uintptr_t>(ptr) & ~SLAB_ALIGNMENT)));
+  if (slab->get_chunk_size() >= size)
+    return ptr;
+
+  // If we need a new chunk we reallocate and copy it over.
+  void *new_ptr = gpu::allocate(size);
+  inline_memcpy(new_ptr, ptr, slab->get_chunk_size());
+  gpu::deallocate(ptr);
+  return new_ptr;
+}
+
+void *aligned_allocate(uint32_t alignment, uint64_t size) {
+  // All alignment values must be a non-zero power of two.
+  if (!impl::is_pow2(alignment))
+    return nullptr;
+
+  // If the requested alignment is less than what we already provide this is
+  // just a normal allocation.
+  if (alignment <= MIN_ALIGNMENT + 1)
+    return gpu::allocate(size);
+
+  // We can't handle alignments greater than 2MiB so we simply fail.
+  if (alignment > SLAB_ALIGNMENT + 1)
+    return nullptr;
+
+  // Trying to handle allocation internally would break the assumption that each
+  // chunk is identical to eachother. Allocate enough memory with worst-case
+  // alignment and then round up. The index logic will round down properly.
+  uint64_t rounded = size + alignment - MIN_ALIGNMENT;
+  void *ptr = gpu::allocate(rounded);
+  return __builtin_align_up(ptr, alignment);
+}
+
 } // namespace gpu
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/GPU/allocator.h b/libc/src/__support/GPU/allocator.h
index 466009aa7198..a7cf8bceef27 100644
--- a/libc/src/__support/GPU/allocator.h
+++ b/libc/src/__support/GPU/allocator.h
@@ -17,6 +17,8 @@ namespace gpu {
 
 void *allocate(uint64_t size);
 void deallocate(void *ptr);
+void *reallocate(void *ptr, uint64_t size);
+void *aligned_allocate(uint32_t alignment, uint64_t size);
 
 } // namespace gpu
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/macros/optimization.h b/libc/src/__support/macros/optimization.h
index db008d323b3a..250a9e060728 100644
--- a/libc/src/__support/macros/optimization.h
+++ b/libc/src/__support/macros/optimization.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
 #define LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
 
-#include "src/__support/macros/attributes.h"          // LIBC_INLINE
+#include "src/__support/macros/attributes.h" // LIBC_INLINE
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/compiler.h" // LIBC_COMPILER_IS_CLANG
 
@@ -30,8 +30,10 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 
 #if defined(LIBC_COMPILER_IS_CLANG)
 #define LIBC_LOOP_NOUNROLL _Pragma("nounroll")
+#define LIBC_LOOP_UNROLL _Pragma("unroll")
 #elif defined(LIBC_COMPILER_IS_GCC)
 #define LIBC_LOOP_NOUNROLL _Pragma("GCC unroll 0")
+#define LIBC_LOOP_UNROLL _Pragma("GCC unroll 2048")
 #else
 #error "Unhandled compiler"
 #endif
diff --git a/libc/src/__support/macros/properties/cpu_features.h b/libc/src/__support/macros/properties/cpu_features.h
index 457a2b7869d4..cdb2df97b2b9 100644
--- a/libc/src/__support/macros/properties/cpu_features.h
+++ b/libc/src/__support/macros/properties/cpu_features.h
@@ -61,7 +61,7 @@
 
 #if defined(__riscv_flen)
 // https://github.com/riscv-non-isa/riscv-c-api-doc/blob/main/src/c-api.adoc
-#if (__riscv_arch_test && __riscv_zfhmin)
+#if defined(__riscv_zfhmin)
 #define LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF
 #define LIBC_TARGET_CPU_HAS_FPU_HALF
 #endif // LIBC_TARGET_CPU_HAS_RISCV_FPU_HALF
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index 6aade4ccc84a..d3fb58ed0c71 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -13,6 +13,7 @@ add_object_library(
   SRCS 
     character_converter.cpp
   DEPENDS
+    libc.hdr.errno_macros
     libc.hdr.types.char8_t
     libc.hdr.types.char32_t
     libc.src.__support.error_or
@@ -27,6 +28,7 @@ add_object_library(
   SRCS 
     wcrtomb.cpp
   DEPENDS
+    libc.hdr.errno_macros
     libc.hdr.types.char32_t
     libc.hdr.types.size_t
     libc.hdr.types.wchar_t
@@ -43,6 +45,7 @@ add_object_library(
   SRCS
     mbrtowc.cpp
   DEPENDS
+  libc.hdr.errno_macros
   libc.hdr.types.wchar_t
   libc.hdr.types.size_t
   libc.src.__support.common
diff --git a/libc/src/__support/wchar/character_converter.cpp b/libc/src/__support/wchar/character_converter.cpp
index 1f81de4248ff..3cacfa5689e4 100644
--- a/libc/src/__support/wchar/character_converter.cpp
+++ b/libc/src/__support/wchar/character_converter.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/errno_macros.h"
 #include "hdr/types/char32_t.h"
 #include "hdr/types/char8_t.h"
 #include "src/__support/CPP/bit.h"
@@ -25,6 +26,9 @@ constexpr size_t ENCODED_BITS_PER_UTF8 = 6;
 // Information not metadata (# of bits excluding the byte headers)
 constexpr uint32_t MASK_ENCODED_BITS =
     mask_trailing_ones<uint32_t, ENCODED_BITS_PER_UTF8>();
+// Maximum value for utf-32 for a utf-8 sequence of a given length
+constexpr char32_t MAX_VALUE_PER_UTF8_LEN[] = {0x7f, 0x7ff, 0xffff, 0x10ffff};
+constexpr int MAX_UTF8_LENGTH = 4;
 
 CharacterConverter::CharacterConverter(mbstate *mbstate) { state = mbstate; }
 
@@ -40,6 +44,17 @@ bool CharacterConverter::isFull() {
 
 bool CharacterConverter::isEmpty() { return state->bytes_stored == 0; }
 
+bool CharacterConverter::isValidState() {
+  if (state->total_bytes > MAX_UTF8_LENGTH)
+    return false;
+
+  const char32_t max_utf32_value =
+      state->total_bytes == 0 ? 0
+                              : MAX_VALUE_PER_UTF8_LEN[state->total_bytes - 1];
+  return state->bytes_stored <= state->total_bytes &&
+         state->partial <= max_utf32_value;
+}
+
 int CharacterConverter::push(char8_t utf8_byte) {
   uint8_t num_ones = static_cast<uint8_t>(cpp::countl_one(utf8_byte));
   // Checking the first byte if first push
@@ -62,7 +77,7 @@ int CharacterConverter::push(char8_t utf8_byte) {
     else {
       // bytes_stored and total_bytes will always be 0 here
       state->partial = static_cast<char32_t>(0);
-      return -1;
+      return EILSEQ;
     }
     state->partial = static_cast<char32_t>(utf8_byte);
     state->bytes_stored++;
@@ -79,7 +94,7 @@ int CharacterConverter::push(char8_t utf8_byte) {
   }
   // Invalid byte -> reset the state
   clear();
-  return -1;
+  return EILSEQ;
 }
 
 int CharacterConverter::push(char32_t utf32) {
@@ -90,9 +105,7 @@ int CharacterConverter::push(char32_t utf32) {
   state->partial = utf32;
 
   // determine number of utf-8 bytes needed to represent this utf32 value
-  constexpr char32_t MAX_VALUE_PER_UTF8_LEN[] = {0x7f, 0x7ff, 0xffff, 0x10ffff};
-  constexpr int NUM_RANGES = 4;
-  for (uint8_t i = 0; i < NUM_RANGES; i++) {
+  for (uint8_t i = 0; i < MAX_UTF8_LENGTH; i++) {
     if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i]) {
       state->total_bytes = i + 1;
       state->bytes_stored = i + 1;
@@ -103,7 +116,7 @@ int CharacterConverter::push(char32_t utf32) {
   // `utf32` contains a value that is too large to actually represent a valid
   // unicode character
   clear();
-  return -1;
+  return EILSEQ;
 }
 
 ErrorOr<char32_t> CharacterConverter::pop_utf32() {
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index be0e6129df23..d9a63fdc0522 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -28,6 +28,7 @@ class CharacterConverter {
   void clear();
   bool isFull();
   bool isEmpty();
+  bool isValidState();
 
   int push(char8_t utf8_byte);
   int push(char32_t utf32);
diff --git a/libc/src/__support/wchar/mbrtowc.cpp b/libc/src/__support/wchar/mbrtowc.cpp
index 954c7458f4df..90ba934c42b6 100644
--- a/libc/src/__support/wchar/mbrtowc.cpp
+++ b/libc/src/__support/wchar/mbrtowc.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/__support/wchar/mbrtowc.h"
+#include "hdr/errno_macros.h"
 #include "hdr/types/mbstate_t.h"
 #include "hdr/types/size_t.h"
 #include "hdr/types/wchar_t.h"
@@ -22,6 +23,8 @@ namespace internal {
 ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict s,
                         size_t n, mbstate *__restrict ps) {
   CharacterConverter char_conv(ps);
+  if (!char_conv.isValidState())
+    return Error(EINVAL);
   if (s == nullptr)
     return 0;
   size_t i = 0;
@@ -29,8 +32,8 @@ ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict s,
   for (; i < n && !char_conv.isFull(); ++i) {
     int err = char_conv.push(static_cast<char8_t>(s[i]));
     // Encoding error
-    if (err == -1)
-      return Error(-1);
+    if (err == EILSEQ)
+      return Error(err);
   }
   auto wc = char_conv.pop_utf32();
   if (wc.has_value()) {
diff --git a/libc/src/__support/wchar/wcrtomb.cpp b/libc/src/__support/wchar/wcrtomb.cpp
index 8ca3d17ad6ce..fc54bbfc93e0 100644
--- a/libc/src/__support/wchar/wcrtomb.cpp
+++ b/libc/src/__support/wchar/wcrtomb.cpp
@@ -11,6 +11,7 @@
 #include "src/__support/wchar/character_converter.h"
 #include "src/__support/wchar/mbstate.h"
 
+#include "hdr/errno_macros.h"
 #include "hdr/types/char32_t.h"
 #include "hdr/types/size_t.h"
 #include "hdr/types/wchar_t.h"
@@ -26,8 +27,8 @@ ErrorOr<size_t> wcrtomb(char *__restrict s, wchar_t wc,
 
   CharacterConverter cr(ps);
 
-  if (s == nullptr)
-    return Error(-1);
+  if (!cr.isValidState())
+    return Error(EINVAL);
 
   int status = cr.push(static_cast<char32_t>(wc));
   if (status != 0)
diff --git a/libc/src/setjmp/arm/CMakeLists.txt b/libc/src/setjmp/arm/CMakeLists.txt
index 55c80b0bede0..77f8471c06bb 100644
--- a/libc/src/setjmp/arm/CMakeLists.txt
+++ b/libc/src/setjmp/arm/CMakeLists.txt
@@ -8,6 +8,22 @@ add_entrypoint_object(
     libc.hdr.types.jmp_buf
 )
 
+if (TARGET libc.src.setjmp.sigsetjmp_epilogue)
+  add_entrypoint_object(
+    sigsetjmp
+    SRCS
+      sigsetjmp.cpp
+    HDRS
+      ../sigsetjmp.h
+    DEPENDS
+      libc.hdr.types.jmp_buf
+      libc.hdr.types.sigset_t
+      libc.hdr.offsetof_macros
+      libc.src.setjmp.sigsetjmp_epilogue
+      libc.src.setjmp.setjmp
+  )
+endif()
+
 add_entrypoint_object(
   longjmp
   SRCS
diff --git a/libc/src/setjmp/arm/sigsetjmp.cpp b/libc/src/setjmp/arm/sigsetjmp.cpp
new file mode 100644
index 000000000000..41c86bda5d6c
--- /dev/null
+++ b/libc/src/setjmp/arm/sigsetjmp.cpp
@@ -0,0 +1,65 @@
+//===-- Implementation of sigsetjmp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/setjmp/sigsetjmp.h"
+#include "hdr/offsetof_macros.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/setjmp/setjmp_impl.h"
+#include "src/setjmp/sigsetjmp_epilogue.h"
+
+namespace LIBC_NAMESPACE_DECL {
+[[gnu::naked]]
+LLVM_LIBC_FUNCTION(int, sigsetjmp, (sigjmp_buf buf)) {
+#if defined(__thumb__) && __ARM_ARCH_ISA_THUMB == 1
+  // Thumb1 does not support the high registers > r7 in stmia, so move them
+  // into lower GPRs first.
+  asm(R"(
+      tst r1, r1
+      bne .Ldosave
+      b %c[setjmp]
+.Ldosave:
+      str r4, [r0, #%c[extra]]
+      mov r4, lr
+      str r4, [r0, #%c[retaddr]]
+      mov r4, r0
+      bl %c[setjmp]
+      mov r1, r0
+      mov r0, r4
+      ldr r4, [r0, #%c[retaddr]]
+      mov lr, r4
+      ldr r4, [r0, #%c[extra]]
+      b %c[epilogue]
+  )" ::[retaddr] "i"(offsetof(__jmp_buf, sig_retaddr)),
+      [extra] "i"(offsetof(__jmp_buf, sig_extra)), [setjmp] "i"(setjmp),
+      [epilogue] "i"(sigsetjmp_epilogue)
+      : "r0", "r1", "r4");
+#else
+  // Some thumb2 linkers do not support conditional branch to PLT.
+  // We branch to local labels instead.
+  asm(R"(
+      tst r1, r1
+      bne .Ldosave
+      b %c[setjmp]
+.Ldosave:
+      str r4, [r0, #%c[extra]]
+      str lr, [r0, #%c[retaddr]]
+      mov r4, r0
+      bl %c[setjmp]
+      mov r1, r0
+      mov r0, r4
+      ldr lr, [r0, #%c[retaddr]]
+      ldr r4, [r0, #%c[extra]]
+      b %c[epilogue]
+  )" ::[retaddr] "i"(offsetof(__jmp_buf, sig_retaddr)),
+      [extra] "i"(offsetof(__jmp_buf, sig_extra)), [setjmp] "X"(setjmp),
+      [epilogue] "X"(sigsetjmp_epilogue)
+      : "r0", "r1", "r4");
+#endif
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/gpu/aligned_alloc.cpp b/libc/src/stdlib/gpu/aligned_alloc.cpp
index cd2c7e55128f..34a7eae618fe 100644
--- a/libc/src/stdlib/gpu/aligned_alloc.cpp
+++ b/libc/src/stdlib/gpu/aligned_alloc.cpp
@@ -15,15 +15,15 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, aligned_alloc, (size_t alignment, size_t size)) {
-  if ((alignment & -alignment) != alignment)
-    return nullptr;
-
-  void *ptr = gpu::allocate(size);
-  if ((reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)) != 0) {
-    gpu::deallocate(ptr);
-    return nullptr;
-  }
-  return ptr;
+  // FIXME: NVIDIA targets currently use the built-in 'malloc' which we cannot
+  // reason with. But we still need to provide this function for compatibility.
+#ifndef LIBC_TARGET_ARCH_IS_NVPTX
+  return gpu::aligned_allocate(static_cast<uint32_t>(alignment), size);
+#else
+  (void)alignment;
+  (void)size;
+  return nullptr;
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/stdlib/gpu/realloc.cpp b/libc/src/stdlib/gpu/realloc.cpp
index 4fd4d6b27817..97ad1b3bbeff 100644
--- a/libc/src/stdlib/gpu/realloc.cpp
+++ b/libc/src/stdlib/gpu/realloc.cpp
@@ -16,17 +16,15 @@
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(void *, realloc, (void *ptr, size_t size)) {
-  if (ptr == nullptr)
-    return gpu::allocate(size);
-
-  void *newmem = gpu::allocate(size);
-  if (newmem == nullptr)
-    return nullptr;
-
-  // This will copy garbage if it goes beyond the old allocation size.
-  inline_memcpy(newmem, ptr, size);
-  gpu::deallocate(ptr);
-  return newmem;
+  // FIXME: NVIDIA targets currently use the built-in 'malloc' which we cannot
+  // reason with. But we still need to provide this function for compatibility.
+#ifndef LIBC_TARGET_ARCH_IS_NVPTX
+  return gpu::reallocate(ptr, size);
+#else
+  (void)ptr;
+  (void)size;
+  return nullptr;
+#endif
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index 08c0b0d34d50..a967247db53f 100644
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -7,6 +7,7 @@ add_header_library(
     aarch64/inline_memcpy.h
     aarch64/inline_memmove.h
     aarch64/inline_memset.h
+    arm/inline_memcpy.h
     generic/aligned_access.h
     generic/byte_per_byte.h
     inline_bcmp.h
diff --git a/libc/src/string/memory_utils/arm/inline_memcpy.h b/libc/src/string/memory_utils/arm/inline_memcpy.h
new file mode 100644
index 000000000000..61efebe29b48
--- /dev/null
+++ b/libc/src/string/memory_utils/arm/inline_memcpy.h
@@ -0,0 +1,217 @@
+//===-- Memcpy implementation for arm ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
+
+#include "src/__support/macros/attributes.h"   // LIBC_INLINE
+#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
+#include "src/string/memory_utils/utils.h" // memcpy_inline, distance_to_align
+
+#include <stddef.h> // size_t
+
+// https://libc.llvm.org/compiler_support.html
+// Support for [[likely]] / [[unlikely]]
+//  [X] GCC 12.2
+//  [X] Clang 12
+//  [ ] Clang 11
+#define LIBC_ATTR_LIKELY [[likely]]
+#define LIBC_ATTR_UNLIKELY [[unlikely]]
+
+#if defined(LIBC_COMPILER_IS_CLANG)
+#if LIBC_COMPILER_CLANG_VER < 1200
+#undef LIBC_ATTR_LIKELY
+#undef LIBC_ATTR_UNLIKELY
+#define LIBC_ATTR_LIKELY
+#define LIBC_ATTR_UNLIKELY
+#endif
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace {
+
+LIBC_INLINE_VAR constexpr size_t kWordSize = sizeof(uint32_t);
+
+enum Strategy {
+  ForceWordLdStChain,
+  AssumeWordAligned,
+  AssumeUnaligned,
+};
+
+template <size_t bytes, Strategy strategy = AssumeUnaligned>
+LIBC_INLINE void copy_and_bump_pointers(Ptr &dst, CPtr &src) {
+  if constexpr (strategy == AssumeUnaligned) {
+    memcpy_inline<bytes>(assume_aligned<1>(dst), assume_aligned<1>(src));
+  } else if constexpr (strategy == AssumeWordAligned) {
+    static_assert(bytes >= kWordSize);
+    memcpy_inline<bytes>(assume_aligned<kWordSize>(dst),
+                         assume_aligned<kWordSize>(src));
+  } else if constexpr (strategy == ForceWordLdStChain) {
+    // We restrict loads/stores to 4 byte to prevent the use of load/store
+    // multiple (LDM, STM) and load/store double (LDRD, STRD). First, they may
+    // fault (see notes below) and second, they use more registers which in turn
+    // adds push/pop instructions in the hot path.
+    static_assert((bytes % kWordSize == 0) && (bytes >= kWordSize));
+    LIBC_LOOP_UNROLL
+    for (size_t i = 0; i < bytes / kWordSize; ++i) {
+      const size_t offset = i * kWordSize;
+      memcpy_inline<kWordSize>(dst + offset, src + offset);
+    }
+  }
+  // In the 1, 2, 4 byte copy case, the compiler can fold pointer offsetting
+  // into the load/store instructions.
+  // e.g.,
+  // ldrb  r3, [r1], #1
+  // strb  r3, [r0], #1
+  dst += bytes;
+  src += bytes;
+}
+
+LIBC_INLINE void copy_bytes_and_bump_pointers(Ptr &dst, CPtr &src,
+                                              const size_t size) {
+  LIBC_LOOP_NOUNROLL
+  for (size_t i = 0; i < size; ++i)
+    *dst++ = *src++;
+}
+
+template <size_t block_size, Strategy strategy>
+LIBC_INLINE void copy_blocks_and_update_args(Ptr &dst, CPtr &src,
+                                             size_t &size) {
+  LIBC_LOOP_NOUNROLL
+  for (size_t i = 0; i < size / block_size; ++i)
+    copy_and_bump_pointers<block_size, strategy>(dst, src);
+  // Update `size` once at the end instead of once per iteration.
+  size %= block_size;
+}
+
+LIBC_INLINE CPtr bitwise_or(CPtr a, CPtr b) {
+  return cpp::bit_cast<CPtr>(cpp::bit_cast<uintptr_t>(a) |
+                             cpp::bit_cast<uintptr_t>(b));
+}
+
+LIBC_INLINE auto misaligned(CPtr a) {
+  return distance_to_align_down<kWordSize>(a);
+}
+
+} // namespace
+
+// Implementation for Cortex-M0, M0+, M1.
+// Notes:
+// - It compiles down to 196 bytes, but 220 bytes when used through `memcpy`
+//   that also needs to return the `dst` ptr.
+// - These cores do not allow for unaligned loads/stores.
+// - When `src` and `dst` are coaligned, we start by aligning them and perform
+//   bulk copies. We let the compiler know the pointers are aligned so it can
+//   use load/store multiple (LDM, STM). This significantly increase throughput
+//   but it also requires more registers and push/pop instructions. This impacts
+//   latency for small size copies.
+// - When `src` and `dst` are misaligned, we align `dst` and recompose words
+//   using multiple aligned loads. `load_aligned` takes care of endianness
+//   issues.
+[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_low_end(Ptr dst, CPtr src,
+                                                            size_t size) {
+  if (size >= 8) {
+    if (const size_t offset = distance_to_align_up<kWordSize>(dst))
+      LIBC_ATTR_UNLIKELY {
+        copy_bytes_and_bump_pointers(dst, src, offset);
+        size -= offset;
+      }
+    const auto src_alignment = distance_to_align_down<kWordSize>(src);
+    if (src_alignment == 0)
+      LIBC_ATTR_LIKELY {
+        // Both `src` and `dst` are now word-aligned.
+        copy_blocks_and_update_args<64, AssumeWordAligned>(dst, src, size);
+        copy_blocks_and_update_args<16, AssumeWordAligned>(dst, src, size);
+        copy_blocks_and_update_args<4, AssumeWordAligned>(dst, src, size);
+      }
+    else {
+      // `dst` is aligned but `src` is not.
+      LIBC_LOOP_NOUNROLL
+      while (size >= kWordSize) {
+        // Recompose word from multiple loads depending on the alignment.
+        const uint32_t value =
+            src_alignment == 2
+                ? load_aligned<uint32_t, uint16_t, uint16_t>(src)
+                : load_aligned<uint32_t, uint8_t, uint16_t, uint8_t>(src);
+        memcpy_inline<kWordSize>(assume_aligned<kWordSize>(dst), &value);
+        dst += kWordSize;
+        src += kWordSize;
+        size -= kWordSize;
+      }
+    }
+    // Up to 3 bytes may still need to be copied.
+    // Handling them with the slow loop below.
+  }
+  copy_bytes_and_bump_pointers(dst, src, size);
+}
+
+// Implementation for Cortex-M3, M4, M7, M23, M33, M35P, M52 with hardware
+// support for unaligned loads and stores.
+// Notes:
+// - It compiles down to 266 bytes.
+// - `dst` and `src` are not `__restrict` to prevent the compiler from
+//   reordering loads/stores.
+// - We keep state variables to a strict minimum to keep everything in the free
+//   registers and prevent costly push / pop.
+// - If unaligned single loads/stores to normal memory are supported, unaligned
+//   accesses for load/store multiple (LDM, STM) and load/store double (LDRD,
+//   STRD) instructions are generally not supported and will still fault so we
+//   make sure to restrict unrolling to word loads/stores.
+[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm_mid_end(Ptr dst, CPtr src,
+                                                            size_t size) {
+  if (misaligned(bitwise_or(src, dst)))
+    LIBC_ATTR_UNLIKELY {
+      if (size < 8)
+        LIBC_ATTR_UNLIKELY {
+          if (size & 1)
+            copy_and_bump_pointers<1>(dst, src);
+          if (size & 2)
+            copy_and_bump_pointers<2>(dst, src);
+          if (size & 4)
+            copy_and_bump_pointers<4>(dst, src);
+          return;
+        }
+      if (misaligned(src))
+        LIBC_ATTR_UNLIKELY {
+          const size_t offset = distance_to_align_up<kWordSize>(dst);
+          if (offset & 1)
+            copy_and_bump_pointers<1>(dst, src);
+          if (offset & 2)
+            copy_and_bump_pointers<2>(dst, src);
+          size -= offset;
+        }
+    }
+  copy_blocks_and_update_args<64, ForceWordLdStChain>(dst, src, size);
+  copy_blocks_and_update_args<16, ForceWordLdStChain>(dst, src, size);
+  copy_blocks_and_update_args<4, AssumeUnaligned>(dst, src, size);
+  if (size & 1)
+    copy_and_bump_pointers<1>(dst, src);
+  if (size & 2)
+    LIBC_ATTR_UNLIKELY
+  copy_and_bump_pointers<2>(dst, src);
+}
+
+[[maybe_unused]] LIBC_INLINE void inline_memcpy_arm(void *__restrict dst_,
+                                                    const void *__restrict src_,
+                                                    size_t size) {
+  Ptr dst = cpp::bit_cast<Ptr>(dst_);
+  CPtr src = cpp::bit_cast<CPtr>(src_);
+#ifdef __ARM_FEATURE_UNALIGNED
+  return inline_memcpy_arm_mid_end(dst, src, size);
+#else
+  return inline_memcpy_arm_low_end(dst, src, size);
+#endif
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+// Cleanup local macros
+#undef LIBC_ATTR_LIKELY
+#undef LIBC_ATTR_UNLIKELY
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCPY_H
diff --git a/libc/src/string/memory_utils/inline_memcpy.h b/libc/src/string/memory_utils/inline_memcpy.h
index f98e55321a9b..13975e6b3bd0 100644
--- a/libc/src/string/memory_utils/inline_memcpy.h
+++ b/libc/src/string/memory_utils/inline_memcpy.h
@@ -22,6 +22,9 @@
 #include "src/string/memory_utils/x86_64/inline_memcpy.h"
 #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY                                    \
   inline_memcpy_x86_maybe_interpose_repmovsb
+#elif defined(LIBC_TARGET_ARCH_IS_ARM)
+#include "src/string/memory_utils/arm/inline_memcpy.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_arm
 #elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
 #include "src/string/memory_utils/aarch64/inline_memcpy.h"
 #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_aarch64
diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h
index bdf0b8652188..c08608c87bb2 100644
--- a/libc/src/string/memory_utils/utils.h
+++ b/libc/src/string/memory_utils/utils.h
@@ -101,7 +101,7 @@ LIBC_INLINE void memcpy_inline(void *__restrict dst,
 }
 
 using Ptr = cpp::byte *;        // Pointer to raw data.
-using CPtr = const cpp::byte *; // Const pointer to raw data.
+using CPtr = const cpp::byte *; // Pointer to const raw data.
 
 // This type makes sure that we don't accidentally promote an integral type to
 // another one. It is only constructible from the exact T type.
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index f390785e5817..f2f4b1d38f0f 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -10,6 +10,17 @@ add_entrypoint_object(
     libc.src.string.string_utils
 )
 
+add_entrypoint_object(
+  wcsnlen
+  SRCS
+    wcsnlen.cpp
+  HDRS
+    wcsnlen.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.types.wchar_t
+)
+
 add_entrypoint_object(
   wctob
   SRCS
@@ -34,6 +45,16 @@ add_entrypoint_object(
     libc.src.__support.wctype_utils
 )
 
+add_entrypoint_object(
+  wcstok
+  SRCS
+    wcstok.cpp
+  HDRS
+    wcstok.h
+  DEPENDS
+    libc.hdr.types.wchar_t
+)
+
 add_entrypoint_object(
   wcrtomb
   SRCS
@@ -48,6 +69,19 @@ add_entrypoint_object(
     libc.src.__support.wchar.mbstate
 )
 
+add_entrypoint_object(
+  wctomb
+  SRCS
+    wctomb.cpp
+  HDRS
+    wctomb.h
+  DEPENDS
+    libc.hdr.types.wchar_t
+    libc.src.__support.wchar.wcrtomb
+    libc.src.__support.wchar.mbstate
+    libc.src.__support.libc_errno
+)
+
 add_entrypoint_object(
   mbrtowc
   SRCS
@@ -179,6 +213,17 @@ add_entrypoint_object(
     libc.hdr.types.size_t
 )
 
+add_entrypoint_object(
+  wcscspn
+  SRCS
+    wcscspn.cpp
+  HDRS
+    wcscspn.h
+  DEPENDS
+    libc.hdr.wchar_macros
+    libc.hdr.types.size_t
+)
+
 add_entrypoint_object(
   wmemcmp
   SRCS
diff --git a/libc/src/wchar/mbrtowc.cpp b/libc/src/wchar/mbrtowc.cpp
index cd429ab8d30e..6ecf4828da9b 100644
--- a/libc/src/wchar/mbrtowc.cpp
+++ b/libc/src/wchar/mbrtowc.cpp
@@ -29,7 +29,7 @@ LLVM_LIBC_FUNCTION(size_t, mbrtowc,
                                    : reinterpret_cast<internal::mbstate *>(ps));
   if (!ret.has_value()) {
     // Encoding failure
-    libc_errno = EILSEQ;
+    libc_errno = ret.error();
     return -1;
   }
   return ret.value();
diff --git a/libc/src/wchar/wcpncpy.h b/libc/src/wchar/wcpncpy.h
index d817239d2d0b..1f67cd46f775 100644
--- a/libc/src/wchar/wcpncpy.h
+++ b/libc/src/wchar/wcpncpy.h
@@ -20,4 +20,4 @@ wchar_t *wcpncpy(wchar_t *__restrict ws1, const wchar_t *__restrict ws2,
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_WCHAR_WCPNsCPY_H
+#endif // LLVM_LIBC_SRC_WCHAR_WCPNCPY_H
diff --git a/libc/src/wchar/wcrtomb.cpp b/libc/src/wchar/wcrtomb.cpp
index 6d604a00599e..3e9df0599431 100644
--- a/libc/src/wchar/wcrtomb.cpp
+++ b/libc/src/wchar/wcrtomb.cpp
@@ -35,7 +35,7 @@ LLVM_LIBC_FUNCTION(size_t, wcrtomb,
                     : reinterpret_cast<internal::mbstate *>(ps));
 
   if (!result.has_value()) {
-    libc_errno = EILSEQ;
+    libc_errno = result.error();
     return -1;
   }
 
diff --git a/libc/src/wchar/wcscspn.cpp b/libc/src/wchar/wcscspn.cpp
new file mode 100644
index 000000000000..8869d84cdfde
--- /dev/null
+++ b/libc/src/wchar/wcscspn.cpp
@@ -0,0 +1,34 @@
+//===-- Implementation of wcscspn -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcscspn.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bool check(wchar_t c, const wchar_t *s2) {
+  for (int n = 0; s2[n]; ++n) {
+    if (s2[n] == c)
+      return false;
+  }
+  return true;
+}
+LLVM_LIBC_FUNCTION(size_t, wcscspn, (const wchar_t *s1, const wchar_t *s2)) {
+  size_t i = 0;
+  for (; s1[i]; ++i) {
+    if (!check(s1[i], s2))
+      return i;
+  }
+  return i;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcscspn.h b/libc/src/wchar/wcscspn.h
new file mode 100644
index 000000000000..cffc2b98c346
--- /dev/null
+++ b/libc/src/wchar/wcscspn.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcscspn ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
+#define LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t wcscspn(const wchar_t *s1, const wchar_t *s2);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSCSPN_H
diff --git a/libc/src/wchar/wcsnlen.cpp b/libc/src/wchar/wcsnlen.cpp
new file mode 100644
index 000000000000..4613006ff203
--- /dev/null
+++ b/libc/src/wchar/wcsnlen.cpp
@@ -0,0 +1,25 @@
+//===-- Implementation of wcsnlen -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcsnlen.h"
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(size_t, wcsnlen, (const wchar_t *src, size_t maxlen)) {
+  size_t i = 0;
+  for (; i < maxlen && src[i]; ++i)
+    ;
+  return i;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcsnlen.h b/libc/src/wchar/wcsnlen.h
new file mode 100644
index 000000000000..5a4c92d368a5
--- /dev/null
+++ b/libc/src/wchar/wcsnlen.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcsnlen ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSNLEN_H
+#define LLVM_LIBC_SRC_WCHAR_WCSNLEN_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t wcsnlen(const wchar_t *src, size_t maxlen);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSNLEN_H
diff --git a/libc/src/wchar/wcsspn.h b/libc/src/wchar/wcsspn.h
index 6dbe65d9a4a9..6e21cbe59457 100644
--- a/libc/src/wchar/wcsspn.h
+++ b/libc/src/wchar/wcsspn.h
@@ -19,4 +19,4 @@ size_t wcsspn(const wchar_t *s1, const wchar_t *s2);
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_WCHAR_WCSCHR_H
+#endif // LLVM_LIBC_SRC_WCHAR_WCSSPN_H
diff --git a/libc/src/wchar/wcstok.cpp b/libc/src/wchar/wcstok.cpp
new file mode 100644
index 000000000000..291efc15e158
--- /dev/null
+++ b/libc/src/wchar/wcstok.cpp
@@ -0,0 +1,50 @@
+//===-- Implementation of wcstok ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcstok.h"
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bool isADelimeter(wchar_t wc, const wchar_t *delimiters) {
+  for (const wchar_t *delim_ptr = delimiters; *delim_ptr != L'\0'; ++delim_ptr)
+    if (wc == *delim_ptr)
+      return true;
+  return false;
+}
+
+LLVM_LIBC_FUNCTION(wchar_t *, wcstok,
+                   (wchar_t *__restrict str, const wchar_t *__restrict delim,
+                    wchar_t **__restrict context)) {
+  if (str == nullptr) {
+    if (*context == nullptr)
+      return nullptr;
+
+    str = *context;
+  }
+
+  wchar_t *tok_start, *tok_end;
+  for (tok_start = str; *tok_start != L'\0' && isADelimeter(*tok_start, delim);
+       ++tok_start)
+    ;
+
+  for (tok_end = tok_start; *tok_end != L'\0' && !isADelimeter(*tok_end, delim);
+       ++tok_end)
+    ;
+
+  if (*tok_end != L'\0') {
+    *tok_end = L'\0';
+    ++tok_end;
+  }
+  *context = tok_end;
+  return *tok_start == L'\0' ? nullptr : tok_start;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcstok.h b/libc/src/wchar/wcstok.h
new file mode 100644
index 000000000000..5e673ff4e89b
--- /dev/null
+++ b/libc/src/wchar/wcstok.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wcstok ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+#define LLVM_LIBC_SRC_WCHAR_WCSTOK_H
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+wchar_t *wcstok(wchar_t *__restrict str, const wchar_t *__restrict delim,
+                wchar_t **__restrict context);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCSTOK_H
diff --git a/libc/src/wchar/wctomb.cpp b/libc/src/wchar/wctomb.cpp
new file mode 100644
index 000000000000..142302e6ae09
--- /dev/null
+++ b/libc/src/wchar/wctomb.cpp
@@ -0,0 +1,35 @@
+//===-- Implementation of wctomb ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wctomb.h"
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/mbstate.h"
+#include "src/__support/wchar/wcrtomb.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, wctomb, (char *s, wchar_t wc)) {
+  static internal::mbstate internal_mbstate;
+  if (s == nullptr)
+    return 0;
+
+  auto result = internal::wcrtomb(s, wc, &internal_mbstate);
+
+  if (!result.has_value()) { // invalid wide character
+    libc_errno = EILSEQ;
+    return -1;
+  }
+
+  return static_cast<int>(result.value());
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wctomb.h b/libc/src/wchar/wctomb.h
new file mode 100644
index 000000000000..02a34e5ad229
--- /dev/null
+++ b/libc/src/wchar/wctomb.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for wctomb ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCTOMB_H
+#define LLVM_LIBC_SRC_WCHAR_WCTOMB_H
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int wctomb(char *s, wchar_t wc);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCTOMB_H
diff --git a/libc/test/integration/src/stdlib/gpu/CMakeLists.txt b/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
index 26c877b1b6ae..5f9a215bb873 100644
--- a/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
+++ b/libc/test/integration/src/stdlib/gpu/CMakeLists.txt
@@ -17,6 +17,36 @@ if(NOT LIBC_TARGET_ARCHITECTURE_IS_NVPTX)
       --blocks 1024
   )
 
+  add_integration_test(
+    realloc
+    SUITE
+      stdlib-gpu-integration-tests
+    SRCS
+      realloc.cpp
+    DEPENDS
+      libc.src.stdlib.malloc
+      libc.src.stdlib.free
+      libc.src.stdlib.realloc
+    LOADER_ARGS
+      --threads 256
+      --blocks 1024
+  )
+
+  add_integration_test(
+    aligned_alloc
+    SUITE
+      stdlib-gpu-integration-tests
+    SRCS
+      aligned_alloc.cpp
+    DEPENDS
+      libc.src.stdlib.aligned_alloc
+      libc.src.stdlib.malloc
+      libc.src.stdlib.free
+    LOADER_ARGS
+      --threads 256
+      --blocks 128
+  )
+
   add_integration_test(
     malloc_stress
     SUITE
diff --git a/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp b/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp
new file mode 100644
index 000000000000..6e00eb86c680
--- /dev/null
+++ b/libc/test/integration/src/stdlib/gpu/aligned_alloc.cpp
@@ -0,0 +1,29 @@
+#include "test/IntegrationTest/test.h"
+
+#include "src/__support/GPU/utils.h"
+#include "src/stdlib/aligned_alloc.h" // Adjust path if needed
+#include "src/stdlib/free.h"
+
+using namespace LIBC_NAMESPACE;
+
+TEST_MAIN(int, char **, char **) {
+  // aligned_alloc with valid alignment and size
+  void *ptr = LIBC_NAMESPACE::aligned_alloc(32, 16);
+  EXPECT_NE(ptr, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(ptr, 32));
+
+  LIBC_NAMESPACE::free(ptr);
+
+  // aligned_alloc fails if alignment is not power of two
+  void *bad_align = LIBC_NAMESPACE::aligned_alloc(30, 99);
+  EXPECT_EQ(bad_align, nullptr);
+
+  // aligned_alloc with a divergent size.
+  size_t alignment = 1 << (__gpu_lane_id() % 8 + 1);
+  void *div =
+      LIBC_NAMESPACE::aligned_alloc(alignment, (gpu::get_thread_id() + 1) * 4);
+  EXPECT_NE(div, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(div, alignment));
+
+  return 0;
+}
diff --git a/libc/test/integration/src/stdlib/gpu/malloc.cpp b/libc/test/integration/src/stdlib/gpu/malloc.cpp
index 7880206b1aaa..a02a6749258c 100644
--- a/libc/test/integration/src/stdlib/gpu/malloc.cpp
+++ b/libc/test/integration/src/stdlib/gpu/malloc.cpp
@@ -24,6 +24,7 @@ TEST_MAIN(int, char **, char **) {
   int *divergent = reinterpret_cast<int *>(
       LIBC_NAMESPACE::malloc((gpu::get_thread_id() + 1) * 16));
   EXPECT_NE(divergent, nullptr);
+  EXPECT_TRUE(__builtin_is_aligned(divergent, 16));
   *divergent = 1;
   EXPECT_EQ(*divergent, 1);
   LIBC_NAMESPACE::free(divergent);
diff --git a/libc/test/integration/src/stdlib/gpu/realloc.cpp b/libc/test/integration/src/stdlib/gpu/realloc.cpp
new file mode 100644
index 000000000000..7cdf29489928
--- /dev/null
+++ b/libc/test/integration/src/stdlib/gpu/realloc.cpp
@@ -0,0 +1,44 @@
+#include "test/IntegrationTest/test.h"
+
+#include "src/__support/GPU/utils.h"
+#include "src/stdlib/free.h"
+#include "src/stdlib/malloc.h"
+#include "src/stdlib/realloc.h"
+
+using namespace LIBC_NAMESPACE;
+
+TEST_MAIN(int, char **, char **) {
+  // realloc(nullptr, size) is equivalent to malloc.
+  int *alloc = reinterpret_cast<int *>(LIBC_NAMESPACE::realloc(nullptr, 32));
+  EXPECT_NE(alloc, nullptr);
+  *alloc = 42;
+  EXPECT_EQ(*alloc, 42);
+
+  // realloc to same size returns the same pointer.
+  void *same = LIBC_NAMESPACE::realloc(alloc, 32);
+  EXPECT_EQ(same, alloc);
+  EXPECT_EQ(reinterpret_cast<int *>(same)[0], 42);
+
+  // realloc to smaller size returns same pointer.
+  void *smaller = LIBC_NAMESPACE::realloc(same, 16);
+  EXPECT_EQ(smaller, alloc);
+  EXPECT_EQ(reinterpret_cast<int *>(smaller)[0], 42);
+
+  // realloc to larger size returns new pointer and preserves contents.
+  int *larger = reinterpret_cast<int *>(LIBC_NAMESPACE::realloc(smaller, 128));
+  EXPECT_NE(larger, nullptr);
+  EXPECT_EQ(larger[0], 42);
+
+  // realloc works when called with a divergent size.
+  int *div = reinterpret_cast<int *>(
+      LIBC_NAMESPACE::malloc((gpu::get_thread_id() + 1) * 16));
+  EXPECT_NE(div, nullptr);
+  div[0] = static_cast<int>(gpu::get_thread_id());
+  int *div_realloc = reinterpret_cast<int *>(
+      LIBC_NAMESPACE::realloc(div, ((gpu::get_thread_id() + 1) * 32)));
+  EXPECT_NE(div_realloc, nullptr);
+  EXPECT_EQ(div_realloc[0], static_cast<int>(gpu::get_thread_id()));
+  LIBC_NAMESPACE::free(div_realloc);
+
+  return 0;
+}
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index a6a7bc4aa6f4..1ad523e14884 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -186,3 +186,45 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, CantPushMidConversion) {
   int err = cr.push(utf32);
   ASSERT_EQ(err, -1);
 }
+
+TEST(LlvmLibcCharacterConverterUTF32To8Test, InvalidState) {
+  LIBC_NAMESPACE::internal::mbstate s1;
+  LIBC_NAMESPACE::internal::CharacterConverter c1(&s1);
+  ASSERT_TRUE(c1.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s2{0, 2, 0};
+  LIBC_NAMESPACE::internal::CharacterConverter c2(&s2);
+  ASSERT_FALSE(c2.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s3{0x7f, 1, 1};
+  LIBC_NAMESPACE::internal::CharacterConverter c3(&s3);
+  ASSERT_TRUE(c3.isValidState());
+  LIBC_NAMESPACE::internal::mbstate s4{0x80, 1, 1};
+  LIBC_NAMESPACE::internal::CharacterConverter c4(&s4);
+  ASSERT_FALSE(c4.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s5{0x7ff, 1, 2};
+  LIBC_NAMESPACE::internal::CharacterConverter c5(&s5);
+  ASSERT_TRUE(c5.isValidState());
+  LIBC_NAMESPACE::internal::mbstate s6{0x800, 1, 2};
+  LIBC_NAMESPACE::internal::CharacterConverter c6(&s6);
+  ASSERT_FALSE(c6.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s7{0xffff, 1, 3};
+  LIBC_NAMESPACE::internal::CharacterConverter c7(&s7);
+  ASSERT_TRUE(c7.isValidState());
+  LIBC_NAMESPACE::internal::mbstate s8{0x10000, 1, 3};
+  LIBC_NAMESPACE::internal::CharacterConverter c8(&s8);
+  ASSERT_FALSE(c8.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s9{0x10ffff, 1, 4};
+  LIBC_NAMESPACE::internal::CharacterConverter c9(&s9);
+  ASSERT_TRUE(c9.isValidState());
+  LIBC_NAMESPACE::internal::mbstate s10{0x110000, 1, 2};
+  LIBC_NAMESPACE::internal::CharacterConverter c10(&s10);
+  ASSERT_FALSE(c10.isValidState());
+
+  LIBC_NAMESPACE::internal::mbstate s11{0, 0, 5};
+  LIBC_NAMESPACE::internal::CharacterConverter c11(&s11);
+  ASSERT_FALSE(c11.isValidState());
+}
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index 36ae7d689cc0..b419fb5d5541 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/errno_macros.h"
 #include "src/__support/error_or.h"
 #include "src/__support/wchar/character_converter.h"
 #include "src/__support/wchar/mbstate.h"
@@ -87,7 +88,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidByte) {
   LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
   int err = char_conv.push(static_cast<char8_t>(ch));
 
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 }
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
@@ -100,12 +101,12 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
 
   LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
   int err = char_conv.push(static_cast<char8_t>(ch[0]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
   err = char_conv.push(static_cast<char8_t>(ch[1]));
   ASSERT_EQ(err, 0);
   // Prev byte was single byte so trying to push another should error.
   err = char_conv.push(static_cast<char8_t>(ch[2]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
   err = char_conv.push(static_cast<char8_t>(ch[3]));
   ASSERT_EQ(err, 0);
 }
@@ -127,7 +128,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidLastByte) {
   err = char_conv.push(static_cast<char8_t>(ch[2]));
   ASSERT_EQ(err, 0);
   err = char_conv.push(static_cast<char8_t>(ch[3]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 }
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
@@ -144,7 +145,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
   ASSERT_EQ(err, 0);
   // Should produce an error on 3rd byte
   err = char_conv.push(static_cast<char8_t>(ch[2]));
-  ASSERT_EQ(err, -1);
+  ASSERT_EQ(err, EILSEQ);
 
   // Should produce an error since mbstate was reset
   auto wch = char_conv.pop_utf32();
diff --git a/libc/test/src/math/tanpif16_test.cpp b/libc/test/src/math/tanpif16_test.cpp
index 629ba963135c..04d1f713574a 100644
--- a/libc/test/src/math/tanpif16_test.cpp
+++ b/libc/test/src/math/tanpif16_test.cpp
@@ -1,4 +1,4 @@
-//===-- Exhaustive test for cospif16 --------------------------------------===//
+//===-- Exhaustive test for tanpif16 --------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index 48688b3bdd1f..657343d07215 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -12,6 +12,18 @@ add_libc_test(
     libc.src.wchar.wcslen
 )
 
+add_libc_test(
+  wcsnlen_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcsnlen_test.cpp
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.types.wchar_t
+    libc.src.wchar.wcsnlen
+)
+
 add_libc_test(
   btowc_test
   SUITE
@@ -31,10 +43,12 @@ add_libc_test(
     mbrtowc_test.cpp
   DEPENDS
     libc.src.__support.libc_errno
+    libc.src.__support.wchar.mbstate
     libc.src.string.memset
     libc.src.wchar.mbrtowc
     libc.hdr.types.mbstate_t
     libc.hdr.types.wchar_t
+    libc.test.UnitTest.ErrnoCheckingTest
 )
 
 add_libc_test(
@@ -72,6 +86,19 @@ add_libc_test(
     libc.hdr.types.wchar_t
     libc.hdr.types.mbstate_t
     libc.src.__support.libc_errno
+    libc.src.__support.wchar.mbstate
+    libc.test.UnitTest.ErrnoCheckingTest
+)
+
+add_libc_test(
+  wctomb_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wctomb_test.cpp
+  DEPENDS
+    libc.src.wchar.wctomb
+    libc.hdr.types.wchar_t
 )
 
 add_libc_test(
@@ -96,6 +123,16 @@ add_libc_test(
     libc.src.wchar.wcschr
 )
 
+add_libc_test(
+  wcstok_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcstok_test.cpp
+  DEPENDS
+    libc.src.wchar.wcstok
+)
+
 add_libc_test(
   wcsncmp_test
   SUITE
@@ -146,6 +183,16 @@ add_libc_test(
     libc.src.wchar.wcsspn
 )
 
+add_libc_test(
+  wcscspn_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcscspn_test.cpp
+  DEPENDS
+    libc.src.wchar.wcscspn
+)
+
 add_libc_test(
   wmemchr_test
   SUITE
diff --git a/libc/test/src/wchar/mbrtowc_test.cpp b/libc/test/src/wchar/mbrtowc_test.cpp
index 69dcf00fde20..5a14d8e25935 100644
--- a/libc/test/src/wchar/mbrtowc_test.cpp
+++ b/libc/test/src/wchar/mbrtowc_test.cpp
@@ -9,11 +9,15 @@
 #include "hdr/types/mbstate_t.h"
 #include "hdr/types/wchar_t.h"
 #include "src/__support/libc_errno.h"
+#include "src/__support/wchar/mbstate.h"
 #include "src/string/memset.h"
 #include "src/wchar/mbrtowc.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
-TEST(LlvmLibcMBRToWC, OneByte) {
+using LlvmLibcMBRToWCTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+
+TEST_F(LlvmLibcMBRToWCTest, OneByte) {
   const char *ch = "A";
   wchar_t dest[2];
   // Testing if it works with nullptr mbstate_t
@@ -21,13 +25,15 @@ TEST(LlvmLibcMBRToWC, OneByte) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
   ASSERT_EQ(static_cast<char>(*dest), 'A');
   ASSERT_EQ(static_cast<int>(n), 1);
+  ASSERT_ERRNO_SUCCESS();
 
   // Should fail since we have not read enough
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 0, mb);
   ASSERT_EQ(static_cast<int>(n), -2);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, TwoByte) {
+TEST_F(LlvmLibcMBRToWCTest, TwoByte) {
   const char ch[2] = {static_cast<char>(0xC2),
                       static_cast<char>(0x8E)}; // Ž car symbol
   wchar_t dest[2];
@@ -36,6 +42,7 @@ TEST(LlvmLibcMBRToWC, TwoByte) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
   ASSERT_EQ(static_cast<int>(*dest), 142);
   ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_ERRNO_SUCCESS();
 
   // Should fail since we have not read enough
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
@@ -44,9 +51,10 @@ TEST(LlvmLibcMBRToWC, TwoByte) {
   n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 1, mb);
   ASSERT_EQ(static_cast<int>(n), 1);
   ASSERT_EQ(static_cast<int>(*dest), 142);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, ThreeByte) {
+TEST_F(LlvmLibcMBRToWCTest, ThreeByte) {
   const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
                       static_cast<char>(0x91)}; // ∑ sigma symbol
   wchar_t dest[2];
@@ -55,17 +63,20 @@ TEST(LlvmLibcMBRToWC, ThreeByte) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
   ASSERT_EQ(static_cast<int>(*dest), 8721);
   ASSERT_EQ(static_cast<int>(n), 3);
+  ASSERT_ERRNO_SUCCESS();
 
   // Should fail since we have not read enough
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
   ASSERT_EQ(static_cast<int>(n), -2);
+  ASSERT_ERRNO_SUCCESS();
   // Should pass after reading two more bytes
   n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 8721);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, FourByte) {
+TEST_F(LlvmLibcMBRToWCTest, FourByte) {
   const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
                       static_cast<char>(0xA4),
                       static_cast<char>(0xA1)}; // 🤡 clown emoji
@@ -75,27 +86,29 @@ TEST(LlvmLibcMBRToWC, FourByte) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
   ASSERT_EQ(static_cast<int>(*dest), 129313);
   ASSERT_EQ(static_cast<int>(n), 4);
-
+  ASSERT_ERRNO_SUCCESS();
   // Should fail since we have not read enough
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
   ASSERT_EQ(static_cast<int>(n), -2);
+  ASSERT_ERRNO_SUCCESS();
   // Should pass after reading two more bytes
   n = LIBC_NAMESPACE::mbrtowc(dest, ch + 2, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 129313);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, InvalidByte) {
+TEST_F(LlvmLibcMBRToWCTest, InvalidByte) {
   const char ch[1] = {static_cast<char>(0x80)};
   wchar_t dest[2];
   mbstate_t *mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
   ASSERT_EQ(static_cast<int>(n), -1);
-  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  ASSERT_ERRNO_EQ(EILSEQ);
 }
 
-TEST(LlvmLibcMBRToWC, InvalidMultiByte) {
+TEST_F(LlvmLibcMBRToWCTest, InvalidMultiByte) {
   const char ch[4] = {static_cast<char>(0x80), static_cast<char>(0x00),
                       static_cast<char>(0x80),
                       static_cast<char>(0x00)}; // invalid sequence of bytes
@@ -105,18 +118,19 @@ TEST(LlvmLibcMBRToWC, InvalidMultiByte) {
   // Trying to push all 4 should error
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
   ASSERT_EQ(static_cast<int>(n), -1);
-  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  ASSERT_ERRNO_EQ(EILSEQ);
   // Trying to push just the first one should error
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
   ASSERT_EQ(static_cast<int>(n), -1);
-  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  ASSERT_ERRNO_EQ(EILSEQ);
   // Trying to push the second and third should correspond to null wc
   n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 0);
   ASSERT_TRUE(*dest == L'\0');
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, InvalidLastByte) {
+TEST_F(LlvmLibcMBRToWCTest, InvalidLastByte) {
   // Last byte is invalid since it does not have correct starting sequence.
   // 0xC0 --> 11000000 starting sequence should be 10xxxxxx
   const char ch[4] = {static_cast<char>(0xF1), static_cast<char>(0x80),
@@ -127,10 +141,10 @@ TEST(LlvmLibcMBRToWC, InvalidLastByte) {
   // Trying to push all 4 should error
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
   ASSERT_EQ(static_cast<int>(n), -1);
-  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  ASSERT_ERRNO_EQ(EILSEQ);
 }
 
-TEST(LlvmLibcMBRToWC, ValidTwoByteWithExtraRead) {
+TEST_F(LlvmLibcMBRToWCTest, ValidTwoByteWithExtraRead) {
   const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0x80)};
   wchar_t dest[2];
@@ -140,9 +154,10 @@ TEST(LlvmLibcMBRToWC, ValidTwoByteWithExtraRead) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 142);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, TwoValidTwoBytes) {
+TEST_F(LlvmLibcMBRToWCTest, TwoValidTwoBytes) {
   const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0xC7), static_cast<char>(0x8C)};
   wchar_t dest[2];
@@ -152,12 +167,14 @@ TEST(LlvmLibcMBRToWC, TwoValidTwoBytes) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 142);
+  ASSERT_ERRNO_SUCCESS();
   n = LIBC_NAMESPACE::mbrtowc(dest + 1, ch + 2, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*(dest + 1)), 460);
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcMBRToWC, NullString) {
+TEST_F(LlvmLibcMBRToWCTest, NullString) {
   wchar_t dest[2] = {L'O', L'K'};
   mbstate_t *mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
@@ -165,8 +182,24 @@ TEST(LlvmLibcMBRToWC, NullString) {
   size_t n = LIBC_NAMESPACE::mbrtowc(dest, nullptr, 2, mb);
   ASSERT_EQ(static_cast<int>(n), 0);
   ASSERT_TRUE(dest[0] == L'O');
+  ASSERT_ERRNO_SUCCESS();
   // reading a null terminator should return 0
   const char *ch = "\0";
   n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
   ASSERT_EQ(static_cast<int>(n), 0);
+  ASSERT_ERRNO_SUCCESS();
+}
+
+TEST_F(LlvmLibcMBRToWCTest, InvalidMBState) {
+  const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+                      static_cast<char>(0xC7), static_cast<char>(0x8C)};
+  wchar_t dest[2] = {L'O', L'K'};
+  mbstate_t *mb;
+  LIBC_NAMESPACE::internal::mbstate inv;
+  inv.total_bytes = 6;
+  mb = reinterpret_cast<mbstate_t *>(&inv);
+  // invalid mbstate should error
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), -1);
+  ASSERT_ERRNO_EQ(EINVAL);
 }
diff --git a/libc/test/src/wchar/wcrtomb_test.cpp b/libc/test/src/wchar/wcrtomb_test.cpp
index c06b39ae0143..b29624e87f7a 100644
--- a/libc/test/src/wchar/wcrtomb_test.cpp
+++ b/libc/test/src/wchar/wcrtomb_test.cpp
@@ -9,11 +9,15 @@
 #include "hdr/types/mbstate_t.h"
 #include "hdr/types/wchar_t.h"
 #include "src/__support/libc_errno.h"
+#include "src/__support/wchar/mbstate.h"
 #include "src/string/memset.h"
 #include "src/wchar/wcrtomb.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
-TEST(LlvmLibcWCRToMBTest, OneByte) {
+using LlvmLibcWCRToMBTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+
+TEST_F(LlvmLibcWCRToMBTest, OneByte) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   wchar_t wc = L'U';
@@ -21,9 +25,10 @@ TEST(LlvmLibcWCRToMBTest, OneByte) {
   size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
   ASSERT_EQ(cnt, static_cast<size_t>(1));
   ASSERT_EQ(mb[0], 'U');
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcWCRToMBTest, TwoByte) {
+TEST_F(LlvmLibcWCRToMBTest, TwoByte) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   // testing utf32: 0xff -> utf8: 0xc3 0xbf
@@ -33,9 +38,10 @@ TEST(LlvmLibcWCRToMBTest, TwoByte) {
   ASSERT_EQ(cnt, static_cast<size_t>(2));
   ASSERT_EQ(mb[0], static_cast<char>(0xc3));
   ASSERT_EQ(mb[1], static_cast<char>(0xbf));
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcWCRToMBTest, ThreeByte) {
+TEST_F(LlvmLibcWCRToMBTest, ThreeByte) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   // testing utf32: 0xac15 -> utf8: 0xea 0xb0 0x95
@@ -46,9 +52,10 @@ TEST(LlvmLibcWCRToMBTest, ThreeByte) {
   ASSERT_EQ(mb[0], static_cast<char>(0xea));
   ASSERT_EQ(mb[1], static_cast<char>(0xb0));
   ASSERT_EQ(mb[2], static_cast<char>(0x95));
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcWCRToMBTest, FourByte) {
+TEST_F(LlvmLibcWCRToMBTest, FourByte) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   // testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
@@ -60,9 +67,10 @@ TEST(LlvmLibcWCRToMBTest, FourByte) {
   ASSERT_EQ(mb[1], static_cast<char>(0x9f));
   ASSERT_EQ(mb[2], static_cast<char>(0xa4));
   ASSERT_EQ(mb[3], static_cast<char>(0xa1));
+  ASSERT_ERRNO_SUCCESS();
 }
 
-TEST(LlvmLibcWCRToMBTest, NullString) {
+TEST_F(LlvmLibcWCRToMBTest, NullString) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   wchar_t wc = L'A';
@@ -70,24 +78,39 @@ TEST(LlvmLibcWCRToMBTest, NullString) {
 
   // should be equivalent to the call wcrtomb(buf, L'\0', state)
   size_t cnt1 = LIBC_NAMESPACE::wcrtomb(nullptr, wc, &state);
+  ASSERT_ERRNO_SUCCESS();
   size_t cnt2 = LIBC_NAMESPACE::wcrtomb(mb, L'\0', &state);
+  ASSERT_ERRNO_SUCCESS();
 
   ASSERT_EQ(cnt1, cnt2);
 }
 
-TEST(LlvmLibcWCRToMBTest, NullState) {
+TEST_F(LlvmLibcWCRToMBTest, NullState) {
   wchar_t wc = L'A';
   char mb[4];
   size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, nullptr);
+  ASSERT_ERRNO_SUCCESS();
   ASSERT_EQ(cnt, static_cast<size_t>(1));
 }
 
-TEST(LlvmLibcWCRToMBTest, InvalidWchar) {
+TEST_F(LlvmLibcWCRToMBTest, InvalidWchar) {
   mbstate_t state;
   LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
   wchar_t wc = 0x12ffff;
   char mb[4];
   size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
   ASSERT_EQ(cnt, static_cast<size_t>(-1));
-  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  ASSERT_ERRNO_EQ(EILSEQ);
+}
+
+TEST_F(LlvmLibcWCRToMBTest, InvalidMBState) {
+  mbstate_t *state;
+  LIBC_NAMESPACE::internal::mbstate inv;
+  inv.total_bytes = 6;
+  state = reinterpret_cast<mbstate_t *>(&inv);
+  wchar_t wc = L'A';
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, state);
+  ASSERT_EQ(cnt, static_cast<size_t>(-1));
+  ASSERT_ERRNO_EQ(EINVAL);
 }
diff --git a/libc/test/src/wchar/wcscspn_test.cpp b/libc/test/src/wchar/wcscspn_test.cpp
new file mode 100644
index 000000000000..6318302d1005
--- /dev/null
+++ b/libc/test/src/wchar/wcscspn_test.cpp
@@ -0,0 +1,67 @@
+//===-- Unittests for wcscspn ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcscspn.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSCSpnTest, EmptyStringShouldReturnZeroLengthSpan) {
+  // The search should not include the null terminator.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"", L""), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"_", L""), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"", L"_"), size_t(0));
+}
+
+TEST(LlvmLibcWCSCSpnTest, ShouldNotSpanAnythingAfterNullTerminator) {
+  const wchar_t src[4] = {L'a', L'b', L'\0', L'c'};
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"de"), size_t(2));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"c"), size_t(2));
+
+  // Same goes for the segment to be searched for.
+  const wchar_t segment[4] = {L'1', L'2', L'\0', L'3'};
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"3", segment), size_t(1));
+}
+
+TEST(LlvmLibcWCSCSpnTest, SpanEachIndividualCharacter) {
+  const wchar_t *src = L"12345";
+  // These are all in the segment.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"1"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"2"), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"3"), size_t(2));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"4"), size_t(3));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"5"), size_t(4));
+}
+
+TEST(LlvmLibcWCSCSpnTest, UnmatchedCharacterShouldReturnLength) {
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"a", L"b"), size_t(1));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"abcdef", L"1"), size_t(6));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"123", L"4"), size_t(3));
+}
+
+TEST(LlvmLibcWCSCSpnTest, NonSequentialCharactersShouldNotSpan) {
+  const wchar_t *src = L"abc456789";
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"_1_abc_2_def_3_"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(src, L"67__34xyz12"), size_t(3));
+}
+
+TEST(LlvmLibcWCSCSpnTest, ReverseCharacters) {
+  // These are all in the string.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"54321"), size_t(0));
+  // 1 is not in the span.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"432"), size_t(1));
+  // 1 is in the span.
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"12345", L"51"), size_t(0));
+}
+
+TEST(LlvmLibcWCSCSpnTest, DuplicatedCharactersToBeSearchedForShouldStillMatch) {
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"a", L"aa"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aa", L"aa"), size_t(0));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aaa", L"bb"), size_t(3));
+  EXPECT_EQ(LIBC_NAMESPACE::wcscspn(L"aaaa", L"bb"), size_t(4));
+}
diff --git a/libc/test/src/wchar/wcsnlen_test.cpp b/libc/test/src/wchar/wcsnlen_test.cpp
new file mode 100644
index 000000000000..efb7198a3115
--- /dev/null
+++ b/libc/test/src/wchar/wcsnlen_test.cpp
@@ -0,0 +1,54 @@
+//===-- Unittests for wcsnlen ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcsnlen.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSNLenTest, EmptyString) {
+  ASSERT_EQ(static_cast<size_t>(0), LIBC_NAMESPACE::wcsnlen(L"", 0));
+  // If N is greater than string length, this should still return 0.
+  ASSERT_EQ(static_cast<size_t>(0), LIBC_NAMESPACE::wcsnlen(L"", 1));
+}
+
+TEST(LlvmLibcWCSNLenTest, OneCharacterString) {
+  const wchar_t *src = L"A";
+  ASSERT_EQ(static_cast<size_t>(1), LIBC_NAMESPACE::wcsnlen(src, 1));
+  // If N is 0, this should return 0.
+  ASSERT_EQ(static_cast<size_t>(0), LIBC_NAMESPACE::wcsnlen(src, 0));
+  // If N is greater than string length, this should still return 1.
+  ASSERT_EQ(static_cast<size_t>(1), LIBC_NAMESPACE::wcsnlen(src, 3));
+}
+
+TEST(LlvmLibcWCSNLenTest, ManyCharacterString) {
+  const wchar_t *src = L"123456789";
+  ASSERT_EQ(static_cast<size_t>(9), LIBC_NAMESPACE::wcsnlen(src, 9));
+  // If N is 0, this should return 0.
+  ASSERT_EQ(static_cast<size_t>(0), LIBC_NAMESPACE::wcsnlen(src, 0));
+  // If N is smaller than the string length, it should return N.
+  ASSERT_EQ(static_cast<size_t>(3), LIBC_NAMESPACE::wcsnlen(src, 3));
+  // If N is greater than string length, this should still return 9.
+  ASSERT_EQ(static_cast<size_t>(9), LIBC_NAMESPACE::wcsnlen(src, 42));
+}
+
+TEST(LlvmLibcWCSNLenTest, IgnoreCharactersAfterNullTerminator) {
+  const wchar_t src[5] = {L'a', L'b', L'c', L'\0', L'd'};
+  ASSERT_EQ(static_cast<size_t>(3), LIBC_NAMESPACE::wcsnlen(src, 3));
+  // This should only read up to the null terminator.
+  ASSERT_EQ(static_cast<size_t>(3), LIBC_NAMESPACE::wcsnlen(src, 4));
+  ASSERT_EQ(static_cast<size_t>(3), LIBC_NAMESPACE::wcsnlen(src, 5));
+}
+
+TEST(LlvmLibcWCSNLenTest, NoNullTerminator) {
+  const wchar_t src[4] = {L'a', L'b', L'c', L'd'};
+  // Should return 4
+  ASSERT_EQ(static_cast<size_t>(4), LIBC_NAMESPACE::wcsnlen(src, 4));
+  // Should return 2 since N is smaller than string length
+  ASSERT_EQ(static_cast<size_t>(2), LIBC_NAMESPACE::wcsnlen(src, 2));
+}
diff --git a/libc/test/src/wchar/wcstok_test.cpp b/libc/test/src/wchar/wcstok_test.cpp
new file mode 100644
index 000000000000..7106e9f2fab5
--- /dev/null
+++ b/libc/test/src/wchar/wcstok_test.cpp
@@ -0,0 +1,181 @@
+//===-- Unittests for wcstok ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/wchar/wcstok.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCSTokReentrantTest, NoTokenFound) {
+  { // Empty source and delimiter string.
+    wchar_t empty[] = L"";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"", &reserve), nullptr);
+  }
+  { // Empty source and single character delimiter string.
+    wchar_t empty[] = L"";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"_", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(empty, L"_", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"_", &reserve), nullptr);
+  }
+  { // Same character source and delimiter string.
+    wchar_t single[] = L"_";
+    wchar_t *reserve = nullptr;
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(single, L"_", &reserve), nullptr);
+    // Another call to ensure that 'reserve' is not in a bad state.
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(single, L"_", &reserve), nullptr);
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L"_", &reserve), nullptr);
+  }
+  { // Multiple character source and single character delimiter string.
+    wchar_t multiple[] = L"1,2";
+    wchar_t *reserve = nullptr;
+    wchar_t *tok = LIBC_NAMESPACE::wcstok(multiple, L":", &reserve);
+    ASSERT_TRUE(tok[0] == L'1');
+    ASSERT_TRUE(tok[1] == L',');
+    ASSERT_TRUE(tok[2] == L'2');
+    ASSERT_TRUE(tok[3] == L'\0');
+    // Another call to ensure that 'reserve' is not in a bad state.
+    tok = LIBC_NAMESPACE::wcstok(multiple, L":", &reserve);
+    ASSERT_TRUE(tok[0] == L'1');
+    ASSERT_TRUE(tok[1] == L',');
+    ASSERT_TRUE(tok[2] == L'2');
+    ASSERT_TRUE(tok[3] == L'\0');
+    ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L":", &reserve), nullptr);
+  }
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterAsFirstCharacterShouldBeIgnored) {
+  wchar_t src[] = L".123";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L".", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L".", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L".", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterIsMiddleCharacter) {
+  wchar_t src[] = L"12,34";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L",", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimiterAsLastCharacterShouldBeIgnored) {
+  wchar_t src[] = L"1234:";
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L":", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'4');
+  ASSERT_TRUE(tok[4] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L":", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'3');
+  ASSERT_TRUE(tok[3] == L'4');
+  ASSERT_TRUE(tok[4] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L":", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, ShouldNotGoPastNullTerminator) {
+  wchar_t src[] = {L'1', L'2', L'\0', L',', L'3'};
+  wchar_t *reserve = nullptr;
+  wchar_t *tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  // Another call to ensure that 'reserve' is not in a bad state.
+  tok = LIBC_NAMESPACE::wcstok(src, L",", &reserve);
+  ASSERT_TRUE(tok[0] == L'1');
+  ASSERT_TRUE(tok[1] == L'2');
+  ASSERT_TRUE(tok[2] == L'\0');
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(nullptr, L",", &reserve), nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest,
+     ShouldReturnNullptrWhenBothSrcAndSaveptrAreNull) {
+  wchar_t *src = nullptr;
+  wchar_t *reserve = nullptr;
+  // Ensure that instead of crashing if src and reserve are null, nullptr is
+  // returned
+  ASSERT_EQ(LIBC_NAMESPACE::wcstok(src, L",", &reserve), nullptr);
+  // And that neither src nor reserve are changed when that happens
+  ASSERT_EQ(src, nullptr);
+  ASSERT_EQ(reserve, nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest,
+     SubsequentCallsShouldFindFollowingDelimiters) {
+  wchar_t src[] = L"12,34.56";
+  wchar_t *reserve = nullptr;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'1');
+  ASSERT_TRUE(token[1] == L'2');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'3');
+  ASSERT_TRUE(token[1] == L'4');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L",.", &reserve);
+  ASSERT_TRUE(token[0] == L'5');
+  ASSERT_TRUE(token[1] == L'6');
+  ASSERT_TRUE(token[2] == L'\0');
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+  // Subsequent calls after hitting the end of the string should also return
+  // nullptr.
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+}
+
+TEST(LlvmLibcWCSTokReentrantTest, DelimitersShouldNotBeIncludedInToken) {
+  wchar_t src[] = L"__ab__:_cd__:__ef__:__";
+  wchar_t *reserve = nullptr;
+  wchar_t *token = LIBC_NAMESPACE::wcstok(src, L"_:", &reserve);
+  ASSERT_TRUE(token[0] == L'a');
+  ASSERT_TRUE(token[1] == L'b');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L":_", &reserve);
+  ASSERT_TRUE(token[0] == L'c');
+  ASSERT_TRUE(token[1] == L'd');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,", &reserve);
+  ASSERT_TRUE(token[0] == L'e');
+  ASSERT_TRUE(token[1] == L'f');
+  ASSERT_TRUE(token[2] == L'\0');
+
+  token = LIBC_NAMESPACE::wcstok(nullptr, L"_:,_", &reserve);
+  ASSERT_EQ(token, nullptr);
+}
diff --git a/libc/test/src/wchar/wctomb_test.cpp b/libc/test/src/wchar/wctomb_test.cpp
new file mode 100644
index 000000000000..09fbf5280622
--- /dev/null
+++ b/libc/test/src/wchar/wctomb_test.cpp
@@ -0,0 +1,73 @@
+//===-- Unittests for wctomb ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/wchar_t.h"
+#include "src/__support/libc_errno.h"
+#include "src/wchar/wctomb.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
+#include "test/UnitTest/Test.h"
+
+using LlvmLibcWCToMBTest = LIBC_NAMESPACE::testing::ErrnoCheckingTest;
+
+TEST(LlvmLibcWCToMBTest, OneByte) {
+  wchar_t wc = L'U';
+  char mb[4];
+  int cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+  ASSERT_EQ(cnt, 1);
+  ASSERT_EQ(mb[0], 'U');
+}
+
+TEST(LlvmLibcWCToMBTest, TwoByte) {
+  // testing utf32: 0xff -> utf8: 0xc3 0xbf
+  wchar_t wc = 0xff;
+  char mb[4];
+  int cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+  ASSERT_EQ(cnt, 2);
+  ASSERT_EQ(mb[0], static_cast<char>(0xc3));
+  ASSERT_EQ(mb[1], static_cast<char>(0xbf));
+}
+
+TEST(LlvmLibcWCToMBTest, ThreeByte) {
+  // testing utf32: 0xac15 -> utf8: 0xea 0xb0 0x95
+  wchar_t wc = 0xac15;
+  char mb[4];
+  int cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+  ASSERT_EQ(cnt, 3);
+  ASSERT_EQ(mb[0], static_cast<char>(0xea));
+  ASSERT_EQ(mb[1], static_cast<char>(0xb0));
+  ASSERT_EQ(mb[2], static_cast<char>(0x95));
+}
+
+TEST(LlvmLibcWCToMBTest, FourByte) {
+  // testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
+  wchar_t wc = 0x1f921;
+  char mb[4];
+  int cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+  ASSERT_EQ(cnt, 4);
+  ASSERT_EQ(mb[0], static_cast<char>(0xf0));
+  ASSERT_EQ(mb[1], static_cast<char>(0x9f));
+  ASSERT_EQ(mb[2], static_cast<char>(0xa4));
+  ASSERT_EQ(mb[3], static_cast<char>(0xa1));
+}
+
+TEST(LlvmLibcWCToMBTest, NullString) {
+  wchar_t wc = L'A';
+
+  int cnt = LIBC_NAMESPACE::wctomb(nullptr, wc);
+
+  // no state-dependent encoding
+  ASSERT_EQ(cnt, 0);
+}
+
+TEST(LlvmLibcWCToMBTest, InvalidWchar) {
+  wchar_t wc = 0x12ffff;
+  char mb[4];
+  int cnt = LIBC_NAMESPACE::wctomb(mb, wc);
+  ASSERT_EQ(cnt, -1);
+  ASSERT_ERRNO_EQ(EILSEQ);
+}
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index c98e2043464d..e2871d1b01a1 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -84,6 +84,9 @@ else()
   endif()
 endif()
 
+# Setup the paths where libclc runtimes should be stored.
+set( LIBCLC_OUTPUT_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR} )
+
 if( EXISTS ${LIBCLC_CUSTOM_LLVM_TOOLS_BINARY_DIR} )
   message( WARNING "Using custom LLVM tools to build libclc: "
     "${LIBCLC_CUSTOM_LLVM_TOOLS_BINARY_DIR}, "
diff --git a/libclc/clc/include/clc/clcmacro.h b/libclc/clc/include/clc/clcmacro.h
index de7b977021f8..5c67c937cb1d 100644
--- a/libclc/clc/include/clc/clcmacro.h
+++ b/libclc/clc/include/clc/clcmacro.h
@@ -12,111 +12,6 @@
 #include <clc/internal/clc.h>
 #include <clc/utils.h>
 
-#define _CLC_UNARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE)          \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x) {                              \
-    return (RET_TYPE##2)(FUNCTION(x.s0), FUNCTION(x.s1));                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x) {                              \
-    return (RET_TYPE##3)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2));      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x) {                              \
-    return (RET_TYPE##4)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),       \
-                         FUNCTION(x.s3));                                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x) {                              \
-    return (RET_TYPE##8)(FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),       \
-                         FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),       \
-                         FUNCTION(x.s6), FUNCTION(x.s7));                      \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x) {                            \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2), FUNCTION(x.s3),        \
-        FUNCTION(x.s4), FUNCTION(x.s5), FUNCTION(x.s6), FUNCTION(x.s7),        \
-        FUNCTION(x.s8), FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb),        \
-        FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se), FUNCTION(x.sf));       \
-  }
-
-#define _CLC_BINARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,         \
-                              ARG2_TYPE)                                       \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x, ARG2_TYPE##2 y) {              \
-    return (RET_TYPE##2)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x, ARG2_TYPE##3 y) {              \
-    return (RET_TYPE##3)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2));                                \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x, ARG2_TYPE##4 y) {              \
-    return (RET_TYPE##4)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x, ARG2_TYPE##8 y) {              \
-    return (RET_TYPE##8)(FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1),           \
-                         FUNCTION(x.s2, y.s2), FUNCTION(x.s3, y.s3),           \
-                         FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),           \
-                         FUNCTION(x.s6, y.s6), FUNCTION(x.s7, y.s7));          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x, ARG2_TYPE##16 y) {           \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0, y.s0), FUNCTION(x.s1, y.s1), FUNCTION(x.s2, y.s2),      \
-        FUNCTION(x.s3, y.s3), FUNCTION(x.s4, y.s4), FUNCTION(x.s5, y.s5),      \
-        FUNCTION(x.s6, y.s6), FUNCTION(x.s7, y.s7), FUNCTION(x.s8, y.s8),      \
-        FUNCTION(x.s9, y.s9), FUNCTION(x.sa, y.sa), FUNCTION(x.sb, y.sb),      \
-        FUNCTION(x.sc, y.sc), FUNCTION(x.sd, y.sd), FUNCTION(x.se, y.se),      \
-        FUNCTION(x.sf, y.sf));                                                 \
-  }
-
-#define _CLC_TERNARY_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,        \
-                               ARG2_TYPE, ARG3_TYPE)                           \
-  DECLSPEC RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x, ARG2_TYPE##2 y,                \
-                                ARG3_TYPE##2 z) {                              \
-    return (RET_TYPE##2)(FUNCTION(x.s0, y.s0, z.s0),                           \
-                         FUNCTION(x.s1, y.s1, z.s1));                          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x, ARG2_TYPE##3 y,                \
-                                ARG3_TYPE##3 z) {                              \
-    return (RET_TYPE##3)(FUNCTION(x.s0, y.s0, z.s0),                           \
-                         FUNCTION(x.s1, y.s1, z.s1),                           \
-                         FUNCTION(x.s2, y.s2, z.s2));                          \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x, ARG2_TYPE##4 y,                \
-                                ARG3_TYPE##4 z) {                              \
-    return (RET_TYPE##4)(                                                      \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3));               \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x, ARG2_TYPE##8 y,                \
-                                ARG3_TYPE##8 z) {                              \
-    return (RET_TYPE##8)(                                                      \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3),                \
-        FUNCTION(x.s4, y.s4, z.s4), FUNCTION(x.s5, y.s5, z.s5),                \
-        FUNCTION(x.s6, y.s6, z.s6), FUNCTION(x.s7, y.s7, z.s7));               \
-  }                                                                            \
-                                                                               \
-  DECLSPEC RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x, ARG2_TYPE##16 y,             \
-                                 ARG3_TYPE##16 z) {                            \
-    return (RET_TYPE##16)(                                                     \
-        FUNCTION(x.s0, y.s0, z.s0), FUNCTION(x.s1, y.s1, z.s1),                \
-        FUNCTION(x.s2, y.s2, z.s2), FUNCTION(x.s3, y.s3, z.s3),                \
-        FUNCTION(x.s4, y.s4, z.s4), FUNCTION(x.s5, y.s5, z.s5),                \
-        FUNCTION(x.s6, y.s6, z.s6), FUNCTION(x.s7, y.s7, z.s7),                \
-        FUNCTION(x.s8, y.s8, z.s8), FUNCTION(x.s9, y.s9, z.s9),                \
-        FUNCTION(x.sa, y.sa, z.sa), FUNCTION(x.sb, y.sb, z.sb),                \
-        FUNCTION(x.sc, y.sc, z.sc), FUNCTION(x.sd, y.sd, z.sd),                \
-        FUNCTION(x.se, y.se, z.se), FUNCTION(x.sf, y.sf, z.sf));               \
-  }
-
 #define _CLC_V_V_VP_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,         \
                               ADDR_SPACE, ARG2_TYPE)                           \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 2)                                          \
@@ -171,117 +66,4 @@
         FUNCTION(x.sf, ptr + 15));                                             \
   }
 
-#define _CLC_DEFINE_BINARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE,     \
-                                   ARG2_TYPE)                                  \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, RET_TYPE, FUNCTION, ARG1_TYPE, \
-                        ARG2_TYPE)
-
-// FIXME: Make _CLC_DEFINE_BINARY_BUILTIN avoid scalarization by default, and
-// introduce an explicit scalarizing version.
-#define _CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(RET_TYPE, FUNCTION, BUILTIN,   \
-                                                ARG1_TYPE, ARG2_TYPE)          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x,                  \
-                                              ARG2_TYPE##2 y) {                \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x,                  \
-                                              ARG2_TYPE##3 y) {                \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x,                  \
-                                              ARG2_TYPE##4 y) {                \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x,                  \
-                                              ARG2_TYPE##8 y) {                \
-    return BUILTIN(x, y);                                                      \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x,                \
-                                               ARG2_TYPE##16 y) {              \
-    return BUILTIN(x, y);                                                      \
-  }
-
-#define _CLC_DEFINE_BINARY_BUILTIN_WITH_SCALAR_SECOND_ARG(                     \
-    RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE, ARG2_TYPE)                         \
-  _CLC_DEFINE_BINARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE,           \
-                             ARG2_TYPE)                                        \
-  _CLC_BINARY_VECTORIZE_SCALAR_SECOND_ARG(_CLC_OVERLOAD _CLC_DEF, RET_TYPE,    \
-                                          FUNCTION, ARG1_TYPE, ARG2_TYPE)
-
-#define _CLC_DEFINE_UNARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE)      \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x) { return BUILTIN(x); } \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x) {                \
-    return BUILTIN(x);                                                         \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x) {                \
-    return BUILTIN(x);                                                         \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x) {                \
-    return BUILTIN(x);                                                         \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x) {                \
-    return BUILTIN(x);                                                         \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##16 FUNCTION(ARG1_TYPE##16 x) {              \
-    return BUILTIN(x);                                                         \
-  }
-
-#define _CLC_DEFINE_TERNARY_BUILTIN(RET_TYPE, FUNCTION, BUILTIN, ARG1_TYPE,    \
-                                    ARG2_TYPE, ARG3_TYPE)                      \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y,           \
-                                           ARG3_TYPE z) {                      \
-    return BUILTIN(x, y, z);                                                   \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##2 FUNCTION(ARG1_TYPE##2 x, ARG2_TYPE##2 y,  \
-                                              ARG3_TYPE##2 z) {                \
-    return BUILTIN(x, y, z);                                                   \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##3 FUNCTION(ARG1_TYPE##3 x, ARG2_TYPE##3 y,  \
-                                              ARG3_TYPE##3 z) {                \
-    return BUILTIN(x, y, z);                                                   \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##4 FUNCTION(ARG1_TYPE##4 x, ARG2_TYPE##4 y,  \
-                                              ARG3_TYPE##4 z) {                \
-    return BUILTIN(x, y, z);                                                   \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##8 FUNCTION(ARG1_TYPE##8 x, ARG2_TYPE##8 y,  \
-                                              ARG3_TYPE##8 z) {                \
-    return BUILTIN(x, y, z);                                                   \
-  }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE##16 FUNCTION(                                \
-      ARG1_TYPE##16 x, ARG2_TYPE##16 y, ARG3_TYPE##16 z) {                     \
-    return BUILTIN(x, y, z);                                                   \
-  }
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-#define _CLC_DEFINE_UNARY_BUILTIN_FP16(FUNCTION)                               \
-  _CLC_DEF _CLC_OVERLOAD half FUNCTION(half x) {                               \
-    return (half)FUNCTION((float)x);                                           \
-  }                                                                            \
-  _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, FUNCTION, half)
-
-#define _CLC_DEFINE_BINARY_BUILTIN_FP16(FUNCTION)                              \
-  _CLC_DEF _CLC_OVERLOAD half FUNCTION(half x, half y) {                       \
-    return (half)FUNCTION((float)x, (float)y);                                 \
-  }                                                                            \
-  _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, FUNCTION, half, half)
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : disable
-
-#else
-
-#define _CLC_DEFINE_UNARY_BUILTIN_FP16(FUNCTION)
-#define _CLC_DEFINE_BINARY_BUILTIN_FP16(FUNCTION)
-
-#endif
-
 #endif // __CLC_CLCMACRO_H__
diff --git a/libclc/clc/include/clc/common/clc_degrees.h b/libclc/clc/include/clc/common/clc_degrees.h
index 5f04fac1f261..e633344c291c 100644
--- a/libclc/clc/include/clc/common/clc_degrees.h
+++ b/libclc/clc/include/clc/common/clc_degrees.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_DEGREES_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_degrees
+#define FUNCTION __clc_degrees
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_DEGREES_H__
diff --git a/libclc/clc/include/clc/common/clc_radians.h b/libclc/clc/include/clc/common/clc_radians.h
index 03d7c4b9e301..0b075efbcce2 100644
--- a/libclc/clc/include/clc/common/clc_radians.h
+++ b/libclc/clc/include/clc/common/clc_radians.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_RADIANS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_radians
+#define FUNCTION __clc_radians
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_RADIANS_H__
diff --git a/libclc/clc/include/clc/common/clc_sign.h b/libclc/clc/include/clc/common/clc_sign.h
index 9737f1e54441..0a8c4cc94d80 100644
--- a/libclc/clc/include/clc/common/clc_sign.h
+++ b/libclc/clc/include/clc/common/clc_sign.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_SIGN_H__
 #define __CLC_COMMON_CLC_SIGN_H__
 
-#define __CLC_FUNCTION __clc_sign
+#define FUNCTION __clc_sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_SIGN_H__
diff --git a/libclc/clc/include/clc/common/clc_step.h b/libclc/clc/include/clc/common/clc_step.h
index 6b093d06896c..96b4759cabf7 100644
--- a/libclc/clc/include/clc/common/clc_step.h
+++ b/libclc/clc/include/clc/common/clc_step.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_STEP_H__
 #define __CLC_COMMON_CLC_STEP_H__
 
-#define __CLC_FUNCTION __clc_step
+#define FUNCTION __clc_step
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_COMMON_CLC_STEP_H__
diff --git a/libclc/clc/include/clc/geometric/binary_decl.inc b/libclc/clc/include/clc/geometric/binary_decl.inc
index 3f3aafdeae6c..342d97a80265 100644
--- a/libclc/clc/include/clc/geometric/binary_decl.inc
+++ b/libclc/clc/include/clc/geometric/binary_decl.inc
@@ -10,7 +10,7 @@
 #if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
      __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
 
-_CLC_OVERLOAD _CLC_DECL __CLC_SCALAR_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                            __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                                      __CLC_GENTYPE b);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/binary_def.inc b/libclc/clc/include/clc/geometric/binary_def.inc
index 8fc875a05221..ad4fa4c01683 100644
--- a/libclc/clc/include/clc/geometric/binary_def.inc
+++ b/libclc/clc/include/clc/geometric/binary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
@@ -18,7 +18,7 @@
 
 _CLC_OVERLOAD _CLC_DEF __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                                      __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/clc_distance.h b/libclc/clc/include/clc/geometric/clc_distance.h
index 1acea411a08b..666e2c881bc9 100644
--- a/libclc/clc/include/clc/geometric/clc_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_distance.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DISTANCE_H__
 #define __CLC_GEOMETRIC_CLC_DISTANCE_H__
 
-#define __CLC_FUNCTION __clc_distance
+#define FUNCTION __clc_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_dot.h b/libclc/clc/include/clc/geometric/clc_dot.h
index a5a97d1bdd2b..4bfed6e3abc5 100644
--- a/libclc/clc/include/clc/geometric/clc_dot.h
+++ b/libclc/clc/include/clc/geometric/clc_dot.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DOT_H__
 #define __CLC_GEOMETRIC_CLC_DOT_H__
 
-#define __CLC_FUNCTION __clc_dot
+#define FUNCTION __clc_dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DOT_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_distance.h b/libclc/clc/include/clc/geometric/clc_fast_distance.h
index 489bea429951..2932d0147a6d 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_distance.h
@@ -10,12 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_fast_distance
+#define FUNCTION __clc_fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_length.h b/libclc/clc/include/clc/geometric/clc_fast_length.h
index 40552634fff9..903ecc8b9175 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_length.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_length.h
@@ -10,12 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_fast_length
+#define FUNCTION __clc_fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_normalize.h b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
index 66eed8b83ab1..a91862fe61cb 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
@@ -11,12 +11,11 @@
 
 #define __FLOAT_ONLY
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define __CLC_FUNCTION __clc_fast_normalize
+#define FUNCTION __clc_fast_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __FLOAT_ONLY
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_length.h b/libclc/clc/include/clc/geometric/clc_length.h
index f8ce12db0787..35b206e6c3fb 100644
--- a/libclc/clc/include/clc/geometric/clc_length.h
+++ b/libclc/clc/include/clc/geometric/clc_length.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_LENGTH_H__
 #define __CLC_GEOMETRIC_CLC_LENGTH_H__
 
-#define __CLC_FUNCTION __clc_length
+#define FUNCTION __clc_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_normalize.h b/libclc/clc/include/clc/geometric/clc_normalize.h
index 3058a72b2bbb..745ed7f4887d 100644
--- a/libclc/clc/include/clc/geometric/clc_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_normalize.h
@@ -10,11 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_NORMALIZE_H__
 
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define __CLC_FUNCTION __clc_normalize
+#define FUNCTION __clc_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
 
 #endif // __CLC_GEOMETRIC_CLC_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/unary_decl.inc b/libclc/clc/include/clc/geometric/unary_decl.inc
index 6da16d2e8abf..cff66000fcf7 100644
--- a/libclc/clc/include/clc/geometric/unary_decl.inc
+++ b/libclc/clc/include/clc/geometric/unary_decl.inc
@@ -16,6 +16,6 @@ _CLC_OVERLOAD _CLC_DECL
 #else
     __CLC_SCALAR_GENTYPE
 #endif
-    __CLC_FUNCTION(__CLC_GENTYPE a);
+    FUNCTION(__CLC_GENTYPE a);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/unary_def.inc b/libclc/clc/include/clc/geometric/unary_def.inc
index 5bedefb66276..ea90de0ec581 100644
--- a/libclc/clc/include/clc/geometric/unary_def.inc
+++ b/libclc/clc/include/clc/geometric/unary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
@@ -23,7 +23,7 @@ _CLC_OVERLOAD _CLC_DEF
     __CLC_SCALAR_GENTYPE
 #endif
     FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/integer/clc_add_sat.h b/libclc/clc/include/clc/integer/clc_add_sat.h
index 8c3495f6c519..be095d38225e 100644
--- a/libclc/clc/include/clc/integer/clc_add_sat.h
+++ b/libclc/clc/include/clc/integer/clc_add_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ADD_SAT_H__
 #define __CLC_INTEGER_CLC_ADD_SAT_H__
 
-#define __CLC_FUNCTION __clc_add_sat
+#define FUNCTION __clc_add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ADD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_clz.h b/libclc/clc/include/clc/integer/clc_clz.h
index dd62cc935f8c..3e0530bad6f1 100644
--- a/libclc/clc/include/clc/integer/clc_clz.h
+++ b/libclc/clc/include/clc/integer/clc_clz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CLZ_H__
 #define __CLC_INTEGER_CLC_CLZ_H__
 
-#define __CLC_FUNCTION __clc_clz
+#define FUNCTION __clc_clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CLZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_ctz.h b/libclc/clc/include/clc/integer/clc_ctz.h
index 3f2079e023f6..1495dc5560f1 100644
--- a/libclc/clc/include/clc/integer/clc_ctz.h
+++ b/libclc/clc/include/clc/integer/clc_ctz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CTZ_H__
 #define __CLC_INTEGER_CLC_CTZ_H__
 
-#define __CLC_FUNCTION __clc_ctz
+#define FUNCTION __clc_ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CTZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_hadd.h b/libclc/clc/include/clc/integer/clc_hadd.h
index 679369d3c9d4..7aaee9b601a2 100644
--- a/libclc/clc/include/clc/integer/clc_hadd.h
+++ b/libclc/clc/include/clc/integer/clc_hadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_HADD_H__
 #define __CLC_INTEGER_CLC_HADD_H__
 
-#define __CLC_FUNCTION __clc_hadd
+#define FUNCTION __clc_hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_HADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad24.h b/libclc/clc/include/clc/integer/clc_mad24.h
index e19bd905d22e..121840f81d63 100644
--- a/libclc/clc/include/clc/integer/clc_mad24.h
+++ b/libclc/clc/include/clc/integer/clc_mad24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD24_H__
 #define __CLC_INTEGER_CLC_MAD24_H__
 
-#define __CLC_FUNCTION __clc_mad24
+#define FUNCTION __clc_mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad_sat.h b/libclc/clc/include/clc/integer/clc_mad_sat.h
index 845378b3e7c4..0c418068ca4a 100644
--- a/libclc/clc/include/clc/integer/clc_mad_sat.h
+++ b/libclc/clc/include/clc/integer/clc_mad_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD_SAT_H__
 #define __CLC_INTEGER_CLC_MAD_SAT_H__
 
-#define __CLC_FUNCTION __clc_mad_sat
+#define FUNCTION __clc_mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul24.h b/libclc/clc/include/clc/integer/clc_mul24.h
index f813de55c71e..9310458d5981 100644
--- a/libclc/clc/include/clc/integer/clc_mul24.h
+++ b/libclc/clc/include/clc/integer/clc_mul24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL24_H__
 #define __CLC_INTEGER_CLC_MUL24_H__
 
-#define __CLC_FUNCTION __clc_mul24
+#define FUNCTION __clc_mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul_hi.h b/libclc/clc/include/clc/integer/clc_mul_hi.h
index e4738e5570e8..6542f237d09b 100644
--- a/libclc/clc/include/clc/integer/clc_mul_hi.h
+++ b/libclc/clc/include/clc/integer/clc_mul_hi.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL_HI_H__
 #define __CLC_INTEGER_CLC_MUL_HI_H__
 
-#define __CLC_FUNCTION __clc_mul_hi
+#define FUNCTION __clc_mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL_HI_H__
diff --git a/libclc/clc/include/clc/integer/clc_popcount.h b/libclc/clc/include/clc/integer/clc_popcount.h
index 55ca6ccfcec2..8ece2e998862 100644
--- a/libclc/clc/include/clc/integer/clc_popcount.h
+++ b/libclc/clc/include/clc/integer/clc_popcount.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_INTEGER_CLC_POPCOUNT_H__
 #define __CLC_INTEGER_CLC_POPCOUNT_H__
 
-#define __CLC_FUNCTION __clc_popcount
+#define FUNCTION __clc_popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_INTRINSIC
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_POPCOUNT_H__
diff --git a/libclc/clc/include/clc/integer/clc_rhadd.h b/libclc/clc/include/clc/integer/clc_rhadd.h
index b8b0b1a3a56e..882f950656df 100644
--- a/libclc/clc/include/clc/integer/clc_rhadd.h
+++ b/libclc/clc/include/clc/integer/clc_rhadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_RHADD_H__
 #define __CLC_INTEGER_CLC_RHADD_H__
 
-#define __CLC_FUNCTION __clc_rhadd
+#define FUNCTION __clc_rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_RHADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_rotate.h b/libclc/clc/include/clc/integer/clc_rotate.h
index 513bf9852623..61cf08788ef3 100644
--- a/libclc/clc/include/clc/integer/clc_rotate.h
+++ b/libclc/clc/include/clc/integer/clc_rotate.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ROTATE_H__
 #define __CLC_INTEGER_CLC_ROTATE_H__
 
-#define __CLC_FUNCTION __clc_rotate
+#define FUNCTION __clc_rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ROTATE_H__
diff --git a/libclc/clc/include/clc/integer/clc_sub_sat.h b/libclc/clc/include/clc/integer/clc_sub_sat.h
index 94a5484adefc..1b694a4a1780 100644
--- a/libclc/clc/include/clc/integer/clc_sub_sat.h
+++ b/libclc/clc/include/clc/integer/clc_sub_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_SUB_SAT_H__
 #define __CLC_INTEGER_CLC_SUB_SAT_H__
 
-#define __CLC_FUNCTION __clc_sub_sat
+#define FUNCTION __clc_sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTEGER_CLC_SUB_SAT_H__
diff --git a/libclc/clc/include/clc/internal/math/clc_sw_fma.h b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
index 5d6c76879ceb..8497f4efd113 100644
--- a/libclc/clc/include/clc/internal/math/clc_sw_fma.h
+++ b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 #define __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 
-#define __CLC_FUNCTION __clc_sw_fma
+#define FUNCTION __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
diff --git a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
index 6694b20a29ec..1b2b9ff33521 100644
--- a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_GENTYPE b);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                               __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
+                                               __CLC_SCALAR_GENTYPE b);
diff --git a/libclc/clc/include/clc/math/binary_def_via_fp32.inc b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
index 0b4108adb061..b53a178bf0f1 100644
--- a/libclc/clc/include/clc/math/binary_def_via_fp32.inc
+++ b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                    __CLC_GENTYPE y) {
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                              __CLC_GENTYPE y) {
   return __CLC_CONVERT_GENTYPE(
-      __CLC_FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
+      FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
 }
diff --git a/libclc/clc/include/clc/math/clc_acos.h b/libclc/clc/include/clc/math/clc_acos.h
index 1d484fa5b56c..130e3111ec1c 100644
--- a/libclc/clc/include/clc/math/clc_acos.h
+++ b/libclc/clc/include/clc/math/clc_acos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acos
+#define FUNCTION __clc_acos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOS_H__
diff --git a/libclc/clc/include/clc/math/clc_acosh.h b/libclc/clc/include/clc/math/clc_acosh.h
index 9492724bb27b..9b8c7184b974 100644
--- a/libclc/clc/include/clc/math/clc_acosh.h
+++ b/libclc/clc/include/clc/math/clc_acosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acosh
+#define FUNCTION __clc_acosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSH_H__
diff --git a/libclc/clc/include/clc/math/clc_acospi.h b/libclc/clc/include/clc/math/clc_acospi.h
index 90cfc06b7dff..6d4dc694900a 100644
--- a/libclc/clc/include/clc/math/clc_acospi.h
+++ b/libclc/clc/include/clc/math/clc_acospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_acospi
+#define FUNCTION __clc_acospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_asin.h b/libclc/clc/include/clc/math/clc_asin.h
index f6246c49f802..f3d69a7bf995 100644
--- a/libclc/clc/include/clc/math/clc_asin.h
+++ b/libclc/clc/include/clc/math/clc_asin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asin
+#define FUNCTION __clc_asin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASIN_H__
diff --git a/libclc/clc/include/clc/math/clc_asinh.h b/libclc/clc/include/clc/math/clc_asinh.h
index 05e44fa438ce..dc4530fa68b8 100644
--- a/libclc/clc/include/clc/math/clc_asinh.h
+++ b/libclc/clc/include/clc/math/clc_asinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asinh
+#define FUNCTION __clc_asinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINH_H__
diff --git a/libclc/clc/include/clc/math/clc_asinpi.h b/libclc/clc/include/clc/math/clc_asinpi.h
index 8f308dd06022..19d8ec3ad0c5 100644
--- a/libclc/clc/include/clc/math/clc_asinpi.h
+++ b/libclc/clc/include/clc/math/clc_asinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_asinpi
+#define FUNCTION __clc_asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_atan.h b/libclc/clc/include/clc/math/clc_atan.h
index 203e46c2b8a4..9f10efeae976 100644
--- a/libclc/clc/include/clc/math/clc_atan.h
+++ b/libclc/clc/include/clc/math/clc_atan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atan
+#define FUNCTION __clc_atan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2.h b/libclc/clc/include/clc/math/clc_atan2.h
index 6edc9dda48f4..99bd0760f3fe 100644
--- a/libclc/clc/include/clc/math/clc_atan2.h
+++ b/libclc/clc/include/clc/math/clc_atan2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_atan2
+#define FUNCTION __clc_atan2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2pi.h b/libclc/clc/include/clc/math/clc_atan2pi.h
index 2f26d1d9ae98..ae044b7bbfaf 100644
--- a/libclc/clc/include/clc/math/clc_atan2pi.h
+++ b/libclc/clc/include/clc/math/clc_atan2pi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2PI_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_atan2pi
+#define FUNCTION __clc_atan2pi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2PI_H__
diff --git a/libclc/clc/include/clc/math/clc_atanh.h b/libclc/clc/include/clc/math/clc_atanh.h
index 85f9fc53be1a..c1a95d4acba1 100644
--- a/libclc/clc/include/clc/math/clc_atanh.h
+++ b/libclc/clc/include/clc/math/clc_atanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atanh
+#define FUNCTION __clc_atanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANH_H__
diff --git a/libclc/clc/include/clc/math/clc_atanpi.h b/libclc/clc/include/clc/math/clc_atanpi.h
index 2b3c6b373ed4..43f752ecfe98 100644
--- a/libclc/clc/include/clc/math/clc_atanpi.h
+++ b/libclc/clc/include/clc/math/clc_atanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_atanpi
+#define FUNCTION __clc_atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_cbrt.inc b/libclc/clc/include/clc/math/clc_cbrt.inc
index fd9b750c8fd8..019b5174a901 100644
--- a/libclc/clc/include/clc/math/clc_cbrt.inc
+++ b/libclc/clc/include/clc/math/clc_cbrt.inc
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CBRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cbrt
+#define FUNCTION __clc_cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_CBRT_H__
diff --git a/libclc/clc/include/clc/math/clc_ceil.h b/libclc/clc/include/clc/math/clc_ceil.h
index 45668224c617..c60b794c83c6 100644
--- a/libclc/clc/include/clc/math/clc_ceil.h
+++ b/libclc/clc/include/clc/math/clc_ceil.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CEIL_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_ceil
+#define FUNCTION __clc_ceil
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_CEIL_H__
diff --git a/libclc/clc/include/clc/math/clc_copysign.h b/libclc/clc/include/clc/math/clc_copysign.h
index 563dd2827839..5976e42de250 100644
--- a/libclc/clc/include/clc/math/clc_copysign.h
+++ b/libclc/clc/include/clc/math/clc_copysign.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COPYSIGN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_copysign
+#define FUNCTION __clc_copysign
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COPYSIGN_H__
diff --git a/libclc/clc/include/clc/math/clc_cos.h b/libclc/clc/include/clc/math/clc_cos.h
index 44681608efc3..3a1ae9342e26 100644
--- a/libclc/clc/include/clc/math/clc_cos.h
+++ b/libclc/clc/include/clc/math/clc_cos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cos
+#define FUNCTION __clc_cos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_cosh.h b/libclc/clc/include/clc/math/clc_cosh.h
index 7bcfa2ea26e0..3033d76000dc 100644
--- a/libclc/clc/include/clc/math/clc_cosh.h
+++ b/libclc/clc/include/clc/math/clc_cosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cosh
+#define FUNCTION __clc_cosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COSH_H__
diff --git a/libclc/clc/include/clc/math/clc_cospi.h b/libclc/clc/include/clc/math/clc_cospi.h
index 4d82100649b0..3e4e73f096a0 100644
--- a/libclc/clc/include/clc/math/clc_cospi.h
+++ b/libclc/clc/include/clc/math/clc_cospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_cospi
+#define FUNCTION __clc_cospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_COSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_erf.h b/libclc/clc/include/clc/math/clc_erf.h
index 01a21b36b352..04b990c65133 100644
--- a/libclc/clc/include/clc/math/clc_erf.h
+++ b/libclc/clc/include/clc/math/clc_erf.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERF_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_erf
+#define FUNCTION __clc_erf
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ERF_H__
diff --git a/libclc/clc/include/clc/math/clc_erfc.h b/libclc/clc/include/clc/math/clc_erfc.h
index efd581542879..be2578b4bfb0 100644
--- a/libclc/clc/include/clc/math/clc_erfc.h
+++ b/libclc/clc/include/clc/math/clc_erfc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERFC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_erfc
+#define FUNCTION __clc_erfc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ERFC_H__
diff --git a/libclc/clc/include/clc/math/clc_exp.h b/libclc/clc/include/clc/math/clc_exp.h
index 84369bb8fbdd..355148934427 100644
--- a/libclc/clc/include/clc/math/clc_exp.h
+++ b/libclc/clc/include/clc/math/clc_exp.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp
+#define FUNCTION __clc_exp
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_exp10.h b/libclc/clc/include/clc/math/clc_exp10.h
index 500271cb0742..c281ee84dd3a 100644
--- a/libclc/clc/include/clc/math/clc_exp10.h
+++ b/libclc/clc/include/clc/math/clc_exp10.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp10
+#define FUNCTION __clc_exp10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_exp2.h b/libclc/clc/include/clc/math/clc_exp2.h
index 4ad0ddea48a7..bb10f12d34e2 100644
--- a/libclc/clc/include/clc/math/clc_exp2.h
+++ b/libclc/clc/include/clc/math/clc_exp2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_exp2
+#define FUNCTION __clc_exp2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_exp_helper.h b/libclc/clc/include/clc/math/clc_exp_helper.h
index 89c725ca0d5c..0c5028bd2385 100644
--- a/libclc/clc/include/clc/math/clc_exp_helper.h
+++ b/libclc/clc/include/clc/math/clc_exp_helper.h
@@ -14,6 +14,4 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif // __CLC_MATH_CLC_EXP_HELPER
diff --git a/libclc/clc/include/clc/math/clc_expm1.h b/libclc/clc/include/clc/math/clc_expm1.h
index 5c60960ca5b0..7a199dca17cb 100644
--- a/libclc/clc/include/clc/math/clc_expm1.h
+++ b/libclc/clc/include/clc/math/clc_expm1.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXPM1_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_expm1
+#define FUNCTION __clc_expm1
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_EXPM1_H__
diff --git a/libclc/clc/include/clc/math/clc_fabs.h b/libclc/clc/include/clc/math/clc_fabs.h
index 1ef38a8c7595..8af3bb572142 100644
--- a/libclc/clc/include/clc/math/clc_fabs.h
+++ b/libclc/clc/include/clc/math/clc_fabs.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FABS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_fabs
+#define FUNCTION __clc_fabs
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FABS_H__
diff --git a/libclc/clc/include/clc/math/clc_fdim.h b/libclc/clc/include/clc/math/clc_fdim.h
index 50ad045db62c..c678eb25f59b 100644
--- a/libclc/clc/include/clc/math/clc_fdim.h
+++ b/libclc/clc/include/clc/math/clc_fdim.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FDIM_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_fdim
+#define FUNCTION __clc_fdim
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FDIM_H__
diff --git a/libclc/clc/include/clc/math/clc_floor.h b/libclc/clc/include/clc/math/clc_floor.h
index d243819e0a42..b3fe8ea95ce1 100644
--- a/libclc/clc/include/clc/math/clc_floor.h
+++ b/libclc/clc/include/clc/math/clc_floor.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FLOOR_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_floor
+#define FUNCTION __clc_floor
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FLOOR_H__
diff --git a/libclc/clc/include/clc/math/clc_fma.h b/libclc/clc/include/clc/math/clc_fma.h
index 5e87f4c47085..efe20bd150eb 100644
--- a/libclc/clc/include/clc/math/clc_fma.h
+++ b/libclc/clc/include/clc/math/clc_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMA_H__
 #define __CLC_MATH_CLC_FMA_H__
 
-#define __CLC_FUNCTION __clc_fma
+#define FUNCTION __clc_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMA_H__
diff --git a/libclc/clc/include/clc/math/clc_fmax.h b/libclc/clc/include/clc/math/clc_fmax.h
index f367a9b6ea1c..fd5d38987ec1 100644
--- a/libclc/clc/include/clc/math/clc_fmax.h
+++ b/libclc/clc/include/clc/math/clc_fmax.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMAX_H__
 #define __CLC_MATH_CLC_FMAX_H__
 
-#define __CLC_FUNCTION __clc_fmax
+#define FUNCTION __clc_fmax
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMAX_H__
diff --git a/libclc/clc/include/clc/math/clc_fmin.h b/libclc/clc/include/clc/math/clc_fmin.h
index 47ed3fad2d8c..9fcdb56475ff 100644
--- a/libclc/clc/include/clc/math/clc_fmin.h
+++ b/libclc/clc/include/clc/math/clc_fmin.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMIN_H__
 #define __CLC_MATH_CLC_FMIN_H__
 
-#define __CLC_FUNCTION __clc_fmin
+#define FUNCTION __clc_fmin
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMIN_H__
diff --git a/libclc/clc/include/clc/math/clc_fmod.h b/libclc/clc/include/clc/math/clc_fmod.h
index 44179babd7fe..840bb850d73c 100644
--- a/libclc/clc/include/clc/math/clc_fmod.h
+++ b/libclc/clc/include/clc/math/clc_fmod.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMOD_H__
 #define __CLC_MATH_CLC_FMOD_H__
 
-#define __CLC_FUNCTION __clc_fmod
+#define FUNCTION __clc_fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FMOD_H__
diff --git a/libclc/clc/include/clc/math/clc_fract.h b/libclc/clc/include/clc/math/clc_fract.h
index af762ecc59ea..26c402c79641 100644
--- a/libclc/clc/include/clc/math/clc_fract.h
+++ b/libclc/clc/include/clc/math/clc_fract.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FRACT_H__
 #define __CLC_MATH_CLC_FRACT_H__
 
-#define __CLC_FUNCTION __clc_fract
+#define FUNCTION __clc_fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FRACT_H__
diff --git a/libclc/clc/include/clc/math/clc_frexp.h b/libclc/clc/include/clc/math/clc_frexp.h
index d6ed2754d0d4..e3c9f6b363dd 100644
--- a/libclc/clc/include/clc/math/clc_frexp.h
+++ b/libclc/clc/include/clc/math/clc_frexp.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_FREXP_H__
 #define __CLC_MATH_CLC_FREXP_H__
 
-#define __CLC_FUNCTION __clc_frexp
+#define FUNCTION __clc_frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_FREXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_cos.h b/libclc/clc/include/clc/math/clc_half_cos.h
index d0efc45793af..0f9f0b61443f 100644
--- a/libclc/clc/include/clc/math/clc_half_cos.h
+++ b/libclc/clc/include/clc/math/clc_half_cos.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_COS_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_cos
+#define FUNCTION __clc_half_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_half_divide.h b/libclc/clc/include/clc/math/clc_half_divide.h
index 5d1e7b9e33e3..44d90f39b4f6 100644
--- a/libclc/clc/include/clc/math/clc_half_divide.h
+++ b/libclc/clc/include/clc/math/clc_half_divide.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_DIVIDE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_divide
+#define FUNCTION __clc_half_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp.h b/libclc/clc/include/clc/math/clc_half_exp.h
index e95f3d42085c..158d28c16aab 100644
--- a/libclc/clc/include/clc/math/clc_half_exp.h
+++ b/libclc/clc/include/clc/math/clc_half_exp.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp
+#define FUNCTION __clc_half_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp10.h b/libclc/clc/include/clc/math/clc_half_exp10.h
index e4cce6e63bf7..4982b618e505 100644
--- a/libclc/clc/include/clc/math/clc_half_exp10.h
+++ b/libclc/clc/include/clc/math/clc_half_exp10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp10
+#define FUNCTION __clc_half_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp2.h b/libclc/clc/include/clc/math/clc_half_exp2.h
index 6fc99735f092..dc0f67659bf9 100644
--- a/libclc/clc/include/clc/math/clc_half_exp2.h
+++ b/libclc/clc/include/clc/math/clc_half_exp2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_EXP2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_exp2
+#define FUNCTION __clc_half_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log.h b/libclc/clc/include/clc/math/clc_half_log.h
index e44e68649966..df527e1a8fb5 100644
--- a/libclc/clc/include/clc/math/clc_half_log.h
+++ b/libclc/clc/include/clc/math/clc_half_log.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log
+#define FUNCTION __clc_half_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log10.h b/libclc/clc/include/clc/math/clc_half_log10.h
index 23e0ba116c72..47db4cd32593 100644
--- a/libclc/clc/include/clc/math/clc_half_log10.h
+++ b/libclc/clc/include/clc/math/clc_half_log10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log10
+#define FUNCTION __clc_half_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log2.h b/libclc/clc/include/clc/math/clc_half_log2.h
index 8ea2439eb3db..2b95b444b3f2 100644
--- a/libclc/clc/include/clc/math/clc_half_log2.h
+++ b/libclc/clc/include/clc/math/clc_half_log2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_LOG2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_log2
+#define FUNCTION __clc_half_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_powr.h b/libclc/clc/include/clc/math/clc_half_powr.h
index 252bfcd78504..2fd62213b824 100644
--- a/libclc/clc/include/clc/math/clc_half_powr.h
+++ b/libclc/clc/include/clc/math/clc_half_powr.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_POWR_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_powr
+#define FUNCTION __clc_half_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_half_recip.h b/libclc/clc/include/clc/math/clc_half_recip.h
index 1adb0bbe249c..d2ace8b7a17e 100644
--- a/libclc/clc/include/clc/math/clc_half_recip.h
+++ b/libclc/clc/include/clc/math/clc_half_recip.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_RECIP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_recip
+#define FUNCTION __clc_half_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_rsqrt.h b/libclc/clc/include/clc/math/clc_half_rsqrt.h
index 6342739ed6f2..a50dabccdaaf 100644
--- a/libclc/clc/include/clc/math/clc_half_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_rsqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_RSQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_rsqrt
+#define FUNCTION __clc_half_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sin.h b/libclc/clc/include/clc/math/clc_half_sin.h
index 31378d290fec..f79f6afc2b4e 100644
--- a/libclc/clc/include/clc/math/clc_half_sin.h
+++ b/libclc/clc/include/clc/math/clc_half_sin.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_SIN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_sin
+#define FUNCTION __clc_half_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sqrt.h b/libclc/clc/include/clc/math/clc_half_sqrt.h
index 0e765fb3e773..df5fc222a5c9 100644
--- a/libclc/clc/include/clc/math/clc_half_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_sqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_SQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_sqrt
+#define FUNCTION __clc_half_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_tan.h b/libclc/clc/include/clc/math/clc_half_tan.h
index 6c890a952522..0f6ef8790722 100644
--- a/libclc/clc/include/clc/math/clc_half_tan.h
+++ b/libclc/clc/include/clc/math/clc_half_tan.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_HALF_TAN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_half_tan
+#define FUNCTION __clc_half_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_hypot.h b/libclc/clc/include/clc/math/clc_hypot.h
index 40b354154fa5..b2d69ebda96e 100644
--- a/libclc/clc/include/clc/math/clc_hypot.h
+++ b/libclc/clc/include/clc/math/clc_hypot.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_HYPOT_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_hypot
+#define FUNCTION __clc_hypot
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_HYPOT_H__
diff --git a/libclc/clc/include/clc/math/clc_ilogb.h b/libclc/clc/include/clc/math/clc_ilogb.h
index 6aeaf6be3ac6..e02108ff1444 100644
--- a/libclc/clc/include/clc/math/clc_ilogb.h
+++ b/libclc/clc/include/clc/math/clc_ilogb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_ILOGB_H__
 #define __CLC_MATH_CLC_ILOGB_H__
 
-#define __CLC_FUNCTION __clc_ilogb
+#define FUNCTION __clc_ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ILOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_lgamma.h b/libclc/clc/include/clc/math/clc_lgamma.h
index f54c5e2b61b6..69b1ff9701ca 100644
--- a/libclc/clc/include/clc/math/clc_lgamma.h
+++ b/libclc/clc/include/clc/math/clc_lgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_lgamma
+#define FUNCTION __clc_lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_lgamma_r.h b/libclc/clc/include/clc/math/clc_lgamma_r.h
index 262e62372227..a98f760d8032 100644
--- a/libclc/clc/include/clc/math/clc_lgamma_r.h
+++ b/libclc/clc/include/clc/math/clc_lgamma_r.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LGAMMA_R_H__
 #define __CLC_MATH_CLC_LGAMMA_R_H__
 
-#define __CLC_FUNCTION __clc_lgamma_r
+#define FUNCTION __clc_lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_R_H__
diff --git a/libclc/clc/include/clc/math/clc_log.h b/libclc/clc/include/clc/math/clc_log.h
index 8b0e44b425a8..6bddb5ff746e 100644
--- a/libclc/clc/include/clc/math/clc_log.h
+++ b/libclc/clc/include/clc/math/clc_log.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG_H__
 #define __CLC_MATH_CLC_LOG_H__
 
-#define __CLC_FUNCTION __clc_log
+#define FUNCTION __clc_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_log10.h b/libclc/clc/include/clc/math/clc_log10.h
index 6e086690c6ad..e3a5b19eb418 100644
--- a/libclc/clc/include/clc/math/clc_log10.h
+++ b/libclc/clc/include/clc/math/clc_log10.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG10_H__
 #define __CLC_MATH_CLC_LOG10_H__
 
-#define __CLC_FUNCTION __clc_log10
+#define FUNCTION __clc_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_log1p.h b/libclc/clc/include/clc/math/clc_log1p.h
index 1be8c8548d48..f502802dafb2 100644
--- a/libclc/clc/include/clc/math/clc_log1p.h
+++ b/libclc/clc/include/clc/math/clc_log1p.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LOG1P_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_log1p
+#define FUNCTION __clc_log1p
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG1P_H__
diff --git a/libclc/clc/include/clc/math/clc_log2.h b/libclc/clc/include/clc/math/clc_log2.h
index c5f9e37f3984..7344cabaeffa 100644
--- a/libclc/clc/include/clc/math/clc_log2.h
+++ b/libclc/clc/include/clc/math/clc_log2.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG2_H__
 #define __CLC_MATH_CLC_LOG2_H__
 
-#define __CLC_FUNCTION __clc_log2
+#define FUNCTION __clc_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_logb.h b/libclc/clc/include/clc/math/clc_logb.h
index 75a2ce7e578b..1339d5819171 100644
--- a/libclc/clc/include/clc/math/clc_logb.h
+++ b/libclc/clc/include/clc/math/clc_logb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_LOGB_H__
 #define __CLC_MATH_CLC_LOGB_H__
 
-#define __CLC_FUNCTION __clc_logb
+#define FUNCTION __clc_logb
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_LOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_mad.h b/libclc/clc/include/clc/math/clc_mad.h
index 117e361380ed..72014b7c0ef7 100644
--- a/libclc/clc/include/clc/math/clc_mad.h
+++ b/libclc/clc/include/clc/math/clc_mad.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAD_H__
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION __clc_mad
+#define FUNCTION __clc_mad
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MAD_H__
diff --git a/libclc/clc/include/clc/math/clc_maxmag.h b/libclc/clc/include/clc/math/clc_maxmag.h
index 857ad7867b05..058230e6ebfc 100644
--- a/libclc/clc/include/clc/math/clc_maxmag.h
+++ b/libclc/clc/include/clc/math/clc_maxmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAXMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_maxmag
+#define FUNCTION __clc_maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MAXMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_minmag.h b/libclc/clc/include/clc/math/clc_minmag.h
index 9615d029bfed..581b95782823 100644
--- a/libclc/clc/include/clc/math/clc_minmag.h
+++ b/libclc/clc/include/clc/math/clc_minmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MINMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_minmag
+#define FUNCTION __clc_minmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MINMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_modf.h b/libclc/clc/include/clc/math/clc_modf.h
index aa8f9e20892a..fbdd3f8ffd4b 100644
--- a/libclc/clc/include/clc/math/clc_modf.h
+++ b/libclc/clc/include/clc/math/clc_modf.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_MODF_H__
 #define __CLC_MATH_CLC_MODF_H__
 
-#define __CLC_FUNCTION __clc_modf
+#define FUNCTION __clc_modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_MODF_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.h b/libclc/clc/include/clc/math/clc_nan.h
index 91901fd9369f..45d91184cf14 100644
--- a/libclc/clc/include/clc/math/clc_nan.h
+++ b/libclc/clc/include/clc/math/clc_nan.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_NAN_H__
 #define __CLC_MATH_CLC_NAN_H__
 
-#define __CLC_FUNCTION __clc_nan
+#define FUNCTION __clc_nan
 #define __CLC_BODY <clc/math/clc_nan.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.inc b/libclc/clc/include/clc/math/clc_nan.inc
index 637928f0839f..22604e0a5785 100644
--- a/libclc/clc/include/clc/math/clc_nan.inc
+++ b/libclc/clc/include/clc/math/clc_nan.inc
@@ -7,3 +7,4 @@
 //===----------------------------------------------------------------------===//
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code);
diff --git a/libclc/clc/include/clc/math/clc_native_cos.h b/libclc/clc/include/clc/math/clc_native_cos.h
index f5837d547913..08d51baca1f6 100644
--- a/libclc/clc/include/clc/math/clc_native_cos.h
+++ b/libclc/clc/include/clc/math/clc_native_cos.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_COS_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_cos
+#define FUNCTION __clc_native_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_native_divide.h b/libclc/clc/include/clc/math/clc_native_divide.h
index ed8118775e98..a10ed512f10d 100644
--- a/libclc/clc/include/clc/math/clc_native_divide.h
+++ b/libclc/clc/include/clc/math/clc_native_divide.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_DIVIDE_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_divide
+#define FUNCTION __clc_native_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp.h b/libclc/clc/include/clc/math/clc_native_exp.h
index b9c4005585da..30801fe2f8ad 100644
--- a/libclc/clc/include/clc/math/clc_native_exp.h
+++ b/libclc/clc/include/clc/math/clc_native_exp.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp
+#define FUNCTION __clc_native_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp10.h b/libclc/clc/include/clc/math/clc_native_exp10.h
index 5c49e46330d6..91cf60ef6a2b 100644
--- a/libclc/clc/include/clc/math/clc_native_exp10.h
+++ b/libclc/clc/include/clc/math/clc_native_exp10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp10
+#define FUNCTION __clc_native_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp2.h b/libclc/clc/include/clc/math/clc_native_exp2.h
index 2fa368f69fef..fe4728053619 100644
--- a/libclc/clc/include/clc/math/clc_native_exp2.h
+++ b/libclc/clc/include/clc/math/clc_native_exp2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_EXP2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_exp2
+#define FUNCTION __clc_native_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log.h b/libclc/clc/include/clc/math/clc_native_log.h
index 61fe0c0902d8..25a7054d2a59 100644
--- a/libclc/clc/include/clc/math/clc_native_log.h
+++ b/libclc/clc/include/clc/math/clc_native_log.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log
+#define FUNCTION __clc_native_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log10.h b/libclc/clc/include/clc/math/clc_native_log10.h
index f9fca1d94aed..d46124d655ee 100644
--- a/libclc/clc/include/clc/math/clc_native_log10.h
+++ b/libclc/clc/include/clc/math/clc_native_log10.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG10_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log10
+#define FUNCTION __clc_native_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log2.h b/libclc/clc/include/clc/math/clc_native_log2.h
index 69aec752b832..8998553b4984 100644
--- a/libclc/clc/include/clc/math/clc_native_log2.h
+++ b/libclc/clc/include/clc/math/clc_native_log2.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_LOG2_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_log2
+#define FUNCTION __clc_native_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_powr.h b/libclc/clc/include/clc/math/clc_native_powr.h
index 8d50f21a3124..932cfa66aeb9 100644
--- a/libclc/clc/include/clc/math/clc_native_powr.h
+++ b/libclc/clc/include/clc/math/clc_native_powr.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_POWR_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_powr
+#define FUNCTION __clc_native_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_native_recip.h b/libclc/clc/include/clc/math/clc_native_recip.h
index 94d494ad07a1..6b94cbc3d6c6 100644
--- a/libclc/clc/include/clc/math/clc_native_recip.h
+++ b/libclc/clc/include/clc/math/clc_native_recip.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_RECIP_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_recip
+#define FUNCTION __clc_native_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_rsqrt.h b/libclc/clc/include/clc/math/clc_native_rsqrt.h
index 5b58d6efd758..5fa870b3f868 100644
--- a/libclc/clc/include/clc/math/clc_native_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_rsqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_RSQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_rsqrt
+#define FUNCTION __clc_native_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sin.h b/libclc/clc/include/clc/math/clc_native_sin.h
index 27f5ecbf40e7..d25968c74409 100644
--- a/libclc/clc/include/clc/math/clc_native_sin.h
+++ b/libclc/clc/include/clc/math/clc_native_sin.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_SIN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_sin
+#define FUNCTION __clc_native_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sqrt.h b/libclc/clc/include/clc/math/clc_native_sqrt.h
index a45f8b7f21a2..e8a28b7f7046 100644
--- a/libclc/clc/include/clc/math/clc_native_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_sqrt.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_SQRT_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_sqrt
+#define FUNCTION __clc_native_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_tan.h b/libclc/clc/include/clc/math/clc_native_tan.h
index 362088a5d58f..36fdd6d8c011 100644
--- a/libclc/clc/include/clc/math/clc_native_tan.h
+++ b/libclc/clc/include/clc/math/clc_native_tan.h
@@ -10,12 +10,11 @@
 #define __CLC_MATH_CLC_NATIVE_TAN_H__
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION __clc_native_tan
+#define FUNCTION __clc_native_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
-#undef __FLOAT_ONLY
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nextafter.h b/libclc/clc/include/clc/math/clc_nextafter.h
index f24742b9e9c3..d080a40a06f0 100644
--- a/libclc/clc/include/clc/math/clc_nextafter.h
+++ b/libclc/clc/include/clc/math/clc_nextafter.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_NEXTAFTER_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_nextafter
+#define FUNCTION __clc_nextafter
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_NEXTAFTER_H__
diff --git a/libclc/clc/include/clc/math/clc_pow.h b/libclc/clc/include/clc/math/clc_pow.h
index 5e37e5bf6da6..b33391e10a42 100644
--- a/libclc/clc/include/clc/math/clc_pow.h
+++ b/libclc/clc/include/clc/math/clc_pow.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POW_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_pow
+#define FUNCTION __clc_pow
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POW_H__
diff --git a/libclc/clc/include/clc/math/clc_pown.h b/libclc/clc/include/clc/math/clc_pown.h
index 67475503f92b..473bbe1603c5 100644
--- a/libclc/clc/include/clc/math/clc_pown.h
+++ b/libclc/clc/include/clc/math/clc_pown.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_POWN_H__
 #define __CLC_MATH_CLC_POWN_H__
 
-#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION __clc_pown
+#define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
+#define FUNCTION __clc_pown
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POWN_H__
diff --git a/libclc/clc/include/clc/math/clc_powr.h b/libclc/clc/include/clc/math/clc_powr.h
index baa494cce698..a820516f1866 100644
--- a/libclc/clc/include/clc/math/clc_powr.h
+++ b/libclc/clc/include/clc/math/clc_powr.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POWR_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION __clc_powr
+#define FUNCTION __clc_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_remainder.h b/libclc/clc/include/clc/math/clc_remainder.h
index a7700c95103b..bc53b043045c 100644
--- a/libclc/clc/include/clc/math/clc_remainder.h
+++ b/libclc/clc/include/clc/math/clc_remainder.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMAINDER_H__
 #define __CLC_MATH_CLC_REMAINDER_H__
 
-#define __CLC_FUNCTION __clc_remainder
+#define FUNCTION __clc_remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_REMAINDER_H__
diff --git a/libclc/clc/include/clc/math/clc_remquo.h b/libclc/clc/include/clc/math/clc_remquo.h
index 48a8844a6e38..b46f94ba9447 100644
--- a/libclc/clc/include/clc/math/clc_remquo.h
+++ b/libclc/clc/include/clc/math/clc_remquo.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMQUO_H__
 #define __CLC_MATH_CLC_REMQUO_H__
 
-#define __CLC_FUNCTION __clc_remquo
+#define FUNCTION __clc_remquo
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_REMQUO_H__
diff --git a/libclc/clc/include/clc/math/clc_rint.h b/libclc/clc/include/clc/math/clc_rint.h
index 17b1549f439b..c446c7576471 100644
--- a/libclc/clc/include/clc/math/clc_rint.h
+++ b/libclc/clc/include/clc/math/clc_rint.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RINT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_rint
+#define FUNCTION __clc_rint
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_RINT_H__
diff --git a/libclc/clc/include/clc/math/clc_rootn.h b/libclc/clc/include/clc/math/clc_rootn.h
index bf9dd5413c3d..fb7817ee250f 100644
--- a/libclc/clc/include/clc/math/clc_rootn.h
+++ b/libclc/clc/include/clc/math/clc_rootn.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_ROOTN_H__
 #define __CLC_MATH_CLC_ROOTN_H__
 
-#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION __clc_rootn
+#define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
+#define FUNCTION __clc_rootn
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ROOTN_H__
diff --git a/libclc/clc/include/clc/math/clc_round.h b/libclc/clc/include/clc/math/clc_round.h
index 78ebf4db99e9..fd1872a5cf22 100644
--- a/libclc/clc/include/clc/math/clc_round.h
+++ b/libclc/clc/include/clc/math/clc_round.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ROUND_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_round
+#define FUNCTION __clc_round
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_ROUND_H__
diff --git a/libclc/clc/include/clc/math/clc_rsqrt.h b/libclc/clc/include/clc/math/clc_rsqrt.h
index 59402fbead2b..1690ede1b892 100644
--- a/libclc/clc/include/clc/math/clc_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_rsqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_rsqrt
+#define FUNCTION __clc_rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_sin.h b/libclc/clc/include/clc/math/clc_sin.h
index de4c722ca123..89ae444eb6e3 100644
--- a/libclc/clc/include/clc/math/clc_sin.h
+++ b/libclc/clc/include/clc/math/clc_sin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sin
+#define FUNCTION __clc_sin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos.h b/libclc/clc/include/clc/math/clc_sincos.h
index e26dc7c024c9..bf3e0806db38 100644
--- a/libclc/clc/include/clc/math/clc_sincos.h
+++ b/libclc/clc/include/clc/math/clc_sincos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINCOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define __CLC_FUNCTION __clc_sincos
+#define FUNCTION __clc_sincos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINCOS_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos_helpers.h b/libclc/clc/include/clc/math/clc_sincos_helpers.h
index e3a2e1c0b605..f9ceba3bf2cf 100644
--- a/libclc/clc/include/clc/math/clc_sincos_helpers.h
+++ b/libclc/clc/include/clc/math/clc_sincos_helpers.h
@@ -14,13 +14,9 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-
 #define __DOUBLE_ONLY
 #define __CLC_BODY <clc/math/clc_sincos_helpers_fp64.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif // __CLC_MATH_CLC_SINCOS_HELPERS_H__
diff --git a/libclc/clc/include/clc/math/clc_sinh.h b/libclc/clc/include/clc/math/clc_sinh.h
index fa9cabe79cb0..c94d8e37acc6 100644
--- a/libclc/clc/include/clc/math/clc_sinh.h
+++ b/libclc/clc/include/clc/math/clc_sinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sinh
+#define FUNCTION __clc_sinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINH_H__
diff --git a/libclc/clc/include/clc/math/clc_sinpi.h b/libclc/clc/include/clc/math/clc_sinpi.h
index db563c2e7e14..725e559600ed 100644
--- a/libclc/clc/include/clc/math/clc_sinpi.h
+++ b/libclc/clc/include/clc/math/clc_sinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sinpi
+#define FUNCTION __clc_sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_sqrt.h b/libclc/clc/include/clc/math/clc_sqrt.h
index 708451553329..74aea8e85809 100644
--- a/libclc/clc/include/clc/math/clc_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_sqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_sqrt
+#define FUNCTION __clc_sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_tan.h b/libclc/clc/include/clc/math/clc_tan.h
index 028ff28ecd69..d16370420a03 100644
--- a/libclc/clc/include/clc/math/clc_tan.h
+++ b/libclc/clc/include/clc/math/clc_tan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tan
+#define FUNCTION __clc_tan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_tanh.h b/libclc/clc/include/clc/math/clc_tanh.h
index 88fce2340d99..2c860c165ca6 100644
--- a/libclc/clc/include/clc/math/clc_tanh.h
+++ b/libclc/clc/include/clc/math/clc_tanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tanh
+#define FUNCTION __clc_tanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TANH_H__
diff --git a/libclc/clc/include/clc/math/clc_tanpi.h b/libclc/clc/include/clc/math/clc_tanpi.h
index 596bb72b4b8d..2691fa696c0a 100644
--- a/libclc/clc/include/clc/math/clc_tanpi.h
+++ b/libclc/clc/include/clc/math/clc_tanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tanpi
+#define FUNCTION __clc_tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_tgamma.h b/libclc/clc/include/clc/math/clc_tgamma.h
index dd76e041875d..d067703c2e9d 100644
--- a/libclc/clc/include/clc/math/clc_tgamma.h
+++ b/libclc/clc/include/clc/math/clc_tgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_tgamma
+#define FUNCTION __clc_tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_trunc.h b/libclc/clc/include/clc/math/clc_trunc.h
index 33445f9fe15d..6a8353a4f6d4 100644
--- a/libclc/clc/include/clc/math/clc_trunc.h
+++ b/libclc/clc/include/clc/math/clc_trunc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TRUNC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION __clc_trunc
+#define FUNCTION __clc_trunc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MATH_CLC_TRUNC_H__
diff --git a/libclc/clc/include/clc/math/gentype.inc b/libclc/clc/include/clc/math/gentype.inc
index a8d8317a9cac..d74a9932bd0a 100644
--- a/libclc/clc/include/clc/math/gentype.inc
+++ b/libclc/clc/include/clc/math/gentype.inc
@@ -349,3 +349,7 @@
 
 #undef __CLC_AS_GENTYPE
 #undef __CLC_CONVERT_GENTYPE
+
+#undef __HALF_ONLY
+#undef __FLOAT_ONLY
+#undef __DOUBLE_ONLY
diff --git a/libclc/clc/include/clc/math/remquo_decl.inc b/libclc/clc/include/clc/math/remquo_decl.inc
index cba28a7244eb..24d96b048805 100644
--- a/libclc/clc/include/clc/math/remquo_decl.inc
+++ b/libclc/clc/include/clc/math/remquo_decl.inc
@@ -6,19 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     private __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               private __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     global __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               global __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     local __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               local __CLC_INTN *q);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y,
-                                                     generic __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
+                                               generic __CLC_INTN *q);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_decl.inc b/libclc/clc/include/clc/math/unary_decl.inc
index 5d742de00b71..ce1a71b8284a 100644
--- a/libclc/clc/include/clc/math/unary_decl.inc
+++ b/libclc/clc/include/clc/math/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
index 8bfe17c3c681..d62046a5292c 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     global __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     local __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     private __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               global __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               local __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               private __CLC_INTN *iptr);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     generic __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               generic __CLC_INTN *iptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
index 333a752ced39..ed8d0a12506f 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
index 63a20e3d21c5..ca0077bec558 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     global __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     local __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_GENTYPE x, private __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               global __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               local __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               private __CLC_GENTYPE *ptr);
 
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_GENTYPE x, generic __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               generic __CLC_GENTYPE *ptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_via_fp32.inc b/libclc/clc/include/clc/math/unary_def_via_fp32.inc
index aefee6170571..f109e8ec4a11 100644
--- a/libclc/clc/include/clc/math/unary_def_via_fp32.inc
+++ b/libclc/clc/include/clc/math/unary_def_via_fp32.inc
@@ -6,6 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x) {
-  return __CLC_CONVERT_GENTYPE(__CLC_FUNCTION(__CLC_CONVERT_FLOATN(x)));
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x) {
+  return __CLC_CONVERT_GENTYPE(FUNCTION(__CLC_CONVERT_FLOATN(x)));
 }
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
index 464c6ffc7c8c..a8ed34ad04bb 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               private __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               global __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               local __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               generic __CLC_INTN *iptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, iptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_return.inc b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
index 33be58bf4584..c7bb84cf5f7c 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/math/unary_def_with_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
index fecec80dde83..c3faa9aab87c 100644
--- a/libclc/clc/include/clc/math/unary_def_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               private __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               global __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               local __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
                                               generic __CLC_GENTYPE *ptr) {
-  return __CLC_FUNCTION(FUNCTION)(x, ptr);
+  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/misc/clc_shuffle.h b/libclc/clc/include/clc/misc/clc_shuffle.h
index ba3d7a2bddd6..1e9ec6828f95 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE_H__
 #define __CLC_MISC_CLC_SHUFFLE_H__
 
-#define __CLC_FUNCTION __clc_shuffle
+#define FUNCTION __clc_shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE_H__
diff --git a/libclc/clc/include/clc/misc/clc_shuffle2.h b/libclc/clc/include/clc/misc/clc_shuffle2.h
index 84bafba12b21..fb5361cfeb95 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle2.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle2.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE2_H__
 #define __CLC_MISC_CLC_SHUFFLE2_H__
 
-#define __CLC_FUNCTION __clc_shuffle2
+#define FUNCTION __clc_shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE2_H__
diff --git a/libclc/clc/include/clc/misc/shuffle2_decl.inc b/libclc/clc/include/clc/misc/shuffle2_decl.inc
index 2fc992ef2c17..47fd5e28a314 100644
--- a/libclc/clc/include/clc/misc/shuffle2_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_decl.inc
@@ -13,16 +13,16 @@
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
-               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
+         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle2_def.inc b/libclc/clc/include/clc/misc/shuffle2_def.inc
index 099a175d665d..0415b8c18729 100644
--- a/libclc/clc/include/clc/misc/shuffle2_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_def.inc
@@ -11,8 +11,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
@@ -21,22 +21,22 @@
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
          __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, y, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_decl.inc b/libclc/clc/include/clc/misc/shuffle_decl.inc
index 5e7e5b24c287..c6c64d5b5fed 100644
--- a/libclc/clc/include/clc/misc/shuffle_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle_decl.inc
@@ -13,12 +13,12 @@
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
+FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_def.inc b/libclc/clc/include/clc/misc/shuffle_def.inc
index 84c873e6458e..d2a088498fc6 100644
--- a/libclc/clc/include/clc/misc/shuffle_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle_def.inc
@@ -11,8 +11,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
@@ -20,19 +20,19 @@
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
 FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask) {
-  return __CLC_FUNCTION(FUNCTION)(x, mask);
+  return __IMPL_FUNCTION(FUNCTION)(x, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/relational/binary_decl.inc b/libclc/clc/include/clc/relational/binary_decl.inc
index b82fb85d9742..bcdf5238b8f5 100644
--- a/libclc/clc/include/clc/relational/binary_decl.inc
+++ b/libclc/clc/include/clc/relational/binary_decl.inc
@@ -6,5 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_FLOATN a,
-                                                  __CLC_FLOATN b);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_FLOATN a, __CLC_FLOATN b);
diff --git a/libclc/clc/include/clc/relational/clc_isfinite.h b/libclc/clc/include/clc/relational/clc_isfinite.h
index 37483edd3761..5e71ec7a0640 100644
--- a/libclc/clc/include/clc/relational/clc_isfinite.h
+++ b/libclc/clc/include/clc/relational/clc_isfinite.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISFINITE_H__
 #define __CLC_RELATIONAL_CLC_ISFINITE_H__
 
-#define __CLC_FUNCTION __clc_isfinite
+#define FUNCTION __clc_isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISFINITE_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreater.h b/libclc/clc/include/clc/relational/clc_isgreater.h
index 4ce18ee63171..e2e6911a80cd 100644
--- a/libclc/clc/include/clc/relational/clc_isgreater.h
+++ b/libclc/clc/include/clc/relational/clc_isgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISGREATER_H__
 
-#define __CLC_FUNCTION __clc_isgreater
+#define FUNCTION __clc_isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreaterequal.h b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
index c0a430cc61e9..3fe8835aff9d 100644
--- a/libclc/clc/include/clc/relational/clc_isgreaterequal.h
+++ b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 
-#define __CLC_FUNCTION __clc_isgreaterequal
+#define FUNCTION __clc_isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isless.h b/libclc/clc/include/clc/relational/clc_isless.h
index e68079397062..01384cf6fa4a 100644
--- a/libclc/clc/include/clc/relational/clc_isless.h
+++ b/libclc/clc/include/clc/relational/clc_isless.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESS_H__
 #define __CLC_RELATIONAL_CLC_ISLESS_H__
 
-#define __CLC_FUNCTION __clc_isless
+#define FUNCTION __clc_isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESS_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessequal.h b/libclc/clc/include/clc/relational/clc_islessequal.h
index 8e95d04bf475..a4b77a451b24 100644
--- a/libclc/clc/include/clc/relational/clc_islessequal.h
+++ b/libclc/clc/include/clc/relational/clc_islessequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 
-#define __CLC_FUNCTION __clc_islessequal
+#define FUNCTION __clc_islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessgreater.h b/libclc/clc/include/clc/relational/clc_islessgreater.h
index 69c43e40b463..9fb6d641bfa1 100644
--- a/libclc/clc/include/clc/relational/clc_islessgreater.h
+++ b/libclc/clc/include/clc/relational/clc_islessgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 
-#define __CLC_FUNCTION __clc_islessgreater
+#define FUNCTION __clc_islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnormal.h b/libclc/clc/include/clc/relational/clc_isnormal.h
index 4fee10a02244..d580fed5a739 100644
--- a/libclc/clc/include/clc/relational/clc_isnormal.h
+++ b/libclc/clc/include/clc/relational/clc_isnormal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNORMAL_H__
 #define __CLC_RELATIONAL_CLC_ISNORMAL_H__
 
-#define __CLC_FUNCTION __clc_isnormal
+#define FUNCTION __clc_isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNORMAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnotequal.h b/libclc/clc/include/clc/relational/clc_isnotequal.h
index 3aac82cfb938..16982fc3c5aa 100644
--- a/libclc/clc/include/clc/relational/clc_isnotequal.h
+++ b/libclc/clc/include/clc/relational/clc_isnotequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 
-#define __CLC_FUNCTION __clc_isnotequal
+#define FUNCTION __clc_isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isordered.h b/libclc/clc/include/clc/relational/clc_isordered.h
index 20b5adb0cce7..7ba26662105f 100644
--- a/libclc/clc/include/clc/relational/clc_isordered.h
+++ b/libclc/clc/include/clc/relational/clc_isordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISORDERED_H__
 
-#define __CLC_FUNCTION __clc_isordered
+#define FUNCTION __clc_isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_isunordered.h b/libclc/clc/include/clc/relational/clc_isunordered.h
index 7e693233ba34..eac158d24519 100644
--- a/libclc/clc/include/clc/relational/clc_isunordered.h
+++ b/libclc/clc/include/clc/relational/clc_isunordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 
-#define __CLC_FUNCTION __clc_isunordered
+#define FUNCTION __clc_isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISUNORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_signbit.h b/libclc/clc/include/clc/relational/clc_signbit.h
index 55159c9cc56d..892263a09e99 100644
--- a/libclc/clc/include/clc/relational/clc_signbit.h
+++ b/libclc/clc/include/clc/relational/clc_signbit.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_SIGNBIT_H__
 #define __CLC_RELATIONAL_CLC_SIGNBIT_H__
 
-#define __CLC_FUNCTION __clc_signbit
+#define FUNCTION __clc_signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_SIGNBIT_H__
diff --git a/libclc/clc/include/clc/relational/unary_decl.inc b/libclc/clc/include/clc/relational/unary_decl.inc
index cff02d0cb9c4..b9fb36c90546 100644
--- a/libclc/clc/include/clc/relational/unary_decl.inc
+++ b/libclc/clc/include/clc/relational/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_FLOATN x);
+_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_FLOATN x);
diff --git a/libclc/clc/include/clc/shared/binary_decl.inc b/libclc/clc/include/clc/shared/binary_decl.inc
index 589a31a7305a..ff4739dcdb8d 100644
--- a/libclc/clc/include/clc/shared/binary_decl.inc
+++ b/libclc/clc/include/clc/shared/binary_decl.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_GENTYPE y);
diff --git a/libclc/clc/include/clc/math/binary_decl_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
similarity index 74%
rename from libclc/clc/include/clc/math/binary_decl_with_int_second_arg.inc
rename to libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
index 1a854abe11d7..a86d89b4fc85 100644
--- a/libclc/clc/include/clc/math/binary_decl_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
@@ -8,5 +8,4 @@
 
 #include <clc/utils.h>
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_INTN y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y);
diff --git a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc b/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
index 6044902a5be7..05a53393170f 100644
--- a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_GENTYPE y);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
-                                                     __CLC_SCALAR_GENTYPE y);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
+                                               __CLC_SCALAR_GENTYPE y);
 #endif
diff --git a/libclc/clc/include/clc/shared/binary_def.inc b/libclc/clc/include/clc/shared/binary_def.inc
index 646e1504f482..9ab5a7c5a79e 100644
--- a/libclc/clc/include/clc/shared/binary_def.inc
+++ b/libclc/clc/include/clc/shared/binary_def.inc
@@ -8,11 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_scalarize.inc b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
new file mode 100644
index 000000000000..53d4d3fa23b1
--- /dev/null
+++ b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_DEF_SPEC
+#define __CLC_DEF_SPEC _CLC_DEF
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
+                                                     __CLC_ARG2_TYPE y) {
+  return __IMPL_FUNCTION(x, y);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+#define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
+                                                      __CLC_ARG2_TYPE2 y) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#undef __CLC_ARG2_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+#define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
+                                                      __CLC_ARG2_TYPE3 y) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+#undef __CLC_ARG2_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+#define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
+                                                      __CLC_ARG2_TYPE4 y) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+#undef __CLC_ARG2_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+#define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
+                                                      __CLC_ARG2_TYPE8 y) {
+  return (
+      __CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0), __IMPL_FUNCTION(x.s1, y.s1),
+                       __IMPL_FUNCTION(x.s2, y.s2), __IMPL_FUNCTION(x.s3, y.s3),
+                       __IMPL_FUNCTION(x.s4, y.s4), __IMPL_FUNCTION(x.s5, y.s5),
+                       __IMPL_FUNCTION(x.s6, y.s6),
+                       __IMPL_FUNCTION(x.s7, y.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+#undef __CLC_ARG2_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+#define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
+                                                       __CLC_ARG2_TYPE16 y) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0),
+                            __IMPL_FUNCTION(x.s1, y.s1),
+                            __IMPL_FUNCTION(x.s2, y.s2),
+                            __IMPL_FUNCTION(x.s3, y.s3),
+                            __IMPL_FUNCTION(x.s4, y.s4),
+                            __IMPL_FUNCTION(x.s5, y.s5),
+                            __IMPL_FUNCTION(x.s6, y.s6),
+                            __IMPL_FUNCTION(x.s7, y.s7),
+                            __IMPL_FUNCTION(x.s8, y.s8),
+                            __IMPL_FUNCTION(x.s9, y.s9),
+                            __IMPL_FUNCTION(x.sa, y.sa),
+                            __IMPL_FUNCTION(x.sb, y.sb),
+                            __IMPL_FUNCTION(x.sc, y.sc),
+                            __IMPL_FUNCTION(x.sd, y.sd),
+                            __IMPL_FUNCTION(x.se, y.se),
+                            __IMPL_FUNCTION(x.sf, y.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+#undef __CLC_ARG2_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/include/clc/math/binary_def_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
similarity index 80%
rename from libclc/clc/include/clc/math/binary_def_with_int_second_arg.inc
rename to libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
index 756bce6538ab..262b6b21cc64 100644
--- a/libclc/clc/include/clc/math/binary_def_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y) {
-  return __CLC_FUNCTION(FUNCTION)(x, y);
+  return __IMPL_FUNCTION(FUNCTION)(x, y);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
index 49fa7d2ce36e..103f775ab87e 100644
--- a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
@@ -8,18 +8,18 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
 
 #ifndef __CLC_SCALAR
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
                                               __CLC_SCALAR_GENTYPE b) {
-  return __CLC_FUNCTION(FUNCTION)(a, (__CLC_GENTYPE)b);
+  return __IMPL_FUNCTION(FUNCTION)(a, (__CLC_GENTYPE)b);
 }
 #endif
diff --git a/libclc/clc/include/clc/shared/ternary_decl.inc b/libclc/clc/include/clc/shared/ternary_decl.inc
index e734c0f89807..a76db1eefb6a 100644
--- a/libclc/clc/include/clc/shared/ternary_decl.inc
+++ b/libclc/clc/include/clc/shared/ternary_decl.inc
@@ -6,6 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_GENTYPE b,
-                                                     __CLC_GENTYPE c);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
+                                               __CLC_GENTYPE c);
diff --git a/libclc/clc/include/clc/shared/ternary_def.inc b/libclc/clc/include/clc/shared/ternary_def.inc
index 36952136e74c..92134e8cb6f2 100644
--- a/libclc/clc/include/clc/shared/ternary_def.inc
+++ b/libclc/clc/include/clc/shared/ternary_def.inc
@@ -8,11 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
                                               __CLC_GENTYPE c) {
-  return __CLC_FUNCTION(FUNCTION)(a, b, c);
+  return __IMPL_FUNCTION(FUNCTION)(a, b, c);
 }
diff --git a/libclc/clc/include/clc/shared/ternary_def_scalarize.inc b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
new file mode 100644
index 000000000000..c216fa4db4ca
--- /dev/null
+++ b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
@@ -0,0 +1,153 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_DEF_SPEC
+#define __CLC_DEF_SPEC _CLC_DEF
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG3_TYPE
+#define __CLC_ARG3_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
+                                                     __CLC_ARG2_TYPE y,
+                                                     __CLC_ARG3_TYPE z) {
+  return __IMPL_FUNCTION(x, y, z);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+#define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
+#define __CLC_ARG3_TYPE2 __CLC_XCONCAT(__CLC_ARG3_TYPE, 2)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
+                                                      __CLC_ARG2_TYPE2 y,
+                                                      __CLC_ARG3_TYPE2 z) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#undef __CLC_ARG2_TYPE2
+#undef __CLC_ARG3_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+#define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
+#define __CLC_ARG3_TYPE3 __CLC_XCONCAT(__CLC_ARG3_TYPE, 3)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
+                                                      __CLC_ARG2_TYPE3 y,
+                                                      __CLC_ARG3_TYPE3 z) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+#undef __CLC_ARG2_TYPE3
+#undef __CLC_ARG3_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+#define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
+#define __CLC_ARG3_TYPE4 __CLC_XCONCAT(__CLC_ARG3_TYPE, 4)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
+                                                      __CLC_ARG2_TYPE4 y,
+                                                      __CLC_ARG3_TYPE4 z) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3, z.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+#undef __CLC_ARG2_TYPE4
+#undef __CLC_ARG3_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+#define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
+#define __CLC_ARG3_TYPE8 __CLC_XCONCAT(__CLC_ARG3_TYPE, 8)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
+                                                      __CLC_ARG2_TYPE8 y,
+                                                      __CLC_ARG3_TYPE8 z) {
+  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                           __IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                           __IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                           __IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                           __IMPL_FUNCTION(x.s7, y.s7, z.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+#undef __CLC_ARG2_TYPE8
+#undef __CLC_ARG3_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+#define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
+#define __CLC_ARG3_TYPE16 __CLC_XCONCAT(__CLC_ARG3_TYPE, 16)
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
+                                                       __CLC_ARG2_TYPE16 y,
+                                                       __CLC_ARG3_TYPE16 z) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                            __IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                            __IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                            __IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                            __IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                            __IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                            __IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                            __IMPL_FUNCTION(x.s7, y.s7, z.s7),
+                            __IMPL_FUNCTION(x.s8, y.s8, z.s8),
+                            __IMPL_FUNCTION(x.s9, y.s9, z.s9),
+                            __IMPL_FUNCTION(x.sa, y.sa, z.sa),
+                            __IMPL_FUNCTION(x.sb, y.sb, z.sb),
+                            __IMPL_FUNCTION(x.sc, y.sc, z.sc),
+                            __IMPL_FUNCTION(x.sd, y.sd, z.sd),
+                            __IMPL_FUNCTION(x.se, y.se, z.se),
+                            __IMPL_FUNCTION(x.sf, y.sf, z.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+#undef __CLC_ARG2_TYPE16
+#undef __CLC_ARG3_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/include/clc/shared/unary_decl.inc b/libclc/clc/include/clc/shared/unary_decl.inc
index 5d742de00b71..ce1a71b8284a 100644
--- a/libclc/clc/include/clc/shared/unary_decl.inc
+++ b/libclc/clc/include/clc/shared/unary_decl.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/shared/unary_def.inc b/libclc/clc/include/clc/shared/unary_def.inc
index 3325d410ede5..e746a6bc89f3 100644
--- a/libclc/clc/include/clc/shared/unary_def.inc
+++ b/libclc/clc/include/clc/shared/unary_def.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __CLC_FUNCTION
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/shared/unary_def_scalarize.inc b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
new file mode 100644
index 000000000000..6c82babad2b5
--- /dev/null
+++ b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/utils.h>
+
+#ifdef __CLC_SCALAR
+
+#ifndef __CLC_MIN_VECSIZE
+#define __CLC_MIN_VECSIZE 2
+#endif
+
+#ifndef __IMPL_FUNCTION
+#define __IMPL_FUNCTION FUNCTION
+#endif
+
+#ifndef __CLC_RET_TYPE
+#define __CLC_RET_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG1_TYPE
+#define __CLC_ARG1_TYPE __CLC_GENTYPE
+#endif
+
+#ifndef __CLC_ARG2_TYPE
+#define __CLC_ARG2_TYPE __CLC_GENTYPE
+#endif
+
+#if __CLC_MIN_VECSIZE == 1
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x) {
+  return __IMPL_FUNCTION(x);
+}
+#endif // __CLC_MIN_VECSIZE == 1
+
+#if __CLC_MIN_VECSIZE <= 2
+#define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
+#define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x) {
+  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1));
+}
+#undef __CLC_RET_TYPE2
+#undef __CLC_ARG1_TYPE2
+#endif // __CLC_MIN_VECSIZE <= 2
+
+#if __CLC_MIN_VECSIZE > 3
+#error "__CLC_MIN_VECSIZE > 3 isn't implemented"
+#endif
+
+#define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
+#define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x) {
+  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2));
+}
+#undef __CLC_RET_TYPE3
+#undef __CLC_ARG1_TYPE3
+
+#define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
+#define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x) {
+  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3));
+}
+#undef __CLC_RET_TYPE4
+#undef __CLC_ARG1_TYPE4
+
+#define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
+#define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x) {
+  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
+                           __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
+                           __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7));
+}
+#undef __CLC_RET_TYPE8
+#undef __CLC_ARG1_TYPE8
+
+#define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
+#define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x) {
+  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
+                            __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
+                            __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
+                            __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7),
+                            __IMPL_FUNCTION(x.s8), __IMPL_FUNCTION(x.s9),
+                            __IMPL_FUNCTION(x.sa), __IMPL_FUNCTION(x.sb),
+                            __IMPL_FUNCTION(x.sc), __IMPL_FUNCTION(x.sd),
+                            __IMPL_FUNCTION(x.se), __IMPL_FUNCTION(x.sf));
+}
+#undef __CLC_RET_TYPE16
+#undef __CLC_ARG1_TYPE16
+
+#endif // __CLC_SCALAR
diff --git a/libclc/clc/lib/amdgcn/math/clc_fmax.cl b/libclc/clc/lib/amdgcn/math/clc_fmax.cl
index 20bdcadb9eab..cea90a7135d5 100644
--- a/libclc/clc/lib/amdgcn/math/clc_fmax.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_fmax.cl
@@ -18,7 +18,6 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmax(float x, float y) {
   y = __builtin_canonicalizef(y);
   return __builtin_fmaxf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -29,8 +28,6 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmax(double x, double y) {
   y = __builtin_canonicalize(y);
   return __builtin_fmax(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmax, double,
-                      double)
 
 #endif
 #ifdef cl_khr_fp16
@@ -44,6 +41,9 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmax(half x, half y) {
     return x;
   return (y < x) ? x : y;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmax, half, half)
 
 #endif
+
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgcn/math/clc_fmin.cl b/libclc/clc/lib/amdgcn/math/clc_fmin.cl
index a5f66dfefa90..12bb0c64429f 100644
--- a/libclc/clc/lib/amdgcn/math/clc_fmin.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_fmin.cl
@@ -18,7 +18,6 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmin(float x, float y) {
   y = __builtin_canonicalizef(y);
   return __builtin_fminf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
 
 #ifdef cl_khr_fp64
 
@@ -29,8 +28,6 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmin(double x, double y) {
   y = __builtin_canonicalize(y);
   return __builtin_fmin(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmin, double,
-                      double)
 
 #endif
 
@@ -45,6 +42,9 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmin(half x, half y) {
     return x;
   return (y < x) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmin, half, half)
 
 #endif
+
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
index 94a6d6ccfa9f..b2990647a00a 100644
--- a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
@@ -10,14 +10,25 @@
 #include <clc/internal/clc.h>
 #include <clc/math/clc_ldexp.h>
 
+#define FUNCTION __clc_ldexp
+#define __CLC_ARG2_TYPE int
+#define __CLC_MIN_VECSIZE 1
+
 #ifdef __HAS_LDEXPF__
 // This defines all the ldexp(floatN, intN) variants.
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_ldexp, __builtin_amdgcn_ldexpf, float, int);
+#define __FLOAT_ONLY
+#define __IMPL_FUNCTION __builtin_amdgcn_ldexpf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __IMPL_FUNCTION
 #endif
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 // This defines all the ldexp(doubleN, intN) variants.
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_ldexp, __builtin_amdgcn_ldexp, double,
-                           int);
+#define __DOUBLE_ONLY
+#define __IMPL_FUNCTION __builtin_amdgcn_ldexp
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __IMPL_FUNCTION
 #endif
diff --git a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
index 76b1850fce57..63f28921dc17 100644
--- a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
@@ -9,8 +9,9 @@
 #include <clc/clcmacro.h>
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF float __clc_native_exp2(float val) {
-  return __builtin_amdgcn_exp2f(val);
-}
-
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_native_exp2, float)
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_native_exp2
+#define __IMPL_FUNCTION __builtin_amdgcn_exp2f
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
index 576525bc47f6..b7cb635a2ae8 100644
--- a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
@@ -43,6 +43,9 @@ _CLC_OVERLOAD _CLC_DEF double __clc_sqrt(double x) {
   return (x == __builtin_inf() || (x == 0.0)) ? v01 : v23;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_sqrt, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_sqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
 
 #endif
diff --git a/libclc/clc/lib/clspv/math/clc_sw_fma.cl b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
index 82978380a63f..266269644721 100644
--- a/libclc/clc/lib/clspv/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
@@ -269,5 +269,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa.lo & 0x7fffff));
 }
 
-_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float,
-                       float, float)
+#define __FLOAT_ONLY
+#define FUNCTION __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_add_sat.cl b/libclc/clc/lib/generic/integer/clc_add_sat.cl
index b0cf51e61597..8241ee5d547a 100644
--- a/libclc/clc/lib/generic/integer/clc_add_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_add_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_add_sat
-#define __CLC_FUNCTION(x) __builtin_elementwise_add_sat
+#define __IMPL_FUNCTION(x) __builtin_elementwise_add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_clz.cl b/libclc/clc/lib/generic/integer/clc_clz.cl
index 74f662375af6..c6e1da680b7b 100644
--- a/libclc/clc/lib/generic/integer/clc_clz.cl
+++ b/libclc/clc/lib/generic/integer/clc_clz.cl
@@ -42,11 +42,6 @@ _CLC_OVERLOAD _CLC_DEF ulong __clc_clz(ulong x) {
   return x ? __builtin_clzl(x) : 64;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, __clc_clz, char)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, __clc_clz, uchar)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, __clc_clz, short)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, __clc_clz, ushort)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, __clc_clz, int)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, __clc_clz, uint)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, __clc_clz, long)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, __clc_clz, ulong)
+#define FUNCTION __clc_clz
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_ctz.cl b/libclc/clc/lib/generic/integer/clc_ctz.cl
index 50fda4a214b2..d82d99d539df 100644
--- a/libclc/clc/lib/generic/integer/clc_ctz.cl
+++ b/libclc/clc/lib/generic/integer/clc_ctz.cl
@@ -38,11 +38,6 @@ _CLC_OVERLOAD _CLC_DEF ulong __clc_ctz(ulong x) {
   return __builtin_ctzg(x, 64);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, char, __clc_ctz, char)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uchar, __clc_ctz, uchar)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, short, __clc_ctz, short)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ushort, __clc_ctz, ushort)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, int, __clc_ctz, int)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, uint, __clc_ctz, uint)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, long, __clc_ctz, long)
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, ulong, __clc_ctz, ulong)
+#define FUNCTION __clc_ctz
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_popcount.cl b/libclc/clc/lib/generic/integer/clc_popcount.cl
index f9e1cdf98204..078ceecc9769 100644
--- a/libclc/clc/lib/generic/integer/clc_popcount.cl
+++ b/libclc/clc/lib/generic/integer/clc_popcount.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_popcount
-#define __CLC_FUNCTION(x) __builtin_elementwise_popcount
+#define __IMPL_FUNCTION(x) __builtin_elementwise_popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_sub_sat.cl b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
index a9aa5f67323f..e664f04b1748 100644
--- a/libclc/clc/lib/generic/integer/clc_sub_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_sub_sat
-#define __CLC_FUNCTION(x) __builtin_elementwise_sub_sat
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_ceil.cl b/libclc/clc/lib/generic/math/clc_ceil.cl
index c4df638d5ce2..b06cb364b58d 100644
--- a/libclc/clc/lib/generic/math/clc_ceil.cl
+++ b/libclc/clc/lib/generic/math/clc_ceil.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_ceil
-#define __CLC_FUNCTION(x) __builtin_elementwise_ceil
+#define __IMPL_FUNCTION(x) __builtin_elementwise_ceil
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_copysign.cl b/libclc/clc/lib/generic/math/clc_copysign.cl
index d336985ebf96..06f126b16dfd 100644
--- a/libclc/clc/lib/generic/math/clc_copysign.cl
+++ b/libclc/clc/lib/generic/math/clc_copysign.cl
@@ -6,30 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/internal/clc.h>
 
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(float, __clc_copysign,
-                                        __builtin_elementwise_copysign, float,
-                                        float)
-
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(double, __clc_copysign,
-                                        __builtin_elementwise_copysign, double,
-                                        double)
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(half, __clc_copysign,
-                                        __builtin_elementwise_copysign, half,
-                                        half)
-
-#endif
+#define FUNCTION __clc_copysign
+#define __IMPL_FUNCTION(x) __builtin_elementwise_copysign
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_erf.cl b/libclc/clc/lib/generic/math/clc_erf.cl
index 3808b9b81411..17fee3e8d11f 100644
--- a/libclc/clc/lib/generic/math/clc_erf.cl
+++ b/libclc/clc/lib/generic/math/clc_erf.cl
@@ -211,7 +211,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_erf(float x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_erf, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_erf
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -496,7 +500,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_erf(double y) {
   return y < 0.0 ? -ret : ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erf, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_erf
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -508,7 +516,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erf, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_erf
+#define FUNCTION __clc_erf
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_erfc.cl b/libclc/clc/lib/generic/math/clc_erfc.cl
index a9edfdbb72f2..77f8959831d2 100644
--- a/libclc/clc/lib/generic/math/clc_erfc.cl
+++ b/libclc/clc/lib/generic/math/clc_erfc.cl
@@ -211,7 +211,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_erfc(float x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_erfc, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_erfc
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -505,7 +509,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_erfc(double x) {
   return ret;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erfc, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_erfc
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -517,7 +525,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_erfc, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_erfc
+#define FUNCTION __clc_erfc
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_fabs.cl b/libclc/clc/lib/generic/math/clc_fabs.cl
index f684c1e76bbb..a0f0f3af07e2 100644
--- a/libclc/clc/lib/generic/math/clc_fabs.cl
+++ b/libclc/clc/lib/generic/math/clc_fabs.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_fabs
-#define __CLC_FUNCTION(x) __builtin_elementwise_abs
+#define __IMPL_FUNCTION(x) __builtin_elementwise_abs
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_floor.cl b/libclc/clc/lib/generic/math/clc_floor.cl
index 0626ba36f3ea..a14adb98297e 100644
--- a/libclc/clc/lib/generic/math/clc_floor.cl
+++ b/libclc/clc/lib/generic/math/clc_floor.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_floor
-#define __CLC_FUNCTION(x) __builtin_elementwise_floor
+#define __IMPL_FUNCTION(x) __builtin_elementwise_floor
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_fmax.cl b/libclc/clc/lib/generic/math/clc_fmax.cl
index 8ee369f57d38..5ebbf1b28df9 100644
--- a/libclc/clc/lib/generic/math/clc_fmax.cl
+++ b/libclc/clc/lib/generic/math/clc_fmax.cl
@@ -10,13 +10,29 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/clc_isnan.h>
 
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_fmax, __builtin_fmaxf, float, float);
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmax
+#define __IMPL_FUNCTION __builtin_fmaxf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_fmax, __builtin_fmax, double, double);
+#define __DOUBLE_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmax
+#define __IMPL_FUNCTION __builtin_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #endif
 
@@ -31,6 +47,12 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmax(half x, half y) {
     return x;
   return (x < y) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmax, half, half)
+
+#define __HALF_ONLY
+#define __CLC_SUPPORTED_VECSIZE_OR_1 2
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_fmin.cl b/libclc/clc/lib/generic/math/clc_fmin.cl
index 2f307274b9be..5bddbb863412 100644
--- a/libclc/clc/lib/generic/math/clc_fmin.cl
+++ b/libclc/clc/lib/generic/math/clc_fmin.cl
@@ -10,13 +10,29 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/clc_isnan.h>
 
-_CLC_DEFINE_BINARY_BUILTIN(float, __clc_fmin, __builtin_fminf, float, float);
+#define __FLOAT_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmin
+#define __IMPL_FUNCTION __builtin_fminf
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_DEFINE_BINARY_BUILTIN(double, __clc_fmin, __builtin_fmin, double, double);
+#define __DOUBLE_ONLY
+#define __CLC_MIN_VECSIZE 1
+#define FUNCTION __clc_fmin
+#define __IMPL_FUNCTION __builtin_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_MIN_VECSIZE
+#undef FUNCTION
+#undef __IMPL_FUNCTION
 
 #endif
 
@@ -31,6 +47,11 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmin(half x, half y) {
     return x;
   return (y < x) ? y : x;
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_fmin, half, half)
+
+#define __HALF_ONLY
+#define __CLC_SUPPORTED_VECSIZE_OR_1 2
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_fmod.cl b/libclc/clc/lib/generic/math/clc_fmod.cl
index 6af84a14f3d1..3162ef60d46e 100644
--- a/libclc/clc/lib/generic/math/clc_fmod.cl
+++ b/libclc/clc/lib/generic/math/clc_fmod.cl
@@ -63,7 +63,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
 
   return xr;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_fmod, float, float);
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmod
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -170,8 +175,13 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
 
   return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double,
-                      double);
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmod
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
+
 #endif
 
 #ifdef cl_khr_fp16
@@ -180,7 +190,7 @@ _CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_fmod, double,
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_fmod
+#define FUNCTION __clc_fmod
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_half_cos.cl b/libclc/clc/lib/generic/math/clc_half_cos.cl
index c2064a00232f..79a7ee7b483f 100644
--- a/libclc/clc/lib/generic/math/clc_half_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_half_cos.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_cos
-#define __CLC_FUNCTION(x) __clc_cos
+#define __IMPL_FUNCTION(x) __clc_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp.cl b/libclc/clc/lib/generic/math/clc_half_exp.cl
index 88336f9df011..b53454580e3c 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp
-#define __CLC_FUNCTION(x) __clc_exp
+#define __IMPL_FUNCTION(x) __clc_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp10.cl b/libclc/clc/lib/generic/math/clc_half_exp10.cl
index b3c0001a8527..7bd107bf59e8 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp10
-#define __CLC_FUNCTION(x) __clc_exp10
+#define __IMPL_FUNCTION(x) __clc_exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp2.cl b/libclc/clc/lib/generic/math/clc_half_exp2.cl
index f3ffcc6876b9..a20d79bfff6e 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_exp2
-#define __CLC_FUNCTION(x) __clc_exp2
+#define __IMPL_FUNCTION(x) __clc_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log.cl b/libclc/clc/lib/generic/math/clc_half_log.cl
index 7371344ab2d1..26b2c756c988 100644
--- a/libclc/clc/lib/generic/math/clc_half_log.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log
-#define __CLC_FUNCTION(x) __clc_log
+#define __IMPL_FUNCTION(x) __clc_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log10.cl b/libclc/clc/lib/generic/math/clc_half_log10.cl
index e9becad210ea..36f5bf55bb53 100644
--- a/libclc/clc/lib/generic/math/clc_half_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log10
-#define __CLC_FUNCTION(x) __clc_log10
+#define __IMPL_FUNCTION(x) __clc_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log2.cl b/libclc/clc/lib/generic/math/clc_half_log2.cl
index aa8d838afad7..cbf1d350dd8e 100644
--- a/libclc/clc/lib/generic/math/clc_half_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_log2
-#define __CLC_FUNCTION(x) __clc_log2
+#define __IMPL_FUNCTION(x) __clc_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_powr.cl b/libclc/clc/lib/generic/math/clc_half_powr.cl
index ade6731db143..f7ef2074b840 100644
--- a/libclc/clc/lib/generic/math/clc_half_powr.cl
+++ b/libclc/clc/lib/generic/math/clc_half_powr.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_powr
-#define __CLC_FUNCTION(x) __clc_powr
+#define __IMPL_FUNCTION(x) __clc_powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
index 07daf7fc3623..7e5d9e052685 100644
--- a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_rsqrt
-#define __CLC_FUNCTION(x) __clc_rsqrt
+#define __IMPL_FUNCTION(x) __clc_rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sin.cl b/libclc/clc/lib/generic/math/clc_half_sin.cl
index c0af5e7bf471..ef333dae3797 100644
--- a/libclc/clc/lib/generic/math/clc_half_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sin.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_sin
-#define __CLC_FUNCTION(x) __clc_sin
+#define __IMPL_FUNCTION(x) __clc_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sqrt.cl b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
index b6b5097e965c..293eb81196b6 100644
--- a/libclc/clc/lib/generic/math/clc_half_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_sqrt
-#define __CLC_FUNCTION(x) __clc_sqrt
+#define __IMPL_FUNCTION(x) __clc_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_tan.cl b/libclc/clc/lib/generic/math/clc_half_tan.cl
index ca27f3c662aa..ecc9f8f32831 100644
--- a/libclc/clc/lib/generic/math/clc_half_tan.cl
+++ b/libclc/clc/lib/generic/math/clc_half_tan.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_half_tan
-#define __CLC_FUNCTION(x) __clc_tan
+#define __IMPL_FUNCTION(x) __clc_tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_ldexp.cl b/libclc/clc/lib/generic/math/clc_ldexp.cl
index 144a8fc897d9..cb4185d89c72 100644
--- a/libclc/clc/lib/generic/math/clc_ldexp.cl
+++ b/libclc/clc/lib/generic/math/clc_ldexp.cl
@@ -86,8 +86,6 @@ _CLC_DEF_ldexp _CLC_OVERLOAD float __clc_ldexp(float x, int n) {
   return val_f;
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, float, __clc_ldexp, float, int);
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -121,8 +119,6 @@ _CLC_DEF_ldexp _CLC_OVERLOAD double __clc_ldexp(double x, int n) {
   return mr;
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, double, __clc_ldexp, double, int);
-
 #endif
 
 #ifdef cl_khr_fp16
@@ -133,6 +129,10 @@ _CLC_OVERLOAD _CLC_DEF_ldexp half __clc_ldexp(half x, int n) {
   return (half)__clc_ldexp((float)x, n);
 }
 
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF_ldexp, half, __clc_ldexp, half, int);
-
 #endif
+
+#define FUNCTION __clc_ldexp
+#define __CLC_DEF_SPEC _CLC_DEF_ldexp
+#define __CLC_ARG2_TYPE int
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_lgamma_r.cl b/libclc/clc/lib/generic/math/clc_lgamma_r.cl
index 96a42bbb6e15..ffacfc17a4d0 100644
--- a/libclc/clc/lib/generic/math/clc_lgamma_r.cl
+++ b/libclc/clc/lib/generic/math/clc_lgamma_r.cl
@@ -611,3 +611,10 @@ _CLC_V_V_VP_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_lgamma_r, half,
 #define __CLC_BODY <clc_lgamma_r.inc>
 #include <clc/math/gentype.inc>
 #undef __CLC_ADDRSPACE
+
+#if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
+#define __CLC_ADDRSPACE generic
+#define __CLC_BODY <clc_lgamma_r.inc>
+#include <clc/math/gentype.inc>
+#undef __CLC_ADDRSPACE
+#endif
diff --git a/libclc/clc/lib/generic/math/clc_log.cl b/libclc/clc/lib/generic/math/clc_log.cl
index 85405faf8b8c..cf5628f206ca 100644
--- a/libclc/clc/lib/generic/math/clc_log.cl
+++ b/libclc/clc/lib/generic/math/clc_log.cl
@@ -19,8 +19,6 @@ _CLC_OVERLOAD _CLC_DEF float __clc_log(float x) {
   return __clc_log2(x) * (1.0f / M_LOG2E_F);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log, float);
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -29,8 +27,6 @@ _CLC_OVERLOAD _CLC_DEF double __clc_log(double x) {
   return __clc_log2(x) * (1.0 / M_LOG2E);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log, double);
-
 #endif // cl_khr_fp64
 
 #ifdef cl_khr_fp16
@@ -41,6 +37,8 @@ _CLC_OVERLOAD _CLC_DEF half __clc_log(half x) {
   return (half)__clc_log2((float)x) * (1.0h / M_LOG2E_H);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log, half);
-
 #endif // cl_khr_fp16
+
+#define FUNCTION __clc_log
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log10.cl b/libclc/clc/lib/generic/math/clc_log10.cl
index 689ec25b74e5..f5f0e8cc7083 100644
--- a/libclc/clc/lib/generic/math/clc_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_log10.cl
@@ -22,12 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG10
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log10, float);
-
-#ifdef cl_khr_fp64
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log10, double);
-#endif // cl_khr_fp64
-
-#ifdef cl_khr_fp16
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log10, half);
-#endif // cl_khr_fp16
+#define FUNCTION __clc_log10
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log2.cl b/libclc/clc/lib/generic/math/clc_log2.cl
index ebd981c9d995..335488af2f3d 100644
--- a/libclc/clc/lib/generic/math/clc_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_log2.cl
@@ -22,12 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG2
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_log2, float);
-
-#ifdef cl_khr_fp64
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_log2, double);
-#endif // cl_khr_fp64
-
-#ifdef cl_khr_fp16
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, half, __clc_log2, half);
-#endif // cl_khr_fp16
+#define FUNCTION __clc_log2
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_nan.inc b/libclc/clc/lib/generic/math/clc_nan.inc
index e934416351db..46e828ba48c7 100644
--- a/libclc/clc/lib/generic/math/clc_nan.inc
+++ b/libclc/clc/lib/generic/math/clc_nan.inc
@@ -20,4 +20,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code) {
   return __CLC_AS_GENTYPE(res);
 }
 
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code) {
+  return __clc_nan(__CLC_AS_U_GENTYPE(code));
+}
+
 #undef NAN_MASK
diff --git a/libclc/clc/lib/generic/math/clc_native_cos.cl b/libclc/clc/lib/generic/math/clc_native_cos.cl
index de56fdec48d2..2e63e9c9493f 100644
--- a/libclc/clc/lib/generic/math/clc_native_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_native_cos.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_cos
-#define __CLC_FUNCTION(x) __builtin_elementwise_cos
+#define __IMPL_FUNCTION(x) __builtin_elementwise_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp.cl b/libclc/clc/lib/generic/math/clc_native_exp.cl
index 400270a6163a..2f50a0aefac9 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_exp
-#define __CLC_FUNCTION(x) __builtin_elementwise_exp
+#define __IMPL_FUNCTION(x) __builtin_elementwise_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp2.cl b/libclc/clc/lib/generic/math/clc_native_exp2.cl
index 427d901fcdb1..25f557306c35 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_exp2
-#define __CLC_FUNCTION(x) __builtin_elementwise_exp2
+#define __IMPL_FUNCTION(x) __builtin_elementwise_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log.cl b/libclc/clc/lib/generic/math/clc_native_log.cl
index 85f188b65428..b9b9d274f8b9 100644
--- a/libclc/clc/lib/generic/math/clc_native_log.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log
-#define __CLC_FUNCTION(x) __builtin_elementwise_log
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log10.cl b/libclc/clc/lib/generic/math/clc_native_log10.cl
index 624018e4481b..221aa406466c 100644
--- a/libclc/clc/lib/generic/math/clc_native_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log10.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log10
-#define __CLC_FUNCTION(x) __builtin_elementwise_log10
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log2.cl b/libclc/clc/lib/generic/math/clc_native_log2.cl
index 2c8c18e61ca5..c3008ce242c7 100644
--- a/libclc/clc/lib/generic/math/clc_native_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log2.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_log2
-#define __CLC_FUNCTION(x) __builtin_elementwise_log2
+#define __IMPL_FUNCTION(x) __builtin_elementwise_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sin.cl b/libclc/clc/lib/generic/math/clc_native_sin.cl
index 22b988bf4375..533f8d726d41 100644
--- a/libclc/clc/lib/generic/math/clc_native_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sin.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_sin
-#define __CLC_FUNCTION(x) __builtin_elementwise_sin
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sqrt.cl b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
index ed022ef1fee1..e0f028e988b4 100644
--- a/libclc/clc/lib/generic/math/clc_native_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
@@ -10,7 +10,7 @@
 
 #define __FLOAT_ONLY
 #define FUNCTION __clc_native_sqrt
-#define __CLC_FUNCTION(x) __builtin_elementwise_sqrt
+#define __IMPL_FUNCTION(x) __builtin_elementwise_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_pow.inc b/libclc/clc/lib/generic/math/clc_pow.inc
index 98e154984aaa..8b1f820268ba 100644
--- a/libclc/clc/lib/generic/math/clc_pow.inc
+++ b/libclc/clc/lib/generic/math/clc_pow.inc
@@ -330,6 +330,15 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
     const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081;
     const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10;
 
+    // If v is so large that we need to return INFINITY, or so small that we
+    // need to return 0, set v to known values that will produce that result. Do
+    // not try to continue the computation with the original v and patch it up
+    // afterwards because v may be so large that temp is out of range of int, in
+    // which case that conversion, and a value based on that conversion being
+    // passed to __clc_ldexp, results in undefined behavior.
+    v = v > max_exp_arg ? 1000.0 : v;
+    v = v < min_exp_arg ? -1000.0 : v;
+
     __CLC_GENTYPE temp = v * sixtyfour_by_lnof2;
     __CLC_INTN n = __CLC_CONVERT_INTN(temp);
     __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n);
@@ -357,10 +366,6 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
 
     expv = __clc_fma(f, q, f2) + f1;
     expv = __clc_ldexp(expv, m);
-
-    expv = v > max_exp_arg ? __CLC_AS_GENTYPE((__CLC_ULONGN)0x7FF0000000000000L)
-                           : expv;
-    expv = v < min_exp_arg ? 0.0 : expv;
   }
 
   // See whether y is an integer.
diff --git a/libclc/clc/lib/generic/math/clc_pown.inc b/libclc/clc/lib/generic/math/clc_pown.inc
index 8bdc407e9ac8..483fd2faf271 100644
--- a/libclc/clc/lib/generic/math/clc_pown.inc
+++ b/libclc/clc/lib/generic/math/clc_pown.inc
@@ -317,6 +317,15 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
     const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081;
     const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10;
 
+    // If v is so large that we need to return INFINITY, or so small that we
+    // need to return 0, set v to known values that will produce that result. Do
+    // not try to continue the computation with the original v and patch it up
+    // afterwards because v may be so large that temp is out of range of int, in
+    // which case that conversion, and a value based on that conversion being
+    // passed to __clc_ldexp, results in undefined behavior.
+    v = v > max_exp_arg ? 1000.0 : v;
+    v = v < min_exp_arg ? -1000.0 : v;
+
     __CLC_GENTYPE temp = v * sixtyfour_by_lnof2;
     __CLC_INTN n = __CLC_CONVERT_INTN(temp);
     __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n);
@@ -344,10 +353,6 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
 
     expv = __clc_fma(f, q, f2) + f1;
     expv = __clc_ldexp(expv, m);
-
-    expv = v > max_exp_arg ? __CLC_AS_GENTYPE((__CLC_ULONGN)0x7FF0000000000000L)
-                           : expv;
-    expv = v < min_exp_arg ? 0.0 : expv;
   }
 
   // See whether y is an integer.
diff --git a/libclc/clc/lib/generic/math/clc_powr.inc b/libclc/clc/lib/generic/math/clc_powr.inc
index fbdf3d85de2b..1244f7f6ac5d 100644
--- a/libclc/clc/lib/generic/math/clc_powr.inc
+++ b/libclc/clc/lib/generic/math/clc_powr.inc
@@ -316,6 +316,15 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
     const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081;
     const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10;
 
+    // If v is so large that we need to return INFINITY, or so small that we
+    // need to return 0, set v to known values that will produce that result. Do
+    // not try to continue the computation with the original v and patch it up
+    // afterwards because v may be so large that temp is out of range of int, in
+    // which case that conversion, and a value based on that conversion being
+    // passed to __clc_ldexp, results in undefined behavior.
+    v = v > max_exp_arg ? 1000.0 : v;
+    v = v < min_exp_arg ? -1000.0 : v;
+
     __CLC_GENTYPE temp = v * sixtyfour_by_lnof2;
     __CLC_INTN n = __CLC_CONVERT_INTN(temp);
     __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n);
@@ -343,10 +352,6 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
 
     expv = __clc_fma(f, q, f2) + f1;
     expv = __clc_ldexp(expv, m);
-
-    expv = v > max_exp_arg ? __CLC_AS_GENTYPE((__CLC_ULONGN)0x7FF0000000000000L)
-                           : expv;
-    expv = v < min_exp_arg ? 0.0 : expv;
   }
 
   // See whether y is an integer.
diff --git a/libclc/clc/lib/generic/math/clc_remainder.cl b/libclc/clc/lib/generic/math/clc_remainder.cl
index c79d271c624e..2b3d185a8bee 100644
--- a/libclc/clc/lib/generic/math/clc_remainder.cl
+++ b/libclc/clc/lib/generic/math/clc_remainder.cl
@@ -73,8 +73,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remainder(float x, float y) {
 
   return xr;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_remainder, float,
-                      float);
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_remainder
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -207,8 +211,13 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y) {
 
   return ret;
 }
-_CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double,
-                      double);
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_remainder
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
+
 #endif
 
 #ifdef cl_khr_fp16
@@ -217,7 +226,7 @@ _CLC_BINARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, double, __clc_remainder, double,
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_remainder
+#define FUNCTION __clc_remainder
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_rint.cl b/libclc/clc/lib/generic/math/clc_rint.cl
index 2188a7d901f0..b3f94490e3b5 100644
--- a/libclc/clc/lib/generic/math/clc_rint.cl
+++ b/libclc/clc/lib/generic/math/clc_rint.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_rint
-#define __CLC_FUNCTION(x) __builtin_elementwise_rint
+#define __IMPL_FUNCTION(x) __builtin_elementwise_rint
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_rootn.inc b/libclc/clc/lib/generic/math/clc_rootn.inc
index 0c459ae5c3cb..996f88f14535 100644
--- a/libclc/clc/lib/generic/math/clc_rootn.inc
+++ b/libclc/clc/lib/generic/math/clc_rootn.inc
@@ -323,6 +323,15 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
     const __CLC_GENTYPE lnof2_by_64_head = 0.010830424260348081;
     const __CLC_GENTYPE lnof2_by_64_tail = -4.359010638708991e-10;
 
+    // If v is so large that we need to return INFINITY, or so small that we
+    // need to return 0, set v to known values that will produce that result. Do
+    // not try to continue the computation with the original v and patch it up
+    // afterwards because v may be so large that temp is out of range of int, in
+    // which case that conversion, and a value based on that conversion being
+    // passed to __clc_ldexp, results in undefined behavior.
+    v = v > max_exp_arg ? 1000.0 : v;
+    v = v < min_exp_arg ? -1000.0 : v;
+
     __CLC_GENTYPE temp = v * sixtyfour_by_lnof2;
     __CLC_INTN n = __CLC_CONVERT_INTN(temp);
     __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(n);
@@ -350,10 +359,6 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
 
     expv = __clc_fma(f, q, f2) + f1;
     expv = __clc_ldexp(expv, m);
-
-    expv = v > max_exp_arg ? __CLC_AS_GENTYPE((__CLC_ULONGN)0x7FF0000000000000L)
-                           : expv;
-    expv = v < min_exp_arg ? 0.0 : expv;
   }
 
   // See whether y is an integer.
diff --git a/libclc/clc/lib/generic/math/clc_round.cl b/libclc/clc/lib/generic/math/clc_round.cl
index e784dbeaf3fa..8c298d8c3483 100644
--- a/libclc/clc/lib/generic/math/clc_round.cl
+++ b/libclc/clc/lib/generic/math/clc_round.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_round
-#define __CLC_FUNCTION(x) __builtin_elementwise_round
+#define __IMPL_FUNCTION(x) __builtin_elementwise_round
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
index c1768e39243f..0ea1195fffa7 100644
--- a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
+++ b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
@@ -32,8 +32,6 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
@@ -52,6 +50,4 @@
 
 #include <clc/math/gentype.inc>
 
-#undef __DOUBLE_ONLY
-
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_sw_fma.cl b/libclc/clc/lib/generic/math/clc_sw_fma.cl
index df5ac9dbd49d..ee4734078d69 100644
--- a/libclc/clc/lib/generic/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/generic/math/clc_sw_fma.cl
@@ -160,5 +160,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa & 0x7fffff));
 }
 
-_CLC_TERNARY_VECTORIZE(_CLC_DEF _CLC_OVERLOAD, float, __clc_sw_fma, float,
-                       float, float)
+#define __FLOAT_ONLY
+#define FUNCTION __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_tgamma.cl b/libclc/clc/lib/generic/math/clc_tgamma.cl
index 5c2d164abeee..e0fed98026ca 100644
--- a/libclc/clc/lib/generic/math/clc_tgamma.cl
+++ b/libclc/clc/lib/generic/math/clc_tgamma.cl
@@ -32,7 +32,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_tgamma(float x) {
   return g;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_tgamma, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_tgamma
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -55,7 +59,11 @@ _CLC_OVERLOAD _CLC_DEF double __clc_tgamma(double x) {
   return g;
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_tgamma, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_tgamma
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
 
@@ -67,7 +75,7 @@ _CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_tgamma, double);
 
 // Forward the half version of this builtin onto the float one
 #define __HALF_ONLY
-#define __CLC_FUNCTION __clc_tgamma
+#define FUNCTION __clc_tgamma
 #define __CLC_BODY <clc/math/unary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_trunc.cl b/libclc/clc/lib/generic/math/clc_trunc.cl
index 0bf2bc44c26f..6261d04cd26d 100644
--- a/libclc/clc/lib/generic/math/clc_trunc.cl
+++ b/libclc/clc/lib/generic/math/clc_trunc.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 
 #define FUNCTION __clc_trunc
-#define __CLC_FUNCTION(x) __builtin_elementwise_trunc
+#define __IMPL_FUNCTION(x) __builtin_elementwise_trunc
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/r600/math/clc_fmax.cl b/libclc/clc/lib/r600/math/clc_fmax.cl
index 7e9c6df789ed..689e51a9829a 100644
--- a/libclc/clc/lib/r600/math/clc_fmax.cl
+++ b/libclc/clc/lib/r600/math/clc_fmax.cl
@@ -17,7 +17,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmax(float x, float y) {
   y = __clc_flush_denormal_if_not_supported(y);
   return __builtin_fmaxf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -26,7 +31,11 @@ _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmax, float, float)
 _CLC_DEF _CLC_OVERLOAD double __clc_fmax(double x, double y) {
   return __builtin_fmax(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmax, double,
-                      double)
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmax
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/r600/math/clc_fmin.cl b/libclc/clc/lib/r600/math/clc_fmin.cl
index cbcf849f7b7e..22cb7046a4ce 100644
--- a/libclc/clc/lib/r600/math/clc_fmin.cl
+++ b/libclc/clc/lib/r600/math/clc_fmin.cl
@@ -18,7 +18,12 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmin(float x, float y) {
   y = __clc_flush_denormal_if_not_supported(y);
   return __builtin_fminf(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
+
+#define __FLOAT_ONLY
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -27,7 +32,11 @@ _CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_fmin, float, float)
 _CLC_DEF _CLC_OVERLOAD double __clc_fmin(double x, double y) {
   return __builtin_fmin(x, y);
 }
-_CLC_BINARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_fmin, double,
-                      double)
+
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_fmin
+#define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
index ee09814eb1e7..b5966570804c 100644
--- a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
+++ b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
@@ -13,4 +13,7 @@ _CLC_OVERLOAD _CLC_DEF float __clc_native_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_native_rsqrt, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_native_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
index 91350d611f0a..75355df56d32 100644
--- a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
+++ b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
@@ -13,7 +13,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, float, __clc_rsqrt, float);
+#define __FLOAT_ONLY
+#define FUNCTION __clc_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -23,6 +27,10 @@ _CLC_OVERLOAD _CLC_DEF double __clc_rsqrt(double x) {
   return __builtin_r600_recipsqrt_ieee(x);
 }
 
-_CLC_UNARY_VECTORIZE(_CLC_OVERLOAD _CLC_DEF, double, __clc_rsqrt, double);
+#define __DOUBLE_ONLY
+#define FUNCTION __clc_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
 
 #endif
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index 597bb642655e..a8160e04d46a 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -120,14 +120,14 @@ function(link_bc)
   endif()
 
   add_custom_command(
-    OUTPUT ${ARG_TARGET}.bc
-    COMMAND ${llvm-link_exe} ${link_flags} -o ${ARG_TARGET}.bc ${LINK_INPUT_ARG}
+    OUTPUT ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc
+    COMMAND ${llvm-link_exe} ${link_flags} -o ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc ${LINK_INPUT_ARG}
     DEPENDS ${llvm-link_target} ${ARG_DEPENDENCIES} ${ARG_INPUTS} ${RSP_FILE}
   )
 
-  add_custom_target( ${ARG_TARGET} ALL DEPENDS ${ARG_TARGET}.bc )
+  add_custom_target( ${ARG_TARGET} ALL DEPENDS ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc )
   set_target_properties( ${ARG_TARGET} PROPERTIES
-    TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${ARG_TARGET}.bc
+    TARGET_FILE ${LIBCLC_ARCH_OBJFILE_DIR}/${ARG_TARGET}.bc
     FOLDER "libclc/Device IR/Linking"
   )
 endfunction()
@@ -356,65 +356,92 @@ function(add_libclc_builtin_set)
 
   set( builtins_link_lib $<TARGET_PROPERTY:${builtins_link_lib_tgt},TARGET_FILE> )
 
+  # For SPIR-V targets we diverage at this point and generate SPIR-V using the
+  # llvm-spirv tool.
   if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
-    set( spv_suffix ${ARG_ARCH_SUFFIX}.spv )
-    add_custom_command( OUTPUT ${spv_suffix}
-      COMMAND ${llvm-spirv_exe} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
+    set( obj_suffix ${ARG_ARCH_SUFFIX}.spv )
+    set( libclc_builtins_lib ${LIBCLC_OUTPUT_LIBRARY_DIR}/${obj_suffix} )
+    add_custom_command( OUTPUT ${libclc_builtins_lib}
+      COMMAND ${llvm-spirv_exe} ${spvflags} -o ${libclc_builtins_lib} ${builtins_link_lib}
       DEPENDS ${llvm-spirv_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
     )
-    add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
-    set_target_properties( "prepare-${spv_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
-    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
-       DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+  else()
+    # Non-SPIR-V targets add an extra step to optimize the bytecode
+    set( builtins_opt_lib_tgt builtins.opt.${ARG_ARCH_SUFFIX} )
 
-    return()
+    add_custom_command( OUTPUT ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+      COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+        ${builtins_link_lib}
+      DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
+    )
+    add_custom_target( ${builtins_opt_lib_tgt}
+      ALL DEPENDS ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+    )
+    set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
+      TARGET_FILE ${LIBCLC_ARCH_OBJFILE_DIR}/${builtins_opt_lib_tgt}.bc
+      FOLDER "libclc/Device IR/Opt"
+    )
+
+    set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+
+    set( obj_suffix ${ARG_ARCH_SUFFIX}.bc )
+    set( libclc_builtins_lib ${LIBCLC_OUTPUT_LIBRARY_DIR}/${obj_suffix} )
+    add_custom_command( OUTPUT ${libclc_builtins_lib}
+      COMMAND ${prepare_builtins_exe} -o ${libclc_builtins_lib} ${builtins_opt_lib}
+      DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target}
+    )
   endif()
 
-  set( builtins_opt_lib_tgt builtins.opt.${ARG_ARCH_SUFFIX} )
+  # Add a 'prepare' target
+  add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${libclc_builtins_lib} )
+  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
 
-  # Add opt target
-  add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
-    COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${builtins_opt_lib_tgt}.bc
-      ${builtins_link_lib}
-    DEPENDS ${opt_target} ${builtins_link_lib} ${builtins_link_lib_tgt}
-  )
-  add_custom_target( ${builtins_opt_lib_tgt}
-    ALL DEPENDS ${builtins_opt_lib_tgt}.bc
-  )
-  set_target_properties( ${builtins_opt_lib_tgt} PROPERTIES
-    TARGET_FILE ${CMAKE_CURRENT_BINARY_DIR}/${builtins_opt_lib_tgt}.bc
-    FOLDER "libclc/Device IR/Opt"
-  )
+  # Also add a 'prepare' target for the triple. Since a triple may have
+  # multiple devices, ensure we only try to create the triple target once. The
+  # triple's target will build all of the bytecode for its constituent devices.
+  if( NOT TARGET prepare-${ARG_TRIPLE} )
+    add_custom_target( prepare-${ARG_TRIPLE} ALL )
+  endif()
+  add_dependencies( prepare-${ARG_TRIPLE} prepare-${obj_suffix} )
 
-  set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+  install(
+    FILES ${libclc_builtins_lib}
+    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
+  )
 
-  # Add prepare target
-  set( obj_suffix ${ARG_ARCH_SUFFIX}.bc )
-  add_custom_command( OUTPUT ${obj_suffix}
-    COMMAND ${prepare_builtins_exe} -o ${obj_suffix} ${builtins_opt_lib}
-    DEPENDS ${builtins_opt_lib} ${builtins_opt_lib_tgt} ${prepare_builtins_target} )
-  add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
-  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
+  # SPIR-V targets can exit early here
+  if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
+    return()
+  endif()
 
-  # nvptx-- targets don't include workitem builtins, and clspv targets don't
-  # include all OpenCL builtins
+  # Add a test for whether or not the libraries contain unresolved calls which
+  # would usually indicate a build problem. Note that we don't perform this
+  # test for all libclc targets:
+  # * nvptx-- targets don't include workitem builtins
+  # * clspv targets don't include all OpenCL builtins
   if( NOT ARG_ARCH MATCHES "^(nvptx|clspv)(64)?$" )
     add_test( NAME external-calls-${obj_suffix}
-      COMMAND ./check_external_calls.sh ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} ${LLVM_TOOLS_BINARY_DIR}
+      COMMAND ./check_external_calls.sh ${libclc_builtins_lib} ${LLVM_TOOLS_BINARY_DIR}
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} )
   endif()
 
-  install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${obj_suffix} DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
   foreach( a ${ARG_ALIASES} )
     set( alias_suffix "${a}-${ARG_TRIPLE}.bc" )
     add_custom_command(
-      OUTPUT ${alias_suffix}
-      COMMAND ${CMAKE_COMMAND} -E create_symlink ${obj_suffix} ${alias_suffix}
-      DEPENDS prepare-${obj_suffix} )
-    add_custom_target( alias-${alias_suffix} ALL DEPENDS ${alias_suffix} )
-    set_target_properties( alias-${alias_suffix} PROPERTIES FOLDER "libclc/Device IR/Aliases" )
-    install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${alias_suffix}
-             DESTINATION "${CMAKE_INSTALL_DATADIR}/clc" )
+      OUTPUT ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      COMMAND ${CMAKE_COMMAND} -E create_symlink ${libclc_builtins_lib} ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      DEPENDS prepare-${obj_suffix}
+    )
+    add_custom_target( alias-${alias_suffix} ALL
+      DEPENDS ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+    )
+    set_target_properties( alias-${alias_suffix}
+      PROPERTIES FOLDER "libclc/Device IR/Aliases"
+    )
+    install(
+      FILES ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
+      DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
+    )
   endforeach( a )
 endfunction(add_libclc_builtin_set)
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_add.h b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
index 62c73d99371f..8b3b8372ca9f 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_add
+#define FUNCTION atom_add
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_and.h b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
index 9dd9466f62f8..8eda27bd0732 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_and
+#define FUNCTION atom_and
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
index 866d8903db81..b8631ceac3a4 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
@@ -10,12 +10,12 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, int)
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, uint)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_ADDRESS_SPACE
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
index 146de3412fc2..3918a7cae006 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
@@ -10,7 +10,7 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(local, long)
 __CLC_DECLARE_ATOM(local, ulong)
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOM(global, ulong)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_max.h b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
index 00b47e6b14fd..778085138f79 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_max
+#define FUNCTION atom_max
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_min.h b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
index 40bed0dc086f..d3169397b89f 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_min
+#define FUNCTION atom_min
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_or.h b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
index 2c0beb850b4d..9a058c813b99 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_or
+#define FUNCTION atom_or
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
index 36074aaef5a3..7cbcce5c8bf5 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_sub
+#define FUNCTION atom_sub
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
index a5043aa4109e..75d9a391d1c7 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define __CLC_FUNCTION atom_xchg
+#define FUNCTION atom_xchg
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
index 0c4b456b8f23..b01b1fba2378 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
@@ -7,18 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define __CLC_FUNCTION atom_xor
+#define FUNCTION atom_xor
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
index 937def3f4a2a..74b3d8ffd8c3 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_add
+#define FUNCTION atomic_add
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
index 5f353f40318f..aec521e07bc2 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_and
+#define FUNCTION atomic_and
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
index 0cfd4c3eab5f..e060e3aaea16 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE)                                  \
-  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 #define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE)                                   \
   __CLC_DECLARE_ATOMIC(global, TYPE)                                           \
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(uint)
 #undef __CLC_DECLARE_ATOMIC_ADDRSPACE
 #undef __CLC_DECLARE_ATOMIC
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
index 13832423cfee..4f23f3858531 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_max
+#define FUNCTION atomic_max
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
index 3bd47a3f7837..1d99600f6d13 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_min
+#define FUNCTION atomic_min
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
index 13f84ea49a5a..7cb920df9976 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_or
+#define FUNCTION atomic_or
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
index eedc9d0e7605..f12e77523770 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_sub
+#define FUNCTION atomic_sub
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
index ec322d56e23d..ef5ad542a2d4 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_xchg
+#define FUNCTION atomic_xchg
 
-_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile local float *, float);
-_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile global float *, float);
+_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile local float *, float);
+_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile global float *, float);
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
index 6f6b4564fdf3..fb6498d9d0b3 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atomic_xor
+#define FUNCTION atomic_xor
 #include <clc/opencl/atomic/atomic_decl.inc>
diff --git a/libclc/opencl/include/clc/opencl/clc.h b/libclc/opencl/include/clc/opencl/clc.h
index 5859a00c3158..7d34f0011351 100644
--- a/libclc/opencl/include/clc/opencl/clc.h
+++ b/libclc/opencl/include/clc/opencl/clc.h
@@ -15,19 +15,7 @@
 
 #pragma OPENCL EXTENSION cl_clang_storage_class_specifiers : enable
 
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-#endif
-
-/* Function Attributes */
-#include <clc/clcfunc.h>
-
-/* 6.1 Supported Data Types */
-#include <clc/clctypes.h>
+#include <clc/opencl/opencl-base.h>
 
 /* 6.2.3 Explicit Conversions */
 #include <clc/opencl/convert.h>
diff --git a/libclc/opencl/include/clc/opencl/common/degrees.h b/libclc/opencl/include/clc/opencl/common/degrees.h
index f442c5fbda85..6511bc9be10d 100644
--- a/libclc/opencl/include/clc/opencl/common/degrees.h
+++ b/libclc/opencl/include/clc/opencl/common/degrees.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION degrees
+#define FUNCTION degrees
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/common/radians.h b/libclc/opencl/include/clc/opencl/common/radians.h
index 2772f7c2f0ae..7cfee9e6d640 100644
--- a/libclc/opencl/include/clc/opencl/common/radians.h
+++ b/libclc/opencl/include/clc/opencl/common/radians.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION radians
+#define FUNCTION radians
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/common/sign.h b/libclc/opencl/include/clc/opencl/common/sign.h
index 2bb95716358f..76b53d27efc8 100644
--- a/libclc/opencl/include/clc/opencl/common/sign.h
+++ b/libclc/opencl/include/clc/opencl/common/sign.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION sign
+#define FUNCTION sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/distance.h b/libclc/opencl/include/clc/opencl/geometric/distance.h
index 28ec8274302e..aa7149d71827 100644
--- a/libclc/opencl/include/clc/opencl/geometric/distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/distance.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION distance
+#define FUNCTION distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/dot.h b/libclc/opencl/include/clc/opencl/geometric/dot.h
index 92d5b56d2e06..af149a7779e4 100644
--- a/libclc/opencl/include/clc/opencl/geometric/dot.h
+++ b/libclc/opencl/include/clc/opencl/geometric/dot.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION dot
+#define FUNCTION dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
index ef401fdf7289..6ae297a49915 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_distance
+#define FUNCTION fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_length.h b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
index b6fba6a8ca30..6bfb439fa222 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_length
+#define FUNCTION fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
index 1af81c66973b..54347d77df6c 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
@@ -7,12 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #define __FLOAT_ONLY
-#define __CLC_FUNCTION fast_normalize
+#define FUNCTION fast_normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __FLOAT_ONLY
diff --git a/libclc/opencl/include/clc/opencl/geometric/length.h b/libclc/opencl/include/clc/opencl/geometric/length.h
index 2105a8f8cece..b9a4788e095b 100644
--- a/libclc/opencl/include/clc/opencl/geometric/length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/length.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION length
+#define FUNCTION length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/geometric/normalize.h b/libclc/opencl/include/clc/opencl/geometric/normalize.h
index a15b7003329b..90d6f332682a 100644
--- a/libclc/opencl/include/clc/opencl/geometric/normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/normalize.h
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION normalize
+#define FUNCTION normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/integer/abs.h b/libclc/opencl/include/clc/opencl/integer/abs.h
index 22ba40fb40d4..3059bef9a7e4 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs.h
+++ b/libclc/opencl/include/clc/opencl/integer/abs.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_ABS_H__
+#define __CLC_OPENCL_INTEGER_ABS_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_BODY <clc/opencl/integer/abs.inc>
 #include <clc/integer/gentype.inc>
+
+#endif // __CLC_OPENCL_INTEGER_ABS_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/abs_diff.h b/libclc/opencl/include/clc/opencl/integer/abs_diff.h
index f22cc4d88afc..9e8fe2b31fed 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs_diff.h
+++ b/libclc/opencl/include/clc/opencl/integer/abs_diff.h
@@ -6,5 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_ABS_DIFF_H__
+#define __CLC_OPENCL_INTEGER_ABS_DIFF_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_BODY <clc/opencl/integer/abs_diff.inc>
 #include <clc/integer/gentype.inc>
+
+#endif // __CLC_OPENCL_INTEGER_ABS_DIFF_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/add_sat.h b/libclc/opencl/include/clc/opencl/integer/add_sat.h
index 1982d1190800..cd2c17a09629 100644
--- a/libclc/opencl/include/clc/opencl/integer/add_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/add_sat.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION add_sat
+#ifndef __CLC_OPENCL_INTEGER_ADD_SAT_H__
+#define __CLC_OPENCL_INTEGER_ADD_SAT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_ADD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/clz.h b/libclc/opencl/include/clc/opencl/integer/clz.h
index b215fc1d0bc9..77032ceeaf0f 100644
--- a/libclc/opencl/include/clc/opencl/integer/clz.h
+++ b/libclc/opencl/include/clc/opencl/integer/clz.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION clz
+#ifndef __CLC_OPENCL_INTEGER_CLZ_H__
+#define __CLC_OPENCL_INTEGER_CLZ_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_CLZ_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/ctz.h b/libclc/opencl/include/clc/opencl/integer/ctz.h
index 2ab643644085..b861225846ee 100644
--- a/libclc/opencl/include/clc/opencl/integer/ctz.h
+++ b/libclc/opencl/include/clc/opencl/integer/ctz.h
@@ -6,13 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_CTZ_H__
+#define __CLC_OPENCL_INTEGER_CTZ_H__
+
 #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
-#define __CLC_FUNCTION ctz
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
 
 #endif // __OPENCL_C_VERSION__ >= CL_VERSION_2_0
+
+#endif // __CLC_OPENCL_INTEGER_CTZ_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/hadd.h b/libclc/opencl/include/clc/opencl/integer/hadd.h
index c05c9d50e67d..c4c8f7daf401 100644
--- a/libclc/opencl/include/clc/opencl/integer/hadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/hadd.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION hadd
+#ifndef __CLC_OPENCL_INTEGER_HADD_H__
+#define __CLC_OPENCL_INTEGER_HADD_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_HADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad24.h b/libclc/opencl/include/clc/opencl/integer/mad24.h
index 9ff998a8604d..daee9a038347 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad24.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad24
+#ifndef __CLC_OPENCL_INTEGER_MAD24_H__
+#define __CLC_OPENCL_INTEGER_MAD24_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_hi.h b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
index e06ab8f3c11f..bbcdcd9c82eb 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad_hi
+#ifndef __CLC_OPENCL_INTEGER_MAD_HI_H__
+#define __CLC_OPENCL_INTEGER_MAD_HI_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_sat.h b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
index 0a9fd7869a6f..28045477975e 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mad_sat
+#ifndef __CLC_OPENCL_INTEGER_MAD_SAT_H__
+#define __CLC_OPENCL_INTEGER_MAD_SAT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MAD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul24.h b/libclc/opencl/include/clc/opencl/integer/mul24.h
index d2aca25fc314..6aa799a28b7e 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul24.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mul24
+#ifndef __CLC_OPENCL_INTEGER_MUL24_H__
+#define __CLC_OPENCL_INTEGER_MUL24_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MUL24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul_hi.h b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
index 4374ec73be32..ed7ac7319cab 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION mul_hi
+#ifndef __CLC_OPENCL_INTEGER_MUL_HI_H__
+#define __CLC_OPENCL_INTEGER_MUL_HI_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_MUL_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/popcount.h b/libclc/opencl/include/clc/opencl/integer/popcount.h
index 4d43f26f489e..43d5bc0f0b33 100644
--- a/libclc/opencl/include/clc/opencl/integer/popcount.h
+++ b/libclc/opencl/include/clc/opencl/integer/popcount.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION popcount
+#ifndef __CLC_OPENCL_INTEGER_POPCOUNT_H__
+#define __CLC_OPENCL_INTEGER_POPCOUNT_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_POPCOUNT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rhadd.h b/libclc/opencl/include/clc/opencl/integer/rhadd.h
index 3da5524df206..2c93c100cf02 100644
--- a/libclc/opencl/include/clc/opencl/integer/rhadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/rhadd.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION rhadd
+#ifndef __CLC_OPENCL_INTEGER_RHADD_H__
+#define __CLC_OPENCL_INTEGER_RHADD_H__
+
+#define FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_RHADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rotate.h b/libclc/opencl/include/clc/opencl/integer/rotate.h
index 1e4de24cbf31..156155ae0638 100644
--- a/libclc/opencl/include/clc/opencl/integer/rotate.h
+++ b/libclc/opencl/include/clc/opencl/integer/rotate.h
@@ -6,9 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION rotate
+#ifndef __CLC_OPENCL_INTEGER_ROTATE_H__
+#define __CLC_OPENCL_INTEGER_ROTATE_H__
+
+#include <clc/opencl/opencl-base.h>
+
+#define FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_ROTATE_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/sub_sat.h b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
index ee5d8c78f3eb..a7861ee1ac4a 100644
--- a/libclc/opencl/include/clc/opencl/integer/sub_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
@@ -6,9 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION sub_sat
+#ifndef __CLC_OPENCL_INTEGER_SUB_SAT_H__
+#define __CLC_OPENCL_INTEGER_SUB_SAT_H__
+
+#define FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
+
+#endif // __CLC_OPENCL_INTEGER_SUB_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/upsample.h b/libclc/opencl/include/clc/opencl/integer/upsample.h
index 0db67dfe4e97..a713f2b392f2 100644
--- a/libclc/opencl/include/clc/opencl/integer/upsample.h
+++ b/libclc/opencl/include/clc/opencl/integer/upsample.h
@@ -6,6 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef __CLC_OPENCL_INTEGER_UPSAMPLE_H__
+#define __CLC_OPENCL_INTEGER_UPSAMPLE_H__
+
+#include <clc/opencl/opencl-base.h>
+
 #define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                       \
   _CLC_OVERLOAD _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
 
@@ -30,3 +35,5 @@ __CLC_UPSAMPLE_TYPES()
 #undef __CLC_UPSAMPLE_TYPES
 #undef __CLC_UPSAMPLE_DECL
 #undef __CLC_UPSAMPLE_VEC
+
+#endif // __CLC_OPENCL_INTEGER_UPSAMPLE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acos.h b/libclc/opencl/include/clc/opencl/math/acos.h
index 695e6560478c..0a9104f4003a 100644
--- a/libclc/opencl/include/clc/opencl/math/acos.h
+++ b/libclc/opencl/include/clc/opencl/math/acos.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acos
+#define FUNCTION acos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/acosh.h b/libclc/opencl/include/clc/opencl/math/acosh.h
index 52197d448d6b..cc1844e41fa9 100644
--- a/libclc/opencl/include/clc/opencl/math/acosh.h
+++ b/libclc/opencl/include/clc/opencl/math/acosh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acosh
+#define FUNCTION acosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/acospi.h b/libclc/opencl/include/clc/opencl/math/acospi.h
index 47e4e5f3e7f1..34175b1a83d8 100644
--- a/libclc/opencl/include/clc/opencl/math/acospi.h
+++ b/libclc/opencl/include/clc/opencl/math/acospi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION acospi
+#define FUNCTION acospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/asin.h b/libclc/opencl/include/clc/opencl/math/asin.h
index eb83d2d2743f..d6055834bc43 100644
--- a/libclc/opencl/include/clc/opencl/math/asin.h
+++ b/libclc/opencl/include/clc/opencl/math/asin.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asin
+#define FUNCTION asin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/asinh.h b/libclc/opencl/include/clc/opencl/math/asinh.h
index 74ce5ba6731c..d29f05829105 100644
--- a/libclc/opencl/include/clc/opencl/math/asinh.h
+++ b/libclc/opencl/include/clc/opencl/math/asinh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asinh
+#define FUNCTION asinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/asinpi.h b/libclc/opencl/include/clc/opencl/math/asinpi.h
index a7bd6469c93a..8d57697f6aca 100644
--- a/libclc/opencl/include/clc/opencl/math/asinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/asinpi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION asinpi
+#define FUNCTION asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/atan.h b/libclc/opencl/include/clc/opencl/math/atan.h
index 4c83c2d4c819..5642000d9166 100644
--- a/libclc/opencl/include/clc/opencl/math/atan.h
+++ b/libclc/opencl/include/clc/opencl/math/atan.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atan
+#define FUNCTION atan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/atan2.h b/libclc/opencl/include/clc/opencl/math/atan2.h
index 5872e41218dc..c6bfb20bacd0 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atan2
+#define FUNCTION atan2
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/atan2pi.h b/libclc/opencl/include/clc/opencl/math/atan2pi.h
index b68cf492e292..87817b993842 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2pi.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2pi.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION atan2pi
+#define FUNCTION atan2pi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/atanh.h b/libclc/opencl/include/clc/opencl/math/atanh.h
index 6ea99c2dedc4..ae0751120f84 100644
--- a/libclc/opencl/include/clc/opencl/math/atanh.h
+++ b/libclc/opencl/include/clc/opencl/math/atanh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atanh
+#define FUNCTION atanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/atanpi.h b/libclc/opencl/include/clc/opencl/math/atanpi.h
index 5c7d0a9130fd..67d9a13a8dd0 100644
--- a/libclc/opencl/include/clc/opencl/math/atanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/atanpi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION atanpi
+#define FUNCTION atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/cbrt.h b/libclc/opencl/include/clc/opencl/math/cbrt.h
index f0599e0805ab..e2dad07b0f00 100644
--- a/libclc/opencl/include/clc/opencl/math/cbrt.h
+++ b/libclc/opencl/include/clc/opencl/math/cbrt.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cbrt
+#define FUNCTION cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/ceil.h b/libclc/opencl/include/clc/opencl/math/ceil.h
index 82e5dc4eec00..a7ca6bf12b85 100644
--- a/libclc/opencl/include/clc/opencl/math/ceil.h
+++ b/libclc/opencl/include/clc/opencl/math/ceil.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION ceil
+#define FUNCTION ceil
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/copysign.h b/libclc/opencl/include/clc/opencl/math/copysign.h
index 3956dcd2fc19..40baac0a586b 100644
--- a/libclc/opencl/include/clc/opencl/math/copysign.h
+++ b/libclc/opencl/include/clc/opencl/math/copysign.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION copysign
+#define FUNCTION copysign
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/cos.h b/libclc/opencl/include/clc/opencl/math/cos.h
index 073282a3e0c1..7837075abb5f 100644
--- a/libclc/opencl/include/clc/opencl/math/cos.h
+++ b/libclc/opencl/include/clc/opencl/math/cos.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cos
+#define FUNCTION cos
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/cosh.h b/libclc/opencl/include/clc/opencl/math/cosh.h
index f156194cd9b1..bc2c1f595de2 100644
--- a/libclc/opencl/include/clc/opencl/math/cosh.h
+++ b/libclc/opencl/include/clc/opencl/math/cosh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cosh
+#define FUNCTION cosh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/cospi.h b/libclc/opencl/include/clc/opencl/math/cospi.h
index 2e9ae1b2ed37..fb55e87d2569 100644
--- a/libclc/opencl/include/clc/opencl/math/cospi.h
+++ b/libclc/opencl/include/clc/opencl/math/cospi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION cospi
+#define FUNCTION cospi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/erf.h b/libclc/opencl/include/clc/opencl/math/erf.h
index cbbb58b6c35a..66737077eef4 100644
--- a/libclc/opencl/include/clc/opencl/math/erf.h
+++ b/libclc/opencl/include/clc/opencl/math/erf.h
@@ -9,8 +9,8 @@
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION erf
+#define FUNCTION erf
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/erfc.h b/libclc/opencl/include/clc/opencl/math/erfc.h
index 74cef50ecc7c..279a44b49346 100644
--- a/libclc/opencl/include/clc/opencl/math/erfc.h
+++ b/libclc/opencl/include/clc/opencl/math/erfc.h
@@ -9,8 +9,8 @@
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION erfc
+#define FUNCTION erfc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/exp.h b/libclc/opencl/include/clc/opencl/math/exp.h
index a6193a9eac84..4eb5d31c157e 100644
--- a/libclc/opencl/include/clc/opencl/math/exp.h
+++ b/libclc/opencl/include/clc/opencl/math/exp.h
@@ -9,8 +9,8 @@
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp
+#define FUNCTION exp
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/exp10.h b/libclc/opencl/include/clc/opencl/math/exp10.h
index 7b48c1ce0348..feef3bfb27fb 100644
--- a/libclc/opencl/include/clc/opencl/math/exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/exp10.h
@@ -9,8 +9,8 @@
 #undef exp10
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp10
+#define FUNCTION exp10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/exp2.h b/libclc/opencl/include/clc/opencl/math/exp2.h
index 7081050769e9..801eecd34d5f 100644
--- a/libclc/opencl/include/clc/opencl/math/exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/exp2.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION exp2
+#define FUNCTION exp2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/expm1.h b/libclc/opencl/include/clc/opencl/math/expm1.h
index 9f6a512df38e..b1b27ec8a1fb 100644
--- a/libclc/opencl/include/clc/opencl/math/expm1.h
+++ b/libclc/opencl/include/clc/opencl/math/expm1.h
@@ -9,8 +9,8 @@
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION expm1
+#define FUNCTION expm1
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fabs.h b/libclc/opencl/include/clc/opencl/math/fabs.h
index 41212d6f597c..d3b4a6cbc2e9 100644
--- a/libclc/opencl/include/clc/opencl/math/fabs.h
+++ b/libclc/opencl/include/clc/opencl/math/fabs.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION fabs
+#define FUNCTION fabs
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fdim.h b/libclc/opencl/include/clc/opencl/math/fdim.h
index fdc261072c16..da05471f0a01 100644
--- a/libclc/opencl/include/clc/opencl/math/fdim.h
+++ b/libclc/opencl/include/clc/opencl/math/fdim.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fdim
+#define FUNCTION fdim
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/floor.h b/libclc/opencl/include/clc/opencl/math/floor.h
index 80f8fcc5ac95..6927366cd5b8 100644
--- a/libclc/opencl/include/clc/opencl/math/floor.h
+++ b/libclc/opencl/include/clc/opencl/math/floor.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION floor
+#define FUNCTION floor
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fma.h b/libclc/opencl/include/clc/opencl/math/fma.h
index 9a201bb98400..ebb60a670d69 100644
--- a/libclc/opencl/include/clc/opencl/math/fma.h
+++ b/libclc/opencl/include/clc/opencl/math/fma.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION fma
+#define FUNCTION fma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fmax.h b/libclc/opencl/include/clc/opencl/math/fmax.h
index 2144bb334f1a..be3fba157506 100644
--- a/libclc/opencl/include/clc/opencl/math/fmax.h
+++ b/libclc/opencl/include/clc/opencl/math/fmax.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define __CLC_FUNCTION fmax
+#define FUNCTION fmax
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fmin.h b/libclc/opencl/include/clc/opencl/math/fmin.h
index 5a2af3725bf2..4867ed894ea1 100644
--- a/libclc/opencl/include/clc/opencl/math/fmin.h
+++ b/libclc/opencl/include/clc/opencl/math/fmin.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define __CLC_FUNCTION fmin
+#define FUNCTION fmin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fmod.h b/libclc/opencl/include/clc/opencl/math/fmod.h
index 10236aec3e9e..9b10fecc0f14 100644
--- a/libclc/opencl/include/clc/opencl/math/fmod.h
+++ b/libclc/opencl/include/clc/opencl/math/fmod.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fmod
+#define FUNCTION fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/fract.h b/libclc/opencl/include/clc/opencl/math/fract.h
index 8443d4968c98..01c06771421e 100644
--- a/libclc/opencl/include/clc/opencl/math/fract.h
+++ b/libclc/opencl/include/clc/opencl/math/fract.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION fract
+#define FUNCTION fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/frexp.h b/libclc/opencl/include/clc/opencl/math/frexp.h
index 796acca1cb75..38d494228470 100644
--- a/libclc/opencl/include/clc/opencl/math/frexp.h
+++ b/libclc/opencl/include/clc/opencl/math/frexp.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION frexp
+#define FUNCTION frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_cos.h b/libclc/opencl/include/clc/opencl/math/half_cos.h
index c9a5a88f8091..63c3216756e3 100644
--- a/libclc/opencl/include/clc/opencl/math/half_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/half_cos.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_cos
+#define FUNCTION half_cos
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_divide.h b/libclc/opencl/include/clc/opencl/math/half_divide.h
index a05eab5b1c78..fd69b94fb5a3 100644
--- a/libclc/opencl/include/clc/opencl/math/half_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/half_divide.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION half_divide
+#define FUNCTION half_divide
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp.h b/libclc/opencl/include/clc/opencl/math/half_exp.h
index 65734878eb4e..a815ebc91eef 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp
+#define FUNCTION half_exp
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp10.h b/libclc/opencl/include/clc/opencl/math/half_exp10.h
index 4df20a7aa622..80b74a2f4228 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp10.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp10
+#define FUNCTION half_exp10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp2.h b/libclc/opencl/include/clc/opencl/math/half_exp2.h
index 2eed851fa698..eaa0cb3b5704 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp2.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_exp2
+#define FUNCTION half_exp2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_log.h b/libclc/opencl/include/clc/opencl/math/half_log.h
index 3d4468057009..ea20a37b96e5 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log
+#define FUNCTION half_log
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_log10.h b/libclc/opencl/include/clc/opencl/math/half_log10.h
index 1c468f3f3398..35217f85bfea 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log10.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log10
+#define FUNCTION half_log10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_log2.h b/libclc/opencl/include/clc/opencl/math/half_log2.h
index 0695e2542962..bbcc0b9d17aa 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log2.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_log2
+#define FUNCTION half_log2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_powr.h b/libclc/opencl/include/clc/opencl/math/half_powr.h
index 27aa2b0fab66..eaa16251b68a 100644
--- a/libclc/opencl/include/clc/opencl/math/half_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/half_powr.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION half_powr
+#define FUNCTION half_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_recip.h b/libclc/opencl/include/clc/opencl/math/half_recip.h
index 8e2d16099f9f..b91b0d3d7c71 100644
--- a/libclc/opencl/include/clc/opencl/math/half_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/half_recip.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_recip
+#define FUNCTION half_recip
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
index e93973bc6457..5fc2764537aa 100644
--- a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
@@ -7,8 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_rsqrt
+#define FUNCTION half_rsqrt
 #define __FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_sin.h b/libclc/opencl/include/clc/opencl/math/half_sin.h
index 50626f9d96fe..279605d15ffc 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sin.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_sin
+#define FUNCTION half_sin
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_sqrt.h b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
index 7298570984ba..5c8080b7c584 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
@@ -7,8 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_sqrt
+#define FUNCTION half_sqrt
 #define __FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/half_tan.h b/libclc/opencl/include/clc/opencl/math/half_tan.h
index 40ae6e6c9aba..e63d919fb600 100644
--- a/libclc/opencl/include/clc/opencl/math/half_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/half_tan.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION half_tan
+#define FUNCTION half_tan
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/hypot.h b/libclc/opencl/include/clc/opencl/math/hypot.h
index a9d464af95c0..4b3dae721cd5 100644
--- a/libclc/opencl/include/clc/opencl/math/hypot.h
+++ b/libclc/opencl/include/clc/opencl/math/hypot.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION hypot
+#define FUNCTION hypot
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/ilogb.h b/libclc/opencl/include/clc/opencl/math/ilogb.h
index 45d8ea064b46..49ef681390aa 100644
--- a/libclc/opencl/include/clc/opencl/math/ilogb.h
+++ b/libclc/opencl/include/clc/opencl/math/ilogb.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION ilogb
+#define FUNCTION ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/ldexp.h b/libclc/opencl/include/clc/opencl/math/ldexp.h
index 6dcd2a9548d0..783ecd1db382 100644
--- a/libclc/opencl/include/clc/opencl/math/ldexp.h
+++ b/libclc/opencl/include/clc/opencl/math/ldexp.h
@@ -6,5 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#define FUNCTION ldexp
+#define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
+#include <clc/math/gentype.inc>
+#undef FUNCTION
+
 #define __CLC_BODY <clc/opencl/math/ldexp.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/include/clc/opencl/math/ldexp.inc b/libclc/opencl/include/clc/opencl/math/ldexp.inc
index 116acdff41d3..b5a5cfcafdd5 100644
--- a/libclc/opencl/include/clc/opencl/math/ldexp.inc
+++ b/libclc/opencl/include/clc/opencl/math/ldexp.inc
@@ -6,10 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE ldexp(__CLC_GENTYPE x, int n);
-
 #ifndef __CLC_SCALAR
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE ldexp(__CLC_GENTYPE x, __CLC_INTN n);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE ldexp(__CLC_GENTYPE x, int n);
 
 #endif
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma.h b/libclc/opencl/include/clc/opencl/math/lgamma.h
index 9f0be722b5f6..63120747a26a 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION lgamma
+#define FUNCTION lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma_r.h b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
index 78438b1be551..1dd7cca89093 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma_r.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION lgamma_r
+#define FUNCTION lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/log.h b/libclc/opencl/include/clc/opencl/math/log.h
index 8fec73d7fb69..ab8e85908cde 100644
--- a/libclc/opencl/include/clc/opencl/math/log.h
+++ b/libclc/opencl/include/clc/opencl/math/log.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log
+#define FUNCTION log
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/log10.h b/libclc/opencl/include/clc/opencl/math/log10.h
index 21cf2a976942..eec17f2da9b5 100644
--- a/libclc/opencl/include/clc/opencl/math/log10.h
+++ b/libclc/opencl/include/clc/opencl/math/log10.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log10
+#define FUNCTION log10
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/log1p.h b/libclc/opencl/include/clc/opencl/math/log1p.h
index 643207ebb947..6334f7d86e4e 100644
--- a/libclc/opencl/include/clc/opencl/math/log1p.h
+++ b/libclc/opencl/include/clc/opencl/math/log1p.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log1p
+#define FUNCTION log1p
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/log2.h b/libclc/opencl/include/clc/opencl/math/log2.h
index 26b407f1019b..ae9992a619ca 100644
--- a/libclc/opencl/include/clc/opencl/math/log2.h
+++ b/libclc/opencl/include/clc/opencl/math/log2.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION log2
+#define FUNCTION log2
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/logb.h b/libclc/opencl/include/clc/opencl/math/logb.h
index a0daa9806381..75e153b50047 100644
--- a/libclc/opencl/include/clc/opencl/math/logb.h
+++ b/libclc/opencl/include/clc/opencl/math/logb.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION logb
+#define FUNCTION logb
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/mad.h b/libclc/opencl/include/clc/opencl/math/mad.h
index 3daf9b9f8a0e..20e199a78a71 100644
--- a/libclc/opencl/include/clc/opencl/math/mad.h
+++ b/libclc/opencl/include/clc/opencl/math/mad.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define __CLC_FUNCTION mad
+#define FUNCTION mad
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/maxmag.h b/libclc/opencl/include/clc/opencl/math/maxmag.h
index d4616e60dc37..ea41128d6b6b 100644
--- a/libclc/opencl/include/clc/opencl/math/maxmag.h
+++ b/libclc/opencl/include/clc/opencl/math/maxmag.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION maxmag
+#define FUNCTION maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/minmag.h b/libclc/opencl/include/clc/opencl/math/minmag.h
index 92c0f23a22b7..59b3916e5693 100644
--- a/libclc/opencl/include/clc/opencl/math/minmag.h
+++ b/libclc/opencl/include/clc/opencl/math/minmag.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION minmag
+#define FUNCTION minmag
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/modf.h b/libclc/opencl/include/clc/opencl/math/modf.h
index b582cae948f2..75bf8d849a70 100644
--- a/libclc/opencl/include/clc/opencl/math/modf.h
+++ b/libclc/opencl/include/clc/opencl/math/modf.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION modf
+#define FUNCTION modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_cos.h b/libclc/opencl/include/clc/opencl/math/native_cos.h
index e301634322b6..fa2a94abe292 100644
--- a/libclc/opencl/include/clc/opencl/math/native_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/native_cos.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_cos
+#define FUNCTION native_cos
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_divide.h b/libclc/opencl/include/clc/opencl/math/native_divide.h
index 5fe0c32f67d8..60742038d9e4 100644
--- a/libclc/opencl/include/clc/opencl/math/native_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/native_divide.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION native_divide
+#define FUNCTION native_divide
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp.h b/libclc/opencl/include/clc/opencl/math/native_exp.h
index bfe24aa12808..c77ffed7f37b 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp
+#define FUNCTION native_exp
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp10.h b/libclc/opencl/include/clc/opencl/math/native_exp10.h
index f775478eef3a..4f1f1c4ee183 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp10.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp10
+#define FUNCTION native_exp10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp2.h b/libclc/opencl/include/clc/opencl/math/native_exp2.h
index c00632c99387..33e2ad37168d 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp2.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_exp2
+#define FUNCTION native_exp2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_log.h b/libclc/opencl/include/clc/opencl/math/native_log.h
index 854f42072374..6564416e8497 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log
+#define FUNCTION native_log
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_log10.h b/libclc/opencl/include/clc/opencl/math/native_log10.h
index 6862124b0f69..ef6e2a7ddbc2 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log10.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log10
+#define FUNCTION native_log10
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_log2.h b/libclc/opencl/include/clc/opencl/math/native_log2.h
index 576f50079d26..c1d7d54eb3de 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log2.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_log2
+#define FUNCTION native_log2
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_powr.h b/libclc/opencl/include/clc/opencl/math/native_powr.h
index a9210092ed38..3e472e313bfc 100644
--- a/libclc/opencl/include/clc/opencl/math/native_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/native_powr.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define __CLC_FUNCTION native_powr
+#define FUNCTION native_powr
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_recip.h b/libclc/opencl/include/clc/opencl/math/native_recip.h
index 7bc99233fe00..8aea05ba9358 100644
--- a/libclc/opencl/include/clc/opencl/math/native_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/native_recip.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_recip
+#define FUNCTION native_recip
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
index 40a1f02cba07..5f73545ce0f7 100644
--- a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_rsqrt
+#define FUNCTION native_rsqrt
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_sin.h b/libclc/opencl/include/clc/opencl/math/native_sin.h
index 798d5b325f6f..c109b5ecad5e 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sin.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_sin
+#define FUNCTION native_sin
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_sqrt.h b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
index a41a0c104cc8..061565ca1b30 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_sqrt
+#define FUNCTION native_sqrt
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/native_tan.h b/libclc/opencl/include/clc/opencl/math/native_tan.h
index 658bb8f93274..5b93b2604b47 100644
--- a/libclc/opencl/include/clc/opencl/math/native_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/native_tan.h
@@ -7,10 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION native_tan
+#define FUNCTION native_tan
 #define __FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef __FLOAT_ONLY
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/nextafter.h b/libclc/opencl/include/clc/opencl/math/nextafter.h
index ce1d549c0659..3b95500b0e9d 100644
--- a/libclc/opencl/include/clc/opencl/math/nextafter.h
+++ b/libclc/opencl/include/clc/opencl/math/nextafter.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION nextafter
+#define FUNCTION nextafter
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/pow.h b/libclc/opencl/include/clc/opencl/math/pow.h
index 5f8479332830..3b62f4c154f3 100644
--- a/libclc/opencl/include/clc/opencl/math/pow.h
+++ b/libclc/opencl/include/clc/opencl/math/pow.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION pow
+#define FUNCTION pow
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/pown.h b/libclc/opencl/include/clc/opencl/math/pown.h
index 1d38c68947ba..ba249712ec84 100644
--- a/libclc/opencl/include/clc/opencl/math/pown.h
+++ b/libclc/opencl/include/clc/opencl/math/pown.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION pown
-#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc>
+#define FUNCTION pown
+#define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/powr.h b/libclc/opencl/include/clc/opencl/math/powr.h
index d128d79edb39..6cbcbc143f40 100644
--- a/libclc/opencl/include/clc/opencl/math/powr.h
+++ b/libclc/opencl/include/clc/opencl/math/powr.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION powr
+#define FUNCTION powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/remainder.h b/libclc/opencl/include/clc/opencl/math/remainder.h
index d366474f4192..0315516a1e5a 100644
--- a/libclc/opencl/include/clc/opencl/math/remainder.h
+++ b/libclc/opencl/include/clc/opencl/math/remainder.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION remainder
+#define FUNCTION remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/remquo.h b/libclc/opencl/include/clc/opencl/math/remquo.h
index 007d380c7cfc..5ce010e2c273 100644
--- a/libclc/opencl/include/clc/opencl/math/remquo.h
+++ b/libclc/opencl/include/clc/opencl/math/remquo.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION remquo
+#define FUNCTION remquo
 
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 #include <clc/math/gentype.inc>
@@ -18,4 +18,4 @@
 #undef __CLC_ADDRESS_SPACE
 #endif
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/rint.h b/libclc/opencl/include/clc/opencl/math/rint.h
index aa31c177882c..bb27f3d32153 100644
--- a/libclc/opencl/include/clc/opencl/math/rint.h
+++ b/libclc/opencl/include/clc/opencl/math/rint.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION rint
+#define FUNCTION rint
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/rootn.h b/libclc/opencl/include/clc/opencl/math/rootn.h
index 789f31596d1c..0751e875948f 100644
--- a/libclc/opencl/include/clc/opencl/math/rootn.h
+++ b/libclc/opencl/include/clc/opencl/math/rootn.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_BODY <clc/math/binary_decl_with_int_second_arg.inc>
-#define __CLC_FUNCTION rootn
+#define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
+#define FUNCTION rootn
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/round.h b/libclc/opencl/include/clc/opencl/math/round.h
index f4209f631d6c..6764c251bdb9 100644
--- a/libclc/opencl/include/clc/opencl/math/round.h
+++ b/libclc/opencl/include/clc/opencl/math/round.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION round
+#define FUNCTION round
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/rsqrt.h b/libclc/opencl/include/clc/opencl/math/rsqrt.h
index 5f674b299adb..4bcda124c8b8 100644
--- a/libclc/opencl/include/clc/opencl/math/rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/rsqrt.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION rsqrt
+#define FUNCTION rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/sin.h b/libclc/opencl/include/clc/opencl/math/sin.h
index 5ef699395689..38761d946cc9 100644
--- a/libclc/opencl/include/clc/opencl/math/sin.h
+++ b/libclc/opencl/include/clc/opencl/math/sin.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sin
+#define FUNCTION sin
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/sincos.h b/libclc/opencl/include/clc/opencl/math/sincos.h
index 7426c23e88af..01d034e97d97 100644
--- a/libclc/opencl/include/clc/opencl/math/sincos.h
+++ b/libclc/opencl/include/clc/opencl/math/sincos.h
@@ -7,6 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define __CLC_FUNCTION __clc_sincos
+#define FUNCTION sincos
 #include <clc/math/gentype.inc>
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/sinh.h b/libclc/opencl/include/clc/opencl/math/sinh.h
index c6d7d9a936e2..194a6acf155a 100644
--- a/libclc/opencl/include/clc/opencl/math/sinh.h
+++ b/libclc/opencl/include/clc/opencl/math/sinh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sinh
+#define FUNCTION sinh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/sinpi.h b/libclc/opencl/include/clc/opencl/math/sinpi.h
index 0d90dbf15351..f4be7d08ab5f 100644
--- a/libclc/opencl/include/clc/opencl/math/sinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/sinpi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sinpi
+#define FUNCTION sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/sqrt.h b/libclc/opencl/include/clc/opencl/math/sqrt.h
index 03faa0513c06..6f842365b469 100644
--- a/libclc/opencl/include/clc/opencl/math/sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/sqrt.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION sqrt
+#define FUNCTION sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/tan.h b/libclc/opencl/include/clc/opencl/math/tan.h
index bbc0b79ba0bb..312561364889 100644
--- a/libclc/opencl/include/clc/opencl/math/tan.h
+++ b/libclc/opencl/include/clc/opencl/math/tan.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tan
+#define FUNCTION tan
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/tanh.h b/libclc/opencl/include/clc/opencl/math/tanh.h
index 9017078740a0..fef93ebe9f7b 100644
--- a/libclc/opencl/include/clc/opencl/math/tanh.h
+++ b/libclc/opencl/include/clc/opencl/math/tanh.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tanh
+#define FUNCTION tanh
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/tanpi.h b/libclc/opencl/include/clc/opencl/math/tanpi.h
index f02b5a9be81f..2df53e8fe99b 100644
--- a/libclc/opencl/include/clc/opencl/math/tanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/tanpi.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tanpi
+#define FUNCTION tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/tgamma.h b/libclc/opencl/include/clc/opencl/math/tgamma.h
index c74d67a857c5..8c0f320c7c34 100644
--- a/libclc/opencl/include/clc/opencl/math/tgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/tgamma.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION tgamma
+#define FUNCTION tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/math/trunc.h b/libclc/opencl/include/clc/opencl/math/trunc.h
index 5832e9e93bd8..41597cb56209 100644
--- a/libclc/opencl/include/clc/opencl/math/trunc.h
+++ b/libclc/opencl/include/clc/opencl/math/trunc.h
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define __CLC_FUNCTION trunc
+#define FUNCTION trunc
 
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle.h b/libclc/opencl/include/clc/opencl/misc/shuffle.h
index f9d84c9633d3..3c3be84a5baf 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION shuffle
+#define FUNCTION shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -16,4 +16,4 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle2.h b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
index 91548ad784fb..865de53029be 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle2.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION shuffle2
+#define FUNCTION shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -16,4 +16,4 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/opencl-base.h b/libclc/opencl/include/clc/opencl/opencl-base.h
new file mode 100644
index 000000000000..ac0dd0f12970
--- /dev/null
+++ b/libclc/opencl/include/clc/opencl/opencl-base.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLC_OPENCL_OPENCL_BASE_H__
+#define __CLC_OPENCL_OPENCL_BASE_H__
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#endif
+
+/* Function Attributes */
+#include <clc/clcfunc.h>
+
+/* 6.1 Supported Data Types */
+#include <clc/clctypes.h>
+
+#endif // __CLC_OPENCL_OPENCL_BASE_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isfinite.h b/libclc/opencl/include/clc/opencl/relational/isfinite.h
index e92a4d40a8cb..3c70ebacd7df 100644
--- a/libclc/opencl/include/clc/opencl/relational/isfinite.h
+++ b/libclc/opencl/include/clc/opencl/relational/isfinite.h
@@ -8,9 +8,9 @@
 
 #undef isfinite
 
-#define __CLC_FUNCTION isfinite
+#define FUNCTION isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreater.h b/libclc/opencl/include/clc/opencl/relational/isgreater.h
index bec1e81f48ba..f64b5d8c13e2 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreater.h
@@ -8,9 +8,9 @@
 
 #undef isgreater
 
-#define __CLC_FUNCTION isgreater
+#define FUNCTION isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
index da02167fc010..e8b6a03e6bf4 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
@@ -8,9 +8,9 @@
 
 #undef isgreaterequal
 
-#define __CLC_FUNCTION isgreaterequal
+#define FUNCTION isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isless.h b/libclc/opencl/include/clc/opencl/relational/isless.h
index c930b6df6d08..d5a8c06beb68 100644
--- a/libclc/opencl/include/clc/opencl/relational/isless.h
+++ b/libclc/opencl/include/clc/opencl/relational/isless.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION isless
+#define FUNCTION isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/islessequal.h b/libclc/opencl/include/clc/opencl/relational/islessequal.h
index 335c29612446..b4c3aee168d5 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessequal.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION islessequal
+#define FUNCTION islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/islessgreater.h b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
index f9a5fa8b0122..f1306b3aac26 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __CLC_FUNCTION islessgreater
+#define FUNCTION islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isnormal.h b/libclc/opencl/include/clc/opencl/relational/isnormal.h
index 33b7a7e89451..92646c342c07 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnormal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnormal.h
@@ -8,9 +8,9 @@
 
 #undef isnormal
 
-#define __CLC_FUNCTION isnormal
+#define FUNCTION isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isnotequal.h b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
index dfb9de1da3a1..f65f3d47720a 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnotequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
@@ -8,9 +8,9 @@
 
 #undef isnotequal
 
-#define __CLC_FUNCTION isnotequal
+#define FUNCTION isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isordered.h b/libclc/opencl/include/clc/opencl/relational/isordered.h
index bc6dbdeba92a..92cefa6d6f36 100644
--- a/libclc/opencl/include/clc/opencl/relational/isordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isordered.h
@@ -8,9 +8,9 @@
 
 #undef isordered
 
-#define __CLC_FUNCTION isordered
+#define FUNCTION isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/isunordered.h b/libclc/opencl/include/clc/opencl/relational/isunordered.h
index 73f05ce9edea..af91e626abd3 100644
--- a/libclc/opencl/include/clc/opencl/relational/isunordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isunordered.h
@@ -8,9 +8,9 @@
 
 #undef isunordered
 
-#define __CLC_FUNCTION isunordered
+#define FUNCTION isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/relational/signbit.h b/libclc/opencl/include/clc/opencl/relational/signbit.h
index 89785f5fc103..1e0894cfba5a 100644
--- a/libclc/opencl/include/clc/opencl/relational/signbit.h
+++ b/libclc/opencl/include/clc/opencl/relational/signbit.h
@@ -8,9 +8,9 @@
 
 #undef signbit
 
-#define __CLC_FUNCTION signbit
+#define FUNCTION signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/relational/floatn.inc>
 
-#undef __CLC_FUNCTION
+#undef FUNCTION
diff --git a/libclc/opencl/lib/clspv/math/fma.cl b/libclc/opencl/lib/clspv/math/fma.cl
index 272201812122..c027b4fcd8f1 100644
--- a/libclc/opencl/lib/clspv/math/fma.cl
+++ b/libclc/opencl/lib/clspv/math/fma.cl
@@ -6,8 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/internal/math/clc_sw_fma.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_TERNARY_BUILTIN(float, fma, __clc_sw_fma, float, float, float)
+#define __FLOAT_ONLY
+#define FUNCTION fma
+#define __IMPL_FUNCTION(x) __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def.inc>
+
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/common/degrees.cl b/libclc/opencl/lib/generic/common/degrees.cl
index 8b17fe432129..a86003c170bf 100644
--- a/libclc/opencl/lib/generic/common/degrees.cl
+++ b/libclc/opencl/lib/generic/common/degrees.cl
@@ -6,22 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/common/clc_degrees.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_UNARY_BUILTIN(float, degrees, __clc_degrees, float)
+#define FUNCTION degrees
+#define __CLC_BODY <clc/shared/unary_def.inc>
 
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_UNARY_BUILTIN(double, degrees, __clc_degrees, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_UNARY_BUILTIN(half, degrees, __clc_degrees, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/common/radians.cl b/libclc/opencl/lib/generic/common/radians.cl
index 1c58c6c4da6f..b45653be2e78 100644
--- a/libclc/opencl/lib/generic/common/radians.cl
+++ b/libclc/opencl/lib/generic/common/radians.cl
@@ -6,22 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/common/clc_radians.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_UNARY_BUILTIN(float, radians, __clc_radians, float)
+#define FUNCTION radians
+#define __CLC_BODY <clc/shared/unary_def.inc>
 
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_UNARY_BUILTIN(double, radians, __clc_radians, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_UNARY_BUILTIN(half, radians, __clc_radians, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/abs.cl b/libclc/opencl/lib/generic/integer/abs.cl
index 5d354d97e71c..0771ec793433 100644
--- a/libclc/opencl/lib/generic/integer/abs.cl
+++ b/libclc/opencl/lib/generic/integer/abs.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_abs.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/abs.h>
 
 #define __CLC_BODY <abs.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/abs_diff.cl b/libclc/opencl/lib/generic/integer/abs_diff.cl
index 73d6631b3a4a..d65b0468a7f2 100644
--- a/libclc/opencl/lib/generic/integer/abs_diff.cl
+++ b/libclc/opencl/lib/generic/integer/abs_diff.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_abs_diff.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/abs_diff.h>
 
 #define __CLC_BODY <abs_diff.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/add_sat.cl b/libclc/opencl/lib/generic/integer/add_sat.cl
index 8fe1d32a8221..50d989a47211 100644
--- a/libclc/opencl/lib/generic/integer/add_sat.cl
+++ b/libclc/opencl/lib/generic/integer/add_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_add_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/add_sat.h>
 
 #define FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/clz.cl b/libclc/opencl/lib/generic/integer/clz.cl
index cf12a9838c6d..58110df1eec2 100644
--- a/libclc/opencl/lib/generic/integer/clz.cl
+++ b/libclc/opencl/lib/generic/integer/clz.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_clz.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/clz.h>
 
 #define FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/ctz.cl b/libclc/opencl/lib/generic/integer/ctz.cl
index 1b0470e08fd3..cbb167275a4a 100644
--- a/libclc/opencl/lib/generic/integer/ctz.cl
+++ b/libclc/opencl/lib/generic/integer/ctz.cl
@@ -9,7 +9,7 @@
 #if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
 #include <clc/integer/clc_ctz.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/ctz.h>
 
 #define FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/hadd.cl b/libclc/opencl/lib/generic/integer/hadd.cl
index 100cb7bf207a..e9edc88bce02 100644
--- a/libclc/opencl/lib/generic/integer/hadd.cl
+++ b/libclc/opencl/lib/generic/integer/hadd.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_hadd.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/hadd.h>
 
 #define FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad24.cl b/libclc/opencl/lib/generic/integer/mad24.cl
index d42873f1dd15..5e9dfb3b3031 100644
--- a/libclc/opencl/lib/generic/integer/mad24.cl
+++ b/libclc/opencl/lib/generic/integer/mad24.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad24.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad24.h>
 
 #define FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_hi.cl b/libclc/opencl/lib/generic/integer/mad_hi.cl
index 68e610d400cc..786614e42bc8 100644
--- a/libclc/opencl/lib/generic/integer/mad_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mad_hi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad_hi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad_hi.h>
 
 #define FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_sat.cl b/libclc/opencl/lib/generic/integer/mad_sat.cl
index d9daf56632b0..54ddc0eed2d1 100644
--- a/libclc/opencl/lib/generic/integer/mad_sat.cl
+++ b/libclc/opencl/lib/generic/integer/mad_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mad_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mad_sat.h>
 
 #define FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul24.cl b/libclc/opencl/lib/generic/integer/mul24.cl
index b0a9ff80f352..e6db6b89c5d9 100644
--- a/libclc/opencl/lib/generic/integer/mul24.cl
+++ b/libclc/opencl/lib/generic/integer/mul24.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mul24.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mul24.h>
 
 #define FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul_hi.cl b/libclc/opencl/lib/generic/integer/mul_hi.cl
index 40cb359f5b29..4c7b8c2caa99 100644
--- a/libclc/opencl/lib/generic/integer/mul_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mul_hi.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_mul_hi.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/mul_hi.h>
 
 #define FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/popcount.cl b/libclc/opencl/lib/generic/integer/popcount.cl
index a349f454813e..3c33c82302c3 100644
--- a/libclc/opencl/lib/generic/integer/popcount.cl
+++ b/libclc/opencl/lib/generic/integer/popcount.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_popcount.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/popcount.h>
 
 #define FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/rhadd.cl b/libclc/opencl/lib/generic/integer/rhadd.cl
index 539def845488..f83551189942 100644
--- a/libclc/opencl/lib/generic/integer/rhadd.cl
+++ b/libclc/opencl/lib/generic/integer/rhadd.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_rhadd.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/rhadd.h>
 
 #define FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/rotate.cl b/libclc/opencl/lib/generic/integer/rotate.cl
index a340ccf77b2e..282270f3b3ff 100644
--- a/libclc/opencl/lib/generic/integer/rotate.cl
+++ b/libclc/opencl/lib/generic/integer/rotate.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_rotate.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/rotate.h>
 
 #define FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/sub_sat.cl b/libclc/opencl/lib/generic/integer/sub_sat.cl
index 46596724358a..0b45df1118f1 100644
--- a/libclc/opencl/lib/generic/integer/sub_sat.cl
+++ b/libclc/opencl/lib/generic/integer/sub_sat.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_sub_sat.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/sub_sat.h>
 
 #define FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
diff --git a/libclc/opencl/lib/generic/integer/upsample.cl b/libclc/opencl/lib/generic/integer/upsample.cl
index 32cb3cffc9eb..5f95c811e058 100644
--- a/libclc/opencl/lib/generic/integer/upsample.cl
+++ b/libclc/opencl/lib/generic/integer/upsample.cl
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <clc/integer/clc_upsample.h>
-#include <clc/opencl/clc.h>
+#include <clc/opencl/integer/upsample.h>
 
 #define __CLC_UPSAMPLE_IMPL(BGENTYPE, GENTYPE, UGENTYPE)                       \
   _CLC_OVERLOAD _CLC_DEF BGENTYPE upsample(GENTYPE hi, UGENTYPE lo) {          \
diff --git a/libclc/opencl/lib/generic/math/fma.cl b/libclc/opencl/lib/generic/math/fma.cl
index ee3395bb2c64..c077357a44f0 100644
--- a/libclc/opencl/lib/generic/math/fma.cl
+++ b/libclc/opencl/lib/generic/math/fma.cl
@@ -6,23 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/math/clc_fma.h>
 #include <clc/math/math.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_TERNARY_BUILTIN(float, fma, __clc_fma, float, float, float)
+#define FUNCTION fma
+#define __CLC_BODY <clc/shared/ternary_def.inc>
 
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_TERNARY_BUILTIN(double, fma, __clc_fma, double, double, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_TERNARY_BUILTIN(half, fma, __clc_fma, half, half, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/ldexp.cl b/libclc/opencl/lib/generic/math/ldexp.cl
index e3b9b2b3f136..81c027f736e3 100644
--- a/libclc/opencl/lib/generic/math/ldexp.cl
+++ b/libclc/opencl/lib/generic/math/ldexp.cl
@@ -6,27 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/math/clc_ldexp.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(float, ldexp, __clc_ldexp, float, int)
+#define FUNCTION ldexp
+#define __IMPL_FUNCTION(x) __clc_ldexp
+#define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(double, ldexp, __clc_ldexp, double, int)
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(half, ldexp, __clc_ldexp, half, int)
-
-#endif
+#include <clc/math/gentype.inc>
 
 // This defines all the ldexp(GENTYPE, int) variants
 #define __CLC_BODY <ldexp.inc>
diff --git a/libclc/opencl/lib/generic/math/mad.cl b/libclc/opencl/lib/generic/math/mad.cl
index 20e690309445..39aa8e884cc0 100644
--- a/libclc/opencl/lib/generic/math/mad.cl
+++ b/libclc/opencl/lib/generic/math/mad.cl
@@ -6,22 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/math/clc_mad.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_TERNARY_BUILTIN(float, mad, __clc_mad, float, float, float)
+#define FUNCTION mad
+#define __CLC_BODY <clc/shared/ternary_def.inc>
 
-#ifdef cl_khr_fp64
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_TERNARY_BUILTIN(double, mad, __clc_mad, double, double, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_TERNARY_BUILTIN(half, mad, __clc_mad, half, half, half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/nextafter.cl b/libclc/opencl/lib/generic/math/nextafter.cl
index ecb187c53069..6fd52802941b 100644
--- a/libclc/opencl/lib/generic/math/nextafter.cl
+++ b/libclc/opencl/lib/generic/math/nextafter.cl
@@ -6,27 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/math/clc_nextafter.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(float, nextafter, __clc_nextafter,
-                                        float, float)
+#define FUNCTION nextafter
+#define __IMPL_FUNCTION(x) __clc_nextafter
+#define __CLC_BODY <clc/shared/binary_def.inc>
 
-#ifdef cl_khr_fp64
-
-#pragma OPENCL EXTENSION cl_khr_fp64 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(double, nextafter, __clc_nextafter,
-                                        double, double)
-
-#endif
-
-#ifdef cl_khr_fp16
-
-#pragma OPENCL EXTENSION cl_khr_fp16 : enable
-
-_CLC_DEFINE_BINARY_BUILTIN_NO_SCALARIZE(half, nextafter, __clc_nextafter, half,
-                                        half)
-
-#endif
+#include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/pown.cl b/libclc/opencl/lib/generic/math/pown.cl
index a2ed523a41f7..115bae3406f0 100644
--- a/libclc/opencl/lib/generic/math/pown.cl
+++ b/libclc/opencl/lib/generic/math/pown.cl
@@ -10,5 +10,5 @@
 #include <clc/opencl/clc.h>
 
 #define FUNCTION pown
-#define __CLC_BODY <clc/math/binary_def_with_int_second_arg.inc>
+#define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/rootn.cl b/libclc/opencl/lib/generic/math/rootn.cl
index 9f737151b390..0e1acc95470d 100644
--- a/libclc/opencl/lib/generic/math/rootn.cl
+++ b/libclc/opencl/lib/generic/math/rootn.cl
@@ -10,5 +10,5 @@
 #include <clc/opencl/clc.h>
 
 #define FUNCTION rootn
-#define __CLC_BODY <clc/math/binary_def_with_int_second_arg.inc>
+#define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/binary_def.inc b/libclc/opencl/lib/generic/relational/binary_def.inc
index a3b90bbd6d41..54bb237b8f8f 100644
--- a/libclc/opencl/lib/generic/relational/binary_def.inc
+++ b/libclc/opencl/lib/generic/relational/binary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_FLOATN a, __CLC_FLOATN b) {
-  return __CLC_FUNCTION(FUNCTION)(a, b);
+  return __IMPL_FUNCTION(FUNCTION)(a, b);
 }
diff --git a/libclc/opencl/lib/generic/relational/unary_def.inc b/libclc/opencl/lib/generic/relational/unary_def.inc
index 80dea211bb30..47bb33ef2da3 100644
--- a/libclc/opencl/lib/generic/relational/unary_def.inc
+++ b/libclc/opencl/lib/generic/relational/unary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#define __CLC_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 _CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_FLOATN a) {
-  return __CLC_FUNCTION(FUNCTION)(a);
+  return __IMPL_FUNCTION(FUNCTION)(a);
 }
diff --git a/libclc/opencl/lib/spirv/math/fma.cl b/libclc/opencl/lib/spirv/math/fma.cl
index 272201812122..c027b4fcd8f1 100644
--- a/libclc/opencl/lib/spirv/math/fma.cl
+++ b/libclc/opencl/lib/spirv/math/fma.cl
@@ -6,8 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include <clc/clcmacro.h>
 #include <clc/internal/math/clc_sw_fma.h>
 #include <clc/opencl/clc.h>
 
-_CLC_DEFINE_TERNARY_BUILTIN(float, fma, __clc_sw_fma, float, float, float)
+#define __FLOAT_ONLY
+#define FUNCTION fma
+#define __IMPL_FUNCTION(x) __clc_sw_fma
+#define __CLC_BODY <clc/shared/ternary_def.inc>
+
+#include <clc/math/gentype.inc>
diff --git a/libcxx/.clang-format b/libcxx/.clang-format
index fe2f2ef0119f..f372ac961999 100644
--- a/libcxx/.clang-format
+++ b/libcxx/.clang-format
@@ -16,8 +16,11 @@ AllowShortLambdasOnASingleLine: All
 AttributeMacros: [
                   '_ALIGNAS_TYPE',
                   '_ALIGNAS',
+                  '_LIBCPP_ACQUIRE_CAPABILITY',
+                  '_LIBCPP_ACQUIRE_SHARED_CAPABILITY',
                   '_LIBCPP_ALIGNOF',
                   '_LIBCPP_ALWAYS_INLINE',
+                  '_LIBCPP_CAPABILITY',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX14',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX17',
                   '_LIBCPP_CONSTEXPR_SINCE_CXX20',
@@ -41,9 +44,13 @@ AttributeMacros: [
                   '_LIBCPP_NO_UNIQUE_ADDRESS',
                   '_LIBCPP_NOALIAS',
                   '_LIBCPP_OVERRIDABLE_FUNC_VIS',
+                  '_LIBCPP_RELEASE_CAPABILITY',
+                  '_LIBCPP_REQUIRES_CAPABILITY',
+                  '_LIBCPP_SCOPED_LOCKABLE',
                   '_LIBCPP_STANDALONE_DEBUG',
                   '_LIBCPP_TEMPLATE_DATA_VIS',
-                  '_LIBCPP_THREAD_SAFETY_ANNOTATION',
+                  '_LIBCPP_TRY_ACQUIRE_CAPABILITY',
+                  '_LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY',
                   '_LIBCPP_USING_IF_EXISTS',
                   '_LIBCPP_WEAK',
                  ]
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 3c635e5e46bb..61805726a4ff 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -198,6 +198,10 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_char8_t``                                      ``201907L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_common_reference``                             ``202302L``
+    ---------------------------------------------------------- -----------------
+    ``__cpp_lib_common_reference_wrapper``                     ``202302L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_concepts``                                     ``202002L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_algorithms``                         ``201806L``
diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index c52bc54f412b..2cb560b3852f 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -47,10 +47,11 @@ Implemented Papers
 - P1222R4: A Standard ``flat_set`` (`Github <https://github.com/llvm/llvm-project/issues/105193>`__)
 - P2897R7: ``aligned_accessor``: An mdspan accessor expressing pointer over-alignment (`Github <https://github.com/llvm/llvm-project/issues/118372>`__)
 - P3247R2: Deprecate the notion of trivial types (`Github <https://github.com/llvm/llvm-project/issues/118387>`__)
-- P3372R3: ``constexpr`` containers and adaptors (`Github <https://github.com/llvm/llvm-project/issues/128673>`__) (Only ``constexpr flat_map`` is implemented)
+- P3372R3: ``constexpr`` containers and adaptors (`Github <https://github.com/llvm/llvm-project/issues/127876>`__) (``forward_list``, ``list``, ``priority_queue``, ``flat_map``, and ``flat_set`` are implemented)
 - P2441R2: ``views::join_with`` (`Github <https://github.com/llvm/llvm-project/issues/105185>`__)
 - P2711R1: Making multi-param constructors of ``views`` ``explicit`` (`Github <https://github.com/llvm/llvm-project/issues/105252>`__)
 - P2770R0: Stashing stashing ``iterators`` for proper flattening (`Github <https://github.com/llvm/llvm-project/issues/105250>`__)
+- P2655R3: ``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type (`Github <https://github.com/llvm/llvm-project/issues/105260>`__)
 
 Improvements and New Features
 -----------------------------
@@ -81,6 +82,9 @@ Improvements and New Features
 - The ``bitset::to_string`` function has been optimized, resulting in a performance improvement of up to 8.3x for bitsets
   with uniformly distributed zeros and ones, and up to 13.5x and 16.1x for sparse and dense bitsets, respectively.
 
+- The ``flat_map::insert`` and ``flat_set::insert_range`` have been optimized, resulting in a performance improvement of up
+  to 10x for inserting elements into a ``flat_map`` when the input range is a ``flat_map`` or a ``zip_view``.
+
 Deprecations and Removals
 -------------------------
 
diff --git a/libcxx/docs/Status/Cxx23Papers.csv b/libcxx/docs/Status/Cxx23Papers.csv
index 574675175a4c..e4fa07d82289 100644
--- a/libcxx/docs/Status/Cxx23Papers.csv
+++ b/libcxx/docs/Status/Cxx23Papers.csv
@@ -113,7 +113,7 @@
 "`P2693R1 <https://wg21.link/P2693R1>`__","Formatting ``thread::id`` and ``stacktrace``","2023-02 (Issaquah)","|Partial|","","The formatter for ``stacktrace`` is not implemented, since ``stacktrace`` is not implemented yet"
 "`P2679R2 <https://wg21.link/P2679R2>`__","Fixing ``std::start_lifetime_as`` for arrays","2023-02 (Issaquah)","","",""
 "`P2674R1 <https://wg21.link/P2674R1>`__","A trait for implicit lifetime types","2023-02 (Issaquah)","|Complete|","20",""
-"`P2655R3 <https://wg21.link/P2655R3>`__","``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type","2023-02 (Issaquah)","","",""
+"`P2655R3 <https://wg21.link/P2655R3>`__","``common_reference_t`` of ``reference_wrapper`` Should Be a Reference Type","2023-02 (Issaquah)","|Complete|","21","The paper is implemented as a DR to C++20"
 "`P2652R2 <https://wg21.link/P2652R2>`__","Disallow User Specialization of ``allocator_traits``","2023-02 (Issaquah)","|Complete|","19",""
 "`P2787R1 <https://wg21.link/P2787R1>`__","``pmr::generator`` - Promise Types are not Values","2023-02 (Issaquah)","","",""
 "`P2614R2 <https://wg21.link/P2614R2>`__","Deprecate ``numeric_limits::has_denorm``","2023-02 (Issaquah)","|Complete|","18",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 2eb192106977..dc18127525a3 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -59,7 +59,7 @@
 "`P2248R8 <https://wg21.link/P2248R8>`__","Enabling list-initialization for algorithms","2024-03 (Tokyo)","","",""
 "`P2810R4 <https://wg21.link/P2810R4>`__","``is_debugger_present`` ``is_replaceable``","2024-03 (Tokyo)","","",""
 "`P1068R11 <https://wg21.link/P1068R11>`__","Vector API for random number generation","2024-03 (Tokyo)","","",""
-"`P2944R3 <https://wg21.link/P2944R3>`__","Comparisons for ``reference_wrapper``","2024-03 (Tokyo)","|Partial|","","The changes to ``optional``, ``tuple`` and ``variant`` are not yet implemented"
+"`P2944R3 <https://wg21.link/P2944R3>`__","Comparisons for ``reference_wrapper``","2024-03 (Tokyo)","|Partial|","","The changes to ``optional`` and ``tuple``'s equality overload from P2165R4 are not yet implemented"
 "`P2642R6 <https://wg21.link/P2642R6>`__","Padded ``mdspan`` layouts","2024-03 (Tokyo)","","",""
 "`P3029R1 <https://wg21.link/P3029R1>`__","Better ``mdspan``'s CTAD","2024-03 (Tokyo)","|Complete|","19",""
 "","","","","",""
diff --git a/libcxx/docs/UserDocumentation.rst b/libcxx/docs/UserDocumentation.rst
index f82554a4869f..79f5908bdc6b 100644
--- a/libcxx/docs/UserDocumentation.rst
+++ b/libcxx/docs/UserDocumentation.rst
@@ -114,11 +114,6 @@ enable or disable extended libc++ behavior.
   ensure that the appropriate experimental library (usually ``libc++experimental.a``)
   is linked into their program.
 
-**_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS**:
-  This macro is used to enable -Wthread-safety annotations on libc++'s
-  ``std::mutex`` and ``std::lock_guard``. By default, these annotations are
-  disabled and must be manually enabled by the user.
-
 **_LIBCPP_HARDENING_MODE**:
   This macro is used to choose the :ref:`hardening mode <using-hardening-modes>`.
 
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index e386f31386b6..248aed666e27 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -498,6 +498,7 @@ set(files
   __iterator/ostreambuf_iterator.h
   __iterator/permutable.h
   __iterator/prev.h
+  __iterator/product_iterator.h
   __iterator/projected.h
   __iterator/ranges_iterator_traits.h
   __iterator/readable_traits.h
@@ -510,13 +511,19 @@ set(files
   __iterator/unreachable_sentinel.h
   __iterator/wrap_iter.h
   __locale
+  __locale_dir/check_grouping.h
+  __locale_dir/get_c_locale.h
   __locale_dir/locale_base_api.h
   __locale_dir/locale_base_api/android.h
   __locale_dir/locale_base_api/bsd_locale_fallbacks.h
   __locale_dir/locale_base_api/ibm.h
   __locale_dir/locale_base_api/musl.h
   __locale_dir/locale_base_api/openbsd.h
+  __locale_dir/messages.h
+  __locale_dir/money.h
+  __locale_dir/num.h
   __locale_dir/pad_and_output.h
+  __locale_dir/scan_keyword.h
   __locale_dir/support/apple.h
   __locale_dir/support/bsd_like.h
   __locale_dir/support/freebsd.h
@@ -525,6 +532,9 @@ set(files
   __locale_dir/support/no_locale/characters.h
   __locale_dir/support/no_locale/strtonum.h
   __locale_dir/support/windows.h
+  __locale_dir/time.h
+  __locale_dir/wbuffer_convert.h
+  __locale_dir/wstring_convert.h
   __math/abs.h
   __math/copysign.h
   __math/error_functions.h
@@ -777,9 +787,8 @@ set(files
   __tuple/tuple_size.h
   __tuple/tuple_types.h
   __type_traits/add_cv_quals.h
-  __type_traits/add_lvalue_reference.h
   __type_traits/add_pointer.h
-  __type_traits/add_rvalue_reference.h
+  __type_traits/add_reference.h
   __type_traits/aligned_storage.h
   __type_traits/aligned_union.h
   __type_traits/alignment_of.h
diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index 86b606d27847..49758397f618 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -50,12 +50,14 @@
 #  include <__format/formatter.h>
 #  include <__format/parser_std_format_spec.h>
 #  include <__format/write_escaped.h>
+#  include <__iterator/istreambuf_iterator.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__locale_dir/time.h>
 #  include <__memory/addressof.h>
 #  include <__type_traits/is_specialization.h>
 #  include <cmath>
 #  include <ctime>
 #  include <limits>
-#  include <locale>
 #  include <sstream>
 #  include <string_view>
 
diff --git a/libcxx/include/__concepts/common_with.h b/libcxx/include/__concepts/common_with.h
index 85abb05efbc2..09082b2295de 100644
--- a/libcxx/include/__concepts/common_with.h
+++ b/libcxx/include/__concepts/common_with.h
@@ -12,7 +12,7 @@
 #include <__concepts/common_reference_with.h>
 #include <__concepts/same_as.h>
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/common_reference.h>
 #include <__type_traits/common_type.h>
 #include <__utility/declval.h>
diff --git a/libcxx/include/__config b/libcxx/include/__config
index af8a297fdf3f..58ca9bab937e 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -903,23 +903,6 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_NO_THREAD_SAFETY_ANALYSIS
 #  endif
 
-// Work around the attribute handling in clang.  When both __declspec and
-// __attribute__ are present, the processing goes awry preventing the definition
-// of the types. In MinGW mode, __declspec evaluates to __attribute__, and thus
-// combining the two does work.
-#  if defined(_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS) && defined(__clang__) &&                                       \
-      __has_attribute(acquire_capability) && !defined(_MSC_VER)
-#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 1
-#  else
-#    define _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS 0
-#  endif
-
-#  if _LIBCPP_HAS_THREAD_SAFETY_ANNOTATIONS
-#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x) __attribute__((x))
-#  else
-#    define _LIBCPP_THREAD_SAFETY_ANNOTATION(x)
-#  endif
-
 #  if _LIBCPP_STD_VER >= 20
 #    define _LIBCPP_CONSTINIT constinit
 #  elif __has_attribute(__require_constant_initialization__)
@@ -1147,6 +1130,60 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_PREFERRED_NAME(x)
 #  endif
 
+#  if __has_cpp_attribute(_Clang::__scoped_lockable__)
+#    define _LIBCPP_SCOPED_LOCKABLE [[_Clang::__scoped_lockable__]]
+#  else
+#    define _LIBCPP_SCOPED_LOCKABLE
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__capability__)
+#    define _LIBCPP_CAPABILITY(...) [[_Clang::__capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_CAPABILITY(...)
+#  endif
+
+#  if __has_attribute(__acquire_capability__)
+#    define _LIBCPP_ACQUIRE_CAPABILITY(...) __attribute__((__acquire_capability__(__VA_ARGS__)))
+#  else
+#    define _LIBCPP_ACQUIRE_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__try_acquire_capability__)
+#    define _LIBCPP_TRY_ACQUIRE_CAPABILITY(...) [[_Clang::__try_acquire_capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_TRY_ACQUIRE_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__acquire_shared_capability__)
+#    define _LIBCPP_ACQUIRE_SHARED_CAPABILITY [[_Clang::__acquire_shared_capability__]]
+#  else
+#    define _LIBCPP_ACQUIRE_SHARED_CAPABILITY
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__try_acquire_shared_capability__)
+#    define _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(...) [[_Clang::__try_acquire_shared_capability__(__VA_ARGS__)]]
+#  else
+#    define _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(...)
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__release_capability__)
+#    define _LIBCPP_RELEASE_CAPABILITY [[_Clang::__release_capability__]]
+#  else
+#    define _LIBCPP_RELEASE_CAPABILITY
+#  endif
+
+#  if __has_cpp_attribute(_Clang::__release_shared_capability__)
+#    define _LIBCPP_RELEASE_SHARED_CAPABILITY [[_Clang::__release_shared_capability__]]
+#  else
+#    define _LIBCPP_RELEASE_SHARED_CAPABILITY
+#  endif
+
+#  if __has_attribute(__requires_capability__)
+#    define _LIBCPP_REQUIRES_CAPABILITY(...) __attribute__((__requires_capability__(__VA_ARGS__)))
+#  else
+#    define _LIBCPP_REQUIRES_CAPABILITY(...)
+#  endif
+
 #  if defined(_LIBCPP_ABI_MICROSOFT) && __has_declspec_attribute(empty_bases)
 #    define _LIBCPP_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
 #  else
diff --git a/libcxx/include/__coroutine/coroutine_handle.h b/libcxx/include/__coroutine/coroutine_handle.h
index 88848544cf6b..b7add258510e 100644
--- a/libcxx/include/__coroutine/coroutine_handle.h
+++ b/libcxx/include/__coroutine/coroutine_handle.h
@@ -172,6 +172,6 @@ struct hash<coroutine_handle<_Tp>> {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__coroutine/coroutine_traits.h b/libcxx/include/__coroutine/coroutine_traits.h
index 78f05341f748..60f7ca2311c6 100644
--- a/libcxx/include/__coroutine/coroutine_traits.h
+++ b/libcxx/include/__coroutine/coroutine_traits.h
@@ -43,6 +43,6 @@ struct coroutine_traits : public __coroutine_traits_sfinae<_Ret> {};
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_COROUTINE_TRAITS_H
diff --git a/libcxx/include/__coroutine/noop_coroutine_handle.h b/libcxx/include/__coroutine/noop_coroutine_handle.h
index c95d85d5484c..2b2838b6bf49 100644
--- a/libcxx/include/__coroutine/noop_coroutine_handle.h
+++ b/libcxx/include/__coroutine/noop_coroutine_handle.h
@@ -94,6 +94,6 @@ inline _LIBCPP_HIDE_FROM_ABI noop_coroutine_handle noop_coroutine() noexcept { r
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // _LIBCPP___COROUTINE_NOOP_COROUTINE_HANDLE_H
diff --git a/libcxx/include/__coroutine/trivial_awaitables.h b/libcxx/include/__coroutine/trivial_awaitables.h
index b604bd3c2d8a..7cb7f4dfeb78 100644
--- a/libcxx/include/__coroutine/trivial_awaitables.h
+++ b/libcxx/include/__coroutine/trivial_awaitables.h
@@ -35,6 +35,6 @@ struct suspend_always {
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // __LIBCPP_STD_VER >= 20
+#endif // _LIBCPP_STD_VER >= 20
 
 #endif // __LIBCPP___COROUTINE_TRIVIAL_AWAITABLES_H
diff --git a/libcxx/include/__filesystem/path.h b/libcxx/include/__filesystem/path.h
index a2c28bfd79bb..381e5678a585 100644
--- a/libcxx/include/__filesystem/path.h
+++ b/libcxx/include/__filesystem/path.h
@@ -29,7 +29,6 @@
 
 #if _LIBCPP_HAS_LOCALIZATION
 #  include <iomanip> // for quoted
-#  include <locale>
 #endif
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__filesystem/u8path.h b/libcxx/include/__filesystem/u8path.h
index e13980298d9e..a701425e4291 100644
--- a/libcxx/include/__filesystem/u8path.h
+++ b/libcxx/include/__filesystem/u8path.h
@@ -13,14 +13,9 @@
 #include <__algorithm/unwrap_iter.h>
 #include <__config>
 #include <__filesystem/path.h>
+#include <__locale>
 #include <string>
 
-// Only required on Windows for __widen_from_utf8, and included conservatively
-// because it requires support for localization.
-#if defined(_LIBCPP_WIN32API)
-#  include <locale>
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
diff --git a/libcxx/include/__flat_map/key_value_iterator.h b/libcxx/include/__flat_map/key_value_iterator.h
index f163dfc28706..d04a23d1f860 100644
--- a/libcxx/include/__flat_map/key_value_iterator.h
+++ b/libcxx/include/__flat_map/key_value_iterator.h
@@ -13,9 +13,12 @@
 #include <__compare/three_way_comparable.h>
 #include <__concepts/convertible_to.h>
 #include <__config>
+#include <__cstddef/size_t.h>
 #include <__iterator/iterator_traits.h>
+#include <__iterator/product_iterator.h>
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
+#include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
 
@@ -57,6 +60,8 @@ struct __key_value_iterator {
   template <class, class, class, bool>
   friend struct __key_value_iterator;
 
+  friend struct __product_iterator_traits<__key_value_iterator>;
+
 public:
   using iterator_concept = random_access_iterator_tag;
   // `__key_value_iterator` only satisfy "Cpp17InputIterator" named requirements, because
@@ -181,6 +186,29 @@ struct __key_value_iterator {
   }
 };
 
+template <class _Owner, class _KeyContainer, class _MappedContainer, bool _Const>
+struct __product_iterator_traits<__key_value_iterator<_Owner, _KeyContainer, _MappedContainer, _Const>> {
+  static constexpr size_t __size = 2;
+
+  template <size_t _Nth, class _Iter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static decltype(auto) __get_iterator_element(_Iter&& __it)
+    requires(_Nth <= 1)
+  {
+    if constexpr (_Nth == 0) {
+      return std::forward<_Iter>(__it).__key_iter_;
+    } else {
+      return std::forward<_Iter>(__it).__mapped_iter_;
+    }
+  }
+
+  template <class _KeyIter, class _MappedIter>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto
+  __make_product_iterator(_KeyIter&& __key_iter, _MappedIter&& __mapped_iter) {
+    return __key_value_iterator<_Owner, _KeyContainer, _MappedContainer, _Const>(
+        std::forward<_KeyIter>(__key_iter), std::forward<_MappedIter>(__mapped_iter));
+  }
+};
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 23
diff --git a/libcxx/include/__flat_map/utils.h b/libcxx/include/__flat_map/utils.h
index 27687ae8de3b..3a05c715660d 100644
--- a/libcxx/include/__flat_map/utils.h
+++ b/libcxx/include/__flat_map/utils.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___FLAT_MAP_UTILS_H
 
 #include <__config>
+#include <__iterator/product_iterator.h>
 #include <__type_traits/container_traits.h>
 #include <__utility/exception_guard.h>
 #include <__utility/forward.h>
@@ -79,8 +80,6 @@ struct __flat_map_utils {
     return typename _Map::iterator(std::move(__key_it), std::move(__mapped_it));
   }
 
-  // TODO: We could optimize this, see
-  // https://github.com/llvm/llvm-project/issues/108624
   template <class _Map, class _InputIterator, class _Sentinel>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static typename _Map::size_type
   __append(_Map& __map, _InputIterator __first, _Sentinel __last) {
@@ -93,6 +92,25 @@ struct __flat_map_utils {
     }
     return __num_appended;
   }
+
+  template <class _Map, class _InputIterator>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static typename _Map::size_type
+  __append(_Map& __map, _InputIterator __first, _InputIterator __last)
+    requires __is_product_iterator_of_size<_InputIterator, 2>::value
+  {
+    auto __s1 = __map.__containers_.keys.size();
+    __map.__containers_.keys.insert(
+        __map.__containers_.keys.end(),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<0>(__first),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<0>(__last));
+
+    __map.__containers_.values.insert(
+        __map.__containers_.values.end(),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<1>(__first),
+        __product_iterator_traits<_InputIterator>::template __get_iterator_element<1>(__last));
+
+    return __map.__containers_.keys.size() - __s1;
+  }
 };
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__flat_set/flat_set.h b/libcxx/include/__flat_set/flat_set.h
index b8edd8eee0cf..95cb998459bc 100644
--- a/libcxx/include/__flat_set/flat_set.h
+++ b/libcxx/include/__flat_set/flat_set.h
@@ -107,13 +107,13 @@ class flat_set {
 
 public:
   // [flat.set.cons], construct/copy/destroy
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set() noexcept(is_nothrow_default_constructible_v<_KeyContainer> && is_nothrow_default_constructible_v<_Compare>)
       : __keys_(), __compare_() {}
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(const flat_set&) = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const flat_set&) = default;
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(flat_set&& __other) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(flat_set&& __other) noexcept(
       is_nothrow_move_constructible_v<_KeyContainer> && is_nothrow_move_constructible_v<_Compare>)
 #  if _LIBCPP_HAS_EXCEPTIONS
       try
@@ -130,14 +130,17 @@ class flat_set {
 #  endif // _LIBCPP_HAS_EXCEPTIONS
   }
 
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(const key_compare& __comp) : __keys_(), __compare_(__comp) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(const key_compare& __comp)
+      : __keys_(), __compare_(__comp) {}
 
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(container_type __keys, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(container_type __keys, const key_compare& __comp = key_compare())
       : __keys_(std::move(__keys)), __compare_(__comp) {
     __sort_and_unique();
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, container_type __keys, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, container_type __keys, const key_compare& __comp = key_compare())
       : __keys_(std::move(__keys)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -145,7 +148,7 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(_InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
       : __keys_(), __compare_(__comp) {
     insert(__first, __last);
@@ -153,7 +156,7 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, _InputIterator __first, _InputIterator __last, const key_compare& __comp = key_compare())
       : __keys_(__first, __last), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -161,48 +164,52 @@ class flat_set {
   }
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg)
       : flat_set(from_range, std::forward<_Range>(__rg), key_compare()) {}
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const key_compare& __comp) : flat_set(__comp) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg, const key_compare& __comp)
+      : flat_set(__comp) {
     insert_range(std::forward<_Range>(__rg));
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : flat_set(__il.begin(), __il.end(), __comp) {}
 
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : flat_set(sorted_unique, __il.begin(), __il.end(), __comp) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI explicit flat_set(const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 explicit flat_set(const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const container_type& __keys, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const container_type& __keys, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
     __sort_and_unique();
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
     __sort_and_unique();
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, const container_type& __keys, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, const container_type& __keys, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_() {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -210,7 +217,7 @@ class flat_set {
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, const container_type& __keys, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __keys)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -219,13 +226,13 @@ class flat_set {
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(const flat_set& __other, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(const flat_set& __other, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __other.__keys_)),
         __compare_(__other.__compare_) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(flat_set&& __other, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(flat_set&& __other, const _Allocator& __alloc)
 #  if _LIBCPP_HAS_EXCEPTIONS
       try
 #  endif // _LIBCPP_HAS_EXCEPTIONS
@@ -241,14 +248,15 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI flat_set(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(_InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
     insert(__first, __last);
   }
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(_InputIterator __first, _InputIterator __last, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
     insert(__first, __last);
@@ -256,7 +264,7 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, _InputIterator __first, _InputIterator __last, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_() {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
@@ -265,12 +273,12 @@ class flat_set {
 
   template <class _InputIterator, class _Allocator>
     requires(__has_input_iterator_category<_InputIterator>::value && uses_allocator<container_type, _Allocator>::value)
-  _LIBCPP_HIDE_FROM_ABI
-  flat_set(sorted_unique_t,
-           _InputIterator __first,
-           _InputIterator __last,
-           const key_compare& __comp,
-           const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(
+      sorted_unique_t,
+      _InputIterator __first,
+      _InputIterator __last,
+      const key_compare& __comp,
+      const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc, __first, __last)), __compare_(__comp) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys_), "Either the key container is not sorted or it contains duplicates");
@@ -278,49 +286,52 @@ class flat_set {
 
   template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set(from_range_t, _Range&& __rg, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_() {
     insert_range(std::forward<_Range>(__rg));
   }
 
   template <_ContainerCompatibleRange<value_type> _Range, class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(from_range_t, _Range&& __rg, const key_compare& __comp, const _Allocator& __alloc)
       : __keys_(std::make_obj_using_allocator<container_type>(__alloc)), __compare_(__comp) {
     insert_range(std::forward<_Range>(__rg));
   }
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(initializer_list<value_type> __il, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(initializer_list<value_type> __il, const _Allocator& __alloc)
       : flat_set(__il.begin(), __il.end(), __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
       : flat_set(__il.begin(), __il.end(), __comp, __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI flat_set(sorted_unique_t, initializer_list<value_type> __il, const _Allocator& __alloc)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
+  flat_set(sorted_unique_t, initializer_list<value_type> __il, const _Allocator& __alloc)
       : flat_set(sorted_unique, __il.begin(), __il.end(), __alloc) {}
 
   template <class _Allocator>
     requires uses_allocator<container_type, _Allocator>::value
-  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26
   flat_set(sorted_unique_t, initializer_list<value_type> __il, const key_compare& __comp, const _Allocator& __alloc)
       : flat_set(sorted_unique, __il.begin(), __il.end(), __comp, __alloc) {}
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(initializer_list<value_type> __il) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(initializer_list<value_type> __il) {
     clear();
     insert(__il);
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(const flat_set&) = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(const flat_set&) = default;
 
-  _LIBCPP_HIDE_FROM_ABI flat_set& operator=(flat_set&& __other) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 flat_set& operator=(flat_set&& __other) noexcept(
       is_nothrow_move_assignable_v<_KeyContainer> && is_nothrow_move_assignable_v<_Compare>) {
     // No matter what happens, we always want to clear the other container before returning
     // since we moved from it
@@ -336,31 +347,53 @@ class flat_set {
   }
 
   // iterators
-  _LIBCPP_HIDE_FROM_ABI iterator begin() noexcept { return iterator(std::as_const(__keys_).begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const noexcept { return const_iterator(__keys_.begin()); }
-  _LIBCPP_HIDE_FROM_ABI iterator end() noexcept { return iterator(std::as_const(__keys_).end()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator end() const noexcept { return const_iterator(__keys_.end()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator begin() noexcept {
+    return iterator(std::as_const(__keys_).begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator begin() const noexcept {
+    return const_iterator(__keys_.begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator end() noexcept {
+    return iterator(std::as_const(__keys_).end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator end() const noexcept {
+    return const_iterator(__keys_.end());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() noexcept { return reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const noexcept { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() noexcept { return reverse_iterator(begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const noexcept { return const_reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rbegin() noexcept {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rbegin() const noexcept {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 reverse_iterator rend() noexcept {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator rend() const noexcept {
+    return const_reverse_iterator(begin());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const noexcept { return begin(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const noexcept { return end(); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const noexcept { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const noexcept { return const_reverse_iterator(begin()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cbegin() const noexcept { return begin(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator cend() const noexcept { return end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crbegin() const noexcept {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_reverse_iterator crend() const noexcept {
+    return const_reverse_iterator(begin());
+  }
 
   // [flat.set.capacity], capacity
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI bool empty() const noexcept { return __keys_.empty(); }
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool empty() const noexcept {
+    return __keys_.empty();
+  }
 
-  _LIBCPP_HIDE_FROM_ABI size_type size() const noexcept { return __keys_.size(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type size() const noexcept { return __keys_.size(); }
 
-  _LIBCPP_HIDE_FROM_ABI size_type max_size() const noexcept { return __keys_.max_size(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type max_size() const noexcept { return __keys_.max_size(); }
 
   // [flat.set.modifiers], modifiers
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> emplace(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> emplace(_Args&&... __args) {
     if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
       return __emplace(std::forward<_Args>(__args)...);
     } else {
@@ -369,7 +402,7 @@ class flat_set {
   }
 
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator emplace_hint(const_iterator __hint, _Args&&... __args) {
     if constexpr (sizeof...(__args) == 1 && (is_same_v<remove_cvref_t<_Args>, _Key> && ...)) {
       return __emplace_hint(std::move(__hint), std::forward<_Args>(__args)...);
     } else {
@@ -377,32 +410,36 @@ class flat_set {
     }
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(const value_type& __x) { return emplace(__x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(const value_type& __x) {
+    return emplace(__x);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(value_type&& __x) { return emplace(std::move(__x)); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(value_type&& __x) {
+    return emplace(std::move(__x));
+  }
 
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && is_constructible_v<value_type, _Kp>)
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> insert(_Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> insert(_Kp&& __x) {
     return __emplace(std::forward<_Kp>(__x));
   }
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, const value_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, const value_type& __x) {
     return emplace_hint(__hint, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, value_type&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, value_type&& __x) {
     return emplace_hint(__hint, std::move(__x));
   }
 
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && is_constructible_v<value_type, _Kp>)
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __hint, _Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator insert(const_iterator __hint, _Kp&& __x) {
     return __emplace_hint(__hint, std::forward<_Kp>(__x));
   }
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(_InputIterator __first, _InputIterator __last) {
     if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
       __reserve(__last - __first);
     }
@@ -411,7 +448,8 @@ class flat_set {
 
   template <class _InputIterator>
     requires __has_input_iterator_category<_InputIterator>::value
-  _LIBCPP_HIDE_FROM_ABI void insert(sorted_unique_t, _InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void
+  insert(sorted_unique_t, _InputIterator __first, _InputIterator __last) {
     if constexpr (sized_sentinel_for<_InputIterator, _InputIterator>) {
       __reserve(__last - __first);
     }
@@ -420,7 +458,7 @@ class flat_set {
   }
 
   template <_ContainerCompatibleRange<value_type> _Range>
-  _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert_range(_Range&& __range) {
     if constexpr (ranges::sized_range<_Range>) {
       __reserve(ranges::size(__range));
     }
@@ -428,19 +466,21 @@ class flat_set {
     __append_sort_merge_unique</*WasSorted = */ false>(std::forward<_Range>(__range));
   }
 
-  _LIBCPP_HIDE_FROM_ABI void insert(initializer_list<value_type> __il) { insert(__il.begin(), __il.end()); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(initializer_list<value_type> __il) {
+    insert(__il.begin(), __il.end());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI void insert(sorted_unique_t, initializer_list<value_type> __il) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void insert(sorted_unique_t, initializer_list<value_type> __il) {
     insert(sorted_unique, __il.begin(), __il.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI container_type extract() && {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 container_type extract() && {
     auto __guard = std::__make_scope_guard([&]() noexcept { clear() /* noexcept */; });
     auto __ret   = std::move(__keys_);
     return __ret;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void replace(container_type&& __keys) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void replace(container_type&& __keys) {
     _LIBCPP_ASSERT_SEMANTIC_REQUIREMENT(
         __is_sorted_and_unique(__keys), "Either the key container is not sorted or it contains duplicates");
     auto __guard = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
@@ -448,7 +488,7 @@ class flat_set {
     __guard.__complete();
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase(iterator __position) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(iterator __position) {
     auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     auto __key_iter   = __keys_.erase(__position.__base());
     __on_failure.__complete();
@@ -458,7 +498,7 @@ class flat_set {
   // The following overload is the same as the iterator overload
   // iterator erase(const_iterator __position);
 
-  _LIBCPP_HIDE_FROM_ABI size_type erase(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(const key_type& __x) {
     auto __iter = find(__x);
     if (__iter != end()) {
       erase(__iter);
@@ -470,21 +510,21 @@ class flat_set {
   template <class _Kp>
     requires(__is_transparent_v<_Compare> && !is_convertible_v<_Kp &&, iterator> &&
              !is_convertible_v<_Kp &&, const_iterator>)
-  _LIBCPP_HIDE_FROM_ABI size_type erase(_Kp&& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type erase(_Kp&& __x) {
     auto [__first, __last] = equal_range(__x);
     auto __res             = __last - __first;
     erase(__first, __last);
     return __res;
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator erase(const_iterator __first, const_iterator __last) {
     auto __on_failure = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     auto __key_it     = __keys_.erase(__first.__base(), __last.__base());
     __on_failure.__complete();
     return iterator(std::move(__key_it));
   }
 
-  _LIBCPP_HIDE_FROM_ABI void swap(flat_set& __y) noexcept {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __y) noexcept {
     // warning: The spec has unconditional noexcept, which means that
     // if any of the following functions throw an exception,
     // std::terminate will be called.
@@ -493,121 +533,134 @@ class flat_set {
     ranges::swap(__keys_, __y.__keys_);
   }
 
-  _LIBCPP_HIDE_FROM_ABI void clear() noexcept { __keys_.clear(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void clear() noexcept { __keys_.clear(); }
 
   // observers
-  _LIBCPP_HIDE_FROM_ABI key_compare key_comp() const { return __compare_; }
-  _LIBCPP_HIDE_FROM_ABI value_compare value_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 key_compare key_comp() const { return __compare_; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 value_compare value_comp() const { return __compare_; }
 
   // set operations
-  _LIBCPP_HIDE_FROM_ABI iterator find(const key_type& __x) { return __find_impl(*this, __x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const key_type& __x) {
+    return __find_impl(*this, __x);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator find(const key_type& __x) const { return __find_impl(*this, __x); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const key_type& __x) const {
+    return __find_impl(*this, __x);
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator find(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator find(const _Kp& __x) {
     return __find_impl(*this, __x);
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator find(const _Kp& __x) const {
     return __find_impl(*this, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI size_type count(const key_type& __x) const { return contains(__x) ? 1 : 0; }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const key_type& __x) const {
+    return contains(__x) ? 1 : 0;
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI size_type count(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 size_type count(const _Kp& __x) const {
     return contains(__x) ? 1 : 0;
   }
 
-  _LIBCPP_HIDE_FROM_ABI bool contains(const key_type& __x) const { return find(__x) != end(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const key_type& __x) const {
+    return find(__x) != end();
+  }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI bool contains(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool contains(const _Kp& __x) const {
     return find(__x) != end();
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const key_type& __x) {
     const auto& __keys = __keys_;
     return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const key_type& __x) const {
     return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator lower_bound(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator lower_bound(const _Kp& __x) {
     const auto& __keys = __keys_;
     return iterator(std::lower_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator lower_bound(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator lower_bound(const _Kp& __x) const {
     return const_iterator(std::lower_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const key_type& __x) {
     const auto& __keys = __keys_;
     return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const key_type& __x) const {
     return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI iterator upper_bound(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator upper_bound(const _Kp& __x) {
     const auto& __keys = __keys_;
     return iterator(std::upper_bound(__keys.begin(), __keys.end(), __x, __compare_));
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI const_iterator upper_bound(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 const_iterator upper_bound(const _Kp& __x) const {
     return const_iterator(std::upper_bound(__keys_.begin(), __keys_.end(), __x, __compare_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const key_type& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const key_type& __x) {
     return __equal_range_impl(*this, __x);
   }
 
-  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const key_type& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
+  equal_range(const key_type& __x) const {
     return __equal_range_impl(*this, __x);
   }
 
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, iterator> equal_range(const _Kp& __x) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, iterator> equal_range(const _Kp& __x) {
     return __equal_range_impl(*this, __x);
   }
   template <class _Kp>
     requires __is_transparent_v<_Compare>
-  _LIBCPP_HIDE_FROM_ABI pair<const_iterator, const_iterator> equal_range(const _Kp& __x) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<const_iterator, const_iterator>
+  equal_range(const _Kp& __x) const {
     return __equal_range_impl(*this, __x);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const flat_set& __x, const flat_set& __y) {
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool operator==(const flat_set& __x, const flat_set& __y) {
     return ranges::equal(__x, __y);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI auto operator<=>(const flat_set& __x, const flat_set& __y) {
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 auto
+  operator<=>(const flat_set& __x, const flat_set& __y) {
     return std::lexicographical_compare_three_way(
         __x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI void swap(flat_set& __x, flat_set& __y) noexcept { __x.swap(__y); }
+  friend _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void swap(flat_set& __x, flat_set& __y) noexcept {
+    __x.swap(__y);
+  }
 
 private:
-  _LIBCPP_HIDE_FROM_ABI bool __is_sorted_and_unique(auto&& __key_container) const {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_sorted_and_unique(auto&& __key_container) const {
     auto __greater_or_equal_to = [this](const auto& __x, const auto& __y) { return !__compare_(__x, __y); };
     return ranges::adjacent_find(__key_container, __greater_or_equal_to) == ranges::end(__key_container);
   }
@@ -615,14 +668,14 @@ class flat_set {
   // This function is only used in constructors. So there is not exception handling in this function.
   // If the function exits via an exception, there will be no flat_set object constructed, thus, there
   // is no invariant state to preserve
-  _LIBCPP_HIDE_FROM_ABI void __sort_and_unique() {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __sort_and_unique() {
     ranges::sort(__keys_, __compare_);
     auto __dup_start = ranges::unique(__keys_, __key_equiv(__compare_)).begin();
     __keys_.erase(__dup_start, __keys_.end());
   }
 
   template <bool _WasSorted, class... _Args>
-  _LIBCPP_HIDE_FROM_ABI void __append_sort_merge_unique(_Args&&... __args) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __append_sort_merge_unique(_Args&&... __args) {
     auto __on_failure    = std::__make_exception_guard([&]() noexcept { clear() /* noexcept */; });
     size_type __old_size = size();
     __flat_set_utils::__append(*this, std::forward<_Args>(__args)...);
@@ -642,7 +695,7 @@ class flat_set {
   }
 
   template <class _Self, class _Kp>
-  _LIBCPP_HIDE_FROM_ABI static auto __find_impl(_Self&& __self, const _Kp& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __find_impl(_Self&& __self, const _Kp& __key) {
     auto __it   = __self.lower_bound(__key);
     auto __last = __self.end();
     if (__it == __last || __self.__compare_(__key, *__it)) {
@@ -652,7 +705,7 @@ class flat_set {
   }
 
   template <class _Self, class _Kp>
-  _LIBCPP_HIDE_FROM_ABI static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto __equal_range_impl(_Self&& __self, const _Kp& __key) {
     using __iter = _If<is_const_v<__libcpp_remove_reference_t<_Self>>, const_iterator, iterator>;
     auto __it    = std::lower_bound(__self.__keys_.begin(), __self.__keys_.end(), __key, __self.__compare_);
     auto __last  = __self.__keys_.end();
@@ -663,7 +716,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __emplace(_Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 pair<iterator, bool> __emplace(_Kp&& __key) {
     auto __it = lower_bound(__key);
     if (__it == end() || __compare_(__key, *__it)) {
       return pair<iterator, bool>(__flat_set_utils::__emplace_exact_pos(*this, __it, std::forward<_Kp>(__key)), true);
@@ -673,7 +726,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI bool __is_hint_correct(const_iterator __hint, _Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool __is_hint_correct(const_iterator __hint, _Kp&& __key) {
     if (__hint != cbegin() && !__compare_(*std::prev(__hint), __key)) {
       return false;
     }
@@ -684,7 +737,7 @@ class flat_set {
   }
 
   template <class _Kp>
-  _LIBCPP_HIDE_FROM_ABI iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 iterator __emplace_hint(const_iterator __hint, _Kp&& __key) {
     if (__is_hint_correct(__hint, __key)) {
       if (__hint == cend() || __compare_(__key, *__hint)) {
         return __flat_set_utils::__emplace_exact_pos(*this, __hint, std::forward<_Kp>(__key));
@@ -697,22 +750,23 @@ class flat_set {
     }
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __reserve(size_t __size) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 void __reserve(size_t __size) {
     if constexpr (__container_traits<_KeyContainer>::__reservable) {
       __keys_.reserve(__size);
     }
   }
 
   template <class _Key2, class _Compare2, class _KeyContainer2, class _Predicate>
-  friend typename flat_set<_Key2, _Compare2, _KeyContainer2>::size_type
+  friend typename flat_set<_Key2, _Compare2, _KeyContainer2>::size_type _LIBCPP_CONSTEXPR_SINCE_CXX26
   erase_if(flat_set<_Key2, _Compare2, _KeyContainer2>&, _Predicate);
 
   _KeyContainer __keys_;
   _LIBCPP_NO_UNIQUE_ADDRESS key_compare __compare_;
 
   struct __key_equiv {
-    _LIBCPP_HIDE_FROM_ABI __key_equiv(key_compare __c) : __comp_(__c) {}
-    _LIBCPP_HIDE_FROM_ABI bool operator()(const_reference __x, const_reference __y) const {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 __key_equiv(key_compare __c) : __comp_(__c) {}
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 bool
+    operator()(const_reference __x, const_reference __y) const {
       return !__comp_(__x, __y) && !__comp_(__y, __x);
     }
     key_compare __comp_;
@@ -799,7 +853,7 @@ struct uses_allocator<flat_set<_Key, _Compare, _KeyContainer>, _Allocator>
     : bool_constant<uses_allocator_v<_KeyContainer, _Allocator>> {};
 
 template <class _Key, class _Compare, class _KeyContainer, class _Predicate>
-_LIBCPP_HIDE_FROM_ABI typename flat_set<_Key, _Compare, _KeyContainer>::size_type
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 typename flat_set<_Key, _Compare, _KeyContainer>::size_type
 erase_if(flat_set<_Key, _Compare, _KeyContainer>& __flat_set, _Predicate __pred) {
   auto __guard = std::__make_exception_guard([&] { __flat_set.clear(); });
   auto __it    = std::remove_if(__flat_set.__keys_.begin(), __flat_set.__keys_.end(), [&](const auto& __e) -> bool {
diff --git a/libcxx/include/__flat_set/utils.h b/libcxx/include/__flat_set/utils.h
index 542bfd886aef..8a7f93a00465 100644
--- a/libcxx/include/__flat_set/utils.h
+++ b/libcxx/include/__flat_set/utils.h
@@ -38,7 +38,8 @@ struct __flat_set_utils {
   // When an exception is thrown during the emplacement, the function will clear the set if the container does not
   // have strong exception safety guarantee on emplacement.
   template <class _Set, class _Iter, class _KeyArg>
-  _LIBCPP_HIDE_FROM_ABI static auto __emplace_exact_pos(_Set& __set, _Iter&& __iter, _KeyArg&& __key) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static auto
+  __emplace_exact_pos(_Set& __set, _Iter&& __iter, _KeyArg&& __key) {
     using _KeyContainer = typename decay_t<_Set>::container_type;
     auto __on_failure   = std::__make_exception_guard([&]() noexcept {
       if constexpr (!__container_traits<_KeyContainer>::__emplacement_has_strong_exception_safety_guarantee) {
@@ -51,12 +52,13 @@ struct __flat_set_utils {
   }
 
   template <class _Set, class _InputIterator>
-  _LIBCPP_HIDE_FROM_ABI static void __append(_Set& __set, _InputIterator __first, _InputIterator __last) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static void
+  __append(_Set& __set, _InputIterator __first, _InputIterator __last) {
     __set.__keys_.insert(__set.__keys_.end(), std::move(__first), std::move(__last));
   }
 
   template <class _Set, class _Range>
-  _LIBCPP_HIDE_FROM_ABI static void __append(_Set& __set, _Range&& __rng) {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX26 static void __append(_Set& __set, _Range&& __rng) {
     if constexpr (requires { __set.__keys_.insert_range(__set.__keys_.end(), std::forward<_Range>(__rng)); }) {
       // C++23 Sequence Container should have insert_range member function
       // Note that not all Sequence Containers provide append_range.
diff --git a/libcxx/include/__functional/boyer_moore_searcher.h b/libcxx/include/__functional/boyer_moore_searcher.h
index 7889232f4b91..6c51c32d2f89 100644
--- a/libcxx/include/__functional/boyer_moore_searcher.h
+++ b/libcxx/include/__functional/boyer_moore_searcher.h
@@ -17,12 +17,10 @@
 #include <__config>
 #include <__functional/hash.h>
 #include <__functional/operations.h>
-#include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/shared_ptr.h>
 #include <__type_traits/make_unsigned.h>
 #include <__utility/pair.h>
-#include <__vector/vector.h>
 #include <array>
 #include <limits>
 #include <unordered_map>
@@ -125,8 +123,8 @@ class boyer_moore_searcher {
   template <class _RandomAccessIterator2>
   _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
   operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
-    static_assert(__is_same_uncvref<typename iterator_traits<_RandomAccessIterator1>::value_type,
-                                    typename iterator_traits<_RandomAccessIterator2>::value_type>::value,
+    static_assert(is_same_v<__remove_cvref_t<typename iterator_traits<_RandomAccessIterator1>::value_type>,
+                            __remove_cvref_t<typename iterator_traits<_RandomAccessIterator2>::value_type>>,
                   "Corpus and Pattern iterators must point to the same type");
     if (__first == __last)
       return std::make_pair(__last, __last);
@@ -196,7 +194,7 @@ class boyer_moore_searcher {
     if (__count == 0)
       return;
 
-    vector<difference_type> __scratch(__count);
+    auto __scratch = std::make_unique<difference_type[]>(__count);
 
     __compute_bm_prefix(__first, __last, __pred, __scratch);
     for (size_t __i = 0; __i <= __count; ++__i)
@@ -256,8 +254,8 @@ class boyer_moore_horspool_searcher {
   template <class _RandomAccessIterator2>
   _LIBCPP_HIDE_FROM_ABI pair<_RandomAccessIterator2, _RandomAccessIterator2>
   operator()(_RandomAccessIterator2 __first, _RandomAccessIterator2 __last) const {
-    static_assert(__is_same_uncvref<typename std::iterator_traits<_RandomAccessIterator1>::value_type,
-                                    typename std::iterator_traits<_RandomAccessIterator2>::value_type>::value,
+    static_assert(is_same_v<__remove_cvref_t<typename std::iterator_traits<_RandomAccessIterator1>::value_type>,
+                            __remove_cvref_t<typename std::iterator_traits<_RandomAccessIterator2>::value_type>>,
                   "Corpus and Pattern iterators must point to the same type");
     if (__first == __last)
       return std::make_pair(__last, __last);
diff --git a/libcxx/include/__functional/reference_wrapper.h b/libcxx/include/__functional/reference_wrapper.h
index c46203a4ca9a..148703b21d84 100644
--- a/libcxx/include/__functional/reference_wrapper.h
+++ b/libcxx/include/__functional/reference_wrapper.h
@@ -11,14 +11,18 @@
 #define _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
 
 #include <__compare/synth_three_way.h>
+#include <__concepts/convertible_to.h>
 #include <__config>
 #include <__functional/weak_result_type.h>
 #include <__memory/addressof.h>
+#include <__type_traits/common_reference.h>
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/is_core_convertible.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/is_specialization.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
@@ -45,7 +49,7 @@ class reference_wrapper : public __weak_result_type<_Tp> {
 public:
   template <class _Up,
             class = __void_t<decltype(__fun(std::declval<_Up>()))>,
-            __enable_if_t<!__is_same_uncvref<_Up, reference_wrapper>::value, int> = 0>
+            __enable_if_t<!is_same<__remove_cvref_t<_Up>, reference_wrapper>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference_wrapper(_Up&& __u)
       _NOEXCEPT_(noexcept(__fun(std::declval<_Up>()))) {
     type& __f = static_cast<_Up&&>(__u);
@@ -155,6 +159,32 @@ template <class _CanonicalTag, class _Operation, class... _Args>
 inline const bool __desugars_to_v<_CanonicalTag, reference_wrapper<_Operation>, _Args...> =
     __desugars_to_v<_CanonicalTag, _Operation, _Args...>;
 
+#if _LIBCPP_STD_VER >= 20
+
+template <class _Tp>
+inline constexpr bool __is_ref_wrapper = __is_specialization_v<_Tp, reference_wrapper>;
+
+template <class _Rp, class _Tp, class _RpQual, class _TpQual>
+concept __ref_wrap_common_reference_exists_with = __is_ref_wrapper<_Rp> && requires {
+  typename common_reference_t<typename _Rp::type&, _TpQual>;
+} && convertible_to<_RpQual, common_reference_t<typename _Rp::type&, _TpQual>>;
+
+template <class _Rp, class _Tp, template <class> class _RpQual, template <class> class _TpQual>
+  requires(__ref_wrap_common_reference_exists_with<_Rp, _Tp, _RpQual<_Rp>, _TpQual<_Tp>> &&
+           !__ref_wrap_common_reference_exists_with<_Tp, _Rp, _TpQual<_Tp>, _RpQual<_Rp>>)
+struct basic_common_reference<_Rp, _Tp, _RpQual, _TpQual> {
+  using type _LIBCPP_NODEBUG = common_reference_t<typename _Rp::type&, _TpQual<_Tp>>;
+};
+
+template <class _Tp, class _Rp, template <class> class _TpQual, template <class> class _RpQual>
+  requires(__ref_wrap_common_reference_exists_with<_Rp, _Tp, _RpQual<_Rp>, _TpQual<_Tp>> &&
+           !__ref_wrap_common_reference_exists_with<_Tp, _Rp, _TpQual<_Tp>, _RpQual<_Rp>>)
+struct basic_common_reference<_Tp, _Rp, _TpQual, _RpQual> {
+  using type _LIBCPP_NODEBUG = common_reference_t<typename _Rp::type&, _TpQual<_Tp>>;
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___FUNCTIONAL_REFERENCE_WRAPPER_H
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index aefa8e19c186..c59d8d5bf383 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -29,6 +29,7 @@
 #include <__memory/unique_ptr.h>
 #include <__new/launder.h>
 #include <__type_traits/can_extract_key.h>
+#include <__type_traits/copy_cvref.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
 #include <__type_traits/is_const.h>
@@ -108,9 +109,22 @@ struct __hash_node_base {
   _LIBCPP_HIDE_FROM_ABI explicit __hash_node_base(__next_pointer __next) _NOEXCEPT : __next_(__next) {}
 };
 
+template <class _Tp>
+struct __get_hash_node_value_type {
+  using type _LIBCPP_NODEBUG = _Tp;
+};
+
+template <class _Key, class _Tp>
+struct __get_hash_node_value_type<__hash_value_type<_Key, _Tp> > {
+  using type _LIBCPP_NODEBUG = pair<const _Key, _Tp>;
+};
+
+template <class _Tp>
+using __get_hash_node_value_type_t _LIBCPP_NODEBUG = typename __get_hash_node_value_type<_Tp>::type;
+
 template <class _Tp, class _VoidPtr>
 struct __hash_node : public __hash_node_base< __rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > > {
-  typedef _Tp __node_value_type;
+  using __node_value_type _LIBCPP_NODEBUG = __get_hash_node_value_type_t<_Tp>;
   using _Base _LIBCPP_NODEBUG          = __hash_node_base<__rebind_pointer_t<_VoidPtr, __hash_node<_Tp, _VoidPtr> > >;
   using __next_pointer _LIBCPP_NODEBUG = typename _Base::__next_pointer;
 
@@ -122,18 +136,20 @@ struct __hash_node : public __hash_node_base< __rebind_pointer_t<_VoidPtr, __has
 
 private:
   union {
-    _Tp __value_;
+    __node_value_type __value_;
   };
 
 public:
-  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+  _LIBCPP_HIDE_FROM_ABI __node_value_type& __get_value() { return __value_; }
 #else
 
 private:
-  _ALIGNAS_TYPE(_Tp) char __buffer_[sizeof(_Tp)];
+  _ALIGNAS_TYPE(__node_value_type) char __buffer_[sizeof(__node_value_type)];
 
 public:
-  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return *std::__launder(reinterpret_cast<_Tp*>(&__buffer_)); }
+  _LIBCPP_HIDE_FROM_ABI __node_value_type& __get_value() {
+    return *std::__launder(reinterpret_cast<__node_value_type*>(&__buffer_));
+  }
 #endif
 
   _LIBCPP_HIDE_FROM_ABI explicit __hash_node(__next_pointer __next, size_t __hash) : _Base(__next), __hash_(__hash) {}
@@ -191,18 +207,18 @@ struct __hash_key_value_types<__hash_value_type<_Key, _Tp> > {
 
   _LIBCPP_HIDE_FROM_ABI static key_type const& __get_key(__container_value_type const& __v) { return __v.first; }
 
-  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __node_value_type>::value, int> = 0>
+  template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __node_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
     return __t.__get_value();
   }
 
-  template <class _Up, __enable_if_t<__is_same_uncvref<_Up, __container_value_type>::value, int> = 0>
+  template <class _Up, __enable_if_t<is_same<__remove_cvref_t<_Up>, __container_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static __container_value_type const& __get_value(_Up& __t) {
     return __t;
   }
 
-  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__node_value_type& __n) {
-    return std::addressof(__n.__get_value());
+  _LIBCPP_HIDE_FROM_ABI static __container_value_type* __get_ptr(__container_value_type& __n) {
+    return std::addressof(__n);
   }
   _LIBCPP_HIDE_FROM_ABI static pair<key_type&&, mapped_type&&> __move(__node_value_type& __v) { return __v.__move(); }
 };
@@ -242,7 +258,7 @@ public:
 
   typedef typename __node_base_type::__next_pointer __next_pointer;
 
-  typedef _Tp __node_value_type;
+  using __node_value_type _LIBCPP_NODEBUG = __get_hash_node_value_type_t<_Tp>;
   typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer;
   typedef __rebind_pointer_t<_VoidPtr, const __node_value_type> __const_node_value_type_pointer;
 
@@ -667,14 +683,14 @@ int __diagnose_unordered_container_requirements(void*);
 template <class _Tp, class _Hash, class _Equal, class _Alloc>
 class __hash_table {
 public:
-  typedef _Tp value_type;
+  using value_type = __get_hash_node_value_type_t<_Tp>;
   typedef _Hash hasher;
   typedef _Equal key_equal;
   typedef _Alloc allocator_type;
 
 private:
   typedef allocator_traits<allocator_type> __alloc_traits;
-  typedef typename __make_hash_node_types<value_type, typename __alloc_traits::void_pointer>::type _NodeTypes;
+  typedef typename __make_hash_node_types<_Tp, typename __alloc_traits::void_pointer>::type _NodeTypes;
 
 public:
   typedef typename _NodeTypes::__node_value_type __node_value_type;
@@ -840,11 +856,27 @@ public:
     return __emplace_unique_key_args(_NodeTypes::__get_key(__x), std::move(__x));
   }
 
-  template <class _Pp, __enable_if_t<!__is_same_uncvref<_Pp, __container_value_type>::value, int> = 0>
+  template <class _Pp, __enable_if_t<!is_same<__remove_cvref_t<_Pp>, __container_value_type>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(_Pp&& __x) {
     return __emplace_unique(std::forward<_Pp>(__x));
   }
 
+  template <class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(value_type&& __value) {
+    using __key_type = typename _NodeTypes::key_type;
+
+    __node_holder __h = __construct_node(const_cast<__key_type&&>(__value.first), std::move(__value.second));
+    __node_insert_unique(__h.get());
+    __h.release();
+  }
+
+  template <class _ValueT = _Tp, __enable_if_t<!__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(value_type&& __value) {
+    __node_holder __h = __construct_node(std::move(__value));
+    __node_insert_unique(__h.get());
+    __h.release();
+  }
+
   template <class _Pp>
   _LIBCPP_HIDE_FROM_ABI iterator __insert_multi(_Pp&& __x) {
     return __emplace_multi(std::forward<_Pp>(__x));
@@ -855,6 +887,22 @@ public:
     return __emplace_hint_multi(__p, std::forward<_Pp>(__x));
   }
 
+  template <class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(value_type&& __value) {
+    using __key_type = typename _NodeTypes::key_type;
+
+    __node_holder __h = __construct_node(const_cast<__key_type&&>(__value.first), std::move(__value.second));
+    __node_insert_multi(__h.get());
+    __h.release();
+  }
+
+  template <class _ValueT = _Tp, __enable_if_t<!__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(value_type&& __value) {
+    __node_holder __h = __construct_node(std::move(__value));
+    __node_insert_multi(__h.get());
+    __h.release();
+  }
+
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __insert_unique(const __container_value_type& __x) {
     return __emplace_unique_key_args(_NodeTypes::__get_key(__x), __x);
   }
@@ -1020,6 +1068,21 @@ private:
   _LIBCPP_HIDE_FROM_ABI void __deallocate_node(__next_pointer __np) _NOEXCEPT;
   _LIBCPP_HIDE_FROM_ABI __next_pointer __detach() _NOEXCEPT;
 
+  template <class _From, class _ValueT = _Tp, __enable_if_t<__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __assign_value(__get_hash_node_value_type_t<_Tp>& __lhs, _From&& __rhs) {
+    using __key_type = typename _NodeTypes::key_type;
+
+    // This is technically UB, since the object was constructed as `const`.
+    // Clang doesn't optimize on this currently though.
+    const_cast<__key_type&>(__lhs.first) = const_cast<__copy_cvref_t<_From, __key_type>&&>(__rhs.first);
+    __lhs.second                         = std::forward<_From>(__rhs).second;
+  }
+
+  template <class _From, class _ValueT = _Tp, __enable_if_t<!__is_hash_value_type<_ValueT>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI void __assign_value(_Tp& __lhs, _From&& __rhs) {
+    __lhs = std::forward<_From>(__rhs);
+  }
+
   template <class, class, class, class, class>
   friend class unordered_map;
   template <class, class, class, class, class>
@@ -1216,8 +1279,8 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
 #endif // _LIBCPP_HAS_EXCEPTIONS
         const_iterator __i = __u.begin();
         while (__cache != nullptr && __u.size() != 0) {
-          __cache->__upcast()->__get_value() = std::move(__u.remove(__i++)->__get_value());
-          __next_pointer __next              = __cache->__next_;
+          __assign_value(__cache->__upcast()->__get_value(), std::move(__u.remove(__i++)->__get_value()));
+          __next_pointer __next = __cache->__next_;
           __node_insert_multi(__cache->__upcast());
           __cache = __next;
         }
@@ -1230,11 +1293,8 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__move_assign(__hash_table& __u,
       __deallocate_node(__cache);
     }
     const_iterator __i = __u.begin();
-    while (__u.size() != 0) {
-      __node_holder __h = __construct_node(_NodeTypes::__move(__u.remove(__i++)->__get_value()));
-      __node_insert_multi(__h.get());
-      __h.release();
-    }
+    while (__u.size() != 0)
+      __insert_multi_from_orphaned_node(std::move(__u.remove(__i++)->__get_value()));
   }
 }
 
@@ -1262,8 +1322,8 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_unique(_InputIterator __
     try {
 #endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __cache != nullptr && __first != __last; ++__first) {
-        __cache->__upcast()->__get_value() = *__first;
-        __next_pointer __next              = __cache->__next_;
+        __assign_value(__cache->__upcast()->__get_value(), *__first);
+        __next_pointer __next = __cache->__next_;
         __node_insert_unique(__cache->__upcast());
         __cache = __next;
       }
@@ -1294,7 +1354,7 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__assign_multi(_InputIterator __f
     try {
 #endif // _LIBCPP_HAS_EXCEPTIONS
       for (; __cache != nullptr && __first != __last; ++__first) {
-        __cache->__upcast()->__get_value() = *__first;
+        __assign_value(__cache->__upcast()->__get_value(), *__first);
         __next_pointer __next              = __cache->__next_;
         __node_insert_multi(__cache->__upcast());
         __cache = __next;
@@ -1771,9 +1831,9 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 template <class _Key>
 typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) {
-  size_t __hash  = hash_function()(__k);
   size_type __bc = bucket_count();
-  if (__bc != 0) {
+  if (__bc != 0 && size() != 0) {
+    size_t __hash       = hash_function()(__k);
     size_t __chash      = std::__constrain_hash(__hash, __bc);
     __next_pointer __nd = __bucket_list_[__chash];
     if (__nd != nullptr) {
@@ -1792,9 +1852,9 @@ template <class _Tp, class _Hash, class _Equal, class _Alloc>
 template <class _Key>
 typename __hash_table<_Tp, _Hash, _Equal, _Alloc>::const_iterator
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::find(const _Key& __k) const {
-  size_t __hash  = hash_function()(__k);
   size_type __bc = bucket_count();
-  if (__bc != 0) {
+  if (__bc != 0 && size() != 0) {
+    size_t __hash       = hash_function()(__k);
     size_t __chash      = std::__constrain_hash(__hash, __bc);
     __next_pointer __nd = __bucket_list_[__chash];
     if (__nd != nullptr) {
diff --git a/libcxx/include/__iterator/product_iterator.h b/libcxx/include/__iterator/product_iterator.h
new file mode 100644
index 000000000000..1db673bafd33
--- /dev/null
+++ b/libcxx/include/__iterator/product_iterator.h
@@ -0,0 +1,76 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
+#define _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
+
+// Product iterators are iterators that contain two or more underlying iterators.
+//
+// For example, std::flat_map stores its data into two separate containers, and its iterator
+// is a proxy over two separate underlying iterators. The concept of product iterators
+// allows algorithms to operate over these underlying iterators separately, opening the
+// door to various optimizations.
+//
+// If __product_iterator_traits can be instantiated, the following functions and associated types must be provided:
+// - static constexpr size_t Traits::__size
+//   The number of underlying iterators inside the product iterator.
+//
+// - template <size_t _N>
+//   static decltype(auto) Traits::__get_iterator_element(It&& __it)
+//   Returns the _Nth iterator element of the given product iterator.
+//
+// - template <class... _Iters>
+//   static _Iterator __make_product_iterator(_Iters&&...);
+//   Creates a product iterator from the given underlying iterators.
+
+#include <__config>
+#include <__cstddef/size_t.h>
+#include <__type_traits/enable_if.h>
+#include <__type_traits/integral_constant.h>
+#include <__utility/declval.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Iterator>
+struct __product_iterator_traits;
+/* exposition-only:
+{
+  static constexpr size_t __size = ...;
+
+  template <size_t _N, class _Iter>
+  static decltype(auto) __get_iterator_element(_Iter&&);
+
+  template <class... _Iters>
+  static _Iterator __make_product_iterator(_Iters&&...);
+};
+*/
+
+template <class _Tp, size_t = 0>
+struct __is_product_iterator : false_type {};
+
+template <class _Tp>
+struct __is_product_iterator<_Tp, sizeof(__product_iterator_traits<_Tp>) * 0> : true_type {};
+
+template <class _Tp, size_t _Size, class = void>
+struct __is_product_iterator_of_size : false_type {};
+
+template <class _Tp, size_t _Size>
+struct __is_product_iterator_of_size<_Tp, _Size, __enable_if_t<__product_iterator_traits<_Tp>::__size == _Size> >
+    : true_type {};
+
+template <class _Iterator, size_t _Nth>
+using __product_iterator_element_t _LIBCPP_NODEBUG =
+    decltype(__product_iterator_traits<_Iterator>::template __get_iterator_element<_Nth>(std::declval<_Iterator>()));
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ITERATOR_PRODUCT_ITERATOR_H
diff --git a/libcxx/include/__locale_dir/check_grouping.h b/libcxx/include/__locale_dir/check_grouping.h
new file mode 100644
index 000000000000..93e9e404bb5f
--- /dev/null
+++ b/libcxx/include/__locale_dir/check_grouping.h
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
+#define _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
+
+#include <__config>
+#include <__fwd/string.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+_LIBCPP_EXPORTED_FROM_ABI void
+__check_grouping(const string& __grouping, unsigned* __g, unsigned* __g_end, ios_base::iostate& __err);
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_CHECK_GROUPING_H
diff --git a/libcxx/include/__locale_dir/get_c_locale.h b/libcxx/include/__locale_dir/get_c_locale.h
new file mode 100644
index 000000000000..e8bac9a87095
--- /dev/null
+++ b/libcxx/include/__locale_dir/get_c_locale.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
+#define _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
+
+#include <__config>
+#include <__locale_dir/locale_base_api.h>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// FIXME: This should really be part of the locale base API
+
+#  if defined(__APPLE__) || defined(__FreeBSD__)
+#    define _LIBCPP_GET_C_LOCALE 0
+#  elif defined(__NetBSD__)
+#    define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
+#  else
+#    define _LIBCPP_GET_C_LOCALE __cloc()
+// Get the C locale object
+_LIBCPP_EXPORTED_FROM_ABI __locale::__locale_t __cloc();
+#    define __cloc_defined
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_GET_C_LOCALE_H
diff --git a/libcxx/include/__locale_dir/messages.h b/libcxx/include/__locale_dir/messages.h
new file mode 100644
index 000000000000..c04bf04025ff
--- /dev/null
+++ b/libcxx/include/__locale_dir/messages.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_MESSAGES_H
+#define _LIBCPP___LOCALE_DIR_MESSAGES_H
+
+#include <__config>
+#include <__iterator/back_insert_iterator.h>
+#include <__locale>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
+// Most unix variants have catopen.  These are the specific ones that don't.
+#    if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
+#      define _LIBCPP_HAS_CATOPEN 1
+#      include <nl_types.h>
+#    else
+#      define _LIBCPP_HAS_CATOPEN 0
+#    endif
+#  else
+#    define _LIBCPP_HAS_CATOPEN 0
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+class _LIBCPP_EXPORTED_FROM_ABI messages_base {
+public:
+  typedef intptr_t catalog;
+
+  _LIBCPP_HIDE_FROM_ABI messages_base() {}
+};
+
+template <class _CharT>
+class messages : public locale::facet, public messages_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI catalog open(const basic_string<char>& __nm, const locale& __loc) const {
+    return do_open(__nm, __loc);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI string_type get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+    return do_get(__c, __set, __msgid, __dflt);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void close(catalog __c) const { do_close(__c); }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages() override {}
+
+  virtual catalog do_open(const basic_string<char>&, const locale&) const;
+  virtual string_type do_get(catalog, int __set, int __msgid, const string_type& __dflt) const;
+  virtual void do_close(catalog) const;
+};
+
+template <class _CharT>
+locale::id messages<_CharT>::id;
+
+template <class _CharT>
+typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
+#  if _LIBCPP_HAS_CATOPEN
+  return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__nm;
+  return -1;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+typename messages<_CharT>::string_type
+messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
+#  if _LIBCPP_HAS_CATOPEN
+  string __ndflt;
+  __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
+      std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
+  nl_catd __cat = (nl_catd)__c;
+  static_assert(sizeof(catalog) >= sizeof(nl_catd), "Unexpected nl_catd type");
+  char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
+  string_type __w;
+  __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
+  return __w;
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+  (void)__set;
+  (void)__msgid;
+  return __dflt;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+template <class _CharT>
+void messages<_CharT>::do_close(catalog __c) const {
+#  if _LIBCPP_HAS_CATOPEN
+  catclose((nl_catd)__c);
+#  else  // !_LIBCPP_HAS_CATOPEN
+  (void)__c;
+#  endif // _LIBCPP_HAS_CATOPEN
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
+#  endif
+
+template <class _CharT>
+class messages_byname : public messages<_CharT> {
+public:
+  typedef messages_base::catalog catalog;
+  typedef basic_string<_CharT> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const char*, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const string&, size_t __refs = 0) : messages<_CharT>(__refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_MESSAGES_H
diff --git a/libcxx/include/__locale_dir/money.h b/libcxx/include/__locale_dir/money.h
new file mode 100644
index 000000000000..c1296665505e
--- /dev/null
+++ b/libcxx/include/__locale_dir/money.h
@@ -0,0 +1,873 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_MONEY_H
+#define _LIBCPP___LOCALE_DIR_MONEY_H
+
+#include <__algorithm/copy.h>
+#include <__algorithm/equal.h>
+#include <__algorithm/find.h>
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__locale>
+#include <__locale_dir/check_grouping.h>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/pad_and_output.h>
+#include <__memory/unique_ptr.h>
+#include <ios>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// money_base
+
+class _LIBCPP_EXPORTED_FROM_ABI money_base {
+public:
+  enum part { none, space, symbol, sign, value };
+  struct pattern {
+    char field[4];
+  };
+
+  _LIBCPP_HIDE_FROM_ABI money_base() {}
+};
+
+// moneypunct
+
+template <class _CharT, bool _International = false>
+class moneypunct : public locale::facet, public money_base {
+public:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
+  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
+  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
+  _LIBCPP_HIDE_FROM_ABI string_type curr_symbol() const { return do_curr_symbol(); }
+  _LIBCPP_HIDE_FROM_ABI string_type positive_sign() const { return do_positive_sign(); }
+  _LIBCPP_HIDE_FROM_ABI string_type negative_sign() const { return do_negative_sign(); }
+  _LIBCPP_HIDE_FROM_ABI int frac_digits() const { return do_frac_digits(); }
+  _LIBCPP_HIDE_FROM_ABI pattern pos_format() const { return do_pos_format(); }
+  _LIBCPP_HIDE_FROM_ABI pattern neg_format() const { return do_neg_format(); }
+
+  static locale::id id;
+  static const bool intl = _International;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct() override {}
+
+  virtual char_type do_decimal_point() const { return numeric_limits<char_type>::max(); }
+  virtual char_type do_thousands_sep() const { return numeric_limits<char_type>::max(); }
+  virtual string do_grouping() const { return string(); }
+  virtual string_type do_curr_symbol() const { return string_type(); }
+  virtual string_type do_positive_sign() const { return string_type(); }
+  virtual string_type do_negative_sign() const { return string_type(1, '-'); }
+  virtual int do_frac_digits() const { return 0; }
+  virtual pattern do_pos_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+  virtual pattern do_neg_format() const {
+    pattern __p = {{symbol, sign, none, value}};
+    return __p;
+  }
+};
+
+template <class _CharT, bool _International>
+locale::id moneypunct<_CharT, _International>::id;
+
+template <class _CharT, bool _International>
+const bool moneypunct<_CharT, _International>::intl;
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
+#  endif
+
+// moneypunct_byname
+
+template <class _CharT, bool _International = false>
+class moneypunct_byname : public moneypunct<_CharT, _International> {
+public:
+  typedef money_base::pattern pattern;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const char* __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const string& __nm, size_t __refs = 0)
+      : moneypunct<_CharT, _International>(__refs) {
+    init(__nm.c_str());
+  }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct_byname() override {}
+
+  char_type do_decimal_point() const override { return __decimal_point_; }
+  char_type do_thousands_sep() const override { return __thousands_sep_; }
+  string do_grouping() const override { return __grouping_; }
+  string_type do_curr_symbol() const override { return __curr_symbol_; }
+  string_type do_positive_sign() const override { return __positive_sign_; }
+  string_type do_negative_sign() const override { return __negative_sign_; }
+  int do_frac_digits() const override { return __frac_digits_; }
+  pattern do_pos_format() const override { return __pos_format_; }
+  pattern do_neg_format() const override { return __neg_format_; }
+
+private:
+  char_type __decimal_point_;
+  char_type __thousands_sep_;
+  string __grouping_;
+  string_type __curr_symbol_;
+  string_type __positive_sign_;
+  string_type __negative_sign_;
+  int __frac_digits_;
+  pattern __pos_format_;
+  pattern __neg_format_;
+
+  void init(const char*);
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
+
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
+template <>
+_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
+#  endif
+
+// money_get
+
+template <class _CharT>
+class __money_get {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_get() {}
+
+  static void __gather_info(
+      bool __intl,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __psn,
+      string_type& __nsn,
+      int& __fd);
+};
+
+template <class _CharT>
+void __money_get<_CharT>::__gather_info(
+    bool __intl,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __psn,
+    string_type& __nsn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    __pat                                   = __mp.neg_format();
+    __nsn                                   = __mp.negative_sign();
+    __psn                                   = __mp.positive_sign();
+    __dp                                    = __mp.decimal_point();
+    __ts                                    = __mp.thousands_sep();
+    __grp                                   = __mp.grouping();
+    __sym                                   = __mp.curr_symbol();
+    __fd                                    = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    __pat                                    = __mp.neg_format();
+    __nsn                                    = __mp.negative_sign();
+    __psn                                    = __mp.positive_sign();
+    __dp                                     = __mp.decimal_point();
+    __ts                                     = __mp.thousands_sep();
+    __grp                                    = __mp.grouping();
+    __sym                                    = __mp.curr_symbol();
+    __fd                                     = __mp.frac_digits();
+  }
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class money_get : public locale::facet, private __money_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+    return do_get(__b, __e, __intl, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_get() override {}
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const;
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const;
+
+private:
+  static bool __do_get(
+      iter_type& __b,
+      iter_type __e,
+      bool __intl,
+      const locale& __loc,
+      ios_base::fmtflags __flags,
+      ios_base::iostate& __err,
+      bool& __neg,
+      const ctype<char_type>& __ct,
+      unique_ptr<char_type, void (*)(void*)>& __wb,
+      char_type*& __wn,
+      char_type* __we);
+};
+
+template <class _CharT, class _InputIterator>
+locale::id money_get<_CharT, _InputIterator>::id;
+
+_LIBCPP_EXPORTED_FROM_ABI void __do_nothing(void*);
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void __double_or_nothing(unique_ptr<_Tp, void (*)(void*)>& __b, _Tp*& __n, _Tp*& __e) {
+  bool __owns      = __b.get_deleter() != __do_nothing;
+  size_t __cur_cap = static_cast<size_t>(__e - __b.get()) * sizeof(_Tp);
+  size_t __new_cap = __cur_cap < numeric_limits<size_t>::max() / 2 ? 2 * __cur_cap : numeric_limits<size_t>::max();
+  if (__new_cap == 0)
+    __new_cap = sizeof(_Tp);
+  size_t __n_off = static_cast<size_t>(__n - __b.get());
+  _Tp* __t       = (_Tp*)std::realloc(__owns ? __b.get() : 0, __new_cap);
+  if (__t == 0)
+    std::__throw_bad_alloc();
+  if (__owns)
+    __b.release();
+  else
+    std::memcpy(__t, __b.get(), __cur_cap);
+  __b = unique_ptr<_Tp, void (*)(void*)>(__t, free);
+  __new_cap /= sizeof(_Tp);
+  __n = __b.get() + __n_off;
+  __e = __b.get() + __new_cap;
+}
+
+// true == success
+template <class _CharT, class _InputIterator>
+bool money_get<_CharT, _InputIterator>::__do_get(
+    iter_type& __b,
+    iter_type __e,
+    bool __intl,
+    const locale& __loc,
+    ios_base::fmtflags __flags,
+    ios_base::iostate& __err,
+    bool& __neg,
+    const ctype<char_type>& __ct,
+    unique_ptr<char_type, void (*)(void*)>& __wb,
+    char_type*& __wn,
+    char_type* __we) {
+  if (__b == __e) {
+    __err |= ios_base::failbit;
+    return false;
+  }
+  const unsigned __bz = 100;
+  unsigned __gbuf[__bz];
+  unique_ptr<unsigned, void (*)(void*)> __gb(__gbuf, __do_nothing);
+  unsigned* __gn = __gb.get();
+  unsigned* __ge = __gn + __bz;
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __psn;
+  string_type __nsn;
+  // Capture the spaces read into money_base::{space,none} so they
+  // can be compared to initial spaces in __sym.
+  string_type __spaces;
+  int __fd;
+  __money_get<_CharT>::__gather_info(__intl, __loc, __pat, __dp, __ts, __grp, __sym, __psn, __nsn, __fd);
+  const string_type* __trailing_sign = 0;
+  __wn                               = __wb.get();
+  for (unsigned __p = 0; __p < 4 && __b != __e; ++__p) {
+    switch (__pat.field[__p]) {
+    case money_base::space:
+      if (__p != 3) {
+        if (__ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+        else {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+      [[__fallthrough__]];
+    case money_base::none:
+      if (__p != 3) {
+        while (__b != __e && __ct.is(ctype_base::space, *__b))
+          __spaces.push_back(*__b++);
+      }
+      break;
+    case money_base::sign:
+      if (__psn.size() > 0 && *__b == __psn[0]) {
+        ++__b;
+        __neg = false;
+        if (__psn.size() > 1)
+          __trailing_sign = std::addressof(__psn);
+        break;
+      }
+      if (__nsn.size() > 0 && *__b == __nsn[0]) {
+        ++__b;
+        __neg = true;
+        if (__nsn.size() > 1)
+          __trailing_sign = std::addressof(__nsn);
+        break;
+      }
+      if (__psn.size() > 0 && __nsn.size() > 0) { // sign is required
+        __err |= ios_base::failbit;
+        return false;
+      }
+      if (__psn.size() == 0 && __nsn.size() == 0)
+        // locale has no way of specifying a sign. Use the initial value of __neg as a default
+        break;
+      __neg = (__nsn.size() == 0);
+      break;
+    case money_base::symbol: {
+      bool __more_needed =
+          __trailing_sign || (__p < 2) || (__p == 2 && __pat.field[3] != static_cast<char>(money_base::none));
+      bool __sb = (__flags & ios_base::showbase) != 0;
+      if (__sb || __more_needed) {
+        typename string_type::const_iterator __sym_space_end = __sym.begin();
+        if (__p > 0 && (__pat.field[__p - 1] == money_base::none || __pat.field[__p - 1] == money_base::space)) {
+          // Match spaces we've already read against spaces at
+          // the beginning of __sym.
+          while (__sym_space_end != __sym.end() && __ct.is(ctype_base::space, *__sym_space_end))
+            ++__sym_space_end;
+          const size_t __num_spaces = __sym_space_end - __sym.begin();
+          if (__num_spaces > __spaces.size() ||
+              !std::equal(__spaces.end() - __num_spaces, __spaces.end(), __sym.begin())) {
+            // No match. Put __sym_space_end back at the
+            // beginning of __sym, which will prevent a
+            // match in the next loop.
+            __sym_space_end = __sym.begin();
+          }
+        }
+        typename string_type::const_iterator __sym_curr_char = __sym_space_end;
+        while (__sym_curr_char != __sym.end() && __b != __e && *__b == *__sym_curr_char) {
+          ++__b;
+          ++__sym_curr_char;
+        }
+        if (__sb && __sym_curr_char != __sym.end()) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+      }
+    } break;
+    case money_base::value: {
+      unsigned __ng = 0;
+      for (; __b != __e; ++__b) {
+        char_type __c = *__b;
+        if (__ct.is(ctype_base::digit, __c)) {
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = __c;
+          ++__ng;
+        } else if (__grp.size() > 0 && __ng > 0 && __c == __ts) {
+          if (__gn == __ge)
+            std::__double_or_nothing(__gb, __gn, __ge);
+          *__gn++ = __ng;
+          __ng    = 0;
+        } else
+          break;
+      }
+      if (__gb.get() != __gn && __ng > 0) {
+        if (__gn == __ge)
+          std::__double_or_nothing(__gb, __gn, __ge);
+        *__gn++ = __ng;
+      }
+      if (__fd > 0) {
+        if (__b == __e || *__b != __dp) {
+          __err |= ios_base::failbit;
+          return false;
+        }
+        for (++__b; __fd > 0; --__fd, ++__b) {
+          if (__b == __e || !__ct.is(ctype_base::digit, *__b)) {
+            __err |= ios_base::failbit;
+            return false;
+          }
+          if (__wn == __we)
+            std::__double_or_nothing(__wb, __wn, __we);
+          *__wn++ = *__b;
+        }
+      }
+      if (__wn == __wb.get()) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    } break;
+    }
+  }
+  if (__trailing_sign) {
+    for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
+      if (__b == __e || *__b != (*__trailing_sign)[__i]) {
+        __err |= ios_base::failbit;
+        return false;
+      }
+    }
+  }
+  if (__gb.get() != __gn) {
+    ios_base::iostate __et = ios_base::goodbit;
+    __check_grouping(__grp, __gb.get(), __gn, __et);
+    if (__et) {
+      __err |= ios_base::failbit;
+      return false;
+    }
+  }
+  return true;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    const char __src[] = "0123456789";
+    char_type __atoms[sizeof(__src) - 1];
+    __ct.widen(__src, __src + (sizeof(__src) - 1), __atoms);
+    char __nbuf[__bz];
+    char* __nc          = __nbuf;
+    const char* __nc_in = __nc;
+    unique_ptr<char, void (*)(void*)> __h(nullptr, free);
+    if (__wn - __wb.get() > __bz - 2) {
+      __h.reset((char*)malloc(static_cast<size_t>(__wn - __wb.get() + 2)));
+      if (__h.get() == nullptr)
+        std::__throw_bad_alloc();
+      __nc    = __h.get();
+      __nc_in = __nc;
+    }
+    if (__neg)
+      *__nc++ = '-';
+    for (const char_type* __w = __wb.get(); __w < __wn; ++__w, ++__nc)
+      *__nc = __src[std::find(__atoms, std::end(__atoms), *__w) - __atoms];
+    *__nc = char();
+    if (sscanf(__nc_in, "%Lf", &__v) != 1)
+      std::__throw_runtime_error("money_get error");
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator money_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
+  const int __bz = 100;
+  char_type __wbuf[__bz];
+  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
+  char_type* __wn;
+  char_type* __we              = __wbuf + __bz;
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = false;
+  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
+    __v.clear();
+    if (__neg)
+      __v.push_back(__ct.widen('-'));
+    char_type __z = __ct.widen('0');
+    char_type* __w;
+    for (__w = __wb.get(); __w < __wn - 1; ++__w)
+      if (*__w != __z)
+        break;
+    __v.append(__w, __wn);
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
+#  endif
+
+// money_put
+
+template <class _CharT>
+class __money_put {
+protected:
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI __money_put() {}
+
+  static void __gather_info(
+      bool __intl,
+      bool __neg,
+      const locale& __loc,
+      money_base::pattern& __pat,
+      char_type& __dp,
+      char_type& __ts,
+      string& __grp,
+      string_type& __sym,
+      string_type& __sn,
+      int& __fd);
+  static void __format(
+      char_type* __mb,
+      char_type*& __mi,
+      char_type*& __me,
+      ios_base::fmtflags __flags,
+      const char_type* __db,
+      const char_type* __de,
+      const ctype<char_type>& __ct,
+      bool __neg,
+      const money_base::pattern& __pat,
+      char_type __dp,
+      char_type __ts,
+      const string& __grp,
+      const string_type& __sym,
+      const string_type& __sn,
+      int __fd);
+};
+
+template <class _CharT>
+void __money_put<_CharT>::__gather_info(
+    bool __intl,
+    bool __neg,
+    const locale& __loc,
+    money_base::pattern& __pat,
+    char_type& __dp,
+    char_type& __ts,
+    string& __grp,
+    string_type& __sym,
+    string_type& __sn,
+    int& __fd) {
+  if (__intl) {
+    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  } else {
+    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
+    if (__neg) {
+      __pat = __mp.neg_format();
+      __sn  = __mp.negative_sign();
+    } else {
+      __pat = __mp.pos_format();
+      __sn  = __mp.positive_sign();
+    }
+    __dp  = __mp.decimal_point();
+    __ts  = __mp.thousands_sep();
+    __grp = __mp.grouping();
+    __sym = __mp.curr_symbol();
+    __fd  = __mp.frac_digits();
+  }
+}
+
+template <class _CharT>
+void __money_put<_CharT>::__format(
+    char_type* __mb,
+    char_type*& __mi,
+    char_type*& __me,
+    ios_base::fmtflags __flags,
+    const char_type* __db,
+    const char_type* __de,
+    const ctype<char_type>& __ct,
+    bool __neg,
+    const money_base::pattern& __pat,
+    char_type __dp,
+    char_type __ts,
+    const string& __grp,
+    const string_type& __sym,
+    const string_type& __sn,
+    int __fd) {
+  __me = __mb;
+  for (char __p : __pat.field) {
+    switch (__p) {
+    case money_base::none:
+      __mi = __me;
+      break;
+    case money_base::space:
+      __mi    = __me;
+      *__me++ = __ct.widen(' ');
+      break;
+    case money_base::sign:
+      if (!__sn.empty())
+        *__me++ = __sn[0];
+      break;
+    case money_base::symbol:
+      if (!__sym.empty() && (__flags & ios_base::showbase))
+        __me = std::copy(__sym.begin(), __sym.end(), __me);
+      break;
+    case money_base::value: {
+      // remember start of value so we can reverse it
+      char_type* __t = __me;
+      // find beginning of digits
+      if (__neg)
+        ++__db;
+      // find end of digits
+      const char_type* __d;
+      for (__d = __db; __d < __de; ++__d)
+        if (!__ct.is(ctype_base::digit, *__d))
+          break;
+      // print fractional part
+      if (__fd > 0) {
+        int __f;
+        for (__f = __fd; __d > __db && __f > 0; --__f)
+          *__me++ = *--__d;
+        char_type __z = __f > 0 ? __ct.widen('0') : char_type();
+        for (; __f > 0; --__f)
+          *__me++ = __z;
+        *__me++ = __dp;
+      }
+      // print units part
+      if (__d == __db) {
+        *__me++ = __ct.widen('0');
+      } else {
+        unsigned __ng = 0;
+        unsigned __ig = 0;
+        unsigned __gl = __grp.empty() ? numeric_limits<unsigned>::max() : static_cast<unsigned>(__grp[__ig]);
+        while (__d != __db) {
+          if (__ng == __gl) {
+            *__me++ = __ts;
+            __ng    = 0;
+            if (++__ig < __grp.size())
+              __gl = __grp[__ig] == numeric_limits<char>::max()
+                       ? numeric_limits<unsigned>::max()
+                       : static_cast<unsigned>(__grp[__ig]);
+          }
+          *__me++ = *--__d;
+          ++__ng;
+        }
+      }
+      // reverse it
+      std::reverse(__t, __me);
+    } break;
+    }
+  }
+  // print rest of sign, if any
+  if (__sn.size() > 1)
+    __me = std::copy(__sn.begin() + 1, __sn.end(), __me);
+  // set alignment
+  if ((__flags & ios_base::adjustfield) == ios_base::left)
+    __mi = __me;
+  else if ((__flags & ios_base::adjustfield) != ios_base::internal)
+    __mi = __mb;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class money_put : public locale::facet, private __money_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit money_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+    return do_put(__s, __intl, __iob, __fl, __units);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+    return do_put(__s, __intl, __iob, __fl, __digits);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_put() override {}
+
+  virtual iter_type do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const;
+  virtual iter_type
+  do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id money_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
+  // convert to char
+  const size_t __bs = 100;
+  char __buf[__bs];
+  char* __bb = __buf;
+  char_type __digits[__bs];
+  char_type* __db = __digits;
+  int __n         = snprintf(__bb, __bs, "%.0Lf", __units);
+  unique_ptr<char, void (*)(void*)> __hn(nullptr, free);
+  unique_ptr<char_type, void (*)(void*)> __hd(0, free);
+  // secure memory for digit storage
+  if (static_cast<size_t>(__n) > __bs - 1) {
+    __n = __locale::__asprintf(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
+    if (__n == -1)
+      std::__throw_bad_alloc();
+    __hn.reset(__bb);
+    __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
+    if (__hd == nullptr)
+      std::__throw_bad_alloc();
+    __db = __hd.get();
+  }
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  __ct.widen(__bb, __bb + __n, __db);
+  bool __neg = __n > 0 && __bb[0] == '-';
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[__bs];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __hw(0, free);
+  size_t __exn = __n > __fd ? (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() +
+                                  static_cast<size_t>(__fd) + 1
+                            : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > __bs) {
+    __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __hw.get();
+    if (__mb == 0)
+      std::__throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb, __mi, __me, __iob.flags(), __db, __db + __n, __ct, __neg, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
+  // gather info
+  locale __loc                 = __iob.getloc();
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
+  bool __neg                   = __digits.size() > 0 && __digits[0] == __ct.widen('-');
+  money_base::pattern __pat;
+  char_type __dp;
+  char_type __ts;
+  string __grp;
+  string_type __sym;
+  string_type __sn;
+  int __fd;
+  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
+  // secure memory for formatting
+  char_type __mbuf[100];
+  char_type* __mb = __mbuf;
+  unique_ptr<char_type, void (*)(void*)> __h(0, free);
+  size_t __exn =
+      static_cast<int>(__digits.size()) > __fd
+          ? (__digits.size() - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() + static_cast<size_t>(__fd) +
+                1
+          : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
+  if (__exn > 100) {
+    __h.reset((char_type*)malloc(__exn * sizeof(char_type)));
+    __mb = __h.get();
+    if (__mb == 0)
+      std::__throw_bad_alloc();
+  }
+  // format
+  char_type* __mi;
+  char_type* __me;
+  this->__format(
+      __mb,
+      __mi,
+      __me,
+      __iob.flags(),
+      __digits.data(),
+      __digits.data() + __digits.size(),
+      __ct,
+      __neg,
+      __pat,
+      __dp,
+      __ts,
+      __grp,
+      __sym,
+      __sn,
+      __fd);
+  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_MONEY_H
diff --git a/libcxx/include/__locale_dir/num.h b/libcxx/include/__locale_dir/num.h
new file mode 100644
index 000000000000..7ca8ffe34895
--- /dev/null
+++ b/libcxx/include/__locale_dir/num.h
@@ -0,0 +1,1072 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_NUM_H
+#define _LIBCPP___LOCALE_DIR_NUM_H
+
+#include <__algorithm/find.h>
+#include <__algorithm/reverse.h>
+#include <__charconv/to_chars_integral.h>
+#include <__charconv/traits.h>
+#include <__config>
+#include <__iterator/istreambuf_iterator.h>
+#include <__iterator/ostreambuf_iterator.h>
+#include <__locale_dir/check_grouping.h>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/pad_and_output.h>
+#include <__locale_dir/scan_keyword.h>
+#include <__memory/unique_ptr.h>
+#include <__system_error/errc.h>
+#include <cerrno>
+#include <ios>
+#include <streambuf>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+// TODO: Properly qualify calls now that the locale base API defines functions instead of macros
+// NOLINTBEGIN(libcpp-robust-against-adl)
+
+_LIBCPP_PUSH_MACROS
+#  include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
+  static const int __num_get_buf_sz = 40;
+
+  static int __get_base(ios_base&);
+  static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
+  // count of leading characters in __src used for parsing integers ("012..X+-")
+  static const size_t __int_chr_cnt = 26;
+  // count of leading characters in __src used for parsing floating-point values ("012..-pP")
+  static const size_t __fp_chr_cnt = 28;
+};
+
+template <class _CharT>
+struct __num_get : protected __num_get_base {
+  static string __stage2_float_prep(ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep);
+
+  static int __stage2_float_loop(
+      _CharT __ct,
+      bool& __in_units,
+      char& __exp,
+      char* __a,
+      char*& __a_end,
+      _CharT __decimal_point,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      unsigned& __dc,
+      _CharT* __atoms);
+
+  [[__deprecated__("This exists only for ABI compatibility")]] static string
+  __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
+  static int __stage2_int_loop(
+      _CharT __ct,
+      int __base,
+      char* __a,
+      char*& __a_end,
+      unsigned& __dc,
+      _CharT __thousands_sep,
+      const string& __grouping,
+      unsigned* __g,
+      unsigned*& __g_end,
+      _CharT* __atoms);
+
+  _LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
+    locale __loc                 = __iob.getloc();
+    const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
+    __thousands_sep              = __np.thousands_sep();
+    return __np.grouping();
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
+    return __do_widen_p(__iob, __atoms);
+  }
+
+private:
+  template <typename _Tp>
+  _LIBCPP_HIDE_FROM_ABI const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
+    locale __loc = __iob.getloc();
+    use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
+    return __atoms;
+  }
+
+  _LIBCPP_HIDE_FROM_ABI const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
+    (void)__iob;
+    (void)__atoms;
+    return __src;
+  }
+};
+
+template <class _CharT>
+string __num_get<_CharT>::__stage2_float_prep(
+    ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
+  locale __loc = __iob.getloc();
+  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __fp_chr_cnt, __atoms);
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
+  __decimal_point              = __np.decimal_point();
+  __thousands_sep              = __np.thousands_sep();
+  return __np.grouping();
+}
+
+template <class _CharT>
+int __num_get<_CharT>::__stage2_int_loop(
+    _CharT __ct,
+    int __base,
+    char* __a,
+    char*& __a_end,
+    unsigned& __dc,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    _CharT* __atoms) {
+  if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
+    *__a_end++ = __ct == __atoms[24] ? '+' : '-';
+    __dc       = 0;
+    return 0;
+  }
+  if (__grouping.size() != 0 && __ct == __thousands_sep) {
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
+  if (__f >= 24)
+    return -1;
+  switch (__base) {
+  case 8:
+  case 10:
+    if (__f >= __base)
+      return -1;
+    break;
+  case 16:
+    if (__f < 22)
+      break;
+    if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
+      __dc       = 0;
+      *__a_end++ = __src[__f];
+      return 0;
+    }
+    return -1;
+  }
+  *__a_end++ = __src[__f];
+  ++__dc;
+  return 0;
+}
+
+template <class _CharT>
+int __num_get<_CharT>::__stage2_float_loop(
+    _CharT __ct,
+    bool& __in_units,
+    char& __exp,
+    char* __a,
+    char*& __a_end,
+    _CharT __decimal_point,
+    _CharT __thousands_sep,
+    const string& __grouping,
+    unsigned* __g,
+    unsigned*& __g_end,
+    unsigned& __dc,
+    _CharT* __atoms) {
+  if (__ct == __decimal_point) {
+    if (!__in_units)
+      return -1;
+    __in_units = false;
+    *__a_end++ = '.';
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+      *__g_end++ = __dc;
+    return 0;
+  }
+  if (__ct == __thousands_sep && __grouping.size() != 0) {
+    if (!__in_units)
+      return -1;
+    if (__g_end - __g < __num_get_buf_sz) {
+      *__g_end++ = __dc;
+      __dc       = 0;
+    }
+    return 0;
+  }
+  ptrdiff_t __f = std::find(__atoms, __atoms + __num_get_base::__fp_chr_cnt, __ct) - __atoms;
+  if (__f >= static_cast<ptrdiff_t>(__num_get_base::__fp_chr_cnt))
+    return -1;
+  char __x = __src[__f];
+  if (__x == '-' || __x == '+') {
+    if (__a_end == __a || (std::toupper(__a_end[-1]) == std::toupper(__exp))) {
+      *__a_end++ = __x;
+      return 0;
+    }
+    return -1;
+  }
+  if (__x == 'x' || __x == 'X')
+    __exp = 'P';
+  else if (std::toupper(__x) == __exp) {
+    __exp = std::tolower(__exp);
+    if (__in_units) {
+      __in_units = false;
+      if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
+        *__g_end++ = __dc;
+    }
+  }
+  *__a_end++ = __x;
+  if (__f >= 22)
+    return 0;
+  ++__dc;
+  return 0;
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
+#  endif
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
+  return __locale::__strtof(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
+  return __locale::__strtod(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <>
+inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
+  return __locale::__strtold(__a, __p2, _LIBCPP_GET_C_LOCALE);
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end, ios_base::iostate& __err) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    _Tp __ld                                                     = std::__do_strtod<_Tp>(__a, &__p2);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE)
+      __err = ios_base::failbit;
+    return __ld;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || __ll < numeric_limits<_Tp>::min() || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      if (__ll > 0)
+        return numeric_limits<_Tp>::max();
+      else
+        return numeric_limits<_Tp>::min();
+    }
+    return static_cast<_Tp>(__ll);
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _Tp
+__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
+  if (__a != __a_end) {
+    const bool __negate = *__a == '-';
+    if (__negate && ++__a == __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    }
+    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
+    errno                                                     = 0;
+    char* __p2;
+    unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
+    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
+    if (__current_errno == 0)
+      errno = __save_errno;
+    if (__p2 != __a_end) {
+      __err = ios_base::failbit;
+      return 0;
+    } else if (__current_errno == ERANGE || numeric_limits<_Tp>::max() < __ll) {
+      __err = ios_base::failbit;
+      return numeric_limits<_Tp>::max();
+    }
+    _Tp __res = static_cast<_Tp>(__ll);
+    if (__negate)
+      __res = -__res;
+    return __res;
+  }
+  __err = ios_base::failbit;
+  return 0;
+}
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class num_get : public locale::facet, private __num_get<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+    return do_get(__b, __e, __iob, __err, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_get() override {}
+
+  template <class _Fp>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_floating_point(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const {
+    // Stage 1, nothing to do
+    // Stage 2
+    char_type __atoms[__num_get_base::__fp_chr_cnt];
+    char_type __decimal_point;
+    char_type __thousands_sep;
+    string __grouping = this->__stage2_float_prep(__iob, __atoms, __decimal_point, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end        = __g;
+    unsigned __dc            = 0;
+    bool __in_units          = true;
+    char __exp               = 'E';
+    bool __is_leading_parsed = false;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_float_loop(
+              *__b,
+              __in_units,
+              __exp,
+              __a,
+              __a_end,
+              __decimal_point,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              __dc,
+              __atoms))
+        break;
+
+      // the leading character excluding the sign must be a decimal digit
+      if (!__is_leading_parsed) {
+        if (__a_end - __a >= 1 && __a[0] != '-' && __a[0] != '+') {
+          if (('0' <= __a[0] && __a[0] <= '9') || __a[0] == '.')
+            __is_leading_parsed = true;
+          else
+            break;
+        } else if (__a_end - __a >= 2 && (__a[0] == '-' || __a[0] == '+')) {
+          if (('0' <= __a[1] && __a[1] <= '9') || __a[1] == '.')
+            __is_leading_parsed = true;
+          else
+            break;
+        }
+      }
+    }
+    if (__grouping.size() != 0 && __in_units && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_float<_Fp>(__a, __a_end, __err);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  template <class _Signed>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
+    // Stage 1
+    int __base = this->__get_base(__iob);
+    // Stage 2
+    char_type __thousands_sep;
+    const int __atoms_size = __num_get_base::__int_chr_cnt;
+    char_type __atoms1[__atoms_size];
+    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end = __g;
+    unsigned __dc     = 0;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_int_loop(
+              *__b,
+              __base,
+              __a,
+              __a_end,
+              __dc,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              const_cast<char_type*>(__atoms)))
+        break;
+    }
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  template <class _Unsigned>
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  __do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
+    // Stage 1
+    int __base = this->__get_base(__iob);
+    // Stage 2
+    char_type __thousands_sep;
+    const int __atoms_size = __num_get_base::__int_chr_cnt;
+    char_type __atoms1[__atoms_size];
+    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
+    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
+    string __buf;
+    __buf.resize(__buf.capacity());
+    char* __a     = &__buf[0];
+    char* __a_end = __a;
+    unsigned __g[__num_get_base::__num_get_buf_sz];
+    unsigned* __g_end = __g;
+    unsigned __dc     = 0;
+    for (; __b != __e; ++__b) {
+      if (__a_end == __a + __buf.size()) {
+        size_t __tmp = __buf.size();
+        __buf.resize(2 * __buf.size());
+        __buf.resize(__buf.capacity());
+        __a     = &__buf[0];
+        __a_end = __a + __tmp;
+      }
+      if (this->__stage2_int_loop(
+              *__b,
+              __base,
+              __a,
+              __a_end,
+              __dc,
+              __thousands_sep,
+              __grouping,
+              __g,
+              __g_end,
+              const_cast<char_type*>(__atoms)))
+        break;
+    }
+    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
+      *__g_end++ = __dc;
+    // Stage 3
+    __v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
+    // Digit grouping checked
+    __check_grouping(__grouping, __g, __g_end, __err);
+    // EOF checked
+    if (__b == __e)
+      __err |= ios_base::eofbit;
+    return __b;
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
+    return this->__do_get_signed(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
+    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type
+  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
+    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
+  }
+
+  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id num_get<_CharT, _InputIterator>::id;
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0) {
+    long __lv = -1;
+    __b       = do_get(__b, __e, __iob, __err, __lv);
+    switch (__lv) {
+    case 0:
+      __v = false;
+      break;
+    case 1:
+      __v = true;
+      break;
+    default:
+      __v   = true;
+      __err = ios_base::failbit;
+      break;
+    }
+    return __b;
+  }
+  const ctype<_CharT>& __ct    = std::use_facet<ctype<_CharT> >(__iob.getloc());
+  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__iob.getloc());
+  typedef typename numpunct<_CharT>::string_type string_type;
+  const string_type __names[2] = {__np.truename(), __np.falsename()};
+  const string_type* __i       = std::__scan_keyword(__b, __e, __names, __names + 2, __ct, __err);
+  __v                          = __i == __names;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator num_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
+  // Stage 1
+  int __base = 16;
+  // Stage 2
+  char_type __atoms[__num_get_base::__int_chr_cnt];
+  char_type __thousands_sep = char_type();
+  string __grouping;
+  std::use_facet<ctype<_CharT> >(__iob.getloc())
+      .widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
+  string __buf;
+  __buf.resize(__buf.capacity());
+  char* __a     = &__buf[0];
+  char* __a_end = __a;
+  unsigned __g[__num_get_base::__num_get_buf_sz];
+  unsigned* __g_end = __g;
+  unsigned __dc     = 0;
+  for (; __b != __e; ++__b) {
+    if (__a_end == __a + __buf.size()) {
+      size_t __tmp = __buf.size();
+      __buf.resize(2 * __buf.size());
+      __buf.resize(__buf.capacity());
+      __a     = &__buf[0];
+      __a_end = __a + __tmp;
+    }
+    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
+      break;
+  }
+  // Stage 3
+  __buf.resize(__a_end - __a);
+  if (__locale::__sscanf(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
+    __err = ios_base::failbit;
+  // EOF checked
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
+#  endif
+
+struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
+protected:
+  static void __format_int(char* __fmt, const char* __len, bool __signd, ios_base::fmtflags __flags);
+  static bool __format_float(char* __fmt, const char* __len, ios_base::fmtflags __flags);
+  static char* __identify_padding(char* __nb, char* __ne, const ios_base& __iob);
+};
+
+template <class _CharT>
+struct __num_put : protected __num_put_base {
+  static void __widen_and_group_int(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+  static void __widen_and_group_float(
+      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
+};
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_int(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  if (__grouping.empty()) {
+    __ct.widen(__nb, __ne, __ob);
+    __oe = __ob + (__ne - __nb);
+  } else {
+    __oe       = __ob;
+    char* __nf = __nb;
+    if (*__nf == '-' || *__nf == '+')
+      *__oe++ = __ct.widen(*__nf++);
+    if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+      *__oe++ = __ct.widen(*__nf++);
+      *__oe++ = __ct.widen(*__nf++);
+    }
+    std::reverse(__nf, __ne);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ne; ++__p) {
+      if (static_cast<unsigned>(__grouping[__dg]) > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+template <class _CharT>
+void __num_put<_CharT>::__widen_and_group_float(
+    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
+  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
+  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
+  string __grouping             = __npt.grouping();
+  __oe                          = __ob;
+  char* __nf                    = __nb;
+  if (*__nf == '-' || *__nf == '+')
+    *__oe++ = __ct.widen(*__nf++);
+  char* __ns;
+  if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
+    *__oe++ = __ct.widen(*__nf++);
+    *__oe++ = __ct.widen(*__nf++);
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!__locale::__isxdigit(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  } else {
+    for (__ns = __nf; __ns < __ne; ++__ns)
+      if (!__locale::__isdigit(*__ns, _LIBCPP_GET_C_LOCALE))
+        break;
+  }
+  if (__grouping.empty()) {
+    __ct.widen(__nf, __ns, __oe);
+    __oe += __ns - __nf;
+  } else {
+    std::reverse(__nf, __ns);
+    _CharT __thousands_sep = __npt.thousands_sep();
+    unsigned __dc          = 0;
+    unsigned __dg          = 0;
+    for (char* __p = __nf; __p < __ns; ++__p) {
+      if (__grouping[__dg] > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
+        *__oe++ = __thousands_sep;
+        __dc    = 0;
+        if (__dg < __grouping.size() - 1)
+          ++__dg;
+      }
+      *__oe++ = __ct.widen(*__p);
+      ++__dc;
+    }
+    std::reverse(__ob + (__nf - __nb), __oe);
+  }
+  for (__nf = __ns; __nf < __ne; ++__nf) {
+    if (*__nf == '.') {
+      *__oe++ = __npt.decimal_point();
+      ++__nf;
+      break;
+    } else
+      *__oe++ = __ct.widen(*__nf);
+  }
+  __ct.widen(__nf, __ne, __oe);
+  __oe += __ne - __nf;
+  if (__np == __ne)
+    __op = __oe;
+  else
+    __op = __ob + (__np - __nb);
+}
+
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class num_put : public locale::facet, private __num_put<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit num_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+    return do_put(__s, __iob, __fl, __v);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_put() override {}
+
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const;
+  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const;
+
+  template <class _Integral>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_integral(iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const;
+
+  template <class _Float>
+  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
+  __do_put_floating_point(iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const;
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id num_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
+  if ((__iob.flags() & ios_base::boolalpha) == 0)
+    return do_put(__s, __iob, __fl, (unsigned long)__v);
+  const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
+  typedef typename numpunct<char_type>::string_type string_type;
+  string_type __nm = __v ? __np.truename() : __np.falsename();
+  for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
+    *__s = *__i;
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Integral>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_integral(
+    iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const {
+  // Stage 1 - Get number in narrow char
+
+  // Worst case is octal, with showbase enabled. Note that octal is always
+  // printed as an unsigned value.
+  using _Unsigned = typename make_unsigned<_Integral>::type;
+  _LIBCPP_CONSTEXPR const unsigned __buffer_size =
+      (numeric_limits<_Unsigned>::digits / 3)          // 1 char per 3 bits
+      + ((numeric_limits<_Unsigned>::digits % 3) != 0) // round up
+      + 2;                                             // base prefix + terminating null character
+
+  char __char_buffer[__buffer_size];
+  char* __buffer_ptr = __char_buffer;
+
+  auto __flags = __iob.flags();
+
+  auto __basefield = (__flags & ios_base::basefield);
+
+  // Extract base
+  int __base = 10;
+  if (__basefield == ios_base::oct)
+    __base = 8;
+  else if (__basefield == ios_base::hex)
+    __base = 16;
+
+  // Print '-' and make the argument unsigned
+  auto __uval = std::__to_unsigned_like(__v);
+  if (__basefield != ios_base::oct && __basefield != ios_base::hex && __v < 0) {
+    *__buffer_ptr++ = '-';
+    __uval          = std::__complement(__uval);
+  }
+
+  // Maybe add '+' prefix
+  if (std::is_signed<_Integral>::value && (__flags & ios_base::showpos) && __basefield != ios_base::oct &&
+      __basefield != ios_base::hex && __v >= 0)
+    *__buffer_ptr++ = '+';
+
+  // Add base prefix
+  if (__v != 0 && __flags & ios_base::showbase) {
+    if (__basefield == ios_base::oct) {
+      *__buffer_ptr++ = '0';
+    } else if (__basefield == ios_base::hex) {
+      *__buffer_ptr++ = '0';
+      *__buffer_ptr++ = (__flags & ios_base::uppercase ? 'X' : 'x');
+    }
+  }
+
+  auto __res = std::__to_chars_integral(__buffer_ptr, __char_buffer + __buffer_size, __uval, __base);
+  _LIBCPP_ASSERT_INTERNAL(__res.__ec == std::errc(0), "to_chars: invalid maximum buffer size computed?");
+
+  // Make letters uppercase
+  if (__flags & ios_base::hex && __flags & ios_base::uppercase) {
+    for (; __buffer_ptr != __res.__ptr; ++__buffer_ptr)
+      *__buffer_ptr = std::__hex_to_upper(*__buffer_ptr);
+  }
+
+  char* __np = this->__identify_padding(__char_buffer, __res.__ptr, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__buffer_size - 1) - 1];
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_int(__char_buffer, __np, __res.__ptr, __o, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
+  return this->__do_put_integral(__s, __iob, __fl, __v);
+}
+
+template <class _CharT, class _OutputIterator>
+template <class _Float>
+_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_floating_point(
+    iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const {
+  // Stage 1 - Get number in narrow char
+  char __fmt[8]            = {'%', 0};
+  bool __specify_precision = this->__format_float(__fmt + 1, __len, __iob.flags());
+  const unsigned __nbuf    = 30;
+  char __nar[__nbuf];
+  char* __nb = __nar;
+  int __nc;
+  _LIBCPP_DIAGNOSTIC_PUSH
+  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
+  if (__specify_precision)
+    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+  else
+    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+  unique_ptr<char, void (*)(void*)> __nbh(nullptr, free);
+  if (__nc > static_cast<int>(__nbuf - 1)) {
+    if (__specify_precision)
+      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
+    else
+      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
+    if (__nc == -1)
+      std::__throw_bad_alloc();
+    __nbh.reset(__nb);
+  }
+  _LIBCPP_DIAGNOSTIC_POP
+  char* __ne = __nb + __nc;
+  char* __np = this->__identify_padding(__nb, __ne, __iob);
+  // Stage 2 - Widen __nar while adding thousands separators
+  char_type __o[2 * (__nbuf - 1) - 1];
+  char_type* __ob = __o;
+  unique_ptr<char_type, void (*)(void*)> __obh(0, free);
+  if (__nb != __nar) {
+    __ob = (char_type*)malloc(2 * static_cast<size_t>(__nc) * sizeof(char_type));
+    if (__ob == 0)
+      std::__throw_bad_alloc();
+    __obh.reset(__ob);
+  }
+  char_type* __op; // pad here
+  char_type* __oe; // end of output
+  this->__widen_and_group_float(__nb, __np, __ne, __ob, __op, __oe, __iob.getloc());
+  // [__o, __oe) contains thousands_sep'd wide number
+  // Stage 3 & 4
+  __s = std::__pad_and_output(__s, __ob, __op, __oe, __iob, __fl);
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
+  return this->__do_put_floating_point(__s, __iob, __fl, __v, "L");
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator
+num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
+  auto __flags = __iob.flags();
+  __iob.flags((__flags & ~ios_base::basefield & ~ios_base::uppercase) | ios_base::hex | ios_base::showbase);
+  auto __res = __do_put_integral(__s, __iob, __fl, reinterpret_cast<uintptr_t>(__v));
+  __iob.flags(__flags);
+  return __res;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+// NOLINTEND(libcpp-robust-against-adl)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_NUM_H
diff --git a/libcxx/include/__locale_dir/scan_keyword.h b/libcxx/include/__locale_dir/scan_keyword.h
new file mode 100644
index 000000000000..78dd0a46a8ee
--- /dev/null
+++ b/libcxx/include/__locale_dir/scan_keyword.h
@@ -0,0 +1,143 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
+#define _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
+
+#include <__config>
+#include <__memory/unique_ptr.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// __scan_keyword
+// Scans [__b, __e) until a match is found in the basic_strings range
+//  [__kb, __ke) or until it can be shown that there is no match in [__kb, __ke).
+//  __b will be incremented (visibly), consuming CharT until a match is found
+//  or proved to not exist.  A keyword may be "", in which will match anything.
+//  If one keyword is a prefix of another, and the next CharT in the input
+//  might match another keyword, the algorithm will attempt to find the longest
+//  matching keyword.  If the longer matching keyword ends up not matching, then
+//  no keyword match is found.  If no keyword match is found, __ke is returned
+//  and failbit is set in __err.
+//  Else an iterator pointing to the matching keyword is found.  If more than
+//  one keyword matches, an iterator to the first matching keyword is returned.
+//  If on exit __b == __e, eofbit is set in __err.  If __case_sensitive is false,
+//  __ct is used to force to lower case before comparing characters.
+//  Examples:
+//  Keywords:  "a", "abb"
+//  If the input is "a", the first keyword matches and eofbit is set.
+//  If the input is "abc", no match is found and "ab" are consumed.
+template <class _InputIterator, class _ForwardIterator, class _Ctype>
+_LIBCPP_HIDE_FROM_ABI _ForwardIterator __scan_keyword(
+    _InputIterator& __b,
+    _InputIterator __e,
+    _ForwardIterator __kb,
+    _ForwardIterator __ke,
+    const _Ctype& __ct,
+    ios_base::iostate& __err,
+    bool __case_sensitive = true) {
+  typedef typename iterator_traits<_InputIterator>::value_type _CharT;
+  size_t __nkw                       = static_cast<size_t>(std::distance(__kb, __ke));
+  const unsigned char __doesnt_match = '\0';
+  const unsigned char __might_match  = '\1';
+  const unsigned char __does_match   = '\2';
+  unsigned char __statbuf[100];
+  unsigned char* __status = __statbuf;
+  unique_ptr<unsigned char, void (*)(void*)> __stat_hold(nullptr, free);
+  if (__nkw > sizeof(__statbuf)) {
+    __status = (unsigned char*)malloc(__nkw);
+    if (__status == nullptr)
+      std::__throw_bad_alloc();
+    __stat_hold.reset(__status);
+  }
+  size_t __n_might_match = __nkw; // At this point, any keyword might match
+  size_t __n_does_match  = 0;     // but none of them definitely do
+  // Initialize all statuses to __might_match, except for "" keywords are __does_match
+  unsigned char* __st = __status;
+  for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+    if (!__ky->empty())
+      *__st = __might_match;
+    else {
+      *__st = __does_match;
+      --__n_might_match;
+      ++__n_does_match;
+    }
+  }
+  // While there might be a match, test keywords against the next CharT
+  for (size_t __indx = 0; __b != __e && __n_might_match > 0; ++__indx) {
+    // Peek at the next CharT but don't consume it
+    _CharT __c = *__b;
+    if (!__case_sensitive)
+      __c = __ct.toupper(__c);
+    bool __consume = false;
+    // For each keyword which might match, see if the __indx character is __c
+    // If a match if found, consume __c
+    // If a match is found, and that is the last character in the keyword,
+    //    then that keyword matches.
+    // If the keyword doesn't match this character, then change the keyword
+    //    to doesn't match
+    __st = __status;
+    for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+      if (*__st == __might_match) {
+        _CharT __kc = (*__ky)[__indx];
+        if (!__case_sensitive)
+          __kc = __ct.toupper(__kc);
+        if (__c == __kc) {
+          __consume = true;
+          if (__ky->size() == __indx + 1) {
+            *__st = __does_match;
+            --__n_might_match;
+            ++__n_does_match;
+          }
+        } else {
+          *__st = __doesnt_match;
+          --__n_might_match;
+        }
+      }
+    }
+    // consume if we matched a character
+    if (__consume) {
+      ++__b;
+      // If we consumed a character and there might be a matched keyword that
+      //   was marked matched on a previous iteration, then such keywords
+      //   which are now marked as not matching.
+      if (__n_might_match + __n_does_match > 1) {
+        __st = __status;
+        for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
+          if (*__st == __does_match && __ky->size() != __indx + 1) {
+            *__st = __doesnt_match;
+            --__n_does_match;
+          }
+        }
+      }
+    }
+  }
+  // We've exited the loop because we hit eof and/or we have no more "might matches".
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  // Return the first matching result
+  for (__st = __status; __kb != __ke; ++__kb, (void)++__st)
+    if (*__st == __does_match)
+      break;
+  if (__kb == __ke)
+    __err |= ios_base::failbit;
+  return __kb;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_SCAN_KEYWORD_H
diff --git a/libcxx/include/__locale_dir/time.h b/libcxx/include/__locale_dir/time.h
new file mode 100644
index 000000000000..5f60d5f36b34
--- /dev/null
+++ b/libcxx/include/__locale_dir/time.h
@@ -0,0 +1,766 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_TIME_H
+#define _LIBCPP___LOCALE_DIR_TIME_H
+
+#include <__algorithm/copy.h>
+#include <__config>
+#include <__locale_dir/get_c_locale.h>
+#include <__locale_dir/scan_keyword.h>
+#include <ios>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _CharT, class _InputIterator>
+_LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
+    _InputIterator& __b, _InputIterator __e, ios_base::iostate& __err, const ctype<_CharT>& __ct, int __n) {
+  // Precondition:  __n >= 1
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return 0;
+  }
+  // get first digit
+  _CharT __c = *__b;
+  if (!__ct.is(ctype_base::digit, __c)) {
+    __err |= ios_base::failbit;
+    return 0;
+  }
+  int __r = __ct.narrow(__c, 0) - '0';
+  for (++__b, (void)--__n; __b != __e && __n > 0; ++__b, (void)--__n) {
+    // get next digit
+    __c = *__b;
+    if (!__ct.is(ctype_base::digit, __c))
+      return __r;
+    __r = __r * 10 + __ct.narrow(__c, 0) - '0';
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __r;
+}
+
+class _LIBCPP_EXPORTED_FROM_ABI time_base {
+public:
+  enum dateorder { no_order, dmy, mdy, ymd, ydm };
+};
+
+template <class _CharT>
+class __time_get_c_storage {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  virtual const string_type* __weeks() const;
+  virtual const string_type* __months() const;
+  virtual const string_type* __am_pm() const;
+  virtual const string_type& __c() const;
+  virtual const string_type& __r() const;
+  virtual const string_type& __x() const;
+  virtual const string_type& __X() const;
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_c_storage() {}
+};
+
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
+
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__months() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__am_pm() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__c() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__r() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
+template <>
+_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
+#  endif
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
+public:
+  typedef _CharT char_type;
+  typedef _InputIterator iter_type;
+  typedef time_base::dateorder dateorder;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get(size_t __refs = 0) : locale::facet(__refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI dateorder date_order() const { return this->do_date_order(); }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_time(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_weekday(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_monthname(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+    return do_get_year(__b, __e, __iob, __err, __tm);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod = 0)
+      const {
+    return do_get(__b, __e, __iob, __err, __tm, __fmt, __mod);
+  }
+
+  iter_type
+  get(iter_type __b,
+      iter_type __e,
+      ios_base& __iob,
+      ios_base::iostate& __err,
+      tm* __tm,
+      const char_type* __fmtb,
+      const char_type* __fmte) const;
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get() override {}
+
+  virtual dateorder do_date_order() const;
+  virtual iter_type
+  do_get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type
+  do_get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
+  virtual iter_type do_get(
+      iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod) const;
+
+private:
+  void __get_white_space(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_percent(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+
+  void __get_weekdayname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_monthname(
+      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_month(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_year4(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_hour(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_12_hour(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_am_pm(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_minute(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_second(int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void
+  __get_weekday(int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+  void __get_day_year_num(
+      int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
+};
+
+template <class _CharT, class _InputIterator>
+locale::id time_get<_CharT, _InputIterator>::id;
+
+// time_get primitives
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekdayname(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __wk = this->__weeks();
+  ptrdiff_t __i           = std::__scan_keyword(__b, __e, __wk, __wk + 14, __ct, __err, false) - __wk;
+  if (__i < 14)
+    __w = __i % 7;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_monthname(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  // Note:  ignoring case comes from the POSIX strptime spec
+  const string_type* __month = this->__months();
+  ptrdiff_t __i              = std::__scan_keyword(__b, __e, __month, __month + 24, __ct, __err, false) - __month;
+  if (__i < 24)
+    __m = __i % 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_month(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2) - 1;
+  if (!(__err & ios_base::failbit) && 0 <= __t && __t <= 11)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit)) {
+    if (__t < 69)
+      __t += 2000;
+    else if (69 <= __t && __t <= 99)
+      __t += 1900;
+    __y = __t - 1900;
+  }
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_year4(
+    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
+  if (!(__err & ios_base::failbit))
+    __y = __t - 1900;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 23)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_12_hour(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 12)
+    __h = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_minute(
+    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 59)
+    __m = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_second(
+    int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
+  if (!(__err & ios_base::failbit) && __t <= 60)
+    __s = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_weekday(
+    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 1);
+  if (!(__err & ios_base::failbit) && __t <= 6)
+    __w = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_day_year_num(
+    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 3);
+  if (!(__err & ios_base::failbit) && __t <= 365)
+    __d = __t;
+  else
+    __err |= ios_base::failbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_white_space(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+    ;
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_am_pm(
+    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  const string_type* __ap = this->__am_pm();
+  if (__ap[0].size() + __ap[1].size() == 0) {
+    __err |= ios_base::failbit;
+    return;
+  }
+  ptrdiff_t __i = std::__scan_keyword(__b, __e, __ap, __ap + 2, __ct, __err, false) - __ap;
+  if (__i == 0 && __h == 12)
+    __h = 0;
+  else if (__i == 1 && __h < 12)
+    __h += 12;
+}
+
+template <class _CharT, class _InputIterator>
+void time_get<_CharT, _InputIterator>::__get_percent(
+    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
+  if (__b == __e) {
+    __err |= ios_base::eofbit | ios_base::failbit;
+    return;
+  }
+  if (__ct.narrow(*__b, 0) != '%')
+    __err |= ios_base::failbit;
+  else if (++__b == __e)
+    __err |= ios_base::eofbit;
+}
+
+// time_get end primitives
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::get(
+    iter_type __b,
+    iter_type __e,
+    ios_base& __iob,
+    ios_base::iostate& __err,
+    tm* __tm,
+    const char_type* __fmtb,
+    const char_type* __fmte) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __err                        = ios_base::goodbit;
+  while (__fmtb != __fmte && __err == ios_base::goodbit) {
+    if (__b == __e) {
+      __err = ios_base::failbit;
+      break;
+    }
+    if (__ct.narrow(*__fmtb, 0) == '%') {
+      if (++__fmtb == __fmte) {
+        __err = ios_base::failbit;
+        break;
+      }
+      char __cmd = __ct.narrow(*__fmtb, 0);
+      char __opt = '\0';
+      if (__cmd == 'E' || __cmd == '0') {
+        if (++__fmtb == __fmte) {
+          __err = ios_base::failbit;
+          break;
+        }
+        __opt = __cmd;
+        __cmd = __ct.narrow(*__fmtb, 0);
+      }
+      __b = do_get(__b, __e, __iob, __err, __tm, __cmd, __opt);
+      ++__fmtb;
+    } else if (__ct.is(ctype_base::space, *__fmtb)) {
+      for (++__fmtb; __fmtb != __fmte && __ct.is(ctype_base::space, *__fmtb); ++__fmtb)
+        ;
+      for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
+        ;
+    } else if (__ct.toupper(*__b) == __ct.toupper(*__fmtb)) {
+      ++__b;
+      ++__fmtb;
+    } else
+      __err = ios_base::failbit;
+  }
+  if (__b == __e)
+    __err |= ios_base::eofbit;
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+typename time_get<_CharT, _InputIterator>::dateorder time_get<_CharT, _InputIterator>::do_date_order() const {
+  return mdy;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_time(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const char_type __fmt[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+  return get(__b, __e, __iob, __err, __tm, __fmt, __fmt + sizeof(__fmt) / sizeof(__fmt[0]));
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_date(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const string_type& __fmt = this->__x();
+  return get(__b, __e, __iob, __err, __tm, __fmt.data(), __fmt.data() + __fmt.size());
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_weekday(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_monthname(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get_year(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  __get_year(__tm->tm_year, __b, __e, __err, __ct);
+  return __b;
+}
+
+template <class _CharT, class _InputIterator>
+_InputIterator time_get<_CharT, _InputIterator>::do_get(
+    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char) const {
+  __err                        = ios_base::goodbit;
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  switch (__fmt) {
+  case 'a':
+  case 'A':
+    __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'b':
+  case 'B':
+  case 'h':
+    __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'c': {
+    const string_type& __fm = this->__c();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'd':
+  case 'e':
+    __get_day(__tm->tm_mday, __b, __e, __err, __ct);
+    break;
+  case 'D': {
+    const char_type __fm[] = {'%', 'm', '/', '%', 'd', '/', '%', 'y'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'F': {
+    const char_type __fm[] = {'%', 'Y', '-', '%', 'm', '-', '%', 'd'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'H':
+    __get_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'I':
+    __get_12_hour(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'j':
+    __get_day_year_num(__tm->tm_yday, __b, __e, __err, __ct);
+    break;
+  case 'm':
+    __get_month(__tm->tm_mon, __b, __e, __err, __ct);
+    break;
+  case 'M':
+    __get_minute(__tm->tm_min, __b, __e, __err, __ct);
+    break;
+  case 'n':
+  case 't':
+    __get_white_space(__b, __e, __err, __ct);
+    break;
+  case 'p':
+    __get_am_pm(__tm->tm_hour, __b, __e, __err, __ct);
+    break;
+  case 'r': {
+    const char_type __fm[] = {'%', 'I', ':', '%', 'M', ':', '%', 'S', ' ', '%', 'p'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'R': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'S':
+    __get_second(__tm->tm_sec, __b, __e, __err, __ct);
+    break;
+  case 'T': {
+    const char_type __fm[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
+    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
+  } break;
+  case 'w':
+    __get_weekday(__tm->tm_wday, __b, __e, __err, __ct);
+    break;
+  case 'x':
+    return do_get_date(__b, __e, __iob, __err, __tm);
+  case 'X': {
+    const string_type& __fm = this->__X();
+    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
+  } break;
+  case 'y':
+    __get_year(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case 'Y':
+    __get_year4(__tm->tm_year, __b, __e, __err, __ct);
+    break;
+  case '%':
+    __get_percent(__b, __e, __err, __ct);
+    break;
+  default:
+    __err |= ios_base::failbit;
+  }
+  return __b;
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_get {
+protected:
+  __locale::__locale_t __loc_;
+
+  __time_get(const char* __nm);
+  __time_get(const string& __nm);
+  ~__time_get();
+};
+
+template <class _CharT>
+class __time_get_storage : public __time_get {
+protected:
+  typedef basic_string<_CharT> string_type;
+
+  string_type __weeks_[14];
+  string_type __months_[24];
+  string_type __am_pm_[2];
+  string_type __c_;
+  string_type __r_;
+  string_type __x_;
+  string_type __X_;
+
+  explicit __time_get_storage(const char* __nm);
+  explicit __time_get_storage(const string& __nm);
+
+  _LIBCPP_HIDE_FROM_ABI ~__time_get_storage() {}
+
+  time_base::dateorder __do_date_order() const;
+
+private:
+  void init(const ctype<_CharT>&);
+  string_type __analyze(char __fmt, const ctype<_CharT>&);
+};
+
+#  define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                      \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;                \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                           \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                             \
+    template <>                                                                                                        \
+    _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(           \
+        char, const ctype<_CharT>&);                                                                                   \
+    extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()       \
+        const;                                                                                                         \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);           \
+    extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);             \
+    extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                  \
+    __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);
+
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
+#  endif
+#  undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
+
+template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
+class time_get_byname : public time_get<_CharT, _InputIterator>, private __time_get_storage<_CharT> {
+public:
+  typedef time_base::dateorder dateorder;
+  typedef _InputIterator iter_type;
+  typedef _CharT char_type;
+  typedef basic_string<char_type> string_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const char* __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const string& __nm, size_t __refs = 0)
+      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get_byname() override {}
+
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL dateorder do_date_order() const override { return this->__do_date_order(); }
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __weeks() const override { return this->__weeks_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __months() const override { return this->__months_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __am_pm() const override { return this->__am_pm_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __c() const override { return this->__c_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __r() const override { return this->__r_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __x() const override { return this->__x_; }
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __X() const override { return this->__X_; }
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
+#  endif
+
+class _LIBCPP_EXPORTED_FROM_ABI __time_put {
+  __locale::__locale_t __loc_;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI __time_put() : __loc_(_LIBCPP_GET_C_LOCALE) {}
+  __time_put(const char* __nm);
+  __time_put(const string& __nm);
+  ~__time_put();
+  void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+  void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
+#  endif
+};
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class time_put : public locale::facet, private __time_put {
+public:
+  typedef _CharT char_type;
+  typedef _OutputIterator iter_type;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(size_t __refs = 0) : locale::facet(__refs) {}
+
+  iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+      const;
+
+  _LIBCPP_HIDE_FROM_ABI iter_type
+  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, char __fmt, char __mod = 0) const {
+    return do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+  }
+
+  static locale::id id;
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put() override {}
+  virtual iter_type do_put(iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const;
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const char* __nm, size_t __refs) : locale::facet(__refs), __time_put(__nm) {}
+  _LIBCPP_HIDE_FROM_ABI explicit time_put(const string& __nm, size_t __refs)
+      : locale::facet(__refs), __time_put(__nm) {}
+};
+
+template <class _CharT, class _OutputIterator>
+locale::id time_put<_CharT, _OutputIterator>::id;
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::put(
+    iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
+    const {
+  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
+  for (; __pb != __pe; ++__pb) {
+    if (__ct.narrow(*__pb, 0) == '%') {
+      if (++__pb == __pe) {
+        *__s++ = __pb[-1];
+        break;
+      }
+      char __mod = 0;
+      char __fmt = __ct.narrow(*__pb, 0);
+      if (__fmt == 'E' || __fmt == 'O') {
+        if (++__pb == __pe) {
+          *__s++ = __pb[-2];
+          *__s++ = __pb[-1];
+          break;
+        }
+        __mod = __fmt;
+        __fmt = __ct.narrow(*__pb, 0);
+      }
+      __s = do_put(__s, __iob, __fl, __tm, __fmt, __mod);
+    } else
+      *__s++ = *__pb;
+  }
+  return __s;
+}
+
+template <class _CharT, class _OutputIterator>
+_OutputIterator time_put<_CharT, _OutputIterator>::do_put(
+    iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const {
+  char_type __nar[100];
+  char_type* __nb = __nar;
+  char_type* __ne = __nb + 100;
+  __do_put(__nb, __ne, __tm, __fmt, __mod);
+  return std::copy(__nb, __ne, __s);
+}
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
+#  endif
+
+template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
+class time_put_byname : public time_put<_CharT, _OutputIterator> {
+public:
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const char* __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const string& __nm, size_t __refs = 0)
+      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put_byname() override {}
+};
+
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
+#  if _LIBCPP_HAS_WIDE_CHARACTERS
+extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
+#  endif
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_TIME_H
diff --git a/libcxx/include/__locale_dir/wbuffer_convert.h b/libcxx/include/__locale_dir/wbuffer_convert.h
new file mode 100644
index 000000000000..a6818aadf5d0
--- /dev/null
+++ b/libcxx/include/__locale_dir/wbuffer_convert.h
@@ -0,0 +1,430 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
+#define _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
+
+#include <__algorithm/reverse.h>
+#include <__config>
+#include <__string/char_traits.h>
+#include <ios>
+#include <streambuf>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+_LIBCPP_PUSH_MACROS
+#    include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Codecvt, class _Elem = wchar_t, class _Tr = char_traits<_Elem> >
+class _LIBCPP_DEPRECATED_IN_CXX17 wbuffer_convert : public basic_streambuf<_Elem, _Tr> {
+public:
+  // types:
+  typedef _Elem char_type;
+  typedef _Tr traits_type;
+  typedef typename traits_type::int_type int_type;
+  typedef typename traits_type::pos_type pos_type;
+  typedef typename traits_type::off_type off_type;
+  typedef typename _Codecvt::state_type state_type;
+
+private:
+  char* __extbuf_;
+  const char* __extbufnext_;
+  const char* __extbufend_;
+  char __extbuf_min_[8];
+  size_t __ebs_;
+  char_type* __intbuf_;
+  size_t __ibs_;
+  streambuf* __bufptr_;
+  _Codecvt* __cv_;
+  state_type __st_;
+  ios_base::openmode __cm_;
+  bool __owns_eb_;
+  bool __owns_ib_;
+  bool __always_noconv_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
+  explicit _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    else
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
+
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf() const { return __bufptr_; }
+  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf(streambuf* __bytebuf) {
+    streambuf* __r = __bufptr_;
+    __bufptr_      = __bytebuf;
+    return __r;
+  }
+
+  wbuffer_convert(const wbuffer_convert&)            = delete;
+  wbuffer_convert& operator=(const wbuffer_convert&) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __st_; }
+
+protected:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type underflow();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type pbackfail(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type overflow(int_type __c = traits_type::eof());
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
+  seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out);
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int sync();
+
+private:
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool __read_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __write_mode();
+  _LIBCPP_HIDE_FROM_ABI_VIRTUAL wbuffer_convert* __close();
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt, state_type __state)
+    : __extbuf_(nullptr),
+      __extbufnext_(nullptr),
+      __extbufend_(nullptr),
+      __ebs_(0),
+      __intbuf_(0),
+      __ibs_(0),
+      __bufptr_(__bytebuf),
+      __cv_(__pcvt),
+      __st_(__state),
+      __cm_(0),
+      __owns_eb_(false),
+      __owns_ib_(false),
+      __always_noconv_(__cv_ ? __cv_->always_noconv() : false) {
+  setbuf(0, 4096);
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>::~wbuffer_convert() {
+  __close();
+  delete __cv_;
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || __bufptr_ == nullptr)
+    return traits_type::eof();
+  bool __initial = __read_mode();
+  char_type __1buf;
+  if (this->gptr() == 0)
+    this->setg(std::addressof(__1buf), std::addressof(__1buf) + 1, std::addressof(__1buf) + 1);
+  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
+  int_type __c            = traits_type::eof();
+  if (this->gptr() == this->egptr()) {
+    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
+      __nmemb            = __bufptr_->sgetn((char*)this->eback() + __unget_sz, __nmemb);
+      if (__nmemb != 0) {
+        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
+        __c = *this->gptr();
+      }
+    } else {
+      if (__extbufend_ != __extbufnext_) {
+        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
+        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
+        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+      }
+      __extbufnext_      = __extbuf_ + (__extbufend_ - __extbufnext_);
+      __extbufend_       = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
+      streamsize __nmemb = std::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
+                                    static_cast<streamsize>(__extbufend_ - __extbufnext_));
+      codecvt_base::result __r;
+      // FIXME: Do we ever need to restore the state here?
+      // state_type __svs = __st_;
+      streamsize __nr = __bufptr_->sgetn(const_cast<char*>(__extbufnext_), __nmemb);
+      if (__nr != 0) {
+        __extbufend_ = __extbufnext_ + __nr;
+        char_type* __inext;
+        __r = __cv_->in(
+            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->egptr(), __inext);
+        if (__r == codecvt_base::noconv) {
+          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
+          __c = *this->gptr();
+        } else if (__inext != this->eback() + __unget_sz) {
+          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
+          __c = *this->gptr();
+        }
+      }
+    }
+  } else
+    __c = *this->gptr();
+  if (this->eback() == std::addressof(__1buf))
+    this->setg(0, 0, 0);
+  return __c;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::pbackfail(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ != 0 && __bufptr_ && this->eback() < this->gptr()) {
+    if (traits_type::eq_int_type(__c, traits_type::eof())) {
+      this->gbump(-1);
+      return traits_type::not_eof(__c);
+    }
+    if (traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
+      this->gbump(-1);
+      *this->gptr() = traits_type::to_char_type(__c);
+      return __c;
+    }
+  }
+  return traits_type::eof();
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return traits_type::eof();
+  __write_mode();
+  char_type __1buf;
+  char_type* __pb_save  = this->pbase();
+  char_type* __epb_save = this->epptr();
+  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
+    if (this->pptr() == 0)
+      this->setp(std::addressof(__1buf), std::addressof(__1buf) + 1);
+    *this->pptr() = traits_type::to_char_type(__c);
+    this->pbump(1);
+  }
+  if (this->pptr() != this->pbase()) {
+    if (__always_noconv_) {
+      streamsize __nmemb = static_cast<streamsize>(this->pptr() - this->pbase());
+      if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+        return traits_type::eof();
+    } else {
+      char* __extbe = __extbuf_;
+      codecvt_base::result __r;
+      do {
+        const char_type* __e;
+        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+        if (__e == this->pbase())
+          return traits_type::eof();
+        if (__r == codecvt_base::noconv) {
+          streamsize __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
+          if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
+            return traits_type::eof();
+        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
+          streamsize __nmemb = static_cast<size_t>(__extbe - __extbuf_);
+          if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+            return traits_type::eof();
+          if (__r == codecvt_base::partial) {
+            this->setp(const_cast<char_type*>(__e), this->pptr());
+            this->__pbump(this->epptr() - this->pbase());
+          }
+        } else
+          return traits_type::eof();
+      } while (__r == codecvt_base::partial);
+    }
+    this->setp(__pb_save, __epb_save);
+  }
+  return traits_type::not_eof(__c);
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+basic_streambuf<_Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::setbuf(char_type* __s, streamsize __n) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  this->setg(0, 0, 0);
+  this->setp(0, 0);
+  if (__owns_eb_)
+    delete[] __extbuf_;
+  if (__owns_ib_)
+    delete[] __intbuf_;
+  __ebs_ = __n;
+  if (__ebs_ > sizeof(__extbuf_min_)) {
+    if (__always_noconv_ && __s) {
+      __extbuf_  = (char*)__s;
+      __owns_eb_ = false;
+    } else {
+      __extbuf_  = new char[__ebs_];
+      __owns_eb_ = true;
+    }
+  } else {
+    __extbuf_  = __extbuf_min_;
+    __ebs_     = sizeof(__extbuf_min_);
+    __owns_eb_ = false;
+  }
+  if (!__always_noconv_) {
+    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
+    if (__s && __ibs_ >= sizeof(__extbuf_min_)) {
+      __intbuf_  = __s;
+      __owns_ib_ = false;
+    } else {
+      __intbuf_  = new char_type[__ibs_];
+      __owns_ib_ = true;
+    }
+  } else {
+    __ibs_     = 0;
+    __intbuf_  = 0;
+    __owns_ib_ = false;
+  }
+  return this;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __om) {
+  int __width = __cv_->encoding();
+  if (__cv_ == 0 || !__bufptr_ || (__width <= 0 && __off != 0) || sync())
+    return pos_type(off_type(-1));
+  // __width > 0 || __off == 0, now check __way
+  if (__way != ios_base::beg && __way != ios_base::cur && __way != ios_base::end)
+    return pos_type(off_type(-1));
+  pos_type __r = __bufptr_->pubseekoff(__width * __off, __way, __om);
+  __r.state(__st_);
+  return __r;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
+wbuffer_convert<_Codecvt, _Elem, _Tr>::seekpos(pos_type __sp, ios_base::openmode __wch) {
+  if (__cv_ == 0 || !__bufptr_ || sync())
+    return pos_type(off_type(-1));
+  if (__bufptr_->pubseekpos(__sp, __wch) == pos_type(off_type(-1)))
+    return pos_type(off_type(-1));
+  return __sp;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+int wbuffer_convert<_Codecvt, _Elem, _Tr>::sync() {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  if (__cv_ == 0 || !__bufptr_)
+    return 0;
+  if (__cm_ & ios_base::out) {
+    if (this->pptr() != this->pbase())
+      if (overflow() == traits_type::eof())
+        return -1;
+    codecvt_base::result __r;
+    do {
+      char* __extbe;
+      __r                = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
+      streamsize __nmemb = static_cast<streamsize>(__extbe - __extbuf_);
+      if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
+        return -1;
+    } while (__r == codecvt_base::partial);
+    if (__r == codecvt_base::error)
+      return -1;
+    if (__bufptr_->pubsync())
+      return -1;
+  } else if (__cm_ & ios_base::in) {
+    off_type __c;
+    if (__always_noconv_)
+      __c = this->egptr() - this->gptr();
+    else {
+      int __width = __cv_->encoding();
+      __c         = __extbufend_ - __extbufnext_;
+      if (__width > 0)
+        __c += __width * (this->egptr() - this->gptr());
+      else {
+        if (this->gptr() != this->egptr()) {
+          std::reverse(this->gptr(), this->egptr());
+          codecvt_base::result __r;
+          const char_type* __e = this->gptr();
+          char* __extbe;
+          do {
+            __r = __cv_->out(__st_, __e, this->egptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
+            switch (__r) {
+            case codecvt_base::noconv:
+              __c += this->egptr() - this->gptr();
+              break;
+            case codecvt_base::ok:
+            case codecvt_base::partial:
+              __c += __extbe - __extbuf_;
+              break;
+            default:
+              return -1;
+            }
+          } while (__r == codecvt_base::partial);
+        }
+      }
+    }
+    if (__bufptr_->pubseekoff(-__c, ios_base::cur, __cm_) == pos_type(off_type(-1)))
+      return -1;
+    this->setg(0, 0, 0);
+    __cm_ = 0;
+  }
+  return 0;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _Tr>
+bool wbuffer_convert<_Codecvt, _Elem, _Tr>::__read_mode() {
+  if (!(__cm_ & ios_base::in)) {
+    this->setp(0, 0);
+    if (__always_noconv_)
+      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
+    else
+      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
+    __cm_ = ios_base::in;
+    return true;
+  }
+  return false;
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+void wbuffer_convert<_Codecvt, _Elem, _Tr>::__write_mode() {
+  if (!(__cm_ & ios_base::out)) {
+    this->setg(0, 0, 0);
+    if (__ebs_ > sizeof(__extbuf_min_)) {
+      if (__always_noconv_)
+        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
+      else
+        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
+    } else
+      this->setp(0, 0);
+    __cm_ = ios_base::out;
+  }
+}
+
+template <class _Codecvt, class _Elem, class _Tr>
+wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__close() {
+  wbuffer_convert* __rt = nullptr;
+  if (__cv_ != nullptr && __bufptr_ != nullptr) {
+    __rt = this;
+    if ((__cm_ & ios_base::out) && sync())
+      __rt = nullptr;
+  }
+  return __rt;
+}
+
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_WBUFFER_CONVERT_H
diff --git a/libcxx/include/__locale_dir/wstring_convert.h b/libcxx/include/__locale_dir/wstring_convert.h
new file mode 100644
index 000000000000..42a56eb8573e
--- /dev/null
+++ b/libcxx/include/__locale_dir/wstring_convert.h
@@ -0,0 +1,254 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
+#define _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
+
+#include <__config>
+#include <__locale>
+#include <__memory/allocator.h>
+#include <string>
+
+#if _LIBCPP_HAS_LOCALIZATION
+
+#  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#    pragma GCC system_header
+#  endif
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+_LIBCPP_PUSH_MACROS
+#    include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Codecvt,
+          class _Elem      = wchar_t,
+          class _WideAlloc = allocator<_Elem>,
+          class _ByteAlloc = allocator<char> >
+class _LIBCPP_DEPRECATED_IN_CXX17 wstring_convert {
+public:
+  typedef basic_string<char, char_traits<char>, _ByteAlloc> byte_string;
+  typedef basic_string<_Elem, char_traits<_Elem>, _WideAlloc> wide_string;
+  typedef typename _Codecvt::state_type state_type;
+  typedef typename wide_string::traits_type::int_type int_type;
+
+private:
+  byte_string __byte_err_string_;
+  wide_string __wide_err_string_;
+  _Codecvt* __cvtptr_;
+  state_type __cvtstate_;
+  size_t __cvtcount_;
+
+public:
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
+  _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
+#    else
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
+#    endif
+
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
+  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
+  wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
+#    ifndef _LIBCPP_CXX03_LANG
+  _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
+#    endif
+  _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
+
+  wstring_convert(const wstring_convert& __wc)            = delete;
+  wstring_convert& operator=(const wstring_convert& __wc) = delete;
+
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(char __byte) { return from_bytes(&__byte, &__byte + 1); }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __ptr) {
+    return from_bytes(__ptr, __ptr + char_traits<char>::length(__ptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const byte_string& __str) {
+    return from_bytes(__str.data(), __str.data() + __str.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __first, const char* __last);
+
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(_Elem __wchar) {
+    return to_bytes(std::addressof(__wchar), std::addressof(__wchar) + 1);
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __wptr) {
+    return to_bytes(__wptr, __wptr + char_traits<_Elem>::length(__wptr));
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const wide_string& __wstr) {
+    return to_bytes(__wstr.data(), __wstr.data() + __wstr.size());
+  }
+  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __first, const _Elem* __last);
+
+  _LIBCPP_HIDE_FROM_ABI size_t converted() const _NOEXCEPT { return __cvtcount_; }
+  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __cvtstate_; }
+};
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt)
+    : __cvtptr_(__pcvt), __cvtstate_(), __cvtcount_(0) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt, state_type __state)
+    : __cvtptr_(__pcvt), __cvtstate_(__state), __cvtcount_(0) {}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
+    const byte_string& __byte_err, const wide_string& __wide_err)
+    : __byte_err_string_(__byte_err), __wide_err_string_(__wide_err), __cvtstate_(), __cvtcount_(0) {
+  __cvtptr_ = new _Codecvt;
+}
+
+#    ifndef _LIBCPP_CXX03_LANG
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
+    : __byte_err_string_(std::move(__wc.__byte_err_string_)),
+      __wide_err_string_(std::move(__wc.__wide_err_string_)),
+      __cvtptr_(__wc.__cvtptr_),
+      __cvtstate_(__wc.__cvtstate_),
+      __cvtcount_(__wc.__cvtcount_) {
+  __wc.__cvtptr_ = nullptr;
+}
+
+#    endif // _LIBCPP_CXX03_LANG
+
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::~wstring_convert() {
+  delete __cvtptr_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wide_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::from_bytes(const char* __frm, const char* __frm_end) {
+  _LIBCPP_SUPPRESS_DEPRECATED_POP
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    wide_string __ws(2 * (__frm_end - __frm), _Elem());
+    if (__frm != __frm_end)
+      __ws.resize(__ws.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      _Elem* __to     = std::addressof(__ws[0]);
+      _Elem* __to_end = __to + __ws.size();
+      const char* __frm_nxt;
+      do {
+        _Elem* __to_nxt;
+        __r = __cvtptr_->in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __ws.resize(__to - std::addressof(__ws[0]));
+          // This only gets executed if _Elem is char
+          __ws.append((const _Elem*)__frm, (const _Elem*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __ws.resize(__to_nxt - std::addressof(__ws[0]));
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - std::addressof(__ws[0]);
+          __ws.resize(2 * __s);
+          __to     = std::addressof(__ws[0]) + __s;
+          __to_end = std::addressof(__ws[0]) + __ws.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok)
+      return __ws;
+  }
+
+  if (__wide_err_string_.empty())
+    std::__throw_range_error("wstring_convert: from_bytes error");
+
+  return __wide_err_string_;
+}
+
+template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
+typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::byte_string
+wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::to_bytes(const _Elem* __frm, const _Elem* __frm_end) {
+  __cvtcount_ = 0;
+  if (__cvtptr_ != nullptr) {
+    byte_string __bs(2 * (__frm_end - __frm), char());
+    if (__frm != __frm_end)
+      __bs.resize(__bs.capacity());
+    codecvt_base::result __r = codecvt_base::ok;
+    state_type __st          = __cvtstate_;
+    if (__frm != __frm_end) {
+      char* __to     = std::addressof(__bs[0]);
+      char* __to_end = __to + __bs.size();
+      const _Elem* __frm_nxt;
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+        __cvtcount_ += __frm_nxt - __frm;
+        if (__frm_nxt == __frm) {
+          __r = codecvt_base::error;
+        } else if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - std::addressof(__bs[0]));
+          // This only gets executed if _Elem is char
+          __bs.append((const char*)__frm, (const char*)__frm_end);
+          __frm = __frm_nxt;
+          __r   = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - std::addressof(__bs[0]));
+          __frm = __frm_nxt;
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __s = __to_nxt - std::addressof(__bs[0]);
+          __bs.resize(2 * __s);
+          __to     = std::addressof(__bs[0]) + __s;
+          __to_end = std::addressof(__bs[0]) + __bs.size();
+          __frm    = __frm_nxt;
+        }
+      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
+    }
+    if (__r == codecvt_base::ok) {
+      size_t __s = __bs.size();
+      __bs.resize(__bs.capacity());
+      char* __to     = std::addressof(__bs[0]) + __s;
+      char* __to_end = __to + __bs.size();
+      do {
+        char* __to_nxt;
+        __r = __cvtptr_->unshift(__st, __to, __to_end, __to_nxt);
+        if (__r == codecvt_base::noconv) {
+          __bs.resize(__to - std::addressof(__bs[0]));
+          __r = codecvt_base::ok;
+        } else if (__r == codecvt_base::ok) {
+          __bs.resize(__to_nxt - std::addressof(__bs[0]));
+        } else if (__r == codecvt_base::partial) {
+          ptrdiff_t __sp = __to_nxt - std::addressof(__bs[0]);
+          __bs.resize(2 * __sp);
+          __to     = std::addressof(__bs[0]) + __sp;
+          __to_end = std::addressof(__bs[0]) + __bs.size();
+        }
+      } while (__r == codecvt_base::partial);
+      if (__r == codecvt_base::ok)
+        return __bs;
+    }
+  }
+
+  if (__byte_err_string_.empty())
+    std::__throw_range_error("wstring_convert: to_bytes error");
+
+  return __byte_err_string_;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
+
+#endif // _LIBCPP_HAS_LOCALIZATION
+
+#endif // _LIBCPP___LOCALE_DIR_WSTRING_CONVERT_H
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index f1b2e2dbfc0c..0cbd99510567 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -34,7 +34,7 @@
 #include <__memory/shared_count.h>
 #include <__memory/uninitialized_algorithms.h>
 #include <__memory/unique_ptr.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/disjunction.h>
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 3e8d3cd6a409..eff24546cdc0 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -24,7 +24,7 @@
 #include <__memory/auto_ptr.h>
 #include <__memory/compressed_pair.h>
 #include <__memory/pointer_traits.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/dependent_type.h>
diff --git a/libcxx/include/__mutex/lock_guard.h b/libcxx/include/__mutex/lock_guard.h
index a529c552a1df..67a25ed2f8a8 100644
--- a/libcxx/include/__mutex/lock_guard.h
+++ b/libcxx/include/__mutex/lock_guard.h
@@ -19,7 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Mutex>
-class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) lock_guard {
+class _LIBCPP_SCOPED_LOCKABLE lock_guard {
 public:
   typedef _Mutex mutex_type;
 
@@ -27,16 +27,14 @@ class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) lock_guard {
   mutex_type& __m_;
 
 public:
-  [[__nodiscard__]]
-  _LIBCPP_HIDE_FROM_ABI explicit lock_guard(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI explicit lock_guard(mutex_type& __m) _LIBCPP_ACQUIRE_CAPABILITY(__m)
       : __m_(__m) {
     __m_.lock();
   }
 
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI lock_guard(mutex_type& __m, adopt_lock_t)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI lock_guard(mutex_type& __m, adopt_lock_t) _LIBCPP_REQUIRES_CAPABILITY(__m)
       : __m_(__m) {}
-  _LIBCPP_HIDE_FROM_ABI ~lock_guard() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI ~lock_guard() { __m_.unlock(); }
 
   lock_guard(lock_guard const&)            = delete;
   lock_guard& operator=(lock_guard const&) = delete;
diff --git a/libcxx/include/__mutex/mutex.h b/libcxx/include/__mutex/mutex.h
index 317320287902..68c8842b35ed 100644
--- a/libcxx/include/__mutex/mutex.h
+++ b/libcxx/include/__mutex/mutex.h
@@ -21,7 +21,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("mutex")) mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("mutex") mutex {
   __libcpp_mutex_t __m_ = _LIBCPP_MUTEX_INITIALIZER;
 
 public:
@@ -36,9 +36,9 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(capability("mut
   ~mutex() _NOEXCEPT;
 #  endif
 
-  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability());
-  bool try_lock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(try_acquire_capability(true));
-  void unlock() _NOEXCEPT _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability());
+  _LIBCPP_ACQUIRE_CAPABILITY() void lock();
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock() _NOEXCEPT;
+  _LIBCPP_RELEASE_CAPABILITY void unlock() _NOEXCEPT;
 
   typedef __libcpp_mutex_t* native_handle_type;
   _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return &__m_; }
diff --git a/libcxx/include/__ostream/basic_ostream.h b/libcxx/include/__ostream/basic_ostream.h
index f7473a36d8cc..effeef491f34 100644
--- a/libcxx/include/__ostream/basic_ostream.h
+++ b/libcxx/include/__ostream/basic_ostream.h
@@ -15,6 +15,9 @@
 
 #  include <__exception/operations.h>
 #  include <__fwd/memory.h>
+#  include <__iterator/ostreambuf_iterator.h>
+#  include <__locale_dir/num.h>
+#  include <__locale_dir/pad_and_output.h>
 #  include <__memory/addressof.h>
 #  include <__memory/unique_ptr.h>
 #  include <__new/exceptions.h>
@@ -27,7 +30,6 @@
 #  include <__utility/declval.h>
 #  include <bitset>
 #  include <ios>
-#  include <locale>
 #  include <streambuf>
 #  include <string_view>
 
diff --git a/libcxx/include/__ostream/print.h b/libcxx/include/__ostream/print.h
index a4d7506cffc4..c5906c41b95b 100644
--- a/libcxx/include/__ostream/print.h
+++ b/libcxx/include/__ostream/print.h
@@ -18,8 +18,8 @@
 #  include <__ostream/basic_ostream.h>
 #  include <format>
 #  include <ios>
-#  include <locale>
 #  include <print>
+#  include <streambuf>
 
 #  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #    pragma GCC system_header
diff --git a/libcxx/include/__ranges/zip_view.h b/libcxx/include/__ranges/zip_view.h
index 6d31f5ce0993..e2a194efcfb4 100644
--- a/libcxx/include/__ranges/zip_view.h
+++ b/libcxx/include/__ranges/zip_view.h
@@ -23,6 +23,7 @@
 #include <__iterator/iter_move.h>
 #include <__iterator/iter_swap.h>
 #include <__iterator/iterator_traits.h>
+#include <__iterator/product_iterator.h>
 #include <__ranges/access.h>
 #include <__ranges/all.h>
 #include <__ranges/concepts.h>
@@ -251,6 +252,10 @@ class zip_view<_Views...>::__iterator : public __zip_view_iterator_category_base
 
   friend class zip_view<_Views...>;
 
+  static constexpr bool __is_zip_view_iterator = true;
+
+  friend struct __product_iterator_traits<__iterator>;
+
 public:
   using iterator_concept = decltype(ranges::__get_zip_view_iterator_tag<_Const, _Views...>());
   using value_type       = tuple<range_value_t<__maybe_const<_Const, _Views>>...>;
@@ -468,6 +473,23 @@ inline constexpr auto zip = __zip::__fn{};
 } // namespace views
 } // namespace ranges
 
+template <class _Iterator>
+  requires _Iterator::__is_zip_view_iterator
+struct __product_iterator_traits<_Iterator> {
+  static constexpr size_t __size = tuple_size<decltype(std::declval<_Iterator>().__current_)>::value;
+
+  template <size_t _Nth, class _Iter>
+    requires(_Nth < __size)
+  _LIBCPP_HIDE_FROM_ABI static constexpr decltype(auto) __get_iterator_element(_Iter&& __it) {
+    return std::get<_Nth>(std::forward<_Iter>(__it).__current_);
+  }
+
+  template <class... _Iters>
+  _LIBCPP_HIDE_FROM_ABI static constexpr _Iterator __make_product_iterator(_Iters&&... __iters) {
+    return _Iterator(std::tuple(std::forward<_Iters>(__iters)...));
+  }
+};
+
 #endif // _LIBCPP_STD_VER >= 23
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__thread/thread.h b/libcxx/include/__thread/thread.h
index 40ea83096e73..1b51571ce302 100644
--- a/libcxx/include/__thread/thread.h
+++ b/libcxx/include/__thread/thread.h
@@ -16,6 +16,7 @@
 #include <__exception/terminate.h>
 #include <__functional/hash.h>
 #include <__functional/unary_function.h>
+#include <__locale>
 #include <__memory/addressof.h>
 #include <__memory/unique_ptr.h>
 #include <__mutex/mutex.h>
@@ -30,7 +31,6 @@
 #include <tuple>
 
 #if _LIBCPP_HAS_LOCALIZATION
-#  include <locale>
 #  include <sstream>
 #endif
 
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 25d098351d57..9d4ba3ad0639 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -505,34 +505,6 @@ struct __is_tree_value_type : false_type {};
 template <class _One>
 struct __is_tree_value_type<_One> : __is_tree_value_type_imp<__remove_cvref_t<_One> > {};
 
-template <class _VoidPtr>
-struct __tree_node_base_types {
-  typedef _VoidPtr __void_pointer;
-
-  typedef __tree_node_base<__void_pointer> __node_base_type;
-  typedef __rebind_pointer_t<_VoidPtr, __node_base_type> __node_base_pointer;
-
-  typedef __tree_end_node<__node_base_pointer> __end_node_type;
-  typedef __rebind_pointer_t<_VoidPtr, __end_node_type> __end_node_pointer;
-  typedef __end_node_pointer __parent_pointer;
-
-// TODO(LLVM 22): Remove this check
-#ifndef _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
-  static_assert(sizeof(__node_base_pointer) == sizeof(__end_node_pointer) && _LIBCPP_ALIGNOF(__node_base_pointer) ==
-                    _LIBCPP_ALIGNOF(__end_node_pointer),
-                "It looks like you are using std::__tree (an implementation detail for (multi)map/set) with a fancy "
-                "pointer type that thas a different representation depending on whether it points to a __tree base "
-                "pointer or a __tree node pointer (both of which are implementation details of the standard library). "
-                "This means that your ABI is being broken between LLVM 19 and LLVM 20. If you don't care about your "
-                "ABI being broken, define the _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB macro to silence this "
-                "diagnostic.");
-#endif
-
-private:
-  static_assert(is_same<typename pointer_traits<_VoidPtr>::element_type, void>::value,
-                "_VoidPtr does not point to unqualified void type");
-};
-
 template <class _Tp>
 struct __get_tree_key_type {
   using type _LIBCPP_NODEBUG = _Tp;
@@ -563,26 +535,13 @@ template <class _NodePtr, class _NodeT = typename pointer_traits<_NodePtr>::elem
 struct __tree_node_types;
 
 template <class _NodePtr, class _Tp, class _VoidPtr>
-struct __tree_node_types<_NodePtr, __tree_node<_Tp, _VoidPtr> > : public __tree_node_base_types<_VoidPtr> {
-  typedef __tree_node_base_types<_VoidPtr> __base;
-
-public:
-  typedef typename pointer_traits<_NodePtr>::element_type __node_type;
-  typedef _NodePtr __node_pointer;
-
-  using __node_value_type _LIBCPP_NODEBUG = __get_node_value_type_t<_Tp>;
-  typedef __rebind_pointer_t<_VoidPtr, __node_value_type> __node_value_type_pointer;
+struct __tree_node_types<_NodePtr, __tree_node<_Tp, _VoidPtr> > {
+  using __node_base_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<_VoidPtr, __tree_node_base<_VoidPtr> >;
+  using __end_node_pointer _LIBCPP_NODEBUG  = __rebind_pointer_t<_VoidPtr, __tree_end_node<__node_base_pointer> >;
 
 private:
-  static_assert(!is_const<__node_type>::value, "_NodePtr should never be a pointer to const");
-  static_assert(is_same<__rebind_pointer_t<_VoidPtr, __node_type>, _NodePtr>::value,
-                "_VoidPtr does not rebind to _NodePtr.");
-};
-
-template <class _ValueTp, class _VoidPtr>
-struct __make_tree_node_types {
-  typedef __rebind_pointer_t<_VoidPtr, __tree_node<_ValueTp, _VoidPtr> > _NodePtr;
-  typedef __tree_node_types<_NodePtr> type;
+  static_assert(is_same<typename pointer_traits<_VoidPtr>::element_type, void>::value,
+                "_VoidPtr does not point to unqualified void type");
 };
 
 // node
@@ -597,20 +556,19 @@ public:
 };
 
 template <class _VoidPtr>
-class _LIBCPP_STANDALONE_DEBUG __tree_node_base : public __tree_node_base_types<_VoidPtr>::__end_node_type {
-  typedef __tree_node_base_types<_VoidPtr> _NodeBaseTypes;
-
+class _LIBCPP_STANDALONE_DEBUG
+__tree_node_base : public __tree_end_node<__rebind_pointer_t<_VoidPtr, __tree_node_base<_VoidPtr> > > {
 public:
-  typedef typename _NodeBaseTypes::__node_base_pointer pointer;
-  typedef typename _NodeBaseTypes::__parent_pointer __parent_pointer;
+  using pointer                            = __rebind_pointer_t<_VoidPtr, __tree_node_base>;
+  using __end_node_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<_VoidPtr, __tree_end_node<pointer> >;
 
   pointer __right_;
-  __parent_pointer __parent_;
+  __end_node_pointer __parent_;
   bool __is_black_;
 
   _LIBCPP_HIDE_FROM_ABI pointer __parent_unsafe() const { return static_cast<pointer>(__parent_); }
 
-  _LIBCPP_HIDE_FROM_ABI void __set_parent(pointer __p) { __parent_ = static_cast<__parent_pointer>(__p); }
+  _LIBCPP_HIDE_FROM_ABI void __set_parent(pointer __p) { __parent_ = static_cast<__end_node_pointer>(__p); }
 
   ~__tree_node_base()                                  = delete;
   __tree_node_base(__tree_node_base const&)            = delete;
@@ -640,7 +598,6 @@ public:
   typedef typename __alloc_traits::pointer pointer;
 
 private:
-  typedef __tree_node_types<pointer> _NodeTypes;
   allocator_type& __na_;
 
 public:
@@ -749,7 +706,8 @@ private:
 template <class _Tp, class _NodePtr, class _DiffType>
 class __tree_const_iterator {
   typedef __tree_node_types<_NodePtr> _NodeTypes;
-  typedef typename _NodeTypes::__node_pointer __node_pointer;
+  // NOLINTNEXTLINE(libcpp-nodebug-on-aliases) lldb relies on this alias for pretty printing
+  using __node_pointer = _NodePtr;
   typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
   typedef typename _NodeTypes::__end_node_pointer __end_node_pointer;
 
@@ -842,35 +800,44 @@ public:
 
 private:
   typedef allocator_traits<allocator_type> __alloc_traits;
-  typedef typename __make_tree_node_types<_Tp, typename __alloc_traits::void_pointer>::type _NodeTypes;
   using key_type = __get_tree_key_type_t<_Tp>;
 
 public:
-  typedef typename _NodeTypes::__node_value_type __node_value_type;
-
   typedef typename __alloc_traits::pointer pointer;
   typedef typename __alloc_traits::const_pointer const_pointer;
   typedef typename __alloc_traits::size_type size_type;
   typedef typename __alloc_traits::difference_type difference_type;
 
 public:
-  typedef typename _NodeTypes::__void_pointer __void_pointer;
+  using __void_pointer _LIBCPP_NODEBUG = typename __alloc_traits::void_pointer;
 
-  typedef typename _NodeTypes::__node_type __node;
-  typedef typename _NodeTypes::__node_pointer __node_pointer;
+  using __node _LIBCPP_NODEBUG = __tree_node<_Tp, __void_pointer>;
+  // NOLINTNEXTLINE(libcpp-nodebug-on-aliases) lldb relies on this alias for pretty printing
+  using __node_pointer = __rebind_pointer_t<__void_pointer, __node>;
 
-  typedef typename _NodeTypes::__node_base_type __node_base;
-  typedef typename _NodeTypes::__node_base_pointer __node_base_pointer;
+  using __node_base _LIBCPP_NODEBUG         = __tree_node_base<__void_pointer>;
+  using __node_base_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<__void_pointer, __node_base>;
 
-  typedef typename _NodeTypes::__end_node_type __end_node_t;
-  typedef typename _NodeTypes::__end_node_pointer __end_node_ptr;
-  using __end_node_pointer _LIBCPP_NODEBUG = typename _NodeTypes::__end_node_pointer;
+  using __end_node_t _LIBCPP_NODEBUG       = __tree_end_node<__node_base_pointer>;
+  using __end_node_pointer _LIBCPP_NODEBUG = __rebind_pointer_t<__void_pointer, __end_node_t>;
 
-  typedef typename _NodeTypes::__parent_pointer __parent_pointer;
+  using __parent_pointer _LIBCPP_NODEBUG = __end_node_pointer; // TODO: Remove this once the uses in <map> are removed
 
   typedef __rebind_alloc<__alloc_traits, __node> __node_allocator;
   typedef allocator_traits<__node_allocator> __node_traits;
 
+// TODO(LLVM 22): Remove this check
+#ifndef _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB
+  static_assert(sizeof(__node_base_pointer) == sizeof(__end_node_pointer) && _LIBCPP_ALIGNOF(__node_base_pointer) ==
+                    _LIBCPP_ALIGNOF(__end_node_pointer),
+                "It looks like you are using std::__tree (an implementation detail for (multi)map/set) with a fancy "
+                "pointer type that thas a different representation depending on whether it points to a __tree base "
+                "pointer or a __tree node pointer (both of which are implementation details of the standard library). "
+                "This means that your ABI is being broken between LLVM 19 and LLVM 20. If you don't care about your "
+                "ABI being broken, define the _LIBCPP_ABI_TREE_REMOVE_NODE_POINTER_UB macro to silence this "
+                "diagnostic.");
+#endif
+
 private:
   // check for sane allocator pointer rebinding semantics. Rebinding the
   // allocator for a new pointer type should be exactly the same as rebinding
@@ -1147,7 +1114,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI size_type __erase_multi(const _Key& __k);
 
   _LIBCPP_HIDE_FROM_ABI void
-  __insert_node_at(__parent_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT;
+  __insert_node_at(__end_node_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT;
 
   template <class _Key>
   _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __v);
@@ -1203,14 +1170,14 @@ public:
   // FIXME: Make this function const qualified. Unfortunately doing so
   // breaks existing code which uses non-const callable comparators.
   template <class _Key>
-  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__parent_pointer& __parent, const _Key& __v);
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__end_node_pointer& __parent, const _Key& __v);
   template <class _Key>
-  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__parent_pointer& __parent, const _Key& __v) const {
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_equal(__end_node_pointer& __parent, const _Key& __v) const {
     return const_cast<__tree*>(this)->__find_equal(__parent, __v);
   }
   template <class _Key>
   _LIBCPP_HIDE_FROM_ABI __node_base_pointer&
-  __find_equal(const_iterator __hint, __parent_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v);
+  __find_equal(const_iterator __hint, __end_node_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v);
 
   _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree& __t) {
     __copy_assign_alloc(__t, integral_constant<bool, __node_traits::propagate_on_container_copy_assignment::value>());
@@ -1224,11 +1191,12 @@ public:
   _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __tree&, false_type) {}
 
 private:
-  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_low(__parent_pointer& __parent, const value_type& __v);
-  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_high(__parent_pointer& __parent, const value_type& __v);
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_low(__end_node_pointer& __parent, const value_type& __v);
+
+  _LIBCPP_HIDE_FROM_ABI __node_base_pointer& __find_leaf_high(__end_node_pointer& __parent, const value_type& __v);
 
   _LIBCPP_HIDE_FROM_ABI __node_base_pointer&
-  __find_leaf(const_iterator __hint, __parent_pointer& __parent, const value_type& __v);
+  __find_leaf(const_iterator __hint, __end_node_pointer& __parent, const value_type& __v);
 
   template <class... _Args>
   _LIBCPP_HIDE_FROM_ABI __node_holder __construct_node(_Args&&... __args);
@@ -1400,8 +1368,8 @@ template <class _InputIterator>
 void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _InputIterator __last) {
   typedef iterator_traits<_InputIterator> _ITraits;
   typedef typename _ITraits::value_type _ItValueType;
-  static_assert((is_same<_ItValueType, value_type>::value || is_same<_ItValueType, __node_value_type>::value),
-                "__assign_multi may only be called with the containers value type or the nodes value type");
+  static_assert(
+      is_same<_ItValueType, value_type>::value, "__assign_multi may only be called with the containers value_type");
   if (size() != 0) {
     _DetachedTreeCache __cache(this);
     for (; __cache.__get() && __first != __last; ++__first) {
@@ -1435,7 +1403,7 @@ __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t) _NOEXCEPT_(
   if (size() == 0)
     __begin_node() = __end_node();
   else {
-    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+    __end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
     __t.__begin_node()               = __t.__end_node();
     __t.__end_node()->__left_        = nullptr;
     __t.size()                       = 0;
@@ -1451,7 +1419,7 @@ __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __
     else {
       __begin_node()                   = __t.__begin_node();
       __end_node()->__left_            = __t.__end_node()->__left_;
-      __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+      __end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
       size()                           = __t.size();
       __t.__begin_node()               = __t.__end_node();
       __t.__end_node()->__left_        = nullptr;
@@ -1474,7 +1442,7 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, true_type)
   if (size() == 0)
     __begin_node() = __end_node();
   else {
-    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+    __end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
     __t.__begin_node()               = __t.__end_node();
     __t.__end_node()->__left_        = nullptr;
     __t.size()                       = 0;
@@ -1547,11 +1515,11 @@ void __tree<_Tp, _Compare, _Allocator>::swap(__tree& __t)
   if (size() == 0)
     __begin_node() = __end_node();
   else
-    __end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__end_node());
+    __end_node()->__left_->__parent_ = __end_node();
   if (__t.size() == 0)
     __t.__begin_node() = __t.__end_node();
   else
-    __t.__end_node()->__left_->__parent_ = static_cast<__parent_pointer>(__t.__end_node());
+    __t.__end_node()->__left_->__parent_ = __t.__end_node();
 }
 
 template <class _Tp, class _Compare, class _Allocator>
@@ -1567,7 +1535,7 @@ void __tree<_Tp, _Compare, _Allocator>::clear() _NOEXCEPT {
 // Return reference to null leaf
 template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
-__tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__parent_pointer& __parent, const value_type& __v) {
+__tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__end_node_pointer& __parent, const value_type& __v) {
   __node_pointer __nd = __root();
   if (__nd != nullptr) {
     while (true) {
@@ -1575,20 +1543,20 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__parent_pointer& __parent, c
         if (__nd->__right_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__right_);
         else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __nd->__right_;
         }
       } else {
         if (__nd->__left_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__left_);
         else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __parent->__left_;
         }
       }
     }
   }
-  __parent = static_cast<__parent_pointer>(__end_node());
+  __parent = __end_node();
   return __parent->__left_;
 }
 
@@ -1597,7 +1565,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__parent_pointer& __parent, c
 // Return reference to null leaf
 template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
-__tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__parent_pointer& __parent, const value_type& __v) {
+__tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__end_node_pointer& __parent, const value_type& __v) {
   __node_pointer __nd = __root();
   if (__nd != nullptr) {
     while (true) {
@@ -1605,20 +1573,20 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__parent_pointer& __parent,
         if (__nd->__left_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__left_);
         else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __parent->__left_;
         }
       } else {
         if (__nd->__right_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__right_);
         else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __nd->__right_;
         }
       }
     }
   }
-  __parent = static_cast<__parent_pointer>(__end_node());
+  __parent = __end_node();
   return __parent->__left_;
 }
 
@@ -1630,7 +1598,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__parent_pointer& __parent,
 // Return reference to null leaf
 template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Compare, _Allocator>::__find_leaf(
-    const_iterator __hint, __parent_pointer& __parent, const value_type& __v) {
+    const_iterator __hint, __end_node_pointer& __parent, const value_type& __v) {
   if (__hint == end() || !value_comp()(*__hint, __v)) // check before
   {
     // __v <= *__hint
@@ -1638,10 +1606,10 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
     if (__prior == begin() || !value_comp()(__v, *--__prior)) {
       // *prev(__hint) <= __v <= *__hint
       if (__hint.__ptr_->__left_ == nullptr) {
-        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        __parent = static_cast<__end_node_pointer>(__hint.__ptr_);
         return __parent->__left_;
       } else {
-        __parent = static_cast<__parent_pointer>(__prior.__ptr_);
+        __parent = static_cast<__end_node_pointer>(__prior.__ptr_);
         return static_cast<__node_base_pointer>(__prior.__ptr_)->__right_;
       }
     }
@@ -1659,7 +1627,7 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
 template <class _Tp, class _Compare, class _Allocator>
 template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer&
-__tree<_Tp, _Compare, _Allocator>::__find_equal(__parent_pointer& __parent, const _Key& __v) {
+__tree<_Tp, _Compare, _Allocator>::__find_equal(__end_node_pointer& __parent, const _Key& __v) {
   __node_pointer __nd           = __root();
   __node_base_pointer* __nd_ptr = __root_ptr();
   if (__nd != nullptr) {
@@ -1669,7 +1637,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(__parent_pointer& __parent, cons
           __nd_ptr = std::addressof(__nd->__left_);
           __nd     = static_cast<__node_pointer>(__nd->__left_);
         } else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __parent->__left_;
         }
       } else if (value_comp()(__nd->__value_, __v)) {
@@ -1677,16 +1645,16 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(__parent_pointer& __parent, cons
           __nd_ptr = std::addressof(__nd->__right_);
           __nd     = static_cast<__node_pointer>(__nd->__right_);
         } else {
-          __parent = static_cast<__parent_pointer>(__nd);
+          __parent = static_cast<__end_node_pointer>(__nd);
           return __nd->__right_;
         }
       } else {
-        __parent = static_cast<__parent_pointer>(__nd);
+        __parent = static_cast<__end_node_pointer>(__nd);
         return *__nd_ptr;
       }
     }
   }
-  __parent = static_cast<__parent_pointer>(__end_node());
+  __parent = __end_node();
   return __parent->__left_;
 }
 
@@ -1700,7 +1668,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(__parent_pointer& __parent, cons
 template <class _Tp, class _Compare, class _Allocator>
 template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Compare, _Allocator>::__find_equal(
-    const_iterator __hint, __parent_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v) {
+    const_iterator __hint, __end_node_pointer& __parent, __node_base_pointer& __dummy, const _Key& __v) {
   if (__hint == end() || value_comp()(__v, *__hint)) // check before
   {
     // __v < *__hint
@@ -1708,10 +1676,10 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
     if (__prior == begin() || value_comp()(*--__prior, __v)) {
       // *prev(__hint) < __v < *__hint
       if (__hint.__ptr_->__left_ == nullptr) {
-        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        __parent = __hint.__ptr_;
         return __parent->__left_;
       } else {
-        __parent = static_cast<__parent_pointer>(__prior.__ptr_);
+        __parent = __prior.__ptr_;
         return static_cast<__node_base_pointer>(__prior.__ptr_)->__right_;
       }
     }
@@ -1724,10 +1692,10 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
     if (__next == end() || value_comp()(__v, *__next)) {
       // *__hint < __v < *std::next(__hint)
       if (__hint.__get_np()->__right_ == nullptr) {
-        __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+        __parent = __hint.__ptr_;
         return static_cast<__node_base_pointer>(__hint.__ptr_)->__right_;
       } else {
-        __parent = static_cast<__parent_pointer>(__next.__ptr_);
+        __parent = __next.__ptr_;
         return __parent->__left_;
       }
     }
@@ -1735,14 +1703,14 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_base_pointer& __tree<_Tp, _Co
     return __find_equal(__parent, __v);
   }
   // else __v == *__hint
-  __parent = static_cast<__parent_pointer>(__hint.__ptr_);
+  __parent = __hint.__ptr_;
   __dummy  = static_cast<__node_base_pointer>(__hint.__ptr_);
   return __dummy;
 }
 
 template <class _Tp, class _Compare, class _Allocator>
 void __tree<_Tp, _Compare, _Allocator>::__insert_node_at(
-    __parent_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT {
+    __end_node_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT {
   __new_node->__left_   = nullptr;
   __new_node->__right_  = nullptr;
   __new_node->__parent_ = __parent;
@@ -1758,7 +1726,7 @@ template <class _Tp, class _Compare, class _Allocator>
 template <class _Key, class... _Args>
 pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
 __tree<_Tp, _Compare, _Allocator>::__emplace_unique_key_args(_Key const& __k, _Args&&... __args) {
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_equal(__parent, __k);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   bool __inserted              = false;
@@ -1776,7 +1744,7 @@ template <class _Key, class... _Args>
 pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
 __tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_key_args(
     const_iterator __p, _Key const& __k, _Args&&... __args) {
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer __dummy;
   __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __k);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
@@ -1806,7 +1774,7 @@ template <class... _Args>
 pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
 __tree<_Tp, _Compare, _Allocator>::__emplace_unique_impl(_Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_equal(__parent, __h->__value_);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   bool __inserted              = false;
@@ -1823,7 +1791,7 @@ template <class... _Args>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_impl(const_iterator __p, _Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer __dummy;
   __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __h->__value_);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
@@ -1839,7 +1807,7 @@ template <class... _Args>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__emplace_multi(_Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf_high(__parent, __h->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
   return iterator(static_cast<__node_pointer>(__h.release()));
@@ -1850,7 +1818,7 @@ template <class... _Args>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf(__p, __parent, __h->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
   return iterator(static_cast<__node_pointer>(__h.release()));
@@ -1859,7 +1827,7 @@ __tree<_Tp, _Compare, _Allocator>::__emplace_hint_multi(const_iterator __p, _Arg
 template <class _Tp, class _Compare, class _Allocator>
 pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
 __tree<_Tp, _Compare, _Allocator>::__node_assign_unique(const value_type& __v, __node_pointer __nd) {
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_equal(__parent, __v);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   bool __inserted              = false;
@@ -1875,7 +1843,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_assign_unique(const value_type& __v, _
 template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(__node_pointer __nd) {
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf_high(__parent, __nd->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
   return iterator(__nd);
@@ -1884,7 +1852,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(__node_pointer __nd) {
 template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(const_iterator __p, __node_pointer __nd) {
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf(__p, __parent, __nd->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
   return iterator(__nd);
@@ -1911,7 +1879,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(_NodeHandle&& __n
     return _InsertReturnType{end(), false, _NodeHandle()};
 
   __node_pointer __ptr = __nh.__ptr_;
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_equal(__parent, __ptr->__value_);
   if (__child != nullptr)
     return _InsertReturnType{iterator(static_cast<__node_pointer>(__child)), false, std::move(__nh)};
@@ -1929,7 +1897,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(const_iterator __
     return end();
 
   __node_pointer __ptr = __nh.__ptr_;
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer __dummy;
   __node_base_pointer& __child = __find_equal(__hint, __parent, __dummy, __ptr->__value_);
   __node_pointer __r           = static_cast<__node_pointer>(__child);
@@ -1965,7 +1933,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merg
 
   for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
     __node_pointer __src_ptr = __i.__get_np();
-    __parent_pointer __parent;
+    __end_node_pointer __parent;
     __node_base_pointer& __child = __find_equal(__parent, __src_ptr->__value_);
     ++__i;
     if (__child != nullptr)
@@ -1982,7 +1950,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(_NodeHandle&& __nh
   if (__nh.empty())
     return end();
   __node_pointer __ptr = __nh.__ptr_;
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf_high(__parent, __ptr->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
   __nh.__release_ptr();
@@ -1997,7 +1965,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(const_iterator __h
     return end();
 
   __node_pointer __ptr = __nh.__ptr_;
-  __parent_pointer __parent;
+  __end_node_pointer __parent;
   __node_base_pointer& __child = __find_leaf(__hint, __parent, __ptr->__value_);
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
   __nh.__release_ptr();
@@ -2011,7 +1979,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merg
 
   for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
     __node_pointer __src_ptr = __i.__get_np();
-    __parent_pointer __parent;
+    __end_node_pointer __parent;
     __node_base_pointer& __child = __find_leaf_high(__parent, __src_ptr->__value_);
     ++__i;
     __source.__remove_node_pointer(__src_ptr);
@@ -2026,7 +1994,7 @@ typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allo
   __node_pointer __np    = __p.__get_np();
   iterator __r           = __remove_node_pointer(__np);
   __node_allocator& __na = __node_alloc();
-  __node_traits::destroy(__na, std::addressof(const_cast<__node_value_type&>(*__p)));
+  __node_traits::destroy(__na, std::addressof(const_cast<value_type&>(*__p)));
   __node_traits::deallocate(__na, __np, 1);
   return __r;
 }
diff --git a/libcxx/include/__type_traits/add_lvalue_reference.h b/libcxx/include/__type_traits/add_lvalue_reference.h
deleted file mode 100644
index 42f91623f980..000000000000
--- a/libcxx/include/__type_traits/add_lvalue_reference.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
-
-#include <__config>
-#include <__type_traits/is_referenceable.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#if __has_builtin(__add_lvalue_reference) && !defined(_LIBCPP_COMPILER_GCC)
-
-template <class _Tp>
-using __add_lvalue_reference_t _LIBCPP_NODEBUG = __add_lvalue_reference(_Tp);
-
-#else
-
-template <class _Tp, bool = __is_referenceable_v<_Tp>>
-struct __add_lvalue_reference_impl {
-  using type _LIBCPP_NODEBUG = _Tp;
-};
-template <class _Tp >
-struct __add_lvalue_reference_impl<_Tp, true> {
-  using type _LIBCPP_NODEBUG = _Tp&;
-};
-
-template <class _Tp>
-using __add_lvalue_reference_t = typename __add_lvalue_reference_impl<_Tp>::type;
-
-#endif // __has_builtin(__add_lvalue_reference)
-
-template <class _Tp>
-struct _LIBCPP_NO_SPECIALIZATIONS add_lvalue_reference {
-  using type _LIBCPP_NODEBUG = __add_lvalue_reference_t<_Tp>;
-};
-
-#if _LIBCPP_STD_VER >= 14
-template <class _Tp>
-using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
-#endif
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___TYPE_TRAITS_ADD_LVALUE_REFERENCE_H
diff --git a/libcxx/include/__type_traits/add_pointer.h b/libcxx/include/__type_traits/add_pointer.h
index f37f9cf5a93c..fa9376699a74 100644
--- a/libcxx/include/__type_traits/add_pointer.h
+++ b/libcxx/include/__type_traits/add_pointer.h
@@ -20,10 +20,20 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer) && !defined(_LIBCPP_COMPILER_GCC)
+#if !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS)
 
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS add_pointer {
+  using type _LIBCPP_NODEBUG = __add_pointer(_Tp);
+};
+
+#  ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __add_pointer_t _LIBCPP_NODEBUG = typename add_pointer<_Tp>::type;
+#  else
 template <class _Tp>
 using __add_pointer_t _LIBCPP_NODEBUG = __add_pointer(_Tp);
+#  endif
 
 #else
 template <class _Tp, bool = __is_referenceable_v<_Tp> || is_void<_Tp>::value>
@@ -38,13 +48,13 @@ struct __add_pointer_impl<_Tp, false> {
 template <class _Tp>
 using __add_pointer_t = typename __add_pointer_impl<_Tp>::type;
 
-#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__add_pointer)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS add_pointer {
   using type _LIBCPP_NODEBUG = __add_pointer_t<_Tp>;
 };
 
+#endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS)
+
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
 using add_pointer_t = __add_pointer_t<_Tp>;
diff --git a/libcxx/include/__type_traits/add_rvalue_reference.h b/libcxx/include/__type_traits/add_reference.h
similarity index 51%
rename from libcxx/include/__type_traits/add_rvalue_reference.h
rename to libcxx/include/__type_traits/add_reference.h
index 33cc00180cc1..c6f5d63d26de 100644
--- a/libcxx/include/__type_traits/add_rvalue_reference.h
+++ b/libcxx/include/__type_traits/add_reference.h
@@ -6,11 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
-#define _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#ifndef _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
+#define _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
 
 #include <__config>
-#include <__type_traits/is_referenceable.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,32 +17,37 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__add_rvalue_reference) && !defined(_LIBCPP_COMPILER_GCC)
-
 template <class _Tp>
-using __add_rvalue_reference_t _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
-
-#else
-
-template <class _Tp, bool = __is_referenceable_v<_Tp> >
-struct __add_rvalue_reference_impl {
-  using type _LIBCPP_NODEBUG = _Tp;
-};
-template <class _Tp >
-struct __add_rvalue_reference_impl<_Tp, true> {
-  using type _LIBCPP_NODEBUG = _Tp&&;
+struct _LIBCPP_NO_SPECIALIZATIONS add_lvalue_reference {
+  using type _LIBCPP_NODEBUG = __add_lvalue_reference(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __add_lvalue_reference_t _LIBCPP_NODEBUG = typename add_lvalue_reference<_Tp>::type;
+#else
 template <class _Tp>
-using __add_rvalue_reference_t = typename __add_rvalue_reference_impl<_Tp>::type;
+using __add_lvalue_reference_t _LIBCPP_NODEBUG = __add_lvalue_reference(_Tp);
+#endif
 
-#endif // __has_builtin(__add_rvalue_reference)
+#if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
+#endif
 
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS add_rvalue_reference {
-  using type _LIBCPP_NODEBUG = __add_rvalue_reference_t<_Tp>;
+  using type _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
+template <class _Tp>
+using __add_rvalue_reference_t _LIBCPP_NODEBUG = typename add_rvalue_reference<_Tp>::type;
+#else
+template <class _Tp>
+using __add_rvalue_reference_t _LIBCPP_NODEBUG = __add_rvalue_reference(_Tp);
+#endif
+
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
 using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
@@ -51,4 +55,4 @@ using add_rvalue_reference_t = __add_rvalue_reference_t<_Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP___TYPE_TRAITS_ADD_RVALUE_REFERENCE_H
+#endif // _LIBCPP___TYPE_TRAITS_ADD_REFERENCE_H
diff --git a/libcxx/include/__type_traits/common_reference.h b/libcxx/include/__type_traits/common_reference.h
index 7df55f7fecfd..59badb64267d 100644
--- a/libcxx/include/__type_traits/common_reference.h
+++ b/libcxx/include/__type_traits/common_reference.h
@@ -10,6 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_COMMON_REFERENCE_H
 
 #include <__config>
+#include <__type_traits/add_pointer.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/copy_cv.h>
 #include <__type_traits/copy_cvref.h>
@@ -139,13 +140,17 @@ struct __common_reference_sub_bullet2 : __common_reference_sub_bullet3<_Tp, _Up>
 template <class _Tp, class _Up>
 struct __common_reference_sub_bullet1 : __common_reference_sub_bullet2<_Tp, _Up> {};
 
-// sub-bullet 1 - If T1 and T2 are reference types and COMMON-REF(T1, T2) is well-formed, then
-// the member typedef `type` denotes that type.
+// sub-bullet 1 - Let R be COMMON-REF(T1, T2). If T1 and T2 are reference types, R is well-formed, and
+// is_convertible_v<add_pointer_t<T1>, add_pointer_t<R>> && is_convertible_v<add_pointer_t<T2>, add_pointer_t<R>> is
+// true, then the member typedef type denotes R.
+
 template <class _Tp, class _Up>
 struct common_reference<_Tp, _Up> : __common_reference_sub_bullet1<_Tp, _Up> {};
 
 template <class _Tp, class _Up>
-  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; }
+  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; } &&
+           is_convertible_v<add_pointer_t<_Tp>, add_pointer_t<__common_ref_t<_Tp, _Up>>> &&
+           is_convertible_v<add_pointer_t<_Up>, add_pointer_t<__common_ref_t<_Tp, _Up>>>
 struct __common_reference_sub_bullet1<_Tp, _Up> {
   using type _LIBCPP_NODEBUG = __common_ref_t<_Tp, _Up>;
 };
diff --git a/libcxx/include/__type_traits/copy_cvref.h b/libcxx/include/__type_traits/copy_cvref.h
index 158e5a5d78bb..39002347d4a9 100644
--- a/libcxx/include/__type_traits/copy_cvref.h
+++ b/libcxx/include/__type_traits/copy_cvref.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_COPY_CVREF_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/copy_cv.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/decay.h b/libcxx/include/__type_traits/decay.h
index 1e663fdcd0ed..661fba53dfaa 100644
--- a/libcxx/include/__type_traits/decay.h
+++ b/libcxx/include/__type_traits/decay.h
@@ -10,14 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_DECAY_H
 
 #include <__config>
-#include <__type_traits/add_pointer.h>
-#include <__type_traits/conditional.h>
-#include <__type_traits/is_array.h>
-#include <__type_traits/is_function.h>
-#include <__type_traits/is_referenceable.h>
-#include <__type_traits/remove_cv.h>
-#include <__type_traits/remove_extent.h>
-#include <__type_traits/remove_reference.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -25,42 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__decay) && !defined(_LIBCPP_COMPILER_GCC)
-template <class _Tp>
-using __decay_t _LIBCPP_NODEBUG = __decay(_Tp);
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS decay {
-  using type _LIBCPP_NODEBUG = __decay_t<_Tp>;
-};
-
-#else
-template <class _Up, bool>
-struct __decay {
-  using type _LIBCPP_NODEBUG = __remove_cv_t<_Up>;
-};
-
-template <class _Up>
-struct __decay<_Up, true> {
-public:
-  using type _LIBCPP_NODEBUG =
-      __conditional_t<is_array<_Up>::value,
-                      __add_pointer_t<__remove_extent_t<_Up> >,
-                      __conditional_t<is_function<_Up>::value, typename add_pointer<_Up>::type, __remove_cv_t<_Up> > >;
+  using type _LIBCPP_NODEBUG = __decay(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-struct decay {
-private:
-  using _Up _LIBCPP_NODEBUG = __libcpp_remove_reference_t<_Tp>;
-
-public:
-  using type _LIBCPP_NODEBUG = typename __decay<_Up, __is_referenceable_v<_Up> >::type;
-};
-
+using __decay_t _LIBCPP_NODEBUG = typename decay<_Tp>::type;
+#else
 template <class _Tp>
-using __decay_t = typename decay<_Tp>::type;
-#endif // __has_builtin(__decay)
+using __decay_t _LIBCPP_NODEBUG = __decay(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/has_unique_object_representation.h b/libcxx/include/__type_traits/has_unique_object_representation.h
index ba81be6aa19b..af9abfe81d41 100644
--- a/libcxx/include/__type_traits/has_unique_object_representation.h
+++ b/libcxx/include/__type_traits/has_unique_object_representation.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/remove_all_extents.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -23,12 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS has_unique_object_representations
-    // TODO: We work around a Clang and GCC bug in __has_unique_object_representations by using remove_all_extents
-    //       even though it should not be necessary. This was reported to the compilers:
-    //         - Clang: https://github.com/llvm/llvm-project/issues/95311
-    //         - GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115476
-    //       remove_all_extents_t can be removed once all the compilers we support have fixed this bug.
-    : public integral_constant<bool, __has_unique_object_representations(remove_all_extents_t<_Tp>)> {};
+    : integral_constant<bool, __has_unique_object_representations(_Tp)> {};
 
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool has_unique_object_representations_v =
diff --git a/libcxx/include/__type_traits/invoke.h b/libcxx/include/__type_traits/invoke.h
index ccf86209d229..ac38b32b0cd7 100644
--- a/libcxx/include/__type_traits/invoke.h
+++ b/libcxx/include/__type_traits/invoke.h
@@ -22,6 +22,7 @@
 #include <__type_traits/is_same.h>
 #include <__type_traits/is_void.h>
 #include <__type_traits/nat.h>
+#include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 
@@ -61,6 +62,112 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+#if __has_builtin(__builtin_invoke)
+
+template <class... _Args>
+using __invoke_result_t _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...));
+
+template <class, class... _Args>
+struct __invoke_result_impl {};
+
+template <class... _Args>
+struct __invoke_result_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> {
+  using type _LIBCPP_NODEBUG = __invoke_result_t<_Args...>;
+};
+
+template <class... _Args>
+using __invoke_result _LIBCPP_NODEBUG = __invoke_result_impl<void, _Args...>;
+
+template <class... _Args>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __invoke_result_t<_Args...> __invoke(_Args&&... __args)
+    _NOEXCEPT_(noexcept(__builtin_invoke(std::forward<_Args>(__args)...))) {
+  return __builtin_invoke(std::forward<_Args>(__args)...);
+}
+
+template <class _Void, class... _Args>
+inline const bool __is_invocable_impl = false;
+
+template <class... _Args>
+inline const bool __is_invocable_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> = true;
+
+template <class... _Args>
+inline const bool __is_invocable_v = __is_invocable_impl<void, _Args...>;
+
+template <class... _Args>
+struct __is_invocable : integral_constant<bool, __is_invocable_v<_Args...> > {};
+
+template <class _Ret, bool, class... _Args>
+inline const bool __is_invocable_r_impl = false;
+
+template <class _Ret, class... _Args>
+inline const bool __is_invocable_r_impl<_Ret, true, _Args...> =
+    __is_core_convertible<__invoke_result_t<_Args...>, _Ret>::value || is_void<_Ret>::value;
+
+template <class _Ret, class... _Args>
+inline const bool __is_invocable_r_v = __is_invocable_r_impl<_Ret, __is_invocable_v<_Args...>, _Args...>;
+
+template <bool __is_invocable, class... _Args>
+inline const bool __is_nothrow_invocable_impl = false;
+
+template <class... _Args>
+inline const bool __is_nothrow_invocable_impl<true, _Args...> = noexcept(__builtin_invoke(std::declval<_Args>()...));
+
+template <class... _Args>
+inline const bool __is_nothrow_invocable_v = __is_nothrow_invocable_impl<__is_invocable_v<_Args...>, _Args...>;
+
+template <bool __is_invocable, class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_impl = false;
+
+template <class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_impl<true, _Ret, _Args...> =
+    __is_nothrow_core_convertible_v<__invoke_result_t<_Args...>, _Ret> || is_void<_Ret>::value;
+
+template <class _Ret, class... _Args>
+inline const bool __is_nothrow_invocable_r_v =
+    __is_nothrow_invocable_r_impl<__is_nothrow_invocable_v<_Args...>, _Ret, _Args...>;
+
+#  if _LIBCPP_STD_VER >= 17
+
+// is_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_invocable : bool_constant<__is_invocable_v<_Fn, _Args...> > {};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_invocable_r : bool_constant<__is_invocable_r_v<_Ret, _Fn, _Args...>> {};
+
+template <class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_invocable_v = __is_invocable_v<_Fn, _Args...>;
+
+template <class _Ret, class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_invocable_r_v = is_invocable_r<_Ret, _Fn, _Args...>::value;
+
+// is_nothrow_invocable
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_nothrow_invocable : bool_constant<__is_nothrow_invocable_v<_Fn, _Args...> > {};
+
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS is_nothrow_invocable_r
+    : integral_constant<bool, __is_nothrow_invocable_r_v<_Ret, _Fn, _Args...>> {};
+
+template <class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_nothrow_invocable_v = __is_nothrow_invocable_v<_Fn, _Args...>;
+
+template <class _Ret, class _Fn, class... _Args>
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_nothrow_invocable_r_v =
+    __is_nothrow_invocable_r_v<_Ret, _Fn, _Args...>;
+
+template <class _Fn, class... _Args>
+struct _LIBCPP_NO_SPECIALIZATIONS invoke_result : __invoke_result<_Fn, _Args...> {};
+
+template <class _Fn, class... _Args>
+using invoke_result_t = __invoke_result_t<_Fn, _Args...>;
+
+#  endif // _LIBCPP_STD_VER >= 17
+
+#else // __has_builtin(__builtin_invoke)
+
 template <class _DecayedFp>
 struct __member_pointer_class_type {};
 
@@ -211,21 +318,21 @@ struct __nothrow_invokable_r_imp<true, false, _Ret, _Fp, _Args...> {
   template <class _Tp>
   static void __test_noexcept(_Tp) _NOEXCEPT;
 
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   static const bool value = false;
-#else
+#  else
   static const bool value =
       noexcept(_ThisT::__test_noexcept<_Ret>(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...)));
-#endif
+#  endif
 };
 
 template <class _Ret, class _Fp, class... _Args>
 struct __nothrow_invokable_r_imp<true, true, _Ret, _Fp, _Args...> {
-#ifdef _LIBCPP_CXX03_LANG
+#  ifdef _LIBCPP_CXX03_LANG
   static const bool value = false;
-#else
+#  else
   static const bool value = noexcept(std::__invoke(std::declval<_Fp>(), std::declval<_Args>()...));
-#endif
+#  endif
 };
 
 template <class _Ret, class _Fp, class... _Args>
@@ -236,22 +343,6 @@ template <class _Fp, class... _Args>
 using __nothrow_invokable _LIBCPP_NODEBUG =
     __nothrow_invokable_r_imp<__is_invocable<_Fp, _Args...>::value, true, void, _Fp, _Args...>;
 
-template <class _Ret, bool = is_void<_Ret>::value>
-struct __invoke_void_return_wrapper {
-  template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
-    return std::__invoke(std::forward<_Args>(__args)...);
-  }
-};
-
-template <class _Ret>
-struct __invoke_void_return_wrapper<_Ret, true> {
-  template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
-    std::__invoke(std::forward<_Args>(__args)...);
-  }
-};
-
 template <class _Func, class... _Args>
 inline const bool __is_invocable_v = __is_invocable<_Func, _Args...>::value;
 
@@ -268,12 +359,7 @@ struct __invoke_result
 template <class _Func, class... _Args>
 using __invoke_result_t _LIBCPP_NODEBUG = typename __invoke_result<_Func, _Args...>::type;
 
-template <class _Ret, class... _Args>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ret __invoke_r(_Args&&... __args) {
-  return __invoke_void_return_wrapper<_Ret>::__call(std::forward<_Args>(__args)...);
-}
-
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 
 // is_invocable
 
@@ -311,7 +397,30 @@ struct _LIBCPP_NO_SPECIALIZATIONS invoke_result : __invoke_result<_Fn, _Args...>
 template <class _Fn, class... _Args>
 using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
 
-#endif // _LIBCPP_STD_VER >= 17
+#  endif // _LIBCPP_STD_VER >= 17
+
+#endif // __has_builtin(__builtin_invoke_r)
+
+template <class _Ret, bool = is_void<_Ret>::value>
+struct __invoke_void_return_wrapper {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
+    return std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Ret>
+struct __invoke_void_return_wrapper<_Ret, true> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
+    std::__invoke(std::forward<_Args>(__args)...);
+  }
+};
+
+template <class _Ret, class... _Args>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Ret __invoke_r(_Args&&... __args) {
+  return __invoke_void_return_wrapper<_Ret>::__call(std::forward<_Args>(__args)...);
+}
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_array.h b/libcxx/include/__type_traits/is_array.h
index 1d94370378c6..e734d1a3043e 100644
--- a/libcxx/include/__type_traits/is_array.h
+++ b/libcxx/include/__type_traits/is_array.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_IS_ARRAY_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -19,32 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_array) &&                                                                                       \
-    (!defined(_LIBCPP_COMPILER_CLANG_BASED) || (defined(_LIBCPP_CLANG_VER) && _LIBCPP_CLANG_VER >= 1900))
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_array : _BoolConstant<__is_array(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_array_v = __is_array(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_array : public false_type {};
-template <class _Tp>
-struct is_array<_Tp[]> : public true_type {};
-template <class _Tp, size_t _Np>
-struct is_array<_Tp[_Np]> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_array_v = is_array<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_array)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_assignable.h b/libcxx/include/__type_traits/is_assignable.h
index 253e86ba774e..da9080db3b76 100644
--- a/libcxx/include/__type_traits/is_assignable.h
+++ b/libcxx/include/__type_traits/is_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_bounded_array.h b/libcxx/include/__type_traits/is_bounded_array.h
index aec488826ec4..8a41e07aa019 100644
--- a/libcxx/include/__type_traits/is_bounded_array.h
+++ b/libcxx/include/__type_traits/is_bounded_array.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_IS_BOUNDED_ARRAY_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -19,26 +18,16 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class>
-inline const bool __is_bounded_array_v = false;
-template <class _Tp, size_t _Np>
-inline const bool __is_bounded_array_v<_Tp[_Np]> = true;
+template <class _Tp>
+inline const bool __is_bounded_array_v = __is_bounded_array(_Tp);
 
 #if _LIBCPP_STD_VER >= 20
 
-template <class>
-struct _LIBCPP_NO_SPECIALIZATIONS is_bounded_array : false_type {};
-
-_LIBCPP_DIAGNOSTIC_PUSH
-#  if __has_warning("-Winvalid-specialization")
-_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winvalid-specialization")
-#  endif
-template <class _Tp, size_t _Np>
-struct is_bounded_array<_Tp[_Np]> : true_type {};
-_LIBCPP_DIAGNOSTIC_POP
+template <class _Tp>
+struct _LIBCPP_NO_SPECIALIZATIONS is_bounded_array : bool_constant<__is_bounded_array(_Tp)> {};
 
 template <class _Tp>
-_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_bounded_array_v = is_bounded_array<_Tp>::value;
+_LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_bounded_array_v = __is_bounded_array(_Tp);
 
 #endif
 
diff --git a/libcxx/include/__type_traits/is_const.h b/libcxx/include/__type_traits/is_const.h
index 4ec354d7f9bf..85cc32bff1ea 100644
--- a/libcxx/include/__type_traits/is_const.h
+++ b/libcxx/include/__type_traits/is_const.h
@@ -18,29 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_const)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_const : _BoolConstant<__is_const(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_const_v = __is_const(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_const : public false_type {};
-template <class _Tp>
-struct is_const<_Tp const> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_const_v = is_const<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_const)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_constructible.h b/libcxx/include/__type_traits/is_constructible.h
index feafb70b1f68..28d3048719b0 100644
--- a/libcxx/include/__type_traits/is_constructible.h
+++ b/libcxx/include/__type_traits/is_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_IS_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_core_convertible.h b/libcxx/include/__type_traits/is_core_convertible.h
index ca3a346c17cd..0b8e94e101db 100644
--- a/libcxx/include/__type_traits/is_core_convertible.h
+++ b/libcxx/include/__type_traits/is_core_convertible.h
@@ -37,6 +37,15 @@ concept __core_convertible_to = __is_core_convertible<_Tp, _Up>::value;
 
 #endif // _LIBCPP_STD_VER >= 20
 
+template <class _Tp, class _Up, bool = __is_core_convertible<_Tp, _Up>::value>
+inline const bool __is_nothrow_core_convertible_v = false;
+
+#ifndef _LIBCPP_CXX03_LANG
+template <class _Tp, class _Up>
+inline const bool __is_nothrow_core_convertible_v<_Tp, _Up, true> =
+    noexcept(static_cast<void (*)(_Up) noexcept>(0)(static_cast<_Tp (*)() noexcept>(0)()));
+#endif
+
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_IS_CORE_CONVERTIBLE_H
diff --git a/libcxx/include/__type_traits/is_nothrow_assignable.h b/libcxx/include/__type_traits/is_nothrow_assignable.h
index 903dead4435e..15ad047c566e 100644
--- a/libcxx/include/__type_traits/is_nothrow_assignable.h
+++ b/libcxx/include/__type_traits/is_nothrow_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_nothrow_constructible.h b/libcxx/include/__type_traits/is_nothrow_constructible.h
index bd14c1c40516..bc4c2ee45b9f 100644
--- a/libcxx/include/__type_traits/is_nothrow_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_pointer.h b/libcxx/include/__type_traits/is_pointer.h
index 3c58656e0e61..1c3a93d1256c 100644
--- a/libcxx/include/__type_traits/is_pointer.h
+++ b/libcxx/include/__type_traits/is_pointer.h
@@ -11,7 +11,6 @@
 
 #include <__config>
 #include <__type_traits/integral_constant.h>
-#include <__type_traits/remove_cv.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -19,8 +18,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_pointer)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_pointer : _BoolConstant<__is_pointer(_Tp)> {};
 
@@ -29,36 +26,6 @@ template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_pointer_v = __is_pointer(_Tp);
 #  endif
 
-#else // __has_builtin(__is_pointer)
-
-template <class _Tp>
-struct __libcpp_is_pointer : false_type {};
-template <class _Tp>
-struct __libcpp_is_pointer<_Tp*> : true_type {};
-
-template <class _Tp>
-struct __libcpp_remove_objc_qualifiers {
-  typedef _Tp type;
-};
-#  if __has_feature(objc_arc)
-// clang-format off
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __autoreleasing> { typedef _Tp type; };
-template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __unsafe_unretained> { typedef _Tp type; };
-// clang-format on
-#  endif
-
-template <class _Tp>
-struct is_pointer : __libcpp_is_pointer<typename __libcpp_remove_objc_qualifiers<__remove_cv_t<_Tp> >::type> {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_pointer)
-
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___TYPE_TRAITS_IS_POINTER_H
diff --git a/libcxx/include/__type_traits/is_swappable.h b/libcxx/include/__type_traits/is_swappable.h
index 2afe784e8d89..5c060402f16d 100644
--- a/libcxx/include/__type_traits/is_swappable.h
+++ b/libcxx/include/__type_traits/is_swappable.h
@@ -11,7 +11,7 @@
 
 #include <__config>
 #include <__cstddef/size_t.h>
-#include <__type_traits/add_lvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_assignable.h>
diff --git a/libcxx/include/__type_traits/is_trivially_assignable.h b/libcxx/include/__type_traits/is_trivially_assignable.h
index 6548de991be4..c95ed9dd57be 100644
--- a/libcxx/include/__type_traits/is_trivially_assignable.h
+++ b/libcxx/include/__type_traits/is_trivially_assignable.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_ASSIGNABLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_trivially_constructible.h b/libcxx/include/__type_traits/is_trivially_constructible.h
index 4e36c910cc04..820b04287c51 100644
--- a/libcxx/include/__type_traits/is_trivially_constructible.h
+++ b/libcxx/include/__type_traits/is_trivially_constructible.h
@@ -10,8 +10,7 @@
 #define _LIBCPP___TYPE_TRAITS_IS_TRIVIALLY_CONSTRUCTIBLE_H
 
 #include <__config>
-#include <__type_traits/add_lvalue_reference.h>
-#include <__type_traits/add_rvalue_reference.h>
+#include <__type_traits/add_reference.h>
 #include <__type_traits/integral_constant.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/__type_traits/is_volatile.h b/libcxx/include/__type_traits/is_volatile.h
index 22e9c297c800..c654ebba9299 100644
--- a/libcxx/include/__type_traits/is_volatile.h
+++ b/libcxx/include/__type_traits/is_volatile.h
@@ -18,29 +18,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_volatile)
-
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS is_volatile : _BoolConstant<__is_volatile(_Tp)> {};
 
-#  if _LIBCPP_STD_VER >= 17
+#if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_NO_SPECIALIZATIONS inline constexpr bool is_volatile_v = __is_volatile(_Tp);
-#  endif
-
-#else
-
-template <class _Tp>
-struct is_volatile : public false_type {};
-template <class _Tp>
-struct is_volatile<_Tp volatile> : public true_type {};
-
-#  if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_volatile_v = is_volatile<_Tp>::value;
-#  endif
-
-#endif // __has_builtin(__is_volatile)
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/remove_all_extents.h b/libcxx/include/__type_traits/remove_all_extents.h
index 2c7aea2bfd18..a9aaad488078 100644
--- a/libcxx/include/__type_traits/remove_all_extents.h
+++ b/libcxx/include/__type_traits/remove_all_extents.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_ALL_EXTENTS_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,31 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__remove_all_extents) && !defined(_LIBCPP_COMPILER_GCC)
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_all_extents {
   using type _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-using __remove_all_extents_t _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
+using __remove_all_extents_t _LIBCPP_NODEBUG = typename remove_all_extents<_Tp>::type;
 #else
 template <class _Tp>
-struct remove_all_extents {
-  typedef _Tp type;
-};
-template <class _Tp>
-struct remove_all_extents<_Tp[]> {
-  typedef typename remove_all_extents<_Tp>::type type;
-};
-template <class _Tp, size_t _Np>
-struct remove_all_extents<_Tp[_Np]> {
-  typedef typename remove_all_extents<_Tp>::type type;
-};
-
-template <class _Tp>
-using __remove_all_extents_t = typename remove_all_extents<_Tp>::type;
-#endif // __has_builtin(__remove_all_extents)
+using __remove_all_extents_t _LIBCPP_NODEBUG = __remove_all_extents(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/remove_cvref.h b/libcxx/include/__type_traits/remove_cvref.h
index 760deddca946..c568e8df3099 100644
--- a/libcxx/include/__type_traits/remove_cvref.h
+++ b/libcxx/include/__type_traits/remove_cvref.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_CVREF_H
 
 #include <__config>
-#include <__type_traits/is_same.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -31,9 +30,6 @@ template <class _Tp>
 using __remove_cvref_t _LIBCPP_NODEBUG = __remove_cvref(_Tp);
 #endif // __has_builtin(__remove_cvref)
 
-template <class _Tp, class _Up>
-using __is_same_uncvref _LIBCPP_NODEBUG = _IsSame<__remove_cvref_t<_Tp>, __remove_cvref_t<_Up> >;
-
 #if _LIBCPP_STD_VER >= 20
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_cvref {
diff --git a/libcxx/include/__type_traits/remove_extent.h b/libcxx/include/__type_traits/remove_extent.h
index 7745af14bdb1..108b867bd0b4 100644
--- a/libcxx/include/__type_traits/remove_extent.h
+++ b/libcxx/include/__type_traits/remove_extent.h
@@ -10,7 +10,6 @@
 #define _LIBCPP___TYPE_TRAITS_REMOVE_EXTENT_H
 
 #include <__config>
-#include <__cstddef/size_t.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -18,31 +17,18 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__remove_extent) && !defined(_LIBCPP_COMPILER_GCC)
 template <class _Tp>
 struct _LIBCPP_NO_SPECIALIZATIONS remove_extent {
   using type _LIBCPP_NODEBUG = __remove_extent(_Tp);
 };
 
+#ifdef _LIBCPP_COMPILER_GCC
 template <class _Tp>
-using __remove_extent_t _LIBCPP_NODEBUG = __remove_extent(_Tp);
+using __remove_extent_t _LIBCPP_NODEBUG = typename remove_extent<_Tp>::type;
 #else
 template <class _Tp>
-struct remove_extent {
-  typedef _Tp type;
-};
-template <class _Tp>
-struct remove_extent<_Tp[]> {
-  typedef _Tp type;
-};
-template <class _Tp, size_t _Np>
-struct remove_extent<_Tp[_Np]> {
-  typedef _Tp type;
-};
-
-template <class _Tp>
-using __remove_extent_t = typename remove_extent<_Tp>::type;
-#endif // __has_builtin(__remove_extent)
+using __remove_extent_t _LIBCPP_NODEBUG = __remove_extent(_Tp);
+#endif
 
 #if _LIBCPP_STD_VER >= 14
 template <class _Tp>
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 6be476e2b69d..d109f27af58d 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -136,6 +136,7 @@ template <size_t N> struct hash<std::bitset<N>>;
 #  include <__algorithm/fill.h>
 #  include <__algorithm/fill_n.h>
 #  include <__algorithm/find.h>
+#  include <__algorithm/min.h>
 #  include <__assert>
 #  include <__bit/countr.h>
 #  include <__bit/invert_if.h>
@@ -146,7 +147,11 @@ template <size_t N> struct hash<std::bitset<N>>;
 #  include <__functional/hash.h>
 #  include <__functional/identity.h>
 #  include <__functional/unary_function.h>
+#  include <__tuple/tuple_indices.h>
+#  include <__type_traits/enable_if.h>
+#  include <__type_traits/integral_constant.h>
 #  include <__type_traits/is_char_like_type.h>
+#  include <__utility/integer_sequence.h>
 #  include <climits>
 #  include <stdexcept>
 #  include <string_view>
@@ -220,11 +225,42 @@ protected:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void operator^=(const __bitset& __v) _NOEXCEPT;
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void flip() _NOEXCEPT;
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const {
-    return to_ulong(integral_constant < bool, _Size< sizeof(unsigned long) * CHAR_BIT>());
+    if _LIBCPP_CONSTEXPR (_Size > sizeof(unsigned long) * CHAR_BIT) {
+      if (auto __e = __make_iter(_Size); std::find(__make_iter(sizeof(unsigned long) * CHAR_BIT), __e, true) != __e)
+        std::__throw_overflow_error("__bitset<_N_words, _Size>::to_ulong overflow error");
+    }
+
+    static_assert(sizeof(__storage_type) >= sizeof(unsigned long),
+                  "libc++ only supports platforms where sizeof(size_t) >= sizeof(unsigned long), such as 32-bit and "
+                  "64-bit platforms. If you're interested in supporting a platform where that is not the case, please "
+                  "contact the libc++ developers.");
+    return static_cast<unsigned long>(__first_[0]);
   }
+
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
-    return to_ullong(integral_constant < bool, _Size< sizeof(unsigned long long) * CHAR_BIT>());
+    // Check for overflow if _Size does not fit in unsigned long long
+    if _LIBCPP_CONSTEXPR (_Size > sizeof(unsigned long long) * CHAR_BIT) {
+      if (auto __e = __make_iter(_Size);
+          std::find(__make_iter(sizeof(unsigned long long) * CHAR_BIT), __e, true) != __e)
+        std::__throw_overflow_error("__bitset<_N_words, _Size>::to_ullong overflow error");
+    }
+
+    // At this point, the effective bitset size (excluding leading zeros) fits in unsigned long long
+
+    if _LIBCPP_CONSTEXPR (sizeof(__storage_type) >= sizeof(unsigned long long)) {
+      // If __storage_type is at least as large as unsigned long long, the result spans only one word
+      return static_cast<unsigned long long>(__first_[0]);
+    } else {
+      // Otherwise, the result spans multiple words which are concatenated
+      const size_t __ull_words = (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1;
+      const size_t __n_words   = _N_words < __ull_words ? _N_words : __ull_words;
+      unsigned long long __r   = static_cast<unsigned long long>(__first_[0]);
+      for (size_t __i = 1; __i < __n_words; ++__i)
+        __r |= static_cast<unsigned long long>(__first_[__i]) << (__bits_per_word * __i);
+      return __r;
+    }
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool all() const _NOEXCEPT { return !__scan_bits(__bit_not()); }
@@ -255,16 +291,6 @@ private:
       return ~__x;
     }
   };
-#  ifdef _LIBCPP_CXX03_LANG
-  void __init(unsigned long long __v, false_type) _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI void __init(unsigned long long __v, true_type) _NOEXCEPT;
-#  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(false_type) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong(true_type) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(false_type) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type, false_type) const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong(true_type, true_type) const;
 
   template <typename _Proj>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool __scan_bits(_Proj __proj) const _NOEXCEPT {
@@ -282,6 +308,15 @@ private:
     }
     return false;
   }
+
+#  ifdef _LIBCPP_CXX03_LANG
+  void __init(unsigned long long __v, false_type) _NOEXCEPT;
+  _LIBCPP_HIDE_FROM_ABI void __init(unsigned long long __v, true_type) _NOEXCEPT;
+#  else
+  template <size_t... _Indices>
+  _LIBCPP_HIDE_FROM_ABI constexpr __bitset(unsigned long long __v, std::__tuple_indices<_Indices...>) _NOEXCEPT
+      : __first_{static_cast<__storage_type>(__v >> (_Indices * __bits_per_word))...} {}
+#  endif // _LIBCPP_CXX03_LANG
 };
 
 template <size_t _N_words, size_t _Size>
@@ -316,21 +351,15 @@ inline _LIBCPP_HIDE_FROM_ABI void __bitset<_N_words, _Size>::__init(unsigned lon
 template <size_t _N_words, size_t _Size>
 inline _LIBCPP_CONSTEXPR __bitset<_N_words, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
 #  ifndef _LIBCPP_CXX03_LANG
-#    if __SIZEOF_SIZE_T__ == 8
-    : __first_{__v}
-#    elif __SIZEOF_SIZE_T__ == 4
-    : __first_{static_cast<__storage_type>(__v),
-               _Size >= 2 * __bits_per_word
-                   ? static_cast<__storage_type>(__v >> __bits_per_word)
-                   : static_cast<__storage_type>((__v >> __bits_per_word) &
-                                                 (__storage_type(1) << (_Size - __bits_per_word)) - 1)}
-#    else
-#      error This constructor has not been ported to this platform
-#    endif
+    : __bitset(__v,
+               std::__make_indices_imp< (_N_words < (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1)
+                                            ? _N_words
+                                            : (sizeof(unsigned long long) - 1) / sizeof(__storage_type) + 1,
+                                        0>{})
 #  endif
 {
 #  ifdef _LIBCPP_CXX03_LANG
-  __init(__v, integral_constant<bool, sizeof(unsigned long long) <= sizeof(__storage_type)>());
+  __init(__v, _BoolConstant<sizeof(unsigned long long) <= sizeof(__storage_type)>());
 #  endif
 }
 
@@ -369,58 +398,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void __bitset<_N_words, _Siz
     *__p ^= (__storage_type(1) << __n) - 1;
 }
 
-template <size_t _N_words, size_t _Size>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long
-__bitset<_N_words, _Size>::to_ulong(false_type) const {
-  __const_iterator __e = __make_iter(_Size);
-  __const_iterator __i = std::find(__make_iter(sizeof(unsigned long) * CHAR_BIT), __e, true);
-  if (__i != __e)
-    std::__throw_overflow_error("bitset to_ulong overflow error");
-
-  return __first_[0];
-}
-
-template <size_t _N_words, size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long
-__bitset<_N_words, _Size>::to_ulong(true_type) const {
-  return __first_[0];
-}
-
-template <size_t _N_words, size_t _Size>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
-__bitset<_N_words, _Size>::to_ullong(false_type) const {
-  __const_iterator __e = __make_iter(_Size);
-  __const_iterator __i = std::find(__make_iter(sizeof(unsigned long long) * CHAR_BIT), __e, true);
-  if (__i != __e)
-    std::__throw_overflow_error("bitset to_ullong overflow error");
-
-  return to_ullong(true_type());
-}
-
-template <size_t _N_words, size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
-__bitset<_N_words, _Size>::to_ullong(true_type) const {
-  return to_ullong(true_type(), integral_constant<bool, sizeof(__storage_type) < sizeof(unsigned long long)>());
-}
-
-template <size_t _N_words, size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
-__bitset<_N_words, _Size>::to_ullong(true_type, false_type) const {
-  return __first_[0];
-}
-
-template <size_t _N_words, size_t _Size>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long
-__bitset<_N_words, _Size>::to_ullong(true_type, true_type) const {
-  unsigned long long __r = __first_[0];
-  _LIBCPP_DIAGNOSTIC_PUSH
-  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wshift-count-overflow")
-  for (size_t __i = 1; __i < sizeof(unsigned long long) / sizeof(__storage_type); ++__i)
-    __r |= static_cast<unsigned long long>(__first_[__i]) << (sizeof(__storage_type) * CHAR_BIT);
-  _LIBCPP_DIAGNOSTIC_POP
-  return __r;
-}
-
 template <size_t _N_words, size_t _Size>
 inline size_t __bitset<_N_words, _Size>::__hash_code() const _NOEXCEPT {
   size_t __h = 0;
@@ -479,8 +456,25 @@ protected:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void flip() _NOEXCEPT;
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const {
+    if _LIBCPP_CONSTEXPR (_Size > sizeof(unsigned long) * CHAR_BIT) {
+      if (auto __e = __make_iter(_Size); std::find(__make_iter(sizeof(unsigned long) * CHAR_BIT), __e, true) != __e)
+        __throw_overflow_error("__bitset<1, _Size>::to_ulong overflow error");
+    }
+    return static_cast<unsigned long>(__first_);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
+    // If _Size exceeds the size of unsigned long long, check for overflow
+    if _LIBCPP_CONSTEXPR (_Size > sizeof(unsigned long long) * CHAR_BIT) {
+      if (auto __e = __make_iter(_Size);
+          std::find(__make_iter(sizeof(unsigned long long) * CHAR_BIT), __e, true) != __e)
+        __throw_overflow_error("__bitset<1, _Size>::to_ullong overflow error");
+    }
+
+    // If _Size fits or no overflow, directly cast to unsigned long long
+    return static_cast<unsigned long long>(__first_);
+  }
 
   template <bool _Sparse, class _CharT, class _Traits, class _Allocator>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
@@ -507,8 +501,10 @@ inline _LIBCPP_CONSTEXPR __bitset<1, _Size>::__bitset() _NOEXCEPT : __first_(0)
 
 template <size_t _Size>
 inline _LIBCPP_CONSTEXPR __bitset<1, _Size>::__bitset(unsigned long long __v) _NOEXCEPT
-    : __first_(_Size == __bits_per_word ? static_cast<__storage_type>(__v)
-                                        : static_cast<__storage_type>(__v) & ((__storage_type(1) << _Size) - 1)) {}
+    // TODO: We must refer to __bits_per_word in order to work around an issue with the GDB pretty-printers.
+    //       Without it, the pretty-printers complain about a missing __bits_per_word member. This needs to
+    //       be investigated further.
+    : __first_(_Size == __bits_per_word ? static_cast<__storage_type>(__v) : static_cast<__storage_type>(__v)) {}
 
 template <size_t _Size>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void
@@ -533,16 +529,6 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void __bitset<1, _Siz
   __first_ ^= ~__storage_type(0) >> (__bits_per_word - _Size);
 }
 
-template <size_t _Size>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long __bitset<1, _Size>::to_ulong() const {
-  return __first_;
-}
-
-template <size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long __bitset<1, _Size>::to_ullong() const {
-  return __first_;
-}
-
 template <size_t _Size>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bool __bitset<1, _Size>::all() const _NOEXCEPT {
   __storage_type __m = ~__storage_type(0) >> (__bits_per_word - _Size);
@@ -633,8 +619,6 @@ class bitset : private __bitset<_Size == 0 ? 0 : (_Size - 1) / (sizeof(size_t) *
 public:
   static const unsigned __n_words = _Size == 0 ? 0 : (_Size - 1) / (sizeof(size_t) * CHAR_BIT) + 1;
   typedef __bitset<__n_words, _Size> __base;
-
-public:
   typedef typename __base::reference reference;
   typedef typename __base::__const_reference __const_reference;
 
@@ -713,8 +697,10 @@ public:
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p < _Size, "bitset::operator[] index out of bounds");
     return __base::__make_ref(__p);
   }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const;
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long to_ulong() const { return __base::to_ulong(); }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long to_ullong() const {
+    return __base::to_ullong();
+  }
   template <class _CharT, class _Traits, class _Allocator>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
   to_string(_CharT __zero = _CharT('0'), _CharT __one = _CharT('1')) const;
@@ -850,16 +836,6 @@ _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 bitset<_Size>& bitset<_Size>
   return *this;
 }
 
-template <size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long bitset<_Size>::to_ulong() const {
-  return __base::to_ulong();
-}
-
-template <size_t _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unsigned long long bitset<_Size>::to_ullong() const {
-  return __base::to_ullong();
-}
-
 template <size_t _Size>
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 basic_string<_CharT, _Traits, _Allocator>
diff --git a/libcxx/include/complex b/libcxx/include/complex
index e9baec04d946..d8ec3d95c10e 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -260,6 +260,7 @@ template<class T> complex<T> tanh (const complex<T>&);
 #  include <__cxx03/complex>
 #else
 #  include <__config>
+#  include <__cstddef/size_t.h>
 #  include <__fwd/complex.h>
 #  include <__fwd/tuple.h>
 #  include <__tuple/tuple_element.h>
diff --git a/libcxx/include/experimental/iterator b/libcxx/include/experimental/iterator
index 565bb83903ac..0ed95188d771 100644
--- a/libcxx/include/experimental/iterator
+++ b/libcxx/include/experimental/iterator
@@ -130,6 +130,11 @@ _LIBCPP_POP_MACROS
 #    include <optional>
 #    include <type_traits>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_EXPERIMENTAL_ITERATOR
diff --git a/libcxx/include/experimental/memory b/libcxx/include/experimental/memory
index 7ebae2847e5e..ffbd8f3a065a 100644
--- a/libcxx/include/experimental/memory
+++ b/libcxx/include/experimental/memory
@@ -57,8 +57,8 @@ public:
 #  include <__cstddef/size_t.h>
 #  include <__functional/hash.h>
 #  include <__functional/operations.h>
-#  include <__type_traits/add_lvalue_reference.h>
 #  include <__type_traits/add_pointer.h>
+#  include <__type_traits/add_reference.h>
 #  include <__type_traits/common_type.h>
 #  include <__type_traits/enable_if.h>
 #  include <__type_traits/is_convertible.h>
diff --git a/libcxx/include/flat_map b/libcxx/include/flat_map
index 255245008173..eea9896165f0 100644
--- a/libcxx/include/flat_map
+++ b/libcxx/include/flat_map
@@ -72,6 +72,15 @@ namespace std {
 #  include <version>
 
 // standard required includes
+
+// [iterator.range]
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+// [flat.map.syn]
 #  include <compare>
 #  include <initializer_list>
 
diff --git a/libcxx/include/flat_set b/libcxx/include/flat_set
index ebbb3a0247f3..66041a42b79c 100644
--- a/libcxx/include/flat_set
+++ b/libcxx/include/flat_set
@@ -65,6 +65,15 @@ namespace std {
 #  include <version>
 
 // standard required includes
+
+// [iterator.range]
+#  include <__iterator/access.h>
+#  include <__iterator/data.h>
+#  include <__iterator/empty.h>
+#  include <__iterator/reverse_access.h>
+#  include <__iterator/size.h>
+
+// [flat.set.syn]
 #  include <compare>
 #  include <initializer_list>
 
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index 00aa00ff7e9c..c86f709bedb8 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -833,40 +833,59 @@ typename basic_filebuf<_CharT, _Traits>::int_type basic_filebuf<_CharT, _Traits>
     *this->pptr() = traits_type::to_char_type(__c);
     this->pbump(1);
   }
-  if (this->pptr() != this->pbase()) {
-    if (__always_noconv_) {
-      size_t __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
-      if (std::fwrite(this->pbase(), sizeof(char_type), __nmemb, __file_) != __nmemb)
+
+  // There is nothing to write, early return
+  if (this->pptr() == this->pbase()) {
+    return traits_type::not_eof(__c);
+  }
+
+  if (__always_noconv_) {
+    size_t __n = static_cast<size_t>(this->pptr() - this->pbase());
+    if (std::fwrite(this->pbase(), sizeof(char_type), __n, __file_) != __n)
+      return traits_type::eof();
+  } else {
+    if (!__cv_)
+      std::__throw_bad_cast();
+
+    // See [filebuf.virtuals]
+    char_type* __b = this->pbase();
+    char_type* __p = this->pptr();
+    const char_type* __end;
+    char* __extbuf_end = __extbuf_;
+    do {
+      codecvt_base::result __r = __cv_->out(__st_, __b, __p, __end, __extbuf_, __extbuf_ + __ebs_, __extbuf_end);
+      if (__end == __b)
         return traits_type::eof();
-    } else {
-      char* __extbe = __extbuf_;
-      codecvt_base::result __r;
-      do {
-        if (!__cv_)
-          std::__throw_bad_cast();
 
-        const char_type* __e;
-        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
-        if (__e == this->pbase())
+      // No conversion needed: output characters directly to the file, done.
+      if (__r == codecvt_base::noconv) {
+        size_t __n = static_cast<size_t>(__p - __b);
+        if (std::fwrite(__b, 1, __n, __file_) != __n)
           return traits_type::eof();
-        if (__r == codecvt_base::noconv) {
-          size_t __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
-          if (std::fwrite(this->pbase(), 1, __nmemb, __file_) != __nmemb)
-            return traits_type::eof();
-        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
-          size_t __nmemb = static_cast<size_t>(__extbe - __extbuf_);
-          if (std::fwrite(__extbuf_, 1, __nmemb, __file_) != __nmemb)
-            return traits_type::eof();
-          if (__r == codecvt_base::partial) {
-            this->setp(const_cast<char_type*>(__e), this->pptr());
-            this->__pbump(this->epptr() - this->pbase());
-          }
-        } else
+        break;
+
+        // Conversion successful: output the converted characters to the file, done.
+      } else if (__r == codecvt_base::ok) {
+        size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_);
+        if (std::fwrite(__extbuf_, 1, __n, __file_) != __n)
           return traits_type::eof();
-      } while (__r == codecvt_base::partial);
-    }
-    this->setp(__pb_save, __epb_save);
+        break;
+
+        // Conversion partially successful: output converted characters to the file and repeat with the
+        // remaining characters.
+      } else if (__r == codecvt_base::partial) {
+        size_t __n = static_cast<size_t>(__extbuf_end - __extbuf_);
+        if (std::fwrite(__extbuf_, 1, __n, __file_) != __n)
+          return traits_type::eof();
+        __b = const_cast<char_type*>(__end);
+        continue;
+
+      } else {
+        return traits_type::eof();
+      }
+    } while (true);
   }
+  this->setp(__pb_save, __epb_save);
   return traits_type::not_eof(__c);
 }
 
diff --git a/libcxx/include/functional b/libcxx/include/functional
index b121a19d6bcb..9ebcd818ec84 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -565,6 +565,7 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #    include <__functional/bind_front.h>
 #    include <__functional/identity.h>
 #    include <__functional/ranges_operations.h>
+#    include <__type_traits/common_reference.h>
 #    include <__type_traits/unwrap_ref.h>
 #  endif
 
@@ -599,6 +600,10 @@ POLICY:  For non-variadic implementations, the number of arguments is limited
 #    include <utility>
 #    include <vector>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 23
+#    include <__vector/vector.h>
+#  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_FUNCTIONAL
diff --git a/libcxx/include/future b/libcxx/include/future
index 3dfcce80a977..31932aeeb6b8 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -393,7 +393,7 @@ template <class R, class Alloc> struct uses_allocator<packaged_task<R>, Alloc>;
 #    include <__system_error/error_code.h>
 #    include <__system_error/error_condition.h>
 #    include <__thread/thread.h>
-#    include <__type_traits/add_lvalue_reference.h>
+#    include <__type_traits/add_reference.h>
 #    include <__type_traits/aligned_storage.h>
 #    include <__type_traits/conditional.h>
 #    include <__type_traits/decay.h>
@@ -1823,7 +1823,16 @@ template <class _Rp, class _Fp>
 _LIBCPP_HIDE_FROM_ABI future<_Rp> __make_async_assoc_state(_Fp&& __f) {
   unique_ptr<__async_assoc_state<_Rp, _Fp>, __release_shared_count> __h(
       new __async_assoc_state<_Rp, _Fp>(std::forward<_Fp>(__f)));
-  std::thread(&__async_assoc_state<_Rp, _Fp>::__execute, __h.get()).detach();
+#    if _LIBCPP_HAS_EXCEPTIONS
+  try {
+#    endif
+    std::thread(&__async_assoc_state<_Rp, _Fp>::__execute, __h.get()).detach();
+#    if _LIBCPP_HAS_EXCEPTIONS
+  } catch (...) {
+    __h->__make_ready();
+    throw;
+  }
+#    endif
   return future<_Rp>(__h.get());
 }
 
diff --git a/libcxx/include/iomanip b/libcxx/include/iomanip
index a97b4a94d232..d65d66d71659 100644
--- a/libcxx/include/iomanip
+++ b/libcxx/include/iomanip
@@ -49,10 +49,12 @@ template <class charT, class traits, class Allocator>
 
 #  if _LIBCPP_HAS_LOCALIZATION
 
+#    include <__iterator/istreambuf_iterator.h>
+#    include <__locale_dir/money.h>
+#    include <__locale_dir/time.h>
 #    include <__ostream/put_character_sequence.h>
 #    include <ios>
 #    include <iosfwd>
-#    include <locale>
 #    include <version>
 
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -564,6 +566,11 @@ _LIBCPP_END_NAMESPACE_STD
 #    include <unordered_map>
 #    include <vector>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_IOMANIP
diff --git a/libcxx/include/istream b/libcxx/include/istream
index 02546902494e..93def61a8b47 100644
--- a/libcxx/include/istream
+++ b/libcxx/include/istream
@@ -167,6 +167,7 @@ template <class Stream, class T>
 
 #    include <__fwd/istream.h>
 #    include <__iterator/istreambuf_iterator.h>
+#    include <__locale_dir/num.h>
 #    include <__ostream/basic_ostream.h>
 #    include <__type_traits/conjunction.h>
 #    include <__type_traits/enable_if.h>
@@ -176,7 +177,7 @@ template <class Stream, class T>
 #    include <__utility/forward.h>
 #    include <bitset>
 #    include <ios>
-#    include <locale>
+#    include <streambuf>
 #    include <version>
 
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -1411,6 +1412,10 @@ _LIBCPP_POP_MACROS
 #    include <type_traits>
 #  endif
 
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_ISTREAM
diff --git a/libcxx/include/locale b/libcxx/include/locale
index b206bf8f162f..b50f9c87c728 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -194,3496 +194,20 @@ template <class charT> class messages_byname;
 
 #  if _LIBCPP_HAS_LOCALIZATION
 
-#    include <__algorithm/copy.h>
-#    include <__algorithm/equal.h>
-#    include <__algorithm/find.h>
-#    include <__algorithm/max.h>
-#    include <__algorithm/reverse.h>
-#    include <__algorithm/unwrap_iter.h>
-#    include <__assert>
-#    include <__charconv/to_chars_integral.h>
-#    include <__charconv/traits.h>
-#    include <__iterator/access.h>
-#    include <__iterator/back_insert_iterator.h>
-#    include <__iterator/istreambuf_iterator.h>
-#    include <__iterator/ostreambuf_iterator.h>
 #    include <__locale>
-#    include <__locale_dir/pad_and_output.h>
-#    include <__memory/addressof.h>
-#    include <__memory/unique_ptr.h>
-#    include <__new/exceptions.h>
-#    include <__system_error/errc.h>
-#    include <__type_traits/make_unsigned.h>
-#    include <cerrno>
-#    include <cstdio>
-#    include <cstdlib>
-#    include <ctime>
+#    include <__locale_dir/messages.h>
+#    include <__locale_dir/money.h>
+#    include <__locale_dir/num.h>
+#    include <__locale_dir/time.h>
+#    include <__locale_dir/wbuffer_convert.h>
+#    include <__locale_dir/wstring_convert.h>
 #    include <ios>
-#    include <limits>
-#    include <streambuf>
 #    include <version>
 
-// TODO: Properly qualify calls now that the locale base API defines functions instead of macros
-// NOLINTBEGIN(libcpp-robust-against-adl)
-
-#    if defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))
-// Most unix variants have catopen.  These are the specific ones that don't.
-#      if !defined(__BIONIC__) && !defined(_NEWLIB_VERSION) && !defined(__EMSCRIPTEN__)
-#        define _LIBCPP_HAS_CATOPEN 1
-#        include <nl_types.h>
-#      else
-#        define _LIBCPP_HAS_CATOPEN 0
-#      endif
-#    else
-#      define _LIBCPP_HAS_CATOPEN 0
-#    endif
-
 #    if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #      pragma GCC system_header
 #    endif
 
-_LIBCPP_PUSH_MACROS
-#    include <__undef_macros>
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-#    if defined(__APPLE__) || defined(__FreeBSD__)
-#      define _LIBCPP_GET_C_LOCALE 0
-#    elif defined(__NetBSD__)
-#      define _LIBCPP_GET_C_LOCALE LC_C_LOCALE
-#    else
-#      define _LIBCPP_GET_C_LOCALE __cloc()
-// Get the C locale object
-_LIBCPP_EXPORTED_FROM_ABI __locale::__locale_t __cloc();
-#      define __cloc_defined
-#    endif
-
-// __scan_keyword
-// Scans [__b, __e) until a match is found in the basic_strings range
-//  [__kb, __ke) or until it can be shown that there is no match in [__kb, __ke).
-//  __b will be incremented (visibly), consuming CharT until a match is found
-//  or proved to not exist.  A keyword may be "", in which will match anything.
-//  If one keyword is a prefix of another, and the next CharT in the input
-//  might match another keyword, the algorithm will attempt to find the longest
-//  matching keyword.  If the longer matching keyword ends up not matching, then
-//  no keyword match is found.  If no keyword match is found, __ke is returned
-//  and failbit is set in __err.
-//  Else an iterator pointing to the matching keyword is found.  If more than
-//  one keyword matches, an iterator to the first matching keyword is returned.
-//  If on exit __b == __e, eofbit is set in __err.  If __case_sensitive is false,
-//  __ct is used to force to lower case before comparing characters.
-//  Examples:
-//  Keywords:  "a", "abb"
-//  If the input is "a", the first keyword matches and eofbit is set.
-//  If the input is "abc", no match is found and "ab" are consumed.
-template <class _InputIterator, class _ForwardIterator, class _Ctype>
-_LIBCPP_HIDE_FROM_ABI _ForwardIterator __scan_keyword(
-    _InputIterator& __b,
-    _InputIterator __e,
-    _ForwardIterator __kb,
-    _ForwardIterator __ke,
-    const _Ctype& __ct,
-    ios_base::iostate& __err,
-    bool __case_sensitive = true) {
-  typedef typename iterator_traits<_InputIterator>::value_type _CharT;
-  size_t __nkw                       = static_cast<size_t>(std::distance(__kb, __ke));
-  const unsigned char __doesnt_match = '\0';
-  const unsigned char __might_match  = '\1';
-  const unsigned char __does_match   = '\2';
-  unsigned char __statbuf[100];
-  unsigned char* __status = __statbuf;
-  unique_ptr<unsigned char, void (*)(void*)> __stat_hold(nullptr, free);
-  if (__nkw > sizeof(__statbuf)) {
-    __status = (unsigned char*)malloc(__nkw);
-    if (__status == nullptr)
-      std::__throw_bad_alloc();
-    __stat_hold.reset(__status);
-  }
-  size_t __n_might_match = __nkw; // At this point, any keyword might match
-  size_t __n_does_match  = 0;     // but none of them definitely do
-  // Initialize all statuses to __might_match, except for "" keywords are __does_match
-  unsigned char* __st = __status;
-  for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-    if (!__ky->empty())
-      *__st = __might_match;
-    else {
-      *__st = __does_match;
-      --__n_might_match;
-      ++__n_does_match;
-    }
-  }
-  // While there might be a match, test keywords against the next CharT
-  for (size_t __indx = 0; __b != __e && __n_might_match > 0; ++__indx) {
-    // Peek at the next CharT but don't consume it
-    _CharT __c = *__b;
-    if (!__case_sensitive)
-      __c = __ct.toupper(__c);
-    bool __consume = false;
-    // For each keyword which might match, see if the __indx character is __c
-    // If a match if found, consume __c
-    // If a match is found, and that is the last character in the keyword,
-    //    then that keyword matches.
-    // If the keyword doesn't match this character, then change the keyword
-    //    to doesn't match
-    __st = __status;
-    for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-      if (*__st == __might_match) {
-        _CharT __kc = (*__ky)[__indx];
-        if (!__case_sensitive)
-          __kc = __ct.toupper(__kc);
-        if (__c == __kc) {
-          __consume = true;
-          if (__ky->size() == __indx + 1) {
-            *__st = __does_match;
-            --__n_might_match;
-            ++__n_does_match;
-          }
-        } else {
-          *__st = __doesnt_match;
-          --__n_might_match;
-        }
-      }
-    }
-    // consume if we matched a character
-    if (__consume) {
-      ++__b;
-      // If we consumed a character and there might be a matched keyword that
-      //   was marked matched on a previous iteration, then such keywords
-      //   which are now marked as not matching.
-      if (__n_might_match + __n_does_match > 1) {
-        __st = __status;
-        for (_ForwardIterator __ky = __kb; __ky != __ke; ++__ky, (void)++__st) {
-          if (*__st == __does_match && __ky->size() != __indx + 1) {
-            *__st = __doesnt_match;
-            --__n_does_match;
-          }
-        }
-      }
-    }
-  }
-  // We've exited the loop because we hit eof and/or we have no more "might matches".
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  // Return the first matching result
-  for (__st = __status; __kb != __ke; ++__kb, (void)++__st)
-    if (*__st == __does_match)
-      break;
-  if (__kb == __ke)
-    __err |= ios_base::failbit;
-  return __kb;
-}
-
-struct _LIBCPP_EXPORTED_FROM_ABI __num_get_base {
-  static const int __num_get_buf_sz = 40;
-
-  static int __get_base(ios_base&);
-  static const char __src[33]; // "0123456789abcdefABCDEFxX+-pPiInN"
-  // count of leading characters in __src used for parsing integers ("012..X+-")
-  static const size_t __int_chr_cnt = 26;
-  // count of leading characters in __src used for parsing floating-point values ("012..-pP")
-  static const size_t __fp_chr_cnt = 28;
-};
-
-_LIBCPP_EXPORTED_FROM_ABI void
-__check_grouping(const string& __grouping, unsigned* __g, unsigned* __g_end, ios_base::iostate& __err);
-
-template <class _CharT>
-struct __num_get : protected __num_get_base {
-  static string __stage2_float_prep(ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep);
-
-  static int __stage2_float_loop(
-      _CharT __ct,
-      bool& __in_units,
-      char& __exp,
-      char* __a,
-      char*& __a_end,
-      _CharT __decimal_point,
-      _CharT __thousands_sep,
-      const string& __grouping,
-      unsigned* __g,
-      unsigned*& __g_end,
-      unsigned& __dc,
-      _CharT* __atoms);
-
-  [[__deprecated__("This exists only for ABI compatibility")]] static string
-  __stage2_int_prep(ios_base& __iob, _CharT* __atoms, _CharT& __thousands_sep);
-  static int __stage2_int_loop(
-      _CharT __ct,
-      int __base,
-      char* __a,
-      char*& __a_end,
-      unsigned& __dc,
-      _CharT __thousands_sep,
-      const string& __grouping,
-      unsigned* __g,
-      unsigned*& __g_end,
-      _CharT* __atoms);
-
-  _LIBCPP_HIDE_FROM_ABI static string __stage2_int_prep(ios_base& __iob, _CharT& __thousands_sep) {
-    locale __loc                 = __iob.getloc();
-    const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__loc);
-    __thousands_sep              = __np.thousands_sep();
-    return __np.grouping();
-  }
-
-  _LIBCPP_HIDE_FROM_ABI const _CharT* __do_widen(ios_base& __iob, _CharT* __atoms) const {
-    return __do_widen_p(__iob, __atoms);
-  }
-
-private:
-  template <typename _Tp>
-  _LIBCPP_HIDE_FROM_ABI const _Tp* __do_widen_p(ios_base& __iob, _Tp* __atoms) const {
-    locale __loc = __iob.getloc();
-    use_facet<ctype<_Tp> >(__loc).widen(__src, __src + __int_chr_cnt, __atoms);
-    return __atoms;
-  }
-
-  _LIBCPP_HIDE_FROM_ABI const char* __do_widen_p(ios_base& __iob, char* __atoms) const {
-    (void)__iob;
-    (void)__atoms;
-    return __src;
-  }
-};
-
-template <class _CharT>
-string __num_get<_CharT>::__stage2_float_prep(
-    ios_base& __iob, _CharT* __atoms, _CharT& __decimal_point, _CharT& __thousands_sep) {
-  locale __loc = __iob.getloc();
-  std::use_facet<ctype<_CharT> >(__loc).widen(__src, __src + __fp_chr_cnt, __atoms);
-  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__loc);
-  __decimal_point              = __np.decimal_point();
-  __thousands_sep              = __np.thousands_sep();
-  return __np.grouping();
-}
-
-template <class _CharT>
-int __num_get<_CharT>::__stage2_int_loop(
-    _CharT __ct,
-    int __base,
-    char* __a,
-    char*& __a_end,
-    unsigned& __dc,
-    _CharT __thousands_sep,
-    const string& __grouping,
-    unsigned* __g,
-    unsigned*& __g_end,
-    _CharT* __atoms) {
-  if (__a_end == __a && (__ct == __atoms[24] || __ct == __atoms[25])) {
-    *__a_end++ = __ct == __atoms[24] ? '+' : '-';
-    __dc       = 0;
-    return 0;
-  }
-  if (__grouping.size() != 0 && __ct == __thousands_sep) {
-    if (__g_end - __g < __num_get_buf_sz) {
-      *__g_end++ = __dc;
-      __dc       = 0;
-    }
-    return 0;
-  }
-  ptrdiff_t __f = std::find(__atoms, __atoms + __int_chr_cnt, __ct) - __atoms;
-  if (__f >= 24)
-    return -1;
-  switch (__base) {
-  case 8:
-  case 10:
-    if (__f >= __base)
-      return -1;
-    break;
-  case 16:
-    if (__f < 22)
-      break;
-    if (__a_end != __a && __a_end - __a <= 2 && __a_end[-1] == '0') {
-      __dc       = 0;
-      *__a_end++ = __src[__f];
-      return 0;
-    }
-    return -1;
-  }
-  *__a_end++ = __src[__f];
-  ++__dc;
-  return 0;
-}
-
-template <class _CharT>
-int __num_get<_CharT>::__stage2_float_loop(
-    _CharT __ct,
-    bool& __in_units,
-    char& __exp,
-    char* __a,
-    char*& __a_end,
-    _CharT __decimal_point,
-    _CharT __thousands_sep,
-    const string& __grouping,
-    unsigned* __g,
-    unsigned*& __g_end,
-    unsigned& __dc,
-    _CharT* __atoms) {
-  if (__ct == __decimal_point) {
-    if (!__in_units)
-      return -1;
-    __in_units = false;
-    *__a_end++ = '.';
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
-      *__g_end++ = __dc;
-    return 0;
-  }
-  if (__ct == __thousands_sep && __grouping.size() != 0) {
-    if (!__in_units)
-      return -1;
-    if (__g_end - __g < __num_get_buf_sz) {
-      *__g_end++ = __dc;
-      __dc       = 0;
-    }
-    return 0;
-  }
-  ptrdiff_t __f = std::find(__atoms, __atoms + __num_get_base::__fp_chr_cnt, __ct) - __atoms;
-  if (__f >= static_cast<ptrdiff_t>(__num_get_base::__fp_chr_cnt))
-    return -1;
-  char __x = __src[__f];
-  if (__x == '-' || __x == '+') {
-    if (__a_end == __a || (std::toupper(__a_end[-1]) == std::toupper(__exp))) {
-      *__a_end++ = __x;
-      return 0;
-    }
-    return -1;
-  }
-  if (__x == 'x' || __x == 'X')
-    __exp = 'P';
-  else if (std::toupper(__x) == __exp) {
-    __exp = std::tolower(__exp);
-    if (__in_units) {
-      __in_units = false;
-      if (__grouping.size() != 0 && __g_end - __g < __num_get_buf_sz)
-        *__g_end++ = __dc;
-    }
-  }
-  *__a_end++ = __x;
-  if (__f >= 22)
-    return 0;
-  ++__dc;
-  return 0;
-}
-
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_get<wchar_t>;
-#    endif
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __do_strtod(const char* __a, char** __p2);
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI float __do_strtod<float>(const char* __a, char** __p2) {
-  return __locale::__strtof(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI double __do_strtod<double>(const char* __a, char** __p2) {
-  return __locale::__strtod(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <>
-inline _LIBCPP_HIDE_FROM_ABI long double __do_strtod<long double>(const char* __a, char** __p2) {
-  return __locale::__strtold(__a, __p2, _LIBCPP_GET_C_LOCALE);
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp __num_get_float(const char* __a, const char* __a_end, ios_base::iostate& __err) {
-  if (__a != __a_end) {
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    _Tp __ld                                                     = std::__do_strtod<_Tp>(__a, &__p2);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE)
-      __err = ios_base::failbit;
-    return __ld;
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__num_get_signed_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
-  if (__a != __a_end) {
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    long long __ll = __locale::__strtoll(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE || __ll < numeric_limits<_Tp>::min() || numeric_limits<_Tp>::max() < __ll) {
-      __err = ios_base::failbit;
-      if (__ll > 0)
-        return numeric_limits<_Tp>::max();
-      else
-        return numeric_limits<_Tp>::min();
-    }
-    return static_cast<_Tp>(__ll);
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI _Tp
-__num_get_unsigned_integral(const char* __a, const char* __a_end, ios_base::iostate& __err, int __base) {
-  if (__a != __a_end) {
-    const bool __negate = *__a == '-';
-    if (__negate && ++__a == __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    }
-    __libcpp_remove_reference_t<decltype(errno)> __save_errno = errno;
-    errno                                                     = 0;
-    char* __p2;
-    unsigned long long __ll = __locale::__strtoull(__a, &__p2, __base, _LIBCPP_GET_C_LOCALE);
-    __libcpp_remove_reference_t<decltype(errno)> __current_errno = errno;
-    if (__current_errno == 0)
-      errno = __save_errno;
-    if (__p2 != __a_end) {
-      __err = ios_base::failbit;
-      return 0;
-    } else if (__current_errno == ERANGE || numeric_limits<_Tp>::max() < __ll) {
-      __err = ios_base::failbit;
-      return numeric_limits<_Tp>::max();
-    }
-    _Tp __res = static_cast<_Tp>(__ll);
-    if (__negate)
-      __res = -__res;
-    return __res;
-  }
-  __err = ios_base::failbit;
-  return 0;
-}
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class num_get : public locale::facet, private __num_get<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit num_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
-    return do_get(__b, __e, __iob, __err, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_get() override {}
-
-  template <class _Fp>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_floating_point(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Fp& __v) const {
-    // Stage 1, nothing to do
-    // Stage 2
-    char_type __atoms[__num_get_base::__fp_chr_cnt];
-    char_type __decimal_point;
-    char_type __thousands_sep;
-    string __grouping = this->__stage2_float_prep(__iob, __atoms, __decimal_point, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end        = __g;
-    unsigned __dc            = 0;
-    bool __in_units          = true;
-    char __exp               = 'E';
-    bool __is_leading_parsed = false;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_float_loop(
-              *__b,
-              __in_units,
-              __exp,
-              __a,
-              __a_end,
-              __decimal_point,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              __dc,
-              __atoms))
-        break;
-
-      // the leading character excluding the sign must be a decimal digit
-      if (!__is_leading_parsed) {
-        if (__a_end - __a >= 1 && __a[0] != '-' && __a[0] != '+') {
-          if (('0' <= __a[0] && __a[0] <= '9') || __a[0] == '.')
-            __is_leading_parsed = true;
-          else
-            break;
-        } else if (__a_end - __a >= 2 && (__a[0] == '-' || __a[0] == '+')) {
-          if (('0' <= __a[1] && __a[1] <= '9') || __a[1] == '.')
-            __is_leading_parsed = true;
-          else
-            break;
-        }
-      }
-    }
-    if (__grouping.size() != 0 && __in_units && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_float<_Fp>(__a, __a_end, __err);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  template <class _Signed>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_signed(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Signed& __v) const {
-    // Stage 1
-    int __base = this->__get_base(__iob);
-    // Stage 2
-    char_type __thousands_sep;
-    const int __atoms_size = __num_get_base::__int_chr_cnt;
-    char_type __atoms1[__atoms_size];
-    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
-    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end = __g;
-    unsigned __dc     = 0;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_int_loop(
-              *__b,
-              __base,
-              __a,
-              __a_end,
-              __dc,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              const_cast<char_type*>(__atoms)))
-        break;
-    }
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_signed_integral<_Signed>(__a, __a_end, __err, __base);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  template <class _Unsigned>
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  __do_get_unsigned(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, _Unsigned& __v) const {
-    // Stage 1
-    int __base = this->__get_base(__iob);
-    // Stage 2
-    char_type __thousands_sep;
-    const int __atoms_size = __num_get_base::__int_chr_cnt;
-    char_type __atoms1[__atoms_size];
-    const char_type* __atoms = this->__do_widen(__iob, __atoms1);
-    string __grouping        = this->__stage2_int_prep(__iob, __thousands_sep);
-    string __buf;
-    __buf.resize(__buf.capacity());
-    char* __a     = &__buf[0];
-    char* __a_end = __a;
-    unsigned __g[__num_get_base::__num_get_buf_sz];
-    unsigned* __g_end = __g;
-    unsigned __dc     = 0;
-    for (; __b != __e; ++__b) {
-      if (__a_end == __a + __buf.size()) {
-        size_t __tmp = __buf.size();
-        __buf.resize(2 * __buf.size());
-        __buf.resize(__buf.capacity());
-        __a     = &__buf[0];
-        __a_end = __a + __tmp;
-      }
-      if (this->__stage2_int_loop(
-              *__b,
-              __base,
-              __a,
-              __a_end,
-              __dc,
-              __thousands_sep,
-              __grouping,
-              __g,
-              __g_end,
-              const_cast<char_type*>(__atoms)))
-        break;
-    }
-    if (__grouping.size() != 0 && __g_end - __g < __num_get_base::__num_get_buf_sz)
-      *__g_end++ = __dc;
-    // Stage 3
-    __v = std::__num_get_unsigned_integral<_Unsigned>(__a, __a_end, __err, __base);
-    // Digit grouping checked
-    __check_grouping(__grouping, __g, __g_end, __err);
-    // EOF checked
-    if (__b == __e)
-      __err |= ios_base::eofbit;
-    return __b;
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const;
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long& __v) const {
-    return this->__do_get_signed(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long long& __v) const {
-    return this->__do_get_signed(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned short& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned int& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, unsigned long long& __v) const {
-    return this->__do_get_unsigned(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, float& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, double& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return this->__do_get_floating_point(__b, __e, __iob, __err, __v);
-  }
-
-  virtual iter_type do_get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const;
-};
-
-template <class _CharT, class _InputIterator>
-locale::id num_get<_CharT, _InputIterator>::id;
-
-template <class _CharT, class _InputIterator>
-_InputIterator num_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, bool& __v) const {
-  if ((__iob.flags() & ios_base::boolalpha) == 0) {
-    long __lv = -1;
-    __b       = do_get(__b, __e, __iob, __err, __lv);
-    switch (__lv) {
-    case 0:
-      __v = false;
-      break;
-    case 1:
-      __v = true;
-      break;
-    default:
-      __v   = true;
-      __err = ios_base::failbit;
-      break;
-    }
-    return __b;
-  }
-  const ctype<_CharT>& __ct    = std::use_facet<ctype<_CharT> >(__iob.getloc());
-  const numpunct<_CharT>& __np = std::use_facet<numpunct<_CharT> >(__iob.getloc());
-  typedef typename numpunct<_CharT>::string_type string_type;
-  const string_type __names[2] = {__np.truename(), __np.falsename()};
-  const string_type* __i       = std::__scan_keyword(__b, __e, __names, __names + 2, __ct, __err);
-  __v                          = __i == __names;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator num_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, void*& __v) const {
-  // Stage 1
-  int __base = 16;
-  // Stage 2
-  char_type __atoms[__num_get_base::__int_chr_cnt];
-  char_type __thousands_sep = char_type();
-  string __grouping;
-  std::use_facet<ctype<_CharT> >(__iob.getloc())
-      .widen(__num_get_base::__src, __num_get_base::__src + __num_get_base::__int_chr_cnt, __atoms);
-  string __buf;
-  __buf.resize(__buf.capacity());
-  char* __a     = &__buf[0];
-  char* __a_end = __a;
-  unsigned __g[__num_get_base::__num_get_buf_sz];
-  unsigned* __g_end = __g;
-  unsigned __dc     = 0;
-  for (; __b != __e; ++__b) {
-    if (__a_end == __a + __buf.size()) {
-      size_t __tmp = __buf.size();
-      __buf.resize(2 * __buf.size());
-      __buf.resize(__buf.capacity());
-      __a     = &__buf[0];
-      __a_end = __a + __tmp;
-    }
-    if (this->__stage2_int_loop(*__b, __base, __a, __a_end, __dc, __thousands_sep, __grouping, __g, __g_end, __atoms))
-      break;
-  }
-  // Stage 3
-  __buf.resize(__a_end - __a);
-  if (__locale::__sscanf(__buf.c_str(), _LIBCPP_GET_C_LOCALE, "%p", &__v) != 1)
-    __err = ios_base::failbit;
-  // EOF checked
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_get<wchar_t>;
-#    endif
-
-struct _LIBCPP_EXPORTED_FROM_ABI __num_put_base {
-protected:
-  static void __format_int(char* __fmt, const char* __len, bool __signd, ios_base::fmtflags __flags);
-  static bool __format_float(char* __fmt, const char* __len, ios_base::fmtflags __flags);
-  static char* __identify_padding(char* __nb, char* __ne, const ios_base& __iob);
-};
-
-template <class _CharT>
-struct __num_put : protected __num_put_base {
-  static void __widen_and_group_int(
-      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
-  static void __widen_and_group_float(
-      char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc);
-};
-
-template <class _CharT>
-void __num_put<_CharT>::__widen_and_group_int(
-    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
-  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
-  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
-  string __grouping             = __npt.grouping();
-  if (__grouping.empty()) {
-    __ct.widen(__nb, __ne, __ob);
-    __oe = __ob + (__ne - __nb);
-  } else {
-    __oe       = __ob;
-    char* __nf = __nb;
-    if (*__nf == '-' || *__nf == '+')
-      *__oe++ = __ct.widen(*__nf++);
-    if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
-      *__oe++ = __ct.widen(*__nf++);
-      *__oe++ = __ct.widen(*__nf++);
-    }
-    std::reverse(__nf, __ne);
-    _CharT __thousands_sep = __npt.thousands_sep();
-    unsigned __dc          = 0;
-    unsigned __dg          = 0;
-    for (char* __p = __nf; __p < __ne; ++__p) {
-      if (static_cast<unsigned>(__grouping[__dg]) > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
-        *__oe++ = __thousands_sep;
-        __dc    = 0;
-        if (__dg < __grouping.size() - 1)
-          ++__dg;
-      }
-      *__oe++ = __ct.widen(*__p);
-      ++__dc;
-    }
-    std::reverse(__ob + (__nf - __nb), __oe);
-  }
-  if (__np == __ne)
-    __op = __oe;
-  else
-    __op = __ob + (__np - __nb);
-}
-
-template <class _CharT>
-void __num_put<_CharT>::__widen_and_group_float(
-    char* __nb, char* __np, char* __ne, _CharT* __ob, _CharT*& __op, _CharT*& __oe, const locale& __loc) {
-  const ctype<_CharT>& __ct     = std::use_facet<ctype<_CharT> >(__loc);
-  const numpunct<_CharT>& __npt = std::use_facet<numpunct<_CharT> >(__loc);
-  string __grouping             = __npt.grouping();
-  __oe                          = __ob;
-  char* __nf                    = __nb;
-  if (*__nf == '-' || *__nf == '+')
-    *__oe++ = __ct.widen(*__nf++);
-  char* __ns;
-  if (__ne - __nf >= 2 && __nf[0] == '0' && (__nf[1] == 'x' || __nf[1] == 'X')) {
-    *__oe++ = __ct.widen(*__nf++);
-    *__oe++ = __ct.widen(*__nf++);
-    for (__ns = __nf; __ns < __ne; ++__ns)
-      if (!__locale::__isxdigit(*__ns, _LIBCPP_GET_C_LOCALE))
-        break;
-  } else {
-    for (__ns = __nf; __ns < __ne; ++__ns)
-      if (!__locale::__isdigit(*__ns, _LIBCPP_GET_C_LOCALE))
-        break;
-  }
-  if (__grouping.empty()) {
-    __ct.widen(__nf, __ns, __oe);
-    __oe += __ns - __nf;
-  } else {
-    std::reverse(__nf, __ns);
-    _CharT __thousands_sep = __npt.thousands_sep();
-    unsigned __dc          = 0;
-    unsigned __dg          = 0;
-    for (char* __p = __nf; __p < __ns; ++__p) {
-      if (__grouping[__dg] > 0 && __dc == static_cast<unsigned>(__grouping[__dg])) {
-        *__oe++ = __thousands_sep;
-        __dc    = 0;
-        if (__dg < __grouping.size() - 1)
-          ++__dg;
-      }
-      *__oe++ = __ct.widen(*__p);
-      ++__dc;
-    }
-    std::reverse(__ob + (__nf - __nb), __oe);
-  }
-  for (__nf = __ns; __nf < __ne; ++__nf) {
-    if (*__nf == '.') {
-      *__oe++ = __npt.decimal_point();
-      ++__nf;
-      break;
-    } else
-      *__oe++ = __ct.widen(*__nf);
-  }
-  __ct.widen(__nf, __ne, __oe);
-  __oe += __ne - __nf;
-  if (__np == __ne)
-    __op = __oe;
-  else
-    __op = __ob + (__np - __nb);
-}
-
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template struct _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __num_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class num_put : public locale::facet, private __num_put<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit num_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
-    return do_put(__s, __iob, __fl, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~num_put() override {}
-
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const;
-  virtual iter_type do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const;
-
-  template <class _Integral>
-  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
-  __do_put_integral(iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const;
-
-  template <class _Float>
-  _LIBCPP_HIDE_FROM_ABI inline _OutputIterator
-  __do_put_floating_point(iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const;
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id num_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, bool __v) const {
-  if ((__iob.flags() & ios_base::boolalpha) == 0)
-    return do_put(__s, __iob, __fl, (unsigned long)__v);
-  const numpunct<char_type>& __np = std::use_facet<numpunct<char_type> >(__iob.getloc());
-  typedef typename numpunct<char_type>::string_type string_type;
-  string_type __nm = __v ? __np.truename() : __np.falsename();
-  for (typename string_type::iterator __i = __nm.begin(); __i != __nm.end(); ++__i, ++__s)
-    *__s = *__i;
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-template <class _Integral>
-_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_integral(
-    iter_type __s, ios_base& __iob, char_type __fl, _Integral __v) const {
-  // Stage 1 - Get number in narrow char
-
-  // Worst case is octal, with showbase enabled. Note that octal is always
-  // printed as an unsigned value.
-  using _Unsigned = typename make_unsigned<_Integral>::type;
-  _LIBCPP_CONSTEXPR const unsigned __buffer_size =
-      (numeric_limits<_Unsigned>::digits / 3)          // 1 char per 3 bits
-      + ((numeric_limits<_Unsigned>::digits % 3) != 0) // round up
-      + 2;                                             // base prefix + terminating null character
-
-  char __char_buffer[__buffer_size];
-  char* __buffer_ptr = __char_buffer;
-
-  auto __flags = __iob.flags();
-
-  auto __basefield = (__flags & ios_base::basefield);
-
-  // Extract base
-  int __base = 10;
-  if (__basefield == ios_base::oct)
-    __base = 8;
-  else if (__basefield == ios_base::hex)
-    __base = 16;
-
-  // Print '-' and make the argument unsigned
-  auto __uval = std::__to_unsigned_like(__v);
-  if (__basefield != ios_base::oct && __basefield != ios_base::hex && __v < 0) {
-    *__buffer_ptr++ = '-';
-    __uval          = std::__complement(__uval);
-  }
-
-  // Maybe add '+' prefix
-  if (std::is_signed<_Integral>::value && (__flags & ios_base::showpos) && __basefield != ios_base::oct &&
-      __basefield != ios_base::hex && __v >= 0)
-    *__buffer_ptr++ = '+';
-
-  // Add base prefix
-  if (__v != 0 && __flags & ios_base::showbase) {
-    if (__basefield == ios_base::oct) {
-      *__buffer_ptr++ = '0';
-    } else if (__basefield == ios_base::hex) {
-      *__buffer_ptr++ = '0';
-      *__buffer_ptr++ = (__flags & ios_base::uppercase ? 'X' : 'x');
-    }
-  }
-
-  auto __res = std::__to_chars_integral(__buffer_ptr, __char_buffer + __buffer_size, __uval, __base);
-  _LIBCPP_ASSERT_INTERNAL(__res.__ec == std::errc(0), "to_chars: invalid maximum buffer size computed?");
-
-  // Make letters uppercase
-  if (__flags & ios_base::hex && __flags & ios_base::uppercase) {
-    for (; __buffer_ptr != __res.__ptr; ++__buffer_ptr)
-      *__buffer_ptr = std::__hex_to_upper(*__buffer_ptr);
-  }
-
-  char* __np = this->__identify_padding(__char_buffer, __res.__ptr, __iob);
-  // Stage 2 - Widen __nar while adding thousands separators
-  char_type __o[2 * (__buffer_size - 1) - 1];
-  char_type* __op; // pad here
-  char_type* __oe; // end of output
-  this->__widen_and_group_int(__char_buffer, __np, __res.__ptr, __o, __op, __oe, __iob.getloc());
-  // [__o, __oe) contains thousands_sep'd wide number
-  // Stage 3 & 4
-  return std::__pad_and_output(__s, __o, __op, __oe, __iob, __fl);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, unsigned long long __v) const {
-  return this->__do_put_integral(__s, __iob, __fl, __v);
-}
-
-template <class _CharT, class _OutputIterator>
-template <class _Float>
-_LIBCPP_HIDE_FROM_ABI inline _OutputIterator num_put<_CharT, _OutputIterator>::__do_put_floating_point(
-    iter_type __s, ios_base& __iob, char_type __fl, _Float __v, char const* __len) const {
-  // Stage 1 - Get number in narrow char
-  char __fmt[8]            = {'%', 0};
-  bool __specify_precision = this->__format_float(__fmt + 1, __len, __iob.flags());
-  const unsigned __nbuf    = 30;
-  char __nar[__nbuf];
-  char* __nb = __nar;
-  int __nc;
-  _LIBCPP_DIAGNOSTIC_PUSH
-  _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
-  _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wformat-nonliteral")
-  if (__specify_precision)
-    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
-  else
-    __nc = __locale::__snprintf(__nb, __nbuf, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-  unique_ptr<char, void (*)(void*)> __nbh(nullptr, free);
-  if (__nc > static_cast<int>(__nbuf - 1)) {
-    if (__specify_precision)
-      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, (int)__iob.precision(), __v);
-    else
-      __nc = __locale::__asprintf(&__nb, _LIBCPP_GET_C_LOCALE, __fmt, __v);
-    if (__nc == -1)
-      std::__throw_bad_alloc();
-    __nbh.reset(__nb);
-  }
-  _LIBCPP_DIAGNOSTIC_POP
-  char* __ne = __nb + __nc;
-  char* __np = this->__identify_padding(__nb, __ne, __iob);
-  // Stage 2 - Widen __nar while adding thousands separators
-  char_type __o[2 * (__nbuf - 1) - 1];
-  char_type* __ob = __o;
-  unique_ptr<char_type, void (*)(void*)> __obh(0, free);
-  if (__nb != __nar) {
-    __ob = (char_type*)malloc(2 * static_cast<size_t>(__nc) * sizeof(char_type));
-    if (__ob == 0)
-      std::__throw_bad_alloc();
-    __obh.reset(__ob);
-  }
-  char_type* __op; // pad here
-  char_type* __oe; // end of output
-  this->__widen_and_group_float(__nb, __np, __ne, __ob, __op, __oe, __iob.getloc());
-  // [__o, __oe) contains thousands_sep'd wide number
-  // Stage 3 & 4
-  __s = std::__pad_and_output(__s, __ob, __op, __oe, __iob, __fl);
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, double __v) const {
-  return this->__do_put_floating_point(__s, __iob, __fl, __v, "");
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, long double __v) const {
-  return this->__do_put_floating_point(__s, __iob, __fl, __v, "L");
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator
-num_put<_CharT, _OutputIterator>::do_put(iter_type __s, ios_base& __iob, char_type __fl, const void* __v) const {
-  auto __flags = __iob.flags();
-  __iob.flags((__flags & ~ios_base::basefield & ~ios_base::uppercase) | ios_base::hex | ios_base::showbase);
-  auto __res = __do_put_integral(__s, __iob, __fl, reinterpret_cast<uintptr_t>(__v));
-  __iob.flags(__flags);
-  return __res;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS num_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _InputIterator>
-_LIBCPP_HIDE_FROM_ABI int __get_up_to_n_digits(
-    _InputIterator& __b, _InputIterator __e, ios_base::iostate& __err, const ctype<_CharT>& __ct, int __n) {
-  // Precondition:  __n >= 1
-  if (__b == __e) {
-    __err |= ios_base::eofbit | ios_base::failbit;
-    return 0;
-  }
-  // get first digit
-  _CharT __c = *__b;
-  if (!__ct.is(ctype_base::digit, __c)) {
-    __err |= ios_base::failbit;
-    return 0;
-  }
-  int __r = __ct.narrow(__c, 0) - '0';
-  for (++__b, (void)--__n; __b != __e && __n > 0; ++__b, (void)--__n) {
-    // get next digit
-    __c = *__b;
-    if (!__ct.is(ctype_base::digit, __c))
-      return __r;
-    __r = __r * 10 + __ct.narrow(__c, 0) - '0';
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __r;
-}
-
-class _LIBCPP_EXPORTED_FROM_ABI time_base {
-public:
-  enum dateorder { no_order, dmy, mdy, ymd, ydm };
-};
-
-template <class _CharT>
-class __time_get_c_storage {
-protected:
-  typedef basic_string<_CharT> string_type;
-
-  virtual const string_type* __weeks() const;
-  virtual const string_type* __months() const;
-  virtual const string_type* __am_pm() const;
-  virtual const string_type& __c() const;
-  virtual const string_type& __r() const;
-  virtual const string_type& __x() const;
-  virtual const string_type& __X() const;
-
-  _LIBCPP_HIDE_FROM_ABI ~__time_get_c_storage() {}
-};
-
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__weeks() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__months() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string* __time_get_c_storage<char>::__am_pm() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__c() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__r() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__x() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const string& __time_get_c_storage<char>::__X() const;
-
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__weeks() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__months() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring* __time_get_c_storage<wchar_t>::__am_pm() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__c() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__r() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__x() const;
-template <>
-_LIBCPP_EXPORTED_FROM_ABI const wstring& __time_get_c_storage<wchar_t>::__X() const;
-#    endif
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class time_get : public locale::facet, public time_base, private __time_get_c_storage<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-  typedef time_base::dateorder dateorder;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI dateorder date_order() const { return this->do_date_order(); }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_time(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_date(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_weekday(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_monthname(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-    return do_get_year(__b, __e, __iob, __err, __tm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod = 0)
-      const {
-    return do_get(__b, __e, __iob, __err, __tm, __fmt, __mod);
-  }
-
-  iter_type
-  get(iter_type __b,
-      iter_type __e,
-      ios_base& __iob,
-      ios_base::iostate& __err,
-      tm* __tm,
-      const char_type* __fmtb,
-      const char_type* __fmte) const;
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get() override {}
-
-  virtual dateorder do_date_order() const;
-  virtual iter_type
-  do_get_time(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_date(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_weekday(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_monthname(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type
-  do_get_year(iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const;
-  virtual iter_type do_get(
-      iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char __mod) const;
-
-private:
-  void __get_white_space(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_percent(iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-
-  void __get_weekdayname(
-      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_monthname(
-      int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_day(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_month(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_year(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_year4(int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_hour(int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_12_hour(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_am_pm(int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_minute(int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_second(int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void
-  __get_weekday(int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-  void __get_day_year_num(
-      int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const;
-};
-
-template <class _CharT, class _InputIterator>
-locale::id time_get<_CharT, _InputIterator>::id;
-
-// time_get primitives
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_weekdayname(
-    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  // Note:  ignoring case comes from the POSIX strptime spec
-  const string_type* __wk = this->__weeks();
-  ptrdiff_t __i           = std::__scan_keyword(__b, __e, __wk, __wk + 14, __ct, __err, false) - __wk;
-  if (__i < 14)
-    __w = __i % 7;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_monthname(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  // Note:  ignoring case comes from the POSIX strptime spec
-  const string_type* __month = this->__months();
-  ptrdiff_t __i              = std::__scan_keyword(__b, __e, __month, __month + 24, __ct, __err, false) - __month;
-  if (__i < 24)
-    __m = __i % 12;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_day(
-    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
-    __d = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_month(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2) - 1;
-  if (!(__err & ios_base::failbit) && 0 <= __t && __t <= 11)
-    __m = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_year(
-    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
-  if (!(__err & ios_base::failbit)) {
-    if (__t < 69)
-      __t += 2000;
-    else if (69 <= __t && __t <= 99)
-      __t += 1900;
-    __y = __t - 1900;
-  }
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_year4(
-    int& __y, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 4);
-  if (!(__err & ios_base::failbit))
-    __y = __t - 1900;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_hour(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 23)
-    __h = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_12_hour(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 12)
-    __h = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_minute(
-    int& __m, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 59)
-    __m = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_second(
-    int& __s, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
-  if (!(__err & ios_base::failbit) && __t <= 60)
-    __s = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_weekday(
-    int& __w, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 1);
-  if (!(__err & ios_base::failbit) && __t <= 6)
-    __w = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_day_year_num(
-    int& __d, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  int __t = std::__get_up_to_n_digits(__b, __e, __err, __ct, 3);
-  if (!(__err & ios_base::failbit) && __t <= 365)
-    __d = __t;
-  else
-    __err |= ios_base::failbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_white_space(
-    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
-    ;
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_am_pm(
-    int& __h, iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  const string_type* __ap = this->__am_pm();
-  if (__ap[0].size() + __ap[1].size() == 0) {
-    __err |= ios_base::failbit;
-    return;
-  }
-  ptrdiff_t __i = std::__scan_keyword(__b, __e, __ap, __ap + 2, __ct, __err, false) - __ap;
-  if (__i == 0 && __h == 12)
-    __h = 0;
-  else if (__i == 1 && __h < 12)
-    __h += 12;
-}
-
-template <class _CharT, class _InputIterator>
-void time_get<_CharT, _InputIterator>::__get_percent(
-    iter_type& __b, iter_type __e, ios_base::iostate& __err, const ctype<char_type>& __ct) const {
-  if (__b == __e) {
-    __err |= ios_base::eofbit | ios_base::failbit;
-    return;
-  }
-  if (__ct.narrow(*__b, 0) != '%')
-    __err |= ios_base::failbit;
-  else if (++__b == __e)
-    __err |= ios_base::eofbit;
-}
-
-// time_get end primitives
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::get(
-    iter_type __b,
-    iter_type __e,
-    ios_base& __iob,
-    ios_base::iostate& __err,
-    tm* __tm,
-    const char_type* __fmtb,
-    const char_type* __fmte) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __err                        = ios_base::goodbit;
-  while (__fmtb != __fmte && __err == ios_base::goodbit) {
-    if (__b == __e) {
-      __err = ios_base::failbit;
-      break;
-    }
-    if (__ct.narrow(*__fmtb, 0) == '%') {
-      if (++__fmtb == __fmte) {
-        __err = ios_base::failbit;
-        break;
-      }
-      char __cmd = __ct.narrow(*__fmtb, 0);
-      char __opt = '\0';
-      if (__cmd == 'E' || __cmd == '0') {
-        if (++__fmtb == __fmte) {
-          __err = ios_base::failbit;
-          break;
-        }
-        __opt = __cmd;
-        __cmd = __ct.narrow(*__fmtb, 0);
-      }
-      __b = do_get(__b, __e, __iob, __err, __tm, __cmd, __opt);
-      ++__fmtb;
-    } else if (__ct.is(ctype_base::space, *__fmtb)) {
-      for (++__fmtb; __fmtb != __fmte && __ct.is(ctype_base::space, *__fmtb); ++__fmtb)
-        ;
-      for (; __b != __e && __ct.is(ctype_base::space, *__b); ++__b)
-        ;
-    } else if (__ct.toupper(*__b) == __ct.toupper(*__fmtb)) {
-      ++__b;
-      ++__fmtb;
-    } else
-      __err = ios_base::failbit;
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-typename time_get<_CharT, _InputIterator>::dateorder time_get<_CharT, _InputIterator>::do_date_order() const {
-  return mdy;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_time(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const char_type __fmt[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
-  return get(__b, __e, __iob, __err, __tm, __fmt, __fmt + sizeof(__fmt) / sizeof(__fmt[0]));
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_date(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const string_type& __fmt = this->__x();
-  return get(__b, __e, __iob, __err, __tm, __fmt.data(), __fmt.data() + __fmt.size());
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_weekday(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_monthname(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get_year(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm) const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  __get_year(__tm->tm_year, __b, __e, __err, __ct);
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator time_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, ios_base& __iob, ios_base::iostate& __err, tm* __tm, char __fmt, char) const {
-  __err                        = ios_base::goodbit;
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  switch (__fmt) {
-  case 'a':
-  case 'A':
-    __get_weekdayname(__tm->tm_wday, __b, __e, __err, __ct);
-    break;
-  case 'b':
-  case 'B':
-  case 'h':
-    __get_monthname(__tm->tm_mon, __b, __e, __err, __ct);
-    break;
-  case 'c': {
-    const string_type& __fm = this->__c();
-    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
-  } break;
-  case 'd':
-  case 'e':
-    __get_day(__tm->tm_mday, __b, __e, __err, __ct);
-    break;
-  case 'D': {
-    const char_type __fm[] = {'%', 'm', '/', '%', 'd', '/', '%', 'y'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'F': {
-    const char_type __fm[] = {'%', 'Y', '-', '%', 'm', '-', '%', 'd'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'H':
-    __get_hour(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'I':
-    __get_12_hour(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'j':
-    __get_day_year_num(__tm->tm_yday, __b, __e, __err, __ct);
-    break;
-  case 'm':
-    __get_month(__tm->tm_mon, __b, __e, __err, __ct);
-    break;
-  case 'M':
-    __get_minute(__tm->tm_min, __b, __e, __err, __ct);
-    break;
-  case 'n':
-  case 't':
-    __get_white_space(__b, __e, __err, __ct);
-    break;
-  case 'p':
-    __get_am_pm(__tm->tm_hour, __b, __e, __err, __ct);
-    break;
-  case 'r': {
-    const char_type __fm[] = {'%', 'I', ':', '%', 'M', ':', '%', 'S', ' ', '%', 'p'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'R': {
-    const char_type __fm[] = {'%', 'H', ':', '%', 'M'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'S':
-    __get_second(__tm->tm_sec, __b, __e, __err, __ct);
-    break;
-  case 'T': {
-    const char_type __fm[] = {'%', 'H', ':', '%', 'M', ':', '%', 'S'};
-    __b                    = get(__b, __e, __iob, __err, __tm, __fm, __fm + sizeof(__fm) / sizeof(__fm[0]));
-  } break;
-  case 'w':
-    __get_weekday(__tm->tm_wday, __b, __e, __err, __ct);
-    break;
-  case 'x':
-    return do_get_date(__b, __e, __iob, __err, __tm);
-  case 'X': {
-    const string_type& __fm = this->__X();
-    __b                     = get(__b, __e, __iob, __err, __tm, __fm.data(), __fm.data() + __fm.size());
-  } break;
-  case 'y':
-    __get_year(__tm->tm_year, __b, __e, __err, __ct);
-    break;
-  case 'Y':
-    __get_year4(__tm->tm_year, __b, __e, __err, __ct);
-    break;
-  case '%':
-    __get_percent(__b, __e, __err, __ct);
-    break;
-  default:
-    __err |= ios_base::failbit;
-  }
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get<wchar_t>;
-#    endif
-
-class _LIBCPP_EXPORTED_FROM_ABI __time_get {
-protected:
-  __locale::__locale_t __loc_;
-
-  __time_get(const char* __nm);
-  __time_get(const string& __nm);
-  ~__time_get();
-};
-
-template <class _CharT>
-class __time_get_storage : public __time_get {
-protected:
-  typedef basic_string<_CharT> string_type;
-
-  string_type __weeks_[14];
-  string_type __months_[24];
-  string_type __am_pm_[2];
-  string_type __c_;
-  string_type __r_;
-  string_type __x_;
-  string_type __X_;
-
-  explicit __time_get_storage(const char* __nm);
-  explicit __time_get_storage(const string& __nm);
-
-  _LIBCPP_HIDE_FROM_ABI ~__time_get_storage() {}
-
-  time_base::dateorder __do_date_order() const;
-
-private:
-  void init(const ctype<_CharT>&);
-  string_type __analyze(char __fmt, const ctype<_CharT>&);
-};
-
-#    define _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(_CharT)                                                    \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order() const;              \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);                           \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);                         \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);                           \
-      template <>                                                                                                      \
-      _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type __time_get_storage<_CharT>::__analyze(         \
-          char, const ctype<_CharT>&);                                                                                 \
-      extern template _LIBCPP_EXPORTED_FROM_ABI time_base::dateorder __time_get_storage<_CharT>::__do_date_order()     \
-          const;                                                                                                       \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const char*);           \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::__time_get_storage(const string&);         \
-      extern template _LIBCPP_EXPORTED_FROM_ABI void __time_get_storage<_CharT>::init(const ctype<_CharT>&);           \
-      extern template _LIBCPP_EXPORTED_FROM_ABI __time_get_storage<_CharT>::string_type                                \
-      __time_get_storage<_CharT>::__analyze(char, const ctype<_CharT>&);                                               \
-      /**/
-
-_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(char)
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-_LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION(wchar_t)
-#    endif
-#    undef _LIBCPP_TIME_GET_STORAGE_EXPLICIT_INSTANTIATION
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class time_get_byname : public time_get<_CharT, _InputIterator>, private __time_get_storage<_CharT> {
-public:
-  typedef time_base::dateorder dateorder;
-  typedef _InputIterator iter_type;
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const char* __nm, size_t __refs = 0)
-      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
-  _LIBCPP_HIDE_FROM_ABI explicit time_get_byname(const string& __nm, size_t __refs = 0)
-      : time_get<_CharT, _InputIterator>(__refs), __time_get_storage<_CharT>(__nm) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_get_byname() override {}
-
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL dateorder do_date_order() const override { return this->__do_date_order(); }
-
-private:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __weeks() const override { return this->__weeks_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __months() const override { return this->__months_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type* __am_pm() const override { return this->__am_pm_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __c() const override { return this->__c_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __r() const override { return this->__r_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __x() const override { return this->__x_; }
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL const string_type& __X() const override { return this->__X_; }
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_get_byname<wchar_t>;
-#    endif
-
-class _LIBCPP_EXPORTED_FROM_ABI __time_put {
-  __locale::__locale_t __loc_;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI __time_put() : __loc_(_LIBCPP_GET_C_LOCALE) {}
-  __time_put(const char* __nm);
-  __time_put(const string& __nm);
-  ~__time_put();
-  void __do_put(char* __nb, char*& __ne, const tm* __tm, char __fmt, char __mod) const;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-  void __do_put(wchar_t* __wb, wchar_t*& __we, const tm* __tm, char __fmt, char __mod) const;
-#    endif
-};
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class time_put : public locale::facet, private __time_put {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  iter_type
-  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
-      const;
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, char __fmt, char __mod = 0) const {
-    return do_put(__s, __iob, __fl, __tm, __fmt, __mod);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put() override {}
-  virtual iter_type do_put(iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const;
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(const char* __nm, size_t __refs) : locale::facet(__refs), __time_put(__nm) {}
-  _LIBCPP_HIDE_FROM_ABI explicit time_put(const string& __nm, size_t __refs)
-      : locale::facet(__refs), __time_put(__nm) {}
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id time_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator time_put<_CharT, _OutputIterator>::put(
-    iter_type __s, ios_base& __iob, char_type __fl, const tm* __tm, const char_type* __pb, const char_type* __pe)
-    const {
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__iob.getloc());
-  for (; __pb != __pe; ++__pb) {
-    if (__ct.narrow(*__pb, 0) == '%') {
-      if (++__pb == __pe) {
-        *__s++ = __pb[-1];
-        break;
-      }
-      char __mod = 0;
-      char __fmt = __ct.narrow(*__pb, 0);
-      if (__fmt == 'E' || __fmt == 'O') {
-        if (++__pb == __pe) {
-          *__s++ = __pb[-2];
-          *__s++ = __pb[-1];
-          break;
-        }
-        __mod = __fmt;
-        __fmt = __ct.narrow(*__pb, 0);
-      }
-      __s = do_put(__s, __iob, __fl, __tm, __fmt, __mod);
-    } else
-      *__s++ = *__pb;
-  }
-  return __s;
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator time_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, ios_base&, char_type, const tm* __tm, char __fmt, char __mod) const {
-  char_type __nar[100];
-  char_type* __nb = __nar;
-  char_type* __ne = __nb + 100;
-  __do_put(__nb, __ne, __tm, __fmt, __mod);
-  return std::copy(__nb, __ne, __s);
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class time_put_byname : public time_put<_CharT, _OutputIterator> {
-public:
-  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const char* __nm, size_t __refs = 0)
-      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit time_put_byname(const string& __nm, size_t __refs = 0)
-      : time_put<_CharT, _OutputIterator>(__nm, __refs) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~time_put_byname() override {}
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS time_put_byname<wchar_t>;
-#    endif
-
-// money_base
-
-class _LIBCPP_EXPORTED_FROM_ABI money_base {
-public:
-  enum part { none, space, symbol, sign, value };
-  struct pattern {
-    char field[4];
-  };
-
-  _LIBCPP_HIDE_FROM_ABI money_base() {}
-};
-
-// moneypunct
-
-template <class _CharT, bool _International = false>
-class moneypunct : public locale::facet, public money_base {
-public:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI char_type decimal_point() const { return do_decimal_point(); }
-  _LIBCPP_HIDE_FROM_ABI char_type thousands_sep() const { return do_thousands_sep(); }
-  _LIBCPP_HIDE_FROM_ABI string grouping() const { return do_grouping(); }
-  _LIBCPP_HIDE_FROM_ABI string_type curr_symbol() const { return do_curr_symbol(); }
-  _LIBCPP_HIDE_FROM_ABI string_type positive_sign() const { return do_positive_sign(); }
-  _LIBCPP_HIDE_FROM_ABI string_type negative_sign() const { return do_negative_sign(); }
-  _LIBCPP_HIDE_FROM_ABI int frac_digits() const { return do_frac_digits(); }
-  _LIBCPP_HIDE_FROM_ABI pattern pos_format() const { return do_pos_format(); }
-  _LIBCPP_HIDE_FROM_ABI pattern neg_format() const { return do_neg_format(); }
-
-  static locale::id id;
-  static const bool intl = _International;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct() override {}
-
-  virtual char_type do_decimal_point() const { return numeric_limits<char_type>::max(); }
-  virtual char_type do_thousands_sep() const { return numeric_limits<char_type>::max(); }
-  virtual string do_grouping() const { return string(); }
-  virtual string_type do_curr_symbol() const { return string_type(); }
-  virtual string_type do_positive_sign() const { return string_type(); }
-  virtual string_type do_negative_sign() const { return string_type(1, '-'); }
-  virtual int do_frac_digits() const { return 0; }
-  virtual pattern do_pos_format() const {
-    pattern __p = {{symbol, sign, none, value}};
-    return __p;
-  }
-  virtual pattern do_neg_format() const {
-    pattern __p = {{symbol, sign, none, value}};
-    return __p;
-  }
-};
-
-template <class _CharT, bool _International>
-locale::id moneypunct<_CharT, _International>::id;
-
-template <class _CharT, bool _International>
-const bool moneypunct<_CharT, _International>::intl;
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<char, true>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct<wchar_t, true>;
-#    endif
-
-// moneypunct_byname
-
-template <class _CharT, bool _International = false>
-class moneypunct_byname : public moneypunct<_CharT, _International> {
-public:
-  typedef money_base::pattern pattern;
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const char* __nm, size_t __refs = 0)
-      : moneypunct<_CharT, _International>(__refs) {
-    init(__nm);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI explicit moneypunct_byname(const string& __nm, size_t __refs = 0)
-      : moneypunct<_CharT, _International>(__refs) {
-    init(__nm.c_str());
-  }
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~moneypunct_byname() override {}
-
-  char_type do_decimal_point() const override { return __decimal_point_; }
-  char_type do_thousands_sep() const override { return __thousands_sep_; }
-  string do_grouping() const override { return __grouping_; }
-  string_type do_curr_symbol() const override { return __curr_symbol_; }
-  string_type do_positive_sign() const override { return __positive_sign_; }
-  string_type do_negative_sign() const override { return __negative_sign_; }
-  int do_frac_digits() const override { return __frac_digits_; }
-  pattern do_pos_format() const override { return __pos_format_; }
-  pattern do_neg_format() const override { return __neg_format_; }
-
-private:
-  char_type __decimal_point_;
-  char_type __thousands_sep_;
-  string __grouping_;
-  string_type __curr_symbol_;
-  string_type __positive_sign_;
-  string_type __negative_sign_;
-  int __frac_digits_;
-  pattern __pos_format_;
-  pattern __neg_format_;
-
-  void init(const char*);
-};
-
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, false>::init(const char*);
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<char, true>::init(const char*);
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<char, true>;
-
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, false>::init(const char*);
-template <>
-_LIBCPP_EXPORTED_FROM_ABI void moneypunct_byname<wchar_t, true>::init(const char*);
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, false>;
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS moneypunct_byname<wchar_t, true>;
-#    endif
-
-// money_get
-
-template <class _CharT>
-class __money_get {
-protected:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI __money_get() {}
-
-  static void __gather_info(
-      bool __intl,
-      const locale& __loc,
-      money_base::pattern& __pat,
-      char_type& __dp,
-      char_type& __ts,
-      string& __grp,
-      string_type& __sym,
-      string_type& __psn,
-      string_type& __nsn,
-      int& __fd);
-};
-
-template <class _CharT>
-void __money_get<_CharT>::__gather_info(
-    bool __intl,
-    const locale& __loc,
-    money_base::pattern& __pat,
-    char_type& __dp,
-    char_type& __ts,
-    string& __grp,
-    string_type& __sym,
-    string_type& __psn,
-    string_type& __nsn,
-    int& __fd) {
-  if (__intl) {
-    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
-    __pat                                   = __mp.neg_format();
-    __nsn                                   = __mp.negative_sign();
-    __psn                                   = __mp.positive_sign();
-    __dp                                    = __mp.decimal_point();
-    __ts                                    = __mp.thousands_sep();
-    __grp                                   = __mp.grouping();
-    __sym                                   = __mp.curr_symbol();
-    __fd                                    = __mp.frac_digits();
-  } else {
-    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
-    __pat                                    = __mp.neg_format();
-    __nsn                                    = __mp.negative_sign();
-    __psn                                    = __mp.positive_sign();
-    __dp                                     = __mp.decimal_point();
-    __ts                                     = __mp.thousands_sep();
-    __grp                                    = __mp.grouping();
-    __sym                                    = __mp.curr_symbol();
-    __fd                                     = __mp.frac_digits();
-  }
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_get<wchar_t>;
-#    endif
-
-template <class _CharT, class _InputIterator = istreambuf_iterator<_CharT> >
-class money_get : public locale::facet, private __money_get<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _InputIterator iter_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit money_get(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-    return do_get(__b, __e, __intl, __iob, __err, __v);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
-    return do_get(__b, __e, __intl, __iob, __err, __v);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_get() override {}
-
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const;
-  virtual iter_type
-  do_get(iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const;
-
-private:
-  static bool __do_get(
-      iter_type& __b,
-      iter_type __e,
-      bool __intl,
-      const locale& __loc,
-      ios_base::fmtflags __flags,
-      ios_base::iostate& __err,
-      bool& __neg,
-      const ctype<char_type>& __ct,
-      unique_ptr<char_type, void (*)(void*)>& __wb,
-      char_type*& __wn,
-      char_type* __we);
-};
-
-template <class _CharT, class _InputIterator>
-locale::id money_get<_CharT, _InputIterator>::id;
-
-_LIBCPP_EXPORTED_FROM_ABI void __do_nothing(void*);
-
-template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void __double_or_nothing(unique_ptr<_Tp, void (*)(void*)>& __b, _Tp*& __n, _Tp*& __e) {
-  bool __owns      = __b.get_deleter() != __do_nothing;
-  size_t __cur_cap = static_cast<size_t>(__e - __b.get()) * sizeof(_Tp);
-  size_t __new_cap = __cur_cap < numeric_limits<size_t>::max() / 2 ? 2 * __cur_cap : numeric_limits<size_t>::max();
-  if (__new_cap == 0)
-    __new_cap = sizeof(_Tp);
-  size_t __n_off = static_cast<size_t>(__n - __b.get());
-  _Tp* __t       = (_Tp*)std::realloc(__owns ? __b.get() : 0, __new_cap);
-  if (__t == 0)
-    std::__throw_bad_alloc();
-  if (__owns)
-    __b.release();
-  else
-    std::memcpy(__t, __b.get(), __cur_cap);
-  __b = unique_ptr<_Tp, void (*)(void*)>(__t, free);
-  __new_cap /= sizeof(_Tp);
-  __n = __b.get() + __n_off;
-  __e = __b.get() + __new_cap;
-}
-
-// true == success
-template <class _CharT, class _InputIterator>
-bool money_get<_CharT, _InputIterator>::__do_get(
-    iter_type& __b,
-    iter_type __e,
-    bool __intl,
-    const locale& __loc,
-    ios_base::fmtflags __flags,
-    ios_base::iostate& __err,
-    bool& __neg,
-    const ctype<char_type>& __ct,
-    unique_ptr<char_type, void (*)(void*)>& __wb,
-    char_type*& __wn,
-    char_type* __we) {
-  if (__b == __e) {
-    __err |= ios_base::failbit;
-    return false;
-  }
-  const unsigned __bz = 100;
-  unsigned __gbuf[__bz];
-  unique_ptr<unsigned, void (*)(void*)> __gb(__gbuf, __do_nothing);
-  unsigned* __gn = __gb.get();
-  unsigned* __ge = __gn + __bz;
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __psn;
-  string_type __nsn;
-  // Capture the spaces read into money_base::{space,none} so they
-  // can be compared to initial spaces in __sym.
-  string_type __spaces;
-  int __fd;
-  __money_get<_CharT>::__gather_info(__intl, __loc, __pat, __dp, __ts, __grp, __sym, __psn, __nsn, __fd);
-  const string_type* __trailing_sign = 0;
-  __wn                               = __wb.get();
-  for (unsigned __p = 0; __p < 4 && __b != __e; ++__p) {
-    switch (__pat.field[__p]) {
-    case money_base::space:
-      if (__p != 3) {
-        if (__ct.is(ctype_base::space, *__b))
-          __spaces.push_back(*__b++);
-        else {
-          __err |= ios_base::failbit;
-          return false;
-        }
-      }
-      [[__fallthrough__]];
-    case money_base::none:
-      if (__p != 3) {
-        while (__b != __e && __ct.is(ctype_base::space, *__b))
-          __spaces.push_back(*__b++);
-      }
-      break;
-    case money_base::sign:
-      if (__psn.size() > 0 && *__b == __psn[0]) {
-        ++__b;
-        __neg = false;
-        if (__psn.size() > 1)
-          __trailing_sign = std::addressof(__psn);
-        break;
-      }
-      if (__nsn.size() > 0 && *__b == __nsn[0]) {
-        ++__b;
-        __neg = true;
-        if (__nsn.size() > 1)
-          __trailing_sign = std::addressof(__nsn);
-        break;
-      }
-      if (__psn.size() > 0 && __nsn.size() > 0) { // sign is required
-        __err |= ios_base::failbit;
-        return false;
-      }
-      if (__psn.size() == 0 && __nsn.size() == 0)
-        // locale has no way of specifying a sign. Use the initial value of __neg as a default
-        break;
-      __neg = (__nsn.size() == 0);
-      break;
-    case money_base::symbol: {
-      bool __more_needed =
-          __trailing_sign || (__p < 2) || (__p == 2 && __pat.field[3] != static_cast<char>(money_base::none));
-      bool __sb = (__flags & ios_base::showbase) != 0;
-      if (__sb || __more_needed) {
-        typename string_type::const_iterator __sym_space_end = __sym.begin();
-        if (__p > 0 && (__pat.field[__p - 1] == money_base::none || __pat.field[__p - 1] == money_base::space)) {
-          // Match spaces we've already read against spaces at
-          // the beginning of __sym.
-          while (__sym_space_end != __sym.end() && __ct.is(ctype_base::space, *__sym_space_end))
-            ++__sym_space_end;
-          const size_t __num_spaces = __sym_space_end - __sym.begin();
-          if (__num_spaces > __spaces.size() ||
-              !std::equal(__spaces.end() - __num_spaces, __spaces.end(), __sym.begin())) {
-            // No match. Put __sym_space_end back at the
-            // beginning of __sym, which will prevent a
-            // match in the next loop.
-            __sym_space_end = __sym.begin();
-          }
-        }
-        typename string_type::const_iterator __sym_curr_char = __sym_space_end;
-        while (__sym_curr_char != __sym.end() && __b != __e && *__b == *__sym_curr_char) {
-          ++__b;
-          ++__sym_curr_char;
-        }
-        if (__sb && __sym_curr_char != __sym.end()) {
-          __err |= ios_base::failbit;
-          return false;
-        }
-      }
-    } break;
-    case money_base::value: {
-      unsigned __ng = 0;
-      for (; __b != __e; ++__b) {
-        char_type __c = *__b;
-        if (__ct.is(ctype_base::digit, __c)) {
-          if (__wn == __we)
-            std::__double_or_nothing(__wb, __wn, __we);
-          *__wn++ = __c;
-          ++__ng;
-        } else if (__grp.size() > 0 && __ng > 0 && __c == __ts) {
-          if (__gn == __ge)
-            std::__double_or_nothing(__gb, __gn, __ge);
-          *__gn++ = __ng;
-          __ng    = 0;
-        } else
-          break;
-      }
-      if (__gb.get() != __gn && __ng > 0) {
-        if (__gn == __ge)
-          std::__double_or_nothing(__gb, __gn, __ge);
-        *__gn++ = __ng;
-      }
-      if (__fd > 0) {
-        if (__b == __e || *__b != __dp) {
-          __err |= ios_base::failbit;
-          return false;
-        }
-        for (++__b; __fd > 0; --__fd, ++__b) {
-          if (__b == __e || !__ct.is(ctype_base::digit, *__b)) {
-            __err |= ios_base::failbit;
-            return false;
-          }
-          if (__wn == __we)
-            std::__double_or_nothing(__wb, __wn, __we);
-          *__wn++ = *__b;
-        }
-      }
-      if (__wn == __wb.get()) {
-        __err |= ios_base::failbit;
-        return false;
-      }
-    } break;
-    }
-  }
-  if (__trailing_sign) {
-    for (unsigned __i = 1; __i < __trailing_sign->size(); ++__i, ++__b) {
-      if (__b == __e || *__b != (*__trailing_sign)[__i]) {
-        __err |= ios_base::failbit;
-        return false;
-      }
-    }
-  }
-  if (__gb.get() != __gn) {
-    ios_base::iostate __et = ios_base::goodbit;
-    __check_grouping(__grp, __gb.get(), __gn, __et);
-    if (__et) {
-      __err |= ios_base::failbit;
-      return false;
-    }
-  }
-  return true;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator money_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, long double& __v) const {
-  const int __bz = 100;
-  char_type __wbuf[__bz];
-  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
-  char_type* __wn;
-  char_type* __we              = __wbuf + __bz;
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = false;
-  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
-    const char __src[] = "0123456789";
-    char_type __atoms[sizeof(__src) - 1];
-    __ct.widen(__src, __src + (sizeof(__src) - 1), __atoms);
-    char __nbuf[__bz];
-    char* __nc          = __nbuf;
-    const char* __nc_in = __nc;
-    unique_ptr<char, void (*)(void*)> __h(nullptr, free);
-    if (__wn - __wb.get() > __bz - 2) {
-      __h.reset((char*)malloc(static_cast<size_t>(__wn - __wb.get() + 2)));
-      if (__h.get() == nullptr)
-        std::__throw_bad_alloc();
-      __nc    = __h.get();
-      __nc_in = __nc;
-    }
-    if (__neg)
-      *__nc++ = '-';
-    for (const char_type* __w = __wb.get(); __w < __wn; ++__w, ++__nc)
-      *__nc = __src[std::find(__atoms, std::end(__atoms), *__w) - __atoms];
-    *__nc = char();
-    if (sscanf(__nc_in, "%Lf", &__v) != 1)
-      std::__throw_runtime_error("money_get error");
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-template <class _CharT, class _InputIterator>
-_InputIterator money_get<_CharT, _InputIterator>::do_get(
-    iter_type __b, iter_type __e, bool __intl, ios_base& __iob, ios_base::iostate& __err, string_type& __v) const {
-  const int __bz = 100;
-  char_type __wbuf[__bz];
-  unique_ptr<char_type, void (*)(void*)> __wb(__wbuf, __do_nothing);
-  char_type* __wn;
-  char_type* __we              = __wbuf + __bz;
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = false;
-  if (__do_get(__b, __e, __intl, __loc, __iob.flags(), __err, __neg, __ct, __wb, __wn, __we)) {
-    __v.clear();
-    if (__neg)
-      __v.push_back(__ct.widen('-'));
-    char_type __z = __ct.widen('0');
-    char_type* __w;
-    for (__w = __wb.get(); __w < __wn - 1; ++__w)
-      if (*__w != __z)
-        break;
-    __v.append(__w, __wn);
-  }
-  if (__b == __e)
-    __err |= ios_base::eofbit;
-  return __b;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_get<wchar_t>;
-#    endif
-
-// money_put
-
-template <class _CharT>
-class __money_put {
-protected:
-  typedef _CharT char_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI __money_put() {}
-
-  static void __gather_info(
-      bool __intl,
-      bool __neg,
-      const locale& __loc,
-      money_base::pattern& __pat,
-      char_type& __dp,
-      char_type& __ts,
-      string& __grp,
-      string_type& __sym,
-      string_type& __sn,
-      int& __fd);
-  static void __format(
-      char_type* __mb,
-      char_type*& __mi,
-      char_type*& __me,
-      ios_base::fmtflags __flags,
-      const char_type* __db,
-      const char_type* __de,
-      const ctype<char_type>& __ct,
-      bool __neg,
-      const money_base::pattern& __pat,
-      char_type __dp,
-      char_type __ts,
-      const string& __grp,
-      const string_type& __sym,
-      const string_type& __sn,
-      int __fd);
-};
-
-template <class _CharT>
-void __money_put<_CharT>::__gather_info(
-    bool __intl,
-    bool __neg,
-    const locale& __loc,
-    money_base::pattern& __pat,
-    char_type& __dp,
-    char_type& __ts,
-    string& __grp,
-    string_type& __sym,
-    string_type& __sn,
-    int& __fd) {
-  if (__intl) {
-    const moneypunct<char_type, true>& __mp = std::use_facet<moneypunct<char_type, true> >(__loc);
-    if (__neg) {
-      __pat = __mp.neg_format();
-      __sn  = __mp.negative_sign();
-    } else {
-      __pat = __mp.pos_format();
-      __sn  = __mp.positive_sign();
-    }
-    __dp  = __mp.decimal_point();
-    __ts  = __mp.thousands_sep();
-    __grp = __mp.grouping();
-    __sym = __mp.curr_symbol();
-    __fd  = __mp.frac_digits();
-  } else {
-    const moneypunct<char_type, false>& __mp = std::use_facet<moneypunct<char_type, false> >(__loc);
-    if (__neg) {
-      __pat = __mp.neg_format();
-      __sn  = __mp.negative_sign();
-    } else {
-      __pat = __mp.pos_format();
-      __sn  = __mp.positive_sign();
-    }
-    __dp  = __mp.decimal_point();
-    __ts  = __mp.thousands_sep();
-    __grp = __mp.grouping();
-    __sym = __mp.curr_symbol();
-    __fd  = __mp.frac_digits();
-  }
-}
-
-template <class _CharT>
-void __money_put<_CharT>::__format(
-    char_type* __mb,
-    char_type*& __mi,
-    char_type*& __me,
-    ios_base::fmtflags __flags,
-    const char_type* __db,
-    const char_type* __de,
-    const ctype<char_type>& __ct,
-    bool __neg,
-    const money_base::pattern& __pat,
-    char_type __dp,
-    char_type __ts,
-    const string& __grp,
-    const string_type& __sym,
-    const string_type& __sn,
-    int __fd) {
-  __me = __mb;
-  for (char __p : __pat.field) {
-    switch (__p) {
-    case money_base::none:
-      __mi = __me;
-      break;
-    case money_base::space:
-      __mi    = __me;
-      *__me++ = __ct.widen(' ');
-      break;
-    case money_base::sign:
-      if (!__sn.empty())
-        *__me++ = __sn[0];
-      break;
-    case money_base::symbol:
-      if (!__sym.empty() && (__flags & ios_base::showbase))
-        __me = std::copy(__sym.begin(), __sym.end(), __me);
-      break;
-    case money_base::value: {
-      // remember start of value so we can reverse it
-      char_type* __t = __me;
-      // find beginning of digits
-      if (__neg)
-        ++__db;
-      // find end of digits
-      const char_type* __d;
-      for (__d = __db; __d < __de; ++__d)
-        if (!__ct.is(ctype_base::digit, *__d))
-          break;
-      // print fractional part
-      if (__fd > 0) {
-        int __f;
-        for (__f = __fd; __d > __db && __f > 0; --__f)
-          *__me++ = *--__d;
-        char_type __z = __f > 0 ? __ct.widen('0') : char_type();
-        for (; __f > 0; --__f)
-          *__me++ = __z;
-        *__me++ = __dp;
-      }
-      // print units part
-      if (__d == __db) {
-        *__me++ = __ct.widen('0');
-      } else {
-        unsigned __ng = 0;
-        unsigned __ig = 0;
-        unsigned __gl = __grp.empty() ? numeric_limits<unsigned>::max() : static_cast<unsigned>(__grp[__ig]);
-        while (__d != __db) {
-          if (__ng == __gl) {
-            *__me++ = __ts;
-            __ng    = 0;
-            if (++__ig < __grp.size())
-              __gl = __grp[__ig] == numeric_limits<char>::max()
-                       ? numeric_limits<unsigned>::max()
-                       : static_cast<unsigned>(__grp[__ig]);
-          }
-          *__me++ = *--__d;
-          ++__ng;
-        }
-      }
-      // reverse it
-      std::reverse(__t, __me);
-    } break;
-    }
-  }
-  // print rest of sign, if any
-  if (__sn.size() > 1)
-    __me = std::copy(__sn.begin() + 1, __sn.end(), __me);
-  // set alignment
-  if ((__flags & ios_base::adjustfield) == ios_base::left)
-    __mi = __me;
-  else if ((__flags & ios_base::adjustfield) != ios_base::internal)
-    __mi = __mb;
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS __money_put<wchar_t>;
-#    endif
-
-template <class _CharT, class _OutputIterator = ostreambuf_iterator<_CharT> >
-class money_put : public locale::facet, private __money_put<_CharT> {
-public:
-  typedef _CharT char_type;
-  typedef _OutputIterator iter_type;
-  typedef basic_string<char_type> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit money_put(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
-    return do_put(__s, __intl, __iob, __fl, __units);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI iter_type
-  put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
-    return do_put(__s, __intl, __iob, __fl, __digits);
-  }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~money_put() override {}
-
-  virtual iter_type do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const;
-  virtual iter_type
-  do_put(iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const;
-};
-
-template <class _CharT, class _OutputIterator>
-locale::id money_put<_CharT, _OutputIterator>::id;
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, long double __units) const {
-  // convert to char
-  const size_t __bs = 100;
-  char __buf[__bs];
-  char* __bb = __buf;
-  char_type __digits[__bs];
-  char_type* __db = __digits;
-  int __n         = snprintf(__bb, __bs, "%.0Lf", __units);
-  unique_ptr<char, void (*)(void*)> __hn(nullptr, free);
-  unique_ptr<char_type, void (*)(void*)> __hd(0, free);
-  // secure memory for digit storage
-  if (static_cast<size_t>(__n) > __bs - 1) {
-    __n = __locale::__asprintf(&__bb, _LIBCPP_GET_C_LOCALE, "%.0Lf", __units);
-    if (__n == -1)
-      std::__throw_bad_alloc();
-    __hn.reset(__bb);
-    __hd.reset((char_type*)malloc(static_cast<size_t>(__n) * sizeof(char_type)));
-    if (__hd == nullptr)
-      std::__throw_bad_alloc();
-    __db = __hd.get();
-  }
-  // gather info
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  __ct.widen(__bb, __bb + __n, __db);
-  bool __neg = __n > 0 && __bb[0] == '-';
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __sn;
-  int __fd;
-  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  // secure memory for formatting
-  char_type __mbuf[__bs];
-  char_type* __mb = __mbuf;
-  unique_ptr<char_type, void (*)(void*)> __hw(0, free);
-  size_t __exn = __n > __fd ? (static_cast<size_t>(__n) - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() +
-                                  static_cast<size_t>(__fd) + 1
-                            : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
-  if (__exn > __bs) {
-    __hw.reset((char_type*)malloc(__exn * sizeof(char_type)));
-    __mb = __hw.get();
-    if (__mb == 0)
-      std::__throw_bad_alloc();
-  }
-  // format
-  char_type* __mi;
-  char_type* __me;
-  this->__format(
-      __mb, __mi, __me, __iob.flags(), __db, __db + __n, __ct, __neg, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
-}
-
-template <class _CharT, class _OutputIterator>
-_OutputIterator money_put<_CharT, _OutputIterator>::do_put(
-    iter_type __s, bool __intl, ios_base& __iob, char_type __fl, const string_type& __digits) const {
-  // gather info
-  locale __loc                 = __iob.getloc();
-  const ctype<char_type>& __ct = std::use_facet<ctype<char_type> >(__loc);
-  bool __neg                   = __digits.size() > 0 && __digits[0] == __ct.widen('-');
-  money_base::pattern __pat;
-  char_type __dp;
-  char_type __ts;
-  string __grp;
-  string_type __sym;
-  string_type __sn;
-  int __fd;
-  this->__gather_info(__intl, __neg, __loc, __pat, __dp, __ts, __grp, __sym, __sn, __fd);
-  // secure memory for formatting
-  char_type __mbuf[100];
-  char_type* __mb = __mbuf;
-  unique_ptr<char_type, void (*)(void*)> __h(0, free);
-  size_t __exn =
-      static_cast<int>(__digits.size()) > __fd
-          ? (__digits.size() - static_cast<size_t>(__fd)) * 2 + __sn.size() + __sym.size() + static_cast<size_t>(__fd) +
-                1
-          : __sn.size() + __sym.size() + static_cast<size_t>(__fd) + 2;
-  if (__exn > 100) {
-    __h.reset((char_type*)malloc(__exn * sizeof(char_type)));
-    __mb = __h.get();
-    if (__mb == 0)
-      std::__throw_bad_alloc();
-  }
-  // format
-  char_type* __mi;
-  char_type* __me;
-  this->__format(
-      __mb,
-      __mi,
-      __me,
-      __iob.flags(),
-      __digits.data(),
-      __digits.data() + __digits.size(),
-      __ct,
-      __neg,
-      __pat,
-      __dp,
-      __ts,
-      __grp,
-      __sym,
-      __sn,
-      __fd);
-  return std::__pad_and_output(__s, __mb, __mi, __me, __iob, __fl);
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS money_put<wchar_t>;
-#    endif
-
-// messages
-
-class _LIBCPP_EXPORTED_FROM_ABI messages_base {
-public:
-  typedef intptr_t catalog;
-
-  _LIBCPP_HIDE_FROM_ABI messages_base() {}
-};
-
-template <class _CharT>
-class messages : public locale::facet, public messages_base {
-public:
-  typedef _CharT char_type;
-  typedef basic_string<_CharT> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages(size_t __refs = 0) : locale::facet(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI catalog open(const basic_string<char>& __nm, const locale& __loc) const {
-    return do_open(__nm, __loc);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI string_type get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-    return do_get(__c, __set, __msgid, __dflt);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI void close(catalog __c) const { do_close(__c); }
-
-  static locale::id id;
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages() override {}
-
-  virtual catalog do_open(const basic_string<char>&, const locale&) const;
-  virtual string_type do_get(catalog, int __set, int __msgid, const string_type& __dflt) const;
-  virtual void do_close(catalog) const;
-};
-
-template <class _CharT>
-locale::id messages<_CharT>::id;
-
-template <class _CharT>
-typename messages<_CharT>::catalog messages<_CharT>::do_open(const basic_string<char>& __nm, const locale&) const {
-#    if _LIBCPP_HAS_CATOPEN
-  return (catalog)catopen(__nm.c_str(), NL_CAT_LOCALE);
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__nm;
-  return -1;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-template <class _CharT>
-typename messages<_CharT>::string_type
-messages<_CharT>::do_get(catalog __c, int __set, int __msgid, const string_type& __dflt) const {
-#    if _LIBCPP_HAS_CATOPEN
-  string __ndflt;
-  __narrow_to_utf8<sizeof(char_type) * __CHAR_BIT__>()(
-      std::back_inserter(__ndflt), __dflt.c_str(), __dflt.c_str() + __dflt.size());
-  nl_catd __cat = (nl_catd)__c;
-  static_assert(sizeof(catalog) >= sizeof(nl_catd), "Unexpected nl_catd type");
-  char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
-  string_type __w;
-  __widen_from_utf8<sizeof(char_type) * __CHAR_BIT__>()(std::back_inserter(__w), __n, __n + std::strlen(__n));
-  return __w;
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__c;
-  (void)__set;
-  (void)__msgid;
-  return __dflt;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-template <class _CharT>
-void messages<_CharT>::do_close(catalog __c) const {
-#    if _LIBCPP_HAS_CATOPEN
-  catclose((nl_catd)__c);
-#    else  // !_LIBCPP_HAS_CATOPEN
-  (void)__c;
-#    endif // _LIBCPP_HAS_CATOPEN
-}
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages<wchar_t>;
-#    endif
-
-template <class _CharT>
-class messages_byname : public messages<_CharT> {
-public:
-  typedef messages_base::catalog catalog;
-  typedef basic_string<_CharT> string_type;
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const char*, size_t __refs = 0) : messages<_CharT>(__refs) {}
-
-  _LIBCPP_HIDE_FROM_ABI explicit messages_byname(const string&, size_t __refs = 0) : messages<_CharT>(__refs) {}
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~messages_byname() override {}
-};
-
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<char>;
-#    if _LIBCPP_HAS_WIDE_CHARACTERS
-extern template class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS messages_byname<wchar_t>;
-#    endif
-
-#    if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
-
-template <class _Codecvt,
-          class _Elem      = wchar_t,
-          class _WideAlloc = allocator<_Elem>,
-          class _ByteAlloc = allocator<char> >
-class _LIBCPP_DEPRECATED_IN_CXX17 wstring_convert {
-public:
-  typedef basic_string<char, char_traits<char>, _ByteAlloc> byte_string;
-  typedef basic_string<_Elem, char_traits<_Elem>, _WideAlloc> wide_string;
-  typedef typename _Codecvt::state_type state_type;
-  typedef typename wide_string::traits_type::int_type int_type;
-
-private:
-  byte_string __byte_err_string_;
-  wide_string __wide_err_string_;
-  _Codecvt* __cvtptr_;
-  state_type __cvtstate_;
-  size_t __cvtcount_;
-
-public:
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wstring_convert() : wstring_convert(new _Codecvt) {}
-  _LIBCPP_HIDE_FROM_ABI explicit wstring_convert(_Codecvt* __pcvt);
-#      else
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_EXPLICIT_SINCE_CXX14 wstring_convert(_Codecvt* __pcvt = new _Codecvt);
-#      endif
-
-  _LIBCPP_HIDE_FROM_ABI wstring_convert(_Codecvt* __pcvt, state_type __state);
-  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
-  wstring_convert(const byte_string& __byte_err, const wide_string& __wide_err = wide_string());
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wstring_convert(wstring_convert&& __wc);
-#      endif
-  _LIBCPP_HIDE_FROM_ABI ~wstring_convert();
-
-  wstring_convert(const wstring_convert& __wc)            = delete;
-  wstring_convert& operator=(const wstring_convert& __wc) = delete;
-
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(char __byte) { return from_bytes(&__byte, &__byte + 1); }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __ptr) {
-    return from_bytes(__ptr, __ptr + char_traits<char>::length(__ptr));
-  }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const byte_string& __str) {
-    return from_bytes(__str.data(), __str.data() + __str.size());
-  }
-  _LIBCPP_HIDE_FROM_ABI wide_string from_bytes(const char* __first, const char* __last);
-
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(_Elem __wchar) { return to_bytes(std::addressof(__wchar), std::addressof(__wchar) + 1); }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __wptr) {
-    return to_bytes(__wptr, __wptr + char_traits<_Elem>::length(__wptr));
-  }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const wide_string& __wstr) {
-    return to_bytes(__wstr.data(), __wstr.data() + __wstr.size());
-  }
-  _LIBCPP_HIDE_FROM_ABI byte_string to_bytes(const _Elem* __first, const _Elem* __last);
-
-  _LIBCPP_HIDE_FROM_ABI size_t converted() const _NOEXCEPT { return __cvtcount_; }
-  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __cvtstate_; }
-};
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt)
-    : __cvtptr_(__pcvt), __cvtstate_(), __cvtcount_(0) {}
-_LIBCPP_SUPPRESS_DEPRECATED_POP
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(_Codecvt* __pcvt, state_type __state)
-    : __cvtptr_(__pcvt), __cvtstate_(__state), __cvtcount_(0) {}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(
-    const byte_string& __byte_err, const wide_string& __wide_err)
-    : __byte_err_string_(__byte_err), __wide_err_string_(__wide_err), __cvtstate_(), __cvtcount_(0) {
-  __cvtptr_ = new _Codecvt;
-}
-
-#      ifndef _LIBCPP_CXX03_LANG
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-inline wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wstring_convert(wstring_convert&& __wc)
-    : __byte_err_string_(std::move(__wc.__byte_err_string_)),
-      __wide_err_string_(std::move(__wc.__wide_err_string_)),
-      __cvtptr_(__wc.__cvtptr_),
-      __cvtstate_(__wc.__cvtstate_),
-      __cvtcount_(__wc.__cvtcount_) {
-  __wc.__cvtptr_ = nullptr;
-}
-
-#      endif // _LIBCPP_CXX03_LANG
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::~wstring_convert() {
-  delete __cvtptr_;
-}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::wide_string
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::from_bytes(const char* __frm, const char* __frm_end) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  __cvtcount_ = 0;
-  if (__cvtptr_ != nullptr) {
-    wide_string __ws(2 * (__frm_end - __frm), _Elem());
-    if (__frm != __frm_end)
-      __ws.resize(__ws.capacity());
-    codecvt_base::result __r = codecvt_base::ok;
-    state_type __st          = __cvtstate_;
-    if (__frm != __frm_end) {
-      _Elem* __to     = std::addressof(__ws[0]);
-      _Elem* __to_end = __to + __ws.size();
-      const char* __frm_nxt;
-      do {
-        _Elem* __to_nxt;
-        __r = __cvtptr_->in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
-        __cvtcount_ += __frm_nxt - __frm;
-        if (__frm_nxt == __frm) {
-          __r = codecvt_base::error;
-        } else if (__r == codecvt_base::noconv) {
-          __ws.resize(__to - std::addressof(__ws[0]));
-          // This only gets executed if _Elem is char
-          __ws.append((const _Elem*)__frm, (const _Elem*)__frm_end);
-          __frm = __frm_nxt;
-          __r   = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __ws.resize(__to_nxt - std::addressof(__ws[0]));
-          __frm = __frm_nxt;
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __s = __to_nxt - std::addressof(__ws[0]);
-          __ws.resize(2 * __s);
-          __to     = std::addressof(__ws[0]) + __s;
-          __to_end = std::addressof(__ws[0]) + __ws.size();
-          __frm    = __frm_nxt;
-        }
-      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
-    }
-    if (__r == codecvt_base::ok)
-      return __ws;
-  }
-
-  if (__wide_err_string_.empty())
-    std::__throw_range_error("wstring_convert: from_bytes error");
-
-  return __wide_err_string_;
-}
-
-template <class _Codecvt, class _Elem, class _WideAlloc, class _ByteAlloc>
-typename wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::byte_string
-wstring_convert<_Codecvt, _Elem, _WideAlloc, _ByteAlloc>::to_bytes(const _Elem* __frm, const _Elem* __frm_end) {
-  __cvtcount_ = 0;
-  if (__cvtptr_ != nullptr) {
-    byte_string __bs(2 * (__frm_end - __frm), char());
-    if (__frm != __frm_end)
-      __bs.resize(__bs.capacity());
-    codecvt_base::result __r = codecvt_base::ok;
-    state_type __st          = __cvtstate_;
-    if (__frm != __frm_end) {
-      char* __to     = std::addressof(__bs[0]);
-      char* __to_end = __to + __bs.size();
-      const _Elem* __frm_nxt;
-      do {
-        char* __to_nxt;
-        __r = __cvtptr_->out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
-        __cvtcount_ += __frm_nxt - __frm;
-        if (__frm_nxt == __frm) {
-          __r = codecvt_base::error;
-        } else if (__r == codecvt_base::noconv) {
-          __bs.resize(__to - std::addressof(__bs[0]));
-          // This only gets executed if _Elem is char
-          __bs.append((const char*)__frm, (const char*)__frm_end);
-          __frm = __frm_nxt;
-          __r   = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __bs.resize(__to_nxt - std::addressof(__bs[0]));
-          __frm = __frm_nxt;
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __s = __to_nxt - std::addressof(__bs[0]);
-          __bs.resize(2 * __s);
-          __to     = std::addressof(__bs[0]) + __s;
-          __to_end = std::addressof(__bs[0]) + __bs.size();
-          __frm    = __frm_nxt;
-        }
-      } while (__r == codecvt_base::partial && __frm_nxt < __frm_end);
-    }
-    if (__r == codecvt_base::ok) {
-      size_t __s = __bs.size();
-      __bs.resize(__bs.capacity());
-      char* __to     = std::addressof(__bs[0]) + __s;
-      char* __to_end = __to + __bs.size();
-      do {
-        char* __to_nxt;
-        __r = __cvtptr_->unshift(__st, __to, __to_end, __to_nxt);
-        if (__r == codecvt_base::noconv) {
-          __bs.resize(__to - std::addressof(__bs[0]));
-          __r = codecvt_base::ok;
-        } else if (__r == codecvt_base::ok) {
-          __bs.resize(__to_nxt - std::addressof(__bs[0]));
-        } else if (__r == codecvt_base::partial) {
-          ptrdiff_t __sp = __to_nxt - std::addressof(__bs[0]);
-          __bs.resize(2 * __sp);
-          __to     = std::addressof(__bs[0]) + __sp;
-          __to_end = std::addressof(__bs[0]) + __bs.size();
-        }
-      } while (__r == codecvt_base::partial);
-      if (__r == codecvt_base::ok)
-        return __bs;
-    }
-  }
-
-  if (__byte_err_string_.empty())
-    std::__throw_range_error("wstring_convert: to_bytes error");
-
-  return __byte_err_string_;
-}
-
-template <class _Codecvt, class _Elem = wchar_t, class _Tr = char_traits<_Elem> >
-class _LIBCPP_DEPRECATED_IN_CXX17 wbuffer_convert : public basic_streambuf<_Elem, _Tr> {
-public:
-  // types:
-  typedef _Elem char_type;
-  typedef _Tr traits_type;
-  typedef typename traits_type::int_type int_type;
-  typedef typename traits_type::pos_type pos_type;
-  typedef typename traits_type::off_type off_type;
-  typedef typename _Codecvt::state_type state_type;
-
-private:
-  char* __extbuf_;
-  const char* __extbufnext_;
-  const char* __extbufend_;
-  char __extbuf_min_[8];
-  size_t __ebs_;
-  char_type* __intbuf_;
-  size_t __ibs_;
-  streambuf* __bufptr_;
-  _Codecvt* __cv_;
-  state_type __st_;
-  ios_base::openmode __cm_;
-  bool __owns_eb_;
-  bool __owns_ib_;
-  bool __always_noconv_;
-
-public:
-#      ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI wbuffer_convert() : wbuffer_convert(nullptr) {}
-  explicit _LIBCPP_HIDE_FROM_ABI
-  wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#      else
-  _LIBCPP_EXPLICIT_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI
-  wbuffer_convert(streambuf* __bytebuf = nullptr, _Codecvt* __pcvt = new _Codecvt, state_type __state = state_type());
-#      endif
-
-  _LIBCPP_HIDE_FROM_ABI ~wbuffer_convert();
-
-  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf() const { return __bufptr_; }
-  _LIBCPP_HIDE_FROM_ABI streambuf* rdbuf(streambuf* __bytebuf) {
-    streambuf* __r = __bufptr_;
-    __bufptr_      = __bytebuf;
-    return __r;
-  }
-
-  wbuffer_convert(const wbuffer_convert&)            = delete;
-  wbuffer_convert& operator=(const wbuffer_convert&) = delete;
-
-  _LIBCPP_HIDE_FROM_ABI state_type state() const { return __st_; }
-
-protected:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type underflow();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type pbackfail(int_type __c = traits_type::eof());
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int_type overflow(int_type __c = traits_type::eof());
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual basic_streambuf<char_type, traits_type>* setbuf(char_type* __s, streamsize __n);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
-  seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __wch = ios_base::in | ios_base::out);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual pos_type
-  seekpos(pos_type __sp, ios_base::openmode __wch = ios_base::in | ios_base::out);
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL virtual int sync();
-
-private:
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL bool __read_mode();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL void __write_mode();
-  _LIBCPP_HIDE_FROM_ABI_VIRTUAL wbuffer_convert* __close();
-};
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>::wbuffer_convert(streambuf* __bytebuf, _Codecvt* __pcvt, state_type __state)
-    : __extbuf_(nullptr),
-      __extbufnext_(nullptr),
-      __extbufend_(nullptr),
-      __ebs_(0),
-      __intbuf_(0),
-      __ibs_(0),
-      __bufptr_(__bytebuf),
-      __cv_(__pcvt),
-      __st_(__state),
-      __cm_(0),
-      __owns_eb_(false),
-      __owns_ib_(false),
-      __always_noconv_(__cv_ ? __cv_->always_noconv() : false) {
-  setbuf(0, 4096);
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>::~wbuffer_convert() {
-  __close();
-  delete __cv_;
-  if (__owns_eb_)
-    delete[] __extbuf_;
-  if (__owns_ib_)
-    delete[] __intbuf_;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow() {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || __bufptr_ == nullptr)
-    return traits_type::eof();
-  bool __initial = __read_mode();
-  char_type __1buf;
-  if (this->gptr() == 0)
-    this->setg(std::addressof(__1buf), std::addressof(__1buf) + 1, std::addressof(__1buf) + 1);
-  const size_t __unget_sz = __initial ? 0 : std::min<size_t>((this->egptr() - this->eback()) / 2, 4);
-  int_type __c            = traits_type::eof();
-  if (this->gptr() == this->egptr()) {
-    std::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
-    if (__always_noconv_) {
-      streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
-      __nmemb            = __bufptr_->sgetn((char*)this->eback() + __unget_sz, __nmemb);
-      if (__nmemb != 0) {
-        this->setg(this->eback(), this->eback() + __unget_sz, this->eback() + __unget_sz + __nmemb);
-        __c = *this->gptr();
-      }
-    } else {
-      if (__extbufend_ != __extbufnext_) {
-        _LIBCPP_ASSERT_NON_NULL(__extbufnext_ != nullptr, "underflow moving from nullptr");
-        _LIBCPP_ASSERT_NON_NULL(__extbuf_ != nullptr, "underflow moving into nullptr");
-        std::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
-      }
-      __extbufnext_      = __extbuf_ + (__extbufend_ - __extbufnext_);
-      __extbufend_       = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
-      streamsize __nmemb = std::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
-                                    static_cast<streamsize>(__extbufend_ - __extbufnext_));
-      codecvt_base::result __r;
-      // FIXME: Do we ever need to restore the state here?
-      // state_type __svs = __st_;
-      streamsize __nr = __bufptr_->sgetn(const_cast<char*>(__extbufnext_), __nmemb);
-      if (__nr != 0) {
-        __extbufend_ = __extbufnext_ + __nr;
-        char_type* __inext;
-        __r = __cv_->in(
-            __st_, __extbuf_, __extbufend_, __extbufnext_, this->eback() + __unget_sz, this->egptr(), __inext);
-        if (__r == codecvt_base::noconv) {
-          this->setg((char_type*)__extbuf_, (char_type*)__extbuf_, (char_type*)const_cast<char*>(__extbufend_));
-          __c = *this->gptr();
-        } else if (__inext != this->eback() + __unget_sz) {
-          this->setg(this->eback(), this->eback() + __unget_sz, __inext);
-          __c = *this->gptr();
-        }
-      }
-    }
-  } else
-    __c = *this->gptr();
-  if (this->eback() == std::addressof(__1buf))
-    this->setg(0, 0, 0);
-  return __c;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::pbackfail(int_type __c) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ != 0 && __bufptr_ && this->eback() < this->gptr()) {
-    if (traits_type::eq_int_type(__c, traits_type::eof())) {
-      this->gbump(-1);
-      return traits_type::not_eof(__c);
-    }
-    if (traits_type::eq(traits_type::to_char_type(__c), this->gptr()[-1])) {
-      this->gbump(-1);
-      *this->gptr() = traits_type::to_char_type(__c);
-      return __c;
-    }
-  }
-  return traits_type::eof();
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::int_type wbuffer_convert<_Codecvt, _Elem, _Tr>::overflow(int_type __c) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || !__bufptr_)
-    return traits_type::eof();
-  __write_mode();
-  char_type __1buf;
-  char_type* __pb_save  = this->pbase();
-  char_type* __epb_save = this->epptr();
-  if (!traits_type::eq_int_type(__c, traits_type::eof())) {
-    if (this->pptr() == 0)
-      this->setp(std::addressof(__1buf), std::addressof(__1buf) + 1);
-    *this->pptr() = traits_type::to_char_type(__c);
-    this->pbump(1);
-  }
-  if (this->pptr() != this->pbase()) {
-    if (__always_noconv_) {
-      streamsize __nmemb = static_cast<streamsize>(this->pptr() - this->pbase());
-      if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
-        return traits_type::eof();
-    } else {
-      char* __extbe = __extbuf_;
-      codecvt_base::result __r;
-      do {
-        const char_type* __e;
-        __r = __cv_->out(__st_, this->pbase(), this->pptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
-        if (__e == this->pbase())
-          return traits_type::eof();
-        if (__r == codecvt_base::noconv) {
-          streamsize __nmemb = static_cast<size_t>(this->pptr() - this->pbase());
-          if (__bufptr_->sputn((const char*)this->pbase(), __nmemb) != __nmemb)
-            return traits_type::eof();
-        } else if (__r == codecvt_base::ok || __r == codecvt_base::partial) {
-          streamsize __nmemb = static_cast<size_t>(__extbe - __extbuf_);
-          if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
-            return traits_type::eof();
-          if (__r == codecvt_base::partial) {
-            this->setp(const_cast<char_type*>(__e), this->pptr());
-            this->__pbump(this->epptr() - this->pbase());
-          }
-        } else
-          return traits_type::eof();
-      } while (__r == codecvt_base::partial);
-    }
-    this->setp(__pb_save, __epb_save);
-  }
-  return traits_type::not_eof(__c);
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-basic_streambuf<_Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::setbuf(char_type* __s, streamsize __n) {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  this->setg(0, 0, 0);
-  this->setp(0, 0);
-  if (__owns_eb_)
-    delete[] __extbuf_;
-  if (__owns_ib_)
-    delete[] __intbuf_;
-  __ebs_ = __n;
-  if (__ebs_ > sizeof(__extbuf_min_)) {
-    if (__always_noconv_ && __s) {
-      __extbuf_  = (char*)__s;
-      __owns_eb_ = false;
-    } else {
-      __extbuf_  = new char[__ebs_];
-      __owns_eb_ = true;
-    }
-  } else {
-    __extbuf_  = __extbuf_min_;
-    __ebs_     = sizeof(__extbuf_min_);
-    __owns_eb_ = false;
-  }
-  if (!__always_noconv_) {
-    __ibs_ = max<streamsize>(__n, sizeof(__extbuf_min_));
-    if (__s && __ibs_ >= sizeof(__extbuf_min_)) {
-      __intbuf_  = __s;
-      __owns_ib_ = false;
-    } else {
-      __intbuf_  = new char_type[__ibs_];
-      __owns_ib_ = true;
-    }
-  } else {
-    __ibs_     = 0;
-    __intbuf_  = 0;
-    __owns_ib_ = false;
-  }
-  return this;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::seekoff(off_type __off, ios_base::seekdir __way, ios_base::openmode __om) {
-  int __width = __cv_->encoding();
-  if (__cv_ == 0 || !__bufptr_ || (__width <= 0 && __off != 0) || sync())
-    return pos_type(off_type(-1));
-  // __width > 0 || __off == 0, now check __way
-  if (__way != ios_base::beg && __way != ios_base::cur && __way != ios_base::end)
-    return pos_type(off_type(-1));
-  pos_type __r = __bufptr_->pubseekoff(__width * __off, __way, __om);
-  __r.state(__st_);
-  return __r;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-typename wbuffer_convert<_Codecvt, _Elem, _Tr>::pos_type
-wbuffer_convert<_Codecvt, _Elem, _Tr>::seekpos(pos_type __sp, ios_base::openmode __wch) {
-  if (__cv_ == 0 || !__bufptr_ || sync())
-    return pos_type(off_type(-1));
-  if (__bufptr_->pubseekpos(__sp, __wch) == pos_type(off_type(-1)))
-    return pos_type(off_type(-1));
-  return __sp;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-int wbuffer_convert<_Codecvt, _Elem, _Tr>::sync() {
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
-  if (__cv_ == 0 || !__bufptr_)
-    return 0;
-  if (__cm_ & ios_base::out) {
-    if (this->pptr() != this->pbase())
-      if (overflow() == traits_type::eof())
-        return -1;
-    codecvt_base::result __r;
-    do {
-      char* __extbe;
-      __r                = __cv_->unshift(__st_, __extbuf_, __extbuf_ + __ebs_, __extbe);
-      streamsize __nmemb = static_cast<streamsize>(__extbe - __extbuf_);
-      if (__bufptr_->sputn(__extbuf_, __nmemb) != __nmemb)
-        return -1;
-    } while (__r == codecvt_base::partial);
-    if (__r == codecvt_base::error)
-      return -1;
-    if (__bufptr_->pubsync())
-      return -1;
-  } else if (__cm_ & ios_base::in) {
-    off_type __c;
-    if (__always_noconv_)
-      __c = this->egptr() - this->gptr();
-    else {
-      int __width = __cv_->encoding();
-      __c         = __extbufend_ - __extbufnext_;
-      if (__width > 0)
-        __c += __width * (this->egptr() - this->gptr());
-      else {
-        if (this->gptr() != this->egptr()) {
-          std::reverse(this->gptr(), this->egptr());
-          codecvt_base::result __r;
-          const char_type* __e = this->gptr();
-          char* __extbe;
-          do {
-            __r = __cv_->out(__st_, __e, this->egptr(), __e, __extbuf_, __extbuf_ + __ebs_, __extbe);
-            switch (__r) {
-            case codecvt_base::noconv:
-              __c += this->egptr() - this->gptr();
-              break;
-            case codecvt_base::ok:
-            case codecvt_base::partial:
-              __c += __extbe - __extbuf_;
-              break;
-            default:
-              return -1;
-            }
-          } while (__r == codecvt_base::partial);
-        }
-      }
-    }
-    if (__bufptr_->pubseekoff(-__c, ios_base::cur, __cm_) == pos_type(off_type(-1)))
-      return -1;
-    this->setg(0, 0, 0);
-    __cm_ = 0;
-  }
-  return 0;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_PUSH
-template <class _Codecvt, class _Elem, class _Tr>
-bool wbuffer_convert<_Codecvt, _Elem, _Tr>::__read_mode() {
-  if (!(__cm_ & ios_base::in)) {
-    this->setp(0, 0);
-    if (__always_noconv_)
-      this->setg((char_type*)__extbuf_, (char_type*)__extbuf_ + __ebs_, (char_type*)__extbuf_ + __ebs_);
-    else
-      this->setg(__intbuf_, __intbuf_ + __ibs_, __intbuf_ + __ibs_);
-    __cm_ = ios_base::in;
-    return true;
-  }
-  return false;
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-void wbuffer_convert<_Codecvt, _Elem, _Tr>::__write_mode() {
-  if (!(__cm_ & ios_base::out)) {
-    this->setg(0, 0, 0);
-    if (__ebs_ > sizeof(__extbuf_min_)) {
-      if (__always_noconv_)
-        this->setp((char_type*)__extbuf_, (char_type*)__extbuf_ + (__ebs_ - 1));
-      else
-        this->setp(__intbuf_, __intbuf_ + (__ibs_ - 1));
-    } else
-      this->setp(0, 0);
-    __cm_ = ios_base::out;
-  }
-}
-
-template <class _Codecvt, class _Elem, class _Tr>
-wbuffer_convert<_Codecvt, _Elem, _Tr>* wbuffer_convert<_Codecvt, _Elem, _Tr>::__close() {
-  wbuffer_convert* __rt = nullptr;
-  if (__cv_ != nullptr && __bufptr_ != nullptr) {
-    __rt = this;
-    if ((__cm_ & ios_base::out) && sync())
-      __rt = nullptr;
-  }
-  return __rt;
-}
-
-_LIBCPP_SUPPRESS_DEPRECATED_POP
-
-#    endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT)
-
-_LIBCPP_END_NAMESPACE_STD
-
-_LIBCPP_POP_MACROS
-
-// NOLINTEND(libcpp-robust-against-adl)
-
 #  endif // _LIBCPP_HAS_LOCALIZATION
 
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
diff --git a/libcxx/include/map b/libcxx/include/map
index 6c271392911d..2070c91cfc46 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -972,31 +972,15 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
 
-  _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) {
-#  ifndef _LIBCPP_CXX03_LANG
-    __tree_ = __m.__tree_;
-#  else
-    if (this != std::addressof(__m)) {
-      __tree_.clear();
-      __tree_.value_comp() = __m.__tree_.value_comp();
-      __tree_.__copy_assign_alloc(__m.__tree_);
-      insert(__m.begin(), __m.end());
-    }
-#  endif
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__m.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__m.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
@@ -1659,31 +1643,16 @@ public:
     insert(__m.begin(), __m.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) {
-#  ifndef _LIBCPP_CXX03_LANG
-    __tree_ = __m.__tree_;
-#  else
-    if (this != std::addressof(__m)) {
-      __tree_.clear();
-      __tree_.value_comp() = __m.__tree_.value_comp();
-      __tree_.__copy_assign_alloc(__m.__tree_);
-      insert(__m.begin(), __m.end());
-    }
-#  endif
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__m.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__m.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multimap&
+  operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index adf80f2ac9ac..45b9c72a05b8 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -65,9 +65,8 @@ module std_core [system] {
 
   module type_traits {
     module add_cv_quals                               { header "__type_traits/add_cv_quals.h" }
-    module add_lvalue_reference                       { header "__type_traits/add_lvalue_reference.h" }
     module add_pointer                                { header "__type_traits/add_pointer.h" }
-    module add_rvalue_reference                       { header "__type_traits/add_rvalue_reference.h" }
+    module add_reference                              { header "__type_traits/add_reference.h" }
     module aligned_storage                            { header "__type_traits/aligned_storage.h" }
     module aligned_union                              { header "__type_traits/aligned_union.h" }
     module alignment_of                               { header "__type_traits/alignment_of.h" }
@@ -869,6 +868,7 @@ module std [system] {
 
     header "array"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module atomic {
@@ -934,7 +934,10 @@ module std [system] {
     module chars_format               { header "__charconv/chars_format.h" }
     module from_chars_floating_point  { header "__charconv/from_chars_floating_point.h" }
     module from_chars_integral        { header "__charconv/from_chars_integral.h" }
-    module from_chars_result          { header "__charconv/from_chars_result.h" }
+    module from_chars_result          {
+      header "__charconv/from_chars_result.h"
+      export std.system_error.errc
+    }
     module tables                     { header "__charconv/tables.h" }
     module to_chars                   { header "__charconv/to_chars.h" }
     module to_chars_base_10           { header "__charconv/to_chars_base_10.h" }
@@ -1223,6 +1226,7 @@ module std [system] {
 
     header "deque"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module exception {
@@ -1299,6 +1303,7 @@ module std [system] {
     header "flat_map"
     export *
     export std.algorithm.ranges_sort
+    export std.iterator.reverse_iterator
     export std.ranges.zip_view
     export std.tuple
   }
@@ -1317,6 +1322,10 @@ module std [system] {
     export std.flat_map.sorted_unique
     export std.flat_map.sorted_equivalent
     export *
+    export std.algorithm.ranges_sort
+    export std.iterator.reverse_iterator
+    export std.ranges.zip_view
+    export std.tuple
   }
 
   module format {
@@ -1372,6 +1381,7 @@ module std [system] {
   module forward_list {
     header "forward_list"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module fstream {
@@ -1522,6 +1532,7 @@ module std [system] {
     }
     module permutable                 { header "__iterator/permutable.h" }
     module prev                       { header "__iterator/prev.h" }
+    module product_iterator           { header "__iterator/product_iterator.h" }
     module projected                  { header "__iterator/projected.h" }
     module ranges_iterator_traits     { header "__iterator/ranges_iterator_traits.h" }
     module readable_traits            { header "__iterator/readable_traits.h" }
@@ -1546,11 +1557,21 @@ module std [system] {
   module list {
     header "list"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module locale {
     header "locale"
-    module pad_and_output { header "__locale_dir/pad_and_output.h" }
+    module check_grouping  { header "__locale_dir/check_grouping.h" }
+    module get_c_locale    { header "__locale_dir/get_c_locale.h" }
+    module messages        { header "__locale_dir/messages.h" }
+    module money           { header "__locale_dir/money.h" }
+    module num             { header "__locale_dir/num.h" }
+    module pad_and_output  { header "__locale_dir/pad_and_output.h" }
+    module scan_keyword    { header "__locale_dir/scan_keyword.h" }
+    module time            { header "__locale_dir/time.h" }
+    module wbuffer_convert { header "__locale_dir/wbuffer_convert.h" }
+    module wstring_convert { header "__locale_dir/wstring_convert.h" }
 
     module support {
       header "__locale_dir/locale_base_api.h"
@@ -1588,6 +1609,7 @@ module std [system] {
     module fwd { header "__fwd/map.h" }
     header "map"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module mdspan {
@@ -1934,6 +1956,7 @@ module std [system] {
   module regex {
     header "regex"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module scoped_allocator {
@@ -1950,6 +1973,7 @@ module std [system] {
     module fwd { header "__fwd/set.h" }
     header "set"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module shared_mutex {
@@ -1967,6 +1991,7 @@ module std [system] {
 
     header "span"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module sstream {
@@ -2019,6 +2044,7 @@ module std [system] {
 
     header "string"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module string_view {
@@ -2026,6 +2052,7 @@ module std [system] {
 
     header "string_view"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module strstream {
@@ -2107,11 +2134,13 @@ module std [system] {
   module unordered_map {
     header "unordered_map"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module unordered_set {
     header "unordered_set"
     export *
+    export std.iterator.reverse_iterator
   }
 
   module utility {
@@ -2200,6 +2229,7 @@ module std [system] {
     }
 
     header "vector"
+    export std.iterator.reverse_iterator
     export *
   }
 
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index f616bad3ac17..dc8e711f0487 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -434,7 +434,7 @@ public:
 };
 
 template <class _Mutex>
-class _LIBCPP_THREAD_SAFETY_ANNOTATION(scoped_lockable) scoped_lock<_Mutex> {
+class _LIBCPP_SCOPED_LOCKABLE scoped_lock<_Mutex> {
 public:
   typedef _Mutex mutex_type;
 
@@ -442,16 +442,15 @@ private:
   mutex_type& __m_;
 
 public:
-  [[nodiscard]]
-  _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(mutex_type& __m) _LIBCPP_THREAD_SAFETY_ANNOTATION(acquire_capability(__m))
+  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(mutex_type& __m) _LIBCPP_ACQUIRE_CAPABILITY(__m)
       : __m_(__m) {
     __m_.lock();
   }
 
-  ~scoped_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(release_capability()) { __m_.unlock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI ~scoped_lock() { __m_.unlock(); }
 
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t, mutex_type& __m)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(requires_capability(__m))
+  [[nodiscard]]
+  _LIBCPP_HIDE_FROM_ABI explicit scoped_lock(adopt_lock_t, mutex_type& __m) _LIBCPP_REQUIRES_CAPABILITY(__m)
       : __m_(__m) {}
 
   scoped_lock(scoped_lock const&)            = delete;
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index 4a627a07a4a2..8ec17b68503f 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -205,6 +205,11 @@ void vprint_nonunicode(ostream& os, string_view fmt, format_args args);
 #    include <stdexcept>
 #    include <type_traits>
 #  endif
+
+#  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 23
+#    include <locale>
+#  endif
+
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
 
 #endif // _LIBCPP_OSTREAM
diff --git a/libcxx/include/set b/libcxx/include/set
index aeea98adf582..480aa0ca0b0c 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -662,14 +662,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
 
-  _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) {
-    __tree_ = __s.__tree_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__s.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 #  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
@@ -1129,14 +1125,10 @@ public:
     insert(__s.begin(), __s.end());
   }
 
-  _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) {
-    __tree_ = __s.__tree_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value)
-      : __tree_(std::move(__s.__tree_)) {}
+  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
 
   _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
 #  endif // _LIBCPP_CXX03_LANG
diff --git a/libcxx/include/shared_mutex b/libcxx/include/shared_mutex
index 6469c02ca587..8c02e348e4de 100644
--- a/libcxx/include/shared_mutex
+++ b/libcxx/include/shared_mutex
@@ -183,7 +183,7 @@ struct _LIBCPP_EXPORTED_FROM_ABI __shared_mutex_base {
 };
 
 #      if _LIBCPP_STD_VER >= 17
-class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_mutex")) shared_mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("shared_mutex") shared_mutex {
   __shared_mutex_base __base_;
 
 public:
@@ -194,35 +194,23 @@ public:
   shared_mutex& operator=(const shared_mutex&) = delete;
 
   // Exclusive ownership
-  _LIBCPP_HIDE_FROM_ABI void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__()) {
-    return __base_.lock();
-  }
-  _LIBCPP_HIDE_FROM_ABI bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
-    return __base_.try_lock();
-  }
-  _LIBCPP_HIDE_FROM_ABI void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__()) {
-    return __base_.unlock();
-  }
+  _LIBCPP_ACQUIRE_CAPABILITY() _LIBCPP_HIDE_FROM_ABI void lock() { return __base_.lock(); }
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool try_lock() { return __base_.try_lock(); }
+  _LIBCPP_RELEASE_CAPABILITY _LIBCPP_HIDE_FROM_ABI void unlock() { return __base_.unlock(); }
 
   // Shared ownership
-  _LIBCPP_HIDE_FROM_ABI void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__()) {
-    return __base_.lock_shared();
-  }
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared()
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_ACQUIRE_SHARED_CAPABILITY _LIBCPP_HIDE_FROM_ABI void lock_shared() { return __base_.lock_shared(); }
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool try_lock_shared() {
     return __base_.try_lock_shared();
   }
-  _LIBCPP_HIDE_FROM_ABI void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__()) {
-    return __base_.unlock_shared();
-  }
+  _LIBCPP_RELEASE_SHARED_CAPABILITY _LIBCPP_HIDE_FROM_ABI void unlock_shared() { return __base_.unlock_shared(); }
 
   //     typedef __shared_mutex_base::native_handle_type native_handle_type;
   //     _LIBCPP_HIDE_FROM_ABI native_handle_type native_handle() { return __base::unlock_shared(); }
 };
 #      endif
 
-class _LIBCPP_EXPORTED_FROM_ABI
-_LIBCPP_THREAD_SAFETY_ANNOTATION(__capability__("shared_timed_mutex")) shared_timed_mutex {
+class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_CAPABILITY("shared_timed_mutex") shared_timed_mutex {
   __shared_mutex_base __base_;
 
 public:
@@ -233,17 +221,17 @@ public:
   shared_timed_mutex& operator=(const shared_timed_mutex&) = delete;
 
   // Exclusive ownership
-  void lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_capability__());
-  bool try_lock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true));
+  void lock() _LIBCPP_ACQUIRE_CAPABILITY();
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) bool try_lock();
   template <class _Rep, class _Period>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_for(const chrono::duration<_Rep, _Period>& __rel_time) {
     return try_lock_until(chrono::steady_clock::now() + __rel_time);
   }
 
   template <class _Clock, class _Duration>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
     unique_lock<mutex> __lk(__base_.__mut_);
     if (__base_.__state_ & __base_.__write_entered_) {
       while (true) {
@@ -270,20 +258,20 @@ public:
     return true;
   }
 
-  void unlock() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_capability__());
+  _LIBCPP_RELEASE_CAPABILITY void unlock();
 
   // Shared ownership
-  void lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__acquire_shared_capability__());
-  bool try_lock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true));
+  _LIBCPP_ACQUIRE_SHARED_CAPABILITY void lock_shared();
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) bool try_lock_shared();
   template <class _Rep, class _Period>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared_for(const chrono::duration<_Rep, _Period>& __rel_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_shared_for(const chrono::duration<_Rep, _Period>& __rel_time) {
     return try_lock_shared_until(chrono::steady_clock::now() + __rel_time);
   }
 
   template <class _Clock, class _Duration>
-  _LIBCPP_HIDE_FROM_ABI bool try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time)
-      _LIBCPP_THREAD_SAFETY_ANNOTATION(__try_acquire_shared_capability__(true)) {
+  _LIBCPP_TRY_ACQUIRE_SHARED_CAPABILITY(true) _LIBCPP_HIDE_FROM_ABI bool
+  try_lock_shared_until(const chrono::time_point<_Clock, _Duration>& __abs_time) {
     unique_lock<mutex> __lk(__base_.__mut_);
     if ((__base_.__state_ & __base_.__write_entered_) ||
         (__base_.__state_ & __base_.__n_readers_) == __base_.__n_readers_) {
@@ -302,7 +290,7 @@ public:
     return true;
   }
 
-  void unlock_shared() _LIBCPP_THREAD_SAFETY_ANNOTATION(__release_shared_capability__());
+  _LIBCPP_RELEASE_SHARED_CAPABILITY void unlock_shared();
 };
 
 template <class _Mutex>
diff --git a/libcxx/include/sstream b/libcxx/include/sstream
index 8462f7585340..682a28fd4dbb 100644
--- a/libcxx/include/sstream
+++ b/libcxx/include/sstream
@@ -325,7 +325,7 @@ typedef basic_stringstream<wchar_t> wstringstream;
 #    include <__utility/swap.h>
 #    include <ios>
 #    include <istream>
-#    include <locale>
+#    include <streambuf>
 #    include <string>
 #    include <string_view>
 #    include <version>
diff --git a/libcxx/include/string b/libcxx/include/string
index c450ddcb4191..514dd91c7c17 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -1126,7 +1126,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
   basic_string(const _Tp& __t, size_type __pos, size_type __n, const allocator_type& __a = allocator_type())
@@ -1138,7 +1138,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1147,7 +1147,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const _Tp& __t, const allocator_type& __a)
       : __alloc_(__a) {
@@ -1206,7 +1206,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1343,7 +1343,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string >::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string >::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator+=(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1372,7 +1372,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& append(const _Tp& __t) {
     __self_view __sv = __t;
@@ -1383,7 +1383,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   append(const _Tp& __t, size_type __pos, size_type __n = npos) {
@@ -1510,7 +1510,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   assign(const _Tp& __t, size_type __pos, size_type __n = npos) {
@@ -1580,7 +1580,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   insert(size_type __pos1, const _Tp& __t, size_type __pos2, size_type __n = npos) {
@@ -1661,7 +1661,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
   replace(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) {
@@ -2006,7 +2006,7 @@ public:
 
   template <class _Tp,
             __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
-                              !__is_same_uncvref<_Tp, basic_string>::value,
+                              !is_same<__remove_cvref_t<_Tp>, basic_string>::value,
                           int> = 0>
   inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 int
   compare(size_type __pos1, size_type __n1, const _Tp& __t, size_type __pos2, size_type __n2 = npos) const {
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index 6e7a430d219e..75021f0ea51f 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -216,6 +216,7 @@ template <class... Types>
 #  include <__compare/common_comparison_category.h>
 #  include <__compare/ordering.h>
 #  include <__compare/synth_three_way.h>
+#  include <__concepts/boolean_testable.h>
 #  include <__config>
 #  include <__cstddef/size_t.h>
 #  include <__fwd/array.h>
@@ -1153,6 +1154,11 @@ struct __tuple_equal<0> {
 };
 
 template <class... _Tp, class... _Up>
+#    if _LIBCPP_STD_VER >= 26
+  requires(__all<requires(const _Tp& __t, const _Up& __u) {
+             { __t == __u } -> __boolean_testable;
+           }...>::value && (sizeof...(_Tp) == sizeof...(_Up)))
+#    endif
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 bool
 operator==(const tuple<_Tp...>& __x, const tuple<_Up...>& __y) {
   static_assert(sizeof...(_Tp) == sizeof...(_Up), "Can't compare tuples of different sizes");
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 9db7b2afb0cf..a6e0c1867566 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -463,9 +463,8 @@ namespace std
 #else
 #  include <__config>
 #  include <__type_traits/add_cv_quals.h>
-#  include <__type_traits/add_lvalue_reference.h>
 #  include <__type_traits/add_pointer.h>
-#  include <__type_traits/add_rvalue_reference.h>
+#  include <__type_traits/add_reference.h>
 #  include <__type_traits/aligned_storage.h>
 #  include <__type_traits/aligned_union.h>
 #  include <__type_traits/alignment_of.h>
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index b7f333e5f117..484f22ce5d72 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -654,9 +654,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
       : _Hash(__h) {}
   _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return *this; }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const {
-    return static_cast<const _Hash&>(*this)(__x.__get_value().first);
-  }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return static_cast<const _Hash&>(*this)(__x.first); }
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return static_cast<const _Hash&>(*this)(__x); }
 #  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
@@ -680,7 +678,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI __unordered_map_hasher(const _Hash& __h) _NOEXCEPT_(is_nothrow_copy_constructible<_Hash>::value)
       : __hash_(__h) {}
   _LIBCPP_HIDE_FROM_ABI const _Hash& hash_function() const _NOEXCEPT { return __hash_; }
-  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return __hash_(__x.__get_value().first); }
+  _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Cp& __x) const { return __hash_(__x.first); }
   _LIBCPP_HIDE_FROM_ABI size_t operator()(const _Key& __x) const { return __hash_(__x); }
 #  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
@@ -713,10 +711,10 @@ public:
       : _Pred(__p) {}
   _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return *this; }
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y.__get_value().first);
+    return static_cast<const _Pred&>(*this)(__x.first, __y.first);
   }
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y);
+    return static_cast<const _Pred&>(*this)(__x.first, __y);
   }
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
     return static_cast<const _Pred&>(*this)(__x, __y.__get_value().first);
@@ -724,7 +722,7 @@ public:
 #  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
-    return static_cast<const _Pred&>(*this)(__x.__get_value().first, __y);
+    return static_cast<const _Pred&>(*this)(__x.first, __y);
   }
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
@@ -755,23 +753,17 @@ public:
   _LIBCPP_HIDE_FROM_ABI __unordered_map_equal(const _Pred& __p) _NOEXCEPT_(is_nothrow_copy_constructible<_Pred>::value)
       : __pred_(__p) {}
   _LIBCPP_HIDE_FROM_ABI const _Pred& key_eq() const _NOEXCEPT { return __pred_; }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const {
-    return __pred_(__x.__get_value().first, __y.__get_value().first);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const {
-    return __pred_(__x.__get_value().first, __y);
-  }
-  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const {
-    return __pred_(__x, __y.__get_value().first);
-  }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Cp& __y) const { return __pred_(__x.first, __y.first); }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _Key& __y) const { return __pred_(__x.first, __y); }
+  _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _Cp& __y) const { return __pred_(__x, __y.first); }
 #  if _LIBCPP_STD_VER >= 20
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Cp& __x, const _K2& __y) const {
-    return __pred_(__x.__get_value().first, __y);
+    return __pred_(__x.first, __y);
   }
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _Cp& __y) const {
-    return __pred_(__x, __y.__get_value().first);
+    return __pred_(__x, __y.first);
   }
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _Key& __x, const _K2& __y) const {
@@ -833,96 +825,16 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
     if (__second_constructed)
-      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().__get_value().second));
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().second));
     if (__first_constructed)
-      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().__get_value().first));
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().first));
     if (__p)
       __alloc_traits::deallocate(__na_, __p, 1);
   }
 };
 
-#  ifndef _LIBCPP_CXX03_LANG
 template <class _Key, class _Tp>
-struct _LIBCPP_STANDALONE_DEBUG __hash_value_type {
-  typedef _Key key_type;
-  typedef _Tp mapped_type;
-  typedef pair<const key_type, mapped_type> value_type;
-  typedef pair<key_type&, mapped_type&> __nc_ref_pair_type;
-  typedef pair<key_type&&, mapped_type&&> __nc_rref_pair_type;
-
-private:
-  value_type __cc_;
-
-public:
-  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() {
-#    if _LIBCPP_STD_VER >= 17
-    return *std::launder(std::addressof(__cc_));
-#    else
-    return __cc_;
-#    endif
-  }
-
-  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const {
-#    if _LIBCPP_STD_VER >= 17
-    return *std::launder(std::addressof(__cc_));
-#    else
-    return __cc_;
-#    endif
-  }
-
-  _LIBCPP_HIDE_FROM_ABI __nc_ref_pair_type __ref() {
-    value_type& __v = __get_value();
-    return __nc_ref_pair_type(const_cast<key_type&>(__v.first), __v.second);
-  }
-
-  _LIBCPP_HIDE_FROM_ABI __nc_rref_pair_type __move() {
-    value_type& __v = __get_value();
-    return __nc_rref_pair_type(std::move(const_cast<key_type&>(__v.first)), std::move(__v.second));
-  }
-
-  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(const __hash_value_type& __v) {
-    __ref() = __v.__get_value();
-    return *this;
-  }
-
-  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(__hash_value_type&& __v) {
-    __ref() = __v.__move();
-    return *this;
-  }
-
-  template <class _ValueTp, __enable_if_t<__is_same_uncvref<_ValueTp, value_type>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI __hash_value_type& operator=(_ValueTp&& __v) {
-    __ref() = std::forward<_ValueTp>(__v);
-    return *this;
-  }
-
-  __hash_value_type(const __hash_value_type& __v) = delete;
-  __hash_value_type(__hash_value_type&& __v)      = delete;
-  template <class... _Args>
-  explicit __hash_value_type(_Args&&... __args) = delete;
-
-  ~__hash_value_type() = delete;
-};
-
-#  else
-
-template <class _Key, class _Tp>
-struct __hash_value_type {
-  typedef _Key key_type;
-  typedef _Tp mapped_type;
-  typedef pair<const key_type, mapped_type> value_type;
-
-private:
-  value_type __cc_;
-
-public:
-  _LIBCPP_HIDE_FROM_ABI value_type& __get_value() { return __cc_; }
-  _LIBCPP_HIDE_FROM_ABI const value_type& __get_value() const { return __cc_; }
-
-  ~__hash_value_type() = delete;
-};
-
-#  endif
+struct __hash_value_type;
 
 template <class _HashIterator>
 class __hash_map_iterator {
@@ -941,8 +853,8 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI __hash_map_iterator(_HashIterator __i) _NOEXCEPT : __i_(__i) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return *__i_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(*__i_); }
 
   _LIBCPP_HIDE_FROM_ABI __hash_map_iterator& operator++() {
     ++__i_;
@@ -995,8 +907,8 @@ public:
   __hash_map_const_iterator(__hash_map_iterator<typename _HashIterator::__non_const_iterator> __i) _NOEXCEPT
       : __i_(__i.__i_) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __i_->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__i_->__get_value()); }
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return *__i_; }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(*__i_); }
 
   _LIBCPP_HIDE_FROM_ABI __hash_map_const_iterator& operator++() {
     ++__i_;
@@ -1053,8 +965,8 @@ public:
 
 private:
   typedef __hash_value_type<key_type, mapped_type> __value_type;
-  typedef __unordered_map_hasher<key_type, __value_type, hasher, key_equal> __hasher;
-  typedef __unordered_map_equal<key_type, __value_type, key_equal, hasher> __key_equal;
+  typedef __unordered_map_hasher<key_type, value_type, hasher, key_equal> __hasher;
+  typedef __unordered_map_equal<key_type, value_type, key_equal, hasher> __key_equal;
   typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
 
   typedef __hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
@@ -1073,9 +985,6 @@ private:
 
   static_assert(__check_valid_allocator<allocator_type>::value, "");
 
-  static_assert(is_same<typename __table::__container_value_type, value_type>::value, "");
-  static_assert(is_same<typename __table::__node_value_type, __value_type>::value, "");
-
 public:
   typedef typename __alloc_traits::pointer pointer;
   typedef typename __alloc_traits::const_pointer const_pointer;
@@ -1680,9 +1589,8 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __
     : __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
   if (__a != __u.get_allocator()) {
     iterator __i = __u.begin();
-    while (__u.size() != 0) {
-      __table_.__emplace_unique(__u.__table_.remove((__i++).__i_)->__get_value().__move());
-    }
+    while (__u.size() != 0)
+      __table_.__insert_unique_from_orphaned_node(std::move(__u.__table_.remove((__i++).__i_)->__get_value()));
   }
 }
 
@@ -1741,8 +1649,7 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k) {
   return __table_
       .__emplace_unique_key_args(__k, piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
-      .first->__get_value()
-      .second;
+      .first->second;
 }
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -1750,8 +1657,7 @@ _Tp& unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](key_type&& __k)
   return __table_
       .__emplace_unique_key_args(
           __k, piecewise_construct, std::forward_as_tuple(std::move(__k)), std::forward_as_tuple())
-      .first->__get_value()
-      .second;
+      .first->second;
 }
 #  else // _LIBCPP_CXX03_LANG
 
@@ -1760,9 +1666,9 @@ typename unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__node_holder
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::__construct_node_with_key(const key_type& __k) {
   __node_allocator& __na = __table_.__node_alloc();
   __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-  __node_traits::construct(__na, std::addressof(__h->__get_value().__get_value().first), __k);
+  __node_traits::construct(__na, std::addressof(__h->__get_value().first), __k);
   __h.get_deleter().__first_constructed = true;
-  __node_traits::construct(__na, std::addressof(__h->__get_value().__get_value().second));
+  __node_traits::construct(__na, std::addressof(__h->__get_value().second));
   __h.get_deleter().__second_constructed = true;
   return __h;
 }
@@ -1869,8 +1775,8 @@ public:
 
 private:
   typedef __hash_value_type<key_type, mapped_type> __value_type;
-  typedef __unordered_map_hasher<key_type, __value_type, hasher, key_equal> __hasher;
-  typedef __unordered_map_equal<key_type, __value_type, key_equal, hasher> __key_equal;
+  typedef __unordered_map_hasher<key_type, value_type, hasher, key_equal> __hasher;
+  typedef __unordered_map_equal<key_type, value_type, key_equal, hasher> __key_equal;
   typedef __rebind_alloc<allocator_traits<allocator_type>, __value_type> __allocator_type;
 
   typedef __hash_table<__value_type, __hasher, __key_equal, __allocator_type> __table;
@@ -2439,9 +2345,8 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
     : __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
   if (__a != __u.get_allocator()) {
     iterator __i = __u.begin();
-    while (__u.size() != 0) {
-      __table_.__insert_multi(__u.__table_.remove((__i++).__i_)->__get_value().__move());
-    }
+    while (__u.size() != 0)
+      __table_.__insert_multi_from_orphaned_node(std::move(__u.__table_.remove((__i++).__i_)->__get_value()));
   }
 }
 
diff --git a/libcxx/include/variant b/libcxx/include/variant
index dac6f786cc19..ede9f486ecc2 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -242,6 +242,7 @@ namespace std {
 #  include <__type_traits/is_assignable.h>
 #  include <__type_traits/is_constructible.h>
 #  include <__type_traits/is_convertible.h>
+#  include <__type_traits/is_core_convertible.h>
 #  include <__type_traits/is_destructible.h>
 #  include <__type_traits/is_nothrow_assignable.h>
 #  include <__type_traits/is_nothrow_constructible.h>
@@ -1442,6 +1443,11 @@ struct __convert_to_bool {
 };
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t == __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator==(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__lhs.index() != __rhs.index())
@@ -1474,6 +1480,11 @@ operator<=>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
 #    endif // _LIBCPP_STD_VER >= 20
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t != __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__lhs.index() != __rhs.index())
@@ -1484,6 +1495,11 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(const variant<_Types...>& __lhs,
 }
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t < __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__rhs.valueless_by_exception())
@@ -1498,6 +1514,11 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator<(const variant<_Types...>& __lhs,
 }
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t > __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__lhs.valueless_by_exception())
@@ -1512,6 +1533,11 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator>(const variant<_Types...>& __lhs,
 }
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t <= __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__lhs.valueless_by_exception())
@@ -1526,6 +1552,11 @@ _LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(const variant<_Types...>& __lhs,
 }
 
 template <class... _Types>
+#    if _LIBCPP_STD_VER >= 26
+  requires(requires(const _Types& __t) {
+    { __t >= __t } -> __core_convertible_to<bool>;
+  } && ...)
+#    endif
 _LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(const variant<_Types...>& __lhs, const variant<_Types...>& __rhs) {
   using __variant_detail::__visitation::__variant;
   if (__rhs.valueless_by_exception())
diff --git a/libcxx/include/version b/libcxx/include/version
index 91fe48351e16..d98049bd5704 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -59,6 +59,8 @@ __cpp_lib_char8_t                                       201907L <atomic> <filesy
 __cpp_lib_chrono                                        201611L <chrono>
 __cpp_lib_chrono_udls                                   201304L <chrono>
 __cpp_lib_clamp                                         201603L <algorithm>
+__cpp_lib_common_reference                              202302L <type_traits>
+__cpp_lib_common_reference_wrapper                      202302L <functional>
 __cpp_lib_complex_udls                                  201309L <complex>
 __cpp_lib_concepts                                      202002L <concepts>
 __cpp_lib_constexpr_algorithms                          202306L <algorithm> <utility>
@@ -402,6 +404,8 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # if _LIBCPP_HAS_CHAR8_T
 #   define __cpp_lib_char8_t                            201907L
 # endif
+# define __cpp_lib_common_reference                     202302L
+# define __cpp_lib_common_reference_wrapper             202302L
 # define __cpp_lib_concepts                             202002L
 # define __cpp_lib_constexpr_algorithms                 201806L
 # define __cpp_lib_constexpr_complex                    201711L
diff --git a/libcxx/modules/std/functional.inc b/libcxx/modules/std/functional.inc
index ddc7d023ee6d..9ef8f584611f 100644
--- a/libcxx/modules/std/functional.inc
+++ b/libcxx/modules/std/functional.inc
@@ -14,6 +14,10 @@ export namespace std {
   using std::invoke_r;
 #endif
 
+#if _LIBCPP_STD_VER >= 20
+  using std::basic_common_reference;
+#endif
+
   // [refwrap], reference_wrapper
   using std::reference_wrapper;
 
diff --git a/libcxx/src/atomic.cpp b/libcxx/src/atomic.cpp
index c1af8d6f95aa..903084da053a 100644
--- a/libcxx/src/atomic.cpp
+++ b/libcxx/src/atomic.cpp
@@ -151,7 +151,10 @@ __libcpp_contention_monitor_for_wait(__cxx_atomic_contention_t volatile* /*__con
 static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __contention_state,
                                      __cxx_atomic_contention_t const volatile* __platform_state,
                                      __cxx_contention_t __old_value) {
-  __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_seq_cst);
+  __cxx_atomic_fetch_add(__contention_state, __cxx_contention_t(1), memory_order_relaxed);
+  // https://github.com/llvm/llvm-project/issues/109290
+  // There are no platform guarantees of a memory barrier in the platform wait implementation
+  __cxx_atomic_thread_fence(memory_order_seq_cst);
   // We sleep as long as the monitored value hasn't changed.
   __libcpp_platform_wait_on_address(__platform_state, __old_value);
   __cxx_atomic_fetch_sub(__contention_state, __cxx_contention_t(1), memory_order_release);
@@ -163,7 +166,7 @@ static void __libcpp_contention_wait(__cxx_atomic_contention_t volatile* __conte
 static void __libcpp_atomic_notify(void const volatile* __location) {
   auto const __entry = __libcpp_contention_state(__location);
   // The value sequence laundering happens on the next line below.
-  __cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_release);
+  __cxx_atomic_fetch_add(&__entry->__platform_state, __cxx_contention_t(1), memory_order_seq_cst);
   __libcpp_contention_notify(
       &__entry->__contention_state,
       &__entry->__platform_state,
diff --git a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
index fb4455c4aa9d..0ff7f15164d8 100644
--- a/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
+++ b/libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h
@@ -13,12 +13,14 @@
 #include <iterator>
 #include <random>
 #include <string>
+#include <ranges>
 #include <type_traits>
 #include <utility>
 #include <vector>
 
 #include "benchmark/benchmark.h"
 #include "../../GenerateInput.h"
+#include "test_macros.h"
 
 namespace support {
 
@@ -57,7 +59,7 @@ void associative_container_benchmarks(std::string container) {
   auto get_key = [](Value const& v) { return adapt_operations<Container>::key_from_value(v); };
 
   auto bench = [&](std::string operation, auto f) {
-    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(32)->Arg(1024)->Arg(8192);
+    benchmark::RegisterBenchmark(container + "::" + operation, f)->Arg(0)->Arg(32)->Arg(1024)->Arg(8192);
   };
 
   static constexpr bool is_multi_key_container =
@@ -66,6 +68,8 @@ void associative_container_benchmarks(std::string container) {
 
   static constexpr bool is_ordered_container = requires(Container c, Key k) { c.lower_bound(k); };
 
+  static constexpr bool is_map_like = requires { typename Container::mapped_type; };
+
   // These benchmarks are structured to perform the operation being benchmarked
   // a small number of times at each iteration, in order to offset the cost of
   // PauseTiming() and ResumeTiming().
@@ -172,7 +176,7 @@ void associative_container_benchmarks(std::string container) {
   // Insertion
   /////////////////////////
   bench("insert(value) (already present)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1;
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value to_insert        = in[in.size() / 2]; // pick any existing value
     std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
@@ -321,11 +325,53 @@ void associative_container_benchmarks(std::string container) {
     }
   });
 
+  if constexpr (is_map_like) {
+    bench("insert(iterator, iterator) (product_iterator from same type)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Value> in  = make_value_types(generate_unique_keys(size + (size / 10)));
+      Container source(in.begin(), in.end());
+
+      Container c;
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.insert(source.begin(), source.end());
+        benchmark::DoNotOptimize(c);
+        benchmark::ClobberMemory();
+
+        st.PauseTiming();
+        c = Container();
+        st.ResumeTiming();
+      }
+    });
+
+#if TEST_STD_VER >= 23
+    bench("insert(iterator, iterator) (product_iterator from zip_view)", [=](auto& st) {
+      const std::size_t size = st.range(0);
+      std::vector<Key> keys  = generate_unique_keys(size + (size / 10));
+      std::sort(keys.begin(), keys.end());
+      std::vector<typename Container::mapped_type> mapped(keys.size());
+
+      auto source = std::views::zip(keys, mapped);
+
+      Container c;
+
+      for ([[maybe_unused]] auto _ : st) {
+        c.insert(source.begin(), source.end());
+        benchmark::DoNotOptimize(c);
+        benchmark::ClobberMemory();
+
+        st.PauseTiming();
+        c = Container();
+        st.ResumeTiming();
+      }
+    });
+#endif
+  }
   /////////////////////////
   // Erasure
   /////////////////////////
   bench("erase(key) (existent)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1; // avoid empty container
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value element          = in[in.size() / 2]; // pick any element
     std::vector<Container> c(BatchSize, Container(in.begin(), in.end()));
@@ -369,7 +415,7 @@ void associative_container_benchmarks(std::string container) {
   });
 
   bench("erase(iterator)", [=](auto& st) {
-    const std::size_t size = st.range(0);
+    const std::size_t size = st.range(0) ? st.range(0) : 1; // avoid empty container
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Value element          = in[in.size() / 2]; // pick any element
 
@@ -448,7 +494,7 @@ void associative_container_benchmarks(std::string container) {
       Container c(in.begin(), in.end());
 
       while (st.KeepRunningBatch(BatchSize)) {
-        for (std::size_t i = 0; i != BatchSize; ++i) {
+        for (std::size_t i = 0; i != keys.size(); ++i) { // possible empty keys when Arg(0)
           auto result = func(c, keys[i]);
           benchmark::DoNotOptimize(c);
           benchmark::DoNotOptimize(result);
diff --git a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
index 82902d50f31e..f3b86554802c 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_map.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: std-at-least-c++26
+// REQUIRES: std-at-least-c++23
 
 #include <flat_map>
 #include <utility>
diff --git a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
index f752f79b3b45..80eaa549042c 100644
--- a/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/flat_multimap.bench.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: std-at-least-c++26
+// REQUIRES: std-at-least-c++23
 
 #include <flat_map>
 
diff --git a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
index 56420bdaadfb..89443a597e85 100644
--- a/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
+++ b/libcxx/test/benchmarks/containers/associative/unordered_set.bench.cpp
@@ -8,6 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
+#include <string>
 #include <unordered_set>
 #include <utility>
 
@@ -27,6 +28,7 @@ struct support::adapt_operations<std::unordered_set<K>> {
 
 int main(int argc, char** argv) {
   support::associative_container_benchmarks<std::unordered_set<int>>("std::unordered_set<int>");
+  support::associative_container_benchmarks<std::unordered_set<std::string>>("std::unordered_set<std::string>");
 
   benchmark::Initialize(&argc, argv);
   benchmark::RunSpecifiedBenchmarks();
diff --git a/libcxx/test/configs/cmake-bridge.cfg.in b/libcxx/test/configs/cmake-bridge.cfg.in
index d7d588669032..20b7c1e9bc35 100644
--- a/libcxx/test/configs/cmake-bridge.cfg.in
+++ b/libcxx/test/configs/cmake-bridge.cfg.in
@@ -12,6 +12,8 @@
 #
 
 import os, site
+import shlex
+import sys
 site.addsitedir(os.path.join('@LIBCXX_SOURCE_DIR@', 'utils'))
 import libcxx.test.format
 
@@ -32,3 +34,4 @@ config.substitutions.append(('%{lib-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIB
 config.substitutions.append(('%{module-dir}', '@LIBCXX_TESTING_INSTALL_PREFIX@/@LIBCXX_INSTALL_MODULES_DIR@'))
 config.substitutions.append(('%{test-tools-dir}', '@LIBCXX_TEST_TOOLS_PATH@'))
 config.substitutions.append(('%{benchmark_flags}', '-I @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/include -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib -L @LIBCXX_BINARY_DIR@/test/benchmarks/google-benchmark/lib64 -l benchmark'))
+config.substitutions.append(("%{python}", shlex.quote(sys.executable)))
diff --git a/libcxx/test/extensions/gnu/hash/specializations.pass.cpp b/libcxx/test/extensions/gnu/hash/specializations.pass.cpp
new file mode 100644
index 000000000000..6661ed0da7da
--- /dev/null
+++ b/libcxx/test/extensions/gnu/hash/specializations.pass.cpp
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Prevent <ext/hash_set> from generating deprecated warnings for this test.
+// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
+
+#include <cassert>
+#include <ext/hash_map>
+#include <string>
+
+int main(int, char**) {
+  char str[] = "test";
+  assert(__gnu_cxx::hash<const char*>()("test") == std::hash<std::string>()("test"));
+  assert(__gnu_cxx::hash<char*>()(str) == std::hash<std::string>()("test"));
+  assert(__gnu_cxx::hash<char>()(42) == 42);
+  assert(__gnu_cxx::hash<signed char>()(42) == 42);
+  assert(__gnu_cxx::hash<unsigned char>()(42) == 42);
+  assert(__gnu_cxx::hash<short>()(42) == 42);
+  assert(__gnu_cxx::hash<unsigned short>()(42) == 42);
+  assert(__gnu_cxx::hash<int>()(42) == 42);
+  assert(__gnu_cxx::hash<unsigned int>()(42) == 42);
+  assert(__gnu_cxx::hash<long>()(42) == 42);
+  assert(__gnu_cxx::hash<unsigned long>()(42) == 42);
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/extensions/hash/specializations.compile.fail.cpp b/libcxx/test/extensions/gnu/hash/specializations.verify.cpp
similarity index 70%
rename from libcxx/test/libcxx/extensions/hash/specializations.compile.fail.cpp
rename to libcxx/test/extensions/gnu/hash/specializations.verify.cpp
index f81ec5dacb91..5a74a52dd395 100644
--- a/libcxx/test/libcxx/extensions/hash/specializations.compile.fail.cpp
+++ b/libcxx/test/extensions/gnu/hash/specializations.verify.cpp
@@ -6,13 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
+
 #include <assert.h>
 #include <ext/hash_map>
 #include <string>
 
-int main(int, char**)
-{
-    assert(__gnu_cxx::hash<std::string>()(std::string()) == 0);  // error
+int main(int, char**) {
+  assert(__gnu_cxx::hash<std::string>()(std::string()) == 0); // expected-error {{does not provide a call operator}}
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/extensions/hash_map/const_iterator.compile.fail.cpp b/libcxx/test/extensions/gnu/hash_map/const_iterator.verify.cpp
similarity index 55%
rename from libcxx/test/libcxx/extensions/hash_map/const_iterator.compile.fail.cpp
rename to libcxx/test/extensions/gnu/hash_map/const_iterator.verify.cpp
index db09e40801a1..eacdacd035b3 100644
--- a/libcxx/test/libcxx/extensions/hash_map/const_iterator.compile.fail.cpp
+++ b/libcxx/test/extensions/gnu/hash_map/const_iterator.verify.cpp
@@ -6,14 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
+
 #include <ext/hash_map>
 
-int main(int, char**)
-{
-    __gnu_cxx::hash_map<int, int> m;
-    m[1] = 1;
-    const __gnu_cxx::hash_map<int, int> &cm = m;
-    cm.find(1)->second = 2;  // error
+int main(int, char**) {
+  __gnu_cxx::hash_map<int, int> m;
+  m[1]                                    = 1;
+  const __gnu_cxx::hash_map<int, int>& cm = m;
+  cm.find(1)->second = 2; // expected-error {{cannot assign to return value because function 'operator->' returns a const value}}
 
   return 0;
 }
diff --git a/libcxx/test/extensions/gnu/lit.local.cfg b/libcxx/test/extensions/gnu/lit.local.cfg
new file mode 100644
index 000000000000..ffc4fc420f4c
--- /dev/null
+++ b/libcxx/test/extensions/gnu/lit.local.cfg
@@ -0,0 +1,5 @@
+
+# Only libc++ and libstdc++ have any gnu extensions
+if ("stdlib=libc++" not in config.available_features and
+    "stdlib=libstdc++" not in config.available_features):
+    config.unsupported = True
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
index e71f35129dda..22e0662cf2c6 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert.temporary.pass.cpp
@@ -21,7 +21,7 @@
 #include "../flat_helpers.h"
 #include "test_macros.h"
 
-bool test() {
+constexpr bool test() {
   using M = std::flat_set<TrackCopyMove>;
   {
     M m;
@@ -43,6 +43,9 @@ bool test() {
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
index 35b9737e8510..c9dcd71127e2 100644
--- a/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
+++ b/libcxx/test/libcxx/containers/container.adaptors/flat.set/insert_range.pass.cpp
@@ -24,23 +24,33 @@
 #include <vector>
 
 #include "../flat_helpers.h"
+#include "test_iterators.h"
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   NotQuiteSequenceContainer<int> v;
   std::flat_set s(v);
-  std::istringstream ints("0 1 1 0");
-  auto r = std::ranges::subrange(std::istream_iterator<int>(ints), std::istream_iterator<int>()) |
-           std::views::transform([](int i) { return i * i; });
+
+  int ar[]   = {0, 1, 1, 0};
+  using Iter = cpp20_input_iterator<const int*>;
+  using Sent = sentinel_wrapper<Iter>;
+  using R    = std::ranges::subrange<Iter, Sent>;
+  auto r     = R(Iter(ar), Sent(Iter(ar + 4)));
+
   static_assert(
       ![](auto& t) { return requires { t.insert_range(t.end(), r); }; }(v),
       "This test is to test the case where the underlying container does not provide insert_range");
   s.insert_range(r);
   assert(std::ranges::equal(s, std::vector<int>{0, 1}));
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/libcxx/extensions/hash/specializations.pass.cpp b/libcxx/test/libcxx/extensions/hash/specializations.pass.cpp
deleted file mode 100644
index 345a2721dc03..000000000000
--- a/libcxx/test/libcxx/extensions/hash/specializations.pass.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: clang-modules-build
-
-// Prevent <ext/hash_set> from generating deprecated warnings for this test.
-// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated
-#include <assert.h>
-#include <ext/hash_map>
-#include <string>
-
-#include "test_macros.h"
-
-int main(int, char**)
-{
-    char str[] = "test";
-    assert(__gnu_cxx::hash<const char *>()("test") ==
-           std::hash<std::string>()("test"));
-    assert(__gnu_cxx::hash<char *>()(str) == std::hash<std::string>()("test"));
-    assert(__gnu_cxx::hash<char>()(42) == 42);
-    assert(__gnu_cxx::hash<signed char>()(42) == 42);
-    assert(__gnu_cxx::hash<unsigned char>()(42) == 42);
-    assert(__gnu_cxx::hash<short>()(42) == 42);
-    assert(__gnu_cxx::hash<unsigned short>()(42) == 42);
-    assert(__gnu_cxx::hash<int>()(42) == 42);
-    assert(__gnu_cxx::hash<unsigned int>()(42) == 42);
-    assert(__gnu_cxx::hash<long>()(42) == 42);
-    assert(__gnu_cxx::hash<unsigned long>()(42) == 42);
-
-  return 0;
-}
diff --git a/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
index c04250987e8e..f17c1483c4a9 100644
--- a/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
+++ b/libcxx/test/libcxx/input.output/iostreams.base/ios.base/ios.base.cons/dtor.uninitialized.pass.cpp
@@ -13,7 +13,7 @@
 
 // The fix for issue 57964 requires an updated dylib due to explicit
 // instantiations. That means Apple backdeployment targets remain broken.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <ios>
 
diff --git a/libcxx/test/libcxx/iterators/product_iterator.pass.cpp b/libcxx/test/libcxx/iterators/product_iterator.pass.cpp
new file mode 100644
index 000000000000..604f3667c36e
--- /dev/null
+++ b/libcxx/test/libcxx/iterators/product_iterator.pass.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+// gcc 15 does not seem to recognize the __product_iterator_traits specializations
+// UNSUPPORTED: gcc
+
+#include <flat_map>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "test_macros.h"
+#include "test_iterators.h"
+
+constexpr bool test() {
+  {
+    // Test that the __get_iterator_element can handle a non-copyable iterator
+    int Date[] = {1, 2, 3, 4};
+    cpp20_input_iterator<int*> iter(Date);
+    sentinel_wrapper<cpp20_input_iterator<int*>> sent{cpp20_input_iterator<int*>(Date + 4)};
+    std::ranges::subrange r1(std::move(iter), std::move(sent));
+    auto v  = std::views::zip(std::move(r1), std::views::iota(0, 4));
+    auto it = v.begin();
+
+    using Iter = decltype(it);
+
+    static_assert(!std::is_copy_constructible_v<Iter>);
+
+    static_assert(std::__product_iterator_traits<Iter>::__size == 2);
+    std::same_as<cpp20_input_iterator<int*>&> decltype(auto) it1 =
+        std::__product_iterator_traits<Iter>::__get_iterator_element<0>(it);
+
+    assert(*it1 == 1);
+  }
+  if (!std::is_constant_evaluated()) {
+    // Test __make_product_iterator
+    using M = std::flat_map<int, int>;
+    M m{{1, 1}, {2, 2}, {3, 3}};
+    using Iter         = std::ranges::iterator_t<const M>;
+    const auto& keys   = m.keys();
+    const auto& values = m.values();
+
+    auto it_keys   = std::ranges::begin(keys);
+    auto it_values = std::ranges::begin(values);
+
+    auto it = std::__product_iterator_traits<Iter>::__make_product_iterator(it_keys, it_values);
+    assert(it->first == 1);
+    assert(it->second == 1);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp b/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
index 1ecf378de5b6..5f618e92375a 100644
--- a/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
+++ b/libcxx/test/libcxx/localization/locale.categories/__scan_keyword.pass.cpp
@@ -35,8 +35,11 @@
 //                const _Ctype& __ct, ios_base::iostate& __err,
 //                bool __case_sensitive = true);
 
-#include <locale>
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
+#include <__locale_dir/scan_keyword.h>
 #include <cassert>
+#include <locale>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
index 5d40fa71a415..f43ad5b8c188 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
@@ -11,9 +11,8 @@
 
 // <mutex>
 
-// This test does not define _LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS so it
-// should compile without any warnings or errors even though this pattern is not
-// understood by the thread safety annotations.
+// This test does not set -Wthread-safety so it should compile without any warnings or errors even though this pattern
+// is not understood by the thread safety annotations.
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
index 1c1f36459c83..9633982774e1 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
index ddfe3269fdfa..965d1444a4d5 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
index bab686e7b4ca..4ae22219b030 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
@@ -6,21 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
 std::mutex m;
 
-void f() {
-  m.lock();
-} // expected-error {{mutex 'm' is still held at the end of function}}
+void f() { m.lock(); } // expected-warning {{mutex 'm' is still held at the end of function}}
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
index 81ce3b25a839..e9794acc8941 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
-
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
 
 // <mutex>
 
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
+
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
 
 #include <mutex>
 
diff --git a/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp b/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
index fee940fff847..d87a70927819 100644
--- a/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.shared_mutex/thread_safety.verify.cpp
@@ -8,12 +8,11 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
 
 // <shared_mutex>
 //
@@ -51,13 +50,13 @@ void f() {
   {
     m.lock_shared();
     read(data); // ok
-    ++data; // expected-error {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
+    ++data;     // expected-warning {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
     m.unlock_shared();
   }
   {
     if (m.try_lock_shared()) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
diff --git a/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp b/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
index baa8bc614f14..9c16caa3631f 100644
--- a/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
+++ b/libcxx/test/libcxx/thread/thread.shared_timed_mutex/thread_safety.verify.cpp
@@ -8,12 +8,11 @@
 
 // UNSUPPORTED: c++03, c++11
 // UNSUPPORTED: no-threads
-// REQUIRES: thread-safety
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS
 
-// On Windows Clang bugs out when both __declspec and __attribute__ are present,
-// the processing goes awry preventing the definition of the types.
-// XFAIL: msvc
+// GCC doesn't have thread safety attributes
+// UNSUPPORTED: gcc
+
+// ADDITIONAL_COMPILE_FLAGS: -Wthread-safety
 
 // <shared_mutex>
 //
@@ -68,27 +67,27 @@ void f(std::chrono::time_point<std::chrono::steady_clock> tp, std::chrono::milli
   {
     m.lock_shared();
     read(data); // ok
-    ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+    ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
     m.unlock_shared();
   }
   {
     if (m.try_lock_shared()) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
   {
     if (m.try_lock_shared_for(d)) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
   {
     if (m.try_lock_shared_until(tp)) {
       read(data); // ok
-      ++data; // expected-error {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
+      ++data;     // expected-warning {{writing variable 'data' requires holding shared_timed_mutex 'm' exclusively}}
       m.unlock_shared();
     }
   }
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index ce8f0261f2b2..5f906338f4b7 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -82,7 +82,6 @@ ccomplex ios
 ccomplex iosfwd
 ccomplex istream
 ccomplex limits
-ccomplex locale
 ccomplex ratio
 ccomplex sstream
 ccomplex stdexcept
@@ -120,7 +119,6 @@ chrono ios
 chrono iosfwd
 chrono istream
 chrono limits
-chrono locale
 chrono optional
 chrono ratio
 chrono sstream
@@ -177,7 +175,6 @@ complex ios
 complex iosfwd
 complex istream
 complex limits
-complex locale
 complex ratio
 complex sstream
 complex stdexcept
@@ -226,7 +223,6 @@ ctgmath ios
 ctgmath iosfwd
 ctgmath istream
 ctgmath limits
-ctgmath locale
 ctgmath ratio
 ctgmath sstream
 ctgmath stdexcept
@@ -278,7 +274,6 @@ experimental/iterator ios
 experimental/iterator iosfwd
 experimental/iterator iterator
 experimental/iterator limits
-experimental/iterator locale
 experimental/iterator ratio
 experimental/iterator stdexcept
 experimental/iterator streambuf
@@ -324,10 +319,8 @@ filesystem iomanip
 filesystem ios
 filesystem iosfwd
 filesystem limits
-filesystem locale
 filesystem ratio
 filesystem stdexcept
-filesystem streambuf
 filesystem string
 filesystem string_view
 filesystem tuple
@@ -412,7 +405,6 @@ fstream ios
 fstream iosfwd
 fstream istream
 fstream limits
-fstream locale
 fstream ratio
 fstream stdexcept
 fstream streambuf
@@ -455,7 +447,6 @@ future ios
 future iosfwd
 future istream
 future limits
-future locale
 future ratio
 future sstream
 future stdexcept
@@ -483,10 +474,8 @@ iomanip initializer_list
 iomanip ios
 iomanip iosfwd
 iomanip limits
-iomanip locale
 iomanip ratio
 iomanip stdexcept
-iomanip streambuf
 iomanip string
 iomanip string_view
 iomanip tuple
@@ -538,7 +527,6 @@ iostream ios
 iostream iosfwd
 iostream istream
 iostream limits
-iostream locale
 iostream optional
 iostream ostream
 iostream print
@@ -568,7 +556,6 @@ istream initializer_list
 istream ios
 istream iosfwd
 istream limits
-istream locale
 istream ratio
 istream stdexcept
 istream streambuf
@@ -724,7 +711,6 @@ ostream initializer_list
 ostream ios
 ostream iosfwd
 ostream limits
-ostream locale
 ostream optional
 ostream print
 ostream ratio
@@ -905,7 +891,6 @@ sstream ios
 sstream iosfwd
 sstream istream
 sstream limits
-sstream locale
 sstream ratio
 sstream stdexcept
 sstream streambuf
@@ -1004,7 +989,6 @@ strstream ios
 strstream iosfwd
 strstream istream
 strstream limits
-strstream locale
 strstream ratio
 strstream stdexcept
 strstream streambuf
@@ -1034,7 +1018,6 @@ syncstream initializer_list
 syncstream ios
 syncstream iosfwd
 syncstream limits
-syncstream locale
 syncstream map
 syncstream optional
 syncstream ostream
@@ -1086,7 +1069,6 @@ thread ios
 thread iosfwd
 thread istream
 thread limits
-thread locale
 thread ratio
 thread sstream
 thread stdexcept
diff --git a/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp b/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
index 48460d1488fd..32f8d45fbbc9 100644
--- a/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
+++ b/libcxx/test/libcxx/utilities/function.objects/func.require/bullet_1_2_3.pass.cpp
@@ -254,8 +254,9 @@ struct ReferenceWrapper {
     static void fun(Type&) noexcept;
     static void fun(Type&&) = delete;
 
-    template <class Type2,
-              class = typename std::enable_if<!std::__is_same_uncvref<Type2, ReferenceWrapper>::value>::type>
+    template <
+        class Type2,
+        class = typename std::enable_if<!std::is_same<std::__remove_cvref_t<Type2>, ReferenceWrapper>::value>::type>
     constexpr ReferenceWrapper(Type2&& t) noexcept : ptr(&t) {}
 
     constexpr Type& get() const noexcept { return *ptr; }
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp b/libcxx/test/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/conditional-compile-flags.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/lit.local.cfg b/libcxx/test/selftest/additional_compile_flags/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/lit.local.cfg
rename to libcxx/test/selftest/additional_compile_flags/lit.local.cfg
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp b/libcxx/test/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/substitutes-in-compile-flags.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp b/libcxx/test/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
rename to libcxx/test/selftest/additional_compile_flags/substitutes-in-run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp b/libcxx/test/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
rename to libcxx/test/selftest/compile.fail.cpp/compile-error.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp b/libcxx/test/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
rename to libcxx/test/selftest/compile.fail.cpp/compile-success.compile.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/compile-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/compile-success.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/link-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/link-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/link-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp b/libcxx/test/selftest/compile.pass.cpp/run-error.compile.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.cpp/run-error.compile.pass.cpp
rename to libcxx/test/selftest/compile.pass.cpp/run-error.compile.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/compile-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/compile-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/compile-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/compile-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/compile-success.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/compile-success.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/compile-success.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/compile-success.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/link-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/link-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/link-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/link-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/compile.pass.mm/run-error.compile.pass.mm b/libcxx/test/selftest/compile.pass.mm/run-error.compile.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/compile.pass.mm/run-error.compile.pass.mm
rename to libcxx/test/selftest/compile.pass.mm/run-error.compile.pass.mm
diff --git a/libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp b/libcxx/test/selftest/convenience_substitutions/build_run.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/convenience_substitutions/build_run.sh.cpp
rename to libcxx/test/selftest/convenience_substitutions/build_run.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/convenience_substitutions/verify.sh.cpp b/libcxx/test/selftest/convenience_substitutions/verify.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/convenience_substitutions/verify.sh.cpp
rename to libcxx/test/selftest/convenience_substitutions/verify.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py b/libcxx/test/selftest/dsl/dsl.sh.py
similarity index 99%
rename from libcxx/test/libcxx/selftest/dsl/dsl.sh.py
rename to libcxx/test/selftest/dsl/dsl.sh.py
index 012759436d89..6d4406b7858e 100644
--- a/libcxx/test/libcxx/selftest/dsl/dsl.sh.py
+++ b/libcxx/test/selftest/dsl/dsl.sh.py
@@ -26,7 +26,7 @@
 
 # Allow importing 'lit' and the 'libcxx' module. Make sure we put the lit
 # path first so we don't find any system-installed version.
-monorepoRoot = dirname(dirname(dirname(dirname(dirname(dirname(__file__))))))
+monorepoRoot = dirname(dirname(dirname(dirname(dirname(__file__)))))
 sys.path = [
     os.path.join(monorepoRoot, "libcxx", "utils"),
     os.path.join(monorepoRoot, "llvm", "utils", "lit"),
diff --git a/libcxx/test/libcxx/selftest/dsl/lit.local.cfg b/libcxx/test/selftest/dsl/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/selftest/dsl/lit.local.cfg
rename to libcxx/test/selftest/dsl/lit.local.cfg
diff --git a/libcxx/test/selftest/file_dependencies/a.txt b/libcxx/test/selftest/file_dependencies/a.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp b/libcxx/test/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
rename to libcxx/test/selftest/file_dependencies/absolute-and-relative-paths.sh.cpp
diff --git a/libcxx/test/selftest/file_dependencies/dir/b.txt b/libcxx/test/selftest/file_dependencies/dir/b.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp b/libcxx/test/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
rename to libcxx/test/selftest/file_dependencies/substitute-in-dependencies.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp b/libcxx/test/selftest/gen.cpp/empty.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp
rename to libcxx/test/selftest/gen.cpp/empty.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp b/libcxx/test/selftest/gen.cpp/one.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp
rename to libcxx/test/selftest/gen.cpp/one.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp b/libcxx/test/selftest/gen.cpp/two.gen.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp
rename to libcxx/test/selftest/gen.cpp/two.gen.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/compile-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/compile-error.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/compile-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/link-error.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/link-error.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/link-error.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp b/libcxx/test/selftest/link.fail.cpp/link-success.link.fail.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.fail.cpp/link-success.link.fail.cpp
rename to libcxx/test/selftest/link.fail.cpp/link-success.link.fail.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/compile-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/compile-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/compile-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/link-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/link-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/link-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/link-success.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/link-success.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/link-success.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp b/libcxx/test/selftest/link.pass.cpp/run-error.link.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.cpp/run-error.link.pass.cpp
rename to libcxx/test/selftest/link.pass.cpp/run-error.link.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/compile-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/compile-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/compile-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/compile-error.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/link-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/link-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/link-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/link-error.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/link-success.link.pass.mm b/libcxx/test/selftest/link.pass.mm/link-success.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/link-success.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/link-success.link.pass.mm
diff --git a/libcxx/test/libcxx/selftest/link.pass.mm/run-error.link.pass.mm b/libcxx/test/selftest/link.pass.mm/run-error.link.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/link.pass.mm/run-error.link.pass.mm
rename to libcxx/test/selftest/link.pass.mm/run-error.link.pass.mm
diff --git a/libcxx/test/libcxx/lit.local.cfg b/libcxx/test/selftest/lit.local.cfg
similarity index 100%
rename from libcxx/test/libcxx/lit.local.cfg
rename to libcxx/test/selftest/lit.local.cfg
diff --git a/libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp b/libcxx/test/selftest/modules/no-modules.sh.cpp
similarity index 73%
rename from libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp
rename to libcxx/test/selftest/modules/no-modules.sh.cpp
index 74e65a9072f7..762b38ccaaeb 100644
--- a/libcxx/test/libcxx/selftest/modules/no-modules.sh.cpp
+++ b/libcxx/test/selftest/modules/no-modules.sh.cpp
@@ -6,6 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // Make sure that the compile flags contain no module information.
 
 // MODULE_DEPENDENCIES:
diff --git a/libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp b/libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp
rename to libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
index d56ebb2961d4..455674d6b994 100644
--- a/libcxx/test/libcxx/selftest/modules/std-and-std.compat-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std-and-std.compat-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/modules/std-module.sh.cpp b/libcxx/test/selftest/modules/std-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std-module.sh.cpp
rename to libcxx/test/selftest/modules/std-module.sh.cpp
index ec43994fa1ef..e96c1e64b727 100644
--- a/libcxx/test/libcxx/selftest/modules/std-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp b/libcxx/test/selftest/modules/std.compat-module.sh.cpp
similarity index 82%
rename from libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp
rename to libcxx/test/selftest/modules/std.compat-module.sh.cpp
index e84709853fbc..4ec0ad8323d8 100644
--- a/libcxx/test/libcxx/selftest/modules/std.compat-module.sh.cpp
+++ b/libcxx/test/selftest/modules/std.compat-module.sh.cpp
@@ -10,6 +10,10 @@
 // UNSUPPORTED: clang-modules-build
 // UNSUPPORTED: gcc
 
+// TODO: This test is currently written in a way that is specific to libc++, but it's really trying to test a property
+//       of the test framework, which isn't libc++ specific.
+// REQUIRES: stdlib=libc++
+
 // XFAIL: has-no-cxx-module-support
 
 // Make sure that the compile flags contain the expected elements.
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp b/libcxx/test/selftest/pass.cpp/compile-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/compile-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/compile-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp b/libcxx/test/selftest/pass.cpp/link-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/link-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/link-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp b/libcxx/test/selftest/pass.cpp/run-error.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/run-error.pass.cpp
rename to libcxx/test/selftest/pass.cpp/run-error.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp b/libcxx/test/selftest/pass.cpp/run-success.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/run-success.pass.cpp
rename to libcxx/test/selftest/pass.cpp/run-success.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp b/libcxx/test/selftest/pass.cpp/werror.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.cpp/werror.pass.cpp
rename to libcxx/test/selftest/pass.cpp/werror.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm b/libcxx/test/selftest/pass.mm/compile-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/compile-error.pass.mm
rename to libcxx/test/selftest/pass.mm/compile-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm b/libcxx/test/selftest/pass.mm/link-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/link-error.pass.mm
rename to libcxx/test/selftest/pass.mm/link-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm b/libcxx/test/selftest/pass.mm/no-arc.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/no-arc.pass.mm
rename to libcxx/test/selftest/pass.mm/no-arc.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm b/libcxx/test/selftest/pass.mm/run-error.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/run-error.pass.mm
rename to libcxx/test/selftest/pass.mm/run-error.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm b/libcxx/test/selftest/pass.mm/run-success.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/run-success.pass.mm
rename to libcxx/test/selftest/pass.mm/run-success.pass.mm
diff --git a/libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm b/libcxx/test/selftest/pass.mm/use-objective-cxx.pass.mm
similarity index 100%
rename from libcxx/test/libcxx/selftest/pass.mm/use-objective-cxx.pass.mm
rename to libcxx/test/selftest/pass.mm/use-objective-cxx.pass.mm
diff --git a/libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp b/libcxx/test/selftest/remote-substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/remote-substitutions.sh.cpp
rename to libcxx/test/selftest/remote-substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp b/libcxx/test/selftest/sh.cpp/run-error.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/run-error.sh.cpp
rename to libcxx/test/selftest/sh.cpp/run-error.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp b/libcxx/test/selftest/sh.cpp/run-success.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/run-success.sh.cpp
rename to libcxx/test/selftest/sh.cpp/run-success.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp b/libcxx/test/selftest/sh.cpp/substitutions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/substitutions.sh.cpp
rename to libcxx/test/selftest/sh.cpp/substitutions.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp b/libcxx/test/selftest/sh.cpp/werror.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/sh.cpp/werror.sh.cpp
rename to libcxx/test/selftest/sh.cpp/werror.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/shell-no-escape-builtins.sh.cpp b/libcxx/test/selftest/shell-no-escape-builtins.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/shell-no-escape-builtins.sh.cpp
rename to libcxx/test/selftest/shell-no-escape-builtins.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp b/libcxx/test/selftest/stdin-is-piped.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp
rename to libcxx/test/selftest/stdin-is-piped.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/test_macros.pass.cpp b/libcxx/test/selftest/test_macros.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/test_macros.pass.cpp
rename to libcxx/test/selftest/test_macros.pass.cpp
diff --git a/libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp b/libcxx/test/selftest/tmpdir-exists.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/tmpdir-exists.sh.cpp
rename to libcxx/test/selftest/tmpdir-exists.sh.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp b/libcxx/test/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-diagnostics-unmarked.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp b/libcxx/test/selftest/verify.cpp/no-diagnostics.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-diagnostics.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-diagnostics.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp b/libcxx/test/selftest/verify.cpp/no-werror.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/no-werror.verify.cpp
rename to libcxx/test/selftest/verify.cpp/no-werror.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp b/libcxx/test/selftest/verify.cpp/right-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/right-diagnostic.verify.cpp
rename to libcxx/test/selftest/verify.cpp/right-diagnostic.verify.cpp
diff --git a/libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp b/libcxx/test/selftest/verify.cpp/wrong-diagnostic.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/selftest/verify.cpp/wrong-diagnostic.verify.cpp
rename to libcxx/test/selftest/verify.cpp/wrong-diagnostic.verify.cpp
diff --git a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
index 66bd4022f321..9bcdc9c1f2b9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp
@@ -18,6 +18,7 @@
 #include <cassert>
 #include <functional>
 #include <deque>
+#include <ranges>
 
 #include "MinSequenceContainer.h"
 #include "../helpers.h"
@@ -95,6 +96,26 @@ constexpr void test() {
       });
 }
 
+constexpr void test_product_iterator() {
+  using M = std::flat_map<int, int>;
+  {
+    M m1{{1, 1}, {2, 1}, {3, 1}};
+    M m2{{4, 1}, {5, 1}, {6, 1}};
+    m1.insert(m2.begin(), m2.end());
+    M expected{{1, 1}, {2, 1}, {3, 1}, {4, 1}, {5, 1}, {6, 1}};
+    assert(m1 == expected);
+  }
+  {
+    std::vector<int> keys{1, 2, 3};
+    std::vector<int> values{1, 1, 1};
+    auto zv = std::views::zip(keys, values);
+    M m;
+    m.insert(zv.begin(), zv.end());
+    M expected{{1, 1}, {2, 1}, {3, 1}};
+    assert(m == expected);
+  }
+}
+
 constexpr bool test() {
   test<std::vector<int>, std::vector<double>>();
 #ifndef __cpp_lib_constexpr_deque
@@ -105,7 +126,7 @@ constexpr bool test() {
   }
   test<MinSequenceContainer<int>, MinSequenceContainer<double>>();
   test<std::vector<int, min_allocator<int>>, std::vector<double, min_allocator<double>>>();
-
+  test_product_iterator();
   if (!TEST_IS_CONSTANT_EVALUATED) {
     auto insert_func = [](auto& m, const auto& newValues) { m.insert(newValues.begin(), newValues.end()); };
     test_insert_range_exception_guarantee(insert_func);
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
index 223b92fc3e8e..e0afa752b5a9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<int>, KeyContainer>;
   M m;
@@ -38,15 +38,23 @@ void test_one() {
   assert(m.empty());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
index dde1f7092e5b..e2f52ab458c4 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp
@@ -24,7 +24,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   {
     using A1 = limited_allocator<int, 10>;
     using C  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
@@ -59,10 +59,15 @@ void test() {
     assert(c.max_size() <= max_dist);
     assert(c.max_size() <= alloc_max_size(std::allocator<char>()));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
index 9f5ffdd06635..4c4de89f58ba 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.capacity/size.pass.cpp
@@ -16,6 +16,7 @@
 #include <deque>
 #include <flat_set>
 #include <functional>
+#include <type_traits>
 #include <vector>
 
 #include "MinSequenceContainer.h"
@@ -23,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
   using S = typename M::size_type;
   {
@@ -46,7 +47,7 @@ void test_one() {
   }
   {
     M m;
-    S s = 1000000;
+    S s = TEST_IS_CONSTANT_EVALUATED ? 100 : 1000000;
     for (auto i = 0u; i < s; ++i) {
       m.emplace(i);
     }
@@ -56,15 +57,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
index d14e883dd5e9..3b7a3550391f 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/alloc.pass.cpp
@@ -14,15 +14,18 @@
 //   explicit flat_set(const Allocator& a);
 
 #include <cassert>
+#include <deque>
 #include <flat_set>
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "../../../test_compare.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // The constructors in this subclause shall not participate in overload
     // resolution unless uses_allocator_v<container_type, Alloc> is true.
@@ -30,8 +33,8 @@ void test() {
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
     using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
+    using V1 = KeyContainer<int, A1>;
+    using V2 = KeyContainer<int, A2>;
     using M1 = std::flat_set<int, C, V1>;
     using M2 = std::flat_set<int, C, V2>;
     static_assert(std::is_constructible_v<M1, const A1&>);
@@ -41,14 +44,14 @@ void test() {
   }
   {
     // explicit
-    using M = std::flat_set<int, std::less<int>, std::vector<int, test_allocator<int>>>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
 
     static_assert(std::is_constructible_v<M, test_allocator<int>>);
     static_assert(!std::is_convertible_v<test_allocator<int>, M>);
   }
   {
     using A = test_allocator<short>;
-    using M = std::flat_set<int, std::less<int>, std::vector<int, test_allocator<int>>>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
     M m(A(0, 5));
     assert(m.empty());
     assert(m.begin() == m.end());
@@ -57,8 +60,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
index ad49b6214903..388a515893bd 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/assign_initializer_list.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -45,17 +45,25 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
index 110757a1bb9a..712f33a7b7f8 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/compare.pass.cpp
@@ -23,27 +23,14 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 #include "test_allocator.h"
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const C&, const A1&>);
-  }
+template <class KeyContainer>
+constexpr void test_compare() {
   {
     using C = test_less<int>;
-    auto m  = std::flat_set<int, C>(C(3));
+    auto m  = std::flat_set<int, C, KeyContainer>(C(3));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(3));
@@ -51,16 +38,20 @@ void test() {
   {
     // The one-argument ctor is explicit.
     using C = test_less<int>;
-    static_assert(std::is_constructible_v<std::flat_set<int, C>, C>);
-    static_assert(!std::is_convertible_v<C, std::flat_set<int, C>>);
+    static_assert(std::is_constructible_v<std::flat_set<int, C, KeyContainer>, C>);
+    static_assert(!std::is_convertible_v<C, std::flat_set<int, C, KeyContainer>>);
 
-    static_assert(std::is_constructible_v<std::flat_set<int>, std::less<int>>);
-    static_assert(!std::is_convertible_v<std::less<int>, std::flat_set<int>>);
+    static_assert(std::is_constructible_v<std::flat_set<int, std::less<int>, KeyContainer>, std::less<int>>);
+    static_assert(!std::is_convertible_v<std::less<int>, std::flat_set<int, std::less<int>, KeyContainer>>);
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_compare_alloc() {
   {
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    auto m   = std::flat_set<int, C, std::vector<int, A1>>(C(4), A1(5));
+    auto m   = std::flat_set<int, C, KeyContainer<int, A1>>(C(4), A1(5));
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
@@ -69,9 +60,9 @@ void test() {
   }
   {
     // explicit(false)
-    using C                                      = test_less<int>;
-    using A1                                     = test_allocator<int>;
-    std::flat_set<int, C, std::deque<int, A1>> m = {C(4), A1(5)};
+    using C                                        = test_less<int>;
+    using A1                                       = test_allocator<int>;
+    std::flat_set<int, C, KeyContainer<int, A1>> m = {C(4), A1(5)};
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp() == C(4));
@@ -80,8 +71,45 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const C&, const A1&>);
+  }
+  test_compare<std::vector<int>>();
+  test_compare<MinSequenceContainer<int>>();
+  test_compare<std::vector<int, min_allocator<int>>>();
+
+  test_compare_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_compare<std::deque<int>>();
+    test_compare_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
index 6b1246885bf5..ea775ebd19be 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/containers.pass.cpp
@@ -36,44 +36,24 @@ void conversion_test(T);
 template <class T, class... Args>
 concept ImplicitlyConstructible = requires(Args&&... args) { conversion_test<T>({std::forward<Args>(args)...}); };
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, const V1&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const V2&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const V1&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const V2&, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, const V1&, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, const V2&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, const V1&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, const V2&, const C&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // flat_set(container_type)
-    using M             = std::flat_set<int>;
-    std::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m              = M(ks);
+    using M              = std::flat_set<int, std::less<int>, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m               = M(ks);
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3}));
 
     // explicit
-    static_assert(std::is_constructible_v<M, const std::vector<int>&>);
-    static_assert(!ImplicitlyConstructible<M, const std::vector<int>&>);
+    static_assert(std::is_constructible_v<M, const KeyContainer<int>&>);
+    static_assert(!ImplicitlyConstructible<M, const KeyContainer<int>&>);
   }
   {
     // flat_set(container_type)
     // move-only
     MoveOnly expected[] = {3, 2, 1};
-    using Ks            = std::deque<MoveOnly, min_allocator<MoveOnly>>;
+    using Ks            = KeyContainer<MoveOnly, min_allocator<MoveOnly>>;
     using M             = std::flat_set<MoveOnly, std::greater<MoveOnly>, Ks>;
     Ks ks;
     ks.push_back(1);
@@ -87,8 +67,8 @@ void test() {
     // flat_set(container_type)
     // container's allocator is used
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
     auto m  = M(std::move(ks));
     assert(ks.empty()); // it was moved-from
     assert((m == M{1, 2, 3}));
@@ -97,22 +77,22 @@ void test() {
   }
   {
     // flat_set(container_type , key_compare)
-    using C             = test_less<int>;
-    using M             = std::flat_set<int, C>;
-    std::vector<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m              = M(ks, C(4));
+    using C              = test_less<int>;
+    using M              = std::flat_set<int, C, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m               = M(ks, C(4));
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3}));
     assert(m.key_comp() == C(4));
 
     // explicit
-    static_assert(std::is_constructible_v<M, const std::vector<int>&, const C&>);
-    static_assert(!ImplicitlyConstructible<M, const std::vector<int>&, const C&>);
+    static_assert(std::is_constructible_v<M, const KeyContainer<int>&, const C&>);
+    static_assert(!ImplicitlyConstructible<M, const KeyContainer<int>&, const C&>);
   }
   {
     // flat_set(container_type , const Allocator&)
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 1, 1, 2, 2, 3, 2, 3, 3}, A(5));
     auto m  = M(ks, A(4)); // replaces the allocators
     assert(!ks.empty());   // it was an lvalue above
     assert((m == M{1, 2, 3}));
@@ -120,7 +100,7 @@ void test() {
     assert(keys.get_allocator() == A(4));
 
     // explicit(false)
-    static_assert(ImplicitlyConstructible<M, const std::deque<int, A>&, const A&>);
+    static_assert(ImplicitlyConstructible<M, const KeyContainer<int, A>&, const A&>);
     M m2 = {ks, A(4)};   // implicit ctor
     assert(!ks.empty()); // it was an lvalue above
     assert(m2 == m);
@@ -129,11 +109,11 @@ void test() {
   }
   {
     // flat_set(container_type , key_compare, const Allocator&)
-    using C                = test_less<int>;
-    using A                = test_allocator<int>;
-    using M                = std::flat_set<int, C, std::vector<int, A>>;
-    std::vector<int, A> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
-    auto m                 = M(ks, C(4), A(5));
+    using C                 = test_less<int>;
+    using A                 = test_allocator<int>;
+    using M                 = std::flat_set<int, C, KeyContainer<int, A>>;
+    KeyContainer<int, A> ks = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+    auto m                  = M(ks, C(4), A(5));
     assert(std::ranges::equal(m, std::vector<int, A>{1, 2, 3}));
     assert(m.key_comp() == C(4));
     auto m_copy = m;
@@ -141,7 +121,7 @@ void test() {
     assert(keys.get_allocator() == A(5));
 
     // explicit(false)
-    static_assert(ImplicitlyConstructible<M, const std::vector<int, A>&, const A&>);
+    static_assert(ImplicitlyConstructible<M, const KeyContainer<int, A>&, const A&>);
     M m2 = {ks, C(4), A(5)};
     assert(m2 == m);
     assert(m2.key_comp() == C(4));
@@ -150,8 +130,44 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, const V1&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V2&, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, const V1&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, const V2&, const C&, const A1&>);
+  }
+
+  test<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
index 1ba550d98f01..5864d7d751b1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy.pass.cpp
@@ -14,6 +14,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <deque>
 #include <flat_set>
 #include <vector>
 
@@ -21,10 +22,11 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
     using M = std::flat_set<int, C, decltype(ks)>;
     auto mo = M(ks, C(5));
     auto m  = mo;
@@ -42,7 +44,7 @@ void test() {
   }
   {
     using C  = test_less<int>;
-    using Ks = std::vector<int, other_allocator<int>>;
+    using Ks = KeyContainer<int, other_allocator<int>>;
     auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
     using M  = std::flat_set<int, C, Ks>;
     auto mo  = M(Ks(ks, other_allocator<int>(6)), C(5));
@@ -61,8 +63,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
index 5011bd200306..108a3cf1c496 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_alloc.pass.cpp
@@ -23,7 +23,27 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
+  using C = test_less<int>;
+  KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+  using M = std::flat_set<int, C, decltype(ks)>;
+  auto mo = M(ks, C(5));
+  auto m  = M(mo, test_allocator<int>(3));
+
+  assert(m.key_comp() == C(5));
+  assert(std::ranges::equal(m, ks));
+  auto keys = std::move(m).extract();
+  assert(keys.get_allocator() == test_allocator<int>(3));
+
+  // mo is unchanged
+  assert(mo.key_comp() == C(5));
+  assert(std::ranges::equal(mo, ks));
+  auto keys2 = std::move(mo).extract();
+  assert(keys2.get_allocator() == test_allocator<int>(6));
+}
+
+constexpr bool test() {
   {
     // The constructors in this subclause shall not participate in overload
     // resolution unless uses_allocator_v<container_type, Alloc> is true.
@@ -40,28 +60,21 @@ void test() {
     static_assert(!std::is_constructible_v<M1, const M1&, const A2&>);
     static_assert(!std::is_constructible_v<M2, const M2&, const A1&>);
   }
-  {
-    using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
-    using M = std::flat_set<int, C, decltype(ks)>;
-    auto mo = M(ks, C(5));
-    auto m  = M(mo, test_allocator<int>(3));
 
-    assert(m.key_comp() == C(5));
-    assert(std::ranges::equal(m, ks));
-    auto keys = std::move(m).extract();
-    assert(keys.get_allocator() == test_allocator<int>(3));
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
 
-    // mo is unchanged
-    assert(mo.key_comp() == C(5));
-    assert(std::ranges::equal(mo, ks));
-    auto keys2 = std::move(mo).extract();
-    assert(keys2.get_allocator() == test_allocator<int>(6));
-  }
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
index 695363e3aeab..59caa9c1f8a4 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/copy_assign.pass.cpp
@@ -13,6 +13,7 @@
 // flat_set& operator=(const flat_set& m);
 
 #include <algorithm>
+#include <deque>
 #include <flat_set>
 #include <functional>
 #include <vector>
@@ -22,11 +23,12 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // test_allocator is not propagated
     using C = test_less<int>;
-    std::vector<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
+    KeyContainer<int, test_allocator<int>> ks({1, 3, 5}, test_allocator<int>(6));
     using M = std::flat_set<int, C, decltype(ks)>;
     auto mo = M(ks, C(5));
     auto m  = M({{3, 4, 5}}, C(3), test_allocator<int>(2));
@@ -46,7 +48,7 @@ void test() {
   {
     // other_allocator is propagated
     using C  = test_less<int>;
-    using Ks = std::vector<int, other_allocator<int>>;
+    using Ks = KeyContainer<int, other_allocator<int>>;
     auto ks  = Ks({1, 3, 5}, other_allocator<int>(6));
     using M  = std::flat_set<int, C, Ks>;
     auto mo  = M(Ks(ks, other_allocator<int>(6)), C(5));
@@ -64,9 +66,9 @@ void test() {
     auto keys2 = std::move(mo).extract();
     assert(keys2.get_allocator() == other_allocator<int>(6));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // comparator is copied and invariant is preserved
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     M mo    = M({1, 2}, std::less<int>());
     M m     = M({1, 2}, std::greater<int>());
     assert(m.key_comp()(2, 1) == true);
@@ -77,7 +79,7 @@ void test() {
   }
   {
     // self-assignment
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     M m     = {{1, 2}};
     m       = static_cast<const M&>(m);
     assert((m == M{{1, 2}}));
@@ -93,8 +95,21 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
index 5cbd557eb856..75eba28b61bd 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/default.pass.cpp
@@ -25,28 +25,29 @@
 #include "test_macros.h"
 
 struct DefaultCtableComp {
-  explicit DefaultCtableComp() { default_constructed_ = true; }
-  bool operator()(int, int) const { return false; }
+  constexpr explicit DefaultCtableComp() { default_constructed_ = true; }
+  constexpr bool operator()(int, int) const { return false; }
   bool default_constructed_ = false;
 };
 
 struct ThrowingCtorComp {
-  ThrowingCtorComp() noexcept(false) {}
-  bool operator()(const auto&, const auto&) const { return false; }
+  constexpr ThrowingCtorComp() noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const { return false; }
 };
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
-    std::flat_set<int> m;
+    std::flat_set<int, std::less<int>, KeyContainer<int>> m;
     assert(m.empty());
   }
   {
     // explicit(false)
-    std::flat_set<int> m = {};
+    std::flat_set<int, std::less<int>, KeyContainer<int>> m = {};
     assert(m.empty());
   }
   {
-    std::flat_set<int, DefaultCtableComp, std::deque<int, min_allocator<int>>> m;
+    std::flat_set<int, DefaultCtableComp, KeyContainer<int, min_allocator<int>>> m;
     assert(m.empty());
     assert(m.begin() == m.end());
     assert(m.key_comp().default_constructed_);
@@ -54,43 +55,56 @@ void test() {
   {
     using A1 = explicit_allocator<int>;
     {
-      std::flat_set<int, DefaultCtableComp, std::vector<int, A1>> m;
+      std::flat_set<int, DefaultCtableComp, KeyContainer<int, A1>> m;
       assert(m.empty());
       assert(m.key_comp().default_constructed_);
     }
     {
       A1 a1;
-      std::flat_set<int, DefaultCtableComp, std::vector<int, A1>> m(a1);
+      std::flat_set<int, DefaultCtableComp, KeyContainer<int, A1>> m(a1);
       assert(m.empty());
       assert(m.key_comp().default_constructed_);
     }
   }
 #if defined(_LIBCPP_VERSION)
   {
-    using C = std::flat_set<MoveOnly>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly>>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
     C c;
   }
   {
-    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, test_allocator<MoveOnly>>>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly, test_allocator<MoveOnly>>>;
     static_assert(std::is_nothrow_default_constructible_v<C>);
     C c;
   }
 #endif // _LIBCPP_VERSION
   {
-    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, std::vector<MoveOnly, other_allocator<MoveOnly>>>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly, other_allocator<MoveOnly>>>;
     static_assert(!std::is_nothrow_default_constructible_v<C>);
     C c;
   }
   {
-    using C = std::flat_set<MoveOnly, ThrowingCtorComp>;
+    using C = std::flat_set<MoveOnly, ThrowingCtorComp, KeyContainer<MoveOnly>>;
     static_assert(!std::is_nothrow_default_constructible_v<C>);
     C c;
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
index fa1e2478af45..810b13b0a5b0 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/dtor_noexcept.pass.cpp
@@ -23,39 +23,53 @@
 #include "test_allocator.h"
 
 struct ThrowingDtorComp {
-  bool operator()(const auto&, const auto&) const;
-  ~ThrowingDtorComp() noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const;
+  constexpr ~ThrowingDtorComp() noexcept(false) {}
 };
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
-    using C = std::flat_set<MoveOnly, MoveOnly>;
+    using C = std::flat_set<MoveOnly, std::less<MoveOnly>, KeyContainer<MoveOnly>>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
   {
-    using V = std::vector<MoveOnly, test_allocator<MoveOnly>>;
+    using V = KeyContainer<MoveOnly, test_allocator<MoveOnly>>;
     using C = std::flat_set<MoveOnly, std::less<MoveOnly>, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
   {
-    using V = std::deque<MoveOnly, other_allocator<MoveOnly>>;
+    using V = KeyContainer<MoveOnly, other_allocator<MoveOnly>>;
     using C = std::flat_set<MoveOnly, std::greater<MoveOnly>, V>;
     static_assert(std::is_nothrow_destructible_v<C>);
     C c;
   }
 #if defined(_LIBCPP_VERSION)
   {
-    using C = std::flat_set<MoveOnly, ThrowingDtorComp>;
+    using C = std::flat_set<MoveOnly, ThrowingDtorComp, KeyContainer<MoveOnly>>;
     static_assert(!std::is_nothrow_destructible_v<C>);
     C c;
   }
 #endif // _LIBCPP_VERSION
 }
 
+constexpr bool test() {
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
index 9aed5c88ee72..2a8e4c40ebe2 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/initializer_list.pass.cpp
@@ -30,57 +30,17 @@
 #include "../../../test_compare.h"
 
 struct DefaultCtableComp {
-  explicit DefaultCtableComp() { default_constructed_ = true; }
-  bool operator()(int, int) const { return false; }
+  constexpr explicit DefaultCtableComp() { default_constructed_ = true; }
+  constexpr bool operator()(int, int) const { return false; }
   bool default_constructed_ = false;
 };
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    using IL = std::initializer_list<int>;
-    static_assert(std::is_constructible_v<M1, IL, const A1&>);
-    static_assert(std::is_constructible_v<M2, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M2, IL, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, IL, const C&, const A1&>);
-  }
-
-  {
-    // initializer_list<value_type> needs to match exactly
-    using M = std::flat_set<int>;
-    using C = typename M::key_compare;
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C, std::allocator<int>>);
-    static_assert(std::is_constructible_v<M, std::initializer_list<int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
-  }
-
+template <template <class...> class KeyContainer>
+constexpr void test() {
   int expected[] = {1, 2, 3, 5};
   {
     // flat_set(initializer_list<value_type>);
-    using M                       = std::flat_set<int>;
+    using M                       = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     std::initializer_list<int> il = {5, 2, 2, 3, 1, 3};
     M m(il);
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
@@ -88,13 +48,13 @@ void test() {
   {
     // flat_set(initializer_list<value_type>);
     // explicit(false)
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     M m     = {5, 2, 2, 3, 1, 3};
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
   }
   {
     // flat_set(initializer_list<value_type>);
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, min_allocator<int>>>;
     M m     = {5, 2, 2, 3, 1, 3};
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
   }
@@ -103,15 +63,14 @@ void test() {
     {
       // flat_set(initializer_list<value_type>);
       // different comparator
-      using M = std::flat_set<int, DefaultCtableComp, std::vector<int, A>>;
+      using M = std::flat_set<int, DefaultCtableComp, KeyContainer<int, A>>;
       M m     = {1, 2, 3};
       assert(m.size() == 1);
-      LIBCPP_ASSERT(*m.begin() == 1);
       assert(m.key_comp().default_constructed_);
     }
     {
       // flat_set(initializer_list<value_type>, const Allocator&);
-      using M = std::flat_set<int, std::greater<int>, std::deque<int, A>>;
+      using M = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
       A a;
       M m({5, 2, 2, 3, 1, 3}, a);
       assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
@@ -120,7 +79,7 @@ void test() {
   {
     // flat_set(initializer_list<value_type>, const key_compare&);
     using C = test_less<int>;
-    using M = std::flat_set<int, C>;
+    using M = std::flat_set<int, C, KeyContainer<int>>;
     auto m  = M({5, 2, 2, 3, 1, 3}, C(10));
     assert(std::equal(m.begin(), m.end(), expected, expected + 4));
     assert(m.key_comp() == C(10));
@@ -130,10 +89,10 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(10));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(initializer_list<value_type>, const key_compare&);
     // Sorting uses the comparator that was passed in
-    using M = std::flat_set<int, std::function<bool(int, int)>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int, min_allocator<int>>>;
     auto m  = M({5, 2, 2, 1, 3, 1}, std::greater<int>());
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
     assert(m.key_comp()(2, 1) == true);
@@ -141,15 +100,69 @@ void test() {
   {
     // flat_set(initializer_list<value_type> il, const key_compare& comp, const Alloc& a);
     using A = explicit_allocator<int>;
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, A>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
     A a;
     M m({5, 2, 2, 3, 1, 3}, {}, a);
     assert(std::equal(m.rbegin(), m.rend(), expected, expected + 4));
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    using IL = std::initializer_list<int>;
+    static_assert(std::is_constructible_v<M1, IL, const A1&>);
+    static_assert(std::is_constructible_v<M2, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M1, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, IL, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, IL, const C&, const A1&>);
+  }
+
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_set<int>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::initializer_list<int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::initializer_list<const int>, std::allocator<int>>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
index 2d0b07c9155f..256f01a7b2f1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/iter_iter.pass.cpp
@@ -23,43 +23,21 @@
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C     = test_less<int>;
-    using A1    = test_allocator<int>;
-    using A2    = other_allocator<int>;
-    using V1    = std::vector<int, A1>;
-    using V2    = std::vector<int, A2>;
-    using M1    = std::flat_set<int, C, V1>;
-    using M2    = std::flat_set<int, C, V2>;
-    using Iter1 = typename M1::iterator;
-    using Iter2 = typename M2::iterator;
-    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
-    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A1&>);
-  }
-
+template <class KeyContainer>
+constexpr void test() {
   int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
   int expected[] = {1, 2, 3};
   {
     // flat_set(InputIterator , InputIterator)
     // cpp17_input_iterator
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     auto m  = M(cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 9));
     assert(std::ranges::equal(m, expected));
 
@@ -70,21 +48,21 @@ void test() {
   {
     // flat_set(InputIterator , InputIterator)
     // greater
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer>;
     auto m  = M(cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 9));
-    assert(std::ranges::equal(m, std::deque<int, min_allocator<int>>{3, 2, 1}));
+    assert(std::ranges::equal(m, KeyContainer{3, 2, 1}));
   }
   {
     // flat_set(InputIterator , InputIterator)
     // Test when the operands are of array type (also contiguous iterator type)
-    using M = std::flat_set<int, std::greater<int>, std::vector<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer>;
     auto m  = M(ar, ar);
     assert(m.empty());
   }
   {
     // flat_set(InputIterator , InputIterator, const key_compare&)
     using C = test_less<int>;
-    using M = std::flat_set<int, C, std::vector<int>>;
+    using M = std::flat_set<int, C, KeyContainer>;
     auto m  = M(ar, ar + 9, C(3));
     assert(std::ranges::equal(m, expected));
     assert(m.key_comp() == C(3));
@@ -94,10 +72,17 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(3));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
+  int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+  int expected[] = {1, 2, 3};
+
   {
     // flat_set(InputIterator , InputIterator, const Allocator&)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     auto m   = M(ar, ar + 9, A1(5));
     assert(std::ranges::equal(m, expected));
     assert(std::move(m).extract().get_allocator() == A1(5));
@@ -106,7 +91,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     M m      = {ar, ar + 9, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
     assert(std::move(m).extract().get_allocator() == A1(5));
@@ -115,7 +100,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const key_compare&, const Allocator&)
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::vector<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     auto m   = M(ar, ar + 9, C(3), A1(5));
     assert(std::ranges::equal(m, expected));
     assert(m.key_comp() == C(3));
@@ -125,7 +110,7 @@ void test() {
     // flat_set(InputIterator , InputIterator, const key_compare&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     M m      = {ar, ar + 9, {}, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
     LIBCPP_ASSERT(std::ranges::equal(m, expected));
@@ -133,8 +118,53 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_set<int, C, V1>;
+    using M2    = std::flat_set<int, C, V2>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const A1&>);
+    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, Iter1, Iter1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, Iter2, Iter2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<MinSequenceContainer<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
index e26f8c8f25ce..fcc15edc2935 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move.pass.cpp
@@ -25,11 +25,12 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C = test_less<int>;
     using A = test_allocator<int>;
-    using M = std::flat_set<int, C, std::deque<int, A>>;
+    using M = std::flat_set<int, C, KeyContainer<int, A>>;
     M mo    = M({1, 2, 3}, C(5), A(7));
     M m     = std::move(mo);
     assert((m == M{1, 2, 3}));
@@ -43,7 +44,7 @@ void test() {
   {
     using C = test_less<int>;
     using A = min_allocator<int>;
-    using M = std::flat_set<int, C, std::vector<int, A>>;
+    using M = std::flat_set<int, C, KeyContainer<int, A>>;
     M mo    = M({1, 2, 3}, C(5), A());
     M m     = std::move(mo);
     assert((m == M{1, 2, 3}));
@@ -54,9 +55,9 @@ void test() {
     assert(mo.key_comp() == C(5));
     assert(std::move(mo).extract().get_allocator() == A());
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // A moved-from flat_set maintains its class invariant in the presence of moved-from comparators.
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     M mo    = M({1, 2, 3}, std::less<int>());
     M m     = std::move(mo);
     assert(m.size() == 3);
@@ -86,47 +87,35 @@ struct ThrowingMoveAllocator {
   using value_type                                    = T;
   explicit ThrowingMoveAllocator()                    = default;
   ThrowingMoveAllocator(const ThrowingMoveAllocator&) = default;
-  ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
-  T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
-  void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
+  constexpr ThrowingMoveAllocator(ThrowingMoveAllocator&&) noexcept(false) {}
+  constexpr T* allocate(std::ptrdiff_t n) { return std::allocator<T>().allocate(n); }
+  constexpr void deallocate(T* p, std::ptrdiff_t n) { return std::allocator<T>().deallocate(p, n); }
   friend bool operator==(ThrowingMoveAllocator, ThrowingMoveAllocator) = default;
 };
 
 struct ThrowingMoveComp {
   ThrowingMoveComp() = default;
-  ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
-  ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
-  bool operator()(const auto&, const auto&) const { return false; }
-};
-
-struct MoveSensitiveComp {
-  MoveSensitiveComp() noexcept(false)                  = default;
-  MoveSensitiveComp(const MoveSensitiveComp&) noexcept = default;
-  MoveSensitiveComp(MoveSensitiveComp&& rhs) { rhs.is_moved_from_ = true; }
-  MoveSensitiveComp& operator=(const MoveSensitiveComp&) noexcept(false) = default;
-  MoveSensitiveComp& operator=(MoveSensitiveComp&& rhs) {
-    rhs.is_moved_from_ = true;
-    return *this;
-  }
-  bool operator()(const auto&, const auto&) const { return false; }
-  bool is_moved_from_ = false;
+  constexpr ThrowingMoveComp(const ThrowingMoveComp&) noexcept(true) {}
+  constexpr ThrowingMoveComp(ThrowingMoveComp&&) noexcept(false) {}
+  constexpr bool operator()(const auto&, const auto&) const { return false; }
 };
 
-void test_move_noexcept() {
+template <template <class...> class KeyContainer>
+constexpr void test_move_noexcept() {
   {
-    using C = std::flat_set<int>;
+    using C = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
   {
-    using C = std::flat_set<int, std::less<int>, std::deque<int, test_allocator<int>>>;
+    using C = std::flat_set<int, std::less<int>, KeyContainer<int, test_allocator<int>>>;
     LIBCPP_STATIC_ASSERT(std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
 #if _LIBCPP_VERSION
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // Container fails to be nothrow-move-constructible; this relies on libc++'s support for non-nothrow-copyable allocators
     using C = std::flat_set<int, std::less<int>, std::deque<int, ThrowingMoveAllocator<int>>>;
     static_assert(!std::is_nothrow_move_constructible_v<std::deque<int, ThrowingMoveAllocator<int>>>);
@@ -137,13 +126,26 @@ void test_move_noexcept() {
 #endif // _LIBCPP_VERSION
   {
     // Comparator fails to be nothrow-move-constructible
-    using C = std::flat_set<int, ThrowingMoveComp>;
+    using C = std::flat_set<int, ThrowingMoveComp, KeyContainer<int>>;
     static_assert(!std::is_nothrow_move_constructible_v<C>);
     C c;
     C d = std::move(c);
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+  test_move_noexcept<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque>();
+    test_move_noexcept<std::deque>();
+  }
+  return true;
+}
+
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
 static int countdown = 0;
 
@@ -179,10 +181,12 @@ void test_move_exception() {
 
 int main(int, char**) {
   test();
-  test_move_noexcept();
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
   test_move_exception();
 #endif
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
index 489b6ff36324..9e88cbb002da 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_alloc.pass.cpp
@@ -24,28 +24,13 @@
 #include "../../../test_compare.h"
 #include "test_allocator.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
-    static_assert(std::is_constructible_v<M2, M2&&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, M2&&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     int expected[] = {1, 2, 3};
     using C        = test_less<int>;
     using A        = test_allocator<int>;
-    using M        = std::flat_set<int, C, std::deque<int, A>>;
+    using M        = std::flat_set<int, C, KeyContainer<int, A>>;
     auto mo        = M(expected, expected + 3, C(5), A(7));
     auto m         = M(std::move(mo), A(3));
 
@@ -72,8 +57,38 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, M1&&, const A1&>);
+    static_assert(std::is_constructible_v<M2, M2&&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, M1&&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, M2&&, const A1&>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
index 0e0ab0aa135f..1c3affe05d10 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/move_assign.pass.cpp
@@ -30,9 +30,9 @@
 struct MoveNegates {
   int value_    = 0;
   MoveNegates() = default;
-  MoveNegates(int v) : value_(v) {}
-  MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
-  MoveNegates& operator=(MoveNegates&& rhs) {
+  constexpr MoveNegates(int v) : value_(v) {}
+  constexpr MoveNegates(MoveNegates&& rhs) : value_(rhs.value_) { rhs.value_ = -rhs.value_; }
+  constexpr MoveNegates& operator=(MoveNegates&& rhs) {
     value_     = rhs.value_;
     rhs.value_ = -rhs.value_;
     return *this;
@@ -44,9 +44,9 @@ struct MoveNegates {
 struct MoveClears {
   int value_   = 0;
   MoveClears() = default;
-  MoveClears(int v) : value_(v) {}
-  MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
-  MoveClears& operator=(MoveClears&& rhs) {
+  constexpr MoveClears(int v) : value_(v) {}
+  constexpr MoveClears(MoveClears&& rhs) : value_(rhs.value_) { rhs.value_ = 0; }
+  constexpr MoveClears& operator=(MoveClears&& rhs) {
     value_     = rhs.value_;
     rhs.value_ = 0;
     return *this;
@@ -68,11 +68,12 @@ struct MoveAssignThrows : std::vector<int> {
 };
 #endif // TEST_HAS_NO_EXCEPTIONS
 
-void test_move_assign_clears() {
+template <template <class...> class KeyContainer>
+constexpr void test_move_assign_clears() {
   // Preserves the class invariant for the moved-from flat_set.
   {
     const int expected[] = {1, 2, 3, 4, 5, 6, 7, 8};
-    using M              = std::flat_set<MoveNegates, std::less<MoveNegates>>;
+    using M              = std::flat_set<MoveNegates, std::less<MoveNegates>, KeyContainer<MoveNegates>>;
     M m                  = M(expected, expected + 8);
     M m2                 = M(expected, expected + 3);
 
@@ -89,7 +90,7 @@ void test_move_assign_clears() {
   }
   {
     const int expected[] = {1, 2, 3, 4, 5, 6, 7, 8};
-    using M              = std::flat_set<MoveClears, std::less<MoveClears>>;
+    using M              = std::flat_set<MoveClears, std::less<MoveClears>, KeyContainer<MoveClears>>;
     M m                  = M(expected, expected + 8);
     M m2                 = M(expected, expected + 3);
 
@@ -106,7 +107,7 @@ void test_move_assign_clears() {
   }
   {
     // moved-from object maintains invariant if one of underlying container does not clear after move
-    using M = std::flat_set<int, std::less<>, std::vector<int>>;
+    using M = std::flat_set<int, std::less<>, CopyOnlyVector<int>>;
     M m1    = M({1, 2, 3});
     M m2    = M({1, 2});
     m2      = std::move(m1);
@@ -115,7 +116,7 @@ void test_move_assign_clears() {
     LIBCPP_ASSERT(m1.empty());
   }
 #if !defined(TEST_HAS_NO_EXCEPTIONS)
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     using M = std::flat_set<int, std::less<>, MoveAssignThrows>;
     M m1    = {1, 2, 3};
     M m2    = {1, 2};
@@ -189,11 +190,12 @@ void test_move_assign_no_except() {
   }
 }
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     using C                           = test_less<int>;
     using A1                          = test_allocator<int>;
-    using M                           = std::flat_set<int, C, std::vector<int, A1>>;
+    using M                           = std::flat_set<int, C, KeyContainer<int, A1>>;
     M mo                              = M({1, 2, 3}, C(5), A1(7));
     M m                               = M({}, C(3), A1(7));
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -207,7 +209,7 @@ void test() {
   {
     using C                           = test_less<int>;
     using A1                          = other_allocator<int>;
-    using M                           = std::flat_set<int, C, std::deque<int, A1>>;
+    using M                           = std::flat_set<int, C, KeyContainer<int, A1>>;
     M mo                              = M({4, 5}, C(5), A1(7));
     M m                               = M({1, 2, 3, 4}, C(3), A1(7));
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -220,7 +222,7 @@ void test() {
   }
   {
     using A                           = min_allocator<int>;
-    using M                           = std::flat_set<int, std::greater<int>, std::vector<int, A>>;
+    using M                           = std::flat_set<int, std::greater<int>, KeyContainer<int, A>>;
     M mo                              = M({5, 4, 3}, A());
     M m                               = M({4, 3, 2, 1}, A());
     std::same_as<M&> decltype(auto) r = m = std::move(mo);
@@ -232,10 +234,27 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test<std::vector>();
+  test_move_assign_clears<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque>();
+    test_move_assign_clears<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
-  test_move_assign_clears();
   test_move_assign_no_except();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
index bd7b5c12432e..031c035bf5bf 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/range.pass.cpp
@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
@@ -56,35 +57,14 @@ static_assert(
     !std::
         is_constructible_v<Set, std::from_range_t, RangeOf<std::pair<int, int>>, std::less<int>, std::allocator<int>>);
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A1&>);
-  }
-
+template <class KeyContainer>
+constexpr void test() {
   int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
   int expected[] = {1, 2, 3};
   {
     // flat_set(from_range_t, R&&)
     // input_range && !common
-    using M    = std::flat_set<int>;
+    using M    = std::flat_set<int, std::less<int>, KeyContainer>;
     using Iter = cpp20_input_iterator<const int*>;
     using Sent = sentinel_wrapper<Iter>;
     using R    = std::ranges::subrange<Iter, Sent>;
@@ -99,17 +79,17 @@ void test() {
   {
     // flat_set(from_range_t, R&&)
     // greater
-    using M    = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M    = std::flat_set<int, std::greater<int>, KeyContainer>;
     using Iter = cpp20_input_iterator<const int*>;
     using Sent = sentinel_wrapper<Iter>;
     using R    = std::ranges::subrange<Iter, Sent>;
     auto m     = M(std::from_range, R(Iter(ar), Sent(Iter(ar + 9))));
-    assert(std::ranges::equal(m, std::deque<int, min_allocator<int>>{3, 2, 1}));
+    assert(std::ranges::equal(m, KeyContainer{3, 2, 1}));
   }
   {
     // flat_set(from_range_t, R&&)
     // contiguous range
-    using M = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     using R = std::ranges::subrange<const int*>;
     auto m  = M(std::from_range, R(ar, ar + 9));
     assert(std::ranges::equal(m, expected));
@@ -117,7 +97,7 @@ void test() {
   {
     // flat_set(from_range_t, R&&, const key_compare&)
     using C = test_less<int>;
-    using M = std::flat_set<int, C, std::vector<int>>;
+    using M = std::flat_set<int, C, KeyContainer>;
     using R = std::ranges::subrange<const int*>;
     auto m  = M(std::from_range, R(ar, ar + 9), C(3));
     assert(std::ranges::equal(m, expected));
@@ -128,10 +108,16 @@ void test() {
     assert(m2 == m);
     assert(m2.key_comp() == C(3));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
+  int ar[]       = {1, 1, 1, 2, 2, 3, 2, 3, 3};
+  int expected[] = {1, 2, 3};
   {
     // flat_set(from_range_t, R&&, const Allocator&)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     auto m   = M(std::from_range, R(ar, ar + 9), A1(5));
     assert(std::ranges::equal(m, expected));
@@ -141,7 +127,7 @@ void test() {
     // flat_set(from_range_t, R&&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     M m      = {std::from_range, R(ar, ar + 9), A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
@@ -151,7 +137,7 @@ void test() {
     // flat_set(from_range_t, R&&, const key_compare&, const Allocator&)
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::vector<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     auto m   = M(std::from_range, R(ar, ar + 9), C(3), A1(5));
     assert(std::ranges::equal(m, expected));
@@ -162,7 +148,7 @@ void test() {
     // flat_set(from_range_t, R&&, const key_compare&, const Allocator&)
     // explicit(false)
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, std::less<int>, std::deque<int, A1>>;
+    using M  = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     using R  = std::ranges::subrange<const int*>;
     M m      = {std::from_range, R(ar, ar + 9), {}, A1(5)}; // implicit ctor
     assert(std::ranges::equal(m, expected));
@@ -170,8 +156,51 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::from_range_t, M1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::from_range_t, M2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<MinSequenceContainer<int>>();
+
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
index 873ff32b6293..5cf6d4c23f4e 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_container.pass.cpp
@@ -30,33 +30,13 @@
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A1&>);
-  }
+template <template <class...> class KeyContainer>
+constexpr void test() {
   {
     // flat_set(sorted_unique_t, container_type)
-    using M             = std::flat_set<int>;
-    std::vector<int> ks = {1, 2, 4, 10};
-    auto ks2            = ks;
+    using M              = std::flat_set<int, std::less<int>, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 2, 4, 10};
+    auto ks2             = ks;
 
     auto m = M(std::sorted_unique, ks);
     assert((m == M{1, 2, 4, 10}));
@@ -71,7 +51,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, container_type)
     // non-default container, comparator and allocator type
-    using Ks = std::deque<int, min_allocator<int>>;
+    using Ks = KeyContainer<int, min_allocator<int>>;
     using M  = std::flat_set<int, std::greater<int>, Ks>;
     Ks ks    = {10, 4, 2, 1};
     auto m   = M(std::sorted_unique, ks);
@@ -84,8 +64,8 @@ void test() {
     // flat_set(sorted_unique_t, container_type)
     // allocator copied into the containers
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 2, 4, 10}, A(4));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 2, 4, 10}, A(4));
     auto m  = M(std::sorted_unique, std::move(ks));
     assert(ks.empty()); // it was moved-from
     assert((m == M{1, 2, 4, 10}));
@@ -93,9 +73,9 @@ void test() {
   }
   {
     // flat_set(sorted_unique_t, container_type ,  key_compare)
-    using C             = test_less<int>;
-    using M             = std::flat_set<int, C>;
-    std::vector<int> ks = {1, 2, 4, 10};
+    using C              = test_less<int>;
+    using M              = std::flat_set<int, C, KeyContainer<int>>;
+    KeyContainer<int> ks = {1, 2, 4, 10};
 
     auto m = M(std::sorted_unique, ks, C(4));
     assert((m == M{1, 2, 4, 10}));
@@ -108,11 +88,11 @@ void test() {
   }
   {
     // flat_set(sorted_unique_t, container_type , key_compare, const Allocator&)
-    using C                = test_less<int>;
-    using A                = test_allocator<int>;
-    using M                = std::flat_set<int, C, std::vector<int, A>>;
-    std::vector<int, A> ks = {1, 2, 4, 10};
-    auto m                 = M(std::sorted_unique, ks, C(4), A(5));
+    using C                 = test_less<int>;
+    using A                 = test_allocator<int>;
+    using M                 = std::flat_set<int, C, KeyContainer<int, A>>;
+    KeyContainer<int, A> ks = {1, 2, 4, 10};
+    auto m                  = M(std::sorted_unique, ks, C(4), A(5));
     assert((m == M{1, 2, 4, 10}));
     assert(m.key_comp() == C(4));
     assert(M(m).extract().get_allocator() == A(5));
@@ -126,8 +106,8 @@ void test() {
   {
     // flat_set(sorted_unique_t, container_type , const Allocator&)
     using A = test_allocator<int>;
-    using M = std::flat_set<int, std::less<int>, std::deque<int, A>>;
-    auto ks = std::deque<int, A>({1, 2, 4, 10}, A(4));
+    using M = std::flat_set<int, std::less<int>, KeyContainer<int, A>>;
+    auto ks = KeyContainer<int, A>({1, 2, 4, 10}, A(4));
     auto m  = M(std::sorted_unique, ks, A(6)); // replaces the allocators
     assert(!ks.empty());                       // it was an lvalue above
     assert((m == M{1, 2, 4, 10}));
@@ -140,8 +120,43 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, const V1&, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, const V2&, const C&, const A1&>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
index a8dac35aefee..00cebb60950f 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_initializer_list.pass.cpp
@@ -31,60 +31,16 @@
 #include "../../../test_compare.h"
 
 template <class T>
-std::initializer_list<T> il = {1, 2, 4, 5};
+constexpr std::initializer_list<T> il = {1, 2, 4, 5};
 
-void test() {
+template <template <class...> class KeyContainer>
+constexpr void test() {
   const auto il1 = il<int>;
   const auto il2 = il<short>;
 
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C  = test_less<int>;
-    using A1 = test_allocator<int>;
-    using A2 = other_allocator<int>;
-    using V1 = std::vector<int, A1>;
-    using V2 = std::vector<int, A2>;
-    using M1 = std::flat_set<int, C, V1>;
-    using M2 = std::flat_set<int, C, V2>;
-    using IL = std::initializer_list<int>;
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A1&>);
-  }
-  {
-    // initializer_list<value_type> needs to match exactly
-    using M = std::flat_set<int>;
-    using C = typename M::key_compare;
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C, std::allocator<int>>);
-    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
-    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
-    static_assert(
-        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
-  }
-
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>);
-    using M       = std::flat_set<int>;
+    using M       = std::flat_set<int, std::less<int>, KeyContainer<int>>;
     auto m        = M(std::sorted_unique, il1);
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
@@ -93,9 +49,9 @@ void test() {
     M m2 = {std::sorted_unique, il1};
     assert(m2 == m);
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&);
-    using M = std::flat_set<int, std::function<bool(int, int)>>;
+    using M = std::flat_set<int, std::function<bool(int, int)>, KeyContainer<int>>;
     auto m  = M(std::sorted_unique, il1, std::less<int>());
     assert(m == M({1, 2, 4, 5}, std::less<>()));
     assert(m.key_comp()(1, 2) == true);
@@ -107,7 +63,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&);
     // greater
-    using M = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M = std::flat_set<int, std::greater<int>, KeyContainer<int, min_allocator<int>>>;
     std::initializer_list<int> il4{5, 4, 2, 1};
     auto m = M(std::sorted_unique, il4, std::greater<int>());
     assert((m == M{5, 4, 2, 1}));
@@ -115,7 +71,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, initializer_list<value_type>,  const Allocator&)
     using A1      = test_allocator<short>;
-    using M       = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M       = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     auto m        = M(std::sorted_unique, il2, A1(5));
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
@@ -130,7 +86,7 @@ void test() {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
     using C  = test_less<int>;
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, C, std::vector<short, A1>>;
+    using M  = std::flat_set<short, C, KeyContainer<short, A1>>;
     auto m   = M(std::sorted_unique, il2, C(3), A1(5));
     assert((m == M{1, 2, 4, 5}));
     assert(m.key_comp() == C(3));
@@ -140,15 +96,73 @@ void test() {
     // flat_set(sorted_unique_t, initializer_list<value_type>, const key_compare&, const Allocator&);
     // explicit(false)
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M  = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     M m      = {std::sorted_unique, il2, {}, A1(5)}; // implicit ctor
     assert((m == M{1, 2, 4, 5}));
     assert(std::move(m).extract().get_allocator() == A1(5));
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C  = test_less<int>;
+    using A1 = test_allocator<int>;
+    using A2 = other_allocator<int>;
+    using V1 = std::vector<int, A1>;
+    using V2 = std::vector<int, A2>;
+    using M1 = std::flat_set<int, C, V1>;
+    using M2 = std::flat_set<int, C, V2>;
+    using IL = std::initializer_list<int>;
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, IL, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, IL, const C&, const A1&>);
+  }
+  {
+    // initializer_list<value_type> needs to match exactly
+    using M = std::flat_set<int>;
+    using C = typename M::key_compare;
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, C, std::allocator<int>>);
+    static_assert(std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>>);
+    static_assert(!std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, C, std::allocator<int>>);
+    static_assert(
+        !std::is_constructible_v<M, std::sorted_unique_t, std::initializer_list<const int>, std::allocator<int>>);
+  }
+
+  test<std::vector>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test<std::deque>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
index b184ee9c3f5a..7ffb09e7347c 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.cons/sorted_iter_iter.pass.cpp
@@ -22,40 +22,20 @@
 #include <functional>
 #include <vector>
 
+#include "MinSequenceContainer.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_iterators.h"
 #include "test_macros.h"
 #include "../../../test_compare.h"
 
-void test() {
-  {
-    // The constructors in this subclause shall not participate in overload
-    // resolution unless uses_allocator_v<container_type, Alloc> is true.
-
-    using C     = test_less<int>;
-    using A1    = test_allocator<int>;
-    using A2    = other_allocator<int>;
-    using V1    = std::vector<int, A1>;
-    using V2    = std::vector<int, A2>;
-    using M1    = std::flat_set<int, C, V1>;
-    using M2    = std::flat_set<int, C, V2>;
-    using Iter1 = typename M1::iterator;
-    using Iter2 = typename M2::iterator;
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A1&>);
-
-    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A1&>);
-    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A2&>);
-    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A1&>);
-  }
+template <class KeyContainer>
+constexpr void test() {
   {
     // flat_set(sorted_unique_t, InputIterator, InputIterator);
     // cpp17_input_iterator
-    using M  = std::flat_set<int>;
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
+
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique, cpp17_input_iterator<const int*>(ar), cpp17_input_iterator<const int*>(ar + 4));
     auto expected = M{1, 2, 4, 5};
@@ -69,16 +49,16 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator);
     // contiguous iterator
     using C       = test_less<int>;
-    using M       = std::flat_set<int, C, std::vector<int, min_allocator<int>>>;
+    using M       = std::flat_set<int, C, KeyContainer>;
     int ar[]      = {1, 2, 4, 5};
     auto m        = M(std::sorted_unique, ar, ar + 4);
     auto expected = M{1, 2, 4, 5};
     assert(m == expected);
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // cpp_17_input_iterator
-    using M  = std::flat_set<int, std::function<bool(int, int)>>;
+    using M  = std::flat_set<int, std::function<bool(int, int)>, KeyContainer>;
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique,
                cpp17_input_iterator<const int*>(ar),
@@ -97,7 +77,7 @@ void test() {
   {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // greater
-    using M  = std::flat_set<int, std::greater<int>, std::deque<int, min_allocator<int>>>;
+    using M  = std::flat_set<int, std::greater<int>, KeyContainer>;
     int ar[] = {5, 4, 2, 1};
     auto m   = M(std::sorted_unique,
                cpp17_input_iterator<const int*>(ar),
@@ -109,16 +89,20 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&);
     // contiguous iterator
     using C   = test_less<int>;
-    using M   = std::flat_set<int, C, std::vector<int, min_allocator<int>>>;
+    using M   = std::flat_set<int, C, KeyContainer>;
     int ar[1] = {42};
     auto m    = M(std::sorted_unique, ar, ar, C(5));
     assert(m.empty());
     assert(m.key_comp() == C(5));
   }
+}
+
+template <template <class...> class KeyContainer>
+constexpr void test_alloc() {
   {
     // flat_set(sorted_unique_t, InputIterator , InputIterator, const Allocator&)
     using A1      = test_allocator<int>;
-    using M       = std::flat_set<int, std::less<int>, std::vector<int, A1>>;
+    using M       = std::flat_set<int, std::less<int>, KeyContainer<int, A1>>;
     int ar[]      = {1, 2, 4, 5};
     auto m        = M(std::sorted_unique, ar, ar + 4, A1(5));
     auto expected = M{1, 2, 4, 5};
@@ -134,7 +118,7 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
     using C  = test_less<int>;
     using A1 = test_allocator<int>;
-    using M  = std::flat_set<int, C, std::deque<int, A1>>;
+    using M  = std::flat_set<int, C, KeyContainer<int, A1>>;
     int ar[] = {1, 2, 4, 5};
     auto m   = M(std::sorted_unique, ar, ar + 4, C(3), A1(5));
     assert((m == M{1, 2, 4, 5}));
@@ -145,7 +129,7 @@ void test() {
     // flat_set(sorted_unique_t, InputIterator, InputIterator, const key_compare&, const Allocator&);
     // explicit(false)
     using A1 = test_allocator<short>;
-    using M  = std::flat_set<short, std::less<int>, std::deque<short, A1>>;
+    using M  = std::flat_set<short, std::less<int>, KeyContainer<short, A1>>;
     int ar[] = {1, 2, 4, 5};
     M m      = {std::sorted_unique, ar, ar + 4, {}, A1(5)}; // implicit ctor
     assert((m == M{1, 2, 4, 5}));
@@ -153,8 +137,52 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  {
+    // The constructors in this subclause shall not participate in overload
+    // resolution unless uses_allocator_v<container_type, Alloc> is true.
+
+    using C     = test_less<int>;
+    using A1    = test_allocator<int>;
+    using A2    = other_allocator<int>;
+    using V1    = std::vector<int, A1>;
+    using V2    = std::vector<int, A2>;
+    using M1    = std::flat_set<int, C, V1>;
+    using M2    = std::flat_set<int, C, V2>;
+    using Iter1 = typename M1::iterator;
+    using Iter2 = typename M2::iterator;
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const A1&>);
+
+    static_assert(std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A1&>);
+    static_assert(std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M1, std::sorted_unique_t, Iter1, Iter1, const C&, const A2&>);
+    static_assert(!std::is_constructible_v<M2, std::sorted_unique_t, Iter2, Iter2, const C&, const A1&>);
+  }
+
+  test<std::vector<int>>();
+  test<std::vector<int, min_allocator<int>>>();
+  test<MinSequenceContainer<int>>();
+  test_alloc<std::vector>();
+
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test<std::deque<int>>();
+    test_alloc<std::deque>();
+  }
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
index d7d7a98f49d7..924fbdab5de9 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.erasure/erase_if.pass.cpp
@@ -32,7 +32,7 @@ static_assert(HasStdErase<std::vector<int>>);
 static_assert(!HasStdErase<std::flat_set<int>>);
 
 template <class M>
-M make(std::initializer_list<int> vals) {
+constexpr M make(std::initializer_list<int> vals) {
   M ret;
   for (int v : vals)
     ret.emplace(v);
@@ -40,8 +40,8 @@ M make(std::initializer_list<int> vals) {
 }
 
 template <class M, class Pred>
-void test0(
-    std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
+constexpr void
+test0(std::initializer_list<int> vals, Pred p, std::initializer_list<int> expected, std::size_t expected_erased_count) {
   M s = make<M>(vals);
   ASSERT_SAME_TYPE(typename M::size_type, decltype(std::erase_if(s, p)));
   assert(expected_erased_count == std::erase_if(s, p));
@@ -50,11 +50,11 @@ void test0(
 
 struct NotBool {
   bool b;
-  explicit operator bool() const { return b; }
+  constexpr explicit operator bool() const { return b; }
 };
 
 template <class S>
-void test_one() {
+constexpr void test_one() {
   // Test all the plausible signatures for this predicate.
   auto is1        = [](typename S::const_reference v) { return v == 1; };
   auto is2        = [](typename S::value_type v) { return v == 2; };
@@ -84,18 +84,28 @@ void test_one() {
   test0<S>({1, 2, 3}, nonBoolIs1, {2, 3}, 1);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::flat_set<int>>();
   test_one<std::flat_set<int, std::less<int>, std::vector<int, min_allocator<int>>>>();
   test_one<std::flat_set<int, std::greater<int>, std::vector<int, test_allocator<int>>>>();
-  test_one<std::flat_set<int, std::less<int>, std::deque<int, min_allocator<int>>>>();
-  test_one<std::flat_set<int, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::flat_set<int, std::less<int>, std::deque<int, min_allocator<int>>>>();
+    test_one<std::flat_set<int, std::greater<int>, std::deque<int, test_allocator<int>>>>();
+  }
   test_one<std::flat_set<long>>();
   test_one<std::flat_set<double>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
index 846bcff63d77..8aafa97818bc 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator.pass.cpp
@@ -30,7 +30,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -67,9 +67,12 @@ void test_one() {
   assert(i == m.begin());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -88,10 +91,15 @@ void test() {
     assert(!(ii1 != cii));
     assert(!(cii != ii1));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
index ad1bc8facc92..fbc519cba98d 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/iterator_comparison.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -141,15 +141,23 @@ void test_one() {
   assert(cri2 <=> cri1 == std::strong_ordering::greater);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
index 4c2482cd0f3d..3725333f1f10 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.iterators/reverse_iterator.pass.cpp
@@ -25,19 +25,22 @@
 
 #include <iterator>
 
+#include "MinSequenceContainer.h"
+#include "min_allocator.h"
 #include "test_macros.h"
 
-void test() {
+template <class KeyContainer>
+constexpr void test_one() {
   {
-    using M     = std::flat_set<int, std::less<int>, std::deque<int>>;
+    using M     = std::flat_set<int, std::less<int>, KeyContainer>;
     M m         = {1, 2, 3, 4};
     const M& cm = m;
-    ASSERT_SAME_TYPE(decltype(m.rbegin()), M::reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.crbegin()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(cm.rbegin()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.rend()), M::reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(m.crend()), M::const_reverse_iterator);
-    ASSERT_SAME_TYPE(decltype(cm.rend()), M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rbegin()), typename M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crbegin()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rbegin()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.rend()), typename M::reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(m.crend()), typename M::const_reverse_iterator);
+    ASSERT_SAME_TYPE(decltype(cm.rend()), typename M::const_reverse_iterator);
     static_assert(noexcept(m.rbegin()));
     static_assert(noexcept(cm.rbegin()));
     static_assert(noexcept(m.crbegin()));
@@ -49,12 +52,12 @@ void test() {
     assert(std::distance(cm.rbegin(), cm.rend()) == 4);
     assert(std::distance(m.crbegin(), m.crend()) == 4);
     assert(std::distance(cm.crbegin(), cm.crend()) == 4);
-    M::reverse_iterator i; // default-construct
+    typename M::reverse_iterator i; // default-construct
     ASSERT_SAME_TYPE(decltype(*i), const int&);
-    i                           = m.rbegin(); // move-assignment
-    M::const_reverse_iterator k = i;          // converting constructor
-    assert(i == k);                           // comparison
-    for (int j = 4; j >= 1; --j, ++i) {       // pre-increment
+    i                                    = m.rbegin(); // move-assignment
+    typename M::const_reverse_iterator k = i;          // converting constructor
+    assert(i == k);                                    // comparison
+    for (int j = 4; j >= 1; --j, ++i) {                // pre-increment
       assert(*i == j);
     }
     assert(i == m.rend());
@@ -65,11 +68,11 @@ void test() {
     assert(i == m.rbegin());
   }
   {
+    using M = std::flat_set<int, std::less<int>, KeyContainer>;
     // N3644 testing
-    using C = std::flat_set<int>;
-    C::reverse_iterator ii1{}, ii2{};
-    C::reverse_iterator ii4 = ii1;
-    C::const_reverse_iterator cii{};
+    typename M::reverse_iterator ii1{}, ii2{};
+    typename M::reverse_iterator ii4 = ii1;
+    typename M::const_reverse_iterator cii{};
     assert(ii1 == ii2);
     assert(ii1 == ii4);
     assert(!(ii1 != ii2));
@@ -81,8 +84,23 @@ void test() {
   }
 }
 
+constexpr bool test() {
+  test_one<std::vector<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
+  test_one<MinSequenceContainer<int>>();
+  test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
+}
+
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
index 5ecdc429b3c5..f92609fc3455 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/clear.pass.cpp
@@ -38,7 +38,7 @@ static_assert(NoExceptClear<std::flat_set<int, std::less<int>, ThrowOnMoveContai
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -58,17 +58,24 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
index 79800e894afb..de8c05c48c24 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace.pass.cpp
@@ -28,7 +28,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -98,7 +98,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_emplaceable() {
+constexpr void test_emplaceable() {
   using M = std::flat_set<Emplaceable, std::less<Emplaceable>, KeyContainer>;
   using R = std::pair<typename M::iterator, bool>;
 
@@ -121,16 +121,24 @@ void test_emplaceable() {
   assert(*m.begin() == Emplaceable(1, 3.5));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_emplaceable<std::vector<Emplaceable>>();
-  test_emplaceable<std::deque<Emplaceable>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_emplaceable<std::deque<Emplaceable>>();
   test_emplaceable<MinSequenceContainer<Emplaceable>>();
   test_emplaceable<std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -141,6 +149,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
index b3bd8adf0c35..0b5aab1b1c61 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/emplace_hint.pass.cpp
@@ -27,7 +27,7 @@
 #include "../helpers.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = M::iterator;
@@ -114,7 +114,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_emplaceable() {
+constexpr void test_emplaceable() {
   using M = std::flat_set<Emplaceable, std::less<Emplaceable>, KeyContainer>;
   using R = M::iterator;
 
@@ -134,16 +134,24 @@ void test_emplaceable() {
   assert(*m.begin() == Emplaceable(1, 3.5));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_emplaceable<std::vector<Emplaceable>>();
-  test_emplaceable<std::vector<Emplaceable>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_emplaceable<std::deque<Emplaceable>>();
   test_emplaceable<MinSequenceContainer<Emplaceable>>();
   test_emplaceable<std::vector<Emplaceable, min_allocator<Emplaceable>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -154,6 +162,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
index 27c6fadb5341..c908f5d63b87 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter.pass.cpp
@@ -27,7 +27,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -94,11 +94,16 @@ void test_one() {
   assert(i8 == m.end());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -109,6 +114,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
index f44a8864bdbc..bc53e306851b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_iter_iter.pass.cpp
@@ -26,7 +26,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using I   = M::iterator;
@@ -78,11 +78,16 @@ void test_one() {
   assert(i5 == m.end());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -93,6 +98,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
index c50b219895b1..c5d4cd1052b1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key.pass.cpp
@@ -26,7 +26,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer, class Compare = std::less<>>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, Compare, KeyContainer>;
 
   auto make = [](std::initializer_list<int> il) {
@@ -74,12 +74,17 @@ void test_one() {
   assert(m.empty());
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<int>, std::greater<>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -95,6 +100,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
index f4b373364cab..b6701e36a55a 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/erase_key_transparent.pass.cpp
@@ -38,10 +38,10 @@ static_assert(!CanErase<const NonTransparentSet>);
 
 template <class Key, class It>
 struct HeterogeneousKey {
-  explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
-  operator It() && { return it_; }
-  auto operator<=>(Key key) const { return key_ <=> key; }
-  friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
+  constexpr explicit HeterogeneousKey(Key key, It it) : key_(key), it_(it) {}
+  constexpr operator It() && { return it_; }
+  constexpr auto operator<=>(Key key) const { return key_ <=> key; }
+  constexpr friend bool operator<(const HeterogeneousKey&, const HeterogeneousKey&) {
     assert(false);
     return false;
   }
@@ -50,7 +50,7 @@ struct HeterogeneousKey {
 };
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -70,7 +70,7 @@ void test_one() {
 }
 
 template <class KeyContainer>
-void test_transparent_comparator() {
+constexpr void test_transparent_comparator() {
   using M = std::flat_set<std::string, TransparentComparator, KeyContainer>;
   {
     M m = {"alpha", "beta", "epsilon", "eta", "gamma"};
@@ -95,14 +95,20 @@ void test_transparent_comparator() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
   test_transparent_comparator<std::vector<std::string>>();
-  test_transparent_comparator<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_transparent_comparator<std::deque<std::string>>();
   test_transparent_comparator<MinSequenceContainer<std::string>>();
   test_transparent_comparator<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -145,6 +151,8 @@ void test() {
     auto n = m.erase("beta");
     assert(n == 1);
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -159,6 +167,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
index 12e63e0d23e1..b554ef815490 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp
@@ -33,7 +33,7 @@ static_assert(!CanExtract<std::flat_set<int> const&>);
 static_assert(!CanExtract<std::flat_set<int> const&&>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
   {
     M m = M({1, 2, 3});
@@ -55,9 +55,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -70,6 +73,8 @@ void test() {
     check_invariant(m);
     LIBCPP_ASSERT(m.empty());
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -97,6 +102,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
index e0cb80f74462..2e59818b4828 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_cv.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -59,11 +59,16 @@ void test_one() {
   assert(*r.first == 3);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -78,6 +83,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
index 992c96ecba1a..ad6f379b091e 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer_list.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = typename M::value_type;
@@ -67,11 +67,16 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -87,6 +92,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
index 76b4d5f0eb7c..710f83f78ecb 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using R   = typename M::iterator;
@@ -55,11 +55,16 @@ void test_one() {
   assert(*r == 3);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -75,6 +80,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
index 18bdb798de01..af25de9b1af3 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp
@@ -37,7 +37,7 @@ static_assert(!CanInsert<Set, int, int>);
 static_assert(!CanInsert<Set, cpp20_input_iterator<int*>, cpp20_input_iterator<int*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using M = std::flat_set<int, std::less<int>, KeyContainer>;
 
   int ar1[] = {
@@ -75,9 +75,12 @@ void test_one() {
   assert(m == expected2);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   {
@@ -86,6 +89,8 @@ void test() {
     m.insert(v.begin(), v.end());
     assert(std::ranges::equal(m, std::vector<int>{1, 2, 3, 4}));
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -96,6 +101,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
index d29de98e4d3d..a1fb10115bff 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp
@@ -22,7 +22,7 @@
 #include "test_macros.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = Key;
@@ -49,15 +49,22 @@ void test_one() {
   assert(*r == V(3));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<MoveOnly>>();
-  test_one<std::deque<int>>();
-  test_one<std::deque<MoveOnly>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::deque<int>>();
+    test_one<std::deque<MoveOnly>>();
+  }
   test_one<MinSequenceContainer<int>>();
   test_one<MinSequenceContainer<MoveOnly>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -73,6 +80,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
index 966a696c096b..a624f9161a98 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp
@@ -39,7 +39,7 @@ static_assert(!CanInsertRange<Set, std::ranges::subrange<std::pair<int, int>*>>)
 static_assert(!CanInsertRange<Set, std::ranges::subrange<std::pair<short, short>*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
 
   {
@@ -65,7 +65,7 @@ void test_one() {
   {
     // The "uniquing" part uses the comparator, not operator==.
     struct ModTen {
-      bool operator()(int a, int b) const { return (a % 10) < (b % 10); }
+      constexpr bool operator()(int a, int b) const { return (a % 10) < (b % 10); }
     };
     using M  = std::flat_set<Key, ModTen, KeyContainer>;
     M m      = {21, 43, 15, 37};
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
   {
@@ -96,6 +99,8 @@ void test() {
     MoveOnly expected[] = {1, 3, 4, 5};
     assert(std::ranges::equal(m, expected));
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -106,6 +111,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
index a179632d6421..b5272f2ae6e2 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp
@@ -25,7 +25,7 @@
 #include "../helpers.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
   using R   = std::pair<typename M::iterator, bool>;
@@ -57,15 +57,22 @@ void test_one() {
   assert(*r.first == V(3));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
   test_one<std::vector<MoveOnly>>();
-  test_one<std::deque<int>>();
-  test_one<std::deque<MoveOnly>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+  {
+    test_one<std::deque<int>>();
+    test_one<std::deque<MoveOnly>>();
+  }
   test_one<MinSequenceContainer<int>>();
   test_one<MinSequenceContainer<MoveOnly>>();
   test_one<std::vector<int, min_allocator<int>>>();
   test_one<std::vector<MoveOnly, min_allocator<MoveOnly>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -81,6 +88,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
index fb0a8b049e72..bc6476bfaaac 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   using V   = Key;
@@ -47,11 +47,16 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -67,6 +72,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
index 6258c4dbfdcb..5287dbec1929 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanInsert<Set, std::sorted_unique_t, int, int>);
 static_assert(!CanInsert<Set, std::sorted_unique_t, cpp20_input_iterator<int*>, cpp20_input_iterator<int*>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -60,11 +60,16 @@ void test_one() {
   assert(m == expected2);
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -77,6 +82,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
index 4b5779a10ee7..3d5c954c5e31 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp
@@ -46,7 +46,7 @@ static_assert(!CanInsert<NonTransparentSet, NonConvertibleTransparent<int>>);
 static_assert(!CanInsert<NonTransparentSet, NonTransparentSetIter, NonConvertibleTransparent<int>>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -105,9 +105,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -131,6 +134,8 @@ void test() {
     assert(!inserted);
     assert(iter == m.begin() + 1);
   }
+
+  return true;
 }
 
 void test_exception() {
@@ -153,6 +158,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
index 591def4c0f93..c15266676c61 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp
@@ -31,7 +31,7 @@ static_assert(CanReplace<Set, std::vector<int>>);
 static_assert(!CanReplace<Set, const std::vector<int>&>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -51,11 +51,16 @@ void test_one() {
   assert(std::ranges::equal(m, expected_keys));
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 void test_exception() {
@@ -81,6 +86,9 @@ void test_exception() {
 int main(int, char**) {
   test();
   test_exception();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
index 686fe93a593f..6b4a65e4701b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp
@@ -38,7 +38,7 @@ static_assert(NoExceptAdlSwap<std::flat_set<int, std::less<int>, ThrowOnMoveCont
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
 
@@ -84,15 +84,22 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
-  test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
index 1eac55d768ce..a3b605ee94cf 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp
@@ -37,7 +37,7 @@ static_assert(NoExceptMemberSwap<std::flat_set<int, std::less<int>, ThrowOnMoveC
 #endif
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<Key>, KeyContainer>;
   {
@@ -82,15 +82,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
index ba2c428c9e30..b1c667a879ce 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp
@@ -21,7 +21,7 @@
 
 #include "test_macros.h"
 
-void test() {
+constexpr bool test() {
   {
     using M    = std::flat_set<int>;
     using Comp = std::less<int>; // the default
@@ -36,7 +36,7 @@ void test() {
     assert(vc(1, 2));
     assert(!vc(2, 1));
   }
-  {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
     using Comp = std::function<bool(int, int)>;
     using M    = std::flat_set<int, Comp>;
     Comp comp  = std::greater<int>();
@@ -67,10 +67,15 @@ void test() {
     assert(vc(1, 2));
     assert(!vc(2, 1));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
index e0416fc261ff..cb2b3da89138 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
index 91f49305cd71..d7e45f6b706b 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/contains_transparent.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!CanContains<NonTransparentSet>);
 static_assert(!CanContains<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -60,9 +60,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -82,10 +85,15 @@ void test() {
     assert(m.contains("beta"));
     assert(!m.contains("eta2"));
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
index 51aec21ca212..b2c646e3ecf2 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count.pass.cpp
@@ -23,7 +23,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using S   = typename KeyContainer::size_type;
 
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
index 84e7b9a18b83..8686db3d400e 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/count_transparent.pass.cpp
@@ -35,7 +35,7 @@ static_assert(!CanCount<NonTransparentSet>);
 static_assert(!CanCount<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
   {
@@ -59,9 +59,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -81,10 +84,15 @@ void test() {
     assert(m.count("beta") == 1);
     assert(m.count("eta2") == 0);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
index 7a8b19273289..69e5f1b18cb2 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M  = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -74,15 +74,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
index 31081eb97039..e6667c363963 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/equal_range_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanEqualRange<NonTransparentSet>);
 static_assert(!CanEqualRange<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -87,9 +87,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -110,10 +113,15 @@ void test() {
     assert(first == m.begin() + 1);
     assert(last == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
index 953b8fe63854..daf9b70fdcc3 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find.pass.cpp
@@ -25,7 +25,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, std::less<>, KeyContainer>;
   {
@@ -50,15 +50,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
index b8d454c58f54..d952ba941896 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/find_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanFind<NonTransparentSet>);
 static_assert(!CanFind<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -77,9 +77,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -101,10 +104,15 @@ void test() {
     auto it2 = m.find("beta2");
     assert(it2 == m.end());
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
index 32796b7a164f..f5e6c7c06e46 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -66,15 +66,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
index 0c3e25f795f3..3b24e051e999 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/lower_bound_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanLowerBound<NonTransparentSet>);
 static_assert(!CanLowerBound<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -107,10 +110,15 @@ void test() {
     auto it2 = m.lower_bound("beta2");
     assert(it2 == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
index 6a967cb2371b..c669746862dd 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp
@@ -24,7 +24,7 @@
 #include "min_allocator.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   {
     using M = std::flat_set<Key, std::less<>, KeyContainer>;
@@ -67,15 +67,23 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
index 36f2a2094649..dda8858f2683 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_transparent.pass.cpp
@@ -36,7 +36,7 @@ static_assert(!CanUpperBound<NonTransparentSet>);
 static_assert(!CanUpperBound<const NonTransparentSet>);
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
   using M   = std::flat_set<Key, TransparentComparator, KeyContainer>;
 
@@ -83,9 +83,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<std::string>>();
-  test_one<std::deque<std::string>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<std::string>>();
   test_one<MinSequenceContainer<std::string>>();
   test_one<std::vector<std::string, min_allocator<std::string>>>();
 
@@ -107,10 +110,15 @@ void test() {
     auto it2 = m.upper_bound("beta2");
     assert(it2 == m.begin() + 2);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h b/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
index 4cc720311cf0..3f0dc403e4bc 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/helpers.h
@@ -20,7 +20,7 @@
 #include "test_macros.h"
 
 template <class... Args>
-void check_invariant(const std::flat_set<Args...>& m) {
+constexpr void check_invariant(const std::flat_set<Args...>& m) {
   assert(std::is_sorted(m.begin(), m.end(), m.key_comp()));
   auto key_equal = [&](const auto& x, const auto& y) {
     const auto& c = m.key_comp();
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
index faf746861df3..8c47a63631ef 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp
@@ -15,6 +15,7 @@
 
 #include <flat_set>
 #include <vector>
+#include "test_macros.h"
 
 struct A {
   using Set = std::flat_set<A>;
@@ -25,12 +26,18 @@ struct A {
 };
 
 // Implement the operator< required in order to instantiate flat_set<A>
-bool operator<(A const& L, A const& R) { return L.data < R.data; }
+constexpr bool operator<(A const& L, A const& R) { return L.data < R.data; }
 
-void test() { A a; }
+constexpr bool test() {
+  A a;
+  return true;
+}
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp b/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
index 3e7aecee77fd..a8284389efd1 100644
--- a/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
+++ b/libcxx/test/std/containers/container.adaptors/flat.set/op_compare.pass.cpp
@@ -31,7 +31,7 @@
 #include "test_container_comparisons.h"
 
 template <class KeyContainer>
-void test_one() {
+constexpr void test_one() {
   using Key = typename KeyContainer::value_type;
 
   {
@@ -69,9 +69,12 @@ void test_one() {
   }
 }
 
-void test() {
+constexpr bool test() {
   test_one<std::vector<int>>();
-  test_one<std::deque<int>>();
+#ifndef __cpp_lib_constexpr_deque
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
+    test_one<std::deque<int>>();
   test_one<MinSequenceContainer<int>>();
   test_one<std::vector<int, min_allocator<int>>>();
 
@@ -86,7 +89,7 @@ void test() {
   {
     // Comparisons use value_type's native operators, not the comparator
     struct StrongComp {
-      bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
+      constexpr bool operator()(double a, double b) const { return std::strong_order(a, b) < 0; }
     };
     using C = std::flat_set<double, StrongComp>;
     C s1    = {1};
@@ -101,10 +104,15 @@ void test() {
     assert(s1 != s2);
     assert((s1 <=> s2) == std::partial_ordering::unordered);
   }
+
+  return true;
 }
 
 int main(int, char**) {
   test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/container.adaptors/flat_helpers.h b/libcxx/test/std/containers/container.adaptors/flat_helpers.h
index 57c4595ccfeb..45860bf15c64 100644
--- a/libcxx/test/std/containers/container.adaptors/flat_helpers.h
+++ b/libcxx/test/std/containers/container.adaptors/flat_helpers.h
@@ -9,6 +9,7 @@
 #ifndef TEST_STD_CONTAINERS_CONTAINER_ADAPTORS_FLAT_HELPERS_H
 #define TEST_STD_CONTAINERS_CONTAINER_ADAPTORS_FLAT_HELPERS_H
 
+#include <cstdint>
 #include <vector>
 
 #include "test_macros.h"
@@ -17,7 +18,7 @@ template <class T>
 struct CopyOnlyVector : std::vector<T> {
   using std::vector<T>::vector;
 
-  CopyOnlyVector(const CopyOnlyVector&) = default;
+  constexpr CopyOnlyVector(const CopyOnlyVector&) = default;
   constexpr CopyOnlyVector(CopyOnlyVector&& other) : CopyOnlyVector(other) {}
   constexpr CopyOnlyVector(CopyOnlyVector&& other, std::vector<T>::allocator_type alloc)
       : CopyOnlyVector(other, alloc) {}
@@ -30,14 +31,14 @@ template <class T>
 struct SillyReserveVector : std::vector<T> {
   using std::vector<T>::vector;
 
-  void reserve(size_t) { this->clear(); }
+  constexpr void reserve(std::size_t) { this->clear(); }
 };
 
 template <class T, bool ConvertibleToT = false>
 struct Transparent {
   T t;
 
-  TEST_CONSTEXPR explicit operator T() const
+  constexpr explicit operator T() const
     requires ConvertibleToT
   {
     return t;
@@ -58,10 +59,10 @@ struct TransparentComparator {
 
   bool* transparent_used  = nullptr;
   TransparentComparator() = default;
-  TEST_CONSTEXPR TransparentComparator(bool& used) : transparent_used(&used) {}
+  constexpr TransparentComparator(bool& used) : transparent_used(&used) {}
 
   template <class T, bool Convertible>
-  TEST_CONSTEXPR bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
+  constexpr bool operator()(const T& t, const Transparent<T, Convertible>& transparent) const {
     if (transparent_used != nullptr) {
       *transparent_used = true;
     }
@@ -69,7 +70,7 @@ struct TransparentComparator {
   }
 
   template <class T, bool Convertible>
-  TEST_CONSTEXPR bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
+  constexpr bool operator()(const Transparent<T, Convertible>& transparent, const T& t) const {
     if (transparent_used != nullptr) {
       *transparent_used = true;
     }
@@ -77,7 +78,7 @@ struct TransparentComparator {
   }
 
   template <class T>
-  TEST_CONSTEXPR bool operator()(const T& t1, const T& t2) const {
+  constexpr bool operator()(const T& t1, const T& t2) const {
     return t1 < t2;
   }
 };
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
index 25851e941a7d..e606e538efcb 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
@@ -20,6 +20,7 @@
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <utility>
 
 #include "test_macros.h"
 #include "../../../test_compare.h"
@@ -109,6 +110,19 @@ int main(int, char**) {
     assert(c.max_load_factor() == 1);
   }
 #endif
+  { // Test with std::pair, since we have some special handling for pairs inside __hash_table
+    struct pair_hash {
+      size_t operator()(std::pair<int, int> val) const TEST_NOEXCEPT { return val.first | val.second; }
+    };
+
+    std::pair<int, int> arr[] = {
+        std::make_pair(1, 2), std::make_pair(2, 3), std::make_pair(3, 4), std::make_pair(4, 5)};
+    std::unordered_set<std::pair<int, int>, pair_hash> a(arr, arr + 4);
+    std::unordered_set<std::pair<int, int>, pair_hash> b;
+
+    b = a;
+    assert(a == b);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
index d7c4088f5be0..4e3afc17ad49 100644
--- a/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
+++ b/libcxx/test/std/input.output/file.streams/fstreams/filebuf.virtuals/setbuf.pass.cpp
@@ -15,7 +15,7 @@
 
 // This test requires the fix to https://github.com/llvm/llvm-project/issues/60509 in the dylib,
 // which landed in 5afb937d8a30445642ccaf33866ee4cdd0713222.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 #include <fstream>
 #include <cstddef>
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
index d468d7154dad..88672851f7b6 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_money.pass.cpp
@@ -15,9 +15,10 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+#include <cassert>
 #include <iomanip>
 #include <istream>
-#include <cassert>
+#include <streambuf>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
index addf53fcdeb3..c6e3e43109a2 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/get_time.pass.cpp
@@ -12,9 +12,11 @@
 
 // template <class charT> T9 get_time(struct tm* tmb, const charT* fmt);
 
+#include <cassert>
+#include <ctime>
 #include <iomanip>
 #include <istream>
-#include <cassert>
+#include <streambuf>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp b/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
index 7de93bef0f7b..5b35b7d1cc81 100644
--- a/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/ext.manip/put_time.pass.cpp
@@ -12,9 +12,10 @@
 
 // template <class charT> T10 put_time(const struct tm* tmb, const charT* fmt);
 
+#include <cassert>
+#include <ctime>
 #include <iomanip>
 #include <ostream>
-#include <cassert>
 
 #include "test_macros.h"
 #include "platform_support.h" // locale name macros
diff --git a/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp b/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
index 3b4354d0916c..3b685950d36a 100644
--- a/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
+++ b/libcxx/test/std/input.output/iostream.format/input.streams/istream.unformatted/sync.pass.cpp
@@ -16,7 +16,7 @@
 // The fix for bug 51497 and bug 51499 require and updated dylib due to
 // explicit instantiations. That means Apple backdeployment targets remain
 // broken.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 #include <istream>
 #include <cassert>
diff --git a/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py
new file mode 100644
index 000000000000..46290b1ed52a
--- /dev/null
+++ b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py
@@ -0,0 +1,180 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+# In addition to being available via inclusion of the <iterator> header,
+# the function templates in [iterator.range] are available when any of the following
+# headers are included: <array>, <deque>, <flat_map>, <flat_set>, <forward_list>,
+# <list>, <map>, <regex>, <set>, <span>, <string>, <string_view>, <unordered_map>,
+# <unordered_set>, <vector>.
+
+# RUN: %{python} %s %{libcxx-dir}/utils
+# END.
+
+import sys
+
+sys.path.append(sys.argv[1])
+from libcxx.header_information import (
+    lit_header_restrictions,
+    lit_header_undeprecations,
+    Header,
+)
+
+headers = list(
+    map(
+        Header,
+        [
+            "array",
+            "deque",
+            "flat_map",
+            "flat_set",
+            "forward_list",
+            "list",
+            "map",
+            "regex",
+            "set",
+            "span",
+            "string",
+            "string_view",
+            "unordered_map",
+            "unordered_set",
+            "vector",
+        ],
+    )
+)
+
+for header in headers:
+    print(
+        f"""\
+//--- {header}.pass.cpp
+{lit_header_restrictions.get(header, '')}
+{lit_header_undeprecations.get(header, '')}
+// UNSUPPORTED: c++03
+
+#include <{header}>
+#include <cassert>
+#include <initializer_list>
+
+#include "test_macros.h"
+
+struct Container {{
+  int a[3] = {{1, 2, 3}};
+
+  int* begin() {{ return &a[0]; }}
+  const int* begin() const {{ return &a[0]; }}
+  int* rbegin() {{ return &a[2]; }}
+  const int* rbegin() const {{ return &a[2]; }}
+  int* end() {{ return &a[3]; }}
+  const int* end() const {{ return &a[3]; }}
+  int* rend() {{ return (&a[0]) - 1; }}
+  const int* rend() const {{ return (&a[0]) - 1; }}
+  std::size_t size() const {{ return 3; }}
+  bool empty() const {{ return false; }}
+  int* data() {{return &a[0]; }}
+  const int* data() const {{ return &a[0]; }}
+}};
+
+int main(int, char**)  {{
+  {{
+    Container c;
+    const auto& cc = c;
+    assert(std::begin(c) == c.begin());
+    assert(std::begin(cc) == cc.begin());
+    assert(std::end(c) == c.end());
+    assert(std::end(cc) == cc.end());
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(c) == cc.begin());
+    assert(std::cbegin(cc) == cc.begin());
+    assert(std::cend(c) == cc.end());
+    assert(std::cend(cc) == cc.end());
+    assert(std::rbegin(c) == c.rbegin());
+    assert(std::rbegin(cc) == cc.rbegin());
+    assert(std::rend(c) == cc.rend());
+    assert(std::rend(cc) == cc.rend());
+    assert(std::crbegin(c) == cc.rbegin());
+    assert(std::crbegin(cc) == cc.rbegin());
+    assert(std::crend(c) == cc.rend());
+    assert(std::crend(cc) == cc.rend());
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::data(c) == c.data());
+    assert(std::data(cc) == cc.data());
+    assert(std::size(cc) == cc.size());
+    assert(std::empty(cc) == cc.empty());
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(cc) == 3);
+#endif
+  }}
+  {{
+    int a[]        = {{1, 2, 3}};
+    const auto& ca = a;
+    assert(std::begin(a) == &a[0]);
+    assert(std::begin(ca) == &ca[0]);
+    assert(std::end(a) == &a[3]);
+    assert(std::end(ca) == &ca[3]);
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(a) == &a[0]);
+    assert(std::cbegin(ca) == &ca[0]);
+    assert(std::cend(a) == &a[3]);
+    assert(std::cend(ca) == &ca[3]);
+    assert(std::rbegin(a) == std::reverse_iterator<int*>(std::end(a)));
+    assert(std::rbegin(ca) == std::reverse_iterator<const int*>(std::end(ca)));
+    assert(std::rend(a) == std::reverse_iterator<int*>(std::begin(a)));
+    assert(std::rend(ca) == std::reverse_iterator<const int*>(std::begin(ca)));
+    assert(std::crbegin(a) == std::reverse_iterator<const int*>(std::end(a)));
+    assert(std::crbegin(ca) == std::reverse_iterator<const int*>(std::end(ca)));
+    assert(std::crend(a) == std::reverse_iterator<const int*>(std::begin(a)));
+    assert(std::crend(ca) == std::reverse_iterator<const int*>(std::begin(ca)));
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::size(ca) == 3);
+    assert(!std::empty(ca));
+    assert(std::data(a) == &a[0]);
+    assert(std::data(ca) == &ca[0]);
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(ca) == 3);
+#endif
+  }}
+  {{
+    auto il         = {{1, 2, 3}};
+    const auto& cil = il;
+    assert(std::begin(il) == il.begin());
+    assert(std::begin(cil) == cil.begin());
+    assert(std::end(il) == il.end());
+    assert(std::end(cil) == cil.end());
+#if TEST_STD_VER >= 14
+    assert(std::cbegin(il) == cil.begin());
+    assert(std::cbegin(cil) == cil.begin());
+    assert(std::cend(il) == cil.end());
+    assert(std::cend(cil) == cil.end());
+    assert(std::rbegin(il) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::rbegin(cil) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::rend(il) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::rend(cil) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::crbegin(il) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::crbegin(cil) == std::reverse_iterator<const int*>(std::end(il)));
+    assert(std::crend(il) == std::reverse_iterator<const int*>(std::begin(il)));
+    assert(std::crend(cil) == std::reverse_iterator<const int*>(std::begin(il)));
+#endif
+#if TEST_STD_VER >= 17
+    assert(std::size(cil) == 3);
+    assert(!std::empty(cil));
+    assert(std::data(il) == &*std::begin(il));
+    assert(std::data(cil) == &*std::begin(il));
+#endif
+#if TEST_STD_VER >= 20
+    assert(std::ssize(cil) == 3);
+#endif
+  }}
+
+  return 0;
+}}
+
+"""
+    )
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
index 8dbbbf432ddb..8c0820681188 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
@@ -32,6 +32,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -94,6 +98,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -165,6 +173,10 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should not be defined before c++20"
 #  endif
@@ -245,6 +257,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++20"
 #  endif
@@ -337,6 +356,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++23"
 #  endif
@@ -441,6 +467,13 @@
 #    error "__cpp_lib_boyer_moore_searcher should have the value 201603L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_functional
 #    error "__cpp_lib_constexpr_functional should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
index 45c51b5807c6..e6c0940ab7fd 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
@@ -28,6 +28,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 #  endif
@@ -130,6 +134,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 #  endif
@@ -250,6 +258,10 @@
 #    error "__cpp_lib_bounded_array_traits should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++17"
 #  endif
@@ -394,6 +406,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++20"
 #  endif
@@ -568,6 +587,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++23"
 #  endif
@@ -763,6 +789,13 @@
 #    error "__cpp_lib_bounded_array_traits should have the value 201902L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_has_unique_object_representations
 #    error "__cpp_lib_has_unique_object_representations should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index e54671914223..222d562a19d6 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -164,6 +164,14 @@
 #    error "__cpp_lib_clamp should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifdef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should not be defined before c++14"
 #  endif
@@ -1057,6 +1065,14 @@
 #    error "__cpp_lib_clamp should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++14"
 #  endif
@@ -2055,6 +2071,14 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should not be defined before c++20"
+#  endif
+
+#  ifdef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should not be defined before c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++17"
 #  endif
@@ -3281,6 +3305,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++20"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++20"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++20"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++20"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++20"
 #  endif
@@ -4726,6 +4764,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++23"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++23"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++23"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++23"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++23"
 #  endif
@@ -6399,6 +6451,20 @@
 #    error "__cpp_lib_clamp should have the value 201603L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_common_reference
+#    error "__cpp_lib_common_reference should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference != 202302L
+#    error "__cpp_lib_common_reference should have the value 202302L in c++26"
+#  endif
+
+#  ifndef __cpp_lib_common_reference_wrapper
+#    error "__cpp_lib_common_reference_wrapper should be defined in c++26"
+#  endif
+#  if __cpp_lib_common_reference_wrapper != 202302L
+#    error "__cpp_lib_common_reference_wrapper should have the value 202302L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_complex_udls
 #    error "__cpp_lib_complex_udls should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
index 4a5b3ee1ca87..ea6b07934510 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.get/locale.money.get.members/get_long_double_fr_FR.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
index 251f6b996571..14745996b9fd 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.money.put/locale.money.put.members/put_long_double_fr_FR.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
index 1e3b4a658882..630b2739c88a 100644
--- a/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.monetary/locale.moneypunct.byname/grouping.pass.cpp
@@ -9,7 +9,7 @@
 // TODO(mordante) Investigate
 // UNSUPPORTED: apple-clang
 
-// XFAIL darwin
+// XFAIL: darwin
 //
 // NetBSD does not support LC_MONETARY at the moment
 // XFAIL: netbsd
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
index f32f7b0c1847..612d3738a373 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_double.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
index f063e67e65a0..58bc9e5abef8 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_float.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
index 29fc7a382fe4..bf8bb651d6bc 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/facet.num.get.members/get_long_double.pass.cpp
@@ -11,7 +11,7 @@
 
 // The fix for LWG2381 (https://github.com/llvm/llvm-project/pull/77948) changed behavior of
 // FP parsing. This requires 3e15c97fa3812993bdc319827a5c6d867b765ae8 in the dylib.
-// XFAIL using-built-library-before-llvm-19
+// XFAIL: using-built-library-before-llvm-19
 
 // <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
index 0749617d056a..3f53490c1bae 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_date.pass.cpp
@@ -22,8 +22,10 @@
 // get_date(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
index 1b19f5c94499..fbffafc5bcee 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_monthname.pass.cpp
@@ -20,8 +20,10 @@
 // get_monthname(iter_type s, iter_type end, ios_base& str,
 //               ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
index 35dba465917e..513f92bc3faf 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_one.pass.cpp
@@ -23,8 +23,10 @@
 // iter_type get(iter_type s, iter_type end, ios_base& f,
 //               ios_base::iostate& err, tm *t, char format, char modifier = 0) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
index aa5b0d04caa1..651c1e16092e 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_time.pass.cpp
@@ -19,9 +19,11 @@
 // get_time(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
index da64b573a41c..0df0b10c9d7c 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_weekday.pass.cpp
@@ -19,8 +19,10 @@
 // get_weekday(iter_type s, iter_type end, ios_base& str,
 //             ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
index ec846093583d..ef0d9230e78b 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get.byname/get_year.pass.cpp
@@ -18,9 +18,11 @@
 // iter_type get_year(iter_type s, iter_type end, ios_base& str,
 //                    ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
index a6c7f7816fdc..1eac586d413f 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_date.pass.cpp
@@ -14,9 +14,11 @@
 // get_date(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
index e8ca74ee4aa0..4a4443d38be3 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_many.pass.cpp
@@ -14,9 +14,11 @@
 // get(iter_type s, iter_type end, ios_base& f, ios_base::iostate& err, tm *t,
 //     const char_type *fmt, const char_type *fmtend) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
index 10769f92c50d..10d91715991b 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_monthname.pass.cpp
@@ -14,8 +14,10 @@
 // get_monthname(iter_type s, iter_type end, ios_base& str,
 //               ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
index a4ef9bd9f12c..16a8eaa20b24 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_one.pass.cpp
@@ -13,8 +13,10 @@
 // iter_type get(iter_type s, iter_type end, ios_base& f,
 //               ios_base::iostate& err, tm *t, char format, char modifier = 0) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
index 009f94d26ee5..4d5b5edcdb69 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_time.pass.cpp
@@ -14,8 +14,10 @@
 // get_time(iter_type s, iter_type end, ios_base& str,
 //          ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
index 23d53e32ef0d..21780e59081d 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_weekday.pass.cpp
@@ -14,8 +14,10 @@
 // get_weekday(iter_type s, iter_type end, ios_base& str,
 //             ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
index 663f7647d5ac..c59dca140bfe 100644
--- a/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.time/locale.time.get/locale.time.get.members/get_year.pass.cpp
@@ -13,9 +13,11 @@
 // iter_type get_year(iter_type s, iter_type end, ios_base& str,
 //                    ios_base::iostate& err, tm* t) const;
 
-#include <locale>
 #include <cassert>
+#include <ctime>
 #include <ios>
+#include <locale>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 
diff --git a/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_element.compile.pass.cpp b/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_element.compile.pass.cpp
index 50c0e0eff1ed..3a220578a074 100644
--- a/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_element.compile.pass.cpp
+++ b/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_element.compile.pass.cpp
@@ -16,7 +16,7 @@
 #include <complex>
 #include <concepts>
 
-template <size_t I, typename C>
+template <std::size_t I, typename C>
 concept HasTupleElement = requires { std::tuple_element<I, C>{}; };
 
 struct SomeObject {};
diff --git a/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_size.compile.pass.cpp b/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_size.compile.pass.cpp
index 28dcbe3fc5ec..1091c6345598 100644
--- a/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_size.compile.pass.cpp
+++ b/libcxx/test/std/numerics/complex.number/complex.tuple/tuple_size.compile.pass.cpp
@@ -28,7 +28,7 @@ void test() {
   using C = std::complex<T>;
 
   static_assert(HasTupleSize<C>);
-  static_assert(std::same_as<typename std::tuple_size<C>::value_type, size_t>);
+  static_assert(std::same_as<typename std::tuple_size<C>::value_type, std::size_t>);
   static_assert(std::tuple_size<C>() == 2);
 }
 
diff --git a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
new file mode 100644
index 000000000000..9ac54a873866
--- /dev/null
+++ b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads, no-exceptions
+
+// ASan seems to try to create threadsm which obviouly doesn't work in this test.
+// UNSUPPORTED: asan, hwasan
+
+// UNSUPPORTED: c++03
+
+// There is no way to limit the number of threads on windows
+// UNSUPPORTED: windows
+
+// AIX, macOS and FreeBSD seem to limit the number of processes, not threads via RLIMIT_NPROC
+// XFAIL: target={{.+}}-aix{{.*}}
+// XFAIL: target={{.+}}-apple-{{.*}}
+// XFAIL: freebsd
+
+// This test makes sure that we fail gracefully in care the thread creation fails. This is only reliably possible on
+// systems that allow limiting the number of threads that can be created. See https://llvm.org/PR125428 for more details
+
+#include <cassert>
+#include <future>
+#include <system_error>
+
+#if __has_include(<sys/resource.h>)
+#  include <sys/resource.h>
+#  ifdef RLIMIT_NPROC
+void force_thread_creation_failure() {
+  rlimit lim = {1, 1};
+  assert(setrlimit(RLIMIT_NPROC, &lim) == 0);
+}
+#  else
+#    error "No known way to force only one thread being available"
+#  endif
+#else
+#  error "No known way to force only one thread being available"
+#endif
+
+int main(int, char**) {
+  force_thread_creation_failure();
+
+  try {
+    std::future<int> fut = std::async(std::launch::async, [] { return 1; });
+    assert(false);
+  } catch (const std::system_error&) {
+  }
+
+  try {
+    std::future<void> fut = std::async(std::launch::async, [] { return; });
+    assert(false);
+  } catch (const std::system_error&) {
+  }
+
+  return 0;
+}
diff --git a/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp b/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
new file mode 100644
index 000000000000..7384a195bb36
--- /dev/null
+++ b/libcxx/test/std/thread/futures/futures.async/wait_on_destruct.pass.cpp
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03
+
+// This test uses std::atomic interfaces that are only available in C++20
+// UNSUPPORTED: c++11, c++14, c++17
+
+// Make sure that the `future` destructor keeps the data alive until the thread finished. This test fails by triggering
+// TSan. It may not be observable by normal means.
+
+// See https://github.com/llvm/llvm-project/pull/125433#issuecomment-2703618927 for more details.
+
+#include <atomic>
+#include <future>
+#include <mutex>
+#include <condition_variable>
+#include <thread>
+
+std::mutex mux;
+
+int main(int, char**) {
+  std::condition_variable cond;
+  std::unique_lock lock(mux);
+  auto v = std::async(std::launch::async, [&cond, value = 1]() mutable {
+    std::unique_lock thread_lock(mux);
+    cond.notify_all();
+    thread_lock.unlock();
+
+    value = 4;
+    (void)value;
+  });
+  cond.wait(lock);
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
index 25eb97a2df4d..16c6986ae670 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.T2.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<int, int>, int>);
-static_assert(CanCompare<std::expected<int, int>, EqualityComparable>);
-static_assert(!CanCompare<std::expected<int, int>, NonComparable>);
+static_assert(HasOperatorEqual<std::expected<int, int>, int>);
+static_assert(HasOperatorEqual<std::expected<int, int>, EqualityComparable>);
+static_assert(!HasOperatorEqual<std::expected<int, int>, NonComparable>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
index f0f549b6b777..218f0c39740e 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.other_expected.pass.cpp
@@ -17,26 +17,26 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 // Test constraint
-static_assert(!CanCompare<NonComparable, NonComparable>);
+static_assert(!HasOperatorEqual<NonComparable, NonComparable>);
 
-static_assert(CanCompare<std::expected<int, int>, std::expected<int, int>>);
-static_assert(CanCompare<std::expected<int, int>, std::expected<short, short>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<short, short>>);
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(!CanCompare<std::expected<int, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<int, int>, std::expected<int, int>>);
-static_assert(!CanCompare<std::expected<NonComparable, int>, std::expected<NonComparable, int>>);
-static_assert(!CanCompare<std::expected<int, NonComparable>, std::expected<int, NonComparable>>);
-static_assert(!CanCompare<std::expected<NonComparable, int>, std::expected<int, NonComparable>>);
-static_assert(!CanCompare<std::expected<int, NonComparable>, std::expected<NonComparable, int>>);
+static_assert(!HasOperatorEqual<std::expected<int, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<int, int>, std::expected<int, int>>);
+static_assert(!HasOperatorEqual<std::expected<NonComparable, int>, std::expected<NonComparable, int>>);
+static_assert(!HasOperatorEqual<std::expected<int, NonComparable>, std::expected<int, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<NonComparable, int>, std::expected<int, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<int, NonComparable>, std::expected<NonComparable, int>>);
 #else
 // Note this is true because other overloads in expected<non-void> are unconstrained
-static_assert(CanCompare<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
 #endif
 constexpr bool test() {
   // x.has_value() && y.has_value()
diff --git a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
index 6c7d2f39514e..153cbbddf306 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/equality/equality.unexpected.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<EqualityComparable, EqualityComparable>, std::unexpected<int>>);
-static_assert(CanCompare<std::expected<EqualityComparable, int>, std::unexpected<EqualityComparable>>);
-static_assert(!CanCompare<std::expected<EqualityComparable, NonComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<EqualityComparable, EqualityComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<EqualityComparable, int>, std::unexpected<EqualityComparable>>);
+static_assert(!HasOperatorEqual<std::expected<EqualityComparable, NonComparable>, std::unexpected<int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp b/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
index b6c3d8deee64..af53d0c00f57 100644
--- a/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.void/equality/equality.other_expected.pass.cpp
@@ -17,26 +17,26 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 struct Foo{};
-static_assert(!CanCompare<Foo, Foo>);
+static_assert(!HasOperatorEqual<Foo, Foo>);
 
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, short>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, short>>);
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(!CanCompare<std::expected<void, int>, std::expected<int, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(CanCompare<std::expected<void, int>, std::expected<void, int>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::expected<void, NonComparable>>);
-static_assert(!CanCompare<std::expected<void, int>, std::expected<void, NonComparable>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::expected<void, int>>);
+static_assert(!HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<void, int>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::expected<void, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, int>, std::expected<void, NonComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::expected<void, int>>);
 #else
 // Note this is true because other overloads in expected<non-void> are unconstrained
-static_assert(CanCompare<std::expected<void, int>, std::expected<int, int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::expected<int, int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp b/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
index f37f38bb7151..8d040d2ab7c0 100644
--- a/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.void/equality/equality.unexpected.pass.cpp
@@ -16,14 +16,14 @@
 #include <type_traits>
 #include <utility>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
-#include "../../types.h"
 
 #if TEST_STD_VER >= 26
 // https://wg21.link/P3379R0
-static_assert(CanCompare<std::expected<void, EqualityComparable>, std::unexpected<int>>);
-static_assert(CanCompare<std::expected<void, int>, std::unexpected<EqualityComparable>>);
-static_assert(!CanCompare<std::expected<void, NonComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<void, EqualityComparable>, std::unexpected<int>>);
+static_assert(HasOperatorEqual<std::expected<void, int>, std::unexpected<EqualityComparable>>);
+static_assert(!HasOperatorEqual<std::expected<void, NonComparable>, std::unexpected<int>>);
 #endif
 
 constexpr bool test() {
diff --git a/libcxx/test/std/utilities/expected/types.h b/libcxx/test/std/utilities/expected/types.h
index 11473ca3d97d..df73ebdfe495 100644
--- a/libcxx/test/std/utilities/expected/types.h
+++ b/libcxx/test/std/utilities/expected/types.h
@@ -336,17 +336,4 @@ struct CheckForInvalidWrites : public CheckForInvalidWritesBase<WithPaddedExpect
   }
 };
 
-struct NonComparable {};
-
-struct EqualityComparable {
-  int i;
-  constexpr EqualityComparable(int ii) : i(ii) {}
-
-  friend constexpr bool operator==(const EqualityComparable& data, int ii) { return data.i == ii; }
-};
-
-// Test constraint
-template <class T1, class T2>
-concept CanCompare = requires(T1 t1, T2 t2) { t1 == t2; };
-
 #endif // TEST_STD_UTILITIES_EXPECTED_TYPES_H
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp
new file mode 100644
index 000000000000..a3630cb0017b
--- /dev/null
+++ b/libcxx/test/std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp
@@ -0,0 +1,152 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <functional>
+//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+//
+// common_reference specializations for reference_wrapper
+
+#include <concepts>
+#include <functional>
+#include <type_traits>
+
+template <class T>
+concept HasType = requires { typename T::type; };
+
+template <class Result, class T1, class T2>
+concept check_XY = std::same_as<Result, std::common_reference_t<T1, T2>>;
+
+template <class Result, class T1, class T2>
+concept check_YX = std::same_as<Result, std::common_reference_t<T2, T1>>;
+
+template <class Result, class T1, class T2>
+concept check = check_XY<Result, T1, T2> && check_YX<Result, T1, T2>;
+
+template <class T1, class T2>
+concept check_none_XY = !HasType<std::common_reference<T1, T2>>;
+template <class T1, class T2>
+concept check_none_YX = !HasType<std::common_reference<T2, T1>>;
+
+template <class T1, class T2>
+concept check_none = check_none_XY<T1, T2> && check_none_YX<T1, T2>;
+
+// https://eel.is/c++draft/meta.trans#other-2.4
+template <class X, class Y>
+using CondRes = decltype(false ? std::declval<X (&)()>()() : std::declval<Y (&)()>()());
+
+template <class X, class Y>
+struct Ternary {};
+
+template <class X, class Y>
+  requires requires() { typename CondRes<X, Y>; }
+struct Ternary<X, Y> {
+  using type = CondRes<X, Y>;
+};
+template <class X, class Y>
+using Ternary_t = typename Ternary<X, Y>::type;
+
+template <class T>
+using Ref = std::reference_wrapper<T>;
+
+using std::common_reference_t;
+using std::same_as;
+
+// clang-format off
+static_assert(check<int &     , Ref<int      >, int &      >);
+static_assert(check<int const&, Ref<int      >, int const& >);
+static_assert(check<int const&, Ref<int const>, int &      >);
+static_assert(check<int const&, Ref<int const>, int const& >);
+static_assert(check<int&,       Ref<int> const&, int& >);
+static_assert(check<const volatile int&, Ref<const volatile int>, const volatile int&>);
+
+// derived-base and implicit convertibles
+struct B {};
+struct D : B {};
+struct C {
+    operator B&() const;
+};
+
+static_assert(check<B&      , Ref<B>,       D &     >);
+static_assert(check<B const&, Ref<B>,       D const&>);
+static_assert(check<B const&, Ref<B const>, D const&>);
+
+static_assert(check<B&      , Ref<D>,       B &     >);
+static_assert(check<B const&, Ref<D>,       B const&>);
+static_assert(check<B const&, Ref<D const>, B const&>);
+
+static_assert(std::same_as<B&,       CondRes<Ref<D>,       B&>>);
+static_assert(std::same_as<B const&, CondRes<Ref<D>,       B const &>>);
+static_assert(std::same_as<B const&, CondRes<Ref<D const>, B const&>>);
+
+static_assert( check<B&        , Ref<B>      , C&      >);
+static_assert( check<B&        , Ref<B>      , C       >);
+static_assert( check<B const&  , Ref<B const>, C       >);
+static_assert(!check<B&        , Ref<C>      , B&      >); // Ref<C> cannot be converted to B&
+static_assert( check<B&        , Ref<B>      , C const&>); // was const B& before P2655R3
+
+
+using Ri   = Ref<int>;
+using RRi  = Ref<Ref<int>>;
+using RRRi = Ref<Ref<Ref<int>>>;
+static_assert(check<Ri&,  Ri&,  RRi>);
+static_assert(check<RRi&, RRi&, RRRi>);
+static_assert(check<Ri,   Ri,   RRi>);
+static_assert(check<RRi,  RRi,  RRRi>);
+
+static_assert(check_none<int&, RRi>);
+static_assert(check_none<int,  RRi>);
+static_assert(check_none<int&, RRRi>);
+static_assert(check_none<int,  RRRi>);
+
+static_assert(check_none<Ri&, RRRi>);
+static_assert(check_none<Ri,  RRRi>);
+
+
+template <typename T>
+struct Test {
+  // Check that reference_wrapper<T> behaves the same as T& in common_reference.
+
+  using R1 = common_reference_t<T&, T&>;
+  using R2 = common_reference_t<T&, T const&>;
+  using R3 = common_reference_t<T&, T&&>;
+  using R4 = common_reference_t<T&, T const&&>;
+  using R5 = common_reference_t<T&, T>;
+
+  static_assert(same_as<R1, common_reference_t<Ref<T>, T&>>);
+  static_assert(same_as<R2, common_reference_t<Ref<T>, T const&>>);
+  static_assert(same_as<R3, common_reference_t<Ref<T>, T&&>>);
+  static_assert(same_as<R4, common_reference_t<Ref<T>, T const&&>>);
+  static_assert(same_as<R5, common_reference_t<Ref<T>, T>>);
+
+  // commute:
+  static_assert(same_as<R1, common_reference_t<T&,        Ref<T>>>);
+  static_assert(same_as<R2, common_reference_t<T const&,  Ref<T>>>);
+  static_assert(same_as<R3, common_reference_t<T&&,       Ref<T>>>);
+  static_assert(same_as<R4, common_reference_t<T const&&, Ref<T>>>);
+  static_assert(same_as<R5, common_reference_t<T,         Ref<T>>>);
+
+  // reference qualification of reference_wrapper is irrelevant 
+  static_assert(same_as<R1, common_reference_t<Ref<T>&,        T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> ,        T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> const&,  T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T>&&,       T&>>);
+  static_assert(same_as<R1, common_reference_t<Ref<T> const&&, T&>>);
+};
+
+// clang-format on
+// Instantiate above checks:
+template struct Test<int>;
+template struct Test<std::reference_wrapper<int>>;
+
+// reference_wrapper as both args is unaffected.
+// subject to simple first rule of
+static_assert(check<Ref<int>&, Ref<int>&, Ref<int>&>);
+
+// double wrap is unaffected.
+static_assert(check<Ref<int>&, Ref<Ref<int>>, Ref<int>&>);
diff --git a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
index e802776f52bf..8e3cad797d5a 100644
--- a/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp
@@ -18,9 +18,7 @@
 #include "test_macros.h"
 
 template <class T>
-constexpr bool has_type = requires {
-  typename T::type;
-};
+constexpr bool has_type = requires { typename T::type; };
 
 // A slightly simplified variation of std::tuple
 template <class...>
@@ -74,8 +72,10 @@ static_assert(std::is_same_v<std::common_reference_t<void (&&)()>, void (&&)()>)
 //  -- Otherwise, if sizeof...(T) is two, let T1 and T2 denote the two types in
 //     the pack T. Then
 // (6.3.1)
-//    -- If T1 and T2 are reference types and COMMON_REF(T1, T2) is well-formed,
-//       then the member typedef type denotes that type.
+//    -- Let R be COMMON-REF(T1, T2). If T1 and T2 are reference types, R is well-formed,
+//       and is_convertible_v<add_pointer_t<T1>, add_pointer_t<R>> && is_convertible_v<add_pointer_t<T2>, add_pointer_t<R>>
+//       is true, then the member typedef type denotes R.
+
 struct B {};
 struct D : B {};
 static_assert(std::is_same_v<std::common_reference_t<B&, D&>, B&>);
@@ -99,7 +99,19 @@ static_assert(std::is_same_v<std::common_reference_t<int const&, int volatile&>,
 static_assert(std::is_same_v<std::common_reference_t<int const volatile&&, int volatile&&>, int const volatile&&>);
 
 static_assert(std::is_same_v<std::common_reference_t<int (&)[10], int (&&)[10]>, int const (&)[10]>);
-static_assert(std::is_same_v<std::common_reference_t<int const (&)[10], int volatile (&)[10]>, int const volatile (&)[10]>);
+static_assert(
+    std::is_same_v<std::common_reference_t<int const (&)[10], int volatile (&)[10]>, int const volatile (&)[10]>);
+
+// when conversion from pointers are not true
+struct E {};
+struct F {
+  operator E&() const;
+};
+
+static_assert(!std::is_convertible_v<F*, E*>);
+
+// The following should not use 6.3.1, but fallback to 6.3.3
+static_assert(std::is_same_v<std::common_reference_t<E&, F>, E&>);
 
 // (6.3.2)
 //    -- Otherwise, if basic_common_reference<remove_cvref_t<T1>,
@@ -136,8 +148,8 @@ static_assert(std::is_same_v<std::common_reference_t<int&, MyIntRef>, MyIntRef>)
 //    -- Otherwise, if common_type_t<T1, T2> is well-formed, then the member
 //       typedef type denotes that type.
 struct moveonly {
-  moveonly() = default;
-  moveonly(moveonly&&) = default;
+  moveonly()                      = default;
+  moveonly(moveonly&&)            = default;
   moveonly& operator=(moveonly&&) = default;
 };
 struct moveonly2 : moveonly {};
@@ -169,14 +181,17 @@ static_assert(!has_type<std::common_reference<int, short, int, char*> >);
 
 #if TEST_STD_VER > 20
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, int>>, std::tuple<int, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, long>, std::tuple<long, int>>, std::tuple<long, long>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::tuple<int, long>, std::tuple<long, int>>, std::tuple<long, long>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, const int&>, std::tuple<const int&, int>>,
                              std::tuple<const int&, int>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, volatile int&>, std::tuple<volatile int&, int>>,
                              std::tuple<volatile int&, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::tuple<int&, const volatile int&>, std::tuple<const volatile int&, int>>,
-                             std::tuple<const volatile int&, int>>);
-static_assert(!has_type<std::common_reference_t<std::tuple<const int&, volatile int&>, std::tuple<volatile int&, const int&>>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::tuple<int&, const volatile int&>, std::tuple<const volatile int&, int>>,
+                   std::tuple<const volatile int&, int>>);
+static_assert(
+    !has_type<std::common_reference_t<std::tuple<const int&, volatile int&>, std::tuple<volatile int&, const int&>>>);
 
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, X2>, std::tuple<int, Y2>>, std::tuple<int, Z2>>);
 static_assert(std::is_same_v<std::common_reference_t<std::tuple<int, X2>, std::tuple<int, Y2>>, std::tuple<int, Z2>>);
@@ -185,26 +200,25 @@ static_assert(!has_type<std::common_reference<std::tuple<int, X2>, std::tuple<fl
 static_assert(!has_type<std::common_reference<std::tuple<int, X2>, int, X2>>);
 
 struct A {};
-template <template<class> class TQual, template<class> class UQual>
+template <template <class> class TQual, template <class> class UQual>
 struct std::basic_common_reference<A, std::tuple<B>, TQual, UQual> {
   using type = tuple<UQual<B>>;
 };
 
 static_assert(std::is_same_v<std::common_reference_t<A, std::tuple<B>, std::tuple<D>>, std::tuple<B>>);
 
-
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int, int>>,
-                             std::pair<int, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int, long>, std::pair<long, int>>,
-                             std::pair<long, long>>);
+static_assert(std::is_same_v<std::common_reference_t<std::pair<int, int>>, std::pair<int, int>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::pair<int, long>, std::pair<long, int>>, std::pair<long, long>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, const int&>, std::pair<const int&, int>>,
                              std::pair<const int&, int>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, volatile int&>, std::pair<volatile int&, int>>,
                              std::pair<volatile int&, int>>);
-static_assert(std::is_same_v<std::common_reference_t<std::pair<int&, const volatile int&>, std::pair<const volatile int&, int>>,
-                             std::pair<const volatile int&, int>>);
-static_assert(!has_type<std::common_reference_t<std::pair<const int&, volatile int&>,
-                        std::pair<volatile int&, const int&>>>);
+static_assert(
+    std::is_same_v<std::common_reference_t<std::pair<int&, const volatile int&>, std::pair<const volatile int&, int>>,
+                   std::pair<const volatile int&, int>>);
+static_assert(
+    !has_type<std::common_reference_t<std::pair<const int&, volatile int&>, std::pair<volatile int&, const int&>>>);
 
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int, X2>, std::pair<int, Y2>>, std::pair<int, Z2>>);
 static_assert(std::is_same_v<std::common_reference_t<std::pair<int, X2>, std::pair<int, Y2>>, std::pair<int, Z2>>);
diff --git a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
index cbc8da96be1b..5df6103d79e7 100644
--- a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
+++ b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
@@ -13,36 +13,38 @@
 #include <type_traits>
 #include <climits>
 #include <cassert>
+#include <stdexcept>
 
 #include "test_macros.h"
 
 template <std::size_t N>
 TEST_CONSTEXPR_CXX23 void test_to_ullong() {
-    const std::size_t M = sizeof(unsigned long long) * CHAR_BIT < N ? sizeof(unsigned long long) * CHAR_BIT : N;
-    const bool is_M_zero = std::integral_constant<bool, M == 0>::value; // avoid compiler warnings
-    const std::size_t X = is_M_zero ? sizeof(unsigned long long) * CHAR_BIT - 1 : sizeof(unsigned long long) * CHAR_BIT - M;
-    const unsigned long long max = is_M_zero ? 0 : (unsigned long long)(-1) >> X;
-    unsigned long long tests[] = {
-        0,
-        std::min<unsigned long long>(1, max),
-        std::min<unsigned long long>(2, max),
-        std::min<unsigned long long>(3, max),
-        std::min(max, max-3),
-        std::min(max, max-2),
-        std::min(max, max-1),
-        max
-    };
-    for (unsigned long long j : tests) {
-         std::bitset<N> v(j);
-        assert(j == v.to_ullong());
-    }
-    { // test values bigger than can fit into the bitset
-        const unsigned long long val = 0x55AAAAFFFFAAAA55ULL;
-        const bool canFit = N < sizeof(unsigned long long) * CHAR_BIT;
-        const unsigned long long mask = canFit ? (1ULL << (canFit ? N : 0)) - 1 : (unsigned long long)(-1); // avoid compiler warnings
-        std::bitset<N> v(val);
-        assert(v.to_ullong() == (val & mask)); // we shouldn't return bit patterns from outside the limits of the bitset.
-    }
+  const std::size_t M  = sizeof(unsigned long long) * CHAR_BIT < N ? sizeof(unsigned long long) * CHAR_BIT : N;
+  const bool is_M_zero = std::integral_constant < bool, M == 0 > ::value; // avoid compiler warnings
+  const std::size_t X =
+      is_M_zero ? sizeof(unsigned long long) * CHAR_BIT - 1 : sizeof(unsigned long long) * CHAR_BIT - M;
+  const unsigned long long max = is_M_zero ? 0 : (unsigned long long)(-1) >> X;
+  unsigned long long tests[]   = {
+      0,
+      std::min<unsigned long long>(1, max),
+      std::min<unsigned long long>(2, max),
+      std::min<unsigned long long>(3, max),
+      std::min(max, max - 3),
+      std::min(max, max - 2),
+      std::min(max, max - 1),
+      max};
+  for (unsigned long long j : tests) {
+    std::bitset<N> v(j);
+    assert(j == v.to_ullong());
+  }
+  { // test values bigger than can fit into the bitset
+    const unsigned long long val = 0x55AAAAFFFFAAAA55ULL;
+    const bool canFit            = N < sizeof(unsigned long long) * CHAR_BIT;
+    const unsigned long long mask =
+        canFit ? (1ULL << (canFit ? N : 0)) - 1 : (unsigned long long)(-1); // avoid compiler warnings
+    std::bitset<N> v(val);
+    assert(v.to_ullong() == (val & mask)); // we shouldn't return bit patterns from outside the limits of the bitset.
+  }
 }
 
 TEST_CONSTEXPR_CXX23 bool test() {
@@ -56,6 +58,22 @@ TEST_CONSTEXPR_CXX23 bool test() {
   test_to_ullong<65>();
   test_to_ullong<1000>();
 
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    // bitset has true bits beyond the size of unsigned long long
+    std::bitset<std::numeric_limits<unsigned long long>::digits + 1> q(0);
+    q.flip();
+    try {
+      q.to_ullong(); // throws
+      assert(false);
+    } catch (const std::overflow_error&) {
+      // expected
+    } catch (...) {
+      assert(false);
+    }
+  }
+#endif // TEST_HAS_NO_EXCEPTIONS
+
   return true;
 }
 
diff --git a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
index 17a5e7eb51aa..d943030a1628 100644
--- a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
+++ b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
@@ -14,37 +14,37 @@
 #include <limits>
 #include <climits>
 #include <cassert>
+#include <stdexcept>
 
 #include "test_macros.h"
 
 template <std::size_t N>
 TEST_CONSTEXPR_CXX23 void test_to_ulong() {
-    const std::size_t M = sizeof(unsigned long) * CHAR_BIT < N ? sizeof(unsigned long) * CHAR_BIT : N;
-    const bool is_M_zero = std::integral_constant<bool, M == 0>::value; // avoid compiler warnings
-    const std::size_t X = is_M_zero ? sizeof(unsigned long) * CHAR_BIT - 1 : sizeof(unsigned long) * CHAR_BIT - M;
-    const std::size_t max = is_M_zero ? 0 : std::size_t(std::numeric_limits<unsigned long>::max()) >> X;
-    std::size_t tests[] = {
-        0,
-        std::min<std::size_t>(1, max),
-        std::min<std::size_t>(2, max),
-        std::min<std::size_t>(3, max),
-        std::min(max, max-3),
-        std::min(max, max-2),
-        std::min(max, max-1),
-        max
-    };
-    for (std::size_t j : tests) {
-        std::bitset<N> v(j);
-        assert(j == v.to_ulong());
-    }
+  const std::size_t M   = sizeof(unsigned long) * CHAR_BIT < N ? sizeof(unsigned long) * CHAR_BIT : N;
+  const bool is_M_zero  = std::integral_constant < bool, M == 0 > ::value; // avoid compiler warnings
+  const std::size_t X   = is_M_zero ? sizeof(unsigned long) * CHAR_BIT - 1 : sizeof(unsigned long) * CHAR_BIT - M;
+  const std::size_t max = is_M_zero ? 0 : std::size_t(std::numeric_limits<unsigned long>::max()) >> X;
+  std::size_t tests[]   = {
+      0,
+      std::min<std::size_t>(1, max),
+      std::min<std::size_t>(2, max),
+      std::min<std::size_t>(3, max),
+      std::min(max, max - 3),
+      std::min(max, max - 2),
+      std::min(max, max - 1),
+      max};
+  for (std::size_t j : tests) {
+    std::bitset<N> v(j);
+    assert(j == v.to_ulong());
+  }
 
-    { // test values bigger than can fit into the bitset
-        const unsigned long val = 0x5AFFFFA5UL;
-        const bool canFit = N < sizeof(unsigned long) * CHAR_BIT;
-        const unsigned long mask = canFit ? (1UL << (canFit ? N : 0)) - 1 : (unsigned long)(-1); // avoid compiler warnings
-        std::bitset<N> v(val);
-        assert(v.to_ulong() == (val & mask)); // we shouldn't return bit patterns from outside the limits of the bitset.
-    }
+  { // test values bigger than can fit into the bitset
+    const unsigned long val  = 0x5AFFFFA5UL;
+    const bool canFit        = N < sizeof(unsigned long) * CHAR_BIT;
+    const unsigned long mask = canFit ? (1UL << (canFit ? N : 0)) - 1 : (unsigned long)(-1); // avoid compiler warnings
+    std::bitset<N> v(val);
+    assert(v.to_ulong() == (val & mask)); // we shouldn't return bit patterns from outside the limits of the bitset.
+  }
 }
 
 TEST_CONSTEXPR_CXX23 bool test() {
@@ -58,6 +58,22 @@ TEST_CONSTEXPR_CXX23 bool test() {
   test_to_ulong<65>();
   test_to_ulong<1000>();
 
+#ifndef TEST_HAS_NO_EXCEPTIONS
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    // bitset has true bits beyond the size of unsigned long
+    std::bitset<std::numeric_limits<unsigned long>::digits + 1> q(0);
+    q.flip();
+    try {
+      q.to_ulong(); // throws
+      assert(false);
+    } catch (const std::overflow_error&) {
+      // expected
+    } catch (...) {
+      assert(false);
+    }
+  }
+#endif // TEST_HAS_NO_EXCEPTIONS
+
   return true;
 }
 
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/eq.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/eq.pass.cpp
index a8de656313d4..779a89b163f0 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/eq.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/eq.pass.cpp
@@ -16,12 +16,33 @@
 
 // UNSUPPORTED: c++03
 
-#include <tuple>
-#include <string>
+#include <array>
 #include <cassert>
+#include <tuple>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
 
+#if TEST_STD_VER >= 26
+
+// Test SFINAE.
+
+static_assert(std::equality_comparable<std::tuple<EqualityComparable>>);
+static_assert(std::equality_comparable<std::tuple<EqualityComparable, EqualityComparable>>);
+
+static_assert(!std::equality_comparable<std::tuple<NonComparable>>);
+static_assert(!std::equality_comparable<std::tuple<EqualityComparable, NonComparable>>);
+static_assert(!std::equality_comparable<std::tuple<NonComparable, EqualityComparable>>);
+static_assert(!std::equality_comparable_with<std::tuple<EqualityComparable>, std::tuple<NonComparable>>);
+static_assert(!std::equality_comparable_with<std::tuple<NonComparable>, std::tuple<EqualityComparable>>);
+// Size mismatch.
+static_assert(
+    !std::equality_comparable_with<std::tuple<EqualityComparable>, std::tuple<EqualityComparable, EqualityComparable>>);
+static_assert(
+    !std::equality_comparable_with<std::tuple<EqualityComparable, EqualityComparable>, std::tuple<EqualityComparable>>);
+
+#endif
+
 int main(int, char**)
 {
     {
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/size_incompatible_comparison.verify.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/size_incompatible_comparison.verify.cpp
index 851f6fcd1fba..c05438f838f1 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/size_incompatible_comparison.verify.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.rel/size_incompatible_comparison.verify.cpp
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+// Disabled in C++26 and later because tuple comparison between different sizes is constrained since P2944R3.
+// UNSUPPORTED: std-at-least-cxx26
+
 // <tuple>
 
 // template <class... Types> class tuple;
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
index c472906c5ed7..226a3587a47f 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.spec/comparison.pass.cpp
@@ -21,28 +21,15 @@
 #include <cassert>
 #include <concepts>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
 
 #if TEST_STD_VER >= 26
 
 // Test SFINAE.
 
-struct EqualityComparable {
-  constexpr EqualityComparable(int value) : value_{value} {};
-
-  friend constexpr bool operator==(const EqualityComparable&, const EqualityComparable&) noexcept = default;
-
-  int value_;
-};
-
-static_assert(std::equality_comparable<EqualityComparable>);
-
 static_assert(std::equality_comparable<std::pair<EqualityComparable, EqualityComparable>>);
 
-struct NonComparable {};
-
-static_assert(!std::equality_comparable<NonComparable>);
-
 static_assert(!std::equality_comparable<std::pair<EqualityComparable, NonComparable>>);
 static_assert(!std::equality_comparable<std::pair<NonComparable, EqualityComparable>>);
 
diff --git a/libcxx/test/std/utilities/variant/variant.relops/relops.pass.cpp b/libcxx/test/std/utilities/variant/variant.relops/relops.pass.cpp
index c1a5b8e474a7..2c0070366268 100644
--- a/libcxx/test/std/utilities/variant/variant.relops/relops.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.relops/relops.pass.cpp
@@ -39,8 +39,57 @@
 #include <utility>
 #include <variant>
 
+#include "test_comparisons.h"
 #include "test_macros.h"
 
+#if TEST_STD_VER >= 26
+
+// Test SFINAE.
+
+// ==
+static_assert(HasOperatorEqual<std::variant<EqualityComparable>>);
+static_assert(HasOperatorEqual<std::variant<EqualityComparable, int, long>>);
+
+static_assert(!HasOperatorEqual<std::variant<NonComparable>>);
+static_assert(!HasOperatorEqual<std::variant<NonComparable, EqualityComparable>>);
+
+// >
+static_assert(HasOperatorGreaterThan<std::variant<ThreeWayComparable>>);
+static_assert(HasOperatorGreaterThan<std::variant<ThreeWayComparable, int, long>>);
+
+static_assert(!HasOperatorGreaterThan<std::variant<NonComparable>>);
+static_assert(!HasOperatorGreaterThan<std::variant<NonComparable, ThreeWayComparable>>);
+
+// >=
+static_assert(HasOperatorGreaterThanEqual<std::variant<ThreeWayComparable>>);
+static_assert(HasOperatorGreaterThanEqual<std::variant<ThreeWayComparable, int, long>>);
+
+static_assert(!HasOperatorGreaterThanEqual<std::variant<NonComparable>>);
+static_assert(!HasOperatorGreaterThanEqual<std::variant<NonComparable, ThreeWayComparable>>);
+
+// <
+static_assert(HasOperatorLessThan<std::variant<ThreeWayComparable>>);
+static_assert(HasOperatorLessThan<std::variant<ThreeWayComparable, int, long>>);
+
+static_assert(!HasOperatorLessThan<std::variant<NonComparable>>);
+static_assert(!HasOperatorLessThan<std::variant<NonComparable, ThreeWayComparable>>);
+
+// <=
+static_assert(HasOperatorLessThanEqual<std::variant<ThreeWayComparable>>);
+static_assert(HasOperatorLessThanEqual<std::variant<ThreeWayComparable, int, long>>);
+
+static_assert(!HasOperatorLessThanEqual<std::variant<NonComparable>>);
+static_assert(!HasOperatorLessThanEqual<std::variant<NonComparable, ThreeWayComparable>>);
+
+// !=
+static_assert(HasOperatorNotEqual<std::variant<EqualityComparable>>);
+static_assert(HasOperatorNotEqual<std::variant<EqualityComparable, int, long>>);
+
+static_assert(!HasOperatorNotEqual<std::variant<NonComparable>>);
+static_assert(!HasOperatorNotEqual<std::variant<NonComparable, EqualityComparable>>);
+
+#endif
+
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct MakeEmptyT {
   MakeEmptyT() = default;
diff --git a/libcxx/test/std/utilities/variant/variant.relops/relops_bool_conv.verify.cpp b/libcxx/test/std/utilities/variant/variant.relops/relops_bool_conv.verify.cpp
index 64248171d114..392a234e6a9b 100644
--- a/libcxx/test/std/utilities/variant/variant.relops/relops_bool_conv.verify.cpp
+++ b/libcxx/test/std/utilities/variant/variant.relops/relops_bool_conv.verify.cpp
@@ -41,7 +41,9 @@
 
 #include "test_macros.h"
 
-
+#if TEST_STD_VER >= 26
+// expected-no-diagnostics
+#else
 struct MyBoolExplicit {
   bool value;
   constexpr explicit MyBoolExplicit(bool v) : value(v) {}
@@ -70,8 +72,7 @@ inline constexpr MyBoolExplicit operator>=(const ComparesToMyBoolExplicit& LHS,
   return MyBoolExplicit(LHS.value >= RHS.value);
 }
 
-
-int main(int, char**) {
+void test() {
   using V = std::variant<int, ComparesToMyBoolExplicit>;
   V v1(42);
   V v2(101);
@@ -83,6 +84,6 @@ int main(int, char**) {
   (void)(v1 <= v2); // expected-note {{here}}
   (void)(v1 > v2); // expected-note {{here}}
   (void)(v1 >= v2); // expected-note {{here}}
-
-  return 0;
 }
+
+#endif
diff --git a/libcxx/test/support/concat_macros.h b/libcxx/test/support/concat_macros.h
index 50b656219739..efec4029f402 100644
--- a/libcxx/test/support/concat_macros.h
+++ b/libcxx/test/support/concat_macros.h
@@ -9,6 +9,7 @@
 #ifndef TEST_SUPPORT_CONCAT_MACROS_H
 #define TEST_SUPPORT_CONCAT_MACROS_H
 
+#include <algorithm>
 #include <cstdio>
 #include <string>
 #include <source_location>
diff --git a/libcxx/test/support/test_comparisons.h b/libcxx/test/support/test_comparisons.h
index d9729e0451b4..e37ab44828c7 100644
--- a/libcxx/test/support/test_comparisons.h
+++ b/libcxx/test/support/test_comparisons.h
@@ -271,12 +271,31 @@ struct PartialOrder {
 template <typename T1, typename T2 = T1>
 concept HasOperatorEqual = requires(T1 t1, T2 t2) { t1 == t2; };
 
+template <typename T1, typename T2 = T1>
+concept HasOperatorGreaterThan = requires(T1 t1, T2 t2) { t1 > t2; };
+
+template <typename T1, typename T2 = T1>
+concept HasOperatorGreaterThanEqual = requires(T1 t1, T2 t2) { t1 >= t2; };
+template <typename T1, typename T2 = T1>
+concept HasOperatorLessThan = requires(T1 t1, T2 t2) { t1 < t2; };
+
+template <typename T1, typename T2 = T1>
+concept HasOperatorLessThanEqual = requires(T1 t1, T2 t2) { t1 <= t2; };
+
+template <typename T1, typename T2 = T1>
+concept HasOperatorNotEqual = requires(T1 t1, T2 t2) { t1 != t2; };
+
 template <typename T1, typename T2 = T1>
 concept HasOperatorSpaceship = requires(T1 t1, T2 t2) { t1 <=> t2; };
 
 struct NonComparable {};
 static_assert(!std::equality_comparable<NonComparable>);
 static_assert(!HasOperatorEqual<NonComparable>);
+static_assert(!HasOperatorGreaterThan<NonComparable>);
+static_assert(!HasOperatorGreaterThanEqual<NonComparable>);
+static_assert(!HasOperatorLessThan<NonComparable>);
+static_assert(!HasOperatorLessThanEqual<NonComparable>);
+static_assert(!HasOperatorNotEqual<NonComparable>);
 static_assert(!HasOperatorSpaceship<NonComparable>);
 
 class EqualityComparable {
@@ -290,6 +309,28 @@ class EqualityComparable {
 };
 static_assert(std::equality_comparable<EqualityComparable>);
 static_assert(HasOperatorEqual<EqualityComparable>);
+static_assert(HasOperatorNotEqual<EqualityComparable>);
+
+class ThreeWayComparable {
+public:
+  constexpr ThreeWayComparable(int value) : value_{value} {};
+
+  friend constexpr bool operator==(const ThreeWayComparable&, const ThreeWayComparable&) noexcept = default;
+  friend constexpr std::strong_ordering
+  operator<=>(const ThreeWayComparable&, const ThreeWayComparable&) noexcept = default;
+
+private:
+  int value_;
+};
+static_assert(std::equality_comparable<ThreeWayComparable>);
+static_assert(std::three_way_comparable<ThreeWayComparable>);
+static_assert(HasOperatorEqual<ThreeWayComparable>);
+static_assert(HasOperatorGreaterThan<ThreeWayComparable>);
+static_assert(HasOperatorGreaterThanEqual<ThreeWayComparable>);
+static_assert(HasOperatorLessThan<ThreeWayComparable>);
+static_assert(HasOperatorLessThanEqual<ThreeWayComparable>);
+static_assert(HasOperatorNotEqual<ThreeWayComparable>);
+static_assert(HasOperatorSpaceship<ThreeWayComparable>);
 
 #endif // TEST_STD_VER >= 20
 
diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
index 897da8b769e6..8ec029f7dba2 100644
--- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
+++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
@@ -101,8 +101,10 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match
             // Looks at the common locations where headers store their data
             // * header
             // * __header/*.h
+            // * __header_dir/*.h // Used for transitioning from __header
             // * __fwd/header.h
             anyOf(isExpansionInFileMatching(("v1/__" + filename_ + "/").str()),
+                  isExpansionInFileMatching(("v1/__" + filename_ + "_dir/").str()),
                   isExpansionInFileMatching(extra_header_),
                   isExpansionInFileMatching(("v1/__fwd/" + filename_ + "\\.h$").str()),
                   isExpansionInFileMatching(("v1/" + filename_ + "$").str())),
diff --git a/libcxx/utils/ci/buildkite-pipeline-trigger.sh b/libcxx/utils/ci/buildkite-pipeline-trigger.sh
new file mode 100755
index 000000000000..4661cd54fad4
--- /dev/null
+++ b/libcxx/utils/ci/buildkite-pipeline-trigger.sh
@@ -0,0 +1,34 @@
+# ===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===----------------------------------------------------------------------===##
+
+#
+# This script determines whether the libc++ Buildkite pipeline should be triggered
+# on a change. This is required because Buildkite gets notified for every PR in the
+# LLVM monorepo, and we make it a no-op unless the libc++ pipeline needs to be triggered.
+#
+
+# Set by buildkite
+: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH:=}
+
+# Fetch origin to have an up to date merge base for the diff.
+git fetch origin
+# List of files affected by this commit
+: ${MODIFIED_FILES:=$(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD)}
+
+echo "Files modified:" >&2
+echo "$MODIFIED_FILES" >&2
+modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
+echo "Directories modified:" >&2
+echo "$modified_dirs" >&2
+
+# If libc++ or one of the runtimes directories changed, trigger the libc++ Buildkite pipeline.
+if echo "$modified_dirs" | grep -q -E "^(libcxx|libcxxabi|libunwind|runtimes|cmake)$"; then
+    buildkite-agent pipeline upload libcxx/utils/ci/buildkite-pipeline.yml
+else
+    echo "No Buildkite jobs to trigger"
+fi
diff --git a/libcxx/utils/gdb/libcxx/printers.py b/libcxx/utils/gdb/libcxx/printers.py
index 90bc54d987ee..1c8ef6d7feb9 100644
--- a/libcxx/utils/gdb/libcxx/printers.py
+++ b/libcxx/utils/gdb/libcxx/printers.py
@@ -65,16 +65,6 @@ def _remove_generics(typename):
     return match.group(1)
 
 
-def _cc_field(node):
-    """Previous versions of libcxx had inconsistent field naming naming. Handle
-    both types.
-    """
-    try:
-        return node["__value_"]["__cc_"]
-    except:
-        return node["__value_"]["__cc"]
-
-
 def _data_field(node):
     """Previous versions of libcxx had inconsistent field naming naming. Handle
     both types.
@@ -829,7 +819,7 @@ class StdUnorderedMapPrinter(AbstractUnorderedCollectionPrinter):
     """Print a std::unordered_(multi)map."""
 
     def _get_key_value(self, node):
-        key_value = _cc_field(node)
+        key_value = node["__value_"]
         return [key_value["first"], key_value["second"]]
 
     def display_hint(self):
@@ -885,7 +875,7 @@ def __init__(self, val):
         self._initialize(val, val["__i_"]["__node_"])
 
     def _get_key_value(self):
-        key_value = _cc_field(self.node)
+        key_value = self.node["__value_"]
         return [key_value["first"], key_value["second"]]
 
     def display_hint(self):
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index edd7b124a1fb..18eb8a862374 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -313,6 +313,16 @@ def add_version_header(tc):
             "values": {"c++17": 201603},
             "headers": ["algorithm"],
         },
+        {
+            "name": "__cpp_lib_common_reference",
+            "values": {"c++20": 202302},
+            "headers": ["type_traits"],
+        },
+        {
+            "name": "__cpp_lib_common_reference_wrapper",
+            "values": {"c++20": 202302},
+            "headers": ["functional"],
+        },
         {
             "name": "__cpp_lib_complex_udls",
             "values": {"c++14": 201309},
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 74746e37d3bc..0cb81546665d 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -134,11 +134,6 @@ def _mingwSupportsModules(cfg):
     Feature(name=lambda cfg: "msvc-{}".format(*_msvcVersion(cfg)), when=_isMSVC),
     Feature(name=lambda cfg: "msvc-{}.{}".format(*_msvcVersion(cfg)), when=_isMSVC),
 
-    Feature(
-        name="thread-safety",
-        when=lambda cfg: hasCompileFlag(cfg, "-Werror=thread-safety"),
-        actions=[AddCompileFlag("-Werror=thread-safety")],
-    ),
     Feature(
         name="diagnose-if-support",
         when=lambda cfg: hasCompileFlag(cfg, "-Wuser-defined-warnings"),
@@ -336,12 +331,22 @@ def _mingwSupportsModules(cfg):
         or platform.system().lower().startswith("aix")
         # Avoid building on platforms that don't support modules properly.
         or not hasCompileFlag(cfg, "-Wno-reserved-module-identifier")
-        or not sourceBuilds(
-            cfg,
-            """
+        # older versions don't support extern "C++", newer versions don't support main in named module.
+        or not (
+            sourceBuilds(
+                cfg,
+                """
+            export module test;
+            extern "C++" int main(int, char**) { return 0; }
+          """,
+            )
+            or sourceBuilds(
+                cfg,
+                """
             export module test;
             int main(int, char**) { return 0; }
           """,
+            )
         ),
     ),
     # The time zone validation tests compare the output of zdump against the
diff --git a/libcxx/utils/libcxx/test/params.py b/libcxx/utils/libcxx/test/params.py
index fc34009d0a55..adfb2a9f6950 100644
--- a/libcxx/utils/libcxx/test/params.py
+++ b/libcxx/utils/libcxx/test/params.py
@@ -170,7 +170,6 @@ def getSuitableClangTidy(cfg):
             [
                 AddFeature("target={}".format(triple)),
                 AddFlagIfSupported("--target={}".format(triple)),
-                AddSubstitution("%{triple}", triple),
             ],
         ),
     ),
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index 3e2f80b81845..975bd75cbb9b 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -192,6 +192,8 @@ if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
   set(LIBCXXABI_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LIBCXXABI_TARGET_SUBDIR})
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX}/${LIBCXXABI_TARGET_SUBDIR} CACHE STRING
       "Path where built libc++abi libraries should be installed.")
+  set(LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR "${CMAKE_INSTALL_INCLUDEDIR}/${LIBCXXABI_TARGET_SUBDIR}/c++/v1" CACHE STRING
+      "Path where target-specific libc++abi headers should be installed.")
   unset(LIBCXXABI_TARGET_SUBDIR)
 else()
   if(LLVM_LIBRARY_OUTPUT_INTDIR)
@@ -203,6 +205,8 @@ else()
   endif()
   set(LIBCXXABI_INSTALL_LIBRARY_DIR lib${LIBCXXABI_LIBDIR_SUFFIX} CACHE STRING
       "Path where built libc++abi libraries should be installed.")
+  set(LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR "${LIBCXXABI_INSTALL_INCLUDE_DIR}" CACHE STRING
+      "Path where target-specific libc++abi headers should be installed.")
 endif()
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIBCXXABI_LIBRARY_DIR})
diff --git a/libcxxabi/src/demangle/ItaniumDemangle.h b/libcxxabi/src/demangle/ItaniumDemangle.h
index 4e7f92dd1991..b306b2013445 100644
--- a/libcxxabi/src/demangle/ItaniumDemangle.h
+++ b/libcxxabi/src/demangle/ItaniumDemangle.h
@@ -4468,7 +4468,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() {
         return nullptr;
       if (!consumeIf('_'))
         return nullptr;
-      return make<BitIntType>(Size, Signed);
+      // The front end expects this to be available for Substitution
+      Result = make<BitIntType>(Size, Signed);
+      break;
     }
     //                ::= Di   # char32_t
     case 'i':
diff --git a/libcxxabi/test/DemangleTestCases.inc b/libcxxabi/test/DemangleTestCases.inc
index 1e3f7459deaa..2721d2aa5504 100644
--- a/libcxxabi/test/DemangleTestCases.inc
+++ b/libcxxabi/test/DemangleTestCases.inc
@@ -6,6 +6,7 @@
 {"_Z1fDU10_", "f(unsigned _BitInt(10))"},
 {"_Z1fIfEvDUstPT__", "void f<float>(unsigned _BitInt(sizeof (float*)))"},
 {"_Z1fIiEvDBstPT__", "void f<int>(_BitInt(sizeof (int*)))"},
+{"_Z6myfuncRDB8_S0_", "myfunc(_BitInt(8)&, _BitInt(8)&)"},
 {"_Z4testI1A1BE1Cv", "C test<A, B>()"},
 {"_Z4testI1A1BET0_T_S3_", "B test<A, B>(A, A)"},
 {"_ZN1SgtEi", "S::operator>(int)"},
diff --git a/libcxxabi/test/configs/cmake-bridge.cfg.in b/libcxxabi/test/configs/cmake-bridge.cfg.in
index 1bdba32405dd..f81dd8afb109 100644
--- a/libcxxabi/test/configs/cmake-bridge.cfg.in
+++ b/libcxxabi/test/configs/cmake-bridge.cfg.in
@@ -32,7 +32,7 @@ config.substitutions.append(('%{llvm}', '@LLVM_MAIN_SRC_DIR@'))
 config.substitutions.append(('%{install-prefix}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@'))
 config.substitutions.append(('%{include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/include'))
 config.substitutions.append(('%{cxx-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_DIR@'))
-config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/include/%{triple}/c++/v1'))
+config.substitutions.append(('%{cxx-target-include}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_INCLUDE_TARGET_DIR@'))
 config.substitutions.append(('%{lib}', '@LIBCXXABI_TESTING_INSTALL_PREFIX@/@LIBCXXABI_INSTALL_LIBRARY_DIR@'))
 config.substitutions.append(('%{benchmark_flags}', ''))
 
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 473943a23942..79b63e5b7236 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -307,6 +307,7 @@ struct Configuration {
   bool warnDebugInfoUnusable = true;
   bool warnLongSectionNames = true;
   bool warnStdcallFixup = true;
+  bool warnExportedDllMain = true;
   bool incremental = true;
   bool integrityCheck = false;
   bool killAt = false;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index f3240b22a144..283aeed1a19c 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -706,11 +706,12 @@ void LinkerDriver::detectWinSysRoot(const opt::InputArgList &Args) {
     if (A->getOption().getID() == OPT_winsysroot)
       path::append(diaPath, "DIA SDK");
   }
-  useWinSysRootLibPath = Args.hasArg(OPT_lldignoreenv) ||
-                         !Process::GetEnv("LIB") ||
-                         Args.getLastArg(OPT_vctoolsdir, OPT_winsysroot);
-  if (Args.hasArg(OPT_lldignoreenv) || !Process::GetEnv("LIB") ||
-      Args.getLastArg(OPT_winsdkdir, OPT_winsysroot)) {
+  useWinSysRootLibPath = !Process::GetEnv("LIB") ||
+                         Args.hasArg(OPT_lldignoreenv, OPT_vctoolsdir,
+                                     OPT_vctoolsversion, OPT_winsysroot);
+  if (!Process::GetEnv("LIB") ||
+      Args.hasArg(OPT_lldignoreenv, OPT_winsdkdir, OPT_winsdkversion,
+                  OPT_winsysroot)) {
     std::optional<StringRef> WinSdkDir, WinSdkVersion;
     if (auto *A = Args.getLastArg(OPT_winsdkdir))
       WinSdkDir = A->getValue();
@@ -1620,7 +1621,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
       // installations when operating in mingw mode. (This also makes LLD ignore
       // winsysroot and vctoolsdir arguments.)
       detectWinSysRoot(args);
-      if (!args.hasArg(OPT_lldignoreenv) && !args.hasArg(OPT_winsysroot))
+      if (!args.hasArg(OPT_lldignoreenv, OPT_winsysroot, OPT_vctoolsdir,
+                       OPT_vctoolsversion, OPT_winsdkdir, OPT_winsdkversion))
         addLibSearchPaths();
     } else {
       if (args.hasArg(OPT_vctoolsdir, OPT_winsysroot))
@@ -1641,6 +1643,8 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
         config->warnLocallyDefinedImported = false;
       else if (s == "longsections")
         config->warnLongSectionNames = false;
+      else if (s == "exporteddllmain")
+        config->warnExportedDllMain = false;
       // Other warning numbers are ignored.
     }
   }
diff --git a/lld/COFF/InputFiles.cpp b/lld/COFF/InputFiles.cpp
index 760ce00dda0d..0b7dbea8cdd9 100644
--- a/lld/COFF/InputFiles.cpp
+++ b/lld/COFF/InputFiles.cpp
@@ -114,6 +114,35 @@ static coff_symbol_generic *cloneSymbol(COFFSymbolRef sym) {
   }
 }
 
+// Skip importing DllMain thunks from import libraries.
+static bool fixupDllMain(COFFLinkerContext &ctx, llvm::object::Archive *file,
+                         const Archive::Symbol &sym, bool &skipDllMain) {
+  if (skipDllMain)
+    return true;
+  const Archive::Child &c =
+      CHECK(sym.getMember(), file->getFileName() +
+                                 ": could not get the member for symbol " +
+                                 toCOFFString(ctx, sym));
+  MemoryBufferRef mb =
+      CHECK(c.getMemoryBufferRef(),
+            file->getFileName() +
+                ": could not get the buffer for a child buffer of the archive");
+  if (identify_magic(mb.getBuffer()) == file_magic::coff_import_library) {
+    if (ctx.config.warnExportedDllMain) {
+      // We won't place DllMain symbols in the symbol table if they are
+      // coming from a import library. This message can be ignored with the flag
+      // '/ignore:exporteddllmain'
+      Warn(ctx)
+          << file->getFileName()
+          << ": skipping exported DllMain symbol [exporteddllmain]\nNOTE: this "
+             "might be a mistake when the DLL/library was produced.";
+    }
+    skipDllMain = true;
+    return true;
+  }
+  return false;
+}
+
 ArchiveFile::ArchiveFile(COFFLinkerContext &ctx, MemoryBufferRef m)
     : InputFile(ctx.symtab, ArchiveKind, m) {}
 
@@ -176,8 +205,17 @@ void ArchiveFile::parse() {
   }
 
   // Read the symbol table to construct Lazy objects.
-  for (const Archive::Symbol &sym : file->symbols())
+  bool skipDllMain = false;
+  for (const Archive::Symbol &sym : file->symbols()) {
+    // If the DllMain symbol was exported by mistake, skip importing it
+    // otherwise we might end up with a import thunk in the final binary which
+    // is wrong.
+    if (sym.getName() == "__imp_DllMain" || sym.getName() == "DllMain") {
+      if (fixupDllMain(ctx, file.get(), sym, skipDllMain))
+        continue;
+    }
     archiveSymtab->addLazyArchive(this, sym);
+  }
 }
 
 // Returns a buffer pointing to a member file containing a given symbol.
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 8a225ed103ee..1812f2af419d 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -11,6 +11,7 @@
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "TargetImpl.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Endian.h"
 
@@ -82,6 +83,7 @@ class AArch64 : public TargetInfo {
                 uint64_t val) const override;
   RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
+  void applyBranchToBranchOpt() const override;
 
 private:
   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
@@ -974,6 +976,62 @@ void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   }
 }
 
+static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
+                                                        Relocation &r) {
+  // Identify a control transfer relocation for the branch-to-branch
+  // optimization. A "control transfer relocation" means a B or BL
+  // target but it also includes relative vtable relocations for example.
+  //
+  // We require the relocation type to be JUMP26, CALL26 or PLT32. With a
+  // relocation type of PLT32 the value may be assumed to be used for branching
+  // directly to the symbol and the addend is only used to produce the relocated
+  // value (hence the effective addend is always 0). This is because if a PLT is
+  // needed the addend will be added to the address of the PLT, and it doesn't
+  // make sense to branch into the middle of a PLT. For example, relative vtable
+  // relocations use PLT32 and 0 or a positive value as the addend but still are
+  // used to branch to the symbol.
+  //
+  // With JUMP26 or CALL26 the only reasonable interpretation of a non-zero
+  // addend is that we are branching to symbol+addend so that becomes the
+  // effective addend.
+  if (r.type == R_AARCH64_PLT32)
+    return 0;
+  if (r.type == R_AARCH64_JUMP26 || r.type == R_AARCH64_CALL26)
+    return r.addend;
+  return std::nullopt;
+}
+
+static std::pair<Relocation *, uint64_t>
+getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
+  auto *i = llvm::partition_point(
+      is.relocations, [&](Relocation &r) { return r.offset < offset; });
+  if (i != is.relocations.end() && i->offset == offset &&
+      i->type == R_AARCH64_JUMP26) {
+    return {i, i->addend};
+  }
+  return {nullptr, 0};
+}
+
+static void redirectControlTransferRelocations(Relocation &r1,
+                                               const Relocation &r2) {
+  r1.expr = r2.expr;
+  r1.sym = r2.sym;
+  // With PLT32 we must respect the original addend as that affects the value's
+  // interpretation. With the other relocation types the original addend is
+  // irrelevant because it referred to an offset within the original target
+  // section so we overwrite it.
+  if (r1.type == R_AARCH64_PLT32)
+    r1.addend += r2.addend;
+  else
+    r1.addend = r2.addend;
+}
+
+void AArch64::applyBranchToBranchOpt() const {
+  applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
+                             getBranchInfoAtTarget,
+                             redirectControlTransferRelocations);
+}
+
 // AArch64 may use security features in variant PLT sequences. These are:
 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index 3c4ad53af1b5..fe804cbb0e69 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -39,8 +39,13 @@ class LoongArch final : public TargetInfo {
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
   bool relaxOnce(int pass) const override;
+  RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
   void finalizeRelax(int passes) const override;
+
+private:
+  void tlsdescToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
+  void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
 };
 } // end anonymous namespace
 
@@ -58,6 +63,7 @@ enum Op {
   LU12I_W = 0x14000000,
   PCADDI = 0x18000000,
   PCADDU12I = 0x1c000000,
+  PCALAU12I = 0x1a000000,
   LD_W = 0x28800000,
   LD_D = 0x28c00000,
   JIRL = 0x4c000000,
@@ -69,6 +75,7 @@ enum Reg {
   R_ZERO = 0,
   R_RA = 1,
   R_TP = 2,
+  R_A0 = 4,
   R_T0 = 12,
   R_T1 = 13,
   R_T2 = 14,
@@ -961,7 +968,8 @@ static bool relax(Ctx &ctx, InputSection &sec) {
     case R_LARCH_TLS_LD_PC_HI20:
     case R_LARCH_TLS_DESC_PC_HI20:
       // The overflow check for i+2 will be carried out in isPairRelaxable.
-      if (isPairRelaxable(relocs, i))
+      if (r.expr != RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC &&
+          r.expr != R_RELAX_TLS_GD_TO_LE && isPairRelaxable(relocs, i))
         relaxPCHi20Lo12(ctx, sec, i, loc, r, relocs[i + 2], remove);
       break;
     case R_LARCH_CALL36:
@@ -1046,6 +1054,104 @@ static void tlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) {
   }
 }
 
+// Convert TLSDESC GD/LD to IE.
+// In normal or medium code model, there are two forms of code sequences:
+//  * pcalau12i  $a0, %desc_pc_hi20(sym_desc)
+//  * addi.d     $a0, $a0, %desc_pc_lo12(sym_desc)
+//  * ld.d       $ra, $a0, %desc_ld(sym_desc)
+//  * jirl       $ra, $ra, %desc_call(sym_desc)
+//  ------
+//  * pcaddi $a0, %desc_pcrel_20(a)
+//  * load $ra, $a0, %desc_ld(a)
+//  * jirl $ra, $ra, %desc_call(a)
+//
+// The code sequence obtained is as follows:
+//  * pcalau12i $a0, %ie_pc_hi20(sym_ie)
+//  * ld.[wd]   $a0, $a0, %ie_pc_lo12(sym_ie)
+//
+// Simplicity, whether tlsdescToIe or tlsdescToLe, we always tend to convert the
+// preceding instructions to NOPs, due to both forms of code sequence
+// (corresponding to relocation combinations:
+// R_LARCH_TLS_DESC_PC_HI20+R_LARCH_TLS_DESC_PC_LO12 and
+// R_LARCH_TLS_DESC_PCREL20_S2) have same process.
+//
+// When relaxation enables, redundant NOPs can be removed.
+void LoongArch::tlsdescToIe(uint8_t *loc, const Relocation &rel,
+                            uint64_t val) const {
+  switch (rel.type) {
+  case R_LARCH_TLS_DESC_PC_HI20:
+  case R_LARCH_TLS_DESC_PC_LO12:
+  case R_LARCH_TLS_DESC_PCREL20_S2:
+    write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    break;
+  case R_LARCH_TLS_DESC_LD:
+    write32le(loc, insn(PCALAU12I, R_A0, 0, 0)); // pcalau12i $a0, %ie_pc_hi20
+    relocateNoSym(loc, R_LARCH_TLS_IE_PC_HI20, val);
+    break;
+  case R_LARCH_TLS_DESC_CALL:
+    write32le(loc, insn(ctx.arg.is64 ? LD_D : LD_W, R_A0, R_A0,
+                        0)); // ld.[wd] $a0, $a0, %ie_pc_lo12
+    relocateNoSym(loc, R_LARCH_TLS_IE_PC_LO12, val);
+    break;
+  default:
+    llvm_unreachable("unsupported relocation for TLSDESC to IE");
+  }
+}
+
+// Convert TLSDESC GD/LD to LE.
+// The code sequence obtained in the normal or medium code model is as follows:
+//  * lu12i.w   $a0, %le_hi20(sym)      # le_hi20 != 0, otherwise NOP
+//  * ori       $a0, src, %le_lo12(sym) # le_hi20 != 0, src = $a0,
+//                                      # otherwise,    src = $zero
+// See the comment in tlsdescToIe for detailed information.
+void LoongArch::tlsdescToLe(uint8_t *loc, const Relocation &rel,
+                            uint64_t val) const {
+  assert(isInt<32>(val) &&
+         "val exceeds the range of medium code model in tlsdescToLe");
+
+  bool isUInt12 = isUInt<12>(val);
+  switch (rel.type) {
+  case R_LARCH_TLS_DESC_PC_HI20:
+  case R_LARCH_TLS_DESC_PC_LO12:
+  case R_LARCH_TLS_DESC_PCREL20_S2:
+    write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    break;
+  case R_LARCH_TLS_DESC_LD:
+    if (isUInt12)
+      write32le(loc, insn(ANDI, R_ZERO, R_ZERO, 0)); // nop
+    else
+      write32le(loc, insn(LU12I_W, R_A0, extractBits(val, 31, 12),
+                          0)); // lu12i.w $a0, %le_hi20
+    break;
+  case R_LARCH_TLS_DESC_CALL:
+    if (isUInt12)
+      write32le(loc, insn(ORI, R_A0, R_ZERO, val)); // ori $a0, $zero, %le_lo12
+    else
+      write32le(loc,
+                insn(ORI, R_A0, R_A0, lo12(val))); // ori $a0, $a0, %le_lo12
+    break;
+  default:
+    llvm_unreachable("unsupported relocation for TLSDESC to LE");
+  }
+}
+
+// During TLSDESC GD_TO_IE, the converted code sequence always includes an
+// instruction related to the Lo12 relocation (ld.[wd]). To obtain correct val
+// in `getRelocTargetVA`, expr of this instruction should be adjusted to
+// R_RELAX_TLS_GD_TO_IE_ABS, while expr of other instructions related to the
+// Hi20 relocation (pcalau12i) should be adjusted to
+// RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC. Specifically, in the normal or
+// medium code model, the instruction with relocation R_LARCH_TLS_DESC_CALL is
+// the candidate of Lo12 relocation.
+RelExpr LoongArch::adjustTlsExpr(RelType type, RelExpr expr) const {
+  if (expr == R_RELAX_TLS_GD_TO_IE) {
+    if (type != R_LARCH_TLS_DESC_CALL)
+      return RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC;
+    return R_RELAX_TLS_GD_TO_IE_ABS;
+  }
+  return expr;
+}
+
 void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   const unsigned bits = ctx.arg.is64 ? 64 : 32;
   uint64_t secAddr = sec.getOutputSection()->addr;
@@ -1074,7 +1180,7 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
         //  * i+2 -- R_LARCH_TLS_IE64_PC_LO20
         //  * i+3 -- R_LARCH_TLS_IE64_PC_HI12
         isExtreme =
-            (i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_IE64_PC_LO20);
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_IE64_PC_LO20;
       }
       if (isExtreme) {
         rel.expr = getRelExpr(rel.type, *rel.sym, loc);
@@ -1088,6 +1194,47 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
         tlsIeToLe(loc, rel, val);
       }
       continue;
+    case RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC:
+      if (rel.type == R_LARCH_TLS_DESC_PC_HI20) {
+        // LoongArch does not support TLSDESC GD/LD to LE/IE optimization in the
+        // extreme code model. In these cases, the relocs are as follows:
+        //
+        //  * i   -- R_LARCH_TLS_DESC_PC_HI20
+        //  * i+1 -- R_LARCH_TLS_DESC_PC_LO12
+        //  * i+2 -- R_LARCH_TLS_DESC64_PC_LO20
+        //  * i+3 -- R_LARCH_TLS_DESC64_PC_HI12
+        isExtreme =
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_DESC64_PC_LO20;
+      }
+      [[fallthrough]];
+    case R_RELAX_TLS_GD_TO_IE_ABS:
+      if (isExtreme) {
+        if (rel.type == R_LARCH_TLS_DESC_CALL)
+          continue;
+        rel.expr = getRelExpr(rel.type, *rel.sym, loc);
+        val = SignExtend64(sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset),
+                           bits);
+        relocateNoSym(loc, rel.type, val);
+      } else {
+        tlsdescToIe(loc, rel, val);
+      }
+      continue;
+    case R_RELAX_TLS_GD_TO_LE:
+      if (rel.type == R_LARCH_TLS_DESC_PC_HI20) {
+        isExtreme =
+            i + 2 < size && relocs[i + 2].type == R_LARCH_TLS_DESC64_PC_LO20;
+      }
+      if (isExtreme) {
+        if (rel.type == R_LARCH_TLS_DESC_CALL)
+          continue;
+        rel.expr = getRelExpr(rel.type, *rel.sym, loc);
+        val = SignExtend64(sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset),
+                           bits);
+        relocateNoSym(loc, rel.type, val);
+      } else {
+        tlsdescToLe(loc, rel, val);
+      }
+      continue;
     default:
       break;
     }
diff --git a/lld/ELF/Arch/TargetImpl.h b/lld/ELF/Arch/TargetImpl.h
new file mode 100644
index 000000000000..90a2b53cbb2a
--- /dev/null
+++ b/lld/ELF/Arch/TargetImpl.h
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_ELF_ARCH_TARGETIMPL_H
+#define LLD_ELF_ARCH_TARGETIMPL_H
+
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "Relocations.h"
+#include "Symbols.h"
+#include "llvm/BinaryFormat/ELF.h"
+
+namespace lld::elf {
+
+// getControlTransferAddend: If this relocation is used for control transfer
+// instructions (e.g. branch, branch-link or call) or code references (e.g.
+// virtual function pointers) and indicates an address-insignificant reference,
+// return the effective addend for the relocation, otherwise return
+// std::nullopt. The effective addend for a relocation is the addend that is
+// used to determine its branch destination.
+//
+// getBranchInfoAtTarget: If a control transfer relocation referring to
+// is+offset directly transfers control to a relocated branch instruction in the
+// specified section, return the relocation for the branch target as well as its
+// effective addend (see above). Otherwise return {nullptr, 0}.
+//
+// redirectControlTransferRelocations: Given r1, a relocation for which
+// getControlTransferAddend() returned a value, and r2, a relocation returned by
+// getBranchInfo(), modify r1 so that it branches directly to the target of r2.
+template <typename GetControlTransferAddend, typename GetBranchInfoAtTarget,
+          typename RedirectControlTransferRelocations>
+inline void applyBranchToBranchOptImpl(
+    Ctx &ctx, GetControlTransferAddend getControlTransferAddend,
+    GetBranchInfoAtTarget getBranchInfoAtTarget,
+    RedirectControlTransferRelocations redirectControlTransferRelocations) {
+  // Needs to run serially because it writes to the relocations array as well as
+  // reading relocations of other sections.
+  for (ELFFileBase *f : ctx.objectFiles) {
+    auto getRelocBranchInfo =
+        [&getBranchInfoAtTarget](
+            Relocation &r,
+            uint64_t addend) -> std::pair<Relocation *, uint64_t> {
+      auto *target = dyn_cast_or_null<Defined>(r.sym);
+      // We don't allow preemptible symbols or ifuncs (may go somewhere else),
+      // absolute symbols (runtime behavior unknown), non-executable or writable
+      // memory (ditto) or non-regular sections (no section data).
+      if (!target || target->isPreemptible || target->isGnuIFunc() ||
+          !target->section ||
+          !(target->section->flags & llvm::ELF::SHF_EXECINSTR) ||
+          (target->section->flags & llvm::ELF::SHF_WRITE) ||
+          target->section->kind() != SectionBase::Regular)
+        return {nullptr, 0};
+      return getBranchInfoAtTarget(*cast<InputSection>(target->section),
+                                   target->value + addend);
+    };
+    for (InputSectionBase *sb : f->getSections()) {
+      auto *s = dyn_cast_or_null<InputSection>(sb);
+      if (!s)
+        continue;
+      for (Relocation &r : s->relocations) {
+        std::optional<uint64_t> addend = getControlTransferAddend(*s, r);
+        if (!addend)
+          continue;
+        std::pair<Relocation *, uint64_t> targetAndAddend =
+            getRelocBranchInfo(r, *addend);
+        if (!targetAndAddend.first)
+          continue;
+        // Avoid getting stuck in an infinite loop if we encounter a branch
+        // that (possibly indirectly) branches to itself. It is unlikely
+        // that more than 5 iterations will ever be needed in practice.
+        size_t iterations = 5;
+        while (iterations--) {
+          std::pair<Relocation *, uint64_t> nextTargetAndAddend =
+              getRelocBranchInfo(*targetAndAddend.first,
+                                 targetAndAddend.second);
+          if (!nextTargetAndAddend.first)
+            break;
+          targetAndAddend = nextTargetAndAddend;
+        }
+        redirectControlTransferRelocations(r, *targetAndAddend.first);
+      }
+    }
+  }
+}
+
+} // namespace lld::elf
+
+#endif
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 974da4d96320..488f4803b2cb 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -11,6 +11,7 @@
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "TargetImpl.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MathExtras.h"
@@ -49,6 +50,7 @@ class X86_64 : public TargetInfo {
   bool deleteFallThruJmpInsn(InputSection &is, InputFile *file,
                              InputSection *nextIS) const override;
   bool relaxOnce(int pass) const override;
+  void applyBranchToBranchOpt() const override;
 
 private:
   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
@@ -318,6 +320,8 @@ bool X86_64::deleteFallThruJmpInsn(InputSection &is, InputFile *file,
 bool X86_64::relaxOnce(int pass) const {
   uint64_t minVA = UINT64_MAX, maxVA = 0;
   for (OutputSection *osec : ctx.outputSections) {
+    if (!(osec->flags & SHF_ALLOC))
+      continue;
     minVA = std::min(minVA, osec->addr);
     maxVA = std::max(maxVA, osec->addr + osec->size);
   }
@@ -1161,6 +1165,72 @@ void X86_64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   }
 }
 
+static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
+                                                        Relocation &r) {
+  // Identify a control transfer relocation for the branch-to-branch
+  // optimization. A "control transfer relocation" usually means a CALL or JMP
+  // target but it also includes relative vtable relocations for example.
+  //
+  // We require the relocation type to be PLT32. With a relocation type of PLT32
+  // the value may be assumed to be used for branching directly to the symbol
+  // and the addend is only used to produce the relocated value (hence the
+  // effective addend is always 0). This is because if a PLT is needed the
+  // addend will be added to the address of the PLT, and it doesn't make sense
+  // to branch into the middle of a PLT. For example, relative vtable
+  // relocations use PLT32 and 0 or a positive value as the addend but still are
+  // used to branch to the symbol.
+  //
+  // STT_SECTION symbols are a special case on x86 because the LLVM assembler
+  // uses them for branches to local symbols which are assembled as referring to
+  // the section symbol with the addend equal to the symbol value - 4.
+  if (r.type == R_X86_64_PLT32) {
+    if (r.sym->isSection())
+      return r.addend + 4;
+    return 0;
+  }
+  return std::nullopt;
+}
+
+static std::pair<Relocation *, uint64_t>
+getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
+  auto content = is.contentMaybeDecompress();
+  if (content.size() > offset && content[offset] == 0xe9) { // JMP immediate
+    auto *i = llvm::partition_point(
+        is.relocations, [&](Relocation &r) { return r.offset < offset + 1; });
+    // Unlike with getControlTransferAddend() it is valid to accept a PC32
+    // relocation here because we know that this is actually a JMP and not some
+    // other reference, so the interpretation is that we add 4 to the addend and
+    // use that as the effective addend.
+    if (i != is.relocations.end() && i->offset == offset + 1 &&
+        (i->type == R_X86_64_PC32 || i->type == R_X86_64_PLT32)) {
+      return {i, i->addend + 4};
+    }
+  }
+  return {nullptr, 0};
+}
+
+static void redirectControlTransferRelocations(Relocation &r1,
+                                               const Relocation &r2) {
+  // The isSection() check handles the STT_SECTION case described above.
+  // In that case the original addend is irrelevant because it referred to an
+  // offset within the original target section so we overwrite it.
+  //
+  // The +4 is here to compensate for r2.addend which will likely be -4,
+  // but may also be addend-4 in case of a PC32 branch to symbol+addend.
+  if (r1.sym->isSection())
+    r1.addend = r2.addend;
+  else
+    r1.addend += r2.addend + 4;
+  r1.expr = r2.expr;
+  r1.sym = r2.sym;
+}
+
+void X86_64::applyBranchToBranchOpt() const {
+  applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
+                             getBranchInfoAtTarget,
+                             redirectControlTransferRelocations);
+}
+
 // If Intel Indirect Branch Tracking is enabled, we have to emit special PLT
 // entries containing endbr64 instructions. A PLT entry will be split into two
 // parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt).
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 2b72d54ba410..4c771e47a0e7 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -275,6 +275,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;
@@ -302,6 +306,7 @@ struct Config {
   bool bpFunctionOrderForCompression = false;
   bool bpDataOrderForCompression = false;
   bool bpVerboseSectionOrderer = false;
+  bool branchToBranch = false;
   bool checkSections;
   bool checkDynamicRelocs;
   std::optional<llvm::DebugCompressionType> compressDebugSections;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 0d47d39bb5d2..56e49eacd8ee 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1396,6 +1396,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =
@@ -1657,6 +1662,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   ctx.arg.zWxneeded = hasZOption(args, "wxneeded");
   setUnresolvedSymbolPolicy(ctx, args);
   ctx.arg.power10Stubs = args.getLastArgValue(OPT_power10_stubs_eq) != "no";
+  ctx.arg.branchToBranch = args.hasFlag(
+      OPT_branch_to_branch, OPT_no_branch_to_branch, ctx.arg.optimize >= 2);
 
   if (opt::Arg *arg = args.getLastArg(OPT_eb, OPT_el)) {
     if (arg->getOption().matches(OPT_eb))
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 0ce0f08d0387..68e3feb1bd04 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -430,8 +430,9 @@ InputSectionBase *InputSection::getRelocatedSection() const {
 
 template <class ELFT, class RelTy>
 void InputSection::copyRelocations(Ctx &ctx, uint8_t *buf) {
-  if (ctx.arg.relax && !ctx.arg.relocatable &&
-      (ctx.arg.emachine == EM_RISCV || ctx.arg.emachine == EM_LOONGARCH)) {
+  bool linkerRelax =
+      ctx.arg.relax && is_contained({EM_RISCV, EM_LOONGARCH}, ctx.arg.emachine);
+  if (!ctx.arg.relocatable && (linkerRelax || ctx.arg.branchToBranch)) {
     // On LoongArch and RISC-V, relaxation might change relocations: copy
     // from internal ones that are updated by relaxation.
     InputSectionBase *sec = getRelocatedSection();
@@ -829,6 +830,7 @@ uint64_t InputSectionBase::getRelocTargetVA(Ctx &ctx, const Relocation &r,
   case R_GOTPLT_PC:
     return r.sym->getGotPltVA(ctx) + a - p;
   case RE_LOONGARCH_GOT_PAGE_PC:
+  case RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC:
     if (r.sym->hasFlag(NEEDS_TLSGD))
       return getLoongArchPageDelta(ctx.in.got->getGlobalDynAddr(*r.sym) + a, p,
                                    r.type);
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index ccb8c79a84b0..cb158362e4c5 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -182,6 +182,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index b4d79f2fc2c3..1a1450117c7e 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -59,6 +59,10 @@ def build_id: J<"build-id=">, HelpText<"Generate build ID note">,
   MetaVarName<"[fast,md5,sha1,uuid,0x<hexstring>]">;
 def : F<"build-id">, Alias<build_id>, AliasArgs<["sha1"]>, HelpText<"Alias for --build-id=sha1">;
 
+defm branch_to_branch: BB<"branch-to-branch",
+    "Enable branch-to-branch optimization (default at -O2)",
+    "Disable branch-to-branch optimization (default at -O0 and -O1)">;
+
 defm check_sections: B<"check-sections",
     "Check section addresses for overlaps (default)",
     "Do not check section addresses for overlaps">;
@@ -710,7 +714,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
     "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
     "Ignore the .llvm.lto section in relocatable object files (default).">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 6c4209a2b81e..cebd564036b2 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1340,22 +1340,10 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
   if (ctx.arg.emachine == EM_MIPS)
     return handleMipsTlsRelocation(ctx, type, sym, *sec, offset, addend, expr);
 
-  // LoongArch does not yet implement transition from TLSDESC to LE/IE, so
-  // generate TLSDESC dynamic relocation for the dynamic linker to handle.
-  if (ctx.arg.emachine == EM_LOONGARCH &&
-      oneof<RE_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
-            R_TLSDESC_CALL>(expr)) {
-    if (expr != R_TLSDESC_CALL) {
-      sym.setFlags(NEEDS_TLSDESC);
-      sec->addReloc({expr, type, offset, addend, &sym});
-    }
-    return 1;
-  }
-
   bool isRISCV = ctx.arg.emachine == EM_RISCV;
 
   if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
-            R_TLSDESC_GOTPLT>(expr) &&
+            R_TLSDESC_GOTPLT, RE_LOONGARCH_TLSDESC_PAGE_PC>(expr) &&
       ctx.arg.shared) {
     // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12_I,CALL} reference a label. Do not
     // set NEEDS_TLSDESC on the label.
@@ -1369,10 +1357,14 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return 1;
   }
 
-  // LoongArch supports IE to LE optimization in non-extreme code model.
+  // LoongArch supports IE to LE, DESC GD/LD to IE/LE optimizations in
+  // non-extreme code model.
   bool execOptimizeInLoongArch =
       ctx.arg.emachine == EM_LOONGARCH &&
-      (type == R_LARCH_TLS_IE_PC_HI20 || type == R_LARCH_TLS_IE_PC_LO12);
+      (type == R_LARCH_TLS_IE_PC_HI20 || type == R_LARCH_TLS_IE_PC_LO12 ||
+       type == R_LARCH_TLS_DESC_PC_HI20 || type == R_LARCH_TLS_DESC_PC_LO12 ||
+       type == R_LARCH_TLS_DESC_LD || type == R_LARCH_TLS_DESC_CALL ||
+       type == R_LARCH_TLS_DESC_PCREL20_S2);
 
   // ARM, Hexagon, LoongArch and RISC-V do not support GD/LD to IE/LE
   // optimizations.
@@ -1431,9 +1423,23 @@ unsigned RelocationScanner::handleTlsRelocation(RelExpr expr, RelType type,
     return 1;
   }
 
+  // LoongArch does not support transition from TLSDESC to LE/IE in the extreme
+  // code model, in which NEEDS_TLSDESC should set, rather than NEEDS_TLSGD. So
+  // we check independently.
+  if (ctx.arg.emachine == EM_LOONGARCH &&
+      oneof<RE_LOONGARCH_TLSDESC_PAGE_PC, R_TLSDESC, R_TLSDESC_PC,
+            R_TLSDESC_CALL>(expr) &&
+      !execOptimize) {
+    if (expr != R_TLSDESC_CALL) {
+      sym.setFlags(NEEDS_TLSDESC);
+      sec->addReloc({expr, type, offset, addend, &sym});
+    }
+    return 1;
+  }
+
   if (oneof<RE_AARCH64_TLSDESC_PAGE, R_TLSDESC, R_TLSDESC_CALL, R_TLSDESC_PC,
             R_TLSDESC_GOTPLT, R_TLSGD_GOT, R_TLSGD_GOTPLT, R_TLSGD_PC,
-            RE_LOONGARCH_TLSGD_PAGE_PC>(expr)) {
+            RE_LOONGARCH_TLSGD_PAGE_PC, RE_LOONGARCH_TLSDESC_PAGE_PC>(expr)) {
     if (!execOptimize) {
       sym.setFlags(NEEDS_TLSGD);
       sec->addReloc({expr, type, offset, addend, &sym});
@@ -1665,9 +1671,10 @@ void RelocationScanner::scan(Relocs<RelTy> rels) {
   }
 
   // Sort relocations by offset for more efficient searching for
-  // R_RISCV_PCREL_HI20 and R_PPC64_ADDR64.
+  // R_RISCV_PCREL_HI20, R_PPC64_ADDR64 and the branch-to-branch optimization.
   if (ctx.arg.emachine == EM_RISCV ||
-      (ctx.arg.emachine == EM_PPC64 && sec->name == ".toc"))
+      (ctx.arg.emachine == EM_PPC64 && sec->name == ".toc") ||
+      ctx.arg.branchToBranch)
     llvm::stable_sort(sec->relocs(),
                       [](const Relocation &lhs, const Relocation &rhs) {
                         return lhs.offset < rhs.offset;
@@ -1958,6 +1965,9 @@ void elf::postScanRelocations(Ctx &ctx) {
   for (ELFFileBase *file : ctx.objectFiles)
     for (Symbol *sym : file->getLocalSymbols())
       fn(*sym);
+
+  if (ctx.arg.branchToBranch)
+    ctx.target->applyBranchToBranchOpt();
 }
 
 static bool mergeCmp(const InputSection *a, const InputSection *b) {
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index d2a77bc95310..02ddf707fd95 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -131,6 +131,7 @@ enum RelExpr {
   RE_LOONGARCH_GOT_PAGE_PC,
   RE_LOONGARCH_TLSGD_PAGE_PC,
   RE_LOONGARCH_TLSDESC_PAGE_PC,
+  RE_LOONGARCH_RELAX_TLS_GD_TO_IE_PAGE_PC,
 };
 
 // Architecture-neutral representation of relocation.
diff --git a/lld/ELF/ScriptParser.cpp b/lld/ELF/ScriptParser.cpp
index 593d5636f245..ddfa24d9cacf 100644
--- a/lld/ELF/ScriptParser.cpp
+++ b/lld/ELF/ScriptParser.cpp
@@ -1229,6 +1229,8 @@ SymbolAssignment *ScriptParser::readSymbolAssignment(StringRef name) {
 // This is an operator-precedence parser to parse a linker
 // script expression.
 Expr ScriptParser::readExpr() {
+  if (atEOF())
+    return []() { return 0; };
   // Our lexer is context-aware. Set the in-expression bit so that
   // they apply different tokenization rules.
   SaveAndRestore saved(lexState, State::Expr);
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index fd1e5d33c438..6dd20b2f0cba 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -101,6 +101,7 @@ class TargetInfo {
 
   virtual void applyJumpInstrMod(uint8_t *loc, JumpModType type,
                                  JumpModType val) const {}
+  virtual void applyBranchToBranchOpt() const {}
 
   virtual ~TargetInfo();
 
diff --git a/lld/docs/DTLTO.rst b/lld/docs/DTLTO.rst
new file mode 100644
index 000000000000..985decf6c7db
--- /dev/null
+++ b/lld/docs/DTLTO.rst
@@ -0,0 +1,42 @@
+Integrated Distributed ThinLTO (DTLTO)
+======================================
+
+Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend
+ThinLTO compilations via external distribution systems, such as Incredibuild,
+during the traditional link step.
+
+The implementation is documented here: https://llvm.org/docs/DTLTO.html.
+
+Currently, DTLTO is only supported in ELF LLD. Support will be added to other
+LLD flavours in the future.
+
+ELF LLD
+-------
+
+The command-line interface is as follows:
+
+- ``--thinlto-distributor=<path>``  
+  Specifies the file to execute as the distributor process. If specified,
+  ThinLTO backend compilations will be distributed.
+
+- ``--thinlto-remote-compiler=<path>``  
+  Specifies the path to the compiler that the distributor process will use for
+  backend compilations. The compiler invoked must match the version of LLD.
+
+- ``--thinlto-distributor-arg=<arg>``  
+  Specifies ``<arg>`` on the command line when invoking the distributor.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``  
+  Appends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+  Options that introduce extra input/output files may cause miscompilation if
+  the distribution system does not automatically handle pushing/fetching them to
+  remote nodes. In such cases, configure the distributor - possibly using
+  ``--thinlto-distributor-arg=`` - to manage these dependencies. See the
+  distributor documentation for details.
+
+Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
+Currently, other options are silently accepted but do not have the intended
+effect. Support for such options will be expanded in the future.
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 064ed0828c31..dabfc961dd5b 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -62,6 +62,10 @@ ELF Improvements
   on executable sections.
   (`#128883 <https://github.com/llvm/llvm-project/pull/128883>`_)
 
+* For AArch64 and X86_64, added ``--branch-to-branch``, which rewrites branches
+  that point to another branch instruction to instead branch directly to the
+  target of the second instruction. Enabled by default at ``-O2``.
+  
 Breaking changes
 ----------------
 * Executable-only and readable-executable sections are now allowed to be placed
diff --git a/lld/docs/index.rst b/lld/docs/index.rst
index 8260461c3690..69792e3b575b 100644
--- a/lld/docs/index.rst
+++ b/lld/docs/index.rst
@@ -147,3 +147,4 @@ document soon.
    ELF/start-stop-gc
    ELF/warn_backrefs
    MachO/index
+   DTLTO
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index f59be56b5cba..87f6134a83d8 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -93,6 +93,11 @@ Bind default visibility defined STB_GLOBAL function symbols locally for
 .Fl shared.
 .It Fl -be8
 Write a Big Endian ELF File using BE8 format(AArch32 only)
+.It Fl -branch-to-branch
+Enable the branch-to-branch optimizations: a branch whose target is
+another branch instruction is rewritten to point to the latter branch
+target (AArch64 and X86_64 only). Enabled by default at
+.Fl O2 Ns .
 .It Fl -build-id Ns = Ns Ar value
 Generate a build ID note.
 .Ar value
@@ -414,7 +419,7 @@ If not specified,
 .Dv a.out
 is used as a default.
 .It Fl O Ns Ar value
-Optimize output file size.
+Optimize output file.
 .Ar value
 may be:
 .Pp
@@ -424,7 +429,7 @@ Disable string merging.
 .It Cm 1
 Enable string merging.
 .It Cm 2
-Enable string tail merging.
+Enable string tail merging and branch-to-branch optimization.
 .El
 .Pp
 .Fl O Ns Cm 1
diff --git a/lld/include/lld/Common/BPSectionOrdererBase.inc b/lld/include/lld/Common/BPSectionOrdererBase.inc
index fbeca187aa76..00c9dc7159dd 100644
--- a/lld/include/lld/Common/BPSectionOrdererBase.inc
+++ b/lld/include/lld/Common/BPSectionOrdererBase.inc
@@ -319,7 +319,7 @@ auto BPOrderer<D>::computeOrder(
     for (auto dupSecIdx : It->getSecond()) {
       const auto *dupIsec = sections[dupSecIdx];
       if (orderedSections.insert(dupIsec)) {
-        duplicateCodeSize += D::getSize(*isec);
+        duplicateCodeSize += D::getSize(*dupIsec);
         ++numDuplicateCodeSections;
       }
     }
@@ -337,7 +337,7 @@ auto BPOrderer<D>::computeOrder(
     for (auto dupSecIdx : It->getSecond()) {
       const auto *dupIsec = sections[dupSecIdx];
       if (orderedSections.insert(dupIsec)) {
-        duplicateDataSize += D::getSize(*isec);
+        duplicateDataSize += D::getSize(*dupIsec);
         ++numDuplicateDataSections;
       }
     }
diff --git a/lld/test/COFF/exported-dllmain.test b/lld/test/COFF/exported-dllmain.test
new file mode 100644
index 000000000000..fcf6ed100537
--- /dev/null
+++ b/lld/test/COFF/exported-dllmain.test
@@ -0,0 +1,57 @@
+REQUIRES: x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows a.s -o a.obj
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows b1.s -o b1.obj
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows b2.s -o b2.obj
+
+### This is the line where our problem occurs. Here, we export the DllMain symbol which shouldn't happen normally.
+RUN: lld-link b1.obj b2.obj -out:b.dll -dll -implib:b.lib -entry:DllMain -export:bar -export:DllMain
+
+RUN: llvm-mc -filetype=obj -triple=x86_64-windows c.s -o c.obj
+RUN: lld-link -lib c.obj -out:c.lib
+
+### Later, if b.lib is provided before other libs/objs that export DllMain statically, we previously were using the dllimported DllMain from b.lib, which is wrong.
+RUN: lld-link a.obj b.lib c.lib -dll -out:out.dll -entry:DllMain 2>&1 | FileCheck -check-prefix=WARN %s
+RUN: lld-link a.obj b.lib c.lib -dll -out:out.dll -entry:DllMain -ignore:exporteddllmain 2>&1 | FileCheck -check-prefix=IGNORED --allow-empty %s
+RUN: llvm-objdump --private-headers -d out.dll | FileCheck -check-prefix=DISASM %s
+
+WARN: lld-link: warning: b.lib: skipping exported DllMain symbol [exporteddllmain]
+IGNORED-NOT: lld-link: warning: b.lib: skipping exported DllMain symbol [exporteddllmain]
+
+DISASM: The Import Tables:
+DISASM: DLL Name: b.dll
+DISASM-NOT: DllMain
+DISASM: bar
+DISASM: Disassembly of section .text:
+DISASM-EMPTY:
+DISASM:      b8 01 00 00 00               movl    $0x1, %eax
+DISASM-NEXT: c3                           retq
+
+#--- a.s
+        .text
+        .globl foo
+foo:
+        call *__imp_bar(%rip)
+        ret
+
+#--- b1.s
+        .text
+        .globl bar
+bar:
+        ret
+
+#--- b2.s
+        .text
+        .globl DllMain
+DllMain:
+        xor %eax, %eax
+        ret
+
+#--- c.s
+        .text
+        .globl DllMain
+DllMain:
+        movl $1, %eax
+        ret
diff --git a/lld/test/COFF/winsysroot.test b/lld/test/COFF/winsysroot.test
index e29e6ca115c8..c2e9f23562d4 100644
--- a/lld/test/COFF/winsysroot.test
+++ b/lld/test/COFF/winsysroot.test
@@ -58,3 +58,7 @@ LIBIGNORED_MINGW: could not open 'libstd32.a'
 
 # RUN: not lld-link -lldmingw %t.obj /defaultlib:std32 /winsysroot:%t.dir/sysroot 2>&1 | FileCheck -check-prefix=IGNORED_ARG %s
 IGNORED_ARG: warning: ignoring /vctoolsdir or /winsysroot flags in MinGW mode
+
+Check that when /winsdkversion is specified, %LIB% is ignored.
+# RUN: env LIB=%t.dir/sysroot/VC/Tools/MSVC/1.1.1.1/lib/x86 not lld-link %t.obj /winsdkversion:99.99.9999.9 /defaultlib:std32 2>&1 | FileCheck -check-prefix=WINSDKVERSION %s
+WINSDKVERSION: could not open 'std32.lib'
diff --git a/lld/test/ELF/aarch64-branch-to-branch.s b/lld/test/ELF/aarch64-branch-to-branch.s
new file mode 100644
index 000000000000..c970fe308579
--- /dev/null
+++ b/lld/test/ELF/aarch64-branch-to-branch.s
@@ -0,0 +1,84 @@
+# REQUIRES: aarch64
+
+## Test that the branch-to-branch optimization follows the links
+## from f1 -> f2 -> f3 and updates all references to point to f3.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-pc-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t --branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+
+## Test that branch-to-branch is disabled by default.
+
+# RUN: ld.lld %t.o -o %t --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --no-branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+## Test that branch-to-branch is disabled for preemptible symbols.
+
+# RUN: ld.lld %t.o -o %t --branch-to-branch -shared --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+.section .rodata.vtable,"a"
+.globl vtable
+vtable:
+# B2B: Contents of section .rodata:
+# RELOC: RELOCATION RECORDS FOR [.rodata]:
+# RELOC-NEXT: OFFSET
+# B2B-NEXT: [[VF:[0-9a-f]{8}]]
+# B2B-RELOC-NEXT: R_AARCH64_PLT32 f3
+# NOB2B-RELOC-NEXT: R_AARCH64_PLT32 f1
+.4byte f1@PLT - vtable
+# B2B-SAME: [[VF]]
+# B2B-RELOC-NEXT: R_AARCH64_PLT32 f3+0x4
+# NOB2B-RELOC-NEXT: R_AARCH64_PLT32 f2+0x4
+.4byte f2@PLT - vtable
+# B2B-SAME: [[VF]]
+# RELOC-NEXT: R_AARCH64_PLT32 f3+0x8
+.4byte f3@PLT - vtable
+
+.section .text._start,"ax"
+.globl _start
+# CHECK: <_start>:
+# RELOC: RELOCATION RECORDS FOR [.text]:
+# RELOC-NEXT: OFFSET
+_start:
+.cfi_startproc
+# B2B: bl {{.*}} <f3>
+# B2B-RELOC-NEXT: R_AARCH64_CALL26 f3
+# NOB2B: bl {{.*}} <f1{{.*}}>
+# NOB2B-RELOC-NEXT: R_AARCH64_CALL26 f1
+bl f1
+# B2B: b {{.*}} <f3>
+# B2B-RELOC-NEXT: R_AARCH64_JUMP26 f3
+# NOB2B: b {{.*}} <f2{{.*}}>
+# NOB2B-RELOC-NEXT: R_AARCH64_JUMP26 f2
+b f2
+.cfi_endproc
+
+.section .text.f1,"ax"
+.globl f1
+f1:
+# B2B-RELOC-NEXT: R_AARCH64_JUMP26 f3
+# NOB2B-RELOC-NEXT: R_AARCH64_JUMP26 f2
+b f2
+
+.section .text.f2,"ax"
+.globl f2
+# CHECK: <f2>:
+f2:
+# CHECK-NEXT: b {{.*}} <f3{{.*}}>
+# RELOC-NEXT: R_AARCH64_JUMP26 f3
+b f3
+
+.section .text.f3,"ax"
+.globl f3
+f3:
+ret
diff --git a/lld/test/ELF/dtlto/files.test b/lld/test/ELF/dtlto/files.test
new file mode 100644
index 000000000000..54bcf587f46b
--- /dev/null
+++ b/lld/test/ELF/dtlto/files.test
@@ -0,0 +1,99 @@
+# REQUIRES: x86
+
+## Test that the LLD options --save-temps, --thinlto-emit-index-files,
+## and --thinlto-emit-imports-files function correctly with DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
+## linker.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+RUN: cp t1.bc t3.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Create response file containing shared ThinLTO linker arguments.
+## --start-lib/--end-lib is used to test the special case where unused lazy
+## bitcode inputs result in empty index/imports files.
+## Note that mock.py does not do any compilation; instead, it simply writes
+## the contents of the object files supplied on the command line into the
+## output object files in job order.
+RUN: echo "t1.bc t2.bc --start-lib t3.bc --end-lib -o my.elf \
+RUN:   --thinlto-distributor=\"%python\" \
+RUN:   --thinlto-distributor-arg=\"%llvm_src_root/utils/dtlto/mock.py\" \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+## Check that without extra flags, no index/imports files are produced and
+## backend temp files are removed.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that index files are created with --thinlto-emit-index-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that imports files are created with --thinlto-emit-imports-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,IMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that both index and imports files are emitted with both flags.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## Check that backend temp files are retained with --save-temps.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that all files are emitted when all options are enabled.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## JSON jobs description, retained with --save-temps.
+## Note that DTLTO temporary files include a PID component.
+NOBACKEND-NOT:      {{^}}my.[[#]].dist-file.json{{$}}
+BACKEND:            {{^}}my.[[#]].dist-file.json{{$}}
+
+## Index/imports files for t1.bc.
+NOIMPORTSFILES-NOT: {{^}}t1.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t1.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t1.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t1.bc.thinlto.bc{{$}}
+
+## Index/imports files for t2.bc.
+NOIMPORTSFILES-NOT: {{^}}t2.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t2.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t2.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t2.bc.thinlto.bc{{$}}
+
+## Empty index/imports files for unused t3.bc.
+NOEMPTYIMPORTS-NOT: {{^}}t3.bc.imports{{$}}
+EMPTYIMPORTS:       {{^}}t3.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t3.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t3.bc.thinlto.bc{{$}}
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/options.test b/lld/test/ELF/dtlto/options.test
new file mode 100644
index 000000000000..1a10d18140f2
--- /dev/null
+++ b/lld/test/ELF/dtlto/options.test
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO options are passed correctly to the distributor and
+## remote compiler.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc foo.ll -o foo.o
+
+## Note: validate.py does not perform any compilation. Instead, it validates the
+## received JSON, pretty-prints the JSON and the supplied arguments, and then
+## exits with an error. This allows FileCheck directives to verify the
+## distributor inputs.
+RUN: not ld.lld foo.o \
+RUN:   -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+RUN:   --thinlto-distributor-arg=darg1=10 \
+RUN:   --thinlto-distributor-arg=darg2=20 \
+RUN:   --thinlto-remote-compiler=my_clang.exe \
+RUN:   --thinlto-remote-compiler-arg=carg1=20 \
+RUN:   --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s
+
+CHECK: distributor_args=['darg1=10', 'darg2=20']
+
+CHECK: "linker_output": "my.elf"
+
+CHECK: "my_clang.exe"
+CHECK: "carg1=20"
+CHECK: "carg2=30"
+
+CHECK: error: DTLTO backend compilation: cannot open native object file:
+
+#--- foo.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+  ret void
+}
diff --git a/lld/test/ELF/dtlto/partitions.test b/lld/test/ELF/dtlto/partitions.test
new file mode 100644
index 000000000000..fbcbc0713ce2
--- /dev/null
+++ b/lld/test/ELF/dtlto/partitions.test
@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO works with more than one LTO partition.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@f/@t1/g' f.ll > t1.ll
+RUN: sed 's/@f/@t2/g' f.ll > t2.ll
+
+## Generate bitcode.
+RUN: opt f.ll -o full.bc
+RUN: opt -thinlto-bc t1.ll -o thin1.bc
+RUN: opt -thinlto-bc t2.ll -o thin2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o thin1.o
+RUN: llc t2.ll --filetype=obj -o thin2.o
+
+## Link with 3 LTO partitions.
+RUN: ld.lld full.bc thin1.bc thin2.bc \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=thin1.o \
+RUN:   --thinlto-distributor-arg=thin2.o \
+RUN:   --save-temps \
+RUN:   --lto-partitions=3
+
+## DTLTO temporary object files include the task number and a PID component. The
+## task number should incorporate the LTO partition number.
+RUN: ls | sort | FileCheck %s
+CHECK: {{^}}thin1.3.[[PID:[a-zA-Z0-9_]+]].native.o{{$}}
+CHECK: {{^}}thin2.4.[[PID]].native.o{{$}}
+
+#--- f.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+  ret void
+}
diff --git a/lld/test/ELF/linkerscript/align-section.test b/lld/test/ELF/linkerscript/align-section.test
index 7a28fef2076e..1324be24a932 100644
--- a/lld/test/ELF/linkerscript/align-section.test
+++ b/lld/test/ELF/linkerscript/align-section.test
@@ -1,7 +1,24 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux /dev/null -o %t.o
-# RUN: ld.lld -o %t --script %s %t.o -shared
+# RUN: rm -rf %t && split-file %s %t && cd %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux /dev/null -o a.o
+# RUN: ld.lld --script a.t a.o -shared
 
 # lld shouldn't crash.
 
+#--- a.t
 SECTIONS { .foo : ALIGN(2M) {} }
+
+# RUN: not ld.lld --script b.t 2>&1 | FileCheck %s --match-full-lines --strict-whitespace
+
+# lld should not crash and report the error properly.
+
+# CHECK:{{.*}} error: b.t:3: malformed number: :
+# CHECK:>>>   S :ALIGN(4096) {}
+# CHECK:>>>     ^
+
+#--- b.t
+SECTIONS
+{
+  S :ALIGN(4096) {}
+}
diff --git a/lld/test/ELF/loongarch-relax-tlsdesc.s b/lld/test/ELF/loongarch-relax-tlsdesc.s
index 9ce7c5881ca9..5f4368343471 100644
--- a/lld/test/ELF/loongarch-relax-tlsdesc.s
+++ b/lld/test/ELF/loongarch-relax-tlsdesc.s
@@ -9,19 +9,18 @@
 # RUN: llvm-readobj -r -x .got a.64.so | FileCheck --check-prefix=GD64-RELA %s
 # RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.so | FileCheck %s --check-prefix=GD64
 
-## FIXME: The transition from TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
-# RUN: ld.lld --relax -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
-# RUN: llvm-objdump --no-show-raw-insn -d -h a.64.le | FileCheck %s --check-prefix=LE64
+## FIXME: IE/LE relaxation have not yet been implemented, --relax/--no-relax obtain the same results.
+## Transition from TLSDESC to IE/LE. Also check --emit-relocs.
+# RUN: ld.lld -e 0 -z now --emit-relocs a.64.o c.64.o -o a.64.le
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.le | FileCheck %s --check-prefix=LE64
 
 # RUN: ld.lld --no-relax -e 0 -z now a.64.o c.64.o -o a.64.le.norelax
 # RUN: llvm-objdump --no-show-raw-insn -d -h a.64.le.norelax | FileCheck %s --check-prefix=LE64-NORELAX
 
-# RUN: ld.lld --relax -e 0 -z now a.64.o c.64.so -o a.64.ie
+# RUN: ld.lld --relax -e 0 -z now --emit-relocs a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
-# RUN: llvm-objdump --no-show-raw-insn -d -h a.64.ie | FileCheck %s --check-prefix=IE64
+# RUN: llvm-objdump --no-show-raw-insn -dr -h a.64.ie | FileCheck %s --check-prefix=IE64
 
 # RUN: ld.lld --no-relax -e 0 -z now a.64.o c.64.so -o a.64.ie.norelax
 # RUN: llvm-objdump --no-show-raw-insn -d -h a.64.ie.norelax | FileCheck %s --check-prefix=IE64-NORELAX
@@ -71,172 +70,199 @@
 # GD64-NEXT:          jirl    $ra, $ra, 0
 # GD64-NEXT:          add.d   $a4, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30280 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30290 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x302A0 R_LARCH_TLS_DESC64 - 0x1000
-# LE64-RELA-NEXT:   0x302B0 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030280 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030290 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x000302a0 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x000302b0 00000000 00000000 00000000 00000000 .
-
-# LE64:   .got    00000040 0000000000030280
-
-## &.got[a]-. = 0x30280 - 0x20228 = 16406<<2
-# LE64:        20228: pcaddi  $a0, 16406
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+# LE64-RELA: could not find section '.got'
+
+## a@tprel = 0x8
+# LE64:        20158: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          ori     $a0, $zero, 8
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL a
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
 # LE64-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30280+48 - 0x20238: 0x10 pages, page offset 0x2b0
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# LE64:        20238: pcalau12i $a0, 16
-# LE64-NEXT:          addi.d  $a0, $a0, 688
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# LE64:        2016c: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 b
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 b
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD b
+# LE64-NEXT:          ori     $a0, $zero, 2047
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL b
 # LE64-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30280+16 - 0x2024c: 0x10 pages, page offset 0x290
+## c@tprel = 0x800
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# LE64:        2024c: pcalau12i $a0, 16
+# LE64:        20180: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 c
 # LE64-NEXT:          addi.d  $t0, $zero, 0
-# LE64-NEXT:          addi.d  $a0, $a0, 656
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 c
 # LE64-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NEXT:          ld.d    $ra, $a0, 0
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD c
 # LE64-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NEXT:          jirl    $ra, $ra, 0
+# LE64-NEXT:          ori     $a0, $zero, 2048
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL c
 # LE64-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30280+32 - 0x2026c = 16397<<2
-# LE64:        2026c: pcaddi  $a0, 16397
-# LE64-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NEXT:          jirl    $ra, $ra, 0
+## d@tprel = 0x1000
+# LE64:        201a0: nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 d
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          nop
+# LE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 d
+# LE64-NEXT:            R_LARCH_RELAX *ABS*
+# LE64-NEXT:          lu12i.w $a0, 1
+# LE64-NEXT:            R_LARCH_TLS_DESC_LD d
+# LE64-NEXT:          ori     $a0, $a0, 0
+# LE64-NEXT:            R_LARCH_TLS_DESC_CALL d
 # LE64-NEXT:          add.d   $a4, $a0, $tp
 
-# LE64-NORELAX: .got    00000040 0000000000030288
-
-## &.got[a]-. = 0x30288 - 0x20228 = 0x10 pages, page offset 0x288
-# LE64-NORELAX:        20228: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 648
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# LE64-NORELAX:        20158: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 8
 # LE64-NORELAX-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30288+48 - 0x2023c: 0x10 pages, page offset 0x2b8
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# LE64-NORELAX:        2023c: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 696
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# LE64-NORELAX:        2016c: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 2047
 # LE64-NORELAX-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30288+16 - 0x20250: 0x10 pages, page offset 0x298
+## c@tprel = 0x800
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# LE64-NORELAX:        20250: pcalau12i $a0, 16
+# LE64-NORELAX:        20180: nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $zero, 0
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 664
+# LE64-NORELAX-NEXT:          nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
+# LE64-NORELAX-NEXT:          nop
 # LE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+# LE64-NORELAX-NEXT:          ori     $a0, $zero, 2048
 # LE64-NORELAX-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30288+32 - 0x20270: 0x10 pages, page offset 0x2a8
-# LE64-NORELAX:        20270: pcalau12i $a0, 16
-# LE64-NORELAX-NEXT:          addi.d  $a0, $a0, 680
-# LE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# LE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## d@tprel = 0x1000
+# LE64-NORELAX:        201a0: nop
+# LE64-NORELAX-NEXT:          nop
+# LE64-NORELAX-NEXT:          lu12i.w $a0, 1
+# LE64-NORELAX-NEXT:          ori     $a0, $a0, 0
 # LE64-NORELAX-NEXT:          add.d   $a4, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x30430 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x30460 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x30440 R_LARCH_TLS_DESC64 c 0x0
-# IE64-RELA-NEXT:   0x30450 R_LARCH_TLS_DESC64 d 0x0
+# IE64-RELA-NEXT:   0x30408 R_LARCH_TLS_TPREL64 c 0x0
+# IE64-RELA-NEXT:   0x30410 R_LARCH_TLS_TPREL64 d 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x00030430 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030440 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030450 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x00030460 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x00030408 00000000 00000000 00000000 00000000 .
 
-# IE64:   .got           00000040 0000000000030430
+# IE64:   .got           00000010 0000000000030408
 
 ## a and b are optimized to use LE. c and d are optimized to IE.
-## &.got[a]-. = 0x30430 - 0x202f8 = 16462<<2
-# IE64:        202f8: pcaddi  $a0, 16462
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# IE64:        202c8: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          ori     $a0, $zero, 8
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL a
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
 # IE64-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30430+48 - 0x20308: 0x10 pages, page offset 0x460
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# IE64:        20308: pcalau12i $a0, 16
-# IE64-NEXT:          addi.d  $a0, $a0, 1120
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# IE64:        202dc: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 b
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 b
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD b
+# IE64-NEXT:          ori     $a0, $zero, 2047
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL b
 # IE64-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30430+16 - 0x2031c: 0x10 pages, page offset 0x440
+## &.got[c]-. = 0x30408 - 0x20300: 0x10 pages, page offset 0x408
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# IE64:        2031c: pcalau12i $a0, 16
+# IE64:        202f0: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 c
 # IE64-NEXT:          addi.d  $t0, $zero, 0
-# IE64-NEXT:          addi.d  $a0, $a0, 1088
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 c
 # IE64-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NEXT:          ld.d    $ra, $a0, 0
+# IE64-NEXT:          pcalau12i $a0, 16
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD c
 # IE64-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NEXT:          jirl    $ra, $ra, 0
+# IE64-NEXT:          ld.d    $a0, $a0, 1032
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL c
 # IE64-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30430+32 - 0x2033c = 16453<<2
-# IE64:        2033c: pcaddi  $a0, 16453
-# IE64-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NEXT:          jirl    $ra, $ra, 0
+## &.got[d]-. = 0x30408+8 - 0x20318: 0x10 pages, page offset 0x410
+# IE64:        20310: nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_HI20 d
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          nop
+# IE64-NEXT:            R_LARCH_TLS_DESC_PC_LO12 d
+# IE64-NEXT:            R_LARCH_RELAX *ABS*
+# IE64-NEXT:          pcalau12i $a0, 16
+# IE64-NEXT:            R_LARCH_TLS_DESC_LD d
+# IE64-NEXT:          ld.d    $a0, $a0, 1040
+# IE64-NEXT:            R_LARCH_TLS_DESC_CALL d
 # IE64-NEXT:          add.d   $a4, $a0, $tp
 
-# IE64-NORELAX: .got    00000040 0000000000030438
+# IE64-NORELAX: .got    00000010 0000000000030408
 
-## &.got[a]-. = 0x30438 - 0x202f8 = 0x10 pages, page offset 0x438
-# IE64-NORELAX:        202f8: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1080
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## a@tprel = 0x8
+# IE64-NORELAX:        202c8: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          ori     $a0, $zero, 8
 # IE64-NORELAX-NEXT:          add.d   $a1, $a0, $tp
 
-## &.got[b]-. = 0x30438+48 - 0x2030c: 0x10 pages, page offset 0x468
-## R_LARCH_RELAX does not appear in pairs. No relaxation.
-# IE64-NORELAX:        2030c: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1128
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## b@tprel = 0x7ff
+# IE64-NORELAX:        202dc: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          ori     $a0, $zero, 2047
 # IE64-NORELAX-NEXT:          add.d   $a2, $a0, $tp
 
-## &.got[c]-. = 0x30438+16 - 0x20320: 0x10 pages, page offset 0x448
+## &.got[c]-. = 0x30408 - 0x20300: 0x10 pages, page offset 0x408
 ## Without R_LARCH_RELAX relocation. No relaxation.
-# IE64-NORELAX:        20320: pcalau12i $a0, 16
+# IE64-NORELAX:        202f0: nop
 # IE64-NORELAX-NEXT:          addi.d  $t0, $zero, 0
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1096
+# IE64-NORELAX-NEXT:          nop
 # IE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
+# IE64-NORELAX-NEXT:          pcalau12i $a0, 16
 # IE64-NORELAX-NEXT:          addi.d  $t0, $t0, 1
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+# IE64-NORELAX-NEXT:          ld.d    $a0, $a0, 1032
 # IE64-NORELAX-NEXT:          add.d   $a3, $a0, $tp
 
-## &.got[d]-. = 0x30438+32 - 0x20340: 0x10 pages, page offset 0x458
-# IE64-NORELAX:        20340: pcalau12i $a0, 16
-# IE64-NORELAX-NEXT:          addi.d  $a0, $a0, 1112
-# IE64-NORELAX-NEXT:          ld.d    $ra, $a0, 0
-# IE64-NORELAX-NEXT:          jirl    $ra, $ra, 0
+## &.got[d]-. = 0x30408+8 - 0x20318: 0x10 pages, page offset 0x410
+# IE64-NORELAX:        20310: nop
+# IE64-NORELAX-NEXT:          nop
+# IE64-NORELAX-NEXT:          pcalau12i $a0, 16
+# IE64-NORELAX-NEXT:          ld.d    $a0, $a0, 1040
 # IE64-NORELAX-NEXT:          add.d   $a4, $a0, $tp
 
 #--- a.s
 la.tls.desc $a0, a
 add.d $a1, $a0, $tp
 
-# ADDI.D does not have R_LARCH_RELAX. No relaxation.
+# ADDI.D does not have R_LARCH_RELAX. No relaxation when it is not optimized to IE/LE (--shared).
 pcalau12i $a0, %desc_pc_hi20(b)
 .reloc .-4, R_LARCH_RELAX, 0
 addi.d $a0, $a0, %desc_pc_lo12(b)
diff --git a/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s b/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
index 99e21d993519..422592980d28 100644
--- a/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
+++ b/lld/test/ELF/loongarch-tlsdesc-pcrel20-s2.s
@@ -14,14 +14,14 @@
 # RUN: ld.lld -shared -z now a.64.o c.64.o -o rel.64.so -z rel
 # RUN: llvm-readobj -r -x .got rel.64.so | FileCheck --check-prefix=GD64-REL %s
 
-## FIXME: The transition frome TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
+## Transition from TLSDESC to IE/LE.
 # RUN: ld.lld -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.le | FileCheck --check-prefix=LE64 %s
 
 # RUN: ld.lld -e 0 -z now a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.ie | FileCheck --check-prefix=IE64 %s
 
 ## 32-bit code is mostly the same. We only test a few variants.
 
@@ -68,25 +68,46 @@
 # GD64-NEXT:          jirl $ra, $ra, 0
 # GD64-NEXT:          add.d $a3, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30240 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30250 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x30260 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030240 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030250 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030260 00000000 00000000 00000000 00000000 .
+# LE64-RELA: could not find section '.got'
+
+# LE64-LABEL: <.text>:
+## st_value(a) = 8
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 8
+# LE64-NEXT:         add.d   $a1, $a0, $tp
+## st_value(b) = 2047
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2047
+# LE64-NEXT:         add.d   $a2, $a0, $tp
+## st_value(c) = 2048
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2048
+# LE64-NEXT:         add.d   $a3, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x303C8 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x303E8 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x303D8 R_LARCH_TLS_DESC64 c 0x0
+# IE64-RELA-NEXT:   0x30398 R_LARCH_TLS_TPREL64 c 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x000303c8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303d8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303e8 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x00030398 00000000 00000000                  .
+
+## a and b are optimized to use LE. c is optimized to IE.
+# IE64-LABEL: <.text>:
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 8
+# IE64-NEXT:         add.d   $a1, $a0, $tp
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 2047
+# IE64-NEXT:         add.d   $a2, $a0, $tp
+## &.got[c]-. = 0x30398 - 0x202ac: 0x10 pages, page offset 0x398
+# IE64-NEXT:         nop
+# IE64-NEXT:  202ac: pcalau12i $a0, 16
+# IE64-NEXT:         ld.d      $a0, $a0, 920
+# IE64-NEXT:         add.d   $a3, $a0, $tp
 
 # GD32-REL:      .rel.dyn {
 # GD32-REL-NEXT:    0x20264 R_LARCH_TLS_DESC32 -
diff --git a/lld/test/ELF/loongarch-tlsdesc.s b/lld/test/ELF/loongarch-tlsdesc.s
index bf09b1e4bbae..3dc31210d7dd 100644
--- a/lld/test/ELF/loongarch-tlsdesc.s
+++ b/lld/test/ELF/loongarch-tlsdesc.s
@@ -14,14 +14,14 @@
 # RUN: ld.lld -shared -z now a.64.o c.64.o -o rel.64.so -z rel
 # RUN: llvm-readobj -r -x .got rel.64.so | FileCheck --check-prefix=GD64-REL %s
 
-## FIXME: The transition frome TLSDESC to IE/LE has not yet been implemented.
-## Keep the dynamic relocations and hand them over to dynamic linker.
-
+## Transition from TLSDESC to IE/LE.
 # RUN: ld.lld -e 0 -z now a.64.o c.64.o -o a.64.le
-# RUN: llvm-readobj -r -x .got a.64.le | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-readobj -r -x .got a.64.le 2>&1 | FileCheck --check-prefix=LE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.le | FileCheck --check-prefix=LE64 %s
 
 # RUN: ld.lld -e 0 -z now a.64.o c.64.so -o a.64.ie
 # RUN: llvm-readobj -r -x .got a.64.ie | FileCheck --check-prefix=IE64-RELA %s
+# RUN: llvm-objdump --no-show-raw-insn -d a.64.ie | FileCheck --check-prefix=IE64 %s
 
 ## 32-bit code is mostly the same. We only test a few variants.
 
@@ -71,25 +71,52 @@
 # GD64-NEXT:          jirl $ra, $ra, 0
 # GD64-NEXT:          add.d $a3, $a0, $tp
 
-# LE64-RELA:      .rela.dyn {
-# LE64-RELA-NEXT:   0x30250 R_LARCH_TLS_DESC64 - 0x8
-# LE64-RELA-NEXT:   0x30260 R_LARCH_TLS_DESC64 - 0x800
-# LE64-RELA-NEXT:   0x30270 R_LARCH_TLS_DESC64 - 0x7FF
-# LE64-RELA-NEXT: }
-# LE64-RELA:      Hex dump of section '.got':
-# LE64-RELA-NEXT: 0x00030250 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030260 00000000 00000000 00000000 00000000 .
-# LE64-RELA-NEXT: 0x00030270 00000000 00000000 00000000 00000000 .
+# LE64-RELA: could not find section '.got'
+
+# LE64-LABEL: <.text>:
+## st_value(a) = 8
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 8
+# LE64-NEXT:         add.d   $a1, $a0, $tp
+## st_value(b) = 2047
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2047
+# LE64-NEXT:         add.d   $a2, $a0, $tp
+## st_value(c) = 2048
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         nop
+# LE64-NEXT:         ori     $a0, $zero, 2048
+# LE64-NEXT:         add.d   $a3, $a0, $tp
 
 # IE64-RELA:      .rela.dyn {
-# IE64-RELA-NEXT:   0x303D8 R_LARCH_TLS_DESC64 - 0x8
-# IE64-RELA-NEXT:   0x303F8 R_LARCH_TLS_DESC64 - 0x7FF
-# IE64-RELA-NEXT:   0x303E8 R_LARCH_TLS_DESC64 c 0x0
+# IE64-RELA-NEXT:   0x303A8 R_LARCH_TLS_TPREL64 c 0x0
 # IE64-RELA-NEXT: }
 # IE64-RELA:      Hex dump of section '.got':
-# IE64-RELA-NEXT: 0x000303d8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303e8 00000000 00000000 00000000 00000000 .
-# IE64-RELA-NEXT: 0x000303f8 00000000 00000000 00000000 00000000 .
+# IE64-RELA-NEXT: 0x000303a8 00000000 00000000                  .
+
+## a and b are optimized to use LE. c is optimized to IE.
+# IE64-LABEL: <.text>:
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 8
+# IE64-NEXT:         add.d   $a1, $a0, $tp
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:         ori     $a0, $zero, 2047
+# IE64-NEXT:         add.d   $a2, $a0, $tp
+## &.got[c]-. = 0x303a8 - 0x202b8: 0x10 pages, page offset 0x3a8
+# IE64-NEXT:         nop
+# IE64-NEXT:         nop
+# IE64-NEXT:  202b8: pcalau12i $a0, 16
+# IE64-NEXT:         ld.d      $a0, $a0, 936
+# IE64-NEXT:         add.d   $a3, $a0, $tp
 
 # GD32-REL:      .rel.dyn {
 # GD32-REL-NEXT:    0x20270 R_LARCH_TLS_DESC32 -
diff --git a/lld/test/ELF/lto/lazy-debug.ll b/lld/test/ELF/lto/lazy-debug.ll
new file mode 100644
index 000000000000..48d176f0f6bf
--- /dev/null
+++ b/lld/test/ELF/lto/lazy-debug.ll
@@ -0,0 +1,107 @@
+; REQUIRES: aarch64
+;; Regression test for a case in lazy debuginfo loading.
+;; The bug would cause ld.lld to crash.
+
+; RUN: split-file %s %t
+; RUN: llvm-as %t/hda_codec.s -o %t/hda_codec.o
+; RUN: llvm-as %t/hda_bind.s -o %t/hda_bind.o
+; RUN: ld.lld -EL -maarch64elf -r %t/hda_bind.o %t/hda_codec.o -o %t/hda_codec
+
+;--- hda_codec.s
+; ModuleID = 'hda_codec.o'
+source_filename = "hda_codec.i"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+%struct.anon = type { i32 }
+
+@hda_set_power_state_codec = hidden local_unnamed_addr global %struct.anon zeroinitializer, align 4, !dbg !0
+
+; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable
+define hidden void @snd_hda_codec_shutdown() local_unnamed_addr #0 !dbg !19 {
+entry:
+  ret void, !dbg !22
+}
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12, !13, !14, !15, !16, !17}
+!llvm.ident = !{!18}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "hda_set_power_state_codec", scope: !2, file: !5, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "hda_codec.i", directory: "/tmp")
+!4 = !{!0}
+!5 = !DIFile(filename: "hda_codec.i", directory: "/tmp", checksumkind: CSK_MD5, checksum: "c192644b468953345ff9647026173a7b")
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !5, line: 1, size: i64 32, offset: i64 0, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "mfg", scope: !6, file: !5, line: 2, baseType: !9, size: i64 32, offset: i64 0)
+!9 = !DIBasicType(name: "int", size: i64 32, encoding: DW_ATE_signed)
+!10 = !{i32 7, !"Dwarf Version", i32 5}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{i32 8, !"PIC Level", i32 2}
+!14 = !{i32 7, !"PIE Level", i32 2}
+!15 = !{i32 7, !"uwtable", i32 2}
+!16 = !{i32 7, !"frame-pointer", i32 1}
+!17 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!18 = !{!"clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)"}
+!19 = distinct !DISubprogram(name: "snd_hda_codec_shutdown", scope: !5, file: !5, line: 4, type: !20, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!20 = !DISubroutineType(types: !21)
+!21 = !{null}
+!22 = !DILocation(line: 4, column: 32, scope: !19)
+
+^0 = module: (path: "hda_codec.o", hash: (1120894731, 3099354915, 309166549, 2100129435, 1932081428))
+^1 = gv: (name: "snd_hda_codec_shutdown", summaries: (function: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 1, funcFlags: (readNone: 1, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0)))) ; guid = 1539195202824839354
+^2 = gv: (name: "hda_set_power_state_codec", summaries: (variable: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), varFlags: (readonly: 1, writeonly: 1, constant: 0)))) ; guid = 10300548032946263328
+^3 = flags: 8
+^4 = blockcount: 0
+
+;--- hda_bind.s
+; ModuleID = 'hda_bind.o'
+source_filename = "hda_bind.i"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define hidden void @hda_codec_driver_shutdown() local_unnamed_addr #0 !dbg !11 {
+entry:
+  tail call void @snd_hda_codec_shutdown() #2, !dbg !15
+  ret void, !dbg !16
+}
+
+declare void @snd_hda_codec_shutdown(...) local_unnamed_addr #1
+
+attributes #0 = { nounwind uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+attributes #1 = { "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "hda_bind.i", directory: "/tmp")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"PIE Level", i32 2}
+!7 = !{i32 7, !"uwtable", i32 2}
+!8 = !{i32 7, !"frame-pointer", i32 1}
+!9 = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+!10 = !{!"clang version 21.0.0git (git@github.com:llvm/llvm-project.git 93849a39c432827473ca6c676f1500da69b3aaa0)"}
+!11 = distinct !DISubprogram(name: "hda_codec_driver_shutdown", scope: !12, file: !12, line: 2, type: !13, scopeLine: 2, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!12 = !DIFile(filename: "hda_bind.i", directory: "/tmp", checksumkind: CSK_MD5, checksum: "5907dd04e8964940b57448f37db201c6")
+!13 = !DISubroutineType(types: !14)
+!14 = !{null}
+!15 = !DILocation(line: 2, column: 36, scope: !11)
+!16 = !DILocation(line: 2, column: 62, scope: !11)
+
+^0 = module: (path: "hda_bind.o", hash: (1958332034, 2012675483, 855691486, 2017350850, 2779827776))
+^1 = gv: (name: "snd_hda_codec_shutdown") ; guid = 1539195202824839354
+^2 = gv: (name: "hda_codec_driver_shutdown", summaries: (function: (module: ^0, flags: (linkage: external, visibility: hidden, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0, importType: definition), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 0, alwaysInline: 0, noUnwind: 1, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1, tail: 1))))) ; guid = 12817427500962331703
+^3 = flags: 8
+^4 = blockcount: 0
diff --git a/lld/test/ELF/x86-64-branch-to-branch.s b/lld/test/ELF/x86-64-branch-to-branch.s
new file mode 100644
index 000000000000..28ee5cad0b46
--- /dev/null
+++ b/lld/test/ELF/x86-64-branch-to-branch.s
@@ -0,0 +1,135 @@
+# REQUIRES: x86
+
+## Test that the branch-to-branch optimization follows the links
+## from f1 -> f2 -> f3 and updates all references to point to f3.
+ 
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t --branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+
+## Test that branch-to-branch is disabled by default.
+
+# RUN: ld.lld %t.o -o %t --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --no-branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+## Test that branch-to-branch is disabled for preemptible symbols.
+
+# RUN: ld.lld %t.o -o %t --branch-to-branch -shared --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+.section .rodata.vtable,"a"
+.globl vtable
+vtable:
+# B2B: Contents of section .rodata:
+# RELOC: RELOCATION RECORDS FOR [.rodata]:
+# RELOC-NEXT: OFFSET
+# B2B-NEXT: [[VF:[0-9a-f]{8}]]
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f1
+.4byte f1@PLT - vtable
+# B2B-SAME: [[VF]]
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3+0x4
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2+0x4
+.4byte f2@PLT - vtable
+# B2B-SAME: [[VF]]
+# RELOC-NEXT: R_X86_64_PLT32 f3+0x8
+.4byte f3@PLT - vtable
+
+# For .rodata.f6
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+ 
+.section .text._start,"ax"
+.globl _start
+# CHECK: <_start>:
+# RELOC: RELOCATION RECORDS FOR [.text]:
+# RELOC-NEXT: OFFSET
+_start:
+.cfi_startproc
+# B2B-NEXT: jmp {{.*}} <f3>
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-NEXT: jmp {{.*}} <f1{{.*}}>
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f1-0x4
+jmp f1
+# B2B-NEXT: jmp {{.*}} <f3>
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-NEXT: jmp {{.*}} <f2{{.*}}>
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2-0x4
+jmp f2
+# This will assemble to a relocation pointing to an STT_SECTION for .text.f4
+# with an addend, which looks similar to the relative vtable cases above but
+# requires different handling of the addend so that we don't think this is
+# branching to the `jmp f3` at the start of the target section.
+# CHECK-NEXT: jmp {{.*}} <f4{{.*}}>
+# RELOC-NEXT: R_X86_64_PLT32 .text+0x2e
+jmp f4
+# B2B-NEXT: jmp 0x[[IPLT:[0-9a-f]*]]
+# RELOC-NEXT: R_X86_64_PLT32 f5-0x4
+jmp f5
+# B2B-NEXT: jmp {{.*}} <f6>
+# RELOC-NEXT: R_X86_64_PLT32 f6-0x4
+jmp f6
+# B2B-NEXT: jmp {{.*}} <f7>
+# RELOC-NEXT: R_X86_64_PLT32 f7-0x4
+jmp f7
+.cfi_endproc
+
+.section .text.f1,"ax"
+.globl f1
+f1:
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2-0x4
+jmp f2
+
+.section .text.f2,"ax"
+.globl f2
+# CHECK: <f2>:
+f2:
+# CHECK-NEXT: jmp {{.*}} <f3{{.*}}>
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .text.f3,"ax"
+.globl f3
+f3:
+# Test that a self-branch doesn't trigger an infinite loop.
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .text.f4,"ax"
+jmp f3
+f4:
+ret
+
+.section .text.f5,"ax"
+.type f5, @gnu_indirect_function
+.globl f5
+f5:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .rodata.f6,"a"
+.globl f6
+f6:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+# RELOC: RELOCATION RECORDS FOR [.wtext.f7]:
+# RELOC-NEXT: OFFSET
+
+.section .wtext.f7,"awx"
+.globl f7
+f7:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+# B2B: <.iplt>:
+# B2B-NEXT: [[IPLT]]:
diff --git a/lld/test/MachO/icf-safe-thunks-dwarf.ll b/lld/test/MachO/icf-safe-thunks-dwarf.ll
index 248f1424a808..7fae4124f793 100644
--- a/lld/test/MachO/icf-safe-thunks-dwarf.ll
+++ b/lld/test/MachO/icf-safe-thunks-dwarf.ll
@@ -186,7 +186,7 @@ entry:
   ret i64 add (i64 add (i64 ptrtoint (ptr @func_A to i64), i64 ptrtoint (ptr @func_B to i64)), i64 ptrtoint (ptr @func_C to i64)), !dbg !27
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!6, !7, !8, !9, !10, !11}
@@ -254,7 +254,7 @@ entry:
   ret i64 add (i64 add (i64 ptrtoint (ptr @func_D to i64), i64 ptrtoint (ptr @func_E to i64)), i64 ptrtoint (ptr @func_F to i64)), !dbg !27
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind ssp willreturn memory(none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+ccpp,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!6, !7, !8, !9, !10, !11}
diff --git a/lld/test/MachO/icf-safe-thunks.ll b/lld/test/MachO/icf-safe-thunks.ll
index 650593003873..12f1e81bdf3e 100644
--- a/lld/test/MachO/icf-safe-thunks.ll
+++ b/lld/test/MachO/icf-safe-thunks.ll
@@ -244,8 +244,8 @@ define void @take_func_addr() local_unnamed_addr #0 {
   ret void
 }
 
-attributes #0 = { mustprogress nofree noinline norecurse nounwind ssp memory(readwrite, argmem: none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #1 = { mustprogress nofree noinline norecurse nounwind ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress nofree noinline norecurse nounwind ssp memory(readwrite, argmem: none) uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #1 = { mustprogress nofree noinline norecurse nounwind ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.module.flags = !{!0, !1, !2, !3}
 
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index 9e6b0e839d9a..10f556567cdc 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -36,6 +36,7 @@
 
 llvm_config.use_default_substitutions()
 llvm_config.use_lld()
+config.substitutions.append(("%llvm_src_root", config.llvm_src_root))
 
 tool_patterns = [
     "llc",
diff --git a/lld/test/wasm/compress-relocs.ll b/lld/test/wasm/compress-relocs.ll
deleted file mode 100644
index cea9f3476e99..000000000000
--- a/lld/test/wasm/compress-relocs.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc -filetype=obj %s -o %t.o
-; RUN: llvm-mc -mattr=+call-indirect-overlong -filetype=obj -triple=wasm32-unknown-unknown %p/Inputs/call-indirect.s -o %t2.o
-; RUN: wasm-ld --export-dynamic -o %t.wasm %t2.o %t.o
-; RUN: obj2yaml %t.wasm | FileCheck %s
-; RUN: wasm-ld --export-dynamic -O2 -o %t-opt.wasm %t2.o %t.o
-; RUN: obj2yaml %t-opt.wasm | FileCheck %s
-; RUN: not wasm-ld --compress-relocations -o %t-compressed.wasm %t2.o %t.o 2>&1 | FileCheck %s -check-prefix=ERROR
-; RUN: wasm-ld --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t2.o %t.o
-; RUN: obj2yaml %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
-
-target triple = "wasm32-unknown-unknown-wasm"
-
-define i32 @foo() {
-entry:
-  ret i32 2
-}
-
-define void @_start() local_unnamed_addr {
-entry:
-  ret void
-}
-
-; ERROR: wasm-ld: error: --compress-relocations is incompatible with output debug information. Please pass --strip-debug or --strip-all
-
-; CHECK:    Body:            41002802808880800011808080800080808080001A41002802848880800011818080800080808080001A0B
-; COMPRESS: Body:            4100280280081100001A4100280284081101001A0B
diff --git a/lld/test/wasm/compress-relocs.s b/lld/test/wasm/compress-relocs.s
new file mode 100644
index 000000000000..41d4ff567d50
--- /dev/null
+++ b/lld/test/wasm/compress-relocs.s
@@ -0,0 +1,129 @@
+# RUN: llvm-mc -mattr=+reference-types,+exception-handling -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
+# RUN: wasm-ld --export-dynamic -o %t.wasm %t.o
+# RUN: llvm-objdump -d %t.wasm | FileCheck %s
+# RUN: wasm-ld --export-dynamic -O2 -o %t-opt.wasm %t.o
+# RUN: llvm-objdump -d %t-opt.wasm | FileCheck %s
+# RUN: not wasm-ld --compress-relocations -o %t-compressed.wasm %t.o 2>&1 | FileCheck %s -check-prefix=ERROR
+# RUN: wasm-ld --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t.o
+# RUN: llvm-objdump -d %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
+
+  .globl _start
+_start:
+  .functype _start () -> ()
+  end_function
+
+  .globl func_ret_i64
+func_ret_i64:
+  .functype func_ret_i64 () -> (i64)
+  i64.const 1
+  end_function
+
+  .globl func_ret_i32
+func_ret_i32:
+  .functype func_ret_i32 () -> (i32)
+  i32.const 2
+  end_function
+
+i32_global:
+  .globaltype i32_global, i32
+
+i32_tag:
+  .tagtype i32_tag i32
+
+  .globl test_memory_and_indirect_call_relocs
+test_memory_and_indirect_call_relocs:
+  .functype test_memory_and_indirect_call_relocs () -> ()
+  i32.const indirect_func_ret_i64 # R_WASM_MEMORY_ADDR_SLEB
+  i32.load  0
+  call_indirect () -> (i64)       # R_WASM_TYPE_INDEX_LEB, R_WASM_TABLE_NUMBER_LEB
+  drop
+  i32.const 0
+  i32.load  indirect_func_ret_i32 # R_WASM_MEMORY_ADDR_LEB
+  call_indirect () -> (i32)
+  drop
+  i32.const func_ret_i64          # R_WASM_TABLE_INDEX_SLEB
+  call_indirect () -> (i64)
+  drop
+  end_function
+
+# CHECK:    test_memory_and_indirect_call_relocs
+# CHECK:      41 90 88 80 80 00                 i32.const      1040
+# CHECK:      11 80 80 80 80 00 80 80 80 80 00  call_indirect  0
+# CHECK:      28 02 94 88 80 80 00              i32.load       1044
+# CHECK:      11 81 80 80 80 00 80 80 80 80 00  call_indirect  1
+# CHECK:      41 81 80 80 80 00                 i32.const      1
+# CHECK:      11 80 80 80 80 00 80 80 80 80 00  call_indirect  0
+# COMPRESS: test_memory_and_indirect_call_relocs
+# COMPRESS:   41 90 08                          i32.const      1040
+# COMPRESS:   11 00 00                          call_indirect  0
+# COMPRESS:   28 02 94 08                       i32.load       1044
+# COMPRESS:   11 01 00                          call_indirect  1
+# COMPRESS:   41 01                             i32.const      1
+# COMPRESS:   11 00 00                          call_indirect  0
+
+  .globl test_simple_index_relocs
+test_simple_index_relocs:
+  .functype test_simple_index_relocs () -> ()
+  call       func_ret_i32 # R_WASM_FUNCTION_INDEX_LEB
+  global.set i32_global   # R_WASM_GLOBAL_INDEX_LEB
+  i32.const  0
+  throw      i32_tag      # R_WASM_TAG_INDEX_LEB
+  end_function
+
+# CHECK:    test_simple_index_relocs
+# CHECK:      10 82 80 80 80 00  call        2
+# CHECK:      24 81 80 80 80 00  global.set  1
+# CHECK:      08 80 80 80 80 00  throw       0
+# COMPRESS: test_simple_index_relocs
+# COMPRESS:   10 02              call        2
+# COMPRESS:   24 01              global.set  1
+# COMPRESS:   08 00              throw       0
+
+  .globl test_relative_relocs
+test_relative_relocs:
+  .functype test_relative_relocs () -> ()
+  i32.const indirect_func_ret_i64@MBREL # R_WASM_MEMORY_ADDR_REL_SLEB
+  drop
+  i32.const func_ret_i32@TBREL          # R_WASM_TABLE_INDEX_REL_SLEB
+  drop
+  i32.const i32_tls_data@TLSREL         # R_WASM_MEMORY_ADDR_TLS_SLEB
+  drop
+  end_function
+
+# CHECK:    test_relative_relocs
+# CHECK:      41 90 88 80 80 00  i32.const  1040
+# CHECK:      41 81 80 80 80 00  i32.const  1
+# CHECK:      41 83 80 80 80 00  i32.const  3
+# COMPRESS: test_relative_relocs
+# COMPRESS:   41 90 08           i32.const  1040
+# COMPRESS:   41 01              i32.const  1
+# COMPRESS:   41 03              i32.const  3
+
+# ERROR: wasm-ld: error: --compress-relocations is incompatible with output debug information. Please pass --strip-debug or --strip-all
+
+  .section .tdata,"T",@
+  .int8 0
+  .int8 0
+  .int8 0
+i32_tls_data:
+  .int32 65
+  .size  i32_tls_data, 4
+
+  .section .data,"",@
+  .p2align 4
+indirect_func_ret_i64:
+  .int32 func_ret_i64
+  .size  indirect_func_ret_i64, 4
+
+indirect_func_ret_i32:
+  .int32 func_ret_i32
+  .size  indirect_func_ret_i32, 4
+
+.section .custom_section.target_features,"",@
+  .int8 2
+  .int8 43
+  .int8 7
+  .ascii  "atomics"
+  .int8 43
+  .int8 11
+  .ascii  "bulk-memory"
diff --git a/lld/test/wasm/compress-relocs64.s b/lld/test/wasm/compress-relocs64.s
new file mode 100644
index 000000000000..44e7a089275b
--- /dev/null
+++ b/lld/test/wasm/compress-relocs64.s
@@ -0,0 +1,92 @@
+# RUN: llvm-mc -filetype=obj -triple=wasm64-unknown-unknown %s -o %t.o
+# RUN: wasm-ld -mwasm64 --export-dynamic -o %t.wasm %t.o
+# RUN: llvm-objdump -d %t.wasm | FileCheck %s
+# RUN: wasm-ld -mwasm64 --export-dynamic -O2 -o %t-opt.wasm %t.o
+# RUN: llvm-objdump -d %t-opt.wasm | FileCheck %s
+# RUN: wasm-ld -mwasm64 --export-dynamic --strip-debug --compress-relocations -o %t-compressed.wasm %t.o
+# RUN: llvm-objdump -d %t-compressed.wasm | FileCheck %s -check-prefix=COMPRESS
+
+  .globl _start
+_start:
+  .functype _start () -> ()
+  end_function
+
+  .globl func_ret_i64
+func_ret_i64:
+  .functype func_ret_i64 () -> (i64)
+  i64.const 1
+  end_function
+
+  .globl func_ret_i32
+func_ret_i32:
+  .functype func_ret_i32 () -> (i32)
+  i32.const 2
+  end_function
+
+  .globl test_memory_and_indirect_call_relocs
+test_memory_and_indirect_call_relocs:
+  .functype test_memory_and_indirect_call_relocs () -> ()
+  i64.const indirect_func_ret_i64 # R_WASM_MEMORY_ADDR_SLEB64
+  drop
+  i64.const 0
+  i64.load  indirect_func_ret_i32 # R_WASM_MEMORY_ADDR_LEB64
+  drop
+  i64.const func_ret_i64          # R_WASM_TABLE_INDEX_SLEB64
+  drop
+  end_function
+
+# CHECK:    test_memory_and_indirect_call_relocs
+# CHECK:      42 90 88 80 80 80 80 80 80 80 00     i64.const 1040
+# CHECK:      29 03 98 88 80 80 80 80 80 80 80 00  i64.load  1048
+# CHECK:      42 81 80 80 80 80 80 80 80 80 00     i64.const 1
+# COMPRESS: test_memory_and_indirect_call_relocs
+# COMPRESS:   42 90 08                             i64.const 1040
+# COMPRESS:   29 03 98 08                          i64.load  1048
+# COMPRESS:   42 01                                i64.const 1
+
+  .globl test_relative_relocs
+test_relative_relocs:
+  .functype test_relative_relocs () -> ()
+  i64.const indirect_func_ret_i64@MBREL # R_WASM_MEMORY_ADDR_REL_SLEB64
+  drop
+  i64.const func_ret_i32@TBREL          # R_WASM_TABLE_INDEX_REL_SLEB64
+  drop
+  i64.const i32_tls_data@TLSREL         # R_WASM_MEMORY_ADDR_TLS_SLEB64
+  drop
+  end_function
+
+# CHECK:    test_relative_relocs
+# CHECK:      42 90 88 80 80 80 80 80 80 80 00  i64.const 1040
+# CHECK:      42 81 80 80 80 80 80 80 80 80 00  i64.const 1
+# CHECK:      42 83 80 80 80 80 80 80 80 80 00  i64.const 3
+# COMPRESS: test_relative_relocs
+# COMPRESS:   42 90 08                          i64.const 1040
+# COMPRESS:   42 01                             i64.const 1
+# COMPRESS:   42 03                             i64.const 3
+
+  .section .tdata,"T",@
+  .int8 0
+  .int8 0
+  .int8 0
+i32_tls_data:
+  .int32 65
+  .size  i32_tls_data, 4
+
+  .section .data,"",@
+  .p2align 4
+indirect_func_ret_i64:
+  .int64 func_ret_i64
+  .size  indirect_func_ret_i64, 8
+
+indirect_func_ret_i32:
+  .int64 func_ret_i32
+  .size  indirect_func_ret_i32, 8
+
+.section .custom_section.target_features,"",@
+  .int8 2
+  .int8 43
+  .int8 7
+  .ascii  "atomics"
+  .int8 43
+  .int8 11
+  .ascii  "bulk-memory"
diff --git a/lld/wasm/InputChunks.cpp b/lld/wasm/InputChunks.cpp
index 855589353e4f..181221a77b10 100644
--- a/lld/wasm/InputChunks.cpp
+++ b/lld/wasm/InputChunks.cpp
@@ -228,7 +228,7 @@ void InputFunction::setTableIndex(uint32_t index) {
 // witten.
 static unsigned writeCompressedReloc(uint8_t *buf, const WasmRelocation &rel,
                                      uint64_t value) {
-  switch (rel.Type) {
+  switch (rel.getType()) {
   case R_WASM_TYPE_INDEX_LEB:
   case R_WASM_FUNCTION_INDEX_LEB:
   case R_WASM_GLOBAL_INDEX_LEB:
@@ -239,16 +239,33 @@ static unsigned writeCompressedReloc(uint8_t *buf, const WasmRelocation &rel,
     return encodeULEB128(value, buf);
   case R_WASM_TABLE_INDEX_SLEB:
   case R_WASM_TABLE_INDEX_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB64:
   case R_WASM_MEMORY_ADDR_SLEB:
   case R_WASM_MEMORY_ADDR_SLEB64:
+  case R_WASM_MEMORY_ADDR_REL_SLEB:
+  case R_WASM_MEMORY_ADDR_REL_SLEB64:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB:
     return encodeSLEB128(static_cast<int64_t>(value), buf);
-  default:
-    llvm_unreachable("unexpected relocation type");
+  case R_WASM_TABLE_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I32:
+  case R_WASM_FUNCTION_OFFSET_I32:
+  case R_WASM_SECTION_OFFSET_I32:
+  case R_WASM_GLOBAL_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I64:
+  case R_WASM_TABLE_INDEX_I64:
+  case R_WASM_FUNCTION_OFFSET_I64:
+  case R_WASM_MEMORY_ADDR_LOCREL_I32:
+  case R_WASM_FUNCTION_INDEX_I32:
+    fatal("relocation compression not supported for " +
+          relocTypeToString(rel.Type));
   }
+  llvm_unreachable("unhandled relocation type");
 }
 
 static unsigned getRelocWidthPadded(const WasmRelocation &rel) {
-  switch (rel.Type) {
+  switch (rel.getType()) {
   case R_WASM_TYPE_INDEX_LEB:
   case R_WASM_FUNCTION_INDEX_LEB:
   case R_WASM_GLOBAL_INDEX_LEB:
@@ -256,15 +273,32 @@ static unsigned getRelocWidthPadded(const WasmRelocation &rel) {
   case R_WASM_MEMORY_ADDR_LEB:
   case R_WASM_TABLE_NUMBER_LEB:
   case R_WASM_TABLE_INDEX_SLEB:
+  case R_WASM_TABLE_INDEX_REL_SLEB:
   case R_WASM_MEMORY_ADDR_SLEB:
+  case R_WASM_MEMORY_ADDR_REL_SLEB:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB:
     return 5;
   case R_WASM_TABLE_INDEX_SLEB64:
+  case R_WASM_TABLE_INDEX_REL_SLEB64:
   case R_WASM_MEMORY_ADDR_LEB64:
   case R_WASM_MEMORY_ADDR_SLEB64:
+  case R_WASM_MEMORY_ADDR_REL_SLEB64:
+  case R_WASM_MEMORY_ADDR_TLS_SLEB64:
     return 10;
-  default:
-    llvm_unreachable("unexpected relocation type");
+  case R_WASM_TABLE_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I32:
+  case R_WASM_FUNCTION_OFFSET_I32:
+  case R_WASM_SECTION_OFFSET_I32:
+  case R_WASM_GLOBAL_INDEX_I32:
+  case R_WASM_MEMORY_ADDR_I64:
+  case R_WASM_TABLE_INDEX_I64:
+  case R_WASM_FUNCTION_OFFSET_I64:
+  case R_WASM_MEMORY_ADDR_LOCREL_I32:
+  case R_WASM_FUNCTION_INDEX_I32:
+    fatal("relocation compression not supported for " +
+          relocTypeToString(rel.Type));
   }
+  llvm_unreachable("unhandled relocation type");
 }
 
 static unsigned getRelocWidth(const WasmRelocation &rel, uint64_t value) {
diff --git a/lldb/bindings/interface/SBDebuggerExtensions.i b/lldb/bindings/interface/SBDebuggerExtensions.i
index ff2a0359b136..7895fcdd7246 100644
--- a/lldb/bindings/interface/SBDebuggerExtensions.i
+++ b/lldb/bindings/interface/SBDebuggerExtensions.i
@@ -45,4 +45,17 @@ STRING_EXTENSION_OUTSIDE(SBDebugger)
     lldb::FileSP GetErrorFileHandle() {
         return self->GetErrorFile().GetFile();
     }
+
+#ifdef SWIGPYTHON
+    %pythoncode %{
+        class staticproperty:
+            def __init__(self, func):
+                self.func = func
+            def __get__(self, instance, owner):
+                return self.func()
+        @staticproperty
+        def version():
+            return SBDebugger.GetVersionString()
+    %}
+#endif
 }
diff --git a/lldb/cmake/modules/LLDBFramework.cmake b/lldb/cmake/modules/LLDBFramework.cmake
index 70010ffbf738..bbd717a982cf 100644
--- a/lldb/cmake/modules/LLDBFramework.cmake
+++ b/lldb/cmake/modules/LLDBFramework.cmake
@@ -112,7 +112,8 @@ file(GLOB lldb_framework_header_staging_list ${lldb_framework_header_staging}/*)
 foreach(header ${lldb_framework_header_staging_list})
 
   set(input_header ${header})
-  set(output_header $<TARGET_FILE_DIR:liblldb>/Headers/${input_header})
+  get_filename_component(header_basename ${input_header} NAME)
+  set(output_header $<TARGET_FILE_DIR:liblldb>/Headers/${header_basename})
 
   add_custom_command(TARGET liblldb POST_BUILD
     COMMAND ${LLDB_SOURCE_DIR}/scripts/framework-header-fix.py -f lldb_main -i ${input_header} -o ${output_header} -p ${unifdef_EXECUTABLE} USWIG
diff --git a/lldb/docs/index.rst b/lldb/docs/index.rst
index 1ffdb08a1ca2..d49d0d8d5a73 100644
--- a/lldb/docs/index.rst
+++ b/lldb/docs/index.rst
@@ -135,6 +135,7 @@ interesting areas to contribute to lldb.
    use/ondemand
    use/aarch64-linux
    use/symbolfilejson
+   use/mcp
    use/troubleshooting
    use/links
    Man Page <man/lldb>
diff --git a/lldb/docs/resources/contributing.rst b/lldb/docs/resources/contributing.rst
index e7d46c9061d5..7e84df41c3fb 100644
--- a/lldb/docs/resources/contributing.rst
+++ b/lldb/docs/resources/contributing.rst
@@ -104,7 +104,7 @@ making sense for the particular pair of plugins). Unfortunately, due to historic
 reasons, not all plugin dependencies follow this rule, which is why we have
 another category called `LLDB_TOLERATED_PLUGIN_DEPENDENCIES`. New dependencies
 are forbidden (even though they are accepted by CMake) and existing ones should
-be removed whereever possible.
+be removed wherever possible.
 
 .. _Error handling:
 
diff --git a/lldb/docs/resources/lldbgdbremote.md b/lldb/docs/resources/lldbgdbremote.md
index 162815148a18..80c68091ecd0 100644
--- a/lldb/docs/resources/lldbgdbremote.md
+++ b/lldb/docs/resources/lldbgdbremote.md
@@ -1998,6 +1998,23 @@ threads (live system debug) / cores (JTAG) in your program have
 stopped and allows LLDB to display and control your program
 correctly.
 
+## qWatchpointSupportInfo
+
+Get the number of hardware watchpoints available on the remote target.
+
+```
+send packet: $qWatchpointSupportInfo:#55
+read packet: $num:4;#f9
+```
+
+`num` is the number of hardware breakpoints, it will be `0` if none are
+available.
+
+**Priority to Implement:** Low. If this packet is not supported, LLDB will assume
+that hardware breakpoints are supported. If that is not the case, LLDB assumes
+that the debug stub will respond with an error when asked to set a hardware
+watchpoint.
+
 ## Stop reply packet extensions
 
 This section describes some of the additional information you can
diff --git a/lldb/docs/use/mcp.md b/lldb/docs/use/mcp.md
new file mode 100644
index 000000000000..375c164fe771
--- /dev/null
+++ b/lldb/docs/use/mcp.md
@@ -0,0 +1,85 @@
+# Model Context Protocol (MCP)
+
+LLDB supports the [Model Context Protocol](https://modelcontextprotocol.io)
+(MCP). This structured, machine-friendly protocol allows AI models to access
+and interact with external tools, for example debuggers. Using MCP, an AI agent
+can execute LLDB commands to control the debugger: set breakpoints, inspect
+memory, step through code. This can range from helping you run a specific
+command you cannot immediately remember, to a fully agent-driven debugging
+experience.
+
+## MCP Server
+
+To start the MCP server in LLDB, use the `protocol-server start` command.
+Specify `MCP` as the protocol and provide a URI to listen on. For example, to
+start listening for local TCP connections on port `59999`, use the following
+command:
+
+```
+(lldb) protocol-server start MCP listen://localhost:59999
+MCP server started with connection listeners: connection://[::1]:59999, connection://[127.0.0.1]:59999
+```
+
+The server will automatically stop when exiting LLDB, or it can be stopped
+explicitly with the `protocol-server stop` command.
+
+```
+(lldb) protocol-server stop MCP
+```
+
+The commands will fail if a server is already running or not running
+respectively.
+
+## MCP Client
+
+MCP uses standard input/output (stdio) for communication between client and
+server. The exact configuration depends on the client, but most applications
+allow you to specify an MCP server as a binary and arguments. This means that
+you need to use something like `netcat` to connect to LLDB's MCP server and
+forward communication over stdio over the network connection.
+
+```
+┌──────────┐               ┌──────────┐               ┌──────────┐
+│          │               │          │               │          │
+│   LLDB   ├─────socket────┤  netcat  ├─────stdio─────┤MCP Client│
+│          │               │          │               │          │
+└──────────┘               └──────────┘               └──────────┘
+```
+
+Configuration example for [Claude Code](https://modelcontextprotocol.io/quickstart/user):
+
+```json
+{
+  "mcpServers": {
+    "tool": {
+      "command": "/usr/bin/nc",
+      "args": ["localhost", "59999"]
+    }
+  }
+}
+```
+
+Configuration example for [Visual Studio Code](https://code.visualstudio.com/docs/copilot/chat/mcp-servers):
+
+```json
+{
+  "mcp": {
+    "servers": {
+      "lldb": {
+        "type": "stdio",
+        "command": "/usr/bin/nc",
+        "args": ["localhost", "59999"]
+      }
+    }
+  }
+}
+```
+
+### Troubleshooting
+
+The MCP server uses the `Host` log channel. You can enable logging with the
+`log enable` command.
+
+```
+(lldb) log enable lldb host
+```
diff --git a/lldb/include/lldb/API/SBBreakpoint.h b/lldb/include/lldb/API/SBBreakpoint.h
index e08df3b6d5ab..18ed3e7226d3 100644
--- a/lldb/include/lldb/API/SBBreakpoint.h
+++ b/lldb/include/lldb/API/SBBreakpoint.h
@@ -148,6 +148,11 @@ class LLDB_API SBBreakpoint {
 
   bool IsHardware() const;
 
+  /// Make this breakpoint a hardware breakpoint. This will replace all existing
+  /// breakpoint locations with hardware breakpoints. Returns an error if this
+  /// fails, e.g. when there aren't enough hardware resources available.
+  lldb::SBError SetIsHardware(bool is_hardware);
+
   // Can only be called from a ScriptedBreakpointResolver...
   SBError
   AddLocation(SBAddress &address);
diff --git a/lldb/include/lldb/API/SBStatisticsOptions.h b/lldb/include/lldb/API/SBStatisticsOptions.h
index 74bea03eff9c..bfff9dc92643 100644
--- a/lldb/include/lldb/API/SBStatisticsOptions.h
+++ b/lldb/include/lldb/API/SBStatisticsOptions.h
@@ -57,8 +57,7 @@ class LLDB_API SBStatisticsOptions {
   /// a JSON array with all commands the user and/or scripts executed during a
   /// debug session.
   ///
-  /// Defaults to true, unless the `SummaryOnly` mode is enabled, in which case
-  /// this is turned off unless specified.
+  /// Defaults to false.
   void SetIncludeTranscript(bool b);
   bool GetIncludeTranscript() const;
 
diff --git a/lldb/include/lldb/Breakpoint/Breakpoint.h b/lldb/include/lldb/Breakpoint/Breakpoint.h
index f623a2e0c295..b200a1e4893d 100644
--- a/lldb/include/lldb/Breakpoint/Breakpoint.h
+++ b/lldb/include/lldb/Breakpoint/Breakpoint.h
@@ -519,6 +519,8 @@ class Breakpoint : public std::enable_shared_from_this<Breakpoint>,
 
   bool IsHardware() const { return m_hardware; }
 
+  llvm::Error SetIsHardware(bool is_hardware);
+
   lldb::BreakpointResolverSP GetResolver() { return m_resolver_sp; }
 
   lldb::SearchFilterSP GetSearchFilter() { return m_filter_sp; }
diff --git a/lldb/include/lldb/Breakpoint/BreakpointLocation.h b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
index 3592291bb2d0..66274e8825ee 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointLocation.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointLocation.h
@@ -69,7 +69,7 @@ class BreakpointLocation
   // The next section deals with various breakpoint options.
 
   /// If \a enabled is \b true, enable the breakpoint, if \b false disable it.
-  void SetEnabled(bool enabled);
+  bool SetEnabled(bool enabled);
 
   /// Check the Enable/Disable state.
   ///
@@ -235,7 +235,7 @@ class BreakpointLocation
   ///     \b true if the target should stop at this breakpoint and \b
   ///     false not.
   bool InvokeCallback(StoppointCallbackContext *context);
-  
+
   /// Report whether the callback for this location is synchronous or not.
   ///
   /// \return
@@ -371,11 +371,8 @@ class BreakpointLocation
   ///     The thread for which this breakpoint location is valid, or
   ///     LLDB_INVALID_THREAD_ID if it is valid for all threads.
   ///
-  /// \param[in] hardware
-  ///     \b true if a hardware breakpoint is requested.
-
   BreakpointLocation(lldb::break_id_t bid, Breakpoint &owner,
-                     const Address &addr, lldb::tid_t tid, bool hardware,
+                     const Address &addr, lldb::tid_t tid,
                      bool check_for_resolver = true);
 
   // Data members:
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolver.h b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
index a0b37c94ac2b..52cd70e934e6 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolver.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
@@ -148,8 +148,7 @@ class BreakpointResolver : public Searcher {
   enum ResolverTy GetResolverTy() {
     if (SubclassID > ResolverTy::LastKnownResolverType)
       return ResolverTy::UnknownResolver;
-    else
-      return (enum ResolverTy)SubclassID;
+    return (enum ResolverTy)SubclassID;
   }
 
   const char *GetResolverName() { return ResolverTyToName(GetResolverTy()); }
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 9f82466a8341..504f936fe317 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -227,6 +227,8 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   const char *GetIOHandlerHelpPrologue();
 
+  void RefreshIOHandler();
+
   void ClearIOHandlers();
 
   bool EnableLog(llvm::StringRef channel,
@@ -602,10 +604,6 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   void FlushProcessOutput(Process &process, bool flush_stdout,
                           bool flush_stderr);
 
-  void AddProtocolServer(lldb::ProtocolServerSP protocol_server_sp);
-  void RemoveProtocolServer(lldb::ProtocolServerSP protocol_server_sp);
-  lldb::ProtocolServerSP GetProtocolServer(llvm::StringRef protocol) const;
-
   SourceManager::SourceFileCache &GetSourceFileCache() {
     return m_source_file_cache;
   }
@@ -776,8 +774,6 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
   mutable std::mutex m_progress_reports_mutex;
   /// @}
 
-  llvm::SmallVector<lldb::ProtocolServerSP> m_protocol_servers;
-
   std::mutex m_destroy_callback_mutex;
   lldb::callback_token_t m_destroy_callback_next_token = 0;
   struct DestroyCallbackInfo {
diff --git a/lldb/include/lldb/Core/EmulateInstruction.h b/lldb/include/lldb/Core/EmulateInstruction.h
index b459476883fc..a9fd4543cbbc 100644
--- a/lldb/include/lldb/Core/EmulateInstruction.h
+++ b/lldb/include/lldb/Core/EmulateInstruction.h
@@ -22,6 +22,8 @@
 #include "lldb/lldb-private-types.h"
 #include "lldb/lldb-types.h"
 
+#include "llvm/Support/Error.h"
+
 #include <cstddef>
 #include <cstdint>
 
@@ -32,6 +34,38 @@ class RegisterValue;
 class Stream;
 class Target;
 class UnwindPlan;
+class EmulateInstruction;
+
+using BreakpointLocations = std::vector<lldb::addr_t>;
+
+class SingleStepBreakpointLocationsPredictor {
+public:
+  SingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up)
+      : m_emulator_up{std::move(emulator_up)} {}
+
+  virtual BreakpointLocations GetBreakpointLocations(Status &status);
+
+  virtual llvm::Expected<unsigned>
+  GetBreakpointSize([[maybe_unused]] lldb::addr_t bp_addr) {
+    return 4;
+  }
+
+  virtual ~SingleStepBreakpointLocationsPredictor() = default;
+
+protected:
+  // This function retrieves the address of the next instruction as it appears
+  // in the binary file. Essentially, it reads the value of the PC register,
+  // determines the size of the current instruction (where the PC is pointing),
+  // and returns the sum of these two values.
+  lldb::addr_t GetNextInstructionAddress(Status &error);
+
+  lldb::addr_t GetBreakpointLocationAddress(lldb::addr_t entry_pc,
+                                            Status &error);
+
+  std::unique_ptr<EmulateInstruction> m_emulator_up;
+  bool m_emulation_result = false;
+};
 
 /// \class EmulateInstruction EmulateInstruction.h
 /// "lldb/Core/EmulateInstruction.h"
@@ -497,7 +531,19 @@ class EmulateInstruction : public PluginInterface {
   static uint32_t GetInternalRegisterNumber(RegisterContext *reg_ctx,
                                             const RegisterInfo &reg_info);
 
+  static std::unique_ptr<SingleStepBreakpointLocationsPredictor>
+  CreateBreakpointLocationPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up);
+
+  // Helper functions
+  std::optional<lldb::addr_t> ReadPC();
+  bool WritePC(lldb::addr_t addr);
+
 protected:
+  using BreakpointLocationsPredictorCreator =
+      std::function<std::unique_ptr<SingleStepBreakpointLocationsPredictor>(
+          std::unique_ptr<EmulateInstruction>)>;
+
   ArchSpec m_arch;
   void *m_baton = nullptr;
   ReadMemoryCallback m_read_mem_callback = &ReadMemoryDefault;
@@ -508,6 +554,21 @@ class EmulateInstruction : public PluginInterface {
   Opcode m_opcode;
 
 private:
+  virtual BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() {
+    if (!m_arch.IsMIPS() && !m_arch.GetTriple().isPPC64() &&
+        !m_arch.GetTriple().isLoongArch()) {
+      // Unsupported architecture
+      return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+        return nullptr;
+      };
+    }
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<SingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
+
   // For EmulateInstruction only
   EmulateInstruction(const EmulateInstruction &) = delete;
   const EmulateInstruction &operator=(const EmulateInstruction &) = delete;
diff --git a/lldb/include/lldb/Core/IOHandler.h b/lldb/include/lldb/Core/IOHandler.h
index 2fb3d7a7c9cc..2672bbe5da2b 100644
--- a/lldb/include/lldb/Core/IOHandler.h
+++ b/lldb/include/lldb/Core/IOHandler.h
@@ -90,6 +90,8 @@ class IOHandler {
 
   virtual void TerminalSizeChanged() {}
 
+  virtual void Refresh() {}
+
   virtual const char *GetPrompt() {
     // Prompt support isn't mandatory
     return nullptr;
@@ -404,6 +406,8 @@ class IOHandlerEditline : public IOHandler {
 
   void PrintAsync(const char *s, size_t len, bool is_stdout) override;
 
+  void Refresh() override;
+
 private:
 #if LLDB_ENABLE_LIBEDIT
   bool IsInputCompleteCallback(Editline *editline, StringList &lines);
diff --git a/lldb/include/lldb/Core/ProtocolServer.h b/lldb/include/lldb/Core/ProtocolServer.h
index fafe46090432..937256c10aec 100644
--- a/lldb/include/lldb/Core/ProtocolServer.h
+++ b/lldb/include/lldb/Core/ProtocolServer.h
@@ -20,8 +20,9 @@ class ProtocolServer : public PluginInterface {
   ProtocolServer() = default;
   virtual ~ProtocolServer() = default;
 
-  static lldb::ProtocolServerSP Create(llvm::StringRef name,
-                                       Debugger &debugger);
+  static ProtocolServer *GetOrCreate(llvm::StringRef name);
+
+  static std::vector<llvm::StringRef> GetSupportedProtocols();
 
   struct Connection {
     Socket::SocketProtocol protocol;
diff --git a/lldb/include/lldb/Core/Statusline.h b/lldb/include/lldb/Core/Statusline.h
index 521b9f2526f6..6bda153f822d 100644
--- a/lldb/include/lldb/Core/Statusline.h
+++ b/lldb/include/lldb/Core/Statusline.h
@@ -36,12 +36,10 @@ class Statusline {
   /// Draw the statusline with the given text.
   void Draw(std::string msg);
 
-  /// Update terminal dimensions.
-  void UpdateTerminalProperties();
-
   enum ScrollWindowMode {
     EnableStatusline,
     DisableStatusline,
+    ResizeStatusline,
   };
 
   /// Set the scroll window for the given mode.
diff --git a/lldb/include/lldb/DataFormatters/FormattersHelpers.h b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
index 42699d0a0b1b..6a47cf37eac6 100644
--- a/lldb/include/lldb/DataFormatters/FormattersHelpers.h
+++ b/lldb/include/lldb/DataFormatters/FormattersHelpers.h
@@ -55,6 +55,14 @@ void AddFilter(TypeCategoryImpl::SharedPointer category_sp,
 
 std::optional<size_t> ExtractIndexFromString(const char *item_name);
 
+/// Prints the summary for the pointer value of a C++
+/// std::unique_ptr/std::shared_ptr/std::weak_ptr.
+void DumpCxxSmartPtrPointerSummary(Stream &stream, ValueObject &ptr,
+                                   const TypeSummaryOptions &options);
+
+bool ContainerSizeSummaryProvider(ValueObject &valobj, Stream &stream,
+                                  const TypeSummaryOptions &options);
+
 Address GetArrayAddressOrPointerValue(ValueObject &valobj);
 
 time_t GetOSXEpoch();
diff --git a/lldb/include/lldb/Expression/DWARFExpression.h b/lldb/include/lldb/Expression/DWARFExpression.h
index 0adbe3e8df2e..37853c0b5a8f 100644
--- a/lldb/include/lldb/Expression/DWARFExpression.h
+++ b/lldb/include/lldb/Expression/DWARFExpression.h
@@ -35,6 +35,8 @@ namespace lldb_private {
 /// location expression or a location list and interprets it.
 class DWARFExpression {
 public:
+  using Stack = std::vector<Value>;
+
   class Delegate {
   public:
     Delegate() = default;
@@ -53,7 +55,7 @@ class DWARFExpression {
     virtual bool ParseVendorDWARFOpcode(uint8_t op,
                                         const DataExtractor &opcodes,
                                         lldb::offset_t &offset,
-                                        std::vector<Value> &stack) const = 0;
+                                        Stack &stack) const = 0;
 
     Delegate(const Delegate &) = delete;
     Delegate &operator=(const Delegate &) = delete;
diff --git a/lldb/include/lldb/Expression/IRMemoryMap.h b/lldb/include/lldb/Expression/IRMemoryMap.h
index abec5442793c..58b95c56c1c3 100644
--- a/lldb/include/lldb/Expression/IRMemoryMap.h
+++ b/lldb/include/lldb/Expression/IRMemoryMap.h
@@ -12,6 +12,7 @@
 #include "lldb/Utility/DataBufferHeap.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-public.h"
+#include "llvm/Support/Error.h"
 
 #include <map>
 
@@ -50,8 +51,13 @@ class IRMemoryMap {
                                  ///only in the process.
   };
 
-  lldb::addr_t Malloc(size_t size, uint8_t alignment, uint32_t permissions,
-                      AllocationPolicy policy, bool zero_memory, Status &error);
+  // If 'policy' is 'eAllocationPolicyMirror' but it is impossible to allocate
+  // memory in the process, 'eAllocationPolicyHostOnly' will be used instead.
+  // The actual policy is returned via 'used_policy'.
+  llvm::Expected<lldb::addr_t> Malloc(size_t size, uint8_t alignment,
+                                      uint32_t permissions,
+                                      AllocationPolicy policy, bool zero_memory,
+                                      AllocationPolicy *used_policy = nullptr);
   void Leak(lldb::addr_t process_address, Status &error);
   void Free(lldb::addr_t process_address, Status &error);
 
diff --git a/lldb/include/lldb/Host/Editline.h b/lldb/include/lldb/Host/Editline.h
index c202a76758e1..947ad3bfe5ec 100644
--- a/lldb/include/lldb/Host/Editline.h
+++ b/lldb/include/lldb/Host/Editline.h
@@ -267,6 +267,8 @@ class Editline {
 
   size_t GetTerminalHeight() { return m_terminal_height; }
 
+  void Refresh();
+
 private:
   /// Sets the lowest line number for multi-line editing sessions.  A value of
   /// zero suppresses line number printing in the prompt.
diff --git a/lldb/include/lldb/Host/PipeBase.h b/lldb/include/lldb/Host/PipeBase.h
index ed8df6bf1e51..6a37f3f2445b 100644
--- a/lldb/include/lldb/Host/PipeBase.h
+++ b/lldb/include/lldb/Host/PipeBase.h
@@ -21,18 +21,14 @@ class PipeBase {
 public:
   virtual ~PipeBase();
 
-  virtual Status CreateNew(bool child_process_inherit) = 0;
-  virtual Status CreateNew(llvm::StringRef name,
-                           bool child_process_inherit) = 0;
+  virtual Status CreateNew() = 0;
+  virtual Status CreateNew(llvm::StringRef name) = 0;
   virtual Status CreateWithUniqueName(llvm::StringRef prefix,
-                                      bool child_process_inherit,
                                       llvm::SmallVectorImpl<char> &name) = 0;
 
-  virtual Status OpenAsReader(llvm::StringRef name,
-                              bool child_process_inherit) = 0;
+  virtual Status OpenAsReader(llvm::StringRef name) = 0;
 
   virtual llvm::Error OpenAsWriter(llvm::StringRef name,
-                                   bool child_process_inherit,
                                    const Timeout<std::micro> &timeout) = 0;
 
   virtual bool CanRead() const = 0;
diff --git a/lldb/include/lldb/Host/posix/HostInfoPosix.h b/lldb/include/lldb/Host/posix/HostInfoPosix.h
index 779b67bd66ed..c4a7f13674f4 100644
--- a/lldb/include/lldb/Host/posix/HostInfoPosix.h
+++ b/lldb/include/lldb/Host/posix/HostInfoPosix.h
@@ -39,6 +39,8 @@ class HostInfoPosix : public HostInfoBase {
   static llvm::VersionTuple GetOSVersion();
   static std::optional<std::string> GetOSBuildString();
 
+  static llvm::Expected<llvm::StringRef> GetSDKRoot(SDKOptions options);
+
 protected:
   static bool ComputeSupportExeDirectory(FileSpec &file_spec);
   static bool ComputeHeaderDirectory(FileSpec &file_spec);
diff --git a/lldb/include/lldb/Host/posix/PipePosix.h b/lldb/include/lldb/Host/posix/PipePosix.h
index effd33fba7eb..0bec2061f316 100644
--- a/lldb/include/lldb/Host/posix/PipePosix.h
+++ b/lldb/include/lldb/Host/posix/PipePosix.h
@@ -32,14 +32,12 @@ class PipePosix : public PipeBase {
 
   ~PipePosix() override;
 
-  Status CreateNew(bool child_process_inherit) override;
-  Status CreateNew(llvm::StringRef name, bool child_process_inherit) override;
+  Status CreateNew() override;
+  Status CreateNew(llvm::StringRef name) override;
   Status CreateWithUniqueName(llvm::StringRef prefix,
-                              bool child_process_inherit,
                               llvm::SmallVectorImpl<char> &name) override;
-  Status OpenAsReader(llvm::StringRef name,
-                      bool child_process_inherit) override;
-  llvm::Error OpenAsWriter(llvm::StringRef name, bool child_process_inherit,
+  Status OpenAsReader(llvm::StringRef name) override;
+  llvm::Error OpenAsWriter(llvm::StringRef name,
                            const Timeout<std::micro> &timeout) override;
 
   bool CanRead() const override;
diff --git a/lldb/include/lldb/Host/windows/MainLoopWindows.h b/lldb/include/lldb/Host/windows/MainLoopWindows.h
index 3937a24645d9..705e7e78ba48 100644
--- a/lldb/include/lldb/Host/windows/MainLoopWindows.h
+++ b/lldb/include/lldb/Host/windows/MainLoopWindows.h
@@ -17,6 +17,8 @@
 
 namespace lldb_private {
 
+using handle_t = void *;
+
 // Windows-specific implementation of the MainLoopBase class. It can monitor
 // socket descriptors for readability using WSAEventSelect. Non-socket file
 // descriptors are not supported.
@@ -31,17 +33,30 @@ class MainLoopWindows : public MainLoopBase {
 
   Status Run() override;
 
+  class IOEvent {
+  public:
+    IOEvent(handle_t event) : m_event(event) {}
+    virtual ~IOEvent() {}
+    virtual void WillPoll() {}
+    virtual void DidPoll() {}
+    virtual void Disarm() {}
+    handle_t GetHandle() { return m_event; }
+
+  protected:
+    handle_t m_event;
+  };
+  using IOEventUP = std::unique_ptr<IOEvent>;
+
 protected:
   void UnregisterReadObject(IOObject::WaitableHandle handle) override;
 
   void Interrupt() override;
 
 private:
-  void ProcessReadObject(IOObject::WaitableHandle handle);
   llvm::Expected<size_t> Poll();
 
   struct FdInfo {
-    void *event;
+    IOEventUP event;
     Callback callback;
   };
   llvm::DenseMap<IOObject::WaitableHandle, FdInfo> m_read_fds;
diff --git a/lldb/include/lldb/Host/windows/PipeWindows.h b/lldb/include/lldb/Host/windows/PipeWindows.h
index 9cf591a2d462..a8bd3cecb9af 100644
--- a/lldb/include/lldb/Host/windows/PipeWindows.h
+++ b/lldb/include/lldb/Host/windows/PipeWindows.h
@@ -29,16 +29,14 @@ class PipeWindows : public PipeBase {
   ~PipeWindows() override;
 
   // Create an unnamed pipe.
-  Status CreateNew(bool child_process_inherit) override;
+  Status CreateNew() override;
 
   // Create a named pipe.
-  Status CreateNew(llvm::StringRef name, bool child_process_inherit) override;
+  Status CreateNew(llvm::StringRef name) override;
   Status CreateWithUniqueName(llvm::StringRef prefix,
-                              bool child_process_inherit,
                               llvm::SmallVectorImpl<char> &name) override;
-  Status OpenAsReader(llvm::StringRef name,
-                      bool child_process_inherit) override;
-  llvm::Error OpenAsWriter(llvm::StringRef name, bool child_process_inherit,
+  Status OpenAsReader(llvm::StringRef name) override;
+  llvm::Error OpenAsWriter(llvm::StringRef name,
                            const Timeout<std::micro> &timeout) override;
 
   bool CanRead() const override;
@@ -72,8 +70,7 @@ class PipeWindows : public PipeBase {
   HANDLE GetWriteNativeHandle();
 
 private:
-  Status OpenNamedPipe(llvm::StringRef name, bool child_process_inherit,
-                       bool is_read);
+  Status OpenNamedPipe(llvm::StringRef name, bool is_read);
 
   HANDLE m_read;
   HANDLE m_write;
diff --git a/lldb/include/lldb/Symbol/ObjectFile.h b/lldb/include/lldb/Symbol/ObjectFile.h
index 43567592dd44..1b9ae1fb31a6 100644
--- a/lldb/include/lldb/Symbol/ObjectFile.h
+++ b/lldb/include/lldb/Symbol/ObjectFile.h
@@ -18,6 +18,7 @@
 #include "lldb/Utility/Endian.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-private.h"
 #include "llvm/Support/Threading.h"
@@ -544,9 +545,9 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
     return false;
   }
 
-  /// Get metadata about threads from the corefile.
+  /// Get metadata about thread ids from the corefile.
   ///
-  /// The corefile may have metadata (e.g. a Mach-O "thread extrainfo"
+  /// The corefile may have metadata (e.g. a Mach-O "process metadata"
   /// LC_NOTE) which for the threads in the process; this method tries
   /// to retrieve them.
   ///
@@ -568,6 +569,18 @@ class ObjectFile : public std::enable_shared_from_this<ObjectFile>,
     return false;
   }
 
+  /// Get process metadata from the corefile in a StructuredData dictionary.
+  ///
+  /// The corefile may have notes (e.g. a Mach-O "process metadata" LC_NOTE)
+  /// which provide metadata about the process and threads in a JSON or
+  /// similar format.
+  ///
+  /// \return
+  ///     A StructuredData object with the metadata in the note, if there is
+  ///     one.  An empty shared pointer is returned if not metadata is found,
+  ///     or a problem parsing it.
+  virtual StructuredData::ObjectSP GetCorefileProcessMetadata() { return {}; }
+
   virtual lldb::RegisterContextSP
   GetThreadContextAtIndex(uint32_t idx, lldb_private::Thread &thread) {
     return lldb::RegisterContextSP();
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index 75c7f230ddf3..b0b608d0a5e7 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -472,6 +472,14 @@ class SymbolFile : public PluginInterface {
     return false;
   };
 
+  /// Get number of loaded/parsed DWO files. This is emitted in "statistics
+  /// dump"
+  ///
+  /// \returns
+  ///     A pair containing (loaded_dwo_count, total_dwo_count). If this
+  ///     symbol file doesn't support DWO files, both counts will be 0.
+  virtual std::pair<uint32_t, uint32_t> GetDwoFileCounts() { return {0, 0}; }
+
   virtual lldb::TypeSP
   MakeType(lldb::user_id_t uid, ConstString name,
            std::optional<uint64_t> byte_size, SymbolContextScope *context,
diff --git a/lldb/include/lldb/Target/MemoryTagManager.h b/lldb/include/lldb/Target/MemoryTagManager.h
index 6bd4180fff70..5b7219692d77 100644
--- a/lldb/include/lldb/Target/MemoryTagManager.h
+++ b/lldb/include/lldb/Target/MemoryTagManager.h
@@ -122,11 +122,15 @@ class MemoryTagManager {
   //
   // 'reader' will always be a wrapper around a CoreFile in real use
   // but allows testing without having to mock a CoreFile.
+  //
+  // This call will fail in the case that the core file segment does not contain
+  // enough data to read all the tags.
   typedef std::function<size_t(lldb::offset_t, size_t, void *)> CoreReaderFn;
-  std::vector<lldb::addr_t> virtual UnpackTagsFromCoreFileSegment(
-      CoreReaderFn reader, lldb::addr_t tag_segment_virtual_address,
-      lldb::addr_t tag_segment_data_address, lldb::addr_t addr,
-      size_t len) const = 0;
+  llvm::
+      Expected<std::vector<lldb::addr_t>> virtual UnpackTagsFromCoreFileSegment(
+          CoreReaderFn reader, lldb::addr_t tag_segment_virtual_address,
+          lldb::addr_t tag_segment_data_address, lldb::addr_t addr,
+          size_t len) const = 0;
 
   // Pack uncompressed tags into their storage format (e.g. for gdb QMemTags).
   // Checks that each tag is within the expected value range.
diff --git a/lldb/include/lldb/Target/Statistics.h b/lldb/include/lldb/Target/Statistics.h
index 2d0d25cd3c75..55dff8861a9a 100644
--- a/lldb/include/lldb/Target/Statistics.h
+++ b/lldb/include/lldb/Target/Statistics.h
@@ -153,6 +153,8 @@ struct ModuleStats {
   bool symtab_stripped = false;
   bool debug_info_had_variable_errors = false;
   bool debug_info_had_incomplete_types = false;
+  uint32_t dwo_file_count = 0;
+  uint32_t loaded_dwo_file_count = 0;
 };
 
 struct ConstStringStats {
@@ -189,11 +191,7 @@ struct StatisticsOptions {
 
   void SetIncludeTranscript(bool value) { m_include_transcript = value; }
   bool GetIncludeTranscript() const {
-    if (m_include_transcript.has_value())
-      return m_include_transcript.value();
-    // `m_include_transcript` has no value set, so return a value based on
-    // `m_summary_only`.
-    return !GetSummaryOnly();
+    return m_include_transcript.value_or(false);
   }
 
   void SetIncludePlugins(bool value) { m_include_plugins = value; }
diff --git a/lldb/include/lldb/Utility/Broadcaster.h b/lldb/include/lldb/Utility/Broadcaster.h
index c6f63f191657..95fbb403f466 100644
--- a/lldb/include/lldb/Utility/Broadcaster.h
+++ b/lldb/include/lldb/Utility/Broadcaster.h
@@ -9,11 +9,11 @@
 #ifndef LLDB_UTILITY_BROADCASTER_H
 #define LLDB_UTILITY_BROADCASTER_H
 
-#include "lldb/Utility/ConstString.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-forward.h"
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 
 #include <cstdint>
 #include <map>
diff --git a/lldb/include/lldb/Utility/Event.h b/lldb/include/lldb/Utility/Event.h
index 4f58f257d4a2..103da8206d90 100644
--- a/lldb/include/lldb/Utility/Event.h
+++ b/lldb/include/lldb/Utility/Event.h
@@ -10,7 +10,6 @@
 #define LLDB_UTILITY_EVENT_H
 
 #include "lldb/Utility/Broadcaster.h"
-#include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Predicate.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/lldb-defines.h"
diff --git a/lldb/include/lldb/Utility/IOObject.h b/lldb/include/lldb/Utility/IOObject.h
index 8cf42992e7be..de6532a63708 100644
--- a/lldb/include/lldb/Utility/IOObject.h
+++ b/lldb/include/lldb/Utility/IOObject.h
@@ -14,6 +14,7 @@
 #include <sys/types.h>
 
 #include "lldb/lldb-private.h"
+#include "lldb/lldb-types.h"
 
 namespace lldb_private {
 
@@ -24,9 +25,9 @@ class IOObject {
     eFDTypeSocket, // Socket requiring send/recv
   };
 
-  // TODO: On Windows this should be a HANDLE, and wait should use
-  // WaitForMultipleObjects
-  typedef int WaitableHandle;
+  // A handle for integrating with the host event loop model.
+  using WaitableHandle = lldb::file_t;
+
   static const WaitableHandle kInvalidHandleValue;
 
   IOObject(FDType type) : m_fd_type(type) {}
diff --git a/lldb/include/lldb/ValueObject/DILEval.h b/lldb/include/lldb/ValueObject/DILEval.h
index 2a0cb548a810..45e29b3ddcd7 100644
--- a/lldb/include/lldb/ValueObject/DILEval.h
+++ b/lldb/include/lldb/ValueObject/DILEval.h
@@ -25,8 +25,7 @@ namespace lldb_private::dil {
 /// evaluating).
 lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
                                      std::shared_ptr<StackFrame> frame_sp,
-                                     lldb::DynamicValueType use_dynamic,
-                                     CompilerType *scope_ptr = nullptr);
+                                     lldb::DynamicValueType use_dynamic);
 
 /// Given the name of an identifier, check to see if it matches the name of a
 /// global variable. If so, find the ValueObject for that global variable, and
@@ -35,8 +34,7 @@ lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
 lldb::ValueObjectSP LookupGlobalIdentifier(llvm::StringRef name_ref,
                                            std::shared_ptr<StackFrame> frame_sp,
                                            lldb::TargetSP target_sp,
-                                           lldb::DynamicValueType use_dynamic,
-                                           CompilerType *scope_ptr = nullptr);
+                                           lldb::DynamicValueType use_dynamic);
 
 class Interpreter : Visitor {
 public:
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 558818e8e230..2bc85a2d2afa 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -391,7 +391,7 @@ typedef std::shared_ptr<lldb_private::Platform> PlatformSP;
 typedef std::shared_ptr<lldb_private::Process> ProcessSP;
 typedef std::shared_ptr<lldb_private::ProcessAttachInfo> ProcessAttachInfoSP;
 typedef std::shared_ptr<lldb_private::ProcessLaunchInfo> ProcessLaunchInfoSP;
-typedef std::shared_ptr<lldb_private::ProtocolServer> ProtocolServerSP;
+typedef std::unique_ptr<lldb_private::ProtocolServer> ProtocolServerUP;
 typedef std::weak_ptr<lldb_private::Process> ProcessWP;
 typedef std::shared_ptr<lldb_private::RegisterCheckpoint> RegisterCheckpointSP;
 typedef std::shared_ptr<lldb_private::RegisterContext> RegisterContextSP;
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index 34eaaa8e581e..249b25c251ac 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -81,8 +81,7 @@ typedef lldb::PlatformSP (*PlatformCreateInstance)(bool force,
 typedef lldb::ProcessSP (*ProcessCreateInstance)(
     lldb::TargetSP target_sp, lldb::ListenerSP listener_sp,
     const FileSpec *crash_file_path, bool can_connect);
-typedef lldb::ProtocolServerSP (*ProtocolServerCreateInstance)(
-    Debugger &debugger);
+typedef lldb::ProtocolServerUP (*ProtocolServerCreateInstance)();
 typedef lldb::RegisterTypeBuilderSP (*RegisterTypeBuilderCreateInstance)(
     Target &target);
 typedef lldb::ScriptInterpreterSP (*ScriptInterpreterCreateInstance)(
diff --git a/lldb/include/lldb/lldb-types.h b/lldb/include/lldb/lldb-types.h
index 15781ad626ef..e309fc8833ce 100644
--- a/lldb/include/lldb/lldb-types.h
+++ b/lldb/include/lldb/lldb-types.h
@@ -42,7 +42,7 @@ typedef void *rwlock_t;
 typedef void *process_t;                          // Process type is HANDLE
 typedef void *thread_t;                           // Host thread type
 typedef void *file_t;                             // Host file type
-typedef unsigned int __w64 socket_t;              // Host socket type
+typedef uintptr_t socket_t;                       // Host socket type
 typedef void *thread_arg_t;                       // Host thread argument type
 typedef unsigned thread_result_t;                 // Host thread result type
 typedef thread_result_t (*thread_func_t)(void *); // Host thread function type
diff --git a/lldb/packages/Python/lldbsuite/test/builders/builder.py b/lldb/packages/Python/lldbsuite/test/builders/builder.py
index de0573246944..ada6f9ff4a54 100644
--- a/lldb/packages/Python/lldbsuite/test/builders/builder.py
+++ b/lldb/packages/Python/lldbsuite/test/builders/builder.py
@@ -247,13 +247,25 @@ def getLLDBObjRoot(self):
     def _getDebugInfoArgs(self, debug_info):
         if debug_info is None:
             return []
-        if debug_info == "dwarf":
-            return ["MAKE_DSYM=NO"]
-        if debug_info == "dwo":
-            return ["MAKE_DSYM=NO", "MAKE_DWO=YES"]
-        if debug_info == "gmodules":
-            return ["MAKE_DSYM=NO", "MAKE_GMODULES=YES"]
-        return None
+
+        debug_options = debug_info if isinstance(debug_info, list) else [debug_info]
+        option_flags = {
+            "dwarf": {"MAKE_DSYM": "NO"},
+            "dwo": {"MAKE_DSYM": "NO", "MAKE_DWO": "YES"},
+            "gmodules": {"MAKE_DSYM": "NO", "MAKE_GMODULES": "YES"},
+            "debug_names": {"MAKE_DEBUG_NAMES": "YES"},
+            "dwp": {"MAKE_DSYM": "NO", "MAKE_DWP": "YES"},
+        }
+
+        # Collect all flags, with later options overriding earlier ones
+        flags = {}
+
+        for option in debug_options:
+            if not option or option not in option_flags:
+                return None  # Invalid options
+            flags.update(option_flags[option])
+
+        return [f"{key}={value}" for key, value in flags.items()]
 
     def getBuildCommand(
         self,
diff --git a/lldb/packages/Python/lldbsuite/test/gdbclientutils.py b/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
index 753de22b9cfe..b603c35c8df0 100644
--- a/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
+++ b/lldb/packages/Python/lldbsuite/test/gdbclientutils.py
@@ -92,6 +92,9 @@ class MockGDBServerResponder:
     class RESPONSE_DISCONNECT:
         pass
 
+    class RESPONSE_NONE:
+        pass
+
     def __init__(self):
         self.packetLog = []
 
@@ -181,6 +184,8 @@ def respond(self, packet):
             return self.qQueryGDBServer()
         if packet == "qHostInfo":
             return self.qHostInfo()
+        if packet.startswith("qEcho"):
+            return self.qEcho(int(packet.split(":")[1]))
         if packet == "qGetWorkingDir":
             return self.qGetWorkingDir()
         if packet == "qOffsets":
@@ -237,6 +242,9 @@ def qProcessInfo(self):
     def qHostInfo(self):
         return "ptrsize:8;endian:little;"
 
+    def qEcho(self):
+        return "E04"
+
     def qQueryGDBServer(self):
         return "E04"
 
@@ -655,6 +663,8 @@ def _handlePacket(self, packet):
         if not isinstance(response, list):
             response = [response]
         for part in response:
+            if part is MockGDBServerResponder.RESPONSE_NONE:
+                continue
             if part is MockGDBServerResponder.RESPONSE_DISCONNECT:
                 raise self.TerminateConnectionException()
             self._sendPacket(part)
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 27e004003437..8112705438c1 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -7,6 +7,7 @@
 # System modules
 import errno
 import io
+import json
 import os
 import re
 import sys
@@ -1704,3 +1705,89 @@ def packetlog_get_dylib_info(log):
                 expect_dylib_info_response = True
 
     return dylib_info
+
+
+# ========================
+# Utilities for simulators
+# ========================
+
+
+def get_latest_apple_simulator(platform_name, log=None):
+    # Run simctl to list all simulators
+    cmd = ["xcrun", "simctl", "list", "-j", "devices"]
+    cmd_str = " ".join(cmd)
+    if log:
+        log(cmd_str)
+    sim_devices_str = subprocess.check_output(cmd).decode("utf-8")
+    sim_devices = json.loads(sim_devices_str)["devices"]
+
+    # Find an available simulator for the requested platform
+    device_uuid = None
+    device_runtime = None
+    for simulator in sim_devices:
+        if isinstance(simulator, dict):
+            runtime = simulator["name"]
+            devices = simulator["devices"]
+        else:
+            runtime = simulator
+            devices = sim_devices[simulator]
+        if not platform_name in runtime.lower():
+            continue
+        for device in devices:
+            if "availability" in device and device["availability"] != "(available)":
+                continue
+            if "isAvailable" in device and not device["isAvailable"]:
+                continue
+            if device_runtime and runtime < device_runtime:
+                continue
+            device_uuid = device["udid"]
+            device_runtime = runtime
+            # Stop searching in this runtime
+            break
+
+    return device_uuid
+
+
+def launch_exe_in_apple_simulator(
+    device_uuid,
+    exe_path,
+    exe_args=[],
+    stderr_lines_to_read=0,
+    stderr_patterns=[],
+    log=None,
+):
+    exe_path = os.path.realpath(exe_path)
+    cmd = [
+        "xcrun",
+        "simctl",
+        "spawn",
+        "-s",
+        device_uuid,
+        exe_path,
+    ] + exe_args
+    if log:
+        log(" ".join(cmd))
+    sim_launcher = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+
+    # Read stderr to try to find matches.
+    # Each pattern will return the value of group[1] of the first match in the stderr.
+    # Will read at most stderr_lines_to_read lines.
+    # Will early terminate when all matches have been found.
+    total_patterns = len(stderr_patterns)
+    matches_found = 0
+    matched_strings = [None] * total_patterns
+    for _ in range(0, stderr_lines_to_read):
+        stderr = sim_launcher.stderr.readline().decode("utf-8")
+        if not stderr:
+            continue
+        for i, pattern in enumerate(stderr_patterns):
+            if matched_strings[i] is not None:
+                continue
+            match = re.match(pattern, stderr)
+            if match:
+                matched_strings[i] = str(match.group(1))
+                matches_found += 1
+        if matches_found == total_patterns:
+            break
+
+    return exe_path, matched_strings
diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 06959f226066..58833e1b0cc7 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -276,6 +276,10 @@ ifeq "$(MAKE_DWO)" "YES"
 	CFLAGS += -gsplit-dwarf
 endif
 
+ifeq "$(MAKE_DEBUG_NAMES)" "YES"
+	CFLAGS += -gpubnames
+endif
+
 ifeq "$(USE_PRIVATE_MODULE_CACHE)" "YES"
 THE_CLANG_MODULE_CACHE_DIR := $(BUILDDIR)/private-module-cache
 else
diff --git a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
index 6dc53a4e88ff..35649b10ce45 100644
--- a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
+++ b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
@@ -58,7 +58,7 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>,
 
   _T1 &first() { return static_cast<_Base1 &>(*this).__get(); }
 };
-#elif COMPRESSED_PAIR_REV == 1
+#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2
 // From libc++ datasizeof.h
 template <class _Tp> struct _FirstPaddingByte {
   _LLDB_NO_UNIQUE_ADDRESS _Tp __v_;
@@ -72,6 +72,9 @@ inline const size_t __datasizeof_v =
 template <class _Tp>
 struct __lldb_is_final : public integral_constant<bool, __is_final(_Tp)> {};
 
+// The legacy layout has been patched, see
+// https://github.com/llvm/llvm-project/pull/142516.
+#if COMPRESSED_PAIR_REV == 1
 template <class _ToPad> class __compressed_pair_padding {
   char __padding_[((is_empty<_ToPad>::value &&
                     !__lldb_is_final<_ToPad>::value) ||
@@ -79,6 +82,25 @@ template <class _ToPad> class __compressed_pair_padding {
                       ? 0
                       : sizeof(_ToPad) - __datasizeof_v<_ToPad>];
 };
+#else
+template <class _ToPad>
+inline const bool __is_reference_or_unpadded_object =
+    (std::is_empty<_ToPad>::value && !__lldb_is_final<_ToPad>::value) ||
+    sizeof(_ToPad) == __datasizeof_v<_ToPad>;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &> = true;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &&> = true;
+
+template <class _ToPad, bool _Empty = __is_reference_or_unpadded_object<_ToPad>>
+class __compressed_pair_padding {
+  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
+};
+
+template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {};
+#endif
 
 #define _LLDB_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)              \
   [[__gnu__::__aligned__(                                                      \
@@ -96,7 +118,7 @@ template <class _ToPad> class __compressed_pair_padding {
   _LLDB_NO_UNIQUE_ADDRESS __compressed_pair_padding<T2> __padding2_;           \
   _LLDB_NO_UNIQUE_ADDRESS T3 Initializer3;                                     \
   _LLDB_NO_UNIQUE_ADDRESS __compressed_pair_padding<T3> __padding3_;
-#elif COMPRESSED_PAIR_REV == 2
+#elif COMPRESSED_PAIR_REV == 3
 #define _LLDB_COMPRESSED_PAIR(T1, Name1, T2, Name2)                            \
   _LLDB_NO_UNIQUE_ADDRESS T1 Name1;                                            \
   _LLDB_NO_UNIQUE_ADDRESS T2 Name2
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index 3b54d598c350..6299caf7631a 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -344,7 +344,12 @@ def stepOver(
         granularity="statement",
         timeout=DEFAULT_TIMEOUT,
     ):
-        self.dap_server.request_next(threadId=threadId, granularity=granularity)
+        response = self.dap_server.request_next(
+            threadId=threadId, granularity=granularity
+        )
+        self.assertTrue(
+            response["success"], f"next request failed: response {response}"
+        )
         if waitForStop:
             return self.dap_server.wait_for_stopped(timeout)
         return None
diff --git a/lldb/scripts/framework-header-fix.py b/lldb/scripts/framework-header-fix.py
index 9528fdb7e30b..6ea8df4c24dd 100755
--- a/lldb/scripts/framework-header-fix.py
+++ b/lldb/scripts/framework-header-fix.py
@@ -20,7 +20,7 @@
 
 # Main header regexes
 INCLUDE_FILENAME_REGEX = re.compile(
-    r'#include "lldb/API/(?P<include_filename>.*){0,1}"'
+    r'#include "lldb/(API/)?(?P<include_filename>.*){0,1}"'
 )
 
 # RPC header regexes
@@ -70,7 +70,7 @@ def modify_main_includes(input_file_path, output_file_path):
                     r"#include <LLDB/" + match.group("include_filename") + ">",
                     file_buffer,
                 )
-                output_file.write(file_buffer)
+            output_file.write(file_buffer)
 
 
 def remove_guards(output_file_path, unifdef_path, unifdef_guards):
@@ -113,6 +113,10 @@ def main():
     # arguments in and of themself, so they need to passed in without dashes.
     unifdef_guards = ["-" + guard for guard in args.unifdef_guards]
 
+    # Create the framework's header dir if it doesn't already exist
+    if not os.path.exists(os.path.dirname(output_file_path)):
+        os.makedirs(os.path.dirname(output_file_path))
+
     if framework_version == "lldb_main":
         modify_main_includes(input_file_path, output_file_path)
     if framework_version == "lldb_rpc":
diff --git a/lldb/scripts/version-header-fix.py b/lldb/scripts/version-header-fix.py
index fb26ee1579e6..98457e6f5b3c 100755
--- a/lldb/scripts/version-header-fix.py
+++ b/lldb/scripts/version-header-fix.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 """
-Usage: <path/to/input-header.h> <path/to/output-header.h> LLDB_MAJOR_VERSION LLDB_MINOR_VERSION LLDB_PATCH_VERSION
+Usage: -i <path/to/input-header.h> -o <path/to/output-header.h> -m LLDB_MAJOR_VERSION -n LLDB_MINOR_VERSION -p LLDB_PATCH_VERSION
 
-This script uncomments and populates the versioning information in lldb-defines.h
+This script uncomments and populates the versioning information in lldb-defines.h. Note that the LLDB version numbering looks like MAJOR.MINOR.PATCH
 """
 
 import argparse
@@ -15,18 +15,19 @@
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("input_path")
-    parser.add_argument("output_path")
-    parser.add_argument("lldb_version_major")
-    parser.add_argument("lldb_version_minor")
-    parser.add_argument("lldb_version_patch")
+    parser = argparse.ArgumentParser(
+        description="This script uncomments and populates the versioning information in lldb-defines.h. Note that the LLDB version numbering looks like MAJOR.MINOR.PATCH"
+    )
+    parser.add_argument("-i", "--input_path", help="The filepath for the input header.")
+    parser.add_argument(
+        "-o", "--output_path", help="The filepath for the output header."
+    )
+    parser.add_argument("-m", "--major", help="The LLDB version major.")
+    parser.add_argument("-n", "--minor", help="The LLDB version minor.")
+    parser.add_argument("-p", "--patch", help="The LLDB version patch number.")
     args = parser.parse_args()
     input_path = str(args.input_path)
     output_path = str(args.output_path)
-    lldb_version_major = args.lldb_version_major
-    lldb_version_minor = args.lldb_version_minor
-    lldb_version_patch = args.lldb_version_patch
 
     with open(input_path, "r") as input_file:
         lines = input_file.readlines()
@@ -38,19 +39,19 @@ def main():
         # e.g. //#define LLDB_VERSION -> #define LLDB_VERSION <LLDB_MAJOR_VERSION>
         file_buffer = re.sub(
             LLDB_VERSION_REGEX,
-            r"#define LLDB_VERSION " + lldb_version_major,
+            r"#define LLDB_VERSION " + args.major,
             file_buffer,
         )
 
         file_buffer = re.sub(
             LLDB_REVISION_REGEX,
-            r"#define LLDB_REVISION " + lldb_version_patch,
+            r"#define LLDB_REVISION " + args.patch,
             file_buffer,
         )
         file_buffer = re.sub(
             LLDB_VERSION_STRING_REGEX,
             r'#define LLDB_VERSION_STRING "{0}.{1}.{2}"'.format(
-                lldb_version_major, lldb_version_minor, lldb_version_patch
+                args.major, args.minor, args.patch
             ),
             file_buffer,
         )
diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index f4a323aaf267..4751ed319b25 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -326,7 +326,7 @@ foreach(header
 endforeach()
 
 add_custom_command(TARGET liblldb POST_BUILD
-  COMMAND "${Python3_EXECUTABLE}" ${LLDB_SOURCE_DIR}/scripts/version-header-fix.py ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h ${lldb_header_staging_dir}/lldb-defines.h ${LLDB_VERSION_MAJOR} ${LLDB_VERSION_MINOR} ${LLDB_VERSION_PATCH}
+  COMMAND "${Python3_EXECUTABLE}" ${LLDB_SOURCE_DIR}/scripts/version-header-fix.py -i ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h -o ${lldb_header_staging_dir}/lldb-defines.h -m ${LLDB_VERSION_MAJOR} -n ${LLDB_VERSION_MINOR} -p ${LLDB_VERSION_PATCH}
 )
 add_custom_target(liblldb-header-staging DEPENDS ${lldb_staged_headers})
 add_dependencies(liblldb liblldb-header-staging)
diff --git a/lldb/source/API/SBBreakpoint.cpp b/lldb/source/API/SBBreakpoint.cpp
index 87fadbcec4f2..397afc1f10f9 100644
--- a/lldb/source/API/SBBreakpoint.cpp
+++ b/lldb/source/API/SBBreakpoint.cpp
@@ -781,6 +781,18 @@ bool SBBreakpoint::IsHardware() const {
   return false;
 }
 
+lldb::SBError SBBreakpoint::SetIsHardware(bool is_hardware) {
+  LLDB_INSTRUMENT_VA(this, is_hardware);
+
+  BreakpointSP bkpt_sp = GetSP();
+  if (bkpt_sp) {
+    std::lock_guard<std::recursive_mutex> guard(
+        bkpt_sp->GetTarget().GetAPIMutex());
+    return SBError(Status::FromError(bkpt_sp->SetIsHardware(is_hardware)));
+  }
+  return SBError();
+}
+
 BreakpointSP SBBreakpoint::GetSP() const { return m_opaque_wp.lock(); }
 
 // This is simple collection of breakpoint id's and their target.
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index 2ed0c9314e3e..8fc93cc8e0e5 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -61,7 +61,7 @@ Breakpoint::Breakpoint(Target &new_target, const Breakpoint &source_bp)
 Breakpoint::~Breakpoint() = default;
 
 BreakpointSP Breakpoint::CopyFromBreakpoint(TargetSP new_target,
-    const Breakpoint& bp_to_copy_from) {
+                                            const Breakpoint &bp_to_copy_from) {
   if (!new_target)
     return BreakpointSP();
 
@@ -163,7 +163,7 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
         std::make_shared<SearchFilterForUnconstrainedSearches>(target_sp);
   else {
     filter_sp = SearchFilter::CreateFromStructuredData(target_sp, *filter_dict,
-        create_error);
+                                                       create_error);
     if (create_error.Fail()) {
       error = Status::FromErrorStringWithFormat(
           "Error creating breakpoint filter from data: %s.",
@@ -174,7 +174,7 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
 
   std::unique_ptr<BreakpointOptions> options_up;
   StructuredData::Dictionary *options_dict;
-  Target& target = *target_sp;
+  Target &target = *target_sp;
   success = breakpoint_dict->GetValueForKeyAsDictionary(
       BreakpointOptions::GetSerializationKey(), options_dict);
   if (success) {
@@ -192,8 +192,8 @@ lldb::BreakpointSP Breakpoint::CreateFromStructuredData(
   success = breakpoint_dict->GetValueForKeyAsBoolean(
       Breakpoint::GetKey(OptionNames::Hardware), hardware);
 
-  result_sp = target.CreateBreakpoint(filter_sp, resolver_sp, false,
-                                      hardware, true);
+  result_sp =
+      target.CreateBreakpoint(filter_sp, resolver_sp, false, hardware, true);
 
   if (result_sp && options_up) {
     result_sp->m_options = *options_up;
@@ -251,6 +251,45 @@ const lldb::TargetSP Breakpoint::GetTargetSP() {
 
 bool Breakpoint::IsInternal() const { return LLDB_BREAK_ID_IS_INTERNAL(m_bid); }
 
+llvm::Error Breakpoint::SetIsHardware(bool is_hardware) {
+  if (is_hardware == m_hardware)
+    return llvm::Error::success();
+
+  // Disable all non-hardware breakpoint locations.
+  std::vector<BreakpointLocationSP> locations;
+  for (BreakpointLocationSP location_sp : m_locations.BreakpointLocations()) {
+    if (!location_sp || !location_sp->IsEnabled())
+      continue;
+
+    lldb::BreakpointSiteSP breakpoint_site_sp =
+        location_sp->GetBreakpointSite();
+    if (!breakpoint_site_sp ||
+        breakpoint_site_sp->GetType() == BreakpointSite::eHardware)
+      continue;
+
+    locations.push_back(location_sp);
+    location_sp->SetEnabled(false);
+  }
+
+  // Toggle the hardware mode.
+  m_hardware = is_hardware;
+
+  // Re-enable all breakpoint locations.
+  size_t num_failures = 0;
+  for (BreakpointLocationSP location_sp : locations) {
+    if (!location_sp->SetEnabled(true))
+      num_failures++;
+  }
+
+  if (num_failures != 0)
+    return llvm::createStringError(
+        "%ull out of %ull breakpoint locations left disabled because they "
+        "couldn't be converted to hardware",
+        num_failures, locations.size());
+
+  return llvm::Error::success();
+}
+
 BreakpointLocationSP Breakpoint::AddLocation(const Address &addr,
                                              bool *new_location) {
   return m_locations.AddLocation(addr, m_resolve_indirect_symbols,
@@ -347,8 +386,7 @@ void Breakpoint::SetThreadID(lldb::tid_t thread_id) {
 lldb::tid_t Breakpoint::GetThreadID() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return LLDB_INVALID_THREAD_ID;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetTID();
+  return m_options.GetThreadSpecNoCreate()->GetTID();
 }
 
 void Breakpoint::SetThreadIndex(uint32_t index) {
@@ -362,8 +400,7 @@ void Breakpoint::SetThreadIndex(uint32_t index) {
 uint32_t Breakpoint::GetThreadIndex() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return 0;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetIndex();
+  return m_options.GetThreadSpecNoCreate()->GetIndex();
 }
 
 void Breakpoint::SetThreadName(const char *thread_name) {
@@ -378,8 +415,7 @@ void Breakpoint::SetThreadName(const char *thread_name) {
 const char *Breakpoint::GetThreadName() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return nullptr;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetName();
+  return m_options.GetThreadSpecNoCreate()->GetName();
 }
 
 void Breakpoint::SetQueueName(const char *queue_name) {
@@ -394,8 +430,7 @@ void Breakpoint::SetQueueName(const char *queue_name) {
 const char *Breakpoint::GetQueueName() const {
   if (m_options.GetThreadSpecNoCreate() == nullptr)
     return nullptr;
-  else
-    return m_options.GetThreadSpecNoCreate()->GetQueueName();
+  return m_options.GetThreadSpecNoCreate()->GetQueueName();
 }
 
 void Breakpoint::SetCondition(const char *condition) {
@@ -851,8 +886,8 @@ void Breakpoint::GetDescription(Stream *s, lldb::DescriptionLevel level,
     if (level == eDescriptionLevelBrief) {
       s->PutCString(GetBreakpointKind());
       return;
-    } else
-      s->Printf("Kind: %s\n", GetBreakpointKind());
+    }
+    s->Printf("Kind: %s\n", GetBreakpointKind());
   }
 
   const size_t num_locations = GetNumLocations();
@@ -956,8 +991,7 @@ void Breakpoint::GetResolverDescription(Stream *s) {
     m_resolver_sp->GetDescription(s);
 }
 
-bool Breakpoint::GetMatchingFileLine(ConstString filename,
-                                     uint32_t line_number,
+bool Breakpoint::GetMatchingFileLine(ConstString filename, uint32_t line_number,
                                      BreakpointLocationCollection &loc_coll) {
   // TODO: To be correct, this method needs to fill the breakpoint location
   // collection
@@ -1014,19 +1048,32 @@ void Breakpoint::SendBreakpointChangedEvent(
 
 const char *Breakpoint::BreakpointEventTypeAsCString(BreakpointEventType type) {
   switch (type) {
-    case eBreakpointEventTypeInvalidType: return "invalid";
-    case eBreakpointEventTypeAdded: return "breakpoint added";
-    case eBreakpointEventTypeRemoved: return "breakpoint removed";
-    case eBreakpointEventTypeLocationsAdded: return "locations added";
-    case eBreakpointEventTypeLocationsRemoved: return "locations removed";
-    case eBreakpointEventTypeLocationsResolved: return "locations resolved";
-    case eBreakpointEventTypeEnabled: return "breakpoint enabled";
-    case eBreakpointEventTypeDisabled: return "breakpoint disabled";
-    case eBreakpointEventTypeCommandChanged: return "command changed";
-    case eBreakpointEventTypeConditionChanged: return "condition changed";
-    case eBreakpointEventTypeIgnoreChanged: return "ignore count changed";
-    case eBreakpointEventTypeThreadChanged: return "thread changed";
-    case eBreakpointEventTypeAutoContinueChanged: return "autocontinue changed";
+  case eBreakpointEventTypeInvalidType:
+    return "invalid";
+  case eBreakpointEventTypeAdded:
+    return "breakpoint added";
+  case eBreakpointEventTypeRemoved:
+    return "breakpoint removed";
+  case eBreakpointEventTypeLocationsAdded:
+    return "locations added";
+  case eBreakpointEventTypeLocationsRemoved:
+    return "locations removed";
+  case eBreakpointEventTypeLocationsResolved:
+    return "locations resolved";
+  case eBreakpointEventTypeEnabled:
+    return "breakpoint enabled";
+  case eBreakpointEventTypeDisabled:
+    return "breakpoint disabled";
+  case eBreakpointEventTypeCommandChanged:
+    return "command changed";
+  case eBreakpointEventTypeConditionChanged:
+    return "condition changed";
+  case eBreakpointEventTypeIgnoreChanged:
+    return "ignore count changed";
+  case eBreakpointEventTypeThreadChanged:
+    return "thread changed";
+  case eBreakpointEventTypeAutoContinueChanged:
+    return "autocontinue changed";
   };
   llvm_unreachable("Fully covered switch above!");
 }
@@ -1064,7 +1111,7 @@ void Breakpoint::BreakpointEventData::Dump(Stream *s) const {
   BreakpointEventType event_type = GetBreakpointEventType();
   break_id_t bkpt_id = GetBreakpoint()->GetID();
   s->Format("bkpt: {0} type: {1}", bkpt_id,
-      BreakpointEventTypeAsCString(event_type));
+            BreakpointEventTypeAsCString(event_type));
 }
 
 const Breakpoint::BreakpointEventData *
@@ -1085,8 +1132,7 @@ Breakpoint::BreakpointEventData::GetBreakpointEventTypeFromEvent(
 
   if (data == nullptr)
     return eBreakpointEventTypeInvalidType;
-  else
-    return data->GetBreakpointEventType();
+  return data->GetBreakpointEventType();
 }
 
 BreakpointSP Breakpoint::BreakpointEventData::GetBreakpointFromEvent(
diff --git a/lldb/source/Breakpoint/BreakpointIDList.cpp b/lldb/source/Breakpoint/BreakpointIDList.cpp
index 5fc9f95a75db..8dde2c31e64a 100644
--- a/lldb/source/Breakpoint/BreakpointIDList.cpp
+++ b/lldb/source/Breakpoint/BreakpointIDList.cpp
@@ -99,8 +99,8 @@ llvm::Error BreakpointIDList::FindAndReplaceIDRanges(
         new_args.Clear();
         return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                        error.AsCString());
-      } else
-        names_found.insert(std::string(current_arg));
+      }
+      names_found.insert(std::string(current_arg));
     } else if ((i + 2 < old_args.size()) &&
                BreakpointID::IsRangeIdentifier(old_args[i + 1].ref()) &&
                BreakpointID::IsValidIDExpression(current_arg) &&
diff --git a/lldb/source/Breakpoint/BreakpointLocation.cpp b/lldb/source/Breakpoint/BreakpointLocation.cpp
index c7ea50407ae1..755331594604 100644
--- a/lldb/source/Breakpoint/BreakpointLocation.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocation.cpp
@@ -31,7 +31,7 @@ using namespace lldb_private;
 
 BreakpointLocation::BreakpointLocation(break_id_t loc_id, Breakpoint &owner,
                                        const Address &addr, lldb::tid_t tid,
-                                       bool hardware, bool check_for_resolver)
+                                       bool check_for_resolver)
     : m_should_resolve_indirect_functions(false), m_is_reexported(false),
       m_is_indirect(false), m_address(addr), m_owner(owner),
       m_condition_hash(0), m_loc_id(loc_id), m_hit_counter() {
@@ -55,8 +55,7 @@ const BreakpointOptions &BreakpointLocation::GetOptionsSpecifyingKind(
     BreakpointOptions::OptionKind kind) const {
   if (m_options_up && m_options_up->IsOptionSet(kind))
     return *m_options_up;
-  else
-    return m_owner.GetOptions();
+  return m_owner.GetOptions();
 }
 
 Address &BreakpointLocation::GetAddress() { return m_address; }
@@ -68,29 +67,25 @@ Target &BreakpointLocation::GetTarget() { return m_owner.GetTarget(); }
 bool BreakpointLocation::IsEnabled() const {
   if (!m_owner.IsEnabled())
     return false;
-  else if (m_options_up != nullptr)
+  if (m_options_up != nullptr)
     return m_options_up->IsEnabled();
-  else
-    return true;
+  return true;
 }
 
-void BreakpointLocation::SetEnabled(bool enabled) {
+bool BreakpointLocation::SetEnabled(bool enabled) {
   GetLocationOptions().SetEnabled(enabled);
-  if (enabled) {
-    ResolveBreakpointSite();
-  } else {
-    ClearBreakpointSite();
-  }
+  const bool success =
+      enabled ? ResolveBreakpointSite() : ClearBreakpointSite();
   SendBreakpointLocationChangedEvent(enabled ? eBreakpointEventTypeEnabled
                                              : eBreakpointEventTypeDisabled);
+  return success;
 }
 
 bool BreakpointLocation::IsAutoContinue() const {
   if (m_options_up &&
       m_options_up->IsOptionSet(BreakpointOptions::eAutoContinue))
     return m_options_up->IsAutoContinue();
-  else
-    return m_owner.IsAutoContinue();
+  return m_owner.IsAutoContinue();
 }
 
 void BreakpointLocation::SetAutoContinue(bool auto_continue) {
@@ -109,8 +104,7 @@ lldb::tid_t BreakpointLocation::GetThreadID() {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetTID();
-  else
-    return LLDB_INVALID_THREAD_ID;
+  return LLDB_INVALID_THREAD_ID;
 }
 
 void BreakpointLocation::SetThreadIndex(uint32_t index) {
@@ -131,8 +125,7 @@ uint32_t BreakpointLocation::GetThreadIndex() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetIndex();
-  else
-    return 0;
+  return 0;
 }
 
 void BreakpointLocation::SetThreadName(const char *thread_name) {
@@ -153,8 +146,7 @@ const char *BreakpointLocation::GetThreadName() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetName();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 void BreakpointLocation::SetQueueName(const char *queue_name) {
@@ -175,22 +167,19 @@ const char *BreakpointLocation::GetQueueName() const {
           .GetThreadSpecNoCreate();
   if (thread_spec)
     return thread_spec->GetQueueName();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 bool BreakpointLocation::InvokeCallback(StoppointCallbackContext *context) {
   if (m_options_up != nullptr && m_options_up->HasCallback())
     return m_options_up->InvokeCallback(context, m_owner.GetID(), GetID());
-  else
-    return m_owner.InvokeCallback(context, GetID());
+  return m_owner.InvokeCallback(context, GetID());
 }
 
 bool BreakpointLocation::IsCallbackSynchronous() {
   if (m_options_up != nullptr && m_options_up->HasCallback())
     return m_options_up->IsCallbackSynchronous();
-  else
-    return m_owner.GetOptions().IsCallbackSynchronous();
+  return m_owner.GetOptions().IsCallbackSynchronous();
 }
 
 void BreakpointLocation::SetCallback(BreakpointHitCallback callback,
@@ -445,10 +434,10 @@ bool BreakpointLocation::ResolveBreakpointSite() {
       process->CreateBreakpointSite(shared_from_this(), m_owner.IsHardware());
 
   if (new_id == LLDB_INVALID_BREAK_ID) {
-    Log *log = GetLog(LLDBLog::Breakpoints);
-    if (log)
-      log->Warning("Failed to add breakpoint site at 0x%" PRIx64,
-                   m_address.GetOpcodeLoadAddress(&m_owner.GetTarget()));
+    LLDB_LOGF(GetLog(LLDBLog::Breakpoints),
+              "Failed to add breakpoint site at 0x%" PRIx64 "(resolved=%s)",
+              m_address.GetOpcodeLoadAddress(&m_owner.GetTarget()),
+              IsResolved() ? "yes" : "no");
   }
 
   return IsResolved();
diff --git a/lldb/source/Breakpoint/BreakpointLocationList.cpp b/lldb/source/Breakpoint/BreakpointLocationList.cpp
index 0240305d6f29..1d8b4c1ccfae 100644
--- a/lldb/source/Breakpoint/BreakpointLocationList.cpp
+++ b/lldb/source/Breakpoint/BreakpointLocationList.cpp
@@ -32,7 +32,7 @@ BreakpointLocationList::Create(const Address &addr,
   lldb::break_id_t bp_loc_id = ++m_next_id;
   BreakpointLocationSP bp_loc_sp(
       new BreakpointLocation(bp_loc_id, m_owner, addr, LLDB_INVALID_THREAD_ID,
-                             m_owner.IsHardware(), resolve_indirect_symbols));
+                             resolve_indirect_symbols));
   m_locations.push_back(bp_loc_sp);
   m_address_to_location[addr] = bp_loc_sp;
   return bp_loc_sp;
@@ -72,8 +72,7 @@ BreakpointLocationList::FindByID(lldb::break_id_t break_id) const {
       llvm::lower_bound(m_locations, break_id, Compare);
   if (pos != end && (*pos)->GetID() == break_id)
     return *(pos);
-  else
-    return BreakpointLocationSP();
+  return BreakpointLocationSP();
 }
 
 size_t BreakpointLocationList::FindInModule(
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index 242b5b30168c..2151c4c2dd20 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -438,7 +438,8 @@ bool BreakpointOptions::InvokeCallback(StoppointCallbackContext *context,
         return m_callback(m_callback_baton_sp ? m_callback_baton_sp->data()
                                           : nullptr,
                       context, break_id, break_loc_id);
-    } else if (IsCallbackSynchronous()) {
+    }
+    if (IsCallbackSynchronous()) {
       return false;
     }
   }
@@ -482,9 +483,8 @@ const char *BreakpointOptions::GetConditionText(size_t *hash) const {
       *hash = m_condition_text_hash;
 
     return m_condition_text.c_str();
-  } else {
-    return nullptr;
   }
+  return nullptr;
 }
 
 const ThreadSpec *BreakpointOptions::GetThreadSpecNoCreate() const {
diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp
index b9c8a1635d84..edde1c91b789 100644
--- a/lldb/source/Breakpoint/BreakpointResolverName.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp
@@ -124,14 +124,14 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData(
     return std::make_shared<BreakpointResolverName>(
         nullptr, RegularExpression(regex_text), language, offset,
         skip_prologue);
-  } else {
-    StructuredData::Array *names_array;
-    success = options_dict.GetValueForKeyAsArray(
-        GetKey(OptionNames::SymbolNameArray), names_array);
-    if (!success) {
-      error = Status::FromErrorString("BRN::CFSD: Missing symbol names entry.");
-      return nullptr;
-    }
+  }
+  StructuredData::Array *names_array;
+  success = options_dict.GetValueForKeyAsArray(
+      GetKey(OptionNames::SymbolNameArray), names_array);
+  if (!success) {
+    error = Status::FromErrorString("BRN::CFSD: Missing symbol names entry.");
+    return nullptr;
+  }
     StructuredData::Array *names_mask_array;
     success = options_dict.GetValueForKeyAsArray(
         GetKey(OptionNames::NameMaskArray), names_mask_array);
@@ -182,7 +182,6 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData(
       resolver_sp->AddNameLookup(ConstString(names[i]), name_masks[i]);
     }
     return resolver_sp;
-  }
 }
 
 StructuredData::ObjectSP BreakpointResolverName::SerializeToStructuredData() {
diff --git a/lldb/source/Breakpoint/Watchpoint.cpp b/lldb/source/Breakpoint/Watchpoint.cpp
index 2df848aaa057..f1366ca53807 100644
--- a/lldb/source/Breakpoint/Watchpoint.cpp
+++ b/lldb/source/Breakpoint/Watchpoint.cpp
@@ -477,8 +477,7 @@ void Watchpoint::SetCondition(const char *condition) {
 const char *Watchpoint::GetConditionText() const {
   if (m_condition_up)
     return m_condition_up->GetUserText();
-  else
-    return nullptr;
+  return nullptr;
 }
 
 void Watchpoint::SendWatchpointChangedEvent(
@@ -534,8 +533,7 @@ Watchpoint::WatchpointEventData::GetWatchpointEventTypeFromEvent(
 
   if (data == nullptr)
     return eWatchpointEventTypeInvalidType;
-  else
-    return data->GetWatchpointEventType();
+  return data->GetWatchpointEventType();
 }
 
 WatchpointSP Watchpoint::WatchpointEventData::GetWatchpointFromEvent(
diff --git a/lldb/source/Breakpoint/WatchpointOptions.cpp b/lldb/source/Breakpoint/WatchpointOptions.cpp
index c27cd62e0220..321aab070019 100644
--- a/lldb/source/Breakpoint/WatchpointOptions.cpp
+++ b/lldb/source/Breakpoint/WatchpointOptions.cpp
@@ -92,8 +92,8 @@ bool WatchpointOptions::InvokeCallback(StoppointCallbackContext *context,
     return m_callback(m_callback_baton_sp ? m_callback_baton_sp->data()
                                           : nullptr,
                       context, watch_id);
-  } else
-    return true;
+  }
+  return true;
 }
 
 bool WatchpointOptions::HasCallback() {
diff --git a/lldb/source/Commands/CommandObjectBreakpoint.cpp b/lldb/source/Commands/CommandObjectBreakpoint.cpp
index a0c39cf670d4..4631c97bf50a 100644
--- a/lldb/source/Commands/CommandObjectBreakpoint.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpoint.cpp
@@ -2485,8 +2485,9 @@ void CommandObjectMultiwordBreakpoint::VerifyIDs(
     Breakpoint *breakpoint =
         target.GetBreakpointByID(cur_bp_id.GetBreakpointID()).get();
     if (breakpoint != nullptr) {
-      const size_t num_locations = breakpoint->GetNumLocations();
-      if (static_cast<size_t>(cur_bp_id.GetLocationID()) > num_locations) {
+      lldb::break_id_t cur_loc_id = cur_bp_id.GetLocationID();
+      // GetLocationID returns 0 when the location isn't specified.
+      if (cur_loc_id != 0 && !breakpoint->FindLocationByID(cur_loc_id)) {
         StreamString id_str;
         BreakpointID::GetCanonicalReference(
             &id_str, cur_bp_id.GetBreakpointID(), cur_bp_id.GetLocationID());
diff --git a/lldb/source/Commands/CommandObjectProtocolServer.cpp b/lldb/source/Commands/CommandObjectProtocolServer.cpp
index 115754769f3e..f11e27f01c8a 100644
--- a/lldb/source/Commands/CommandObjectProtocolServer.cpp
+++ b/lldb/source/Commands/CommandObjectProtocolServer.cpp
@@ -23,20 +23,6 @@ using namespace lldb_private;
 #define LLDB_OPTIONS_mcp
 #include "CommandOptions.inc"
 
-static std::vector<llvm::StringRef> GetSupportedProtocols() {
-  std::vector<llvm::StringRef> supported_protocols;
-  size_t i = 0;
-
-  for (llvm::StringRef protocol_name =
-           PluginManager::GetProtocolServerPluginNameAtIndex(i++);
-       !protocol_name.empty();
-       protocol_name = PluginManager::GetProtocolServerPluginNameAtIndex(i++)) {
-    supported_protocols.push_back(protocol_name);
-  }
-
-  return supported_protocols;
-}
-
 class CommandObjectProtocolServerStart : public CommandObjectParsed {
 public:
   CommandObjectProtocolServerStart(CommandInterpreter &interpreter)
@@ -57,12 +43,11 @@ class CommandObjectProtocolServerStart : public CommandObjectParsed {
     }
 
     llvm::StringRef protocol = args.GetArgumentAtIndex(0);
-    std::vector<llvm::StringRef> supported_protocols = GetSupportedProtocols();
-    if (llvm::find(supported_protocols, protocol) ==
-        supported_protocols.end()) {
+    ProtocolServer *server = ProtocolServer::GetOrCreate(protocol);
+    if (!server) {
       result.AppendErrorWithFormatv(
           "unsupported protocol: {0}. Supported protocols are: {1}", protocol,
-          llvm::join(GetSupportedProtocols(), ", "));
+          llvm::join(ProtocolServer::GetSupportedProtocols(), ", "));
       return;
     }
 
@@ -72,10 +57,6 @@ class CommandObjectProtocolServerStart : public CommandObjectParsed {
     }
     llvm::StringRef connection_uri = args.GetArgumentAtIndex(1);
 
-    ProtocolServerSP server_sp = GetDebugger().GetProtocolServer(protocol);
-    if (!server_sp)
-      server_sp = ProtocolServer::Create(protocol, GetDebugger());
-
     const char *connection_error =
         "unsupported connection specifier, expected 'accept:///path' or "
         "'listen://[host]:port', got '{0}'.";
@@ -94,23 +75,25 @@ class CommandObjectProtocolServerStart : public CommandObjectParsed {
 
     ProtocolServer::Connection connection;
     connection.protocol = protocol_and_mode->first;
-    connection.name =
-        formatv("[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
-                uri->port.value_or(0));
-
-    if (llvm::Error error = server_sp->Start(connection)) {
+    if (connection.protocol == Socket::SocketProtocol::ProtocolUnixDomain)
+      connection.name = uri->path;
+    else
+      connection.name = formatv(
+          "[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
+          uri->port.value_or(0));
+
+    if (llvm::Error error = server->Start(connection)) {
       result.AppendErrorWithFormatv("{0}", llvm::fmt_consume(std::move(error)));
       return;
     }
 
-    GetDebugger().AddProtocolServer(server_sp);
-
-    if (Socket *socket = server_sp->GetSocket()) {
+    if (Socket *socket = server->GetSocket()) {
       std::string address =
           llvm::join(socket->GetListeningConnectionURI(), ", ");
       result.AppendMessageWithFormatv(
           "{0} server started with connection listeners: {1}", protocol,
           address);
+      result.SetStatus(eReturnStatusSuccessFinishNoResult);
     }
   }
 };
@@ -134,30 +117,18 @@ class CommandObjectProtocolServerStop : public CommandObjectParsed {
     }
 
     llvm::StringRef protocol = args.GetArgumentAtIndex(0);
-    std::vector<llvm::StringRef> supported_protocols = GetSupportedProtocols();
-    if (llvm::find(supported_protocols, protocol) ==
-        supported_protocols.end()) {
+    ProtocolServer *server = ProtocolServer::GetOrCreate(protocol);
+    if (!server) {
       result.AppendErrorWithFormatv(
           "unsupported protocol: {0}. Supported protocols are: {1}", protocol,
-          llvm::join(GetSupportedProtocols(), ", "));
-      return;
-    }
-
-    Debugger &debugger = GetDebugger();
-
-    ProtocolServerSP server_sp = debugger.GetProtocolServer(protocol);
-    if (!server_sp) {
-      result.AppendError(
-          llvm::formatv("no {0} protocol server running", protocol).str());
+          llvm::join(ProtocolServer::GetSupportedProtocols(), ", "));
       return;
     }
 
-    if (llvm::Error error = server_sp->Stop()) {
+    if (llvm::Error error = server->Stop()) {
       result.AppendErrorWithFormatv("{0}", llvm::fmt_consume(std::move(error)));
       return;
     }
-
-    debugger.RemoveProtocolServer(server_sp);
   }
 };
 
diff --git a/lldb/source/Commands/CommandObjectStats.cpp b/lldb/source/Commands/CommandObjectStats.cpp
index b77c44bdf5d0..08283ef9d169 100644
--- a/lldb/source/Commands/CommandObjectStats.cpp
+++ b/lldb/source/Commands/CommandObjectStats.cpp
@@ -9,6 +9,7 @@
 #include "CommandObjectStats.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Host/OptionParser.h"
+#include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandOptionArgumentTable.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Interpreter/OptionArgParser.h"
@@ -147,9 +148,18 @@ class CommandObjectStatsDump : public CommandObjectParsed {
     if (!m_options.m_all_targets)
       target = m_exe_ctx.GetTargetPtr();
 
+    // Check if transcript is requested but transcript saving is disabled
+    const StatisticsOptions &stats_options = m_options.GetStatisticsOptions();
+    if (stats_options.GetIncludeTranscript() &&
+        !GetDebugger().GetCommandInterpreter().GetSaveTranscript()) {
+      result.AppendWarning(
+          "transcript requested but none was saved. Enable with "
+          "'settings set interpreter.save-transcript true'");
+    }
+
     result.AppendMessageWithFormatv(
-        "{0:2}", DebuggerStats::ReportStatistics(
-                     GetDebugger(), target, m_options.GetStatisticsOptions()));
+        "{0:2}",
+        DebuggerStats::ReportStatistics(GetDebugger(), target, stats_options));
     result.SetStatus(eReturnStatusSuccessFinishResult);
   }
 
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index a4ced37649ea..fe421adfe870 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -4086,8 +4086,6 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
     // Dump all sections for all modules images
 
     if (command.GetArgumentCount() == 0) {
-      ModuleSP current_module;
-
       // Where it is possible to look in the current symbol context first,
       // try that.  If this search was successful and --all was not passed,
       // don't print anything else.
@@ -4110,8 +4108,7 @@ class CommandObjectTargetModulesLookup : public CommandObjectParsed {
       }
 
       for (ModuleSP module_sp : target_modules.ModulesNoLocking()) {
-        if (module_sp != current_module &&
-            LookupInModule(m_interpreter, module_sp.get(), result,
+        if (LookupInModule(m_interpreter, module_sp.get(), result,
                            syntax_error)) {
           result.GetOutputStream().EOL();
           num_successful_lookups++;
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index 75bdffe16c52..5327647d65cc 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -1482,13 +1482,15 @@ let Command = "statistics dump" in {
     "this is turned off unless specified. "
     "In default mode, if both '--targets' and '--modules' are 'true', a list "
     "of module identifiers will be added to the 'targets' section.">;
-  def statistics_dump_transcript: Option<"transcript", "t">, Group<1>,
-    Arg<"Boolean">,
-    Desc<"If the setting interpreter.save-transcript is enabled and this "
-    "option is 'true', include a JSON array with all commands the user and/or "
-    "scripts executed during a debug session. "
-    "Defaults to true, unless the '--summary' mode is enabled, in which case "
-    "this is turned off unless specified.">;
+  def statistics_dump_transcript
+      : Option<"transcript", "t">,
+        Group<1>,
+        Arg<"Boolean">,
+        Desc<"If the setting interpreter.save-transcript is enabled and this "
+             "option is 'true', include a JSON array with all commands the "
+             "user and/or "
+             "scripts executed during a debug session. "
+             "Defaults to false. ">;
   def statistics_dump_plugins
       : Option<"plugins", "p">,
         Group<1>,
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 33d1053fd8a6..ed674ee1275c 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -1445,6 +1445,13 @@ bool Debugger::PopIOHandler(const IOHandlerSP &pop_reader_sp) {
   return true;
 }
 
+void Debugger::RefreshIOHandler() {
+  std::lock_guard<std::recursive_mutex> guard(m_io_handler_stack.GetMutex());
+  IOHandlerSP reader_sp(m_io_handler_stack.Top());
+  if (reader_sp)
+    reader_sp->Refresh();
+}
+
 StreamUP Debugger::GetAsyncOutputStream() {
   return std::make_unique<StreamAsynchronousIO>(*this,
                                                 StreamAsynchronousIO::STDOUT);
@@ -2376,26 +2383,3 @@ llvm::ThreadPoolInterface &Debugger::GetThreadPool() {
          "Debugger::GetThreadPool called before Debugger::Initialize");
   return *g_thread_pool;
 }
-
-void Debugger::AddProtocolServer(lldb::ProtocolServerSP protocol_server_sp) {
-  assert(protocol_server_sp &&
-         GetProtocolServer(protocol_server_sp->GetPluginName()) == nullptr);
-  m_protocol_servers.push_back(protocol_server_sp);
-}
-
-void Debugger::RemoveProtocolServer(lldb::ProtocolServerSP protocol_server_sp) {
-  auto it = llvm::find(m_protocol_servers, protocol_server_sp);
-  if (it != m_protocol_servers.end())
-    m_protocol_servers.erase(it);
-}
-
-lldb::ProtocolServerSP
-Debugger::GetProtocolServer(llvm::StringRef protocol) const {
-  for (ProtocolServerSP protocol_server_sp : m_protocol_servers) {
-    if (!protocol_server_sp)
-      continue;
-    if (protocol_server_sp->GetPluginName() == protocol)
-      return protocol_server_sp;
-  }
-  return nullptr;
-}
diff --git a/lldb/source/Core/EmulateInstruction.cpp b/lldb/source/Core/EmulateInstruction.cpp
index d240b4d3b331..634ce1075d54 100644
--- a/lldb/source/Core/EmulateInstruction.cpp
+++ b/lldb/source/Core/EmulateInstruction.cpp
@@ -588,7 +588,99 @@ EmulateInstruction::GetInternalRegisterNumber(RegisterContext *reg_ctx,
   return LLDB_INVALID_REGNUM;
 }
 
+std::unique_ptr<SingleStepBreakpointLocationsPredictor>
+EmulateInstruction::CreateBreakpointLocationPredictor(
+    std::unique_ptr<EmulateInstruction> emulator_up) {
+  auto creator =
+      emulator_up->GetSingleStepBreakpointLocationsPredictorCreator();
+  return creator(std::move(emulator_up));
+}
+
+std::optional<lldb::addr_t> EmulateInstruction::ReadPC() {
+  bool success = false;
+  auto addr = ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
+                                   LLDB_INVALID_ADDRESS, &success);
+  return success ? std::optional<addr_t>(addr) : std::nullopt;
+}
+
+bool EmulateInstruction::WritePC(lldb::addr_t addr) {
+  EmulateInstruction::Context ctx;
+  ctx.type = eContextAdvancePC;
+  ctx.SetNoArgs();
+  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
+                               LLDB_REGNUM_GENERIC_PC, addr);
+}
+
 bool EmulateInstruction::CreateFunctionEntryUnwind(UnwindPlan &unwind_plan) {
   unwind_plan.Clear();
   return false;
 }
+
+BreakpointLocations
+SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(Status &status) {
+  if (!m_emulator_up->ReadInstruction()) {
+    // try to get at least the size of next instruction to set breakpoint.
+    lldb::addr_t next_pc = GetNextInstructionAddress(status);
+    return BreakpointLocations{next_pc};
+  }
+
+  auto entry_pc = m_emulator_up->ReadPC();
+  if (!entry_pc) {
+    status = Status("Can't read PC");
+    return {};
+  }
+
+  m_emulation_result = m_emulator_up->EvaluateInstruction(
+      eEmulateInstructionOptionAutoAdvancePC);
+
+  lldb::addr_t next_pc = GetBreakpointLocationAddress(*entry_pc, status);
+  return BreakpointLocations{next_pc};
+}
+
+lldb::addr_t SingleStepBreakpointLocationsPredictor::GetNextInstructionAddress(
+    Status &error) {
+  auto instr_size = m_emulator_up->GetLastInstrSize();
+  if (!instr_size) {
+    error = Status("Read instruction failed!");
+    return LLDB_INVALID_ADDRESS;
+  }
+
+  auto pc = m_emulator_up->ReadPC();
+  if (!pc) {
+    error = Status("Can't read PC");
+    return LLDB_INVALID_ADDRESS;
+  }
+
+  lldb::addr_t next_pc = *pc + *instr_size;
+  return next_pc;
+}
+
+lldb::addr_t
+SingleStepBreakpointLocationsPredictor::GetBreakpointLocationAddress(
+    lldb::addr_t entry_pc, Status &error) {
+  auto addr = m_emulator_up->ReadPC();
+  if (!addr) {
+    error = Status("Can't read PC");
+    return LLDB_INVALID_ADDRESS;
+  }
+  lldb::addr_t pc = *addr;
+
+  if (m_emulation_result) {
+    assert(entry_pc != pc && "Emulation was successfull but PC wasn't updated");
+    return pc;
+  }
+
+  if (entry_pc == pc) {
+    // Emulate instruction failed and it hasn't changed PC. Advance PC with
+    // the size of the current opcode because the emulation of all
+    // PC modifying instruction should be successful. The failure most
+    // likely caused by an unsupported instruction which does not modify PC.
+    return pc + m_emulator_up->GetOpcode().GetByteSize();
+  }
+
+  // The instruction emulation failed after it modified the PC. It is an
+  // unknown error where we can't continue because the next instruction is
+  // modifying the PC but we don't  know how.
+  error = Status("Instruction emulation failed unexpectedly.");
+  return LLDB_INVALID_ADDRESS;
+}
diff --git a/lldb/source/Core/FormatEntity.cpp b/lldb/source/Core/FormatEntity.cpp
index 8e3c3c18863f..ca389bc88cf7 100644
--- a/lldb/source/Core/FormatEntity.cpp
+++ b/lldb/source/Core/FormatEntity.cpp
@@ -470,9 +470,7 @@ static bool DumpAddressAndContent(Stream &s, const SymbolContext *sc,
                                   bool print_file_addr_or_load_addr) {
   Target *target = Target::GetTargetFromContexts(exe_ctx, sc);
 
-  addr_t vaddr = LLDB_INVALID_ADDRESS;
-  if (target && target->HasLoadedSections())
-    vaddr = addr.GetLoadAddress(target);
+  addr_t vaddr = addr.GetLoadAddress(target);
   if (vaddr == LLDB_INVALID_ADDRESS)
     vaddr = addr.GetFileAddress();
   if (vaddr == LLDB_INVALID_ADDRESS)
diff --git a/lldb/source/Core/IOHandler.cpp b/lldb/source/Core/IOHandler.cpp
index 8aac507eaa0c..f65a1113f359 100644
--- a/lldb/source/Core/IOHandler.cpp
+++ b/lldb/source/Core/IOHandler.cpp
@@ -663,3 +663,10 @@ void IOHandlerEditline::PrintAsync(const char *s, size_t len, bool is_stdout) {
 #endif
   }
 }
+
+void IOHandlerEditline::Refresh() {
+#if LLDB_ENABLE_LIBEDIT
+  if (m_editline_up)
+    m_editline_up->Refresh();
+#endif
+}
diff --git a/lldb/source/Core/ProtocolServer.cpp b/lldb/source/Core/ProtocolServer.cpp
index d57a047afa7b..41636cdacdec 100644
--- a/lldb/source/Core/ProtocolServer.cpp
+++ b/lldb/source/Core/ProtocolServer.cpp
@@ -12,10 +12,36 @@
 using namespace lldb_private;
 using namespace lldb;
 
-ProtocolServerSP ProtocolServer::Create(llvm::StringRef name,
-                                        Debugger &debugger) {
+ProtocolServer *ProtocolServer::GetOrCreate(llvm::StringRef name) {
+  static std::mutex g_mutex;
+  static llvm::StringMap<ProtocolServerUP> g_protocol_server_instances;
+
+  std::lock_guard<std::mutex> guard(g_mutex);
+
+  auto it = g_protocol_server_instances.find(name);
+  if (it != g_protocol_server_instances.end())
+    return it->second.get();
+
   if (ProtocolServerCreateInstance create_callback =
-          PluginManager::GetProtocolCreateCallbackForPluginName(name))
-    return create_callback(debugger);
+          PluginManager::GetProtocolCreateCallbackForPluginName(name)) {
+    auto pair =
+        g_protocol_server_instances.try_emplace(name, create_callback());
+    return pair.first->second.get();
+  }
+
   return nullptr;
 }
+
+std::vector<llvm::StringRef> ProtocolServer::GetSupportedProtocols() {
+  std::vector<llvm::StringRef> supported_protocols;
+  size_t i = 0;
+
+  for (llvm::StringRef protocol_name =
+           PluginManager::GetProtocolServerPluginNameAtIndex(i++);
+       !protocol_name.empty();
+       protocol_name = PluginManager::GetProtocolServerPluginNameAtIndex(i++)) {
+    supported_protocols.push_back(protocol_name);
+  }
+
+  return supported_protocols;
+}
diff --git a/lldb/source/Core/Statusline.cpp b/lldb/source/Core/Statusline.cpp
index 8a8640805cac..393d42724102 100644
--- a/lldb/source/Core/Statusline.cpp
+++ b/lldb/source/Core/Statusline.cpp
@@ -24,6 +24,7 @@
 #define ANSI_SAVE_CURSOR ESCAPE "7"
 #define ANSI_RESTORE_CURSOR ESCAPE "8"
 #define ANSI_CLEAR_BELOW ESCAPE "[J"
+#define ANSI_CLEAR_SCREEN ESCAPE "[2J"
 #define ANSI_SET_SCROLL_ROWS ESCAPE "[1;%ur"
 #define ANSI_TO_START_OF_ROW ESCAPE "[%u;1f"
 #define ANSI_REVERSE_VIDEO ESCAPE "[7m"
@@ -41,10 +42,12 @@ Statusline::Statusline(Debugger &debugger)
 Statusline::~Statusline() { Disable(); }
 
 void Statusline::TerminalSizeChanged() {
-  UpdateTerminalProperties();
+  m_terminal_width = m_debugger.GetTerminalWidth();
+  m_terminal_height = m_debugger.GetTerminalHeight();
+
+  UpdateScrollWindow(ResizeStatusline);
 
-  // This definitely isn't signal safe, but the best we can do, until we
-  // have proper signal-catching thread.
+  // Draw the old statusline.
   Redraw(/*update=*/false);
 }
 
@@ -85,13 +88,6 @@ void Statusline::Draw(std::string str) {
   locked_stream << ANSI_RESTORE_CURSOR;
 }
 
-void Statusline::UpdateTerminalProperties() {
-  UpdateScrollWindow(DisableStatusline);
-  m_terminal_width = m_debugger.GetTerminalWidth();
-  m_terminal_height = m_debugger.GetTerminalHeight();
-  UpdateScrollWindow(EnableStatusline);
-}
-
 void Statusline::UpdateScrollWindow(ScrollWindowMode mode) {
   assert(m_terminal_width != 0 && m_terminal_height != 0);
 
@@ -99,24 +95,36 @@ void Statusline::UpdateScrollWindow(ScrollWindowMode mode) {
   if (!stream_sp)
     return;
 
-  const unsigned scroll_height =
-      (mode == DisableStatusline) ? m_terminal_height : m_terminal_height - 1;
-
+  const unsigned reduced_scroll_window = m_terminal_height - 1;
   LockedStreamFile locked_stream = stream_sp->Lock();
-  locked_stream << ANSI_SAVE_CURSOR;
-  locked_stream.Printf(ANSI_SET_SCROLL_ROWS, scroll_height);
-  locked_stream << ANSI_RESTORE_CURSOR;
+
   switch (mode) {
   case EnableStatusline:
     // Move everything on the screen up.
-    locked_stream.Printf(ANSI_UP_ROWS, 1);
     locked_stream << '\n';
+    locked_stream.Printf(ANSI_UP_ROWS, 1);
+    // Reduce the scroll window.
+    locked_stream << ANSI_SAVE_CURSOR;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, reduced_scroll_window);
+    locked_stream << ANSI_RESTORE_CURSOR;
     break;
   case DisableStatusline:
+    // Reset the scroll window.
+    locked_stream << ANSI_SAVE_CURSOR;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, 0);
+    locked_stream << ANSI_RESTORE_CURSOR;
     // Clear the screen below to hide the old statusline.
     locked_stream << ANSI_CLEAR_BELOW;
     break;
+  case ResizeStatusline:
+    // Clear the screen and update the scroll window.
+    // FIXME: Find a better solution (#146919).
+    locked_stream << ANSI_CLEAR_SCREEN;
+    locked_stream.Printf(ANSI_SET_SCROLL_ROWS, reduced_scroll_window);
+    break;
   }
+
+  m_debugger.RefreshIOHandler();
 }
 
 void Statusline::Redraw(bool update) {
diff --git a/lldb/source/Core/Telemetry.cpp b/lldb/source/Core/Telemetry.cpp
index 6434b11a63ad..2f2e1c1d56d5 100644
--- a/lldb/source/Core/Telemetry.cpp
+++ b/lldb/source/Core/Telemetry.cpp
@@ -119,6 +119,19 @@ llvm::Error TelemetryManager::preDispatch(TelemetryInfo *entry) {
   return llvm::Error::success();
 }
 
+// Helper for extracting time field from a Dictionary.
+static std::optional<std::chrono::nanoseconds>
+GetAsNanosec(StructuredData::Dictionary *dict, llvm::StringRef key) {
+  auto value = dict->GetValueForKey(key);
+  if (!value->IsValid()) {
+    LLDB_LOG(GetLog(LLDBLog::Object),
+             "Cannot determine {0} from client-telemetry entry", key);
+    return std::nullopt;
+  }
+
+  return std::chrono::nanoseconds(value->GetUnsignedIntegerValue(0));
+}
+
 void TelemetryManager::DispatchClientTelemetry(
     const lldb_private::StructuredDataImpl &entry, Debugger *debugger) {
   if (!m_config->enable_client_telemetry)
@@ -148,23 +161,12 @@ void TelemetryManager::DispatchClientTelemetry(
     LLDB_LOG(GetLog(LLDBLog::Object),
              "Cannot determine client_data from client-telemetry entry");
 
-  int64_t start_time;
-  if (dict->GetValueForKeyAsInteger("start_time", start_time)) {
-    client_info.start_time +=
-        std::chrono::nanoseconds(static_cast<size_t>(start_time));
-  } else {
-    LLDB_LOG(GetLog(LLDBLog::Object),
-             "Cannot determine start-time from client-telemetry entry");
-  }
+  if (auto maybe_start_time = GetAsNanosec(dict, "start_time"))
+    client_info.start_time += *maybe_start_time;
 
-  int64_t end_time;
-  if (dict->GetValueForKeyAsInteger("end_time", end_time)) {
+  if (auto maybe_end_time = GetAsNanosec(dict, "end_time")) {
     SteadyTimePoint epoch;
-    client_info.end_time =
-        epoch + std::chrono::nanoseconds(static_cast<size_t>(end_time));
-  } else {
-    LLDB_LOG(GetLog(LLDBLog::Object),
-             "Cannot determine end-time from client-telemetry entry");
+    client_info.end_time = epoch + *maybe_end_time;
   }
 
   llvm::StringRef error_msg;
diff --git a/lldb/source/DataFormatters/FormattersHelpers.cpp b/lldb/source/DataFormatters/FormattersHelpers.cpp
index d7b058d91c4a..028fc5da73dc 100644
--- a/lldb/source/DataFormatters/FormattersHelpers.cpp
+++ b/lldb/source/DataFormatters/FormattersHelpers.cpp
@@ -126,3 +126,28 @@ lldb_private::formatters::GetArrayAddressOrPointerValue(ValueObject &valobj) {
 
   return data_addr.address;
 }
+
+void lldb_private::formatters::DumpCxxSmartPtrPointerSummary(
+    Stream &stream, ValueObject &ptr, const TypeSummaryOptions &options) {
+  if (ptr.GetValueAsUnsigned(0) == 0) {
+    stream.Printf("nullptr");
+    return;
+  }
+
+  Status error;
+  ValueObjectSP pointee_sp = ptr.Dereference(error);
+  if (!pointee_sp || !error.Success())
+    return;
+
+  if (!pointee_sp->DumpPrintableRepresentation(
+          stream, ValueObject::eValueObjectRepresentationStyleSummary,
+          lldb::eFormatInvalid,
+          ValueObject::PrintableRepresentationSpecialCases::eDisable, false))
+    stream.Printf("ptr = 0x%" PRIx64, ptr.GetValueAsUnsigned(0));
+}
+
+bool lldb_private::formatters::ContainerSizeSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return FormatEntity::FormatStringRef("size=${svar%#}", stream, nullptr,
+                                       nullptr, nullptr, &valobj, false, false);
+}
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 661324338e80..efc3dd1c968a 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -588,7 +588,7 @@ bool DWARFExpression::LinkThreadLocalStorage(
   return true;
 }
 
-static llvm::Error Evaluate_DW_OP_entry_value(std::vector<Value> &stack,
+static llvm::Error Evaluate_DW_OP_entry_value(DWARFExpression::Stack &stack,
                                               ExecutionContext *exe_ctx,
                                               RegisterContext *reg_ctx,
                                               const DataExtractor &opcodes,
@@ -653,7 +653,7 @@ static llvm::Error Evaluate_DW_OP_entry_value(std::vector<Value> &stack,
   addr_t return_pc = LLDB_INVALID_ADDRESS;
   uint32_t current_frame_idx = current_frame->GetFrameIndex();
 
-  for (uint32_t parent_frame_idx = current_frame_idx + 1;;parent_frame_idx++) {
+  for (uint32_t parent_frame_idx = current_frame_idx + 1;; parent_frame_idx++) {
     parent_frame = thread->GetStackFrameAtIndex(parent_frame_idx);
     // If this is null, we're at the end of the stack.
     if (!parent_frame)
@@ -859,6 +859,66 @@ ResolveLoadAddress(ExecutionContext *exe_ctx, lldb::ModuleSP &module_sp,
   return load_addr;
 }
 
+static llvm::Error Evaluate_DW_OP_deref(DWARFExpression::Stack &stack,
+                                        ExecutionContext *exe_ctx,
+                                        lldb::ModuleSP module_sp,
+                                        Process *process) {
+  if (stack.empty())
+    return llvm::createStringError("expression stack empty for DW_OP_deref");
+
+  const Value::ValueType value_type = stack.back().GetValueType();
+  switch (value_type) {
+  case Value::ValueType::HostAddress: {
+    void *src = (void *)stack.back().GetScalar().ULongLong();
+    intptr_t ptr;
+    ::memcpy(&ptr, src, sizeof(void *));
+    stack.back().GetScalar() = ptr;
+    stack.back().ClearContext();
+  } break;
+  case Value::ValueType::FileAddress: {
+    auto file_addr = stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+    Address so_addr;
+    auto maybe_load_addr = ResolveLoadAddress(exe_ctx, module_sp, "DW_OP_deref",
+                                              file_addr, so_addr);
+    if (!maybe_load_addr)
+      return maybe_load_addr.takeError();
+    stack.back().GetScalar() = *maybe_load_addr;
+    // Fall through to load address promotion code below.
+  }
+    [[fallthrough]];
+  case Value::ValueType::Scalar:
+    // Promote Scalar to LoadAddress and fall through.
+    stack.back().SetValueType(Value::ValueType::LoadAddress);
+    [[fallthrough]];
+  case Value::ValueType::LoadAddress: {
+    if (!exe_ctx)
+      return llvm::createStringError("NULL execution context for DW_OP_deref");
+    if (!process)
+      return llvm::createStringError("NULL process for DW_OP_deref");
+    lldb::addr_t pointer_addr =
+        stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
+    Status error;
+    lldb::addr_t pointer_value =
+        process->ReadPointerFromMemory(pointer_addr, error);
+    if (pointer_value == LLDB_INVALID_ADDRESS)
+      return llvm::joinErrors(
+          llvm::createStringError(
+              "Failed to dereference pointer from 0x%" PRIx64
+              " for DW_OP_deref",
+              pointer_addr),
+          error.takeError());
+    if (ABISP abi_sp = process->GetABI())
+      pointer_value = abi_sp->FixCodeAddress(pointer_value);
+    stack.back().GetScalar() = pointer_value;
+    stack.back().ClearContext();
+  } break;
+  case Value::ValueType::Invalid:
+    return llvm::createStringError("invalid value type for DW_OP_deref");
+  }
+
+  return llvm::Error::success();
+}
+
 /// Helper function to move common code used to load sized data from a uint8_t
 /// buffer.
 ///
@@ -889,7 +949,8 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
   if (opcodes.GetByteSize() == 0)
     return llvm::createStringError(
         "no location, value may have been optimized out");
-  std::vector<Value> stack;
+
+  Stack stack;
 
   Process *process = nullptr;
   StackFrame *frame = nullptr;
@@ -1018,69 +1079,9 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
     // retrieved from the dereferenced address is the size of an address on the
     // target machine.
     case DW_OP_deref: {
-      if (stack.empty())
-        return llvm::createStringError(
-            "expression stack empty for DW_OP_deref");
-      Value::ValueType value_type = stack.back().GetValueType();
-      switch (value_type) {
-      case Value::ValueType::HostAddress: {
-        void *src = (void *)stack.back().GetScalar().ULongLong();
-        intptr_t ptr;
-        ::memcpy(&ptr, src, sizeof(void *));
-        stack.back().GetScalar() = ptr;
-        stack.back().ClearContext();
-      } break;
-      case Value::ValueType::FileAddress: {
-        auto file_addr = stack.back().GetScalar().ULongLong(
-            LLDB_INVALID_ADDRESS);
-
-        Address so_addr;
-        auto maybe_load_addr = ResolveLoadAddress(
-            exe_ctx, module_sp, "DW_OP_deref", file_addr, so_addr);
-
-        if (!maybe_load_addr)
-          return maybe_load_addr.takeError();
-
-        stack.back().GetScalar() = *maybe_load_addr;
-        // Fall through to load address promotion code below.
-      }
-        [[fallthrough]];
-      case Value::ValueType::Scalar:
-        // Promote Scalar to LoadAddress and fall through.
-        stack.back().SetValueType(Value::ValueType::LoadAddress);
-        [[fallthrough]];
-      case Value::ValueType::LoadAddress:
-        if (exe_ctx) {
-          if (process) {
-            lldb::addr_t pointer_addr =
-                stack.back().GetScalar().ULongLong(LLDB_INVALID_ADDRESS);
-            Status error;
-            lldb::addr_t pointer_value =
-                process->ReadPointerFromMemory(pointer_addr, error);
-            if (pointer_value != LLDB_INVALID_ADDRESS) {
-              if (ABISP abi_sp = process->GetABI())
-                pointer_value = abi_sp->FixCodeAddress(pointer_value);
-              stack.back().GetScalar() = pointer_value;
-              stack.back().ClearContext();
-            } else {
-              return llvm::createStringError(
-                  "Failed to dereference pointer from 0x%" PRIx64
-                  " for DW_OP_deref: %s\n",
-                  pointer_addr, error.AsCString());
-            }
-          } else {
-            return llvm::createStringError("NULL process for DW_OP_deref");
-          }
-        } else {
-          return llvm::createStringError(
-              "NULL execution context for DW_OP_deref");
-        }
-        break;
-
-      case Value::ValueType::Invalid:
-        return llvm::createStringError("invalid value type for DW_OP_deref");
-      }
-
+      if (llvm::Error err =
+              Evaluate_DW_OP_deref(stack, exe_ctx, module_sp, process))
+        return err;
     } break;
 
     // OPCODE: DW_OP_deref_size
@@ -1965,8 +1966,7 @@ llvm::Expected<Value> DWARFExpression::Evaluate(
           case Value::ValueType::Scalar: {
             uint32_t bit_size = piece_byte_size * 8;
             uint32_t bit_offset = 0;
-            if (!scalar.ExtractBitfield(
-                    bit_size, bit_offset)) {
+            if (!scalar.ExtractBitfield(bit_size, bit_offset)) {
               return llvm::createStringError(
                   "unable to extract %" PRIu64 " bytes from a %" PRIu64
                   " byte scalar value.",
@@ -2408,8 +2408,7 @@ bool DWARFExpression::MatchesOperand(
     return MatchUnaryOp(
         MatchOpType(Instruction::Operand::Type::Dereference),
         MatchBinaryOp(MatchOpType(Instruction::Operand::Type::Sum),
-                      MatchRegOp(*reg),
-                      MatchImmOp(offset)))(operand);
+                      MatchRegOp(*reg), MatchImmOp(offset)))(operand);
   } else {
     return MatchRegOp(*reg)(operand);
   }
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index 06b0cb7769f6..e445fa883302 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -57,12 +57,14 @@ IRExecutionUnit::IRExecutionUnit(std::unique_ptr<llvm::LLVMContext> &context_up,
 lldb::addr_t IRExecutionUnit::WriteNow(const uint8_t *bytes, size_t size,
                                        Status &error) {
   const bool zero_memory = false;
-  lldb::addr_t allocation_process_addr =
+  auto address_or_error =
       Malloc(size, 8, lldb::ePermissionsWritable | lldb::ePermissionsReadable,
-             eAllocationPolicyMirror, zero_memory, error);
-
-  if (!error.Success())
+             eAllocationPolicyMirror, zero_memory);
+  if (!address_or_error) {
+    error = Status::FromError(address_or_error.takeError());
     return LLDB_INVALID_ADDRESS;
+  }
+  lldb::addr_t allocation_process_addr = *address_or_error;
 
   WriteMemory(allocation_process_addr, bytes, size, error);
 
@@ -1102,9 +1104,12 @@ bool IRExecutionUnit::CommitOneAllocation(lldb::ProcessSP &process_sp,
     break;
   default:
     const bool zero_memory = false;
-    record.m_process_address =
-        Malloc(record.m_size, record.m_alignment, record.m_permissions,
-               eAllocationPolicyProcessOnly, zero_memory, error);
+    if (auto address_or_error =
+            Malloc(record.m_size, record.m_alignment, record.m_permissions,
+                   eAllocationPolicyProcessOnly, zero_memory))
+      record.m_process_address = *address_or_error;
+    else
+      error = Status::FromError(address_or_error.takeError());
     break;
   }
 
diff --git a/lldb/source/Expression/IRMemoryMap.cpp b/lldb/source/Expression/IRMemoryMap.cpp
index 65b5d11413c8..150699352a2e 100644
--- a/lldb/source/Expression/IRMemoryMap.cpp
+++ b/lldb/source/Expression/IRMemoryMap.cpp
@@ -319,11 +319,11 @@ IRMemoryMap::Allocation::Allocation(lldb::addr_t process_alloc,
   }
 }
 
-lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
-                                 uint32_t permissions, AllocationPolicy policy,
-                                 bool zero_memory, Status &error) {
+llvm::Expected<lldb::addr_t>
+IRMemoryMap::Malloc(size_t size, uint8_t alignment, uint32_t permissions,
+                    AllocationPolicy policy, bool zero_memory,
+                    AllocationPolicy *used_policy) {
   lldb_private::Log *log(GetLog(LLDBLog::Expressions));
-  error.Clear();
 
   lldb::ProcessSP process_sp;
   lldb::addr_t allocation_address = LLDB_INVALID_ADDRESS;
@@ -347,15 +347,14 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
 
   switch (policy) {
   default:
-    error =
-        Status::FromErrorString("Couldn't malloc: invalid allocation policy");
-    return LLDB_INVALID_ADDRESS;
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        "Couldn't malloc: invalid allocation policy");
   case eAllocationPolicyHostOnly:
     allocation_address = FindSpace(allocation_size);
-    if (allocation_address == LLDB_INVALID_ADDRESS) {
-      error = Status::FromErrorString("Couldn't malloc: address space is full");
-      return LLDB_INVALID_ADDRESS;
-    }
+    if (allocation_address == LLDB_INVALID_ADDRESS)
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Couldn't malloc: address space is full");
     break;
   case eAllocationPolicyMirror:
     process_sp = m_process_wp.lock();
@@ -366,6 +365,7 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               process_sp && process_sp->CanJIT() ? "true" : "false",
               process_sp && process_sp->IsAlive() ? "true" : "false");
     if (process_sp && process_sp->CanJIT() && process_sp->IsAlive()) {
+      Status error;
       if (!zero_memory)
         allocation_address =
             process_sp->AllocateMemory(allocation_size, permissions, error);
@@ -374,7 +374,7 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
             process_sp->CallocateMemory(allocation_size, permissions, error);
 
       if (!error.Success())
-        return LLDB_INVALID_ADDRESS;
+        return error.takeError();
     } else {
       LLDB_LOGF(log,
                 "IRMemoryMap::%s switching to eAllocationPolicyHostOnly "
@@ -382,17 +382,17 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
                 __FUNCTION__);
       policy = eAllocationPolicyHostOnly;
       allocation_address = FindSpace(allocation_size);
-      if (allocation_address == LLDB_INVALID_ADDRESS) {
-        error =
-            Status::FromErrorString("Couldn't malloc: address space is full");
-        return LLDB_INVALID_ADDRESS;
-      }
+      if (allocation_address == LLDB_INVALID_ADDRESS)
+        return llvm::createStringError(
+            llvm::inconvertibleErrorCode(),
+            "Couldn't malloc: address space is full");
     }
     break;
   case eAllocationPolicyProcessOnly:
     process_sp = m_process_wp.lock();
     if (process_sp) {
       if (process_sp->CanJIT() && process_sp->IsAlive()) {
+        Status error;
         if (!zero_memory)
           allocation_address =
               process_sp->AllocateMemory(allocation_size, permissions, error);
@@ -401,17 +401,16 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               process_sp->CallocateMemory(allocation_size, permissions, error);
 
         if (!error.Success())
-          return LLDB_INVALID_ADDRESS;
+          return error.takeError();
       } else {
-        error = Status::FromErrorString(
+        return llvm::createStringError(
+            llvm::inconvertibleErrorCode(),
             "Couldn't malloc: process doesn't support allocating memory");
-        return LLDB_INVALID_ADDRESS;
       }
     } else {
-      error = Status::FromErrorString(
-          "Couldn't malloc: process doesn't exist, and this "
-          "memory must be in the process");
-      return LLDB_INVALID_ADDRESS;
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Couldn't malloc: process doesn't exist, "
+                                     "and this memory must be in the process");
     }
     break;
   }
@@ -455,6 +454,9 @@ lldb::addr_t IRMemoryMap::Malloc(size_t size, uint8_t alignment,
               (uint64_t)permissions, policy_string, aligned_address);
   }
 
+  if (used_policy)
+    *used_policy = policy;
+
   return aligned_address;
 }
 
diff --git a/lldb/source/Expression/LLVMUserExpression.cpp b/lldb/source/Expression/LLVMUserExpression.cpp
index fac3ce6f5799..2d59194027b5 100644
--- a/lldb/source/Expression/LLVMUserExpression.cpp
+++ b/lldb/source/Expression/LLVMUserExpression.cpp
@@ -313,25 +313,22 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
 
   if (m_jit_start_addr != LLDB_INVALID_ADDRESS || m_can_interpret) {
     if (m_materialized_address == LLDB_INVALID_ADDRESS) {
-      Status alloc_error;
-
       IRMemoryMap::AllocationPolicy policy =
           m_can_interpret ? IRMemoryMap::eAllocationPolicyHostOnly
                           : IRMemoryMap::eAllocationPolicyMirror;
 
       const bool zero_memory = false;
-
-      m_materialized_address = m_execution_unit_sp->Malloc(
-          m_materializer_up->GetStructByteSize(),
-          m_materializer_up->GetStructAlignment(),
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable, policy,
-          zero_memory, alloc_error);
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = m_execution_unit_sp->Malloc(
+              m_materializer_up->GetStructByteSize(),
+              m_materializer_up->GetStructAlignment(),
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable, policy,
+              zero_memory)) {
+        m_materialized_address = *address_or_error;
+      } else {
         diagnostic_manager.Printf(
             lldb::eSeverityError,
             "Couldn't allocate space for materialized struct: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return false;
       }
     }
@@ -339,8 +336,6 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
     struct_address = m_materialized_address;
 
     if (m_can_interpret && m_stack_frame_bottom == LLDB_INVALID_ADDRESS) {
-      Status alloc_error;
-
       size_t stack_frame_size = target->GetExprAllocSize();
       if (stack_frame_size == 0) {
         ABISP abi_sp;
@@ -351,19 +346,17 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
       }
 
       const bool zero_memory = false;
-
-      m_stack_frame_bottom = m_execution_unit_sp->Malloc(
-          stack_frame_size, 8,
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-          IRMemoryMap::eAllocationPolicyHostOnly, zero_memory, alloc_error);
-
-      m_stack_frame_top = m_stack_frame_bottom + stack_frame_size;
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = m_execution_unit_sp->Malloc(
+              stack_frame_size, 8,
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+              IRMemoryMap::eAllocationPolicyHostOnly, zero_memory)) {
+        m_stack_frame_bottom = *address_or_error;
+        m_stack_frame_top = m_stack_frame_bottom + stack_frame_size;
+      } else {
         diagnostic_manager.Printf(
             lldb::eSeverityError,
             "Couldn't allocate space for the stack frame: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return false;
       }
     }
@@ -382,4 +375,3 @@ bool LLVMUserExpression::PrepareToExecuteJITExpression(
   }
   return true;
 }
-
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 79c804c6c421..17ea1596806d 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -74,22 +74,21 @@ class EntityPersistentVariable : public Materializer::Entity {
     // Allocate a spare memory area to store the persistent variable's
     // contents.
 
-    Status allocate_error;
     const bool zero_memory = false;
-
-    lldb::addr_t mem = map.Malloc(
+    IRMemoryMap::AllocationPolicy used_policy;
+    auto address_or_error = map.Malloc(
         llvm::expectedToOptional(m_persistent_variable_sp->GetByteSize())
             .value_or(0),
         8, lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-        IRMemoryMap::eAllocationPolicyMirror, zero_memory, allocate_error);
-
-    if (!allocate_error.Success()) {
+        IRMemoryMap::eAllocationPolicyMirror, zero_memory, &used_policy);
+    if (!address_or_error) {
       err = Status::FromErrorStringWithFormat(
           "couldn't allocate a memory area to store %s: %s",
           m_persistent_variable_sp->GetName().GetCString(),
-          allocate_error.AsCString());
+          toString(address_or_error.takeError()).c_str());
       return;
     }
+    lldb::addr_t mem = *address_or_error;
 
     LLDB_LOGF(log, "Allocated %s (0x%" PRIx64 ") successfully",
               m_persistent_variable_sp->GetName().GetCString(), mem);
@@ -103,14 +102,22 @@ class EntityPersistentVariable : public Materializer::Entity {
         m_persistent_variable_sp->GetName(), mem, eAddressTypeLoad,
         map.GetAddressByteSize());
 
-    // Clear the flag if the variable will never be deallocated.
-
     if (m_persistent_variable_sp->m_flags &
         ExpressionVariable::EVKeepInTarget) {
-      Status leak_error;
-      map.Leak(mem, leak_error);
-      m_persistent_variable_sp->m_flags &=
-          ~ExpressionVariable::EVNeedsAllocation;
+      if (used_policy == IRMemoryMap::eAllocationPolicyMirror) {
+        // Clear the flag if the variable will never be deallocated.
+        Status leak_error;
+        map.Leak(mem, leak_error);
+        m_persistent_variable_sp->m_flags &=
+            ~ExpressionVariable::EVNeedsAllocation;
+      } else {
+        // If the variable cannot be kept in target, clear this flag...
+        m_persistent_variable_sp->m_flags &=
+            ~ExpressionVariable::EVKeepInTarget;
+        // ...and set the flag to copy the value during dematerialization.
+        m_persistent_variable_sp->m_flags |=
+            ExpressionVariable::EVNeedsFreezeDry;
+      }
     }
 
     // Write the contents of the variable to the area.
@@ -329,22 +336,10 @@ class EntityPersistentVariable : public Materializer::Entity {
       return;
     }
 
-    lldb::ProcessSP process_sp =
-        map.GetBestExecutionContextScope()->CalculateProcess();
-    if (!process_sp || !process_sp->CanJIT()) {
-      // Allocations are not persistent so persistent variables cannot stay
-      // materialized.
-
-      m_persistent_variable_sp->m_flags |=
-          ExpressionVariable::EVNeedsAllocation;
-
-      DestroyAllocation(map, err);
-      if (!err.Success())
-        return;
-    } else if (m_persistent_variable_sp->m_flags &
-                   ExpressionVariable::EVNeedsAllocation &&
-               !(m_persistent_variable_sp->m_flags &
-                 ExpressionVariable::EVKeepInTarget)) {
+    if (m_persistent_variable_sp->m_flags &
+            ExpressionVariable::EVNeedsAllocation &&
+        !(m_persistent_variable_sp->m_flags &
+          ExpressionVariable::EVKeepInTarget)) {
       DestroyAllocation(map, err);
       if (!err.Success())
         return;
@@ -565,26 +560,25 @@ class EntityVariableBase : public Materializer::Entity {
 
         size_t byte_align = (*opt_bit_align + 7) / 8;
 
-        Status alloc_error;
         const bool zero_memory = false;
-
-        m_temporary_allocation = map.Malloc(
-            data.GetByteSize(), byte_align,
-            lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-            IRMemoryMap::eAllocationPolicyMirror, zero_memory, alloc_error);
+        if (auto address_or_error = map.Malloc(
+                data.GetByteSize(), byte_align,
+                lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+                IRMemoryMap::eAllocationPolicyMirror, zero_memory)) {
+          m_temporary_allocation = *address_or_error;
+        } else {
+          err = Status::FromErrorStringWithFormat(
+              "couldn't allocate a temporary region for %s: %s",
+              GetName().AsCString(),
+              toString(address_or_error.takeError()).c_str());
+          return;
+        }
 
         m_temporary_allocation_size = data.GetByteSize();
 
         m_original_data = std::make_shared<DataBufferHeap>(data.GetDataStart(),
                                                            data.GetByteSize());
 
-        if (!alloc_error.Success()) {
-          err = Status::FromErrorStringWithFormat(
-              "couldn't allocate a temporary region for %s: %s",
-              GetName().AsCString(), alloc_error.AsCString());
-          return;
-        }
-
         Status write_error;
 
         map.WriteMemory(m_temporary_allocation, data.GetDataStart(),
@@ -967,22 +961,21 @@ class EntityResultVariable : public Materializer::Entity {
 
       size_t byte_align = (*opt_bit_align + 7) / 8;
 
-      Status alloc_error;
       const bool zero_memory = true;
-
-      m_temporary_allocation = map.Malloc(
-          byte_size, byte_align,
-          lldb::ePermissionsReadable | lldb::ePermissionsWritable,
-          IRMemoryMap::eAllocationPolicyMirror, zero_memory, alloc_error);
-      m_temporary_allocation_size = byte_size;
-
-      if (!alloc_error.Success()) {
+      if (auto address_or_error = map.Malloc(
+              byte_size, byte_align,
+              lldb::ePermissionsReadable | lldb::ePermissionsWritable,
+              IRMemoryMap::eAllocationPolicyMirror, zero_memory)) {
+        m_temporary_allocation = *address_or_error;
+      } else {
         err = Status::FromErrorStringWithFormat(
             "couldn't allocate a temporary region for the result: %s",
-            alloc_error.AsCString());
+            toString(address_or_error.takeError()).c_str());
         return;
       }
 
+      m_temporary_allocation_size = byte_size;
+
       Status pointer_write_error;
 
       map.WritePointerToMemory(load_addr, m_temporary_allocation,
@@ -1086,9 +1079,8 @@ class EntityResultVariable : public Materializer::Entity {
       m_delegate->DidDematerialize(ret);
     }
 
-    bool can_persist =
-        (m_is_program_reference && process_sp && process_sp->CanJIT() &&
-         !(address >= frame_bottom && address < frame_top));
+    bool can_persist = m_is_program_reference &&
+                       !(address >= frame_bottom && address < frame_top);
 
     if (can_persist && m_keep_in_memory) {
       ret->m_live_sp = ValueObjectConstResult::Create(exe_scope, m_type, name,
@@ -1118,7 +1110,9 @@ class EntityResultVariable : public Materializer::Entity {
         map.Free(m_temporary_allocation, free_error);
       }
     } else {
-      ret->m_flags |= ExpressionVariable::EVIsLLDBAllocated;
+      ret->m_flags |= m_is_program_reference
+                          ? ExpressionVariable::EVIsProgramReference
+                          : ExpressionVariable::EVIsLLDBAllocated;
     }
 
     m_temporary_allocation = LLDB_INVALID_ADDRESS;
diff --git a/lldb/source/Host/common/Editline.cpp b/lldb/source/Host/common/Editline.cpp
index b251ded6c379..5ed30fbb231d 100644
--- a/lldb/source/Host/common/Editline.cpp
+++ b/lldb/source/Host/common/Editline.cpp
@@ -122,22 +122,21 @@ static int GetOperation(HistoryOperation op) {
   //  - The H_FIRST returns the most recent entry in the history.
   //
   // The naming of the enum entries match the semantic meaning.
-  switch(op) {
-    case HistoryOperation::Oldest:
-      return H_LAST;
-    case HistoryOperation::Older:
-      return H_NEXT;
-    case HistoryOperation::Current:
-      return H_CURR;
-    case HistoryOperation::Newer:
-      return H_PREV;
-    case HistoryOperation::Newest:
-      return H_FIRST;
+  switch (op) {
+  case HistoryOperation::Oldest:
+    return H_LAST;
+  case HistoryOperation::Older:
+    return H_NEXT;
+  case HistoryOperation::Current:
+    return H_CURR;
+  case HistoryOperation::Newer:
+    return H_PREV;
+  case HistoryOperation::Newest:
+    return H_FIRST;
   }
   llvm_unreachable("Fully covered switch!");
 }
 
-
 EditLineStringType CombineLines(const std::vector<EditLineStringType> &lines) {
   EditLineStringStreamType combined_stream;
   for (EditLineStringType line : lines) {
@@ -313,8 +312,8 @@ class EditlineHistory {
   /// Path to the history file.
   std::string m_path;
 };
-}
-}
+} // namespace line_editor
+} // namespace lldb_private
 
 // Editline private methods
 
@@ -1151,7 +1150,8 @@ unsigned char Editline::TabCommand(int ch) {
       to_add.push_back(' ');
       el_deletestr(m_editline, request.GetCursorArgumentPrefix().size());
       el_insertstr(m_editline, to_add.c_str());
-      // Clear all the autosuggestion parts if the only single space can be completed.
+      // Clear all the autosuggestion parts if the only single space can be
+      // completed.
       if (to_add == " ")
         return CC_REDISPLAY;
       return CC_REFRESH;
@@ -1709,6 +1709,13 @@ void Editline::PrintAsync(lldb::LockableStreamFileSP stream_sp, const char *s,
   }
 }
 
+void Editline::Refresh() {
+  if (!m_editline || !m_output_stream_sp)
+    return;
+  LockedStreamFile locked_stream = m_output_stream_sp->Lock();
+  el_set(m_editline, EL_REFRESH);
+}
+
 bool Editline::CompleteCharacter(char ch, EditLineGetCharType &out) {
 #if !LLDB_EDITLINE_USE_WCHAR
   if (ch == (char)EOF)
diff --git a/lldb/source/Host/common/File.cpp b/lldb/source/Host/common/File.cpp
index 9aa95ffda44c..23b6dc9fe850 100644
--- a/lldb/source/Host/common/File.cpp
+++ b/lldb/source/Host/common/File.cpp
@@ -274,7 +274,11 @@ int NativeFile::GetDescriptor() const {
 }
 
 IOObject::WaitableHandle NativeFile::GetWaitableHandle() {
+#ifdef _WIN32
+  return (HANDLE)_get_osfhandle(GetDescriptor());
+#else
   return GetDescriptor();
+#endif
 }
 
 FILE *NativeFile::GetStream() {
diff --git a/lldb/source/Host/common/JSONTransport.cpp b/lldb/source/Host/common/JSONTransport.cpp
index 1a0851d5c436..546c12c8f711 100644
--- a/lldb/source/Host/common/JSONTransport.cpp
+++ b/lldb/source/Host/common/JSONTransport.cpp
@@ -42,7 +42,8 @@ ReadFull(IOObject &descriptor, size_t length,
   if (timeout && timeout_supported) {
     SelectHelper sh;
     sh.SetTimeout(*timeout);
-    sh.FDSetRead(descriptor.GetWaitableHandle());
+    sh.FDSetRead(
+        reinterpret_cast<lldb::socket_t>(descriptor.GetWaitableHandle()));
     Status status = sh.Select();
     if (status.Fail()) {
       // Convert timeouts into a specific error.
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index 2b23fd1e6e57..3511cde8bb36 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -69,7 +69,7 @@ SharedSocket::SharedSocket(const Socket *socket, Status &error) {
   m_fd = kInvalidFD;
 
   // Create a pipe to transfer WSAPROTOCOL_INFO to the child process.
-  error = m_socket_pipe.CreateNew(true);
+  error = m_socket_pipe.CreateNew();
   if (error.Fail())
     return;
 
@@ -313,8 +313,7 @@ Socket::DecodeHostAndPort(llvm::StringRef host_and_port) {
 }
 
 IOObject::WaitableHandle Socket::GetWaitableHandle() {
-  // TODO: On Windows, use WSAEventSelect
-  return m_socket;
+  return (IOObject::WaitableHandle)m_socket;
 }
 
 Status Socket::Read(void *buf, size_t &num_bytes) {
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index e187bf98188a..3c1d1179963d 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -595,7 +595,9 @@ DataExtractor data(arg_data.GetBytes(), arg_data_size,
       const llvm::Triple::ArchType triple_arch = triple.getArch();
       const bool check_for_ios_simulator =
           (triple_arch == llvm::Triple::x86 ||
-           triple_arch == llvm::Triple::x86_64);
+           triple_arch == llvm::Triple::x86_64 ||
+           triple_arch == llvm::Triple::aarch64);
+
       const char *cstr = data.GetCStr(&offset);
       if (cstr) {
         process_info.GetExecutableFile().SetFile(cstr, FileSpec::Style::native);
@@ -621,21 +623,20 @@ DataExtractor data(arg_data.GetBytes(), arg_data_size,
           }
 
           Environment &proc_env = process_info.GetEnvironment();
-          while ((cstr = data.GetCStr(&offset))) {
-            if (cstr[0] == '\0')
-              break;
-
-            if (check_for_ios_simulator) {
-              if (strncmp(cstr, "SIMULATOR_UDID=", strlen("SIMULATOR_UDID=")) ==
-                  0)
-                process_info.GetArchitecture().GetTriple().setOS(
-                    llvm::Triple::IOS);
-              else
-                process_info.GetArchitecture().GetTriple().setOS(
-                    llvm::Triple::MacOSX);
-            }
-
-            proc_env.insert(cstr);
+          bool is_simulator = false;
+          llvm::StringRef env_var;
+          while (!(env_var = data.GetCStr(&offset)).empty()) {
+            if (check_for_ios_simulator &&
+                env_var.starts_with("SIMULATOR_UDID="))
+              is_simulator = true;
+            proc_env.insert(env_var);
+          }
+          llvm::Triple &triple = process_info.GetArchitecture().GetTriple();
+          if (is_simulator) {
+            triple.setOS(llvm::Triple::IOS);
+            triple.setEnvironment(llvm::Triple::Simulator);
+          } else {
+            triple.setOS(llvm::Triple::MacOSX);
           }
           return true;
         }
@@ -741,8 +742,8 @@ static bool GetMacOSXProcessUserAndGroup(ProcessInstanceInfo &process_info) {
         !match_info.ProcessIDsMatch(process_info))
       continue;
 
-    // Get CPU type first so we can know to look for iOS simulator is we have
-    // x86 or x86_64
+    // Get CPU type first so we can know to look for iOS simulator if we have
+    // a compatible type.
     if (GetMacOSXProcessCPUType(process_info)) {
       if (GetMacOSXProcessArgs(&match_info, process_info)) {
         if (match_info.Matches(process_info))
diff --git a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
index 57dce44812c8..e8bf04e30844 100644
--- a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
+++ b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
@@ -92,7 +92,7 @@ void ConnectionFileDescriptor::OpenCommandPipe() {
 
   Log *log = GetLog(LLDBLog::Connection);
   // Make the command file descriptor here:
-  Status result = m_pipe.CreateNew(/*child_processes_inherit=*/false);
+  Status result = m_pipe.CreateNew();
   if (!result.Success()) {
     LLDB_LOGF(log,
               "%p ConnectionFileDescriptor::OpenCommandPipe () - could not "
@@ -272,13 +272,11 @@ size_t ConnectionFileDescriptor::Read(void *dst, size_t dst_len,
   error = m_io_sp->Read(dst, bytes_read);
 
   if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionFileDescriptor::Read()  fd = %" PRIu64
-              ", dst = %p, dst_len = %" PRIu64 ") => %" PRIu64 ", error = %s",
-              static_cast<void *>(this),
-              static_cast<uint64_t>(m_io_sp->GetWaitableHandle()),
-              static_cast<void *>(dst), static_cast<uint64_t>(dst_len),
-              static_cast<uint64_t>(bytes_read), error.AsCString());
+    LLDB_LOG(log,
+             "{0} ConnectionFileDescriptor::Read()  fd = {1}"
+             ", dst = {2}, dst_len = {3}) => {4}, error = {5}",
+             this, m_io_sp->GetWaitableHandle(), dst, dst_len, bytes_read,
+             error.AsCString());
   }
 
   if (bytes_read == 0) {
@@ -376,13 +374,11 @@ size_t ConnectionFileDescriptor::Write(const void *src, size_t src_len,
   error = m_io_sp->Write(src, bytes_sent);
 
   if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionFileDescriptor::Write(fd = %" PRIu64
-              ", src = %p, src_len = %" PRIu64 ") => %" PRIu64 " (error = %s)",
-              static_cast<void *>(this),
-              static_cast<uint64_t>(m_io_sp->GetWaitableHandle()),
-              static_cast<const void *>(src), static_cast<uint64_t>(src_len),
-              static_cast<uint64_t>(bytes_sent), error.AsCString());
+    LLDB_LOG(log,
+             "{0} ConnectionFileDescriptor::Write(fd = {1}"
+             ", src = {2}, src_len = {3}) => {4} (error = {5})",
+             this, m_io_sp->GetWaitableHandle(), src, src_len, bytes_sent,
+             error.AsCString());
   }
 
   if (error_ptr)
@@ -451,7 +447,8 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout<std::micro> &timeout,
     if (timeout)
       select_helper.SetTimeout(*timeout);
 
-    select_helper.FDSetRead(handle);
+    // FIXME: Migrate to MainLoop.
+    select_helper.FDSetRead(reinterpret_cast<socket_t>(handle));
 #if defined(_WIN32)
     // select() won't accept pipes on Windows.  The entire Windows codepath
     // needs to be converted over to using WaitForMultipleObjects and event
@@ -493,7 +490,7 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout<std::micro> &timeout,
           break; // Lets keep reading to until we timeout
         }
       } else {
-        if (select_helper.FDIsSetRead(handle))
+        if (select_helper.FDIsSetRead((lldb::socket_t)handle))
           return eConnectionStatusSuccess;
 
         if (select_helper.FDIsSetRead(pipe_fd)) {
diff --git a/lldb/source/Host/posix/HostInfoPosix.cpp b/lldb/source/Host/posix/HostInfoPosix.cpp
index 879dccfd353b..704f77891c82 100644
--- a/lldb/source/Host/posix/HostInfoPosix.cpp
+++ b/lldb/source/Host/posix/HostInfoPosix.cpp
@@ -140,6 +140,12 @@ std::optional<std::string> PosixUserIDResolver::DoGetGroupName(id_t gid) {
   return std::nullopt;
 }
 
+/// The SDK is the directory where the system C headers, libraries, can be
+/// found. On POSIX platforms this is simply the root directory.
+llvm::Expected<llvm::StringRef> HostInfoPosix::GetSDKRoot(SDKOptions options) {
+  return "/";
+}
+
 static llvm::ManagedStatic<PosixUserIDResolver> g_user_id_resolver;
 
 UserIDResolver &HostInfoPosix::GetUserIDResolver() {
diff --git a/lldb/source/Host/posix/MainLoopPosix.cpp b/lldb/source/Host/posix/MainLoopPosix.cpp
index b4d629708eb3..19a7128fbe40 100644
--- a/lldb/source/Host/posix/MainLoopPosix.cpp
+++ b/lldb/source/Host/posix/MainLoopPosix.cpp
@@ -208,7 +208,7 @@ void MainLoopPosix::RunImpl::ProcessReadEvents() {
 #endif
 
 MainLoopPosix::MainLoopPosix() {
-  Status error = m_interrupt_pipe.CreateNew(/*child_process_inherit=*/false);
+  Status error = m_interrupt_pipe.CreateNew();
   assert(error.Success());
 
   // Make the write end of the pipe non-blocking.
diff --git a/lldb/source/Host/posix/PipePosix.cpp b/lldb/source/Host/posix/PipePosix.cpp
index a8c4f8df333a..68b02a9aa465 100644
--- a/lldb/source/Host/posix/PipePosix.cpp
+++ b/lldb/source/Host/posix/PipePosix.cpp
@@ -79,24 +79,22 @@ PipePosix &PipePosix::operator=(PipePosix &&pipe_posix) {
 
 PipePosix::~PipePosix() { Close(); }
 
-Status PipePosix::CreateNew(bool child_processes_inherit) {
+Status PipePosix::CreateNew() {
   std::scoped_lock<std::mutex, std::mutex> guard(m_read_mutex, m_write_mutex);
   if (CanReadUnlocked() || CanWriteUnlocked())
     return Status(EINVAL, eErrorTypePOSIX);
 
   Status error;
 #if PIPE2_SUPPORTED
-  if (::pipe2(m_fds, (child_processes_inherit) ? 0 : O_CLOEXEC) == 0)
+  if (::pipe2(m_fds, O_CLOEXEC) == 0)
     return error;
 #else
   if (::pipe(m_fds) == 0) {
 #ifdef FD_CLOEXEC
-    if (!child_processes_inherit) {
-      if (!SetCloexecFlag(m_fds[0]) || !SetCloexecFlag(m_fds[1])) {
-        error = Status::FromErrno();
-        CloseUnlocked();
-        return error;
-      }
+    if (!SetCloexecFlag(m_fds[0]) || !SetCloexecFlag(m_fds[1])) {
+      error = Status::FromErrno();
+      CloseUnlocked();
+      return error;
     }
 #endif
     return error;
@@ -109,7 +107,7 @@ Status PipePosix::CreateNew(bool child_processes_inherit) {
   return error;
 }
 
-Status PipePosix::CreateNew(llvm::StringRef name, bool child_process_inherit) {
+Status PipePosix::CreateNew(llvm::StringRef name) {
   std::scoped_lock<std::mutex, std::mutex> guard(m_read_mutex, m_write_mutex);
   if (CanReadUnlocked() || CanWriteUnlocked())
     return Status::FromErrorString("Pipe is already opened");
@@ -121,7 +119,6 @@ Status PipePosix::CreateNew(llvm::StringRef name, bool child_process_inherit) {
 }
 
 Status PipePosix::CreateWithUniqueName(llvm::StringRef prefix,
-                                       bool child_process_inherit,
                                        llvm::SmallVectorImpl<char> &name) {
   llvm::SmallString<128> named_pipe_path;
   llvm::SmallString<128> pipe_spec((prefix + ".%%%%%%").str());
@@ -137,7 +134,7 @@ Status PipePosix::CreateWithUniqueName(llvm::StringRef prefix,
   do {
     llvm::sys::fs::createUniquePath(tmpdir_file_spec.GetPath(), named_pipe_path,
                                     /*MakeAbsolute=*/false);
-    error = CreateNew(named_pipe_path, child_process_inherit);
+    error = CreateNew(named_pipe_path);
   } while (error.GetError() == EEXIST);
 
   if (error.Success())
@@ -145,16 +142,13 @@ Status PipePosix::CreateWithUniqueName(llvm::StringRef prefix,
   return error;
 }
 
-Status PipePosix::OpenAsReader(llvm::StringRef name,
-                               bool child_process_inherit) {
+Status PipePosix::OpenAsReader(llvm::StringRef name) {
   std::scoped_lock<std::mutex, std::mutex> guard(m_read_mutex, m_write_mutex);
 
   if (CanReadUnlocked() || CanWriteUnlocked())
     return Status::FromErrorString("Pipe is already opened");
 
-  int flags = O_RDONLY | O_NONBLOCK;
-  if (!child_process_inherit)
-    flags |= O_CLOEXEC;
+  int flags = O_RDONLY | O_NONBLOCK | O_CLOEXEC;
 
   Status error;
   int fd = FileSystem::Instance().Open(name.str().c_str(), flags);
@@ -167,15 +161,12 @@ Status PipePosix::OpenAsReader(llvm::StringRef name,
 }
 
 llvm::Error PipePosix::OpenAsWriter(llvm::StringRef name,
-                                    bool child_process_inherit,
                                     const Timeout<std::micro> &timeout) {
   std::lock_guard<std::mutex> guard(m_write_mutex);
   if (CanReadUnlocked() || CanWriteUnlocked())
     return llvm::createStringError("Pipe is already opened");
 
-  int flags = O_WRONLY | O_NONBLOCK;
-  if (!child_process_inherit)
-    flags |= O_CLOEXEC;
+  int flags = O_WRONLY | O_NONBLOCK | O_CLOEXEC;
 
   using namespace std::chrono;
   std::optional<time_point<steady_clock>> finish_time;
diff --git a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
index 698524349e16..15a809784fbc 100644
--- a/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
+++ b/lldb/source/Host/posix/ProcessLauncherPosixFork.cpp
@@ -255,8 +255,7 @@ ProcessLauncherPosixFork::LaunchProcess(const ProcessLaunchInfo &launch_info,
                                         Status &error) {
   // A pipe used by the child process to report errors.
   PipePosix pipe;
-  const bool child_processes_inherit = false;
-  error = pipe.CreateNew(child_processes_inherit);
+  error = pipe.CreateNew();
   if (error.Fail())
     return HostProcess();
 
diff --git a/lldb/source/Host/windows/MainLoopWindows.cpp b/lldb/source/Host/windows/MainLoopWindows.cpp
index f3ab2a710cd0..b3322e8b3ae5 100644
--- a/lldb/source/Host/windows/MainLoopWindows.cpp
+++ b/lldb/source/Host/windows/MainLoopWindows.cpp
@@ -8,13 +8,19 @@
 
 #include "lldb/Host/windows/MainLoopWindows.h"
 #include "lldb/Host/Config.h"
+#include "lldb/Host/Socket.h"
+#include "lldb/Host/windows/windows.h"
 #include "lldb/Utility/Status.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/WindowsError.h"
 #include <algorithm>
 #include <cassert>
 #include <cerrno>
 #include <csignal>
 #include <ctime>
+#include <io.h>
+#include <thread>
 #include <vector>
 #include <winsock2.h>
 
@@ -31,6 +37,122 @@ static DWORD ToTimeout(std::optional<MainLoopWindows::TimePoint> point) {
   return ceil<milliseconds>(dur).count();
 }
 
+namespace {
+
+class PipeEvent : public MainLoopWindows::IOEvent {
+public:
+  explicit PipeEvent(HANDLE handle)
+      : IOEvent(CreateEventW(NULL, /*bManualReset=*/FALSE,
+                             /*bInitialState=*/FALSE, NULL)),
+        m_handle(handle), m_ready(CreateEventW(NULL, /*bManualReset=*/FALSE,
+                                               /*bInitialState=*/FALSE, NULL)) {
+    assert(m_event && m_ready);
+  }
+
+  ~PipeEvent() override {
+    if (m_monitor_thread.joinable()) {
+      m_stopped = true;
+      SetEvent(m_ready);
+      // Keep trying to cancel ReadFile() until the thread exits.
+      do {
+        CancelIoEx(m_handle, /*lpOverlapped=*/NULL);
+      } while (WaitForSingleObject(m_monitor_thread.native_handle(), 1) ==
+               WAIT_TIMEOUT);
+      m_monitor_thread.join();
+    }
+    CloseHandle(m_event);
+    CloseHandle(m_ready);
+  }
+
+  void WillPoll() override {
+    if (!m_monitor_thread.joinable())
+      m_monitor_thread = std::thread(&PipeEvent::Monitor, this);
+  }
+
+  void Disarm() override { SetEvent(m_ready); }
+
+  /// Monitors the handle performing a zero byte read to determine when data is
+  /// avaiable.
+  void Monitor() {
+    do {
+      char buf[1];
+      DWORD bytes_read = 0;
+      OVERLAPPED ov;
+      ZeroMemory(&ov, sizeof(ov));
+      // Block on a 0-byte read; this will only resume when data is
+      // available in the pipe. The pipe must be PIPE_WAIT or this thread
+      // will spin.
+      BOOL success =
+          ReadFile(m_handle, buf, /*nNumberOfBytesToRead=*/0, &bytes_read, &ov);
+      DWORD bytes_available = 0;
+      DWORD err = GetLastError();
+      if (!success && err == ERROR_IO_PENDING) {
+        success = GetOverlappedResult(m_handle, &ov, &bytes_read,
+                                      /*bWait=*/TRUE);
+        err = GetLastError();
+      }
+      if (success) {
+        success =
+            PeekNamedPipe(m_handle, NULL, 0, NULL, &bytes_available, NULL);
+        err = GetLastError();
+      }
+      if (success) {
+        if (bytes_available == 0) {
+          // This can happen with a zero-byte write. Try again.
+          continue;
+        }
+      } else if (err == ERROR_NO_DATA) {
+        // The pipe is nonblocking. Try again.
+        Sleep(0);
+        continue;
+      } else if (err == ERROR_OPERATION_ABORTED) {
+        // Read may have been cancelled, try again.
+        continue;
+      }
+
+      SetEvent(m_event);
+
+      // Wait until the current read is consumed before doing the next read.
+      WaitForSingleObject(m_ready, INFINITE);
+    } while (!m_stopped);
+  }
+
+private:
+  HANDLE m_handle;
+  HANDLE m_ready;
+  std::thread m_monitor_thread;
+  std::atomic<bool> m_stopped = false;
+};
+
+class SocketEvent : public MainLoopWindows::IOEvent {
+public:
+  explicit SocketEvent(SOCKET socket)
+      : IOEvent(WSACreateEvent()), m_socket(socket) {
+    assert(m_event != WSA_INVALID_EVENT);
+  }
+
+  ~SocketEvent() override { WSACloseEvent(m_event); }
+
+  void WillPoll() override {
+    int result =
+        WSAEventSelect(m_socket, m_event, FD_READ | FD_ACCEPT | FD_CLOSE);
+    assert(result == 0);
+    UNUSED_IF_ASSERT_DISABLED(result);
+  }
+
+  void DidPoll() override {
+    int result = WSAEventSelect(m_socket, WSA_INVALID_EVENT, 0);
+    assert(result == 0);
+    UNUSED_IF_ASSERT_DISABLED(result);
+  }
+
+  void Disarm() override { WSAResetEvent(m_event); }
+
+  SOCKET m_socket;
+};
+
+} // namespace
+
 MainLoopWindows::MainLoopWindows() {
   m_interrupt_event = WSACreateEvent();
   assert(m_interrupt_event != WSA_INVALID_EVENT);
@@ -44,14 +166,11 @@ MainLoopWindows::~MainLoopWindows() {
 }
 
 llvm::Expected<size_t> MainLoopWindows::Poll() {
-  std::vector<WSAEVENT> events;
+  std::vector<HANDLE> events;
   events.reserve(m_read_fds.size() + 1);
-  for (auto &[fd, info] : m_read_fds) {
-    int result = WSAEventSelect(fd, info.event, FD_READ | FD_ACCEPT | FD_CLOSE);
-    assert(result == 0);
-    UNUSED_IF_ASSERT_DISABLED(result);
-
-    events.push_back(info.event);
+  for (auto &[_, fd_info] : m_read_fds) {
+    fd_info.event->WillPoll();
+    events.push_back(fd_info.event->GetHandle());
   }
   events.push_back(m_interrupt_event);
 
@@ -59,11 +178,8 @@ llvm::Expected<size_t> MainLoopWindows::Poll() {
       WSAWaitForMultipleEvents(events.size(), events.data(), FALSE,
                                ToTimeout(GetNextWakeupTime()), FALSE);
 
-  for (auto &fd : m_read_fds) {
-    int result = WSAEventSelect(fd.first, WSA_INVALID_EVENT, 0);
-    assert(result == 0);
-    UNUSED_IF_ASSERT_DISABLED(result);
-  }
+  for (auto &[_, fd_info] : m_read_fds)
+    fd_info.event->DidPoll();
 
   if (result >= WSA_WAIT_EVENT_0 && result < WSA_WAIT_EVENT_0 + events.size())
     return result - WSA_WAIT_EVENT_0;
@@ -83,49 +199,42 @@ MainLoopWindows::RegisterReadObject(const IOObjectSP &object_sp,
     error = Status::FromErrorString("IO object is not valid.");
     return nullptr;
   }
-  if (object_sp->GetFdType() != IOObject::eFDTypeSocket) {
-    error = Status::FromErrorString(
-        "MainLoopWindows: non-socket types unsupported on Windows");
-    return nullptr;
-  }
 
-  WSAEVENT event = WSACreateEvent();
-  if (event == WSA_INVALID_EVENT) {
-    error =
-        Status::FromErrorStringWithFormat("Cannot create monitoring event.");
-    return nullptr;
-  }
+  IOObject::WaitableHandle waitable_handle = object_sp->GetWaitableHandle();
+  assert(waitable_handle != IOObject::kInvalidHandleValue);
 
-  const bool inserted =
-      m_read_fds
-          .try_emplace(object_sp->GetWaitableHandle(), FdInfo{event, callback})
-          .second;
-  if (!inserted) {
-    WSACloseEvent(event);
+  if (m_read_fds.find(waitable_handle) != m_read_fds.end()) {
     error = Status::FromErrorStringWithFormat(
-        "File descriptor %d already monitored.",
-        object_sp->GetWaitableHandle());
+        "File descriptor %d already monitored.", waitable_handle);
     return nullptr;
   }
 
+  if (object_sp->GetFdType() == IOObject::eFDTypeSocket) {
+    m_read_fds[waitable_handle] = {
+        std::make_unique<SocketEvent>(
+            reinterpret_cast<SOCKET>(waitable_handle)),
+        callback};
+  } else {
+    DWORD file_type = GetFileType(waitable_handle);
+    if (file_type != FILE_TYPE_PIPE) {
+      error = Status::FromErrorStringWithFormat("Unsupported file type %d",
+                                                file_type);
+      return nullptr;
+    }
+
+    m_read_fds[waitable_handle] = {std::make_unique<PipeEvent>(waitable_handle),
+                                   callback};
+  }
+
   return CreateReadHandle(object_sp);
 }
 
 void MainLoopWindows::UnregisterReadObject(IOObject::WaitableHandle handle) {
   auto it = m_read_fds.find(handle);
   assert(it != m_read_fds.end());
-  BOOL result = WSACloseEvent(it->second.event);
-  assert(result == TRUE);
-  UNUSED_IF_ASSERT_DISABLED(result);
   m_read_fds.erase(it);
 }
 
-void MainLoopWindows::ProcessReadObject(IOObject::WaitableHandle handle) {
-  auto it = m_read_fds.find(handle);
-  if (it != m_read_fds.end())
-    it->second.callback(*this); // Do the work
-}
-
 Status MainLoopWindows::Run() {
   m_terminate_request = false;
 
@@ -138,8 +247,8 @@ Status MainLoopWindows::Run() {
 
     if (*signaled_event < m_read_fds.size()) {
       auto &KV = *std::next(m_read_fds.begin(), *signaled_event);
-      WSAResetEvent(KV.second.event);
-      ProcessReadObject(KV.first);
+      KV.second.event->Disarm();
+      KV.second.callback(*this); // Do the work.
     } else {
       assert(*signaled_event == m_read_fds.size());
       WSAResetEvent(m_interrupt_event);
diff --git a/lldb/source/Host/windows/PipeWindows.cpp b/lldb/source/Host/windows/PipeWindows.cpp
index 30b9d1c41695..0b495fff69df 100644
--- a/lldb/source/Host/windows/PipeWindows.cpp
+++ b/lldb/source/Host/windows/PipeWindows.cpp
@@ -66,7 +66,7 @@ PipeWindows::PipeWindows(pipe_t read, pipe_t write)
 
 PipeWindows::~PipeWindows() { Close(); }
 
-Status PipeWindows::CreateNew(bool child_process_inherit) {
+Status PipeWindows::CreateNew() {
   // Even for anonymous pipes, we open a named pipe.  This is because you
   // cannot get overlapped i/o on Windows without using a named pipe.  So we
   // synthesize a unique name.
@@ -74,11 +74,10 @@ Status PipeWindows::CreateNew(bool child_process_inherit) {
   std::string pipe_name = llvm::formatv(
       "lldb.pipe.{0}.{1}.{2}", GetCurrentProcessId(), &g_pipe_serial, serial);
 
-  return CreateNew(pipe_name.c_str(), child_process_inherit);
+  return CreateNew(pipe_name.c_str());
 }
 
-Status PipeWindows::CreateNew(llvm::StringRef name,
-                              bool child_process_inherit) {
+Status PipeWindows::CreateNew(llvm::StringRef name) {
   if (name.empty())
     return Status(ERROR_INVALID_PARAMETER, eErrorTypeWin32);
 
@@ -109,7 +108,7 @@ Status PipeWindows::CreateNew(llvm::StringRef name,
 
   // Open the write end of the pipe. Note that closing either the read or 
   // write end of the pipe could directly close the pipe itself.
-  Status result = OpenNamedPipe(name, child_process_inherit, false);
+  Status result = OpenNamedPipe(name, false);
   if (!result.Success()) {
     CloseReadFileDescriptor();
     return result;
@@ -119,7 +118,6 @@ Status PipeWindows::CreateNew(llvm::StringRef name,
 }
 
 Status PipeWindows::CreateWithUniqueName(llvm::StringRef prefix,
-                                         bool child_process_inherit,
                                          llvm::SmallVectorImpl<char> &name) {
   llvm::SmallString<128> pipe_name;
   Status error;
@@ -133,7 +131,7 @@ Status PipeWindows::CreateWithUniqueName(llvm::StringRef prefix,
     pipe_name += "-";
     pipe_name += reinterpret_cast<char *>(unique_string);
     ::RpcStringFreeA(&unique_string);
-    error = CreateNew(pipe_name, child_process_inherit);
+    error = CreateNew(pipe_name);
   } else {
     error = Status(status, eErrorTypeWin32);
   }
@@ -142,25 +140,22 @@ Status PipeWindows::CreateWithUniqueName(llvm::StringRef prefix,
   return error;
 }
 
-Status PipeWindows::OpenAsReader(llvm::StringRef name,
-                                 bool child_process_inherit) {
+Status PipeWindows::OpenAsReader(llvm::StringRef name) {
   if (CanRead())
     return Status(); // Note the name is ignored.
 
-  return OpenNamedPipe(name, child_process_inherit, true);
+  return OpenNamedPipe(name, true);
 }
 
 llvm::Error PipeWindows::OpenAsWriter(llvm::StringRef name,
-                                      bool child_process_inherit,
                                       const Timeout<std::micro> &timeout) {
   if (CanWrite())
     return llvm::Error::success(); // Note the name is ignored.
 
-  return OpenNamedPipe(name, child_process_inherit, false).takeError();
+  return OpenNamedPipe(name, false).takeError();
 }
 
-Status PipeWindows::OpenNamedPipe(llvm::StringRef name,
-                                  bool child_process_inherit, bool is_read) {
+Status PipeWindows::OpenNamedPipe(llvm::StringRef name, bool is_read) {
   if (name.empty())
     return Status(ERROR_INVALID_PARAMETER, eErrorTypeWin32);
 
diff --git a/lldb/source/Interpreter/ScriptInterpreter.cpp b/lldb/source/Interpreter/ScriptInterpreter.cpp
index 63655cc5a50c..ae913e68feb1 100644
--- a/lldb/source/Interpreter/ScriptInterpreter.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreter.cpp
@@ -220,7 +220,7 @@ ScriptInterpreterIORedirect::ScriptInterpreterIORedirect(
     m_input_file_sp = debugger.GetInputFileSP();
 
     Pipe pipe;
-    Status pipe_result = pipe.CreateNew(false);
+    Status pipe_result = pipe.CreateNew();
 #if defined(_WIN32)
     lldb::file_t read_file = pipe.GetReadNativeHandle();
     pipe.ReleaseReadFileDescriptor();
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
index 32975d88af46..89da4d200699 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.cpp
@@ -14471,3 +14471,16 @@ bool EmulateInstructionARM::CreateFunctionEntryUnwind(UnwindPlan &unwind_plan) {
   unwind_plan.SetReturnAddressRegister(dwarf_lr);
   return true;
 }
+
+llvm::Expected<unsigned>
+ARMSingleStepBreakpointLocationsPredictor::GetBreakpointSize(
+    lldb::addr_t bp_addr) {
+  auto flags = m_emulator_up->ReadRegisterUnsigned(
+      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS, LLDB_INVALID_ADDRESS,
+      nullptr);
+  if (flags == LLDB_INVALID_ADDRESS)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Reading flags failed!");
+
+  return (flags & 0x20) ? /* Thumb mode */ 2 : /* Arm mode */ 4;
+}
diff --git a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
index 9eca0288607c..7931e0e648fb 100644
--- a/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
+++ b/lldb/source/Plugins/Instruction/ARM/EmulateInstructionARM.h
@@ -16,6 +16,16 @@
 
 namespace lldb_private {
 
+class ARMSingleStepBreakpointLocationsPredictor
+    : public SingleStepBreakpointLocationsPredictor {
+public:
+  ARMSingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator_up)
+      : SingleStepBreakpointLocationsPredictor{std::move(emulator_up)} {}
+
+  llvm::Expected<unsigned> GetBreakpointSize(lldb::addr_t bp_addr) override;
+};
+
 // ITSession - Keep track of the IT Block progression.
 class ITSession {
 public:
@@ -770,6 +780,14 @@ class EmulateInstructionARM : public EmulateInstruction {
   // B6.2.13 SUBS PC, LR and related instructions
   bool EmulateSUBSPcLrEtc(const uint32_t opcode, const ARMEncoding encoding);
 
+  BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() override {
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<ARMSingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
+
   uint32_t m_arm_isa;
   Mode m_opcode_mode;
   uint32_t m_opcode_cpsr;
diff --git a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
index 37f08592d62b..db32bf8a4dd5 100644
--- a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
+++ b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.cpp
@@ -86,7 +86,6 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
   uint32_t inst_size = m_opcode.GetByteSize();
   uint32_t inst = m_opcode.GetOpcode32();
   bool increase_pc = options & eEmulateInstructionOptionAutoAdvancePC;
-  bool success = false;
 
   Opcode *opcode_data = GetOpcodeForInstruction(inst);
   if (!opcode_data)
@@ -94,9 +93,10 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
 
   lldb::addr_t old_pc = 0;
   if (increase_pc) {
-    old_pc = ReadPC(&success);
-    if (!success)
+    auto addr = ReadPC();
+    if (!addr)
       return false;
+    old_pc = *addr;
   }
 
   // Call the Emulate... function.
@@ -104,9 +104,10 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
     return false;
 
   if (increase_pc) {
-    lldb::addr_t new_pc = ReadPC(&success);
-    if (!success)
+    auto addr = ReadPC();
+    if (!addr)
       return false;
+    lldb::addr_t new_pc = *addr;
 
     if (new_pc == old_pc && !WritePC(old_pc + inst_size))
       return false;
@@ -115,13 +116,14 @@ bool EmulateInstructionLoongArch::EvaluateInstruction(uint32_t options) {
 }
 
 bool EmulateInstructionLoongArch::ReadInstruction() {
-  bool success = false;
-  m_addr = ReadPC(&success);
-  if (!success) {
+  auto addr = ReadPC();
+  if (!addr) {
     m_addr = LLDB_INVALID_ADDRESS;
     return false;
   }
+  m_addr = *addr;
 
+  bool success = false;
   Context ctx;
   ctx.type = eContextReadOpcode;
   ctx.SetNoArgs();
@@ -131,19 +133,6 @@ bool EmulateInstructionLoongArch::ReadInstruction() {
   return true;
 }
 
-lldb::addr_t EmulateInstructionLoongArch::ReadPC(bool *success) {
-  return ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
-                              LLDB_INVALID_ADDRESS, success);
-}
-
-bool EmulateInstructionLoongArch::WritePC(lldb::addr_t pc) {
-  EmulateInstruction::Context ctx;
-  ctx.type = eContextAdvancePC;
-  ctx.SetNoArgs();
-  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
-                               LLDB_REGNUM_GENERIC_PC, pc);
-}
-
 std::optional<RegisterInfo>
 EmulateInstructionLoongArch::GetRegisterInfo(lldb::RegisterKind reg_kind,
                                              uint32_t reg_index) {
@@ -273,9 +262,12 @@ bool EmulateInstructionLoongArch::EmulateNonJMP(uint32_t inst) { return false; }
 bool EmulateInstructionLoongArch::EmulateBEQZ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -293,9 +285,12 @@ bool EmulateInstructionLoongArch::EmulateBEQZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBNEZ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -313,9 +308,12 @@ bool EmulateInstructionLoongArch::EmulateBNEZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBCEQZ64(uint32_t inst) {
   bool success = false;
   uint32_t cj = Bits32(inst, 7, 5) + fpr_fcc0_loongarch;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint8_t cj_val =
       (uint8_t)ReadRegisterUnsigned(eRegisterKindLLDB, cj, 0, &success);
@@ -335,9 +333,12 @@ bool EmulateInstructionLoongArch::EmulateBCEQZ64(uint32_t inst) {
 bool EmulateInstructionLoongArch::EmulateBCNEZ64(uint32_t inst) {
   bool success = false;
   uint32_t cj = Bits32(inst, 7, 5) + fpr_fcc0_loongarch;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs21 = Bits32(inst, 25, 10) + (Bits32(inst, 4, 0) << 16);
   uint8_t cj_val =
       (uint8_t)ReadRegisterUnsigned(eRegisterKindLLDB, cj, 0, &success);
@@ -358,9 +359,12 @@ bool EmulateInstructionLoongArch::EmulateJIRL64(uint32_t inst) {
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
   bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   EmulateInstruction::Context ctx;
   if (!WriteRegisterUnsigned(ctx, eRegisterKindLLDB, rd, pc + 4))
     return false;
@@ -374,10 +378,11 @@ bool EmulateInstructionLoongArch::EmulateJIRL64(uint32_t inst) {
 // b offs26
 // PC = PC + SignExtend({offs26, 2' b0}, GRLEN)
 bool EmulateInstructionLoongArch::EmulateB64(uint32_t inst) {
-  bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint32_t offs26 = Bits32(inst, 25, 10) + (Bits32(inst, 9, 0) << 16);
   uint64_t next_pc = pc + llvm::SignExtend64<28>(offs26 << 2);
   return WritePC(next_pc);
@@ -387,10 +392,11 @@ bool EmulateInstructionLoongArch::EmulateB64(uint32_t inst) {
 // GR[1] = PC + 4
 // PC = PC + SignExtend({offs26, 2'b0}, GRLEN)
 bool EmulateInstructionLoongArch::EmulateBL64(uint32_t inst) {
-  bool success = false;
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   EmulateInstruction::Context ctx;
   if (!WriteRegisterUnsigned(ctx, eRegisterKindLLDB, gpr_r1_loongarch, pc + 4))
     return false;
@@ -406,9 +412,12 @@ bool EmulateInstructionLoongArch::EmulateBEQ64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -429,9 +438,12 @@ bool EmulateInstructionLoongArch::EmulateBNE64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -452,9 +464,12 @@ bool EmulateInstructionLoongArch::EmulateBLT64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   int64_t rj_val =
       (int64_t)ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -477,9 +492,12 @@ bool EmulateInstructionLoongArch::EmulateBGE64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   int64_t rj_val =
       (int64_t)ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
@@ -502,9 +520,12 @@ bool EmulateInstructionLoongArch::EmulateBLTU64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
@@ -525,9 +546,12 @@ bool EmulateInstructionLoongArch::EmulateBGEU64(uint32_t inst) {
   bool success = false;
   uint32_t rj = Bits32(inst, 9, 5);
   uint32_t rd = Bits32(inst, 4, 0);
-  uint64_t pc = ReadPC(&success);
-  if (!success)
+
+  auto addr = ReadPC();
+  if (!addr)
     return false;
+  uint64_t pc = *addr;
+
   uint64_t rj_val = ReadRegisterUnsigned(eRegisterKindLLDB, rj, 0, &success);
   if (!success)
     return false;
diff --git a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
index 47fe454fcd88..2a8acba42f49 100644
--- a/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
+++ b/lldb/source/Plugins/Instruction/LoongArch/EmulateInstructionLoongArch.h
@@ -57,8 +57,6 @@ class EmulateInstructionLoongArch : public EmulateInstruction {
 
   std::optional<RegisterInfo> GetRegisterInfo(lldb::RegisterKind reg_kind,
                                               uint32_t reg_num) override;
-  lldb::addr_t ReadPC(bool *success);
-  bool WritePC(lldb::addr_t pc);
   bool IsLoongArch64() { return m_arch_subtype == llvm::Triple::loongarch64; }
   bool TestExecute(uint32_t inst);
 
diff --git a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
index badc7ba36f01..2adde02aca3a 100644
--- a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
+++ b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.cpp
@@ -650,9 +650,10 @@ std::optional<DecodeResult> EmulateInstructionRISCV::Decode(uint32_t inst) {
   for (const InstrPattern &pat : PATTERNS) {
     if ((inst & pat.type_mask) == pat.eigen &&
         (inst_type & pat.inst_type) != 0) {
-      LLDB_LOGF(
-          log, "EmulateInstructionRISCV::%s: inst(%x at %" PRIx64 ") was decoded to %s",
-          __FUNCTION__, inst, m_addr, pat.name);
+      LLDB_LOGF(log,
+                "EmulateInstructionRISCV::%s: inst(%x at %" PRIx64
+                ") was decoded to %s",
+                __FUNCTION__, inst, m_addr, pat.name);
       auto decoded = is_16b ? pat.decode(try_rvc) : pat.decode(inst);
       return DecodeResult{decoded, inst, is_16b, pat};
     }
@@ -1649,21 +1650,6 @@ bool EmulateInstructionRISCV::ReadInstruction() {
   return true;
 }
 
-std::optional<addr_t> EmulateInstructionRISCV::ReadPC() {
-  bool success = false;
-  auto addr = ReadRegisterUnsigned(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC,
-                                   LLDB_INVALID_ADDRESS, &success);
-  return success ? std::optional<addr_t>(addr) : std::nullopt;
-}
-
-bool EmulateInstructionRISCV::WritePC(addr_t pc) {
-  EmulateInstruction::Context ctx;
-  ctx.type = eContextAdvancePC;
-  ctx.SetNoArgs();
-  return WriteRegisterUnsigned(ctx, eRegisterKindGeneric,
-                               LLDB_REGNUM_GENERIC_PC, pc);
-}
-
 RoundingMode EmulateInstructionRISCV::GetRoundingMode() {
   bool success = false;
   auto fcsr = ReadRegisterUnsigned(eRegisterKindLLDB, fpr_fcsr_riscv,
@@ -1792,4 +1778,125 @@ bool EmulateInstructionRISCV::SupportsThisArch(const ArchSpec &arch) {
   return arch.GetTriple().isRISCV();
 }
 
+BreakpointLocations
+RISCVSingleStepBreakpointLocationsPredictor::GetBreakpointLocations(
+    Status &status) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  auto pc = riscv_emulator->ReadPC();
+  if (!pc) {
+    status = Status("Can't read PC");
+    return {};
+  }
+
+  auto inst = riscv_emulator->ReadInstructionAt(*pc);
+  if (!inst) {
+    // Can't read instruction. Try default handler.
+    return SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(
+        status);
+  }
+
+  if (FoundLoadReserve(inst->decoded))
+    return HandleAtomicSequence(*pc, status);
+
+  if (FoundStoreConditional(inst->decoded)) {
+    // Ill-formed atomic sequence (SC doesn't have corresponding LR
+    // instruction). Consider SC instruction like a non-atomic store and set a
+    // breakpoint at the next instruction.
+    Log *log = GetLog(LLDBLog::Unwind);
+    LLDB_LOGF(log,
+              "RISCVSingleStepBreakpointLocationsPredictor::%s: can't find "
+              "corresponding load reserve insturuction",
+              __FUNCTION__);
+    return {*pc + (inst->is_rvc ? 2u : 4u)};
+  }
+
+  return SingleStepBreakpointLocationsPredictor::GetBreakpointLocations(status);
+}
+
+llvm::Expected<unsigned>
+RISCVSingleStepBreakpointLocationsPredictor::GetBreakpointSize(
+    lldb::addr_t bp_addr) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  if (auto inst = riscv_emulator->ReadInstructionAt(bp_addr); inst)
+    return inst->is_rvc ? 2 : 4;
+
+  // Try last instruction size.
+  if (auto last_instr_size = riscv_emulator->GetLastInstrSize();
+      last_instr_size)
+    return *last_instr_size;
+
+  // Just place non-compressed software trap.
+  return 4;
+}
+
+BreakpointLocations
+RISCVSingleStepBreakpointLocationsPredictor::HandleAtomicSequence(
+    lldb::addr_t pc, Status &error) {
+  EmulateInstructionRISCV *riscv_emulator =
+      static_cast<EmulateInstructionRISCV *>(m_emulator_up.get());
+
+  // Handle instructions between LR and SC. According to unprivilleged
+  // RISC-V ISA there can be at most 16 instructions in the sequence.
+
+  lldb::addr_t entry_pc = pc; // LR instruction address
+  auto lr_inst = riscv_emulator->ReadInstructionAt(entry_pc);
+  pc += lr_inst->is_rvc ? 2 : 4;
+
+  size_t atomic_length = 0;
+  std::optional<DecodeResult> inst;
+  std::vector<lldb::addr_t> bp_addrs;
+  do {
+    inst = riscv_emulator->ReadInstructionAt(pc);
+    if (!inst) {
+      error = Status("Can't read instruction");
+      return {};
+    }
+
+    if (B *branch = std::get_if<B>(&inst->decoded))
+      bp_addrs.push_back(pc + SignExt(branch->imm));
+
+    unsigned addent = inst->is_rvc ? 2 : 4;
+    pc += addent;
+    atomic_length += addent;
+  } while ((atomic_length < s_max_atomic_sequence_length) &&
+           !FoundStoreConditional(inst->decoded));
+
+  if (atomic_length >= s_max_atomic_sequence_length) {
+    // Ill-formed atomic sequence (LR doesn't have corresponding SC
+    // instruction). In this case consider LR like a non-atomic load instruction
+    // and set a breakpoint at the next after LR instruction.
+    Log *log = GetLog(LLDBLog::Unwind);
+    LLDB_LOGF(log,
+              "RISCVSingleStepBreakpointLocationsPredictor::%s: can't find "
+              "corresponding store conditional insturuction",
+              __FUNCTION__);
+    return {entry_pc + (lr_inst->is_rvc ? 2u : 4u)};
+  }
+
+  lldb::addr_t exit_pc = pc;
+
+  // Check if we have a branch to the start of the atomic sequence after SC
+  // instruction. If we have such branch, consider it as a part of the atomic
+  // sequence.
+  inst = riscv_emulator->ReadInstructionAt(exit_pc);
+  if (inst) {
+    B *branch = std::get_if<B>(&inst->decoded);
+    if (branch && (exit_pc + SignExt(branch->imm)) == entry_pc)
+      exit_pc += inst->is_rvc ? 2 : 4;
+  }
+
+  // Set breakpoints at the jump addresses of the forward branches that points
+  // after the end of the atomic sequence.
+  llvm::erase_if(
+      bp_addrs, [exit_pc](lldb::addr_t bp_addr) { return exit_pc >= bp_addr; });
+
+  // Set breakpoint at the end of atomic sequence.
+  bp_addrs.push_back(exit_pc);
+  return bp_addrs;
+}
+
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
index 53ac11c2e110..3578a4ab0305 100644
--- a/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
+++ b/lldb/source/Plugins/Instruction/RISCV/EmulateInstructionRISCV.h
@@ -20,6 +20,33 @@
 
 namespace lldb_private {
 
+class RISCVSingleStepBreakpointLocationsPredictor
+    : public SingleStepBreakpointLocationsPredictor {
+public:
+  RISCVSingleStepBreakpointLocationsPredictor(
+      std::unique_ptr<EmulateInstruction> emulator)
+      : SingleStepBreakpointLocationsPredictor{std::move(emulator)} {}
+
+  BreakpointLocations GetBreakpointLocations(Status &status) override;
+
+  llvm::Expected<unsigned> GetBreakpointSize(lldb::addr_t bp_addr) override;
+
+private:
+  static bool FoundLoadReserve(const RISCVInst &inst) {
+    return std::holds_alternative<LR_W>(inst) ||
+           std::holds_alternative<LR_D>(inst);
+  }
+
+  static bool FoundStoreConditional(const RISCVInst &inst) {
+    return std::holds_alternative<SC_W>(inst) ||
+           std::holds_alternative<SC_D>(inst);
+  }
+
+  BreakpointLocations HandleAtomicSequence(lldb::addr_t pc, Status &error);
+
+  static constexpr size_t s_max_atomic_sequence_length = 64;
+};
+
 class EmulateInstructionRISCV : public EmulateInstruction {
 public:
   static llvm::StringRef GetPluginNameStatic() { return "riscv"; }
@@ -67,9 +94,6 @@ class EmulateInstructionRISCV : public EmulateInstruction {
   std::optional<RegisterInfo> GetRegisterInfo(lldb::RegisterKind reg_kind,
                                               uint32_t reg_num) override;
 
-  std::optional<lldb::addr_t> ReadPC();
-  bool WritePC(lldb::addr_t pc);
-
   std::optional<DecodeResult> ReadInstructionAt(lldb::addr_t addr);
   std::optional<DecodeResult> Decode(uint32_t inst);
   bool Execute(DecodeResult inst, bool ignore_cond);
@@ -98,6 +122,13 @@ class EmulateInstructionRISCV : public EmulateInstruction {
   bool SetAccruedExceptions(llvm::APFloatBase::opStatus);
 
 private:
+  BreakpointLocationsPredictorCreator
+  GetSingleStepBreakpointLocationsPredictorCreator() override {
+    return [](std::unique_ptr<EmulateInstruction> emulator_up) {
+      return std::make_unique<RISCVSingleStepBreakpointLocationsPredictor>(
+          std::move(emulator_up));
+    };
+  }
   /// Last decoded instruction from m_opcode
   DecodeResult m_decoded;
   /// Last decoded instruction size estimate.
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 7ae2e141a63e..06412e34eb77 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -376,9 +376,9 @@ static bool PrintDemangledArgumentList(Stream &s, const SymbolContext &sc) {
 
   auto info_or_err = GetAndValidateInfo(sc);
   if (!info_or_err) {
-    LLDB_LOG_ERROR(
-        GetLog(LLDBLog::Language), info_or_err.takeError(),
-        "Failed to handle ${{function.basename}} frame-format variable: {0}");
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Language), info_or_err.takeError(),
+                   "Failed to handle ${{function.formatted-arguments}} "
+                   "frame-format variable: {0}");
     return false;
   }
   auto [demangled_name, info] = *info_or_err;
@@ -1016,15 +1016,15 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(false);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::bitset summary provider",
                 "^std::__[[:alnum:]]+::bitset<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::vector summary provider",
                 "^std::__[[:alnum:]]+::vector<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::valarray summary provider",
                 "^std::__[[:alnum:]]+::valarray<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
@@ -1033,16 +1033,16 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "^std::__[[:alnum:]]+::slice_array<.+>$", stl_summary_flags,
                 true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ summary provider for the valarray proxy arrays",
                 "^std::__[[:alnum:]]+::(gslice|mask|indirect)_array<.+>$",
                 stl_summary_flags, true);
   AddCXXSummary(
-      cpp_category_sp, lldb_private::formatters::LibcxxContainerSummaryProvider,
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
       "libc++ std::list summary provider",
       "^std::__[[:alnum:]]+::forward_list<.+>$", stl_summary_flags, true);
   AddCXXSummary(
-      cpp_category_sp, lldb_private::formatters::LibcxxContainerSummaryProvider,
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
       "libc++ std::list summary provider",
       // A POSIX variant of: "^std::__(?!cxx11:)[[:alnum:]]+::list<.+>$"
       // so that it does not clash with: "^std::(__cxx11::)?list<.+>$"
@@ -1050,35 +1050,35 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       "cxx11[[:alnum:]])[[:alnum:]]*::list<.+>$",
       stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::map summary provider",
                 "^std::__[[:alnum:]]+::map<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::deque summary provider",
                 "^std::__[[:alnum:]]+::deque<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::queue summary provider",
                 "^std::__[[:alnum:]]+::queue<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::set summary provider",
                 "^std::__[[:alnum:]]+::set<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::multiset summary provider",
                 "^std::__[[:alnum:]]+::multiset<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::multimap summary provider",
                 "^std::__[[:alnum:]]+::multimap<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::unordered containers summary provider",
                 "^std::__[[:alnum:]]+::unordered_(multi)?(map|set)<.+> >$",
                 stl_summary_flags, true);
-  AddCXXSummary(cpp_category_sp, LibcxxContainerSummaryProvider,
+  AddCXXSummary(cpp_category_sp, ContainerSizeSummaryProvider,
                 "libc++ std::tuple summary provider",
                 "^std::__[[:alnum:]]+::tuple<.*>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
@@ -1094,7 +1094,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "libc++ std::variant summary provider",
                 "^std::__[[:alnum:]]+::variant<.+>$", stl_summary_flags, true);
   AddCXXSummary(cpp_category_sp,
-                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
                 "libc++ std::span summary provider",
                 "^std::__[[:alnum:]]+::span<.+>$", stl_summary_flags, true);
 
@@ -1344,38 +1344,31 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       .SetShowMembersOneLiner(false)
       .SetHideItemNames(false);
 
-  lldb::TypeSummaryImplSP std_string_summary_sp(
-      new StringSummaryFormat(stl_summary_flags, "${var._M_dataplus._M_p}"));
-
-  lldb::TypeSummaryImplSP cxx11_string_summary_sp(new CXXFunctionSummaryFormat(
+  lldb::TypeSummaryImplSP string_summary_sp(new CXXFunctionSummaryFormat(
       stl_summary_flags, LibStdcppStringSummaryProvider,
-      "libstdc++ c++11 std::string summary provider"));
-  lldb::TypeSummaryImplSP cxx11_wstring_summary_sp(new CXXFunctionSummaryFormat(
-      stl_summary_flags, LibStdcppWStringSummaryProvider,
-      "libstdc++ c++11 std::wstring summary provider"));
+      "libstdc++ std::(w)string summary provider"));
 
   cpp_category_sp->AddTypeSummary("std::string", eFormatterMatchExact,
-                                  std_string_summary_sp);
+                                  string_summary_sp);
   cpp_category_sp->AddTypeSummary("std::basic_string<char>",
-                                  eFormatterMatchExact, std_string_summary_sp);
+                                  eFormatterMatchExact, string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::basic_string<char,std::char_traits<char>,std::allocator<char> >",
-      eFormatterMatchExact, std_string_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::basic_string<char, std::char_traits<char>, std::allocator<char> >",
-      eFormatterMatchExact, std_string_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
 
   cpp_category_sp->AddTypeSummary("std::__cxx11::string", eFormatterMatchExact,
-                                  cxx11_string_summary_sp);
+                                  string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::__cxx11::basic_string<char, std::char_traits<char>, "
       "std::allocator<char> >",
-      eFormatterMatchExact, cxx11_string_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
   cpp_category_sp->AddTypeSummary("std::__cxx11::basic_string<unsigned char, "
                                   "std::char_traits<unsigned char>, "
                                   "std::allocator<unsigned char> >",
-                                  eFormatterMatchExact,
-                                  cxx11_string_summary_sp);
+                                  eFormatterMatchExact, string_summary_sp);
 
   // making sure we force-pick the summary for printing wstring (_M_p is a
   // wchar_t*)
@@ -1395,11 +1388,11 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       eFormatterMatchExact, std_wstring_summary_sp);
 
   cpp_category_sp->AddTypeSummary("std::__cxx11::wstring", eFormatterMatchExact,
-                                  cxx11_wstring_summary_sp);
+                                  string_summary_sp);
   cpp_category_sp->AddTypeSummary(
       "std::__cxx11::basic_string<wchar_t, std::char_traits<wchar_t>, "
       "std::allocator<wchar_t> >",
-      eFormatterMatchExact, cxx11_wstring_summary_sp);
+      eFormatterMatchExact, string_summary_sp);
 
   SyntheticChildren::Flags stl_synth_flags;
   stl_synth_flags.SetCascades(true).SetSkipPointers(false).SetSkipReferences(
@@ -1463,44 +1456,54 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
 
   stl_summary_flags.SetDontShowChildren(false);
   stl_summary_flags.SetSkipPointers(false);
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?bitset<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?vector<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?map<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?set<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?deque<.+>(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?multimap<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary("^std::(__debug::)?multiset<.+> >(( )?&)?$",
-                                  eFormatterMatchRegex,
-                                  TypeSummaryImplSP(new StringSummaryFormat(
-                                      stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary(
-      "^std::(__debug::)?unordered_(multi)?(map|set)<.+> >$",
-      eFormatterMatchRegex,
-      TypeSummaryImplSP(
-          new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
-  cpp_category_sp->AddTypeSummary(
-      "^std::((__debug::)?|(__cxx11::)?)list<.+>(( )?&)?$",
-      eFormatterMatchRegex,
-      TypeSummaryImplSP(
-          new StringSummaryFormat(stl_summary_flags, "size=${svar%#}")));
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::bitset summary provider",
+      "^std::(__debug::)?bitset<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::vector summary provider",
+      "^std::(__debug::)?vector<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::map summary provider",
+      "^std::(__debug::)?map<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::set summary provider",
+      "^std::(__debug::)?set<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::deque summary provider",
+      "^std::(__debug::)?deque<.+>(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::multimap summary provider",
+      "^std::(__debug::)?multimap<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(
+      cpp_category_sp, lldb_private::formatters::ContainerSizeSummaryProvider,
+      "libstdc++ std::multiset summary provider",
+      "^std::(__debug::)?multiset<.+> >(( )?&)?$", stl_summary_flags, true);
+
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
+                "libstdc++ std unordered container summary provider",
+                "^std::(__debug::)?unordered_(multi)?(map|set)<.+> >$",
+                stl_summary_flags, true);
+
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::ContainerSizeSummaryProvider,
+                "libstdc++ std::list summary provider",
+                "^std::((__debug::)?|(__cxx11::)?)list<.+>(( )?&)?$",
+                stl_summary_flags, true);
+
   cpp_category_sp->AddTypeSummary(
       "^std::((__debug::)?|(__cxx11::)?)forward_list<.+>(( )?&)?$",
       eFormatterMatchRegex,
@@ -1522,8 +1525,8 @@ static void LoadLibStdcppFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
   AddCXXSynthetic(
       cpp_category_sp,
       lldb_private::formatters::LibstdcppMapIteratorSyntheticFrontEndCreator,
-      "std::map iterator synthetic children", "^std::_Rb_tree_iterator<.+>$",
-      stl_synth_flags, true);
+      "std::map iterator synthetic children",
+      "^std::_Rb_tree_(const_)?iterator<.+>$", stl_synth_flags, true);
 
   AddCXXSynthetic(
       cpp_category_sp,
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 7143089209dd..3079de4936a8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -157,39 +157,43 @@ bool lldb_private::formatters::LibcxxSmartPointerSummaryProvider(
   ValueObjectSP valobj_sp(valobj.GetNonSyntheticValue());
   if (!valobj_sp)
     return false;
-  ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_"));
-  ValueObjectSP count_sp(
-      valobj_sp->GetChildAtNamePath({"__cntrl_", "__shared_owners_"}));
-  ValueObjectSP weakcount_sp(
-      valobj_sp->GetChildAtNamePath({"__cntrl_", "__shared_weak_owners_"}));
 
-  if (!ptr_sp)
+  ValueObjectSP ptr_sp(valobj_sp->GetChildMemberWithName("__ptr_"));
+  ValueObjectSP ctrl_sp(valobj_sp->GetChildMemberWithName("__cntrl_"));
+  if (!ctrl_sp || !ptr_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
+
+  bool success;
+  uint64_t ctrl_addr = ctrl_sp->GetValueAsUnsigned(0, &success);
+  // Empty control field. We're done.
+  if (!success || ctrl_addr == 0)
     return true;
-  } else {
-    bool print_pointee = false;
-    Status error;
-    ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-    if (pointee_sp && error.Success()) {
-      if (pointee_sp->DumpPrintableRepresentation(
-              stream, ValueObject::eValueObjectRepresentationStyleSummary,
-              lldb::eFormatInvalid,
-              ValueObject::PrintableRepresentationSpecialCases::eDisable,
-              false))
-        print_pointee = true;
-    }
-    if (!print_pointee)
-      stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
+
+  if (auto count_sp = ctrl_sp->GetChildMemberWithName("__shared_owners_")) {
+    bool success;
+    uint64_t count = count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
+
+    // std::shared_ptr releases the underlying resource when the
+    // __shared_owners_ count hits -1. So `__shared_owners_ == 0` indicates 1
+    // owner. Hence add +1 here.
+    stream.Printf(" strong=%" PRIu64, count + 1);
   }
 
-  if (count_sp)
-    stream.Printf(" strong=%" PRIu64, 1 + count_sp->GetValueAsUnsigned(0));
+  if (auto weak_count_sp =
+          ctrl_sp->GetChildMemberWithName("__shared_weak_owners_")) {
+    bool success;
+    uint64_t count = weak_count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
 
-  if (weakcount_sp)
-    stream.Printf(" weak=%" PRIu64, 1 + weakcount_sp->GetValueAsUnsigned(0));
+    // Unlike __shared_owners_, __shared_weak_owners_ indicates the exact
+    // std::weak_ptr reference count.
+    stream.Printf(" weak=%" PRIu64, count);
+  }
 
   return true;
 }
@@ -210,24 +214,7 @@ bool lldb_private::formatters::LibcxxUniquePointerSummaryProvider(
   if (!ptr_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
-    return true;
-  } else {
-    bool print_pointee = false;
-    Status error;
-    ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-    if (pointee_sp && error.Success()) {
-      if (pointee_sp->DumpPrintableRepresentation(
-              stream, ValueObject::eValueObjectRepresentationStyleSummary,
-              lldb::eFormatInvalid,
-              ValueObject::PrintableRepresentationSpecialCases::eDisable,
-              false))
-        print_pointee = true;
-    }
-    if (!print_pointee)
-      stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
-  }
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
 
   return true;
 }
@@ -251,7 +238,8 @@ lldb_private::formatters::LibCxxVectorIteratorSyntheticFrontEndCreator(
 
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     LibcxxSharedPtrSyntheticFrontEnd(lldb::ValueObjectSP valobj_sp)
-    : SyntheticChildrenFrontEnd(*valobj_sp), m_cntrl(nullptr) {
+    : SyntheticChildrenFrontEnd(*valobj_sp), m_cntrl(nullptr),
+      m_ptr_obj(nullptr) {
   if (valobj_sp)
     Update();
 }
@@ -264,7 +252,7 @@ llvm::Expected<uint32_t> lldb_private::formatters::
 lldb::ValueObjectSP
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
     uint32_t idx) {
-  if (!m_cntrl)
+  if (!m_cntrl || !m_ptr_obj)
     return lldb::ValueObjectSP();
 
   ValueObjectSP valobj_sp = m_backend.GetSP();
@@ -272,20 +260,17 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
     return lldb::ValueObjectSP();
 
   if (idx == 0)
-    return valobj_sp->GetChildMemberWithName("__ptr_");
+    return m_ptr_obj->GetSP();
 
   if (idx == 1) {
-    if (auto ptr_sp = valobj_sp->GetChildMemberWithName("__ptr_")) {
-      Status status;
-      auto value_type_sp =
-            valobj_sp->GetCompilerType()
-              .GetTypeTemplateArgument(0).GetPointerType();
-      ValueObjectSP cast_ptr_sp = ptr_sp->Cast(value_type_sp);
-      ValueObjectSP value_sp = cast_ptr_sp->Dereference(status);
-      if (status.Success()) {
-        return value_sp;
-      }
-    }
+    Status status;
+    auto value_type_sp = valobj_sp->GetCompilerType()
+                             .GetTypeTemplateArgument(0)
+                             .GetPointerType();
+    ValueObjectSP cast_ptr_sp = m_ptr_obj->Cast(value_type_sp);
+    ValueObjectSP value_sp = cast_ptr_sp->Dereference(status);
+    if (status.Success())
+      return value_sp;
   }
 
   return lldb::ValueObjectSP();
@@ -294,6 +279,7 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::GetChildAtIndex(
 lldb::ChildCacheState
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
   m_cntrl = nullptr;
+  m_ptr_obj = nullptr;
 
   ValueObjectSP valobj_sp = m_backend.GetSP();
   if (!valobj_sp)
@@ -303,6 +289,12 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
   if (!target_sp)
     return lldb::ChildCacheState::eRefetch;
 
+  auto ptr_obj_sp = valobj_sp->GetChildMemberWithName("__ptr_");
+  if (!ptr_obj_sp)
+    return lldb::ChildCacheState::eRefetch;
+
+  m_ptr_obj = ptr_obj_sp->Clone(ConstString("pointer")).get();
+
   lldb::ValueObjectSP cntrl_sp(valobj_sp->GetChildMemberWithName("__cntrl_"));
 
   m_cntrl = cntrl_sp.get(); // need to store the raw pointer to avoid a circular
@@ -313,10 +305,12 @@ lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::Update() {
 llvm::Expected<size_t>
 lldb_private::formatters::LibcxxSharedPtrSyntheticFrontEnd::
     GetIndexOfChildWithName(ConstString name) {
-  if (name == "__ptr_")
+  if (name == "__ptr_" || name == "pointer")
     return 0;
-  if (name == "$$dereference$$")
+
+  if (name == "object" || name == "$$dereference$$")
     return 1;
+
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
@@ -417,18 +411,12 @@ lldb_private::formatters::LibcxxUniquePtrSyntheticFrontEnd::
     return 0;
   if (name == "deleter")
     return 1;
-  if (name == "$$dereference$$")
+  if (name == "obj" || name == "object" || name == "$$dereference$$")
     return 2;
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
 
-bool lldb_private::formatters::LibcxxContainerSummaryProvider(
-    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  return FormatEntity::FormatStringRef("size=${svar%#}", stream, nullptr,
-                                       nullptr, nullptr, &valobj, false, false);
-}
-
 /// The field layout in a libc++ string (cap, side, data or data, size, cap).
 namespace {
 enum class StringLayout { CSD, DSC };
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index 56501302d116..d88a6ecb1fa8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -81,9 +81,6 @@ SyntheticChildrenFrontEnd *
 LibcxxVectorBoolSyntheticFrontEndCreator(CXXSyntheticChildren *,
                                          lldb::ValueObjectSP);
 
-bool LibcxxContainerSummaryProvider(ValueObject &valobj, Stream &stream,
-                                    const TypeSummaryOptions &options);
-
 /// Formatter for libc++ std::span<>.
 bool LibcxxSpanSummaryProvider(ValueObject &valobj, Stream &stream,
                                const TypeSummaryOptions &options);
@@ -108,6 +105,7 @@ class LibcxxSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
 
 private:
   ValueObject *m_cntrl;
+  ValueObject *m_ptr_obj;
 };
 
 class LibcxxUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
index ffc33395830b..501fd0945b82 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxUnorderedMap.cpp
@@ -113,8 +113,10 @@ CompilerType lldb_private::formatters::LibcxxStdUnorderedMapSyntheticFrontEnd::
   // wraps a std::pair. Peel away the internal wrapper type - whose structure is
   // of no value to users, to expose the std::pair. This matches the structure
   // returned by the std::map synthetic provider.
-  if (isUnorderedMap(
-          m_backend.GetCompilerType().GetCanonicalType().GetTypeName())) {
+  if (isUnorderedMap(m_backend.GetCompilerType()
+                         .GetNonReferenceType()
+                         .GetCanonicalType()
+                         .GetTypeName())) {
     std::string name;
     CompilerType field_type =
         element_type.GetFieldAtIndex(0, name, nullptr, nullptr, nullptr);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 77e0755607a0..84442273aebd 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -10,6 +10,7 @@
 #include "LibCxx.h"
 
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
+#include "lldb/DataFormatters/FormattersHelpers.h"
 #include "lldb/DataFormatters/StringPrinter.h"
 #include "lldb/DataFormatters/VectorIterator.h"
 #include "lldb/Target/Target.h"
@@ -77,8 +78,7 @@ class LibStdcppSharedPtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
   // objects are only destroyed when every shared pointer to any of them
   // is destroyed, so we must not store a shared pointer to any ValueObject
   // derived from our backend ValueObject (since we're in the same cluster).
-  ValueObject* m_ptr_obj = nullptr; // Underlying pointer (held, not owned)
-  ValueObject* m_obj_obj = nullptr; // Underlying object (held, not owned)
+  ValueObject *m_ptr_obj = nullptr; // Underlying pointer (held, not owned)
 };
 
 } // end of anonymous namespace
@@ -239,122 +239,12 @@ VectorIteratorSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
 
 bool lldb_private::formatters::LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  const bool scalar_is_load_addr = true;
-  auto [addr_of_string, addr_type] =
-      valobj.IsPointerOrReferenceType()
-          ? valobj.GetPointerValue()
-          : valobj.GetAddressOf(scalar_is_load_addr);
-  if (addr_of_string != LLDB_INVALID_ADDRESS) {
-    switch (addr_type) {
-    case eAddressTypeLoad: {
-      ProcessSP process_sp(valobj.GetProcessSP());
-      if (!process_sp)
-        return false;
-
-      StringPrinter::ReadStringAndDumpToStreamOptions options(valobj);
-      Status error;
-      lldb::addr_t addr_of_data =
-          process_sp->ReadPointerFromMemory(addr_of_string, error);
-      if (error.Fail() || addr_of_data == 0 ||
-          addr_of_data == LLDB_INVALID_ADDRESS)
-        return false;
-      options.SetLocation(addr_of_data);
-      options.SetTargetSP(valobj.GetTargetSP());
-      options.SetStream(&stream);
-      options.SetNeedsZeroTermination(false);
-      options.SetBinaryZeroIsTerminator(true);
-      lldb::addr_t size_of_data = process_sp->ReadPointerFromMemory(
-          addr_of_string + process_sp->GetAddressByteSize(), error);
-      if (error.Fail())
-        return false;
-      options.SetSourceSize(size_of_data);
-      options.SetHasSourceSize(true);
-
-      if (!StringPrinter::ReadStringAndDumpToStream<
-              StringPrinter::StringElementType::UTF8>(options)) {
-        stream.Printf("Summary Unavailable");
-        return true;
-      } else
-        return true;
-    } break;
-    case eAddressTypeHost:
-      break;
-    case eAddressTypeInvalid:
-    case eAddressTypeFile:
-      break;
-    }
-  }
-  return false;
-}
+  ValueObjectSP ptr = valobj.GetChildAtNamePath({"_M_dataplus", "_M_p"});
+  if (!ptr)
+    return false;
 
-bool lldb_private::formatters::LibStdcppWStringSummaryProvider(
-    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
-  const bool scalar_is_load_addr = true;
-  auto [addr_of_string, addr_type] = valobj.GetAddressOf(scalar_is_load_addr);
-  if (addr_of_string != LLDB_INVALID_ADDRESS) {
-    switch (addr_type) {
-    case eAddressTypeLoad: {
-      ProcessSP process_sp(valobj.GetProcessSP());
-      if (!process_sp)
-        return false;
-
-      CompilerType wchar_compiler_type =
-          valobj.GetCompilerType().GetBasicTypeFromAST(lldb::eBasicTypeWChar);
-
-      if (!wchar_compiler_type)
-        return false;
-
-      // Safe to pass nullptr for exe_scope here.
-      std::optional<uint64_t> size =
-          llvm::expectedToOptional(wchar_compiler_type.GetBitSize(nullptr));
-      if (!size)
-        return false;
-      const uint32_t wchar_size = *size;
-
-      StringPrinter::ReadStringAndDumpToStreamOptions options(valobj);
-      Status error;
-      lldb::addr_t addr_of_data =
-          process_sp->ReadPointerFromMemory(addr_of_string, error);
-      if (error.Fail() || addr_of_data == 0 ||
-          addr_of_data == LLDB_INVALID_ADDRESS)
-        return false;
-      options.SetLocation(addr_of_data);
-      options.SetTargetSP(valobj.GetTargetSP());
-      options.SetStream(&stream);
-      options.SetNeedsZeroTermination(false);
-      options.SetBinaryZeroIsTerminator(false);
-      lldb::addr_t size_of_data = process_sp->ReadPointerFromMemory(
-          addr_of_string + process_sp->GetAddressByteSize(), error);
-      if (error.Fail())
-        return false;
-      options.SetSourceSize(size_of_data);
-      options.SetHasSourceSize(true);
-      options.SetPrefixToken("L");
-
-      switch (wchar_size) {
-      case 8:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF8>(options);
-      case 16:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF16>(options);
-      case 32:
-        return StringPrinter::ReadStringAndDumpToStream<
-            StringPrinter::StringElementType::UTF32>(options);
-      default:
-        stream.Printf("size for wchar_t is not valid");
-        return true;
-      }
-      return true;
-    } break;
-    case eAddressTypeHost:
-      break;
-    case eAddressTypeInvalid:
-    case eAddressTypeFile:
-      break;
-    }
-  }
-  return false;
+  stream << ptr->GetSummaryAsCString();
+  return true;
 }
 
 LibStdcppSharedPtrSyntheticFrontEnd::LibStdcppSharedPtrSyntheticFrontEnd(
@@ -376,15 +266,20 @@ LibStdcppSharedPtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
 
   if (idx == 0)
     return m_ptr_obj->GetSP();
+
   if (idx == 1) {
-    if (m_ptr_obj && !m_obj_obj) {
-      Status error;
-      ValueObjectSP obj_obj = m_ptr_obj->Dereference(error);
-      if (error.Success())
-        m_obj_obj = obj_obj->Clone(ConstString("object")).get();
-    }
-    if (m_obj_obj)
-      return m_obj_obj->GetSP();
+    ValueObjectSP valobj_sp = m_backend.GetSP();
+    if (!valobj_sp)
+      return nullptr;
+
+    Status status;
+    auto value_type_sp = valobj_sp->GetCompilerType()
+                             .GetTypeTemplateArgument(0)
+                             .GetPointerType();
+    ValueObjectSP cast_ptr_sp = m_ptr_obj->Cast(value_type_sp);
+    ValueObjectSP value_sp = cast_ptr_sp->Dereference(status);
+    if (status.Success())
+      return value_sp;
   }
   return lldb::ValueObjectSP();
 }
@@ -403,7 +298,6 @@ lldb::ChildCacheState LibStdcppSharedPtrSyntheticFrontEnd::Update() {
     return lldb::ChildCacheState::eRefetch;
 
   m_ptr_obj = ptr_obj_sp->Clone(ConstString("pointer")).get();
-  m_obj_obj = nullptr;
 
   return lldb::ChildCacheState::eRefetch;
 }
@@ -412,8 +306,10 @@ llvm::Expected<size_t>
 LibStdcppSharedPtrSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
   if (name == "pointer")
     return 0;
+
   if (name == "object" || name == "$$dereference$$")
     return 1;
+
   return llvm::createStringError("Type has no child named '%s'",
                                  name.AsCString());
 }
@@ -435,29 +331,37 @@ bool lldb_private::formatters::LibStdcppSmartPointerSummaryProvider(
   if (!ptr_sp)
     return false;
 
-  ValueObjectSP usecount_sp(
-      valobj_sp->GetChildAtNamePath({"_M_refcount", "_M_pi", "_M_use_count"}));
-  if (!usecount_sp)
+  DumpCxxSmartPtrPointerSummary(stream, *ptr_sp, options);
+
+  ValueObjectSP pi_sp = valobj_sp->GetChildAtNamePath({"_M_refcount", "_M_pi"});
+  if (!pi_sp)
     return false;
 
-  if (ptr_sp->GetValueAsUnsigned(0) == 0 ||
-      usecount_sp->GetValueAsUnsigned(0) == 0) {
-    stream.Printf("nullptr");
+  bool success;
+  uint64_t pi_addr = pi_sp->GetValueAsUnsigned(0, &success);
+  // Empty control field. We're done.
+  if (!success || pi_addr == 0)
     return true;
+
+  int64_t shared_count = 0;
+  if (auto count_sp = pi_sp->GetChildMemberWithName("_M_use_count")) {
+    bool success;
+    shared_count = count_sp->GetValueAsSigned(0, &success);
+    if (!success)
+      return false;
+
+    stream.Printf(" strong=%" PRId64, shared_count);
   }
 
-  Status error;
-  ValueObjectSP pointee_sp = ptr_sp->Dereference(error);
-  if (pointee_sp && error.Success()) {
-    if (pointee_sp->DumpPrintableRepresentation(
-            stream, ValueObject::eValueObjectRepresentationStyleSummary,
-            lldb::eFormatInvalid,
-            ValueObject::PrintableRepresentationSpecialCases::eDisable,
-            false)) {
-      return true;
-    }
+  // _M_weak_count is the number of weak references + (_M_use_count != 0).
+  if (auto weak_count_sp = pi_sp->GetChildMemberWithName("_M_weak_count")) {
+    bool success;
+    int64_t count = weak_count_sp->GetValueAsUnsigned(0, &success);
+    if (!success)
+      return false;
+
+    stream.Printf(" weak=%" PRId64, count - (shared_count != 0));
   }
 
-  stream.Printf("ptr = 0x%" PRIx64, ptr_sp->GetValueAsUnsigned(0));
   return true;
 }
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
index c7b2130275ad..8d4d777edee8 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.h
@@ -18,11 +18,7 @@ namespace lldb_private {
 namespace formatters {
 bool LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream,
-    const TypeSummaryOptions &options); // libcstdc++ c++11 std::string
-
-bool LibStdcppWStringSummaryProvider(
-    ValueObject &valobj, Stream &stream,
-    const TypeSummaryOptions &options); // libcstdc++ c++11 std::wstring
+    const TypeSummaryOptions &options); // libstdc++ std::string
 
 bool LibStdcppSmartPointerSummaryProvider(
     ValueObject &valobj, Stream &stream,
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
index b47ff579b6a5..4daba2e36416 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcppUniquePointer.cpp
@@ -42,8 +42,7 @@ class LibStdcppUniquePtrSyntheticFrontEnd : public SyntheticChildrenFrontEnd {
   // objects are only destroyed when every shared pointer to any of them
   // is destroyed, so we must not store a shared pointer to any ValueObject
   // derived from our backend ValueObject (since we're in the same cluster).
-  ValueObject* m_ptr_obj = nullptr;
-  ValueObject* m_obj_obj = nullptr;
+  ValueObject *m_ptr_obj = nullptr;
   ValueObject* m_del_obj = nullptr;
 
   ValueObjectSP GetTuple();
@@ -107,7 +106,6 @@ lldb::ChildCacheState LibStdcppUniquePtrSyntheticFrontEnd::Update() {
     if (del_obj)
       m_del_obj = del_obj->Clone(ConstString("deleter")).get();
   }
-  m_obj_obj = nullptr;
 
   return lldb::ChildCacheState::eRefetch;
 }
@@ -119,15 +117,13 @@ LibStdcppUniquePtrSyntheticFrontEnd::GetChildAtIndex(uint32_t idx) {
   if (idx == 1 && m_del_obj)
     return m_del_obj->GetSP();
   if (idx == 2) {
-    if (m_ptr_obj && !m_obj_obj) {
-      Status error;
-      ValueObjectSP obj_obj = m_ptr_obj->Dereference(error);
-      if (error.Success()) {
-        m_obj_obj = obj_obj->Clone(ConstString("object")).get();
+    if (m_ptr_obj) {
+      Status status;
+      auto value_sp = m_ptr_obj->Dereference(status);
+      if (status.Success()) {
+        return value_sp;
       }
     }
-    if (m_obj_obj)
-      return m_obj_obj->GetSP();
   }
   return lldb::ValueObjectSP();
 }
@@ -156,14 +152,8 @@ bool LibStdcppUniquePtrSyntheticFrontEnd::GetSummary(
   if (!m_ptr_obj)
     return false;
 
-  bool success;
-  uint64_t ptr_value = m_ptr_obj->GetValueAsUnsigned(0, &success);
-  if (!success)
-    return false;
-  if (ptr_value == 0)
-    stream.Printf("nullptr");
-  else
-    stream.Printf("0x%" PRIx64, ptr_value);
+  DumpCxxSmartPtrPointerSummary(stream, *m_ptr_obj, options);
+
   return true;
 }
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
index dac93931bab1..cc0c9e728964 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCClassDescriptorV2.cpp
@@ -552,9 +552,8 @@ bool ClassDescriptorV2::Describe(
     } else {
       std::optional<method_list_t> base_method_list =
           GetMethodList(process, class_ro->m_baseMethods_ptr);
-      if (!base_method_list)
-        return false;
-      if (!ProcessMethodList(instance_method_func, *base_method_list))
+      if (base_method_list &&
+          !ProcessMethodList(instance_method_func, *base_method_list))
         return false;
     }
   }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index e459cb281793..4fcdebe5bac6 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -1505,15 +1505,24 @@ AppleObjCRuntimeV2::GetClassDescriptorFromISA(ObjCISA isa) {
 
 ObjCLanguageRuntime::ClassDescriptorSP
 AppleObjCRuntimeV2::GetClassDescriptor(ValueObject &valobj) {
+  ValueObjectSet seen;
+  return GetClassDescriptorImpl(valobj, seen);
+}
+
+ObjCLanguageRuntime::ClassDescriptorSP
+AppleObjCRuntimeV2::GetClassDescriptorImpl(ValueObject &valobj,
+                                           ValueObjectSet &seen) {
+  seen.insert(&valobj);
+
   ClassDescriptorSP objc_class_sp;
   if (valobj.IsBaseClass()) {
     ValueObject *parent = valobj.GetParent();
-    // if I am my own parent, bail out of here fast..
-    if (parent && parent != &valobj) {
-      ClassDescriptorSP parent_descriptor_sp = GetClassDescriptor(*parent);
-      if (parent_descriptor_sp)
-        return parent_descriptor_sp->GetSuperclass();
-    }
+    // Fail if there's a cycle in our parent chain.
+    if (!parent || seen.count(parent))
+      return nullptr;
+    if (ClassDescriptorSP parent_descriptor_sp =
+            GetClassDescriptorImpl(*parent, seen))
+      return parent_descriptor_sp->GetSuperclass();
     return nullptr;
   }
   // if we get an invalid VO (which might still happen when playing around with
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
index 79840f9be79b..b99bda2f7e25 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.h
@@ -20,6 +20,7 @@
 #include "Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.h"
 
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallSet.h"
 
 class RemoteNXMapTable;
 
@@ -421,6 +422,10 @@ class AppleObjCRuntimeV2 : public AppleObjCRuntime {
 
   lldb::addr_t GetISAHashTablePointer();
 
+  using ValueObjectSet = llvm::SmallPtrSet<ValueObject *, 8>;
+  ClassDescriptorSP GetClassDescriptorImpl(ValueObject &valobj,
+                                           ValueObjectSet &seen);
+
   /// Update the generation count of realized classes. This is not an exact
   /// count but rather a value that is incremented when new classes are realized
   /// or destroyed. Unlike the count in gdb_objc_realized_classes, it will
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index b1741926b74a..13df6e2f26b5 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -11,7 +11,6 @@
 
 #include "Plugins/Process/Utility/RegisterContextDarwin_arm.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_arm64.h"
-#include "Plugins/Process/Utility/RegisterContextDarwin_i386.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_riscv32.h"
 #include "Plugins/Process/Utility/RegisterContextDarwin_x86_64.h"
 #include "lldb/Core/Debugger.h"
@@ -81,9 +80,6 @@
 #ifdef CPU_TYPE_ARM64_32
 #undef CPU_TYPE_ARM64_32
 #endif
-#ifdef CPU_TYPE_I386
-#undef CPU_TYPE_I386
-#endif
 #ifdef CPU_TYPE_X86_64
 #undef CPU_TYPE_X86_64
 #endif
@@ -184,46 +180,32 @@ class RegisterContextDarwin_x86_64_Mach : public RegisterContextDarwin_x86_64 {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
 
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       if (flavor == 0)
-        done = true;
-      else {
-        uint32_t i;
-        uint32_t count = data.GetU32(&offset);
-        switch (flavor) {
-        case GPRRegSet:
-          for (i = 0; i < count; ++i)
-            (&gpr.rax)[i] = data.GetU64(&offset);
-          SetError(GPRRegSet, Read, 0);
-          done = true;
-
-          break;
-        case FPURegSet:
-          // TODO: fill in FPU regs....
-          // SetError (FPURegSet, Read, -1);
-          done = true;
-
-          break;
-        case EXCRegSet:
-          exc.trapno = data.GetU32(&offset);
-          exc.err = data.GetU32(&offset);
-          exc.faultvaddr = data.GetU64(&offset);
-          SetError(EXCRegSet, Read, 0);
-          done = true;
-          break;
-        case 7:
-        case 8:
-        case 9:
-          // fancy flavors that encapsulate of the above flavors...
-          break;
-
-        default:
-          done = true;
-          break;
-        }
+        break;
+      uint32_t count = data.GetU32(&offset);
+      switch (flavor) {
+      case GPRRegSet: {
+        uint32_t *gpr_data = reinterpret_cast<uint32_t *>(&gpr.rax);
+        for (uint32_t i = 0; i < count && offset < data.GetByteSize(); ++i)
+          gpr_data[i] = data.GetU32(&offset);
+        SetError(GPRRegSet, Read, 0);
+      } break;
+      case FPURegSet:
+        // TODO: fill in FPU regs....
+        SetError(FPURegSet, Read, -1);
+        break;
+      case EXCRegSet:
+        exc.trapno = data.GetU32(&offset);
+        exc.err = data.GetU32(&offset);
+        exc.faultvaddr = data.GetU64(&offset);
+        SetError(EXCRegSet, Read, 0);
+        break;
+      default:
+        offset += count * 4;
+        break;
       }
     }
   }
@@ -353,127 +335,11 @@ class RegisterContextDarwin_x86_64_Mach : public RegisterContextDarwin_x86_64 {
   }
 
 protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return 0; }
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return 0; }
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return 0; }
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override {
-    return 0;
-  }
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override {
-    return 0;
-  }
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override {
-    return 0;
-  }
-};
-
-class RegisterContextDarwin_i386_Mach : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextDarwin_i386_Mach(lldb_private::Thread &thread,
-                                  const DataExtractor &data)
-      : RegisterContextDarwin_i386(thread, 0) {
-    SetRegisterDataFrom_LC_THREAD(data);
-  }
-
-  void InvalidateAllRegisters() override {
-    // Do nothing... registers are always valid...
-  }
-
-  void SetRegisterDataFrom_LC_THREAD(const DataExtractor &data) {
-    lldb::offset_t offset = 0;
-    SetError(GPRRegSet, Read, -1);
-    SetError(FPURegSet, Read, -1);
-    SetError(EXCRegSet, Read, -1);
-    bool done = false;
-
-    while (!done) {
-      int flavor = data.GetU32(&offset);
-      if (flavor == 0)
-        done = true;
-      else {
-        uint32_t i;
-        uint32_t count = data.GetU32(&offset);
-        switch (flavor) {
-        case GPRRegSet:
-          for (i = 0; i < count; ++i)
-            (&gpr.eax)[i] = data.GetU32(&offset);
-          SetError(GPRRegSet, Read, 0);
-          done = true;
-
-          break;
-        case FPURegSet:
-          // TODO: fill in FPU regs....
-          // SetError (FPURegSet, Read, -1);
-          done = true;
-
-          break;
-        case EXCRegSet:
-          exc.trapno = data.GetU32(&offset);
-          exc.err = data.GetU32(&offset);
-          exc.faultvaddr = data.GetU32(&offset);
-          SetError(EXCRegSet, Read, 0);
-          done = true;
-          break;
-        case 7:
-        case 8:
-        case 9:
-          // fancy flavors that encapsulate of the above flavors...
-          break;
-
-        default:
-          done = true;
-          break;
-        }
-      }
-    }
-  }
-
-  static bool Create_LC_THREAD(Thread *thread, Stream &data) {
-    RegisterContextSP reg_ctx_sp(thread->GetRegisterContext());
-    if (reg_ctx_sp) {
-      RegisterContext *reg_ctx = reg_ctx_sp.get();
-
-      data.PutHex32(GPRRegSet); // Flavor
-      data.PutHex32(GPRWordCount);
-      PrintRegisterValue(reg_ctx, "eax", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ebx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ecx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "edx", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "edi", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "esi", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ebp", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "esp", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ss", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "eflags", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "eip", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "cs", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "ds", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "es", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "fs", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "gs", nullptr, 4, data);
-
-      // Write out the EXC registers
-      data.PutHex32(EXCRegSet);
-      data.PutHex32(EXCWordCount);
-      PrintRegisterValue(reg_ctx, "trapno", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "err", nullptr, 4, data);
-      PrintRegisterValue(reg_ctx, "faultvaddr", nullptr, 4, data);
-      return true;
-    }
-    return false;
-  }
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return 0; }
+  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override { return -1; }
 
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return 0; }
+  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override { return -1; }
 
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return 0; }
+  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override { return -1; }
 
   int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override {
     return 0;
@@ -505,12 +371,11 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
 
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRAltRegSet:
       case GPRRegSet: {
@@ -524,9 +389,7 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
 
           SetError(GPRRegSet, Read, 0);
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
 
       case FPURegSet: {
         uint8_t *fpu_reg_buf = (uint8_t *)&fpu.floats;
@@ -536,12 +399,8 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
           offset += fpu_reg_buf_size;
           fpu.fpscr = data.GetU32(&offset);
           SetError(FPURegSet, Read, 0);
-        } else {
-          done = true;
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
 
       case EXCRegSet:
         if (count == 3) {
@@ -550,14 +409,9 @@ class RegisterContextDarwin_arm_Mach : public RegisterContextDarwin_arm {
           exc.far = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        done = true;
-        offset = next_thread_state;
         break;
-
-      // Unknown register set flavor, stop trying to parse.
-      default:
-        done = true;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -640,11 +494,10 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
     SetError(GPRRegSet, Read, -1);
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
-    bool done = false;
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRRegSet:
         // x0-x29 + fp + lr + sp + pc (== 33 64-bit registers) plus cpsr (1
@@ -659,7 +512,6 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
           gpr.cpsr = data.GetU32(&offset);
           SetError(GPRRegSet, Read, 0);
         }
-        offset = next_thread_state;
         break;
       case FPURegSet: {
         uint8_t *fpu_reg_buf = (uint8_t *)&fpu.v[0];
@@ -668,12 +520,8 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
             data.ExtractBytes(offset, fpu_reg_buf_size, eByteOrderLittle,
                               fpu_reg_buf) == fpu_reg_buf_size) {
           SetError(FPURegSet, Read, 0);
-        } else {
-          done = true;
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
       case EXCRegSet:
         if (count == 4) {
           exc.far = data.GetU64(&offset);
@@ -681,12 +529,9 @@ class RegisterContextDarwin_arm64_Mach : public RegisterContextDarwin_arm64 {
           exc.exception = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        offset = next_thread_state;
-        break;
-      default:
-        done = true;
         break;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -789,11 +634,10 @@ class RegisterContextDarwin_riscv32_Mach
     SetError(FPURegSet, Read, -1);
     SetError(EXCRegSet, Read, -1);
     SetError(CSRRegSet, Read, -1);
-    bool done = false;
-    while (!done) {
+    while (offset < data.GetByteSize()) {
       int flavor = data.GetU32(&offset);
       uint32_t count = data.GetU32(&offset);
-      lldb::offset_t next_thread_state = offset + (count * 4);
+      offset_t next_thread_state = offset + (count * 4);
       switch (flavor) {
       case GPRRegSet:
         // x0-x31 + pc
@@ -803,7 +647,6 @@ class RegisterContextDarwin_riscv32_Mach
           gpr.pc = data.GetU32(&offset);
           SetError(GPRRegSet, Read, 0);
         }
-        offset = next_thread_state;
         break;
       case FPURegSet: {
         // f0-f31 + fcsr
@@ -813,9 +656,7 @@ class RegisterContextDarwin_riscv32_Mach
           fpr.fcsr = data.GetU32(&offset);
           SetError(FPURegSet, Read, 0);
         }
-      }
-        offset = next_thread_state;
-        break;
+      } break;
       case EXCRegSet:
         if (count == 3) {
           exc.exception = data.GetU32(&offset);
@@ -823,12 +664,9 @@ class RegisterContextDarwin_riscv32_Mach
           exc.far = data.GetU32(&offset);
           SetError(EXCRegSet, Read, 0);
         }
-        offset = next_thread_state;
-        break;
-      default:
-        done = true;
         break;
       }
+      offset = next_thread_state;
     }
   }
 
@@ -5422,16 +5260,6 @@ lldb_private::Address ObjectFileMachO::GetEntryPointAddress() {
               done = true;
             }
             break;
-          case llvm::MachO::CPU_TYPE_I386:
-            if (flavor ==
-                1) // x86_THREAD_STATE32 from mach/i386/thread_status.h
-            {
-              offset += 40; // This is the offset of eip in the GPR thread state
-                            // data structure.
-              start_address = m_data.GetU32(&offset);
-              done = true;
-            }
-            break;
           case llvm::MachO::CPU_TYPE_X86_64:
             if (flavor ==
                 4) // x86_THREAD_STATE64 from mach/i386/thread_status.h
@@ -5794,27 +5622,8 @@ bool ObjectFileMachO::GetCorefileThreadExtraInfos(
     std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
 
     Log *log(GetLog(LLDBLog::Object | LLDBLog::Process | LLDBLog::Thread));
-    auto lc_notes = FindLC_NOTEByName("process metadata");
-    for (auto lc_note : lc_notes) {
-      offset_t payload_offset = std::get<0>(lc_note);
-      offset_t strsize = std::get<1>(lc_note);
-      std::string buf(strsize, '\0');
-      if (m_data.CopyData(payload_offset, strsize, buf.data()) != strsize) {
-        LLDB_LOGF(log,
-                  "Unable to read %" PRIu64
-                  " bytes of 'process metadata' LC_NOTE JSON contents",
-                  strsize);
-        return false;
-      }
-      while (buf.back() == '\0')
-        buf.resize(buf.size() - 1);
-      StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(buf);
+    if (StructuredData::ObjectSP object_sp = GetCorefileProcessMetadata()) {
       StructuredData::Dictionary *dict = object_sp->GetAsDictionary();
-      if (!dict) {
-        LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
-                       "get a dictionary.");
-        return false;
-      }
       StructuredData::Array *threads;
       if (!dict->GetValueForKeyAsArray("threads", threads) || !threads) {
         LLDB_LOGF(log,
@@ -5857,6 +5666,49 @@ bool ObjectFileMachO::GetCorefileThreadExtraInfos(
   return false;
 }
 
+StructuredData::ObjectSP ObjectFileMachO::GetCorefileProcessMetadata() {
+  ModuleSP module_sp(GetModule());
+  if (!module_sp)
+    return {};
+
+  Log *log(GetLog(LLDBLog::Object | LLDBLog::Process | LLDBLog::Thread));
+  std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
+  auto lc_notes = FindLC_NOTEByName("process metadata");
+  if (lc_notes.size() == 0)
+    return {};
+
+  if (lc_notes.size() > 1)
+    LLDB_LOGF(
+        log,
+        "Multiple 'process metadata' LC_NOTEs found, only using the first.");
+
+  auto [payload_offset, strsize] = lc_notes[0];
+  std::string buf(strsize, '\0');
+  if (m_data.CopyData(payload_offset, strsize, buf.data()) != strsize) {
+    LLDB_LOGF(log,
+              "Unable to read %" PRIu64
+              " bytes of 'process metadata' LC_NOTE JSON contents",
+              strsize);
+    return {};
+  }
+  while (buf.back() == '\0')
+    buf.resize(buf.size() - 1);
+  StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(buf);
+  if (!object_sp) {
+    LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
+                   "parse as valid JSON.");
+    return {};
+  }
+  StructuredData::Dictionary *dict = object_sp->GetAsDictionary();
+  if (!dict) {
+    LLDB_LOGF(log, "Unable to read 'process metadata' LC_NOTE, did not "
+                   "get a dictionary.");
+    return {};
+  }
+
+  return object_sp;
+}
+
 lldb::RegisterContextSP
 ObjectFileMachO::GetThreadContextAtIndex(uint32_t idx,
                                          lldb_private::Thread &thread) {
@@ -5887,11 +5739,6 @@ ObjectFileMachO::GetThreadContextAtIndex(uint32_t idx,
             std::make_shared<RegisterContextDarwin_arm_Mach>(thread, data);
         break;
 
-      case llvm::MachO::CPU_TYPE_I386:
-        reg_ctx_sp =
-            std::make_shared<RegisterContextDarwin_i386_Mach>(thread, data);
-        break;
-
       case llvm::MachO::CPU_TYPE_X86_64:
         reg_ctx_sp =
             std::make_shared<RegisterContextDarwin_x86_64_Mach>(thread, data);
@@ -6759,11 +6606,6 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
                   thread_sp.get(), LC_THREAD_datas[thread_idx]);
               break;
 
-            case llvm::MachO::CPU_TYPE_I386:
-              RegisterContextDarwin_i386_Mach::Create_LC_THREAD(
-                  thread_sp.get(), LC_THREAD_datas[thread_idx]);
-              break;
-
             case llvm::MachO::CPU_TYPE_X86_64:
               RegisterContextDarwin_x86_64_Mach::Create_LC_THREAD(
                   thread_sp.get(), LC_THREAD_datas[thread_idx]);
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index 7f67f5e04f1d..7e3a6754dd0b 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -133,6 +133,8 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
 
   bool GetCorefileThreadExtraInfos(std::vector<lldb::tid_t> &tids) override;
 
+  lldb_private::StructuredData::ObjectSP GetCorefileProcessMetadata() override;
+
   bool LoadCoreFileImages(lldb_private::Process &process) override;
 
   lldb::RegisterContextSP
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index 262a7dc73171..1db7bc78013d 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -1030,6 +1030,40 @@ PlatformDarwin::ExtractAppSpecificInfo(Process &process) {
   return dict_sp;
 }
 
+static llvm::Expected<lldb_private::FileSpec>
+ResolveSDKPathFromDebugInfo(lldb_private::Target *target) {
+
+  ModuleSP exe_module_sp = target->GetExecutableModule();
+  if (!exe_module_sp)
+    return llvm::createStringError("Failed to get module from target");
+
+  SymbolFile *sym_file = exe_module_sp->GetSymbolFile();
+  if (!sym_file)
+    return llvm::createStringError("Failed to get symbol file from module");
+
+  XcodeSDK merged_sdk;
+  for (unsigned i = 0; i < sym_file->GetNumCompileUnits(); ++i) {
+    if (auto cu_sp = sym_file->GetCompileUnitAtIndex(i)) {
+      auto cu_sdk = sym_file->ParseXcodeSDK(*cu_sp);
+      merged_sdk.Merge(cu_sdk);
+    }
+  }
+
+  // TODO: The result of this loop is almost equivalent to deriving the SDK
+  // from the target triple, which would be a lot cheaper.
+  FileSpec sdk_path = merged_sdk.GetSysroot();
+  if (FileSystem::Instance().Exists(sdk_path)) {
+    return sdk_path;
+  }
+  auto path_or_err = HostInfo::GetSDKRoot(HostInfo::SDKOptions{merged_sdk});
+  if (!path_or_err)
+    return llvm::createStringError(
+        llvm::formatv("Failed to resolve SDK path: {0}",
+                      llvm::toString(path_or_err.takeError())));
+
+  return FileSpec(*path_or_err);
+}
+
 void PlatformDarwin::AddClangModuleCompilationOptionsForSDKType(
     Target *target, std::vector<std::string> &options, XcodeSDK::Type sdk_type) {
   const std::vector<std::string> apple_arguments = {
@@ -1129,15 +1163,13 @@ void PlatformDarwin::AddClangModuleCompilationOptionsForSDKType(
   FileSpec sysroot_spec;
 
   if (target) {
-    if (ModuleSP exe_module_sp = target->GetExecutableModule()) {
-      auto path_or_err = ResolveSDKPathFromDebugInfo(*exe_module_sp);
-      if (path_or_err) {
-        sysroot_spec = FileSpec(*path_or_err);
-      } else {
-        LLDB_LOG_ERROR(GetLog(LLDBLog::Types | LLDBLog::Host),
-                       path_or_err.takeError(),
-                       "Failed to resolve SDK path: {0}");
-      }
+    auto sysroot_spec_or_err = ::ResolveSDKPathFromDebugInfo(target);
+    if (!sysroot_spec_or_err) {
+      LLDB_LOG_ERROR(GetLog(LLDBLog::Types | LLDBLog::Host),
+                     sysroot_spec_or_err.takeError(),
+                     "Failed to resolve sysroot: {0}");
+    } else {
+      sysroot_spec = *sysroot_spec_or_err;
     }
   }
 
diff --git a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
index bf552e19742c..a22083a8a090 100644
--- a/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/NativeProcessFreeBSD.cpp
@@ -324,12 +324,14 @@ void NativeProcessFreeBSD::MonitorSIGTRAP(lldb::pid_t pid) {
         auto thread_info =
             m_threads_stepping_with_breakpoint.find(thread->GetID());
         if (thread_info != m_threads_stepping_with_breakpoint.end() &&
-            thread_info->second == regctx.GetPC()) {
+            llvm::is_contained(thread_info->second, regctx.GetPC())) {
           thread->SetStoppedByTrace();
-          Status brkpt_error = RemoveBreakpoint(thread_info->second);
-          if (brkpt_error.Fail())
-            LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
-                     thread_info->first, brkpt_error);
+          for (auto &&bp_addr : thread_info->second) {
+            Status brkpt_error = RemoveBreakpoint(bp_addr);
+            if (brkpt_error.Fail())
+              LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
+                       thread_info->first, brkpt_error);
+          }
           m_threads_stepping_with_breakpoint.erase(thread_info);
         } else
           thread->SetStoppedByBreakpoint();
diff --git a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
index 7f2aba0e4eb2..9c798cb1cc8f 100644
--- a/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp
@@ -833,7 +833,7 @@ void NativeProcessLinux::MonitorBreakpoint(NativeThreadLinux &thread) {
   auto stepping_with_bp_it =
       m_threads_stepping_with_breakpoint.find(thread.GetID());
   if (stepping_with_bp_it != m_threads_stepping_with_breakpoint.end() &&
-      stepping_with_bp_it->second == reg_ctx.GetPC())
+      llvm::is_contained(stepping_with_bp_it->second, reg_ctx.GetPC()))
     thread.SetStoppedByTrace();
 
   StopRunningThreads(thread.GetID());
@@ -1960,10 +1960,12 @@ void NativeProcessLinux::SignalIfAllThreadsStopped() {
   // Clear any temporary breakpoints we used to implement software single
   // stepping.
   for (const auto &thread_info : m_threads_stepping_with_breakpoint) {
-    Status error = RemoveBreakpoint(thread_info.second);
-    if (error.Fail())
-      LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
-               thread_info.first, error);
+    for (auto &&bp_addr : thread_info.second) {
+      Status error = RemoveBreakpoint(bp_addr);
+      if (error.Fail())
+        LLDB_LOG(log, "pid = {0} remove stepping breakpoint: {1}",
+                 thread_info.first, error);
+    }
   }
   m_threads_stepping_with_breakpoint.clear();
 
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt b/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
index ddce25c62046..f26e14f580cb 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/CMakeLists.txt
@@ -12,7 +12,6 @@ add_lldb_library(lldbPluginProcessMacOSXKernel PLUGIN
   ProcessKDPLog.cpp
   RegisterContextKDP_arm.cpp
   RegisterContextKDP_arm64.cpp
-  RegisterContextKDP_i386.cpp
   RegisterContextKDP_x86_64.cpp
   ThreadKDP.cpp
 
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
deleted file mode 100644
index 61dfeae6ddf4..000000000000
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-//===-- RegisterContextKDP_i386.cpp ---------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "RegisterContextKDP_i386.h"
-#include "ProcessKDP.h"
-#include "ThreadKDP.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-RegisterContextKDP_i386::RegisterContextKDP_i386(ThreadKDP &thread,
-                                                 uint32_t concrete_frame_idx)
-    : RegisterContextDarwin_i386(thread, concrete_frame_idx),
-      m_kdp_thread(thread) {}
-
-RegisterContextKDP_i386::~RegisterContextKDP_i386() = default;
-
-int RegisterContextKDP_i386::DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, GPRRegSet, &gpr, sizeof(gpr),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, FPURegSet, &fpu, sizeof(fpu),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestReadRegisters(tid, EXCRegSet, &exc, sizeof(exc),
-                                      error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteGPR(lldb::tid_t tid, int flavor,
-                                        const GPR &gpr) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, GPRRegSet, &gpr, sizeof(gpr),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteFPU(lldb::tid_t tid, int flavor,
-                                        const FPU &fpu) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, FPURegSet, &fpu, sizeof(fpu),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
-
-int RegisterContextKDP_i386::DoWriteEXC(lldb::tid_t tid, int flavor,
-                                        const EXC &exc) {
-  ProcessSP process_sp(CalculateProcess());
-  if (process_sp) {
-    Status error;
-    if (static_cast<ProcessKDP *>(process_sp.get())
-            ->GetCommunication()
-            .SendRequestWriteRegisters(tid, EXCRegSet, &exc, sizeof(exc),
-                                       error)) {
-      if (error.Success())
-        return 0;
-    }
-  }
-  return -1;
-}
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h b/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
deleted file mode 100644
index 04868e96191f..000000000000
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/RegisterContextKDP_i386.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===-- RegisterContextKDP_i386.h -------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
-
-#include "Plugins/Process/Utility/RegisterContextDarwin_i386.h"
-
-class ThreadKDP;
-
-class RegisterContextKDP_i386 : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextKDP_i386(ThreadKDP &thread, uint32_t concrete_frame_idx);
-
-  ~RegisterContextKDP_i386() override;
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override;
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override;
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override;
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override;
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override;
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override;
-
-  ThreadKDP &m_kdp_thread;
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_MACOSX_KERNEL_REGISTERCONTEXTKDP_I386_H
diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
index 1349de879c69..69d4cfdae4ad 100644
--- a/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
+++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ThreadKDP.cpp
@@ -26,7 +26,6 @@
 #include "ProcessKDPLog.h"
 #include "RegisterContextKDP_arm.h"
 #include "RegisterContextKDP_arm64.h"
-#include "RegisterContextKDP_i386.h"
 #include "RegisterContextKDP_x86_64.h"
 
 #include <memory>
@@ -105,10 +104,6 @@ ThreadKDP::CreateRegisterContextForFrame(StackFrame *frame) {
         reg_ctx_sp = std::make_shared<RegisterContextKDP_arm64>(
             *this, concrete_frame_idx);
         break;
-      case llvm::MachO::CPU_TYPE_I386:
-        reg_ctx_sp = std::make_shared<RegisterContextKDP_i386>(
-            *this, concrete_frame_idx);
-        break;
       case llvm::MachO::CPU_TYPE_X86_64:
         reg_ctx_sp = std::make_shared<RegisterContextKDP_x86_64>(
             *this, concrete_frame_idx);
diff --git a/lldb/source/Plugins/Process/Utility/CMakeLists.txt b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
index 48646b784f93..5d99c22dafe1 100644
--- a/lldb/source/Plugins/Process/Utility/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/Utility/CMakeLists.txt
@@ -22,7 +22,6 @@ add_lldb_library(lldbPluginProcessUtility
   RegisterContext_x86.cpp
   RegisterContextDarwin_arm.cpp
   RegisterContextDarwin_arm64.cpp
-  RegisterContextDarwin_i386.cpp
   RegisterContextDarwin_riscv32.cpp
   RegisterContextDarwin_x86_64.cpp
   RegisterContextDummy.cpp
@@ -35,7 +34,6 @@ add_lldb_library(lldbPluginProcessUtility
   RegisterContextLinux_x86_64.cpp
   RegisterContextLinux_s390x.cpp
   RegisterContextMach_arm.cpp
-  RegisterContextMach_i386.cpp
   RegisterContextMach_x86_64.cpp
   RegisterContextMemory.cpp
   RegisterContextNetBSD_i386.cpp
diff --git a/lldb/source/Plugins/Process/Utility/LinuxSignals.cpp b/lldb/source/Plugins/Process/Utility/LinuxSignals.cpp
index 15a77ce227ec..5346babc1857 100644
--- a/lldb/source/Plugins/Process/Utility/LinuxSignals.cpp
+++ b/lldb/source/Plugins/Process/Utility/LinuxSignals.cpp
@@ -65,6 +65,10 @@
 // See siginfo.h in the Linux Kernel, these codes can be sent for any signal.
 #define ADD_LINUX_SIGNAL(signo, name, ...)                                     \
   AddSignal(signo, name, __VA_ARGS__);                                         \
+  ADD_SIGCODE(signo, signo, SI_USER, 0, "sent by kill, sigsend or raise",      \
+              SignalCodePrintOption::Sender);                                  \
+  ADD_SIGCODE(signo, signo, SI_KERNEL, 0x80, "sent by kernel (SI_KERNEL)",     \
+              SignalCodePrintOption::Sender);                                  \
   ADD_SIGCODE(signo, signo, SI_QUEUE, -1, "sent by sigqueue",                  \
               SignalCodePrintOption::Sender);                                  \
   ADD_SIGCODE(signo, signo, SI_TIMER, -2, "sent by timer expiration",          \
diff --git a/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.cpp b/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.cpp
index 7e25bc4ea2a2..9f60675e5190 100644
--- a/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.cpp
+++ b/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.cpp
@@ -247,7 +247,7 @@ MemoryTagManagerAArch64MTE::UnpackTagsData(const std::vector<uint8_t> &tags,
   return unpacked;
 }
 
-std::vector<lldb::addr_t>
+llvm::Expected<std::vector<lldb::addr_t>>
 MemoryTagManagerAArch64MTE::UnpackTagsFromCoreFileSegment(
     CoreReaderFn reader, lldb::addr_t tag_segment_virtual_address,
     lldb::addr_t tag_segment_data_address, lldb::addr_t addr,
@@ -290,8 +290,12 @@ MemoryTagManagerAArch64MTE::UnpackTagsFromCoreFileSegment(
   const size_t bytes_copied =
       reader(tag_segment_data_address + file_offset_in_bytes, tag_bytes_to_read,
              tag_data.data());
-  UNUSED_IF_ASSERT_DISABLED(bytes_copied);
-  assert(bytes_copied == tag_bytes_to_read);
+  if (bytes_copied != tag_bytes_to_read) {
+    return llvm::createStringError(
+        llvm::inconvertibleErrorCode(),
+        "Could not read tags from core file segment. Segment "
+        "is missing some or all tag data.");
+  }
 
   std::vector<lldb::addr_t> tags;
   tags.reserve(2 * tag_data.size());
diff --git a/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h b/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h
index 365e176e5b1d..79d24ce78ece 100644
--- a/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h
+++ b/lldb/source/Plugins/Process/Utility/MemoryTagManagerAArch64MTE.h
@@ -44,7 +44,7 @@ class MemoryTagManagerAArch64MTE : public MemoryTagManager {
   UnpackTagsData(const std::vector<uint8_t> &tags,
                  size_t granules = 0) const override;
 
-  std::vector<lldb::addr_t>
+  llvm::Expected<std::vector<lldb::addr_t>>
   UnpackTagsFromCoreFileSegment(CoreReaderFn reader,
                                 lldb::addr_t tag_segment_virtual_address,
                                 lldb::addr_t tag_segment_data_address,
diff --git a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
index 31d59e387d25..eddf4b97baba 100644
--- a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
+++ b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.cpp
@@ -87,34 +87,10 @@ static size_t WriteMemoryCallback(EmulateInstruction *instruction, void *baton,
   return length;
 }
 
-static lldb::addr_t ReadFlags(NativeRegisterContext &regsiter_context) {
-  const RegisterInfo *flags_info = regsiter_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS);
-  return regsiter_context.ReadRegisterAsUnsigned(flags_info,
-                                                 LLDB_INVALID_ADDRESS);
-}
-
-static int GetSoftwareBreakpointSize(const ArchSpec &arch,
-                                     lldb::addr_t next_flags) {
-  if (arch.GetMachine() == llvm::Triple::arm) {
-    if (next_flags & 0x20)
-      // Thumb mode
-      return 2;
-    // Arm mode
-    return 4;
-  }
-  if (arch.IsMIPS() || arch.GetTriple().isPPC64() ||
-      arch.GetTriple().isRISCV() || arch.GetTriple().isLoongArch())
-    return 4;
-  return 0;
-}
-
-static Status SetSoftwareBreakpointOnPC(const ArchSpec &arch, lldb::addr_t pc,
-                                        lldb::addr_t next_flags,
-                                        NativeProcessProtocol &process) {
-  int size_hint = GetSoftwareBreakpointSize(arch, next_flags);
+static Status SetSoftwareBreakpoint(lldb::addr_t bp_addr, unsigned bp_size,
+                                    NativeProcessProtocol &process) {
   Status error;
-  error = process.SetBreakpoint(pc, size_hint, /*hardware=*/false);
+  error = process.SetBreakpoint(bp_addr, bp_size, /*hardware=*/false);
 
   // If setting the breakpoint fails because pc is out of the address
   // space, ignore it and let the debugee segfault.
@@ -136,7 +112,6 @@ Status NativeProcessSoftwareSingleStep::SetupSoftwareSingleStepping(
   std::unique_ptr<EmulateInstruction> emulator_up(
       EmulateInstruction::FindPlugin(arch, eInstructionTypePCModifying,
                                      nullptr));
-
   if (emulator_up == nullptr)
     return Status::FromErrorString("Instruction emulator not found!");
 
@@ -147,65 +122,24 @@ Status NativeProcessSoftwareSingleStep::SetupSoftwareSingleStepping(
   emulator_up->SetWriteMemCallback(&WriteMemoryCallback);
   emulator_up->SetWriteRegCallback(&WriteRegisterCallback);
 
-  if (!emulator_up->ReadInstruction()) {
-    // try to get at least the size of next instruction to set breakpoint.
-    auto instr_size = emulator_up->GetLastInstrSize();
-    if (!instr_size)
-      return Status::FromErrorString("Read instruction failed!");
-    bool success = false;
-    auto pc = emulator_up->ReadRegisterUnsigned(eRegisterKindGeneric,
-                                                LLDB_REGNUM_GENERIC_PC,
-                                                LLDB_INVALID_ADDRESS, &success);
-    if (!success)
-      return Status::FromErrorString("Reading pc failed!");
-    lldb::addr_t next_pc = pc + *instr_size;
-    auto result =
-        SetSoftwareBreakpointOnPC(arch, next_pc, /* next_flags */ 0x0, process);
-    m_threads_stepping_with_breakpoint.insert({thread.GetID(), next_pc});
-    return result;
-  }
+  auto bp_locaions_predictor =
+      EmulateInstruction::CreateBreakpointLocationPredictor(
+          std::move(emulator_up));
+
+  auto bp_locations = bp_locaions_predictor->GetBreakpointLocations(error);
+  if (error.Fail())
+    return error;
+
+  for (auto &&bp_addr : bp_locations) {
+    auto bp_size = bp_locaions_predictor->GetBreakpointSize(bp_addr);
+    if (auto err = bp_size.takeError())
+      return Status(toString(std::move(err)));
 
-  bool emulation_result =
-      emulator_up->EvaluateInstruction(eEmulateInstructionOptionAutoAdvancePC);
-
-  const RegisterInfo *reg_info_pc = register_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_PC);
-  const RegisterInfo *reg_info_flags = register_context.GetRegisterInfo(
-      eRegisterKindGeneric, LLDB_REGNUM_GENERIC_FLAGS);
-
-  auto pc_it =
-      baton.m_register_values.find(reg_info_pc->kinds[eRegisterKindDWARF]);
-  auto flags_it = reg_info_flags == nullptr
-                      ? baton.m_register_values.end()
-                      : baton.m_register_values.find(
-                            reg_info_flags->kinds[eRegisterKindDWARF]);
-
-  lldb::addr_t next_pc;
-  lldb::addr_t next_flags;
-  if (emulation_result) {
-    assert(pc_it != baton.m_register_values.end() &&
-           "Emulation was successfull but PC wasn't updated");
-    next_pc = pc_it->second.GetAsUInt64();
-
-    if (flags_it != baton.m_register_values.end())
-      next_flags = flags_it->second.GetAsUInt64();
-    else
-      next_flags = ReadFlags(register_context);
-  } else if (pc_it == baton.m_register_values.end()) {
-    // Emulate instruction failed and it haven't changed PC. Advance PC with
-    // the size of the current opcode because the emulation of all
-    // PC modifying instruction should be successful. The failure most
-    // likely caused by a not supported instruction which don't modify PC.
-    next_pc = register_context.GetPC() + emulator_up->GetOpcode().GetByteSize();
-    next_flags = ReadFlags(register_context);
-  } else {
-    // The instruction emulation failed after it modified the PC. It is an
-    // unknown error where we can't continue because the next instruction is
-    // modifying the PC but we don't  know how.
-    return Status::FromErrorString(
-        "Instruction emulation failed unexpectedly.");
+    error = SetSoftwareBreakpoint(bp_addr, *bp_size, process);
+    if (error.Fail())
+      return error;
   }
-  auto result = SetSoftwareBreakpointOnPC(arch, next_pc, next_flags, process);
-  m_threads_stepping_with_breakpoint.insert({thread.GetID(), next_pc});
-  return result;
+
+  m_threads_stepping_with_breakpoint.insert({thread.GetID(), bp_locations});
+  return error;
 }
diff --git a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
index f9435b7a84ba..e30e9594359f 100644
--- a/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
+++ b/lldb/source/Plugins/Process/Utility/NativeProcessSoftwareSingleStep.h
@@ -23,7 +23,8 @@ class NativeProcessSoftwareSingleStep {
 protected:
   // List of thread ids stepping with a breakpoint with the address of
   // the relevan breakpoint
-  std::map<lldb::tid_t, lldb::addr_t> m_threads_stepping_with_breakpoint;
+  std::map<lldb::tid_t, std::vector<lldb::addr_t>>
+      m_threads_stepping_with_breakpoint;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp
deleted file mode 100644
index 174a5a567d2d..000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.cpp
+++ /dev/null
@@ -1,958 +0,0 @@
-//===-- RegisterContextDarwin_i386.cpp ------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Utility/DataBufferHeap.h"
-#include "lldb/Utility/DataExtractor.h"
-#include "lldb/Utility/Endian.h"
-#include "lldb/Utility/Log.h"
-#include "lldb/Utility/RegisterValue.h"
-#include "lldb/Utility/Scalar.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Compiler.h"
-
-#include <cstddef>
-
-#include <memory>
-
-#include "RegisterContextDarwin_i386.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-enum {
-  gpr_eax = 0,
-  gpr_ebx,
-  gpr_ecx,
-  gpr_edx,
-  gpr_edi,
-  gpr_esi,
-  gpr_ebp,
-  gpr_esp,
-  gpr_ss,
-  gpr_eflags,
-  gpr_eip,
-  gpr_cs,
-  gpr_ds,
-  gpr_es,
-  gpr_fs,
-  gpr_gs,
-
-  fpu_fcw,
-  fpu_fsw,
-  fpu_ftw,
-  fpu_fop,
-  fpu_ip,
-  fpu_cs,
-  fpu_dp,
-  fpu_ds,
-  fpu_mxcsr,
-  fpu_mxcsrmask,
-  fpu_stmm0,
-  fpu_stmm1,
-  fpu_stmm2,
-  fpu_stmm3,
-  fpu_stmm4,
-  fpu_stmm5,
-  fpu_stmm6,
-  fpu_stmm7,
-  fpu_xmm0,
-  fpu_xmm1,
-  fpu_xmm2,
-  fpu_xmm3,
-  fpu_xmm4,
-  fpu_xmm5,
-  fpu_xmm6,
-  fpu_xmm7,
-
-  exc_trapno,
-  exc_err,
-  exc_faultvaddr,
-
-  k_num_registers,
-
-  // Aliases
-  fpu_fctrl = fpu_fcw,
-  fpu_fstat = fpu_fsw,
-  fpu_ftag = fpu_ftw,
-  fpu_fiseg = fpu_cs,
-  fpu_fioff = fpu_ip,
-  fpu_foseg = fpu_ds,
-  fpu_fooff = fpu_dp
-};
-
-enum {
-  ehframe_eax = 0,
-  ehframe_ecx,
-  ehframe_edx,
-  ehframe_ebx,
-  ehframe_ebp,
-  ehframe_esp,
-  ehframe_esi,
-  ehframe_edi,
-  ehframe_eip,
-  ehframe_eflags
-};
-
-enum {
-  dwarf_eax = 0,
-  dwarf_ecx,
-  dwarf_edx,
-  dwarf_ebx,
-  dwarf_esp,
-  dwarf_ebp,
-  dwarf_esi,
-  dwarf_edi,
-  dwarf_eip,
-  dwarf_eflags,
-  dwarf_stmm0 = 11,
-  dwarf_stmm1,
-  dwarf_stmm2,
-  dwarf_stmm3,
-  dwarf_stmm4,
-  dwarf_stmm5,
-  dwarf_stmm6,
-  dwarf_stmm7,
-  dwarf_xmm0 = 21,
-  dwarf_xmm1,
-  dwarf_xmm2,
-  dwarf_xmm3,
-  dwarf_xmm4,
-  dwarf_xmm5,
-  dwarf_xmm6,
-  dwarf_xmm7
-};
-
-#define GPR_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::GPR, reg))
-#define FPU_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::FPU, reg) +             \
-   sizeof(RegisterContextDarwin_i386::GPR))
-#define EXC_OFFSET(reg)                                                        \
-  (LLVM_EXTENSION offsetof(RegisterContextDarwin_i386::EXC, reg) +             \
-   sizeof(RegisterContextDarwin_i386::GPR) +                                   \
-   sizeof(RegisterContextDarwin_i386::FPU))
-
-// These macros will auto define the register name, alt name, register size,
-// register offset, encoding, format and native register. This ensures that the
-// register state structures are defined correctly and have the correct sizes
-// and offsets.
-#define DEFINE_GPR(reg, alt)                                                   \
-  #reg, alt, sizeof(((RegisterContextDarwin_i386::GPR *) NULL)->reg),          \
-                    GPR_OFFSET(reg), eEncodingUint, eFormatHex
-#define DEFINE_FPU_UINT(reg)                                                   \
-  #reg, NULL, sizeof(((RegisterContextDarwin_i386::FPU *) NULL)->reg),         \
-                     FPU_OFFSET(reg), eEncodingUint, eFormatHex
-#define DEFINE_FPU_VECT(reg, i)                                                \
-  #reg #i, NULL,                                                               \
-      sizeof(((RegisterContextDarwin_i386::FPU *) NULL)->reg[i].bytes),        \
-              FPU_OFFSET(reg[i]), eEncodingVector, eFormatVectorOfUInt8,       \
-                         {LLDB_INVALID_REGNUM, dwarf_##reg##i,                 \
-                          LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,            \
-                          fpu_##reg##i },                                      \
-                          nullptr, nullptr, nullptr,
-
-#define DEFINE_EXC(reg)                                                        \
-  #reg, NULL, sizeof(((RegisterContextDarwin_i386::EXC *) NULL)->reg),         \
-                     EXC_OFFSET(reg), eEncodingUint, eFormatHex
-#define REG_CONTEXT_SIZE                                                       \
-  (sizeof(RegisterContextDarwin_i386::GPR) +                                   \
-   sizeof(RegisterContextDarwin_i386::FPU) +                                   \
-   sizeof(RegisterContextDarwin_i386::EXC))
-
-static RegisterInfo g_register_infos[] = {
-    {DEFINE_GPR(eax, nullptr),
-     {ehframe_eax, dwarf_eax, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_eax},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ebx, nullptr),
-     {ehframe_ebx, dwarf_ebx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_ebx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ecx, nullptr),
-     {ehframe_ecx, dwarf_ecx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_ecx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(edx, nullptr),
-     {ehframe_edx, dwarf_edx, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_edx},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(edi, nullptr),
-     {ehframe_edi, dwarf_edi, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_edi},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(esi, nullptr),
-     {ehframe_esi, dwarf_esi, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      gpr_esi},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ebp, "fp"),
-     {ehframe_ebp, dwarf_ebp, LLDB_REGNUM_GENERIC_FP, LLDB_INVALID_REGNUM,
-      gpr_ebp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(esp, "sp"),
-     {ehframe_esp, dwarf_esp, LLDB_REGNUM_GENERIC_SP, LLDB_INVALID_REGNUM,
-      gpr_esp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ss, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_ss},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(eflags, "flags"),
-     {ehframe_eflags, dwarf_eflags, LLDB_REGNUM_GENERIC_FLAGS,
-      LLDB_INVALID_REGNUM, gpr_eflags},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(eip, "pc"),
-     {ehframe_eip, dwarf_eip, LLDB_REGNUM_GENERIC_PC, LLDB_INVALID_REGNUM,
-      gpr_eip},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(cs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_cs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(ds, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_ds},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(es, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_es},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(fs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_fs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_GPR(gs, nullptr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, gpr_gs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-
-    {DEFINE_FPU_UINT(fcw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fcw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(fsw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fsw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ftw),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ftw},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(fop),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_fop},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ip),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ip},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(cs),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_cs},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(dp),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_dp},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(ds),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_ds},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(mxcsr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_mxcsr},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_UINT(mxcsrmask),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, fpu_mxcsrmask},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_FPU_VECT(stmm, 0)},
-    {DEFINE_FPU_VECT(stmm, 1)},
-    {DEFINE_FPU_VECT(stmm, 2)},
-    {DEFINE_FPU_VECT(stmm, 3)},
-    {DEFINE_FPU_VECT(stmm, 4)},
-    {DEFINE_FPU_VECT(stmm, 5)},
-    {DEFINE_FPU_VECT(stmm, 6)},
-    {DEFINE_FPU_VECT(stmm, 7)},
-    {DEFINE_FPU_VECT(xmm, 0)},
-    {DEFINE_FPU_VECT(xmm, 1)},
-    {DEFINE_FPU_VECT(xmm, 2)},
-    {DEFINE_FPU_VECT(xmm, 3)},
-    {DEFINE_FPU_VECT(xmm, 4)},
-    {DEFINE_FPU_VECT(xmm, 5)},
-    {DEFINE_FPU_VECT(xmm, 6)},
-    {DEFINE_FPU_VECT(xmm, 7)},
-
-    {DEFINE_EXC(trapno),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_trapno},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_EXC(err),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_err},
-     nullptr,
-     nullptr,
-     nullptr,
-    },
-    {DEFINE_EXC(faultvaddr),
-     {LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM, LLDB_INVALID_REGNUM,
-      LLDB_INVALID_REGNUM, exc_faultvaddr},
-     nullptr,
-     nullptr,
-     nullptr,
-     }};
-
-static size_t k_num_register_infos = std::size(g_register_infos);
-
-RegisterContextDarwin_i386::RegisterContextDarwin_i386(
-    Thread &thread, uint32_t concrete_frame_idx)
-    : RegisterContext(thread, concrete_frame_idx), gpr(), fpu(), exc() {
-  uint32_t i;
-  for (i = 0; i < kNumErrors; i++) {
-    gpr_errs[i] = -1;
-    fpu_errs[i] = -1;
-    exc_errs[i] = -1;
-  }
-}
-
-RegisterContextDarwin_i386::~RegisterContextDarwin_i386() = default;
-
-void RegisterContextDarwin_i386::InvalidateAllRegisters() {
-  InvalidateAllRegisterStates();
-}
-
-size_t RegisterContextDarwin_i386::GetRegisterCount() {
-  assert(k_num_register_infos == k_num_registers);
-  return k_num_registers;
-}
-
-const RegisterInfo *
-RegisterContextDarwin_i386::GetRegisterInfoAtIndex(size_t reg) {
-  assert(k_num_register_infos == k_num_registers);
-  if (reg < k_num_registers)
-    return &g_register_infos[reg];
-  return nullptr;
-}
-
-size_t RegisterContextDarwin_i386::GetRegisterInfosCount() {
-  return k_num_register_infos;
-}
-
-const RegisterInfo *RegisterContextDarwin_i386::GetRegisterInfos() {
-  return g_register_infos;
-}
-
-// General purpose registers
-static uint32_t g_gpr_regnums[] = {
-    gpr_eax, gpr_ebx,    gpr_ecx, gpr_edx, gpr_edi, gpr_esi, gpr_ebp, gpr_esp,
-    gpr_ss,  gpr_eflags, gpr_eip, gpr_cs,  gpr_ds,  gpr_es,  gpr_fs,  gpr_gs};
-
-// Floating point registers
-static uint32_t g_fpu_regnums[] = {
-    fpu_fcw,   fpu_fsw,   fpu_ftw,   fpu_fop,       fpu_ip,    fpu_cs,
-    fpu_dp,    fpu_ds,    fpu_mxcsr, fpu_mxcsrmask, fpu_stmm0, fpu_stmm1,
-    fpu_stmm2, fpu_stmm3, fpu_stmm4, fpu_stmm5,     fpu_stmm6, fpu_stmm7,
-    fpu_xmm0,  fpu_xmm1,  fpu_xmm2,  fpu_xmm3,      fpu_xmm4,  fpu_xmm5,
-    fpu_xmm6,  fpu_xmm7};
-
-// Exception registers
-
-static uint32_t g_exc_regnums[] = {exc_trapno, exc_err, exc_faultvaddr};
-
-// Number of registers in each register set
-const size_t k_num_gpr_registers = std::size(g_gpr_regnums);
-const size_t k_num_fpu_registers = std::size(g_fpu_regnums);
-const size_t k_num_exc_registers = std::size(g_exc_regnums);
-
-// Register set definitions. The first definitions at register set index of
-// zero is for all registers, followed by other registers sets. The register
-// information for the all register set need not be filled in.
-static const RegisterSet g_reg_sets[] = {
-    {
-        "General Purpose Registers", "gpr", k_num_gpr_registers, g_gpr_regnums,
-    },
-    {"Floating Point Registers", "fpu", k_num_fpu_registers, g_fpu_regnums},
-    {"Exception State Registers", "exc", k_num_exc_registers, g_exc_regnums}};
-
-const size_t k_num_regsets = std::size(g_reg_sets);
-
-size_t RegisterContextDarwin_i386::GetRegisterSetCount() {
-  return k_num_regsets;
-}
-
-const RegisterSet *RegisterContextDarwin_i386::GetRegisterSet(size_t reg_set) {
-  if (reg_set < k_num_regsets)
-    return &g_reg_sets[reg_set];
-  return nullptr;
-}
-
-// Register information definitions for 32 bit i386.
-int RegisterContextDarwin_i386::GetSetForNativeRegNum(int reg_num) {
-  if (reg_num < fpu_fcw)
-    return GPRRegSet;
-  else if (reg_num < exc_trapno)
-    return FPURegSet;
-  else if (reg_num < k_num_registers)
-    return EXCRegSet;
-  return -1;
-}
-
-void RegisterContextDarwin_i386::LogGPR(Log *log, const char *title) {
-  if (log) {
-    if (title)
-      LLDB_LOGF(log, "%s", title);
-    for (uint32_t i = 0; i < k_num_gpr_registers; i++) {
-      uint32_t reg = gpr_eax + i;
-      LLDB_LOGF(log, "%12s = 0x%8.8x", g_register_infos[reg].name,
-                (&gpr.eax)[reg]);
-    }
-  }
-}
-
-int RegisterContextDarwin_i386::ReadGPR(bool force) {
-  int set = GPRRegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadGPR(GetThreadID(), set, gpr));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::ReadFPU(bool force) {
-  int set = FPURegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadFPU(GetThreadID(), set, fpu));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::ReadEXC(bool force) {
-  int set = EXCRegSet;
-  if (force || !RegisterSetIsCached(set)) {
-    SetError(set, Read, DoReadEXC(GetThreadID(), set, exc));
-  }
-  return GetError(set, Read);
-}
-
-int RegisterContextDarwin_i386::WriteGPR() {
-  int set = GPRRegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteGPR(GetThreadID(), set, gpr));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::WriteFPU() {
-  int set = FPURegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteFPU(GetThreadID(), set, fpu));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::WriteEXC() {
-  int set = EXCRegSet;
-  if (!RegisterSetIsCached(set)) {
-    SetError(set, Write, -1);
-    return -1;
-  }
-  SetError(set, Write, DoWriteEXC(GetThreadID(), set, exc));
-  SetError(set, Read, -1);
-  return GetError(set, Write);
-}
-
-int RegisterContextDarwin_i386::ReadRegisterSet(uint32_t set, bool force) {
-  switch (set) {
-  case GPRRegSet:
-    return ReadGPR(force);
-  case FPURegSet:
-    return ReadFPU(force);
-  case EXCRegSet:
-    return ReadEXC(force);
-  default:
-    break;
-  }
-  return -1;
-}
-
-int RegisterContextDarwin_i386::WriteRegisterSet(uint32_t set) {
-  // Make sure we have a valid context to set.
-  if (RegisterSetIsCached(set)) {
-    switch (set) {
-    case GPRRegSet:
-      return WriteGPR();
-    case FPURegSet:
-      return WriteFPU();
-    case EXCRegSet:
-      return WriteEXC();
-    default:
-      break;
-    }
-  }
-  return -1;
-}
-
-bool RegisterContextDarwin_i386::ReadRegister(const RegisterInfo *reg_info,
-                                              RegisterValue &value) {
-  const uint32_t reg = reg_info->kinds[eRegisterKindLLDB];
-  int set = RegisterContextDarwin_i386::GetSetForNativeRegNum(reg);
-
-  if (set == -1)
-    return false;
-
-  if (ReadRegisterSet(set, false) != 0)
-    return false;
-
-  switch (reg) {
-  case gpr_eax:
-  case gpr_ebx:
-  case gpr_ecx:
-  case gpr_edx:
-  case gpr_edi:
-  case gpr_esi:
-  case gpr_ebp:
-  case gpr_esp:
-  case gpr_ss:
-  case gpr_eflags:
-  case gpr_eip:
-  case gpr_cs:
-  case gpr_ds:
-  case gpr_es:
-  case gpr_fs:
-  case gpr_gs:
-    value = (&gpr.eax)[reg - gpr_eax];
-    break;
-
-  case fpu_fcw:
-    value = fpu.fcw;
-    break;
-
-  case fpu_fsw:
-    value = fpu.fsw;
-    break;
-
-  case fpu_ftw:
-    value = fpu.ftw;
-    break;
-
-  case fpu_fop:
-    value = fpu.fop;
-    break;
-
-  case fpu_ip:
-    value = fpu.ip;
-    break;
-
-  case fpu_cs:
-    value = fpu.cs;
-    break;
-
-  case fpu_dp:
-    value = fpu.dp;
-    break;
-
-  case fpu_ds:
-    value = fpu.ds;
-    break;
-
-  case fpu_mxcsr:
-    value = fpu.mxcsr;
-    break;
-
-  case fpu_mxcsrmask:
-    value = fpu.mxcsrmask;
-    break;
-
-  case fpu_stmm0:
-  case fpu_stmm1:
-  case fpu_stmm2:
-  case fpu_stmm3:
-  case fpu_stmm4:
-  case fpu_stmm5:
-  case fpu_stmm6:
-  case fpu_stmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    //::memcpy (reg_value.value.vector.uint8, fpu.stmm[reg - fpu_stmm0].bytes,
-    //10);
-    return false;
-
-  case fpu_xmm0:
-  case fpu_xmm1:
-  case fpu_xmm2:
-  case fpu_xmm3:
-  case fpu_xmm4:
-  case fpu_xmm5:
-  case fpu_xmm6:
-  case fpu_xmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    //::memcpy (reg_value.value.vector.uint8, fpu.xmm[reg - fpu_xmm0].bytes,
-    //16);
-    return false;
-
-  case exc_trapno:
-    value = exc.trapno;
-    break;
-
-  case exc_err:
-    value = exc.err;
-    break;
-
-  case exc_faultvaddr:
-    value = exc.faultvaddr;
-    break;
-
-  default:
-    return false;
-  }
-  return true;
-}
-
-bool RegisterContextDarwin_i386::WriteRegister(const RegisterInfo *reg_info,
-                                               const RegisterValue &value) {
-  const uint32_t reg = reg_info->kinds[eRegisterKindLLDB];
-  int set = GetSetForNativeRegNum(reg);
-
-  if (set == -1)
-    return false;
-
-  if (ReadRegisterSet(set, false) != 0)
-    return false;
-
-  switch (reg) {
-  case gpr_eax:
-  case gpr_ebx:
-  case gpr_ecx:
-  case gpr_edx:
-  case gpr_edi:
-  case gpr_esi:
-  case gpr_ebp:
-  case gpr_esp:
-  case gpr_ss:
-  case gpr_eflags:
-  case gpr_eip:
-  case gpr_cs:
-  case gpr_ds:
-  case gpr_es:
-  case gpr_fs:
-  case gpr_gs:
-    (&gpr.eax)[reg - gpr_eax] = value.GetAsUInt32();
-    break;
-
-  case fpu_fcw:
-    fpu.fcw = value.GetAsUInt16();
-    break;
-
-  case fpu_fsw:
-    fpu.fsw = value.GetAsUInt16();
-    break;
-
-  case fpu_ftw:
-    fpu.ftw = value.GetAsUInt8();
-    break;
-
-  case fpu_fop:
-    fpu.fop = value.GetAsUInt16();
-    break;
-
-  case fpu_ip:
-    fpu.ip = value.GetAsUInt32();
-    break;
-
-  case fpu_cs:
-    fpu.cs = value.GetAsUInt16();
-    break;
-
-  case fpu_dp:
-    fpu.dp = value.GetAsUInt32();
-    break;
-
-  case fpu_ds:
-    fpu.ds = value.GetAsUInt16();
-    break;
-
-  case fpu_mxcsr:
-    fpu.mxcsr = value.GetAsUInt32();
-    break;
-
-  case fpu_mxcsrmask:
-    fpu.mxcsrmask = value.GetAsUInt32();
-    break;
-
-  case fpu_stmm0:
-  case fpu_stmm1:
-  case fpu_stmm2:
-  case fpu_stmm3:
-  case fpu_stmm4:
-  case fpu_stmm5:
-  case fpu_stmm6:
-  case fpu_stmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    ::memcpy(fpu.stmm[reg - fpu_stmm0].bytes, value.GetBytes(),
-             value.GetByteSize());
-    return false;
-
-  case fpu_xmm0:
-  case fpu_xmm1:
-  case fpu_xmm2:
-  case fpu_xmm3:
-  case fpu_xmm4:
-  case fpu_xmm5:
-  case fpu_xmm6:
-  case fpu_xmm7:
-    // These values don't fit into scalar types,
-    // RegisterContext::ReadRegisterBytes() must be used for these registers
-    ::memcpy(fpu.xmm[reg - fpu_xmm0].bytes, value.GetBytes(),
-             value.GetByteSize());
-    return false;
-
-  case exc_trapno:
-    exc.trapno = value.GetAsUInt32();
-    break;
-
-  case exc_err:
-    exc.err = value.GetAsUInt32();
-    break;
-
-  case exc_faultvaddr:
-    exc.faultvaddr = value.GetAsUInt32();
-    break;
-
-  default:
-    return false;
-  }
-  return WriteRegisterSet(set) == 0;
-}
-
-bool RegisterContextDarwin_i386::ReadAllRegisterValues(
-    lldb::WritableDataBufferSP &data_sp) {
-  data_sp = std::make_shared<DataBufferHeap>(REG_CONTEXT_SIZE, 0);
-  if (ReadGPR(false) == 0 && ReadFPU(false) == 0 && ReadEXC(false) == 0) {
-    uint8_t *dst = data_sp->GetBytes();
-    ::memcpy(dst, &gpr, sizeof(gpr));
-    dst += sizeof(gpr);
-
-    ::memcpy(dst, &fpu, sizeof(fpu));
-    dst += sizeof(gpr);
-
-    ::memcpy(dst, &exc, sizeof(exc));
-    return true;
-  }
-  return false;
-}
-
-bool RegisterContextDarwin_i386::WriteAllRegisterValues(
-    const lldb::DataBufferSP &data_sp) {
-  if (data_sp && data_sp->GetByteSize() == REG_CONTEXT_SIZE) {
-    const uint8_t *src = data_sp->GetBytes();
-    ::memcpy(&gpr, src, sizeof(gpr));
-    src += sizeof(gpr);
-
-    ::memcpy(&fpu, src, sizeof(fpu));
-    src += sizeof(gpr);
-
-    ::memcpy(&exc, src, sizeof(exc));
-    uint32_t success_count = 0;
-    if (WriteGPR() == 0)
-      ++success_count;
-    if (WriteFPU() == 0)
-      ++success_count;
-    if (WriteEXC() == 0)
-      ++success_count;
-    return success_count == 3;
-  }
-  return false;
-}
-
-uint32_t RegisterContextDarwin_i386::ConvertRegisterKindToRegisterNumber(
-    lldb::RegisterKind kind, uint32_t reg) {
-  if (kind == eRegisterKindGeneric) {
-    switch (reg) {
-    case LLDB_REGNUM_GENERIC_PC:
-      return gpr_eip;
-    case LLDB_REGNUM_GENERIC_SP:
-      return gpr_esp;
-    case LLDB_REGNUM_GENERIC_FP:
-      return gpr_ebp;
-    case LLDB_REGNUM_GENERIC_FLAGS:
-      return gpr_eflags;
-    case LLDB_REGNUM_GENERIC_RA:
-    default:
-      break;
-    }
-  } else if (kind == eRegisterKindEHFrame || kind == eRegisterKindDWARF) {
-    switch (reg) {
-    case dwarf_eax:
-      return gpr_eax;
-    case dwarf_ecx:
-      return gpr_ecx;
-    case dwarf_edx:
-      return gpr_edx;
-    case dwarf_ebx:
-      return gpr_ebx;
-    case dwarf_esp:
-      return gpr_esp;
-    case dwarf_ebp:
-      return gpr_ebp;
-    case dwarf_esi:
-      return gpr_esi;
-    case dwarf_edi:
-      return gpr_edi;
-    case dwarf_eip:
-      return gpr_eip;
-    case dwarf_eflags:
-      return gpr_eflags;
-    case dwarf_stmm0:
-      return fpu_stmm0;
-    case dwarf_stmm1:
-      return fpu_stmm1;
-    case dwarf_stmm2:
-      return fpu_stmm2;
-    case dwarf_stmm3:
-      return fpu_stmm3;
-    case dwarf_stmm4:
-      return fpu_stmm4;
-    case dwarf_stmm5:
-      return fpu_stmm5;
-    case dwarf_stmm6:
-      return fpu_stmm6;
-    case dwarf_stmm7:
-      return fpu_stmm7;
-    case dwarf_xmm0:
-      return fpu_xmm0;
-    case dwarf_xmm1:
-      return fpu_xmm1;
-    case dwarf_xmm2:
-      return fpu_xmm2;
-    case dwarf_xmm3:
-      return fpu_xmm3;
-    case dwarf_xmm4:
-      return fpu_xmm4;
-    case dwarf_xmm5:
-      return fpu_xmm5;
-    case dwarf_xmm6:
-      return fpu_xmm6;
-    case dwarf_xmm7:
-      return fpu_xmm7;
-    default:
-      break;
-    }
-  } else if (kind == eRegisterKindLLDB) {
-    return reg;
-  }
-  return LLDB_INVALID_REGNUM;
-}
-
-bool RegisterContextDarwin_i386::HardwareSingleStep(bool enable) {
-  if (ReadGPR(false) != 0)
-    return false;
-
-  const uint32_t trace_bit = 0x100u;
-  if (enable) {
-    // If the trace bit is already set, there is nothing to do
-    if (gpr.eflags & trace_bit)
-      return true;
-    else
-      gpr.eflags |= trace_bit;
-  } else {
-    // If the trace bit is already cleared, there is nothing to do
-    if (gpr.eflags & trace_bit)
-      gpr.eflags &= ~trace_bit;
-    else
-      return true;
-  }
-
-  return WriteGPR() == 0;
-}
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
deleted file mode 100644
index be933f3be266..000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_i386.h
+++ /dev/null
@@ -1,208 +0,0 @@
-//===-- RegisterContextDarwin_i386.h ----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
-
-#include "lldb/Target/RegisterContext.h"
-#include "lldb/lldb-private.h"
-
-class RegisterContextDarwin_i386 : public lldb_private::RegisterContext {
-public:
-  RegisterContextDarwin_i386(lldb_private::Thread &thread,
-                             uint32_t concrete_frame_idx);
-
-  ~RegisterContextDarwin_i386() override;
-
-  void InvalidateAllRegisters() override;
-
-  size_t GetRegisterCount() override;
-
-  const lldb_private::RegisterInfo *GetRegisterInfoAtIndex(size_t reg) override;
-
-  size_t GetRegisterSetCount() override;
-
-  const lldb_private::RegisterSet *GetRegisterSet(size_t set) override;
-
-  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
-                    lldb_private::RegisterValue &value) override;
-
-  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
-                     const lldb_private::RegisterValue &value) override;
-
-  bool ReadAllRegisterValues(lldb::WritableDataBufferSP &data_sp) override;
-
-  bool WriteAllRegisterValues(const lldb::DataBufferSP &data_sp) override;
-
-  uint32_t ConvertRegisterKindToRegisterNumber(lldb::RegisterKind kind,
-                                               uint32_t num) override;
-
-  bool HardwareSingleStep(bool enable) override;
-
-  struct GPR {
-    uint32_t eax;
-    uint32_t ebx;
-    uint32_t ecx;
-    uint32_t edx;
-    uint32_t edi;
-    uint32_t esi;
-    uint32_t ebp;
-    uint32_t esp;
-    uint32_t ss;
-    uint32_t eflags;
-    uint32_t eip;
-    uint32_t cs;
-    uint32_t ds;
-    uint32_t es;
-    uint32_t fs;
-    uint32_t gs;
-  };
-
-  struct MMSReg {
-    uint8_t bytes[10];
-    uint8_t pad[6];
-  };
-
-  struct XMMReg {
-    uint8_t bytes[16];
-  };
-
-  struct FPU {
-    uint32_t pad[2];
-    uint16_t fcw;
-    uint16_t fsw;
-    uint8_t ftw;
-    uint8_t pad1;
-    uint16_t fop;
-    uint32_t ip;
-    uint16_t cs;
-    uint16_t pad2;
-    uint32_t dp;
-    uint16_t ds;
-    uint16_t pad3;
-    uint32_t mxcsr;
-    uint32_t mxcsrmask;
-    MMSReg stmm[8];
-    XMMReg xmm[8];
-    uint8_t pad4[14 * 16];
-    int pad5;
-  };
-
-  struct EXC {
-    uint32_t trapno;
-    uint32_t err;
-    uint32_t faultvaddr;
-  };
-
-protected:
-  enum { GPRRegSet = 1, FPURegSet = 2, EXCRegSet = 3 };
-
-  enum {
-    GPRWordCount = sizeof(GPR) / sizeof(uint32_t),
-    FPUWordCount = sizeof(FPU) / sizeof(uint32_t),
-    EXCWordCount = sizeof(EXC) / sizeof(uint32_t)
-  };
-
-  enum { Read = 0, Write = 1, kNumErrors = 2 };
-
-  GPR gpr;
-  FPU fpu;
-  EXC exc;
-  int gpr_errs[2]; // Read/Write errors
-  int fpu_errs[2]; // Read/Write errors
-  int exc_errs[2]; // Read/Write errors
-
-  void InvalidateAllRegisterStates() {
-    SetError(GPRRegSet, Read, -1);
-    SetError(FPURegSet, Read, -1);
-    SetError(EXCRegSet, Read, -1);
-  }
-
-  int GetError(int flavor, uint32_t err_idx) const {
-    if (err_idx < kNumErrors) {
-      switch (flavor) {
-      // When getting all errors, just OR all values together to see if
-      // we got any kind of error.
-      case GPRRegSet:
-        return gpr_errs[err_idx];
-      case FPURegSet:
-        return fpu_errs[err_idx];
-      case EXCRegSet:
-        return exc_errs[err_idx];
-      default:
-        break;
-      }
-    }
-    return -1;
-  }
-
-  bool SetError(int flavor, uint32_t err_idx, int err) {
-    if (err_idx < kNumErrors) {
-      switch (flavor) {
-      case GPRRegSet:
-        gpr_errs[err_idx] = err;
-        return true;
-
-      case FPURegSet:
-        fpu_errs[err_idx] = err;
-        return true;
-
-      case EXCRegSet:
-        exc_errs[err_idx] = err;
-        return true;
-
-      default:
-        break;
-      }
-    }
-    return false;
-  }
-
-  bool RegisterSetIsCached(int set) const { return GetError(set, Read) == 0; }
-
-  void LogGPR(lldb_private::Log *log, const char *title);
-
-  int ReadGPR(bool force);
-
-  int ReadFPU(bool force);
-
-  int ReadEXC(bool force);
-
-  int WriteGPR();
-
-  int WriteFPU();
-
-  int WriteEXC();
-
-  // Subclasses override these to do the actual reading.
-  virtual int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) = 0;
-
-  virtual int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) = 0;
-
-  virtual int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) = 0;
-
-  virtual int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) = 0;
-
-  virtual int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) = 0;
-
-  virtual int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) = 0;
-
-  int ReadRegisterSet(uint32_t set, bool force);
-
-  int WriteRegisterSet(uint32_t set);
-
-  static uint32_t GetRegisterNumber(uint32_t reg_kind, uint32_t reg_num);
-
-  static int GetSetForNativeRegNum(int reg_num);
-
-  static size_t GetRegisterInfosCount();
-
-  static const lldb_private::RegisterInfo *GetRegisterInfos();
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTDARWIN_I386_H
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
deleted file mode 100644
index fe5cecef1b0c..000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- RegisterContextMach_i386.cpp --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#if defined(__APPLE__)
-
-#include <mach/thread_act.h>
-
-#include "RegisterContextMach_i386.h"
-
-using namespace lldb;
-using namespace lldb_private;
-
-RegisterContextMach_i386::RegisterContextMach_i386(Thread &thread,
-                                                   uint32_t concrete_frame_idx)
-    : RegisterContextDarwin_i386(thread, concrete_frame_idx) {}
-
-RegisterContextMach_i386::~RegisterContextMach_i386() = default;
-
-int RegisterContextMach_i386::DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) {
-  mach_msg_type_number_t count = GPRWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&gpr, &count);
-}
-
-int RegisterContextMach_i386::DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) {
-  mach_msg_type_number_t count = FPUWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&fpu, &count);
-}
-
-int RegisterContextMach_i386::DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) {
-  mach_msg_type_number_t count = EXCWordCount;
-  return ::thread_get_state(tid, flavor, (thread_state_t)&exc, &count);
-}
-
-int RegisterContextMach_i386::DoWriteGPR(lldb::tid_t tid, int flavor,
-                                         const GPR &gpr) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<GPR *>(&gpr)),
-      GPRWordCount);
-}
-
-int RegisterContextMach_i386::DoWriteFPU(lldb::tid_t tid, int flavor,
-                                         const FPU &fpu) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<FPU *>(&fpu)),
-      FPUWordCount);
-}
-
-int RegisterContextMach_i386::DoWriteEXC(lldb::tid_t tid, int flavor,
-                                         const EXC &exc) {
-  return ::thread_set_state(
-      tid, flavor, reinterpret_cast<thread_state_t>(const_cast<EXC *>(&exc)),
-      EXCWordCount);
-}
-
-#endif
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h b/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
deleted file mode 100644
index 8bdac083863d..000000000000
--- a/lldb/source/Plugins/Process/Utility/RegisterContextMach_i386.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===-- RegisterContextMach_i386.h ------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
-#define LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
-
-#include "RegisterContextDarwin_i386.h"
-
-class RegisterContextMach_i386 : public RegisterContextDarwin_i386 {
-public:
-  RegisterContextMach_i386(lldb_private::Thread &thread,
-                           uint32_t concrete_frame_idx);
-
-  ~RegisterContextMach_i386() override;
-
-protected:
-  int DoReadGPR(lldb::tid_t tid, int flavor, GPR &gpr) override;
-
-  int DoReadFPU(lldb::tid_t tid, int flavor, FPU &fpu) override;
-
-  int DoReadEXC(lldb::tid_t tid, int flavor, EXC &exc) override;
-
-  int DoWriteGPR(lldb::tid_t tid, int flavor, const GPR &gpr) override;
-
-  int DoWriteFPU(lldb::tid_t tid, int flavor, const FPU &fpu) override;
-
-  int DoWriteEXC(lldb::tid_t tid, int flavor, const EXC &exc) override;
-};
-
-#endif // LLDB_SOURCE_PLUGINS_PROCESS_UTILITY_REGISTERCONTEXTMACH_I386_H
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.cpp
index 394b62559da7..406fa06ea011 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.cpp
@@ -180,7 +180,7 @@ bool GDBRemoteClientBase::Interrupt(std::chrono::seconds interrupt_timeout) {
 GDBRemoteCommunication::PacketResult
 GDBRemoteClientBase::SendPacketAndWaitForResponse(
     llvm::StringRef payload, StringExtractorGDBRemote &response,
-    std::chrono::seconds interrupt_timeout) {
+    std::chrono::seconds interrupt_timeout, bool sync_on_timeout) {
   Lock lock(*this, interrupt_timeout);
   if (!lock) {
     if (Log *log = GetLog(GDBRLog::Process))
@@ -191,7 +191,7 @@ GDBRemoteClientBase::SendPacketAndWaitForResponse(
     return PacketResult::ErrorSendFailed;
   }
 
-  return SendPacketAndWaitForResponseNoLock(payload, response);
+  return SendPacketAndWaitForResponseNoLock(payload, response, sync_on_timeout);
 }
 
 GDBRemoteCommunication::PacketResult
@@ -236,14 +236,15 @@ GDBRemoteClientBase::SendPacketAndReceiveResponseWithOutputSupport(
 
 GDBRemoteCommunication::PacketResult
 GDBRemoteClientBase::SendPacketAndWaitForResponseNoLock(
-    llvm::StringRef payload, StringExtractorGDBRemote &response) {
+    llvm::StringRef payload, StringExtractorGDBRemote &response,
+    bool sync_on_timeout) {
   PacketResult packet_result = SendPacketNoLock(payload);
   if (packet_result != PacketResult::Success)
     return packet_result;
 
   const size_t max_response_retries = 3;
   for (size_t i = 0; i < max_response_retries; ++i) {
-    packet_result = ReadPacket(response, GetPacketTimeout(), true);
+    packet_result = ReadPacket(response, GetPacketTimeout(), sync_on_timeout);
     // Make sure we received a response
     if (packet_result != PacketResult::Success)
       return packet_result;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.h
index af2abdf4da5c..9c17a8c1de05 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteClientBase.h
@@ -61,7 +61,8 @@ class GDBRemoteClientBase : public GDBRemoteCommunication, public Broadcaster {
   // ErrorReplyTimeout.
   PacketResult SendPacketAndWaitForResponse(
       llvm::StringRef payload, StringExtractorGDBRemote &response,
-      std::chrono::seconds interrupt_timeout = std::chrono::seconds(0));
+      std::chrono::seconds interrupt_timeout = std::chrono::seconds(0),
+      bool sync_on_timeout = true);
 
   PacketResult ReadPacketWithOutputSupport(
       StringExtractorGDBRemote &response, Timeout<std::micro> timeout,
@@ -104,7 +105,8 @@ class GDBRemoteClientBase : public GDBRemoteCommunication, public Broadcaster {
 protected:
   PacketResult
   SendPacketAndWaitForResponseNoLock(llvm::StringRef payload,
-                                     StringExtractorGDBRemote &response);
+                                     StringExtractorGDBRemote &response,
+                                     bool sync_on_timeout = true);
 
   virtual void OnRunPacketSent(bool first);
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index 0d3ead840b08..d7e4b2b9546b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -11,7 +11,6 @@
 #include "lldb/Host/Config.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/Host.h"
-#include "lldb/Host/HostInfo.h"
 #include "lldb/Host/Pipe.h"
 #include "lldb/Host/ProcessLaunchInfo.h"
 #include "lldb/Host/Socket.h"
@@ -31,14 +30,7 @@
 #include <climits>
 #include <cstring>
 #include <sys/stat.h>
-
-#if defined(__APPLE__)
-#define DEBUGSERVER_BASENAME "debugserver"
-#elif defined(_WIN32)
-#define DEBUGSERVER_BASENAME "lldb-server.exe"
-#else
-#define DEBUGSERVER_BASENAME "lldb-server"
-#endif
+#include <variant>
 
 #if HAVE_LIBCOMPRESSION
 #include <compression.h>
@@ -354,8 +346,9 @@ GDBRemoteCommunication::WaitForPacketNoLock(StringExtractorGDBRemote &packet,
             disconnected = true;
             Disconnect();
           }
+        } else {
+          timed_out = true;
         }
-        timed_out = true;
         break;
       case eConnectionStatusSuccess:
         // printf ("status = success but error = %s\n",
@@ -834,166 +827,67 @@ GDBRemoteCommunication::CheckForPacket(const uint8_t *src, size_t src_len,
   return GDBRemoteCommunication::PacketType::Invalid;
 }
 
-FileSpec GDBRemoteCommunication::GetDebugserverPath(Platform *platform) {
-  Log *log = GetLog(GDBRLog::Process);
-  // If we locate debugserver, keep that located version around
-  static FileSpec g_debugserver_file_spec;
-  FileSpec debugserver_file_spec;
-
-  Environment host_env = Host::GetEnvironment();
-
-  // Always check to see if we have an environment override for the path to the
-  // debugserver to use and use it if we do.
-  std::string env_debugserver_path = host_env.lookup("LLDB_DEBUGSERVER_PATH");
-  if (!env_debugserver_path.empty()) {
-    debugserver_file_spec.SetFile(env_debugserver_path,
-                                  FileSpec::Style::native);
-    LLDB_LOGF(log,
-              "GDBRemoteCommunication::%s() gdb-remote stub exe path set "
-              "from environment variable: %s",
-              __FUNCTION__, env_debugserver_path.c_str());
-  } else
-    debugserver_file_spec = g_debugserver_file_spec;
-  bool debugserver_exists =
-      FileSystem::Instance().Exists(debugserver_file_spec);
-  if (!debugserver_exists) {
-    // The debugserver binary is in the LLDB.framework/Resources directory.
-    debugserver_file_spec = HostInfo::GetSupportExeDir();
-    if (debugserver_file_spec) {
-      debugserver_file_spec.AppendPathComponent(DEBUGSERVER_BASENAME);
-      debugserver_exists = FileSystem::Instance().Exists(debugserver_file_spec);
-      if (debugserver_exists) {
-        LLDB_LOGF(log,
-                  "GDBRemoteCommunication::%s() found gdb-remote stub exe '%s'",
-                  __FUNCTION__, debugserver_file_spec.GetPath().c_str());
-
-        g_debugserver_file_spec = debugserver_file_spec;
-      } else {
-        if (platform)
-          debugserver_file_spec =
-              platform->LocateExecutable(DEBUGSERVER_BASENAME);
-        else
-          debugserver_file_spec.Clear();
-        if (debugserver_file_spec) {
-          // Platform::LocateExecutable() wouldn't return a path if it doesn't
-          // exist
-          debugserver_exists = true;
-        } else {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunication::%s() could not find "
-                    "gdb-remote stub exe '%s'",
-                    __FUNCTION__, debugserver_file_spec.GetPath().c_str());
-        }
-        // Don't cache the platform specific GDB server binary as it could
-        // change from platform to platform
-        g_debugserver_file_spec.Clear();
-      }
-    }
-  }
-  return debugserver_file_spec;
-}
-
 Status GDBRemoteCommunication::StartDebugserverProcess(
-    const char *url, Platform *platform, ProcessLaunchInfo &launch_info,
-    uint16_t *port, const Args *inferior_args, shared_fd_t pass_comm_fd) {
+    std::variant<llvm::StringRef, shared_fd_t> comm,
+    ProcessLaunchInfo &launch_info, const Args *inferior_args) {
   Log *log = GetLog(GDBRLog::Process);
-  LLDB_LOG(log, "Starting debug server: url={0}, port={1}",
-           url ? url : "<empty>", port ? *port : uint16_t(0));
-
-  FileSpec debugserver_file_spec = GetDebugserverPath(platform);
-  if (!debugserver_file_spec)
-    return Status::FromErrorString("unable to locate " DEBUGSERVER_BASENAME);
-
-  launch_info.SetExecutableFile(debugserver_file_spec,
-                                /*add_exe_file_as_first_arg=*/true);
 
   Args &debugserver_args = launch_info.GetArguments();
 
 #if !defined(__APPLE__)
   // First argument to lldb-server must be mode in which to run.
-  debugserver_args.AppendArgument(llvm::StringRef("gdbserver"));
+  debugserver_args.AppendArgument("gdbserver");
 #endif
 
-  // If a url is supplied then use it
-  if (url && url[0])
-    debugserver_args.AppendArgument(llvm::StringRef(url));
-
-  if (pass_comm_fd != SharedSocket::kInvalidFD) {
-    StreamString fd_arg;
-    fd_arg.Printf("--fd=%" PRIi64, (int64_t)pass_comm_fd);
-    debugserver_args.AppendArgument(fd_arg.GetString());
-    // Send "pass_comm_fd" down to the inferior so it can use it to
-    // communicate back with this process. Ignored on Windows.
-    launch_info.AppendDuplicateFileAction((int64_t)pass_comm_fd,
-                                          (int64_t)pass_comm_fd);
-  }
-
   // use native registers, not the GDB registers
-  debugserver_args.AppendArgument(llvm::StringRef("--native-regs"));
+  debugserver_args.AppendArgument("--native-regs");
 
   if (launch_info.GetLaunchInSeparateProcessGroup())
-    debugserver_args.AppendArgument(llvm::StringRef("--setsid"));
+    debugserver_args.AppendArgument("--setsid");
 
   llvm::SmallString<128> named_pipe_path;
   // socket_pipe is used by debug server to communicate back either
-  // TCP port or domain socket name which it listens on.
-  // The second purpose of the pipe to serve as a synchronization point -
+  // TCP port or domain socket name which it listens on. However, we're not
+  // interested in the actualy value here.
+  // The only reason for using the pipe is to serve as a synchronization point -
   // once data is written to the pipe, debug server is up and running.
   Pipe socket_pipe;
 
-  std::unique_ptr<TCPSocket> sock_up;
+  // If a url is supplied then use it
+  if (shared_fd_t *comm_fd = std::get_if<shared_fd_t>(&comm)) {
+    LLDB_LOG(log, "debugserver communicates over fd {0}", comm_fd);
+    assert(*comm_fd != SharedSocket::kInvalidFD);
+    debugserver_args.AppendArgument(llvm::formatv("--fd={0}", *comm_fd).str());
+    // Send "comm_fd" down to the inferior so it can use it to communicate back
+    // with this process.
+    launch_info.AppendDuplicateFileAction((int64_t)*comm_fd, (int64_t)*comm_fd);
+  } else {
+    llvm::StringRef url = std::get<llvm::StringRef>(comm);
+    LLDB_LOG(log, "debugserver listens on: {0}", url);
+    debugserver_args.AppendArgument(url);
 
-  // port is null when debug server should listen on domain socket - we're
-  // not interested in port value but rather waiting for debug server to
-  // become available.
-  if (pass_comm_fd == SharedSocket::kInvalidFD) {
-    if (url) {
-// Create a temporary file to get the stdout/stderr and redirect the output of
-// the command into this file. We will later read this file if all goes well
-// and fill the data into "command_output_ptr"
 #if defined(__APPLE__)
-      // Binding to port zero, we need to figure out what port it ends up
-      // using using a named pipe...
-      Status error = socket_pipe.CreateWithUniqueName("debugserver-named-pipe",
-                                                      false, named_pipe_path);
-      if (error.Fail()) {
-        LLDB_LOG(log, "named pipe creation failed: {0}", error);
-        return error;
-      }
-      debugserver_args.AppendArgument(llvm::StringRef("--named-pipe"));
-      debugserver_args.AppendArgument(named_pipe_path);
+    // Using a named pipe as debugserver does not support --pipe.
+    Status error = socket_pipe.CreateWithUniqueName("debugserver-named-pipe",
+                                                    named_pipe_path);
+    if (error.Fail()) {
+      LLDB_LOG(log, "named pipe creation failed: {0}", error);
+      return error;
+    }
+    debugserver_args.AppendArgument(llvm::StringRef("--named-pipe"));
+    debugserver_args.AppendArgument(named_pipe_path);
 #else
-      // Binding to port zero, we need to figure out what port it ends up
-      // using using an unnamed pipe...
-      Status error = socket_pipe.CreateNew(true);
-      if (error.Fail()) {
-        LLDB_LOG(log, "unnamed pipe creation failed: {0}", error);
-        return error;
-      }
-      pipe_t write = socket_pipe.GetWritePipe();
-      debugserver_args.AppendArgument(llvm::StringRef("--pipe"));
-      debugserver_args.AppendArgument(llvm::to_string(write));
-      launch_info.AppendCloseFileAction(socket_pipe.GetReadFileDescriptor());
-#endif
-    } else {
-      // No host and port given, so lets listen on our end and make the
-      // debugserver connect to us..
-      if (llvm::Expected<std::unique_ptr<TCPSocket>> expected_sock =
-              Socket::TcpListen("127.0.0.1:0"))
-        sock_up = std::move(*expected_sock);
-      else
-        return Status::FromError(expected_sock.takeError());
-
-      uint16_t port_ = sock_up->GetLocalPortNumber();
-      // Send the host and port down that debugserver and specify an option
-      // so that it connects back to the port we are listening to in this
-      // process
-      debugserver_args.AppendArgument(llvm::StringRef("--reverse-connect"));
-      debugserver_args.AppendArgument(
-          llvm::formatv("127.0.0.1:{0}", port_).str());
-      if (port)
-        *port = port_;
+    // Using an unnamed pipe as it's simpler.
+    Status error = socket_pipe.CreateNew();
+    if (error.Fail()) {
+      LLDB_LOG(log, "unnamed pipe creation failed: {0}", error);
+      return error;
     }
+    pipe_t write = socket_pipe.GetWritePipe();
+    debugserver_args.AppendArgument(llvm::StringRef("--pipe"));
+    debugserver_args.AppendArgument(llvm::to_string(write));
+    launch_info.AppendDuplicateFileAction((int64_t)write, (int64_t)write);
+#endif
   }
 
   Environment host_env = Host::GetEnvironment();
@@ -1070,12 +964,12 @@ Status GDBRemoteCommunication::StartDebugserverProcess(
     return error;
   }
 
-  if (pass_comm_fd != SharedSocket::kInvalidFD)
+  if (std::holds_alternative<shared_fd_t>(comm))
     return Status();
 
   Status error;
   if (named_pipe_path.size() > 0) {
-    error = socket_pipe.OpenAsReader(named_pipe_path, false);
+    error = socket_pipe.OpenAsReader(named_pipe_path);
     if (error.Fail()) {
       LLDB_LOG(log, "failed to open named pipe {0} for reading: {1}",
                named_pipe_path, error);
@@ -1084,55 +978,30 @@ Status GDBRemoteCommunication::StartDebugserverProcess(
 
   if (socket_pipe.CanWrite())
     socket_pipe.CloseWriteFileDescriptor();
-  if (socket_pipe.CanRead()) {
-    // Read port from pipe with 10 second timeout.
-    std::string port_str;
-    while (error.Success()) {
-      char buf[10];
-      if (llvm::Expected<size_t> num_bytes =
-              socket_pipe.Read(buf, std::size(buf), std::chrono::seconds(10))) {
-        if (*num_bytes == 0)
-          break;
-        port_str.append(buf, *num_bytes);
-      } else {
-        error = Status::FromError(num_bytes.takeError());
-      }
-    }
-    if (error.Success() && (port != nullptr)) {
-      // NB: Deliberately using .c_str() to stop at embedded '\0's
-      llvm::StringRef port_ref = port_str.c_str();
-      uint16_t child_port = 0;
-      // FIXME: improve error handling
-      llvm::to_integer(port_ref, child_port);
-      if (*port == 0 || *port == child_port) {
-        *port = child_port;
-        LLDB_LOG(log, "debugserver listens on port {0}", *port);
-      } else {
-        LLDB_LOG(log,
-                 "debugserver listening on port {0} but requested port was {1}",
-                 child_port, (*port));
-      }
+  assert(socket_pipe.CanRead());
+
+  // Read data from the pipe -- and ignore it (see comment above).
+  while (error.Success()) {
+    char buf[10];
+    if (llvm::Expected<size_t> num_bytes =
+            socket_pipe.Read(buf, std::size(buf), std::chrono::seconds(10))) {
+      if (*num_bytes == 0)
+        break;
     } else {
-      LLDB_LOG(log, "failed to read a port value from pipe {0}: {1}",
-               named_pipe_path, error);
+      error = Status::FromError(num_bytes.takeError());
     }
-    socket_pipe.Close();
   }
+  if (error.Fail()) {
+    LLDB_LOG(log, "failed to synchronize on pipe {0}: {1}", named_pipe_path,
+             error);
+  }
+  socket_pipe.Close();
 
   if (named_pipe_path.size() > 0) {
     if (Status err = socket_pipe.Delete(named_pipe_path); err.Fail())
       LLDB_LOG(log, "failed to delete pipe {0}: {1}", named_pipe_path, err);
   }
 
-  if (error.Success() && sock_up) {
-    Socket *accepted_socket = nullptr;
-    error = sock_up->Accept(/*timeout=*/std::nullopt, accepted_socket);
-    if (accepted_socket) {
-      SetConnection(std::make_unique<ConnectionFileDescriptor>(
-          std::unique_ptr<Socket>(accepted_socket)));
-    }
-  }
-
   return error;
 }
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
index fc86f801f0d8..35bf5eb2e3f0 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.h
@@ -134,18 +134,12 @@ class GDBRemoteCommunication : public Communication {
 
   std::chrono::seconds GetPacketTimeout() const { return m_packet_timeout; }
 
-  // Get the debugserver path and check that it exist.
-  FileSpec GetDebugserverPath(Platform *platform);
-
   // Start a debugserver instance on the current host using the
   // supplied connection URL.
-  Status StartDebugserverProcess(
-      const char *url,
-      Platform *platform, // If non nullptr, then check with the platform for
-                          // the GDB server binary if it can't be located
-      ProcessLaunchInfo &launch_info, uint16_t *port, const Args *inferior_args,
-      shared_fd_t pass_comm_fd); // Communication file descriptor to pass during
-                                 // fork/exec to avoid having to connect/accept
+  static Status
+  StartDebugserverProcess(std::variant<llvm::StringRef, shared_fd_t> comm,
+                          ProcessLaunchInfo &launch_info,
+                          const Args *inferior_args);
 
   void DumpHistory(Stream &strm);
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index adbf06b9a19a..7d2bd452acca 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -406,7 +406,7 @@ void GDBRemoteCommunicationClient::GetRemoteQSupported() {
         m_supports_qXfer_memory_map_read = eLazyBoolYes;
       else if (x == "qXfer:siginfo:read+")
         m_supports_qXfer_siginfo_read = eLazyBoolYes;
-      else if (x == "qEcho")
+      else if (x == "qEcho+")
         m_supports_qEcho = eLazyBoolYes;
       else if (x == "QPassSignals+")
         m_supports_QPassSignals = eLazyBoolYes;
@@ -4358,7 +4358,9 @@ llvm::Expected<int> GDBRemoteCommunicationClient::KillProcess(lldb::pid_t pid) {
   StringExtractorGDBRemote response;
   GDBRemoteCommunication::ScopedTimeout(*this, seconds(3));
 
-  if (SendPacketAndWaitForResponse("k", response, GetPacketTimeout()) !=
+  // LLDB server typically sends no response for "k", so we shouldn't try
+  // to sync on timeout.
+  if (SendPacketAndWaitForResponse("k", response, GetPacketTimeout(), false) !=
       PacketResult::Success)
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "failed to send k packet");
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
index 89fdfa74bc02..5876c3a9434a 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
@@ -46,9 +46,10 @@ using namespace lldb_private;
 
 // GDBRemoteCommunicationServerPlatform constructor
 GDBRemoteCommunicationServerPlatform::GDBRemoteCommunicationServerPlatform(
-    const Socket::SocketProtocol socket_protocol, uint16_t gdbserver_port)
-    : GDBRemoteCommunicationServerCommon(), m_socket_protocol(socket_protocol),
-      m_gdbserver_port(gdbserver_port) {
+    FileSpec debugserver_path, const Socket::SocketProtocol socket_protocol,
+    uint16_t gdbserver_port)
+    : m_debugserver_path(std::move(debugserver_path)),
+      m_socket_protocol(socket_protocol), m_gdbserver_port(gdbserver_port) {
 
   RegisterMemberFunctionHandler(
       StringExtractorGDBRemote::eServerPacketType_qC,
@@ -94,41 +95,40 @@ GDBRemoteCommunicationServerPlatform::~GDBRemoteCommunicationServerPlatform() =
 Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
     const lldb_private::Args &args, lldb::pid_t &pid, std::string &socket_name,
     shared_fd_t fd) {
-  std::ostringstream url;
+  Log *log = GetLog(LLDBLog::Platform);
+
+  ProcessLaunchInfo debugserver_launch_info;
+  // Do not run in a new session so that it can not linger after the platform
+  // closes.
+  debugserver_launch_info.SetLaunchInSeparateProcessGroup(false);
+  debugserver_launch_info.SetMonitorProcessCallback(
+      [](lldb::pid_t, int, int) {});
+  if (!FileSystem::Instance().Exists(m_debugserver_path))
+    return Status::FromErrorString("debugserver does not exist");
+  debugserver_launch_info.SetExecutableFile(m_debugserver_path,
+                                            /*add_exe_file_as_first_arg=*/true);
+
+  Status error;
   if (fd == SharedSocket::kInvalidFD) {
     if (m_socket_protocol == Socket::ProtocolTcp) {
-      // Just check that GDBServer exists. GDBServer must be launched after
-      // accepting the connection.
-      if (!GetDebugserverPath(nullptr))
-        return Status::FromErrorString("unable to locate debugserver");
+      // The server will be launched after accepting the connection.
       return Status();
     }
 
+    std::ostringstream url;
     // debugserver does not accept the URL scheme prefix.
 #if !defined(__APPLE__)
     url << Socket::FindSchemeByProtocol(m_socket_protocol) << "://";
 #endif
     socket_name = GetDomainSocketPath("gdbserver").GetPath();
     url << socket_name;
+    error = StartDebugserverProcess(url.str(), debugserver_launch_info, &args);
   } else {
     if (m_socket_protocol != Socket::ProtocolTcp)
       return Status::FromErrorString("protocol must be tcp");
+    error = StartDebugserverProcess(fd, debugserver_launch_info, &args);
   }
 
-  // Spawn a debugserver and try to get the port it listens to.
-  ProcessLaunchInfo debugserver_launch_info;
-  Log *log = GetLog(LLDBLog::Platform);
-  LLDB_LOG(log, "Launching debugserver url='{0}', fd={1}...", url.str(), fd);
-
-  // Do not run in a new session so that it can not linger after the platform
-  // closes.
-  debugserver_launch_info.SetLaunchInSeparateProcessGroup(false);
-  debugserver_launch_info.SetMonitorProcessCallback(
-      [](lldb::pid_t, int, int) {});
-
-  Status error = StartDebugserverProcess(
-      url.str().c_str(), nullptr, debugserver_launch_info, nullptr, &args, fd);
-
   if (error.Success()) {
     pid = debugserver_launch_info.GetProcessID();
     AddSpawnedProcess(pid);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
index beee61ea8048..bcfb6d3f3ed0 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
@@ -26,7 +26,8 @@ class GDBRemoteCommunicationServerPlatform
     : public GDBRemoteCommunicationServerCommon {
 public:
   GDBRemoteCommunicationServerPlatform(
-      const Socket::SocketProtocol socket_protocol, uint16_t gdbserver_port);
+      FileSpec debugserver_path, const Socket::SocketProtocol socket_protocol,
+      uint16_t gdbserver_port);
 
   ~GDBRemoteCommunicationServerPlatform() override;
 
@@ -40,6 +41,7 @@ class GDBRemoteCommunicationServerPlatform
   void SetPendingGdbServer(const std::string &socket_name);
 
 protected:
+  const FileSpec m_debugserver_path;
   const Socket::SocketProtocol m_socket_protocol;
   std::recursive_mutex m_spawned_pids_mutex;
   std::set<lldb::pid_t> m_spawned_pids;
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 4e70fe8ac159..a2c34ddfc252 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -34,6 +34,7 @@
 #include "lldb/DataFormatters/FormatManager.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
 #include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
 #include "lldb/Host/HostThread.h"
 #include "lldb/Host/PosixApi.h"
 #include "lldb/Host/PseudoTerminal.h"
@@ -92,7 +93,14 @@
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
 
+#if defined(__APPLE__)
 #define DEBUGSERVER_BASENAME "debugserver"
+#elif defined(_WIN32)
+#define DEBUGSERVER_BASENAME "lldb-server.exe"
+#else
+#define DEBUGSERVER_BASENAME "lldb-server"
+#endif
+
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
@@ -3448,6 +3456,51 @@ ProcessGDBRemote::EstablishConnectionIfNeeded(const ProcessInfo &process_info) {
   return error;
 }
 
+static FileSpec GetDebugserverPath(Platform &platform) {
+  Log *log = GetLog(GDBRLog::Process);
+  // If we locate debugserver, keep that located version around
+  static FileSpec g_debugserver_file_spec;
+  FileSpec debugserver_file_spec;
+
+  Environment host_env = Host::GetEnvironment();
+
+  // Always check to see if we have an environment override for the path to the
+  // debugserver to use and use it if we do.
+  std::string env_debugserver_path = host_env.lookup("LLDB_DEBUGSERVER_PATH");
+  if (!env_debugserver_path.empty()) {
+    debugserver_file_spec.SetFile(env_debugserver_path,
+                                  FileSpec::Style::native);
+    LLDB_LOG(log, "gdb-remote stub exe path set from environment variable: {0}",
+             env_debugserver_path);
+  } else
+    debugserver_file_spec = g_debugserver_file_spec;
+  if (FileSystem::Instance().Exists(debugserver_file_spec))
+    return debugserver_file_spec;
+
+  // The debugserver binary is in the LLDB.framework/Resources directory.
+  debugserver_file_spec = HostInfo::GetSupportExeDir();
+  if (debugserver_file_spec) {
+    debugserver_file_spec.AppendPathComponent(DEBUGSERVER_BASENAME);
+    if (FileSystem::Instance().Exists(debugserver_file_spec)) {
+      LLDB_LOG(log, "found gdb-remote stub exe '{0}'", debugserver_file_spec);
+
+      g_debugserver_file_spec = debugserver_file_spec;
+    } else {
+      debugserver_file_spec = platform.LocateExecutable(DEBUGSERVER_BASENAME);
+      if (!debugserver_file_spec) {
+        // Platform::LocateExecutable() wouldn't return a path if it doesn't
+        // exist
+        LLDB_LOG(log, "could not find gdb-remote stub exe '{0}'",
+                 debugserver_file_spec);
+      }
+      // Don't cache the platform specific GDB server binary as it could
+      // change from platform to platform
+      g_debugserver_file_spec.Clear();
+    }
+  }
+  return debugserver_file_spec;
+}
+
 Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
     const ProcessInfo &process_info) {
   using namespace std::placeholders; // For _1, _2, etc.
@@ -3466,6 +3519,8 @@ Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
       std::bind(MonitorDebugserverProcess, this_wp, _1, _2, _3));
   debugserver_launch_info.SetUserID(process_info.GetUserID());
 
+  FileSpec debugserver_path = GetDebugserverPath(*GetTarget().GetPlatform());
+
 #if defined(__APPLE__)
   // On macOS 11, we need to support x86_64 applications translated to
   // arm64. We check whether a binary is translated and spawn the correct
@@ -3478,12 +3533,12 @@ Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
              NULL, 0) == 0 &&
       bufsize > 0) {
     if (processInfo.kp_proc.p_flag & P_TRANSLATED) {
-      FileSpec rosetta_debugserver(
-          "/Library/Apple/usr/libexec/oah/debugserver");
-      debugserver_launch_info.SetExecutableFile(rosetta_debugserver, false);
+      debugserver_path = FileSpec("/Library/Apple/usr/libexec/oah/debugserver");
     }
   }
 #endif
+  debugserver_launch_info.SetExecutableFile(debugserver_path,
+                                            /*add_exe_file_as_first_arg=*/true);
 
   llvm::Expected<Socket::Pair> socket_pair = Socket::CreatePair();
   if (!socket_pair)
@@ -3494,9 +3549,8 @@ Status ProcessGDBRemote::LaunchAndConnectToDebugserver(
   if (error.Fail())
     return error;
 
-  error = m_gdb_comm.StartDebugserverProcess(
-      nullptr, GetTarget().GetPlatform().get(), debugserver_launch_info,
-      nullptr, nullptr, shared_socket.GetSendableFD());
+  error = m_gdb_comm.StartDebugserverProcess(shared_socket.GetSendableFD(),
+                                             debugserver_launch_info, nullptr);
 
   if (error.Fail()) {
     Log *log = GetLog(GDBRLog::Process);
diff --git a/lldb/source/Plugins/Process/mach-core/CMakeLists.txt b/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
index a1ea85ec4c72..926f06d23c64 100644
--- a/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/mach-core/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_library(lldbPluginProcessMachCore PLUGIN
   ProcessMachCore.cpp
   ThreadMachCore.cpp
+  RegisterContextUnifiedCore.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp
new file mode 100644
index 000000000000..449f85bc3f52
--- /dev/null
+++ b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.cpp
@@ -0,0 +1,312 @@
+//===-- RegisterContextUnifiedCore.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RegisterContextUnifiedCore.h"
+#include "lldb/Target/DynamicRegisterInfo.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Utility/DataExtractor.h"
+#include "lldb/Utility/RegisterValue.h"
+#include "lldb/Utility/StructuredData.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+RegisterContextUnifiedCore::RegisterContextUnifiedCore(
+    Thread &thread, uint32_t concrete_frame_idx,
+    RegisterContextSP core_thread_regctx_sp,
+    StructuredData::ObjectSP metadata_thread_registers)
+    : RegisterContext(thread, concrete_frame_idx) {
+
+  ProcessSP process_sp(thread.GetProcess());
+  Target &target = process_sp->GetTarget();
+  StructuredData::Dictionary *metadata_registers_dict = nullptr;
+
+  // If we have thread metadata, check if the keys for register
+  // definitions are present; if not, clear the ObjectSP.
+  if (metadata_thread_registers &&
+      metadata_thread_registers->GetAsDictionary() &&
+      metadata_thread_registers->GetAsDictionary()->HasKey("register_info")) {
+    metadata_registers_dict = metadata_thread_registers->GetAsDictionary()
+                                  ->GetValueForKey("register_info")
+                                  ->GetAsDictionary();
+    if (metadata_registers_dict)
+      if (!metadata_registers_dict->HasKey("sets") ||
+          !metadata_registers_dict->HasKey("registers"))
+        metadata_registers_dict = nullptr;
+  }
+
+  // When creating a register set list from the two sources,
+  // the LC_THREAD aka core_thread_regctx_sp register sets
+  // will be used at the same indexes.
+  // Any additional sets named by the thread metadata registers
+  // will be added after them.  If the thread metadata
+  // specify a set with the same name as LC_THREAD, the already-used
+  // index from the core register context will be used in
+  // the RegisterInfo.
+  std::map<size_t, size_t> metadata_regset_to_combined_regset;
+
+  // Calculate the total size of the register store buffer we need
+  // for all registers.  The corefile register definitions may include
+  // RegisterInfo descriptions of registers that aren't actually
+  // available.  For simplicity, calculate the size of all registers
+  // as if they are available, so we can maintain the same offsets into
+  // the buffer.
+  uint32_t core_buffer_end = 0;
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterCount(); idx++) {
+    const RegisterInfo *reginfo =
+        core_thread_regctx_sp->GetRegisterInfoAtIndex(idx);
+    core_buffer_end =
+        std::max(reginfo->byte_offset + reginfo->byte_size, core_buffer_end);
+  }
+
+  // Add metadata register sizes to the total buffer size.
+  uint32_t combined_buffer_end = core_buffer_end;
+  if (metadata_registers_dict) {
+    StructuredData::Array *registers = nullptr;
+    if (metadata_registers_dict->GetValueForKeyAsArray("registers", registers))
+      registers->ForEach(
+          [&combined_buffer_end](StructuredData::Object *ent) -> bool {
+            uint32_t bitsize;
+            if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("bitsize",
+                                                                 bitsize))
+              return false;
+            combined_buffer_end += (bitsize / 8);
+            return true;
+          });
+  }
+  m_register_data.resize(combined_buffer_end, 0);
+
+  // Copy the core register values into our combined data buffer,
+  // skip registers that are contained within another (e.g. w0 vs. x0)
+  // and registers that return as "unavailable".
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterCount(); idx++) {
+    const RegisterInfo *reginfo =
+        core_thread_regctx_sp->GetRegisterInfoAtIndex(idx);
+    RegisterValue val;
+    if (!reginfo->value_regs &&
+        core_thread_regctx_sp->ReadRegister(reginfo, val))
+      memcpy(m_register_data.data() + reginfo->byte_offset, val.GetBytes(),
+             val.GetByteSize());
+  }
+
+  // Set 'offset' fields for each register definition into our combined
+  // register data buffer. DynamicRegisterInfo needs this field set to
+  // parse the JSON.
+  // Also copy the values of the registers into our register data buffer.
+  if (metadata_registers_dict) {
+    size_t offset = core_buffer_end;
+    ByteOrder byte_order = core_thread_regctx_sp->GetByteOrder();
+    StructuredData::Array *registers;
+    if (metadata_registers_dict->GetValueForKeyAsArray("registers", registers))
+      registers->ForEach([this, &offset,
+                          byte_order](StructuredData::Object *ent) -> bool {
+        uint64_t bitsize;
+        uint64_t value;
+        if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("bitsize",
+                                                             bitsize))
+          return false;
+        if (!ent->GetAsDictionary()->GetValueForKeyAsInteger("value", value)) {
+          // We had a bitsize but no value, so move the offset forward I guess.
+          offset += (bitsize / 8);
+          return false;
+        }
+        ent->GetAsDictionary()->AddIntegerItem("offset", offset);
+        Status error;
+        const int bytesize = bitsize / 8;
+        switch (bytesize) {
+        case 2: {
+          Scalar value_scalar((uint16_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        case 4: {
+          Scalar value_scalar((uint32_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        case 8: {
+          Scalar value_scalar((uint64_t)value);
+          value_scalar.GetAsMemoryData(m_register_data.data() + offset,
+                                       bytesize, byte_order, error);
+          offset += bytesize;
+        } break;
+        }
+        return true;
+      });
+  }
+
+  // Create a DynamicRegisterInfo from the metadata JSON.
+  std::unique_ptr<DynamicRegisterInfo> additional_reginfo_up;
+  if (metadata_registers_dict)
+    additional_reginfo_up = DynamicRegisterInfo::Create(
+        *metadata_registers_dict, target.GetArchitecture());
+
+  // Put the RegisterSet names in the constant string pool,
+  // to sidestep lifetime issues of char*'s.
+  auto copy_regset_name = [](RegisterSet &dst, const RegisterSet &src) {
+    dst.name = ConstString(src.name).AsCString();
+    if (src.short_name)
+      dst.short_name = ConstString(src.short_name).AsCString();
+    else
+      dst.short_name = nullptr;
+  };
+
+  // Copy the core thread register sets into our combined register set list.
+  // RegisterSet indexes will be identical for the LC_THREAD RegisterContext.
+  for (size_t idx = 0; idx < core_thread_regctx_sp->GetRegisterSetCount();
+       idx++) {
+    RegisterSet new_set;
+    const RegisterSet *old_set = core_thread_regctx_sp->GetRegisterSet(idx);
+    copy_regset_name(new_set, *old_set);
+    m_register_sets.push_back(new_set);
+  }
+
+  // Add any additional metadata RegisterSets to our combined RegisterSet array.
+  if (additional_reginfo_up) {
+    for (size_t idx = 0; idx < additional_reginfo_up->GetNumRegisterSets();
+         idx++) {
+      // See if this metadata RegisterSet name matches one already present
+      // from the LC_THREAD RegisterContext.
+      bool found_match = false;
+      const RegisterSet *old_set = additional_reginfo_up->GetRegisterSet(idx);
+      for (size_t jdx = 0; jdx < m_register_sets.size(); jdx++) {
+        if (strcmp(m_register_sets[jdx].name, old_set->name) == 0) {
+          metadata_regset_to_combined_regset[idx] = jdx;
+          found_match = true;
+          break;
+        }
+      }
+      // This metadata RegisterSet is a new one.
+      // Add it to the combined RegisterSet array.
+      if (!found_match) {
+        RegisterSet new_set;
+        copy_regset_name(new_set, *old_set);
+        metadata_regset_to_combined_regset[idx] = m_register_sets.size();
+        m_register_sets.push_back(new_set);
+      }
+    }
+  }
+
+  // Set up our combined RegisterInfo array, one RegisterSet at a time.
+  for (size_t combined_regset_idx = 0;
+       combined_regset_idx < m_register_sets.size(); combined_regset_idx++) {
+    uint32_t registers_this_regset = 0;
+
+    // Copy all LC_THREAD RegisterInfos that have a value into our
+    // combined RegisterInfo array.  (the LC_THREAD RegisterContext
+    // may describe registers that were not provided in this thread)
+    //
+    // LC_THREAD register set indexes are identical to the combined
+    // register set indexes.  The combined register set array may have
+    // additional entries.
+    if (combined_regset_idx < core_thread_regctx_sp->GetRegisterSetCount()) {
+      const RegisterSet *regset =
+          core_thread_regctx_sp->GetRegisterSet(combined_regset_idx);
+      // Copy all the registers that have values in.
+      for (size_t j = 0; j < regset->num_registers; j++) {
+        uint32_t reg_idx = regset->registers[j];
+        const RegisterInfo *reginfo =
+            core_thread_regctx_sp->GetRegisterInfoAtIndex(reg_idx);
+        RegisterValue val;
+        if (!reginfo->value_regs &&
+            core_thread_regctx_sp->ReadRegister(reginfo, val)) {
+          m_regset_regnum_collection[combined_regset_idx].push_back(
+              m_register_infos.size());
+          m_register_infos.push_back(*reginfo);
+          registers_this_regset++;
+        }
+      }
+    }
+
+    // Copy all the metadata RegisterInfos into our combined combined
+    // RegisterInfo array.
+    // The metadata may add registers to one of the LC_THREAD register sets,
+    // or its own newly added register sets.  metadata_regset_to_combined_regset
+    // has the association of the RegisterSet indexes between the two.
+    if (additional_reginfo_up) {
+      // Find the register set in the metadata that matches this register
+      // set, then copy all its RegisterInfos.
+      for (size_t setidx = 0;
+           setidx < additional_reginfo_up->GetNumRegisterSets(); setidx++) {
+        if (metadata_regset_to_combined_regset[setidx] == combined_regset_idx) {
+          const RegisterSet *regset =
+              additional_reginfo_up->GetRegisterSet(setidx);
+          for (size_t j = 0; j < regset->num_registers; j++) {
+            uint32_t reg_idx = regset->registers[j];
+            const RegisterInfo *reginfo =
+                additional_reginfo_up->GetRegisterInfoAtIndex(reg_idx);
+            m_regset_regnum_collection[combined_regset_idx].push_back(
+                m_register_infos.size());
+            m_register_infos.push_back(*reginfo);
+            registers_this_regset++;
+          }
+        }
+      }
+    }
+    m_register_sets[combined_regset_idx].num_registers = registers_this_regset;
+    m_register_sets[combined_regset_idx].registers =
+        m_regset_regnum_collection[combined_regset_idx].data();
+  }
+}
+
+size_t RegisterContextUnifiedCore::GetRegisterCount() {
+  return m_register_infos.size();
+}
+
+const RegisterInfo *
+RegisterContextUnifiedCore::GetRegisterInfoAtIndex(size_t reg) {
+  if (reg < m_register_infos.size())
+    return &m_register_infos[reg];
+  return nullptr;
+}
+
+size_t RegisterContextUnifiedCore::GetRegisterSetCount() {
+  return m_register_sets.size();
+}
+
+const RegisterSet *RegisterContextUnifiedCore::GetRegisterSet(size_t set) {
+  if (set < m_register_sets.size())
+    return &m_register_sets[set];
+  return nullptr;
+}
+
+bool RegisterContextUnifiedCore::ReadRegister(
+    const lldb_private::RegisterInfo *reg_info,
+    lldb_private::RegisterValue &value) {
+  if (!reg_info)
+    return false;
+  if (ProcessSP process_sp = m_thread.GetProcess()) {
+    DataExtractor regdata(m_register_data.data(), m_register_data.size(),
+                          process_sp->GetByteOrder(),
+                          process_sp->GetAddressByteSize());
+    offset_t offset = reg_info->byte_offset;
+    switch (reg_info->byte_size) {
+    case 2:
+      value.SetUInt16(regdata.GetU16(&offset));
+      break;
+    case 4:
+      value.SetUInt32(regdata.GetU32(&offset));
+      break;
+    case 8:
+      value.SetUInt64(regdata.GetU64(&offset));
+      break;
+    default:
+      return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+bool RegisterContextUnifiedCore::WriteRegister(
+    const lldb_private::RegisterInfo *reg_info,
+    const lldb_private::RegisterValue &value) {
+  return false;
+}
diff --git a/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h
new file mode 100644
index 000000000000..971d5cb56846
--- /dev/null
+++ b/lldb/source/Plugins/Process/mach-core/RegisterContextUnifiedCore.h
@@ -0,0 +1,57 @@
+//===-- RegisterContextUnifiedCore.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
+
+#include <string>
+#include <vector>
+
+#include "lldb/Target/RegisterContext.h"
+#include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/StructuredData.h"
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-private.h"
+
+namespace lldb_private {
+
+class RegisterContextUnifiedCore : public RegisterContext {
+public:
+  RegisterContextUnifiedCore(
+      Thread &thread, uint32_t concrete_frame_idx,
+      lldb::RegisterContextSP core_thread_regctx_sp,
+      lldb_private::StructuredData::ObjectSP metadata_thread_registers);
+
+  void InvalidateAllRegisters() override {};
+
+  size_t GetRegisterCount() override;
+
+  const lldb_private::RegisterInfo *GetRegisterInfoAtIndex(size_t reg) override;
+
+  size_t GetRegisterSetCount() override;
+
+  const lldb_private::RegisterSet *GetRegisterSet(size_t set) override;
+
+  bool ReadRegister(const lldb_private::RegisterInfo *reg_info,
+                    lldb_private::RegisterValue &value) override;
+
+  bool WriteRegister(const lldb_private::RegisterInfo *reg_info,
+                     const lldb_private::RegisterValue &value) override;
+
+private:
+  std::vector<lldb_private::RegisterSet> m_register_sets;
+  std::vector<lldb_private::RegisterInfo> m_register_infos;
+  /// For each register set, an array of register numbers included.
+  std::map<size_t, std::vector<uint32_t>> m_regset_regnum_collection;
+  /// Bytes of the register contents.
+  std::vector<uint8_t> m_register_data;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_PLUGINS_PROCESS_REGISTERCONTEXT_UNIFIED_CORE_H
diff --git a/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
index 003cdd0fcaa5..de7de9ee7c5c 100644
--- a/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
+++ b/lldb/source/Plugins/Process/mach-core/ThreadMachCore.cpp
@@ -6,12 +6,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "RegisterContextUnifiedCore.h"
 #include "ThreadMachCore.h"
 
 #include "lldb/Breakpoint/Watchpoint.h"
 #include "lldb/Host/SafeMachO.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Target/AppleArm64ExceptionClass.h"
+#include "lldb/Target/DynamicRegisterInfo.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/Target/StopInfo.h"
@@ -22,6 +28,7 @@
 #include "lldb/Utility/RegisterValue.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
+#include "lldb/Utility/StructuredData.h"
 
 #include "ProcessMachCore.h"
 //#include "RegisterContextKDP_arm.h"
@@ -70,27 +77,50 @@ lldb::RegisterContextSP ThreadMachCore::GetRegisterContext() {
 
 lldb::RegisterContextSP
 ThreadMachCore::CreateRegisterContextForFrame(StackFrame *frame) {
-  lldb::RegisterContextSP reg_ctx_sp;
   uint32_t concrete_frame_idx = 0;
 
   if (frame)
     concrete_frame_idx = frame->GetConcreteFrameIndex();
+  if (concrete_frame_idx > 0)
+    return GetUnwinder().CreateRegisterContextForFrame(frame);
 
-  if (concrete_frame_idx == 0) {
-    if (!m_thread_reg_ctx_sp) {
-      ProcessSP process_sp(GetProcess());
+  if (m_thread_reg_ctx_sp)
+    return m_thread_reg_ctx_sp;
 
-      ObjectFile *core_objfile =
-          static_cast<ProcessMachCore *>(process_sp.get())->GetCoreObjectFile();
-      if (core_objfile)
-        m_thread_reg_ctx_sp = core_objfile->GetThreadContextAtIndex(
-            m_objfile_lc_thread_idx, *this);
+  ProcessSP process_sp(GetProcess());
+  assert(process_sp);
+
+  ObjectFile *core_objfile =
+      static_cast<ProcessMachCore *>(process_sp.get())->GetCoreObjectFile();
+  if (!core_objfile)
+    return {};
+
+  RegisterContextSP core_thread_regctx_sp =
+      core_objfile->GetThreadContextAtIndex(m_objfile_lc_thread_idx, *this);
+
+  if (!core_thread_regctx_sp)
+    return {};
+
+  StructuredData::ObjectSP process_md_sp =
+      core_objfile->GetCorefileProcessMetadata();
+
+  StructuredData::ObjectSP thread_md_sp;
+  if (process_md_sp && process_md_sp->GetAsDictionary() &&
+      process_md_sp->GetAsDictionary()->HasKey("threads")) {
+    StructuredData::Array *threads = process_md_sp->GetAsDictionary()
+                                         ->GetValueForKey("threads")
+                                         ->GetAsArray();
+    if (threads && threads->GetSize() == core_objfile->GetNumThreadContexts()) {
+      StructuredData::ObjectSP thread_sp =
+          threads->GetItemAtIndex(m_objfile_lc_thread_idx);
+      if (thread_sp && thread_sp->GetAsDictionary())
+        thread_md_sp = thread_sp;
     }
-    reg_ctx_sp = m_thread_reg_ctx_sp;
-  } else {
-    reg_ctx_sp = GetUnwinder().CreateRegisterContextForFrame(frame);
   }
-  return reg_ctx_sp;
+  m_thread_reg_ctx_sp = std::make_shared<RegisterContextUnifiedCore>(
+      *this, concrete_frame_idx, core_thread_regctx_sp, thread_md_sp);
+
+  return m_thread_reg_ctx_sp;
 }
 
 static bool IsCrashExceptionClass(AppleArm64ExceptionClass EC) {
diff --git a/lldb/source/Plugins/Protocol/MCP/Protocol.h b/lldb/source/Plugins/Protocol/MCP/Protocol.h
index e31589940657..cb790dc4e559 100644
--- a/lldb/source/Plugins/Protocol/MCP/Protocol.h
+++ b/lldb/source/Plugins/Protocol/MCP/Protocol.h
@@ -123,6 +123,8 @@ using Message = std::variant<Request, Response, Notification, Error>;
 bool fromJSON(const llvm::json::Value &, Message &, llvm::json::Path);
 llvm::json::Value toJSON(const Message &);
 
+using ToolArguments = std::variant<std::monostate, llvm::json::Value>;
+
 } // namespace lldb_private::mcp::protocol
 
 #endif
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
index c3cd9a88c20b..3180341b50b9 100644
--- a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
@@ -24,8 +24,7 @@ LLDB_PLUGIN_DEFINE(ProtocolServerMCP)
 
 static constexpr size_t kChunkSize = 1024;
 
-ProtocolServerMCP::ProtocolServerMCP(Debugger &debugger)
-    : ProtocolServer(), m_debugger(debugger) {
+ProtocolServerMCP::ProtocolServerMCP() : ProtocolServer() {
   AddRequestHandler("initialize",
                     std::bind(&ProtocolServerMCP::InitializeHandler, this,
                               std::placeholders::_1));
@@ -39,8 +38,10 @@ ProtocolServerMCP::ProtocolServerMCP(Debugger &debugger)
       "notifications/initialized", [](const protocol::Notification &) {
         LLDB_LOG(GetLog(LLDBLog::Host), "MCP initialization complete");
       });
-  AddTool(std::make_unique<LLDBCommandTool>(
-      "lldb_command", "Run an lldb command.", m_debugger));
+  AddTool(
+      std::make_unique<CommandTool>("lldb_command", "Run an lldb command."));
+  AddTool(std::make_unique<DebuggerListTool>(
+      "lldb_debugger_list", "List debugger instances with their debugger_id."));
 }
 
 ProtocolServerMCP::~ProtocolServerMCP() { llvm::consumeError(Stop()); }
@@ -54,8 +55,8 @@ void ProtocolServerMCP::Terminate() {
   PluginManager::UnregisterPlugin(CreateInstance);
 }
 
-lldb::ProtocolServerSP ProtocolServerMCP::CreateInstance(Debugger &debugger) {
-  return std::make_shared<ProtocolServerMCP>(debugger);
+lldb::ProtocolServerUP ProtocolServerMCP::CreateInstance() {
+  return std::make_unique<ProtocolServerMCP>();
 }
 
 llvm::StringRef ProtocolServerMCP::GetPluginDescriptionStatic() {
@@ -145,7 +146,7 @@ llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
   std::lock_guard<std::mutex> guard(m_server_mutex);
 
   if (m_running)
-    return llvm::createStringError("server already running");
+    return llvm::createStringError("the MCP server is already running");
 
   Status status;
   m_listener = Socket::Create(connection.protocol, status);
@@ -162,10 +163,10 @@ llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
   if (llvm::Error error = handles.takeError())
     return error;
 
+  m_running = true;
   m_listen_handlers = std::move(*handles);
   m_loop_thread = std::thread([=] {
-    llvm::set_thread_name(
-        llvm::formatv("debugger-{0}.mcp.runloop", m_debugger.GetID()));
+    llvm::set_thread_name("protocol-server.mcp");
     m_loop.Run();
   });
 
@@ -175,6 +176,8 @@ llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
 llvm::Error ProtocolServerMCP::Stop() {
   {
     std::lock_guard<std::mutex> guard(m_server_mutex);
+    if (!m_running)
+      return createStringError("the MCP sever is not running");
     m_running = false;
   }
 
@@ -311,11 +314,12 @@ ProtocolServerMCP::ToolsCallHandler(const protocol::Request &request) {
   if (it == m_tools.end())
     return llvm::createStringError(llvm::formatv("no tool \"{0}\"", tool_name));
 
-  const json::Value *args = param_obj->get("arguments");
-  if (!args)
-    return llvm::createStringError("no tool arguments");
+  protocol::ToolArguments tool_args;
+  if (const json::Value *args = param_obj->get("arguments"))
+    tool_args = *args;
 
-  llvm::Expected<protocol::TextResult> text_result = it->second->Call(*args);
+  llvm::Expected<protocol::TextResult> text_result =
+      it->second->Call(tool_args);
   if (!text_result)
     return text_result.takeError();
 
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
index 52bb92a04a80..d55882cc8ab0 100644
--- a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
@@ -21,7 +21,7 @@ namespace lldb_private::mcp {
 
 class ProtocolServerMCP : public ProtocolServer {
 public:
-  ProtocolServerMCP(Debugger &debugger);
+  ProtocolServerMCP();
   virtual ~ProtocolServerMCP() override;
 
   virtual llvm::Error Start(ProtocolServer::Connection connection) override;
@@ -33,7 +33,7 @@ class ProtocolServerMCP : public ProtocolServer {
   static llvm::StringRef GetPluginNameStatic() { return "MCP"; }
   static llvm::StringRef GetPluginDescriptionStatic();
 
-  static lldb::ProtocolServerSP CreateInstance(Debugger &debugger);
+  static lldb::ProtocolServerUP CreateInstance();
 
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
 
@@ -71,8 +71,6 @@ class ProtocolServerMCP : public ProtocolServer {
   llvm::StringLiteral kName = "lldb-mcp";
   llvm::StringLiteral kVersion = "0.1.0";
 
-  Debugger &m_debugger;
-
   bool m_running = false;
 
   MainLoop m_loop;
diff --git a/lldb/source/Plugins/Protocol/MCP/Tool.cpp b/lldb/source/Plugins/Protocol/MCP/Tool.cpp
index de8fcc8f3cb4..5c4626cf66b3 100644
--- a/lldb/source/Plugins/Protocol/MCP/Tool.cpp
+++ b/lldb/source/Plugins/Protocol/MCP/Tool.cpp
@@ -7,22 +7,38 @@
 //===----------------------------------------------------------------------===//
 
 #include "Tool.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
 
 using namespace lldb_private::mcp;
 using namespace llvm;
 
-struct LLDBCommandToolArguments {
+namespace {
+struct CommandToolArguments {
+  uint64_t debugger_id;
   std::string arguments;
 };
 
-bool fromJSON(const llvm::json::Value &V, LLDBCommandToolArguments &A,
+bool fromJSON(const llvm::json::Value &V, CommandToolArguments &A,
               llvm::json::Path P) {
   llvm::json::ObjectMapper O(V, P);
-  return O && O.map("arguments", A.arguments);
+  return O && O.map("debugger_id", A.debugger_id) &&
+         O.mapOptional("arguments", A.arguments);
 }
 
+/// Helper function to create a TextResult from a string output.
+static lldb_private::mcp::protocol::TextResult
+createTextResult(std::string output, bool is_error = false) {
+  lldb_private::mcp::protocol::TextResult text_result;
+  text_result.content.emplace_back(
+      lldb_private::mcp::protocol::TextContent{{std::move(output)}});
+  text_result.isError = is_error;
+  return text_result;
+}
+
+} // namespace
+
 Tool::Tool(std::string name, std::string description)
     : m_name(std::move(name)), m_description(std::move(description)) {}
 
@@ -37,22 +53,27 @@ protocol::ToolDefinition Tool::GetDefinition() const {
   return definition;
 }
 
-LLDBCommandTool::LLDBCommandTool(std::string name, std::string description,
-                                 Debugger &debugger)
-    : Tool(std::move(name), std::move(description)), m_debugger(debugger) {}
-
 llvm::Expected<protocol::TextResult>
-LLDBCommandTool::Call(const llvm::json::Value &args) {
-  llvm::json::Path::Root root;
+CommandTool::Call(const protocol::ToolArguments &args) {
+  if (!std::holds_alternative<json::Value>(args))
+    return createStringError("CommandTool requires arguments");
+
+  json::Path::Root root;
 
-  LLDBCommandToolArguments arguments;
-  if (!fromJSON(args, arguments, root))
+  CommandToolArguments arguments;
+  if (!fromJSON(std::get<json::Value>(args), arguments, root))
     return root.getError();
 
+  lldb::DebuggerSP debugger_sp =
+      Debugger::GetDebuggerAtIndex(arguments.debugger_id);
+  if (!debugger_sp)
+    return createStringError(
+        llvm::formatv("no debugger with id {0}", arguments.debugger_id));
+
   // FIXME: Disallow certain commands and their aliases.
   CommandReturnObject result(/*colors=*/false);
-  m_debugger.GetCommandInterpreter().HandleCommand(arguments.arguments.c_str(),
-                                                   eLazyBoolYes, result);
+  debugger_sp->GetCommandInterpreter().HandleCommand(
+      arguments.arguments.c_str(), eLazyBoolYes, result);
 
   std::string output;
   llvm::StringRef output_str = result.GetOutputString();
@@ -66,16 +87,64 @@ LLDBCommandTool::Call(const llvm::json::Value &args) {
     output += err_str;
   }
 
-  mcp::protocol::TextResult text_result;
-  text_result.content.emplace_back(mcp::protocol::TextContent{{output}});
-  text_result.isError = !result.Succeeded();
-  return text_result;
+  return createTextResult(output, !result.Succeeded());
 }
 
-std::optional<llvm::json::Value> LLDBCommandTool::GetSchema() const {
+std::optional<llvm::json::Value> CommandTool::GetSchema() const {
+  llvm::json::Object id_type{{"type", "number"}};
   llvm::json::Object str_type{{"type", "string"}};
-  llvm::json::Object properties{{"arguments", std::move(str_type)}};
+  llvm::json::Object properties{{"debugger_id", std::move(id_type)},
+                                {"arguments", std::move(str_type)}};
+  llvm::json::Array required{"debugger_id"};
   llvm::json::Object schema{{"type", "object"},
-                            {"properties", std::move(properties)}};
+                            {"properties", std::move(properties)},
+                            {"required", std::move(required)}};
   return schema;
 }
+
+llvm::Expected<protocol::TextResult>
+DebuggerListTool::Call(const protocol::ToolArguments &args) {
+  if (!std::holds_alternative<std::monostate>(args))
+    return createStringError("DebuggerListTool takes no arguments");
+
+  llvm::json::Path::Root root;
+
+  // Return a nested Markdown list with debuggers and target.
+  // Example output:
+  //
+  // - debugger 0
+  //     - target 0 /path/to/foo
+  //     - target 1
+  // - debugger 1
+  //     - target 0 /path/to/bar
+  //
+  // FIXME: Use Structured Content when we adopt protocol version 2025-06-18.
+  std::string output;
+  llvm::raw_string_ostream os(output);
+
+  const size_t num_debuggers = Debugger::GetNumDebuggers();
+  for (size_t i = 0; i < num_debuggers; ++i) {
+    lldb::DebuggerSP debugger_sp = Debugger::GetDebuggerAtIndex(i);
+    if (!debugger_sp)
+      continue;
+
+    os << "- debugger " << i << '\n';
+
+    TargetList &target_list = debugger_sp->GetTargetList();
+    const size_t num_targets = target_list.GetNumTargets();
+    for (size_t j = 0; j < num_targets; ++j) {
+      lldb::TargetSP target_sp = target_list.GetTargetAtIndex(j);
+      if (!target_sp)
+        continue;
+      os << "    - target " << j;
+      if (target_sp == target_list.GetSelectedTarget())
+        os << " (selected)";
+      // Append the module path if we have one.
+      if (Module *exe_module = target_sp->GetExecutableModulePointer())
+        os << " " << exe_module->GetFileSpec().GetPath();
+      os << '\n';
+    }
+  }
+
+  return createTextResult(output);
+}
diff --git a/lldb/source/Plugins/Protocol/MCP/Tool.h b/lldb/source/Plugins/Protocol/MCP/Tool.h
index 57a5125813b7..74ab04b47252 100644
--- a/lldb/source/Plugins/Protocol/MCP/Tool.h
+++ b/lldb/source/Plugins/Protocol/MCP/Tool.h
@@ -22,10 +22,10 @@ class Tool {
   virtual ~Tool() = default;
 
   virtual llvm::Expected<protocol::TextResult>
-  Call(const llvm::json::Value &args) = 0;
+  Call(const protocol::ToolArguments &args) = 0;
 
   virtual std::optional<llvm::json::Value> GetSchema() const {
-    return std::nullopt;
+    return llvm::json::Object{{"type", "object"}};
   }
 
   protocol::ToolDefinition GetDefinition() const;
@@ -37,20 +37,26 @@ class Tool {
   std::string m_description;
 };
 
-class LLDBCommandTool : public mcp::Tool {
+class CommandTool : public mcp::Tool {
 public:
-  LLDBCommandTool(std::string name, std::string description,
-                  Debugger &debugger);
-  ~LLDBCommandTool() = default;
+  using mcp::Tool::Tool;
+  ~CommandTool() = default;
 
   virtual llvm::Expected<protocol::TextResult>
-  Call(const llvm::json::Value &args) override;
+  Call(const protocol::ToolArguments &args) override;
 
   virtual std::optional<llvm::json::Value> GetSchema() const override;
+};
 
-private:
-  Debugger &m_debugger;
+class DebuggerListTool : public mcp::Tool {
+public:
+  using mcp::Tool::Tool;
+  ~DebuggerListTool() = default;
+
+  virtual llvm::Expected<protocol::TextResult>
+  Call(const protocol::ToolArguments &args) override;
 };
+
 } // namespace lldb_private::mcp
 
 #endif
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
index c5876502b8f2..a0b8acb58563 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.cpp
@@ -7,20 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "DWARFDataExtractor.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/ArrayRef.h"
 
 namespace lldb_private {
 
-uint64_t
-DWARFDataExtractor::GetDWARFInitialLength(lldb::offset_t *offset_ptr) const {
-  return GetU32(offset_ptr);
-}
-
-dw_offset_t
-DWARFDataExtractor::GetDWARFOffset(lldb::offset_t *offset_ptr) const {
-  return GetMaxU64(offset_ptr, GetDWARFSizeOfOffset());
-}
-
 llvm::DWARFDataExtractor DWARFDataExtractor::GetAsLLVMDWARF() const {
   return llvm::DWARFDataExtractor(llvm::ArrayRef(GetDataStart(), GetByteSize()),
                                   GetByteOrder() == lldb::eByteOrderLittle,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
index 41b8e9ad0217..f6bcf9b2d27f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDataExtractor.h
@@ -9,7 +9,6 @@
 #ifndef LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDATAEXTRACTOR_H
 #define LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DWARFDATAEXTRACTOR_H
 
-#include "lldb/Core/dwarf.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 
@@ -23,13 +22,6 @@ class DWARFDataExtractor : public DataExtractor {
                      lldb::offset_t length)
       : DataExtractor(data, offset, length) {}
 
-  uint64_t GetDWARFInitialLength(lldb::offset_t *offset_ptr) const;
-
-  dw_offset_t GetDWARFOffset(lldb::offset_t *offset_ptr) const;
-
-  size_t GetDWARFSizeofInitialLength() const { return 4; }
-  size_t GetDWARFSizeOfOffset() const { return 4; }
-
   llvm::DWARFDataExtractor GetAsLLVMDWARF() const;
   llvm::DataExtractor GetAsLLVM() const;
 };
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
index fd3d45cef4c5..a159b923f8df 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFFormValue.cpp
@@ -36,7 +36,6 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
   bool indirect = false;
   bool is_block = false;
   m_value.data = nullptr;
-  uint8_t ref_addr_size;
   // Read the value for the form into value and follow and DW_FORM_indirect
   // instances we run into
   do {
@@ -77,7 +76,9 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
     case DW_FORM_strp:
     case DW_FORM_line_strp:
     case DW_FORM_sec_offset:
-      m_value.uval = data.GetMaxU64(offset_ptr, 4);
+      assert(m_unit);
+      m_value.uval = data.GetMaxU64(
+          offset_ptr, m_unit->GetFormParams().getDwarfOffsetByteSize());
       break;
     case DW_FORM_addrx1:
     case DW_FORM_strx1:
@@ -119,11 +120,8 @@ bool DWARFFormValue::ExtractValue(const DWARFDataExtractor &data,
       break;
     case DW_FORM_ref_addr:
       assert(m_unit);
-      if (m_unit->GetVersion() <= 2)
-        ref_addr_size = m_unit->GetAddressByteSize();
-      else
-        ref_addr_size = 4;
-      m_value.uval = data.GetMaxU64(offset_ptr, ref_addr_size);
+      m_value.uval = data.GetMaxU64(
+          offset_ptr, m_unit->GetFormParams().getRefAddrByteSize());
       break;
     case DW_FORM_indirect:
       m_form = static_cast<dw_form_t>(data.GetULEB128(offset_ptr));
@@ -165,7 +163,7 @@ static FormSize g_form_sizes[] = {
     {1, 1}, // 0x0b DW_FORM_data1
     {1, 1}, // 0x0c DW_FORM_flag
     {0, 0}, // 0x0d DW_FORM_sdata
-    {1, 4}, // 0x0e DW_FORM_strp
+    {0, 0}, // 0x0e DW_FORM_strp (4 bytes for DWARF32, 8 bytes for DWARF64)
     {0, 0}, // 0x0f DW_FORM_udata
     {0, 0}, // 0x10 DW_FORM_ref_addr (addr size for DWARF2 and earlier, 4 bytes
             // for DWARF32, 8 bytes for DWARF32 in DWARF 3 and later
@@ -175,7 +173,7 @@ static FormSize g_form_sizes[] = {
     {1, 8},  // 0x14 DW_FORM_ref8
     {0, 0},  // 0x15 DW_FORM_ref_udata
     {0, 0},  // 0x16 DW_FORM_indirect
-    {1, 4},  // 0x17 DW_FORM_sec_offset
+    {0, 0}, // 0x17 DW_FORM_sec_offset (4 bytes for DWARF32,8 bytes for DWARF64)
     {0, 0},  // 0x18 DW_FORM_exprloc
     {1, 0},  // 0x19 DW_FORM_flag_present
     {0, 0},  // 0x1a DW_FORM_strx (ULEB128)
@@ -183,7 +181,7 @@ static FormSize g_form_sizes[] = {
     {1, 4},  // 0x1c DW_FORM_ref_sup4
     {0, 0},  // 0x1d DW_FORM_strp_sup (4 bytes for DWARF32, 8 bytes for DWARF64)
     {1, 16}, // 0x1e DW_FORM_data16
-    {1, 4},  // 0x1f DW_FORM_line_strp
+    {0, 0}, // 0x1f DW_FORM_line_strp (4 bytes for DWARF32, 8 bytes for DWARF64)
     {1, 8},  // 0x20 DW_FORM_ref_sig8
 };
 
@@ -209,7 +207,6 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
                                const DWARFDataExtractor &debug_info_data,
                                lldb::offset_t *offset_ptr,
                                const DWARFUnit *unit) {
-  uint8_t ref_addr_size;
   switch (form) {
   // Blocks if inlined data that have a length field and the data bytes inlined
   // in the .debug_info
@@ -246,14 +243,9 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
     return true;
 
   case DW_FORM_ref_addr:
-    ref_addr_size = 4;
     assert(unit); // Unit must be valid for DW_FORM_ref_addr objects or we will
                   // get this wrong
-    if (unit->GetVersion() <= 2)
-      ref_addr_size = unit->GetAddressByteSize();
-    else
-      ref_addr_size = 4;
-    *offset_ptr += ref_addr_size;
+    *offset_ptr += unit->GetFormParams().getRefAddrByteSize();
     return true;
 
   // 0 bytes values (implied from DW_FORM)
@@ -288,7 +280,8 @@ bool DWARFFormValue::SkipValue(dw_form_t form,
     case DW_FORM_sec_offset:
     case DW_FORM_strp:
     case DW_FORM_line_strp:
-      *offset_ptr += 4;
+      assert(unit);
+      *offset_ptr += unit->GetFormParams().getDwarfOffsetByteSize();
       return true;
 
     // 4 byte values
@@ -419,12 +412,8 @@ void DWARFFormValue::Dump(Stream &s) const {
   case DW_FORM_ref_addr: {
     assert(m_unit); // Unit must be valid for DW_FORM_ref_addr objects or we
                     // will get this wrong
-    if (m_unit->GetVersion() <= 2)
-      DumpAddress(s.AsRawOstream(), uvalue, sizeof(uint64_t) * 2);
-    else
-      DumpAddress(s.AsRawOstream(), uvalue,
-                  4 * 2); // 4 for DWARF32, 8 for DWARF64, but we don't
-                          // support DWARF64 yet
+    DumpAddress(s.AsRawOstream(), uvalue,
+                m_unit->GetFormParams().getRefAddrByteSize());
     break;
   }
   case DW_FORM_ref1:
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index ffd6f1dd52af..f216ab13e893 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -1073,20 +1073,7 @@ const lldb_private::DWARFDataExtractor &DWARFUnit::GetData() const {
              : m_dwarf.GetDWARFContext().getOrLoadDebugInfoData();
 }
 
-uint32_t DWARFUnit::GetHeaderByteSize() const {
-  switch (m_header.getUnitType()) {
-  case llvm::dwarf::DW_UT_compile:
-  case llvm::dwarf::DW_UT_partial:
-    return GetVersion() < 5 ? 11 : 12;
-  case llvm::dwarf::DW_UT_skeleton:
-  case llvm::dwarf::DW_UT_split_compile:
-    return 20;
-  case llvm::dwarf::DW_UT_type:
-  case llvm::dwarf::DW_UT_split_type:
-    return GetVersion() < 5 ? 23 : 24;
-  }
-  llvm_unreachable("invalid UnitType.");
-}
+uint32_t DWARFUnit::GetHeaderByteSize() const { return m_header.getSize(); }
 
 std::optional<uint64_t>
 DWARFUnit::GetStringOffsetSectionItem(uint32_t index) const {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index c05bba36ed74..f55400eeaa44 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -119,6 +119,9 @@ class DWARFUnit : public DWARFExpression::Delegate, public UserID {
   // Size of the CU data incl. header but without initial length.
   dw_offset_t GetLength() const { return m_header.getLength(); }
   uint16_t GetVersion() const override { return m_header.getVersion(); }
+  const llvm::dwarf::FormParams &GetFormParams() const {
+    return m_header.getFormParams();
+  }
   const llvm::DWARFAbbreviationDeclarationSet *GetAbbreviations() const;
   dw_offset_t GetAbbrevOffset() const;
   uint8_t GetAddressByteSize() const override {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 71f204c03a42..c83779c40a05 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4420,3 +4420,32 @@ void SymbolFileDWARF::GetCompileOptions(
     args.insert({comp_unit, Args(flags)});
   }
 }
+
+std::pair<uint32_t, uint32_t> SymbolFileDWARF::GetDwoFileCounts() {
+  uint32_t total_dwo_count = 0;
+  uint32_t loaded_dwo_count = 0;
+
+  DWARFDebugInfo &info = DebugInfo();
+  const size_t num_cus = info.GetNumUnits();
+  for (size_t cu_idx = 0; cu_idx < num_cus; cu_idx++) {
+    DWARFUnit *dwarf_cu = info.GetUnitAtIndex(cu_idx);
+    if (dwarf_cu == nullptr)
+      continue;
+
+    // Check if this is a DWO unit by checking if it has a DWO ID.
+    if (!dwarf_cu->GetDWOId().has_value())
+      continue;
+
+    total_dwo_count++;
+
+    // If we have a DWO symbol file, that means we were able to successfully
+    // load it.
+    SymbolFile *dwo_symfile =
+        dwarf_cu->GetDwoSymbolFile(/*load_all_debug_info=*/false);
+    if (dwo_symfile) {
+      loaded_dwo_count++;
+    }
+  }
+
+  return {loaded_dwo_count, total_dwo_count};
+}
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index d2d30d7decb1..8335f1171287 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -282,6 +282,11 @@ class SymbolFileDWARF : public SymbolFileCommon {
   bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
                             bool errors_only) override;
 
+  // Gets a pair of loaded and total dwo file counts.
+  // For split-dwarf files, this reports the counts for successfully loaded DWO
+  // CUs and total DWO CUs. For non-split-dwarf files, this reports 0 for both.
+  std::pair<uint32_t, uint32_t> GetDwoFileCounts() override;
+
   DWARFContext &GetDWARFContext() { return m_context; }
 
   const std::shared_ptr<SymbolFileDWARFDwo> &GetDwpSymbolFile();
diff --git a/lldb/source/Symbol/SaveCoreOptions.cpp b/lldb/source/Symbol/SaveCoreOptions.cpp
index 0f9dbb73c172..f93b58f59cf9 100644
--- a/lldb/source/Symbol/SaveCoreOptions.cpp
+++ b/lldb/source/Symbol/SaveCoreOptions.cpp
@@ -23,7 +23,7 @@ Status SaveCoreOptions::SetPluginName(const char *name) {
 
   std::vector<llvm::StringRef> plugin_names =
       PluginManager::GetSaveCorePluginNames();
-  if (llvm::find(plugin_names, name) == plugin_names.end()) {
+  if (!llvm::is_contained(plugin_names, name)) {
     StreamString stream;
     stream.Printf("plugin name '%s' is not a valid ObjectFile plugin name.",
                   name);
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 61a3d05bc374..bba1230c7992 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -4614,7 +4614,7 @@ class IOHandlerProcessSTDIO : public IOHandler {
         m_process(process),
         m_read_file(GetInputFD(), File::eOpenOptionReadOnly, false),
         m_write_file(write_fd, File::eOpenOptionWriteOnly, false) {
-    m_pipe.CreateNew(false);
+    m_pipe.CreateNew();
   }
 
   ~IOHandlerProcessSTDIO() override = default;
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index 52a54020bc46..d97a81495218 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -562,6 +562,7 @@ ValueObjectSP StackFrame::DILGetValueForVariableExpressionPath(
     return ValueObjectConstResult::Create(nullptr, std::move(error));
   }
 
+  var_sp = (*valobj_or_error)->GetVariable();
   return *valobj_or_error;
 }
 
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index 6ec8f8963baf..909f335687b2 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -73,6 +73,8 @@ json::Value ModuleStats::ToJSON() const {
                      debug_info_had_incomplete_types);
   module.try_emplace("symbolTableStripped", symtab_stripped);
   module.try_emplace("symbolTableSymbolCount", symtab_symbol_count);
+  module.try_emplace("dwoFileCount", dwo_file_count);
+  module.try_emplace("loadedDwoFileCount", loaded_dwo_file_count);
 
   if (!symbol_locator_time.map.empty()) {
     json::Object obj;
@@ -86,7 +88,7 @@ json::Value ModuleStats::ToJSON() const {
 
   if (!symfile_modules.empty()) {
     json::Array symfile_ids;
-    for (const auto symfile_id: symfile_modules)
+    for (const auto symfile_id : symfile_modules)
       symfile_ids.emplace_back(symfile_id);
     module.try_emplace("symbolFileModuleIdentifiers", std::move(symfile_ids));
   }
@@ -322,6 +324,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   uint32_t num_modules_with_incomplete_types = 0;
   uint32_t num_stripped_modules = 0;
   uint32_t symtab_symbol_count = 0;
+  uint32_t total_loaded_dwo_file_count = 0;
+  uint32_t total_dwo_file_count = 0;
   for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
     Module *module = target != nullptr
                          ? target->GetImages().GetModuleAtIndex(image_idx).get()
@@ -353,6 +357,10 @@ llvm::json::Value DebuggerStats::ReportStatistics(
         for (const auto &symbol_module : symbol_modules.Modules())
           module_stat.symfile_modules.push_back((intptr_t)symbol_module.get());
       }
+      std::tie(module_stat.loaded_dwo_file_count, module_stat.dwo_file_count) =
+          sym_file->GetDwoFileCounts();
+      total_dwo_file_count += module_stat.dwo_file_count;
+      total_loaded_dwo_file_count += module_stat.loaded_dwo_file_count;
       module_stat.debug_info_index_loaded_from_cache =
           sym_file->GetDebugInfoIndexWasLoadedFromCache();
       if (module_stat.debug_info_index_loaded_from_cache)
@@ -427,6 +435,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
       {"totalDebugInfoEnabled", num_debug_info_enabled_modules},
       {"totalSymbolTableStripped", num_stripped_modules},
       {"totalSymbolTableSymbolCount", symtab_symbol_count},
+      {"totalLoadedDwoFileCount", total_loaded_dwo_file_count},
+      {"totalDwoFileCount", total_dwo_file_count},
   };
 
   if (include_targets) {
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 45a9e1196a04..7f569173eba2 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -1510,15 +1510,18 @@ bool Target::IgnoreWatchpointByID(lldb::watch_id_t watch_id,
 }
 
 ModuleSP Target::GetExecutableModule() {
-  // search for the first executable in the module list
-  for (ModuleSP module_sp : m_images.Modules()) {
+  std::lock_guard<std::recursive_mutex> lock(m_images.GetMutex());
+
+  // Search for the first executable in the module list.
+  for (ModuleSP module_sp : m_images.ModulesNoLocking()) {
     lldb_private::ObjectFile *obj = module_sp->GetObjectFile();
     if (obj == nullptr)
       continue;
     if (obj->GetType() == ObjectFile::Type::eTypeExecutable)
       return module_sp;
   }
-  // as fall back return the first module loaded
+
+  // If there is none, fall back return the first module loaded.
   return m_images.GetModuleAtIndex(0);
 }
 
@@ -3546,6 +3549,7 @@ llvm::Expected<TraceSP> Target::GetTraceOrCreate() {
 }
 
 Status Target::Attach(ProcessAttachInfo &attach_info, Stream *stream) {
+  Progress attach_progress("Waiting to attach to process");
   m_stats.SetLaunchOrAttachTime();
   auto state = eStateInvalid;
   auto process_sp = GetProcessSP();
diff --git a/lldb/source/Utility/IOObject.cpp b/lldb/source/Utility/IOObject.cpp
index 964edce0ce10..c0c07cc0b68e 100644
--- a/lldb/source/Utility/IOObject.cpp
+++ b/lldb/source/Utility/IOObject.cpp
@@ -8,7 +8,16 @@
 
 #include "lldb/Utility/IOObject.h"
 
+#ifdef _WIN32
+#include "lldb/Host/windows/windows.h"
+#endif
+
 using namespace lldb_private;
 
+#ifdef _WIN32
+const IOObject::WaitableHandle IOObject::kInvalidHandleValue =
+    INVALID_HANDLE_VALUE;
+#else
 const IOObject::WaitableHandle IOObject::kInvalidHandleValue = -1;
+#endif
 IOObject::~IOObject() = default;
diff --git a/lldb/source/Utility/SelectHelper.cpp b/lldb/source/Utility/SelectHelper.cpp
index 34c1b62228fd..3044cdae110e 100644
--- a/lldb/source/Utility/SelectHelper.cpp
+++ b/lldb/source/Utility/SelectHelper.cpp
@@ -177,7 +177,7 @@ lldb_private::Status SelectHelper::Select() {
 #endif
   // Set the FD bits in the fdsets for read/write/error
   for (auto &pair : m_fd_map) {
-    const lldb::socket_t fd = pair.first;
+    lldb::socket_t fd = pair.first;
 
     if (pair.second.read_set)
       FD_SET(fd, read_fdset_ptr);
diff --git a/lldb/source/ValueObject/DILEval.cpp b/lldb/source/ValueObject/DILEval.cpp
index b2bb4e20ddc2..8ca9b4215985 100644
--- a/lldb/source/ValueObject/DILEval.cpp
+++ b/lldb/source/ValueObject/DILEval.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/ValueObject/DILEval.h"
+#include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/ValueObject/DILAST.h"
@@ -18,70 +19,26 @@
 
 namespace lldb_private::dil {
 
-static lldb::ValueObjectSP LookupStaticIdentifier(
-    VariableList &variable_list, std::shared_ptr<StackFrame> exe_scope,
-    llvm::StringRef name_ref, llvm::StringRef unqualified_name) {
-  // First look for an exact match to the (possibly) qualified name.
-  for (const lldb::VariableSP &var_sp : variable_list) {
-    lldb::ValueObjectSP valobj_sp(
-        ValueObjectVariable::Create(exe_scope.get(), var_sp));
-    if (valobj_sp && valobj_sp->GetVariable() &&
-        (valobj_sp->GetVariable()->NameMatches(ConstString(name_ref))))
-      return valobj_sp;
-  }
-
-  // If the qualified name is the same as the unqualfied name, there's nothing
-  // more to be done.
-  if (name_ref == unqualified_name)
-    return nullptr;
-
-  // We didn't match the qualified name; try to match the unqualified name.
-  for (const lldb::VariableSP &var_sp : variable_list) {
-    lldb::ValueObjectSP valobj_sp(
-        ValueObjectVariable::Create(exe_scope.get(), var_sp));
-    if (valobj_sp && valobj_sp->GetVariable() &&
-        (valobj_sp->GetVariable()->NameMatches(ConstString(unqualified_name))))
-      return valobj_sp;
-  }
-
-  return nullptr;
-}
-
 static lldb::VariableSP DILFindVariable(ConstString name,
-                                        lldb::VariableListSP variable_list) {
+                                        VariableList &variable_list) {
   lldb::VariableSP exact_match;
   std::vector<lldb::VariableSP> possible_matches;
 
-  for (lldb::VariableSP var_sp : *variable_list) {
+  for (lldb::VariableSP var_sp : variable_list) {
     llvm::StringRef str_ref_name = var_sp->GetName().GetStringRef();
-    // Check for global vars, which might start with '::'.
-    str_ref_name.consume_front("::");
-
-    if (str_ref_name == name.GetStringRef())
-      possible_matches.push_back(var_sp);
-    else if (var_sp->NameMatches(name))
-      possible_matches.push_back(var_sp);
-  }
-
-  // Look for exact matches (favors local vars over global vars)
-  auto exact_match_it =
-      llvm::find_if(possible_matches, [&](lldb::VariableSP var_sp) {
-        return var_sp->GetName() == name;
-      });
-
-  if (exact_match_it != possible_matches.end())
-    return *exact_match_it;
 
-  // Look for a global var exact match.
-  for (auto var_sp : possible_matches) {
-    llvm::StringRef str_ref_name = var_sp->GetName().GetStringRef();
     str_ref_name.consume_front("::");
+    // Check for the exact same match
     if (str_ref_name == name.GetStringRef())
       return var_sp;
+
+    // Check for possible matches by base name
+    if (var_sp->NameMatches(name))
+      possible_matches.push_back(var_sp);
   }
 
-  // If there's a single non-exact match, take it.
-  if (possible_matches.size() == 1)
+  // If there's a non-exact match, take it.
+  if (possible_matches.size() > 0)
     return possible_matches[0];
 
   return nullptr;
@@ -89,38 +46,23 @@ static lldb::VariableSP DILFindVariable(ConstString name,
 
 lldb::ValueObjectSP LookupGlobalIdentifier(
     llvm::StringRef name_ref, std::shared_ptr<StackFrame> stack_frame,
-    lldb::TargetSP target_sp, lldb::DynamicValueType use_dynamic,
-    CompilerType *scope_ptr) {
-  // First look for match in "local" global variables.
-  lldb::VariableListSP variable_list(stack_frame->GetInScopeVariableList(true));
-  name_ref.consume_front("::");
+    lldb::TargetSP target_sp, lldb::DynamicValueType use_dynamic) {
+  // Get a global variables list without the locals from the current frame
+  SymbolContext symbol_context =
+      stack_frame->GetSymbolContext(lldb::eSymbolContextCompUnit);
+  lldb::VariableListSP variable_list =
+      symbol_context.comp_unit->GetVariableList(true);
 
+  name_ref.consume_front("::");
   lldb::ValueObjectSP value_sp;
   if (variable_list) {
     lldb::VariableSP var_sp =
-        DILFindVariable(ConstString(name_ref), variable_list);
+        DILFindVariable(ConstString(name_ref), *variable_list);
     if (var_sp)
       value_sp =
           stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
   }
 
-  if (value_sp)
-    return value_sp;
-
-  // Also check for static global vars.
-  if (variable_list) {
-    const char *type_name = "";
-    if (scope_ptr)
-      type_name = scope_ptr->GetCanonicalType().GetTypeName().AsCString();
-    std::string name_with_type_prefix =
-        llvm::formatv("{0}::{1}", type_name, name_ref).str();
-    value_sp = LookupStaticIdentifier(*variable_list, stack_frame,
-                                      name_with_type_prefix, name_ref);
-    if (!value_sp)
-      value_sp = LookupStaticIdentifier(*variable_list, stack_frame, name_ref,
-                                        name_ref);
-  }
-
   if (value_sp)
     return value_sp;
 
@@ -129,28 +71,22 @@ lldb::ValueObjectSP LookupGlobalIdentifier(
   target_sp->GetImages().FindGlobalVariables(
       ConstString(name_ref), std::numeric_limits<uint32_t>::max(),
       modules_var_list);
-  if (modules_var_list.Empty())
-    return nullptr;
 
-  for (const lldb::VariableSP &var_sp : modules_var_list) {
-    std::string qualified_name = llvm::formatv("::{0}", name_ref).str();
-    if (var_sp->NameMatches(ConstString(name_ref)) ||
-        var_sp->NameMatches(ConstString(qualified_name))) {
+  if (!modules_var_list.Empty()) {
+    lldb::VariableSP var_sp =
+        DILFindVariable(ConstString(name_ref), modules_var_list);
+    if (var_sp)
       value_sp = ValueObjectVariable::Create(stack_frame.get(), var_sp);
-      break;
-    }
-  }
-
-  if (value_sp)
-    return value_sp;
 
+    if (value_sp)
+      return value_sp;
+  }
   return nullptr;
 }
 
 lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
                                      std::shared_ptr<StackFrame> stack_frame,
-                                     lldb::DynamicValueType use_dynamic,
-                                     CompilerType *scope_ptr) {
+                                     lldb::DynamicValueType use_dynamic) {
   // Support $rax as a special syntax for accessing registers.
   // Will return an invalid value in case the requested register doesn't exist.
   if (name_ref.consume_front("$")) {
@@ -164,38 +100,34 @@ lldb::ValueObjectSP LookupIdentifier(llvm::StringRef name_ref,
     return nullptr;
   }
 
-  lldb::VariableListSP variable_list(
-      stack_frame->GetInScopeVariableList(false));
-
   if (!name_ref.contains("::")) {
-    if (!scope_ptr || !scope_ptr->IsValid()) {
-      // Lookup in the current frame.
-      // Try looking for a local variable in current scope.
-      lldb::ValueObjectSP value_sp;
-      if (variable_list) {
-        lldb::VariableSP var_sp =
-            DILFindVariable(ConstString(name_ref), variable_list);
-        if (var_sp)
-          value_sp =
-              stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
-      }
-      if (!value_sp)
-        value_sp = stack_frame->FindVariable(ConstString(name_ref));
-
-      if (value_sp)
-        return value_sp;
-
-      // Try looking for an instance variable (class member).
-      SymbolContext sc = stack_frame->GetSymbolContext(
-          lldb::eSymbolContextFunction | lldb::eSymbolContextBlock);
-      llvm::StringRef ivar_name = sc.GetInstanceVariableName();
-      value_sp = stack_frame->FindVariable(ConstString(ivar_name));
-      if (value_sp)
-        value_sp = value_sp->GetChildMemberWithName(name_ref);
-
-      if (value_sp)
-        return value_sp;
+    // Lookup in the current frame.
+    // Try looking for a local variable in current scope.
+    lldb::VariableListSP variable_list(
+        stack_frame->GetInScopeVariableList(false));
+
+    lldb::ValueObjectSP value_sp;
+    if (variable_list) {
+      lldb::VariableSP var_sp =
+          variable_list->FindVariable(ConstString(name_ref));
+      if (var_sp)
+        value_sp =
+            stack_frame->GetValueObjectForFrameVariable(var_sp, use_dynamic);
     }
+
+    if (value_sp)
+      return value_sp;
+
+    // Try looking for an instance variable (class member).
+    SymbolContext sc = stack_frame->GetSymbolContext(
+        lldb::eSymbolContextFunction | lldb::eSymbolContextBlock);
+    llvm::StringRef ivar_name = sc.GetInstanceVariableName();
+    value_sp = stack_frame->FindVariable(ConstString(ivar_name));
+    if (value_sp)
+      value_sp = value_sp->GetChildMemberWithName(name_ref);
+
+    if (value_sp)
+      return value_sp;
   }
   return nullptr;
 }
@@ -253,6 +185,12 @@ Interpreter::Visit(const UnaryOpNode *node) {
       rhs = dynamic_rhs;
 
     lldb::ValueObjectSP child_sp = rhs->Dereference(error);
+    if (!child_sp && m_use_synthetic) {
+      if (lldb::ValueObjectSP synth_obj_sp = rhs->GetSyntheticValue()) {
+        error.Clear();
+        child_sp = synth_obj_sp->Dereference(error);
+      }
+    }
     if (error.Fail())
       return llvm::make_error<DILDiagnosticError>(m_expr, error.AsCString(),
                                                   node->GetLocation());
@@ -280,6 +218,7 @@ Interpreter::Visit(const MemberOfNode *node) {
   auto base_or_err = Evaluate(node->GetBase());
   if (!base_or_err)
     return base_or_err;
+  bool expr_is_ptr = node->GetIsArrow();
   lldb::ValueObjectSP base = *base_or_err;
 
   // Perform some basic type & correctness checking.
@@ -319,11 +258,11 @@ Interpreter::Visit(const MemberOfNode *node) {
         return llvm::make_error<DILDiagnosticError>(
             m_expr, errMsg, node->GetLocation(), node->GetFieldName().size());
       }
+      expr_is_ptr = false;
     }
   }
 
   if (m_check_ptr_vs_member) {
-    bool expr_is_ptr = node->GetIsArrow();
     bool base_is_ptr = base->IsPointerType();
 
     if (expr_is_ptr != base_is_ptr) {
@@ -356,8 +295,8 @@ Interpreter::Visit(const MemberOfNode *node) {
 
     if (!m_use_synthetic || !field_obj) {
       std::string errMsg = llvm::formatv(
-          "no member named '{0}' in {1}", node->GetFieldName(),
-          base->GetCompilerType().GetFullyUnqualifiedType().TypeDescription());
+          "\"{0}\" is not a member of \"({1}) {2}\"", node->GetFieldName(),
+          base->GetTypeName().AsCString("<invalid type>"), base->GetName());
       return llvm::make_error<DILDiagnosticError>(
           m_expr, errMsg, node->GetLocation(), node->GetFieldName().size());
     }
@@ -376,9 +315,9 @@ Interpreter::Visit(const MemberOfNode *node) {
   CompilerType base_type = base->GetCompilerType();
   if (node->GetIsArrow() && base->IsPointerType())
     base_type = base_type.GetPointeeType();
-  std::string errMsg =
-      llvm::formatv("no member named '{0}' in {1}", node->GetFieldName(),
-                    base_type.GetFullyUnqualifiedType().TypeDescription());
+  std::string errMsg = llvm::formatv(
+      "\"{0}\" is not a member of \"({1}) {2}\"", node->GetFieldName(),
+      base->GetTypeName().AsCString("<invalid type>"), base->GetName());
   return llvm::make_error<DILDiagnosticError>(
       m_expr, errMsg, node->GetLocation(), node->GetFieldName().size());
 }
diff --git a/lldb/source/ValueObject/DILParser.cpp b/lldb/source/ValueObject/DILParser.cpp
index 5abbfeabcea3..eac41fab9076 100644
--- a/lldb/source/ValueObject/DILParser.cpp
+++ b/lldb/source/ValueObject/DILParser.cpp
@@ -183,12 +183,14 @@ ASTNodeUP DILParser::ParsePostfixExpression() {
 //    "(" expression ")"
 //
 ASTNodeUP DILParser::ParsePrimaryExpression() {
-  if (CurToken().IsOneOf({Token::coloncolon, Token::identifier})) {
+  if (CurToken().IsOneOf(
+          {Token::coloncolon, Token::identifier, Token::l_paren})) {
     // Save the source location for the diagnostics message.
     uint32_t loc = CurToken().GetLocation();
-    auto identifier = ParseIdExpression();
+    std::string identifier = ParseIdExpression();
 
-    return std::make_unique<IdentifierNode>(loc, identifier);
+    if (!identifier.empty())
+      return std::make_unique<IdentifierNode>(loc, identifier);
   }
 
   if (CurToken().Is(Token::l_paren)) {
@@ -232,16 +234,15 @@ std::string DILParser::ParseNestedNameSpecifier() {
         m_dil_lexer.LookAhead(4).Is(Token::coloncolon)) {
       m_dil_lexer.Advance(4);
 
-      assert(
-          (CurToken().Is(Token::identifier) || CurToken().Is(Token::l_paren)) &&
-          "Expected an identifier or anonymous namespace, but not found.");
+      Expect(Token::coloncolon);
+      m_dil_lexer.Advance();
+      if (!CurToken().Is(Token::identifier) && !CurToken().Is(Token::l_paren)) {
+        BailOut("Expected an identifier or anonymous namespace, but not found.",
+                CurToken().GetLocation(), CurToken().GetSpelling().length());
+      }
       // Continue parsing the nested_namespace_specifier.
       std::string identifier2 = ParseNestedNameSpecifier();
-      if (identifier2.empty()) {
-        Expect(Token::identifier);
-        identifier2 = CurToken().GetSpelling();
-        m_dil_lexer.Advance();
-      }
+
       return "(anonymous namespace)::" + identifier2;
     }
 
@@ -301,6 +302,9 @@ std::string DILParser::ParseIdExpression() {
                          nested_name_specifier, unqualified_id);
   }
 
+  if (!CurToken().Is(Token::identifier))
+    return "";
+
   // No nested_name_specifier, but with global scope -- this is also a
   // qualified_id production. Follow the second production rule.
   if (global_scope) {
diff --git a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
index ca2d2d0177ee..5c2f1f454c49 100644
--- a/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
+++ b/lldb/test/API/commands/expression/call-function/TestCallStdStringFunction.py
@@ -12,7 +12,6 @@ class ExprCommandCallFunctionTestCase(TestBase):
     @expectedFailureAll(
         compiler="icc", bugnumber="llvm.org/pr14437, fails with ICC 13.1"
     )
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr21765")
     @skipIf(compiler="clang", compiler_version=["<", "9.0"])
     def test_with(self):
         """Test calling std::String member function."""
diff --git a/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py b/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
index 9f9848912a11..c22987550efa 100644
--- a/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
+++ b/lldb/test/API/commands/expression/expr-in-syscall/TestExpressionInSyscall.py
@@ -7,16 +7,10 @@
 
 
 class ExprSyscallTestCase(TestBase):
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr21765, getpid() does not exist on Windows",
-    )
     @expectedFailureNetBSD
     def test_setpgid(self):
         self.build()
-        self.expr_syscall()
 
-    def expr_syscall(self):
         # Create a target by the debugger.
         target = self.createTestTarget()
 
@@ -65,7 +59,12 @@ def expr_syscall(self):
 
         # try evaluating a couple of expressions in this state
         self.expect_expr("release_flag = 1", result_value="1")
-        self.expect_expr("(int)getpid()", result_value=str(process.GetProcessID()))
+        func = (
+            "GetCurrentProcessId"
+            if lldbplatformutil.getPlatform() == "windows"
+            else "getpid"
+        )
+        self.expect_expr(f"(int){func}()", result_value=str(process.GetProcessID()))
 
         # and run the process to completion
         process.Continue()
diff --git a/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
index 168c4994b39b..85755a62018c 100644
--- a/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContentFromStdModule.py
@@ -23,7 +23,7 @@ def test(self):
         self.expect_expr(
             "s",
             result_type="std::shared_ptr<Foo>",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("s->a", result_type="int", result_value="3")
         self.expect_expr("s->a = 5", result_type="int", result_value="5")
diff --git a/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
index 1704099cfd85..35d62ae50f6a 100644
--- a/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/shared_ptr/TestSharedPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestSharedPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,8 +24,8 @@ def test(self):
         self.expect_expr(
             "s",
             result_type="std::shared_ptr<int>",
-            result_summary="3 strong=1 weak=1",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_summary="3 strong=1 weak=0",
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*s", result_type="element_type", result_value="3")
         self.expect_expr("*s = 5", result_type="element_type", result_value="5")
diff --git a/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
index f07b0633d32a..e95214df3b38 100644
--- a/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestDbgInfoContentWeakPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,7 +24,7 @@ def test(self):
         self.expect_expr(
             "w",
             result_type="std::weak_ptr<Foo>",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*w.lock()", result_type="element_type")
         self.expect_expr("w.lock()->a", result_type="int", result_value="3")
diff --git a/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
index a9c53548a93c..0bc0fd466acd 100644
--- a/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/weak_ptr/TestWeakPtrFromStdModule.py
@@ -10,7 +10,7 @@
 class TestSharedPtr(TestBase):
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(compiler="clang", compiler_version=['<', '17.0'])
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
     @skipUnlessDarwin
     def test(self):
         self.build()
@@ -24,8 +24,8 @@ def test(self):
         self.expect_expr(
             "w",
             result_type="std::weak_ptr<int>",
-            result_summary="3 strong=1 weak=2",
-            result_children=[ValueCheck(name="__ptr_")],
+            result_summary="3 strong=1 weak=1",
+            result_children=[ValueCheck(name="pointer")],
         )
         self.expect_expr("*w.lock()", result_type="element_type", result_value="3")
         self.expect_expr("*w.lock() = 5", result_type="element_type", result_value="5")
diff --git a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
index 0515876734ca..23188ef898d5 100644
--- a/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
+++ b/lldb/test/API/commands/expression/ir-interpreter/TestIRInterpreter.py
@@ -104,9 +104,11 @@ def build_and_run(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
+    @expectedFailureAll(
+        oslist=["windows"],
+        bugnumber="llvm.org/pr21765",
+    )
     @add_test_categories(["pyapi"])
-    # getpid() is POSIX, among other problems, see bug
-    @expectedFailureAll(oslist=["windows"], bugnumber="http://llvm.org/pr21765")
     def test_ir_interpreter(self):
         self.build_and_run()
 
@@ -137,6 +139,14 @@ def test_ir_interpreter(self):
         for expression in set_up_expressions:
             self.frame().EvaluateExpression(expression, options)
 
+        func_call = "(int)getpid()"
+        if lldbplatformutil.getPlatform() == "windows":
+            func_call = "(int)GetCurrentProcessId()"
+
+        for expression in expressions:
+            interp_expression = expression
+            jit_expression = func_call + "; " + expression
+
         for expression in expressions:
             interp_expression = expression
             jit_expression = "(int)getpid(); " + expression
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile b/lldb/test/API/commands/frame/var-dil/basics/BitField/Makefile
similarity index 75%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile
rename to lldb/test/API/commands/frame/var-dil/basics/BitField/Makefile
index c5ac05605bdc..99998b20bcb0 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/Makefile
+++ b/lldb/test/API/commands/frame/var-dil/basics/BitField/Makefile
@@ -1,4 +1,3 @@
-USE_LIBCPP := 1
 CXX_SOURCES := main.cpp
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/BitField/TestFrameVarDILBitField.py b/lldb/test/API/commands/frame/var-dil/basics/BitField/TestFrameVarDILBitField.py
new file mode 100644
index 000000000000..0e3506951909
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/BitField/TestFrameVarDILBitField.py
@@ -0,0 +1,43 @@
+"""
+Make sure 'frame var' using DIL parser/evaluator works for bit fields.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test import lldbutil
+
+import os
+import shutil
+import time
+
+
+class TestFrameVarDILBitField(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("bf.a", value="1023")
+        self.expect_var_path("bf.b", value="9")
+        self.expect_var_path("bf.c", value="false")
+        self.expect_var_path("bf.d", value="true")
+
+        self.expect_var_path("abf.a", value="1023")
+        self.expect_var_path("abf.b", value="'\\x0f'")
+        self.expect_var_path("abf.c", value="3")
+
+        # Perform an operation to ensure we actually read the value.
+        # Address-of is not allowed for bit-fields.
+        self.expect(
+            "frame variable '&bf.a'",
+            error=True,
+            substrs=["'bf.a' doesn't have a valid address"],
+        )
diff --git a/lldb/test/API/commands/frame/var-dil/basics/BitField/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/BitField/main.cpp
new file mode 100644
index 000000000000..5705cfb2df64
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/BitField/main.cpp
@@ -0,0 +1,42 @@
+#include <cstdint>
+
+int main(int argc, char **argv) {
+  enum BitFieldEnum : uint32_t { kZero, kOne };
+
+  struct BitFieldStruct {
+    uint16_t a : 10;
+    uint32_t b : 4;
+    bool c : 1;
+    bool d : 1;
+    int32_t e : 32;
+    uint32_t f : 32;
+    uint32_t g : 31;
+    uint64_t h : 31;
+    uint64_t i : 33;
+    BitFieldEnum j : 10;
+  };
+
+  BitFieldStruct bf;
+  bf.a = 0b1111111111;
+  bf.b = 0b1001;
+  bf.c = 0b0;
+  bf.d = 0b1;
+  bf.e = 0b1;
+  bf.f = 0b1;
+  bf.g = 0b1;
+  bf.h = 0b1;
+  bf.i = 0b1;
+  bf.j = BitFieldEnum::kOne;
+
+  struct AlignedBitFieldStruct {
+    uint16_t a : 10;
+    uint8_t b : 4;
+    unsigned char : 0;
+    uint16_t c : 2;
+  };
+
+  uint32_t data = ~0;
+  AlignedBitFieldStruct abf = (AlignedBitFieldStruct &)data;
+
+  return 0; // Set a breakpoint here
+}
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
index 99998b20bcb0..c39bce9a94dc 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/Makefile
@@ -1,3 +1,3 @@
-CXX_SOURCES := main.cpp
+CXX_SOURCES := main.cpp extern.cpp
 
 include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
index edb013c7b352..9cb166e3d0dc 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/TestFrameVarDILGlobalVariableLookup.py
@@ -18,6 +18,7 @@ class TestFrameVarDILGlobalVariableLookup(TestBase):
     # each debug info format.
     NO_DEBUG_INFO_TESTCASE = True
 
+    @skipIf(macos_version=["<", "15.0"], archs=["arm64", "arm64e"])
     def test_frame_var(self):
         self.build()
         lldbutil.run_to_source_breakpoint(
@@ -32,20 +33,20 @@ def test_frame_var(self):
         self.expect_var_path("::globalPtr", type="int *")
         self.expect_var_path("::globalRef", type="int &")
 
-        self.expect(
-            "frame variable 'externGlobalVar'",
-            error=True,
-            substrs=["use of undeclared identifier"],
-        )  # 0x00C0FFEE
-        self.expect(
-            "frame variable '::externGlobalVar'",
-            error=True,
-            substrs=["use of undeclared identifier"],
-        )  # ["12648430"])
-
         self.expect_var_path("ns::globalVar", value="13")
         self.expect_var_path("ns::globalPtr", type="int *")
         self.expect_var_path("ns::globalRef", type="int &")
         self.expect_var_path("::ns::globalVar", value="13")
         self.expect_var_path("::ns::globalPtr", type="int *")
         self.expect_var_path("::ns::globalRef", type="int &")
+
+        self.expect_var_path("externGlobalVar", value="2")
+        self.expect_var_path("::externGlobalVar", value="2")
+        self.expect_var_path("ext::externGlobalVar", value="4")
+        self.expect_var_path("::ext::externGlobalVar", value="4")
+
+        self.expect_var_path("ExtStruct::static_inline", value="16")
+
+        # Test local variable priority over global
+        self.expect_var_path("foo", value="1")
+        self.expect_var_path("::foo", value="2")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp
new file mode 100644
index 000000000000..c451191dafc5
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/extern.cpp
@@ -0,0 +1,10 @@
+int externGlobalVar = 2;
+
+namespace ext {
+int externGlobalVar = 4;
+} // namespace ext
+
+struct ExtStruct {
+private:
+  static constexpr inline int static_inline = 16;
+} es;
diff --git a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
index 5bae4fd423e3..a8ecbe2d8fc4 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
+++ b/lldb/test/API/commands/frame/var-dil/basics/GlobalVariableLookup/main.cpp
@@ -10,6 +10,9 @@ int *globalPtr = &globalVar;
 int &globalRef = globalVar;
 } // namespace ns
 
+int foo = 2;
+
 int main(int argc, char **argv) {
+  int foo = 1;
   return 0; // Set a breakpoint here
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile b/lldb/test/API/commands/frame/var-dil/basics/Indirection/Makefile
similarity index 70%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
rename to lldb/test/API/commands/frame/var-dil/basics/Indirection/Makefile
index bf8e6b8703f3..99998b20bcb0 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/Makefile
+++ b/lldb/test/API/commands/frame/var-dil/basics/Indirection/Makefile
@@ -1,5 +1,3 @@
 CXX_SOURCES := main.cpp
 
-USE_LIBSTDCPP := 1
-
 include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/Indirection/TestFrameVarDILIndirection.py b/lldb/test/API/commands/frame/var-dil/basics/Indirection/TestFrameVarDILIndirection.py
new file mode 100644
index 000000000000..38c72131d797
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/Indirection/TestFrameVarDILIndirection.py
@@ -0,0 +1,46 @@
+"""
+Make sure 'frame var' using DIL parser/evaluator works for indirection.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test import lldbutil
+
+import os
+import shutil
+import time
+
+
+class TestFrameVarDILIndirection(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("*p", value="1")
+        self.expect_var_path("p", type="int *")
+        self.expect_var_path("*my_p", value="1")
+        self.expect_var_path("my_p", type="myp")
+        self.expect_var_path("*my_pr", type="int *")
+        self.expect_var_path("my_pr", type="mypr")
+
+        self.expect(
+            "frame variable '*1'",
+            error=True,
+            substrs=["Unexpected token: <'1' (numeric_constant)>"],
+        )
+        self.expect(
+            "frame variable '*val'",
+            error=True,
+            substrs=[
+                "dereference failed: not a pointer, reference or array type: (int) val"
+            ],
+        )
diff --git a/lldb/test/API/commands/frame/var-dil/basics/Indirection/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/Indirection/main.cpp
new file mode 100644
index 000000000000..9fbd99b219ce
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/Indirection/main.cpp
@@ -0,0 +1,12 @@
+int main(int argc, char **argv) {
+  int val = 1;
+  int *p = &val;
+
+  typedef int *myp;
+  myp my_p = &val;
+
+  typedef int *&mypr;
+  mypr my_pr = p;
+
+  return 0; // Set a breakpoint here
+}
diff --git a/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py b/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
index bb16c1f82489..ca6754a556d8 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/MemberOf/TestFrameVarDILMemberOf.py
@@ -31,12 +31,19 @@ def test_frame_var(self):
         self.expect_var_path("sp->x", value="1")
         self.expect_var_path("sp->r", type="int &")
 
-        self.expect("frame variable 'sp->foo'", error=True,
-                    substrs=["no member named 'foo' in 'Sx *'"])
-
-        self.expect("frame variable 'sp.x'", error=True,
-                    substrs=["member reference type 'Sx *' is a "
-                             "pointer; did you mean to use '->'"])
+        self.expect(
+            "frame variable 'sp->foo'",
+            error=True,
+            substrs=['"foo" is not a member of "(Sx *) sp"'],
+        )
+
+        self.expect(
+            "frame variable 'sp.x'",
+            error=True,
+            substrs=[
+                "member reference type 'Sx *' is a pointer; did you mean to use '->'"
+            ],
+        )
 
         # Test for record typedefs.
         self.expect_var_path("sa.x", value="3")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py b/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
index 1bde4706da90..856b3a069eec 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/MemberOfAnonymousMember/TestFrameVarDILMemberOfAnonymousMember.py
@@ -27,8 +27,11 @@ def test_frame_var(self):
         self.expect_var_path("a.x", value="1")
         self.expect_var_path("a.y", value="2")
 
-        self.expect("frame variable 'b.x'", error=True,
-                    substrs=["no member named 'x' in 'B'"])
+        self.expect(
+            "frame variable 'b.x'",
+            error=True,
+            substrs=['"x" is not a member of "(B) b"'],
+        )
         #self.expect_var_path("b.y", value="0")
         self.expect_var_path("b.z", value="3")
         self.expect_var_path("b.w", value="4")
@@ -43,19 +46,31 @@ def test_frame_var(self):
         self.expect_var_path("d.z", value="9")
         self.expect_var_path("d.w", value="10")
 
-        self.expect("frame variable 'e.x'", error=True,
-                    substrs=["no member named 'x' in 'E'"])
-        self.expect("frame variable 'f.x'", error=True,
-                    substrs=["no member named 'x' in 'F'"])
+        self.expect(
+            "frame variable 'e.x'",
+            error=True,
+            substrs=['"x" is not a member of "(E) e"'],
+        )
+        self.expect(
+            "frame variable 'f.x'",
+            error=True,
+            substrs=['"x" is not a member of "(F) f"'],
+        )
         self.expect_var_path("f.named_field.x", value="12")
 
         self.expect_var_path("unnamed_derived.y", value="2")
         self.expect_var_path("unnamed_derived.z", value="13")
 
-        self.expect("frame variable 'derb.x'", error=True,
-                    substrs=["no member named 'x' in 'DerivedB'"])
-        self.expect("frame variable 'derb.y'", error=True,
-                    substrs=["no member named 'y' in 'DerivedB'"])
+        self.expect(
+            "frame variable 'derb.x'",
+            error=True,
+            substrs=['"x" is not a member of "(DerivedB) derb"'],
+        )
+        self.expect(
+            "frame variable 'derb.y'",
+            error=True,
+            substrs=['"y" is not a member of "(DerivedB) derb"'],
+        )
         self.expect_var_path("derb.w", value="14")
         self.expect_var_path("derb.k", value="15")
         self.expect_var_path("derb.a.x", value="1")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/Makefile b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/TestFrameVarDILPointerDereference.py b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/TestFrameVarDILPointerDereference.py
new file mode 100644
index 000000000000..ffb447441e98
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/TestFrameVarDILPointerDereference.py
@@ -0,0 +1,48 @@
+"""
+Test DIL pointer dereferencing.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test import lldbutil
+
+import os
+import shutil
+import time
+
+
+class TestFrameVarDILPointerDereference(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("*p_int0", value="0")
+        self.expect_var_path("*cp_int5", value="5")
+        self.expect_var_path("&pp_void0[2]", type="void **")
+        self.expect_var_path("**pp_int0", value="0")
+        self.expect_var_path("&**pp_int0", type="int *")
+        self.expect(
+            "frame variable '&p_void[0]'",
+            error=True,
+            substrs=["subscript of pointer to incomplete type 'void'"],
+        )
+
+        # Verify some of the returned values.
+        pp_void0_2_got = self.expect_var_path("&pp_void0[2]", type="void **")
+        # Initialized in C++ code to point to the same value
+        pp_void0_2_exp = self.expect_var_path("pp_void0_2", type="void **")
+        self.assertEqual(
+            pp_void0_2_got.GetValueAsAddress(), pp_void0_2_exp.GetValueAsAddress()
+        )
+        pp_int0_2stars_got = self.expect_var_path("&**pp_int0", type="int *")
+        pp_int0_2stars_exp = self.expect_var_path("pp_int0_2stars", type="int *")
+        self.assertEqual(
+            pp_int0_2stars_got.GetValueAsAddress(),
+            pp_int0_2stars_exp.GetValueAsAddress(),
+        )
diff --git a/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/main.cpp
new file mode 100644
index 000000000000..0c39609dd591
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/PointerDereference/main.cpp
@@ -0,0 +1,34 @@
+#include <cstddef>
+
+int main(int argc, char **argv) {
+  int *p_null = nullptr;
+  const char *p_char1 = "hello";
+
+  typedef const char *my_char_ptr;
+  my_char_ptr my_p_char1 = p_char1;
+
+  int offset = 5;
+  int array[10];
+  array[0] = 0;
+  array[offset] = offset;
+
+  int (&array_ref)[10] = array;
+
+  int *p_int0 = &array[0];
+  int **pp_int0 = &p_int0;
+  const int *cp_int0 = &array[0];
+  const int *cp_int5 = &array[offset];
+
+  typedef int *td_int_ptr_t;
+  td_int_ptr_t td_int_ptr0 = &array[0];
+
+  void *p_void = (void *)p_char1;
+  void **pp_void0 = &p_void;
+  void **pp_void1 = pp_void0 + 1;
+
+  void **pp_void0_2 = &pp_void0[2];
+  int *pp_int0_2stars = &**pp_int0;
+  std::nullptr_t std_nullptr_t = nullptr;
+
+  return 0; // Set a breakpoint here
+}
diff --git a/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/Makefile b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/TestFrameVarDILQualifiedId.py b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/TestFrameVarDILQualifiedId.py
new file mode 100644
index 000000000000..b2ce9602e6a5
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/TestFrameVarDILQualifiedId.py
@@ -0,0 +1,31 @@
+"""
+Make sure 'frame var' using DIL parser/evaluator works for namespaces.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test import lldbutil
+
+import os
+import shutil
+import time
+
+
+class TestFrameVarDILQualifiedId(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("::ns::i", value="1")
+        self.expect_var_path("ns::i", value="1")
+        self.expect_var_path("::ns::ns::i", value="2")
+        self.expect_var_path("ns::ns::i", value="2")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/main.cpp
new file mode 100644
index 000000000000..8a5c47a6f364
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/QualifiedId/main.cpp
@@ -0,0 +1,16 @@
+namespace ns {
+
+int i = 1;
+
+namespace ns {
+
+int i = 2;
+
+} // namespace ns
+
+} // namespace ns
+
+int main(int argc, char **argv) {
+
+  return 0; // Set a breakpoint here
+}
diff --git a/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/Makefile b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/TestFrameVarDILSyntheticDereference.py b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/TestFrameVarDILSyntheticDereference.py
new file mode 100644
index 000000000000..e67f63846175
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/TestFrameVarDILSyntheticDereference.py
@@ -0,0 +1,34 @@
+"""
+Test code for dereferencing synthetic wrapped pointers.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class DILSyntheticDereferenceTestCase(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_frame_var(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(
+            self, "Set a breakpoint here", lldb.SBFileSpec("main.cpp")
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.runCmd("script from wrapPtrSynthProvider import *")
+        self.runCmd("type synth add -l wrapPtrSynthProvider wrap_ptr")
+
+        self.expect_var_path("ptr_node->value", value="1")
+        self.expect_var_path("ptr_node->next->value", value="2")
+        self.expect_var_path("(*ptr_node).value", value="1")
+        self.expect_var_path("(*(*ptr_node).next).value", value="2")
+
+        self.expect_var_path("ptr_node.ptr", type="NodeS *")
+        self.expect_var_path("ptr_node.ptr->value", value="1")
+        self.expect_var_path("ptr_node.ptr->next.ptr->value", value="2")
diff --git a/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/main.cpp
new file mode 100644
index 000000000000..dc9c01b46d58
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/main.cpp
@@ -0,0 +1,26 @@
+#include <memory>
+
+struct NodeS;
+
+// Class to wrap pointers.
+class wrap_ptr {
+public:
+  NodeS *ptr;
+
+  wrap_ptr(NodeS *n_ptr) : ptr(n_ptr) {}
+};
+
+struct NodeS {
+  wrap_ptr next;
+  int value;
+
+  NodeS(NodeS *n_ptr, int val) : next(wrap_ptr(n_ptr)), value(val) {}
+};
+
+int main(int argc, char **argv) {
+
+  // Make a short linked list of fake smart pointers.
+  auto ptr_node = wrap_ptr(new NodeS(new NodeS(nullptr, 2), 1));
+
+  return 0; // Set a breakpoint here
+}
diff --git a/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/wrapPtrSynthProvider.py b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/wrapPtrSynthProvider.py
new file mode 100644
index 000000000000..f89e426c0fd7
--- /dev/null
+++ b/lldb/test/API/commands/frame/var-dil/basics/SyntheticDereference/wrapPtrSynthProvider.py
@@ -0,0 +1,29 @@
+import lldb
+
+
+class wrapPtrSynthProvider:
+    def __init__(self, valobj, dict):
+        self.valobj = valobj
+
+    def num_children(self):
+        return 1
+
+    def get_child_at_index(self, index):
+        if index == 0:
+            return self.valobj.GetChildMemberWithName("ptr")
+        if index == 1:
+            internal_child = self.valobj.GetChildMemberWithName("ptr")
+            if not internal_child:
+                return None
+            return internal_child.Dereference()
+        return None
+
+    def get_child_index(self, name):
+        if name == "ptr":
+            return 0
+        if name == "$$dereference$$":
+            return 1
+        return -1
+
+    def update(self):
+        return True
diff --git a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
index c365bc993e33..45d855f5c193 100644
--- a/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
+++ b/lldb/test/API/commands/platform/launchgdbserver/TestPlatformLaunchGDBServer.py
@@ -1,8 +1,3 @@
-""" Check that errors while handling qLaunchGDBServer are reported to the user.
-    Though this isn't a platform command in itself, the best way to test it is
-    from Python because we can juggle multiple processes more easily.
-"""
-
 import os
 import socket
 import shutil
@@ -15,14 +10,7 @@
 class TestPlatformProcessLaunchGDBServer(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
-    @skipIfRemote
-    # Windows cannot delete the executable while it is running.
-    # On Darwin we may be using debugserver.
-    @skipUnlessPlatform(["linux"])
-    @add_test_categories(["lldb-server"])
-    def test_platform_process_launch_gdb_server(self):
-        self.build()
-
+    def _launch_and_connect(self, exe):
         hostname = socket.getaddrinfo("localhost", 0, proto=socket.IPPROTO_TCP)[0][4][0]
         listen_url = "[%s]:0" % hostname
 
@@ -33,16 +21,9 @@ def test_platform_process_launch_gdb_server(self):
             listen_url,
             "--socket-file",
             port_file,
-            "--",
-            self.getBuildArtifact("a.out"),
-            "foo",
         ]
 
-        # Run lldb-server from a new location.
-        new_lldb_server = self.getBuildArtifact("lldb-server")
-        shutil.copy(lldbgdbserverutils.get_lldb_server_exe(), new_lldb_server)
-
-        self.spawnSubprocess(new_lldb_server, commandline_args)
+        self.spawnSubprocess(exe, commandline_args)
         socket_id = lldbutil.wait_for_file_on_target(self, port_file)
 
         new_platform = lldb.SBPlatform("remote-" + self.getPlatform())
@@ -51,10 +32,52 @@ def test_platform_process_launch_gdb_server(self):
         connect_url = "connect://[%s]:%s" % (hostname, socket_id)
         self.runCmd("platform connect %s" % connect_url)
 
-        # First connect to lldb-server which spawn a process to handle the connection.
-        # Then remove our new lldb-server so that when it tries to invoke itself as a
+        wd = self.getBuildArtifact("wd")
+        os.mkdir(wd)
+        new_platform.SetWorkingDirectory(wd)
+
+    @skipIfRemote
+    # Windows cannot delete the executable while it is running.
+    # On Darwin we may be using debugserver.
+    @skipUnlessPlatform(["linux"])
+    @add_test_categories(["lldb-server"])
+    def test_launch_error(self):
+        """
+        Check that errors while handling qLaunchGDBServer are reported to the
+        user.  Though this isn't a platform command in itself, the best way to
+        test it is from Python because we can juggle multiple processes more
+        easily.
+        """
+
+        self.build()
+
+        # Run lldb-server from a new location.
+        new_lldb_server = self.getBuildArtifact("lldb-server")
+        shutil.copy(lldbgdbserverutils.get_lldb_server_exe(), new_lldb_server)
+        self._launch_and_connect(new_lldb_server)
+
+        # Now, remove our new lldb-server so that when it tries to invoke itself as a
         # gdbserver, it fails.
         os.remove(new_lldb_server)
 
         self.runCmd("target create {}".format(self.getBuildArtifact("a.out")))
         self.expect("run", substrs=["unable to launch a GDB server on"], error=True)
+
+    @skipIfRemote
+    @skipIfDarwin  # Uses debugserver for debugging
+    @add_test_categories(["lldb-server"])
+    def test_launch_with_unusual_process_name(self):
+        """
+        Test that lldb-server can launch a debug session when running under an
+        unusual name (or under a symlink which resolves to an unusal name).
+        """
+
+        self.build()
+
+        # Run lldb-server from a new location.
+        new_lldb_server = self.getBuildArtifact("obfuscated-server")
+        shutil.copy(lldbgdbserverutils.get_lldb_server_exe(), new_lldb_server)
+        self._launch_and_connect(new_lldb_server)
+
+        self.runCmd("target create {}".format(self.getBuildArtifact("a.out")))
+        self.expect("run", substrs=["exited with status = 0"])
diff --git a/lldb/test/API/commands/protocol/TestMCPUnixSocket.py b/lldb/test/API/commands/protocol/TestMCPUnixSocket.py
new file mode 100644
index 000000000000..ea9255cc60ef
--- /dev/null
+++ b/lldb/test/API/commands/protocol/TestMCPUnixSocket.py
@@ -0,0 +1,34 @@
+import os
+import tempfile
+import unittest
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+# To be safe and portable, Unix domain socket paths should be kept at or below
+# 108 characters on Linux, and around 104 characters on macOS:
+MAX_SOCKET_PATH_LENGTH = 104
+
+
+class MCPUnixSocketCommandTestCase(TestBase):
+    @skipIfWindows
+    @skipIfRemote
+    @no_debug_info_test
+    def test_unix_socket(self):
+        """
+        Test if we can start an MCP protocol-server accepting unix sockets
+        """
+
+        temp_directory = tempfile.TemporaryDirectory()
+        socket_file = os.path.join(temp_directory.name, "mcp.sock")
+
+        if len(socket_file) >= MAX_SOCKET_PATH_LENGTH:
+            self.skipTest(
+                f"Socket path {socket_file} exceeds the {MAX_SOCKET_PATH_LENGTH} character limit"
+            )
+
+        self.expect(
+            f"protocol-server start MCP accept://{socket_file}",
+            startstr="MCP server started with connection listeners:",
+            substrs=[f"unix-connect://{socket_file}"],
+        )
diff --git a/lldb/test/API/commands/statistics/basic/TestStats.py b/lldb/test/API/commands/statistics/basic/TestStats.py
index 83132b40d85d..e9ee8b8661e5 100644
--- a/lldb/test/API/commands/statistics/basic/TestStats.py
+++ b/lldb/test/API/commands/statistics/basic/TestStats.py
@@ -177,6 +177,8 @@ def test_default_no_run(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoParseTime",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         if self.getPlatform() != "windows":
@@ -287,6 +289,8 @@ def test_default_with_run(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoParseTime",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         stats = debug_stats["targets"][0]
@@ -325,6 +329,8 @@ def test_memory(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 
@@ -377,6 +383,8 @@ def test_modules(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         stats = debug_stats["targets"][0]
@@ -397,6 +405,8 @@ def test_modules(self):
             "symbolTableLoadedFromCache",
             "symbolTableParseTime",
             "symbolTableSavedToCache",
+            "dwoFileCount",
+            "loadedDwoFileCount",
             "triple",
             "uuid",
         ]
@@ -485,6 +495,8 @@ def test_breakpoints(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         target_stats = debug_stats["targets"][0]
@@ -513,6 +525,136 @@ def test_breakpoints(self):
                 breakpoint, 'target_stats["breakpoints"]', bp_keys_exist, None
             )
 
+    @add_test_categories(["dwo"])
+    def test_non_split_dwarf_has_no_dwo_files(self):
+        """
+        Test "statistics dump" and the dwo file count.
+        Builds a binary without split-dwarf mode, and then
+        verifies the dwo file count is zero after running "statistics dump"
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+
+        # Verify that the dwo file count is zero
+        self.assertEqual(debug_stats["totalDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+
+    @add_test_categories(["dwo"])
+    def test_no_debug_names_eager_loads_dwo_files(self):
+        """
+        Test the eager loading behavior of DWO files when debug_names is absent by
+        building a split-dwarf binary without debug_names and then running "statistics dump".
+        DWO file loading behavior:
+        - With debug_names: DebugNamesDWARFIndex allows for lazy loading.
+          DWO files are loaded on-demand when symbols are actually looked up
+        - Without debug_names: ManualDWARFIndex uses eager loading.
+          All DWO files are loaded upfront during the first symbol lookup to build a manual index.
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["dwo"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+
+        # Verify that all DWO files are loaded
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+
+    @add_test_categories(["dwo"])
+    def test_split_dwarf_dwo_file_count(self):
+        """
+        Test "statistics dump" and the dwo file count.
+        Builds a binary w/ separate .dwo files and debug_names, and then
+        verifies the loaded dwo file count is the expected count after running
+        various commands
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        # -gsplit-dwarf creates separate .dwo files,
+        # -gpubnames enables the debug_names accelerator tables for faster symbol lookup
+        #  and lazy loading of DWO files
+        # Expected output: third.dwo (contains main) and baz.dwo (contains Baz struct/function)
+        self.build(dictionary=da, debug_info=["dwo", "debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+
+        # 1) 2 DWO files available but none loaded yet
+        self.assertEqual(len(debug_stats["modules"]), 1)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Since there's only one module, module stats should have the same counts as total counts
+        self.assertIn("dwoFileCount", debug_stats["modules"][0])
+        self.assertIn("loadedDwoFileCount", debug_stats["modules"][0])
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+        # 2) Setting breakpoint in main triggers loading of third.dwo (contains main function)
+        self.runCmd("b main")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+        # 3) Type lookup forces loading of baz.dwo (contains struct Baz definition)
+        self.runCmd("type lookup Baz")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 2)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+    @add_test_categories(["dwo"])
+    def test_dwp_dwo_file_count(self):
+        """
+        Test "statistics dump" and the loaded dwo file count.
+        Builds a binary w/ a separate .dwp file and debug_names, and then
+        verifies the loaded dwo file count is the expected count after running
+        various commands.
+
+        We expect the DWO file counters to reflect the number of compile units
+        loaded from the DWP file (each representing what was originally a separate DWO file)
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["dwp", "debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+
+        # Initially: 2 DWO files available but none loaded yet
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Setting breakpoint in main triggers parsing of the CU within a.dwp corresponding to third.dwo (contains main function)
+        self.runCmd("b main")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Type lookup forces parsing of the CU within a.dwp corresponding to baz.dwo (contains struct Baz definition)
+        self.runCmd("type lookup Baz")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+
     @skipUnlessDarwin
     @no_debug_info_test
     def test_dsym_binary_has_symfile_in_stats(self):
@@ -707,6 +849,30 @@ def test_transcript_happy_path(self):
         # The second "statistics dump" in the transcript should have no output
         self.assertNotIn("output", transcript[2])
 
+    def test_transcript_warning_when_disabled(self):
+        """
+        Test that "statistics dump --transcript=true" shows a warning when
+        transcript saving is disabled.
+        """
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+
+        # Ensure transcript saving is disabled (this is the default)
+        self.runCmd("settings set interpreter.save-transcript false")
+
+        # Request transcript in statistics dump and check for warning
+        interpreter = self.dbg.GetCommandInterpreter()
+        res = lldb.SBCommandReturnObject()
+        interpreter.HandleCommand("statistics dump --transcript=true", res)
+        self.assertTrue(res.Succeeded())
+        # We should warn about transcript being requested but not saved
+        self.assertIn(
+            "transcript requested but none was saved. Enable with "
+            "'settings set interpreter.save-transcript true'",
+            res.GetError(),
+        )
+
     def verify_stats(self, stats, expectation, options):
         for field_name in expectation:
             idx = field_name.find(".")
@@ -754,7 +920,7 @@ def get_test_cases_for_sections_existence(self):
                     "targets.frameVariable": True,
                     "targets.totalSharedLibraryEventHitCount": True,
                     "modules": True,
-                    "transcript": True,
+                    "transcript": False,
                 },
             },
             {  # Summary mode
@@ -841,6 +1007,24 @@ def get_test_cases_for_sections_existence(self):
                     "transcript": False,
                 },
             },
+            {  # Default mode without modules and with transcript
+                "command_options": " --modules=false --transcript=true",
+                "api_options": {
+                    "SetIncludeModules": False,
+                    "SetIncludeTranscript": True,
+                },
+                "expect": {
+                    "commands": True,
+                    "targets": True,
+                    "targets.moduleIdentifiers": False,
+                    "targets.breakpoints": True,
+                    "targets.expressionEvaluation": True,
+                    "targets.frameVariable": True,
+                    "targets.totalSharedLibraryEventHitCount": True,
+                    "modules": False,
+                    "transcript": True,
+                },
+            },
             {  # Default mode without modules
                 "command_options": " --modules=false",
                 "api_options": {
@@ -855,6 +1039,23 @@ def get_test_cases_for_sections_existence(self):
                     "targets.frameVariable": True,
                     "targets.totalSharedLibraryEventHitCount": True,
                     "modules": False,
+                    "transcript": False,
+                },
+            },
+            {  # Default mode with transcript
+                "command_options": " --transcript=true",
+                "api_options": {
+                    "SetIncludeTranscript": True,
+                },
+                "expect": {
+                    "commands": True,
+                    "targets": True,
+                    "targets.moduleIdentifiers": True,
+                    "targets.breakpoints": True,
+                    "targets.expressionEvaluation": True,
+                    "targets.frameVariable": True,
+                    "targets.totalSharedLibraryEventHitCount": True,
+                    "modules": True,
                     "transcript": True,
                 },
             },
diff --git a/lldb/test/API/commands/statistics/basic/baz.cpp b/lldb/test/API/commands/statistics/basic/baz.cpp
new file mode 100644
index 000000000000..536758b17d83
--- /dev/null
+++ b/lldb/test/API/commands/statistics/basic/baz.cpp
@@ -0,0 +1,12 @@
+// Helper that the lldb command `statistics dump` works in split-dwarf mode.
+
+struct Baz {
+  int x;
+  bool y;
+};
+
+void baz() {
+  Baz b;
+  b.x = 1;
+  b.y = true;
+}
diff --git a/lldb/test/API/commands/statistics/basic/third.cpp b/lldb/test/API/commands/statistics/basic/third.cpp
new file mode 100644
index 000000000000..3943b9c2faaf
--- /dev/null
+++ b/lldb/test/API/commands/statistics/basic/third.cpp
@@ -0,0 +1,7 @@
+// Test that the lldb command `statistics dump` works.
+
+void baz();
+int main(void) {
+  baz();
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/Makefile
new file mode 100644
index 000000000000..22f1051530f8
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/Makefile
@@ -0,0 +1 @@
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/TestLocationsAfterRebuild.py b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/TestLocationsAfterRebuild.py
new file mode 100644
index 000000000000..1c7bb538d5df
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/TestLocationsAfterRebuild.py
@@ -0,0 +1,59 @@
+"""
+When a rebuild causes a location to be removed, make sure
+we still handle the remaining locations correctly.
+"""
+
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import skipIfWindows
+import os
+
+
+class TestLocationsAfterRebuild(TestBase):
+    # If your test case doesn't stress debug info, then
+    # set this to true.  That way it won't be run once for
+    # each debug info format.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    # On Windows we cannot remove a file that lldb is debugging.
+    @skipIfWindows
+    def test_remaining_location_spec(self):
+        """If we rebuild a couple of times some of the old locations
+        get removed.  Make sure the command-line breakpoint id
+        validator still works correctly."""
+        self.build(dictionary={"C_SOURCES": "main.c", "EXE": "a.out"})
+
+        path_to_exe = self.getBuildArtifact()
+
+        (target, process, thread, bkpt) = lldbutil.run_to_name_breakpoint(self, "main")
+
+        # Let the process continue to exit:
+        process.Continue()
+        self.assertEqual(process.GetState(), lldb.eStateExited, "Ran to completion")
+        os.remove(path_to_exe)
+
+        # We have to rebuild twice with changed sources to get
+        # us to remove the first set of locations:
+        self.build(dictionary={"C_SOURCES": "second_main.c", "EXE": "a.out"})
+
+        (target, process, thread, bkpt) = lldbutil.run_to_breakpoint_do_run(
+            self, target, bkpt
+        )
+
+        # Let the process continue to exit:
+        process.Continue()
+        self.assertEqual(process.GetState(), lldb.eStateExited, "Ran to completion")
+
+        os.remove(path_to_exe)
+
+        self.build(dictionary={"C_SOURCES": "third_main.c", "EXE": "a.out"})
+
+        (target, process, thread, bkpt) = lldbutil.run_to_breakpoint_do_run(
+            self, target, bkpt
+        )
+
+        bkpt_id = bkpt.GetID()
+        loc_string = f"{bkpt_id}.3"
+        self.runCmd(f"break disable {loc_string}")
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/main.c
new file mode 100644
index 000000000000..d06ee7a053bf
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/main.c
@@ -0,0 +1 @@
+int main() { return 1111; }
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/second_main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/second_main.c
new file mode 100644
index 000000000000..a3090f9a8632
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/second_main.c
@@ -0,0 +1 @@
+int main() { return 22222; }
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/third_main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/third_main.c
new file mode 100644
index 000000000000..07027eaa1605
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_locations/after_rebuild/third_main.c
@@ -0,0 +1 @@
+int main() { return 33333; }
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
index 0ab5dd0f910f..0ad903befc65 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/base.py
@@ -14,5 +14,15 @@ def supports_hw_breakpoints(self):
         self.runCmd("breakpoint set -b main --hardware")
         self.runCmd("run")
         if "stopped" in self.res.GetOutput():
+            return True
+        return False
+
+    def hw_breakpoints_supported(self):
+        if self.supports_hw_breakpoints():
             return "Hardware breakpoints are supported"
         return None
+
+    def hw_breakpoints_unsupported(self):
+        if not self.supports_hw_breakpoints():
+            return "Hardware breakpoints are unsupported"
+        return None
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
index 1a0515aa04c0..4632c3bed189 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/hardware_breakpoint_on_multiple_threads/TestHWBreakMultiThread.py
@@ -12,16 +12,13 @@
 
 
 class HardwareBreakpointMultiThreadTestCase(HardwareBreakpointTestBase):
-    def does_not_support_hw_breakpoints(self):
-        return not super().supports_hw_breakpoints()
-
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_unsupported)
     def test_hw_break_set_delete_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
         self.break_multi_thread("delete")
 
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_unsupported)
     def test_hw_break_set_disable_multi_thread_macos(self):
         self.build()
         self.setTearDownCleanup()
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
index 89d57683a800..a8c9cdeea936 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/require_hw_breakpoints/TestRequireHWBreakpoints.py
@@ -26,7 +26,7 @@ def test_breakpoint(self):
         breakpoint = target.BreakpointCreateByLocation("main.c", 1)
         self.assertTrue(breakpoint.IsHardware())
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_range(self):
         """Test stepping when hardware breakpoints are required."""
         self.build()
@@ -49,7 +49,7 @@ def test_step_range(self):
             "Could not create hardware breakpoint for thread plan", error.GetCString()
         )
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_out(self):
         """Test stepping out when hardware breakpoints are required."""
         self.build()
@@ -71,7 +71,7 @@ def test_step_out(self):
             "Could not create hardware breakpoint for thread plan", error.GetCString()
         )
 
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_over(self):
         """Test stepping over when hardware breakpoints are required."""
         self.build()
@@ -91,7 +91,7 @@ def test_step_over(self):
 
     # Was reported to sometimes pass on certain hardware.
     @skipIf(oslist=["linux"], archs=["arm$"])
-    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.hw_breakpoints_supported)
     def test_step_until(self):
         """Test stepping until when hardware breakpoints are required."""
         self.build()
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile
new file mode 100644
index 000000000000..304633c2dca1
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/Makefile
@@ -0,0 +1,7 @@
+C_SOURCES := main.c
+
+ifeq ($(CC_TYPE), icc)
+    CFLAGS_EXTRAS := -debug inline-debug-info
+endif
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py
new file mode 100644
index 000000000000..ccbb23507c1b
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/TestSimpleHWBreakpoints.py
@@ -0,0 +1,50 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+from functionalities.breakpoint.hardware_breakpoints.base import *
+
+
+class SimpleHWBreakpointTest(HardwareBreakpointTestBase):
+    def does_not_support_hw_breakpoints(self):
+        # FIXME: Use HardwareBreakpointTestBase.supports_hw_breakpoints
+        if super().supports_hw_breakpoints() is None:
+            return "Hardware breakpoints are unsupported"
+        return None
+
+    @skipTestIfFn(does_not_support_hw_breakpoints)
+    def test(self):
+        """Test SBBreakpoint::SetIsHardware"""
+        self.build()
+
+        # Set a breakpoint on main.
+        target, process, _, main_bp = lldbutil.run_to_source_breakpoint(
+            self, "main", lldb.SBFileSpec("main.c")
+        )
+
+        break_on_me_bp = target.BreakpointCreateByLocation("main.c", 1)
+
+        self.assertFalse(main_bp.IsHardware())
+        self.assertFalse(break_on_me_bp.IsHardware())
+        self.assertGreater(break_on_me_bp.GetNumResolvedLocations(), 0)
+
+        error = break_on_me_bp.SetIsHardware(True)
+
+        # Regardless of whether we succeeded in updating all the locations, the
+        # breakpoint will be marked as a hardware breakpoint.
+        self.assertTrue(break_on_me_bp.IsHardware())
+
+        if super().supports_hw_breakpoints():
+            self.assertSuccess(error)
+
+            # Continue to our Hardware breakpoint and verify that's the reason
+            # we're stopped.
+            process.Continue()
+            self.expect(
+                "thread list",
+                STOPPED_DUE_TO_BREAKPOINT,
+                substrs=["stopped", "stop reason = breakpoint"],
+            )
+        else:
+            self.assertFailure(error)
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c
new file mode 100644
index 000000000000..4f1c9df56200
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/simple_hw_breakpoints/main.c
@@ -0,0 +1,7 @@
+int break_on_me() {
+  int i = 10;
+  i++;
+  return i;
+}
+
+int main() { return break_on_me(); }
diff --git a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
index e4e25ae877de..c82ae24a6d9a 100644
--- a/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
+++ b/lldb/test/API/functionalities/breakpoint/hardware_breakpoints/write_memory_with_hw_breakpoint/TestWriteMemoryWithHWBreakpoint.py
@@ -12,10 +12,9 @@
 
 
 class WriteMemoryWithHWBreakpoint(HardwareBreakpointTestBase):
-    def does_not_support_hw_breakpoints(self):
-        return not super().supports_hw_breakpoints()
 
-    @skipTestIfFn(does_not_support_hw_breakpoints)
+    @skipTestIfFn(HardwareBreakpointTestBase.supports_hw_breakpoints)
+    @skip
     def test_copy_memory_with_hw_break(self):
         self.build()
         exe = self.getBuildArtifact("a.out")
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
similarity index 89%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
index c6592ede0314..8186e1d66985 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/TestLibCxxAtomic.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/TestDataFormatterStdAtomic.py
@@ -9,18 +9,15 @@
 from lldbsuite.test import lldbutil
 
 
-class LibCxxAtomicTestCase(TestBase):
+class StdAtomicTestCase(TestBase):
     def get_variable(self, name):
         var = self.frame().FindVariable(name)
         var.SetPreferDynamicValue(lldb.eDynamicCanRunTarget)
         var.SetPreferSyntheticValue(True)
         return var
 
-    @skipIf(compiler=["gcc"])
-    @add_test_categories(["libc++"])
-    def test(self):
-        """Test that std::atomic as defined by libc++ is correctly printed by LLDB"""
-        self.build()
+    def do_test(self):
+        """Test that std::atomic is correctly printed by LLDB"""
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         bkpt = self.target().FindBreakpointByID(
@@ -31,8 +28,6 @@ def test(self):
 
         self.runCmd("run", RUN_SUCCEEDED)
 
-        lldbutil.skip_if_library_missing(self, self.target(), re.compile(r"libc\+\+"))
-
         # The stop reason of the thread should be breakpoint.
         self.expect(
             "thread list",
@@ -66,3 +61,9 @@ def test(self):
         self.expect(
             "frame var p.child.parent", substrs=["p.child.parent = {\n  Value = 0x"]
         )
+
+    @skipIf(compiler=["gcc"])
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
similarity index 51%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
index 760d2384a772..ee77a880a0c5 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/atomic/main.cpp
@@ -6,28 +6,26 @@ struct Child {
   // This should point to the parent which in turn owns this
   // child instance. This cycle should not cause LLDB to infinite loop
   // during printing.
-  std::atomic<Parent*> parent{nullptr};
+  std::atomic<Parent *> parent{nullptr};
 };
 struct Parent {
   Child child;
 };
 
 struct S {
-    int x = 1;
-    int y = 2;
+  int x = 1;
+  int y = 2;
 };
 
-int main ()
-{
-    std::atomic<S> s;
-    s.store(S());
-    std::atomic<int> i;
-    i.store(5);
+int main() {
+  std::atomic<S> s;
+  s.store(S());
+  std::atomic<int> i;
+  i.store(5);
 
-    Parent p;
-    // Let the child node know what its parent is.
-    p.child.parent = &p;
+  Parent p;
+  // Let the child node know what its parent is.
+  p.child.parent = &p;
 
-    return 0; // Set break point at this line.
+  return 0; // Set break point at this line.
 }
-
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
similarity index 82%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
index 57540fc1a281..4f79c0a900c3 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/Makefile
@@ -1,6 +1,4 @@
 CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
 CXXFLAGS_EXTRAS := -std=c++20
+
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
similarity index 98%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
index 0737a5bc7e6e..013a1787bc4b 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/TestDataFormatterStdChrono.py
@@ -2,20 +2,16 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxChronoDataFormatterTestCase(TestBase):
-    @add_test_categories(["libc++"])
-    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
-    def test_with_run_command(self):
+class StdChronoDataFormatterTestCase(TestBase):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
         isNotWindowsHost = lldbplatformutil.getHostPlatform() != "windows"
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp", False)
         )
@@ -433,3 +429,9 @@ def test_with_run_command(self):
                 "ymwdl_2024_last_tuesday_january = year=2024 month=January weekday=Tuesday index=last"
             ],
         )
+
+    @skipIf(compiler="clang", compiler_version=["<", "17.0"])
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/main.cpp
similarity index 100%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/chrono/main.cpp
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
index abe2a2b7895d..f52f6f7ec3ff 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/TestDataFormatterGenericDeque.py
@@ -21,9 +21,44 @@ def check_size(self, var_name, size):
         var = self.findVariable(var_name)
         self.assertEqual(var.GetNumChildren(), size)
 
+    def check_numbers(self, var_name, show_ptr=False):
+        patterns = []
+        substrs = [
+            "[0] = 1",
+            "[1] = 12",
+            "[2] = 123",
+            "[3] = 1234",
+            "[4] = 12345",
+            "[5] = 123456",
+            "[6] = 1234567",
+            "}",
+        ]
+        if show_ptr:
+            patterns = [var_name + " = 0x.* size=7"]
+        else:
+            substrs.insert(0, var_name + " = size=7")
+        self.expect(
+            "frame variable " + var_name,
+            patterns=patterns,
+            substrs=substrs,
+        )
+        self.expect_expr(
+            var_name,
+            result_summary="size=7",
+            result_children=[
+                ValueCheck(value="1"),
+                ValueCheck(value="12"),
+                ValueCheck(value="123"),
+                ValueCheck(value="1234"),
+                ValueCheck(value="12345"),
+                ValueCheck(value="123456"),
+                ValueCheck(value="1234567"),
+            ],
+        )
+
     def do_test(self, stdlib_type):
         self.build(dictionary={stdlib_type: "1"})
-        lldbutil.run_to_source_breakpoint(
+        (_, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp")
         )
 
@@ -83,6 +118,21 @@ def do_test(self, stdlib_type):
                 ],
             )
 
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        # first value added
+        self.expect("frame variable empty", substrs=["empty = size=1", "[0] = 1", "}"])
+
+        # add remaining values
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        self.check_numbers("empty")
+
+        # clear out the deque
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        self.expect_expr("empty", result_children=[])
+
     @add_test_categories(["libstdcxx"])
     def test_libstdcpp(self):
         self.do_test(USE_LIBSTDCPP)
@@ -90,3 +140,25 @@ def test_libstdcpp(self):
     @add_test_categories(["libc++"])
     def test_libcpp(self):
         self.do_test(USE_LIBCPP)
+
+    def do_test_ref_and_ptr(self, stdlib_type: str):
+        """Test formatting of std::deque& and std::deque*"""
+        self.build(dictionary={stdlib_type: "1"})
+        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "stop here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        # The reference should display the same was as the value did
+        self.check_numbers("ref", True)
+
+        # The pointer should just show the right number of elements:
+        self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
+        self.expect("expression ptr", substrs=["$", "size=7"])
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcpp_ref_and_ptr(self):
+        self.do_test_ref_and_ptr(USE_LIBSTDCPP)
+
+    @add_test_categories(["libc++"])
+    def test_libcpp_ref_and_ptr(self):
+        self.do_test_ref_and_ptr(USE_LIBCPP)
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
index b948fe1b4b37..a1c5b443daae 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/deque/main.cpp
@@ -26,6 +26,11 @@ template <typename T> T fill(T deque) {
   return deque;
 }
 
+void by_ref_and_ptr(std::deque<int> &ref, std::deque<int> *ptr) {
+  puts("stop here");
+  return;
+}
+
 int main() {
   std::deque<int> empty;
   std::deque<int> deque_1 = {1};
@@ -37,5 +42,23 @@ int main() {
   std::deque<Foo_large> deque_200_large;
   deque_200_large = fill<std::deque<Foo_large>>(deque_200_large);
 
-  return empty.size() + deque_1.front() + deque_3.front(); // break here
+  puts("break here");
+
+  empty.push_back(1);
+  puts("break here");
+
+  (empty.push_back(12));
+  (empty.push_back(123));
+  (empty.push_back(1234));
+  (empty.push_back(12345));
+  (empty.push_back(123456));
+  (empty.push_back(1234567));
+  puts("break here");
+
+  by_ref_and_ptr(empty, &empty);
+
+  empty.clear();
+  puts("break here");
+
+  return empty.size() + deque_1.front() + deque_3.front();
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
similarity index 94%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
index b2b83a3b4611..5851588b59b5 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/TestDataFormatterStdMap.py
@@ -2,14 +2,13 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxMapDataFormatterTestCase(TestBase):
+class StdMapDataFormatterTestCase(TestBase):
     def setUp(self):
         TestBase.setUp(self)
         ns = "ndk" if lldbplatformutil.target_is_android() else ""
@@ -22,10 +21,8 @@ def check_pair(self, first_value, second_value):
         ]
         return ValueCheck(children=pair_children)
 
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         bkpt = self.target().FindBreakpointByID(
@@ -326,3 +323,23 @@ def cleanup():
         lldbutil.continue_to_breakpoint(self.process(), bkpt)
 
         self.expect("frame variable ss", substrs=["%s::map" % ns, "size=0", "{}"])
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @expectedFailureAll(
+        bugnumber="Don't support formatting __gnu_debug::_Safe_iterator yet"
+    )
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp
new file mode 100644
index 000000000000..91bdf0b58d9e
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/map/main.cpp
@@ -0,0 +1,80 @@
+#include <map>
+#include <string>
+
+#define intint_map std::map<int, int>
+#define strint_map std::map<std::string, int>
+#define intstr_map std::map<int, std::string>
+#define strstr_map std::map<std::string, std::string>
+
+int g_the_foo = 0;
+
+int thefoo_rw(int arg = 1) {
+  if (arg < 0)
+    arg = 0;
+  if (!arg)
+    arg = 1;
+  g_the_foo += arg;
+  return g_the_foo;
+}
+
+int main() {
+  intint_map ii;
+
+  ii[0] = 0; // Set break point at this line.
+  ii[1] = 1;
+
+  intint_map::iterator it = ii.begin();
+  intint_map::const_iterator const_it = ii.cbegin();
+  std::printf("%d %d\n", it->second, const_it->second);
+
+  thefoo_rw(1); // Set break point at this line.
+  ii[2] = 0;
+  ii[3] = 1;
+  thefoo_rw(1); // Set break point at this line.
+  ii[4] = 0;
+  ii[5] = 1;
+  ii[6] = 0;
+  ii[7] = 1;
+  thefoo_rw(1); // Set break point at this line.
+  ii[85] = 1234567;
+
+  ii.clear();
+
+  strint_map si;
+  thefoo_rw(1); // Set break point at this line.
+
+  si["zero"] = 0;
+  thefoo_rw(1); // Set break point at this line.
+  si["one"] = 1;
+  si["two"] = 2;
+  si["three"] = 3;
+  thefoo_rw(1); // Set break point at this line.
+  si["four"] = 4;
+
+  si.clear();
+  thefoo_rw(1); // Set break point at this line.
+
+  intstr_map is;
+  thefoo_rw(1); // Set break point at this line.
+  is[85] = "goofy";
+  is[1] = "is";
+  is[2] = "smart";
+  is[3] = "!!!";
+  thefoo_rw(1); // Set break point at this line.
+
+  is.clear();
+  thefoo_rw(1); // Set break point at this line.
+
+  strstr_map ss;
+  thefoo_rw(1); // Set break point at this line.
+
+  ss["ciao"] = "hello";
+  ss["casa"] = "house";
+  ss["gatto"] = "cat";
+  thefoo_rw(1); // Set break point at this line.
+  ss["a Mac.."] = "..is always a Mac!";
+
+  ss.clear();
+  thefoo_rw(1); // Set break point at this line.
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
index 03e18ff4335e..d3d6ef55c6ff 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/set/TestDataFormatterGenericSet.py
@@ -137,7 +137,7 @@ def test_with_run_command_libcpp(self):
 
     def do_test_ref_and_ptr(self, stdlib_type):
         """Test that the data formatters work on ref and ptr."""
-        self.build()
+        self.build(dictionary={stdlib_type: "1"})
         (self.target, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self,
             "Stop here to check by ref and ptr.",
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
similarity index 54%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
index 23011c951e21..8b641c964395 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/TestDataFormatterLibcxxSharedPtr.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/TestDataFormatterStdSharedPtr.py
@@ -1,8 +1,7 @@
 """
-Test lldb data formatter for libc++ std::shared_ptr.
+Test lldb data formatter for std::shared_ptr.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -10,12 +9,9 @@
 
 
 class TestCase(TestBase):
-    @add_test_categories(["libc++"])
-    def test_shared_ptr_variables(self):
+    def do_test(self):
         """Test `frame variable` output for `std::shared_ptr` types."""
-        self.build()
-
-        lldbutil.run_to_source_breakpoint(
+        (_, process, _, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "// break here", lldb.SBFileSpec("main.cpp")
         )
 
@@ -23,7 +19,7 @@ def test_shared_ptr_variables(self):
             "sp_empty",
             type="std::shared_ptr<int>",
             summary="nullptr",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
         self.assertEqual(
             valobj.child[0].GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS), 0
@@ -36,45 +32,37 @@ def test_shared_ptr_variables(self):
         valobj = self.expect_var_path(
             "sp_int",
             type="std::shared_ptr<int>",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
         valobj = self.expect_var_path(
             "sp_int_ref",
             type="std::shared_ptr<int> &",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
         valobj = self.expect_var_path(
             "sp_int_ref_ref",
             type="std::shared_ptr<int> &&",
-            children=[ValueCheck(name="__ptr_")],
+            children=[ValueCheck(name="pointer")],
         )
-        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=1$")
+        self.assertRegex(valobj.summary, r"^10( strong=1)? weak=0$")
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            string_type = "std::string"
-        else:
-            string_type = "std::basic_string<char, std::char_traits<char>, std::allocator<char> > "
-
         valobj = self.expect_var_path(
             "sp_str",
-            type="std::shared_ptr<" + string_type + ">",
-            children=[ValueCheck(name="__ptr_", summary='"hello"')],
+            children=[ValueCheck(name="pointer", summary='"hello"')],
         )
-        self.assertRegex(valobj.summary, r'^"hello"( strong=1)? weak=1$')
+        self.assertRegex(valobj.summary, r'^"hello"( strong=1)? weak=0$')
 
         valobj = self.expect_var_path("sp_user", type="std::shared_ptr<User>")
         self.assertRegex(
             valobj.summary,
-            "^std(::__[^:]*)?::shared_ptr<User>::element_type @ 0x0*[1-9a-f][0-9a-f]+( strong=1)? weak=1",
+            "element_type @ 0x0*[1-9a-f][0-9a-f]+( strong=1)? weak=0",
         )
         self.assertNotEqual(valobj.child[0].unsigned, 0)
 
@@ -86,7 +74,43 @@ def test_shared_ptr_variables(self):
                 ValueCheck(name="name", summary='"steph"'),
             ],
         )
-        self.assertEqual(str(valobj), '(User) *__ptr_ = (id = 30, name = "steph")')
+        self.assertEqual(str(valobj), '(User) *pointer = (id = 30, name = "steph")')
 
         self.expect_var_path("sp_user->id", type="int", value="30")
         self.expect_var_path("sp_user->name", type="std::string", summary='"steph"')
+
+        valobj = self.expect_var_path(
+            "si", type="std::shared_ptr<int>", summary="47 strong=2 weak=0"
+        )
+
+        valobj = self.expect_var_path(
+            "sie", type="std::shared_ptr<int>", summary="nullptr strong=2 weak=0"
+        )
+
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        valobj = self.expect_var_path(
+            "si", type="std::shared_ptr<int>", summary="47 strong=2 weak=2"
+        )
+        valobj = self.expect_var_path(
+            "sie", type="std::shared_ptr<int>", summary="nullptr strong=2 weak=2"
+        )
+        valobj = self.expect_var_path(
+            "wie", type="std::weak_ptr<int>", summary="nullptr strong=2 weak=2"
+        )
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("ptr_node->value", value="1")
+        self.expect_var_path("ptr_node->next->value", value="2")
+        self.expect_var_path("(*ptr_node).value", value="1")
+        self.expect_var_path("(*(*ptr_node).next).value", value="2")
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
new file mode 100644
index 000000000000..cfefaf37a5db
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/shared_ptr/main.cpp
@@ -0,0 +1,38 @@
+#include <cstdio>
+#include <memory>
+#include <string>
+
+struct User {
+  int id = 30;
+  std::string name = "steph";
+};
+
+struct NodeS {
+  std::shared_ptr<NodeS> next;
+  int value;
+};
+
+int main() {
+  std::shared_ptr<int> sp_empty;
+  std::shared_ptr<int> sp_int = std::make_shared<int>(10);
+  std::shared_ptr<std::string> sp_str = std::make_shared<std::string>("hello");
+  std::shared_ptr<int> &sp_int_ref = sp_int;
+  std::shared_ptr<int> &&sp_int_ref_ref = std::make_shared<int>(10);
+  std::shared_ptr<User> sp_user = std::make_shared<User>();
+  std::shared_ptr<NodeS> ptr_node =
+      std::shared_ptr<NodeS>(new NodeS{nullptr, 2});
+  ptr_node = std::shared_ptr<NodeS>(new NodeS{std::move(ptr_node), 1});
+
+  // Construct empty shared_ptr with non-null control field.
+  std::shared_ptr<int> si(new int(47));
+  std::shared_ptr<int> sie(si, nullptr);
+
+  std::puts("// break here");
+
+  std::weak_ptr<int> wie = sie;
+  std::weak_ptr<int> wie2 = sie;
+
+  std::puts("// break here");
+
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile
new file mode 100644
index 000000000000..a27336ffd9ac
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/Makefile
@@ -0,0 +1,4 @@
+CXX_SOURCES := main.cpp
+CXXFLAGS_EXTRAS := -std=c++14
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
new file mode 100644
index 000000000000..1b204c7d6ef0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/TestDataFormatterStdUniquePtr.py
@@ -0,0 +1,157 @@
+"""
+Test lldb data formatter for std::unique_ptr.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def do_test(self):
+        """Test `frame variable` output for `std::unique_ptr` types."""
+
+        lldbutil.run_to_source_breakpoint(
+            self, "// break here", lldb.SBFileSpec("main.cpp")
+        )
+
+        valobj = self.expect_var_path(
+            "up_empty",
+            summary="nullptr",
+            children=[ValueCheck(name="pointer")],
+        )
+        self.assertEqual(
+            valobj.child[0].GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS), 0
+        )
+
+        self.expect(
+            "frame variable *up_empty", substrs=["(int) *up_empty = <parent is NULL>"]
+        )
+
+        valobj = self.expect_var_path(
+            "up_int",
+            summary="10",
+            children=[ValueCheck(name="pointer")],
+        )
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
+
+        valobj = self.expect_var_path(
+            "up_int_ref",
+            summary="10",
+            children=[ValueCheck(name="pointer")],
+        )
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
+
+        valobj = self.expect_var_path(
+            "up_int_ref_ref",
+            summary="10",
+            children=[ValueCheck(name="pointer")],
+        )
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
+
+        valobj = self.expect_var_path(
+            "up_str",
+            summary='"hello"',
+            children=[ValueCheck(name="pointer", summary='"hello"')],
+        )
+
+        valobj = self.expect_var_path("up_user")
+        self.assertRegex(valobj.summary, "^User @ 0x0*[1-9a-f][0-9a-f]+$")
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
+
+        valobj = self.expect_var_path(
+            "*up_user",
+            type="User",
+            children=[
+                ValueCheck(name="id", value="30"),
+                ValueCheck(name="name", summary='"steph"'),
+            ],
+        )
+        self.assertEqual(str(valobj), '(User) *pointer = (id = 30, name = "steph")')
+
+        valobj = self.expect_var_path(
+            "up_non_empty_deleter",
+            type="std::unique_ptr<int, NonEmptyIntDeleter>",
+            summary="1234",
+            children=[
+                ValueCheck(name="pointer"),
+                ValueCheck(
+                    name="deleter", children=[ValueCheck(name="dummy_", value="9999")]
+                ),
+            ],
+        )
+        self.assertNotEqual(valobj.child[0].unsigned, 0)
+
+        self.expect_var_path("up_user->id", type="int", value="30")
+        self.expect_var_path("up_user->name", type="std::string", summary='"steph"')
+
+        self.runCmd("settings set target.experimental.use-DIL true")
+        self.expect_var_path("ptr_node->value", value="1")
+        self.expect_var_path("ptr_node->next->value", value="2")
+        self.expect_var_path("(*ptr_node).value", value="1")
+        self.expect_var_path("(*(*ptr_node).next).value", value="2")
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    def do_test_recursive_unique_ptr(self):
+        # Tests that LLDB can handle when we have a loop in the unique_ptr
+        # reference chain and that it correctly handles the different options
+        # for the frame variable command in this case.
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
+        self.runCmd("run", RUN_SUCCEEDED)
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        self.expect("frame variable f1->next", substrs=["next = NodeU @"])
+        self.expect(
+            "frame variable --ptr-depth=1 f1->next",
+            substrs=["next = NodeU @", "value = 2"],
+        )
+        self.expect(
+            "frame variable --ptr-depth=2 f1->next",
+            substrs=["next = NodeU @", "value = 1", "value = 2"],
+        )
+
+        frame = self.frame()
+        self.assertTrue(frame.IsValid())
+        self.assertEqual(
+            2,
+            frame.GetValueForVariablePath("f1->next.object.value").GetValueAsUnsigned(),
+        )
+        self.assertEqual(
+            2, frame.GetValueForVariablePath("f1->next->value").GetValueAsUnsigned()
+        )
+        self.assertEqual(
+            1,
+            frame.GetValueForVariablePath(
+                "f1->next.object.next.obj.value"
+            ).GetValueAsUnsigned(),
+        )
+        self.assertEqual(
+            1,
+            frame.GetValueForVariablePath("f1->next->next->value").GetValueAsUnsigned(),
+        )
+
+    @add_test_categories(["libstdcxx"])
+    def test_recursive_unique_ptr_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_recursive_unique_ptr()
+
+    @add_test_categories(["libc++"])
+    def test_recursive_unique_ptr_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_recursive_unique_ptr()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
similarity index 58%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
index ab65d95b248e..15e9f70dbd6a 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/unique_ptr/main.cpp
@@ -6,15 +6,29 @@ struct User {
   std::string name = "steph";
 };
 
+struct NodeU {
+  std::unique_ptr<NodeU> next;
+  int value;
+};
+
 // libc++ stores unique_ptr data in a compressed pair, which has a specialized
 // representation when the type of the second element is an empty class. So
 // we need a deleter class with a dummy data member to trigger the other path.
 struct NonEmptyIntDeleter {
-  void operator()(int* ptr) { delete ptr; }
+  void operator()(int *ptr) { delete ptr; }
 
   int dummy_ = 9999;
 };
 
+static void recursive() {
+  // Set up a structure where we have a loop in the unique_ptr chain.
+  NodeU *f1 = new NodeU{nullptr, 1};
+  NodeU *f2 = new NodeU{nullptr, 2};
+  f1->next.reset(f2);
+  f2->next.reset(f1);
+  std::puts("Set break point at this line.");
+}
+
 int main() {
   std::unique_ptr<int> up_empty;
   std::unique_ptr<int> up_int = std::make_unique<int>(10);
@@ -24,6 +38,13 @@ int main() {
   std::unique_ptr<User> up_user = std::make_unique<User>();
   auto up_non_empty_deleter =
       std::unique_ptr<int, NonEmptyIntDeleter>(new int(1234));
+  std::unique_ptr<NodeU> ptr_node =
+      std::unique_ptr<NodeU>(new NodeU{nullptr, 2});
+  ptr_node = std::unique_ptr<NodeU>(new NodeU{std::move(ptr_node), 1});
+
+  std::puts("// break here");
+
+  recursive();
 
-  return 0; // break here
+  return 0;
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile
similarity index 72%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile
index bf8e6b8703f3..65e9dd2fa9e3 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/Makefile
@@ -1,5 +1,4 @@
 CXX_SOURCES := main.cpp
 
-USE_LIBSTDCPP := 1
-
 include Makefile.rules
+
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
similarity index 81%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
index f3371bc014b1..56c86d1edde2 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/TestDataFormatterStdVBool.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/TestDataFormatterStdVBool.py
@@ -2,8 +2,6 @@
 Test lldb data formatter subsystem.
 """
 
-
-from typing import Optional
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -17,19 +15,8 @@ def setUp(self):
         # Find the line number to break at.
         self.line = line_number("main.cpp", "// Set break point at this line.")
 
-    @skip
-    @add_test_categories(["libstdcxx"])
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command(build_args)
-
-    def with_run_command(self, dictionary: Optional[dict] = None):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
         self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
 
         lldbutil.run_break_set_by_file_and_line(
@@ -52,6 +39,7 @@ def cleanup():
             self.runCmd("type summary clear", check=False)
             self.runCmd("type filter clear", check=False)
             self.runCmd("type synth clear", check=False)
+            self.runCmd("settings set target.max-children-count 24", check=False)
 
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
@@ -83,3 +71,20 @@ def cleanup():
                 "[48] = true",
             ],
         )
+
+    @add_test_categories(["libc++"])
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp
new file mode 100644
index 000000000000..22fc6c89ca8a
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vbool/main.cpp
@@ -0,0 +1,65 @@
+#include <cstdio>
+#include <string>
+#include <vector>
+
+int main() {
+  std::vector<bool> vBool;
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(false);
+  vBool.push_back(true);
+  vBool.push_back(true);
+
+  std::puts("// Set break point at this line.");
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile
new file mode 100644
index 000000000000..99998b20bcb0
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
similarity index 84%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
index 13341a9b274b..ba8b10450f4f 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/TestDataFormatterLibcxxVector.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/TestDataFormatterStdVector.py
@@ -2,14 +2,13 @@
 Test lldb data formatter subsystem.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
 
-class LibcxxVectorDataFormatterTestCase(TestBase):
+class StdVectorDataFormatterTestCase(TestBase):
     def check_numbers(self, var_name, show_ptr=False):
         patterns = []
         substrs = [
@@ -52,10 +51,8 @@ def check_numbers(self, var_name, show_ptr=False):
         self.expect("frame variable " + var_name + "[2]", substrs=["123"])
         self.expect("frame variable " + var_name + "[3]", substrs=["1234"])
 
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
+    def do_test(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "break here", lldb.SBFileSpec("main.cpp", False)
         )
@@ -170,10 +167,25 @@ def cleanup():
 
         self.expect("frame variable strings", substrs=["vector has 0 items"])
 
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test()
+
+    @add_test_categories(["libstdcxx"])
+    def test_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test()
+
     @add_test_categories(["libc++"])
-    def test_ref_and_ptr(self):
+    def test_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test()
+
+    def do_test_ref_and_ptr(self):
         """Test that that file and class static variables display correctly."""
-        self.build()
         (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
             self, "Stop here to check by ref", lldb.SBFileSpec("main.cpp", False)
         )
@@ -186,3 +198,20 @@ def test_ref_and_ptr(self):
         self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
 
         self.expect("expression ptr", substrs=["$", "size=7"])
+
+    @add_test_categories(["libstdcxx"])
+    def test_ref_and_ptr_libstdcxx(self):
+        self.build(dictionary={"USE_LIBSTDCPP": 1})
+        self.do_test_ref_and_ptr()
+
+    @add_test_categories(["libstdcxx"])
+    def test_ref_and_ptr_libstdcxx_debug(self):
+        self.build(
+            dictionary={"USE_LIBSTDCPP": 1, "CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
+        )
+        self.do_test_ref_and_ptr()
+
+    @add_test_categories(["libc++"])
+    def test_ref_and_ptr_libcxx(self):
+        self.build(dictionary={"USE_LIBCPP": 1})
+        self.do_test_ref_and_ptr()
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp
new file mode 100644
index 000000000000..ec7ef7f84310
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/vector/main.cpp
@@ -0,0 +1,40 @@
+#include <stdio.h>
+#include <string>
+#include <vector>
+typedef std::vector<int> int_vect;
+typedef std::vector<std::string> string_vect;
+
+template <class T>
+void by_ref_and_ptr(std::vector<T> &ref, std::vector<T> *ptr) {
+  // Stop here to check by ref
+  return;
+}
+
+int main() {
+  int_vect numbers;
+  numbers.push_back(1);  // break here
+  numbers.push_back(12); // break here
+  numbers.push_back(123);
+  numbers.push_back(1234);
+  numbers.push_back(12345); // break here
+  numbers.push_back(123456);
+  numbers.push_back(1234567);
+  by_ref_and_ptr(numbers, &numbers);
+
+  printf("break here");
+  numbers.clear();
+
+  numbers.push_back(7); // break here
+
+  string_vect strings;
+  strings.push_back(std::string("goofy"));
+  strings.push_back(std::string("is"));
+  strings.push_back(std::string("smart"));
+  printf("break here");
+  strings.push_back(std::string("!!!"));
+
+  printf("break here");
+  strings.clear();
+
+  return 0; // break here
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
index 3a745883fb49..f10811817c0d 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/invalid-vector/main.cpp
@@ -1,4 +1,4 @@
-#define COMPRESSED_PAIR_REV 2
+#define COMPRESSED_PAIR_REV 3
 #include <libcxx-simulators-common/compressed_pair.h>
 
 namespace std {
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
index c996fff4fb7a..c8d9c2e389a0 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/TestDataFormatterLibcxxStringSimulator.py
@@ -28,7 +28,7 @@ def _run_test(self, defines):
 
 for v in [None, "ALTERNATE_LAYOUT"]:
     for r in range(6):
-        for c in range(3):
+        for c in range(4):
             name = "test_r%d_c%d" % (r, c)
             defines = ["REVISION=%d" % r, "COMPRESSED_PAIR_REV=%d" % c]
             if v:
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
index fdf8d53a3413..cf431e524069 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/string/main.cpp
@@ -209,7 +209,7 @@ template <class _CharT, class _Traits, class _Allocator> class basic_string {
   __long &getLongRep() {
 #if COMPRESSED_PAIR_REV == 0
     return __r_.first().__l;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
     return __rep_.__l;
 #endif
   }
@@ -217,14 +217,14 @@ template <class _CharT, class _Traits, class _Allocator> class basic_string {
   __short &getShortRep() {
 #if COMPRESSED_PAIR_REV == 0
     return __r_.first().__s;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
     return __rep_.__s;
 #endif
   }
 
 #if COMPRESSED_PAIR_REV == 0
   std::__lldb::__compressed_pair<__rep, allocator_type> __r_;
-#elif COMPRESSED_PAIR_REV <= 2
+#elif COMPRESSED_PAIR_REV <= 3
   _LLDB_COMPRESSED_PAIR(__rep, __rep_, allocator_type, __alloc_);
 #endif
 
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
index 4ee97e10de84..e623c3a1413b 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/TestDataFormatterLibcxxUniquePtrSimulator.py
@@ -26,7 +26,7 @@ def _run_test(self, defines):
         )
 
 
-for r in range(3):
+for r in range(4):
     name = "test_r%d" % r
     defines = ["COMPRESSED_PAIR_REV=%d" % r]
 
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
index 91a019566aff..3d174a91cc26 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx-simulators/unique_ptr/main.cpp
@@ -20,7 +20,8 @@ template <class _Tp, class _Dp = default_delete<_Tp>> class unique_ptr {
   std::__lldb::__compressed_pair<pointer, deleter_type> __ptr_;
   explicit unique_ptr(pointer __p) noexcept
       : __ptr_(__p, std::__lldb::__value_init_tag()) {}
-#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2
+#elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2 ||                  \
+    COMPRESSED_PAIR_REV == 3
   _LLDB_COMPRESSED_PAIR(pointer, __ptr_, deleter_type, __deleter_);
   explicit unique_ptr(pointer __p) noexcept : __ptr_(__p), __deleter_() {}
 #endif
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
deleted file mode 100644
index b016f006747d..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-CXX_SOURCES := main.cpp
-CXXFLAGS_EXTRAS := -std=c++11
-USE_LIBCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
deleted file mode 100644
index ece1e4f5f049..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/TestDataFormatterLibcxxDeque.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""
-Test LLDB's data formatter for libcxx's std::deque.
-"""
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class LibcxxDequeDataFormatterTestCase(TestBase):
-    def check_numbers(self, var_name, show_ptr=False):
-        patterns = []
-        substrs = [
-            "[0] = 1",
-            "[1] = 12",
-            "[2] = 123",
-            "[3] = 1234",
-            "[4] = 12345",
-            "[5] = 123456",
-            "[6] = 1234567",
-            "}",
-        ]
-        if show_ptr:
-            patterns = [var_name + " = 0x.* size=7"]
-        else:
-            substrs.insert(0, var_name + " = size=7")
-        self.expect(
-            "frame variable " + var_name,
-            patterns=patterns,
-            substrs=substrs,
-        )
-        self.expect_expr(
-            var_name,
-            result_summary="size=7",
-            result_children=[
-                ValueCheck(value="1"),
-                ValueCheck(value="12"),
-                ValueCheck(value="123"),
-                ValueCheck(value="1234"),
-                ValueCheck(value="12345"),
-                ValueCheck(value="123456"),
-                ValueCheck(value="1234567"),
-            ],
-        )
-
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
-        """Test basic formatting of std::deque"""
-        self.build()
-        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "break here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 1", "}"]
-        )
-
-        # add remaining values
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        self.check_numbers("numbers")
-
-        # clear out the deque
-        lldbutil.continue_to_breakpoint(process, bkpt)
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-    @add_test_categories(["libc++"])
-    def test_ref_and_ptr(self):
-        """Test formatting of std::deque& and std::deque*"""
-        self.build()
-        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
-            self, "stop here", lldb.SBFileSpec("main.cpp", False)
-        )
-
-        # The reference should display the same was as the value did
-        self.check_numbers("ref", True)
-
-        # The pointer should just show the right number of elements:
-        self.expect("frame variable ptr", substrs=["ptr =", " size=7"])
-        self.expect("expression ptr", substrs=["$", "size=7"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp
deleted file mode 100644
index 025c823d1f5b..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/deque/main.cpp
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <cstdio>
-#include <deque>
-typedef std::deque<int> int_deq;
-
-void by_ref_and_ptr(std::deque<int> &ref, std::deque<int> *ptr) {
-  printf("stop here");
-  return;
-}
-
-int main() {
-  int_deq numbers;
-  printf("break here");
-
-  (numbers.push_back(1));
-  printf("break here");
-
-  (numbers.push_back(12));
-  (numbers.push_back(123));
-  (numbers.push_back(1234));
-  (numbers.push_back(12345));
-  (numbers.push_back(123456));
-  (numbers.push_back(1234567));
-  by_ref_and_ptr(numbers, &numbers);
-  printf("break here");
-
-  numbers.clear();
-  printf("break here");
-
-  return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
similarity index 98%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
index d87cf7d40278..c5df567e01a2 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/Makefile
@@ -1,4 +1,5 @@
 CXX_SOURCES := main.cpp
+
 USE_LIBCPP := 1
-include Makefile.rules
 
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py
new file mode 100644
index 000000000000..ae8e0ac08c2b
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/TestDataFormatterLibcxxInvalidString.py
@@ -0,0 +1,38 @@
+"""
+Test lldb behaves sanely when formatting corrupted `std::string`s.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class LibcxxInvalidStringDataFormatterTestCase(TestBase):
+    @add_test_categories(["libc++"])
+    @skipIf(oslist=[lldbplatformutil.getDarwinOSTriples()], archs=["arm", "aarch64"])
+    def test(self):
+        self.build()
+
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Set break point at this line.", lldb.SBFileSpec("main.cpp")
+        )
+        frame = thread.frames[0]
+
+        if not self.process().GetAddressByteSize() == 8:
+            self.skip()
+
+        # The test assumes that std::string is in its cap-size-data layout.
+        self.expect(
+            "frame variable garbage1", substrs=["garbage1 = Summary Unavailable"]
+        )
+        self.expect(
+            "frame variable garbage2", substrs=[r'garbage2 = "\xfa\xfa\xfa\xfa"']
+        )
+        self.expect("frame variable garbage3", substrs=[r'garbage3 = "\xf0\xf0"'])
+        self.expect(
+            "frame variable garbage4", substrs=["garbage4 = Summary Unavailable"]
+        )
+        self.expect(
+            "frame variable garbage5", substrs=["garbage5 = Summary Unavailable"]
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp
new file mode 100644
index 000000000000..eb3efe1bcb7e
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/invalid-string/main.cpp
@@ -0,0 +1,110 @@
+#include <cstdio>
+#include <cstdlib>
+#include <stdint.h>
+#include <string>
+
+// For more information about libc++'s std::string ABI, see:
+//
+//   https://joellaity.com/2020/01/31/string.html
+
+// A corrupt string which hits the SSO code path, but has an invalid size.
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  // Set the size of this short-mode string to 116. Note that in short mode,
+  // the size is encoded as `size << 1`.
+  unsigned char size = 232;
+
+  // 23 garbage bytes for the inline string payload.
+  char inline_buf[23] = {0};
+#else  // _LIBCPP_ABI_VERSION == 1
+  // Like above, but data comes first, and use bitfields to indicate size.
+  char inline_buf[23] = {0};
+  unsigned char size : 7 = 116;
+  unsigned char is_long : 1 = 0;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_short_mode;
+
+// A corrupt libcxx string in long mode with a payload that contains a utf8
+// sequence that's inherently too long.
+static unsigned char garbage_utf8_payload1[] = {
+    250, // This means that we expect a 5-byte sequence, this is invalid. LLDB
+         // should fall back to ASCII printing.
+    250, 250, 250};
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 4;
+  unsigned char *data = &garbage_utf8_payload1[0];
+#else  // _LIBCPP_ABI_VERSION == 1
+  unsigned char *data = &garbage_utf8_payload1[0];
+  uint64_t size = 4;
+  uint64_t cap : 63 = 4;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode1;
+
+// A corrupt libcxx string in long mode with a payload that contains a utf8
+// sequence that's too long to fit in the buffer.
+static unsigned char garbage_utf8_payload2[] = {
+    240, // This means that we expect a 4-byte sequence, but the buffer is too
+         // small for this. LLDB should fall back to ASCII printing.
+    240};
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 3;
+  uint64_t size = 2;
+  unsigned char *data = &garbage_utf8_payload2[0];
+#else  // _LIBCPP_ABI_VERSION == 1
+  unsigned char *data = &garbage_utf8_payload2[0];
+  uint64_t size = 2;
+  uint64_t cap : 63 = 3;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode2;
+
+// A corrupt libcxx string which has an invalid size (i.e. a size greater than
+// the capacity of the string).
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 7;
+  const char *data = "foo";
+#else  // _LIBCPP_ABI_VERSION == 1
+  const char *data = "foo";
+  uint64_t size = 7;
+  uint64_t cap : 63 = 5;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode3;
+
+// A corrupt libcxx string in long mode with a payload that would trigger a
+// buffer overflow.
+static struct {
+#if _LIBCPP_ABI_VERSION == 1
+  uint64_t cap = 5;
+  uint64_t size = 2;
+  uint64_t data = 0xfffffffffffffffeULL;
+#else  // _LIBCPP_ABI_VERSION == 1
+  uint64_t data = 0xfffffffffffffffeULL;
+  uint64_t size = 2;
+  uint64_t cap : 63 = 5;
+  uint64_t is_long : 1 = 1;
+#endif // #if _LIBCPP_ABI_VERSION == 1
+} garbage_string_long_mode4;
+
+int main() {
+  std::string garbage1, garbage2, garbage3, garbage4, garbage5;
+  if (sizeof(std::string) == sizeof(garbage_string_short_mode))
+    memcpy((void *)&garbage1, &garbage_string_short_mode, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode1))
+    memcpy((void *)&garbage2, &garbage_string_long_mode1, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode2))
+    memcpy((void *)&garbage3, &garbage_string_long_mode2, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode3))
+    memcpy((void *)&garbage4, &garbage_string_long_mode3, sizeof(std::string));
+  if (sizeof(std::string) == sizeof(garbage_string_long_mode4))
+    memcpy((void *)&garbage5, &garbage_string_long_mode4, sizeof(std::string));
+
+  std::puts("// Set break point at this line.");
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
deleted file mode 100644
index 43ce6aeefd2a..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/main.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-#include <string>
-#include <map>
-
-#define intint_map std::map<int, int> 
-#define strint_map std::map<std::string, int> 
-#define intstr_map std::map<int, std::string> 
-#define strstr_map std::map<std::string, std::string> 
-
-int g_the_foo = 0;
-
-int thefoo_rw(int arg = 1)
-{
-	if (arg < 0)
-		arg = 0;
-	if (!arg)
-		arg = 1;
-	g_the_foo += arg;
-	return g_the_foo;
-}
-
-int main()
-{
-    intint_map ii;
-    
-    ii[0] = 0; // Set break point at this line.
-    ii[1] = 1;
-
-    intint_map::iterator it = ii.begin();
-    intint_map::const_iterator const_it = ii.cbegin();
-    std::printf("%d %d\n", it->second, const_it->second);
-
-    thefoo_rw(1); // Set break point at this line.
-    ii[2] = 0;
-    ii[3] = 1;
-	thefoo_rw(1);  // Set break point at this line.
-	ii[4] = 0;
-    ii[5] = 1;
-    ii[6] = 0;
-    ii[7] = 1;
-    thefoo_rw(1);  // Set break point at this line.
-    ii[85] = 1234567;
-
-    ii.clear();
-    
-    strint_map si;
-    thefoo_rw(1);  // Set break point at this line.
-	
-    si["zero"] = 0;
-	thefoo_rw(1);  // Set break point at this line.
-    si["one"] = 1;
-    si["two"] = 2;
-    si["three"] = 3;
-	thefoo_rw(1);  // Set break point at this line.
-    si["four"] = 4;
-
-    si.clear();
-    thefoo_rw(1);  // Set break point at this line.
-	
-    intstr_map is;
-    thefoo_rw(1);  // Set break point at this line.
-    is[85] = "goofy";
-    is[1] = "is";
-    is[2] = "smart";
-    is[3] = "!!!";
-    thefoo_rw(1);  // Set break point at this line.
-	
-    is.clear();
-    thefoo_rw(1);  // Set break point at this line.
-	
-    strstr_map ss;
-    thefoo_rw(1);  // Set break point at this line.
-	
-    ss["ciao"] = "hello";
-    ss["casa"] = "house";
-    ss["gatto"] = "cat";
-    thefoo_rw(1);  // Set break point at this line.
-    ss["a Mac.."] = "..is always a Mac!";
-    
-    ss.clear();
-    thefoo_rw(1);  // Set break point at this line.    
-    return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile
deleted file mode 100644
index c1c8b4a2a0a5..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-# We need debug info tuning for lldb in order to emit the preferred name for
-# std::string. See https://reviews.llvm.org/D145803.
-CXXFLAGS_EXTRAS := -std=c++14 -glldb
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp
deleted file mode 100644
index 026bfe24e72e..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/shared_ptr/main.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-#include <memory>
-#include <string>
-
-struct User {
-  int id = 30;
-  std::string name = "steph";
-};
-
-int main() {
-  std::shared_ptr<int> sp_empty;
-  std::shared_ptr<int> sp_int = std::make_shared<int>(10);
-  std::shared_ptr<std::string> sp_str = std::make_shared<std::string>("hello");
-  std::shared_ptr<int> &sp_int_ref = sp_int;
-  std::shared_ptr<int> &&sp_int_ref_ref = std::make_shared<int>(10);
-  std::shared_ptr<User> sp_user = std::make_shared<User>();
-
-  return 0; // break here
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
index 32764629d65a..2f7fc88e0f44 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
@@ -42,8 +42,6 @@ def cleanup():
             self.runCmd("type filter clear", check=False)
             self.runCmd("type synth clear", check=False)
 
-        is_64_bit = self.process().GetAddressByteSize() == 8
-
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
 
@@ -126,25 +124,6 @@ def cleanup():
             ],
         )
 
-        # The test assumes that std::string is in its cap-size-data layout.
-        is_alternate_layout = (
-            "arm" in self.getArchitecture()
-        ) and self.platformIsDarwin()
-        if is_64_bit and not is_alternate_layout:
-            self.expect(
-                "frame variable garbage1", substrs=["garbage1 = Summary Unavailable"]
-            )
-            self.expect(
-                "frame variable garbage2", substrs=[r'garbage2 = "\xfa\xfa\xfa\xfa"']
-            )
-            self.expect("frame variable garbage3", substrs=[r'garbage3 = "\xf0\xf0"'])
-            self.expect(
-                "frame variable garbage4", substrs=["garbage4 = Summary Unavailable"]
-            )
-            self.expect(
-                "frame variable garbage5", substrs=["garbage5 = Summary Unavailable"]
-            )
-
         # Finally, make sure that if the string is not readable, we give an error:
         bkpt_2 = target.BreakpointCreateBySourceRegex(
             "Break here to look at bad string", self.main_spec
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
index f9f1c0802e51..373f817a80f7 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
@@ -1,97 +1,6 @@
 #include <string>
 #include <stdint.h>
 
-// For more information about libc++'s std::string ABI, see:
-//
-//   https://joellaity.com/2020/01/31/string.html
-
-// A corrupt string which hits the SSO code path, but has an invalid size.
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  // Set the size of this short-mode string to 116. Note that in short mode,
-  // the size is encoded as `size << 1`.
-  unsigned char size = 232;
-
-  // 23 garbage bytes for the inline string payload.
-  char inline_buf[23] = {0};
-#else  // _LIBCPP_ABI_VERSION == 1
-  // Like above, but data comes first, and use bitfields to indicate size.
-  char inline_buf[23] = {0};
-  unsigned char size : 7 = 116;
-  unsigned char is_long : 1 = 0;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_short_mode;
-
-// A corrupt libcxx string in long mode with a payload that contains a utf8
-// sequence that's inherently too long.
-static unsigned char garbage_utf8_payload1[] = {
-  250, // This means that we expect a 5-byte sequence, this is invalid. LLDB
-       // should fall back to ASCII printing.
-  250, 250, 250
-};
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 4;
-  unsigned char *data = &garbage_utf8_payload1[0];
-#else  // _LIBCPP_ABI_VERSION == 1
-  unsigned char *data = &garbage_utf8_payload1[0];
-  uint64_t size = 4;
-  uint64_t cap : 63 = 4;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode1;
-
-// A corrupt libcxx string in long mode with a payload that contains a utf8
-// sequence that's too long to fit in the buffer.
-static unsigned char garbage_utf8_payload2[] = {
-  240, // This means that we expect a 4-byte sequence, but the buffer is too
-       // small for this. LLDB should fall back to ASCII printing.
-  240
-};
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 3;
-  uint64_t size = 2;
-  unsigned char *data = &garbage_utf8_payload2[0];
-#else  // _LIBCPP_ABI_VERSION == 1
-  unsigned char *data = &garbage_utf8_payload2[0];
-  uint64_t size = 2;
-  uint64_t cap : 63 = 3;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode2;
-
-// A corrupt libcxx string which has an invalid size (i.e. a size greater than
-// the capacity of the string).
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 7;
-  const char *data = "foo";
-#else  // _LIBCPP_ABI_VERSION == 1
-  const char *data = "foo";
-  uint64_t size = 7;
-  uint64_t cap : 63 = 5;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode3;
-
-// A corrupt libcxx string in long mode with a payload that would trigger a
-// buffer overflow.
-static struct {
-#if _LIBCPP_ABI_VERSION == 1
-  uint64_t cap = 5;
-  uint64_t size = 2;
-  uint64_t data = 0xfffffffffffffffeULL;
-#else  // _LIBCPP_ABI_VERSION == 1
-  uint64_t data = 0xfffffffffffffffeULL;
-  uint64_t size = 2;
-  uint64_t cap : 63 = 5;
-  uint64_t is_long : 1 = 1;
-#endif // #if _LIBCPP_ABI_VERSION == 1
-} garbage_string_long_mode4;
-
 size_t touch_string(std::string &in_str)
 {
   return in_str.size(); // Break here to look at bad string
@@ -115,18 +24,6 @@ int main()
     std::u32string u32_empty(U"");
     std::string *null_str = nullptr;
 
-    std::string garbage1, garbage2, garbage3, garbage4, garbage5;
-    if (sizeof(std::string) == sizeof(garbage_string_short_mode))
-      memcpy((void *)&garbage1, &garbage_string_short_mode, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode1))
-      memcpy((void *)&garbage2, &garbage_string_long_mode1, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode2))
-      memcpy((void *)&garbage3, &garbage_string_long_mode2, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode3))
-      memcpy((void *)&garbage4, &garbage_string_long_mode3, sizeof(std::string));
-    if (sizeof(std::string) == sizeof(garbage_string_long_mode4))
-      memcpy((void *)&garbage5, &garbage_string_long_mode4, sizeof(std::string));
-
     S.assign(L"!!!!!"); // Set break point at this line.
     std::string *not_a_string = (std::string *) 0x0;
     touch_string(*not_a_string);
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile
deleted file mode 100644
index c1c8b4a2a0a5..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-# We need debug info tuning for lldb in order to emit the preferred name for
-# std::string. See https://reviews.llvm.org/D145803.
-CXXFLAGS_EXTRAS := -std=c++14 -glldb
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
deleted file mode 100644
index 6a726e025348..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unique_ptr/TestDataFormatterLibcxxUniquePtr.py
+++ /dev/null
@@ -1,117 +0,0 @@
-"""
-Test lldb data formatter for libc++ std::unique_ptr.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class TestCase(TestBase):
-    def make_expected_type(self, pointee_type: str, qualifiers: str = "") -> str:
-        if qualifiers:
-            qualifiers = " " + qualifiers
-
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            return f"std::unique_ptr<{pointee_type}>{qualifiers}"
-        else:
-            return f"std::unique_ptr<{pointee_type}, std::default_delete<{pointee_type}> >{qualifiers}"
-
-    def make_expected_basic_string_ptr(self) -> str:
-        if self.expectedCompiler(["clang"]) and self.expectedCompilerVersion(
-            [">", "16.0"]
-        ):
-            return f"std::unique_ptr<std::string>"
-        else:
-            return (
-                "std::unique_ptr<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, "
-                "std::default_delete<std::basic_string<char, std::char_traits<char>, std::allocator<char> > > >"
-            )
-
-    @add_test_categories(["libc++"])
-    def test_unique_ptr_variables(self):
-        """Test `frame variable` output for `std::unique_ptr` types."""
-        self.build()
-
-        lldbutil.run_to_source_breakpoint(
-            self, "// break here", lldb.SBFileSpec("main.cpp")
-        )
-
-        valobj = self.expect_var_path(
-            "up_empty",
-            type=self.make_expected_type("int"),
-            summary="nullptr",
-            children=[ValueCheck(name="pointer")],
-        )
-        self.assertEqual(
-            valobj.child[0].GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS), 0
-        )
-
-        self.expect(
-            "frame variable *up_empty", substrs=["(int) *up_empty = <parent is NULL>"]
-        )
-
-        valobj = self.expect_var_path(
-            "up_int",
-            type=self.make_expected_type("int"),
-            summary="10",
-            children=[ValueCheck(name="pointer")],
-        )
-        self.assertNotEqual(valobj.child[0].unsigned, 0)
-
-        valobj = self.expect_var_path(
-            "up_int_ref",
-            type=self.make_expected_type("int", qualifiers="&"),
-            summary="10",
-            children=[ValueCheck(name="pointer")],
-        )
-        self.assertNotEqual(valobj.child[0].unsigned, 0)
-
-        valobj = self.expect_var_path(
-            "up_int_ref_ref",
-            type=self.make_expected_type("int", qualifiers="&&"),
-            summary="10",
-            children=[ValueCheck(name="pointer")],
-        )
-        self.assertNotEqual(valobj.child[0].unsigned, 0)
-
-        valobj = self.expect_var_path(
-            "up_str",
-            type=self.make_expected_basic_string_ptr(),
-            summary='"hello"',
-            children=[ValueCheck(name="pointer", summary='"hello"')],
-        )
-
-        valobj = self.expect_var_path("up_user", type=self.make_expected_type("User"))
-        self.assertRegex(valobj.summary, "^User @ 0x0*[1-9a-f][0-9a-f]+$")
-        self.assertNotEqual(valobj.child[0].unsigned, 0)
-
-        valobj = self.expect_var_path(
-            "*up_user",
-            type="User",
-            children=[
-                ValueCheck(name="id", value="30"),
-                ValueCheck(name="name", summary='"steph"'),
-            ],
-        )
-        self.assertEqual(str(valobj), '(User) *pointer = (id = 30, name = "steph")')
-
-        valobj = self.expect_var_path(
-            "up_non_empty_deleter",
-            type="std::unique_ptr<int, NonEmptyIntDeleter>",
-            summary="1234",
-            children=[
-                ValueCheck(name="pointer"),
-                ValueCheck(
-                    name="deleter", children=[ValueCheck(name="dummy_", value="9999")]
-                ),
-            ],
-        )
-        self.assertNotEqual(valobj.child[0].unsigned, 0)
-
-        self.expect_var_path("up_user->id", type="int", value="30")
-        self.expect_var_path("up_user->name", type="std::string", summary='"steph"')
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/Makefile
similarity index 100%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/Makefile
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
similarity index 51%
rename from lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py
rename to lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
index c021a46a17b5..a338e3c12598 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/TestDataFormatterLibccUnorderedMap.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/TestDataFormatterLibccUnorderedMap.py
@@ -9,6 +9,29 @@
 
 
 class LibcxxUnorderedMapDataFormatterTestCase(TestBase):
+    def check_ptr_or_ref(self, var_name: str):
+        var = self.frame().FindVariable(var_name)
+        self.assertTrue(var)
+
+        pair = var.GetChildAtIndex(0)
+        self.assertTrue(pair)
+
+        self.assertEqual(pair.GetChildAtIndex(0).summary, '"Hello"')
+        self.assertEqual(pair.GetChildAtIndex(1).summary, '"World"')
+
+    def check_ptr_ptr(self, var_name: str):
+        var = self.frame().FindVariable(var_name)
+        self.assertTrue(var)
+
+        ptr = var.GetChildAtIndex(0)
+        self.assertTrue(ptr)
+
+        pair = ptr.GetChildAtIndex(0)
+        self.assertTrue(pair)
+
+        self.assertEqual(pair.GetChildAtIndex(0).summary, '"Hello"')
+        self.assertEqual(pair.GetChildAtIndex(1).summary, '"World"')
+
     @add_test_categories(["libc++"])
     def test_iterator_formatters(self):
         """Test that std::unordered_map related structures are formatted correctly when printed.
@@ -64,3 +87,43 @@ def test_iterator_formatters(self):
                 ValueCheck(name="second", summary='"Qux"'),
             ],
         )
+
+        lldbutil.continue_to_breakpoint(process, bkpt)
+
+        # Test references to std::unordered_map
+        self.check_ptr_or_ref("ref1")
+        self.check_ptr_or_ref("ref2")
+        self.check_ptr_or_ref("ref3")
+        self.check_ptr_or_ref("ref4")
+        self.check_ptr_or_ref("ref5")
+        self.check_ptr_or_ref("ref6")
+
+        # FIXME: we're getting this wrong.
+        self.expect_var_path(
+            "ref7",
+            summary="size=0",
+            type="const StringMapT *const &",
+        )
+
+    @expectedFailureAll(
+        bugnumber="https://github.com/llvm/llvm-project/issues/146040",
+        compiler="clang",
+        compiler_version=["<", "21"],
+    )
+    @add_test_categories(["libc++"])
+    def test_ptr_formatters(self):
+        """
+        Test that pointers to std::unordered_map are formatted correctly.
+        """
+
+        self.build()
+        (self.target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
+            self, "Stop here", lldb.SBFileSpec("main.cpp", False)
+        )
+
+        self.check_ptr_or_ref("ptr1")
+        self.check_ptr_or_ref("ptr2")
+        self.check_ptr_or_ref("ptr3")
+        self.check_ptr_ptr("ptr4")
+        self.check_ptr_ptr("ptr5")
+        self.check_ptr_ptr("ptr6")
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
new file mode 100644
index 000000000000..5397e54d9954
--- /dev/null
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map-iterator/main.cpp
@@ -0,0 +1,53 @@
+#include <cstdio>
+#include <string>
+#include <unordered_map>
+
+using StringMapT = std::unordered_map<std::string, std::string>;
+using StringMapTRef = const StringMapT &;
+using StringMapTPtr = const StringMapT *;
+
+static void check_references(const StringMapT &ref1, StringMapT &ref2,
+                             StringMapTRef ref3, StringMapTRef &ref4,
+                             const StringMapT &&ref5, StringMapT &&ref6,
+                             const StringMapT *const &ref7) {
+  std::printf("Break here");
+}
+
+static void check_pointer(const StringMapT *ptr1, StringMapT *ptr2,
+                          StringMapTPtr ptr3, StringMapTPtr *ptr4,
+                          const StringMapT *const *ptr5, StringMapT **ptr6) {
+  std::printf("Stop here");
+}
+
+int main() {
+  StringMapT string_map;
+  {
+    auto empty_iter = string_map.begin();
+    auto const_empty_iter = string_map.cbegin();
+    std::printf("Break here");
+  }
+  string_map["Foo"] = "Bar";
+  string_map["Baz"] = "Qux";
+
+  {
+    auto foo = string_map.find("Foo");
+    auto invalid = string_map.find("Invalid");
+
+    StringMapT::const_iterator const_baz = string_map.find("Baz");
+    auto bucket_it = string_map.begin(string_map.bucket("Baz"));
+    auto const_bucket_it = string_map.cbegin(string_map.bucket("Baz"));
+
+    std::printf("Break here");
+
+    StringMapT tmp{{"Hello", "World"}};
+    check_references(tmp, tmp, tmp, tmp, StringMapT{tmp}, StringMapT{tmp},
+                     &tmp);
+
+    StringMapT *tmp_ptr = &tmp;
+    const StringMapT *const_tmp_ptr = &tmp;
+    check_pointer(tmp_ptr, tmp_ptr, tmp_ptr, &const_tmp_ptr, &tmp_ptr,
+                  &tmp_ptr);
+  }
+
+  return 0;
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile
deleted file mode 100644
index 564cbada74e0..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-CXXFLAGS_EXTRAS := -O0
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp
deleted file mode 100644
index adcea6962977..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/unordered_map/main.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-#include <cstdio>
-#include <string>
-#include <unordered_map>
-
-using StringMapT = std::unordered_map<std::string, std::string>;
-
-int main() {
-  StringMapT string_map;
-  {
-    auto empty_iter = string_map.begin();
-    auto const_empty_iter = string_map.cbegin();
-    std::printf("Break here");
-  }
-  string_map["Foo"] = "Bar";
-  string_map["Baz"] = "Qux";
-
-  {
-    auto foo = string_map.find("Foo");
-    auto invalid = string_map.find("Invalid");
-
-    StringMapT::const_iterator const_baz = string_map.find("Baz");
-    auto bucket_it = string_map.begin(string_map.bucket("Baz"));
-    auto const_bucket_it = string_map.cbegin(string_map.bucket("Baz"));
-    std::printf("Break here");
-  }
-
-  return 0;
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
deleted file mode 100644
index 24dddee62e1e..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/TestDataFormatterLibcxxVBool.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class LibcxxVBoolDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libc++"])
-    def test_with_run_command(self):
-        """Test that that file and class static variables display correctly."""
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_file_and_line(
-            self, "main.cpp", self.line, num_expected_locations=-1
-        )
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-            self.runCmd("settings set target.max-children-count 24", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        self.expect(
-            "frame variable -A vBool",
-            substrs=[
-                "size=49",
-                "[0] = false",
-                "[1] = true",
-                "[18] = false",
-                "[27] = true",
-                "[36] = false",
-                "[47] = true",
-                "[48] = true",
-            ],
-        )
-
-        self.expect(
-            "expr -A -- vBool",
-            substrs=[
-                "size=49",
-                "[0] = false",
-                "[1] = true",
-                "[18] = false",
-                "[27] = true",
-                "[36] = false",
-                "[47] = true",
-                "[48] = true",
-            ],
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
deleted file mode 100644
index 026cfc863f2c..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/main.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-#include <string>
-#include <vector>
-
-int main()
-{
-    std::vector<bool> vBool;
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(true);
-
-    printf ("size: %d", (int) vBool.size()); // Set break point at this line.
-    return 0; 
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
deleted file mode 100644
index 564cbada74e0..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-USE_LIBCPP := 1
-
-CXXFLAGS_EXTRAS := -O0
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
deleted file mode 100644
index 0e1dbe4f03e2..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/vector/main.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-#include <stdio.h>
-#include <string>
-#include <vector>
-typedef std::vector<int> int_vect;
-typedef std::vector<std::string> string_vect;
-
-template <class T>
-void by_ref_and_ptr(std::vector<T> &ref, std::vector<T> *ptr) {
-  // Stop here to check by ref
-  return;  
-}
-
-int main()
-{
-    int_vect numbers;
-    (numbers.push_back(1));  // break here
-    (numbers.push_back(12));  // break here
-    (numbers.push_back(123));
-    (numbers.push_back(1234));
-    (numbers.push_back(12345)); // break here
-    (numbers.push_back(123456));
-    (numbers.push_back(1234567));
-    by_ref_and_ptr(numbers, &numbers);
-    
-    printf("break here");
-    numbers.clear();
-    
-    (numbers.push_back(7)); // break here
-
-    string_vect strings;
-    (strings.push_back(std::string("goofy")));
-    (strings.push_back(std::string("is")));
-    (strings.push_back(std::string("smart")));
-    printf("break here");
-    (strings.push_back(std::string("!!!")));
-     
-    printf("break here");
-    strings.clear();
-    
-    return 0;  // break here
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
deleted file mode 100644
index 2d76c02657a7..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/TestDataFormatterStdMap.py
+++ /dev/null
@@ -1,301 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-from typing import Optional
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdMapDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command("__debug::", build_args)
-
-    def with_run_command(self, namespace: str = "", dictionary: Optional[dict] = None):
-        """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        self.runCmd("frame variable ii --show-types")
-
-        self.runCmd(
-            f'type summary add -x "std::{namespace}map<" --summary-string "map has ${{svar%#}} items" -e'
-        )
-
-        self.expect("frame variable ii", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 2 items",
-                "[0] = ",
-                "first = 0",
-                "second = 0",
-                "[1] = ",
-                "first = 1",
-                "second = 1",
-            ],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 4 items",
-                "[2] = ",
-                "first = 2",
-                "second = 0",
-                "[3] = ",
-                "first = 3",
-                "second = 1",
-            ],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ii",
-            substrs=[
-                "map has 9 items",
-                "[5] = ",
-                "first = 5",
-                "second = 0",
-                "[7] = ",
-                "first = 7",
-                "second = 1",
-            ],
-        )
-
-        self.expect(
-            "expression ii",
-            substrs=[
-                "map has 9 items",
-                "[5] = ",
-                "first = 5",
-                "second = 0",
-                "[7] = ",
-                "first = 7",
-                "second = 1",
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable ii[0]", substrs=["first = 0", "second = 0"])
-        self.expect("frame variable ii[3]", substrs=["first =", "second ="])
-
-        self.expect("frame variable ii[8]", matching=True, substrs=["1234567"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("ii").MightHaveChildren(),
-            "ii.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression ii[8]", matching=False, error=True,
-        #            substrs = ['1234567'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable ii", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable si --show-types")
-
-        self.expect("frame variable si", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable si",
-            substrs=["map has 1 items", "[0] = ", 'first = "zero"', "second = 0"],
-        )
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable si",
-            substrs=[
-                "map has 5 items",
-                '[0] = (first = "four", second = 4)',
-                '[1] = (first = "one", second = 1)',
-                '[2] = (first = "three", second = 3)',
-                '[3] = (first = "two", second = 2)',
-                '[4] = (first = "zero", second = 0)',
-            ],
-        )
-
-        self.expect(
-            "expression si",
-            substrs=[
-                "map has 5 items",
-                '[0] = (first = "four", second = 4)',
-                '[1] = (first = "one", second = 1)',
-                '[2] = (first = "three", second = 3)',
-                '[3] = (first = "two", second = 2)',
-                '[4] = (first = "zero", second = 0)',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable si[0]", substrs=["first = ", "four", "second = 4"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("si").MightHaveChildren(),
-            "si.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression si[0]", matching=False, error=True,
-        #            substrs = ['first = ', 'zero'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable si", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable is --show-types")
-
-        self.expect("frame variable is", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable is",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = 1, second = "is")',
-                '[1] = (first = 2, second = "smart")',
-                '[2] = (first = 3, second = "!!!")',
-                '[3] = (first = 85, second = "goofy")',
-            ],
-        )
-
-        self.expect(
-            "expression is",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = 1, second = "is")',
-                '[1] = (first = 2, second = "smart")',
-                '[2] = (first = 3, second = "!!!")',
-                '[3] = (first = 85, second = "goofy")',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable is[0]", substrs=["first = ", "second ="])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("is").MightHaveChildren(),
-            "is.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression is[0]", matching=False, error=True,
-        #            substrs = ['first = ', 'goofy'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable is", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("frame variable ss --show-types")
-
-        self.expect("frame variable ss", substrs=["map has 0 items", "{}"])
-
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable ss",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = "a Mac..", second = "..is always a Mac!")',
-                '[1] = (first = "casa", second = "house")',
-                '[2] = (first = "ciao", second = "hello")',
-                '[3] = (first = "gatto", second = "cat")',
-            ],
-        )
-
-        self.expect(
-            "expression ss",
-            substrs=[
-                "map has 4 items",
-                '[0] = (first = "a Mac..", second = "..is always a Mac!")',
-                '[1] = (first = "casa", second = "house")',
-                '[2] = (first = "ciao", second = "hello")',
-                '[3] = (first = "gatto", second = "cat")',
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable ss[3]", substrs=["gatto", "cat"])
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("ss").MightHaveChildren(),
-            "ss.MightHaveChildren() says False for non empty!",
-        )
-
-        # check that the expression parser does not make use of
-        # synthetic children instead of running code
-        # TOT clang has a fix for this, which makes the expression command here succeed
-        # since this would make the test fail or succeed depending on clang version in use
-        # this is safer commented for the time being
-        # self.expect("expression ss[3]", matching=False, error=True,
-        #            substrs = ['gatto'])
-
-        self.runCmd("c")
-
-        self.expect("frame variable ss", substrs=["map has 0 items", "{}"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
deleted file mode 100644
index d5e5b212782d..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/map/main.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-#include <map>
-#include <string>
-
-#define intint_map std::map<int, int> 
-#define strint_map std::map<std::string, int> 
-#define intstr_map std::map<int, std::string> 
-#define strstr_map std::map<std::string, std::string> 
-
-
-int main()
-{
-    intint_map ii;
-    
-    ii[0] = 0; // Set break point at this line.
-    ii[1] = 1;
-    ii[2] = 0;// Set break point at this line.
-    ii[3] = 1;
-    ii[4] = 0;// Set break point at this line.
-    ii[5] = 1;
-    ii[6] = 0;
-    ii[7] = 1;
-    ii[85] = 1234567;
-    
-    ii.clear();// Set break point at this line.
-    
-    strint_map si;
-    
-    si["zero"] = 0;// Set break point at this line.
-    si["one"] = 1;// Set break point at this line.
-    si["two"] = 2;
-    si["three"] = 3;
-    si["four"] = 4;
-
-    si.clear();// Set break point at this line.
-    
-    intstr_map is;
-    
-    is[85] = "goofy";// Set break point at this line.
-    is[1] = "is";
-    is[2] = "smart";
-    is[3] = "!!!";
-    
-    is.clear();// Set break point at this line.
-    
-    strstr_map ss;
-    
-    ss["ciao"] = "hello";// Set break point at this line.
-    ss["casa"] = "house";
-    ss["gatto"] = "cat";
-    ss["a Mac.."] = "..is always a Mac!";
-    
-    ss.clear();// Set break point at this line.
-    
-    return 0;// Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
deleted file mode 100644
index 654e4b15bd56..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CXXFLAGS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
deleted file mode 100644
index a6856177858a..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/TestDataFormatterStdSmartPtr.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdSmartPtrDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        self.expect("frame variable nsp", substrs=["nsp = nullptr"])
-        self.expect("frame variable isp", substrs=["isp = 123"])
-        self.expect("frame variable ssp", substrs=['ssp = "foobar"'])
-
-        self.expect("frame variable nwp", substrs=["nwp = nullptr"])
-        self.expect("frame variable iwp", substrs=["iwp = 123"])
-        self.expect("frame variable swp", substrs=['swp = "foobar"'])
-
-        self.expect("frame variable *nsp", substrs=["*nsp = <parent is NULL>"])
-        self.expect("frame variable *isp", substrs=["*isp = 123"])
-        self.expect("frame variable *ssp", substrs=['*ssp = "foobar"'])
-        self.expect("frame variable *fsp", substrs=["*fsp = (mem = 5)"])
-
-        self.expect("frame variable fsp->mem", substrs=["(int) fsp->mem = 5"])
-
-        self.runCmd("continue")
-
-        self.expect("frame variable nsp", substrs=["nsp = nullptr"])
-        self.expect("frame variable isp", substrs=["isp = nullptr"])
-        self.expect("frame variable ssp", substrs=["ssp = nullptr"])
-
-        self.expect("frame variable nwp", substrs=["nwp = nullptr"])
-        self.expect("frame variable iwp", substrs=["iwp = nullptr"])
-        self.expect("frame variable swp", substrs=["swp = nullptr"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
deleted file mode 100644
index 6e4b869e1c8e..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/smart_ptr/main.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#include <memory>
-#include <string>
-
-struct Foo {
-  int mem = 5;
-};
-
-int
-main()
-{
-    std::shared_ptr<char> nsp;
-    std::shared_ptr<int> isp(new int{123});
-    std::shared_ptr<std::string> ssp = std::make_shared<std::string>("foobar");
-    std::shared_ptr<Foo> fsp = std::make_shared<Foo>();
-
-    std::weak_ptr<char> nwp;
-    std::weak_ptr<int> iwp = isp;
-    std::weak_ptr<std::string> swp = ssp;
-
-    nsp.reset(); // Set break point at this line.
-    isp.reset();
-    ssp.reset();
-    fsp.reset();
-
-    return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
deleted file mode 100644
index 2301e5edfbd9..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/TestDataFormatterStdUniquePtr.py
+++ /dev/null
@@ -1,134 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdUniquePtrDataFormatterTestCase(TestBase):
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    @skipIf(oslist=["linux"], archs=["arm$", "aarch64"])
-    def test_with_run_command(self):
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        frame = self.frame()
-        self.assertTrue(frame.IsValid())
-
-        self.expect("frame variable nup", substrs=["nup = nullptr"])
-        self.expect("frame variable iup", substrs=["iup = 0x"])
-        self.expect("frame variable sup", substrs=["sup = 0x"])
-
-        self.expect("frame variable ndp", substrs=["ndp = nullptr"])
-        self.expect(
-            "frame variable idp", substrs=["idp = 0x", "deleter = ", "a = 1", "b = 2"]
-        )
-        self.expect(
-            "frame variable sdp", substrs=["sdp = 0x", "deleter = ", "a = 3", "b = 4"]
-        )
-
-        self.assertEqual(
-            123, frame.GetValueForVariablePath("iup.object").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            123, frame.GetValueForVariablePath("*iup").GetValueAsUnsigned()
-        )
-        self.assertFalse(frame.GetValueForVariablePath("iup.deleter").IsValid())
-
-        self.assertEqual(
-            '"foobar"', frame.GetValueForVariablePath("sup.object").GetSummary()
-        )
-        self.assertEqual('"foobar"', frame.GetValueForVariablePath("*sup").GetSummary())
-        self.assertFalse(frame.GetValueForVariablePath("sup.deleter").IsValid())
-
-        self.assertEqual(
-            456, frame.GetValueForVariablePath("idp.object").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            456, frame.GetValueForVariablePath("*idp").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            '"baz"', frame.GetValueForVariablePath("sdp.object").GetSummary()
-        )
-        self.assertEqual('"baz"', frame.GetValueForVariablePath("*sdp").GetSummary())
-
-        idp_deleter = frame.GetValueForVariablePath("idp.deleter")
-        self.assertTrue(idp_deleter.IsValid())
-        self.assertEqual(
-            1, idp_deleter.GetChildMemberWithName("a").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            2, idp_deleter.GetChildMemberWithName("b").GetValueAsUnsigned()
-        )
-
-        sdp_deleter = frame.GetValueForVariablePath("sdp.deleter")
-        self.assertTrue(sdp_deleter.IsValid())
-        self.assertEqual(
-            3, sdp_deleter.GetChildMemberWithName("a").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            4, sdp_deleter.GetChildMemberWithName("b").GetValueAsUnsigned()
-        )
-
-    @skipIfFreeBSD
-    @skipIfWindows  # libstdcpp not ported to Windows
-    @skipIfDarwin  # doesn't compile on Darwin
-    @skipIfwatchOS  # libstdcpp not ported to watchos
-    @skipIf(oslist=["linux"], archs=["arm$", "aarch64"])
-    @add_test_categories(["libstdcxx"])
-    def test_recursive_unique_ptr(self):
-        # Tests that LLDB can handle when we have a loop in the unique_ptr
-        # reference chain and that it correctly handles the different options
-        # for the frame variable command in this case.
-        self.build()
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-        self.runCmd("run", RUN_SUCCEEDED)
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        self.expect("frame variable f1->fp", substrs=["fp = 0x"])
-        self.expect(
-            "frame variable --ptr-depth=1 f1->fp", substrs=["data = 2", "fp = 0x"]
-        )
-        self.expect(
-            "frame variable --ptr-depth=2 f1->fp",
-            substrs=["data = 2", "fp = 0x", "data = 1"],
-        )
-
-        frame = self.frame()
-        self.assertTrue(frame.IsValid())
-        self.assertEqual(
-            2, frame.GetValueForVariablePath("f1->fp.object.data").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            2, frame.GetValueForVariablePath("f1->fp->data").GetValueAsUnsigned()
-        )
-        self.assertEqual(
-            1,
-            frame.GetValueForVariablePath(
-                "f1->fp.object.fp.object.data"
-            ).GetValueAsUnsigned(),
-        )
-        self.assertEqual(
-            1, frame.GetValueForVariablePath("f1->fp->fp->data").GetValueAsUnsigned()
-        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
deleted file mode 100644
index 9cacfce98976..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/TestDataFormatterInvalidStdUniquePtr.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import lldbsuite.test.lldbinline as lldbinline
-from lldbsuite.test.decorators import *
-
-lldbinline.MakeInlineTest(__file__, globals(), [no_debug_info_test])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
deleted file mode 100644
index b12cab231695..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/invalid/main.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-// Test that we don't crash when trying to pretty-print structures that don't
-// have the layout our data formatters expect.
-namespace std {
-template<typename T, typename Deleter = void>
-class unique_ptr {};
-}
-
-int main() {
-  std::unique_ptr<int> U;
-  return 0; //% self.expect("frame variable U", substrs=["unique_ptr", "{}"])
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
deleted file mode 100644
index dd0072764d4e..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/unique_ptr/main.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include <memory>
-#include <string>
-
-struct Deleter {
-  void operator()(void *) {}
-
-  int a;
-  int b;
-};
-
-struct Foo {
-  int data;
-  std::unique_ptr<Foo> fp;
-};
-
-int main() {
-  std::unique_ptr<char> nup;
-  std::unique_ptr<int> iup(new int{123});
-  std::unique_ptr<std::string> sup(new std::string("foobar"));
-
-  std::unique_ptr<char, Deleter> ndp;
-  std::unique_ptr<int, Deleter> idp(new int{456}, Deleter{1, 2});
-  std::unique_ptr<std::string, Deleter> sdp(new std::string("baz"),
-                                            Deleter{3, 4});
-
-  std::unique_ptr<Foo> fp(new Foo{3});
-
-  // Set up a structure where we have a loop in the unique_ptr chain.
-  Foo* f1 = new Foo{1};
-  Foo* f2 = new Foo{2};
-  f1->fp.reset(f2);
-  f2->fp.reset(f1);
-
-  return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
index 394e221809f7..dae9b24fbbcf 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/TestDataFormatterLibStdcxxVariant.py
@@ -2,7 +2,6 @@
 Test lldb data formatter for LibStdC++ std::variant.
 """
 
-
 import lldb
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
@@ -62,17 +61,13 @@ def test_with_run_command(self):
             "frame variable v3",
             substrs=["v3 =  Active Type = char  {", "Value = 'A'", "}"],
         )
-        """
-        TODO: temporarily disable No Value tests as they seem to fail on ubuntu/debian
-        bots. Pending reproduce and investigation.
 
-        self.expect("frame variable v_no_value", substrs=["v_no_value =  No Value"])
+        self.expect("frame variable v_valueless", substrs=["v_valueless =  No Value"])
 
         self.expect(
-            "frame variable v_many_types_no_value",
-            substrs=["v_many_types_no_value =  No Value"],
+            "frame variable v_many_types_valueless",
+            substrs=["v_many_types_valueless =  No Value"],
         )
-        """
 
     @add_test_categories(["libstdcxx"])
     def test_invalid_variant_index(self):
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp
index 36e0f74f831f..235928264add 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/variant/main.cpp
@@ -4,7 +4,9 @@
 #include <vector>
 
 struct S {
-  operator int() { throw 42; }
+  S() = default;
+  S(S &&) { throw 42; }
+  S &operator=(S &&) = default;
 };
 
 int main() {
@@ -21,7 +23,7 @@ int main() {
   std::variant<int, double, char> v2;
   std::variant<int, double, char> v3;
   std::variant<std::variant<int, double, char>> v_v1;
-  std::variant<int, double, char> v_no_value;
+  std::variant<int, char, S> v_valueless = 5;
   // The next variant has many types, meaning the type index does not fit in
   // a byte and must be `unsigned short` instead of `unsigned char` when
   // using the unstable libc++ ABI. With stable libc++ ABI, the type index
@@ -43,8 +45,11 @@ int main() {
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
       int, int, int, int, int, int, int, int, int, int, int, int, int, int, int,
-      int, int, int, int, int, int, int, int, int, int, int, int>
-      v_many_types_no_value;
+      int, int, int, int, int, int, int, int, int, int, int, int, S>
+      v_many_types_valueless;
+
+  v_valueless = 5;
+  v_many_types_valueless.emplace<0>(10);
 
   v1 = 12; // v contains int
   v1_typedef = v1;
@@ -67,18 +72,22 @@ int main() {
   printf("%f\n", d); // break here
 
   try {
-    v_no_value.emplace<0>(S());
+    // Exception in type-changing move-assignment is guaranteed to put
+    // std::variant into a valueless state.
+    v_valueless = S();
   } catch (...) {
   }
 
-  printf("%zu\n", v_no_value.index());
+  printf("%d\n", v_valueless.valueless_by_exception());
 
   try {
-    v_many_types_no_value.emplace<0>(S());
+    // Exception in move-assignment is guaranteed to put std::variant into a
+    // valueless state.
+    v_many_types_valueless = S();
   } catch (...) {
   }
 
-  printf("%zu\n", v_many_types_no_value.index());
+  printf("%d\n", v_many_types_valueless.valueless_by_exception());
 
   return 0; // break here
 }
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
deleted file mode 100644
index c825977b1a5d..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CFLAGS_EXTRAS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
deleted file mode 100644
index 73956dd3fda3..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/main.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-#include <vector>
-
-int main()
-{
-    std::vector<bool> vBool;
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(false);
-    vBool.push_back(true);
-    vBool.push_back(true);
-
-    return 0; // Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
deleted file mode 100644
index 654e4b15bd56..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-CXX_SOURCES := main.cpp
-
-CXXFLAGS := -O0
-USE_LIBSTDCPP := 1
-
-include Makefile.rules
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
deleted file mode 100644
index 6cfd17a39304..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/TestDataFormatterStdVector.py
+++ /dev/null
@@ -1,220 +0,0 @@
-"""
-Test lldb data formatter subsystem.
-"""
-
-
-from typing import Optional
-import lldb
-from lldbsuite.test.decorators import *
-from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
-
-
-class StdVectorDataFormatterTestCase(TestBase):
-    def setUp(self):
-        # Call super's setUp().
-        TestBase.setUp(self)
-        # Find the line number to break at.
-        self.line = line_number("main.cpp", "// Set break point at this line.")
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command(self):
-        self.with_run_command()
-
-    @add_test_categories(["libstdcxx"])
-    @expectedFailureAll(bugnumber="llvm.org/pr50861", compiler="gcc")
-    def test_with_run_command_debug(self):
-        build_args = {"CXXFLAGS_EXTRAS": "-D_GLIBCXX_DEBUG"}
-        self.with_run_command(build_args)
-
-    def with_run_command(self, dictionary: Optional[dict] = None):
-        """Test that that file and class static variables display correctly."""
-        self.build(dictionary=dictionary)
-        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
-
-        lldbutil.run_break_set_by_source_regexp(self, "Set break point at this line.")
-
-        self.runCmd("run", RUN_SUCCEEDED)
-
-        # The stop reason of the thread should be breakpoint.
-        self.expect(
-            "thread list",
-            STOPPED_DUE_TO_BREAKPOINT,
-            substrs=["stopped", "stop reason = breakpoint"],
-        )
-
-        # This is the function to remove the custom formats in order to have a
-        # clean slate for the next test case.
-        def cleanup():
-            self.runCmd("type format clear", check=False)
-            self.runCmd("type summary clear", check=False)
-            self.runCmd("type filter clear", check=False)
-            self.runCmd("type synth clear", check=False)
-
-        # Execute the cleanup function during test case tear down.
-        self.addTearDownHook(cleanup)
-
-        # empty vectors (and storage pointers SHOULD BOTH BE NULL..)
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        self.runCmd("c")
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 1", "}"]
-        )
-
-        # add some more data
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable numbers",
-            substrs=[
-                "numbers = size=4",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "}",
-            ],
-        )
-
-        self.expect(
-            "expression numbers",
-            substrs=[
-                "$",
-                "size=4",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "}",
-            ],
-        )
-
-        # check access to synthetic children
-        self.runCmd(
-            'type summary add --summary-string "item 0 is ${var[0]}" std::int_vect int_vect'
-        )
-        self.expect("frame variable numbers", substrs=["item 0 is 1"])
-
-        self.runCmd(
-            'type summary add --summary-string "item 0 is ${svar[0]}" std::int_vect int_vect'
-        )
-        # import time
-        # time.sleep(19)
-        self.expect("frame variable numbers", substrs=["item 0 is 1"])
-        # move on with synths
-        self.runCmd("type summary delete std::int_vect")
-        self.runCmd("type summary delete int_vect")
-
-        # add some more data
-        self.runCmd("c")
-
-        self.expect(
-            "frame variable numbers",
-            substrs=[
-                "numbers = size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
-        )
-
-        self.expect(
-            "expression numbers",
-            substrs=[
-                "$",
-                "size=7",
-                "[0] = 1",
-                "[1] = 12",
-                "[2] = 123",
-                "[3] = 1234",
-                "[4] = 12345",
-                "[5] = 123456",
-                "[6] = 1234567",
-                "}",
-            ],
-        )
-
-        # check access-by-index
-        self.expect("frame variable numbers[0]", substrs=["1"])
-        self.expect("frame variable numbers[1]", substrs=["12"])
-        self.expect("frame variable numbers[2]", substrs=["123"])
-        self.expect("frame variable numbers[3]", substrs=["1234"])
-
-        # but check that expression does not rely on us
-        # (when expression gets to call into STL code correctly, we will have to find
-        # another way to check this)
-        self.expect(
-            "expression numbers[6]", matching=False, error=True, substrs=["1234567"]
-        )
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("numbers").MightHaveChildren(),
-            "numbers.MightHaveChildren() says False for non empty!",
-        )
-
-        # clear out the vector and see that we do the right thing once again
-        self.runCmd("c")
-
-        self.expect("frame variable numbers", substrs=["numbers = size=0"])
-
-        self.runCmd("c")
-
-        # first value added
-        self.expect(
-            "frame variable numbers", substrs=["numbers = size=1", "[0] = 7", "}"]
-        )
-
-        # check if we can display strings
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["goofy", "is", "smart"])
-
-        self.expect("expression strings", substrs=["goofy", "is", "smart"])
-
-        # test summaries based on synthetic children
-        self.runCmd(
-            'type summary add std::string_vect string_vect --summary-string "vector has ${svar%#} items" -e'
-        )
-        self.expect(
-            "frame variable strings",
-            substrs=["vector has 3 items", "goofy", "is", "smart"],
-        )
-
-        self.expect(
-            "expression strings", substrs=["vector has 3 items", "goofy", "is", "smart"]
-        )
-
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["vector has 4 items"])
-
-        # check access-by-index
-        self.expect("frame variable strings[0]", substrs=["goofy"])
-        self.expect("frame variable strings[1]", substrs=["is"])
-
-        # but check that expression does not rely on us
-        # (when expression gets to call into STL code correctly, we will have to find
-        # another way to check this)
-        self.expect(
-            "expression strings[0]", matching=False, error=True, substrs=["goofy"]
-        )
-
-        # check that MightHaveChildren() gets it right
-        self.assertTrue(
-            self.frame().FindVariable("strings").MightHaveChildren(),
-            "strings.MightHaveChildren() says False for non empty!",
-        )
-
-        self.runCmd("c")
-
-        self.expect("frame variable strings", substrs=["vector has 0 items"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
deleted file mode 100644
index 010917995e40..000000000000
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libstdcpp/vector/main.cpp
+++ /dev/null
@@ -1,31 +0,0 @@
-#include <string>
-#include <vector>
-typedef std::vector<int> int_vect;
-typedef std::vector<std::string> string_vect;
-
-int main()
-{
-    int_vect numbers;
-    numbers.push_back(1);  // Set break point at this line.
-    numbers.push_back(12);  // Set break point at this line.
-    numbers.push_back(123);
-    numbers.push_back(1234);
-    numbers.push_back(12345);  // Set break point at this line.
-    numbers.push_back(123456);
-    numbers.push_back(1234567);
-    
-    numbers.clear();  // Set break point at this line.
-    
-    numbers.push_back(7);  // Set break point at this line.
-
-    string_vect strings;  // Set break point at this line.
-    strings.push_back(std::string("goofy"));
-    strings.push_back(std::string("is"));
-    strings.push_back(std::string("smart"));
-    
-    strings.push_back(std::string("!!!"));  // Set break point at this line.
-    
-    strings.clear();  // Set break point at this line.
-    
-    return 0;// Set break point at this line.
-}
diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
index 08ac9290ee85..12b464d3397e 100644
--- a/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestGDBRemoteClient.py
@@ -356,6 +356,78 @@ def A(self, packet):
             ["vRun;%s;61726731;61726732;61726733" % (exe_hex,)]
         )
 
+    def test_launch_lengthy_vRun(self):
+        class MyResponder(MockGDBServerResponder):
+            def __init__(self, *args, **kwargs):
+                self.started = False
+                return super().__init__(*args, **kwargs)
+
+            def qC(self):
+                if self.started:
+                    return "QCp10.10"
+                else:
+                    return "E42"
+
+            def qfThreadInfo(self):
+                if self.started:
+                    return "mp10.10"
+                else:
+                    return "E42"
+
+            def qsThreadInfo(self):
+                return "l"
+
+            def qEcho(self, num):
+                resp = "qEcho:" + str(num)
+                if num >= 2:
+                    # We have launched our program
+                    self.started = True
+                    return [resp, "T13"]
+
+                return resp
+
+            def qSupported(self, client_supported):
+                return "PacketSize=3fff;QStartNoAckMode+;qEcho+;"
+
+            def qHostInfo(self):
+                return "default_packet_timeout:1;"
+
+            def vRun(self, packet):
+                return [self.RESPONSE_NONE]
+
+            def A(self, packet):
+                return "E28"
+
+        self.server.responder = MyResponder()
+
+        target = self.createTarget("a.yaml")
+        # NB: apparently GDB packets are using "/" on Windows too
+        exe_path = self.getBuildArtifact("a").replace(os.path.sep, "/")
+        exe_hex = binascii.b2a_hex(exe_path.encode()).decode()
+        process = self.connect(target)
+        lldbutil.expect_state_changes(
+            self, self.dbg.GetListener(), process, [lldb.eStateConnected]
+        )
+
+        process = target.Launch(
+            lldb.SBListener(),
+            ["arg1", "arg2", "arg3"],  # argv
+            [],  # envp
+            None,  # stdin_path
+            None,  # stdout_path
+            None,  # stderr_path
+            None,  # working_directory
+            0,  # launch_flags
+            True,  # stop_at_entry
+            lldb.SBError(),
+        )  # error
+        self.assertTrue(process, PROCESS_IS_VALID)
+        self.assertEqual(process.GetProcessID(), 16)
+
+        self.assertPacketLogContains(
+            ["vRun;%s;61726731;61726732;61726733" % (exe_hex,)]
+        )
+
     def test_launch_QEnvironment(self):
         class MyResponder(MockGDBServerResponder):
             def qC(self):
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py b/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py
new file mode 100644
index 000000000000..dd03a0cc836a
--- /dev/null
+++ b/lldb/test/API/functionalities/postmortem/elf-core/expr/TestExpr.py
@@ -0,0 +1,52 @@
+"""
+Test evaluating expressions when debugging core file.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+@skipIfLLVMTargetMissing("X86")
+class CoreExprTestCase(TestBase):
+    def setUp(self):
+        TestBase.setUp(self)
+        self.target = self.dbg.CreateTarget("linux-x86_64.out")
+        self.process = self.target.LoadCore("linux-x86_64.core")
+        self.assertTrue(self.process, PROCESS_IS_VALID)
+
+    def test_result_var(self):
+        """Test that the result variable can be used in subsequent expressions."""
+
+        self.expect_expr(
+            "outer",
+            result_type="Outer",
+            result_children=[ValueCheck(name="inner", type="Inner")],
+        )
+        self.expect_expr(
+            "$0.inner",
+            result_type="Inner",
+            result_children=[ValueCheck(name="val", type="int", value="5")],
+        )
+        self.expect_expr("$1.val", result_type="int", result_value="5")
+
+    def test_persist_var(self):
+        """Test that user-defined variables can be used in subsequent expressions."""
+
+        self.target.EvaluateExpression("int $my_int = 5")
+        self.expect_expr("$my_int * 2", result_type="int", result_value="10")
+
+    def test_context_object(self):
+        """Test expression evaluation in context of an object."""
+
+        val_outer = self.expect_expr("outer", result_type="Outer")
+
+        val_inner = val_outer.EvaluateExpression("inner")
+        self.assertTrue(val_inner.IsValid())
+        self.assertEqual("Inner", val_inner.GetDisplayTypeName())
+
+        val_val = val_inner.EvaluateExpression("this->val")
+        self.assertTrue(val_val.IsValid())
+        self.assertEqual("int", val_val.GetDisplayTypeName())
+        self.assertEqual(val_val.GetValueAsSigned(), 5)
diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.core
new file mode 100644
index 0000000000000000000000000000000000000000..3bd2916c64e9ff9facc4626ab53d785c47c6dcdf
GIT binary patch
literal 40960
zcmeHQ3v^V)8NT}pD-v!%gCGct2RYc3HBnI_#O}=lXJOIEL$N+W%tNRo!DKH)@ql3o
za=k=TZLLxt_{LLft=1Z?kAxsWkB>k+fKnAfFd<e^N_dv-_uo6SSvH839(ztbGjR8x
z|2Z??KQp@%Hq4^D&;+N$p@`yEE>^lZMF|K+91SRAm0p~8bGk0xDbDu%-f;}7!k)~n
zdWz5oM8&2;o<PaZLl!=IK}iW%q{quK{H*qT+|n25;bIAn;g>`TpQTll{IM@b7x6j~
zAFm6_{i&Z;djYObA9u8$kP8~;XN~VR?l(8V&&TD0azFCPN2Ank&Sg{b$m(Qz;U&H<
zWc^<}MN#o2<Jr40S%`l$x@txOcK=~Z79Mf7f`6RHi}G?Np9|-4+axEUp(yupyEHEU
zp~Q}2&%0%CcHt2iD^qy1q<~e~1tmR+*v^?<u2+w)9(zH-1?T%Kbj5#8f+XzYfliDN
zCb!`UUJ{%t`X+KN`hQ}PPuz?8G#8E2`VoB^F2eD7EpoNnT4Y$xTpvAT4x=g;#}xgz
zzL-<W1(g8bsjM{zeyk<sdj;xZ&WL@Bs}--PaC(y9`CJkhQckJ#v5A#qs)|)3^nDZb
zr*U48LA(<DrM(tT=e(0sYtX3gsIc`amtV#02XlT%0`+k|k<hqT;>imICx{#f%wk}3
z?0W2_VsG^Y5W(JCtP1Cq^nk^RzMm5=n|n@1kBP1PP`Q;W1f{s5&x>N6wwjE++y=m+
z(HJW$KT!l|9|&L$I58<g!Aqyl059jnPb=Apd-L~=PV771XD9U%0f~S_KqAl+fkg@D
zf2TPA^Ur<(?h+W;?Mvr>j}pMgN7XuSiqG7(Ma;uuXYj*^pyD$piLK9h;sd|<tjXy-
zJD4J9yr0#cj~n;`c7@ZGTr8;Y%ij?$KAztQ?kA{a3)$uHi=9EZ&GCpB&hc!dP9h)?
zkO)WwBmxoviGV~vA|Mfv2uK7Z0ulj<fJ8tdAQ6xVNCYGT|IY}}&-cZ5w3hMjU`-M~
z=Z7v{FTNW#AWkpFZv8s}XZSxSC@uW^F~a7f1YUm6$VTRW^*tkH@Vw67?_H6b;ci?y
zYh-2MLHnz2B0Z8y1SA3y0f~S_Kq4R!kO)WwBmxoviGV~vA|Mfv2uK7Z0ulj<z=<Ky
z^Vf-${3~;_?BBdR#xEP6SS!*?A|Mfv2uK7Z0ulj<fJ8tdAQ6xVNCYGT5&?;TL_i`S
z5s(N-1SA3y0f~S_Kq4R!kO)WwBmxoviGV~vA|Mfv2uK7Z0ulj<fJ8tdAQ6xVNCYGT
z5&?;TL_i`S5s(N-1SA3y0f~S_Kq4R!kO)WwBmxqF9|eKL`~O`0{=Y_k>0N-g+8TGU
z9o_%uj=TRa{{QDq?Br1HC#blO&j~}iV{a?J*Ex}<@c-ynP;pN;`OveQU(lCGr$I+W
zP;RD(`~PxzXA^12c2fiue$+j><D1|h*Vz5wsQ1Q0;}Ca*)7{|IZMw%+^q2f87E#iX
zJ?Wc9Jl9Vn&h5=}j&R&2{3zza=^x*uygm52k5X1VH07@JSxYC??o7UO=;Dd*c<214
z`k`9~J(g2`$IsS|9QoY!C1o>iKkn_<qY&>*^l&Z(Mw!C#ruO<n{v5w__YUe9N){fn
zZ{-)87h6(Axd4>x6Xi5cvQLbY%IyQZ-O)S_-P=rY=JJcuS8_SsXHIbv?S~8dg#On+
z-cR|a$Ji%w`v8xBHh3zR<FVo=$d!VEdBs%~1viw0!?VjuDoTqBZx%8Vh<eeSilQ5Z
zRS#ihNtKWxP|;jna8)-{7nGETODgeNG^cQ0i85z)xn4b*+VtaZhJBfJ!MT0{81v!S
ze`xO5e;#?`tK?H*NqF5vE+~{sCtkbsXI-SDLikV-_t6X6xMghV#XR>>5R|<~q7_7E
zaXz-T`MeQ#VXotPo)*O<Cls+32HLE0IBLgR#r^-qc|P5g-JD;}_1x`8Bh-H}<`AU<
z{FO;b*7~-Fgcl*FPLM5OO5MBU(&@^U2@k)}w*J6?#>F)+^z%O9y6g7!S33r~#xdw=
zE!+x!S~Kr}QtUYx=yK4*2($+D=Kypw=yDj3fX!jh@u2D0Sf+uB{5fu!qByEk9E1Dx
z^43F6d83@uemDV?3dz#(gdlsE1Ssd@$;Go7GHXre^-Wpo3??UK-{}g~xkJl6ERy7W
zws&*(JK67Mzo)H5d<{vXShwOyM?b|{r7_jx2|;G<8&<5G<fPzTt~=dzp5;lAUe2{5
zRQCHk*fl*Qh(Y7}JD!=?Z;C-hA^AV?G(axaD(Obx>4PzR0QmzX&}bZJTZ_1ymr|$u
zBvGGPT~_9Y(P-zOkeNIdm5}KnUUeKOtvA)o_riV63v0Xd&sa-i6s4IpZS%2{T7&yO
zV@7Mp(a6+ScK@Qe)UOU(Laiu~s6$V+ylu<XFtcqUQD<0<L|tL8h}F5+Onr4-G>SgF
z2x4rE>^#!tS&OP8yd@!q`g-ag$u!M#H%QQEG}iC?Fxb{^tY>XoYd0`6c`@W>>Jrkb
ztdZ^8#?)t8edcMI8}%ocW1ZcJ`aSB<gVTe?!Jx5+89THqg0EZMR+NA?J@^_85oM7!
zEmU{bBMs5$R8|{Rmplj?%^WhKtJ$H}t^{LxmO&ZW4|WyU_`ZgQ?_z1GWl%HIvy_Oj
z(M%>jH2kYj?YDX_rvB+YbjVD;#xfdn6^Le@IwG@8U%}eiSyuF|731a&$k@O}D{rqD
z7g_DFMf1OAj*UiR&^VXPZfu);eF|%8A4z<+H=}WK?G`=rYV8{BTJ5@Pn`^Q@f5{$+
zx?~eNxTn5jJow0dXZXs<epfiqFcZ>{u_<JHqIx=8SY7f9Ur-`apMhUZ)|1?usk!UT
zJGR1BGfp<|{@kieb$8y|z6Ki+Ggh<O#_AV3PDX4q^_#U6VazTNV^ib{r1B7|j_^J4
z@L)P1mD|WP&C>#MtW>@PxtaPhY2#BlAafH^S$C=z7<c^c2oS9sb9g6=UbV)Gys^Kb
zhg1T2reEuN6ZyFVoSF3@%-Uzh`m8QH1b5`=O~{d_Pl!;Shz||l$!b5+uQuLBrXYLK
zvdMZLL^ChQ&HPG#lsQ_ND{JLnqfulDDRX%qK%Hl<Uyr;Q8=2!shlVDRi0jeWj18Sn
zm8!#Xkr~WKR_7T*?schtv{92a3Jl63pud`#0UzdI*u{)q&FV1Aemc$EE43@M>DsiZ
ztmsn~`O+IXq^h^Shp{$b(JRJ>p}N%1UW`WDwi>HzwneZ>s!KkGqI-&minausw!0@g
z+VadZwXQXQ>M0m#X0{viTCa{z8t@LLP4NcJoKei!q?r>3hK7HQ1-0$sN}A2@V>UxL
zBxo-1X=Z;d^8@`4-#3vrG2>!pOu)=#ku_dEakYYLn=wb#C6A*gG*7dQPa9~`sn~Uo
zveIg9aT7%9l08rZ)eFI`nyIHn4!QNWG58NK_>R61GDB}|A8-g2Q*Xs|#iXr%x??nH
z>T;Z{X{$TrXw}T#T}=*s2s6i0zszbcGZuO^^P<cH`u2>*ju|ko$*TVgh}Gx>fV`)^
zbDf!bxS0ZF?FET!JE9q_T^`RG%zAZ+2Ge*D?U~<B_GzAXL4rn;y@_>nZ(^@OZl=CL
zT9q|*Z(>?ztNyT<`g5(Re}?08w0X!KD(dWB0KvM*ZU7Qeb2kT@cDOIA8=sbEo~L!K
zMS2fHL1<{^0(bkggYk%{%kDwUVA@pV_!2wElaS*ttQ@P0H^9?c4aof@q<N7q&vfNw
zuGJr#?AS$Xp{+gW${rPT-Qu&?fV%WPxEyl_JIJ$c4Crvu8jLhMf!~<zs2ZECbkq&$
zm239Wx*8q&<&({mgXSc!IV?A`S?>rsRtL?Cg06|)kmG$;bHLgtPhmA3VxP1{^Z$YO
z9OF_ibL?kE64G}+7Cz?4x7Gsed^BT2M;OTgp=nUs{pQ8jS0t*=&&Q%%`8+yRmwNv;
zY-k5+vVIK~1<Nw8)mmk3Tc5rMD@z};YG&Bipq!MYu-dNhP(HvG>!V(?xwsurbna-_
z0it@EGa8%g;|i_+h6PaVQCs%YLhnrrHmU;f(<qv!sl5-4K{#3OUs(TBjdfTll?<z7
zehxDyr?8ghQ3%5#;|D2v24MBFnnU!_R{xTjgRsFvi8rcE%)B<GCCBTwY!?hv^x+we
zSHIQVunR-$>>{m^<1IB7^eHu}`Z8l`O6O)~;LRSd+gW70$|9fd&d)!LeQBWjY<sko
z)fEp(iR@PvzS40cLbU9QMrj*bjL&Tz&&%N61UC+k$JGJum6&b|xIf2q{q{$rPsVgE
zaP+2Y`#l2g?wIZ#a7Ik`_<?A2VN7=~I6bDz!m9sOOm_~r!kDfUAFQv4>1KeN9MkET
zC)wZzU<__IMyY0CA^%WH3B4pI=OW)IeC|ElmyzK=*PpH^{;FA3;mUB~^@{(7a@{|x
zuxgg#FTT0F>gF<=hAVB&ypqbQ*%jqSF$KkzO<~#WBE?@`5iU{uC9?`jD+|j?3T73<
zQE-YsTv8oY{DpJEv&+$*UKB>Gs%}9+Wy#z*g+(Q0I9)0JqKY!Bf<gb61rygg=L0&c
zh;xykjLXGw1?$HuG#{zZ`KJqK8sRU_QG#A$S=r|astSKDR_NSJF@(Q3X9+r%-X2k?
zJ*o=-fh0sZ0}u60_*)Dqc!y1@#}ytfCOHc=EQ478#kow-)48P>zi3C$bD_s>74sM8
zK0(KFf9gN^i2if2qM<??kBGmOR|FM54-$9_e>w|Lq4PhE46*Tl*UyK@mWqgDbBdCW
z3SlC~^IbnD684Gy*CqJZ<Ge>j(DVerMgMIpoV!sW6KnGYZR8a}^AjAzxP^Z?_ZR(7
zp$Uy5D7_WNmxS&pi=d2ee;fA~bbP!sw~ZJ5mdmhH996{6|N0*#wc8?*=m$}e`*_oI
kKG28ag>ilfV#!JbBmxoviGV~vA|Mfv2uK7Z0>?n$-++g3-~a#s

literal 0
HcmV?d00001

diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out b/lldb/test/API/functionalities/postmortem/elf-core/expr/linux-x86_64.out
new file mode 100755
index 0000000000000000000000000000000000000000..33e60f025d210798b0ca9425211c301eabe42da0
GIT binary patch
literal 10816
zcmeHN&u<$=6n?YowViC7I87l*1I?zbQA(xF4^l{JXk8MwZbbtLN)@4&tsQR~OR*i<
z>nKo4i-JH=K#;$H9yoA@OSy0<qLnysgx;xCBm_aUQV>Ytz1bOOF`*O*^)w^BdGCGi
zdo#PAXKh8<%c&QRYm9*$8XN(h#giQJXMp4rl7TS{gMo7#S~xefRl=y&!O>~4wDj1-
z2f+qKA_uXb^oyJYAid;KAK%Xs1S{zW!e14z^s?`z+g|Q|efvb8^qh;>1onkR?LnNp
zod@A6j#5A=pcGIFC<T-PN&%&SQa~x76i^B%1s;|HQ|Xo4>E$2e>Gv=F@df=serFd>
z=^Lqax`Om-YW*wdOD}&JKX<*Qdkmn)U%9lgQoWmAy|l4fy?f^oB6HTP-O~Lw^}-fk
za#I_tr|+&r&t2zrH7Ny@0!jg;fKosypcGIFC<T-PN&%&SQa~y2kQ88D`eFK?w@&-5
zZ*C^h^`k$J{yh3C=xOZq9kACYfi1;ZS5qYVA^tP|F~rHP{k*w(fH=pR<L_vrvG6Ps
zL~R89O|16>>u0y>XA%Dn>yHNO6MlUvTP=E3Yi2%EykIT5<x0L(v<42@1NMN`J5c4-
zfj0+-?EWEpa6jC`B@AG+GrpS8BO**RLinIs6PnhKldgwZ$cTm@Y=mB>aumW*h(s<@
zSSA2MMkEBxFnFdzI09OP-bxT<9XmB@)rN3t01|<nX#lfPTYIAkVuWJ25QAo}kI6^|
zcIGp}M{#P>)|Q%o!XT!tc?LxIKBh)uurs%CHnWoTYmKFG7T=O@4y=_W5^aW_D7VMk
zyArKzYHy|8YsbrKfs<``qx(Ql95GE3Z>WtuZR!creAYaSsS(4<$u>%cF%8>HPf#+_
zd9)KCVqp6a22l#|25r{bwIiJD@JU2z90E<G#4sfz*H%oNK%mLsb$^+dpU?(GhL6z<
zRQ(zBjIrOFW8~;94>SzADFmH$Yc0lZ7NO+H{mVBx^BYV0#fGQueVrHy;Z<q!y%F7H
zMnmKBw?=5$Xb7J&SlE0{3*$<h)gMpN3eaKfYK+>(&hUs~nbTYK^P=8nbTod?%ezfm
z)3E(Np2I>WU$kcy7NE~7Ec6Y)%v`1n`C`#6!(wJ0BM(mU?8T~wm{ajGWe=QJ#t(8}
zbTF0Y87J|q#EIEbsQ}(wzQP|m5>DQ@yft|XV&mF{(0L+0>4{q41AjngJvyGVB(LCv
zqYrHu?I_w9+9|Ygv`MsSv{%twv<li=Xz!xEhxU)d-|m;q?Fb?GJKo5N__SKdzqIVZ
zYzJMu_=NbggxKR~ar*Y^Phzw+_G7pmP7nQd_(8<?;QCU=9<dZXp@d6W`F9l6YQj3s
ziSa2Xm3EF#9y^_K(qoh0IN3@?ex`0N<7L2ci@E!LzF=2gD|neKdavyJIT?BGk_Was
z=ggKf1=pF&mA5jmXG(>FiwDZix!LLkCtsW`$vBhEmfb~}&F71*jB#lhW^%Ni-%J#|
z@LFCjUYO5Uyo#)2$tlgwR$PSsm(hi*&>5+{V<kRc`Q8?N(*{TV`&INgQ9qs-f5>xV
z`RP6ZOGigPIOxD*|AOrA75%-I&sqwSG3LK40qhXQv_GAz;J9k}8Lb~2#HAtq^MRNb
z#Pp5j2fqbDjAfsM>EW!$sCKi*XI%4LZLa6iZ*M(D<STppmQQ?Fo9nrASRt(b{(JZO
zEo-$-Kal;>(=n29kovjkw}Rj{+{*&1-~X2AuRkwxFX=znW=-1i2hmUe4G9j}C$Wz(
zsK=;=zqfqQmuPU*b19c+FUU)a{&fL1qo=bg&tHeY8_??_4!YrlpG^cojO+JrMQpqN
L7196czx4kJjnh(U

literal 0
HcmV?d00001

diff --git a/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp b/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp
new file mode 100644
index 000000000000..32a9c63e9b71
--- /dev/null
+++ b/lldb/test/API/functionalities/postmortem/elf-core/expr/main.cpp
@@ -0,0 +1,15 @@
+struct Inner {
+  Inner(int val) : val(val) {}
+  int val;
+};
+
+struct Outer {
+  Outer(int val) : inner(val) {}
+  Inner inner;
+};
+
+extern "C" void _start(void) {
+  Outer outer(5);
+  char *boom = (char *)0;
+  *boom = 47;
+}
diff --git a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
index 9af53845ca1b..583e187cb6d3 100644
--- a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
+++ b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
@@ -16,6 +16,28 @@ def setUp(self):
             self.broadcaster, lldb.SBDebugger.eBroadcastBitProgress
         )
 
+    def test_wait_attach_progress_reporting(self):
+        """Test that progress reports for wait attaching work as intended."""
+        target = self.dbg.CreateTarget(None)
+
+        # The waiting to attach progress message will get emitted upon
+        # trying to attach, but it's not going to be the event picked
+        # up by checking with fetch_next_event, so go through all emitted
+        # progress events and check that the waiting to attach one was emitted at all.
+        target.AttachToProcessWithName(
+            self.listener,
+            "wait-attach-progress-report",
+            False,
+            lldb.SBError(),
+        )
+        event = lldb.SBEvent()
+        events = []
+        while self.listener.GetNextEventForBroadcaster(self.broadcaster, event):
+            progress_data = lldb.SBDebugger.GetProgressDataFromEvent(event)
+            message = progress_data.GetValueForKey("message").GetStringValue(100)
+            events.append(message)
+        self.assertTrue("Waiting to attach to process" in events)
+
     def test_dwarf_symbol_loading_progress_report(self):
         """Test that we are able to fetch dwarf symbol loading progress events"""
         self.build()
diff --git a/lldb/test/API/functionalities/statusline/TestStatusline.py b/lldb/test/API/functionalities/statusline/TestStatusline.py
index 68c5ad3090cf..087e62b387d7 100644
--- a/lldb/test/API/functionalities/statusline/TestStatusline.py
+++ b/lldb/test/API/functionalities/statusline/TestStatusline.py
@@ -27,10 +27,12 @@ def do_setup(self):
         self.expect("run", substrs=["stop reason"])
         self.resize()
 
-    def resize(self):
+    def resize(self, height=None, width=None):
+        height = self.TERMINAL_HEIGHT if not height else height
+        width = self.TERMINAL_WIDTH if not width else width
         # Change the terminal dimensions. When we launch the tests, we reset
         # all the settings, leaving the terminal dimensions unset.
-        self.child.setwinsize(self.TERMINAL_HEIGHT, self.TERMINAL_WIDTH)
+        self.child.setwinsize(height, width)
 
     def test(self):
         """Basic test for the statusline."""
@@ -65,9 +67,7 @@ def test(self):
         self.expect('set set separator "| "')
 
         # Hide the statusline and check or the control character.
-        self.expect(
-            "set set show-statusline false", ["\x1b[1;{}r".format(self.TERMINAL_HEIGHT)]
-        )
+        self.expect("set set show-statusline false", ["\x1b[1;0r"])
 
     def test_no_color(self):
         """Basic test for the statusline with colors disabled."""
@@ -104,3 +104,14 @@ def test_no_target(self):
         self.resize()
 
         self.expect("set set show-statusline true", ["no target"])
+
+    @skipIfEditlineSupportMissing
+    def test_resize(self):
+        """Test that move the cursor when resizing."""
+        self.launch(timeout=self.TIMEOUT)
+        self.resize()
+        self.expect("set set show-statusline true", ["no target"])
+        self.resize(20, 60)
+        # Check for the escape code to resize the scroll window.
+        self.child.expect(re.escape("\x1b[1;19r"))
+        self.child.expect("(lldb)")
diff --git a/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py b/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
index 4eb5351eefc8..08f09b317b21 100644
--- a/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
+++ b/lldb/test/API/lang/cpp/char8_t/TestCxxChar8_t.py
@@ -24,9 +24,7 @@ def test_without_process(self):
 
         self.expect_expr("a", result_type="char8_t", result_summary="0x61 u8'a'")
         self.expect_expr("ab", result_type="const char8_t *", result_summary='u8"你好"')
-
-        # FIXME: This should work too.
-        self.expect("expr abc", substrs=['u8"你好"'], matching=False)
+        self.expect_expr("abc", result_type="char8_t[9]", result_summary='u8"你好"')
 
     @skipIf(compiler="clang", compiler_version=["<", "7.0"])
     def test_with_process(self):
diff --git a/lldb/test/API/lang/objc/class-without-methods/Makefile b/lldb/test/API/lang/objc/class-without-methods/Makefile
new file mode 100644
index 000000000000..f6f336c01bbc
--- /dev/null
+++ b/lldb/test/API/lang/objc/class-without-methods/Makefile
@@ -0,0 +1,5 @@
+OBJC_SOURCES := Point.m main.m
+include Makefile.rules
+
+# Only objc metadata, no debug info, for Point.m
+Point.o: CFLAGS_EXTRAS += -g0
diff --git a/lldb/test/API/lang/objc/class-without-methods/Point.h b/lldb/test/API/lang/objc/class-without-methods/Point.h
new file mode 100644
index 000000000000..a352259954ef
--- /dev/null
+++ b/lldb/test/API/lang/objc/class-without-methods/Point.h
@@ -0,0 +1,4 @@
+#import <objc/NSObject.h>
+
+@interface Point : NSObject
+@end
diff --git a/lldb/test/API/lang/objc/class-without-methods/Point.m b/lldb/test/API/lang/objc/class-without-methods/Point.m
new file mode 100644
index 000000000000..770a857c80dc
--- /dev/null
+++ b/lldb/test/API/lang/objc/class-without-methods/Point.m
@@ -0,0 +1,7 @@
+#import "Point.h"
+
+@implementation Point {
+  float _x;
+  float _y;
+}
+@end
diff --git a/lldb/test/API/lang/objc/class-without-methods/TestObjCClassWithoutMethods.py b/lldb/test/API/lang/objc/class-without-methods/TestObjCClassWithoutMethods.py
new file mode 100644
index 000000000000..f83b0352bf1a
--- /dev/null
+++ b/lldb/test/API/lang/objc/class-without-methods/TestObjCClassWithoutMethods.py
@@ -0,0 +1,11 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.m"))
+        self.expect("frame var -P1 p", substrs=["_x = 0", "_y = 0"])
diff --git a/lldb/test/API/lang/objc/class-without-methods/main.m b/lldb/test/API/lang/objc/class-without-methods/main.m
new file mode 100644
index 000000000000..0b6e14371bc5
--- /dev/null
+++ b/lldb/test/API/lang/objc/class-without-methods/main.m
@@ -0,0 +1,7 @@
+#import "Point.h"
+
+int main() {
+  Point *p = [Point new];
+  // break here
+  return 0;
+}
diff --git a/lldb/test/API/linux/aarch64/mte_core_file/TestAArch64LinuxMTEMemoryTagCoreFile.py b/lldb/test/API/linux/aarch64/mte_core_file/TestAArch64LinuxMTEMemoryTagCoreFile.py
index a9879f67d8b8..bfdc8229094f 100644
--- a/lldb/test/API/linux/aarch64/mte_core_file/TestAArch64LinuxMTEMemoryTagCoreFile.py
+++ b/lldb/test/API/linux/aarch64/mte_core_file/TestAArch64LinuxMTEMemoryTagCoreFile.py
@@ -248,3 +248,26 @@ def test_mte_ctrl_register(self):
                     "TCF: 0 = TCF_NONE, 1 = TCF_SYNC, 2 = TCF_ASYNC, 3 = TCF_ASYMM"
                 ],
             )
+
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_mte_no_tags(self):
+        """Test that we handle there being a tag segment but that segment does
+        not contain any tag data. This can happen when the core is dumped
+        with a restrictive limit or filter."""
+        self.runCmd("target create --core core.mte.notags")
+
+        mte_buf_addr = 0xFFFFA4AF3000
+
+        # We can see which memory was tagged.
+        self.expect(
+            f"memory region {mte_buf_addr}", substrs=["memory tagging: enabled"]
+        )
+
+        # We cannot read those tags.
+        self.expect(
+            f"memory tag read {mte_buf_addr}",
+            substrs=[
+                "Could not read tags from core file segment. Segment is missing some or all tag data."
+            ],
+            error=True,
+        )
diff --git a/lldb/test/API/linux/aarch64/mte_core_file/core.mte.notags b/lldb/test/API/linux/aarch64/mte_core_file/core.mte.notags
new file mode 100644
index 0000000000000000000000000000000000000000..8f9d60668a84d99b425ab7bfe20c02b67e153542
GIT binary patch
literal 32768
zcmeI5dr(x@8Nk2Yy9-Oix;|2gA>r~cqGMK2V<b(7%dV)j6dkh3WRgjjbr*#IyShNC
z(TuA?d~BKu$+XpEMo>|gaT2FvTCwdw%%dizQ-#`T)6N9VLo=ogU>>02?f2b#4(u+F
zv?iTN+jH>V^Z3qp&Ub$2JLlf(A7?k_7cMp$48Vg4eg(rh0EG)8Vg(k$6qYx!xQQns
z>QOQyF}i?Pqdv?elIcd_FCVtH>Zz!^mE~AWdRiMYul4r7tQSWrT+e!gdL_?Z8Qo5!
zZcv1s(c^2_uc~^LsQb>O5qesI%xdinkFM9l<PrTvaS67LuBVLDi{jEThMxQ-dfj8_
zb&S-D;vyd&-OeK;c}CeOzc{)c;{g#|qV%$EP)`{{Zxk+3^H36_r!XEoj<8eXQo`g-
zF?up9=W&Fd)?bjxJ7V-$@vz_P1+<PdE<skI`v&#oq`^`B+8RB7m|Ly&#QXcQd0=J{
zlWk}$+3Q$VoKMB$=*B{1nwL$-%JFzne1;cd{QQK73?|Rq9-SX6$M?Di|2jIO-z4aJ
z*U=UF-f$CZ$UWb^a&A^u7TK-^HfApW7B83ec6)A{7R8e5@%VJYPosSPVv1gWNIqWQ
z(-bzBQ!H#xJrcb(`<dQpR-c<ua=YH%HKv!v@@v`nuwDt(<^5@MgLw%rJ~vQIH88{}
zT67VGbj8kz+vcyWu~pTSYPA4tbB0P}R`iu3vMq>L!IFaexNY255?IUDv3!upzwSu0
z3E|PY<2I%8PMUNdn8)&~C+P{c%wenA@G)`M)+~vbV4_|Pyey`<prEklskS#@exX*%
zp1GLj4B}ZiuzlFV^DO4|AYS(+&XDGm0)M>o{CK@sn{!HXALsI^OwOZQSYlzC?t9Z%
ze$zQu(3Awh+gP5DD~07hnxog}OL{5Ozm>_8V`7=*V<VcA0B%nCg$0ZGobdI@84tai
zSYzK&*gHh*y9Ce6x}2{U0QL?L`%LF~_RQYS75FopJ(Fo4)B?*xx4y2Q;^q8VdisT^
zeF8ezzQy*1S*#ggd4MBY{n@O1PMN=|bdJ~kSoP}itnA^12{$09tq#noEMGCl?GAX#
z7R<l1vfNj@_RdwlS}OI}s{OVFFz1_7$;%96=8}EcUufUgqoA)#dSv3ip}k>xOu*gJ
zwW4<AJ@WL`LoNy(`lRvoq2+o2CiIN_BUumVPirmK7BAj-tT!d@-%tXy_tUm-z)iid
zpy>VwF)cq3{Z2x4EVv&SBkEZBrj4@y=fK%Wg3SmwAU5=_V2WU)V&Fa(7g4S&Z)9z3
z)ZR(xz2T;aI{dk;F-KWKgzT!Szjbigga{dbA0Zii8CxS{m=!?~5CjAPK|l}?1Ox#=
zKoAfF1OY)n5D)|e0YN|z5CjAPK|l}?1Ox#=KoAfF1OY)n5D)|e0YN|z5CjAPK|l}?
z1Ox#=KoAfF1OY)n5D)|e0YN|z5CjAPK|l}?1Ox#=KoAfF1OY)n5D)|e0YN|z5CjAP
zK|l}?1Ox#=KoAfF1OY)n5csbourcQRe`$>K|NqPn^2Jfo`Tr8$$K%1Fbo4Bpp*vPc
zVtPE5Sw%YM6p!|Ue(mhhk@3(W)9h?pel9WT(6y5q7$tV<#lT~#OM%WurLzUeR%(*Y
z7mN(l96v*q>qN?V`Uy--gx(P+`P;C3682EY4lt}_3X3qdV)@@#IUfg|ZAkT-7`(%5
zoDZ>bU=_D9qex8kxsg9-;|G_kqBP(ut#o-R{hkWfs?wU8@~YC|VpiZ^>2<FgE*+{I
zD6JkY4b-@*%U1<TYia|&;ZkfN;P!d_RiGc<`<NF`#nleR)ea$b&9yDC&C5aij1hkZ
zq)dO!?f)20=cm(o%vE?%ps^V7zWtOdo}aBwTqzKHDx+a9DY_pF%Ub51SGOL2(fhM0
zTb{l)ZO43gi{_5cQ6ZftjCrueX>+P!#M@9g$-W?@+zV?in!%X{!D5efQ?Xs{nC87O
zXmx<}s1gUxCqZd@1<~JOhP&Hyq_dk%O4|w(%ukV}vnRU0b2cB0Z@%xD9%=_=pnka=
zdg`zeZppNVw`MqIQvGeH{|;CxULI7P3M8JoAjRYH%j0&O&)wB&uRhoZneop-#v8j+
zz<8;^0M7SvcQx*R>R2DOPZy3f<T1fSk{?o4XN!^^nhp>Kw6{ZY{VDTX!A>0RA1S>B
zZ8O4=NgKgf(Jg^97xho8>QtBM)<83WGsOyPHeFYpG`=^n95yJ=#9@821sdwD=Fn!e
zb+bVq=Rq9L41j@yCX7umKx5g8au>u`yo_y7|J~RI@o+(01+k+3KE^WY$KkzhO?%|J
z)dQP%m!h6IL2q{j@w=i9@mq}iKJeTTdI$TnY}nAX5-g!gWcWHVT*Vkxp?*7z*BI8J
zJd)$P$Z@mP9wv?s;)yJY+3R2o8PUcRkS=|QW9M8+X4iO18qX6-S}2Y2JWNcGMf^t_
zx`I{+)x+|(dP%oC4weoWO-nTn+ps*vV6HIWd;s!!A8n{BTh*@sg!0jb6WB)zm`C)h
z&!buE*Makx&iZ{C`z3b2Wbt`oh<Un9U4vHH3`gsc!9MiEmbhU*G{z76VI)WL!6##U
zupNEy9QvT0`Jf$r5Ser?LVNAV&W_w%V7fFDeGtdf*us3!4ra~&NJjqmI@;|yzP;Fy
zxHn+mu<tqhwUu3Q76`imUgAFK{rw;wWqo9l{DbRO@=5YXfV$nrj=VNx-3G7bwaF%o
zaeh9*u}1kV&K0qE70WYk9`<c2+k)I*QRCS<T;66>hy5sW9frDd8=g#B-nKP)dE2AY
z^TMUg_AvQx2=zA_B+ZZPaMP&|q_|mjNZ3I>M80E}<DRQavWM*%zuRqxxKInuV;tJ(
zit!;k`4{>Sl@pT#${nF!)11*>Vg#r6wL#pCj!dFGMasW!-HHroUn!lmsaVCe;z1TJ
zFh81#tZ8j37=yg2vnjr*FAv~oZp8X7IhognYkAfcY=3FQS|{J2{pX9~vWYssvyz-L
z)|HzsogG|%EIcSd!{z%7;ZGz(>!%MJ!=Fmf`k7>$MeSd$Hiz-t{oZSm)T-vfK+uwR
zT%HKdlG#clD8_~qlhV5l^LrFyuidib@k5yY+#`h-{`R^R(v_T)WC(Oy!I^q994*PP
z=iu79`||f|evu24FUdQf+5MxSp<$rH6wX6l-Sc2T#`v5c+DbBvIUp$mn;gnO3K%ZA
zCxf#--Ja7uPZ=;I20PQheCn!nbE^yB(iODVzB9P{d>p()HuLLkBOJZ_uAyPq7DIze
zg<jfYyQN!O9qQt)^U~zjbJEn-y#UV+V60qtyz!x_O5=J7Lb~jh)@~`e)d9iIU07#N
zx-rL$>EjZ-L^hFaI9HeQaBO>K7&W^-_c+1>6^p`GtMkK`@5>8+`mjAbxc=m!cH~TZ
zG(P=Wi6t>?!acJa?LQ|?X+1A}d)CP`eZEH1J1<RZt(VLpS2FY_SitF!;fx7wabI(E
zrJ^sU_B_$mROan3smSPWFU#n!lc6CO*Q^`;)7|q#V+-a>(xIUR3?Z~3lxI;I_kwl7
zKHOJQt#C$K1jpL($~8eRK3tqwzAn{*<r%QH9j{I3m(8H$7yxqG@e1NRte1?Tha}@E
ze4397BAXWU(ai(uT}`H7;~y}MY<d-o@^}Y6iz0o6+9y6uhG64sSdW;E#$hamu0-fP
ziEYq4y3k(bs_Hx^8C$!sEh?ilJ>N>Qpur7>25C~T(`d%$J9PAIkzio+IA}}3v64@{
zjC`mc+>1K*GFz@;9CYZ<JcB9N8ARWqX9=Gl(mO|MHZfRiMn9ynKTLi5m9`WUtZ7Eu
z$RD^byn!<E>2CDtLQL<$wj<?egSOwhGNiK>?+2YHfBaUT#df&wP~uPe61;!!-2TE_
zeFd{$>?=4>-)FkbQj~F9QqlB7O5u{VN@A7?CRnHKED9#?FKSM1DLRpSpy=J?gGF`5
zU@_D+7kQFppT|<-v)An_%1%!6r9LowS?c21ev++0S^5K+%hDHT`iJE+P(H(-DjR%u
z#pv^tNWQPsP6=yAmV9*<SmVaIg1Y@hsdX(y_PPT_p1OlYAVcAWx}G1RUpC+~vbT9a
zbuL+p9~bYzG6P;lyd=ELc*WsG>nQk{s$RoQaWYseM?Y89ewOa(SJh8g`n>^F?P2MK
zS5@`->#90UJK=ozcS-m=DgT`n{;J5|FE6qJ_L>eIxBw{d?}sFo=Re4>Ll+PIMiIT|
zQ$U~cQ~aQj?g02hKU$t;p@-$a#|!jo|FJv&?i#CuB@V|ua%NtjO2)Tv+3ELJddl48
zJ~?~7ZLY1jv@%=HwAYqbdS!Q@>P{@YYwpaMx-eUwn>BZSc6K&?64<KCs%ru@?iFAw
zb61yv&AZN5y{<}+YXV@y9|EyGRvM@-_xqwUE-%=8e*ASHTTSWO8nC%5Ys!7tRIR55
zt5$0bxU0%NU<>%Y?ix4PJpQVxQeVwj<N0^X=_eCt&iQn3Kj3i|Ymmz=EYJOhewT;Z
z;BxLKJU+zg({~QJ5ZCAaatoG`jO%m1<1zO$tfB?xhCB<Yn8M@x{al~>6_5GnJeSAD
z^gT;H%FDUm@^~TZiQCBS=LUQi6ZFZ;^<%%+@dF|%aBjTNfe9KDUuPN&5>kg4jwR?Z
zzso$q;sT6GpZ9-^{XYYb?~SP+%a6Xpsc+WZi2tY18jq0k@$-GegEDH9efC2?`(Y#=
z%P5NsT%X4kC?tLEXS^=|f0TS)z+!HHWXx@ktojI>_}GQQ^&;zvv>+e|2m*qDARq_`
S0)l`bAP5Kog24X{fqw$@NM;oP

literal 0
HcmV?d00001

diff --git a/lldb/test/API/linux/aarch64/mte_core_file/main.c b/lldb/test/API/linux/aarch64/mte_core_file/main.c
index 89027e0ea75d..6537edd7bdb9 100644
--- a/lldb/test/API/linux/aarch64/mte_core_file/main.c
+++ b/lldb/test/API/linux/aarch64/mte_core_file/main.c
@@ -5,9 +5,14 @@
 //
 // Compile with:
 // <gcc or clang> -march=armv8.5-a+memtag -g main.c -o a.out.mte
+// (use a.out.mte to generate core.mte.notags and core.mte)
 // <gcc or clang> -march=armv8.5-a+memtag -g main.c -DNO_MTE -o a.out.nomte
 //
-// /proc/self/coredump_filter was set to 2 when the core files were made.
+// Set /proc/self/coredump_filter to the following values when generating the
+// core files:
+// * core.mte - 3
+// * core.mte.notags - 2
+// * core.nomte - 3
 
 #include <arm_acle.h>
 #include <asm/mman.h>
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/Makefile b/lldb/test/API/macosx/lc-note/additional-registers/Makefile
new file mode 100644
index 000000000000..e232ec5d6967
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/Makefile
@@ -0,0 +1,11 @@
+MAKE_DSYM := NO
+C_SOURCES := main.c
+CXXFLAGS_EXTRAS := -std=c++17
+
+all: a.out add-lcnote
+
+add-lcnote:
+	"$(MAKE)" -f "$(MAKEFILE_RULES)" EXE=add-lcnote \
+	    CXX=$(CC) CXXFLAGS_EXTRAS="$(CXXFLAGS_EXTRAS)" CXX_SOURCES=add-lcnote.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py b/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py
new file mode 100644
index 000000000000..cf9f055bd78f
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/TestMetadataRegisters.py
@@ -0,0 +1,150 @@
+"""Test that lldb will read additional registers from Mach-O LC_NOTE metadata."""
+
+import os
+import re
+import subprocess
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestMetadataRegisters(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipUnlessDarwin
+    @skipIfRemote
+    def test_add_registers_via_metadata(self):
+        self.build()
+        self.aout_exe = self.getBuildArtifact("a.out")
+        lldb_corefile = self.getBuildArtifact("lldb.core")
+        metadata_corefile = self.getBuildArtifact("metadata.core")
+        bad_metadata1_corefile = self.getBuildArtifact("bad-metadata1.core")
+        bad_metadata2_corefile = self.getBuildArtifact("bad-metadata2.core")
+        add_lcnote = self.getBuildArtifact("add-lcnote")
+
+        (target, process, t, bp) = lldbutil.run_to_source_breakpoint(
+            self, "break here", lldb.SBFileSpec("main.c")
+        )
+
+        self.assertTrue(process.IsValid())
+
+        if self.TraceOn():
+            self.runCmd("bt")
+            self.runCmd("reg read -a")
+
+        self.runCmd("process save-core " + lldb_corefile)
+        process.Kill()
+        target.Clear()
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % metadata_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{"threads":[{"register_info":
+                   {"sets":["Special Registers", "General Purpose Registers"],
+                    "registers":[
+                       {"name":"jar", "value":10, "bitsize": 32, "set": 0},
+                       {"name":"bar", "value":65537, "bitsize":16, "set":0},
+                       {"name":"mar", "value":65537, "bitsize":32, "set":0},
+                       {"name":"anotherpc", "value":55, "bitsize":64, "set": 1}]}}]}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % bad_metadata1_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{lol im bad json}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        cmd = (
+            add_lcnote
+            + " -r"
+            + " -i '%s'" % lldb_corefile
+            + " -o '%s'" % bad_metadata2_corefile
+            + " -n 'process metadata' "
+            + " -s '"
+            + """{"threads":[
+                   {"register_info":
+                     {"sets":["a"],"registers":[{"name":"a", "value":1, "bitsize": 32, "set": 0}]}},
+                   {"register_info":
+                     {"sets":["a"],"registers":[{"name":"a", "value":1, "bitsize": 32, "set": 0}]}}
+                 ]}"""
+            + "'"
+        )
+        call(cmd, shell=True)
+
+        # Now load the corefile
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(metadata_corefile)
+        self.assertTrue(process.IsValid())
+        if self.TraceOn():
+            self.runCmd("bt")
+            self.runCmd("reg read -a")
+
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+
+        # Register sets will be
+        #    from LC_THREAD:
+        #       General Purpose Registers
+        #       Floating Point Registers
+        #       Exception State Registers
+        #    from LC_NOTE metadata:
+        #       Special Registers
+        self.assertEqual(frame.registers[0].GetName(), "General Purpose Registers")
+        self.assertEqual(frame.registers[3].GetName(), "Special Registers")
+
+        anotherpc = frame.registers[0].GetChildMemberWithName("anotherpc")
+        self.assertTrue(anotherpc.IsValid())
+        self.assertEqual(anotherpc.GetValueAsUnsigned(), 0x37)
+
+        jar = frame.registers[3].GetChildMemberWithName("jar")
+        self.assertTrue(jar.IsValid())
+        self.assertEqual(jar.GetValueAsUnsigned(), 10)
+        self.assertEqual(jar.GetByteSize(), 4)
+
+        bar = frame.registers[3].GetChildMemberWithName("bar")
+        self.assertTrue(bar.IsValid())
+        self.assertEqual(bar.GetByteSize(), 2)
+
+        mar = frame.registers[3].GetChildMemberWithName("mar")
+        self.assertTrue(mar.IsValid())
+        self.assertEqual(mar.GetValueAsUnsigned(), 0x10001)
+        self.assertEqual(mar.GetByteSize(), 4)
+
+        # bad1 has invalid JSON, no additional registers
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(bad_metadata1_corefile)
+        self.assertTrue(process.IsValid())
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+        gpr_found = False
+        for regset in frame.registers:
+            if regset.GetName() == "General Purpose Registers":
+                gpr_found = True
+        self.assertTrue(gpr_found)
+
+        # bad2 has invalid JSON, more process metadata threads than
+        # LC_THREADs, and should be rejected.
+        target = self.dbg.CreateTarget("")
+        process = target.LoadCore(bad_metadata2_corefile)
+        self.assertTrue(process.IsValid())
+        thread = process.GetSelectedThread()
+        frame = thread.GetFrameAtIndex(0)
+        metadata_regset_found = False
+        for regset in frame.registers:
+            if regset.GetName() == "a" or regset.GetName() == "b":
+                metadata_regset_found = True
+        self.assertFalse(metadata_regset_found)
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp b/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp
new file mode 100644
index 000000000000..c5ab732c0c7c
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/add-lcnote.cpp
@@ -0,0 +1,385 @@
+#include <getopt.h>
+#include <mach-o/loader.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include <iostream>
+#include <optional>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+[[noreturn]] void print_help(void) {
+  fprintf(stderr, "Append an LC_NOTE to a corefile.  Usage: \n");
+  fprintf(stderr, "   -i|--input <corefile>\n");
+  fprintf(stderr, "   -o|--output <corefile>\n");
+  fprintf(stderr, "   -n|--name <LC_NOTE name>\n");
+  fprintf(
+      stderr,
+      "   -r|--remove-dups  remove existing LC_NOTEs with this same name\n");
+  fprintf(stderr, "  One of:\n");
+  fprintf(stderr, "   -f|--file <file to embed as LC_NOTE payload>\n");
+  fprintf(stderr, "   -s|--str <string to embed as LC_NOTE payload>\n");
+  exit(1);
+}
+
+void parse_args(int argc, char **argv, string &infile, string &outfile,
+                string &note_name, vector<uint8_t> &payload,
+                bool &remove_dups) {
+  const char *const short_opts = "i:o:n:f:s:hr";
+  const option long_opts[] = {{"input", required_argument, nullptr, 'i'},
+                              {"output", required_argument, nullptr, 'o'},
+                              {"name", required_argument, nullptr, 'n'},
+                              {"file", required_argument, nullptr, 'f'},
+                              {"str", required_argument, nullptr, 's'},
+                              {"remove-dups", no_argument, nullptr, 'r'},
+                              {"help", no_argument, nullptr, 'h'},
+                              {nullptr, no_argument, nullptr, 0}};
+  optional<string> infile_str, outfile_str, name_str, payload_file_str,
+      payload_str;
+  remove_dups = false;
+  while (true) {
+    const auto opt = getopt_long(argc, argv, short_opts, long_opts, nullptr);
+    if (opt == -1)
+      break;
+    switch (opt) {
+    case 'i':
+      infile_str = optarg;
+      break;
+    case 'o':
+      outfile_str = optarg;
+      break;
+    case 'n':
+      name_str = optarg;
+      break;
+    case 'f':
+      payload_file_str = optarg;
+      break;
+    case 's':
+      payload_str = optarg;
+      break;
+    case 'r':
+      remove_dups = true;
+      break;
+    case 'h':
+      print_help();
+    }
+  }
+
+  if (!infile_str || !outfile_str || !name_str ||
+      (!payload_file_str && !payload_str))
+    print_help();
+
+  infile = *infile_str;
+  outfile = *outfile_str;
+  note_name = *name_str;
+  if (payload_str) {
+    payload.resize(payload_str->size(), 0);
+    memcpy(payload.data(), payload_str->c_str(), payload_str->size());
+  } else {
+    struct stat sb;
+    if (stat(payload_file_str->c_str(), &sb)) {
+      fprintf(stderr, "File '%s' does not exist.\n", payload_file_str->c_str());
+      exit(1);
+    }
+    payload.resize(sb.st_size, 0);
+    FILE *f = fopen(payload_file_str->c_str(), "r");
+    fread(payload.data(), 1, sb.st_size, f);
+    fclose(f);
+  }
+}
+
+struct all_image_infos_header {
+  uint32_t version;         // currently 1
+  uint32_t imgcount;        // number of binary images
+  uint64_t entries_fileoff; // file offset in the corefile of where the array of
+                            // struct entry's begin.
+  uint32_t entry_size;      // size of 'struct entry'.
+  uint32_t unused;          // set to 0
+};
+
+struct image_entry {
+  uint64_t filepath_offset; // corefile offset of the c-string filepath,
+                            // if available, else this should be set
+                            // to UINT64_MAX.
+  uuid_t uuid;              // uint8_t[16].  should be set to all zeroes if
+                            // uuid is unknown.
+  uint64_t
+      load_address; // virtual addr of mach-o header, UINT64_MAX if unknown.
+  uint64_t seg_addrs_offset; // corefile offset to the array of struct
+                             // segment_vmaddr's, UINT64_MAX if none.
+  uint32_t segment_count; // The number of segments for this binary, 0 if none.
+  uint32_t
+      executing; // Set to 0 if executing status is unknown by corefile
+                 // creator.
+                 // Set to 1 if this binary was executing on any thread,
+                 // so it can be force-loaded by the corefile reader.
+                 // Set to 2 if this binary was not executing on any thread.
+};
+
+int count_lc_notes_with_name(FILE *in, std::string name) {
+  fseeko(in, 0, SEEK_SET);
+
+  uint8_t magic[4];
+  if (fread(magic, 1, 4, in) != 4) {
+    printf("Failed to read magic number\n");
+    return 0;
+  }
+  uint8_t magic_32_le[] = {0xce, 0xfa, 0xed, 0xfe};
+  uint8_t magic_64_le[] = {0xcf, 0xfa, 0xed, 0xfe};
+
+  if (memcmp(magic, magic_32_le, 4) != 0 &&
+      memcmp(magic, magic_64_le, 4) != 0) {
+    return 0;
+  }
+
+  fseeko(in, 0, SEEK_SET);
+
+  int number_of_load_cmds = 0;
+  size_t size_of_mach_header = 0;
+  if (memcmp(magic, magic_64_le, 4) == 0) {
+    struct mach_header_64 mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return 0;
+    }
+    number_of_load_cmds = mh.ncmds;
+  } else {
+    struct mach_header mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return 0;
+    }
+    number_of_load_cmds = mh.ncmds;
+  }
+
+  int notes_seen = 0;
+  fseeko(in, size_of_mach_header, SEEK_SET);
+  for (int i = 0; i < number_of_load_cmds; i++) {
+    off_t cmd_start = ftello(in);
+    uint32_t cmd, cmdsize;
+    fread(&cmd, sizeof(uint32_t), 1, in);
+    fread(&cmdsize, sizeof(uint32_t), 1, in);
+
+    fseeko(in, cmd_start, SEEK_SET);
+    off_t next_cmd = cmd_start + cmdsize;
+    if (cmd == LC_NOTE) {
+      struct note_command note;
+      fread(&note, sizeof(note), 1, in);
+      if (strncmp(name.c_str(), note.data_owner, 16) == 0)
+        notes_seen++;
+    }
+    fseeko(in, next_cmd, SEEK_SET);
+  }
+  return notes_seen;
+}
+
+void copy_and_add_note(FILE *in, FILE *out, std::string lc_note_name,
+                       vector<uint8_t> payload_data, bool remove_dups) {
+  int number_of_load_cmds = 0;
+  off_t header_start = ftello(in);
+
+  int notes_to_remove = 0;
+  if (remove_dups)
+    notes_to_remove = count_lc_notes_with_name(in, lc_note_name);
+  fseeko(in, header_start, SEEK_SET);
+
+  uint8_t magic[4];
+  if (fread(magic, 1, 4, in) != 4) {
+    printf("Failed to read magic number\n");
+    return;
+  }
+  uint8_t magic_32_le[] = {0xce, 0xfa, 0xed, 0xfe};
+  uint8_t magic_64_le[] = {0xcf, 0xfa, 0xed, 0xfe};
+
+  if (memcmp(magic, magic_32_le, 4) != 0 &&
+      memcmp(magic, magic_64_le, 4) != 0) {
+    return;
+  }
+
+  fseeko(in, header_start, SEEK_SET);
+
+  off_t end_of_infine_loadcmds;
+  size_t size_of_mach_header = 0;
+  if (memcmp(magic, magic_64_le, 4) == 0) {
+    struct mach_header_64 mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return;
+    }
+    number_of_load_cmds = mh.ncmds;
+    end_of_infine_loadcmds = sizeof(mh) + mh.sizeofcmds;
+
+    mh.ncmds += 1;
+    mh.ncmds -= notes_to_remove;
+    mh.sizeofcmds += sizeof(struct note_command);
+    mh.sizeofcmds -= notes_to_remove * sizeof(struct note_command);
+    fseeko(out, header_start, SEEK_SET);
+    fwrite(&mh, sizeof(mh), 1, out);
+  } else {
+    struct mach_header mh;
+    size_of_mach_header = sizeof(mh);
+    if (fread(&mh, sizeof(mh), 1, in) != 1) {
+      fprintf(stderr, "unable to read mach header\n");
+      return;
+    }
+    number_of_load_cmds = mh.ncmds;
+    end_of_infine_loadcmds = sizeof(mh) + mh.sizeofcmds;
+
+    mh.ncmds += 1;
+    mh.ncmds -= notes_to_remove;
+    mh.sizeofcmds += sizeof(struct note_command);
+    mh.sizeofcmds -= notes_to_remove * sizeof(struct note_command);
+    fseeko(out, header_start, SEEK_SET);
+    fwrite(&mh, sizeof(mh), 1, out);
+  }
+
+  off_t start_of_infile_load_cmds = ftello(in);
+  fseek(in, 0, SEEK_END);
+  off_t infile_size = ftello(in);
+
+  // LC_SEGMENT may be aligned to 4k boundaries, let's maintain
+  // that alignment by putting 4096 minus the size of the added
+  // LC_NOTE load command after the output file's load commands.
+  off_t end_of_outfile_loadcmds =
+      end_of_infine_loadcmds - (notes_to_remove * sizeof(struct note_command)) +
+      4096 - sizeof(struct note_command);
+  off_t slide = end_of_outfile_loadcmds - end_of_infine_loadcmds;
+
+  off_t all_image_infos_infile_offset = 0;
+
+  fseek(in, start_of_infile_load_cmds, SEEK_SET);
+  fseek(out, start_of_infile_load_cmds, SEEK_SET);
+  // Copy all the load commands from IN to OUT, updating any file offsets by
+  // SLIDE.
+  for (int cmd_num = 0; cmd_num < number_of_load_cmds; cmd_num++) {
+    off_t cmd_start = ftello(in);
+    uint32_t cmd, cmdsize;
+    fread(&cmd, sizeof(uint32_t), 1, in);
+    fread(&cmdsize, sizeof(uint32_t), 1, in);
+
+    fseeko(in, cmd_start, SEEK_SET);
+    off_t next_cmd = cmd_start + cmdsize;
+
+    switch (cmd) {
+    case LC_SEGMENT: {
+      struct segment_command segcmd;
+      fread(&segcmd, sizeof(segcmd), 1, in);
+      segcmd.fileoff += slide;
+      fwrite(&segcmd, cmdsize, 1, out);
+    } break;
+    case LC_SEGMENT_64: {
+      struct segment_command_64 segcmd;
+      fread(&segcmd, sizeof(segcmd), 1, in);
+      segcmd.fileoff += slide;
+      fwrite(&segcmd, cmdsize, 1, out);
+    } break;
+    case LC_NOTE: {
+      struct note_command notecmd;
+      fread(&notecmd, sizeof(notecmd), 1, in);
+      if ((strncmp(lc_note_name.c_str(), notecmd.data_owner, 16) == 0) &&
+          remove_dups) {
+        fseeko(in, next_cmd, SEEK_SET);
+        continue;
+      }
+      if (strncmp("all image infos", notecmd.data_owner, 16) == 0)
+        all_image_infos_infile_offset = notecmd.offset;
+      notecmd.offset += slide;
+      fwrite(&notecmd, cmdsize, 1, out);
+    } break;
+    default: {
+      vector<uint8_t> buf(cmdsize);
+      fread(buf.data(), cmdsize, 1, in);
+      fwrite(buf.data(), cmdsize, 1, out);
+    }
+    }
+    fseeko(in, next_cmd, SEEK_SET);
+  }
+
+  // Now add our additional LC_NOTE load command.
+  struct note_command note;
+  note.cmd = LC_NOTE;
+  note.cmdsize = sizeof(struct note_command);
+  memset(&note.data_owner, 0, 16);
+  // data_owner may not be nul terminated if all 16 characters
+  // are used, intentionally using strncpy here.
+  strncpy(note.data_owner, lc_note_name.c_str(), 16);
+  note.offset = infile_size + slide;
+  note.size = payload_data.size();
+  fwrite(&note, sizeof(struct note_command), 1, out);
+
+  fseeko(in, end_of_infine_loadcmds, SEEK_SET);
+  fseeko(out, end_of_outfile_loadcmds, SEEK_SET);
+
+  // Copy the rest of the corefile contents
+  vector<uint8_t> data_buf(1024 * 1024);
+  while (!feof(in)) {
+    size_t read_bytes = fread(data_buf.data(), 1, data_buf.size(), in);
+    if (read_bytes > 0) {
+      fwrite(data_buf.data(), read_bytes, 1, out);
+    } else {
+      break;
+    }
+  }
+
+  fwrite(payload_data.data(), payload_data.size(), 1, out);
+
+  // The "all image infos" LC_NOTE payload has file offsets hardcoded
+  // in it, unfortunately.  We've shifted the contents of the corefile
+  // and these offsets need to be updated in the ouput file.
+  // Re-copy them into the outfile with corrected file offsets.
+  off_t infile_image_entry_base = 0;
+  if (all_image_infos_infile_offset != 0) {
+    off_t all_image_infos_outfile_offset =
+        all_image_infos_infile_offset + slide;
+    fseeko(in, all_image_infos_infile_offset, SEEK_SET);
+    struct all_image_infos_header header;
+    fread(&header, sizeof(header), 1, in);
+    infile_image_entry_base = header.entries_fileoff;
+    header.entries_fileoff += slide;
+    fseeko(out, all_image_infos_outfile_offset, SEEK_SET);
+    fwrite(&header, sizeof(header), 1, out);
+
+    for (int i = 0; i < header.imgcount; i++) {
+      off_t infile_entries_fileoff = header.entries_fileoff - slide;
+      off_t outfile_entries_fileoff = header.entries_fileoff;
+
+      struct image_entry ent;
+      fseeko(in, infile_entries_fileoff + (header.entry_size * i), SEEK_SET);
+      fread(&ent, sizeof(ent), 1, in);
+      ent.filepath_offset += slide;
+      ent.seg_addrs_offset += slide;
+      fseeko(out, outfile_entries_fileoff + (header.entry_size * i), SEEK_SET);
+      fwrite(&ent, sizeof(ent), 1, out);
+    }
+  }
+}
+
+int main(int argc, char **argv) {
+  string infile, outfile, name;
+  vector<uint8_t> payload;
+  bool remove_dups;
+  parse_args(argc, argv, infile, outfile, name, payload, remove_dups);
+
+  FILE *in = fopen(infile.c_str(), "r");
+  if (!in) {
+    fprintf(stderr, "Unable to open %s for reading\n", infile.c_str());
+    exit(1);
+  }
+  FILE *out = fopen(outfile.c_str(), "w");
+  if (!out) {
+    fprintf(stderr, "Unable to open %s for reading\n", outfile.c_str());
+    exit(1);
+  }
+
+  copy_and_add_note(in, out, name, payload, remove_dups);
+
+  fclose(in);
+  fclose(out);
+}
diff --git a/lldb/test/API/macosx/lc-note/additional-registers/main.c b/lldb/test/API/macosx/lc-note/additional-registers/main.c
new file mode 100644
index 000000000000..70679942daf0
--- /dev/null
+++ b/lldb/test/API/macosx/lc-note/additional-registers/main.c
@@ -0,0 +1,11 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int main(int argc, char **argv) {
+
+  char *heap_buf = (char *)malloc(80);
+  strcpy(heap_buf, "this is a string on the heap");
+
+  return 0; // break here
+}
diff --git a/lldb/test/API/macosx/simulator/Makefile b/lldb/test/API/macosx/simulator/Makefile
index 421e85117613..ce3490d28ee8 100644
--- a/lldb/test/API/macosx/simulator/Makefile
+++ b/lldb/test/API/macosx/simulator/Makefile
@@ -1,3 +1,6 @@
-C_SOURCES := hello.c
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS -D__STDC_FORMAT_MACROS
+ENABLE_THREADS := YES
+CXX_SOURCES := hello.cpp
+MAKE_DSYM := NO
 
 include Makefile.rules
diff --git a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
index 74ba0ee6c83b..b17ee83ea04f 100644
--- a/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
+++ b/lldb/test/API/macosx/simulator/TestSimulatorPlatform.py
@@ -39,7 +39,9 @@ def check_debugserver(self, log, expected_platform, expected_version):
         if expected_version:
             self.assertEqual(aout_info["min_version_os_sdk"], expected_version)
 
-    def run_with(self, arch, os, vers, env, expected_load_command):
+    def run_with(
+        self, arch, os, vers, env, expected_load_command, expected_platform=None
+    ):
         env_list = [env] if env else []
         triple = "-".join([arch, "apple", os + vers] + env_list)
         sdk = lldbutil.get_xcode_sdk(os, env)
@@ -72,12 +74,47 @@ def run_with(self, arch, os, vers, env, expected_load_command):
         log = self.getBuildArtifact("packets.log")
         self.expect("log enable gdb-remote packets -f " + log)
         lldbutil.run_to_source_breakpoint(
-            self, "break here", lldb.SBFileSpec("hello.c")
+            self, "break here", lldb.SBFileSpec("hello.cpp")
         )
         triple_re = "-".join([arch, "apple", os + vers + ".*"] + env_list)
         self.expect("image list -b -t", patterns=[r"a\.out " + triple_re])
         self.check_debugserver(log, os + env, vers)
 
+        if expected_platform is not None:
+            # Verify the platform name.
+            self.expect(
+                "platform status",
+                patterns=[r"Platform: " + expected_platform + "-simulator"],
+            )
+
+            # Launch exe in simulator and verify that `platform process list` can find the process.
+            # This separate launch is needed because the command ignores processes which are being debugged.
+            device_udid = lldbutil.get_latest_apple_simulator(
+                expected_platform, self.trace
+            )
+            _, matched_strings = lldbutil.launch_exe_in_apple_simulator(
+                device_udid,
+                self.getBuildArtifact("a.out"),
+                exe_args=[],
+                stderr_lines_to_read=1,  # in hello.cpp, the pid is printed first
+                stderr_patterns=[r"PID: (.*)"],
+                log=self.trace,
+            )
+
+            # Make sure we found the PID.
+            self.assertIsNotNone(matched_strings[0])
+            pid = int(matched_strings[0])
+
+            # Verify that processes on the platform can be listed.
+            self.expect(
+                "platform process list",
+                patterns=[
+                    r"\d+ matching processes were found on \"%s-simulator\""
+                    % expected_platform,
+                    r"%d .+ a.out" % pid,
+                ],
+            )
+
     @skipIfAsan
     @skipUnlessDarwin
     @skipIfDarwinEmbedded
@@ -90,6 +127,7 @@ def test_ios(self):
             vers="",
             env="simulator",
             expected_load_command="LC_BUILD_VERSION",
+            expected_platform="ios",
         )
 
     @skipIfAsan
diff --git a/lldb/test/API/macosx/simulator/hello.c b/lldb/test/API/macosx/simulator/hello.c
deleted file mode 100644
index 9010d0e29515..000000000000
--- a/lldb/test/API/macosx/simulator/hello.c
+++ /dev/null
@@ -1,5 +0,0 @@
-void puts(char *);
-int main(int argc, char **argv) {
-  puts("break here\n");
-  return 0;
-}
diff --git a/lldb/test/API/macosx/simulator/hello.cpp b/lldb/test/API/macosx/simulator/hello.cpp
new file mode 100644
index 000000000000..e202125da092
--- /dev/null
+++ b/lldb/test/API/macosx/simulator/hello.cpp
@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <thread>
+#include <unistd.h>
+
+static void print_pid() { fprintf(stderr, "PID: %d\n", getpid()); }
+
+static void sleep() { std::this_thread::sleep_for(std::chrono::seconds(10)); }
+
+int main(int argc, char **argv) {
+  print_pid();
+  puts("break here\n");
+  sleep();
+  return 0;
+}
diff --git a/lldb/test/API/python_api/debugger/TestDebuggerAPI.py b/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
index 646ccce36530..43f45f330ee2 100644
--- a/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
+++ b/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
@@ -286,3 +286,11 @@ def remove_bar(dbg_id):
             ('remove bar ret', False), # remove_bar should fail, because it's already invoked and removed
             ('foo called', original_dbg_id), # foo should be called
         ])
+
+    def test_version(self):
+        instance_str = self.dbg.GetVersionString()
+        class_str = lldb.SBDebugger.GetVersionString()
+        property_str = lldb.SBDebugger.version
+
+        self.assertEqual(instance_str, class_str)
+        self.assertEqual(class_str, property_str)
diff --git a/lldb/test/API/riscv/step/Makefile b/lldb/test/API/riscv/step/Makefile
new file mode 100644
index 000000000000..10495940055b
--- /dev/null
+++ b/lldb/test/API/riscv/step/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/riscv/step/TestSoftwareStep.py b/lldb/test/API/riscv/step/TestSoftwareStep.py
new file mode 100644
index 000000000000..279c4c1b797e
--- /dev/null
+++ b/lldb/test/API/riscv/step/TestSoftwareStep.py
@@ -0,0 +1,79 @@
+"""
+Test software step-inst, also known as instruction level single step, in risc-v atomic sequence.
+For more information about atomic sequences, see the RISC-V Unprivileged ISA specification.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestSoftwareStep(TestBase):
+    def do_sequence_test(self, filename, bkpt_name):
+        source_file = filename + ".c"
+        exe_file = filename + ".x"
+
+        self.build(dictionary={"C_SOURCES": source_file, "EXE": exe_file})
+        (target, process, cur_thread, bkpt) = lldbutil.run_to_name_breakpoint(
+            self, bkpt_name, exe_name=exe_file
+        )
+        entry_pc = cur_thread.GetFrameAtIndex(0).GetPC()
+
+        self.runCmd("thread step-inst")
+        self.expect(
+            "thread list",
+            substrs=["stopped", "stop reason = instruction step into"],
+        )
+
+        pc = cur_thread.GetFrameAtIndex(0).GetPC()
+
+        return pc - entry_pc
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_cas(self):
+        """
+        This test verifies LLDB instruction step handling of a proper lr/sc pair.
+        """
+        difference = self.do_sequence_test("main", "cas")
+        self.assertEqual(difference, 0x1A)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_branch_cas(self):
+        """
+        LLDB cannot predict the actual state of registers within a critical section (i.e., inside an atomic
+        sequence). Therefore, it should identify all forward branches inside the atomic sequence and set
+        breakpoints at every jump address that lies beyond the end of the sequence (after the sc instruction).
+        This ensures that if any such branch is taken, execution will pause at its target address.
+
+        This test includes an lr/sc sequence containing an active forward branch with a jump address located
+        after the end of the atomic section. LLDB should correctly stop at this branch's target address. The
+        test is nearly identical to the previous one, except for the branch condition, which is inverted and
+        will result in a taken jump.
+        """
+        difference = self.do_sequence_test("branch", "branch_cas")
+        self.assertEqual(difference, 0x1A)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_incomplete_sequence_without_lr(self):
+        """
+        This test verifies the behavior of a standalone sc instruction without a preceding lr. Since the sc
+        lacks the required lr pairing, LLDB should treat it as a non-atomic store rather than part of an
+        atomic sequence.
+        """
+        difference = self.do_sequence_test(
+            "incomplete_sequence_without_lr", "incomplete_cas"
+        )
+        self.assertEqual(difference, 0x4)
+
+    @skipIf(archs=no_match("^rv.*"))
+    def test_incomplete_sequence_without_sc(self):
+        """
+        This test checks the behavior of a standalone lr instruction without a subsequent sc. Since the lr
+        lacks its required sc counterpart, LLDB should treat it as a non-atomic load rather than part of an
+        atomic sequence.
+        """
+        difference = self.do_sequence_test(
+            "incomplete_sequence_without_sc", "incomplete_cas"
+        )
+        self.assertEqual(difference, 0x4)
diff --git a/lldb/test/API/riscv/step/branch.c b/lldb/test/API/riscv/step/branch.c
new file mode 100644
index 000000000000..79bf0ca005db
--- /dev/null
+++ b/lldb/test/API/riscv/step/branch.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) branch_cas(int *a, int *b) {
+  // Stop at the first instruction. The atomic sequence contains active forward
+  // branch (bne a5, a1, 2f). After step instruction lldb should stop at the
+  // branch's target address (ret instruction).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "bne a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "beqz a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  branch_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c b/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c
new file mode 100644
index 000000000000..eda313c6d69c
--- /dev/null
+++ b/lldb/test/API/riscv/step/incomplete_sequence_without_lr.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) incomplete_cas(int *a, int *b) {
+  // Stop at the first instruction (an sc without a corresponding lr), then make
+  // a step instruction and ensure that execution stops at the next instruction
+  // (and).
+  asm volatile("1:\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "bnez a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  incomplete_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c b/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c
new file mode 100644
index 000000000000..952dc7d2804e
--- /dev/null
+++ b/lldb/test/API/riscv/step/incomplete_sequence_without_sc.c
@@ -0,0 +1,21 @@
+void __attribute__((naked)) incomplete_cas(int *a, int *b) {
+  // Stop at the first instruction (an lr without a corresponding sc), then make
+  // a step instruction and ensure that execution stops at the next instruction
+  // (and).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "bnez a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  incomplete_cas(&a, &b);
+}
diff --git a/lldb/test/API/riscv/step/main.c b/lldb/test/API/riscv/step/main.c
new file mode 100644
index 000000000000..35e8aee2cae4
--- /dev/null
+++ b/lldb/test/API/riscv/step/main.c
@@ -0,0 +1,22 @@
+void __attribute__((naked)) cas(int *a, int *b) {
+  // This atomic sequence implements a copy-and-swap function. This test should
+  // at the first instruction, and after step instruction, we should stop at the
+  // end of the sequence (on the ret instruction).
+  asm volatile("1:\n\t"
+               "lr.w a2, (a0)\n\t"
+               "and a5, a2, a4\n\t"
+               "beq a5, a1, 2f\n\t"
+               "xor a5, a2, a0\n\t"
+               "and a5, a5, a4\n\t"
+               "xor a5, a2, a5\n\t"
+               "sc.w a5, a1, (a3)\n\t"
+               "beqz a5, 1b\n\t"
+               "2:\n\t"
+               "ret\n\t");
+}
+
+int main() {
+  int a = 4;
+  int b = 2;
+  cas(&a, &b);
+}
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py b/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
index 8bccc2bcf415..674bfe4199b4 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
@@ -67,19 +67,19 @@ def test_break_on_invalid_source_reference(self):
             "Invalid sourceReference.",
         )
 
-        # Verify that setting a breakpoint on a source reference without a symbol also fails
+        # Verify that setting a breakpoint on a source reference that is not created fails
         response = self.dap_server.request_setBreakpoints(
-            Source(source_reference=0), [1]
+            Source(source_reference=200), [1]
         )
         self.assertIsNotNone(response)
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(len(breakpoints), 1)
-        breakpoint = breakpoints[0]
+        break_point = breakpoints[0]
         self.assertFalse(
-            breakpoint["verified"], "Expected breakpoint to not be verified"
+            break_point["verified"], "Expected breakpoint to not be verified"
         )
-        self.assertIn("message", breakpoint, "Expected message to be present")
+        self.assertIn("message", break_point, "Expected message to be present")
         self.assertEqual(
-            breakpoint["message"],
-            "Breakpoints in assembly without a valid symbol are not supported yet.",
+            break_point["message"],
+            "Invalid sourceReference.",
         )
diff --git a/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
index 7b4d1adbb207..811843dfdf7a 100644
--- a/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
+++ b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
@@ -144,9 +144,9 @@ def test_exit_status_message_sigterm(self):
         )
 
         # Verify the exit status message is printed.
-        self.assertIn(
-            "exited with status = -1 (0xffffffff) debugserver died with signal SIGTERM",
+        self.assertRegex(
             console_output,
+            ".*exited with status = -1 .* died with signal SIGTERM.*",
             "Exit status does not contain message 'exited with status'",
         )
 
diff --git a/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
index 3ba7deb285de..35810feb4836 100644
--- a/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
+++ b/lldb/test/API/tools/lldb-dap/restart/TestDAP_restart_runInTerminal.py
@@ -5,7 +5,7 @@
 from typing import Dict, Any, List
 
 import lldbdap_testcase
-from lldbsuite.test.decorators import skipIfWindows, skipIf, skipIfBuildType
+from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import line_number
 
 
@@ -32,6 +32,7 @@ def verify_stopped_on_entry(self, stopped_events: List[Dict[str, Any]]):
 
         self.assertEqual(seen_stopped_event, 1, "expect only one stopped entry event.")
 
+    @skipIfAsan
     @skipIfWindows
     @skipIf(oslist=["linux"], archs=["arm$"])  # Always times out on buildbot
     def test_basic_functionality(self):
@@ -80,6 +81,7 @@ def test_basic_functionality(self):
         )
         self.continue_to_exit()
 
+    @skipIfAsan
     @skipIfWindows
     @skipIf(oslist=["linux"], archs=["arm$"])  # Always times out on buildbot
     def test_stopOnEntry(self):
diff --git a/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py b/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
index 3d07cd8b20e2..af8b6b140da4 100644
--- a/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
+++ b/lldb/test/API/tools/lldb-dap/runInTerminal/TestDAP_runInTerminal.py
@@ -2,7 +2,7 @@
 Test lldb-dap runInTerminal reverse request
 """
 
-from lldbsuite.test.decorators import skipIfBuildType, skipIfWindows, skipIf, no_match
+from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import line_number
 import lldbdap_testcase
 import os
@@ -26,6 +26,7 @@ def read_error_message(fifo_file):
         with open(fifo_file, "r") as file:
             return file.readline()
 
+    @skipIfAsan
     @skipIfWindows
     @skipIf(oslist=["linux"], archs=no_match(["x86_64"]))
     def test_runInTerminal(self):
@@ -73,6 +74,8 @@ def test_runInTerminal(self):
 
         self.continue_to_exit()
 
+    @skipIfAsan
+    @skipIfWindows
     @skipIf(oslist=["linux"], archs=no_match(["x86_64"]))
     def test_runInTerminalWithObjectEnv(self):
         """
diff --git a/lldb/test/API/tools/lldb-dap/runInTerminal/main.c b/lldb/test/API/tools/lldb-dap/runInTerminal/main.c
index 0cc25d374d08..40aa484b5381 100644
--- a/lldb/test/API/tools/lldb-dap/runInTerminal/main.c
+++ b/lldb/test/API/tools/lldb-dap/runInTerminal/main.c
@@ -1,6 +1,10 @@
 #include <stdio.h>
 #include <stdlib.h>
+#ifdef _WIN32
+#include <stdlib.h>
+#else
 #include <unistd.h>
+#endif
 
 int main(int argc, char *argv[]) {
   const char *foo = getenv("FOO");
diff --git a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
index 592a4cfb0a88..e01320c25b15 100644
--- a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
+++ b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
@@ -101,7 +101,9 @@ def test_server_interrupt(self):
         # Interrupt the server which should disconnect all clients.
         process.send_signal(signal.SIGINT)
 
-        self.dap_server.wait_for_terminated()
+        # Wait for both events since they can happen in any order.
+        self.dap_server.wait_for_event(["terminated", "exited"])
+        self.dap_server.wait_for_event(["terminated", "exited"])
         self.assertIsNotNone(
             self.dap_server.exit_status,
             "Process exited before interrupting lldb-dap server",
diff --git a/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
index 42a39e3c8c08..5c055f679aed 100644
--- a/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
+++ b/lldb/test/API/tools/lldb-dap/step/TestDAP_step.py
@@ -83,3 +83,41 @@ def test_step(self):
 
                     # only step one thread that is at the breakpoint and stop
                     break
+
+    def test_step_over_inlined_function(self):
+        """
+        Test stepping over when the program counter is in another file.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+        source = "main.cpp"
+        breakpoint_lines = [line_number(source, "// breakpoint 2")]
+        step_over_pos = line_number(source, "// position_after_step_over")
+        breakpoint_ids = self.set_source_breakpoints(source, breakpoint_lines)
+        self.assertEqual(
+            len(breakpoint_ids),
+            len(breakpoint_lines),
+            "expect correct number of breakpoints.",
+        )
+        self.continue_to_breakpoints(breakpoint_ids)
+
+        thread_id = self.dap_server.get_thread_id()
+        self.stepOver(thread_id)
+        levels = 1
+        frames = self.get_stackFrames(thread_id, 0, levels)
+        self.assertEqual(len(frames), levels, "expect current number of frame levels.")
+        top_frame = frames[0]
+        self.assertEqual(
+            top_frame["source"]["name"], source, "expect we are in the same file."
+        )
+        self.assertTrue(
+            top_frame["source"]["path"].endswith(source),
+            f"expect path ending with '{source}'.",
+        )
+        self.assertEqual(
+            top_frame["line"],
+            step_over_pos,
+            f"expect step_over on line {step_over_pos}",
+        )
+
+        self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-dap/step/main.cpp b/lldb/test/API/tools/lldb-dap/step/main.cpp
index 8905beb5e7ef..7320e83154f5 100644
--- a/lldb/test/API/tools/lldb-dap/step/main.cpp
+++ b/lldb/test/API/tools/lldb-dap/step/main.cpp
@@ -1,3 +1,5 @@
+#include "other.h"
+
 int function(int x) {
   if ((x % 2) == 0)
     return function(x - 1) + x; // breakpoint 1
@@ -5,4 +7,14 @@ int function(int x) {
     return x;
 }
 
-int main(int argc, char const *argv[]) { return function(2); }
+int function2() {
+  int volatile value = 3; // breakpoint 2
+  inlined_fn();           // position_after_step_over
+
+  return value;
+}
+
+int main(int argc, char const *argv[]) {
+  int func_result = function2();
+  return function(2) - func_result; // returns 0
+}
diff --git a/lldb/test/API/tools/lldb-dap/step/other.h b/lldb/test/API/tools/lldb-dap/step/other.h
new file mode 100644
index 000000000000..c71cc373fbdf
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/step/other.h
@@ -0,0 +1,7 @@
+#ifndef OTHER_H
+#define OTHER_H
+
+__attribute__((noinline)) void not_inlined_fn() {};
+
+__attribute__((always_inline)) inline void inlined_fn() { not_inlined_fn(); }
+#endif // OTHER_H
\ No newline at end of file
diff --git a/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py b/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
index ed47f94e9492..4fee27823f1f 100644
--- a/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
+++ b/lldb/test/API/tools/lldb-server/TestAppleSimulatorOSType.py
@@ -14,40 +14,12 @@ class TestAppleSimulatorOSType(gdbremote_testcase.GdbRemoteTestCaseBase):
     READ_LINES = 10
 
     def check_simulator_ostype(self, sdk, platform_name, arch=platform.machine()):
-        cmd = ["xcrun", "simctl", "list", "-j", "devices"]
-        cmd_str = " ".join(cmd)
-        self.trace(cmd_str)
-        sim_devices_str = subprocess.check_output(cmd).decode("utf-8")
+        # Get simulator
+        deviceUDID = None
         try:
-            sim_devices = json.loads(sim_devices_str)["devices"]
+            deviceUDID = lldbutil.get_latest_apple_simulator(platform_name, self.trace)
         except json.decoder.JSONDecodeError:
-            self.fail(
-                "Could not parse '{}' output. Authorization denied?".format(cmd_str)
-            )
-        # Find an available simulator for the requested platform
-        deviceUDID = None
-        deviceRuntime = None
-        for simulator in sim_devices:
-            if isinstance(simulator, dict):
-                runtime = simulator["name"]
-                devices = simulator["devices"]
-            else:
-                runtime = simulator
-                devices = sim_devices[simulator]
-            if not platform_name in runtime.lower():
-                continue
-            for device in devices:
-                if "availability" in device and device["availability"] != "(available)":
-                    continue
-                if "isAvailable" in device and not device["isAvailable"]:
-                    continue
-                if deviceRuntime and runtime < deviceRuntime:
-                    continue
-                deviceUDID = device["udid"]
-                deviceRuntime = runtime
-                # Stop searching in this runtime
-                break
-
+            self.fail("Could not parse output. Authorization denied?")
         if not deviceUDID:
             self.skipTest(
                 "Could not find a simulator for {} ({})".format(platform_name, arch)
@@ -78,34 +50,20 @@ def check_simulator_ostype(self, sdk, platform_name, arch=platform.machine()):
             },
             compiler=clang,
         )
-        exe_path = os.path.realpath(self.getBuildArtifact(exe_name))
-        cmd = [
-            "xcrun",
-            "simctl",
-            "spawn",
-            "-s",
+
+        # Launch the executable in the simulator
+        exe_path, matched_groups = lldbutil.launch_exe_in_apple_simulator(
             deviceUDID,
-            exe_path,
-            "print-pid",
-            "sleep:10",
-        ]
-        self.trace(" ".join(cmd))
-        sim_launcher = subprocess.Popen(cmd, stderr=subprocess.PIPE)
-        # Get the PID from the process output
-        pid = None
-
-        # Read the first READ_LINES to try to find the PID.
-        for _ in range(0, self.READ_LINES):
-            stderr = sim_launcher.stderr.readline().decode("utf-8")
-            if not stderr:
-                continue
-            match = re.match(r"PID: (.*)", stderr)
-            if match:
-                pid = int(match.group(1))
-                break
+            self.getBuildArtifact(exe_name),
+            ["print-pid", "sleep:10"],
+            self.READ_LINES,
+            [r"PID: (.*)"],
+            self.trace,
+        )
 
         # Make sure we found the PID.
-        self.assertIsNotNone(pid)
+        self.assertIsNotNone(matched_groups[0])
+        pid = int(matched_groups[0])
 
         # Launch debug monitor attaching to the simulated process
         server = self.connect_to_debug_monitor(attach_pid=pid)
diff --git a/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h b/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h
index fecc69687cd7..a15dd9d2a942 100644
--- a/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h
+++ b/lldb/test/Shell/Scripts/Inputs/Main/SBAddress.h
@@ -6,6 +6,7 @@
 // e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
 #include "lldb/API/SBDefines.h"
 #include "lldb/API/SBModule.h"
+#include "lldb/lldb-types.h"
 
 // Any include guards specified at the command line must be removed.
 #ifndef SWIG
diff --git a/lldb/test/Shell/Scripts/TestFrameworkFixScript.test b/lldb/test/Shell/Scripts/TestFrameworkFixScript.test
index e90c3bdfc5ad..5c48b796efda 100644
--- a/lldb/test/Shell/Scripts/TestFrameworkFixScript.test
+++ b/lldb/test/Shell/Scripts/TestFrameworkFixScript.test
@@ -9,3 +9,4 @@ RUN: cat %t/Outputs/SBAddress.h | FileCheck %s
 # e.g. #include "lldb/API/SBDefines.h" -> #include <LLDB/SBDefines.h>
 CHECK: #include <LLDB/SBDefines.h>
 CHECK: #include <LLDB/SBModule.h>
+CHECK: #include <LLDB/lldb-types.h>
diff --git a/lldb/test/Shell/Scripts/TestVersionFixScript.test b/lldb/test/Shell/Scripts/TestVersionFixScript.test
index 78cc98726307..9467d1655b05 100644
--- a/lldb/test/Shell/Scripts/TestVersionFixScript.test
+++ b/lldb/test/Shell/Scripts/TestVersionFixScript.test
@@ -1,6 +1,6 @@
 # Create a temp dir for output and run the version fix script on the truncated version of lldb-defines.h in the inputs dir.
 RUN: mkdir -p %t/Outputs
-RUN: %python %p/../../../scripts/version-header-fix.py %p/Inputs/lldb-defines.h %t/Outputs/lldb-defines.h 21 0 12
+RUN: %python %p/../../../scripts/version-header-fix.py --input_path %p/Inputs/lldb-defines.h --output_path %t/Outputs/lldb-defines.h --major 21 --minor 0 --patch 12
 
 # Check the output
 RUN: cat %t/Outputs/lldb-defines.h | FileCheck %s
diff --git a/lldb/tools/lldb-dap/Breakpoint.cpp b/lldb/tools/lldb-dap/Breakpoint.cpp
index ef5646c4c3d1..b4e593eb83d2 100644
--- a/lldb/tools/lldb-dap/Breakpoint.cpp
+++ b/lldb/tools/lldb-dap/Breakpoint.cpp
@@ -64,8 +64,8 @@ protocol::Breakpoint Breakpoint::ToProtocolBreakpoint() {
         "0x" + llvm::utohexstr(bp_addr.GetLoadAddress(m_bp.GetTarget()));
     breakpoint.instructionReference = formatted_addr;
 
-    auto source = CreateSource(bp_addr, m_dap.target);
-    if (!IsAssemblySource(source)) {
+    std::optional<protocol::Source> source = m_dap.ResolveSource(bp_addr);
+    if (source && !IsAssemblySource(*source)) {
       auto line_entry = bp_addr.GetLineEntry();
       const auto line = line_entry.GetLine();
       if (line != LLDB_INVALID_LINE_NUMBER)
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index c171b55951cb..a26beed491e9 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -497,6 +497,27 @@ DAP::SendFormattedOutput(OutputType o, const char *format, ...) {
       o, llvm::StringRef(buffer, std::min<int>(actual_length, sizeof(buffer))));
 }
 
+int32_t DAP::CreateSourceReference(lldb::addr_t address) {
+  std::lock_guard<std::mutex> guard(m_source_references_mutex);
+  auto iter = llvm::find(m_source_references, address);
+  if (iter != m_source_references.end())
+    return std::distance(m_source_references.begin(), iter) + 1;
+
+  m_source_references.emplace_back(address);
+  return static_cast<int32_t>(m_source_references.size());
+}
+
+std::optional<lldb::addr_t> DAP::GetSourceReferenceAddress(int32_t reference) {
+  std::lock_guard<std::mutex> guard(m_source_references_mutex);
+  if (reference <= LLDB_DAP_INVALID_SRC_REF)
+    return std::nullopt;
+
+  if (static_cast<size_t>(reference) > m_source_references.size())
+    return std::nullopt;
+
+  return m_source_references[reference - 1];
+}
+
 ExceptionBreakpoint *DAP::GetExceptionBPFromStopReason(lldb::SBThread &thread) {
   const auto num = thread.GetStopReasonDataCount();
   // Check to see if have hit an exception breakpoint and change the
@@ -602,6 +623,66 @@ ReplMode DAP::DetectReplMode(lldb::SBFrame frame, std::string &expression,
   llvm_unreachable("enum cases exhausted.");
 }
 
+std::optional<protocol::Source> DAP::ResolveSource(const lldb::SBFrame &frame) {
+  if (!frame.IsValid())
+    return std::nullopt;
+
+  const lldb::SBAddress frame_pc = frame.GetPCAddress();
+  if (DisplayAssemblySource(debugger, frame_pc))
+    return ResolveAssemblySource(frame_pc);
+
+  return CreateSource(frame.GetLineEntry().GetFileSpec());
+}
+
+std::optional<protocol::Source> DAP::ResolveSource(lldb::SBAddress address) {
+  if (DisplayAssemblySource(debugger, address))
+    return ResolveAssemblySource(address);
+
+  lldb::SBLineEntry line_entry = GetLineEntryForAddress(target, address);
+  if (!line_entry.IsValid())
+    return std::nullopt;
+
+  return CreateSource(line_entry.GetFileSpec());
+}
+
+std::optional<protocol::Source>
+DAP::ResolveAssemblySource(lldb::SBAddress address) {
+  lldb::SBSymbol symbol = address.GetSymbol();
+  lldb::addr_t load_addr = LLDB_INVALID_ADDRESS;
+  std::string name;
+  if (symbol.IsValid()) {
+    load_addr = symbol.GetStartAddress().GetLoadAddress(target);
+    name = symbol.GetName();
+  } else {
+    load_addr = address.GetLoadAddress(target);
+    name = GetLoadAddressString(load_addr);
+  }
+
+  if (load_addr == LLDB_INVALID_ADDRESS)
+    return std::nullopt;
+
+  protocol::Source source;
+  source.sourceReference = CreateSourceReference(load_addr);
+  lldb::SBModule module = address.GetModule();
+  if (module.IsValid()) {
+    lldb::SBFileSpec file_spec = module.GetFileSpec();
+    if (file_spec.IsValid()) {
+      std::string path = GetSBFileSpecPath(file_spec);
+      if (!path.empty())
+        source.path = path + '`' + name;
+    }
+  }
+
+  source.name = std::move(name);
+
+  // Mark the source as deemphasized since users will only be able to view
+  // assembly for these frames.
+  source.presentationHint =
+      protocol::Source::eSourcePresentationHintDeemphasize;
+
+  return source;
+}
+
 bool DAP::RunLLDBCommands(llvm::StringRef prefix,
                           llvm::ArrayRef<std::string> commands) {
   bool required_command_failed = false;
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 5ca5822f9bce..af4aabaafaae 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -219,7 +219,9 @@ struct DAP {
   void __attribute__((format(printf, 3, 4)))
   SendFormattedOutput(OutputType o, const char *format, ...);
 
-  static int64_t GetNextSourceReference();
+  int32_t CreateSourceReference(lldb::addr_t address);
+
+  std::optional<lldb::addr_t> GetSourceReferenceAddress(int32_t reference);
 
   ExceptionBreakpoint *GetExceptionBPFromStopReason(lldb::SBThread &thread);
 
@@ -252,6 +254,40 @@ struct DAP {
   ReplMode DetectReplMode(lldb::SBFrame frame, std::string &expression,
                           bool partial_expression);
 
+  /// Create a `protocol::Source` object as described in the debug adapter
+  /// definition.
+  ///
+  /// \param[in] frame
+  ///     The frame to use when populating the "Source" object.
+  ///
+  /// \return
+  ///     A `protocol::Source` object that follows the formal JSON
+  ///     definition outlined by Microsoft.
+  std::optional<protocol::Source> ResolveSource(const lldb::SBFrame &frame);
+
+  /// Create a "Source" JSON object as described in the debug adapter
+  /// definition.
+  ///
+  /// \param[in] address
+  ///     The address to use when populating out the "Source" object.
+  ///
+  /// \return
+  ///     An optional "Source" JSON object that follows the formal JSON
+  ///     definition outlined by Microsoft.
+  std::optional<protocol::Source> ResolveSource(lldb::SBAddress address);
+
+  /// Create a "Source" JSON object as described in the debug adapter
+  /// definition.
+  ///
+  /// \param[in] address
+  ///     The address to use when populating out the "Source" object.
+  ///
+  /// \return
+  ///     An optional "Source" JSON object that follows the formal JSON
+  ///     definition outlined by Microsoft.
+  std::optional<protocol::Source>
+  ResolveAssemblySource(lldb::SBAddress address);
+
   /// \return
   ///   \b false if a fatal error was found while executing these commands,
   ///   according to the rules of \a LLDBUtils::RunLLDBCommands.
@@ -406,6 +442,10 @@ struct DAP {
   std::thread progress_event_thread;
   /// @}
 
+  /// List of addresses mapped by sourceReference.
+  std::vector<lldb::addr_t> m_source_references;
+  std::mutex m_source_references_mutex;
+
   /// Queue for all incoming messages.
   std::deque<protocol::Message> m_queue;
   std::mutex m_queue_mutex;
diff --git a/lldb/tools/lldb-dap/DAPError.h b/lldb/tools/lldb-dap/DAPError.h
index e18614fe7193..26b1daae5934 100644
--- a/lldb/tools/lldb-dap/DAPError.h
+++ b/lldb/tools/lldb-dap/DAPError.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLDB_TOOLS_LLDB_DAP_DAPERROR_H
+#define LLDB_TOOLS_LLDB_DAP_DAPERROR_H
+
 #include "llvm/Support/Error.h"
 #include <optional>
 #include <string>
@@ -50,3 +53,5 @@ class NotStoppedError : public llvm::ErrorInfo<NotStoppedError> {
 };
 
 } // namespace lldb_dap
+
+#endif // LLDB_TOOLS_LLDB_DAP_DAPERROR_H
diff --git a/lldb/tools/lldb-dap/Handler/DisassembleRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/DisassembleRequestHandler.cpp
index 85214b84b5c9..f66c87fa9893 100644
--- a/lldb/tools/lldb-dap/Handler/DisassembleRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/DisassembleRequestHandler.cpp
@@ -85,7 +85,8 @@ static lldb::SBAddress GetDisassembleStartAddress(lldb::SBTarget target,
 }
 
 static DisassembledInstruction ConvertSBInstructionToDisassembledInstruction(
-    lldb::SBTarget &target, lldb::SBInstruction &inst, bool resolve_symbols) {
+    DAP &dap, lldb::SBInstruction &inst, bool resolve_symbols) {
+  lldb::SBTarget target = dap.target;
   if (!inst.IsValid())
     return GetInvalidInstruction();
 
@@ -138,14 +139,14 @@ static DisassembledInstruction ConvertSBInstructionToDisassembledInstruction(
     si << " ; " << c;
   }
 
-  protocol::Source source = CreateSource(addr, target);
+  std::optional<protocol::Source> source = dap.ResolveSource(addr);
   lldb::SBLineEntry line_entry = GetLineEntryForAddress(target, addr);
 
   // If the line number is 0 then the entry represents a compiler generated
   // location.
-  if (!IsAssemblySource(source) && line_entry.GetStartAddress() == addr &&
-      line_entry.IsValid() && line_entry.GetFileSpec().IsValid() &&
-      line_entry.GetLine() != 0) {
+  if (source && !IsAssemblySource(*source) &&
+      line_entry.GetStartAddress() == addr && line_entry.IsValid() &&
+      line_entry.GetFileSpec().IsValid() && line_entry.GetLine() != 0) {
 
     disassembled_inst.location = std::move(source);
     const auto line = line_entry.GetLine();
@@ -221,7 +222,7 @@ DisassembleRequestHandler::Run(const DisassembleArguments &args) const {
       original_address_index = i;
 
     instructions.push_back(ConvertSBInstructionToDisassembledInstruction(
-        dap.target, inst, resolve_symbols));
+        dap, inst, resolve_symbols));
   }
 
   // Check if we miss instructions at the beginning.
diff --git a/lldb/tools/lldb-dap/Handler/LocationsRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/LocationsRequestHandler.cpp
index 804bdcd622a2..cf9b5a3dbd06 100644
--- a/lldb/tools/lldb-dap/Handler/LocationsRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/LocationsRequestHandler.cpp
@@ -137,7 +137,16 @@ void LocationsRequestHandler::operator()(
       return;
     }
 
-    body.try_emplace("source", CreateSource(line_entry.GetFileSpec()));
+    const std::optional<protocol::Source> source =
+        CreateSource(line_entry.GetFileSpec());
+    if (!source) {
+      response["success"] = false;
+      response["message"] = "Failed to resolve file path for location";
+      dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
+
+    body.try_emplace("source", *source);
     if (int line = line_entry.GetLine())
       body.try_emplace("line", line);
     if (int column = line_entry.GetColumn())
@@ -152,7 +161,16 @@ void LocationsRequestHandler::operator()(
       return;
     }
 
-    body.try_emplace("source", CreateSource(decl.GetFileSpec()));
+    const std::optional<protocol::Source> source =
+        CreateSource(decl.GetFileSpec());
+    if (!source) {
+      response["success"] = false;
+      response["message"] = "Failed to resolve file path for location";
+      dap.SendJSON(llvm::json::Value(std::move(response)));
+      return;
+    }
+
+    body.try_emplace("source", *source);
     if (int line = decl.GetLine())
       body.try_emplace("line", line);
     if (int column = decl.GetColumn())
diff --git a/lldb/tools/lldb-dap/Handler/SourceRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/SourceRequestHandler.cpp
index 353d3365564f..755ad206abe2 100644
--- a/lldb/tools/lldb-dap/Handler/SourceRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/SourceRequestHandler.cpp
@@ -29,34 +29,42 @@ namespace lldb_dap {
 /// the source code for a given source reference.
 llvm::Expected<protocol::SourceResponseBody>
 SourceRequestHandler::Run(const protocol::SourceArguments &args) const {
-  const auto source =
+
+  uint32_t source_ref =
       args.source->sourceReference.value_or(args.sourceReference);
+  const std::optional<lldb::addr_t> source_addr_opt =
+      dap.GetSourceReferenceAddress(source_ref);
 
-  if (!source)
+  if (!source_addr_opt)
     return llvm::make_error<DAPError>(
-        "invalid arguments, expected source.sourceReference to be set");
+        llvm::formatv("Unknown source reference {}", source_ref));
 
-  lldb::SBAddress address(source, dap.target);
+  lldb::SBAddress address(*source_addr_opt, dap.target);
   if (!address.IsValid())
     return llvm::make_error<DAPError>("source not found");
 
   lldb::SBSymbol symbol = address.GetSymbol();
-
-  lldb::SBStream stream;
-  lldb::SBExecutionContext exe_ctx(dap.target);
+  lldb::SBInstructionList insts;
 
   if (symbol.IsValid()) {
-    lldb::SBInstructionList insts = symbol.GetInstructions(dap.target);
-    insts.GetDescription(stream, exe_ctx);
+    insts = symbol.GetInstructions(dap.target);
   } else {
     // No valid symbol, just return the disassembly.
-    lldb::SBInstructionList insts = dap.target.ReadInstructions(
+    insts = dap.target.ReadInstructions(
         address, dap.k_number_of_assembly_lines_for_nodebug);
-    insts.GetDescription(stream, exe_ctx);
   }
 
+  if (!insts || insts.GetSize() == 0)
+    return llvm::make_error<DAPError>(
+        llvm::formatv("no instruction source for address {}",
+                      address.GetLoadAddress(dap.target)));
+
+  lldb::SBStream stream;
+  lldb::SBExecutionContext exe_ctx(dap.target);
+  insts.GetDescription(stream, exe_ctx);
   return protocol::SourceResponseBody{/*content=*/stream.GetData(),
-                                      /*mimeType=*/"text/x-lldb.disassembly"};
+                                      /*mimeType=*/
+                                      "text/x-lldb.disassembly"};
 }
 
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/StackTraceRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/StackTraceRequestHandler.cpp
index 4ea4cd1e517d..77ef952a1e34 100644
--- a/lldb/tools/lldb-dap/Handler/StackTraceRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/StackTraceRequestHandler.cpp
@@ -69,7 +69,7 @@ static bool FillStackFrames(DAP &dap, lldb::SBThread &thread,
       break;
     }
 
-    stack_frames.emplace_back(CreateStackFrame(frame, frame_format));
+    stack_frames.emplace_back(CreateStackFrame(dap, frame, frame_format));
   }
 
   if (include_all && reached_end_of_stack) {
diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp
index cf7db41559b8..e72d93ee3457 100644
--- a/lldb/tools/lldb-dap/JSONUtils.cpp
+++ b/lldb/tools/lldb-dap/JSONUtils.cpp
@@ -543,7 +543,7 @@ llvm::json::Object CreateEventObject(const llvm::StringRef event_name) {
 //   },
 //   "required": [ "id", "name", "line", "column" ]
 // }
-llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
+llvm::json::Value CreateStackFrame(DAP &dap, lldb::SBFrame &frame,
                                    lldb::SBFormat &format) {
   llvm::json::Object object;
   int64_t frame_id = MakeDAPFrameID(frame);
@@ -572,9 +572,9 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
 
   EmplaceSafeString(object, "name", frame_name);
 
-  auto target = frame.GetThread().GetProcess().GetTarget();
-  auto source = CreateSource(frame.GetPCAddress(), target);
-  if (!IsAssemblySource(source)) {
+  std::optional<protocol::Source> source = dap.ResolveSource(frame);
+
+  if (source && !IsAssemblySource(*source)) {
     // This is a normal source with a valid line entry.
     auto line_entry = frame.GetLineEntry();
     object.try_emplace("line", line_entry.GetLine());
@@ -584,8 +584,7 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
     // This is a source where the disassembly is used, but there is a valid
     // symbol. Calculate the line of the current PC from the start of the
     // current symbol.
-    lldb::SBTarget target = frame.GetThread().GetProcess().GetTarget();
-    lldb::SBInstructionList inst_list = target.ReadInstructions(
+    lldb::SBInstructionList inst_list = dap.target.ReadInstructions(
         frame.GetSymbol().GetStartAddress(), frame.GetPCAddress(), nullptr);
     size_t inst_line = inst_list.GetSize();
 
@@ -598,7 +597,8 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
     object.try_emplace("column", 1);
   }
 
-  object.try_emplace("source", std::move(source));
+  if (source)
+    object.try_emplace("source", std::move(source).value());
 
   const auto pc = frame.GetPC();
   if (pc != LLDB_INVALID_ADDRESS) {
diff --git a/lldb/tools/lldb-dap/JSONUtils.h b/lldb/tools/lldb-dap/JSONUtils.h
index 69da0725bd05..fd9a06931ebf 100644
--- a/lldb/tools/lldb-dap/JSONUtils.h
+++ b/lldb/tools/lldb-dap/JSONUtils.h
@@ -234,6 +234,9 @@ llvm::json::Object CreateEventObject(const llvm::StringRef event_name);
 ///   "line" - the source file line number as an integer
 ///   "column" - the source file column number as an integer
 ///
+/// \param[in] dap
+///     The DAP session associated with the stopped thread.
+///
 /// \param[in] frame
 ///     The LLDB stack frame to use when populating out the "StackFrame"
 ///     object.
@@ -245,7 +248,7 @@ llvm::json::Object CreateEventObject(const llvm::StringRef event_name);
 /// \return
 ///     A "StackFrame" JSON object with that follows the formal JSON
 ///     definition outlined by Microsoft.
-llvm::json::Value CreateStackFrame(lldb::SBFrame &frame,
+llvm::json::Value CreateStackFrame(DAP &dap, lldb::SBFrame &frame,
                                    lldb::SBFormat &format);
 
 /// Create a "StackFrame" label object for a LLDB thread.
diff --git a/lldb/tools/lldb-dap/ProgressEvent.h b/lldb/tools/lldb-dap/ProgressEvent.h
index d1b9b9dd887c..9dfed4c301a8 100644
--- a/lldb/tools/lldb-dap/ProgressEvent.h
+++ b/lldb/tools/lldb-dap/ProgressEvent.h
@@ -6,6 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLDB_TOOLS_LLDB_DAP_PROGRESS_EVENT_H
+#define LLDB_TOOLS_LLDB_DAP_PROGRESS_EVENT_H
+
 #include <atomic>
 #include <chrono>
 #include <mutex>
@@ -13,8 +16,6 @@
 #include <queue>
 #include <thread>
 
-#include "DAPForward.h"
-
 #include "llvm/Support/JSON.h"
 
 namespace lldb_dap {
@@ -129,8 +130,12 @@ class ProgressEventReporter {
 public:
   /// \param[in] report_callback
   ///     Function to invoke to report the event to the IDE.
-  ProgressEventReporter(ProgressEventReportCallback report_callback);
+  explicit ProgressEventReporter(ProgressEventReportCallback report_callback);
 
+  ProgressEventReporter(const ProgressEventReporter &) = delete;
+  ProgressEventReporter(ProgressEventReporter &&) = delete;
+  ProgressEventReporter &operator=(const ProgressEventReporter &) = delete;
+  ProgressEventReporter &operator=(ProgressEventReporter &&) = delete;
   ~ProgressEventReporter();
 
   /// Add a new event to the internal queue and report the event if
@@ -156,3 +161,5 @@ class ProgressEventReporter {
 };
 
 } // namespace lldb_dap
+
+#endif // LLDB_TOOLS_LLDB_DAP_PROGRESS_EVENT_H
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index 7d9a99fdacce..d4b816c72679 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -468,7 +468,7 @@ struct SourceArguments {
   /// The reference to the source. This is the same as `source.sourceReference`.
   /// This is provided for backward compatibility since old clients do not
   /// understand the `source` attribute.
-  int64_t sourceReference;
+  int64_t sourceReference = LLDB_DAP_INVALID_SRC_REF;
 };
 bool fromJSON(const llvm::json::Value &, SourceArguments &, llvm::json::Path);
 
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
index 7f96c07faae1..f3635202175a 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
@@ -65,7 +65,7 @@ llvm::json::Value toJSON(const Source &S) {
     result.insert({"name", *S.name});
   if (S.path)
     result.insert({"path", *S.path});
-  if (S.sourceReference)
+  if (S.sourceReference && (*S.sourceReference > LLDB_DAP_INVALID_SRC_REF))
     result.insert({"sourceReference", *S.sourceReference});
   if (S.presentationHint)
     result.insert({"presentationHint", *S.presentationHint});
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
index 7fe745411399..6adfe3b7211b 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
@@ -28,6 +28,7 @@
 #include <string>
 
 #define LLDB_DAP_INVALID_VARRERF UINT64_MAX
+#define LLDB_DAP_INVALID_SRC_REF 0
 
 namespace lldb_dap::protocol {
 
@@ -328,7 +329,7 @@ struct Source {
   /// `source` request (even if a path is specified). Since a `sourceReference`
   /// is only valid for a session, it can not be used to persist a source. The
   /// value should be less than or equal to 2147483647 (2^31-1).
-  std::optional<int64_t> sourceReference;
+  std::optional<int32_t> sourceReference;
 
   /// A hint for how to present the source in the UI. A value of `deemphasize`
   /// can be used to indicate that the source is not available or that it is
diff --git a/lldb/tools/lldb-dap/ProtocolUtils.cpp b/lldb/tools/lldb-dap/ProtocolUtils.cpp
index cb1ee6a42400..724d85110792 100644
--- a/lldb/tools/lldb-dap/ProtocolUtils.cpp
+++ b/lldb/tools/lldb-dap/ProtocolUtils.cpp
@@ -46,70 +46,33 @@ static bool ShouldDisplayAssemblySource(
   return false;
 }
 
-static protocol::Source CreateAssemblySource(const lldb::SBTarget &target,
-                                             lldb::SBAddress address) {
-  protocol::Source source;
-
-  auto symbol = address.GetSymbol();
-  std::string name;
-  if (symbol.IsValid()) {
-    source.sourceReference = symbol.GetStartAddress().GetLoadAddress(target);
-    name = symbol.GetName();
-  } else {
-    const auto load_addr = address.GetLoadAddress(target);
-    source.sourceReference = load_addr;
-    name = GetLoadAddressString(load_addr);
-  }
+std::optional<protocol::Source> CreateSource(const lldb::SBFileSpec &file) {
+  if (!file.IsValid())
+    return std::nullopt;
 
-  lldb::SBModule module = address.GetModule();
-  if (module.IsValid()) {
-    lldb::SBFileSpec file_spec = module.GetFileSpec();
-    if (file_spec.IsValid()) {
-      std::string path = GetSBFileSpecPath(file_spec);
-      if (!path.empty())
-        source.path = path + '`' + name;
-    }
-  }
-
-  source.name = std::move(name);
-
-  // Mark the source as deemphasized since users will only be able to view
-  // assembly for these frames.
-  source.presentationHint =
-      protocol::Source::PresentationHint::eSourcePresentationHintDeemphasize;
-
-  return source;
-}
-
-protocol::Source CreateSource(const lldb::SBFileSpec &file) {
   protocol::Source source;
-  if (file.IsValid()) {
-    if (const char *name = file.GetFilename())
-      source.name = name;
-    char path[PATH_MAX] = "";
-    if (file.GetPath(path, sizeof(path)) &&
-        lldb::SBFileSpec::ResolvePath(path, path, PATH_MAX))
-      source.path = path;
-  }
+  if (const char *name = file.GetFilename())
+    source.name = name;
+  char path[PATH_MAX] = "";
+  if (file.GetPath(path, sizeof(path)) &&
+      lldb::SBFileSpec::ResolvePath(path, path, PATH_MAX))
+    source.path = path;
   return source;
 }
 
-protocol::Source CreateSource(lldb::SBAddress address, lldb::SBTarget &target) {
-  lldb::SBDebugger debugger = target.GetDebugger();
-  lldb::StopDisassemblyType stop_disassembly_display =
-      GetStopDisassemblyDisplay(debugger);
-  if (ShouldDisplayAssemblySource(address, stop_disassembly_display))
-    return CreateAssemblySource(target, address);
-
-  lldb::SBLineEntry line_entry = GetLineEntryForAddress(target, address);
-  return CreateSource(line_entry.GetFileSpec());
-}
-
 bool IsAssemblySource(const protocol::Source &source) {
   // According to the specification, a source must have either `path` or
   // `sourceReference` specified. We use `path` for sources with known source
   // code, and `sourceReferences` when falling back to assembly.
-  return source.sourceReference.value_or(0) != 0;
+  return source.sourceReference.value_or(LLDB_DAP_INVALID_SRC_REF) >
+         LLDB_DAP_INVALID_SRC_REF;
+}
+
+bool DisplayAssemblySource(lldb::SBDebugger &debugger,
+                           lldb::SBAddress address) {
+  const lldb::StopDisassemblyType stop_disassembly_display =
+      GetStopDisassemblyDisplay(debugger);
+  return ShouldDisplayAssemblySource(address, stop_disassembly_display);
 }
 
 std::string GetLoadAddressString(const lldb::addr_t addr) {
diff --git a/lldb/tools/lldb-dap/ProtocolUtils.h b/lldb/tools/lldb-dap/ProtocolUtils.h
index 788d2fd054e2..f36bf0fb60a8 100644
--- a/lldb/tools/lldb-dap/ProtocolUtils.h
+++ b/lldb/tools/lldb-dap/ProtocolUtils.h
@@ -26,26 +26,15 @@ namespace lldb_dap {
 ///     The SBFileSpec to use when populating out the "Source" object
 ///
 /// \return
-///     A "Source" JSON object that follows the formal JSON
+///     An optional "Source" JSON object that follows the formal JSON
 ///     definition outlined by Microsoft.
-protocol::Source CreateSource(const lldb::SBFileSpec &file);
-
-/// Create a "Source" JSON object as described in the debug adapter definition.
-///
-/// \param[in] address
-///     The address to use when populating out the "Source" object.
-///
-/// \param[in] target
-///     The target that has the address.
-///
-/// \return
-///     A "Source" JSON object that follows the formal JSON
-///     definition outlined by Microsoft.
-protocol::Source CreateSource(lldb::SBAddress address, lldb::SBTarget &target);
+std::optional<protocol::Source> CreateSource(const lldb::SBFileSpec &file);
 
 /// Checks if the given source is for assembly code.
 bool IsAssemblySource(const protocol::Source &source);
 
+bool DisplayAssemblySource(lldb::SBDebugger &debugger, lldb::SBAddress address);
+
 /// Get the address as a 16-digit hex string, e.g. "0x0000000000012345"
 std::string GetLoadAddressString(const lldb::addr_t addr);
 
diff --git a/lldb/tools/lldb-dap/SourceBreakpoint.cpp b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
index a4327ae18cf6..5fce9fe0ddbb 100644
--- a/lldb/tools/lldb-dap/SourceBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
@@ -46,7 +46,13 @@ llvm::Error SourceBreakpoint::SetBreakpoint(const protocol::Source &source) {
 
   if (source.sourceReference) {
     // Breakpoint set by assembly source.
-    lldb::SBAddress source_address(*source.sourceReference, m_dap.target);
+    std::optional<lldb::addr_t> raw_addr =
+        m_dap.GetSourceReferenceAddress(*source.sourceReference);
+    if (!raw_addr)
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "Invalid sourceReference.");
+
+    lldb::SBAddress source_address(*raw_addr, m_dap.target);
     if (!source_address.IsValid())
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                      "Invalid sourceReference.");
diff --git a/lldb/tools/lldb-dap/tool/lldb-dap.cpp b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
index 2799d10ae84b..8bba4162aa7b 100644
--- a/lldb/tools/lldb-dap/tool/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
@@ -351,8 +351,6 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
                      << " disconnected failed: "
                      << llvm::toString(std::move(error)) << "\n";
       }
-      // Close the socket to ensure the DAP::Loop read finishes.
-      sock->Close();
     }
   }
 
diff --git a/lldb/tools/lldb-server/lldb-gdbserver.cpp b/lldb/tools/lldb-server/lldb-gdbserver.cpp
index 843354147f2d..e1dc15d83069 100644
--- a/lldb/tools/lldb-server/lldb-gdbserver.cpp
+++ b/lldb/tools/lldb-server/lldb-gdbserver.cpp
@@ -185,7 +185,7 @@ Status writeSocketIdToPipe(const char *const named_pipe_path,
                            llvm::StringRef socket_id) {
   Pipe port_name_pipe;
   // Wait for 10 seconds for pipe to be opened.
-  if (llvm::Error err = port_name_pipe.OpenAsWriter(named_pipe_path, false,
+  if (llvm::Error err = port_name_pipe.OpenAsWriter(named_pipe_path,
                                                     std::chrono::seconds{10}))
     return Status::FromError(std::move(err));
 
@@ -218,8 +218,8 @@ void ConnectToRemote(MainLoop &mainloop,
                                     error.AsCString());
       exit(-1);
     }
-    connection_up = std::unique_ptr<Connection>(new ConnectionFileDescriptor(
-        new TCPSocket(sockfd, /*should_close=*/true)));
+    connection_up = std::make_unique<ConnectionFileDescriptor>(
+        std::make_unique<TCPSocket>(sockfd, /*should_close=*/true));
 #else
     url = llvm::formatv("fd://{0}", connection_fd).str();
 
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index 3c79dc001f65..0bd928507ba8 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -30,6 +30,7 @@
 #include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h"
 #include "Plugins/Process/gdb-remote/ProcessGDBRemoteLog.h"
 #include "lldb/Host/ConnectionFileDescriptor.h"
+#include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostGetOpt.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/MainLoop.h"
@@ -192,14 +193,15 @@ static Status ListenGdbConnectionsIfNeeded(
 }
 
 static llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>>
-AcceptGdbConnectionsIfNeeded(const Socket::SocketProtocol protocol,
+AcceptGdbConnectionsIfNeeded(const FileSpec &debugserver_path,
+                             const Socket::SocketProtocol protocol,
                              std::unique_ptr<TCPSocket> &gdb_sock,
                              MainLoop &main_loop, const uint16_t gdbserver_port,
                              const lldb_private::Args &args) {
   if (protocol != Socket::ProtocolTcp)
     return std::vector<MainLoopBase::ReadHandleUP>();
 
-  return gdb_sock->Accept(main_loop, [gdbserver_port,
+  return gdb_sock->Accept(main_loop, [debugserver_path, gdbserver_port,
                                       &args](std::unique_ptr<Socket> sock_up) {
     Log *log = GetLog(LLDBLog::Platform);
     Status error;
@@ -210,8 +212,8 @@ AcceptGdbConnectionsIfNeeded(const Socket::SocketProtocol protocol,
     }
     lldb::pid_t child_pid = LLDB_INVALID_PROCESS_ID;
     std::string socket_name;
-    GDBRemoteCommunicationServerPlatform platform(Socket::ProtocolTcp,
-                                                  gdbserver_port);
+    GDBRemoteCommunicationServerPlatform platform(
+        debugserver_path, Socket::ProtocolTcp, gdbserver_port);
     error = platform.LaunchGDBServer(args, child_pid, socket_name,
                                      shared_socket.GetSendableFD());
     if (error.Success() && child_pid != LLDB_INVALID_PROCESS_ID) {
@@ -341,6 +343,24 @@ static Status spawn_process(const char *progname, const FileSpec &prog,
   return Status();
 }
 
+static FileSpec GetDebugserverPath() {
+  if (const char *p = getenv("LLDB_DEBUGSERVER_PATH")) {
+    FileSpec candidate(p);
+    if (FileSystem::Instance().Exists(candidate))
+      return candidate;
+  }
+#if defined(__APPLE__)
+  FileSpec candidate = HostInfo::GetSupportExeDir();
+  candidate.AppendPathComponent("debugserver");
+  if (FileSystem::Instance().Exists(candidate))
+    return candidate;
+  return FileSpec();
+#else
+  // On non-apple platforms, *we* are the debug server.
+  return HostInfo::GetProgramFileSpec();
+#endif
+}
+
 // main
 int main_platform(int argc, char *argv[]) {
   const char *progname = argv[0];
@@ -453,6 +473,12 @@ int main_platform(int argc, char *argv[]) {
   lldb_private::Args inferior_arguments;
   inferior_arguments.SetArguments(argc, const_cast<const char **>(argv));
 
+  FileSpec debugserver_path = GetDebugserverPath();
+  if (!debugserver_path) {
+    WithColor::error(errs()) << "Could not find debug server executable.";
+    return EXIT_FAILURE;
+  }
+
   Log *log = GetLog(LLDBLog::Platform);
   if (fd != SharedSocket::kInvalidFD) {
     // Child process will handle the connection and exit.
@@ -483,8 +509,8 @@ int main_platform(int argc, char *argv[]) {
 #endif
     }
 
-    GDBRemoteCommunicationServerPlatform platform(socket->GetSocketProtocol(),
-                                                  gdbserver_port);
+    GDBRemoteCommunicationServerPlatform platform(
+        debugserver_path, socket->GetSocketProtocol(), gdbserver_port);
     platform.SetConnection(
         std::make_unique<ConnectionFileDescriptor>(std::move(socket)));
     client_handle(platform, inferior_arguments);
@@ -576,8 +602,9 @@ int main_platform(int argc, char *argv[]) {
     }
 
     llvm::Expected<std::vector<MainLoopBase::ReadHandleUP>> gdb_handles =
-        AcceptGdbConnectionsIfNeeded(protocol, gdb_sock, main_loop,
-                                     gdbserver_port, inferior_arguments);
+        AcceptGdbConnectionsIfNeeded(debugserver_path, protocol, gdb_sock,
+                                     main_loop, gdbserver_port,
+                                     inferior_arguments);
     if (!gdb_handles) {
       printf("Failed to accept gdb: %s\n",
              llvm::toString(gdb_handles.takeError()).c_str());
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 6743ce543dc6..3f198d963a93 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -1112,13 +1112,13 @@ bool opts::irmemorymap::evalMalloc(StringRef Line,
   // Issue the malloc in the target process with "-rw" permissions.
   const uint32_t Permissions = 0x3;
   const bool ZeroMemory = false;
-  Status ST;
-  addr_t Addr =
-      State.Map.Malloc(Size, Alignment, Permissions, AP, ZeroMemory, ST);
-  if (ST.Fail()) {
-    outs() << formatv("Malloc error: {0}\n", ST);
+  auto AddrOrErr =
+      State.Map.Malloc(Size, Alignment, Permissions, AP, ZeroMemory);
+  if (!AddrOrErr) {
+    outs() << formatv("Malloc error: {0}\n", toString(AddrOrErr.takeError()));
     return true;
   }
+  addr_t Addr = *AddrOrErr;
 
   // Print the result of the allocation before checking its validity.
   outs() << formatv("Malloc: address = {0:x}\n", Addr);
diff --git a/lldb/unittests/Core/CommunicationTest.cpp b/lldb/unittests/Core/CommunicationTest.cpp
index 51e218af8baf..f8c584ae142b 100644
--- a/lldb/unittests/Core/CommunicationTest.cpp
+++ b/lldb/unittests/Core/CommunicationTest.cpp
@@ -141,8 +141,7 @@ TEST_F(CommunicationTest, SynchronizeWhileClosing) {
 #if LLDB_ENABLE_POSIX
 TEST_F(CommunicationTest, WriteAll) {
   Pipe pipe;
-  ASSERT_THAT_ERROR(pipe.CreateNew(/*child_process_inherit=*/false).ToError(),
-                    llvm::Succeeded());
+  ASSERT_THAT_ERROR(pipe.CreateNew().ToError(), llvm::Succeeded());
 
   // Make the write end non-blocking in order to easily reproduce a partial
   // write.
diff --git a/lldb/unittests/DAP/TestBase.cpp b/lldb/unittests/DAP/TestBase.cpp
index 27ad42686fbb..d5d36158d68e 100644
--- a/lldb/unittests/DAP/TestBase.cpp
+++ b/lldb/unittests/DAP/TestBase.cpp
@@ -29,7 +29,7 @@ using lldb_private::NativeFile;
 using lldb_private::Pipe;
 
 void TransportBase::SetUp() {
-  PipeTest::SetUp();
+  PipePairTest::SetUp();
   to_dap = std::make_unique<Transport>(
       "to_dap", nullptr,
       std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
diff --git a/lldb/unittests/DAP/TestBase.h b/lldb/unittests/DAP/TestBase.h
index 25d37013954d..50884b1d7feb 100644
--- a/lldb/unittests/DAP/TestBase.h
+++ b/lldb/unittests/DAP/TestBase.h
@@ -18,7 +18,7 @@ namespace lldb_dap_tests {
 
 /// A base class for tests that need transport configured for communicating DAP
 /// messages.
-class TransportBase : public PipeTest {
+class TransportBase : public PipePairTest {
 protected:
   std::unique_ptr<lldb_dap::Transport> to_dap;
   std::unique_ptr<lldb_dap::Transport> from_dap;
diff --git a/lldb/unittests/Host/CMakeLists.txt b/lldb/unittests/Host/CMakeLists.txt
index 3b20f1d723d1..5591edda38ac 100644
--- a/lldb/unittests/Host/CMakeLists.txt
+++ b/lldb/unittests/Host/CMakeLists.txt
@@ -37,7 +37,9 @@ add_lldb_unittest(HostTests
     lldbUtilityHelpers
     lldbHostHelpers
     LLVMTestingSupport
-    LLVMTargetParser
+
+  LINK_COMPONENTS
+    TargetParser
   )
 
 add_subdirectory(common)
diff --git a/lldb/unittests/Host/FileTest.cpp b/lldb/unittests/Host/FileTest.cpp
index 35c87bb200fa..d973d1943059 100644
--- a/lldb/unittests/Host/FileTest.cpp
+++ b/lldb/unittests/Host/FileTest.cpp
@@ -14,6 +14,10 @@
 #include "llvm/Support/Program.h"
 #include "gtest/gtest.h"
 
+#ifdef _WIN32
+#include "lldb/Host/windows/windows.h"
+#endif
+
 using namespace lldb;
 using namespace lldb_private;
 
@@ -32,7 +36,11 @@ TEST(File, GetWaitableHandleFileno) {
   ASSERT_TRUE(stream);
 
   NativeFile file(stream, true);
-  EXPECT_EQ(file.GetWaitableHandle(), fd);
+#ifdef _WIN32
+  EXPECT_EQ(file.GetWaitableHandle(), (HANDLE)_get_osfhandle(fd));
+#else
+  EXPECT_EQ(file.GetWaitableHandle(), (file_t)fd);
+#endif
 }
 
 TEST(File, GetStreamFromDescriptor) {
@@ -53,5 +61,9 @@ TEST(File, GetStreamFromDescriptor) {
   ASSERT_TRUE(stream != NULL);
 
   EXPECT_EQ(file.GetDescriptor(), fd);
-  EXPECT_EQ(file.GetWaitableHandle(), fd);
+#ifdef _WIN32
+  EXPECT_EQ(file.GetWaitableHandle(), (HANDLE)_get_osfhandle(fd));
+#else
+  EXPECT_EQ(file.GetWaitableHandle(), (file_t)fd);
+#endif
 }
diff --git a/lldb/unittests/Host/HostTest.cpp b/lldb/unittests/Host/HostTest.cpp
index dca32c7547cf..1424e4472edd 100644
--- a/lldb/unittests/Host/HostTest.cpp
+++ b/lldb/unittests/Host/HostTest.cpp
@@ -145,8 +145,7 @@ TEST(Host, LaunchProcessDuplicatesHandle) {
     exit(1);
   }
   Pipe pipe;
-  ASSERT_THAT_ERROR(pipe.CreateNew(/*child_process_inherit=*/false).takeError(),
-                    llvm::Succeeded());
+  ASSERT_THAT_ERROR(pipe.CreateNew().takeError(), llvm::Succeeded());
   SCOPED_TRACE(llvm::formatv("Pipe handles are: {0}/{1}",
                              (uint64_t)pipe.GetReadPipe(),
                              (uint64_t)pipe.GetWritePipe())
diff --git a/lldb/unittests/Host/JSONTransportTest.cpp b/lldb/unittests/Host/JSONTransportTest.cpp
index 4621869887ac..2f0846471688 100644
--- a/lldb/unittests/Host/JSONTransportTest.cpp
+++ b/lldb/unittests/Host/JSONTransportTest.cpp
@@ -14,12 +14,12 @@ using namespace llvm;
 using namespace lldb_private;
 
 namespace {
-template <typename T> class JSONTransportTest : public PipeTest {
+template <typename T> class JSONTransportTest : public PipePairTest {
 protected:
   std::unique_ptr<JSONTransport> transport;
 
   void SetUp() override {
-    PipeTest::SetUp();
+    PipePairTest::SetUp();
     transport = std::make_unique<T>(
         std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
                                      File::eOpenOptionReadOnly,
diff --git a/lldb/unittests/Host/MainLoopTest.cpp b/lldb/unittests/Host/MainLoopTest.cpp
index e18489978e90..502028ae1a34 100644
--- a/lldb/unittests/Host/MainLoopTest.cpp
+++ b/lldb/unittests/Host/MainLoopTest.cpp
@@ -79,7 +79,69 @@ TEST_F(MainLoopTest, ReadObject) {
   ASSERT_EQ(1u, callback_count);
 }
 
-TEST_F(MainLoopTest, NoSpuriousReads) {
+TEST_F(MainLoopTest, ReadPipeObject) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  MainLoop loop;
+
+  char X = 'X';
+  size_t len = sizeof(X);
+  ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::HasValue(1));
+
+  Status error;
+  auto handle = loop.RegisterReadObject(
+      std::make_shared<NativeFile>(pipe.GetReadFileDescriptor(),
+                                   File::eOpenOptionReadOnly, false),
+      make_callback(), error);
+  ASSERT_TRUE(error.Success());
+  ASSERT_TRUE(handle);
+  ASSERT_TRUE(loop.Run().Success());
+  ASSERT_EQ(1u, callback_count);
+}
+
+TEST_F(MainLoopTest, NoSpuriousPipeReads) {
+  Pipe pipe;
+
+  ASSERT_TRUE(pipe.CreateNew().Success());
+
+  char X = 'X';
+  size_t len = sizeof(X);
+  ASSERT_THAT_EXPECTED(pipe.Write(&X, len), llvm::Succeeded());
+
+  lldb::IOObjectSP r = std::make_shared<NativeFile>(
+      pipe.GetReadFileDescriptor(), File::eOpenOptionReadOnly, false);
+
+  MainLoop loop;
+
+  Status error;
+  auto handle = loop.RegisterReadObject(
+      r,
+      [&](MainLoopBase &) {
+        if (callback_count == 0) {
+          // Read the byte back the first time we're called. After that, the
+          // pipe is empty, and we should not be called anymore.
+          char X;
+          size_t len = sizeof(X);
+          ASSERT_THAT_ERROR(r->Read(&X, len).ToError(), llvm::Succeeded());
+          EXPECT_EQ(len, sizeof(X));
+          EXPECT_EQ(X, 'X');
+        }
+        ++callback_count;
+      },
+      error);
+  ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
+  // Terminate the loop after one second.
+  loop.AddCallback([](MainLoopBase &loop) { loop.RequestTermination(); },
+                   std::chrono::seconds(1));
+  ASSERT_THAT_ERROR(loop.Run().ToError(), llvm::Succeeded());
+
+  // Make sure the callback was called only once.
+  ASSERT_EQ(1u, callback_count);
+}
+
+TEST_F(MainLoopTest, NoSpuriousSocketReads) {
   // Write one byte into the socket.
   char X = 'X';
   size_t len = sizeof(X);
@@ -99,6 +161,7 @@ TEST_F(MainLoopTest, NoSpuriousReads) {
           EXPECT_THAT_ERROR(socketpair[1]->Read(&X, len).ToError(),
                             llvm::Succeeded());
           EXPECT_EQ(len, sizeof(X));
+          EXPECT_EQ(X, 'X');
         }
         ++callback_count;
       },
@@ -164,9 +227,8 @@ TEST_F(MainLoopTest, PendingCallbackCalledOnlyOnce) {
       [&](MainLoopBase &loop) {
         // Add one pending callback on the first iteration.
         if (callback_count == 0) {
-          loop.AddPendingCallback([&](MainLoopBase &loop) {
-            callback_count++;
-          });
+          loop.AddPendingCallback(
+              [&](MainLoopBase &loop) { callback_count++; });
         }
         // Terminate the loop on second iteration.
         if (callback_count++ >= 1)
@@ -321,7 +383,7 @@ TEST_F(MainLoopTest, UnmonitoredSignal) {
   MainLoop loop;
   Status error;
   struct sigaction sa;
-  sa.sa_sigaction = [](int, siginfo_t *, void *) { };
+  sa.sa_sigaction = [](int, siginfo_t *, void *) {};
   sa.sa_flags = SA_SIGINFO; // important: no SA_RESTART
   sigemptyset(&sa.sa_mask);
   ASSERT_EQ(0, sigaction(SIGUSR2, &sa, nullptr));
diff --git a/lldb/unittests/Host/PipeTest.cpp b/lldb/unittests/Host/PipeTest.cpp
index d33797adb7fc..4ee3b53cc552 100644
--- a/lldb/unittests/Host/PipeTest.cpp
+++ b/lldb/unittests/Host/PipeTest.cpp
@@ -29,11 +29,10 @@ class PipeTest : public testing::Test {
 TEST_F(PipeTest, CreateWithUniqueName) {
   Pipe pipe;
   llvm::SmallString<0> name;
-  ASSERT_THAT_ERROR(pipe.CreateWithUniqueName("PipeTest-CreateWithUniqueName",
-                                              /*child_process_inherit=*/false,
-                                              name)
-                        .ToError(),
-                    llvm::Succeeded());
+  ASSERT_THAT_ERROR(
+      pipe.CreateWithUniqueName("PipeTest-CreateWithUniqueName", name)
+          .ToError(),
+      llvm::Succeeded());
 }
 
 // Test broken
@@ -41,19 +40,15 @@ TEST_F(PipeTest, CreateWithUniqueName) {
 TEST_F(PipeTest, OpenAsReader) {
   Pipe pipe;
   llvm::SmallString<0> name;
-  ASSERT_THAT_ERROR(pipe.CreateWithUniqueName("PipeTest-OpenAsReader",
-                                              /*child_process_inherit=*/false,
-                                              name)
-                        .ToError(),
-                    llvm::Succeeded());
+  ASSERT_THAT_ERROR(
+      pipe.CreateWithUniqueName("PipeTest-OpenAsReader", name).ToError(),
+      llvm::Succeeded());
 
   // Ensure name is not null-terminated
   size_t name_len = name.size();
   name += "foobar";
   llvm::StringRef name_ref(name.data(), name_len);
-  ASSERT_THAT_ERROR(
-      pipe.OpenAsReader(name_ref, /*child_process_inherit=*/false).ToError(),
-      llvm::Succeeded());
+  ASSERT_THAT_ERROR(pipe.OpenAsReader(name_ref).ToError(), llvm::Succeeded());
 
   ASSERT_TRUE(pipe.CanRead());
 }
@@ -63,7 +58,7 @@ TEST_F(PipeTest, OpenAsReader) {
 #ifndef _WIN32
 TEST_F(PipeTest, WriteWithTimeout) {
   Pipe pipe;
-  ASSERT_THAT_ERROR(pipe.CreateNew(false).ToError(), llvm::Succeeded());
+  ASSERT_THAT_ERROR(pipe.CreateNew().ToError(), llvm::Succeeded());
 
   // The pipe buffer is 1024 for PipeWindows and at least 512 on Darwin.
   // In Linux versions before 2.6.11, the capacity of a pipe was the same as the
@@ -153,7 +148,7 @@ TEST_F(PipeTest, WriteWithTimeout) {
 
 TEST_F(PipeTest, ReadWithTimeout) {
   Pipe pipe;
-  ASSERT_THAT_ERROR(pipe.CreateNew(false).ToError(), llvm::Succeeded());
+  ASSERT_THAT_ERROR(pipe.CreateNew().ToError(), llvm::Succeeded());
 
   char buf[100];
   // The pipe is initially empty. A polling read returns immediately.
diff --git a/lldb/unittests/Host/SocketTest.cpp b/lldb/unittests/Host/SocketTest.cpp
index 9fefb2a6ae0e..5d0085210dd4 100644
--- a/lldb/unittests/Host/SocketTest.cpp
+++ b/lldb/unittests/Host/SocketTest.cpp
@@ -199,9 +199,7 @@ TEST_P(SocketTest, TCPAcceptTimeout) {
   if (!HostSupportsProtocol())
     return;
 
-  const bool child_processes_inherit = false;
-  auto listen_socket_up =
-      std::make_unique<TCPSocket>(true, child_processes_inherit);
+  auto listen_socket_up = std::make_unique<TCPSocket>(true);
   Status error = listen_socket_up->Listen(
       llvm::formatv("[{0}]:0", GetParam().localhost_ip).str(), 5);
   ASSERT_THAT_ERROR(error.ToError(), llvm::Succeeded());
diff --git a/lldb/unittests/Host/posix/SupportTest.cpp b/lldb/unittests/Host/posix/SupportTest.cpp
index e4d7ba89fece..5393ad83ad8a 100644
--- a/lldb/unittests/Host/posix/SupportTest.cpp
+++ b/lldb/unittests/Host/posix/SupportTest.cpp
@@ -7,10 +7,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/posix/Support.h"
-#include "lldb/Host/aix/Support.h"
 #include "llvm/Support/Threading.h"
 #include "gtest/gtest.h"
 
+#if defined(_AIX)
+#include "lldb/Host/aix/Support.h"
+#elif defined(__linux__)
+#include "lldb/Host/linux/Support.h"
+#endif
+
 using namespace lldb_private;
 
 #ifndef __APPLE__
@@ -21,10 +26,16 @@ TEST(Support, getProcFile_Pid) {
 }
 #endif // #ifndef __APPLE__
 
-#if defined(_AIX) && defined(LLVM_ENABLE_THREADING)
+#if defined(_AIX) || defined(__linux__)
 TEST(Support, getProcFile_Tid) {
-  auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(), "lwpstatus");
+  auto BufferOrError = getProcFile(getpid(), llvm::get_threadid(),
+#ifdef _AIX
+                                   "lwpstatus"
+#else
+                                   "status"
+#endif
+  );
   ASSERT_TRUE(BufferOrError);
   ASSERT_TRUE(*BufferOrError);
 }
-#endif // #ifdef _AIX && LLVM_ENABLE_THREADING
+#endif // #if defined(_AIX) || defined(__linux__)
diff --git a/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp b/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
index f9372ded0133..f3b0d65c2cbf 100644
--- a/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
+++ b/lldb/unittests/Instruction/LoongArch/TestLoongArchEmulator.cpp
@@ -166,7 +166,6 @@ using EncoderBZcond = uint32_t (*)(uint32_t rj, int32_t offs21);
 using EncoderBCZcond = uint32_t (*)(uint8_t cj, int32_t offs21);
 
 TEST_F(LoongArch64EmulatorTester, testJIRL) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // JIRL r1, r12, 0x10
@@ -177,14 +176,13 @@ TEST_F(LoongArch64EmulatorTester, testJIRL) {
   gpr.gpr[12] = 0x12000400;
   ASSERT_TRUE(TestExecute(inst));
   auto r1 = gpr.gpr[1];
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
   ASSERT_EQ(r1, old_pc + 4);
-  ASSERT_EQ(pc, gpr.gpr[12] + (offs16 * 4));
+  ASSERT_EQ(*pc, gpr.gpr[12] + (offs16 * 4));
 }
 
 TEST_F(LoongArch64EmulatorTester, testB) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // B  0x10010
@@ -193,13 +191,12 @@ TEST_F(LoongArch64EmulatorTester, testB) {
   uint32_t inst = 0b01010000000000000100000000000001;
   uint32_t offs26 = 0x10010;
   ASSERT_TRUE(TestExecute(inst));
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (offs26 * 4));
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (offs26 * 4));
 }
 
 TEST_F(LoongArch64EmulatorTester, testBL) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   WritePC(old_pc);
   // BL  0x10010
@@ -209,16 +206,15 @@ TEST_F(LoongArch64EmulatorTester, testBL) {
   uint32_t offs26 = 0x10010;
   ASSERT_TRUE(TestExecute(inst));
   auto r1 = gpr.gpr[1];
-  auto pc = ReadPC(&success);
-  ASSERT_TRUE(success);
+  auto pc = ReadPC();
+  ASSERT_TRUE(pc);
   ASSERT_EQ(r1, old_pc + 4);
-  ASSERT_EQ(pc, old_pc + (offs26 * 4));
+  ASSERT_EQ(*pc, old_pc + (offs26 * 4));
 }
 
 static void testBcondBranch(LoongArch64EmulatorTester *tester,
                             EncoderBcond encoder, bool branched,
                             uint64_t rj_val, uint64_t rd_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->gpr.gpr[12] = rj_val;
@@ -226,39 +222,37 @@ static void testBcondBranch(LoongArch64EmulatorTester *tester,
   // b<cmp> r12, r13, (-256)
   uint32_t inst = encoder(12, 13, -256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (-256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (-256 * 4) : 4));
 }
 
 static void testBZcondBranch(LoongArch64EmulatorTester *tester,
                              EncoderBZcond encoder, bool branched,
                              uint64_t rj_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->gpr.gpr[4] = rj_val;
   // b<cmp>z  r4, (-256)
   uint32_t inst = encoder(4, -256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (-256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (-256 * 4) : 4));
 }
 
 static void testBCZcondBranch(LoongArch64EmulatorTester *tester,
                               EncoderBCZcond encoder, bool branched,
                               uint32_t cj_val) {
-  bool success = false;
   addr_t old_pc = 0x12000600;
   tester->WritePC(old_pc);
   tester->fpr.fcc = cj_val;
   // bc<cmp>z fcc0, 256
   uint32_t inst = encoder(0, 256);
   ASSERT_TRUE(tester->TestExecute(inst));
-  auto pc = tester->ReadPC(&success);
-  ASSERT_TRUE(success);
-  ASSERT_EQ(pc, old_pc + (branched ? (256 * 4) : 4));
+  auto pc = tester->ReadPC();
+  ASSERT_TRUE(pc);
+  ASSERT_EQ(*pc, old_pc + (branched ? (256 * 4) : 4));
 }
 
 GEN_BCOND_TEST(64, BEQ, 1, 1, 0)
diff --git a/lldb/unittests/Process/Utility/MemoryTagManagerAArch64MTETest.cpp b/lldb/unittests/Process/Utility/MemoryTagManagerAArch64MTETest.cpp
index 40d7c3601ccf..30199bfe5c25 100644
--- a/lldb/unittests/Process/Utility/MemoryTagManagerAArch64MTETest.cpp
+++ b/lldb/unittests/Process/Utility/MemoryTagManagerAArch64MTETest.cpp
@@ -87,31 +87,38 @@ TEST(MemoryTagManagerAArch64MTETest, UnpackTagsFromCoreFileSegment) {
   std::vector<uint8_t> tags_data;
   MemoryTagManager::CoreReaderFn reader =
       [&tags_data](lldb::offset_t offset, size_t length, void *dst) {
+        if ((offset + length) >= tags_data.size())
+          length = tags_data.size() - offset;
+
         std::memcpy(dst, tags_data.data() + offset, length);
         return length;
       };
 
   // Zero length is ok.
-  std::vector<lldb::addr_t> tags =
+  llvm::Expected<std::vector<lldb::addr_t>> tags =
       manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 0, 0);
-  ASSERT_EQ(tags.size(), (size_t)0);
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_EQ(tags->size(), (size_t)0);
 
   // In the simplest case we read 2 tags which are in the same byte.
   tags_data.push_back(0x21);
   // The least significant bits are the first tag in memory.
   std::vector<lldb::addr_t> expected{1, 2};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 0, 32);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // If we read just one then it will have to trim off the second one.
   expected = std::vector<lldb::addr_t>{1};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 0, 16);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // If we read the second tag only then the first one must be trimmed.
   expected = std::vector<lldb::addr_t>{2};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 16, 16);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // This trimming logic applies if you read a larger set of tags.
   tags_data = std::vector<uint8_t>{0x21, 0x43, 0x65, 0x87};
@@ -119,31 +126,55 @@ TEST(MemoryTagManagerAArch64MTETest, UnpackTagsFromCoreFileSegment) {
   // Trailing tag should be trimmed.
   expected = std::vector<lldb::addr_t>{1, 2, 3};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 0, 48);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // Leading tag should be trimmed.
   expected = std::vector<lldb::addr_t>{2, 3, 4};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 16, 48);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // Leading and trailing trimmmed.
   expected = std::vector<lldb::addr_t>{2, 3, 4, 5};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 16, 64);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // The address given is an offset into the whole file so the address requested
   // from the reader should be beyond that.
   tags_data = std::vector<uint8_t>{0xFF, 0xFF, 0x21, 0x43, 0x65, 0x87};
   expected = std::vector<lldb::addr_t>{1, 2};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 2, 0, 32);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
 
   // addr is a virtual address that we expect to be >= the tag segment's
   // starting virtual address. So again an offset must be made from the
   // difference.
   expected = std::vector<lldb::addr_t>{3, 4};
   tags = manager.UnpackTagsFromCoreFileSegment(reader, 32, 2, 64, 32);
-  ASSERT_THAT(expected, testing::ContainerEq(tags));
+  ASSERT_THAT_EXPECTED(tags, llvm::Succeeded());
+  ASSERT_THAT(expected, testing::ContainerEq(*tags));
+
+  // Error when there is not enough data to decode tags.
+
+  // Read 1 tag from an offset just outside the segment's data.
+  tags_data = {0xAB};
+  tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 32, 16);
+  const char *expected_err = "Could not read tags from core file segment. "
+                             "Segment is missing some or all tag data.";
+  EXPECT_THAT_EXPECTED(tags, llvm::FailedWithMessage(expected_err));
+
+  // First 2 tags come from the segment, second 2 cannot be read.
+  tags_data.push_back(0xCD);
+  tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 32, 64);
+  EXPECT_THAT_EXPECTED(tags, llvm::FailedWithMessage(expected_err));
+
+  // Segment is completely empty.
+  tags_data.clear();
+  tags = manager.UnpackTagsFromCoreFileSegment(reader, 0, 0, 0, 16);
+  EXPECT_THAT_EXPECTED(tags, llvm::FailedWithMessage(expected_err));
 }
 
 TEST(MemoryTagManagerAArch64MTETest, GetLogicalTag) {
diff --git a/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp b/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
index 72b8c7b1fd82..8e61379b5c73 100644
--- a/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
+++ b/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
@@ -46,9 +46,10 @@ class TestTool : public mcp::Tool {
   using mcp::Tool::Tool;
 
   virtual llvm::Expected<mcp::protocol::TextResult>
-  Call(const llvm::json::Value &args) override {
+  Call(const ToolArguments &args) override {
     std::string argument;
-    if (const json::Object *args_obj = args.getAsObject()) {
+    if (const json::Object *args_obj =
+            std::get<json::Value>(args).getAsObject()) {
       if (const json::Value *s = args_obj->get("arguments")) {
         argument = s->getAsString().value_or("");
       }
@@ -66,7 +67,7 @@ class ErrorTool : public mcp::Tool {
   using mcp::Tool::Tool;
 
   virtual llvm::Expected<mcp::protocol::TextResult>
-  Call(const llvm::json::Value &args) override {
+  Call(const ToolArguments &args) override {
     return llvm::createStringError("error");
   }
 };
@@ -77,7 +78,7 @@ class FailTool : public mcp::Tool {
   using mcp::Tool::Tool;
 
   virtual llvm::Expected<mcp::protocol::TextResult>
-  Call(const llvm::json::Value &args) override {
+  Call(const ToolArguments &args) override {
     mcp::protocol::TextResult text_result;
     text_result.content.emplace_back(mcp::protocol::TextContent{{"failed"}});
     text_result.isError = true;
@@ -115,7 +116,7 @@ class ProtocolServerMCPTest : public ::testing::Test {
     ProtocolServer::Connection connection;
     connection.protocol = Socket::SocketProtocol::ProtocolTcp;
     connection.name = llvm::formatv("{0}:0", k_localhost).str();
-    m_server_up = std::make_unique<TestProtocolServerMCP>(*m_debugger_sp);
+    m_server_up = std::make_unique<TestProtocolServerMCP>();
     m_server_up->AddTool(std::make_unique<TestTool>("test", "test tool"));
     ASSERT_THAT_ERROR(m_server_up->Start(connection), llvm::Succeeded());
 
@@ -145,7 +146,7 @@ class ProtocolServerMCPTest : public ::testing::Test {
 
 TEST_F(ProtocolServerMCPTest, Intialization) {
   llvm::StringLiteral request =
-      R"json({"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"claude-ai","version":"0.1.0"}},"jsonrpc":"2.0","id":0})json";
+      R"json({"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lldb-unit","version":"0.1.0"}},"jsonrpc":"2.0","id":0})json";
   llvm::StringLiteral response =
       R"json({"jsonrpc":"2.0","id":0,"result":{"capabilities":{"tools":{"listChanged":true}},"protocolVersion":"2024-11-05","serverInfo":{"name":"lldb-mcp","version":"0.1.0"}}})json";
 
@@ -167,7 +168,7 @@ TEST_F(ProtocolServerMCPTest, ToolsList) {
   llvm::StringLiteral request =
       R"json({"method":"tools/list","params":{},"jsonrpc":"2.0","id":1})json";
   llvm::StringLiteral response =
-      R"json({"id":1,"jsonrpc":"2.0","result":{"tools":[{"description":"test tool","name":"test"},{"description":"Run an lldb command.","inputSchema":{"properties":{"arguments":{"type":"string"}},"type":"object"},"name":"lldb_command"}]}})json";
+      R"json( {"id":1,"jsonrpc":"2.0","result":{"tools":[{"description":"test tool","inputSchema":{"type":"object"},"name":"test"},{"description":"List debugger instances with their debugger_id.","inputSchema":{"type":"object"},"name":"lldb_debugger_list"},{"description":"Run an lldb command.","inputSchema":{"properties":{"arguments":{"type":"string"},"debugger_id":{"type":"number"}},"required":["debugger_id"],"type":"object"},"name":"lldb_command"}]}})json";
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
 
@@ -205,7 +206,7 @@ TEST_F(ProtocolServerMCPTest, ResourcesList) {
 
 TEST_F(ProtocolServerMCPTest, ToolsCall) {
   llvm::StringLiteral request =
-      R"json({"method":"tools/call","params":{"name":"test","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+      R"json({"method":"tools/call","params":{"name":"test","arguments":{"arguments":"foo","debugger_id":0}},"jsonrpc":"2.0","id":11})json";
   llvm::StringLiteral response =
       R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"foo","type":"text"}],"isError":false}})json";
 
@@ -227,7 +228,7 @@ TEST_F(ProtocolServerMCPTest, ToolsCallError) {
   m_server_up->AddTool(std::make_unique<ErrorTool>("error", "error tool"));
 
   llvm::StringLiteral request =
-      R"json({"method":"tools/call","params":{"name":"error","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+      R"json({"method":"tools/call","params":{"name":"error","arguments":{"arguments":"foo","debugger_id":0}},"jsonrpc":"2.0","id":11})json";
   llvm::StringLiteral response =
       R"json({"error":{"code":-1,"message":"error"},"id":11,"jsonrpc":"2.0"})json";
 
@@ -249,7 +250,7 @@ TEST_F(ProtocolServerMCPTest, ToolsCallFail) {
   m_server_up->AddTool(std::make_unique<FailTool>("fail", "fail tool"));
 
   llvm::StringLiteral request =
-      R"json({"method":"tools/call","params":{"name":"fail","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+      R"json({"method":"tools/call","params":{"name":"fail","arguments":{"arguments":"foo","debugger_id":0}},"jsonrpc":"2.0","id":11})json";
   llvm::StringLiteral response =
       R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"failed","type":"text"}],"isError":true}})json";
 
diff --git a/lldb/unittests/Protocol/ProtocolMCPTest.cpp b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
index 00959f3ce20b..14cc240dd362 100644
--- a/lldb/unittests/Protocol/ProtocolMCPTest.cpp
+++ b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
@@ -133,3 +133,100 @@ TEST(ProtocolMCPTest, ToolDefinition) {
   EXPECT_EQ(tool_definition.inputSchema,
             deserialized_tool_definition->inputSchema);
 }
+
+TEST(ProtocolMCPTest, MessageWithRequest) {
+  Request request;
+  request.id = 1;
+  request.method = "test_method";
+  request.params = llvm::json::Object{{"param", "value"}};
+
+  Message message = request;
+
+  llvm::Expected<Message> deserialized_message = roundtripJSON(message);
+  ASSERT_THAT_EXPECTED(deserialized_message, llvm::Succeeded());
+
+  ASSERT_TRUE(std::holds_alternative<Request>(*deserialized_message));
+  const Request &deserialized_request =
+      std::get<Request>(*deserialized_message);
+
+  EXPECT_EQ(request.id, deserialized_request.id);
+  EXPECT_EQ(request.method, deserialized_request.method);
+  EXPECT_EQ(request.params, deserialized_request.params);
+}
+
+TEST(ProtocolMCPTest, MessageWithResponse) {
+  Response response;
+  response.id = 2;
+  response.result = llvm::json::Object{{"result", "success"}};
+
+  Message message = response;
+
+  llvm::Expected<Message> deserialized_message = roundtripJSON(message);
+  ASSERT_THAT_EXPECTED(deserialized_message, llvm::Succeeded());
+
+  ASSERT_TRUE(std::holds_alternative<Response>(*deserialized_message));
+  const Response &deserialized_response =
+      std::get<Response>(*deserialized_message);
+
+  EXPECT_EQ(response.id, deserialized_response.id);
+  EXPECT_EQ(response.result, deserialized_response.result);
+}
+
+TEST(ProtocolMCPTest, MessageWithNotification) {
+  Notification notification;
+  notification.method = "notification_method";
+  notification.params = llvm::json::Object{{"notify", "data"}};
+
+  Message message = notification;
+
+  llvm::Expected<Message> deserialized_message = roundtripJSON(message);
+  ASSERT_THAT_EXPECTED(deserialized_message, llvm::Succeeded());
+
+  ASSERT_TRUE(std::holds_alternative<Notification>(*deserialized_message));
+  const Notification &deserialized_notification =
+      std::get<Notification>(*deserialized_message);
+
+  EXPECT_EQ(notification.method, deserialized_notification.method);
+  EXPECT_EQ(notification.params, deserialized_notification.params);
+}
+
+TEST(ProtocolMCPTest, MessageWithError) {
+  ErrorInfo error_info;
+  error_info.code = -32603;
+  error_info.message = "Internal error";
+
+  Error error;
+  error.id = 3;
+  error.error = error_info;
+
+  Message message = error;
+
+  llvm::Expected<Message> deserialized_message = roundtripJSON(message);
+  ASSERT_THAT_EXPECTED(deserialized_message, llvm::Succeeded());
+
+  ASSERT_TRUE(std::holds_alternative<Error>(*deserialized_message));
+  const Error &deserialized_error = std::get<Error>(*deserialized_message);
+
+  EXPECT_EQ(error.id, deserialized_error.id);
+  EXPECT_EQ(error.error.code, deserialized_error.error.code);
+  EXPECT_EQ(error.error.message, deserialized_error.error.message);
+}
+
+TEST(ProtocolMCPTest, ResponseWithError) {
+  ErrorInfo error_info;
+  error_info.code = -32700;
+  error_info.message = "Parse error";
+
+  Response response;
+  response.id = 4;
+  response.error = error_info;
+
+  llvm::Expected<Response> deserialized_response = roundtripJSON(response);
+  ASSERT_THAT_EXPECTED(deserialized_response, llvm::Succeeded());
+
+  EXPECT_EQ(response.id, deserialized_response->id);
+  EXPECT_FALSE(deserialized_response->result.has_value());
+  ASSERT_TRUE(deserialized_response->error.has_value());
+  EXPECT_EQ(response.error->code, deserialized_response->error->code);
+  EXPECT_EQ(response.error->message, deserialized_response->error->message);
+}
diff --git a/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt b/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
index cf8702209a7c..eb2e00adba64 100644
--- a/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
+++ b/lldb/unittests/SymbolFile/DWARF/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_unittest(SymbolFileDWARFTests
   DWARFUnitTest.cpp
   SymbolFileDWARFTests.cpp
   XcodeSDKModuleTests.cpp
+  DWARF64UnitTest.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp b/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp
new file mode 100644
index 000000000000..2303f68674e4
--- /dev/null
+++ b/lldb/unittests/SymbolFile/DWARF/DWARF64UnitTest.cpp
@@ -0,0 +1,86 @@
+//===-- DWARF64UnitTest.cpp------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/SymbolFile/DWARF/DWARFDebugInfo.h"
+#include "Plugins/SymbolFile/DWARF/DWARFUnit.h"
+#include "TestingSupport/Symbol/YAMLModuleTester.h"
+
+using namespace lldb_private;
+using namespace lldb_private::dwarf;
+using namespace lldb_private::plugin::dwarf;
+
+TEST(DWARF64UnitTest, DWARF64DebugInfoAndCU) {
+  const char *yamldata = R"(
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_PPC64
+DWARF:
+  debug_str:
+    - 'clang version 18.1.8 (clang-18.1.8-1)'
+    - 'main'
+  debug_abbrev:
+    - Table:
+        - Code:            0x1
+          Tag:             DW_TAG_compile_unit
+          Children:        DW_CHILDREN_yes
+          Attributes:
+            - Attribute:       DW_AT_producer
+              Form:            DW_FORM_strp
+            - Attribute:       DW_AT_language
+              Form:            DW_FORM_data2
+            - Attribute:       DW_AT_stmt_list 
+              Form:            DW_FORM_sec_offset 
+        - Code:            0x02
+          Tag:             DW_TAG_subprogram 
+          Children:        DW_CHILDREN_no
+          Attributes:
+            - Attribute:       DW_AT_name 
+              Form:            DW_FORM_strp
+  debug_info:
+    - Format:          DWARF64
+      Version:         4
+      AbbrOffset:      0x0
+      AddrSize:        8
+      Entries:
+        - AbbrCode:        0x1
+          Values:
+            - Value:           0x0
+            - Value:           0x04
+            - Value:           0x0
+        - AbbrCode:        0x2
+          Values:
+            - Value:           0x1
+        - AbbrCode:	   0x0
+)";
+
+  YAMLModuleTester t(yamldata);
+  auto *symbol_file =
+      llvm::cast<SymbolFileDWARF>(t.GetModule()->GetSymbolFile());
+  DWARFUnit *unit = symbol_file->DebugInfo().GetUnitAtIndex(0);
+  ASSERT_TRUE(unit);
+  ASSERT_EQ(unit->GetFormParams().Format, DwarfFormat::DWARF64);
+  ASSERT_EQ(unit->GetVersion(), 4);
+  ASSERT_EQ(unit->GetAddressByteSize(), 8);
+
+  DWARFFormValue form_value;
+  const DWARFDebugInfoEntry *cu_entry = unit->DIE().GetDIE();
+  ASSERT_EQ(cu_entry->Tag(), DW_TAG_compile_unit);
+  ASSERT_EQ(unit->GetProducer(), eProducerClang);
+  ASSERT_EQ(unit->GetDWARFLanguageType(), DW_LANG_C_plus_plus);
+  auto attrs = cu_entry->GetAttributes(unit, DWARFDebugInfoEntry::Recurse::yes);
+  attrs.ExtractFormValueAtIndex(2, form_value); // Validate DW_AT_stmt_list
+  ASSERT_EQ(form_value.Unsigned(), 0UL);
+
+  DWARFDIE cu_die(unit, cu_entry);
+  auto declaration = cu_die.GetFirstChild();
+  ASSERT_TRUE(declaration.IsValid());
+  ASSERT_EQ(declaration.Tag(), DW_TAG_subprogram);
+}
diff --git a/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h b/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
index 50d5d4117c89..4d86e7962201 100644
--- a/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
+++ b/lldb/unittests/TestingSupport/Host/PipeTestUtilities.h
@@ -14,14 +14,14 @@
 #include "gtest/gtest.h"
 
 /// A base class for tests that need a pair of pipes for communication.
-class PipeTest : public testing::Test {
+class PipePairTest : public testing::Test {
 protected:
   lldb_private::Pipe input;
   lldb_private::Pipe output;
 
   void SetUp() override {
-    ASSERT_THAT_ERROR(input.CreateNew(false).ToError(), llvm::Succeeded());
-    ASSERT_THAT_ERROR(output.CreateNew(false).ToError(), llvm::Succeeded());
+    ASSERT_THAT_ERROR(input.CreateNew().ToError(), llvm::Succeeded());
+    ASSERT_THAT_ERROR(output.CreateNew().ToError(), llvm::Succeeded());
   }
 };
 
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 41e2f685a5da..27c139e0dfe2 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -110,7 +110,7 @@ endif()
 # This allows an easy way of setting up a build directory for llvm and another
 # one for llvm+clang+... using the same sources.
 # These projects will be included when "all" is included in LLVM_ENABLE_PROJECTS.
-set(LLVM_ALL_PROJECTS "bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libclc;lld;lldb;mlir;openmp;polly;pstl")
+set(LLVM_ALL_PROJECTS "bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libclc;lld;lldb;mlir;openmp;polly")
 if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
   # Disallow 'openmp' as a LLVM PROJECT on AIX as the supported way is to use
   # LLVM_ENABLE_RUNTIMES.
@@ -131,6 +131,14 @@ set(LLVM_ENABLE_PROJECTS "" CACHE STRING
 if( LLVM_ENABLE_PROJECTS STREQUAL "all" )
   set( LLVM_ENABLE_PROJECTS ${LLVM_ALL_PROJECTS})
 endif()
+
+# The pstl project was removed. Ignore it to pacify certain buildbots.
+list(FIND LLVM_ENABLE_PROJECTS "pstl" FOUND_INDEX)
+if(NOT "${FOUND_INDEX}" STREQUAL "-1")
+  message(WARNING "The pstl project has been removed and will not be built")
+  list(REMOVE_ITEM LLVM_ENABLE_PROJECTS "pstl")
+endif()
+
 foreach(proj ${LLVM_ENABLE_PROJECTS})
   if (NOT proj STREQUAL "llvm" AND NOT "${proj}" IN_LIST LLVM_KNOWN_PROJECTS)
      MESSAGE(FATAL_ERROR "${proj} isn't a known project: ${LLVM_KNOWN_PROJECTS}. Did you mean to enable it as a runtime in LLVM_ENABLE_RUNTIMES?")
@@ -142,7 +150,7 @@ endforeach()
 # As we migrate runtimes to using the bootstrapping build, the set of default runtimes
 # should grow as we remove those runtimes from LLVM_ENABLE_PROJECTS above.
 set(LLVM_DEFAULT_RUNTIMES "libcxx;libcxxabi;libunwind")
-set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc")
+set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all")
@@ -1058,6 +1066,13 @@ else()
 endif()
 option(LLVM_ENABLE_PLUGINS "Enable plugin support" ${LLVM_ENABLE_PLUGINS_default})
 
+set(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS OFF)
+if (LLVM_BUILD_LLVM_DYLIB OR LLVM_BUILD_SHARED_LIBS OR LLVM_ENABLE_PLUGINS)
+  # Export annotations for LLVM must be enabled if building as a shared lib or
+  # enabling plugins.
+  set(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS ON)
+endif()
+
 set(LLVM_ENABLE_NEW_PASS_MANAGER TRUE CACHE BOOL
   "Enable the new pass manager by default.")
 if(NOT LLVM_ENABLE_NEW_PASS_MANAGER)
diff --git a/llvm/Maintainers.md b/llvm/Maintainers.md
index 7bf722d5f18a..1ae6155119c8 100644
--- a/llvm/Maintainers.md
+++ b/llvm/Maintainers.md
@@ -263,8 +263,6 @@ amaclean@nvidia.com (email), [AlexMaclean](https://github.com/AlexMaclean) (GitH
 
 #### PowerPC backend
 
-Zheng Chen \
-czhengsz@cn.ibm.com (email), [chenzheng1030](https://github.com/chenzheng1030) (GitHub)
 
 #### RISCV backend
 
@@ -522,7 +520,8 @@ Hans Wennborg (hans@chromium.org, [zmodem](https://github.com/zmodem)) -- Releas
 Kostya Serebryany ([kcc](https://github.com/kcc)) -- Sanitizers \
 Michael Spencer (bigcheesegs@gmail.com), [Bigcheese](https://github.com/Bigcheese)) -- Windows support in object tools \
 Alexei Starovoitov (alexei.starovoitov@gmail.com, [4ast](https://github.com/4ast)) -- BPF backend \
-Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers
+Evgeniy Stepanov ([eugenis](https://github.com/eugenis)) -- Sanitizers \
+Zheng Chen (czhengsz@cn.ibm.com, [chenzheng1030](https://github.com/chenzheng1030)) -- PowerPC backend
 
 ### Former maintainers of removed components
 
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 8d8a94d1cddc..83772ed8d2b1 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -137,7 +137,7 @@ function(add_llvm_symbol_exports target_name export_file)
       set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                    LINK_FLAGS "  -Wl,--version-script,\"${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}\"")
     endif()
-  elseif(WIN32)
+  elseif(WIN32 OR CYGWIN)
     set(native_export_file "${target_name}.def")
 
     add_custom_command(OUTPUT ${native_export_file}
@@ -153,7 +153,7 @@ function(add_llvm_symbol_exports target_name export_file)
     elseif(CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC")
       # clang in msvc mode, calling a link.exe/lld-link style linker
       set(export_file_linker_flag "-Wl,/DEF:${export_file_linker_flag}")
-    elseif(MINGW)
+    elseif(MINGW OR CYGWIN)
       # ${export_file_linker_flag}, which is the plain file name, works as is
       # when passed to the compiler driver, which then passes it on to the
       # linker as an input file.
@@ -666,7 +666,7 @@ function(llvm_add_library name)
     # When building shared objects for each target there are some internal APIs
     # that are used across shared objects which we can't hide.
     if (LLVM_BUILD_LLVM_DYLIB_VIS AND NOT BUILD_SHARED_LIBS AND NOT APPLE AND
-        (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
+        (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
         NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
         NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
 
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 96ed6242524e..7cd67c02b2b9 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -465,7 +465,7 @@ if( LLVM_ENABLE_PIC )
 endif()
 
 if((NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX")) AND
-   (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
+   (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
   # GCC for MinGW does nothing about -fvisibility-inlines-hidden, but warns
   # about use of the attributes. As long as we don't use the attributes (to
   # override the default) we shouldn't set the command line options either.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index ef6285381688..3c2ac85131d0 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1452,6 +1452,7 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    It is preferred over llvm.amdgcn.mov.dpp.`<type>` for future use.
                                                    `llvm.amdgcn.update.dpp.<type> <old> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>`
                                                    Should be equivalent to:
+
                                                    - `v_mov_b32 <dest> <old>`
                                                    - `v_mov_b32 <dest> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>`
 
@@ -5992,7 +5993,7 @@ GFX6-GFX8
   available in dispatch packet. For M0, it is also possible to use maximum
   possible value of LDS for given target (0x7FFF for GFX6 and 0xFFFF for
   GFX7-GFX8).
-GFX9-GFX11
+GFX9 and later
   The M0 register is not used for range checking LDS accesses and so does not
   need to be initialized in the prolog.
 
@@ -16599,25 +16600,25 @@ scratch address space.
 
 On entry to a function:
 
-1.  SGPR0-3 contain a V# with the following properties (see
+#.  SGPR0-3 contain a V# with the following properties (see
     :ref:`amdgpu-amdhsa-kernel-prolog-private-segment-buffer`):
 
     * Base address pointing to the beginning of the wavefront scratch backing
       memory.
     * Swizzled with dword element size and stride of wavefront size elements.
 
-2.  The FLAT_SCRATCH register pair is setup. See
+#.  The FLAT_SCRATCH register pair is setup. See
     :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`.
-3.  GFX6-GFX8: M0 register set to the size of LDS in bytes. See
+#.  GFX6-GFX8: M0 register set to the size of LDS in bytes. See
     :ref:`amdgpu-amdhsa-kernel-prolog-m0`.
-4.  The EXEC register is set to the lanes active on entry to the function.
-5.  MODE register: *TBD*
-6.  VGPR0-31 and SGPR4-29 are used to pass function input arguments as described
+#.  The EXEC register is set to the lanes active on entry to the function.
+#.  MODE register: *TBD*
+#.  VGPR0-31 and SGPR4-29 are used to pass function input arguments as described
     below.
-7.  SGPR30-31 return address (RA). The code address that the function must
+#.  SGPR30-31 return address (RA). The code address that the function must
     return to when it completes. The value is undefined if the function is *no
     return*.
-8.  SGPR32 is used for the stack pointer (SP). It is an unswizzled scratch
+#.  SGPR32 is used for the stack pointer (SP). It is an unswizzled scratch
     offset relative to the beginning of the wavefront scratch backing memory.
 
     The unswizzled SP can be used with buffer instructions as an unswizzled SGPR
@@ -16654,19 +16655,19 @@ On entry to a function:
     arguments after the last local allocation and adjust SGPR32 to the address
     after the last local allocation.
 
-9.  All other registers are unspecified.
-10. Any necessary ``s_waitcnt`` has been performed to ensure memory is available
-    to the function.
-11. Use pass-by-reference (byref) in stead of pass-by-value (byval) for struct
-    arguments in C ABI. Callee is responsible for allocating stack memory and
-    copying the value of the struct if modified. Note that the backend still
-    supports byval for struct arguments.
+#. All other registers are unspecified.
+#. Any necessary ``s_waitcnt`` has been performed to ensure memory is available
+   to the function.
+#. Use pass-by-reference (byref) in stead of pass-by-value (byval) for struct
+   arguments in C ABI. Callee is responsible for allocating stack memory and
+   copying the value of the struct if modified. Note that the backend still
+   supports byval for struct arguments.
 
 On exit from a function:
 
-1.  VGPR0-31 and SGPR4-29 are used to pass function result arguments as
+#.  VGPR0-31 and SGPR4-29 are used to pass function result arguments as
     described below. Any registers used are considered clobbered registers.
-2.  The following registers are preserved and have the same value as on entry:
+#.  The following registers are preserved and have the same value as on entry:
 
     * FLAT_SCRATCH
     * EXEC
@@ -16701,10 +16702,10 @@ On exit from a function:
       preserved if it can be determined that the called function does not change
       their value.
 
-2.  The PC is set to the RA provided on entry.
-3.  MODE register: *TBD*.
-4.  All other registers are clobbered.
-5.  Any necessary ``s_waitcnt`` has been performed to ensure memory accessed by
+#.  The PC is set to the RA provided on entry.
+#.  MODE register: *TBD*.
+#.  All other registers are clobbered.
+#.  Any necessary ``s_waitcnt`` has been performed to ensure memory accessed by
     function is available to the caller.
 
 .. TODO::
diff --git a/llvm/docs/AliasAnalysis.rst b/llvm/docs/AliasAnalysis.rst
index 7afe0e277bd4..1830cca91504 100644
--- a/llvm/docs/AliasAnalysis.rst
+++ b/llvm/docs/AliasAnalysis.rst
@@ -256,20 +256,6 @@ analysis run method (``run`` for a ``Pass``, ``runOnFunction`` for a
     return false;
   }
 
-Required methods to override
-----------------------------
-
-You must override the ``getAdjustedAnalysisPointer`` method on all subclasses
-of ``AliasAnalysis``. An example implementation of this method would look like:
-
-.. code-block:: c++
-
-  void *getAdjustedAnalysisPointer(const void* ID) override {
-    if (ID == &AliasAnalysis::ID)
-      return (AliasAnalysis*)this;
-    return this;
-  }
-
 Interfaces which may be specified
 ---------------------------------
 
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 72f19fd35392..2f03421c9223 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -588,7 +588,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
   The full list is:
 
-  ``bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libc;libclc;lld;lldb;mlir;openmp;polly;pstl``
+  ``bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libc;libclc;lld;lldb;mlir;openmp;polly``
 
   .. note::
     Some projects listed here can also go in ``LLVM_ENABLE_RUNTIMES``. They
@@ -610,7 +610,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
   The full list is:
 
-  ``libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;openmp;llvm-libgcc;offload``
+  ``libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload``
 
   To enable all of them, use:
 
@@ -710,7 +710,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   For example, ar will be symlinked to llvm-ar.
 
 **LLVM_INSTALL_CCTOOLS_SYMLINKS**:BOOL
-  Install symliks from the cctools tool names to the corresponding LLVM tools.
+  Install symlinks from the cctools tool names to the corresponding LLVM tools.
   For example, lipo will be symlinked to llvm-lipo.
 
 **LLVM_INSTALL_OCAMLDOC_HTML_DIR**:STRING
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index ad772f69127a..c614a6d7ace9 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -591,6 +591,8 @@ rather than C-style casts. There are two exceptions to this:
 
 * When casting to ``void`` to suppress warnings about unused variables (as an
   alternative to ``[[maybe_unused]]``). Prefer C-style casts in this instance.
+  Note that if the variable is unused because it's used only in ``assert``, use
+  ``[[maybe_unused]]`` instead of a C-style void cast.
 
 * When casting between integral types (including enums that are not strongly-
   typed), functional-style casts are permitted as an alternative to
@@ -682,7 +684,7 @@ something notionally equivalent. Examples:
   };
 
   // The Foo constructor call is reading a file, don't use braces to call it.
-  std::fill(foo.begin(), foo.end(), Foo("name"));
+  llvm::fill(foo, Foo("name"));
 
   // The pair is being constructed like an aggregate, use braces.
   bar_map.insert({my_key, my_value});
@@ -1288,17 +1290,26 @@ These are two interesting different cases. In the first case, the call to
 ``V.size()`` is only useful for the assert, and we don't want it executed when
 assertions are disabled.  Code like this should move the call into the assert
 itself.  In the second case, the side effects of the call must happen whether
-the assert is enabled or not.  In this case, the value should be cast to void to
-disable the warning.  To be specific, it is preferred to write the code like
-this:
+the assert is enabled or not. In this case, the value should be defined using
+the ``[[maybe_unused]]`` attribute to suppress the warning. To be specific, it is
+preferred to write the code like this:
 
 .. code-block:: c++
 
   assert(V.size() > 42 && "Vector smaller than it should be");
 
-  bool NewToSet = Myset.insert(Value); (void)NewToSet;
+  [[maybe_unused]] bool NewToSet = Myset.insert(Value);
   assert(NewToSet && "The value shouldn't be in the set yet");
 
+In C code where ``[[maybe_unused]]`` is not supported, use ``void`` cast to
+suppress unused variable warning as follows:
+
+.. code-block:: c
+
+    LLVMValueRef Value = LLVMMetadataAsValue(Context, NodeMD);
+    assert(LLVMIsAValueAsMetadata(Value) != NULL);
+    (void)Value;
+
 Do Not Use ``using namespace std``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index 9680e59b2d65..e28b23d12ce6 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -279,7 +279,7 @@ OPTIONS
   any analysis with a special representation (i.e. BlockFrequency,
   BranchProbability, etc) are printed as raw hex values.
 
-  Only works with PowerPC objects or X86 linked images.
+  Only supported for AArch64, BPF, PowerPC, and X86.
 
   Example:
     A non-symbolized branch instruction with a local target and pc-relative memory access like
diff --git a/llvm/docs/DeveloperPolicy.rst b/llvm/docs/DeveloperPolicy.rst
index 6b60812df83b..eb59c4953dc2 100644
--- a/llvm/docs/DeveloperPolicy.rst
+++ b/llvm/docs/DeveloperPolicy.rst
@@ -16,90 +16,88 @@ to eliminate miscommunication, rework, and confusion that might arise from the
 distributed nature of LLVM's development.  By stating the policy in clear terms,
 we hope each developer can know ahead of time what to expect when making LLVM
 contributions.  This policy covers all llvm.org subprojects, including Clang,
-LLDB, libc++, etc.
+LLDB, libc++, MLIR, etc.
 
-This policy is also designed to accomplish the following objectives:
+The developer policy supports the following LLVM project objectives:
 
-#. Attract both users and developers to the LLVM project.
+#. Attract both users and new contributors to the LLVM project.
 
-#. Make life as simple and easy for contributors as possible.
+#. Help people contribute to LLVM by documenting our development practices.
 
-#. Keep the top of tree as stable as possible.
+#. Maintain the stability, performance, and quality of the ``main`` branch.
 
-#. Establish awareness of the project's :ref:`copyright, license, and patent
-   policies <copyright-license-patents>` with contributors to the project.
-
-This policy is aimed at frequent contributors to LLVM. People interested in
-contributing one-off patches can do so in an informal way by sending them to the
-`llvm-commits mailing list
-<http://lists.llvm.org/mailman/listinfo/llvm-commits>`_ and engaging another
-developer to see it through the process.
+#. Establish the project's :ref:`copyright, license, and patent
+   policies <copyright-license-patents>` policies.
 
 Developer Policies
 ==================
 
-This section contains policies that pertain to frequent LLVM developers.  We
-always welcome `one-off patches`_ from people who do not routinely contribute to
-LLVM, but we expect more from frequent contributors to keep the system as
-efficient as possible for everyone.  Frequent LLVM contributors are expected to
-meet the following requirements in order for LLVM to maintain a high standard of
-quality.
-
-Stay Informed
--------------
-
-Developers should stay informed by reading the `LLVM Discourse forums`_ and subscribing
-to the categories of interest for notifications.
-
-Paying attention to changes being made by others is a good way to see what other people
-are interested in and watching the flow of the project as a whole.
+Communication Channels
+----------------------
 
-Contributions to the project are made through :ref:`GitHub Pull Requests <github-reviews>`.
-You can subscribe to notification for areas of the codebase by joining
+LLVM is a large project with many subcomponents, and it has a wide array of
+communication channels that you can use to keep track of recent developments,
+upcoming projects, new designs, enhancements, and other community business.
+
+First and foremost is the `LLVM Discourse forums`_, which is the successor
+to our former mailing lists (llvm-dev@, cfe-dev@, lldb-dev@, etc). This is
+probably the most vital and active communication channel to our highly
+distributed open source project. It enables long-form asynchronous text
+communication, and this is where people tend to propose major changes or
+propose new designs in the form of RFCs (Request For Comment), which are
+described later. Please be aware that the Discourse forums are public and
+archived, and that notices of confidentiality or non-disclosure cannot be
+respected.
+
+We accept code contributions as :ref:`GitHub Pull Requests <github-reviews>`.
+Our project is generally too large to subscribe to all github notifications, so
+if you want to be notified of pull requests affecting a specific parts of the
+code, you can join
 one of the `pr-subscribers-* <https://github.com/orgs/llvm/teams?query=pr-subscribers>`_
 GitHub teams. This `mapping <https://github.com/llvm/llvm-project/blob/main/.github/new-prs-labeler.yml>`_
-indicates which team is associated with a particular paths in the repository.
-
-You can also subscribe to the "commits" mailing list for a subproject you're interested in,
-such as `llvm-commits
-<http://lists.llvm.org/mailman/listinfo/llvm-commits>`_, `cfe-commits
-<http://lists.llvm.org/mailman/listinfo/cfe-commits>`_, or `lldb-commits
-<http://lists.llvm.org/mailman/listinfo/lldb-commits>`_.
-
-Missing features and bugs are tracked through our `GitHub issue tracker <https://github.com/llvm/llvm-project/issues>`_
-and assigned labels. We recommend that active developers monitor incoming issues.
-You can subscribe for notification for specific components by joining
-one of the `issue-subscribers-* <https://github.com/orgs/llvm/teams?query=issue-subscribers>`_
-teams.
-You may also subscribe to the `llvm-bugs
-<http://lists.llvm.org/mailman/listinfo/llvm-bugs>`_ email list to keep track
-of bugs and enhancements occurring in the entire project.  We really appreciate people
-who are proactive at catching incoming bugs in their components and dealing with them
-promptly.
-
-Please be aware that all public LLVM mailing lists and discourse forums are public and archived, and
-that notices of confidentiality or non-disclosure cannot be respected.
+documents the paths that trigger notifications for each of the listed teams.
+
+Missing features and bugs are tracked through our `GitHub issue tracker
+<https://github.com/llvm/llvm-project/issues>`_. You can subscribe for
+notification for specific components by joining one of the `issue-subscribers-*
+<https://github.com/orgs/llvm/teams?query=issue-subscribers>`_ teams. You may
+also subscribe to the `llvm-bugs
+<http://lists.llvm.org/mailman/listinfo/llvm-bugs>`_ email list to subscribe to
+the firehose of all issue notifications, which some community members use to
+perform custom filtering.
+
+Beyond the asynchronous written communication channels, LLVM has a Discord
+server for real-time chat communication, as well as a community calendar with
+many regular workgroup video calls and office hours. See :doc:`GettingInvolved`
+for more information on other ways to engage with the community.
 
 .. _patch:
-.. _one-off patches:
 
 Making and Submitting a Patch
 -----------------------------
 
-When making a patch for review, the goal is to make it as easy for the reviewer
-to read it as possible.  As such, we recommend that you:
+Patches are submitted to GitHub and reviewed using Pull Requests. Follow the
+:ref:`Getting Started Guide <sources>` to check out sources, make a patch, and
+then follow the :ref:`GitHub Pull Request <github-reviews>` guide to upload a
+pull request.
+
+Here are some tips to enable a successful code review:
 
-#. Make your patch against git main, not a branch, and not an old version
-   of LLVM.  This makes it easy to apply the patch.  For information on how to
-   clone from git, please see the :ref:`Getting Started Guide <sources>`.
+* :ref:`Include a test <include a testcase>`. This tends to be one of the first
+  things a reviewer will ask for and look at to understand what a new patch
+  does.
 
-#. Similarly, patches should be submitted soon after they are generated.  Old
-   patches may not apply correctly if the underlying code changes between the
-   time the patch was created and the time it is applied.
+* Identify 2-3 individuals to review the patch. Look through the relevant
+  :ref:`Maintainers` file or browse git blame for likely stakeholders for the
+  code you want to modify, and add ``@username`` to a PR comment to notify them
+  of your PR if you are unable to add reviewers yourself due to GitHub permissions.
 
-#. Once you have created your patch, create a
-   :ref:`GitHub Pull Request <github-reviews>` for
-   it (or commit it directly if applicable).
+* To avoid precommit CI failures due to merge conflicts, base your patches on a
+  recent commit from ``main``. If you want to make changes to a release branch,
+  land a change in ``main`` first and then follow the
+  :ref:`backporting instructions <backporting>`.
+
+See :doc:`CodeReview` for more info on what to expect.
 
 When submitting patches, please do not add confidentiality or non-disclosure
 notices to the patches themselves.  These notices conflict with the LLVM
@@ -115,19 +113,20 @@ GitHub platform about their past contributions. Primarily, our buildbot
 infrastructure uses emails to contact contributors about build and test
 failures.
 
-Therefore, the LLVM community requires contributors to have a public
-email address associated with their GitHub commits, so please ensure that "Keep
-my email addresses private" is disabled in your
-`account settings <https://github.com/settings/emails>`_.
+Therefore, the LLVM community requires contributors to have a public email
+address associated with their GitHub commits, so please ensure that "Keep my
+email addresses private" is disabled in your `account settings
+<https://github.com/settings/emails>`_. There are many free email forwarding
+services available if you wish to keep your identity private.
 
 .. _code review:
 
 Code Reviews
 ------------
 
-LLVM has a code-review policy. Code review is one way to increase the quality of
-software. Please see :doc:`CodeReview` for more information on LLVM's code-review
-process.
+LLVM uses code review, which is a generally accepted software engineering best
+practice for maintaining high code quality. Please see :doc:`CodeReview` for
+more information on LLVM's code review process.
 
 .. _maintainers:
 
@@ -147,17 +146,17 @@ Community members can find active and inactive maintainers for a project in the
 Maintainers are volunteering to take on the following shared responsibilities
 within an area of a project:
 
-    * ensure that commits receive high-quality review, either by the maintainer
-      or by someone else,
-    * help to confirm and comment on issues,
-    * mediate code review disagreements through collaboration with other
-      maintainers (and other reviewers) to come to a consensus on how best to
-      proceed with disputed changes,
-    * actively engage with relevant RFCs,
-    * aid release managers with backporting and other release-related
-      activities,
-    * be a point of contact for contributors who need help (answering questions
-      on Discord/Discourse or holding office hours).
+* ensure that commits receive high-quality review, either by the maintainer
+  or by someone else,
+* help to confirm and comment on issues,
+* mediate code review disagreements through collaboration with other
+  maintainers (and other reviewers) to come to a consensus on how best to
+  proceed with disputed changes,
+* actively engage with relevant RFCs,
+* aid release managers with backporting and other release-related
+  activities,
+* be a point of contact for contributors who need help (answering questions
+  on Discord/Discourse or holding office hours).
 
 Each top-level project in the monorepo will specify one or more
 lead maintainers who are responsible for ensuring community needs are
@@ -173,10 +172,10 @@ project should be discontinued.
 All contributors with commit access to the LLVM Project are eligible to be a
 maintainer. However, we are looking for people who can commit to:
 
-    * engaging in their responsibilities the majority of the days in a month,
-    * ensuring that they, and the community members they interact with, abide by
-      the LLVM Community Code of Conduct, and
-    * performing these duties for at least three months.
+* engaging in their responsibilities the majority of the days in a month,
+* ensuring that they, and the community members they interact with, abide by the
+  :ref:`LLVM Community Code of Conduct`, and
+* performing these duties for at least three months.
 
 We recognize that priorities shift, job changes happen, burnout is real,
 extended vacations are a blessing, and people's lives are generally complex.
@@ -220,27 +219,47 @@ Test Cases
 Developers are required to create test cases for any bugs fixed and any new
 features added.  Some tips for getting your testcase approved:
 
-* All feature and regression test cases are added to the ``llvm/test``
-  directory. The appropriate sub-directory should be selected (see the
+* All feature and regression test cases are added to the ``test`` subdirectory
+  of each LLVM subproject, i.e. ``llvm-project/llvm/test`` for LLVM itself. The
+  appropriate sub-directory should be selected (see the
   :doc:`Testing Guide <TestingGuide>` for details).
 
-* Test cases should be written in :doc:`LLVM assembly language <LangRef>`.
-
-* Test cases, especially for regressions, should be reduced as much as possible,
-  by :doc:`bugpoint <Bugpoint>` or manually. It is unacceptable to place an
-  entire failing program into ``llvm/test`` as this creates a *time-to-test*
-  burden on all developers. Please keep them short.
+* We prefer that functional changes are tested using ``FileCheck`` and the tool
+  that fits most closely with the code being modified. For example, ``opt`` is
+  used to test IR transformations, ``llc`` for backend changes, and ``clang``
+  for frontend changes. Some components have scripts for generating and
+  updating golden tests in the ``utils/`` subproject directory, i.e.
+  `mlir/utils/generate-test-checks.py <https://github.com/llvm/llvm-project/blob/main/mlir/utils/generate-test-checks.py>`_
+  and `llvm/utils/update_llc_test_checks.py <https://github.com/llvm/llvm-project/blob/main/llvm/utils/update_llc_test_checks.py>`_
+
+* Some subprojects such as ``clang`` and ``clangd`` have project specific
+  testing tools, like the ``clang -verify`` flag (`docs
+  <https://clang.llvm.org/docs/InternalsManual.html#verifying-diagnostics>`_)
+  and the ``clangd -lit-test``
+  flag, which are preferred over ``FileCheck``.
+
+* Changes to libraries, such as Support, which are not directly observable
+  through tool invocations, are often best tested with unit tests. Unit tests
+  are located under the ``unittests`` subdirectory of each subproject.
+
+* Test cases should be targeted. Large inputs exhibiting bugs should be reduced
+  with tools like ``llvm-reduce`` before committing them to the suite. It is not
+  acceptable to place an entire failing program into ``llvm/test`` as this
+  creates a *time-to-test* burden on all developers. Please keep them short.
 
 * Avoid adding links to resources that are not available to the entire
   community, such as links to private bug trackers, internal corporate
   documentation, etc. Instead, add sufficient comments to the test to provide
   the context behind such links.
 
-Note that llvm/test and clang/test are designed for regression and small feature
-tests only. More extensive test cases (e.g., entire applications, benchmarks,
-etc) should be added to the ``llvm-test`` test suite.  The llvm-test suite is
-for coverage (correctness, performance, etc) testing, not feature or regression
-testing.
+As a project, we prefer to separate tests into small in-tree tests, and large
+out-of-tree integration tests. More extensive integration test cases (e.g.,
+entire applications, benchmarks, etc) should be added to the `llvm-test-suite
+<https://github.com/llvm/llvm-test-suite>`_ repository.  The
+``llvm-test-suite`` repository is for integration and application testing
+(correctness, performance, etc) testing, not feature or regression testing. It
+also serves to separate out third party code that falls under a different
+license.
 
 Release Notes
 -------------
@@ -336,12 +355,12 @@ you follow these guidelines to help review, search in logs, email formatting
 and so on. These guidelines are very similar to rules used by other open source
 projects.
 
-Most importantly, the contents of the message should be carefully written to
-convey the rationale of the change (without delving too much in detail). It
-also should avoid being vague or overly specific. For example, "bits were not
-set right" will leave the reviewer wondering about which bits, and why they
-weren't right, while "Correctly set overflow bits in TargetInfo" conveys almost
-all there is to the change.
+Commit messages should communicate briefly what the change does, but they
+should really emphasize why a change is being made and provide useful context.
+Commit messages should be thoughtfully written and specific, rather than vague.
+For example, "bits were not set right" will leave the reviewer wondering about
+which bits, and why they weren't right, while "Correctly set overflow bits in
+TargetInfo" conveys almost all there is to the change.
 
 Below are some guidelines about the format of the message itself:
 
@@ -369,8 +388,6 @@ Below are some guidelines about the format of the message itself:
   or "[OpenMP] ...". This helps email filters and searches for post-commit
   reviews.
 
-* The body, if it exists, should be separated from the title by an empty line.
-
 * The body should be concise, but explanatory, including a complete
   reasoning.  Unless it is required to understand the change, examples,
   code snippets and gory details should be left to bug comments, web
@@ -382,7 +399,7 @@ Below are some guidelines about the format of the message itself:
 * If the commit is a bug fix on top of another recently committed patch, or a
   revert or reapply of a patch, include the git commit hash of the prior
   related commit. This could be as simple as "Revert commit NNNN because it
-  caused PR#".
+  caused issue #".
 
 * If the patch has been reviewed, add a link to its review page, as shown
   `here <https://www.llvm.org/docs/Phabricator.html#committing-a-change>`__.
@@ -397,9 +414,15 @@ Below are some guidelines about the format of the message itself:
   message self-explanatory. Note that such non-public links should not be
   included in the submitted code.
 
+LLVM uses a squash workflow for pull requests, so as the pull request evolves
+during review, it's important to update the pull request description over the
+course of a review. GitHub uses the initial commit message to create the pull
+request description, but it ignores all subsequent commit messages. Authors and
+reviewers should make a final editing pass over the squashed commit description when
+squashing and merging PRs.
+
 For minor violations of these recommendations, the community normally favors
-reminding the contributor of this policy over reverting. Minor corrections and
-omissions can be handled by sending a reply to the commits mailing list.
+reminding the contributor of this policy over reverting.
 
 .. _revert_policy:
 
@@ -512,11 +535,12 @@ someone with commit access commits on your behalf. When doing so, please
 provide the name and email address you would like to use in the Author
 property of the commit.
 
-For external tracking purposes, committed changes are automatically reflected
-on a commits mailing list soon after the commit lands (e.g. llvm-commits_).
-Note that these mailing lists are moderated, and it is not unusual for a large
-commit to require a moderator to approve the email, so do not be concerned if a
-commit does not immediately appear in the archives.
+For external tracking purposes, committed changes are automatically reflected on
+a commits mailing list soon after the commit lands (e.g.
+llvm-commits@lists.llvm.org). Note that these mailing lists are moderated, and
+it is not unusual for a large commit to require a moderator to approve the
+email, so do not be concerned if a commit does not immediately appear in the
+archives.
 
 If you have recently been granted commit access, these policies apply:
 
@@ -550,28 +574,69 @@ after they are committed, depending on the nature of the change).  You are
 encouraged to review other peoples' patches as well, but you aren't required
 to do so.
 
-.. _discuss the change/gather consensus:
+Obtaining Other Access or Permissions
+-------------------------------------
 
-Making a Major Change
----------------------
+To obtain access other than commit access, you can raise an issue like the one
+for obtaining commit access. However, instead of including PRs you have authored,
+include evidence of your need for the type of access you want.
 
-When a developer begins a major new project with the aim of contributing it back
-to LLVM, they should inform the community with a post to the `LLVM Discourse forums`_, to the extent
-possible. The reason for this is to:
+For example, if you are helping to triage issues and want the ability to add
+labels, include links to issues you have triaged previously and explain how
+having this ability would help that work.
 
-#. keep the community informed about future changes to LLVM,
+.. _discuss the change/gather consensus:
 
-#. avoid duplication of effort by preventing multiple parties working on the
-   same thing and not knowing about it, and
+Proposing Major Changes (RFCs)
+------------------------------
 
-#. ensure that any technical issues around the proposed work are discussed and
-   resolved before any significant work is done.
+LLVM is a large community with many stakeholders, and before landing any major
+change, it is important to discuss the design of a change publicly with the
+community. This is done by posting an Request For Comments (RFC) on the `LLVM
+Discourse forums`_.
 
 The design of LLVM is carefully controlled to ensure that all the pieces fit
 together well and are as consistent as possible. If you plan to make a major
 change to the way LLVM works or want to add a major new extension, it is a good
-idea to get consensus with the development community before you start working on
-it.
+idea to get consensus with the development community before you invest
+significant effort in an implementation. Prototype implementations, however, can
+often be helpful in making design discussions more concrete by demonstrating
+what is possible.
+
+These are some suggestions for how to get a major change accepted:
+
+* Make it targeted, and avoid touching components irrelevant to the task.
+
+* Explain how the change improves LLVM for other stakeholders rather than
+  focusing on your specific use case.
+
+* As discussion evolves, periodically summarize the current state of the
+  discussion and clearly separate points where consensus seems to emerge from
+  those where further discussion is necessary.
+
+* Compilers are foundational infrastructure, so there is a high quality bar,
+  and the burden of proof is on the proposer. If reviewers repeatedly ask for
+  an unreasonable amount of evidence or data, proposal authors can escalate to
+  the area team to resolve disagreements.
+
+After posting a major proposal, it is common to receive lots of conflicting
+feedback from different parties, or no feedback at all, leaving authors without
+clear next steps. As a community, we are aiming for `"rough consensus"
+<https://en.wikipedia.org/wiki/Rough_consensus>`_, similar in spirit to what is
+described in `IETF RFC7282 <https://datatracker.ietf.org/doc/html/rfc7282>`_.
+This requires considering and addressing all of the objections to the RFC, and
+confirming that we can all live with the tradeoffs embodied in the proposal.
+
+The LLVM Area Teams (defined in `LP0004
+<https://github.com/llvm/llvm-www/blob/main/proposals/LP0004-project-governance.md>`_)
+are responsible for facilitating project decision making. In cases were there
+isn't obvious agreement, area teams should step in to restate their perceived
+consensus. In cases of deeper disagreement, area teams should try to identify
+the next steps for the proposal, such as gathering more data, changing the
+proposal, or rejecting it outright. They can also act as moderators by
+scheduling calls for participants to speak directly to resolve disagreements,
+subject to normal :ref:`Code of Conduct <LLVM Community Code of Conduct>`
+guidelines.
 
 Once the design of the new feature is finalized, the work itself should be done
 as a series of `incremental changes`_, not as a long-term development branch.
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index ea267842cdc3..f92eac3209c2 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -430,27 +430,6 @@ Example:
    .uleb128  .LBB_END0_1-.LBB0_1          # BB_1 size
    .byte     y                            # BB_1 metadata
 
-Version 0: basic block address offsets are computed relative to the function
-address. This uses the unversioned ``SHT_LLVM_BB_ADDR_MAP_V0`` section type and
-is semantically equivalent to using ``SHT_LLVM_BB_ADDR_MAP`` with a zero
-version field.
-
-Example:
-
-.. code-block:: gas
-
-  .section  ".llvm_bb_addr_map","",@llvm_bb_addr_map_v0
-  .quad     .Lfunc_begin0                 # address of the function
-  .byte     2                             # number of basic blocks
-  # BB record for BB_0
-   .uleb128  .Lfunc_beign0-.Lfunc_begin0  # BB_0 offset relative to the function entry (always zero)
-   .uleb128  .LBB_END0_0-.Lfunc_begin0    # BB_0 size
-   .byte     x                            # BB_0 metadata
-  # BB record for BB_1
-   .uleb128  .LBB0_1-.Lfunc_begin0        # BB_1 offset relative to the function entry
-   .uleb128  .LBB_END0_1-.LBB0_1          # BB_1 size
-   .byte     y                            # BB_1 metadata
-
 PGO Analysis Map
 """"""""""""""""
 
diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 31dee0bbb17b..f43792be6f93 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -537,6 +537,8 @@ See more in-depth information about how to contribute in the following documenta
 Releases
 ========
 
+.. _backporting:
+
 Backporting Fixes to the Release Branches
 -----------------------------------------
 You can use special comments on issues or pull requests to make backport
diff --git a/llvm/docs/HowToAddABuilder.rst b/llvm/docs/HowToAddABuilder.rst
index 32a00ed43d3c..bbc87e5841dd 100644
--- a/llvm/docs/HowToAddABuilder.rst
+++ b/llvm/docs/HowToAddABuilder.rst
@@ -28,7 +28,7 @@ There are two buildmasters running.
   commits from the llvm-zorg repository.
 
 In order to remain connected to the main buildmaster (and thus notify
-developers of failures), a builbot must:
+developers of failures), a buildbot must:
 
 * Be building a supported configuration.  Builders for experimental backends
   should generally be attached to staging buildmaster.
@@ -74,9 +74,9 @@ Here are the steps you can follow to do so:
    of parallelism (-j param) would give the fastest build.  You can build
    multiple configurations on one computer.
 
-#. Install buildbot-worker (currently we are using buildbot version 2.8.4).
+#. Install buildbot-worker (currently we are using buildbot version 3.11.7).
    This specific version can be installed using ``pip``, with a command such
-   as ``pip3 install buildbot-worker==2.8.4``.
+   as ``pip3 install buildbot-worker==3.11.7``.
 
 #. Create a designated user account, your buildbot-worker will be running under,
    and set appropriate permissions.
@@ -414,4 +414,3 @@ Some tasks don't give immediate feedback, so if nothing happens within a short
 time, try again with the browser's web console open. Sometimes you will see
 403 errors and other messages that might indicate you don't have the correct
 details set up.
-
diff --git a/llvm/docs/HowToReleaseLLVM.rst b/llvm/docs/HowToReleaseLLVM.rst
index ca55ee573c00..dd4bb08b81d1 100644
--- a/llvm/docs/HowToReleaseLLVM.rst
+++ b/llvm/docs/HowToReleaseLLVM.rst
@@ -38,8 +38,8 @@ Releases should be tagged on Tuesdays.
 =============================== =========================
 Release                         Approx. Date
 =============================== =========================
-*release branch: even releases* *4th Tue in January*
-*release branch: odd releases*  *4th Tue in July*
+*release branch: even releases* *2nd Tue in January*
+*release branch: odd releases*  *2nd Tue in July*
 X.1.0-rc1                       3 days after branch.
 X.1.0-rc2                       2 weeks after branch.
 X.1.0-rc3                       4 weeks after branch
@@ -49,7 +49,11 @@ X.1.0-rc3                       4 weeks after branch
 **X.1.3**                       **12 weeks after branch**
 **X.1.4**                       **14 weeks after branch**
 **X.1.5**                       **16 weeks after branch**
-**X.1.6 (if necessary)**        **18 weeks after branch**
+**X.1.6**                       **18 weeks after branch**
+**X.1.7**                       **20 weeks after branch**
+**X.1.8**                       **22 weeks after branch**
+**X.1.9** (If necessary)        **24 weeks after branch**
+**Next release branches**       **~25 weeks after branch**
 =============================== =========================
 
 Release Process Summary
@@ -341,10 +345,10 @@ Below are the rules regarding patching the release branch:
    was created.  As with all phases, release managers and maintainers can reject
    patches that are deemed too invasive.
 
-#. *Before RC2* Patches should be limited to bug fixes or backend specific
+#. *Before RC2/RC3* Patches should be limited to bug fixes or backend specific
    improvements that are determined to be very safe.
 
-#. *Before RC3/Final Major Release* Patches should be limited to critical
+#. *Before Final Major Release* Patches should be limited to critical
    bugs or regressions.
 
 #. *Bug fix releases* Patches should be limited to bug fixes or very safe
diff --git a/llvm/docs/HowToUpdateDebugInfo.rst b/llvm/docs/HowToUpdateDebugInfo.rst
index a87efe7e6e43..c3262a96b62e 100644
--- a/llvm/docs/HowToUpdateDebugInfo.rst
+++ b/llvm/docs/HowToUpdateDebugInfo.rst
@@ -420,7 +420,41 @@ tests. Changes to this pass are not allowed to break existing tests.
    check lines. In cases where this can't be avoided (say, if a test wouldn't
    be precise enough), moving the test to its own file is preferred.
 
-.. _MIRDebugify:
+Using Coverage Tracking to remove false positives
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As described :ref:`above<WhenToDropLocation>`, there are valid reasons for
+instructions to not have source locations. Therefore, when detecting dropped or
+not-generated source locations, it may be preferable to avoid detecting cases
+where the missing source location is intentional. For this, you can use the
+"coverage tracking" feature in LLVM to prevent these from appearing in the
+``debugify`` output. This is enabled in a build of LLVM by setting the CMake
+flag ``-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=COVERAGE``. When this has been
+set, LLVM will enable runtime tracking of
+:ref:`DebugLoc annotations<NewInstLocations>`, allowing ``debugify`` to ignore
+instructions that have an explicitly recorded reason given for not having a
+source location.
+
+For triaging source location bugs detected with ``debugify``, you may find it
+helpful to instead set the CMake flag to enable "origin tracking",
+``-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=COVERAGE_AND_ORIGIN``. This flag adds
+more detail to ``debugify``'s output, by including one or more stacktraces with
+every missing source location, capturing the point at which the empty source
+location was created, and every point at which it was copied to an instruction,
+making it trivial in most cases to find the origin of the underlying bug. If
+using origin tracking, it is recommended to also build LLVM with debug info
+enabled, so that the stacktrace can be accurately symbolized.
+
+.. note::
+
+   The coverage tracking feature has been designed primarily for use with the
+   :ref:`original debug info preservation<OriginalDI>` mode of ``debugify``, and
+   so may not be reliable in other settings. When using this mode, the
+   stacktraces produced by the ``COVERAGE_AND_ORIGIN`` setting will be printed
+   in an easy-to-read format as part of the reports generated by the
+   ``llvm-original-di-preservation.py`` script.
+
+.. _OriginalDI:
 
 Test original debug info preservation in optimizations
 ------------------------------------------------------
@@ -478,6 +512,8 @@ as follows:
 Please do note that there are some known false positives, for source locations
 and debug record checking, so that will be addressed as a future work.
 
+.. _MIRDebugify:
+
 Mutation testing for MIR-level transformations
 ----------------------------------------------
 
diff --git a/llvm/docs/InterfaceExportAnnotations.rst b/llvm/docs/InterfaceExportAnnotations.rst
index afea5b461d16..ec1f907b5f64 100644
--- a/llvm/docs/InterfaceExportAnnotations.rst
+++ b/llvm/docs/InterfaceExportAnnotations.rst
@@ -222,7 +222,7 @@ method in a C++ class, it may be annotated for export.
 Friend Functions
 ~~~~~~~~~~~~~~~~
 Friend functions declared in a class, struct or union must be annotated with
-``LLVM_ABI_FRIEND`` if the corresponding function declaration is annotated with
+``LLVM_ABI`` if the corresponding function declaration is annotated with
 ``LLVM_ABI``. This requirement applies even when the class containing the friend
 declaration is annotated with ``LLVM_ABI``.
 
@@ -236,14 +236,13 @@ declaration is annotated with ``LLVM_ABI``.
    class ExampleClass {
      // Friend declaration of a function must be annotated the same as the actual
      // function declaration.
-     LLVM_ABI_FRIEND friend int friend_function(ExampleClass &obj);
+     LLVM_ABI friend int friend_function(ExampleClass &obj);
    };
 
 .. note::
 
    Annotating the friend declaration avoids an “inconsistent dll linkage”
-   compiler error when building a DLL for Windows. The ``LLVM_ABI_FRIEND``
-   annotation is a no-op when building ELF or Mach-O shared libraries.
+   compiler error when building a DLL for Windows.
 
 Virtual Table and Type Info
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index a7f1f0b8e300..26475ce57996 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1234,8 +1234,7 @@ Currently, only the following parameter attributes are defined:
     ``byval`` parameters). This is not a valid attribute for return
     values.
 
-    The byval type argument indicates the in-memory value type, and
-    must be the same as the pointee type of the argument.
+    The byval type argument indicates the in-memory value type.
 
     The byval attribute also supports specifying an alignment with the
     align attribute. It indicates the alignment of the stack slot to
@@ -1283,8 +1282,7 @@ Currently, only the following parameter attributes are defined:
     any parameter must have a ``"preallocated"`` operand bundle. A ``musttail``
     function call cannot have a ``"preallocated"`` operand bundle.
 
-    The preallocated attribute requires a type argument, which must be
-    the same as the pointee type of the argument.
+    The preallocated attribute requires a type argument.
 
     The preallocated attribute also supports specifying an alignment with the
     align attribute. It indicates the alignment of the stack slot to
@@ -1318,8 +1316,7 @@ Currently, only the following parameter attributes are defined:
     must be cleared off with :ref:`llvm.stackrestore
     <int_stackrestore>`.
 
-    The inalloca attribute requires a type argument, which must be the
-    same as the pointee type of the argument.
+    The inalloca attribute requires a type argument.
 
     See :doc:`InAlloca` for more information on how to use this
     attribute.
@@ -1331,8 +1328,7 @@ Currently, only the following parameter attributes are defined:
     loads and stores to the structure may be assumed by the callee not
     to trap and to be properly aligned.
 
-    The sret type argument specifies the in memory type, which must be
-    the same as the pointee type of the argument.
+    The sret type argument specifies the in memory type.
 
     A function that accepts an ``sret`` argument must return ``void``.
     A return value may not be ``sret``.
@@ -1741,6 +1737,23 @@ Currently, only the following parameter attributes are defined:
 
     This attribute cannot be applied to return values.
 
+``dead_on_return``
+    This attribute indicates that the memory pointed to by the argument is dead
+    upon function return, both upon normal return and if the calls unwinds, meaning
+    that the caller will not depend on its contents. Stores that would be observable
+    either on the return path or on the unwind path may be elided.
+
+    Specifically, the behavior is as-if any memory written through the pointer
+    during the execution of the function is overwritten with a poison value
+    upon function return. The caller may access the memory, but any load
+    not preceded by a store will return poison.
+
+    This attribute does not imply aliasing properties. For pointer arguments that
+    do not alias other memory locations, ``noalias`` attribute may be used in
+    conjunction. Conversely, this attribute always implies ``dead_on_unwind``.
+
+    This attribute cannot be applied to return values.
+
 ``range(<ty> <a>, <b>)``
     This attribute expresses the possible range of the parameter or return value.
     If the value is not in the specified range, it is converted to poison.
@@ -6976,16 +6989,25 @@ appear in the included source file.
 DILabel
 """""""
 
-``DILabel`` nodes represent labels within a :ref:`DISubprogram`. All fields of
-a ``DILabel`` are mandatory. The ``scope:`` field must be one of either a
-:ref:`DILexicalBlockFile`, a :ref:`DILexicalBlock`, or a :ref:`DISubprogram`.
-The ``name:`` field is the label identifier. The ``file:`` field is the
-:ref:`DIFile` the label is present in. The ``line:`` field is the source line
+``DILabel`` nodes represent labels within a :ref:`DISubprogram`. The ``scope:``
+field must be one of either a :ref:`DILexicalBlockFile`, a
+:ref:`DILexicalBlock`, or a :ref:`DISubprogram`. The ``name:`` field is the
+label identifier. The ``file:`` field is the :ref:`DIFile` the label is
+present in. The ``line:`` and ``column:`` field are the source line and column
 within the file where the label is declared.
 
+Furthermore, a label can be marked as artificial, i.e. compiler-generated,
+using ``isArtificial:``. Such artificial labels are generated, e.g., by
+the ``CoroSplit`` pass. In addition, the ``CoroSplit`` pass also uses the
+``coroSuspendIdx:`` field to identify the coroutine suspend points.
+
+``scope:``, ``name:``, ``file:`` and ``line:`` are mandatory. The remaining
+fields are optional.
+
 .. code-block:: text
 
-  !2 = !DILabel(scope: !0, name: "foo", file: !1, line: 7)
+  !2 = !DILabel(scope: !0, name: "foo", file: !1, line: 7, column: 4)
+  !3 = !DILabel(scope: !0, name: "__coro_resume_3", file: !1, line: 9, column: 3, isArtificial: true, coroSuspendIdx: 3)
 
 DICommonBlock
 """""""""""""
@@ -13162,7 +13184,7 @@ This instruction requires several arguments:
    -  All ABI-impacting function attributes, such as sret, byval, inreg,
       returned, and inalloca, must match.
    -  The caller and callee prototypes must match. Pointer types of parameters
-      or return types may differ in pointee type, but not in address space.
+      or return types do not differ in address space.
 
    On the other hand, if the calling convention is `swifttailcc` or `tailcc`:
 
diff --git a/llvm/docs/MLGO.rst b/llvm/docs/MLGO.rst
index 4f8fb3f59ca1..ed0769bebeac 100644
--- a/llvm/docs/MLGO.rst
+++ b/llvm/docs/MLGO.rst
@@ -15,11 +15,13 @@ Currently the following heuristics feature such integration:
 
 This document is an outline of the tooling and APIs facilitating MLGO.
 
-Note that tools for orchestrating ML training are not part of LLVM, as they are
-dependency-heavy - both on the ML infrastructure choice, as well as choices of
-distributed computing. For the training scenario, LLVM only contains facilities
-enabling it, such as corpus extraction, training data extraction, and evaluation
-of models during training.
+.. note::
+    
+  The tools for orchestrating ML training are not part of LLVM, as they are
+  dependency-heavy - both on the ML infrastructure choice, as well as choices of
+  distributed computing. For the training scenario, LLVM only contains facilities
+  enabling it, such as corpus extraction, training data extraction, and evaluation
+  of models during training.
 
 
 .. contents::
@@ -329,8 +331,10 @@ We currently feature 4 implementations:
   the neural network, together with its weights (essentially, loops performing
   matrix multiplications)
 
-NOTE: we are actively working on replacing this with an EmitC implementation
-requiring no out of tree build-time dependencies.
+.. note::
+    
+  we are actively working on replacing this with an EmitC implementation
+  requiring no out of tree build-time dependencies.
 
 - ``InteractiveModelRunner``. This is intended for training scenarios where the
   training algorithm drives compilation. This model runner has no special
@@ -448,7 +452,16 @@ downstream tasks, including ML-guided compiler optimizations.
 
 The core components are:
   - **Vocabulary**: A mapping from IR entities (opcodes, types, etc.) to their
-    vector representations. This is managed by ``IR2VecVocabAnalysis``.
+    vector representations. This is managed by ``IR2VecVocabAnalysis``. The 
+    vocabulary (.json file) contains three sections -- Opcodes, Types, and 
+    Arguments, each containing the representations of the corresponding 
+    entities.
+
+    .. note::
+      
+      It is mandatory to have these three sections present in the vocabulary file 
+      for it to be valid; order in which they appear does not matter.
+
   - **Embedder**: A class (``ir2vec::Embedder``) that uses the vocabulary to
     compute embeddings for instructions, basic blocks, and functions.
 
@@ -479,14 +492,9 @@ embeddings can be computed and accessed via an ``ir2vec::Embedder`` instance.
 
       // Assuming F is an llvm::Function&
       // For example, using IR2VecKind::Symbolic:
-      Expected<std::unique_ptr<ir2vec::Embedder>> EmbOrErr =
+      std::unique_ptr<ir2vec::Embedder> Emb =
           ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary);
 
-      if (auto Err = EmbOrErr.takeError()) {
-        // Handle error in embedder creation
-        return;
-      }
-      std::unique_ptr<ir2vec::Embedder> Emb = std::move(*EmbOrErr);
 
 3. **Compute and Access Embeddings**:
    Call ``getFunctionVector()`` to get the embedding for the function. 
@@ -522,9 +530,11 @@ implementation details.
 Building with ML support
 ========================
 
-**NOTE** For up to date information on custom builds, see the ``ml-*``
-`build bots <http://lab.llvm.org>`_. They are set up using 
-`like this <https://github.com/google/ml-compiler-opt/blob/main/buildbot/buildbot_init.sh>`_.
+.. note::
+  
+  For up to date information on custom builds, see the ``ml-*``
+  `build bots <http://lab.llvm.org>`_. They are set up using 
+  `like this <https://github.com/google/ml-compiler-opt/blob/main/buildbot/buildbot_init.sh>`_.
 
 Embed pre-trained models (aka "release" mode)
 ---------------------------------------------
@@ -558,9 +568,11 @@ You can also specify a URL for the path, and it is also possible to pre-compile
 the header and object and then just point to the precompiled artifacts. See for
 example ``LLVM_OVERRIDE_MODEL_HEADER_INLINERSIZEMODEL``.
 
-**Note** that we are transitioning away from the AOT compiler shipping with the
-tensorflow package, and to a EmitC, in-tree solution, so these details will
-change soon.
+.. note::
+
+  We are transitioning away from the AOT compiler shipping with the
+  tensorflow package, and to a EmitC, in-tree solution, so these details will
+  change soon.
 
 Using TFLite (aka "development" mode)
 -------------------------------------
diff --git a/llvm/docs/QualGroup.rst b/llvm/docs/QualGroup.rst
new file mode 100644
index 000000000000..ca5fe2137f47
--- /dev/null
+++ b/llvm/docs/QualGroup.rst
@@ -0,0 +1,60 @@
+========================
+LLVM Qualification Group
+========================
+
+Introduction
+============
+
+The LLVM Qualification Group is an open working group within the LLVM community. 
+It was created to coordinate efforts around enabling the use of LLVM components 
+in safety-critical applications such as those governed by ISO 26262 (automotive), 
+DO-178C (aerospace), and EN 50128 (railways).
+
+Motivation
+==========
+
+LLVM is increasingly used in safety-critical domains (e.g., automotive, aerospace, medical),
+but currently lacks a shared structure to address the specific needs of functional safety, 
+such as systematic assurance arguments, tool qualification artifacts, and their associated 
+documentation.
+
+An more open, upstream, reusable, and collaborative approach would benefit the wider ecosystem.
+This group serves as a public forum for those interested in improving LLVM’s suitability for
+use in such environments.
+
+Goals
+=====
+
+The Qualification Group aims to:
+
+* Facilitate open discussion around tool confidence and qualification practices
+* Identify areas for upstream improvements (e.g., traceability hooks, quality evidence)
+* Share safety-relevant documentation and argumentation templates
+* Coordinate efforts across users and vendors working toward similar goals
+* Act as a point of contact for safety-related collaboration
+
+The group is non-enforcing and does not control any part of the codebase.
+All technical decisions remain subject to the standard LLVM review and governance process.
+
+Participation
+=============
+
+Participation is open to anyone interested. The group currently coordinates via:
+
+* LLVM Discourse (under the "Community" category): https://discourse.llvm.org/
+* Public working sessions and collaborative documents
+* [Details on working sessions will be shared soon on the GettingInvolved page]
+* Informal discussions on the LLVM Community Discord server (channel to be announced)
+
+Initial Contributors
+====================
+
+The initial participants include community members and industry contributors with experience
+in compiler development, tool qualification, and functional safety. (To be filled in after PR review.)
+
+Contact
+=======
+
+For more information or to get involved, please refer to the ongoing Discourse thread:
+https://discourse.llvm.org/t/rfc-proposal-to-establish-a-safety-group-in-llvm/86916 
+
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 81684ba30f12..158a20cce7f8 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -498,6 +498,9 @@ The current vendor extensions supported are:
 ``experimental-Xqcisync``
   LLVM implements `version 0.3 of the Qualcomm uC Sync Delay extension specification <https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.13.0>`__ by Qualcomm. These instructions are only available for riscv32.
 
+``Xmipscbop``
+  LLVM implements MIPS prefetch extension `p8700 processor <https://mips.com/products/hardware/p8700/>`__ by MIPS.
+
 ``Xmipscmov``
   LLVM implements conditional move for the `p8700 processor <https://mips.com/products/hardware/p8700/>`__ by MIPS.
 
@@ -516,6 +519,9 @@ The current vendor extensions supported are:
 ``XAndesVBFHCvt``
   LLVM implements `version 5.0.0 of the Andes Vector BFLOAT16 Conversion Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
 
+``XAndesVSINTLoad``
+  LLVM implements `version 5.0.0 of the Andes Vector INT4 Load Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
+
 ``XAndesVPackFPH``
   LLVM implements `version 5.0.0 of the Andes Vector Packed FP16 Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
 
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 4a1005667692..daf822388a2f 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -212,6 +212,8 @@ Changes to the RISC-V Backend
 * `-mtune=andes-45-series` was added.
 * Adds assembler support for the Andes `XAndesvbfhcvt` (Andes Vector BFLOAT16 Conversion extension).
 * `-mcpu=andes-ax45mpv` was added.
+* Removed -mattr=+no-rvc-hints that could be used to disable parsing and generation of RVC hints.
+* Adds assembler support for the Andes `XAndesvsintload` (Andes Vector INT4 Load extension).
 
 Changes to the WebAssembly Backend
 ----------------------------------
@@ -272,6 +274,7 @@ Changes to the LLVM tools
 * In llvm-objcopy/llvm-strip's ELF port, `--discard-locals` and `--discard-all` now allow and preserve symbols referenced by relocations.
   ([#47468](https://github.com/llvm/llvm-project/issues/47468))
 * llvm-addr2line now supports a `+` prefix when specifying an address.
+* Support for `SHT_LLVM_BB_ADDR_MAP` versions 0 and 1 has been dropped.
 
 Changes to LLDB
 ---------------------------------
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 1858bda6160d..1f563fbfb725 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -217,6 +217,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
      - Adds an instruction to compute the matrix product of an M x K matrix with a K x N matrix and then add an M x N matrix. 
    * - ``SPV_INTEL_int4``
      - Adds support for 4-bit integer type, and allow this type to be used in cooperative matrices.
+   * - ``SPV_KHR_float_controls2``
+     - Adds ability to specify the floating-point environment in shaders. It can be used on whole modules and individual instructions.
 
 To enable multiple extensions, list them separated by comma. For example, to enable support for atomic operations on floating-point numbers and arbitrary precision integers, use:
 
@@ -255,6 +257,7 @@ using target extension types and are represented as follows:
      SPIR-V Type        LLVM type name          LLVM type arguments
      ================== ======================= ===========================================================================================
      OpTypeImage        ``spirv.Image``         sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
+     OpTypeImage        ``spirv.SignedImage``   sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeSampler      ``spirv.Sampler``       (none)
      OpTypeSampledImage ``spirv.SampledImage``  sampled type, dimensionality, depth, arrayed, MS, sampled, image format, [access qualifier]
      OpTypeEvent        ``spirv.Event``         (none)
@@ -275,6 +278,12 @@ parameters of its underlying image type, so that a sampled image for the
 previous type has the representation
 ``target("spirv.SampledImage, void, 1, 1, 0, 0, 0, 0, 0)``.
 
+The differences between ``spirv.Image`` and ``spirv.SignedImage`` is that the
+backend will generate code assuming that the format of the image is a signed
+integer instead of unsigned. This is required because llvm-ir will create the
+same sampled type for signed and unsigned integers. If the image format is
+unknown, the backend cannot distinguish the two case.
+
 See `wg-hlsl proposal 0018 <https://github.com/llvm/wg-hlsl/blob/main/proposals/0018-spirv-resource-representation.md>`_
 for details on ``spirv.VulkanBuffer``.
 
diff --git a/llvm/docs/SourceLevelDebugging.rst b/llvm/docs/SourceLevelDebugging.rst
index 8a11dcf5254a..dfc8c53edbb8 100644
--- a/llvm/docs/SourceLevelDebugging.rst
+++ b/llvm/docs/SourceLevelDebugging.rst
@@ -143,6 +143,42 @@ debugging information influences optimization passes then it will be reported
 as a failure.  See :doc:`TestingGuide` for more information on LLVM test
 infrastructure and how to run various tests.
 
+.. _variables_and_variable_fragments:
+
+Variables and Variable Fragments
+================================
+
+In this document "variable" refers generally to any source language object
+which can have a value, including at least:
+
+- Variables
+- Constants
+- Formal parameters
+
+.. note::
+
+   There is no special provision for "true" constants in LLVM today, and
+   they are instead treated as local or global variables.
+
+A variable is represented by a `local variable <LangRef.html#dilocalvariable>`_
+or `global variable <LangRef.html#diglobalvariable>`_ metadata node.
+
+A "variable fragment" (or just "fragment") is a contiguous span of bits of a
+variable.
+
+A :ref:`debug record <debug_records>` which refers to a ``DIExpression`` ending
+with a ``DW_OP_LLVM_fragment`` operation describes a fragment of the variable
+it refers to.
+
+The operands of the ``DW_OP_LLVM_fragment`` operation encode the bit offset of
+the fragment relative to the start of the variable, and the size of the
+fragment in bits, respectively.
+
+.. note::
+
+   The ``DW_OP_LLVM_fragment`` operation acts only to encode the fragment
+   information, and does not have an effect on the semantics of the expression.
+
 .. _format:
 
 Debugging information format
@@ -510,10 +546,23 @@ values through compilation, when objects are promoted to SSA values a
 ``#dbg_value`` record is created for each assignment, recording the
 variable's new location. Compared with the ``#dbg_declare`` record:
 
-* A #dbg_value terminates the effect of any preceding #dbg_values for (any
-  overlapping fragments of) the specified variable.
-* The #dbg_value's position in the IR defines where in the instruction stream
-  the variable's value changes.
+* A ``#dbg_value`` terminates the effects that any preceding records have on
+  any common bits of a common variable.
+
+  .. note::
+
+    The current implementation generally terminates the effect of every
+    record in its entirety if any of its effects would be terminated, rather
+    than carrying forward the effect of previous records for non-overlapping
+    bits as it would be permitted to do by this definition. This is allowed
+    just as dropping any debug information at any point in the compilation is
+    allowed.
+
+    One exception to this is :doc:`AssignmentTracking` where certain
+    memory-based locations are carried forward partially in some situations.
+
+* The ``#dbg_value``'s position in the IR defines where in the instruction
+  stream the variable's value changes.
 * Operands can be constants, indicating the variable is assigned a
   constant value.
 
diff --git a/llvm/docs/index.rst b/llvm/docs/index.rst
index b480729aaa5d..fe0c84569598 100644
--- a/llvm/docs/index.rst
+++ b/llvm/docs/index.rst
@@ -86,6 +86,7 @@ LLVM welcomes contributions of all kinds. To learn more, see the following artic
    :hidden:
 
    GettingInvolved
+   QualGroup
 
 * :doc:`GettingInvolved`
 * :ref:`development-process`
@@ -97,6 +98,9 @@ LLVM welcomes contributions of all kinds. To learn more, see the following artic
 
 * :ref:`report-security-issue`
 
+* :doc:`QualGroup`  
+  LLVM Qualification Group
+
 Indices and tables
 ==================
 
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
index 73a7b5b38a38..7800dca5fd37 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
@@ -21,6 +21,7 @@
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
index d2bedd2c7270..1789ab1e20fe 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
@@ -22,6 +22,7 @@
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
index 09b87b0181f9..643c089b4ed8 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
@@ -25,6 +25,7 @@
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
index 30c0e239a6cb..0e2e9fe729bb 100644
--- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
@@ -24,6 +24,7 @@
 #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
index 65f94ed7a80d..1fc89fca1e1d 100644
--- a/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
+++ b/llvm/examples/Kaleidoscope/include/KaleidoscopeJIT.h
@@ -22,6 +22,7 @@
 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl/LLJITWithExecutorProcessControl.cpp b/llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl/LLJITWithExecutorProcessControl.cpp
index 043150f47d92..4f58d1aa82a6 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl/LLJITWithExecutorProcessControl.cpp
+++ b/llvm/examples/OrcV2Examples/LLJITWithExecutorProcessControl/LLJITWithExecutorProcessControl.cpp
@@ -27,6 +27,7 @@
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp b/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
index c55aa73d5027..84f17871c03e 100644
--- a/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
+++ b/llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries/LLJITWithThinLTOSummaries.cpp
@@ -169,7 +169,9 @@ Expected<ThreadSafeModule> loadModule(StringRef Path,
 
   MemoryBufferRef BitcodeBufferRef = (**BitcodeBuffer).getMemBufferRef();
   Expected<std::unique_ptr<Module>> M =
-      parseBitcodeFile(BitcodeBufferRef, *TSCtx.getContext());
+      TSCtx.withContextDo([&](LLVMContext *Ctx) {
+        return parseBitcodeFile(BitcodeBufferRef, *Ctx);
+      });
   if (!M)
     return M.takeError();
 
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
index 286fa8baac4f..b95462f340f2 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsBasicUsage/OrcV2CBindingsBasicUsage.c
@@ -22,11 +22,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -57,6 +54,9 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Free the builder.
   LLVMDisposeBuilder(Builder);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
index 1b4102625fa1..42a27c7054d4 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsDumpObjects/OrcV2CBindingsDumpObjects.c
@@ -31,8 +31,7 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  LLVMContextRef Ctx = LLVMContextCreate();
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
   LLVMTypeRef ParamTypes[] = {LLVMInt32Type(), LLVMInt32Type()};
   LLVMTypeRef SumFunctionType =
@@ -45,6 +44,8 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   LLVMValueRef SumArg1 = LLVMGetParam(SumFunction, 1);
   LLVMValueRef Result = LLVMBuildAdd(Builder, SumArg0, SumArg1, "result");
   LLVMBuildRet(Builder, Result);
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
   LLVMOrcDisposeThreadSafeContext(TSCtx);
   return TSM;
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
index 41ae6e53db1d..62904d006da6 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsIRTransforms/OrcV2CBindingsIRTransforms.c
@@ -32,8 +32,7 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  LLVMContextRef Ctx = LLVMContextCreate();
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
   LLVMTypeRef ParamTypes[] = {LLVMInt32Type(), LLVMInt32Type()};
   LLVMTypeRef SumFunctionType =
@@ -47,6 +46,7 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   LLVMValueRef Result = LLVMBuildAdd(Builder, SumArg0, SumArg1, "result");
   LLVMBuildRet(Builder, Result);
   LLVMDisposeBuilder(Builder);
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
   LLVMOrcDisposeThreadSafeContext(TSCtx);
   return TSM;
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
index 33398c8cb981..9c31f9389920 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsLazy/OrcV2CBindingsLazy.c
@@ -67,11 +67,9 @@ const char MainMod[] =
 LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
                                 const char *Name,
                                 LLVMOrcThreadSafeModuleRef *TSM) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
 
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext for the Module.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Wrap Source in a MemoryBuffer
   LLVMMemoryBufferRef MB =
@@ -85,6 +83,10 @@ LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
     // TODO: LLVMDisposeMessage(ErrMsg);
   }
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   *TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
index f85430bcfda4..6962c6980e78 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsMCJITLikeMemoryManager/OrcV2CBindingsMCJITLikeMemoryManager.c
@@ -150,11 +150,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -182,6 +179,10 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Build the return instruction.
   LLVMBuildRet(Builder, Result);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
index 7f84a3d41343..7f8a9cd334c6 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsRemovableCode/OrcV2CBindingsRemovableCode.c
@@ -22,11 +22,8 @@ int handleError(LLVMErrorRef Err) {
 }
 
 LLVMOrcThreadSafeModuleRef createDemoModule(void) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Create a new LLVM module.
   LLVMModuleRef M = LLVMModuleCreateWithNameInContext("demo", Ctx);
@@ -57,6 +54,10 @@ LLVMOrcThreadSafeModuleRef createDemoModule(void) {
   //  - Free the builder.
   LLVMDisposeBuilder(Builder);
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx =
+      LLVMOrcCreateNewThreadSafeContextFromLLVMContext(Ctx);
+
   // Our demo module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   LLVMOrcThreadSafeModuleRef TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c b/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
index 85651f728399..3c1ff8392eff 100644
--- a/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
+++ b/llvm/examples/OrcV2Examples/OrcV2CBindingsVeryLazy/OrcV2CBindingsVeryLazy.c
@@ -74,11 +74,8 @@ LLVMErrorRef applyDataLayout(void *Ctx, LLVMModuleRef M) {
 LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
                                 const char *Name,
                                 LLVMOrcThreadSafeModuleRef *TSM) {
-  // Create a new ThreadSafeContext and underlying LLVMContext.
-  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
-
-  // Get a reference to the underlying LLVMContext.
-  LLVMContextRef Ctx = LLVMOrcThreadSafeContextGetContext(TSCtx);
+  // Create an LLVMContext.
+  LLVMContextRef Ctx = LLVMContextCreate();
 
   // Wrap Source in a MemoryBuffer
   LLVMMemoryBufferRef MB =
@@ -93,6 +90,9 @@ LLVMErrorRef parseExampleModule(const char *Source, size_t Len,
     return Err;
   }
 
+  // Create a new ThreadSafeContext to hold the context.
+  LLVMOrcThreadSafeContextRef TSCtx = LLVMOrcCreateNewThreadSafeContext();
+
   // Our module is now complete. Wrap it and our ThreadSafeContext in a
   // ThreadSafeModule.
   *TSM = LLVMOrcCreateNewThreadSafeModule(M, TSCtx);
diff --git a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
index acdbce4f6e3e..15dca0abe52d 100644
--- a/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
+++ b/llvm/examples/SpeculativeJIT/SpeculativeJIT.cpp
@@ -7,6 +7,7 @@
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/SpeculateAnalyses.h"
 #include "llvm/ExecutionEngine/Orc/Speculation.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h"
diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h
index ec3e14214064..e1e7b2bd70ad 100644
--- a/llvm/include/llvm-c/DebugInfo.h
+++ b/llvm/include/llvm-c/DebugInfo.h
@@ -1427,6 +1427,16 @@ void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP);
  */
 unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram);
 
+/**
+ * Replace the subprogram subroutine type.
+ * \param Subprogram        The subprogram object.
+ * \param SubroutineType    The new subroutine type.
+ *
+ * @see DISubprogram::replaceType()
+ */
+void LLVMDISubprogramReplaceType(LLVMMetadataRef Subprogram,
+                                 LLVMMetadataRef SubroutineType);
+
 /**
  * Get the debug location for the given instruction.
  *
diff --git a/llvm/include/llvm-c/Orc.h b/llvm/include/llvm-c/Orc.h
index 743ba1d58178..ee80f6f9f989 100644
--- a/llvm/include/llvm-c/Orc.h
+++ b/llvm/include/llvm-c/Orc.h
@@ -1062,20 +1062,32 @@ LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath(
     const char *FileName);
 
 /**
- * Create a ThreadSafeContext containing a new LLVMContext.
+ * Create a ThreadSafeContextRef containing a new LLVMContext.
  *
  * Ownership of the underlying ThreadSafeContext data is shared: Clients
- * can and should dispose of their ThreadSafeContext as soon as they no longer
- * need to refer to it directly. Other references (e.g. from ThreadSafeModules)
- * will keep the data alive as long as it is needed.
+ * can and should dispose of their ThreadSafeContextRef as soon as they no
+ * longer need to refer to it directly. Other references (e.g. from
+ * ThreadSafeModules) will keep the underlying data alive as long as it is
+ * needed.
  */
 LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void);
 
 /**
- * Get a reference to the wrapped LLVMContext.
+ * Create a ThreadSafeContextRef from a given LLVMContext, which must not be
+ * associated with any existing ThreadSafeContext.
+ *
+ * The underlying ThreadSafeContext will take ownership of the LLVMContext
+ * object, so clients should not free the LLVMContext passed to this
+ * function.
+ *
+ * Ownership of the underlying ThreadSafeContext data is shared: Clients
+ * can and should dispose of their ThreadSafeContextRef as soon as they no
+ * longer need to refer to it directly. Other references (e.g. from
+ * ThreadSafeModules) will keep the underlying data alive as long as it is
+ * needed.
  */
-LLVMContextRef
-LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx);
+LLVMOrcThreadSafeContextRef
+LLVMOrcCreateNewThreadSafeContextFromLLVMContext(LLVMContextRef Ctx);
 
 /**
  * Dispose of a ThreadSafeContext.
diff --git a/llvm/include/llvm-c/lto.h b/llvm/include/llvm-c/lto.h
index 5ceb02224d2b..91195b0fe945 100644
--- a/llvm/include/llvm-c/lto.h
+++ b/llvm/include/llvm-c/lto.h
@@ -46,7 +46,7 @@ typedef bool lto_bool_t;
  * @{
  */
 
-#define LTO_API_VERSION 29
+#define LTO_API_VERSION 30
 
 /**
  * \since prior to LTO_API_VERSION=3
@@ -286,6 +286,21 @@ lto_module_get_symbol_name(lto_module_t mod, unsigned int index);
 extern lto_symbol_attributes
 lto_module_get_symbol_attribute(lto_module_t mod, unsigned int index);
 
+/**
+ * Returns the number of asm undefined symbols in the object module.
+ *
+ * \since prior to LTO_API_VERSION=30
+ */
+extern unsigned int lto_module_get_num_asm_undef_symbols(lto_module_t mod);
+
+/**
+ * Returns the name of the ith asm undefined symbol in the object module.
+ *
+ * \since prior to LTO_API_VERSION=30
+ */
+extern const char *lto_module_get_asm_undef_symbol_name(lto_module_t mod,
+                                                        unsigned int index);
+
 /**
  * Returns the module's linker options.
  *
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index 13df838da3da..e1589544787c 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -572,7 +572,7 @@ class IEEEFloat final {
   /// emphasizes producing different codes for different inputs in order to
   /// be used in canonicalization and memoization. As such, equality is
   /// bitwiseIsEqual, and 0 != -0.
-  LLVM_ABI_FRIEND friend hash_code hash_value(const IEEEFloat &Arg);
+  LLVM_ABI friend hash_code hash_value(const IEEEFloat &Arg);
 
   /// Converts this value into a decimal string.
   ///
@@ -629,13 +629,12 @@ class IEEEFloat final {
   ///   0   -> \c IEK_Zero
   ///   Inf -> \c IEK_Inf
   ///
-  LLVM_ABI_FRIEND friend int ilogb(const IEEEFloat &Arg);
+  LLVM_ABI friend int ilogb(const IEEEFloat &Arg);
 
   /// Returns: X * 2^Exp for integral exponents.
-  LLVM_ABI_FRIEND friend IEEEFloat scalbn(IEEEFloat X, int Exp, roundingMode);
+  LLVM_ABI friend IEEEFloat scalbn(IEEEFloat X, int Exp, roundingMode);
 
-  LLVM_ABI_FRIEND friend IEEEFloat frexp(const IEEEFloat &X, int &Exp,
-                                         roundingMode);
+  LLVM_ABI friend IEEEFloat frexp(const IEEEFloat &X, int &Exp, roundingMode);
 
   /// \name Special value setters.
   /// @{
@@ -673,7 +672,9 @@ class IEEEFloat final {
 
   integerPart addSignificand(const IEEEFloat &);
   integerPart subtractSignificand(const IEEEFloat &, integerPart);
-  lostFraction addOrSubtractSignificand(const IEEEFloat &, bool subtract);
+  // Exported for IEEEFloatUnitTestHelper.
+  LLVM_ABI lostFraction addOrSubtractSignificand(const IEEEFloat &,
+                                                 bool subtract);
   lostFraction multiplySignificand(const IEEEFloat &, IEEEFloat,
                                    bool ignoreAddend = false);
   lostFraction multiplySignificand(const IEEEFloat&);
@@ -906,11 +907,11 @@ class DoubleAPFloat final {
   LLVM_ABI LLVM_READONLY int getExactLog2() const;
   LLVM_ABI LLVM_READONLY int getExactLog2Abs() const;
 
-  LLVM_ABI_FRIEND friend DoubleAPFloat scalbn(const DoubleAPFloat &X, int Exp,
-                                              roundingMode);
-  LLVM_ABI_FRIEND friend DoubleAPFloat frexp(const DoubleAPFloat &X, int &Exp,
-                                             roundingMode);
-  LLVM_ABI_FRIEND friend hash_code hash_value(const DoubleAPFloat &Arg);
+  LLVM_ABI friend DoubleAPFloat scalbn(const DoubleAPFloat &X, int Exp,
+                                       roundingMode);
+  LLVM_ABI friend DoubleAPFloat frexp(const DoubleAPFloat &X, int &Exp,
+                                      roundingMode);
+  LLVM_ABI friend hash_code hash_value(const DoubleAPFloat &Arg);
 };
 
 LLVM_ABI hash_code hash_value(const DoubleAPFloat &Arg);
@@ -1518,7 +1519,7 @@ class APFloat : public APFloatBase {
     APFLOAT_DISPATCH_ON_SEMANTICS(getExactLog2());
   }
 
-  LLVM_ABI_FRIEND friend hash_code hash_value(const APFloat &Arg);
+  LLVM_ABI friend hash_code hash_value(const APFloat &Arg);
   friend int ilogb(const APFloat &Arg) { return ilogb(Arg.getIEEE()); }
   friend APFloat scalbn(APFloat X, int Exp, roundingMode RM);
   friend APFloat frexp(const APFloat &X, int &Exp, roundingMode RM);
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index fe48c60466e9..295506393a1c 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -561,7 +561,7 @@ class [[nodiscard]] APInt {
   }
 
   /// Overload to compute a hash_code for an APInt value.
-  friend hash_code hash_value(const APInt &Arg);
+  LLVM_ABI friend hash_code hash_value(const APInt &Arg);
 
   /// This function returns a pointer to the internal storage of the APInt.
   /// This is useful for writing out the APInt in binary form without any
diff --git a/llvm/include/llvm/ADT/ArrayRef.h b/llvm/include/llvm/ADT/ArrayRef.h
index 4819c8847134..8091be203664 100644
--- a/llvm/include/llvm/ADT/ArrayRef.h
+++ b/llvm/include/llvm/ADT/ArrayRef.h
@@ -67,7 +67,8 @@ namespace llvm {
     /*implicit*/ ArrayRef() = default;
 
     /// Construct an empty ArrayRef from std::nullopt.
-    /*implicit*/ ArrayRef(std::nullopt_t) {}
+    /*implicit*/ LLVM_DEPRECATED("Use {} or ArrayRef<T>() instead", "{}")
+    ArrayRef(std::nullopt_t) {}
 
     /// Construct an ArrayRef from a single element.
     /*implicit*/ ArrayRef(const T &OneElt LLVM_LIFETIME_BOUND)
@@ -84,23 +85,19 @@ namespace llvm {
       assert(begin <= end);
     }
 
-    /// Construct an ArrayRef from a SmallVector. This is templated in order to
-    /// avoid instantiating SmallVectorTemplateCommon<T> whenever we
-    /// copy-construct an ArrayRef.
-    template<typename U>
-    /*implicit*/ ArrayRef(const SmallVectorTemplateCommon<T, U> &Vec)
-      : Data(Vec.data()), Length(Vec.size()) {
-    }
-
-    /// Construct an ArrayRef from a std::vector.
-    template<typename A>
-    /*implicit*/ ArrayRef(const std::vector<T, A> &Vec)
-      : Data(Vec.data()), Length(Vec.size()) {}
-
-    /// Construct an ArrayRef from a std::array
-    template <size_t N>
-    /*implicit*/ constexpr ArrayRef(const std::array<T, N> &Arr)
-        : Data(Arr.data()), Length(N) {}
+    /// Construct an ArrayRef from a type that has a data() method that returns
+    /// a pointer convertible to const T *.
+    template <
+        typename C,
+        typename = std::enable_if_t<
+            std::conjunction_v<
+                std::is_convertible<
+                    decltype(std::declval<const C &>().data()) *,
+                    const T *const *>,
+                std::is_integral<decltype(std::declval<const C &>().size())>>,
+            void>>
+    /*implicit*/ constexpr ArrayRef(const C &V)
+        : Data(V.data()), Length(V.size()) {}
 
     /// Construct an ArrayRef from a C array.
     template <size_t N>
@@ -123,32 +120,6 @@ namespace llvm {
 #pragma GCC diagnostic pop
 #endif
 
-    /// Construct an ArrayRef<const T*> from ArrayRef<T*>. This uses SFINAE to
-    /// ensure that only ArrayRefs of pointers can be converted.
-    template <typename U>
-    ArrayRef(const ArrayRef<U *> &A,
-             std::enable_if_t<std::is_convertible<U *const *, T const *>::value>
-                 * = nullptr)
-        : Data(A.data()), Length(A.size()) {}
-
-    /// Construct an ArrayRef<const T*> from a SmallVector<T*>. This is
-    /// templated in order to avoid instantiating SmallVectorTemplateCommon<T>
-    /// whenever we copy-construct an ArrayRef.
-    template <typename U, typename DummyT>
-    /*implicit*/ ArrayRef(
-        const SmallVectorTemplateCommon<U *, DummyT> &Vec,
-        std::enable_if_t<std::is_convertible<U *const *, T const *>::value> * =
-            nullptr)
-        : Data(Vec.data()), Length(Vec.size()) {}
-
-    /// Construct an ArrayRef<const T*> from std::vector<T*>. This uses SFINAE
-    /// to ensure that only vectors of pointers can be converted.
-    template <typename U, typename A>
-    ArrayRef(const std::vector<U *, A> &Vec,
-             std::enable_if_t<std::is_convertible<U *const *, T const *>::value>
-                 * = nullptr)
-        : Data(Vec.data()), Length(Vec.size()) {}
-
     /// Construct an ArrayRef<T> from iterator_range<U*>. This uses SFINAE
     /// to ensure that this is only used for iterator ranges over plain pointer
     /// iterators.
@@ -187,6 +158,20 @@ namespace llvm {
       return Data[Length-1];
     }
 
+    /// consume_front() - Returns the first element and drops it from ArrayRef.
+    const T &consume_front() {
+      const T &Ret = front();
+      *this = drop_front();
+      return Ret;
+    }
+
+    /// consume_back() - Returns the last element and drops it from ArrayRef.
+    const T &consume_back() {
+      const T &Ret = back();
+      *this = drop_back();
+      return Ret;
+    }
+
     // copy - Allocate copy in Allocator and return ArrayRef<T> to it.
     template <typename Allocator> MutableArrayRef<T> copy(Allocator &A) {
       T *Buff = A.template Allocate<T>(Length);
@@ -333,7 +318,8 @@ namespace llvm {
     /*implicit*/ MutableArrayRef() = default;
 
     /// Construct an empty MutableArrayRef from std::nullopt.
-    /*implicit*/ MutableArrayRef(std::nullopt_t) : ArrayRef<T>() {}
+    /*implicit*/ LLVM_DEPRECATED("Use {} or MutableArrayRef<T>() instead", "{}")
+    MutableArrayRef(std::nullopt_t) : ArrayRef<T>() {}
 
     /// Construct a MutableArrayRef from a single element.
     /*implicit*/ MutableArrayRef(T &OneElt) : ArrayRef<T>(OneElt) {}
@@ -345,18 +331,16 @@ namespace llvm {
     /// Construct a MutableArrayRef from a range.
     MutableArrayRef(T *begin, T *end) : ArrayRef<T>(begin, end) {}
 
-    /// Construct a MutableArrayRef from a SmallVector.
-    /*implicit*/ MutableArrayRef(SmallVectorImpl<T> &Vec)
-    : ArrayRef<T>(Vec) {}
-
-    /// Construct a MutableArrayRef from a std::vector.
-    /*implicit*/ MutableArrayRef(std::vector<T> &Vec)
-    : ArrayRef<T>(Vec) {}
-
-    /// Construct a MutableArrayRef from a std::array
-    template <size_t N>
-    /*implicit*/ constexpr MutableArrayRef(std::array<T, N> &Arr)
-        : ArrayRef<T>(Arr) {}
+    /// Construct a MutableArrayRef from a type that has a data() method that
+    /// returns a pointer convertible to T *.
+    template <typename C,
+              typename = std::enable_if_t<
+                  std::conjunction_v<
+                      std::is_convertible<
+                          decltype(std::declval<C &>().data()) *, T *const *>,
+                      std::is_integral<decltype(std::declval<C &>().size())>>,
+                  void>>
+    /*implicit*/ constexpr MutableArrayRef(const C &V) : ArrayRef<T>(V) {}
 
     /// Construct a MutableArrayRef from a C array.
     template <size_t N>
@@ -382,6 +366,20 @@ namespace llvm {
       return data()[this->size()-1];
     }
 
+    /// consume_front() - Returns the first element and drops it from ArrayRef.
+    T &consume_front() {
+      T &Ret = front();
+      *this = drop_front();
+      return Ret;
+    }
+
+    /// consume_back() - Returns the last element and drops it from ArrayRef.
+    T &consume_back() {
+      T &Ret = back();
+      *this = drop_back();
+      return Ret;
+    }
+
     /// slice(n, m) - Chop off the first N elements of the array, and keep M
     /// elements in the array.
     MutableArrayRef<T> slice(size_t N, size_t M) const {
diff --git a/llvm/include/llvm/ADT/BitVector.h b/llvm/include/llvm/ADT/BitVector.h
index 0eaa77b6dd81..70de4c239c1c 100644
--- a/llvm/include/llvm/ADT/BitVector.h
+++ b/llvm/include/llvm/ADT/BitVector.h
@@ -795,9 +795,7 @@ class BitVector {
     set_unused_bits(false);
   }
 
-  void init_words(bool t) {
-    std::fill(Bits.begin(), Bits.end(), 0 - (BitWord)t);
-  }
+  void init_words(bool t) { llvm::fill(Bits, 0 - (BitWord)t); }
 
   template<bool AddBits, bool InvertMask>
   void applyMask(const uint32_t *Mask, unsigned MaskWords) {
diff --git a/llvm/include/llvm/ADT/Bitset.h b/llvm/include/llvm/ADT/Bitset.h
index 4ba5c63535b2..55b96e47f2ed 100644
--- a/llvm/include/llvm/ADT/Bitset.h
+++ b/llvm/include/llvm/ADT/Bitset.h
@@ -50,7 +50,7 @@ class Bitset {
   }
 
   Bitset &set() {
-    std::fill(std::begin(Bits), std::end(Bits), -BitWord(0));
+    llvm::fill(Bits, -BitWord(0));
     return *this;
   }
 
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index 48c5c6eac901..07015b4a8dd8 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -16,6 +16,7 @@
 #ifndef LLVM_ADT_DYNAMICAPINT_H
 #define LLVM_ADT_DYNAMICAPINT_H
 
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/SlowDynamicAPInt.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
@@ -116,6 +117,14 @@ class DynamicAPInt {
       : ValSmall(Val) {
     ValLarge.Val.BitWidth = 0;
   }
+  LLVM_ATTRIBUTE_ALWAYS_INLINE explicit DynamicAPInt(const APInt &Val) {
+    if (Val.getBitWidth() <= 64) {
+      ValSmall = Val.getSExtValue();
+      ValLarge.Val.BitWidth = 0;
+    } else {
+      new (&ValLarge) detail::SlowDynamicAPInt(Val);
+    }
+  }
   LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt() : DynamicAPInt(0) {}
   LLVM_ATTRIBUTE_ALWAYS_INLINE ~DynamicAPInt() {
     if (LLVM_UNLIKELY(isLarge()))
@@ -212,7 +221,7 @@ class DynamicAPInt {
   friend DynamicAPInt operator/(int64_t A, const DynamicAPInt &B);
   friend DynamicAPInt operator%(int64_t A, const DynamicAPInt &B);
 
-  friend hash_code hash_value(const DynamicAPInt &x); // NOLINT
+  LLVM_ABI friend hash_code hash_value(const DynamicAPInt &x); // NOLINT
 
   LLVM_ABI void static_assert_layout(); // NOLINT
 
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index a252570c7eb7..dbc0e5176cbb 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1759,6 +1759,12 @@ bool none_of(R &&Range, UnaryPredicate P) {
   return std::none_of(adl_begin(Range), adl_end(Range), P);
 }
 
+/// Provide wrappers to std::fill which take ranges instead of having to pass
+/// begin/end explicitly.
+template <typename R, typename T> void fill(R &&Range, T &&Value) {
+  std::fill(adl_begin(Range), adl_end(Range), std::forward<T>(Value));
+}
+
 /// Provide wrappers to std::find which take ranges instead of having to pass
 /// begin/end explicitly.
 template <typename R, typename T> auto find(R &&Range, const T &Val) {
diff --git a/llvm/include/llvm/ADT/STLForwardCompat.h b/llvm/include/llvm/ADT/STLForwardCompat.h
index ec2df4547354..6a38d85739a8 100644
--- a/llvm/include/llvm/ADT/STLForwardCompat.h
+++ b/llvm/include/llvm/ADT/STLForwardCompat.h
@@ -37,74 +37,18 @@ template <typename T>
 using remove_cvref_t // NOLINT(readability-identifier-naming)
     = typename llvm::remove_cvref<T>::type;
 
+// TODO: Remove this in favor of std::type_identity<T> once we switch to C++23.
 template <typename T>
 struct type_identity // NOLINT(readability-identifier-naming)
 {
   using type = T;
 };
 
-//===----------------------------------------------------------------------===//
-//     Features from C++17
-//===----------------------------------------------------------------------===//
-
-template <typename T>
-struct negation // NOLINT(readability-identifier-naming)
-    : std::integral_constant<bool, !bool(T::value)> {};
-
-template <typename...>
-struct conjunction // NOLINT(readability-identifier-naming)
-    : std::true_type {};
-template <typename B1> struct conjunction<B1> : B1 {};
-template <typename B1, typename... Bn>
-struct conjunction<B1, Bn...>
-    : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
-
-template <typename...>
-struct disjunction // NOLINT(readability-identifier-naming)
-    : std::false_type {};
-template <typename B1> struct disjunction<B1> : B1 {};
-template <typename B1, typename... Bn>
-struct disjunction<B1, Bn...>
-    : std::conditional<bool(B1::value), B1, disjunction<Bn...>>::type {};
-
-struct in_place_t // NOLINT(readability-identifier-naming)
-{
-  explicit in_place_t() = default;
-};
-/// \warning This must not be odr-used, as it cannot be made \c inline in C++14.
-constexpr in_place_t in_place; // NOLINT(readability-identifier-naming)
-
-template <typename T>
-struct in_place_type_t // NOLINT(readability-identifier-naming)
-{
-  explicit in_place_type_t() = default;
-};
-/// \warning This must not be odr-used, as it cannot be made \c inline in C++14.
+// TODO: Remove this in favor of std::type_identity_t<T> once we switch to
+// C++23.
 template <typename T>
-constexpr in_place_type_t<T>
-    in_place_type; // NOLINT(readability-identifier-naming)
-
-template <std::size_t I>
-struct in_place_index_t // NOLINT(readability-identifier-naming)
-{
-  explicit in_place_index_t() = default;
-};
-/// \warning This must not be odr-used, as it cannot be made \c inline in C++14.
-template <std::size_t I>
-constexpr in_place_index_t<I>
-    in_place_index; // NOLINT(readability-identifier-naming)
-
-/// Implementation of std::experimental::make_array. Should be deleted in favor
-/// of the C++17 deduction guide for std::array.
-template <typename ExplicitT = void, typename... ArgTs>
-static constexpr auto
-make_array(ArgTs &&...Args) // NOLINT(readability-identifier-naming)
-    -> std::array<typename std::conditional_t<std::is_same<ExplicitT, void>::value,
-                                              std::common_type<ArgTs...>,
-                                              type_identity<ExplicitT>>::type,
-                  sizeof...(ArgTs)> {
-  return {std::forward<ArgTs>(Args)...};
-}
+using type_identity_t // NOLINT(readability-identifier-naming)
+    = typename llvm::type_identity<T>::type;
 
 //===----------------------------------------------------------------------===//
 //     Features from C++23
diff --git a/llvm/include/llvm/ADT/SlowDynamicAPInt.h b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
index 702e0398e40f..a9db8a3ae00c 100644
--- a/llvm/include/llvm/ADT/SlowDynamicAPInt.h
+++ b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
@@ -62,18 +62,17 @@ class SlowDynamicAPInt {
   LLVM_ABI SlowDynamicAPInt &operator++();
   LLVM_ABI SlowDynamicAPInt &operator--();
 
-  LLVM_ABI_FRIEND friend SlowDynamicAPInt abs(const SlowDynamicAPInt &X);
-  LLVM_ABI_FRIEND friend SlowDynamicAPInt ceilDiv(const SlowDynamicAPInt &LHS,
-                                                  const SlowDynamicAPInt &RHS);
-  LLVM_ABI_FRIEND friend SlowDynamicAPInt floorDiv(const SlowDynamicAPInt &LHS,
-                                                   const SlowDynamicAPInt &RHS);
+  LLVM_ABI friend SlowDynamicAPInt abs(const SlowDynamicAPInt &X);
+  LLVM_ABI friend SlowDynamicAPInt ceilDiv(const SlowDynamicAPInt &LHS,
+                                           const SlowDynamicAPInt &RHS);
+  LLVM_ABI friend SlowDynamicAPInt floorDiv(const SlowDynamicAPInt &LHS,
+                                            const SlowDynamicAPInt &RHS);
   /// The operands must be non-negative for gcd.
-  LLVM_ABI_FRIEND friend SlowDynamicAPInt gcd(const SlowDynamicAPInt &A,
-                                              const SlowDynamicAPInt &B);
+  LLVM_ABI friend SlowDynamicAPInt gcd(const SlowDynamicAPInt &A,
+                                       const SlowDynamicAPInt &B);
 
   /// Overload to compute a hash_code for a SlowDynamicAPInt value.
-  LLVM_ABI_FRIEND friend hash_code
-  hash_value(const SlowDynamicAPInt &X); // NOLINT
+  LLVM_ABI friend hash_code hash_value(const SlowDynamicAPInt &X); // NOLINT
 
   // Make DynamicAPInt a friend so it can access Val directly.
   friend DynamicAPInt;
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 2ce934321c45..892040ef4cab 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -80,7 +80,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   explicit SmallPtrSetImplBase(const void **SmallStorage, unsigned SmallSize)
       : CurArray(SmallStorage), CurArraySize(SmallSize), NumNonEmpty(0),
         NumTombstones(0), IsSmall(true) {
-    assert(SmallSize && (SmallSize & (SmallSize-1)) == 0 &&
+    assert(llvm::has_single_bit(SmallSize) &&
            "Initial size must be a power of two!");
   }
 
diff --git a/llvm/include/llvm/ADT/TinyPtrVector.h b/llvm/include/llvm/ADT/TinyPtrVector.h
index 501500793509..8d7a07b5e9eb 100644
--- a/llvm/include/llvm/ADT/TinyPtrVector.h
+++ b/llvm/include/llvm/ADT/TinyPtrVector.h
@@ -132,47 +132,17 @@ class TinyPtrVector {
                        : Count == 1 ? PtrUnion(Value)
                                     : PtrUnion(new VecTy(Count, Value))) {}
 
-  // implicit conversion operator to ArrayRef.
-  operator ArrayRef<EltTy>() const {
-    if (Val.isNull())
-      return {};
-    if (isa<EltTy>(Val))
-      return *Val.getAddrOfPtr1();
-    return *cast<VecTy *>(Val);
-  }
-
-  // implicit conversion operator to MutableArrayRef.
-  operator MutableArrayRef<EltTy>() {
-    if (Val.isNull())
-      return {};
-    if (isa<EltTy>(Val))
-      return *Val.getAddrOfPtr1();
-    return *cast<VecTy *>(Val);
-  }
-
-  // Implicit conversion to ArrayRef<U> if EltTy* implicitly converts to U*.
-  template <
-      typename U,
-      std::enable_if_t<std::is_convertible<ArrayRef<EltTy>, ArrayRef<U>>::value,
-                       bool> = false>
-  operator ArrayRef<U>() const {
-    return operator ArrayRef<EltTy>();
-  }
-
   bool empty() const {
     // This vector can be empty if it contains no element, or if it
     // contains a pointer to an empty vector.
-    if (Val.isNull()) return true;
-    if (VecTy *Vec = dyn_cast_if_present<VecTy *>(Val))
-      return Vec->empty();
-    return false;
+    if (isa<EltTy>(Val))
+      return Val.isNull();
+    return cast<VecTy *>(Val)->empty();
   }
 
   unsigned size() const {
-    if (empty())
-      return 0;
     if (isa<EltTy>(Val))
-      return 1;
+      return Val.isNull() ? 0 : 1;
     return cast<VecTy *>(Val)->size();
   }
 
@@ -214,6 +184,9 @@ class TinyPtrVector {
     return const_reverse_iterator(begin());
   }
 
+  EltTy *data() { return begin(); }
+  const EltTy *data() const { return begin(); }
+
   EltTy operator[](unsigned i) const {
     assert(!Val.isNull() && "can't index into an empty vector");
     if (isa<EltTy>(Val)) {
@@ -250,8 +223,8 @@ class TinyPtrVector {
     // If we have a single value, convert to a vector.
     if (isa<EltTy>(Val)) {
       EltTy V = cast<EltTy>(Val);
-      Val = new VecTy();
-      cast<VecTy *>(Val)->push_back(V);
+      Val = new VecTy({V, NewVal});
+      return;
     }
 
     // Add the new value, we know we have a vector.
@@ -261,20 +234,19 @@ class TinyPtrVector {
   void pop_back() {
     // If we have a single value, convert to empty.
     if (isa<EltTy>(Val))
-      Val = (EltTy)nullptr;
-    else if (VecTy *Vec = cast<VecTy *>(Val))
-      Vec->pop_back();
+      Val = EltTy();
+    else
+      cast<VecTy *>(Val)->pop_back();
   }
 
   void clear() {
     // If we have a single value, convert to empty.
     if (isa<EltTy>(Val)) {
       Val = EltTy();
-    } else if (VecTy *Vec = dyn_cast_if_present<VecTy *>(Val)) {
+    } else {
       // If we have a vector form, just clear it.
-      Vec->clear();
+      cast<VecTy *>(Val)->clear();
     }
-    // Otherwise, we're already empty.
   }
 
   iterator erase(iterator I) {
@@ -285,10 +257,10 @@ class TinyPtrVector {
     if (isa<EltTy>(Val)) {
       if (I == begin())
         Val = EltTy();
-    } else if (VecTy *Vec = dyn_cast_if_present<VecTy *>(Val)) {
+    } else {
       // multiple items in a vector; just do the erase, there is no
       // benefit to collapsing back to a pointer
-      return Vec->erase(I);
+      return cast<VecTy *>(Val)->erase(I);
     }
     return end();
   }
@@ -301,8 +273,8 @@ class TinyPtrVector {
     if (isa<EltTy>(Val)) {
       if (S == begin() && S != E)
         Val = EltTy();
-    } else if (VecTy *Vec = dyn_cast_if_present<VecTy *>(Val)) {
-      return Vec->erase(S, E);
+    } else {
+      return cast<VecTy *>(Val)->erase(S, E);
     }
     return end();
   }
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 66ea5ecbc4bf..c9de5a4fa921 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -72,9 +72,6 @@ namespace bfi_detail {
 
 struct IrreducibleGraph;
 
-// This is part of a workaround for a GCC 4.7 crash on lambdas.
-template <class BT> struct BlockEdgesAdder;
-
 /// Mass of a block.
 ///
 /// This class implements a sort of fixed-point fraction always between 0.0 and
@@ -843,9 +840,6 @@ void IrreducibleGraph::addEdges(const BlockNode &Node,
 ///         (Running this until fixed point would "solve" the geometric
 ///         series by simulation.)
 template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
-  // This is part of a workaround for a GCC 4.7 crash on lambdas.
-  friend struct bfi_detail::BlockEdgesAdder<BT>;
-
   using BlockT = typename bfi_detail::TypeMap<BT>::BlockT;
   using BlockKeyT = typename bfi_detail::TypeMap<BT>::BlockKeyT;
   using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
@@ -1632,29 +1626,6 @@ BlockFrequencyInfoImplBase::Scaled64 BlockFrequencyInfoImpl<BT>::discrepancy(
 }
 #endif
 
-/// \note This should be a lambda, but that crashes GCC 4.7.
-namespace bfi_detail {
-
-template <class BT> struct BlockEdgesAdder {
-  using BlockT = BT;
-  using LoopData = BlockFrequencyInfoImplBase::LoopData;
-  using Successor = GraphTraits<const BlockT *>;
-
-  const BlockFrequencyInfoImpl<BT> &BFI;
-
-  explicit BlockEdgesAdder(const BlockFrequencyInfoImpl<BT> &BFI)
-      : BFI(BFI) {}
-
-  void operator()(IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr,
-                  const LoopData *OuterLoop) {
-    const BlockT *BB = BFI.RPOT[Irr.Node.Index];
-    for (const auto *Succ : children<const BlockT *>(BB))
-      G.addEdge(Irr, BFI.getNode(Succ), OuterLoop);
-  }
-};
-
-} // end namespace bfi_detail
-
 template <class BT>
 void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
     LoopData *OuterLoop, std::list<LoopData>::iterator Insert) {
@@ -1665,9 +1636,12 @@ void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
 
   using namespace bfi_detail;
 
-  // Ideally, addBlockEdges() would be declared here as a lambda, but that
-  // crashes GCC 4.7.
-  BlockEdgesAdder<BT> addBlockEdges(*this);
+  auto addBlockEdges = [&](IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr,
+                           const LoopData *OuterLoop) {
+    const BlockT *BB = RPOT[Irr.Node.Index];
+    for (const auto *Succ : children<const BlockT *>(BB))
+      G.addEdge(Irr, getNode(Succ), OuterLoop);
+  };
   IrreducibleGraph G(*this, OuterLoop, addBlockEdges);
 
   for (auto &L : analyzeIrreducible(G, OuterLoop, Insert))
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index 83a92c5eb39a..cf57f2c95b55 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -411,8 +411,8 @@ class BranchProbabilityInfo {
   std::optional<uint32_t> getInitialEstimatedBlockWeight(const BasicBlock *BB);
 
   // Computes estimated weights for all blocks in \p F.
-  void computeEestimateBlockWeight(const Function &F, DominatorTree *DT,
-                                   PostDominatorTree *PDT);
+  void estimateBlockWeights(const Function &F, DominatorTree *DT,
+                            PostDominatorTree *PDT);
 
   /// Based on computed weights by \p computeEstimatedBlockWeight set
   /// probabilities on branches.
diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index cfc21b3ec202..9e2dc1ad771c 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -34,7 +34,7 @@ namespace dxil {
 
 // Returns the resource name from dx_resource_handlefrombinding or
 // dx_resource_handlefromimplicitbinding call
-StringRef getResourceNameFromBindingCall(CallInst *CI);
+LLVM_ABI StringRef getResourceNameFromBindingCall(CallInst *CI);
 
 /// The dx.RawBuffer target extension type
 ///
@@ -387,7 +387,7 @@ class ResourceInfo {
 
   const ResourceBinding &getBinding() const { return Binding; }
   TargetExtType *getHandleTy() const { return HandleTy; }
-  const StringRef getName() const { return Name; }
+  StringRef getName() const { return Name; }
 
   bool hasSymbol() const { return Symbol; }
   LLVM_ABI GlobalVariable *createSymbol(Module &M, StructType *Ty);
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
index babb6d9d6cf0..38177491d6b7 100644
--- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -15,6 +15,7 @@
 #define LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/Compiler.h"
@@ -32,17 +33,19 @@ class FunctionPropertiesInfo {
   void updateAggregateStats(const Function &F, const LoopInfo &LI);
   void reIncludeBB(const BasicBlock &BB);
 
+  ir2vec::Embedding FunctionEmbedding = ir2vec::Embedding(0.0);
+  std::optional<ir2vec::Vocab> IR2VecVocab;
+
 public:
   LLVM_ABI static FunctionPropertiesInfo
   getFunctionPropertiesInfo(const Function &F, const DominatorTree &DT,
-                            const LoopInfo &LI);
+                            const LoopInfo &LI,
+                            const IR2VecVocabResult *VocabResult);
 
   LLVM_ABI static FunctionPropertiesInfo
   getFunctionPropertiesInfo(Function &F, FunctionAnalysisManager &FAM);
 
-  bool operator==(const FunctionPropertiesInfo &FPI) const {
-    return std::memcmp(this, &FPI, sizeof(FunctionPropertiesInfo)) == 0;
-  }
+  bool operator==(const FunctionPropertiesInfo &FPI) const;
 
   bool operator!=(const FunctionPropertiesInfo &FPI) const {
     return !(*this == FPI);
@@ -137,6 +140,19 @@ class FunctionPropertiesInfo {
   int64_t CallReturnsVectorPointerCount = 0;
   int64_t CallWithManyArgumentsCount = 0;
   int64_t CallWithPointerArgumentCount = 0;
+
+  const ir2vec::Embedding &getFunctionEmbedding() const {
+    return FunctionEmbedding;
+  }
+
+  const std::optional<ir2vec::Vocab> &getIR2VecVocab() const {
+    return IR2VecVocab;
+  }
+
+  // Helper intended to be useful for unittests
+  void setFunctionEmbeddingForTest(const ir2vec::Embedding &Embedding) {
+    FunctionEmbedding = Embedding;
+  }
 };
 
 // Analysis pass
@@ -193,6 +209,7 @@ class FunctionPropertiesUpdater {
   DominatorTree &getUpdatedDominatorTree(FunctionAnalysisManager &FAM) const;
 
   DenseSet<const BasicBlock *> Successors;
+  DenseSet<const BasicBlock *> CallUsers;
 
   // Edges we might potentially need to remove from the dominator tree.
   SmallVector<DominatorTree::UpdateType, 2> DomTreeUpdates;
diff --git a/llvm/include/llvm/Analysis/IR2Vec.h b/llvm/include/llvm/Analysis/IR2Vec.h
index de67955d85d7..ef8f630d7feb 100644
--- a/llvm/include/llvm/Analysis/IR2Vec.h
+++ b/llvm/include/llvm/Analysis/IR2Vec.h
@@ -32,6 +32,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/JSON.h"
 #include <map>
@@ -57,9 +58,9 @@ enum class IR2VecKind { Symbolic };
 
 namespace ir2vec {
 
-extern cl::opt<float> OpcWeight;
-extern cl::opt<float> TypeWeight;
-extern cl::opt<float> ArgWeight;
+LLVM_ABI extern cl::opt<float> OpcWeight;
+LLVM_ABI extern cl::opt<float> TypeWeight;
+LLVM_ABI extern cl::opt<float> ArgWeight;
 
 /// Embedding is a datatype that wraps std::vector<double>. It provides
 /// additional functionality for arithmetic and comparison operations.
@@ -106,16 +107,23 @@ struct Embedding {
   const std::vector<double> &getData() const { return Data; }
 
   /// Arithmetic operators
-  Embedding &operator+=(const Embedding &RHS);
-  Embedding &operator-=(const Embedding &RHS);
+  LLVM_ABI Embedding &operator+=(const Embedding &RHS);
+  LLVM_ABI Embedding operator+(const Embedding &RHS) const;
+  LLVM_ABI Embedding &operator-=(const Embedding &RHS);
+  LLVM_ABI Embedding operator-(const Embedding &RHS) const;
+  LLVM_ABI Embedding &operator*=(double Factor);
+  LLVM_ABI Embedding operator*(double Factor) const;
 
   /// Adds Src Embedding scaled by Factor with the called Embedding.
   /// Called_Embedding += Src * Factor
-  Embedding &scaleAndAdd(const Embedding &Src, float Factor);
+  LLVM_ABI Embedding &scaleAndAdd(const Embedding &Src, float Factor);
 
   /// Returns true if the embedding is approximately equal to the RHS embedding
   /// within the specified tolerance.
-  bool approximatelyEquals(const Embedding &RHS, double Tolerance = 1e-6) const;
+  LLVM_ABI bool approximatelyEquals(const Embedding &RHS,
+                                    double Tolerance = 1e-4) const;
+
+  void print(raw_ostream &OS) const;
 };
 
 using InstEmbeddingsMap = DenseMap<const Instruction *, Embedding>;
@@ -148,7 +156,7 @@ class Embedder {
   mutable BBEmbeddingsMap BBVecMap;
   mutable InstEmbeddingsMap InstVecMap;
 
-  Embedder(const Function &F, const Vocab &Vocabulary);
+  LLVM_ABI Embedder(const Function &F, const Vocab &Vocabulary);
 
   /// Helper function to compute embeddings. It generates embeddings for all
   /// the instructions and basic blocks in the function F. Logic of computing
@@ -161,38 +169,38 @@ class Embedder {
 
   /// Lookup vocabulary for a given Key. If the key is not found, it returns a
   /// zero vector.
-  Embedding lookupVocab(const std::string &Key) const;
+  LLVM_ABI Embedding lookupVocab(const std::string &Key) const;
 
 public:
   virtual ~Embedder() = default;
 
   /// Factory method to create an Embedder object.
-  static Expected<std::unique_ptr<Embedder>>
+  LLVM_ABI static std::unique_ptr<Embedder>
   create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary);
 
   /// Returns a map containing instructions and the corresponding embeddings for
   /// the function F if it has been computed. If not, it computes the embeddings
   /// for the function and returns the map.
-  const InstEmbeddingsMap &getInstVecMap() const;
+  LLVM_ABI const InstEmbeddingsMap &getInstVecMap() const;
 
   /// Returns a map containing basic block and the corresponding embeddings for
   /// the function F if it has been computed. If not, it computes the embeddings
   /// for the function and returns the map.
-  const BBEmbeddingsMap &getBBVecMap() const;
+  LLVM_ABI const BBEmbeddingsMap &getBBVecMap() const;
 
   /// Returns the embedding for a given basic block in the function F if it has
   /// been computed. If not, it computes the embedding for the basic block and
   /// returns it.
-  const Embedding &getBBVector(const BasicBlock &BB) const;
+  LLVM_ABI const Embedding &getBBVector(const BasicBlock &BB) const;
 
   /// Computes and returns the embedding for the current function.
-  const Embedding &getFunctionVector() const;
+  LLVM_ABI const Embedding &getFunctionVector() const;
 };
 
 /// Class for computing the Symbolic embeddings of IR2Vec.
 /// Symbolic embeddings are constructed based on the entity-level
 /// representations obtained from the Vocabulary.
-class SymbolicEmbedder : public Embedder {
+class LLVM_ABI SymbolicEmbedder : public Embedder {
 private:
   /// Utility function to compute the embedding for a given type.
   Embedding getTypeEmbedding(const Type *Ty) const;
@@ -219,13 +227,13 @@ class IR2VecVocabResult {
 
 public:
   IR2VecVocabResult() = default;
-  IR2VecVocabResult(ir2vec::Vocab &&Vocabulary);
+  LLVM_ABI IR2VecVocabResult(ir2vec::Vocab &&Vocabulary);
 
   bool isValid() const { return Valid; }
-  const ir2vec::Vocab &getVocabulary() const;
-  unsigned getDimension() const;
-  bool invalidate(Module &M, const PreservedAnalyses &PA,
-                  ModuleAnalysisManager::Invalidator &Inv) const;
+  LLVM_ABI const ir2vec::Vocab &getVocabulary() const;
+  LLVM_ABI unsigned getDimension() const;
+  LLVM_ABI bool invalidate(Module &M, const PreservedAnalyses &PA,
+                           ModuleAnalysisManager::Invalidator &Inv) const;
 };
 
 /// This analysis provides the vocabulary for IR2Vec. The vocabulary provides a
@@ -234,25 +242,36 @@ class IR2VecVocabResult {
 class IR2VecVocabAnalysis : public AnalysisInfoMixin<IR2VecVocabAnalysis> {
   ir2vec::Vocab Vocabulary;
   Error readVocabulary();
+  Error parseVocabSection(StringRef Key, const json::Value &ParsedVocabValue,
+                          ir2vec::Vocab &TargetVocab, unsigned &Dim);
   void emitError(Error Err, LLVMContext &Ctx);
 
 public:
-  static AnalysisKey Key;
+  LLVM_ABI static AnalysisKey Key;
   IR2VecVocabAnalysis() = default;
-  explicit IR2VecVocabAnalysis(const ir2vec::Vocab &Vocab);
-  explicit IR2VecVocabAnalysis(ir2vec::Vocab &&Vocab);
+  LLVM_ABI explicit IR2VecVocabAnalysis(const ir2vec::Vocab &Vocab);
+  LLVM_ABI explicit IR2VecVocabAnalysis(ir2vec::Vocab &&Vocab);
   using Result = IR2VecVocabResult;
-  Result run(Module &M, ModuleAnalysisManager &MAM);
+  LLVM_ABI Result run(Module &M, ModuleAnalysisManager &MAM);
 };
 
 /// This pass prints the IR2Vec embeddings for instructions, basic blocks, and
 /// functions.
 class IR2VecPrinterPass : public PassInfoMixin<IR2VecPrinterPass> {
   raw_ostream &OS;
-  void printVector(const ir2vec::Embedding &Vec) const;
 
 public:
   explicit IR2VecPrinterPass(raw_ostream &OS) : OS(OS) {}
+  LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+  static bool isRequired() { return true; }
+};
+
+/// This pass prints the embeddings in the vocabulary
+class IR2VecVocabPrinterPass : public PassInfoMixin<IR2VecVocabPrinterPass> {
+  raw_ostream &OS;
+
+public:
+  explicit IR2VecVocabPrinterPass(raw_ostream &OS) : OS(OS) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
   static bool isRequired() { return true; }
 };
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 463249461483..3b92cbff28de 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -54,6 +54,9 @@ enum class RecurKind {
   FMulAdd,  ///< Sum of float products with llvm.fmuladd(a * b + sum).
   AnyOf,    ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
             ///< loop invariant, and both x and y are integer type.
+  FindFirstIVSMin, /// FindFirst reduction with select(icmp(),x,y) where one of
+                   ///< (x,y) is a decreasing loop induction, and both x and y
+                   ///< are integer type, producing a SMin reduction.
   FindLastIVSMax, ///< FindLast reduction with select(cmp(),x,y) where one of
                   ///< (x,y) is increasing loop induction, and both x and y
                   ///< are integer type, producing a SMax reduction.
@@ -165,13 +168,13 @@ class RecurrenceDescriptor {
   /// Returns a struct describing whether the instruction is either a
   ///   Select(ICmp(A, B), X, Y), or
   ///   Select(FCmp(A, B), X, Y)
-  /// where one of (X, Y) is an increasing loop induction variable, and the
-  /// other is a PHI value.
+  /// where one of (X, Y) is an increasing (FindLast) or decreasing (FindFirst)
+  /// loop induction variable, and the other is a PHI value.
   // TODO: Support non-monotonic variable. FindLast does not need be restricted
   // to increasing loop induction variables.
-  LLVM_ABI static InstDesc isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
-                                               Instruction *I,
-                                               ScalarEvolution &SE);
+  LLVM_ABI static InstDesc isFindIVPattern(RecurKind Kind, Loop *TheLoop,
+                                           PHINode *OrigPhi, Instruction *I,
+                                           ScalarEvolution &SE);
 
   /// Returns a struct describing if the instruction is a
   /// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
@@ -259,6 +262,12 @@ class RecurrenceDescriptor {
     return Kind == RecurKind::AnyOf;
   }
 
+  /// Returns true if the recurrence kind is of the form
+  ///   select(cmp(),x,y) where one of (x,y) is decreasing loop induction.
+  static bool isFindFirstIVRecurrenceKind(RecurKind Kind) {
+    return Kind == RecurKind::FindFirstIVSMin;
+  }
+
   /// Returns true if the recurrence kind is of the form
   ///   select(cmp(),x,y) where one of (x,y) is increasing loop induction.
   static bool isFindLastIVRecurrenceKind(RecurKind Kind) {
@@ -269,22 +278,35 @@ class RecurrenceDescriptor {
   /// Returns true if recurrece kind is a signed redux kind.
   static bool isSignedRecurrenceKind(RecurKind Kind) {
     return Kind == RecurKind::SMax || Kind == RecurKind::SMin ||
+           Kind == RecurKind::FindFirstIVSMin ||
            Kind == RecurKind::FindLastIVSMax;
   }
 
+  /// Returns true if the recurrence kind is of the form
+  ///   select(cmp(),x,y) where one of (x,y) is an increasing or decreasing loop
+  ///   induction.
+  static bool isFindIVRecurrenceKind(RecurKind Kind) {
+    return isFindFirstIVRecurrenceKind(Kind) ||
+           isFindLastIVRecurrenceKind(Kind);
+  }
+
   /// Returns the type of the recurrence. This type can be narrower than the
   /// actual type of the Phi if the recurrence has been type-promoted.
   Type *getRecurrenceType() const { return RecurrenceType; }
 
-  /// Returns the sentinel value for FindLastIV recurrences to replace the start
-  /// value.
+  /// Returns the sentinel value for FindFirstIV & FindLastIV recurrences to
+  /// replace the start value.
   Value *getSentinelValue() const {
-    assert(isFindLastIVRecurrenceKind(Kind) && "Unexpected recurrence kind");
     Type *Ty = StartValue->getType();
     unsigned BW = Ty->getIntegerBitWidth();
+    if (isFindLastIVRecurrenceKind(Kind)) {
+      return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
+                                      ? APInt::getSignedMinValue(BW)
+                                      : APInt::getMinValue(BW));
+    }
     return ConstantInt::get(Ty, isSignedRecurrenceKind(Kind)
-                                    ? APInt::getSignedMinValue(BW)
-                                    : APInt::getMinValue(BW));
+                                    ? APInt::getSignedMaxValue(BW)
+                                    : APInt::getMaxValue(BW));
   }
 
   /// Returns a reference to the instructions used for type-promoting the
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 9d15136e81d1..50ba3c13da70 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -331,6 +331,10 @@ class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
   };
 
   Result run(Module &M, ModuleAnalysisManager &MAM) { return Result(M, MAM); }
+
+private:
+  static bool initializeIR2VecVocabIfRequested(Module &M,
+                                               ModuleAnalysisManager &MAM);
 };
 
 /// Printer pass for the InlineAdvisorAnalysis results.
diff --git a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
index 961d5091bf9f..25a35df3efe2 100644
--- a/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
+++ b/llvm/include/llvm/Analysis/InlineModelFeatureMaps.h
@@ -132,6 +132,11 @@ constexpr bool isHeuristicInlineCostFeature(InlineCostFeatureIndex Feature) {
     "not "                                                                     \
     "fully inlined by ElimAvailExtern)")
 
+// Not all features listed in FeatureIndex are used by the ML model.
+// Specifically, callee_embedding and caller_embedding are used only when the
+// usage of IR2Vec embeddings is explicitly enabled. Meaning, the size/number of
+// features is not static. So, we cannot determine number of features based on
+// the number of elements in this enum.
 // clang-format off
 enum class FeatureIndex : size_t {
 #define POPULATE_INDICES(DTYPE, SHAPE, NAME, COMMENT) NAME,
@@ -142,7 +147,11 @@ enum class FeatureIndex : size_t {
   INLINE_FEATURE_ITERATOR(POPULATE_INDICES)
 #undef POPULATE_INDICES
 
-  NumberOfFeatures
+// IR2Vec embeddings
+// Dimensions of embeddings are not known in the compile time (until vocab is 
+// read). Hence macros cannot be used here.
+  callee_embedding,
+  caller_embedding
 };
 // clang-format on
 
@@ -151,10 +160,7 @@ inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature) {
   return static_cast<FeatureIndex>(static_cast<size_t>(Feature));
 }
 
-constexpr size_t NumberOfFeatures =
-    static_cast<size_t>(FeatureIndex::NumberOfFeatures);
-
-LLVM_ABI extern const std::vector<TensorSpec> FeatureMap;
+LLVM_ABI extern std::vector<TensorSpec> FeatureMap;
 
 LLVM_ABI extern const char *const DecisionName;
 LLVM_ABI extern const TensorSpec InlineDecisionSpec;
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 1faf279ae201..1415da14a349 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -396,8 +396,6 @@ class MemoryDepChecker {
     uint64_t MaxStride;
     std::optional<uint64_t> CommonStride;
 
-    bool ShouldRetryWithRuntimeCheck;
-
     /// TypeByteSize is either the common store size of both accesses, or 0 when
     /// store sizes mismatch.
     uint64_t TypeByteSize;
@@ -407,11 +405,9 @@ class MemoryDepChecker {
 
     DepDistanceStrideAndSizeInfo(const SCEV *Dist, uint64_t MaxStride,
                                  std::optional<uint64_t> CommonStride,
-                                 bool ShouldRetryWithRuntimeCheck,
                                  uint64_t TypeByteSize, bool AIsWrite,
                                  bool BIsWrite)
         : Dist(Dist), MaxStride(MaxStride), CommonStride(CommonStride),
-          ShouldRetryWithRuntimeCheck(ShouldRetryWithRuntimeCheck),
           TypeByteSize(TypeByteSize), AIsWrite(AIsWrite), BIsWrite(BIsWrite) {}
   };
 
diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
index 580dd5e95d76..8262dd0846ed 100644
--- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
@@ -82,6 +82,7 @@ class MLInlineAdvisor : public InlineAdvisor {
   int64_t NodeCount = 0;
   int64_t EdgeCount = 0;
   int64_t EdgesOfLastSeenNodes = 0;
+  const bool UseIR2Vec;
 
   std::map<const LazyCallGraph::Node *, unsigned> FunctionLevels;
   const int32_t InitialIRSize = 0;
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
index 8e9d7e0b7214..09e3945f5a8f 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
@@ -64,6 +64,9 @@ inline class_match<const SCEV> m_SCEV() { return class_match<const SCEV>(); }
 inline class_match<const SCEVConstant> m_SCEVConstant() {
   return class_match<const SCEVConstant>();
 }
+inline class_match<const SCEVVScale> m_SCEVVScale() {
+  return class_match<const SCEVVScale>();
+}
 
 template <typename Class> struct bind_ty {
   Class *&VR;
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index ead06481b87f..4df086b277e4 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -136,7 +136,7 @@ class TargetLibraryInfoImpl {
     AMDLIBM      // AMD Math Vector library.
   };
 
-  LLVM_ABI TargetLibraryInfoImpl();
+  LLVM_ABI TargetLibraryInfoImpl() = delete;
   LLVM_ABI explicit TargetLibraryInfoImpl(const Triple &T);
 
   // Provide value semantics.
@@ -294,6 +294,8 @@ class TargetLibraryInfo {
   std::bitset<NumLibFuncs> OverrideAsUnavailable;
 
 public:
+  TargetLibraryInfo() = delete;
+
   explicit TargetLibraryInfo(const TargetLibraryInfoImpl &Impl,
                              std::optional<const Function *> F = std::nullopt)
       : Impl(&Impl) {
@@ -649,7 +651,11 @@ class LLVM_ABI TargetLibraryInfoWrapperPass : public ImmutablePass {
 
 public:
   static char ID;
+
+  /// The default constructor should not be used and is only for pass manager
+  /// initialization purposes.
   TargetLibraryInfoWrapperPass();
+
   explicit TargetLibraryInfoWrapperPass(const Triple &T);
   explicit TargetLibraryInfoWrapperPass(const TargetLibraryInfoImpl &TLI);
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 90d92e0fcf55..c43870392361 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1325,9 +1325,21 @@ class TargetTransformInfo {
   /// \return The cost of a partial reduction, which is a reduction from a
   /// vector to another vector with fewer elements of larger size. They are
   /// represented by the llvm.experimental.partial.reduce.add intrinsic, which
-  /// takes an accumulator and a binary operation operand that itself is fed by
-  /// two extends. An example of an operation that uses a partial reduction is a
-  /// dot product, which reduces two vectors to another of 4 times fewer and 4
+  /// takes an accumulator of type \p AccumType and a second vector operand to
+  /// be accumulated, whose element count is specified by \p VF. The type of
+  /// reduction is specified by \p Opcode. The second operand passed to the
+  /// intrinsic could be the result of an extend, such as sext or zext. In
+  /// this case \p BinOp is nullopt, \p InputTypeA represents the type being
+  /// extended and \p OpAExtend the operation, i.e. sign- or zero-extend.
+  /// Also, \p InputTypeB should be nullptr and OpBExtend should be None.
+  /// Alternatively, the second operand could be the result of a binary
+  /// operation performed on two extends, i.e.
+  ///   mul(zext i8 %a -> i32, zext i8 %b -> i32).
+  /// In this case \p BinOp may specify the opcode of the binary operation,
+  /// \p InputTypeA and \p InputTypeB the types being extended, and
+  /// \p OpAExtend, \p OpBExtend the form of extensions. An example of an
+  /// operation that uses a partial reduction is a dot product, which reduces
+  /// two vectors in binary mul operation to another of 4 times fewer and 4
   /// times larger elements.
   LLVM_ABI InstructionCost getPartialReductionCost(
       unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index c22928c9bcd9..12f87226c5f5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1534,11 +1534,15 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       ArrayRef<int> Mask = Shuffle->getShuffleMask();
       int NumSubElts, SubIndex;
 
+      // Treat undef/poison mask as free (no matter the length).
+      if (all_of(Mask, [](int M) { return M < 0; }))
+        return TTI::TCC_Free;
+
       // TODO: move more of this inside improveShuffleKindFromMask.
       if (Shuffle->changesLength()) {
         // Treat a 'subvector widening' as a free shuffle.
         if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding())
-          return 0;
+          return TTI::TCC_Free;
 
         if (Shuffle->isExtractSubvectorMask(SubIndex))
           return TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, VecTy,
@@ -1599,18 +1603,13 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
       }
 
       if (Shuffle->isIdentity())
-        return 0;
+        return TTI::TCC_Free;
 
       if (Shuffle->isReverse())
         return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, VecSrcTy, Mask,
                                          CostKind, 0, nullptr, Operands,
                                          Shuffle);
 
-      if (Shuffle->isSelect())
-        return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, VecSrcTy, Mask,
-                                         CostKind, 0, nullptr, Operands,
-                                         Shuffle);
-
       if (Shuffle->isTranspose())
         return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, VecSrcTy,
                                          Mask, CostKind, 0, nullptr, Operands,
@@ -1632,6 +1631,11 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
             FixedVectorType::get(VecTy->getScalarType(), NumSubElts), Operands,
             Shuffle);
 
+      if (Shuffle->isSelect())
+        return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, VecSrcTy, Mask,
+                                         CostKind, 0, nullptr, Operands,
+                                         Shuffle);
+
       if (Shuffle->isSplice(SubIndex))
         return TargetTTI->getShuffleCost(TTI::SK_Splice, VecTy, VecSrcTy, Mask,
                                          CostKind, SubIndex, nullptr, Operands,
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 4596b2563c1d..02990a3cb44f 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -41,6 +41,7 @@ class LoopInfo;
 class MDNode;
 class StringRef;
 class TargetLibraryInfo;
+class IntrinsicInst;
 template <typename T> class ArrayRef;
 
 constexpr unsigned MaxAnalysisRecursionDepth = 6;
@@ -311,11 +312,11 @@ LLVM_ABI std::optional<bool> computeKnownFPSignBit(const Value *V,
 
 /// Return true if the sign bit of the FP value can be ignored by the user when
 /// the value is zero.
-bool canIgnoreSignBitOfZero(const Use &U);
+LLVM_ABI bool canIgnoreSignBitOfZero(const Use &U);
 
 /// Return true if the sign bit of the FP value can be ignored by the user when
 /// the value is NaN.
-bool canIgnoreSignBitOfNaN(const Use &U);
+LLVM_ABI bool canIgnoreSignBitOfNaN(const Use &U);
 
 /// If the specified value can be set by repeating the same byte in memory,
 /// return the i8 value that it is represented with. This is true for all i8
@@ -726,6 +727,9 @@ LLVM_ABI bool isGuaranteedToExecuteForEveryIteration(const Instruction *I,
 /// getGuaranteedNonPoisonOp.
 LLVM_ABI bool propagatesPoison(const Use &PoisonOp);
 
+/// Return whether this intrinsic propagates poison for all operands.
+LLVM_ABI bool intrinsicPropagatesPoison(Intrinsic::ID IID);
+
 /// Return true if the given instruction must trigger undefined behavior
 /// when I is executed with any operands which appear in KnownPoison holding
 /// a poison value at the point of execution.
@@ -965,6 +969,21 @@ LLVM_ABI bool matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
 LLVM_ABI bool matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
                                     Value *&Start, Value *&Step);
 
+/// Attempt to match a simple value-accumulating recurrence of the form:
+///   %llvm.intrinsic.acc = phi Ty [%Init, %Entry], [%llvm.intrinsic, %backedge]
+///   %llvm.intrinsic = call Ty @llvm.intrinsic(%OtherOp, %llvm.intrinsic.acc)
+/// OR
+///   %llvm.intrinsic.acc = phi Ty [%Init, %Entry], [%llvm.intrinsic, %backedge]
+///   %llvm.intrinsic = call Ty @llvm.intrinsic(%llvm.intrinsic.acc, %OtherOp)
+///
+/// The recurrence relation is of kind:
+///   X_0 = %a (initial value),
+///   X_i = call @llvm.binary.intrinsic(X_i-1, %b)
+/// Where %b is not required to be loop-invariant.
+LLVM_ABI bool matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
+                                                   PHINode *&P, Value *&Init,
+                                                   Value *&OtherOp);
+
 /// Return true if RHS is known to be implied true by LHS.  Return false if
 /// RHS is known to be implied false by LHS.  Otherwise, return std::nullopt if
 /// no implication can be made. A & B must be i1 (boolean) values or a vector of
diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 56c9e5330867..89abca02efef 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -157,23 +157,33 @@ enum class FeatureFlags : uint64_t {
 static_assert((uint64_t)FeatureFlags::NextUnusedBit <= 1ull << 63,
               "Shader flag bits exceed enum size.");
 
-#define ROOT_ELEMENT_FLAG(Num, Val) Val = Num,
-enum class RootElementFlag : uint32_t {
+#define ROOT_SIGNATURE_FLAG(Num, Val) Val = Num,
+enum class RootFlags : uint32_t {
 #include "DXContainerConstants.def"
+
+  LLVM_MARK_AS_BITMASK_ENUM(SamplerHeapDirectlyIndexed)
 };
 
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) Val = Num,
-enum class RootDescriptorFlag : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<RootFlags>> getRootFlags();
+
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag) Enum = Num,
+enum class RootDescriptorFlags : uint32_t {
 #include "DXContainerConstants.def"
+
+  LLVM_MARK_AS_BITMASK_ENUM(DataStatic)
 };
 
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) Val = Num,
-enum class DescriptorRangeFlag : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<RootDescriptorFlags>> getRootDescriptorFlags();
+
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag) Enum = Num,
+enum class DescriptorRangeFlags : uint32_t {
 #include "DXContainerConstants.def"
 
-  LLVM_MARK_AS_BITMASK_ENUM(DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+  LLVM_MARK_AS_BITMASK_ENUM(DescriptorsStaticKeepingBufferBoundsChecks)
 };
 
+LLVM_ABI ArrayRef<EnumEntry<DescriptorRangeFlags>> getDescriptorRangeFlags();
+
 #define ROOT_PARAMETER(Val, Enum) Enum = Val,
 enum class RootParameterType : uint32_t {
 #include "DXContainerConstants.def"
@@ -186,7 +196,7 @@ enum class DescriptorRangeType : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-ArrayRef<EnumEntry<DescriptorRangeType>> getDescriptorRangeTypes();
+LLVM_ABI ArrayRef<EnumEntry<DescriptorRangeType>> getDescriptorRangeTypes();
 
 #define ROOT_PARAMETER(Val, Enum)                                              \
   case Val:                                                                    \
@@ -215,26 +225,34 @@ inline bool isValidShaderVisibility(uint32_t V) {
   return false;
 }
 
-#define STATIC_SAMPLER_FILTER(Val, Enum) Enum = Val,
-enum class StaticSamplerFilter : uint32_t {
+#define FILTER(Val, Enum) Enum = Val,
+enum class SamplerFilter : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<SamplerFilter>> getSamplerFilters();
+
 #define TEXTURE_ADDRESS_MODE(Val, Enum) Enum = Val,
 enum class TextureAddressMode : uint32_t {
 #include "DXContainerConstants.def"
 };
 
-#define COMPARISON_FUNCTION(Val, Enum) Enum = Val,
-enum class SamplersComparisonFunction : uint32_t {
+LLVM_ABI ArrayRef<EnumEntry<TextureAddressMode>> getTextureAddressModes();
+
+#define COMPARISON_FUNC(Val, Enum) Enum = Val,
+enum class ComparisonFunc : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<ComparisonFunc>> getComparisonFuncs();
+
 #define STATIC_BORDER_COLOR(Val, Enum) Enum = Val,
-enum class SamplersBorderColor : uint32_t {
+enum class StaticBorderColor : uint32_t {
 #include "DXContainerConstants.def"
 };
 
+LLVM_ABI ArrayRef<EnumEntry<StaticBorderColor>> getStaticBorderColors();
+
 LLVM_ABI PartType parsePartType(StringRef S);
 
 struct VertexPSVInfo {
@@ -740,16 +758,31 @@ struct RootDescriptor : public v1::RootDescriptor {
   }
 };
 
-struct DescriptorRange : public v1::DescriptorRange {
+struct DescriptorRange {
+  uint32_t RangeType;
+  uint32_t NumDescriptors;
+  uint32_t BaseShaderRegister;
+  uint32_t RegisterSpace;
   uint32_t Flags;
+  uint32_t OffsetInDescriptorsFromTableStart;
   void swapBytes() {
-    v1::DescriptorRange::swapBytes();
+    sys::swapByteOrder(RangeType);
+    sys::swapByteOrder(NumDescriptors);
+    sys::swapByteOrder(BaseShaderRegister);
+    sys::swapByteOrder(RegisterSpace);
+    sys::swapByteOrder(OffsetInDescriptorsFromTableStart);
     sys::swapByteOrder(Flags);
   }
 };
 } // namespace v2
 } // namespace RTS0
 
+// D3D_ROOT_SIGNATURE_VERSION
+enum class RootSignatureVersion {
+  V1_0 = 0x1,
+  V1_1 = 0x2,
+};
+
 } // namespace dxbc
 } // namespace llvm
 
diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index a281256a44ae..8730820dd8b0 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -55,47 +55,52 @@ SHADER_FEATURE_FLAG(33, 39, NextUnusedBit, "Next reserved shader flag bit (not a
 #undef SHADER_FEATURE_FLAG
 #endif // SHADER_FEATURE_FLAG
 
-
-// ROOT_ELEMENT_FLAG(bit offset for the flag, name).
-#ifdef ROOT_ELEMENT_FLAG
-
-ROOT_ELEMENT_FLAG(0, NONE)
-ROOT_ELEMENT_FLAG(0x1, AllowInputAssemblerInputLayout)
-ROOT_ELEMENT_FLAG(0x2, DenyVertexShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x4, DenyHullShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x8, DenyDomainShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x10, DenyGeometryShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x20, DenyPixelShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x40, AllowStreamOutput)
-ROOT_ELEMENT_FLAG(0x80, LocalRootSignature)
-ROOT_ELEMENT_FLAG(0x100, DenyAmplificationShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x200, DenyMeshShaderRootAccess)
-ROOT_ELEMENT_FLAG(0x400, CBVSRVUAVHeapDirectlyIndexed)
-ROOT_ELEMENT_FLAG(0x800, SamplerHeapDirectlyIndexed)
-#undef ROOT_ELEMENT_FLAG
-#endif // ROOT_ELEMENT_FLAG
-
+// Below is the definition of root signature enumerations and flags. The
+// definitions of all values here correspond to their description in the
+// d3d12.h header and are carried over from their values in DXC. For reference:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/
+
+// ROOT_SIGNATURE_FLAG(flag value, name).
+#ifdef ROOT_SIGNATURE_FLAG
+
+ROOT_SIGNATURE_FLAG(0, None)
+ROOT_SIGNATURE_FLAG(0x1, AllowInputAssemblerInputLayout)
+ROOT_SIGNATURE_FLAG(0x2, DenyVertexShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x4, DenyHullShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x8, DenyDomainShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x10, DenyGeometryShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x20, DenyPixelShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x40, AllowStreamOutput)
+ROOT_SIGNATURE_FLAG(0x80, LocalRootSignature)
+ROOT_SIGNATURE_FLAG(0x100, DenyAmplificationShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x200, DenyMeshShaderRootAccess)
+ROOT_SIGNATURE_FLAG(0x400, CBVSRVUAVHeapDirectlyIndexed)
+ROOT_SIGNATURE_FLAG(0x800, SamplerHeapDirectlyIndexed)
+
+#undef ROOT_SIGNATURE_FLAG
+#endif // ROOT_SIGNATURE_FLAG
  
-// ROOT_DESCRIPTOR_FLAG(bit offset for the flag, name).
+// ROOT_DESCRIPTOR_FLAG(flag value, name, flag).
 #ifdef ROOT_DESCRIPTOR_FLAG
 
-ROOT_DESCRIPTOR_FLAG(0, NONE)
-ROOT_DESCRIPTOR_FLAG(0x2, DATA_VOLATILE)
-ROOT_DESCRIPTOR_FLAG(0x4, DATA_STATIC_WHILE_SET_AT_EXECUTE)
-ROOT_DESCRIPTOR_FLAG(0x8, DATA_STATIC)
+ROOT_DESCRIPTOR_FLAG(0, None, NONE)
+ROOT_DESCRIPTOR_FLAG(0x2, DataVolatile, DATA_VOLATILE)
+ROOT_DESCRIPTOR_FLAG(0x4, DataStaticWhileSetAtExecute, DATA_STATIC_WHILE_SET_AT_EXECUTE)
+ROOT_DESCRIPTOR_FLAG(0x8, DataStatic, DATA_STATIC)
+
 #undef ROOT_DESCRIPTOR_FLAG
 #endif // ROOT_DESCRIPTOR_FLAG
 
-
-// DESCRIPTOR_RANGE_FLAG(bit offset for the flag, name).
+// DESCRIPTOR_RANGE_FLAG(flag value, name, flag).
 #ifdef DESCRIPTOR_RANGE_FLAG
 
-DESCRIPTOR_RANGE_FLAG(0, NONE)
-DESCRIPTOR_RANGE_FLAG(0x1, DESCRIPTORS_VOLATILE)
-DESCRIPTOR_RANGE_FLAG(0x2, DATA_VOLATILE)
-DESCRIPTOR_RANGE_FLAG(0x4, DATA_STATIC_WHILE_SET_AT_EXECUTE)
-DESCRIPTOR_RANGE_FLAG(0x8, DATA_STATIC)
-DESCRIPTOR_RANGE_FLAG(0x10000, DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+DESCRIPTOR_RANGE_FLAG(0, None, NONE)
+DESCRIPTOR_RANGE_FLAG(0x1, DescriptorsVolatile, DESCRIPTORS_VOLATILE)
+DESCRIPTOR_RANGE_FLAG(0x2, DataVolatile, DATA_VOLATILE)
+DESCRIPTOR_RANGE_FLAG(0x4, DataStaticWhileSetAtExecute, DATA_STATIC_WHILE_SET_AT_EXECUTE)
+DESCRIPTOR_RANGE_FLAG(0x8, DataStatic, DATA_STATIC)
+DESCRIPTOR_RANGE_FLAG(0x10000, DescriptorsStaticKeepingBufferBoundsChecks, DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
+
 #undef DESCRIPTOR_RANGE_FLAG
 #endif // DESCRIPTOR_RANGE_FLAG
 
@@ -119,6 +124,7 @@ ROOT_PARAMETER(4, UAV)
 #undef ROOT_PARAMETER
 #endif // ROOT_PARAMETER
 
+// SHADER_VISIBILITY(value, enum).
 #ifdef SHADER_VISIBILITY
 
 SHADER_VISIBILITY(0, All)
@@ -129,79 +135,89 @@ SHADER_VISIBILITY(4, Geometry)
 SHADER_VISIBILITY(5, Pixel)
 SHADER_VISIBILITY(6, Amplification)
 SHADER_VISIBILITY(7, Mesh)
+
 #undef SHADER_VISIBILITY
 #endif // SHADER_VISIBILITY
 
-#ifdef STATIC_SAMPLER_FILTER
-
-STATIC_SAMPLER_FILTER(0, MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x1, MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x4, MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x5, MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x10, MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x11, MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x14, MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x15, MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x55, ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x80, COMPARISON_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x81, COMPARISON_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x84, COMPARISON_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x85, COMPARISON_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x90, COMPARISON_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x91, COMPARISON_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x94, COMPARISON_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x95, COMPARISON_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0xd5, COMPARISON_ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x100, MINIMUM_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x101, MINIMUM_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x104, MINIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x105, MINIMUM_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x110, MINIMUM_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x111, MINIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x114, MINIMUM_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x115, MINIMUM_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x155, MINIMUM_ANISOTROPIC)
-STATIC_SAMPLER_FILTER(0x180, MAXIMUM_MIN_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x181, MAXIMUM_MIN_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x184, MAXIMUM_MIN_POINT_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x185, MAXIMUM_MIN_POINT_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x190, MAXIMUM_MIN_LINEAR_MAG_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x191, MAXIMUM_MIN_LINEAR_MAG_POINT_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x194, MAXIMUM_MIN_MAG_LINEAR_MIP_POINT)
-STATIC_SAMPLER_FILTER(0x195, MAXIMUM_MIN_MAG_MIP_LINEAR)
-STATIC_SAMPLER_FILTER(0x1d5, MAXIMUM_ANISOTROPIC)
-#undef STATIC_SAMPLER_FILTER
-#endif // STATIC_SAMPLER_FILTER
-
+// FILTER(value, enum).
+#ifdef FILTER
+
+FILTER(0, MinMagMipPoint)
+FILTER(0x1, MinMagPointMipLinear)
+FILTER(0x4, MinPointMagLinearMipPoint)
+FILTER(0x5, MinPointMagMipLinear)
+FILTER(0x10, MinLinearMagMipPoint)
+FILTER(0x11, MinLinearMagPointMipLinear)
+FILTER(0x14, MinMagLinearMipPoint)
+FILTER(0x15, MinMagMipLinear)
+FILTER(0x55, Anisotropic)
+FILTER(0x80, ComparisonMinMagMipPoint)
+FILTER(0x81, ComparisonMinMagPointMipLinear)
+FILTER(0x84, ComparisonMinPointMagLinearMipPoint)
+FILTER(0x85, ComparisonMinPointMagMipLinear)
+FILTER(0x90, ComparisonMinLinearMagMipPoint)
+FILTER(0x91, ComparisonMinLinearMagPointMipLinear)
+FILTER(0x94, ComparisonMinMagLinearMipPoint)
+FILTER(0x95, ComparisonMinMagMipLinear)
+FILTER(0xd5, ComparisonAnisotropic)
+FILTER(0x100, MinimumMinMagMipPoint)
+FILTER(0x101, MinimumMinMagPointMipLinear)
+FILTER(0x104, MinimumMinPointMagLinearMipPoint)
+FILTER(0x105, MinimumMinPointMagMipLinear)
+FILTER(0x110, MinimumMinLinearMagMipPoint)
+FILTER(0x111, MinimumMinLinearMagPointMipLinear)
+FILTER(0x114, MinimumMinMagLinearMipPoint)
+FILTER(0x115, MinimumMinMagMipLinear)
+FILTER(0x155, MinimumAnisotropic)
+FILTER(0x180, MaximumMinMagMipPoint)
+FILTER(0x181, MaximumMinMagPointMipLinear)
+FILTER(0x184, MaximumMinPointMagLinearMipPoint)
+FILTER(0x185, MaximumMinPointMagMipLinear)
+FILTER(0x190, MaximumMinLinearMagMipPoint)
+FILTER(0x191, MaximumMinLinearMagPointMipLinear)
+FILTER(0x194, MaximumMinMagLinearMipPoint)
+FILTER(0x195, MaximumMinMagMipLinear)
+FILTER(0x1d5, MaximumAnisotropic)
+
+#undef FILTER
+#endif // FILTER
+
+// TEXTURE_ADDRESS_MODE(value, enum).
 #ifdef TEXTURE_ADDRESS_MODE
 
-TEXTURE_ADDRESS_MODE( 1,  Wrap)
-TEXTURE_ADDRESS_MODE( 2,  Mirror)
-TEXTURE_ADDRESS_MODE( 3,  Clamp)
-TEXTURE_ADDRESS_MODE( 4,  Border)
-TEXTURE_ADDRESS_MODE( 5,  MirrorOnce)
+TEXTURE_ADDRESS_MODE(1,  Wrap)
+TEXTURE_ADDRESS_MODE(2,  Mirror)
+TEXTURE_ADDRESS_MODE(3,  Clamp)
+TEXTURE_ADDRESS_MODE(4,  Border)
+TEXTURE_ADDRESS_MODE(5,  MirrorOnce)
+
 #undef TEXTURE_ADDRESS_MODE
 #endif // TEXTURE_ADDRESS_MODE
 
-#ifdef COMPARISON_FUNCTION
+// COMPARISON_FUNC(value, enum).
+#ifdef COMPARISON_FUNC
+
+COMPARISON_FUNC(1,  Never)
+COMPARISON_FUNC(2,  Less)
+COMPARISON_FUNC(3,  Equal)
+COMPARISON_FUNC(4,  LessEqual)
+COMPARISON_FUNC(5,  Greater)
+COMPARISON_FUNC(6,  NotEqual)
+COMPARISON_FUNC(7,  GreaterEqual)
+COMPARISON_FUNC(8,  Always)
 
-COMPARISON_FUNCTION( 1,  Never)
-COMPARISON_FUNCTION( 2,  Less)
-COMPARISON_FUNCTION( 3,  Equal)
-COMPARISON_FUNCTION( 4,  LessEqual)
-COMPARISON_FUNCTION( 5,  Greater)
-COMPARISON_FUNCTION( 6,  NotEqual)
-COMPARISON_FUNCTION( 7,  GreaterEqual)
-COMPARISON_FUNCTION( 8,  Always)
-#undef COMPARISON_FUNCTION
-#endif // COMPARISON_FUNCTION
+#undef COMPARISON_FUNC
+#endif // COMPARISON_FUNC
 
+// STATIC_BORDER_COLOR(value, enum).
 #ifdef STATIC_BORDER_COLOR
-STATIC_BORDER_COLOR( 0,  TransparentBlack)
-STATIC_BORDER_COLOR( 1,  OpaqueBlack)
-STATIC_BORDER_COLOR( 2,  OpaqueWhite)
-STATIC_BORDER_COLOR( 3,  OpaqueBlackUint)
-STATIC_BORDER_COLOR( 4,  OpaqueWhiteUint)
+
+STATIC_BORDER_COLOR(0,  TransparentBlack)
+STATIC_BORDER_COLOR(1,  OpaqueBlack)
+STATIC_BORDER_COLOR(2,  OpaqueWhite)
+STATIC_BORDER_COLOR(3,  OpaqueBlackUint)
+STATIC_BORDER_COLOR(4,  OpaqueWhiteUint)
+
 #undef STATIC_BORDER_COLOR
 #endif // STATIC_BORDER_COLOR
 
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index e2063d2dfef5..a3f9d8b63598 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -489,7 +489,7 @@ HANDLE_DW_AT(0x2029, HP_is_result_param, 0, HP)
 // HANDLE_DW_AT(0x2004, CPQ_split_lifetimes_rtn, 0, COMPAQ)
 // HANDLE_DW_AT(0x2005, CPQ_prologue_length, 0, COMPAQ)
 
-HANDLE_DW_AT(0x2026, DW_AT_INTEL_other_endian, 0, INTEL)
+HANDLE_DW_AT(0x2026, INTEL_other_endian, 0, INTEL)
 
 // Green Hills.
 HANDLE_DW_AT(0x2083, GHS_rsm, 0, GHS)
@@ -638,13 +638,14 @@ HANDLE_DW_AT(0x3e09, LLVM_ptrauth_authenticates_null_values, 0, LLVM)
 HANDLE_DW_AT(0x3e0a, LLVM_ptrauth_authentication_mode, 0, LLVM)
 HANDLE_DW_AT(0x3e0b, LLVM_num_extra_inhabitants, 0, LLVM)
 HANDLE_DW_AT(0x3e0c, LLVM_stmt_sequence, 0, LLVM)
+HANDLE_DW_AT(0x3e0d, LLVM_coro_suspend_idx, 0, LLVM)
 // Heterogeneous Debugging Extension defined at
 // https://llvm.org/docs/AMDGPUDwarfProposalForHeterogeneousDebugging.html.
-HANDLE_DW_AT(0x3e0d, LLVM_memory_space, 0, LLVM)
-HANDLE_DW_AT(0x3e0e, LLVM_address_space, 0, LLVM)
-HANDLE_DW_AT(0x3e0f, LLVM_lanes, 0, LLVM)
-HANDLE_DW_AT(0x3e10, LLVM_lane_pc, 0, LLVM)
-HANDLE_DW_AT(0x3e11, LLVM_vector_size, 0, LLVM)
+HANDLE_DW_AT(0x3e0e, LLVM_memory_space, 0, LLVM)
+HANDLE_DW_AT(0x3e0f, LLVM_address_space, 0, LLVM)
+HANDLE_DW_AT(0x3e10, LLVM_lanes, 0, LLVM)
+HANDLE_DW_AT(0x3e11, LLVM_lane_pc, 0, LLVM)
+HANDLE_DW_AT(0x3e12, LLVM_vector_size, 0, LLVM)
 
 // https://www.llvm.org/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.html#a-7-15-memory-space-encodings
 HANDLE_DW_MSPACE(0x0, none)
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index 48b48b56ef62..f836cb85b838 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -1046,6 +1046,7 @@ LLVM_ABI unsigned getSubOperationEncoding(unsigned OpEncoding,
 LLVM_ABI unsigned getVirtuality(StringRef VirtualityString);
 LLVM_ABI unsigned getEnumKind(StringRef EnumKindString);
 LLVM_ABI unsigned getLanguage(StringRef LanguageString);
+LLVM_ABI unsigned getMemorySpace(StringRef LanguageString);
 LLVM_ABI unsigned getCallingConvention(StringRef LanguageString);
 LLVM_ABI unsigned getMemorySpace(StringRef LanguageString);
 LLVM_ABI unsigned getAttributeEncoding(StringRef EncodingString);
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index f5f236cf9806..ebb257ab3382 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1153,9 +1153,7 @@ enum : unsigned {
   SHT_LLVM_SYMPART = 0x6fff4c05,   // Symbol partition specification.
   SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition.
   SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition.
-  SHT_LLVM_BB_ADDR_MAP_V0 =
-      0x6fff4c08, // LLVM Basic Block Address Map (old version kept for
-                  // backward-compatibility).
+  // SHT_LLVM_BB_ADDR_MAP_V0 = 0x6fff4c08,  // Do not use.
   SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c09, // LLVM Call Graph Profile.
   SHT_LLVM_BB_ADDR_MAP = 0x6fff4c0a,        // LLVM Basic Block Address Map.
   SHT_LLVM_OFFLOADING = 0x6fff4c0b,         // LLVM device offloading data.
diff --git a/llvm/include/llvm/BinaryFormat/GOFF.h b/llvm/include/llvm/BinaryFormat/GOFF.h
index 443bcfc9479a..49d2809cb652 100644
--- a/llvm/include/llvm/BinaryFormat/GOFF.h
+++ b/llvm/include/llvm/BinaryFormat/GOFF.h
@@ -169,6 +169,91 @@ enum SubsectionKind : uint8_t {
   SK_PPA1 = 2,
   SK_PPA2 = 4,
 };
+
+// The standard System/390 convention is to name the high-order (leftmost) bit
+// in a byte as bit zero. The Flags type helps to set bits in byte according
+// to this numeration order.
+class Flags {
+  uint8_t Val = 0;
+
+  constexpr static uint8_t bits(uint8_t BitIndex, uint8_t Length, uint8_t Value,
+                                uint8_t OldValue) {
+    uint8_t Pos = 8 - BitIndex - Length;
+    uint8_t Mask = ((1 << Length) - 1) << Pos;
+    Value = Value << Pos;
+    return (OldValue & ~Mask) | Value;
+  }
+
+public:
+  constexpr Flags() = default;
+  constexpr Flags(uint8_t BitIndex, uint8_t Length, uint8_t Value)
+      : Val(bits(BitIndex, Length, Value, 0)) {}
+
+  template <typename T>
+  constexpr void set(uint8_t BitIndex, uint8_t Length, T NewValue) {
+    Val = bits(BitIndex, Length, static_cast<uint8_t>(NewValue), Val);
+  }
+
+  template <typename T>
+  constexpr T get(uint8_t BitIndex, uint8_t Length) const {
+    return static_cast<T>((Val >> (8 - BitIndex - Length)) &
+                          ((1 << Length) - 1));
+  }
+
+  constexpr operator uint8_t() const { return Val; }
+};
+
+// Structure for the flag field of a symbol. See
+// https://www.ibm.com/docs/en/zos/3.1.0?topic=formats-external-symbol-definition-record
+// at offset 41 for the definition.
+struct SymbolFlags {
+  Flags SymFlags;
+
+#define GOFF_SYMBOL_FLAG(NAME, TYPE, BITINDEX, LENGTH)                         \
+  void set##NAME(TYPE Val) { SymFlags.set<TYPE>(BITINDEX, LENGTH, Val); }      \
+  TYPE get##NAME() const { return SymFlags.get<TYPE>(BITINDEX, LENGTH); }
+
+  GOFF_SYMBOL_FLAG(FillBytePresence, bool, 0, 1)
+  GOFF_SYMBOL_FLAG(Mangled, bool, 1, 1)
+  GOFF_SYMBOL_FLAG(Renameable, bool, 2, 1)
+  GOFF_SYMBOL_FLAG(RemovableClass, bool, 3, 1)
+  GOFF_SYMBOL_FLAG(ReservedQwords, ESDReserveQwords, 5, 3)
+
+#undef GOFF_SYMBOL_FLAG
+
+  constexpr operator uint8_t() const { return static_cast<uint8_t>(SymFlags); }
+};
+
+// Structure for the behavioral attributes. See
+// https://www.ibm.com/docs/en/zos/3.1.0?topic=record-external-symbol-definition-behavioral-attributes
+// for the definition.
+struct BehavioralAttributes {
+  Flags Attr[10];
+
+#define GOFF_BEHAVIORAL_ATTRIBUTE(NAME, TYPE, ATTRIDX, BITINDEX, LENGTH)       \
+  void set##NAME(TYPE Val) { Attr[ATTRIDX].set<TYPE>(BITINDEX, LENGTH, Val); } \
+  TYPE get##NAME() const { return Attr[ATTRIDX].get<TYPE>(BITINDEX, LENGTH); }
+
+  GOFF_BEHAVIORAL_ATTRIBUTE(Amode, GOFF::ESDAmode, 0, 0, 8)
+  GOFF_BEHAVIORAL_ATTRIBUTE(Rmode, GOFF::ESDRmode, 1, 0, 8)
+  GOFF_BEHAVIORAL_ATTRIBUTE(TextStyle, GOFF::ESDTextStyle, 2, 0, 4)
+  GOFF_BEHAVIORAL_ATTRIBUTE(BindingAlgorithm, GOFF::ESDBindingAlgorithm, 2, 4,
+                            4)
+  GOFF_BEHAVIORAL_ATTRIBUTE(TaskingBehavior, GOFF::ESDTaskingBehavior, 3, 0, 3)
+  GOFF_BEHAVIORAL_ATTRIBUTE(ReadOnly, bool, 3, 4, 1)
+  GOFF_BEHAVIORAL_ATTRIBUTE(Executable, GOFF::ESDExecutable, 3, 5, 3)
+  GOFF_BEHAVIORAL_ATTRIBUTE(DuplicateSymbolSeverity,
+                            GOFF::ESDDuplicateSymbolSeverity, 4, 2, 2)
+  GOFF_BEHAVIORAL_ATTRIBUTE(BindingStrength, GOFF::ESDBindingStrength, 4, 4, 4)
+  GOFF_BEHAVIORAL_ATTRIBUTE(LoadingBehavior, GOFF::ESDLoadingBehavior, 5, 0, 2)
+  GOFF_BEHAVIORAL_ATTRIBUTE(COMMON, bool, 5, 2, 1)
+  GOFF_BEHAVIORAL_ATTRIBUTE(IndirectReference, bool, 5, 3, 1)
+  GOFF_BEHAVIORAL_ATTRIBUTE(BindingScope, GOFF::ESDBindingScope, 5, 4, 4)
+  GOFF_BEHAVIORAL_ATTRIBUTE(LinkageType, GOFF::ESDLinkageType, 6, 2, 1)
+  GOFF_BEHAVIORAL_ATTRIBUTE(Alignment, GOFF::ESDAlignment, 6, 3, 5)
+
+#undef GOFF_BEHAVIORAL_ATTRIBUTE
+};
 } // end namespace GOFF
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index 604104ff3cca..cf90a43d0d7e 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -253,7 +253,7 @@ const unsigned WASM_SYMBOL_ABSOLUTE = 0x200;
 
 #define WASM_RELOC(name, value) name = value,
 
-enum : unsigned {
+enum WasmRelocType : unsigned {
 #include "WasmRelocs.def"
 };
 
@@ -452,6 +452,8 @@ struct WasmRelocation {
   uint32_t Index;  // Index into either symbol or type index space.
   uint64_t Offset; // Offset from the start of the section.
   int64_t Addend;  // A value to add to the symbol.
+
+  WasmRelocType getType() const { return static_cast<WasmRelocType>(Type); }
 };
 
 struct WasmInitFunc {
diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h
index e7451a3f1bac..4f839d4cd157 100644
--- a/llvm/include/llvm/Bitcode/BitcodeReader.h
+++ b/llvm/include/llvm/Bitcode/BitcodeReader.h
@@ -120,7 +120,7 @@ struct ParserCallbacks {
           IdentificationBit(IdentificationBit), ModuleBit(ModuleBit) {}
 
     // Calls the ctor.
-    LLVM_ABI_FRIEND friend Expected<BitcodeFileContents>
+    LLVM_ABI friend Expected<BitcodeFileContents>
     getBitcodeFileContents(MemoryBufferRef Buffer);
 
     Expected<std::unique_ptr<Module>>
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 86f8dd836a18..f21bc811280a 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -801,6 +801,7 @@ enum AttributeKindCodes {
   ATTR_KIND_NO_DIVERGENCE_SOURCE = 100,
   ATTR_KIND_SANITIZE_TYPE = 101,
   ATTR_KIND_CAPTURES = 102,
+  ATTR_KIND_DEAD_ON_RETURN = 103,
 
   // TODO: Get rid of this.
   // There really shouldn't be incompatible bitcode changes specific to AMD
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index cabecbec175b..2b9be43eadb7 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1105,6 +1105,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                               VectorType *&SubTy) const {
     if (Mask.empty())
       return Kind;
+    int NumDstElts = Mask.size();
     int NumSrcElts = SrcTy->getElementCount().getKnownMinValue();
     switch (Kind) {
     case TTI::SK_PermuteSingleSrc: {
@@ -1115,8 +1116,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       if (isSplatMask(Mask, NumSrcElts, Index))
         return TTI::SK_Broadcast;
       if (ShuffleVectorInst::isExtractSubvectorMask(Mask, NumSrcElts, Index) &&
-          (Index + Mask.size()) <= (size_t)NumSrcElts) {
-        SubTy = FixedVectorType::get(SrcTy->getElementType(), Mask.size());
+          (Index + NumDstElts) <= NumSrcElts) {
+        SubTy = FixedVectorType::get(SrcTy->getElementType(), NumDstElts);
         return TTI::SK_ExtractSubvector;
       }
       break;
@@ -1126,8 +1127,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
         return improveShuffleKindFromMask(TTI::SK_PermuteSingleSrc, Mask, SrcTy,
                                           Index, SubTy);
       int NumSubElts;
-      if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask(
-                                 Mask, NumSrcElts, NumSubElts, Index)) {
+      if (NumDstElts > 2 && ShuffleVectorInst::isInsertSubvectorMask(
+                                Mask, NumSrcElts, NumSubElts, Index)) {
         if (Index + NumSubElts > NumSrcElts)
           return Kind;
         SubTy = FixedVectorType::get(SrcTy->getElementType(), NumSubElts);
@@ -2242,6 +2243,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::log2:
       ISD = ISD::FLOG2;
       break;
+    case Intrinsic::ldexp:
+      ISD = ISD::FLDEXP;
+      break;
     case Intrinsic::fabs:
       ISD = ISD::FABS;
       break;
@@ -2296,6 +2300,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::roundeven:
       ISD = ISD::FROUNDEVEN;
       break;
+    case Intrinsic::lround:
+      ISD = ISD::LROUND;
+      break;
+    case Intrinsic::llround:
+      ISD = ISD::LLROUND;
+      break;
     case Intrinsic::pow:
       ISD = ISD::FPOW;
       break;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 109dc8812c24..75c051712ae4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -609,6 +609,15 @@ class LLVM_ABI CallLowering {
   virtual bool isTypeIsValidForThisReturn(EVT Ty) const { return false; }
 };
 
+extern template LLVM_ABI void
+CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
+                                    const DataLayout &DL,
+                                    const Function &FuncInfo) const;
+
+extern template LLVM_ABI void
+CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,
+                                    const DataLayout &DL,
+                                    const CallBase &FuncInfo) const;
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index c15263e0b06f..7d7b5364d6b6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -693,18 +693,19 @@ class CombinerHelper {
   /// feeding a G_AND instruction \p MI.
   bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const;
 
-  /// Given an G_UDIV \p MI expressing a divide by constant, return an
-  /// expression that implements it by multiplying by a magic number.
+  /// Given an G_UDIV \p MI or G_UREM \p MI expressing a divide by constant,
+  /// return an expression that implements it by multiplying by a magic number.
   /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
-  MachineInstr *buildUDivUsingMul(MachineInstr &MI) const;
-  /// Combine G_UDIV by constant into a multiply by magic constant.
-  bool matchUDivByConst(MachineInstr &MI) const;
-  void applyUDivByConst(MachineInstr &MI) const;
+  MachineInstr *buildUDivorURemUsingMul(MachineInstr &MI) const;
+  /// Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
+  bool matchUDivorURemByConst(MachineInstr &MI) const;
+  void applyUDivorURemByConst(MachineInstr &MI) const;
 
   /// Given an G_SDIV \p MI expressing a signed divide by constant, return an
   /// expression that implements it by multiplying by a magic number.
   /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
   MachineInstr *buildSDivUsingMul(MachineInstr &MI) const;
+  /// Combine G_SDIV by constant into a multiply by magic constant.
   bool matchSDivByConst(MachineInstr &MI) const;
   void applySDivByConst(MachineInstr &MI) const;
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 4106be4c81ce..ea0873f41ebb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -295,6 +295,11 @@ class LegalizerHelper {
   getNeutralElementForVecReduce(unsigned Opcode, MachineIRBuilder &MIRBuilder,
                                 LLT Ty);
 
+  LegalizeResult emitSincosLibcall(MachineInstr &MI,
+                                   MachineIRBuilder &MIRBuilder, unsigned Size,
+                                   Type *OpType,
+                                   LostDebugLocObserver &LocObserver);
+
 public:
   /// Return the alignment to use for a stack temporary object with the given
   /// type.
diff --git a/llvm/include/llvm/CodeGen/MachineOperand.h b/llvm/include/llvm/CodeGen/MachineOperand.h
index 161a1cdbff98..646588a2a92a 100644
--- a/llvm/include/llvm/CodeGen/MachineOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineOperand.h
@@ -764,7 +764,7 @@ class MachineOperand {
   /// isIdenticalTo uses for comparison. It is thus suited for use in hash
   /// tables which use that function for equality comparisons only. This must
   /// stay exactly in sync with isIdenticalTo above.
-  LLVM_ABI_FRIEND friend hash_code hash_value(const MachineOperand &MO);
+  LLVM_ABI friend hash_code hash_value(const MachineOperand &MO);
 
   /// ChangeToImmediate - Replace this operand with a new immediate operand of
   /// the specified value.  If an operand is known to be an immediate already,
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 990452fa11fe..18df5d657064 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -618,6 +618,9 @@ LLVM_ABI FunctionPass *createSelectOptimizePass();
 
 LLVM_ABI FunctionPass *createCallBrPass();
 
+/// Creates Windows Secure Hot Patch pass. \see WindowsSecureHotPatching.cpp
+ModulePass *createWindowsSecureHotPatchingPass();
+
 /// Lowers KCFI operand bundles for indirect calls.
 LLVM_ABI FunctionPass *createKCFIPass();
 } // namespace llvm
diff --git a/llvm/include/llvm/CodeGen/PseudoSourceValue.h b/llvm/include/llvm/CodeGen/PseudoSourceValue.h
index 01402c31a1e3..aec19005278a 100644
--- a/llvm/include/llvm/CodeGen/PseudoSourceValue.h
+++ b/llvm/include/llvm/CodeGen/PseudoSourceValue.h
@@ -46,8 +46,8 @@ class LLVM_ABI PseudoSourceValue {
 private:
   unsigned Kind;
   unsigned AddressSpace;
-  LLVM_ABI_FRIEND friend raw_ostream &
-  llvm::operator<<(raw_ostream &OS, const PseudoSourceValue *PSV);
+  LLVM_ABI friend raw_ostream &llvm::operator<<(raw_ostream &OS,
+                                                const PseudoSourceValue *PSV);
 
   friend class MachineMemOperand; // For printCustom().
   friend class MIRFormatter;      // For printCustom().
diff --git a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
index 7481ed5b80b3..451459eda25e 100644
--- a/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
+++ b/llvm/include/llvm/CodeGen/RuntimeLibcallUtil.h
@@ -56,6 +56,10 @@ LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getPOWI(EVT RetVT);
 
+/// getPOW - Return the POW_* value for the given types, or
+/// UNKNOWN_LIBCALL if there is none.
+LLVM_ABI Libcall getPOW(EVT RetVT);
+
 /// getLDEXP - Return the LDEXP_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
 LLVM_ABI Libcall getLDEXP(EVT RetVT);
diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 9eb6dd45f912..7c5cdbbeb0ca 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -108,6 +108,23 @@ inline Value_match m_Specific(SDValue N) {
   return Value_match(N);
 }
 
+template <unsigned ResNo, typename Pattern> struct Result_match {
+  Pattern P;
+
+  explicit Result_match(const Pattern &P) : P(P) {}
+
+  template <typename MatchContext>
+  bool match(const MatchContext &Ctx, SDValue N) {
+    return N.getResNo() == ResNo && P.match(Ctx, N);
+  }
+};
+
+/// Match only if the SDValue is a certain result at ResNo.
+template <unsigned ResNo, typename Pattern>
+inline Result_match<ResNo, Pattern> m_Result(const Pattern &P) {
+  return Result_match<ResNo, Pattern>(P);
+}
+
 struct DeferredValue_match {
   SDValue &MatchVal;
 
@@ -277,6 +294,34 @@ inline auto m_SpecificVT(EVT RefVT) {
 inline auto m_Glue() { return m_SpecificVT(MVT::Glue); }
 inline auto m_OtherVT() { return m_SpecificVT(MVT::Other); }
 
+/// Match a scalar ValueType.
+template <typename Pattern>
+inline auto m_SpecificScalarVT(EVT RefVT, const Pattern &P) {
+  return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
+                         P};
+}
+inline auto m_SpecificScalarVT(EVT RefVT) {
+  return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
+                         m_Value()};
+}
+
+/// Match a vector ValueType.
+template <typename Pattern>
+inline auto m_SpecificVectorElementVT(EVT RefVT, const Pattern &P) {
+  return ValueType_match{[=](EVT VT) {
+                           return VT.isVector() &&
+                                  VT.getVectorElementType() == RefVT;
+                         },
+                         P};
+}
+inline auto m_SpecificVectorElementVT(EVT RefVT) {
+  return ValueType_match{[=](EVT VT) {
+                           return VT.isVector() &&
+                                  VT.getVectorElementType() == RefVT;
+                         },
+                         m_Value()};
+}
+
 /// Match any integer ValueTypes.
 template <typename Pattern> inline auto m_IntegerVT(const Pattern &P) {
   return ValueType_match{[](EVT VT) { return VT.isInteger(); }, P};
@@ -513,6 +558,13 @@ m_VSelect(const T0_P &Cond, const T1_P &T, const T2_P &F) {
   return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::VSELECT, Cond, T, F);
 }
 
+template <typename T0_P, typename T1_P, typename T2_P>
+inline Result_match<0, TernaryOpc_match<T0_P, T1_P, T2_P>>
+m_Load(const T0_P &Ch, const T1_P &Ptr, const T2_P &Offset) {
+  return m_Result<0>(
+      TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::LOAD, Ch, Ptr, Offset));
+}
+
 template <typename T0_P, typename T1_P, typename T2_P>
 inline TernaryOpc_match<T0_P, T1_P, T2_P>
 m_InsertElt(const T0_P &Vec, const T1_P &Val, const T2_P &Idx) {
@@ -1048,19 +1100,46 @@ inline SpecificInt_match m_SpecificInt(uint64_t V) {
   return SpecificInt_match(APInt(64, V));
 }
 
-inline SpecificInt_match m_Zero() { return m_SpecificInt(0U); }
-inline SpecificInt_match m_One() { return m_SpecificInt(1U); }
+struct Zero_match {
+  bool AllowUndefs;
+
+  explicit Zero_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
+
+  template <typename MatchContext>
+  bool match(const MatchContext &, SDValue N) const {
+    return isZeroOrZeroSplat(N, AllowUndefs);
+  }
+};
+
+struct Ones_match {
+  bool AllowUndefs;
+
+  Ones_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
+
+  template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
+    return isOnesOrOnesSplat(N, AllowUndefs);
+  }
+};
 
 struct AllOnes_match {
+  bool AllowUndefs;
 
-  AllOnes_match() = default;
+  AllOnes_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
 
   template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
-    return isAllOnesOrAllOnesSplat(N);
+    return isAllOnesOrAllOnesSplat(N, AllowUndefs);
   }
 };
 
-inline AllOnes_match m_AllOnes() { return AllOnes_match(); }
+inline Ones_match m_One(bool AllowUndefs = false) {
+  return Ones_match(AllowUndefs);
+}
+inline Zero_match m_Zero(bool AllowUndefs = false) {
+  return Zero_match(AllowUndefs);
+}
+inline AllOnes_match m_AllOnes(bool AllowUndefs = false) {
+  return AllOnes_match(AllowUndefs);
+}
 
 /// Match true boolean value based on the information provided by
 /// TargetLowering.
@@ -1137,7 +1216,7 @@ inline CondCode_match m_SpecificCondCode(ISD::CondCode CC) {
 
 /// Match a negate as a sub(0, v)
 template <typename ValTy>
-inline BinaryOpc_match<SpecificInt_match, ValTy> m_Neg(const ValTy &V) {
+inline BinaryOpc_match<Zero_match, ValTy, false> m_Neg(const ValTy &V) {
   return m_Sub(m_Zero(), V);
 }
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index a98e46c58727..7d8a0c4ce8e4 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2030,6 +2030,11 @@ class SelectionDAG {
   LLVM_ABI SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
                                       ArrayRef<SDValue> Ops);
 
+  /// Fold BUILD_VECTOR of constants/undefs to the destination type
+  /// BUILD_VECTOR of constants/undefs elements.
+  LLVM_ABI SDValue FoldConstantBuildVector(BuildVectorSDNode *BV,
+                                           const SDLoc &DL, EVT DstEltVT);
+
   /// Constant fold a setcc to true or false.
   LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
                              const SDLoc &dl);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 375d2f31e283..5241a51dd8cd 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -435,9 +435,9 @@ class SelectionDAGISel {
   /// It runs node predicate number PredNo and returns true if it succeeds or
   /// false if it fails.  The number is a private implementation detail to the
   /// code tblgen produces.
-  virtual bool CheckNodePredicateWithOperands(
-      SDValue Op, unsigned PredNo,
-      const SmallVectorImpl<SDValue> &Operands) const {
+  virtual bool
+  CheckNodePredicateWithOperands(SDValue Op, unsigned PredNo,
+                                 ArrayRef<SDValue> Operands) const {
     llvm_unreachable("Tblgen should generate the implementation of this!");
   }
 
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 92da4ef7f055..5d9937f83239 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1497,6 +1497,14 @@ class MemSDNode : public SDNode {
     MMO->refineAlignment(NewMMO);
   }
 
+  void refineRanges(const MachineMemOperand *NewMMO) {
+    // If this node has range metadata that is different than NewMMO, clear the
+    // range metadata.
+    // FIXME: Union the ranges instead?
+    if (getRanges() && getRanges() != NewMMO->getRanges())
+      MMO->clearRanges();
+  }
+
   const SDValue &getChain() const { return getOperand(0); }
 
   const SDValue &getBasePtr() const {
@@ -1929,6 +1937,16 @@ LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
 /// Does not permit build vector implicit truncation.
 LLVM_ABI bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
 
+/// Return true if the value is a constant 1 integer or a splatted vector of a
+/// constant 1 integer (with no undefs).
+/// Does not permit build vector implicit truncation.
+LLVM_ABI bool isOnesOrOnesSplat(SDValue N, bool AllowUndefs = false);
+
+/// Return true if the value is a constant 0 integer or a splatted vector of a
+/// constant 0 integer (with no undefs).
+/// Does not permit build vector implicit truncation.
+LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs = false);
+
 /// Return true if \p V is either a integer or FP constant.
 inline bool isIntOrFPConstant(SDValue V) {
   return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 727526055e59..420f1d5fb20c 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1116,10 +1116,7 @@ class LLVM_ABI TargetLoweringBase {
     LegalizeTypeAction ValueTypeActions[MVT::VALUETYPE_SIZE];
 
   public:
-    ValueTypeActionImpl() {
-      std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
-                TypeLegal);
-    }
+    ValueTypeActionImpl() { llvm::fill(ValueTypeActions, TypeLegal); }
 
     LegalizeTypeAction getTypeAction(MVT VT) const {
       return ValueTypeActions[VT.SimpleTy];
@@ -3558,13 +3555,8 @@ class LLVM_ABI TargetLoweringBase {
     return nullptr;
   }
 
-  /// Rename the default libcall routine name for the specified libcall.
-  void setLibcallName(RTLIB::Libcall Call, const char *Name) {
-    Libcalls.setLibcallName(Call, Name);
-  }
-
-  void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
-    Libcalls.setLibcallName(Calls, Name);
+  void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) {
+    Libcalls.setLibcallImpl(Call, Impl);
   }
 
   /// Get the libcall routine name for the specified libcall.
@@ -5091,9 +5083,6 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     return nullptr;
   }
 
-  bool verifyReturnAddressArgumentIsConstant(SDValue Op,
-                                             SelectionDAG &DAG) const;
-
   //===--------------------------------------------------------------------===//
   // Inline Asm Support hooks
   //
@@ -5800,6 +5789,8 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
 private:
   SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                            const SDLoc &DL, DAGCombinerInfo &DCI) const;
+  SDValue foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
+                          const SDLoc &DL, DAGCombinerInfo &DCI) const;
   SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                              const SDLoc &DL, DAGCombinerInfo &DCI) const;
 
diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 00e681e6bf53..fe450b3c1a3a 100644
--- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -316,10 +316,15 @@ class TargetLoweringObjectFileXCOFF : public TargetLoweringObjectFile {
 };
 
 class TargetLoweringObjectFileGOFF : public TargetLoweringObjectFile {
+  std::string DefaultRootSDName;
+  std::string DefaultADAPRName;
+
 public:
   TargetLoweringObjectFileGOFF();
   ~TargetLoweringObjectFileGOFF() override = default;
 
+  void getModuleMetadata(Module &M) override;
+
   MCSection *SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind,
                                     const TargetMachine &TM) const override;
   MCSection *getExplicitSectionGlobal(const GlobalObject *GO, SectionKind Kind,
diff --git a/llvm/include/llvm/CodeGenTypes/LowLevelType.h b/llvm/include/llvm/CodeGenTypes/LowLevelType.h
index 06879e1f8d15..d8e0848aff84 100644
--- a/llvm/include/llvm/CodeGenTypes/LowLevelType.h
+++ b/llvm/include/llvm/CodeGenTypes/LowLevelType.h
@@ -28,6 +28,7 @@
 
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/CodeGenTypes/MachineValueType.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include <cassert>
 
@@ -140,7 +141,7 @@ class LLT {
   explicit constexpr LLT()
       : IsScalar(false), IsPointer(false), IsVector(false), RawData(0) {}
 
-  explicit LLT(MVT VT);
+  LLVM_ABI explicit LLT(MVT VT);
 
   constexpr bool isValid() const { return IsScalar || RawData != 0; }
   constexpr bool isScalar() const { return IsScalar; }
@@ -282,7 +283,7 @@ class LLT {
       return scalar(getScalarSizeInBits());
   }
 
-  void print(raw_ostream &OS) const;
+  LLVM_ABI void print(raw_ostream &OS) const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   LLVM_DUMP_METHOD void dump() const;
diff --git a/llvm/include/llvm/CodeGenTypes/MachineValueType.h b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
index c14abca02735..b8e91a022ec5 100644
--- a/llvm/include/llvm/CodeGenTypes/MachineValueType.h
+++ b/llvm/include/llvm/CodeGenTypes/MachineValueType.h
@@ -17,6 +17,7 @@
 #define LLVM_CODEGEN_MACHINEVALUETYPE_H
 
 #include "llvm/ADT/Sequence.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/TypeSize.h"
@@ -65,10 +66,10 @@ namespace llvm {
     bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
 
     /// Support for debugging, callable in GDB: VT.dump()
-    void dump() const;
+    LLVM_ABI void dump() const;
 
     /// Implement operator<<.
-    void print(raw_ostream &OS) const;
+    LLVM_ABI void print(raw_ostream &OS) const;
 
     /// Return true if this is a valid simple valuetype.
     bool isValid() const {
@@ -509,11 +510,11 @@ namespace llvm {
     /// otherwise they are invalid.
     /// NB: This includes pointer types, which require a DataLayout to convert
     /// to a concrete value type.
-    static MVT getVT(Type *Ty, bool HandleUnknown = false);
+    LLVM_ABI static MVT getVT(Type *Ty, bool HandleUnknown = false);
 
     /// Returns an APFloat semantics tag appropriate for the value type. If this
     /// is a vector type, the element semantics are returned.
-    const fltSemantics &getFltSemantics() const;
+    LLVM_ABI const fltSemantics &getFltSemantics() const;
 
   public:
     /// SimpleValueType Iteration
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 831bfa5699ab..1a6a45d8a5a3 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -17,7 +17,7 @@
 /* The number of commits in the linear history from the
  * start of the universe up to the latest llvm main commit
  * that has been merged */
-#define LLVM_MAIN_REVISION 541644
+#define LLVM_MAIN_REVISION 543426
 
 /* Define if LLVM_ENABLE_DUMP is enabled */
 #cmakedefine LLVM_ENABLE_DUMP
@@ -115,6 +115,9 @@
 /* Define if building LLVM with BUILD_SHARED_LIBS */
 #cmakedefine LLVM_BUILD_SHARED_LIBS
 
+/* Define if exporting LLVM public interface for shared library */
+#cmakedefine LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS
+
 /* Define if building LLVM with LLVM_FORCE_USE_OLD_TOOLCHAIN_LIBS */
 #cmakedefine LLVM_FORCE_USE_OLD_TOOLCHAIN ${LLVM_FORCE_USE_OLD_TOOLCHAIN}
 
diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def b/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
index 9d85acc49fa0..b38bdb482df4 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
+++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
@@ -256,6 +256,8 @@ SYMBOL_RECORD_ALIAS(S_GTHREAD32     , 0x1113, GlobalTLS, ThreadLocalDataSym)
 SYMBOL_RECORD(S_UNAMESPACE    , 0x1124, UsingNamespaceSym)
 SYMBOL_RECORD(S_ANNOTATION    , 0x1019, AnnotationSym)
 
+SYMBOL_RECORD(S_HOTPATCHFUNC  , 0x1169, HotPatchFuncSym)
+
 #undef CV_SYMBOL
 #undef SYMBOL_RECORD
 #undef SYMBOL_RECORD_ALIAS
diff --git a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
index 5b4f0d31e642..c2897891544a 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/SymbolRecord.h
@@ -177,6 +177,21 @@ class CallerSym : public SymbolRecord {
   uint32_t RecordOffset = 0;
 };
 
+class HotPatchFuncSym : public SymbolRecord {
+public:
+  explicit HotPatchFuncSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {}
+  HotPatchFuncSym(uint32_t RecordOffset)
+      : SymbolRecord(SymbolRecordKind::HotPatchFuncSym),
+        RecordOffset(RecordOffset) {}
+
+  // This is an ItemID in the IPI stream, which points to an LF_FUNC_ID or
+  // LF_MFUNC_ID record.
+  TypeIndex Function;
+  StringRef Name;
+
+  uint32_t RecordOffset = 0;
+};
+
 struct DecodedAnnotation {
   StringRef Name;
   ArrayRef<uint8_t> Bytes;
@@ -225,8 +240,7 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t FirstByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t FirstByte = Annotations.consume_front();
 
     if ((FirstByte & 0x80) == 0x00)
       return FirstByte;
@@ -234,8 +248,7 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t SecondByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t SecondByte = Annotations.consume_front();
 
     if ((FirstByte & 0xC0) == 0x80)
       return ((FirstByte & 0x3F) << 8) | SecondByte;
@@ -243,14 +256,12 @@ struct BinaryAnnotationIterator
     if (Annotations.empty())
       return -1;
 
-    uint8_t ThirdByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t ThirdByte = Annotations.consume_front();
 
     if (Annotations.empty())
       return -1;
 
-    uint8_t FourthByte = Annotations.front();
-    Annotations = Annotations.drop_front();
+    uint8_t FourthByte = Annotations.consume_front();
 
     if ((FirstByte & 0xE0) == 0xC0)
       return ((FirstByte & 0x1F) << 24) | (SecondByte << 16) |
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
index 32e8247ac4c2..4723f00a4d24 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIPrinter.h
@@ -10,6 +10,7 @@
 #define LLVM_DEBUGINFO_DWARF_DWARFCFIPRINTER_H
 
 #include "llvm/DebugInfo/DWARF/DWARFCFIProgram.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
@@ -17,9 +18,10 @@ struct DIDumpOptions;
 
 namespace dwarf {
 
-void printCFIProgram(const CFIProgram &P, raw_ostream &OS,
-                     const DIDumpOptions &DumpOpts, unsigned IndentLevel,
-                     std::optional<uint64_t> Address);
+LLVM_ABI void printCFIProgram(const CFIProgram &P, raw_ostream &OS,
+                              const DIDumpOptions &DumpOpts,
+                              unsigned IndentLevel,
+                              std::optional<uint64_t> Address);
 
 } // end namespace dwarf
 
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
index ad7358c28f16..e377115e849e 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFCFIProgram.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
-#define LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
@@ -106,17 +106,11 @@ class CFIProgram {
   };
 
   /// Get the OperandType as a "const char *".
-  static const char *operandTypeString(OperandType OT);
+  LLVM_ABI static const char *operandTypeString(OperandType OT);
 
   /// Retrieve the array describing the types of operands according to the enum
   /// above. This is indexed by opcode.
-  static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
-
-private:
-  std::vector<Instruction> Instructions;
-  const uint64_t CodeAlignmentFactor;
-  const int64_t DataAlignmentFactor;
-  Triple::ArchType Arch;
+  LLVM_ABI static ArrayRef<OperandType[MaxOperands]> getOperandTypes();
 
   /// Convenience method to add a new instruction with the given opcode.
   void addInstruction(uint8_t Opcode) {
@@ -144,10 +138,16 @@ class CFIProgram {
     Instructions.back().Ops.push_back(Operand2);
     Instructions.back().Ops.push_back(Operand3);
   }
+
+private:
+  std::vector<Instruction> Instructions;
+  const uint64_t CodeAlignmentFactor;
+  const int64_t DataAlignmentFactor;
+  Triple::ArchType Arch;
 };
 
 } // end namespace dwarf
 
 } // end namespace llvm
 
-#endif // LLVM_DEBUGINFO_DWARF_DWARFCFIPROGRAM_H
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFCFIPROGRAM_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
index 4c9a81093ca8..7ef62a6383f9 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDataExtractorSimple.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
-#define LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTORSIMPLE_H
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
 
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Compiler.h"
@@ -195,4 +195,4 @@ class DWARFDataExtractorSimple
 };
 
 } // end namespace llvm
-#endif // LLVM_DEBUGINFO_DWARF_DWARFDATAEXTRACTOR_H
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFDATAEXTRACTORSIMPLE_H
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
index 3b367009a379..89d12437a7d3 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
@@ -138,6 +138,8 @@ class UnwindLocation {
     return *AddrSpace;
   }
   int32_t getConstant() const { return Offset; }
+  bool getDereference() const { return Dereference; }
+
   /// Some opcodes will modify the CFA location's register only, so we need
   /// to be able to modify the CFA register when evaluating DWARF Call Frame
   /// Information opcodes.
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 44432d321911..28ec9c291ea0 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
-#define LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
+#ifndef LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
+#define LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator.h"
@@ -186,8 +186,9 @@ class DWARFExpressionPrinter {
   /// \param E to be printed
   /// \param OS to this stream
   /// \param GetNameForDWARFReg callback to return dwarf register name
-  static void print(const DWARFExpression *E, raw_ostream &OS,
-                    DIDumpOptions DumpOpts, DWARFUnit *U, bool IsEH = false);
+  LLVM_ABI static void print(const DWARFExpression *E, raw_ostream &OS,
+                             DIDumpOptions DumpOpts, DWARFUnit *U,
+                             bool IsEH = false);
 
   /// Print the expression in a format intended to be compact and useful to a
   /// user, but not perfectly unambiguous, or capable of representing every
@@ -199,9 +200,10 @@ class DWARFExpressionPrinter {
   /// \param GetNameForDWARFReg callback to return dwarf register name
   ///
   /// \returns true if the expression was successfully printed
-  static bool printCompact(const DWARFExpression *E, raw_ostream &OS,
-                           std::function<StringRef(uint64_t RegNum, bool IsEH)>
-                               GetNameForDWARFReg = nullptr);
+  LLVM_ABI static bool printCompact(
+      const DWARFExpression *E, raw_ostream &OS,
+      std::function<StringRef(uint64_t RegNum, bool IsEH)> GetNameForDWARFReg =
+          nullptr);
 
   /// Pretty print a register opcode and operands.
   /// \param U within the context of this Dwarf unit, if any.
@@ -211,9 +213,10 @@ class DWARFExpressionPrinter {
   /// \param Operands to the opcode
   ///
   /// returns true if the Op was successfully printed
-  static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
-                                    DIDumpOptions DumpOpts, uint8_t Opcode,
-                                    ArrayRef<uint64_t> Operands);
+  LLVM_ABI static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
+                                             DIDumpOptions DumpOpts,
+                                             uint8_t Opcode,
+                                             ArrayRef<uint64_t> Operands);
 
 private:
   static bool printOp(const DWARFExpression::Operation *Op, raw_ostream &OS,
@@ -228,4 +231,4 @@ class DWARFExpressionPrinter {
 
 } // end namespace llvm
 
-#endif // LLVM_DEBUGINFO_DWARF_DWARFEXPRESSION_H
+#endif // LLVM_DEBUGINFO_DWARF_LOWLEVEL_DWARFEXPRESSION_H
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
index be64cdaea3d7..ee9a87e25e15 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVObject.h
@@ -84,6 +84,14 @@ using LVTypes = SmallVector<LVType *, 8>;
 
 using LVOffsets = SmallVector<LVOffset, 8>;
 
+// The following DWARF documents detail the 'tombstone' concept:
+//   https://dwarfstd.org/issues/231013.1.html
+//   https://dwarfstd.org/issues/200609.1.html
+//
+// The value of the largest representable address offset (for example,
+// 0xffffffff when the size of an address is 32 bits).
+//
+// -1 (0xffffffff) => Valid tombstone
 const LVAddress MaxAddress = std::numeric_limits<uint64_t>::max();
 
 enum class LVBinaryType { NONE, ELF, COFF };
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h
index 52e4aa9701e4..f4ebfc914ad0 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSourceLanguage.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/CodeView/CodeView.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 namespace logicalview {
@@ -52,7 +53,7 @@ struct LVSourceLanguage {
 
   bool isValid() const { return Language != Invalid; }
   TaggedLanguage get() const { return Language; }
-  StringRef getName() const;
+  LLVM_ABI StringRef getName() const;
 
 private:
   TaggedLanguage Language = Invalid;
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
index d64723ced2e7..2abc18b8daca 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVDWARFReader.h
@@ -58,6 +58,9 @@ class LVDWARFReader final : public LVBinaryReader {
   bool FoundLowPC = false;
   bool FoundHighPC = false;
 
+  // The value is updated for each Compile Unit that is processed.
+  std::optional<LVAddress> TombstoneAddress;
+
   // Cross references (Elements).
   using LVElementSet = std::unordered_set<LVElement *>;
   struct LVElementEntry {
@@ -127,6 +130,12 @@ class LVDWARFReader final : public LVBinaryReader {
   LVAddress getCUHighAddress() const { return CUHighAddress; }
   void setCUHighAddress(LVAddress Address) { CUHighAddress = Address; }
 
+  void setTombstoneAddress(LVAddress Address) { TombstoneAddress = Address; }
+  LVAddress getTombstoneAddress() const {
+    assert(TombstoneAddress && "Unset tombstone value");
+    return TombstoneAddress.value();
+  }
+
   const LVSymbols &GetSymbolsWithLocations() const {
     return SymbolsWithLocations;
   }
diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index f4569850b093..5533652736dc 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -4468,7 +4468,9 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() {
         return nullptr;
       if (!consumeIf('_'))
         return nullptr;
-      return make<BitIntType>(Size, Signed);
+      // The front end expects this to be available for Substitution
+      Result = make<BitIntType>(Size, Signed);
+      break;
     }
     //                ::= Di   # char32_t
     case 'i':
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index d72fb47cd9b0..a9cfe726a73d 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -226,35 +226,51 @@ enum class TagKind { Class, Struct, Union, Enum };
 
 enum class NodeKind {
   Unknown,
-  Md5Symbol,
-  PrimitiveType,
-  FunctionSignature,
-  Identifier,
-  NamedIdentifier,
-  VcallThunkIdentifier,
-  LocalStaticGuardIdentifier,
-  IntrinsicFunctionIdentifier,
-  ConversionOperatorIdentifier,
+
+  SymbolStart,
+  Md5Symbol = SymbolStart,
+  EncodedStringLiteral,
+  FunctionSymbol,
+  LocalStaticGuardVariable,
+  SpecialTableSymbol,
+  VariableSymbol,
+  SymbolEnd = VariableSymbol,
+
+  IdentifierStart,
+  ConversionOperatorIdentifier = IdentifierStart,
   DynamicStructorIdentifier,
-  StructorIdentifier,
+  IntrinsicFunctionIdentifier,
   LiteralOperatorIdentifier,
+  LocalStaticGuardIdentifier,
+  NamedIdentifier,
+  RttiBaseClassDescriptor,
+  StructorIdentifier,
+  VcallThunkIdentifier,
+  IdentifierEnd = VcallThunkIdentifier,
+
+  TypeStart,
+  ArrayType = TypeStart,
+  Custom,
+
+  FunctionSignature,
   ThunkSignature,
+  FunctionSignatureEnd = ThunkSignature,
+
+  IntrinsicType,
   PointerType,
+  PrimitiveType,
   TagType,
-  ArrayType,
-  Custom,
-  IntrinsicType,
+  TypeEnd = TagType,
+
+  PointerAuthQualifier,
+
+  IntegerLiteral,
+
   NodeArray,
+
   QualifiedName,
+
   TemplateParameterReference,
-  EncodedStringLiteral,
-  IntegerLiteral,
-  RttiBaseClassDescriptor,
-  LocalStaticGuardVariable,
-  FunctionSymbol,
-  VariableSymbol,
-  SpecialTableSymbol,
-  PointerAuthQualifier,
 };
 
 struct Node {
@@ -309,6 +325,10 @@ struct TypeNode : public Node {
     outputPost(OB, Flags);
   }
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::TypeStart && N->kind() <= NodeKind::TypeEnd;
+  }
+
   Qualifiers Quals = Q_None;
 };
 
@@ -319,6 +339,10 @@ struct PrimitiveTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override {}
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PrimitiveType;
+  }
+
   PrimitiveKind PrimKind;
 };
 
@@ -329,6 +353,11 @@ struct FunctionSignatureNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::FunctionSignature &&
+           N->kind() <= NodeKind::FunctionSignatureEnd;
+  }
+
   // Valid if this FunctionTypeNode is the Pointee of a PointerType or
   // MemberPointerType.
   PointerAffinity Affinity = PointerAffinity::None;
@@ -357,6 +386,11 @@ struct FunctionSignatureNode : public TypeNode {
 struct IdentifierNode : public Node {
   explicit IdentifierNode(NodeKind K) : Node(K) {}
 
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::IdentifierStart &&
+           N->kind() <= NodeKind::IdentifierEnd;
+  }
+
   NodeArrayNode *TemplateParams = nullptr;
 
 protected:
@@ -368,6 +402,10 @@ struct VcallThunkIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::VcallThunkIdentifier;
+  }
+
   uint64_t OffsetInVTable = 0;
 };
 
@@ -377,6 +415,10 @@ struct DynamicStructorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::DynamicStructorIdentifier;
+  }
+
   VariableSymbolNode *Variable = nullptr;
   QualifiedNameNode *Name = nullptr;
   bool IsDestructor = false;
@@ -387,6 +429,10 @@ struct NamedIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::NamedIdentifier;
+  }
+
   std::string_view Name;
 };
 
@@ -397,6 +443,10 @@ struct IntrinsicFunctionIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntrinsicFunctionIdentifier;
+  }
+
   IntrinsicFunctionKind Operator;
 };
 
@@ -406,6 +456,10 @@ struct LiteralOperatorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LiteralOperatorIdentifier;
+  }
+
   std::string_view Name;
 };
 
@@ -415,6 +469,10 @@ struct LocalStaticGuardIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LocalStaticGuardIdentifier;
+  }
+
   bool IsThread = false;
   uint32_t ScopeIndex = 0;
 };
@@ -425,6 +483,10 @@ struct ConversionOperatorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ConversionOperatorIdentifier;
+  }
+
   // The type that this operator converts too.
   TypeNode *TargetType = nullptr;
 };
@@ -437,6 +499,10 @@ struct StructorIdentifierNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::StructorIdentifier;
+  }
+
   // The name of the class that this is a structor of.
   IdentifierNode *Class = nullptr;
   bool IsDestructor = false;
@@ -448,6 +514,10 @@ struct ThunkSignatureNode : public FunctionSignatureNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ThunkSignature;
+  }
+
   struct ThisAdjustor {
     uint32_t StaticOffset = 0;
     int32_t VBPtrOffset = 0;
@@ -463,6 +533,10 @@ struct PointerTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PointerType;
+  }
+
   // Is this a pointer, reference, or rvalue-reference?
   PointerAffinity Affinity = PointerAffinity::None;
 
@@ -482,6 +556,8 @@ struct TagTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) { return N->kind() == NodeKind::TagType; }
+
   QualifiedNameNode *QualifiedName = nullptr;
   TagKind Tag;
 };
@@ -495,6 +571,10 @@ struct ArrayTypeNode : public TypeNode {
   void outputDimensionsImpl(OutputBuffer &OB, OutputFlags Flags) const;
   void outputOneDimension(OutputBuffer &OB, OutputFlags Flags, Node *N) const;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::ArrayType;
+  }
+
   // A list of array dimensions.  e.g. [3,4,5] in `int Foo[3][4][5]`
   NodeArrayNode *Dimensions = nullptr;
 
@@ -505,6 +585,10 @@ struct ArrayTypeNode : public TypeNode {
 struct IntrinsicNode : public TypeNode {
   IntrinsicNode() : TypeNode(NodeKind::IntrinsicType) {}
   void output(OutputBuffer &OB, OutputFlags Flags) const override {}
+
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntrinsicType;
+  }
 };
 
 struct CustomTypeNode : public TypeNode {
@@ -513,6 +597,8 @@ struct CustomTypeNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) { return N->kind() == NodeKind::Custom; }
+
   IdentifierNode *Identifier = nullptr;
 };
 
@@ -524,6 +610,10 @@ struct NodeArrayNode : public Node {
   void output(OutputBuffer &OB, OutputFlags Flags,
               std::string_view Separator) const;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::NodeArray;
+  }
+
   Node **Nodes = nullptr;
   size_t Count = 0;
 };
@@ -533,6 +623,10 @@ struct QualifiedNameNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::QualifiedName;
+  }
+
   NodeArrayNode *Components = nullptr;
 
   IdentifierNode *getUnqualifiedIdentifier() {
@@ -547,6 +641,10 @@ struct TemplateParameterReferenceNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::TemplateParameterReference;
+  }
+
   SymbolNode *Symbol = nullptr;
 
   int ThunkOffsetCount = 0;
@@ -562,6 +660,10 @@ struct IntegerLiteralNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::IntegerLiteral;
+  }
+
   uint64_t Value = 0;
   bool IsNegative = false;
 };
@@ -572,6 +674,10 @@ struct RttiBaseClassDescriptorNode : public IdentifierNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::RttiBaseClassDescriptor;
+  }
+
   uint32_t NVOffset = 0;
   int32_t VBPtrOffset = 0;
   uint32_t VBTableOffset = 0;
@@ -581,6 +687,12 @@ struct RttiBaseClassDescriptorNode : public IdentifierNode {
 struct SymbolNode : public Node {
   explicit SymbolNode(NodeKind K) : Node(K) {}
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
+
+  static bool classof(const Node *N) {
+    return N->kind() >= NodeKind::SymbolStart &&
+           N->kind() <= NodeKind::SymbolEnd;
+  }
+
   QualifiedNameNode *Name = nullptr;
 };
 
@@ -589,6 +701,11 @@ struct SpecialTableSymbolNode : public SymbolNode {
       : SymbolNode(NodeKind::SpecialTableSymbol) {}
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
+
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::SpecialTableSymbol;
+  }
+
   QualifiedNameNode *TargetName = nullptr;
   Qualifiers Quals = Qualifiers::Q_None;
 };
@@ -599,6 +716,10 @@ struct LocalStaticGuardVariableNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::LocalStaticGuardVariable;
+  }
+
   bool IsVisible = false;
 };
 
@@ -607,6 +728,10 @@ struct EncodedStringLiteralNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::EncodedStringLiteral;
+  }
+
   std::string_view DecodedString;
   bool IsTruncated = false;
   CharKind Char = CharKind::Char;
@@ -617,6 +742,10 @@ struct VariableSymbolNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::VariableSymbol;
+  }
+
   StorageClass SC = StorageClass::None;
   TypeNode *Type = nullptr;
 };
@@ -626,6 +755,10 @@ struct FunctionSymbolNode : public SymbolNode {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::FunctionSymbol;
+  }
+
   FunctionSignatureNode *Signature = nullptr;
 };
 
@@ -641,6 +774,10 @@ struct PointerAuthQualifierNode : public Node {
 
   void output(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  static bool classof(const Node *N) {
+    return N->kind() == NodeKind::PointerAuthQualifier;
+  }
+
   // List of arguments.
   NodeArrayNode *Components = nullptr;
 };
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/COFF.h b/llvm/include/llvm/ExecutionEngine/Orc/COFF.h
index adc9e9e17116..42a6c85a577f 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/COFF.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/COFF.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_EXECUTIONENGINE_ORC_COFF_H
 #define LLVM_EXECUTIONENGINE_ORC_COFF_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 
@@ -31,8 +32,8 @@ class COFFImportFileScanner {
 public:
   COFFImportFileScanner(std::set<std::string> &ImportedDynamicLibraries)
       : ImportedDynamicLibraries(ImportedDynamicLibraries) {}
-  Expected<bool> operator()(object::Archive &A, MemoryBufferRef MemberBuf,
-                            size_t Index) const;
+  LLVM_ABI Expected<bool>
+  operator()(object::Archive &A, MemoryBufferRef MemberBuf, size_t Index) const;
 
 private:
   std::set<std::string> &ImportedDynamicLibraries;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h
index bc1f5f0eb70c..c69b6f736651 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h
@@ -23,7 +23,7 @@
 namespace llvm {
 namespace orc {
 
-class EPCGenericMemoryAccess : public ExecutorProcessControl::MemoryAccess {
+class EPCGenericMemoryAccess : public MemoryAccess {
 public:
   /// Function addresses for memory access.
   struct FuncAddrs {
@@ -31,8 +31,15 @@ class EPCGenericMemoryAccess : public ExecutorProcessControl::MemoryAccess {
     ExecutorAddr WriteUInt16s;
     ExecutorAddr WriteUInt32s;
     ExecutorAddr WriteUInt64s;
-    ExecutorAddr WriteBuffers;
     ExecutorAddr WritePointers;
+    ExecutorAddr WriteBuffers;
+    ExecutorAddr ReadUInt8s;
+    ExecutorAddr ReadUInt16s;
+    ExecutorAddr ReadUInt32s;
+    ExecutorAddr ReadUInt64s;
+    ExecutorAddr ReadPointers;
+    ExecutorAddr ReadBuffers;
+    ExecutorAddr ReadStrings;
   };
 
   /// Create an EPCGenericMemoryAccess instance from a given set of
@@ -68,6 +75,13 @@ class EPCGenericMemoryAccess : public ExecutorProcessControl::MemoryAccess {
         FAs.WriteUInt64s, std::move(OnWriteComplete), Ws);
   }
 
+  void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
+                          WriteResultFn OnWriteComplete) override {
+    using namespace shared;
+    EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessPointerWrite>)>(
+        FAs.WritePointers, std::move(OnWriteComplete), Ws);
+  }
+
   void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws,
                          WriteResultFn OnWriteComplete) override {
     using namespace shared;
@@ -75,11 +89,116 @@ class EPCGenericMemoryAccess : public ExecutorProcessControl::MemoryAccess {
         FAs.WriteBuffers, std::move(OnWriteComplete), Ws);
   }
 
-  void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
-                          WriteResultFn OnWriteComplete) override {
+  void readUInt8sAsync(ArrayRef<ExecutorAddr> Rs,
+                       OnReadUIntsCompleteFn<uint8_t> OnComplete) override {
     using namespace shared;
-    EPC.callSPSWrapperAsync<void(SPSSequence<SPSMemoryAccessPointerWrite>)>(
-        FAs.WritePointers, std::move(OnWriteComplete), Ws);
+    EPC.callSPSWrapperAsync<SPSSequence<uint8_t>(SPSSequence<SPSExecutorAddr>)>(
+        FAs.ReadUInt8s,
+        [OnComplete = std::move(OnComplete)](
+            Error Err, ReadUIntsResult<uint8_t> Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readUInt16sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint16_t> OnComplete) override {
+    using namespace shared;
+    EPC.callSPSWrapperAsync<SPSSequence<uint16_t>(
+        SPSSequence<SPSExecutorAddr>)>(
+        FAs.ReadUInt16s,
+        [OnComplete = std::move(OnComplete)](
+            Error Err, ReadUIntsResult<uint16_t> Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readUInt32sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint32_t> OnComplete) override {
+    using namespace shared;
+    EPC.callSPSWrapperAsync<SPSSequence<uint32_t>(
+        SPSSequence<SPSExecutorAddr>)>(
+        FAs.ReadUInt32s,
+        [OnComplete = std::move(OnComplete)](
+            Error Err, ReadUIntsResult<uint32_t> Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readUInt64sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint64_t> OnComplete) override {
+    using namespace shared;
+    EPC.callSPSWrapperAsync<SPSSequence<uint64_t>(
+        SPSSequence<SPSExecutorAddr>)>(
+        FAs.ReadUInt64s,
+        [OnComplete = std::move(OnComplete)](
+            Error Err, ReadUIntsResult<uint64_t> Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readPointersAsync(ArrayRef<ExecutorAddr> Rs,
+                         OnReadPointersCompleteFn OnComplete) override {
+    using namespace shared;
+    using SPSSig = SPSSequence<SPSExecutorAddr>(SPSSequence<SPSExecutorAddr>);
+    EPC.callSPSWrapperAsync<SPSSig>(
+        FAs.ReadPointers,
+        [OnComplete = std::move(OnComplete)](
+            Error Err, ReadPointersResult Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readBuffersAsync(ArrayRef<ExecutorAddrRange> Rs,
+                        OnReadBuffersCompleteFn OnComplete) override {
+    using namespace shared;
+    using SPSSig =
+        SPSSequence<SPSSequence<uint8_t>>(SPSSequence<SPSExecutorAddrRange>);
+    EPC.callSPSWrapperAsync<SPSSig>(
+        FAs.ReadBuffers,
+        [OnComplete = std::move(OnComplete)](Error Err,
+                                             ReadBuffersResult Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
+  }
+
+  void readStringsAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadStringsCompleteFn OnComplete) override {
+    using namespace shared;
+    using SPSSig = SPSSequence<SPSString>(SPSSequence<SPSExecutorAddr>);
+    EPC.callSPSWrapperAsync<SPSSig>(
+        FAs.ReadStrings,
+        [OnComplete = std::move(OnComplete)](Error Err,
+                                             ReadStringsResult Result) mutable {
+          if (Err)
+            OnComplete(std::move(Err));
+          else
+            OnComplete(std::move(Result));
+        },
+        Rs);
   }
 
 private:
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
index e63f5f784252..90b0b4fbb259 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutorProcessControl.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/DylibManager.h"
+#include "llvm/ExecutionEngine/Orc/MemoryAccess.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h"
@@ -23,16 +24,13 @@
 #include "llvm/ExecutionEngine/Orc/TargetProcess/UnwindInfoManager.h"
 #include "llvm/ExecutionEngine/Orc/TaskDispatch.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Support/MSVCErrorWorkarounds.h"
 #include "llvm/TargetParser/Triple.h"
 
 #include <future>
 #include <mutex>
 #include <vector>
 
-namespace llvm {
-namespace orc {
+namespace llvm::orc {
 
 class ExecutionSession;
 
@@ -99,81 +97,6 @@ class LLVM_ABI ExecutorProcessControl {
     TaskDispatcher &D;
   };
 
-  /// APIs for manipulating memory in the target process.
-  class LLVM_ABI MemoryAccess {
-  public:
-    /// Callback function for asynchronous writes.
-    using WriteResultFn = unique_function<void(Error)>;
-
-    virtual ~MemoryAccess();
-
-    virtual void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
-                                  WriteResultFn OnWriteComplete) = 0;
-
-    virtual void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws,
-                                   WriteResultFn OnWriteComplete) = 0;
-
-    virtual void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws,
-                                   WriteResultFn OnWriteComplete) = 0;
-
-    virtual void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws,
-                                   WriteResultFn OnWriteComplete) = 0;
-
-    virtual void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws,
-                                   WriteResultFn OnWriteComplete) = 0;
-
-    virtual void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
-                                    WriteResultFn OnWriteComplete) = 0;
-
-    Error writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writeUInt8sAsync(Ws,
-                       [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-
-    Error writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writeUInt16sAsync(Ws,
-                        [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-
-    Error writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writeUInt32sAsync(Ws,
-                        [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-
-    Error writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writeUInt64sAsync(Ws,
-                        [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-
-    Error writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writeBuffersAsync(Ws,
-                        [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-
-    Error writePointers(ArrayRef<tpctypes::PointerWrite> Ws) {
-      std::promise<MSVCPError> ResultP;
-      auto ResultF = ResultP.get_future();
-      writePointersAsync(Ws,
-                         [&](Error Err) { ResultP.set_value(std::move(Err)); });
-      return ResultF.get();
-    }
-  };
-
   /// Contains the address of the dispatch function and context that the ORC
   /// runtime can use to call functions in the JIT.
   struct JITDispatchInfo {
@@ -399,124 +322,6 @@ class LLVM_ABI ExecutorProcessControl {
   StringMap<ExecutorAddr> BootstrapSymbols;
 };
 
-class LLVM_ABI InProcessMemoryAccess
-    : public ExecutorProcessControl::MemoryAccess {
-public:
-  InProcessMemoryAccess(bool IsArch64Bit) : IsArch64Bit(IsArch64Bit) {}
-  void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
-                        WriteResultFn OnWriteComplete) override;
-
-  void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws,
-                         WriteResultFn OnWriteComplete) override;
-
-  void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws,
-                         WriteResultFn OnWriteComplete) override;
-
-  void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws,
-                         WriteResultFn OnWriteComplete) override;
-
-  void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws,
-                         WriteResultFn OnWriteComplete) override;
-
-  void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
-                          WriteResultFn OnWriteComplete) override;
-
-private:
-  bool IsArch64Bit;
-};
-
-/// A ExecutorProcessControl instance that asserts if any of its methods are
-/// used. Suitable for use is unit tests, and by ORC clients who haven't moved
-/// to ExecutorProcessControl-based APIs yet.
-class UnsupportedExecutorProcessControl : public ExecutorProcessControl,
-                                          private InProcessMemoryAccess {
-public:
-  UnsupportedExecutorProcessControl(
-      std::shared_ptr<SymbolStringPool> SSP = nullptr,
-      std::unique_ptr<TaskDispatcher> D = nullptr, const std::string &TT = "",
-      unsigned PageSize = 0)
-      : ExecutorProcessControl(
-            SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>(),
-            D ? std::move(D) : std::make_unique<InPlaceTaskDispatcher>()),
-        InProcessMemoryAccess(Triple(TT).isArch64Bit()) {
-    this->TargetTriple = Triple(TT);
-    this->PageSize = PageSize;
-    this->MemAccess = this;
-  }
-
-  Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr,
-                              ArrayRef<std::string> Args) override {
-    llvm_unreachable("Unsupported");
-  }
-
-  Expected<int32_t> runAsVoidFunction(ExecutorAddr VoidFnAddr) override {
-    llvm_unreachable("Unsupported");
-  }
-
-  Expected<int32_t> runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) override {
-    llvm_unreachable("Unsupported");
-  }
-
-  void callWrapperAsync(ExecutorAddr WrapperFnAddr,
-                        IncomingWFRHandler OnComplete,
-                        ArrayRef<char> ArgBuffer) override {
-    llvm_unreachable("Unsupported");
-  }
-
-  Error disconnect() override { return Error::success(); }
-};
-
-/// A ExecutorProcessControl implementation targeting the current process.
-class LLVM_ABI SelfExecutorProcessControl : public ExecutorProcessControl,
-                                            private InProcessMemoryAccess,
-                                            private DylibManager {
-public:
-  SelfExecutorProcessControl(
-      std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
-      Triple TargetTriple, unsigned PageSize,
-      std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr);
-
-  /// Create a SelfExecutorProcessControl with the given symbol string pool and
-  /// memory manager.
-  /// If no symbol string pool is given then one will be created.
-  /// If no memory manager is given a jitlink::InProcessMemoryManager will
-  /// be created and used by default.
-  static Expected<std::unique_ptr<SelfExecutorProcessControl>>
-  Create(std::shared_ptr<SymbolStringPool> SSP = nullptr,
-         std::unique_ptr<TaskDispatcher> D = nullptr,
-         std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr = nullptr);
-
-  Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr,
-                              ArrayRef<std::string> Args) override;
-
-  Expected<int32_t> runAsVoidFunction(ExecutorAddr VoidFnAddr) override;
-
-  Expected<int32_t> runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) override;
-
-  void callWrapperAsync(ExecutorAddr WrapperFnAddr,
-                        IncomingWFRHandler OnComplete,
-                        ArrayRef<char> ArgBuffer) override;
-
-  Error disconnect() override;
-
-private:
-  static shared::CWrapperFunctionResult
-  jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag,
-                                       const char *Data, size_t Size);
-
-  Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) override;
-
-  void lookupSymbolsAsync(ArrayRef<LookupRequest> Request,
-                          SymbolLookupCompleteFn F) override;
-
-  std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr;
-#ifdef __APPLE__
-  std::unique_ptr<UnwindInfoManager> UnwindInfoMgr;
-#endif // __APPLE__
-  char GlobalManglingPrefix = 0;
-};
-
-} // end namespace orc
-} // end namespace llvm
+} // namespace llvm::orc
 
 #endif // LLVM_EXECUTIONENGINE_ORC_EXECUTORPROCESSCONTROL_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h b/llvm/include/llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h
new file mode 100644
index 000000000000..eb68495469ec
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h
@@ -0,0 +1,68 @@
+//===-- InProcessMemoryAccess.h - Direct, in-process mem access -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Accesses memory in the current process.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_INPROCESSMEMORYACCESS_H
+#define LLVM_EXECUTIONENGINE_ORC_INPROCESSMEMORYACCESS_H
+
+#include "llvm/ExecutionEngine/Orc/MemoryAccess.h"
+
+namespace llvm::orc {
+
+class LLVM_ABI InProcessMemoryAccess : public MemoryAccess {
+public:
+  InProcessMemoryAccess(bool IsArch64Bit) : IsArch64Bit(IsArch64Bit) {}
+  void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
+                        WriteResultFn OnWriteComplete) override;
+
+  void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws,
+                         WriteResultFn OnWriteComplete) override;
+
+  void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws,
+                         WriteResultFn OnWriteComplete) override;
+
+  void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws,
+                         WriteResultFn OnWriteComplete) override;
+
+  void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
+                          WriteResultFn OnWriteComplete) override;
+
+  void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws,
+                         WriteResultFn OnWriteComplete) override;
+
+  void readUInt8sAsync(ArrayRef<ExecutorAddr> Rs,
+                       OnReadUIntsCompleteFn<uint8_t> OnComplete) override;
+
+  void readUInt16sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint16_t> OnComplete) override;
+
+  void readUInt32sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint32_t> OnComplete) override;
+
+  void readUInt64sAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadUIntsCompleteFn<uint64_t> OnComplete) override;
+
+  void readPointersAsync(ArrayRef<ExecutorAddr> Rs,
+                         OnReadPointersCompleteFn OnComplete) override;
+
+  void readBuffersAsync(ArrayRef<ExecutorAddrRange> Rs,
+                        OnReadBuffersCompleteFn OnComplete) override;
+
+  void readStringsAsync(ArrayRef<ExecutorAddr> Rs,
+                        OnReadStringsCompleteFn OnComplete) override;
+
+private:
+  bool IsArch64Bit;
+};
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_INPROCESSMEMORYACCESS_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
index cb8fdd650a2a..9c0d6e86d279 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -42,8 +42,7 @@ class ExecutorProcessControl;
 class LLVM_ABI LLJIT {
   template <typename, typename, typename> friend class LLJITBuilderSetters;
 
-  LLVM_ABI_FRIEND friend Expected<JITDylibSP>
-  setUpGenericLLVMIRPlatform(LLJIT &J);
+  LLVM_ABI friend Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J);
 
 public:
   /// Initializer support for LLJIT.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MemoryAccess.h b/llvm/include/llvm/ExecutionEngine/Orc/MemoryAccess.h
new file mode 100644
index 000000000000..1935f3c56443
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MemoryAccess.h
@@ -0,0 +1,195 @@
+//===------- MemoryAccess.h - Executor memory access APIs -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilities for accessing memory in the executor processes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_MEMORYACCESS_H
+#define LLVM_EXECUTIONENGINE_ORC_MEMORYACCESS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
+#include "llvm/Support/MSVCErrorWorkarounds.h"
+
+#include <future>
+
+namespace llvm::orc {
+
+/// APIs for manipulating memory in the target process.
+class LLVM_ABI MemoryAccess {
+public:
+  /// Callback function for asynchronous writes.
+  using WriteResultFn = unique_function<void(Error)>;
+
+  template <typename T> using ReadUIntsResult = std::vector<T>;
+  template <typename T>
+  using OnReadUIntsCompleteFn =
+      unique_function<void(Expected<ReadUIntsResult<T>>)>;
+
+  using ReadPointersResult = std::vector<ExecutorAddr>;
+  using OnReadPointersCompleteFn =
+      unique_function<void(Expected<ReadPointersResult>)>;
+
+  using ReadBuffersResult = std::vector<std::vector<uint8_t>>;
+  using OnReadBuffersCompleteFn =
+      unique_function<void(Expected<ReadBuffersResult>)>;
+
+  using ReadStringsResult = std::vector<std::string>;
+  using OnReadStringsCompleteFn =
+      unique_function<void(Expected<ReadStringsResult>)>;
+
+  virtual ~MemoryAccess();
+
+  virtual void writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
+                                WriteResultFn OnWriteComplete) = 0;
+
+  virtual void writeUInt16sAsync(ArrayRef<tpctypes::UInt16Write> Ws,
+                                 WriteResultFn OnWriteComplete) = 0;
+
+  virtual void writeUInt32sAsync(ArrayRef<tpctypes::UInt32Write> Ws,
+                                 WriteResultFn OnWriteComplete) = 0;
+
+  virtual void writeUInt64sAsync(ArrayRef<tpctypes::UInt64Write> Ws,
+                                 WriteResultFn OnWriteComplete) = 0;
+
+  virtual void writePointersAsync(ArrayRef<tpctypes::PointerWrite> Ws,
+                                  WriteResultFn OnWriteComplete) = 0;
+
+  virtual void writeBuffersAsync(ArrayRef<tpctypes::BufferWrite> Ws,
+                                 WriteResultFn OnWriteComplete) = 0;
+
+  virtual void readUInt8sAsync(ArrayRef<ExecutorAddr> Rs,
+                               OnReadUIntsCompleteFn<uint8_t> OnComplete) = 0;
+
+  virtual void readUInt16sAsync(ArrayRef<ExecutorAddr> Rs,
+                                OnReadUIntsCompleteFn<uint16_t> OnComplete) = 0;
+
+  virtual void readUInt32sAsync(ArrayRef<ExecutorAddr> Rs,
+                                OnReadUIntsCompleteFn<uint32_t> OnComplete) = 0;
+
+  virtual void readUInt64sAsync(ArrayRef<ExecutorAddr> Rs,
+                                OnReadUIntsCompleteFn<uint64_t> OnComplete) = 0;
+
+  virtual void readPointersAsync(ArrayRef<ExecutorAddr> Rs,
+                                 OnReadPointersCompleteFn OnComplete) = 0;
+
+  virtual void readBuffersAsync(ArrayRef<ExecutorAddrRange> Rs,
+                                OnReadBuffersCompleteFn OnComplete) = 0;
+
+  virtual void readStringsAsync(ArrayRef<ExecutorAddr> Rs,
+                                OnReadStringsCompleteFn OnComplete) = 0;
+
+  Error writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writeUInt8sAsync(Ws, [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Error writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writeUInt16sAsync(Ws,
+                      [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Error writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writeUInt32sAsync(Ws,
+                      [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Error writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writeUInt64sAsync(Ws,
+                      [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Error writePointers(ArrayRef<tpctypes::PointerWrite> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writePointersAsync(Ws,
+                       [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Error writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws) {
+    std::promise<MSVCPError> ResultP;
+    auto ResultF = ResultP.get_future();
+    writeBuffersAsync(Ws,
+                      [&](Error Err) { ResultP.set_value(std::move(Err)); });
+    return ResultF.get();
+  }
+
+  Expected<ReadUIntsResult<uint8_t>> readUInt8s(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadUIntsResult<uint8_t>>> P;
+    readUInt8sAsync(Rs, [&](Expected<ReadUIntsResult<uint8_t>> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadUIntsResult<uint16_t>> readUInt16s(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadUIntsResult<uint16_t>>> P;
+    readUInt16sAsync(Rs, [&](Expected<ReadUIntsResult<uint16_t>> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadUIntsResult<uint32_t>> readUInt32s(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadUIntsResult<uint32_t>>> P;
+    readUInt32sAsync(Rs, [&](Expected<ReadUIntsResult<uint32_t>> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadUIntsResult<uint64_t>> readUInt64s(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadUIntsResult<uint64_t>>> P;
+    readUInt64sAsync(Rs, [&](Expected<ReadUIntsResult<uint64_t>> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadPointersResult> readPointers(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadPointersResult>> P;
+    readPointersAsync(Rs, [&](Expected<ReadPointersResult> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadBuffersResult> readBuffers(ArrayRef<ExecutorAddrRange> Rs) {
+    std::promise<MSVCPExpected<ReadBuffersResult>> P;
+    readBuffersAsync(Rs, [&](Expected<ReadBuffersResult> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+
+  Expected<ReadStringsResult> readStrings(ArrayRef<ExecutorAddr> Rs) {
+    std::promise<MSVCPExpected<ReadStringsResult>> P;
+    readStringsAsync(Rs, [&](Expected<ReadStringsResult> Result) {
+      P.set_value(std::move(Result));
+    });
+    return P.get_future().get();
+  }
+};
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_MEMORYACCESS_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h
new file mode 100644
index 000000000000..805ac89aa912
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h
@@ -0,0 +1,76 @@
+//===-- SelfExecutorProcessControl.h - EPC for in-process JITs --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Executor process control implementation for JITs that run JIT'd code in the
+// same process.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_SELFEXECUTORPROCESSCONTROL_H
+#define LLVM_EXECUTIONENGINE_ORC_SELFEXECUTORPROCESSCONTROL_H
+
+#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h"
+
+#include <memory>
+
+namespace llvm::orc {
+
+/// A ExecutorProcessControl implementation targeting the current process.
+class LLVM_ABI SelfExecutorProcessControl : public ExecutorProcessControl,
+                                            private DylibManager {
+public:
+  SelfExecutorProcessControl(
+      std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
+      Triple TargetTriple, unsigned PageSize,
+      std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr);
+
+  /// Create a SelfExecutorProcessControl with the given symbol string pool and
+  /// memory manager.
+  /// If no symbol string pool is given then one will be created.
+  /// If no memory manager is given a jitlink::InProcessMemoryManager will
+  /// be created and used by default.
+  static Expected<std::unique_ptr<SelfExecutorProcessControl>>
+  Create(std::shared_ptr<SymbolStringPool> SSP = nullptr,
+         std::unique_ptr<TaskDispatcher> D = nullptr,
+         std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr = nullptr);
+
+  Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr,
+                              ArrayRef<std::string> Args) override;
+
+  Expected<int32_t> runAsVoidFunction(ExecutorAddr VoidFnAddr) override;
+
+  Expected<int32_t> runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) override;
+
+  void callWrapperAsync(ExecutorAddr WrapperFnAddr,
+                        IncomingWFRHandler OnComplete,
+                        ArrayRef<char> ArgBuffer) override;
+
+  Error disconnect() override;
+
+private:
+  static shared::CWrapperFunctionResult
+  jitDispatchViaWrapperFunctionManager(void *Ctx, const void *FnTag,
+                                       const char *Data, size_t Size);
+
+  Expected<tpctypes::DylibHandle> loadDylib(const char *DylibPath) override;
+
+  void lookupSymbolsAsync(ArrayRef<LookupRequest> Request,
+                          SymbolLookupCompleteFn F) override;
+
+  std::unique_ptr<jitlink::JITLinkMemoryManager> OwnedMemMgr;
+#ifdef __APPLE__
+  std::unique_ptr<UnwindInfoManager> UnwindInfoMgr;
+#endif // __APPLE__
+  char GlobalManglingPrefix = 0;
+  InProcessMemoryAccess IPMA;
+};
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_SELFEXECUTORPROCESSCONTROL_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
index 017ef70469e2..2bc6c1218ae7 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h
@@ -44,8 +44,16 @@ LLVM_ABI extern const char *MemoryWriteUInt8sWrapperName;
 LLVM_ABI extern const char *MemoryWriteUInt16sWrapperName;
 LLVM_ABI extern const char *MemoryWriteUInt32sWrapperName;
 LLVM_ABI extern const char *MemoryWriteUInt64sWrapperName;
-LLVM_ABI extern const char *MemoryWriteBuffersWrapperName;
 LLVM_ABI extern const char *MemoryWritePointersWrapperName;
+LLVM_ABI extern const char *MemoryWriteBuffersWrapperName;
+
+LLVM_ABI extern const char *MemoryReadUInt8sWrapperName;
+LLVM_ABI extern const char *MemoryReadUInt16sWrapperName;
+LLVM_ABI extern const char *MemoryReadUInt32sWrapperName;
+LLVM_ABI extern const char *MemoryReadUInt64sWrapperName;
+LLVM_ABI extern const char *MemoryReadPointersWrapperName;
+LLVM_ABI extern const char *MemoryReadBuffersWrapperName;
+LLVM_ABI extern const char *MemoryReadStringsWrapperName;
 
 LLVM_ABI extern const char *RegisterEHFrameSectionAllocActionName;
 LLVM_ABI extern const char *DeregisterEHFrameSectionAllocActionName;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
index e91d8d926d88..adb07ba29b92 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h
@@ -93,11 +93,11 @@ using UInt64Write = UIntWrite<uint64_t>;
 /// For use with TargetProcessControl::MemoryAccess objects.
 struct BufferWrite {
   BufferWrite() = default;
-  BufferWrite(ExecutorAddr Addr, StringRef Buffer)
+  BufferWrite(ExecutorAddr Addr, ArrayRef<char> Buffer)
       : Addr(Addr), Buffer(Buffer) {}
 
   ExecutorAddr Addr;
-  StringRef Buffer;
+  ArrayRef<char> Buffer;
 };
 
 /// Describes a write to a pointer.
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
index d92c029ff84d..ed6ea961bccf 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
@@ -36,8 +36,8 @@ class SymbolStringPool {
   friend class SymbolStringPoolEntryUnsafe;
 
   // Implemented in DebugUtils.h.
-  LLVM_ABI_FRIEND friend raw_ostream &operator<<(raw_ostream &OS,
-                                                 const SymbolStringPool &SSP);
+  LLVM_ABI friend raw_ostream &operator<<(raw_ostream &OS,
+                                          const SymbolStringPool &SSP);
 
 public:
   /// Destroy a SymbolStringPool.
@@ -94,8 +94,8 @@ class SymbolStringPtrBase {
     return LHS.S < RHS.S;
   }
 
-  LLVM_ABI_FRIEND friend raw_ostream &
-  operator<<(raw_ostream &OS, const SymbolStringPtrBase &Sym);
+  LLVM_ABI friend raw_ostream &operator<<(raw_ostream &OS,
+                                          const SymbolStringPtrBase &Sym);
 
 #ifndef NDEBUG
   // Returns true if the pool entry's ref count is above zero (or if the entry
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index b61c8b8563a1..03165895b85d 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -36,16 +36,6 @@ class ThreadSafeContext {
   };
 
 public:
-  // RAII based lock for ThreadSafeContext.
-  class [[nodiscard]] Lock {
-  public:
-    Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {}
-
-  private:
-    std::shared_ptr<State> S;
-    std::unique_lock<std::recursive_mutex> L;
-  };
-
   /// Construct a null context.
   ThreadSafeContext() = default;
 
@@ -56,17 +46,20 @@ class ThreadSafeContext {
            "Can not construct a ThreadSafeContext from a nullptr");
   }
 
-  /// Returns a pointer to the LLVMContext that was used to construct this
-  /// instance, or null if the instance was default constructed.
-  LLVMContext *getContext() { return S ? S->Ctx.get() : nullptr; }
-
-  /// Returns a pointer to the LLVMContext that was used to construct this
-  /// instance, or null if the instance was default constructed.
-  const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; }
+  template <typename Func> decltype(auto) withContextDo(Func &&F) {
+    if (auto TmpS = S) {
+      std::lock_guard<std::recursive_mutex> Lock(TmpS->Mutex);
+      return F(TmpS->Ctx.get());
+    } else
+      return F((LLVMContext *)nullptr);
+  }
 
-  Lock getLock() const {
-    assert(S && "Can not lock an empty ThreadSafeContext");
-    return Lock(S);
+  template <typename Func> decltype(auto) withContextDo(Func &&F) const {
+    if (auto TmpS = S) {
+      std::lock_guard<std::recursive_mutex> Lock(TmpS->Mutex);
+      return F(const_cast<const LLVMContext *>(TmpS->Ctx.get()));
+    } else
+      return F((const LLVMContext *)nullptr);
   }
 
 private:
@@ -89,10 +82,7 @@ class ThreadSafeModule {
     // *before* the context that it depends on.
     // We also need to lock the context to make sure the module tear-down
     // does not overlap any other work on the context.
-    if (M) {
-      auto L = TSCtx.getLock();
-      M = nullptr;
-    }
+    TSCtx.withContextDo([this](LLVMContext *Ctx) { M = nullptr; });
     M = std::move(Other.M);
     TSCtx = std::move(Other.TSCtx);
     return *this;
@@ -111,45 +101,39 @@ class ThreadSafeModule {
 
   ~ThreadSafeModule() {
     // We need to lock the context while we destruct the module.
-    if (M) {
-      auto L = TSCtx.getLock();
-      M = nullptr;
-    }
+    TSCtx.withContextDo([this](LLVMContext *Ctx) { M = nullptr; });
   }
 
   /// Boolean conversion: This ThreadSafeModule will evaluate to true if it
   /// wraps a non-null module.
-  explicit operator bool() const {
-    if (M) {
-      assert(TSCtx.getContext() &&
-             "Non-null module must have non-null context");
-      return true;
-    }
-    return false;
-  }
+  explicit operator bool() const { return !!M; }
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
   template <typename Func> decltype(auto) withModuleDo(Func &&F) {
-    assert(M && "Can not call on null module");
-    auto Lock = TSCtx.getLock();
-    return F(*M);
+    return TSCtx.withContextDo([&](LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(*M);
+    });
   }
 
   /// Locks the associated ThreadSafeContext and calls the given function
   /// on the contained Module.
   template <typename Func> decltype(auto) withModuleDo(Func &&F) const {
-    assert(M && "Can not call on null module");
-    auto Lock = TSCtx.getLock();
-    return F(*M);
+    return TSCtx.withContextDo([&](const LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(*M);
+    });
   }
 
   /// Locks the associated ThreadSafeContext and calls the given function,
   /// passing the contained std::unique_ptr<Module>. The given function should
   /// consume the Module.
   template <typename Func> decltype(auto) consumingModuleDo(Func &&F) {
-    auto Lock = TSCtx.getLock();
-    return F(std::move(M));
+    return TSCtx.withContextDo([&](LLVMContext *) {
+      assert(M && "Can not call on null module");
+      return F(std::move(M));
+    });
   }
 
   /// Get a raw pointer to the contained module without locking the context.
@@ -169,6 +153,12 @@ class ThreadSafeModule {
 using GVPredicate = std::function<bool(const GlobalValue &)>;
 using GVModifier = std::function<void(GlobalValue &)>;
 
+/// Clones teh given module onto the given context.
+LLVM_ABI ThreadSafeModule
+cloneToContext(const ThreadSafeModule &TSMW, ThreadSafeContext TSCtx,
+               GVPredicate ShouldCloneDef = GVPredicate(),
+               GVModifier UpdateClonedDefSource = GVModifier());
+
 /// Clones the given module on to a new context.
 LLVM_ABI ThreadSafeModule cloneToNewContext(
     const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef = GVPredicate(),
diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
index c1fab1199f51..57387f029b6c 100644
--- a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
+++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h
@@ -287,7 +287,7 @@ class RuntimeDyld {
   LLVM_ABI void finalizeWithMemoryManagerLocking();
 
 private:
-  LLVM_ABI_FRIEND friend void jitLinkForORC(
+  LLVM_ABI friend void jitLinkForORC(
       object::OwningBinary<object::ObjectFile> O,
       RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver,
       bool ProcessAllSections,
diff --git a/llvm/include/llvm/Frontend/Directive/Spelling.h b/llvm/include/llvm/Frontend/Directive/Spelling.h
index a10148960325..a13e26e88823 100644
--- a/llvm/include/llvm/Frontend/Directive/Spelling.h
+++ b/llvm/include/llvm/Frontend/Directive/Spelling.h
@@ -10,6 +10,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Compiler.h"
 
 #include <limits>
 #include <tuple>
@@ -33,7 +34,8 @@ struct Spelling {
   VersionRange Versions;
 };
 
-StringRef FindName(llvm::iterator_range<const Spelling *>, unsigned Version);
+LLVM_ABI StringRef FindName(llvm::iterator_range<const Spelling *>,
+                            unsigned Version);
 
 } // namespace llvm::directive
 
diff --git a/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h b/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
index f0168c040788..17ba28b6de44 100644
--- a/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
+++ b/llvm/include/llvm/Frontend/Driver/CodeGenOptions.h
@@ -63,7 +63,7 @@ enum ProfileInstrKind {
 };
 
 // Default filename used for profile generation.
-std::string getDefaultProfileGenName();
+LLVM_ABI std::string getDefaultProfileGenName();
 } // end namespace llvm::driver
 
 #endif
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index 9dfbd3cb6892..e44612af071b 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -14,8 +14,10 @@
 #ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 #define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATURE_H
 
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DXILABI.h"
+#include "llvm/Support/raw_ostream.h"
 #include <limits>
 #include <variant>
 
@@ -23,126 +25,6 @@ namespace llvm {
 namespace hlsl {
 namespace rootsig {
 
-// Definition of the various enumerations and flags. The definitions of all
-// values here correspond to their description in the d3d12.h header and are
-// carried over from their values in DXC. For reference:
-// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/
-
-enum class RootFlags : uint32_t {
-  None = 0,
-  AllowInputAssemblerInputLayout = 0x1,
-  DenyVertexShaderRootAccess = 0x2,
-  DenyHullShaderRootAccess = 0x4,
-  DenyDomainShaderRootAccess = 0x8,
-  DenyGeometryShaderRootAccess = 0x10,
-  DenyPixelShaderRootAccess = 0x20,
-  AllowStreamOutput = 0x40,
-  LocalRootSignature = 0x80,
-  DenyAmplificationShaderRootAccess = 0x100,
-  DenyMeshShaderRootAccess = 0x200,
-  CBVSRVUAVHeapDirectlyIndexed = 0x400,
-  SamplerHeapDirectlyIndexed = 0x800,
-  ValidFlags = 0x00000fff
-};
-
-enum class RootDescriptorFlags : unsigned {
-  None = 0,
-  DataVolatile = 0x2,
-  DataStaticWhileSetAtExecute = 0x4,
-  DataStatic = 0x8,
-  ValidFlags = 0xe,
-};
-
-enum class DescriptorRangeFlags : unsigned {
-  None = 0,
-  DescriptorsVolatile = 0x1,
-  DataVolatile = 0x2,
-  DataStaticWhileSetAtExecute = 0x4,
-  DataStatic = 0x8,
-  DescriptorsStaticKeepingBufferBoundsChecks = 0x10000,
-  ValidFlags = 0x1000f,
-  ValidSamplerFlags = DescriptorsVolatile,
-};
-
-enum class ShaderVisibility {
-  All = 0,
-  Vertex = 1,
-  Hull = 2,
-  Domain = 3,
-  Geometry = 4,
-  Pixel = 5,
-  Amplification = 6,
-  Mesh = 7,
-};
-
-// D3D12_FILTER enumeration:
-// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_filter
-enum class SamplerFilter {
-  MinMagMipPoint = 0,
-  MinMagPointMipLinear = 0x1,
-  MinPointMagLinearMipPoint = 0x4,
-  MinPointMagMipLinear = 0x5,
-  MinLinearMagMipPoint = 0x10,
-  MinLinearMagPointMipLinear = 0x11,
-  MinMagLinearMipPoint = 0x14,
-  MinMagMipLinear = 0x15,
-  Anisotropic = 0x55,
-  ComparisonMinMagMipPoint = 0x80,
-  ComparisonMinMagPointMipLinear = 0x81,
-  ComparisonMinPointMagLinearMipPoint = 0x84,
-  ComparisonMinPointMagMipLinear = 0x85,
-  ComparisonMinLinearMagMipPoint = 0x90,
-  ComparisonMinLinearMagPointMipLinear = 0x91,
-  ComparisonMinMagLinearMipPoint = 0x94,
-  ComparisonMinMagMipLinear = 0x95,
-  ComparisonAnisotropic = 0xd5,
-  MinimumMinMagMipPoint = 0x100,
-  MinimumMinMagPointMipLinear = 0x101,
-  MinimumMinPointMagLinearMipPoint = 0x104,
-  MinimumMinPointMagMipLinear = 0x105,
-  MinimumMinLinearMagMipPoint = 0x110,
-  MinimumMinLinearMagPointMipLinear = 0x111,
-  MinimumMinMagLinearMipPoint = 0x114,
-  MinimumMinMagMipLinear = 0x115,
-  MinimumAnisotropic = 0x155,
-  MaximumMinMagMipPoint = 0x180,
-  MaximumMinMagPointMipLinear = 0x181,
-  MaximumMinPointMagLinearMipPoint = 0x184,
-  MaximumMinPointMagMipLinear = 0x185,
-  MaximumMinLinearMagMipPoint = 0x190,
-  MaximumMinLinearMagPointMipLinear = 0x191,
-  MaximumMinMagLinearMipPoint = 0x194,
-  MaximumMinMagMipLinear = 0x195,
-  MaximumAnisotropic = 0x1d5
-};
-
-enum class TextureAddressMode {
-  Wrap = 1,
-  Mirror = 2,
-  Clamp = 3,
-  Border = 4,
-  MirrorOnce = 5
-};
-
-enum class ComparisonFunc : unsigned {
-  Never = 1,
-  Less = 2,
-  Equal = 3,
-  LessEqual = 4,
-  Greater = 5,
-  NotEqual = 6,
-  GreaterEqual = 7,
-  Always = 8
-};
-
-enum class StaticBorderColor {
-  TransparentBlack = 0,
-  OpaqueBlack = 1,
-  OpaqueWhite = 2,
-  OpaqueBlackUint = 3,
-  OpaqueWhiteUint = 4
-};
-
 // Definitions of the in-memory data layout structures
 
 // Models the different registers: bReg | tReg | uReg | sReg
@@ -157,7 +39,7 @@ struct RootConstants {
   uint32_t Num32BitConstants;
   Register Reg;
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
 };
 
 enum class DescriptorType : uint8_t { SRV = 0, UAV, CBuffer };
@@ -166,17 +48,24 @@ struct RootDescriptor {
   DescriptorType Type;
   Register Reg;
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
-  RootDescriptorFlags Flags;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
+  dxbc::RootDescriptorFlags Flags;
 
-  void setDefaultFlags() {
+  void setDefaultFlags(dxbc::RootSignatureVersion Version) {
+    if (Version == dxbc::RootSignatureVersion::V1_0) {
+      Flags = dxbc::RootDescriptorFlags::DataVolatile;
+      return;
+    }
+
+    assert(Version == llvm::dxbc::RootSignatureVersion::V1_1 &&
+           "Specified an invalid root signature version");
     switch (Type) {
     case DescriptorType::CBuffer:
     case DescriptorType::SRV:
-      Flags = RootDescriptorFlags::DataStaticWhileSetAtExecute;
+      Flags = dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute;
       break;
     case DescriptorType::UAV:
-      Flags = RootDescriptorFlags::DataVolatile;
+      Flags = dxbc::RootDescriptorFlags::DataVolatile;
       break;
     }
   }
@@ -184,7 +73,7 @@ struct RootDescriptor {
 
 // Models the end of a descriptor table and stores its visibility
 struct DescriptorTable {
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
   // Denotes that the previous NumClauses in the RootElement array
   // are the clauses in the table.
   uint32_t NumClauses = 0;
@@ -200,19 +89,28 @@ struct DescriptorTableClause {
   uint32_t NumDescriptors = 1;
   uint32_t Space = 0;
   uint32_t Offset = DescriptorTableOffsetAppend;
-  DescriptorRangeFlags Flags;
+  dxbc::DescriptorRangeFlags Flags;
+
+  void setDefaultFlags(dxbc::RootSignatureVersion Version) {
+    if (Version == dxbc::RootSignatureVersion::V1_0) {
+      Flags = dxbc::DescriptorRangeFlags::DescriptorsVolatile;
+      if (Type != ClauseType::Sampler)
+        Flags |= dxbc::DescriptorRangeFlags::DataVolatile;
+      return;
+    }
 
-  void setDefaultFlags() {
+    assert(Version == dxbc::RootSignatureVersion::V1_1 &&
+           "Specified an invalid root signature version");
     switch (Type) {
     case ClauseType::CBuffer:
     case ClauseType::SRV:
-      Flags = DescriptorRangeFlags::DataStaticWhileSetAtExecute;
+      Flags = dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute;
       break;
     case ClauseType::UAV:
-      Flags = DescriptorRangeFlags::DataVolatile;
+      Flags = dxbc::DescriptorRangeFlags::DataVolatile;
       break;
     case ClauseType::Sampler:
-      Flags = DescriptorRangeFlags::None;
+      Flags = dxbc::DescriptorRangeFlags::None;
       break;
     }
   }
@@ -220,18 +118,18 @@ struct DescriptorTableClause {
 
 struct StaticSampler {
   Register Reg;
-  SamplerFilter Filter = SamplerFilter::Anisotropic;
-  TextureAddressMode AddressU = TextureAddressMode::Wrap;
-  TextureAddressMode AddressV = TextureAddressMode::Wrap;
-  TextureAddressMode AddressW = TextureAddressMode::Wrap;
+  dxbc::SamplerFilter Filter = dxbc::SamplerFilter::Anisotropic;
+  dxbc::TextureAddressMode AddressU = dxbc::TextureAddressMode::Wrap;
+  dxbc::TextureAddressMode AddressV = dxbc::TextureAddressMode::Wrap;
+  dxbc::TextureAddressMode AddressW = dxbc::TextureAddressMode::Wrap;
   float MipLODBias = 0.f;
   uint32_t MaxAnisotropy = 16;
-  ComparisonFunc CompFunc = ComparisonFunc::LessEqual;
-  StaticBorderColor BorderColor = StaticBorderColor::OpaqueWhite;
+  dxbc::ComparisonFunc CompFunc = dxbc::ComparisonFunc::LessEqual;
+  dxbc::StaticBorderColor BorderColor = dxbc::StaticBorderColor::OpaqueWhite;
   float MinLOD = 0.f;
   float MaxLOD = std::numeric_limits<float>::max();
   uint32_t Space = 0;
-  ShaderVisibility Visibility = ShaderVisibility::All;
+  dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
 };
 
 /// Models RootElement : RootFlags | RootConstants | RootParam
@@ -251,8 +149,23 @@ struct StaticSampler {
 /// RootElements in the array, and it holds a data member for the Visibility
 /// parameter.
 using RootElement =
-    std::variant<RootFlags, RootConstants, RootDescriptor, DescriptorTable,
-                 DescriptorTableClause, StaticSampler>;
+    std::variant<dxbc::RootFlags, RootConstants, RootDescriptor,
+                 DescriptorTable, DescriptorTableClause, StaticSampler>;
+
+/// The following contains the serialization interface for root elements
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const dxbc::RootFlags &Flags);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const RootConstants &Constants);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const DescriptorTableClause &Clause);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const RootDescriptor &Descriptor);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
+                                 const StaticSampler &StaticSampler);
+LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element);
+
+LLVM_ABI void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
 
 } // namespace rootsig
 } // namespace hlsl
diff --git a/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
new file mode 100644
index 000000000000..0aa122f668ef
--- /dev/null
+++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureMetadata.h
@@ -0,0 +1,56 @@
+//===- RootSignatureMetadata.h - HLSL Root Signature helpers --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains a library for working with HLSL Root Signatures and
+/// their metadata representation.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
+#define LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
+
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+
+namespace llvm {
+class LLVMContext;
+class MDNode;
+class Metadata;
+
+namespace hlsl {
+namespace rootsig {
+
+class MetadataBuilder {
+public:
+  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
+      : Ctx(Ctx), Elements(Elements) {}
+
+  /// Iterates through elements and dispatches onto the correct Build* method
+  ///
+  /// Accumulates the root signature and returns the Metadata node that is just
+  /// a list of all the elements
+  LLVM_ABI MDNode *BuildRootSignature();
+
+private:
+  /// Define the various builders for the different metadata types
+  MDNode *BuildRootFlags(const dxbc::RootFlags &Flags);
+  MDNode *BuildRootConstants(const RootConstants &Constants);
+  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
+  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
+  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
+  MDNode *BuildStaticSampler(const StaticSampler &Sampler);
+
+  llvm::LLVMContext &Ctx;
+  ArrayRef<RootElement> Elements;
+  SmallVector<Metadata *> GeneratedMetadata;
+};
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
+
+#endif // LLVM_FRONTEND_HLSL_ROOTSIGNATUREMETADATA_H
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
similarity index 50%
rename from llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
rename to llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
index b871d3839fe0..ee4e3cc90118 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
+++ b/llvm/include/llvm/Frontend/HLSL/RootSignatureValidations.h
@@ -1,4 +1,4 @@
-//===- HLSLRootSignatureUtils.h - HLSL Root Signature helpers -------------===//
+//===- RootSignatureValidations.h - HLSL Root Signature helpers -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -11,68 +11,16 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
-#define LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+#ifndef LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
+#define LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/IntervalMap.h"
 #include "llvm/Frontend/HLSL/HLSLRootSignature.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
-class LLVMContext;
-class MDNode;
-class Metadata;
-
 namespace hlsl {
 namespace rootsig {
 
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootFlags &Flags);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const RootConstants &Constants);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const DescriptorTableClause &Clause);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const RootDescriptor &Descriptor);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS,
-                                 const StaticSampler &StaticSampler);
-
-LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element);
-
-LLVM_ABI void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements);
-
-class MetadataBuilder {
-public:
-  MetadataBuilder(llvm::LLVMContext &Ctx, ArrayRef<RootElement> Elements)
-      : Ctx(Ctx), Elements(Elements) {}
-
-  /// Iterates through the elements and dispatches onto the correct Build method
-  ///
-  /// Accumulates the root signature and returns the Metadata node that is just
-  /// a list of all the elements
-  LLVM_ABI MDNode *BuildRootSignature();
-
-private:
-  /// Define the various builders for the different metadata types
-  MDNode *BuildRootFlags(const RootFlags &Flags);
-  MDNode *BuildRootConstants(const RootConstants &Constants);
-  MDNode *BuildRootDescriptor(const RootDescriptor &Descriptor);
-  MDNode *BuildDescriptorTable(const DescriptorTable &Table);
-  MDNode *BuildDescriptorTableClause(const DescriptorTableClause &Clause);
-  MDNode *BuildStaticSampler(const StaticSampler &Sampler);
-
-  llvm::LLVMContext &Ctx;
-  ArrayRef<RootElement> Elements;
-  SmallVector<Metadata *> GeneratedMetadata;
-};
-
 struct RangeInfo {
   const static uint32_t Unbounded = ~0u;
 
@@ -83,7 +31,7 @@ struct RangeInfo {
   // Information retained for diagnostics
   llvm::dxil::ResourceClass Class;
   uint32_t Space;
-  ShaderVisibility Visibility;
+  llvm::dxbc::ShaderVisibility Visibility;
 };
 
 class ResourceRange {
@@ -99,13 +47,14 @@ class ResourceRange {
 
   // Returns a reference to the first RangeInfo that overlaps with
   // [Info.LowerBound;Info.UpperBound], or, std::nullopt if there is no overlap
-  std::optional<const RangeInfo *> getOverlapping(const RangeInfo &Info) const;
+  LLVM_ABI std::optional<const RangeInfo *>
+  getOverlapping(const RangeInfo &Info) const;
 
   // Return the mapped RangeInfo at X or nullptr if no mapping exists
-  const RangeInfo *lookup(uint32_t X) const;
+  LLVM_ABI const RangeInfo *lookup(uint32_t X) const;
 
   // Removes all entries of the ResourceRange
-  void clear();
+  LLVM_ABI void clear();
 
   // Insert the required (sub-)intervals such that the interval of [a;b] =
   // [Info.LowerBound, Info.UpperBound] is covered and points to a valid
@@ -133,11 +82,11 @@ class ResourceRange {
   // Returns a reference to the first RangeInfo that overlaps with
   // [Info.LowerBound;Info.UpperBound], or, std::nullopt if there is no overlap
   // (equivalent to getOverlapping)
-  std::optional<const RangeInfo *> insert(const RangeInfo &Info);
+  LLVM_ABI std::optional<const RangeInfo *> insert(const RangeInfo &Info);
 };
 
 } // namespace rootsig
 } // namespace hlsl
 } // namespace llvm
 
-#endif // LLVM_FRONTEND_HLSL_HLSLROOTSIGNATUREUTILS_H
+#endif // LLVM_FRONTEND_HLSL_ROOTSIGNATUREVALIDATIONS_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h b/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h
new file mode 100644
index 000000000000..898c32aa4b8f
--- /dev/null
+++ b/llvm/include/llvm/Frontend/OpenMP/DirectiveNameParser.h
@@ -0,0 +1,77 @@
+//===- DirectiveNameParser.h  ------------------------------------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
+#define LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <memory>
+
+namespace llvm::omp {
+/// Parser class for OpenMP directive names. It only recognizes names listed
+/// in OMP.td, in particular it does not recognize Fortran's end-directives
+/// if they are not explicitly listed in OMP.td.
+///
+/// The class itself may be a singleton, once it's constructed it never
+/// changes.
+///
+/// Usage:
+/// {
+///   DirectiveNameParser Parser;   // Could be static const.
+///
+///   DirectiveNameParser::State *S = Parser.initial();
+///   for (StringRef Token : Tokens)
+///     S = Parser.consume(S, Token); // Passing nullptr is ok.
+///
+///   if (S == nullptr) {
+///     // Error: ended up in a state from which there is no possible path
+///     // to a successful parse.
+///   } else if (S->Value == OMPD_unknown) {
+///     // Parsed a sequence of tokens that are not a complete name, but
+///     // parsing more tokens could lead to a successful parse.
+///   } else {
+///     // Success.
+///     ParsedId = S->Value;
+///   }
+/// }
+struct DirectiveNameParser {
+  DirectiveNameParser(SourceLanguage L = SourceLanguage::C);
+
+  struct State {
+    Directive Value = Directive::OMPD_unknown;
+
+  private:
+    using TransitionMapTy = StringMap<State>;
+    std::unique_ptr<TransitionMapTy> Transition;
+
+    State *next(StringRef Tok);
+    const State *next(StringRef Tok) const;
+    bool isValid() const {
+      return Value != Directive::OMPD_unknown || !Transition->empty();
+    }
+    friend struct DirectiveNameParser;
+  };
+
+  const State *initial() const { return &InitialState; }
+  const State *consume(const State *Current, StringRef Tok) const;
+
+  static SmallVector<StringRef> tokenize(StringRef N);
+
+private:
+  void insertName(StringRef Name, Directive D);
+  State *insertTransition(State *From, StringRef Tok);
+
+  State InitialState;
+};
+} // namespace llvm::omp
+
+#endif // LLVM_FRONTEND_OPENMP_DIRECTIVENAMEPARSER_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index a87111cb5a11..f2610011a7e0 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -355,9 +355,16 @@ def OMPC_NumTeams : Clause<[Spelling<"num_teams">]> {
   let clangClass = "OMPNumTeamsClause";
   let flangClass = "ScalarIntExpr";
 }
+def OMP_NUMTHREADS_Strict : EnumVal<"strict", 1, 1> {}
+def OMP_NUMTHREADS_Unknown : EnumVal<"unknown", 2, 0> { let isDefault = 1; }
 def OMPC_NumThreads : Clause<[Spelling<"num_threads">]> {
   let clangClass = "OMPNumThreadsClause";
   let flangClass = "ScalarIntExpr";
+  let enumClauseValue = "NumThreadsType";
+  let allowedClauseValues = [
+    OMP_NUMTHREADS_Strict,
+    OMP_NUMTHREADS_Unknown
+  ];
 }
 def OMPC_OMPX_Attribute : Clause<[Spelling<"ompx_attribute">]> {
   let clangClass = "OMPXAttributeClause";
@@ -644,7 +651,8 @@ def OMP_EndAssumes : Directive<[Spelling<"end assumes">]> {
   let category = OMP_BeginAssumes.category;
   let languages = OMP_BeginAssumes.languages;
 }
-def OMP_BeginDeclareTarget : Directive<[Spelling<"begin declare target">]> {
+def OMP_BeginDeclareTarget : Directive<[Spelling<"begin declare target", 1, 52>,
+                                        Spelling<"begin declare_target", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_DeviceType>,
     VersionedClause<OMPC_Indirect, 51>,
@@ -655,17 +663,21 @@ def OMP_BeginDeclareTarget : Directive<[Spelling<"begin declare target">]> {
   let category = CA_Declarative;
   let languages = [L_C];
 }
-def OMP_EndDeclareTarget : Directive<[Spelling<"end declare target">]> {
+def OMP_EndDeclareTarget : Directive<[Spelling<"end declare target", 1, 52>,
+                                      Spelling<"end declare_target", 60>]> {
   let association = AS_Delimited;
   let category = OMP_BeginDeclareTarget.category;
   let languages = OMP_BeginDeclareTarget.languages;
 }
-def OMP_BeginDeclareVariant : Directive<[Spelling<"begin declare variant">]> {
+def OMP_BeginDeclareVariant
+    : Directive<[Spelling<"begin declare variant", 1, 52>,
+                 Spelling<"begin declare_variant", 60>]> {
   let association = AS_Delimited;
   let category = CA_Declarative;
   let languages = [L_C];
 }
-def OMP_EndDeclareVariant : Directive<[Spelling<"end declare variant">]> {
+def OMP_EndDeclareVariant : Directive<[Spelling<"end declare variant", 1, 52>,
+                                       Spelling<"end declare_variant", 60>]> {
   let association = AS_Delimited;
   let category = OMP_BeginDeclareVariant.category;
   let languages = OMP_BeginDeclareVariant.languages;
@@ -678,7 +690,8 @@ def OMP_Cancel : Directive<[Spelling<"cancel">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_CancellationPoint : Directive<[Spelling<"cancellation point">]> {
+def OMP_CancellationPoint : Directive<[Spelling<"cancellation point", 1, 52>,
+                                       Spelling<"cancellation_point", 60>]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_CancellationConstructType>,
   ];
@@ -692,21 +705,24 @@ def OMP_Critical : Directive<[Spelling<"critical">]> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_DeclareMapper : Directive<[Spelling<"declare mapper">]> {
+def OMP_DeclareMapper : Directive<[Spelling<"declare mapper", 1, 52>,
+                                   Spelling<"declare_mapper", 60>]> {
   let requiredClauses = [
     VersionedClause<OMPC_Map, 45>,
   ];
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareReduction : Directive<[Spelling<"declare reduction">]> {
+def OMP_DeclareReduction : Directive<[Spelling<"declare reduction", 1, 52>,
+                                      Spelling<"declare_reduction", 60>]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Initializer>,
   ];
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareSimd : Directive<[Spelling<"declare simd">]> {
+def OMP_DeclareSimd : Directive<[Spelling<"declare simd", 1, 52>,
+                                 Spelling<"declare_simd", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Linear>,
@@ -722,7 +738,8 @@ def OMP_DeclareSimd : Directive<[Spelling<"declare simd">]> {
   let association = AS_Declaration;
   let category = CA_Declarative;
 }
-def OMP_DeclareTarget : Directive<[Spelling<"declare target">]> {
+def OMP_DeclareTarget : Directive<[Spelling<"declare target", 1, 52>,
+                                   Spelling<"declare_target", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_Enter, 52>,
     VersionedClause<OMPC_Indirect, 51>,
@@ -735,7 +752,8 @@ def OMP_DeclareTarget : Directive<[Spelling<"declare target">]> {
   let association = AS_None;
   let category = CA_Declarative;
 }
-def OMP_DeclareVariant : Directive<[Spelling<"declare variant">]> {
+def OMP_DeclareVariant : Directive<[Spelling<"declare variant", 1, 52>,
+                                    Spelling<"declare_variant", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_AdjustArgs, 51>,
   ];
@@ -1094,7 +1112,8 @@ def OMP_Target : Directive<[Spelling<"target">]> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TargetData : Directive<[Spelling<"target data">]> {
+def OMP_TargetData : Directive<[Spelling<"target data", 1, 52>,
+                                Spelling<"target_data", 60>]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_If>,
@@ -1107,7 +1126,8 @@ def OMP_TargetData : Directive<[Spelling<"target data">]> {
   let association = AS_Block;
   let category = CA_Executable;
 }
-def OMP_TargetEnterData : Directive<[Spelling<"target enter data">]> {
+def OMP_TargetEnterData : Directive<[Spelling<"target enter data", 1, 52>,
+                                     Spelling<"target_enter_data", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
   ];
@@ -1122,7 +1142,8 @@ def OMP_TargetEnterData : Directive<[Spelling<"target enter data">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_TargetExitData : Directive<[Spelling<"target exit data">]> {
+def OMP_TargetExitData : Directive<[Spelling<"target exit data", 1, 52>,
+                                    Spelling<"target_exit_data", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
   ];
@@ -1137,7 +1158,8 @@ def OMP_TargetExitData : Directive<[Spelling<"target exit data">]> {
   let association = AS_None;
   let category = CA_Executable;
 }
-def OMP_TargetUpdate : Directive<[Spelling<"target update">]> {
+def OMP_TargetUpdate : Directive<[Spelling<"target update", 1, 52>,
+                                  Spelling<"target_update", 60>]> {
   let allowedClauses = [
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_From>,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 5e614a598345..bcc5658b9148 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -485,7 +485,7 @@ class OpenMPIRBuilder {
   /// not have an effect on \p M (see initialize)
   OpenMPIRBuilder(Module &M)
       : M(M), Builder(M.getContext()), OffloadInfoManager(this),
-        T(M.getTargetTriple()) {}
+        T(M.getTargetTriple()), IsFinalized(false) {}
   LLVM_ABI ~OpenMPIRBuilder();
 
   class AtomicInfo : public llvm::AtomicInfo {
@@ -522,6 +522,10 @@ class OpenMPIRBuilder {
   ///                              all functions are finalized.
   LLVM_ABI void finalize(Function *Fn = nullptr);
 
+  /// Check whether the finalize function has already run
+  /// \return true if the finalize function has already run
+  LLVM_ABI bool isFinalized();
+
   /// Add attributes known for \p FnID to \p Fn.
   LLVM_ABI void addAttributes(omp::RuntimeFunction FnID, Function &Fn);
 
@@ -3301,6 +3305,8 @@ class OpenMPIRBuilder {
   Value *emitRMWOpAsInstruction(Value *Src1, Value *Src2,
                                 AtomicRMWInst::BinOp RMWOp);
 
+  bool IsFinalized;
+
 public:
   /// a struct to pack relevant information while generating atomic Ops
   struct AtomicOpValue {
diff --git a/llvm/include/llvm/IR/Argument.h b/llvm/include/llvm/IR/Argument.h
index 60854b17094b..b9a73b3eb5fc 100644
--- a/llvm/include/llvm/IR/Argument.h
+++ b/llvm/include/llvm/IR/Argument.h
@@ -78,6 +78,9 @@ class Argument final : public Value {
   /// Return true if this argument has the byval attribute.
   LLVM_ABI bool hasByValAttr() const;
 
+  /// Return true if this argument has the dead_on_return attribute.
+  LLVM_ABI bool hasDeadOnReturnAttr() const;
+
   /// Return true if this argument has the byref attribute.
   LLVM_ABI bool hasByRefAttr() const;
 
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 0c759953c5f1..e1123edffb2a 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -198,6 +198,9 @@ def NoFree : EnumAttr<"nofree", IntersectAnd, [FnAttr, ParamAttr]>;
 /// Argument is dead if the call unwinds.
 def DeadOnUnwind : EnumAttr<"dead_on_unwind", IntersectAnd, [ParamAttr]>;
 
+/// Argument is dead upon function return.
+def DeadOnReturn : EnumAttr<"dead_on_return", IntersectAnd, [ParamAttr]>;
+
 /// Disable implicit floating point insts.
 def NoImplicitFloat : EnumAttr<"noimplicitfloat", IntersectPreserve, [FnAttr]>;
 
@@ -392,6 +395,16 @@ def CoroDestroyOnlyWhenComplete : EnumAttr<"coro_only_destroy_when_complete", In
 /// pipeline to perform elide on the call or invoke instruction.
 def CoroElideSafe : EnumAttr<"coro_elide_safe", IntersectPreserve, [FnAttr]>;
 
+/// Function is marked for Windows Hot Patching
+def MarkedForWindowsSecureHotPatching
+    : StrBoolAttr<"marked_for_windows_hot_patching">;
+
+/// Global variable should not be accessed through a "__ref_" global variable in
+/// a hot patching function This attribute is applied to the global variable
+/// decl, not the hotpatched function.
+def AllowDirectAccessInHotPatchFunction
+    : StrBoolAttr<"allow_direct_access_in_hot_patch_function">;
+
 /// Target-independent string attributes.
 def LessPreciseFPMAD : StrBoolAttr<"less-precise-fpmad">;
 def NoInfsFPMath : StrBoolAttr<"no-infs-fp-math">;
diff --git a/llvm/include/llvm/IR/CMakeLists.txt b/llvm/include/llvm/IR/CMakeLists.txt
index 468d663796ed..c5c4cc4d21b8 100644
--- a/llvm/include/llvm/IR/CMakeLists.txt
+++ b/llvm/include/llvm/IR/CMakeLists.txt
@@ -1,6 +1,9 @@
 set(LLVM_TARGET_DEFINITIONS Attributes.td)
 tablegen(LLVM Attributes.inc -gen-attrs)
 
+set(LLVM_TARGET_DEFINITIONS RuntimeLibcalls.td)
+tablegen(LLVM RuntimeLibcalls.inc -gen-runtime-libcalls)
+
 set(LLVM_TARGET_DEFINITIONS Intrinsics.td)
 tablegen(LLVM IntrinsicImpl.inc -gen-intrinsic-impl)
 tablegen(LLVM IntrinsicEnums.inc -gen-intrinsic-enums)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index ca818c8d502d..da2c54a20f45 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -295,12 +295,11 @@ namespace llvm {
     /// \param DWARFMemorySpace  DWARF memory space. (optional)
     /// \param Name              Pointer type name. (optional)
     /// \param Annotations       Member annotations.
-    LLVM_ABI DIDerivedType *
-    createPointerType(DIType *PointeeTy, uint64_t SizeInBits,
-                      uint32_t AlignInBits = 0,
-                      std::optional<unsigned> DWARFAddressSpace = std::nullopt,
-                      dwarf::MemorySpace DWARFMemorySpace = dwarf::DW_MSPACE_LLVM_none,
-                      StringRef Name = "", DINodeArray Annotations = nullptr);
+    LLVM_ABI DIDerivedType *createPointerType(
+        DIType *PointeeTy, uint64_t SizeInBits, uint32_t AlignInBits = 0,
+        std::optional<unsigned> DWARFAddressSpace = std::nullopt,
+        dwarf::MemorySpace DWARFMemorySpace = dwarf::DW_MSPACE_LLVM_none,
+        StringRef Name = "", DINodeArray Annotations = nullptr);
 
     /// Create a __ptrauth qualifier.
     LLVM_ABI DIDerivedType *
@@ -376,6 +375,22 @@ namespace llvm {
                                               uint32_t VBPtrOffset,
                                               DINode::DIFlags Flags);
 
+    /// Create debugging information entry for a member.
+    /// \param Scope        Member scope.
+    /// \param Name         Member name.
+    /// \param File         File where this member is defined.
+    /// \param LineNo       Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param OffsetInBits Member offset.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Ty           Parent type.
+    /// \param Annotations  Member annotations.
+    LLVM_ABI DIDerivedType *createMemberType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo,
+        Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
+        DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations = nullptr);
+
     /// Create debugging information entry for a member.
     /// \param Scope        Member scope.
     /// \param Name         Member name.
@@ -428,6 +443,23 @@ namespace llvm {
                                                     Constant *Discriminant,
                                                     DIType *Ty);
 
+    /// Create debugging information entry for a bit field member.
+    /// \param Scope               Member scope.
+    /// \param Name                Member name.
+    /// \param File                File where this member is defined.
+    /// \param LineNo              Line number.
+    /// \param SizeInBits          Member size.
+    /// \param OffsetInBits        Member offset.
+    /// \param StorageOffsetInBits Member storage offset.
+    /// \param Flags               Flags to encode member attribute.
+    /// \param Ty                  Parent type.
+    /// \param Annotations         Member annotations.
+    LLVM_ABI DIDerivedType *createBitFieldMemberType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo,
+        Metadata *SizeInBits, Metadata *OffsetInBits,
+        uint64_t StorageOffsetInBits, DINode::DIFlags Flags, DIType *Ty,
+        DINodeArray Annotations = nullptr);
+
     /// Create debugging information entry for a bit field member.
     /// \param Scope               Member scope.
     /// \param Name                Member name.
@@ -519,6 +551,29 @@ namespace llvm {
         unsigned RunTimeLang = 0, DIType *VTableHolder = nullptr,
         MDNode *TemplateParms = nullptr, StringRef UniqueIdentifier = "");
 
+    /// Create debugging information entry for a struct.
+    /// \param Scope        Scope in which this struct is defined.
+    /// \param Name         Struct name.
+    /// \param File         File where this member is defined.
+    /// \param LineNumber   Line number.
+    /// \param SizeInBits   Member size.
+    /// \param AlignInBits  Member alignment.
+    /// \param Flags        Flags to encode member attribute, e.g. private
+    /// \param Elements     Struct elements.
+    /// \param RunTimeLang  Optional parameter, Objective-C runtime version.
+    /// \param UniqueIdentifier A unique identifier for the struct.
+    /// \param Specification The type that this type completes. This is used by
+    /// Swift to represent generic types.
+    /// \param NumExtraInhabitants The number of extra inhabitants of the type.
+    /// An extra inhabitant is a bit pattern that does not represent a valid
+    /// value for instances of a given type. This is used by the Swift language.
+    LLVM_ABI DICompositeType *createStructType(
+        DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+        Metadata *SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags,
+        DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang = 0,
+        DIType *VTableHolder = nullptr, StringRef UniqueIdentifier = "",
+        DIType *Specification = nullptr, uint32_t NumExtraInhabitants = 0);
+
     /// Create debugging information entry for a struct.
     /// \param Scope        Scope in which this struct is defined.
     /// \param Name         Struct name.
@@ -879,7 +934,10 @@ namespace llvm {
     /// \c Scope must be a \a DILocalScope, and thus its scope chain eventually
     /// leads to a \a DISubprogram.
     LLVM_ABI DILabel *createLabel(DIScope *Scope, StringRef Name, DIFile *File,
-                                  unsigned LineNo, bool AlwaysPreserve = false);
+                                  unsigned LineNo, unsigned Column,
+                                  bool IsArtificial,
+                                  std::optional<unsigned> CoroSuspendIdx,
+                                  bool AlwaysPreserve = false);
 
     /// Create a new descriptor for a parameter variable.
     ///
@@ -929,6 +987,8 @@ namespace llvm {
     /// \param Annotations   Attribute Annotations.
     /// \param TargetFuncName The name of the target function if this is
     ///                       a trampoline.
+    /// \param UseKeyInstructions Instruct DWARF emission to interpret Key
+    /// Instructions metadata on instructions to determine is_stmt placement.
     LLVM_ABI DISubprogram *createFunction(
         DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
         unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
@@ -936,7 +996,8 @@ namespace llvm {
         DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
         DITemplateParameterArray TParams = nullptr,
         DISubprogram *Decl = nullptr, DITypeArray ThrownTypes = nullptr,
-        DINodeArray Annotations = nullptr, StringRef TargetFuncName = "");
+        DINodeArray Annotations = nullptr, StringRef TargetFuncName = "",
+        bool UseKeyInstructions = false);
 
     /// Identical to createFunction,
     /// except that the resulting DbgNode is meant to be RAUWed.
@@ -967,15 +1028,15 @@ namespace llvm {
     /// \param SPFlags       Additional flags specific to subprograms.
     /// \param TParams       Function template parameters.
     /// \param ThrownTypes   Exception types this function may throw.
-    LLVM_ABI DISubprogram *
-    createMethod(DIScope *Scope, StringRef Name, StringRef LinkageName,
-                 DIFile *File, unsigned LineNo, DISubroutineType *Ty,
-                 unsigned VTableIndex = 0, int ThisAdjustment = 0,
-                 DIType *VTableHolder = nullptr,
-                 DINode::DIFlags Flags = DINode::FlagZero,
-                 DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
-                 DITemplateParameterArray TParams = nullptr,
-                 DITypeArray ThrownTypes = nullptr);
+    /// \param UseKeyInstructions Enable Key Instructions debug info.
+    LLVM_ABI DISubprogram *createMethod(
+        DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File,
+        unsigned LineNo, DISubroutineType *Ty, unsigned VTableIndex = 0,
+        int ThisAdjustment = 0, DIType *VTableHolder = nullptr,
+        DINode::DIFlags Flags = DINode::FlagZero,
+        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagZero,
+        DITemplateParameterArray TParams = nullptr,
+        DITypeArray ThrownTypes = nullptr, bool UseKeyInstructions = false);
 
     /// Create common block entry for a Fortran common block.
     /// \param Scope       Scope of this common block.
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 273057ede5b6..8ab645dd282d 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -719,40 +719,33 @@ std::optional<StringRef> DIScope::getSource() const {
 class DIType : public DIScope {
   unsigned Line;
   DIFlags Flags;
-  uint64_t SizeInBits;
-  uint64_t OffsetInBits;
   uint32_t NumExtraInhabitants;
 
 protected:
+  static constexpr unsigned N_OPERANDS = 5;
+
   DIType(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
-         unsigned Line, uint64_t SizeInBits, uint32_t AlignInBits,
-         uint64_t OffsetInBits, uint32_t NumExtraInhabitants, DIFlags Flags,
-         ArrayRef<Metadata *> Ops)
+         unsigned Line, uint32_t AlignInBits, uint32_t NumExtraInhabitants,
+         DIFlags Flags, ArrayRef<Metadata *> Ops)
       : DIScope(C, ID, Storage, Tag, Ops) {
-    init(Line, SizeInBits, AlignInBits, OffsetInBits, NumExtraInhabitants,
-         Flags);
+    init(Line, AlignInBits, NumExtraInhabitants, Flags);
   }
   ~DIType() = default;
 
-  void init(unsigned Line, uint64_t SizeInBits, uint32_t AlignInBits,
-            uint64_t OffsetInBits, uint32_t NumExtraInhabitants,
+  void init(unsigned Line, uint32_t AlignInBits, uint32_t NumExtraInhabitants,
             DIFlags Flags) {
     this->Line = Line;
     this->Flags = Flags;
-    this->SizeInBits = SizeInBits;
     this->SubclassData32 = AlignInBits;
-    this->OffsetInBits = OffsetInBits;
     this->NumExtraInhabitants = NumExtraInhabitants;
   }
 
   /// Change fields in place.
-  void mutate(unsigned Tag, unsigned Line, uint64_t SizeInBits,
-              uint32_t AlignInBits, uint64_t OffsetInBits,
+  void mutate(unsigned Tag, unsigned Line, uint32_t AlignInBits,
               uint32_t NumExtraInhabitants, DIFlags Flags) {
     assert(isDistinct() && "Only distinct nodes can mutate");
     setTag(Tag);
-    init(Line, SizeInBits, AlignInBits, OffsetInBits, NumExtraInhabitants,
-         Flags);
+    init(Line, AlignInBits, NumExtraInhabitants, Flags);
   }
 
 public:
@@ -761,10 +754,8 @@ class DIType : public DIScope {
   }
 
   unsigned getLine() const { return Line; }
-  uint64_t getSizeInBits() const { return SizeInBits; }
   LLVM_ABI uint32_t getAlignInBits() const;
   uint32_t getAlignInBytes() const { return getAlignInBits() / CHAR_BIT; }
-  uint64_t getOffsetInBits() const { return OffsetInBits; }
   uint32_t getNumExtraInhabitants() const { return NumExtraInhabitants; }
   DIFlags getFlags() const { return Flags; }
 
@@ -774,6 +765,24 @@ class DIType : public DIScope {
   Metadata *getRawScope() const { return getOperand(1); }
   MDString *getRawName() const { return getOperandAs<MDString>(2); }
 
+  Metadata *getRawSizeInBits() const { return getOperand(3); }
+  uint64_t getSizeInBits() const {
+    if (auto *MD = dyn_cast_or_null<ConstantAsMetadata>(getRawSizeInBits())) {
+      if (ConstantInt *CI = dyn_cast_or_null<ConstantInt>(MD->getValue()))
+        return CI->getZExtValue();
+    }
+    return 0;
+  }
+
+  Metadata *getRawOffsetInBits() const { return getOperand(4); }
+  uint64_t getOffsetInBits() const {
+    if (auto *MD = dyn_cast_or_null<ConstantAsMetadata>(getRawOffsetInBits())) {
+      if (ConstantInt *CI = dyn_cast_or_null<ConstantInt>(MD->getValue()))
+        return CI->getZExtValue();
+    }
+    return 0;
+  }
+
   /// Returns a new temporary DIType with updated Flags
   TempDIType cloneWithFlags(DIFlags NewFlags) const {
     auto NewTy = clone();
@@ -839,18 +848,18 @@ class DIBasicType : public DIType {
 
 protected:
   DIBasicType(LLVMContext &C, StorageType Storage, unsigned Tag,
-              uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+              uint32_t AlignInBits, unsigned Encoding,
               uint32_t NumExtraInhabitants, DIFlags Flags,
               ArrayRef<Metadata *> Ops)
-      : DIType(C, DIBasicTypeKind, Storage, Tag, 0, SizeInBits, AlignInBits, 0,
+      : DIType(C, DIBasicTypeKind, Storage, Tag, 0, AlignInBits,
                NumExtraInhabitants, Flags, Ops),
         Encoding(Encoding) {}
   DIBasicType(LLVMContext &C, unsigned ID, StorageType Storage, unsigned Tag,
-              uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+              uint32_t AlignInBits, unsigned Encoding,
               uint32_t NumExtraInhabitants, DIFlags Flags,
               ArrayRef<Metadata *> Ops)
-      : DIType(C, ID, Storage, Tag, 0, SizeInBits, AlignInBits, 0,
-               NumExtraInhabitants, Flags, Ops),
+      : DIType(C, ID, Storage, Tag, 0, AlignInBits, NumExtraInhabitants, Flags,
+               Ops),
         Encoding(Encoding) {}
   ~DIBasicType() = default;
 
@@ -868,11 +877,21 @@ class DIBasicType : public DIType {
                                        uint32_t AlignInBits, unsigned Encoding,
                                        uint32_t NumExtraInhabitants,
                                        DIFlags Flags, StorageType Storage,
-                                       bool ShouldCreate = true);
+                                       bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    return getImpl(Context, Tag, Name, SizeInBitsNode, AlignInBits, Encoding,
+                   NumExtraInhabitants, Flags, Storage, ShouldCreate);
+  }
+  static DIBasicType *getImpl(LLVMContext &Context, unsigned Tag,
+                              MDString *Name, Metadata *SizeInBits,
+                              uint32_t AlignInBits, unsigned Encoding,
+                              uint32_t NumExtraInhabitants, DIFlags Flags,
+                              StorageType Storage, bool ShouldCreate = true);
 
   TempDIBasicType cloneImpl() const {
-    return getTemporary(getContext(), getTag(), getName(), getSizeInBits(),
-                        getAlignInBits(), getEncoding(),
+    return getTemporary(getContext(), getTag(), getRawName(),
+                        getRawSizeInBits(), getAlignInBits(), getEncoding(),
                         getNumExtraInhabitants(), getFlags());
   }
 
@@ -905,6 +924,12 @@ class DIBasicType : public DIType {
                      uint32_t NumExtraInhabitants, DIFlags Flags),
                     (Tag, Name, SizeInBits, AlignInBits, Encoding,
                      NumExtraInhabitants, Flags))
+  DEFINE_MDNODE_GET(DIBasicType,
+                    (unsigned Tag, MDString *Name, Metadata *SizeInBits,
+                     uint32_t AlignInBits, unsigned Encoding,
+                     uint32_t NumExtraInhabitants, DIFlags Flags),
+                    (Tag, Name, SizeInBits, AlignInBits, Encoding,
+                     NumExtraInhabitants, Flags))
 
   TempDIBasicType clone() const { return cloneImpl(); }
 
@@ -936,29 +961,28 @@ class DIFixedPointType : public DIBasicType {
   APInt Denominator;
 
   DIFixedPointType(LLVMContext &C, StorageType Storage, unsigned Tag,
-                   uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
-                   DIFlags Flags, unsigned Kind, int Factor,
-                   ArrayRef<Metadata *> Ops)
-      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, SizeInBits,
-                    AlignInBits, Encoding, 0, Flags, Ops),
+                   uint32_t AlignInBits, unsigned Encoding, DIFlags Flags,
+                   unsigned Kind, int Factor, ArrayRef<Metadata *> Ops)
+      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, AlignInBits,
+                    Encoding, 0, Flags, Ops),
         Kind(Kind), Factor(Factor) {
     assert(Kind == FixedPointBinary || Kind == FixedPointDecimal);
   }
   DIFixedPointType(LLVMContext &C, StorageType Storage, unsigned Tag,
-                   uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
-                   DIFlags Flags, unsigned Kind, APInt Numerator,
-                   APInt Denominator, ArrayRef<Metadata *> Ops)
-      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, SizeInBits,
-                    AlignInBits, Encoding, 0, Flags, Ops),
+                   uint32_t AlignInBits, unsigned Encoding, DIFlags Flags,
+                   unsigned Kind, APInt Numerator, APInt Denominator,
+                   ArrayRef<Metadata *> Ops)
+      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, AlignInBits,
+                    Encoding, 0, Flags, Ops),
         Kind(Kind), Factor(0), Numerator(Numerator), Denominator(Denominator) {
     assert(Kind == FixedPointRational);
   }
   DIFixedPointType(LLVMContext &C, StorageType Storage, unsigned Tag,
-                   uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
-                   DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
+                   uint32_t AlignInBits, unsigned Encoding, DIFlags Flags,
+                   unsigned Kind, int Factor, APInt Numerator,
                    APInt Denominator, ArrayRef<Metadata *> Ops)
-      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, SizeInBits,
-                    AlignInBits, Encoding, 0, Flags, Ops),
+      : DIBasicType(C, DIFixedPointTypeKind, Storage, Tag, AlignInBits,
+                    Encoding, 0, Flags, Ops),
         Kind(Kind), Factor(Factor), Numerator(Numerator),
         Denominator(Denominator) {}
   ~DIFixedPointType() = default;
@@ -968,6 +992,17 @@ class DIFixedPointType : public DIBasicType {
           uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
           DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
           APInt Denominator, StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    return getImpl(Context, Tag, getCanonicalMDString(Context, Name),
+                   SizeInBitsNode, AlignInBits, Encoding, Flags, Kind, Factor,
+                   Numerator, Denominator, Storage, ShouldCreate);
+  }
+  static DIFixedPointType *
+  getImpl(LLVMContext &Context, unsigned Tag, StringRef Name,
+          Metadata *SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+          DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
+          APInt Denominator, StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name),
                    SizeInBits, AlignInBits, Encoding, Flags, Kind, Factor,
                    Numerator, Denominator, Storage, ShouldCreate);
@@ -976,12 +1011,23 @@ class DIFixedPointType : public DIBasicType {
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
           uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
           DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
+          APInt Denominator, StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    return getImpl(Context, Tag, Name, SizeInBitsNode, AlignInBits, Encoding,
+                   Flags, Kind, Factor, Numerator, Denominator, Storage,
+                   ShouldCreate);
+  }
+  static DIFixedPointType *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
+          Metadata *SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+          DIFlags Flags, unsigned Kind, int Factor, APInt Numerator,
           APInt Denominator, StorageType Storage, bool ShouldCreate = true);
 
   TempDIFixedPointType cloneImpl() const {
-    return getTemporary(getContext(), getTag(), getName(), getSizeInBits(),
-                        getAlignInBits(), getEncoding(), getFlags(), Kind,
-                        Factor, Numerator, Denominator);
+    return getTemporary(getContext(), getTag(), getRawName(),
+                        getRawSizeInBits(), getAlignInBits(), getEncoding(),
+                        getFlags(), Kind, Factor, Numerator, Denominator);
   }
 
 public:
@@ -1013,6 +1059,13 @@ class DIFixedPointType : public DIBasicType {
                      APInt Denominator),
                     (Tag, Name, SizeInBits, AlignInBits, Encoding, Flags, Kind,
                      Factor, Numerator, Denominator))
+  DEFINE_MDNODE_GET(DIFixedPointType,
+                    (unsigned Tag, MDString *Name, Metadata *SizeInBits,
+                     uint32_t AlignInBits, unsigned Encoding, DIFlags Flags,
+                     unsigned Kind, int Factor, APInt Numerator,
+                     APInt Denominator),
+                    (Tag, Name, SizeInBits, AlignInBits, Encoding, Flags, Kind,
+                     Factor, Numerator, Denominator))
 
   TempDIFixedPointType clone() const { return cloneImpl(); }
 
@@ -1052,13 +1105,15 @@ class DIStringType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
+  static constexpr unsigned MY_FIRST_OPERAND = DIType::N_OPERANDS;
+
   unsigned Encoding;
 
   DIStringType(LLVMContext &C, StorageType Storage, unsigned Tag,
-               uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
+               uint32_t AlignInBits, unsigned Encoding,
                ArrayRef<Metadata *> Ops)
-      : DIType(C, DIStringTypeKind, Storage, Tag, 0, SizeInBits, AlignInBits, 0,
-               0, FlagZero, Ops),
+      : DIType(C, DIStringTypeKind, Storage, Tag, 0, AlignInBits, 0, FlagZero,
+               Ops),
         Encoding(Encoding) {}
   ~DIStringType() = default;
 
@@ -1068,20 +1123,34 @@ class DIStringType : public DIType {
                                uint64_t SizeInBits, uint32_t AlignInBits,
                                unsigned Encoding, StorageType Storage,
                                bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name),
-                   StringLength, StrLenExp, StrLocationExp, SizeInBits,
+                   StringLength, StrLenExp, StrLocationExp, SizeInBitsNode,
                    AlignInBits, Encoding, Storage, ShouldCreate);
   }
   LLVM_ABI static DIStringType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
           Metadata *StringLength, Metadata *StrLenExp, Metadata *StrLocationExp,
           uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding,
-          StorageType Storage, bool ShouldCreate = true);
+          StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    return getImpl(Context, Tag, Name, StringLength, StrLenExp, StrLocationExp,
+                   SizeInBitsNode, AlignInBits, Encoding, Storage,
+                   ShouldCreate);
+  }
+  static DIStringType *getImpl(LLVMContext &Context, unsigned Tag,
+                               MDString *Name, Metadata *StringLength,
+                               Metadata *StrLenExp, Metadata *StrLocationExp,
+                               Metadata *SizeInBits, uint32_t AlignInBits,
+                               unsigned Encoding, StorageType Storage,
+                               bool ShouldCreate = true);
 
   TempDIStringType cloneImpl() const {
     return getTemporary(getContext(), getTag(), getRawName(),
                         getRawStringLength(), getRawStringLengthExp(),
-                        getRawStringLocationExp(), getSizeInBits(),
+                        getRawStringLocationExp(), getRawSizeInBits(),
                         getAlignInBits(), getEncoding());
   }
 
@@ -1105,6 +1174,13 @@ class DIStringType : public DIType {
                      unsigned Encoding),
                     (Tag, Name, StringLength, StringLengthExp,
                      StringLocationExp, SizeInBits, AlignInBits, Encoding))
+  DEFINE_MDNODE_GET(DIStringType,
+                    (unsigned Tag, MDString *Name, Metadata *StringLength,
+                     Metadata *StringLengthExp, Metadata *StringLocationExp,
+                     Metadata *SizeInBits, uint32_t AlignInBits,
+                     unsigned Encoding),
+                    (Tag, Name, StringLength, StringLengthExp,
+                     StringLocationExp, SizeInBits, AlignInBits, Encoding))
 
   TempDIStringType clone() const { return cloneImpl(); }
 
@@ -1126,11 +1202,15 @@ class DIStringType : public DIType {
 
   unsigned getEncoding() const { return Encoding; }
 
-  Metadata *getRawStringLength() const { return getOperand(3); }
+  Metadata *getRawStringLength() const { return getOperand(MY_FIRST_OPERAND); }
 
-  Metadata *getRawStringLengthExp() const { return getOperand(4); }
+  Metadata *getRawStringLengthExp() const {
+    return getOperand(MY_FIRST_OPERAND + 1);
+  }
 
-  Metadata *getRawStringLocationExp() const { return getOperand(5); }
+  Metadata *getRawStringLocationExp() const {
+    return getOperand(MY_FIRST_OPERAND + 2);
+  }
 };
 
 /// Derived types.
@@ -1172,18 +1252,20 @@ class DIDerivedType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
+  static constexpr unsigned MY_FIRST_OPERAND = DIType::N_OPERANDS;
+
   /// The DWARF address space of the memory pointed to or referenced by a
   /// pointer or reference type respectively.
   std::optional<unsigned> DWARFAddressSpace;
   dwarf::MemorySpace DWARFMemorySpace;
 
   DIDerivedType(LLVMContext &C, StorageType Storage, unsigned Tag,
-                unsigned Line, uint64_t SizeInBits, uint32_t AlignInBits,
-                uint64_t OffsetInBits,
-                std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS, 
-                std::optional<PtrAuthData> PtrAuthData, DIFlags Flags, ArrayRef<Metadata *> Ops)
-      : DIType(C, DIDerivedTypeKind, Storage, Tag, Line, SizeInBits,
-               AlignInBits, OffsetInBits, 0, Flags, Ops),
+                unsigned Line, uint32_t AlignInBits,
+                std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+                std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+                ArrayRef<Metadata *> Ops)
+      : DIType(C, DIDerivedTypeKind, Storage, Tag, Line, AlignInBits, 0, Flags,
+               Ops),
         DWARFAddressSpace(DWARFAddressSpace), DWARFMemorySpace(MS) {
     if (PtrAuthData)
       SubclassData32 = PtrAuthData->RawData;
@@ -1193,9 +1275,44 @@ class DIDerivedType : public DIType {
   getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, DIFile *File,
           unsigned Line, DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
           uint32_t AlignInBits, uint64_t OffsetInBits,
-          std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS, 
-          std::optional<PtrAuthData> PtrAuthData, DIFlags Flags, Metadata *ExtraData, DINodeArray Annotations,
-          StorageType Storage, bool ShouldCreate = true) {
+          std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+          std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+          Metadata *ExtraData, DINodeArray Annotations, StorageType Storage,
+          bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    auto *OffsetInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), OffsetInBits));
+    return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File,
+                   Line, Scope, BaseType, SizeInBitsNode, AlignInBits,
+                   OffsetInBitsNode, DWARFAddressSpace, MS, PtrAuthData, Flags,
+                   ExtraData, Annotations.get(), Storage, ShouldCreate);
+  }
+  static DIDerivedType *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, DIFile *File,
+          unsigned Line, DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
+          uint32_t AlignInBits, uint64_t OffsetInBits,
+          std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+          std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+          Metadata *ExtraData, DINodeArray Annotations, StorageType Storage,
+          bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    auto *OffsetInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), OffsetInBits));
+    return getImpl(Context, Tag, Name, File, Line, Scope, BaseType,
+                   SizeInBitsNode, AlignInBits, OffsetInBitsNode,
+                   DWARFAddressSpace, MS, PtrAuthData, Flags, ExtraData,
+                   Annotations.get(), Storage, ShouldCreate);
+  }
+  static DIDerivedType *
+  getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, DIFile *File,
+          unsigned Line, DIScope *Scope, DIType *BaseType, Metadata *SizeInBits,
+          uint32_t AlignInBits, Metadata *OffsetInBits,
+          std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+          std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+          Metadata *ExtraData, DINodeArray Annotations, StorageType Storage,
+          bool ShouldCreate = true) {
     return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File,
                    Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits,
                    DWARFAddressSpace, MS, PtrAuthData, Flags, ExtraData, Annotations.get(),
@@ -1204,31 +1321,55 @@ class DIDerivedType : public DIType {
   LLVM_ABI static DIDerivedType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
           unsigned Line, Metadata *Scope, Metadata *BaseType,
-          uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+          Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
           std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
           std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
           Metadata *ExtraData, Metadata *Annotations, StorageType Storage,
           bool ShouldCreate = true);
 
   TempDIDerivedType cloneImpl() const {
-    return getTemporary(getContext(), getTag(), getName(), getFile(), getLine(),
-                        getScope(), getBaseType(), getSizeInBits(),
-                        getAlignInBits(), getOffsetInBits(),
-                        getDWARFAddressSpace(), getDWARFMemorySpace(), getPtrAuthData(),
-                        getFlags(), getExtraData(), getAnnotations());
+    return getTemporary(
+        getContext(), getTag(), getRawName(), getFile(), getLine(), getScope(),
+        getBaseType(), getRawSizeInBits(), getAlignInBits(),
+        getRawOffsetInBits(), getDWARFAddressSpace(), getDWARFMemorySpace(), getPtrAuthData(),
+        getFlags(), getExtraData(), getRawAnnotations());
   }
 
 public:
-  DEFINE_MDNODE_GET(
-      DIDerivedType,
-      (unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
-       Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
-       uint32_t AlignInBits, uint64_t OffsetInBits,
-       std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS, 
-       std::optional<PtrAuthData> PtrAuthData, DIFlags Flags, Metadata *ExtraData = nullptr,
-       Metadata *Annotations = nullptr),
-      (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
-       OffsetInBits, DWARFAddressSpace, MS, PtrAuthData, Flags, ExtraData, Annotations))
+  DEFINE_MDNODE_GET(DIDerivedType,
+                    (unsigned Tag, MDString *Name, Metadata *File,
+                     unsigned Line, Metadata *Scope, Metadata *BaseType,
+                     Metadata *SizeInBits, uint32_t AlignInBits,
+                     Metadata *OffsetInBits,
+                     std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+                     std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+                     Metadata *ExtraData = nullptr,
+                     Metadata *Annotations = nullptr),
+                    (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                     AlignInBits, OffsetInBits, DWARFAddressSpace, MS, PtrAuthData,
+                     Flags, ExtraData, Annotations))
+  DEFINE_MDNODE_GET(DIDerivedType,
+                    (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
+                     DIScope *Scope, DIType *BaseType, Metadata *SizeInBits,
+                     uint32_t AlignInBits, Metadata *OffsetInBits,
+                     std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+                     std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+                     Metadata *ExtraData = nullptr,
+                     DINodeArray Annotations = nullptr),
+                    (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                     AlignInBits, OffsetInBits, DWARFAddressSpace, MS, PtrAuthData,
+                     Flags, ExtraData, Annotations))
+  DEFINE_MDNODE_GET(DIDerivedType,
+                    (unsigned Tag, MDString *Name, DIFile *File, unsigned Line,
+                     DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
+                     uint32_t AlignInBits, uint64_t OffsetInBits,
+                     std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
+                     std::optional<PtrAuthData> PtrAuthData, DIFlags Flags,
+                     Metadata *ExtraData = nullptr,
+                     DINodeArray Annotations = nullptr),
+                    (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
+                     AlignInBits, OffsetInBits, DWARFAddressSpace, MS, PtrAuthData,
+                     Flags, ExtraData, Annotations))
   DEFINE_MDNODE_GET(DIDerivedType,
                     (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
                      DIScope *Scope, DIType *BaseType, uint64_t SizeInBits,
@@ -1251,7 +1392,7 @@ class DIDerivedType : public DIType {
 
   /// Get the base type this is derived from.
   DIType *getBaseType() const { return cast_or_null<DIType>(getRawBaseType()); }
-  Metadata *getRawBaseType() const { return getOperand(3); }
+  Metadata *getRawBaseType() const { return getOperand(MY_FIRST_OPERAND); }
 
   /// \returns The DWARF address space of the memory pointed to or referenced by
   /// a pointer or reference type respectively.
@@ -1275,7 +1416,7 @@ class DIDerivedType : public DIType {
   /// TODO: Separate out types that need this extra operand: pointer-to-member
   /// types and member fields (static members and ivars).
   Metadata *getExtraData() const { return getRawExtraData(); }
-  Metadata *getRawExtraData() const { return getOperand(4); }
+  Metadata *getRawExtraData() const { return getOperand(MY_FIRST_OPERAND + 1); }
 
   /// Get the template parameters from a template alias.
   DITemplateParameterArray getTemplateParams() const {
@@ -1286,7 +1427,9 @@ class DIDerivedType : public DIType {
   DINodeArray getAnnotations() const {
     return cast_or_null<MDTuple>(getRawAnnotations());
   }
-  Metadata *getRawAnnotations() const { return getOperand(5); }
+  Metadata *getRawAnnotations() const {
+    return getOperand(MY_FIRST_OPERAND + 2);
+  }
 
   /// Get casted version of extra data.
   /// @{
@@ -1330,9 +1473,10 @@ class DISubrangeType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
+  static constexpr unsigned MY_FIRST_OPERAND = DIType::N_OPERANDS;
+
   DISubrangeType(LLVMContext &C, StorageType Storage, unsigned Line,
-                 uint64_t SizeInBits, uint32_t AlignInBits, DIFlags Flags,
-                 ArrayRef<Metadata *> Ops);
+                 uint32_t AlignInBits, DIFlags Flags, ArrayRef<Metadata *> Ops);
 
   ~DISubrangeType() = default;
 
@@ -1342,21 +1486,23 @@ class DISubrangeType : public DIType {
           DIFlags Flags, DIType *BaseType, Metadata *LowerBound,
           Metadata *UpperBound, Metadata *Stride, Metadata *Bias,
           StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
     return getImpl(Context, getCanonicalMDString(Context, Name), File, Line,
-                   Scope, SizeInBits, AlignInBits, Flags, BaseType, LowerBound,
-                   UpperBound, Stride, Bias, Storage, ShouldCreate);
+                   Scope, SizeInBitsNode, AlignInBits, Flags, BaseType,
+                   LowerBound, UpperBound, Stride, Bias, Storage, ShouldCreate);
   }
 
   LLVM_ABI static DISubrangeType *
   getImpl(LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line,
-          Metadata *Scope, uint64_t SizeInBits, uint32_t AlignInBits,
+          Metadata *Scope, Metadata *SizeInBits, uint32_t AlignInBits,
           DIFlags Flags, Metadata *BaseType, Metadata *LowerBound,
           Metadata *UpperBound, Metadata *Stride, Metadata *Bias,
           StorageType Storage, bool ShouldCreate = true);
 
   TempDISubrangeType cloneImpl() const {
-    return getTemporary(getContext(), getName(), getFile(), getLine(),
-                        getScope(), getSizeInBits(), getAlignInBits(),
+    return getTemporary(getContext(), getRawName(), getFile(), getLine(),
+                        getScope(), getRawSizeInBits(), getAlignInBits(),
                         getFlags(), getBaseType(), getRawLowerBound(),
                         getRawUpperBound(), getRawStride(), getRawBias());
   }
@@ -1366,9 +1512,10 @@ class DISubrangeType : public DIType {
 public:
   DEFINE_MDNODE_GET(DISubrangeType,
                     (MDString * Name, Metadata *File, unsigned Line,
-                     Metadata *Scope, uint64_t SizeInBits, uint32_t AlignInBits,
-                     DIFlags Flags, Metadata *BaseType, Metadata *LowerBound,
-                     Metadata *UpperBound, Metadata *Stride, Metadata *Bias),
+                     Metadata *Scope, Metadata *SizeInBits,
+                     uint32_t AlignInBits, DIFlags Flags, Metadata *BaseType,
+                     Metadata *LowerBound, Metadata *UpperBound,
+                     Metadata *Stride, Metadata *Bias),
                     (Name, File, Line, Scope, SizeInBits, AlignInBits, Flags,
                      BaseType, LowerBound, UpperBound, Stride, Bias))
   DEFINE_MDNODE_GET(DISubrangeType,
@@ -1383,15 +1530,23 @@ class DISubrangeType : public DIType {
 
   /// Get the base type this is derived from.
   DIType *getBaseType() const { return cast_or_null<DIType>(getRawBaseType()); }
-  Metadata *getRawBaseType() const { return getOperand(3); }
+  Metadata *getRawBaseType() const { return getOperand(MY_FIRST_OPERAND); }
 
-  Metadata *getRawLowerBound() const { return getOperand(4).get(); }
+  Metadata *getRawLowerBound() const {
+    return getOperand(MY_FIRST_OPERAND + 1).get();
+  }
 
-  Metadata *getRawUpperBound() const { return getOperand(5).get(); }
+  Metadata *getRawUpperBound() const {
+    return getOperand(MY_FIRST_OPERAND + 2).get();
+  }
 
-  Metadata *getRawStride() const { return getOperand(6).get(); }
+  Metadata *getRawStride() const {
+    return getOperand(MY_FIRST_OPERAND + 3).get();
+  }
 
-  Metadata *getRawBias() const { return getOperand(7).get(); }
+  Metadata *getRawBias() const {
+    return getOperand(MY_FIRST_OPERAND + 4).get();
+  }
 
   BoundType getLowerBound() const {
     return convertRawToBound(getRawLowerBound());
@@ -1418,31 +1573,30 @@ class DICompositeType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
+  static constexpr unsigned MY_FIRST_OPERAND = DIType::N_OPERANDS;
+
   unsigned RuntimeLang;
   std::optional<uint32_t> EnumKind;
 
   DICompositeType(LLVMContext &C, StorageType Storage, unsigned Tag,
-                  unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits,
-                  uint32_t AlignInBits, uint64_t OffsetInBits,
+                  unsigned Line, unsigned RuntimeLang, uint32_t AlignInBits,
                   uint32_t NumExtraInhabitants,
                   std::optional<uint32_t> EnumKind, DIFlags Flags,
                   ArrayRef<Metadata *> Ops)
-      : DIType(C, DICompositeTypeKind, Storage, Tag, Line, SizeInBits,
-               AlignInBits, OffsetInBits, NumExtraInhabitants, Flags, Ops),
+      : DIType(C, DICompositeTypeKind, Storage, Tag, Line, AlignInBits,
+               NumExtraInhabitants, Flags, Ops),
         RuntimeLang(RuntimeLang), EnumKind(EnumKind) {}
   ~DICompositeType() = default;
 
   /// Change fields in place.
   void mutate(unsigned Tag, unsigned Line, unsigned RuntimeLang,
-              uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
-              uint32_t NumExtraInhabitants, std::optional<uint32_t> EnumKind,
-              DIFlags Flags) {
+              uint32_t AlignInBits, uint32_t NumExtraInhabitants,
+              std::optional<uint32_t> EnumKind, DIFlags Flags) {
     assert(isDistinct() && "Only distinct nodes can mutate");
     assert(getRawIdentifier() && "Only ODR-uniqued nodes should mutate");
     this->RuntimeLang = RuntimeLang;
     this->EnumKind = EnumKind;
-    DIType::mutate(Tag, Line, SizeInBits, AlignInBits, OffsetInBits,
-                   NumExtraInhabitants, Flags);
+    DIType::mutate(Tag, Line, AlignInBits, NumExtraInhabitants, Flags);
   }
 
   static DICompositeType *
@@ -1456,6 +1610,52 @@ class DICompositeType : public DIType {
           Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
           Metadata *Rank, DINodeArray Annotations, Metadata *BitStride,
           StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    auto *OffsetInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), OffsetInBits));
+    return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File,
+                   Line, Scope, BaseType, SizeInBitsNode, AlignInBits,
+                   OffsetInBitsNode, Flags, Elements.get(), RuntimeLang,
+                   EnumKind, VTableHolder, TemplateParams.get(),
+                   getCanonicalMDString(Context, Identifier), Discriminator,
+                   DataLocation, Associated, Allocated, Rank, Annotations.get(),
+                   Specification, NumExtraInhabitants, BitStride, Storage,
+                   ShouldCreate);
+  }
+  static DICompositeType *
+  getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
+          unsigned Line, Metadata *Scope, Metadata *BaseType,
+          uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+          DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
+          std::optional<uint32_t> EnumKind, Metadata *VTableHolder,
+          Metadata *TemplateParams, MDString *Identifier,
+          Metadata *Discriminator, Metadata *DataLocation, Metadata *Associated,
+          Metadata *Allocated, Metadata *Rank, Metadata *Annotations,
+          Metadata *Specification, uint32_t NumExtraInhabitants,
+          Metadata *BitStride, StorageType Storage, bool ShouldCreate = true) {
+    auto *SizeInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), SizeInBits));
+    auto *OffsetInBitsNode = ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), OffsetInBits));
+    return getImpl(Context, Tag, Name, File, Line, Scope, BaseType,
+                   SizeInBitsNode, AlignInBits, OffsetInBitsNode, Flags,
+                   Elements, RuntimeLang, EnumKind, VTableHolder,
+                   TemplateParams, Identifier, Discriminator, DataLocation,
+                   Associated, Allocated, Rank, Annotations, Specification,
+                   NumExtraInhabitants, BitStride, Storage, ShouldCreate);
+  }
+  static DICompositeType *
+  getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, Metadata *File,
+          unsigned Line, DIScope *Scope, DIType *BaseType, Metadata *SizeInBits,
+          uint32_t AlignInBits, Metadata *OffsetInBits, DIType *Specification,
+          uint32_t NumExtraInhabitants, DIFlags Flags, DINodeArray Elements,
+          unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
+          DIType *VTableHolder, DITemplateParameterArray TemplateParams,
+          StringRef Identifier, DIDerivedType *Discriminator,
+          Metadata *DataLocation, Metadata *Associated, Metadata *Allocated,
+          Metadata *Rank, DINodeArray Annotations, Metadata *BitStride,
+          StorageType Storage, bool ShouldCreate = true) {
     return getImpl(
         Context, Tag, getCanonicalMDString(Context, Name), File, Line, Scope,
         BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements.get(),
@@ -1467,7 +1667,7 @@ class DICompositeType : public DIType {
   LLVM_ABI static DICompositeType *
   getImpl(LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
           unsigned Line, Metadata *Scope, Metadata *BaseType,
-          uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+          Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
           DIFlags Flags, Metadata *Elements, unsigned RuntimeLang,
           std::optional<uint32_t> EnumKind, Metadata *VTableHolder,
           Metadata *TemplateParams, MDString *Identifier,
@@ -1478,13 +1678,14 @@ class DICompositeType : public DIType {
 
   TempDICompositeType cloneImpl() const {
     return getTemporary(
-        getContext(), getTag(), getName(), getFile(), getLine(), getScope(),
-        getBaseType(), getSizeInBits(), getAlignInBits(), getOffsetInBits(),
-        getFlags(), getElements(), getRuntimeLang(), getEnumKind(),
-        getVTableHolder(), getTemplateParams(), getIdentifier(),
-        getDiscriminator(), getRawDataLocation(), getRawAssociated(),
-        getRawAllocated(), getRawRank(), getAnnotations(), getSpecification(),
-        getNumExtraInhabitants(), getRawBitStride());
+        getContext(), getTag(), getRawName(), getFile(), getLine(), getScope(),
+        getBaseType(), getRawSizeInBits(), getAlignInBits(),
+        getRawOffsetInBits(), getFlags(), getRawElements(), getRuntimeLang(),
+        getEnumKind(), getVTableHolder(), getRawTemplateParams(),
+        getRawIdentifier(), getDiscriminator(), getRawDataLocation(),
+        getRawAssociated(), getRawAllocated(), getRawRank(),
+        getRawAnnotations(), getSpecification(), getNumExtraInhabitants(),
+        getRawBitStride());
   }
 
 public:
@@ -1524,6 +1725,42 @@ class DICompositeType : public DIType {
        TemplateParams, Identifier, Discriminator, DataLocation, Associated,
        Allocated, Rank, Annotations, Specification, NumExtraInhabitants,
        BitStride))
+  DEFINE_MDNODE_GET(
+      DICompositeType,
+      (unsigned Tag, StringRef Name, DIFile *File, unsigned Line,
+       DIScope *Scope, DIType *BaseType, Metadata *SizeInBits,
+       uint32_t AlignInBits, Metadata *OffsetInBits, DIFlags Flags,
+       DINodeArray Elements, unsigned RuntimeLang,
+       std::optional<uint32_t> EnumKind, DIType *VTableHolder,
+       DITemplateParameterArray TemplateParams = nullptr,
+       StringRef Identifier = "", DIDerivedType *Discriminator = nullptr,
+       Metadata *DataLocation = nullptr, Metadata *Associated = nullptr,
+       Metadata *Allocated = nullptr, Metadata *Rank = nullptr,
+       DINodeArray Annotations = nullptr, DIType *Specification = nullptr,
+       uint32_t NumExtraInhabitants = 0, Metadata *BitStride = nullptr),
+      (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+       OffsetInBits, Specification, NumExtraInhabitants, Flags, Elements,
+       RuntimeLang, EnumKind, VTableHolder, TemplateParams, Identifier,
+       Discriminator, DataLocation, Associated, Allocated, Rank, Annotations,
+       BitStride))
+  DEFINE_MDNODE_GET(
+      DICompositeType,
+      (unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
+       Metadata *Scope, Metadata *BaseType, Metadata *SizeInBits,
+       uint32_t AlignInBits, Metadata *OffsetInBits, DIFlags Flags,
+       Metadata *Elements, unsigned RuntimeLang,
+       std::optional<uint32_t> EnumKind, Metadata *VTableHolder,
+       Metadata *TemplateParams = nullptr, MDString *Identifier = nullptr,
+       Metadata *Discriminator = nullptr, Metadata *DataLocation = nullptr,
+       Metadata *Associated = nullptr, Metadata *Allocated = nullptr,
+       Metadata *Rank = nullptr, Metadata *Annotations = nullptr,
+       Metadata *Specification = nullptr, uint32_t NumExtraInhabitants = 0,
+       Metadata *BitStride = nullptr),
+      (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits,
+       OffsetInBits, Flags, Elements, RuntimeLang, EnumKind, VTableHolder,
+       TemplateParams, Identifier, Discriminator, DataLocation, Associated,
+       Allocated, Rank, Annotations, Specification, NumExtraInhabitants,
+       BitStride))
 
   TempDICompositeType clone() const { return cloneImpl(); }
 
@@ -1537,8 +1774,8 @@ class DICompositeType : public DIType {
   LLVM_ABI static DICompositeType *
   getODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag,
              MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
-             Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
-             uint64_t OffsetInBits, Metadata *Specification,
+             Metadata *BaseType, Metadata *SizeInBits, uint32_t AlignInBits,
+             Metadata *OffsetInBits, Metadata *Specification,
              uint32_t NumExtraInhabitants, DIFlags Flags, Metadata *Elements,
              unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
              Metadata *VTableHolder, Metadata *TemplateParams,
@@ -1560,8 +1797,8 @@ class DICompositeType : public DIType {
   LLVM_ABI static DICompositeType *
   buildODRType(LLVMContext &Context, MDString &Identifier, unsigned Tag,
                MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
-               Metadata *BaseType, uint64_t SizeInBits, uint32_t AlignInBits,
-               uint64_t OffsetInBits, Metadata *Specification,
+               Metadata *BaseType, Metadata *SizeInBits, uint32_t AlignInBits,
+               Metadata *OffsetInBits, Metadata *Specification,
                uint32_t NumExtraInhabitants, DIFlags Flags, Metadata *Elements,
                unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
                Metadata *VTableHolder, Metadata *TemplateParams,
@@ -1579,41 +1816,55 @@ class DICompositeType : public DIType {
   DITemplateParameterArray getTemplateParams() const {
     return cast_or_null<MDTuple>(getRawTemplateParams());
   }
-  StringRef getIdentifier() const { return getStringOperand(7); }
+  StringRef getIdentifier() const {
+    return getStringOperand(MY_FIRST_OPERAND + 4);
+  }
   unsigned getRuntimeLang() const { return RuntimeLang; }
   std::optional<uint32_t> getEnumKind() const { return EnumKind; }
 
-  Metadata *getRawBaseType() const { return getOperand(3); }
-  Metadata *getRawElements() const { return getOperand(4); }
-  Metadata *getRawVTableHolder() const { return getOperand(5); }
-  Metadata *getRawTemplateParams() const { return getOperand(6); }
-  MDString *getRawIdentifier() const { return getOperandAs<MDString>(7); }
-  Metadata *getRawDiscriminator() const { return getOperand(8); }
+  Metadata *getRawBaseType() const { return getOperand(MY_FIRST_OPERAND); }
+  Metadata *getRawElements() const { return getOperand(MY_FIRST_OPERAND + 1); }
+  Metadata *getRawVTableHolder() const {
+    return getOperand(MY_FIRST_OPERAND + 2);
+  }
+  Metadata *getRawTemplateParams() const {
+    return getOperand(MY_FIRST_OPERAND + 3);
+  }
+  MDString *getRawIdentifier() const {
+    return getOperandAs<MDString>(MY_FIRST_OPERAND + 4);
+  }
+  Metadata *getRawDiscriminator() const {
+    return getOperand(MY_FIRST_OPERAND + 5);
+  }
   DIDerivedType *getDiscriminator() const {
-    return getOperandAs<DIDerivedType>(8);
+    return getOperandAs<DIDerivedType>(MY_FIRST_OPERAND + 5);
+  }
+  Metadata *getRawDataLocation() const {
+    return getOperand(MY_FIRST_OPERAND + 6);
   }
-  Metadata *getRawDataLocation() const { return getOperand(9); }
   DIVariable *getDataLocation() const {
     return dyn_cast_or_null<DIVariable>(getRawDataLocation());
   }
   DIExpression *getDataLocationExp() const {
     return dyn_cast_or_null<DIExpression>(getRawDataLocation());
   }
-  Metadata *getRawAssociated() const { return getOperand(10); }
+  Metadata *getRawAssociated() const {
+    return getOperand(MY_FIRST_OPERAND + 7);
+  }
   DIVariable *getAssociated() const {
     return dyn_cast_or_null<DIVariable>(getRawAssociated());
   }
   DIExpression *getAssociatedExp() const {
     return dyn_cast_or_null<DIExpression>(getRawAssociated());
   }
-  Metadata *getRawAllocated() const { return getOperand(11); }
+  Metadata *getRawAllocated() const { return getOperand(MY_FIRST_OPERAND + 8); }
   DIVariable *getAllocated() const {
     return dyn_cast_or_null<DIVariable>(getRawAllocated());
   }
   DIExpression *getAllocatedExp() const {
     return dyn_cast_or_null<DIExpression>(getRawAllocated());
   }
-  Metadata *getRawRank() const { return getOperand(12); }
+  Metadata *getRawRank() const { return getOperand(MY_FIRST_OPERAND + 9); }
   ConstantInt *getRankConst() const {
     if (auto *MD = dyn_cast_or_null<ConstantAsMetadata>(getRawRank()))
       return dyn_cast_or_null<ConstantInt>(MD->getValue());
@@ -1623,17 +1874,23 @@ class DICompositeType : public DIType {
     return dyn_cast_or_null<DIExpression>(getRawRank());
   }
 
-  Metadata *getRawAnnotations() const { return getOperand(13); }
+  Metadata *getRawAnnotations() const {
+    return getOperand(MY_FIRST_OPERAND + 10);
+  }
   DINodeArray getAnnotations() const {
     return cast_or_null<MDTuple>(getRawAnnotations());
   }
 
-  Metadata *getRawSpecification() const { return getOperand(14); }
+  Metadata *getRawSpecification() const {
+    return getOperand(MY_FIRST_OPERAND + 11);
+  }
   DIType *getSpecification() const {
     return cast_or_null<DIType>(getRawSpecification());
   }
 
-  Metadata *getRawBitStride() const { return getOperand(15); }
+  Metadata *getRawBitStride() const {
+    return getOperand(MY_FIRST_OPERAND + 12);
+  }
   ConstantInt *getBitStrideConst() const {
     if (auto *MD = dyn_cast_or_null<ConstantAsMetadata>(getRawBitStride()))
       return dyn_cast_or_null<ConstantInt>(MD->getValue());
@@ -1652,15 +1909,15 @@ class DICompositeType : public DIType {
       assert(is_contained(Elements->operands(), Op) &&
              "Lost a member during member list replacement");
 #endif
-    replaceOperandWith(4, Elements.get());
+    replaceOperandWith(MY_FIRST_OPERAND + 1, Elements.get());
   }
 
   void replaceVTableHolder(DIType *VTableHolder) {
-    replaceOperandWith(5, VTableHolder);
+    replaceOperandWith(MY_FIRST_OPERAND + 2, VTableHolder);
   }
 
   void replaceTemplateParams(DITemplateParameterArray TemplateParams) {
-    replaceOperandWith(6, TemplateParams.get());
+    replaceOperandWith(MY_FIRST_OPERAND + 3, TemplateParams.get());
   }
   /// @}
 
@@ -1676,6 +1933,8 @@ class DISubroutineType : public DIType {
   friend class LLVMContextImpl;
   friend class MDNode;
 
+  static constexpr unsigned MY_FIRST_OPERAND = DIType::N_OPERANDS;
+
   /// The calling convention used with DW_AT_calling_convention. Actually of
   /// type dwarf::CallingConvention.
   uint8_t CC;
@@ -1721,7 +1980,7 @@ class DISubroutineType : public DIType {
     return cast_or_null<MDTuple>(getRawTypeArray());
   }
 
-  Metadata *getRawTypeArray() const { return getOperand(3); }
+  Metadata *getRawTypeArray() const { return getOperand(MY_FIRST_OPERAND); }
 
   static bool classof(const Metadata *MD) {
     return MD->getMetadataID() == DISubroutineTypeKind;
@@ -1971,7 +2230,7 @@ class DILocalScope : public DIScope {
   }
 };
 
-/// Subprogram description.
+/// Subprogram description. Uses SubclassData1.
 class DISubprogram : public DILocalScope {
   friend class LLVMContextImpl;
   friend class MDNode;
@@ -2018,7 +2277,8 @@ class DISubprogram : public DILocalScope {
 
   DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
                unsigned ScopeLine, unsigned VirtualIndex, int ThisAdjustment,
-               DIFlags Flags, DISPFlags SPFlags, ArrayRef<Metadata *> Ops);
+               DIFlags Flags, DISPFlags SPFlags, bool UsesKeyInstructions,
+               ArrayRef<Metadata *> Ops);
   ~DISubprogram() = default;
 
   static DISubprogram *
@@ -2030,15 +2290,17 @@ class DISubprogram : public DILocalScope {
           DITemplateParameterArray TemplateParams, DISubprogram *Declaration,
           DINodeArray RetainedNodes, DITypeArray ThrownTypes,
           DINodeArray Annotations, StringRef TargetFuncName,
-          StorageType Storage, bool ShouldCreate = true) {
+          bool UsesKeyInstructions, StorageType Storage,
+          bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name),
                    getCanonicalMDString(Context, LinkageName), File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags, Unit, TemplateParams.get(), Declaration,
                    RetainedNodes.get(), ThrownTypes.get(), Annotations.get(),
                    getCanonicalMDString(Context, TargetFuncName),
-                   Storage, ShouldCreate);
+                   UsesKeyInstructions, Storage, ShouldCreate);
   }
+
   LLVM_ABI static DISubprogram *
   getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
           MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type,
@@ -2046,8 +2308,8 @@ class DISubprogram : public DILocalScope {
           int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
           Metadata *TemplateParams, Metadata *Declaration,
           Metadata *RetainedNodes, Metadata *ThrownTypes, Metadata *Annotations,
-          MDString *TargetFuncName, StorageType Storage,
-          bool ShouldCreate = true);
+          MDString *TargetFuncName, bool UsesKeyInstructions,
+          StorageType Storage, bool ShouldCreate = true);
 
   TempDISubprogram cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getLinkageName(),
@@ -2056,7 +2318,7 @@ class DISubprogram : public DILocalScope {
                         getThisAdjustment(), getFlags(), getSPFlags(),
                         getUnit(), getTemplateParams(), getDeclaration(),
                         getRetainedNodes(), getThrownTypes(), getAnnotations(),
-                        getTargetFuncName());
+                        getTargetFuncName(), getKeyInstructionsEnabled());
   }
 
 public:
@@ -2069,10 +2331,11 @@ class DISubprogram : public DILocalScope {
        DITemplateParameterArray TemplateParams = nullptr,
        DISubprogram *Declaration = nullptr, DINodeArray RetainedNodes = nullptr,
        DITypeArray ThrownTypes = nullptr, DINodeArray Annotations = nullptr,
-       StringRef TargetFuncName = ""),
+       StringRef TargetFuncName = "", bool UsesKeyInstructions = false),
       (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType,
        VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams,
-       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName))
+       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName,
+       UsesKeyInstructions))
 
   DEFINE_MDNODE_GET(
       DISubprogram,
@@ -2082,10 +2345,12 @@ class DISubprogram : public DILocalScope {
        DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
        Metadata *TemplateParams = nullptr, Metadata *Declaration = nullptr,
        Metadata *RetainedNodes = nullptr, Metadata *ThrownTypes = nullptr,
-       Metadata *Annotations = nullptr, MDString *TargetFuncName = nullptr),
+       Metadata *Annotations = nullptr, MDString *TargetFuncName = nullptr,
+       bool UsesKeyInstructions = false),
       (Scope, Name, LinkageName, File, Line, Type, ScopeLine, ContainingType,
        VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit, TemplateParams,
-       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName))
+       Declaration, RetainedNodes, ThrownTypes, Annotations, TargetFuncName,
+       UsesKeyInstructions))
 
   TempDISubprogram clone() const { return cloneImpl(); }
 
@@ -2096,6 +2361,8 @@ class DISubprogram : public DILocalScope {
     return NewSP;
   }
 
+  bool getKeyInstructionsEnabled() const { return SubclassData1; }
+
 public:
   unsigned getLine() const { return Line; }
   unsigned getVirtuality() const { return getSPFlags() & SPFlagVirtuality; }
@@ -4555,35 +4822,49 @@ class DILabel : public DINode {
   friend class LLVMContextImpl;
   friend class MDNode;
 
-  DILabel(LLVMContext &C, StorageType Storage, unsigned Line,
+  unsigned Column;
+  std::optional<unsigned> CoroSuspendIdx;
+  bool IsArtificial;
+
+  DILabel(LLVMContext &C, StorageType Storage, unsigned Line, unsigned Column,
+          bool IsArtificial, std::optional<unsigned> CoroSuspendIdx,
           ArrayRef<Metadata *> Ops);
   ~DILabel() = default;
 
   static DILabel *getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name,
-                          DIFile *File, unsigned Line, StorageType Storage,
-                          bool ShouldCreate = true) {
+                          DIFile *File, unsigned Line, unsigned Column,
+                          bool IsArtificial,
+                          std::optional<unsigned> CoroSuspendIdx,
+                          StorageType Storage, bool ShouldCreate = true) {
     return getImpl(Context, Scope, getCanonicalMDString(Context, Name), File,
-                   Line, Storage, ShouldCreate);
+                   Line, Column, IsArtificial, CoroSuspendIdx, Storage,
+                   ShouldCreate);
   }
-  LLVM_ABI static DILabel *getImpl(LLVMContext &Context, Metadata *Scope,
-                                   MDString *Name, Metadata *File,
-                                   unsigned Line, StorageType Storage,
-                                   bool ShouldCreate = true);
+  LLVM_ABI static DILabel *
+  getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, Metadata *File,
+          unsigned Line, unsigned Column, bool IsArtificial,
+          std::optional<unsigned> CoroSuspendIdx, StorageType Storage,
+          bool ShouldCreate = true);
 
   TempDILabel cloneImpl() const {
     return getTemporary(getContext(), getScope(), getName(), getFile(),
-                        getLine());
+                        getLine(), getColumn(), isArtificial(),
+                        getCoroSuspendIdx());
   }
 
 public:
   DEFINE_MDNODE_GET(DILabel,
                     (DILocalScope * Scope, StringRef Name, DIFile *File,
-                     unsigned Line),
-                    (Scope, Name, File, Line))
+                     unsigned Line, unsigned Column, bool IsArtificial,
+                     std::optional<unsigned> CoroSuspendIdx),
+                    (Scope, Name, File, Line, Column, IsArtificial,
+                     CoroSuspendIdx))
   DEFINE_MDNODE_GET(DILabel,
                     (Metadata * Scope, MDString *Name, Metadata *File,
-                     unsigned Line),
-                    (Scope, Name, File, Line))
+                     unsigned Line, unsigned Column, bool IsArtificial,
+                     std::optional<unsigned> CoroSuspendIdx),
+                    (Scope, Name, File, Line, Column, IsArtificial,
+                     CoroSuspendIdx))
 
   TempDILabel clone() const { return cloneImpl(); }
 
@@ -4594,8 +4875,11 @@ class DILabel : public DINode {
     return cast_or_null<DILocalScope>(getRawScope());
   }
   unsigned getLine() const { return SubclassData32; }
+  unsigned getColumn() const { return Column; }
   StringRef getName() const { return getStringOperand(1); }
   DIFile *getFile() const { return cast_or_null<DIFile>(getRawFile()); }
+  bool isArtificial() const { return IsArtificial; }
+  std::optional<unsigned> getCoroSuspendIdx() const { return CoroSuspendIdx; }
 
   Metadata *getRawScope() const { return getOperand(0); }
   MDString *getRawName() const { return getOperandAs<MDString>(1); }
diff --git a/llvm/include/llvm/IR/DebugLoc.h b/llvm/include/llvm/IR/DebugLoc.h
index 999e03b6374a..6d79aa6b2aa0 100644
--- a/llvm/include/llvm/IR/DebugLoc.h
+++ b/llvm/include/llvm/IR/DebugLoc.h
@@ -27,6 +27,21 @@ namespace llvm {
   class Function;
 
 #if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+  struct DbgLocOrigin {
+    static constexpr unsigned long MaxDepth = 16;
+    using StackTracesTy =
+        SmallVector<std::pair<int, std::array<void *, MaxDepth>>, 0>;
+    StackTracesTy StackTraces;
+    DbgLocOrigin(bool ShouldCollectTrace);
+    void addTrace();
+    const StackTracesTy &getOriginStackTraces() const { return StackTraces; };
+  };
+#else
+  struct DbgLocOrigin {
+    DbgLocOrigin(bool) {}
+  };
+#endif
   // Used to represent different "kinds" of DebugLoc, expressing that the
   // instruction it is part of is either normal and should contain a valid
   // DILocation, or otherwise describing the reason why the instruction does
@@ -55,22 +70,29 @@ namespace llvm {
     Temporary
   };
 
-  // Extends TrackingMDNodeRef to also store a DebugLocKind, allowing Debugify
-  // to ignore intentionally-empty DebugLocs.
-  class DILocAndCoverageTracking : public TrackingMDNodeRef {
+  // Extends TrackingMDNodeRef to also store a DebugLocKind and Origin,
+  // allowing Debugify to ignore intentionally-empty DebugLocs and display the
+  // code responsible for generating unintentionally-empty DebugLocs.
+  // Currently we only need to track the Origin of this DILoc when using a
+  // DebugLoc that is not annotated (i.e. has DebugLocKind::Normal) and has a
+  // null DILocation, so only collect the origin stacktrace in those cases.
+  class DILocAndCoverageTracking : public TrackingMDNodeRef,
+                                   public DbgLocOrigin {
   public:
     DebugLocKind Kind;
     // Default constructor for empty DebugLocs.
     DILocAndCoverageTracking()
-        : TrackingMDNodeRef(nullptr), Kind(DebugLocKind::Normal) {}
-    // Valid or nullptr MDNode*, normal DebugLocKind.
+        : TrackingMDNodeRef(nullptr), DbgLocOrigin(true),
+          Kind(DebugLocKind::Normal) {}
+    // Valid or nullptr MDNode*, no annotative DebugLocKind.
     DILocAndCoverageTracking(const MDNode *Loc)
-        : TrackingMDNodeRef(const_cast<MDNode *>(Loc)),
+        : TrackingMDNodeRef(const_cast<MDNode *>(Loc)), DbgLocOrigin(!Loc),
           Kind(DebugLocKind::Normal) {}
     LLVM_ABI DILocAndCoverageTracking(const DILocation *Loc);
     // Explicit DebugLocKind, which always means a nullptr MDNode*.
     DILocAndCoverageTracking(DebugLocKind Kind)
-        : TrackingMDNodeRef(nullptr), Kind(Kind) {}
+        : TrackingMDNodeRef(nullptr),
+          DbgLocOrigin(Kind == DebugLocKind::Normal), Kind(Kind) {}
   };
   template <> struct simplify_type<DILocAndCoverageTracking> {
     using SimpleType = MDNode *;
@@ -187,6 +209,19 @@ namespace llvm {
 #endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
     }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+    const DbgLocOrigin::StackTracesTy &getOriginStackTraces() const {
+      return Loc.getOriginStackTraces();
+    }
+    DebugLoc getCopied() const {
+      DebugLoc NewDL = *this;
+      NewDL.Loc.addTrace();
+      return NewDL;
+    }
+#else
+    DebugLoc getCopied() const { return *this; }
+#endif
+
     /// Get the underlying \a DILocation.
     ///
     /// \pre !*this or \c isa<DILocation>(getAsMDNode()).
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index c361be3e752a..d3497716ca84 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -783,9 +783,8 @@ class LLVM_ABI Function : public GlobalObject, public ilist_node<Function> {
 
 private:
   // These need access to the underlying BB list.
-  LLVM_ABI_FRIEND friend void BasicBlock::removeFromParent();
-  LLVM_ABI_FRIEND friend iplist<BasicBlock>::iterator
-  BasicBlock::eraseFromParent();
+  LLVM_ABI friend void BasicBlock::removeFromParent();
+  LLVM_ABI friend iplist<BasicBlock>::iterator BasicBlock::eraseFromParent();
   template <class BB_t, class BB_i_t, class BI_t, class II_t>
   friend class InstIterator;
   friend class llvm::SymbolTableListTraits<llvm::BasicBlock>;
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 8fe610835dca..66ab2fa5610f 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1575,10 +1575,14 @@ class IRBuilderBase {
     return Accum;
   }
 
-  Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") {
+  Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "",
+                  bool IsDisjoint = false) {
     if (auto *V = Folder.FoldBinOp(Instruction::Or, LHS, RHS))
       return V;
-    return Insert(BinaryOperator::CreateOr(LHS, RHS), Name);
+    return Insert(
+        IsDisjoint ? BinaryOperator::CreateDisjoint(Instruction::Or, LHS, RHS)
+                   : BinaryOperator::CreateOr(LHS, RHS),
+        Name);
   }
 
   Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") {
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 8e1ef2422678..ef382a9168f2 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -507,7 +507,7 @@ class Instruction : public User,
   LLVM_ABI bool extractProfTotalWeight(uint64_t &TotalVal) const;
 
   /// Set the debug location information for this instruction.
-  void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); }
+  void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc).getCopied(); }
 
   /// Return the debug location for this node as a DebugLoc.
   const DebugLoc &getDebugLoc() const { return DbgLoc; }
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index d00c173e21d6..1a1bd2233acb 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1848,11 +1848,11 @@ def int_ubsantrap : Intrinsic<[], [llvm_i8_ty],
 
 // Return true if ubsan check is allowed.
 def int_allow_ubsan_check : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i8_ty],
-    [IntrInaccessibleMemOnly, IntrWriteMem, ImmArg<ArgIndex<0>>, NoUndef<RetIndex>]>;
+    [IntrInaccessibleMemOnly, ImmArg<ArgIndex<0>>, NoUndef<RetIndex>]>;
 
 // Return true if runtime check is allowed.
 def int_allow_runtime_check : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_metadata_ty],
-    [IntrInaccessibleMemOnly, IntrWriteMem, NoUndef<RetIndex>]>,
+    [IntrInaccessibleMemOnly, NoUndef<RetIndex>]>,
     ClangBuiltin<"__builtin_allow_runtime_check">;
 
 // Support for dynamic deoptimization (or de-specialization)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 0ec5f5163118..ca6e2128812f 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -612,7 +612,7 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                   [IntrReadMem, IntrArgMemOnly]>;
   class AdvSIMD_1Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
 
   class AdvSIMD_2Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
@@ -626,11 +626,11 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
   class AdvSIMD_2Vec_Store_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                      llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
   class AdvSIMD_2Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
 
   class AdvSIMD_3Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
@@ -644,12 +644,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
   class AdvSIMD_3Vec_Store_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                      LLVMMatchType<0>, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
   class AdvSIMD_3Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
 
   class AdvSIMD_4Vec_Load_Intrinsic
     : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
@@ -667,12 +667,12 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
   class AdvSIMD_4Vec_Store_Lane_Intrinsic
     : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                  LLVMMatchType<0>, LLVMMatchType<0>,
                  llvm_i64_ty, llvm_anyptr_ty],
-                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
+                [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
 }
 
 // Memory ops
@@ -761,7 +761,7 @@ let TargetPrefix = "aarch64" in {
   class RNDR_Intrinsic
     : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>;
   class FPMR_Set_Intrinsic
-    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrInaccessibleMemOnly]>; 
+    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;
 }
 
 // FP environment registers.
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index e6f0bf627608..f7a9b6585469 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -588,6 +588,14 @@ def int_amdgcn_ds_ordered_swap : AMDGPUDSOrderedIntrinsic;
 def int_amdgcn_ds_append : AMDGPUDSAppendConsumedIntrinsic;
 def int_amdgcn_ds_consume : AMDGPUDSAppendConsumedIntrinsic;
 
+def int_amdgcn_cvt_pk_f16_fp8 : DefaultAttrsIntrinsic<
+  [llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrSpeculatable]
+>, ClangBuiltin<"__builtin_amdgcn_cvt_pk_f16_fp8">;
+
+def int_amdgcn_cvt_pk_f16_bf8 : DefaultAttrsIntrinsic<
+  [llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrSpeculatable]
+>, ClangBuiltin<"__builtin_amdgcn_cvt_pk_f16_bf8">;
+
 class AMDGPUCvtScaleF32Intrinsic<LLVMType DstTy, LLVMType Src0Ty, string name> : DefaultAttrsIntrinsic<
   [DstTy], [Src0Ty, llvm_float_ty], [IntrNoMem, IntrSpeculatable]
 >, ClangBuiltin<"__builtin_amdgcn_"#name>;
@@ -2988,6 +2996,12 @@ class AMDGPULoadIntrinsic<LLVMType ptr_ty>:
 
 def int_amdgcn_global_load_tr_b64  : AMDGPULoadIntrinsic<global_ptr_ty>;
 def int_amdgcn_global_load_tr_b128 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr4_b64 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_global_load_tr6_b96 : AMDGPULoadIntrinsic<global_ptr_ty>;
+def int_amdgcn_ds_load_tr8_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr16_b128   : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr4_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
+def int_amdgcn_ds_load_tr6_b96     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr4_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr6_b96     : AMDGPULoadIntrinsic<local_ptr_ty>;
 def int_amdgcn_ds_read_tr8_b64     : AMDGPULoadIntrinsic<local_ptr_ty>;
@@ -3486,6 +3500,40 @@ def int_amdgcn_ashr_pk_u8_i32 : ClangBuiltin<"__builtin_amdgcn_ashr_pk_u8_i32">,
   DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
+//===----------------------------------------------------------------------===//
+// gfx1250 intrinsics
+// ===----------------------------------------------------------------------===//
+
+def int_amdgcn_ds_atomic_async_barrier_arrive_b64 :
+  ClangBuiltin<"__builtin_amdgcn_ds_atomic_async_barrier_arrive_b64">,
+  Intrinsic<[], [local_ptr_ty],
+            // Atomically updates LDS and also ASYNC_CNT which is modeled as InaccessibleMem.
+            [IntrConvergent, IntrWillReturn, IntrInaccessibleMemOrArgMemOnly],
+            "", [SDNPMemOperand]>;
+
+def int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 :
+  ClangBuiltin<"__builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64">,
+  Intrinsic<[llvm_i64_ty], [local_ptr_ty, llvm_i64_ty],
+            [IntrConvergent, IntrWillReturn, IntrArgMemOnly, NoCapture<ArgIndex<0>>],
+            "", [SDNPMemOperand]>;
+
+def int_amdgcn_s_monitor_sleep :
+  ClangBuiltin<"__builtin_amdgcn_s_monitor_sleep">,
+  DefaultAttrsIntrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem,
+                                            IntrHasSideEffects]>;
+
+// llvm.amdgcn.cvt.f16.fp8 half vdst, int srcA, imm byte_sel [0..3]
+def int_amdgcn_cvt_f16_fp8 : ClangBuiltin<"__builtin_amdgcn_cvt_f16_fp8">,
+  DefaultAttrsIntrinsic<[llvm_half_ty],
+            [llvm_i32_ty, llvm_i32_ty],
+            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
+// llvm.amdgcn.cvt.f16.bf8 half vdst, int srcA, imm byte_sel [0..3]
+def int_amdgcn_cvt_f16_bf8 : ClangBuiltin<"__builtin_amdgcn_cvt_f16_bf8">,
+  DefaultAttrsIntrinsic<[llvm_half_ty],
+            [llvm_i32_ty, llvm_i32_ty],
+            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
 //===----------------------------------------------------------------------===//
 // Special Intrinsics for backend internal use only. No frontend
 // should emit calls to these.
@@ -3532,6 +3580,41 @@ def int_amdgcn_fdiv_fast : DefaultAttrsIntrinsic<
   [IntrNoMem, IntrSpeculatable]
 >;
 
+class AMDGPUTensorLoadStore:
+  Intrinsic<
+    [],
+    [llvm_v4i32_ty, // D# group 0
+     llvm_v8i32_ty, // D# group 1
+     llvm_v4i32_ty, // D# group 2
+     llvm_v4i32_ty, // D# group 3
+     llvm_i32_ty],  // cachepolicy:
+                    //   bits [0-2] = th
+                    //   bits [3-4] = scope
+    [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<4>>, IntrWillReturn, IntrConvergent, IntrNoCallback, IntrNoFree],
+    "", [SDNPMemOperand]
+  >;
+
+class AMDGPUTensorLoadStoreD2:
+  Intrinsic<
+    [],
+    [llvm_v4i32_ty,  // D# group 0
+     llvm_v8i32_ty,  // D# group 1
+     llvm_i32_ty],   // cachepolicy:
+                     //   bits [0-2] = th
+                     //   bits [3-4] = scope
+    [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<2>>, IntrWillReturn, IntrConvergent, IntrNoCallback, IntrNoFree],
+    "", [SDNPMemOperand]
+  >;
+
+def int_amdgcn_tensor_load_to_lds :
+  ClangBuiltin<"__builtin_amdgcn_tensor_load_to_lds">, AMDGPUTensorLoadStore;
+def int_amdgcn_tensor_store_from_lds :
+  ClangBuiltin<"__builtin_amdgcn_tensor_store_from_lds">, AMDGPUTensorLoadStore;
+def int_amdgcn_tensor_load_to_lds_d2 :
+  ClangBuiltin<"__builtin_amdgcn_tensor_load_to_lds_d2">, AMDGPUTensorLoadStoreD2;
+def int_amdgcn_tensor_store_from_lds_d2 :
+  ClangBuiltin<"__builtin_amdgcn_tensor_store_from_lds_d2">, AMDGPUTensorLoadStoreD2;
+
 /// Emit an addrspacecast without null pointer checking.
 /// Should only be inserted by a pass based on analysis of an addrspacecast's src.
 def int_amdgcn_addrspacecast_nonnull : DefaultAttrsIntrinsic<
diff --git a/llvm/include/llvm/IR/IntrinsicsARM.td b/llvm/include/llvm/IR/IntrinsicsARM.td
index 3ee69b72cc5c..1219ce2f86da 100644
--- a/llvm/include/llvm/IR/IntrinsicsARM.td
+++ b/llvm/include/llvm/IR/IntrinsicsARM.td
@@ -451,9 +451,6 @@ class Neon_3Arg_Long_Intrinsic
                            LLVMTruncatedType<0>],
                           [IntrNoMem]>;
 
-class Neon_1FloatArg_Intrinsic
-  : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
-
 class Neon_CvtFxToFP_Intrinsic
   : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
                           [IntrNoMem]>;
@@ -677,14 +674,6 @@ def int_arm_neon_vtbx2 : Neon_Tbl4Arg_Intrinsic;
 def int_arm_neon_vtbx3 : Neon_Tbl5Arg_Intrinsic;
 def int_arm_neon_vtbx4 : Neon_Tbl6Arg_Intrinsic;
 
-// Vector and Scalar Rounding.
-def int_arm_neon_vrintn : Neon_1FloatArg_Intrinsic;
-def int_arm_neon_vrintx : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrinta : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintz : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintm : Neon_1Arg_Intrinsic;
-def int_arm_neon_vrintp : Neon_1Arg_Intrinsic;
-
 // De-interleaving vector loads from N-element structures.
 // Source operands are the address and alignment.
 def int_arm_neon_vld1 : DefaultAttrsIntrinsic<
@@ -744,32 +733,33 @@ def int_arm_neon_vld4dup : DefaultAttrsIntrinsic<
 // Interleaving vector stores from N-element structures.
 // Source operands are: the address, the N vectors, and the alignment.
 def int_arm_neon_vst1 : DefaultAttrsIntrinsic<
-    [], [llvm_anyptr_ty, llvm_anyvector_ty, llvm_i32_ty], [IntrArgMemOnly]>;
+    [], [llvm_anyptr_ty, llvm_anyvector_ty, llvm_i32_ty],
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst2 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst3 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst4 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 
 def int_arm_neon_vst1x2 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 def int_arm_neon_vst1x3 : DefaultAttrsIntrinsic<
     [], [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 def int_arm_neon_vst1x4 : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>],
-    [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 
 // Vector store N-element structure from one lane.
 // Source operands are: the address, the N vectors, the lane number, and
@@ -778,17 +768,17 @@ def int_arm_neon_vst2lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, llvm_i32_ty,
      llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst3lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      llvm_i32_ty, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 def int_arm_neon_vst4lane : DefaultAttrsIntrinsic<
     [],
     [llvm_anyptr_ty, llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
      LLVMMatchType<1>, llvm_i32_ty, llvm_i32_ty],
-    [IntrArgMemOnly]>;
+    [IntrWriteMem, IntrArgMemOnly]>;
 
 // Vector bitwise select.
 def int_arm_neon_vbsl : DefaultAttrsIntrinsic<
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 84c26599b5b7..7dd9ff7f08b8 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -655,6 +655,14 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
       DefaultAttrsIntrinsic<[llvm_v1i128_ty],[llvm_v1i128_ty],[IntrNoMem]>;
 
   // BCD intrinsics.
+  def int_ppc_national2packed: ClangBuiltin<"__builtin_ppc_national2packed">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty],[llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+  def int_ppc_packed2national: ClangBuiltin<"__builtin_ppc_packed2national">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty],[llvm_v16i8_ty], [IntrNoMem]>;
+  def int_ppc_packed2zoned: ClangBuiltin<"__builtin_ppc_packed2zoned">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty],[llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+  def int_ppc_zoned2packed: ClangBuiltin<"__builtin_ppc_zoned2packed">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty],[llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<ArgIndex<1>>]>;
   def int_ppc_cdtbcdd : ClangBuiltin<"__builtin_ppc_cdtbcd">,
     DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem]>;
   def int_ppc_cbcdtdd: ClangBuiltin<"__builtin_ppc_cbcdtd">,
diff --git a/llvm/include/llvm/IR/LegacyPassManagers.h b/llvm/include/llvm/IR/LegacyPassManagers.h
index 1e5b92636d0a..d25bcafc9cc0 100644
--- a/llvm/include/llvm/IR/LegacyPassManagers.h
+++ b/llvm/include/llvm/IR/LegacyPassManagers.h
@@ -333,8 +333,7 @@ class LLVM_ABI PMDataManager {
   /// Initialize available analysis information.
   void initializeAnalysisInfo() {
     AvailableAnalysis.clear();
-    for (auto &IA : InheritedAnalysis)
-      IA = nullptr;
+    llvm::fill(InheritedAnalysis, nullptr);
   }
 
   // Return true if P preserves high level analysis used by other
diff --git a/llvm/include/llvm/IR/OptBisect.h b/llvm/include/llvm/IR/OptBisect.h
index be6aef3298b2..ea3c1defeb10 100644
--- a/llvm/include/llvm/IR/OptBisect.h
+++ b/llvm/include/llvm/IR/OptBisect.h
@@ -28,8 +28,8 @@ class OptPassGate {
 
   /// IRDescription is a textual description of the IR unit the pass is running
   /// over.
-  virtual bool shouldRunPass(const StringRef PassName,
-                             StringRef IRDescription) {
+  virtual bool shouldRunPass(StringRef PassName,
+                             StringRef IRDescription) const {
     return true;
   }
 
@@ -62,8 +62,8 @@ class LLVM_ABI OptBisect : public OptPassGate {
   /// Most passes should not call this routine directly. Instead, it is called
   /// through helper routines provided by the base classes of the pass. For
   /// instance, function passes should call FunctionPass::skipFunction().
-  bool shouldRunPass(const StringRef PassName,
-                     StringRef IRDescription) override;
+  bool shouldRunPass(StringRef PassName,
+                     StringRef IRDescription) const override;
 
   /// isEnabled() should return true before calling shouldRunPass().
   bool isEnabled() const override { return BisectLimit != Disabled; }
@@ -75,11 +75,11 @@ class LLVM_ABI OptBisect : public OptPassGate {
     LastBisectNum = 0;
   }
 
-  static const int Disabled = std::numeric_limits<int>::max();
+  static constexpr int Disabled = std::numeric_limits<int>::max();
 
 private:
   int BisectLimit = Disabled;
-  int LastBisectNum = 0;
+  mutable int LastBisectNum = 0;
 };
 
 /// Singleton instance of the OptBisect class, so multiple pass managers don't
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 1f86cdfd94e1..ed9b83d5d436 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -2119,6 +2119,13 @@ m_IntToPtr(const OpTy &Op) {
   return CastOperator_match<OpTy, Instruction::IntToPtr>(Op);
 }
 
+/// Matches any cast or self. Used to ignore casts.
+template <typename OpTy>
+inline match_combine_or<CastInst_match<OpTy, CastInst>, OpTy>
+m_CastOrSelf(const OpTy &Op) {
+  return m_CombineOr(CastInst_match<OpTy, CastInst>(Op), Op);
+}
+
 /// Matches Trunc.
 template <typename OpTy>
 inline CastInst_match<OpTy, TruncInst> m_Trunc(const OpTy &Op) {
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index 8e8d069b836f..bc3b046dadb4 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -21,6 +21,14 @@
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
+struct MDProfLabels {
+  static const char *BranchWeights;
+  static const char *ValueProfile;
+  static const char *FunctionEntryCount;
+  static const char *SyntheticFunctionEntryCount;
+  static const char *ExpectedBranchWeights;
+  static const char *UnknownBranchWeightsMarker;
+};
 
 /// Checks if an Instruction has MD_prof Metadata
 LLVM_ABI bool hasProfMD(const Instruction &I);
@@ -28,6 +36,9 @@ LLVM_ABI bool hasProfMD(const Instruction &I);
 /// Checks if an MDNode contains Branch Weight Metadata
 LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData);
 
+/// Checks if an MDNode contains value profiling Metadata
+LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData);
+
 /// Checks if an instructions has Branch Weight Metadata
 ///
 /// \param I The instruction to check
@@ -133,6 +144,18 @@ LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
 LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
                                bool IsExpected);
 
+/// Specify that the branch weights for this terminator cannot be known at
+/// compile time. This should only be called by passes, and never as a default
+/// behavior in e.g. MDBuilder. The goal is to use this info to validate passes
+/// do not accidentally drop profile info, and this API is called in cases where
+/// the pass explicitly cannot provide that info. Defaulting it in would hide
+/// bugs where the pass forgets to transfer over or otherwise specify profile
+/// info.
+LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I);
+
+LLVM_ABI bool isExplicitlyUnknownBranchWeightsMetadata(const MDNode &MD);
+LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I);
+
 /// Scaling the profile data attached to 'I' using the ratio of S/T.
 LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T);
 
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
deleted file mode 100644
index 247643525ff4..000000000000
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ /dev/null
@@ -1,701 +0,0 @@
-//===-- llvm/RuntimeLibcalls.def - File that describes libcalls -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines all of the runtime library calls the backend can emit.
-// The various long double types cannot be merged, because 80-bit library
-// functions use "xf" and 128-bit use "tf".
-//
-// When adding PPCF128 functions here, note that their names generally need
-// to be overridden for Darwin with the xxx$LDBL128 form.  See
-// PPCISelLowering.cpp.
-//
-//===----------------------------------------------------------------------===//
-
-// NOTE: NO INCLUDE GUARD DESIRED!
-
-// Provide definitions of macros so that users of this file do not have to
-// define everything to use it...
-
-// Declare the enumerator for each libcall, along with its default name. Some
-// libcalls have different names on particular OSes or architectures. These
-// are set in InitLibcalls() in TargetLoweringBase.cpp and/or by targets
-// using TargetLoweringBase::setLibcallName()
-#ifndef HANDLE_LIBCALL
-#error "HANDLE_LIBCALL must be defined"
-#endif
-
-// Integer
-HANDLE_LIBCALL(SHL_I16, "__ashlhi3")
-HANDLE_LIBCALL(SHL_I32, "__ashlsi3")
-HANDLE_LIBCALL(SHL_I64, "__ashldi3")
-HANDLE_LIBCALL(SHL_I128, "__ashlti3")
-HANDLE_LIBCALL(SRL_I16, "__lshrhi3")
-HANDLE_LIBCALL(SRL_I32, "__lshrsi3")
-HANDLE_LIBCALL(SRL_I64, "__lshrdi3")
-HANDLE_LIBCALL(SRL_I128, "__lshrti3")
-HANDLE_LIBCALL(SRA_I16, "__ashrhi3")
-HANDLE_LIBCALL(SRA_I32, "__ashrsi3")
-HANDLE_LIBCALL(SRA_I64, "__ashrdi3")
-HANDLE_LIBCALL(SRA_I128, "__ashrti3")
-HANDLE_LIBCALL(MUL_I8, "__mulqi3")
-HANDLE_LIBCALL(MUL_I16, "__mulhi3")
-HANDLE_LIBCALL(MUL_I32, "__mulsi3")
-HANDLE_LIBCALL(MUL_I64, "__muldi3")
-HANDLE_LIBCALL(MUL_I128, "__multi3")
-HANDLE_LIBCALL(MULO_I32, "__mulosi4")
-HANDLE_LIBCALL(MULO_I64, "__mulodi4")
-HANDLE_LIBCALL(MULO_I128, "__muloti4")
-HANDLE_LIBCALL(SDIV_I8, "__divqi3")
-HANDLE_LIBCALL(SDIV_I16, "__divhi3")
-HANDLE_LIBCALL(SDIV_I32, "__divsi3")
-HANDLE_LIBCALL(SDIV_I64, "__divdi3")
-HANDLE_LIBCALL(SDIV_I128, "__divti3")
-HANDLE_LIBCALL(UDIV_I8, "__udivqi3")
-HANDLE_LIBCALL(UDIV_I16, "__udivhi3")
-HANDLE_LIBCALL(UDIV_I32, "__udivsi3")
-HANDLE_LIBCALL(UDIV_I64, "__udivdi3")
-HANDLE_LIBCALL(UDIV_I128, "__udivti3")
-HANDLE_LIBCALL(SREM_I8, "__modqi3")
-HANDLE_LIBCALL(SREM_I16, "__modhi3")
-HANDLE_LIBCALL(SREM_I32, "__modsi3")
-HANDLE_LIBCALL(SREM_I64, "__moddi3")
-HANDLE_LIBCALL(SREM_I128, "__modti3")
-HANDLE_LIBCALL(UREM_I8, "__umodqi3")
-HANDLE_LIBCALL(UREM_I16, "__umodhi3")
-HANDLE_LIBCALL(UREM_I32, "__umodsi3")
-HANDLE_LIBCALL(UREM_I64, "__umoddi3")
-HANDLE_LIBCALL(UREM_I128, "__umodti3")
-HANDLE_LIBCALL(SDIVREM_I8, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SDIVREM_I16, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SDIVREM_I32, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SDIVREM_I64, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SDIVREM_I128, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(UDIVREM_I8, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(UDIVREM_I16, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(UDIVREM_I32, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(UDIVREM_I64, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(UDIVREM_I128, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(NEG_I32, "__negsi2")
-HANDLE_LIBCALL(NEG_I64, "__negdi2")
-HANDLE_LIBCALL(CTLZ_I32, "__clzsi2")
-HANDLE_LIBCALL(CTLZ_I64, "__clzdi2")
-HANDLE_LIBCALL(CTLZ_I128, "__clzti2")
-HANDLE_LIBCALL(CTPOP_I32, "__popcountsi2")
-HANDLE_LIBCALL(CTPOP_I64, "__popcountdi2")
-HANDLE_LIBCALL(CTPOP_I128, "__popcountti2")
-
-// Floating-point
-HANDLE_LIBCALL(ADD_F32, "__addsf3")
-HANDLE_LIBCALL(ADD_F64, "__adddf3")
-HANDLE_LIBCALL(ADD_F80, "__addxf3")
-HANDLE_LIBCALL(ADD_F128, "__addtf3")
-HANDLE_LIBCALL(ADD_PPCF128, "__gcc_qadd")
-HANDLE_LIBCALL(SUB_F32, "__subsf3")
-HANDLE_LIBCALL(SUB_F64, "__subdf3")
-HANDLE_LIBCALL(SUB_F80, "__subxf3")
-HANDLE_LIBCALL(SUB_F128, "__subtf3")
-HANDLE_LIBCALL(SUB_PPCF128, "__gcc_qsub")
-HANDLE_LIBCALL(MUL_F32, "__mulsf3")
-HANDLE_LIBCALL(MUL_F64, "__muldf3")
-HANDLE_LIBCALL(MUL_F80, "__mulxf3")
-HANDLE_LIBCALL(MUL_F128, "__multf3")
-HANDLE_LIBCALL(MUL_PPCF128, "__gcc_qmul")
-HANDLE_LIBCALL(DIV_F32, "__divsf3")
-HANDLE_LIBCALL(DIV_F64, "__divdf3")
-HANDLE_LIBCALL(DIV_F80, "__divxf3")
-HANDLE_LIBCALL(DIV_F128, "__divtf3")
-HANDLE_LIBCALL(DIV_PPCF128, "__gcc_qdiv")
-HANDLE_LIBCALL(REM_F32, "fmodf")
-HANDLE_LIBCALL(REM_F64, "fmod")
-HANDLE_LIBCALL(REM_F80, "fmodl")
-HANDLE_LIBCALL(REM_F128, "fmodl")
-HANDLE_LIBCALL(REM_PPCF128, "fmodl")
-HANDLE_LIBCALL(FMA_F32, "fmaf")
-HANDLE_LIBCALL(FMA_F64, "fma")
-HANDLE_LIBCALL(FMA_F80, "fmal")
-HANDLE_LIBCALL(FMA_F128, "fmal")
-HANDLE_LIBCALL(FMA_PPCF128, "fmal")
-HANDLE_LIBCALL(POWI_F32, "__powisf2")
-HANDLE_LIBCALL(POWI_F64, "__powidf2")
-HANDLE_LIBCALL(POWI_F80, "__powixf2")
-HANDLE_LIBCALL(POWI_F128, "__powitf2")
-HANDLE_LIBCALL(POWI_PPCF128, "__powitf2")
-HANDLE_LIBCALL(SQRT_F32, "sqrtf")
-HANDLE_LIBCALL(SQRT_F64, "sqrt")
-HANDLE_LIBCALL(SQRT_F80, "sqrtl")
-HANDLE_LIBCALL(SQRT_F128, "sqrtl")
-HANDLE_LIBCALL(SQRT_PPCF128, "sqrtl")
-HANDLE_LIBCALL(CBRT_F32, "cbrtf")
-HANDLE_LIBCALL(CBRT_F64, "cbrt")
-HANDLE_LIBCALL(CBRT_F80, "cbrtl")
-HANDLE_LIBCALL(CBRT_F128, "cbrtl")
-HANDLE_LIBCALL(CBRT_PPCF128, "cbrtl")
-HANDLE_LIBCALL(LOG_F32, "logf")
-HANDLE_LIBCALL(LOG_F64, "log")
-HANDLE_LIBCALL(LOG_F80, "logl")
-HANDLE_LIBCALL(LOG_F128, "logl")
-HANDLE_LIBCALL(LOG_PPCF128, "logl")
-HANDLE_LIBCALL(LOG_FINITE_F32, "__logf_finite")
-HANDLE_LIBCALL(LOG_FINITE_F64, "__log_finite")
-HANDLE_LIBCALL(LOG_FINITE_F80, "__logl_finite")
-HANDLE_LIBCALL(LOG_FINITE_F128, "__logl_finite")
-HANDLE_LIBCALL(LOG_FINITE_PPCF128, "__logl_finite")
-HANDLE_LIBCALL(LOG2_F32, "log2f")
-HANDLE_LIBCALL(LOG2_F64, "log2")
-HANDLE_LIBCALL(LOG2_F80, "log2l")
-HANDLE_LIBCALL(LOG2_F128, "log2l")
-HANDLE_LIBCALL(LOG2_PPCF128, "log2l")
-HANDLE_LIBCALL(LOG2_FINITE_F32, "__log2f_finite")
-HANDLE_LIBCALL(LOG2_FINITE_F64, "__log2_finite")
-HANDLE_LIBCALL(LOG2_FINITE_F80, "__log2l_finite")
-HANDLE_LIBCALL(LOG2_FINITE_F128, "__log2l_finite")
-HANDLE_LIBCALL(LOG2_FINITE_PPCF128, "__log2l_finite")
-HANDLE_LIBCALL(LOG10_F32, "log10f")
-HANDLE_LIBCALL(LOG10_F64, "log10")
-HANDLE_LIBCALL(LOG10_F80, "log10l")
-HANDLE_LIBCALL(LOG10_F128, "log10l")
-HANDLE_LIBCALL(LOG10_PPCF128, "log10l")
-HANDLE_LIBCALL(LOG10_FINITE_F32, "__log10f_finite")
-HANDLE_LIBCALL(LOG10_FINITE_F64, "__log10_finite")
-HANDLE_LIBCALL(LOG10_FINITE_F80, "__log10l_finite")
-HANDLE_LIBCALL(LOG10_FINITE_F128, "__log10l_finite")
-HANDLE_LIBCALL(LOG10_FINITE_PPCF128, "__log10l_finite")
-HANDLE_LIBCALL(EXP_F32, "expf")
-HANDLE_LIBCALL(EXP_F64, "exp")
-HANDLE_LIBCALL(EXP_F80, "expl")
-HANDLE_LIBCALL(EXP_F128, "expl")
-HANDLE_LIBCALL(EXP_PPCF128, "expl")
-HANDLE_LIBCALL(EXP_FINITE_F32, "__expf_finite")
-HANDLE_LIBCALL(EXP_FINITE_F64, "__exp_finite")
-HANDLE_LIBCALL(EXP_FINITE_F80, "__expl_finite")
-HANDLE_LIBCALL(EXP_FINITE_F128, "__expl_finite")
-HANDLE_LIBCALL(EXP_FINITE_PPCF128, "__expl_finite")
-HANDLE_LIBCALL(EXP2_F32, "exp2f")
-HANDLE_LIBCALL(EXP2_F64, "exp2")
-HANDLE_LIBCALL(EXP2_F80, "exp2l")
-HANDLE_LIBCALL(EXP2_F128, "exp2l")
-HANDLE_LIBCALL(EXP2_PPCF128, "exp2l")
-HANDLE_LIBCALL(EXP2_FINITE_F32, "__exp2f_finite")
-HANDLE_LIBCALL(EXP2_FINITE_F64, "__exp2_finite")
-HANDLE_LIBCALL(EXP2_FINITE_F80, "__exp2l_finite")
-HANDLE_LIBCALL(EXP2_FINITE_F128, "__exp2l_finite")
-HANDLE_LIBCALL(EXP2_FINITE_PPCF128, "__exp2l_finite")
-HANDLE_LIBCALL(EXP10_F32, "exp10f")
-HANDLE_LIBCALL(EXP10_F64, "exp10")
-HANDLE_LIBCALL(EXP10_F80, "exp10l")
-HANDLE_LIBCALL(EXP10_F128, "exp10l")
-HANDLE_LIBCALL(EXP10_PPCF128, "exp10l")
-HANDLE_LIBCALL(SIN_F32, "sinf")
-HANDLE_LIBCALL(SIN_F64, "sin")
-HANDLE_LIBCALL(SIN_F80, "sinl")
-HANDLE_LIBCALL(SIN_F128, "sinl")
-HANDLE_LIBCALL(SIN_PPCF128, "sinl")
-HANDLE_LIBCALL(COS_F32, "cosf")
-HANDLE_LIBCALL(COS_F64, "cos")
-HANDLE_LIBCALL(COS_F80, "cosl")
-HANDLE_LIBCALL(COS_F128, "cosl")
-HANDLE_LIBCALL(COS_PPCF128, "cosl")
-HANDLE_LIBCALL(TAN_F32, "tanf")
-HANDLE_LIBCALL(TAN_F64, "tan")
-HANDLE_LIBCALL(TAN_F80, "tanl")
-HANDLE_LIBCALL(TAN_F128,"tanl")
-HANDLE_LIBCALL(TAN_PPCF128, "tanl")
-HANDLE_LIBCALL(SINH_F32, "sinhf")
-HANDLE_LIBCALL(SINH_F64, "sinh")
-HANDLE_LIBCALL(SINH_F80, "sinhl")
-HANDLE_LIBCALL(SINH_F128, "sinhl")
-HANDLE_LIBCALL(SINH_PPCF128, "sinhl")
-HANDLE_LIBCALL(COSH_F32, "coshf")
-HANDLE_LIBCALL(COSH_F64, "cosh")
-HANDLE_LIBCALL(COSH_F80, "coshl")
-HANDLE_LIBCALL(COSH_F128, "coshl")
-HANDLE_LIBCALL(COSH_PPCF128, "coshl")
-HANDLE_LIBCALL(TANH_F32, "tanhf")
-HANDLE_LIBCALL(TANH_F64, "tanh")
-HANDLE_LIBCALL(TANH_F80, "tanhl")
-HANDLE_LIBCALL(TANH_F128,"tanhl")
-HANDLE_LIBCALL(TANH_PPCF128, "tanhl")
-HANDLE_LIBCALL(ASIN_F32, "asinf")
-HANDLE_LIBCALL(ASIN_F64, "asin")
-HANDLE_LIBCALL(ASIN_F80, "asinl")
-HANDLE_LIBCALL(ASIN_F128, "asinl")
-HANDLE_LIBCALL(ASIN_PPCF128, "asinl")
-HANDLE_LIBCALL(ACOS_F32, "acosf")
-HANDLE_LIBCALL(ACOS_F64, "acos")
-HANDLE_LIBCALL(ACOS_F80, "acosl")
-HANDLE_LIBCALL(ACOS_F128, "acosl")
-HANDLE_LIBCALL(ACOS_PPCF128, "acosl")
-HANDLE_LIBCALL(ATAN_F32, "atanf")
-HANDLE_LIBCALL(ATAN_F64, "atan")
-HANDLE_LIBCALL(ATAN_F80, "atanl")
-HANDLE_LIBCALL(ATAN_F128,"atanl")
-HANDLE_LIBCALL(ATAN_PPCF128, "atanl")
-HANDLE_LIBCALL(ATAN2_F32, "atan2f")
-HANDLE_LIBCALL(ATAN2_F64, "atan2")
-HANDLE_LIBCALL(ATAN2_F80, "atan2l")
-HANDLE_LIBCALL(ATAN2_F128,"atan2l")
-HANDLE_LIBCALL(ATAN2_PPCF128, "atan2l")
-HANDLE_LIBCALL(SINCOS_F32, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_F64, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_F80, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_F128, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_PPCF128, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_STRET_F32, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(SINCOS_STRET_F64, LIBCALL_NO_NAME)
-HANDLE_LIBCALL(POW_F32, "powf")
-HANDLE_LIBCALL(POW_F64, "pow")
-HANDLE_LIBCALL(POW_F80, "powl")
-HANDLE_LIBCALL(POW_F128, "powl")
-HANDLE_LIBCALL(POW_PPCF128, "powl")
-HANDLE_LIBCALL(POW_FINITE_F32, "__powf_finite")
-HANDLE_LIBCALL(POW_FINITE_F64, "__pow_finite")
-HANDLE_LIBCALL(POW_FINITE_F80, "__powl_finite")
-HANDLE_LIBCALL(POW_FINITE_F128, "__powl_finite")
-HANDLE_LIBCALL(POW_FINITE_PPCF128, "__powl_finite")
-HANDLE_LIBCALL(CEIL_F32, "ceilf")
-HANDLE_LIBCALL(CEIL_F64, "ceil")
-HANDLE_LIBCALL(CEIL_F80, "ceill")
-HANDLE_LIBCALL(CEIL_F128, "ceill")
-HANDLE_LIBCALL(CEIL_PPCF128, "ceill")
-HANDLE_LIBCALL(TRUNC_F32, "truncf")
-HANDLE_LIBCALL(TRUNC_F64, "trunc")
-HANDLE_LIBCALL(TRUNC_F80, "truncl")
-HANDLE_LIBCALL(TRUNC_F128, "truncl")
-HANDLE_LIBCALL(TRUNC_PPCF128, "truncl")
-HANDLE_LIBCALL(RINT_F32, "rintf")
-HANDLE_LIBCALL(RINT_F64, "rint")
-HANDLE_LIBCALL(RINT_F80, "rintl")
-HANDLE_LIBCALL(RINT_F128, "rintl")
-HANDLE_LIBCALL(RINT_PPCF128, "rintl")
-HANDLE_LIBCALL(NEARBYINT_F32, "nearbyintf")
-HANDLE_LIBCALL(NEARBYINT_F64, "nearbyint")
-HANDLE_LIBCALL(NEARBYINT_F80, "nearbyintl")
-HANDLE_LIBCALL(NEARBYINT_F128, "nearbyintl")
-HANDLE_LIBCALL(NEARBYINT_PPCF128, "nearbyintl")
-HANDLE_LIBCALL(ROUND_F32, "roundf")
-HANDLE_LIBCALL(ROUND_F64, "round")
-HANDLE_LIBCALL(ROUND_F80, "roundl")
-HANDLE_LIBCALL(ROUND_F128, "roundl")
-HANDLE_LIBCALL(ROUND_PPCF128, "roundl")
-HANDLE_LIBCALL(ROUNDEVEN_F32, "roundevenf")
-HANDLE_LIBCALL(ROUNDEVEN_F64, "roundeven")
-HANDLE_LIBCALL(ROUNDEVEN_F80, "roundevenl")
-HANDLE_LIBCALL(ROUNDEVEN_F128, "roundevenl")
-HANDLE_LIBCALL(ROUNDEVEN_PPCF128, "roundevenl")
-HANDLE_LIBCALL(FLOOR_F32, "floorf")
-HANDLE_LIBCALL(FLOOR_F64, "floor")
-HANDLE_LIBCALL(FLOOR_F80, "floorl")
-HANDLE_LIBCALL(FLOOR_F128, "floorl")
-HANDLE_LIBCALL(FLOOR_PPCF128, "floorl")
-HANDLE_LIBCALL(COPYSIGN_F32, "copysignf")
-HANDLE_LIBCALL(COPYSIGN_F64, "copysign")
-HANDLE_LIBCALL(COPYSIGN_F80, "copysignl")
-HANDLE_LIBCALL(COPYSIGN_F128, "copysignl")
-HANDLE_LIBCALL(COPYSIGN_PPCF128, "copysignl")
-HANDLE_LIBCALL(FMIN_F32, "fminf")
-HANDLE_LIBCALL(FMIN_F64, "fmin")
-HANDLE_LIBCALL(FMIN_F80, "fminl")
-HANDLE_LIBCALL(FMIN_F128, "fminl")
-HANDLE_LIBCALL(FMIN_PPCF128, "fminl")
-HANDLE_LIBCALL(FMAX_F32, "fmaxf")
-HANDLE_LIBCALL(FMAX_F64, "fmax")
-HANDLE_LIBCALL(FMAX_F80, "fmaxl")
-HANDLE_LIBCALL(FMAX_F128, "fmaxl")
-HANDLE_LIBCALL(FMAX_PPCF128, "fmaxl")
-HANDLE_LIBCALL(FMINIMUM_F32, "fminimumf")
-HANDLE_LIBCALL(FMINIMUM_F64, "fminimum")
-HANDLE_LIBCALL(FMINIMUM_F80, "fminimuml")
-HANDLE_LIBCALL(FMINIMUM_F128, "fminimuml")
-HANDLE_LIBCALL(FMINIMUM_PPCF128, "fminimuml")
-HANDLE_LIBCALL(FMAXIMUM_F32, "fmaximumf")
-HANDLE_LIBCALL(FMAXIMUM_F64, "fmaximum")
-HANDLE_LIBCALL(FMAXIMUM_F80, "fmaximuml")
-HANDLE_LIBCALL(FMAXIMUM_F128, "fmaximuml")
-HANDLE_LIBCALL(FMAXIMUM_PPCF128, "fmaximum_numl")
-HANDLE_LIBCALL(FMINIMUM_NUM_F32, "fminimum_numf")
-HANDLE_LIBCALL(FMINIMUM_NUM_F64, "fminimum_num")
-HANDLE_LIBCALL(FMINIMUM_NUM_F80, "fminimum_numl")
-HANDLE_LIBCALL(FMINIMUM_NUM_F128, "fminimum_numl")
-HANDLE_LIBCALL(FMINIMUM_NUM_PPCF128, "fminimum_numl")
-HANDLE_LIBCALL(FMAXIMUM_NUM_F32, "fmaximum_numf")
-HANDLE_LIBCALL(FMAXIMUM_NUM_F64, "fmaximum_num")
-HANDLE_LIBCALL(FMAXIMUM_NUM_F80, "fmaximum_numl")
-HANDLE_LIBCALL(FMAXIMUM_NUM_F128, "fmaximum_numl")
-HANDLE_LIBCALL(FMAXIMUM_NUM_PPCF128, "fmaximum_numl")
-HANDLE_LIBCALL(LROUND_F32, "lroundf")
-HANDLE_LIBCALL(LROUND_F64, "lround")
-HANDLE_LIBCALL(LROUND_F80, "lroundl")
-HANDLE_LIBCALL(LROUND_F128, "lroundl")
-HANDLE_LIBCALL(LROUND_PPCF128, "lroundl")
-HANDLE_LIBCALL(LLROUND_F32, "llroundf")
-HANDLE_LIBCALL(LLROUND_F64, "llround")
-HANDLE_LIBCALL(LLROUND_F80, "llroundl")
-HANDLE_LIBCALL(LLROUND_F128, "llroundl")
-HANDLE_LIBCALL(LLROUND_PPCF128, "llroundl")
-HANDLE_LIBCALL(LRINT_F32, "lrintf")
-HANDLE_LIBCALL(LRINT_F64, "lrint")
-HANDLE_LIBCALL(LRINT_F80, "lrintl")
-HANDLE_LIBCALL(LRINT_F128, "lrintl")
-HANDLE_LIBCALL(LRINT_PPCF128, "lrintl")
-HANDLE_LIBCALL(LLRINT_F32, "llrintf")
-HANDLE_LIBCALL(LLRINT_F64, "llrint")
-HANDLE_LIBCALL(LLRINT_F80, "llrintl")
-HANDLE_LIBCALL(LLRINT_F128, "llrintl")
-HANDLE_LIBCALL(LLRINT_PPCF128, "llrintl")
-HANDLE_LIBCALL(LDEXP_F32, "ldexpf")
-HANDLE_LIBCALL(LDEXP_F64, "ldexp")
-HANDLE_LIBCALL(LDEXP_F80, "ldexpl")
-HANDLE_LIBCALL(LDEXP_F128, "ldexpl")
-HANDLE_LIBCALL(LDEXP_PPCF128, "ldexpl")
-HANDLE_LIBCALL(FREXP_F32, "frexpf")
-HANDLE_LIBCALL(FREXP_F64, "frexp")
-HANDLE_LIBCALL(FREXP_F80, "frexpl")
-HANDLE_LIBCALL(FREXP_F128, "frexpl")
-HANDLE_LIBCALL(FREXP_PPCF128, "frexpl")
-HANDLE_LIBCALL(SINCOSPI_F32, "sincospif")
-HANDLE_LIBCALL(SINCOSPI_F64, "sincospi")
-HANDLE_LIBCALL(SINCOSPI_F80, "sincospil")
-HANDLE_LIBCALL(SINCOSPI_F128, "sincospil")
-HANDLE_LIBCALL(SINCOSPI_PPCF128, "sincospil")
-HANDLE_LIBCALL(MODF_F32, "modff")
-HANDLE_LIBCALL(MODF_F64, "modf")
-HANDLE_LIBCALL(MODF_F80, "modfl")
-HANDLE_LIBCALL(MODF_F128, "modfl")
-HANDLE_LIBCALL(MODF_PPCF128, "modfl")
-
-// Floating point environment
-HANDLE_LIBCALL(FEGETENV, "fegetenv")
-HANDLE_LIBCALL(FESETENV, "fesetenv")
-
-// Floating point control modes
-HANDLE_LIBCALL(FEGETMODE, "fegetmode")
-HANDLE_LIBCALL(FESETMODE, "fesetmode")
-
-// Conversion
-HANDLE_LIBCALL(FPEXT_BF16_F32, "__extendbfsf2")
-HANDLE_LIBCALL(FPEXT_F32_PPCF128, "__gcc_stoq")
-HANDLE_LIBCALL(FPEXT_F64_PPCF128, "__gcc_dtoq")
-HANDLE_LIBCALL(FPEXT_F80_F128, "__extendxftf2")
-HANDLE_LIBCALL(FPEXT_F64_F128, "__extenddftf2")
-HANDLE_LIBCALL(FPEXT_F32_F128, "__extendsftf2")
-HANDLE_LIBCALL(FPEXT_F16_F128, "__extendhftf2")
-HANDLE_LIBCALL(FPEXT_F16_F80, "__extendhfxf2")
-HANDLE_LIBCALL(FPEXT_F32_F64, "__extendsfdf2")
-HANDLE_LIBCALL(FPEXT_F16_F64, "__extendhfdf2")
-HANDLE_LIBCALL(FPEXT_F16_F32, "__extendhfsf2")
-HANDLE_LIBCALL(FPROUND_F32_F16, "__truncsfhf2")
-HANDLE_LIBCALL(FPROUND_F64_F16, "__truncdfhf2")
-HANDLE_LIBCALL(FPROUND_F80_F16, "__truncxfhf2")
-HANDLE_LIBCALL(FPROUND_F128_F16, "__trunctfhf2")
-HANDLE_LIBCALL(FPROUND_PPCF128_F16, "__trunctfhf2")
-HANDLE_LIBCALL(FPROUND_F32_BF16, "__truncsfbf2")
-HANDLE_LIBCALL(FPROUND_F64_BF16, "__truncdfbf2")
-HANDLE_LIBCALL(FPROUND_F80_BF16, "__truncxfbf2")
-HANDLE_LIBCALL(FPROUND_F128_BF16, "__trunctfbf2")
-HANDLE_LIBCALL(FPROUND_F64_F32, "__truncdfsf2")
-HANDLE_LIBCALL(FPROUND_F80_F32, "__truncxfsf2")
-HANDLE_LIBCALL(FPROUND_F128_F32, "__trunctfsf2")
-HANDLE_LIBCALL(FPROUND_PPCF128_F32, "__gcc_qtos")
-HANDLE_LIBCALL(FPROUND_F80_F64, "__truncxfdf2")
-HANDLE_LIBCALL(FPROUND_F128_F64, "__trunctfdf2")
-HANDLE_LIBCALL(FPROUND_PPCF128_F64, "__gcc_qtod")
-HANDLE_LIBCALL(FPROUND_F128_F80, "__trunctfxf2")
-HANDLE_LIBCALL(FPTOSINT_F16_I32, "__fixhfsi")
-HANDLE_LIBCALL(FPTOSINT_F16_I64, "__fixhfdi")
-HANDLE_LIBCALL(FPTOSINT_F16_I128, "__fixhfti")
-HANDLE_LIBCALL(FPTOSINT_F32_I32, "__fixsfsi")
-HANDLE_LIBCALL(FPTOSINT_F32_I64, "__fixsfdi")
-HANDLE_LIBCALL(FPTOSINT_F32_I128, "__fixsfti")
-HANDLE_LIBCALL(FPTOSINT_F64_I32, "__fixdfsi")
-HANDLE_LIBCALL(FPTOSINT_F64_I64, "__fixdfdi")
-HANDLE_LIBCALL(FPTOSINT_F64_I128, "__fixdfti")
-HANDLE_LIBCALL(FPTOSINT_F80_I32, "__fixxfsi")
-HANDLE_LIBCALL(FPTOSINT_F80_I64, "__fixxfdi")
-HANDLE_LIBCALL(FPTOSINT_F80_I128, "__fixxfti")
-HANDLE_LIBCALL(FPTOSINT_F128_I32, "__fixtfsi")
-HANDLE_LIBCALL(FPTOSINT_F128_I64, "__fixtfdi")
-HANDLE_LIBCALL(FPTOSINT_F128_I128, "__fixtfti")
-HANDLE_LIBCALL(FPTOSINT_PPCF128_I32, "__gcc_qtou")
-HANDLE_LIBCALL(FPTOSINT_PPCF128_I64, "__fixtfdi")
-HANDLE_LIBCALL(FPTOSINT_PPCF128_I128, "__fixtfti")
-HANDLE_LIBCALL(FPTOUINT_F16_I32, "__fixunshfsi")
-HANDLE_LIBCALL(FPTOUINT_F16_I64, "__fixunshfdi")
-HANDLE_LIBCALL(FPTOUINT_F16_I128, "__fixunshfti")
-HANDLE_LIBCALL(FPTOUINT_F32_I32, "__fixunssfsi")
-HANDLE_LIBCALL(FPTOUINT_F32_I64, "__fixunssfdi")
-HANDLE_LIBCALL(FPTOUINT_F32_I128, "__fixunssfti")
-HANDLE_LIBCALL(FPTOUINT_F64_I32, "__fixunsdfsi")
-HANDLE_LIBCALL(FPTOUINT_F64_I64, "__fixunsdfdi")
-HANDLE_LIBCALL(FPTOUINT_F64_I128, "__fixunsdfti")
-HANDLE_LIBCALL(FPTOUINT_F80_I32, "__fixunsxfsi")
-HANDLE_LIBCALL(FPTOUINT_F80_I64, "__fixunsxfdi")
-HANDLE_LIBCALL(FPTOUINT_F80_I128, "__fixunsxfti")
-HANDLE_LIBCALL(FPTOUINT_F128_I32, "__fixunstfsi")
-HANDLE_LIBCALL(FPTOUINT_F128_I64, "__fixunstfdi")
-HANDLE_LIBCALL(FPTOUINT_F128_I128, "__fixunstfti")
-HANDLE_LIBCALL(FPTOUINT_PPCF128_I32, "__fixunstfsi")
-HANDLE_LIBCALL(FPTOUINT_PPCF128_I64, "__fixunstfdi")
-HANDLE_LIBCALL(FPTOUINT_PPCF128_I128, "__fixunstfti")
-HANDLE_LIBCALL(SINTTOFP_I32_F16, "__floatsihf")
-HANDLE_LIBCALL(SINTTOFP_I32_F32, "__floatsisf")
-HANDLE_LIBCALL(SINTTOFP_I32_F64, "__floatsidf")
-HANDLE_LIBCALL(SINTTOFP_I32_F80, "__floatsixf")
-HANDLE_LIBCALL(SINTTOFP_I32_F128, "__floatsitf")
-HANDLE_LIBCALL(SINTTOFP_I32_PPCF128, "__gcc_itoq")
-HANDLE_LIBCALL(SINTTOFP_I64_BF16, "__floatdibf")
-HANDLE_LIBCALL(SINTTOFP_I64_F16, "__floatdihf")
-HANDLE_LIBCALL(SINTTOFP_I64_F32, "__floatdisf")
-HANDLE_LIBCALL(SINTTOFP_I64_F64, "__floatdidf")
-HANDLE_LIBCALL(SINTTOFP_I64_F80, "__floatdixf")
-HANDLE_LIBCALL(SINTTOFP_I64_F128, "__floatditf")
-HANDLE_LIBCALL(SINTTOFP_I64_PPCF128, "__floatditf")
-HANDLE_LIBCALL(SINTTOFP_I128_F16, "__floattihf")
-HANDLE_LIBCALL(SINTTOFP_I128_F32, "__floattisf")
-HANDLE_LIBCALL(SINTTOFP_I128_F64, "__floattidf")
-HANDLE_LIBCALL(SINTTOFP_I128_F80, "__floattixf")
-HANDLE_LIBCALL(SINTTOFP_I128_F128, "__floattitf")
-HANDLE_LIBCALL(SINTTOFP_I128_PPCF128, "__floattitf")
-HANDLE_LIBCALL(UINTTOFP_I32_F16, "__floatunsihf")
-HANDLE_LIBCALL(UINTTOFP_I32_F32, "__floatunsisf")
-HANDLE_LIBCALL(UINTTOFP_I32_F64, "__floatunsidf")
-HANDLE_LIBCALL(UINTTOFP_I32_F80, "__floatunsixf")
-HANDLE_LIBCALL(UINTTOFP_I32_F128, "__floatunsitf")
-HANDLE_LIBCALL(UINTTOFP_I32_PPCF128, "__gcc_utoq")
-HANDLE_LIBCALL(UINTTOFP_I64_BF16, "__floatundibf")
-HANDLE_LIBCALL(UINTTOFP_I64_F16, "__floatundihf")
-HANDLE_LIBCALL(UINTTOFP_I64_F32, "__floatundisf")
-HANDLE_LIBCALL(UINTTOFP_I64_F64, "__floatundidf")
-HANDLE_LIBCALL(UINTTOFP_I64_F80, "__floatundixf")
-HANDLE_LIBCALL(UINTTOFP_I64_F128, "__floatunditf")
-HANDLE_LIBCALL(UINTTOFP_I64_PPCF128, "__floatunditf")
-HANDLE_LIBCALL(UINTTOFP_I128_F16, "__floatuntihf")
-HANDLE_LIBCALL(UINTTOFP_I128_F32, "__floatuntisf")
-HANDLE_LIBCALL(UINTTOFP_I128_F64, "__floatuntidf")
-HANDLE_LIBCALL(UINTTOFP_I128_F80, "__floatuntixf")
-HANDLE_LIBCALL(UINTTOFP_I128_F128, "__floatuntitf")
-HANDLE_LIBCALL(UINTTOFP_I128_PPCF128, "__floatuntitf")
-HANDLE_LIBCALL(CONVERT_F128_PPCF128, "__extendkftf2")
-HANDLE_LIBCALL(CONVERT_PPCF128_F128, "__trunctfkf2")
-
-// Comparison
-HANDLE_LIBCALL(OEQ_F32, "__eqsf2")
-HANDLE_LIBCALL(OEQ_F64, "__eqdf2")
-HANDLE_LIBCALL(OEQ_F128, "__eqtf2")
-HANDLE_LIBCALL(OEQ_PPCF128, "__gcc_qeq")
-HANDLE_LIBCALL(UNE_F32, "__nesf2")
-HANDLE_LIBCALL(UNE_F64, "__nedf2")
-HANDLE_LIBCALL(UNE_F128, "__netf2")
-HANDLE_LIBCALL(UNE_PPCF128, "__gcc_qne")
-HANDLE_LIBCALL(OGE_F32, "__gesf2")
-HANDLE_LIBCALL(OGE_F64, "__gedf2")
-HANDLE_LIBCALL(OGE_F128, "__getf2")
-HANDLE_LIBCALL(OGE_PPCF128, "__gcc_qge")
-HANDLE_LIBCALL(OLT_F32, "__ltsf2")
-HANDLE_LIBCALL(OLT_F64, "__ltdf2")
-HANDLE_LIBCALL(OLT_F128, "__lttf2")
-HANDLE_LIBCALL(OLT_PPCF128, "__gcc_qlt")
-HANDLE_LIBCALL(OLE_F32, "__lesf2")
-HANDLE_LIBCALL(OLE_F64, "__ledf2")
-HANDLE_LIBCALL(OLE_F128, "__letf2")
-HANDLE_LIBCALL(OLE_PPCF128, "__gcc_qle")
-HANDLE_LIBCALL(OGT_F32, "__gtsf2")
-HANDLE_LIBCALL(OGT_F64, "__gtdf2")
-HANDLE_LIBCALL(OGT_F128, "__gttf2")
-HANDLE_LIBCALL(OGT_PPCF128, "__gcc_qgt")
-HANDLE_LIBCALL(UO_F32, "__unordsf2")
-HANDLE_LIBCALL(UO_F64, "__unorddf2")
-HANDLE_LIBCALL(UO_F128, "__unordtf2")
-HANDLE_LIBCALL(UO_PPCF128, "__gcc_qunord")
-
-// Memory
-HANDLE_LIBCALL(MEMCPY, "memcpy")
-HANDLE_LIBCALL(MEMMOVE, "memmove")
-HANDLE_LIBCALL(MEMSET, "memset")
-// DSEPass can emit calloc if it finds a pair of malloc/memset
-HANDLE_LIBCALL(CALLOC, "calloc")
-HANDLE_LIBCALL(BZERO, LIBCALL_NO_NAME)
-
-// Element-wise unordered-atomic memory of different sizes
-HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memcpy_element_unordered_atomic_1")
-HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memcpy_element_unordered_atomic_2")
-HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memcpy_element_unordered_atomic_4")
-HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memcpy_element_unordered_atomic_8")
-HANDLE_LIBCALL(MEMCPY_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memcpy_element_unordered_atomic_16")
-HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memmove_element_unordered_atomic_1")
-HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memmove_element_unordered_atomic_2")
-HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memmove_element_unordered_atomic_4")
-HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memmove_element_unordered_atomic_8")
-HANDLE_LIBCALL(MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memmove_element_unordered_atomic_16")
-HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_1, "__llvm_memset_element_unordered_atomic_1")
-HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_2, "__llvm_memset_element_unordered_atomic_2")
-HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_4, "__llvm_memset_element_unordered_atomic_4")
-HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_8, "__llvm_memset_element_unordered_atomic_8")
-HANDLE_LIBCALL(MEMSET_ELEMENT_UNORDERED_ATOMIC_16, "__llvm_memset_element_unordered_atomic_16")
-
-// Exception handling
-HANDLE_LIBCALL(UNWIND_RESUME, "_Unwind_Resume")
-HANDLE_LIBCALL(CXA_END_CLEANUP, "__cxa_end_cleanup")
-
-// Note: there are two sets of atomics libcalls; see
-// <https://llvm.org/docs/Atomics.html> for more info on the
-// difference between them.
-
-// Atomic '__sync_*' libcalls.
-HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_1, "__sync_val_compare_and_swap_1")
-HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_2, "__sync_val_compare_and_swap_2")
-HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_4, "__sync_val_compare_and_swap_4")
-HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_8, "__sync_val_compare_and_swap_8")
-HANDLE_LIBCALL(SYNC_VAL_COMPARE_AND_SWAP_16, "__sync_val_compare_and_swap_16")
-HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_1, "__sync_lock_test_and_set_1")
-HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_2, "__sync_lock_test_and_set_2")
-HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_4, "__sync_lock_test_and_set_4")
-HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_8, "__sync_lock_test_and_set_8")
-HANDLE_LIBCALL(SYNC_LOCK_TEST_AND_SET_16, "__sync_lock_test_and_set_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_1, "__sync_fetch_and_add_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_2, "__sync_fetch_and_add_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_4, "__sync_fetch_and_add_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_8, "__sync_fetch_and_add_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_ADD_16, "__sync_fetch_and_add_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_1, "__sync_fetch_and_sub_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_2, "__sync_fetch_and_sub_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_4, "__sync_fetch_and_sub_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_8, "__sync_fetch_and_sub_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_SUB_16, "__sync_fetch_and_sub_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_AND_1, "__sync_fetch_and_and_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_AND_2, "__sync_fetch_and_and_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_AND_4, "__sync_fetch_and_and_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_AND_8, "__sync_fetch_and_and_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_AND_16, "__sync_fetch_and_and_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_OR_1, "__sync_fetch_and_or_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_OR_2, "__sync_fetch_and_or_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_OR_4, "__sync_fetch_and_or_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_OR_8, "__sync_fetch_and_or_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_OR_16, "__sync_fetch_and_or_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_1, "__sync_fetch_and_xor_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_2, "__sync_fetch_and_xor_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_4, "__sync_fetch_and_xor_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_8, "__sync_fetch_and_xor_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_XOR_16, "__sync_fetch_and_xor_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_1, "__sync_fetch_and_nand_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_2, "__sync_fetch_and_nand_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_4, "__sync_fetch_and_nand_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_8, "__sync_fetch_and_nand_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_NAND_16, "__sync_fetch_and_nand_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_1, "__sync_fetch_and_max_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_2, "__sync_fetch_and_max_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_4, "__sync_fetch_and_max_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_8, "__sync_fetch_and_max_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MAX_16, "__sync_fetch_and_max_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_1, "__sync_fetch_and_umax_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_2, "__sync_fetch_and_umax_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_4, "__sync_fetch_and_umax_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_8, "__sync_fetch_and_umax_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMAX_16, "__sync_fetch_and_umax_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_1, "__sync_fetch_and_min_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_2, "__sync_fetch_and_min_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_4, "__sync_fetch_and_min_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_8, "__sync_fetch_and_min_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_MIN_16, "__sync_fetch_and_min_16")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_1, "__sync_fetch_and_umin_1")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_2, "__sync_fetch_and_umin_2")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_4, "__sync_fetch_and_umin_4")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_8, "__sync_fetch_and_umin_8")
-HANDLE_LIBCALL(SYNC_FETCH_AND_UMIN_16, "__sync_fetch_and_umin_16")
-
-// Atomic `__atomic_*' libcalls.
-HANDLE_LIBCALL(ATOMIC_LOAD, "__atomic_load")
-HANDLE_LIBCALL(ATOMIC_LOAD_1, "__atomic_load_1")
-HANDLE_LIBCALL(ATOMIC_LOAD_2, "__atomic_load_2")
-HANDLE_LIBCALL(ATOMIC_LOAD_4, "__atomic_load_4")
-HANDLE_LIBCALL(ATOMIC_LOAD_8, "__atomic_load_8")
-HANDLE_LIBCALL(ATOMIC_LOAD_16, "__atomic_load_16")
-
-HANDLE_LIBCALL(ATOMIC_STORE, "__atomic_store")
-HANDLE_LIBCALL(ATOMIC_STORE_1, "__atomic_store_1")
-HANDLE_LIBCALL(ATOMIC_STORE_2, "__atomic_store_2")
-HANDLE_LIBCALL(ATOMIC_STORE_4, "__atomic_store_4")
-HANDLE_LIBCALL(ATOMIC_STORE_8, "__atomic_store_8")
-HANDLE_LIBCALL(ATOMIC_STORE_16, "__atomic_store_16")
-
-HANDLE_LIBCALL(ATOMIC_EXCHANGE, "__atomic_exchange")
-HANDLE_LIBCALL(ATOMIC_EXCHANGE_1, "__atomic_exchange_1")
-HANDLE_LIBCALL(ATOMIC_EXCHANGE_2, "__atomic_exchange_2")
-HANDLE_LIBCALL(ATOMIC_EXCHANGE_4, "__atomic_exchange_4")
-HANDLE_LIBCALL(ATOMIC_EXCHANGE_8, "__atomic_exchange_8")
-HANDLE_LIBCALL(ATOMIC_EXCHANGE_16, "__atomic_exchange_16")
-
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE, "__atomic_compare_exchange")
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_1, "__atomic_compare_exchange_1")
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_2, "__atomic_compare_exchange_2")
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_4, "__atomic_compare_exchange_4")
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_8, "__atomic_compare_exchange_8")
-HANDLE_LIBCALL(ATOMIC_COMPARE_EXCHANGE_16, "__atomic_compare_exchange_16")
-
-HANDLE_LIBCALL(ATOMIC_FETCH_ADD_1, "__atomic_fetch_add_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_ADD_2, "__atomic_fetch_add_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_ADD_4, "__atomic_fetch_add_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_ADD_8, "__atomic_fetch_add_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_ADD_16, "__atomic_fetch_add_16")
-HANDLE_LIBCALL(ATOMIC_FETCH_SUB_1, "__atomic_fetch_sub_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_SUB_2, "__atomic_fetch_sub_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_SUB_4, "__atomic_fetch_sub_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_SUB_8, "__atomic_fetch_sub_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_SUB_16, "__atomic_fetch_sub_16")
-HANDLE_LIBCALL(ATOMIC_FETCH_AND_1, "__atomic_fetch_and_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_AND_2, "__atomic_fetch_and_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_AND_4, "__atomic_fetch_and_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_AND_8, "__atomic_fetch_and_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_AND_16, "__atomic_fetch_and_16")
-HANDLE_LIBCALL(ATOMIC_FETCH_OR_1, "__atomic_fetch_or_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_OR_2, "__atomic_fetch_or_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_OR_4, "__atomic_fetch_or_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_OR_8, "__atomic_fetch_or_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_OR_16, "__atomic_fetch_or_16")
-HANDLE_LIBCALL(ATOMIC_FETCH_XOR_1, "__atomic_fetch_xor_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_XOR_2, "__atomic_fetch_xor_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_XOR_4, "__atomic_fetch_xor_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_XOR_8, "__atomic_fetch_xor_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_XOR_16, "__atomic_fetch_xor_16")
-HANDLE_LIBCALL(ATOMIC_FETCH_NAND_1, "__atomic_fetch_nand_1")
-HANDLE_LIBCALL(ATOMIC_FETCH_NAND_2, "__atomic_fetch_nand_2")
-HANDLE_LIBCALL(ATOMIC_FETCH_NAND_4, "__atomic_fetch_nand_4")
-HANDLE_LIBCALL(ATOMIC_FETCH_NAND_8, "__atomic_fetch_nand_8")
-HANDLE_LIBCALL(ATOMIC_FETCH_NAND_16, "__atomic_fetch_nand_16")
-
-// Out-of-line atomics libcalls
-#define HLCALLS(A, N)                                                          \
-  HANDLE_LIBCALL(A##N##_RELAX, LIBCALL_NO_NAME)                                \
-  HANDLE_LIBCALL(A##N##_ACQ, LIBCALL_NO_NAME)                                  \
-  HANDLE_LIBCALL(A##N##_REL, LIBCALL_NO_NAME)                                  \
-  HANDLE_LIBCALL(A##N##_ACQ_REL, LIBCALL_NO_NAME)
-#define HLCALL5(A)                                                             \
-  HLCALLS(A, 1) HLCALLS(A, 2) HLCALLS(A, 4) HLCALLS(A, 8) HLCALLS(A, 16)
-HLCALL5(OUTLINE_ATOMIC_CAS)
-HLCALL5(OUTLINE_ATOMIC_SWP)
-HLCALL5(OUTLINE_ATOMIC_LDADD)
-HLCALL5(OUTLINE_ATOMIC_LDSET)
-HLCALL5(OUTLINE_ATOMIC_LDCLR)
-HLCALL5(OUTLINE_ATOMIC_LDEOR)
-#undef HLCALLS
-#undef HLCALL5
-
-// Stack Protector Fail
-HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail")
-
-// Deoptimization
-HANDLE_LIBCALL(DEOPTIMIZE, "__llvm_deoptimize")
-
-// Return address
-HANDLE_LIBCALL(RETURN_ADDRESS, LIBCALL_NO_NAME)
-
-// Clear cache
-HANDLE_LIBCALL(CLEAR_CACHE, "__clear_cache")
-HANDLE_LIBCALL(RISCV_FLUSH_ICACHE, "__riscv_flush_icache")
-
-HANDLE_LIBCALL(UNKNOWN_LIBCALL, LIBCALL_NO_NAME)
-
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index 2a095be58a49..50b9791111ce 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -23,23 +23,15 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
-namespace llvm {
-namespace RTLIB {
+/// TableGen will produce 2 enums, RTLIB::Libcall and
+/// RTLIB::LibcallImpl. RTLIB::Libcall describes abstract functionality the
+/// compiler may choose to access, RTLIB::LibcallImpl describes a particular ABI
+/// implementation, which includes a name and type signature.
+#define GET_RUNTIME_LIBCALL_ENUM
+#include "llvm/IR/RuntimeLibcalls.inc"
+#undef GET_RUNTIME_LIBCALL_ENUM
 
-/// RTLIB::Libcall enum - This enum defines all of the runtime library calls
-/// the backend can emit.  The various long double types cannot be merged,
-/// because 80-bit library functions use "xf" and 128-bit use "tf".
-///
-/// When adding PPCF128 functions here, note that their names generally need
-/// to be overridden for Darwin with the xxx$LDBL128 form.  See
-/// PPCISelLowering.cpp.
-///
-enum Libcall {
-#define HANDLE_LIBCALL(code, name) code,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-} // namespace RTLIB
+namespace llvm {
 
 template <> struct enum_iteration_traits<RTLIB::Libcall> {
   static constexpr bool is_iterable = true;
@@ -59,37 +51,47 @@ struct RuntimeLibcallsInfo {
       ExceptionHandling ExceptionModel = ExceptionHandling::None,
       FloatABI::ABIType FloatABI = FloatABI::Default,
       EABI EABIVersion = EABI::Default, StringRef ABIName = "") {
+    initSoftFloatCmpLibcallPredicates();
+    initDefaultLibCallImpls();
     initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName);
   }
 
   /// Rename the default libcall routine name for the specified libcall.
-  void setLibcallName(RTLIB::Libcall Call, const char *Name) {
-    LibcallRoutineNames[Call] = Name;
-  }
-
-  void setLibcallName(ArrayRef<RTLIB::Libcall> Calls, const char *Name) {
-    for (auto Call : Calls)
-      setLibcallName(Call, Name);
+  void setLibcallImpl(RTLIB::Libcall Call, RTLIB::LibcallImpl Impl) {
+    LibcallImpls[Call] = Impl;
   }
 
   /// Get the libcall routine name for the specified libcall.
+  // FIXME: This should be removed. Only LibcallImpl should have a name.
   const char *getLibcallName(RTLIB::Libcall Call) const {
-    return LibcallRoutineNames[Call];
+    return LibCallImplNames[LibcallImpls[Call]];
+  }
+
+  /// Get the libcall routine name for the specified libcall implementation.
+  const char *getLibcallImplName(RTLIB::LibcallImpl CallImpl) const {
+    return LibCallImplNames[CallImpl];
+  }
+
+  /// Return the lowering's selection of implementation call for \p Call
+  RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const {
+    return LibcallImpls[Call];
   }
 
   /// Set the CallingConv that should be used for the specified libcall.
+  // FIXME: This should be a function of RTLIB::LibcallImpl
   void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
     LibcallCallingConvs[Call] = CC;
   }
 
   /// Get the CallingConv that should be used for the specified libcall.
+  // FIXME: This should be a function of RTLIB::LibcallImpl
   CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
     return LibcallCallingConvs[Call];
   }
 
-  ArrayRef<const char *> getLibcallNames() const {
-    // Trim UNKNOWN_LIBCALL from the end
-    return ArrayRef(LibcallRoutineNames).drop_back();
+  ArrayRef<RTLIB::LibcallImpl> getLibcallImpls() const {
+    // Trim Unsupported from the start
+    return ArrayRef(LibcallImpls).drop_front();
   }
 
   /// Get the comparison predicate that's to be used to test the result of the
@@ -101,6 +103,7 @@ struct RuntimeLibcallsInfo {
   }
 
   // FIXME: This should be removed. This should be private constant.
+  // FIXME: This should be a function of RTLIB::LibcallImpl
   void setSoftFloatCmpLibcallPredicate(RTLIB::Libcall Call,
                                        CmpInst::Predicate Pred) {
     SoftFloatCompareLibcallPredicates[Call] = Pred;
@@ -117,8 +120,12 @@ struct RuntimeLibcallsInfo {
   }
 
 private:
-  /// Stores the name each libcall.
-  const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1] = {nullptr};
+  static const RTLIB::LibcallImpl
+      DefaultLibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1];
+
+  /// Stores the implementation choice for each each libcall.
+  RTLIB::LibcallImpl LibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {
+      RTLIB::Unsupported};
 
   static_assert(static_cast<int>(CallingConv::C) == 0,
                 "default calling conv should be encoded as 0");
@@ -134,6 +141,13 @@ struct RuntimeLibcallsInfo {
   // opcode.
   CmpInst::Predicate SoftFloatCompareLibcallPredicates[RTLIB::UNKNOWN_LIBCALL];
 
+  /// Names of concrete implementations of runtime calls. e.g. __ashlsi3 for
+  /// SHL_I32
+  static const char *const LibCallImplNames[RTLIB::NumLibcallImpls];
+
+  /// Map from a concrete LibcallImpl implementation to its RTLIB::Libcall kind.
+  static const RTLIB::Libcall ImplToLibcall[RTLIB::NumLibcallImpls];
+
   static bool darwinHasSinCosStret(const Triple &TT) {
     assert(TT.isOSDarwin() && "should be called with darwin triple");
     // Don't bother with 32 bit x86.
@@ -149,12 +163,25 @@ struct RuntimeLibcallsInfo {
     return true;
   }
 
+  static bool darwinHasExp10(const Triple &TT);
+
   /// Return true if the target has sincosf/sincos/sincosl functions
   static bool hasSinCos(const Triple &TT) {
     return TT.isGNUEnvironment() || TT.isOSFuchsia() ||
            (TT.isAndroid() && !TT.isAndroidVersionLT(9));
   }
 
+  void initDefaultLibCallImpls();
+
+  /// Generated by tablegen.
+  void setPPCLibCallNameOverrides();
+
+  /// Generated by tablegen.
+  void setZOSLibCallNameOverrides();
+
+  /// Generated by tablegen.
+  void setWindowsArm64LibCallNameOverrides();
+
   void initSoftFloatCmpLibcallPredicates();
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
new file mode 100644
index 000000000000..1b54be8dca41
--- /dev/null
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -0,0 +1,1661 @@
+//===-- llvm/RuntimeLibcallList.td - Describe libcalls --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+//--------------------------------------------------------------------
+// Utility classes
+//--------------------------------------------------------------------
+
+class DuplicateLibcallImplWithPrefix<RuntimeLibcallImpl Impl, string prefix>
+    : RuntimeLibcallImpl<Impl.Provides, prefix#Impl.LibCallFuncName>;
+
+//--------------------------------------------------------------------
+// Declare all kinds of used libcalls
+//--------------------------------------------------------------------
+
+// Integer
+foreach IntTy = ["I16", "I32", "I64", "I128"] in {
+  def SHL_#IntTy : RuntimeLibcall;
+  def SRL_#IntTy : RuntimeLibcall;
+  def SRA_#IntTy : RuntimeLibcall;
+}
+
+foreach IntTy = ["I8", "I16", "I32", "I64", "I128"] in {
+  def MUL_#IntTy : RuntimeLibcall;
+}
+
+foreach IntTy = ["I32", "I64", "I128" ] in {
+  def MULO_#IntTy : RuntimeLibcall;
+}
+
+foreach IntTy = ["I8", "I16", "I32", "I64", "I128"] in {
+  def SDIV_#IntTy : RuntimeLibcall;
+  def UDIV_#IntTy : RuntimeLibcall;
+  def SREM_#IntTy : RuntimeLibcall;
+  def UREM_#IntTy : RuntimeLibcall;
+  def SDIVREM_#IntTy : RuntimeLibcall;
+  def UDIVREM_#IntTy : RuntimeLibcall;
+}
+
+foreach IntTy = ["I32", "I64" ] in {
+  def NEG_#IntTy : RuntimeLibcall;
+}
+
+foreach IntTy = ["I32", "I64", "I128"] in {
+  def CTLZ_#IntTy : RuntimeLibcall;
+  def CTPOP_#IntTy : RuntimeLibcall;
+}
+
+foreach FPTy = ["F32", "F64", "F80", "F128", "PPCF128"] in {
+  def ADD_#FPTy : RuntimeLibcall;
+  def SUB_#FPTy : RuntimeLibcall;
+  def MUL_#FPTy : RuntimeLibcall;
+  def DIV_#FPTy : RuntimeLibcall;
+  def REM_#FPTy : RuntimeLibcall;
+  def FMA_#FPTy : RuntimeLibcall;
+  def POWI_#FPTy : RuntimeLibcall;
+  def SQRT_#FPTy : RuntimeLibcall;
+  def CBRT_#FPTy : RuntimeLibcall;
+  def LOG_#FPTy : RuntimeLibcall;
+  def LOG_FINITE_#FPTy : RuntimeLibcall;
+  def LOG2_#FPTy : RuntimeLibcall;
+  def LOG2_FINITE_#FPTy : RuntimeLibcall;
+  def LOG10_#FPTy : RuntimeLibcall;
+  def LOG10_FINITE_#FPTy : RuntimeLibcall;
+  def EXP_#FPTy : RuntimeLibcall;
+  def EXP_FINITE_#FPTy : RuntimeLibcall;
+  def EXP2_#FPTy : RuntimeLibcall;
+  def EXP2_FINITE_#FPTy : RuntimeLibcall;
+  def EXP10_#FPTy : RuntimeLibcall;
+  def EXP10_FINITE_#FPTy : RuntimeLibcall;
+  def SIN_#FPTy : RuntimeLibcall;
+  def COS_#FPTy : RuntimeLibcall;
+  def TAN_#FPTy : RuntimeLibcall;
+  def SINH_#FPTy : RuntimeLibcall;
+  def COSH_#FPTy : RuntimeLibcall;
+  def TANH_#FPTy : RuntimeLibcall;
+  def ASIN_#FPTy : RuntimeLibcall;
+  def ACOS_#FPTy : RuntimeLibcall;
+  def ATAN_#FPTy : RuntimeLibcall;
+  def ATAN2_#FPTy : RuntimeLibcall;
+  def SINCOS_#FPTy : RuntimeLibcall;
+}
+
+foreach FPTy = [ "F32", "F64" ] in {
+  def SINCOS_STRET_#FPTy : RuntimeLibcall;
+}
+
+foreach FPTy = ["F32", "F64", "F80", "F128", "PPCF128"] in {
+  def POW_#FPTy : RuntimeLibcall;
+  def POW_FINITE_#FPTy : RuntimeLibcall;
+  def CEIL_#FPTy : RuntimeLibcall;
+  def TRUNC_#FPTy : RuntimeLibcall;
+  def RINT_#FPTy : RuntimeLibcall;
+  def NEARBYINT_#FPTy : RuntimeLibcall;
+  def ROUND_#FPTy : RuntimeLibcall;
+  def ROUNDEVEN_#FPTy : RuntimeLibcall;
+  def FLOOR_#FPTy : RuntimeLibcall;
+  def COPYSIGN_#FPTy : RuntimeLibcall;
+  def FMIN_#FPTy : RuntimeLibcall;
+  def FMAX_#FPTy : RuntimeLibcall;
+  def FMINIMUM_#FPTy : RuntimeLibcall;
+  def FMAXIMUM_#FPTy : RuntimeLibcall;
+  def FMINIMUM_NUM_#FPTy : RuntimeLibcall;
+  def FMAXIMUM_NUM_#FPTy : RuntimeLibcall;
+  def LROUND_#FPTy : RuntimeLibcall;
+  def LLROUND_#FPTy : RuntimeLibcall;
+  def LRINT_#FPTy : RuntimeLibcall;
+  def LLRINT_#FPTy : RuntimeLibcall;
+  def LDEXP_#FPTy : RuntimeLibcall;
+  def FREXP_#FPTy : RuntimeLibcall;
+  def SINCOSPI_#FPTy : RuntimeLibcall;
+  def MODF_#FPTy : RuntimeLibcall;
+}
+
+def FEGETENV : RuntimeLibcall;
+def FESETENV : RuntimeLibcall;
+
+def FEGETMODE : RuntimeLibcall;
+def FESETMODE : RuntimeLibcall;
+
+def FPEXT_BF16_F32 : RuntimeLibcall;
+def FPEXT_F32_PPCF128 : RuntimeLibcall;
+def FPEXT_F64_PPCF128 : RuntimeLibcall;
+def FPEXT_F80_F128 : RuntimeLibcall;
+def FPEXT_F64_F128 : RuntimeLibcall;
+def FPEXT_F32_F128 : RuntimeLibcall;
+def FPEXT_F16_F128 : RuntimeLibcall;
+def FPEXT_F16_F80 : RuntimeLibcall;
+def FPEXT_F32_F64 : RuntimeLibcall;
+def FPEXT_F16_F64 : RuntimeLibcall;
+def FPEXT_F16_F32 : RuntimeLibcall;
+def FPROUND_F32_F16 : RuntimeLibcall;
+def FPROUND_F64_F16 : RuntimeLibcall;
+def FPROUND_F80_F16 : RuntimeLibcall;
+def FPROUND_F128_F16 : RuntimeLibcall;
+def FPROUND_PPCF128_F16 : RuntimeLibcall;
+def FPROUND_F32_BF16 : RuntimeLibcall;
+def FPROUND_F64_BF16 : RuntimeLibcall;
+def FPROUND_F80_BF16 : RuntimeLibcall;
+def FPROUND_F128_BF16 : RuntimeLibcall;
+def FPROUND_F64_F32 : RuntimeLibcall;
+def FPROUND_F80_F32 : RuntimeLibcall;
+def FPROUND_F128_F32 : RuntimeLibcall;
+def FPROUND_PPCF128_F32 : RuntimeLibcall;
+def FPROUND_F80_F64 : RuntimeLibcall;
+def FPROUND_F128_F64 : RuntimeLibcall;
+def FPROUND_PPCF128_F64 : RuntimeLibcall;
+def FPROUND_F128_F80 : RuntimeLibcall;
+def FPTOSINT_F16_I32 : RuntimeLibcall;
+def FPTOSINT_F16_I64 : RuntimeLibcall;
+def FPTOSINT_F16_I128 : RuntimeLibcall;
+def FPTOSINT_F32_I32 : RuntimeLibcall;
+def FPTOSINT_F32_I64 : RuntimeLibcall;
+def FPTOSINT_F32_I128 : RuntimeLibcall;
+def FPTOSINT_F64_I32 : RuntimeLibcall;
+def FPTOSINT_F64_I64 : RuntimeLibcall;
+def FPTOSINT_F64_I128 : RuntimeLibcall;
+def FPTOSINT_F80_I32 : RuntimeLibcall;
+def FPTOSINT_F80_I64 : RuntimeLibcall;
+def FPTOSINT_F80_I128 : RuntimeLibcall;
+def FPTOSINT_F128_I32 : RuntimeLibcall;
+def FPTOSINT_F128_I64 : RuntimeLibcall;
+def FPTOSINT_F128_I128 : RuntimeLibcall;
+def FPTOSINT_PPCF128_I32 : RuntimeLibcall;
+def FPTOSINT_PPCF128_I64 : RuntimeLibcall;
+def FPTOSINT_PPCF128_I128 : RuntimeLibcall;
+def FPTOUINT_F16_I32 : RuntimeLibcall;
+def FPTOUINT_F16_I64 : RuntimeLibcall;
+def FPTOUINT_F16_I128 : RuntimeLibcall;
+def FPTOUINT_F32_I32 : RuntimeLibcall;
+def FPTOUINT_F32_I64 : RuntimeLibcall;
+def FPTOUINT_F32_I128 : RuntimeLibcall;
+def FPTOUINT_F64_I32 : RuntimeLibcall;
+def FPTOUINT_F64_I64 : RuntimeLibcall;
+def FPTOUINT_F64_I128 : RuntimeLibcall;
+def FPTOUINT_F80_I32 : RuntimeLibcall;
+def FPTOUINT_F80_I64 : RuntimeLibcall;
+def FPTOUINT_F80_I128 : RuntimeLibcall;
+def FPTOUINT_F128_I32 : RuntimeLibcall;
+def FPTOUINT_F128_I64 : RuntimeLibcall;
+def FPTOUINT_F128_I128 : RuntimeLibcall;
+def FPTOUINT_PPCF128_I32 : RuntimeLibcall;
+def FPTOUINT_PPCF128_I64 : RuntimeLibcall;
+def FPTOUINT_PPCF128_I128 : RuntimeLibcall;
+def SINTTOFP_I32_F16 : RuntimeLibcall;
+def SINTTOFP_I32_F32 : RuntimeLibcall;
+def SINTTOFP_I32_F64 : RuntimeLibcall;
+def SINTTOFP_I32_F80 : RuntimeLibcall;
+def SINTTOFP_I32_F128 : RuntimeLibcall;
+def SINTTOFP_I32_PPCF128 : RuntimeLibcall;
+def SINTTOFP_I64_BF16 : RuntimeLibcall;
+def SINTTOFP_I64_F16 : RuntimeLibcall;
+def SINTTOFP_I64_F32 : RuntimeLibcall;
+def SINTTOFP_I64_F64 : RuntimeLibcall;
+def SINTTOFP_I64_F80 : RuntimeLibcall;
+def SINTTOFP_I64_F128 : RuntimeLibcall;
+def SINTTOFP_I64_PPCF128 : RuntimeLibcall;
+def SINTTOFP_I128_F16 : RuntimeLibcall;
+def SINTTOFP_I128_F32 : RuntimeLibcall;
+def SINTTOFP_I128_F64 : RuntimeLibcall;
+def SINTTOFP_I128_F80 : RuntimeLibcall;
+def SINTTOFP_I128_F128 : RuntimeLibcall;
+def SINTTOFP_I128_PPCF128 : RuntimeLibcall;
+def UINTTOFP_I32_F16 : RuntimeLibcall;
+def UINTTOFP_I32_F32 : RuntimeLibcall;
+def UINTTOFP_I32_F64 : RuntimeLibcall;
+def UINTTOFP_I32_F80 : RuntimeLibcall;
+def UINTTOFP_I32_F128 : RuntimeLibcall;
+def UINTTOFP_I32_PPCF128 : RuntimeLibcall;
+def UINTTOFP_I64_BF16 : RuntimeLibcall;
+def UINTTOFP_I64_F16 : RuntimeLibcall;
+def UINTTOFP_I64_F32 : RuntimeLibcall;
+def UINTTOFP_I64_F64 : RuntimeLibcall;
+def UINTTOFP_I64_F80 : RuntimeLibcall;
+def UINTTOFP_I64_F128 : RuntimeLibcall;
+def UINTTOFP_I64_PPCF128 : RuntimeLibcall;
+def UINTTOFP_I128_F16 : RuntimeLibcall;
+def UINTTOFP_I128_F32 : RuntimeLibcall;
+def UINTTOFP_I128_F64 : RuntimeLibcall;
+def UINTTOFP_I128_F80 : RuntimeLibcall;
+def UINTTOFP_I128_F128 : RuntimeLibcall;
+def UINTTOFP_I128_PPCF128 : RuntimeLibcall;
+def CONVERT_F128_PPCF128 : RuntimeLibcall;
+def CONVERT_PPCF128_F128 : RuntimeLibcall;
+
+// Comparisons
+foreach FPTy = ["F32", "F64", "F128", "PPCF128"] in {
+  def OEQ_#FPTy : RuntimeLibcall;
+  def UNE_#FPTy : RuntimeLibcall;
+  def OGE_#FPTy : RuntimeLibcall;
+  def OLT_#FPTy : RuntimeLibcall;
+  def OLE_#FPTy : RuntimeLibcall;
+  def OGT_#FPTy : RuntimeLibcall;
+  def UO_#FPTy : RuntimeLibcall;
+}
+
+// Memory
+def MEMCPY : RuntimeLibcall;
+def MEMMOVE : RuntimeLibcall;
+def MEMSET : RuntimeLibcall;
+def CALLOC : RuntimeLibcall;
+def BZERO : RuntimeLibcall;
+
+// Element-wise unordered-atomic memory of different sizes
+foreach MemSize = [1, 2, 4, 8, 16] in {
+  def MEMCPY_ELEMENT_UNORDERED_ATOMIC_#MemSize : RuntimeLibcall;
+  def MEMMOVE_ELEMENT_UNORDERED_ATOMIC_#MemSize : RuntimeLibcall;
+  def MEMSET_ELEMENT_UNORDERED_ATOMIC_#MemSize : RuntimeLibcall;
+}
+
+// Exception handling
+def UNWIND_RESUME : RuntimeLibcall;
+def CXA_END_CLEANUP : RuntimeLibcall;
+
+// Note: there are two sets of atomics libcalls; see
+// <https://llvm.org/docs/Atomics.html> for more info on the
+// difference between them.
+
+// Atomic '__sync_*' libcalls.
+defset list<RuntimeLibcall> LibCalls__sync = {
+  foreach MemSize = [1, 2, 4, 8, 16] in {
+    def SYNC_VAL_COMPARE_AND_SWAP_#MemSize : RuntimeLibcall;
+    def SYNC_LOCK_TEST_AND_SET_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_ADD_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_SUB_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_AND_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_OR_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_XOR_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_NAND_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_MAX_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_UMAX_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_MIN_#MemSize : RuntimeLibcall;
+    def SYNC_FETCH_AND_UMIN_#MemSize : RuntimeLibcall;
+  }
+}
+
+defset list<RuntimeLibcall> LibCalls__atomic = {
+  // Atomic `__atomic_*' libcalls.
+  foreach MemSize = ["", "_1", "_2", "_4", "_8", "_16"] in {
+    def ATOMIC_LOAD#MemSize : RuntimeLibcall;
+    def ATOMIC_STORE#MemSize : RuntimeLibcall;
+    def ATOMIC_EXCHANGE#MemSize : RuntimeLibcall;
+    def ATOMIC_COMPARE_EXCHANGE#MemSize : RuntimeLibcall;
+  }
+
+  foreach MemSize = [1, 2, 4, 8, 16] in {
+    def ATOMIC_FETCH_ADD_#MemSize : RuntimeLibcall;
+    def ATOMIC_FETCH_SUB_#MemSize : RuntimeLibcall;
+    def ATOMIC_FETCH_AND_#MemSize : RuntimeLibcall;
+    def ATOMIC_FETCH_OR_#MemSize : RuntimeLibcall;
+    def ATOMIC_FETCH_XOR_#MemSize : RuntimeLibcall;
+    def ATOMIC_FETCH_NAND_#MemSize : RuntimeLibcall;
+  }
+}
+
+multiclass AtomicOrderSizeLibcall {
+  def _RELAX : RuntimeLibcall;
+  def _ACQ : RuntimeLibcall;
+  def _REL : RuntimeLibcall;
+  def _ACQ_REL : RuntimeLibcall;
+}
+
+multiclass AtomicOrderSizeLibcallImpl<string ProvidesBase> {
+  def _relax : RuntimeLibcallImpl<!cast<RuntimeLibcall>(ProvidesBase#"_RELAX")>;
+  def _acq : RuntimeLibcallImpl<!cast<RuntimeLibcall>(ProvidesBase#"_ACQ")>;
+  def _rel : RuntimeLibcallImpl<!cast<RuntimeLibcall>(ProvidesBase#"_REL")>;
+  def _acq_rel : RuntimeLibcallImpl<!cast<RuntimeLibcall>(ProvidesBase#"_ACQ_REL")>;
+}
+
+// Out-of-line atomics libcalls
+defset list<RuntimeLibcall> LibCalls__OutOfLineAtomic = {
+  foreach MemSize = [1, 2, 4, 8, 16] in {
+    defm OUTLINE_ATOMIC_CAS#MemSize : AtomicOrderSizeLibcall;
+    defm OUTLINE_ATOMIC_SWP#MemSize : AtomicOrderSizeLibcall;
+    defm OUTLINE_ATOMIC_LDADD#MemSize : AtomicOrderSizeLibcall;
+    defm OUTLINE_ATOMIC_LDSET#MemSize : AtomicOrderSizeLibcall;
+    defm OUTLINE_ATOMIC_LDCLR#MemSize : AtomicOrderSizeLibcall;
+    defm OUTLINE_ATOMIC_LDEOR#MemSize : AtomicOrderSizeLibcall;
+  }
+}
+
+// Stack Protector Fail
+def STACKPROTECTOR_CHECK_FAIL : RuntimeLibcall;
+
+// Deoptimization
+def DEOPTIMIZE : RuntimeLibcall;
+
+// Return address
+def RETURN_ADDRESS : RuntimeLibcall;
+
+// Clear cache
+def CLEAR_CACHE : RuntimeLibcall;
+def RISCV_FLUSH_ICACHE : RuntimeLibcall;
+
+// Mips16 calls
+def MIPS16_RET_DC : RuntimeLibcall;
+def MIPS16_RET_DF : RuntimeLibcall;
+def MIPS16_RET_SC : RuntimeLibcall;
+def MIPS16_RET_SF : RuntimeLibcall;
+
+multiclass LibmLongDoubleLibCall<string libcall_basename = !toupper(NAME),
+                                 string rtbasename = NAME> {
+  def NAME#"_f128"
+      : RuntimeLibcallImpl<!cast<RuntimeLibcall>(libcall_basename#"_F128"),
+                           !strconcat(rtbasename, "l")>;
+  def NAME#"_ppcf128"
+      : RuntimeLibcallImpl<!cast<RuntimeLibcall>(libcall_basename#"_PPCF128"),
+                           !strconcat(rtbasename, "l")>;
+  def NAME#"_f80"
+      : RuntimeLibcallImpl<!cast<RuntimeLibcall>(libcall_basename#"_F80"),
+                           !strconcat(rtbasename, "l")>;
+}
+
+// AArch64 calls
+def SC_MEMCPY : RuntimeLibcall;
+def SC_MEMMOVE : RuntimeLibcall;
+def SC_MEMSET : RuntimeLibcall;
+
+// ARM EABI calls
+def AEABI_MEMCPY4 : RuntimeLibcall; // Align 4
+def AEABI_MEMCPY8 : RuntimeLibcall; // Align 8
+def AEABI_MEMMOVE4 : RuntimeLibcall;
+def AEABI_MEMMOVE8 : RuntimeLibcall;
+def AEABI_MEMSET4 : RuntimeLibcall;
+def AEABI_MEMSET8 : RuntimeLibcall;
+def AEABI_MEMCLR : RuntimeLibcall;
+def AEABI_MEMCLR4 : RuntimeLibcall;
+def AEABI_MEMCLR8 : RuntimeLibcall;
+
+// Hexagon calls
+def HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES : RuntimeLibcall;
+
+// XCore calls
+def MEMCPY_ALIGN_4 : RuntimeLibcall;
+
+//--------------------------------------------------------------------
+// Define implementation default libcalls
+//--------------------------------------------------------------------
+
+defset list<RuntimeLibcallImpl> DefaultRuntimeLibcallImpls = {
+let IsDefault = true in {
+
+//--------------------------------------------------------------------
+// compiler-rt / libgcc
+//--------------------------------------------------------------------
+
+def __ashlhi3 : RuntimeLibcallImpl<SHL_I16>;
+def __ashlsi3 : RuntimeLibcallImpl<SHL_I32>;
+def __ashldi3 : RuntimeLibcallImpl<SHL_I64>;
+def __ashlti3 : RuntimeLibcallImpl<SHL_I128>;
+
+def __lshrhi3 : RuntimeLibcallImpl<SRL_I16>;
+def __lshrsi3 : RuntimeLibcallImpl<SRL_I32>;
+def __lshrdi3 : RuntimeLibcallImpl<SRL_I64>;
+def __lshrti3 : RuntimeLibcallImpl<SRL_I128>;
+
+def __ashrhi3 : RuntimeLibcallImpl<SRA_I16>;
+def __ashrsi3 : RuntimeLibcallImpl<SRA_I32>;
+def __ashrdi3 : RuntimeLibcallImpl<SRA_I64>;
+def __ashrti3 : RuntimeLibcallImpl<SRA_I128>;
+
+def __mulqi3 : RuntimeLibcallImpl<MUL_I8>;
+def __mulhi3 : RuntimeLibcallImpl<MUL_I16>;
+def __mulsi3 : RuntimeLibcallImpl<MUL_I32>;
+def __muldi3 : RuntimeLibcallImpl<MUL_I64>;
+def __multi3 : RuntimeLibcallImpl<MUL_I128>;
+
+def __mulosi4 : RuntimeLibcallImpl<MULO_I32>;
+def __mulodi4 : RuntimeLibcallImpl<MULO_I64>;
+def __muloti4 : RuntimeLibcallImpl<MULO_I128>;
+
+def __divqi3 : RuntimeLibcallImpl<SDIV_I8>;
+def __divhi3 : RuntimeLibcallImpl<SDIV_I16>;
+def __divsi3 : RuntimeLibcallImpl<SDIV_I32>;
+def __divdi3 : RuntimeLibcallImpl<SDIV_I64>;
+def __divti3 : RuntimeLibcallImpl<SDIV_I128>;
+
+def __udivqi3 : RuntimeLibcallImpl<UDIV_I8>;
+def __udivhi3 : RuntimeLibcallImpl<UDIV_I16>;
+def __udivsi3 : RuntimeLibcallImpl<UDIV_I32>;
+def __udivdi3 : RuntimeLibcallImpl<UDIV_I64>;
+def __udivti3 : RuntimeLibcallImpl<UDIV_I128>;
+
+def __modqi3 : RuntimeLibcallImpl<SREM_I8>;
+def __modhi3 : RuntimeLibcallImpl<SREM_I16>;
+def __modsi3 : RuntimeLibcallImpl<SREM_I32>;
+def __moddi3 : RuntimeLibcallImpl<SREM_I64>;
+def __modti3 : RuntimeLibcallImpl<SREM_I128>;
+
+def __umodqi3 : RuntimeLibcallImpl<UREM_I8>;
+def __umodhi3 : RuntimeLibcallImpl<UREM_I16>;
+def __umodsi3 : RuntimeLibcallImpl<UREM_I32>;
+def __umoddi3 : RuntimeLibcallImpl<UREM_I64>;
+def __umodti3 : RuntimeLibcallImpl<UREM_I128>;
+
+def __negsi2 : RuntimeLibcallImpl<NEG_I32>;
+def __negdi2 : RuntimeLibcallImpl<NEG_I64>;
+
+def __clzsi2 : RuntimeLibcallImpl<CTLZ_I32>;
+def __clzdi2 : RuntimeLibcallImpl<CTLZ_I64>;
+def __clzti2 : RuntimeLibcallImpl<CTLZ_I128>;
+
+def __popcountsi2 : RuntimeLibcallImpl<CTPOP_I32>;
+def __popcountdi2 : RuntimeLibcallImpl<CTPOP_I64>;
+def __popcountti2 : RuntimeLibcallImpl<CTPOP_I128>;
+
+def __addsf3 : RuntimeLibcallImpl<ADD_F32>;
+def __adddf3 : RuntimeLibcallImpl<ADD_F64>;
+def __addxf3 : RuntimeLibcallImpl<ADD_F80>;
+def __addtf3 : RuntimeLibcallImpl<ADD_F128>;
+def __gcc_qadd : RuntimeLibcallImpl<ADD_PPCF128>;
+
+def __subsf3 : RuntimeLibcallImpl<SUB_F32>;
+def __subdf3 : RuntimeLibcallImpl<SUB_F64>;
+def __subxf3 : RuntimeLibcallImpl<SUB_F80>;
+def __subtf3 : RuntimeLibcallImpl<SUB_F128>;
+def __gcc_qsub : RuntimeLibcallImpl<SUB_PPCF128>;
+
+def __mulsf3 : RuntimeLibcallImpl<MUL_F32>;
+def __muldf3 : RuntimeLibcallImpl<MUL_F64>;
+def __mulxf3 : RuntimeLibcallImpl<MUL_F80>;
+def __multf3 : RuntimeLibcallImpl<MUL_F128>;
+def __gcc_qmul : RuntimeLibcallImpl<MUL_PPCF128>;
+
+def __divsf3 : RuntimeLibcallImpl<DIV_F32>;
+def __divdf3 : RuntimeLibcallImpl<DIV_F64>;
+def __divxf3 : RuntimeLibcallImpl<DIV_F80>;
+def __divtf3 : RuntimeLibcallImpl<DIV_F128>;
+def __gcc_qdiv : RuntimeLibcallImpl<DIV_PPCF128>;
+
+def __powisf2 : RuntimeLibcallImpl<POWI_F32>;
+def __powidf2 : RuntimeLibcallImpl<POWI_F64>;
+def __powixf2 : RuntimeLibcallImpl<POWI_F80>;
+def __powitf2_f128 : RuntimeLibcallImpl<POWI_F128, "__powitf2">;
+def __powitf2_ppc128 : RuntimeLibcallImpl<POWI_PPCF128, "__powitf2">;
+
+// Conversion
+def __extendbfsf2 : RuntimeLibcallImpl<FPEXT_BF16_F32>;
+def __gcc_stoq : RuntimeLibcallImpl<FPEXT_F32_PPCF128>;
+def __gcc_dtoq : RuntimeLibcallImpl<FPEXT_F64_PPCF128>;
+def __extendxftf2 : RuntimeLibcallImpl<FPEXT_F80_F128>;
+def __extenddftf2 : RuntimeLibcallImpl<FPEXT_F64_F128>;
+def __extendsftf2 : RuntimeLibcallImpl<FPEXT_F32_F128>;
+def __extendhftf2 : RuntimeLibcallImpl<FPEXT_F16_F128>;
+def __extendhfxf2 : RuntimeLibcallImpl<FPEXT_F16_F80>;
+def __extendsfdf2 : RuntimeLibcallImpl<FPEXT_F32_F64>;
+def __extendhfdf2 : RuntimeLibcallImpl<FPEXT_F16_F64>;
+def __extendhfsf2 : RuntimeLibcallImpl<FPEXT_F16_F32>;
+def __truncsfhf2 : RuntimeLibcallImpl<FPROUND_F32_F16>;
+def __truncdfhf2 : RuntimeLibcallImpl<FPROUND_F64_F16>;
+def __truncxfhf2 : RuntimeLibcallImpl<FPROUND_F80_F16>;
+def __trunctfhf2_f128 : RuntimeLibcallImpl<FPROUND_F128_F16, "__trunctfhf2">;
+def __trunctfhf2_ppcf128 : RuntimeLibcallImpl<FPROUND_PPCF128_F16, "__trunctfhf2">;
+def __truncsfbf2 : RuntimeLibcallImpl<FPROUND_F32_BF16>;
+def __truncdfbf2 : RuntimeLibcallImpl<FPROUND_F64_BF16>;
+def __truncxfbf2 : RuntimeLibcallImpl<FPROUND_F80_BF16>;
+def __trunctfbf2 : RuntimeLibcallImpl<FPROUND_F128_BF16>;
+def __truncdfsf2 : RuntimeLibcallImpl<FPROUND_F64_F32>;
+def __truncxfsf2 : RuntimeLibcallImpl<FPROUND_F80_F32>;
+def __trunctfsf2 : RuntimeLibcallImpl<FPROUND_F128_F32>;
+def __gcc_qtos : RuntimeLibcallImpl<FPROUND_PPCF128_F32>;
+def __truncxfdf2 : RuntimeLibcallImpl<FPROUND_F80_F64>;
+def __trunctfdf2 : RuntimeLibcallImpl<FPROUND_F128_F64>;
+def __gcc_qtod : RuntimeLibcallImpl<FPROUND_PPCF128_F64>;
+def __trunctfxf2 : RuntimeLibcallImpl<FPROUND_F128_F80>;
+def __fixhfsi : RuntimeLibcallImpl<FPTOSINT_F16_I32>;
+def __fixhfdi : RuntimeLibcallImpl<FPTOSINT_F16_I64>;
+def __fixhfti : RuntimeLibcallImpl<FPTOSINT_F16_I128>;
+def __fixsfsi : RuntimeLibcallImpl<FPTOSINT_F32_I32>;
+def __fixsfdi : RuntimeLibcallImpl<FPTOSINT_F32_I64>;
+def __fixsfti : RuntimeLibcallImpl<FPTOSINT_F32_I128>;
+def __fixdfsi : RuntimeLibcallImpl<FPTOSINT_F64_I32>;
+def __fixdfdi : RuntimeLibcallImpl<FPTOSINT_F64_I64>;
+def __fixdfti : RuntimeLibcallImpl<FPTOSINT_F64_I128>;
+def __fixxfsi : RuntimeLibcallImpl<FPTOSINT_F80_I32>;
+def __fixxfdi : RuntimeLibcallImpl<FPTOSINT_F80_I64>;
+def __fixxfti : RuntimeLibcallImpl<FPTOSINT_F80_I128>;
+def __fixtfsi : RuntimeLibcallImpl<FPTOSINT_F128_I32>;
+def __fixtfdi_f128 : RuntimeLibcallImpl<FPTOSINT_F128_I64, "__fixtfdi">;
+def __fixtfti_f128 : RuntimeLibcallImpl<FPTOSINT_F128_I128, "__fixtfti">;
+def __gcc_qtou : RuntimeLibcallImpl<FPTOSINT_PPCF128_I32>;
+def __fixtfdi_ppcf128 : RuntimeLibcallImpl<FPTOSINT_PPCF128_I64, "__fixtfdi">;
+def __fixtfti_ppcf128 : RuntimeLibcallImpl<FPTOSINT_PPCF128_I128, "__fixtfti">;
+def __fixunshfsi : RuntimeLibcallImpl<FPTOUINT_F16_I32>;
+def __fixunshfdi : RuntimeLibcallImpl<FPTOUINT_F16_I64>;
+def __fixunshfti : RuntimeLibcallImpl<FPTOUINT_F16_I128>;
+def __fixunssfsi : RuntimeLibcallImpl<FPTOUINT_F32_I32>;
+def __fixunssfdi : RuntimeLibcallImpl<FPTOUINT_F32_I64>;
+def __fixunssfti : RuntimeLibcallImpl<FPTOUINT_F32_I128>;
+def __fixunsdfsi : RuntimeLibcallImpl<FPTOUINT_F64_I32>;
+def __fixunsdfdi : RuntimeLibcallImpl<FPTOUINT_F64_I64>;
+def __fixunsdfti : RuntimeLibcallImpl<FPTOUINT_F64_I128>;
+def __fixunsxfsi : RuntimeLibcallImpl<FPTOUINT_F80_I32>;
+def __fixunsxfdi : RuntimeLibcallImpl<FPTOUINT_F80_I64>;
+def __fixunsxfti : RuntimeLibcallImpl<FPTOUINT_F80_I128>;
+def __fixunstfsi_f128 : RuntimeLibcallImpl<FPTOUINT_F128_I32, "__fixunstfsi">;
+def __fixunstfdi_f128 : RuntimeLibcallImpl<FPTOUINT_F128_I64, "__fixunstfdi">;
+def __fixunstfti_f128 : RuntimeLibcallImpl<FPTOUINT_F128_I128, "__fixunstfti">;
+def __fixunstfsi_ppcf128 : RuntimeLibcallImpl<FPTOUINT_PPCF128_I32, "__fixunstfsi">;
+def __fixunstfdi_ppcf128 : RuntimeLibcallImpl<FPTOUINT_PPCF128_I64, "__fixunstfdi">;
+def __fixunstfti_ppcf128 : RuntimeLibcallImpl<FPTOUINT_PPCF128_I128, "__fixunstfti">;
+def __floatsihf : RuntimeLibcallImpl<SINTTOFP_I32_F16>;
+def __floatsisf : RuntimeLibcallImpl<SINTTOFP_I32_F32>;
+def __floatsidf : RuntimeLibcallImpl<SINTTOFP_I32_F64>;
+def __floatsixf : RuntimeLibcallImpl<SINTTOFP_I32_F80>;
+def __floatsitf : RuntimeLibcallImpl<SINTTOFP_I32_F128>;
+def __gcc_itoq : RuntimeLibcallImpl<SINTTOFP_I32_PPCF128>;
+def __floatdibf : RuntimeLibcallImpl<SINTTOFP_I64_BF16>;
+def __floatdihf : RuntimeLibcallImpl<SINTTOFP_I64_F16>;
+def __floatdisf : RuntimeLibcallImpl<SINTTOFP_I64_F32>;
+def __floatdidf : RuntimeLibcallImpl<SINTTOFP_I64_F64>;
+def __floatdixf : RuntimeLibcallImpl<SINTTOFP_I64_F80>;
+def __floatditf_f128 : RuntimeLibcallImpl<SINTTOFP_I64_F128, "__floatditf">;
+def __floatditf_ppcf128 : RuntimeLibcallImpl<SINTTOFP_I64_PPCF128, "__floatditf">;
+def __floattihf : RuntimeLibcallImpl<SINTTOFP_I128_F16>;
+def __floattisf : RuntimeLibcallImpl<SINTTOFP_I128_F32>;
+def __floattidf : RuntimeLibcallImpl<SINTTOFP_I128_F64>;
+def __floattixf : RuntimeLibcallImpl<SINTTOFP_I128_F80>;
+def __floattitf_f128 : RuntimeLibcallImpl<SINTTOFP_I128_F128, "__floattitf">;
+def __floattitf_ppcf128 : RuntimeLibcallImpl<SINTTOFP_I128_PPCF128, "__floattitf">;
+def __floatunsihf : RuntimeLibcallImpl<UINTTOFP_I32_F16>;
+def __floatunsisf : RuntimeLibcallImpl<UINTTOFP_I32_F32>;
+def __floatunsidf : RuntimeLibcallImpl<UINTTOFP_I32_F64>;
+def __floatunsixf : RuntimeLibcallImpl<UINTTOFP_I32_F80>;
+def __floatunsitf : RuntimeLibcallImpl<UINTTOFP_I32_F128>;
+def __gcc_utoq : RuntimeLibcallImpl<UINTTOFP_I32_PPCF128>;
+def __floatundibf : RuntimeLibcallImpl<UINTTOFP_I64_BF16>;
+def __floatundihf : RuntimeLibcallImpl<UINTTOFP_I64_F16>;
+def __floatundisf : RuntimeLibcallImpl<UINTTOFP_I64_F32>;
+def __floatundidf : RuntimeLibcallImpl<UINTTOFP_I64_F64>;
+def __floatundixf : RuntimeLibcallImpl<UINTTOFP_I64_F80>;
+def __floatunditf_f128 : RuntimeLibcallImpl<UINTTOFP_I64_F128, "__floatunditf">;
+def __floatunditf_ppcf128 : RuntimeLibcallImpl<UINTTOFP_I64_PPCF128, "__floatunditf">;
+def __floatuntihf : RuntimeLibcallImpl<UINTTOFP_I128_F16>;
+def __floatuntisf : RuntimeLibcallImpl<UINTTOFP_I128_F32>;
+def __floatuntidf : RuntimeLibcallImpl<UINTTOFP_I128_F64>;
+def __floatuntixf : RuntimeLibcallImpl<UINTTOFP_I128_F80>;
+def __floatuntitf_f128 : RuntimeLibcallImpl<UINTTOFP_I128_F128, "__floatuntitf">;
+def __floatuntitf_ppcf128 : RuntimeLibcallImpl<UINTTOFP_I128_PPCF128, "__floatuntitf">;
+def __extendkftf2 : RuntimeLibcallImpl<CONVERT_F128_PPCF128>;
+def __trunctfkf2 : RuntimeLibcallImpl<CONVERT_PPCF128_F128>;
+
+// Comparison
+def __eqsf2 : RuntimeLibcallImpl<OEQ_F32>;
+def __eqdf2 : RuntimeLibcallImpl<OEQ_F64>;
+def __eqtf2 : RuntimeLibcallImpl<OEQ_F128>;
+def __gcc_qeq : RuntimeLibcallImpl<OEQ_PPCF128>;
+def __nesf2 : RuntimeLibcallImpl<UNE_F32>;
+def __nedf2 : RuntimeLibcallImpl<UNE_F64>;
+def __netf2 : RuntimeLibcallImpl<UNE_F128>;
+def __gcc_qne : RuntimeLibcallImpl<UNE_PPCF128>;
+def __gesf2 : RuntimeLibcallImpl<OGE_F32>;
+def __gedf2 : RuntimeLibcallImpl<OGE_F64>;
+def __getf2 : RuntimeLibcallImpl<OGE_F128>;
+def __gcc_qge : RuntimeLibcallImpl<OGE_PPCF128>;
+def __ltsf2 : RuntimeLibcallImpl<OLT_F32>;
+def __ltdf2 : RuntimeLibcallImpl<OLT_F64>;
+def __lttf2 : RuntimeLibcallImpl<OLT_F128>;
+def __gcc_qlt : RuntimeLibcallImpl<OLT_PPCF128>;
+def __lesf2 : RuntimeLibcallImpl<OLE_F32>;
+def __ledf2 : RuntimeLibcallImpl<OLE_F64>;
+def __letf2 : RuntimeLibcallImpl<OLE_F128>;
+def __gcc_qle : RuntimeLibcallImpl<OLE_PPCF128>;
+def __gtsf2 : RuntimeLibcallImpl<OGT_F32>;
+def __gtdf2 : RuntimeLibcallImpl<OGT_F64>;
+def __gttf2 : RuntimeLibcallImpl<OGT_F128>;
+def __gcc_qgt : RuntimeLibcallImpl<OGT_PPCF128>;
+def __unordsf2 : RuntimeLibcallImpl<UO_F32>;
+def __unorddf2 : RuntimeLibcallImpl<UO_F64>;
+def __unordtf2 : RuntimeLibcallImpl<UO_F128>;
+def __gcc_qunord : RuntimeLibcallImpl<UO_PPCF128>;
+
+// Element-wise unordered-atomic memory of different sizes
+foreach MemSize = [ 1, 2, 4, 8, 16 ] in {
+  def __llvm_memcpy_element_unordered_atomic_#MemSize : RuntimeLibcallImpl<
+    !cast<RuntimeLibcall>("MEMCPY_ELEMENT_UNORDERED_ATOMIC_"#MemSize)>;
+
+  def __llvm_memmove_element_unordered_atomic_#MemSize : RuntimeLibcallImpl<
+    !cast<RuntimeLibcall>("MEMMOVE_ELEMENT_UNORDERED_ATOMIC_"#MemSize)>;
+
+  def __llvm_memset_element_unordered_atomic_#MemSize : RuntimeLibcallImpl<
+    !cast<RuntimeLibcall>("MEMSET_ELEMENT_UNORDERED_ATOMIC_"#MemSize)>;
+}
+
+// Exception handling
+def _Unwind_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
+def __cxa_end_cleanup : RuntimeLibcallImpl<CXA_END_CLEANUP>;
+
+// Atomic '__sync_*' libcalls.
+foreach lc = LibCalls__sync in {
+  def __#!tolower(!cast<string>(lc)) : RuntimeLibcallImpl<lc>;
+}
+
+// Atomic `__atomic_*' libcalls.
+foreach lc = LibCalls__atomic in {
+  def __#!tolower(!cast<string>(lc)) : RuntimeLibcallImpl<lc>;
+}
+
+// Stack Protector Fail
+def __stack_chk_fail : RuntimeLibcallImpl<STACKPROTECTOR_CHECK_FAIL>;
+
+// Deoptimization
+def __llvm_deoptimize : RuntimeLibcallImpl<DEOPTIMIZE>;
+
+// Clear cache
+def __clear_cache : RuntimeLibcallImpl<CLEAR_CACHE>;
+def __riscv_flush_icache : RuntimeLibcallImpl<RISCV_FLUSH_ICACHE>;
+
+//--------------------------------------------------------------------
+// libm
+//--------------------------------------------------------------------
+
+def fmodf : RuntimeLibcallImpl<REM_F32>;
+def fmod : RuntimeLibcallImpl<REM_F64>;
+def fmodl_f128 : RuntimeLibcallImpl<REM_F128, "fmodl">;
+def fmodl_f80 : RuntimeLibcallImpl<REM_F80, "fmodl">;
+def fmodl_ppc128 : RuntimeLibcallImpl<REM_PPCF128, "fmodl">;
+
+def fmaf : RuntimeLibcallImpl<FMA_F32>;
+def fma : RuntimeLibcallImpl<FMA_F64>;
+defm fma : LibmLongDoubleLibCall;
+
+def sqrtf : RuntimeLibcallImpl<SQRT_F32>;
+def sqrt : RuntimeLibcallImpl<SQRT_F64>;
+defm sqrt : LibmLongDoubleLibCall;
+
+def cbrtf : RuntimeLibcallImpl<CBRT_F32>;
+def cbrt : RuntimeLibcallImpl<CBRT_F64>;
+defm cbrt : LibmLongDoubleLibCall;
+
+def logf : RuntimeLibcallImpl<LOG_F32>;
+def log : RuntimeLibcallImpl<LOG_F64>;
+defm log : LibmLongDoubleLibCall;
+
+def __logf_finite : RuntimeLibcallImpl<LOG_FINITE_F32>;
+def __log_finite : RuntimeLibcallImpl<LOG_FINITE_F64>;
+def __logl_finite_f80 : RuntimeLibcallImpl<LOG_FINITE_F80, "__logl_finite">;
+def __logl_finite_f128 : RuntimeLibcallImpl<LOG_FINITE_F128, "__logl_finite">;
+def __logl_finite_ppcf128 : RuntimeLibcallImpl<LOG_FINITE_PPCF128, "__logl_finite">;
+
+def log2f : RuntimeLibcallImpl<LOG2_F32>;
+def log2 : RuntimeLibcallImpl<LOG2_F64>;
+defm log2 : LibmLongDoubleLibCall;
+
+def __log2f_finite : RuntimeLibcallImpl<LOG2_FINITE_F32>;
+def __log2_finite : RuntimeLibcallImpl<LOG2_FINITE_F64>;
+def __log2l_finite_f80 : RuntimeLibcallImpl<LOG2_FINITE_F80, "__log2l_finite">;
+def __log2l_finite_f128 : RuntimeLibcallImpl<LOG2_FINITE_F128, "__log2l_finite">;
+def __log2l_finite_ppcf128 : RuntimeLibcallImpl<LOG2_FINITE_PPCF128, "__log2l_finite">;
+
+def log10f : RuntimeLibcallImpl<LOG10_F32>;
+def log10 : RuntimeLibcallImpl<LOG10_F64>;
+defm log10 : LibmLongDoubleLibCall;
+
+def __log10f_finite : RuntimeLibcallImpl<LOG10_FINITE_F32>;
+def __log10_finite : RuntimeLibcallImpl<LOG10_FINITE_F64>;
+def __log10l_finite_f80 : RuntimeLibcallImpl<LOG10_FINITE_F80, "__log10l_finite">;
+def __log10l_finite_f128 : RuntimeLibcallImpl<LOG10_FINITE_F128, "__log10l_finite">;
+def __log10l_finite_ppcf128 : RuntimeLibcallImpl<LOG10_FINITE_PPCF128, "__log10l_finite">;
+
+def expf : RuntimeLibcallImpl<EXP_F32>;
+def exp : RuntimeLibcallImpl<EXP_F64>;
+defm exp : LibmLongDoubleLibCall<"EXP", "exp">;
+
+def __expf_finite : RuntimeLibcallImpl<EXP_FINITE_F32>;
+def __exp_finite : RuntimeLibcallImpl<EXP_FINITE_F64>;
+def __expl_finite_f80 : RuntimeLibcallImpl<EXP_FINITE_F80, "__expl_finite">;
+def __expl_finite_f128 : RuntimeLibcallImpl<EXP_FINITE_F128, "__expl_finite">;
+def __expl_finite_ppcf128 : RuntimeLibcallImpl<EXP_FINITE_PPCF128, "__expl_finite">;
+
+def exp2f : RuntimeLibcallImpl<EXP2_F32>;
+def exp2 : RuntimeLibcallImpl<EXP2_F64>;
+defm exp2 : LibmLongDoubleLibCall<"EXP2", "exp2">;
+
+def __exp2f_finite : RuntimeLibcallImpl<EXP2_FINITE_F32>;
+def __exp2_finite : RuntimeLibcallImpl<EXP2_FINITE_F64>;
+def __exp2l_finite_f80 : RuntimeLibcallImpl<EXP2_FINITE_F80, "__exp2l_finite">;
+def __exp2l_finite_f128 : RuntimeLibcallImpl<EXP2_FINITE_F128, "__exp2l_finite">;
+def __exp2l_finite_ppcf128 : RuntimeLibcallImpl<EXP2_FINITE_PPCF128, "__exp2l_finite">;
+
+def exp10f : RuntimeLibcallImpl<EXP10_F32>;
+def exp10 : RuntimeLibcallImpl<EXP10_F64>;
+def exp10l_f80 : RuntimeLibcallImpl<EXP10_F80, "exp10l">;
+def exp10l_f128 : RuntimeLibcallImpl<EXP10_F128, "exp10l">;
+def exp10l_ppcf128 : RuntimeLibcallImpl<EXP10_PPCF128, "exp10l">;
+
+def sinf : RuntimeLibcallImpl<SIN_F32>;
+def sin : RuntimeLibcallImpl<SIN_F64>;
+defm sin : LibmLongDoubleLibCall;
+
+def cosf : RuntimeLibcallImpl<COS_F32>;
+def cos : RuntimeLibcallImpl<COS_F64>;
+defm cos : LibmLongDoubleLibCall;
+
+def tanf : RuntimeLibcallImpl<TAN_F32>;
+def tan : RuntimeLibcallImpl<TAN_F64>;
+defm tan : LibmLongDoubleLibCall;
+
+def sinhf : RuntimeLibcallImpl<SINH_F32>;
+def sinh : RuntimeLibcallImpl<SINH_F64>;
+defm sinh : LibmLongDoubleLibCall;
+
+def coshf : RuntimeLibcallImpl<COSH_F32>;
+def cosh : RuntimeLibcallImpl<COSH_F64>;
+defm cosh : LibmLongDoubleLibCall;
+
+def tanhf : RuntimeLibcallImpl<TANH_F32>;
+def tanh : RuntimeLibcallImpl<TANH_F64>;
+defm tanh : LibmLongDoubleLibCall;
+
+def asinf : RuntimeLibcallImpl<ASIN_F32>;
+def asin : RuntimeLibcallImpl<ASIN_F64>;
+defm asin : LibmLongDoubleLibCall;
+
+def acosf : RuntimeLibcallImpl<ACOS_F32>;
+def acos : RuntimeLibcallImpl<ACOS_F64>;
+defm acos : LibmLongDoubleLibCall;
+
+def atanf : RuntimeLibcallImpl<ATAN_F32>;
+def atan : RuntimeLibcallImpl<ATAN_F64>;
+defm atan : LibmLongDoubleLibCall;
+
+def atan2f : RuntimeLibcallImpl<ATAN2_F32>;
+def atan2 : RuntimeLibcallImpl<ATAN2_F64>;
+defm atan2 : LibmLongDoubleLibCall;
+
+def powf : RuntimeLibcallImpl<POW_F32>;
+def pow : RuntimeLibcallImpl<POW_F64>;
+defm pow : LibmLongDoubleLibCall;
+
+def __powf_finite : RuntimeLibcallImpl<POW_FINITE_F32>;
+def __pow_finite : RuntimeLibcallImpl<POW_FINITE_F64>;
+def __powl_finite_f80 : RuntimeLibcallImpl<POW_FINITE_F80, "__powl_finite">;
+def __powl_finite_f128 : RuntimeLibcallImpl<POW_FINITE_F128, "__powl_finite">;
+def __powl_finite_ppcf128 : RuntimeLibcallImpl<POW_FINITE_PPCF128, "__powl_finite">;
+
+def ceilf : RuntimeLibcallImpl<CEIL_F32>;
+def ceil : RuntimeLibcallImpl<CEIL_F64>;
+defm ceil : LibmLongDoubleLibCall;
+
+def truncf : RuntimeLibcallImpl<TRUNC_F32>;
+def trunc : RuntimeLibcallImpl<TRUNC_F64>;
+defm trunc : LibmLongDoubleLibCall;
+
+def rintf : RuntimeLibcallImpl<RINT_F32>;
+def rint : RuntimeLibcallImpl<RINT_F64>;
+defm rint : LibmLongDoubleLibCall;
+
+def nearbyintf : RuntimeLibcallImpl<NEARBYINT_F32>;
+def nearbyint : RuntimeLibcallImpl<NEARBYINT_F64>;
+defm nearbyint : LibmLongDoubleLibCall;
+
+def roundf : RuntimeLibcallImpl<ROUND_F32>;
+def round : RuntimeLibcallImpl<ROUND_F64>;
+defm round : LibmLongDoubleLibCall;
+
+def roundevenf : RuntimeLibcallImpl<ROUNDEVEN_F32>;
+def roundeven : RuntimeLibcallImpl<ROUNDEVEN_F64>;
+defm roundeven : LibmLongDoubleLibCall;
+
+def floorf : RuntimeLibcallImpl<FLOOR_F32>;
+def floor : RuntimeLibcallImpl<FLOOR_F64>;
+defm floor : LibmLongDoubleLibCall;
+
+def copysignf : RuntimeLibcallImpl<COPYSIGN_F32>;
+def copysign : RuntimeLibcallImpl<COPYSIGN_F64>;
+defm copysign : LibmLongDoubleLibCall;
+
+def fminf : RuntimeLibcallImpl<FMIN_F32>;
+def fmin : RuntimeLibcallImpl<FMIN_F64>;
+defm fmin : LibmLongDoubleLibCall;
+
+def fmaxf : RuntimeLibcallImpl<FMAX_F32>;
+def fmax : RuntimeLibcallImpl<FMAX_F64>;
+defm fmax : LibmLongDoubleLibCall;
+
+def fminimumf : RuntimeLibcallImpl<FMINIMUM_F32>;
+def fminimum : RuntimeLibcallImpl<FMINIMUM_F64>;
+defm fminimum : LibmLongDoubleLibCall;
+
+def fmaximumf : RuntimeLibcallImpl<FMAXIMUM_F32>;
+def fmaximum : RuntimeLibcallImpl<FMAXIMUM_F64>;
+defm fmaximum : LibmLongDoubleLibCall;
+
+def fminimum_numf : RuntimeLibcallImpl<FMINIMUM_NUM_F32>;
+def fminimum_num : RuntimeLibcallImpl<FMINIMUM_NUM_F64>;
+defm fminimum_num : LibmLongDoubleLibCall;
+
+def fmaximum_numf : RuntimeLibcallImpl<FMAXIMUM_NUM_F32>;
+def fmaximum_num : RuntimeLibcallImpl<FMAXIMUM_NUM_F64>;
+defm fmaximum_num : LibmLongDoubleLibCall;
+
+def lroundf : RuntimeLibcallImpl<LROUND_F32>;
+def lround : RuntimeLibcallImpl<LROUND_F64>;
+defm lround : LibmLongDoubleLibCall;
+
+def llroundf : RuntimeLibcallImpl<LLROUND_F32>;
+def llround : RuntimeLibcallImpl<LLROUND_F64>;
+defm llround : LibmLongDoubleLibCall;
+
+def lrintf : RuntimeLibcallImpl<LRINT_F32>;
+def lrint : RuntimeLibcallImpl<LRINT_F64>;
+defm lrint : LibmLongDoubleLibCall;
+
+def llrintf : RuntimeLibcallImpl<LLRINT_F32>;
+def llrint : RuntimeLibcallImpl<LLRINT_F64>;
+defm llrint : LibmLongDoubleLibCall;
+
+def ldexpf : RuntimeLibcallImpl<LDEXP_F32>;
+def ldexp : RuntimeLibcallImpl<LDEXP_F64>;
+defm ldexp : LibmLongDoubleLibCall;
+
+def frexpf : RuntimeLibcallImpl<FREXP_F32>;
+def frexp : RuntimeLibcallImpl<FREXP_F64>;
+defm frexp : LibmLongDoubleLibCall;
+
+def sincospif : RuntimeLibcallImpl<SINCOSPI_F32>;
+def sincospi : RuntimeLibcallImpl<SINCOSPI_F64>;
+defm sincospi : LibmLongDoubleLibCall;
+
+def modff : RuntimeLibcallImpl<MODF_F32>;
+def modf : RuntimeLibcallImpl<MODF_F64>;
+defm modf : LibmLongDoubleLibCall;
+
+// Floating point environment
+def fegetenv : RuntimeLibcallImpl<FEGETENV>;
+def fesetenv : RuntimeLibcallImpl<FESETENV>;
+
+// Floating point control modes
+def fegetmode : RuntimeLibcallImpl<FEGETMODE>;
+def fesetmode : RuntimeLibcallImpl<FESETMODE>;
+
+//--------------------------------------------------------------------
+// libc
+//--------------------------------------------------------------------
+
+// Memory
+def memcpy : RuntimeLibcallImpl<MEMCPY>;
+def memmove : RuntimeLibcallImpl<MEMMOVE>;
+def memset : RuntimeLibcallImpl<MEMSET>;
+
+// DSEPass can emit calloc if it finds a pair of malloc/memset
+def calloc : RuntimeLibcallImpl<CALLOC>;
+
+} // End let IsDefault = true
+} // End defset DefaultRuntimeLibcallImpls
+
+//--------------------------------------------------------------------
+// Define implementation other libcalls
+//--------------------------------------------------------------------
+
+// TODO: Define other custom names targets use to override
+
+def __exp10f : RuntimeLibcallImpl<EXP10_F32>;
+def __exp10 : RuntimeLibcallImpl<EXP10_F64>;
+
+def __sincosf_stret : RuntimeLibcallImpl<SINCOS_STRET_F32>;
+def __sincos_stret : RuntimeLibcallImpl<SINCOS_STRET_F64>;
+
+def sincosf : RuntimeLibcallImpl<SINCOS_F32>;
+def sincos : RuntimeLibcallImpl<SINCOS_F64>;
+defm sincos : LibmLongDoubleLibCall;
+
+def bzero : RuntimeLibcallImpl<BZERO>;
+def __bzero : RuntimeLibcallImpl<BZERO>;
+def _Unwind_SjLj_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
+
+//===----------------------------------------------------------------------===//
+// F128 libm Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+def logf128 : RuntimeLibcallImpl<LOG_F128>;
+def log2f128 : RuntimeLibcallImpl<LOG2_F128>;
+def log10f128 : RuntimeLibcallImpl<LOG10_F128>;
+def expf128 : RuntimeLibcallImpl<EXP_F128>;
+def exp2f128 : RuntimeLibcallImpl<EXP2_F128>;
+def exp10f128 : RuntimeLibcallImpl<EXP10_F128>;
+def sinf128 : RuntimeLibcallImpl<SIN_F128>;
+def cosf128 : RuntimeLibcallImpl<COS_F128>;
+def tanf128 : RuntimeLibcallImpl<TAN_F128>;
+def tanhf128 : RuntimeLibcallImpl<TANH_F128>;
+def sincosf128 : RuntimeLibcallImpl<SINCOS_F128>;
+def powf128 : RuntimeLibcallImpl<POW_F128>;
+def fminf128 : RuntimeLibcallImpl<FMIN_F128>;
+def fmaxf128 : RuntimeLibcallImpl<FMAX_F128>;
+def fmodf128 : RuntimeLibcallImpl<REM_F128>;
+def sqrtf128 : RuntimeLibcallImpl<SQRT_F128>;
+def ceilf128 : RuntimeLibcallImpl<CEIL_F128>;
+def floorf128 : RuntimeLibcallImpl<FLOOR_F128>;
+def truncf128 : RuntimeLibcallImpl<TRUNC_F128>;
+def roundf128 : RuntimeLibcallImpl<ROUND_F128>;
+def lroundf128 : RuntimeLibcallImpl<LROUND_F128>;
+def llroundf128 : RuntimeLibcallImpl<LLROUND_F128>;
+def rintf128 : RuntimeLibcallImpl<RINT_F128>;
+def lrintf128 : RuntimeLibcallImpl<LRINT_F128>;
+def llrintf128 : RuntimeLibcallImpl<LLRINT_F128>;
+def nearbyintf128 : RuntimeLibcallImpl<NEARBYINT_F128>;
+def fmaf128 : RuntimeLibcallImpl<FMA_F128>;
+def frexpf128 : RuntimeLibcallImpl<FREXP_F128>;
+
+def cbrtf128 : RuntimeLibcallImpl<CBRT_F128>;
+def fminimumf128 : RuntimeLibcallImpl<FMINIMUM_F128>;
+def fmaximumf128 : RuntimeLibcallImpl<FMAXIMUM_F128>;
+def fminimum_numf128 : RuntimeLibcallImpl<FMINIMUM_NUM_F128>;
+def fmaximum_numf128 : RuntimeLibcallImpl<FMAXIMUM_NUM_F128>;
+def asinf128 : RuntimeLibcallImpl<ASIN_F128>;
+def acosf128 : RuntimeLibcallImpl<ACOS_F128>;
+def atanf128 : RuntimeLibcallImpl<ATAN_F128>;
+def atan2f128 : RuntimeLibcallImpl<ATAN2_F128>;
+def ldexpf128 : RuntimeLibcallImpl<LDEXP_F128>;
+def roundevenf128 : RuntimeLibcallImpl<ROUNDEVEN_F128>;
+def modff128 : RuntimeLibcallImpl<MODF_F128>;
+def sinhf128 : RuntimeLibcallImpl<SINH_F128>;
+def coshf128 : RuntimeLibcallImpl<COSH_F128>;
+def copysignf128 : RuntimeLibcallImpl<COPYSIGN_F128>;
+
+def __logf128_finite : RuntimeLibcallImpl<LOG_FINITE_F128>;
+def __log2f128_finite : RuntimeLibcallImpl<LOG2_FINITE_F128>;
+def __log10f128_finite : RuntimeLibcallImpl<LOG10_FINITE_F128>;
+def __expf128_finite : RuntimeLibcallImpl<EXP_FINITE_F128>;
+def __exp2f128_finite : RuntimeLibcallImpl<EXP2_FINITE_F128>;
+def __exp10f128_finite : RuntimeLibcallImpl<EXP10_FINITE_F128>;
+def __powf128_finite : RuntimeLibcallImpl<POW_FINITE_F128>;
+
+//===----------------------------------------------------------------------===//
+// AArch64 Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+defset list<RuntimeLibcallImpl> AArch64LibcallImpls = {
+  foreach MemSize = [1, 2, 4, 8, 16] in {
+    defm __aarch64_cas#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_CAS"#MemSize>;
+  }
+
+  foreach MemSize = [1, 2, 4, 8] in {
+    defm __aarch64_swp#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_SWP"#MemSize>;
+    defm __aarch64_ldadd#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDADD"#MemSize>;
+    defm __aarch64_ldset#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDSET"#MemSize>;
+    defm __aarch64_ldclr#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDCLR"#MemSize>;
+    defm __aarch64_ldeor#MemSize
+        : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDEOR"#MemSize>;
+  }
+
+  def __arm_sc_memcpy : RuntimeLibcallImpl<SC_MEMCPY>;
+  def __arm_sc_memmove : RuntimeLibcallImpl<SC_MEMMOVE>;
+  def __arm_sc_memset : RuntimeLibcallImpl<SC_MEMSET>;
+}
+
+foreach libcall = AArch64LibcallImpls in {
+  def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+}
+
+foreach libcall = DefaultRuntimeLibcallImpls in {
+  def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+}
+
+//===----------------------------------------------------------------------===//
+// ARM Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// if (isTargetMachO()) {
+// if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
+//  Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
+
+def __addsf3vfp : RuntimeLibcallImpl<ADD_F32>;
+def __subsf3vfp : RuntimeLibcallImpl<SUB_F32>;
+def __mulsf3vfp : RuntimeLibcallImpl<MUL_F32>;
+def __divsf3vfp : RuntimeLibcallImpl<DIV_F32>;
+
+// Double-precision floating-point arithmetic.
+def __adddf3vfp : RuntimeLibcallImpl<ADD_F64>;
+def __subdf3vfp : RuntimeLibcallImpl<SUB_F64>;
+def __muldf3vfp : RuntimeLibcallImpl<MUL_F64>;
+def __divdf3vfp : RuntimeLibcallImpl<DIV_F64>;
+
+// Single-precision comparisons.
+
+// TODO: Track setcc type
+def __eqsf2vfp : RuntimeLibcallImpl<OEQ_F32>; // CmpInst::ICMP_NE
+def __nesf2vfp : RuntimeLibcallImpl<UNE_F32>; // CmpInst::ICMP_NE
+def __ltsf2vfp : RuntimeLibcallImpl<OLT_F32>; // CmpInst::ICMP_NE
+def __lesf2vfp : RuntimeLibcallImpl<OLE_F32>; // CmpInst::ICMP_NE
+def __gesf2vfp : RuntimeLibcallImpl<OGE_F32>; // CmpInst::ICMP_NE
+def __gtsf2vfp : RuntimeLibcallImpl<OGT_F32>; // CmpInst::ICMP_NE
+def __unordsf2vfp : RuntimeLibcallImpl<UO_F32>; // CmpInst::ICMP_NE
+
+// Double-precision comparisons.
+def __eqdf2vfp : RuntimeLibcallImpl<OEQ_F64>; // CmpInst::ICMP_NE
+def __nedf2vfp : RuntimeLibcallImpl<UNE_F64>; // CmpInst::ICMP_NE
+def __ltdf2vfp : RuntimeLibcallImpl<OLT_F64>; // CmpInst::ICMP_NE
+def __ledf2vfp : RuntimeLibcallImpl<OLE_F64>; // CmpInst::ICMP_NE
+def __gedf2vfp : RuntimeLibcallImpl<OGE_F64>; // CmpInst::ICMP_NE
+def __gtdf2vfp : RuntimeLibcallImpl<OGT_F64>; // CmpInst::ICMP_NE
+def __unorddf2vfp : RuntimeLibcallImpl<UO_F64>; // CmpInst::ICMP_NE
+
+// Floating-point to integer conversions.
+// i64 conversions are done via library routines even when generating VFP
+// instructions, so use the same ones.
+def __fixdfsivfp : RuntimeLibcallImpl<FPTOSINT_F64_I32>;
+def __fixunsdfsivfp : RuntimeLibcallImpl<FPTOUINT_F64_I32>;
+def __fixsfsivfp : RuntimeLibcallImpl<FPTOSINT_F32_I32>;
+def __fixunssfsivfp : RuntimeLibcallImpl<FPTOUINT_F32_I32>;
+
+// Conversions between floating types.
+def __truncdfsf2vfp : RuntimeLibcallImpl<FPROUND_F64_F32>;
+def __extendsfdf2vfp : RuntimeLibcallImpl<FPEXT_F32_F64>;
+
+// Integer to floating-point conversions.
+// i64 conversions are done via library routines even when generating VFP
+// instructions, so use the same ones.
+// FIXME: There appears to be some naming inconsistency in ARM libgcc:
+// e.g., __floatunsidf vs. __floatunssidfvfp.
+def __floatsidfvfp : RuntimeLibcallImpl<SINTTOFP_I32_F64>;
+def __floatunssidfvfp : RuntimeLibcallImpl<UINTTOFP_I32_F64>;
+def __floatsisfvfp : RuntimeLibcallImpl<SINTTOFP_I32_F32>;
+def __floatunssisfvfp : RuntimeLibcallImpl<UINTTOFP_I32_F32>;
+
+//  // RTLIB
+//  if (isAAPCS_ABI() &&
+//      (isTargetAEABI() || isTargetGNUAEABI() ||
+//       isTargetMuslAEABI() || isTargetAndroid())) {
+
+// TODO: Set CallingConv = ARM_AAPCS
+
+// Double-precision floating-point arithmetic helper functions
+// RTABI chapter 4.1.2, Table 2
+def __aeabi_dadd : RuntimeLibcallImpl<ADD_F64>; // CallingConv::ARM_AAPCS
+def __aeabi_ddiv : RuntimeLibcallImpl<DIV_F64>; // CallingConv::ARM_AAPCS
+def __aeabi_dmul : RuntimeLibcallImpl<MUL_F64>; // CallingConv::ARM_AAPCS
+def __aeabi_dsub : RuntimeLibcallImpl<SUB_F64>; // CallingConv::ARM_AAPCS
+
+// Double-precision floating-point comparison helper functions
+// RTABI chapter 4.1.2, Table 3
+def __aeabi_dcmpeq__ne : RuntimeLibcallImpl<OEQ_F64, "__aeabi_dcmpeq">; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_dcmpeq__eq : RuntimeLibcallImpl<UNE_F64, "__aeabi_dcmpeq">; // CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ
+def __aeabi_dcmplt : RuntimeLibcallImpl<OLT_F64>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_dcmple : RuntimeLibcallImpl<OLE_F64>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_dcmpge : RuntimeLibcallImpl<OGE_F64>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_dcmpgt : RuntimeLibcallImpl<OGT_F64>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_dcmpun : RuntimeLibcallImpl<UO_F64>; // CallingConv::ARM_AAPCS
+
+// Single-precision floating-point arithmetic helper functions
+// RTABI chapter 4.1.2, Table 4
+def __aeabi_fadd : RuntimeLibcallImpl<ADD_F32>; // CallingConv::ARM_AAPCS
+def __aeabi_fdiv : RuntimeLibcallImpl<DIV_F32>; // CallingConv::ARM_AAPCS
+def __aeabi_fmul : RuntimeLibcallImpl<MUL_F32>; // CallingConv::ARM_AAPCS
+def __aeabi_fsub : RuntimeLibcallImpl<SUB_F32>; // CallingConv::ARM_AAPCS
+
+// Single-precision floating-point comparison helper functions
+// RTABI chapter 4.1.2, Table 5
+def __aeabi_fcmpeq__ne : RuntimeLibcallImpl<OEQ_F32, "__aeabi_fcmpeq">; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_fcmpeq__eq : RuntimeLibcallImpl<UNE_F32, "__aeabi_fcmpeq">; // CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ
+def __aeabi_fcmplt : RuntimeLibcallImpl<OLT_F32>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_fcmple : RuntimeLibcallImpl<OLE_F32>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_fcmpge : RuntimeLibcallImpl<OGE_F32>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_fcmpgt : RuntimeLibcallImpl<OGT_F32>; // CallingConv::ARM_AAPCS, CmpInst::ICMP_NE
+def __aeabi_fcmpun : RuntimeLibcallImpl<UO_F32>; // CallingConv::ARM_AAPCS
+
+// Floating-point to integer conversions.
+// RTABI chapter 4.1.2, Table 6
+def __aeabi_d2iz : RuntimeLibcallImpl<FPTOSINT_F64_I32>; //  CallingConv::ARM_AAPCS
+def __aeabi_d2uiz : RuntimeLibcallImpl<FPTOUINT_F64_I32>; // CallingConv::ARM_AAPCS
+def __aeabi_d2lz : RuntimeLibcallImpl<FPTOSINT_F64_I64>; //  CallingConv::ARM_AAPCS
+def __aeabi_d2ulz : RuntimeLibcallImpl<FPTOUINT_F64_I64>; // CallingConv::ARM_AAPCS
+def __aeabi_f2iz : RuntimeLibcallImpl<FPTOSINT_F32_I32>; //  CallingConv::ARM_AAPCS
+def __aeabi_f2uiz : RuntimeLibcallImpl<FPTOUINT_F32_I32>; // CallingConv::ARM_AAPCS
+def __aeabi_f2lz : RuntimeLibcallImpl<FPTOSINT_F32_I64>; //  CallingConv::ARM_AAPCS
+def __aeabi_f2ulz : RuntimeLibcallImpl<FPTOUINT_F32_I64>; // CallingConv::ARM_AAPCS
+
+// Conversions between floating types.
+// RTABI chapter 4.1.2, Table 7
+def __aeabi_d2f : RuntimeLibcallImpl<FPROUND_F64_F32>; // CallingConv::ARM_AAPCS
+def __aeabi_d2h : RuntimeLibcallImpl<FPROUND_F64_F16>; // CallingConv::ARM_AAPCS
+def  __aeabi_f2d : RuntimeLibcallImpl<FPEXT_F32_F64>; // CallingConv::ARM_AAPCS
+
+// Integer to floating-point conversions.
+// RTABI chapter 4.1.2, Table 8
+def __aeabi_i2d : RuntimeLibcallImpl<SINTTOFP_I32_F64>; //  CallingConv::ARM_AAPCS
+def __aeabi_ui2d : RuntimeLibcallImpl<UINTTOFP_I32_F64>; // CallingConv::ARM_AAPCS
+def __aeabi_l2d : RuntimeLibcallImpl<SINTTOFP_I64_F64>; //  CallingConv::ARM_AAPCS
+def __aeabi_ul2d : RuntimeLibcallImpl<UINTTOFP_I64_F64>; // CallingConv::ARM_AAPCS
+def __aeabi_i2f : RuntimeLibcallImpl<SINTTOFP_I32_F32>; //  CallingConv::ARM_AAPCS
+def __aeabi_ui2f : RuntimeLibcallImpl<UINTTOFP_I32_F32>; // CallingConv::ARM_AAPCS
+def __aeabi_l2f : RuntimeLibcallImpl<SINTTOFP_I64_F32>; //  CallingConv::ARM_AAPCS
+def __aeabi_ul2f : RuntimeLibcallImpl<UINTTOFP_I64_F32>; // CallingConv::ARM_AAPCS
+
+// Long long helper functions
+// RTABI chapter 4.2, Table 9
+def __aeabi_lmul : RuntimeLibcallImpl<MUL_I64>; // CallingConv::ARM_AAPCS
+def __aeabi_llsl : RuntimeLibcallImpl<SHL_I64>; // CallingConv::ARM_AAPCS
+def __aeabi_llsr : RuntimeLibcallImpl<SRL_I64>; // CallingConv::ARM_AAPCS
+def __aeabi_lasr : RuntimeLibcallImpl<SRA_I64>; // CallingConv::ARM_AAPCS
+
+// Integer division functions
+// RTABI chapter 4.3.1
+def __aeabi_idiv__i8 : RuntimeLibcallImpl<SDIV_I8, "__aeabi_idiv">; // CallingConv::ARM_AAPCS
+def __aeabi_idiv__i16 : RuntimeLibcallImpl<SDIV_I16, "__aeabi_idiv">; // CallingConv::ARM_AAPCS
+def __aeabi_idiv__i32 : RuntimeLibcallImpl<SDIV_I32, "__aeabi_idiv">; // CallingConv::ARM_AAPCS
+def __aeabi_ldivmod : RuntimeLibcallImpl<SDIVREM_I64>; //  CallingConv::ARM_AAPCS
+def __aeabi_uidiv__i8 : RuntimeLibcallImpl<UDIV_I8, "__aeabi_uidiv">; // CallingConv::ARM_AAPCS
+def __aeabi_uidiv__i16 : RuntimeLibcallImpl<UDIV_I16, "__aeabi_uidiv">; // CallingConv::ARM_AAPCS
+def __aeabi_uidiv__i32 : RuntimeLibcallImpl<UDIV_I32, "__aeabi_uidiv">; // CallingConv::ARM_AAPCS
+def __aeabi_uldivmod : RuntimeLibcallImpl<UDIVREM_I64>; // CallingConv::ARM_AAPCS
+
+def __aeabi_idivmod : RuntimeLibcallImpl<SDIVREM_I32>; //  CallingConv::ARM_AAPCS
+def __aeabi_uidivmod : RuntimeLibcallImpl<UDIVREM_I32>; //  CallingConv::ARM_AAPCS
+
+// EABI dependent RTLIB
+
+// Memory operations
+// RTABI chapter 4.3.4
+def __aeabi_memcpy : RuntimeLibcallImpl<MEMCPY>; // CallingConv::ARM_AAPCS
+def __aeabi_memcpy4 : RuntimeLibcallImpl<AEABI_MEMCPY4>;
+def __aeabi_memcpy8 : RuntimeLibcallImpl<AEABI_MEMCPY8>;
+
+def __aeabi_memmove : RuntimeLibcallImpl<MEMMOVE>; // CallingConv::ARM_AAPCS
+def __aeabi_memmove4 : RuntimeLibcallImpl<AEABI_MEMMOVE4>;
+def __aeabi_memmove8 : RuntimeLibcallImpl<AEABI_MEMMOVE8>;
+
+def __aeabi_memset : RuntimeLibcallImpl<MEMSET>; // CallingConv::ARM_AAPCS
+def __aeabi_memset4 : RuntimeLibcallImpl<AEABI_MEMSET4>;
+def __aeabi_memset8 : RuntimeLibcallImpl<AEABI_MEMSET8>;
+
+def __aeabi_memclr : RuntimeLibcallImpl<AEABI_MEMCLR>;
+def __aeabi_memclr4 : RuntimeLibcallImpl<AEABI_MEMCLR4>;
+def __aeabi_memclr8 : RuntimeLibcallImpl<AEABI_MEMCLR8>;
+
+// isTargetWindows()
+def __stoi64 : RuntimeLibcallImpl<FPTOSINT_F32_I64>; // CallingConv::ARM_AAPCS_VFP
+def __dtoi64 : RuntimeLibcallImpl<FPTOSINT_F64_I64>; // CallingConv::ARM_AAPCS_VFP
+def __stou64 : RuntimeLibcallImpl<FPTOUINT_F32_I64>; // CallingConv::ARM_AAPCS_VFP
+def __dtou64 : RuntimeLibcallImpl<FPTOUINT_F64_I64>; // CallingConv::ARM_AAPCS_VFP
+def __i64tos : RuntimeLibcallImpl<SINTTOFP_I64_F32>; // CallingConv::ARM_AAPCS_VFP
+def __i64tod : RuntimeLibcallImpl<SINTTOFP_I64_F64>; // CallingConv::ARM_AAPCS_VFP
+def __u64tos : RuntimeLibcallImpl<UINTTOFP_I64_F32>; // CallingConv::ARM_AAPCS_VFP
+def __u64tod : RuntimeLibcallImpl<UINTTOFP_I64_F64>; // CallingConv::ARM_AAPCS_VFP
+
+def __rt_sdiv : RuntimeLibcallImpl<SDIVREM_I32>; // CallingConv::ARM_AAPCS
+def __rt_sdiv64 : RuntimeLibcallImpl<SDIVREM_I64>; // CallingConv::ARM_AAPCS
+def __rt_udiv : RuntimeLibcallImpl<UDIVREM_I32>; // CallingConv::ARM_AAPCS
+def __rt_udiv64 : RuntimeLibcallImpl<UDIVREM_I64>; // CallingConv::ARM_AAPCS
+
+// Use divmod compiler-rt calls for iOS 5.0 and later.
+//  isTargetMachO() &&
+// !(isTargetIOS() && isOSVersionLT(5, 0))
+def __divmodsi4 : RuntimeLibcallImpl<SDIVREM_I32>;
+def __udivmodsi4 : RuntimeLibcallImpl<UDIVREM_I32>;
+
+// FIXME: Change calling convention of FPROUND_F32_F16,
+// RTLIB::FPROUND_F64_F16, FPEXT_F16_F32 for !Subtarget->isTargetWatchABI())
+
+// In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
+// a __gnu_ prefix (which is the default).
+// isTargetAEABI()
+def __aeabi_f2h : RuntimeLibcallImpl<FPROUND_F32_F16>; // CallingConv::ARM_AAPCS
+//def __aeabi_d2h : RuntimeLibcallImpl<FPROUND_F64_F16>; // CallingConv::ARM_AAPCS
+def __aeabi_h2f : RuntimeLibcallImpl<FPEXT_F16_F32>; // CallingConv::ARM_AAPCS
+
+// !isTargetMachO()
+def __gnu_f2h_ieee : RuntimeLibcallImpl<FPROUND_F32_F16>;
+def __gnu_h2f_ieee : RuntimeLibcallImpl<FPEXT_F16_F32>;
+
+//===----------------------------------------------------------------------===//
+// AVR Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// Several of the runtime library functions use a special calling conv
+def __divmodqi4 : RuntimeLibcallImpl<SDIVREM_I8>; // CallingConv::AVR_BUILTIN
+def __divmodhi4 : RuntimeLibcallImpl<SDIVREM_I16>; // CallingConv::AVR_BUILTIN
+//def __divmodsi4 : RuntimeLibcallImpl<SDIVREM_I32>;
+def __udivmodqi4 : RuntimeLibcallImpl<UDIVREM_I8>; // CallingConv::AVR_BUILTIN
+def __udivmodhi4 : RuntimeLibcallImpl<UDIVREM_I16>; // CallingConv::AVR_BUILTIN
+//def __udivmodsi4 : RuntimeLibcallImpl<UDIVREM_I32>;
+
+// Standard sinf/cosf name replaced with "sin" and "cos". Define a
+// separate Impl so we can set a different type signature.
+def avr_sin : RuntimeLibcallImpl<SIN_F32, "sin">;
+def avr_cos : RuntimeLibcallImpl<COS_F32, "cos">;
+
+//===----------------------------------------------------------------------===//
+// Hexagon Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+def __hexagon_divsi3 : RuntimeLibcallImpl<SDIV_I32>;
+def __hexagon_divdi3 : RuntimeLibcallImpl<SDIV_I64>;
+def __hexagon_udivsi3 : RuntimeLibcallImpl<UDIV_I32>;
+def __hexagon_udivdi3 : RuntimeLibcallImpl<UDIV_I64>;
+def __hexagon_modsi3 : RuntimeLibcallImpl<SREM_I32>;
+def __hexagon_moddi3 : RuntimeLibcallImpl<SREM_I64>;
+def __hexagon_umodsi3 : RuntimeLibcallImpl<UREM_I32>;
+def __hexagon_umoddi3 : RuntimeLibcallImpl<UREM_I64>;
+
+// FIXME: "Fast" versions should be treated as a separate RTLIB::FAST_* function
+def __hexagon_adddf3 : RuntimeLibcallImpl<ADD_F64>;
+def __hexagon_fast_adddf3 : RuntimeLibcallImpl<ADD_F64>;
+
+def __hexagon_subdf3 : RuntimeLibcallImpl<SUB_F64>;
+def __hexagon_fast_subdf3 : RuntimeLibcallImpl<SUB_F64>;
+
+def __hexagon_muldf3 : RuntimeLibcallImpl<MUL_F64>;
+def __hexagon_fast_muldf3 : RuntimeLibcallImpl<MUL_F64>;
+
+def __hexagon_divdf3 : RuntimeLibcallImpl<DIV_F64>;
+def __hexagon_fast_divdf3 : RuntimeLibcallImpl<DIV_F64>;
+
+def __hexagon_divsf3 : RuntimeLibcallImpl<DIV_F32>;
+def __hexagon_fast_divsf3 : RuntimeLibcallImpl<DIV_F32>;
+
+def __hexagon_sqrtf : RuntimeLibcallImpl<SQRT_F32>;
+def __hexagon_fast2_sqrtf : RuntimeLibcallImpl<SQRT_F32>;
+
+// This is the only fast library function for sqrtd.
+def __hexagon_fast2_sqrtdf2 : RuntimeLibcallImpl<SQRT_F64>;
+
+def __hexagon_memcpy_likely_aligned_min32bytes_mult8bytes
+    : RuntimeLibcallImpl<HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES>;
+
+//===----------------------------------------------------------------------===//
+// Mips16 Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// Hard float libcalls
+def __mips16_adddf3 : RuntimeLibcallImpl<ADD_F64>;
+def __mips16_addsf3 : RuntimeLibcallImpl<ADD_F32>;
+def __mips16_divdf3 : RuntimeLibcallImpl<DIV_F64>;
+def __mips16_divsf3 : RuntimeLibcallImpl<DIV_F32>;
+def __mips16_eqdf2 : RuntimeLibcallImpl<OEQ_F64>;
+def __mips16_eqsf2 : RuntimeLibcallImpl<OEQ_F32>;
+def __mips16_extendsfdf2 : RuntimeLibcallImpl<FPEXT_F32_F64>;
+def __mips16_fix_truncdfsi : RuntimeLibcallImpl<FPTOSINT_F64_I32>;
+def __mips16_fix_truncsfsi : RuntimeLibcallImpl<FPTOSINT_F32_I32>;
+def __mips16_floatsidf : RuntimeLibcallImpl<SINTTOFP_I32_F64>;
+def __mips16_floatsisf : RuntimeLibcallImpl<SINTTOFP_I32_F32>;
+def __mips16_floatunsidf : RuntimeLibcallImpl<UINTTOFP_I32_F64>;
+def __mips16_floatunsisf : RuntimeLibcallImpl<UINTTOFP_I32_F32>;
+def __mips16_gedf2 : RuntimeLibcallImpl<OGE_F64>;
+def __mips16_gesf2 : RuntimeLibcallImpl<OGE_F32>;
+def __mips16_gtdf2 : RuntimeLibcallImpl<OGT_F64>;
+def __mips16_gtsf2 : RuntimeLibcallImpl<OGT_F32>;
+def __mips16_ledf2 : RuntimeLibcallImpl<OLE_F64>;
+def __mips16_lesf2 : RuntimeLibcallImpl<OLE_F32>;
+def __mips16_ltdf2 : RuntimeLibcallImpl<OLT_F64>;
+def __mips16_ltsf2 : RuntimeLibcallImpl<OLT_F32>;
+def __mips16_muldf3 : RuntimeLibcallImpl<MUL_F64>;
+def __mips16_mulsf3 : RuntimeLibcallImpl<MUL_F32>;
+def __mips16_nedf2 : RuntimeLibcallImpl<UNE_F64>;
+def __mips16_nesf2 : RuntimeLibcallImpl<UNE_F32>;
+def __mips16_ret_dc : RuntimeLibcallImpl<MIPS16_RET_DC>;
+def __mips16_ret_df : RuntimeLibcallImpl<MIPS16_RET_DF>;
+def __mips16_ret_sc : RuntimeLibcallImpl<MIPS16_RET_SC>;
+def __mips16_ret_sf : RuntimeLibcallImpl<MIPS16_RET_SF>;
+def __mips16_subdf3 : RuntimeLibcallImpl<SUB_F64>;
+def __mips16_subsf3 : RuntimeLibcallImpl<SUB_F32>;
+def __mips16_truncdfsf2 : RuntimeLibcallImpl<FPROUND_F64_F32>;
+def __mips16_unorddf2 : RuntimeLibcallImpl<UO_F64>;
+def __mips16_unordsf2 : RuntimeLibcallImpl<UO_F32>;
+
+//===----------------------------------------------------------------------===//
+// MSP430 Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// EABI Libcalls - EABI Section 6.2
+
+// Floating point conversions - EABI Table 6
+def __mspabi_cvtdf : RuntimeLibcallImpl<FPROUND_F64_F32>;
+def __mspabi_cvtfd : RuntimeLibcallImpl<FPEXT_F32_F64>;
+
+// The following is NOT implemented in libgcc
+//def __mspabi_fixdi : RuntimeLibcallImpl<FPTOSINT_F64_I16>;
+def __mspabi_fixdli : RuntimeLibcallImpl<FPTOSINT_F64_I32>;
+def __mspabi_fixdlli : RuntimeLibcallImpl<FPTOSINT_F64_I64>;
+
+// The following is NOT implemented in libgcc
+//def __mspabi_fixdu : RuntimeLibcallImpl<FPTOUINT_F64_I16>;
+def __mspabi_fixdul : RuntimeLibcallImpl<FPTOUINT_F64_I32>;
+def __mspabi_fixdull : RuntimeLibcallImpl<FPTOUINT_F64_I64>;
+
+// The following is NOT implemented in libgcc
+//def __mspabi_fixfi : RuntimeLibcallImpl<FPTOSINT_F32_I16>;
+def __mspabi_fixfli : RuntimeLibcallImpl<FPTOSINT_F32_I32>;
+def __mspabi_fixflli : RuntimeLibcallImpl<FPTOSINT_F32_I64>;
+
+// The following is NOT implemented in libgcc
+//def __mspabi_fixfu : RuntimeLibcallImpl<FPTOUINT_F32_I16>;
+def __mspabi_fixful : RuntimeLibcallImpl<FPTOUINT_F32_I32>;
+def __mspabi_fixfull : RuntimeLibcallImpl<FPTOUINT_F32_I64>;
+
+// TODO The following IS implemented in libgcc
+//def __mspabi_fltid : RuntimeLibcallImpl<SINTTOFP_I16_F64>;
+def __mspabi_fltlid : RuntimeLibcallImpl<SINTTOFP_I32_F64>;
+
+// TODO The following IS implemented in libgcc but is not in the EABI
+def __mspabi_fltllid : RuntimeLibcallImpl<SINTTOFP_I64_F64>;
+
+// TODO The following IS implemented in libgcc
+//def __mspabi_fltud : RuntimeLibcallImpl<UINTTOFP_I16_F64>;
+def __mspabi_fltuld : RuntimeLibcallImpl<UINTTOFP_I32_F64>;
+
+// The following IS implemented in libgcc but is not in the EABI
+def __mspabi_fltulld : RuntimeLibcallImpl<UINTTOFP_I64_F64>;
+
+// TODO The following IS implemented in libgcc
+//def __mspabi_fltif : RuntimeLibcallImpl<SINTTOFP_I16_F32>;
+def __mspabi_fltlif : RuntimeLibcallImpl<SINTTOFP_I32_F32>;
+
+// TODO The following IS implemented in libgcc but is not in the EABI
+def __mspabi_fltllif : RuntimeLibcallImpl<SINTTOFP_I64_F32>;
+
+// TODO The following IS implemented in libgcc
+//def __mspabi_fltuf : RuntimeLibcallImpl<UINTTOFP_I16_F32>;
+def __mspabi_fltulf : RuntimeLibcallImpl<UINTTOFP_I32_F32>;
+
+// The following IS implemented in libgcc but is not in the EABI
+def __mspabi_fltullf : RuntimeLibcallImpl<UINTTOFP_I64_F32>;
+
+// Floating point comparisons - EABI Table 7
+def __mspabi_cmpd__oeq : RuntimeLibcallImpl<OEQ_F64, "__mspabi_cmpd">;
+def __mspabi_cmpd__une : RuntimeLibcallImpl<UNE_F64, "__mspabi_cmpd">;
+def __mspabi_cmpd__oge : RuntimeLibcallImpl<OGE_F64, "__mspabi_cmpd">;
+def __mspabi_cmpd__olt : RuntimeLibcallImpl<OLT_F64, "__mspabi_cmpd">;
+def __mspabi_cmpd__ole : RuntimeLibcallImpl<OLE_F64, "__mspabi_cmpd">;
+def __mspabi_cmpd__ogt : RuntimeLibcallImpl<OGT_F64, "__mspabi_cmpd">;
+def __mspabi_cmpf__oeq : RuntimeLibcallImpl<OEQ_F32, "__mspabi_cmpf">;
+def __mspabi_cmpf__une : RuntimeLibcallImpl<UNE_F32, "__mspabi_cmpf">;
+def __mspabi_cmpf__oge : RuntimeLibcallImpl<OGE_F32, "__mspabi_cmpf">;
+def __mspabi_cmpf__olt : RuntimeLibcallImpl<OLT_F32, "__mspabi_cmpf">;
+def __mspabi_cmpf__ole : RuntimeLibcallImpl<OLE_F32, "__mspabi_cmpf">;
+def __mspabi_cmpf__ogt : RuntimeLibcallImpl<OGT_F32, "__mspabi_cmpf">;
+
+// Floating point arithmetic - EABI Table 8
+def __mspabi_addd : RuntimeLibcallImpl<ADD_F64>;
+def __mspabi_addf : RuntimeLibcallImpl<ADD_F32>;
+def __mspabi_divd : RuntimeLibcallImpl<DIV_F64>;
+def __mspabi_divf : RuntimeLibcallImpl<DIV_F32>;
+def __mspabi_mpyd : RuntimeLibcallImpl<MUL_F64>;
+def __mspabi_mpyf : RuntimeLibcallImpl<MUL_F32>;
+def __mspabi_subd : RuntimeLibcallImpl<SUB_F64>;
+def __mspabi_subf : RuntimeLibcallImpl<SUB_F32>;
+
+// The following are NOT implemented in libgcc
+// def __mspabi_negd : RuntimeLibcallImpl<NEG_F64>;
+// def __mspabi_negf : RuntimeLibcallImpl<NEG_F32>;
+
+// Universal Integer Operations - EABI Table 9
+def __mspabi_divi : RuntimeLibcallImpl<SDIV_I16>;
+def __mspabi_divli : RuntimeLibcallImpl<SDIV_I32>;
+def __mspabi_divlli : RuntimeLibcallImpl<SDIV_I64>;
+def __mspabi_divu : RuntimeLibcallImpl<UDIV_I16>;
+def __mspabi_divul : RuntimeLibcallImpl<UDIV_I32>;
+def __mspabi_divull : RuntimeLibcallImpl<UDIV_I64>;
+def __mspabi_remi : RuntimeLibcallImpl<SREM_I16>;
+def __mspabi_remli : RuntimeLibcallImpl<SREM_I32>;
+def __mspabi_remlli : RuntimeLibcallImpl<SREM_I64>;
+def __mspabi_remu : RuntimeLibcallImpl<UREM_I16>;
+def __mspabi_remul : RuntimeLibcallImpl<UREM_I32>;
+def __mspabi_remull : RuntimeLibcallImpl<UREM_I64>;
+
+// Bitwise Operations - EABI Table 10
+// TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
+def __mspabi_srll : RuntimeLibcallImpl<SRL_I32>;
+def __mspabi_sral : RuntimeLibcallImpl<SRA_I32>;
+def __mspabi_slll : RuntimeLibcallImpl<SHL_I32>;
+// __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
+
+// hasHWMult16()
+// Integer Multiply - EABI Table 9
+//def __mspabi_mpyi_hw : RuntimeLibcallImpl<MUL_I16>;
+def __mspabi_mpyl_hw : RuntimeLibcallImpl<MUL_I32>;
+def __mspabi_mpyll_hw : RuntimeLibcallImpl<MUL_I64>;
+// TODO The __mspabi_mpysl*_hw functions ARE implemented in libgcc
+// TODO The __mspabi_mpyul*_hw functions ARE implemented in libgcc
+
+// hasHWMult32()
+// Integer Multiply - EABI Table 9
+def __mspabi_mpyi_hw : RuntimeLibcallImpl<MUL_I16>;
+def __mspabi_mpyl_hw32 : RuntimeLibcallImpl<MUL_I32>;
+def __mspabi_mpyll_hw32 : RuntimeLibcallImpl<MUL_I64>;
+// TODO The __mspabi_mpysl*_hw32 functions ARE implemented in libgcc
+// TODO The __mspabi_mpyul*_hw32 functions ARE implemented in libgcc
+
+// hasHWMultF5()
+// Integer Multiply - EABI Table 9
+def __mspabi_mpyi_f5hw : RuntimeLibcallImpl<MUL_I16>;
+def __mspabi_mpyl_f5hw : RuntimeLibcallImpl<MUL_I32>;
+def __mspabi_mpyll_f5hw : RuntimeLibcallImpl<MUL_I64>;
+// TODO The __mspabi_mpysl*_f5hw functions ARE implemented in libgcc
+// TODO The __mspabi_mpyul*_f5hw functions ARE implemented in libgcc
+
+// NoHWMult
+// Integer Multiply - EABI Table 9
+def __mspabi_mpyi : RuntimeLibcallImpl<MUL_I16>;
+def __mspabi_mpyl : RuntimeLibcallImpl<MUL_I32>;
+def __mspabi_mpyll : RuntimeLibcallImpl<MUL_I64>;
+// The __mspabi_mpysl* functions are NOT implemented in libgcc
+// The __mspabi_mpyul* functions are NOT implemented in libgcc
+
+// setLibcallCallingConv(MUL_I64, CallingConv::MSP430_BUILTIN);
+
+//===----------------------------------------------------------------------===//
+// PPC Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+class PPCRuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME>
+    : RuntimeLibcallImpl<P, Name>;
+
+// For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
+def __addkf3 : PPCRuntimeLibcallImpl<ADD_F128>;
+def __subkf3 : PPCRuntimeLibcallImpl<SUB_F128>;
+def __mulkf3 : PPCRuntimeLibcallImpl<MUL_F128>;
+def __divkf3 : PPCRuntimeLibcallImpl<DIV_F128>;
+def __powikf2 : PPCRuntimeLibcallImpl<POWI_F128>;
+def __extendsfkf2 : PPCRuntimeLibcallImpl<FPEXT_F32_F128>;
+def __extenddfkf2 : PPCRuntimeLibcallImpl<FPEXT_F64_F128>;
+def __trunckfhf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F16>;
+def __trunckfsf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F32>;
+def __trunckfdf2 : PPCRuntimeLibcallImpl<FPROUND_F128_F64>;
+def __fixkfsi : PPCRuntimeLibcallImpl<FPTOSINT_F128_I32>;
+def __fixkfdi : PPCRuntimeLibcallImpl<FPTOSINT_F128_I64>;
+def __fixkfti : PPCRuntimeLibcallImpl<FPTOSINT_F128_I128>;
+def __fixunskfsi : PPCRuntimeLibcallImpl<FPTOUINT_F128_I32>;
+def __fixunskfdi : PPCRuntimeLibcallImpl<FPTOUINT_F128_I64>;
+def __fixunskfti : PPCRuntimeLibcallImpl<FPTOUINT_F128_I128>;
+def __floatsikf : PPCRuntimeLibcallImpl<SINTTOFP_I32_F128>;
+def __floatdikf : PPCRuntimeLibcallImpl<SINTTOFP_I64_F128>;
+def __floattikf : PPCRuntimeLibcallImpl<SINTTOFP_I128_F128>;
+def __floatunsikf : PPCRuntimeLibcallImpl<UINTTOFP_I32_F128>;
+def __floatundikf : PPCRuntimeLibcallImpl<UINTTOFP_I64_F128>;
+def __floatuntikf : PPCRuntimeLibcallImpl<UINTTOFP_I128_F128>;
+def __eqkf2 : PPCRuntimeLibcallImpl<OEQ_F128>;
+def __nekf2 : PPCRuntimeLibcallImpl<UNE_F128>;
+def __gekf2 : PPCRuntimeLibcallImpl<OGE_F128>;
+def __ltkf2 : PPCRuntimeLibcallImpl<OLT_F128>;
+def __lekf2 : PPCRuntimeLibcallImpl<OLE_F128>;
+def __gtkf2 : PPCRuntimeLibcallImpl<OGT_F128>;
+def __unordkf2 : PPCRuntimeLibcallImpl<UO_F128>;
+
+// PPC64 && Subtarget.isAIXABI()
+def ___memmove64 : RuntimeLibcallImpl<MEMCPY>;
+def ___memset64 : RuntimeLibcallImpl<MEMSET>;
+def ___bzero64 : RuntimeLibcallImpl<BZERO>;
+
+// !PPC64 && Subtarget.isAIXABI()
+def ___memmove : RuntimeLibcallImpl<MEMMOVE>;
+def ___memset : RuntimeLibcallImpl<MEMSET>;
+def ___bzero : RuntimeLibcallImpl<BZERO>;
+
+//===----------------------------------------------------------------------===//
+// SPARC Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// Subtarget->useSoftMulDiv()
+def sparc_umul : RuntimeLibcallImpl<MUL_I32, ".umul">;
+def sparc_div : RuntimeLibcallImpl<SDIV_I32, ".div">;
+def sparc_udiv : RuntimeLibcallImpl<UDIV_I32, ".udiv">;
+def sparc_rem : RuntimeLibcallImpl<SREM_I32, ".rem">;
+def sparc_urem : RuntimeLibcallImpl<UREM_I32, ".urem">;
+
+// SPARC64 && !Subtarget->useSoftFloat()
+def _Qp_add : RuntimeLibcallImpl<ADD_F128>;
+def _Qp_sub : RuntimeLibcallImpl<SUB_F128>;
+def _Qp_mul : RuntimeLibcallImpl<MUL_F128>;
+def _Qp_div : RuntimeLibcallImpl<DIV_F128>;
+def _Qp_sqrt : RuntimeLibcallImpl<SQRT_F128>;
+def _Qp_qtoi : RuntimeLibcallImpl<FPTOSINT_F128_I32>;
+def _Qp_qtoui : RuntimeLibcallImpl<FPTOUINT_F128_I32>;
+def _Qp_itoq : RuntimeLibcallImpl<SINTTOFP_I32_F128>;
+def _Qp_uitoq : RuntimeLibcallImpl<UINTTOFP_I32_F128>;
+def _Qp_qtox : RuntimeLibcallImpl<FPTOSINT_F128_I64>;
+def _Qp_qtoux : RuntimeLibcallImpl<FPTOUINT_F128_I64>;
+def _Qp_xtoq : RuntimeLibcallImpl<SINTTOFP_I64_F128>;
+def _Qp_uxtoq : RuntimeLibcallImpl<UINTTOFP_I64_F128>;
+def _Qp_stoq : RuntimeLibcallImpl<FPEXT_F32_F128>;
+def _Qp_dtoq : RuntimeLibcallImpl<FPEXT_F64_F128>;
+def _Qp_qtos : RuntimeLibcallImpl<FPROUND_F128_F32>;
+def _Qp_qtod : RuntimeLibcallImpl<FPROUND_F128_F64>;
+
+// SPARC32 && !Subtarget->useSoftFloat()
+def _Q_add : RuntimeLibcallImpl<ADD_F128>;
+def _Q_sub : RuntimeLibcallImpl<SUB_F128>;
+def _Q_mul : RuntimeLibcallImpl<MUL_F128>;
+def _Q_div : RuntimeLibcallImpl<DIV_F128>;
+def _Q_sqrt : RuntimeLibcallImpl<SQRT_F128>;
+def _Q_qtoi : RuntimeLibcallImpl<FPTOSINT_F128_I32>;
+def _Q_qtou : RuntimeLibcallImpl<FPTOUINT_F128_I32>;
+def _Q_itoq : RuntimeLibcallImpl<SINTTOFP_I32_F128>;
+def _Q_utoq : RuntimeLibcallImpl<UINTTOFP_I32_F128>;
+def _Q_stoq : RuntimeLibcallImpl<FPEXT_F32_F128>;
+def _Q_dtoq : RuntimeLibcallImpl<FPEXT_F64_F128>;
+def _Q_qtos : RuntimeLibcallImpl<FPROUND_F128_F32>;
+def _Q_qtod : RuntimeLibcallImpl<FPROUND_F128_F64>;
+
+// SPARC32 && (Subtarget->hasHardQuad() || !Subtarget->useSoftFloat())
+def _Q_qtoll : RuntimeLibcallImpl<FPTOSINT_F128_I64>;
+def _Q_qtoull : RuntimeLibcallImpl<FPTOUINT_F128_I64>;
+def _Q_lltoq : RuntimeLibcallImpl<SINTTOFP_I64_F128>;
+def _Q_ulltoq : RuntimeLibcallImpl<UINTTOFP_I64_F128>;
+
+//===----------------------------------------------------------------------===//
+// Windows Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()
+def _alldiv : RuntimeLibcallImpl<SDIV_I64>; // CallingConv::X86_StdCall
+def _aulldiv : RuntimeLibcallImpl<UDIV_I64>; // CallingConv::X86_StdCall
+def _allrem : RuntimeLibcallImpl<SREM_I64>; // CallingConv::X86_StdCall
+def _aullrem : RuntimeLibcallImpl<UREM_I64>; // CallingConv::X86_StdCall
+def _allmul : RuntimeLibcallImpl<MUL_I64>; // CallingConv::X86_StdCall
+
+//===----------------------------------------------------------------------===//
+// XCore Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+def __memcpy_4 : RuntimeLibcallImpl<MEMCPY_ALIGN_4>;
+
+//===----------------------------------------------------------------------===//
+// ZOS Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+class ZOSRuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME>
+    : RuntimeLibcallImpl<P, Name>;
+
+def zos___TRNC_B : ZOSRuntimeLibcallImpl<TRUNC_F64, "@@TRNC@B">;
+def zos___FTRC_B : ZOSRuntimeLibcallImpl<TRUNC_F32, "@@FTRC@B">;
+def zos___LTRC_B : ZOSRuntimeLibcallImpl<TRUNC_F128, "@@LTRC@B">;
+def zos___WSQT_B : ZOSRuntimeLibcallImpl<SQRT_F64, "@@WSQT@B">;
+def zos___FSQT_B : ZOSRuntimeLibcallImpl<SQRT_F32, "@@FSQT@B">;
+def zos___LSQT_B : ZOSRuntimeLibcallImpl<SQRT_F128, "@@LSQT@B">;
+def zos___SSIN_B : ZOSRuntimeLibcallImpl<SIN_F64, "@@SSIN@B">;
+def zos___FSIN_B : ZOSRuntimeLibcallImpl<SIN_F32, "@@FSIN@B">;
+def zos___LSIN_B : ZOSRuntimeLibcallImpl<SIN_F128, "@@LSIN@B">;
+def zos___ROUN_B : ZOSRuntimeLibcallImpl<ROUND_F64, "@@ROUN@B">;
+def zos___ROUNFB : ZOSRuntimeLibcallImpl<ROUND_F32, "@@ROUNFB">;
+def zos___ROUNLB : ZOSRuntimeLibcallImpl<ROUND_F128, "@@ROUNLB">;
+def zos___SRNT_B : ZOSRuntimeLibcallImpl<RINT_F64, "@@SRNT@B">;
+def zos___RINTFB : ZOSRuntimeLibcallImpl<RINT_F32, "@@RINTFB">;
+def zos___RINTLB : ZOSRuntimeLibcallImpl<RINT_F128, "@@RINTLB">;
+def zos___WFMD_B : ZOSRuntimeLibcallImpl<REM_F64, "@@WFMD@B">;
+def zos___FFMD_B : ZOSRuntimeLibcallImpl<REM_F32, "@@FFMD@B">;
+def zos___LFMD_B : ZOSRuntimeLibcallImpl<REM_F128, "@@LFMD@B">;
+def zos___WPOW_B : ZOSRuntimeLibcallImpl<POW_F64, "@@WPOW@B">;
+def zos___FPOW_B : ZOSRuntimeLibcallImpl<POW_F32, "@@FPOW@B">;
+def zos___LPOW_B : ZOSRuntimeLibcallImpl<POW_F128, "@@LPOW@B">;
+def zos___NBYI_B : ZOSRuntimeLibcallImpl<NEARBYINT_F64, "@@NBYI@B">;
+def zos___NBYIFB : ZOSRuntimeLibcallImpl<NEARBYINT_F32, "@@NBYIFB">;
+def zos___NBYILB : ZOSRuntimeLibcallImpl<NEARBYINT_F128, "@@NBYILB">;
+def zos___ROND_B : ZOSRuntimeLibcallImpl<LROUND_F64, "@@ROND@B">;
+def zos___FRND_B : ZOSRuntimeLibcallImpl<LROUND_F32, "@@FRND@B">;
+def zos___LRND_B : ZOSRuntimeLibcallImpl<LROUND_F128, "@@LRND@B">;
+def zos___LRNT_B : ZOSRuntimeLibcallImpl<LRINT_F64, "@@LRNT@B">;
+def zos___LRNTFB : ZOSRuntimeLibcallImpl<LRINT_F32, "@@LRNTFB">;
+def zos___LRNTLB : ZOSRuntimeLibcallImpl<LRINT_F128, "@@LRNTLB">;
+def zos___WLOG_B : ZOSRuntimeLibcallImpl<LOG_F64, "@@WLOG@B">;
+def zos___FLOG_B : ZOSRuntimeLibcallImpl<LOG_F32, "@@FLOG@B">;
+def zos___LLOG_B : ZOSRuntimeLibcallImpl<LOG_F128, "@@LLOG@B">;
+def zos___LOG2_B : ZOSRuntimeLibcallImpl<LOG2_F64, "@@LOG2@B">;
+def zos___FLG2_B : ZOSRuntimeLibcallImpl<LOG2_F32, "@@FLG2@B">;
+def zos___LLG2_B : ZOSRuntimeLibcallImpl<LOG2_F128, "@@LLG2@B">;
+def zos___WLG1_B : ZOSRuntimeLibcallImpl<LOG10_F64, "@@WLG1@B">;
+def zos___FLG1_B : ZOSRuntimeLibcallImpl<LOG10_F32, "@@FLG1@B">;
+def zos___LLG1_B : ZOSRuntimeLibcallImpl<LOG10_F128, "@@LLG1@B">;
+def zos___LLRD_B : ZOSRuntimeLibcallImpl<LLROUND_F64, "@@LLRD@B">;
+def zos___LLRDFB : ZOSRuntimeLibcallImpl<LLROUND_F32, "@@LLRDFB">;
+def zos___LLRDLB : ZOSRuntimeLibcallImpl<LLROUND_F128, "@@LLRDLB">;
+def zos___LLRT_B : ZOSRuntimeLibcallImpl<LLRINT_F64, "@@LLRT@B">;
+def zos___LLRTFB : ZOSRuntimeLibcallImpl<LLRINT_F32, "@@LLRTFB">;
+def zos___LLRTLB : ZOSRuntimeLibcallImpl<LLRINT_F128, "@@LLRTLB">;
+def zos___SLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F64, "@@SLXP@B">;
+def zos___FLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F32, "@@FLXP@B">;
+def zos___LLXP_B : ZOSRuntimeLibcallImpl<LDEXP_F128, "@@LLXP@B">;
+def zos___SFXP_B : ZOSRuntimeLibcallImpl<FREXP_F64, "@@SFXP@B">;
+def zos___FFXP_B : ZOSRuntimeLibcallImpl<FREXP_F32, "@@FFXP@B">;
+def zos___LFXP_B : ZOSRuntimeLibcallImpl<FREXP_F128, "@@LFXP@B">;
+def zos___FMIN_B : ZOSRuntimeLibcallImpl<FMIN_F64, "@@FMIN@B">;
+def zos___FMINFB : ZOSRuntimeLibcallImpl<FMIN_F32, "@@FMINFB">;
+def zos___FMINLB : ZOSRuntimeLibcallImpl<FMIN_F128, "@@FMINLB">;
+def zos___FMA_B : ZOSRuntimeLibcallImpl<FMA_F64, "@@FMA@B">;
+def zos___FMAFB : ZOSRuntimeLibcallImpl<FMA_F32, "@@FMAFB">;
+def zos___FMALB : ZOSRuntimeLibcallImpl<FMA_F128, "@@FMALB">;
+def zos___FMAX_B : ZOSRuntimeLibcallImpl<FMAX_F64, "@@FMAX@B">;
+def zos___FMAXFB : ZOSRuntimeLibcallImpl<FMAX_F32, "@@FMAXFB">;
+def zos___FMAXLB : ZOSRuntimeLibcallImpl<FMAX_F128, "@@FMAXLB">;
+def zos___SFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F64, "@@SFLR@B">;
+def zos___FFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F32, "@@FFLR@B">;
+def zos___LFLR_B : ZOSRuntimeLibcallImpl<FLOOR_F128, "@@LFLR@B">;
+def zos___WEXP_B : ZOSRuntimeLibcallImpl<EXP_F64, "@@WEXP@B">;
+def zos___FEXP_B : ZOSRuntimeLibcallImpl<EXP_F32, "@@FEXP@B">;
+def zos___LEXP_B : ZOSRuntimeLibcallImpl<EXP_F128, "@@LEXP@B">;
+def zos___EXP2_B : ZOSRuntimeLibcallImpl<EXP2_F64, "@@EXP2@B">;
+def zos___FXP2_B : ZOSRuntimeLibcallImpl<EXP2_F32, "@@FXP2@B">;
+def zos___LXP2_B : ZOSRuntimeLibcallImpl<EXP2_F128, "@@LXP2@B">;
+def zos___SCOS_B : ZOSRuntimeLibcallImpl<COS_F64, "@@SCOS@B">;
+def zos___FCOS_B : ZOSRuntimeLibcallImpl<COS_F32, "@@FCOS@B">;
+def zos___LCOS_B : ZOSRuntimeLibcallImpl<COS_F128, "@@LCOS@B">;
+def zos___DCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F64, "@@DCPY@B">;
+def zos___FCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F32, "@@FCPY@B">;
+def zos___LCPY_B : ZOSRuntimeLibcallImpl<COPYSIGN_F128, "@@LCPY@B">;
+def zos___SCEL_B : ZOSRuntimeLibcallImpl<CEIL_F64, "@@SCEL@B">;
+def zos___FCEL_B : ZOSRuntimeLibcallImpl<CEIL_F32, "@@FCEL@B">;
+def zos___LCEL_B : ZOSRuntimeLibcallImpl<CEIL_F128, "@@LCEL@B">;
+def zos___SCRT_B : ZOSRuntimeLibcallImpl<CBRT_F64, "@@SCRT@B">;
+def zos___FCBT_B : ZOSRuntimeLibcallImpl<CBRT_F32, "@@FCBT@B">;
+def zos___LCBT_B : ZOSRuntimeLibcallImpl<CBRT_F128, "@@LCBT@B">;
+
+//===----------------------------------------------------------------------===//
+// WebAssembly Runtime Libcalls
+//===----------------------------------------------------------------------===//
+
+// Define the emscripten name for return address helper.
+// TODO: when implementing other Wasm backends, make this generic or only do
+// this on emscripten depending on what they end up doing.
+def emscripten_return_address : RuntimeLibcallImpl<RETURN_ADDRESS>;
diff --git a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
new file mode 100644
index 000000000000..9444a0a838cd
--- /dev/null
+++ b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
@@ -0,0 +1,32 @@
+//===-- llvm/RuntimeLibcalls.td - File that describes libcalls ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+/// Abstract definition for functionality the compiler may need to
+/// emit a call to. Emits the RTLIB::Libcall enum - This enum defines
+/// all of the runtime library calls the backend can emit. The various
+/// long double types cannot be merged, because 80-bit library
+/// functions use "xf" and 128-bit use "tf".
+///
+/// When adding PPCF128 functions here, note that their names generally need
+/// to be overridden for Darwin with the xxx$LDBL128 form.  See
+/// PPCISelLowering.cpp.
+class RuntimeLibcall {
+  string Name = NAME;
+
+  // TODO: Record type signature and attributes
+}
+
+// Concrete implementation of a libcall, which may have a different
+// name and only be valid on some subset of targets.
+
+// TODO: Do we need a way to conditionally select calling convention?
+class RuntimeLibcallImpl<RuntimeLibcall P, string Name = NAME> {
+  RuntimeLibcall Provides = P;
+  string LibCallFuncName = Name;
+  bit IsDefault = false;
+}
diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h
index fd987179645d..74dd49072974 100644
--- a/llvm/include/llvm/IR/Type.h
+++ b/llvm/include/llvm/IR/Type.h
@@ -515,7 +515,7 @@ inline Type **unwrap(LLVMTypeRef* Tys) {
 }
 
 inline LLVMTypeRef *wrap(Type **Tys) {
-  return reinterpret_cast<LLVMTypeRef*>(const_cast<Type**>(Tys));
+  return reinterpret_cast<LLVMTypeRef *>(Tys);
 }
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 1b5b1d588882..1c4ed3843b39 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -336,6 +336,7 @@ LLVM_ABI void initializeVerifierLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegMapWrapperLegacyPass(PassRegistry &);
 LLVM_ABI void initializeVirtRegRewriterLegacyPass(PassRegistry &);
 LLVM_ABI void initializeWasmEHPreparePass(PassRegistry &);
+LLVM_ABI void initializeWindowsSecureHotPatchingPass(PassRegistry &);
 LLVM_ABI void initializeWinEHPreparePass(PassRegistry &);
 LLVM_ABI void initializeWriteBitcodePassPass(PassRegistry &);
 LLVM_ABI void initializeXRayInstrumentationLegacyPass(PassRegistry &);
diff --git a/llvm/include/llvm/LTO/legacy/LTOModule.h b/llvm/include/llvm/LTO/legacy/LTOModule.h
index 2b6a8734e78f..e4d52ac067a6 100644
--- a/llvm/include/llvm/LTO/legacy/LTOModule.h
+++ b/llvm/include/llvm/LTO/legacy/LTOModule.h
@@ -143,6 +143,14 @@ struct LTOModule {
     return StringRef();
   }
 
+  uint32_t getAsmUndefSymbolCount() { return _asm_undefines.size(); }
+
+  StringRef getAsmUndefSymbolName(uint32_t index) {
+    if (index < _asm_undefines.size())
+      return _asm_undefines[index];
+    return StringRef();
+  }
+
   const GlobalValue *getSymbolGV(uint32_t index) {
     if (index < _symbols.size())
       return _symbols[index].symbol;
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 5965be676ea6..bae7f0da7022 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -47,6 +47,10 @@
 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 #include <cstdlib>
 
+namespace llvm {
+class Triple;
+}
+
 namespace {
 struct ForcePassLinking {
   ForcePassLinking() {
@@ -147,7 +151,7 @@ struct ForcePassLinking {
     llvm::Function::Create(nullptr, llvm::GlobalValue::ExternalLinkage)
         ->viewCFGOnly();
     llvm::RGPassManager RGM;
-    llvm::TargetLibraryInfoImpl TLII;
+    llvm::TargetLibraryInfoImpl TLII((llvm::Triple()));
     llvm::TargetLibraryInfo TLI(TLII);
     llvm::AliasAnalysis AA(TLI);
     llvm::BatchAAResults BAA(AA);
diff --git a/llvm/include/llvm/MC/DXContainerRootSignature.h b/llvm/include/llvm/MC/DXContainerRootSignature.h
index 4c2ca063158c..4b6b42f7d74f 100644
--- a/llvm/include/llvm/MC/DXContainerRootSignature.h
+++ b/llvm/include/llvm/MC/DXContainerRootSignature.h
@@ -68,7 +68,7 @@ struct RootParametersContainer {
     Tables.push_back(Table);
   }
 
-  const std::pair<uint32_t, uint32_t>
+  std::pair<uint32_t, uint32_t>
   getTypeAndLocForParameter(uint32_t Location) const {
     const RootParameterInfo &Info = ParametersInfo[Location];
     return {Info.Header.ParameterType, Info.Location};
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index e49e786a10f5..ebc98ebfce9f 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -28,7 +28,6 @@ class MCSymbol;
 class MCAssembler;
 class MCContext;
 struct MCDwarfFrameInfo;
-struct MCFixupKindInfo;
 class MCInst;
 class MCObjectStreamer;
 class MCObjectTargetWriter;
@@ -39,6 +38,23 @@ class raw_pwrite_stream;
 class StringRef;
 class raw_ostream;
 
+/// Target independent information on a fixup kind.
+struct MCFixupKindInfo {
+  /// A target specific name for the fixup kind. The names will be unique for
+  /// distinct kinds on any given target.
+  const char *Name;
+
+  /// The bit offset to write the relocation into.
+  uint8_t TargetOffset;
+
+  /// The number of bits written by this fixup. The bits are assumed to be
+  /// contiguous.
+  uint8_t TargetSize;
+
+  /// Flags describing additional information on this fixup kind.
+  unsigned Flags;
+};
+
 /// Generic interface to target specific assembler backends.
 class LLVM_ABI MCAsmBackend {
 protected: // Can only create subclasses.
@@ -91,11 +107,6 @@ class LLVM_ABI MCAsmBackend {
   /// Get information on a fixup kind.
   virtual MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const;
 
-  // Hook used by the default `addReloc` to check if a relocation is needed.
-  virtual bool shouldForceRelocation(const MCFixup &, const MCValue &) {
-    return false;
-  }
-
   /// Hook to check if extra nop bytes must be inserted for alignment directive.
   /// For some targets this may be necessary in order to support linker
   /// relaxation. The number of bytes to insert are returned in Size.
@@ -111,14 +122,18 @@ class LLVM_ABI MCAsmBackend {
     return false;
   }
 
-  virtual bool evaluateTargetFixup(const MCFixup &Fixup, const MCValue &Target,
-                                   uint64_t &Value) {
-    llvm_unreachable("Need to implement hook if target has custom fixups");
+  // Evaluate a fixup, returning std::nullopt to use default handling for
+  // `Value` and `IsResolved`. Otherwise, returns `IsResolved` with the
+  // expectation that the hook updates `Value`.
+  virtual std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &,
+                                            MCValue &, uint64_t &) {
+    return {};
   }
 
-  virtual bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                        uint64_t &FixedValue, bool IsResolved);
+  void maybeAddReloc(const MCFragment &, const MCFixup &, const MCValue &,
+                     uint64_t &Value, bool IsResolved);
 
+  /// Determine if a relocation is required. In addition,
   /// Apply the \p Value for given \p Fixup into the provided data fragment, at
   /// the offset specified by the fixup and following the fixup kind as
   /// appropriate. Errors (such as an out of range fixup value) should be
@@ -159,7 +174,11 @@ class LLVM_ABI MCAsmBackend {
   /// instruction.
   /// \param STI the subtarget information for the associated instruction.
   virtual void relaxInstruction(MCInst &Inst,
-                                const MCSubtargetInfo &STI) const {};
+                                const MCSubtargetInfo &STI) const {
+    llvm_unreachable(
+        "Needed if fixupNeedsRelaxation/fixupNeedsRelaxationAdvanced may "
+        "return true");
+  }
 
   // Defined by linker relaxation targets.
   virtual bool relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index b09f42deee53..f2b3d6cad74d 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -76,7 +76,6 @@ class LLVM_ABI MCAsmInfo {
     uint32_t Kind;
     StringRef Name;
   };
-  using VariantKindDesc = AtSpecifier;
 
 protected:
   //===------------------------------------------------------------------===//
@@ -432,9 +431,9 @@ class LLVM_ABI MCAsmInfo {
   // If true, use Motorola-style integers in Assembly (ex. $0ac).
   bool UseMotorolaIntegers = false;
 
-  llvm::DenseMap<uint32_t, StringRef> SpecifierToName;
-  llvm::StringMap<uint32_t> NameToSpecifier;
-  void initializeVariantKinds(ArrayRef<VariantKindDesc> Descs);
+  llvm::DenseMap<uint32_t, StringRef> AtSpecifierToName;
+  llvm::StringMap<uint32_t> NameToAtSpecifier;
+  void initializeAtSpecifiers(ArrayRef<AtSpecifier>);
 
 public:
   explicit MCAsmInfo();
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index 5e009090311c..1015992cedf2 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -101,21 +101,18 @@ class MCAssembler {
   /// out.
   /// \param RecordReloc Record relocation if needed.
   /// relocation.
-  bool evaluateFixup(const MCFragment *F, const MCFixup &Fixup, MCValue &Target,
+  bool evaluateFixup(const MCFragment &F, MCFixup &Fixup, MCValue &Target,
                      uint64_t &Value, bool RecordReloc,
                      MutableArrayRef<char> Contents) const;
 
   /// Check whether a fixup can be satisfied, or whether it needs to be relaxed
   /// (increased in size, in order to hold its value correctly).
-  bool fixupNeedsRelaxation(const MCFixup &Fixup, const MCRelaxableFragment *DF) const;
-
-  /// Check whether the given fragment needs relaxation.
-  bool fragmentNeedsRelaxation(const MCRelaxableFragment *IF) const;
+  bool fixupNeedsRelaxation(const MCRelaxableFragment &, const MCFixup &) const;
 
   void layoutSection(MCSection &Sec);
-  /// Perform one layout iteration and return true if any offsets
-  /// were adjusted.
-  bool relaxOnce();
+  /// Perform one layout iteration and return the index of the first stable
+  /// section for subsequent optimization.
+  unsigned relaxOnce(unsigned FirstStable);
 
   /// Perform relaxation on a single fragment.
   bool relaxFragment(MCFragment &F);
diff --git a/llvm/include/llvm/MC/MCCodeView.h b/llvm/include/llvm/MC/MCCodeView.h
index 2a57e04b2c88..88f84a246284 100644
--- a/llvm/include/llvm/MC/MCCodeView.h
+++ b/llvm/include/llvm/MC/MCCodeView.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include <deque>
 #include <map>
 #include <vector>
 
@@ -268,6 +269,10 @@ class CodeViewContext {
   /// Indicate whether we have already laid out the checksum table addresses or
   /// not.
   bool ChecksumOffsetsAssigned = false;
+
+  /// Append-only storage of MCCVDefRangeFragment::Ranges.
+  std::deque<SmallVector<std::pair<const MCSymbol *, const MCSymbol *>, 0>>
+      DefRangeStorage;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 76a750db3fa1..d7b81af4a785 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -19,8 +19,10 @@
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/MC/MCAsmMacro.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSectionGOFF.h"
 #include "llvm/MC/MCSymbolTableEntry.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Support/Allocator.h"
@@ -54,7 +56,6 @@ class MCSection;
 class MCSectionCOFF;
 class MCSectionDXContainer;
 class MCSectionELF;
-class MCSectionGOFF;
 class MCSectionMachO;
 class MCSectionSPIRV;
 class MCSectionWasm;
@@ -356,6 +357,11 @@ class MCContext {
   MCSymbolXCOFF *createXCOFFSymbolImpl(const MCSymbolTableEntry *Name,
                                        bool IsTemporary);
 
+  template <typename TAttr>
+  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                TAttr SDAttributes, MCSection *Parent,
+                                bool IsVirtual);
+
   /// Map of currently defined macros.
   StringMap<MCAsmMacro> MacroMap;
 
@@ -606,9 +612,12 @@ class MCContext {
   getELFUniqueIDForEntsize(StringRef SectionName, unsigned Flags,
                            unsigned EntrySize);
 
-  LLVM_ABI MCSectionGOFF *getGOFFSection(StringRef Section, SectionKind Kind,
-                                         MCSection *Parent,
-                                         uint32_t Subsection = 0);
+  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                GOFF::SDAttr SDAttributes);
+  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                GOFF::EDAttr EDAttributes, MCSection *Parent);
+  MCSectionGOFF *getGOFFSection(SectionKind Kind, StringRef Name,
+                                GOFF::PRAttr PRAttributes, MCSection *Parent);
 
   LLVM_ABI MCSectionCOFF *
   getCOFFSection(StringRef Section, unsigned Characteristics,
diff --git a/llvm/include/llvm/MC/MCDXContainerStreamer.h b/llvm/include/llvm/MC/MCDXContainerStreamer.h
index 8d29bb3d03f6..37e3c7ddaad2 100644
--- a/llvm/include/llvm/MC/MCDXContainerStreamer.h
+++ b/llvm/include/llvm/MC/MCDXContainerStreamer.h
@@ -35,9 +35,6 @@ class MCDXContainerStreamer : public MCObjectStreamer {
 
   bool emitSymbolAttribute(MCSymbol *, MCSymbolAttr) override { return false; }
   void emitCommonSymbol(MCSymbol *, uint64_t, Align) override {}
-
-private:
-  void emitInstToData(const MCInst &, const MCSubtargetInfo &) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index 8d66ed22875d..e460c95ea4df 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -33,6 +33,10 @@ class MCSymbolRefExpr;
 /// needed for parsing.
 class MCExpr {
 public:
+  // Allow MC classes to access the private `print` function.
+  friend class MCAsmInfo;
+  friend class MCFragment;
+  friend class MCOperand;
   enum ExprKind : uint8_t {
     Binary,    ///< Binary expressions.
     Constant,  ///< Constant expressions.
@@ -53,10 +57,13 @@ class MCExpr {
   unsigned SubclassData : NumSubclassDataBits;
   SMLoc Loc;
 
+  void print(raw_ostream &OS, const MCAsmInfo *MAI,
+             int SurroundingPrec = 0) const;
   bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm,
                           bool InSet) const;
 
 protected:
+  using Spec = uint16_t;
   explicit MCExpr(ExprKind Kind, SMLoc Loc, unsigned SubclassData = 0)
       : Kind(Kind), SubclassData(SubclassData), Loc(Loc) {
     assert(SubclassData < (1 << NumSubclassDataBits) &&
@@ -82,9 +89,6 @@ class MCExpr {
   /// \name Utility Methods
   /// @{
 
-  // TODO: Make this private. Users should call MCAsmInfo::printExpr instead.
-  LLVM_ABI void print(raw_ostream &OS, const MCAsmInfo *MAI,
-                      int SurroundingPrec = 0) const;
   LLVM_ABI void dump() const;
 
   /// @}
@@ -134,11 +138,6 @@ class MCExpr {
                                            MCValue &);
 };
 
-inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) {
-  E.print(OS, nullptr);
-  return OS;
-}
-
 ////  Represent a constant integer expression.
 class MCConstantExpr : public MCExpr {
   int64_t Value;
@@ -193,15 +192,10 @@ class MCSymbolRefExpr : public MCExpr {
   // VariantKind isn't ideal for encoding relocation operators because:
   // (a) other expressions, like MCConstantExpr (e.g., 4@l) and MCBinaryExpr
   // (e.g., (a+1)@l), also need it; (b) semantics become unclear (e.g., folding
-  // expressions with @). MCTargetExpr, as used by AArch64 and RISC-V, offers a
-  // cleaner approach.
+  // expressions with @). MCSpecifierExpr, as used by AArch64 and RISC-V, offers
+  // a cleaner approach.
   enum VariantKind : uint16_t {
-    VK_None,
-
-    VK_SECREL,
-    VK_WEAKREF, // The link between the symbols in .weakref foo, bar
-
-    VK_COFF_IMGREL32, // symbol@imgrel (image-relative)
+    VK_COFF_IMGREL32 = 3, // symbol@imgrel (image-relative)
 
     FirstTargetSpecifier,
   };
@@ -210,7 +204,7 @@ class MCSymbolRefExpr : public MCExpr {
   /// The symbol being referenced.
   const MCSymbol *Symbol;
 
-  explicit MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind,
+  explicit MCSymbolRefExpr(const MCSymbol *Symbol, Spec specifier,
                            const MCAsmInfo *MAI, SMLoc Loc = SMLoc());
 
 public:
@@ -219,17 +213,12 @@ class MCSymbolRefExpr : public MCExpr {
 
   static const MCSymbolRefExpr *create(const MCSymbol *Symbol, MCContext &Ctx,
                                        SMLoc Loc = SMLoc()) {
-    return MCSymbolRefExpr::create(Symbol, VK_None, Ctx, Loc);
+    return MCSymbolRefExpr::create(Symbol, 0, Ctx, Loc);
   }
 
   LLVM_ABI static const MCSymbolRefExpr *create(const MCSymbol *Symbol,
-                                                VariantKind Kind,
-                                                MCContext &Ctx,
+                                                Spec specifier, MCContext &Ctx,
                                                 SMLoc Loc = SMLoc());
-  static const MCSymbolRefExpr *create(const MCSymbol *Symbol, uint16_t Kind,
-                                       MCContext &Ctx, SMLoc Loc = SMLoc()) {
-    return MCSymbolRefExpr::create(Symbol, VariantKind(Kind), Ctx, Loc);
-  }
 
   /// @}
   /// \name Accessors
@@ -352,8 +341,8 @@ class MCBinaryExpr : public MCExpr {
                                              SMLoc Loc = SMLoc());
 
   static const MCBinaryExpr *createAdd(const MCExpr *LHS, const MCExpr *RHS,
-                                       MCContext &Ctx) {
-    return create(Add, LHS, RHS, Ctx);
+                                       MCContext &Ctx, SMLoc Loc = SMLoc()) {
+    return create(Add, LHS, RHS, Ctx, Loc);
   }
 
   static const MCBinaryExpr *createAnd(const MCExpr *LHS, const MCExpr *RHS,
@@ -505,7 +494,6 @@ class LLVM_ABI MCTargetExpr : public MCExpr {
 /// MCExprs are bump pointer allocated and not destructed.
 class LLVM_ABI MCSpecifierExpr : public MCExpr {
 protected:
-  using Spec = uint16_t;
   const MCExpr *Expr;
   // Target-specific relocation specifier code
   const Spec specifier;
diff --git a/llvm/include/llvm/MC/MCFixup.h b/llvm/include/llvm/MC/MCFixup.h
index 7575202eaec5..c1fbe7028491 100644
--- a/llvm/include/llvm/MC/MCFixup.h
+++ b/llvm/include/llvm/MC/MCFixup.h
@@ -18,7 +18,8 @@ namespace llvm {
 class MCExpr;
 
 /// Extensible enumeration to represent the type of a fixup.
-enum MCFixupKind : uint16_t {
+using MCFixupKind = uint16_t;
+enum {
   // [0, FirstLiteralRelocationKind) encodes raw relocation types.
 
   // [FirstLiteralRelocationKind, FK_NONE) encodes raw relocation types coming
@@ -34,10 +35,6 @@ enum MCFixupKind : uint16_t {
   FK_Data_4,      ///< A four-byte fixup.
   FK_Data_8,      ///< A eight-byte fixup.
   FK_Data_leb128, ///< A leb128 fixup.
-  FK_PCRel_1,     ///< A one-byte pc relative fixup.
-  FK_PCRel_2,     ///< A two-byte pc relative fixup.
-  FK_PCRel_4,     ///< A four-byte pc relative fixup.
-  FK_PCRel_8,     ///< A eight-byte pc relative fixup.
   FK_SecRel_1,    ///< A one-byte section relative fixup.
   FK_SecRel_2,    ///< A two-byte section relative fixup.
   FK_SecRel_4,    ///< A four-byte section relative fixup.
@@ -73,31 +70,29 @@ class MCFixup {
   /// determine how the operand value should be encoded into the instruction.
   MCFixupKind Kind = FK_NONE;
 
+  /// True if this is a PC-relative fixup. The relocatable expression is
+  /// typically resolved When SymB is nullptr and SymA is a local symbol defined
+  /// within the current section.
+  bool PCRel = false;
+
   /// Used by RISC-V style linker relaxation. Whether the fixup is
   /// linker-relaxable.
   bool LinkerRelaxable = false;
 
   /// Consider bit fields if we need more flags.
 
-  /// The source location which gave rise to the fixup, if any.
-  SMLoc Loc;
 public:
-  static MCFixup create(uint32_t Offset, const MCExpr *Value,
-                        MCFixupKind Kind, SMLoc Loc = SMLoc()) {
+  static MCFixup create(uint32_t Offset, const MCExpr *Value, MCFixupKind Kind,
+                        bool PCRel = false) {
     MCFixup FI;
     FI.Value = Value;
     FI.Offset = Offset;
     FI.Kind = Kind;
-    FI.Loc = Loc;
+    FI.PCRel = PCRel;
     return FI;
   }
-  static MCFixup create(uint32_t Offset, const MCExpr *Value, unsigned Kind,
-                        SMLoc Loc = SMLoc()) {
-    return create(Offset, Value, MCFixupKind(Kind), Loc);
-  }
 
   MCFixupKind getKind() const { return Kind; }
-
   unsigned getTargetKind() const { return Kind; }
 
   uint32_t getOffset() const { return Offset; }
@@ -105,6 +100,8 @@ class MCFixup {
 
   const MCExpr *getValue() const { return Value; }
 
+  bool isPCRel() const { return PCRel; }
+  void setPCRel() { PCRel = true; }
   bool isLinkerRelaxable() const { return LinkerRelaxable; }
   void setLinkerRelaxable() { LinkerRelaxable = true; }
 
@@ -124,7 +121,7 @@ class MCFixup {
     }
   }
 
-  SMLoc getLoc() const { return Loc; }
+  SMLoc getLoc() const;
 };
 
 namespace mc {
diff --git a/llvm/include/llvm/MC/MCFixupKindInfo.h b/llvm/include/llvm/MC/MCFixupKindInfo.h
deleted file mode 100644
index f11cd5caa1d8..000000000000
--- a/llvm/include/llvm/MC/MCFixupKindInfo.h
+++ /dev/null
@@ -1,47 +0,0 @@
-//===-- llvm/MC/MCFixupKindInfo.h - Fixup Descriptors -----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCFIXUPKINDINFO_H
-#define LLVM_MC_MCFIXUPKINDINFO_H
-
-#include <cstdint>
-
-namespace llvm {
-
-/// Target independent information on a fixup kind.
-struct MCFixupKindInfo {
-  enum FixupKindFlags {
-    /// Is this fixup kind PCrelative? This is used by the assembler backend to
-    /// evaluate fixup values in a target independent manner when possible.
-    FKF_IsPCRel = (1 << 0),
-
-    /// Should this fixup kind force a 4-byte aligned effective PC value?
-    FKF_IsAlignedDownTo32Bits = (1 << 1),
-
-    /// Should this fixup be evaluated in a target dependent manner?
-    FKF_IsTarget = (1 << 2),
-  };
-
-  /// A target specific name for the fixup kind. The names will be unique for
-  /// distinct kinds on any given target.
-  const char *Name;
-
-  /// The bit offset to write the relocation into.
-  uint8_t TargetOffset;
-
-  /// The number of bits written by this fixup. The bits are assumed to be
-  /// contiguous.
-  uint8_t TargetSize;
-
-  /// Flags describing additional information on this fixup kind.
-  unsigned Flags;
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
deleted file mode 100644
index 473dec6477df..000000000000
--- a/llvm/include/llvm/MC/MCFragment.h
+++ /dev/null
@@ -1,576 +0,0 @@
-//===- MCFragment.h - Fragment type hierarchy -------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_MC_MCFRAGMENT_H
-#define LLVM_MC_MCFRAGMENT_H
-
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/ilist_node.h"
-#include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/Support/Alignment.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/SMLoc.h"
-#include <cstdint>
-#include <utility>
-
-namespace llvm {
-
-class MCAssembler;
-class MCObjectStreamer;
-class MCSection;
-class MCSubtargetInfo;
-class MCSymbol;
-
-class MCFragment {
-  friend class MCAssembler;
-  friend class MCObjectStreamer;
-  friend class MCSection;
-
-public:
-  enum FragmentType : uint8_t {
-    FT_Align,
-    FT_Data,
-    FT_Fill,
-    FT_Nops,
-    FT_Relaxable,
-    FT_Org,
-    FT_Dwarf,
-    FT_DwarfFrame,
-    FT_LEB,
-    FT_BoundaryAlign,
-    FT_SymbolId,
-    FT_CVInlineLines,
-    FT_CVDefRange,
-    FT_PseudoProbe,
-  };
-
-private:
-  // The next fragment within the section.
-  MCFragment *Next = nullptr;
-
-  /// The data for the section this fragment is in.
-  MCSection *Parent = nullptr;
-
-  /// The offset of this fragment in its section.
-  uint64_t Offset = 0;
-
-  /// The layout order of this fragment.
-  unsigned LayoutOrder = 0;
-
-  FragmentType Kind;
-
-protected:
-  /// Used by subclasses for better packing.
-  ///
-  /// MCEncodedFragment
-  bool HasInstructions : 1;
-  bool AlignToBundleEnd : 1;
-  /// MCDataFragment
-  bool LinkerRelaxable : 1;
-  /// MCRelaxableFragment: x86-specific
-  bool AllowAutoPadding : 1;
-
-  LLVM_ABI MCFragment(FragmentType Kind, bool HasInstructions);
-
-public:
-  MCFragment() = delete;
-  MCFragment(const MCFragment &) = delete;
-  MCFragment &operator=(const MCFragment &) = delete;
-
-  /// Destroys the current fragment.
-  ///
-  /// This must be used instead of delete as MCFragment is non-virtual.
-  /// This method will dispatch to the appropriate subclass.
-  LLVM_ABI void destroy();
-
-  MCFragment *getNext() const { return Next; }
-
-  FragmentType getKind() const { return Kind; }
-
-  MCSection *getParent() const { return Parent; }
-  void setParent(MCSection *Value) { Parent = Value; }
-
-  LLVM_ABI const MCSymbol *getAtom() const;
-
-  unsigned getLayoutOrder() const { return LayoutOrder; }
-  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
-
-  /// Does this fragment have instructions emitted into it? By default
-  /// this is false, but specific fragment types may set it to true.
-  bool hasInstructions() const { return HasInstructions; }
-
-  LLVM_ABI void dump() const;
-};
-
-/// Interface implemented by fragments that contain encoded instructions and/or
-/// data.
-///
-class MCEncodedFragment : public MCFragment {
-  uint8_t BundlePadding = 0;
-
-protected:
-  MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions)
-      : MCFragment(FType, HasInstructions) {}
-
-  /// The MCSubtargetInfo in effect when the instruction was encoded.
-  /// It must be non-null for instructions.
-  const MCSubtargetInfo *STI = nullptr;
-
-public:
-  static bool classof(const MCFragment *F) {
-    MCFragment::FragmentType Kind = F->getKind();
-    switch (Kind) {
-    default:
-      return false;
-    case MCFragment::FT_Relaxable:
-    case MCFragment::FT_Data:
-    case MCFragment::FT_Dwarf:
-    case MCFragment::FT_DwarfFrame:
-    case MCFragment::FT_PseudoProbe:
-      return true;
-    }
-  }
-
-  /// Should this fragment be placed at the end of an aligned bundle?
-  bool alignToBundleEnd() const { return AlignToBundleEnd; }
-  void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; }
-
-  /// Get the padding size that must be inserted before this fragment.
-  /// Used for bundling. By default, no padding is inserted.
-  /// Note that padding size is restricted to 8 bits. This is an optimization
-  /// to reduce the amount of space used for each fragment. In practice, larger
-  /// padding should never be required.
-  uint8_t getBundlePadding() const { return BundlePadding; }
-
-  /// Set the padding size for this fragment. By default it's a no-op,
-  /// and only some fragments have a meaningful implementation.
-  void setBundlePadding(uint8_t N) { BundlePadding = N; }
-
-  /// Retrieve the MCSubTargetInfo in effect when the instruction was encoded.
-  /// Guaranteed to be non-null if hasInstructions() == true
-  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
-
-  /// Record that the fragment contains instructions with the MCSubtargetInfo in
-  /// effect when the instruction was encoded.
-  void setHasInstructions(const MCSubtargetInfo &STI) {
-    HasInstructions = true;
-    this->STI = &STI;
-  }
-};
-
-/// Interface implemented by fragments that contain encoded instructions and/or
-/// data and also have fixups registered.
-///
-template <unsigned ContentsSize, unsigned FixupsSize>
-class MCEncodedFragmentWithFixups : public MCEncodedFragment {
-  SmallVector<char, ContentsSize> Contents;
-
-  /// The list of fixups in this fragment.
-  SmallVector<MCFixup, FixupsSize> Fixups;
-
-protected:
-  MCEncodedFragmentWithFixups(MCFragment::FragmentType FType,
-                              bool HasInstructions)
-      : MCEncodedFragment(FType, HasInstructions) {}
-
-public:
-  SmallVectorImpl<char> &getContents() { return Contents; }
-  const SmallVectorImpl<char> &getContents() const { return Contents; }
-
-  void appendContents(ArrayRef<char> C) { Contents.append(C.begin(), C.end()); }
-  void appendContents(size_t Num, char Elt) { Contents.append(Num, Elt); }
-  void setContents(ArrayRef<char> C) { Contents.assign(C.begin(), C.end()); }
-
-  SmallVectorImpl<MCFixup> &getFixups() { return Fixups; }
-  const SmallVectorImpl<MCFixup> &getFixups() const { return Fixups; }
-
-  static bool classof(const MCFragment *F) {
-    MCFragment::FragmentType Kind = F->getKind();
-    return Kind == MCFragment::FT_Relaxable || Kind == MCFragment::FT_Data ||
-           Kind == MCFragment::FT_CVDefRange || Kind == MCFragment::FT_Dwarf ||
-           Kind == MCFragment::FT_DwarfFrame;
-  }
-};
-
-/// Fragment for data and encoded instructions.
-///
-class MCDataFragment : public MCEncodedFragmentWithFixups<32, 4> {
-public:
-  MCDataFragment() : MCEncodedFragmentWithFixups<32, 4>(FT_Data, false) {}
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Data;
-  }
-
-  bool isLinkerRelaxable() const { return LinkerRelaxable; }
-  void setLinkerRelaxable() { LinkerRelaxable = true; }
-};
-
-/// A relaxable fragment holds on to its MCInst, since it may need to be
-/// relaxed during the assembler layout and relaxation stage.
-///
-class MCRelaxableFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The instruction this is a fragment for.
-  MCInst Inst;
-
-public:
-  MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI)
-      : MCEncodedFragmentWithFixups(FT_Relaxable, true), Inst(Inst) {
-    this->STI = &STI;
-  }
-
-  const MCInst &getInst() const { return Inst; }
-  void setInst(const MCInst &Value) { Inst = Value; }
-
-  bool getAllowAutoPadding() const { return AllowAutoPadding; }
-  void setAllowAutoPadding(bool V) { AllowAutoPadding = V; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Relaxable;
-  }
-};
-
-class MCAlignFragment : public MCFragment {
-  /// The alignment to ensure, in bytes.
-  Align Alignment;
-
-  /// Flag to indicate that (optimal) NOPs should be emitted instead
-  /// of using the provided value. The exact interpretation of this flag is
-  /// target dependent.
-  bool EmitNops : 1;
-
-  /// Value to use for filling padding bytes.
-  int64_t Value;
-
-  /// The size of the integer (in bytes) of \p Value.
-  unsigned ValueSize;
-
-  /// The maximum number of bytes to emit; if the alignment
-  /// cannot be satisfied in this width then this fragment is ignored.
-  unsigned MaxBytesToEmit;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo *STI = nullptr;
-
-public:
-  MCAlignFragment(Align Alignment, int64_t Value, unsigned ValueSize,
-                  unsigned MaxBytesToEmit)
-      : MCFragment(FT_Align, false), Alignment(Alignment), EmitNops(false),
-        Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {}
-
-  Align getAlignment() const { return Alignment; }
-
-  int64_t getValue() const { return Value; }
-
-  unsigned getValueSize() const { return ValueSize; }
-
-  unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; }
-
-  bool hasEmitNops() const { return EmitNops; }
-  void setEmitNops(bool Value, const MCSubtargetInfo *STI) {
-    EmitNops = Value;
-    this->STI = STI;
-  }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Align;
-  }
-};
-
-class MCFillFragment : public MCFragment {
-  uint8_t ValueSize;
-  /// Value to use for filling bytes.
-  uint64_t Value;
-  /// The number of bytes to insert.
-  const MCExpr &NumValues;
-  uint64_t Size = 0;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-public:
-  MCFillFragment(uint64_t Value, uint8_t VSize, const MCExpr &NumValues,
-                 SMLoc Loc)
-      : MCFragment(FT_Fill, false), ValueSize(VSize), Value(Value),
-        NumValues(NumValues), Loc(Loc) {}
-
-  uint64_t getValue() const { return Value; }
-  uint8_t getValueSize() const { return ValueSize; }
-  const MCExpr &getNumValues() const { return NumValues; }
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Fill;
-  }
-};
-
-class MCNopsFragment : public MCFragment {
-  /// The number of bytes to insert.
-  int64_t Size;
-  /// Maximum number of bytes allowed in each NOP instruction.
-  int64_t ControlledNopLength;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo &STI;
-
-public:
-  MCNopsFragment(int64_t NumBytes, int64_t ControlledNopLength, SMLoc L,
-                 const MCSubtargetInfo &STI)
-      : MCFragment(FT_Nops, false), Size(NumBytes),
-        ControlledNopLength(ControlledNopLength), Loc(L), STI(STI) {}
-
-  int64_t getNumBytes() const { return Size; }
-  int64_t getControlledNopLength() const { return ControlledNopLength; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Nops;
-  }
-};
-
-class MCOrgFragment : public MCFragment {
-  /// Value to use for filling bytes.
-  int8_t Value;
-
-  /// The offset this fragment should start at.
-  const MCExpr *Offset;
-
-  /// Source location of the directive that this fragment was created for.
-  SMLoc Loc;
-
-public:
-  MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc)
-      : MCFragment(FT_Org, false), Value(Value), Offset(&Offset), Loc(Loc) {}
-
-  const MCExpr &getOffset() const { return *Offset; }
-
-  uint8_t getValue() const { return Value; }
-
-  SMLoc getLoc() const { return Loc; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Org;
-  }
-};
-
-class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> {
-  /// True if this is a sleb128, false if uleb128.
-  bool IsSigned;
-
-  /// The value this fragment should contain.
-  const MCExpr *Value;
-
-public:
-  MCLEBFragment(const MCExpr &Value, bool IsSigned)
-      : MCEncodedFragmentWithFixups<8, 0>(FT_LEB, false), IsSigned(IsSigned),
-        Value(&Value) {
-    getContents().push_back(0);
-  }
-
-  const MCExpr &getValue() const { return *Value; }
-  void setValue(const MCExpr *Expr) { Value = Expr; }
-
-  bool isSigned() const { return IsSigned; }
-
-  /// @}
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_LEB;
-  }
-};
-
-class MCDwarfLineAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The value of the difference between the two line numbers
-  /// between two .loc dwarf directives.
-  int64_t LineDelta;
-
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .loc dwarf directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_Dwarf, false),
-        LineDelta(LineDelta), AddrDelta(&AddrDelta) {}
-
-  int64_t getLineDelta() const { return LineDelta; }
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_Dwarf;
-  }
-};
-
-class MCDwarfCallFrameFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .cfi_* dwarf directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCDwarfCallFrameFragment(const MCExpr &AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_DwarfFrame, false),
-        AddrDelta(&AddrDelta) {}
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-  void setAddrDelta(const MCExpr *E) { AddrDelta = E; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_DwarfFrame;
-  }
-};
-
-/// Represents a symbol table index fragment.
-class MCSymbolIdFragment : public MCFragment {
-  const MCSymbol *Sym;
-
-public:
-  MCSymbolIdFragment(const MCSymbol *Sym)
-      : MCFragment(FT_SymbolId, false), Sym(Sym) {}
-
-  const MCSymbol *getSymbol() { return Sym; }
-  const MCSymbol *getSymbol() const { return Sym; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_SymbolId;
-  }
-};
-
-/// Fragment representing the binary annotations produced by the
-/// .cv_inline_linetable directive.
-class MCCVInlineLineTableFragment : public MCFragment {
-  unsigned SiteFuncId;
-  unsigned StartFileId;
-  unsigned StartLineNum;
-  const MCSymbol *FnStartSym;
-  const MCSymbol *FnEndSym;
-  SmallString<8> Contents;
-
-  /// CodeViewContext has the real knowledge about this format, so let it access
-  /// our members.
-  friend class CodeViewContext;
-
-public:
-  MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId,
-                              unsigned StartLineNum, const MCSymbol *FnStartSym,
-                              const MCSymbol *FnEndSym)
-      : MCFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
-        StartFileId(StartFileId), StartLineNum(StartLineNum),
-        FnStartSym(FnStartSym), FnEndSym(FnEndSym) {}
-
-  const MCSymbol *getFnStartSym() const { return FnStartSym; }
-  const MCSymbol *getFnEndSym() const { return FnEndSym; }
-
-  SmallString<8> &getContents() { return Contents; }
-  const SmallString<8> &getContents() const { return Contents; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_CVInlineLines;
-  }
-};
-
-/// Fragment representing the .cv_def_range directive.
-class MCCVDefRangeFragment : public MCEncodedFragmentWithFixups<32, 4> {
-  SmallVector<std::pair<const MCSymbol *, const MCSymbol *>, 2> Ranges;
-  SmallString<32> FixedSizePortion;
-
-  /// CodeViewContext has the real knowledge about this format, so let it access
-  /// our members.
-  friend class CodeViewContext;
-
-public:
-  MCCVDefRangeFragment(
-      ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
-      StringRef FixedSizePortion)
-      : MCEncodedFragmentWithFixups<32, 4>(FT_CVDefRange, false),
-        Ranges(Ranges), FixedSizePortion(FixedSizePortion) {}
-
-  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const {
-    return Ranges;
-  }
-
-  StringRef getFixedSizePortion() const { return FixedSizePortion.str(); }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_CVDefRange;
-  }
-};
-
-/// Represents required padding such that a particular other set of fragments
-/// does not cross a particular power-of-two boundary. The other fragments must
-/// follow this one within the same section.
-class MCBoundaryAlignFragment : public MCFragment {
-  /// The alignment requirement of the branch to be aligned.
-  Align AlignBoundary;
-  /// The last fragment in the set of fragments to be aligned.
-  const MCFragment *LastFragment = nullptr;
-  /// The size of the fragment.  The size is lazily set during relaxation, and
-  /// is not meaningful before that.
-  uint64_t Size = 0;
-
-  /// When emitting Nops some subtargets have specific nop encodings.
-  const MCSubtargetInfo &STI;
-
-public:
-  MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
-      : MCFragment(FT_BoundaryAlign, false), AlignBoundary(AlignBoundary),
-        STI(STI) {}
-
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
-
-  Align getAlignment() const { return AlignBoundary; }
-  void setAlignment(Align Value) { AlignBoundary = Value; }
-
-  const MCFragment *getLastFragment() const { return LastFragment; }
-  void setLastFragment(const MCFragment *F) {
-    assert(!F || getParent() == F->getParent());
-    LastFragment = F;
-  }
-
-  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_BoundaryAlign;
-  }
-};
-
-class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
-  /// The expression for the difference of the two symbols that
-  /// make up the address delta between two .pseudoprobe directives.
-  const MCExpr *AddrDelta;
-
-public:
-  MCPseudoProbeAddrFragment(const MCExpr *AddrDelta)
-      : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false),
-        AddrDelta(AddrDelta) {}
-
-  const MCExpr &getAddrDelta() const { return *AddrDelta; }
-
-  static bool classof(const MCFragment *F) {
-    return F->getKind() == MCFragment::FT_PseudoProbe;
-  }
-};
-} // end namespace llvm
-
-#endif // LLVM_MC_MCFRAGMENT_H
diff --git a/llvm/include/llvm/MC/MCGOFFAttributes.h b/llvm/include/llvm/MC/MCGOFFAttributes.h
new file mode 100644
index 000000000000..c996f0cae2c6
--- /dev/null
+++ b/llvm/include/llvm/MC/MCGOFFAttributes.h
@@ -0,0 +1,92 @@
+//===- MCGOFFAttributes.h - Attributes of GOFF symbols --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the various attribute collections defining GOFF symbols.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCGOFFATTRIBUTES_H
+#define LLVM_MC_MCGOFFATTRIBUTES_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include <cstdint>
+
+namespace llvm {
+namespace GOFF {
+// An "External Symbol Definition" in the GOFF file has a type, and depending on
+// the type a different subset of the fields is used.
+//
+// Unlike other formats, a 2 dimensional structure is used to define the
+// location of data. For example, the equivalent of the ELF .text section is
+// made up of a Section Definition (SD) and a class (Element Definition; ED).
+// The name of the SD symbol depends on the application, while the class has the
+// predefined name C_CODE/C_CODE64 in AMODE31 and AMODE64 respectively.
+//
+// Data can be placed into this structure in 2 ways. First, the data (in a text
+// record) can be associated with an ED symbol. To refer to data, a Label
+// Definition (LD) is used to give an offset into the data a name. When binding,
+// the whole data is pulled into the resulting executable, and the addresses
+// given by the LD symbols are resolved.
+//
+// The alternative is to use a Part Definition (PR). In this case, the data (in
+// a text record) is associated with the part. When binding, only the data of
+// referenced PRs is pulled into the resulting binary.
+//
+// Both approaches are used. SD, ED, and PR elements are modelled by nested
+// MCSectionGOFF instances, while LD elements are associated with MCSymbolGOFF
+// instances.
+
+// Attributes for SD symbols.
+struct SDAttr {
+  GOFF::ESDTaskingBehavior TaskingBehavior = GOFF::ESD_TA_Unspecified;
+  GOFF::ESDBindingScope BindingScope = GOFF::ESD_BSC_Unspecified;
+};
+
+// Attributes for ED symbols.
+struct EDAttr {
+  bool IsReadOnly = false;
+  GOFF::ESDRmode Rmode;
+  GOFF::ESDNameSpaceId NameSpace = GOFF::ESD_NS_NormalName;
+  GOFF::ESDTextStyle TextStyle = GOFF::ESD_TS_ByteOriented;
+  GOFF::ESDBindingAlgorithm BindAlgorithm = GOFF::ESD_BA_Concatenate;
+  GOFF::ESDLoadingBehavior LoadBehavior = GOFF::ESD_LB_Initial;
+  GOFF::ESDReserveQwords ReservedQwords = GOFF::ESD_RQ_0;
+  GOFF::ESDAlignment Alignment = GOFF::ESD_ALIGN_Doubleword;
+  uint8_t FillByteValue = 0;
+};
+
+// Attributes for LD symbols.
+struct LDAttr {
+  bool IsRenamable = false;
+  GOFF::ESDExecutable Executable = GOFF::ESD_EXE_Unspecified;
+  GOFF::ESDBindingStrength BindingStrength = GOFF::ESD_BST_Strong;
+  GOFF::ESDLinkageType Linkage = GOFF::ESD_LT_XPLink;
+  GOFF::ESDAmode Amode;
+  GOFF::ESDBindingScope BindingScope = GOFF::ESD_BSC_Unspecified;
+};
+
+// Attributes for PR symbols.
+struct PRAttr {
+  bool IsRenamable = false;
+  GOFF::ESDExecutable Executable = GOFF::ESD_EXE_Unspecified;
+  GOFF::ESDLinkageType Linkage = GOFF::ESD_LT_XPLink;
+  GOFF::ESDBindingScope BindingScope = GOFF::ESD_BSC_Unspecified;
+  uint32_t SortKey = 0;
+};
+
+// Predefined GOFF class names.
+constexpr StringLiteral CLASS_CODE = "C_CODE64";
+constexpr StringLiteral CLASS_WSA = "C_WSA64";
+constexpr StringLiteral CLASS_DATA = "C_DATA64";
+constexpr StringLiteral CLASS_PPA2 = "C_@@QPPA2";
+
+} // namespace GOFF
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/MC/MCGOFFObjectWriter.h b/llvm/include/llvm/MC/MCGOFFObjectWriter.h
index 85316a6fd5d0..78a491522dbd 100644
--- a/llvm/include/llvm/MC/MCGOFFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCGOFFObjectWriter.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCGOFFOBJECTWRITER_H
 
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCValue.h"
 
 namespace llvm {
 class MCObjectWriter;
@@ -29,6 +30,25 @@ class MCGOFFObjectTargetWriter : public MCObjectTargetWriter {
   }
 };
 
+class GOFFObjectWriter : public MCObjectWriter {
+  // The target specific GOFF writer instance.
+  std::unique_ptr<MCGOFFObjectTargetWriter> TargetObjectWriter;
+
+  // The stream used to write the GOFF records.
+  raw_pwrite_stream &OS;
+
+public:
+  GOFFObjectWriter(std::unique_ptr<MCGOFFObjectTargetWriter> MOTW,
+                   raw_pwrite_stream &OS);
+  ~GOFFObjectWriter() override;
+
+  // Implementation of the MCObjectWriter interface.
+  void recordRelocation(const MCFragment &F, const MCFixup &Fixup,
+                        MCValue Target, uint64_t &FixedValue) override {}
+
+  uint64_t writeObject() override;
+};
+
 /// \brief Construct a new GOFF writer instance.
 ///
 /// \param MOTW - The target-specific GOFF writer subclass.
diff --git a/llvm/include/llvm/MC/MCGOFFStreamer.h b/llvm/include/llvm/MC/MCGOFFStreamer.h
index 7f8f33d8ee4b..6d029f6bd4a2 100644
--- a/llvm/include/llvm/MC/MCGOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCGOFFStreamer.h
@@ -13,8 +13,10 @@
 #include "llvm/MC/MCObjectWriter.h"
 
 namespace llvm {
+class GOFFObjectWriter;
 
 class MCGOFFStreamer : public MCObjectStreamer {
+
 public:
   MCGOFFStreamer(MCContext &Context, std::unique_ptr<MCAsmBackend> MAB,
                  std::unique_ptr<MCObjectWriter> OW,
@@ -24,12 +26,15 @@ class MCGOFFStreamer : public MCObjectStreamer {
 
   ~MCGOFFStreamer() override;
 
+  void changeSection(MCSection *Section, uint32_t Subsection = 0) override;
+
+  GOFFObjectWriter &getWriter();
+
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override {
     return false;
   }
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override {}
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override {}
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCInst.h b/llvm/include/llvm/MC/MCInst.h
index 581dc609b913..f26af88d9214 100644
--- a/llvm/include/llvm/MC/MCInst.h
+++ b/llvm/include/llvm/MC/MCInst.h
@@ -27,6 +27,7 @@
 
 namespace llvm {
 
+class MCContext;
 class MCExpr;
 class MCInst;
 class MCInstPrinter;
@@ -175,8 +176,7 @@ class MCOperand {
     return Op;
   }
 
-  LLVM_ABI void print(raw_ostream &OS,
-                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void print(raw_ostream &OS, const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump() const;
   LLVM_ABI bool isBareSymbolRef() const;
   LLVM_ABI bool evaluateAsConstantImm(int64_t &Imm) const;
@@ -228,8 +228,7 @@ class MCInst {
     return Operands.insert(I, Op);
   }
 
-  LLVM_ABI void print(raw_ostream &OS,
-                      const MCRegisterInfo *RegInfo = nullptr) const;
+  LLVM_ABI void print(raw_ostream &OS, const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump() const;
 
   /// Dump the MCInst as prettily as possible using the additional MC
@@ -238,10 +237,10 @@ class MCInst {
   LLVM_ABI void dump_pretty(raw_ostream &OS,
                             const MCInstPrinter *Printer = nullptr,
                             StringRef Separator = " ",
-                            const MCRegisterInfo *RegInfo = nullptr) const;
+                            const MCContext *Ctx = nullptr) const;
   LLVM_ABI void dump_pretty(raw_ostream &OS, StringRef Name,
                             StringRef Separator = " ",
-                            const MCRegisterInfo *RegInfo = nullptr) const;
+                            const MCContext *Ctx = nullptr) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const MCOperand &MO) {
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index a6ccdf06eb91..51e4df59bea4 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -191,8 +191,6 @@ class LLVM_ABI MachObjectWriter final : public MCObjectWriter {
   /// \name Utility Methods
   /// @{
 
-  bool isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind);
-
   std::vector<IndirectSymbolData> &getIndirectSymbols() {
     return IndirectSymbols;
   }
diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h
index 20c819bdfd96..5ce58ae0a836 100644
--- a/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -231,8 +231,6 @@ class LLVM_ABI MCObjectFileInfo {
   MCSection *GLJMPSection = nullptr;
 
   // GOFF specific sections.
-  MCSection *PPA1Section = nullptr;
-  MCSection *PPA2Section = nullptr;
   MCSection *PPA2ListSection = nullptr;
   MCSection *ADASection = nullptr;
   MCSection *IDRLSection = nullptr;
@@ -439,8 +437,6 @@ class LLVM_ABI MCObjectFileInfo {
   MCSection *getGLJMPSection() const { return GLJMPSection; }
 
   // GOFF specific sections.
-  MCSection *getPPA1Section() const { return PPA1Section; }
-  MCSection *getPPA2Section() const { return PPA2Section; }
   MCSection *getPPA2ListSection() const { return PPA2ListSection; }
   MCSection *getADASection() const { return ADASection; }
   MCSection *getIDRLSection() const { return IDRLSection; }
diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h
index c987bc2426e9..58c18af40615 100644
--- a/llvm/include/llvm/MC/MCObjectStreamer.h
+++ b/llvm/include/llvm/MC/MCObjectStreamer.h
@@ -12,7 +12,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 
@@ -60,7 +59,7 @@ class MCObjectStreamer : public MCStreamer {
   DenseMap<const MCSymbol *, SmallVector<PendingAssignment, 1>>
       pendingAssignments;
 
-  virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0;
+  virtual void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &);
   void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override;
   void emitInstructionImpl(const MCInst &Inst, const MCSubtargetInfo &STI);
@@ -126,7 +125,7 @@ class MCObjectStreamer : public MCStreamer {
 
   /// Emit an instruction to a special fragment, because this instruction
   /// can change its size during relaxation.
-  virtual void emitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &);
+  void emitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &);
 
   void emitBundleAlignMode(Align Alignment) override;
   void emitBundleLock(bool AlignToEnd) override;
diff --git a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index fd52fbac5199..27ed2a627649 100644
--- a/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/llvm/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -16,6 +16,7 @@
 
 namespace llvm {
 
+class MCAsmInfo;
 class MCRegister;
 class raw_ostream;
 
@@ -89,20 +90,11 @@ class LLVM_ABI MCParsedAsmOperand {
   virtual SMLoc getOffsetOfLoc() const { return SMLoc(); }
 
   /// print - Print a debug representation of the operand to the given stream.
-  virtual void print(raw_ostream &OS) const = 0;
+  virtual void print(raw_ostream &, const MCAsmInfo &) const = 0;
 
   /// dump - Print to the debug stream.
   virtual void dump() const;
 };
-
-//===----------------------------------------------------------------------===//
-// Debugging Support
-
-inline raw_ostream& operator<<(raw_ostream &OS, const MCParsedAsmOperand &MO) {
-  MO.print(OS);
-  return OS;
-}
-
 } // end namespace llvm
 
 #endif // LLVM_MC_MCPARSER_MCPARSEDASMOPERAND_H
diff --git a/llvm/include/llvm/MC/MCSPIRVStreamer.h b/llvm/include/llvm/MC/MCSPIRVStreamer.h
index ee0544147073..42559f7522b5 100644
--- a/llvm/include/llvm/MC/MCSPIRVStreamer.h
+++ b/llvm/include/llvm/MC/MCSPIRVStreamer.h
@@ -35,9 +35,6 @@ class MCSPIRVStreamer : public MCObjectStreamer {
   }
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override {}
-
-private:
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCSchedule.h b/llvm/include/llvm/MC/MCSchedule.h
index eb71f4581bd6..d41a7412a983 100644
--- a/llvm/include/llvm/MC/MCSchedule.h
+++ b/llvm/include/llvm/MC/MCSchedule.h
@@ -15,6 +15,7 @@
 #define LLVM_MC_MCSCHEDULE_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringTable.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -124,7 +125,7 @@ struct MCSchedClassDesc {
   static const unsigned short VariantNumMicroOps = InvalidNumMicroOps - 1;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  const char* Name;
+  uint32_t NameOff;
 #endif
   uint16_t NumMicroOps : 13;
   uint16_t BeginGroup : 1;
@@ -324,6 +325,7 @@ struct MCSchedModel {
   const MCSchedClassDesc *SchedClassTable;
   unsigned NumProcResourceKinds;
   unsigned NumSchedClasses;
+  const StringTable *SchedClassNames;
   // Instruction itinerary tables used by InstrItineraryData.
   friend class InstrItineraryData;
   const InstrItinerary *InstrItineraries;
@@ -368,6 +370,14 @@ struct MCSchedModel {
     return &SchedClassTable[SchedClassIdx];
   }
 
+  StringRef getSchedClassName(unsigned SchedClassIdx) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+    return (*SchedClassNames)[SchedClassTable[SchedClassIdx].NameOff];
+#else
+    return "<unknown>";
+#endif
+  }
+
   /// Returns the latency value for the scheduling class.
   LLVM_ABI static int computeInstrLatency(const MCSubtargetInfo &STI,
                                           const MCSchedClassDesc &SCDesc);
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index 7a94a7d65613..3abe5e0fe3c6 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -13,8 +13,12 @@
 #ifndef LLVM_MC_MCSECTION_H
 #define LLVM_MC_MCSECTION_H
 
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/MC/MCFragment.h"
+#include "llvm/MC/MCFixup.h"
+#include "llvm/MC/MCInst.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Compiler.h"
@@ -29,15 +33,106 @@ class MCContext;
 class MCExpr;
 class MCObjectStreamer;
 class MCSymbol;
+class MCSection;
+class MCSubtargetInfo;
 class raw_ostream;
 class Triple;
 
+// Represents a contiguous piece of code or data within a section. Its size is
+// determined by MCAssembler::layout. All subclasses (except
+// MCRelaxableFragment, which stores a MCInst) must have trivial destructors.
+//
+// Declaration order: MCFragment, MCSection, then MCFragment's derived classes.
+// This allows MCSection's inline functions to access MCFragment members and
+// allows MCFragment's derived classes to access MCSection.
+class MCFragment {
+  friend class MCAssembler;
+  friend class MCObjectStreamer;
+  friend class MCSection;
+
+public:
+  enum FragmentType : uint8_t {
+    FT_Data,
+    FT_Relaxable,
+    FT_Align,
+    FT_Fill,
+    FT_LEB,
+    FT_Nops,
+    FT_Org,
+    FT_Dwarf,
+    FT_DwarfFrame,
+    FT_BoundaryAlign,
+    FT_SymbolId,
+    FT_CVInlineLines,
+    FT_CVDefRange,
+    FT_PseudoProbe,
+  };
+
+private:
+  // The next fragment within the section.
+  MCFragment *Next = nullptr;
+
+  /// The data for the section this fragment is in.
+  MCSection *Parent = nullptr;
+
+  /// The offset of this fragment in its section.
+  uint64_t Offset = 0;
+
+  /// The layout order of this fragment.
+  unsigned LayoutOrder = 0;
+
+  FragmentType Kind;
+
+protected:
+  /// Used by subclasses for better packing.
+  ///
+  /// MCEncodedFragment
+  bool HasInstructions : 1;
+  bool AlignToBundleEnd : 1;
+  /// MCDataFragment
+  bool LinkerRelaxable : 1;
+  /// MCRelaxableFragment: x86-specific
+  bool AllowAutoPadding : 1;
+
+  LLVM_ABI MCFragment(FragmentType Kind, bool HasInstructions);
+
+public:
+  MCFragment() = delete;
+  MCFragment(const MCFragment &) = delete;
+  MCFragment &operator=(const MCFragment &) = delete;
+
+  /// Destroys the current fragment.
+  ///
+  /// This must be used instead of delete as MCFragment is non-virtual.
+  /// This method will dispatch to the appropriate subclass.
+  LLVM_ABI void destroy();
+
+  MCFragment *getNext() const { return Next; }
+
+  FragmentType getKind() const { return Kind; }
+
+  MCSection *getParent() const { return Parent; }
+  void setParent(MCSection *Value) { Parent = Value; }
+
+  LLVM_ABI const MCSymbol *getAtom() const;
+
+  unsigned getLayoutOrder() const { return LayoutOrder; }
+  void setLayoutOrder(unsigned Value) { LayoutOrder = Value; }
+
+  /// Does this fragment have instructions emitted into it? By default
+  /// this is false, but specific fragment types may set it to true.
+  bool hasInstructions() const { return HasInstructions; }
+
+  LLVM_ABI void dump() const;
+};
+
 /// Instances of this class represent a uniqued identifier for a section in the
 /// current translation unit.  The MCContext class uniques and creates these.
 class LLVM_ABI MCSection {
 public:
   friend MCAssembler;
   friend MCObjectStreamer;
+  friend class MCEncodedFragment;
   static constexpr unsigned NonUniqueID = ~0U;
 
   enum SectionVariant {
@@ -115,6 +210,10 @@ class LLVM_ABI MCSection {
   // subsections.
   SmallVector<std::pair<unsigned, FragList>, 1> Subsections;
 
+  // Content and fixup storage for fragments
+  SmallVector<char, 0> ContentStorage;
+  SmallVector<MCFixup, 0> FixupStorage;
+
 protected:
   // TODO Make Name private when possible.
   StringRef Name;
@@ -182,7 +281,8 @@ class LLVM_ABI MCSection {
   iterator begin() const { return iterator(CurFragList->Head); }
   iterator end() const { return {}; }
 
-  void dump() const;
+  void dump(DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>>
+                *FragToSyms = nullptr) const;
 
   virtual void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
                                     raw_ostream &OS,
@@ -199,6 +299,480 @@ class LLVM_ABI MCSection {
   virtual StringRef getVirtualSectionKind() const;
 };
 
+/// Interface implemented by fragments that contain encoded instructions and/or
+/// data.
+class MCEncodedFragment : public MCFragment {
+  uint8_t BundlePadding = 0;
+  uint32_t ContentStart = 0;
+  uint32_t ContentEnd = 0;
+  uint32_t FixupStart = 0;
+  uint32_t FixupEnd = 0;
+
+protected:
+  MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions)
+      : MCFragment(FType, HasInstructions) {}
+
+  /// The MCSubtargetInfo in effect when the instruction was encoded.
+  /// It must be non-null for instructions.
+  const MCSubtargetInfo *STI = nullptr;
+
+public:
+  static bool classof(const MCFragment *F) {
+    MCFragment::FragmentType Kind = F->getKind();
+    switch (Kind) {
+    default:
+      return false;
+    case MCFragment::FT_Relaxable:
+    case MCFragment::FT_Data:
+    case MCFragment::FT_Dwarf:
+    case MCFragment::FT_DwarfFrame:
+    case MCFragment::FT_LEB:
+    case MCFragment::FT_PseudoProbe:
+    case MCFragment::FT_CVInlineLines:
+    case MCFragment::FT_CVDefRange:
+      return true;
+    }
+  }
+
+  /// Should this fragment be placed at the end of an aligned bundle?
+  bool alignToBundleEnd() const { return AlignToBundleEnd; }
+  void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; }
+
+  /// Get the padding size that must be inserted before this fragment.
+  /// Used for bundling. By default, no padding is inserted.
+  /// Note that padding size is restricted to 8 bits. This is an optimization
+  /// to reduce the amount of space used for each fragment. In practice, larger
+  /// padding should never be required.
+  uint8_t getBundlePadding() const { return BundlePadding; }
+
+  /// Set the padding size for this fragment. By default it's a no-op,
+  /// and only some fragments have a meaningful implementation.
+  void setBundlePadding(uint8_t N) { BundlePadding = N; }
+
+  /// Retrieve the MCSubTargetInfo in effect when the instruction was encoded.
+  /// Guaranteed to be non-null if hasInstructions() == true
+  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
+
+  /// Record that the fragment contains instructions with the MCSubtargetInfo in
+  /// effect when the instruction was encoded.
+  void setHasInstructions(const MCSubtargetInfo &STI) {
+    HasInstructions = true;
+    this->STI = &STI;
+  }
+
+  // Content-related functions manage parent's storage using ContentStart and
+  // ContentSize.
+  void clearContents() { ContentEnd = ContentStart; }
+  // Get a SmallVector reference. The caller should call doneAppending to update
+  // `ContentEnd`.
+  SmallVectorImpl<char> &getContentsForAppending() {
+    SmallVectorImpl<char> &S = getParent()->ContentStorage;
+    if (LLVM_UNLIKELY(ContentEnd != S.size())) {
+      // Move the elements to the end. Reserve space to avoid invalidating
+      // S.begin()+I for `append`.
+      auto Size = ContentEnd - ContentStart;
+      auto I = std::exchange(ContentStart, S.size());
+      S.reserve(S.size() + Size);
+      S.append(S.begin() + I, S.begin() + I + Size);
+    }
+    return S;
+  }
+  void doneAppending() { ContentEnd = getParent()->ContentStorage.size(); }
+  void appendContents(ArrayRef<char> Contents) {
+    getContentsForAppending().append(Contents.begin(), Contents.end());
+    doneAppending();
+  }
+  void appendContents(size_t Num, char Elt) {
+    getContentsForAppending().append(Num, Elt);
+    doneAppending();
+  }
+  void setContents(ArrayRef<char> Contents);
+  MutableArrayRef<char> getContents() {
+    return MutableArrayRef(getParent()->ContentStorage)
+        .slice(ContentStart, ContentEnd - ContentStart);
+  }
+  ArrayRef<char> getContents() const {
+    return ArrayRef(getParent()->ContentStorage)
+        .slice(ContentStart, ContentEnd - ContentStart);
+  }
+
+  // Fixup-related functions manage parent's storage using FixupStart and
+  // FixupSize.
+  void clearFixups() { FixupEnd = FixupStart; }
+  void addFixup(MCFixup Fixup);
+  void appendFixups(ArrayRef<MCFixup> Fixups);
+  void setFixups(ArrayRef<MCFixup> Fixups);
+  MutableArrayRef<MCFixup> getFixups() {
+    return MutableArrayRef(getParent()->FixupStorage)
+        .slice(FixupStart, FixupEnd - FixupStart);
+  }
+  ArrayRef<MCFixup> getFixups() const {
+    return ArrayRef(getParent()->FixupStorage)
+        .slice(FixupStart, FixupEnd - FixupStart);
+  }
+};
+
+/// Fragment for data and encoded instructions.
+///
+class MCDataFragment : public MCEncodedFragment {
+public:
+  MCDataFragment() : MCEncodedFragment(FT_Data, false) {}
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Data;
+  }
+
+  bool isLinkerRelaxable() const { return LinkerRelaxable; }
+  void setLinkerRelaxable() { LinkerRelaxable = true; }
+};
+
+/// A relaxable fragment holds on to its MCInst, since it may need to be
+/// relaxed during the assembler layout and relaxation stage.
+///
+class MCRelaxableFragment : public MCEncodedFragment {
+  /// The instruction this is a fragment for.
+  MCInst Inst;
+
+public:
+  MCRelaxableFragment(const MCInst &Inst, const MCSubtargetInfo &STI)
+      : MCEncodedFragment(FT_Relaxable, true), Inst(Inst) {
+    this->STI = &STI;
+  }
+
+  const MCInst &getInst() const { return Inst; }
+  void setInst(const MCInst &Value) { Inst = Value; }
+
+  bool getAllowAutoPadding() const { return AllowAutoPadding; }
+  void setAllowAutoPadding(bool V) { AllowAutoPadding = V; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Relaxable;
+  }
+};
+
+class MCAlignFragment : public MCFragment {
+  /// The alignment to ensure, in bytes.
+  Align Alignment;
+
+  /// Flag to indicate that (optimal) NOPs should be emitted instead
+  /// of using the provided value. The exact interpretation of this flag is
+  /// target dependent.
+  bool EmitNops : 1;
+
+  /// Value to use for filling padding bytes.
+  int64_t Value;
+
+  /// The size of the integer (in bytes) of \p Value.
+  unsigned ValueSize;
+
+  /// The maximum number of bytes to emit; if the alignment
+  /// cannot be satisfied in this width then this fragment is ignored.
+  unsigned MaxBytesToEmit;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo *STI = nullptr;
+
+public:
+  MCAlignFragment(Align Alignment, int64_t Value, unsigned ValueSize,
+                  unsigned MaxBytesToEmit)
+      : MCFragment(FT_Align, false), Alignment(Alignment), EmitNops(false),
+        Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {}
+
+  Align getAlignment() const { return Alignment; }
+
+  int64_t getValue() const { return Value; }
+
+  unsigned getValueSize() const { return ValueSize; }
+
+  unsigned getMaxBytesToEmit() const { return MaxBytesToEmit; }
+
+  bool hasEmitNops() const { return EmitNops; }
+  void setEmitNops(bool Value, const MCSubtargetInfo *STI) {
+    EmitNops = Value;
+    this->STI = STI;
+  }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Align;
+  }
+};
+
+class MCFillFragment : public MCFragment {
+  uint8_t ValueSize;
+  /// Value to use for filling bytes.
+  uint64_t Value;
+  /// The number of bytes to insert.
+  const MCExpr &NumValues;
+  uint64_t Size = 0;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+public:
+  MCFillFragment(uint64_t Value, uint8_t VSize, const MCExpr &NumValues,
+                 SMLoc Loc)
+      : MCFragment(FT_Fill, false), ValueSize(VSize), Value(Value),
+        NumValues(NumValues), Loc(Loc) {}
+
+  uint64_t getValue() const { return Value; }
+  uint8_t getValueSize() const { return ValueSize; }
+  const MCExpr &getNumValues() const { return NumValues; }
+  uint64_t getSize() const { return Size; }
+  void setSize(uint64_t Value) { Size = Value; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Fill;
+  }
+};
+
+class MCNopsFragment : public MCFragment {
+  /// The number of bytes to insert.
+  int64_t Size;
+  /// Maximum number of bytes allowed in each NOP instruction.
+  int64_t ControlledNopLength;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo &STI;
+
+public:
+  MCNopsFragment(int64_t NumBytes, int64_t ControlledNopLength, SMLoc L,
+                 const MCSubtargetInfo &STI)
+      : MCFragment(FT_Nops, false), Size(NumBytes),
+        ControlledNopLength(ControlledNopLength), Loc(L), STI(STI) {}
+
+  int64_t getNumBytes() const { return Size; }
+  int64_t getControlledNopLength() const { return ControlledNopLength; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Nops;
+  }
+};
+
+class MCOrgFragment : public MCFragment {
+  /// Value to use for filling bytes.
+  int8_t Value;
+
+  /// The offset this fragment should start at.
+  const MCExpr *Offset;
+
+  /// Source location of the directive that this fragment was created for.
+  SMLoc Loc;
+
+public:
+  MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc)
+      : MCFragment(FT_Org, false), Value(Value), Offset(&Offset), Loc(Loc) {}
+
+  const MCExpr &getOffset() const { return *Offset; }
+
+  uint8_t getValue() const { return Value; }
+
+  SMLoc getLoc() const { return Loc; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Org;
+  }
+};
+
+class MCLEBFragment final : public MCEncodedFragment {
+  /// True if this is a sleb128, false if uleb128.
+  bool IsSigned;
+
+  /// The value this fragment should contain.
+  const MCExpr *Value;
+
+public:
+  MCLEBFragment(const MCExpr &Value, bool IsSigned)
+      : MCEncodedFragment(FT_LEB, false), IsSigned(IsSigned), Value(&Value) {}
+
+  const MCExpr &getValue() const { return *Value; }
+  void setValue(const MCExpr *Expr) { Value = Expr; }
+
+  bool isSigned() const { return IsSigned; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_LEB;
+  }
+};
+
+class MCDwarfLineAddrFragment : public MCEncodedFragment {
+  /// The value of the difference between the two line numbers
+  /// between two .loc dwarf directives.
+  int64_t LineDelta;
+
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .loc dwarf directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta)
+      : MCEncodedFragment(FT_Dwarf, false), LineDelta(LineDelta),
+        AddrDelta(&AddrDelta) {}
+
+  int64_t getLineDelta() const { return LineDelta; }
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_Dwarf;
+  }
+};
+
+class MCDwarfCallFrameFragment : public MCEncodedFragment {
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .cfi_* dwarf directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCDwarfCallFrameFragment(const MCExpr &AddrDelta)
+      : MCEncodedFragment(FT_DwarfFrame, false), AddrDelta(&AddrDelta) {}
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+  void setAddrDelta(const MCExpr *E) { AddrDelta = E; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_DwarfFrame;
+  }
+};
+
+/// Represents a symbol table index fragment.
+class MCSymbolIdFragment : public MCFragment {
+  const MCSymbol *Sym;
+
+public:
+  MCSymbolIdFragment(const MCSymbol *Sym)
+      : MCFragment(FT_SymbolId, false), Sym(Sym) {}
+
+  const MCSymbol *getSymbol() { return Sym; }
+  const MCSymbol *getSymbol() const { return Sym; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_SymbolId;
+  }
+};
+
+/// Fragment representing the binary annotations produced by the
+/// .cv_inline_linetable directive.
+class MCCVInlineLineTableFragment : public MCEncodedFragment {
+  unsigned SiteFuncId;
+  unsigned StartFileId;
+  unsigned StartLineNum;
+  const MCSymbol *FnStartSym;
+  const MCSymbol *FnEndSym;
+
+  /// CodeViewContext has the real knowledge about this format, so let it access
+  /// our members.
+  friend class CodeViewContext;
+
+public:
+  MCCVInlineLineTableFragment(unsigned SiteFuncId, unsigned StartFileId,
+                              unsigned StartLineNum, const MCSymbol *FnStartSym,
+                              const MCSymbol *FnEndSym)
+      : MCEncodedFragment(FT_CVInlineLines, false), SiteFuncId(SiteFuncId),
+        StartFileId(StartFileId), StartLineNum(StartLineNum),
+        FnStartSym(FnStartSym), FnEndSym(FnEndSym) {}
+
+  const MCSymbol *getFnStartSym() const { return FnStartSym; }
+  const MCSymbol *getFnEndSym() const { return FnEndSym; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_CVInlineLines;
+  }
+};
+
+/// Fragment representing the .cv_def_range directive.
+class MCCVDefRangeFragment : public MCEncodedFragment {
+  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges;
+  StringRef FixedSizePortion;
+
+  /// CodeViewContext has the real knowledge about this format, so let it access
+  /// our members.
+  friend class CodeViewContext;
+
+public:
+  MCCVDefRangeFragment(
+      ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
+      StringRef FixedSizePortion)
+      : MCEncodedFragment(FT_CVDefRange, false),
+        Ranges(Ranges.begin(), Ranges.end()),
+        FixedSizePortion(FixedSizePortion) {}
+
+  ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> getRanges() const {
+    return Ranges;
+  }
+
+  StringRef getFixedSizePortion() const { return FixedSizePortion; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_CVDefRange;
+  }
+};
+
+/// Represents required padding such that a particular other set of fragments
+/// does not cross a particular power-of-two boundary. The other fragments must
+/// follow this one within the same section.
+class MCBoundaryAlignFragment : public MCFragment {
+  /// The alignment requirement of the branch to be aligned.
+  Align AlignBoundary;
+  /// The last fragment in the set of fragments to be aligned.
+  const MCFragment *LastFragment = nullptr;
+  /// The size of the fragment.  The size is lazily set during relaxation, and
+  /// is not meaningful before that.
+  uint64_t Size = 0;
+
+  /// When emitting Nops some subtargets have specific nop encodings.
+  const MCSubtargetInfo &STI;
+
+public:
+  MCBoundaryAlignFragment(Align AlignBoundary, const MCSubtargetInfo &STI)
+      : MCFragment(FT_BoundaryAlign, false), AlignBoundary(AlignBoundary),
+        STI(STI) {}
+
+  uint64_t getSize() const { return Size; }
+  void setSize(uint64_t Value) { Size = Value; }
+
+  Align getAlignment() const { return AlignBoundary; }
+  void setAlignment(Align Value) { AlignBoundary = Value; }
+
+  const MCFragment *getLastFragment() const { return LastFragment; }
+  void setLastFragment(const MCFragment *F) {
+    assert(!F || getParent() == F->getParent());
+    LastFragment = F;
+  }
+
+  const MCSubtargetInfo *getSubtargetInfo() const { return &STI; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_BoundaryAlign;
+  }
+};
+
+class MCPseudoProbeAddrFragment : public MCEncodedFragment {
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .pseudoprobe directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCPseudoProbeAddrFragment(const MCExpr *AddrDelta)
+      : MCEncodedFragment(FT_PseudoProbe, false), AddrDelta(AddrDelta) {}
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_PseudoProbe;
+  }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_MC_MCSECTION_H
diff --git a/llvm/include/llvm/MC/MCSectionGOFF.h b/llvm/include/llvm/MC/MCSectionGOFF.h
index 11c0f9536403..1e6a331202f3 100644
--- a/llvm/include/llvm/MC/MCSectionGOFF.h
+++ b/llvm/include/llvm/MC/MCSectionGOFF.h
@@ -16,7 +16,9 @@
 #define LLVM_MC_MCSECTIONGOFF_H
 
 #include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCSection.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
@@ -24,26 +26,102 @@ namespace llvm {
 class MCExpr;
 
 class MCSectionGOFF final : public MCSection {
-private:
-  MCSection *Parent;
-  uint32_t Subsection;
+  // Parent of this section. Implies that the parent is emitted first.
+  MCSectionGOFF *Parent;
+
+  // The attributes of the GOFF symbols.
+  union {
+    GOFF::SDAttr SDAttributes;
+    GOFF::EDAttr EDAttributes;
+    GOFF::PRAttr PRAttributes;
+  };
+
+  // The type of this section.
+  GOFF::ESDSymbolType SymbolType;
+
+  // This section is a BSS section.
+  unsigned IsBSS : 1;
+
+  // Indicates that the PR symbol needs to set the length of the section to a
+  // non-zero value. This is only a problem with the ADA PR - the binder will
+  // generate an error in this case.
+  unsigned RequiresNonZeroLength : 1;
+
+  // Set to true if the section definition was already emitted.
+  mutable unsigned Emitted : 1;
 
   friend class MCContext;
-  MCSectionGOFF(StringRef Name, SectionKind K, MCSection *P, uint32_t Sub)
-      : MCSection(SV_GOFF, Name, K.isText(), /*IsVirtual=*/false, nullptr),
-        Parent(P), Subsection(Sub) {}
+  friend class MCSymbolGOFF;
+
+  MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual,
+                GOFF::SDAttr SDAttributes, MCSectionGOFF *Parent)
+      : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr),
+        Parent(Parent), SDAttributes(SDAttributes),
+        SymbolType(GOFF::ESD_ST_SectionDefinition), IsBSS(K.isBSS()),
+        RequiresNonZeroLength(0), Emitted(0) {}
+
+  MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual,
+                GOFF::EDAttr EDAttributes, MCSectionGOFF *Parent)
+      : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr),
+        Parent(Parent), EDAttributes(EDAttributes),
+        SymbolType(GOFF::ESD_ST_ElementDefinition), IsBSS(K.isBSS()),
+        RequiresNonZeroLength(0), Emitted(0) {}
+
+  MCSectionGOFF(StringRef Name, SectionKind K, bool IsVirtual,
+                GOFF::PRAttr PRAttributes, MCSectionGOFF *Parent)
+      : MCSection(SV_GOFF, Name, K.isText(), IsVirtual, nullptr),
+        Parent(Parent), PRAttributes(PRAttributes),
+        SymbolType(GOFF::ESD_ST_PartReference), IsBSS(K.isBSS()),
+        RequiresNonZeroLength(0), Emitted(0) {}
 
 public:
   void printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
                             raw_ostream &OS,
-                            uint32_t /*Subsection*/) const override {
-    OS << "\t.section\t\"" << getName() << "\"\n";
-  }
+                            uint32_t Subsection) const override;
 
   bool useCodeAlign() const override { return false; }
 
-  MCSection *getParent() const { return Parent; }
-  uint32_t getSubsection() const { return Subsection; }
+  // Return the parent section.
+  MCSectionGOFF *getParent() const { return Parent; }
+
+  // Returns true if this is a BSS section.
+  bool isBSS() const { return IsBSS; }
+
+  // Returns the type of this section.
+  GOFF::ESDSymbolType getSymbolType() const { return SymbolType; }
+
+  bool isSD() const { return SymbolType == GOFF::ESD_ST_SectionDefinition; }
+  bool isED() const { return SymbolType == GOFF::ESD_ST_ElementDefinition; }
+  bool isPR() const { return SymbolType == GOFF::ESD_ST_PartReference; }
+
+  // Accessors to the attributes.
+  GOFF::SDAttr getSDAttributes() const {
+    assert(isSD() && "Not a SD section");
+    return SDAttributes;
+  }
+  GOFF::EDAttr getEDAttributes() const {
+    assert(isED() && "Not a ED section");
+    return EDAttributes;
+  }
+  GOFF::PRAttr getPRAttributes() const {
+    assert(isPR() && "Not a PR section");
+    return PRAttributes;
+  }
+
+  // Returns the text style for a section. Only defined for ED and PR sections.
+  GOFF::ESDTextStyle getTextStyle() const {
+    assert((isED() || isPR() || isVirtualSection()) && "Expect ED or PR section");
+    if (isED())
+      return EDAttributes.TextStyle;
+    if (isPR())
+      return getParent()->getEDAttributes().TextStyle;
+    // Virtual sections have no data, so byte orientation is fine.
+    return GOFF::ESD_TS_ByteOriented;
+  }
+
+  bool requiresNonZeroLength() const { return RequiresNonZeroLength; }
+
+  void setName(StringRef SectionName) { Name = SectionName; }
 
   static bool classof(const MCSection *S) { return S->getVariant() == SV_GOFF; }
 };
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 455796aeff9f..f2f6a2a6231b 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -19,9 +19,9 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCDwarf.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCPseudoProbe.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCWinEH.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -45,11 +45,9 @@ class MCAsmBackend;
 class MCAssembler;
 class MCContext;
 class MCExpr;
-class MCFragment;
 class MCInst;
 class MCInstPrinter;
 class MCRegister;
-class MCSection;
 class MCStreamer;
 class MCSubtargetInfo;
 class MCSymbol;
diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index b02bc111955a..dc191f881921 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -16,7 +16,7 @@
 #include "llvm/ADT/StringMapEntry.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
+#include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSymbolTableEntry.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/include/llvm/MC/MCSymbolGOFF.h b/llvm/include/llvm/MC/MCSymbolGOFF.h
index 9ca53a565310..d8c570d2de24 100644
--- a/llvm/include/llvm/MC/MCSymbolGOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolGOFF.h
@@ -13,15 +13,41 @@
 #ifndef LLVM_MC_MCSYMBOLGOFF_H
 #define LLVM_MC_MCSYMBOLGOFF_H
 
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/MC/MCGOFFAttributes.h"
+#include "llvm/MC/MCSectionGOFF.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolTableEntry.h"
 
 namespace llvm {
 
 class MCSymbolGOFF : public MCSymbol {
+  // Associated data area of the section. Needs to be emitted first.
+  MCSectionGOFF *ADA;
+
+  GOFF::LDAttr LDAttributes;
+
+  enum SymbolFlags : uint16_t {
+    SF_LD = 0x01, // LD attributes are set.
+  };
+
 public:
   MCSymbolGOFF(const MCSymbolTableEntry *Name, bool IsTemporary)
       : MCSymbol(SymbolKindGOFF, Name, IsTemporary) {}
+
+  void setLDAttributes(GOFF::LDAttr Attr) {
+    modifyFlags(SF_LD, SF_LD);
+    LDAttributes = Attr;
+  }
+  GOFF::LDAttr getLDAttributes() const { return LDAttributes; }
+  bool hasLDAttributes() const { return getFlags() & SF_LD; }
+
+  void setADA(MCSectionGOFF *AssociatedDataArea) {
+    ADA = AssociatedDataArea;
+    AssociatedDataArea->RequiresNonZeroLength = true;
+  }
+  MCSectionGOFF *getADA() const { return ADA; }
+
   static bool classof(const MCSymbol *S) { return S->isGOFF(); }
 };
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCWasmStreamer.h b/llvm/include/llvm/MC/MCWasmStreamer.h
index bf5b5f6c3447..2598c261ea02 100644
--- a/llvm/include/llvm/MC/MCWasmStreamer.h
+++ b/llvm/include/llvm/MC/MCWasmStreamer.h
@@ -58,8 +58,6 @@ class MCWasmStreamer : public MCObjectStreamer {
   void finishImpl() override;
 
 private:
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
-
   bool SeenIdent;
 };
 
diff --git a/llvm/include/llvm/MC/MCWinCOFFStreamer.h b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
index b474ececb21d..05def07a7d9d 100644
--- a/llvm/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCWinCOFFStreamer.h
@@ -74,8 +74,6 @@ class MCWinCOFFStreamer : public MCObjectStreamer {
 protected:
   const MCSymbol *CurSymbol;
 
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override;
-
   void finalizeCGProfileEntry(const MCSymbolRefExpr *&S);
 
 private:
diff --git a/llvm/include/llvm/MC/MCXCOFFStreamer.h b/llvm/include/llvm/MC/MCXCOFFStreamer.h
index 684c229407a8..870d48fe4200 100644
--- a/llvm/include/llvm/MC/MCXCOFFStreamer.h
+++ b/llvm/include/llvm/MC/MCXCOFFStreamer.h
@@ -25,7 +25,6 @@ class MCXCOFFStreamer : public MCObjectStreamer {
   bool emitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
   void emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         Align ByteAlignment) override;
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override;
   void emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym, uint64_t Size,
                                   MCSymbol *CsectSym, Align Alignment) override;
   void emitXCOFFSymbolLinkageWithVisibility(MCSymbol *Symbol,
diff --git a/llvm/include/llvm/MCA/CodeEmitter.h b/llvm/include/llvm/MCA/CodeEmitter.h
index cc978c78244f..c28a5cd2fec5 100644
--- a/llvm/include/llvm/MCA/CodeEmitter.h
+++ b/llvm/include/llvm/MCA/CodeEmitter.h
@@ -34,7 +34,6 @@ namespace mca {
 /// strings. Encodings are cached internally for later usage.
 class CodeEmitter {
   const MCSubtargetInfo &STI;
-  const MCAsmBackend &MAB;
   const MCCodeEmitter &MCE;
 
   SmallString<256> Code;
@@ -52,7 +51,7 @@ class CodeEmitter {
 public:
   CodeEmitter(const MCSubtargetInfo &ST, const MCAsmBackend &AB,
               const MCCodeEmitter &CE, ArrayRef<MCInst> S)
-      : STI(ST), MAB(AB), MCE(CE), Sequence(S), Encodings(S.size()) {}
+      : STI(ST), MCE(CE), Sequence(S), Encodings(S.size()) {}
 
   StringRef getEncoding(unsigned MCID) {
     EncodingInfo EI = getOrCreateEncodingInfo(MCID);
diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index 4417061da936..51f570da6df5 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -178,19 +178,14 @@ struct RootDescriptorView : RootParameterView {
     return readParameter<dxbc::RTS0::v2::RootDescriptor>();
   }
 };
-
-struct DescriptorTable {
+template <typename T> struct DescriptorTable {
   uint32_t NumRanges;
   uint32_t RangesOffset;
-  ViewArray<dxbc::RTS0::v2::DescriptorRange> Ranges;
+  ViewArray<T> Ranges;
 
-  typename ViewArray<dxbc::RTS0::v2::DescriptorRange>::iterator begin() const {
-    return Ranges.begin();
-  }
+  typename ViewArray<T>::iterator begin() const { return Ranges.begin(); }
 
-  typename ViewArray<dxbc::RTS0::v2::DescriptorRange>::iterator end() const {
-    return Ranges.end();
-  }
+  typename ViewArray<T>::iterator end() const { return Ranges.end(); }
 };
 
 struct DescriptorTableView : RootParameterView {
@@ -200,9 +195,9 @@ struct DescriptorTableView : RootParameterView {
   }
 
   // Define a type alias to access the template parameter from inside classof
-  llvm::Expected<DescriptorTable> read(uint32_t Version) {
+  template <typename T> llvm::Expected<DescriptorTable<T>> read() {
     const char *Current = ParamData.begin();
-    DescriptorTable Table;
+    DescriptorTable<T> Table;
 
     Table.NumRanges =
         support::endian::read<uint32_t, llvm::endianness::little>(Current);
@@ -212,13 +207,8 @@ struct DescriptorTableView : RootParameterView {
         support::endian::read<uint32_t, llvm::endianness::little>(Current);
     Current += sizeof(uint32_t);
 
-    size_t RangeSize = sizeof(dxbc::RTS0::v1::DescriptorRange);
-    if (Version > 1)
-      RangeSize = sizeof(dxbc::RTS0::v2::DescriptorRange);
-
-    Table.Ranges.Stride = RangeSize;
-    Table.Ranges.Data =
-        ParamData.substr(2 * sizeof(uint32_t), Table.NumRanges * RangeSize);
+    Table.Ranges.Data = ParamData.substr(2 * sizeof(uint32_t),
+                                         Table.NumRanges * Table.Ranges.Stride);
     return Table;
   }
 };
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index 8d7545144dfd..0b362d389c17 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -931,7 +931,7 @@ Expected<typename ELFT::ShdrRange> ELFFile<ELFT>::sections() const {
   const uintX_t SectionTableOffset = getHeader().e_shoff;
   if (SectionTableOffset == 0) {
     if (!FakeSections.empty())
-      return ArrayRef(FakeSections.data(), FakeSections.size());
+      return ArrayRef(FakeSections);
     return ArrayRef<Elf_Shdr>();
   }
 
diff --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index c235112dacf7..7e0a4c6b0703 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -87,7 +87,7 @@ struct RootDescriptorYaml {
 
   LLVM_ABI uint32_t getEncodedFlags() const;
 
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) bool Val = false;
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag) bool Enum = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
@@ -100,7 +100,7 @@ struct DescriptorRangeYaml {
 
   LLVM_ABI uint32_t getEncodedFlags() const;
 
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) bool Val = false;
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag) bool Enum = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
@@ -165,16 +165,16 @@ struct RootParameterYamlDesc {
 };
 
 struct StaticSamplerYamlDesc {
-  uint32_t Filter = llvm::to_underlying(dxbc::StaticSamplerFilter::ANISOTROPIC);
+  uint32_t Filter = llvm::to_underlying(dxbc::SamplerFilter::Anisotropic);
   uint32_t AddressU = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   uint32_t AddressV = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   uint32_t AddressW = llvm::to_underlying(dxbc::TextureAddressMode::Wrap);
   float MipLODBias = 0.f;
   uint32_t MaxAnisotropy = 16u;
   uint32_t ComparisonFunc =
-      llvm::to_underlying(dxbc::SamplersComparisonFunction::LessEqual);
+      llvm::to_underlying(dxbc::ComparisonFunc::LessEqual);
   uint32_t BorderColor =
-      llvm::to_underlying(dxbc::SamplersBorderColor::OpaqueWhite);
+      llvm::to_underlying(dxbc::StaticBorderColor::OpaqueWhite);
   float MinLOD = 0.f;
   float MaxLOD = std::numeric_limits<float>::max();
   uint32_t ShaderRegister;
@@ -203,7 +203,7 @@ struct RootSignatureYamlDesc {
   LLVM_ABI static llvm::Expected<DXContainerYAML::RootSignatureYamlDesc>
   create(const object::DirectX::RootSignature &Data);
 
-#define ROOT_ELEMENT_FLAG(Num, Val) bool Val = false;
+#define ROOT_SIGNATURE_FLAG(Num, Val) bool Val = false;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h
index 0b09064f8445..a716ea88740d 100644
--- a/llvm/include/llvm/Pass.h
+++ b/llvm/include/llvm/Pass.h
@@ -177,11 +177,6 @@ class LLVM_ABI Pass {
   /// longer used.
   virtual void releaseMemory();
 
-  /// getAdjustedAnalysisPointer - This method is used when a pass implements
-  /// an analysis interface through multiple inheritance.  If needed, it should
-  /// override this to adjust the this pointer as needed for the specified pass
-  /// info.
-  virtual void *getAdjustedAnalysisPointer(AnalysisID ID);
   virtual ImmutablePass *getAsImmutablePass();
   virtual PMDataManager *getAsPMDataManager();
 
@@ -278,7 +273,7 @@ class LLVM_ABI ModulePass : public Pass {
 protected:
   /// Optional passes call this function to check whether the pass should be
   /// skipped. This is the case when optimization bisect is over the limit.
-  bool skipModule(Module &M) const;
+  bool skipModule(const Module &M) const;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/PassAnalysisSupport.h b/llvm/include/llvm/PassAnalysisSupport.h
index 02abb00b66b5..e81c0c4642d2 100644
--- a/llvm/include/llvm/PassAnalysisSupport.h
+++ b/llvm/include/llvm/PassAnalysisSupport.h
@@ -214,15 +214,7 @@ AnalysisType *Pass::getAnalysisIfAvailable() const {
   assert(Resolver && "Pass not resident in a PassManager object!");
 
   const void *PI = &AnalysisType::ID;
-
-  Pass *ResultPass = Resolver->getAnalysisIfAvailable(PI);
-  if (!ResultPass) return nullptr;
-
-  // Because the AnalysisType may not be a subclass of pass (for
-  // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially
-  // adjust the return pointer (because the class may multiply inherit, once
-  // from pass, once from AnalysisType).
-  return (AnalysisType*)ResultPass->getAdjustedAnalysisPointer(PI);
+  return (AnalysisType *)Resolver->getAnalysisIfAvailable(PI);
 }
 
 /// getAnalysis<AnalysisType>() - This function is used by subclasses to get
@@ -245,12 +237,7 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI) const {
   assert(ResultPass &&
          "getAnalysis*() called on an analysis that was not "
          "'required' by pass!");
-
-  // Because the AnalysisType may not be a subclass of pass (for
-  // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially
-  // adjust the return pointer (because the class may multiply inherit, once
-  // from pass, once from AnalysisType).
-  return *(AnalysisType*)ResultPass->getAdjustedAnalysisPointer(PI);
+  return *(AnalysisType *)ResultPass;
 }
 
 /// getAnalysis<AnalysisType>() - This function is used by subclasses to get
@@ -282,12 +269,7 @@ AnalysisType &Pass::getAnalysisID(AnalysisID PI, Function &F, bool *Changed) {
   else
     assert(!LocalChanged &&
            "A pass trigged a code update but the update status is lost");
-
-  // Because the AnalysisType may not be a subclass of pass (for
-  // AnalysisGroups), we use getAdjustedAnalysisPointer here to potentially
-  // adjust the return pointer (because the class may multiply inherit, once
-  // from pass, once from AnalysisType).
-  return *(AnalysisType*)ResultPass->getAdjustedAnalysisPointer(PI);
+  return *(AnalysisType *)ResultPass;
 }
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index d1230b0ba7c5..7d1a85ba528f 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -31,6 +31,7 @@
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -494,6 +495,15 @@ struct MCDCRecord {
     return TV[TestVectorIndex].first[PosToID[Condition]];
   }
 
+  /// Return the number of True and False decisions for all executed test
+  /// vectors.
+  std::pair<unsigned, unsigned> getDecisions() const {
+    const unsigned TrueDecisions =
+        llvm::count(llvm::make_second_range(TV), CondState::MCDC_True);
+
+    return {TrueDecisions, TV.size() - TrueDecisions};
+  }
+
   /// Return the Result evaluation for an executed test vector.
   /// See MCDCRecordProcessor::RecordTestVector().
   CondState getTVResult(unsigned TestVectorIndex) {
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 3b12505d3326..deb5cd17d8fd 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -824,7 +824,7 @@ class LLVM_ABI IndexedInstrProfReader : public InstrProfReader {
   /// functions, MismatchedFuncSum returns the maximum. If \c FuncName is not
   /// found, try to lookup \c DeprecatedFuncName to handle profiles built by
   /// older compilers.
-  Expected<InstrProfRecord>
+  Expected<NamedInstrProfRecord>
   getInstrProfRecord(StringRef FuncName, uint64_t FuncHash,
                      StringRef DeprecatedFuncName = "",
                      uint64_t *MismatchedFuncSum = nullptr);
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 09dc7a294415..4e1f1131b710 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -289,8 +289,8 @@ struct LineLocation {
   LLVM_ABI void serialize(raw_ostream &OS);
 
   bool operator<(const LineLocation &O) const {
-    return LineOffset < O.LineOffset ||
-           (LineOffset == O.LineOffset && Discriminator < O.Discriminator);
+    return std::tie(LineOffset, Discriminator) <
+           std::tie(O.LineOffset, O.Discriminator);
   }
 
   bool operator==(const LineLocation &O) const {
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 146d051f770a..e84b2095efd7 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -193,7 +193,7 @@ class LLVM_ABI SampleProfileWriterText : public SampleProfileWriter {
   /// cannot be skipped.
   bool MarkFlatProfiles = false;
 
-  LLVM_ABI_FRIEND friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
+  LLVM_ABI friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
   SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
                               SampleProfileFormat Format);
 };
@@ -225,7 +225,7 @@ class LLVM_ABI SampleProfileWriterBinary : public SampleProfileWriter {
   void addNames(const FunctionSamples &S);
 
 private:
-  LLVM_ABI_FRIEND friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
+  LLVM_ABI friend ErrorOr<std::unique_ptr<SampleProfileWriter>>
   SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
                               SampleProfileFormat Format);
 };
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
index dd0bc76db3e3..8e50614d6a6d 100644
--- a/llvm/include/llvm/SandboxIR/Value.h
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -33,6 +33,7 @@ class IntrinsicInst;
 class Operator;
 class OverflowingBinaryOperator;
 class FPMathOperator;
+class Region;
 
 /// Iterator for the `Use` edges of a Value's users.
 /// \Returns a `Use` when dereferenced.
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index 0de789ec68c4..f6d0d3591c33 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -133,7 +133,7 @@
 #endif
 
 #if (!(defined(_WIN32) || defined(__CYGWIN__)) ||                              \
-     (defined(__MINGW32__) && defined(__clang__)))
+     ((defined(__MINGW32__) || defined(__CYGWIN__)) && defined(__clang__)))
 #define LLVM_LIBRARY_VISIBILITY LLVM_ATTRIBUTE_VISIBILITY_HIDDEN
 // Clang compilers older then 15 do not support gnu style attributes on
 // namespaces.
@@ -171,11 +171,6 @@
 /// for both functions and classes. On windows its turned in to dllimport for
 /// library consumers, for other platforms its a default visibility attribute.
 ///
-/// LLVM_ABI_FRIEND is for annotating friend function declarations when the
-/// target function's original declaration is annotated with LLVM_ABI. This
-/// macro matches the LLVM_ABI macro on Windows, on other platforms it does
-/// nothing.
-///
 /// LLVM_C_ABI is used to annotated functions and data that need to be exported
 /// for the libllvm-c API. This used both for the llvm-c headers and for the
 /// functions declared in the different Target's c++ source files that don't
@@ -184,19 +179,10 @@
 // Marker to add to classes or functions in public headers that should not have
 // export macros added to them by the clang tool
 #define LLVM_ABI_NOT_EXPORTED
-#if defined(LLVM_BUILD_LLVM_DYLIB) || defined(LLVM_BUILD_SHARED_LIBS) ||       \
-    defined(LLVM_ENABLE_PLUGINS)
-// Some libraries like those for tablegen are linked in to tools that used
-// in the build so can't depend on the llvm shared library. If export macros
-// were left enabled when building these we would get duplicate or
-// missing symbol linker errors on windows.
-#if defined(LLVM_BUILD_STATIC)
-#define LLVM_ABI
-#define LLVM_ABI_FRIEND
-#define LLVM_TEMPLATE_ABI
-#define LLVM_EXPORT_TEMPLATE
-#define LLVM_ABI_EXPORT
-#elif defined(_WIN32) && !defined(__MINGW32__)
+// TODO(https://github.com/llvm/llvm-project/issues/145406): eliminate need for
+// two preprocessor definitions to gate LLVM_ABI macro definitions.
+#if defined(LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS) && !defined(LLVM_BUILD_STATIC)
+#if defined(_WIN32) && !defined(__MINGW32__)
 #if defined(LLVM_EXPORTS)
 #define LLVM_ABI __declspec(dllexport)
 #define LLVM_TEMPLATE_ABI
@@ -206,25 +192,24 @@
 #define LLVM_TEMPLATE_ABI __declspec(dllimport)
 #define LLVM_EXPORT_TEMPLATE
 #endif
-#define LLVM_ABI_FRIEND LLVM_ABI
 #define LLVM_ABI_EXPORT __declspec(dllexport)
-#elif defined(__ELF__) || defined(__MINGW32__) || defined(_AIX) ||             \
+#elif __has_attribute(visibility)
+#if defined(__ELF__) || defined(__MINGW32__) || defined(_AIX) ||               \
     defined(__MVS__) || defined(__CYGWIN__)
-#define LLVM_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
-#define LLVM_ABI_FRIEND
-#define LLVM_TEMPLATE_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define LLVM_ABI __attribute__((visibility("default")))
+#define LLVM_TEMPLATE_ABI LLVM_ABI
 #define LLVM_EXPORT_TEMPLATE
-#define LLVM_ABI_EXPORT LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define LLVM_ABI_EXPORT LLVM_ABI
 #elif defined(__MACH__) || defined(__WASM__) || defined(__EMSCRIPTEN__)
-#define LLVM_ABI LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
-#define LLVM_ABI_FRIEND
+#define LLVM_ABI __attribute__((visibility("default")))
 #define LLVM_TEMPLATE_ABI
 #define LLVM_EXPORT_TEMPLATE
-#define LLVM_ABI_EXPORT LLVM_ATTRIBUTE_VISIBILITY_DEFAULT
+#define LLVM_ABI_EXPORT LLVM_ABI
 #endif
-#else
+#endif
+#endif
+#if !defined(LLVM_ABI)
 #define LLVM_ABI
-#define LLVM_ABI_FRIEND
 #define LLVM_TEMPLATE_ABI
 #define LLVM_EXPORT_TEMPLATE
 #define LLVM_ABI_EXPORT
diff --git a/llvm/include/llvm/Support/ConvertUTF.h b/llvm/include/llvm/Support/ConvertUTF.h
index 3bb238e7df2e..bb1723518a49 100644
--- a/llvm/include/llvm/Support/ConvertUTF.h
+++ b/llvm/include/llvm/Support/ConvertUTF.h
@@ -346,9 +346,9 @@ LLVM_ABI bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out);
 LLVM_ABI bool convertUTF8ToUTF16String(StringRef SrcUTF8,
                                        SmallVectorImpl<UTF16> &DstUTF16);
 
-bool IsSingleCodeUnitUTF8Codepoint(unsigned);
-bool IsSingleCodeUnitUTF16Codepoint(unsigned);
-bool IsSingleCodeUnitUTF32Codepoint(unsigned);
+LLVM_ABI bool IsSingleCodeUnitUTF8Codepoint(unsigned);
+LLVM_ABI bool IsSingleCodeUnitUTF16Codepoint(unsigned);
+LLVM_ABI bool IsSingleCodeUnitUTF32Codepoint(unsigned);
 
 #if defined(_WIN32)
 namespace sys {
diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index 9bb952ceb0dc..ed83e6e3a40e 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -1191,7 +1191,7 @@ class ExpectedAsOutParameter {
 /// (or Expected) and you want to call code that still returns
 /// std::error_codes.
 class LLVM_ABI ECError : public ErrorInfo<ECError> {
-  LLVM_ABI_FRIEND friend Error errorCodeToError(std::error_code);
+  LLVM_ABI friend Error errorCodeToError(std::error_code);
 
   void anchor() override;
 
diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index ae4a212b553c..31fedc37bf77 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -220,7 +220,7 @@ class basic_file_status {
 
 /// Represents the result of a call to sys::fs::status().
 class file_status : public basic_file_status {
-  LLVM_ABI_FRIEND friend bool equivalent(file_status A, file_status B);
+  LLVM_ABI friend bool equivalent(file_status A, file_status B);
 
 #if defined(LLVM_ON_UNIX)
   dev_t fs_st_dev = 0;
diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h
index 962d79de8dd1..74858ec55993 100644
--- a/llvm/include/llvm/Support/JSON.h
+++ b/llvm/include/llvm/Support/JSON.h
@@ -531,7 +531,7 @@ class Value {
                                       llvm::StringRef, std::string, json::Array,
                                       json::Object>
       Union;
-  LLVM_ABI_FRIEND friend bool operator==(const Value &, const Value &);
+  LLVM_ABI friend bool operator==(const Value &, const Value &);
 };
 
 LLVM_ABI bool operator==(const Value &, const Value &);
@@ -713,7 +713,7 @@ class Path::Root {
   llvm::StringLiteral ErrorMessage;
   std::vector<Path::Segment> ErrorPath; // Only valid in error state. Reversed.
 
-  LLVM_ABI_FRIEND friend void Path::report(llvm::StringLiteral Message);
+  LLVM_ABI friend void Path::report(llvm::StringLiteral Message);
 
 public:
   Root(llvm::StringRef Name = "") : Name(Name), ErrorMessage("") {}
diff --git a/llvm/include/llvm/Support/KnownFPClass.h b/llvm/include/llvm/Support/KnownFPClass.h
index 9ebdf260e0ec..b3c18bcf6b34 100644
--- a/llvm/include/llvm/Support/KnownFPClass.h
+++ b/llvm/include/llvm/Support/KnownFPClass.h
@@ -15,6 +15,7 @@
 #define LLVM_SUPPORT_KNOWNFPCLASS_H
 
 #include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/Support/Compiler.h"
 #include <optional>
 
 namespace llvm {
@@ -78,13 +79,13 @@ struct KnownFPClass {
   /// Return true if it's know this can never be interpreted as a zero. This
   /// extends isKnownNeverZero to cover the case where the assumed
   /// floating-point mode for the function interprets denormals as zero.
-  bool isKnownNeverLogicalZero(DenormalMode Mode) const;
+  LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const;
 
   /// Return true if it's know this can never be interpreted as a negative zero.
-  bool isKnownNeverLogicalNegZero(DenormalMode Mode) const;
+  LLVM_ABI bool isKnownNeverLogicalNegZero(DenormalMode Mode) const;
 
   /// Return true if it's know this can never be interpreted as a positive zero.
-  bool isKnownNeverLogicalPosZero(DenormalMode Mode) const;
+  LLVM_ABI bool isKnownNeverLogicalPosZero(DenormalMode Mode) const;
 
   static constexpr FPClassTest OrderedLessThanZeroMask =
       fcNegSubnormal | fcNegNormal | fcNegInf;
@@ -209,7 +210,7 @@ struct KnownFPClass {
   ///
   /// This assumes a copy-like operation and will replace any currently known
   /// information.
-  void propagateDenormal(const KnownFPClass &Src, DenormalMode Mode);
+  LLVM_ABI void propagateDenormal(const KnownFPClass &Src, DenormalMode Mode);
 
   /// Report known classes if \p Src is evaluated through a potentially
   /// canonicalizing operation. We can assume signaling nans will not be
@@ -217,7 +218,8 @@ struct KnownFPClass {
   ///
   /// This assumes a copy-like operation and will replace any currently known
   /// information.
-  void propagateCanonicalizingSrc(const KnownFPClass &Src, DenormalMode Mode);
+  LLVM_ABI void propagateCanonicalizingSrc(const KnownFPClass &Src,
+                                           DenormalMode Mode);
 
   void resetAll() { *this = KnownFPClass(); }
 };
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index ae3150e5602e..7bbf1b3aab76 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -79,7 +79,7 @@ constexpr float ef          = 0x1.5bf0a8P+1F, // (2.71828183) https://oeis.org/A
 
 /// Create a bitmask with the N right-most bits set to 1, and all other
 /// bits set to 0.  Only unsigned types are allowed.
-template <typename T> T maskTrailingOnes(unsigned N) {
+template <typename T> constexpr T maskTrailingOnes(unsigned N) {
   static_assert(std::is_unsigned_v<T>, "Invalid type!");
   const unsigned Bits = CHAR_BIT * sizeof(T);
   assert(N <= Bits && "Invalid bit index");
@@ -90,19 +90,19 @@ template <typename T> T maskTrailingOnes(unsigned N) {
 
 /// Create a bitmask with the N left-most bits set to 1, and all other
 /// bits set to 0.  Only unsigned types are allowed.
-template <typename T> T maskLeadingOnes(unsigned N) {
+template <typename T> constexpr T maskLeadingOnes(unsigned N) {
   return ~maskTrailingOnes<T>(CHAR_BIT * sizeof(T) - N);
 }
 
 /// Create a bitmask with the N right-most bits set to 0, and all other
 /// bits set to 1.  Only unsigned types are allowed.
-template <typename T> T maskTrailingZeros(unsigned N) {
+template <typename T> constexpr T maskTrailingZeros(unsigned N) {
   return maskLeadingOnes<T>(CHAR_BIT * sizeof(T) - N);
 }
 
 /// Create a bitmask with the N left-most bits set to 0, and all other
 /// bits set to 1.  Only unsigned types are allowed.
-template <typename T> T maskLeadingZeros(unsigned N) {
+template <typename T> constexpr T maskLeadingZeros(unsigned N) {
   return maskTrailingOnes<T>(CHAR_BIT * sizeof(T) - N);
 }
 
@@ -120,7 +120,7 @@ static const unsigned char BitReverseTable256[256] = {
 };
 
 /// Reverse the bits in \p Val.
-template <typename T> T reverseBits(T Val) {
+template <typename T> constexpr T reverseBits(T Val) {
 #if __has_builtin(__builtin_bitreverse8)
   if constexpr (std::is_same_v<T, uint8_t>)
     return __builtin_bitreverse8(Val);
@@ -217,7 +217,7 @@ constexpr bool isShiftedUInt(uint64_t x) {
 }
 
 /// Gets the maximum value for a N-bit unsigned integer.
-inline uint64_t maxUIntN(uint64_t N) {
+inline constexpr uint64_t maxUIntN(uint64_t N) {
   assert(N <= 64 && "integer width out of range");
 
   // uint64_t(1) << 64 is undefined behavior, so we can't do
@@ -233,7 +233,7 @@ inline uint64_t maxUIntN(uint64_t N) {
 }
 
 /// Gets the minimum value for a N-bit signed integer.
-inline int64_t minIntN(int64_t N) {
+inline constexpr int64_t minIntN(int64_t N) {
   assert(N <= 64 && "integer width out of range");
 
   if (N == 0)
@@ -242,7 +242,7 @@ inline int64_t minIntN(int64_t N) {
 }
 
 /// Gets the maximum value for a N-bit signed integer.
-inline int64_t maxIntN(int64_t N) {
+inline constexpr int64_t maxIntN(int64_t N) {
   assert(N <= 64 && "integer width out of range");
 
   // This relies on two's complement wraparound when N == 64, so we convert to
@@ -253,12 +253,12 @@ inline int64_t maxIntN(int64_t N) {
 }
 
 /// Checks if an unsigned integer fits into the given (dynamic) bit width.
-inline bool isUIntN(unsigned N, uint64_t x) {
+inline constexpr bool isUIntN(unsigned N, uint64_t x) {
   return N >= 64 || x <= maxUIntN(N);
 }
 
 /// Checks if an signed integer fits into the given (dynamic) bit width.
-inline bool isIntN(unsigned N, int64_t x) {
+inline constexpr bool isIntN(unsigned N, int64_t x) {
   return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
 }
 
diff --git a/llvm/include/llvm/Support/Path.h b/llvm/include/llvm/Support/Path.h
index 32144a0d5348..0cb517146c04 100644
--- a/llvm/include/llvm/Support/Path.h
+++ b/llvm/include/llvm/Support/Path.h
@@ -80,8 +80,8 @@ class const_iterator
   Style S = Style::native; ///< The path style to use.
 
   // An end iterator has Position = Path.size() + 1.
-  LLVM_ABI_FRIEND friend const_iterator begin(StringRef path, Style style);
-  LLVM_ABI_FRIEND friend const_iterator end(StringRef path);
+  LLVM_ABI friend const_iterator begin(StringRef path, Style style);
+  LLVM_ABI friend const_iterator end(StringRef path);
 
 public:
   reference operator*() const { return Component; }
@@ -105,8 +105,8 @@ class reverse_iterator
   size_t    Position = 0;  ///< The iterators current position within Path.
   Style S = Style::native; ///< The path style to use.
 
-  LLVM_ABI_FRIEND friend reverse_iterator rbegin(StringRef path, Style style);
-  LLVM_ABI_FRIEND friend reverse_iterator rend(StringRef path);
+  LLVM_ABI friend reverse_iterator rbegin(StringRef path, Style style);
+  LLVM_ABI friend reverse_iterator rend(StringRef path);
 
 public:
   reference operator*() const { return Component; }
@@ -223,6 +223,8 @@ LLVM_ABI bool remove_dots(SmallVectorImpl<char> &path,
 ///   /foo  + bar/f => /foo/bar/f
 ///   /foo/ + bar/f => /foo/bar/f
 ///   foo   + bar/f => foo/bar/f
+///   foo   + /bar/f => foo/bar/f (FIXME: will be changed to /bar/f to align
+///                                with C++17 std::filesystem::path::append)
 /// @endcode
 ///
 /// @param path Set to \a path + \a component.
diff --git a/llvm/include/llvm/Support/PrettyStackTrace.h b/llvm/include/llvm/Support/PrettyStackTrace.h
index 33d95f1c5ec7..cf695089b44e 100644
--- a/llvm/include/llvm/Support/PrettyStackTrace.h
+++ b/llvm/include/llvm/Support/PrettyStackTrace.h
@@ -51,7 +51,7 @@ namespace llvm {
   /// constructed and destructed, they will add their symbolic frames to a
   /// virtual stack trace.  This gets dumped out if the program crashes.
   class LLVM_ABI PrettyStackTraceEntry {
-    LLVM_ABI_FRIEND friend PrettyStackTraceEntry *
+    LLVM_ABI friend PrettyStackTraceEntry *
     ReverseStackTrace(PrettyStackTraceEntry *);
 
     PrettyStackTraceEntry *NextEntry;
diff --git a/llvm/include/llvm/Support/ProgramStack.h b/llvm/include/llvm/Support/ProgramStack.h
index 55964c977920..0dd8235b90c0 100644
--- a/llvm/include/llvm/Support/ProgramStack.h
+++ b/llvm/include/llvm/Support/ProgramStack.h
@@ -10,6 +10,7 @@
 #define LLVM_SUPPORT_PROGRAMSTACK_H
 
 #include "llvm/ADT/STLFunctionalExtras.h"
+#include "llvm/Support/Compiler.h"
 
 // LLVM_HAS_SPLIT_STACKS is exposed in the header because CrashRecoveryContext
 // needs to know if it's running on another thread or not.
@@ -28,12 +29,12 @@ namespace llvm {
 ///
 /// The value is not guaranteed to point to anything specific. It can be used to
 /// estimate how much stack space has been used since the previous call.
-uintptr_t getStackPointer();
+LLVM_ABI uintptr_t getStackPointer();
 
 /// \returns the default stack size for this platform.
 ///
 /// Based on \p RLIMIT_STACK or the equivalent.
-unsigned getDefaultStackSize();
+LLVM_ABI unsigned getDefaultStackSize();
 
 /// Runs Fn on a new stack of at least the given size.
 ///
@@ -42,7 +43,7 @@ unsigned getDefaultStackSize();
 ///
 /// The preferred implementation is split stacks on platforms that have a good
 /// debugging experience for them. On other platforms a new thread is used.
-void runOnNewStack(unsigned StackSize, function_ref<void()> Fn);
+LLVM_ABI void runOnNewStack(unsigned StackSize, function_ref<void()> Fn);
 
 template <typename R, typename... Ts>
 std::enable_if_t<!std::is_same_v<R, void>, R>
diff --git a/llvm/include/llvm/Support/Signals.h b/llvm/include/llvm/Support/Signals.h
index 6ce26acdd458..21b425fffef5 100644
--- a/llvm/include/llvm/Support/Signals.h
+++ b/llvm/include/llvm/Support/Signals.h
@@ -14,10 +14,23 @@
 #ifndef LLVM_SUPPORT_SIGNALS_H
 #define LLVM_SUPPORT_SIGNALS_H
 
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Support/Compiler.h"
 #include <cstdint>
 #include <string>
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
+namespace llvm {
+// Typedefs that are convenient but only used by the stack-trace-collection code
+// added if DebugLoc origin-tracking is enabled.
+using AddressSet = DenseSet<void *>;
+using SymbolizedAddressMap = DenseMap<void *, SmallVector<std::string, 0>>;
+} // namespace llvm
+#endif
+
 namespace llvm {
 class StringRef;
 class raw_ostream;
@@ -57,6 +70,28 @@ LLVM_ABI void DisableSystemDialogsOnCrash();
 ///        specified, the entire frame is printed.
 LLVM_ABI void PrintStackTrace(raw_ostream &OS, int Depth = 0);
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#ifdef NDEBUG
+#error DebugLoc origin-tracking should not be enabled in Release builds.
+#endif
+/// Populates the given array with a stack trace of the current program, up to
+/// MaxDepth frames. Returns the number of frames returned, which will be
+/// inserted into \p StackTrace from index 0. All entries after the returned
+/// depth will be unmodified. NB: This is only intended to be used for
+/// introspection of LLVM by Debugify, will not be enabled in release builds,
+/// and should not be relied on for other purposes.
+template <unsigned long MaxDepth>
+int getStackTrace(std::array<void *, MaxDepth> &StackTrace);
+
+/// Takes a set of \p Addresses, symbolizes them and stores the result in the
+/// provided \p SymbolizedAddresses map.
+/// NB: This is only intended to be used for introspection of LLVM by
+/// Debugify, will not be enabled in release builds, and should not be relied
+/// on for other purposes.
+void symbolizeAddresses(AddressSet &Addresses,
+                        SymbolizedAddressMap &SymbolizedAddresses);
+#endif
+
 // Run all registered signal handlers.
 LLVM_ABI void RunSignalHandlers();
 
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index e204b95dd2dd..8a304910aa5d 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorOr.h"
 
 #include <string>
@@ -92,8 +93,8 @@ class TextEncodingConverter {
   /// \param[in] From the source character encoding
   /// \param[in] To the target character encoding
   /// \return a TextEncodingConverter instance or an error code
-  static ErrorOr<TextEncodingConverter> create(TextEncoding From,
-                                               TextEncoding To);
+  LLVM_ABI static ErrorOr<TextEncodingConverter> create(TextEncoding From,
+                                                        TextEncoding To);
 
   /// Creates a TextEncodingConverter instance.
   /// Returns std::errc::invalid_argument in case the requested conversion is
@@ -101,7 +102,8 @@ class TextEncodingConverter {
   /// \param[in] From name of the source character encoding
   /// \param[in] To name of the target character encoding
   /// \return a TextEncodingConverter instance or an error code
-  static ErrorOr<TextEncodingConverter> create(StringRef From, StringRef To);
+  LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From,
+                                                        StringRef To);
 
   TextEncodingConverter(const TextEncodingConverter &) = delete;
   TextEncodingConverter &operator=(const TextEncodingConverter &) = delete;
diff --git a/llvm/include/llvm/Support/Timer.h b/llvm/include/llvm/Support/Timer.h
index 36890c75a65e..40709d49db01 100644
--- a/llvm/include/llvm/Support/Timer.h
+++ b/llvm/include/llvm/Support/Timer.h
@@ -258,7 +258,7 @@ class TimerGroup {
 
 private:
   friend class Timer;
-  friend void PrintStatisticsJSON(raw_ostream &OS);
+  LLVM_ABI friend void PrintStatisticsJSON(raw_ostream &OS);
   void addTimer(Timer &T);
   void removeTimer(Timer &T);
   void prepareToPrintList(bool reset_time = false);
diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h
index f25f2311a81a..d7211a930ae4 100644
--- a/llvm/include/llvm/Support/TrailingObjects.h
+++ b/llvm/include/llvm/Support/TrailingObjects.h
@@ -228,12 +228,18 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
 
   using ParentType::getTrailingObjectsImpl;
 
-  // This function contains only a static_assert BaseTy is final. The
-  // static_assert must be in a function, and not at class-level
-  // because BaseTy isn't complete at class instantiation time, but
-  // will be by the time this function is instantiated.
-  static void verifyTrailingObjectsAssertions() {
+  template <bool Strict> static void verifyTrailingObjectsAssertions() {
+    // The static_assert for BaseTy must be in a function, and not at
+    // class-level  because BaseTy isn't complete at class instantiation time,
+    // but will be by the time this function is instantiated.
     static_assert(std::is_final<BaseTy>(), "BaseTy must be final.");
+
+    // Verify that templated getTrailingObjects() is used only with multiple
+    // trailing types. Use getTrailingObjectsNonStrict() which does not check
+    // this.
+    static_assert(!Strict || sizeof...(TrailingTys) > 1,
+                  "Use templated getTrailingObjects() only when there are "
+                  "multiple trailing types");
   }
 
   // These two methods are the base of the recursion for this method.
@@ -283,7 +289,7 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
   /// (which must be one of those specified in the class template). The
   /// array may have zero or more elements in it.
   template <typename T> const T *getTrailingObjects() const {
-    verifyTrailingObjectsAssertions();
+    verifyTrailingObjectsAssertions<true>();
     // Forwards to an impl function with overloads, since member
     // function templates can't be specialized.
     return this->getTrailingObjectsImpl(
@@ -295,7 +301,7 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
   /// (which must be one of those specified in the class template). The
   /// array may have zero or more elements in it.
   template <typename T> T *getTrailingObjects() {
-    verifyTrailingObjectsAssertions();
+    verifyTrailingObjectsAssertions<true>();
     // Forwards to an impl function with overloads, since member
     // function templates can't be specialized.
     return this->getTrailingObjectsImpl(
@@ -310,14 +316,20 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
     static_assert(sizeof...(TrailingTys) == 1,
                   "Can use non-templated getTrailingObjects() only when there "
                   "is a single trailing type");
-    return getTrailingObjects<FirstTrailingType>();
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<const BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<FirstTrailingType>());
   }
 
   FirstTrailingType *getTrailingObjects() {
     static_assert(sizeof...(TrailingTys) == 1,
                   "Can use non-templated getTrailingObjects() only when there "
                   "is a single trailing type");
-    return getTrailingObjects<FirstTrailingType>();
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<FirstTrailingType>());
   }
 
   // Functions that return the trailing objects as ArrayRefs.
@@ -337,6 +349,31 @@ class TrailingObjects : private trailing_objects_internal::TrailingObjectsImpl<
     return ArrayRef(getTrailingObjects(), N);
   }
 
+  // Non-strict forms of templated `getTrailingObjects` that work with single
+  // trailing type.
+  template <typename T> const T *getTrailingObjectsNonStrict() const {
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<const BaseTy *>(this),
+        TrailingObjectsBase::OverloadToken<T>());
+  }
+
+  template <typename T> T *getTrailingObjectsNonStrict() {
+    verifyTrailingObjectsAssertions<false>();
+    return this->getTrailingObjectsImpl(
+        static_cast<BaseTy *>(this), TrailingObjectsBase::OverloadToken<T>());
+  }
+
+  template <typename T>
+  MutableArrayRef<T> getTrailingObjectsNonStrict(size_t N) {
+    return MutableArrayRef(getTrailingObjectsNonStrict<T>(), N);
+  }
+
+  template <typename T>
+  ArrayRef<T> getTrailingObjectsNonStrict(size_t N) const {
+    return ArrayRef(getTrailingObjectsNonStrict<T>(), N);
+  }
+
   /// Returns the size of the trailing data, if an object were
   /// allocated with the given counts (The counts are in the same order
   /// as the template arguments). This does not include the size of the
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index 5849344bcb0b..a2b86eb8e7ca 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -868,7 +868,7 @@ class UnOpInit final : public OpInit, public FoldingSetNode {
   UnaryOp getOpcode() const { return (UnaryOp)Opc; }
   const Init *getOperand() const { return LHS; }
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec, bool IsFinal = false) const;
 
@@ -940,7 +940,7 @@ class BinOpInit final : public OpInit, public FoldingSetNode {
   std::optional<bool> CompareInit(unsigned Opc, const Init *LHS,
                                   const Init *RHS) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -990,7 +990,7 @@ class TernOpInit final : public OpInit, public FoldingSetNode {
   const Init *getMHS() const { return MHS; }
   const Init *getRHS() const { return RHS; }
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -1096,7 +1096,7 @@ class FoldOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec) const;
 
@@ -1129,7 +1129,7 @@ class IsAOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold() const;
 
@@ -1163,7 +1163,7 @@ class ExistsOpInit final : public TypedInit, public FoldingSetNode {
 
   void Profile(FoldingSetNodeID &ID) const;
 
-  // Fold - If possible, fold this to a simpler init.  Return this if not
+  // Fold - If possible, fold this to a simpler init. Return this if not
   // possible to fold.
   const Init *Fold(const Record *CurRec, bool IsFinal = false) const;
 
@@ -1412,8 +1412,8 @@ class FieldInit final : public TypedInit {
   }
 };
 
-/// (v a, b) - Represent a DAG tree value.  DAG inits are required
-/// to have at least one value then a (possibly empty) list of arguments.  Each
+/// (v a, b) - Represent a DAG tree value. DAG inits are required
+/// to have at least one value then a (possibly empty) list of arguments. Each
 /// argument can have a name associated with it.
 class DagInit final
     : public TypedInit,
diff --git a/llvm/include/llvm/TableGen/SetTheory.h b/llvm/include/llvm/TableGen/SetTheory.h
index 771dcff2f214..53c78e457292 100644
--- a/llvm/include/llvm/TableGen/SetTheory.h
+++ b/llvm/include/llvm/TableGen/SetTheory.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SetTheory class that computes ordered sets of
-// Records from DAG expressions.  Operators for standard set operations are
+// Records from DAG expressions. Operators for standard set operations are
 // predefined, and it is possible to add special purpose set operators as well.
 //
 // The user may define named sets as Records of predefined classes. Set
@@ -112,7 +112,7 @@ class SetTheory {
   void addExpander(StringRef ClassName, std::unique_ptr<Expander>);
 
   /// addFieldExpander - Add an expander for ClassName that simply evaluates
-  /// FieldName in the Record to get the set elements.  That is all that is
+  /// FieldName in the Record to get the set elements. That is all that is
   /// needed for a class like:
   ///
   ///   class Set<dag d> {
@@ -134,7 +134,7 @@ class SetTheory {
       evaluate(*begin++, Elts, Loc);
   }
 
-  /// expand - Expand a record into a set of elements if possible.  Return a
+  /// expand - Expand a record into a set of elements if possible. Return a
   /// pointer to the expanded elements, or NULL if Set cannot be expanded
   /// further.
   const RecVec *expand(const Record *Set);
diff --git a/llvm/include/llvm/TableGen/StringMatcher.h b/llvm/include/llvm/TableGen/StringMatcher.h
index 49769883a98b..70554de1a6f1 100644
--- a/llvm/include/llvm/TableGen/StringMatcher.h
+++ b/llvm/include/llvm/TableGen/StringMatcher.h
@@ -26,7 +26,7 @@ class raw_ostream;
 /// simple switch tree to classify the input string.
 ///
 /// If a match is found, the code in Matches[i].second is executed; control must
-/// not exit this code fragment.  If nothing matches, execution falls through.
+/// not exit this code fragment. If nothing matches, execution falls through.
 class StringMatcher {
 public:
   using StringPair = std::pair<std::string, std::string>;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 4a92dc16c1bf..6033d80e717d 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1132,8 +1132,8 @@ def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg,
 def udiv_by_const : GICombineRule<
   (defs root:$root),
   (match (wip_match_opcode G_UDIV):$root,
-   [{ return Helper.matchUDivByConst(*${root}); }]),
-  (apply [{ Helper.applyUDivByConst(*${root}); }])>;
+   [{ return Helper.matchUDivorURemByConst(*${root}); }]),
+  (apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
 
 def sdiv_by_const : GICombineRule<
   (defs root:$root),
@@ -1156,6 +1156,14 @@ def udiv_by_pow2 : GICombineRule<
 def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const,
                                       sdiv_by_pow2, udiv_by_pow2]>;
 
+def urem_by_const : GICombineRule<
+  (defs root:$root),
+  (match (G_UREM $dst, $x, $y):$root,
+   [{ return Helper.matchUDivorURemByConst(*${root}); }]),
+  (apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
+
+def intrem_combines : GICombineGroup<[urem_by_const]>;
+
 def reassoc_ptradd : GICombineRule<
   (defs root:$root, build_fn_matchinfo:$matchinfo),
   (match (wip_match_opcode G_PTR_ADD):$root,
@@ -2048,7 +2056,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     constant_fold_cast_op, fabs_fneg_fold,
     intdiv_combines, mulh_combines, redundant_neg_operands,
     and_or_disjoint_mask, fma_combines, fold_binop_into_select,
-    sub_add_reg, select_to_minmax,
+    intrem_combines, sub_add_reg, select_to_minmax,
     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
     simplify_neg_minmax, combine_concat_vector,
     sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index ce9a2b275196..4c83f8a580aa 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -538,11 +538,11 @@ include "llvm/Target/TargetInstrPredicate.td"
 include "llvm/Target/TargetSchedule.td"
 
 class InstructionEncoding {
-  // Size of encoded instruction.
+  // Size of encoded instruction in bytes.
   int Size;
 
-  // The "namespace" in which this instruction exists, on targets like ARM
-  // which multiple ISA namespaces exist.
+  // The "namespace" in which this instruction exists (on targets like ARM
+  // where multiple ISA namespaces exist).
   string DecoderNamespace = "";
 
   // List of predicates which will be turned into isel matching code.
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 6bdbf8faa1a0..a2321b842545 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -39,6 +39,7 @@ using ModulePassManager = PassManager<Module>;
 
 class Function;
 class GlobalValue;
+class MachineInstr;
 class MachineModuleInfoWrapperPass;
 struct MachineSchedContext;
 class Mangler;
@@ -529,6 +530,15 @@ class LLVM_ABI TargetMachine {
 
   // MachineRegisterInfo callback function
   virtual void registerMachineRegisterInfoCallback(MachineFunction &MF) const {}
+
+  /// Remove all Linker Optimization Hints (LOH) associated with instructions in
+  /// \p MIs and \return the number of hints removed. This is useful in
+  /// transformations that cause these hints to be illegal, like in the machine
+  /// outliner.
+  virtual size_t clearLinkerOptimizationHints(
+      const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+    return 0;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/TargetParser/SubtargetFeature.h b/llvm/include/llvm/TargetParser/SubtargetFeature.h
index cdcfcdd0e802..a48b18745352 100644
--- a/llvm/include/llvm/TargetParser/SubtargetFeature.h
+++ b/llvm/include/llvm/TargetParser/SubtargetFeature.h
@@ -56,7 +56,7 @@ class FeatureBitset {
   }
 
   FeatureBitset &set() {
-    std::fill(std::begin(Bits), std::end(Bits), -1ULL);
+    llvm::fill(Bits, -1ULL);
     return *this;
   }
 
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index 2ea7829d668a..2f68d66dee90 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -40,89 +40,89 @@ enum GPUKind : uint32_t {
   GK_NONE = 0,
 
   // R600-based processors.
-  GK_R600 = 1,
-  GK_R630 = 2,
-  GK_RS880 = 3,
-  GK_RV670 = 4,
-  GK_RV710 = 5,
-  GK_RV730 = 6,
-  GK_RV770 = 7,
-  GK_CEDAR = 8,
-  GK_CYPRESS = 9,
-  GK_JUNIPER = 10,
-  GK_REDWOOD = 11,
-  GK_SUMO = 12,
-  GK_BARTS = 13,
-  GK_CAICOS = 14,
-  GK_CAYMAN = 15,
-  GK_TURKS = 16,
+  GK_R600,
+  GK_R630,
+  GK_RS880,
+  GK_RV670,
+  GK_RV710,
+  GK_RV730,
+  GK_RV770,
+  GK_CEDAR,
+  GK_CYPRESS,
+  GK_JUNIPER,
+  GK_REDWOOD,
+  GK_SUMO,
+  GK_BARTS,
+  GK_CAICOS,
+  GK_CAYMAN,
+  GK_TURKS,
 
   GK_R600_FIRST = GK_R600,
   GK_R600_LAST = GK_TURKS,
 
   // AMDGCN-based processors.
-  GK_GFX600 = 32,
-  GK_GFX601 = 33,
-  GK_GFX602 = 34,
-
-  GK_GFX700 = 40,
-  GK_GFX701 = 41,
-  GK_GFX702 = 42,
-  GK_GFX703 = 43,
-  GK_GFX704 = 44,
-  GK_GFX705 = 45,
-
-  GK_GFX801 = 50,
-  GK_GFX802 = 51,
-  GK_GFX803 = 52,
-  GK_GFX805 = 53,
-  GK_GFX810 = 54,
-
-  GK_GFX900 = 60,
-  GK_GFX902 = 61,
-  GK_GFX904 = 62,
-  GK_GFX906 = 63,
-  GK_GFX908 = 64,
-  GK_GFX909 = 65,
-  GK_GFX90A = 66,
-  GK_GFX90C = 67,
-  GK_GFX942 = 70,
-  GK_GFX950 = 71,
-
-  GK_GFX1010 = 72,
-  GK_GFX1011 = 73,
-  GK_GFX1012 = 74,
-  GK_GFX1013 = 75,
-  GK_GFX1030 = 76,
-  GK_GFX1031 = 77,
-  GK_GFX1032 = 78,
-  GK_GFX1033 = 79,
-  GK_GFX1034 = 80,
-  GK_GFX1035 = 81,
-  GK_GFX1036 = 82,
-
-  GK_GFX1100 = 90,
-  GK_GFX1101 = 91,
-  GK_GFX1102 = 92,
-  GK_GFX1103 = 93,
-  GK_GFX1150 = 94,
-  GK_GFX1151 = 95,
-  GK_GFX1152 = 96,
-  GK_GFX1153 = 97,
-
-  GK_GFX1200 = 100,
-  GK_GFX1201 = 101,
-  GK_GFX1250 = 102,
+  GK_GFX600,
+  GK_GFX601,
+  GK_GFX602,
+
+  GK_GFX700,
+  GK_GFX701,
+  GK_GFX702,
+  GK_GFX703,
+  GK_GFX704,
+  GK_GFX705,
+
+  GK_GFX801,
+  GK_GFX802,
+  GK_GFX803,
+  GK_GFX805,
+  GK_GFX810,
+
+  GK_GFX900,
+  GK_GFX902,
+  GK_GFX904,
+  GK_GFX906,
+  GK_GFX908,
+  GK_GFX909,
+  GK_GFX90A,
+  GK_GFX90C,
+  GK_GFX942,
+  GK_GFX950,
+
+  GK_GFX1010,
+  GK_GFX1011,
+  GK_GFX1012,
+  GK_GFX1013,
+  GK_GFX1030,
+  GK_GFX1031,
+  GK_GFX1032,
+  GK_GFX1033,
+  GK_GFX1034,
+  GK_GFX1035,
+  GK_GFX1036,
+
+  GK_GFX1100,
+  GK_GFX1101,
+  GK_GFX1102,
+  GK_GFX1103,
+  GK_GFX1150,
+  GK_GFX1151,
+  GK_GFX1152,
+  GK_GFX1153,
+
+  GK_GFX1200,
+  GK_GFX1201,
+  GK_GFX1250,
 
   GK_AMDGCN_FIRST = GK_GFX600,
   GK_AMDGCN_LAST = GK_GFX1250,
 
-  GK_GFX9_GENERIC = 192,
-  GK_GFX10_1_GENERIC = 193,
-  GK_GFX10_3_GENERIC = 194,
-  GK_GFX11_GENERIC = 195,
-  GK_GFX12_GENERIC = 196,
-  GK_GFX9_4_GENERIC = 197,
+  GK_GFX9_GENERIC,
+  GK_GFX10_1_GENERIC,
+  GK_GFX10_3_GENERIC,
+  GK_GFX11_GENERIC,
+  GK_GFX12_GENERIC,
+  GK_GFX9_4_GENERIC,
 
   GK_AMDGCN_GENERIC_FIRST = GK_GFX9_GENERIC,
   GK_AMDGCN_GENERIC_LAST = GK_GFX9_4_GENERIC,
@@ -214,7 +214,7 @@ struct BasicSubtargetSubTypeKV {
   }
 };
 
-std::optional<llvm::StringMap<bool>>
+LLVM_ABI std::optional<llvm::StringMap<bool>>
 getCPUDefaultTargetFeatures(StringRef CPU,
                             ArrayRef<BasicSubtargetSubTypeKV> ProcDesc,
                             ArrayRef<BasicSubtargetFeatureKV> ProcFeatures);
diff --git a/llvm/include/llvm/TargetParser/Triple.h b/llvm/include/llvm/TargetParser/Triple.h
index d6fa4537ee3b..657f4230379e 100644
--- a/llvm/include/llvm/TargetParser/Triple.h
+++ b/llvm/include/llvm/TargetParser/Triple.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_TARGETPARSER_TRIPLE_H
 #define LLVM_TARGETPARSER_TRIPLE_H
 
-#include "llvm/ADT/Twine.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/VersionTuple.h"
 
@@ -20,6 +20,8 @@
 #undef sparc
 
 namespace llvm {
+enum class ExceptionHandling;
+class Twine;
 
 /// Triple - Helper class for working with autoconf configuration names. For
 /// historical reasons, we also call these 'triples' (they used to contain
@@ -349,7 +351,12 @@ class Triple {
   /// triple fields unknown.
   Triple() = default;
 
+  LLVM_ABI explicit Triple(std::string &&Str);
+  explicit Triple(StringRef Str) : Triple(Str.str()) {}
+  explicit Triple(const char *Str) : Triple(std::string(Str)) {}
+  explicit Triple(const std::string &Str) : Triple(std::string(Str)) {}
   LLVM_ABI explicit Triple(const Twine &Str);
+
   LLVM_ABI Triple(const Twine &ArchStr, const Twine &VendorStr,
                   const Twine &OSStr);
   LLVM_ABI Triple(const Twine &ArchStr, const Twine &VendorStr,
@@ -581,6 +588,9 @@ class Triple {
   /// Is this an Apple XROS triple.
   bool isXROS() const { return getOS() == Triple::XROS; }
 
+  /// Is this an Apple BridgeOS triple.
+  bool isBridgeOS() const { return getOS() == Triple::BridgeOS; }
+
   /// Is this an Apple DriverKit triple.
   bool isDriverKit() const { return getOS() == Triple::DriverKit; }
 
@@ -591,9 +601,11 @@ class Triple {
     return (getVendor() == Triple::Apple) && isOSBinFormatMachO();
   }
 
-  /// Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
+  /// Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, DriverKit, XROS, or
+  /// bridgeOS).
   bool isOSDarwin() const {
-    return isMacOSX() || isiOS() || isWatchOS() || isDriverKit() || isXROS();
+    return isMacOSX() || isiOS() || isWatchOS() || isDriverKit() || isXROS() ||
+           isBridgeOS();
   }
 
   bool isSimulatorEnvironment() const {
@@ -1310,6 +1322,8 @@ class Triple {
   /// Returns whether an OS version is invalid and would not map to an Apple OS.
   LLVM_ABI static bool isValidVersionForOS(OSType OSKind,
                                            const VersionTuple &Version);
+
+  LLVM_ABI ExceptionHandling getDefaultExceptionHandling() const;
 };
 
 } // End llvm namespace
diff --git a/llvm/include/llvm/Telemetry/Telemetry.h b/llvm/include/llvm/Telemetry/Telemetry.h
index bc0056ec7e84..708ec439ed40 100644
--- a/llvm/include/llvm/Telemetry/Telemetry.h
+++ b/llvm/include/llvm/Telemetry/Telemetry.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 #include <map>
 #include <memory>
@@ -101,7 +102,7 @@ struct EntryKind {
 /// For example, The LLDB debugger can define a DebugCommandInfo subclass
 /// which has additional fields about the debug-command being instrumented,
 /// such as `CommandArguments` or `CommandName`.
-struct TelemetryInfo {
+struct LLVM_ABI TelemetryInfo {
   // This represents a unique-id, conventionally corresponding to
   // a tool's session - i.e., every time the tool starts until it exits.
   //
@@ -141,10 +142,15 @@ class Destination {
 /// and this framework.
 /// It is responsible for collecting telemetry data from the tool being
 /// monitored and transmitting the data elsewhere.
-class Manager {
+class LLVM_ABI Manager {
 public:
+  Manager() = default;
   virtual ~Manager() = default;
 
+  // Explicitly non-copyable.
+  Manager(Manager const &) = delete;
+  Manager &operator=(Manager const &) = delete;
+
   // Dispatch Telemetry data to the Destination(s).
   // The argument is non-const because the Manager may add or remove
   // data from the entry.
diff --git a/llvm/include/llvm/Testing/Demangle/DemangleTestCases.inc b/llvm/include/llvm/Testing/Demangle/DemangleTestCases.inc
index 1e3f7459deaa..2721d2aa5504 100644
--- a/llvm/include/llvm/Testing/Demangle/DemangleTestCases.inc
+++ b/llvm/include/llvm/Testing/Demangle/DemangleTestCases.inc
@@ -6,6 +6,7 @@
 {"_Z1fDU10_", "f(unsigned _BitInt(10))"},
 {"_Z1fIfEvDUstPT__", "void f<float>(unsigned _BitInt(sizeof (float*)))"},
 {"_Z1fIiEvDBstPT__", "void f<int>(_BitInt(sizeof (int*)))"},
+{"_Z6myfuncRDB8_S0_", "myfunc(_BitInt(8)&, _BitInt(8)&)"},
 {"_Z4testI1A1BE1Cv", "C test<A, B>()"},
 {"_Z4testI1A1BET0_T_S3_", "B test<A, B>(A, A)"},
 {"_ZN1SgtEi", "S::operator>(int)"},
diff --git a/llvm/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h b/llvm/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
index d144f62f1cc1..30a260c52f1d 100644
--- a/llvm/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
+++ b/llvm/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
@@ -14,10 +14,12 @@
 #ifndef LLVM_TOOLDRIVERS_LLVM_DLLTOOL_DLLTOOLDRIVER_H
 #define LLVM_TOOLDRIVERS_LLVM_DLLTOOL_DLLTOOLDRIVER_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 template <typename T> class ArrayRef;
 
-int dlltoolDriverMain(ArrayRef<const char *> ArgsArr);
+LLVM_ABI int dlltoolDriverMain(ArrayRef<const char *> ArgsArr);
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/ToolDrivers/llvm-lib/LibDriver.h b/llvm/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
index 23a2fc348a89..43fc5d72d3a8 100644
--- a/llvm/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
+++ b/llvm/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
@@ -14,11 +14,12 @@
 #ifndef LLVM_TOOLDRIVERS_LLVM_LIB_LIBDRIVER_H
 #define LLVM_TOOLDRIVERS_LLVM_LIB_LIBDRIVER_H
 
+#include "llvm/Support/Compiler.h"
+
 namespace llvm {
 template <typename T> class ArrayRef;
 
-int libDriverMain(ArrayRef<const char *> ARgs);
-
+LLVM_ABI int libDriverMain(ArrayRef<const char *> ARgs);
 }
 
 #endif
diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
index 20850ba4c02d..a9a370b27988 100644
--- a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
+++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h
@@ -41,6 +41,13 @@ class HipStdParAllocationInterpositionPass
   static bool isRequired() { return true; }
 };
 
+class HipStdParMathFixupPass : public PassInfoMixin<HipStdParMathFixupPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
+
+  static bool isRequired() { return true; }
+};
+
 } // namespace llvm
 
 #endif // LLVM_TRANSFORMS_HIPSTDPAR_HIPSTDPAR_H
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index e6eb756df987..f19f3292c479 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -6335,6 +6335,47 @@ struct AAUnderlyingObjects : AbstractAttribute {
                           AA::ValueScope Scope = AA::Interprocedural) const = 0;
 };
 
+/// An abstract interface for identifying pointers from which loads can be
+/// marked invariant.
+struct AAInvariantLoadPointer : public AbstractAttribute {
+  AAInvariantLoadPointer(const IRPosition &IRP) : AbstractAttribute(IRP) {}
+
+  /// See AbstractAttribute::isValidIRPositionForInit
+  static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+    if (!IRP.getAssociatedType()->isPointerTy())
+      return false;
+
+    return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+  }
+
+  /// Create an abstract attribute view for the position \p IRP.
+  static AAInvariantLoadPointer &createForPosition(const IRPosition &IRP,
+                                                   Attributor &A);
+
+  /// Return true if the pointer's contents are known to remain invariant.
+  virtual bool isKnownInvariant() const = 0;
+  virtual bool isKnownLocallyInvariant() const = 0;
+
+  /// Return true if the pointer's contents are assumed to remain invariant.
+  virtual bool isAssumedInvariant() const = 0;
+  virtual bool isAssumedLocallyInvariant() const = 0;
+
+  /// See AbstractAttribute::getName().
+  StringRef getName() const override { return "AAInvariantLoadPointer"; }
+
+  /// See AbstractAttribute::getIdAddr().
+  const char *getIdAddr() const override { return &ID; }
+
+  /// This function should return true if the type of the \p AA is
+  /// AAInvariantLoadPointer
+  static bool classof(const AbstractAttribute *AA) {
+    return (AA->getIdAddr() == &ID);
+  }
+
+  /// Unique ID (due to the unique address).
+  static const char ID;
+};
+
 /// An abstract interface for address space information.
 struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   AAAddressSpace(const IRPosition &IRP, Attributor &A)
diff --git a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
index 7b095ea24b4b..37bc72877983 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/LowerAllowCheckPass.h
@@ -27,6 +27,7 @@ class LowerAllowCheckPass : public PassInfoMixin<LowerAllowCheckPass> {
 public:
   struct Options {
     std::vector<unsigned int> cutoffs;
+    unsigned int runtime_check = 0;
   };
 
   explicit LowerAllowCheckPass(LowerAllowCheckPass::Options Opts)
diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index 1619fa31fb6f..c243e236901d 100644
--- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -52,11 +52,10 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/ilist.h"
-#include "llvm/ADT/ilist_node.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
@@ -79,7 +78,7 @@ struct PredicateConstraint {
 
 // Base class for all predicate information we provide.
 // All of our predicate information has at least a comparison.
-class PredicateBase : public ilist_node<PredicateBase> {
+class PredicateBase {
 public:
   PredicateType Type;
   // The original operand before we renamed it.
@@ -96,7 +95,6 @@ class PredicateBase : public ilist_node<PredicateBase> {
   PredicateBase(const PredicateBase &) = delete;
   PredicateBase &operator=(const PredicateBase &) = delete;
   PredicateBase() = delete;
-  virtual ~PredicateBase() = default;
   static bool classof(const PredicateBase *PB) {
     return PB->Type == PT_Assume || PB->Type == PT_Branch ||
            PB->Type == PT_Switch;
@@ -177,7 +175,8 @@ class PredicateSwitch : public PredicateWithEdge {
 /// accesses.
 class PredicateInfo {
 public:
-  LLVM_ABI PredicateInfo(Function &, DominatorTree &, AssumptionCache &);
+  LLVM_ABI PredicateInfo(Function &, DominatorTree &, AssumptionCache &,
+                         BumpPtrAllocator &);
   LLVM_ABI ~PredicateInfo();
 
   LLVM_ABI void verifyPredicateInfo() const;
@@ -197,9 +196,6 @@ class PredicateInfo {
 private:
   Function &F;
 
-  // This owns the all the predicate infos in the function, placed or not.
-  iplist<PredicateBase> AllInfos;
-
   // This maps from copy operands to Predicate Info. Note that it does not own
   // the Predicate Info, they belong to the ValueInfo structs in the ValueInfos
   // vector.
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index d654ac3ec927..cba37363d047 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -301,6 +301,14 @@ class LoopVectorizationLegality {
   /// Returns the reduction variables found in the loop.
   const ReductionList &getReductionVars() const { return Reductions; }
 
+  /// Returns the recurrence descriptor associated with a given phi node \p PN,
+  /// expecting one to exist.
+  const RecurrenceDescriptor &getRecurrenceDescriptor(PHINode *PN) const {
+    assert(isReductionVariable(PN) &&
+           "only reductions have recurrence descriptors");
+    return Reductions.find(PN)->second;
+  }
+
   /// Returns the induction variables found in the loop.
   const InductionList &getInductionVars() const { return Inductions; }
 
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index 6d2144b14bb0..ec70350691ab 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -307,7 +307,8 @@ class SeedCollector {
   }
 
 public:
-  LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE);
+  LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
+                         bool CollectStores, bool CollectLoads);
   LLVM_ABI ~SeedCollector();
 
   iterator_range<SeedContainer::iterator> getStoreSeeds() {
diff --git a/llvm/include/llvm/WindowsDriver/MSVCPaths.h b/llvm/include/llvm/WindowsDriver/MSVCPaths.h
index 51ffd6b6bc2c..076c2eaf9c3f 100644
--- a/llvm/include/llvm/WindowsDriver/MSVCPaths.h
+++ b/llvm/include/llvm/WindowsDriver/MSVCPaths.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 #include <optional>
 #include <string>
@@ -36,58 +37,64 @@ enum class ToolsetLayout {
 // Windows SDKs and VC Toolchains group their contents into subdirectories based
 // on the target architecture. This function converts an llvm::Triple::ArchType
 // to the corresponding subdirectory name.
-const char *archToWindowsSDKArch(llvm::Triple::ArchType Arch);
+LLVM_ABI const char *archToWindowsSDKArch(llvm::Triple::ArchType Arch);
 
 // Similar to the above function, but for Visual Studios before VS2017.
-const char *archToLegacyVCArch(llvm::Triple::ArchType Arch);
+LLVM_ABI const char *archToLegacyVCArch(llvm::Triple::ArchType Arch);
 
 // Similar to the above function, but for DevDiv internal builds.
-const char *archToDevDivInternalArch(llvm::Triple::ArchType Arch);
+LLVM_ABI const char *archToDevDivInternalArch(llvm::Triple::ArchType Arch);
 
-bool appendArchToWindowsSDKLibPath(int SDKMajor, llvm::SmallString<128> LibPath,
-                                   llvm::Triple::ArchType Arch,
-                                   std::string &path);
+LLVM_ABI bool appendArchToWindowsSDKLibPath(int SDKMajor,
+                                            llvm::SmallString<128> LibPath,
+                                            llvm::Triple::ArchType Arch,
+                                            std::string &path);
 
 // Get the path to a specific subdirectory in the current toolchain for
 // a given target architecture.
 // VS2017 changed the VC toolchain layout, so this should be used instead
 // of hardcoding paths.
-std::string getSubDirectoryPath(SubDirectoryType Type, ToolsetLayout VSLayout,
-                                const std::string &VCToolChainPath,
-                                llvm::Triple::ArchType TargetArch,
-                                llvm::StringRef SubdirParent = "");
+LLVM_ABI std::string getSubDirectoryPath(SubDirectoryType Type,
+                                         ToolsetLayout VSLayout,
+                                         const std::string &VCToolChainPath,
+                                         llvm::Triple::ArchType TargetArch,
+                                         llvm::StringRef SubdirParent = "");
 
 // Check if the Include path of a specified version of Visual Studio contains
 // specific header files. If not, they are probably shipped with Universal CRT.
-bool useUniversalCRT(ToolsetLayout VSLayout, const std::string &VCToolChainPath,
-                     llvm::Triple::ArchType TargetArch,
-                     llvm::vfs::FileSystem &VFS);
+LLVM_ABI bool useUniversalCRT(ToolsetLayout VSLayout,
+                              const std::string &VCToolChainPath,
+                              llvm::Triple::ArchType TargetArch,
+                              llvm::vfs::FileSystem &VFS);
 
 /// Get Windows SDK installation directory.
-bool getWindowsSDKDir(vfs::FileSystem &VFS,
+LLVM_ABI bool getWindowsSDKDir(vfs::FileSystem &VFS,
+                               std::optional<llvm::StringRef> WinSdkDir,
+                               std::optional<llvm::StringRef> WinSdkVersion,
+                               std::optional<llvm::StringRef> WinSysRoot,
+                               std::string &Path, int &Major,
+                               std::string &WindowsSDKIncludeVersion,
+                               std::string &WindowsSDKLibVersion);
+
+LLVM_ABI bool
+getUniversalCRTSdkDir(vfs::FileSystem &VFS,
                       std::optional<llvm::StringRef> WinSdkDir,
                       std::optional<llvm::StringRef> WinSdkVersion,
                       std::optional<llvm::StringRef> WinSysRoot,
-                      std::string &Path, int &Major,
-                      std::string &WindowsSDKIncludeVersion,
-                      std::string &WindowsSDKLibVersion);
-
-bool getUniversalCRTSdkDir(vfs::FileSystem &VFS,
-                           std::optional<llvm::StringRef> WinSdkDir,
-                           std::optional<llvm::StringRef> WinSdkVersion,
-                           std::optional<llvm::StringRef> WinSysRoot,
-                           std::string &Path, std::string &UCRTVersion);
+                      std::string &Path, std::string &UCRTVersion);
 
 // Check command line arguments to try and find a toolchain.
-bool findVCToolChainViaCommandLine(
-    vfs::FileSystem &VFS, std::optional<llvm::StringRef> VCToolsDir,
-    std::optional<llvm::StringRef> VCToolsVersion,
-    std::optional<llvm::StringRef> WinSysRoot, std::string &Path,
-    ToolsetLayout &VSLayout);
+LLVM_ABI bool
+findVCToolChainViaCommandLine(vfs::FileSystem &VFS,
+                              std::optional<llvm::StringRef> VCToolsDir,
+                              std::optional<llvm::StringRef> VCToolsVersion,
+                              std::optional<llvm::StringRef> WinSysRoot,
+                              std::string &Path, ToolsetLayout &VSLayout);
 
 // Check various environment variables to try and find a toolchain.
-bool findVCToolChainViaEnvironment(vfs::FileSystem &VFS, std::string &Path,
-                                   ToolsetLayout &VSLayout);
+LLVM_ABI bool findVCToolChainViaEnvironment(vfs::FileSystem &VFS,
+                                            std::string &Path,
+                                            ToolsetLayout &VSLayout);
 
 // Query the Setup Config server for installs, then pick the newest version
 // and find its default VC toolchain. If `VCToolsVersion` is specified, that
@@ -95,7 +102,7 @@ bool findVCToolChainViaEnvironment(vfs::FileSystem &VFS, std::string &Path,
 //
 // This is the preferred way to discover new Visual Studios, as they're no
 // longer listed in the registry.
-bool
+LLVM_ABI bool
 findVCToolChainViaSetupConfig(vfs::FileSystem &VFS,
                               std::optional<llvm::StringRef> VCToolsVersion,
                               std::string &Path, ToolsetLayout &VSLayout);
@@ -103,7 +110,8 @@ findVCToolChainViaSetupConfig(vfs::FileSystem &VFS,
 // Look in the registry for Visual Studio installs, and use that to get
 // a toolchain path. VS2017 and newer don't get added to the registry.
 // So if we find something here, we know that it's an older version.
-bool findVCToolChainViaRegistry(std::string &Path, ToolsetLayout &VSLayout);
+LLVM_ABI bool findVCToolChainViaRegistry(std::string &Path,
+                                         ToolsetLayout &VSLayout);
 
 } // namespace llvm
 
diff --git a/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h b/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h
index 2da74bb9dce8..0c938aceeebb 100644
--- a/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h
+++ b/llvm/include/llvm/WindowsManifest/WindowsManifestMerger.h
@@ -25,6 +25,7 @@
 #ifndef LLVM_WINDOWSMANIFEST_WINDOWSMANIFESTMERGER_H
 #define LLVM_WINDOWSMANIFEST_WINDOWSMANIFESTMERGER_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
 
 namespace llvm {
@@ -34,9 +35,10 @@ class MemoryBufferRef;
 
 namespace windows_manifest {
 
-bool isAvailable();
+LLVM_ABI bool isAvailable();
 
-class WindowsManifestError : public ErrorInfo<WindowsManifestError, ECError> {
+class LLVM_ABI WindowsManifestError
+    : public ErrorInfo<WindowsManifestError, ECError> {
 public:
   static char ID;
   WindowsManifestError(const Twine &Msg);
@@ -48,13 +50,13 @@ class WindowsManifestError : public ErrorInfo<WindowsManifestError, ECError> {
 
 class WindowsManifestMerger {
 public:
-  WindowsManifestMerger();
-  ~WindowsManifestMerger();
-  Error merge(MemoryBufferRef Manifest);
+  LLVM_ABI WindowsManifestMerger();
+  LLVM_ABI ~WindowsManifestMerger();
+  LLVM_ABI Error merge(MemoryBufferRef Manifest);
 
   // Returns vector containing merged xml manifest, or uninitialized vector for
   // empty manifest.
-  std::unique_ptr<MemoryBuffer> getMergedManifest();
+  LLVM_ABI std::unique_ptr<MemoryBuffer> getMergedManifest();
 
 private:
   class WindowsManifestMergerImpl;
diff --git a/llvm/include/llvm/XRay/InstrumentationMap.h b/llvm/include/llvm/XRay/InstrumentationMap.h
index 54737e226df8..b5371478680e 100644
--- a/llvm/include/llvm/XRay/InstrumentationMap.h
+++ b/llvm/include/llvm/XRay/InstrumentationMap.h
@@ -85,7 +85,7 @@ class InstrumentationMap {
   FunctionAddressMap FunctionAddresses;
   FunctionAddressReverseMap FunctionIds;
 
-  LLVM_ABI_FRIEND friend Expected<InstrumentationMap>
+  LLVM_ABI friend Expected<InstrumentationMap>
       loadInstrumentationMap(StringRef);
 
 public:
diff --git a/llvm/include/llvm/XRay/Trace.h b/llvm/include/llvm/XRay/Trace.h
index af1d35c67817..5e4e40aa7ca9 100644
--- a/llvm/include/llvm/XRay/Trace.h
+++ b/llvm/include/llvm/XRay/Trace.h
@@ -51,7 +51,7 @@ class Trace {
 
   typedef std::vector<XRayRecord>::const_iterator citerator;
 
-  LLVM_ABI_FRIEND friend Expected<Trace> loadTrace(const DataExtractor &, bool);
+  LLVM_ABI friend Expected<Trace> loadTrace(const DataExtractor &, bool);
 
 public:
   using size_type = RecordVector::size_type;
diff --git a/llvm/include/module.modulemap b/llvm/include/module.modulemap
index 9d7dbb8e4b07..a4bd6b9ed370 100644
--- a/llvm/include/module.modulemap
+++ b/llvm/include/module.modulemap
@@ -268,7 +268,6 @@ module LLVM_IR {
   textual header "llvm/IR/FixedMetadataKinds.def"
   textual header "llvm/IR/Value.def"
   textual header "llvm/IR/VPIntrinsics.def"
-  textual header "llvm/IR/RuntimeLibcalls.def"
 }
 
 module LLVM_IRReader {
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 31611dfe4fd2..86a2edbd8bd4 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -839,6 +839,10 @@ MemoryEffects BasicAAResult::getMemoryEffects(const CallBase *Call,
       FuncME |= MemoryEffects::readOnly();
     if (Call->hasClobberingOperandBundles())
       FuncME |= MemoryEffects::writeOnly();
+    if (Call->isVolatile()) {
+      // Volatile operations also access inaccessible memory.
+      FuncME |= MemoryEffects::inaccessibleMemOnly();
+    }
     Min &= FuncME;
   }
 
diff --git a/llvm/lib/Analysis/BranchProbabilityInfo.cpp b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
index 0ff16dbec24e..671e023415a6 100644
--- a/llvm/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/llvm/lib/Analysis/BranchProbabilityInfo.cpp
@@ -798,8 +798,9 @@ BranchProbabilityInfo::getInitialEstimatedBlockWeight(const BasicBlock *BB) {
 // Does RPO traversal over all blocks in \p F and assigns weights to
 // 'unreachable', 'noreturn', 'cold', 'unwind' blocks. In addition it does its
 // best to propagate the weight to up/down the IR.
-void BranchProbabilityInfo::computeEestimateBlockWeight(
-    const Function &F, DominatorTree *DT, PostDominatorTree *PDT) {
+void BranchProbabilityInfo::estimateBlockWeights(const Function &F,
+                                                 DominatorTree *DT,
+                                                 PostDominatorTree *PDT) {
   SmallVector<BasicBlock *, 8> BlockWorkList;
   SmallVector<LoopBlock, 8> LoopWorkList;
   SmallDenseMap<LoopData, SmallVector<BasicBlock *, 4>> LoopExitBlocks;
@@ -1244,7 +1245,7 @@ void BranchProbabilityInfo::calculate(const Function &F, const LoopInfo &LoopI,
     PDT = PDTPtr.get();
   }
 
-  computeEestimateBlockWeight(F, DT, PDT);
+  estimateBlockWeights(F, DT, PDT);
 
   // Walk the basic blocks in post-order so that we can build up state about
   // the successors of a block iteratively.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index b58f9b26a865..6e469c034d9c 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1963,7 +1963,7 @@ inline bool llvm_fenv_testexcept() {
   return false;
 }
 
-static const APFloat FTZPreserveSign(const APFloat &V) {
+static APFloat FTZPreserveSign(const APFloat &V) {
   if (V.isDenormal())
     return APFloat::getZero(V.getSemantics(), V.isNegative());
   return V;
@@ -2229,17 +2229,6 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
     return nullptr;
   }
 
-  if (isa<PoisonValue>(Operands[0])) {
-    // TODO: All of these operations should probably propagate poison.
-    switch (IntrinsicID) {
-    case Intrinsic::canonicalize:
-    case Intrinsic::sqrt:
-      return PoisonValue::get(Ty);
-    default:
-      break;
-    }
-  }
-
   if (isa<UndefValue>(Operands[0])) {
     // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN.
     // ctpop() is between 0 and bitwidth, pick 0 for undef.
@@ -2553,6 +2542,9 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
       case Intrinsic::cosh:
         return ConstantFoldFP(cosh, APF, Ty);
       case Intrinsic::atan:
+        // Implement optional behavior from C's Annex F for +/-0.0.
+        if (U.isZero())
+          return ConstantFP::get(Ty->getContext(), U);
         return ConstantFoldFP(atan, APF, Ty);
       case Intrinsic::sqrt:
         return ConstantFoldFP(sqrt, APF, Ty);
@@ -2606,6 +2598,9 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
       break;
     case LibFunc_atan:
     case LibFunc_atanf:
+      // Implement optional behavior from C's Annex F for +/-0.0.
+      if (U.isZero())
+        return ConstantFP::get(Ty->getContext(), U);
       if (TLI->has(Func))
         return ConstantFoldFP(atan, APF, Ty);
       break;
@@ -3222,11 +3217,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
     case Intrinsic::smin:
     case Intrinsic::umax:
     case Intrinsic::umin:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3239,9 +3229,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
 
     case Intrinsic::scmp:
     case Intrinsic::ucmp:
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 || !C1)
         return ConstantInt::get(Ty, 0);
 
@@ -3308,11 +3295,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
     }
     case Intrinsic::uadd_sat:
     case Intrinsic::sadd_sat:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3323,11 +3305,6 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
         return ConstantInt::get(Ty, C0->sadd_sat(*C1));
     case Intrinsic::usub_sat:
     case Intrinsic::ssub_sat:
-      // This is the same as for binary ops - poison propagates.
-      // TODO: Poison handling should be consolidated.
-      if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-        return PoisonValue::get(Ty);
-
       if (!C0 && !C1)
         return UndefValue::get(Ty);
       if (!C0 || !C1)
@@ -3586,11 +3563,6 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
 
   if (IntrinsicID == Intrinsic::smul_fix ||
       IntrinsicID == Intrinsic::smul_fix_sat) {
-    // poison * C -> poison
-    // C * poison -> poison
-    if (isa<PoisonValue>(Operands[0]) || isa<PoisonValue>(Operands[1]))
-      return PoisonValue::get(Ty);
-
     const APInt *C0, *C1;
     if (!getConstIntOrUndef(Operands[0], C0) ||
         !getConstIntOrUndef(Operands[1], C1))
@@ -3664,6 +3636,11 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
                                         ArrayRef<Constant *> Operands,
                                         const TargetLibraryInfo *TLI,
                                         const CallBase *Call) {
+  if (IntrinsicID != Intrinsic::not_intrinsic &&
+      any_of(Operands, IsaPred<PoisonValue>) &&
+      intrinsicPropagatesPoison(IntrinsicID))
+    return PoisonValue::get(Ty);
+
   if (Operands.size() == 1)
     return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
 
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index c1b1d002c997..5b85060f9caa 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -2372,20 +2372,17 @@ bool DependenceInfo::testMIV(const SCEV *Src, const SCEV *Dst,
     banerjeeMIVtest(Src, Dst, Loops, Result);
 }
 
-
 // Given a product, e.g., 10*X*Y, returns the first constant operand,
-// in this case 10. If there is no constant part, returns NULL.
-static
-const SCEVConstant *getConstantPart(const SCEV *Expr) {
+// in this case 10. If there is no constant part, returns std::nullopt.
+static std::optional<APInt> getConstantPart(const SCEV *Expr) {
   if (const auto *Constant = dyn_cast<SCEVConstant>(Expr))
-    return Constant;
-  else if (const auto *Product = dyn_cast<SCEVMulExpr>(Expr))
+    return Constant->getAPInt();
+  if (const auto *Product = dyn_cast<SCEVMulExpr>(Expr))
     if (const auto *Constant = dyn_cast<SCEVConstant>(Product->getOperand(0)))
-      return Constant;
-  return nullptr;
+      return Constant->getAPInt();
+  return std::nullopt;
 }
 
-
 //===----------------------------------------------------------------------===//
 // gcdMIVtest -
 // Tests an MIV subscript pair for dependence.
@@ -2421,11 +2418,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    const auto *Constant = getConstantPart(Coeff);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+    if (!ConstCoeff)
       return false;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     Coefficients = AddRec->getStart();
   }
   const SCEV *SrcConst = Coefficients;
@@ -2440,11 +2436,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    const auto *Constant = getConstantPart(Coeff);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+    if (!ConstCoeff)
       return false;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     Coefficients = AddRec->getStart();
   }
   const SCEV *DstConst = Coefficients;
@@ -2463,12 +2458,10 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) {
         // Search for constant operand to participate in GCD;
         // If none found; return false.
-        const SCEVConstant *ConstOp = getConstantPart(Product);
+        std::optional<APInt> ConstOp = getConstantPart(Product);
         if (!ConstOp)
           return false;
-        APInt ConstOpValue = ConstOp->getAPInt();
-        ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD,
-                                                   ConstOpValue.abs());
+        ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD, ConstOp->abs());
       }
       else
         return false;
@@ -2520,11 +2513,11 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else {
         // If the coefficient is the product of a constant and other stuff,
         // we can use the constant in the GCD computation.
-        Constant = getConstantPart(Coeff);
-        if (!Constant)
+        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+        if (!ConstCoeff)
           return false;
-        APInt ConstCoeff = Constant->getAPInt();
-        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+        RunningGCD =
+            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
       }
       Inner = AddRec->getStart();
     }
@@ -2537,24 +2530,23 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       else {
         // If the coefficient is the product of a constant and other stuff,
         // we can use the constant in the GCD computation.
-        Constant = getConstantPart(Coeff);
-        if (!Constant)
+        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
+        if (!ConstCoeff)
           return false;
-        APInt ConstCoeff = Constant->getAPInt();
-        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+        RunningGCD =
+            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
       }
       Inner = AddRec->getStart();
     }
     Delta = SE->getMinusSCEV(SrcCoeff, DstCoeff);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
-    Constant = getConstantPart(Delta);
-    if (!Constant)
+    std::optional<APInt> ConstCoeff = getConstantPart(Delta);
+    if (!ConstCoeff)
       // The difference of the two coefficients might not be a product
       // or constant, in which case we give up on this direction.
       continue;
-    APInt ConstCoeff = Constant->getAPInt();
-    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
     LLVM_DEBUG(dbgs() << "\tRunningGCD = " << RunningGCD << "\n");
     if (RunningGCD != 0) {
       Remainder = ConstDelta.srem(RunningGCD);
@@ -4188,67 +4180,65 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
     }
   }
 
-  if (Coupled.count()) {
-    // test coupled subscript groups
-    SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
-    for (unsigned II = 0; II <= MaxLevels; ++II)
-      Constraints[II].setAny(SE);
-    for (unsigned SI : Coupled.set_bits()) {
-      SmallBitVector Group(Pair[SI].Group);
-      SmallBitVector Sivs(Pairs);
-      SmallBitVector Mivs(Pairs);
-      SmallBitVector ConstrainedLevels(MaxLevels + 1);
-      for (unsigned SJ : Group.set_bits()) {
-        if (Pair[SJ].Classification == Subscript::SIV)
-          Sivs.set(SJ);
-        else
-          Mivs.set(SJ);
+  assert(!Coupled.empty() && "coupled expected non-empty");
+
+  // test coupled subscript groups
+  SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
+  for (unsigned II = 0; II <= MaxLevels; ++II)
+    Constraints[II].setAny(SE);
+  for (unsigned SI : Coupled.set_bits()) {
+    SmallBitVector Group(Pair[SI].Group);
+    SmallBitVector Sivs(Pairs);
+    SmallBitVector Mivs(Pairs);
+    SmallBitVector ConstrainedLevels(MaxLevels + 1);
+    for (unsigned SJ : Group.set_bits()) {
+      if (Pair[SJ].Classification == Subscript::SIV)
+        Sivs.set(SJ);
+      else
+        Mivs.set(SJ);
+    }
+    while (Sivs.any()) {
+      bool Changed = false;
+      for (unsigned SJ : Sivs.set_bits()) {
+        // SJ is an SIV subscript that's part of the current coupled group
+        unsigned Level;
+        const SCEV *SplitIter = nullptr;
+        (void)testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level, Result, NewConstraint,
+                      SplitIter);
+        if (Level == SplitLevel && SplitIter)
+          return SplitIter;
+        ConstrainedLevels.set(Level);
+        if (intersectConstraints(&Constraints[Level], &NewConstraint))
+          Changed = true;
+        Sivs.reset(SJ);
       }
-      while (Sivs.any()) {
-        bool Changed = false;
-        for (unsigned SJ : Sivs.set_bits()) {
-          // SJ is an SIV subscript that's part of the current coupled group
-          unsigned Level;
-          const SCEV *SplitIter = nullptr;
-          (void) testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level,
-                         Result, NewConstraint, SplitIter);
-          if (Level == SplitLevel && SplitIter)
-            return SplitIter;
-          ConstrainedLevels.set(Level);
-          if (intersectConstraints(&Constraints[Level], &NewConstraint))
-            Changed = true;
-          Sivs.reset(SJ);
-        }
-        if (Changed) {
-          // propagate, possibly creating new SIVs and ZIVs
-          for (unsigned SJ : Mivs.set_bits()) {
-            // SJ is an MIV subscript that's part of the current coupled group
-            if (propagate(Pair[SJ].Src, Pair[SJ].Dst,
-                          Pair[SJ].Loops, Constraints, Result.Consistent)) {
-              Pair[SJ].Classification =
-                classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()),
-                             Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()),
-                             Pair[SJ].Loops);
-              switch (Pair[SJ].Classification) {
-              case Subscript::ZIV:
-                Mivs.reset(SJ);
-                break;
-              case Subscript::SIV:
-                Sivs.set(SJ);
-                Mivs.reset(SJ);
-                break;
-              case Subscript::RDIV:
-              case Subscript::MIV:
-                break;
-              default:
-                llvm_unreachable("bad subscript classification");
-              }
-            }
-          }
+      if (!Changed)
+        continue;
+      // propagate, possibly creating new SIVs and ZIVs
+      for (unsigned SJ : Mivs.set_bits()) {
+        // SJ is an MIV subscript that's part of the current coupled group
+        if (!propagate(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Constraints,
+                       Result.Consistent))
+          continue;
+        Pair[SJ].Classification = classifyPair(
+            Pair[SJ].Src, LI->getLoopFor(Src->getParent()), Pair[SJ].Dst,
+            LI->getLoopFor(Dst->getParent()), Pair[SJ].Loops);
+        switch (Pair[SJ].Classification) {
+        case Subscript::ZIV:
+          Mivs.reset(SJ);
+          break;
+        case Subscript::SIV:
+          Sivs.set(SJ);
+          Mivs.reset(SJ);
+          break;
+        case Subscript::RDIV:
+        case Subscript::MIV:
+          break;
+        default:
+          llvm_unreachable("bad subscript classification");
         }
       }
     }
   }
   llvm_unreachable("somehow reached end of routine");
-  return nullptr;
 }
diff --git a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
index e7e8f2ac1ff2..790e00e1b3b0 100644
--- a/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/DevelopmentModeInlineAdvisor.cpp
@@ -27,8 +27,8 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ManagedStatic.h"
 
-#include <vector>
 #include <optional>
+#include <vector>
 
 using namespace llvm;
 
@@ -260,9 +260,8 @@ static const std::vector<TensorSpec> TrainingOnlyFeatures{
 
 static const std::vector<TensorSpec> getInputFeatures() {
   std::vector<TensorSpec> InputSpecs;
-  for (size_t I = 0; I < NumberOfFeatures; ++I)
-    InputSpecs.push_back(
-        TensorSpec(TFFeedPrefix + FeatureMap[I].name(), FeatureMap[I]));
+  for (const auto &Feature : FeatureMap)
+    InputSpecs.push_back(TensorSpec(TFFeedPrefix + Feature.name(), Feature));
   append_range(InputSpecs, TrainingOnlyFeatures);
   return InputSpecs;
 }
@@ -299,7 +298,8 @@ void TrainingLogger::logInlineEvent(const InlineEvent &Event,
                                     const MLModelRunner &ModelRunner) {
   L->startObservation();
   size_t CurrentFeature = 0;
-  for (; CurrentFeature < NumberOfFeatures; ++CurrentFeature)
+  size_t FeatureMapSize = FeatureMap.size();
+  for (; CurrentFeature < FeatureMapSize; ++CurrentFeature)
     L->logTensorValue(CurrentFeature,
                       reinterpret_cast<const char *>(
                           ModelRunner.getTensorUntyped(CurrentFeature)));
diff --git a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
index 9d044c8a3591..dd4eb7f0df05 100644
--- a/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
+++ b/llvm/lib/Analysis/FunctionPropertiesAnalysis.cpp
@@ -199,6 +199,25 @@ void FunctionPropertiesInfo::updateForBB(const BasicBlock &BB,
 #undef CHECK_OPERAND
     }
   }
+
+  if (IR2VecVocab) {
+    // We instantiate the IR2Vec embedder each time, as having an unique
+    // pointer to the embedder as member of the class would make it
+    // non-copyable. Instantiating the embedder in itself is not costly.
+    auto Embedder = ir2vec::Embedder::create(IR2VecKind::Symbolic,
+                                             *BB.getParent(), *IR2VecVocab);
+    if (!Embedder) {
+      BB.getContext().emitError("Error creating IR2Vec embeddings");
+      return;
+    }
+    const auto &BBEmbedding = Embedder->getBBVector(BB);
+    // Subtract BBEmbedding from Function embedding if the direction is -1,
+    // and add it if the direction is +1.
+    if (Direction == -1)
+      FunctionEmbedding -= BBEmbedding;
+    else
+      FunctionEmbedding += BBEmbedding;
+  }
 }
 
 void FunctionPropertiesInfo::updateAggregateStats(const Function &F,
@@ -220,14 +239,24 @@ void FunctionPropertiesInfo::updateAggregateStats(const Function &F,
 
 FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
     Function &F, FunctionAnalysisManager &FAM) {
+  // We use the cached result of the IR2VecVocabAnalysis run by
+  // InlineAdvisorAnalysis. If the IR2VecVocabAnalysis is not run, we don't
+  // use IR2Vec embeddings.
+  auto VocabResult = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                         .getCachedResult<IR2VecVocabAnalysis>(*F.getParent());
   return getFunctionPropertiesInfo(F, FAM.getResult<DominatorTreeAnalysis>(F),
-                                   FAM.getResult<LoopAnalysis>(F));
+                                   FAM.getResult<LoopAnalysis>(F), VocabResult);
 }
 
 FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
-    const Function &F, const DominatorTree &DT, const LoopInfo &LI) {
+    const Function &F, const DominatorTree &DT, const LoopInfo &LI,
+    const IR2VecVocabResult *VocabResult) {
 
   FunctionPropertiesInfo FPI;
+  if (VocabResult && VocabResult->isValid()) {
+    FPI.IR2VecVocab = VocabResult->getVocabulary();
+    FPI.FunctionEmbedding = ir2vec::Embedding(VocabResult->getDimension(), 0.0);
+  }
   for (const auto &BB : F)
     if (DT.isReachableFromEntry(&BB))
       FPI.reIncludeBB(BB);
@@ -235,6 +264,66 @@ FunctionPropertiesInfo FunctionPropertiesInfo::getFunctionPropertiesInfo(
   return FPI;
 }
 
+bool FunctionPropertiesInfo::operator==(
+    const FunctionPropertiesInfo &FPI) const {
+  if (BasicBlockCount != FPI.BasicBlockCount ||
+      BlocksReachedFromConditionalInstruction !=
+          FPI.BlocksReachedFromConditionalInstruction ||
+      Uses != FPI.Uses ||
+      DirectCallsToDefinedFunctions != FPI.DirectCallsToDefinedFunctions ||
+      LoadInstCount != FPI.LoadInstCount ||
+      StoreInstCount != FPI.StoreInstCount ||
+      MaxLoopDepth != FPI.MaxLoopDepth ||
+      TopLevelLoopCount != FPI.TopLevelLoopCount ||
+      TotalInstructionCount != FPI.TotalInstructionCount ||
+      BasicBlocksWithSingleSuccessor != FPI.BasicBlocksWithSingleSuccessor ||
+      BasicBlocksWithTwoSuccessors != FPI.BasicBlocksWithTwoSuccessors ||
+      BasicBlocksWithMoreThanTwoSuccessors !=
+          FPI.BasicBlocksWithMoreThanTwoSuccessors ||
+      BasicBlocksWithSinglePredecessor !=
+          FPI.BasicBlocksWithSinglePredecessor ||
+      BasicBlocksWithTwoPredecessors != FPI.BasicBlocksWithTwoPredecessors ||
+      BasicBlocksWithMoreThanTwoPredecessors !=
+          FPI.BasicBlocksWithMoreThanTwoPredecessors ||
+      BigBasicBlocks != FPI.BigBasicBlocks ||
+      MediumBasicBlocks != FPI.MediumBasicBlocks ||
+      SmallBasicBlocks != FPI.SmallBasicBlocks ||
+      CastInstructionCount != FPI.CastInstructionCount ||
+      FloatingPointInstructionCount != FPI.FloatingPointInstructionCount ||
+      IntegerInstructionCount != FPI.IntegerInstructionCount ||
+      ConstantIntOperandCount != FPI.ConstantIntOperandCount ||
+      ConstantFPOperandCount != FPI.ConstantFPOperandCount ||
+      ConstantOperandCount != FPI.ConstantOperandCount ||
+      InstructionOperandCount != FPI.InstructionOperandCount ||
+      BasicBlockOperandCount != FPI.BasicBlockOperandCount ||
+      GlobalValueOperandCount != FPI.GlobalValueOperandCount ||
+      InlineAsmOperandCount != FPI.InlineAsmOperandCount ||
+      ArgumentOperandCount != FPI.ArgumentOperandCount ||
+      UnknownOperandCount != FPI.UnknownOperandCount ||
+      CriticalEdgeCount != FPI.CriticalEdgeCount ||
+      ControlFlowEdgeCount != FPI.ControlFlowEdgeCount ||
+      UnconditionalBranchCount != FPI.UnconditionalBranchCount ||
+      IntrinsicCount != FPI.IntrinsicCount ||
+      DirectCallCount != FPI.DirectCallCount ||
+      IndirectCallCount != FPI.IndirectCallCount ||
+      CallReturnsIntegerCount != FPI.CallReturnsIntegerCount ||
+      CallReturnsFloatCount != FPI.CallReturnsFloatCount ||
+      CallReturnsPointerCount != FPI.CallReturnsPointerCount ||
+      CallReturnsVectorIntCount != FPI.CallReturnsVectorIntCount ||
+      CallReturnsVectorFloatCount != FPI.CallReturnsVectorFloatCount ||
+      CallReturnsVectorPointerCount != FPI.CallReturnsVectorPointerCount ||
+      CallWithManyArgumentsCount != FPI.CallWithManyArgumentsCount ||
+      CallWithPointerArgumentCount != FPI.CallWithPointerArgumentCount) {
+    return false;
+  }
+  // Check the equality of the function embeddings. We don't check the equality
+  // of Vocabulary as it remains the same.
+  if (!FunctionEmbedding.approximatelyEquals(FPI.FunctionEmbedding))
+    return false;
+
+  return true;
+}
+
 void FunctionPropertiesInfo::print(raw_ostream &OS) const {
 #define PRINT_PROPERTY(PROP_NAME) OS << #PROP_NAME ": " << PROP_NAME << "\n";
 
@@ -322,6 +411,16 @@ FunctionPropertiesUpdater::FunctionPropertiesUpdater(
   // The caller's entry BB may change due to new alloca instructions.
   LikelyToChangeBBs.insert(&*Caller.begin());
 
+  // The users of the value returned by call instruction can change
+  // leading to the change in embeddings being computed, when used.
+  // We conservatively add the BBs with such uses to LikelyToChangeBBs.
+  for (const auto *User : CB.users())
+    CallUsers.insert(dyn_cast<Instruction>(User)->getParent());
+  // CallSiteBB can be removed from CallUsers if present, it's taken care
+  // separately.
+  CallUsers.erase(&CallSiteBB);
+  LikelyToChangeBBs.insert_range(CallUsers);
+
   // The successors may become unreachable in the case of `invoke` inlining.
   // We track successors separately, too, because they form a boundary, together
   // with the CB BB ('Entry') between which the inlined callee will be pasted.
@@ -435,6 +534,9 @@ void FunctionPropertiesUpdater::finish(FunctionAnalysisManager &FAM) const {
   if (&CallSiteBB != &*Caller.begin())
     Reinclude.insert(&*Caller.begin());
 
+  // Reinclude the BBs which use the values returned by call instruction
+  Reinclude.insert_range(CallUsers);
+
   // Distribute the successors to the 2 buckets.
   for (const auto *Succ : Successors)
     if (DT.isReachableFromEntry(Succ))
@@ -486,6 +588,9 @@ bool FunctionPropertiesUpdater::isUpdateValid(Function &F,
     return false;
   DominatorTree DT(F);
   LoopInfo LI(DT);
-  auto Fresh = FunctionPropertiesInfo::getFunctionPropertiesInfo(F, DT, LI);
+  auto VocabResult = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                         .getCachedResult<IR2VecVocabAnalysis>(*F.getParent());
+  auto Fresh =
+      FunctionPropertiesInfo::getFunctionPropertiesInfo(F, DT, LI, VocabResult);
   return FPI == Fresh;
 }
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 1a5dfbe4bed6..2cc3ad5f1848 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -247,16 +247,10 @@ KnownBits ValueEvolution::compute(const Value *V) {
 }
 
 bool ValueEvolution::computeEvolutions(ArrayRef<PhiStepPair> PhiEvolutions) {
-  for (unsigned I = 0; I < TripCount; ++I) {
-    for (auto [Phi, Step] : PhiEvolutions) {
-      KnownBits KnownAtIter = computeInstr(Step);
-      if (KnownAtIter.getBitWidth() < I + 1) {
-        ErrStr = "Loop iterations exceed bitwidth of result";
-        return false;
-      }
-      KnownPhis.emplace_or_assign(Phi, KnownAtIter);
-    }
-  }
+  for (unsigned I = 0; I < TripCount; ++I)
+    for (auto [Phi, Step] : PhiEvolutions)
+      KnownPhis.emplace_or_assign(Phi, computeInstr(Step));
+
   return ErrStr.empty();
 }
 
@@ -369,7 +363,7 @@ RecurrenceInfo::digRecurrence(Instruction *V,
 /// A Conditional Recurrence is a recurrence of the form:
 ///
 /// loop:
-///    %rec = [%start, %entry], [%step, %loop]
+///    %rec = phi [%start, %entry], [%step, %loop]
 ///    ...
 ///    %step = select _, %tv, %fv
 ///
@@ -497,42 +491,46 @@ CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
   return Table;
 }
 
-/// Checks if \p Reference is reachable from \p Needle on the use-def chain, and
-/// that there are no stray PHI nodes while digging the use-def chain. \p
-/// BOToMatch is a CRC peculiarity: at least one of the Users of Needle needs to
-/// match this OpCode, which is XOR for CRC.
-static bool arePHIsIntertwined(
-    const PHINode *Needle, const PHINode *Reference, const Loop &L,
-    Instruction::BinaryOps BOToMatch = Instruction::BinaryOpsEnd) {
-  // Initialize the worklist with Users of the Needle.
+/// Checks that \p P1 and \p P2 are used together in an XOR in the use-def chain
+/// of \p SI's condition, ignoring any casts. The purpose of this function is to
+/// ensure that LHSAux from the SimpleRecurrence is used correctly in the CRC
+/// computation. We cannot check the correctness of casts at this point, and
+/// rely on the KnownBits propagation to check correctness of the CRC
+/// computation.
+///
+/// In other words, it checks for the following pattern:
+///
+/// loop:
+///   %P1 = phi [_, %entry], [%P1.next, %loop]
+///   %P2 = phi [_, %entry], [%P2.next, %loop]
+///   ...
+///   %xor = xor (CastOrSelf %P1), (CastOrSelf %P2)
+///
+/// where %xor is in the use-def chain of \p SI's condition.
+static bool isConditionalOnXorOfPHIs(const SelectInst *SI, const PHINode *P1,
+                                     const PHINode *P2, const Loop &L) {
   SmallVector<const Instruction *> Worklist;
-  for (const User *U : Needle->users()) {
-    if (auto *UI = dyn_cast<Instruction>(U))
-      if (L.contains(UI))
-        Worklist.push_back(UI);
-  }
 
-  // BOToMatch is usually XOR for CRC.
-  if (BOToMatch != Instruction::BinaryOpsEnd) {
-    if (count_if(Worklist, [BOToMatch](const Instruction *I) {
-          return I->getOpcode() == BOToMatch;
-        }) != 1)
-      return false;
-  }
+  // matchConditionalRecurrence has already ensured that the SelectInst's
+  // condition is an Instruction.
+  Worklist.push_back(cast<Instruction>(SI->getCondition()));
 
   while (!Worklist.empty()) {
     const Instruction *I = Worklist.pop_back_val();
 
-    // Since Needle is never pushed onto the Worklist, I must either be the
-    // Reference PHI node (in which case we're done), or a stray PHI node (in
-    // which case we abort).
+    // Don't add a PHI's operands to the Worklist.
     if (isa<PHINode>(I))
-      return I == Reference;
+      continue;
 
+    // If we match an XOR of the two PHIs ignoring casts, we're done.
+    if (match(I, m_c_Xor(m_CastOrSelf(m_Specific(P1)),
+                         m_CastOrSelf(m_Specific(P2)))))
+      return true;
+
+    // Continue along the use-def chain.
     for (const Use &U : I->operands())
       if (auto *UI = dyn_cast<Instruction>(U))
-        // Don't push Needle back onto the Worklist.
-        if (UI != Needle && L.contains(UI))
+        if (L.contains(UI))
           Worklist.push_back(UI);
   }
   return false;
@@ -586,11 +584,28 @@ HashRecognize::recognizeCRC() const {
   if (SimpleRecurrence) {
     if (isBigEndianBitShift(SimpleRecurrence.BO, SE) != ByteOrderSwapped)
       return "Loop with non-unit bitshifts";
-    if (!arePHIsIntertwined(SimpleRecurrence.Phi, ConditionalRecurrence.Phi, L,
-                            Instruction::BinaryOps::Xor))
-      return "Simple recurrence doesn't use conditional recurrence with XOR";
+
+    // Ensure that the PHIs have exactly two uses:
+    // the bit-shift, and the XOR (or a cast feeding into the XOR).
+    if (!ConditionalRecurrence.Phi->hasNUses(2) ||
+        !SimpleRecurrence.Phi->hasNUses(2))
+      return "Recurrences have stray uses";
+
+    // Check that the SelectInst ConditionalRecurrence.Step is conditional on
+    // the XOR of SimpleRecurrence.Phi and ConditionalRecurrence.Phi.
+    if (!isConditionalOnXorOfPHIs(cast<SelectInst>(ConditionalRecurrence.Step),
+                                  SimpleRecurrence.Phi,
+                                  ConditionalRecurrence.Phi, L))
+      return "Recurrences not intertwined with XOR";
   }
 
+  // Make sure that the TC doesn't exceed the bitwidth of LHSAux, or LHS.
+  Value *LHS = ConditionalRecurrence.Start;
+  Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
+  if (TC > (LHSAux ? LHSAux->getType()->getIntegerBitWidth()
+                   : LHS->getType()->getIntegerBitWidth()))
+    return "Loop iterations exceed bitwidth of data";
+
   // Make sure that the computed value is used in the exit block: this should be
   // true even if it is only really used in an outer loop's exit block, since
   // the loop is in LCSSA form.
@@ -619,13 +634,13 @@ HashRecognize::recognizeCRC() const {
     return VE.getError();
   KnownBits ResultBits = VE.KnownPhis.at(ConditionalRecurrence.Phi);
 
+  unsigned N = std::min(TC, ResultBits.getBitWidth());
   auto IsZero = [](const KnownBits &K) { return K.isZero(); };
-  if (!checkExtractBits(ResultBits, TC, IsZero, *ByteOrderSwapped))
+  if (!checkExtractBits(ResultBits, N, IsZero, *ByteOrderSwapped))
     return ErrBits(ResultBits, TC, *ByteOrderSwapped);
 
-  Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
-  return PolynomialInfo(TC, ConditionalRecurrence.Start, GenPoly, ComputedValue,
-                        *ByteOrderSwapped, LHSAux);
+  return PolynomialInfo(TC, LHS, GenPoly, ComputedValue, *ByteOrderSwapped,
+                        LHSAux);
 }
 
 void CRCTable::print(raw_ostream &OS) const {
diff --git a/llvm/lib/Analysis/IR2Vec.cpp b/llvm/lib/Analysis/IR2Vec.cpp
index fa38c35796a0..bf456102bb61 100644
--- a/llvm/lib/Analysis/IR2Vec.cpp
+++ b/llvm/lib/Analysis/IR2Vec.cpp
@@ -70,7 +70,6 @@ inline bool fromJSON(const llvm::json::Value &E, Embedding &Out,
 // ==----------------------------------------------------------------------===//
 // Embedding
 //===----------------------------------------------------------------------===//
-
 Embedding &Embedding::operator+=(const Embedding &RHS) {
   assert(this->size() == RHS.size() && "Vectors must have the same dimension");
   std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
@@ -78,6 +77,12 @@ Embedding &Embedding::operator+=(const Embedding &RHS) {
   return *this;
 }
 
+Embedding Embedding::operator+(const Embedding &RHS) const {
+  Embedding Result(*this);
+  Result += RHS;
+  return Result;
+}
+
 Embedding &Embedding::operator-=(const Embedding &RHS) {
   assert(this->size() == RHS.size() && "Vectors must have the same dimension");
   std::transform(this->begin(), this->end(), RHS.begin(), this->begin(),
@@ -85,6 +90,24 @@ Embedding &Embedding::operator-=(const Embedding &RHS) {
   return *this;
 }
 
+Embedding Embedding::operator-(const Embedding &RHS) const {
+  Embedding Result(*this);
+  Result -= RHS;
+  return Result;
+}
+
+Embedding &Embedding::operator*=(double Factor) {
+  std::transform(this->begin(), this->end(), this->begin(),
+                 [Factor](double Elem) { return Elem * Factor; });
+  return *this;
+}
+
+Embedding Embedding::operator*(double Factor) const {
+  Embedding Result(*this);
+  Result *= Factor;
+  return Result;
+}
+
 Embedding &Embedding::scaleAndAdd(const Embedding &Src, float Factor) {
   assert(this->size() == Src.size() && "Vectors must have the same dimension");
   for (size_t Itr = 0; Itr < this->size(); ++Itr)
@@ -101,6 +124,13 @@ bool Embedding::approximatelyEquals(const Embedding &RHS,
   return true;
 }
 
+void Embedding::print(raw_ostream &OS) const {
+  OS << " [";
+  for (const auto &Elem : Data)
+    OS << " " << format("%.2f", Elem) << " ";
+  OS << "]\n";
+}
+
 // ==----------------------------------------------------------------------===//
 // Embedder and its subclasses
 //===----------------------------------------------------------------------===//
@@ -110,13 +140,13 @@ Embedder::Embedder(const Function &F, const Vocab &Vocabulary)
       Dimension(Vocabulary.begin()->second.size()), OpcWeight(::OpcWeight),
       TypeWeight(::TypeWeight), ArgWeight(::ArgWeight) {}
 
-Expected<std::unique_ptr<Embedder>>
-Embedder::create(IR2VecKind Mode, const Function &F, const Vocab &Vocabulary) {
+std::unique_ptr<Embedder> Embedder::create(IR2VecKind Mode, const Function &F,
+                                           const Vocab &Vocabulary) {
   switch (Mode) {
   case IR2VecKind::Symbolic:
     return std::make_unique<SymbolicEmbedder>(F, Vocabulary);
   }
-  return make_error<StringError>("Unknown IR2VecKind", errc::invalid_argument);
+  return nullptr;
 }
 
 // FIXME: Currently lookups are string based. Use numeric Keys
@@ -196,18 +226,12 @@ void SymbolicEmbedder::computeEmbeddings(const BasicBlock &BB) const {
   for (const auto &I : BB.instructionsWithoutDebug()) {
     Embedding InstVector(Dimension, 0);
 
-    const auto OpcVec = lookupVocab(I.getOpcodeName());
-    InstVector.scaleAndAdd(OpcVec, OpcWeight);
-
     // FIXME: Currently lookups are string based. Use numeric Keys
     // for efficiency.
-    const auto Type = I.getType();
-    const auto TypeVec = getTypeEmbedding(Type);
-    InstVector.scaleAndAdd(TypeVec, TypeWeight);
-
+    InstVector += lookupVocab(I.getOpcodeName());
+    InstVector += getTypeEmbedding(I.getType());
     for (const auto &Op : I.operands()) {
-      const auto OperandVec = getOperandEmbedding(Op.get());
-      InstVector.scaleAndAdd(OperandVec, ArgWeight);
+      InstVector += getOperandEmbedding(Op.get());
     }
     InstVecMap[&I] = InstVector;
     BBVector += InstVector;
@@ -251,6 +275,43 @@ bool IR2VecVocabResult::invalidate(
   return !(PAC.preservedWhenStateless());
 }
 
+Error IR2VecVocabAnalysis::parseVocabSection(
+    StringRef Key, const json::Value &ParsedVocabValue,
+    ir2vec::Vocab &TargetVocab, unsigned &Dim) {
+  json::Path::Root Path("");
+  const json::Object *RootObj = ParsedVocabValue.getAsObject();
+  if (!RootObj)
+    return createStringError(errc::invalid_argument,
+                             "JSON root is not an object");
+
+  const json::Value *SectionValue = RootObj->get(Key);
+  if (!SectionValue)
+    return createStringError(errc::invalid_argument,
+                             "Missing '" + std::string(Key) +
+                                 "' section in vocabulary file");
+  if (!json::fromJSON(*SectionValue, TargetVocab, Path))
+    return createStringError(errc::illegal_byte_sequence,
+                             "Unable to parse '" + std::string(Key) +
+                                 "' section from vocabulary");
+
+  Dim = TargetVocab.begin()->second.size();
+  if (Dim == 0)
+    return createStringError(errc::illegal_byte_sequence,
+                             "Dimension of '" + std::string(Key) +
+                                 "' section of the vocabulary is zero");
+
+  if (!std::all_of(TargetVocab.begin(), TargetVocab.end(),
+                   [Dim](const std::pair<StringRef, Embedding> &Entry) {
+                     return Entry.second.size() == Dim;
+                   }))
+    return createStringError(
+        errc::illegal_byte_sequence,
+        "All vectors in the '" + std::string(Key) +
+            "' section of the vocabulary are not of the same dimension");
+
+  return Error::success();
+}
+
 // FIXME: Make this optional. We can avoid file reads
 // by auto-generating a default vocabulary during the build time.
 Error IR2VecVocabAnalysis::readVocabulary() {
@@ -259,32 +320,40 @@ Error IR2VecVocabAnalysis::readVocabulary() {
     return createFileError(VocabFile, BufOrError.getError());
 
   auto Content = BufOrError.get()->getBuffer();
-  json::Path::Root Path("");
+
   Expected<json::Value> ParsedVocabValue = json::parse(Content);
   if (!ParsedVocabValue)
     return ParsedVocabValue.takeError();
 
-  bool Res = json::fromJSON(*ParsedVocabValue, Vocabulary, Path);
-  if (!Res)
-    return createStringError(errc::illegal_byte_sequence,
-                             "Unable to parse the vocabulary");
+  ir2vec::Vocab OpcodeVocab, TypeVocab, ArgVocab;
+  unsigned OpcodeDim = 0, TypeDim = 0, ArgDim = 0;
+  if (auto Err = parseVocabSection("Opcodes", *ParsedVocabValue, OpcodeVocab,
+                                   OpcodeDim))
+    return Err;
 
-  if (Vocabulary.empty())
-    return createStringError(errc::illegal_byte_sequence,
-                             "Vocabulary is empty");
+  if (auto Err =
+          parseVocabSection("Types", *ParsedVocabValue, TypeVocab, TypeDim))
+    return Err;
 
-  unsigned Dim = Vocabulary.begin()->second.size();
-  if (Dim == 0)
+  if (auto Err =
+          parseVocabSection("Arguments", *ParsedVocabValue, ArgVocab, ArgDim))
+    return Err;
+
+  if (!(OpcodeDim == TypeDim && TypeDim == ArgDim))
     return createStringError(errc::illegal_byte_sequence,
-                             "Dimension of vocabulary is zero");
+                             "Vocabulary sections have different dimensions");
 
-  if (!std::all_of(Vocabulary.begin(), Vocabulary.end(),
-                   [Dim](const std::pair<StringRef, Embedding> &Entry) {
-                     return Entry.second.size() == Dim;
-                   }))
-    return createStringError(
-        errc::illegal_byte_sequence,
-        "All vectors in the vocabulary are not of the same dimension");
+  auto scaleVocabSection = [](ir2vec::Vocab &Vocab, double Weight) {
+    for (auto &Entry : Vocab)
+      Entry.second *= Weight;
+  };
+  scaleVocabSection(OpcodeVocab, OpcWeight);
+  scaleVocabSection(TypeVocab, TypeWeight);
+  scaleVocabSection(ArgVocab, ArgWeight);
+
+  Vocabulary.insert(OpcodeVocab.begin(), OpcodeVocab.end());
+  Vocabulary.insert(TypeVocab.begin(), TypeVocab.end());
+  Vocabulary.insert(ArgVocab.begin(), ArgVocab.end());
 
   return Error::success();
 }
@@ -304,7 +373,6 @@ void IR2VecVocabAnalysis::emitError(Error Err, LLVMContext &Ctx) {
 IR2VecVocabAnalysis::Result
 IR2VecVocabAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
   auto Ctx = &M.getContext();
-  // FIXME: Scale the vocabulary once. This would avoid scaling per use later.
   // If vocabulary is already populated by the constructor, use it.
   if (!Vocabulary.empty())
     return IR2VecVocabResult(std::move(Vocabulary));
@@ -323,16 +391,9 @@ IR2VecVocabAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
 }
 
 // ==----------------------------------------------------------------------===//
-// IR2VecPrinterPass
+// Printer Passes
 //===----------------------------------------------------------------------===//
 
-void IR2VecPrinterPass::printVector(const Embedding &Vec) const {
-  OS << " [";
-  for (const auto &Elem : Vec)
-    OS << " " << format("%.2f", Elem) << " ";
-  OS << "]\n";
-}
-
 PreservedAnalyses IR2VecPrinterPass::run(Module &M,
                                          ModuleAnalysisManager &MAM) {
   auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
@@ -340,20 +401,16 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
 
   auto Vocab = IR2VecVocabResult.getVocabulary();
   for (Function &F : M) {
-    Expected<std::unique_ptr<Embedder>> EmbOrErr =
+    std::unique_ptr<Embedder> Emb =
         Embedder::create(IR2VecKind::Symbolic, F, Vocab);
-    if (auto Err = EmbOrErr.takeError()) {
-      handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
-        OS << "Error creating IR2Vec embeddings: " << EI.message() << "\n";
-      });
+    if (!Emb) {
+      OS << "Error creating IR2Vec embeddings \n";
       continue;
     }
 
-    std::unique_ptr<Embedder> Emb = std::move(*EmbOrErr);
-
     OS << "IR2Vec embeddings for function " << F.getName() << ":\n";
     OS << "Function vector: ";
-    printVector(Emb->getFunctionVector());
+    Emb->getFunctionVector().print(OS);
 
     OS << "Basic block vectors:\n";
     const auto &BBMap = Emb->getBBVecMap();
@@ -361,7 +418,7 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
       auto It = BBMap.find(&BB);
       if (It != BBMap.end()) {
         OS << "Basic block: " << BB.getName() << ":\n";
-        printVector(It->second);
+        It->second.print(OS);
       }
     }
 
@@ -373,10 +430,24 @@ PreservedAnalyses IR2VecPrinterPass::run(Module &M,
         if (It != InstMap.end()) {
           OS << "Instruction: ";
           I.print(OS);
-          printVector(It->second);
+          It->second.print(OS);
         }
       }
     }
   }
   return PreservedAnalyses::all();
 }
+
+PreservedAnalyses IR2VecVocabPrinterPass::run(Module &M,
+                                              ModuleAnalysisManager &MAM) {
+  auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
+  assert(IR2VecVocabResult.isValid() && "IR2Vec Vocabulary is invalid");
+
+  auto Vocab = IR2VecVocabResult.getVocabulary();
+  for (const auto &Entry : Vocab) {
+    OS << "Key: " << Entry.first << ": ";
+    Entry.second.print(OS);
+  }
+
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index c8e97e5ec0e5..b275b1064cef 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -50,6 +50,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
   case RecurKind::UMax:
   case RecurKind::UMin:
   case RecurKind::AnyOf:
+  case RecurKind::FindFirstIVSMin:
   case RecurKind::FindLastIVSMax:
   case RecurKind::FindLastIVUMax:
     return true;
@@ -684,8 +685,9 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
 // value of the data type or a non-constant value by using mask and multiple
 // reduction operations.
 RecurrenceDescriptor::InstDesc
-RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
-                                          Instruction *I, ScalarEvolution &SE) {
+RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
+                                      PHINode *OrigPhi, Instruction *I,
+                                      ScalarEvolution &SE) {
   // TODO: Support the vectorization of FindLastIV when the reduction phi is
   // used by more than one select instruction. This vectorization is only
   // performed when the SCEV of each increasing induction variable used by the
@@ -713,25 +715,42 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
       return std::nullopt;
 
     const SCEV *Step = AR->getStepRecurrence(SE);
-    if (!SE.isKnownPositive(Step))
+    if ((isFindFirstIVRecurrenceKind(Kind) && !SE.isKnownNegative(Step)) ||
+        (isFindLastIVRecurrenceKind(Kind) && !SE.isKnownPositive(Step)))
       return std::nullopt;
 
     // Keep the minimum value of the recurrence type as the sentinel value.
     // The maximum acceptable range for the increasing induction variable,
     // called the valid range, will be defined as
+
+    // Keep the minimum (FindLast) or maximum (FindFirst) value of the
+    // recurrence type as the sentinel value. The maximum acceptable range for
+    // the induction variable, called the valid range, will be defined as
     //   [<sentinel value> + 1, <sentinel value>)
-    // where <sentinel value> is [Signed|Unsigned]Min(<recurrence type>)
+    // where <sentinel value> is [Signed|Unsigned]Min(<recurrence type>) for
+    // FindLastIV or [Signed|Unsigned]Max(<recurrence type>) for FindFirstIV.
     // TODO: This range restriction can be lifted by adding an additional
     // virtual OR reduction.
     auto CheckRange = [&](bool IsSigned) {
       const ConstantRange IVRange =
           IsSigned ? SE.getSignedRange(AR) : SE.getUnsignedRange(AR);
       unsigned NumBits = Ty->getIntegerBitWidth();
-      const APInt Sentinel = IsSigned ? APInt::getSignedMinValue(NumBits)
-                                      : APInt::getMinValue(NumBits);
-      const ConstantRange ValidRange =
-          ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
-      LLVM_DEBUG(dbgs() << "LV: FindLastIV valid range is " << ValidRange
+      ConstantRange ValidRange = ConstantRange::getEmpty(NumBits);
+      if (isFindLastIVRecurrenceKind(Kind)) {
+        APInt Sentinel = IsSigned ? APInt::getSignedMinValue(NumBits)
+                                  : APInt::getMinValue(NumBits);
+        ValidRange = ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
+      } else {
+        assert(IsSigned && "Only FindFirstIV with SMax is supported currently");
+        ValidRange =
+            ConstantRange::getNonEmpty(APInt::getSignedMinValue(NumBits),
+                                       APInt::getSignedMaxValue(NumBits) - 1);
+      }
+
+      LLVM_DEBUG(dbgs() << "LV: "
+                        << (isFindLastIVRecurrenceKind(Kind) ? "FindLastIV"
+                                                             : "FindFirstIV")
+                        << " valid range is " << ValidRange
                         << ", and the range of " << *AR << " is " << IVRange
                         << "\n");
 
@@ -739,10 +758,18 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
       // its range is fully contained within the valid range.
       return ValidRange.contains(IVRange);
     };
+    if (isFindLastIVRecurrenceKind(Kind)) {
+      if (CheckRange(true))
+        return RecurKind::FindLastIVSMax;
+      if (CheckRange(false))
+        return RecurKind::FindLastIVUMax;
+      return std::nullopt;
+    }
+    assert(isFindFirstIVRecurrenceKind(Kind) &&
+           "Kind must either be a FindLastIV or FindFirstIV");
+
     if (CheckRange(true))
-      return RecurKind::FindLastIVSMax;
-    if (CheckRange(false))
-      return RecurKind::FindLastIVUMax;
+      return RecurKind::FindFirstIVSMin;
     return std::nullopt;
   };
 
@@ -888,8 +915,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
     if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
         Kind == RecurKind::Add || Kind == RecurKind::Mul)
       return isConditionalRdxPattern(I);
-    if (isFindLastIVRecurrenceKind(Kind) && SE)
-      return isFindLastIVPattern(L, OrigPhi, I, *SE);
+    if (isFindIVRecurrenceKind(Kind) && SE)
+      return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
     [[fallthrough]];
   case Instruction::FCmp:
   case Instruction::ICmp:
@@ -1003,6 +1030,11 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
     LLVM_DEBUG(dbgs() << "Found a FindLastIV reduction PHI." << *Phi << "\n");
     return true;
   }
+  if (AddReductionVar(Phi, RecurKind::FindFirstIVSMin, TheLoop, FMF, RedDes, DB,
+                      AC, DT, SE)) {
+    LLVM_DEBUG(dbgs() << "Found a FindFirstIV reduction PHI." << *Phi << "\n");
+    return true;
+  }
   if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT,
                       SE)) {
     LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
@@ -1150,6 +1182,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
   case RecurKind::Mul:
     return Instruction::Mul;
   case RecurKind::AnyOf:
+  case RecurKind::FindFirstIVSMin:
   case RecurKind::FindLastIVSMax:
   case RecurKind::FindLastIVUMax:
   case RecurKind::Or:
diff --git a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
index 90065e053b62..7b93474e4dc7 100644
--- a/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
+++ b/llvm/lib/Analysis/IndirectCallPromotionAnalysis.cpp
@@ -31,12 +31,18 @@ static cl::opt<unsigned> ICPRemainingPercentThreshold(
 
 // The percent threshold for the direct-call target (this call site vs the
 // total call count) for it to be considered as the promotion target.
-static cl::opt<unsigned>
+static cl::opt<uint64_t>
     ICPTotalPercentThreshold("icp-total-percent-threshold", cl::init(5),
                              cl::Hidden,
                              cl::desc("The percentage threshold against total "
                                       "count for the promotion"));
 
+// Set the minimum absolute count threshold for indirect call promotion.
+// Candidates with counts below this threshold will not be promoted.
+static cl::opt<unsigned> ICPMinimumCountThreshold(
+    "icp-minimum-count-threshold", cl::init(0), cl::Hidden,
+    cl::desc("Minimum absolute count for promotion candidate"));
+
 // Set the maximum number of targets to promote for a single indirect-call
 // callsite.
 static cl::opt<unsigned>
@@ -51,7 +57,8 @@ cl::opt<unsigned> MaxNumVTableAnnotations(
 bool ICallPromotionAnalysis::isPromotionProfitable(uint64_t Count,
                                                    uint64_t TotalCount,
                                                    uint64_t RemainingCount) {
-  return Count * 100 >= ICPRemainingPercentThreshold * RemainingCount &&
+  return Count >= ICPMinimumCountThreshold &&
+         Count * 100 >= ICPRemainingPercentThreshold * RemainingCount &&
          Count * 100 >= ICPTotalPercentThreshold * TotalCount;
 }
 
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index 7325c71c0106..6f78ca083e6c 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/EphemeralValuesCache.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
@@ -64,6 +65,13 @@ static cl::opt<bool>
                         cl::desc("If true, annotate inline advisor remarks "
                                  "with LTO and pass information."));
 
+// This flag is used to enable IR2Vec embeddings in the ML inliner; Only valid
+// with ML inliner. The vocab file is used to initialize the embeddings.
+static cl::opt<std::string> IR2VecVocabFile(
+    "ml-inliner-ir2vec-vocab-file", cl::Hidden,
+    cl::desc("Vocab file for IR2Vec; Setting this enables "
+             "configuring the model to use IR2Vec embeddings."));
+
 namespace llvm {
 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
 } // namespace llvm
@@ -206,6 +214,20 @@ void InlineAdvice::recordInliningWithCalleeDeleted() {
 AnalysisKey InlineAdvisorAnalysis::Key;
 AnalysisKey PluginInlineAdvisorAnalysis::Key;
 
+bool InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(
+    Module &M, ModuleAnalysisManager &MAM) {
+  if (!IR2VecVocabFile.empty()) {
+    auto IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);
+    if (!IR2VecVocabResult.isValid()) {
+      M.getContext().emitError("Failed to load IR2Vec vocabulary");
+      return false;
+    }
+  }
+  // No vocab file specified is OK; We just don't use IR2Vec
+  // embeddings.
+  return true;
+}
+
 bool InlineAdvisorAnalysis::Result::tryCreate(
     InlineParams Params, InliningAdvisorMode Mode,
     const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
@@ -231,14 +253,21 @@ bool InlineAdvisorAnalysis::Result::tryCreate(
                                              /* EmitRemarks =*/true, IC);
     }
     break;
+    // Run IR2VecVocabAnalysis once per module to get the vocabulary.
+    // We run it here because it is immutable and we want to avoid running it
+    // multiple times.
   case InliningAdvisorMode::Development:
 #ifdef LLVM_HAVE_TFLITE
     LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
+    if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
+      return false;
     Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
 #endif
     break;
   case InliningAdvisorMode::Release:
     LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
+    if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))
+      return false;
     Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
     break;
   }
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index fe1ceb74429c..773a60479ae2 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -2316,9 +2316,18 @@ bool CallAnalyzer::visitStore(StoreInst &I) {
 }
 
 bool CallAnalyzer::visitExtractValue(ExtractValueInst &I) {
-  // Constant folding for extract value is trivial.
-  if (simplifyInstruction(I))
-    return true;
+  Value *Op = I.getAggregateOperand();
+
+  // Special handling, because we want to simplify extractvalue with a
+  // potential insertvalue from the caller.
+  if (Value *SimpleOp = getSimplifiedValueUnchecked(Op)) {
+    SimplifyQuery SQ(DL);
+    Value *SimpleV = simplifyExtractValueInst(SimpleOp, I.getIndices(), SQ);
+    if (SimpleV) {
+      SimplifiedValues[&I] = SimpleV;
+      return true;
+    }
+  }
 
   // SROA can't look through these, but they may be free.
   return Base::visitExtractValue(I);
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index cb1dae92faf9..82530e7d5b6c 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6484,9 +6484,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
     break;
   }
   case Intrinsic::ptrmask: {
-    if (isa<PoisonValue>(Op0) || isa<PoisonValue>(Op1))
-      return PoisonValue::get(Op0->getType());
-
     // NOTE: We can't apply this simplifications based on the value of Op1
     // because we need to preserve provenance.
     if (Q.isUndefValue(Op0) || match(Op0, m_Zero()))
@@ -6533,10 +6530,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
     if (match(Op0, m_ImmConstant()))
       std::swap(Op0, Op1);
 
-    // Propagate poison.
-    if (isa<PoisonValue>(Op1))
-      return Op1;
-
     // Assume undef is the limit value.
     if (Q.isUndefValue(Op1))
       return ConstantInt::get(
@@ -6696,9 +6689,6 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
       return Op1;
     break;
   case Intrinsic::is_fpclass: {
-    if (isa<PoisonValue>(Op0))
-      return PoisonValue::get(ReturnType);
-
     uint64_t Mask = cast<ConstantInt>(Op1)->getZExtValue();
     // If all tests are made, it doesn't matter what the value is.
     if ((Mask & fcAllFlags) == fcAllFlags)
@@ -6793,6 +6783,9 @@ static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,
   Function *F = cast<Function>(Callee);
   Intrinsic::ID IID = F->getIntrinsicID();
 
+  if (IID != Intrinsic::not_intrinsic && intrinsicPropagatesPoison(IID) &&
+      any_of(Args, IsaPred<PoisonValue>))
+    return PoisonValue::get(F->getReturnType());
   // Most of the intrinsics with no operands have some kind of side effect.
   // Don't simplify.
   if (!NumOperands) {
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 2a562484fc37..922f25de54e9 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1745,14 +1745,10 @@ LazyValueInfoImpl &LazyValueInfo::getOrCreateImpl(const Module *M) {
         Intrinsic::getDeclarationIfExists(M, Intrinsic::experimental_guard);
     PImpl = new LazyValueInfoImpl(AC, DL, GuardDecl);
   }
-  return *static_cast<LazyValueInfoImpl *>(PImpl);
+  return *PImpl;
 }
 
-LazyValueInfoImpl *LazyValueInfo::getImpl() {
-  if (!PImpl)
-    return nullptr;
-  return static_cast<LazyValueInfoImpl *>(PImpl);
-}
+LazyValueInfoImpl *LazyValueInfo::getImpl() { return PImpl; }
 
 LazyValueInfo::~LazyValueInfo() { releaseMemory(); }
 
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 212b3bf196c9..7b71dad23948 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -190,9 +190,9 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup(
 
 /// Returns \p A + \p B, if it is guaranteed not to unsigned wrap. Otherwise
 /// return nullptr. \p A and \p B must have the same type.
-static const SCEV *addSCEVOverflow(const SCEV *A, const SCEV *B,
-                                   ScalarEvolution &SE) {
-  if (!SE.willNotOverflow(Instruction::Add, false, A, B))
+static const SCEV *addSCEVNoOverflow(const SCEV *A, const SCEV *B,
+                                     ScalarEvolution &SE) {
+  if (!SE.willNotOverflow(Instruction::Add, /*IsSigned=*/false, A, B))
     return nullptr;
   return SE.getAddExpr(A, B);
 }
@@ -201,7 +201,7 @@ static const SCEV *addSCEVOverflow(const SCEV *A, const SCEV *B,
 /// return nullptr. \p A and \p B must have the same type.
 static const SCEV *mulSCEVOverflow(const SCEV *A, const SCEV *B,
                                    ScalarEvolution &SE) {
-  if (!SE.willNotOverflow(Instruction::Mul, false, A, B))
+  if (!SE.willNotOverflow(Instruction::Mul, /*IsSigned=*/false, A, B))
     return nullptr;
   return SE.getMulExpr(A, B);
 }
@@ -240,11 +240,11 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR,
       SE.getMinusSCEV(AR->getStart(), StartPtr), WiderTy);
 
   const SCEV *OffsetAtLastIter =
-      mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, false), SE);
+      mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE);
   if (!OffsetAtLastIter)
     return false;
 
-  const SCEV *OffsetEndBytes = addSCEVOverflow(
+  const SCEV *OffsetEndBytes = addSCEVNoOverflow(
       OffsetAtLastIter, SE.getNoopOrZeroExtend(EltSize, WiderTy), SE);
   if (!OffsetEndBytes)
     return false;
@@ -253,7 +253,7 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR,
     // For positive steps, check if
     //  (AR->getStart() - StartPtr) + (MaxBTC  * Step) + EltSize <= DerefBytes,
     // while making sure none of the computations unsigned wrap themselves.
-    const SCEV *EndBytes = addSCEVOverflow(StartOffset, OffsetEndBytes, SE);
+    const SCEV *EndBytes = addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE);
     if (!EndBytes)
       return false;
     return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes,
@@ -949,18 +949,12 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
   int64_t Size = AllocSize.getFixedValue();
 
   // Huge step value - give up.
-  if (APStepVal->getBitWidth() > 64)
+  std::optional<int64_t> StepVal = APStepVal->trySExtValue();
+  if (!StepVal)
     return std::nullopt;
 
-  int64_t StepVal = APStepVal->getSExtValue();
-
   // Strided access.
-  int64_t Stride = StepVal / Size;
-  int64_t Rem = StepVal % Size;
-  if (Rem)
-    return std::nullopt;
-
-  return Stride;
+  return *StepVal % Size ? std::nullopt : std::make_optional(*StepVal / Size);
 }
 
 /// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use
@@ -2087,14 +2081,14 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
   if (StrideAScaled == StrideBScaled)
     CommonStride = StrideAScaled;
 
-  // TODO: Historically, we don't retry with runtime checks unless the
-  // (unscaled) strides are the same. Fix this once the condition for runtime
-  // checks in isDependent is fixed.
-  bool ShouldRetryWithRuntimeCheck = StrideAPtrInt == StrideBPtrInt;
+  // TODO: FoundNonConstantDistanceDependence is used as a necessary condition
+  // to consider retrying with runtime checks. Historically, we did not set it
+  // when (unscaled) strides were different but there is no inherent reason to.
+  if (!isa<SCEVConstant>(Dist))
+    FoundNonConstantDistanceDependence |= StrideAPtrInt == StrideBPtrInt;
 
   return DepDistanceStrideAndSizeInfo(Dist, MaxStride, CommonStride,
-                                      ShouldRetryWithRuntimeCheck, TypeByteSize,
-                                      AIsWrite, BIsWrite);
+                                      TypeByteSize, AIsWrite, BIsWrite);
 }
 
 MemoryDepChecker::Dependence::DepType
@@ -2109,15 +2103,11 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
   if (std::holds_alternative<Dependence::DepType>(Res))
     return std::get<Dependence::DepType>(Res);
 
-  auto &[Dist, MaxStride, CommonStride, ShouldRetryWithRuntimeCheck,
-         TypeByteSize, AIsWrite, BIsWrite] =
+  auto &[Dist, MaxStride, CommonStride, TypeByteSize, AIsWrite, BIsWrite] =
       std::get<DepDistanceStrideAndSizeInfo>(Res);
   bool HasSameSize = TypeByteSize > 0;
 
   if (isa<SCEVCouldNotCompute>(Dist)) {
-    // TODO: Relax requirement that there is a common unscaled stride to retry
-    // with non-constant distance dependencies.
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
     LLVM_DEBUG(dbgs() << "LAA: Dependence because of uncomputable distance.\n");
     return Dependence::Unknown;
   }
@@ -2135,15 +2125,18 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
           DL, SE, *(PSE.getSymbolicMaxBackedgeTakenCount()), *Dist, MaxStride))
     return Dependence::NoDep;
 
-  // Attempt to prove strided accesses independent.
-  const APInt *ConstDist = nullptr;
-  if (match(Dist, m_scev_APInt(ConstDist))) {
-    uint64_t Distance = ConstDist->abs().getZExtValue();
+  // The rest of this function relies on ConstDist being at most 64-bits, which
+  // is checked earlier. Will assert if the calling code changes.
+  const APInt *APDist = nullptr;
+  uint64_t ConstDist =
+      match(Dist, m_scev_APInt(APDist)) ? APDist->abs().getZExtValue() : 0;
 
+  // Attempt to prove strided accesses independent.
+  if (APDist) {
     // If the distance between accesses and their strides are known constants,
     // check whether the accesses interlace each other.
-    if (Distance > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
-        areStridedAccessesIndependent(Distance, *CommonStride, TypeByteSize)) {
+    if (ConstDist > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
+        areStridedAccessesIndependent(ConstDist, *CommonStride, TypeByteSize)) {
       LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
       return Dependence::NoDep;
     }
@@ -2176,16 +2169,10 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     // forward dependency will allow vectorization using any width.
 
     if (IsTrueDataDependence && EnableForwardingConflictDetection) {
-      if (!ConstDist) {
-        // TODO: FoundNonConstantDistanceDependence is used as a necessary
-        // condition to consider retrying with runtime checks. Historically, we
-        // did not set it when strides were different but there is no inherent
-        // reason to.
-        FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
+      if (!ConstDist)
         return Dependence::Unknown;
-      }
-      if (!HasSameSize || couldPreventStoreLoadForward(
-                              ConstDist->abs().getZExtValue(), TypeByteSize)) {
+      if (!HasSameSize ||
+          couldPreventStoreLoadForward(ConstDist, TypeByteSize)) {
         LLVM_DEBUG(
             dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
         return Dependence::ForwardButPreventsForwarding;
@@ -2198,22 +2185,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
 
   int64_t MinDistance = SE.getSignedRangeMin(Dist).getSExtValue();
   // Below we only handle strictly positive distances.
-  if (MinDistance <= 0) {
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
+  if (MinDistance <= 0)
     return Dependence::Unknown;
-  }
-
-  if (!ConstDist) {
-    // Previously this case would be treated as Unknown, possibly setting
-    // FoundNonConstantDistanceDependence to force re-trying with runtime
-    // checks. Until the TODO below is addressed, set it here to preserve
-    // original behavior w.r.t. re-trying with runtime checks.
-    // TODO: FoundNonConstantDistanceDependence is used as a necessary
-    // condition to consider retrying with runtime checks. Historically, we
-    // did not set it when strides were different but there is no inherent
-    // reason to.
-    FoundNonConstantDistanceDependence |= ShouldRetryWithRuntimeCheck;
-  }
 
   if (!HasSameSize) {
     LLVM_DEBUG(dbgs() << "LAA: ReadWrite-Write positive dependency with "
@@ -2283,22 +2256,6 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
     return Dependence::Backward;
   }
 
-  // Positive distance bigger than max vectorization factor.
-  // FIXME: Should use max factor instead of max distance in bytes, which could
-  // not handle different types.
-  // E.g. Assume one char is 1 byte in memory and one int is 4 bytes.
-  //      void foo (int *A, char *B) {
-  //        for (unsigned i = 0; i < 1024; i++) {
-  //          A[i+2] = A[i] + 1;
-  //          B[i+2] = B[i] + 1;
-  //        }
-  //      }
-  //
-  // This case is currently unsafe according to the max safe distance. If we
-  // analyze the two accesses on array B, the max safe dependence distance
-  // is 2. Then we analyze the accesses on array A, the minimum distance needed
-  // is 8, which is less than 2 and forbidden vectorization, But actually
-  // both A and B could be vectorized by 2 iterations.
   MinDepDistBytes =
       std::min(static_cast<uint64_t>(MinDistance), MinDepDistBytes);
 
diff --git a/llvm/lib/Analysis/LoopPass.cpp b/llvm/lib/Analysis/LoopPass.cpp
index 5d0f9abbb5ce..d8680aac74b2 100644
--- a/llvm/lib/Analysis/LoopPass.cpp
+++ b/llvm/lib/Analysis/LoopPass.cpp
@@ -373,7 +373,7 @@ bool LoopPass::skipLoop(const Loop *L) const {
   if (!F)
     return false;
   // Check the opt bisect limit.
-  OptPassGate &Gate = F->getContext().getOptPassGate();
+  const OptPassGate &Gate = F->getContext().getOptPassGate();
   if (Gate.isEnabled() &&
       !Gate.shouldRunPass(this->getPassName(), getDescription(*L)))
     return true;
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index 81a3bc94a6ad..8853a13972be 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -107,7 +107,7 @@ static cl::opt<bool> KeepFPICache(
     cl::init(false));
 
 // clang-format off
-const std::vector<TensorSpec> llvm::FeatureMap{
+std::vector<TensorSpec> llvm::FeatureMap{
 #define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
 // InlineCost features - these must come first
   INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES)
@@ -144,6 +144,7 @@ MLInlineAdvisor::MLInlineAdvisor(
           M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
       ModelRunner(std::move(Runner)), GetDefaultAdvice(GetDefaultAdvice),
       CG(MAM.getResult<LazyCallGraphAnalysis>(M)),
+      UseIR2Vec(MAM.getCachedResult<IR2VecVocabAnalysis>(M) != nullptr),
       InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize),
       PSI(MAM.getResult<ProfileSummaryAnalysis>(M)) {
   assert(ModelRunner);
@@ -186,6 +187,19 @@ MLInlineAdvisor::MLInlineAdvisor(
     EdgeCount += getLocalCalls(KVP.first->getFunction());
   }
   NodeCount = AllNodes.size();
+
+  if (auto IR2VecVocabResult = MAM.getCachedResult<IR2VecVocabAnalysis>(M)) {
+    if (!IR2VecVocabResult->isValid()) {
+      M.getContext().emitError("IR2VecVocabAnalysis is not valid");
+      return;
+    }
+    // Add the IR2Vec features to the feature map
+    auto IR2VecDim = IR2VecVocabResult->getDimension();
+    FeatureMap.push_back(
+        TensorSpec::createSpec<float>("callee_embedding", {IR2VecDim}));
+    FeatureMap.push_back(
+        TensorSpec::createSpec<float>("caller_embedding", {IR2VecDim}));
+  }
 }
 
 unsigned MLInlineAdvisor::getInitialFunctionLevel(const Function &F) const {
@@ -433,6 +447,22 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   *ModelRunner->getTensor<int64_t>(FeatureIndex::is_caller_avail_external) =
       Caller.hasAvailableExternallyLinkage();
 
+  if (UseIR2Vec) {
+    // Python side expects float embeddings. The IR2Vec embeddings are doubles
+    // as of now due to the restriction of fromJSON method used by the
+    // readVocabulary method in ir2vec::Embeddings.
+    auto setEmbedding = [&](const ir2vec::Embedding &Embedding,
+                            FeatureIndex Index) {
+      llvm::transform(Embedding, ModelRunner->getTensor<float>(Index),
+                      [](double Val) { return static_cast<float>(Val); });
+    };
+
+    setEmbedding(CalleeBefore.getFunctionEmbedding(),
+                 FeatureIndex::callee_embedding);
+    setEmbedding(CallerBefore.getFunctionEmbedding(),
+                 FeatureIndex::caller_embedding);
+  }
+
   // Add the cost features
   for (size_t I = 0;
        I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) {
@@ -441,8 +471,7 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
   }
   // This one would have been set up to be right at the end.
   if (!InteractiveChannelBaseName.empty() && InteractiveIncludeDefault)
-    *ModelRunner->getTensor<int64_t>(FeatureIndex::NumberOfFeatures) =
-        GetDefaultAdvice(CB);
+    *ModelRunner->getTensor<int64_t>(FeatureMap.size()) = GetDefaultAdvice(CB);
   return getAdviceFromModel(CB, ORE);
 }
 
@@ -520,7 +549,7 @@ void MLInlineAdvice::reportContextForRemark(
     DiagnosticInfoOptimizationBase &OR) {
   using namespace ore;
   OR << NV("Callee", Callee->getName());
-  for (size_t I = 0; I < NumberOfFeatures; ++I)
+  for (size_t I = 0; I < FeatureMap.size(); ++I)
     OR << NV(FeatureMap[I].name(),
              *getAdvisor()->getModelRunner().getTensor<int64_t>(I));
   OR << NV("ShouldInline", isInliningRecommended());
diff --git a/llvm/lib/Analysis/RegionPass.cpp b/llvm/lib/Analysis/RegionPass.cpp
index 9ea7d711918f..ae1d84659de8 100644
--- a/llvm/lib/Analysis/RegionPass.cpp
+++ b/llvm/lib/Analysis/RegionPass.cpp
@@ -282,7 +282,7 @@ static std::string getDescription(const Region &R) {
 
 bool RegionPass::skipRegion(Region &R) const {
   Function &F = *R.getEntry()->getParent();
-  OptPassGate &Gate = F.getContext().getOptPassGate();
+  const OptPassGate &Gate = F.getContext().getOptPassGate();
   if (Gate.isEnabled() &&
       !Gate.shouldRunPass(this->getPassName(), getDescription(R)))
     return true;
diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp
index 28a5cdb5561d..973c9b1d30ca 100644
--- a/llvm/lib/Analysis/TargetLibraryInfo.cpp
+++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp
@@ -929,12 +929,6 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
   initializeLibCalls(TLI, T, StandardNames);
 }
 
-TargetLibraryInfoImpl::TargetLibraryInfoImpl() {
-  // Default to nothing being available.
-  memset(AvailableArray, 0, sizeof(AvailableArray));
-  initializeBase(*this, Triple());
-}
-
 TargetLibraryInfoImpl::TargetLibraryInfoImpl(const Triple &T) {
   // Default to everything being available.
   memset(AvailableArray, -1, sizeof(AvailableArray));
@@ -1499,7 +1493,7 @@ unsigned TargetLibraryInfoImpl::getSizeTSize(const Module &M) const {
 }
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass()
-    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl()) {}
+    : ImmutablePass(ID), TLA(TargetLibraryInfoImpl(Triple())) {}
 
 TargetLibraryInfoWrapperPass::TargetLibraryInfoWrapperPass(const Triple &T)
     : ImmutablePass(ID), TLA(TargetLibraryInfoImpl(T)) {}
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 2389322a2d54..f0b7d7dd75f8 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -362,6 +362,11 @@ static void computeKnownBitsAddSub(bool Add, const Value *Op0, const Value *Op1,
 
   computeKnownBits(Op0, DemandedElts, Known2, Q, Depth + 1);
   KnownOut = KnownBits::computeForAddSub(Add, NSW, NUW, Known2, KnownOut);
+
+  if (!Add && NSW && !KnownOut.isNonNegative() &&
+      isImpliedByDomCondition(ICmpInst::ICMP_SLE, Op1, Op0, Q.CxtI, Q.DL)
+          .value_or(false))
+    KnownOut.makeNonNegative();
 }
 
 static void computeKnownBitsMul(const Value *Op0, const Value *Op1, bool NSW,
@@ -3031,14 +3036,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         return isKnownNonZero(TI->getOperand(0), DemandedElts, Q, Depth);
     break;
 
+  // Iff x - y != 0, then x ^ y != 0
+  // Therefore we can do the same exact checks
+  case Instruction::Xor:
   case Instruction::Sub:
     return isNonZeroSub(DemandedElts, Q, BitWidth, I->getOperand(0),
                         I->getOperand(1), Depth);
-  case Instruction::Xor:
-    // (X ^ (X != 0)) is non zero
-    if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
-      return true;
-    break;
   case Instruction::Or:
     // (X | (X != 0)) is non zero
     if (matchOpWithOpEqZero(I->getOperand(0), I->getOperand(1)))
@@ -7874,6 +7877,51 @@ bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I,
   llvm_unreachable("Instruction not contained in its own parent basic block.");
 }
 
+bool llvm::intrinsicPropagatesPoison(Intrinsic::ID IID) {
+  switch (IID) {
+  // TODO: Add more intrinsics.
+  case Intrinsic::sadd_with_overflow:
+  case Intrinsic::ssub_with_overflow:
+  case Intrinsic::smul_with_overflow:
+  case Intrinsic::uadd_with_overflow:
+  case Intrinsic::usub_with_overflow:
+  case Intrinsic::umul_with_overflow:
+    // If an input is a vector containing a poison element, the
+    // two output vectors (calculated results, overflow bits)'
+    // corresponding lanes are poison.
+    return true;
+  case Intrinsic::ctpop:
+  case Intrinsic::ctlz:
+  case Intrinsic::cttz:
+  case Intrinsic::abs:
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin:
+  case Intrinsic::scmp:
+  case Intrinsic::is_fpclass:
+  case Intrinsic::ptrmask:
+  case Intrinsic::ucmp:
+  case Intrinsic::bitreverse:
+  case Intrinsic::bswap:
+  case Intrinsic::sadd_sat:
+  case Intrinsic::ssub_sat:
+  case Intrinsic::sshl_sat:
+  case Intrinsic::uadd_sat:
+  case Intrinsic::usub_sat:
+  case Intrinsic::ushl_sat:
+  case Intrinsic::smul_fix:
+  case Intrinsic::smul_fix_sat:
+  case Intrinsic::pow:
+  case Intrinsic::powi:
+  case Intrinsic::canonicalize:
+  case Intrinsic::sqrt:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool llvm::propagatesPoison(const Use &PoisonOp) {
   const Operator *I = cast<Operator>(PoisonOp.getUser());
   switch (I->getOpcode()) {
@@ -7884,38 +7932,8 @@ bool llvm::propagatesPoison(const Use &PoisonOp) {
   case Instruction::Select:
     return PoisonOp.getOperandNo() == 0;
   case Instruction::Call:
-    if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-      switch (II->getIntrinsicID()) {
-      // TODO: Add more intrinsics.
-      case Intrinsic::sadd_with_overflow:
-      case Intrinsic::ssub_with_overflow:
-      case Intrinsic::smul_with_overflow:
-      case Intrinsic::uadd_with_overflow:
-      case Intrinsic::usub_with_overflow:
-      case Intrinsic::umul_with_overflow:
-        // If an input is a vector containing a poison element, the
-        // two output vectors (calculated results, overflow bits)'
-        // corresponding lanes are poison.
-        return true;
-      case Intrinsic::ctpop:
-      case Intrinsic::ctlz:
-      case Intrinsic::cttz:
-      case Intrinsic::abs:
-      case Intrinsic::smax:
-      case Intrinsic::smin:
-      case Intrinsic::umax:
-      case Intrinsic::umin:
-      case Intrinsic::bitreverse:
-      case Intrinsic::bswap:
-      case Intrinsic::sadd_sat:
-      case Intrinsic::ssub_sat:
-      case Intrinsic::sshl_sat:
-      case Intrinsic::uadd_sat:
-      case Intrinsic::usub_sat:
-      case Intrinsic::ushl_sat:
-        return true;
-      }
-    }
+    if (auto *II = dyn_cast<IntrinsicInst>(I))
+      return intrinsicPropagatesPoison(II->getIntrinsicID());
     return false;
   case Instruction::ICmp:
   case Instruction::FCmp:
@@ -9065,46 +9083,43 @@ llvm::canConvertToMinOrMaxIntrinsic(ArrayRef<Value *> VL) {
   return {Intrinsic::not_intrinsic, false};
 }
 
-bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
-                                 Value *&Start, Value *&Step) {
+template <typename InstTy>
+static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
+                                    Value *&Init, Value *&OtherOp) {
   // Handle the case of a simple two-predecessor recurrence PHI.
   // There's a lot more that could theoretically be done here, but
   // this is sufficient to catch some interesting cases.
   // TODO: Expand list -- gep, uadd.sat etc.
-  if (P->getNumIncomingValues() != 2)
+  if (PN->getNumIncomingValues() != 2)
     return false;
 
-  for (unsigned i = 0; i != 2; ++i) {
-    Value *L = P->getIncomingValue(i);
-    Value *R = P->getIncomingValue(!i);
-    auto *LU = dyn_cast<BinaryOperator>(L);
-    if (!LU)
-      continue;
-    Value *LL = LU->getOperand(0);
-    Value *LR = LU->getOperand(1);
-
-    // Find a recurrence.
-    if (LL == P)
-      L = LR;
-    else if (LR == P)
-      L = LL;
-    else
-      continue; // Check for recurrence with L and R flipped.
-
-    // We have matched a recurrence of the form:
-    //   %iv = [R, %entry], [%iv.next, %backedge]
-    //   %iv.next = binop %iv, L
-    // OR
-    //   %iv = [R, %entry], [%iv.next, %backedge]
-    //   %iv.next = binop L, %iv
-    BO = LU;
-    Start = R;
-    Step = L;
-    return true;
+  for (unsigned I = 0; I != 2; ++I) {
+    if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(I))) {
+      Value *LHS = Operation->getOperand(0);
+      Value *RHS = Operation->getOperand(1);
+      if (LHS != PN && RHS != PN)
+        continue;
+
+      Inst = Operation;
+      Init = PN->getIncomingValue(!I);
+      OtherOp = (LHS == PN) ? RHS : LHS;
+      return true;
+    }
   }
   return false;
 }
 
+bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO,
+                                 Value *&Start, Value *&Step) {
+  // We try to match a recurrence of the form:
+  //   %iv = [Start, %entry], [%iv.next, %backedge]
+  //   %iv.next = binop %iv, Step
+  // Or:
+  //   %iv = [Start, %entry], [%iv.next, %backedge]
+  //   %iv.next = binop Step, %iv
+  return matchTwoInputRecurrence(P, BO, Start, Step);
+}
+
 bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
                                  Value *&Start, Value *&Step) {
   BinaryOperator *BO = nullptr;
@@ -9114,6 +9129,22 @@ bool llvm::matchSimpleRecurrence(const BinaryOperator *I, PHINode *&P,
   return P && matchSimpleRecurrence(P, BO, Start, Step) && BO == I;
 }
 
+bool llvm::matchSimpleBinaryIntrinsicRecurrence(const IntrinsicInst *I,
+                                                PHINode *&P, Value *&Init,
+                                                Value *&OtherOp) {
+  // Binary intrinsics only supported for now.
+  if (I->arg_size() != 2 || I->getType() != I->getArgOperand(0)->getType() ||
+      I->getType() != I->getArgOperand(1)->getType())
+    return false;
+
+  IntrinsicInst *II = nullptr;
+  P = dyn_cast<PHINode>(I->getArgOperand(0));
+  if (!P)
+    P = dyn_cast<PHINode>(I->getArgOperand(1));
+
+  return P && matchTwoInputRecurrence(P, II, Init, OtherOp) && II == I;
+}
+
 /// Return true if "icmp Pred LHS RHS" is always true.
 static bool isTruePredicate(CmpInst::Predicate Pred, const Value *LHS,
                             const Value *RHS) {
diff --git a/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json b/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
index 65ca240deba1..a0d539afd529 100644
--- a/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
+++ b/llvm/lib/Analysis/models/seedEmbeddingVocab75D.json
@@ -1,65 +1,72 @@
 {
-"add":[0.09571152, 0.08334279, 0.07070262, 0.14084256, 0.04825167, -0.12893851, -0.12180215, -0.07194936, -0.05329844, -0.28276998, 0.00202254, 0.07488817, -0.16139749, 0.07618549, -0.06084673, 0.10536429, -0.03031596, 0.16402271, 0.22730531, -0.0015432, 0.01986631, -0.15808797, 0.03030382, -0.04201249, 0.01689876, -0.30041003, -0.06479812, 0.1282431, -0.09032831, 0.14007935, -0.23540203, 0.00552223, 0.15325953, 0.13147154, -0.15204638, 0.1616615, 0.05758473, -0.1385157, 0.16438901, -0.14482859, 0.07895052, 0.18407233, 0.15283448, -0.00081216, -0.17934133, -0.16779658, 0.09044863, -0.18453072, -0.00552684, 0.01191218, 0.18504514, 0.13140059, -0.0174882, -0.18127315, 0.02269725, -0.02048657, 0.10858727, 0.0074029, 0.09485064, -0.13431476, 0.07491371, -0.17498694, -0.32914585, -0.14159656, -0.03594542, 0.04091231, 0.00298631, -0.08933277, -0.07178984, 0.04144038, 0.12151367, -0.09504163, 0.13691336, 0.07825345, 0.13958281],
-"alloca":[0.12644756, -0.20912014, 0.07298578, 0.13963142, 0.03932726, -0.12778169, -0.13084207, -0.09090705, -0.01497083, 0.0646746, 0.14847252, 0.08546949, -0.11579383, 0.07812478, -0.16431178, 0.08578802, 0.06094746, -0.1239017, -0.02789196, -0.01074964, -0.1738938, -0.1629063, -0.07137732, -0.02845656, 0.1728318, 0.13779363, -0.06250989, 0.03479109, -0.08423121, 0.14009888, 0.09620709, -0.02287545, -0.04616966, -0.19591278, -0.19636007, 0.21825573, 0.07949521, -0.14614704, 0.17752613, -0.15099092, -0.04518029, 0.17774306, 0.18947133, -0.00197501, -0.12619424, -0.18458585, -0.09960615, 0.01162648, 0.21306573, 0.0468024, 0.201505, 0.09970672, 0.06599383, 0.17757463, -0.11899275, 0.10299123, 0.06038174, -0.07116731, -0.15743989, -0.1258558, 0.1535076, 0.01872367, -0.14336765, -0.19708565, -0.26796287, 0.02628843, -0.0009325, -0.08809827, -0.12970725, 0.17223337, -0.10651242, -0.09162157, 0.03264611, 0.0911452, 0.09506542],
-"and":[-0.12902537, 0.07848564, 0.07048505, 0.13578993, 0.03570583, -0.1242408, -0.12619697, -0.06914084, -0.0515872, 0.13225996, 0.01673339, 0.0763608, 0.18325369, 0.07690141, -0.09591792, 0.10353354, -0.02401061, 0.16493416, 0.21942355, -0.00400911, 0.03811587, -0.15406959, -0.07134877, -0.02946596, -0.03854588, 0.28656727, -0.06625008, 0.12780443, -0.04631379, 0.13919763, -0.15808977, -0.00312698, 0.14692454, 0.18495218, -0.14863448, 0.13449706, 0.06471325, -0.13883992, 0.17630532, -0.16833898, 0.07391811, 0.17909151, 0.18229873, -0.00381102, -0.17680968, -0.1645554, 0.10016466, -0.07963493, 0.00130218, 0.05646244, 0.18222143, 0.10511146, -0.0191175, -0.08713559, 0.25423968, -0.02557301, 0.04319789, 0.03259414, 0.07209402, -0.13169754, 0.07424775, -0.17216511, -0.32057068, -0.13833733, -0.0658454, 0.02420194, -0.04393166, -0.08238692, -0.07023077, 0.04014502, 0.20101993, -0.09093616, 0.13076238, 0.09114857, -0.06483845],
-"ashr":[0.16414718, -0.02078147, 0.07353837, 0.14210321, 0.08068018, -0.13586298, -0.15728961, -0.10365791, 0.00359197, 0.04608767, 0.35770142, 0.08003625, 0.00944533, 0.08471741, 0.05809571, 0.09202059, -0.18349345, 0.22169511, 0.24418135, 0.19192688, -0.1956347, -0.17401719, -0.07583863, -0.02781139, 0.0952112, -0.01444751, -0.27259794, 0.14392436, -0.13541143, 0.1410963, 0.04314299, -0.01641751, -0.05448177, -0.22287542, -0.2131822, 0.25112826, 0.06918604, -0.14414005, 0.060288, -0.09658266, -0.09122665, -0.02779044, 0.20349248, 0.0041391, 0.10610974, -0.20890503, -0.09881835, -0.07368057, 0.22467837, 0.00075097, 0.2147348, -0.02612463, -0.01696278, -0.0649786, 0.15041149, 0.02361087, 0.0211603, -0.03706622, 0.18296233, -0.14298625, 0.14614436, 0.02273145, 0.0209446, -0.21062987, 0.16911499, -0.03668665, 0.00197532, -0.09607943, -0.08398084, 0.02006913, 0.05739584, -0.07919859, 0.19634944, 0.11082727, -0.06584227],
-"bitcast":[0.1675692, -0.12870269, 0.04048132, -0.1670965, -0.1279611, 0.02615386, -0.16829294, -0.09034907, 0.10913523, -0.07819421, 0.23986322, -0.05966561, 0.08410738, 0.19072439, 0.06047394, 0.02999627, -0.16747619, -0.06076627, -0.02673951, -0.1619169, 0.06443421, -0.13788716, -0.05644303, 0.01361013, -0.06858975, -0.06005004, 0.10011288, -0.05508338, -0.10613093, -0.11281271, -0.00758647, -0.12425531, 0.05333719, -0.16881412, -0.20088236, 0.06015657, -0.16405901, 0.06226884, 0.09171099, -0.09500738, 0.07907875, 0.0776544, -0.18457057, -0.11278627, -0.12111131, -0.10180638, 0.18328871, 0.18770072, 0.20346186, 0.10305139, -0.18344335, 0.10693213, -0.10920919, -0.05994263, 0.20354497, -0.03093485, 0.14214055, 0.00580597, 0.10480052, -0.09955201, -0.134185, -0.02904563, 0.00175069, 0.17646657, 0.01348841, -0.02030338, -0.06537742, 0.10032237, 0.15315783, 0.0102667, 0.07717734, -0.01060431, 0.14727928, -0.16261302, -0.06770234],
-"br":[0.11499645, 0.0824301, 0.07218035, 0.13258332, -0.13419574, -0.12916718, -0.12626709, -0.06741403, -0.02857593, -0.2543893, 0.02193022, 0.0760875, -0.1562702, 0.07712954, 0.3149361, 0.10217083, -0.041038, 0.16601022, 0.01906607, -0.02043359, 0.05471838, -0.15233372, -0.06945753, -0.02313732, -0.07342829, 0.3331809, -0.05246306, -0.0269839, -0.05435036, 0.13908924, 0.32604694, 0.00170966, 0.14997493, 0.13026518, -0.14908995, 0.12238151, -0.06773318, -0.13566032, 0.16068587, -0.12842499, 0.04970508, 0.17827724, 0.09729939, -0.00447832, -0.1739753, -0.16429187, 0.09886666, -0.08058207, 0.00714044, 0.04585538, 0.13424252, 0.11376464, -0.01675582, 0.17901348, -0.00653374, -0.01570439, 0.13032894, 0.01734108, 0.16833901, -0.1173776, 0.07662185, -0.15942436, -0.21173944, -0.10505079, 0.0597497, 0.03491669, 0.00338842, -0.04969047, -0.07644061, 0.04528612, 0.254365, -0.09514527, 0.12015092, 0.08262096, -0.02352029],
-"call":[0.10815012, -0.12419116, 0.18759736, -0.1905027, 0.01619313, -0.13483052, -0.12278763, -0.07051246, -0.0083437, 0.25107145, -0.16601063, 0.08127163, -0.17432374, -0.18380919, 0.24335551, 0.07208319, -0.04401246, 0.11606008, -0.02733191, 0.02098145, 0.019888, -0.13705409, -0.07569158, -0.03072285, 0.16870692, -0.09787013, -0.09340432, 0.01931342, -0.03557841, 0.14359893, -0.1592094, -0.00055867, 0.159316, 0.1099042, -0.11837319, 0.08741318, 0.03364393, -0.12831019, 0.10450637, -0.12699029, -0.20213994, 0.18390144, 0.11092624, -0.00209971, -0.13063665, 0.19996215, 0.09006448, -0.07840014, 0.22549215, 0.02587176, 0.13374338, 0.11009877, -0.01874998, -0.21446206, 0.02377797, -0.01036531, 0.05427047, 0.01418843, 0.00771817, -0.12639529, -0.10334941, -0.12244401, 0.30014148, -0.09857437, 0.21212636, 0.03429029, -0.04947309, 0.1023307, -0.07743628, 0.03006962, -0.24868701, -0.02357339, 0.11574048, 0.06895301, -0.363474],
-"constant":[-0.2850312, -0.22839, 0.12669143, -0.0674703, -0.12639391, -0.00477266, 0.04786542, 0.06336267, 0.08660185, 0.12805316, -0.07146342, 0.21539183, -0.0624397, -0.02638953, -0.28688517, 0.28374302, 0.05338082, 0.05559688, -0.13133128, 0.12440272, -0.03583231, 0.29848817, 0.13930812, 0.15453401, 0.0538353, -0.06874479, 0.00262802, 0.27964258, 0.19028014, -0.16371843, -0.05762961, 0.20059372, -0.20804578, -0.06549844, 0.09732475, -0.01551855, 0.21226783, 0.05889762, -0.07560658, 0.11312829, -0.04594622, -0.27309528, -0.05293005, 0.18953343, 0.05463868, -0.31045213, -0.04364616, 0.11005993, 0.12489324, -0.05413342, -0.05814561, -0.26131225, -0.18863814, 0.31165487, -0.08796364, -0.19958755, -0.10849535, 0.14899114, -0.01385941, 0.29359573, -0.01349372, 0.0562498, 0.10977754, 0.08993197, 0.06231657, -0.13509868, -0.20968516, 0.03578237, 0.15356435, -0.17766887, -0.11509016, 0.06898113, -0.06665445, -0.14065051, 0.34711906],
-"extractelement":[-1.62098512e-01, -2.00751424e-02, 1.71776459e-01, 1.38723463e-01, -1.60769299e-01, 9.30800363e-02, -1.24304645e-01, 1.45934001e-01, -5.04420400e-02, -7.54220188e-02, -5.30924797e-02, 8.55294541e-02, 1.17488159e-02, -1.82809234e-01, 9.37484950e-02, 8.38199258e-02, 1.26266479e-01, 9.31843743e-02, -2.26742271e-02, 6.50950940e-03, 5.02749532e-03, -1.14133619e-01, 1.80842891e-01, -9.63811725e-02, 1.67923152e-01, -5.84629439e-02, 1.04362026e-01, 8.92093012e-05, 7.93750137e-02, 1.39107972e-01, -8.40696543e-02, -1.59506593e-02, 1.99361086e-01, 1.93857521e-01, -1.46850392e-01, -1.78695560e-01, 9.88712162e-03, -1.40836969e-01, 1.77736521e-01, -1.61047727e-01, 3.51648539e-01, 1.79638579e-01, 1.49385989e-01, -6.06128015e-03, -1.72102928e-01, -1.81016047e-02, 1.01466164e-01, -1.28312945e-01, -8.05212185e-02, 6.28385469e-02, 1.15654446e-01, 1.91848025e-01, 3.03963851e-02, 3.55794169e-02, 1.40873834e-01, -1.44319711e-02, 1.85423180e-01, 1.16111919e-01, -7.47816712e-02, -1.14719503e-01, 1.02934733e-01, -1.83810964e-01, -2.64400076e-02, -8.99282843e-02, -1.90383971e-01, 7.31386840e-02, -4.36249487e-02, -4.71482053e-02, 1.07486300e-01, 1.09736443e-01, 4.15226035e-02, -1.42309800e-01, 8.96709636e-02, 6.64985999e-02, -6.13647103e-02],
-"extractvalue":[0.08820406, -0.0182279, 0.01493346, -0.17219813, 0.02927338, -0.17379265, -0.05663937, -0.06805898, -0.21235467, -0.01969833, -0.15152055, -0.18049374, 0.01062911, 0.07935719, -0.01993761, 0.12405304, -0.03198355, 0.13872959, 0.18017697, -0.0100032, 0.22617207, -0.21553156, -0.18403597, 0.10906533, 0.17709404, 0.07438782, 0.0875822, -0.22567064, -0.00636844, 0.06691552, 0.05859367, -0.06149556, 0.15721355, 0.01889951, 0.2164152, -0.07916643, -0.03927743, -0.02665933, 0.13756634, -0.08835335, -0.01159343, -0.19749148, 0.09438482, -0.1696074, -0.18197437, 0.19907822, -0.20855112, 0.21903005, -0.2204043, -0.07439119, 0.1308343, 0.10343243, -0.10401972, 0.02784402, -0.12545246, -0.18498465, 0.02385085, 0.22105947, -0.01296441, 0.19779074, -0.09320201, -0.07345647, -0.06985376, -0.0657003, -0.15406364, 0.14981052, 0.01336148, 0.18544021, -0.07248794, 0.0511543, -0.00308717, 0.01915094, 0.079139, 0.19975829, -0.06093699],
-"fadd":[0.0851364, -0.01757606, 0.20865884, 0.17407735, -0.17661206, 0.16181652, -0.11895371, -0.06142977, -0.05126655, 0.04448467, -0.03589934, 0.11921146, 0.0143434, -0.18255684, 0.05196553, 0.10967412, 0.15810761, 0.13785522, -0.01255003, -0.00134187, 0.03563518, -0.12446801, 0.18817455, -0.08609993, 0.1893248, 0.13434686, 0.25903776, -0.00467659, 0.09800702, 0.14749499, 0.03458502, -0.00238901, -0.1395337, 0.19306736, -0.11638073, 0.10491668, 0.04428702, 0.18128034, 0.1797104, -0.1929282, 0.07353631, 0.19062985, 0.10042762, -0.0040207, -0.18379262, 0.07060277, 0.09594815, -0.18272707, -0.12928157, 0.07631373, 0.13005428, 0.24108389, -0.01617561, -0.05803563, 0.02686582, -0.02223067, 0.21217403, 0.17594706, 0.00500477, -0.07345455, -0.08176229, -0.19486704, 0.02177307, -0.09052874, -0.14158368, 0.0890988, -0.06339125, -0.04612265, 0.10199347, 0.11777309, 0.07302366, -0.13303186, 0.09683178, 0.05031883, -0.05881981],
-"fcmp":[1.02981135e-01, -2.43355539e-02, 5.85255250e-02, 1.09401904e-01, -1.87941462e-01, 1.74530044e-01, -1.20533399e-01, -8.64934921e-02, -5.54629341e-02, 1.41877215e-02, 4.80354428e-02, 1.56707644e-01, 2.56910548e-02, -1.94340080e-01, 5.11265621e-02, 8.82050097e-02, 1.67082191e-01, 1.14665776e-01, -2.00159680e-02, -1.82088744e-02, 2.09244549e-01, -1.15076765e-01, 2.03621030e-01, -8.38057026e-02, -4.32270877e-02, 6.39017820e-02, -6.50197491e-02, -2.72039324e-03, 8.68055448e-02, 1.25798717e-01, -4.21847135e-01, 2.64590848e-02, 1.56372517e-01, 2.14883089e-01, -1.12701654e-01, 9.43767577e-02, -1.97456375e-01, -1.72886148e-01, 2.04119727e-01, -1.56766623e-01, 7.38008767e-02, 2.05716744e-01, 3.48394439e-02, -2.48072352e-02, -2.03255862e-01, 5.07165631e-03, 1.12850599e-01, -1.94697857e-01, -6.29984289e-02, 8.21878910e-02, 1.23685144e-01, 1.40430763e-01, -2.20402889e-02, -6.55216798e-02, 2.90598303e-01, -7.66792744e-02, 2.16097474e-01, 6.28568381e-02, 2.69129931e-04, -1.17002167e-01, -1.60454452e-01, -2.07048669e-01, 2.62669493e-02, -9.61467475e-02, -1.88501909e-01, 8.80606323e-02, -5.39420024e-02, -5.08916602e-02, 1.09891914e-01, 1.21375419e-01, 5.10863326e-02, -1.43003076e-01, 8.39024931e-02, 9.51330177e-03, -6.65674359e-02],
-"fdiv":[-0.14435205, -0.02003789, 0.07479589, 0.13465217, -0.18792528, 0.16504793, -0.12187403, 0.00537306, -0.04864043, 0.04958175, -0.14890797, 0.10156095, 0.01710029, -0.1809697, 0.06765524, 0.18452686, 0.16923915, 0.11264159, -0.01412142, -0.00287484, 0.01297035, 0.06755029, 0.18517323, -0.08975757, 0.16194478, 0.04174679, 0.10903918, 0.00072231, 0.10999232, 0.14989018, 0.05669819, -0.00733533, 0.20606744, 0.22580206, -0.14587933, 0.13480443, 0.05984581, -0.19580479, 0.20258778, -0.17146486, 0.07523733, 0.19139282, 0.03738374, -0.01550991, -0.21507922, 0.07425626, 0.10224851, -0.18437587, -0.06662318, 0.07977687, 0.12355862, 0.1434375, -0.01888226, 0.03870944, 0.07555477, -0.01191964, 0.23488866, 0.06922436, -0.07203218, -0.06884275, 0.07191673, -0.19515742, 0.02168754, -0.08983592, -0.20972523, 0.08903123, -0.05095004, -0.04967143, 0.0985181, 0.15771264, 0.09441628, -0.14769785, -0.16405083, 0.05066825, -0.058015],
-"floatTy":[-0.04556743, 0.03822431, -0.08842288, -0.12055657, 0.0960574, 0.2371231, 0.09947137, 0.11064581, -0.12371849, -0.06109007, 0.06440615, 0.14924356, 0.13218448, -0.04400141, 0.01761996, -0.13240574, 0.21056518, -0.14215694, -0.14700417, -0.16416645, -0.07137518, 0.05240471, -0.10071051, 0.12556827, 0.08208757, 0.06504779, 0.1578807, -0.09501757, 0.00659264, -0.00527758, -0.01085964, -0.17290565, 0.00501871, -0.10288252, 0.09658113, -0.107755, -0.1718967, 0.16306138, -0.080802, 0.10203532, -0.02934794, -0.02899184, -0.17915869, -0.19883338, -0.05124965, 0.16747124, -0.05587617, 0.14325564, -0.18054038, -0.161391, -0.1149613, 0.01600737, -0.0704658, 0.01935361, -0.04516762, -0.1305524, 0.04487818, -0.07881694, -0.18929696, 0.05364102, -0.17827097, 0.09934752, -0.04212811, 0.15036745, -0.18782069, 0.19947147, 0.1102478, -0.1304183, 0.17145109, 0.22485495, 0.02270225, 0.08804839, -0.12520337, -0.07780997, 0.00057234],
-"fmul":[-0.05690098, -0.02700638, 0.20705073, 0.16630849, -0.17719169, 0.16012336, -0.12955493, 0.19698587, -0.04035136, -0.03197788, 0.08116172, 0.1013689, 0.01416616, -0.17855197, 0.15503113, 0.13990149, 0.15355295, 0.12397105, -0.01912402, 0.01684221, 0.00669227, -0.11869009, 0.17604592, -0.05788622, 0.10467764, -0.0807798, 0.10300152, 0.10780353, 0.08334652, 0.1446782, 0.03546653, -0.00421023, 0.19682531, 0.19603632, -0.1466306, 0.12923796, 0.06296455, -0.14397427, 0.183406, -0.19625223, 0.08244827, 0.1860209, 0.10110238, -0.002166, -0.18034802, 0.06406105, 0.09568714, -0.17962225, -0.05463478, 0.04498994, 0.11928298, 0.1694295, -0.01222703, 0.14453876, 0.13506988, -0.01149808, 0.20273286, 0.05309585, -0.07139173, -0.07370458, 0.09957068, -0.19369006, 0.02577178, -0.09011577, -0.21375039, 0.07624438, -0.04969844, -0.04844297, 0.08409201, 0.11113621, 0.14570779, -0.13936704, 0.07202481, 0.05272412, -0.05747499],
-"fneg":[-0.04075071, -0.09381824, 0.17104743, 0.0744117, -0.15771168, 0.16444896, -0.11586417, 0.00200867, 0.17323531, -0.10421973, -0.05619403, 0.07496857, 0.05573901, -0.18903743, 0.06586835, 0.03499747, 0.17244707, 0.02476844, -0.02552369, 0.00063759, 0.04312319, -0.04501259, 0.1850581, -0.14339973, -0.04433612, 0.01609048, 0.10461161, -0.12243931, 0.07718915, 0.15313269, 0.12116063, -0.03062805, 0.21040255, 0.21582894, -0.10389283, 0.09344483, -0.20219979, -0.20113027, 0.19945832, -0.17248249, 0.07845841, 0.19814597, 0.06085683, -0.02476168, -0.18729973, 0.0674377, 0.1058675, -0.04973035, -0.10124692, 0.12834774, 0.0900365, -0.17527112, 0.09901146, -0.05764115, 0.03708475, 0.13003437, 0.2325445, 0.19712777, -0.07403741, -0.05580256, 0.18192469, -0.19122703, -0.05592606, 0.09128745, -0.15161441, 0.08734331, -0.0963953, -0.0377482, 0.18287936, 0.16522603, 0.03490613, -0.13519889, 0.08505893, 0.02470117, -0.06158587],
-"fpext":[0.1312062, -0.0272035, 0.1562131, 0.11956131, -0.18370572, 0.16904335, -0.18538451, -0.0905571, -0.05687085, -0.12067703, 0.15717801, 0.06977441, 0.04604319, -0.18692835, 0.07965986, 0.03840762, 0.17120989, 0.03048515, -0.02593113, -0.02722386, 0.0637762, -0.08012746, 0.03968937, -0.09751038, -0.04150679, -0.05797326, 0.09053179, -0.06100635, 0.01466092, 0.1521789, 0.0178679, 0.00468684, 0.09077137, 0.22244066, -0.21890686, 0.09471557, -0.18849488, 0.06589299, 0.20329474, -0.20006016, 0.06712949, 0.19578859, -0.00594554, -0.03142307, -0.18011327, 0.00516408, 0.18754548, -0.18884791, 0.2184627, 0.09512166, 0.07513344, 0.12641825, -0.0237517, -0.06020509, 0.15404698, -0.02384854, 0.22717056, 0.11279562, 0.16759154, -0.09342691, -0.09703359, -0.19393681, -0.06580188, 0.190172, -0.21666776, 0.09756065, -0.06633368, -0.04663929, 0.16539453, 0.16365078, 0.04338961, -0.1387515, 0.07042016, 0.03850469, -0.06256048],
-"fptosi":[-0.09471355, -0.01413281, 0.20098655, 0.17478812, -0.0103814, 0.11870264, -0.10561193, 0.19406088, -0.04215302, 0.09856935, -0.05858651, 0.11069191, -0.02667271, -0.17638545, 0.00591833, 0.12538631, 0.1609231, 0.143028, 0.00411847, 0.06113226, -0.1577999, -0.12172183, 0.18763137, -0.04490714, 0.0262733, 0.12005143, 0.08305117, -0.00109107, 0.03972085, 0.13256785, 0.12486281, -0.00920427, 0.19794717, 0.13488762, -0.08073806, 0.11790548, -0.1865304, -0.18761554, 0.15602915, -0.1440169, -0.01538679, 0.18490645, 0.05900477, -0.01250085, -0.17872328, 0.07583775, 0.09906318, -0.16909951, -0.12062778, 0.03089247, 0.11974704, 0.23940024, -0.00561649, 0.03862159, 0.13437317, -0.0128777, 0.2149731, 0.07683202, -0.10922348, -0.04304254, -0.20104879, -0.16371554, -0.08326109, -0.08942039, -0.03989649, 0.07596397, 0.18141732, 0.08706582, -0.06605583, 0.09984156, -0.05919264, -0.13918152, -0.11102735, -0.04074531, -0.05989955],
-"fptoui":[0.13186027, -0.02009563, 0.05399049, 0.22092278, 0.0764356, -0.12116565, -0.10575569, -0.08263665, -0.05279351, 0.00392524, -0.06554782, 0.15926358, 0.01012308, -0.18726377, 0.02134197, 0.12730621, -0.05603928, 0.1433069, -0.21253656, -0.1455532, 0.23767456, -0.13623604, 0.21878564, -0.05533312, -0.02895407, 0.03911315, -0.2773476, -0.00738798, 0.08483762, -0.14270262, 0.04603437, 0.21681121, -0.20425414, 0.12532364, -0.04144387, -0.1759266, 0.06860236, -0.14378744, 0.16322674, -0.09162073, -0.04966446, 0.1867569, 0.11250684, 0.04158355, -0.20126882, 0.0963328, 0.06281191, -0.17627244, -0.16096021, 0.00596877, 0.12892367, 0.2650723, 0.0032763, -0.05754196, 0.14840715, -0.09938947, 0.13039768, 0.04905574, -0.04759435, -0.18545668, -0.22984591, -0.22227132, -0.02666637, -0.09771203, -0.15587328, 0.07147411, 0.21007161, 0.09932306, -0.06597851, 0.04478595, -0.00686691, -0.14754876, -0.16366987, -0.22182341, -0.05699158],
-"fptrunc":[0.1180348, -0.01865118, 0.19759397, 0.00099745, -0.20069243, 0.15863162, -0.10404988, -0.03822596, -0.04113388, 0.02389156, -0.15024376, 0.10156121, 0.01125149, -0.17805247, -0.01032893, 0.11541142, 0.16151612, 0.08652765, -0.01421094, -0.02742836, 0.03890061, 0.16992694, -0.19120258, -0.08296357, 0.16136265, 0.13252915, 0.08675185, -0.05066127, 0.14796199, 0.16224207, 0.07839199, -0.0724185, 0.17859218, 0.20112462, -0.11284997, 0.04566038, -0.19799039, -0.17690767, 0.18034197, -0.14002973, -0.0558032, 0.18783137, -0.14857374, -0.03776158, -0.21491641, 0.04907086, 0.10290487, -0.17882703, -0.06566726, 0.09571292, -0.1965507, 0.1673165, -0.03019422, 0.04259395, 0.10474471, -0.01224911, 0.2133804, 0.08484804, -0.01221132, -0.04942748, -0.06522352, -0.16705054, -0.07432696, -0.01365143, -0.14369024, 0.09237107, -0.06833915, -0.04292256, 0.10805372, 0.10829192, -0.04953781, -0.16152975, 0.08096681, 0.12343118, -0.04967185],
-"freeze":[-0.10947239, 0.16156663, 0.0653074, -0.16102187, 0.05866997, -0.11313032, 0.18013522, 0.00849657, -0.13109621, 0.00250287, 0.12335391, -0.19474623, 0.00160397, 0.18526912, -0.02094408, 0.10131565, -0.17163642, 0.1624736, 0.20699912, 0.1198829, -0.15380894, -0.13946483, -0.0666218, -0.02034315, 0.06200019, 0.04227962, 0.07729523, 0.1244237, -0.07141764, 0.12154905, 0.0300131, 0.01618693, 0.13966976, 0.08918943, -0.07834358, 0.19349128, -0.06925047, -0.11661758, 0.01764905, -0.06631834, -0.19618054, 0.07797705, -0.18398666, -0.08675893, -0.14833811, -0.1468038, 0.18818453, -0.06729261, -0.00497526, -0.08958972, 0.16964634, -0.10325264, -0.10212341, -0.04745837, 0.12241088, -0.03629779, -0.00606445, 0.01766711, 0.0036858, -0.10732682, -0.13172524, -0.10182981, -0.06431175, -0.13082966, 0.04738441, 0.02668242, 0.15197244, -0.12880892, -0.06579075, 0.03165649, -0.00145413, -0.02683991, 0.13963282, -0.14597963, -0.05104417],
-"fsub":[-0.15878558, -0.01853204, 0.07496438, 0.13546987, -0.1904581, 0.16564278, -0.1120265, -0.05418859, -0.08075086, 0.1190263, -0.030006, 0.07342444, 0.01767328, -0.18650489, 0.04966565, 0.17190282, 0.1616953, 0.10582477, -0.00931773, 0.00157398, 0.00862398, 0.08480153, 0.17171694, -0.0916861, 0.17196383, 0.03703162, 0.11399966, -0.12855959, 0.05810672, 0.15144408, 0.05618297, -0.00800864, 0.20814295, 0.20056027, -0.1153914, 0.10849892, -0.1970011, -0.1813781, 0.1921156, -0.16984123, 0.07333071, 0.19433162, -0.01313439, -0.02830642, -0.17687869, 0.05937865, 0.1070236, -0.18605019, -0.07227098, 0.10597191, 0.1225975, 0.12388534, -0.02131494, -0.02420847, 0.00146665, -0.01353394, 0.22151577, 0.19731705, -0.07575841, 0.08710405, -0.07411117, -0.17442936, -0.1831972, -0.0886184, -0.17084508, 0.10038229, -0.06385624, -0.04644644, 0.10368951, 0.12130242, 0.24595715, -0.14661786, 0.09260331, 0.0455667, -0.05950425],
-"function":[-0.08280793, -0.18925415, -0.1866685, -0.04932962, 0.13478681, 0.203617, 0.05739383, 0.09856346, -0.17600478, 0.07619468, -0.12438924, -0.02529626, -0.07468224, 0.03905433, 0.11325707, -0.04229602, -0.16354711, -0.08952031, -0.15966323, 0.11114867, 0.07319006, 0.02678379, -0.13364255, 0.15032487, 0.0822423, 0.03195171, -0.00871139, -0.1128016, 0.19261838, -0.07204764, -0.05823177, 0.1990249, -0.2057403, -0.08260711, 0.09775644, -0.12436705, 0.15814295, 0.0463977, -0.0493853, 0.1450742, -0.07108644, -0.03192589, -0.11917686, -0.19753116, 0.09248564, 0.13522607, 0.16347136, 0.11065657, -0.0850426, -0.19110996, -0.15060848, -0.25829738, 0.11464069, -0.15613398, -0.07256057, 0.16283846, -0.11428182, -0.1879146, -0.10537492, 0.03637636, -0.02642156, 0.0344548, -0.00722412, 0.13349552, 0.05913915, 0.17941767, -0.20852967, 0.04403022, -0.13914339, -0.17848521, -0.07228794, 0.23070297, -0.11848032, -0.14098541, -0.23762096],
-"getelementptr":[0.09870283, 0.07089636, 0.07129486, 0.13141522, 0.0512748, -0.12164738, -0.10822044, -0.06830852, -0.03589667, 0.23281692, 0.00221914, 0.07347875, -0.14414017, -0.1762956, 0.2280886, 0.09720317, -0.02157139, 0.1186724, 0.16809724, 0.01354192, 0.00843806, -0.1289462, -0.06982945, -0.03921828, 0.11689314, -0.27946517, -0.06115843, 0.0435549, -0.04346889, 0.13182928, -0.13756175, -0.00165133, 0.14864644, 0.1256485, -0.10831792, 0.1638358, 0.05359713, -0.12713233, 0.15040766, -0.14046451, 0.06688947, 0.17097251, 0.11004995, 0.00303981, -0.17120391, -0.15561967, -0.0996208, -0.07566627, -0.0092614, 0.03340579, 0.12288038, 0.11154807, -0.01677553, 0.161112, 0.01968472, -0.01360116, 0.05185669, 0.03974737, 0.08288419, -0.11881893, 0.09566113, -0.14562541, 0.27193576, -0.08737413, -0.13493136, 0.06975145, -0.01905861, -0.04629398, -0.06382514, 0.03648283, 0.32102475, -0.07868497, 0.11246872, 0.06408555, 0.21986632],
-"icmp":[0.10752141, 0.08221146, 0.07407488, 0.13139327, -0.17970721, 0.02754223, -0.13538423, -0.06698897, -0.02524848, 0.29738554, 0.00894943, 0.07728788, 0.01191065, 0.07896648, 0.26208735, 0.10498706, -0.03250087, 0.1684631, 0.23128033, 0.00553169, 0.05936556, -0.15531996, -0.06056314, -0.03261358, 0.09216364, -0.31626984, -0.07434817, 0.12870958, -0.08664963, 0.14354071, 0.2156426, -0.0043262, 0.16025512, 0.13890502, -0.153901, 0.12319393, 0.01295959, -0.14369431, 0.17033054, -0.12931693, 0.07678536, 0.19203088, 0.11493351, -0.00601692, -0.18671934, -0.16845967, 0.11059918, -0.08381938, -0.00690406, 0.04286021, 0.13590424, 0.12681794, -0.01967513, -0.21233313, 0.02998303, -0.01612004, 0.17794448, 0.02153118, 0.09134272, -0.12414944, 0.08073039, -0.17339677, -0.0417021, -0.1026428, -0.0270442, 0.02959586, -0.05658696, -0.05483485, -0.07289126, 0.036765, 0.20820136, -0.0922353, 0.1438483, 0.06928346, -0.37188163],
-"insertelement":[-0.08810953, 0.19707826, 0.23040843, 0.15133125, -0.22085261, 0.18197243, -0.13767323, 0.00611759, -0.03750934, 0.05303819, -0.00140541, 0.13628025, 0.01222425, -0.19765897, 0.02197562, 0.17957552, 0.17369562, 0.13818526, -0.01018569, 0.07335348, -0.00122391, -0.05268616, 0.03798695, -0.07208619, 0.13593383, -0.08763747, 0.10625125, 0.12024429, -0.01286297, 0.16619073, 0.03609107, 0.00169814, 0.21330379, 0.22023544, -0.11971844, 0.23055589, -0.21922874, -0.18983133, 0.21032487, -0.19545083, 0.39695147, 0.2116961, 0.03465794, -0.01642283, -0.23929499, 0.05254361, 0.1186865, -0.20114459, -0.01001151, 0.06816892, 0.13270052, 0.13516793, -0.02193176, 0.13708963, 0.15222688, -0.01366894, 0.22591114, 0.07127486, -0.08461929, -0.08140475, 0.06961198, -0.18172547, 0.03001983, -0.09530003, -0.03038635, 0.09877217, -0.06264398, -0.06664777, 0.08449566, 0.13536829, 0.04701871, -0.1011354, 0.08900025, 0.0484806, -0.0840841],
-"insertvalue":[-0.18927452, 0.0174884, -0.14182499, 0.13622768, 0.0860026, 0.09214608, 0.19103362, -0.21386355, -0.08520557, 0.0039752, 0.11396001, -0.20266375, 0.17875974, -0.19441572, 0.06652557, -0.1076987, -0.15872131, -0.20631677, -0.06141828, 0.01331364, 0.02354101, 0.08883859, 0.18617646, -0.1524484, -0.00485444, -0.06658852, -0.06168855, -0.15898797, 0.02933339, -0.19443747, -0.08171376, -0.10284599, -0.14775087, 0.20585667, -0.03085143, -0.20518455, 0.05644068, 0.18632361, -0.14686832, 0.03718346, 0.10764548, -0.01129938, 0.19590198, 0.02194332, -0.04761516, 0.0276675, 0.01006511, -0.05134218, -0.10854474, 0.23528893, 0.09621219, -0.17192629, 0.10312149, -0.14351319, 0.15347119, 0.19321586, -0.20537563, 0.19100618, 0.11280359, 0.07242113, 0.18901348, 0.19133772, 0.02062779, -0.07726028, 0.00578185, -0.02625765, -0.10374335, -0.02086248, 0.18915856, 0.00862997, 0.04699488, -0.0631949, 0.18942626, 0.01894571, 0.02351614],
-"integerTy":[-0.02392217, 0.13147527, -0.12684862, -0.00286194, 0.18320721, -0.03383226, 0.10688385, 0.20315196, 0.16910973, 0.01749469, 0.09163074, 0.18359211, 0.08457439, 0.17821081, -0.01359864, -0.09173124, -0.1060688, -0.0317139, 0.089481, -0.13937937, 0.1258349, -0.00968946, -0.00211832, -0.13140695, -0.11206195, -0.0249014, -0.09646855, 0.16041876, -0.20260467, -0.1008801, -0.02906649, -0.148828, -0.01750175, -0.10806648, 0.10094456, -0.09347772, -0.11941694, 0.0714565, -0.13869469, 0.09965917, -0.02859109, -0.04663063, -0.03677037, -0.17767252, 0.19819601, -0.00728577, -0.07156438, 0.14130634, -0.16161495, -0.14514765, -0.0230404, 0.14609018, 0.12496789, -0.02587292, 0.06375326, -0.1703029, -0.14651431, -0.15148057, -0.11655853, 0.00689279, -0.21302195, 0.10692985, 0.00105841, 0.05685481, -0.05294221, 0.13358746, 0.17953119, 0.01200722, 0.00904744, 0.13893746, -0.01024614, 0.14703822, -0.08035436, 0.18355256, -0.00407319],
-"inttoptr":[0.04721976, 0.16880296, 0.06448112, -0.17660472, 0.02998716, -0.1156636, 0.13852204, -0.06320232, -0.03162635, -0.06644673, -0.136013, -0.06916367, -0.02801188, 0.10646518, 0.05828808, 0.03782522, -0.02227561, 0.02234134, 0.16369314, -0.08557447, 0.03853939, -0.14319475, -0.05599201, -0.01060566, -0.1136315, 0.0201032, 0.01836221, -0.05016, -0.1216938, 0.12626994, 0.03450989, -0.19574732, 0.14352062, -0.17880194, -0.13339539, -0.04630446, 0.03136644, -0.13006681, 0.1445991, -0.09556036, 0.09764113, 0.16694036, 0.09833785, 0.05145761, 0.17970908, -0.14575475, -0.13704927, -0.06876626, -0.00955433, -0.16089469, -0.14152728, 0.1012365, -0.18291287, -0.07371435, -0.13809073, 0.02210419, 0.090786, 0.02507522, 0.15311833, -0.11663677, 0.10619268, 0.1758742, -0.04052401, 0.19525726, 0.15682434, -0.19402866, 0.08792165, 0.19677114, 0.1986662, -0.2065629, 0.04007441, -0.01697999, 0.17015004, 0.07224851, -0.06042317],
-"invoke":[0.11604629, -0.02348489, -0.11863736, -0.16840768, -0.01534824, -0.13232489, -0.09663513, -0.1576129, -0.2348036, 0.00802987, -0.0517984, -0.05321365, 0.01174416, -0.11655734, 0.05259984, -0.16816719, -0.09482966, -0.00228002, -0.12152751, -0.00498575, 0.03465938, -0.02950593, 0.04545004, -0.2172498, -0.07073424, -0.08918925, 0.0623108, -0.20881031, -0.04182759, -0.20031597, -0.1351732, 0.069332, -0.00361818, 0.11065516, 0.13967985, -0.08796342, -0.00256544, -0.11508698, 0.09178918, 0.2073513, 0.04010937, -0.07716485, 0.06940974, 0.20694868, -0.08160415, 0.1820004, -0.12159622, 0.04531517, -0.00216078, -0.11625812, -0.10868325, 0.11549117, 0.19036727, 0.0365869, -0.09580094, 0.22367325, 0.11756624, -0.2184826, 0.27060437, 0.14232084, 0.01686368, -0.10716032, 0.02151413, -0.02634864, 0.19014603, -0.22087607, -0.05267305, -0.2422757, -0.06234193, -0.0299236, 0.01296355, 0.0901266, 0.09916983, -0.00811885, -0.02482844],
-"label":[-0.06294985, -0.07393633, -0.07793023, 0.19617933, -0.25141802, -0.16856542, 0.05546749, 0.2207709, -0.12299901, -0.10727913, -0.04701709, -0.05162717, 0.2220566, 0.14470658, -0.07780463, -0.02452197, 0.21929401, 0.10171317, -0.09900579, -0.11624218, -0.21602261, 0.11818817, 0.23674524, -0.0676048, -0.15250057, -0.06562828, 0.14570533, -0.0063743, 0.12755615, 0.18377739, -0.05544094, -0.03191924, -0.12098482, -0.04439923, 0.0412157, -0.13413347, -0.03934726, 0.04211318, -0.02061427, 0.04409805, -0.06535667, -0.10373748, -0.11245634, -0.07654405, 0.1734968, -0.01024228, -0.05300638, 0.08171037, 0.1142247, 0.13233529, -0.04136537, -0.00692873, -0.13803534, 0.0900873, -0.18191165, 0.05589029, -0.05146271, -0.11442295, 0.17379795, 0.11092487, 0.18033165, 0.03263773, -0.09051663, 0.05547883, 0.09321848, -0.13325489, 0.07476224, -0.16582265, -0.14871986, -0.11099493, 0.08359679, -0.2101298, -0.05699859, -0.08379642, 0.10380454],
-"landingpad":[0.12086448, 0.16615215, -0.11424457, -0.08484724, -0.09069939, -0.15767984, 0.10221493, -0.17647022, -0.18533581, -0.02190816, 0.1541339, -0.15779817, 0.03144602, -0.10866319, -0.01497394, -0.1721984, -0.07559271, -0.11889227, -0.1177194, -0.02792813, 0.04734257, -0.03713308, -0.17093344, 0.17252314, 0.02917121, 0.01523044, -0.08545186, -0.20184867, -0.08280238, -0.04850229, 0.03705949, 0.07665683, 0.01846915, 0.00205453, 0.1709319, -0.10548064, -0.07096241, 0.12575251, -0.1177155, 0.17698564, 0.03965896, -0.10821396, 0.07917166, 0.17810985, 0.01715738, 0.16967225, -0.16826203, 0.16891196, -0.15257767, 0.10498544, -0.10490264, 0.06432237, 0.18670914, -0.00099873, -0.13853344, 0.21604276, 0.13351974, -0.15664832, -0.08585881, 0.1024329, 0.02737338, 0.11940238, -0.01797596, -0.00314033, 0.12023867, 0.11616865, 0.05652065, 0.1415307, -0.15827312, -0.03440882, 0.02146849, 0.01478402, -0.13532415, -0.02099262, 0.00462938],
-"load":[0.06383128, 0.0803676, 0.07070765, 0.1284488, 0.053152, -0.11878661, -0.09626942, -0.05779593, -0.04309646, -0.31707463, -0.01623556, 0.0741919, -0.0051238, 0.07403108, -0.25466156, 0.09440999, -0.01665624, 0.14732535, 0.17144501, -0.00878042, 0.04688795, -0.15015227, -0.06462591, -0.04178058, -0.0378243, 0.33024567, -0.07477789, -0.04902143, -0.0476392, 0.13162148, -0.17537315, -0.01750547, 0.14473993, 0.1732234, -0.1423584, 0.11269526, 0.03678195, -0.1315726, 0.15341991, -0.10186778, 0.00435855, 0.16657132, 0.17959148, -0.00966169, -0.16982812, -0.02318787, 0.09485581, -0.07465204, -0.06222337, 0.01743078, 0.12776802, 0.09864764, -0.01487866, 0.03529362, 0.13232034, -0.01328148, 0.09440556, 0.06241368, 0.03618008, -0.11342508, 0.09918994, -0.15651965, -0.07195813, -0.09057574, -0.10321549, 0.06875013, -0.00129774, -0.04545839, -0.07164564, 0.05073486, 0.34142092, -0.08255292, 0.12089149, 0.08211686, 0.21363957],
-"lshr":[0.1238757, -0.02033522, 0.05549098, 0.11646183, 0.08761991, -0.12349915, -0.18941326, -0.08920491, 0.09122052, 0.03471606, 0.2897465, 0.07269037, 0.01238082, 0.07473343, 0.02699653, 0.08085709, -0.04658483, 0.2076412, 0.22355735, -0.03676752, 0.18093167, -0.15845503, -0.0710993, -0.02861892, -0.24865082, -0.00123379, -0.23955399, 0.13371961, -0.11877617, 0.07246234, 0.01312172, 0.01923358, -0.04966315, 0.1200467, -0.2137276, 0.10946998, 0.07409713, -0.12598129, 0.02809012, -0.09355168, 0.06742698, 0.08053191, 0.18937954, 0.00296521, 0.10038757, -0.19544551, 0.08368544, -0.07828917, 0.20687084, 0.01422358, 0.19612609, 0.03323235, -0.01736557, -0.2952406, 0.15087438, -0.07250606, 0.01719518, -0.01060284, 0.16445734, -0.16287296, 0.07179926, -0.14751296, 0.02352273, -0.17620452, 0.12871586, 0.01027796, -0.00240841, -0.08637159, -0.07440444, 0.02873684, 0.05644969, -0.07749362, 0.17137784, 0.09177881, -0.06086624],
-"mul":[9.14543942e-02, 7.25680292e-02, 1.87143609e-01, 1.56122670e-01, 8.18791837e-02, -1.13938227e-01, -1.17102735e-01, 1.78440675e-01, -5.71394851e-03, 3.98688577e-02, -1.22035742e-02, 9.62941721e-02, 9.87940375e-03, 6.54331893e-02, -1.18242744e-02, 1.01057030e-01, -3.37237865e-02, 1.82104349e-01, 1.98254153e-01, 4.89455797e-02, -1.38579145e-01, -1.43827185e-01, -7.30315745e-02, -2.81286202e-02, -3.69295813e-02, -7.32784346e-02, -7.52561465e-02, 1.15158796e-01, -8.05188417e-02, 1.22851163e-01, 1.18959583e-01, -6.19116612e-03, 1.14142060e-01, 1.15443885e-01, -1.32043362e-01, 1.91231743e-01, 6.71850219e-02, -1.18121214e-01, 1.39134541e-01, -9.38180089e-02, 7.00822175e-02, 1.52467072e-01, 1.72531351e-01, 2.39370289e-04, 8.84519666e-02, -1.74602866e-01, 7.58597329e-02, -7.24562407e-02, 2.30801646e-02, 2.31582299e-02, 1.77945897e-01, 2.19754413e-01, -1.05254715e-02, -5.34405783e-02, 1.28701046e-01, -8.88467114e-03, 2.39838846e-02, -2.52107698e-02, 4.09773886e-02, -1.43636853e-01, 9.37281698e-02, -1.36706889e-01, 3.18117648e-01, -1.72676995e-01, 9.86104552e-03, 1.59371197e-02, -2.31163911e-04, -7.84823969e-02, -7.37200677e-02, 3.27852108e-02, 1.85986701e-02, -7.93214887e-02, 1.19079717e-01, 9.70832258e-02, -5.25254011e-02],
-"or":[-0.15830125, 0.08118854, 0.06019896, 0.1351512, 0.0696514, -0.12829824, -0.12040509, -0.07483122, -0.05365232, -0.21453764, 0.01100464, 0.07939477, 0.01512126, 0.07680329, 0.20671816, 0.10159217, -0.03416578, 0.18844754, 0.2232867, 0.00390219, 0.17268041, -0.16189471, -0.07818128, -0.02745846, -0.03785938, -0.28434664, -0.23963155, 0.13414037, -0.03098747, 0.1290441, -0.04272285, 0.00627589, 0.15120292, 0.13505426, -0.14509544, 0.1633035, 0.07651295, -0.1345261, 0.15650204, -0.10334036, 0.08041331, 0.17568573, 0.18555732, -0.00240171, 0.09882613, -0.1697568, 0.08491044, -0.07938255, -0.00502023, 0.04242367, 0.19246665, 0.10571583, -0.01717717, -0.0940207, 0.19319834, -0.06771892, 0.03723063, 0.00987004, 0.08289062, -0.14470372, 0.07547856, -0.14796427, 0.03079064, -0.18788183, -0.06784894, 0.02374648, 0.00680319, -0.08787614, -0.0762485, 0.04015802, -0.31986576, -0.08643831, 0.14352895, 0.1116055, -0.09564826],
-"phi":[-0.17533, -0.14122052, 0.25121832, -0.20197222, -0.02265841, 0.08878156, -0.12254399, -0.05334119, 0.17813319, 0.25582585, -0.24652013, 0.08741698, -0.20831233, -0.12509026, -0.19707216, 0.15587404, -0.07037015, 0.00634299, 0.02692973, 0.20398706, 0.09076547, 0.2106457, 0.02665563, -0.01256549, 0.20547326, -0.01408568, -0.11248078, 0.17035946, 0.00720961, -0.24428545, -0.19424082, 0.02588499, -0.09168262, 0.12671804, -0.13895495, 0.11016023, 0.06885135, -0.1585114, 0.1442796, -0.09209647, -0.25246575, -0.18171434, 0.07803512, 0.01651475, -0.05719611, -0.18895395, 0.13336438, -0.09516547, 0.04140934, -0.13787958, 0.14149408, -0.15089944, -0.02157312, 0.2192603, 0.02856141, -0.05521443, 0.0077216, 0.2731221, -0.09575493, 0.2032519, -0.12885517, -0.18518618, 0.16917716, -0.11144329, 0.25175345, -0.30578047, -0.07883886, 0.14204925, 0.26059705, 0.01635692, -0.2599762, 0.1671248, 0.07727495, 0.06118969, 0.27382395],
-"pointer":[0.01122629, 0.03439064, 0.12280786, -0.03425089, 0.17276576, 0.21717595, 0.09126496, -0.17714174, 0.27793115, 0.08274158, -0.08821795, -0.16903219, -0.06300986, -0.23226759, 0.1235508, -0.11923615, -0.21793933, -0.17721081, 0.07182363, 0.27753174, 0.0920836, 0.06713749, -0.04808864, 0.15279293, 0.05296317, 0.04695908, -0.00205181, -0.14607918, 0.177891, -0.01903534, -0.05020404, 0.20370306, 0.03707821, -0.08771795, 0.10304545, 0.13130909, 0.18650576, 0.05210593, -0.09720453, 0.14389311, -0.05903525, 0.01644695, -0.11466363, 0.18700723, 0.07347484, 0.13623913, 0.13876125, 0.10103971, 0.11578573, -0.21065098, -0.07865446, -0.02068869, 0.16991298, -0.16359806, -0.07672877, 0.11908785, -0.14677979, 0.16077666, -0.21696989, 0.03678338, -0.02173937, 0.08764295, 0.18778177, 0.0974953, 0.05379327, -0.10313212, -0.2190201, 0.25079602, -0.1224342, -0.17659691, -0.10354815, 0.12724441, -0.11835013, -0.15596658, -0.262892],
-"pointerTy":[-5.16731367e-02, 1.41998947e-01, -1.03678465e-01, -2.35795856e-01, 1.67232469e-01, -3.50435115e-02, 1.31282926e-01, 8.77518952e-02, -1.61656141e-01, -1.89439468e-02, 6.89205006e-02, -1.47259831e-01, -1.85588151e-02, 1.95201755e-01, 1.53539265e-02, -1.11738130e-01, 7.46774301e-02, -1.60889581e-01, -1.51122361e-01, -1.64823771e-01, -7.43334070e-02, 1.79255735e-02, -8.14272910e-02, -1.60337687e-01, 7.73334429e-02, 1.45058706e-02, -1.51032144e-02, -1.70947760e-01, -1.48217022e-01, -7.63990656e-02, -8.62834305e-02, -1.66838810e-01, 4.88951942e-03, -1.25664428e-01, 1.13860749e-01, -2.08933145e-01, -1.30075082e-01, 7.92812705e-02, -7.94121996e-02, 1.08121462e-01, -2.62738355e-02, -4.07571979e-02, -9.88764390e-02, 1.00676276e-01, -2.82900538e-02, 7.71642849e-02, -2.50683606e-01, -1.28315210e-01, -1.78229675e-01, -2.19585538e-01, -8.51736143e-02, -1.43008991e-04, 1.66052416e-01, 6.89304620e-02, -1.85903355e-01, 1.39382124e-01, -1.14323251e-01, 6.72841072e-02, -1.73793092e-01, 1.37479026e-02, -6.61165267e-02, 1.28709331e-01, -3.70565802e-02, 1.15480699e-01, -1.72476366e-01, 1.86700657e-01, 1.71808466e-01, 1.25800788e-01, 1.42911309e-02, -1.19691528e-01, 1.48811145e-02, -1.66594520e-01, -1.34164989e-01, 1.62748635e-01, 3.45790684e-02],
-"ptrtoint":[0.13572109, 0.07109733, -0.01410782, 0.13761328, 0.06249858, -0.12742682, -0.03786812, -0.08852433, -0.00890332, -0.12647949, -0.23919484, 0.07803643, -0.29866624, 0.07854372, 0.0690124, 0.08534926, -0.03230691, -0.11528994, 0.2259676, -0.09829737, 0.10256737, -0.16430771, -0.0642257, -0.01210832, -0.12542038, 0.01087978, -0.2651048, 0.10121775, -0.12749597, -0.00404411, 0.08437401, -0.01521534, -0.05364794, -0.20650637, -0.15205501, 0.16177334, 0.08448336, -0.13946855, 0.15794401, -0.08243193, -0.05839634, 0.17580664, 0.19553919, 0.02601502, 0.14116347, -0.17160966, -0.09532923, -0.07157483, 0.20540062, -0.00341362, 0.20416199, 0.09911527, -0.01078249, -0.0624228, 0.14551571, 0.07151687, 0.04763724, -0.02011446, 0.16194499, -0.18160547, 0.13115011, 0.01976459, -0.01537652, -0.19834353, -0.14665273, -0.20727639, 0.03673945, -0.09134132, -0.15628532, -0.04004635, -0.02747078, 0.14147465, 0.17462312, 0.11279432, -0.07561264],
-"resume":[0.16280797, -0.14593223, -0.12683636, -0.1562535, -0.03300981, -0.13968605, 0.16718441, -0.19515003, 0.01863662, -0.05534162, -0.01442544, 0.0425216, 0.0093952, -0.16576275, 0.03578432, -0.15065056, -0.06406866, -0.16523188, -0.05909818, 0.06044476, -0.0871481, 0.15487999, -0.18122765, 0.1956721, -0.04706432, 0.08488349, 0.04219154, 0.02599989, -0.02276746, -0.18513387, 0.12803178, -0.09826294, -0.11191379, 0.04426758, 0.11935913, -0.14695078, -0.00907954, 0.15606298, -0.13066523, 0.01243626, 0.07249606, -0.13709828, 0.10189614, 0.18291259, -0.04105699, 0.1736963, -0.12169949, 0.09442408, 0.20398152, 0.22682573, -0.08286698, -0.13903227, 0.09133334, 0.03095248, 0.11325783, 0.1569758, -0.17479569, -0.12558746, -0.12676957, -0.12449513, 0.17823087, 0.16940698, -0.06061083, -0.1153774, 0.11211671, 0.03377679, -0.05404395, 0.10726969, -0.120761, 0.01132012, 0.00449077, 0.00602859, -0.13664234, 0.00466423, -0.02339442],
-"ret":[0.14368142, 0.0712266, 0.05669672, 0.13478386, -0.14595962, -0.12663847, -0.12403987, -0.09207936, 0.00078773, -0.20975904, 0.08747889, 0.07683876, -0.14989862, -0.18156855, 0.38113984, 0.05973466, -0.05483145, 0.00884139, -0.013859, -0.03841434, 0.05703019, -0.04955313, 0.04577729, -0.05112582, -0.25303054, 0.13709044, -0.06985793, -0.04175208, -0.06085019, 0.12376258, 0.08259401, -0.01146284, 0.05020906, 0.18525098, -0.15137392, 0.07154816, 0.06121736, -0.14056544, 0.16508178, -0.12329083, 0.22687763, 0.17450114, 0.18714464, 0.01847176, -0.02590629, -0.15461633, 0.0910584, -0.07975882, 0.03339317, 0.08106077, 0.10348453, 0.12415495, -0.01464526, -0.20166221, 0.23424618, -0.0058443, 0.13251036, -0.01039419, 0.19171973, -0.14341427, 0.11032798, -0.17445254, -0.21265155, -0.11035369, 0.09180369, -0.01410731, -0.05983369, -0.0549278, 0.11081824, 0.02046819, -0.04516792, -0.09715447, 0.14870624, 0.05914663, 0.10917712],
-"sdiv":[7.54674971e-02, 2.70023555e-01, 2.15832815e-01, 1.51102781e-01, 3.86169888e-02, -1.27587810e-01, -1.64553642e-01, 2.08858803e-01, 2.49577500e-03, 3.39335836e-02, 2.95089424e-01, 7.44504854e-02, 1.75000988e-02, 7.48216212e-02, 4.57024798e-02, 1.14481322e-01, -5.15110120e-02, 1.97094589e-01, 2.28497684e-01, 1.89727977e-01, -2.03567356e-01, -1.53853878e-01, -7.04708546e-02, -2.87290998e-02, -4.15551029e-02, -8.37634057e-02, 2.48096928e-01, 1.39052629e-01, -1.18507713e-01, 1.37168303e-01, 4.37297896e-02, -2.56327417e-04, 1.03771754e-01, 1.18692242e-01, -1.97971940e-01, 2.48502731e-01, 5.65373003e-02, -8.69503245e-02, -9.03292373e-03, -2.25797057e-01, 7.20400363e-02, 4.09315303e-02, 1.83778912e-01, -2.47538835e-03, -2.83727050e-02, -2.00868785e-01, 8.99481177e-02, -7.83472061e-02, 4.22467515e-02, 2.67557669e-02, 1.77338138e-01, -1.42202228e-02, -1.59657858e-02, -6.30925298e-02, 3.13569121e-02, -2.22823601e-02, 6.28621271e-03, -1.85963176e-02, 1.17826000e-01, -1.24646917e-01, 1.02576114e-01, -1.52709767e-01, 2.57164501e-02, -1.40907198e-01, 1.74567476e-01, 1.61048099e-02, -5.03884954e-03, -8.63459408e-02, -7.52566457e-02, 3.38615589e-02, 1.36013612e-01, -8.77554864e-02, 1.03453219e-01, 7.68808499e-02, -6.49084374e-02],
-"select":[0.05093412, 0.07767349, 0.07669216, 0.1312176, 0.0579063, 0.1616888, -0.11760561, 0.00115628, -0.04131674, -0.29780418, -0.01691337, 0.0748211, 0.0864482, 0.07403006, 0.25878662, 0.10279866, -0.01487916, 0.15694329, 0.22207598, 0.01221279, 0.02520609, -0.1521685, -0.06417656, -0.03086487, 0.12268542, 0.3565569, -0.08125142, 0.1072576, -0.02930279, 0.14090922, -0.14309497, -0.00379189, 0.15369308, 0.12072479, -0.11386253, 0.17081785, 0.0359251, -0.13158213, 0.15140608, -0.08365479, 0.04333765, 0.17864218, 0.09735203, -0.01021584, -0.1817566, -0.17009453, 0.10480749, -0.07720621, -0.06324099, 0.01645014, 0.13028447, 0.10660475, -0.02038397, -0.05975712, 0.02713404, -0.01552912, 0.07295278, 0.05679769, 0.02740115, -0.11240587, 0.07162624, -0.15834542, -0.32627442, -0.08725446, -0.07952213, 0.07846557, -0.01865849, -0.04569499, -0.0728619, 0.0374458, -0.27716625, -0.07809193, 0.11488006, 0.06656368, -0.06777868],
-"sext":[-0.03718835, 0.08533189, 0.21354397, 0.13909821, 0.04430787, -0.13043992, -0.12707381, 0.14811471, -0.01397946, -0.24312808, -0.01538438, 0.08007284, -0.20212536, 0.07967917, 0.16868502, 0.10560705, -0.01859606, 0.16425593, 0.23104599, 0.04822871, -0.15584451, -0.16572693, -0.07367796, -0.02386475, -0.04161641, -0.28961626, 0.26639512, 0.13398732, -0.09589424, 0.14068675, 0.16089684, -0.01488031, 0.16961882, 0.11251253, -0.15109386, 0.20157905, 0.04388804, -0.14243662, 0.17028852, -0.17495492, 0.08312923, 0.18138759, 0.19254005, -0.00167047, 0.1016985, -0.17693631, 0.09644604, -0.07967581, -0.01462245, 0.00063264, 0.19925497, 0.10426001, -0.01645718, -0.0897459, 0.02190982, -0.00871549, 0.03981987, 0.01151098, 0.07275884, -0.13166158, 0.11194995, -0.15796044, -0.02378252, -0.17802656, -0.05159309, 0.03235529, 0.00206553, -0.0869719, -0.07645953, 0.05517219, 0.01411109, -0.08648386, 0.1488923, 0.11236069, -0.14168048],
-"shl":[-1.21411718e-01, 8.12682733e-02, 7.26464316e-02, 1.38484403e-01, 8.17928985e-02, -1.24320365e-01, -1.29878476e-01, 1.89510286e-01, -6.82524173e-03, 4.69798781e-02, 7.76179358e-02, 7.87331015e-02, -1.99450791e-01, 7.40546957e-02, 1.56282917e-01, 1.49098694e-01, -3.80089432e-02, 1.97095275e-01, 2.20074877e-01, 9.85120013e-02, -1.51519790e-01, -1.56306505e-01, -6.62787706e-02, -2.89262943e-02, -8.50795880e-02, -3.77611667e-01, -8.08443353e-02, 1.29505768e-01, -9.56492350e-02, 1.34987608e-01, 1.17273629e-01, -8.01561028e-03, 2.01825440e-01, 1.25738055e-01, -1.46153197e-01, 2.18217447e-01, 7.04046711e-02, -1.39466003e-01, 6.33412227e-02, -1.05604395e-01, 7.39852190e-02, 1.70712352e-01, 1.86249197e-01, 1.75760695e-04, 9.54905152e-02, -1.82715610e-01, 8.42701495e-02, -7.70764053e-02, 3.44463770e-04, 2.22381260e-02, 1.92770571e-01, 6.70530349e-02, -8.68750829e-03, -5.87612167e-02, 1.35470673e-01, -9.91571508e-03, 3.54958773e-02, -1.26978569e-02, 3.80189577e-03, -1.57232955e-01, 1.05788536e-01, -1.48979709e-01, -6.13744035e-02, -1.79459959e-01, 1.58743337e-02, 1.66407768e-02, 3.78276110e-02, -8.67255181e-02, -7.59592950e-02, 3.41174081e-02, -1.17456488e-01, -8.46017748e-02, 1.42491326e-01, 8.54357481e-02, -6.05630279e-02],
-"shufflevector":[-1.64276063e-01, -1.40854018e-02, 2.16989160e-01, 1.48834497e-01, -1.81232363e-01, 1.71089604e-01, -1.51962206e-01, 9.34694987e-03, -6.01843521e-02, 1.57123819e-01, 1.49263054e-01, 1.57006048e-02, 8.54564831e-03, -1.46307275e-01, -9.97107551e-02, 2.03836277e-01, -1.10085299e-02, 1.58312604e-01, 3.98286656e-02, 1.22957818e-01, -4.54752985e-03, -1.51905298e-01, 2.56421398e-02, -9.66223106e-02, 1.35936990e-01, 1.13613196e-01, 8.88981670e-02, 1.39288589e-01, 2.35020537e-02, 1.54158428e-01, 1.38836756e-01, -7.45062644e-05, 1.85534820e-01, 2.31408417e-01, -1.86368376e-01, 2.18553007e-01, -1.25940442e-01, 6.19565435e-02, 2.09610596e-01, -2.28297710e-01, 2.30249181e-01, 1.89029738e-01, -6.89853504e-02, -3.19687016e-02, -1.80848792e-01, 4.24141204e-03, 1.89942136e-01, -1.91565320e-01, -2.52943840e-02, 5.65835461e-02, 1.32085189e-01, 1.23422973e-01, -2.14130040e-02, -2.95120943e-02, 1.43252522e-01, -2.44392790e-02, 1.89641267e-01, 3.69141959e-02, 5.39628556e-03, -1.15757883e-01, 6.42104596e-02, -2.11238548e-01, 1.83569789e-02, -9.20821503e-02, -9.48524103e-02, 8.56385827e-02, -5.02256379e-02, -8.02270398e-02, 8.46109912e-02, 7.74170980e-02, 3.27805057e-02, -1.08585857e-01, 1.16238609e-01, 5.87845854e-02, 1.14851862e-01],
-"sitofp":[0.05548016, 0.16714495, 0.21498686, 0.14356636, -0.16821964, 0.16527581, -0.15967155, 0.21070401, -0.01250376, -0.11125261, -0.06026782, 0.07956001, -0.13129172, -0.18479359, 0.08652773, 0.08131526, 0.15925482, 0.08148654, -0.00736642, 0.19463325, -0.20742258, -0.11578189, 0.02654804, -0.08633579, -0.05612461, 0.01452674, 0.11379433, 0.03724489, -0.12955493, 0.15072383, -0.0232376, -0.01797887, 0.20284739, 0.20505717, -0.18686672, 0.25398365, -0.11697914, -0.09353511, 0.19472948, -0.18606324, 0.13722429, 0.19458178, 0.03767281, -0.01490554, -0.15784514, -0.0030121, 0.1061831, -0.18870111, -0.04370302, 0.06840985, 0.12218294, 0.133711, -0.01636076, -0.05479606, -0.09695652, -0.01025795, 0.2093504, 0.08853228, -0.07362932, -0.06125485, 0.10777287, -0.17920133, 0.03073885, -0.09303376, -0.18211918, 0.09700941, -0.05694851, -0.0481168, 0.10195274, 0.11961144, 0.14820483, -0.10259698, 0.07829086, 0.06143153, -0.06575584],
-"srem":[0.12748273, -0.01354557, 0.07656507, 0.13586837, 0.09215222, -0.12957662, -0.22408909, 0.21526249, 0.0899937, 0.03087448, 0.23878594, 0.07816898, 0.0165252, 0.07721527, 0.0565106, 0.08858381, -0.13782543, 0.2185999, 0.2350998, 0.20486632, -0.22521928, -0.16895708, -0.07120457, -0.03142672, -0.193668, -0.06093899, 0.098878, 0.13780232, -0.13592266, 0.13956556, 0.03245056, -0.01644238, 0.1208498, 0.11854354, -0.20298819, 0.27215648, 0.06262159, -0.13207191, -0.22367987, 0.18101601, 0.07210222, -0.035157, 0.19500516, -0.0082911, 0.04821047, -0.2052546, 0.0944626, -0.08101162, 0.05265743, -0.00286297, 0.2012179, -0.02560825, -0.01475919, -0.06332223, -0.21597023, -0.00943558, 0.00891592, -0.04312498, 0.11415429, 0.08646774, 0.10881357, -0.15501237, 0.02134495, -0.15064926, 0.14675559, 0.01146032, 0.0020576, 0.09925999, -0.07733848, 0.03300545, 0.05376984, -0.08198962, 0.17412202, 0.08533774, -0.06574512],
-"store":[0.09998547, 0.07416256, 0.06472399, 0.12126696, 0.03879403, -0.12036075, -0.11504382, -0.06913467, -0.03769127, -0.33070236, -0.00644616, 0.07156026, -0.02137089, 0.07216891, -0.05470073, 0.09415297, -0.04082007, 0.1056991, -0.0013265, -0.00926145, 0.04998971, -0.14578499, -0.05182505, -0.02615017, 0.08348413, 0.31976137, -0.07019657, -0.05163203, -0.04972167, 0.12658256, 0.04289434, -0.0016057, 0.14311627, 0.12263075, -0.13941614, 0.13760482, 0.04816094, -0.12741822, 0.14878556, -0.11523844, 0.05825484, 0.1649721, 0.09197503, -0.0012878, -0.1629092, -0.1414969, 0.09324066, -0.07395344, -0.00317431, 0.01789259, 0.12094813, 0.10764453, -0.01553114, 0.11882594, 0.02442449, -0.01568008, 0.11022279, 0.03307666, 0.08506706, -0.10825977, 0.06488108, -0.15164001, -0.30984077, -0.08653892, -0.06201477, 0.01810287, -0.04473237, -0.04719033, -0.06153039, 0.03287021, 0.29479045, -0.07162003, 0.13225412, 0.05114779, -0.33301073],
-"structTy":[-5.22783101e-02, 1.48193806e-01, -1.73157901e-01, -8.37061405e-02, 1.75303683e-01, 1.41080663e-01, 1.32847771e-01, 9.91027430e-02, -1.22380503e-01, -5.10943905e-02, 7.63119832e-02, -1.28850952e-01, 1.66698098e-01, 1.32489190e-01, -4.90079597e-02, -1.37256354e-01, 4.85256799e-02, -1.78637013e-01, -1.86148569e-01, -1.64286211e-01, -6.26875609e-02, 8.13696086e-02, -1.20131724e-01, -1.59142628e-01, 3.36435549e-02, 2.20832378e-02, -2.72354912e-02, -1.38390273e-01, -1.38179898e-01, -1.23813316e-01, -9.40988287e-02, -1.55397400e-01, -1.58612743e-01, -1.14575408e-01, 1.55034050e-01, -1.50801614e-01, -1.07854791e-01, 1.55085072e-01, -1.68802753e-01, 1.61272466e-01, -2.05100384e-02, -8.52123126e-02, -9.58884135e-02, -1.21472001e-01, 7.77167231e-02, 1.36054471e-01, 1.28906265e-01, 1.30557746e-01, -1.81647420e-01, 1.45003811e-01, -1.11549668e-01, -1.79680258e-01, 1.72399670e-01, 1.51102387e-04, -4.99001816e-02, 1.28895119e-01, -1.17917232e-01, -9.13353860e-02, -1.21147677e-01, 9.76096392e-02, -1.77910581e-01, 1.50892287e-01, -2.25245636e-02, 1.24061532e-01, -1.16433218e-01, -6.76782504e-02, 1.45374849e-01, 2.35193707e-02, -1.35173321e-01, 1.46973729e-01, -1.22685850e-01, 1.27605781e-01, -1.51997164e-01, 1.60292000e-01, 7.63390437e-02],
-"sub":[-0.13264543, 0.07865644, 0.0717167, 0.1342318, 0.05076935, -0.12322044, -0.12266772, 0.00524008, -0.00201612, 0.05315073, 0.00274359, 0.0742974, 0.0123115, 0.07594631, 0.21857306, 0.10421466, -0.02275163, 0.17884067, 0.21873002, 0.00927466, 0.00893607, -0.154156, -0.06820294, -0.02856713, -0.03866831, -0.2793768, -0.06824332, 0.13080892, -0.08803294, 0.13524993, 0.21245252, -0.01117309, 0.20007995, 0.12509634, -0.14541206, 0.19173518, 0.0613077, -0.13841055, 0.15438707, -0.14274515, 0.07382213, 0.17015299, 0.18052574, -0.00206174, -0.15260881, -0.17803495, 0.08986183, -0.07603207, -0.0163452, 0.02869452, 0.186938, 0.07764658, -0.01459899, 0.10072613, 0.02469735, -0.01351501, 0.03415474, 0.0135162, 0.04113324, -0.12325168, 0.11012445, -0.14946231, 0.04155808, -0.1440628, -0.06376708, 0.01838643, 0.00198335, -0.07941123, -0.07066227, 0.0376416, 0.24301553, -0.0862716, 0.1351057, 0.0891021, -0.3352],
-"switch":[-0.16398937, -0.1353046, 0.08524002, 0.1561178, -0.22488637, -0.14185904, -0.10160676, 0.17193311, -0.05567684, -0.01794963, -0.22830375, 0.08948597, 0.1698675, 0.08543108, -0.18870668, 0.13391507, 0.1591491, 0.21046144, 0.02533695, -0.05875827, -0.15551412, 0.19997434, 0.19250119, -0.03397122, -0.05150964, -0.15923595, 0.0701151, 0.14852029, -0.01560177, 0.15765266, -0.2304326, -0.01416684, -0.05115065, 0.12840182, -0.17300242, -0.03878057, 0.07419615, -0.16339742, 0.18437031, -0.14087205, -0.2714784, -0.17547078, -0.01152283, -0.00396952, 0.11226111, -0.18184493, 0.1166721, -0.09108577, 0.23214185, 0.07350235, 0.14699937, 0.08840676, -0.01411979, 0.2082167, -0.10539865, -0.01378542, 0.04486391, -0.01463741, 0.14504644, 0.19353367, 0.11889812, -0.19718692, -0.04798679, -0.12421229, 0.29124823, -0.32081923, -0.00218598, -0.1037329, -0.08444938, 0.04092982, -0.03067664, -0.1606894, 0.07095047, 0.08663001, 0.25936154],
-"trunc":[0.11878826, 0.08377917, 0.05513672, 0.14270523, 0.07370453, -0.14921571, -0.16542384, -0.07493053, -0.00377195, -0.20950364, 0.12958477, 0.07888171, -0.00595982, 0.07865481, 0.16519356, 0.10834014, -0.04326443, 0.19355299, 0.23304398, 0.07385943, -0.11257231, -0.16919497, -0.082918, -0.0285878, -0.05886409, -0.25552744, 0.08462781, 0.13503358, -0.09682789, 0.12824969, 0.08024148, -0.01201469, 0.15677036, 0.14443342, -0.1907149, 0.23856063, -0.00528661, -0.14044978, 0.16222948, -0.10748056, 0.08127003, 0.17631039, 0.19577822, -0.01176626, 0.09836395, -0.16917923, 0.10726047, -0.10182234, 0.20931682, 0.04668214, 0.2017859, 0.11306392, -0.018347, -0.06246869, 0.24337131, -0.07117373, 0.03053473, -0.00658053, 0.03026612, -0.16877551, 0.10491424, -0.17555256, 0.14634104, -0.18189284, -0.11786141, 0.02876267, -0.00210879, -0.08721989, -0.07824761, 0.03958495, 0.20243248, -0.09376816, 0.11082744, 0.12096685, -0.12131985],
-"udiv":[0.08154505, 0.17510523, -0.19362096, 0.08671232, 0.11330891, -0.19157314, -0.2145219, -0.06051004, 0.1371807, 0.04572224, -0.06733968, 0.07146493, 0.00681373, 0.07493957, -0.02997438, 0.08688152, -0.04591569, 0.23223788, 0.22175628, 0.01500544, 0.14273156, -0.16111672, -0.08531193, 0.19467805, -0.00517966, -0.06020893, -0.06077334, 0.13201268, -0.12358853, 0.00868457, 0.06033499, 0.01196855, 0.14756525, 0.11571166, -0.11228922, 0.13696799, 0.21417981, 0.20094916, -0.21627706, 0.18220317, -0.06209835, 0.1076472, 0.20395155, 0.00197578, 0.13577849, -0.20195886, 0.0833538, -0.07554612, 0.00410905, 0.00158071, 0.1938367, 0.02883399, -0.01710817, 0.03711469, 0.13843457, -0.06804645, -0.00379174, -0.00144552, 0.13146468, -0.18377277, 0.07351629, -0.14530504, 0.01042824, -0.1766743, 0.01462858, 0.01206683, -0.12541275, 0.09634796, -0.07832152, 0.02791307, 0.02765011, 0.19757621, 0.16267568, 0.12346987, -0.0413983],
-"uitofp":[-0.15078814, 0.1826288, 0.11165173, 0.13953921, -0.1842304, 0.16399114, -0.11256697, -0.05694728, -0.04820754, -0.17459138, -0.12022546, 0.08202647, 0.05924561, -0.17365019, 0.1614662, 0.07272313, 0.14744736, 0.09281041, -0.00596566, -0.01817355, 0.09689283, -0.0900236, 0.03450679, -0.0848825, 0.17167987, -0.07416291, 0.09088556, 0.07786386, 0.08914284, 0.14832556, -0.07537761, 0.00234627, 0.15249911, 0.19451788, -0.15017514, 0.14780688, 0.11764529, -0.02274906, 0.1920297, -0.17293543, 0.22891366, 0.1852529, 0.0128901, -0.00331702, -0.18161494, -0.0123833, 0.10250985, -0.1877487, -0.02335341, 0.06747307, 0.12578565, 0.12541758, -0.01838141, -0.08433985, 0.26602918, -0.01791255, 0.18907419, 0.04615575, 0.22383852, -0.13418758, 0.07736338, -0.18702815, 0.06279426, -0.09547485, -0.14973089, 0.09295561, -0.04817062, -0.05371717, -0.01254407, 0.12052637, 0.14356695, -0.1381116, 0.17813013, 0.0863996, -0.10018482],
-"unreachable":[1.6576199e-01, -1.7907834e-02, -5.7712890e-02, 9.4339609e-02, -9.3380272e-02, -1.9464681e-01, 7.7050277e-03, -1.9517663e-01, 2.6435368e-03, 9.1913566e-02, 1.3639191e-02, 7.0288680e-02, -3.2517979e-01, 7.4262276e-02, -8.9507200e-02, -1.7191246e-01, -5.2555885e-02, -1.4688279e-01, 1.0220077e-01, -4.8814032e-02, -1.3367063e-01, -6.1610699e-02, -1.8777388e-01, 2.0192388e-01, 1.7102915e-01, 1.3610463e-01, -2.6177135e-01, -3.3939917e-02, -5.4787207e-02, 1.1302987e-01, 1.0523112e-01, 8.6424664e-02, -3.6340270e-02, 4.7679577e-02, 1.0370419e-02, 3.9357133e-03, 1.5754176e-02, -4.6523277e-02, -1.2651871e-01, -5.0250064e-03, -2.1445632e-01, -1.5226135e-01, 1.7948903e-01, 1.9091401e-01, 1.1402745e-01, -1.9978039e-01, -8.4826022e-02, -3.7182060e-03, 2.1397528e-01, -1.9490053e-03, 3.8303059e-02, 5.5778958e-02, -1.1436568e-02, 7.7554718e-02, 3.7896037e-02, 7.5804755e-02, -1.7246161e-02, -1.4188807e-01, 4.9074650e-02, -1.9401214e-01, 1.9030276e-01, 1.1096356e-01, -1.5253071e-01, -1.9243342e-01, 1.6415399e-01, 2.7355237e-02, 7.6516271e-03, -9.3317330e-02, -1.7084104e-01, 3.9767768e-02, -1.1136789e-01, -9.4884530e-02, 2.1568926e-04, 1.3515340e-01, 8.2443848e-02],
-"urem":[0.1196936, 0.18855006, -0.20257552, 0.01738752, 0.15161008, -0.14784957, -0.02992235, -0.0674265, 0.14330001, -0.00208072, -0.08096982, -0.06099883, 0.01042207, 0.1028357, 0.05679855, 0.08465453, -0.05227959, 0.21925992, 0.22556421, -0.059273, 0.1825322, 0.17631407, -0.08938511, -0.0277707, -0.2572769, -0.06195913, -0.24406035, 0.13713413, -0.13801506, 0.02009896, 0.02613169, 0.01552499, 0.14668229, 0.12454724, -0.14865053, 0.09922966, 0.2130675, -0.11827406, 0.01006878, -0.09671234, 0.06660412, 0.0945213, 0.20740281, 0.00239918, 0.14173026, -0.1632753, 0.08587474, -0.07923763, 0.02992029, 0.2015828, 0.2007573, -0.11056472, -0.01871637, -0.06330916, 0.14348209, -0.07483656, 0.00636646, -0.01837422, 0.17499712, -0.17495239, 0.07653105, -0.1475833, 0.02116938, -0.17941074, 0.1734715, -0.01114943, 0.03780977, -0.15019819, -0.07816306, -0.08282368, 0.05129486, 0.19702028, 0.18443331, 0.10329096, -0.06307837],
-"variable":[-0.00726075, -0.19133486, 0.11386976, -0.02404008, 0.18218324, 0.2147853, 0.05924276, -0.14287908, 0.2636781, 0.08127253, -0.08004244, -0.151082, -0.05468801, -0.14080277, 0.09922418, -0.09297107, -0.2022257, -0.0631255, 0.08962703, 0.2616532, -0.05850444, 0.00352755, -0.1646196, 0.12995376, 0.04720275, -0.04833887, 0.0210924, 0.23275656, 0.20620209, -0.01313339, -0.04938933, 0.18796188, 0.03722439, -0.0570973, 0.07931395, 0.12533784, 0.19275506, 0.04549298, -0.06122055, -0.27613187, -0.03735771, 0.0223406, -0.09128818, 0.15596932, -0.02494176, 0.10621183, -0.03121548, 0.09133425, 0.11494032, 0.1616893, -0.05987769, -0.01591593, -0.15986548, -0.13643607, -0.07331984, -0.21545175, -0.13974416, 0.14965075, -0.19167605, 0.03849319, -0.01465273, 0.04467848, 0.16016744, 0.08067682, 0.05409591, -0.1172289, -0.20374107, 0.21127956, 0.14046785, -0.15151982, -0.07432669, 0.1618207, -0.09742998, -0.14227344, 0.07137881],
-"vectorTy":[-0.03294902, 0.14949119, -0.09443742, 0.01450515, 0.19603784, 0.25139496, 0.07845286, 0.11937284, 0.16359204, 0.06099452, 0.09465881, -0.1069911, 0.09603978, 0.16709542, -0.00486909, -0.07722043, 0.22341618, -0.03857102, 0.09912027, -0.12520577, 0.13405597, 0.01585144, 0.08918274, -0.19392243, 0.02501151, 0.00475938, 0.15233898, 0.17524292, -0.19524728, -0.01515534, 0.08875454, -0.14749287, 0.01111115, -0.0826562, 0.09034086, -0.04994484, -0.10947519, 0.11695202, -0.15125127, 0.10647643, 0.03638166, -0.01116771, -0.14471221, -0.14233877, -0.05823282, 0.16917565, -0.06123812, -0.2417991, -0.17791323, -0.11769169, -0.07800104, -0.08785082, 0.11596312, -0.02590095, 0.06086191, -0.18129952, -0.15834333, -0.13579826, -0.14008556, 0.05263453, -0.1810565, 0.05645358, 0.05425795, 0.11036766, -0.15751451, 0.20713197, 0.15679651, -0.17410725, 0.00938685, 0.2128002, 0.05759892, 0.03854589, -0.12531556, 0.19853209, -0.00149427],
-"voidTy":[0.01605093, -0.0537584, -0.20252232, -0.08176173, -0.01439075, -0.0382277, 0.15691687, -0.104895, 0.06642336, -0.06327856, 0.09566005, 0.1357376, -0.06892086, -0.00750719, 0.00474408, -0.20349212, -0.05127234, -0.17626588, -0.17354688, -0.18703477, -0.10801363, 0.09327657, 0.10076355, 0.12450968, -0.14411868, 0.09044765, 0.01168226, -0.15345524, -0.14508787, -0.04046924, 0.04365833, -0.11735097, -0.1433241, -0.1147697, 0.18176553, -0.18558215, -0.13961768, 0.12200661, -0.13001354, 0.16781498, 0.01012573, -0.09670691, -0.10543029, 0.09294511, 0.08037758, 0.14874886, -0.1797814, -0.12390587, -0.0998297, 0.12955485, -0.15814209, -0.03254232, 0.17421001, 0.10237212, -0.07504516, -0.112073, -0.11584745, -0.20020956, 0.11506493, 0.01263643, -0.10729685, 0.12432116, -0.1172537, 0.10430267, 0.06826, -0.06138111, 0.13390729, -0.1675781, 0.00966638, 0.13989863, -0.00030718, 0.10629188, -0.15517116, -0.08125519, 0.03691618],
-"xor":[0.10254283, 0.18058334, 0.05268734, 0.1032732, 0.10089786, -0.1597949, -0.11678528, -0.07392009, 0.00164178, -0.3411041, -0.05760791, 0.14810716, 0.01395016, 0.07759612, 0.0478027, 0.10309431, -0.04646098, 0.22321403, 0.22373965, -0.0018288, 0.20699704, -0.16503152, -0.18555428, -0.02829313, -0.04340088, -0.09629976, -0.07565032, 0.12829013, -0.1184216, 0.05327054, -0.2091556, 0.01997479, 0.14617224, 0.12249397, -0.11211042, 0.17021945, 0.21050954, -0.1381692, 0.0287271, -0.09731391, 0.07712973, 0.0957914, 0.19754209, 0.00280226, -0.22004068, -0.19650316, 0.08103254, -0.07884329, 0.01431314, 0.01774853, 0.20215395, 0.04173042, -0.01757454, -0.06132229, 0.14705223, -0.07607429, 0.01276137, -0.02355135, 0.1687547, -0.16499719, 0.07755739, -0.15042275, 0.04714211, -0.19073205, 0.01791589, 0.00800123, -0.00147503, -0.08897253, -0.07817551, 0.02938494, 0.07296588, -0.08095124, 0.19651005, 0.12599285, -0.06209126],
-"zext":[0.07005657, 0.08003729, 0.07546207, 0.13528231, 0.02506938, -0.12476956, -0.14581475, 0.00614589, -0.00503257, -0.24861531, 0.09526423, 0.07557298, 0.01302374, 0.07628334, 0.04175617, 0.09726538, -0.02312024, 0.16171065, 0.22212435, 0.0079543, 0.07804493, -0.15814242, -0.06840046, -0.03052379, -0.0414926, 0.26214772, -0.07542547, 0.12756911, -0.08823611, 0.13954116, -0.18143456, -0.01209466, 0.15410358, 0.20636739, -0.1831482, 0.19437483, 0.06973958, -0.13698913, 0.19432417, -0.16362838, 0.09032086, 0.18127388, 0.18287036, -0.01740397, -0.17513882, -0.16302226, 0.10199076, -0.07994232, 0.029813, 0.04575716, 0.19189098, 0.10111669, -0.01755602, -0.0832833, 0.1540967, -0.01496344, 0.04155953, 0.00536162, 0.0699449, -0.13122603, 0.10352977, -0.18122241, 0.17052825, -0.15459219, -0.06903189, 0.05887637, -0.00081859, -0.08415917, -0.07304011, 0.08401199, -0.25388402, -0.09779947, 0.12506351, 0.09557163, -0.12431429]
+    "Opcodes":{
+        "add":[0.09571152, 0.08334279, 0.07070262, 0.14084256, 0.04825167, -0.12893851, -0.12180215, -0.07194936, -0.05329844, -0.28276998, 0.00202254, 0.07488817, -0.16139749, 0.07618549, -0.06084673, 0.10536429, -0.03031596, 0.16402271, 0.22730531, -0.0015432, 0.01986631, -0.15808797, 0.03030382, -0.04201249, 0.01689876, -0.30041003, -0.06479812, 0.1282431, -0.09032831, 0.14007935, -0.23540203, 0.00552223, 0.15325953, 0.13147154, -0.15204638, 0.1616615, 0.05758473, -0.1385157, 0.16438901, -0.14482859, 0.07895052, 0.18407233, 0.15283448, -0.00081216, -0.17934133, -0.16779658, 0.09044863, -0.18453072, -0.00552684, 0.01191218, 0.18504514, 0.13140059, -0.0174882, -0.18127315, 0.02269725, -0.02048657, 0.10858727, 0.0074029, 0.09485064, -0.13431476, 0.07491371, -0.17498694, -0.32914585, -0.14159656, -0.03594542, 0.04091231, 0.00298631, -0.08933277, -0.07178984, 0.04144038, 0.12151367, -0.09504163, 0.13691336, 0.07825345, 0.13958281],
+        "alloca":[0.12644756, -0.20912014, 0.07298578, 0.13963142, 0.03932726, -0.12778169, -0.13084207, -0.09090705, -0.01497083, 0.0646746, 0.14847252, 0.08546949, -0.11579383, 0.07812478, -0.16431178, 0.08578802, 0.06094746, -0.1239017, -0.02789196, -0.01074964, -0.1738938, -0.1629063, -0.07137732, -0.02845656, 0.1728318, 0.13779363, -0.06250989, 0.03479109, -0.08423121, 0.14009888, 0.09620709, -0.02287545, -0.04616966, -0.19591278, -0.19636007, 0.21825573, 0.07949521, -0.14614704, 0.17752613, -0.15099092, -0.04518029, 0.17774306, 0.18947133, -0.00197501, -0.12619424, -0.18458585, -0.09960615, 0.01162648, 0.21306573, 0.0468024, 0.201505, 0.09970672, 0.06599383, 0.17757463, -0.11899275, 0.10299123, 0.06038174, -0.07116731, -0.15743989, -0.1258558, 0.1535076, 0.01872367, -0.14336765, -0.19708565, -0.26796287, 0.02628843, -0.0009325, -0.08809827, -0.12970725, 0.17223337, -0.10651242, -0.09162157, 0.03264611, 0.0911452, 0.09506542],
+        "and":[-0.12902537, 0.07848564, 0.07048505, 0.13578993, 0.03570583, -0.1242408, -0.12619697, -0.06914084, -0.0515872, 0.13225996, 0.01673339, 0.0763608, 0.18325369, 0.07690141, -0.09591792, 0.10353354, -0.02401061, 0.16493416, 0.21942355, -0.00400911, 0.03811587, -0.15406959, -0.07134877, -0.02946596, -0.03854588, 0.28656727, -0.06625008, 0.12780443, -0.04631379, 0.13919763, -0.15808977, -0.00312698, 0.14692454, 0.18495218, -0.14863448, 0.13449706, 0.06471325, -0.13883992, 0.17630532, -0.16833898, 0.07391811, 0.17909151, 0.18229873, -0.00381102, -0.17680968, -0.1645554, 0.10016466, -0.07963493, 0.00130218, 0.05646244, 0.18222143, 0.10511146, -0.0191175, -0.08713559, 0.25423968, -0.02557301, 0.04319789, 0.03259414, 0.07209402, -0.13169754, 0.07424775, -0.17216511, -0.32057068, -0.13833733, -0.0658454, 0.02420194, -0.04393166, -0.08238692, -0.07023077, 0.04014502, 0.20101993, -0.09093616, 0.13076238, 0.09114857, -0.06483845],
+        "ashr":[0.16414718, -0.02078147, 0.07353837, 0.14210321, 0.08068018, -0.13586298, -0.15728961, -0.10365791, 0.00359197, 0.04608767, 0.35770142, 0.08003625, 0.00944533, 0.08471741, 0.05809571, 0.09202059, -0.18349345, 0.22169511, 0.24418135, 0.19192688, -0.1956347, -0.17401719, -0.07583863, -0.02781139, 0.0952112, -0.01444751, -0.27259794, 0.14392436, -0.13541143, 0.1410963, 0.04314299, -0.01641751, -0.05448177, -0.22287542, -0.2131822, 0.25112826, 0.06918604, -0.14414005, 0.060288, -0.09658266, -0.09122665, -0.02779044, 0.20349248, 0.0041391, 0.10610974, -0.20890503, -0.09881835, -0.07368057, 0.22467837, 0.00075097, 0.2147348, -0.02612463, -0.01696278, -0.0649786, 0.15041149, 0.02361087, 0.0211603, -0.03706622, 0.18296233, -0.14298625, 0.14614436, 0.02273145, 0.0209446, -0.21062987, 0.16911499, -0.03668665, 0.00197532, -0.09607943, -0.08398084, 0.02006913, 0.05739584, -0.07919859, 0.19634944, 0.11082727, -0.06584227],
+        "bitcast":[0.1675692, -0.12870269, 0.04048132, -0.1670965, -0.1279611, 0.02615386, -0.16829294, -0.09034907, 0.10913523, -0.07819421, 0.23986322, -0.05966561, 0.08410738, 0.19072439, 0.06047394, 0.02999627, -0.16747619, -0.06076627, -0.02673951, -0.1619169, 0.06443421, -0.13788716, -0.05644303, 0.01361013, -0.06858975, -0.06005004, 0.10011288, -0.05508338, -0.10613093, -0.11281271, -0.00758647, -0.12425531, 0.05333719, -0.16881412, -0.20088236, 0.06015657, -0.16405901, 0.06226884, 0.09171099, -0.09500738, 0.07907875, 0.0776544, -0.18457057, -0.11278627, -0.12111131, -0.10180638, 0.18328871, 0.18770072, 0.20346186, 0.10305139, -0.18344335, 0.10693213, -0.10920919, -0.05994263, 0.20354497, -0.03093485, 0.14214055, 0.00580597, 0.10480052, -0.09955201, -0.134185, -0.02904563, 0.00175069, 0.17646657, 0.01348841, -0.02030338, -0.06537742, 0.10032237, 0.15315783, 0.0102667, 0.07717734, -0.01060431, 0.14727928, -0.16261302, -0.06770234],
+        "br":[0.11499645, 0.0824301, 0.07218035, 0.13258332, -0.13419574, -0.12916718, -0.12626709, -0.06741403, -0.02857593, -0.2543893, 0.02193022, 0.0760875, -0.1562702, 0.07712954, 0.3149361, 0.10217083, -0.041038, 0.16601022, 0.01906607, -0.02043359, 0.05471838, -0.15233372, -0.06945753, -0.02313732, -0.07342829, 0.3331809, -0.05246306, -0.0269839, -0.05435036, 0.13908924, 0.32604694, 0.00170966, 0.14997493, 0.13026518, -0.14908995, 0.12238151, -0.06773318, -0.13566032, 0.16068587, -0.12842499, 0.04970508, 0.17827724, 0.09729939, -0.00447832, -0.1739753, -0.16429187, 0.09886666, -0.08058207, 0.00714044, 0.04585538, 0.13424252, 0.11376464, -0.01675582, 0.17901348, -0.00653374, -0.01570439, 0.13032894, 0.01734108, 0.16833901, -0.1173776, 0.07662185, -0.15942436, -0.21173944, -0.10505079, 0.0597497, 0.03491669, 0.00338842, -0.04969047, -0.07644061, 0.04528612, 0.254365, -0.09514527, 0.12015092, 0.08262096, -0.02352029],
+        "call":[0.10815012, -0.12419116, 0.18759736, -0.1905027, 0.01619313, -0.13483052, -0.12278763, -0.07051246, -0.0083437, 0.25107145, -0.16601063, 0.08127163, -0.17432374, -0.18380919, 0.24335551, 0.07208319, -0.04401246, 0.11606008, -0.02733191, 0.02098145, 0.019888, -0.13705409, -0.07569158, -0.03072285, 0.16870692, -0.09787013, -0.09340432, 0.01931342, -0.03557841, 0.14359893, -0.1592094, -0.00055867, 0.159316, 0.1099042, -0.11837319, 0.08741318, 0.03364393, -0.12831019, 0.10450637, -0.12699029, -0.20213994, 0.18390144, 0.11092624, -0.00209971, -0.13063665, 0.19996215, 0.09006448, -0.07840014, 0.22549215, 0.02587176, 0.13374338, 0.11009877, -0.01874998, -0.21446206, 0.02377797, -0.01036531, 0.05427047, 0.01418843, 0.00771817, -0.12639529, -0.10334941, -0.12244401, 0.30014148, -0.09857437, 0.21212636, 0.03429029, -0.04947309, 0.1023307, -0.07743628, 0.03006962, -0.24868701, -0.02357339, 0.11574048, 0.06895301, -0.363474],
+        "extractelement":[-1.62098512e-01, -2.00751424e-02, 1.71776459e-01, 1.38723463e-01, -1.60769299e-01, 9.30800363e-02, -1.24304645e-01, 1.45934001e-01, -5.04420400e-02, -7.54220188e-02, -5.30924797e-02, 8.55294541e-02, 1.17488159e-02, -1.82809234e-01, 9.37484950e-02, 8.38199258e-02, 1.26266479e-01, 9.31843743e-02, -2.26742271e-02, 6.50950940e-03, 5.02749532e-03, -1.14133619e-01, 1.80842891e-01, -9.63811725e-02, 1.67923152e-01, -5.84629439e-02, 1.04362026e-01, 8.92093012e-05, 7.93750137e-02, 1.39107972e-01, -8.40696543e-02, -1.59506593e-02, 1.99361086e-01, 1.93857521e-01, -1.46850392e-01, -1.78695560e-01, 9.88712162e-03, -1.40836969e-01, 1.77736521e-01, -1.61047727e-01, 3.51648539e-01, 1.79638579e-01, 1.49385989e-01, -6.06128015e-03, -1.72102928e-01, -1.81016047e-02, 1.01466164e-01, -1.28312945e-01, -8.05212185e-02, 6.28385469e-02, 1.15654446e-01, 1.91848025e-01, 3.03963851e-02, 3.55794169e-02, 1.40873834e-01, -1.44319711e-02, 1.85423180e-01, 1.16111919e-01, -7.47816712e-02, -1.14719503e-01, 1.02934733e-01, -1.83810964e-01, -2.64400076e-02, -8.99282843e-02, -1.90383971e-01, 7.31386840e-02, -4.36249487e-02, -4.71482053e-02, 1.07486300e-01, 1.09736443e-01, 4.15226035e-02, -1.42309800e-01, 8.96709636e-02, 6.64985999e-02, -6.13647103e-02],
+        "extractvalue":[0.08820406, -0.0182279, 0.01493346, -0.17219813, 0.02927338, -0.17379265, -0.05663937, -0.06805898, -0.21235467, -0.01969833, -0.15152055, -0.18049374, 0.01062911, 0.07935719, -0.01993761, 0.12405304, -0.03198355, 0.13872959, 0.18017697, -0.0100032, 0.22617207, -0.21553156, -0.18403597, 0.10906533, 0.17709404, 0.07438782, 0.0875822, -0.22567064, -0.00636844, 0.06691552, 0.05859367, -0.06149556, 0.15721355, 0.01889951, 0.2164152, -0.07916643, -0.03927743, -0.02665933, 0.13756634, -0.08835335, -0.01159343, -0.19749148, 0.09438482, -0.1696074, -0.18197437, 0.19907822, -0.20855112, 0.21903005, -0.2204043, -0.07439119, 0.1308343, 0.10343243, -0.10401972, 0.02784402, -0.12545246, -0.18498465, 0.02385085, 0.22105947, -0.01296441, 0.19779074, -0.09320201, -0.07345647, -0.06985376, -0.0657003, -0.15406364, 0.14981052, 0.01336148, 0.18544021, -0.07248794, 0.0511543, -0.00308717, 0.01915094, 0.079139, 0.19975829, -0.06093699],
+        "fadd":[0.0851364, -0.01757606, 0.20865884, 0.17407735, -0.17661206, 0.16181652, -0.11895371, -0.06142977, -0.05126655, 0.04448467, -0.03589934, 0.11921146, 0.0143434, -0.18255684, 0.05196553, 0.10967412, 0.15810761, 0.13785522, -0.01255003, -0.00134187, 0.03563518, -0.12446801, 0.18817455, -0.08609993, 0.1893248, 0.13434686, 0.25903776, -0.00467659, 0.09800702, 0.14749499, 0.03458502, -0.00238901, -0.1395337, 0.19306736, -0.11638073, 0.10491668, 0.04428702, 0.18128034, 0.1797104, -0.1929282, 0.07353631, 0.19062985, 0.10042762, -0.0040207, -0.18379262, 0.07060277, 0.09594815, -0.18272707, -0.12928157, 0.07631373, 0.13005428, 0.24108389, -0.01617561, -0.05803563, 0.02686582, -0.02223067, 0.21217403, 0.17594706, 0.00500477, -0.07345455, -0.08176229, -0.19486704, 0.02177307, -0.09052874, -0.14158368, 0.0890988, -0.06339125, -0.04612265, 0.10199347, 0.11777309, 0.07302366, -0.13303186, 0.09683178, 0.05031883, -0.05881981],
+        "fcmp":[1.02981135e-01, -2.43355539e-02, 5.85255250e-02, 1.09401904e-01, -1.87941462e-01, 1.74530044e-01, -1.20533399e-01, -8.64934921e-02, -5.54629341e-02, 1.41877215e-02, 4.80354428e-02, 1.56707644e-01, 2.56910548e-02, -1.94340080e-01, 5.11265621e-02, 8.82050097e-02, 1.67082191e-01, 1.14665776e-01, -2.00159680e-02, -1.82088744e-02, 2.09244549e-01, -1.15076765e-01, 2.03621030e-01, -8.38057026e-02, -4.32270877e-02, 6.39017820e-02, -6.50197491e-02, -2.72039324e-03, 8.68055448e-02, 1.25798717e-01, -4.21847135e-01, 2.64590848e-02, 1.56372517e-01, 2.14883089e-01, -1.12701654e-01, 9.43767577e-02, -1.97456375e-01, -1.72886148e-01, 2.04119727e-01, -1.56766623e-01, 7.38008767e-02, 2.05716744e-01, 3.48394439e-02, -2.48072352e-02, -2.03255862e-01, 5.07165631e-03, 1.12850599e-01, -1.94697857e-01, -6.29984289e-02, 8.21878910e-02, 1.23685144e-01, 1.40430763e-01, -2.20402889e-02, -6.55216798e-02, 2.90598303e-01, -7.66792744e-02, 2.16097474e-01, 6.28568381e-02, 2.69129931e-04, -1.17002167e-01, -1.60454452e-01, -2.07048669e-01, 2.62669493e-02, -9.61467475e-02, -1.88501909e-01, 8.80606323e-02, -5.39420024e-02, -5.08916602e-02, 1.09891914e-01, 1.21375419e-01, 5.10863326e-02, -1.43003076e-01, 8.39024931e-02, 9.51330177e-03, -6.65674359e-02],
+        "fdiv":[-0.14435205, -0.02003789, 0.07479589, 0.13465217, -0.18792528, 0.16504793, -0.12187403, 0.00537306, -0.04864043, 0.04958175, -0.14890797, 0.10156095, 0.01710029, -0.1809697, 0.06765524, 0.18452686, 0.16923915, 0.11264159, -0.01412142, -0.00287484, 0.01297035, 0.06755029, 0.18517323, -0.08975757, 0.16194478, 0.04174679, 0.10903918, 0.00072231, 0.10999232, 0.14989018, 0.05669819, -0.00733533, 0.20606744, 0.22580206, -0.14587933, 0.13480443, 0.05984581, -0.19580479, 0.20258778, -0.17146486, 0.07523733, 0.19139282, 0.03738374, -0.01550991, -0.21507922, 0.07425626, 0.10224851, -0.18437587, -0.06662318, 0.07977687, 0.12355862, 0.1434375, -0.01888226, 0.03870944, 0.07555477, -0.01191964, 0.23488866, 0.06922436, -0.07203218, -0.06884275, 0.07191673, -0.19515742, 0.02168754, -0.08983592, -0.20972523, 0.08903123, -0.05095004, -0.04967143, 0.0985181, 0.15771264, 0.09441628, -0.14769785, -0.16405083, 0.05066825, -0.058015],
+        "fmul":[-0.05690098, -0.02700638, 0.20705073, 0.16630849, -0.17719169, 0.16012336, -0.12955493, 0.19698587, -0.04035136, -0.03197788, 0.08116172, 0.1013689, 0.01416616, -0.17855197, 0.15503113, 0.13990149, 0.15355295, 0.12397105, -0.01912402, 0.01684221, 0.00669227, -0.11869009, 0.17604592, -0.05788622, 0.10467764, -0.0807798, 0.10300152, 0.10780353, 0.08334652, 0.1446782, 0.03546653, -0.00421023, 0.19682531, 0.19603632, -0.1466306, 0.12923796, 0.06296455, -0.14397427, 0.183406, -0.19625223, 0.08244827, 0.1860209, 0.10110238, -0.002166, -0.18034802, 0.06406105, 0.09568714, -0.17962225, -0.05463478, 0.04498994, 0.11928298, 0.1694295, -0.01222703, 0.14453876, 0.13506988, -0.01149808, 0.20273286, 0.05309585, -0.07139173, -0.07370458, 0.09957068, -0.19369006, 0.02577178, -0.09011577, -0.21375039, 0.07624438, -0.04969844, -0.04844297, 0.08409201, 0.11113621, 0.14570779, -0.13936704, 0.07202481, 0.05272412, -0.05747499],
+        "fneg":[-0.04075071, -0.09381824, 0.17104743, 0.0744117, -0.15771168, 0.16444896, -0.11586417, 0.00200867, 0.17323531, -0.10421973, -0.05619403, 0.07496857, 0.05573901, -0.18903743, 0.06586835, 0.03499747, 0.17244707, 0.02476844, -0.02552369, 0.00063759, 0.04312319, -0.04501259, 0.1850581, -0.14339973, -0.04433612, 0.01609048, 0.10461161, -0.12243931, 0.07718915, 0.15313269, 0.12116063, -0.03062805, 0.21040255, 0.21582894, -0.10389283, 0.09344483, -0.20219979, -0.20113027, 0.19945832, -0.17248249, 0.07845841, 0.19814597, 0.06085683, -0.02476168, -0.18729973, 0.0674377, 0.1058675, -0.04973035, -0.10124692, 0.12834774, 0.0900365, -0.17527112, 0.09901146, -0.05764115, 0.03708475, 0.13003437, 0.2325445, 0.19712777, -0.07403741, -0.05580256, 0.18192469, -0.19122703, -0.05592606, 0.09128745, -0.15161441, 0.08734331, -0.0963953, -0.0377482, 0.18287936, 0.16522603, 0.03490613, -0.13519889, 0.08505893, 0.02470117, -0.06158587],
+        "fpext":[0.1312062, -0.0272035, 0.1562131, 0.11956131, -0.18370572, 0.16904335, -0.18538451, -0.0905571, -0.05687085, -0.12067703, 0.15717801, 0.06977441, 0.04604319, -0.18692835, 0.07965986, 0.03840762, 0.17120989, 0.03048515, -0.02593113, -0.02722386, 0.0637762, -0.08012746, 0.03968937, -0.09751038, -0.04150679, -0.05797326, 0.09053179, -0.06100635, 0.01466092, 0.1521789, 0.0178679, 0.00468684, 0.09077137, 0.22244066, -0.21890686, 0.09471557, -0.18849488, 0.06589299, 0.20329474, -0.20006016, 0.06712949, 0.19578859, -0.00594554, -0.03142307, -0.18011327, 0.00516408, 0.18754548, -0.18884791, 0.2184627, 0.09512166, 0.07513344, 0.12641825, -0.0237517, -0.06020509, 0.15404698, -0.02384854, 0.22717056, 0.11279562, 0.16759154, -0.09342691, -0.09703359, -0.19393681, -0.06580188, 0.190172, -0.21666776, 0.09756065, -0.06633368, -0.04663929, 0.16539453, 0.16365078, 0.04338961, -0.1387515, 0.07042016, 0.03850469, -0.06256048],
+        "fptosi":[-0.09471355, -0.01413281, 0.20098655, 0.17478812, -0.0103814, 0.11870264, -0.10561193, 0.19406088, -0.04215302, 0.09856935, -0.05858651, 0.11069191, -0.02667271, -0.17638545, 0.00591833, 0.12538631, 0.1609231, 0.143028, 0.00411847, 0.06113226, -0.1577999, -0.12172183, 0.18763137, -0.04490714, 0.0262733, 0.12005143, 0.08305117, -0.00109107, 0.03972085, 0.13256785, 0.12486281, -0.00920427, 0.19794717, 0.13488762, -0.08073806, 0.11790548, -0.1865304, -0.18761554, 0.15602915, -0.1440169, -0.01538679, 0.18490645, 0.05900477, -0.01250085, -0.17872328, 0.07583775, 0.09906318, -0.16909951, -0.12062778, 0.03089247, 0.11974704, 0.23940024, -0.00561649, 0.03862159, 0.13437317, -0.0128777, 0.2149731, 0.07683202, -0.10922348, -0.04304254, -0.20104879, -0.16371554, -0.08326109, -0.08942039, -0.03989649, 0.07596397, 0.18141732, 0.08706582, -0.06605583, 0.09984156, -0.05919264, -0.13918152, -0.11102735, -0.04074531, -0.05989955],
+        "fptoui":[0.13186027, -0.02009563, 0.05399049, 0.22092278, 0.0764356, -0.12116565, -0.10575569, -0.08263665, -0.05279351, 0.00392524, -0.06554782, 0.15926358, 0.01012308, -0.18726377, 0.02134197, 0.12730621, -0.05603928, 0.1433069, -0.21253656, -0.1455532, 0.23767456, -0.13623604, 0.21878564, -0.05533312, -0.02895407, 0.03911315, -0.2773476, -0.00738798, 0.08483762, -0.14270262, 0.04603437, 0.21681121, -0.20425414, 0.12532364, -0.04144387, -0.1759266, 0.06860236, -0.14378744, 0.16322674, -0.09162073, -0.04966446, 0.1867569, 0.11250684, 0.04158355, -0.20126882, 0.0963328, 0.06281191, -0.17627244, -0.16096021, 0.00596877, 0.12892367, 0.2650723, 0.0032763, -0.05754196, 0.14840715, -0.09938947, 0.13039768, 0.04905574, -0.04759435, -0.18545668, -0.22984591, -0.22227132, -0.02666637, -0.09771203, -0.15587328, 0.07147411, 0.21007161, 0.09932306, -0.06597851, 0.04478595, -0.00686691, -0.14754876, -0.16366987, -0.22182341, -0.05699158],
+        "fptrunc":[0.1180348, -0.01865118, 0.19759397, 0.00099745, -0.20069243, 0.15863162, -0.10404988, -0.03822596, -0.04113388, 0.02389156, -0.15024376, 0.10156121, 0.01125149, -0.17805247, -0.01032893, 0.11541142, 0.16151612, 0.08652765, -0.01421094, -0.02742836, 0.03890061, 0.16992694, -0.19120258, -0.08296357, 0.16136265, 0.13252915, 0.08675185, -0.05066127, 0.14796199, 0.16224207, 0.07839199, -0.0724185, 0.17859218, 0.20112462, -0.11284997, 0.04566038, -0.19799039, -0.17690767, 0.18034197, -0.14002973, -0.0558032, 0.18783137, -0.14857374, -0.03776158, -0.21491641, 0.04907086, 0.10290487, -0.17882703, -0.06566726, 0.09571292, -0.1965507, 0.1673165, -0.03019422, 0.04259395, 0.10474471, -0.01224911, 0.2133804, 0.08484804, -0.01221132, -0.04942748, -0.06522352, -0.16705054, -0.07432696, -0.01365143, -0.14369024, 0.09237107, -0.06833915, -0.04292256, 0.10805372, 0.10829192, -0.04953781, -0.16152975, 0.08096681, 0.12343118, -0.04967185],
+        "freeze":[-0.10947239, 0.16156663, 0.0653074, -0.16102187, 0.05866997, -0.11313032, 0.18013522, 0.00849657, -0.13109621, 0.00250287, 0.12335391, -0.19474623, 0.00160397, 0.18526912, -0.02094408, 0.10131565, -0.17163642, 0.1624736, 0.20699912, 0.1198829, -0.15380894, -0.13946483, -0.0666218, -0.02034315, 0.06200019, 0.04227962, 0.07729523, 0.1244237, -0.07141764, 0.12154905, 0.0300131, 0.01618693, 0.13966976, 0.08918943, -0.07834358, 0.19349128, -0.06925047, -0.11661758, 0.01764905, -0.06631834, -0.19618054, 0.07797705, -0.18398666, -0.08675893, -0.14833811, -0.1468038, 0.18818453, -0.06729261, -0.00497526, -0.08958972, 0.16964634, -0.10325264, -0.10212341, -0.04745837, 0.12241088, -0.03629779, -0.00606445, 0.01766711, 0.0036858, -0.10732682, -0.13172524, -0.10182981, -0.06431175, -0.13082966, 0.04738441, 0.02668242, 0.15197244, -0.12880892, -0.06579075, 0.03165649, -0.00145413, -0.02683991, 0.13963282, -0.14597963, -0.05104417],
+        "fsub":[-0.15878558, -0.01853204, 0.07496438, 0.13546987, -0.1904581, 0.16564278, -0.1120265, -0.05418859, -0.08075086, 0.1190263, -0.030006, 0.07342444, 0.01767328, -0.18650489, 0.04966565, 0.17190282, 0.1616953, 0.10582477, -0.00931773, 0.00157398, 0.00862398, 0.08480153, 0.17171694, -0.0916861, 0.17196383, 0.03703162, 0.11399966, -0.12855959, 0.05810672, 0.15144408, 0.05618297, -0.00800864, 0.20814295, 0.20056027, -0.1153914, 0.10849892, -0.1970011, -0.1813781, 0.1921156, -0.16984123, 0.07333071, 0.19433162, -0.01313439, -0.02830642, -0.17687869, 0.05937865, 0.1070236, -0.18605019, -0.07227098, 0.10597191, 0.1225975, 0.12388534, -0.02131494, -0.02420847, 0.00146665, -0.01353394, 0.22151577, 0.19731705, -0.07575841, 0.08710405, -0.07411117, -0.17442936, -0.1831972, -0.0886184, -0.17084508, 0.10038229, -0.06385624, -0.04644644, 0.10368951, 0.12130242, 0.24595715, -0.14661786, 0.09260331, 0.0455667, -0.05950425],
+        "getelementptr":[0.09870283, 0.07089636, 0.07129486, 0.13141522, 0.0512748, -0.12164738, -0.10822044, -0.06830852, -0.03589667, 0.23281692, 0.00221914, 0.07347875, -0.14414017, -0.1762956, 0.2280886, 0.09720317, -0.02157139, 0.1186724, 0.16809724, 0.01354192, 0.00843806, -0.1289462, -0.06982945, -0.03921828, 0.11689314, -0.27946517, -0.06115843, 0.0435549, -0.04346889, 0.13182928, -0.13756175, -0.00165133, 0.14864644, 0.1256485, -0.10831792, 0.1638358, 0.05359713, -0.12713233, 0.15040766, -0.14046451, 0.06688947, 0.17097251, 0.11004995, 0.00303981, -0.17120391, -0.15561967, -0.0996208, -0.07566627, -0.0092614, 0.03340579, 0.12288038, 0.11154807, -0.01677553, 0.161112, 0.01968472, -0.01360116, 0.05185669, 0.03974737, 0.08288419, -0.11881893, 0.09566113, -0.14562541, 0.27193576, -0.08737413, -0.13493136, 0.06975145, -0.01905861, -0.04629398, -0.06382514, 0.03648283, 0.32102475, -0.07868497, 0.11246872, 0.06408555, 0.21986632],
+        "icmp":[0.10752141, 0.08221146, 0.07407488, 0.13139327, -0.17970721, 0.02754223, -0.13538423, -0.06698897, -0.02524848, 0.29738554, 0.00894943, 0.07728788, 0.01191065, 0.07896648, 0.26208735, 0.10498706, -0.03250087, 0.1684631, 0.23128033, 0.00553169, 0.05936556, -0.15531996, -0.06056314, -0.03261358, 0.09216364, -0.31626984, -0.07434817, 0.12870958, -0.08664963, 0.14354071, 0.2156426, -0.0043262, 0.16025512, 0.13890502, -0.153901, 0.12319393, 0.01295959, -0.14369431, 0.17033054, -0.12931693, 0.07678536, 0.19203088, 0.11493351, -0.00601692, -0.18671934, -0.16845967, 0.11059918, -0.08381938, -0.00690406, 0.04286021, 0.13590424, 0.12681794, -0.01967513, -0.21233313, 0.02998303, -0.01612004, 0.17794448, 0.02153118, 0.09134272, -0.12414944, 0.08073039, -0.17339677, -0.0417021, -0.1026428, -0.0270442, 0.02959586, -0.05658696, -0.05483485, -0.07289126, 0.036765, 0.20820136, -0.0922353, 0.1438483, 0.06928346, -0.37188163],
+        "insertelement":[-0.08810953, 0.19707826, 0.23040843, 0.15133125, -0.22085261, 0.18197243, -0.13767323, 0.00611759, -0.03750934, 0.05303819, -0.00140541, 0.13628025, 0.01222425, -0.19765897, 0.02197562, 0.17957552, 0.17369562, 0.13818526, -0.01018569, 0.07335348, -0.00122391, -0.05268616, 0.03798695, -0.07208619, 0.13593383, -0.08763747, 0.10625125, 0.12024429, -0.01286297, 0.16619073, 0.03609107, 0.00169814, 0.21330379, 0.22023544, -0.11971844, 0.23055589, -0.21922874, -0.18983133, 0.21032487, -0.19545083, 0.39695147, 0.2116961, 0.03465794, -0.01642283, -0.23929499, 0.05254361, 0.1186865, -0.20114459, -0.01001151, 0.06816892, 0.13270052, 0.13516793, -0.02193176, 0.13708963, 0.15222688, -0.01366894, 0.22591114, 0.07127486, -0.08461929, -0.08140475, 0.06961198, -0.18172547, 0.03001983, -0.09530003, -0.03038635, 0.09877217, -0.06264398, -0.06664777, 0.08449566, 0.13536829, 0.04701871, -0.1011354, 0.08900025, 0.0484806, -0.0840841],
+        "insertvalue":[-0.18927452, 0.0174884, -0.14182499, 0.13622768, 0.0860026, 0.09214608, 0.19103362, -0.21386355, -0.08520557, 0.0039752, 0.11396001, -0.20266375, 0.17875974, -0.19441572, 0.06652557, -0.1076987, -0.15872131, -0.20631677, -0.06141828, 0.01331364, 0.02354101, 0.08883859, 0.18617646, -0.1524484, -0.00485444, -0.06658852, -0.06168855, -0.15898797, 0.02933339, -0.19443747, -0.08171376, -0.10284599, -0.14775087, 0.20585667, -0.03085143, -0.20518455, 0.05644068, 0.18632361, -0.14686832, 0.03718346, 0.10764548, -0.01129938, 0.19590198, 0.02194332, -0.04761516, 0.0276675, 0.01006511, -0.05134218, -0.10854474, 0.23528893, 0.09621219, -0.17192629, 0.10312149, -0.14351319, 0.15347119, 0.19321586, -0.20537563, 0.19100618, 0.11280359, 0.07242113, 0.18901348, 0.19133772, 0.02062779, -0.07726028, 0.00578185, -0.02625765, -0.10374335, -0.02086248, 0.18915856, 0.00862997, 0.04699488, -0.0631949, 0.18942626, 0.01894571, 0.02351614],
+        "inttoptr":[0.04721976, 0.16880296, 0.06448112, -0.17660472, 0.02998716, -0.1156636, 0.13852204, -0.06320232, -0.03162635, -0.06644673, -0.136013, -0.06916367, -0.02801188, 0.10646518, 0.05828808, 0.03782522, -0.02227561, 0.02234134, 0.16369314, -0.08557447, 0.03853939, -0.14319475, -0.05599201, -0.01060566, -0.1136315, 0.0201032, 0.01836221, -0.05016, -0.1216938, 0.12626994, 0.03450989, -0.19574732, 0.14352062, -0.17880194, -0.13339539, -0.04630446, 0.03136644, -0.13006681, 0.1445991, -0.09556036, 0.09764113, 0.16694036, 0.09833785, 0.05145761, 0.17970908, -0.14575475, -0.13704927, -0.06876626, -0.00955433, -0.16089469, -0.14152728, 0.1012365, -0.18291287, -0.07371435, -0.13809073, 0.02210419, 0.090786, 0.02507522, 0.15311833, -0.11663677, 0.10619268, 0.1758742, -0.04052401, 0.19525726, 0.15682434, -0.19402866, 0.08792165, 0.19677114, 0.1986662, -0.2065629, 0.04007441, -0.01697999, 0.17015004, 0.07224851, -0.06042317],
+        "invoke":[0.11604629, -0.02348489, -0.11863736, -0.16840768, -0.01534824, -0.13232489, -0.09663513, -0.1576129, -0.2348036, 0.00802987, -0.0517984, -0.05321365, 0.01174416, -0.11655734, 0.05259984, -0.16816719, -0.09482966, -0.00228002, -0.12152751, -0.00498575, 0.03465938, -0.02950593, 0.04545004, -0.2172498, -0.07073424, -0.08918925, 0.0623108, -0.20881031, -0.04182759, -0.20031597, -0.1351732, 0.069332, -0.00361818, 0.11065516, 0.13967985, -0.08796342, -0.00256544, -0.11508698, 0.09178918, 0.2073513, 0.04010937, -0.07716485, 0.06940974, 0.20694868, -0.08160415, 0.1820004, -0.12159622, 0.04531517, -0.00216078, -0.11625812, -0.10868325, 0.11549117, 0.19036727, 0.0365869, -0.09580094, 0.22367325, 0.11756624, -0.2184826, 0.27060437, 0.14232084, 0.01686368, -0.10716032, 0.02151413, -0.02634864, 0.19014603, -0.22087607, -0.05267305, -0.2422757, -0.06234193, -0.0299236, 0.01296355, 0.0901266, 0.09916983, -0.00811885, -0.02482844],
+        "landingpad":[0.12086448, 0.16615215, -0.11424457, -0.08484724, -0.09069939, -0.15767984, 0.10221493, -0.17647022, -0.18533581, -0.02190816, 0.1541339, -0.15779817, 0.03144602, -0.10866319, -0.01497394, -0.1721984, -0.07559271, -0.11889227, -0.1177194, -0.02792813, 0.04734257, -0.03713308, -0.17093344, 0.17252314, 0.02917121, 0.01523044, -0.08545186, -0.20184867, -0.08280238, -0.04850229, 0.03705949, 0.07665683, 0.01846915, 0.00205453, 0.1709319, -0.10548064, -0.07096241, 0.12575251, -0.1177155, 0.17698564, 0.03965896, -0.10821396, 0.07917166, 0.17810985, 0.01715738, 0.16967225, -0.16826203, 0.16891196, -0.15257767, 0.10498544, -0.10490264, 0.06432237, 0.18670914, -0.00099873, -0.13853344, 0.21604276, 0.13351974, -0.15664832, -0.08585881, 0.1024329, 0.02737338, 0.11940238, -0.01797596, -0.00314033, 0.12023867, 0.11616865, 0.05652065, 0.1415307, -0.15827312, -0.03440882, 0.02146849, 0.01478402, -0.13532415, -0.02099262, 0.00462938],
+        "load":[0.06383128, 0.0803676, 0.07070765, 0.1284488, 0.053152, -0.11878661, -0.09626942, -0.05779593, -0.04309646, -0.31707463, -0.01623556, 0.0741919, -0.0051238, 0.07403108, -0.25466156, 0.09440999, -0.01665624, 0.14732535, 0.17144501, -0.00878042, 0.04688795, -0.15015227, -0.06462591, -0.04178058, -0.0378243, 0.33024567, -0.07477789, -0.04902143, -0.0476392, 0.13162148, -0.17537315, -0.01750547, 0.14473993, 0.1732234, -0.1423584, 0.11269526, 0.03678195, -0.1315726, 0.15341991, -0.10186778, 0.00435855, 0.16657132, 0.17959148, -0.00966169, -0.16982812, -0.02318787, 0.09485581, -0.07465204, -0.06222337, 0.01743078, 0.12776802, 0.09864764, -0.01487866, 0.03529362, 0.13232034, -0.01328148, 0.09440556, 0.06241368, 0.03618008, -0.11342508, 0.09918994, -0.15651965, -0.07195813, -0.09057574, -0.10321549, 0.06875013, -0.00129774, -0.04545839, -0.07164564, 0.05073486, 0.34142092, -0.08255292, 0.12089149, 0.08211686, 0.21363957],
+        "lshr":[0.1238757, -0.02033522, 0.05549098, 0.11646183, 0.08761991, -0.12349915, -0.18941326, -0.08920491, 0.09122052, 0.03471606, 0.2897465, 0.07269037, 0.01238082, 0.07473343, 0.02699653, 0.08085709, -0.04658483, 0.2076412, 0.22355735, -0.03676752, 0.18093167, -0.15845503, -0.0710993, -0.02861892, -0.24865082, -0.00123379, -0.23955399, 0.13371961, -0.11877617, 0.07246234, 0.01312172, 0.01923358, -0.04966315, 0.1200467, -0.2137276, 0.10946998, 0.07409713, -0.12598129, 0.02809012, -0.09355168, 0.06742698, 0.08053191, 0.18937954, 0.00296521, 0.10038757, -0.19544551, 0.08368544, -0.07828917, 0.20687084, 0.01422358, 0.19612609, 0.03323235, -0.01736557, -0.2952406, 0.15087438, -0.07250606, 0.01719518, -0.01060284, 0.16445734, -0.16287296, 0.07179926, -0.14751296, 0.02352273, -0.17620452, 0.12871586, 0.01027796, -0.00240841, -0.08637159, -0.07440444, 0.02873684, 0.05644969, -0.07749362, 0.17137784, 0.09177881, -0.06086624],
+        "mul":[9.14543942e-02, 7.25680292e-02, 1.87143609e-01, 1.56122670e-01, 8.18791837e-02, -1.13938227e-01, -1.17102735e-01, 1.78440675e-01, -5.71394851e-03, 3.98688577e-02, -1.22035742e-02, 9.62941721e-02, 9.87940375e-03, 6.54331893e-02, -1.18242744e-02, 1.01057030e-01, -3.37237865e-02, 1.82104349e-01, 1.98254153e-01, 4.89455797e-02, -1.38579145e-01, -1.43827185e-01, -7.30315745e-02, -2.81286202e-02, -3.69295813e-02, -7.32784346e-02, -7.52561465e-02, 1.15158796e-01, -8.05188417e-02, 1.22851163e-01, 1.18959583e-01, -6.19116612e-03, 1.14142060e-01, 1.15443885e-01, -1.32043362e-01, 1.91231743e-01, 6.71850219e-02, -1.18121214e-01, 1.39134541e-01, -9.38180089e-02, 7.00822175e-02, 1.52467072e-01, 1.72531351e-01, 2.39370289e-04, 8.84519666e-02, -1.74602866e-01, 7.58597329e-02, -7.24562407e-02, 2.30801646e-02, 2.31582299e-02, 1.77945897e-01, 2.19754413e-01, -1.05254715e-02, -5.34405783e-02, 1.28701046e-01, -8.88467114e-03, 2.39838846e-02, -2.52107698e-02, 4.09773886e-02, -1.43636853e-01, 9.37281698e-02, -1.36706889e-01, 3.18117648e-01, -1.72676995e-01, 9.86104552e-03, 1.59371197e-02, -2.31163911e-04, -7.84823969e-02, -7.37200677e-02, 3.27852108e-02, 1.85986701e-02, -7.93214887e-02, 1.19079717e-01, 9.70832258e-02, -5.25254011e-02],
+        "or":[-0.15830125, 0.08118854, 0.06019896, 0.1351512, 0.0696514, -0.12829824, -0.12040509, -0.07483122, -0.05365232, -0.21453764, 0.01100464, 0.07939477, 0.01512126, 0.07680329, 0.20671816, 0.10159217, -0.03416578, 0.18844754, 0.2232867, 0.00390219, 0.17268041, -0.16189471, -0.07818128, -0.02745846, -0.03785938, -0.28434664, -0.23963155, 0.13414037, -0.03098747, 0.1290441, -0.04272285, 0.00627589, 0.15120292, 0.13505426, -0.14509544, 0.1633035, 0.07651295, -0.1345261, 0.15650204, -0.10334036, 0.08041331, 0.17568573, 0.18555732, -0.00240171, 0.09882613, -0.1697568, 0.08491044, -0.07938255, -0.00502023, 0.04242367, 0.19246665, 0.10571583, -0.01717717, -0.0940207, 0.19319834, -0.06771892, 0.03723063, 0.00987004, 0.08289062, -0.14470372, 0.07547856, -0.14796427, 0.03079064, -0.18788183, -0.06784894, 0.02374648, 0.00680319, -0.08787614, -0.0762485, 0.04015802, -0.31986576, -0.08643831, 0.14352895, 0.1116055, -0.09564826],
+        "phi":[-0.17533, -0.14122052, 0.25121832, -0.20197222, -0.02265841, 0.08878156, -0.12254399, -0.05334119, 0.17813319, 0.25582585, -0.24652013, 0.08741698, -0.20831233, -0.12509026, -0.19707216, 0.15587404, -0.07037015, 0.00634299, 0.02692973, 0.20398706, 0.09076547, 0.2106457, 0.02665563, -0.01256549, 0.20547326, -0.01408568, -0.11248078, 0.17035946, 0.00720961, -0.24428545, -0.19424082, 0.02588499, -0.09168262, 0.12671804, -0.13895495, 0.11016023, 0.06885135, -0.1585114, 0.1442796, -0.09209647, -0.25246575, -0.18171434, 0.07803512, 0.01651475, -0.05719611, -0.18895395, 0.13336438, -0.09516547, 0.04140934, -0.13787958, 0.14149408, -0.15089944, -0.02157312, 0.2192603, 0.02856141, -0.05521443, 0.0077216, 0.2731221, -0.09575493, 0.2032519, -0.12885517, -0.18518618, 0.16917716, -0.11144329, 0.25175345, -0.30578047, -0.07883886, 0.14204925, 0.26059705, 0.01635692, -0.2599762, 0.1671248, 0.07727495, 0.06118969, 0.27382395],
+        "ptrtoint":[0.13572109, 0.07109733, -0.01410782, 0.13761328, 0.06249858, -0.12742682, -0.03786812, -0.08852433, -0.00890332, -0.12647949, -0.23919484, 0.07803643, -0.29866624, 0.07854372, 0.0690124, 0.08534926, -0.03230691, -0.11528994, 0.2259676, -0.09829737, 0.10256737, -0.16430771, -0.0642257, -0.01210832, -0.12542038, 0.01087978, -0.2651048, 0.10121775, -0.12749597, -0.00404411, 0.08437401, -0.01521534, -0.05364794, -0.20650637, -0.15205501, 0.16177334, 0.08448336, -0.13946855, 0.15794401, -0.08243193, -0.05839634, 0.17580664, 0.19553919, 0.02601502, 0.14116347, -0.17160966, -0.09532923, -0.07157483, 0.20540062, -0.00341362, 0.20416199, 0.09911527, -0.01078249, -0.0624228, 0.14551571, 0.07151687, 0.04763724, -0.02011446, 0.16194499, -0.18160547, 0.13115011, 0.01976459, -0.01537652, -0.19834353, -0.14665273, -0.20727639, 0.03673945, -0.09134132, -0.15628532, -0.04004635, -0.02747078, 0.14147465, 0.17462312, 0.11279432, -0.07561264],
+        "resume":[0.16280797, -0.14593223, -0.12683636, -0.1562535, -0.03300981, -0.13968605, 0.16718441, -0.19515003, 0.01863662, -0.05534162, -0.01442544, 0.0425216, 0.0093952, -0.16576275, 0.03578432, -0.15065056, -0.06406866, -0.16523188, -0.05909818, 0.06044476, -0.0871481, 0.15487999, -0.18122765, 0.1956721, -0.04706432, 0.08488349, 0.04219154, 0.02599989, -0.02276746, -0.18513387, 0.12803178, -0.09826294, -0.11191379, 0.04426758, 0.11935913, -0.14695078, -0.00907954, 0.15606298, -0.13066523, 0.01243626, 0.07249606, -0.13709828, 0.10189614, 0.18291259, -0.04105699, 0.1736963, -0.12169949, 0.09442408, 0.20398152, 0.22682573, -0.08286698, -0.13903227, 0.09133334, 0.03095248, 0.11325783, 0.1569758, -0.17479569, -0.12558746, -0.12676957, -0.12449513, 0.17823087, 0.16940698, -0.06061083, -0.1153774, 0.11211671, 0.03377679, -0.05404395, 0.10726969, -0.120761, 0.01132012, 0.00449077, 0.00602859, -0.13664234, 0.00466423, -0.02339442],
+        "ret":[0.14368142, 0.0712266, 0.05669672, 0.13478386, -0.14595962, -0.12663847, -0.12403987, -0.09207936, 0.00078773, -0.20975904, 0.08747889, 0.07683876, -0.14989862, -0.18156855, 0.38113984, 0.05973466, -0.05483145, 0.00884139, -0.013859, -0.03841434, 0.05703019, -0.04955313, 0.04577729, -0.05112582, -0.25303054, 0.13709044, -0.06985793, -0.04175208, -0.06085019, 0.12376258, 0.08259401, -0.01146284, 0.05020906, 0.18525098, -0.15137392, 0.07154816, 0.06121736, -0.14056544, 0.16508178, -0.12329083, 0.22687763, 0.17450114, 0.18714464, 0.01847176, -0.02590629, -0.15461633, 0.0910584, -0.07975882, 0.03339317, 0.08106077, 0.10348453, 0.12415495, -0.01464526, -0.20166221, 0.23424618, -0.0058443, 0.13251036, -0.01039419, 0.19171973, -0.14341427, 0.11032798, -0.17445254, -0.21265155, -0.11035369, 0.09180369, -0.01410731, -0.05983369, -0.0549278, 0.11081824, 0.02046819, -0.04516792, -0.09715447, 0.14870624, 0.05914663, 0.10917712],
+        "sdiv":[7.54674971e-02, 2.70023555e-01, 2.15832815e-01, 1.51102781e-01, 3.86169888e-02, -1.27587810e-01, -1.64553642e-01, 2.08858803e-01, 2.49577500e-03, 3.39335836e-02, 2.95089424e-01, 7.44504854e-02, 1.75000988e-02, 7.48216212e-02, 4.57024798e-02, 1.14481322e-01, -5.15110120e-02, 1.97094589e-01, 2.28497684e-01, 1.89727977e-01, -2.03567356e-01, -1.53853878e-01, -7.04708546e-02, -2.87290998e-02, -4.15551029e-02, -8.37634057e-02, 2.48096928e-01, 1.39052629e-01, -1.18507713e-01, 1.37168303e-01, 4.37297896e-02, -2.56327417e-04, 1.03771754e-01, 1.18692242e-01, -1.97971940e-01, 2.48502731e-01, 5.65373003e-02, -8.69503245e-02, -9.03292373e-03, -2.25797057e-01, 7.20400363e-02, 4.09315303e-02, 1.83778912e-01, -2.47538835e-03, -2.83727050e-02, -2.00868785e-01, 8.99481177e-02, -7.83472061e-02, 4.22467515e-02, 2.67557669e-02, 1.77338138e-01, -1.42202228e-02, -1.59657858e-02, -6.30925298e-02, 3.13569121e-02, -2.22823601e-02, 6.28621271e-03, -1.85963176e-02, 1.17826000e-01, -1.24646917e-01, 1.02576114e-01, -1.52709767e-01, 2.57164501e-02, -1.40907198e-01, 1.74567476e-01, 1.61048099e-02, -5.03884954e-03, -8.63459408e-02, -7.52566457e-02, 3.38615589e-02, 1.36013612e-01, -8.77554864e-02, 1.03453219e-01, 7.68808499e-02, -6.49084374e-02],
+        "select":[0.05093412, 0.07767349, 0.07669216, 0.1312176, 0.0579063, 0.1616888, -0.11760561, 0.00115628, -0.04131674, -0.29780418, -0.01691337, 0.0748211, 0.0864482, 0.07403006, 0.25878662, 0.10279866, -0.01487916, 0.15694329, 0.22207598, 0.01221279, 0.02520609, -0.1521685, -0.06417656, -0.03086487, 0.12268542, 0.3565569, -0.08125142, 0.1072576, -0.02930279, 0.14090922, -0.14309497, -0.00379189, 0.15369308, 0.12072479, -0.11386253, 0.17081785, 0.0359251, -0.13158213, 0.15140608, -0.08365479, 0.04333765, 0.17864218, 0.09735203, -0.01021584, -0.1817566, -0.17009453, 0.10480749, -0.07720621, -0.06324099, 0.01645014, 0.13028447, 0.10660475, -0.02038397, -0.05975712, 0.02713404, -0.01552912, 0.07295278, 0.05679769, 0.02740115, -0.11240587, 0.07162624, -0.15834542, -0.32627442, -0.08725446, -0.07952213, 0.07846557, -0.01865849, -0.04569499, -0.0728619, 0.0374458, -0.27716625, -0.07809193, 0.11488006, 0.06656368, -0.06777868],
+        "sext":[-0.03718835, 0.08533189, 0.21354397, 0.13909821, 0.04430787, -0.13043992, -0.12707381, 0.14811471, -0.01397946, -0.24312808, -0.01538438, 0.08007284, -0.20212536, 0.07967917, 0.16868502, 0.10560705, -0.01859606, 0.16425593, 0.23104599, 0.04822871, -0.15584451, -0.16572693, -0.07367796, -0.02386475, -0.04161641, -0.28961626, 0.26639512, 0.13398732, -0.09589424, 0.14068675, 0.16089684, -0.01488031, 0.16961882, 0.11251253, -0.15109386, 0.20157905, 0.04388804, -0.14243662, 0.17028852, -0.17495492, 0.08312923, 0.18138759, 0.19254005, -0.00167047, 0.1016985, -0.17693631, 0.09644604, -0.07967581, -0.01462245, 0.00063264, 0.19925497, 0.10426001, -0.01645718, -0.0897459, 0.02190982, -0.00871549, 0.03981987, 0.01151098, 0.07275884, -0.13166158, 0.11194995, -0.15796044, -0.02378252, -0.17802656, -0.05159309, 0.03235529, 0.00206553, -0.0869719, -0.07645953, 0.05517219, 0.01411109, -0.08648386, 0.1488923, 0.11236069, -0.14168048],
+        "shl":[-1.21411718e-01, 8.12682733e-02, 7.26464316e-02, 1.38484403e-01, 8.17928985e-02, -1.24320365e-01, -1.29878476e-01, 1.89510286e-01, -6.82524173e-03, 4.69798781e-02, 7.76179358e-02, 7.87331015e-02, -1.99450791e-01, 7.40546957e-02, 1.56282917e-01, 1.49098694e-01, -3.80089432e-02, 1.97095275e-01, 2.20074877e-01, 9.85120013e-02, -1.51519790e-01, -1.56306505e-01, -6.62787706e-02, -2.89262943e-02, -8.50795880e-02, -3.77611667e-01, -8.08443353e-02, 1.29505768e-01, -9.56492350e-02, 1.34987608e-01, 1.17273629e-01, -8.01561028e-03, 2.01825440e-01, 1.25738055e-01, -1.46153197e-01, 2.18217447e-01, 7.04046711e-02, -1.39466003e-01, 6.33412227e-02, -1.05604395e-01, 7.39852190e-02, 1.70712352e-01, 1.86249197e-01, 1.75760695e-04, 9.54905152e-02, -1.82715610e-01, 8.42701495e-02, -7.70764053e-02, 3.44463770e-04, 2.22381260e-02, 1.92770571e-01, 6.70530349e-02, -8.68750829e-03, -5.87612167e-02, 1.35470673e-01, -9.91571508e-03, 3.54958773e-02, -1.26978569e-02, 3.80189577e-03, -1.57232955e-01, 1.05788536e-01, -1.48979709e-01, -6.13744035e-02, -1.79459959e-01, 1.58743337e-02, 1.66407768e-02, 3.78276110e-02, -8.67255181e-02, -7.59592950e-02, 3.41174081e-02, -1.17456488e-01, -8.46017748e-02, 1.42491326e-01, 8.54357481e-02, -6.05630279e-02],
+        "shufflevector":[-1.64276063e-01, -1.40854018e-02, 2.16989160e-01, 1.48834497e-01, -1.81232363e-01, 1.71089604e-01, -1.51962206e-01, 9.34694987e-03, -6.01843521e-02, 1.57123819e-01, 1.49263054e-01, 1.57006048e-02, 8.54564831e-03, -1.46307275e-01, -9.97107551e-02, 2.03836277e-01, -1.10085299e-02, 1.58312604e-01, 3.98286656e-02, 1.22957818e-01, -4.54752985e-03, -1.51905298e-01, 2.56421398e-02, -9.66223106e-02, 1.35936990e-01, 1.13613196e-01, 8.88981670e-02, 1.39288589e-01, 2.35020537e-02, 1.54158428e-01, 1.38836756e-01, -7.45062644e-05, 1.85534820e-01, 2.31408417e-01, -1.86368376e-01, 2.18553007e-01, -1.25940442e-01, 6.19565435e-02, 2.09610596e-01, -2.28297710e-01, 2.30249181e-01, 1.89029738e-01, -6.89853504e-02, -3.19687016e-02, -1.80848792e-01, 4.24141204e-03, 1.89942136e-01, -1.91565320e-01, -2.52943840e-02, 5.65835461e-02, 1.32085189e-01, 1.23422973e-01, -2.14130040e-02, -2.95120943e-02, 1.43252522e-01, -2.44392790e-02, 1.89641267e-01, 3.69141959e-02, 5.39628556e-03, -1.15757883e-01, 6.42104596e-02, -2.11238548e-01, 1.83569789e-02, -9.20821503e-02, -9.48524103e-02, 8.56385827e-02, -5.02256379e-02, -8.02270398e-02, 8.46109912e-02, 7.74170980e-02, 3.27805057e-02, -1.08585857e-01, 1.16238609e-01, 5.87845854e-02, 1.14851862e-01],
+        "sitofp":[0.05548016, 0.16714495, 0.21498686, 0.14356636, -0.16821964, 0.16527581, -0.15967155, 0.21070401, -0.01250376, -0.11125261, -0.06026782, 0.07956001, -0.13129172, -0.18479359, 0.08652773, 0.08131526, 0.15925482, 0.08148654, -0.00736642, 0.19463325, -0.20742258, -0.11578189, 0.02654804, -0.08633579, -0.05612461, 0.01452674, 0.11379433, 0.03724489, -0.12955493, 0.15072383, -0.0232376, -0.01797887, 0.20284739, 0.20505717, -0.18686672, 0.25398365, -0.11697914, -0.09353511, 0.19472948, -0.18606324, 0.13722429, 0.19458178, 0.03767281, -0.01490554, -0.15784514, -0.0030121, 0.1061831, -0.18870111, -0.04370302, 0.06840985, 0.12218294, 0.133711, -0.01636076, -0.05479606, -0.09695652, -0.01025795, 0.2093504, 0.08853228, -0.07362932, -0.06125485, 0.10777287, -0.17920133, 0.03073885, -0.09303376, -0.18211918, 0.09700941, -0.05694851, -0.0481168, 0.10195274, 0.11961144, 0.14820483, -0.10259698, 0.07829086, 0.06143153, -0.06575584],
+        "srem":[0.12748273, -0.01354557, 0.07656507, 0.13586837, 0.09215222, -0.12957662, -0.22408909, 0.21526249, 0.0899937, 0.03087448, 0.23878594, 0.07816898, 0.0165252, 0.07721527, 0.0565106, 0.08858381, -0.13782543, 0.2185999, 0.2350998, 0.20486632, -0.22521928, -0.16895708, -0.07120457, -0.03142672, -0.193668, -0.06093899, 0.098878, 0.13780232, -0.13592266, 0.13956556, 0.03245056, -0.01644238, 0.1208498, 0.11854354, -0.20298819, 0.27215648, 0.06262159, -0.13207191, -0.22367987, 0.18101601, 0.07210222, -0.035157, 0.19500516, -0.0082911, 0.04821047, -0.2052546, 0.0944626, -0.08101162, 0.05265743, -0.00286297, 0.2012179, -0.02560825, -0.01475919, -0.06332223, -0.21597023, -0.00943558, 0.00891592, -0.04312498, 0.11415429, 0.08646774, 0.10881357, -0.15501237, 0.02134495, -0.15064926, 0.14675559, 0.01146032, 0.0020576, 0.09925999, -0.07733848, 0.03300545, 0.05376984, -0.08198962, 0.17412202, 0.08533774, -0.06574512],
+        "store":[0.09998547, 0.07416256, 0.06472399, 0.12126696, 0.03879403, -0.12036075, -0.11504382, -0.06913467, -0.03769127, -0.33070236, -0.00644616, 0.07156026, -0.02137089, 0.07216891, -0.05470073, 0.09415297, -0.04082007, 0.1056991, -0.0013265, -0.00926145, 0.04998971, -0.14578499, -0.05182505, -0.02615017, 0.08348413, 0.31976137, -0.07019657, -0.05163203, -0.04972167, 0.12658256, 0.04289434, -0.0016057, 0.14311627, 0.12263075, -0.13941614, 0.13760482, 0.04816094, -0.12741822, 0.14878556, -0.11523844, 0.05825484, 0.1649721, 0.09197503, -0.0012878, -0.1629092, -0.1414969, 0.09324066, -0.07395344, -0.00317431, 0.01789259, 0.12094813, 0.10764453, -0.01553114, 0.11882594, 0.02442449, -0.01568008, 0.11022279, 0.03307666, 0.08506706, -0.10825977, 0.06488108, -0.15164001, -0.30984077, -0.08653892, -0.06201477, 0.01810287, -0.04473237, -0.04719033, -0.06153039, 0.03287021, 0.29479045, -0.07162003, 0.13225412, 0.05114779, -0.33301073],
+        "sub":[-0.13264543, 0.07865644, 0.0717167, 0.1342318, 0.05076935, -0.12322044, -0.12266772, 0.00524008, -0.00201612, 0.05315073, 0.00274359, 0.0742974, 0.0123115, 0.07594631, 0.21857306, 0.10421466, -0.02275163, 0.17884067, 0.21873002, 0.00927466, 0.00893607, -0.154156, -0.06820294, -0.02856713, -0.03866831, -0.2793768, -0.06824332, 0.13080892, -0.08803294, 0.13524993, 0.21245252, -0.01117309, 0.20007995, 0.12509634, -0.14541206, 0.19173518, 0.0613077, -0.13841055, 0.15438707, -0.14274515, 0.07382213, 0.17015299, 0.18052574, -0.00206174, -0.15260881, -0.17803495, 0.08986183, -0.07603207, -0.0163452, 0.02869452, 0.186938, 0.07764658, -0.01459899, 0.10072613, 0.02469735, -0.01351501, 0.03415474, 0.0135162, 0.04113324, -0.12325168, 0.11012445, -0.14946231, 0.04155808, -0.1440628, -0.06376708, 0.01838643, 0.00198335, -0.07941123, -0.07066227, 0.0376416, 0.24301553, -0.0862716, 0.1351057, 0.0891021, -0.3352],
+        "switch":[-0.16398937, -0.1353046, 0.08524002, 0.1561178, -0.22488637, -0.14185904, -0.10160676, 0.17193311, -0.05567684, -0.01794963, -0.22830375, 0.08948597, 0.1698675, 0.08543108, -0.18870668, 0.13391507, 0.1591491, 0.21046144, 0.02533695, -0.05875827, -0.15551412, 0.19997434, 0.19250119, -0.03397122, -0.05150964, -0.15923595, 0.0701151, 0.14852029, -0.01560177, 0.15765266, -0.2304326, -0.01416684, -0.05115065, 0.12840182, -0.17300242, -0.03878057, 0.07419615, -0.16339742, 0.18437031, -0.14087205, -0.2714784, -0.17547078, -0.01152283, -0.00396952, 0.11226111, -0.18184493, 0.1166721, -0.09108577, 0.23214185, 0.07350235, 0.14699937, 0.08840676, -0.01411979, 0.2082167, -0.10539865, -0.01378542, 0.04486391, -0.01463741, 0.14504644, 0.19353367, 0.11889812, -0.19718692, -0.04798679, -0.12421229, 0.29124823, -0.32081923, -0.00218598, -0.1037329, -0.08444938, 0.04092982, -0.03067664, -0.1606894, 0.07095047, 0.08663001, 0.25936154],
+        "trunc":[0.11878826, 0.08377917, 0.05513672, 0.14270523, 0.07370453, -0.14921571, -0.16542384, -0.07493053, -0.00377195, -0.20950364, 0.12958477, 0.07888171, -0.00595982, 0.07865481, 0.16519356, 0.10834014, -0.04326443, 0.19355299, 0.23304398, 0.07385943, -0.11257231, -0.16919497, -0.082918, -0.0285878, -0.05886409, -0.25552744, 0.08462781, 0.13503358, -0.09682789, 0.12824969, 0.08024148, -0.01201469, 0.15677036, 0.14443342, -0.1907149, 0.23856063, -0.00528661, -0.14044978, 0.16222948, -0.10748056, 0.08127003, 0.17631039, 0.19577822, -0.01176626, 0.09836395, -0.16917923, 0.10726047, -0.10182234, 0.20931682, 0.04668214, 0.2017859, 0.11306392, -0.018347, -0.06246869, 0.24337131, -0.07117373, 0.03053473, -0.00658053, 0.03026612, -0.16877551, 0.10491424, -0.17555256, 0.14634104, -0.18189284, -0.11786141, 0.02876267, -0.00210879, -0.08721989, -0.07824761, 0.03958495, 0.20243248, -0.09376816, 0.11082744, 0.12096685, -0.12131985],
+        "udiv":[0.08154505, 0.17510523, -0.19362096, 0.08671232, 0.11330891, -0.19157314, -0.2145219, -0.06051004, 0.1371807, 0.04572224, -0.06733968, 0.07146493, 0.00681373, 0.07493957, -0.02997438, 0.08688152, -0.04591569, 0.23223788, 0.22175628, 0.01500544, 0.14273156, -0.16111672, -0.08531193, 0.19467805, -0.00517966, -0.06020893, -0.06077334, 0.13201268, -0.12358853, 0.00868457, 0.06033499, 0.01196855, 0.14756525, 0.11571166, -0.11228922, 0.13696799, 0.21417981, 0.20094916, -0.21627706, 0.18220317, -0.06209835, 0.1076472, 0.20395155, 0.00197578, 0.13577849, -0.20195886, 0.0833538, -0.07554612, 0.00410905, 0.00158071, 0.1938367, 0.02883399, -0.01710817, 0.03711469, 0.13843457, -0.06804645, -0.00379174, -0.00144552, 0.13146468, -0.18377277, 0.07351629, -0.14530504, 0.01042824, -0.1766743, 0.01462858, 0.01206683, -0.12541275, 0.09634796, -0.07832152, 0.02791307, 0.02765011, 0.19757621, 0.16267568, 0.12346987, -0.0413983],
+        "uitofp":[-0.15078814, 0.1826288, 0.11165173, 0.13953921, -0.1842304, 0.16399114, -0.11256697, -0.05694728, -0.04820754, -0.17459138, -0.12022546, 0.08202647, 0.05924561, -0.17365019, 0.1614662, 0.07272313, 0.14744736, 0.09281041, -0.00596566, -0.01817355, 0.09689283, -0.0900236, 0.03450679, -0.0848825, 0.17167987, -0.07416291, 0.09088556, 0.07786386, 0.08914284, 0.14832556, -0.07537761, 0.00234627, 0.15249911, 0.19451788, -0.15017514, 0.14780688, 0.11764529, -0.02274906, 0.1920297, -0.17293543, 0.22891366, 0.1852529, 0.0128901, -0.00331702, -0.18161494, -0.0123833, 0.10250985, -0.1877487, -0.02335341, 0.06747307, 0.12578565, 0.12541758, -0.01838141, -0.08433985, 0.26602918, -0.01791255, 0.18907419, 0.04615575, 0.22383852, -0.13418758, 0.07736338, -0.18702815, 0.06279426, -0.09547485, -0.14973089, 0.09295561, -0.04817062, -0.05371717, -0.01254407, 0.12052637, 0.14356695, -0.1381116, 0.17813013, 0.0863996, -0.10018482],
+        "unreachable":[1.6576199e-01, -1.7907834e-02, -5.7712890e-02, 9.4339609e-02, -9.3380272e-02, -1.9464681e-01, 7.7050277e-03, -1.9517663e-01, 2.6435368e-03, 9.1913566e-02, 1.3639191e-02, 7.0288680e-02, -3.2517979e-01, 7.4262276e-02, -8.9507200e-02, -1.7191246e-01, -5.2555885e-02, -1.4688279e-01, 1.0220077e-01, -4.8814032e-02, -1.3367063e-01, -6.1610699e-02, -1.8777388e-01, 2.0192388e-01, 1.7102915e-01, 1.3610463e-01, -2.6177135e-01, -3.3939917e-02, -5.4787207e-02, 1.1302987e-01, 1.0523112e-01, 8.6424664e-02, -3.6340270e-02, 4.7679577e-02, 1.0370419e-02, 3.9357133e-03, 1.5754176e-02, -4.6523277e-02, -1.2651871e-01, -5.0250064e-03, -2.1445632e-01, -1.5226135e-01, 1.7948903e-01, 1.9091401e-01, 1.1402745e-01, -1.9978039e-01, -8.4826022e-02, -3.7182060e-03, 2.1397528e-01, -1.9490053e-03, 3.8303059e-02, 5.5778958e-02, -1.1436568e-02, 7.7554718e-02, 3.7896037e-02, 7.5804755e-02, -1.7246161e-02, -1.4188807e-01, 4.9074650e-02, -1.9401214e-01, 1.9030276e-01, 1.1096356e-01, -1.5253071e-01, -1.9243342e-01, 1.6415399e-01, 2.7355237e-02, 7.6516271e-03, -9.3317330e-02, -1.7084104e-01, 3.9767768e-02, -1.1136789e-01, -9.4884530e-02, 2.1568926e-04, 1.3515340e-01, 8.2443848e-02],
+        "urem":[0.1196936, 0.18855006, -0.20257552, 0.01738752, 0.15161008, -0.14784957, -0.02992235, -0.0674265, 0.14330001, -0.00208072, -0.08096982, -0.06099883, 0.01042207, 0.1028357, 0.05679855, 0.08465453, -0.05227959, 0.21925992, 0.22556421, -0.059273, 0.1825322, 0.17631407, -0.08938511, -0.0277707, -0.2572769, -0.06195913, -0.24406035, 0.13713413, -0.13801506, 0.02009896, 0.02613169, 0.01552499, 0.14668229, 0.12454724, -0.14865053, 0.09922966, 0.2130675, -0.11827406, 0.01006878, -0.09671234, 0.06660412, 0.0945213, 0.20740281, 0.00239918, 0.14173026, -0.1632753, 0.08587474, -0.07923763, 0.02992029, 0.2015828, 0.2007573, -0.11056472, -0.01871637, -0.06330916, 0.14348209, -0.07483656, 0.00636646, -0.01837422, 0.17499712, -0.17495239, 0.07653105, -0.1475833, 0.02116938, -0.17941074, 0.1734715, -0.01114943, 0.03780977, -0.15019819, -0.07816306, -0.08282368, 0.05129486, 0.19702028, 0.18443331, 0.10329096, -0.06307837],
+        "xor":[0.10254283, 0.18058334, 0.05268734, 0.1032732, 0.10089786, -0.1597949, -0.11678528, -0.07392009, 0.00164178, -0.3411041, -0.05760791, 0.14810716, 0.01395016, 0.07759612, 0.0478027, 0.10309431, -0.04646098, 0.22321403, 0.22373965, -0.0018288, 0.20699704, -0.16503152, -0.18555428, -0.02829313, -0.04340088, -0.09629976, -0.07565032, 0.12829013, -0.1184216, 0.05327054, -0.2091556, 0.01997479, 0.14617224, 0.12249397, -0.11211042, 0.17021945, 0.21050954, -0.1381692, 0.0287271, -0.09731391, 0.07712973, 0.0957914, 0.19754209, 0.00280226, -0.22004068, -0.19650316, 0.08103254, -0.07884329, 0.01431314, 0.01774853, 0.20215395, 0.04173042, -0.01757454, -0.06132229, 0.14705223, -0.07607429, 0.01276137, -0.02355135, 0.1687547, -0.16499719, 0.07755739, -0.15042275, 0.04714211, -0.19073205, 0.01791589, 0.00800123, -0.00147503, -0.08897253, -0.07817551, 0.02938494, 0.07296588, -0.08095124, 0.19651005, 0.12599285, -0.06209126],
+        "zext":[0.07005657, 0.08003729, 0.07546207, 0.13528231, 0.02506938, -0.12476956, -0.14581475, 0.00614589, -0.00503257, -0.24861531, 0.09526423, 0.07557298, 0.01302374, 0.07628334, 0.04175617, 0.09726538, -0.02312024, 0.16171065, 0.22212435, 0.0079543, 0.07804493, -0.15814242, -0.06840046, -0.03052379, -0.0414926, 0.26214772, -0.07542547, 0.12756911, -0.08823611, 0.13954116, -0.18143456, -0.01209466, 0.15410358, 0.20636739, -0.1831482, 0.19437483, 0.06973958, -0.13698913, 0.19432417, -0.16362838, 0.09032086, 0.18127388, 0.18287036, -0.01740397, -0.17513882, -0.16302226, 0.10199076, -0.07994232, 0.029813, 0.04575716, 0.19189098, 0.10111669, -0.01755602, -0.0832833, 0.1540967, -0.01496344, 0.04155953, 0.00536162, 0.0699449, -0.13122603, 0.10352977, -0.18122241, 0.17052825, -0.15459219, -0.06903189, 0.05887637, -0.00081859, -0.08415917, -0.07304011, 0.08401199, -0.25388402, -0.09779947, 0.12506351, 0.09557163, -0.12431429]
+    },
+    "Types": {
+        "floatTy":[-0.04556743, 0.03822431, -0.08842288, -0.12055657, 0.0960574, 0.2371231, 0.09947137, 0.11064581, -0.12371849, -0.06109007, 0.06440615, 0.14924356, 0.13218448, -0.04400141, 0.01761996, -0.13240574, 0.21056518, -0.14215694, -0.14700417, -0.16416645, -0.07137518, 0.05240471, -0.10071051, 0.12556827, 0.08208757, 0.06504779, 0.1578807, -0.09501757, 0.00659264, -0.00527758, -0.01085964, -0.17290565, 0.00501871, -0.10288252, 0.09658113, -0.107755, -0.1718967, 0.16306138, -0.080802, 0.10203532, -0.02934794, -0.02899184, -0.17915869, -0.19883338, -0.05124965, 0.16747124, -0.05587617, 0.14325564, -0.18054038, -0.161391, -0.1149613, 0.01600737, -0.0704658, 0.01935361, -0.04516762, -0.1305524, 0.04487818, -0.07881694, -0.18929696, 0.05364102, -0.17827097, 0.09934752, -0.04212811, 0.15036745, -0.18782069, 0.19947147, 0.1102478, -0.1304183, 0.17145109, 0.22485495, 0.02270225, 0.08804839, -0.12520337, -0.07780997, 0.00057234],
+        "integerTy":[-0.02392217, 0.13147527, -0.12684862, -0.00286194, 0.18320721, -0.03383226, 0.10688385, 0.20315196, 0.16910973, 0.01749469, 0.09163074, 0.18359211, 0.08457439, 0.17821081, -0.01359864, -0.09173124, -0.1060688, -0.0317139, 0.089481, -0.13937937, 0.1258349, -0.00968946, -0.00211832, -0.13140695, -0.11206195, -0.0249014, -0.09646855, 0.16041876, -0.20260467, -0.1008801, -0.02906649, -0.148828, -0.01750175, -0.10806648, 0.10094456, -0.09347772, -0.11941694, 0.0714565, -0.13869469, 0.09965917, -0.02859109, -0.04663063, -0.03677037, -0.17767252, 0.19819601, -0.00728577, -0.07156438, 0.14130634, -0.16161495, -0.14514765, -0.0230404, 0.14609018, 0.12496789, -0.02587292, 0.06375326, -0.1703029, -0.14651431, -0.15148057, -0.11655853, 0.00689279, -0.21302195, 0.10692985, 0.00105841, 0.05685481, -0.05294221, 0.13358746, 0.17953119, 0.01200722, 0.00904744, 0.13893746, -0.01024614, 0.14703822, -0.08035436, 0.18355256, -0.00407319],
+        "pointerTy":[-5.16731367e-02, 1.41998947e-01, -1.03678465e-01, -2.35795856e-01, 1.67232469e-01, -3.50435115e-02, 1.31282926e-01, 8.77518952e-02, -1.61656141e-01, -1.89439468e-02, 6.89205006e-02, -1.47259831e-01, -1.85588151e-02, 1.95201755e-01, 1.53539265e-02, -1.11738130e-01, 7.46774301e-02, -1.60889581e-01, -1.51122361e-01, -1.64823771e-01, -7.43334070e-02, 1.79255735e-02, -8.14272910e-02, -1.60337687e-01, 7.73334429e-02, 1.45058706e-02, -1.51032144e-02, -1.70947760e-01, -1.48217022e-01, -7.63990656e-02, -8.62834305e-02, -1.66838810e-01, 4.88951942e-03, -1.25664428e-01, 1.13860749e-01, -2.08933145e-01, -1.30075082e-01, 7.92812705e-02, -7.94121996e-02, 1.08121462e-01, -2.62738355e-02, -4.07571979e-02, -9.88764390e-02, 1.00676276e-01, -2.82900538e-02, 7.71642849e-02, -2.50683606e-01, -1.28315210e-01, -1.78229675e-01, -2.19585538e-01, -8.51736143e-02, -1.43008991e-04, 1.66052416e-01, 6.89304620e-02, -1.85903355e-01, 1.39382124e-01, -1.14323251e-01, 6.72841072e-02, -1.73793092e-01, 1.37479026e-02, -6.61165267e-02, 1.28709331e-01, -3.70565802e-02, 1.15480699e-01, -1.72476366e-01, 1.86700657e-01, 1.71808466e-01, 1.25800788e-01, 1.42911309e-02, -1.19691528e-01, 1.48811145e-02, -1.66594520e-01, -1.34164989e-01, 1.62748635e-01, 3.45790684e-02],
+        "structTy":[-5.22783101e-02, 1.48193806e-01, -1.73157901e-01, -8.37061405e-02, 1.75303683e-01, 1.41080663e-01, 1.32847771e-01, 9.91027430e-02, -1.22380503e-01, -5.10943905e-02, 7.63119832e-02, -1.28850952e-01, 1.66698098e-01, 1.32489190e-01, -4.90079597e-02, -1.37256354e-01, 4.85256799e-02, -1.78637013e-01, -1.86148569e-01, -1.64286211e-01, -6.26875609e-02, 8.13696086e-02, -1.20131724e-01, -1.59142628e-01, 3.36435549e-02, 2.20832378e-02, -2.72354912e-02, -1.38390273e-01, -1.38179898e-01, -1.23813316e-01, -9.40988287e-02, -1.55397400e-01, -1.58612743e-01, -1.14575408e-01, 1.55034050e-01, -1.50801614e-01, -1.07854791e-01, 1.55085072e-01, -1.68802753e-01, 1.61272466e-01, -2.05100384e-02, -8.52123126e-02, -9.58884135e-02, -1.21472001e-01, 7.77167231e-02, 1.36054471e-01, 1.28906265e-01, 1.30557746e-01, -1.81647420e-01, 1.45003811e-01, -1.11549668e-01, -1.79680258e-01, 1.72399670e-01, 1.51102387e-04, -4.99001816e-02, 1.28895119e-01, -1.17917232e-01, -9.13353860e-02, -1.21147677e-01, 9.76096392e-02, -1.77910581e-01, 1.50892287e-01, -2.25245636e-02, 1.24061532e-01, -1.16433218e-01, -6.76782504e-02, 1.45374849e-01, 2.35193707e-02, -1.35173321e-01, 1.46973729e-01, -1.22685850e-01, 1.27605781e-01, -1.51997164e-01, 1.60292000e-01, 7.63390437e-02],
+        "vectorTy":[-0.03294902, 0.14949119, -0.09443742, 0.01450515, 0.19603784, 0.25139496, 0.07845286, 0.11937284, 0.16359204, 0.06099452, 0.09465881, -0.1069911, 0.09603978, 0.16709542, -0.00486909, -0.07722043, 0.22341618, -0.03857102, 0.09912027, -0.12520577, 0.13405597, 0.01585144, 0.08918274, -0.19392243, 0.02501151, 0.00475938, 0.15233898, 0.17524292, -0.19524728, -0.01515534, 0.08875454, -0.14749287, 0.01111115, -0.0826562, 0.09034086, -0.04994484, -0.10947519, 0.11695202, -0.15125127, 0.10647643, 0.03638166, -0.01116771, -0.14471221, -0.14233877, -0.05823282, 0.16917565, -0.06123812, -0.2417991, -0.17791323, -0.11769169, -0.07800104, -0.08785082, 0.11596312, -0.02590095, 0.06086191, -0.18129952, -0.15834333, -0.13579826, -0.14008556, 0.05263453, -0.1810565, 0.05645358, 0.05425795, 0.11036766, -0.15751451, 0.20713197, 0.15679651, -0.17410725, 0.00938685, 0.2128002, 0.05759892, 0.03854589, -0.12531556, 0.19853209, -0.00149427],
+        "voidTy":[0.01605093, -0.0537584, -0.20252232, -0.08176173, -0.01439075, -0.0382277, 0.15691687, -0.104895, 0.06642336, -0.06327856, 0.09566005, 0.1357376, -0.06892086, -0.00750719, 0.00474408, -0.20349212, -0.05127234, -0.17626588, -0.17354688, -0.18703477, -0.10801363, 0.09327657, 0.10076355, 0.12450968, -0.14411868, 0.09044765, 0.01168226, -0.15345524, -0.14508787, -0.04046924, 0.04365833, -0.11735097, -0.1433241, -0.1147697, 0.18176553, -0.18558215, -0.13961768, 0.12200661, -0.13001354, 0.16781498, 0.01012573, -0.09670691, -0.10543029, 0.09294511, 0.08037758, 0.14874886, -0.1797814, -0.12390587, -0.0998297, 0.12955485, -0.15814209, -0.03254232, 0.17421001, 0.10237212, -0.07504516, -0.112073, -0.11584745, -0.20020956, 0.11506493, 0.01263643, -0.10729685, 0.12432116, -0.1172537, 0.10430267, 0.06826, -0.06138111, 0.13390729, -0.1675781, 0.00966638, 0.13989863, -0.00030718, 0.10629188, -0.15517116, -0.08125519, 0.03691618]
+        
+    },
+    "Arguments": {
+        "constant":[-0.2850312, -0.22839, 0.12669143, -0.0674703, -0.12639391, -0.00477266, 0.04786542, 0.06336267, 0.08660185, 0.12805316, -0.07146342, 0.21539183, -0.0624397, -0.02638953, -0.28688517, 0.28374302, 0.05338082, 0.05559688, -0.13133128, 0.12440272, -0.03583231, 0.29848817, 0.13930812, 0.15453401, 0.0538353, -0.06874479, 0.00262802, 0.27964258, 0.19028014, -0.16371843, -0.05762961, 0.20059372, -0.20804578, -0.06549844, 0.09732475, -0.01551855, 0.21226783, 0.05889762, -0.07560658, 0.11312829, -0.04594622, -0.27309528, -0.05293005, 0.18953343, 0.05463868, -0.31045213, -0.04364616, 0.11005993, 0.12489324, -0.05413342, -0.05814561, -0.26131225, -0.18863814, 0.31165487, -0.08796364, -0.19958755, -0.10849535, 0.14899114, -0.01385941, 0.29359573, -0.01349372, 0.0562498, 0.10977754, 0.08993197, 0.06231657, -0.13509868, -0.20968516, 0.03578237, 0.15356435, -0.17766887, -0.11509016, 0.06898113, -0.06665445, -0.14065051, 0.34711906],
+        "function":[-0.08280793, -0.18925415, -0.1866685, -0.04932962, 0.13478681, 0.203617, 0.05739383, 0.09856346, -0.17600478, 0.07619468, -0.12438924, -0.02529626, -0.07468224, 0.03905433, 0.11325707, -0.04229602, -0.16354711, -0.08952031, -0.15966323, 0.11114867, 0.07319006, 0.02678379, -0.13364255, 0.15032487, 0.0822423, 0.03195171, -0.00871139, -0.1128016, 0.19261838, -0.07204764, -0.05823177, 0.1990249, -0.2057403, -0.08260711, 0.09775644, -0.12436705, 0.15814295, 0.0463977, -0.0493853, 0.1450742, -0.07108644, -0.03192589, -0.11917686, -0.19753116, 0.09248564, 0.13522607, 0.16347136, 0.11065657, -0.0850426, -0.19110996, -0.15060848, -0.25829738, 0.11464069, -0.15613398, -0.07256057, 0.16283846, -0.11428182, -0.1879146, -0.10537492, 0.03637636, -0.02642156, 0.0344548, -0.00722412, 0.13349552, 0.05913915, 0.17941767, -0.20852967, 0.04403022, -0.13914339, -0.17848521, -0.07228794, 0.23070297, -0.11848032, -0.14098541, -0.23762096],
+        "label":[-0.06294985, -0.07393633, -0.07793023, 0.19617933, -0.25141802, -0.16856542, 0.05546749, 0.2207709, -0.12299901, -0.10727913, -0.04701709, -0.05162717, 0.2220566, 0.14470658, -0.07780463, -0.02452197, 0.21929401, 0.10171317, -0.09900579, -0.11624218, -0.21602261, 0.11818817, 0.23674524, -0.0676048, -0.15250057, -0.06562828, 0.14570533, -0.0063743, 0.12755615, 0.18377739, -0.05544094, -0.03191924, -0.12098482, -0.04439923, 0.0412157, -0.13413347, -0.03934726, 0.04211318, -0.02061427, 0.04409805, -0.06535667, -0.10373748, -0.11245634, -0.07654405, 0.1734968, -0.01024228, -0.05300638, 0.08171037, 0.1142247, 0.13233529, -0.04136537, -0.00692873, -0.13803534, 0.0900873, -0.18191165, 0.05589029, -0.05146271, -0.11442295, 0.17379795, 0.11092487, 0.18033165, 0.03263773, -0.09051663, 0.05547883, 0.09321848, -0.13325489, 0.07476224, -0.16582265, -0.14871986, -0.11099493, 0.08359679, -0.2101298, -0.05699859, -0.08379642, 0.10380454],
+        "pointer":[0.01122629, 0.03439064, 0.12280786, -0.03425089, 0.17276576, 0.21717595, 0.09126496, -0.17714174, 0.27793115, 0.08274158, -0.08821795, -0.16903219, -0.06300986, -0.23226759, 0.1235508, -0.11923615, -0.21793933, -0.17721081, 0.07182363, 0.27753174, 0.0920836, 0.06713749, -0.04808864, 0.15279293, 0.05296317, 0.04695908, -0.00205181, -0.14607918, 0.177891, -0.01903534, -0.05020404, 0.20370306, 0.03707821, -0.08771795, 0.10304545, 0.13130909, 0.18650576, 0.05210593, -0.09720453, 0.14389311, -0.05903525, 0.01644695, -0.11466363, 0.18700723, 0.07347484, 0.13623913, 0.13876125, 0.10103971, 0.11578573, -0.21065098, -0.07865446, -0.02068869, 0.16991298, -0.16359806, -0.07672877, 0.11908785, -0.14677979, 0.16077666, -0.21696989, 0.03678338, -0.02173937, 0.08764295, 0.18778177, 0.0974953, 0.05379327, -0.10313212, -0.2190201, 0.25079602, -0.1224342, -0.17659691, -0.10354815, 0.12724441, -0.11835013, -0.15596658, -0.262892],
+        "variable":[-0.00726075, -0.19133486, 0.11386976, -0.02404008, 0.18218324, 0.2147853, 0.05924276, -0.14287908, 0.2636781, 0.08127253, -0.08004244, -0.151082, -0.05468801, -0.14080277, 0.09922418, -0.09297107, -0.2022257, -0.0631255, 0.08962703, 0.2616532, -0.05850444, 0.00352755, -0.1646196, 0.12995376, 0.04720275, -0.04833887, 0.0210924, 0.23275656, 0.20620209, -0.01313339, -0.04938933, 0.18796188, 0.03722439, -0.0570973, 0.07931395, 0.12533784, 0.19275506, 0.04549298, -0.06122055, -0.27613187, -0.03735771, 0.0223406, -0.09128818, 0.15596932, -0.02494176, 0.10621183, -0.03121548, 0.09133425, 0.11494032, 0.1616893, -0.05987769, -0.01591593, -0.15986548, -0.13643607, -0.07331984, -0.21545175, -0.13974416, 0.14965075, -0.19167605, 0.03849319, -0.01465273, 0.04467848, 0.16016744, 0.08067682, 0.05409591, -0.1172289, -0.20374107, 0.21127956, 0.14046785, -0.15151982, -0.07432669, 0.1618207, -0.09742998, -0.14227344, 0.07137881]
+    }
 }
\ No newline at end of file
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index ad59182e7d15..dccafbd1ffb1 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -633,7 +633,7 @@ bool LLParser::parseTargetDefinition(std::string &TentativeDLStr,
     if (parseToken(lltok::equal, "expected '=' after target triple") ||
         parseStringConstant(Str))
       return true;
-    M->setTargetTriple(Triple(Str));
+    M->setTargetTriple(Triple(std::move(Str)));
     return false;
   case lltok::kw_datalayout:
     Lex.Lex();
@@ -1234,14 +1234,12 @@ bool LLParser::parseAliasOrIFunc(const std::string &Name, unsigned NameID,
   std::unique_ptr<GlobalIFunc> GI;
   GlobalValue *GV;
   if (IsAlias) {
-    GA.reset(GlobalAlias::create(Ty, AddrSpace,
-                                 (GlobalValue::LinkageTypes)Linkage, Name,
-                                 Aliasee, /*Parent*/ nullptr));
+    GA.reset(GlobalAlias::create(Ty, AddrSpace, Linkage, Name, Aliasee,
+                                 /*Parent=*/nullptr));
     GV = GA.get();
   } else {
-    GI.reset(GlobalIFunc::create(Ty, AddrSpace,
-                                 (GlobalValue::LinkageTypes)Linkage, Name,
-                                 Aliasee, /*Parent*/ nullptr));
+    GI.reset(GlobalIFunc::create(Ty, AddrSpace, Linkage, Name, Aliasee,
+                                 /*Parent=*/nullptr));
     GV = GI.get();
   }
   GV->setThreadLocalMode(TLM);
@@ -1705,14 +1703,6 @@ bool LLParser::parseFnAttributeValuePairs(AttrBuilder &B,
       continue;
     }
 
-    if (Token == lltok::kw_sanitized_padded_global) {
-      if (parseToken(lltok::kw_sanitized_padded_global,
-                     "expected 'sanitized_padded_global'"))
-        return true;
-      B.addAttribute(Attribute::SanitizedPaddedGlobal);
-      continue;
-    }
-
     SMLoc Loc = Lex.getLoc();
     if (Token == lltok::kw_builtin)
       BuiltinLoc = Loc;
@@ -4846,6 +4836,34 @@ struct MDSignedOrMDField : MDEitherFieldImpl<MDSignedField, MDField> {
   }
 };
 
+struct MDUnsignedOrMDField : MDEitherFieldImpl<MDUnsignedField, MDField> {
+  MDUnsignedOrMDField(uint64_t Default = 0, bool AllowNull = true)
+      : ImplTy(MDUnsignedField(Default), MDField(AllowNull)) {}
+
+  MDUnsignedOrMDField(uint64_t Default, uint64_t Max, bool AllowNull = true)
+      : ImplTy(MDUnsignedField(Default, Max), MDField(AllowNull)) {}
+
+  bool isMDUnsignedField() const { return WhatIs == IsTypeA; }
+  bool isMDField() const { return WhatIs == IsTypeB; }
+  uint64_t getMDUnsignedValue() const {
+    assert(isMDUnsignedField() && "Wrong field type");
+    return A.Val;
+  }
+  Metadata *getMDFieldValue() const {
+    assert(isMDField() && "Wrong field type");
+    return B.Val;
+  }
+
+  Metadata *getValueAsMetadata(LLVMContext &Context) const {
+    if (isMDUnsignedField())
+      return ConstantAsMetadata::get(
+          ConstantInt::get(Type::getInt64Ty(Context), getMDUnsignedValue()));
+    if (isMDField())
+      return getMDFieldValue();
+    return nullptr;
+  }
+};
+
 } // end anonymous namespace
 
 namespace llvm {
@@ -5249,6 +5267,29 @@ bool LLParser::parseMDField(LocTy Loc, StringRef Name,
   return true;
 }
 
+template <>
+bool LLParser::parseMDField(LocTy Loc, StringRef Name,
+                            MDUnsignedOrMDField &Result) {
+  // Try to parse an unsigned int.
+  if (Lex.getKind() == lltok::APSInt) {
+    MDUnsignedField Res = Result.A;
+    if (!parseMDField(Loc, Name, Res)) {
+      Result.assign(Res);
+      return false;
+    }
+    return true;
+  }
+
+  // Otherwise, try to parse as an MDField.
+  MDField Res = Result.B;
+  if (!parseMDField(Loc, Name, Res)) {
+    Result.assign(Res);
+    return false;
+  }
+
+  return true;
+}
+
 template <>
 bool LLParser::parseMDField(LocTy Loc, StringRef Name, MDStringField &Result) {
   LocTy ValueLoc = Lex.getLoc();
@@ -5430,7 +5471,7 @@ bool LLParser::parseDISubrangeType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(line, LineField, );                                                 \
   OPTIONAL(scope, MDField, );                                                  \
   OPTIONAL(baseType, MDField, );                                               \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
   OPTIONAL(flags, DIFlagField, );                                              \
   OPTIONAL(lowerBound, MDSignedOrMDField, );                                   \
@@ -5454,10 +5495,10 @@ bool LLParser::parseDISubrangeType(MDNode *&Result, bool IsDistinct) {
   Metadata *Stride = convToMetadata(stride);
   Metadata *Bias = convToMetadata(bias);
 
-  Result = GET_OR_DISTINCT(DISubrangeType,
-                           (Context, name.Val, file.Val, line.Val, scope.Val,
-                            size.Val, align.Val, flags.Val, baseType.Val,
-                            LowerBound, UpperBound, Stride, Bias));
+  Result = GET_OR_DISTINCT(
+      DISubrangeType, (Context, name.Val, file.Val, line.Val, scope.Val,
+                       size.getValueAsMetadata(Context), align.Val, flags.Val,
+                       baseType.Val, LowerBound, UpperBound, Stride, Bias));
 
   return false;
 }
@@ -5565,7 +5606,7 @@ bool LLParser::parseDIBasicType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_base_type));                     \
   OPTIONAL(name, MDStringField, );                                             \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
   OPTIONAL(encoding, DwarfAttEncodingField, );                                 \
   OPTIONAL(num_extra_inhabitants, MDUnsignedField, (0, UINT32_MAX));           \
@@ -5573,7 +5614,8 @@ bool LLParser::parseDIBasicType(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  Result = GET_OR_DISTINCT(DIBasicType, (Context, tag.Val, name.Val, size.Val,
+  Result = GET_OR_DISTINCT(DIBasicType, (Context, tag.Val, name.Val,
+                                         size.getValueAsMetadata(Context),
                                          align.Val, encoding.Val,
                                          num_extra_inhabitants.Val, flags.Val));
   return false;
@@ -5588,7 +5630,7 @@ bool LLParser::parseDIFixedPointType(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   OPTIONAL(tag, DwarfTagField, (dwarf::DW_TAG_base_type));                     \
   OPTIONAL(name, MDStringField, );                                             \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
   OPTIONAL(encoding, DwarfAttEncodingField, );                                 \
   OPTIONAL(flags, DIFlagField, );                                              \
@@ -5600,7 +5642,8 @@ bool LLParser::parseDIFixedPointType(MDNode *&Result, bool IsDistinct) {
 #undef VISIT_MD_FIELDS
 
   Result = GET_OR_DISTINCT(DIFixedPointType,
-                           (Context, tag.Val, name.Val, size.Val, align.Val,
+                           (Context, tag.Val, name.Val,
+                            size.getValueAsMetadata(Context), align.Val,
                             encoding.Val, flags.Val, kind.Val, factor.Val,
                             numerator.Val, denominator.Val));
   return false;
@@ -5615,7 +5658,7 @@ bool LLParser::parseDIStringType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(stringLength, MDField, );                                           \
   OPTIONAL(stringLengthExpression, MDField, );                                 \
   OPTIONAL(stringLocationExpression, MDField, );                               \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
   OPTIONAL(encoding, DwarfAttEncodingField, );
   PARSE_MD_FIELDS();
@@ -5624,7 +5667,8 @@ bool LLParser::parseDIStringType(MDNode *&Result, bool IsDistinct) {
   Result = GET_OR_DISTINCT(
       DIStringType,
       (Context, tag.Val, name.Val, stringLength.Val, stringLengthExpression.Val,
-       stringLocationExpression.Val, size.Val, align.Val, encoding.Val));
+       stringLocationExpression.Val, size.getValueAsMetadata(Context),
+       align.Val, encoding.Val));
   return false;
 }
 
@@ -5646,9 +5690,9 @@ bool LLParser::parseDIDerivedType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(line, LineField, );                                                 \
   OPTIONAL(scope, MDField, );                                                  \
   REQUIRED(baseType, MDField, );                                               \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
-  OPTIONAL(offset, MDUnsignedField, (0, UINT64_MAX));                          \
+  OPTIONAL(offset, MDUnsignedOrMDField, (0, UINT64_MAX));                      \
   OPTIONAL(flags, DIFlagField, );                                              \
   OPTIONAL(extraData, MDField, );                                              \
   OPTIONAL(addressSpace, MDUnsignedField, (UINT32_MAX, UINT32_MAX));      \
@@ -5673,11 +5717,12 @@ bool LLParser::parseDIDerivedType(MDNode *&Result, bool IsDistinct) {
         (unsigned)ptrAuthExtraDiscriminator.Val, ptrAuthIsaPointer.Val,
         ptrAuthAuthenticatesNullValues.Val);
 
-  Result = GET_OR_DISTINCT(DIDerivedType,
-                           (Context, tag.Val, name.Val, file.Val, line.Val,
-                            scope.Val, baseType.Val, size.Val, align.Val,
-                            offset.Val, DWARFAddressSpace, memorySpace.val(), PtrAuthData,
-                            flags.Val, extraData.Val, annotations.Val));
+  Result = GET_OR_DISTINCT(
+      DIDerivedType, (Context, tag.Val, name.Val, file.Val, line.Val, scope.Val,
+                      baseType.Val, size.getValueAsMetadata(Context), align.Val,
+                      offset.getValueAsMetadata(Context), DWARFAddressSpace,
+                      memorySpace.val(),
+                      PtrAuthData, flags.Val, extraData.Val, annotations.Val));
   return false;
 }
 
@@ -5689,9 +5734,9 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(line, LineField, );                                                 \
   OPTIONAL(scope, MDField, );                                                  \
   OPTIONAL(baseType, MDField, );                                               \
-  OPTIONAL(size, MDUnsignedField, (0, UINT64_MAX));                            \
+  OPTIONAL(size, MDUnsignedOrMDField, (0, UINT64_MAX));                        \
   OPTIONAL(align, MDUnsignedField, (0, UINT32_MAX));                           \
-  OPTIONAL(offset, MDUnsignedField, (0, UINT64_MAX));                          \
+  OPTIONAL(offset, MDUnsignedOrMDField, (0, UINT64_MAX));                      \
   OPTIONAL(flags, DIFlagField, );                                              \
   OPTIONAL(elements, MDField, );                                               \
   OPTIONAL(runtimeLang, DwarfLangField, );                                     \
@@ -5726,12 +5771,12 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
   if (identifier.Val)
     if (auto *CT = DICompositeType::buildODRType(
             Context, *identifier.Val, tag.Val, name.Val, file.Val, line.Val,
-            scope.Val, baseType.Val, size.Val, align.Val, offset.Val,
-            specification.Val, num_extra_inhabitants.Val, flags.Val,
-            elements.Val, runtimeLang.Val, EnumKind, vtableHolder.Val,
-            templateParams.Val, discriminator.Val, dataLocation.Val,
-            associated.Val, allocated.Val, Rank, annotations.Val,
-            bitStride.Val)) {
+            scope.Val, baseType.Val, size.getValueAsMetadata(Context),
+            align.Val, offset.getValueAsMetadata(Context), specification.Val,
+            num_extra_inhabitants.Val, flags.Val, elements.Val, runtimeLang.Val,
+            EnumKind, vtableHolder.Val, templateParams.Val, discriminator.Val,
+            dataLocation.Val, associated.Val, allocated.Val, Rank,
+            annotations.Val, bitStride.Val)) {
       Result = CT;
       return false;
     }
@@ -5741,7 +5786,8 @@ bool LLParser::parseDICompositeType(MDNode *&Result, bool IsDistinct) {
   Result = GET_OR_DISTINCT(
       DICompositeType,
       (Context, tag.Val, name.Val, file.Val, line.Val, scope.Val, baseType.Val,
-       size.Val, align.Val, offset.Val, flags.Val, elements.Val,
+       size.getValueAsMetadata(Context), align.Val,
+       offset.getValueAsMetadata(Context), flags.Val, elements.Val,
        runtimeLang.Val, EnumKind, vtableHolder.Val, templateParams.Val,
        identifier.Val, discriminator.Val, dataLocation.Val, associated.Val,
        allocated.Val, Rank, annotations.Val, specification.Val,
@@ -5872,7 +5918,8 @@ bool LLParser::parseDISubprogram(MDNode *&Result, bool IsDistinct) {
   OPTIONAL(retainedNodes, MDField, );                                          \
   OPTIONAL(thrownTypes, MDField, );                                            \
   OPTIONAL(annotations, MDField, );                                            \
-  OPTIONAL(targetFuncName, MDStringField, );
+  OPTIONAL(targetFuncName, MDStringField, );                                   \
+  OPTIONAL(keyInstructions, MDBoolField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
@@ -5892,7 +5939,7 @@ bool LLParser::parseDISubprogram(MDNode *&Result, bool IsDistinct) {
        type.Val, scopeLine.Val, containingType.Val, virtualIndex.Val,
        thisAdjustment.Val, flags.Val, SPFlags, unit.Val, templateParams.Val,
        declaration.Val, retainedNodes.Val, thrownTypes.Val, annotations.Val,
-       targetFuncName.Val));
+       targetFuncName.Val, keyInstructions.Val));
   return false;
 }
 
@@ -6113,18 +6160,26 @@ bool LLParser::parseDILocalVariable(MDNode *&Result, bool IsDistinct) {
 }
 
 /// parseDILabel:
-///   ::= !DILabel(scope: !0, name: "foo", file: !1, line: 7)
+///   ::= !DILabel(scope: !0, name: "foo", file: !1, line: 7, column: 4)
 bool LLParser::parseDILabel(MDNode *&Result, bool IsDistinct) {
 #define VISIT_MD_FIELDS(OPTIONAL, REQUIRED)                                    \
   REQUIRED(scope, MDField, (/* AllowNull */ false));                           \
   REQUIRED(name, MDStringField, );                                             \
   REQUIRED(file, MDField, );                                                   \
-  REQUIRED(line, LineField, );
+  REQUIRED(line, LineField, );                                                 \
+  OPTIONAL(column, ColumnField, );                                             \
+  OPTIONAL(isArtificial, MDBoolField, );                                       \
+  OPTIONAL(coroSuspendIdx, MDUnsignedField, );
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
+  std::optional<unsigned> CoroSuspendIdx =
+      coroSuspendIdx.Seen ? std::optional<unsigned>(coroSuspendIdx.Val)
+                          : std::nullopt;
+
   Result = GET_OR_DISTINCT(DILabel,
-                           (Context, scope.Val, name.Val, file.Val, line.Val));
+                           (Context, scope.Val, name.Val, file.Val, line.Val,
+                            column.Val, isArtificial.Val, CoroSuspendIdx));
   return false;
 }
 
diff --git a/llvm/lib/BinaryFormat/DXContainer.cpp b/llvm/lib/BinaryFormat/DXContainer.cpp
index 8e7b7d313706..36d10d0b6307 100644
--- a/llvm/lib/BinaryFormat/DXContainer.cpp
+++ b/llvm/lib/BinaryFormat/DXContainer.cpp
@@ -60,6 +60,35 @@ ArrayRef<EnumEntry<SigComponentType>> dxbc::getSigComponentTypes() {
   return ArrayRef(SigComponentTypes);
 }
 
+static const EnumEntry<RootFlags> RootFlagNames[] = {
+#define ROOT_SIGNATURE_FLAG(Val, Enum) {#Enum, RootFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootFlags>> dxbc::getRootFlags() {
+  return ArrayRef(RootFlagNames);
+}
+
+static const EnumEntry<RootDescriptorFlags> RootDescriptorFlagNames[] = {
+#define ROOT_DESCRIPTOR_FLAG(Val, Enum, Flag)                                  \
+  {#Enum, RootDescriptorFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<RootDescriptorFlags>> dxbc::getRootDescriptorFlags() {
+  return ArrayRef(RootDescriptorFlagNames);
+}
+
+static const EnumEntry<DescriptorRangeFlags> DescriptorRangeFlagNames[] = {
+#define DESCRIPTOR_RANGE_FLAG(Val, Enum, Flag)                                 \
+  {#Enum, DescriptorRangeFlags::Enum},
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<DescriptorRangeFlags>> dxbc::getDescriptorRangeFlags() {
+  return ArrayRef(DescriptorRangeFlagNames);
+}
+
 #define SHADER_VISIBILITY(Val, Enum) {#Enum, ShaderVisibility::Enum},
 
 static const EnumEntry<ShaderVisibility> ShaderVisibilityValues[] = {
@@ -70,6 +99,46 @@ ArrayRef<EnumEntry<ShaderVisibility>> dxbc::getShaderVisibility() {
   return ArrayRef(ShaderVisibilityValues);
 }
 
+#define FILTER(Val, Enum) {#Enum, SamplerFilter::Enum},
+
+static const EnumEntry<SamplerFilter> SamplerFilterNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<SamplerFilter>> dxbc::getSamplerFilters() {
+  return ArrayRef(SamplerFilterNames);
+}
+
+#define TEXTURE_ADDRESS_MODE(Val, Enum) {#Enum, TextureAddressMode::Enum},
+
+static const EnumEntry<TextureAddressMode> TextureAddressModeNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<TextureAddressMode>> dxbc::getTextureAddressModes() {
+  return ArrayRef(TextureAddressModeNames);
+}
+
+#define COMPARISON_FUNC(Val, Enum) {#Enum, ComparisonFunc::Enum},
+
+static const EnumEntry<ComparisonFunc> ComparisonFuncNames[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<ComparisonFunc>> dxbc::getComparisonFuncs() {
+  return ArrayRef(ComparisonFuncNames);
+}
+
+#define STATIC_BORDER_COLOR(Val, Enum) {#Enum, StaticBorderColor::Enum},
+
+static const EnumEntry<StaticBorderColor> StaticBorderColorValues[] = {
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+};
+
+ArrayRef<EnumEntry<StaticBorderColor>> dxbc::getStaticBorderColors() {
+  return ArrayRef(StaticBorderColorValues);
+}
+
 #define ROOT_PARAMETER(Val, Enum) {#Enum, RootParameterType::Enum},
 
 static const EnumEntry<RootParameterType> RootParameterTypes[] = {
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index dd44bfd08eff..bbabe2544565 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2246,6 +2246,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::NoExt;
   case bitc::ATTR_KIND_CAPTURES:
     return Attribute::Captures;
+  case bitc::ATTR_KIND_DEAD_ON_RETURN:
+    return Attribute::DeadOnReturn;
   }
 }
 
@@ -4697,7 +4699,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
       std::string S;
       if (convertToString(Record, 0, S))
         return error("Invalid record");
-      TheModule->setTargetTriple(Triple(S));
+      TheModule->setTargetTriple(Triple(std::move(S)));
       break;
     }
     case bitc::MODULE_CODE_DATALAYOUT: {  // DATALAYOUT: [strchr x N]
@@ -5084,7 +5086,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
 
       unsigned Line = Record[0], Col = Record[1];
       unsigned ScopeID = Record[2], IAID = Record[3];
-      bool isImplicitCode = Record.size() == 5 && Record[4];
+      bool isImplicitCode = Record.size() >= 5 && Record[4];
+      uint64_t AtomGroup = Record.size() == 7 ? Record[5] : 0;
+      uint8_t AtomRank = Record.size() == 7 ? Record[6] : 0;
 
       MDNode *Scope = nullptr, *IA = nullptr;
       if (ScopeID) {
@@ -5099,8 +5103,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         if (!IA)
           return error("Invalid record");
       }
+
       LastLoc = DILocation::get(Scope->getContext(), Line, Col, Scope, IA,
-                                isImplicitCode);
+                                isImplicitCode, AtomGroup, AtomRank);
       I->setDebugLoc(LastLoc);
       I = nullptr;
       continue;
@@ -7032,7 +7037,7 @@ Error BitcodeReader::materialize(GlobalValue *GV) {
         MDString *MDS = cast<MDString>(MD->getOperand(0));
         StringRef ProfName = MDS->getString();
         // Check consistency of !prof branch_weights metadata.
-        if (ProfName != "branch_weights")
+        if (ProfName != MDProfLabels::BranchWeights)
           continue;
         unsigned ExpectedNumOperands = 0;
         if (BranchInst *BI = dyn_cast<BranchInst>(&I))
@@ -7512,11 +7517,9 @@ std::vector<FunctionSummary::ParamAccess>
 ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) {
   auto ReadRange = [&]() {
     APInt Lower(FunctionSummary::ParamAccess::RangeWidth,
-                BitcodeReader::decodeSignRotatedValue(Record.front()));
-    Record = Record.drop_front();
+                BitcodeReader::decodeSignRotatedValue(Record.consume_front()));
     APInt Upper(FunctionSummary::ParamAccess::RangeWidth,
-                BitcodeReader::decodeSignRotatedValue(Record.front()));
-    Record = Record.drop_front();
+                BitcodeReader::decodeSignRotatedValue(Record.consume_front()));
     ConstantRange Range{Lower, Upper};
     assert(!Range.isFullSet());
     assert(!Range.isUpperSignWrapped());
@@ -7527,16 +7530,13 @@ ModuleSummaryIndexBitcodeReader::parseParamAccesses(ArrayRef<uint64_t> Record) {
   while (!Record.empty()) {
     PendingParamAccesses.emplace_back();
     FunctionSummary::ParamAccess &ParamAccess = PendingParamAccesses.back();
-    ParamAccess.ParamNo = Record.front();
-    Record = Record.drop_front();
+    ParamAccess.ParamNo = Record.consume_front();
     ParamAccess.Use = ReadRange();
-    ParamAccess.Calls.resize(Record.front());
-    Record = Record.drop_front();
+    ParamAccess.Calls.resize(Record.consume_front());
     for (auto &Call : ParamAccess.Calls) {
-      Call.ParamNo = Record.front();
-      Record = Record.drop_front();
-      Call.Callee = std::get<0>(getValueInfoFromValueId(Record.front()));
-      Record = Record.drop_front();
+      Call.ParamNo = Record.consume_front();
+      Call.Callee =
+          std::get<0>(getValueInfoFromValueId(Record.consume_front()));
       Call.Offsets = ReadRange();
     }
   }
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index f4413d762c89..731e418e9830 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1171,8 +1171,10 @@ void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata(
   assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString");
   // Lookup first if the metadata hasn't already been loaded.
   if (auto *MD = MetadataList.lookup(ID)) {
-    auto *N = cast<MDNode>(MD);
-    if (!N->isTemporary())
+    auto *N = dyn_cast<MDNode>(MD);
+    // If the node is not an MDNode, or if it is not temporary, then
+    // we're done.
+    if (!N || !N->isTemporary())
       return;
   }
   SmallVector<uint64_t, 64> Record;
@@ -1480,6 +1482,14 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     return MetadataList.upgradeTypeRef(getMDOrNull(ID));
   };
 
+  auto getMetadataOrConstant = [&](bool IsMetadata,
+                                   uint64_t Entry) -> Metadata * {
+    if (IsMetadata)
+      return getMDOrNull(Entry);
+    return ConstantAsMetadata::get(
+        ConstantInt::get(Type::getInt64Ty(Context), Entry));
+  };
+
 #define GET_OR_DISTINCT(CLASS, ARGS)                                           \
   (IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS)
 
@@ -1609,7 +1619,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_LOCATION: {
-    if (Record.size() != 5 && Record.size() != 6)
+    // 5: inlinedAt, 6: isImplicit, 8: Key Instructions fields.
+    if (Record.size() != 5 && Record.size() != 6 && Record.size() != 8)
       return error("Invalid record");
 
     IsDistinct = Record[0];
@@ -1617,10 +1628,12 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     unsigned Column = Record[2];
     Metadata *Scope = getMD(Record[3]);
     Metadata *InlinedAt = getMDOrNull(Record[4]);
-    bool ImplicitCode = Record.size() == 6 && Record[5];
+    bool ImplicitCode = Record.size() >= 6 && Record[5];
+    uint64_t AtomGroup = Record.size() == 8 ? Record[6] : 0;
+    uint8_t AtomRank = Record.size() == 8 ? Record[7] : 0;
     MetadataList.assignValue(
         GET_OR_DISTINCT(DILocation, (Context, Line, Column, Scope, InlinedAt,
-                                     ImplicitCode)),
+                                     ImplicitCode, AtomGroup, AtomRank)),
         NextMetadataNo);
     NextMetadataNo++;
     break;
@@ -1718,15 +1731,18 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (Record.size() < 6 || Record.size() > 8)
       return error("Invalid record");
 
-    IsDistinct = Record[0];
+    IsDistinct = Record[0] & 1;
+    bool SizeIsMetadata = Record[0] & 2;
     DINode::DIFlags Flags = (Record.size() > 6)
                                 ? static_cast<DINode::DIFlags>(Record[6])
                                 : DINode::FlagZero;
     uint32_t NumExtraInhabitants = (Record.size() > 7) ? Record[7] : 0;
 
+    Metadata *SizeInBits = getMetadataOrConstant(SizeIsMetadata, Record[3]);
+
     MetadataList.assignValue(
         GET_OR_DISTINCT(DIBasicType,
-                        (Context, Record[1], getMDString(Record[2]), Record[3],
+                        (Context, Record[1], getMDString(Record[2]), SizeInBits,
                          Record[4], Record[5], NumExtraInhabitants, Flags)),
         NextMetadataNo);
     NextMetadataNo++;
@@ -1736,9 +1752,12 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (Record.size() < 11)
       return error("Invalid record");
 
-    IsDistinct = Record[0];
+    IsDistinct = Record[0] & 1;
+    bool SizeIsMetadata = Record[0] & 2;
     DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[6]);
 
+    Metadata *SizeInBits = getMetadataOrConstant(SizeIsMetadata, Record[3]);
+
     size_t Offset = 9;
 
     auto ReadWideInt = [&]() {
@@ -1758,7 +1777,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
 
     MetadataList.assignValue(
         GET_OR_DISTINCT(DIFixedPointType,
-                        (Context, Record[1], getMDString(Record[2]), Record[3],
+                        (Context, Record[1], getMDString(Record[2]), SizeInBits,
                          Record[4], Record[5], Flags, Record[7], Record[8],
                          Numerator, Denominator)),
         NextMetadataNo);
@@ -1769,17 +1788,21 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (Record.size() > 9 || Record.size() < 8)
       return error("Invalid record");
 
-    IsDistinct = Record[0];
+    IsDistinct = Record[0] & 1;
+    bool SizeIsMetadata = Record[0] & 2;
     bool SizeIs8 = Record.size() == 8;
     // StringLocationExp (i.e. Record[5]) is added at a later time
     // than the other fields. The code here enables backward compatibility.
     Metadata *StringLocationExp = SizeIs8 ? nullptr : getMDOrNull(Record[5]);
     unsigned Offset = SizeIs8 ? 5 : 6;
+    Metadata *SizeInBits =
+        getMetadataOrConstant(SizeIsMetadata, Record[Offset]);
+
     MetadataList.assignValue(
         GET_OR_DISTINCT(DIStringType,
                         (Context, Record[1], getMDString(Record[2]),
                          getMDOrNull(Record[3]), getMDOrNull(Record[4]),
-                         StringLocationExp, Record[Offset], Record[Offset + 1],
+                         StringLocationExp, SizeInBits, Record[Offset + 1],
                          Record[Offset + 2])),
         NextMetadataNo);
     NextMetadataNo++;
@@ -1812,15 +1835,20 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (!MSpace)
       return MSpace.takeError();
 
-    IsDistinct = Record[0];
+    IsDistinct = Record[0] & 1;
+    bool SizeIsMetadata = Record[0] & 2;
     DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]);
+
+    Metadata *SizeInBits = getMetadataOrConstant(SizeIsMetadata, Record[7]);
+    Metadata *OffsetInBits = getMetadataOrConstant(SizeIsMetadata, Record[9]);
+
     MetadataList.assignValue(
         GET_OR_DISTINCT(DIDerivedType,
                         (Context, Record[1], getMDString(Record[2]),
                          getMDOrNull(Record[3]), Record[4],
                          getDITypeRefOrNull(Record[5]),
-                         getDITypeRefOrNull(Record[6]), Record[7], Record[8],
-                         Record[9], DWARFAddressSpace, *MSpace, PtrAuthData, Flags,
+                         getDITypeRefOrNull(Record[6]), SizeInBits, Record[8],
+                         OffsetInBits, DWARFAddressSpace, *MSpace, PtrAuthData, Flags,
                          getDITypeRefOrNull(Record[11]), Annotations)),
         NextMetadataNo);
     NextMetadataNo++;
@@ -1830,13 +1858,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (Record.size() != 13)
       return error("Invalid record");
 
-    IsDistinct = Record[0];
+    IsDistinct = Record[0] & 1;
+    bool SizeIsMetadata = Record[0] & 2;
     DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[7]);
+
+    Metadata *SizeInBits = getMetadataOrConstant(SizeIsMetadata, Record[5]);
+
     MetadataList.assignValue(
         GET_OR_DISTINCT(DISubrangeType,
                         (Context, getMDString(Record[1]),
                          getMDOrNull(Record[2]), Record[3],
-                         getMDOrNull(Record[4]), Record[5], Record[6], Flags,
+                         getMDOrNull(Record[4]), SizeInBits, Record[6], Flags,
                          getDITypeRefOrNull(Record[8]), getMDOrNull(Record[9]),
                          getMDOrNull(Record[10]), getMDOrNull(Record[11]),
                          getMDOrNull(Record[12]))),
@@ -1851,18 +1883,18 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     // If we have a UUID and this is not a forward declaration, lookup the
     // mapping.
     IsDistinct = Record[0] & 0x1;
-    bool IsNotUsedInTypeRef = Record[0] >= 2;
+    bool IsNotUsedInTypeRef = Record[0] & 2;
+    bool SizeIsMetadata = Record[0] & 4;
     unsigned Tag = Record[1];
     MDString *Name = getMDString(Record[2]);
     Metadata *File = getMDOrNull(Record[3]);
     unsigned Line = Record[4];
     Metadata *Scope = getDITypeRefOrNull(Record[5]);
     Metadata *BaseType = nullptr;
-    uint64_t SizeInBits = Record[7];
     if (Record[8] > (uint64_t)std::numeric_limits<uint32_t>::max())
       return error("Alignment value is too large");
     uint32_t AlignInBits = Record[8];
-    uint64_t OffsetInBits = 0;
+    Metadata *OffsetInBits = nullptr;
     uint32_t NumExtraInhabitants = (Record.size() > 22) ? Record[22] : 0;
     DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]);
     Metadata *Elements = nullptr;
@@ -1909,7 +1941,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
         TemplateParams = getMDOrNull(Record[14]);
     } else {
       BaseType = getDITypeRefOrNull(Record[6]);
-      OffsetInBits = Record[9];
+
+      OffsetInBits = getMetadataOrConstant(SizeIsMetadata, Record[9]);
+
       Elements = getMDOrNull(Record[11]);
       VTableHolder = getDITypeRefOrNull(Record[13]);
       TemplateParams = getMDOrNull(Record[14]);
@@ -1937,6 +1971,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     if (Record.size() > 24 && Record[24] != dwarf::DW_APPLE_ENUM_KIND_invalid)
       EnumKind = Record[24];
 
+    Metadata *SizeInBits = getMetadataOrConstant(SizeIsMetadata, Record[7]);
+
     DICompositeType *CT = nullptr;
     if (Identifier)
       CT = DICompositeType::buildODRType(
@@ -2054,7 +2090,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_SUBPROGRAM: {
-    if (Record.size() < 18 || Record.size() > 21)
+    if (Record.size() < 18 || Record.size() > 22)
       return error("Invalid record");
 
     bool HasSPFlags = Record[0] & 4;
@@ -2106,6 +2142,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     bool HasTargetFuncName = false;
     unsigned OffsetA = 0;
     unsigned OffsetB = 0;
+    // Key instructions won't be enabled in old-format bitcode, so only
+    // check it if HasSPFlags is true.
+    bool UsesKeyInstructions = false;
     if (!HasSPFlags) {
       OffsetA = 2;
       OffsetB = 2;
@@ -2118,7 +2157,9 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     } else {
       HasAnnotations = Record.size() >= 19;
       HasTargetFuncName = Record.size() >= 20;
+      UsesKeyInstructions = Record.size() >= 21 ? Record[20] : 0;
     }
+
     Metadata *CUorFn = getMDOrNull(Record[12 + OffsetB]);
     DISubprogram *SP = GET_OR_DISTINCT(
         DISubprogram,
@@ -2144,8 +2185,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
          HasAnnotations ? getMDOrNull(Record[18 + OffsetB])
                         : nullptr, // annotations
          HasTargetFuncName ? getMDString(Record[19 + OffsetB])
-                           : nullptr // targetFuncName
-         ));
+                           : nullptr, // targetFuncName
+         UsesKeyInstructions));
     MetadataList.assignValue(SP, NextMetadataNo);
     NextMetadataNo++;
 
@@ -2419,15 +2460,28 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
     break;
   }
   case bitc::METADATA_LABEL: {
-    if (Record.size() != 5)
+    if (Record.size() < 5 || Record.size() > 7)
       return error("Invalid record");
 
     IsDistinct = Record[0] & 1;
+    uint64_t Line = Record[4];
+    uint64_t Column = Record.size() > 5 ? Record[5] : 0;
+    bool IsArtificial = Record[0] & 2;
+    std::optional<unsigned> CoroSuspendIdx;
+    if (Record.size() > 6) {
+      uint64_t RawSuspendIdx = Record[6];
+      if (RawSuspendIdx != std::numeric_limits<uint64_t>::max()) {
+        if (RawSuspendIdx > (uint64_t)std::numeric_limits<unsigned>::max())
+          return error("CoroSuspendIdx value is too large");
+        CoroSuspendIdx = RawSuspendIdx;
+      }
+    }
+
     MetadataList.assignValue(
         GET_OR_DISTINCT(DILabel,
                         (Context, getMDOrNull(Record[1]),
-                         getMDString(Record[2]),
-                         getMDOrNull(Record[3]), Record[4])),
+                         getMDString(Record[2]), getMDOrNull(Record[3]), Line,
+                         Column, IsArtificial, CoroSuspendIdx)),
         NextMetadataNo);
     NextMetadataNo++;
     break;
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index c20f43166227..201f3a96dd1f 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -141,6 +141,7 @@ enum {
 
   // FUNCTION_BLOCK abbrev id's.
   FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
+  FUNCTION_INST_STORE_ABBREV,
   FUNCTION_INST_UNOP_ABBREV,
   FUNCTION_INST_UNOP_FLAGS_ABBREV,
   FUNCTION_INST_BINOP_ABBREV,
@@ -149,9 +150,14 @@ enum {
   FUNCTION_INST_CAST_FLAGS_ABBREV,
   FUNCTION_INST_RET_VOID_ABBREV,
   FUNCTION_INST_RET_VAL_ABBREV,
+  FUNCTION_INST_BR_UNCOND_ABBREV,
+  FUNCTION_INST_BR_COND_ABBREV,
   FUNCTION_INST_UNREACHABLE_ABBREV,
   FUNCTION_INST_GEP_ABBREV,
+  FUNCTION_INST_CMP_ABBREV,
+  FUNCTION_INST_CMP_FLAGS_ABBREV,
   FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
+  FUNCTION_DEBUG_LOC_ABBREV,
 };
 
 /// Abstract class to manage the bitcode writing, subclassed for each bitcode
@@ -945,6 +951,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_NO_EXT;
   case Attribute::Captures:
     return bitc::ATTR_KIND_CAPTURES;
+  case Attribute::DeadOnReturn:
+    return bitc::ATTR_KIND_DEAD_ON_RETURN;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
@@ -1806,12 +1814,14 @@ unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
   // location (it's never more expensive than building an array size 1).
   auto Abbv = std::make_shared<BitCodeAbbrev>();
   Abbv->Add(BitCodeAbbrevOp(bitc::METADATA_LOCATION));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
-  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isDistinct
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // line
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));   // column
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // scope
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // inlinedAt
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isImplicitCode
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // atomGroup
+  Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // atomRank
   return Stream.EmitAbbrev(std::move(Abbv));
 }
 
@@ -1827,7 +1837,8 @@ void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
   Record.push_back(VE.getMetadataID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getInlinedAt()));
   Record.push_back(N->isImplicitCode());
-
+  Record.push_back(N->getAtomGroup());
+  Record.push_back(N->getAtomRank());
   Stream.EmitRecord(bitc::METADATA_LOCATION, Record, Abbrev);
   Record.clear();
 }
@@ -2149,6 +2160,7 @@ void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
   Record.push_back(VE.getMetadataOrNullID(N->getThrownTypes().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getAnnotations().get()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawTargetFuncName()));
+  Record.push_back(N->getKeyInstructionsEnabled());
 
   Stream.EmitRecord(bitc::METADATA_SUBPROGRAM, Record, Abbrev);
   Record.clear();
@@ -2347,11 +2359,16 @@ void ModuleBitcodeWriter::writeDILocalVariable(
 void ModuleBitcodeWriter::writeDILabel(
     const DILabel *N, SmallVectorImpl<uint64_t> &Record,
     unsigned Abbrev) {
-  Record.push_back((uint64_t)N->isDistinct());
+  uint64_t IsArtificialFlag = uint64_t(N->isArtificial()) << 1;
+  Record.push_back((uint64_t)N->isDistinct() | IsArtificialFlag);
   Record.push_back(VE.getMetadataOrNullID(N->getScope()));
   Record.push_back(VE.getMetadataOrNullID(N->getRawName()));
   Record.push_back(VE.getMetadataOrNullID(N->getFile()));
   Record.push_back(N->getLine());
+  Record.push_back(N->getColumn());
+  Record.push_back(N->getCoroSuspendIdx().has_value()
+                       ? (uint64_t)N->getCoroSuspendIdx().value()
+                       : std::numeric_limits<uint64_t>::max());
 
   Stream.EmitRecord(bitc::METADATA_LABEL, Record, Abbrev);
   Record.clear();
@@ -3279,12 +3296,17 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
   case Instruction::FCmp: {
     // compare returning Int1Ty or vector of Int1Ty
     Code = bitc::FUNC_CODE_INST_CMP2;
-    pushValueAndType(I.getOperand(0), InstID, Vals);
+    AbbrevToUse = FUNCTION_INST_CMP_ABBREV;
+    if (pushValueAndType(I.getOperand(0), InstID, Vals))
+      AbbrevToUse = 0;
     pushValue(I.getOperand(1), InstID, Vals);
     Vals.push_back(cast<CmpInst>(I).getPredicate());
     uint64_t Flags = getOptimizationFlags(&I);
-    if (Flags != 0)
+    if (Flags != 0) {
       Vals.push_back(Flags);
+      if (AbbrevToUse)
+        AbbrevToUse = FUNCTION_INST_CMP_FLAGS_ABBREV;
+    }
     break;
   }
 
@@ -3306,11 +3328,13 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
   case Instruction::Br:
     {
       Code = bitc::FUNC_CODE_INST_BR;
+      AbbrevToUse = FUNCTION_INST_BR_UNCOND_ABBREV;
       const BranchInst &II = cast<BranchInst>(I);
       Vals.push_back(VE.getValueID(II.getSuccessor(0)));
       if (II.isConditional()) {
         Vals.push_back(VE.getValueID(II.getSuccessor(1)));
         pushValue(II.getCondition(), InstID, Vals);
+        AbbrevToUse = FUNCTION_INST_BR_COND_ABBREV;
       }
     }
     break;
@@ -3531,12 +3555,16 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
     }
     break;
   case Instruction::Store:
-    if (cast<StoreInst>(I).isAtomic())
+    if (cast<StoreInst>(I).isAtomic()) {
       Code = bitc::FUNC_CODE_INST_STOREATOMIC;
-    else
+    } else {
       Code = bitc::FUNC_CODE_INST_STORE;
-    pushValueAndType(I.getOperand(1), InstID, Vals); // ptrty + ptr
-    pushValueAndType(I.getOperand(0), InstID, Vals); // valty + val
+      AbbrevToUse = FUNCTION_INST_STORE_ABBREV;
+    }
+    if (pushValueAndType(I.getOperand(1), InstID, Vals)) // ptrty + ptr
+      AbbrevToUse = 0;
+    if (pushValueAndType(I.getOperand(0), InstID, Vals)) // valty + val
+      AbbrevToUse = 0;
     Vals.push_back(getEncodedAlign(cast<StoreInst>(I).getAlign()));
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
@@ -3759,7 +3787,7 @@ void ModuleBitcodeWriter::writeFunction(
   // in the VST.
   FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
 
-  Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 4);
+  Stream.EnterSubblock(bitc::FUNCTION_BLOCK_ID, 5);
   VE.incorporateFunction(F);
 
   SmallVector<unsigned, 64> Vals;
@@ -3808,7 +3836,10 @@ void ModuleBitcodeWriter::writeFunction(
           Vals.push_back(VE.getMetadataOrNullID(DL->getScope()));
           Vals.push_back(VE.getMetadataOrNullID(DL->getInlinedAt()));
           Vals.push_back(DL->isImplicitCode());
-          Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals);
+          Vals.push_back(DL->getAtomGroup());
+          Vals.push_back(DL->getAtomRank());
+          Stream.EmitRecord(bitc::FUNC_CODE_DEBUG_LOC, Vals,
+                            FUNCTION_DEBUG_LOC_ABBREV);
           Vals.clear();
           LastDL = DL;
         }
@@ -3935,6 +3966,12 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   // Other blocks can define their abbrevs inline.
   Stream.EnterBlockInfoBlock();
 
+  // Encode type indices using fixed size based on number of types.
+  BitCodeAbbrevOp TypeAbbrevOp(BitCodeAbbrevOp::Fixed,
+                               VE.computeBitsRequiredForTypeIndices());
+  // Encode value indices as 6-bit VBR.
+  BitCodeAbbrevOp ValAbbrevOp(BitCodeAbbrevOp::VBR, 6);
+
   { // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
@@ -3980,8 +4017,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // SETTYPE abbrev for CONSTANTS_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(TypeAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::CONSTANTS_BLOCK_ID, Abbv) !=
         CONSTANTS_SETTYPE_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
@@ -4021,19 +4057,29 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_LOAD abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(ValAbbrevOp); // Ptr
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_LOAD_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_STORE));
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4));   // align
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_STORE_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   { // INST_UNOP abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
+    Abbv->Add(ValAbbrevOp); // LHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_UNOP_ABBREV)
@@ -4042,7 +4088,7 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
+    Abbv->Add(ValAbbrevOp); // LHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -4052,8 +4098,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_BINOP abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
+    Abbv->Add(ValAbbrevOp); // LHS
+    Abbv->Add(ValAbbrevOp); // RHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_BINOP_ABBREV)
@@ -4062,8 +4108,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
+    Abbv->Add(ValAbbrevOp); // LHS
+    Abbv->Add(ValAbbrevOp); // RHS
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -4073,10 +4119,9 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_CAST abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));    // OpVal
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,       // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4));  // opc
+    Abbv->Add(ValAbbrevOp); // OpVal
+    Abbv->Add(TypeAbbrevOp); // dest ty
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_CAST_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
@@ -4084,9 +4129,8 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,    // dest ty
-                              VE.computeBitsRequiredForTypeIndices()));
+    Abbv->Add(ValAbbrevOp); // OpVal
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
@@ -4104,11 +4148,31 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   { // INST_RET abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID
+    Abbv->Add(ValAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_RET_VAL_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
+    // TODO: Use different abbrev for absolute value reference (succ0)?
+    Abbv->Add(ValAbbrevOp); // succ0
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_BR_UNCOND_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BR));
+    // TODO: Use different abbrev for absolute value references (succ0, succ1)?
+    Abbv->Add(ValAbbrevOp); // succ0
+    Abbv->Add(ValAbbrevOp); // succ1
+    Abbv->Add(ValAbbrevOp); // cond
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_BR_COND_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   { // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
@@ -4119,26 +4183,61 @@ void ModuleBitcodeWriter::writeBlockInfo() {
   {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
-                              Log2_32_Ceil(VE.getTypes().size() + 1)));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // flags
+    Abbv->Add(TypeAbbrevOp); // dest ty
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(ValAbbrevOp);
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_INST_GEP_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering!");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_CMP_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CMP2));
+    Abbv->Add(ValAbbrevOp);                                // op0
+    Abbv->Add(ValAbbrevOp);                                // op1
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 6)); // pred
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_INST_CMP_FLAGS_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   {
     auto Abbv = std::make_shared<BitCodeAbbrev>();
     Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
     Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
-    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // val
+    Abbv->Add(ValAbbrevOp); // val
     if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
         FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
       llvm_unreachable("Unexpected abbrev ordering! 1");
   }
+  {
+    auto Abbv = std::make_shared<BitCodeAbbrev>();
+    Abbv->Add(BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_LOC));
+    // NOTE: No IsDistinct field for FUNC_CODE_DEBUG_LOC.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Atom group.
+    Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 3)); // Atom rank.
+    if (Stream.EmitBlockInfoAbbrev(bitc::FUNCTION_BLOCK_ID, Abbv) !=
+        FUNCTION_DEBUG_LOC_ABBREV)
+      llvm_unreachable("Unexpected abbrev ordering!");
+  }
   Stream.ExitBlock();
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 4ed16c365336..c710795c2d52 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1530,16 +1530,13 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
     }
 
     if (!Features.OmitBBEntries) {
-      // TODO: Remove this check when version 1 is deprecated.
-      if (BBAddrMapVersion > 1) {
-        OutStreamer->AddComment("BB id");
-        // Emit the BB ID for this basic block.
-        // We only emit BaseID since CloneID is unset for
-        // -basic-block-adress-map.
-        // TODO: Emit the full BBID when labels and sections can be mixed
-        // together.
-        OutStreamer->emitULEB128IntValue(MBB.getBBID()->BaseID);
-      }
+      OutStreamer->AddComment("BB id");
+      // Emit the BB ID for this basic block.
+      // We only emit BaseID since CloneID is unset for
+      // -basic-block-adress-map.
+      // TODO: Emit the full BBID when labels and sections can be mixed
+      // together.
+      OutStreamer->emitULEB128IntValue(MBB.getBBID()->BaseID);
       // Emit the basic block offset relative to the end of the previous block.
       // This is zero unless the block is padded due to alignment.
       emitLabelDifferenceAsULEB128(MBBSymbol, PrevMBBEndSymbol);
@@ -2183,16 +2180,20 @@ void AsmPrinter::emitFunctionBody() {
 }
 
 /// Compute the number of Global Variables that uses a Constant.
-static unsigned getNumGlobalVariableUses(const Constant *C) {
-  if (!C)
+static unsigned getNumGlobalVariableUses(const Constant *C,
+                                         bool &HasNonGlobalUsers) {
+  if (!C) {
+    HasNonGlobalUsers = true;
     return 0;
+  }
 
   if (isa<GlobalVariable>(C))
     return 1;
 
   unsigned NumUses = 0;
   for (const auto *CU : C->users())
-    NumUses += getNumGlobalVariableUses(dyn_cast<Constant>(CU));
+    NumUses +=
+        getNumGlobalVariableUses(dyn_cast<Constant>(CU), HasNonGlobalUsers);
 
   return NumUses;
 }
@@ -2203,7 +2204,8 @@ static unsigned getNumGlobalVariableUses(const Constant *C) {
 /// candidates are skipped and are emitted later in case at least one cstexpr
 /// isn't replaced by a PC relative GOT entry access.
 static bool isGOTEquivalentCandidate(const GlobalVariable *GV,
-                                     unsigned &NumGOTEquivUsers) {
+                                     unsigned &NumGOTEquivUsers,
+                                     bool &HasNonGlobalUsers) {
   // Global GOT equivalents are unnamed private globals with a constant
   // pointer initializer to another global symbol. They must point to a
   // GlobalVariable or Function, i.e., as GlobalValue.
@@ -2215,7 +2217,8 @@ static bool isGOTEquivalentCandidate(const GlobalVariable *GV,
   // To be a got equivalent, at least one of its users need to be a constant
   // expression used by another global variable.
   for (const auto *U : GV->users())
-    NumGOTEquivUsers += getNumGlobalVariableUses(dyn_cast<Constant>(U));
+    NumGOTEquivUsers +=
+        getNumGlobalVariableUses(dyn_cast<Constant>(U), HasNonGlobalUsers);
 
   return NumGOTEquivUsers > 0;
 }
@@ -2233,9 +2236,13 @@ void AsmPrinter::computeGlobalGOTEquivs(Module &M) {
 
   for (const auto &G : M.globals()) {
     unsigned NumGOTEquivUsers = 0;
-    if (!isGOTEquivalentCandidate(&G, NumGOTEquivUsers))
+    bool HasNonGlobalUsers = false;
+    if (!isGOTEquivalentCandidate(&G, NumGOTEquivUsers, HasNonGlobalUsers))
       continue;
-
+    // If non-global variables use it, we still need to emit it.
+    // Add 1 here, then emit it in `emitGlobalGOTEquivs`.
+    if (HasNonGlobalUsers)
+      NumGOTEquivUsers += 1;
     const MCSymbol *GOTEquivSym = getSymbol(&G);
     GlobalGOTEquivs[GOTEquivSym] = std::make_pair(&G, NumGOTEquivUsers);
   }
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index ea57a8fa1f79..d37a22f321ce 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -669,6 +669,8 @@ void CodeViewDebug::endModule() {
   if (!Asm)
     return;
 
+  emitSecureHotPatchInformation();
+
   emitInlineeLinesSubsection();
 
   // Emit per-function debug information.
@@ -823,6 +825,28 @@ void CodeViewDebug::emitObjName() {
   endSymbolRecord(CompilerEnd);
 }
 
+void CodeViewDebug::emitSecureHotPatchInformation() {
+  MCSymbol *hotPatchInfo = nullptr;
+
+  for (const auto &F : MMI->getModule()->functions()) {
+    if (!F.isDeclarationForLinker() &&
+        F.hasFnAttribute("marked_for_windows_hot_patching")) {
+      if (hotPatchInfo == nullptr)
+        hotPatchInfo = beginCVSubsection(DebugSubsectionKind::Symbols);
+      MCSymbol *HotPatchEnd = beginSymbolRecord(SymbolKind::S_HOTPATCHFUNC);
+      auto *SP = F.getSubprogram();
+      OS.AddComment("Function");
+      OS.emitInt32(getFuncIdForSubprogram(SP).getIndex());
+      OS.AddComment("Name");
+      emitNullTerminatedSymbolName(OS, F.getName());
+      endSymbolRecord(HotPatchEnd);
+    }
+  }
+
+  if (hotPatchInfo != nullptr)
+    endCVSubsection(hotPatchInfo);
+}
+
 namespace {
 struct Version {
   int Part[4];
@@ -2061,8 +2085,8 @@ TypeIndex CodeViewDebug::lowerTypeFunction(const DISubroutineType *Ty) {
   ArrayRef<TypeIndex> ArgTypeIndices = {};
   if (!ReturnAndArgTypeIndices.empty()) {
     auto ReturnAndArgTypesRef = ArrayRef(ReturnAndArgTypeIndices);
-    ReturnTypeIndex = ReturnAndArgTypesRef.front();
-    ArgTypeIndices = ReturnAndArgTypesRef.drop_front();
+    ReturnTypeIndex = ReturnAndArgTypesRef.consume_front();
+    ArgTypeIndices = ReturnAndArgTypesRef;
   }
 
   ArgListRecord ArgListRec(TypeRecordKind::ArgList, ArgTypeIndices);
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index 5f4f30271d9c..c862802d835d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -337,6 +337,8 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
 
   void emitCompilerInformation();
 
+  void emitSecureHotPatchInformation();
+
   void emitBuildInfo();
 
   void emitInlineeLinesSubsection();
diff --git a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
index 4bbf66206bfb..a2252d8f00f0 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -472,7 +472,10 @@ unsigned DIEExpr::sizeOf(const dwarf::FormParams &FormParams,
 }
 
 LLVM_DUMP_METHOD
-void DIEExpr::print(raw_ostream &O) const { O << "Expr: " << *Expr; }
+void DIEExpr::print(raw_ostream &O) const {
+  O << "Expr: ";
+  MCAsmInfo().printExpr(O, *Expr);
+}
 
 //===----------------------------------------------------------------------===//
 // DIELabel Implementation
diff --git a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
index 5442fb15202e..171fb8394990 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
@@ -374,6 +374,18 @@ static void handleNewDebugValue(InlinedEntity Var, const MachineInstr &DV,
                                 DbgValueHistoryMap &HistMap) {
   EntryIndex NewIndex;
   if (HistMap.startDbgValue(Var, DV, NewIndex)) {
+    // As we already need to iterate all LiveEntries when handling a DbgValue,
+    // we use this map to avoid a more expensive check against RegVars. There
+    // is an assert that we handle this correctly in addRegDescribedVar.
+    //
+    // In other terms, the presence in this map indicates the presence of a
+    // corresponding entry in RegVars.
+    //
+    // The bool value then tracks whether an entry is to be retained (true) or
+    // removed (false); as we end previous entries we speculatively assume they
+    // can be dropped from RegVars, but we then also visit the new entry whose
+    // set of debug register operands may overlap and "save" a reg from being
+    // dropped.
     SmallDenseMap<unsigned, bool, 4> TrackedRegs;
 
     // If we have created a new debug value entry, close all preceding
@@ -467,9 +479,6 @@ void llvm::calculateDbgEntityHistory(const MachineFunction *MF,
     for (const auto &MI : MBB) {
       if (MI.isDebugValue()) {
         assert(MI.getNumOperands() > 1 && "Invalid DBG_VALUE instruction!");
-        // Use the base variable (without any DW_OP_piece expressions)
-        // as index into History. The full variables including the
-        // piece expressions are attached to the MI.
         const DILocalVariable *RawVar = MI.getDebugVariable();
         assert(RawVar->isValidLocationForIntrinsic(MI.getDebugLoc()) &&
                "Expected inlined-at fields to agree");
@@ -493,8 +502,7 @@ void llvm::calculateDbgEntityHistory(const MachineFunction *MF,
       if (MI.isMetaInstruction())
         continue;
 
-      // Not a DBG_VALUE instruction. It may clobber registers which describe
-      // some variables.
+      // Other instructions may clobber registers which describe some variables.
       for (const MachineOperand &MO : MI.operands()) {
         if (MO.isReg() && MO.isDef() && MO.getReg()) {
           // Ignore call instructions that claim to clobber SP. The AArch64
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index c512113ae2ac..772a09adb314 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -452,7 +452,7 @@ DIE *DwarfCompileUnit::getOrCreateCommonBlock(
   addString(NDie, dwarf::DW_AT_name, Name);
   addGlobalName(Name, NDie, CB->getScope());
   if (CB->getFile())
-    addSourceLine(NDie, CB->getLineNo(), CB->getFile());
+    addSourceLine(NDie, CB->getLineNo(), /*Column*/ 0, CB->getFile());
   if (DIGlobalVariable *V = CB->getDecl())
     getCU().addLocationAttribute(&NDie, V, GlobalExprs);
   return &NDie;
@@ -1467,7 +1467,7 @@ DIE *DwarfCompileUnit::constructImportedEntityDIE(
   else
     EntityDie = getDIE(Entity);
   assert(EntityDie);
-  addSourceLine(*IMDie, Module->getLine(), Module->getFile());
+  addSourceLine(*IMDie, Module->getLine(), /*Column*/ 0, Module->getFile());
   addDIEEntry(*IMDie, dwarf::DW_AT_import, *EntityDie);
   StringRef Name = Module->getName();
   if (!Name.empty()) {
@@ -1786,6 +1786,11 @@ void DwarfCompileUnit::applyLabelAttributes(const DbgLabel &Label,
     addString(LabelDie, dwarf::DW_AT_name, Name);
   const auto *DILabel = Label.getLabel();
   addSourceLine(LabelDie, DILabel);
+  if (DILabel->isArtificial())
+    addFlag(LabelDie, dwarf::DW_AT_artificial);
+  if (DILabel->getCoroSuspendIdx())
+    addUInt(LabelDie, dwarf::DW_AT_LLVM_coro_suspend_idx, std::nullopt,
+            *DILabel->getCoroSuspendIdx());
 }
 
 /// Add a Dwarf expression attribute data and value.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 68ef84fe4ce1..5fe141545283 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -169,8 +169,10 @@ static cl::opt<DwarfDebug::MinimizeAddrInV5> MinimizeAddrInV5Option(
                           "Stuff")),
     cl::init(DwarfDebug::MinimizeAddrInV5::Default));
 
-static cl::opt<bool> KeyInstructionsAreStmts("dwarf-use-key-instructions",
-                                             cl::Hidden, cl::init(false));
+/// Set to false to ignore Key Instructions metadata.
+static cl::opt<bool> KeyInstructionsAreStmts(
+    "dwarf-use-key-instructions", cl::Hidden, cl::init(true),
+    cl::desc("Set to false to ignore Key Instructions metadata"));
 
 static constexpr unsigned ULEB128PadSize = 4;
 
@@ -2153,8 +2155,17 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
   unsigned LastAsmLine =
       Asm->OutStreamer->getContext().getCurrentDwarfLoc().getLine();
 
+  // There may be a mixture of scopes using and not using Key Instructions.
+  // Not-Key-Instructions functions inlined into Key Instructions functions
+  // should use not-key is_stmt handling. Key Instructions functions inlined
+  // into Not-Key-Instructions functions should use Key Instructions is_stmt
+  // handling.
+  bool ScopeUsesKeyInstructions =
+      KeyInstructionsAreStmts && DL &&
+      DL->getScope()->getSubprogram()->getKeyInstructionsEnabled();
+
   bool IsKey = false;
-  if (KeyInstructionsAreStmts && DL && DL.getLine())
+  if (ScopeUsesKeyInstructions && DL && DL.getLine())
     IsKey = KeyInstructions.contains(MI);
 
   if (!DL && MI == PrologEndLoc) {
@@ -2234,7 +2245,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
     PrologEndLoc = nullptr;
   }
 
-  if (KeyInstructionsAreStmts) {
+  if (ScopeUsesKeyInstructions) {
     if (IsKey)
       Flags |= DWARF2_FLAG_IS_STMT;
   } else {
@@ -2448,6 +2459,8 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
            std::pair<uint8_t, SmallVector<const MachineInstr *, 2>>>
       GroupCandidates;
 
+  const auto &TII = *MF->getSubtarget().getInstrInfo();
+
   // For each instruction:
   //   * Skip insts without DebugLoc, AtomGroup or AtomRank, and line zeros.
   //   * Check if insts in this group have been seen already in GroupCandidates.
@@ -2476,24 +2489,20 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
       if (MI.isMetaInstruction())
         continue;
 
-      if (!MI.getDebugLoc() || !MI.getDebugLoc().getLine())
+      const DILocation *Loc = MI.getDebugLoc().get();
+      if (!Loc || !Loc->getLine())
         continue;
 
       // Reset the Buoy to this instruction if it has a different line number.
-      if (!Buoy ||
-          Buoy->getDebugLoc().getLine() != MI.getDebugLoc().getLine()) {
+      if (!Buoy || Buoy->getDebugLoc().getLine() != Loc->getLine()) {
         Buoy = &MI;
         BuoyAtom = 0; // Set later when we know which atom the buoy is used by.
       }
 
       // Call instructions are handled specially - we always mark them as key
       // regardless of atom info.
-      const auto &TII =
-          *MI.getParent()->getParent()->getSubtarget().getInstrInfo();
       bool IsCallLike = MI.isCall() || TII.isTailCall(MI);
       if (IsCallLike) {
-        assert(MI.getDebugLoc() && "Unexpectedly missing DL");
-
         // Calls are always key. Put the buoy (may not be the call) into
         // KeyInstructions directly rather than the candidate map to avoid it
         // being erased (and we may not have a group number for the call).
@@ -2503,14 +2512,13 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
         Buoy = nullptr;
         BuoyAtom = 0;
 
-        if (!MI.getDebugLoc()->getAtomGroup() ||
-            !MI.getDebugLoc()->getAtomRank())
+        if (!Loc->getAtomGroup() || !Loc->getAtomRank())
           continue;
       }
 
-      auto *InlinedAt = MI.getDebugLoc()->getInlinedAt();
-      uint64_t Group = MI.getDebugLoc()->getAtomGroup();
-      uint8_t Rank = MI.getDebugLoc()->getAtomRank();
+      auto *InlinedAt = Loc->getInlinedAt();
+      uint64_t Group = Loc->getAtomGroup();
+      uint8_t Rank = Loc->getAtomRank();
       if (!Group || !Rank)
         continue;
 
@@ -2552,8 +2560,8 @@ void DwarfDebug::computeKeyInstructions(const MachineFunction *MF) {
         CandidateInsts.push_back(Buoy);
         CandidateRank = Rank;
 
-        assert(!BuoyAtom || BuoyAtom == MI.getDebugLoc()->getAtomGroup());
-        BuoyAtom = MI.getDebugLoc()->getAtomGroup();
+        assert(!BuoyAtom || BuoyAtom == Loc->getAtomGroup());
+        BuoyAtom = Loc->getAtomGroup();
       } else {
         // Don't add calls, because they've been dealt with already. This means
         // CandidateInsts might now be empty - handle that.
@@ -2727,10 +2735,12 @@ void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) {
   PrologEndLoc = emitInitialLocDirective(
       *MF, Asm->OutStreamer->getContext().getDwarfCompileUnitID());
 
+  // Run both `findForceIsStmtInstrs` and `computeKeyInstructions` because
+  // Not-Key-Instructions functions may be inlined into Key Instructions
+  // functions and vice versa.
   if (KeyInstructionsAreStmts)
     computeKeyInstructions(MF);
-  else
-    findForceIsStmtInstrs(MF);
+  findForceIsStmtInstrs(MF);
 }
 
 unsigned
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 344b5e676925..41249c3936d9 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -526,6 +526,8 @@ bool DwarfExpression::addExpression(
   // and not any other parts of the following DWARF expression.
   assert(!IsEmittingEntryValue && "Can't emit entry value around expression");
 
+  if (!IsImplemented)
+    return false;
   IsPoisonedExpr = false;
 
   std::optional<DIExpression::ExprOperand> PrevConvertOp;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 9b76fd3c63ce..b925b6acfce4 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -460,49 +460,53 @@ void DwarfUnit::addBlock(DIE &Die, dwarf::Attribute Attribute,
   addBlock(Die, Attribute, Block->BestForm(), Block);
 }
 
-void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, const DIFile *File) {
+void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, unsigned Column,
+                              const DIFile *File) {
   if (Line == 0)
     return;
 
   unsigned FileID = getOrCreateSourceID(File);
   addUInt(Die, dwarf::DW_AT_decl_file, std::nullopt, FileID);
   addUInt(Die, dwarf::DW_AT_decl_line, std::nullopt, Line);
+
+  if (Column != 0)
+    addUInt(Die, dwarf::DW_AT_decl_column, std::nullopt, Column);
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DILocalVariable *V) {
   assert(V);
 
-  addSourceLine(Die, V->getLine(), V->getFile());
+  addSourceLine(Die, V->getLine(), /*Column*/ 0, V->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIGlobalVariable *G) {
   assert(G);
 
-  addSourceLine(Die, G->getLine(), G->getFile());
+  addSourceLine(Die, G->getLine(), /*Column*/ 0, G->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DISubprogram *SP) {
   assert(SP);
 
-  addSourceLine(Die, SP->getLine(), SP->getFile());
+  addSourceLine(Die, SP->getLine(), /*Column*/ 0, SP->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DILabel *L) {
   assert(L);
 
-  addSourceLine(Die, L->getLine(), L->getFile());
+  addSourceLine(Die, L->getLine(), L->getColumn(), L->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIType *Ty) {
   assert(Ty);
 
-  addSourceLine(Die, Ty->getLine(), Ty->getFile());
+  addSourceLine(Die, Ty->getLine(), /*Column*/ 0, Ty->getFile());
 }
 
 void DwarfUnit::addSourceLine(DIE &Die, const DIObjCProperty *Ty) {
   assert(Ty);
 
-  addSourceLine(Die, Ty->getLine(), Ty->getFile());
+  addSourceLine(Die, Ty->getLine(), /*Column*/ 0, Ty->getFile());
 }
 
 void DwarfUnit::addConstantFPValue(DIE &Die, const ConstantFP *CFP) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index fbb3df167c4a..28ef4b06eee8 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -218,7 +218,8 @@ class DwarfUnit : public DIEUnit {
                 DIEBlock *Block);
 
   /// Add location information to specified debug information entry.
-  void addSourceLine(DIE &Die, unsigned Line, const DIFile *File);
+  void addSourceLine(DIE &Die, unsigned Line, unsigned Column,
+                     const DIFile *File);
   void addSourceLine(DIE &Die, const DILocalVariable *V);
   void addSourceLine(DIE &Die, const DIGlobalVariable *G);
   void addSourceLine(DIE &Die, const DISubprogram *SP);
diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
index 55d1350e446a..dccd71fffe05 100644
--- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -308,10 +308,8 @@ void WinException::endFuncletImpl() {
 const MCExpr *WinException::create32bitRef(const MCSymbol *Value) {
   if (!Value)
     return MCConstantExpr::create(0, Asm->OutContext);
-  return MCSymbolRefExpr::create(Value, useImageRel32
-                                            ? MCSymbolRefExpr::VK_COFF_IMGREL32
-                                            : MCSymbolRefExpr::VK_None,
-                                 Asm->OutContext);
+  auto Spec = useImageRel32 ? uint16_t(MCSymbolRefExpr::VK_COFF_IMGREL32) : 0;
+  return MCSymbolRefExpr::create(Value, Spec, Asm->OutContext);
 }
 
 const MCExpr *WinException::create32bitRef(const GlobalValue *GV) {
diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
index 7da01e331b5a..ff265b505b5d 100644
--- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
+++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp
@@ -328,9 +328,6 @@ getDerefOffsetInBytes(const DIExpression *DIExpr) {
 
 /// A whole (unfragmented) source variable.
 using DebugAggregate = std::pair<const DILocalVariable *, const DILocation *>;
-static DebugAggregate getAggregate(const DbgVariableIntrinsic *DII) {
-  return DebugAggregate(DII->getVariable(), DII->getDebugLoc().getInlinedAt());
-}
 static DebugAggregate getAggregate(const DebugVariable &Var) {
   return DebugAggregate(Var.getVariable(), Var.getInlinedAt());
 }
@@ -1028,14 +1025,13 @@ class AssignmentTrackingLowering {
   /// i.e. for all values x and y where x != y:
   /// join(x, x) = x
   /// join(x, y) = NoneOrPhi
-  using AssignRecord = PointerUnion<DbgAssignIntrinsic *, DbgVariableRecord *>;
   struct Assignment {
     enum S { Known, NoneOrPhi } Status;
     /// ID of the assignment. nullptr if Status is not Known.
     DIAssignID *ID;
     /// The dbg.assign that marks this dbg-def. Mem-defs don't use this field.
     /// May be nullptr.
-    AssignRecord Source;
+    DbgVariableRecord *Source = nullptr;
 
     bool isSameSourceAssignment(const Assignment &Other) const {
       // Don't include Source in the equality check. Assignments are
@@ -1050,26 +1046,18 @@ class AssignmentTrackingLowering {
       else
         OS << "null";
       OS << ", s=";
-      if (Source.isNull())
+      if (!Source)
         OS << "null";
-      else if (const auto *DAI = dyn_cast<DbgAssignIntrinsic *>(Source))
-        OS << DAI;
       else
-        OS << cast<DbgVariableRecord *>(Source);
+        OS << Source;
       OS << ")";
     }
 
-    static Assignment make(DIAssignID *ID, DbgAssignIntrinsic *Source) {
-      return Assignment(Known, ID, Source);
-    }
     static Assignment make(DIAssignID *ID, DbgVariableRecord *Source) {
-      assert(Source->isDbgAssign() &&
+      assert((!Source || Source->isDbgAssign()) &&
              "Cannot make an assignment from a non-assign DbgVariableRecord");
       return Assignment(Known, ID, Source);
     }
-    static Assignment make(DIAssignID *ID, AssignRecord Source) {
-      return Assignment(Known, ID, Source);
-    }
     static Assignment makeFromMemDef(DIAssignID *ID) {
       return Assignment(Known, ID);
     }
@@ -1080,21 +1068,11 @@ class AssignmentTrackingLowering {
       // If the Status is Known then we expect there to be an assignment ID.
       assert(Status == NoneOrPhi || ID);
     }
-    Assignment(S Status, DIAssignID *ID, DbgAssignIntrinsic *Source)
-        : Status(Status), ID(ID), Source(Source) {
-      // If the Status is Known then we expect there to be an assignment ID.
-      assert(Status == NoneOrPhi || ID);
-    }
     Assignment(S Status, DIAssignID *ID, DbgVariableRecord *Source)
         : Status(Status), ID(ID), Source(Source) {
       // If the Status is Known then we expect there to be an assignment ID.
       assert(Status == NoneOrPhi || ID);
     }
-    Assignment(S Status, DIAssignID *ID, AssignRecord Source)
-        : Status(Status), ID(ID), Source(Source) {
-      // If the Status is Known then we expect there to be an assignment ID.
-      assert(Status == NoneOrPhi || ID);
-    }
   };
 
   using AssignmentMap = SmallVector<Assignment>;
@@ -1127,15 +1105,7 @@ class AssignmentTrackingLowering {
   void resetInsertionPoint(Instruction &After);
   void resetInsertionPoint(DbgVariableRecord &After);
 
-  // emitDbgValue can be called with:
-  //   Source=[AssignRecord|DbgValueInst*|DbgAssignIntrinsic*|DbgVariableRecord*]
-  // Since AssignRecord can be cast to one of the latter two types, and all
-  // other types have a shared interface, we use a template to handle the latter
-  // three types, and an explicit overload for AssignRecord that forwards to
-  // the template version with the right type.
-  void emitDbgValue(LocKind Kind, AssignRecord Source, VarLocInsertPt After);
-  template <typename T>
-  void emitDbgValue(LocKind Kind, const T Source, VarLocInsertPt After);
+  void emitDbgValue(LocKind Kind, DbgVariableRecord *, VarLocInsertPt After);
 
   static bool mapsAreEqual(const BitVector &Mask, const AssignmentMap &A,
                            const AssignmentMap &B) {
@@ -1353,7 +1323,6 @@ class AssignmentTrackingLowering {
   /// location information).
   ///@{
   void processNonDbgInstruction(Instruction &I, BlockInfo *LiveSet);
-  void processDbgInstruction(DbgInfoIntrinsic &I, BlockInfo *LiveSet);
   /// Update \p LiveSet after encountering an instruction with a DIAssignID
   /// attachment, \p I.
   void processTaggedInstruction(Instruction &I, BlockInfo *LiveSet);
@@ -1362,11 +1331,9 @@ class AssignmentTrackingLowering {
   void processUntaggedInstruction(Instruction &I, BlockInfo *LiveSet);
   void processUnknownStoreToVariable(Instruction &I, VariableID &Var,
                                      BlockInfo *LiveSet);
-  void processDbgAssign(AssignRecord Assign, BlockInfo *LiveSet);
+  void processDbgAssign(DbgVariableRecord *Assign, BlockInfo *LiveSet);
   void processDbgVariableRecord(DbgVariableRecord &DVR, BlockInfo *LiveSet);
-  void processDbgValue(
-      PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgValueRecord,
-      BlockInfo *LiveSet);
+  void processDbgValue(DbgVariableRecord *DbgValue, BlockInfo *LiveSet);
   /// Add an assignment to memory for the variable /p Var.
   void addMemDef(BlockInfo *LiveSet, VariableID Var, const Assignment &AV);
   /// Add an assignment to the variable /p Var.
@@ -1462,10 +1429,6 @@ static DIAssignID *getIDFromInst(const Instruction &I) {
   return cast<DIAssignID>(I.getMetadata(LLVMContext::MD_DIAssignID));
 }
 
-static DIAssignID *getIDFromMarker(const DbgAssignIntrinsic &DAI) {
-  return cast<DIAssignID>(DAI.getAssignID());
-}
-
 static DIAssignID *getIDFromMarker(const DbgVariableRecord &DVR) {
   assert(DVR.isDbgAssign() &&
          "Cannot get a DIAssignID from a non-assign DbgVariableRecord!");
@@ -1520,27 +1483,8 @@ VarLocInsertPt getNextNode(VarLocInsertPt InsertPt) {
   return getNextNode(cast<const DbgRecord *>(InsertPt));
 }
 
-DbgAssignIntrinsic *CastToDbgAssign(DbgVariableIntrinsic *DVI) {
-  return cast<DbgAssignIntrinsic>(DVI);
-}
-
-DbgVariableRecord *CastToDbgAssign(DbgVariableRecord *DVR) {
-  assert(DVR->isDbgAssign() &&
-         "Attempted to cast non-assign DbgVariableRecord to DVRAssign.");
-  return DVR;
-}
-
 void AssignmentTrackingLowering::emitDbgValue(
-    AssignmentTrackingLowering::LocKind Kind,
-    AssignmentTrackingLowering::AssignRecord Source, VarLocInsertPt After) {
-  if (isa<DbgAssignIntrinsic *>(Source))
-    emitDbgValue(Kind, cast<DbgAssignIntrinsic *>(Source), After);
-  else
-    emitDbgValue(Kind, cast<DbgVariableRecord *>(Source), After);
-}
-template <typename T>
-void AssignmentTrackingLowering::emitDbgValue(
-    AssignmentTrackingLowering::LocKind Kind, const T Source,
+    AssignmentTrackingLowering::LocKind Kind, DbgVariableRecord *Source,
     VarLocInsertPt After) {
 
   DILocation *DL = Source->getDebugLoc();
@@ -1566,7 +1510,8 @@ void AssignmentTrackingLowering::emitDbgValue(
 
   // NOTE: This block can mutate Kind.
   if (Kind == LocKind::Mem) {
-    const auto *Assign = CastToDbgAssign(Source);
+    assert(Source->isDbgAssign());
+    const DbgVariableRecord *Assign = Source;
     // Check the address hasn't been dropped (e.g. the debug uses may not have
     // been replaced before deleting a Value).
     if (Assign->isKillAddress()) {
@@ -1734,17 +1679,16 @@ void AssignmentTrackingLowering::processUntaggedInstruction(
 
 void AssignmentTrackingLowering::processTaggedInstruction(
     Instruction &I, AssignmentTrackingLowering::BlockInfo *LiveSet) {
-  auto Linked = at::getAssignmentMarkers(&I);
   auto LinkedDPAssigns = at::getDVRAssignmentMarkers(&I);
   // No dbg.assign intrinsics linked.
   // FIXME: All vars that have a stack slot this store modifies that don't have
   // a dbg.assign linked to it should probably treat this like an untagged
   // store.
-  if (Linked.empty() && LinkedDPAssigns.empty())
+  if (LinkedDPAssigns.empty())
     return;
 
   LLVM_DEBUG(dbgs() << "processTaggedInstruction on " << I << "\n");
-  auto ProcessLinkedAssign = [&](auto *Assign) {
+  for (DbgVariableRecord *Assign : LinkedDPAssigns) {
     VariableID Var = getVariableID(DebugVariable(Assign));
     // Something has gone wrong if VarsWithStackSlot doesn't contain a variable
     // that is linked to a store.
@@ -1815,112 +1759,82 @@ void AssignmentTrackingLowering::processTaggedInstruction(
       setLocKind(LiveSet, Var, LocKind::None);
     } break;
     }
-  };
-  for (DbgAssignIntrinsic *DAI : Linked)
-    ProcessLinkedAssign(DAI);
-  for (DbgVariableRecord *DVR : LinkedDPAssigns)
-    ProcessLinkedAssign(DVR);
+  }
 }
 
-void AssignmentTrackingLowering::processDbgAssign(AssignRecord Assign,
+void AssignmentTrackingLowering::processDbgAssign(DbgVariableRecord *DbgAssign,
                                                   BlockInfo *LiveSet) {
-  auto ProcessDbgAssignImpl = [&](auto *DbgAssign) {
-    // Only bother tracking variables that are at some point stack homed. Other
-    // variables can be dealt with trivially later.
-    if (!VarsWithStackSlot->count(getAggregate(DbgAssign)))
-      return;
-
-    VariableID Var = getVariableID(DebugVariable(DbgAssign));
-    Assignment AV = Assignment::make(getIDFromMarker(*DbgAssign), DbgAssign);
-    addDbgDef(LiveSet, Var, AV);
-
-    LLVM_DEBUG(dbgs() << "processDbgAssign on " << *DbgAssign << "\n";);
-    LLVM_DEBUG(dbgs() << "   LiveLoc " << locStr(getLocKind(LiveSet, Var))
-                      << " -> ");
+  // Only bother tracking variables that are at some point stack homed. Other
+  // variables can be dealt with trivially later.
+  if (!VarsWithStackSlot->count(getAggregate(DbgAssign)))
+    return;
 
-    // Check if the DebugValue and StackHomeValue both hold the same
-    // Assignment.
-    if (hasVarWithAssignment(LiveSet, BlockInfo::Stack, Var, AV)) {
-      // They match. We can use the stack home because the debug intrinsics
-      // state that an assignment happened here, and we know that specific
-      // assignment was the last one to take place in memory for this variable.
-      LocKind Kind;
-      if (DbgAssign->isKillAddress()) {
-        LLVM_DEBUG(
-            dbgs()
-                << "Val, Stack matches Debug program but address is killed\n";);
-        Kind = LocKind::Val;
-      } else {
-        LLVM_DEBUG(dbgs() << "Mem, Stack matches Debug program\n";);
-        Kind = LocKind::Mem;
-      };
-      setLocKind(LiveSet, Var, Kind);
-      emitDbgValue(Kind, DbgAssign, DbgAssign);
+  VariableID Var = getVariableID(DebugVariable(DbgAssign));
+  Assignment AV = Assignment::make(getIDFromMarker(*DbgAssign), DbgAssign);
+  addDbgDef(LiveSet, Var, AV);
+
+  LLVM_DEBUG(dbgs() << "processDbgAssign on " << *DbgAssign << "\n";);
+  LLVM_DEBUG(dbgs() << "   LiveLoc " << locStr(getLocKind(LiveSet, Var))
+                    << " -> ");
+
+  // Check if the DebugValue and StackHomeValue both hold the same
+  // Assignment.
+  if (hasVarWithAssignment(LiveSet, BlockInfo::Stack, Var, AV)) {
+    // They match. We can use the stack home because the debug intrinsics
+    // state that an assignment happened here, and we know that specific
+    // assignment was the last one to take place in memory for this variable.
+    LocKind Kind;
+    if (DbgAssign->isKillAddress()) {
+      LLVM_DEBUG(
+          dbgs()
+              << "Val, Stack matches Debug program but address is killed\n";);
+      Kind = LocKind::Val;
     } else {
-      // The last assignment to the memory location isn't the one that we want
-      // to show to the user so emit a dbg.value(Value). Value may be undef.
-      LLVM_DEBUG(dbgs() << "Val, Stack contents is unknown\n";);
-      setLocKind(LiveSet, Var, LocKind::Val);
-      emitDbgValue(LocKind::Val, DbgAssign, DbgAssign);
-    }
-  };
-  if (isa<DbgVariableRecord *>(Assign))
-    return ProcessDbgAssignImpl(cast<DbgVariableRecord *>(Assign));
-  return ProcessDbgAssignImpl(cast<DbgAssignIntrinsic *>(Assign));
+      LLVM_DEBUG(dbgs() << "Mem, Stack matches Debug program\n";);
+      Kind = LocKind::Mem;
+    };
+    setLocKind(LiveSet, Var, Kind);
+    emitDbgValue(Kind, DbgAssign, DbgAssign);
+  } else {
+    // The last assignment to the memory location isn't the one that we want
+    // to show to the user so emit a dbg.value(Value). Value may be undef.
+    LLVM_DEBUG(dbgs() << "Val, Stack contents is unknown\n";);
+    setLocKind(LiveSet, Var, LocKind::Val);
+    emitDbgValue(LocKind::Val, DbgAssign, DbgAssign);
+  }
 }
 
-void AssignmentTrackingLowering::processDbgValue(
-    PointerUnion<DbgValueInst *, DbgVariableRecord *> DbgValueRecord,
-    BlockInfo *LiveSet) {
-  auto ProcessDbgValueImpl = [&](auto *DbgValue) {
-    // Only other tracking variables that are at some point stack homed.
-    // Other variables can be dealt with trivally later.
-    if (!VarsWithStackSlot->count(getAggregate(DbgValue)))
-      return;
-
-    VariableID Var = getVariableID(DebugVariable(DbgValue));
-    // We have no ID to create an Assignment with so we mark this assignment as
-    // NoneOrPhi. Note that the dbg.value still exists, we just cannot determine
-    // the assignment responsible for setting this value.
-    // This is fine; dbg.values are essentially interchangable with unlinked
-    // dbg.assigns, and some passes such as mem2reg and instcombine add them to
-    // PHIs for promoted variables.
-    Assignment AV = Assignment::makeNoneOrPhi();
-    addDbgDef(LiveSet, Var, AV);
-
-    LLVM_DEBUG(dbgs() << "processDbgValue on " << *DbgValue << "\n";);
-    LLVM_DEBUG(dbgs() << "   LiveLoc " << locStr(getLocKind(LiveSet, Var))
-                      << " -> Val, dbg.value override");
+void AssignmentTrackingLowering::processDbgValue(DbgVariableRecord *DbgValue,
+                                                 BlockInfo *LiveSet) {
+  // Only other tracking variables that are at some point stack homed.
+  // Other variables can be dealt with trivally later.
+  if (!VarsWithStackSlot->count(getAggregate(DbgValue)))
+    return;
 
-    setLocKind(LiveSet, Var, LocKind::Val);
-    emitDbgValue(LocKind::Val, DbgValue, DbgValue);
-  };
-  if (isa<DbgVariableRecord *>(DbgValueRecord))
-    return ProcessDbgValueImpl(cast<DbgVariableRecord *>(DbgValueRecord));
-  return ProcessDbgValueImpl(cast<DbgValueInst *>(DbgValueRecord));
+  VariableID Var = getVariableID(DebugVariable(DbgValue));
+  // We have no ID to create an Assignment with so we mark this assignment as
+  // NoneOrPhi. Note that the dbg.value still exists, we just cannot determine
+  // the assignment responsible for setting this value.
+  // This is fine; dbg.values are essentially interchangable with unlinked
+  // dbg.assigns, and some passes such as mem2reg and instcombine add them to
+  // PHIs for promoted variables.
+  Assignment AV = Assignment::makeNoneOrPhi();
+  addDbgDef(LiveSet, Var, AV);
+
+  LLVM_DEBUG(dbgs() << "processDbgValue on " << *DbgValue << "\n";);
+  LLVM_DEBUG(dbgs() << "   LiveLoc " << locStr(getLocKind(LiveSet, Var))
+                    << " -> Val, dbg.value override");
+
+  setLocKind(LiveSet, Var, LocKind::Val);
+  emitDbgValue(LocKind::Val, DbgValue, DbgValue);
 }
 
-template <typename T> static bool hasZeroSizedFragment(T &DbgValue) {
+static bool hasZeroSizedFragment(DbgVariableRecord &DbgValue) {
   if (auto F = DbgValue.getExpression()->getFragmentInfo())
     return F->SizeInBits == 0;
   return false;
 }
 
-void AssignmentTrackingLowering::processDbgInstruction(
-    DbgInfoIntrinsic &I, AssignmentTrackingLowering::BlockInfo *LiveSet) {
-  auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I);
-  if (!DVI)
-    return;
-
-  // Ignore assignments to zero bits of the variable.
-  if (hasZeroSizedFragment(*DVI))
-    return;
-
-  if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
-    processDbgAssign(DAI, LiveSet);
-  else if (auto *DVI = dyn_cast<DbgValueInst>(&I))
-    processDbgValue(DVI, LiveSet);
-}
 void AssignmentTrackingLowering::processDbgVariableRecord(
     DbgVariableRecord &DVR, AssignmentTrackingLowering::BlockInfo *LiveSet) {
   // Ignore assignments to zero bits of the variable.
@@ -1964,14 +1878,12 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) {
       // attached DbgRecords, or a non-debug instruction with attached processed
       // DbgRecords.
       // II has not been processed.
-      if (!isa<DbgInfoIntrinsic>(&*II)) {
-        if (II->isTerminator())
-          break;
-        resetInsertionPoint(*II);
-        processNonDbgInstruction(*II, LiveSet);
-        assert(LiveSet->isValid());
-        ++II;
-      }
+      if (II->isTerminator())
+        break;
+      resetInsertionPoint(*II);
+      processNonDbgInstruction(*II, LiveSet);
+      assert(LiveSet->isValid());
+      ++II;
     }
     // II is now either a debug intrinsic, a non-debug instruction with no
     // attached DbgRecords, or a non-debug instruction with attached unprocessed
@@ -1987,15 +1899,6 @@ void AssignmentTrackingLowering::process(BasicBlock &BB, BlockInfo *LiveSet) {
       }
     }
     ProcessedLeadingDbgRecords = true;
-    while (II != EI) {
-      auto *Dbg = dyn_cast<DbgInfoIntrinsic>(&*II);
-      if (!Dbg)
-        break;
-      resetInsertionPoint(*II);
-      processDbgInstruction(*Dbg, LiveSet);
-      assert(LiveSet->isValid());
-      ++II;
-    }
     // II is now a non-debug instruction either with no attached DbgRecords, or
     // with attached processed DbgRecords. II has not been processed, and all
     // debug instructions or DbgRecords in the frame preceding II have been
@@ -2056,24 +1959,16 @@ AssignmentTrackingLowering::joinAssignment(const Assignment &A,
   // Here the same assignment (!1) was performed in both preds in the source,
   // but we can't use either one unless they are identical (e.g. .we don't
   // want to arbitrarily pick between constant values).
-  auto JoinSource = [&]() -> AssignRecord {
+  auto JoinSource = [&]() -> DbgVariableRecord * {
     if (A.Source == B.Source)
       return A.Source;
     if (!A.Source || !B.Source)
-      return AssignRecord();
-    assert(isa<DbgVariableRecord *>(A.Source) ==
-           isa<DbgVariableRecord *>(B.Source));
-    if (isa<DbgVariableRecord *>(A.Source) &&
-        cast<DbgVariableRecord *>(A.Source)->isEquivalentTo(
-            *cast<DbgVariableRecord *>(B.Source)))
-      return A.Source;
-    if (isa<DbgAssignIntrinsic *>(A.Source) &&
-        cast<DbgAssignIntrinsic *>(A.Source)->isIdenticalTo(
-            cast<DbgAssignIntrinsic *>(B.Source)))
+      return nullptr;
+    if (A.Source->isEquivalentTo(*B.Source))
       return A.Source;
-    return AssignRecord();
+    return nullptr;
   };
-  AssignRecord Source = JoinSource();
+  DbgVariableRecord *Source = JoinSource();
   assert(A.Status == B.Status && A.Status == Assignment::Known);
   assert(A.ID == B.ID);
   return Assignment::make(A.ID, Source);
@@ -2202,14 +2097,6 @@ AllocaInst *getUnknownStore(const Instruction &I, const DataLayout &Layout) {
   return dyn_cast<AllocaInst>(Base);
 }
 
-DbgDeclareInst *DynCastToDbgDeclare(DbgVariableIntrinsic *DVI) {
-  return dyn_cast<DbgDeclareInst>(DVI);
-}
-
-DbgVariableRecord *DynCastToDbgDeclare(DbgVariableRecord *DVR) {
-  return DVR->isDbgDeclare() ? DVR : nullptr;
-}
-
 /// Build a map of {Variable x: Variables y} where all variable fragments
 /// contained within the variable fragment x are in set y. This means that
 /// y does not contain all overlaps because partial overlaps are excluded.
@@ -2246,11 +2133,10 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
   //                     we can't determine the fragment overlap.
   // We need to add fragments for untagged stores too so that we can correctly
   // clobber overlapped fragment locations later.
-  SmallVector<DbgDeclareInst *> InstDeclares;
   SmallVector<DbgVariableRecord *> DPDeclares;
-  auto ProcessDbgRecord = [&](auto *Record, auto &DeclareList) {
-    if (auto *Declare = DynCastToDbgDeclare(Record)) {
-      DeclareList.push_back(Declare);
+  auto ProcessDbgRecord = [&](DbgVariableRecord *Record) {
+    if (Record->isDbgDeclare()) {
+      DPDeclares.push_back(Record);
       return;
     }
     DebugVariable DV = DebugVariable(Record);
@@ -2263,13 +2149,10 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
   for (auto &BB : Fn) {
     for (auto &I : BB) {
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
-        ProcessDbgRecord(&DVR, DPDeclares);
-      if (auto *DII = dyn_cast<DbgVariableIntrinsic>(&I)) {
-        ProcessDbgRecord(DII, InstDeclares);
-      } else if (auto Info = getUntaggedStoreAssignmentInfo(
-                     I, Fn.getDataLayout())) {
+        ProcessDbgRecord(&DVR);
+      if (auto Info = getUntaggedStoreAssignmentInfo(I, Fn.getDataLayout())) {
         // Find markers linked to this alloca.
-        auto HandleDbgAssignForStore = [&](auto *Assign) {
+        auto HandleDbgAssignForStore = [&](DbgVariableRecord *Assign) {
           std::optional<DIExpression::FragmentInfo> FragInfo;
 
           // Skip this assignment if the affected bits are outside of the
@@ -2302,13 +2185,11 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
           if (Seen.insert(DV).second)
             FragmentMap[DA].push_back(DV);
         };
-        for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(Info->Base))
-          HandleDbgAssignForStore(DAI);
         for (DbgVariableRecord *DVR : at::getDVRAssignmentMarkers(Info->Base))
           HandleDbgAssignForStore(DVR);
       } else if (auto *AI = getUnknownStore(I, Fn.getDataLayout())) {
         // Find markers linked to this alloca.
-        auto HandleDbgAssignForUnknownStore = [&](auto *Assign) {
+        auto HandleDbgAssignForUnknownStore = [&](DbgVariableRecord *Assign) {
           // Because we can't currently represent the fragment info for this
           // store, we treat it as an unusable store to the whole variable.
           DebugVariable DV =
@@ -2321,8 +2202,6 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
           // Cache this info for later.
           UnknownStoreVars[&I].push_back(FnVarLocs->insertVariable(DV));
         };
-        for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(AI))
-          HandleDbgAssignForUnknownStore(DAI);
         for (DbgVariableRecord *DVR : at::getDVRAssignmentMarkers(AI))
           HandleDbgAssignForUnknownStore(DVR);
       }
@@ -2371,9 +2250,6 @@ static AssignmentTrackingLowering::OverlapMap buildOverlapMapAndRecordDeclares(
 
   // Finally, insert the declares afterwards, so the first IDs are all
   // partially stack homed vars.
-  for (auto *DDI : InstDeclares)
-    FnVarLocs->addSingleLocVar(DebugVariable(DDI), DDI->getExpression(),
-                               DDI->getDebugLoc(), DDI->getWrappedLocation());
   for (auto *DVR : DPDeclares)
     FnVarLocs->addSingleLocVar(DebugVariable(DVR), DVR->getExpression(),
                                DVR->getDebugLoc(),
@@ -2556,7 +2432,7 @@ bool AssignmentTrackingLowering::emitPromotedVarLocs(
   bool InsertedAnyIntrinsics = false;
   // Go through every block, translating debug intrinsics for fully promoted
   // variables into FnVarLocs location defs. No analysis required for these.
-  auto TranslateDbgRecord = [&](auto *Record) {
+  auto TranslateDbgRecord = [&](DbgVariableRecord *Record) {
     // Skip variables that haven't been promoted - we've dealt with those
     // already.
     if (VarsWithStackSlot->contains(getAggregate(Record)))
@@ -2574,9 +2450,6 @@ bool AssignmentTrackingLowering::emitPromotedVarLocs(
       for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
         if (DVR.isDbgValue() || DVR.isDbgAssign())
           TranslateDbgRecord(&DVR);
-      auto *DVI = dyn_cast<DbgValueInst>(&I);
-      if (DVI)
-        TranslateDbgRecord(DVI);
     }
   }
   return InsertedAnyIntrinsics;
@@ -2598,10 +2471,8 @@ removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB,
   // FnVarLocs, because wedges in FnVarLocs may only be separated by debug
   // instructions.
   for (const Instruction &I : reverse(*BB)) {
-    if (!isa<DbgVariableIntrinsic>(I)) {
-      // Sequence of consecutive defs ended. Clear map for the next one.
-      VariableDefinedBytes.clear();
-    }
+    // Sequence of consecutive defs ended. Clear map for the next one.
+    VariableDefinedBytes.clear();
 
     auto HandleLocsForWedge = [&](auto *WedgePosition) {
       // Get the location defs that start just before this instruction.
@@ -2854,9 +2725,6 @@ static DenseSet<DebugAggregate> findVarsWithStackSlot(Function &Fn) {
       // DIAssignID might get dropped from an alloca but not stores. In that
       // case, we need to consider the variable interesting for NFC behaviour
       // with this change. TODO: Consider only looking at allocas.
-      for (DbgAssignIntrinsic *DAI : at::getAssignmentMarkers(&I)) {
-        Result.insert({DAI->getVariable(), DAI->getDebugLoc().getInlinedAt()});
-      }
       for (DbgVariableRecord *DVR : at::getDVRAssignmentMarkers(&I)) {
         Result.insert({DVR->getVariable(), DVR->getDebugLoc().getInlinedAt()});
       }
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index ff9f0ff5d5bc..3b3e7a418feb 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -42,6 +42,7 @@
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
@@ -933,7 +934,13 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
 
   // Sort by hash value so that blocks with identical end sequences sort
   // together.
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+  // If origin-tracking is enabled then MergePotentialElt is no longer a POD
+  // type, so we need std::sort instead.
+  std::sort(MergePotentials.begin(), MergePotentials.end());
+#else
   array_pod_sort(MergePotentials.begin(), MergePotentials.end());
+#endif
 
   // Walk through equivalence sets looking for actual exact matches.
   while (MergePotentials.size() > 1) {
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 5dd641343125..f8f9bbba53e4 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -250,6 +250,7 @@ add_llvm_component_library(LLVMCodeGen
   VirtRegMap.cpp
   WasmEHPrepare.cpp
   WindowScheduler.cpp
+  WindowsSecureHotPatching.cpp
   WinEHPrepare.cpp
   XRayInstrumentation.cpp
   ${GeneratedMLSources}
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 43574a54c37d..9bbb89e37865 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -6092,6 +6092,13 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
       }
 
       if (!ResultIndex) {
+        auto PtrInst = dyn_cast<Instruction>(ResultPtr);
+        // We know that we have a pointer without any offsets. If this pointer
+        // originates from a different basic block than the current one, we
+        // must be able to recreate it in the current basic block.
+        // We do not support the recreation of any instructions yet.
+        if (PtrInst && PtrInst->getParent() != MemoryInst->getParent())
+          return Modified;
         SunkAddr = ResultPtr;
       } else {
         if (ResultPtr->getType() != I8PtrTy)
diff --git a/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp b/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
index 4a3503a2da7d..e54419758410 100644
--- a/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
+++ b/llvm/lib/CodeGen/CodeGenTargetMachineImpl.cpp
@@ -80,6 +80,10 @@ void CodeGenTargetMachineImpl::initAsmInfo() {
 
   TmpAsmInfo->setFullRegisterNames(Options.MCOptions.PPCUseFullRegisterNames);
 
+  assert(TmpAsmInfo->getExceptionHandlingType() ==
+             getTargetTriple().getDefaultExceptionHandling() &&
+         "MCAsmInfo and Triple disagree on default exception handling type");
+
   if (Options.ExceptionModel != ExceptionHandling::None)
     TmpAsmInfo->setExceptionsType(Options.ExceptionModel);
 
@@ -197,7 +201,7 @@ CodeGenTargetMachineImpl::createMCStreamer(raw_pwrite_stream &Out,
       return make_error<StringError>("createMCAsmBackend failed",
                                      inconvertibleErrorCode());
 
-    Triple T(getTargetTriple().str());
+    Triple T(getTargetTriple());
     AsmStreamer.reset(getTarget().createMCObjectStreamer(
         T, Context, std::unique_ptr<MCAsmBackend>(MAB),
         DwoOut ? MAB->createDwoObjectWriter(Out, *DwoOut)
diff --git a/llvm/lib/CodeGen/FaultMaps.cpp b/llvm/lib/CodeGen/FaultMaps.cpp
index 3f8fe2402d65..79025e3ae80d 100644
--- a/llvm/lib/CodeGen/FaultMaps.cpp
+++ b/llvm/lib/CodeGen/FaultMaps.cpp
@@ -86,16 +86,8 @@ void FaultMaps::emitFunctionInfo(const MCSymbol *FnLabel,
   OS.emitInt32(0); // Reserved
 
   for (const auto &Fault : FFI) {
-    LLVM_DEBUG(dbgs() << WFMP << "    fault type: "
-                      << faultTypeToString(Fault.Kind) << "\n");
     OS.emitInt32(Fault.Kind);
-
-    LLVM_DEBUG(dbgs() << WFMP << "    faulting PC offset: "
-                      << *Fault.FaultingOffsetExpr << "\n");
     OS.emitValue(Fault.FaultingOffsetExpr, 4);
-
-    LLVM_DEBUG(dbgs() << WFMP << "    fault handler PC offset: "
-                      << *Fault.HandlerOffsetExpr << "\n");
     OS.emitValue(Fault.HandlerOffsetExpr, 4);
   }
 }
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index b1e851183de0..3b11d0848d30 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -3487,6 +3487,13 @@ bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
   if (!DstTy.getElementCount().isKnownMultipleOf(UnmergeSrcTy.getNumElements()))
     return false;
 
+  // Check the unmerge source and destination element types match
+  LLT UnmergeSrcEltTy = UnmergeSrcTy.getElementType();
+  Register UnmergeDstReg = UnmergeMI->getOperand(0).getReg();
+  LLT UnmergeDstEltTy = MRI.getType(UnmergeDstReg);
+  if (UnmergeSrcEltTy != UnmergeDstEltTy)
+    return false;
+
   // Only generate legal instructions post-legalizer
   if (!IsPreLegalize) {
     LLT MidTy = DstTy.changeElementType(UnmergeSrcTy.getScalarType());
@@ -5288,12 +5295,13 @@ bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
   return false;
 }
 
-MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
-  assert(MI.getOpcode() == TargetOpcode::G_UDIV);
-  auto &UDiv = cast<GenericMachineInstr>(MI);
-  Register Dst = UDiv.getReg(0);
-  Register LHS = UDiv.getReg(1);
-  Register RHS = UDiv.getReg(2);
+MachineInstr *CombinerHelper::buildUDivorURemUsingMul(MachineInstr &MI) const {
+  unsigned Opcode = MI.getOpcode();
+  assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
+  auto &UDivorRem = cast<GenericMachineInstr>(MI);
+  Register Dst = UDivorRem.getReg(0);
+  Register LHS = UDivorRem.getReg(1);
+  Register RHS = UDivorRem.getReg(2);
   LLT Ty = MRI.getType(Dst);
   LLT ScalarTy = Ty.getScalarType();
   const unsigned EltBits = ScalarTy.getScalarSizeInBits();
@@ -5446,11 +5454,18 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
   auto IsOne = MIB.buildICmp(
       CmpInst::Predicate::ICMP_EQ,
       Ty.isScalar() ? LLT::scalar(1) : Ty.changeElementSize(1), RHS, One);
-  return MIB.buildSelect(Ty, IsOne, LHS, Q);
+  auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);
+
+  if (Opcode == TargetOpcode::G_UREM) {
+    auto Prod = MIB.buildMul(Ty, ret, RHS);
+    return MIB.buildSub(Ty, LHS, Prod);
+  }
+  return ret;
 }
 
-bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
-  assert(MI.getOpcode() == TargetOpcode::G_UDIV);
+bool CombinerHelper::matchUDivorURemByConst(MachineInstr &MI) const {
+  unsigned Opcode = MI.getOpcode();
+  assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
   Register Dst = MI.getOperand(0).getReg();
   Register RHS = MI.getOperand(2).getReg();
   LLT DstTy = MRI.getType(Dst);
@@ -5467,7 +5482,8 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
   if (MF.getFunction().hasMinSize())
     return false;
 
-  if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
+  if (Opcode == TargetOpcode::G_UDIV &&
+      MI.getFlag(MachineInstr::MIFlag::IsExact)) {
     return matchUnaryPredicate(
         MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
   }
@@ -5487,14 +5503,17 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
              {DstTy.isVector() ? DstTy.changeElementSize(1) : LLT::scalar(1),
               DstTy}}))
       return false;
+    if (Opcode == TargetOpcode::G_UREM &&
+        !isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
+      return false;
   }
 
   return matchUnaryPredicate(
       MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
 }
 
-void CombinerHelper::applyUDivByConst(MachineInstr &MI) const {
-  auto *NewMI = buildUDivUsingMul(MI);
+void CombinerHelper::applyUDivorURemByConst(MachineInstr &MI) const {
+  auto *NewMI = buildUDivorURemUsingMul(MI);
   replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index d777567c6387..dca2a502e711 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1681,7 +1681,7 @@ bool IRTranslator::translateGetElementPtr(const User &U,
     auto OffsetMIB =
         MIRBuilder.buildConstant(OffsetTy, Offset);
 
-    if (int64_t(Offset) >= 0 && cast<GEPOperator>(U).isInBounds())
+    if (Offset >= 0 && cast<GEPOperator>(U).isInBounds())
       Flags |= MachineInstr::MIFlag::NoUWrap;
 
     MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 5f5af5cad778..cdf192f9e7e3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -469,6 +469,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     RTLIBCASE(COSH_F);
   case TargetOpcode::G_FTANH:
     RTLIBCASE(TANH_F);
+  case TargetOpcode::G_FSINCOS:
+    RTLIBCASE(SINCOS_F);
   case TargetOpcode::G_FLOG10:
     RTLIBCASE(LOG10_F);
   case TargetOpcode::G_FLOG:
@@ -648,6 +650,54 @@ simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
                        LocObserver, &MI);
 }
 
+LegalizerHelper::LegalizeResult LegalizerHelper::emitSincosLibcall(
+    MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType,
+    LostDebugLocObserver &LocObserver) {
+  MachineFunction &MF = *MI.getMF();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  Register DstSin = MI.getOperand(0).getReg();
+  Register DstCos = MI.getOperand(1).getReg();
+  Register Src = MI.getOperand(2).getReg();
+  LLT DstTy = MRI.getType(DstSin);
+
+  int MemSize = DstTy.getSizeInBytes();
+  Align Alignment = getStackTemporaryAlignment(DstTy);
+  const DataLayout &DL = MIRBuilder.getDataLayout();
+  unsigned AddrSpace = DL.getAllocaAddrSpace();
+  MachinePointerInfo PtrInfo;
+
+  Register StackPtrSin =
+      createStackTemporary(TypeSize::getFixed(MemSize), Alignment, PtrInfo)
+          .getReg(0);
+  Register StackPtrCos =
+      createStackTemporary(TypeSize::getFixed(MemSize), Alignment, PtrInfo)
+          .getReg(0);
+
+  auto &Ctx = MF.getFunction().getContext();
+  auto LibcallResult =
+      createLibcall(MIRBuilder, getRTLibDesc(MI.getOpcode(), Size),
+                    {{0}, Type::getVoidTy(Ctx), 0},
+                    {{Src, OpType, 0},
+                     {StackPtrSin, PointerType::get(Ctx, AddrSpace), 1},
+                     {StackPtrCos, PointerType::get(Ctx, AddrSpace), 2}},
+                    LocObserver, &MI);
+
+  if (LibcallResult != LegalizeResult::Legalized)
+    return LegalizerHelper::UnableToLegalize;
+
+  MachineMemOperand *LoadMMOSin = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOLoad, MemSize, Alignment);
+  MachineMemOperand *LoadMMOCos = MF.getMachineMemOperand(
+      PtrInfo, MachineMemOperand::MOLoad, MemSize, Alignment);
+
+  MIRBuilder.buildLoad(DstSin, StackPtrSin, *LoadMMOSin);
+  MIRBuilder.buildLoad(DstCos, StackPtrCos, *LoadMMOCos);
+  MI.eraseFromParent();
+
+  return LegalizerHelper::Legalized;
+}
+
 LegalizerHelper::LegalizeResult
 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                        MachineInstr &MI, LostDebugLocObserver &LocObserver) {
@@ -1275,6 +1325,16 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
       return Status;
     break;
   }
+  case TargetOpcode::G_FSINCOS: {
+    LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
+    unsigned Size = LLTy.getSizeInBits();
+    Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
+    if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
+      LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
+      return UnableToLegalize;
+    }
+    return emitSincosLibcall(MI, MIRBuilder, Size, HLTy, LocObserver);
+  }
   case TargetOpcode::G_LROUND:
   case TargetOpcode::G_LLROUND:
   case TargetOpcode::G_INTRINSIC_LRINT:
@@ -7635,6 +7695,22 @@ LegalizerHelper::lowerU64ToF64BitFloatOps(MachineInstr &MI) {
   return Legalized;
 }
 
+/// i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. We cannot
+/// convert fpround f64->f16 without double-rounding, so we manually perform the
+/// lowering here where we know it is valid.
+static LegalizerHelper::LegalizeResult
+loweri64tof16ITOFP(MachineInstr &MI, Register Dst, LLT DstTy, Register Src,
+                   LLT SrcTy, MachineIRBuilder &MIRBuilder) {
+  auto M1 = MI.getOpcode() == TargetOpcode::G_UITOFP
+                ? MIRBuilder.buildUITOFP(SrcTy, Src)
+                : MIRBuilder.buildSITOFP(SrcTy, Src);
+  LLT S32Ty = SrcTy.changeElementSize(32);
+  auto M2 = MIRBuilder.buildFPTrunc(S32Ty, M1);
+  MIRBuilder.buildFPTrunc(Dst, M2);
+  MI.eraseFromParent();
+  return LegalizerHelper::Legalized;
+}
+
 LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
   auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
 
@@ -7646,6 +7722,9 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
     return Legalized;
   }
 
+  if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+    return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
   if (SrcTy != LLT::scalar(64))
     return UnableToLegalize;
 
@@ -7677,6 +7756,9 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
     return Legalized;
   }
 
+  if (DstTy.getScalarSizeInBits() == 16 && SrcTy.getScalarSizeInBits() == 64)
+    return loweri64tof16ITOFP(MI, Dst, DstTy, Src, SrcTy, MIRBuilder);
+
   if (SrcTy != S64)
     return UnableToLegalize;
 
@@ -10039,10 +10121,6 @@ LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) {
   }
 
   bool IsVolatile = MemOp->isVolatile();
-  if (Opc == TargetOpcode::G_MEMCPY_INLINE)
-    return lowerMemcpyInline(MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
-                             IsVolatile);
-
   // Don't try to optimize volatile.
   if (IsVolatile)
     return UnableToLegalize;
diff --git a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
index 1c4150127a90..b2f84351c9a9 100644
--- a/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp
@@ -458,7 +458,7 @@ bool LoadStoreOpt::processMergeCandidate(StoreMergeCandidate &C) {
     for (auto AliasInfo : reverse(C.PotentialAliases)) {
       MachineInstr *PotentialAliasOp = AliasInfo.first;
       unsigned PreCheckedIdx = AliasInfo.second;
-      if (static_cast<unsigned>(Idx) < PreCheckedIdx) {
+      if (Idx < PreCheckedIdx) {
         // Once our store index is lower than the index associated with the
         // potential alias, we know that we've already checked for this alias
         // and all of the earlier potential aliases too.
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 2584175121d6..f48bfc06c14b 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -245,7 +245,7 @@ static void reportGISelDiagnostic(DiagnosticSeverity Severity,
     R << (" (in function: " + MF.getName() + ")").str();
 
   if (IsFatal)
-    report_fatal_error(Twine(R.getMsg()));
+    reportFatalUsageError(Twine(R.getMsg()));
   else
     MORE.emit(R);
 }
diff --git a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
index 84dc4ab0a552..92ecfadf97c9 100644
--- a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
+++ b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/CGData/CodeGenData.h"
 #include "llvm/CGData/CodeGenDataWriter.h"
+#include "llvm/CodeGen/Passes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/InitializePasses.h"
diff --git a/llvm/lib/CodeGen/IfConversion.cpp b/llvm/lib/CodeGen/IfConversion.cpp
index 5265bd74d2db..f80e1e8b683b 100644
--- a/llvm/lib/CodeGen/IfConversion.cpp
+++ b/llvm/lib/CodeGen/IfConversion.cpp
@@ -117,7 +117,11 @@ namespace {
     /// IsAnalyzed      - True if BB has been analyzed (info is still valid).
     /// IsEnqueued      - True if BB has been enqueued to be ifcvt'ed.
     /// IsBrAnalyzable  - True if analyzeBranch() returns false.
-    /// HasFallThrough  - True if BB may fallthrough to the following BB.
+    /// HasFallThrough  - True if BB has fallthrough to the following BB.
+    ///                   Note that BB may have a fallthrough if both
+    ///                   !HasFallThrough and !IsBrAnalyzable is true. Also note
+    ///                   that blockNeverFallThrough() can be used to prove that
+    ///                   there is no fall through.
     /// IsUnpredicable  - True if BB is known to be unpredicable.
     /// ClobbersPred    - True if BB could modify predicates (e.g. has
     ///                   cmp, call, etc.)
@@ -125,7 +129,10 @@ namespace {
     /// ExtraCost       - Extra cost for multi-cycle instructions.
     /// ExtraCost2      - Some instructions are slower when predicated
     /// BB              - Corresponding MachineBasicBlock.
-    /// TrueBB / FalseBB- See analyzeBranch().
+    /// TrueBB / FalseBB- See analyzeBranch(), but note that FalseBB can be set
+    ///                   by AnalyzeBranches even if there is a fallthrough. So
+    ///                   it doesn't correspond exactly to the result from
+    ///                   TTI::analyzeBranch.
     /// BrCond          - Conditions for end of block conditional branches.
     /// Predicate       - Predicate used in the BB.
     struct BBInfo {
@@ -397,6 +404,21 @@ namespace {
       return BBI.IsBrAnalyzable && BBI.TrueBB == nullptr;
     }
 
+    /// Returns true if Block is known not to fallthrough to the following BB.
+    bool blockNeverFallThrough(BBInfo &BBI) const {
+      // Trust "HasFallThrough" if we could analyze branches.
+      if (BBI.IsBrAnalyzable)
+        return !BBI.HasFallThrough;
+      // If this is the last MBB in the function, or if the textual successor
+      // isn't in the successor list, then there is no fallthrough.
+      MachineFunction::iterator PI = BBI.BB->getIterator();
+      MachineFunction::iterator I = std::next(PI);
+      if (I == BBI.BB->getParent()->end() || !PI->isSuccessor(&*I))
+        return true;
+      // Could not prove that there is no fallthrough.
+      return false;
+    }
+
     /// Used to sort if-conversion candidates.
     static bool IfcvtTokenCmp(const std::unique_ptr<IfcvtToken> &C1,
                               const std::unique_ptr<IfcvtToken> &C2) {
@@ -1715,9 +1737,8 @@ bool IfConverter::IfConvertTriangle(BBInfo &BBI, IfcvtKind Kind) {
     // Only merge them if the true block does not fallthrough to the false
     // block. By not merging them, we make it possible to iteratively
     // ifcvt the blocks.
-    if (!HasEarlyExit &&
-        NextMBB.pred_size() == 1 && !NextBBI->HasFallThrough &&
-        !NextMBB.hasAddressTaken()) {
+    if (!HasEarlyExit && NextMBB.pred_size() == 1 &&
+        blockNeverFallThrough(*NextBBI) && !NextMBB.hasAddressTaken()) {
       MergeBlocks(BBI, *NextBBI);
       FalseBBDead = true;
     } else {
@@ -2052,8 +2073,8 @@ bool IfConverter::IfConvertDiamond(BBInfo &BBI, IfcvtKind Kind,
     BBI.BB->removeSuccessor(FalseBBI.BB, true);
 
     BBInfo &TailBBI = BBAnalysis[TailBB->getNumber()];
-    bool CanMergeTail = !TailBBI.HasFallThrough &&
-      !TailBBI.BB->hasAddressTaken();
+    bool CanMergeTail =
+        blockNeverFallThrough(TailBBI) && !TailBBI.BB->hasAddressTaken();
     // The if-converted block can still have a predicated terminator
     // (e.g. a predicated return). If that is the case, we cannot merge
     // it with the tail block.
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 49f1504d244e..9c4c86cebe7e 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -629,173 +629,12 @@ static unsigned getIntrinsicFactor(const IntrinsicInst *II) {
   }
 }
 
-// For an (de)interleave tree like this:
-//
-//   A   C B   D
-//   |___| |___|
-//     |_____|
-//        |
-//     A B C D
-//
-//  We will get ABCD at the end while the leaf operands/results
-//  are ACBD, which are also what we initially collected in
-//  getVectorInterleaveFactor / getVectorDeinterleaveFactor. But TLI
-//  hooks (e.g. lowerDeinterleaveIntrinsicToLoad) expect ABCD, so we need
-//  to reorder them by interleaving these values.
-static void interleaveLeafValues(MutableArrayRef<Value *> SubLeaves) {
-  unsigned NumLeaves = SubLeaves.size();
-  assert(isPowerOf2_32(NumLeaves) && NumLeaves > 1);
-  if (NumLeaves == 2)
-    return;
-
-  const unsigned HalfLeaves = NumLeaves / 2;
-  // Visit the sub-trees.
-  interleaveLeafValues(SubLeaves.take_front(HalfLeaves));
-  interleaveLeafValues(SubLeaves.drop_front(HalfLeaves));
-
-  SmallVector<Value *, 8> Buffer;
-  //    a0 a1 a2 a3 b0 b1 b2 b3
-  // -> a0 b0 a1 b1 a2 b2 a3 b3
-  for (unsigned i = 0U; i < NumLeaves; ++i)
-    Buffer.push_back(SubLeaves[i / 2 + (i % 2 ? HalfLeaves : 0)]);
-
-  llvm::copy(Buffer, SubLeaves.begin());
-}
-
-static bool
-getVectorInterleaveFactor(IntrinsicInst *II, SmallVectorImpl<Value *> &Operands,
-                          SmallVectorImpl<Instruction *> &DeadInsts) {
-  assert(isInterleaveIntrinsic(II->getIntrinsicID()));
-
-  // Visit with BFS
-  SmallVector<IntrinsicInst *, 8> Queue;
-  Queue.push_back(II);
-  while (!Queue.empty()) {
-    IntrinsicInst *Current = Queue.front();
-    Queue.erase(Queue.begin());
-
-    // All the intermediate intrinsics will be deleted.
-    DeadInsts.push_back(Current);
-
-    for (unsigned I = 0; I < getIntrinsicFactor(Current); ++I) {
-      Value *Op = Current->getOperand(I);
-      if (auto *OpII = dyn_cast<IntrinsicInst>(Op))
-        if (OpII->getIntrinsicID() == Intrinsic::vector_interleave2) {
-          Queue.push_back(OpII);
-          continue;
-        }
-
-      // If this is not a perfectly balanced tree, the leaf
-      // result types would be different.
-      if (!Operands.empty() && Op->getType() != Operands.back()->getType())
-        return false;
-
-      Operands.push_back(Op);
-    }
-  }
-
-  const unsigned Factor = Operands.size();
-  // Currently we only recognize factors 2...8 and other powers of 2.
-  // FIXME: should we assert here instead?
-  if (Factor <= 1 ||
-      (!isPowerOf2_32(Factor) && Factor != getIntrinsicFactor(II)))
-    return false;
-
-  // Recursively interleaved factors need to have their values reordered
-  // TODO: Remove once the loop vectorizer no longer recursively interleaves
-  // factors 4 + 8
-  if (isPowerOf2_32(Factor) && getIntrinsicFactor(II) == 2)
-    interleaveLeafValues(Operands);
-  return true;
-}
-
-static bool
-getVectorDeinterleaveFactor(IntrinsicInst *II,
-                            SmallVectorImpl<Value *> &Results,
-                            SmallVectorImpl<Instruction *> &DeadInsts) {
-  assert(isDeinterleaveIntrinsic(II->getIntrinsicID()));
-  using namespace PatternMatch;
-  if (!II->hasNUses(getIntrinsicFactor(II)))
-    return false;
-
-  // Visit with BFS
-  SmallVector<IntrinsicInst *, 8> Queue;
-  Queue.push_back(II);
-  while (!Queue.empty()) {
-    IntrinsicInst *Current = Queue.front();
-    Queue.erase(Queue.begin());
-    assert(Current->hasNUses(getIntrinsicFactor(Current)));
-
-    // All the intermediate intrinsics will be deleted from the bottom-up.
-    DeadInsts.insert(DeadInsts.begin(), Current);
-
-    SmallVector<ExtractValueInst *> EVs(getIntrinsicFactor(Current), nullptr);
-    for (User *Usr : Current->users()) {
-      if (!isa<ExtractValueInst>(Usr))
-        return 0;
-
-      auto *EV = cast<ExtractValueInst>(Usr);
-      // Intermediate ExtractValue instructions will also be deleted.
-      DeadInsts.insert(DeadInsts.begin(), EV);
-      ArrayRef<unsigned> Indices = EV->getIndices();
-      if (Indices.size() != 1)
-        return false;
-
-      if (!EVs[Indices[0]])
-        EVs[Indices[0]] = EV;
-      else
-        return false;
-    }
-
-    // We have legal indices. At this point we're either going
-    // to continue the traversal or push the leaf values into Results.
-    for (ExtractValueInst *EV : EVs) {
-      // Continue the traversal. We're playing safe here and matching only the
-      // expression consisting of a perfectly balanced binary tree in which all
-      // intermediate values are only used once.
-      if (EV->hasOneUse() &&
-          match(EV->user_back(),
-                m_Intrinsic<Intrinsic::vector_deinterleave2>()) &&
-          EV->user_back()->hasNUses(2)) {
-        auto *EVUsr = cast<IntrinsicInst>(EV->user_back());
-        Queue.push_back(EVUsr);
-        continue;
-      }
-
-      // If this is not a perfectly balanced tree, the leaf
-      // result types would be different.
-      if (!Results.empty() && EV->getType() != Results.back()->getType())
-        return false;
-
-      // Save the leaf value.
-      Results.push_back(EV);
-    }
-  }
-
-  const unsigned Factor = Results.size();
-  // Currently we only recognize factors of 2...8 and other powers of 2.
-  // FIXME: should we assert here instead?
-  if (Factor <= 1 ||
-      (!isPowerOf2_32(Factor) && Factor != getIntrinsicFactor(II)))
-    return 0;
-
-  // Recursively interleaved factors need to have their values reordered
-  // TODO: Remove once the loop vectorizer no longer recursively interleaves
-  // factors 4 + 8
-  if (isPowerOf2_32(Factor) && getIntrinsicFactor(II) == 2)
-    interleaveLeafValues(Results);
-  return true;
-}
-
 static Value *getMask(Value *WideMask, unsigned Factor,
                       ElementCount LeafValueEC) {
   if (auto *IMI = dyn_cast<IntrinsicInst>(WideMask)) {
-    SmallVector<Value *, 8> Operands;
-    SmallVector<Instruction *, 8> DeadInsts;
-    if (getVectorInterleaveFactor(IMI, Operands, DeadInsts)) {
-      assert(!Operands.empty());
-      if (Operands.size() == Factor && llvm::all_equal(Operands))
-        return Operands[0];
+    if (isInterleaveIntrinsic(IMI->getIntrinsicID()) &&
+        getIntrinsicFactor(IMI) == Factor && llvm::all_equal(IMI->args())) {
+      return IMI->getArgOperand(0);
     }
   }
 
@@ -830,13 +669,19 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
   if (!LoadedVal->hasOneUse() || !isa<LoadInst, VPIntrinsic>(LoadedVal))
     return false;
 
-  SmallVector<Value *, 8> DeinterleaveValues;
-  SmallVector<Instruction *, 8> DeinterleaveDeadInsts;
-  if (!getVectorDeinterleaveFactor(DI, DeinterleaveValues,
-                                   DeinterleaveDeadInsts))
+  const unsigned Factor = getIntrinsicFactor(DI);
+  if (!DI->hasNUses(Factor))
     return false;
-
-  const unsigned Factor = DeinterleaveValues.size();
+  SmallVector<Value *, 8> DeinterleaveValues(Factor);
+  for (auto *User : DI->users()) {
+    auto *Extract = dyn_cast<ExtractValueInst>(User);
+    if (!Extract || Extract->getNumIndices() != 1)
+      return false;
+    unsigned Idx = Extract->getIndices()[0];
+    if (DeinterleaveValues[Idx])
+      return false;
+    DeinterleaveValues[Idx] = Extract;
+  }
 
   if (auto *VPLoad = dyn_cast<VPIntrinsic>(LoadedVal)) {
     if (VPLoad->getIntrinsicID() != Intrinsic::vp_load)
@@ -869,7 +714,9 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
       return false;
   }
 
-  DeadInsts.insert_range(DeinterleaveDeadInsts);
+  for (Value *V : DeinterleaveValues)
+    DeadInsts.insert(cast<Instruction>(V));
+  DeadInsts.insert(DI);
   // We now have a target-specific load, so delete the old one.
   DeadInsts.insert(cast<Instruction>(LoadedVal));
   return true;
@@ -883,12 +730,8 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
   if (!isa<StoreInst, VPIntrinsic>(StoredBy))
     return false;
 
-  SmallVector<Value *, 8> InterleaveValues;
-  SmallVector<Instruction *, 8> InterleaveDeadInsts;
-  if (!getVectorInterleaveFactor(II, InterleaveValues, InterleaveDeadInsts))
-    return false;
-
-  const unsigned Factor = InterleaveValues.size();
+  SmallVector<Value *, 8> InterleaveValues(II->args());
+  const unsigned Factor = getIntrinsicFactor(II);
 
   if (auto *VPStore = dyn_cast<VPIntrinsic>(StoredBy)) {
     if (VPStore->getIntrinsicID() != Intrinsic::vp_store)
@@ -922,7 +765,7 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
 
   // We now have a target-specific store, so delete the old one.
   DeadInsts.insert(cast<Instruction>(StoredBy));
-  DeadInsts.insert_range(InterleaveDeadInsts);
+  DeadInsts.insert(II);
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index d1cc12baa60e..238e9c5157e7 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -368,6 +368,7 @@ class TransferTracker {
     SmallVector<ResolvedDbgOp> ResolvedDbgOps;
     bool IsValueValid = true;
     unsigned LastUseBeforeDef = 0;
+    bool DbgLocAvailableAndIsEntryVal = false;
 
     // If every value used by the incoming DbgValue is available at block
     // entry, ResolvedDbgOps will contain the machine locations/constants for
@@ -421,6 +422,8 @@ class TransferTracker {
       // live range.
       LocIdx M = ValuesPreferredLoc->second.getLoc();
       ResolvedDbgOps.push_back(M);
+      if (Value.Properties.DIExpr->isEntryValue())
+        DbgLocAvailableAndIsEntryVal = true;
     }
 
     // If we cannot produce a valid value for the LiveIn value within this
@@ -434,6 +437,16 @@ class TransferTracker {
       return;
     }
 
+    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
+    PendingDbgValues.push_back(
+        std::make_pair(VarID, &*MTracker->emitLoc(ResolvedDbgOps, Var, DILoc,
+                                                  Value.Properties)));
+
+    // If the location is available at block entry and is an entry value, skip
+    // tracking and recording thr transfer.
+    if (DbgLocAvailableAndIsEntryVal)
+      return;
+
     // The LiveIn value is available at block entry, begin tracking and record
     // the transfer.
     for (const ResolvedDbgOp &Op : ResolvedDbgOps)
@@ -443,10 +456,6 @@ class TransferTracker {
     auto Result = ActiveVLocs.insert(std::make_pair(VarID, NewValue));
     if (!Result.second)
       Result.first->second = NewValue;
-    auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
-    PendingDbgValues.push_back(
-        std::make_pair(VarID, &*MTracker->emitLoc(ResolvedDbgOps, Var, DILoc,
-                                                  Value.Properties)));
   }
 
   /// Load object with live-in variable values. \p mlocs contains the live-in
@@ -677,6 +686,16 @@ class TransferTracker {
 
     auto &[Var, DILoc] = DVMap.lookupDVID(VarID);
 
+    // If the expression is a DW_OP_entry_value, emit the variable location
+    // as-is.
+    if (DIExpr->isEntryValue()) {
+      Register Reg = MTracker->LocIdxToLocID[Num.getLoc()];
+      MachineOperand MO = MachineOperand::CreateReg(Reg, false);
+      PendingDbgValues.push_back(std::make_pair(
+          VarID, &*emitMOLoc(MO, Var, {DIExpr, Prop.Indirect, false})));
+      return true;
+    }
+
     // Is the variable appropriate for entry values (i.e., is a parameter).
     if (!isEntryValueVariable(Var, DIExpr))
       return false;
@@ -703,7 +722,7 @@ class TransferTracker {
     DebugVariableID VarID = DVMap.getDVID(Var);
 
     // Ignore non-register locations, we don't transfer those.
-    if (MI.isUndefDebugValue() ||
+    if (MI.isUndefDebugValue() || MI.getDebugExpression()->isEntryValue() ||
         all_of(MI.debug_operands(),
                [](const MachineOperand &MO) { return !MO.isReg(); })) {
       auto It = ActiveVLocs.find(VarID);
diff --git a/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp b/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp
index 72e4be0165bf..15a175d6391d 100644
--- a/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp
+++ b/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp
@@ -14,21 +14,22 @@
 #include "llvm/CodeGen/MachineDomTreeUpdater.h"
 #include "llvm/Analysis/GenericDomTreeUpdaterImpl.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/Support/Compiler.h"
 
 namespace llvm {
 
-template class GenericDomTreeUpdater<
+template class LLVM_EXPORT_TEMPLATE GenericDomTreeUpdater<
     MachineDomTreeUpdater, MachineDominatorTree, MachinePostDominatorTree>;
 
-template void
+template LLVM_EXPORT_TEMPLATE void
 GenericDomTreeUpdater<MachineDomTreeUpdater, MachineDominatorTree,
                       MachinePostDominatorTree>::recalculate(MachineFunction
                                                                  &MF);
 
-template void GenericDomTreeUpdater<
+template LLVM_EXPORT_TEMPLATE void GenericDomTreeUpdater<
     MachineDomTreeUpdater, MachineDominatorTree,
     MachinePostDominatorTree>::applyUpdatesImpl</*IsForward=*/true>();
-template void GenericDomTreeUpdater<
+template LLVM_EXPORT_TEMPLATE void GenericDomTreeUpdater<
     MachineDomTreeUpdater, MachineDominatorTree,
     MachinePostDominatorTree>::applyUpdatesImpl</*IsForward=*/false>();
 
diff --git a/llvm/lib/CodeGen/MachineDominators.cpp b/llvm/lib/CodeGen/MachineDominators.cpp
index 917519f12a03..b221fa8b6de8 100644
--- a/llvm/lib/CodeGen/MachineDominators.cpp
+++ b/llvm/lib/CodeGen/MachineDominators.cpp
@@ -17,6 +17,7 @@
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
 
 using namespace llvm;
@@ -35,24 +36,29 @@ static cl::opt<bool, true> VerifyMachineDomInfoX(
     cl::desc("Verify machine dominator info (time consuming)"));
 
 namespace llvm {
-template class DomTreeNodeBase<MachineBasicBlock>;
-template class DominatorTreeBase<MachineBasicBlock, false>; // DomTreeBase
+template class LLVM_EXPORT_TEMPLATE DomTreeNodeBase<MachineBasicBlock>;
+template class LLVM_EXPORT_TEMPLATE
+    DominatorTreeBase<MachineBasicBlock, false>; // DomTreeBase
 
 namespace DomTreeBuilder {
-template void Calculate<MBBDomTree>(MBBDomTree &DT);
-template void CalculateWithUpdates<MBBDomTree>(MBBDomTree &DT, MBBUpdates U);
+template LLVM_EXPORT_TEMPLATE void Calculate<MBBDomTree>(MBBDomTree &DT);
+template LLVM_EXPORT_TEMPLATE void
+CalculateWithUpdates<MBBDomTree>(MBBDomTree &DT, MBBUpdates U);
 
-template void InsertEdge<MBBDomTree>(MBBDomTree &DT, MachineBasicBlock *From,
-                                     MachineBasicBlock *To);
+template LLVM_EXPORT_TEMPLATE void
+InsertEdge<MBBDomTree>(MBBDomTree &DT, MachineBasicBlock *From,
+                       MachineBasicBlock *To);
 
-template void DeleteEdge<MBBDomTree>(MBBDomTree &DT, MachineBasicBlock *From,
-                                     MachineBasicBlock *To);
+template LLVM_EXPORT_TEMPLATE void
+DeleteEdge<MBBDomTree>(MBBDomTree &DT, MachineBasicBlock *From,
+                       MachineBasicBlock *To);
 
-template void ApplyUpdates<MBBDomTree>(MBBDomTree &DT, MBBDomTreeGraphDiff &,
-                                       MBBDomTreeGraphDiff *);
+template LLVM_EXPORT_TEMPLATE void
+ApplyUpdates<MBBDomTree>(MBBDomTree &DT, MBBDomTreeGraphDiff &,
+                         MBBDomTreeGraphDiff *);
 
-template bool Verify<MBBDomTree>(const MBBDomTree &DT,
-                                 MBBDomTree::VerificationLevel VL);
+template LLVM_EXPORT_TEMPLATE bool
+Verify<MBBDomTree>(const MBBDomTree &DT, MBBDomTree::VerificationLevel VL);
 } // namespace DomTreeBuilder
 }
 
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index c9079170ca57..f1811c47e5ad 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -400,7 +400,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
     // Estimate register pressure during pre-regalloc pass.
     unsigned NumRPS = TRI->getNumRegPressureSets();
     RegPressure.resize(NumRPS);
-    std::fill(RegPressure.begin(), RegPressure.end(), 0);
+    llvm::fill(RegPressure, 0);
     RegLimit.resize(NumRPS);
     for (unsigned i = 0, e = NumRPS; i != e; ++i)
       RegLimit[i] = TRI->getRegPressureSetLimit(MF, i);
@@ -941,7 +941,7 @@ static bool isOperandKill(const MachineOperand &MO, MachineRegisterInfo *MRI) {
 /// initialize the starting "register pressure". Note this does not count live
 /// through (livein but not used) registers.
 void MachineLICMImpl::InitRegPressure(MachineBasicBlock *BB) {
-  std::fill(RegPressure.begin(), RegPressure.end(), 0);
+  llvm::fill(RegPressure, 0);
 
   // If the preheader has only a single predecessor and it ends with a
   // fallthrough or an unconditional branch, then scan its predecessor for live
@@ -1219,7 +1219,7 @@ bool MachineLICMImpl::HasHighOperandLatency(MachineInstr &MI, unsigned DefIdx,
 /// Return true if the instruction is marked "cheap" or the operand latency
 /// between its def and a use is one or less.
 bool MachineLICMImpl::IsCheapInstruction(MachineInstr &MI) const {
-  if (TII->isAsCheapAsAMove(MI) || MI.isCopyLike())
+  if (TII->isAsCheapAsAMove(MI) || MI.isSubregToReg())
     return true;
 
   bool isCheap = false;
diff --git a/llvm/lib/CodeGen/MachineLoopInfo.cpp b/llvm/lib/CodeGen/MachineLoopInfo.cpp
index 1c97e5c9063e..fdb1a470493c 100644
--- a/llvm/lib/CodeGen/MachineLoopInfo.cpp
+++ b/llvm/lib/CodeGen/MachineLoopInfo.cpp
@@ -22,13 +22,16 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/GenericLoopInfoImpl.h"
 
 using namespace llvm;
 
 // Explicitly instantiate methods in LoopInfoImpl.h for MI-level Loops.
-template class llvm::LoopBase<MachineBasicBlock, MachineLoop>;
-template class llvm::LoopInfoBase<MachineBasicBlock, MachineLoop>;
+template class LLVM_EXPORT_TEMPLATE
+    llvm::LoopBase<MachineBasicBlock, MachineLoop>;
+template class LLVM_EXPORT_TEMPLATE
+    llvm::LoopInfoBase<MachineBasicBlock, MachineLoop>;
 
 AnalysisKey MachineLoopAnalysis::Key;
 
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index e48612369a5d..b0bce2c21a47 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -67,6 +67,7 @@
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/IRBuilder.h"
@@ -77,6 +78,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/SuffixTree.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <tuple>
 #include <vector>
@@ -104,6 +106,7 @@ STATISTIC(StableHashAttempts,
           "Count of hashing attempts made for outlined functions");
 STATISTIC(StableHashDropped,
           "Count of unsuccessful hashing attempts for outlined functions");
+STATISTIC(NumRemovedLOHs, "Total number of Linker Optimization Hints removed");
 
 // Set to true if the user wants the outliner to run on linkonceodr linkage
 // functions. This is false by default because the linker can dedupe linkonceodr
@@ -426,6 +429,7 @@ struct MachineOutliner : public ModulePass {
   static char ID;
 
   MachineModuleInfo *MMI = nullptr;
+  const TargetMachine *TM = nullptr;
 
   /// Set to true if the outliner should consider functions with
   /// linkonceodr linkage.
@@ -461,6 +465,7 @@ struct MachineOutliner : public ModulePass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
     AU.addPreserved<MachineModuleInfoWrapperPass>();
     AU.addUsedIfAvailable<ImmutableModuleSummaryIndexWrapperPass>();
     AU.setPreservesAll();
@@ -1075,6 +1080,17 @@ bool MachineOutliner::outline(
                       << " B) > threshold (" << OutlinerBenefitThreshold
                       << " B)\n");
 
+    // Remove all Linker Optimization Hints from the candidates.
+    // TODO: The intersection of the LOHs from all candidates should be legal in
+    // the outlined function.
+    SmallPtrSet<MachineInstr *, 2> MIs;
+    for (Candidate &C : OF->Candidates) {
+      for (MachineInstr &MI : C)
+        MIs.insert(&MI);
+      NumRemovedLOHs += TM->clearLinkerOptimizationHints(MIs);
+      MIs.clear();
+    }
+
     // It's beneficial. Create the function and outline its sequence's
     // occurrences.
     OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum);
@@ -1386,6 +1402,7 @@ bool MachineOutliner::runOnModule(Module &M) {
   initializeOutlinerMode(M);
 
   MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
+  TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
 
   // Number to append to the current outlined function.
   unsigned OutlinedFunctionNum = 0;
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index bbe386507fcd..6e445f678790 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -17,20 +17,21 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManagerImpl.h"
+#include "llvm/Support/Compiler.h"
 
 using namespace llvm;
 
 AnalysisKey FunctionAnalysisManagerMachineFunctionProxy::Key;
 
 namespace llvm {
-template class AnalysisManager<MachineFunction>;
+template class LLVM_EXPORT_TEMPLATE AnalysisManager<MachineFunction>;
 template class PassManager<MachineFunction>;
-template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
-                                         Module>;
-template class InnerAnalysisManagerProxy<MachineFunctionAnalysisManager,
-                                         Function>;
-template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
-                                         MachineFunction>;
+template class LLVM_EXPORT_TEMPLATE
+    InnerAnalysisManagerProxy<MachineFunctionAnalysisManager, Module>;
+template class LLVM_EXPORT_TEMPLATE
+    InnerAnalysisManagerProxy<MachineFunctionAnalysisManager, Function>;
+template class LLVM_EXPORT_TEMPLATE
+    OuterAnalysisManagerProxy<ModuleAnalysisManager, MachineFunction>;
 } // namespace llvm
 
 bool FunctionAnalysisManagerMachineFunctionProxy::Result::invalidate(
diff --git a/llvm/lib/CodeGen/MachinePostDominators.cpp b/llvm/lib/CodeGen/MachinePostDominators.cpp
index 51637130addc..d6b5ee0e46d7 100644
--- a/llvm/lib/CodeGen/MachinePostDominators.cpp
+++ b/llvm/lib/CodeGen/MachinePostDominators.cpp
@@ -18,22 +18,25 @@
 using namespace llvm;
 
 namespace llvm {
-template class DominatorTreeBase<MachineBasicBlock, true>; // PostDomTreeBase
+template class LLVM_EXPORT_TEMPLATE
+    DominatorTreeBase<MachineBasicBlock, true>; // PostDomTreeBase
 
 namespace DomTreeBuilder {
 
-template void Calculate<MBBPostDomTree>(MBBPostDomTree &DT);
-template void InsertEdge<MBBPostDomTree>(MBBPostDomTree &DT,
-                                         MachineBasicBlock *From,
-                                         MachineBasicBlock *To);
-template void DeleteEdge<MBBPostDomTree>(MBBPostDomTree &DT,
-                                         MachineBasicBlock *From,
-                                         MachineBasicBlock *To);
-template void ApplyUpdates<MBBPostDomTree>(MBBPostDomTree &DT,
-                                           MBBPostDomTreeGraphDiff &,
-                                           MBBPostDomTreeGraphDiff *);
-template bool Verify<MBBPostDomTree>(const MBBPostDomTree &DT,
-                                     MBBPostDomTree::VerificationLevel VL);
+template LLVM_EXPORT_TEMPLATE void
+Calculate<MBBPostDomTree>(MBBPostDomTree &DT);
+template LLVM_EXPORT_TEMPLATE void
+InsertEdge<MBBPostDomTree>(MBBPostDomTree &DT, MachineBasicBlock *From,
+                           MachineBasicBlock *To);
+template LLVM_EXPORT_TEMPLATE void
+DeleteEdge<MBBPostDomTree>(MBBPostDomTree &DT, MachineBasicBlock *From,
+                           MachineBasicBlock *To);
+template LLVM_EXPORT_TEMPLATE void
+ApplyUpdates<MBBPostDomTree>(MBBPostDomTree &DT, MBBPostDomTreeGraphDiff &,
+                             MBBPostDomTreeGraphDiff *);
+template LLVM_EXPORT_TEMPLATE bool
+Verify<MBBPostDomTree>(const MBBPostDomTree &DT,
+                       MBBPostDomTree::VerificationLevel VL);
 
 } // namespace DomTreeBuilder
 extern bool VerifyMachineDomInfo;
@@ -96,8 +99,8 @@ MachineBasicBlock *MachinePostDominatorTree::findNearestCommonDominator(
     ArrayRef<MachineBasicBlock *> Blocks) const {
   assert(!Blocks.empty());
 
-  MachineBasicBlock *NCD = Blocks.front();
-  for (MachineBasicBlock *BB : Blocks.drop_front()) {
+  MachineBasicBlock *NCD = Blocks.consume_front();
+  for (MachineBasicBlock *BB : Blocks) {
     NCD = Base::findNearestCommonDominator(NCD, BB);
 
     // Stop when the root is reached.
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 7b4f29e82fe6..76cba2949af6 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -1290,13 +1290,13 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceTopDown() const {
                    SchedModel.getWriteProcResEnd(SC)));
 
     if (MISchedSortResourcesInTrace)
-      llvm::stable_sort(ResourcesIt,
-                        [](const MCWriteProcResEntry &LHS,
-                           const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
-                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
-                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
-                        });
+      llvm::stable_sort(
+          ResourcesIt,
+          [](const MCWriteProcResEntry &LHS,
+             const MCWriteProcResEntry &RHS) -> bool {
+            return std::tie(LHS.AcquireAtCycle, LHS.ReleaseAtCycle) <
+                   std::tie(RHS.AcquireAtCycle, RHS.ReleaseAtCycle);
+          });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
@@ -1371,13 +1371,13 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const {
                    SchedModel.getWriteProcResEnd(SC)));
 
     if (MISchedSortResourcesInTrace)
-      llvm::stable_sort(ResourcesIt,
-                        [](const MCWriteProcResEntry &LHS,
-                           const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
-                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
-                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
-                        });
+      llvm::stable_sort(
+          ResourcesIt,
+          [](const MCWriteProcResEntry &LHS,
+             const MCWriteProcResEntry &RHS) -> bool {
+            return std::tie(LHS.AcquireAtCycle, LHS.ReleaseAtCycle) <
+                   std::tie(RHS.AcquireAtCycle, RHS.ReleaseAtCycle);
+          });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
diff --git a/llvm/lib/CodeGen/RegAllocScore.cpp b/llvm/lib/CodeGen/RegAllocScore.cpp
index 8c140261c11c..b86647dbe0a4 100644
--- a/llvm/lib/CodeGen/RegAllocScore.cpp
+++ b/llvm/lib/CodeGen/RegAllocScore.cpp
@@ -23,13 +23,16 @@
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
-cl::opt<double> CopyWeight("regalloc-copy-weight", cl::init(0.2), cl::Hidden);
-cl::opt<double> LoadWeight("regalloc-load-weight", cl::init(4.0), cl::Hidden);
-cl::opt<double> StoreWeight("regalloc-store-weight", cl::init(1.0), cl::Hidden);
-cl::opt<double> CheapRematWeight("regalloc-cheap-remat-weight", cl::init(0.2),
-                                 cl::Hidden);
-cl::opt<double> ExpensiveRematWeight("regalloc-expensive-remat-weight",
-                                     cl::init(1.0), cl::Hidden);
+LLVM_ABI cl::opt<double> CopyWeight("regalloc-copy-weight", cl::init(0.2),
+                                    cl::Hidden);
+LLVM_ABI cl::opt<double> LoadWeight("regalloc-load-weight", cl::init(4.0),
+                                    cl::Hidden);
+LLVM_ABI cl::opt<double> StoreWeight("regalloc-store-weight", cl::init(1.0),
+                                     cl::Hidden);
+LLVM_ABI cl::opt<double> CheapRematWeight("regalloc-cheap-remat-weight",
+                                          cl::init(0.2), cl::Hidden);
+LLVM_ABI cl::opt<double> ExpensiveRematWeight("regalloc-expensive-remat-weight",
+                                              cl::init(1.0), cl::Hidden);
 #define DEBUG_TYPE "regalloc-score"
 
 RegAllocScore &RegAllocScore::operator+=(const RegAllocScore &Other) {
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index 2d25f097348a..2d7987a2e198 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -4152,10 +4152,8 @@ void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
   if (JoinGlobalCopies) {
     SmallVector<MachineInstr *, 2> LocalTerminals;
     SmallVector<MachineInstr *, 2> GlobalTerminals;
-    // Coalesce copies bottom-up to coalesce local defs before local uses. They
-    // are not inherently easier to resolve, but slightly preferable until we
-    // have local live range splitting. In particular this is required by
-    // cmp+jmp macro fusion.
+    // Coalesce copies top-down to propagate coalescing and rematerialization
+    // forward.
     for (MachineInstr &MI : *MBB) {
       if (!MI.isCopyLike())
         continue;
@@ -4177,6 +4175,8 @@ void RegisterCoalescer::copyCoalesceInMBB(MachineBasicBlock *MBB) {
     WorkList.append(GlobalTerminals.begin(), GlobalTerminals.end());
   } else {
     SmallVector<MachineInstr *, 2> Terminals;
+    // Coalesce copies top-down to propagate coalescing and rematerialization
+    // forward.
     for (MachineInstr &MII : *MBB)
       if (MII.isCopyLike()) {
         if (applyTerminalRule(MII))
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 591f4d2e5473..c90d48dcdb0c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -421,6 +421,7 @@ namespace {
     SDValue visitADDLike(SDNode *N);
     SDValue visitADDLikeCommutative(SDValue N0, SDValue N1,
                                     SDNode *LocReference);
+    SDValue visitPTRADD(SDNode *N);
     SDValue visitSUB(SDNode *N);
     SDValue visitADDSAT(SDNode *N);
     SDValue visitSUBSAT(SDNode *N);
@@ -637,7 +638,6 @@ namespace {
     SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex);
     SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
     SDValue combineInsertEltToLoad(SDNode *N, unsigned InsIndex);
-    SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
     SDValue BuildSDIV(SDNode *N);
     SDValue BuildSDIVPow2(SDNode *N);
     SDValue BuildUDIV(SDNode *N);
@@ -1140,7 +1140,7 @@ bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
       return true;
   }
 
-  if (Opc != ISD::ADD)
+  if (Opc != ISD::ADD && Opc != ISD::PTRADD)
     return false;
 
   auto *C2 = dyn_cast<ConstantSDNode>(N1);
@@ -1346,6 +1346,40 @@ SDValue DAGCombiner::reassociateReduction(unsigned RedOpc, unsigned Opc,
                        DAG.getNode(Opc, DL, N0.getOperand(0).getValueType(),
                                    N0.getOperand(0), N1.getOperand(0)));
   }
+
+  // Reassociate op(op(vecreduce(a), b), op(vecreduce(c), d)) into
+  // op(vecreduce(op(a, c)), op(b, d)), to combine the reductions into a
+  // single node.
+  SDValue A, B, C, D, RedA, RedB;
+  if (sd_match(N0, m_OneUse(m_c_BinOp(
+                       Opc,
+                       m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(A))),
+                               m_Value(RedA)),
+                       m_Value(B)))) &&
+      sd_match(N1, m_OneUse(m_c_BinOp(
+                       Opc,
+                       m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(C))),
+                               m_Value(RedB)),
+                       m_Value(D)))) &&
+      !sd_match(B, m_UnaryOp(RedOpc, m_Value())) &&
+      !sd_match(D, m_UnaryOp(RedOpc, m_Value())) &&
+      A.getValueType() == C.getValueType() &&
+      hasOperation(Opc, A.getValueType()) &&
+      TLI.shouldReassociateReduction(RedOpc, VT)) {
+    if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
+        (!N0->getFlags().hasAllowReassociation() ||
+         !N1->getFlags().hasAllowReassociation() ||
+         !RedA->getFlags().hasAllowReassociation() ||
+         !RedB->getFlags().hasAllowReassociation()))
+      return SDValue();
+    SelectionDAG::FlagInserter FlagsInserter(
+        DAG, Flags & N0->getFlags() & N1->getFlags() & RedA->getFlags() &
+                 RedB->getFlags());
+    SDValue Op = DAG.getNode(Opc, DL, A.getValueType(), A, C);
+    SDValue Red = DAG.getNode(RedOpc, DL, VT, Op);
+    SDValue Op2 = DAG.getNode(Opc, DL, VT, B, D);
+    return DAG.getNode(Opc, DL, VT, Red, Op2);
+  }
   return SDValue();
 }
 
@@ -1860,6 +1894,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::TokenFactor:        return visitTokenFactor(N);
   case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
   case ISD::ADD:                return visitADD(N);
+  case ISD::PTRADD:             return visitPTRADD(N);
   case ISD::SUB:                return visitSUB(N);
   case ISD::SADDSAT:
   case ISD::UADDSAT:            return visitADDSAT(N);
@@ -2630,6 +2665,86 @@ SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
   return SDValue();
 }
 
+/// Try to fold a pointer arithmetic node.
+/// This needs to be done separately from normal addition, because pointer
+/// addition is not commutative.
+SDValue DAGCombiner::visitPTRADD(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  EVT PtrVT = N0.getValueType();
+  EVT IntVT = N1.getValueType();
+  SDLoc DL(N);
+
+  // This is already ensured by an assert in SelectionDAG::getNode(). Several
+  // combines here depend on this assumption.
+  assert(PtrVT == IntVT &&
+         "PTRADD with different operand types is not supported");
+
+  // fold (ptradd x, 0) -> x
+  if (isNullConstant(N1))
+    return N0;
+
+  // fold (ptradd 0, x) -> x
+  if (PtrVT == IntVT && isNullConstant(N0))
+    return N1;
+
+  if (N0.getOpcode() != ISD::PTRADD ||
+      reassociationCanBreakAddressingModePattern(ISD::PTRADD, DL, N, N0, N1))
+    return SDValue();
+
+  SDValue X = N0.getOperand(0);
+  SDValue Y = N0.getOperand(1);
+  SDValue Z = N1;
+  bool N0OneUse = N0.hasOneUse();
+  bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+  bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+
+  // (ptradd (ptradd x, y), z) -> (ptradd x, (add y, z)) if:
+  //   * y is a constant and (ptradd x, y) has one use; or
+  //   * y and z are both constants.
+  if ((YIsConstant && N0OneUse) || (YIsConstant && ZIsConstant)) {
+    // If both additions in the original were NUW, the new ones are as well.
+    SDNodeFlags Flags =
+        (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+    SDValue Add = DAG.getNode(ISD::ADD, DL, IntVT, {Y, Z}, Flags);
+    AddToWorklist(Add.getNode());
+    return DAG.getMemBasePlusOffset(X, Add, DL, Flags);
+  }
+
+  // TODO: There is another possible fold here that was proven useful.
+  // It would be this:
+  //
+  // (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y) if:
+  //   * (ptradd x, y) has one use; and
+  //   * y is a constant; and
+  //   * z is not a constant.
+  //
+  // In some cases, specifically in AArch64's FEAT_CPA, it exposes the
+  // opportunity to select more complex instructions such as SUBPT and
+  // MSUBPT. However, a hypothetical corner case has been found that we could
+  // not avoid. Consider this (pseudo-POSIX C):
+  //
+  // char *foo(char *x, int z) {return (x + LARGE_CONSTANT) + z;}
+  // char *p = mmap(LARGE_CONSTANT);
+  // char *q = foo(p, -LARGE_CONSTANT);
+  //
+  // Then x + LARGE_CONSTANT is one-past-the-end, so valid, and a
+  // further + z takes it back to the start of the mapping, so valid,
+  // regardless of the address mmap gave back. However, if mmap gives you an
+  // address < LARGE_CONSTANT (ignoring high bits), x - LARGE_CONSTANT will
+  // borrow from the high bits (with the subsequent + z carrying back into
+  // the high bits to give you a well-defined pointer) and thus trip
+  // FEAT_CPA's pointer corruption checks.
+  //
+  // We leave this fold as an opportunity for future work, addressing the
+  // corner case for FEAT_CPA, as well as reconciling the solution with the
+  // more general application of pointer arithmetic in other future targets.
+  // For now each architecture that wants this fold must implement it in the
+  // target-specific code (see e.g. SITargetLowering::performPtrAddCombine)
+
+  return SDValue();
+}
+
 /// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
 /// a shift and add with a different constant.
 static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL,
@@ -4165,7 +4280,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     return V;
 
   // (A - B) - 1  ->  add (xor B, -1), A
-  if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))), m_One())))
+  if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))),
+                        m_One(/*AllowUndefs=*/true))))
     return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT));
 
   // Look for:
@@ -7402,7 +7518,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     // vector as a scalar and use the splat value.
     APInt Constant = APInt::getZero(1);
     if (const ConstantSDNode *C = isConstOrConstSplat(
-            N1, /*AllowUndef=*/false, /*AllowTruncation=*/true)) {
+            N1, /*AllowUndefs=*/false, /*AllowTruncation=*/true)) {
       Constant = C->getAPIntValue();
     } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
       unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
@@ -11286,18 +11402,25 @@ SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
   SDValue AbsOp0 = N->getOperand(0);
   unsigned Opc0 = Op0.getOpcode();
 
-  // Check if the operands of the sub are (zero|sign)-extended.
-  // TODO: Should we use ValueTracking instead?
+  // Check if the operands of the sub are (zero|sign)-extended, otherwise
+  // fallback to ValueTracking.
   if (Opc0 != Op1.getOpcode() ||
       (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND &&
        Opc0 != ISD::SIGN_EXTEND_INREG)) {
     // fold (abs (sub nsw x, y)) -> abds(x, y)
     // Don't fold this for unsupported types as we lose the NSW handling.
-    if (AbsOp0->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) &&
-        TLI.preferABDSToABSWithNSW(VT)) {
+    if (hasOperation(ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
+        (AbsOp0->getFlags().hasNoSignedWrap() ||
+         DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1))) {
       SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1);
       return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
     }
+    // fold (abs (sub x, y)) -> abdu(x, y)
+    if (hasOperation(ISD::ABDU, VT) && DAG.SignBitIsZero(Op0) &&
+        DAG.SignBitIsZero(Op1)) {
+      SDValue ABD = DAG.getNode(ISD::ABDU, DL, VT, Op0, Op1);
+      return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
+    }
     return SDValue();
   }
 
@@ -12953,6 +13076,98 @@ SDValue DAGCombiner::visitVP_SELECT(SDNode *N) {
   return SDValue();
 }
 
+static SDValue combineVSelectWithAllOnesOrZeros(SDValue Cond, SDValue TVal,
+                                                SDValue FVal,
+                                                const TargetLowering &TLI,
+                                                SelectionDAG &DAG,
+                                                const SDLoc &DL) {
+  EVT VT = TVal.getValueType();
+  if (!TLI.isTypeLegal(VT))
+    return SDValue();
+
+  EVT CondVT = Cond.getValueType();
+  assert(CondVT.isVector() && "Vector select expects a vector selector!");
+
+  bool IsTAllZero = ISD::isBuildVectorAllZeros(TVal.getNode());
+  bool IsTAllOne = ISD::isBuildVectorAllOnes(TVal.getNode());
+  bool IsFAllZero = ISD::isBuildVectorAllZeros(FVal.getNode());
+  bool IsFAllOne = ISD::isBuildVectorAllOnes(FVal.getNode());
+
+  // no vselect(cond, 0/-1, X) or vselect(cond, X, 0/-1), return
+  if (!IsTAllZero && !IsTAllOne && !IsFAllZero && !IsFAllOne)
+    return SDValue();
+
+  // select Cond, 0, 0 → 0
+  if (IsTAllZero && IsFAllZero) {
+    return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, VT)
+                                : DAG.getConstant(0, DL, VT);
+  }
+
+  // check select(setgt lhs, -1), 1, -1 --> or (sra lhs, bitwidth - 1), 1
+  APInt TValAPInt;
+  if (Cond.getOpcode() == ISD::SETCC &&
+      Cond.getOperand(2) == DAG.getCondCode(ISD::SETGT) &&
+      Cond.getOperand(0).getValueType() == VT && VT.isSimple() &&
+      ISD::isConstantSplatVector(TVal.getNode(), TValAPInt) &&
+      TValAPInt.isOne() &&
+      ISD::isConstantSplatVectorAllOnes(Cond.getOperand(1).getNode()) &&
+      ISD::isConstantSplatVectorAllOnes(FVal.getNode())) {
+    return SDValue();
+  }
+
+  // To use the condition operand as a bitwise mask, it must have elements that
+  // are the same size as the select elements. i.e, the condition operand must
+  // have already been promoted from the IR select condition type <N x i1>.
+  // Don't check if the types themselves are equal because that excludes
+  // vector floating-point selects.
+  if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
+    return SDValue();
+
+  // Cond value must be 'sign splat' to be converted to a logical op.
+  if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
+    return SDValue();
+
+  // Try inverting Cond and swapping T/F if it gives all-ones/all-zeros form
+  if (!IsTAllOne && !IsFAllZero && Cond.hasOneUse() &&
+      Cond.getOpcode() == ISD::SETCC &&
+      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
+          CondVT) {
+    if (IsTAllZero || IsFAllOne) {
+      SDValue CC = Cond.getOperand(2);
+      ISD::CondCode InverseCC = ISD::getSetCCInverse(
+          cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
+      Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
+                          InverseCC);
+      std::swap(TVal, FVal);
+      std::swap(IsTAllOne, IsFAllOne);
+      std::swap(IsTAllZero, IsFAllZero);
+    }
+  }
+
+  assert(DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits() &&
+         "Select condition no longer all-sign bits");
+
+  // select Cond, -1, 0 → bitcast Cond
+  if (IsTAllOne && IsFAllZero)
+    return DAG.getBitcast(VT, Cond);
+
+  // select Cond, -1, x → or Cond, x
+  if (IsTAllOne) {
+    SDValue X = DAG.getBitcast(CondVT, FVal);
+    SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, X);
+    return DAG.getBitcast(VT, Or);
+  }
+
+  // select Cond, x, 0 → and Cond, x
+  if (IsFAllZero) {
+    SDValue X = DAG.getBitcast(CondVT, TVal);
+    SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, X);
+    return DAG.getBitcast(VT, And);
+  }
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitVSELECT(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -13221,6 +13436,9 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
   if (SimplifyDemandedVectorElts(SDValue(N, 0)))
     return SDValue(N, 0);
 
+  if (SDValue V = combineVSelectWithAllOnesOrZeros(N0, N1, N2, TLI, DAG, DL))
+    return V;
+
   return SDValue();
 }
 
@@ -15061,6 +15279,7 @@ SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
   default:
     break;
   case ISD::ADD:
+  case ISD::PTRADD:
   case ISD::SUB: {
     unsigned AlignShift = Log2(AL);
     SDValue LHS = N0.getOperand(0);
@@ -16212,8 +16431,8 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
         TLI.isTypeLegal(VT.getVectorElementType()))) &&
       N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() &&
       cast<BuildVectorSDNode>(N0)->isConstant())
-    return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(),
-                                             VT.getVectorElementType());
+    return DAG.FoldConstantBuildVector(cast<BuildVectorSDNode>(N0), SDLoc(N),
+                                       VT.getVectorElementType());
 
   // If the input is a constant, let getNode fold it.
   if (isIntOrFPConstant(N0)) {
@@ -16494,8 +16713,7 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
   // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
   // Try to push freeze through instructions that propagate but don't produce
   // poison as far as possible. If an operand of freeze follows three
-  // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
-  // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
+  // conditions 1) one-use, and 2) does not produce poison then push
   // the freeze through to the operands that are not guaranteed non-poison.
   // NOTE: we will strip poison-generating flags, so ignore them here.
   if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false,
@@ -16503,13 +16721,6 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       N0->getNumValues() != 1 || !N0->hasOneUse())
     return SDValue();
 
-  bool AllowMultipleMaybePoisonOperands =
-      N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
-      N0.getOpcode() == ISD::BUILD_VECTOR ||
-      N0.getOpcode() == ISD::BUILD_PAIR ||
-      N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
-      N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL;
-
   // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
   // ones" or "constant" into something that depends on FrozenUndef. We can
   // instead pick undef values to keep those properties, while at the same time
@@ -16542,10 +16753,6 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       MaybePoisonOperandNumbers.push_back(OpNo);
     if (!HadMaybePoisonOperands)
       continue;
-    if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
-      // Multiple maybe-poison ops when not allowed - bail out.
-      return SDValue();
-    }
   }
   // NOTE: the whole op may be not guaranteed to not be undef or poison because
   // it could create undef or poison due to it's poison-generating flags.
@@ -16599,105 +16806,25 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
       Op = DAG.getFreeze(Op);
   }
 
-  SDValue R;
-  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0)) {
-    // Special case handling for ShuffleVectorSDNode nodes.
-    R = DAG.getVectorShuffle(N0.getValueType(), SDLoc(N0), Ops[0], Ops[1],
-                             SVN->getMask());
-  } else {
-    // NOTE: this strips poison generating flags.
-    // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
-    // ninf, nsz, or fast.
-    // However, contract, reassoc, afn, and arcp should be preserved,
-    // as these fast-math flags do not introduce poison values.
-    SDNodeFlags SrcFlags = N0->getFlags();
-    SDNodeFlags SafeFlags;
-    SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
-    SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
-    SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
-    SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
-    R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops, SafeFlags);
-  }
-  assert(DAG.isGuaranteedNotToBeUndefOrPoison(R, /*PoisonOnly*/ false) &&
-         "Can't create node that may be undef/poison!");
-  return R;
-}
-
-/// We know that BV is a build_vector node with Constant, ConstantFP or Undef
-/// operands. DstEltVT indicates the destination element value type.
-SDValue DAGCombiner::
-ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
-  EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
-
-  // If this is already the right type, we're done.
-  if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
-
-  unsigned SrcBitSize = SrcEltVT.getSizeInBits();
-  unsigned DstBitSize = DstEltVT.getSizeInBits();
-
-  // If this is a conversion of N elements of one type to N elements of another
-  // type, convert each element.  This handles FP<->INT cases.
-  if (SrcBitSize == DstBitSize) {
-    SmallVector<SDValue, 8> Ops;
-    for (SDValue Op : BV->op_values()) {
-      // If the vector element type is not legal, the BUILD_VECTOR operands
-      // are promoted and implicitly truncated.  Make that explicit here.
-      if (Op.getValueType() != SrcEltVT)
-        Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
-      Ops.push_back(DAG.getBitcast(DstEltVT, Op));
-      AddToWorklist(Ops.back().getNode());
-    }
-    EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT,
-                              BV->getValueType(0).getVectorNumElements());
-    return DAG.getBuildVector(VT, SDLoc(BV), Ops);
-  }
-
-  // Otherwise, we're growing or shrinking the elements.  To avoid having to
-  // handle annoying details of growing/shrinking FP values, we convert them to
-  // int first.
-  if (SrcEltVT.isFloatingPoint()) {
-    // Convert the input float vector to a int vector where the elements are the
-    // same sizes.
-    EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), SrcEltVT.getSizeInBits());
-    BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
-    SrcEltVT = IntVT;
-  }
-
-  // Now we know the input is an integer vector.  If the output is a FP type,
-  // convert to integer first, then to FP of the right size.
-  if (DstEltVT.isFloatingPoint()) {
-    EVT TmpVT = EVT::getIntegerVT(*DAG.getContext(), DstEltVT.getSizeInBits());
-    SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
-
-    // Next, convert to FP elements of the same size.
-    return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
-  }
-
-  // Okay, we know the src/dst types are both integers of differing types.
-  assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
-
-  // TODO: Should ConstantFoldBITCASTofBUILD_VECTOR always take a
-  // BuildVectorSDNode?
-  auto *BVN = cast<BuildVectorSDNode>(BV);
-
-  // Extract the constant raw bit data.
-  BitVector UndefElements;
-  SmallVector<APInt> RawBits;
-  bool IsLE = DAG.getDataLayout().isLittleEndian();
-  if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
-    return SDValue();
-
-  SDLoc DL(BV);
-  SmallVector<SDValue, 8> Ops;
-  for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
-    if (UndefElements[I])
-      Ops.push_back(DAG.getUNDEF(DstEltVT));
-    else
-      Ops.push_back(DAG.getConstant(RawBits[I], DL, DstEltVT));
-  }
+  SDLoc DL(N0);
 
-  EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size());
-  return DAG.getBuildVector(VT, DL, Ops);
+  // Special case handling for ShuffleVectorSDNode nodes.
+  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0))
+    return DAG.getVectorShuffle(N0.getValueType(), DL, Ops[0], Ops[1],
+                                SVN->getMask());
+
+  // NOTE: this strips poison generating flags.
+  // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
+  // ninf, nsz, or fast.
+  // However, contract, reassoc, afn, and arcp should be preserved,
+  // as these fast-math flags do not introduce poison values.
+  SDNodeFlags SrcFlags = N0->getFlags();
+  SDNodeFlags SafeFlags;
+  SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
+  SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
+  SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
+  SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
+  return DAG.getNode(N0.getOpcode(), DL, N0->getVTList(), Ops, SafeFlags);
 }
 
 // Returns true if floating point contraction is allowed on the FMUL-SDValue
@@ -16705,7 +16832,7 @@ ConstantFoldBITCASTofBUILD_VECTOR(SDNode *BV, EVT DstEltVT) {
 static bool isContractableFMUL(const TargetOptions &Options, SDValue N) {
   assert(N.getOpcode() == ISD::FMUL);
 
-  return Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath ||
+  return Options.AllowFPOpFusion == FPOpFusion::Fast ||
          N->getFlags().hasAllowContract();
 }
 
@@ -16740,8 +16867,8 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
   if (!HasFMAD && !HasFMA)
     return SDValue();
 
-  bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
-                              Options.UnsafeFPMath || HasFMAD);
+  bool AllowFusionGlobally =
+      Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD;
   // If the addition is not contractable, do not combine.
   if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
     return SDValue();
@@ -16978,8 +17105,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     return SDValue();
 
   const SDNodeFlags Flags = N->getFlags();
-  bool AllowFusionGlobally = (Options.AllowFPOpFusion == FPOpFusion::Fast ||
-                              Options.UnsafeFPMath || HasFMAD);
+  bool AllowFusionGlobally =
+      (Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD);
 
   // If the subtraction is not contractable, do not combine.
   if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
@@ -17134,13 +17261,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
     }
   }
 
-  auto isReassociable = [&Options](SDNode *N) {
-    return Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
-  };
-
-  auto isContractableAndReassociableFMUL = [&isContractableFMUL,
-                                            &isReassociable](SDValue N) {
-    return isContractableFMUL(N) && isReassociable(N.getNode());
+  auto isContractableAndReassociableFMUL = [&isContractableFMUL](SDValue N) {
+    return isContractableFMUL(N) && N->getFlags().hasAllowReassociation();
   };
 
   auto isFusedOp = [&](SDValue N) {
@@ -17148,8 +17270,8 @@ SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
   };
 
   // More folding opportunities when target permits.
-  if (Aggressive && isReassociable(N)) {
-    bool CanFuse = Options.UnsafeFPMath || N->getFlags().hasAllowContract();
+  if (Aggressive && N->getFlags().hasAllowReassociation()) {
+    bool CanFuse = N->getFlags().hasAllowContract();
     // fold (fsub (fma x, y, (fmul u, v)), z)
     //   -> (fma x, y (fma u, v, (fneg z)))
     if (CanFuse && isFusedOp(N0) &&
@@ -17306,8 +17428,7 @@ SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
 
   // Floating-point multiply-add with intermediate rounding. This can result
   // in a less precise result due to the changed rounding order.
-  bool HasFMAD = Options.UnsafeFPMath &&
-                 (LegalOperations && TLI.isFMADLegal(DAG, N));
+  bool HasFMAD = LegalOperations && TLI.isFMADLegal(DAG, N);
 
   // No valid opcode, do not combine.
   if (!HasFMAD && !HasFMA)
@@ -17557,12 +17678,15 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
                            DAG.getConstantFP(4.0, DL, VT));
       }
     }
+  } // enable-unsafe-fp-math && AllowNewConst
 
+  if (((Options.UnsafeFPMath && Options.NoSignedZerosFPMath) ||
+       (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros()))) {
     // Fold fadd(vecreduce(x), vecreduce(y)) -> vecreduce(fadd(x, y))
     if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FADD, ISD::FADD, DL,
                                           VT, N0, N1, Flags))
       return SD;
-  } // enable-unsafe-fp-math
+  }
 
   // FADD -> FMA combines:
   if (SDValue Fused = visitFADDForFMACombine<EmptyMatchContext>(N)) {
@@ -17947,6 +18071,7 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) {
   SDValue N2 = N->getOperand(2);
   ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
+  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
   const TargetOptions &Options = DAG.getTarget().Options;
@@ -17976,11 +18101,16 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) {
   }
 
   // FIXME: use fast math flags instead of Options.UnsafeFPMath
-  if (Options.UnsafeFPMath) {
-    if (N0CFP && N0CFP->isZero())
-      return N2;
-    if (N1CFP && N1CFP->isZero())
-      return N2;
+  // TODO: Finally migrate away from global TargetOptions.
+  if ((Options.NoNaNsFPMath && Options.NoInfsFPMath) ||
+      (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs())) {
+    if (Options.NoSignedZerosFPMath || N->getFlags().hasNoSignedZeros() ||
+        (N2CFP && !N2CFP->isExactlyValue(-0.0))) {
+      if (N0CFP && N0CFP->isZero())
+        return N2;
+      if (N1CFP && N1CFP->isZero())
+        return N2;
+    }
   }
 
   // FIXME: Support splat of constant.
@@ -18196,8 +18326,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
     // Only do the transform if the reciprocal is a legal fp immediate that
     // isn't too nasty (eg NaN, denormal, ...).
     if (((st == APFloat::opOK && !Recip.isDenormal()) ||
-         (st == APFloat::opInexact &&
-          (Options.UnsafeFPMath || Flags.hasAllowReciprocal()))) &&
+         (st == APFloat::opInexact && Flags.hasAllowReciprocal())) &&
         (!LegalOperations ||
          // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
          // backend)... we should handle this gracefully after Legalize.
@@ -18208,7 +18337,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
                          DAG.getConstantFP(Recip, DL, VT));
   }
 
-  if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) {
+  if (Flags.hasAllowReciprocal()) {
     // If this FDIV is part of a reciprocal square root, it may be folded
     // into a target-specific square root estimate instruction.
     if (N1.getOpcode() == ISD::FSQRT) {
@@ -18283,7 +18412,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
 
   // Fold X/Sqrt(X) -> Sqrt(X)
   if ((Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
-      (Options.UnsafeFPMath || Flags.hasAllowReassociation()))
+      Flags.hasAllowReassociation())
     if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0))
       return N1;
 
@@ -23070,13 +23199,16 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
 
     // Ensure all the operands are the same value type, fill any missing
     // operands with UNDEF and create the BUILD_VECTOR.
-    auto CanonicalizeBuildVector = [&](SmallVectorImpl<SDValue> &Ops) {
+    auto CanonicalizeBuildVector = [&](SmallVectorImpl<SDValue> &Ops,
+                                       bool FreezeUndef = false) {
       assert(Ops.size() == NumElts && "Unexpected vector size");
+      SDValue UndefOp = FreezeUndef ? DAG.getFreeze(DAG.getUNDEF(MaxEltVT))
+                                    : DAG.getUNDEF(MaxEltVT);
       for (SDValue &Op : Ops) {
         if (Op)
           Op = VT.isInteger() ? DAG.getAnyExtOrTrunc(Op, DL, MaxEltVT) : Op;
         else
-          Op = DAG.getUNDEF(MaxEltVT);
+          Op = UndefOp;
       }
       return DAG.getBuildVector(VT, DL, Ops);
     };
@@ -23090,6 +23222,10 @@ SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
       if (CurVec.isUndef())
         return CanonicalizeBuildVector(Ops);
 
+      // FREEZE(UNDEF) - build new BUILD_VECTOR from already inserted operands.
+      if (ISD::isFreezeUndef(CurVec.getNode()) && CurVec.hasOneUse())
+        return CanonicalizeBuildVector(Ops, /*FreezeUndef=*/true);
+
       // BUILD_VECTOR - insert unused operands and build new BUILD_VECTOR.
       if (CurVec.getOpcode() == ISD::BUILD_VECTOR && CurVec.hasOneUse()) {
         for (unsigned I = 0; I != NumElts; ++I)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 66971ad661da..f9095d3e7465 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -631,16 +631,17 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, VRBaseMapType &VRBaseMap,
 void
 InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
                                      VRBaseMapType &VRBaseMap) {
-  Register VReg = getVR(Node->getOperand(0), VRBaseMap);
-
   // Create the new VReg in the destination class and emit a copy.
   unsigned DstRCIdx = Node->getConstantOperandVal(1);
   const TargetRegisterClass *DstRC =
     TRI->getAllocatableClass(TRI->getRegClass(DstRCIdx));
   Register NewVReg = MRI->createVirtualRegister(DstRC);
-  BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
-    NewVReg).addReg(VReg);
+  const MCInstrDesc &II = TII->get(TargetOpcode::COPY);
+  MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II, NewVReg);
+  AddOperand(MIB, Node->getOperand(0), 1, &II, VRBaseMap, /*IsDebug=*/false,
+             /*IsClone=*/false, /*IsCloned*/ false);
 
+  MBB->insert(InsertPos, MIB);
   SDValue Op(Node, 0);
   bool isNew = VRBaseMap.insert(std::make_pair(Op, NewVReg)).second;
   (void)isNew; // Silence compiler warning.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 7dfb43164be3..2a947f329774 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -704,42 +704,41 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
-  return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
-                                               RTLIB::POW_F32,
-                                               RTLIB::POW_F64,
-                                               RTLIB::POW_F80,
-                                               RTLIB::POW_F128,
-                                               RTLIB::POW_PPCF128));
+  return SoftenFloatRes_Binary(N, RTLIB::getPOW(N->getValueType(0)));
 }
 
 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
   bool IsStrict = N->isStrictFPOpcode();
   unsigned Offset = IsStrict ? 1 : 0;
-  assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
-          N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
-         "Unsupported power type!");
   bool IsPowI =
       N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
+  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
 
   RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
                              : RTLIB::getLDEXP(N->getValueType(0));
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
   if (!TLI.getLibcallName(LC)) {
+    RTLIB::Libcall NewLC = RTLIB::getPOW(N->getValueType(0));
+    assert(NewLC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpow type");
+
     // Some targets don't have a powi libcall; use pow instead.
     // FIXME: Implement this if some target needs it.
-    DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
-    return DAG.getUNDEF(N->getValueType(0));
+    DAG.getContext()->emitError("do not know how to soften fpowi to fpow");
+    if (IsStrict)
+      ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
+    return DAG.getPOISON(NVT);
   }
 
   if (DAG.getLibInfo().getIntSize() !=
       N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
     // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
     // would use the wrong type for the argument.
-    DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
-    return DAG.getUNDEF(N->getValueType(0));
+    DAG.getContext()->emitError("powi exponent does not match sizeof(int)");
+    if (IsStrict)
+      ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
+    return DAG.getPOISON(NVT);
   }
 
-  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
                      N->getOperand(1 + Offset) };
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
@@ -760,20 +759,21 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
   EVT VT0 = N->getValueType(0);
   EVT VT1 = N->getValueType(1);
   RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
+  EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
+  SDLoc DL(N);
 
   if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
     // If the exponent does not match with sizeof(int) a libcall would use the
     // wrong type for the argument.
     // TODO: Should be able to handle mismatches.
     DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
-    return DAG.getUNDEF(N->getValueType(0));
+    SDValue PoisonExp = DAG.getPOISON(VT1);
+    ReplaceValueWith(SDValue(N, 1), PoisonExp);
+    return DAG.getMergeValues({DAG.getPOISON(NVT0), PoisonExp}, DL);
   }
 
-  EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
   SDValue StackSlot = DAG.CreateStackTemporary(VT1);
 
-  SDLoc DL(N);
-
   auto PointerTy = PointerType::getUnqual(*DAG.getContext());
   TargetLowering::MakeLibCallOptions CallOptions;
   SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
diff --git a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
index e0e8d503ca92..0a449fd011e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
@@ -54,8 +54,8 @@ ResourcePriorityQueue::ResourcePriorityQueue(SelectionDAGISel *IS)
   unsigned NumRC = TRI->getNumRegClasses();
   RegLimit.resize(NumRC);
   RegPressure.resize(NumRC);
-  std::fill(RegLimit.begin(), RegLimit.end(), 0);
-  std::fill(RegPressure.begin(), RegPressure.end(), 0);
+  llvm::fill(RegLimit, 0);
+  llvm::fill(RegPressure, 0);
   for (const TargetRegisterClass *RC : TRI->regclasses())
     RegLimit[RC->getID()] = TRI->getRegPressureLimit(RC, *IS->MF);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 2c51a01044cd..a570b71ecd28 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -1769,8 +1769,8 @@ class RegReductionPQBase : public SchedulingPriorityQueue {
       unsigned NumRC = TRI->getNumRegClasses();
       RegLimit.resize(NumRC);
       RegPressure.resize(NumRC);
-      std::fill(RegLimit.begin(), RegLimit.end(), 0);
-      std::fill(RegPressure.begin(), RegPressure.end(), 0);
+      llvm::fill(RegLimit, 0);
+      llvm::fill(RegPressure, 0);
       for (const TargetRegisterClass *RC : TRI->regclasses())
         RegLimit[RC->getID()] = tri->getRegPressureLimit(RC, MF);
     }
@@ -1793,7 +1793,7 @@ class RegReductionPQBase : public SchedulingPriorityQueue {
   void releaseState() override {
     SUnits = nullptr;
     SethiUllmanNumbers.clear();
-    std::fill(RegPressure.begin(), RegPressure.end(), 0);
+    llvm::fill(RegPressure, 0);
   }
 
   unsigned getNodePriority(const SUnit *SU) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9a087582a8e0..b46203e8b635 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1275,6 +1275,8 @@ SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
       // to replace the dead one with the existing one.  This can cause
       // recursive merging of other unrelated nodes down the line.
       Existing->intersectFlagsWith(N->getFlags());
+      if (auto *MemNode = dyn_cast<MemSDNode>(Existing))
+        MemNode->refineRanges(cast<MemSDNode>(N)->getMemOperand());
       ReplaceAllUsesWith(N, Existing);
 
       // N is now dead. Inform the listeners and delete it.
@@ -1455,8 +1457,8 @@ void SelectionDAG::clear() {
   TargetExternalSymbols.clear();
   MCSymbols.clear();
   SDEI.clear();
-  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), nullptr);
-  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), nullptr);
+  llvm::fill(CondCodeNodes, nullptr);
+  llvm::fill(ValueTypeNodes, nullptr);
 
   EntryNode.UseList = nullptr;
   InsertNode(&EntryNode);
@@ -2517,8 +2519,11 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
   EVT VT = Node->getValueType(0);
   unsigned NumResults = Node->getNumValues();
 
+  if (LC == RTLIB::UNKNOWN_LIBCALL)
+    return false;
+
   const char *LCName = TLI->getLibcallName(LC);
-  if (!LC || !LCName)
+  if (!LCName)
     return false;
 
   auto getVecDesc = [&]() -> VecDesc const * {
@@ -5520,6 +5525,13 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
 
   unsigned Opcode = Op.getOpcode();
   switch (Opcode) {
+  case ISD::AssertSext:
+  case ISD::AssertZext:
+  case ISD::AssertAlign:
+  case ISD::AssertNoFPClass:
+    // Assertion nodes can create poison if the assertion fails.
+    return true;
+
   case ISD::FREEZE:
   case ISD::CONCAT_VECTORS:
   case ISD::INSERT_SUBVECTOR:
@@ -5541,6 +5553,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::FSHL:
   case ISD::FSHR:
   case ISD::BSWAP:
+  case ISD::CTTZ:
+  case ISD::CTLZ:
   case ISD::CTPOP:
   case ISD::BITREVERSE:
   case ISD::PARITY:
@@ -5553,7 +5567,23 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::BUILD_VECTOR:
   case ISD::BUILD_PAIR:
   case ISD::SPLAT_VECTOR:
-  case ISD::VSELECT:
+    return false;
+
+  case ISD::ADDC:
+  case ISD::SUBC:
+  case ISD::ADDE:
+  case ISD::SUBE:
+  case ISD::SADDO:
+  case ISD::SSUBO:
+  case ISD::SMULO:
+  case ISD::SADDO_CARRY:
+  case ISD::SSUBO_CARRY:
+  case ISD::UADDO:
+  case ISD::USUBO:
+  case ISD::UMULO:
+  case ISD::UADDO_CARRY:
+  case ISD::USUBO_CARRY:
+    // No poison on result or overflow flags.
     return false;
 
   case ISD::SELECT_CC:
@@ -5577,6 +5607,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
 
   case ISD::OR:
   case ISD::ZERO_EXTEND:
+  case ISD::SELECT:
+  case ISD::VSELECT:
   case ISD::ADD:
   case ISD::SUB:
   case ISD::MUL:
@@ -5594,9 +5626,13 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::SRA:
     // If the max shift amount isn't in range, then the shift can
     // create poison.
-    return !isGuaranteedNotToBeUndefOrPoison(Op.getOperand(1), DemandedElts,
-                                             PoisonOnly, Depth + 1) ||
-           !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
+    return !getValidMaximumShiftAmount(Op, DemandedElts, Depth + 1);
+
+  case ISD::CTTZ_ZERO_UNDEF:
+  case ISD::CTLZ_ZERO_UNDEF:
+    // If the amount is zero then the result will be poison.
+    // TODO: Add isKnownNeverZero DemandedElts handling.
+    return !isKnownNeverZero(Op.getOperand(0), Depth + 1);
 
   case ISD::SCALAR_TO_VECTOR:
     // Check if we demand any upper (undef) elements.
@@ -5607,12 +5643,8 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
     // Ensure that the element index is in bounds.
     EVT VecVT = Op.getOperand(0).getValueType();
     SDValue Idx = Op.getOperand(Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
-    if (isGuaranteedNotToBeUndefOrPoison(Idx, DemandedElts, PoisonOnly,
-                                         Depth + 1)) {
-      KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
-      return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
-    }
-    return true;
+    KnownBits KnownIdx = computeKnownBits(Idx, Depth + 1);
+    return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
   }
 
   case ISD::VECTOR_SHUFFLE: {
@@ -7251,6 +7283,78 @@ SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
   return SDValue();
 }
 
+SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
+                                              const SDLoc &DL, EVT DstEltVT) {
+  EVT SrcEltVT = BV->getValueType(0).getVectorElementType();
+
+  // If this is already the right type, we're done.
+  if (SrcEltVT == DstEltVT)
+    return SDValue(BV, 0);
+
+  unsigned SrcBitSize = SrcEltVT.getSizeInBits();
+  unsigned DstBitSize = DstEltVT.getSizeInBits();
+
+  // If this is a conversion of N elements of one type to N elements of another
+  // type, convert each element.  This handles FP<->INT cases.
+  if (SrcBitSize == DstBitSize) {
+    SmallVector<SDValue, 8> Ops;
+    for (SDValue Op : BV->op_values()) {
+      // If the vector element type is not legal, the BUILD_VECTOR operands
+      // are promoted and implicitly truncated.  Make that explicit here.
+      if (Op.getValueType() != SrcEltVT)
+        Op = getNode(ISD::TRUNCATE, DL, SrcEltVT, Op);
+      Ops.push_back(getBitcast(DstEltVT, Op));
+    }
+    EVT VT = EVT::getVectorVT(*getContext(), DstEltVT,
+                              BV->getValueType(0).getVectorNumElements());
+    return getBuildVector(VT, DL, Ops);
+  }
+
+  // Otherwise, we're growing or shrinking the elements.  To avoid having to
+  // handle annoying details of growing/shrinking FP values, we convert them to
+  // int first.
+  if (SrcEltVT.isFloatingPoint()) {
+    // Convert the input float vector to a int vector where the elements are the
+    // same sizes.
+    EVT IntEltVT = EVT::getIntegerVT(*getContext(), SrcEltVT.getSizeInBits());
+    if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
+      return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,
+                                     DstEltVT);
+    return SDValue();
+  }
+
+  // Now we know the input is an integer vector.  If the output is a FP type,
+  // convert to integer first, then to FP of the right size.
+  if (DstEltVT.isFloatingPoint()) {
+    EVT IntEltVT = EVT::getIntegerVT(*getContext(), DstEltVT.getSizeInBits());
+    if (SDValue Tmp = FoldConstantBuildVector(BV, DL, IntEltVT))
+      return FoldConstantBuildVector(cast<BuildVectorSDNode>(Tmp), DL,
+                                     DstEltVT);
+    return SDValue();
+  }
+
+  // Okay, we know the src/dst types are both integers of differing types.
+  assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
+
+  // Extract the constant raw bit data.
+  BitVector UndefElements;
+  SmallVector<APInt> RawBits;
+  bool IsLE = getDataLayout().isLittleEndian();
+  if (!BV->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
+    return SDValue();
+
+  SmallVector<SDValue, 8> Ops;
+  for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
+    if (UndefElements[I])
+      Ops.push_back(getUNDEF(DstEltVT));
+    else
+      Ops.push_back(getConstant(RawBits[I], DL, DstEltVT));
+  }
+
+  EVT VT = EVT::getVectorVT(*getContext(), DstEltVT, Ops.size());
+  return getBuildVector(VT, DL, Ops);
+}
+
 SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
   assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
 
@@ -9109,8 +9213,9 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   ID.AddInteger(MMO->getFlags());
   void* IP = nullptr;
-  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
-    cast<AtomicSDNode>(E)->refineAlignment(MMO);
+  if (auto *E = cast_or_null<AtomicSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
+    E->refineAlignment(MMO);
+    E->refineRanges(MMO);
     return SDValue(E, 0);
   }
 
@@ -9401,8 +9506,9 @@ SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   ID.AddInteger(MMO->getFlags());
   void *IP = nullptr;
-  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
-    cast<LoadSDNode>(E)->refineAlignment(MMO);
+  if (auto *E = cast_or_null<LoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
+    E->refineAlignment(MMO);
+    E->refineRanges(MMO);
     return SDValue(E, 0);
   }
   auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -9622,8 +9728,9 @@ SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   ID.AddInteger(MMO->getFlags());
   void *IP = nullptr;
-  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
-    cast<VPLoadSDNode>(E)->refineAlignment(MMO);
+  if (auto *E = cast_or_null<VPLoadSDNode>(FindNodeOrInsertPos(ID, dl, IP))) {
+    E->refineAlignment(MMO);
+    E->refineRanges(MMO);
     return SDValue(E, 0);
   }
   auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
@@ -12566,6 +12673,18 @@ bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
   return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
 }
 
+bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
+  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
+  return C && APInt::isSameValue(C->getAPIntValue(),
+                                 APInt(C->getAPIntValue().getBitWidth(), 1));
+}
+
+bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
+  N = peekThroughBitcasts(N);
+  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, true);
+  return C && C->isZero();
+}
+
 HandleSDNode::~HandleSDNode() {
   DropOperands();
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index f2ab88851b78..da92aaa860b2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -231,6 +231,7 @@ static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
         }
       break;
     case ISD::ADD:
+    case ISD::PTRADD:
       if (auto *C = dyn_cast<ConstantSDNode>(Base->getOperand(1))) {
         Offset += C->getSExtValue();
         Base = DAG.getTargetLoweringInfo().unwrapAddress(Base->getOperand(0));
@@ -259,7 +260,7 @@ static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
     break;
   }
 
-  if (Base->getOpcode() == ISD::ADD) {
+  if (Base->isAnyAdd()) {
     // TODO: The following code appears to be needless as it just
     //       bails on some Ptrs early, reducing the cases where we
     //       find equivalence. We should be able to remove this.
@@ -282,8 +283,7 @@ static BaseIndexOffset matchLSNode(const LSBaseSDNode *N,
     }
 
     // Check if Index Offset pattern
-    if (Index->getOpcode() != ISD::ADD ||
-        !isa<ConstantSDNode>(Index->getOperand(1)))
+    if (!Index->isAnyAdd() || !isa<ConstantSDNode>(Index->getOperand(1)))
       return BaseIndexOffset(PotentialBase, Index, Offset, IsIndexSignExt);
 
     Offset += cast<ConstantSDNode>(Index->getOperand(1))->getSExtValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c3df95e70991..c7e2120d17ee 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4336,19 +4336,13 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
   auto &TLI = DAG.getTargetLoweringInfo();
   GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
 
-  // Normalize Vector GEP - all scalar operands should be converted to the
-  // splat vector.
+  // For a vector GEP, keep the prefix scalar as long as possible, then
+  // convert any scalars encountered after the first vector operand to vectors.
   bool IsVectorGEP = I.getType()->isVectorTy();
   ElementCount VectorElementCount =
       IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
                   : ElementCount::getFixed(0);
 
-  if (IsVectorGEP && !N.getValueType().isVector()) {
-    LLVMContext &Context = *DAG.getContext();
-    EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
-    N = DAG.getSplat(VT, dl, N);
-  }
-
   for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
        GTI != E; ++GTI) {
     const Value *Idx = GTI.getOperand();
@@ -4396,7 +4390,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
         APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
         LLVMContext &Context = *DAG.getContext();
         SDValue OffsVal;
-        if (IsVectorGEP)
+        if (N.getValueType().isVector())
           OffsVal = DAG.getConstant(
               Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
         else
@@ -4418,10 +4412,16 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
       // N = N + Idx * ElementMul;
       SDValue IdxN = getValue(Idx);
 
-      if (!IdxN.getValueType().isVector() && IsVectorGEP) {
-        EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
-                                  VectorElementCount);
-        IdxN = DAG.getSplat(VT, dl, IdxN);
+      if (IdxN.getValueType().isVector() != N.getValueType().isVector()) {
+        if (N.getValueType().isVector()) {
+          EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
+                                    VectorElementCount);
+          IdxN = DAG.getSplat(VT, dl, IdxN);
+        } else {
+          EVT VT =
+              EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
+          N = DAG.getSplat(VT, dl, N);
+        }
       }
 
       // If the index is smaller or larger than intptr_t, truncate or extend
@@ -4442,7 +4442,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
         SDValue VScale = DAG.getNode(
             ISD::VSCALE, dl, VScaleTy,
             DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
-        if (IsVectorGEP)
+        if (N.getValueType().isVector())
           VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
         IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
                            ScaleFlags);
@@ -4475,6 +4475,11 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
     }
   }
 
+  if (IsVectorGEP && !N.getValueType().isVector()) {
+    EVT VT = EVT::getVectorVT(*Context, N.getValueType(), VectorElementCount);
+    N = DAG.getSplat(VT, dl, N);
+  }
+
   MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
   MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
   if (IsVectorGEP) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 66717135c9ad..45ab7526c3a3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/IR/DataLayout.h"
@@ -37,6 +38,7 @@
 #include <cctype>
 #include <deque>
 using namespace llvm;
+using namespace llvm::SDPatternMatch;
 
 /// NOTE: The TargetMachine owns TLOF.
 TargetLowering::TargetLowering(const TargetMachine &tm)
@@ -179,10 +181,12 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
     Args.push_back(Entry);
   }
 
-  if (LC == RTLIB::UNKNOWN_LIBCALL)
-    report_fatal_error("Unsupported library call operation!");
-  SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
-                                         getPointerTy(DAG.getDataLayout()));
+  const char *LibcallName = getLibcallName(LC);
+  if (LC == RTLIB::UNKNOWN_LIBCALL || !LibcallName)
+    reportFatalInternalError("unsupported library call operation");
+
+  SDValue Callee =
+      DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -3587,6 +3591,19 @@ bool TargetLowering::SimplifyDemandedVectorElts(
         DemandedRHS.setBit(M - NumElts);
     }
 
+    // If either side isn't demanded, replace it by UNDEF. We handle this
+    // explicitly here to also simplify in case of multiple uses (on the
+    // contrary to the SimplifyDemandedVectorElts calls below).
+    bool FoldLHS = !DemandedLHS && !LHS.isUndef();
+    bool FoldRHS = !DemandedRHS && !RHS.isUndef();
+    if (FoldLHS || FoldRHS) {
+      LHS = FoldLHS ? TLO.DAG.getUNDEF(LHS.getValueType()) : LHS;
+      RHS = FoldRHS ? TLO.DAG.getUNDEF(RHS.getValueType()) : RHS;
+      SDValue NewOp =
+          TLO.DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, ShuffleMask);
+      return TLO.CombineTo(Op, NewOp);
+    }
+
     // See if we can simplify either shuffle operand.
     APInt UndefLHS, ZeroLHS;
     APInt UndefRHS, ZeroRHS;
@@ -4212,6 +4229,42 @@ SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
   return SDValue();
 }
 
+/// This helper function of SimplifySetCC tries to optimize the comparison when
+/// either operand of the SetCC node is a bitwise-or instruction.
+/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
+SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
+                                        ISD::CondCode Cond, const SDLoc &DL,
+                                        DAGCombinerInfo &DCI) const {
+  if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
+    std::swap(N0, N1);
+
+  SelectionDAG &DAG = DCI.DAG;
+  EVT OpVT = N0.getValueType();
+  if (!N0.hasOneUse() || !OpVT.isInteger() ||
+      (Cond != ISD::SETEQ && Cond != ISD::SETNE))
+    return SDValue();
+
+  // (X | Y) == Y
+  // (X | Y) != Y
+  SDValue X;
+  if (sd_match(N0, m_Or(m_Value(X), m_Specific(N1))) && hasAndNotCompare(X)) {
+    // If the target supports an 'and-not' or 'and-complement' logic operation,
+    // try to use that to make a comparison operation more efficient.
+
+    // Bail out if the compare operand that we want to turn into a zero is
+    // already a zero (otherwise, infinite loop).
+    if (isNullConstant(N1))
+      return SDValue();
+
+    // Transform this into: X & ~Y ==/!= 0.
+    SDValue NotY = DAG.getNOT(SDLoc(N1), N1, OpVT);
+    SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, X, NotY);
+    return DAG.getSetCC(DL, VT, NewAnd, DAG.getConstant(0, DL, OpVT), Cond);
+  }
+
+  return SDValue();
+}
+
 /// There are multiple IR patterns that could be checking whether certain
 /// truncation of a signed number would be lossy or not. The pattern which is
 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
@@ -5507,6 +5560,9 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
 
     if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
       return V;
+
+    if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, dl, DCI))
+      return V;
   }
 
   // Fold remainder of division by a constant.
@@ -7356,17 +7412,6 @@ TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
   return Blended;
 }
 
-bool TargetLowering::
-verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
-  if (!isa<ConstantSDNode>(Op.getOperand(0))) {
-    DAG.getContext()->emitError("argument to '__builtin_return_address' must "
-                                "be a constant integer");
-    return true;
-  }
-
-  return false;
-}
-
 SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
                                          const DenormalMode &Mode) const {
   SDLoc DL(Op);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index cda41a91a372..4f722005f405 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -396,6 +396,10 @@ RTLIB::Libcall RTLIB::getPOWI(EVT RetVT) {
                       POWI_PPCF128);
 }
 
+RTLIB::Libcall RTLIB::getPOW(EVT RetVT) {
+  return getFPLibCall(RetVT, POW_F32, POW_F64, POW_F80, POW_F128, POW_PPCF128);
+}
+
 RTLIB::Libcall RTLIB::getLDEXP(EVT RetVT) {
   return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
                       LDEXP_PPCF128);
@@ -681,9 +685,8 @@ void TargetLoweringBase::initActions() {
   memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
   memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
   memset(CondCodeActions, 0, sizeof(CondCodeActions));
-  std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
-  std::fill(std::begin(TargetDAGCombineArray),
-            std::end(TargetDAGCombineArray), 0);
+  llvm::fill(RegClassForVT, nullptr);
+  llvm::fill(TargetDAGCombineArray, 0);
 
   // Let extending atomic loads be unsupported by default.
   for (MVT ValVT : MVT::all_valuetypes())
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index c804a179d886..5454cd475f5e 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -19,6 +19,7 @@
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/CodeGen/BasicBlockSectionUtils.h"
@@ -47,6 +48,7 @@
 #include "llvm/MC/MCAsmInfoDarwin.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionGOFF.h"
@@ -56,6 +58,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
+#include "llvm/MC/MCSymbolGOFF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/ProfileData/InstrProf.h"
@@ -64,6 +67,7 @@
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/TargetParser/Triple.h"
@@ -2775,6 +2779,29 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForLSDA(
 //===----------------------------------------------------------------------===//
 TargetLoweringObjectFileGOFF::TargetLoweringObjectFileGOFF() = default;
 
+void TargetLoweringObjectFileGOFF::getModuleMetadata(Module &M) {
+  // Construct the default names for the root SD and the ADA PR symbol.
+  StringRef FileName = sys::path::stem(M.getSourceFileName());
+  if (FileName.size() > 1 && FileName.starts_with('<') &&
+      FileName.ends_with('>'))
+    FileName = FileName.substr(1, FileName.size() - 2);
+  DefaultRootSDName = Twine(FileName).concat("#C").str();
+  DefaultADAPRName = Twine(FileName).concat("#S").str();
+  MCSectionGOFF *RootSD =
+      static_cast<MCSectionGOFF *>(TextSection)->getParent();
+  MCSectionGOFF *ADAPR = static_cast<MCSectionGOFF *>(ADASection);
+  RootSD->setName(DefaultRootSDName);
+  ADAPR->setName(DefaultADAPRName);
+  // Initialize the label for the text section.
+  MCSymbolGOFF *TextLD = static_cast<MCSymbolGOFF *>(
+      getContext().getOrCreateSymbol(RootSD->getName()));
+  TextLD->setLDAttributes(GOFF::LDAttr{
+      false, GOFF::ESD_EXE_CODE, GOFF::ESD_BST_Strong, GOFF::ESD_LT_XPLink,
+      GOFF::ESD_AMODE_64, GOFF::ESD_BSC_Section});
+  TextLD->setADA(ADAPR);
+  TextSection->setBeginSymbol(TextLD);
+}
+
 MCSection *TargetLoweringObjectFileGOFF::getExplicitSectionGlobal(
     const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
   return SelectSectionForGlobal(GO, Kind, TM);
@@ -2783,15 +2810,56 @@ MCSection *TargetLoweringObjectFileGOFF::getExplicitSectionGlobal(
 MCSection *TargetLoweringObjectFileGOFF::getSectionForLSDA(
     const Function &F, const MCSymbol &FnSym, const TargetMachine &TM) const {
   std::string Name = ".gcc_exception_table." + F.getName().str();
-  return getContext().getGOFFSection(Name, SectionKind::getData(), nullptr, 0);
+
+  MCSectionGOFF *WSA = getContext().getGOFFSection(
+      SectionKind::getMetadata(), GOFF::CLASS_WSA,
+      GOFF::EDAttr{false, GOFF::ESD_RMODE_64, GOFF::ESD_NS_Parts,
+                   GOFF::ESD_TS_ByteOriented, GOFF::ESD_BA_Merge,
+                   GOFF::ESD_LB_Initial, GOFF::ESD_RQ_0,
+                   GOFF::ESD_ALIGN_Fullword, 0},
+      static_cast<MCSectionGOFF *>(TextSection)->getParent());
+  return getContext().getGOFFSection(SectionKind::getData(), Name,
+                                     GOFF::PRAttr{true, GOFF::ESD_EXE_DATA,
+                                                  GOFF::ESD_LT_XPLink,
+                                                  GOFF::ESD_BSC_Section, 0},
+                                     WSA);
 }
 
 MCSection *TargetLoweringObjectFileGOFF::SelectSectionForGlobal(
     const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const {
   auto *Symbol = TM.getSymbol(GO);
-  if (Kind.isBSS())
-    return getContext().getGOFFSection(Symbol->getName(), SectionKind::getBSS(),
-                                       nullptr, 0);
 
-  return getContext().getObjectFileInfo()->getTextSection();
+  if (Kind.isBSS() || Kind.isData()) {
+    GOFF::ESDBindingScope PRBindingScope =
+        GO->hasExternalLinkage()
+            ? (GO->hasDefaultVisibility() ? GOFF::ESD_BSC_ImportExport
+                                          : GOFF::ESD_BSC_Library)
+            : GOFF::ESD_BSC_Section;
+    GOFF::ESDBindingScope SDBindingScope =
+        PRBindingScope == GOFF::ESD_BSC_Section ? GOFF::ESD_BSC_Section
+                                                : GOFF::ESD_BSC_Unspecified;
+    MaybeAlign Alignment;
+    if (auto *F = dyn_cast<Function>(GO))
+      Alignment = F->getAlign();
+    else if (auto *V = dyn_cast<GlobalVariable>(GO))
+      Alignment = V->getAlign();
+    GOFF::ESDAlignment Align =
+        Alignment ? static_cast<GOFF::ESDAlignment>(Log2(*Alignment))
+                  : GOFF::ESD_ALIGN_Doubleword;
+    MCSectionGOFF *SD = getContext().getGOFFSection(
+        SectionKind::getMetadata(), Symbol->getName(),
+        GOFF::SDAttr{GOFF::ESD_TA_Unspecified, SDBindingScope});
+    MCSectionGOFF *ED = getContext().getGOFFSection(
+        SectionKind::getMetadata(), GOFF::CLASS_WSA,
+        GOFF::EDAttr{false, GOFF::ESD_RMODE_64, GOFF::ESD_NS_Parts,
+                     GOFF::ESD_TS_ByteOriented, GOFF::ESD_BA_Merge,
+                     GOFF::ESD_LB_Deferred, GOFF::ESD_RQ_0, Align, 0},
+        SD);
+    return getContext().getGOFFSection(Kind, Symbol->getName(),
+                                       GOFF::PRAttr{false, GOFF::ESD_EXE_DATA,
+                                                    GOFF::ESD_LT_XPLink,
+                                                    PRBindingScope, 0},
+                                       ED);
+  }
+  return TextSection;
 }
diff --git a/llvm/lib/CodeGen/TargetOptionsImpl.cpp b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
index def325efd544..5eb86e740ff7 100644
--- a/llvm/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/llvm/lib/CodeGen/TargetOptionsImpl.cpp
@@ -39,12 +39,11 @@ bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
 
 bool TargetOptions::FramePointerIsReserved(const MachineFunction &MF) const {
   const Function &F = MF.getFunction();
-
-  if (!F.hasFnAttribute("frame-pointer"))
+  Attribute FPAttr = F.getFnAttribute("frame-pointer");
+  if (!FPAttr.isValid())
     return false;
 
-  StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString();
-  return StringSwitch<bool>(FP)
+  return StringSwitch<bool>(FPAttr.getValueAsString())
       .Cases("all", "non-leaf", "reserved", true)
       .Case("none", false);
 }
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 4ae52b056d84..7d7c6e743fa7 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -893,6 +893,9 @@ void TargetPassConfig::addIRPasses() {
 
   if (EnableGlobalMergeFunc)
     addPass(createGlobalMergeFuncPass());
+
+  if (TM->getTargetTriple().isOSWindows())
+    addPass(createWindowsSecureHotPatchingPass());
 }
 
 /// Turn exception handling constructs into something the code generators can
diff --git a/llvm/lib/CodeGen/WindowsSecureHotPatching.cpp b/llvm/lib/CodeGen/WindowsSecureHotPatching.cpp
new file mode 100644
index 000000000000..6267207dbcbf
--- /dev/null
+++ b/llvm/lib/CodeGen/WindowsSecureHotPatching.cpp
@@ -0,0 +1,617 @@
+//===------ WindowsHotPatch.cpp - Support for Windows hotpatching ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides support for the Windows "Secure Hot-Patching" feature.
+//
+// Windows contains technology, called "Secure Hot-Patching" (SHP), for securely
+// applying hot-patches to a running system. Hot-patches may be applied to the
+// kernel, kernel-mode components, device drivers, user-mode system services,
+// etc.
+//
+// SHP relies on integration between many tools, including compiler, linker,
+// hot-patch generation tools, and the Windows kernel. This file implements that
+// part of the workflow needed in compilers / code generators.
+//
+// SHP is not intended for productivity scenarios such as Edit-and-Continue or
+// interactive development. SHP is intended to minimize downtime during
+// installation of Windows OS patches.
+//
+// In order to work with SHP, LLVM must do all of the following:
+//
+// * On some architectures (X86, AMD64), the function prolog must begin with
+//   hot-patchable instructions. This is handled by the MSVC `/hotpatch` option
+//   and the equivalent `-fms-hotpatch` function. This is necessary because we
+//   generally cannot anticipate which functions will need to be patched in the
+//   future. This option ensures that a function can be hot-patched in the
+//   future, but does not actually generate any hot-patch for it.
+//
+// * For a selected set of functions that are being hot-patched (which are
+//   identified using command-line options), LLVM must generate the
+//   `S_HOTPATCHFUNC` CodeView record (symbol). This record indicates that a
+//   function was compiled with hot-patching enabled.
+//
+//   This implementation uses the `MarkedForWindowsHotPatching` attribute to
+//   annotate those functions that were marked for hot-patching by command-line
+//   parameters. The attribute may be specified by a language front-end by
+//   setting an attribute when a function is created in LLVM IR, or it may be
+//   set by passing LLVM arguments.
+//
+// * For those functions that are hot-patched, LLVM must rewrite references to
+//   global variables so that they are indirected through a `__ref_*` pointer
+//   variable.  For each global variable, that is accessed by a hot-patched
+//   function, e.g. `FOO`, a `__ref_FOO` global pointer variable is created and
+//   all references to the original `FOO` are rewritten as dereferences of the
+//   `__ref_FOO` pointer.
+//
+//   Some globals do not need `__ref_*` indirection. The pointer indirection
+//   behavior can be disabled for these globals by marking them with the
+//   `AllowDirectAccessInHotPatchFunction`.
+//
+// Rewriting references to global variables has some complexity.
+//
+// For ordinary instructions that reference GlobalVariables, we rewrite the
+// operand of the instruction to a Load of the __ref_* variable.
+//
+// For constant expressions, we have to convert the constant expression (and
+// transitively all constant expressions in its parent chain) to non-constant
+// expressions, i.e. to a sequence of instructions.
+//
+// Pass 1:
+//   * Enumerate all instructions in all basic blocks.
+//
+//   * If an instruction references a GlobalVariable (and it is not marked
+//     as being ignored), then we create (if necessary) the __ref_* variable
+//     for the GlobalVariable reference. However, we do not yet modify the
+//     Instruction.
+//
+//   * If an instruction has an operand that is a ConstantExpr and the
+//     ConstantExpression tree contains a reference to a GlobalVariable, then
+//     we similarly create __ref_*. Similarly, we do not yet modify the
+//     Instruction or the ConstantExpr tree.
+//
+// After Pass 1 completes, we will know whether we found any references to
+// globals in this pass.  If the function does not use any globals (and most
+// functions do not use any globals), then we return immediately.
+//
+// If a function does reference globals, then we iterate the list of globals
+// used by this function and we generate Load instructions for each (unique)
+// global.
+//
+// Next, we do another pass over all instructions:
+//
+// Pass 2:
+//   * Re-visit the instructions that were found in Pass 1.
+//
+//   * If an instruction operand is a GlobalVariable, then look up the
+//   replacement
+//     __ref_* global variable and the Value that came from the Load instruction
+//     for it.  Replace the operand of the GlobalVariable with the Load Value.
+//
+//   * If an instruction operand is a ConstantExpr, then recursively examine the
+//     operands of all instructions in the ConstantExpr tree.  If an operand is
+//     a GlobalVariable, then replace the operand with the result of the load
+//     *and* convert the ConstantExpr to a non-constant instruction.  This
+//     instruction will need to be inserted into the BB of the instruction whose
+//     operand is being modified, ideally immediately before the instruction
+//     being modified.
+//
+// Limitations
+//
+// This feature is not intended to work in every situation. There are many
+// legitimate code changes (patches) for which it is not possible to generate
+// a hot-patch. Developers who are writing hot-patches are expected to
+// understand the limitations.
+//
+// Tools which generate hot-patch metadata may also check that certain
+// variables are upheld, and some of these invariants may be global (may require
+// whole-program knowledge, not available in any single compiland). However,
+// such tools are not required to be perfect; they are also best-effort.
+//
+// For these reasons, the hot-patching support implemented in this file is
+// "best effort". It does not recognize every possible code pattern that could
+// be patched, nor does it generate diagnostics for certain code patterns that
+// could result in a binary that does not work with hot-patching. For example,
+// const GlobalVariables that point to other non-const GlobalVariables are not
+// compatible with hot-patching because they cannot use __ref_*-based
+// redirection.
+//
+// References
+//
+// * "Hotpatching on Windows":
+//   https://techcommunity.microsoft.com/blog/windowsosplatform/hotpatching-on-windows/2959541
+//
+// * "Hotpatch for Windows client now available":
+//   https://techcommunity.microsoft.com/blog/windows-itpro-blog/hotpatch-for-windows-client-now-available/4399808
+//
+// * "Get hotpatching for Windows Server":
+//   https://www.microsoft.com/en-us/windows-server/blog/2025/04/24/tired-of-all-the-restarts-get-hotpatching-for-windows-server/
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "windows-secure-hot-patch"
+
+// A file containing list of mangled function names to mark for hot patching.
+static cl::opt<std::string> LLVMMSSecureHotPatchFunctionsFile(
+    "ms-secure-hotpatch-functions-file", cl::value_desc("filename"),
+    cl::desc("A file containing list of mangled function names to mark for "
+             "Windows Secure Hot-Patching"));
+
+// A list of mangled function names to mark for hot patching.
+static cl::list<std::string> LLVMMSSecureHotPatchFunctionsList(
+    "ms-secure-hotpatch-functions-list", cl::value_desc("list"),
+    cl::desc("A list of mangled function names to mark for Windows Secure "
+             "Hot-Patching"),
+    cl::CommaSeparated);
+
+namespace {
+
+struct GlobalVariableUse {
+  // GlobalVariable *GV;
+  Instruction *User;
+  unsigned Op;
+};
+
+class WindowsSecureHotPatching : public ModulePass {
+public:
+  static char ID;
+
+  WindowsSecureHotPatching() : ModulePass(ID) {
+    initializeWindowsSecureHotPatchingPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+  }
+
+  bool doInitialization(Module &) override;
+  bool runOnModule(Module &M) override { return false; }
+
+private:
+  bool
+  runOnFunction(Function &F,
+                SmallDenseMap<GlobalVariable *, GlobalVariable *> &RefMapping);
+};
+
+} // end anonymous namespace
+
+char WindowsSecureHotPatching::ID = 0;
+
+INITIALIZE_PASS(WindowsSecureHotPatching, "windows-secure-hot-patch",
+                "Mark functions for Windows hot patch support", false, false)
+ModulePass *llvm::createWindowsSecureHotPatchingPass() {
+  return new WindowsSecureHotPatching();
+}
+
+// Find functions marked with Attribute::MarkedForWindowsHotPatching and modify
+// their code (if necessary) to account for accesses to global variables.
+//
+// This runs during doInitialization() instead of runOnModule() because it needs
+// to run before CodeViewDebug::collectGlobalVariableInfo().
+bool WindowsSecureHotPatching::doInitialization(Module &M) {
+  // The front end may have already marked functions for hot-patching. However,
+  // we also allow marking functions by passing -ms-hotpatch-functions-file or
+  // -ms-hotpatch-functions-list directly to LLVM. This allows hot-patching to
+  // work with languages that have not yet updated their front-ends.
+  if (!LLVMMSSecureHotPatchFunctionsFile.empty() ||
+      !LLVMMSSecureHotPatchFunctionsList.empty()) {
+    std::vector<std::string> HotPatchFunctionsList;
+
+    if (!LLVMMSSecureHotPatchFunctionsFile.empty()) {
+      auto BufOrErr = MemoryBuffer::getFile(LLVMMSSecureHotPatchFunctionsFile);
+      if (BufOrErr) {
+        const MemoryBuffer &FileBuffer = **BufOrErr;
+        for (line_iterator I(FileBuffer.getMemBufferRef(), true), E; I != E;
+             ++I)
+          HotPatchFunctionsList.push_back(std::string{*I});
+      } else {
+        M.getContext().diagnose(DiagnosticInfoGeneric{
+            Twine("failed to open hotpatch functions file "
+                  "(--ms-hotpatch-functions-file): ") +
+            LLVMMSSecureHotPatchFunctionsFile + Twine(" : ") +
+            BufOrErr.getError().message()});
+      }
+    }
+
+    if (!LLVMMSSecureHotPatchFunctionsList.empty())
+      for (const auto &FuncName : LLVMMSSecureHotPatchFunctionsList)
+        HotPatchFunctionsList.push_back(FuncName);
+
+    // Build a set for quick lookups. This points into HotPatchFunctionsList, so
+    // HotPatchFunctionsList must live longer than HotPatchFunctionsSet.
+    SmallSet<StringRef, 16> HotPatchFunctionsSet;
+    for (const auto &FuncName : HotPatchFunctionsList)
+      HotPatchFunctionsSet.insert(StringRef{FuncName});
+
+    // Iterate through all of the functions and check whether they need to be
+    // marked for hotpatching using the list provided directly to LLVM.
+    for (auto &F : M.functions()) {
+      // Ignore declarations that are not definitions.
+      if (F.isDeclarationForLinker())
+        continue;
+
+      if (HotPatchFunctionsSet.contains(F.getName()))
+        F.addFnAttr("marked_for_windows_hot_patching");
+    }
+  }
+
+  SmallDenseMap<GlobalVariable *, GlobalVariable *> RefMapping;
+  bool MadeChanges = false;
+  for (auto &F : M.functions()) {
+    if (F.hasFnAttribute("marked_for_windows_hot_patching")) {
+      if (runOnFunction(F, RefMapping))
+        MadeChanges = true;
+    }
+  }
+  return MadeChanges;
+}
+
+static bool TypeContainsPointers(Type *ty) {
+  switch (ty->getTypeID()) {
+  case Type::PointerTyID:
+    return true;
+
+  case Type::ArrayTyID:
+    return TypeContainsPointers(ty->getArrayElementType());
+
+  case Type::StructTyID: {
+    unsigned NumElements = ty->getStructNumElements();
+    for (unsigned I = 0; I < NumElements; ++I) {
+      if (TypeContainsPointers(ty->getStructElementType(I))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  default:
+    return false;
+  }
+}
+
+// Returns true if GV needs redirection through a __ref_* variable.
+static bool globalVariableNeedsRedirect(GlobalVariable *GV) {
+  // If a global variable is explictly marked as allowing access in hot-patched
+  // functions, then do not redirect it.
+  if (GV->hasAttribute("allow_direct_access_in_hot_patch_function"))
+    return false;
+
+  // If the global variable is not a constant, then we want to redirect it.
+  if (!GV->isConstant()) {
+    if (GV->getName().starts_with("??_R")) {
+      // This is the name mangling prefix that MSVC uses for RTTI data.
+      // Clang is currently generating RTTI data that is marked non-constant.
+      // We override that and treat it like it is constant.
+      return false;
+    }
+
+    // In general, if a global variable is not a constant, then redirect it.
+    return true;
+  }
+
+  // If the type of GV cannot contain pointers, then it cannot point to
+  // other global variables. In this case, there is no need for redirects.
+  // For example, string literals do not contain pointers.
+  return TypeContainsPointers(GV->getValueType());
+}
+
+// Get or create a new global variable that points to the old one and whose
+// name begins with `__ref_`.
+//
+// In hot-patched images, the __ref_* variables point to global variables in
+// the original (unpatched) image. Hot-patched functions in the hot-patch
+// image use these __ref_* variables to access global variables. This ensures
+// that all code (both unpatched and patched) is using the same instances of
+// global variables.
+//
+// The Windows hot-patch infrastructure handles modifying these __ref_*
+// variables. By default, they are initialized with pointers to the equivalent
+// global variables, so when a hot-patch module is loaded *as* a base image
+// (such as after a system reboot), hot-patch functions will access the
+// instances of global variables that are compiled into the hot-patch image.
+// This is the desired outcome, since in this situation (normal boot) the
+// hot-patch image *is* the base image.
+//
+// When we create the GlobalVariable for the __ref_* variable, we must create
+// it as a *non-constant* global variable. The __ref_* pointers will not change
+// during the runtime of the program, so it is tempting to think that they
+// should be constant. However, they still need to be updateable by the
+// hot-patching infrastructure. Also, if the GlobalVariable is created as a
+// constant, then the LLVM optimizer will assume that it can dereference the
+// definition of the __ref_* variable at compile time, which defeats the
+// purpose of the indirection (pointer).
+//
+// The RefMapping table spans the entire module, not just a single function.
+static GlobalVariable *getOrCreateRefVariable(
+    Function &F, SmallDenseMap<GlobalVariable *, GlobalVariable *> &RefMapping,
+    GlobalVariable *GV) {
+  GlobalVariable *&ReplaceWithRefGV = RefMapping.try_emplace(GV).first->second;
+  if (ReplaceWithRefGV != nullptr) {
+    // We have already created a __ref_* pointer for this GlobalVariable.
+    return ReplaceWithRefGV;
+  }
+
+  Module *M = F.getParent();
+
+  const DISubprogram *Subprogram = F.getSubprogram();
+  DICompileUnit *Unit = Subprogram != nullptr ? Subprogram->getUnit() : nullptr;
+  DIFile *File = Subprogram != nullptr ? Subprogram->getFile() : nullptr;
+  DIBuilder DebugInfo{*F.getParent(), true, Unit};
+
+  auto PtrTy = PointerType::get(M->getContext(), 0);
+
+  Constant *AddrOfOldGV =
+      ConstantExpr::getGetElementPtr(PtrTy, GV, ArrayRef<Value *>{});
+
+  GlobalVariable *RefGV =
+      new GlobalVariable(*M, PtrTy, false, GlobalValue::LinkOnceAnyLinkage,
+                         AddrOfOldGV, Twine("__ref_").concat(GV->getName()),
+                         nullptr, GlobalVariable::NotThreadLocal);
+
+  // Create debug info for the replacement global variable.
+  DataLayout Layout = M->getDataLayout();
+  DIType *DebugType = DebugInfo.createPointerType(
+      nullptr, Layout.getTypeSizeInBits(GV->getValueType()));
+  DIGlobalVariableExpression *GVE = DebugInfo.createGlobalVariableExpression(
+      Unit, RefGV->getName(), StringRef{}, File,
+      /*LineNo*/ 0, DebugType,
+      /*IsLocalToUnit*/ false);
+  RefGV->addDebugInfo(GVE);
+
+  // Store the __ref_* in RefMapping so that future calls use the same RefGV.
+  ReplaceWithRefGV = RefGV;
+
+  return RefGV;
+}
+
+// Given a ConstantExpr, this searches for GlobalVariable references within
+// the expression tree.  If found, it will generate instructions and will
+// return a non-null Value* that points to the new root instruction.
+//
+// If C does not contain any GlobalVariable references, this returns nullptr.
+//
+// If this function creates new instructions, then it will insert them
+// before InsertionPoint.
+static Value *rewriteGlobalVariablesInConstant(
+    Constant *C, SmallDenseMap<GlobalVariable *, Value *> &GVLoadMap,
+    IRBuilder<> &IRBuilderAtEntry) {
+  if (C->getValueID() == Value::GlobalVariableVal) {
+    GlobalVariable *GV = cast<GlobalVariable>(C);
+    if (globalVariableNeedsRedirect(GV)) {
+      return GVLoadMap.at(GV);
+    } else {
+      return nullptr;
+    }
+  }
+
+  // Scan the operands of this expression.
+
+  SmallVector<Value *, 8> ReplacedValues;
+  bool ReplacedAnyOperands = false;
+
+  unsigned NumOperands = C->getNumOperands();
+  for (unsigned OpIndex = 0; OpIndex < NumOperands; ++OpIndex) {
+    Value *OldValue = C->getOperand(OpIndex);
+    Value *ReplacedValue = nullptr;
+    if (Constant *OldConstant = dyn_cast<Constant>(OldValue)) {
+      ReplacedValue = rewriteGlobalVariablesInConstant(OldConstant, GVLoadMap,
+                                                       IRBuilderAtEntry);
+    }
+    // Do not use short-circuiting, here. We need to traverse the whole tree.
+    ReplacedAnyOperands |= ReplacedValue != nullptr;
+    ReplacedValues.push_back(ReplacedValue);
+  }
+
+  // If none of our operands were replaced, then don't rewrite this expression.
+  if (!ReplacedAnyOperands) {
+    return nullptr;
+  }
+
+  // We need to rewrite this expression. Convert this constant expression
+  // to an instruction, then replace any operands as needed.
+  Instruction *NewInst = cast<ConstantExpr>(C)->getAsInstruction();
+  for (unsigned OpIndex = 0; OpIndex < NumOperands; ++OpIndex) {
+    Value *ReplacedValue = ReplacedValues[OpIndex];
+    if (ReplacedValue != nullptr) {
+      NewInst->setOperand(OpIndex, ReplacedValue);
+    }
+  }
+
+  // Insert the new instruction before the reference instruction.
+  IRBuilderAtEntry.Insert(NewInst);
+
+  return NewInst;
+}
+
+static bool searchConstantExprForGlobalVariables(
+    Value *V, SmallDenseMap<GlobalVariable *, Value *> &GVLoadMap,
+    SmallVector<GlobalVariableUse> &GVUses) {
+
+  SmallVector<Value *, 8> ReplacedOperands;
+
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) {
+    if (globalVariableNeedsRedirect(GV)) {
+      GVLoadMap[GV] = nullptr;
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  if (User *U = dyn_cast<User>(V)) {
+    unsigned NumOperands = U->getNumOperands();
+    bool FoundAny = false;
+    for (unsigned OpIndex = 0; OpIndex < NumOperands; ++OpIndex) {
+      Value *Op = U->getOperand(OpIndex);
+      // Do not use short-circuiting, here. We need to traverse the whole tree.
+      FoundAny |= searchConstantExprForGlobalVariables(Op, GVLoadMap, GVUses);
+    }
+    return FoundAny;
+  } else {
+    return false;
+  }
+}
+
+// Processes a function that is marked for hot-patching.
+//
+// If a function is marked for hot-patching, we generate an S_HOTPATCHFUNC
+// CodeView debug symbol. Tools that generate hot-patches look for
+// S_HOTPATCHFUNC in final PDBs so that they can find functions that have been
+// hot-patched and so that they can distinguish hot-patched functions from
+// non-hot-patched functions.
+//
+// Also, in functions that are hot-patched, we must indirect all access to
+// (mutable) global variables through a pointer. This pointer may point into the
+// unpatched ("base") binary or may point into the patched image, depending on
+// whether a hot-patch was loaded as a patch or as a base image.  These
+// indirections go through a new global variable, named `__ref_<Foo>` where
+// `<Foo>` is the original symbol name of the global variable.
+//
+// This function handles rewriting accesses to global variables, but the
+// generation of S_HOTPATCHFUNC occurs in
+// CodeViewDebug::emitHotPatchInformation().
+//
+// Returns true if any global variable references were found and rewritten.
+bool WindowsSecureHotPatching::runOnFunction(
+    Function &F,
+    SmallDenseMap<GlobalVariable *, GlobalVariable *> &RefMapping) {
+  // Scan the function for references to global variables. If we find such a
+  // reference, create (if necessary) the __ref_* variable, then add an entry
+  // to the GVUses table.
+  //
+  // We ignore references to global variables if the variable is marked with
+  // AllowDirectAccessInHotPatchFunction.
+
+  SmallDenseMap<GlobalVariable *, Value *> GVLoadMap;
+  SmallVector<GlobalVariableUse> GVUses;
+
+  for (auto &I : instructions(F)) {
+    unsigned NumOperands = I.getNumOperands();
+    for (unsigned OpIndex = 0; OpIndex < NumOperands; ++OpIndex) {
+      Value *V = I.getOperand(OpIndex);
+
+      bool FoundAnyGVUses = false;
+
+      switch (V->getValueID()) {
+      case Value::GlobalVariableVal: {
+        // Discover all uses of GlobalVariable, these will need to be replaced.
+        GlobalVariable *GV = cast<GlobalVariable>(V);
+        if (globalVariableNeedsRedirect(GV)) {
+          GVLoadMap.insert(std::make_pair(GV, nullptr));
+          FoundAnyGVUses = true;
+        }
+        break;
+      }
+
+      case Value::ConstantExprVal: {
+        ConstantExpr *CE = cast<ConstantExpr>(V);
+        if (searchConstantExprForGlobalVariables(CE, GVLoadMap, GVUses)) {
+          FoundAnyGVUses = true;
+        }
+        break;
+      }
+
+      default:
+        break;
+      }
+
+      if (FoundAnyGVUses) {
+        GVUses.push_back(GlobalVariableUse{&I, OpIndex});
+      }
+    }
+  }
+
+  // If this function did not reference any global variables then we have no
+  // work to do. Most functions do not access global variables.
+  if (GVUses.empty()) {
+    return false;
+  }
+
+  // We know that there is at least one instruction that needs to be rewritten.
+  // Generate a Load instruction for each unique GlobalVariable used by this
+  // function. The Load instructions are inserted at the beginning of the
+  // entry block. Since entry blocks cannot contain PHI instructions, there is
+  // no need to skip PHI instructions.
+
+  // We use a single IRBuilder for inserting Load instructions as well as the
+  // constants that we convert to instructions. Because constants do not
+  // depend on any dynamic values (they're constant, after all!), it is safe
+  // to move them to the start of entry BB.
+
+  auto &EntryBlock = F.getEntryBlock();
+  IRBuilder<> IRBuilderAtEntry(&EntryBlock, EntryBlock.begin());
+
+  for (auto &[GV, LoadValue] : GVLoadMap) {
+    assert(LoadValue == nullptr);
+    GlobalVariable *RefGV = getOrCreateRefVariable(F, RefMapping, GV);
+    LoadValue = IRBuilderAtEntry.CreateLoad(RefGV->getValueType(), RefGV);
+  }
+
+  const DISubprogram *Subprogram = F.getSubprogram();
+  DICompileUnit *Unit = Subprogram != nullptr ? Subprogram->getUnit() : nullptr;
+  DIBuilder DebugInfo{*F.getParent(), true, Unit};
+
+  // Go back to the instructions and rewrite their uses of GlobalVariable.
+  // Because a ConstantExpr can be a tree, it may reference more than one
+  // GlobalVariable.
+
+  for (auto &GVUse : GVUses) {
+    Value *OldOperandValue = GVUse.User->getOperand(GVUse.Op);
+    Value *NewOperandValue;
+
+    switch (OldOperandValue->getValueID()) {
+    case Value::GlobalVariableVal: {
+      // This is easy. Look up the replacement value and store the operand.
+      Value *OperandValue = GVUse.User->getOperand(GVUse.Op);
+      GlobalVariable *GV = cast<GlobalVariable>(OperandValue);
+      NewOperandValue = GVLoadMap.at(GV);
+      break;
+    }
+
+    case Value::ConstantExprVal: {
+      // Walk the recursive tree of the ConstantExpr. If we find a
+      // GlobalVariable then replace it with the loaded value and rewrite
+      // the ConstantExpr to an Instruction and insert it before the
+      // current instruction.
+      Value *OperandValue = GVUse.User->getOperand(GVUse.Op);
+      ConstantExpr *CE = cast<ConstantExpr>(OperandValue);
+      NewOperandValue =
+          rewriteGlobalVariablesInConstant(CE, GVLoadMap, IRBuilderAtEntry);
+      assert(NewOperandValue != nullptr);
+      break;
+    }
+
+    default:
+      // We should only ever get here because a GVUse was created in the first
+      // pass, and this only happens for GlobalVariableVal and ConstantExprVal.
+      llvm_unreachable_internal(
+          "unexpected Value in second pass of hot-patching");
+      break;
+    }
+
+    GVUse.User->setOperand(GVUse.Op, NewOperandValue);
+  }
+
+  return true;
+}
diff --git a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
index d291be6c6439..8d065bf793f3 100644
--- a/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
+++ b/llvm/lib/DWARFLinker/Parallel/DWARFLinkerCompileUnit.cpp
@@ -101,10 +101,8 @@ void CompileUnit::maybeResetToLoadedStage() {
   OutUnitDIE = nullptr;
   DebugAddrIndexMap.clear();
 
-  for (uint64_t &Offset : OutDieOffsetArray)
-    Offset = 0;
-  for (TypeEntry *&Name : TypeEntries)
-    Name = nullptr;
+  llvm::fill(OutDieOffsetArray, 0);
+  llvm::fill(TypeEntries, nullptr);
   eraseSections();
 
   setStage(Stage::CreatedNotLoaded);
diff --git a/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp b/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp
index f56739db7c75..9cb3bca8d6e5 100644
--- a/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp
+++ b/llvm/lib/DebugInfo/CodeView/SymbolDumper.cpp
@@ -672,6 +672,13 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR,
   return Error::success();
 }
 
+Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR,
+                                           HotPatchFuncSym &HotPatchFunc) {
+  printTypeIndex("Function", HotPatchFunc.Function);
+  W.printString("Name", HotPatchFunc.Name);
+  return Error::success();
+}
+
 Error CVSymbolDumperImpl::visitUnknownSymbol(CVSymbol &CVR) {
   W.printNumber("Length", CVR.length());
   return Error::success();
diff --git a/llvm/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp b/llvm/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
index b5e366b965a9..525343b90a3a 100644
--- a/llvm/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
+++ b/llvm/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
@@ -496,6 +496,13 @@ Error SymbolRecordMapping::visitKnownRecord(CVSymbol &CVR,
   return Error::success();
 }
 
+Error SymbolRecordMapping::visitKnownRecord(CVSymbol &CVR,
+                                            HotPatchFuncSym &HotPatchFunc) {
+  error(IO.mapInteger(HotPatchFunc.Function));
+  error(IO.mapStringZ(HotPatchFunc.Name));
+  return Error::success();
+}
+
 RegisterId codeview::decodeFramePtrReg(EncodedFramePtrReg EncodedReg,
                                        CPUType CPU) {
   assert(unsigned(EncodedReg) < 4);
diff --git a/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt b/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt
new file mode 100644
index 000000000000..b92327778330
--- /dev/null
+++ b/llvm/lib/DebugInfo/DWARF/LowLevel/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_llvm_component_library(LLVMDebugInfoDWARFLowLevel
+  DWARFCFIProgram.cpp
+  DWARFExpression.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/DebugInfo/DWARF/LowLevel
+
+  # This code should have almost no dependencies. Anything that needs
+  # more should use the higher-level DebugInfo/DWARF/ library.
+  LINK_COMPONENTS
+  BinaryFormat
+  Support
+  TargetParser
+  )
diff --git a/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp b/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
index 8cd4e22d34fc..4af8d3044f4d 100644
--- a/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
+++ b/llvm/lib/DebugInfo/GSYM/CallSiteInfo.cpp
@@ -12,6 +12,7 @@
 #include "llvm/DebugInfo/GSYM/GsymCreator.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -231,13 +232,7 @@ Error CallSiteInfoLoader::processYAMLFunctions(
 raw_ostream &gsym::operator<<(raw_ostream &OS, const CallSiteInfo &CSI) {
   OS << "  Return=" << HEX64(CSI.ReturnOffset);
   OS << "  Flags=" << HEX8(CSI.Flags);
-
-  OS << "  RegEx=";
-  for (uint32_t i = 0; i < CSI.MatchRegex.size(); ++i) {
-    if (i > 0)
-      OS << ",";
-    OS << CSI.MatchRegex[i];
-  }
+  OS << "  RegEx=" << llvm::interleaved(CSI.MatchRegex, ",");
   return OS;
 }
 
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
index 55880fab5e88..93a567e89f77 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVScope.cpp
@@ -2097,7 +2097,7 @@ Error LVScopeRoot::doPrintMatches(bool Split, raw_ostream &OS,
     print(OS);
 
     for (LVScope *Scope : *Scopes) {
-      getReader().setCompileUnit(const_cast<LVScope *>(Scope));
+      getReader().setCompileUnit(Scope);
 
       // If 'Split', we use the scope name (CU name) as the ouput file; the
       // delimiters in the pathname, must be replaced by a normal character.
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 7ff96ae90a7f..95b7007d119b 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -214,10 +214,11 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
         }
       }
       if (FoundLowPC) {
-        if (CurrentLowPC == MaxAddress)
+        if (CurrentLowPC == getTombstoneAddress())
           CurrentElement->setIsDiscarded();
-        // Consider the case of WebAssembly.
-        CurrentLowPC += WasmCodeSectionOffset;
+        else
+          // Consider the case of WebAssembly.
+          CurrentLowPC += WasmCodeSectionOffset;
         if (CurrentElement->isCompileUnit())
           setCUBaseAddress(CurrentLowPC);
       }
@@ -271,7 +272,8 @@ void LVDWARFReader::processOneAttribute(const DWARFDie &Die,
       DWARFAddressRangesVector Ranges = RangesOrError.get();
       for (DWARFAddressRange &Range : Ranges) {
         // This seems to be a tombstone for empty ranges.
-        if (Range.LowPC == Range.HighPC)
+        if ((Range.LowPC == Range.HighPC) ||
+            (Range.LowPC = getTombstoneAddress()))
           continue;
         // Store the real upper limit for the address range.
         if (UpdateHighAddress && Range.HighPC > 0)
@@ -629,6 +631,11 @@ Error LVDWARFReader::createScopes() {
                                          : DwarfContext->dwo_compile_units();
   for (const std::unique_ptr<DWARFUnit> &CU : CompileUnits) {
 
+    // Take into account the address byte size for a correct 'tombstone'
+    // value identification.
+    setTombstoneAddress(
+        dwarf::computeTombstoneAddress(CU->getAddressByteSize()));
+
     // Deduction of index used for the line records.
     //
     // For the following test case: test.cpp
diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 09cb355ca731..c9a1eebb8a85 100644
--- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -119,7 +119,7 @@ static CVSymbol serializePublic(uint8_t *Mem, const BulkPublic &Pub) {
   memcpy(NameMem, Pub.Name, NameLen);
   // Zero the null terminator and remaining bytes.
   memset(&NameMem[NameLen], 0, Size - sizeof(PublicSym32Layout) - NameLen);
-  return CVSymbol(ArrayRef(reinterpret_cast<uint8_t *>(Mem), Size));
+  return CVSymbol(ArrayRef(Mem, Size));
 }
 
 uint32_t GSIHashStreamBuilder::calculateSerializedLength() const {
diff --git a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
index 679881054f9d..37d1a28b4f35 100644
--- a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
+++ b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
@@ -19,7 +19,6 @@
 #include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h"
-#include "llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypePointer.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h"
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index 9455a3847692..01456d54465d 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -952,8 +952,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue floatZero;
         floatZero.FloatVal = 0.f;
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  floatZero);
+        llvm::fill(Result.AggregateVal, floatZero);
         break;
       }
       if(CV) {
@@ -974,8 +973,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue doubleZero;
         doubleZero.DoubleVal = 0.0;
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  doubleZero);
+        llvm::fill(Result.AggregateVal, doubleZero);
         break;
       }
       if(CV) {
@@ -996,8 +994,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       if (CAZ) {
         GenericValue intZero;
         intZero.IntVal = APInt(ElemTy->getScalarSizeInBits(), 0ull);
-        std::fill(Result.AggregateVal.begin(), Result.AggregateVal.end(),
-                  intZero);
+        llvm::fill(Result.AggregateVal, intZero);
         break;
       }
       if(CV) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
index b617fe222df0..cd2eed748acd 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp
@@ -523,7 +523,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
     case ELFTLSDescCall:
       return "ELFTLSDescCall";
     default:
-      return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+      return getGenericEdgeKindName(R);
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 3af0c0cdeb7c..308dac48cbf7 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -545,7 +545,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
     case MachONegDelta64:
       return "MachONegDelta64";
     default:
-      return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+      return getGenericEdgeKindName(R);
     }
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/XCOFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/XCOFFLinkGraphBuilder.cpp
index 13099295edde..f0a14c6b1c24 100644
--- a/llvm/lib/ExecutionEngine/JITLink/XCOFFLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/XCOFFLinkGraphBuilder.cpp
@@ -296,7 +296,7 @@ Error XCOFFLinkGraphBuilder::processCsectsAndSymbols() {
       if (!CsectSymbolAddr)
         return CsectSymbolAddr.takeError();
 
-      ArrayRef<char> SectionBuffer{Data->data(), Data->size()};
+      ArrayRef<char> SectionBuffer{*Data};
       auto Offset = *CsectSymbolAddr - SectionRef.getAddress();
 
       LLVM_DEBUG(dbgs() << "      symbol entry: offset = " << Offset
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
index fbeb920f4e18..b19b839aab7f 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp
@@ -85,7 +85,7 @@ const char *getEdgeKindName(Edge::Kind R) {
   case RequestTLSDescEntryAndTransformToPageOffset12:
     return "RequestTLSDescEntryAndTransformToPageOffset12";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+    return getGenericEdgeKindName(R);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
index 27484aaf2059..3280c5eed525 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp
@@ -137,7 +137,7 @@ const char *getEdgeKindName(Edge::Kind K) {
   case RequestTLSDescInGOTAndTransformToDelta34:
     return "RequestTLSDescInGOTAndTransformToDelta34";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(K));
+    return getGenericEdgeKindName(K);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
index 6ac991651f08..9d312a281315 100644
--- a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp
@@ -75,7 +75,7 @@ const char *getEdgeKindName(Edge::Kind K) {
   case RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable:
     return "RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable";
   default:
-    return getGenericEdgeKindName(static_cast<Edge::Kind>(K));
+    return getGenericEdgeKindName(K);
   }
 }
 
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index 6c86f252bf7e..f159d59fa3ac 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -27,6 +27,7 @@ add_llvm_component_library(LLVMOrcJIT
   ObjectFileInterface.cpp
   GetDylibInterface.cpp
   IndirectionUtils.cpp
+  InProcessMemoryAccess.cpp
   IRCompileLayer.cpp
   IRTransformLayer.cpp
   IRPartitionLayer.cpp
@@ -52,6 +53,7 @@ add_llvm_component_library(LLVMOrcJIT
   OrcV2CBindings.cpp
   RTDyldObjectLinkingLayer.cpp
   SectCreate.cpp
+  SelfExecutorProcessControl.cpp
   SimpleRemoteEPC.cpp
   Speculation.cpp
   SpeculateAnalyses.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
index 45cb28af5605..9b3bae4e6d2a 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp
@@ -7,222 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/DylibManager.h"
+#include "llvm/ExecutionEngine/Orc/MemoryAccess.h"
 
-#include "llvm/ExecutionEngine/Orc/Core.h"
-#include "llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h"
-#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h"
-#include "llvm/Support/Process.h"
-#include "llvm/TargetParser/Host.h"
-
-#define DEBUG_TYPE "orc"
-
-namespace llvm {
-namespace orc {
+namespace llvm::orc {
 
 DylibManager::~DylibManager() = default;
 
-ExecutorProcessControl::MemoryAccess::~MemoryAccess() = default;
-
 ExecutorProcessControl::~ExecutorProcessControl() = default;
 
-SelfExecutorProcessControl::SelfExecutorProcessControl(
-    std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
-    Triple TargetTriple, unsigned PageSize,
-    std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr)
-    : ExecutorProcessControl(std::move(SSP), std::move(D)),
-      InProcessMemoryAccess(TargetTriple.isArch64Bit()) {
-
-  OwnedMemMgr = std::move(MemMgr);
-  if (!OwnedMemMgr)
-    OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(
-        sys::Process::getPageSizeEstimate());
-
-  this->TargetTriple = std::move(TargetTriple);
-  this->PageSize = PageSize;
-  this->MemMgr = OwnedMemMgr.get();
-  this->MemAccess = this;
-  this->DylibMgr = this;
-  this->JDI = {ExecutorAddr::fromPtr(jitDispatchViaWrapperFunctionManager),
-               ExecutorAddr::fromPtr(this)};
-
-  if (this->TargetTriple.isOSBinFormatMachO())
-    GlobalManglingPrefix = '_';
-
-  addDefaultBootstrapValuesForHostProcess(BootstrapMap, BootstrapSymbols);
-
-#ifdef __APPLE__
-  // FIXME: Don't add an UnwindInfoManager by default -- it's redundant when
-  //        the ORC runtime is loaded. We'll need a way to document this and
-  //        allow clients to choose.
-  if (UnwindInfoManager::TryEnable())
-    UnwindInfoManager::addBootstrapSymbols(this->BootstrapSymbols);
-#endif // __APPLE__
-}
-
-Expected<std::unique_ptr<SelfExecutorProcessControl>>
-SelfExecutorProcessControl::Create(
-    std::shared_ptr<SymbolStringPool> SSP,
-    std::unique_ptr<TaskDispatcher> D,
-    std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) {
-
-  if (!SSP)
-    SSP = std::make_shared<SymbolStringPool>();
-
-  if (!D)
-    D = std::make_unique<InPlaceTaskDispatcher>();
-
-  auto PageSize = sys::Process::getPageSize();
-  if (!PageSize)
-    return PageSize.takeError();
-
-  Triple TT(sys::getProcessTriple());
-
-  return std::make_unique<SelfExecutorProcessControl>(
-      std::move(SSP), std::move(D), std::move(TT), *PageSize,
-      std::move(MemMgr));
-}
-
-Expected<tpctypes::DylibHandle>
-SelfExecutorProcessControl::loadDylib(const char *DylibPath) {
-  std::string ErrMsg;
-  auto Dylib = sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg);
-  if (!Dylib.isValid())
-    return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
-  return ExecutorAddr::fromPtr(Dylib.getOSSpecificHandle());
-}
-
-void SelfExecutorProcessControl::lookupSymbolsAsync(
-    ArrayRef<LookupRequest> Request,
-    DylibManager::SymbolLookupCompleteFn Complete) {
-  std::vector<tpctypes::LookupResult> R;
-
-  for (auto &Elem : Request) {
-    sys::DynamicLibrary Dylib(Elem.Handle.toPtr<void *>());
-    R.push_back(std::vector<ExecutorSymbolDef>());
-    for (auto &KV : Elem.Symbols) {
-      auto &Sym = KV.first;
-      std::string Tmp((*Sym).data() + !!GlobalManglingPrefix,
-                      (*Sym).size() - !!GlobalManglingPrefix);
-      void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str());
-      if (!Addr && KV.second == SymbolLookupFlags::RequiredSymbol) {
-        // FIXME: Collect all failing symbols before erroring out.
-        SymbolNameVector MissingSymbols;
-        MissingSymbols.push_back(Sym);
-        return Complete(
-            make_error<SymbolsNotFound>(SSP, std::move(MissingSymbols)));
-      }
-      // FIXME: determine accurate JITSymbolFlags.
-      R.back().push_back(
-          {ExecutorAddr::fromPtr(Addr), JITSymbolFlags::Exported});
-    }
-  }
-
-  Complete(std::move(R));
-}
-
-Expected<int32_t>
-SelfExecutorProcessControl::runAsMain(ExecutorAddr MainFnAddr,
-                                      ArrayRef<std::string> Args) {
-  using MainTy = int (*)(int, char *[]);
-  return orc::runAsMain(MainFnAddr.toPtr<MainTy>(), Args);
-}
-
-Expected<int32_t>
-SelfExecutorProcessControl::runAsVoidFunction(ExecutorAddr VoidFnAddr) {
-  using VoidTy = int (*)();
-  return orc::runAsVoidFunction(VoidFnAddr.toPtr<VoidTy>());
-}
-
-Expected<int32_t>
-SelfExecutorProcessControl::runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) {
-  using IntTy = int (*)(int);
-  return orc::runAsIntFunction(IntFnAddr.toPtr<IntTy>(), Arg);
-}
-
-void SelfExecutorProcessControl::callWrapperAsync(ExecutorAddr WrapperFnAddr,
-                                                  IncomingWFRHandler SendResult,
-                                                  ArrayRef<char> ArgBuffer) {
-  using WrapperFnTy =
-      shared::CWrapperFunctionResult (*)(const char *Data, size_t Size);
-  auto *WrapperFn = WrapperFnAddr.toPtr<WrapperFnTy>();
-  SendResult(WrapperFn(ArgBuffer.data(), ArgBuffer.size()));
-}
-
-Error SelfExecutorProcessControl::disconnect() {
-  D->shutdown();
-  return Error::success();
-}
-
-void InProcessMemoryAccess::writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
-                                             WriteResultFn OnWriteComplete) {
-  for (auto &W : Ws)
-    *W.Addr.toPtr<uint8_t *>() = W.Value;
-  OnWriteComplete(Error::success());
-}
-
-void InProcessMemoryAccess::writeUInt16sAsync(
-    ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) {
-  for (auto &W : Ws)
-    *W.Addr.toPtr<uint16_t *>() = W.Value;
-  OnWriteComplete(Error::success());
-}
-
-void InProcessMemoryAccess::writeUInt32sAsync(
-    ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) {
-  for (auto &W : Ws)
-    *W.Addr.toPtr<uint32_t *>() = W.Value;
-  OnWriteComplete(Error::success());
-}
-
-void InProcessMemoryAccess::writeUInt64sAsync(
-    ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) {
-  for (auto &W : Ws)
-    *W.Addr.toPtr<uint64_t *>() = W.Value;
-  OnWriteComplete(Error::success());
-}
-
-void InProcessMemoryAccess::writeBuffersAsync(
-    ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) {
-  for (auto &W : Ws)
-    memcpy(W.Addr.toPtr<char *>(), W.Buffer.data(), W.Buffer.size());
-  OnWriteComplete(Error::success());
-}
-
-void InProcessMemoryAccess::writePointersAsync(
-    ArrayRef<tpctypes::PointerWrite> Ws, WriteResultFn OnWriteComplete) {
-  if (IsArch64Bit) {
-    for (auto &W : Ws)
-      *W.Addr.toPtr<uint64_t *>() = W.Value.getValue();
-  } else {
-    for (auto &W : Ws)
-      *W.Addr.toPtr<uint32_t *>() = static_cast<uint32_t>(W.Value.getValue());
-  }
-
-  OnWriteComplete(Error::success());
-}
-
-shared::CWrapperFunctionResult
-SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager(
-    void *Ctx, const void *FnTag, const char *Data, size_t Size) {
-
-  LLVM_DEBUG({
-    dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size
-           << " byte payload.\n";
-  });
-
-  std::promise<shared::WrapperFunctionResult> ResultP;
-  auto ResultF = ResultP.get_future();
-  static_cast<SelfExecutorProcessControl *>(Ctx)
-      ->getExecutionSession()
-      .runJITDispatchHandler(
-          [ResultP = std::move(ResultP)](
-              shared::WrapperFunctionResult Result) mutable {
-            ResultP.set_value(std::move(Result));
-          },
-          ExecutorAddr::fromPtr(FnTag), {Data, Size});
-
-  return ResultF.get().release();
-}
-
-} // end namespace orc
-} // end namespace llvm
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/InProcessMemoryAccess.cpp b/llvm/lib/ExecutionEngine/Orc/InProcessMemoryAccess.cpp
new file mode 100644
index 000000000000..261eedc5461e
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/InProcessMemoryAccess.cpp
@@ -0,0 +1,134 @@
+//===------ InProcessMemoryAccess.cpp - Direct, in-process mem access -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm::orc {
+
+MemoryAccess::~MemoryAccess() = default;
+
+void InProcessMemoryAccess::writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws,
+                                             WriteResultFn OnWriteComplete) {
+  for (auto &W : Ws)
+    *W.Addr.toPtr<uint8_t *>() = W.Value;
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::writeUInt16sAsync(
+    ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) {
+  for (auto &W : Ws)
+    *W.Addr.toPtr<uint16_t *>() = W.Value;
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::writeUInt32sAsync(
+    ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) {
+  for (auto &W : Ws)
+    *W.Addr.toPtr<uint32_t *>() = W.Value;
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::writeUInt64sAsync(
+    ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) {
+  for (auto &W : Ws)
+    *W.Addr.toPtr<uint64_t *>() = W.Value;
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::writePointersAsync(
+    ArrayRef<tpctypes::PointerWrite> Ws, WriteResultFn OnWriteComplete) {
+  if (IsArch64Bit) {
+    for (auto &W : Ws)
+      *W.Addr.toPtr<uint64_t *>() = W.Value.getValue();
+  } else {
+    for (auto &W : Ws)
+      *W.Addr.toPtr<uint32_t *>() = static_cast<uint32_t>(W.Value.getValue());
+  }
+
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::writeBuffersAsync(
+    ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) {
+  for (auto &W : Ws)
+    memcpy(W.Addr.toPtr<char *>(), W.Buffer.data(), W.Buffer.size());
+  OnWriteComplete(Error::success());
+}
+
+void InProcessMemoryAccess::readUInt8sAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadUIntsCompleteFn<uint8_t> OnComplete) {
+  ReadUIntsResult<uint8_t> Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs)
+    Result.push_back(*R.toPtr<uint8_t *>());
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readUInt16sAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadUIntsCompleteFn<uint16_t> OnComplete) {
+  ReadUIntsResult<uint16_t> Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs)
+    Result.push_back(*R.toPtr<uint16_t *>());
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readUInt32sAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadUIntsCompleteFn<uint32_t> OnComplete) {
+  ReadUIntsResult<uint32_t> Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs)
+    Result.push_back(*R.toPtr<uint32_t *>());
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readUInt64sAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadUIntsCompleteFn<uint64_t> OnComplete) {
+  ReadUIntsResult<uint64_t> Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs)
+    Result.push_back(*R.toPtr<uint64_t *>());
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readPointersAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadPointersCompleteFn OnComplete) {
+  ReadPointersResult Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs)
+    Result.push_back(ExecutorAddr::fromPtr(*R.toPtr<void **>()));
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readBuffersAsync(
+    ArrayRef<ExecutorAddrRange> Rs, OnReadBuffersCompleteFn OnComplete) {
+  ReadBuffersResult Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs) {
+    Result.push_back({});
+    Result.back().resize(R.size());
+    memcpy(Result.back().data(), R.Start.toPtr<char *>(), R.size());
+  }
+  OnComplete(std::move(Result));
+}
+
+void InProcessMemoryAccess::readStringsAsync(
+    ArrayRef<ExecutorAddr> Rs, OnReadStringsCompleteFn OnComplete) {
+  ReadStringsResult Result;
+  Result.reserve(Rs.size());
+  for (auto &R : Rs) {
+    Result.push_back({});
+    for (auto *P = R.toPtr<char *>(); *P; ++P)
+      Result.back().push_back(*P);
+  }
+  OnComplete(std::move(Result));
+}
+
+} // end namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index f1a71e4acb46..67bb7dd8ad08 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h"
 #include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
@@ -631,16 +632,19 @@ Error ORCPlatformSupport::initialize(orc::JITDylib &JD) {
       int32_t result;
       auto E = ES.callSPSWrapper<SPSDLUpdateSig>(WrapperAddr->getAddress(),
                                                  result, DSOHandles[&JD]);
-      if (result)
+      if (E)
+        return E;
+      else if (result)
         return make_error<StringError>("dlupdate failed",
                                        inconvertibleErrorCode());
-      return E;
-    }
-    return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
-                                           DSOHandles[&JD], JD.getName(),
-                                           int32_t(ORC_RT_RTLD_LAZY));
+    } else
+      return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(),
+                                             DSOHandles[&JD], JD.getName(),
+                                             int32_t(ORC_RT_RTLD_LAZY));
   } else
     return WrapperAddr.takeError();
+
+  return Error::success();
 }
 
 Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) {
diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
index 9999e1ff3bf0..fd805fbf01fb 100644
--- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp
@@ -729,9 +729,9 @@ LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) {
   return wrap(new ThreadSafeContext(std::make_unique<LLVMContext>()));
 }
 
-LLVMContextRef
-LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx) {
-  return wrap(unwrap(TSCtx)->getContext());
+LLVMOrcThreadSafeContextRef
+LLVMOrcCreateNewThreadSafeContextFromLLVMContext(LLVMContextRef Ctx) {
+  return wrap(new ThreadSafeContext(std::unique_ptr<LLVMContext>(unwrap(Ctx))));
 }
 
 void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx) {
diff --git a/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp b/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
index c50654cf47a2..1e889e141ef9 100644
--- a/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SectCreate.cpp
@@ -23,8 +23,7 @@ void SectCreateMaterializationUnit::materialize(
       SubtargetFeatures(), getGenericEdgeKindName);
 
   auto &Sect = G->createSection(SectName, MP);
-  auto Content = G->allocateContent(
-      ArrayRef<char>(Data->getBuffer().data(), Data->getBuffer().size()));
+  auto Content = G->allocateContent(ArrayRef<char>(Data->getBuffer()));
   auto &B = G->createContentBlock(Sect, Content, ExecutorAddr(), Alignment, 0);
 
   for (auto &[Name, Info] : ExtraSymbols) {
diff --git a/llvm/lib/ExecutionEngine/Orc/SelfExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/SelfExecutorProcessControl.cpp
new file mode 100644
index 000000000000..78045f1491f0
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/SelfExecutorProcessControl.cpp
@@ -0,0 +1,172 @@
+//===------ SelfExecutorProcessControl.cpp -- EPC for in-process JITs -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/DefaultHostBootstrapValues.h"
+#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h"
+#include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/Process.h"
+#include "llvm/TargetParser/Host.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm::orc {
+
+SelfExecutorProcessControl::SelfExecutorProcessControl(
+    std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
+    Triple TargetTriple, unsigned PageSize,
+    std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr)
+    : ExecutorProcessControl(std::move(SSP), std::move(D)),
+      IPMA(TargetTriple.isArch64Bit()) {
+
+  OwnedMemMgr = std::move(MemMgr);
+  if (!OwnedMemMgr)
+    OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(
+        sys::Process::getPageSizeEstimate());
+
+  this->TargetTriple = std::move(TargetTriple);
+  this->PageSize = PageSize;
+  this->MemMgr = OwnedMemMgr.get();
+  this->MemAccess = &IPMA;
+  this->DylibMgr = this;
+  this->JDI = {ExecutorAddr::fromPtr(jitDispatchViaWrapperFunctionManager),
+               ExecutorAddr::fromPtr(this)};
+
+  if (this->TargetTriple.isOSBinFormatMachO())
+    GlobalManglingPrefix = '_';
+
+  addDefaultBootstrapValuesForHostProcess(BootstrapMap, BootstrapSymbols);
+
+#ifdef __APPLE__
+  // FIXME: Don't add an UnwindInfoManager by default -- it's redundant when
+  //        the ORC runtime is loaded. We'll need a way to document this and
+  //        allow clients to choose.
+  if (UnwindInfoManager::TryEnable())
+    UnwindInfoManager::addBootstrapSymbols(this->BootstrapSymbols);
+#endif // __APPLE__
+}
+
+Expected<std::unique_ptr<SelfExecutorProcessControl>>
+SelfExecutorProcessControl::Create(
+    std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D,
+    std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) {
+
+  if (!SSP)
+    SSP = std::make_shared<SymbolStringPool>();
+
+  if (!D)
+    D = std::make_unique<InPlaceTaskDispatcher>();
+
+  auto PageSize = sys::Process::getPageSize();
+  if (!PageSize)
+    return PageSize.takeError();
+
+  Triple TT(sys::getProcessTriple());
+
+  return std::make_unique<SelfExecutorProcessControl>(
+      std::move(SSP), std::move(D), std::move(TT), *PageSize,
+      std::move(MemMgr));
+}
+
+Expected<tpctypes::DylibHandle>
+SelfExecutorProcessControl::loadDylib(const char *DylibPath) {
+  std::string ErrMsg;
+  auto Dylib = sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg);
+  if (!Dylib.isValid())
+    return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
+  return ExecutorAddr::fromPtr(Dylib.getOSSpecificHandle());
+}
+
+void SelfExecutorProcessControl::lookupSymbolsAsync(
+    ArrayRef<LookupRequest> Request,
+    DylibManager::SymbolLookupCompleteFn Complete) {
+  std::vector<tpctypes::LookupResult> R;
+
+  for (auto &Elem : Request) {
+    sys::DynamicLibrary Dylib(Elem.Handle.toPtr<void *>());
+    R.push_back(std::vector<ExecutorSymbolDef>());
+    for (auto &KV : Elem.Symbols) {
+      auto &Sym = KV.first;
+      std::string Tmp((*Sym).data() + !!GlobalManglingPrefix,
+                      (*Sym).size() - !!GlobalManglingPrefix);
+      void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str());
+      if (!Addr && KV.second == SymbolLookupFlags::RequiredSymbol) {
+        // FIXME: Collect all failing symbols before erroring out.
+        SymbolNameVector MissingSymbols;
+        MissingSymbols.push_back(Sym);
+        return Complete(
+            make_error<SymbolsNotFound>(SSP, std::move(MissingSymbols)));
+      }
+      // FIXME: determine accurate JITSymbolFlags.
+      R.back().push_back(
+          {ExecutorAddr::fromPtr(Addr), JITSymbolFlags::Exported});
+    }
+  }
+
+  Complete(std::move(R));
+}
+
+Expected<int32_t>
+SelfExecutorProcessControl::runAsMain(ExecutorAddr MainFnAddr,
+                                      ArrayRef<std::string> Args) {
+  using MainTy = int (*)(int, char *[]);
+  return orc::runAsMain(MainFnAddr.toPtr<MainTy>(), Args);
+}
+
+Expected<int32_t>
+SelfExecutorProcessControl::runAsVoidFunction(ExecutorAddr VoidFnAddr) {
+  using VoidTy = int (*)();
+  return orc::runAsVoidFunction(VoidFnAddr.toPtr<VoidTy>());
+}
+
+Expected<int32_t>
+SelfExecutorProcessControl::runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) {
+  using IntTy = int (*)(int);
+  return orc::runAsIntFunction(IntFnAddr.toPtr<IntTy>(), Arg);
+}
+
+void SelfExecutorProcessControl::callWrapperAsync(ExecutorAddr WrapperFnAddr,
+                                                  IncomingWFRHandler SendResult,
+                                                  ArrayRef<char> ArgBuffer) {
+  using WrapperFnTy =
+      shared::CWrapperFunctionResult (*)(const char *Data, size_t Size);
+  auto *WrapperFn = WrapperFnAddr.toPtr<WrapperFnTy>();
+  SendResult(WrapperFn(ArgBuffer.data(), ArgBuffer.size()));
+}
+
+Error SelfExecutorProcessControl::disconnect() {
+  D->shutdown();
+  return Error::success();
+}
+
+shared::CWrapperFunctionResult
+SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager(
+    void *Ctx, const void *FnTag, const char *Data, size_t Size) {
+
+  LLVM_DEBUG({
+    dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size
+           << " byte payload.\n";
+  });
+
+  std::promise<shared::WrapperFunctionResult> ResultP;
+  auto ResultF = ResultP.get_future();
+  static_cast<SelfExecutorProcessControl *>(Ctx)
+      ->getExecutionSession()
+      .runJITDispatchHandler(
+          [ResultP = std::move(ResultP)](
+              shared::WrapperFunctionResult Result) mutable {
+            ResultP.set_value(std::move(Result));
+          },
+          ExecutorAddr::fromPtr(FnTag), {Data, Size});
+
+  return ResultF.get().release();
+}
+
+} // namespace llvm::orc
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
index 4e5b800100b9..123651fc623e 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp
@@ -47,10 +47,25 @@ const char *MemoryWriteUInt32sWrapperName =
     "__llvm_orc_bootstrap_mem_write_uint32s_wrapper";
 const char *MemoryWriteUInt64sWrapperName =
     "__llvm_orc_bootstrap_mem_write_uint64s_wrapper";
-const char *MemoryWriteBuffersWrapperName =
-    "__llvm_orc_bootstrap_mem_write_buffers_wrapper";
 const char *MemoryWritePointersWrapperName =
     "__llvm_orc_bootstrap_mem_write_pointers_wrapper";
+const char *MemoryWriteBuffersWrapperName =
+    "__llvm_orc_bootstrap_mem_write_buffers_wrapper";
+
+const char *MemoryReadUInt8sWrapperName =
+    "__llvm_orc_bootstrap_mem_read_uint8s_wrapper";
+const char *MemoryReadUInt16sWrapperName =
+    "__llvm_orc_bootstrap_mem_read_uint16s_wrapper";
+const char *MemoryReadUInt32sWrapperName =
+    "__llvm_orc_bootstrap_mem_read_uint32s_wrapper";
+const char *MemoryReadUInt64sWrapperName =
+    "__llvm_orc_bootstrap_mem_read_uint64s_wrapper";
+const char *MemoryReadPointersWrapperName =
+    "__llvm_orc_bootstrap_mem_read_pointers_wrapper";
+const char *MemoryReadBuffersWrapperName =
+    "__llvm_orc_bootstrap_mem_read_buffers_wrapper";
+const char *MemoryReadStringsWrapperName =
+    "__llvm_orc_bootstrap_mem_read_strings_wrapper";
 
 const char *RegisterEHFrameSectionAllocActionName =
     "llvm_orc_registerEHFrameAllocAction";
diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
index 3d816785cb76..87d757805a64 100644
--- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp
@@ -224,7 +224,7 @@ SimpleRemoteEPC::createDefaultMemoryManager(SimpleRemoteEPC &SREPC) {
   return std::make_unique<EPCGenericJITLinkMemoryManager>(SREPC, SAs);
 }
 
-Expected<std::unique_ptr<ExecutorProcessControl::MemoryAccess>>
+Expected<std::unique_ptr<MemoryAccess>>
 SimpleRemoteEPC::createDefaultMemoryAccess(SimpleRemoteEPC &SREPC) {
   EPCGenericMemoryAccess::FuncAddrs FAs;
   if (auto Err = SREPC.getBootstrapSymbols(
@@ -233,7 +233,13 @@ SimpleRemoteEPC::createDefaultMemoryAccess(SimpleRemoteEPC &SREPC) {
            {FAs.WriteUInt32s, rt::MemoryWriteUInt32sWrapperName},
            {FAs.WriteUInt64s, rt::MemoryWriteUInt64sWrapperName},
            {FAs.WriteBuffers, rt::MemoryWriteBuffersWrapperName},
-           {FAs.WritePointers, rt::MemoryWritePointersWrapperName}}))
+           {FAs.WritePointers, rt::MemoryWritePointersWrapperName},
+           {FAs.ReadUInt8s, rt::MemoryReadUInt8sWrapperName},
+           {FAs.ReadUInt16s, rt::MemoryReadUInt16sWrapperName},
+           {FAs.ReadUInt32s, rt::MemoryReadUInt32sWrapperName},
+           {FAs.ReadUInt64s, rt::MemoryReadUInt64sWrapperName},
+           {FAs.ReadBuffers, rt::MemoryReadBuffersWrapperName},
+           {FAs.ReadStrings, rt::MemoryReadStringsWrapperName}}))
     return std::move(Err);
 
   return std::make_unique<EPCGenericMemoryAccess>(SREPC, FAs);
diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
index 74b9eb29bdcc..fee94b96a9e8 100644
--- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp
@@ -60,8 +60,6 @@ void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
                               ThreadSafeModule TSM) {
 
   assert(TSM && "Speculation Layer received Null Module ?");
-  assert(TSM.getContext().getContext() != nullptr &&
-         "Module with null LLVMContext?");
 
   // Instrumentation of runtime calls, lock the Module
   TSM.withModuleDo([this, &R](Module &M) {
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
index c4f201b353d2..06cb717750e3 100644
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp
@@ -33,6 +33,18 @@ writeUIntsWrapper(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
+static llvm::orc::shared::CWrapperFunctionResult
+writePointersWrapper(const char *ArgData, size_t ArgSize) {
+  return WrapperFunction<void(SPSSequence<SPSMemoryAccessPointerWrite>)>::
+      handle(ArgData, ArgSize,
+             [](std::vector<tpctypes::PointerWrite> Ws) {
+               for (auto &W : Ws)
+                 *W.Addr.template toPtr<void **>() =
+                     W.Value.template toPtr<void *>();
+             })
+          .release();
+}
+
 static llvm::orc::shared::CWrapperFunctionResult
 writeBuffersWrapper(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<void(SPSSequence<SPSMemoryAccessBufferWrite>)>::handle(
@@ -45,16 +57,70 @@ writeBuffersWrapper(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
+template <typename ReadT>
 static llvm::orc::shared::CWrapperFunctionResult
-writePointersWrapper(const char *ArgData, size_t ArgSize) {
-  return WrapperFunction<void(SPSSequence<SPSMemoryAccessPointerWrite>)>::
-      handle(ArgData, ArgSize,
-             [](std::vector<tpctypes::PointerWrite> Ws) {
-               for (auto &W : Ws)
-                 *W.Addr.template toPtr<void **>() =
-                     W.Value.template toPtr<void *>();
+readUIntsWrapper(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<ReadT>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(ArgData, ArgSize,
+                                         [](std::vector<ExecutorAddr> Rs) {
+                                           std::vector<ReadT> Result;
+                                           Result.reserve(Rs.size());
+                                           for (auto &R : Rs)
+                                             Result.push_back(
+                                                 *R.toPtr<ReadT *>());
+                                           return Result;
+                                         })
+      .release();
+}
+
+static llvm::orc::shared::CWrapperFunctionResult
+readPointersWrapper(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<SPSExecutorAddr>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(
+             ArgData, ArgSize,
+             [](std::vector<ExecutorAddr> Rs) {
+               std::vector<ExecutorAddr> Result;
+               Result.reserve(Rs.size());
+               for (auto &R : Rs)
+                 Result.push_back(ExecutorAddr::fromPtr(*R.toPtr<void **>()));
+               return Result;
              })
-          .release();
+      .release();
+}
+
+static llvm::orc::shared::CWrapperFunctionResult
+readBuffersWrapper(const char *ArgData, size_t ArgSize) {
+  using SPSSig =
+      SPSSequence<SPSSequence<uint8_t>>(SPSSequence<SPSExecutorAddrRange>);
+  return WrapperFunction<SPSSig>::handle(
+             ArgData, ArgSize,
+             [](std::vector<ExecutorAddrRange> Rs) {
+               std::vector<std::vector<uint8_t>> Result;
+               Result.reserve(Rs.size());
+               for (auto &R : Rs) {
+                 Result.push_back({});
+                 Result.back().resize(R.size());
+                 memcpy(reinterpret_cast<char *>(Result.back().data()),
+                        R.Start.toPtr<char *>(), R.size());
+               }
+               return Result;
+             })
+      .release();
+}
+
+static llvm::orc::shared::CWrapperFunctionResult
+readStringsWrapper(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<SPSString>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(ArgData, ArgSize,
+                                         [](std::vector<ExecutorAddr> Rs) {
+                                           std::vector<std::string> Result;
+                                           Result.reserve(Rs.size());
+                                           for (auto &R : Rs)
+                                             Result.push_back(
+                                                 R.toPtr<char *>());
+                                           return Result;
+                                         })
+      .release();
 }
 
 static llvm::orc::shared::CWrapperFunctionResult
@@ -102,10 +168,24 @@ void addTo(StringMap<ExecutorAddr> &M) {
   M[rt::MemoryWriteUInt64sWrapperName] = ExecutorAddr::fromPtr(
       &writeUIntsWrapper<tpctypes::UInt64Write,
                          shared::SPSMemoryAccessUInt64Write>);
-  M[rt::MemoryWriteBuffersWrapperName] =
-      ExecutorAddr::fromPtr(&writeBuffersWrapper);
   M[rt::MemoryWritePointersWrapperName] =
       ExecutorAddr::fromPtr(&writePointersWrapper);
+  M[rt::MemoryWriteBuffersWrapperName] =
+      ExecutorAddr::fromPtr(&writeBuffersWrapper);
+  M[rt::MemoryReadUInt8sWrapperName] =
+      ExecutorAddr::fromPtr(&readUIntsWrapper<uint8_t>);
+  M[rt::MemoryReadUInt16sWrapperName] =
+      ExecutorAddr::fromPtr(&readUIntsWrapper<uint16_t>);
+  M[rt::MemoryReadUInt32sWrapperName] =
+      ExecutorAddr::fromPtr(&readUIntsWrapper<uint32_t>);
+  M[rt::MemoryReadUInt64sWrapperName] =
+      ExecutorAddr::fromPtr(&readUIntsWrapper<uint64_t>);
+  M[rt::MemoryReadPointersWrapperName] =
+      ExecutorAddr::fromPtr(&readPointersWrapper);
+  M[rt::MemoryReadBuffersWrapperName] =
+      ExecutorAddr::fromPtr(&readBuffersWrapper);
+  M[rt::MemoryReadStringsWrapperName] =
+      ExecutorAddr::fromPtr(&readStringsWrapper);
   M[rt::RunAsMainWrapperName] = ExecutorAddr::fromPtr(&runAsMainWrapper);
   M[rt::RunAsVoidFunctionWrapperName] =
       ExecutorAddr::fromPtr(&runAsVoidFunctionWrapper);
diff --git a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
index 2e128dd23744..19c000e2472a 100644
--- a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
@@ -1,5 +1,4 @@
-//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities
-//h-===//
+//===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -15,49 +14,63 @@
 namespace llvm {
 namespace orc {
 
-ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM,
-                                   GVPredicate ShouldCloneDef,
-                                   GVModifier UpdateClonedDefSource) {
+ThreadSafeModule cloneToContext(const ThreadSafeModule &TSM,
+                                ThreadSafeContext TSCtx,
+                                GVPredicate ShouldCloneDef,
+                                GVModifier UpdateClonedDefSource) {
   assert(TSM && "Can not clone null module");
 
   if (!ShouldCloneDef)
     ShouldCloneDef = [](const GlobalValue &) { return true; };
 
-  return TSM.withModuleDo([&](Module &M) {
-    SmallVector<char, 1> ClonedModuleBuffer;
+  // First copy the source module into a buffer.
+  std::string ModuleName;
+  SmallVector<char, 1> ClonedModuleBuffer;
+  TSM.withModuleDo([&](Module &M) {
+    ModuleName = M.getModuleIdentifier();
+    std::set<GlobalValue *> ClonedDefsInSrc;
+    ValueToValueMapTy VMap;
+    auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
+      if (ShouldCloneDef(*GV)) {
+        ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
+        return true;
+      }
+      return false;
+    });
 
-    {
-      std::set<GlobalValue *> ClonedDefsInSrc;
-      ValueToValueMapTy VMap;
-      auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
-        if (ShouldCloneDef(*GV)) {
-          ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
-          return true;
-        }
-        return false;
-      });
+    if (UpdateClonedDefSource)
+      for (auto *GV : ClonedDefsInSrc)
+        UpdateClonedDefSource(*GV);
 
-      if (UpdateClonedDefSource)
-        for (auto *GV : ClonedDefsInSrc)
-          UpdateClonedDefSource(*GV);
+    BitcodeWriter BCWriter(ClonedModuleBuffer);
+    BCWriter.writeModule(*Tmp);
+    BCWriter.writeSymtab();
+    BCWriter.writeStrtab();
+  });
+
+  MemoryBufferRef ClonedModuleBufferRef(
+      StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
+      "cloned module buffer");
 
-      BitcodeWriter BCWriter(ClonedModuleBuffer);
+  // Then parse the buffer into the new Module.
+  auto M = TSCtx.withContextDo([&](LLVMContext *Ctx) {
+    assert(Ctx && "No LLVMContext provided");
+    auto TmpM = cantFail(parseBitcodeFile(ClonedModuleBufferRef, *Ctx));
+    TmpM->setModuleIdentifier(ModuleName);
+    return TmpM;
+  });
 
-      BCWriter.writeModule(*Tmp);
-      BCWriter.writeSymtab();
-      BCWriter.writeStrtab();
-    }
+  return ThreadSafeModule(std::move(M), std::move(TSCtx));
+}
 
-    MemoryBufferRef ClonedModuleBufferRef(
-        StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
-        "cloned module buffer");
-    ThreadSafeContext NewTSCtx(std::make_unique<LLVMContext>());
+ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM,
+                                   GVPredicate ShouldCloneDef,
+                                   GVModifier UpdateClonedDefSource) {
+  assert(TSM && "Can not clone null module");
 
-    auto ClonedModule = cantFail(
-        parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext()));
-    ClonedModule->setModuleIdentifier(M.getName());
-    return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx));
-  });
+  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
+  return cloneToContext(TSM, std::move(TSCtx), std::move(ShouldCloneDef),
+                        std::move(UpdateClonedDefSource));
 }
 
 } // end namespace orc
diff --git a/llvm/lib/FileCheck/FileCheck.cpp b/llvm/lib/FileCheck/FileCheck.cpp
index bcca499322ae..b79f6ec5b4f0 100644
--- a/llvm/lib/FileCheck/FileCheck.cpp
+++ b/llvm/lib/FileCheck/FileCheck.cpp
@@ -264,7 +264,7 @@ BinaryOperation::getImplicitFormat(const SourceMgr &SM) const {
                                                          : *RightFormat;
 }
 
-Expected<std::string> NumericSubstitution::getResult() const {
+Expected<std::string> NumericSubstitution::getResultRegex() const {
   assert(ExpressionPointer->getAST() != nullptr &&
          "Substituting empty expression");
   Expected<APInt> EvaluatedValue = ExpressionPointer->getAST()->eval();
@@ -274,7 +274,18 @@ Expected<std::string> NumericSubstitution::getResult() const {
   return Format.getMatchingString(*EvaluatedValue);
 }
 
-Expected<std::string> StringSubstitution::getResult() const {
+Expected<std::string> NumericSubstitution::getResultForDiagnostics() const {
+  // The "regex" returned by getResultRegex() is just a numeric value
+  // like '42', '0x2A', '-17', 'DEADBEEF' etc. This is already suitable for use
+  // in diagnostics.
+  Expected<std::string> Literal = getResultRegex();
+  if (!Literal)
+    return Literal;
+
+  return "\"" + std::move(*Literal) + "\"";
+}
+
+Expected<std::string> StringSubstitution::getResultRegex() const {
   // Look up the value and escape it so that we can put it into the regex.
   Expected<StringRef> VarVal = Context->getPatternVarValue(FromStr);
   if (!VarVal)
@@ -282,6 +293,36 @@ Expected<std::string> StringSubstitution::getResult() const {
   return Regex::escape(*VarVal);
 }
 
+Expected<std::string> StringSubstitution::getResultForDiagnostics() const {
+  Expected<StringRef> VarVal = Context->getPatternVarValue(FromStr);
+  if (!VarVal)
+    return VarVal.takeError();
+
+  std::string Result;
+  Result.reserve(VarVal->size() + 2);
+  raw_string_ostream OS(Result);
+
+  OS << '"';
+  // Escape the string if it contains any characters that
+  // make it hard to read, such as non-printable characters (including all
+  // whitespace except space) and double quotes. These are the characters that
+  // are escaped by write_escaped(), except we do not include backslashes,
+  // because they are common in Windows paths and escaping them would make the
+  // output harder to read. However, when we do escape, backslashes are escaped
+  // as well, otherwise the output would be ambiguous.
+  const bool NeedsEscaping =
+      llvm::any_of(*VarVal, [](char C) { return !isPrint(C) || C == '"'; });
+  if (NeedsEscaping)
+    OS.write_escaped(*VarVal);
+  else
+    OS << *VarVal;
+  OS << '"';
+  if (NeedsEscaping)
+    OS << " (escaped value)";
+
+  return Result;
+}
+
 bool Pattern::isValidVarNameStart(char C) { return C == '_' || isAlpha(C); }
 
 Expected<Pattern::VariableProperties>
@@ -1106,7 +1147,7 @@ Pattern::MatchResult Pattern::match(StringRef Buffer,
     Error Errs = Error::success();
     for (const auto &Substitution : Substitutions) {
       // Substitute and check for failure (e.g. use of undefined variable).
-      Expected<std::string> Value = Substitution->getResult();
+      Expected<std::string> Value = Substitution->getResultRegex();
       if (!Value) {
         // Convert to an ErrorDiagnostic to get location information. This is
         // done here rather than printMatch/printNoMatch since now we know which
@@ -1210,7 +1251,8 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
       SmallString<256> Msg;
       raw_svector_ostream OS(Msg);
 
-      Expected<std::string> MatchedValue = Substitution->getResult();
+      Expected<std::string> MatchedValue =
+          Substitution->getResultForDiagnostics();
       // Substitution failures are handled in printNoMatch().
       if (!MatchedValue) {
         consumeError(MatchedValue.takeError());
@@ -1218,8 +1260,8 @@ void Pattern::printSubstitutions(const SourceMgr &SM, StringRef Buffer,
       }
 
       OS << "with \"";
-      OS.write_escaped(Substitution->getFromString()) << "\" equal to \"";
-      OS.write_escaped(*MatchedValue) << "\"";
+      OS.write_escaped(Substitution->getFromString()) << "\" equal to ";
+      OS << *MatchedValue;
 
       // We report only the start of the match/search range to suggest we are
       // reporting the substitutions as set at the start of the match/search.
diff --git a/llvm/lib/FileCheck/FileCheckImpl.h b/llvm/lib/FileCheck/FileCheckImpl.h
index b3cd2af5d57e..5cf5548cdfde 100644
--- a/llvm/lib/FileCheck/FileCheckImpl.h
+++ b/llvm/lib/FileCheck/FileCheckImpl.h
@@ -366,9 +366,15 @@ class Substitution {
   /// \returns the index where the substitution is to be performed in RegExStr.
   size_t getIndex() const { return InsertIdx; }
 
+  /// \returns a regular expression string that matches the result of the
+  /// substitution represented by this class instance or an error if
+  /// substitution failed.
+  virtual Expected<std::string> getResultRegex() const = 0;
+
   /// \returns a string containing the result of the substitution represented
-  /// by this class instance or an error if substitution failed.
-  virtual Expected<std::string> getResult() const = 0;
+  /// by this class instance in a form suitable for diagnostics, or an error if
+  /// substitution failed.
+  virtual Expected<std::string> getResultForDiagnostics() const = 0;
 };
 
 class StringSubstitution : public Substitution {
@@ -379,7 +385,12 @@ class StringSubstitution : public Substitution {
 
   /// \returns the text that the string variable in this substitution matched
   /// when defined, or an error if the variable is undefined.
-  Expected<std::string> getResult() const override;
+  Expected<std::string> getResultRegex() const override;
+
+  /// \returns the text that the string variable in this substitution matched
+  /// when defined, in a form suitable for diagnostics, or an error if the
+  /// variable is undefined.
+  Expected<std::string> getResultForDiagnostics() const override;
 };
 
 class NumericSubstitution : public Substitution {
@@ -397,7 +408,12 @@ class NumericSubstitution : public Substitution {
 
   /// \returns a string containing the result of evaluating the expression in
   /// this substitution, or an error if evaluation failed.
-  Expected<std::string> getResult() const override;
+  Expected<std::string> getResultRegex() const override;
+
+  /// \returns a string containing the result of evaluating the expression in
+  /// this substitution, in a form suitable for diagnostics, or an error if
+  /// evaluation failed.
+  Expected<std::string> getResultForDiagnostics() const override;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Frontend/HLSL/CMakeLists.txt b/llvm/lib/Frontend/HLSL/CMakeLists.txt
index dfebe354807b..534346920ff1 100644
--- a/llvm/lib/Frontend/HLSL/CMakeLists.txt
+++ b/llvm/lib/Frontend/HLSL/CMakeLists.txt
@@ -1,7 +1,9 @@
 add_llvm_component_library(LLVMFrontendHLSL
   CBuffer.cpp
   HLSLResource.cpp
-  HLSLRootSignatureUtils.cpp
+  HLSLRootSignature.cpp
+  RootSignatureMetadata.cpp
+  RootSignatureValidations.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
@@ -11,6 +13,7 @@ add_llvm_component_library(LLVMFrontendHLSL
   intrinsics_gen
 
   LINK_COMPONENTS
+  BinaryFormat
   Core
   Support
   )
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
new file mode 100644
index 000000000000..78c20a6c5c9f
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
@@ -0,0 +1,244 @@
+//===- HLSLRootSignature.cpp - HLSL Root Signature helpers ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains helpers for working with HLSL Root Signatures.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+template <typename T>
+static std::optional<StringRef> getEnumName(const T Value,
+                                            ArrayRef<EnumEntry<T>> Enums) {
+  for (const auto &EnumItem : Enums)
+    if (EnumItem.Value == Value)
+      return EnumItem.Name;
+  return std::nullopt;
+}
+
+template <typename T>
+static raw_ostream &printEnum(raw_ostream &OS, const T Value,
+                              ArrayRef<EnumEntry<T>> Enums) {
+  auto MaybeName = getEnumName(Value, Enums);
+  if (MaybeName)
+    OS << *MaybeName;
+  return OS;
+}
+
+template <typename T>
+static raw_ostream &printFlags(raw_ostream &OS, const T Value,
+                               ArrayRef<EnumEntry<T>> Flags) {
+  bool FlagSet = false;
+  unsigned Remaining = llvm::to_underlying(Value);
+  while (Remaining) {
+    unsigned Bit = 1u << llvm::countr_zero(Remaining);
+    if (Remaining & Bit) {
+      if (FlagSet)
+        OS << " | ";
+
+      auto MaybeFlag = getEnumName(T(Bit), Flags);
+      if (MaybeFlag)
+        OS << *MaybeFlag;
+      else
+        OS << "invalid: " << Bit;
+
+      FlagSet = true;
+    }
+    Remaining &= ~Bit;
+  }
+
+  if (!FlagSet)
+    OS << "None";
+  return OS;
+}
+
+static const EnumEntry<RegisterType> RegisterNames[] = {
+    {"b", RegisterType::BReg},
+    {"t", RegisterType::TReg},
+    {"u", RegisterType::UReg},
+    {"s", RegisterType::SReg},
+};
+
+static raw_ostream &operator<<(raw_ostream &OS, const Register &Reg) {
+  printEnum(OS, Reg.ViewType, ArrayRef(RegisterNames));
+  OS << Reg.Number;
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::ShaderVisibility &Visibility) {
+  printEnum(OS, Visibility, dxbc::getShaderVisibility());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::SamplerFilter &Filter) {
+  printEnum(OS, Filter, dxbc::getSamplerFilters());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::TextureAddressMode &Address) {
+  printEnum(OS, Address, dxbc::getTextureAddressModes());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::ComparisonFunc &CompFunc) {
+  printEnum(OS, CompFunc, dxbc::getComparisonFuncs());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::StaticBorderColor &BorderColor) {
+  printEnum(OS, BorderColor, dxbc::getStaticBorderColors());
+
+  return OS;
+}
+
+static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
+    {"CBV", dxil::ResourceClass::CBuffer},
+    {"SRV", dxil::ResourceClass::SRV},
+    {"UAV", dxil::ResourceClass::UAV},
+    {"Sampler", dxil::ResourceClass::Sampler},
+};
+
+static raw_ostream &operator<<(raw_ostream &OS, const ClauseType &Type) {
+  printEnum(OS, dxil::ResourceClass(llvm::to_underlying(Type)),
+            ArrayRef(ResourceClassNames));
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxbc::RootDescriptorFlags &Flags) {
+  printFlags(OS, Flags, dxbc::getRootDescriptorFlags());
+
+  return OS;
+}
+
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const llvm::dxbc::DescriptorRangeFlags &Flags) {
+  printFlags(OS, Flags, dxbc::getDescriptorRangeFlags());
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const dxbc::RootFlags &Flags) {
+  OS << "RootFlags(";
+  printFlags(OS, Flags, dxbc::getRootFlags());
+  OS << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const RootConstants &Constants) {
+  OS << "RootConstants(num32BitConstants = " << Constants.Num32BitConstants
+     << ", " << Constants.Reg << ", space = " << Constants.Space
+     << ", visibility = " << Constants.Visibility << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table) {
+  OS << "DescriptorTable(numClauses = " << Table.NumClauses
+     << ", visibility = " << Table.Visibility << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause) {
+  OS << Clause.Type << "(" << Clause.Reg << ", numDescriptors = ";
+  if (Clause.NumDescriptors == NumDescriptorsUnbounded)
+    OS << "unbounded";
+  else
+    OS << Clause.NumDescriptors;
+  OS << ", space = " << Clause.Space << ", offset = ";
+  if (Clause.Offset == DescriptorTableOffsetAppend)
+    OS << "DescriptorTableOffsetAppend";
+  else
+    OS << Clause.Offset;
+  OS << ", flags = " << Clause.Flags << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const RootDescriptor &Descriptor) {
+  ClauseType Type = ClauseType(llvm::to_underlying(Descriptor.Type));
+  OS << "Root" << Type << "(" << Descriptor.Reg
+     << ", space = " << Descriptor.Space
+     << ", visibility = " << Descriptor.Visibility
+     << ", flags = " << Descriptor.Flags << ")";
+
+  return OS;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const StaticSampler &Sampler) {
+  OS << "StaticSampler(" << Sampler.Reg << ", filter = " << Sampler.Filter
+     << ", addressU = " << Sampler.AddressU
+     << ", addressV = " << Sampler.AddressV
+     << ", addressW = " << Sampler.AddressW
+     << ", mipLODBias = " << Sampler.MipLODBias
+     << ", maxAnisotropy = " << Sampler.MaxAnisotropy
+     << ", comparisonFunc = " << Sampler.CompFunc
+     << ", borderColor = " << Sampler.BorderColor
+     << ", minLOD = " << Sampler.MinLOD << ", maxLOD = " << Sampler.MaxLOD
+     << ", space = " << Sampler.Space << ", visibility = " << Sampler.Visibility
+     << ")";
+  return OS;
+}
+
+namespace {
+
+// We use the OverloadVisit with std::visit to ensure the compiler catches if a
+// new RootElement variant type is added but it's operator<< isn't handled.
+template <class... Ts> struct OverloadedVisit : Ts... {
+  using Ts::operator()...;
+};
+template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
+
+} // namespace
+
+raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element) {
+  const auto Visitor = OverloadedVisit{
+      [&OS](const dxbc::RootFlags &Flags) { OS << Flags; },
+      [&OS](const RootConstants &Constants) { OS << Constants; },
+      [&OS](const RootDescriptor &Descriptor) { OS << Descriptor; },
+      [&OS](const DescriptorTableClause &Clause) { OS << Clause; },
+      [&OS](const DescriptorTable &Table) { OS << Table; },
+      [&OS](const StaticSampler &Sampler) { OS << Sampler; },
+  };
+  std::visit(Visitor, Element);
+  return OS;
+}
+
+void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements) {
+  OS << " RootElements{";
+  bool First = true;
+  for (const RootElement &Element : Elements) {
+    if (!First)
+      OS << ",";
+    OS << " " << Element;
+    First = false;
+  }
+  OS << "}";
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
deleted file mode 100644
index 22a761ba3b13..000000000000
--- a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
+++ /dev/null
@@ -1,577 +0,0 @@
-//===- HLSLRootSignatureUtils.cpp - HLSL Root Signature helpers -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file This file contains helpers for working with HLSL Root Signatures.
-///
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/bit.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Metadata.h"
-#include "llvm/Support/ScopedPrinter.h"
-
-namespace llvm {
-namespace hlsl {
-namespace rootsig {
-
-template <typename T>
-static std::optional<StringRef> getEnumName(const T Value,
-                                            ArrayRef<EnumEntry<T>> Enums) {
-  for (const auto &EnumItem : Enums)
-    if (EnumItem.Value == Value)
-      return EnumItem.Name;
-  return std::nullopt;
-}
-
-template <typename T>
-static raw_ostream &printEnum(raw_ostream &OS, const T Value,
-                              ArrayRef<EnumEntry<T>> Enums) {
-  auto MaybeName = getEnumName(Value, Enums);
-  if (MaybeName)
-    OS << *MaybeName;
-  return OS;
-}
-
-template <typename T>
-static raw_ostream &printFlags(raw_ostream &OS, const T Value,
-                               ArrayRef<EnumEntry<T>> Flags) {
-  bool FlagSet = false;
-  unsigned Remaining = llvm::to_underlying(Value);
-  while (Remaining) {
-    unsigned Bit = 1u << llvm::countr_zero(Remaining);
-    if (Remaining & Bit) {
-      if (FlagSet)
-        OS << " | ";
-
-      auto MaybeFlag = getEnumName(T(Bit), Flags);
-      if (MaybeFlag)
-        OS << *MaybeFlag;
-      else
-        OS << "invalid: " << Bit;
-
-      FlagSet = true;
-    }
-    Remaining &= ~Bit;
-  }
-
-  if (!FlagSet)
-    OS << "None";
-  return OS;
-}
-
-static const EnumEntry<RegisterType> RegisterNames[] = {
-    {"b", RegisterType::BReg},
-    {"t", RegisterType::TReg},
-    {"u", RegisterType::UReg},
-    {"s", RegisterType::SReg},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const Register &Reg) {
-  printEnum(OS, Reg.ViewType, ArrayRef(RegisterNames));
-  OS << Reg.Number;
-
-  return OS;
-}
-
-static const EnumEntry<ShaderVisibility> VisibilityNames[] = {
-    {"All", ShaderVisibility::All},
-    {"Vertex", ShaderVisibility::Vertex},
-    {"Hull", ShaderVisibility::Hull},
-    {"Domain", ShaderVisibility::Domain},
-    {"Geometry", ShaderVisibility::Geometry},
-    {"Pixel", ShaderVisibility::Pixel},
-    {"Amplification", ShaderVisibility::Amplification},
-    {"Mesh", ShaderVisibility::Mesh},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const ShaderVisibility &Visibility) {
-  printEnum(OS, Visibility, ArrayRef(VisibilityNames));
-
-  return OS;
-}
-
-static const EnumEntry<SamplerFilter> SamplerFilterNames[] = {
-    {"MinMagMipPoint", SamplerFilter::MinMagMipPoint},
-    {"MinMagPointMipLinear", SamplerFilter::MinMagPointMipLinear},
-    {"MinPointMagLinearMipPoint", SamplerFilter::MinPointMagLinearMipPoint},
-    {"MinPointMagMipLinear", SamplerFilter::MinPointMagMipLinear},
-    {"MinLinearMagMipPoint", SamplerFilter::MinLinearMagMipPoint},
-    {"MinLinearMagPointMipLinear", SamplerFilter::MinLinearMagPointMipLinear},
-    {"MinMagLinearMipPoint", SamplerFilter::MinMagLinearMipPoint},
-    {"MinMagMipLinear", SamplerFilter::MinMagMipLinear},
-    {"Anisotropic", SamplerFilter::Anisotropic},
-    {"ComparisonMinMagMipPoint", SamplerFilter::ComparisonMinMagMipPoint},
-    {"ComparisonMinMagPointMipLinear",
-     SamplerFilter::ComparisonMinMagPointMipLinear},
-    {"ComparisonMinPointMagLinearMipPoint",
-     SamplerFilter::ComparisonMinPointMagLinearMipPoint},
-    {"ComparisonMinPointMagMipLinear",
-     SamplerFilter::ComparisonMinPointMagMipLinear},
-    {"ComparisonMinLinearMagMipPoint",
-     SamplerFilter::ComparisonMinLinearMagMipPoint},
-    {"ComparisonMinLinearMagPointMipLinear",
-     SamplerFilter::ComparisonMinLinearMagPointMipLinear},
-    {"ComparisonMinMagLinearMipPoint",
-     SamplerFilter::ComparisonMinMagLinearMipPoint},
-    {"ComparisonMinMagMipLinear", SamplerFilter::ComparisonMinMagMipLinear},
-    {"ComparisonAnisotropic", SamplerFilter::ComparisonAnisotropic},
-    {"MinimumMinMagMipPoint", SamplerFilter::MinimumMinMagMipPoint},
-    {"MinimumMinMagPointMipLinear", SamplerFilter::MinimumMinMagPointMipLinear},
-    {"MinimumMinPointMagLinearMipPoint",
-     SamplerFilter::MinimumMinPointMagLinearMipPoint},
-    {"MinimumMinPointMagMipLinear", SamplerFilter::MinimumMinPointMagMipLinear},
-    {"MinimumMinLinearMagMipPoint", SamplerFilter::MinimumMinLinearMagMipPoint},
-    {"MinimumMinLinearMagPointMipLinear",
-     SamplerFilter::MinimumMinLinearMagPointMipLinear},
-    {"MinimumMinMagLinearMipPoint", SamplerFilter::MinimumMinMagLinearMipPoint},
-    {"MinimumMinMagMipLinear", SamplerFilter::MinimumMinMagMipLinear},
-    {"MinimumAnisotropic", SamplerFilter::MinimumAnisotropic},
-    {"MaximumMinMagMipPoint", SamplerFilter::MaximumMinMagMipPoint},
-    {"MaximumMinMagPointMipLinear", SamplerFilter::MaximumMinMagPointMipLinear},
-    {"MaximumMinPointMagLinearMipPoint",
-     SamplerFilter::MaximumMinPointMagLinearMipPoint},
-    {"MaximumMinPointMagMipLinear", SamplerFilter::MaximumMinPointMagMipLinear},
-    {"MaximumMinLinearMagMipPoint", SamplerFilter::MaximumMinLinearMagMipPoint},
-    {"MaximumMinLinearMagPointMipLinear",
-     SamplerFilter::MaximumMinLinearMagPointMipLinear},
-    {"MaximumMinMagLinearMipPoint", SamplerFilter::MaximumMinMagLinearMipPoint},
-    {"MaximumMinMagMipLinear", SamplerFilter::MaximumMinMagMipLinear},
-    {"MaximumAnisotropic", SamplerFilter::MaximumAnisotropic},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const SamplerFilter &Filter) {
-  printEnum(OS, Filter, ArrayRef(SamplerFilterNames));
-
-  return OS;
-}
-
-static const EnumEntry<TextureAddressMode> TextureAddressModeNames[] = {
-    {"Wrap", TextureAddressMode::Wrap},
-    {"Mirror", TextureAddressMode::Mirror},
-    {"Clamp", TextureAddressMode::Clamp},
-    {"Border", TextureAddressMode::Border},
-    {"MirrorOnce", TextureAddressMode::MirrorOnce},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const TextureAddressMode &Address) {
-  printEnum(OS, Address, ArrayRef(TextureAddressModeNames));
-
-  return OS;
-}
-
-static const EnumEntry<ComparisonFunc> ComparisonFuncNames[] = {
-    {"Never", ComparisonFunc::Never},
-    {"Less", ComparisonFunc::Less},
-    {"Equal", ComparisonFunc::Equal},
-    {"LessEqual", ComparisonFunc::LessEqual},
-    {"Greater", ComparisonFunc::Greater},
-    {"NotEqual", ComparisonFunc::NotEqual},
-    {"GreaterEqual", ComparisonFunc::GreaterEqual},
-    {"Always", ComparisonFunc::Always},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const ComparisonFunc &CompFunc) {
-  printEnum(OS, CompFunc, ArrayRef(ComparisonFuncNames));
-
-  return OS;
-}
-
-static const EnumEntry<StaticBorderColor> StaticBorderColorNames[] = {
-    {"TransparentBlack", StaticBorderColor::TransparentBlack},
-    {"OpaqueBlack", StaticBorderColor::OpaqueBlack},
-    {"OpaqueWhite", StaticBorderColor::OpaqueWhite},
-    {"OpaqueBlackUint", StaticBorderColor::OpaqueBlackUint},
-    {"OpaqueWhiteUint", StaticBorderColor::OpaqueWhiteUint},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const StaticBorderColor &BorderColor) {
-  printEnum(OS, BorderColor, ArrayRef(StaticBorderColorNames));
-
-  return OS;
-}
-
-static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
-    {"CBV", dxil::ResourceClass::CBuffer},
-    {"SRV", dxil::ResourceClass::SRV},
-    {"UAV", dxil::ResourceClass::UAV},
-    {"Sampler", dxil::ResourceClass::Sampler},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const ClauseType &Type) {
-  printEnum(OS, dxil::ResourceClass(llvm::to_underlying(Type)),
-            ArrayRef(ResourceClassNames));
-
-  return OS;
-}
-
-static const EnumEntry<RootDescriptorFlags> RootDescriptorFlagNames[] = {
-    {"DataVolatile", RootDescriptorFlags::DataVolatile},
-    {"DataStaticWhileSetAtExecute",
-     RootDescriptorFlags::DataStaticWhileSetAtExecute},
-    {"DataStatic", RootDescriptorFlags::DataStatic},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const RootDescriptorFlags &Flags) {
-  printFlags(OS, Flags, ArrayRef(RootDescriptorFlagNames));
-
-  return OS;
-}
-
-static const EnumEntry<DescriptorRangeFlags> DescriptorRangeFlagNames[] = {
-    {"DescriptorsVolatile", DescriptorRangeFlags::DescriptorsVolatile},
-    {"DataVolatile", DescriptorRangeFlags::DataVolatile},
-    {"DataStaticWhileSetAtExecute",
-     DescriptorRangeFlags::DataStaticWhileSetAtExecute},
-    {"DataStatic", DescriptorRangeFlags::DataStatic},
-    {"DescriptorsStaticKeepingBufferBoundsChecks",
-     DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS,
-                               const DescriptorRangeFlags &Flags) {
-  printFlags(OS, Flags, ArrayRef(DescriptorRangeFlagNames));
-
-  return OS;
-}
-
-static const EnumEntry<RootFlags> RootFlagNames[] = {
-    {"AllowInputAssemblerInputLayout",
-     RootFlags::AllowInputAssemblerInputLayout},
-    {"DenyVertexShaderRootAccess", RootFlags::DenyVertexShaderRootAccess},
-    {"DenyHullShaderRootAccess", RootFlags::DenyHullShaderRootAccess},
-    {"DenyDomainShaderRootAccess", RootFlags::DenyDomainShaderRootAccess},
-    {"DenyGeometryShaderRootAccess", RootFlags::DenyGeometryShaderRootAccess},
-    {"DenyPixelShaderRootAccess", RootFlags::DenyPixelShaderRootAccess},
-    {"AllowStreamOutput", RootFlags::AllowStreamOutput},
-    {"LocalRootSignature", RootFlags::LocalRootSignature},
-    {"DenyAmplificationShaderRootAccess",
-     RootFlags::DenyAmplificationShaderRootAccess},
-    {"DenyMeshShaderRootAccess", RootFlags::DenyMeshShaderRootAccess},
-    {"CBVSRVUAVHeapDirectlyIndexed", RootFlags::CBVSRVUAVHeapDirectlyIndexed},
-    {"SamplerHeapDirectlyIndexed", RootFlags::SamplerHeapDirectlyIndexed},
-};
-
-raw_ostream &operator<<(raw_ostream &OS, const RootFlags &Flags) {
-  OS << "RootFlags(";
-  printFlags(OS, Flags, ArrayRef(RootFlagNames));
-  OS << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const RootConstants &Constants) {
-  OS << "RootConstants(num32BitConstants = " << Constants.Num32BitConstants
-     << ", " << Constants.Reg << ", space = " << Constants.Space
-     << ", visibility = " << Constants.Visibility << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTable &Table) {
-  OS << "DescriptorTable(numClauses = " << Table.NumClauses
-     << ", visibility = " << Table.Visibility << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause) {
-  OS << Clause.Type << "(" << Clause.Reg << ", numDescriptors = ";
-  if (Clause.NumDescriptors == NumDescriptorsUnbounded)
-    OS << "unbounded";
-  else
-    OS << Clause.NumDescriptors;
-  OS << ", space = " << Clause.Space << ", offset = ";
-  if (Clause.Offset == DescriptorTableOffsetAppend)
-    OS << "DescriptorTableOffsetAppend";
-  else
-    OS << Clause.Offset;
-  OS << ", flags = " << Clause.Flags << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const RootDescriptor &Descriptor) {
-  ClauseType Type = ClauseType(llvm::to_underlying(Descriptor.Type));
-  OS << "Root" << Type << "(" << Descriptor.Reg
-     << ", space = " << Descriptor.Space
-     << ", visibility = " << Descriptor.Visibility
-     << ", flags = " << Descriptor.Flags << ")";
-
-  return OS;
-}
-
-raw_ostream &operator<<(raw_ostream &OS, const StaticSampler &Sampler) {
-  OS << "StaticSampler(" << Sampler.Reg << ", filter = " << Sampler.Filter
-     << ", addressU = " << Sampler.AddressU
-     << ", addressV = " << Sampler.AddressV
-     << ", addressW = " << Sampler.AddressW
-     << ", mipLODBias = " << Sampler.MipLODBias
-     << ", maxAnisotropy = " << Sampler.MaxAnisotropy
-     << ", comparisonFunc = " << Sampler.CompFunc
-     << ", borderColor = " << Sampler.BorderColor
-     << ", minLOD = " << Sampler.MinLOD << ", maxLOD = " << Sampler.MaxLOD
-     << ", space = " << Sampler.Space << ", visibility = " << Sampler.Visibility
-     << ")";
-  return OS;
-}
-
-namespace {
-
-// We use the OverloadVisit with std::visit to ensure the compiler catches if a
-// new RootElement variant type is added but it's operator<< or metadata
-// generation isn't handled.
-template <class... Ts> struct OverloadedVisit : Ts... {
-  using Ts::operator()...;
-};
-template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
-
-} // namespace
-
-raw_ostream &operator<<(raw_ostream &OS, const RootElement &Element) {
-  const auto Visitor = OverloadedVisit{
-      [&OS](const RootFlags &Flags) { OS << Flags; },
-      [&OS](const RootConstants &Constants) { OS << Constants; },
-      [&OS](const RootDescriptor &Descriptor) { OS << Descriptor; },
-      [&OS](const DescriptorTableClause &Clause) { OS << Clause; },
-      [&OS](const DescriptorTable &Table) { OS << Table; },
-      [&OS](const StaticSampler &Sampler) { OS << Sampler; },
-  };
-  std::visit(Visitor, Element);
-  return OS;
-}
-
-void dumpRootElements(raw_ostream &OS, ArrayRef<RootElement> Elements) {
-  OS << " RootElements{";
-  bool First = true;
-  for (const RootElement &Element : Elements) {
-    if (!First)
-      OS << ",";
-    OS << " " << Element;
-    First = false;
-  }
-  OS << "}";
-}
-
-MDNode *MetadataBuilder::BuildRootSignature() {
-  const auto Visitor = OverloadedVisit{
-      [this](const RootFlags &Flags) -> MDNode * {
-        return BuildRootFlags(Flags);
-      },
-      [this](const RootConstants &Constants) -> MDNode * {
-        return BuildRootConstants(Constants);
-      },
-      [this](const RootDescriptor &Descriptor) -> MDNode * {
-        return BuildRootDescriptor(Descriptor);
-      },
-      [this](const DescriptorTableClause &Clause) -> MDNode * {
-        return BuildDescriptorTableClause(Clause);
-      },
-      [this](const DescriptorTable &Table) -> MDNode * {
-        return BuildDescriptorTable(Table);
-      },
-      [this](const StaticSampler &Sampler) -> MDNode * {
-        return BuildStaticSampler(Sampler);
-      },
-  };
-
-  for (const RootElement &Element : Elements) {
-    MDNode *ElementMD = std::visit(Visitor, Element);
-    assert(ElementMD != nullptr &&
-           "Root Element must be initialized and validated");
-    GeneratedMetadata.push_back(ElementMD);
-  }
-
-  return MDNode::get(Ctx, GeneratedMetadata);
-}
-
-MDNode *MetadataBuilder::BuildRootFlags(const RootFlags &Flags) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "RootFlags"),
-      ConstantAsMetadata::get(Builder.getInt32(llvm::to_underlying(Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "RootConstants"),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Constants.Visibility))),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Space)),
-      ConstantAsMetadata::get(Builder.getInt32(Constants.Num32BitConstants)),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
-  IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> TypeName =
-      getEnumName(dxil::ResourceClass(llvm::to_underlying(Descriptor.Type)),
-                  ArrayRef(ResourceClassNames));
-  assert(TypeName && "Provided an invalid Resource Class");
-  llvm::SmallString<7> Name({"Root", *TypeName});
-  Metadata *Operands[] = {
-      MDString::get(Ctx, Name),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Descriptor.Visibility))),
-      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Space)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Descriptor.Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
-  IRBuilder<> Builder(Ctx);
-  SmallVector<Metadata *> TableOperands;
-  // Set the mandatory arguments
-  TableOperands.push_back(MDString::get(Ctx, "DescriptorTable"));
-  TableOperands.push_back(ConstantAsMetadata::get(
-      Builder.getInt32(llvm::to_underlying(Table.Visibility))));
-
-  // Remaining operands are references to the table's clauses. The in-memory
-  // representation of the Root Elements created from parsing will ensure that
-  // the previous N elements are the clauses for this table.
-  assert(Table.NumClauses <= GeneratedMetadata.size() &&
-         "Table expected all owned clauses to be generated already");
-  // So, add a refence to each clause to our operands
-  TableOperands.append(GeneratedMetadata.end() - Table.NumClauses,
-                       GeneratedMetadata.end());
-  // Then, remove those clauses from the general list of Root Elements
-  GeneratedMetadata.pop_back_n(Table.NumClauses);
-
-  return MDNode::get(Ctx, TableOperands);
-}
-
-MDNode *MetadataBuilder::BuildDescriptorTableClause(
-    const DescriptorTableClause &Clause) {
-  IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> Name =
-      getEnumName(dxil::ResourceClass(llvm::to_underlying(Clause.Type)),
-                  ArrayRef(ResourceClassNames));
-  assert(Name && "Provided an invalid Resource Class");
-  Metadata *Operands[] = {
-      MDString::get(Ctx, *Name),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.NumDescriptors)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Space)),
-      ConstantAsMetadata::get(Builder.getInt32(Clause.Offset)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Clause.Flags))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
-  IRBuilder<> Builder(Ctx);
-  Metadata *Operands[] = {
-      MDString::get(Ctx, "StaticSampler"),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.Filter))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressU))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressV))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.AddressW))),
-      ConstantAsMetadata::get(llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx),
-                                                    Sampler.MipLODBias)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.MaxAnisotropy)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.CompFunc))),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.BorderColor))),
-      ConstantAsMetadata::get(
-          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MinLOD)),
-      ConstantAsMetadata::get(
-          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MaxLOD)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.Reg.Number)),
-      ConstantAsMetadata::get(Builder.getInt32(Sampler.Space)),
-      ConstantAsMetadata::get(
-          Builder.getInt32(llvm::to_underlying(Sampler.Visibility))),
-  };
-  return MDNode::get(Ctx, Operands);
-}
-
-std::optional<const RangeInfo *>
-ResourceRange::getOverlapping(const RangeInfo &Info) const {
-  MapT::const_iterator Interval = Intervals.find(Info.LowerBound);
-  if (!Interval.valid() || Info.UpperBound < Interval.start())
-    return std::nullopt;
-  return Interval.value();
-}
-
-const RangeInfo *ResourceRange::lookup(uint32_t X) const {
-  return Intervals.lookup(X, nullptr);
-}
-
-void ResourceRange::clear() { return Intervals.clear(); }
-
-std::optional<const RangeInfo *> ResourceRange::insert(const RangeInfo &Info) {
-  uint32_t LowerBound = Info.LowerBound;
-  uint32_t UpperBound = Info.UpperBound;
-
-  std::optional<const RangeInfo *> Res = std::nullopt;
-  MapT::iterator Interval = Intervals.begin();
-
-  while (true) {
-    if (UpperBound < LowerBound)
-      break;
-
-    Interval.advanceTo(LowerBound);
-    if (!Interval.valid()) // No interval found
-      break;
-
-    // Let Interval = [x;y] and [LowerBound;UpperBound] = [a;b] and note that
-    // a <= y implicitly from Intervals.find(LowerBound)
-    if (UpperBound < Interval.start())
-      break; // found interval does not overlap with inserted one
-
-    if (!Res.has_value()) // Update to be the first found intersection
-      Res = Interval.value();
-
-    if (Interval.start() <= LowerBound && UpperBound <= Interval.stop()) {
-      // x <= a <= b <= y implies that [a;b] is covered by [x;y]
-      //  -> so we don't need to insert this, report an overlap
-      return Res;
-    } else if (LowerBound <= Interval.start() &&
-               Interval.stop() <= UpperBound) {
-      // a <= x <= y <= b implies that [x;y] is covered by [a;b]
-      //  -> so remove the existing interval that we will cover with the
-      //  overwrite
-      Interval.erase();
-    } else if (LowerBound < Interval.start() && UpperBound <= Interval.stop()) {
-      // a < x <= b <= y implies that [a; x] is not covered but [x;b] is
-      //  -> so set b = x - 1 such that [a;x-1] is now the interval to insert
-      UpperBound = Interval.start() - 1;
-    } else if (Interval.start() <= LowerBound && Interval.stop() < UpperBound) {
-      // a < x <= b <= y implies that [y; b] is not covered but [a;y] is
-      //  -> so set a = y + 1 such that [y+1;b] is now the interval to insert
-      LowerBound = Interval.stop() + 1;
-    }
-  }
-
-  assert(LowerBound <= UpperBound && "Attempting to insert an empty interval");
-  Intervals.insert(LowerBound, UpperBound, &Info);
-  return Res;
-}
-
-} // namespace rootsig
-} // namespace hlsl
-} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
new file mode 100644
index 000000000000..f7669f09dcec
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
@@ -0,0 +1,194 @@
+//===- RootSignatureMetadata.h - HLSL Root Signature helpers --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file implements a library for working with HLSL Root Signatures
+/// and their metadata representation.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/Support/ScopedPrinter.h"
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
+    {"CBV", dxil::ResourceClass::CBuffer},
+    {"SRV", dxil::ResourceClass::SRV},
+    {"UAV", dxil::ResourceClass::UAV},
+    {"Sampler", dxil::ResourceClass::Sampler},
+};
+
+static std::optional<StringRef> getResourceName(dxil::ResourceClass Class) {
+  for (const auto &ClassEnum : ResourceClassNames)
+    if (ClassEnum.Value == Class)
+      return ClassEnum.Name;
+  return std::nullopt;
+}
+
+namespace {
+
+// We use the OverloadVisit with std::visit to ensure the compiler catches if a
+// new RootElement variant type is added but it's metadata generation isn't
+// handled.
+template <class... Ts> struct OverloadedVisit : Ts... {
+  using Ts::operator()...;
+};
+template <class... Ts> OverloadedVisit(Ts...) -> OverloadedVisit<Ts...>;
+
+} // namespace
+
+MDNode *MetadataBuilder::BuildRootSignature() {
+  const auto Visitor = OverloadedVisit{
+      [this](const dxbc::RootFlags &Flags) -> MDNode * {
+        return BuildRootFlags(Flags);
+      },
+      [this](const RootConstants &Constants) -> MDNode * {
+        return BuildRootConstants(Constants);
+      },
+      [this](const RootDescriptor &Descriptor) -> MDNode * {
+        return BuildRootDescriptor(Descriptor);
+      },
+      [this](const DescriptorTableClause &Clause) -> MDNode * {
+        return BuildDescriptorTableClause(Clause);
+      },
+      [this](const DescriptorTable &Table) -> MDNode * {
+        return BuildDescriptorTable(Table);
+      },
+      [this](const StaticSampler &Sampler) -> MDNode * {
+        return BuildStaticSampler(Sampler);
+      },
+  };
+
+  for (const RootElement &Element : Elements) {
+    MDNode *ElementMD = std::visit(Visitor, Element);
+    assert(ElementMD != nullptr &&
+           "Root Element must be initialized and validated");
+    GeneratedMetadata.push_back(ElementMD);
+  }
+
+  return MDNode::get(Ctx, GeneratedMetadata);
+}
+
+MDNode *MetadataBuilder::BuildRootFlags(const dxbc::RootFlags &Flags) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "RootFlags"),
+      ConstantAsMetadata::get(Builder.getInt32(llvm::to_underlying(Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "RootConstants"),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Constants.Visibility))),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Space)),
+      ConstantAsMetadata::get(Builder.getInt32(Constants.Num32BitConstants)),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
+  IRBuilder<> Builder(Ctx);
+  std::optional<StringRef> ResName = getResourceName(
+      dxil::ResourceClass(llvm::to_underlying(Descriptor.Type)));
+  assert(ResName && "Provided an invalid Resource Class");
+  llvm::SmallString<7> Name({"Root", *ResName});
+  Metadata *Operands[] = {
+      MDString::get(Ctx, Name),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Descriptor.Visibility))),
+      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Descriptor.Space)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Descriptor.Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
+  IRBuilder<> Builder(Ctx);
+  SmallVector<Metadata *> TableOperands;
+  // Set the mandatory arguments
+  TableOperands.push_back(MDString::get(Ctx, "DescriptorTable"));
+  TableOperands.push_back(ConstantAsMetadata::get(
+      Builder.getInt32(llvm::to_underlying(Table.Visibility))));
+
+  // Remaining operands are references to the table's clauses. The in-memory
+  // representation of the Root Elements created from parsing will ensure that
+  // the previous N elements are the clauses for this table.
+  assert(Table.NumClauses <= GeneratedMetadata.size() &&
+         "Table expected all owned clauses to be generated already");
+  // So, add a refence to each clause to our operands
+  TableOperands.append(GeneratedMetadata.end() - Table.NumClauses,
+                       GeneratedMetadata.end());
+  // Then, remove those clauses from the general list of Root Elements
+  GeneratedMetadata.pop_back_n(Table.NumClauses);
+
+  return MDNode::get(Ctx, TableOperands);
+}
+
+MDNode *MetadataBuilder::BuildDescriptorTableClause(
+    const DescriptorTableClause &Clause) {
+  IRBuilder<> Builder(Ctx);
+  std::optional<StringRef> ResName =
+      getResourceName(dxil::ResourceClass(llvm::to_underlying(Clause.Type)));
+  assert(ResName && "Provided an invalid Resource Class");
+  Metadata *Operands[] = {
+      MDString::get(Ctx, *ResName),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.NumDescriptors)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Space)),
+      ConstantAsMetadata::get(Builder.getInt32(Clause.Offset)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Clause.Flags))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+MDNode *MetadataBuilder::BuildStaticSampler(const StaticSampler &Sampler) {
+  IRBuilder<> Builder(Ctx);
+  Metadata *Operands[] = {
+      MDString::get(Ctx, "StaticSampler"),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Filter))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressU))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressV))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.AddressW))),
+      ConstantAsMetadata::get(llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx),
+                                                    Sampler.MipLODBias)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.MaxAnisotropy)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.CompFunc))),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.BorderColor))),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MinLOD)),
+      ConstantAsMetadata::get(
+          llvm::ConstantFP::get(llvm::Type::getFloatTy(Ctx), Sampler.MaxLOD)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Reg.Number)),
+      ConstantAsMetadata::get(Builder.getInt32(Sampler.Space)),
+      ConstantAsMetadata::get(
+          Builder.getInt32(llvm::to_underlying(Sampler.Visibility))),
+  };
+  return MDNode::get(Ctx, Operands);
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
new file mode 100644
index 000000000000..9825946d5969
--- /dev/null
+++ b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
@@ -0,0 +1,84 @@
+//===- HLSLRootSignatureValidations.cpp - HLSL Root Signature helpers -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains helpers for working with HLSL Root Signatures.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
+
+namespace llvm {
+namespace hlsl {
+namespace rootsig {
+
+std::optional<const RangeInfo *>
+ResourceRange::getOverlapping(const RangeInfo &Info) const {
+  MapT::const_iterator Interval = Intervals.find(Info.LowerBound);
+  if (!Interval.valid() || Info.UpperBound < Interval.start())
+    return std::nullopt;
+  return Interval.value();
+}
+
+const RangeInfo *ResourceRange::lookup(uint32_t X) const {
+  return Intervals.lookup(X, nullptr);
+}
+
+void ResourceRange::clear() { return Intervals.clear(); }
+
+std::optional<const RangeInfo *> ResourceRange::insert(const RangeInfo &Info) {
+  uint32_t LowerBound = Info.LowerBound;
+  uint32_t UpperBound = Info.UpperBound;
+
+  std::optional<const RangeInfo *> Res = std::nullopt;
+  MapT::iterator Interval = Intervals.begin();
+
+  while (true) {
+    if (UpperBound < LowerBound)
+      break;
+
+    Interval.advanceTo(LowerBound);
+    if (!Interval.valid()) // No interval found
+      break;
+
+    // Let Interval = [x;y] and [LowerBound;UpperBound] = [a;b] and note that
+    // a <= y implicitly from Intervals.find(LowerBound)
+    if (UpperBound < Interval.start())
+      break; // found interval does not overlap with inserted one
+
+    if (!Res.has_value()) // Update to be the first found intersection
+      Res = Interval.value();
+
+    if (Interval.start() <= LowerBound && UpperBound <= Interval.stop()) {
+      // x <= a <= b <= y implies that [a;b] is covered by [x;y]
+      //  -> so we don't need to insert this, report an overlap
+      return Res;
+    } else if (LowerBound <= Interval.start() &&
+               Interval.stop() <= UpperBound) {
+      // a <= x <= y <= b implies that [x;y] is covered by [a;b]
+      //  -> so remove the existing interval that we will cover with the
+      //  overwrite
+      Interval.erase();
+    } else if (LowerBound < Interval.start() && UpperBound <= Interval.stop()) {
+      // a < x <= b <= y implies that [a; x] is not covered but [x;b] is
+      //  -> so set b = x - 1 such that [a;x-1] is now the interval to insert
+      UpperBound = Interval.start() - 1;
+    } else if (Interval.start() <= LowerBound && Interval.stop() < UpperBound) {
+      // a < x <= b <= y implies that [y; b] is not covered but [a;y] is
+      //  -> so set a = y + 1 such that [y+1;b] is now the interval to insert
+      LowerBound = Interval.stop() + 1;
+    }
+  }
+
+  assert(LowerBound <= UpperBound && "Attempting to insert an empty interval");
+  Intervals.insert(LowerBound, UpperBound, &Info);
+  return Res;
+}
+
+} // namespace rootsig
+} // namespace hlsl
+} // namespace llvm
diff --git a/llvm/lib/Frontend/OpenMP/CMakeLists.txt b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
index 5bf15ca3a899..e60b59c1203b 100644
--- a/llvm/lib/Frontend/OpenMP/CMakeLists.txt
+++ b/llvm/lib/Frontend/OpenMP/CMakeLists.txt
@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMFrontendOpenMP
   OMP.cpp
   OMPContext.cpp
   OMPIRBuilder.cpp
+  DirectiveNameParser.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Frontend
diff --git a/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp b/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp
new file mode 100644
index 000000000000..62e24825111f
--- /dev/null
+++ b/llvm/lib/Frontend/OpenMP/DirectiveNameParser.cpp
@@ -0,0 +1,83 @@
+//===- DirectiveNameParser.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+
+#include <cassert>
+#include <memory>
+
+namespace llvm::omp {
+DirectiveNameParser::DirectiveNameParser(SourceLanguage L) {
+  // Take every directive, get its name in every version, break the name up
+  // into whitespace-separated tokens, and insert each token.
+  for (size_t I : llvm::seq<size_t>(Directive_enumSize)) {
+    auto D = static_cast<Directive>(I);
+    if (D == Directive::OMPD_unknown || !(getDirectiveLanguages(D) & L))
+      continue;
+    for (unsigned Ver : getOpenMPVersions())
+      insertName(getOpenMPDirectiveName(D, Ver), D);
+  }
+}
+
+const DirectiveNameParser::State *
+DirectiveNameParser::consume(const State *Current, StringRef Tok) const {
+  if (!Current)
+    return Current;
+  assert(Current->isValid() && "Invalid input state");
+  if (const State *Next = Current->next(Tok))
+    return Next->isValid() ? Next : nullptr;
+  return nullptr;
+}
+
+SmallVector<StringRef> DirectiveNameParser::tokenize(StringRef Str) {
+  SmallVector<StringRef> Tokens;
+  SplitString(Str, Tokens);
+  return Tokens;
+}
+
+void DirectiveNameParser::insertName(StringRef Name, Directive D) {
+  State *Where = &InitialState;
+
+  for (StringRef Tok : tokenize(Name))
+    Where = insertTransition(Where, Tok);
+
+  Where->Value = D;
+}
+
+DirectiveNameParser::State *
+DirectiveNameParser::insertTransition(State *From, StringRef Tok) {
+  assert(From && "Expecting state");
+  if (!From->Transition)
+    From->Transition = std::make_unique<State::TransitionMapTy>();
+  if (State *Next = From->next(Tok))
+    return Next;
+
+  auto [Where, DidIt] = From->Transition->try_emplace(Tok, State());
+  assert(DidIt && "Map insertion failed");
+  return &Where->second;
+}
+
+const DirectiveNameParser::State *
+DirectiveNameParser::State::next(StringRef Tok) const {
+  if (!Transition)
+    return nullptr;
+  auto F = Transition->find(Tok);
+  return F != Transition->end() ? &F->second : nullptr;
+}
+
+DirectiveNameParser::State *DirectiveNameParser::State::next(StringRef Tok) {
+  if (!Transition)
+    return nullptr;
+  auto F = Transition->find(Tok);
+  return F != Transition->end() ? &F->second : nullptr;
+}
+} // namespace llvm::omp
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ba8ea147f78f..909314b27e21 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -856,8 +856,12 @@ void OpenMPIRBuilder::finalize(Function *Fn) {
         M.getGlobalVariable("__openmp_nvptx_data_transfer_temporary_storage")};
     emitUsed("llvm.compiler.used", LLVMCompilerUsed);
   }
+
+  IsFinalized = true;
 }
 
+bool OpenMPIRBuilder::isFinalized() { return IsFinalized; }
+
 OpenMPIRBuilder::~OpenMPIRBuilder() {
   assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
 }
@@ -6943,7 +6947,6 @@ static void FixupDebugInfoForOutlinedFunction(
   if (!NewSP)
     return;
 
-  DenseMap<const MDNode *, MDNode *> Cache;
   SmallDenseMap<DILocalVariable *, DILocalVariable *> RemappedVariables;
 
   auto GetUpdatedDIVariable = [&](DILocalVariable *OldVar, unsigned arg) {
@@ -6953,13 +6956,11 @@ static void FixupDebugInfoForOutlinedFunction(
     if (NewVar && (arg == NewVar->getArg()))
       return NewVar;
 
-    DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-        *OldVar->getScope(), *NewSP, Builder.getContext(), Cache);
     NewVar = llvm::DILocalVariable::get(
-        Builder.getContext(), NewScope, OldVar->getName(), OldVar->getFile(),
-        OldVar->getLine(), OldVar->getType(), arg, OldVar->getFlags(),
-        OldVar->getDWARFMemorySpace(), OldVar->getAlignInBits(),
-        OldVar->getAnnotations());
+        Builder.getContext(), OldVar->getScope(), OldVar->getName(),
+        OldVar->getFile(), OldVar->getLine(), OldVar->getType(), arg,
+        OldVar->getFlags(), OldVar->getDWARFMemorySpace(),
+        OldVar->getAlignInBits(), OldVar->getAnnotations());
     return NewVar;
   };
 
@@ -7020,7 +7021,8 @@ static void FixupDebugInfoForOutlinedFunction(
       }
     }
 
-    DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
+    if (ArgNo != 0)
+      DR->setVariable(GetUpdatedDIVariable(OldVar, ArgNo));
   };
 
   // The location and scope of variable intrinsics and records still point to
@@ -7099,36 +7101,9 @@ static Expected<Function *> createOutlinedFunction(
 
   // Save insert point.
   IRBuilder<>::InsertPointGuard IPG(Builder);
-  // If there's a DISubprogram associated with current function, then
-  // generate one for the outlined function.
-  if (Function *ParentFunc = BB->getParent()) {
-    if (DISubprogram *SP = ParentFunc->getSubprogram()) {
-      DICompileUnit *CU = SP->getUnit();
-      DIBuilder DB(*M, true, CU);
-      DebugLoc DL = Builder.getCurrentDebugLocation();
-      if (DL) {
-        // TODO: We are using nullopt for arguments at the moment. This will
-        // need to be updated when debug data is being generated for variables.
-        DISubroutineType *Ty =
-            DB.createSubroutineType(DB.getOrCreateTypeArray({}));
-        DISubprogram::DISPFlags SPFlags = DISubprogram::SPFlagDefinition |
-                                          DISubprogram::SPFlagOptimized |
-                                          DISubprogram::SPFlagLocalToUnit;
-
-        DISubprogram *OutlinedSP = DB.createFunction(
-            CU, FuncName, FuncName, SP->getFile(), DL.getLine(), Ty,
-            DL.getLine(), DINode::DIFlags::FlagArtificial, SPFlags);
-
-        // Attach subprogram to the function.
-        Func->setSubprogram(OutlinedSP);
-        // Update the CurrentDebugLocation in the builder so that right scope
-        // is used for things inside outlined function.
-        Builder.SetCurrentDebugLocation(
-            DILocation::get(Func->getContext(), DL.getLine(), DL.getCol(),
-                            OutlinedSP, DL.getInlinedAt()));
-      }
-    }
-  }
+  // We will generate the entries in the outlined function but the debug
+  // location may still be pointing to the parent function. Reset it now.
+  Builder.SetCurrentDebugLocation(llvm::DebugLoc());
 
   // Generate the region into the function.
   BasicBlock *EntryBB = BasicBlock::Create(Builder.getContext(), "entry", Func);
@@ -9951,7 +9926,7 @@ void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
 TargetRegionEntryInfo
 OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
                                           StringRef ParentName) {
-  sys::fs::UniqueID ID;
+  sys::fs::UniqueID ID(0xdeadf17e, 0);
   auto FileIDInfo = CallBack();
   uint64_t FileID = 0;
   std::error_code EC = sys::fs::getUniqueID(std::get<0>(FileIDInfo), ID);
diff --git a/llvm/lib/FuzzMutate/IRMutator.cpp b/llvm/lib/FuzzMutate/IRMutator.cpp
index 0070fc1a6c7e..d1abf78222c8 100644
--- a/llvm/lib/FuzzMutate/IRMutator.cpp
+++ b/llvm/lib/FuzzMutate/IRMutator.cpp
@@ -20,6 +20,8 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PassInstrumentation.h"
@@ -115,9 +117,41 @@ InjectorIRStrategy::chooseOperation(Value *Src, RandomIRBuilder &IB) {
   return *RS;
 }
 
+static inline Instruction *getEffectiveTerminator(BasicBlock &BB) {
+  if (Instruction *I = BB.getTerminatingMustTailCall()) {
+    return I;
+  } else {
+    // Certain intrinsics, such as @llvm.amdgcn.cs.chain, must be immediately
+    // followed by an unreachable instruction..
+    if (UnreachableInst *UI = dyn_cast<UnreachableInst>(BB.getTerminator())) {
+      if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(UI->getPrevNode())) {
+        return II;
+      }
+    }
+  }
+
+  return BB.getTerminator();
+}
+
+static inline BasicBlock::iterator getEndIterator(BasicBlock &BB) {
+  auto End = BB.end();
+
+  if (BB.empty()) {
+    return End;
+  }
+
+  Instruction *EffectiveTerminator = getEffectiveTerminator(BB);
+  if (EffectiveTerminator != BB.getTerminator()) {
+    // Adjust range for special cases such as tail call.
+    End = std::prev(BB.end());
+  }
+
+  return End;
+}
+
 static inline iterator_range<BasicBlock::iterator>
 getInsertionRange(BasicBlock &BB) {
-  auto End = BB.getTerminatingMustTailCall() ? std::prev(BB.end()) : BB.end();
+  auto End = getEndIterator(BB);
   return make_range(BB.getFirstInsertionPt(), End);
 }
 
@@ -409,6 +443,12 @@ static bool isUnsupportedFunction(Function *F) {
     return true;
   }
 
+  // This intrinsic has specific requirements for its parameters and the caller
+  // must adhere to certain calling conventions.
+  if (F->isIntrinsic() && F->getIntrinsicID() == Intrinsic::amdgcn_cs_chain) {
+    return true;
+  }
+
   return false;
 }
 
@@ -641,8 +681,9 @@ void ShuffleBlockStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
   std::map<size_t, Instruction *> AliveInsts;
   std::map<Instruction *, size_t> AliveInstsLookup;
   size_t InsertIdx = 0;
-  for (auto &I : make_early_inc_range(make_range(
-           BB.getFirstInsertionPt(), BB.getTerminator()->getIterator()))) {
+  for (auto &I : make_early_inc_range(
+           make_range(BB.getFirstInsertionPt(),
+                      getEffectiveTerminator(BB)->getIterator()))) {
     // First gather all instructions that can be shuffled. Don't take
     // terminator.
     AliveInsts.insert({InsertIdx, &I});
@@ -702,7 +743,7 @@ void ShuffleBlockStrategy::mutate(BasicBlock &BB, RandomIRBuilder &IB) {
     }
   }
 
-  Instruction *Terminator = BB.getTerminator();
+  Instruction *Terminator = getEffectiveTerminator(BB);
   // Then put instructions back.
   for (Instruction *I : Insts) {
     I->insertBefore(Terminator->getIterator());
diff --git a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
index 5e793997aee6..8b034137e470 100644
--- a/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/llvm/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -108,6 +108,55 @@ Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
   return findOrCreateSource(BB, Insts, {}, anyType());
 }
 
+// Adapts the current pointer for a legal mem operation on the target arch.
+static Value *buildTargetLegalPtr(Module *M, Value *Ptr, InsertPosition IP,
+                                  const Twine &Name,
+                                  SmallVector<Instruction *> *NewInsts) {
+  if (M && M->getTargetTriple().isAMDGCN()) {
+    // Check if we should perform an address space cast
+    PointerType *pointerType = dyn_cast<PointerType>(Ptr->getType());
+    if (pointerType && pointerType->getAddressSpace() == 8) {
+      // Perform address space cast from address space 8 to address space 7
+      auto NewPtr = new AddrSpaceCastInst(
+          Ptr, PointerType::get(M->getContext(), 7), Name + ".ASC", IP);
+      if (NewInsts)
+        NewInsts->push_back(NewPtr);
+      return NewPtr;
+    }
+  }
+
+  return Ptr;
+}
+
+// Stores a value to memory, considering the target triple's restrictions.
+static Instruction *buildTargetLegalStore(Value *Val, Value *Ptr,
+                                          InsertPosition IP, Module *M) {
+  Value *StorePtr = buildTargetLegalPtr(M, Ptr, IP, "", nullptr);
+  Instruction *Store = new StoreInst(Val, StorePtr, IP);
+  return Store;
+}
+
+// Loads a value from memory, considering the target triple's restrictions.
+static std::pair<Instruction *, SmallVector<Instruction *>>
+buildTargetLegalLoad(Type *AccessTy, Value *Ptr, InsertPosition IP, Module *M,
+                     const Twine &LoadName) {
+  SmallVector<Instruction *> NewInsts;
+
+  Value *LoadPtr = buildTargetLegalPtr(M, Ptr, IP, LoadName, &NewInsts);
+
+  Instruction *Load = new LoadInst(AccessTy, LoadPtr, LoadName, IP);
+  NewInsts.push_back(Load);
+
+  return std::make_pair(Load, NewInsts);
+}
+
+static void eraseNewInstructions(SmallVector<Instruction *> &NewInsts) {
+  // Remove in reverse order (uses before defs)
+  for (auto it = NewInsts.rbegin(); it != NewInsts.rend(); ++it) {
+    (*it)->eraseFromParent();
+  }
+}
+
 Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
                                            ArrayRef<Instruction *> Insts,
                                            ArrayRef<Value *> Srcs,
@@ -158,19 +207,20 @@ Value *RandomIRBuilder::findOrCreateSource(BasicBlock &BB,
       Module *M = BB.getParent()->getParent();
       auto [GV, DidCreate] = findOrCreateGlobalVariable(M, Srcs, Pred);
       Type *Ty = GV->getValueType();
-      LoadInst *LoadGV = nullptr;
-      if (BB.getTerminator()) {
-        LoadGV = new LoadInst(Ty, GV, "LGV", BB.getFirstInsertionPt());
-      } else {
-        LoadGV = new LoadInst(Ty, GV, "LGV", &BB);
-      }
+      InsertPosition IP = BB.getTerminator()
+                              ? InsertPosition(BB.getFirstInsertionPt())
+                              : InsertPosition(&BB);
+      // Build a legal load and track new instructions in case a rollback is
+      // needed.
+      auto [LoadGV, NewInsts] = buildTargetLegalLoad(Ty, GV, IP, M, "LGV");
       // Because we might be generating new values, we have to check if it
       // matches again.
       if (DidCreate) {
         if (Pred.matches(Srcs, LoadGV)) {
           return LoadGV;
         }
-        LoadGV->eraseFromParent();
+        // Remove newly inserted instructions
+        eraseNewInstructions(NewInsts);
         // If no one is using this GlobalVariable, delete it too.
         if (GV->use_empty()) {
           GV->eraseFromParent();
@@ -208,13 +258,18 @@ Value *RandomIRBuilder::newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
     }
     // Pick the type independently.
     Type *AccessTy = RS.getSelection()->getType();
-    auto *NewLoad = new LoadInst(AccessTy, Ptr, "L", IP);
+    // Build a legal load and track new instructions in case a rollback is
+    // needed.
+    auto [NewLoad, NewInsts] =
+        buildTargetLegalLoad(AccessTy, Ptr, IP, BB.getModule(), "L");
 
     // Only sample this load if it really matches the descriptor
     if (Pred.matches(Srcs, NewLoad))
       RS.sample(NewLoad, RS.totalWeight());
-    else
-      NewLoad->eraseFromParent();
+    else {
+      // Remove newly inserted instructions
+      eraseNewInstructions(NewInsts);
+    }
   }
 
   Value *newSrc = RS.getSelection();
@@ -320,8 +375,10 @@ Instruction *RandomIRBuilder::connectToSink(BasicBlock &BB,
       std::shuffle(Dominators.begin(), Dominators.end(), Rand);
       for (BasicBlock *Dom : Dominators) {
         for (Instruction &I : *Dom) {
-          if (isa<PointerType>(I.getType()))
-            return new StoreInst(V, &I, Insts.back()->getIterator());
+          if (isa<PointerType>(I.getType())) {
+            return buildTargetLegalStore(V, &I, Insts.back()->getIterator(),
+                                         I.getModule());
+          }
         }
       }
       break;
@@ -344,10 +401,10 @@ Instruction *RandomIRBuilder::connectToSink(BasicBlock &BB,
       /// TODO: allocate a new stack memory.
       return newSink(BB, Insts, V);
     case SinkToGlobalVariable: {
-      Module *M = BB.getParent()->getParent();
+      Module *M = BB.getModule();
       auto [GV, DidCreate] =
           findOrCreateGlobalVariable(M, {}, fuzzerop::onlyType(V->getType()));
-      return new StoreInst(V, GV, Insts.back()->getIterator());
+      return buildTargetLegalStore(V, GV, Insts.back()->getIterator(), M);
     }
     case EndOfValueSink:
     default:
@@ -369,7 +426,8 @@ Instruction *RandomIRBuilder::newSink(BasicBlock &BB,
     }
   }
 
-  return new StoreInst(V, Ptr, Insts.back()->getIterator());
+  return buildTargetLegalStore(V, Ptr, Insts.back()->getIterator(),
+                               BB.getModule());
 }
 
 Value *RandomIRBuilder::findPointer(BasicBlock &BB,
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index 33e127f3be17..18d7a460ffdf 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -447,8 +447,7 @@ void llvm::printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name) {
       // in the range 0-255.  This is important when building with MSVC because
       // its implementation will assert.  This situation can arise when dealing
       // with UTF-8 multibyte characters.
-      if (!isalnum(static_cast<unsigned char>(C)) && C != '-' && C != '.' &&
-          C != '_') {
+      if (!isalnum(C) && C != '-' && C != '.' && C != '_') {
         NeedsQuotes = true;
         break;
       }
@@ -2481,6 +2480,7 @@ static void writeDISubprogram(raw_ostream &Out, const DISubprogram *N,
   Printer.printMetadata("thrownTypes", N->getRawThrownTypes());
   Printer.printMetadata("annotations", N->getRawAnnotations());
   Printer.printString("targetFuncName", N->getTargetFuncName());
+  Printer.printBool("keyInstructions", N->getKeyInstructionsEnabled(), false);
   Out << ")";
 }
 
@@ -2635,6 +2635,11 @@ static void writeDILabel(raw_ostream &Out, const DILabel *N,
   Printer.printString("name", N->getName());
   Printer.printMetadata("file", N->getRawFile());
   Printer.printInt("line", N->getLine());
+  Printer.printInt("column", N->getColumn());
+  Printer.printBool("isArtificial", N->isArtificial(), false);
+  if (N->getCoroSuspendIdx())
+    Printer.printInt("coroSuspendIdx", *N->getCoroSuspendIdx(),
+                     /* ShouldSkipZero */ false);
   Out << ")";
 }
 
@@ -4228,6 +4233,9 @@ void AssemblyWriter::printFunction(const Function *F) {
       Out << "; Function Attrs: " << AttrStr << '\n';
   }
 
+  if (F->isIntrinsic() && F->getIntrinsicID() == Intrinsic::not_intrinsic)
+    Out << "; Unknown intrinsic\n";
+
   Machine.incorporateFunction(F);
 
   if (F->isDeclaration()) {
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 2189b11c5697..73025d21190a 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -2431,7 +2431,8 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty, AttributeSet AS,
           .addAttribute(Attribute::Writable)
           .addAttribute(Attribute::DeadOnUnwind)
           .addAttribute(Attribute::Initializes)
-          .addAttribute(Attribute::Captures);
+          .addAttribute(Attribute::Captures)
+          .addAttribute(Attribute::DeadOnReturn);
     if (ASK & ASK_UNSAFE_TO_DROP)
       Incompatible.addAttribute(Attribute::Nest)
           .addAttribute(Attribute::SwiftError)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 840294b08683..273bd9fd24d9 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -719,6 +719,12 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
                              .StartsWith("vqaddu.", Intrinsic::uadd_sat)
                              .StartsWith("vqsubs.", Intrinsic::ssub_sat)
                              .StartsWith("vqsubu.", Intrinsic::usub_sat)
+                             .StartsWith("vrinta.", Intrinsic::round)
+                             .StartsWith("vrintn.", Intrinsic::roundeven)
+                             .StartsWith("vrintm.", Intrinsic::floor)
+                             .StartsWith("vrintp.", Intrinsic::ceil)
+                             .StartsWith("vrintx.", Intrinsic::rint)
+                             .StartsWith("vrintz.", Intrinsic::trunc)
                              .Default(Intrinsic::not_intrinsic);
       if (ID != Intrinsic::not_intrinsic) {
         NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID,
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 17a1f71148e5..2bb6e97b42fe 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -325,22 +325,23 @@ DIStringType *DIBuilder::createStringType(StringRef Name,
 }
 
 DIDerivedType *DIBuilder::createQualifiedType(unsigned Tag, DIType *FromTy) {
-  return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, FromTy, 0,
-                            0, 0, std::nullopt, dwarf::DW_MSPACE_LLVM_none, std::nullopt, DINode::FlagZero);
+  return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, FromTy,
+                            (uint64_t)0, 0, (uint64_t)0, std::nullopt, dwarf::DW_MSPACE_LLVM_none,
+                            std::nullopt, DINode::FlagZero);
 }
 
 DIDerivedType *DIBuilder::createPtrAuthQualifiedType(
     DIType *FromTy, unsigned Key, bool IsAddressDiscriminated,
     unsigned ExtraDiscriminator, bool IsaPointer,
     bool AuthenticatesNullValues) {
-  return DIDerivedType::get(VMContext, dwarf::DW_TAG_LLVM_ptrauth_type, "",
-                            nullptr, 0, nullptr, FromTy, 0, 0, 0, std::nullopt,
-                            dwarf::DW_MSPACE_LLVM_none, 
-                            std::optional<DIDerivedType::PtrAuthData>(
-                                std::in_place, Key, IsAddressDiscriminated,
-                                ExtraDiscriminator, IsaPointer,
-                                AuthenticatesNullValues),
-                            DINode::FlagZero);
+  return DIDerivedType::get(
+      VMContext, dwarf::DW_TAG_LLVM_ptrauth_type, "", nullptr, 0, nullptr,
+      FromTy, (uint64_t)0, 0, (uint64_t)0, std::nullopt,
+      dwarf::DW_MSPACE_LLVM_none,
+      std::optional<DIDerivedType::PtrAuthData>(
+          std::in_place, Key, IsAddressDiscriminated, ExtraDiscriminator,
+          IsaPointer, AuthenticatesNullValues),
+      DINode::FlagZero);
 }
 
 DIDerivedType *
@@ -383,9 +384,9 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
   return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
-                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
-                            AlignInBits, 0, std::nullopt, dwarf::DW_MSPACE_LLVM_none, std::nullopt, 
-                            Flags, nullptr, Annotations);
+                            LineNo, getNonCompileUnitScope(Context), Ty,
+                            (uint64_t)0, AlignInBits, (uint64_t)0, std::nullopt, dwarf::DW_MSPACE_LLVM_none,
+                            std::nullopt, Flags, nullptr, Annotations);
 }
 
 DIDerivedType *
@@ -394,19 +395,19 @@ DIBuilder::createTemplateAlias(DIType *Ty, StringRef Name, DIFile *File,
                                DINodeArray TParams, uint32_t AlignInBits,
                                DINode::DIFlags Flags, DINodeArray Annotations) {
   return DIDerivedType::get(VMContext, dwarf::DW_TAG_template_alias, Name, File,
-                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
-                            AlignInBits, 0, std::nullopt,
+                            LineNo, getNonCompileUnitScope(Context), Ty,
+                            (uint64_t)0, AlignInBits, (uint64_t)0, std::nullopt,
                             dwarf::DW_MSPACE_LLVM_none,
-                            std::nullopt, Flags,
-                            TParams.get(), Annotations);
+                            std::nullopt, Flags, TParams.get(), Annotations);
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
   assert(Ty && "Invalid type!");
   assert(FriendTy && "Invalid friend type!");
   return DIDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, Ty,
-                            FriendTy, 0, 0, 0, std::nullopt,
-                            dwarf::DW_MSPACE_LLVM_none, std::nullopt, DINode::FlagZero);
+                            FriendTy, (uint64_t)0, 0, (uint64_t)0, std::nullopt,
+                            dwarf::DW_MSPACE_LLVM_none,
+                            std::nullopt, DINode::FlagZero);
 }
 
 DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy,
@@ -431,6 +432,17 @@ DIDerivedType *DIBuilder::createMemberType(
       std::nullopt, dwarf::DW_MSPACE_LLVM_none, std::nullopt, Flags, nullptr, Annotations);
 }
 
+DIDerivedType *DIBuilder::createMemberType(
+    DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+    Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
+    DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations) {
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File,
+                            LineNumber, getNonCompileUnitScope(Scope), Ty,
+                            SizeInBits, AlignInBits, OffsetInBits, std::nullopt,
+                            dwarf::DW_MSPACE_LLVM_none,
+                            std::nullopt, Flags, nullptr, Annotations);
+}
+
 static ConstantAsMetadata *getConstantOrNull(Constant *C) {
   if (C)
     return ConstantAsMetadata::get(C);
@@ -456,14 +468,29 @@ DIDerivedType *DIBuilder::createVariantMemberType(DIScope *Scope,
                                                   Constant *Discriminant,
                                                   DIType *Ty) {
   auto *V = DICompositeType::get(VMContext, dwarf::DW_TAG_variant, {}, nullptr,
-                                 0, getNonCompileUnitScope(Scope), {}, 0, 0, 0,
-                                 DINode::FlagZero, Elements, 0, {}, nullptr);
+                                 0, getNonCompileUnitScope(Scope), {},
+                                 (uint64_t)0, 0, (uint64_t)0, DINode::FlagZero,
+                                 Elements, 0, {}, nullptr);
 
   trackIfUnresolved(V);
   return createVariantMemberType(Scope, {}, nullptr, 0, 0, 0, 0, Discriminant,
                                  DINode::FlagZero, V);
 }
 
+DIDerivedType *DIBuilder::createBitFieldMemberType(
+    DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
+    Metadata *SizeInBits, Metadata *OffsetInBits, uint64_t StorageOffsetInBits,
+    DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations) {
+  Flags |= DINode::FlagBitField;
+  return DIDerivedType::get(
+      VMContext, dwarf::DW_TAG_member, Name, File, LineNumber,
+      getNonCompileUnitScope(Scope), Ty, SizeInBits, /*AlignInBits=*/0,
+      OffsetInBits, std::nullopt, dwarf::DW_MSPACE_LLVM_none, std::nullopt, Flags,
+      ConstantAsMetadata::get(ConstantInt::get(IntegerType::get(VMContext, 64),
+                                               StorageOffsetInBits)),
+      Annotations);
+}
+
 DIDerivedType *DIBuilder::createBitFieldMemberType(
     DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint64_t OffsetInBits, uint64_t StorageOffsetInBits,
@@ -485,9 +512,10 @@ DIBuilder::createStaticMemberType(DIScope *Scope, StringRef Name, DIFile *File,
                                   unsigned Tag, uint32_t AlignInBits) {
   Flags |= DINode::FlagStaticMember;
   return DIDerivedType::get(VMContext, Tag, Name, File, LineNumber,
-                            getNonCompileUnitScope(Scope), Ty, 0, AlignInBits,
-                            0, std::nullopt, dwarf::DW_MSPACE_LLVM_none, std::nullopt, 
-                            Flags, getConstantOrNull(Val));
+                            getNonCompileUnitScope(Scope), Ty, (uint64_t)0,
+                            AlignInBits, (uint64_t)0, std::nullopt,
+                            dwarf::DW_MSPACE_LLVM_none,
+                            std::nullopt, Flags, getConstantOrNull(Val));
 }
 
 DIDerivedType *
@@ -568,6 +596,22 @@ DICompositeType *DIBuilder::createClassType(
   return R;
 }
 
+DICompositeType *DIBuilder::createStructType(
+    DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber,
+    Metadata *SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags,
+    DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang,
+    DIType *VTableHolder, StringRef UniqueIdentifier, DIType *Specification,
+    uint32_t NumExtraInhabitants) {
+  auto *R = DICompositeType::get(
+      VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber,
+      getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
+      Flags, Elements, RunTimeLang, /*EnumKind=*/std::nullopt, VTableHolder,
+      nullptr, UniqueIdentifier, nullptr, nullptr, nullptr, nullptr, nullptr,
+      nullptr, Specification, NumExtraInhabitants);
+  trackIfUnresolved(R);
+  return R;
+}
+
 DICompositeType *DIBuilder::createStructType(
     DIScope *Context, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags,
@@ -924,9 +968,13 @@ DILocalVariable *DIBuilder::createParameterVariable(
 }
 
 DILabel *DIBuilder::createLabel(DIScope *Context, StringRef Name, DIFile *File,
-                                 unsigned LineNo, bool AlwaysPreserve) {
+                                unsigned LineNo, unsigned Column,
+                                bool IsArtificial,
+                                std::optional<unsigned> CoroSuspendIdx,
+                                bool AlwaysPreserve) {
   auto *Scope = cast<DILocalScope>(Context);
-  auto *Node = DILabel::get(VMContext, Scope, Name, File, LineNo);
+  auto *Node = DILabel::get(VMContext, Scope, Name, File, LineNo, Column,
+                            IsArtificial, CoroSuspendIdx);
 
   if (AlwaysPreserve) {
     /// The optimizer may remove labels. If there is an interest
@@ -953,14 +1001,14 @@ DISubprogram *DIBuilder::createFunction(
     unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine,
     DINode::DIFlags Flags, DISubprogram::DISPFlags SPFlags,
     DITemplateParameterArray TParams, DISubprogram *Decl,
-    DITypeArray ThrownTypes, DINodeArray Annotations,
-    StringRef TargetFuncName) {
+    DITypeArray ThrownTypes, DINodeArray Annotations, StringRef TargetFuncName,
+    bool UseKeyInstructions) {
   bool IsDefinition = SPFlags & DISubprogram::SPFlagDefinition;
   auto *Node = getSubprogram(
       /*IsDistinct=*/IsDefinition, VMContext, getNonCompileUnitScope(Context),
       Name, LinkageName, File, LineNo, Ty, ScopeLine, nullptr, 0, 0, Flags,
       SPFlags, IsDefinition ? CUNode : nullptr, TParams, Decl, nullptr,
-      ThrownTypes, Annotations, TargetFuncName);
+      ThrownTypes, Annotations, TargetFuncName, UseKeyInstructions);
 
   AllSubprograms.push_back(Node);
   trackIfUnresolved(Node);
@@ -987,7 +1035,7 @@ DISubprogram *DIBuilder::createMethod(
     unsigned LineNo, DISubroutineType *Ty, unsigned VIndex, int ThisAdjustment,
     DIType *VTableHolder, DINode::DIFlags Flags,
     DISubprogram::DISPFlags SPFlags, DITemplateParameterArray TParams,
-    DITypeArray ThrownTypes) {
+    DITypeArray ThrownTypes, bool UseKeyInstructions) {
   assert(getNonCompileUnitScope(Context) &&
          "Methods should have both a Context and a context that isn't "
          "the compile unit.");
@@ -997,7 +1045,7 @@ DISubprogram *DIBuilder::createMethod(
       /*IsDistinct=*/IsDefinition, VMContext, cast<DIScope>(Context), Name,
       LinkageName, F, LineNo, Ty, LineNo, VTableHolder, VIndex, ThisAdjustment,
       Flags, SPFlags, IsDefinition ? CUNode : nullptr, TParams, nullptr,
-      nullptr, ThrownTypes);
+      nullptr, ThrownTypes, nullptr, "", IsDefinition && UseKeyInstructions);
 
   AllSubprograms.push_back(SP);
   trackIfUnresolved(SP);
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index d9c3b2e738f2..afa01811344b 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -50,7 +50,7 @@ TinyPtrVector<DbgDeclareInst *> llvm::findDbgDeclares(Value *V) {
   // DenseMap lookup. This check is a bitfield datamember lookup.
   if (!V->isUsedByMetadata())
     return {};
-  auto *L = LocalAsMetadata::getIfExists(V);
+  auto *L = ValueAsMetadata::getIfExists(V);
   if (!L)
     return {};
   auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L);
@@ -69,7 +69,7 @@ TinyPtrVector<DbgVariableRecord *> llvm::findDVRDeclares(Value *V) {
   // DenseMap lookup. This check is a bitfield datamember lookup.
   if (!V->isUsedByMetadata())
     return {};
-  auto *L = LocalAsMetadata::getIfExists(V);
+  auto *L = ValueAsMetadata::getIfExists(V);
   if (!L)
     return {};
 
@@ -86,7 +86,7 @@ TinyPtrVector<DbgVariableRecord *> llvm::findDVRValues(Value *V) {
   // DenseMap lookup. This check is a bitfield datamember lookup.
   if (!V->isUsedByMetadata())
     return {};
-  auto *L = LocalAsMetadata::getIfExists(V);
+  auto *L = ValueAsMetadata::getIfExists(V);
   if (!L)
     return {};
 
@@ -376,7 +376,7 @@ bool DebugInfoFinder::addType(DIType *DT) {
   if (!NodesSeen.insert(DT).second)
     return false;
 
-  TYs.push_back(const_cast<DIType *>(DT));
+  TYs.push_back(DT);
   return true;
 }
 
@@ -1832,6 +1832,12 @@ unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram) {
   return unwrapDI<DISubprogram>(Subprogram)->getLine();
 }
 
+void LLVMDISubprogramReplaceType(LLVMMetadataRef Subprogram,
+                                 LLVMMetadataRef SubroutineType) {
+  unwrapDI<DISubprogram>(Subprogram)
+      ->replaceType(unwrapDI<DISubroutineType>(SubroutineType));
+}
+
 LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst) {
   return wrap(unwrap<Instruction>(Inst)->getDebugLoc().getAsMDNode());
 }
@@ -1850,7 +1856,8 @@ LLVMMetadataRef LLVMDIBuilderCreateLabel(LLVMDIBuilderRef Builder,
                                          LLVMBool AlwaysPreserve) {
   return wrap(unwrap(Builder)->createLabel(
       unwrapDI<DIScope>(Context), StringRef(Name, NameLen),
-      unwrapDI<DIFile>(File), LineNo, AlwaysPreserve));
+      unwrapDI<DIFile>(File), LineNo, /*Column*/ 0, /*IsArtificial*/ false,
+      /*CoroSuspendIdx*/ std::nullopt, AlwaysPreserve));
 }
 
 LLVMDbgRecordRef LLVMDIBuilderInsertLabelBefore(LLVMDIBuilderRef Builder,
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index a01027e3fe8f..d25bd0fd06c4 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -825,25 +825,23 @@ DIGenericSubrange::BoundType DIGenericSubrange::getStride() const {
 }
 
 DISubrangeType::DISubrangeType(LLVMContext &C, StorageType Storage,
-                               unsigned Line, uint64_t SizeInBits,
-                               uint32_t AlignInBits, DIFlags Flags,
-                               ArrayRef<Metadata *> Ops)
+                               unsigned Line, uint32_t AlignInBits,
+                               DIFlags Flags, ArrayRef<Metadata *> Ops)
     : DIType(C, DISubrangeTypeKind, Storage, dwarf::DW_TAG_subrange_type, Line,
-             SizeInBits, AlignInBits, 0, 0, Flags, Ops) {}
+             AlignInBits, 0, Flags, Ops) {}
 
 DISubrangeType *DISubrangeType::getImpl(
     LLVMContext &Context, MDString *Name, Metadata *File, unsigned Line,
-    Metadata *Scope, uint64_t SizeInBits, uint32_t AlignInBits, DIFlags Flags,
+    Metadata *Scope, Metadata *SizeInBits, uint32_t AlignInBits, DIFlags Flags,
     Metadata *BaseType, Metadata *LowerBound, Metadata *UpperBound,
     Metadata *Stride, Metadata *Bias, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(DISubrangeType, (Name, File, Line, Scope, SizeInBits,
                                          AlignInBits, Flags, BaseType,
                                          LowerBound, UpperBound, Stride, Bias));
-  Metadata *Ops[] = {File,       Scope,      Name,   BaseType,
-                     LowerBound, UpperBound, Stride, Bias};
-  DEFINE_GETIMPL_STORE(DISubrangeType, (Line, SizeInBits, AlignInBits, Flags),
-                       Ops);
+  Metadata *Ops[] = {File,     Scope,      Name,       SizeInBits, nullptr,
+                     BaseType, LowerBound, UpperBound, Stride,     Bias};
+  DEFINE_GETIMPL_STORE(DISubrangeType, (Line, AlignInBits, Flags), Ops);
 }
 
 DISubrangeType::BoundType
@@ -883,18 +881,17 @@ DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, const APInt &Value,
 }
 
 DIBasicType *DIBasicType::getImpl(LLVMContext &Context, unsigned Tag,
-                                  MDString *Name, uint64_t SizeInBits,
+                                  MDString *Name, Metadata *SizeInBits,
                                   uint32_t AlignInBits, unsigned Encoding,
                                   uint32_t NumExtraInhabitants, DIFlags Flags,
                                   StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(DIBasicType, (Tag, Name, SizeInBits, AlignInBits,
                                       Encoding, NumExtraInhabitants, Flags));
-  Metadata *Ops[] = {nullptr, nullptr, Name};
-  DEFINE_GETIMPL_STORE(
-      DIBasicType,
-      (Tag, SizeInBits, AlignInBits, Encoding, NumExtraInhabitants, Flags),
-      Ops);
+  Metadata *Ops[] = {nullptr, nullptr, Name, SizeInBits, nullptr};
+  DEFINE_GETIMPL_STORE(DIBasicType,
+                       (Tag, AlignInBits, Encoding, NumExtraInhabitants, Flags),
+                       Ops);
 }
 
 std::optional<DIBasicType::Signedness> DIBasicType::getSignedness() const {
@@ -914,18 +911,18 @@ std::optional<DIBasicType::Signedness> DIBasicType::getSignedness() const {
 
 DIFixedPointType *
 DIFixedPointType::getImpl(LLVMContext &Context, unsigned Tag, MDString *Name,
-                          uint64_t SizeInBits, uint32_t AlignInBits,
+                          Metadata *SizeInBits, uint32_t AlignInBits,
                           unsigned Encoding, DIFlags Flags, unsigned Kind,
                           int Factor, APInt Numerator, APInt Denominator,
                           StorageType Storage, bool ShouldCreate) {
   DEFINE_GETIMPL_LOOKUP(DIFixedPointType,
                         (Tag, Name, SizeInBits, AlignInBits, Encoding, Flags,
                          Kind, Factor, Numerator, Denominator));
-  Metadata *Ops[] = {nullptr, nullptr, Name};
-  DEFINE_GETIMPL_STORE(DIFixedPointType,
-                       (Tag, SizeInBits, AlignInBits, Encoding, Flags, Kind,
-                        Factor, Numerator, Denominator),
-                       Ops);
+  Metadata *Ops[] = {nullptr, nullptr, Name, SizeInBits, nullptr};
+  DEFINE_GETIMPL_STORE(
+      DIFixedPointType,
+      (Tag, AlignInBits, Encoding, Flags, Kind, Factor, Numerator, Denominator),
+      Ops);
 }
 
 bool DIFixedPointType::isSigned() const {
@@ -957,17 +954,17 @@ DIStringType *DIStringType::getImpl(LLVMContext &Context, unsigned Tag,
                                     MDString *Name, Metadata *StringLength,
                                     Metadata *StringLengthExp,
                                     Metadata *StringLocationExp,
-                                    uint64_t SizeInBits, uint32_t AlignInBits,
+                                    Metadata *SizeInBits, uint32_t AlignInBits,
                                     unsigned Encoding, StorageType Storage,
                                     bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(DIStringType,
                         (Tag, Name, StringLength, StringLengthExp,
                          StringLocationExp, SizeInBits, AlignInBits, Encoding));
-  Metadata *Ops[] = {nullptr,      nullptr,         Name,
-                     StringLength, StringLengthExp, StringLocationExp};
-  DEFINE_GETIMPL_STORE(DIStringType, (Tag, SizeInBits, AlignInBits, Encoding),
-                       Ops);
+  Metadata *Ops[] = {nullptr,         nullptr,          Name,
+                     SizeInBits,      nullptr,          StringLength,
+                     StringLengthExp, StringLocationExp};
+  DEFINE_GETIMPL_STORE(DIStringType, (Tag, AlignInBits, Encoding), Ops);
 }
 DIType *DIDerivedType::getClassType() const {
   assert(getTag() == dwarf::DW_TAG_ptr_to_member_type);
@@ -1004,21 +1001,21 @@ Constant *DIDerivedType::getDiscriminantValue() const {
 
 DIDerivedType *DIDerivedType::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
-    unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
-    uint32_t AlignInBits, uint64_t OffsetInBits,
+    unsigned Line, Metadata *Scope, Metadata *BaseType, Metadata *SizeInBits,
+    uint32_t AlignInBits, Metadata *OffsetInBits,
     std::optional<unsigned> DWARFAddressSpace, dwarf::MemorySpace MS,
-    std::optional<PtrAuthData> PtrAuthData, DIFlags Flags, Metadata *ExtraData, Metadata *Annotations,
-    StorageType Storage, bool ShouldCreate) {
+    std::optional<PtrAuthData> PtrAuthData, DIFlags Flags, Metadata *ExtraData,
+    Metadata *Annotations, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   DEFINE_GETIMPL_LOOKUP(DIDerivedType,
                         (Tag, Name, File, Line, Scope, BaseType, SizeInBits,
                          AlignInBits, OffsetInBits, DWARFAddressSpace, MS, 
                          PtrAuthData, Flags, ExtraData, Annotations));
-  Metadata *Ops[] = {File, Scope, Name, BaseType, ExtraData, Annotations};
-  DEFINE_GETIMPL_STORE(DIDerivedType,
-                       (Tag, Line, SizeInBits, AlignInBits, OffsetInBits,
-                        DWARFAddressSpace, MS, PtrAuthData, Flags),
-                       Ops);
+  Metadata *Ops[] = {File,         Scope,    Name,      SizeInBits,
+                     OffsetInBits, BaseType, ExtraData, Annotations};
+  DEFINE_GETIMPL_STORE(
+      DIDerivedType,
+      (Tag, Line, AlignInBits, DWARFAddressSpace, MS, PtrAuthData, Flags), Ops);
 }
 
 std::optional<DIDerivedType::PtrAuthData>
@@ -1030,8 +1027,8 @@ DIDerivedType::getPtrAuthData() const {
 
 DICompositeType *DICompositeType::getImpl(
     LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *File,
-    unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
-    uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags,
+    unsigned Line, Metadata *Scope, Metadata *BaseType, Metadata *SizeInBits,
+    uint32_t AlignInBits, Metadata *OffsetInBits, DIFlags Flags,
     Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
     Metadata *VTableHolder, Metadata *TemplateParams, MDString *Identifier,
     Metadata *Discriminator, Metadata *DataLocation, Metadata *Associated,
@@ -1047,20 +1044,21 @@ DICompositeType *DICompositeType::getImpl(
        OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams,
        Identifier, Discriminator, DataLocation, Associated, Allocated, Rank,
        Annotations, Specification, NumExtraInhabitants, BitStride));
-  Metadata *Ops[] = {File,          Scope,        Name,           BaseType,
-                     Elements,      VTableHolder, TemplateParams, Identifier,
-                     Discriminator, DataLocation, Associated,     Allocated,
-                     Rank,          Annotations,  Specification,  BitStride};
+  Metadata *Ops[] = {File,           Scope,      Name,          SizeInBits,
+                     OffsetInBits,   BaseType,   Elements,      VTableHolder,
+                     TemplateParams, Identifier, Discriminator, DataLocation,
+                     Associated,     Allocated,  Rank,          Annotations,
+                     Specification,  BitStride};
   DEFINE_GETIMPL_STORE(DICompositeType,
-                       (Tag, Line, RuntimeLang, SizeInBits, AlignInBits,
-                        OffsetInBits, NumExtraInhabitants, EnumKind, Flags),
+                       (Tag, Line, RuntimeLang, AlignInBits,
+                        NumExtraInhabitants, EnumKind, Flags),
                        Ops);
 }
 
 DICompositeType *DICompositeType::buildODRType(
     LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name,
     Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType,
-    uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+    Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
     Metadata *Specification, uint32_t NumExtraInhabitants, DIFlags Flags,
     Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
     Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
@@ -1086,12 +1084,13 @@ DICompositeType *DICompositeType::buildODRType(
     return CT;
 
   // Mutate CT in place.  Keep this in sync with getImpl.
-  CT->mutate(Tag, Line, RuntimeLang, SizeInBits, AlignInBits, OffsetInBits,
-             NumExtraInhabitants, EnumKind, Flags);
-  Metadata *Ops[] = {File,          Scope,        Name,           BaseType,
-                     Elements,      VTableHolder, TemplateParams, &Identifier,
-                     Discriminator, DataLocation, Associated,     Allocated,
-                     Rank,          Annotations,  Specification,  BitStride};
+  CT->mutate(Tag, Line, RuntimeLang, AlignInBits, NumExtraInhabitants, EnumKind,
+             Flags);
+  Metadata *Ops[] = {File,           Scope,       Name,          SizeInBits,
+                     OffsetInBits,   BaseType,    Elements,      VTableHolder,
+                     TemplateParams, &Identifier, Discriminator, DataLocation,
+                     Associated,     Allocated,   Rank,          Annotations,
+                     Specification,  BitStride};
   assert((std::end(Ops) - std::begin(Ops)) == (int)CT->getNumOperands() &&
          "Mismatched number of operands");
   for (unsigned I = 0, E = CT->getNumOperands(); I != E; ++I)
@@ -1103,7 +1102,7 @@ DICompositeType *DICompositeType::buildODRType(
 DICompositeType *DICompositeType::getODRType(
     LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name,
     Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType,
-    uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits,
+    Metadata *SizeInBits, uint32_t AlignInBits, Metadata *OffsetInBits,
     Metadata *Specification, uint32_t NumExtraInhabitants, DIFlags Flags,
     Metadata *Elements, unsigned RuntimeLang, std::optional<uint32_t> EnumKind,
     Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator,
@@ -1138,7 +1137,7 @@ DISubroutineType::DISubroutineType(LLVMContext &C, StorageType Storage,
                                    DIFlags Flags, uint8_t CC,
                                    ArrayRef<Metadata *> Ops)
     : DIType(C, DISubroutineTypeKind, Storage, dwarf::DW_TAG_subroutine_type, 0,
-             0, 0, 0, 0, Flags, Ops),
+             0, 0, Flags, Ops),
       CC(CC) {}
 
 DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, DIFlags Flags,
@@ -1146,7 +1145,7 @@ DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, DIFlags Flags,
                                             StorageType Storage,
                                             bool ShouldCreate) {
   DEFINE_GETIMPL_LOOKUP(DISubroutineType, (Flags, CC, TypeArray));
-  Metadata *Ops[] = {nullptr, nullptr, nullptr, TypeArray};
+  Metadata *Ops[] = {nullptr, nullptr, nullptr, nullptr, nullptr, TypeArray};
   DEFINE_GETIMPL_STORE(DISubroutineType, (Flags, CC), Ops);
 }
 
@@ -1293,11 +1292,12 @@ const char *DICompileUnit::nameTableKindString(DebugNameTableKind NTK) {
 DISubprogram::DISubprogram(LLVMContext &C, StorageType Storage, unsigned Line,
                            unsigned ScopeLine, unsigned VirtualIndex,
                            int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags,
-                           ArrayRef<Metadata *> Ops)
+                           bool UsesKeyInstructions, ArrayRef<Metadata *> Ops)
     : DILocalScope(C, DISubprogramKind, Storage, dwarf::DW_TAG_subprogram, Ops),
       Line(Line), ScopeLine(ScopeLine), VirtualIndex(VirtualIndex),
       ThisAdjustment(ThisAdjustment), Flags(Flags), SPFlags(SPFlags) {
   static_assert(dwarf::DW_VIRTUALITY_max < 4, "Virtuality out of range");
+  SubclassData1 = UsesKeyInstructions;
 }
 DISubprogram::DISPFlags
 DISubprogram::toSPFlags(bool IsLocalToUnit, bool IsDefinition, bool IsOptimized,
@@ -1396,7 +1396,7 @@ DISubprogram *DISubprogram::getImpl(
     int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, Metadata *Unit,
     Metadata *TemplateParams, Metadata *Declaration, Metadata *RetainedNodes,
     Metadata *ThrownTypes, Metadata *Annotations, MDString *TargetFuncName,
-    StorageType Storage, bool ShouldCreate) {
+    bool UsesKeyInstructions, StorageType Storage, bool ShouldCreate) {
   assert(isCanonical(Name) && "Expected canonical MDString");
   assert(isCanonical(LinkageName) && "Expected canonical MDString");
   assert(isCanonical(TargetFuncName) && "Expected canonical MDString");
@@ -1405,7 +1405,7 @@ DISubprogram *DISubprogram::getImpl(
                          ContainingType, VirtualIndex, ThisAdjustment, Flags,
                          SPFlags, Unit, TemplateParams, Declaration,
                          RetainedNodes, ThrownTypes, Annotations,
-                         TargetFuncName));
+                         TargetFuncName, UsesKeyInstructions));
   SmallVector<Metadata *, 13> Ops = {
       File,           Scope,          Name,        LinkageName,
       Type,           Unit,           Declaration, RetainedNodes,
@@ -1425,10 +1425,10 @@ DISubprogram *DISubprogram::getImpl(
       }
     }
   }
-  DEFINE_GETIMPL_STORE_N(
-      DISubprogram,
-      (Line, ScopeLine, VirtualIndex, ThisAdjustment, Flags, SPFlags), Ops,
-      Ops.size());
+  DEFINE_GETIMPL_STORE_N(DISubprogram,
+                         (Line, ScopeLine, VirtualIndex, ThisAdjustment, Flags,
+                          SPFlags, UsesKeyInstructions),
+                         Ops, Ops.size());
 }
 
 bool DISubprogram::describes(const Function *F) const {
@@ -1622,18 +1622,27 @@ std::optional<uint64_t> DIVariable::getSizeInBits() const {
 }
 
 DILabel::DILabel(LLVMContext &C, StorageType Storage, unsigned Line,
+                 unsigned Column, bool IsArtificial,
+                 std::optional<unsigned> CoroSuspendIdx,
                  ArrayRef<Metadata *> Ops)
     : DINode(C, DILabelKind, Storage, dwarf::DW_TAG_label, Ops) {
-  SubclassData32 = Line;
+  this->SubclassData32 = Line;
+  this->Column = Column;
+  this->IsArtificial = IsArtificial;
+  this->CoroSuspendIdx = CoroSuspendIdx;
 }
 DILabel *DILabel::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name,
-                          Metadata *File, unsigned Line, StorageType Storage,
-                          bool ShouldCreate) {
+                          Metadata *File, unsigned Line, unsigned Column,
+                          bool IsArtificial,
+                          std::optional<unsigned> CoroSuspendIdx,
+                          StorageType Storage, bool ShouldCreate) {
   assert(Scope && "Expected scope");
   assert(isCanonical(Name) && "Expected canonical MDString");
-  DEFINE_GETIMPL_LOOKUP(DILabel, (Scope, Name, File, Line));
+  DEFINE_GETIMPL_LOOKUP(
+      DILabel, (Scope, Name, File, Line, Column, IsArtificial, CoroSuspendIdx));
   Metadata *Ops[] = {Scope, Name, File};
-  DEFINE_GETIMPL_STORE(DILabel, (Line), Ops);
+  DEFINE_GETIMPL_STORE(DILabel, (Line, Column, IsArtificial, CoroSuspendIdx),
+                       Ops);
 }
 
 DIExpression *DIExpression::getImpl(LLVMContext &Context,
diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp
index ffeeeb6f1e4b..79c5b896f8f2 100644
--- a/llvm/lib/IR/DebugLoc.cpp
+++ b/llvm/lib/IR/DebugLoc.cpp
@@ -9,11 +9,35 @@
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DebugInfo.h"
+
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#include "llvm/Support/Signals.h"
+
+namespace llvm {
+DbgLocOrigin::DbgLocOrigin(bool ShouldCollectTrace) {
+  if (!ShouldCollectTrace)
+    return;
+  auto &[Depth, StackTrace] = StackTraces.emplace_back();
+  Depth = sys::getStackTrace(StackTrace);
+}
+void DbgLocOrigin::addTrace() {
+  // We only want to add new stacktraces if we already have one: addTrace exists
+  // to provide more context to how missing DebugLocs have propagated through
+  // the program, but by design if there is no existing stacktrace then we have
+  // decided not to track this DebugLoc as being "missing".
+  if (StackTraces.empty())
+    return;
+  auto &[Depth, StackTrace] = StackTraces.emplace_back();
+  Depth = sys::getStackTrace(StackTrace);
+}
+} // namespace llvm
+#endif
+
 using namespace llvm;
 
 #if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 DILocAndCoverageTracking::DILocAndCoverageTracking(const DILocation *L)
-    : TrackingMDNodeRef(const_cast<DILocation *>(L)),
+    : TrackingMDNodeRef(const_cast<DILocation *>(L)), DbgLocOrigin(!L),
       Kind(DebugLocKind::Normal) {}
 #endif // LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
 
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 28fb81055baf..7a03663e129d 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -37,6 +37,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/IR/ProfDataUtils.h"
 #include "llvm/IR/SymbolTableListTraits.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
@@ -129,6 +130,12 @@ bool Argument::hasByValAttr() const {
   return hasAttribute(Attribute::ByVal);
 }
 
+bool Argument::hasDeadOnReturnAttr() const {
+  if (!getType()->isPointerTy())
+    return false;
+  return hasAttribute(Attribute::DeadOnReturn);
+}
+
 bool Argument::hasByRefAttr() const {
   if (!getType()->isPointerTy())
     return false;
@@ -1115,7 +1122,7 @@ std::optional<ProfileCount> Function::getEntryCount(bool AllowSynthetic) const {
   MDNode *MD = getMetadata(LLVMContext::MD_prof);
   if (MD && MD->getOperand(0))
     if (MDString *MDS = dyn_cast<MDString>(MD->getOperand(0))) {
-      if (MDS->getString() == "function_entry_count") {
+      if (MDS->getString() == MDProfLabels::FunctionEntryCount) {
         ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(1));
         uint64_t Count = CI->getValue().getZExtValue();
         // A value of -1 is used for SamplePGO when there were no samples.
@@ -1124,7 +1131,8 @@ std::optional<ProfileCount> Function::getEntryCount(bool AllowSynthetic) const {
           return std::nullopt;
         return ProfileCount(Count, PCT_Real);
       } else if (AllowSynthetic &&
-                 MDS->getString() == "synthetic_function_entry_count") {
+                 MDS->getString() ==
+                     MDProfLabels::SyntheticFunctionEntryCount) {
         ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(1));
         uint64_t Count = CI->getValue().getZExtValue();
         return ProfileCount(Count, PCT_Synthetic);
@@ -1137,7 +1145,7 @@ DenseSet<GlobalValue::GUID> Function::getImportGUIDs() const {
   DenseSet<GlobalValue::GUID> R;
   if (MDNode *MD = getMetadata(LLVMContext::MD_prof))
     if (MDString *MDS = dyn_cast<MDString>(MD->getOperand(0)))
-      if (MDS->getString() == "function_entry_count")
+      if (MDS->getString() == MDProfLabels::FunctionEntryCount)
         for (unsigned i = 2; i < MD->getNumOperands(); i++)
           R.insert(mdconst::extract<ConstantInt>(MD->getOperand(i))
                        ->getValue()
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index beefd5e7d7ee..28037d7ec561 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -51,7 +51,7 @@ GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
       *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
       StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
-  GV->setAlignment(Align(1));
+  GV->setAlignment(M->getDataLayout().getPrefTypeAlign(getInt8Ty()));
   return GV;
 }
 
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 2d89ec1b0a8d..3cfee89a1fdf 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -643,6 +643,10 @@ MemoryEffects CallBase::getMemoryEffects() const {
       if (hasClobberingOperandBundles())
         FnME |= MemoryEffects::writeOnly();
     }
+    if (isVolatile()) {
+      // Volatile operations also access inaccessible memory.
+      FnME |= MemoryEffects::inaccessibleMemOnly();
+    }
     ME &= FnME;
   }
   return ME;
@@ -2578,7 +2582,7 @@ Type *ExtractValueInst::getIndexedType(Type *Agg,
       return nullptr;
     }
   }
-  return const_cast<Type*>(Agg);
+  return Agg;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 9eddcf5e7fbe..541eb4b756be 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -491,27 +491,28 @@ template <> struct MDNodeKeyImpl<DIEnumerator> {
 template <> struct MDNodeKeyImpl<DIBasicType> {
   unsigned Tag;
   MDString *Name;
-  uint64_t SizeInBits;
+  Metadata *SizeInBits;
   uint32_t AlignInBits;
   unsigned Encoding;
   uint32_t NumExtraInhabitants;
   unsigned Flags;
 
-  MDNodeKeyImpl(unsigned Tag, MDString *Name, uint64_t SizeInBits,
+  MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *SizeInBits,
                 uint32_t AlignInBits, unsigned Encoding,
                 uint32_t NumExtraInhabitants, unsigned Flags)
       : Tag(Tag), Name(Name), SizeInBits(SizeInBits), AlignInBits(AlignInBits),
         Encoding(Encoding), NumExtraInhabitants(NumExtraInhabitants),
         Flags(Flags) {}
   MDNodeKeyImpl(const DIBasicType *N)
-      : Tag(N->getTag()), Name(N->getRawName()), SizeInBits(N->getSizeInBits()),
-        AlignInBits(N->getAlignInBits()), Encoding(N->getEncoding()),
+      : Tag(N->getTag()), Name(N->getRawName()),
+        SizeInBits(N->getRawSizeInBits()), AlignInBits(N->getAlignInBits()),
+        Encoding(N->getEncoding()),
         NumExtraInhabitants(N->getNumExtraInhabitants()), Flags(N->getFlags()) {
   }
 
   bool isKeyOf(const DIBasicType *RHS) const {
     return Tag == RHS->getTag() && Name == RHS->getRawName() &&
-           SizeInBits == RHS->getSizeInBits() &&
+           SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() &&
            Encoding == RHS->getEncoding() &&
            NumExtraInhabitants == RHS->getNumExtraInhabitants() &&
@@ -526,7 +527,7 @@ template <> struct MDNodeKeyImpl<DIBasicType> {
 template <> struct MDNodeKeyImpl<DIFixedPointType> {
   unsigned Tag;
   MDString *Name;
-  uint64_t SizeInBits;
+  Metadata *SizeInBits;
   uint32_t AlignInBits;
   unsigned Encoding;
   unsigned Flags;
@@ -535,20 +536,21 @@ template <> struct MDNodeKeyImpl<DIFixedPointType> {
   APInt Numerator;
   APInt Denominator;
 
-  MDNodeKeyImpl(unsigned Tag, MDString *Name, uint64_t SizeInBits,
+  MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *SizeInBits,
                 uint32_t AlignInBits, unsigned Encoding, unsigned Flags,
                 unsigned Kind, int Factor, APInt Numerator, APInt Denominator)
       : Tag(Tag), Name(Name), SizeInBits(SizeInBits), AlignInBits(AlignInBits),
         Encoding(Encoding), Flags(Flags), Kind(Kind), Factor(Factor),
         Numerator(Numerator), Denominator(Denominator) {}
   MDNodeKeyImpl(const DIFixedPointType *N)
-      : Tag(N->getTag()), Name(N->getRawName()), SizeInBits(N->getSizeInBits()),
-        AlignInBits(N->getAlignInBits()), Encoding(N->getEncoding()),
-        Flags(N->getFlags()), Kind(N->getKind()), Factor(N->getFactorRaw()),
-        Numerator(N->getNumeratorRaw()), Denominator(N->getDenominatorRaw()) {}
+      : Tag(N->getTag()), Name(N->getRawName()),
+        SizeInBits(N->getRawSizeInBits()), AlignInBits(N->getAlignInBits()),
+        Encoding(N->getEncoding()), Flags(N->getFlags()), Kind(N->getKind()),
+        Factor(N->getFactorRaw()), Numerator(N->getNumeratorRaw()),
+        Denominator(N->getDenominatorRaw()) {}
 
   bool isKeyOf(const DIFixedPointType *RHS) const {
-    return Name == RHS->getRawName() && SizeInBits == RHS->getSizeInBits() &&
+    return Name == RHS->getRawName() && SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() && Kind == RHS->getKind() &&
            (RHS->isRational() ? (Numerator == RHS->getNumerator() &&
                                  Denominator == RHS->getDenominator())
@@ -566,13 +568,13 @@ template <> struct MDNodeKeyImpl<DIStringType> {
   Metadata *StringLength;
   Metadata *StringLengthExp;
   Metadata *StringLocationExp;
-  uint64_t SizeInBits;
+  Metadata *SizeInBits;
   uint32_t AlignInBits;
   unsigned Encoding;
 
   MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *StringLength,
                 Metadata *StringLengthExp, Metadata *StringLocationExp,
-                uint64_t SizeInBits, uint32_t AlignInBits, unsigned Encoding)
+                Metadata *SizeInBits, uint32_t AlignInBits, unsigned Encoding)
       : Tag(Tag), Name(Name), StringLength(StringLength),
         StringLengthExp(StringLengthExp), StringLocationExp(StringLocationExp),
         SizeInBits(SizeInBits), AlignInBits(AlignInBits), Encoding(Encoding) {}
@@ -581,7 +583,7 @@ template <> struct MDNodeKeyImpl<DIStringType> {
         StringLength(N->getRawStringLength()),
         StringLengthExp(N->getRawStringLengthExp()),
         StringLocationExp(N->getRawStringLocationExp()),
-        SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()),
+        SizeInBits(N->getRawSizeInBits()), AlignInBits(N->getAlignInBits()),
         Encoding(N->getEncoding()) {}
 
   bool isKeyOf(const DIStringType *RHS) const {
@@ -589,7 +591,7 @@ template <> struct MDNodeKeyImpl<DIStringType> {
            StringLength == RHS->getRawStringLength() &&
            StringLengthExp == RHS->getRawStringLengthExp() &&
            StringLocationExp == RHS->getRawStringLocationExp() &&
-           SizeInBits == RHS->getSizeInBits() &&
+           SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() &&
            Encoding == RHS->getEncoding();
   }
@@ -609,8 +611,8 @@ template <> struct MDNodeKeyImpl<DIDerivedType> {
   unsigned Line;
   Metadata *Scope;
   Metadata *BaseType;
-  uint64_t SizeInBits;
-  uint64_t OffsetInBits;
+  Metadata *SizeInBits;
+  Metadata *OffsetInBits;
   uint32_t AlignInBits;
   std::optional<unsigned> DWARFAddressSpace;
   dwarf::MemorySpace DWARFMemorySpace;
@@ -620,8 +622,8 @@ template <> struct MDNodeKeyImpl<DIDerivedType> {
   Metadata *Annotations;
 
   MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
-                Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
-                uint32_t AlignInBits, uint64_t OffsetInBits,
+                Metadata *Scope, Metadata *BaseType, Metadata *SizeInBits,
+                uint32_t AlignInBits, Metadata *OffsetInBits,
                 std::optional<unsigned> DWARFAddressSpace,
                 dwarf::MemorySpace DWARFMemorySpace, std::optional<DIDerivedType::PtrAuthData> PtrAuthData,
                 unsigned Flags, Metadata *ExtraData, Metadata *Annotations)
@@ -633,9 +635,9 @@ template <> struct MDNodeKeyImpl<DIDerivedType> {
   MDNodeKeyImpl(const DIDerivedType *N)
       : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()),
         Line(N->getLine()), Scope(N->getRawScope()),
-        BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()),
-        OffsetInBits(N->getOffsetInBits()), AlignInBits(N->getAlignInBits()),
-        DWARFAddressSpace(N->getDWARFAddressSpace()), DWARFMemorySpace(N->getDWARFMemorySpace()), 
+        BaseType(N->getRawBaseType()), SizeInBits(N->getRawSizeInBits()),
+        OffsetInBits(N->getRawOffsetInBits()), AlignInBits(N->getAlignInBits()),
+        DWARFAddressSpace(N->getDWARFAddressSpace()), DWARFMemorySpace(N->getDWARFMemorySpace()),
         PtrAuthData(N->getPtrAuthData()), Flags(N->getFlags()),
         ExtraData(N->getRawExtraData()), Annotations(N->getRawAnnotations()) {}
 
@@ -643,9 +645,9 @@ template <> struct MDNodeKeyImpl<DIDerivedType> {
     return Tag == RHS->getTag() && Name == RHS->getRawName() &&
            File == RHS->getRawFile() && Line == RHS->getLine() &&
            Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() &&
-           SizeInBits == RHS->getSizeInBits() &&
+           SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() &&
-           OffsetInBits == RHS->getOffsetInBits() &&
+           OffsetInBits == RHS->getRawOffsetInBits() &&
            DWARFAddressSpace == RHS->getDWARFAddressSpace() &&
            DWARFMemorySpace == RHS->getDWARFMemorySpace() && PtrAuthData == RHS->getPtrAuthData() &&
            Flags == RHS->getFlags() && ExtraData == RHS->getRawExtraData() &&
@@ -674,7 +676,7 @@ template <> struct MDNodeKeyImpl<DISubrangeType> {
   Metadata *File;
   unsigned Line;
   Metadata *Scope;
-  uint64_t SizeInBits;
+  Metadata *SizeInBits;
   uint32_t AlignInBits;
   unsigned Flags;
   Metadata *BaseType;
@@ -684,7 +686,7 @@ template <> struct MDNodeKeyImpl<DISubrangeType> {
   Metadata *Bias;
 
   MDNodeKeyImpl(MDString *Name, Metadata *File, unsigned Line, Metadata *Scope,
-                uint64_t SizeInBits, uint32_t AlignInBits, unsigned Flags,
+                Metadata *SizeInBits, uint32_t AlignInBits, unsigned Flags,
                 Metadata *BaseType, Metadata *LowerBound, Metadata *UpperBound,
                 Metadata *Stride, Metadata *Bias)
       : Name(Name), File(File), Line(Line), Scope(Scope),
@@ -693,7 +695,7 @@ template <> struct MDNodeKeyImpl<DISubrangeType> {
         Stride(Stride), Bias(Bias) {}
   MDNodeKeyImpl(const DISubrangeType *N)
       : Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()),
-        Scope(N->getRawScope()), SizeInBits(N->getSizeInBits()),
+        Scope(N->getRawScope()), SizeInBits(N->getRawSizeInBits()),
         AlignInBits(N->getAlignInBits()), Flags(N->getFlags()),
         BaseType(N->getRawBaseType()), LowerBound(N->getRawLowerBound()),
         UpperBound(N->getRawUpperBound()), Stride(N->getRawStride()),
@@ -717,7 +719,7 @@ template <> struct MDNodeKeyImpl<DISubrangeType> {
 
     return Name == RHS->getRawName() && File == RHS->getRawFile() &&
            Line == RHS->getLine() && Scope == RHS->getRawScope() &&
-           SizeInBits == RHS->getSizeInBits() &&
+           SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() && Flags == RHS->getFlags() &&
            BaseType == RHS->getRawBaseType() &&
            BoundsEqual(LowerBound, RHS->getRawLowerBound()) &&
@@ -785,8 +787,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
   unsigned Line;
   Metadata *Scope;
   Metadata *BaseType;
-  uint64_t SizeInBits;
-  uint64_t OffsetInBits;
+  Metadata *SizeInBits;
+  Metadata *OffsetInBits;
   uint32_t AlignInBits;
   unsigned Flags;
   Metadata *Elements;
@@ -805,8 +807,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
   Metadata *BitStride;
 
   MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line,
-                Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits,
-                uint32_t AlignInBits, uint64_t OffsetInBits, unsigned Flags,
+                Metadata *Scope, Metadata *BaseType, Metadata *SizeInBits,
+                uint32_t AlignInBits, Metadata *OffsetInBits, unsigned Flags,
                 Metadata *Elements, unsigned RuntimeLang,
                 Metadata *VTableHolder, Metadata *TemplateParams,
                 MDString *Identifier, Metadata *Discriminator,
@@ -826,8 +828,8 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
   MDNodeKeyImpl(const DICompositeType *N)
       : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()),
         Line(N->getLine()), Scope(N->getRawScope()),
-        BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()),
-        OffsetInBits(N->getOffsetInBits()), AlignInBits(N->getAlignInBits()),
+        BaseType(N->getRawBaseType()), SizeInBits(N->getRawSizeInBits()),
+        OffsetInBits(N->getRawOffsetInBits()), AlignInBits(N->getAlignInBits()),
         Flags(N->getFlags()), Elements(N->getRawElements()),
         RuntimeLang(N->getRuntimeLang()), VTableHolder(N->getRawVTableHolder()),
         TemplateParams(N->getRawTemplateParams()),
@@ -844,10 +846,10 @@ template <> struct MDNodeKeyImpl<DICompositeType> {
     return Tag == RHS->getTag() && Name == RHS->getRawName() &&
            File == RHS->getRawFile() && Line == RHS->getLine() &&
            Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() &&
-           SizeInBits == RHS->getSizeInBits() &&
+           SizeInBits == RHS->getRawSizeInBits() &&
            AlignInBits == RHS->getAlignInBits() &&
-           OffsetInBits == RHS->getOffsetInBits() && Flags == RHS->getFlags() &&
-           Elements == RHS->getRawElements() &&
+           OffsetInBits == RHS->getRawOffsetInBits() &&
+           Flags == RHS->getFlags() && Elements == RHS->getRawElements() &&
            RuntimeLang == RHS->getRuntimeLang() &&
            VTableHolder == RHS->getRawVTableHolder() &&
            TemplateParams == RHS->getRawTemplateParams() &&
@@ -923,8 +925,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
   MDString *LinkageName;
   Metadata *File;
   unsigned Line;
-  Metadata *Type;
   unsigned ScopeLine;
+  Metadata *Type;
   Metadata *ContainingType;
   unsigned VirtualIndex;
   int ThisAdjustment;
@@ -937,6 +939,7 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
   Metadata *ThrownTypes;
   Metadata *Annotations;
   MDString *TargetFuncName;
+  bool UsesKeyInstructions;
 
   MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *LinkageName,
                 Metadata *File, unsigned Line, Metadata *Type,
@@ -945,18 +948,19 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
                 unsigned SPFlags, Metadata *Unit, Metadata *TemplateParams,
                 Metadata *Declaration, Metadata *RetainedNodes,
                 Metadata *ThrownTypes, Metadata *Annotations,
-                MDString *TargetFuncName)
+                MDString *TargetFuncName, bool UsesKeyInstructions)
       : Scope(Scope), Name(Name), LinkageName(LinkageName), File(File),
-        Line(Line), Type(Type), ScopeLine(ScopeLine),
+        Line(Line), ScopeLine(ScopeLine), Type(Type),
         ContainingType(ContainingType), VirtualIndex(VirtualIndex),
         ThisAdjustment(ThisAdjustment), Flags(Flags), SPFlags(SPFlags),
         Unit(Unit), TemplateParams(TemplateParams), Declaration(Declaration),
         RetainedNodes(RetainedNodes), ThrownTypes(ThrownTypes),
-        Annotations(Annotations), TargetFuncName(TargetFuncName) {}
+        Annotations(Annotations), TargetFuncName(TargetFuncName),
+        UsesKeyInstructions(UsesKeyInstructions) {}
   MDNodeKeyImpl(const DISubprogram *N)
       : Scope(N->getRawScope()), Name(N->getRawName()),
         LinkageName(N->getRawLinkageName()), File(N->getRawFile()),
-        Line(N->getLine()), Type(N->getRawType()), ScopeLine(N->getScopeLine()),
+        Line(N->getLine()), ScopeLine(N->getScopeLine()), Type(N->getRawType()),
         ContainingType(N->getRawContainingType()),
         VirtualIndex(N->getVirtualIndex()),
         ThisAdjustment(N->getThisAdjustment()), Flags(N->getFlags()),
@@ -966,7 +970,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
         RetainedNodes(N->getRawRetainedNodes()),
         ThrownTypes(N->getRawThrownTypes()),
         Annotations(N->getRawAnnotations()),
-        TargetFuncName(N->getRawTargetFuncName()) {}
+        TargetFuncName(N->getRawTargetFuncName()),
+        UsesKeyInstructions(N->getKeyInstructionsEnabled()) {}
 
   bool isKeyOf(const DISubprogram *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getRawName() &&
@@ -983,7 +988,8 @@ template <> struct MDNodeKeyImpl<DISubprogram> {
            RetainedNodes == RHS->getRawRetainedNodes() &&
            ThrownTypes == RHS->getRawThrownTypes() &&
            Annotations == RHS->getRawAnnotations() &&
-           TargetFuncName == RHS->getRawTargetFuncName();
+           TargetFuncName == RHS->getRawTargetFuncName() &&
+           UsesKeyInstructions == RHS->getKeyInstructionsEnabled();
   }
 
   bool isDefinition() const { return SPFlags & DISubprogram::SPFlagDefinition; }
@@ -1338,20 +1344,33 @@ template <> struct MDNodeKeyImpl<DILabel> {
   MDString *Name;
   Metadata *File;
   unsigned Line;
+  unsigned Column;
+  bool IsArtificial;
+  std::optional<unsigned> CoroSuspendIdx;
 
-  MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line)
-      : Scope(Scope), Name(Name), File(File), Line(Line) {}
+  MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line,
+                unsigned Column, bool IsArtificial,
+                std::optional<unsigned> CoroSuspendIdx)
+      : Scope(Scope), Name(Name), File(File), Line(Line), Column(Column),
+        IsArtificial(IsArtificial), CoroSuspendIdx(CoroSuspendIdx) {}
   MDNodeKeyImpl(const DILabel *N)
       : Scope(N->getRawScope()), Name(N->getRawName()), File(N->getRawFile()),
-        Line(N->getLine()) {}
+        Line(N->getLine()), Column(N->getColumn()),
+        IsArtificial(N->isArtificial()),
+        CoroSuspendIdx(N->getCoroSuspendIdx()) {}
 
   bool isKeyOf(const DILabel *RHS) const {
     return Scope == RHS->getRawScope() && Name == RHS->getRawName() &&
-           File == RHS->getRawFile() && Line == RHS->getLine();
+           File == RHS->getRawFile() && Line == RHS->getLine() &&
+           Column == RHS->getColumn() && IsArtificial == RHS->isArtificial() &&
+           CoroSuspendIdx == RHS->getCoroSuspendIdx();
   }
 
   /// Using name and line to get hash value. It should already be mostly unique.
-  unsigned getHashValue() const { return hash_combine(Scope, Name, Line); }
+  unsigned getHashValue() const {
+    return hash_combine(Scope, Name, Line, Column, IsArtificial,
+                        CoroSuspendIdx);
+  }
 };
 
 template <> struct MDNodeKeyImpl<DIExpression> {
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index c8f1606ea06c..1d9a61ce4a10 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -120,7 +120,7 @@ void PMDataManager::emitInstrCountChangedRemark(
   // If no function was passed in, then we're either a module pass or an
   // CGSCC pass.
   if (!CouldOnlyImpactOneFunction)
-    std::for_each(M.begin(), M.end(), UpdateFunctionChanges);
+    llvm::for_each(M, UpdateFunctionChanges);
   else
     UpdateFunctionChanges(*F);
 
@@ -197,9 +197,7 @@ void PMDataManager::emitInstrCountChangedRemark(
   // Are we looking at more than one function? If so, emit remarks for all of
   // the functions in the module. Otherwise, only emit one remark.
   if (!CouldOnlyImpactOneFunction)
-    std::for_each(FunctionToInstrCount.keys().begin(),
-                  FunctionToInstrCount.keys().end(),
-                  EmitFunctionSizeChangedRemark);
+    llvm::for_each(FunctionToInstrCount.keys(), EmitFunctionSizeChangedRemark);
   else
     EmitFunctionSizeChangedRemark(F->getName().str());
 }
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index b6aa8844a7ea..893f99a19b93 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -15,6 +15,7 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/IR/ProfDataUtils.h"
 using namespace llvm;
 
 MDString *MDBuilder::createString(StringRef Str) {
@@ -55,9 +56,9 @@ MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights,
 
   unsigned int Offset = IsExpected ? 2 : 1;
   SmallVector<Metadata *, 4> Vals(Weights.size() + Offset);
-  Vals[0] = createString("branch_weights");
+  Vals[0] = createString(MDProfLabels::BranchWeights);
   if (IsExpected)
-    Vals[1] = createString("expected");
+    Vals[1] = createString(MDProfLabels::ExpectedBranchWeights);
 
   Type *Int32Ty = Type::getInt32Ty(Context);
   for (unsigned i = 0, e = Weights.size(); i != e; ++i)
@@ -74,9 +75,9 @@ MDNode *MDBuilder::createFunctionEntryCount(
   Type *Int64Ty = Type::getInt64Ty(Context);
   SmallVector<Metadata *, 8> Ops;
   if (Synthetic)
-    Ops.push_back(createString("synthetic_function_entry_count"));
+    Ops.push_back(createString(MDProfLabels::SyntheticFunctionEntryCount));
   else
-    Ops.push_back(createString("function_entry_count"));
+    Ops.push_back(createString(MDProfLabels::FunctionEntryCount));
   Ops.push_back(createConstant(ConstantInt::get(Int64Ty, Count)));
   if (Imports) {
     SmallVector<GlobalValue::GUID, 2> OrderID(Imports->begin(), Imports->end());
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index cbb6da23dae0..46708e26f318 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -699,7 +699,7 @@ MDNode::Header::~Header() {
   }
   MDOperand *O = reinterpret_cast<MDOperand *>(this);
   for (MDOperand *E = O - SmallSize; O != E; --O)
-    (void)(O - 1)->~MDOperand();
+    (O - 1)->~MDOperand();
 }
 
 void *MDNode::Header::getSmallPtr() {
@@ -1202,14 +1202,15 @@ MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
          "first operand should be a non-null MDString");
   StringRef AProfName = AMDS->getString();
   StringRef BProfName = BMDS->getString();
-  if (AProfName == "branch_weights" && BProfName == "branch_weights") {
+  if (AProfName == MDProfLabels::BranchWeights &&
+      BProfName == MDProfLabels::BranchWeights) {
     ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
         A->getOperand(getBranchWeightOffset(A)));
     ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
         B->getOperand(getBranchWeightOffset(B)));
     assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
     return MDNode::get(Ctx,
-                       {MDHelper.createString("branch_weights"),
+                       {MDHelper.createString(MDProfLabels::BranchWeights),
                         MDHelper.createConstant(ConstantInt::get(
                             Type::getInt64Ty(Ctx),
                             SaturatingAdd(AInstrWeight->getZExtValue(),
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 2d31481f62c6..70d364176062 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -119,26 +119,30 @@ Module::~Module() {
 }
 
 void Module::removeDebugIntrinsicDeclarations() {
-  auto *DeclareIntrinsicFn =
-      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_declare);
-  assert((!isMaterialized() || DeclareIntrinsicFn->hasZeroLiveUses()) &&
-         "Debug declare intrinsic should have had uses removed.");
-  DeclareIntrinsicFn->eraseFromParent();
-  auto *ValueIntrinsicFn =
-      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_value);
-  assert((!isMaterialized() || ValueIntrinsicFn->hasZeroLiveUses()) &&
-         "Debug value intrinsic should have had uses removed.");
-  ValueIntrinsicFn->eraseFromParent();
-  auto *AssignIntrinsicFn =
-      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_assign);
-  assert((!isMaterialized() || AssignIntrinsicFn->hasZeroLiveUses()) &&
-         "Debug assign intrinsic should have had uses removed.");
-  AssignIntrinsicFn->eraseFromParent();
-  auto *LabelntrinsicFn =
-      Intrinsic::getOrInsertDeclaration(this, Intrinsic::dbg_label);
-  assert((!isMaterialized() || LabelntrinsicFn->hasZeroLiveUses()) &&
-         "Debug label intrinsic should have had uses removed.");
-  LabelntrinsicFn->eraseFromParent();
+  if (auto *DeclareIntrinsicFn =
+          Intrinsic::getDeclarationIfExists(this, Intrinsic::dbg_declare)) {
+    assert((!isMaterialized() || DeclareIntrinsicFn->hasZeroLiveUses()) &&
+           "Debug declare intrinsic should have had uses removed.");
+    DeclareIntrinsicFn->eraseFromParent();
+  }
+  if (auto *ValueIntrinsicFn =
+          Intrinsic::getDeclarationIfExists(this, Intrinsic::dbg_value)) {
+    assert((!isMaterialized() || ValueIntrinsicFn->hasZeroLiveUses()) &&
+           "Debug value intrinsic should have had uses removed.");
+    ValueIntrinsicFn->eraseFromParent();
+  }
+  if (auto *AssignIntrinsicFn =
+          Intrinsic::getDeclarationIfExists(this, Intrinsic::dbg_assign)) {
+    assert((!isMaterialized() || AssignIntrinsicFn->hasZeroLiveUses()) &&
+           "Debug assign intrinsic should have had uses removed.");
+    AssignIntrinsicFn->eraseFromParent();
+  }
+  if (auto *LabelntrinsicFn =
+          Intrinsic::getDeclarationIfExists(this, Intrinsic::dbg_label)) {
+    assert((!isMaterialized() || LabelntrinsicFn->hasZeroLiveUses()) &&
+           "Debug label intrinsic should have had uses removed.");
+    LabelntrinsicFn->eraseFromParent();
+  }
 }
 
 std::unique_ptr<RandomNumberGenerator>
diff --git a/llvm/lib/IR/OptBisect.cpp b/llvm/lib/IR/OptBisect.cpp
index 559b19944536..427e8b78fd03 100644
--- a/llvm/lib/IR/OptBisect.cpp
+++ b/llvm/lib/IR/OptBisect.cpp
@@ -37,15 +37,15 @@ static cl::opt<bool> OptBisectVerbose(
     cl::desc("Show verbose output when opt-bisect-limit is set"), cl::Hidden,
     cl::init(true), cl::Optional);
 
-static void printPassMessage(const StringRef &Name, int PassNum,
-                             StringRef TargetDesc, bool Running) {
+static void printPassMessage(StringRef Name, int PassNum, StringRef TargetDesc,
+                             bool Running) {
   StringRef Status = Running ? "" : "NOT ";
-  errs() << "BISECT: " << Status << "running pass "
-         << "(" << PassNum << ") " << Name << " on " << TargetDesc << "\n";
+  errs() << "BISECT: " << Status << "running pass (" << PassNum << ") " << Name
+         << " on " << TargetDesc << '\n';
 }
 
-bool OptBisect::shouldRunPass(const StringRef PassName,
-                              StringRef IRDescription) {
+bool OptBisect::shouldRunPass(StringRef PassName,
+                              StringRef IRDescription) const {
   assert(isEnabled());
 
   int CurBisectNum = ++LastBisectNum;
@@ -55,6 +55,4 @@ bool OptBisect::shouldRunPass(const StringRef PassName,
   return ShouldRun;
 }
 
-const int OptBisect::Disabled;
-
 OptPassGate &llvm::getGlobalPassGate() { return getOptBisector(); }
diff --git a/llvm/lib/IR/Pass.cpp b/llvm/lib/IR/Pass.cpp
index 5b0a7e90445e..c21da0ed10b6 100644
--- a/llvm/lib/IR/Pass.cpp
+++ b/llvm/lib/IR/Pass.cpp
@@ -60,8 +60,8 @@ static std::string getDescription(const Module &M) {
   return "module (" + M.getName().str() + ")";
 }
 
-bool ModulePass::skipModule(Module &M) const {
-  OptPassGate &Gate = M.getContext().getOptPassGate();
+bool ModulePass::skipModule(const Module &M) const {
+  const OptPassGate &Gate = M.getContext().getOptPassGate();
   return Gate.isEnabled() &&
          !Gate.shouldRunPass(this->getPassName(), getDescription(M));
 }
@@ -107,10 +107,6 @@ void Pass::verifyAnalysis() const {
   // By default, don't do anything.
 }
 
-void *Pass::getAdjustedAnalysisPointer(AnalysisID AID) {
-  return this;
-}
-
 ImmutablePass *Pass::getAsImmutablePass() {
   return nullptr;
 }
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index 21524eb84053..b1b5f67689e6 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -88,16 +88,24 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
 
 namespace llvm {
 
+const char *MDProfLabels::BranchWeights = "branch_weights";
+const char *MDProfLabels::ExpectedBranchWeights = "expected";
+const char *MDProfLabels::ValueProfile = "VP";
+const char *MDProfLabels::FunctionEntryCount = "function_entry_count";
+const char *MDProfLabels::SyntheticFunctionEntryCount =
+    "synthetic_function_entry_count";
+const char *MDProfLabels::UnknownBranchWeightsMarker = "unknown";
+
 bool hasProfMD(const Instruction &I) {
   return I.hasMetadata(LLVMContext::MD_prof);
 }
 
 bool isBranchWeightMD(const MDNode *ProfileData) {
-  return isTargetMD(ProfileData, "branch_weights", MinBWOps);
+  return isTargetMD(ProfileData, MDProfLabels::BranchWeights, MinBWOps);
 }
 
-static bool isValueProfileMD(const MDNode *ProfileData) {
-  return isTargetMD(ProfileData, "VP", MinVPOps);
+bool isValueProfileMD(const MDNode *ProfileData) {
+  return isTargetMD(ProfileData, MDProfLabels::ValueProfile, MinVPOps);
 }
 
 bool hasBranchWeightMD(const Instruction &I) {
@@ -131,7 +139,8 @@ bool hasBranchWeightOrigin(const MDNode *ProfileData) {
   // NOTE: if we ever have more types of branch weight provenance,
   // we need to check the string value is "expected". For now, we
   // supply a more generic API, and avoid the spurious comparisons.
-  assert(ProfDataName == nullptr || ProfDataName->getString() == "expected");
+  assert(ProfDataName == nullptr ||
+         ProfDataName->getString() == MDProfLabels::ExpectedBranchWeights);
   return ProfDataName != nullptr;
 }
 
@@ -210,7 +219,7 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
   if (!ProfDataName)
     return false;
 
-  if (ProfDataName->getString() == "branch_weights") {
+  if (ProfDataName->getString() == MDProfLabels::BranchWeights) {
     unsigned Offset = getBranchWeightOffset(ProfileData);
     for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) {
       auto *V = mdconst::extract<ConstantInt>(ProfileData->getOperand(Idx));
@@ -219,7 +228,8 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
     return true;
   }
 
-  if (ProfDataName->getString() == "VP" && ProfileData->getNumOperands() > 3) {
+  if (ProfDataName->getString() == MDProfLabels::ValueProfile &&
+      ProfileData->getNumOperands() > 3) {
     TotalVal = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2))
                    ->getValue()
                    .getZExtValue();
@@ -232,6 +242,27 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) {
   return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
 }
 
+void setExplicitlyUnknownBranchWeights(Instruction &I) {
+  MDBuilder MDB(I.getContext());
+  I.setMetadata(
+      LLVMContext::MD_prof,
+      MDNode::get(I.getContext(),
+                  MDB.createString(MDProfLabels::UnknownBranchWeightsMarker)));
+}
+
+bool isExplicitlyUnknownBranchWeightsMetadata(const MDNode &MD) {
+  if (MD.getNumOperands() != 1)
+    return false;
+  return MD.getOperand(0).equalsStr(MDProfLabels::UnknownBranchWeightsMarker);
+}
+
+bool hasExplicitlyUnknownBranchWeights(const Instruction &I) {
+  auto *MD = I.getMetadata(LLVMContext::MD_prof);
+  if (!MD)
+    return false;
+  return isExplicitlyUnknownBranchWeightsMetadata(*MD);
+}
+
 void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
                       bool IsExpected) {
   MDBuilder MDB(I.getContext());
@@ -246,8 +277,9 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
     return;
 
   auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));
-  if (!ProfDataName || (ProfDataName->getString() != "branch_weights" &&
-                        ProfDataName->getString() != "VP"))
+  if (!ProfDataName ||
+      (ProfDataName->getString() != MDProfLabels::BranchWeights &&
+       ProfDataName->getString() != MDProfLabels::ValueProfile))
     return;
 
   if (!hasCountTypeMD(I))
@@ -259,7 +291,7 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
   SmallVector<Metadata *, 3> Vals;
   Vals.push_back(ProfileData->getOperand(0));
   APInt APS(128, S), APT(128, T);
-  if (ProfDataName->getString() == "branch_weights" &&
+  if (ProfDataName->getString() == MDProfLabels::BranchWeights &&
       ProfileData->getNumOperands() > 0) {
     // Using APInt::div may be expensive, but most cases should fit 64 bits.
     APInt Val(128,
@@ -270,7 +302,7 @@ void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
     Val *= APS;
     Vals.push_back(MDB.createConstant(ConstantInt::get(
         Type::getInt32Ty(C), Val.udiv(APT).getLimitedValue(UINT32_MAX))));
-  } else if (ProfDataName->getString() == "VP")
+  } else if (ProfDataName->getString() == MDProfLabels::ValueProfile)
     for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx += 2) {
       // The first value is the key of the value profile, which will not change.
       Vals.push_back(ProfileData->getOperand(Idx));
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 702e0a51357f..712f1a48d0b7 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -12,6 +12,12 @@
 using namespace llvm;
 using namespace RTLIB;
 
+#define GET_INIT_RUNTIME_LIBCALL_UTILS
+#define GET_INIT_RUNTIME_LIBCALL_NAMES
+#include "llvm/IR/RuntimeLibcalls.inc"
+#undef GET_INIT_RUNTIME_LIBCALL_UTILS
+#undef GET_INIT_RUNTIME_LIBCALL_NAMES
+
 static cl::opt<bool>
     HexagonEnableFastMathRuntimeCalls("hexagon-fast-math", cl::Hidden,
                                       cl::desc("Enable Fast Math processing"));
@@ -19,10 +25,10 @@ static cl::opt<bool>
 static void setAArch64LibcallNames(RuntimeLibcallsInfo &Info,
                                    const Triple &TT) {
 #define LCALLNAMES(A, B, N)                                                    \
-  Info.setLibcallName(A##N##_RELAX, #B #N "_relax");                           \
-  Info.setLibcallName(A##N##_ACQ, #B #N "_acq");                               \
-  Info.setLibcallName(A##N##_REL, #B #N "_rel");                               \
-  Info.setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
+  Info.setLibcallImpl(A##N##_RELAX, B##N##_relax);                             \
+  Info.setLibcallImpl(A##N##_ACQ, B##N##_acq);                                 \
+  Info.setLibcallImpl(A##N##_REL, B##N##_rel);                                 \
+  Info.setLibcallImpl(A##N##_ACQ_REL, B##N##_acq_rel);
 #define LCALLNAME4(A, B)                                                       \
   LCALLNAMES(A, B, 1)                                                          \
   LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
@@ -30,36 +36,22 @@ static void setAArch64LibcallNames(RuntimeLibcallsInfo &Info,
   LCALLNAMES(A, B, 1)                                                          \
   LCALLNAMES(A, B, 2)                                                          \
   LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
-  LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
-  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
-  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
-  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
-  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
-  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
 
   if (TT.isWindowsArm64EC()) {
-    // FIXME: are there calls we need to exclude from this?
-#define HANDLE_LIBCALL(code, name)                                             \
-  if (sizeof(name) != 1) {                                                     \
-    const char *libcallName = Info.getLibcallName(RTLIB::code);                \
-    if (libcallName && libcallName[0] != '#') {                                \
-      assert(strcmp(libcallName, name) == 0 && "Unexpected name");             \
-      Info.setLibcallName(RTLIB::code, "#" name);                              \
-    }                                                                          \
-  }
-#define LIBCALL_NO_NAME ""
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-#undef LIBCALL_NO_NAME
-
-    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, #__aarch64_cas)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, #__aarch64_swp)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, #__aarch64_ldadd)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, #__aarch64_ldset)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, #__aarch64_ldclr)
-    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, #__aarch64_ldeor)
+    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, RTLIB::arm64ec___aarch64_cas)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, RTLIB::arm64ec___aarch64_swp)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, RTLIB::arm64ec___aarch64_ldadd)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, RTLIB::arm64ec___aarch64_ldset)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, RTLIB::arm64ec___aarch64_ldclr)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, RTLIB::arm64ec___aarch64_ldeor)
+  } else {
+    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, RTLIB::__aarch64_cas)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, RTLIB::__aarch64_swp)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, RTLIB::__aarch64_ldadd)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, RTLIB::__aarch64_ldset)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, RTLIB::__aarch64_ldclr)
+    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, RTLIB::__aarch64_ldeor)
   }
-
 #undef LCALLNAMES
 #undef LCALLNAME4
 #undef LCALLNAME5
@@ -82,33 +74,33 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
     if (TT.isOSWindows()) {
       const struct {
         const RTLIB::Libcall Op;
-        const char *const Name;
+        const RTLIB::LibcallImpl Impl;
         const CallingConv::ID CC;
       } LibraryCalls[] = {
-          {RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS},
-          {RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS},
-          {RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS},
-          {RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I32, RTLIB::__rt_sdiv, CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I64, RTLIB::__rt_sdiv64, CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I32, RTLIB::__rt_udiv, CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I64, RTLIB::__rt_udiv64, CallingConv::ARM_AAPCS},
       };
 
       for (const auto &LC : LibraryCalls) {
-        Info.setLibcallName(LC.Op, LC.Name);
+        Info.setLibcallImpl(LC.Op, LC.Impl);
         Info.setLibcallCallingConv(LC.Op, LC.CC);
       }
     } else {
       const struct {
         const RTLIB::Libcall Op;
-        const char *const Name;
+        const RTLIB::LibcallImpl Impl;
         const CallingConv::ID CC;
       } LibraryCalls[] = {
-          {RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS},
-          {RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS},
-          {RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS},
-          {RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I32, RTLIB::__aeabi_idivmod, CallingConv::ARM_AAPCS},
+          {RTLIB::SDIVREM_I64, RTLIB::__aeabi_ldivmod, CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I32, RTLIB::__aeabi_uidivmod, CallingConv::ARM_AAPCS},
+          {RTLIB::UDIVREM_I64, RTLIB::__aeabi_uldivmod, CallingConv::ARM_AAPCS},
       };
 
       for (const auto &LC : LibraryCalls) {
-        Info.setLibcallName(LC.Op, LC.Name);
+        Info.setLibcallImpl(LC.Op, LC.Impl);
         Info.setLibcallCallingConv(LC.Op, LC.CC);
       }
     }
@@ -117,29 +109,29 @@ static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
   if (TT.isOSWindows()) {
     static const struct {
       const RTLIB::Libcall Op;
-      const char *const Name;
+      const RTLIB::LibcallImpl Impl;
       const CallingConv::ID CC;
     } LibraryCalls[] = {
-        {RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP},
-        {RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOSINT_F32_I64, RTLIB::__stoi64, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOSINT_F64_I64, RTLIB::__dtoi64, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOUINT_F32_I64, RTLIB::__stou64, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::FPTOUINT_F64_I64, RTLIB::__dtou64, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::SINTTOFP_I64_F32, RTLIB::__i64tos, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::SINTTOFP_I64_F64, RTLIB::__i64tod, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::UINTTOFP_I64_F32, RTLIB::__u64tos, CallingConv::ARM_AAPCS_VFP},
+        {RTLIB::UINTTOFP_I64_F64, RTLIB::__u64tod, CallingConv::ARM_AAPCS_VFP},
     };
 
     for (const auto &LC : LibraryCalls) {
-      Info.setLibcallName(LC.Op, LC.Name);
+      Info.setLibcallImpl(LC.Op, LC.Impl);
       Info.setLibcallCallingConv(LC.Op, LC.CC);
     }
   }
 
   // Use divmod compiler-rt calls for iOS 5.0 and later.
   if (TT.isOSBinFormatMachO() && (!TT.isiOS() || !TT.isOSVersionLT(5, 0))) {
-    Info.setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
-    Info.setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
+    Info.setLibcallImpl(RTLIB::SDIVREM_I32, RTLIB::__divmodsi4);
+    Info.setLibcallImpl(RTLIB::UDIVREM_I32, RTLIB::__udivmodsi4);
   }
 }
 
@@ -147,99 +139,99 @@ static void setMSP430Libcalls(RuntimeLibcallsInfo &Info, const Triple &TT) {
   // EABI Libcalls - EABI Section 6.2
   const struct {
     const RTLIB::Libcall Op;
-    const char *const Name;
+    const RTLIB::LibcallImpl Impl;
   } LibraryCalls[] = {
       // Floating point conversions - EABI Table 6
-      {RTLIB::FPROUND_F64_F32, "__mspabi_cvtdf"},
-      {RTLIB::FPEXT_F32_F64, "__mspabi_cvtfd"},
+      {RTLIB::FPROUND_F64_F32, RTLIB::__mspabi_cvtdf},
+      {RTLIB::FPEXT_F32_F64, RTLIB::__mspabi_cvtfd},
       // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOSINT_F64_I16,  "__mspabi_fixdi" },
-      {RTLIB::FPTOSINT_F64_I32, "__mspabi_fixdli"},
-      {RTLIB::FPTOSINT_F64_I64, "__mspabi_fixdlli"},
+      //{ RTLIB::FPTOSINT_F64_I16,  RTLIB::__mspabi_fixdi },
+      {RTLIB::FPTOSINT_F64_I32, RTLIB::__mspabi_fixdli},
+      {RTLIB::FPTOSINT_F64_I64, RTLIB::__mspabi_fixdlli},
       // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOUINT_F64_I16,  "__mspabi_fixdu" },
-      {RTLIB::FPTOUINT_F64_I32, "__mspabi_fixdul"},
-      {RTLIB::FPTOUINT_F64_I64, "__mspabi_fixdull"},
+      //{ RTLIB::FPTOUINT_F64_I16,  RTLIB::__mspabi_fixdu },
+      {RTLIB::FPTOUINT_F64_I32, RTLIB::__mspabi_fixdul},
+      {RTLIB::FPTOUINT_F64_I64, RTLIB::__mspabi_fixdull},
       // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOSINT_F32_I16,  "__mspabi_fixfi" },
-      {RTLIB::FPTOSINT_F32_I32, "__mspabi_fixfli"},
-      {RTLIB::FPTOSINT_F32_I64, "__mspabi_fixflli"},
+      //{ RTLIB::FPTOSINT_F32_I16,  RTLIB::__mspabi_fixfi },
+      {RTLIB::FPTOSINT_F32_I32, RTLIB::__mspabi_fixfli},
+      {RTLIB::FPTOSINT_F32_I64, RTLIB::__mspabi_fixflli},
       // The following is NOT implemented in libgcc
-      //{ RTLIB::FPTOUINT_F32_I16,  "__mspabi_fixfu" },
-      {RTLIB::FPTOUINT_F32_I32, "__mspabi_fixful"},
-      {RTLIB::FPTOUINT_F32_I64, "__mspabi_fixfull"},
+      //{ RTLIB::FPTOUINT_F32_I16,  RTLIB::__mspabi_fixfu },
+      {RTLIB::FPTOUINT_F32_I32, RTLIB::__mspabi_fixful},
+      {RTLIB::FPTOUINT_F32_I64, RTLIB::__mspabi_fixfull},
       // TODO The following IS implemented in libgcc
-      //{ RTLIB::SINTTOFP_I16_F64,  "__mspabi_fltid" },
-      {RTLIB::SINTTOFP_I32_F64, "__mspabi_fltlid"},
+      //{ RTLIB::SINTTOFP_I16_F64,  RTLIB::__mspabi_fltid },
+      {RTLIB::SINTTOFP_I32_F64, RTLIB::__mspabi_fltlid},
       // TODO The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::SINTTOFP_I64_F64, "__mspabi_fltllid"},
+      {RTLIB::SINTTOFP_I64_F64, RTLIB::__mspabi_fltllid},
       // TODO The following IS implemented in libgcc
-      //{ RTLIB::UINTTOFP_I16_F64,  "__mspabi_fltud" },
-      {RTLIB::UINTTOFP_I32_F64, "__mspabi_fltuld"},
+      //{ RTLIB::UINTTOFP_I16_F64,  RTLIB::__mspabi_fltud },
+      {RTLIB::UINTTOFP_I32_F64, RTLIB::__mspabi_fltuld},
       // The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::UINTTOFP_I64_F64, "__mspabi_fltulld"},
+      {RTLIB::UINTTOFP_I64_F64, RTLIB::__mspabi_fltulld},
       // TODO The following IS implemented in libgcc
-      //{ RTLIB::SINTTOFP_I16_F32,  "__mspabi_fltif" },
-      {RTLIB::SINTTOFP_I32_F32, "__mspabi_fltlif"},
+      //{ RTLIB::SINTTOFP_I16_F32,  RTLIB::__mspabi_fltif },
+      {RTLIB::SINTTOFP_I32_F32, RTLIB::__mspabi_fltlif},
       // TODO The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::SINTTOFP_I64_F32, "__mspabi_fltllif"},
+      {RTLIB::SINTTOFP_I64_F32, RTLIB::__mspabi_fltllif},
       // TODO The following IS implemented in libgcc
-      //{ RTLIB::UINTTOFP_I16_F32,  "__mspabi_fltuf" },
-      {RTLIB::UINTTOFP_I32_F32, "__mspabi_fltulf"},
+      //{ RTLIB::UINTTOFP_I16_F32,  RTLIB::__mspabi_fltuf },
+      {RTLIB::UINTTOFP_I32_F32, RTLIB::__mspabi_fltulf},
       // The following IS implemented in libgcc but is not in the EABI
-      {RTLIB::UINTTOFP_I64_F32, "__mspabi_fltullf"},
+      {RTLIB::UINTTOFP_I64_F32, RTLIB::__mspabi_fltullf},
 
       // Floating point comparisons - EABI Table 7
-      {RTLIB::OEQ_F64, "__mspabi_cmpd"},
-      {RTLIB::UNE_F64, "__mspabi_cmpd"},
-      {RTLIB::OGE_F64, "__mspabi_cmpd"},
-      {RTLIB::OLT_F64, "__mspabi_cmpd"},
-      {RTLIB::OLE_F64, "__mspabi_cmpd"},
-      {RTLIB::OGT_F64, "__mspabi_cmpd"},
-      {RTLIB::OEQ_F32, "__mspabi_cmpf"},
-      {RTLIB::UNE_F32, "__mspabi_cmpf"},
-      {RTLIB::OGE_F32, "__mspabi_cmpf"},
-      {RTLIB::OLT_F32, "__mspabi_cmpf"},
-      {RTLIB::OLE_F32, "__mspabi_cmpf"},
-      {RTLIB::OGT_F32, "__mspabi_cmpf"},
+      {RTLIB::OEQ_F64, RTLIB::__mspabi_cmpd__oeq},
+      {RTLIB::UNE_F64, RTLIB::__mspabi_cmpd__une},
+      {RTLIB::OGE_F64, RTLIB::__mspabi_cmpd__oge},
+      {RTLIB::OLT_F64, RTLIB::__mspabi_cmpd__olt},
+      {RTLIB::OLE_F64, RTLIB::__mspabi_cmpd__ole},
+      {RTLIB::OGT_F64, RTLIB::__mspabi_cmpd__ogt},
+      {RTLIB::OEQ_F32, RTLIB::__mspabi_cmpf__oeq},
+      {RTLIB::UNE_F32, RTLIB::__mspabi_cmpf__une},
+      {RTLIB::OGE_F32, RTLIB::__mspabi_cmpf__oge},
+      {RTLIB::OLT_F32, RTLIB::__mspabi_cmpf__olt},
+      {RTLIB::OLE_F32, RTLIB::__mspabi_cmpf__ole},
+      {RTLIB::OGT_F32, RTLIB::__mspabi_cmpf__ogt},
 
       // Floating point arithmetic - EABI Table 8
-      {RTLIB::ADD_F64, "__mspabi_addd"},
-      {RTLIB::ADD_F32, "__mspabi_addf"},
-      {RTLIB::DIV_F64, "__mspabi_divd"},
-      {RTLIB::DIV_F32, "__mspabi_divf"},
-      {RTLIB::MUL_F64, "__mspabi_mpyd"},
-      {RTLIB::MUL_F32, "__mspabi_mpyf"},
-      {RTLIB::SUB_F64, "__mspabi_subd"},
-      {RTLIB::SUB_F32, "__mspabi_subf"},
+      {RTLIB::ADD_F64, RTLIB::__mspabi_addd},
+      {RTLIB::ADD_F32, RTLIB::__mspabi_addf},
+      {RTLIB::DIV_F64, RTLIB::__mspabi_divd},
+      {RTLIB::DIV_F32, RTLIB::__mspabi_divf},
+      {RTLIB::MUL_F64, RTLIB::__mspabi_mpyd},
+      {RTLIB::MUL_F32, RTLIB::__mspabi_mpyf},
+      {RTLIB::SUB_F64, RTLIB::__mspabi_subd},
+      {RTLIB::SUB_F32, RTLIB::__mspabi_subf},
       // The following are NOT implemented in libgcc
-      // { RTLIB::NEG_F64,  "__mspabi_negd" },
-      // { RTLIB::NEG_F32,  "__mspabi_negf" },
+      // { RTLIB::NEG_F64,  RTLIB::__mspabi_negd },
+      // { RTLIB::NEG_F32,  RTLIB::__mspabi_negf },
 
       // Universal Integer Operations - EABI Table 9
-      {RTLIB::SDIV_I16, "__mspabi_divi"},
-      {RTLIB::SDIV_I32, "__mspabi_divli"},
-      {RTLIB::SDIV_I64, "__mspabi_divlli"},
-      {RTLIB::UDIV_I16, "__mspabi_divu"},
-      {RTLIB::UDIV_I32, "__mspabi_divul"},
-      {RTLIB::UDIV_I64, "__mspabi_divull"},
-      {RTLIB::SREM_I16, "__mspabi_remi"},
-      {RTLIB::SREM_I32, "__mspabi_remli"},
-      {RTLIB::SREM_I64, "__mspabi_remlli"},
-      {RTLIB::UREM_I16, "__mspabi_remu"},
-      {RTLIB::UREM_I32, "__mspabi_remul"},
-      {RTLIB::UREM_I64, "__mspabi_remull"},
+      {RTLIB::SDIV_I16, RTLIB::__mspabi_divi},
+      {RTLIB::SDIV_I32, RTLIB::__mspabi_divli},
+      {RTLIB::SDIV_I64, RTLIB::__mspabi_divlli},
+      {RTLIB::UDIV_I16, RTLIB::__mspabi_divu},
+      {RTLIB::UDIV_I32, RTLIB::__mspabi_divul},
+      {RTLIB::UDIV_I64, RTLIB::__mspabi_divull},
+      {RTLIB::SREM_I16, RTLIB::__mspabi_remi},
+      {RTLIB::SREM_I32, RTLIB::__mspabi_remli},
+      {RTLIB::SREM_I64, RTLIB::__mspabi_remlli},
+      {RTLIB::UREM_I16, RTLIB::__mspabi_remu},
+      {RTLIB::UREM_I32, RTLIB::__mspabi_remul},
+      {RTLIB::UREM_I64, RTLIB::__mspabi_remull},
 
       // Bitwise Operations - EABI Table 10
       // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
-      {RTLIB::SRL_I32, "__mspabi_srll"},
-      {RTLIB::SRA_I32, "__mspabi_sral"},
-      {RTLIB::SHL_I32, "__mspabi_slll"},
+      {RTLIB::SRL_I32, RTLIB::__mspabi_srll},
+      {RTLIB::SRA_I32, RTLIB::__mspabi_sral},
+      {RTLIB::SHL_I32, RTLIB::__mspabi_slll},
       // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
   };
 
   for (const auto &LC : LibraryCalls)
-    Info.setLibcallName(LC.Op, LC.Name);
+    Info.setLibcallImpl(LC.Op, LC.Impl);
 
   // Several of the runtime library functions use a special calling conv
   Info.setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::MSP430_BUILTIN);
@@ -293,133 +285,107 @@ void RuntimeLibcallsInfo::initSoftFloatCmpLibcallPredicates() {
 
 static void setLongDoubleIsF128Libm(RuntimeLibcallsInfo &Info,
                                     bool FiniteOnlyFuncs = false) {
-  Info.setLibcallName(RTLIB::REM_F128, "fmodf128");
-  Info.setLibcallName(RTLIB::FMA_F128, "fmaf128");
-  Info.setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-  Info.setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
-  Info.setLibcallName(RTLIB::LOG_F128, "logf128");
-  Info.setLibcallName(RTLIB::LOG2_F128, "log2f128");
-  Info.setLibcallName(RTLIB::LOG10_F128, "log10f128");
-  Info.setLibcallName(RTLIB::EXP_F128, "expf128");
-  Info.setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-  Info.setLibcallName(RTLIB::EXP10_F128, "exp10f128");
-  Info.setLibcallName(RTLIB::SIN_F128, "sinf128");
-  Info.setLibcallName(RTLIB::COS_F128, "cosf128");
-  Info.setLibcallName(RTLIB::TAN_F128, "tanf128");
-  Info.setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-  Info.setLibcallName(RTLIB::ASIN_F128, "asinf128");
-  Info.setLibcallName(RTLIB::ACOS_F128, "acosf128");
-  Info.setLibcallName(RTLIB::ATAN_F128, "atanf128");
-  Info.setLibcallName(RTLIB::ATAN2_F128, "atan2f128");
-  Info.setLibcallName(RTLIB::SINH_F128, "sinhf128");
-  Info.setLibcallName(RTLIB::COSH_F128, "coshf128");
-  Info.setLibcallName(RTLIB::TANH_F128, "tanhf128");
-  Info.setLibcallName(RTLIB::POW_F128, "powf128");
-  Info.setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-  Info.setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-  Info.setLibcallName(RTLIB::RINT_F128, "rintf128");
-  Info.setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-  Info.setLibcallName(RTLIB::ROUND_F128, "roundf128");
-  Info.setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
-  Info.setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-  Info.setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
-  Info.setLibcallName(RTLIB::FMIN_F128, "fminf128");
-  Info.setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-  Info.setLibcallName(RTLIB::FMINIMUM_F128, "fminimumf128");
-  Info.setLibcallName(RTLIB::FMAXIMUM_F128, "fmaximumf128");
-  Info.setLibcallName(RTLIB::FMINIMUM_NUM_F128, "fminimum_numf128");
-  Info.setLibcallName(RTLIB::FMAXIMUM_NUM_F128, "fmaximum_numf128");
-  Info.setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-  Info.setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-  Info.setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-  Info.setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-  Info.setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
-  Info.setLibcallName(RTLIB::FREXP_F128, "frexpf128");
-  Info.setLibcallName(RTLIB::MODF_F128, "modff128");
+  Info.setLibcallImpl(RTLIB::REM_F128, RTLIB::fmodf128);
+  Info.setLibcallImpl(RTLIB::FMA_F128, RTLIB::fmaf128);
+  Info.setLibcallImpl(RTLIB::SQRT_F128, RTLIB::sqrtf128);
+  Info.setLibcallImpl(RTLIB::CBRT_F128, RTLIB::cbrtf128);
+  Info.setLibcallImpl(RTLIB::LOG_F128, RTLIB::logf128);
+  Info.setLibcallImpl(RTLIB::LOG2_F128, RTLIB::log2f128);
+  Info.setLibcallImpl(RTLIB::LOG10_F128, RTLIB::log10f128);
+  Info.setLibcallImpl(RTLIB::EXP_F128, RTLIB::expf128);
+  Info.setLibcallImpl(RTLIB::EXP2_F128, RTLIB::exp2f128);
+  Info.setLibcallImpl(RTLIB::EXP10_F128, RTLIB::exp10f128);
+  Info.setLibcallImpl(RTLIB::SIN_F128, RTLIB::sinf128);
+  Info.setLibcallImpl(RTLIB::COS_F128, RTLIB::cosf128);
+  Info.setLibcallImpl(RTLIB::TAN_F128, RTLIB::tanf128);
+  Info.setLibcallImpl(RTLIB::SINCOS_F128, RTLIB::sincosf128);
+  Info.setLibcallImpl(RTLIB::ASIN_F128, RTLIB::asinf128);
+  Info.setLibcallImpl(RTLIB::ACOS_F128, RTLIB::acosf128);
+  Info.setLibcallImpl(RTLIB::ATAN_F128, RTLIB::atanf128);
+  Info.setLibcallImpl(RTLIB::ATAN2_F128, RTLIB::atan2f128);
+  Info.setLibcallImpl(RTLIB::SINH_F128, RTLIB::sinhf128);
+  Info.setLibcallImpl(RTLIB::COSH_F128, RTLIB::coshf128);
+  Info.setLibcallImpl(RTLIB::TANH_F128, RTLIB::tanhf128);
+  Info.setLibcallImpl(RTLIB::POW_F128, RTLIB::powf128);
+  Info.setLibcallImpl(RTLIB::CEIL_F128, RTLIB::ceilf128);
+  Info.setLibcallImpl(RTLIB::TRUNC_F128, RTLIB::truncf128);
+  Info.setLibcallImpl(RTLIB::RINT_F128, RTLIB::rintf128);
+  Info.setLibcallImpl(RTLIB::NEARBYINT_F128, RTLIB::nearbyintf128);
+  Info.setLibcallImpl(RTLIB::ROUND_F128, RTLIB::roundf128);
+  Info.setLibcallImpl(RTLIB::ROUNDEVEN_F128, RTLIB::roundevenf128);
+  Info.setLibcallImpl(RTLIB::FLOOR_F128, RTLIB::floorf128);
+  Info.setLibcallImpl(RTLIB::COPYSIGN_F128, RTLIB::copysignf128);
+  Info.setLibcallImpl(RTLIB::FMIN_F128, RTLIB::fminf128);
+  Info.setLibcallImpl(RTLIB::FMAX_F128, RTLIB::fmaxf128);
+  Info.setLibcallImpl(RTLIB::FMINIMUM_F128, RTLIB::fminimumf128);
+  Info.setLibcallImpl(RTLIB::FMAXIMUM_F128, RTLIB::fmaximumf128);
+  Info.setLibcallImpl(RTLIB::FMINIMUM_NUM_F128, RTLIB::fminimum_numf128);
+  Info.setLibcallImpl(RTLIB::FMAXIMUM_NUM_F128, RTLIB::fmaximum_numf128);
+  Info.setLibcallImpl(RTLIB::LROUND_F128, RTLIB::lroundf128);
+  Info.setLibcallImpl(RTLIB::LLROUND_F128, RTLIB::llroundf128);
+  Info.setLibcallImpl(RTLIB::LRINT_F128, RTLIB::lrintf128);
+  Info.setLibcallImpl(RTLIB::LLRINT_F128, RTLIB::llrintf128);
+  Info.setLibcallImpl(RTLIB::LDEXP_F128, RTLIB::ldexpf128);
+  Info.setLibcallImpl(RTLIB::FREXP_F128, RTLIB::frexpf128);
+  Info.setLibcallImpl(RTLIB::MODF_F128, RTLIB::modff128);
 
   if (FiniteOnlyFuncs) {
-    Info.setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
-    Info.setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
-    Info.setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
-    Info.setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
-    Info.setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
-    Info.setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
+    Info.setLibcallImpl(RTLIB::LOG_FINITE_F128, RTLIB::__logf128_finite);
+    Info.setLibcallImpl(RTLIB::LOG2_FINITE_F128, RTLIB::__log2f128_finite);
+    Info.setLibcallImpl(RTLIB::LOG10_FINITE_F128, RTLIB::__log10f128_finite);
+    Info.setLibcallImpl(RTLIB::EXP_FINITE_F128, RTLIB::__expf128_finite);
+    Info.setLibcallImpl(RTLIB::EXP2_FINITE_F128, RTLIB::__exp2f128_finite);
+    Info.setLibcallImpl(RTLIB::POW_FINITE_F128, RTLIB::__powf128_finite);
   } else {
-    Info.setLibcallName(RTLIB::LOG_FINITE_F128, nullptr);
-    Info.setLibcallName(RTLIB::LOG2_FINITE_F128, nullptr);
-    Info.setLibcallName(RTLIB::LOG10_FINITE_F128, nullptr);
-    Info.setLibcallName(RTLIB::EXP_FINITE_F128, nullptr);
-    Info.setLibcallName(RTLIB::EXP2_FINITE_F128, nullptr);
-    Info.setLibcallName(RTLIB::POW_FINITE_F128, nullptr);
+    Info.setLibcallImpl(RTLIB::LOG_FINITE_F128, RTLIB::Unsupported);
+    Info.setLibcallImpl(RTLIB::LOG2_FINITE_F128, RTLIB::Unsupported);
+    Info.setLibcallImpl(RTLIB::LOG10_FINITE_F128, RTLIB::Unsupported);
+    Info.setLibcallImpl(RTLIB::EXP_FINITE_F128, RTLIB::Unsupported);
+    Info.setLibcallImpl(RTLIB::EXP2_FINITE_F128, RTLIB::Unsupported);
+    Info.setLibcallImpl(RTLIB::POW_FINITE_F128, RTLIB::Unsupported);
   }
 }
 
+void RTLIB::RuntimeLibcallsInfo::initDefaultLibCallImpls() {
+  std::memcpy(LibcallImpls, DefaultLibcallImpls, sizeof(LibcallImpls));
+  static_assert(sizeof(LibcallImpls) == sizeof(DefaultLibcallImpls),
+                "libcall array size should match");
+}
+
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
 void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
                                        ExceptionHandling ExceptionModel,
                                        FloatABI::ABIType FloatABI,
                                        EABI EABIVersion, StringRef ABIName) {
-  initSoftFloatCmpLibcallPredicates();
-
-  initSoftFloatCmpLibcallPredicates();
-
-#define HANDLE_LIBCALL(code, name) setLibcallName(RTLIB::code, name);
-#define LIBCALL_NO_NAME nullptr
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-#undef LIBCALL_NO_NAME
-
   // Use the f128 variants of math functions on x86
   if (TT.isX86() && TT.isGNUEnvironment())
     setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/true);
 
   if (TT.isX86() || TT.isVE()) {
     if (ExceptionModel == ExceptionHandling::SjLj)
-      setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
+      setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
   }
 
-  // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
   if (TT.isPPC()) {
-    setLibcallName(RTLIB::ADD_F128, "__addkf3");
-    setLibcallName(RTLIB::SUB_F128, "__subkf3");
-    setLibcallName(RTLIB::MUL_F128, "__mulkf3");
-    setLibcallName(RTLIB::DIV_F128, "__divkf3");
-    setLibcallName(RTLIB::POWI_F128, "__powikf2");
-    setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
-    setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
-    setLibcallName(RTLIB::FPROUND_F128_F16, "__trunckfhf2");
-    setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
-    setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
-    setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
-    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
-    setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
-    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
-    setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
-    setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
-    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
-    setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
-    setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
-    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
-    setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
-    setLibcallName(RTLIB::UNE_F128, "__nekf2");
-    setLibcallName(RTLIB::OGE_F128, "__gekf2");
-    setLibcallName(RTLIB::OLT_F128, "__ltkf2");
-    setLibcallName(RTLIB::OLE_F128, "__lekf2");
-    setLibcallName(RTLIB::OGT_F128, "__gtkf2");
-    setLibcallName(RTLIB::UO_F128, "__unordkf2");
+    setPPCLibCallNameOverrides();
 
     // TODO: Do the finite only functions exist?
     setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/false);
 
+    // TODO: Tablegen predicate support
     if (TT.isOSAIX()) {
-      bool isPPC64 = TT.isPPC64();
-      setLibcallName(RTLIB::MEMCPY, nullptr);
-      setLibcallName(RTLIB::MEMMOVE, isPPC64 ? "___memmove64" : "___memmove");
-      setLibcallName(RTLIB::MEMSET, isPPC64 ? "___memset64" : "___memset");
-      setLibcallName(RTLIB::BZERO, isPPC64 ? "___bzero64" : "___bzero");
+      if (TT.isPPC64()) {
+        setLibcallImpl(RTLIB::MEMCPY, RTLIB::Unsupported);
+        setLibcallImpl(RTLIB::MEMMOVE, RTLIB::___memmove64);
+        setLibcallImpl(RTLIB::MEMSET, RTLIB::___memset64);
+        setLibcallImpl(RTLIB::BZERO, RTLIB::___bzero64);
+      } else {
+        setLibcallImpl(RTLIB::MEMCPY, RTLIB::Unsupported);
+        setLibcallImpl(RTLIB::MEMMOVE, RTLIB::___memmove);
+        setLibcallImpl(RTLIB::MEMSET, RTLIB::___memset);
+        setLibcallImpl(RTLIB::BZERO, RTLIB::___bzero);
+      }
     }
   }
 
@@ -428,27 +394,19 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
     // For f16/f32 conversions, Darwin uses the standard naming scheme,
     // instead of the gnueabi-style __gnu_*_ieee.
     // FIXME: What about other targets?
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
+    setLibcallImpl(RTLIB::FPEXT_F16_F32, RTLIB::__extendhfsf2);
+    setLibcallImpl(RTLIB::FPROUND_F32_F16, RTLIB::__truncsfhf2);
 
     // Some darwins have an optimized __bzero/bzero function.
-    switch (TT.getArch()) {
-    case Triple::x86:
-    case Triple::x86_64:
+    if (TT.isX86()) {
       if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
-        setLibcallName(RTLIB::BZERO, "__bzero");
-      break;
-    case Triple::aarch64:
-    case Triple::aarch64_32:
-      setLibcallName(RTLIB::BZERO, "bzero");
-      break;
-    default:
-      break;
-    }
+        setLibcallImpl(RTLIB::BZERO, RTLIB::__bzero);
+    } else if (TT.isAArch64())
+      setLibcallImpl(RTLIB::BZERO, RTLIB::bzero);
 
     if (darwinHasSinCosStret(TT)) {
-      setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
-      setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
+      setLibcallImpl(RTLIB::SINCOS_STRET_F32, RTLIB::__sincosf_stret);
+      setLibcallImpl(RTLIB::SINCOS_STRET_F64, RTLIB::__sincos_stret);
       if (TT.isWatchABI()) {
         setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
                               CallingConv::ARM_AAPCS_VFP);
@@ -457,80 +415,56 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
       }
     }
 
-    switch (TT.getOS()) {
-    case Triple::MacOSX:
-      if (TT.isMacOSXVersionLT(10, 9)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-      } else {
-        setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-        setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      }
-      break;
-    case Triple::IOS:
-      if (TT.isOSVersionLT(7, 0)) {
-        setLibcallName(RTLIB::EXP10_F32, nullptr);
-        setLibcallName(RTLIB::EXP10_F64, nullptr);
-        break;
-      }
-      [[fallthrough]];
-    case Triple::DriverKit:
-    case Triple::TvOS:
-    case Triple::WatchOS:
-    case Triple::XROS:
-      setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-      setLibcallName(RTLIB::EXP10_F64, "__exp10");
-      break;
-    default:
-      break;
+    if (darwinHasExp10(TT)) {
+      setLibcallImpl(RTLIB::EXP10_F32, RTLIB::__exp10f);
+      setLibcallImpl(RTLIB::EXP10_F64, RTLIB::__exp10);
+    } else {
+      setLibcallImpl(RTLIB::EXP10_F32, RTLIB::Unsupported);
+      setLibcallImpl(RTLIB::EXP10_F64, RTLIB::Unsupported);
     }
-  } else if (TT.getOS() == Triple::BridgeOS) {
-    // TODO: BridgeOS should be included in isOSDarwin.
-    setLibcallName(RTLIB::EXP10_F32, "__exp10f");
-    setLibcallName(RTLIB::EXP10_F64, "__exp10");
   }
 
   if (hasSinCos(TT)) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
-    setLibcallName(RTLIB::SINCOS_F80, "sincosl");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosl");
-    setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
+    setLibcallImpl(RTLIB::SINCOS_F32, RTLIB::sincosf);
+    setLibcallImpl(RTLIB::SINCOS_F64, RTLIB::sincos);
+    setLibcallImpl(RTLIB::SINCOS_F80, RTLIB::sincos_f80);
+    setLibcallImpl(RTLIB::SINCOS_F128, RTLIB::sincos_f128);
+    setLibcallImpl(RTLIB::SINCOS_PPCF128, RTLIB::sincos_ppcf128);
   }
 
   if (TT.isPS()) {
-    setLibcallName(RTLIB::SINCOS_F32, "sincosf");
-    setLibcallName(RTLIB::SINCOS_F64, "sincos");
+    setLibcallImpl(RTLIB::SINCOS_F32, RTLIB::sincosf);
+    setLibcallImpl(RTLIB::SINCOS_F64, RTLIB::sincos);
   }
 
   if (TT.isOSOpenBSD()) {
-    setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
+    setLibcallImpl(RTLIB::STACKPROTECTOR_CHECK_FAIL, RTLIB::Unsupported);
   }
 
   if (TT.isOSWindows() && !TT.isOSCygMing()) {
-    setLibcallName(RTLIB::LDEXP_F32, nullptr);
-    setLibcallName(RTLIB::LDEXP_F80, nullptr);
-    setLibcallName(RTLIB::LDEXP_F128, nullptr);
-    setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
-
-    setLibcallName(RTLIB::FREXP_F32, nullptr);
-    setLibcallName(RTLIB::FREXP_F80, nullptr);
-    setLibcallName(RTLIB::FREXP_F128, nullptr);
-    setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
+    setLibcallImpl(RTLIB::LDEXP_F32, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::LDEXP_F80, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::LDEXP_F128, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::LDEXP_PPCF128, RTLIB::Unsupported);
+
+    setLibcallImpl(RTLIB::FREXP_F32, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::FREXP_F80, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::FREXP_F128, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::FREXP_PPCF128, RTLIB::Unsupported);
   }
 
   // Disable most libcalls on AMDGPU and NVPTX.
   if (TT.isAMDGPU() || TT.isNVPTX()) {
     for (RTLIB::Libcall LC : RTLIB::libcalls()) {
-      if (LC < RTLIB::ATOMIC_LOAD || LC > RTLIB::ATOMIC_FETCH_NAND_16)
-        setLibcallName(LC, nullptr);
+      if (!isAtomicLibCall(LC))
+        setLibcallImpl(LC, RTLIB::Unsupported);
     }
   }
 
   if (TT.isOSMSVCRT()) {
     // MSVCRT doesn't have powi; fall back to pow
-    setLibcallName(RTLIB::POWI_F32, nullptr);
-    setLibcallName(RTLIB::POWI_F64, nullptr);
+    setLibcallImpl(RTLIB::POWI_F32, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::POWI_F64, RTLIB::Unsupported);
   }
 
   // Setup Windows compiler runtime calls.
@@ -538,50 +472,61 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
       (TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment())) {
     static const struct {
       const RTLIB::Libcall Op;
-      const char *const Name;
+      const RTLIB::LibcallImpl Impl;
       const CallingConv::ID CC;
     } LibraryCalls[] = {
-        {RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall},
-        {RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall},
-        {RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall},
-        {RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall},
-        {RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall},
+        {RTLIB::SDIV_I64, RTLIB::_alldiv, CallingConv::X86_StdCall},
+        {RTLIB::UDIV_I64, RTLIB::_aulldiv, CallingConv::X86_StdCall},
+        {RTLIB::SREM_I64, RTLIB::_allrem, CallingConv::X86_StdCall},
+        {RTLIB::UREM_I64, RTLIB::_aullrem, CallingConv::X86_StdCall},
+        {RTLIB::MUL_I64, RTLIB::_allmul, CallingConv::X86_StdCall},
     };
 
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
       setLibcallCallingConv(LC.Op, LC.CC);
     }
   }
 
-  if (TT.isAArch64())
+  if (TT.isAArch64()) {
+    if (TT.isWindowsArm64EC()) {
+      setWindowsArm64LibCallNameOverrides();
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::arm64ec___arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::arm64ec___arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::arm64ec___arm_sc_memset);
+    } else {
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::__arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::__arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::__arm_sc_memset);
+    }
+
     setAArch64LibcallNames(*this, TT);
-  else if (TT.isARM() || TT.isThumb())
+  } else if (TT.isARM() || TT.isThumb()) {
     setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
-  else if (TT.getArch() == Triple::ArchType::avr) {
+  } else if (TT.getArch() == Triple::ArchType::avr) {
     // Division rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SDIV_I8, nullptr);
-    setLibcallName(RTLIB::SDIV_I16, nullptr);
-    setLibcallName(RTLIB::SDIV_I32, nullptr);
-    setLibcallName(RTLIB::UDIV_I8, nullptr);
-    setLibcallName(RTLIB::UDIV_I16, nullptr);
-    setLibcallName(RTLIB::UDIV_I32, nullptr);
+    setLibcallImpl(RTLIB::SDIV_I8, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::SDIV_I16, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::SDIV_I32, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UDIV_I8, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UDIV_I16, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UDIV_I32, RTLIB::Unsupported);
 
     // Modulus rtlib functions (not supported), use divmod functions instead
-    setLibcallName(RTLIB::SREM_I8, nullptr);
-    setLibcallName(RTLIB::SREM_I16, nullptr);
-    setLibcallName(RTLIB::SREM_I32, nullptr);
-    setLibcallName(RTLIB::UREM_I8, nullptr);
-    setLibcallName(RTLIB::UREM_I16, nullptr);
-    setLibcallName(RTLIB::UREM_I32, nullptr);
+    setLibcallImpl(RTLIB::SREM_I8, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::SREM_I16, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::SREM_I32, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UREM_I8, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UREM_I16, RTLIB::Unsupported);
+    setLibcallImpl(RTLIB::UREM_I32, RTLIB::Unsupported);
 
     // Division and modulus rtlib functions
-    setLibcallName(RTLIB::SDIVREM_I8, "__divmodqi4");
-    setLibcallName(RTLIB::SDIVREM_I16, "__divmodhi4");
-    setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
-    setLibcallName(RTLIB::UDIVREM_I8, "__udivmodqi4");
-    setLibcallName(RTLIB::UDIVREM_I16, "__udivmodhi4");
-    setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
+    setLibcallImpl(RTLIB::SDIVREM_I8, RTLIB::__divmodqi4);
+    setLibcallImpl(RTLIB::SDIVREM_I16, RTLIB::__divmodhi4);
+    setLibcallImpl(RTLIB::SDIVREM_I32, RTLIB::__divmodsi4);
+    setLibcallImpl(RTLIB::UDIVREM_I8, RTLIB::__udivmodqi4);
+    setLibcallImpl(RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4);
+    setLibcallImpl(RTLIB::UDIVREM_I32, RTLIB::__udivmodsi4);
 
     // Several of the runtime library functions use a special calling conv
     setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::AVR_BUILTIN);
@@ -590,78 +535,95 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
     setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::AVR_BUILTIN);
 
     // Trigonometric rtlib functions
-    setLibcallName(RTLIB::SIN_F32, "sin");
-    setLibcallName(RTLIB::COS_F32, "cos");
+    setLibcallImpl(RTLIB::SIN_F32, RTLIB::avr_sin);
+    setLibcallImpl(RTLIB::COS_F32, RTLIB::avr_cos);
   }
 
   if (!TT.isWasm()) {
     // These libcalls are only available in compiler-rt, not libgcc.
     if (TT.isArch32Bit()) {
-      setLibcallName(RTLIB::SHL_I128, nullptr);
-      setLibcallName(RTLIB::SRL_I128, nullptr);
-      setLibcallName(RTLIB::SRA_I128, nullptr);
-      setLibcallName(RTLIB::MUL_I128, nullptr);
-      setLibcallName(RTLIB::MULO_I64, nullptr);
+      setLibcallImpl(RTLIB::SHL_I128, RTLIB::Unsupported);
+      setLibcallImpl(RTLIB::SRL_I128, RTLIB::Unsupported);
+      setLibcallImpl(RTLIB::SRA_I128, RTLIB::Unsupported);
+      setLibcallImpl(RTLIB::MUL_I128, RTLIB::Unsupported);
+      setLibcallImpl(RTLIB::MULO_I64, RTLIB::Unsupported);
     }
-    setLibcallName(RTLIB::MULO_I128, nullptr);
+
+    setLibcallImpl(RTLIB::MULO_I128, RTLIB::Unsupported);
   } else {
     // Define the emscripten name for return address helper.
     // TODO: when implementing other Wasm backends, make this generic or only do
     // this on emscripten depending on what they end up doing.
-    setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
-  }
-
-  if (TT.isSystemZ() && TT.isOSzOS()) {
-    struct RTLibCallMapping {
-      RTLIB::Libcall Code;
-      const char *Name;
-    };
-    static RTLibCallMapping RTLibCallCommon[] = {
-#define HANDLE_LIBCALL(code, name) {RTLIB::code, name},
-#include "ZOSLibcallNames.def"
-    };
-    for (auto &E : RTLibCallCommon)
-      setLibcallName(E.Code, E.Name);
+    setLibcallImpl(RTLIB::RETURN_ADDRESS, RTLIB::emscripten_return_address);
   }
 
   if (TT.getArch() == Triple::ArchType::hexagon) {
-    setLibcallName(RTLIB::SDIV_I32, "__hexagon_divsi3");
-    setLibcallName(RTLIB::SDIV_I64, "__hexagon_divdi3");
-    setLibcallName(RTLIB::UDIV_I32, "__hexagon_udivsi3");
-    setLibcallName(RTLIB::UDIV_I64, "__hexagon_udivdi3");
-    setLibcallName(RTLIB::SREM_I32, "__hexagon_modsi3");
-    setLibcallName(RTLIB::SREM_I64, "__hexagon_moddi3");
-    setLibcallName(RTLIB::UREM_I32, "__hexagon_umodsi3");
-    setLibcallName(RTLIB::UREM_I64, "__hexagon_umoddi3");
+    setLibcallImpl(RTLIB::SDIV_I32, RTLIB::__hexagon_divsi3);
+    setLibcallImpl(RTLIB::SDIV_I64, RTLIB::__hexagon_divdi3);
+    setLibcallImpl(RTLIB::UDIV_I32, RTLIB::__hexagon_udivsi3);
+    setLibcallImpl(RTLIB::UDIV_I64, RTLIB::__hexagon_udivdi3);
+    setLibcallImpl(RTLIB::SREM_I32, RTLIB::__hexagon_modsi3);
+    setLibcallImpl(RTLIB::SREM_I64, RTLIB::__hexagon_moddi3);
+    setLibcallImpl(RTLIB::UREM_I32, RTLIB::__hexagon_umodsi3);
+    setLibcallImpl(RTLIB::UREM_I64, RTLIB::__hexagon_umoddi3);
 
     const bool FastMath = HexagonEnableFastMathRuntimeCalls;
     // This is the only fast library function for sqrtd.
     if (FastMath)
-      setLibcallName(RTLIB::SQRT_F64, "__hexagon_fast2_sqrtdf2");
+      setLibcallImpl(RTLIB::SQRT_F64, RTLIB::__hexagon_fast2_sqrtdf2);
 
     // Prefix is: nothing  for "slow-math",
     //            "fast2_" for V5+ fast-math double-precision
     // (actually, keep fast-math and fast-math2 separate for now)
     if (FastMath) {
-      setLibcallName(RTLIB::ADD_F64, "__hexagon_fast_adddf3");
-      setLibcallName(RTLIB::SUB_F64, "__hexagon_fast_subdf3");
-      setLibcallName(RTLIB::MUL_F64, "__hexagon_fast_muldf3");
-      setLibcallName(RTLIB::DIV_F64, "__hexagon_fast_divdf3");
-      setLibcallName(RTLIB::DIV_F32, "__hexagon_fast_divsf3");
+      setLibcallImpl(RTLIB::ADD_F64, RTLIB::__hexagon_fast_adddf3);
+      setLibcallImpl(RTLIB::SUB_F64, RTLIB::__hexagon_fast_subdf3);
+      setLibcallImpl(RTLIB::MUL_F64, RTLIB::__hexagon_fast_muldf3);
+      setLibcallImpl(RTLIB::DIV_F64, RTLIB::__hexagon_fast_divdf3);
+      setLibcallImpl(RTLIB::DIV_F32, RTLIB::__hexagon_fast_divsf3);
     } else {
-      setLibcallName(RTLIB::ADD_F64, "__hexagon_adddf3");
-      setLibcallName(RTLIB::SUB_F64, "__hexagon_subdf3");
-      setLibcallName(RTLIB::MUL_F64, "__hexagon_muldf3");
-      setLibcallName(RTLIB::DIV_F64, "__hexagon_divdf3");
-      setLibcallName(RTLIB::DIV_F32, "__hexagon_divsf3");
+      setLibcallImpl(RTLIB::ADD_F64, RTLIB::__hexagon_adddf3);
+      setLibcallImpl(RTLIB::SUB_F64, RTLIB::__hexagon_subdf3);
+      setLibcallImpl(RTLIB::MUL_F64, RTLIB::__hexagon_muldf3);
+      setLibcallImpl(RTLIB::DIV_F64, RTLIB::__hexagon_divdf3);
+      setLibcallImpl(RTLIB::DIV_F32, RTLIB::__hexagon_divsf3);
     }
 
     if (FastMath)
-      setLibcallName(RTLIB::SQRT_F32, "__hexagon_fast2_sqrtf");
+      setLibcallImpl(RTLIB::SQRT_F32, RTLIB::__hexagon_fast2_sqrtf);
     else
-      setLibcallName(RTLIB::SQRT_F32, "__hexagon_sqrtf");
+      setLibcallImpl(RTLIB::SQRT_F32, RTLIB::__hexagon_sqrtf);
+
+    setLibcallImpl(
+        RTLIB::HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES,
+        RTLIB::__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes);
   }
 
   if (TT.getArch() == Triple::ArchType::msp430)
     setMSP430Libcalls(*this, TT);
+
+  if (TT.isSystemZ() && TT.isOSzOS())
+    setZOSLibCallNameOverrides();
+
+  if (TT.getArch() == Triple::ArchType::xcore)
+    setLibcallImpl(RTLIB::MEMCPY_ALIGN_4, RTLIB::__memcpy_4);
+}
+
+bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) {
+  assert(TT.isOSDarwin() && "should be called with darwin triple");
+
+  switch (TT.getOS()) {
+  case Triple::MacOSX:
+    return !TT.isMacOSXVersionLT(10, 9);
+  case Triple::IOS:
+    return !TT.isOSVersionLT(7, 0);
+  case Triple::DriverKit:
+  case Triple::TvOS:
+  case Triple::WatchOS:
+  case Triple::XROS:
+  case Triple::BridgeOS:
+    return true;
+  default:
+    return false;
+  }
 }
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index d10e8b2a0bb6..1a25a6a67ccf 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -985,7 +985,7 @@ struct TargetTypeInfo {
 static TargetTypeInfo getTargetTypeInfo(const TargetExtType *Ty) {
   LLVMContext &C = Ty->getContext();
   StringRef Name = Ty->getName();
-  if (Name == "spirv.Image")
+  if (Name == "spirv.Image" || Name == "spirv.SignedImage")
     return TargetTypeInfo(PointerType::get(C, 0), TargetExtType::CanBeGlobal,
                           TargetExtType::CanBeLocal);
   if (Name == "spirv.Type") {
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 7b08fdc0fb7c..0b6e0e40527c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -112,6 +112,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
@@ -1180,6 +1181,10 @@ void Verifier::visitDISubrangeType(const DISubrangeType &N) {
   CheckDI(!Bias || isa<ConstantAsMetadata>(Bias) || isa<DIVariable>(Bias) ||
               isa<DIExpression>(Bias),
           "Bias must be signed constant or DIVariable or DIExpression", &N);
+  // Subrange types currently only support constant size.
+  auto *Size = N.getRawSizeInBits();
+  CheckDI(!Size || isa<ConstantAsMetadata>(Size),
+          "SizeInBits must be a constant");
 }
 
 void Verifier::visitDISubrange(const DISubrange &N) {
@@ -1241,6 +1246,10 @@ void Verifier::visitDIBasicType(const DIBasicType &N) {
               N.getTag() == dwarf::DW_TAG_unspecified_type ||
               N.getTag() == dwarf::DW_TAG_string_type,
           "invalid tag", &N);
+  // Basic types currently only support constant size.
+  auto *Size = N.getRawSizeInBits();
+  CheckDI(!Size || isa<ConstantAsMetadata>(Size),
+          "SizeInBits must be a constant");
 }
 
 void Verifier::visitDIFixedPointType(const DIFixedPointType &N) {
@@ -1322,6 +1331,11 @@ void Verifier::visitDIDerivedType(const DIDerivedType &N) {
             &N);
   }
 
+  auto *Size = N.getRawSizeInBits();
+  CheckDI(!Size || isa<ConstantAsMetadata>(Size) || isa<DIVariable>(Size) ||
+              isa<DIExpression>(Size),
+          "SizeInBits must be a constant or DIVariable or DIExpression");
+
   if (N.getDWARFMemorySpace() != dwarf::DW_MSPACE_LLVM_none) {
     CheckDI(N.getTag() == dwarf::DW_TAG_pointer_type ||
                 N.getTag() == dwarf::DW_TAG_reference_type ||
@@ -1416,6 +1430,11 @@ void Verifier::visitDICompositeType(const DICompositeType &N) {
   if (N.getTag() == dwarf::DW_TAG_array_type) {
     CheckDI(N.getRawBaseType(), "array types must have a base type", &N);
   }
+
+  auto *Size = N.getRawSizeInBits();
+  CheckDI(!Size || isa<ConstantAsMetadata>(Size) || isa<DIVariable>(Size) ||
+              isa<DIExpression>(Size),
+          "SizeInBits must be a constant or DIVariable or DIExpression");
 }
 
 void Verifier::visitDISubroutineType(const DISubroutineType &N) {
@@ -2524,6 +2543,12 @@ void Verifier::verifyFunctionMetadata(
   for (const auto &Pair : MDs) {
     if (Pair.first == LLVMContext::MD_prof) {
       MDNode *MD = Pair.second;
+      if (isExplicitlyUnknownBranchWeightsMetadata(*MD)) {
+        CheckFailed("'unknown' !prof metadata should appear only on "
+                    "instructions supporting the 'branch_weights' metadata",
+                    MD);
+        continue;
+      }
       Check(MD->getNumOperands() >= 2,
             "!prof annotations should have no less than 2 operands", MD);
 
@@ -2534,8 +2559,8 @@ void Verifier::verifyFunctionMetadata(
             "expected string with name of the !prof annotation", MD);
       MDString *MDS = cast<MDString>(MD->getOperand(0));
       StringRef ProfName = MDS->getString();
-      Check(ProfName == "function_entry_count" ||
-                ProfName == "synthetic_function_entry_count",
+      Check(ProfName == MDProfLabels::FunctionEntryCount ||
+                ProfName == MDProfLabels::SyntheticFunctionEntryCount,
             "first operand should be 'function_entry_count'"
             " or 'synthetic_function_entry_count'",
             MD);
@@ -3176,6 +3201,12 @@ void Verifier::visitFunction(const Function &F) {
     CheckDI(SP->describes(&F),
             "!dbg attachment points at wrong subprogram for function", N, &F,
             &I, DL, Scope, SP);
+
+    if (DL->getAtomGroup())
+      CheckDI(DL->getScope()->getSubprogram()->getKeyInstructionsEnabled(),
+              "DbgLoc uses atomGroup but DISubprogram doesn't have Key "
+              "Instructions enabled",
+              DL, DL->getScope()->getSubprogram());
   };
   for (auto &BB : F)
     for (auto &I : BB) {
@@ -4980,9 +5011,24 @@ void Verifier::visitDereferenceableMetadata(Instruction& I, MDNode* MD) {
 }
 
 void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
-  Check(MD->getNumOperands() >= 2,
-        "!prof annotations should have no less than 2 operands", MD);
-
+  auto GetBranchingTerminatorNumOperands = [&]() {
+    unsigned ExpectedNumOperands = 0;
+    if (BranchInst *BI = dyn_cast<BranchInst>(&I))
+      ExpectedNumOperands = BI->getNumSuccessors();
+    else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I))
+      ExpectedNumOperands = SI->getNumSuccessors();
+    else if (isa<CallInst>(&I))
+      ExpectedNumOperands = 1;
+    else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I))
+      ExpectedNumOperands = IBI->getNumDestinations();
+    else if (isa<SelectInst>(&I))
+      ExpectedNumOperands = 2;
+    else if (CallBrInst *CI = dyn_cast<CallBrInst>(&I))
+      ExpectedNumOperands = CI->getNumSuccessors();
+    return ExpectedNumOperands;
+  };
+  Check(MD->getNumOperands() >= 1,
+        "!prof annotations should have at least 1 operand", MD);
   // Check first operand.
   Check(MD->getOperand(0) != nullptr, "first operand should not be null", MD);
   Check(isa<MDString>(MD->getOperand(0)),
@@ -4990,27 +5036,28 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
   MDString *MDS = cast<MDString>(MD->getOperand(0));
   StringRef ProfName = MDS->getString();
 
+  if (ProfName == MDProfLabels::UnknownBranchWeightsMarker) {
+    Check(GetBranchingTerminatorNumOperands() != 0 || isa<InvokeInst>(I),
+          "'unknown' !prof should only appear on instructions on which "
+          "'branch_weights' would",
+          MD);
+    Check(MD->getNumOperands() == 1,
+          "'unknown' !prof should have no additional operands", MD);
+    return;
+  }
+
+  Check(MD->getNumOperands() >= 2,
+        "!prof annotations should have no less than 2 operands", MD);
+
   // Check consistency of !prof branch_weights metadata.
-  if (ProfName == "branch_weights") {
+  if (ProfName == MDProfLabels::BranchWeights) {
     unsigned NumBranchWeights = getNumBranchWeights(*MD);
     if (isa<InvokeInst>(&I)) {
       Check(NumBranchWeights == 1 || NumBranchWeights == 2,
             "Wrong number of InvokeInst branch_weights operands", MD);
     } else {
-      unsigned ExpectedNumOperands = 0;
-      if (BranchInst *BI = dyn_cast<BranchInst>(&I))
-        ExpectedNumOperands = BI->getNumSuccessors();
-      else if (SwitchInst *SI = dyn_cast<SwitchInst>(&I))
-        ExpectedNumOperands = SI->getNumSuccessors();
-      else if (isa<CallInst>(&I))
-        ExpectedNumOperands = 1;
-      else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(&I))
-        ExpectedNumOperands = IBI->getNumDestinations();
-      else if (isa<SelectInst>(&I))
-        ExpectedNumOperands = 2;
-      else if (CallBrInst *CI = dyn_cast<CallBrInst>(&I))
-        ExpectedNumOperands = CI->getNumSuccessors();
-      else
+      const unsigned ExpectedNumOperands = GetBranchingTerminatorNumOperands();
+      if (ExpectedNumOperands == 0)
         CheckFailed("!prof branch_weights are not allowed for this instruction",
                     MD);
 
@@ -5024,6 +5071,27 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
       Check(mdconst::dyn_extract<ConstantInt>(MDO),
             "!prof brunch_weights operand is not a const int");
     }
+  } else if (ProfName == MDProfLabels::ValueProfile) {
+    Check(isValueProfileMD(MD), "invalid value profiling metadata", MD);
+    ConstantInt *KindInt = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
+    Check(KindInt, "VP !prof missing kind argument", MD);
+
+    auto Kind = KindInt->getZExtValue();
+    Check(Kind >= InstrProfValueKind::IPVK_First &&
+              Kind <= InstrProfValueKind::IPVK_Last,
+          "Invalid VP !prof kind", MD);
+    Check(MD->getNumOperands() % 2 == 1,
+          "VP !prof should have an even number "
+          "of arguments after 'VP'",
+          MD);
+    if (Kind == InstrProfValueKind::IPVK_IndirectCallTarget ||
+        Kind == InstrProfValueKind::IPVK_MemOPSize)
+      Check(isa<CallBase>(I),
+            "VP !prof indirect call or memop size expected to be applied to "
+            "CallBase instructions only",
+            MD);
+  } else {
+    CheckFailed("expected either branch_weights or VP profile name", MD);
   }
 }
 
@@ -5533,7 +5601,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
                   Call.getOperand(Elem.Begin + 1)->getType()->isPointerTy(),
               "arguments to separate_storage assumptions should be pointers",
               Call);
-        return;
+        continue;
       }
       Check(Elem.Tag->getKey() == "ignore" ||
                 Attribute::isExistingAttribute(Elem.Tag->getKey()),
@@ -5550,7 +5618,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
         if (ArgCount == 3)
           Check(Call.getOperand(Elem.Begin + 2)->getType()->isIntegerTy(),
                 "third argument should be an integer if present", Call);
-        return;
+        continue;
       }
       Check(ArgCount <= 2, "too many arguments", Call);
       if (Kind == Attribute::None)
diff --git a/llvm/lib/IR/ZOSLibcallNames.def b/llvm/lib/IR/ZOSLibcallNames.def
deleted file mode 100644
index 12a01522a7e6..000000000000
--- a/llvm/lib/IR/ZOSLibcallNames.def
+++ /dev/null
@@ -1,100 +0,0 @@
-//===-- ZOSLibcallNames.def ----------------------------------- -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines all of the runtime library functions on z/OS which can be
-// generated during instruction selection.
-//
-//===----------------------------------------------------------------------===//
-
-#if !defined(HANDLE_LIBCALL)
-#error "HANDLE_LIBCALL must be defined"
-#endif
-
-HANDLE_LIBCALL(TRUNC_F64, "@@TRNC@B")
-HANDLE_LIBCALL(TRUNC_F32, "@@FTRC@B")
-HANDLE_LIBCALL(TRUNC_F128, "@@LTRC@B")
-HANDLE_LIBCALL(SQRT_F64, "@@WSQT@B")
-HANDLE_LIBCALL(SQRT_F32, "@@FSQT@B")
-HANDLE_LIBCALL(SQRT_F128, "@@LSQT@B")
-HANDLE_LIBCALL(SIN_F64, "@@SSIN@B")
-HANDLE_LIBCALL(SIN_F32, "@@FSIN@B")
-HANDLE_LIBCALL(SIN_F128, "@@LSIN@B")
-HANDLE_LIBCALL(ROUND_F64, "@@ROUN@B")
-HANDLE_LIBCALL(ROUND_F32, "@@ROUNFB")
-HANDLE_LIBCALL(ROUND_F128, "@@ROUNLB")
-HANDLE_LIBCALL(RINT_F64, "@@SRNT@B")
-HANDLE_LIBCALL(RINT_F32, "@@RINTFB")
-HANDLE_LIBCALL(RINT_F128, "@@RINTLB")
-HANDLE_LIBCALL(REM_F64, "@@WFMD@B")
-HANDLE_LIBCALL(REM_F32, "@@FFMD@B")
-HANDLE_LIBCALL(REM_F128, "@@LFMD@B")
-HANDLE_LIBCALL(POW_F64, "@@WPOW@B")
-HANDLE_LIBCALL(POW_F32, "@@FPOW@B")
-HANDLE_LIBCALL(POW_F128, "@@LPOW@B")
-HANDLE_LIBCALL(NEARBYINT_F64, "@@NBYI@B")
-HANDLE_LIBCALL(NEARBYINT_F32, "@@NBYIFB")
-HANDLE_LIBCALL(NEARBYINT_F128, "@@NBYILB")
-HANDLE_LIBCALL(LROUND_F64, "@@ROND@B")
-HANDLE_LIBCALL(LROUND_F32, "@@FRND@B")
-HANDLE_LIBCALL(LROUND_F128, "@@LRND@B")
-HANDLE_LIBCALL(LRINT_F64, "@@LRNT@B")
-HANDLE_LIBCALL(LRINT_F32, "@@LRNTFB")
-HANDLE_LIBCALL(LRINT_F128, "@@LRNTLB")
-HANDLE_LIBCALL(LOG_F64, "@@WLOG@B")
-HANDLE_LIBCALL(LOG_F32, "@@FLOG@B")
-HANDLE_LIBCALL(LOG_F128, "@@LLOG@B")
-HANDLE_LIBCALL(LOG2_F64, "@@LOG2@B")
-HANDLE_LIBCALL(LOG2_F32, "@@FLG2@B")
-HANDLE_LIBCALL(LOG2_F128, "@@LLG2@B")
-HANDLE_LIBCALL(LOG10_F64, "@@WLG1@B")
-HANDLE_LIBCALL(LOG10_F32, "@@FLG1@B")
-HANDLE_LIBCALL(LOG10_F128, "@@LLG1@B")
-HANDLE_LIBCALL(LLROUND_F64, "@@LLRD@B")
-HANDLE_LIBCALL(LLROUND_F32, "@@LLRDFB")
-HANDLE_LIBCALL(LLROUND_F128, "@@LLRDLB")
-HANDLE_LIBCALL(LLRINT_F64, "@@LLRT@B")
-HANDLE_LIBCALL(LLRINT_F32, "@@LLRTFB")
-HANDLE_LIBCALL(LLRINT_F128, "@@LLRTLB")
-HANDLE_LIBCALL(LDEXP_F64, "@@SLXP@B")
-HANDLE_LIBCALL(LDEXP_F32, "@@FLXP@B")
-HANDLE_LIBCALL(LDEXP_F128, "@@LLXP@B")
-HANDLE_LIBCALL(FREXP_F64, "@@SFXP@B")
-HANDLE_LIBCALL(FREXP_F32, "@@FFXP@B")
-HANDLE_LIBCALL(FREXP_F128, "@@LFXP@B")
-HANDLE_LIBCALL(FMIN_F64, "@@FMIN@B")
-HANDLE_LIBCALL(FMIN_F32, "@@FMINFB")
-HANDLE_LIBCALL(FMIN_F128, "@@FMINLB")
-HANDLE_LIBCALL(FMA_F64, "@@FMA@B")
-HANDLE_LIBCALL(FMA_F32, "@@FMAFB")
-HANDLE_LIBCALL(FMA_F128, "@@FMALB")
-HANDLE_LIBCALL(FMAX_F64, "@@FMAX@B")
-HANDLE_LIBCALL(FMAX_F32, "@@FMAXFB")
-HANDLE_LIBCALL(FMAX_F128, "@@FMAXLB")
-HANDLE_LIBCALL(FLOOR_F64, "@@SFLR@B")
-HANDLE_LIBCALL(FLOOR_F32, "@@FFLR@B")
-HANDLE_LIBCALL(FLOOR_F128, "@@LFLR@B")
-HANDLE_LIBCALL(EXP_F64, "@@WEXP@B")
-HANDLE_LIBCALL(EXP_F32, "@@FEXP@B")
-HANDLE_LIBCALL(EXP_F128, "@@LEXP@B")
-HANDLE_LIBCALL(EXP2_F64, "@@EXP2@B")
-HANDLE_LIBCALL(EXP2_F32, "@@FXP2@B")
-HANDLE_LIBCALL(EXP2_F128, "@@LXP2@B")
-HANDLE_LIBCALL(COS_F64, "@@SCOS@B")
-HANDLE_LIBCALL(COS_F32, "@@FCOS@B")
-HANDLE_LIBCALL(COS_F128, "@@LCOS@B")
-HANDLE_LIBCALL(COPYSIGN_F64, "@@DCPY@B")
-HANDLE_LIBCALL(COPYSIGN_F32, "@@FCPY@B")
-HANDLE_LIBCALL(COPYSIGN_F128, "@@LCPY@B")
-HANDLE_LIBCALL(CEIL_F64, "@@SCEL@B")
-HANDLE_LIBCALL(CEIL_F32, "@@FCEL@B")
-HANDLE_LIBCALL(CEIL_F128, "@@LCEL@B")
-HANDLE_LIBCALL(CBRT_F64, "@@SCRT@B")
-HANDLE_LIBCALL(CBRT_F32, "@@FCBT@B")
-HANDLE_LIBCALL(CBRT_F128, "@@LCBT@B")
-
-#undef HANDLE_LIBCALL
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index adf995cbc9b1..73e79c08a56c 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -135,7 +135,7 @@ std::string llvm::computeLTOCacheKey(
     Hasher.update(Data);
   };
   auto AddUint8 = [&](const uint8_t I) {
-    Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&I, 1));
+    Hasher.update(ArrayRef<uint8_t>(&I, 1));
   };
   AddString(Conf.CPU);
   // FIXME: Hash more of Options. For now all clients initialize Options from
@@ -1389,8 +1389,14 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
 SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
   RTLIB::RuntimeLibcallsInfo Libcalls(TT);
   SmallVector<const char *> LibcallSymbols;
-  copy_if(Libcalls.getLibcallNames(), std::back_inserter(LibcallSymbols),
-          [](const char *Name) { return Name; });
+  ArrayRef<RTLIB::LibcallImpl> LibcallImpls = Libcalls.getLibcallImpls();
+  LibcallSymbols.reserve(LibcallImpls.size());
+
+  for (RTLIB::LibcallImpl Impl : LibcallImpls) {
+    if (Impl != RTLIB::Unsupported)
+      LibcallSymbols.push_back(Libcalls.getLibcallImplName(Impl));
+  }
+
   return LibcallSymbols;
 }
 
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index f49f14c848b9..d662c42c522f 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -46,6 +46,7 @@ add_llvm_component_library(LLVMMC
   MCSectionCOFF.cpp
   MCSectionDXContainer.cpp
   MCSectionELF.cpp
+  MCSectionGOFF.cpp
   MCSectionMachO.cpp
   MCSectionWasm.cpp
   MCSectionXCOFF.cpp
diff --git a/llvm/lib/MC/ConstantPools.cpp b/llvm/lib/MC/ConstantPools.cpp
index 824d2463f30f..bf74ad42f91f 100644
--- a/llvm/lib/MC/ConstantPools.cpp
+++ b/llvm/lib/MC/ConstantPools.cpp
@@ -59,7 +59,7 @@ const MCExpr *ConstantPool::addEntry(const MCExpr *Value, MCContext &Context,
   MCSymbol *CPEntryLabel = Context.createTempSymbol();
 
   Entries.push_back(ConstantPoolEntry(CPEntryLabel, Value, Size, Loc));
-  const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context);
+  const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context, Loc);
   if (C)
     CachedConstantEntries[std::make_pair(C->getValue(), Size)] = SymRef;
   if (S)
diff --git a/llvm/lib/MC/DXContainerRootSignature.cpp b/llvm/lib/MC/DXContainerRootSignature.cpp
index 6c71823a51f8..482280b5ef28 100644
--- a/llvm/lib/MC/DXContainerRootSignature.cpp
+++ b/llvm/lib/MC/DXContainerRootSignature.cpp
@@ -71,12 +71,16 @@ void RootSignatureDesc::write(raw_ostream &OS) const {
   BOS.reserveExtraSpace(getSize());
 
   const uint32_t NumParameters = ParametersContainer.size();
-
+  const uint32_t NumSamplers = StaticSamplers.size();
   support::endian::write(BOS, Version, llvm::endianness::little);
   support::endian::write(BOS, NumParameters, llvm::endianness::little);
   support::endian::write(BOS, RootParameterOffset, llvm::endianness::little);
-  support::endian::write(BOS, NumStaticSamplers, llvm::endianness::little);
-  support::endian::write(BOS, StaticSamplersOffset, llvm::endianness::little);
+  support::endian::write(BOS, NumSamplers, llvm::endianness::little);
+  uint32_t SSO = StaticSamplersOffset;
+  if (NumSamplers > 0)
+    SSO = writePlaceholder(BOS);
+  else
+    support::endian::write(BOS, SSO, llvm::endianness::little);
   support::endian::write(BOS, Flags, llvm::endianness::little);
 
   SmallVector<uint32_t> ParamsOffsets;
@@ -133,29 +137,32 @@ void RootSignatureDesc::write(raw_ostream &OS) const {
                                llvm::endianness::little);
         support::endian::write(BOS, Range.RegisterSpace,
                                llvm::endianness::little);
-        support::endian::write(BOS, Range.OffsetInDescriptorsFromTableStart,
-                               llvm::endianness::little);
         if (Version > 1)
           support::endian::write(BOS, Range.Flags, llvm::endianness::little);
+        support::endian::write(BOS, Range.OffsetInDescriptorsFromTableStart,
+                               llvm::endianness::little);
       }
       break;
     }
     }
   }
-  for (const auto &S : StaticSamplers) {
-    support::endian::write(BOS, S.Filter, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressU, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressV, llvm::endianness::little);
-    support::endian::write(BOS, S.AddressW, llvm::endianness::little);
-    support::endian::write(BOS, S.MipLODBias, llvm::endianness::little);
-    support::endian::write(BOS, S.MaxAnisotropy, llvm::endianness::little);
-    support::endian::write(BOS, S.ComparisonFunc, llvm::endianness::little);
-    support::endian::write(BOS, S.BorderColor, llvm::endianness::little);
-    support::endian::write(BOS, S.MinLOD, llvm::endianness::little);
-    support::endian::write(BOS, S.MaxLOD, llvm::endianness::little);
-    support::endian::write(BOS, S.ShaderRegister, llvm::endianness::little);
-    support::endian::write(BOS, S.RegisterSpace, llvm::endianness::little);
-    support::endian::write(BOS, S.ShaderVisibility, llvm::endianness::little);
+  if (NumSamplers > 0) {
+    rewriteOffsetToCurrentByte(BOS, SSO);
+    for (const auto &S : StaticSamplers) {
+      support::endian::write(BOS, S.Filter, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressU, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressV, llvm::endianness::little);
+      support::endian::write(BOS, S.AddressW, llvm::endianness::little);
+      support::endian::write(BOS, S.MipLODBias, llvm::endianness::little);
+      support::endian::write(BOS, S.MaxAnisotropy, llvm::endianness::little);
+      support::endian::write(BOS, S.ComparisonFunc, llvm::endianness::little);
+      support::endian::write(BOS, S.BorderColor, llvm::endianness::little);
+      support::endian::write(BOS, S.MinLOD, llvm::endianness::little);
+      support::endian::write(BOS, S.MaxLOD, llvm::endianness::little);
+      support::endian::write(BOS, S.ShaderRegister, llvm::endianness::little);
+      support::endian::write(BOS, S.RegisterSpace, llvm::endianness::little);
+      support::endian::write(BOS, S.ShaderVisibility, llvm::endianness::little);
+    }
   }
   assert(Storage.size() == getSize());
   OS.write(Storage.data(), Storage.size());
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 0b67f79c17f7..9f52b3e3e85c 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -27,8 +27,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
@@ -121,7 +119,7 @@ struct ELFWriter {
   } Mode;
 
   uint64_t symbolValue(const MCSymbol &Sym);
-  bool isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed);
+  bool isInSymtab(const MCSymbolELF &Symbol);
 
   /// Helper struct for containing some precomputed information on symbols.
   struct ELFSymbolData {
@@ -399,8 +397,9 @@ static bool isIFunc(const MCSymbolELF *Symbol) {
     const MCSymbolRefExpr *Value;
     if (!Symbol->isVariable() ||
         !(Value = dyn_cast<MCSymbolRefExpr>(Symbol->getVariableValue())) ||
-        Value->getKind() != MCSymbolRefExpr::VK_None ||
-        mergeTypeForSet(Symbol->getType(), ELF::STT_GNU_IFUNC) != ELF::STT_GNU_IFUNC)
+        Value->getSpecifier() ||
+        mergeTypeForSet(Symbol->getType(), ELF::STT_GNU_IFUNC) !=
+            ELF::STT_GNU_IFUNC)
       return false;
     Symbol = &cast<MCSymbolELF>(Value->getSymbol());
   }
@@ -468,7 +467,13 @@ void ELFWriter::writeSymbol(SymbolTableWriter &Writer, uint32_t StringIndex,
                      IsReserved);
 }
 
-bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed) {
+bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol) {
+  if (Symbol.isUsedInReloc() || Symbol.isSignature())
+    return true;
+
+  if (OWriter.Renames.count(&Symbol))
+    return false;
+
   if (Symbol.isVariable()) {
     const MCExpr *Expr = Symbol.getVariableValue();
     // Target Expressions that are always inlined do not appear in the symtab
@@ -478,27 +483,18 @@ bool ELFWriter::isInSymtab(const MCSymbolELF &Symbol, bool Used, bool Renamed) {
     // The .weakref alias does not appear in the symtab.
     if (Symbol.isWeakref())
       return false;
-  }
-
-  if (Used)
-    return true;
 
-  if (Renamed)
-    return false;
-
-  if (Symbol.isVariable() && Symbol.isUndefined()) {
-    // FIXME: this is here just to diagnose the case of a var = commmon_sym.
-    Asm.getBaseSymbol(Symbol);
-    return false;
+    if (Symbol.isUndefined()) {
+      // FIXME: this is here just to diagnose the case of a var = commmon_sym.
+      Asm.getBaseSymbol(Symbol);
+      return false;
+    }
   }
 
   if (Symbol.isTemporary())
     return false;
 
-  if (Symbol.getType() == ELF::STT_SECTION)
-    return false;
-
-  return true;
+  return Symbol.getType() != ELF::STT_SECTION;
 }
 
 void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
@@ -528,10 +524,7 @@ void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
   bool HasLargeSectionIndex = false;
   for (auto It : llvm::enumerate(Asm.symbols())) {
     const auto &Symbol = cast<MCSymbolELF>(It.value());
-    bool Used = Symbol.isUsedInReloc();
-    bool isSignature = Symbol.isSignature();
-    if (!isInSymtab(Symbol, Used || isSignature,
-                    OWriter.Renames.count(&Symbol)))
+    if (!isInSymtab(Symbol))
       continue;
 
     if (Symbol.isTemporary() && Symbol.isUndefined()) {
@@ -556,7 +549,7 @@ void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
         MSD.SectionIndex = ELF::SHN_COMMON;
       }
     } else if (Symbol.isUndefined()) {
-      if (isSignature && !Used) {
+      if (Symbol.isSignature() && !Symbol.isUsedInReloc()) {
         MSD.SectionIndex = RevGroupMap.lookup(&Symbol);
         if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
           HasLargeSectionIndex = true;
@@ -1332,7 +1325,6 @@ bool ELFObjectWriter::checkRelocation(SMLoc Loc, const MCSectionELF *From,
 void ELFObjectWriter::recordRelocation(const MCFragment &F,
                                        const MCFixup &Fixup, MCValue Target,
                                        uint64_t &FixedValue) {
-  MCAsmBackend &Backend = Asm->getBackend();
   const MCSectionELF &Section = cast<MCSectionELF>(*F.getParent());
   MCContext &Ctx = getContext();
 
@@ -1343,8 +1335,7 @@ void ELFObjectWriter::recordRelocation(const MCFragment &F,
   if (DwoOS && !checkRelocation(Fixup.getLoc(), &Section, SecA))
     return;
 
-  bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
+  bool IsPCRel = Fixup.isPCRel();
   uint64_t FixupOffset = Asm->getFragmentOffset(F) + Fixup.getOffset();
   uint64_t Addend = Target.getConstant();
   if (auto *RefB = Target.getSubSym()) {
diff --git a/llvm/lib/MC/GOFFObjectWriter.cpp b/llvm/lib/MC/GOFFObjectWriter.cpp
index c17b2ed8101d..1871f5fe507e 100644
--- a/llvm/lib/MC/GOFFObjectWriter.cpp
+++ b/llvm/lib/MC/GOFFObjectWriter.cpp
@@ -12,8 +12,13 @@
 
 #include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/MC/MCAssembler.h"
+#include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCGOFFObjectWriter.h"
+#include "llvm/MC/MCSectionGOFF.h"
+#include "llvm/MC/MCSymbolGOFF.h"
 #include "llvm/MC/MCValue.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/ConvertEBCDIC.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/raw_ostream.h"
@@ -23,44 +28,13 @@ using namespace llvm;
 #define DEBUG_TYPE "goff-writer"
 
 namespace {
-
-// The standard System/390 convention is to name the high-order (leftmost) bit
-// in a byte as bit zero. The Flags type helps to set bits in a byte according
-// to this numeration order.
-class Flags {
-  uint8_t Val;
-
-  constexpr static uint8_t bits(uint8_t BitIndex, uint8_t Length, uint8_t Value,
-                                uint8_t OldValue) {
-    assert(BitIndex < 8 && "Bit index out of bounds!");
-    assert(Length + BitIndex <= 8 && "Bit length too long!");
-
-    uint8_t Mask = ((1 << Length) - 1) << (8 - BitIndex - Length);
-    Value = Value << (8 - BitIndex - Length);
-    assert((Value & Mask) == Value && "Bits set outside of range!");
-
-    return (OldValue & ~Mask) | Value;
-  }
-
-public:
-  constexpr Flags() : Val(0) {}
-  constexpr Flags(uint8_t BitIndex, uint8_t Length, uint8_t Value)
-      : Val(bits(BitIndex, Length, Value, 0)) {}
-
-  void set(uint8_t BitIndex, uint8_t Length, uint8_t Value) {
-    Val = bits(BitIndex, Length, Value, Val);
-  }
-
-  constexpr operator uint8_t() const { return Val; }
-};
-
 // Common flag values on records.
 
 // Flag: This record is continued.
-constexpr uint8_t RecContinued = Flags(7, 1, 1);
+constexpr uint8_t RecContinued = GOFF::Flags(7, 1, 1);
 
 // Flag: This record is a continuation.
-constexpr uint8_t RecContinuation = Flags(6, 1, 1);
+constexpr uint8_t RecContinuation = GOFF::Flags(6, 1, 1);
 
 // The GOFFOstream is responsible to write the data into the fixed physical
 // records of the format. A user of this class announces the begin of a new
@@ -223,33 +197,163 @@ void GOFFOstream::finalizeRecord() {
 }
 
 namespace {
+// A GOFFSymbol holds all the data required for writing an ESD record.
+class GOFFSymbol {
+public:
+  std::string Name;
+  uint32_t EsdId;
+  uint32_t ParentEsdId;
+  uint64_t Offset = 0; // Offset of the symbol into the section. LD only.
+                       // Offset is only 32 bit, the larger type is used to
+                       // enable error checking.
+  GOFF::ESDSymbolType SymbolType;
+  GOFF::ESDNameSpaceId NameSpace = GOFF::ESD_NS_ProgramManagementBinder;
+
+  GOFF::BehavioralAttributes BehavAttrs;
+  GOFF::SymbolFlags SymbolFlags;
+  uint32_t SortKey = 0;
+  uint32_t SectionLength = 0;
+  uint32_t ADAEsdId = 0;
+  uint32_t EASectionEDEsdId = 0;
+  uint32_t EASectionOffset = 0;
+  uint8_t FillByteValue = 0;
+
+  GOFFSymbol() : EsdId(0), ParentEsdId(0) {}
+
+  GOFFSymbol(StringRef Name, uint32_t EsdID, const GOFF::SDAttr &Attr)
+      : Name(Name.data(), Name.size()), EsdId(EsdID), ParentEsdId(0),
+        SymbolType(GOFF::ESD_ST_SectionDefinition) {
+    BehavAttrs.setTaskingBehavior(Attr.TaskingBehavior);
+    BehavAttrs.setBindingScope(Attr.BindingScope);
+  }
 
-class GOFFObjectWriter : public MCObjectWriter {
-  // The target specific GOFF writer instance.
-  std::unique_ptr<MCGOFFObjectTargetWriter> TargetObjectWriter;
+  GOFFSymbol(StringRef Name, uint32_t EsdID, uint32_t ParentEsdID,
+             const GOFF::EDAttr &Attr)
+      : Name(Name.data(), Name.size()), EsdId(EsdID), ParentEsdId(ParentEsdID),
+        SymbolType(GOFF::ESD_ST_ElementDefinition) {
+    this->NameSpace = Attr.NameSpace;
+    // We always set a fill byte value.
+    this->FillByteValue = Attr.FillByteValue;
+    SymbolFlags.setFillBytePresence(1);
+    SymbolFlags.setReservedQwords(Attr.ReservedQwords);
+    // TODO Do we need/should set the "mangled" flag?
+    BehavAttrs.setReadOnly(Attr.IsReadOnly);
+    BehavAttrs.setRmode(Attr.Rmode);
+    BehavAttrs.setTextStyle(Attr.TextStyle);
+    BehavAttrs.setBindingAlgorithm(Attr.BindAlgorithm);
+    BehavAttrs.setLoadingBehavior(Attr.LoadBehavior);
+    BehavAttrs.setAlignment(Attr.Alignment);
+  }
 
-  // The stream used to write the GOFF records.
-  GOFFOstream OS;
+  GOFFSymbol(StringRef Name, uint32_t EsdID, uint32_t ParentEsdID,
+             GOFF::ESDNameSpaceId NameSpace, const GOFF::LDAttr &Attr)
+      : Name(Name.data(), Name.size()), EsdId(EsdID), ParentEsdId(ParentEsdID),
+        SymbolType(GOFF::ESD_ST_LabelDefinition), NameSpace(NameSpace) {
+    SymbolFlags.setRenameable(Attr.IsRenamable);
+    BehavAttrs.setExecutable(Attr.Executable);
+    BehavAttrs.setBindingStrength(Attr.BindingStrength);
+    BehavAttrs.setLinkageType(Attr.Linkage);
+    BehavAttrs.setAmode(Attr.Amode);
+    BehavAttrs.setBindingScope(Attr.BindingScope);
+  }
 
-public:
-  GOFFObjectWriter(std::unique_ptr<MCGOFFObjectTargetWriter> MOTW,
-                   raw_pwrite_stream &OS)
-      : TargetObjectWriter(std::move(MOTW)), OS(OS) {}
+  GOFFSymbol(StringRef Name, uint32_t EsdID, uint32_t ParentEsdID,
+             const GOFF::EDAttr &EDAttr, const GOFF::PRAttr &Attr)
+      : Name(Name.data(), Name.size()), EsdId(EsdID), ParentEsdId(ParentEsdID),
+        SymbolType(GOFF::ESD_ST_PartReference), NameSpace(EDAttr.NameSpace) {
+    SymbolFlags.setRenameable(Attr.IsRenamable);
+    BehavAttrs.setExecutable(Attr.Executable);
+    BehavAttrs.setLinkageType(Attr.Linkage);
+    BehavAttrs.setBindingScope(Attr.BindingScope);
+    BehavAttrs.setAlignment(EDAttr.Alignment);
+  }
+};
 
-  ~GOFFObjectWriter() override {}
+class GOFFWriter {
+  GOFFOstream OS;
+  MCAssembler &Asm;
 
-  // Write GOFF records.
   void writeHeader();
+  void writeSymbol(const GOFFSymbol &Symbol);
+  void writeText(const MCSectionGOFF *MC);
   void writeEnd();
 
-  // Implementation of the MCObjectWriter interface.
-  void recordRelocation(const MCFragment &F, const MCFixup &Fixup,
-                        MCValue Target, uint64_t &FixedValue) override {}
-  uint64_t writeObject() override;
+  void defineSectionSymbols(const MCSectionGOFF &Section);
+  void defineLabel(const MCSymbolGOFF &Symbol);
+  void defineSymbols();
+
+public:
+  GOFFWriter(raw_pwrite_stream &OS, MCAssembler &Asm);
+  uint64_t writeObject();
 };
-} // end anonymous namespace
+} // namespace
+
+GOFFWriter::GOFFWriter(raw_pwrite_stream &OS, MCAssembler &Asm)
+    : OS(OS), Asm(Asm) {}
+
+void GOFFWriter::defineSectionSymbols(const MCSectionGOFF &Section) {
+  if (Section.isSD()) {
+    GOFFSymbol SD(Section.getName(), Section.getOrdinal(),
+                  Section.getSDAttributes());
+    writeSymbol(SD);
+  }
+
+  if (Section.isED()) {
+    GOFFSymbol ED(Section.getName(), Section.getOrdinal(),
+                  Section.getParent()->getOrdinal(), Section.getEDAttributes());
+    ED.SectionLength = Asm.getSectionAddressSize(Section);
+    writeSymbol(ED);
+  }
 
-void GOFFObjectWriter::writeHeader() {
+  if (Section.isPR()) {
+    MCSectionGOFF *Parent = Section.getParent();
+    GOFFSymbol PR(Section.getName(), Section.getOrdinal(), Parent->getOrdinal(),
+                  Parent->getEDAttributes(), Section.getPRAttributes());
+    PR.SectionLength = Asm.getSectionAddressSize(Section);
+    if (Section.requiresNonZeroLength()) {
+      // We cannot have a zero-length section for data.  If we do,
+      // artificially inflate it. Use 2 bytes to avoid odd alignments. Note:
+      // if this is ever changed, you will need to update the code in
+      // SystemZAsmPrinter::emitCEEMAIN and SystemZAsmPrinter::emitCELQMAIN to
+      // generate -1 if there is no ADA
+      if (!PR.SectionLength)
+        PR.SectionLength = 2;
+    }
+    writeSymbol(PR);
+  }
+}
+
+void GOFFWriter::defineLabel(const MCSymbolGOFF &Symbol) {
+  MCSectionGOFF &Section = static_cast<MCSectionGOFF &>(Symbol.getSection());
+  GOFFSymbol LD(Symbol.getName(), Symbol.getIndex(), Section.getOrdinal(),
+                Section.getEDAttributes().NameSpace, Symbol.getLDAttributes());
+  if (Symbol.getADA())
+    LD.ADAEsdId = Symbol.getADA()->getOrdinal();
+  writeSymbol(LD);
+}
+
+void GOFFWriter::defineSymbols() {
+  unsigned Ordinal = 0;
+  // Process all sections.
+  for (MCSection &S : Asm) {
+    auto &Section = cast<MCSectionGOFF>(S);
+    Section.setOrdinal(++Ordinal);
+    defineSectionSymbols(Section);
+  }
+
+  // Process all symbols
+  for (const MCSymbol &Sym : Asm.symbols()) {
+    if (Sym.isTemporary())
+      continue;
+    auto &Symbol = cast<MCSymbolGOFF>(Sym);
+    if (Symbol.hasLDAttributes()) {
+      Symbol.setIndex(++Ordinal);
+      defineLabel(Symbol);
+    }
+  }
+}
+
+void GOFFWriter::writeHeader() {
   OS.newRecord(GOFF::RT_HDR);
   OS.write_zeros(1);       // Reserved
   OS.writebe<uint32_t>(0); // Target Hardware Environment
@@ -263,7 +367,120 @@ void GOFFObjectWriter::writeHeader() {
   OS.write_zeros(6);       // Reserved
 }
 
-void GOFFObjectWriter::writeEnd() {
+void GOFFWriter::writeSymbol(const GOFFSymbol &Symbol) {
+  if (Symbol.Offset >= (((uint64_t)1) << 31))
+    report_fatal_error("ESD offset out of range");
+
+  // All symbol names are in EBCDIC.
+  SmallString<256> Name;
+  ConverterEBCDIC::convertToEBCDIC(Symbol.Name, Name);
+
+  // Check length here since this number is technically signed but we need uint
+  // for writing to records.
+  if (Name.size() >= GOFF::MaxDataLength)
+    report_fatal_error("Symbol max name length exceeded");
+  uint16_t NameLength = Name.size();
+
+  OS.newRecord(GOFF::RT_ESD);
+  OS.writebe<uint8_t>(Symbol.SymbolType);   // Symbol Type
+  OS.writebe<uint32_t>(Symbol.EsdId);       // ESDID
+  OS.writebe<uint32_t>(Symbol.ParentEsdId); // Parent or Owning ESDID
+  OS.writebe<uint32_t>(0);                  // Reserved
+  OS.writebe<uint32_t>(
+      static_cast<uint32_t>(Symbol.Offset));     // Offset or Address
+  OS.writebe<uint32_t>(0);                       // Reserved
+  OS.writebe<uint32_t>(Symbol.SectionLength);    // Length
+  OS.writebe<uint32_t>(Symbol.EASectionEDEsdId); // Extended Attribute ESDID
+  OS.writebe<uint32_t>(Symbol.EASectionOffset);  // Extended Attribute Offset
+  OS.writebe<uint32_t>(0);                       // Reserved
+  OS.writebe<uint8_t>(Symbol.NameSpace);         // Name Space ID
+  OS.writebe<uint8_t>(Symbol.SymbolFlags);       // Flags
+  OS.writebe<uint8_t>(Symbol.FillByteValue);     // Fill-Byte Value
+  OS.writebe<uint8_t>(0);                        // Reserved
+  OS.writebe<uint32_t>(Symbol.ADAEsdId);         // ADA ESDID
+  OS.writebe<uint32_t>(Symbol.SortKey);          // Sort Priority
+  OS.writebe<uint64_t>(0);                       // Reserved
+  for (auto F : Symbol.BehavAttrs.Attr)
+    OS.writebe<uint8_t>(F);          // Behavioral Attributes
+  OS.writebe<uint16_t>(NameLength);  // Name Length
+  OS.write(Name.data(), NameLength); // Name
+}
+
+namespace {
+/// Adapter stream to write a text section.
+class TextStream : public raw_ostream {
+  /// The underlying GOFFOstream.
+  GOFFOstream &OS;
+
+  /// The buffer size is the maximum number of bytes in a TXT section.
+  static constexpr size_t BufferSize = GOFF::MaxDataLength;
+
+  /// Static allocated buffer for the stream, used by the raw_ostream class. The
+  /// buffer is sized to hold the payload of a logical TXT record.
+  char Buffer[BufferSize];
+
+  /// The offset for the next TXT record. This is equal to the number of bytes
+  /// written.
+  size_t Offset;
+
+  /// The Esdid of the GOFF section.
+  const uint32_t EsdId;
+
+  /// The record style.
+  const GOFF::ESDTextStyle RecordStyle;
+
+  /// See raw_ostream::write_impl.
+  void write_impl(const char *Ptr, size_t Size) override;
+
+  uint64_t current_pos() const override { return Offset; }
+
+public:
+  explicit TextStream(GOFFOstream &OS, uint32_t EsdId,
+                      GOFF::ESDTextStyle RecordStyle)
+      : OS(OS), Offset(0), EsdId(EsdId), RecordStyle(RecordStyle) {
+    SetBuffer(Buffer, sizeof(Buffer));
+  }
+
+  ~TextStream() { flush(); }
+};
+} // namespace
+
+void TextStream::write_impl(const char *Ptr, size_t Size) {
+  size_t WrittenLength = 0;
+
+  // We only have signed 32bits of offset.
+  if (Offset + Size > std::numeric_limits<int32_t>::max())
+    report_fatal_error("TXT section too large");
+
+  while (WrittenLength < Size) {
+    size_t ToWriteLength =
+        std::min(Size - WrittenLength, size_t(GOFF::MaxDataLength));
+
+    OS.newRecord(GOFF::RT_TXT);
+    OS.writebe<uint8_t>(GOFF::Flags(4, 4, RecordStyle)); // Text Record Style
+    OS.writebe<uint32_t>(EsdId);                         // Element ESDID
+    OS.writebe<uint32_t>(0);                             // Reserved
+    OS.writebe<uint32_t>(static_cast<uint32_t>(Offset)); // Offset
+    OS.writebe<uint32_t>(0);                      // Text Field True Length
+    OS.writebe<uint16_t>(0);                      // Text Encoding
+    OS.writebe<uint16_t>(ToWriteLength);          // Data Length
+    OS.write(Ptr + WrittenLength, ToWriteLength); // Data
+
+    WrittenLength += ToWriteLength;
+    Offset += ToWriteLength;
+  }
+}
+
+void GOFFWriter::writeText(const MCSectionGOFF *Section) {
+  // A BSS section contains only zeros, no need to write this.
+  if (Section->isBSS())
+    return;
+
+  TextStream S(OS, Section->getOrdinal(), Section->getTextStyle());
+  Asm.writeSectionData(S, Section);
+}
+
+void GOFFWriter::writeEnd() {
   uint8_t F = GOFF::END_EPR_None;
   uint8_t AMODE = 0;
   uint32_t ESDID = 0;
@@ -271,9 +488,9 @@ void GOFFObjectWriter::writeEnd() {
   // TODO Set Flags/AMODE/ESDID for entry point.
 
   OS.newRecord(GOFF::RT_END);
-  OS.writebe<uint8_t>(Flags(6, 2, F)); // Indicator flags
-  OS.writebe<uint8_t>(AMODE);          // AMODE
-  OS.write_zeros(3);                   // Reserved
+  OS.writebe<uint8_t>(GOFF::Flags(6, 2, F)); // Indicator flags
+  OS.writebe<uint8_t>(AMODE);                // AMODE
+  OS.write_zeros(3);                         // Reserved
   // The record count is the number of logical records. In principle, this value
   // is available as OS.logicalRecords(). However, some tools rely on this field
   // being zero.
@@ -281,8 +498,14 @@ void GOFFObjectWriter::writeEnd() {
   OS.writebe<uint32_t>(ESDID); // ESDID (of entry point)
 }
 
-uint64_t GOFFObjectWriter::writeObject() {
+uint64_t GOFFWriter::writeObject() {
   writeHeader();
+
+  defineSymbols();
+
+  for (const MCSection &Section : Asm)
+    writeText(static_cast<const MCSectionGOFF *>(&Section));
+
   writeEnd();
 
   // Make sure all records are written.
@@ -294,6 +517,17 @@ uint64_t GOFFObjectWriter::writeObject() {
   return OS.getWrittenSize();
 }
 
+GOFFObjectWriter::GOFFObjectWriter(
+    std::unique_ptr<MCGOFFObjectTargetWriter> MOTW, raw_pwrite_stream &OS)
+    : TargetObjectWriter(std::move(MOTW)), OS(OS) {}
+
+GOFFObjectWriter::~GOFFObjectWriter() {}
+
+uint64_t GOFFObjectWriter::writeObject() {
+  uint64_t Size = GOFFWriter(OS, *Asm).writeObject();
+  return Size;
+}
+
 std::unique_ptr<MCObjectWriter>
 llvm::createGOFFObjectWriter(std::unique_ptr<MCGOFFObjectTargetWriter> MOTW,
                              raw_pwrite_stream &OS) {
diff --git a/llvm/lib/MC/MCAsmBackend.cpp b/llvm/lib/MC/MCAsmBackend.cpp
index c69e42dfc9fe..39ef52103106 100644
--- a/llvm/lib/MC/MCAsmBackend.cpp
+++ b/llvm/lib/MC/MCAsmBackend.cpp
@@ -10,7 +10,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCDXContainerWriter.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCGOFFObjectWriter.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -87,6 +86,7 @@ std::optional<MCFixupKind> MCAsmBackend::getFixupKind(StringRef Name) const {
 }
 
 MCFixupKindInfo MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+  // clang-format off
   static const MCFixupKindInfo Builtins[] = {
       {"FK_NONE", 0, 0, 0},
       {"FK_Data_1", 0, 8, 0},
@@ -94,15 +94,12 @@ MCFixupKindInfo MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"FK_Data_4", 0, 32, 0},
       {"FK_Data_8", 0, 64, 0},
       {"FK_Data_leb128", 0, 0, 0},
-      {"FK_PCRel_1", 0, 8, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_2", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_4", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"FK_PCRel_8", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
       {"FK_SecRel_1", 0, 8, 0},
       {"FK_SecRel_2", 0, 16, 0},
       {"FK_SecRel_4", 0, 32, 0},
       {"FK_SecRel_8", 0, 64, 0},
   };
+  // clang-format on
 
   assert(size_t(Kind - FK_NONE) < std::size(Builtins) && "Unknown fixup kind");
   return Builtins[Kind - FK_NONE];
@@ -116,14 +113,11 @@ bool MCAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
   return fixupNeedsRelaxation(Fixup, Value);
 }
 
-bool MCAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
-                            const MCValue &Target, uint64_t &FixedValue,
-                            bool IsResolved) {
-  if (IsResolved && shouldForceRelocation(Fixup, Target))
-    IsResolved = false;
+void MCAsmBackend::maybeAddReloc(const MCFragment &F, const MCFixup &Fixup,
+                                 const MCValue &Target, uint64_t &Value,
+                                 bool IsResolved) {
   if (!IsResolved)
-    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
-  return IsResolved;
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
 }
 
 bool MCAsmBackend::isDarwinCanonicalPersonality(const MCSymbol *Sym) const {
@@ -144,21 +138,9 @@ bool MCAsmBackend::isDarwinCanonicalPersonality(const MCSymbol *Sym) const {
 
 const MCSubtargetInfo *MCAsmBackend::getSubtargetInfo(const MCFragment &F) {
   const MCSubtargetInfo *STI = nullptr;
-  switch (F.getKind()) {
-  case MCFragment::FT_Data: {
-    auto &DF = cast<MCDataFragment>(F);
-    STI = DF.getSubtargetInfo();
-    assert(!DF.hasInstructions() || STI != nullptr);
-    break;
-  }
-  case MCFragment::FT_Relaxable: {
-    auto &RF = cast<MCRelaxableFragment>(F);
-    STI = RF.getSubtargetInfo();
-    assert(!RF.hasInstructions() || STI != nullptr);
-    break;
-  }
-  default:
-    break;
+  if (auto *DF = dyn_cast<MCEncodedFragment>(&F)) {
+    STI = DF->getSubtargetInfo();
+    assert(!DF->hasInstructions() || STI != nullptr);
   }
   return STI;
 }
diff --git a/llvm/lib/MC/MCAsmInfo.cpp b/llvm/lib/MC/MCAsmInfo.cpp
index ba672d2fc2ec..19032634f43d 100644
--- a/llvm/lib/MC/MCAsmInfo.cpp
+++ b/llvm/lib/MC/MCAsmInfo.cpp
@@ -124,29 +124,28 @@ bool MCAsmInfo::shouldOmitSectionDirective(StringRef SectionName) const {
         (SectionName == ".bss" && !usesELFSectionDirectiveForBSS());
 }
 
-void MCAsmInfo::initializeVariantKinds(ArrayRef<VariantKindDesc> Descs) {
-  assert(SpecifierToName.empty() && "cannot initialize twice");
+void MCAsmInfo::initializeAtSpecifiers(ArrayRef<AtSpecifier> Descs) {
+  assert(AtSpecifierToName.empty() && "cannot initialize twice");
   for (auto Desc : Descs) {
     [[maybe_unused]] auto It =
-        SpecifierToName.try_emplace(Desc.Kind, Desc.Name);
+        AtSpecifierToName.try_emplace(Desc.Kind, Desc.Name);
     assert(It.second && "duplicate Kind");
     [[maybe_unused]] auto It2 =
-        NameToSpecifier.try_emplace(Desc.Name.lower(), Desc.Kind);
-    // Workaround for VK_PPC_L/VK_PPC_LO ("l").
-    assert(It2.second || Desc.Name == "l");
+        NameToAtSpecifier.try_emplace(Desc.Name.lower(), Desc.Kind);
+    assert(It2.second);
   }
 }
 
 StringRef MCAsmInfo::getSpecifierName(uint32_t S) const {
-  auto It = SpecifierToName.find(S);
-  assert(It != SpecifierToName.end() &&
+  auto It = AtSpecifierToName.find(S);
+  assert(It != AtSpecifierToName.end() &&
          "ensure the specifier is set in initializeVariantKinds");
   return It->second;
 }
 
 std::optional<uint32_t> MCAsmInfo::getSpecifierForName(StringRef Name) const {
-  auto It = NameToSpecifier.find(Name.lower());
-  if (It != NameToSpecifier.end())
+  auto It = NameToAtSpecifier.find(Name.lower());
+  if (It != NameToAtSpecifier.end())
     return It->second;
   return {};
 }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 65a80865bb0a..951fd21d4310 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCCodeView.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -705,22 +704,14 @@ void MCAsmStreamer::emitDarwinTargetVariantBuildVersion(
 }
 
 void MCAsmStreamer::emitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
-  // Do not emit a .set on inlined target assignments.
-  bool EmitSet = true;
-  if (auto *E = dyn_cast<MCTargetExpr>(Value))
-    if (E->inlineAssignedExpr())
-      EmitSet = false;
-  if (EmitSet) {
-    bool UseSet = MAI->usesSetToEquateSymbol();
-    if (UseSet)
-      OS << ".set ";
-    Symbol->print(OS, MAI);
-    OS << (UseSet ? ", " : " = ");
-    MAI->printExpr(OS, *Value);
-
-    EmitEOL();
-  }
+  bool UseSet = MAI->usesSetToEquateSymbol();
+  if (UseSet)
+    OS << ".set ";
+  Symbol->print(OS, MAI);
+  OS << (UseSet ? ", " : " = ");
+  MAI->printExpr(OS, *Value);
 
+  EmitEOL();
   MCStreamer::emitAssignment(Symbol, Value);
 }
 
@@ -1244,7 +1235,7 @@ void MCAsmStreamer::PrintQuotedString(StringRef Data, raw_ostream &OS) const {
         continue;
       }
 
-      if (isPrint((unsigned char)C)) {
+      if (isPrint(C)) {
         OS << (char)C;
         continue;
       }
@@ -2498,9 +2489,14 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
     auto Kind = F.getKind();
     if (mc::isRelocation(Kind))
       OS << ", relocation type: " << Kind;
-    else
-      OS << ", kind: "
-         << getAssembler().getBackend().getFixupKindInfo(Kind).Name;
+    else {
+      OS << ", kind: ";
+      auto Info = getAssembler().getBackend().getFixupKindInfo(Kind);
+      if (F.isPCRel() && StringRef(Info.Name).starts_with("FK_Data_"))
+        OS << "FK_PCRel_" << (Info.TargetSize / 8);
+      else
+        OS << Info.Name;
+    }
     OS << '\n';
   }
 }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 1866c5b9e0e8..496d66b1876b 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -21,8 +21,6 @@
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -129,7 +127,7 @@ bool MCAssembler::isThumbFunc(const MCSymbol *Symbol) const {
   if (!Expr->evaluateAsRelocatable(V, nullptr))
     return false;
 
-  if (V.getSubSym() || V.getSpecifier() != MCSymbolRefExpr::VK_None)
+  if (V.getSubSym() || V.getSpecifier())
     return false;
 
   auto *Sym = V.getAddSym();
@@ -143,7 +141,7 @@ bool MCAssembler::isThumbFunc(const MCSymbol *Symbol) const {
   return true;
 }
 
-bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
+bool MCAssembler::evaluateFixup(const MCFragment &F, MCFixup &Fixup,
                                 MCValue &Target, uint64_t &Value,
                                 bool RecordReloc,
                                 MutableArrayRef<char> Contents) const {
@@ -163,38 +161,25 @@ bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
   }
 
   bool IsResolved = false;
-  unsigned FixupFlags = getBackend().getFixupKindInfo(Fixup.getKind()).Flags;
-  if (FixupFlags & MCFixupKindInfo::FKF_IsTarget) {
-    IsResolved = getBackend().evaluateTargetFixup(Fixup, Target, Value);
+  if (auto State = getBackend().evaluateFixup(F, Fixup, Target, Value)) {
+    IsResolved = *State;
   } else {
     const MCSymbol *Add = Target.getAddSym();
     const MCSymbol *Sub = Target.getSubSym();
-    Value = Target.getConstant();
+    Value += Target.getConstant();
     if (Add && Add->isDefined())
       Value += getSymbolOffset(*Add);
     if (Sub && Sub->isDefined())
       Value -= getSymbolOffset(*Sub);
 
-    bool IsPCRel = FixupFlags & MCFixupKindInfo::FKF_IsPCRel;
-    bool ShouldAlignPC =
-        FixupFlags & MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
-    if (IsPCRel) {
-      uint64_t Offset = getFragmentOffset(*DF) + Fixup.getOffset();
-
-      // A number of ARM fixups in Thumb mode require that the effective PC
-      // address be determined as the 32-bit aligned version of the actual
-      // offset.
-      if (ShouldAlignPC)
-        Offset &= ~0x3;
-      Value -= Offset;
-
+    if (Fixup.isPCRel()) {
+      Value -= getFragmentOffset(F) + Fixup.getOffset();
       if (Add && !Sub && !Add->isUndefined() && !Add->isAbsolute()) {
         IsResolved = getWriter().isSymbolRefDifferenceFullyResolvedImpl(
-            *Add, *DF, false, true);
+            *Add, F, false, true);
       }
     } else {
       IsResolved = Target.isAbsolute();
-      assert(!ShouldAlignPC && "FKF_IsAlignedDownTo32Bits must be PC-relative");
     }
   }
 
@@ -203,8 +188,7 @@ bool MCAssembler::evaluateFixup(const MCFragment *DF, const MCFixup &Fixup,
 
   if (IsResolved && mc::isRelocRelocation(Fixup.getKind()))
     IsResolved = false;
-  IsResolved = getBackend().addReloc(*DF, Fixup, Target, Value, IsResolved);
-  getBackend().applyFixup(*DF, Fixup, Target, Contents, Value, IsResolved);
+  getBackend().applyFixup(F, Fixup, Target, Contents, Value, IsResolved);
   return true;
 }
 
@@ -212,9 +196,14 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
   assert(getBackendPtr() && "Requires assembler backend");
   switch (F.getKind()) {
   case MCFragment::FT_Data:
-    return cast<MCDataFragment>(F).getContents().size();
   case MCFragment::FT_Relaxable:
-    return cast<MCRelaxableFragment>(F).getContents().size();
+  case MCFragment::FT_LEB:
+  case MCFragment::FT_Dwarf:
+  case MCFragment::FT_DwarfFrame:
+  case MCFragment::FT_CVInlineLines:
+  case MCFragment::FT_CVDefRange:
+  case MCFragment::FT_PseudoProbe:
+    return cast<MCEncodedFragment>(F).getContents().size();
   case MCFragment::FT_Fill: {
     auto &FF = cast<MCFillFragment>(F);
     int64_t NumValues = 0;
@@ -233,9 +222,6 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
   case MCFragment::FT_Nops:
     return cast<MCNopsFragment>(F).getNumBytes();
 
-  case MCFragment::FT_LEB:
-    return cast<MCLEBFragment>(F).getContents().size();
-
   case MCFragment::FT_BoundaryAlign:
     return cast<MCBoundaryAlignFragment>(F).getSize();
 
@@ -291,17 +277,6 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
     }
     return Size;
   }
-
-  case MCFragment::FT_Dwarf:
-    return cast<MCDwarfLineAddrFragment>(F).getContents().size();
-  case MCFragment::FT_DwarfFrame:
-    return cast<MCDwarfCallFrameFragment>(F).getContents().size();
-  case MCFragment::FT_CVInlineLines:
-    return cast<MCCVInlineLineTableFragment>(F).getContents().size();
-  case MCFragment::FT_CVDefRange:
-    return cast<MCCVDefRangeFragment>(F).getContents().size();
-  case MCFragment::FT_PseudoProbe:
-    return cast<MCPseudoProbeAddrFragment>(F).getContents().size();
   }
 
   llvm_unreachable("invalid fragment kind");
@@ -559,6 +534,22 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
   ++stats::EmittedFragments;
 
   switch (F.getKind()) {
+  case MCFragment::FT_Data:
+  case MCFragment::FT_Relaxable:
+  case MCFragment::FT_LEB:
+  case MCFragment::FT_Dwarf:
+  case MCFragment::FT_DwarfFrame:
+  case MCFragment::FT_CVInlineLines:
+  case MCFragment::FT_CVDefRange:
+  case MCFragment::FT_PseudoProbe: {
+    if (F.getKind() == MCFragment::FT_Data)
+      ++stats::EmittedDataFragments;
+    else if (F.getKind() == MCFragment::FT_Relaxable)
+      ++stats::EmittedRelaxableFragments;
+    const auto &EF = cast<MCEncodedFragment>(F);
+    OS << StringRef(EF.getContents().data(), EF.getContents().size());
+    break;
+  }
   case MCFragment::FT_Align: {
     ++stats::EmittedAlignFragments;
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
@@ -605,16 +596,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_Data:
-    ++stats::EmittedDataFragments;
-    OS << cast<MCDataFragment>(F).getContents();
-    break;
-
-  case MCFragment::FT_Relaxable:
-    ++stats::EmittedRelaxableFragments;
-    OS << cast<MCRelaxableFragment>(F).getContents();
-    break;
-
   case MCFragment::FT_Fill: {
     ++stats::EmittedFillFragments;
     const MCFillFragment &FF = cast<MCFillFragment>(F);
@@ -690,12 +671,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_LEB: {
-    const MCLEBFragment &LF = cast<MCLEBFragment>(F);
-    OS << LF.getContents();
-    break;
-  }
-
   case MCFragment::FT_BoundaryAlign: {
     const MCBoundaryAlignFragment &BF = cast<MCBoundaryAlignFragment>(F);
     if (!Asm.getBackend().writeNopData(OS, FragmentSize, BF.getSubtargetInfo()))
@@ -720,31 +695,6 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     break;
   }
 
-  case MCFragment::FT_Dwarf: {
-    const MCDwarfLineAddrFragment &OF = cast<MCDwarfLineAddrFragment>(F);
-    OS << OF.getContents();
-    break;
-  }
-  case MCFragment::FT_DwarfFrame: {
-    const MCDwarfCallFrameFragment &CF = cast<MCDwarfCallFrameFragment>(F);
-    OS << CF.getContents();
-    break;
-  }
-  case MCFragment::FT_CVInlineLines: {
-    const auto &OF = cast<MCCVInlineLineTableFragment>(F);
-    OS << OF.getContents();
-    break;
-  }
-  case MCFragment::FT_CVDefRange: {
-    const auto &DRF = cast<MCCVDefRangeFragment>(F);
-    OS << DRF.getContents();
-    break;
-  }
-  case MCFragment::FT_PseudoProbe: {
-    const MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(F);
-    OS << PF.getContents();
-    break;
-  }
   }
 
   assert(OS.tell() - Start == FragmentSize &&
@@ -812,9 +762,10 @@ void MCAssembler::writeSectionData(raw_ostream &OS,
 
 void MCAssembler::layout() {
   assert(getBackendPtr() && "Expected assembler backend");
-  DEBUG_WITH_TYPE("mc-dump", {
-      errs() << "assembler backend - pre-layout\n--\n";
-      dump(); });
+  DEBUG_WITH_TYPE("mc-dump-pre", {
+    errs() << "assembler backend - pre-layout\n--\n";
+    dump();
+  });
 
   // Assign section ordinals.
   unsigned SectionIndex = 0;
@@ -844,14 +795,11 @@ void MCAssembler::layout() {
   this->HasLayout = true;
   for (MCSection &Sec : *this)
     layoutSection(Sec);
-  while (relaxOnce())
+  unsigned FirstStable = Sections.size();
+  while ((FirstStable = relaxOnce(FirstStable)) > 0)
     if (getContext().hadError())
       return;
 
-  DEBUG_WITH_TYPE("mc-dump", {
-      errs() << "assembler backend - post-relaxation\n--\n";
-      dump(); });
-
   // Some targets might want to adjust fragment offsets. If so, perform another
   // layout iteration.
   if (getBackend().finishLayout(*this))
@@ -875,69 +823,20 @@ void MCAssembler::layout() {
   // Evaluate and apply the fixups, generating relocation entries as necessary.
   for (MCSection &Sec : *this) {
     for (MCFragment &Frag : Sec) {
-      MutableArrayRef<MCFixup> Fixups;
-      MutableArrayRef<char> Contents;
-
-      // Process MCAlignFragment and MCEncodedFragmentWithFixups here.
-      switch (Frag.getKind()) {
-      default:
-        continue;
-      case MCFragment::FT_Align: {
-        MCAlignFragment &AF = cast<MCAlignFragment>(Frag);
-        // Insert fixup type for code alignment if the target define
-        // shouldInsertFixupForCodeAlign target hook.
-        if (Sec.useCodeAlign() && AF.hasEmitNops())
-          getBackend().shouldInsertFixupForCodeAlign(*this, AF);
-        continue;
-      }
-      case MCFragment::FT_Data: {
-        MCDataFragment &DF = cast<MCDataFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_Relaxable: {
-        MCRelaxableFragment &RF = cast<MCRelaxableFragment>(Frag);
-        Fixups = RF.getFixups();
-        Contents = RF.getContents();
-        break;
-      }
-      case MCFragment::FT_CVDefRange: {
-        MCCVDefRangeFragment &CF = cast<MCCVDefRangeFragment>(Frag);
-        Fixups = CF.getFixups();
-        Contents = CF.getContents();
-        break;
-      }
-      case MCFragment::FT_Dwarf: {
-        MCDwarfLineAddrFragment &DF = cast<MCDwarfLineAddrFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_DwarfFrame: {
-        MCDwarfCallFrameFragment &DF = cast<MCDwarfCallFrameFragment>(Frag);
-        Fixups = DF.getFixups();
-        Contents = DF.getContents();
-        break;
-      }
-      case MCFragment::FT_LEB: {
-        auto &LF = cast<MCLEBFragment>(Frag);
-        Fixups = LF.getFixups();
-        Contents = LF.getContents();
-        break;
-      }
-      case MCFragment::FT_PseudoProbe: {
-        MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(Frag);
-        Fixups = PF.getFixups();
-        Contents = PF.getContents();
-        break;
-      }
-      }
-      for (const MCFixup &Fixup : Fixups) {
-        uint64_t FixedValue;
-        MCValue Target;
-        evaluateFixup(&Frag, Fixup, Target, FixedValue,
-                      /*RecordReloc=*/true, Contents);
+      // Process fragments with fixups here.
+      if (auto *F = dyn_cast<MCEncodedFragment>(&Frag)) {
+        auto Contents = F->getContents();
+        for (MCFixup &Fixup : F->getFixups()) {
+          uint64_t FixedValue;
+          MCValue Target;
+          evaluateFixup(Frag, Fixup, Target, FixedValue,
+                        /*RecordReloc=*/true, Contents);
+        }
+      } else if (auto *AF = dyn_cast<MCAlignFragment>(&Frag)) {
+        // For RISC-V linker relaxation, an alignment relocation might be
+        // needed.
+        if (AF->hasEmitNops())
+          getBackend().shouldInsertFixupForCodeAlign(*this, *AF);
       }
     }
   }
@@ -953,54 +852,45 @@ void MCAssembler::Finish() {
   assert(PendingErrors.empty());
 }
 
-bool MCAssembler::fixupNeedsRelaxation(const MCFixup &Fixup,
-                                       const MCRelaxableFragment *DF) const {
+bool MCAssembler::fixupNeedsRelaxation(const MCRelaxableFragment &F,
+                                       const MCFixup &Fixup) const {
   assert(getBackendPtr() && "Expected assembler backend");
   MCValue Target;
   uint64_t Value;
-  bool Resolved = evaluateFixup(DF, const_cast<MCFixup &>(Fixup), Target, Value,
+  bool Resolved = evaluateFixup(F, const_cast<MCFixup &>(Fixup), Target, Value,
                                 /*RecordReloc=*/false, {});
   return getBackend().fixupNeedsRelaxationAdvanced(Fixup, Target, Value,
                                                    Resolved);
 }
 
-bool MCAssembler::fragmentNeedsRelaxation(const MCRelaxableFragment *F) const {
-  assert(getBackendPtr() && "Expected assembler backend");
+bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) {
+  assert(getEmitterPtr() &&
+         "Expected CodeEmitter defined for relaxInstruction");
   // If this inst doesn't ever need relaxation, ignore it. This occurs when we
   // are intentionally pushing out inst fragments, or because we relaxed a
   // previous instruction to one that doesn't need relaxation.
-  if (!getBackend().mayNeedRelaxation(F->getInst(), *F->getSubtargetInfo()))
+  if (!getBackend().mayNeedRelaxation(F.getInst(), *F.getSubtargetInfo()))
     return false;
 
-  for (const MCFixup &Fixup : F->getFixups())
-    if (fixupNeedsRelaxation(Fixup, F))
-      return true;
-
-  return false;
-}
-
-bool MCAssembler::relaxInstruction(MCRelaxableFragment &F) {
-  assert(getEmitterPtr() &&
-         "Expected CodeEmitter defined for relaxInstruction");
-  if (!fragmentNeedsRelaxation(&F))
+  bool DoRelax = false;
+  for (const MCFixup &Fixup : F.getFixups())
+    if ((DoRelax = fixupNeedsRelaxation(F, Fixup)))
+      break;
+  if (!DoRelax)
     return false;
 
   ++stats::RelaxedInstructions;
 
-  // FIXME-PERF: We could immediately lower out instructions if we can tell
-  // they are fully resolved, to avoid retesting on later passes.
-
-  // Relax the fragment.
-
   MCInst Relaxed = F.getInst();
   getBackend().relaxInstruction(Relaxed, *F.getSubtargetInfo());
 
   // Encode the new instruction.
   F.setInst(Relaxed);
-  F.getFixups().clear();
-  F.getContents().clear();
-  getEmitter().encodeInstruction(Relaxed, F.getContents(), F.getFixups(),
-                                 *F.getSubtargetInfo());
+  SmallVector<char, 16> Data;
+  SmallVector<MCFixup, 1> Fixups;
+  getEmitter().encodeInstruction(Relaxed, Data, Fixups, *F.getSubtargetInfo());
+  F.setContents(Data);
+  F.setFixups(Fixups);
   return true;
 }
 
@@ -1008,8 +898,7 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
   const unsigned OldSize = static_cast<unsigned>(LF.getContents().size());
   unsigned PadTo = OldSize;
   int64_t Value;
-  SmallVectorImpl<char> &Data = LF.getContents();
-  LF.getFixups().clear();
+  LF.clearFixups();
   // Use evaluateKnownAbsolute for Mach-O as a hack: .subsections_via_symbols
   // requires that .uleb128 A-B is foldable where A and B reside in different
   // fragments. This is used by __gcc_except_table.
@@ -1030,17 +919,18 @@ bool MCAssembler::relaxLEB(MCLEBFragment &LF) {
     if (UseZeroPad)
       Value = 0;
   }
-  Data.clear();
-  raw_svector_ostream OSE(Data);
+  uint8_t Data[16];
+  size_t Size = 0;
   // The compiler can generate EH table assembly that is impossible to assemble
   // without either adding padding to an LEB fragment or adding extra padding
   // to a later alignment fragment. To accommodate such tables, relaxation can
   // only increase an LEB fragment size here, not decrease it. See PR35809.
   if (LF.isSigned())
-    encodeSLEB128(Value, OSE, PadTo);
+    Size = encodeSLEB128(Value, Data, PadTo);
   else
-    encodeULEB128(Value, OSE, PadTo);
-  return OldSize != LF.getContents().size();
+    Size = encodeULEB128(Value, Data, PadTo);
+  LF.setContents({reinterpret_cast<char *>(Data), Size});
+  return OldSize != Size;
 }
 
 /// Check if the branch crosses the boundary.
@@ -1110,19 +1000,19 @@ bool MCAssembler::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF) {
     return WasRelaxed;
 
   MCContext &Context = getContext();
-  uint64_t OldSize = DF.getContents().size();
+  auto OldSize = DF.getContents().size();
   int64_t AddrDelta;
   bool Abs = DF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
   assert(Abs && "We created a line delta with an invalid expression");
   (void)Abs;
   int64_t LineDelta;
   LineDelta = DF.getLineDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  Data.clear();
-  DF.getFixups().clear();
+  SmallVector<char, 8> Data;
 
   MCDwarfLineAddr::encode(Context, getDWARFLinetableParams(), LineDelta,
                           AddrDelta, Data);
+  DF.setContents(Data);
+  DF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1141,12 +1031,11 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCDwarfCallFrameFragment &DF) {
     return false;
   }
 
-  SmallVectorImpl<char> &Data = DF.getContents();
-  uint64_t OldSize = Data.size();
-  Data.clear();
-  DF.getFixups().clear();
-
+  auto OldSize = DF.getContents().size();
+  SmallVector<char, 8> Data;
   MCDwarfFrameEmitter::encodeAdvanceLoc(Context, Value, Data);
+  DF.setContents(Data);
+  DF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1176,13 +1065,13 @@ bool MCAssembler::relaxPseudoProbeAddr(MCPseudoProbeAddrFragment &PF) {
   bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
   assert(Abs && "We created a pseudo probe with an invalid expression");
   (void)Abs;
-  SmallVectorImpl<char> &Data = PF.getContents();
-  Data.clear();
+  SmallVector<char, 8> Data;
   raw_svector_ostream OSE(Data);
-  PF.getFixups().clear();
 
   // AddrDelta is a signed integer
   encodeSLEB128(AddrDelta, OSE, OldSize);
+  PF.setContents(Data);
+  PF.clearFixups();
   return OldSize != Data.size();
 }
 
@@ -1229,17 +1118,15 @@ void MCAssembler::layoutSection(MCSection &Sec) {
   }
 }
 
-bool MCAssembler::relaxOnce() {
+unsigned MCAssembler::relaxOnce(unsigned FirstStable) {
   ++stats::RelaxationSteps;
   PendingErrors.clear();
 
-  // Size of fragments in one section can depend on the size of fragments in
-  // another. If any fragment has changed size, we have to re-layout (and
-  // as a result possibly further relax) all sections.
-  bool ChangedAny = false;
-  for (MCSection &Sec : *this) {
+  unsigned Res = 0;
+  for (unsigned I = 0; I != FirstStable; ++I) {
     // Assume each iteration finalizes at least one extra fragment. If the
     // layout does not converge after N+1 iterations, bail out.
+    auto &Sec = *Sections[I];
     auto MaxIter = Sec.curFragList()->Tail->getLayoutOrder() + 1;
     for (;;) {
       bool Changed = false;
@@ -1247,13 +1134,20 @@ bool MCAssembler::relaxOnce() {
         if (relaxFragment(F))
           Changed = true;
 
-      ChangedAny |= Changed;
-      if (!Changed || --MaxIter == 0)
+      if (!Changed)
+        break;
+      // If any fragment changed size, it might impact the layout of subsequent
+      // sections. Therefore, we must re-evaluate all sections.
+      FirstStable = Sections.size();
+      Res = I;
+      if (--MaxIter == 0)
         break;
       layoutSection(Sec);
     }
   }
-  return ChangedAny;
+  // The subsequent relaxOnce call only needs to visit Sections [0,Res) if no
+  // change occurred.
+  return Res;
 }
 
 void MCAssembler::reportError(SMLoc L, const Twine &Msg) const {
@@ -1273,31 +1167,26 @@ void MCAssembler::flushPendingErrors() const {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCAssembler::dump() const{
   raw_ostream &OS = errs();
-
-  OS << "<MCAssembler\n";
-  OS << "  Sections:[\n    ";
-  bool First = true;
+  DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> FragToSyms;
+  // Scan symbols and build a map of fragments to their corresponding symbols.
+  // For variable symbols, we don't want to call their getFragment, which might
+  // modify `Fragment`.
+  for (const MCSymbol &Sym : symbols())
+    if (!Sym.isVariable())
+      if (auto *F = Sym.getFragment())
+        FragToSyms.try_emplace(F).first->second.push_back(&Sym);
+
+  OS << "Sections:[";
   for (const MCSection &Sec : *this) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n    ";
-    Sec.dump();
-  }
-  OS << "],\n";
-  OS << "  Symbols:[";
-
-  First = true;
-  for (const MCSymbol &Sym : symbols()) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n           ";
-    OS << "(";
-    Sym.dump();
-    OS << ", Index:" << Sym.getIndex() << ", ";
-    OS << ")";
+    OS << '\n';
+    Sec.dump(&FragToSyms);
   }
-  OS << "]>\n";
+  OS << "\n]\n";
 }
 #endif
+
+SMLoc MCFixup::getLoc() const {
+  if (auto *E = getValue())
+    return E->getLoc();
+  return {};
+}
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index e78f5a082925..5d7914396e09 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -442,10 +442,13 @@ MCFragment *CodeViewContext::emitDefRange(
     MCObjectStreamer &OS,
     ArrayRef<std::pair<const MCSymbol *, const MCSymbol *>> Ranges,
     StringRef FixedSizePortion) {
+  // Store `Ranges` and `FixedSizePortion` in the context, returning references,
+  // as MCCVDefRangeFragment does not own these objects.
+  FixedSizePortion = MCCtx->allocateString(FixedSizePortion);
+  auto &Saved = DefRangeStorage.emplace_back(Ranges.begin(), Ranges.end());
   // Create and insert a fragment into the current section that will be encoded
   // later.
-  auto *F =
-      MCCtx->allocFragment<MCCVDefRangeFragment>(Ranges, FixedSizePortion);
+  auto *F = MCCtx->allocFragment<MCCVDefRangeFragment>(Saved, FixedSizePortion);
   OS.insert(F);
   return F;
 }
@@ -453,9 +456,8 @@ MCFragment *CodeViewContext::emitDefRange(
 static unsigned computeLabelDiff(const MCAssembler &Asm, const MCSymbol *Begin,
                                  const MCSymbol *End) {
   MCContext &Ctx = Asm.getContext();
-  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *BeginRef = MCSymbolRefExpr::create(Begin, Variant, Ctx),
-               *EndRef = MCSymbolRefExpr::create(End, Variant, Ctx);
+  const MCExpr *BeginRef = MCSymbolRefExpr::create(Begin, Ctx),
+               *EndRef = MCSymbolRefExpr::create(End, Ctx);
   const MCExpr *AddrDelta =
       MCBinaryExpr::create(MCBinaryExpr::Sub, EndRef, BeginRef, Ctx);
   int64_t Result;
@@ -508,8 +510,7 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
 
   MCCVFunctionInfo *SiteInfo = getCVFunctionInfo(Frag.SiteFuncId);
 
-  SmallVectorImpl<char> &Buffer = Frag.getContents();
-  Buffer.clear(); // Clear old contents if we went through relaxation.
+  SmallVector<char, 0> Buffer;
   for (const MCCVLoc &Loc : Locs) {
     // Exit early if our line table would produce an oversized InlineSiteSym
     // record. Account for the ChangeCodeLength annotation emitted after the
@@ -602,15 +603,14 @@ void CodeViewContext::encodeInlineLineTable(const MCAssembler &Asm,
 
   compressAnnotation(BinaryAnnotationsOpCode::ChangeCodeLength, Buffer);
   compressAnnotation(std::min(EndSymLength, LocAfterLength), Buffer);
+  Frag.setContents(Buffer);
 }
 
 void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
                                      MCCVDefRangeFragment &Frag) {
   MCContext &Ctx = Asm.getContext();
-  SmallVectorImpl<char> &Contents = Frag.getContents();
-  Contents.clear();
-  SmallVectorImpl<MCFixup> &Fixups = Frag.getFixups();
-  Fixups.clear();
+  SmallVector<char, 0> Contents;
+  SmallVector<MCFixup, 0> Fixups;
   raw_svector_ostream OS(Contents);
 
   // Compute all the sizes up front.
@@ -690,4 +690,7 @@ void CodeViewContext::encodeDefRange(const MCAssembler &Asm,
       GapStartOffset += GapSize + RangeSize;
     }
   }
+
+  Frag.setContents(Contents);
+  Frag.setFixups(Fixups);
 }
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 99c74f6f5c97..b0d1cb41fac1 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -14,13 +14,13 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeView.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCLabel.h"
 #include "llvm/MC/MCSectionCOFF.h"
@@ -74,6 +74,8 @@ MCContext::MCContext(const Triple &TheTriple, const MCAsmInfo *mai,
       CurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_IS_STMT, 0, 0),
       AutoReset(DoAutoReset), TargetOptions(TargetOpts) {
   SaveTempLabels = TargetOptions && TargetOptions->MCSaveTempLabels;
+  if (SaveTempLabels)
+    setUseNamesOnTempLabels(true);
   SecureLogFile = TargetOptions ? TargetOptions->AsSecureLogFile : "";
 
   if (SrcMgr && SrcMgr->getNumBuffers())
@@ -720,22 +722,52 @@ MCContext::getELFUniqueIDForEntsize(StringRef SectionName, unsigned Flags,
                                       : std::nullopt;
 }
 
-MCSectionGOFF *MCContext::getGOFFSection(StringRef Section, SectionKind Kind,
-                                         MCSection *Parent,
-                                         uint32_t Subsection) {
+template <typename TAttr>
+MCSectionGOFF *MCContext::getGOFFSection(SectionKind Kind, StringRef Name,
+                                         TAttr Attributes, MCSection *Parent,
+                                         bool IsVirtual) {
+  std::string UniqueName(Name);
+  if (Parent) {
+    UniqueName.append("/").append(Parent->getName());
+    if (auto *P = static_cast<MCSectionGOFF *>(Parent)->getParent())
+      UniqueName.append("/").append(P->getName());
+  }
   // Do the lookup. If we don't have a hit, return a new section.
-  auto [Iter, Inserted] = GOFFUniquingMap.try_emplace(Section.str());
-  if (!Inserted)
+  auto IterBool = GOFFUniquingMap.insert(std::make_pair(UniqueName, nullptr));
+  auto Iter = IterBool.first;
+  if (!IterBool.second)
     return Iter->second;
 
-  StringRef CachedName = Iter->first;
+  StringRef CachedName = StringRef(Iter->first.c_str(), Name.size());
   MCSectionGOFF *GOFFSection = new (GOFFAllocator.Allocate())
-      MCSectionGOFF(CachedName, Kind, Parent, Subsection);
+      MCSectionGOFF(CachedName, Kind, IsVirtual, Attributes,
+                    static_cast<MCSectionGOFF *>(Parent));
   Iter->second = GOFFSection;
   allocInitialFragment(*GOFFSection);
   return GOFFSection;
 }
 
+MCSectionGOFF *MCContext::getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::SDAttr SDAttributes) {
+  return getGOFFSection<GOFF::SDAttr>(Kind, Name, SDAttributes, nullptr,
+                                      /*IsVirtual=*/true);
+}
+
+MCSectionGOFF *MCContext::getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::EDAttr EDAttributes,
+                                         MCSection *Parent) {
+  return getGOFFSection<GOFF::EDAttr>(
+      Kind, Name, EDAttributes, Parent,
+      /*IsVirtual=*/EDAttributes.BindAlgorithm == GOFF::ESD_BA_Merge);
+}
+
+MCSectionGOFF *MCContext::getGOFFSection(SectionKind Kind, StringRef Name,
+                                         GOFF::PRAttr PRAttributes,
+                                         MCSection *Parent) {
+  return getGOFFSection<GOFF::PRAttr>(Kind, Name, PRAttributes, Parent,
+                                      /*IsVirtual=*/false);
+}
+
 MCSectionCOFF *MCContext::getCOFFSection(StringRef Section,
                                          unsigned Characteristics,
                                          StringRef COMDATSymName, int Selection,
diff --git a/llvm/lib/MC/MCDXContainerStreamer.cpp b/llvm/lib/MC/MCDXContainerStreamer.cpp
index a596c9a16d49..88c6c8b7e978 100644
--- a/llvm/lib/MC/MCDXContainerStreamer.cpp
+++ b/llvm/lib/MC/MCDXContainerStreamer.cpp
@@ -16,9 +16,6 @@
 
 using namespace llvm;
 
-void MCDXContainerStreamer::emitInstToData(const MCInst &,
-                                           const MCSubtargetInfo &) {}
-
 MCStreamer *llvm::createDXContainerStreamer(
     MCContext &Context, std::unique_ptr<MCAsmBackend> &&MAB,
     std::unique_ptr<MCObjectWriter> &&OW, std::unique_ptr<MCCodeEmitter> &&CE) {
diff --git a/llvm/lib/MC/MCDisassembler/Disassembler.h b/llvm/lib/MC/MCDisassembler/Disassembler.h
index 3cb2479d388f..4c34d86437b1 100644
--- a/llvm/lib/MC/MCDisassembler/Disassembler.h
+++ b/llvm/lib/MC/MCDisassembler/Disassembler.h
@@ -98,7 +98,7 @@ class LLVMDisasmContext {
         MAI(std::move(MAI)), MRI(std::move(MRI)), MSI(std::move(MSI)),
         MII(std::move(MII)), Ctx(std::move(Ctx)), DisAsm(std::move(DisAsm)),
         IP(std::move(IP)), Options(0), CommentStream(CommentsToEmit) {}
-  const std::string &getTripleName() const { return TripleName; }
+  StringRef getTripleName() const { return TripleName; }
   void *getDisInfo() const { return DisInfo; }
   int getTagType() const { return TagType; }
   LLVMOpInfoCallback getGetOpInfo() const { return GetOpInfo; }
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 5be14785d157..fc5e6de10b72 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -120,9 +120,8 @@ static inline const MCExpr *makeEndMinusStartExpr(MCContext &Ctx,
                                                   const MCSymbol &Start,
                                                   const MCSymbol &End,
                                                   int IntVal) {
-  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *Res = MCSymbolRefExpr::create(&End, Variant, Ctx);
-  const MCExpr *RHS = MCSymbolRefExpr::create(&Start, Variant, Ctx);
+  const MCExpr *Res = MCSymbolRefExpr::create(&End, Ctx);
+  const MCExpr *RHS = MCSymbolRefExpr::create(&Start, Ctx);
   const MCExpr *Res1 = MCBinaryExpr::create(MCBinaryExpr::Sub, Res, RHS, Ctx);
   const MCExpr *Res2 = MCConstantExpr::create(IntVal, Ctx);
   const MCExpr *Res3 = MCBinaryExpr::create(MCBinaryExpr::Sub, Res1, Res2, Ctx);
@@ -134,8 +133,7 @@ static inline const MCExpr *makeEndMinusStartExpr(MCContext &Ctx,
 //
 static inline const MCExpr *
 makeStartPlusIntExpr(MCContext &Ctx, const MCSymbol &Start, int IntVal) {
-  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *LHS = MCSymbolRefExpr::create(&Start, Variant, Ctx);
+  const MCExpr *LHS = MCSymbolRefExpr::create(&Start, Ctx);
   const MCExpr *RHS = MCConstantExpr::create(IntVal, Ctx);
   const MCExpr *Res = MCBinaryExpr::create(MCBinaryExpr::Add, LHS, RHS, Ctx);
   return Res;
@@ -2113,7 +2111,7 @@ struct CIEKey {
   unsigned LsdaEncoding = -1;
   bool IsSignalFrame = false;
   bool IsSimple = false;
-  unsigned RAReg = static_cast<unsigned>(UINT_MAX);
+  unsigned RAReg = UINT_MAX;
   bool IsBKeyFrame = false;
   bool IsMTETaggedFrame = false;
 };
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 5a514f12ce54..e84b91c5c41b 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
@@ -449,11 +448,14 @@ void MCELFStreamer::emitInstToData(const MCInst &Inst,
   // Emit instruction directly into data fragment.
   size_t FixupStartIndex = DF->getFixups().size();
   size_t CodeOffset = DF->getContents().size();
-  Assembler.getEmitter().encodeInstruction(Inst, DF->getContents(),
-                                           DF->getFixups(), STI);
-
-  auto Fixups = MutableArrayRef(DF->getFixups()).slice(FixupStartIndex);
-  for (auto &Fixup : Fixups) {
+  SmallVector<MCFixup, 1> Fixups;
+  Assembler.getEmitter().encodeInstruction(Inst, DF->getContentsForAppending(),
+                                           Fixups, STI);
+  DF->doneAppending();
+  if (!Fixups.empty())
+    DF->appendFixups(Fixups);
+
+  for (auto &Fixup : MutableArrayRef(DF->getFixups()).slice(FixupStartIndex)) {
     Fixup.setOffset(Fixup.getOffset() + CodeOffset);
     if (Fixup.isLinkerRelaxable()) {
       DF->setLinkerRelaxable();
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index 8919a2627cf6..79c3ec625cb7 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -91,7 +91,7 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
     Sym.print(OS, MAI);
 
     const MCSymbolRefExpr::VariantKind Kind = SRE.getKind();
-    if (Kind != MCSymbolRefExpr::VK_None) {
+    if (Kind) {
       if (!MAI) // should only be used by dump()
         OS << "@<variant " << Kind << '>';
       else if (MAI->useParensForSpecifier()) // ARM
@@ -179,7 +179,9 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
       return MAI->printSpecifierExpr(OS, SE);
     // Used by dump features like -show-inst. Regular MCAsmStreamer output must
     // set MAI.
-    OS << "specifier(" << SE.getSpecifier() << ',' << *SE.getSubExpr() << ')';
+    OS << "specifier(" << SE.getSpecifier() << ',';
+    SE.getSubExpr()->print(OS, nullptr);
+    OS << ')';
     return;
   }
   }
@@ -189,7 +191,7 @@ void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI,
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCExpr::dump() const {
-  dbgs() << *this;
+  print(dbgs(), nullptr);
   dbgs() << '\n';
 }
 #endif
@@ -215,16 +217,16 @@ const MCConstantExpr *MCConstantExpr::create(int64_t Value, MCContext &Ctx,
 
 /* *** */
 
-MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind,
+MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, Spec specifier,
                                  const MCAsmInfo *MAI, SMLoc Loc)
-    : MCExpr(MCExpr::SymbolRef, Loc, Kind), Symbol(Symbol) {
+    : MCExpr(MCExpr::SymbolRef, Loc, specifier), Symbol(Symbol) {
   assert(Symbol);
 }
 
 const MCSymbolRefExpr *MCSymbolRefExpr::create(const MCSymbol *Sym,
-                                               VariantKind Kind,
+                                               uint16_t specifier,
                                                MCContext &Ctx, SMLoc Loc) {
-  return new (Ctx) MCSymbolRefExpr(Sym, Kind, Ctx.getAsmInfo(), Loc);
+  return new (Ctx) MCSymbolRefExpr(Sym, specifier, Ctx.getAsmInfo(), Loc);
 }
 
 /* *** */
@@ -487,8 +489,7 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
       }
       return false;
     }
-    if (Sym.isVariable() && (Kind == MCSymbolRefExpr::VK_None || Layout) &&
-        !Sym.isWeakExternal()) {
+    if (Sym.isVariable() && (Kind == 0 || Layout) && !Sym.isWeakExternal()) {
       Sym.setIsResolving(true);
       auto _ = make_scope_exit([&] { Sym.setIsResolving(false); });
       bool IsMachO =
@@ -502,7 +503,7 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
       auto *A = Res.getAddSym();
       auto *B = Res.getSubSym();
       if (InSet || !(A && !B && A->isInSection())) {
-        if (Kind != MCSymbolRefExpr::VK_None) {
+        if (Kind) {
           if (Res.isAbsolute()) {
             Res = MCValue::get(&Sym, nullptr, 0, Kind);
             return true;
@@ -510,8 +511,8 @@ bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm,
           // If the reference has a variant kind, we can only handle expressions
           // which evaluate exactly to a single unadorned symbol. Attach the
           // original VariantKind to SymA of the result.
-          if (Res.getSpecifier() != MCSymbolRefExpr::VK_None ||
-              !Res.getAddSym() || Res.getSubSym() || Res.getConstant())
+          if (Res.getSpecifier() || !Res.getAddSym() || Res.getSubSym() ||
+              Res.getConstant())
             return false;
           Res.Specifier = Kind;
         }
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index aa4dec0a8e9d..5cf47e3325ff 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/MC/MCFragment.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
@@ -20,145 +19,90 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
+#include <type_traits>
 #include <utility>
 
 using namespace llvm;
 
+static_assert(std::is_trivially_destructible_v<MCDataFragment>,
+              "fragment classes must be trivially destructible");
+
 MCFragment::MCFragment(FragmentType Kind, bool HasInstructions)
     : Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false),
       LinkerRelaxable(false), AllowAutoPadding(false) {}
 
-void MCFragment::destroy() {
-  switch (Kind) {
-    case FT_Align:
-      cast<MCAlignFragment>(this)->~MCAlignFragment();
-      return;
-    case FT_Data:
-      cast<MCDataFragment>(this)->~MCDataFragment();
-      return;
-    case FT_Fill:
-      cast<MCFillFragment>(this)->~MCFillFragment();
-      return;
-    case FT_Nops:
-      cast<MCNopsFragment>(this)->~MCNopsFragment();
-      return;
-    case FT_Relaxable:
-      cast<MCRelaxableFragment>(this)->~MCRelaxableFragment();
-      return;
-    case FT_Org:
-      cast<MCOrgFragment>(this)->~MCOrgFragment();
-      return;
-    case FT_Dwarf:
-      cast<MCDwarfLineAddrFragment>(this)->~MCDwarfLineAddrFragment();
-      return;
-    case FT_DwarfFrame:
-      cast<MCDwarfCallFrameFragment>(this)->~MCDwarfCallFrameFragment();
-      return;
-    case FT_LEB:
-      cast<MCLEBFragment>(this)->~MCLEBFragment();
-      return;
-    case FT_BoundaryAlign:
-      cast<MCBoundaryAlignFragment>(this)->~MCBoundaryAlignFragment();
-      return;
-    case FT_SymbolId:
-      cast<MCSymbolIdFragment>(this)->~MCSymbolIdFragment();
-      return;
-    case FT_CVInlineLines:
-      cast<MCCVInlineLineTableFragment>(this)->~MCCVInlineLineTableFragment();
-      return;
-    case FT_CVDefRange:
-      cast<MCCVDefRangeFragment>(this)->~MCCVDefRangeFragment();
-      return;
-    case FT_PseudoProbe:
-      cast<MCPseudoProbeAddrFragment>(this)->~MCPseudoProbeAddrFragment();
-      return;
-  }
-}
-
 const MCSymbol *MCFragment::getAtom() const {
   return cast<MCSectionMachO>(Parent)->getAtom(LayoutOrder);
 }
 
-// Debugging methods
-
-namespace llvm {
-
-raw_ostream &operator<<(raw_ostream &OS, const MCFixup &AF) {
-  OS << "<MCFixup" << " Offset:" << AF.getOffset()
-     << " Value:" << *AF.getValue()
-     << " Kind:" << AF.getKind() << ">";
-  return OS;
-}
-
-} // end namespace llvm
-
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void MCFragment::dump() const {
   raw_ostream &OS = errs();
 
-  OS << "<";
+  OS << Offset << ' ';
   switch (getKind()) {
-  case MCFragment::FT_Align: OS << "MCAlignFragment"; break;
-  case MCFragment::FT_Data:  OS << "MCDataFragment"; break;
-  case MCFragment::FT_Fill:  OS << "MCFillFragment"; break;
-  case MCFragment::FT_Nops:
-    OS << "MCFNopsFragment";
-    break;
-  case MCFragment::FT_Relaxable:  OS << "MCRelaxableFragment"; break;
-  case MCFragment::FT_Org:   OS << "MCOrgFragment"; break;
-  case MCFragment::FT_Dwarf: OS << "MCDwarfFragment"; break;
-  case MCFragment::FT_DwarfFrame: OS << "MCDwarfCallFrameFragment"; break;
-  case MCFragment::FT_LEB:   OS << "MCLEBFragment"; break;
-  case MCFragment::FT_BoundaryAlign: OS<<"MCBoundaryAlignFragment"; break;
-  case MCFragment::FT_SymbolId:    OS << "MCSymbolIdFragment"; break;
-  case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break;
-  case MCFragment::FT_CVDefRange: OS << "MCCVDefRangeTableFragment"; break;
-  case MCFragment::FT_PseudoProbe:
-    OS << "MCPseudoProbe";
-    break;
+    // clang-format off
+  case MCFragment::FT_Align:         OS << "Align"; break;
+  case MCFragment::FT_Data:          OS << "Data"; break;
+  case MCFragment::FT_Fill:          OS << "Fill"; break;
+  case MCFragment::FT_Nops:          OS << "Nops"; break;
+  case MCFragment::FT_Relaxable:     OS << "Relaxable"; break;
+  case MCFragment::FT_Org:           OS << "Org"; break;
+  case MCFragment::FT_Dwarf:         OS << "Dwarf"; break;
+  case MCFragment::FT_DwarfFrame:    OS << "DwarfCallFrame"; break;
+  case MCFragment::FT_LEB:           OS << "LEB"; break;
+  case MCFragment::FT_BoundaryAlign: OS<<"BoundaryAlign"; break;
+  case MCFragment::FT_SymbolId:      OS << "SymbolId"; break;
+  case MCFragment::FT_CVInlineLines: OS << "CVInlineLineTable"; break;
+  case MCFragment::FT_CVDefRange:    OS << "CVDefRangeTable"; break;
+  case MCFragment::FT_PseudoProbe:   OS << "PseudoProbe"; break;
+    // clang-format on
   }
 
-  OS << "<MCFragment " << (const void *)this << " LayoutOrder:" << LayoutOrder
-     << " Offset:" << Offset << " HasInstructions:" << hasInstructions();
   if (const auto *EF = dyn_cast<MCEncodedFragment>(this))
-    OS << " BundlePadding:" << static_cast<unsigned>(EF->getBundlePadding());
-  OS << ">";
+    if (auto Pad = static_cast<unsigned>(EF->getBundlePadding()))
+      OS << " BundlePadding:" << Pad;
+
+  auto printFixups = [&](llvm::ArrayRef<MCFixup> Fixups) {
+    if (Fixups.empty())
+      return;
+    for (auto [I, F] : llvm::enumerate(Fixups)) {
+      OS << "\n  Fixup @" << F.getOffset() << " Value:";
+      F.getValue()->print(OS, nullptr);
+      OS << " Kind:" << F.getKind();
+    }
+  };
 
   switch (getKind()) {
   case MCFragment::FT_Align: {
     const auto *AF = cast<MCAlignFragment>(this);
+    OS << " Align:" << AF->getAlignment().value() << " Value:" << AF->getValue()
+       << " ValueSize:" << AF->getValueSize()
+       << " MaxBytesToEmit:" << AF->getMaxBytesToEmit();
     if (AF->hasEmitNops())
-      OS << " (emit nops)";
-    OS << "\n       ";
-    OS << " Alignment:" << AF->getAlignment().value()
-       << " Value:" << AF->getValue() << " ValueSize:" << AF->getValueSize()
-       << " MaxBytesToEmit:" << AF->getMaxBytesToEmit() << ">";
+      OS << " Nops";
     break;
   }
   case MCFragment::FT_Data:  {
-    const auto *DF = cast<MCDataFragment>(this);
-    OS << "\n       ";
-    OS << " Contents:[";
-    const SmallVectorImpl<char> &Contents = DF->getContents();
+    const auto *F = cast<MCDataFragment>(this);
+    if (F->isLinkerRelaxable())
+      OS << " LinkerRelaxable";
+    auto Contents = F->getContents();
+    OS << " Size:" << Contents.size() << " [";
     for (unsigned i = 0, e = Contents.size(); i != e; ++i) {
       if (i) OS << ",";
-      OS << hexdigit((Contents[i] >> 4) & 0xF) << hexdigit(Contents[i] & 0xF);
-    }
-    OS << "] (" << Contents.size() << " bytes)";
-
-    if (DF->getFixups().size()) {
-      OS << ",\n       ";
-      OS << " Fixups:[";
-      interleave(DF->getFixups(), OS, ",\n                ");
-      OS << "]";
+      OS << format("%02x", uint8_t(Contents[i]));
     }
+    OS << ']';
+    printFixups(F->getFixups());
     break;
   }
   case MCFragment::FT_Fill:  {
     const auto *FF = cast<MCFillFragment>(this);
     OS << " Value:" << static_cast<unsigned>(FF->getValue())
        << " ValueSize:" << static_cast<unsigned>(FF->getValueSize())
-       << " NumValues:" << FF->getNumValues();
+       << " NumValues:";
+    FF->getNumValues().print(OS, nullptr);
     break;
   }
   case MCFragment::FT_Nops: {
@@ -169,41 +113,40 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_Relaxable:  {
     const auto *F = cast<MCRelaxableFragment>(this);
-    OS << "\n       ";
-    OS << " Inst:";
+    OS << " Size:" << F->getContents().size() << ' ';
     F->getInst().dump_pretty(OS);
-    OS << " (" << F->getContents().size() << " bytes)";
+    printFixups(F->getFixups());
     break;
   }
   case MCFragment::FT_Org:  {
     const auto *OF = cast<MCOrgFragment>(this);
-    OS << "\n       ";
-    OS << " Offset:" << OF->getOffset()
-       << " Value:" << static_cast<unsigned>(OF->getValue());
+    OS << " Offset:";
+    OF->getOffset().print(OS, nullptr);
+    OS << " Value:" << static_cast<unsigned>(OF->getValue());
     break;
   }
   case MCFragment::FT_Dwarf:  {
     const auto *OF = cast<MCDwarfLineAddrFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << OF->getAddrDelta()
-       << " LineDelta:" << OF->getLineDelta();
+    OS << " AddrDelta:";
+    OF->getAddrDelta().print(OS, nullptr);
+    OS << " LineDelta:" << OF->getLineDelta();
     break;
   }
   case MCFragment::FT_DwarfFrame:  {
     const auto *CF = cast<MCDwarfCallFrameFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << CF->getAddrDelta();
+    OS << " AddrDelta:";
+    CF->getAddrDelta().print(OS, nullptr);
     break;
   }
   case MCFragment::FT_LEB: {
     const auto *LF = cast<MCLEBFragment>(this);
-    OS << "\n       ";
-    OS << " Value:" << LF->getValue() << " Signed:" << LF->isSigned();
+    OS << " Value:";
+    LF->getValue().print(OS, nullptr);
+    OS << " Signed:" << LF->isSigned();
     break;
   }
   case MCFragment::FT_BoundaryAlign: {
     const auto *BF = cast<MCBoundaryAlignFragment>(this);
-    OS << "\n       ";
     OS << " BoundarySize:" << BF->getAlignment().value()
        << " LastFragment:" << BF->getLastFragment()
        << " Size:" << BF->getSize();
@@ -211,19 +154,17 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_SymbolId: {
     const auto *F = cast<MCSymbolIdFragment>(this);
-    OS << "\n       ";
     OS << " Sym:" << F->getSymbol();
     break;
   }
   case MCFragment::FT_CVInlineLines: {
     const auto *F = cast<MCCVInlineLineTableFragment>(this);
-    OS << "\n       ";
     OS << " Sym:" << *F->getFnStartSym();
     break;
   }
   case MCFragment::FT_CVDefRange: {
     const auto *F = cast<MCCVDefRangeFragment>(this);
-    OS << "\n       ";
+    OS << "\n   ";
     for (std::pair<const MCSymbol *, const MCSymbol *> RangeStartEnd :
          F->getRanges()) {
       OS << " RangeStart:" << RangeStartEnd.first;
@@ -233,11 +174,10 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   }
   case MCFragment::FT_PseudoProbe: {
     const auto *OF = cast<MCPseudoProbeAddrFragment>(this);
-    OS << "\n       ";
-    OS << " AddrDelta:" << OF->getAddrDelta();
+    OS << " AddrDelta:";
+    OF->getAddrDelta().print(OS, nullptr);
     break;
   }
   }
-  OS << ">";
 }
 #endif
diff --git a/llvm/lib/MC/MCGOFFStreamer.cpp b/llvm/lib/MC/MCGOFFStreamer.cpp
index 2b6d5c8e75a7..b7021915e7b7 100644
--- a/llvm/lib/MC/MCGOFFStreamer.cpp
+++ b/llvm/lib/MC/MCGOFFStreamer.cpp
@@ -15,12 +15,32 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCGOFFObjectWriter.h"
 #include "llvm/MC/TargetRegistry.h"
 
 using namespace llvm;
 
 MCGOFFStreamer::~MCGOFFStreamer() {}
 
+GOFFObjectWriter &MCGOFFStreamer::getWriter() {
+  return static_cast<GOFFObjectWriter &>(getAssembler().getWriter());
+}
+
+// Make sure that all section are registered in the correct order.
+static void registerSectionHierarchy(MCAssembler &Asm, MCSectionGOFF *Section) {
+  if (Section->isRegistered())
+    return;
+  if (Section->getParent())
+    registerSectionHierarchy(Asm, Section->getParent());
+  Asm.registerSection(*Section);
+}
+
+void MCGOFFStreamer::changeSection(MCSection *Section, uint32_t Subsection) {
+  registerSectionHierarchy(getAssembler(),
+                           static_cast<MCSectionGOFF *>(Section));
+  MCObjectStreamer::changeSection(Section, Subsection);
+}
+
 MCStreamer *llvm::createGOFFStreamer(MCContext &Context,
                                      std::unique_ptr<MCAsmBackend> &&MAB,
                                      std::unique_ptr<MCObjectWriter> &&OW,
diff --git a/llvm/lib/MC/MCInst.cpp b/llvm/lib/MC/MCInst.cpp
index 832d25060f88..46a6a18e1596 100644
--- a/llvm/lib/MC/MCInst.cpp
+++ b/llvm/lib/MC/MCInst.cpp
@@ -8,6 +8,8 @@
 
 #include "llvm/MC/MCInst.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -18,14 +20,14 @@
 
 using namespace llvm;
 
-void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
+void MCOperand::print(raw_ostream &OS, const MCContext *Ctx) const {
   OS << "<MCOperand ";
   if (!isValid())
     OS << "INVALID";
   else if (isReg()) {
     OS << "Reg:";
-    if (RegInfo)
-      OS << RegInfo->getName(getReg());
+    if (Ctx && Ctx->getRegisterInfo())
+      OS << Ctx->getRegisterInfo()->getName(getReg());
     else
       OS << getReg();
   } else if (isImm())
@@ -35,11 +37,15 @@ void MCOperand::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
   else if (isDFPImm())
     OS << "DFPImm:" << bit_cast<double>(getDFPImm());
   else if (isExpr()) {
-    OS << "Expr:" << *getExpr();
+    OS << "Expr:";
+    if (Ctx)
+      Ctx->getAsmInfo()->printExpr(OS, *getExpr());
+    else
+      getExpr()->print(OS, nullptr);
   } else if (isInst()) {
     OS << "Inst:(";
     if (const auto *Inst = getInst())
-      Inst->print(OS, RegInfo);
+      Inst->print(OS, Ctx);
     else
       OS << "NULL";
     OS << ")";
@@ -62,7 +68,7 @@ bool MCOperand::isBareSymbolRef() const {
   const MCExpr *Expr = getExpr();
   MCExpr::ExprKind Kind = getExpr()->getKind();
   return Kind == MCExpr::SymbolRef &&
-    cast<MCSymbolRefExpr>(Expr)->getKind() == MCSymbolRefExpr::VK_None;
+         cast<MCSymbolRefExpr>(Expr)->getSpecifier() == 0;
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -72,24 +78,23 @@ LLVM_DUMP_METHOD void MCOperand::dump() const {
 }
 #endif
 
-void MCInst::print(raw_ostream &OS, const MCRegisterInfo *RegInfo) const {
+void MCInst::print(raw_ostream &OS, const MCContext *Ctx) const {
   OS << "<MCInst " << getOpcode();
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
     OS << " ";
-    getOperand(i).print(OS, RegInfo);
+    getOperand(i).print(OS, Ctx);
   }
   OS << ">";
 }
 
 void MCInst::dump_pretty(raw_ostream &OS, const MCInstPrinter *Printer,
-                         StringRef Separator,
-                         const MCRegisterInfo *RegInfo) const {
+                         StringRef Separator, const MCContext *Ctx) const {
   StringRef InstName = Printer ? Printer->getOpcodeName(getOpcode()) : "";
-  dump_pretty(OS, InstName, Separator, RegInfo);
+  dump_pretty(OS, InstName, Separator, Ctx);
 }
 
 void MCInst::dump_pretty(raw_ostream &OS, StringRef Name, StringRef Separator,
-                         const MCRegisterInfo *RegInfo) const {
+                         const MCContext *Ctx) const {
   OS << "<MCInst #" << getOpcode();
 
   // Show the instruction opcode name if we have it.
@@ -98,7 +103,7 @@ void MCInst::dump_pretty(raw_ostream &OS, StringRef Name, StringRef Separator,
 
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
     OS << Separator;
-    getOperand(i).print(OS, RegInfo);
+    getOperand(i).print(OS, Ctx);
   }
   OS << ">";
 }
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 4cf602b7e456..a54dfb9cbaea 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -59,8 +58,6 @@ class MCMachOStreamer : public MCObjectStreamer {
   /// labels in the middle of the section.
   DenseMap<const MCSection*, bool> HasSectionLabel;
 
-  void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &STI) override;
-
   void emitDataRegion(MachO::DataRegionType Kind);
   void emitDataRegionEnd();
 
@@ -423,23 +420,6 @@ void MCMachOStreamer::emitTBSSSymbol(MCSection *Section, MCSymbol *Symbol,
   emitZerofill(Section, Symbol, Size, ByteAlignment);
 }
 
-void MCMachOStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCMachOStreamer::finishImpl() {
   emitFrames(&getAssembler().getBackend());
 
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index 9ad56aaf05bc..0069d12dce39 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -13,6 +13,7 @@
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCGOFFAttributes.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionDXContainer.h"
@@ -546,18 +547,51 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
 }
 
 void MCObjectFileInfo::initGOFFMCObjectFileInfo(const Triple &T) {
-  TextSection = Ctx->getGOFFSection(".text", SectionKind::getText(), nullptr);
-  BSSSection = Ctx->getGOFFSection(".bss", SectionKind::getBSS(), nullptr);
-  PPA1Section = Ctx->getGOFFSection(".ppa1", SectionKind::getMetadata(),
-                                    TextSection, GOFF::SK_PPA1);
-  PPA2Section = Ctx->getGOFFSection(".ppa2", SectionKind::getMetadata(),
-                                    TextSection, GOFF::SK_PPA2);
-
-  PPA2ListSection =
-      Ctx->getGOFFSection(".ppa2list", SectionKind::getData(), nullptr);
-
-  ADASection = Ctx->getGOFFSection(".ada", SectionKind::getData(), nullptr);
-  IDRLSection = Ctx->getGOFFSection("B_IDRL", SectionKind::getData(), nullptr);
+  MCSectionGOFF *RootSDSection = Ctx->getGOFFSection(
+      SectionKind::getMetadata(), "#C",
+      GOFF::SDAttr{GOFF::ESD_TA_Rent, GOFF::ESD_BSC_Section});
+
+  MCSectionGOFF *ADAEDSection = Ctx->getGOFFSection(
+      SectionKind::getMetadata(), GOFF::CLASS_WSA,
+      GOFF::EDAttr{false, GOFF::ESD_RMODE_64, GOFF::ESD_NS_Parts,
+                   GOFF::ESD_TS_ByteOriented, GOFF::ESD_BA_Merge,
+                   GOFF::ESD_LB_Deferred, GOFF::ESD_RQ_1,
+                   GOFF::ESD_ALIGN_Quadword, 0},
+      RootSDSection);
+  ADASection = Ctx->getGOFFSection(SectionKind::getData(), "#S",
+                                   GOFF::PRAttr{false, GOFF::ESD_EXE_DATA,
+                                                GOFF::ESD_LT_XPLink,
+                                                GOFF::ESD_BSC_Section, 0},
+                                   ADAEDSection);
+
+  TextSection = Ctx->getGOFFSection(
+      SectionKind::getText(), GOFF::CLASS_CODE,
+      GOFF::EDAttr{true, GOFF::ESD_RMODE_64, GOFF::ESD_NS_NormalName,
+                   GOFF::ESD_TS_ByteOriented, GOFF::ESD_BA_Concatenate,
+                   GOFF::ESD_LB_Initial, GOFF::ESD_RQ_0,
+                   GOFF::ESD_ALIGN_Doubleword, 0},
+      RootSDSection);
+
+  MCSectionGOFF *PPA2ListEDSection = Ctx->getGOFFSection(
+      SectionKind::getMetadata(), GOFF::CLASS_PPA2,
+      GOFF::EDAttr{true, GOFF::ESD_RMODE_64, GOFF::ESD_NS_Parts,
+                   GOFF::ESD_TS_ByteOriented, GOFF::ESD_BA_Merge,
+                   GOFF::ESD_LB_Initial, GOFF::ESD_RQ_0,
+                   GOFF::ESD_ALIGN_Doubleword, 0},
+      RootSDSection);
+  PPA2ListSection = Ctx->getGOFFSection(SectionKind::getData(), ".&ppa2",
+                                        GOFF::PRAttr{true, GOFF::ESD_EXE_DATA,
+                                                     GOFF::ESD_LT_OS,
+                                                     GOFF::ESD_BSC_Section, 0},
+                                        PPA2ListEDSection);
+
+  IDRLSection = Ctx->getGOFFSection(
+      SectionKind::getData(), "B_IDRL",
+      GOFF::EDAttr{true, GOFF::ESD_RMODE_64, GOFF::ESD_NS_NormalName,
+                   GOFF::ESD_TS_Structured, GOFF::ESD_BA_Concatenate,
+                   GOFF::ESD_LB_NoLoad, GOFF::ESD_RQ_0,
+                   GOFF::ESD_ALIGN_Doubleword, 0},
+      RootSDSection);
 }
 
 void MCObjectFileInfo::initCOFFMCObjectFileInfo(const Triple &T) {
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index e959a242dfcf..bd1a77467679 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -63,28 +63,14 @@ void MCObjectStreamer::resolvePendingFixups() {
     PendingFixup.Fixup.setOffset(PendingFixup.Sym->getOffset() +
                                  PendingFixup.Fixup.getOffset());
 
-    // If the location symbol to relocate is in MCEncodedFragmentWithFixups,
+    // If the location symbol to relocate is in MCEncodedFragment,
     // put the Fixup into location symbol's fragment. Otherwise
     // put into PendingFixup.DF
     MCFragment *SymFragment = PendingFixup.Sym->getFragment();
-    switch (SymFragment->getKind()) {
-    case MCFragment::FT_Relaxable:
-    case MCFragment::FT_Dwarf:
-    case MCFragment::FT_PseudoProbe:
-      cast<MCEncodedFragmentWithFixups<8, 1>>(SymFragment)
-          ->getFixups()
-          .push_back(PendingFixup.Fixup);
-      break;
-    case MCFragment::FT_Data:
-    case MCFragment::FT_CVDefRange:
-      cast<MCEncodedFragmentWithFixups<32, 4>>(SymFragment)
-          ->getFixups()
-          .push_back(PendingFixup.Fixup);
-      break;
-    default:
-      PendingFixup.DF->getFixups().push_back(PendingFixup.Fixup);
-      break;
-    }
+    if (auto *F = dyn_cast<MCEncodedFragment>(SymFragment))
+      F->addFixup(PendingFixup.Fixup);
+    else
+      PendingFixup.DF->addFixup(PendingFixup.Fixup);
   }
   PendingFixups.clear();
 }
@@ -201,8 +187,8 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
     emitIntValue(AbsValue, Size);
     return;
   }
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixup::getDataKindForSize(Size), Loc));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               MCFixup::getDataKindForSize(Size)));
   DF->appendContents(Size, 0);
 }
 
@@ -390,6 +376,22 @@ void MCObjectStreamer::emitInstructionImpl(const MCInst &Inst,
   emitInstToFragment(Inst, STI);
 }
 
+void MCObjectStreamer::emitInstToData(const MCInst &Inst,
+                                      const MCSubtargetInfo &STI) {
+  MCDataFragment *DF = getOrCreateDataFragment();
+  SmallVector<MCFixup, 1> Fixups;
+  SmallString<256> Code;
+  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
+
+  auto CodeOffset = DF->getContents().size();
+  for (MCFixup &Fixup : Fixups)
+    Fixup.setOffset(Fixup.getOffset() + CodeOffset);
+  if (!Fixups.empty())
+    DF->appendFixups(Fixups);
+  DF->setHasInstructions(STI);
+  DF->appendContents(Code);
+}
+
 void MCObjectStreamer::emitInstToFragment(const MCInst &Inst,
                                           const MCSubtargetInfo &STI) {
   // Always create a new, separate fragment here, because its size can change
@@ -398,8 +400,11 @@ void MCObjectStreamer::emitInstToFragment(const MCInst &Inst,
       getContext().allocFragment<MCRelaxableFragment>(Inst, STI);
   insert(IF);
 
-  getAssembler().getEmitter().encodeInstruction(Inst, IF->getContents(),
-                                                IF->getFixups(), STI);
+  SmallVector<MCFixup, 1> Fixups;
+  getAssembler().getEmitter().encodeInstruction(
+      Inst, IF->getContentsForAppending(), Fixups, STI);
+  IF->doneAppending();
+  IF->appendFixups(Fixups);
 }
 
 #ifndef NDEBUG
@@ -436,9 +441,8 @@ void MCObjectStreamer::emitDwarfLocDirective(unsigned FileNo, unsigned Line,
 static const MCExpr *buildSymbolDiff(MCObjectStreamer &OS, const MCSymbol *A,
                                      const MCSymbol *B, SMLoc Loc) {
   MCContext &Context = OS.getContext();
-  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
-  const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
+  const MCExpr *ARef = MCSymbolRefExpr::create(A, Context);
+  const MCExpr *BRef = MCSymbolRefExpr::create(B, Context);
   const MCExpr *AddrDelta =
       MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context, Loc);
   return AddrDelta;
@@ -692,8 +696,7 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
   if (OffsetVal.isAbsolute()) {
     if (OffsetVal.getConstant() < 0)
       return std::make_pair(false, std::string(".reloc offset is negative"));
-    DF->getFixups().push_back(
-        MCFixup::create(OffsetVal.getConstant(), Expr, Kind, Loc));
+    DF->addFixup(MCFixup::create(OffsetVal.getConstant(), Expr, Kind));
     return std::nullopt;
   }
   if (OffsetVal.getSubSym())
@@ -709,14 +712,13 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name,
     if (Error != std::nullopt)
       return Error;
 
-    DF->getFixups().push_back(
-        MCFixup::create(SymbolOffset + OffsetVal.getConstant(),
-                        Expr, Kind, Loc));
+    DF->addFixup(
+        MCFixup::create(SymbolOffset + OffsetVal.getConstant(), Expr, Kind));
     return std::nullopt;
   }
 
   PendingFixups.emplace_back(
-      &Symbol, DF, MCFixup::create(OffsetVal.getConstant(), Expr, Kind, Loc));
+      &Symbol, DF, MCFixup::create(OffsetVal.getConstant(), Expr, Kind));
   return std::nullopt;
 }
 
diff --git a/llvm/lib/MC/MCObjectWriter.cpp b/llvm/lib/MC/MCObjectWriter.cpp
index 86be52d57ef7..7cbd0b489d9f 100644
--- a/llvm/lib/MC/MCObjectWriter.cpp
+++ b/llvm/lib/MC/MCObjectWriter.cpp
@@ -10,7 +10,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 namespace llvm {
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index cb33f5dfbfd3..0bf2b065cac6 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1215,17 +1215,14 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     if (SymbolName.empty())
       return Error(getLexer().getLoc(), "expected a symbol reference");
 
-    MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-
-    // Lookup the symbol variant if used.
+    // Lookup the @specifier if used.
+    uint16_t Spec = 0;
     if (!Split.second.empty()) {
-      auto MaybeVariant = MAI.getSpecifierForName(Split.second);
-      if (MaybeVariant) {
+      auto MaybeSpecifier = MAI.getSpecifierForName(Split.second);
+      if (MaybeSpecifier) {
         SymbolName = Split.first;
-        Variant = MCSymbolRefExpr::VariantKind(*MaybeVariant);
-      } else if (MAI.doesAllowAtInName()) {
-        Variant = MCSymbolRefExpr::VK_None;
-      } else {
+        Spec = *MaybeSpecifier;
+      } else if (!MAI.doesAllowAtInName()) {
         return Error(SMLoc::getFromPointer(Split.second.begin()),
                      "invalid variant '" + Split.second + "'");
       }
@@ -1240,11 +1237,11 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     // semantics in the face of reassignment.
     if (Sym->isVariable()) {
       auto V = Sym->getVariableValue();
-      bool DoInline = isa<MCConstantExpr>(V) && !Variant;
+      bool DoInline = isa<MCConstantExpr>(V) && !Spec;
       if (auto TV = dyn_cast<MCTargetExpr>(V))
         DoInline = TV->inlineAssignedExpr();
       if (DoInline) {
-        if (Variant)
+        if (Spec)
           return Error(EndLoc, "unexpected modifier on variable reference");
         Res = Sym->getVariableValue();
         return false;
@@ -1252,7 +1249,7 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
     }
 
     // Otherwise create a symbol ref.
-    Res = MCSymbolRefExpr::create(Sym, Variant, getContext(), FirstTokenLoc);
+    Res = MCSymbolRefExpr::create(Sym, Spec, getContext(), FirstTokenLoc);
     return false;
   }
   case AsmToken::BigNum:
@@ -1268,18 +1265,18 @@ bool AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
       StringRef IDVal = getTok().getString();
       // Lookup the symbol variant if used.
       std::pair<StringRef, StringRef> Split = IDVal.split('@');
-      MCSymbolRefExpr::VariantKind Spec = MCSymbolRefExpr::VK_None;
+      uint16_t Spec = 0;
       if (Split.first.size() != IDVal.size()) {
         auto MaybeSpec = MAI.getSpecifierForName(Split.second);
         if (!MaybeSpec)
           return TokError("invalid variant '" + Split.second + "'");
         IDVal = Split.first;
-        Spec = MCSymbolRefExpr::VariantKind(*MaybeSpec);
+        Spec = *MaybeSpec;
       }
       if (IDVal == "f" || IDVal == "b") {
         MCSymbol *Sym =
             Ctx.getDirectionalLocalSymbol(IntVal, IDVal == "b");
-        Res = MCSymbolRefExpr::create(Sym, Spec, getContext());
+        Res = MCSymbolRefExpr::create(Sym, Spec, getContext(), Loc);
         if (IDVal == "b" && Sym->isUndefined())
           return Error(Loc, "directional label undefined");
         DirLabels.push_back(std::make_tuple(Loc, CppHashInfo, Sym));
@@ -1361,13 +1358,14 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
   case MCExpr::SymbolRef: {
     const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(E);
 
-    if (SRE->getKind() != MCSymbolRefExpr::VK_None) {
+    if (SRE->getSpecifier()) {
       TokError("invalid variant on expression '" + getTok().getIdentifier() +
                "' (already modified)");
       return E;
     }
 
-    return MCSymbolRefExpr::create(&SRE->getSymbol(), Spec, getContext());
+    return MCSymbolRefExpr::create(&SRE->getSymbol(), Spec, getContext(),
+                                   SRE->getLoc());
   }
 
   case MCExpr::Unary: {
@@ -1375,7 +1373,8 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
     const MCExpr *Sub = applySpecifier(UE->getSubExpr(), Spec);
     if (!Sub)
       return nullptr;
-    return MCUnaryExpr::create(UE->getOpcode(), Sub, getContext());
+    return MCUnaryExpr::create(UE->getOpcode(), Sub, getContext(),
+                               UE->getLoc());
   }
 
   case MCExpr::Binary: {
@@ -1391,7 +1390,8 @@ const MCExpr *MCAsmParser::applySpecifier(const MCExpr *E, uint32_t Spec) {
     if (!RHS)
       RHS = BE->getRHS();
 
-    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, getContext());
+    return MCBinaryExpr::create(BE->getOpcode(), LHS, RHS, getContext(),
+                                BE->getLoc());
   }
   }
 
@@ -2279,7 +2279,7 @@ bool AsmParser::parseAndMatchAndEmitTargetInstruction(ParseStatementInfo &Info,
     for (unsigned i = 0; i != Info.ParsedOperands.size(); ++i) {
       if (i != 0)
         OS << ", ";
-      Info.ParsedOperands[i]->print(OS);
+      Info.ParsedOperands[i]->print(OS, MAI);
     }
     OS << "]";
 
diff --git a/llvm/lib/MC/MCParser/MCAsmParser.cpp b/llvm/lib/MC/MCParser/MCAsmParser.cpp
index 729ea4f4480d..68b9cab2492f 100644
--- a/llvm/lib/MC/MCParser/MCAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/MCAsmParser.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
@@ -165,6 +166,7 @@ bool MCAsmParser::parseGNUAttribute(SMLoc L, int64_t &Tag,
 void MCParsedAsmOperand::dump() const {
   // Cannot completely remove virtual function even in release mode.
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  dbgs() << "  " << *this;
+  dbgs() << "  ";
+  print(dbgs(), MCAsmInfo());
 #endif
 }
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index 9ab19b18576c..ff1486c887ca 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -2316,7 +2316,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     for (unsigned i = 0; i != Info.ParsedOperands.size(); ++i) {
       if (i != 0)
         OS << ", ";
-      Info.ParsedOperands[i]->print(OS);
+      Info.ParsedOperands[i]->print(OS, MAI);
     }
     OS << "]";
 
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index 2a3761b2cfe7..f87d27f4adcc 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -13,7 +13,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSymbol.h"
@@ -41,9 +40,8 @@ int MCPseudoProbeTable::DdgPrintIndent = 0;
 static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
                                      const MCSymbol *B) {
   MCContext &Context = MCOS->getContext();
-  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
-  const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
-  const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
+  const MCExpr *ARef = MCSymbolRefExpr::create(A, Context);
+  const MCExpr *BRef = MCSymbolRefExpr::create(B, Context);
   const MCExpr *AddrDelta =
       MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
   return AddrDelta;
diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp
index 4a9bacdbc8fc..ba9ef00f9f0d 100644
--- a/llvm/lib/MC/MCRegisterInfo.cpp
+++ b/llvm/lib/MC/MCRegisterInfo.cpp
@@ -93,7 +93,7 @@ ArrayRef<MCPhysReg> MCRegisterInfo::getCachedAliasesOf(MCRegister R) const {
 
   sort(Aliases);
   Aliases.erase(unique(Aliases), Aliases.end());
-  assert(none_of(Aliases, [&](auto &Cur) { return R == Cur; }) &&
+  assert(!llvm::is_contained(Aliases, R) &&
          "MCRegAliasIteratorImpl includes Self!");
 
   // Always put "self" at the end, so the iterator can choose to ignore it.
diff --git a/llvm/lib/MC/MCSPIRVStreamer.cpp b/llvm/lib/MC/MCSPIRVStreamer.cpp
index 3b8cbee70d20..d2534375e501 100644
--- a/llvm/lib/MC/MCSPIRVStreamer.cpp
+++ b/llvm/lib/MC/MCSPIRVStreamer.cpp
@@ -16,21 +16,6 @@
 
 using namespace llvm;
 
-void MCSPIRVStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 0> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Append the encoded instruction to the current data fragment (or create a
-  // new such fragment if the current fragment is not a data fragment).
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 MCStreamer *llvm::createSPIRVStreamer(MCContext &Context,
                                       std::unique_ptr<MCAsmBackend> &&MAB,
                                       std::unique_ptr<MCObjectWriter> &&OW,
diff --git a/llvm/lib/MC/MCSchedule.cpp b/llvm/lib/MC/MCSchedule.cpp
index 8aea08919f46..527ccf3fc36e 100644
--- a/llvm/lib/MC/MCSchedule.cpp
+++ b/llvm/lib/MC/MCSchedule.cpp
@@ -37,6 +37,7 @@ const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth,
                                             0,
                                             0,
                                             nullptr,
+                                            nullptr,
                                             nullptr};
 
 int MCSchedModel::computeInstrLatency(const MCSubtargetInfo &STI,
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index b61738d8837a..09d09e8f997e 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -10,8 +10,8 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
@@ -37,10 +37,13 @@ MCSymbol *MCSection::getEndSymbol(MCContext &Ctx) {
 bool MCSection::hasEnded() const { return End && End->isInSection(); }
 
 MCSection::~MCSection() {
+  // If ~MCRelaxableFragment becomes trivial (no longer store a MCInst member),
+  // this dtor can be made empty.
   for (auto &[_, Chain] : Subsections) {
     for (MCFragment *X = Chain.Head, *Y; X; X = Y) {
       Y = X->Next;
-      X->destroy();
+      if (auto *F = dyn_cast<MCRelaxableFragment>(X))
+        F->~MCRelaxableFragment();
     }
   }
 }
@@ -67,19 +70,61 @@ void MCSection::setBundleLockState(BundleLockStateType NewState) {
 StringRef MCSection::getVirtualSectionKind() const { return "virtual"; }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void MCSection::dump() const {
+LLVM_DUMP_METHOD void MCSection::dump(
+    DenseMap<const MCFragment *, SmallVector<const MCSymbol *, 0>> *FragToSyms)
+    const {
   raw_ostream &OS = errs();
 
-  OS << "<MCSection Name:" << getName();
-  OS << " Fragments:[\n      ";
-  bool First = true;
+  OS << "MCSection Name:" << getName();
   for (auto &F : *this) {
-    if (First)
-      First = false;
-    else
-      OS << ",\n      ";
+    OS << '\n';
     F.dump();
+    if (!FragToSyms)
+      continue;
+    auto It = FragToSyms->find(&F);
+    if (It == FragToSyms->end())
+      continue;
+    for (auto *Sym : It->second) {
+      OS << "\n  Symbol @" << Sym->getOffset() << ' ' << Sym->getName();
+      if (Sym->isTemporary())
+        OS << " Temporary";
+    }
   }
-  OS << "]>";
 }
 #endif
+
+void MCEncodedFragment::setContents(ArrayRef<char> Contents) {
+  auto &S = getParent()->ContentStorage;
+  if (ContentStart + Contents.size() > ContentEnd) {
+    ContentStart = S.size();
+    S.resize_for_overwrite(S.size() + Contents.size());
+  }
+  ContentEnd = ContentStart + Contents.size();
+  llvm::copy(Contents, S.begin() + ContentStart);
+}
+
+void MCEncodedFragment::addFixup(MCFixup Fixup) { appendFixups({Fixup}); }
+
+void MCEncodedFragment::appendFixups(ArrayRef<MCFixup> Fixups) {
+  auto &S = getParent()->FixupStorage;
+  if (LLVM_UNLIKELY(FixupEnd != S.size())) {
+    // Move the elements to the end. Reserve space to avoid invalidating
+    // S.begin()+I for `append`.
+    auto Size = FixupEnd - FixupStart;
+    auto I = std::exchange(FixupStart, S.size());
+    S.reserve(S.size() + Size);
+    S.append(S.begin() + I, S.begin() + I + Size);
+  }
+  S.append(Fixups.begin(), Fixups.end());
+  FixupEnd = S.size();
+}
+
+void MCEncodedFragment::setFixups(ArrayRef<MCFixup> Fixups) {
+  auto &S = getParent()->FixupStorage;
+  if (FixupStart + Fixups.size() > FixupEnd) {
+    FixupStart = S.size();
+    S.resize_for_overwrite(S.size() + Fixups.size());
+  }
+  FixupEnd = FixupStart + Fixups.size();
+  llvm::copy(Fixups, S.begin() + FixupStart);
+}
diff --git a/llvm/lib/MC/MCSectionELF.cpp b/llvm/lib/MC/MCSectionELF.cpp
index 72a959b1c920..cc7cdf2fe4d1 100644
--- a/llvm/lib/MC/MCSectionELF.cpp
+++ b/llvm/lib/MC/MCSectionELF.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCSectionELF.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
@@ -169,8 +170,6 @@ void MCSectionELF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
     OS << "llvm_sympart";
   else if (Type == ELF::SHT_LLVM_BB_ADDR_MAP)
     OS << "llvm_bb_addr_map";
-  else if (Type == ELF::SHT_LLVM_BB_ADDR_MAP_V0)
-    OS << "llvm_bb_addr_map_v0";
   else if (Type == ELF::SHT_LLVM_OFFLOADING)
     OS << "llvm_offloading";
   else if (Type == ELF::SHT_LLVM_LTO)
diff --git a/llvm/lib/MC/MCSectionGOFF.cpp b/llvm/lib/MC/MCSectionGOFF.cpp
new file mode 100644
index 000000000000..8163e5bbd632
--- /dev/null
+++ b/llvm/lib/MC/MCSectionGOFF.cpp
@@ -0,0 +1,143 @@
+//===- MCSectionGOFF.cpp - GOFF Code Section Representation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCSectionGOFF.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static void emitCATTR(raw_ostream &OS, StringRef Name, GOFF::ESDRmode Rmode,
+                      GOFF::ESDAlignment Alignment,
+                      GOFF::ESDLoadingBehavior LoadBehavior,
+                      GOFF::ESDExecutable Executable, bool IsReadOnly,
+                      uint32_t SortKey, uint8_t FillByteValue,
+                      StringRef PartName) {
+  OS << Name << " CATTR ";
+  OS << "ALIGN(" << static_cast<unsigned>(Alignment) << "),"
+     << "FILL(" << static_cast<unsigned>(FillByteValue) << ")";
+  switch (LoadBehavior) {
+  case GOFF::ESD_LB_Deferred:
+    OS << ",DEFLOAD";
+    break;
+  case GOFF::ESD_LB_NoLoad:
+    OS << ",NOLOAD";
+    break;
+  default:
+    break;
+  }
+  switch (Executable) {
+  case GOFF::ESD_EXE_CODE:
+    OS << ",EXECUTABLE";
+    break;
+  case GOFF::ESD_EXE_DATA:
+    OS << ",NOTEXECUTABLE";
+    break;
+  default:
+    break;
+  }
+  if (IsReadOnly)
+    OS << ",READONLY";
+  if (Rmode != GOFF::ESD_RMODE_None) {
+    OS << ',';
+    OS << "RMODE(";
+    switch (Rmode) {
+    case GOFF::ESD_RMODE_24:
+      OS << "24";
+      break;
+    case GOFF::ESD_RMODE_31:
+      OS << "31";
+      break;
+    case GOFF::ESD_RMODE_64:
+      OS << "64";
+      break;
+    case GOFF::ESD_RMODE_None:
+      break;
+    }
+    OS << ')';
+  }
+  if (SortKey)
+    OS << ",PRIORITY(" << SortKey << ")";
+  if (!PartName.empty())
+    OS << ",PART(" << PartName << ")";
+  OS << '\n';
+}
+
+static void emitXATTR(raw_ostream &OS, StringRef Name,
+                      GOFF::ESDLinkageType Linkage,
+                      GOFF::ESDExecutable Executable,
+                      GOFF::ESDBindingScope BindingScope) {
+  OS << Name << " XATTR ";
+  OS << "LINKAGE(" << (Linkage == GOFF::ESD_LT_OS ? "OS" : "XPLINK") << "),";
+  if (Executable != GOFF::ESD_EXE_Unspecified)
+    OS << "REFERENCE(" << (Executable == GOFF::ESD_EXE_CODE ? "CODE" : "DATA")
+       << "),";
+  if (BindingScope != GOFF::ESD_BSC_Unspecified) {
+    OS << "SCOPE(";
+    switch (BindingScope) {
+    case GOFF::ESD_BSC_Section:
+      OS << "SECTION";
+      break;
+    case GOFF::ESD_BSC_Module:
+      OS << "MODULE";
+      break;
+    case GOFF::ESD_BSC_Library:
+      OS << "LIBRARY";
+      break;
+    case GOFF::ESD_BSC_ImportExport:
+      OS << "EXPORT";
+      break;
+    default:
+      break;
+    }
+    OS << ')';
+  }
+  OS << '\n';
+}
+
+void MCSectionGOFF::printSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
+                                         raw_ostream &OS,
+                                         uint32_t Subsection) const {
+  switch (SymbolType) {
+  case GOFF::ESD_ST_SectionDefinition: {
+    OS << Name << " CSECT\n";
+    Emitted = true;
+    break;
+  }
+  case GOFF::ESD_ST_ElementDefinition: {
+    getParent()->printSwitchToSection(MAI, T, OS, Subsection);
+    if (!Emitted) {
+      emitCATTR(OS, Name, EDAttributes.Rmode, EDAttributes.Alignment,
+                EDAttributes.LoadBehavior, GOFF::ESD_EXE_Unspecified,
+                EDAttributes.IsReadOnly, 0, EDAttributes.FillByteValue,
+                StringRef());
+      Emitted = true;
+    } else
+      OS << Name << " CATTR\n";
+    break;
+  }
+  case GOFF::ESD_ST_PartReference: {
+    MCSectionGOFF *ED = getParent();
+    ED->getParent()->printSwitchToSection(MAI, T, OS, Subsection);
+    if (!Emitted) {
+      emitCATTR(OS, ED->getName(), ED->getEDAttributes().Rmode,
+                ED->EDAttributes.Alignment, ED->EDAttributes.LoadBehavior,
+                PRAttributes.Executable, ED->EDAttributes.IsReadOnly,
+                PRAttributes.SortKey, ED->EDAttributes.FillByteValue, Name);
+      emitXATTR(OS, Name, PRAttributes.Linkage, PRAttributes.Executable,
+                PRAttributes.BindingScope);
+      ED->Emitted = true;
+      Emitted = true;
+    } else
+      OS << ED->getName() << " CATTR PART(" << Name << ")\n";
+    break;
+  }
+  default:
+    llvm_unreachable("Wrong section type");
+  }
+}
\ No newline at end of file
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index cdcf279e6f90..369ac1ba63d9 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1507,10 +1507,9 @@ static VersionTuple getMachoBuildVersionSupportedOS(const Triple &Target) {
   case Triple::WatchOS:
     return VersionTuple(5);
   case Triple::DriverKit:
-    // DriverKit always uses the build version load command.
-    return VersionTuple();
+  case Triple::BridgeOS:
   case Triple::XROS:
-    // XROS always uses the build version load command.
+    // DriverKit/BridgeOS/XROS always use the build version load command.
     return VersionTuple();
   default:
     break;
@@ -1541,6 +1540,8 @@ getMachoBuildVersionPlatformType(const Triple &Target) {
   case Triple::XROS:
     return Target.isSimulatorEnvironment() ? MachO::PLATFORM_XROS_SIMULATOR
                                            : MachO::PLATFORM_XROS;
+  case Triple::BridgeOS:
+    return MachO::PLATFORM_BRIDGEOS;
   default:
     break;
   }
@@ -1574,6 +1575,7 @@ void MCStreamer::emitVersionForTarget(
     Version = Target.getDriverKitVersion();
     break;
   case Triple::XROS:
+  case Triple::BridgeOS:
     Version = Target.getOSVersion();
     break;
   default:
diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp
index d86eaad48420..89a08327dd25 100644
--- a/llvm/lib/MC/MCSubtargetInfo.cpp
+++ b/llvm/lib/MC/MCSubtargetInfo.cpp
@@ -9,6 +9,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/MC/MCSchedule.h"
 #include "llvm/Support/Format.h"
diff --git a/llvm/lib/MC/MCSymbol.cpp b/llvm/lib/MC/MCSymbol.cpp
index 5cba901d33d5..baf1ff89c018 100644
--- a/llvm/lib/MC/MCSymbol.cpp
+++ b/llvm/lib/MC/MCSymbol.cpp
@@ -11,7 +11,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/lib/MC/MCWasmStreamer.cpp b/llvm/lib/MC/MCWasmStreamer.cpp
index cc731fe02f7d..602d40df4cc3 100644
--- a/llvm/lib/MC/MCWasmStreamer.cpp
+++ b/llvm/lib/MC/MCWasmStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionWasm.h"
@@ -151,26 +150,6 @@ void MCWasmStreamer::emitIdent(StringRef IdentString) {
   // sections in the object format
 }
 
-void MCWasmStreamer::emitInstToData(const MCInst &Inst,
-                                    const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Append the encoded instruction to the current data fragment (or create a
-  // new such fragment if the current fragment is not a data fragment).
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCWasmStreamer::finishImpl() {
   emitFrames(nullptr);
 
diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp
index 7fc8919d1045..8cc74db9beea 100644
--- a/llvm/lib/MC/MCWin64EH.cpp
+++ b/llvm/lib/MC/MCWin64EH.cpp
@@ -338,7 +338,7 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
       auto *MCE = MCUnwindV2EpilogTargetExpr::create(*info, Epilog.second,
                                                      EpilogSize, context);
       MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_2);
-      DF->getFixups().push_back(Fixup);
+      DF->addFixup(Fixup);
       DF->appendContents(2, 0);
     }
   }
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index d95e0cb8f043..0c0866f9e36a 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -136,23 +135,6 @@ WinCOFFObjectWriter &MCWinCOFFStreamer::getWriter() {
   return static_cast<WinCOFFObjectWriter &>(getAssembler().getWriter());
 }
 
-void MCWinCOFFStreamer::emitInstToData(const MCInst &Inst,
-                                       const MCSubtargetInfo &STI) {
-  MCDataFragment *DF = getOrCreateDataFragment();
-
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  for (MCFixup &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixup);
-  }
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCWinCOFFStreamer::initSections(bool NoExecStack,
                                      const MCSubtargetInfo &STI) {
   // FIXME: this is identical to the ELF one.
@@ -299,7 +281,7 @@ void MCWinCOFFStreamer::emitCOFFSectionIndex(const MCSymbol *Symbol) {
   MCDataFragment *DF = getOrCreateDataFragment();
   const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext());
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, FK_SecRel_2);
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   DF->appendContents(2, 0);
 }
 
@@ -316,7 +298,7 @@ void MCWinCOFFStreamer::emitCOFFSecRel32(const MCSymbol *Symbol,
   // Build the secrel32 relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_SecRel_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -335,7 +317,7 @@ void MCWinCOFFStreamer::emitCOFFImgRel32(const MCSymbol *Symbol,
   // Build the imgrel relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -349,7 +331,7 @@ void MCWinCOFFStreamer::emitCOFFSecNumber(MCSymbol const *Symbol) {
   // Build the relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
@@ -363,7 +345,7 @@ void MCWinCOFFStreamer::emitCOFFSecOffset(MCSymbol const *Symbol) {
   // Build the relocation.
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), MCE, FK_Data_4);
   // Record the relocation.
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
   // Emit 4 bytes (zeros) to the object file.
   DF->appendContents(4, 0);
 }
diff --git a/llvm/lib/MC/MCXCOFFStreamer.cpp b/llvm/lib/MC/MCXCOFFStreamer.cpp
index 15de5d83bfe1..12e6680bcf07 100644
--- a/llvm/lib/MC/MCXCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCXCOFFStreamer.cpp
@@ -98,7 +98,7 @@ void MCXCOFFStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
 
   MCFixupKind Kind = *MaybeKind;
   MCFixup Fixup = MCFixup::create(DF->getContents().size(), SRE, Kind);
-  DF->getFixups().push_back(Fixup);
+  DF->addFixup(Fixup);
 }
 
 void MCXCOFFStreamer::emitXCOFFRenameDirective(const MCSymbol *Name,
@@ -138,26 +138,6 @@ void MCXCOFFStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   emitZeros(Size);
 }
 
-void MCXCOFFStreamer::emitInstToData(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI) {
-  MCAssembler &Assembler = getAssembler();
-  SmallVector<MCFixup, 4> Fixups;
-  SmallString<256> Code;
-  Assembler.getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
-
-  // Add the fixups and data.
-  MCDataFragment *DF = getOrCreateDataFragment(&STI);
-  const size_t ContentsSize = DF->getContents().size();
-  auto &DataFragmentFixups = DF->getFixups();
-  for (auto &Fixup : Fixups) {
-    Fixup.setOffset(Fixup.getOffset() + ContentsSize);
-    DataFragmentFixups.push_back(Fixup);
-  }
-
-  DF->setHasInstructions(STI);
-  DF->appendContents(Code);
-}
-
 void MCXCOFFStreamer::emitXCOFFLocalCommonSymbol(MCSymbol *LabelSym,
                                                  uint64_t Size,
                                                  MCSymbol *CsectSym,
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 9377a74b377e..fb8e1fdf62f5 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -15,8 +15,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -86,12 +84,6 @@ MachSymbolData::operator<(const MachSymbolData &RHS) const {
   return Symbol->getName() < RHS.Symbol->getName();
 }
 
-bool MachObjectWriter::isFixupKindPCRel(const MCAssembler &Asm, unsigned Kind) {
-  MCFixupKindInfo FKI = Asm.getBackend().getFixupKindInfo((MCFixupKind)Kind);
-
-  return FKI.Flags & MCFixupKindInfo::FKF_IsPCRel;
-}
-
 uint64_t
 MachObjectWriter::getFragmentAddress(const MCAssembler &Asm,
                                      const MCFragment *Fragment) const {
@@ -808,8 +800,8 @@ uint64_t MachObjectWriter::writeObject() {
     MCSection *CGProfileSection = getContext().getMachOSection(
         "__LLVM", "__cg_profile", 0, SectionKind::getMetadata());
     auto &Frag = cast<MCDataFragment>(*CGProfileSection->begin());
-    Frag.getContents().clear();
-    raw_svector_ostream OS(Frag.getContents());
+    Frag.clearContents();
+    raw_svector_ostream OS(Frag.getContentsForAppending());
     for (const MCObjectWriter::CGProfileEntry &CGPE : CGProfile) {
       uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
       uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -817,6 +809,7 @@ uint64_t MachObjectWriter::writeObject() {
       support::endian::write(OS, ToIndex, W.Endian);
       support::endian::write(OS, CGPE.Count, W.Endian);
     }
+    Frag.doneAppending();
   }
 
   unsigned NumSections = Asm.end() - Asm.begin();
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 3b82fb782f88..d148b521ff9f 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCSymbolWasm.h"
@@ -479,8 +478,7 @@ void WasmObjectWriter::recordRelocation(const MCFragment &F,
                                         const MCFixup &Fixup, MCValue Target,
                                         uint64_t &FixedValue) {
   // The WebAssembly backend should never generate FKF_IsPCRel fixups
-  assert(!(Asm->getBackend().getFixupKindInfo(Fixup.getKind()).Flags &
-           MCFixupKindInfo::FKF_IsPCRel));
+  assert(!Fixup.isPCRel());
 
   const auto &FixupSection = cast<MCSectionWasm>(*F.getParent());
   uint64_t C = Target.getConstant();
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 2ed0aa3d20fc..c2ef430984ed 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionCOFF.h"
@@ -1071,7 +1070,7 @@ uint64_t WinCOFFWriter::writeObject() {
     auto *Sec = getContext().getCOFFSection(".llvm_addrsig",
                                             COFF::IMAGE_SCN_LNK_REMOVE);
     auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
-    raw_svector_ostream OS(Frag->getContents());
+    raw_svector_ostream OS(Frag->getContentsForAppending());
     for (const MCSymbol *S : OWriter.AddrsigSyms) {
       if (!S->isRegistered())
         continue;
@@ -1086,6 +1085,7 @@ uint64_t WinCOFFWriter::writeObject() {
              "executePostLayoutBinding!");
       encodeULEB128(SectionMap[TargetSection]->Symbol->getIndex(), OS);
     }
+    Frag->doneAppending();
   }
 
   // Create the contents of the .llvm.call-graph-profile section.
@@ -1093,7 +1093,7 @@ uint64_t WinCOFFWriter::writeObject() {
     auto *Sec = getContext().getCOFFSection(".llvm.call-graph-profile",
                                             COFF::IMAGE_SCN_LNK_REMOVE);
     auto *Frag = cast<MCDataFragment>(Sec->curFragList()->Head);
-    raw_svector_ostream OS(Frag->getContents());
+    raw_svector_ostream OS(Frag->getContentsForAppending());
     for (const auto &CGPE : OWriter.getCGProfile()) {
       uint32_t FromIndex = CGPE.From->getSymbol().getIndex();
       uint32_t ToIndex = CGPE.To->getSymbol().getIndex();
@@ -1101,6 +1101,7 @@ uint64_t WinCOFFWriter::writeObject() {
       support::endian::write(OS, ToIndex, W.Endian);
       support::endian::write(OS, CGPE.Count, W.Endian);
     }
+    Frag->doneAppending();
   }
 
   assignFileOffsets();
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index ddb2ca8a03e2..2f6785fe7b80 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionXCOFF.h"
 #include "llvm/MC/MCSymbolXCOFF.h"
@@ -686,15 +685,10 @@ void XCOFFWriter::recordRelocation(const MCFragment &F, const MCFixup &Fixup,
   };
 
   const MCSymbol *const SymA = Target.getAddSym();
-
-  MCAsmBackend &Backend = Asm->getBackend();
-  bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
-
   uint8_t Type;
   uint8_t SignAndSize;
-  std::tie(Type, SignAndSize) =
-      TargetObjectWriter->getRelocTypeAndSignSize(Target, Fixup, IsPCRel);
+  std::tie(Type, SignAndSize) = TargetObjectWriter->getRelocTypeAndSignSize(
+      Target, Fixup, Fixup.isPCRel());
 
   const MCSectionXCOFF *SymASec = getContainingCsect(cast<MCSymbolXCOFF>(SymA));
   assert(SectionMap.contains(SymASec) &&
diff --git a/llvm/lib/MCA/CodeEmitter.cpp b/llvm/lib/MCA/CodeEmitter.cpp
index b3936a95c06a..5bc48d9227f4 100644
--- a/llvm/lib/MCA/CodeEmitter.cpp
+++ b/llvm/lib/MCA/CodeEmitter.cpp
@@ -22,12 +22,8 @@ CodeEmitter::EncodingInfo CodeEmitter::getOrCreateEncodingInfo(unsigned MCID) {
 
   SmallVector<llvm::MCFixup, 2> Fixups;
   const MCInst &Inst = Sequence[MCID];
-  MCInst Relaxed(Sequence[MCID]);
-  if (MAB.mayNeedRelaxation(Inst, STI))
-    MAB.relaxInstruction(Relaxed, STI);
-
   EI.first = Code.size();
-  MCE.encodeInstruction(Relaxed, Code, Fixups, STI);
+  MCE.encodeInstruction(Inst, Code, Fixups, STI);
   EI.second = Code.size() - EI.first;
   return EI;
 }
diff --git a/llvm/lib/MCA/InstrBuilder.cpp b/llvm/lib/MCA/InstrBuilder.cpp
index 2bac99b6309a..cad25a6ddd3f 100644
--- a/llvm/lib/MCA/InstrBuilder.cpp
+++ b/llvm/lib/MCA/InstrBuilder.cpp
@@ -75,7 +75,8 @@ static void initializeUsedResources(InstrDesc &ID,
       WithColor::warning()
           << "Ignoring invalid write of zero cycles on processor resource "
           << PR.Name << "\n";
-      WithColor::note() << "found in scheduling class " << SCDesc.Name
+      WithColor::note() << "found in scheduling class "
+                        << SM.getSchedClassName(ID.SchedClassID)
                         << " (write index #" << I << ")\n";
 #endif
       continue;
diff --git a/llvm/lib/ObjCopy/COFF/COFFWriter.cpp b/llvm/lib/ObjCopy/COFF/COFFWriter.cpp
index 734a1698d515..350c4aec572c 100644
--- a/llvm/lib/ObjCopy/COFF/COFFWriter.cpp
+++ b/llvm/lib/ObjCopy/COFF/COFFWriter.cpp
@@ -120,8 +120,11 @@ void COFFWriter::layoutSections() {
 
 Expected<size_t> COFFWriter::finalizeStringTable() {
   for (const auto &S : Obj.getSections())
-    if (S.Name.size() > COFF::NameSize)
-      StrTabBuilder.add(S.Name);
+    if (S.Name.size() > COFF::NameSize) {
+      // Put the section name at the start of strtab to ensure its offset is
+      // less than Max7DecimalOffset. Otherwise, lldb/gdb will not read it.
+      StrTabBuilder.add(S.Name, /*Priority=*/UINT8_MAX);
+    }
 
   for (const auto &S : Obj.getSymbols())
     if (S.Name.size() > COFF::NameSize)
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index c61ba868efe6..6fc0889afc6a 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -700,7 +700,7 @@ static bool isECObject(object::SymbolicFile &Obj) {
         getBitcodeTargetTriple(Obj.getMemoryBufferRef());
     if (!TripleStr)
       return false;
-    Triple T(*TripleStr);
+    Triple T(std::move(*TripleStr));
     return T.isWindowsArm64EC() || T.getArch() == Triple::x86_64;
   }
 
@@ -719,7 +719,7 @@ static bool isAnyArm64COFF(object::SymbolicFile &Obj) {
         getBitcodeTargetTriple(Obj.getMemoryBufferRef());
     if (!TripleStr)
       return false;
-    Triple T(*TripleStr);
+    Triple T(std::move(*TripleStr));
     return T.isOSWindows() && T.getArch() == Triple::aarch64;
   }
 
diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp
index 6ee33d94ee86..ff99dc25abe0 100644
--- a/llvm/lib/Object/ELF.cpp
+++ b/llvm/lib/Object/ELF.cpp
@@ -317,7 +317,6 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) {
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_SYMPART);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_PART_EHDR);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_PART_PHDR);
-    STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_BB_ADDR_MAP_V0);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_BB_ADDR_MAP);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_OFFLOADING);
     STRINGIFY_ENUM_CASE(ELF, SHT_LLVM_LTO);
@@ -833,33 +832,24 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
   BBAddrMap::Features FeatEnable{};
   while (!ULEBSizeErr && !MetadataDecodeErr && Cur &&
          Cur.tell() < Content.size()) {
-    if (Sec.sh_type == ELF::SHT_LLVM_BB_ADDR_MAP) {
-      Version = Data.getU8(Cur);
-      if (!Cur)
-        break;
-      if (Version > 3)
-        return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
-                           Twine(static_cast<int>(Version)));
-      Feature = Data.getU8(Cur); // Feature byte
-      if (!Cur)
-        break;
-      auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature);
-      if (!FeatEnableOrErr)
-        return FeatEnableOrErr.takeError();
-      FeatEnable = *FeatEnableOrErr;
-      if (FeatEnable.hasPGOAnalysis() && Version < 2)
-        return createError(
-            "version should be >= 2 for SHT_LLVM_BB_ADDR_MAP when "
-            "PGO features are enabled: version = " +
-            Twine(static_cast<int>(Version)) +
-            " feature = " + Twine(static_cast<int>(Feature)));
-      if (FeatEnable.CallsiteOffsets && Version < 3)
-        return createError(
-            "version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when "
-            "callsite offsets feature is enabled: version = " +
-            Twine(static_cast<int>(Version)) +
-            " feature = " + Twine(static_cast<int>(Feature)));
-    }
+    Version = Data.getU8(Cur);
+    if (!Cur)
+      break;
+    if (Version < 2 || Version > 3)
+      return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
+                         Twine(static_cast<int>(Version)));
+    Feature = Data.getU8(Cur); // Feature byte
+    if (!Cur)
+      break;
+    auto FeatEnableOrErr = BBAddrMap::Features::decode(Feature);
+    if (!FeatEnableOrErr)
+      return FeatEnableOrErr.takeError();
+    FeatEnable = *FeatEnableOrErr;
+    if (FeatEnable.CallsiteOffsets && Version < 3)
+      return createError("version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when "
+                         "callsite offsets feature is enabled: version = " +
+                         Twine(static_cast<int>(Version)) +
+                         " feature = " + Twine(static_cast<int>(Feature)));
     uint32_t NumBlocksInBBRange = 0;
     uint32_t NumBBRanges = 1;
     typename ELFFile<ELFT>::uintX_t RangeBaseAddress = 0;
@@ -895,9 +885,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
         for (uint32_t BlockIndex = 0; !MetadataDecodeErr && !ULEBSizeErr &&
                                       Cur && (BlockIndex < NumBlocksInBBRange);
              ++BlockIndex) {
-          uint32_t ID = Version >= 2
-                            ? readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr)
-                            : BlockIndex;
+          uint32_t ID = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
           uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
           // Read the callsite offsets.
           uint32_t LastCallsiteOffset = 0;
@@ -917,19 +905,15 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
           uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr) +
                           LastCallsiteOffset;
           uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
-          if (Version >= 1) {
-            // Offset is calculated relative to the end of the previous BB.
-            Offset += PrevBBEndOffset;
-            PrevBBEndOffset = Offset + Size;
-          }
           Expected<BBAddrMap::BBEntry::Metadata> MetadataOrErr =
               BBAddrMap::BBEntry::Metadata::decode(MD);
           if (!MetadataOrErr) {
             MetadataDecodeErr = MetadataOrErr.takeError();
             break;
           }
-          BBEntries.push_back(
-              {ID, Offset, Size, *MetadataOrErr, CallsiteOffsets});
+          BBEntries.push_back({ID, Offset + PrevBBEndOffset, Size,
+                               *MetadataOrErr, CallsiteOffsets});
+          PrevBBEndOffset += Offset + Size;
         }
         TotalNumBlocks += BBEntries.size();
       }
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index cce5c74aa4d1..0e13d32bbe52 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -904,8 +904,7 @@ Expected<std::vector<BBAddrMap>> static readBBAddrMapImpl(
 
   const auto &Sections = cantFail(EF.sections());
   auto IsMatch = [&](const Elf_Shdr &Sec) -> Expected<bool> {
-    if (Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP &&
-        Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP_V0)
+    if (Sec.sh_type != ELF::SHT_LLVM_BB_ADDR_MAP)
       return false;
     if (!TextSectionIndex)
       return true;
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 494ec089d7bd..2579fa37935f 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -48,7 +48,7 @@ static cl::opt<bool> DisableBitcodeVersionUpgrade(
 
 static const char *PreservedSymbols[] = {
     // There are global variables, so put it here instead of in
-    // RuntimeLibcalls.def.
+    // RuntimeLibcalls.td.
     // TODO: Are there similar such variables?
     "__ssp_canary_word",
     "__stack_chk_guard",
@@ -218,9 +218,9 @@ static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
                                          std::end(PreservedSymbols));
   // FIXME: Do we need to pass in ABI fields from TargetOptions?
   RTLIB::RuntimeLibcallsInfo Libcalls(TT);
-  for (const char *Name : Libcalls.getLibcallNames()) {
-    if (Name)
-      PreservedSymbolSet.insert(Name);
+  for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
+    if (Impl != RTLIB::Unsupported)
+      PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
   }
   return PreservedSymbolSet;
 }
diff --git a/llvm/lib/Object/OffloadBundle.cpp b/llvm/lib/Object/OffloadBundle.cpp
index a4c1e0547a5b..1e1042ce2bc2 100644
--- a/llvm/lib/Object/OffloadBundle.cpp
+++ b/llvm/lib/Object/OffloadBundle.cpp
@@ -339,8 +339,7 @@ CompressedOffloadBundle::decompress(llvm::MemoryBufferRef &Input,
     HashRecalcTimer.startTimer();
     llvm::MD5 Hash;
     llvm::MD5::MD5Result Result;
-    Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData.data(),
-                                        DecompressedData.size()));
+    Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData));
     Hash.final(Result);
     uint64_t RecalculatedHash = Result.low();
     HashRecalcTimer.stopTimer();
diff --git a/llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp b/llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
index b15919f68725..305625180930 100644
--- a/llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
+++ b/llvm/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
@@ -605,6 +605,11 @@ template <> void SymbolRecordImpl<JumpTableSym>::map(IO &IO) {
   IO.mapRequired("EntriesCount", Symbol.EntriesCount);
 }
 
+template <> void SymbolRecordImpl<HotPatchFuncSym>::map(IO &IO) {
+  IO.mapRequired("Function", Symbol.Function);
+  IO.mapRequired("Name", Symbol.Name);
+}
+
 } // end namespace detail
 } // end namespace CodeViewYAML
 } // end namespace llvm
diff --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index 3bca25f3fc6c..263f7bdf37bc 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -33,6 +33,47 @@ DXContainerYAML::ShaderFeatureFlags::ShaderFeatureFlags(uint64_t FlagData) {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
+template <typename T>
+static llvm::Error
+readDescriptorRanges(DXContainerYAML::RootParameterHeaderYaml &Header,
+                     DXContainerYAML::RootSignatureYamlDesc &RootSigDesc,
+                     object::DirectX::DescriptorTableView *DTV) {
+
+  llvm::Expected<object::DirectX::DescriptorTable<T>> TableOrErr =
+      DTV->read<T>();
+  if (Error E = TableOrErr.takeError())
+    return E;
+  auto Table = *TableOrErr;
+
+  DXContainerYAML::RootParameterLocationYaml Location(Header);
+  DXContainerYAML::DescriptorTableYaml &TableYaml =
+      RootSigDesc.Parameters.getOrInsertTable(Location);
+  RootSigDesc.Parameters.insertLocation(Location);
+
+  TableYaml.NumRanges = Table.NumRanges;
+  TableYaml.RangesOffset = Table.RangesOffset;
+
+  for (const auto &R : Table.Ranges) {
+    DXContainerYAML::DescriptorRangeYaml NewR;
+    NewR.OffsetInDescriptorsFromTableStart =
+        R.OffsetInDescriptorsFromTableStart;
+    NewR.NumDescriptors = R.NumDescriptors;
+    NewR.BaseShaderRegister = R.BaseShaderRegister;
+    NewR.RegisterSpace = R.RegisterSpace;
+    NewR.RangeType = R.RangeType;
+    if constexpr (std::is_same_v<T, dxbc::RTS0::v2::DescriptorRange>) {
+      // Set all flag fields for v2
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  NewR.Enum =                                                                  \
+      (R.Flags & llvm::to_underlying(dxbc::DescriptorRangeFlags::Enum)) != 0;
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    }
+    TableYaml.Ranges.push_back(NewR);
+  }
+
+  return Error::success();
+}
+
 llvm::Expected<DXContainerYAML::RootSignatureYamlDesc>
 DXContainerYAML::RootSignatureYamlDesc::create(
     const object::DirectX::RootSignature &Data) {
@@ -99,44 +140,24 @@ DXContainerYAML::RootSignatureYamlDesc::create(
       YamlDescriptor.ShaderRegister = Descriptor.ShaderRegister;
       YamlDescriptor.RegisterSpace = Descriptor.RegisterSpace;
       if (Version > 1) {
-#define ROOT_DESCRIPTOR_FLAG(Num, Val)                                         \
-  YamlDescriptor.Val =                                                         \
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  YamlDescriptor.Enum =                                                        \
       (Descriptor.Flags &                                                      \
-       llvm::to_underlying(dxbc::RootDescriptorFlag::Val)) > 0;
+       llvm::to_underlying(dxbc::RootDescriptorFlags::Enum)) > 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
       }
     } else if (auto *DTV =
                    dyn_cast<object::DirectX::DescriptorTableView>(&ParamView)) {
-      llvm::Expected<object::DirectX::DescriptorTable> TableOrErr =
-          DTV->read(Version);
-      if (Error E = TableOrErr.takeError())
-        return std::move(E);
-      auto Table = *TableOrErr;
-      RootParameterLocationYaml Location(Header);
-      DescriptorTableYaml &TableYaml =
-          RootSigDesc.Parameters.getOrInsertTable(Location);
-      RootSigDesc.Parameters.insertLocation(Location);
-
-      TableYaml.NumRanges = Table.NumRanges;
-      TableYaml.RangesOffset = Table.RangesOffset;
-
-      for (const auto &R : Table) {
-        DescriptorRangeYaml NewR;
-
-        NewR.OffsetInDescriptorsFromTableStart =
-            R.OffsetInDescriptorsFromTableStart;
-        NewR.NumDescriptors = R.NumDescriptors;
-        NewR.BaseShaderRegister = R.BaseShaderRegister;
-        NewR.RegisterSpace = R.RegisterSpace;
-        NewR.RangeType = R.RangeType;
-        if (Version > 1) {
-#define DESCRIPTOR_RANGE_FLAG(Num, Val)                                        \
-  NewR.Val =                                                                   \
-      (R.Flags & llvm::to_underlying(dxbc::DescriptorRangeFlag::Val)) > 0;
-#include "llvm/BinaryFormat/DXContainerConstants.def"
-        }
-        TableYaml.Ranges.push_back(NewR);
-      }
+      if (Version == 1) {
+        if (Error E = readDescriptorRanges<dxbc::RTS0::v1::DescriptorRange>(
+                Header, RootSigDesc, DTV))
+          return std::move(E);
+      } else if (Version == 2) {
+        if (Error E = readDescriptorRanges<dxbc::RTS0::v2::DescriptorRange>(
+                Header, RootSigDesc, DTV))
+          return std::move(E);
+      } else
+        llvm_unreachable("Unknown version for DescriptorRanges");
     }
   }
 
@@ -159,38 +180,37 @@ DXContainerYAML::RootSignatureYamlDesc::create(
     RootSigDesc.StaticSamplers.push_back(NewS);
   }
 
-#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
-  RootSigDesc.Val =                                                            \
-      (Flags & llvm::to_underlying(dxbc::RootElementFlag::Val)) > 0;
+#define ROOT_SIGNATURE_FLAG(Num, Val)                                          \
+  RootSigDesc.Val = (Flags & llvm::to_underlying(dxbc::RootFlags::Val)) > 0;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
   return RootSigDesc;
 }
 
 uint32_t DXContainerYAML::RootDescriptorYaml::getEncodedFlags() const {
-  uint64_t Flag = 0;
-#define ROOT_DESCRIPTOR_FLAG(Num, Val)                                         \
-  if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::RootDescriptorFlag::Val;
+  uint64_t Flags = 0;
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  if (Enum)                                                                    \
+    Flags |= (uint32_t)dxbc::RootDescriptorFlags::Enum;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
-  return Flag;
+  return Flags;
 }
 
 uint32_t DXContainerYAML::RootSignatureYamlDesc::getEncodedFlags() {
   uint64_t Flag = 0;
-#define ROOT_ELEMENT_FLAG(Num, Val)                                            \
+#define ROOT_SIGNATURE_FLAG(Num, Val)                                          \
   if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::RootElementFlag::Val;
+    Flag |= (uint32_t)dxbc::RootFlags::Val;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
   return Flag;
 }
 
 uint32_t DXContainerYAML::DescriptorRangeYaml::getEncodedFlags() const {
-  uint64_t Flag = 0;
-#define DESCRIPTOR_RANGE_FLAG(Num, Val)                                        \
-  if (Val)                                                                     \
-    Flag |= (uint32_t)dxbc::DescriptorRangeFlag::Val;
+  uint64_t Flags = 0;
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  if (Enum)                                                                    \
+    Flags |= (uint32_t)dxbc::DescriptorRangeFlags::Enum;
 #include "llvm/BinaryFormat/DXContainerConstants.def"
-  return Flag;
+  return Flags;
 }
 
 uint64_t DXContainerYAML::ShaderFeatureFlags::getEncodedFlags() {
@@ -361,7 +381,7 @@ void MappingTraits<DXContainerYAML::RootSignatureYamlDesc>::mapping(
   IO.mapRequired("StaticSamplersOffset", S.StaticSamplersOffset);
   IO.mapRequired("Parameters", S.Parameters.Locations, S);
   IO.mapOptional("Samplers", S.StaticSamplers);
-#define ROOT_ELEMENT_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
+#define ROOT_SIGNATURE_FLAG(Num, Val) IO.mapOptional(#Val, S.Val, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
@@ -385,7 +405,8 @@ void MappingTraits<llvm::DXContainerYAML::DescriptorRangeYaml>::mapping(
   IO.mapRequired("RegisterSpace", R.RegisterSpace);
   IO.mapRequired("OffsetInDescriptorsFromTableStart",
                  R.OffsetInDescriptorsFromTableStart);
-#define DESCRIPTOR_RANGE_FLAG(Num, Val) IO.mapOptional(#Val, R.Val, false);
+#define DESCRIPTOR_RANGE_FLAG(Num, Enum, Flag)                                 \
+  IO.mapOptional(#Flag, R.Enum, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
@@ -438,7 +459,8 @@ void MappingTraits<llvm::DXContainerYAML::RootDescriptorYaml>::mapping(
     IO &IO, llvm::DXContainerYAML::RootDescriptorYaml &D) {
   IO.mapRequired("RegisterSpace", D.RegisterSpace);
   IO.mapRequired("ShaderRegister", D.ShaderRegister);
-#define ROOT_DESCRIPTOR_FLAG(Num, Val) IO.mapOptional(#Val, D.Val, false);
+#define ROOT_DESCRIPTOR_FLAG(Num, Enum, Flag)                                  \
+  IO.mapOptional(#Flag, D.Enum, false);
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index b28191730f01..970811f40f91 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -720,7 +720,6 @@ void ScalarEnumerationTraits<ELFYAML::ELF_SHT>::enumeration(
   ECase(SHT_LLVM_SYMPART);
   ECase(SHT_LLVM_PART_EHDR);
   ECase(SHT_LLVM_PART_PHDR);
-  ECase(SHT_LLVM_BB_ADDR_MAP_V0);
   ECase(SHT_LLVM_BB_ADDR_MAP);
   ECase(SHT_LLVM_OFFLOADING);
   ECase(SHT_LLVM_LTO);
diff --git a/llvm/lib/Option/Arg.cpp b/llvm/lib/Option/Arg.cpp
index 2d52b947aaed..3aab7c0768e1 100644
--- a/llvm/lib/Option/Arg.cpp
+++ b/llvm/lib/Option/Arg.cpp
@@ -6,13 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Option/Arg.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Config/llvm-config.h"
-#include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -72,13 +73,7 @@ std::string Arg::getAsString(const ArgList &Args) const {
 
   ArgStringList ASL;
   render(Args, ASL);
-  for (ArgStringList::iterator
-         it = ASL.begin(), ie = ASL.end(); it != ie; ++it) {
-    if (it != ASL.begin())
-      OS << ' ';
-    OS << *it;
-  }
-
+  OS << llvm::interleaved(ASL, " ");
   return std::string(OS.str());
 }
 
@@ -100,11 +95,7 @@ void Arg::render(const ArgList &Args, ArgStringList &Output) const {
   case Option::RenderCommaJoinedStyle: {
     SmallString<256> Res;
     raw_svector_ostream OS(Res);
-    OS << getSpelling();
-    for (unsigned i = 0, e = getNumValues(); i != e; ++i) {
-      if (i) OS << ',';
-      OS << getValue(i);
-    }
+    OS << getSpelling() << llvm::interleaved(getValues(), ",");
     Output.push_back(Args.MakeArgString(OS.str()));
     break;
   }
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 7c0dc3290e10..874fce05841e 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -374,121 +374,6 @@
 
 using namespace llvm;
 
-// AOCC begin
-static cl::opt<bool>
-    EnableNaNsForSQRT("enable-nans-for-sqrt", cl::init(false), cl::Hidden,
-                       cl::desc("Enable NaNs for SQRT."));
-
-static cl::opt<bool> EnableBranchFuse(
-            "phi-elim-preserve-cmpjmp-glue", cl::init(false), cl::Hidden,
-                cl::desc("Place copy statements of PHI nodes before cmp"
-                         "statement during PHI elimination"));
-
-static cl::opt<bool> DelayVectorizationToLTO(
-    "delay-vectorization-to-lto", cl::init(false), cl::Hidden,
-    cl::desc("Delay Vectorization to LTO for resolving memory dependencies."));
-
-
-static cl::opt<bool> EnablePartialUnswitching(
-    "enable-partial-unswitch", cl::init(false), cl::Hidden,
-    cl::desc("Enable experimental partial loop unswitcing"));
-
-static cl::opt<bool> EnableAggressiveUnswitching(
-    "aggressive-loop-unswitch", cl::init(false), cl::Hidden,
-    cl::desc("Enable experimental aggressive loop unswitching"));
-
-static cl::opt<bool>
-    ConvertPowExpToInt("convert-pow-exp-to-int", cl::Hidden,
-                         cl::init(true),
-                         cl::desc("Allow converting exponent of pow "
-                                  "from float to int and call powi"));
-static cl::opt<bool> RunGlobalSLPVectorization(
-    "global-vectorize-slp", cl::init(false), cl::Hidden,
-    cl::desc("Run the Global SLP vectorization passes"));
-static cl::opt<bool> MoveLoadSliceGSLP(
-    "move-load-slice-gslp", cl::init(false), cl::Hidden,
-    cl::desc("Move Load Slices to aid global slp vectorization"));
-static cl::opt<int> EnableReduceArrayComputations(
-    "reduce-array-computations", cl::init(0), cl::Hidden,
-    cl::desc("Enable reduction of array computations"));
-
-static cl::opt<bool>
-RunArrayRemap("remap-arrays", cl::init(false), cl::Hidden,
-                     cl::desc("Run the Array Remap passes"));
-
-static cl::opt<int>
-InputStructPeelMemBlockSize("struct-peel-mem-block-size",
-    cl::init(0), cl::Hidden,
-    cl::desc("structure peeling memory block size."));
-
-static cl::opt<bool> EnableLVFunctionSpecialization(
-    "lv-function-specialization", cl::init(false), cl::Hidden,
-    cl::desc("Enable Linktime Function Specialization For Vectorization"));
-
-static cl::opt<bool> DisableI2DCallPromotion(
-    "disable-itodcalls", cl::init(false), cl::Hidden,
-    cl::desc("Disable indirect calls to direct calls promotion"));
-
-static cl::opt<bool> DisableI2DCallPromotionByClone(
-    "disable-itodcallsbyclone", cl::init(false), cl::Hidden,
-    cl::desc("Disable indirect calls to direct calls promotion by function cloning"));
-
-static cl::opt<bool> rvBoscc(
-    "rv-boscc", cl::init(true), cl::Hidden,
-    cl::desc("enable BOSCC transform with region vectorization"));
-
-static cl::opt<bool> rvVectorize(
-    "region-vectorize", cl::init(false), cl::Hidden,
-    cl::desc("enable region vectorization"));
-
-static cl::opt<bool> markOutlined(
-    "mark-rv-outline", cl::init(true), cl::Hidden,
-    cl::desc("mark outlined functions as inline"));
-
-static cl::opt<bool> rvOutline(
-    "rv-outline", cl::init(false), cl::Hidden,
-    cl::desc("outline conditional code in Region vectorization"));
-
-static cl::opt<unsigned>
-rvDepth("rv-depth", cl::init(0),
-        cl::Hidden,
-        cl::desc("vector factor for the RV vectorized loop"));
-
-static cl::opt<int>
-rvMaxVectorRegSizeOption("rv-max-reg-size", cl::init(128), cl::Hidden,
-    cl::desc("Attempt to vectorize for this register size in bits"));
-
-static cl::opt<bool> EnableBranchCombine("enable-branch-combine",
-    cl::desc("Enable branch fusion pass"),
-    cl::init(false), cl::Hidden);
-
-
-static cl::opt<bool>
-    noStoreSink("simplifycfg-no-storesink", cl::Hidden, cl::init(false),
-               cl::desc("Sink store instructions down to the end block generating phi nodes for address and value"));
-
-
-enum PrefetchLevel {
-    zero, one
-};
-
-static cl::opt<PrefetchLevel> EnablePrefetch("enable-X86-prefetching",
-    cl::desc("enable software prefetching on X86"),
-    cl::ValueOptional, cl::init(PrefetchLevel::zero));
-
-static cl::opt<bool>
-    EnableVRP("enable-licm-vrp", cl::Hidden, cl::init(false),
-    cl::desc("enable register pressure awareness in LICM pass"));
-
-static cl::opt<bool> DoFunctionSpecialize("function-specialize",
-    cl::init(true), cl::desc("Specialize function calls."), cl::Hidden);
-
-// AOCC end
-
-static const Regex DefaultAliasRegex(
-    "^(default|default-post-link|thinlto-pre-link|thinlto|lto-pre-link|lto)"
-    "<(O[0123sz])>$");
-
 cl::opt<bool> llvm::PrintPipelinePasses(
     "print-pipeline-passes",
     cl::desc("Print a '-passes' compatible string describing the pipeline "
@@ -730,6 +615,15 @@ static std::optional<OptimizationLevel> parseOptLevel(StringRef S) {
       .Default(std::nullopt);
 }
 
+static Expected<OptimizationLevel> parseOptLevelParam(StringRef S) {
+  std::optional<OptimizationLevel> OptLevel = parseOptLevel(S);
+  if (OptLevel)
+    return *OptLevel;
+  return make_error<StringError>(
+      formatv("invalid optimization level '{}'", S).str(),
+      inconvertibleErrorCode());
+}
+
 Expected<bool> PassBuilder::parseSinglePassOption(StringRef Params,
                                                   StringRef OptionName,
                                                   StringRef PassName) {
@@ -1044,6 +938,19 @@ parseLowerAllowCheckPassOptions(StringRef Params) {
 
         Result.cutoffs[index] = cutoff;
       }
+    } else if (ParamName.starts_with("runtime_check")) {
+      StringRef ValueString;
+      std::tie(std::ignore, ValueString) = ParamName.split("=");
+      int runtime_check;
+      if (ValueString.getAsInteger(0, runtime_check)) {
+        return make_error<StringError>(
+            formatv("invalid LowerAllowCheck pass runtime_check parameter '{}' "
+                    "({})",
+                    ValueString, Params)
+                .str(),
+            inconvertibleErrorCode());
+      }
+      Result.runtime_check = runtime_check;
     } else {
       return make_error<StringError>(
           formatv("invalid LowerAllowCheck pass parameter '{}'", ParamName)
@@ -1617,15 +1524,43 @@ Expected<bool> parseVirtRegRewriterPassOptions(StringRef Params) {
   return ClearVirtRegs;
 }
 
-} // namespace
+struct FatLTOOptions {
+  OptimizationLevel OptLevel;
+  bool ThinLTO = false;
+  bool EmitSummary = false;
+};
+
+Expected<FatLTOOptions> parseFatLTOOptions(StringRef Params) {
+  FatLTOOptions Result;
+  bool HaveOptLevel = false;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
 
-/// Tests whether a pass name starts with a valid prefix for a default pipeline
-/// alias.
-static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
-  return Name.starts_with("default") || Name.starts_with("thinlto") ||
-         Name.starts_with("lto");
+    if (ParamName == "thinlto") {
+      Result.ThinLTO = true;
+    } else if (ParamName == "emit-summary") {
+      Result.EmitSummary = true;
+    } else if (std::optional<OptimizationLevel> OptLevel =
+                   parseOptLevel(ParamName)) {
+      Result.OptLevel = *OptLevel;
+      HaveOptLevel = true;
+    } else {
+      return make_error<StringError>(
+          formatv("invalid fatlto-pre-link pass parameter '{}'", ParamName)
+              .str(),
+          inconvertibleErrorCode());
+    }
+  }
+  if (!HaveOptLevel)
+    return make_error<StringError>(
+        "missing optimization level for fatlto-pre-link pipeline",
+        inconvertibleErrorCode());
+  return Result;
 }
 
+} // namespace
+
 /// Tests whether registered callbacks will accept a given pass name.
 ///
 /// When parsing a pipeline text, the type of the outermost pipeline may be
@@ -1647,10 +1582,6 @@ static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
 
 template <typename CallbacksT>
 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
-  // Manually handle aliases for pre-configured pipeline fragments.
-  if (startsWithDefaultPipelineAliasPrefix(Name))
-    return DefaultAliasRegex.match(Name);
-
   StringRef NameNoBracket = Name.take_until([](char C) { return C == '<'; });
 
   // Explicitly handle pass manager names.
@@ -1849,6 +1780,15 @@ PassBuilder::parsePipelineText(StringRef Text) {
   return {std::move(ResultPipeline)};
 }
 
+static void setupOptionsForPipelineAlias(PipelineTuningOptions &PTO,
+                                         OptimizationLevel L) {
+  // This is consistent with old pass manager invoked via opt, but
+  // inconsistent with clang. Clang doesn't enable loop vectorization
+  // but does enable slp vectorization at Oz.
+  PTO.LoopVectorization = L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
+  PTO.SLPVectorization = L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
+}
+
 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
                                    const PipelineElement &E) {
   auto &Name = E.Name;
@@ -1901,50 +1841,6 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
     ;
   }
 
-  // Manually handle aliases for pre-configured pipeline fragments.
-  if (startsWithDefaultPipelineAliasPrefix(Name)) {
-    SmallVector<StringRef, 3> Matches;
-    if (!DefaultAliasRegex.match(Name, &Matches))
-      return make_error<StringError>(
-          formatv("unknown default pipeline alias '{}'", Name).str(),
-          inconvertibleErrorCode());
-
-    assert(Matches.size() == 3 && "Must capture two matched strings!");
-
-    OptimizationLevel L = *parseOptLevel(Matches[2]);
-
-    // This is consistent with old pass manager invoked via opt, but
-    // inconsistent with clang. Clang doesn't enable loop vectorization
-    // but does enable slp vectorization at Oz.
-    PTO.LoopVectorization =
-        L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
-    PTO.SLPVectorization =
-        L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
-
-    if (Matches[1] == "default") {
-      MPM.addPass(buildPerModuleDefaultPipeline(L));
-    } else if (Matches[1] == "default-post-link") {
-      MPM.addPass(buildPerModuleDefaultPipeline(
-          L, ThinOrFullLTOPhase::CustomLTOPostLink));
-    } else if (Matches[1] == "thinlto-pre-link") {
-      MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
-    } else if (Matches[1] == "thinlto") {
-      MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
-    } else if (Matches[1] == "lto-pre-link") {
-      if (PTO.UnifiedLTO)
-        // When UnifiedLTO is enabled, use the ThinLTO pre-link pipeline. This
-        // avoids compile-time performance regressions and keeps the pre-link
-        // LTO pipeline "unified" for both LTO modes.
-        MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L));
-      else
-        MPM.addPass(buildLTOPreLinkDefaultPipeline(L));
-    } else {
-      assert(Matches[1] == "lto" && "Not one of the matched options!");
-      MPM.addPass(buildLTODefaultPipeline(L, nullptr));
-    }
-    return Error::success();
-  }
-
   // Finally expand the basic registered passes from the .inc file.
 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
   if (Name == NAME) {                                                          \
@@ -2322,9 +2218,18 @@ Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
 Error PassBuilder::parseMachinePass(MachineFunctionPassManager &MFPM,
                                     const PipelineElement &E) {
   StringRef Name = E.Name;
-  if (!E.InnerPipeline.empty())
+  // Handle any nested pass managers.
+  if (!E.InnerPipeline.empty()) {
+    if (E.Name == "machine-function") {
+      MachineFunctionPassManager NestedPM;
+      if (auto Err = parseMachinePassPipeline(NestedPM, E.InnerPipeline))
+        return Err;
+      MFPM.addPass(std::move(NestedPM));
+      return Error::success();
+    }
     return make_error<StringError>("invalid pipeline",
                                    inconvertibleErrorCode());
+  }
 
 #define MACHINE_MODULE_PASS(NAME, CREATE_PASS)                                 \
   if (Name == NAME) {                                                          \
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 9ac80c6c77e4..171da1d8154a 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -696,9 +696,6 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 
   LPM2.addPass(LoopDeletionPass());
 
-  if (PTO.LoopInterchange)
-    LPM2.addPass(LoopInterchangePass());
-
   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
   // because it changes IR to makes profile annotation in back compile
   // inaccurate. The normal unroller doesn't pay attention to forced full unroll
@@ -1555,6 +1552,10 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   //        this may need to be revisited once we run GVN before loop deletion
   //        in the simplification pipeline.
   LPM.addPass(LoopDeletionPass());
+
+  if (PTO.LoopInterchange)
+    LPM.addPass(LoopInterchangePass());
+
   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
       std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false));
 
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index ec14c6a9211d..06d416d11251 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -63,7 +63,8 @@ MODULE_PASS("coro-early", CoroEarlyPass())
 MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
 MODULE_PASS("ctx-instr-gen",
             PGOInstrumentationGen(PGOInstrumentationType::CTXPROF))
-MODULE_PASS("ctx-prof-flatten", PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false))
+MODULE_PASS("ctx-prof-flatten",
+            PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false))
 MODULE_PASS("ctx-prof-flatten-prethinlink",
             PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true))
 MODULE_PASS("noinline-nonprevailing", NoinlineNonPrevailing())
@@ -74,13 +75,15 @@ MODULE_PASS("dot-callgraph", CallGraphDOTPrinterPass())
 MODULE_PASS("dxil-upgrade", DXILUpgradePass())
 MODULE_PASS("elim-avail-extern", EliminateAvailableExternallyPass())
 MODULE_PASS("extract-blocks", BlockExtractorPass({}, false))
-MODULE_PASS("expand-variadics", ExpandVariadicsPass(ExpandVariadicsMode::Disable))
+MODULE_PASS("expand-variadics",
+            ExpandVariadicsPass(ExpandVariadicsMode::Disable))
 MODULE_PASS("forceattrs", ForceFunctionAttrsPass())
 MODULE_PASS("function-import", FunctionImportPass())
 MODULE_PASS("global-merge-func", GlobalMergeFuncPass())
 MODULE_PASS("globalopt", GlobalOptPass())
 MODULE_PASS("globalsplit", GlobalSplitPass())
 MODULE_PASS("hipstdpar-interpose-alloc", HipStdParAllocationInterpositionPass())
+MODULE_PASS("hipstdpar-math-fixup", HipStdParMathFixupPass())
 MODULE_PASS("hipstdpar-select-accelerator-code",
             HipStdParAcceleratorCodeSelectionPass())
 MODULE_PASS("hotcoldsplit", HotColdSplittingPass())
@@ -104,7 +107,10 @@ MODULE_PASS("lower-ifunc", LowerIFuncPass())
 MODULE_PASS("simplify-type-tests", SimplifyTypeTestsPass())
 MODULE_PASS("lowertypetests", LowerTypeTestsPass())
 MODULE_PASS("fatlto-cleanup", FatLtoCleanup())
-MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default))
+MODULE_PASS("pgo-force-function-attrs",
+            PGOForceFunctionAttrsPass(PGOOpt
+                                          ? PGOOpt->ColdOptType
+                                          : PGOOptions::ColdFuncOpt::Default))
 MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation())
 MODULE_PASS("memprof-module", ModuleMemProfilerPass())
 MODULE_PASS("mergefunc", MergeFunctionsPass())
@@ -136,6 +142,7 @@ MODULE_PASS("print<dxil-metadata>", DXILMetadataAnalysisPrinterPass(errs()))
 MODULE_PASS("print<dxil-resources>", DXILResourcePrinterPass(errs()))
 MODULE_PASS("print<inline-advisor>", InlineAdvisorAnalysisPrinterPass(errs()))
 MODULE_PASS("print<ir2vec>", IR2VecPrinterPass(errs()))
+MODULE_PASS("print<ir2vec-vocab>", IR2VecVocabPrinterPass(errs()))
 MODULE_PASS("print<module-debuginfo>", ModuleDebugInfoPrinterPass(errs()))
 MODULE_PASS("print<reg-usage>", PhysicalRegisterUsageInfoPrinterPass(errs()))
 MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
@@ -175,10 +182,10 @@ MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass())
 MODULE_PASS_WITH_PARAMS(
     "asan", "AddressSanitizerPass",
     [](AddressSanitizerOptions Opts) { return AddressSanitizerPass(Opts); },
-    parseASanPassOptions, "kernel")
+    parseASanPassOptions, "kernel;use-after-scope")
 MODULE_PASS_WITH_PARAMS(
     "cg-profile", "CGProfilePass",
-    [](bool InLTOPostLink) { return CGProfilePass(InLTOPostLink);},
+    [](bool InLTOPostLink) { return CGProfilePass(InLTOPostLink); },
     parseCGProfilePassOptions, "in-lto-post-link")
 MODULE_PASS_WITH_PARAMS(
     "global-merge", "GlobalMergePass",
@@ -187,7 +194,7 @@ MODULE_PASS_WITH_PARAMS(
     "group-by-use;ignore-single-use;max-offset=N;merge-const;"
     "merge-const-aggressive;merge-external;no-group-by-use;"
     "no-ignore-single-use;no-merge-const;no-merge-const-aggressive;"
-    "no-merge-external;size-only")
+    "no-merge-external")
 MODULE_PASS_WITH_PARAMS(
     "embed-bitcode", "EmbedBitcodePass",
     [](EmbedBitcodeOptions Opts) { return EmbedBitcodePass(Opts); },
@@ -195,7 +202,7 @@ MODULE_PASS_WITH_PARAMS(
 MODULE_PASS_WITH_PARAMS(
     "globaldce", "GlobalDCEPass",
     [](bool InLTOPostLink) { return GlobalDCEPass(InLTOPostLink); },
-    parseGlobalDCEPassOptions, "in-lto-post-link")
+    parseGlobalDCEPassOptions, "vfe-linkage-unit-visibility")
 MODULE_PASS_WITH_PARAMS(
     "hwasan", "HWAddressSanitizerPass",
     [](HWAddressSanitizerOptions Opts) { return HWAddressSanitizerPass(Opts); },
@@ -240,6 +247,57 @@ MODULE_PASS_WITH_PARAMS(
     },
     parseStructuralHashPrinterPassOptions, "detailed;call-target-ignored")
 
+MODULE_PASS_WITH_PARAMS(
+    "default", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      return buildPerModuleDefaultPipeline(L);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "default-post-link", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      return buildPerModuleDefaultPipeline(
+          L, ThinOrFullLTOPhase::CustomLTOPostLink);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "thinlto-pre-link", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      return buildThinLTOPreLinkDefaultPipeline(L);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "thinlto", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      return buildThinLTODefaultPipeline(L, nullptr);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "lto-pre-link", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      if (PTO.UnifiedLTO) {
+        // When UnifiedLTO is enabled, use the ThinLTO pre-link pipeline. This
+        // avoids compile-time performance regressions and keeps the pre-link
+        // LTO pipeline "unified" for both LTO modes.
+        return buildThinLTOPreLinkDefaultPipeline(L);
+      }
+      return buildLTOPreLinkDefaultPipeline(L);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "lto", "", [&](OptimizationLevel L) {
+      setupOptionsForPipelineAlias(PTO, L);
+      return buildLTODefaultPipeline(L, nullptr);
+    },
+    parseOptLevelParam, "O0;O1;O2;O3;Os;Oz")
+MODULE_PASS_WITH_PARAMS(
+    "fatlto-pre-link", "", [&](FatLTOOptions Opts) {
+      setupOptionsForPipelineAlias(PTO, Opts.OptLevel);
+      return buildFatLTODefaultPipeline(Opts.OptLevel, Opts.ThinLTO,
+                                        Opts.EmitSummary);
+    },
+    parseFatLTOOptions, "O0;O1;O2;O3;Os;Oz;thinlto;emit-summary")
+
 #undef MODULE_PASS_WITH_PARAMS
 
 #ifndef CGSCC_ANALYSIS
@@ -287,7 +345,8 @@ CGSCC_PASS_WITH_PARAMS(
 FUNCTION_ANALYSIS("aa", AAManager())
 FUNCTION_ANALYSIS("access-info", LoopAccessAnalysis())
 FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis())
-FUNCTION_ANALYSIS("bb-sections-profile-reader", BasicBlockSectionsProfileReaderAnalysis(TM))
+FUNCTION_ANALYSIS("bb-sections-profile-reader",
+                  BasicBlockSectionsProfileReaderAnalysis(TM))
 FUNCTION_ANALYSIS("block-freq", BlockFrequencyAnalysis())
 FUNCTION_ANALYSIS("branch-prob", BranchProbabilityAnalysis())
 FUNCTION_ANALYSIS("cycles", CycleAnalysis())
@@ -377,7 +436,7 @@ FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM))
 FUNCTION_PASS("expand-fp", ExpandFpPass(TM))
 FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(TM))
 FUNCTION_PASS("extra-vector-passes",
-                  ExtraFunctionPassManager<ShouldRunExtraVectorPasses>())
+              ExtraFunctionPassManager<ShouldRunExtraVectorPasses>())
 FUNCTION_PASS("fix-irreducible", FixIrreduciblePass())
 FUNCTION_PASS("flatten-cfg", FlattenCFGPass())
 FUNCTION_PASS("float2int", Float2IntPass())
@@ -544,12 +603,10 @@ FUNCTION_PASS_WITH_PARAMS(
     "instcombine", "InstCombinePass",
     [](InstCombineOptions Opts) { return InstCombinePass(Opts); },
     parseInstCombineOptions,
-    "no-use-loop-info;use-loop-info;no-verify-fixpoint;verify-fixpoint;"
-    "max-iterations=N")
+    "no-verify-fixpoint;verify-fixpoint;max-iterations=N")
 FUNCTION_PASS_WITH_PARAMS(
     "lint", "LintPass",
-    [](bool AbortOnError) { return LintPass(AbortOnError); },
-    parseLintOptions,
+    [](bool AbortOnError) { return LintPass(AbortOnError); }, parseLintOptions,
     "abort-on-error")
 FUNCTION_PASS_WITH_PARAMS(
     "loop-unroll", "LoopUnrollPass",
@@ -567,7 +624,7 @@ FUNCTION_PASS_WITH_PARAMS(
 FUNCTION_PASS_WITH_PARAMS(
     "lower-allow-check", "LowerAllowCheckPass",
     [](LowerAllowCheckPass::Options Opts) { return LowerAllowCheckPass(Opts); },
-    parseLowerAllowCheckPassOptions, "")
+    parseLowerAllowCheckPassOptions, "cutoffs[indices]=N")
 FUNCTION_PASS_WITH_PARAMS(
     "lower-matrix-intrinsics", "LowerMatrixIntrinsicsPass",
     [](bool Minimal) { return LowerMatrixIntrinsicsPass(Minimal); },
@@ -576,7 +633,8 @@ FUNCTION_PASS_WITH_PARAMS(
     "normalize", "IRNormalizerPass",
     [](IRNormalizerOptions Options) { return IRNormalizerPass(Options); },
     parseIRNormalizerPassOptions,
-    "no-preserve-order;preserve-order;no-rename-all;rename-all;no-fold-all;fold-all;no-reorder-operands;reorder-operands")
+    "no-preserve-order;preserve-order;no-rename-all;rename-all;no-fold-all;"
+    "fold-all;no-reorder-operands;reorder-operands")
 FUNCTION_PASS_WITH_PARAMS(
     "mldst-motion", "MergedLoadStoreMotionPass",
     [](MergedLoadStoreMotionOptions Opts) {
@@ -590,7 +648,7 @@ FUNCTION_PASS_WITH_PARAMS(
     },
     [](StringRef Params) {
       return PassBuilder::parseSinglePassOption(Params, "allow-partial",
-						"LoopAccessInfoPrinterPass");
+                                                "LoopAccessInfoPrinterPass");
     },
     "allow-partial")
 FUNCTION_PASS_WITH_PARAMS(
@@ -625,10 +683,14 @@ FUNCTION_PASS_WITH_PARAMS(
     "simplifycfg", "SimplifyCFGPass",
     [](SimplifyCFGOptions Opts) { return SimplifyCFGPass(Opts); },
     parseSimplifyCFGOptions,
-    "no-forward-switch-cond;forward-switch-cond;no-switch-range-to-icmp;"
-    "switch-range-to-icmp;no-switch-to-lookup;switch-to-lookup;no-keep-loops;"
-    "keep-loops;no-hoist-common-insts;hoist-common-insts;no-sink-common-insts;"
-    "sink-common-insts;bonus-inst-threshold=N")
+    "no-speculate-blocks;speculate-blocks;no-simplify-cond-branch;"
+    "simplify-cond-branch;no-forward-switch-cond;forward-switch-cond;"
+    "no-switch-range-to-icmp;switch-range-to-icmp;no-switch-to-lookup;"
+    "switch-to-lookup;no-keep-loops;keep-loops;no-hoist-common-insts;"
+    "hoist-common-insts;no-hoist-loads-stores-with-cond-faulting;"
+    "hoist-loads-stores-with-cond-faulting;no-sink-common-insts;"
+    "sink-common-insts;no-speculate-unpredictables;speculate-unpredictables;"
+    "bonus-inst-threshold=N")
 FUNCTION_PASS_WITH_PARAMS(
     "speculative-execution", "SpeculativeExecutionPass",
     [](bool OnlyIfDivergentTarget) {
@@ -640,11 +702,11 @@ FUNCTION_PASS_WITH_PARAMS(
     [](SROAOptions PreserveCFG) { return SROAPass(PreserveCFG); },
     parseSROAOptions, "preserve-cfg;modify-cfg")
 FUNCTION_PASS_WITH_PARAMS(
-  "structurizecfg", "StructurizeCFG",
-  [](bool SkipUniformRegions) {
-    return StructurizeCFGPass(SkipUniformRegions);
-  },
-  parseStructurizeCFGPassOptions, "skip-uniform-regions")
+    "structurizecfg", "StructurizeCFG",
+    [](bool SkipUniformRegions) {
+      return StructurizeCFGPass(SkipUniformRegions);
+    },
+    parseStructurizeCFGPassOptions, "skip-uniform-regions")
 FUNCTION_PASS_WITH_PARAMS(
     "win-eh-prepare", "WinEHPreparePass",
     [](bool DemoteCatchSwitchPHIOnly) {
@@ -656,7 +718,8 @@ FUNCTION_PASS_WITH_PARAMS(
     [](BoundsCheckingPass::Options Options) {
       return BoundsCheckingPass(Options);
     },
-    parseBoundsCheckingOptions, "trap")
+    parseBoundsCheckingOptions,
+    "trap;rt;rt-abort;min-rt;min-rt-abort;merge;guard=N")
 #undef FUNCTION_PASS_WITH_PARAMS
 
 #ifndef LOOPNEST_PASS
@@ -676,7 +739,7 @@ LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
 LOOP_ANALYSIS("should-run-extra-simple-loop-unswitch",
-                  ShouldRunExtraSimpleLoopUnswitch())
+              ShouldRunExtraSimpleLoopUnswitch())
 
 #undef LOOP_ANALYSIS
 
@@ -688,7 +751,7 @@ LOOP_PASS("dot-ddg", DDGDotPrinterPass())
 LOOP_PASS("evl-iv-simplify", EVLIndVarSimplifyPass())
 LOOP_PASS("guard-widening", GuardWideningPass())
 LOOP_PASS("extra-simple-loop-unswitch-passes",
-              ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch>())
+          ExtraLoopPassManager<ShouldRunExtraSimpleLoopUnswitch>())
 LOOP_PASS("indvars", IndVarSimplifyPass())
 LOOP_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 LOOP_PASS("loop-bound-split", LoopBoundSplitPass())
@@ -716,10 +779,10 @@ LOOP_PASS("print<loopnest>", LoopNestPrinterPass(errs()))
 #endif
 LOOP_PASS_WITH_PARAMS(
     "licm", "LICMPass", [](LICMOptions Params) { return LICMPass(Params); },
-    parseLICMOptions, "allowspeculation")
+    parseLICMOptions, "allowspeculation;no-allowspeculation")
 LOOP_PASS_WITH_PARAMS(
     "lnicm", "LNICMPass", [](LICMOptions Params) { return LNICMPass(Params); },
-    parseLICMOptions, "allowspeculation")
+    parseLICMOptions, "allowspeculation;no-allowspeculation")
 LOOP_PASS_WITH_PARAMS(
     "loop-rotate", "LoopRotatePass",
     [](std::pair<bool, bool> Params) {
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 4e8abc43572d..0623e6677204 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -1074,7 +1074,7 @@ bool OptPassGateInstrumentation::shouldRun(StringRef PassName, Any IR) {
 
 void OptPassGateInstrumentation::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
-  OptPassGate &PassGate = Context.getOptPassGate();
+  const OptPassGate &PassGate = Context.getOptPassGate();
   if (!PassGate.isEnabled())
     return;
 
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 368e3535fe90..bd3964ce127f 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -27,6 +27,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ProfDataUtils.h"
 #include "llvm/IR/Type.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/Casting.h"
@@ -616,28 +617,24 @@ Error readAndDecodeStrings(StringRef NameStrings,
 }
 
 Error InstrProfSymtab::create(StringRef NameStrings) {
-  return readAndDecodeStrings(
-      NameStrings,
-      std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1));
+  return readAndDecodeStrings(NameStrings,
+                              [&](StringRef S) { return addFuncName(S); });
 }
 
 Error InstrProfSymtab::create(StringRef FuncNameStrings,
                               StringRef VTableNameStrings) {
-  if (Error E = readAndDecodeStrings(FuncNameStrings,
-                                     std::bind(&InstrProfSymtab::addFuncName,
-                                               this, std::placeholders::_1)))
+  if (Error E = readAndDecodeStrings(
+          FuncNameStrings, [&](StringRef S) { return addFuncName(S); }))
     return E;
 
-  return readAndDecodeStrings(
-      VTableNameStrings,
-      std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+  return readAndDecodeStrings(VTableNameStrings,
+                              [&](StringRef S) { return addVTableName(S); });
 }
 
 Error InstrProfSymtab::initVTableNamesFromCompressedStrings(
     StringRef CompressedVTableStrings) {
-  return readAndDecodeStrings(
-      CompressedVTableStrings,
-      std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1));
+  return readAndDecodeStrings(CompressedVTableStrings,
+                              [&](StringRef S) { return addVTableName(S); });
 }
 
 StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) {
@@ -1358,7 +1355,7 @@ void annotateValueSite(Module &M, Instruction &Inst,
   MDBuilder MDHelper(Ctx);
   SmallVector<Metadata *, 3> Vals;
   // Tag
-  Vals.push_back(MDHelper.createString("VP"));
+  Vals.push_back(MDHelper.createString(MDProfLabels::ValueProfile));
   // Value Kind
   Vals.push_back(MDHelper.createConstant(
       ConstantInt::get(Type::getInt32Ty(Ctx), ValueKind)));
@@ -1389,7 +1386,7 @@ MDNode *mayHaveValueProfileOfKind(const Instruction &Inst,
     return nullptr;
 
   MDString *Tag = cast<MDString>(MD->getOperand(0));
-  if (!Tag || Tag->getString() != "VP")
+  if (!Tag || Tag->getString() != MDProfLabels::ValueProfile)
     return nullptr;
 
   // Now check kind:
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index ab109cd5b13a..5c7b9e054403 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1369,7 +1369,7 @@ InstrProfSymtab &IndexedInstrProfReader::getSymtab() {
   return *Symtab;
 }
 
-Expected<InstrProfRecord> IndexedInstrProfReader::getInstrProfRecord(
+Expected<NamedInstrProfRecord> IndexedInstrProfReader::getInstrProfRecord(
     StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName,
     uint64_t *MismatchedFuncSum) {
   ArrayRef<NamedInstrProfRecord> Data;
@@ -1572,7 +1572,7 @@ memprof::AllMemProfData IndexedMemProfReader::getAllMemProfData() const {
 Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,
                                                 uint64_t FuncHash,
                                                 std::vector<uint64_t> &Counts) {
-  Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
+  auto Record = getInstrProfRecord(FuncName, FuncHash);
   if (Error E = Record.takeError())
     return error(std::move(E));
 
@@ -1583,7 +1583,7 @@ Error IndexedInstrProfReader::getFunctionCounts(StringRef FuncName,
 Error IndexedInstrProfReader::getFunctionBitmap(StringRef FuncName,
                                                 uint64_t FuncHash,
                                                 BitVector &Bitmap) {
-  Expected<InstrProfRecord> Record = getInstrProfRecord(FuncName, FuncHash);
+  auto Record = getInstrProfRecord(FuncName, FuncHash);
   if (Error E = Record.takeError())
     return error(std::move(E));
 
diff --git a/llvm/lib/Support/Signals.cpp b/llvm/lib/Support/Signals.cpp
index 9f9030e79d10..f8a14a45ddc3 100644
--- a/llvm/lib/Support/Signals.cpp
+++ b/llvm/lib/Support/Signals.cpp
@@ -31,7 +31,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include <array>
 #include <cmath>
-#include <vector>
 
 //===----------------------------------------------------------------------===//
 //=== WARNING: Implementation here must contain only TRULY operating system
@@ -137,47 +136,28 @@ static FormattedNumber format_ptr(void *PC) {
   return format_hex((uint64_t)PC, PtrWidth);
 }
 
-/// Helper that launches llvm-symbolizer and symbolizes a backtrace.
-LLVM_ATTRIBUTE_USED
-static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
-                                      int Depth, llvm::raw_ostream &OS) {
-  if (DisableSymbolicationFlag || getenv(DisableSymbolizationEnv))
-    return false;
-
-  // Don't recursively invoke the llvm-symbolizer binary.
-  if (Argv0.contains("llvm-symbolizer"))
-    return false;
-
-  // FIXME: Subtract necessary number from StackTrace entries to turn return addresses
-  // into actual instruction addresses.
-  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
-  // alongside our binary, then in $PATH.
-  ErrorOr<std::string> LLVMSymbolizerPathOrErr = std::error_code();
-  if (const char *Path = getenv(LLVMSymbolizerPathEnv)) {
-    LLVMSymbolizerPathOrErr = sys::findProgramByName(Path);
-  } else if (!Argv0.empty()) {
-    StringRef Parent = llvm::sys::path::parent_path(Argv0);
-    if (!Parent.empty())
-      LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer", Parent);
-  }
-  if (!LLVMSymbolizerPathOrErr)
-    LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer");
-  if (!LLVMSymbolizerPathOrErr)
-    return false;
-  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
-
-  // If we don't know argv0 or the address of main() at this point, try
-  // to guess it anyway (it's possible on some platforms).
-  std::string MainExecutableName =
-      sys::fs::exists(Argv0) ? (std::string)std::string(Argv0)
-                             : sys::fs::getMainExecutable(nullptr, nullptr);
+/// Reads a file \p Filename written by llvm-symbolizer containing function
+/// names and source locations for the addresses in \p AddressList and returns
+/// the strings in a vector of pairs, where the first pair element is the index
+/// of the corresponding entry in AddressList and the second is the symbolized
+/// frame, in a format based on the sanitizer stack trace printer, with the
+/// exception that it does not write out frame numbers (i.e. "#2 " for the
+/// third address), as it is not assumed that \p AddressList corresponds to a
+/// single stack trace.
+/// There may be multiple returned entries for a single \p AddressList entry if
+/// that frame address corresponds to one or more inlined frames; in this case,
+/// all frames for an address will appear contiguously and in-order.
+std::optional<SmallVector<std::pair<unsigned, std::string>, 0>>
+collectAddressSymbols(void **AddressList, unsigned AddressCount,
+                      const char *MainExecutableName,
+                      const std::string &LLVMSymbolizerPath) {
   BumpPtrAllocator Allocator;
   StringSaver StrPool(Allocator);
-  std::vector<const char *> Modules(Depth, nullptr);
-  std::vector<intptr_t> Offsets(Depth, 0);
-  if (!findModulesAndOffsets(StackTrace, Depth, Modules.data(), Offsets.data(),
-                             MainExecutableName.c_str(), StrPool))
-    return false;
+  SmallVector<const char *, 0> Modules(AddressCount, nullptr);
+  SmallVector<intptr_t, 0> Offsets(AddressCount, 0);
+  if (!findModulesAndOffsets(AddressList, AddressCount, Modules.data(),
+                             Offsets.data(), MainExecutableName, StrPool))
+    return {};
   int InputFD;
   SmallString<32> InputFile, OutputFile;
   sys::fs::createTemporaryFile("symbolizer-input", "", InputFD, InputFile);
@@ -187,9 +167,9 @@ static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
 
   {
     raw_fd_ostream Input(InputFD, true);
-    for (int i = 0; i < Depth; i++) {
-      if (Modules[i])
-        Input << Modules[i] << " " << (void*)Offsets[i] << "\n";
+    for (unsigned AddrIdx = 0; AddrIdx < AddressCount; AddrIdx++) {
+      if (Modules[AddrIdx])
+        Input << Modules[AddrIdx] << " " << (void *)Offsets[AddrIdx] << "\n";
     }
   }
 
@@ -206,53 +186,149 @@ static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
   int RunResult =
       sys::ExecuteAndWait(LLVMSymbolizerPath, Args, std::nullopt, Redirects);
   if (RunResult != 0)
-    return false;
+    return {};
 
-  // This report format is based on the sanitizer stack trace printer.  See
-  // sanitizer_stacktrace_printer.cc in compiler-rt.
+  SmallVector<std::pair<unsigned, std::string>, 0> Result;
   auto OutputBuf = MemoryBuffer::getFile(OutputFile.c_str());
   if (!OutputBuf)
-    return false;
+    return {};
   StringRef Output = OutputBuf.get()->getBuffer();
   SmallVector<StringRef, 32> Lines;
   Output.split(Lines, "\n");
-  auto CurLine = Lines.begin();
-  int frame_no = 0;
-  for (int i = 0; i < Depth; i++) {
-    auto PrintLineHeader = [&]() {
-      OS << right_justify(formatv("#{0}", frame_no++).str(),
-                          std::log10(Depth) + 2)
-         << ' ' << format_ptr(StackTrace[i]) << ' ';
-    };
-    if (!Modules[i]) {
-      PrintLineHeader();
-      OS << '\n';
+  auto *CurLine = Lines.begin();
+  // Lines contains the output from llvm-symbolizer, which should contain for
+  // each address with a module in order of appearance, one or more lines
+  // containing the function name and line associated with that address,
+  // followed by an empty line.
+  // For each address, adds an output entry for every real or inlined frame at
+  // that address. For addresses without known modules, we have a single entry
+  // containing just the formatted address; for all other output entries, we
+  // output the function entry if it is known, and either the line number if it
+  // is known or the module+address offset otherwise.
+  for (unsigned AddrIdx = 0; AddrIdx < AddressCount; AddrIdx++) {
+    if (!Modules[AddrIdx]) {
+      auto &SymbolizedFrame = Result.emplace_back(std::make_pair(AddrIdx, ""));
+      raw_string_ostream OS(SymbolizedFrame.second);
+      OS << format_ptr(AddressList[AddrIdx]);
       continue;
     }
     // Read pairs of lines (function name and file/line info) until we
     // encounter empty line.
     for (;;) {
       if (CurLine == Lines.end())
-        return false;
+        return {};
       StringRef FunctionName = *CurLine++;
       if (FunctionName.empty())
         break;
-      PrintLineHeader();
+      auto &SymbolizedFrame = Result.emplace_back(std::make_pair(AddrIdx, ""));
+      raw_string_ostream OS(SymbolizedFrame.second);
+      OS << format_ptr(AddressList[AddrIdx]) << ' ';
       if (!FunctionName.starts_with("??"))
         OS << FunctionName << ' ';
       if (CurLine == Lines.end())
-        return false;
+        return {};
       StringRef FileLineInfo = *CurLine++;
-      if (!FileLineInfo.starts_with("??"))
+      if (!FileLineInfo.starts_with("??")) {
         OS << FileLineInfo;
-      else
-        OS << "(" << Modules[i] << '+' << format_hex(Offsets[i], 0) << ")";
-      OS << "\n";
+      } else {
+        OS << "(" << Modules[AddrIdx] << '+' << format_hex(Offsets[AddrIdx], 0)
+           << ")";
+      }
     }
   }
+  return Result;
+}
+
+ErrorOr<std::string> getLLVMSymbolizerPath(StringRef Argv0 = {}) {
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = std::error_code();
+  if (const char *Path = getenv(LLVMSymbolizerPathEnv)) {
+    LLVMSymbolizerPathOrErr = sys::findProgramByName(Path);
+  } else if (!Argv0.empty()) {
+    StringRef Parent = llvm::sys::path::parent_path(Argv0);
+    if (!Parent.empty())
+      LLVMSymbolizerPathOrErr =
+          sys::findProgramByName("llvm-symbolizer", Parent);
+  }
+  if (!LLVMSymbolizerPathOrErr)
+    LLVMSymbolizerPathOrErr = sys::findProgramByName("llvm-symbolizer");
+  return LLVMSymbolizerPathOrErr;
+}
+
+/// Helper that launches llvm-symbolizer and symbolizes a backtrace.
+LLVM_ATTRIBUTE_USED
+static bool printSymbolizedStackTrace(StringRef Argv0, void **StackTrace,
+                                      int Depth, llvm::raw_ostream &OS) {
+  if (DisableSymbolicationFlag || getenv(DisableSymbolizationEnv))
+    return false;
+
+  // Don't recursively invoke the llvm-symbolizer binary.
+  if (Argv0.contains("llvm-symbolizer"))
+    return false;
+
+  // FIXME: Subtract necessary number from StackTrace entries to turn return
+  // addresses into actual instruction addresses.
+  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
+  // alongside our binary, then in $PATH.
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = getLLVMSymbolizerPath(Argv0);
+  if (!LLVMSymbolizerPathOrErr)
+    return false;
+  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
+
+  // If we don't know argv0 or the address of main() at this point, try
+  // to guess it anyway (it's possible on some platforms).
+  std::string MainExecutableName =
+      sys::fs::exists(Argv0) ? std::string(Argv0)
+                             : sys::fs::getMainExecutable(nullptr, nullptr);
+
+  auto SymbolizedAddressesOpt = collectAddressSymbols(
+      StackTrace, Depth, MainExecutableName.c_str(), LLVMSymbolizerPath);
+  if (!SymbolizedAddressesOpt)
+    return false;
+  for (unsigned FrameNo = 0; FrameNo < SymbolizedAddressesOpt->size();
+       ++FrameNo) {
+    OS << right_justify(formatv("#{0}", FrameNo).str(), std::log10(Depth) + 2)
+       << ' ' << (*SymbolizedAddressesOpt)[FrameNo].second << '\n';
+  }
   return true;
 }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+void sys::symbolizeAddresses(AddressSet &Addresses,
+                             SymbolizedAddressMap &SymbolizedAddresses) {
+  assert(!DisableSymbolicationFlag && !getenv(DisableSymbolizationEnv) &&
+         "Debugify origin stacktraces require symbolization to be enabled.");
+
+  // Convert Set of Addresses to ordered list.
+  SmallVector<void *, 0> AddressList(Addresses.begin(), Addresses.end());
+  if (AddressList.empty())
+    return;
+  llvm::sort(AddressList);
+
+  // Use llvm-symbolizer tool to symbolize the stack traces. First look for it
+  // alongside our binary, then in $PATH.
+  ErrorOr<std::string> LLVMSymbolizerPathOrErr = getLLVMSymbolizerPath();
+  if (!LLVMSymbolizerPathOrErr)
+    report_fatal_error("Debugify origin stacktraces require llvm-symbolizer");
+  const std::string &LLVMSymbolizerPath = *LLVMSymbolizerPathOrErr;
+
+  // Try to guess the main executable name, since we don't have argv0 available
+  // here.
+  std::string MainExecutableName = sys::fs::getMainExecutable(nullptr, nullptr);
+
+  auto SymbolizedAddressesOpt =
+      collectAddressSymbols(AddressList.begin(), AddressList.size(),
+                            MainExecutableName.c_str(), LLVMSymbolizerPath);
+  if (!SymbolizedAddressesOpt)
+    return;
+  for (auto SymbolizedFrame : *SymbolizedAddressesOpt) {
+    SmallVector<std::string, 0> &SymbolizedAddrs =
+        SymbolizedAddresses[AddressList[SymbolizedFrame.first]];
+    SymbolizedAddrs.push_back(SymbolizedFrame.second);
+  }
+  return;
+}
+#endif
+
 static bool printMarkupContext(raw_ostream &OS, const char *MainExecutableName);
 
 LLVM_ATTRIBUTE_USED
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index b675966c8d82..b4ee0f8ee8bf 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -271,7 +271,7 @@ TextEncodingConverterIconv::convertString(StringRef Source,
     // Setup the input. Use nullptr to reset iconv state if input length is
     // zero.
     size_t InputLength = Source.size();
-    char *Input = InputLength ? const_cast<char *>(Source.data()) : "";
+    char *Input = const_cast<char *>(InputLength ? Source.data() : "");
     Ret = iconv(ConvDesc, &Input, &InputLength, &Output, &OutputLength);
     if (Ret != 0) {
       if (auto EC = HandleError(Ret))
diff --git a/llvm/lib/Support/Unix/Signals.inc b/llvm/lib/Support/Unix/Signals.inc
index 6668a2953b3b..a4525a590364 100644
--- a/llvm/lib/Support/Unix/Signals.inc
+++ b/llvm/lib/Support/Unix/Signals.inc
@@ -507,6 +507,21 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) {
   return 0;
 }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#if !defined(HAVE_BACKTRACE)
+#error DebugLoc origin-tracking currently requires `backtrace()`.
+#endif
+namespace llvm {
+namespace sys {
+template <unsigned long MaxDepth>
+int getStackTrace(std::array<void *, MaxDepth> &StackTrace) {
+  return backtrace(StackTrace.data(), MaxDepth);
+}
+template int getStackTrace<16ul>(std::array<void *, 16ul> &);
+} // namespace sys
+} // namespace llvm
+#endif
+
 /// If this is an ELF platform, we can find all loaded modules and their virtual
 /// addresses with dl_iterate_phdr.
 static bool findModulesAndOffsets(void **StackTrace, int Depth,
diff --git a/llvm/lib/Support/Unix/Threading.inc b/llvm/lib/Support/Unix/Threading.inc
index 562971aae32f..7854d6d22915 100644
--- a/llvm/lib/Support/Unix/Threading.inc
+++ b/llvm/lib/Support/Unix/Threading.inc
@@ -62,6 +62,10 @@
 #include <unistd.h>      // For syscall()
 #endif
 
+#if defined(__CYGWIN__)
+#include <sys/cpuset.h>
+#endif
+
 #if defined(__HAIKU__)
 #include <OS.h> // For B_OS_NAME_LENGTH
 #endif
@@ -165,6 +169,8 @@ static constexpr uint32_t get_max_thread_name_length_impl() {
   return 16;
 #elif defined(__OpenBSD__)
   return 24;
+#elif defined(__CYGWIN__)
+  return 16;
 #else
   return 0;
 #endif
@@ -241,7 +247,7 @@ void llvm::get_thread_name(SmallVectorImpl<char> &Name) {
   }
   free(kp);
   return;
-#elif defined(__linux__) && HAVE_PTHREAD_GETNAME_NP
+#elif (defined(__linux__) || defined(__CYGWIN__)) && HAVE_PTHREAD_GETNAME_NP
   constexpr uint32_t len = get_max_thread_name_length_impl();
   char Buffer[len] = {'\0'}; // FIXME: working around MSan false positive.
   if (0 == ::pthread_getname_np(::pthread_self(), Buffer, len))
@@ -263,7 +269,7 @@ void llvm::get_thread_name(SmallVectorImpl<char> &Name) {
 }
 
 SetThreadPriorityResult llvm::set_thread_priority(ThreadPriority Priority) {
-#if defined(__linux__) && defined(SCHED_IDLE)
+#if (defined(__linux__) || defined(__CYGWIN__)) && defined(SCHED_IDLE)
   // Some *really* old glibcs are missing SCHED_IDLE.
   // http://man7.org/linux/man-pages/man3/pthread_setschedparam.3.html
   // http://man7.org/linux/man-pages/man2/sched_setscheduler.2.html
@@ -316,7 +322,7 @@ static int computeHostNumHardwareThreads() {
   if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask),
                          &mask) == 0)
     return CPU_COUNT(&mask);
-#elif defined(__linux__)
+#elif (defined(__linux__) || defined(__CYGWIN__))
   cpu_set_t Set;
   CPU_ZERO(&Set);
   if (sched_getaffinity(0, sizeof(Set), &Set) == 0)
@@ -338,7 +344,8 @@ llvm::BitVector llvm::get_thread_affinity_mask() {
 
 unsigned llvm::get_cpus() { return 1; }
 
-#if defined(__linux__) && (defined(__i386__) || defined(__x86_64__))
+#if (defined(__linux__) || defined(__CYGWIN__)) &&                             \
+    (defined(__i386__) || defined(__x86_64__))
 // On Linux, the number of physical cores can be computed from /proc/cpuinfo,
 // using the number of unique physical/core id pairs. The following
 // implementation reads the /proc/cpuinfo format on an x86_64 system.
diff --git a/llvm/lib/Support/Windows/Process.inc b/llvm/lib/Support/Windows/Process.inc
index d525f5b16e86..c4d18ca7375f 100644
--- a/llvm/lib/Support/Windows/Process.inc
+++ b/llvm/lib/Support/Windows/Process.inc
@@ -37,11 +37,6 @@
 //===          and must not be UNIX code
 //===----------------------------------------------------------------------===//
 
-#ifdef __MINGW32__
-// This ban should be lifted when MinGW 1.0+ has defined this value.
-#define _HEAPOK (-2)
-#endif
-
 using namespace llvm;
 
 Process::Pid Process::getProcessId() {
diff --git a/llvm/lib/Support/Windows/Signals.inc b/llvm/lib/Support/Windows/Signals.inc
index f11ad09f3713..dad0fa306686 100644
--- a/llvm/lib/Support/Windows/Signals.inc
+++ b/llvm/lib/Support/Windows/Signals.inc
@@ -9,6 +9,7 @@
 // This file provides the Win32 specific implementation of the Signals class.
 //
 //===----------------------------------------------------------------------===//
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ExitCodes.h"
 #include "llvm/Support/FileSystem.h"
@@ -47,64 +48,6 @@
 // back on at the end of the file.
 #pragma GCC diagnostic ignored "-Wformat"
 #pragma GCC diagnostic ignored "-Wformat-extra-args"
-
-#if !defined(__MINGW64_VERSION_MAJOR)
-// MinGW.org does not have updated support for the 64-bit versions of the
-// DebugHlp APIs. So we will have to load them manually. The structures and
-// method signatures were pulled from DbgHelp.h in the Windows Platform SDK,
-// and adjusted for brevity.
-typedef struct _IMAGEHLP_LINE64 {
-  DWORD SizeOfStruct;
-  PVOID Key;
-  DWORD LineNumber;
-  PCHAR FileName;
-  DWORD64 Address;
-} IMAGEHLP_LINE64, *PIMAGEHLP_LINE64;
-
-typedef struct _IMAGEHLP_SYMBOL64 {
-  DWORD SizeOfStruct;
-  DWORD64 Address;
-  DWORD Size;
-  DWORD Flags;
-  DWORD MaxNameLength;
-  CHAR Name[1];
-} IMAGEHLP_SYMBOL64, *PIMAGEHLP_SYMBOL64;
-
-typedef struct _tagADDRESS64 {
-  DWORD64 Offset;
-  WORD Segment;
-  ADDRESS_MODE Mode;
-} ADDRESS64, *LPADDRESS64;
-
-typedef struct _KDHELP64 {
-  DWORD64 Thread;
-  DWORD ThCallbackStack;
-  DWORD ThCallbackBStore;
-  DWORD NextCallback;
-  DWORD FramePointer;
-  DWORD64 KiCallUserMode;
-  DWORD64 KeUserCallbackDispatcher;
-  DWORD64 SystemRangeStart;
-  DWORD64 KiUserExceptionDispatcher;
-  DWORD64 StackBase;
-  DWORD64 StackLimit;
-  DWORD64 Reserved[5];
-} KDHELP64, *PKDHELP64;
-
-typedef struct _tagSTACKFRAME64 {
-  ADDRESS64 AddrPC;
-  ADDRESS64 AddrReturn;
-  ADDRESS64 AddrFrame;
-  ADDRESS64 AddrStack;
-  ADDRESS64 AddrBStore;
-  PVOID FuncTableEntry;
-  DWORD64 Params[4];
-  BOOL Far;
-  BOOL Virtual;
-  DWORD64 Reserved[3];
-  KDHELP64 KdHelp;
-} STACKFRAME64, *LPSTACKFRAME64;
-#endif // !defined(__MINGW64_VERSION_MAJOR)
 #endif // __MINGW32__
 
 typedef BOOL(__stdcall *PREAD_PROCESS_MEMORY_ROUTINE64)(
@@ -536,10 +479,8 @@ void sys::PrintStackTraceOnErrorSignal(StringRef Argv0,
 }
 } // namespace llvm
 
-#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR)
-// Provide a prototype for RtlCaptureContext, mingw32 from mingw.org is
-// missing it but mingw-w64 has it.
-extern "C" VOID WINAPI RtlCaptureContext(PCONTEXT ContextRecord);
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+#error DebugLoc origin-tracking currently unimplemented for Windows.
 #endif
 
 static void LocalPrintStackTrace(raw_ostream &OS, PCONTEXT C) {
diff --git a/llvm/lib/Support/Z3Solver.cpp b/llvm/lib/Support/Z3Solver.cpp
index 27027093a0c6..056a17ad8012 100644
--- a/llvm/lib/Support/Z3Solver.cpp
+++ b/llvm/lib/Support/Z3Solver.cpp
@@ -932,7 +932,8 @@ class Z3Statistics final : public SMTSolverStatistics {
   };
   unsigned getUnsigned(StringRef Key) const override {
     auto It = UnsignedValues.find(Key.str());
-    assert(It != UnsignedValues.end());
+    if (It == UnsignedValues.end())
+      return 0;
     return It->second;
   };
 
diff --git a/llvm/lib/TableGen/DetailedRecordsBackend.cpp b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
index cf697599e53a..1ed64356b7c6 100644
--- a/llvm/lib/TableGen/DetailedRecordsBackend.cpp
+++ b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
@@ -1,4 +1,4 @@
-//===- DetailedRecordBackend.cpp - Detailed Records Report      -*- C++ -*-===//
+//===- DetailedRecordBackend.cpp - Detailed Records Report ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This Tablegen backend prints a report that includes all the global 
+// This Tablegen backend prints a report that includes all the global
 // variables, classes, and records in complete detail. It includes more
 // detail than the default TableGen printer backend.
 //
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index 91423664a84c..de0c4c971efe 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -24,7 +24,7 @@ namespace llvm {
 SourceMgr SrcMgr;
 unsigned ErrorsPrinted = 0;
 
-static void PrintMessage(ArrayRef<SMLoc> Loc, SourceMgr::DiagKind Kind,
+static void PrintMessage(ArrayRef<SMLoc> Locs, SourceMgr::DiagKind Kind,
                          const Twine &Msg) {
   // Count the total number of errors printed.
   // This is used to exit with an error code if there were any errors.
@@ -32,11 +32,11 @@ static void PrintMessage(ArrayRef<SMLoc> Loc, SourceMgr::DiagKind Kind,
     ++ErrorsPrinted;
 
   SMLoc NullLoc;
-  if (Loc.empty())
-    Loc = NullLoc;
-  SrcMgr.PrintMessage(Loc.front(), Kind, Msg);
-  for (unsigned i = 1; i < Loc.size(); ++i)
-    SrcMgr.PrintMessage(Loc[i], SourceMgr::DK_Note,
+  if (Locs.empty())
+    Locs = NullLoc;
+  SrcMgr.PrintMessage(Locs.consume_front(), Kind, Msg);
+  for (SMLoc Loc : Locs)
+    SrcMgr.PrintMessage(Loc, SourceMgr::DK_Note,
                         "instantiated from multiclass");
 }
 
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index ea716215e067..f545706d6fe3 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -8,7 +8,7 @@
 //
 // TableGen is a tool which can be used to build up a description of something,
 // then invoke one or more "tablegen backends" to emit information about the
-// description in some predefined format.  In practice, this is used by the LLVM
+// description in some predefined format. In practice, this is used by the LLVM
 // code generators to automate generation of a code generator through a
 // high-level description of the target.
 //
@@ -156,7 +156,7 @@ int llvm::TableGenMain(const char *argv0,
     return 1;
 
   // Always write the depfile, even if the main output hasn't changed.
-  // If it's missing, Ninja considers the output dirty.  If this was below
+  // If it's missing, Ninja considers the output dirty. If this was below
   // the early exit below and someone deleted the .inc.d file but not the .inc
   // file, tablegen would never write the depfile.
   if (!DependFilename.empty()) {
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 7f2ed77a7409..1f3e5dc68f1d 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -500,7 +500,7 @@ const Init *BitsInit::convertInitializerTo(const RecTy *Ty) const {
   }
 
   if (auto *BRT = dyn_cast<BitsRecTy>(Ty)) {
-    // If the number of bits is right, return it.  Otherwise we need to expand
+    // If the number of bits is right, return it. Otherwise we need to expand
     // or truncate.
     if (getNumBits() != BRT->getNumBits()) return nullptr;
     return this;
@@ -944,7 +944,7 @@ const Init *UnOpInit::Fold(const Record *CurRec, bool IsFinal) const {
   case TAIL:
     if (const auto *LHSl = dyn_cast<ListInit>(LHS)) {
       assert(!LHSl->empty() && "Empty list in tail");
-      // Note the slice(1).  We can't just pass the result of getElements()
+      // Note the slice(1). We can't just pass the result of getElements()
       // directly.
       return ListInit::get(LHSl->getElements().slice(1),
                            LHSl->getElementType());
@@ -2921,16 +2921,16 @@ void Record::setName(const Init *NewName) {
   Name = NewName;
   checkName();
   // DO NOT resolve record values to the name at this point because
-  // there might be default values for arguments of this def.  Those
+  // there might be default values for arguments of this def. Those
   // arguments might not have been resolved yet so we don't want to
   // prematurely assume values for those arguments were not passed to
   // this def.
   //
   // Nonetheless, it may be that some of this Record's values
-  // reference the record name.  Indeed, the reason for having the
-  // record name be an Init is to provide this flexibility.  The extra
+  // reference the record name. Indeed, the reason for having the
+  // record name be an Init is to provide this flexibility. The extra
   // resolve steps after completely instantiating defs takes care of
-  // this.  See TGParser::ParseDef and TGParser::ParseDefm.
+  // this. See TGParser::ParseDef and TGParser::ParseDefm.
 }
 
 void Record::resolveReferences(Resolver &R, const RecordVal *SkipVal) {
diff --git a/llvm/lib/TableGen/StringMatcher.cpp b/llvm/lib/TableGen/StringMatcher.cpp
index b85849e40bb5..eb9036625796 100644
--- a/llvm/lib/TableGen/StringMatcher.cpp
+++ b/llvm/lib/TableGen/StringMatcher.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TableGen/StringMatcher.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
@@ -23,14 +23,15 @@
 using namespace llvm;
 
 /// FindFirstNonCommonLetter - Find the first character in the keys of the
-/// string pairs that is not shared across the whole set of strings.  All
+/// string pairs that is not shared across the whole set of strings. All
 /// strings are assumed to have the same length.
 static unsigned
 FindFirstNonCommonLetter(ArrayRef<const StringMatcher::StringPair *> Matches) {
   assert(!Matches.empty());
   for (auto [Idx, Letter] : enumerate(Matches[0]->first)) {
-    // Check to see if `Letter` is the same across the set.
-    for (const StringMatcher::StringPair *Match : Matches)
+    // Check to see if `Letter` is the same across the set. Since the letter is
+    // from `Matches[0]`, we can skip `Matches[0]` in the loop below.
+    for (const StringMatcher::StringPair *Match : Matches.drop_front())
       if (Match->first[Idx] != Letter)
         return Idx;
   }
diff --git a/llvm/lib/TableGen/StringToOffsetTable.cpp b/llvm/lib/TableGen/StringToOffsetTable.cpp
index d73b5749ad7d..0db3f35f1c4b 100644
--- a/llvm/lib/TableGen/StringToOffsetTable.cpp
+++ b/llvm/lib/TableGen/StringToOffsetTable.cpp
@@ -44,8 +44,8 @@ void StringToOffsetTable::EmitStringTableDef(raw_ostream &OS, const Twine &Name,
   bool UseChars = !EmitLongStrLiterals && AggregateString.size() > (64 * 1024);
   OS << (UseChars ? "{\n" : "\n");
 
-  llvm::ListSeparator LineSep(UseChars ? ",\n" : "\n");
-  llvm::SmallVector<StringRef> Strings(split(AggregateString, '\0'));
+  ListSeparator LineSep(UseChars ? ",\n" : "\n");
+  SmallVector<StringRef> Strings(split(AggregateString, '\0'));
   // We should always have an empty string at the start, and because these are
   // null terminators rather than separators, we'll have one at the end as
   // well. Skip the end one.
@@ -63,7 +63,7 @@ void StringToOffsetTable::EmitStringTableDef(raw_ostream &OS, const Twine &Name,
       continue;
     }
 
-    llvm::ListSeparator CharSep(", ");
+    ListSeparator CharSep(", ");
     for (char C : Str) {
       OS << CharSep << "'";
       OS.write_escaped(StringRef(&C, 1));
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 46487cba9453..c8e020d791e0 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -102,7 +102,7 @@ SMRange TGLexer::getLocRange() const {
 }
 
 /// ReturnError - Set the error to the specified string at the specified
-/// location.  This is defined to always return tgtok::Error.
+/// location. This is defined to always return tgtok::Error.
 tgtok::TokKind TGLexer::ReturnError(SMLoc Loc, const Twine &Msg) {
   PrintError(Loc, Msg);
   return tgtok::Error;
@@ -116,7 +116,7 @@ bool TGLexer::processEOF() {
   SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
   if (ParentIncludeLoc != SMLoc()) {
     // If prepExitInclude() detects a problem with the preprocessing
-    // control stack, it will return false.  Pretend that we reached
+    // control stack, it will return false. Pretend that we reached
     // the final EOF and stop lexing more tokens by returning false
     // to LexToken().
     if (!prepExitInclude(false))
@@ -147,7 +147,7 @@ int TGLexer::getNextChar() {
 
   case 0: {
     // A NUL character in the stream is either the end of the current buffer or
-    // a spurious NUL in the file.  Disambiguate that here.
+    // a spurious NUL in the file. Disambiguate that here.
     if (CurPtr - 1 == CurBuf.end()) {
       --CurPtr; // Arrange for another call to return EOF again.
       return EOF;
@@ -160,7 +160,7 @@ int TGLexer::getNextChar() {
   case '\n':
   case '\r':
     // Handle the newline character by ignoring it and incrementing the line
-    // count.  However, be careful about 'dos style' files with \n\r in them.
+    // count. However, be careful about 'dos style' files with \n\r in them.
     // Only treat a \n\r or \r\n as a single line.
     if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
         *CurPtr != CurChar)
@@ -259,7 +259,7 @@ tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
     int NextChar = 0;
     if (isDigit(CurChar)) {
       // Allow identifiers to start with a number if it is followed by
-      // an identifier.  This can happen with paste operations like
+      // an identifier. This can happen with paste operations like
       // foo#8i.
       int i = 0;
       do {
@@ -422,7 +422,7 @@ tgtok::TokKind TGLexer::LexIdentifier() {
   return Kind;
 }
 
-/// LexInclude - We just read the "include" token.  Get the string token that
+/// LexInclude - We just read the "include" token. Get the string token that
 /// comes next and enter the include.
 bool TGLexer::LexInclude() {
   // The token after the include must be a string.
@@ -461,7 +461,7 @@ void TGLexer::SkipBCPLComment() {
   CurPtr = (EOLPos == StringRef::npos) ? CurBuf.end() : CurBuf.data() + EOLPos;
 }
 
-/// SkipCComment - This skips C-style /**/ comments.  The only difference from C
+/// SkipCComment - This skips C-style /**/ comments. The only difference from C
 /// is that we allow nesting.
 bool TGLexer::SkipCComment() {
   ++CurPtr;  // skip the star.
@@ -554,8 +554,8 @@ tgtok::TokKind TGLexer::LexNumber() {
   return Base == 2 ? tgtok::BinaryIntVal : tgtok::IntVal;
 }
 
-/// LexBracket - We just read '['.  If this is a code block, return it,
-/// otherwise return the bracket.  Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
+/// LexBracket - We just read '['. If this is a code block, return it,
+/// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
 tgtok::TokKind TGLexer::LexBracket() {
   if (CurPtr[0] != '{')
     return tgtok::l_square;
@@ -687,7 +687,7 @@ tgtok::TokKind TGLexer::prepIsDirective() const {
         NextChar == '\n' ||
         // It looks like TableGen does not support '\r' as the actual
         // carriage return, e.g. getNextChar() treats a single '\r'
-        // as '\n'.  So we do the same here.
+        // as '\n'. So we do the same here.
         NextChar == '\r')
       return Kind;
 
@@ -700,7 +700,7 @@ tgtok::TokKind TGLexer::prepIsDirective() const {
     //     #define/**/ AND #define//
     //
     // These cases will be reported as incorrect after calling
-    // prepLexMacroName().  We could have supported C-style comments
+    // prepLexMacroName(). We could have supported C-style comments
     // after #ifdef/#define, but this would complicate the code
     // for little benefit.
     if (NextChar == '/') {
@@ -733,7 +733,7 @@ void TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
 
 tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
                                         bool ReturnNextLiveToken) {
-  // We must be looking at a preprocessing directive.  Eat it!
+  // We must be looking at a preprocessing directive. Eat it!
   prepEatPreprocessorDirective(Kind);
 
   if (Kind == tgtok::Ifdef || Kind == tgtok::Ifndef) {
@@ -879,7 +879,7 @@ bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
     tgtok::TokKind Kind = prepIsDirective();
 
     // If we did not find a preprocessing directive or it is #define,
-    // then just skip to the next line.  We do not have to do anything
+    // then just skip to the next line. We do not have to do anything
     // for #define in the line-skipping mode.
     if (Kind == tgtok::Error || Kind == tgtok::Define)
       continue;
@@ -909,7 +909,7 @@ bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
   } while (CurPtr != CurBuf.end());
 
   // We have reached the end of the file, but never left the lines-skipping
-  // mode.  This means there is no matching #endif.
+  // mode. This means there is no matching #endif.
   prepReportPreprocessorStackError();
   return false;
 }
@@ -939,8 +939,8 @@ bool TGLexer::prepSkipLineBegin() {
         // Skip C-style comment.
         // Note that we do not care about skipping the C++-style comments.
         // If the line contains "//", it may not contain any processable
-        // preprocessing directive.  Just return CurPtr pointing to
-        // the first '/' in this case.  We also do not care about
+        // preprocessing directive. Just return CurPtr pointing to
+        // the first '/' in this case. We also do not care about
         // incorrect symbols after the first '/' - we are in lines-skipping
         // mode, so incorrect code is allowed to some extent.
 
@@ -968,7 +968,7 @@ bool TGLexer::prepSkipLineBegin() {
     ++CurPtr;
   }
 
-  // We have reached the end of the file.  Return to the lines skipping
+  // We have reached the end of the file. Return to the lines skipping
   // code, and allow it to handle the EOF as needed.
   return true;
 }
@@ -994,7 +994,7 @@ bool TGLexer::prepSkipDirectiveEnd() {
         SkipBCPLComment();
       } else if (NextChar == '*') {
         // When we are skipping C-style comment at the end of a preprocessing
-        // directive, we can skip several lines.  If any meaningful TD token
+        // directive, we can skip several lines. If any meaningful TD token
         // follows the end of the C-style comment on the same line, it will
         // be considered as an invalid usage of TD token.
         // For example, we want to forbid usages like this one:
@@ -1003,7 +1003,7 @@ bool TGLexer::prepSkipDirectiveEnd() {
         //     #define MACRO /* This macro is used
         //                      to ... */ class Class {}
         // One can argue that this should be allowed, but it does not seem
-        // to be worth of the complication.  Moreover, this matches
+        // to be worth of the complication. Moreover, this matches
         // the C preprocessor behavior.
 
         // Set TokStart to the beginning of the comment to enable proper
diff --git a/llvm/lib/TableGen/TGLexer.h b/llvm/lib/TableGen/TGLexer.h
index a5b10b37e988..ed7d8f3baae5 100644
--- a/llvm/lib/TableGen/TGLexer.h
+++ b/llvm/lib/TableGen/TGLexer.h
@@ -61,7 +61,7 @@ enum TokKind {
   // Integer value.
   IntVal,
 
-  // Binary constant.  Note that these are sized according to the number of
+  // Binary constant. Note that these are sized according to the number of
   // bits given.
   BinaryIntVal,
 
@@ -280,7 +280,7 @@ class TGLexer {
   //
   // An ordered list of preprocessing controls defined by #ifdef/#else
   // directives that are in effect currently is called preprocessing
-  // control stack.  It is represented as a vector of PreprocessorControlDesc's.
+  // control stack. It is represented as a vector of PreprocessorControlDesc's.
   //
   // The control stack is updated according to the following rules:
   //
@@ -321,9 +321,9 @@ class TGLexer {
   //     EOF
   //
   // To do this, we clear the preprocessing control stack on entry
-  // to each of the included file.  PrepIncludeStack is used to store
+  // to each of the included file. PrepIncludeStack is used to store
   // preprocessing control stacks for the current file and all its
-  // parent files.  The back() element is the preprocessing control
+  // parent files. The back() element is the preprocessing control
   // stack for the current file.
   SmallVector<SmallVector<PreprocessorControlDesc>> PrepIncludeStack;
 
@@ -332,7 +332,7 @@ class TGLexer {
   //
   // If IncludeStackMustBeEmpty is true, the include stack must be empty
   // after the popping, otherwise, the include stack must not be empty
-  // after the popping.  Basically, the include stack must be empty
+  // after the popping. Basically, the include stack must be empty
   // only if we exit the "top-level" file (i.e. finish lexing).
   //
   // The method returns false, if the current preprocessing control stack
@@ -340,8 +340,8 @@ class TGLexer {
   // true - otherwise.
   bool prepExitInclude(bool IncludeStackMustBeEmpty);
 
-  // Look ahead for a preprocessing directive starting from CurPtr.  The caller
-  // must only call this method, if *(CurPtr - 1) is '#'.  If the method matches
+  // Look ahead for a preprocessing directive starting from CurPtr. The caller
+  // must only call this method, if *(CurPtr - 1) is '#'. If the method matches
   // a preprocessing directive word followed by a whitespace, then it returns
   // one of the internal token kinds, i.e. Ifdef, Else, Endif, Define.
   //
@@ -353,26 +353,26 @@ class TGLexer {
   //
   // We use look-ahead prepIsDirective() and prepEatPreprocessorDirective()
   // to avoid adjusting CurPtr before we are sure that '#' is followed
-  // by a preprocessing directive.  If it is not, then we fall back to
+  // by a preprocessing directive. If it is not, then we fall back to
   // tgtok::paste interpretation of '#'.
   void prepEatPreprocessorDirective(tgtok::TokKind Kind);
 
   // The main "exit" point from the token parsing to preprocessor.
   //
   // The method is called for CurPtr, when prepIsDirective() returns
-  // true.  The first parameter matches the result of prepIsDirective(),
+  // true. The first parameter matches the result of prepIsDirective(),
   // denoting the actual preprocessor directive to be processed.
   //
   // If the preprocessing directive disables the tokens processing, e.g.:
   //     #ifdef NAME // NAME is undefined
   // then lexPreprocessor() enters the lines-skipping mode.
   // In this mode, it does not parse any tokens, because the code under
-  // the #ifdef may not even be a correct tablegen code.  The preprocessor
+  // the #ifdef may not even be a correct tablegen code. The preprocessor
   // looks for lines containing other preprocessing directives, which
-  // may be prepended with whitespaces and C-style comments.  If the line
+  // may be prepended with whitespaces and C-style comments. If the line
   // does not contain a preprocessing directive, it is skipped completely.
   // Otherwise, the preprocessing directive is processed by recursively
-  // calling lexPreprocessor().  The processing of the encountered
+  // calling lexPreprocessor(). The processing of the encountered
   // preprocessing directives includes updating preprocessing control stack
   // and adding new macros into DefinedMacros set.
   //
@@ -390,38 +390,38 @@ class TGLexer {
 
   // Worker method for lexPreprocessor() to skip lines after some
   // preprocessing directive up to the buffer end or to the directive
-  // that re-enables token processing.  The method returns true
+  // that re-enables token processing. The method returns true
   // upon processing the next directive that re-enables tokens
-  // processing.  False is returned if an error was encountered.
+  // processing. False is returned if an error was encountered.
   //
   // Note that prepSkipRegion() calls lexPreprocessor() to process
-  // encountered preprocessing directives.  In this case, the second
-  // parameter to lexPreprocessor() is set to false.  Being passed
+  // encountered preprocessing directives. In this case, the second
+  // parameter to lexPreprocessor() is set to false. Being passed
   // false ReturnNextLiveToken, lexPreprocessor() must never call
-  // prepSkipRegion().  We assert this by passing ReturnNextLiveToken
+  // prepSkipRegion(). We assert this by passing ReturnNextLiveToken
   // to prepSkipRegion() and checking that it is never set to false.
   bool prepSkipRegion(bool MustNeverBeFalse);
 
-  // Lex name of the macro after either #ifdef or #define.  We could have used
+  // Lex name of the macro after either #ifdef or #define. We could have used
   // LexIdentifier(), but it has special handling of "include" word, which
-  // could result in awkward diagnostic errors.  Consider:
+  // could result in awkward diagnostic errors. Consider:
   // ----
   // #ifdef include
   // class ...
   // ----
   // LexIdentifier() will engage LexInclude(), which will complain about
-  // missing file with name "class".  Instead, prepLexMacroName() will treat
+  // missing file with name "class". Instead, prepLexMacroName() will treat
   // "include" as a normal macro name.
   //
   // On entry, CurPtr points to the end of a preprocessing directive word.
   // The method allows for whitespaces between the preprocessing directive
-  // and the macro name.  The allowed whitespaces are ' ' and '\t'.
+  // and the macro name. The allowed whitespaces are ' ' and '\t'.
   //
   // If the first non-whitespace symbol after the preprocessing directive
   // is a valid start symbol for an identifier (i.e. [a-zA-Z_]), then
   // the method updates TokStart to the position of the first non-whitespace
   // symbol, sets CurPtr to the position of the macro name's last symbol,
-  // and returns a string reference to the macro name.  Otherwise,
+  // and returns a string reference to the macro name. Otherwise,
   // TokStart is set to the first non-whitespace symbol after the preprocessing
   // directive, and the method returns an empty string reference.
   //
@@ -429,10 +429,10 @@ class TGLexer {
   // the preprocessing directive.
   StringRef prepLexMacroName();
 
-  // Skip any whitespaces starting from CurPtr.  The method is used
+  // Skip any whitespaces starting from CurPtr. The method is used
   // only in the lines-skipping mode to find the first non-whitespace
-  // symbol after or at CurPtr.  Allowed whitespaces are ' ', '\t', '\n'
-  // and '\r'.  The method skips C-style comments as well, because
+  // symbol after or at CurPtr. Allowed whitespaces are ' ', '\t', '\n'
+  // and '\r'. The method skips C-style comments as well, because
   // it is used to find the beginning of the preprocessing directive.
   // If we do not handle C-style comments the following code would
   // result in incorrect detection of a preprocessing directive:
@@ -445,13 +445,13 @@ class TGLexer {
   //        second line comment */ #ifdef NAME
   //
   // The method returns true upon reaching the first non-whitespace symbol
-  // or EOF, CurPtr is set to point to this symbol.  The method returns false,
+  // or EOF, CurPtr is set to point to this symbol. The method returns false,
   // if an error occurred during skipping of a C-style comment.
   bool prepSkipLineBegin();
 
   // Skip any whitespaces or comments after a preprocessing directive.
   // The method returns true upon reaching either end of the line
-  // or end of the file.  If there is a multiline C-style comment
+  // or end of the file. If there is a multiline C-style comment
   // after the preprocessing directive, the method skips
   // the comment, so the final CurPtr may point to one of the next lines.
   // The method returns false, if an error occurred during skipping
@@ -459,7 +459,7 @@ class TGLexer {
   // after the preprocessing directive.
   //
   // The method maybe called both during lines-skipping and tokens
-  // processing.  It actually verifies that only whitespaces or/and
+  // processing. It actually verifies that only whitespaces or/and
   // comments follow a preprocessing directive.
   //
   // After the execution of this mehod, CurPtr points either to new line
@@ -475,7 +475,7 @@ class TGLexer {
   bool prepIsProcessingEnabled();
 
   // Report an error, if we reach EOF with non-empty preprocessing control
-  // stack.  This means there is no matching #endif for the previous
+  // stack. This means there is no matching #endif for the previous
   // #ifdef/#else.
   void prepReportPreprocessorStackError();
 };
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 87a1fb64943c..62c535565414 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -240,17 +240,15 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, const Init *ValName,
     return Error(Loc, "Value '" + ValName->getAsUnquotedString() +
                  "' unknown!");
 
-  // Do not allow assignments like 'X = X'.  This will just cause infinite loops
+  // Do not allow assignments like 'X = X'. This will just cause infinite loops
   // in the resolution machinery.
   if (BitList.empty())
     if (const auto *VI = dyn_cast<VarInit>(V))
       if (VI->getNameInit() == ValName && !AllowSelfAssignment)
         return Error(Loc, "Recursion / self-assignment forbidden");
 
-  // If we are assigning to a subset of the bits in the value... then we must be
-  // assigning to a field of BitsRecTy, which must have a BitsInit
-  // initializer.
-  //
+  // If we are assigning to a subset of the bits in the value we must be
+  // assigning to a field of BitsRecTy, which must have a BitsInit initializer.
   if (!BitList.empty()) {
     const auto *CurVal = dyn_cast<BitsInit>(RV->getValue());
     if (!CurVal)
@@ -427,9 +425,9 @@ bool TGParser::resolve(const ForeachLoop &Loop, SubstStack &Substs,
   const Init *List = Loop.ListValue->resolveReferences(R);
 
   // For if-then-else blocks, we lower to a foreach loop whose list is a
-  // ternary selection between lists of different length.  Since we don't
+  // ternary selection between lists of different length. Since we don't
   // have a means to track variable length record lists, we *must* resolve
-  // the condition here.  We want to defer final resolution of the arms
+  // the condition here. We want to defer final resolution of the arms
   // until the resulting records are finalized.
   // e.g. !if(!exists<SchedWrite>("__does_not_exist__"), [1], [])
   if (const auto *TI = dyn_cast<TernOpInit>(List);
@@ -700,7 +698,7 @@ const Init *TGParser::ParseObjectName(MultiClass *CurMultiClass) {
   return Name;
 }
 
-/// ParseClassID - Parse and resolve a reference to a class name.  This returns
+/// ParseClassID - Parse and resolve a reference to a class name. This returns
 /// null on error.
 ///
 ///    ClassID ::= ID
@@ -788,7 +786,7 @@ ParseSubClassReference(Record *CurRec, bool isDefm) {
 }
 
 /// ParseSubMultiClassReference - Parse a reference to a subclass or to a
-/// templated submulticlass.  This returns a SubMultiClassRefTy with a null
+/// templated submulticlass. This returns a SubMultiClassRefTy with a null
 /// Record* on error.
 ///
 ///  SubMultiClassRef ::= MultiClassID
@@ -1078,7 +1076,7 @@ bool TGParser::ParseOptionalBitList(SmallVectorImpl<unsigned> &Ranges) {
   return false;
 }
 
-/// ParseType - Parse and return a tblgen type.  This returns null on error.
+/// ParseType - Parse and return a tblgen type. This returns null on error.
 ///
 ///   Type ::= STRING                       // string type
 ///   Type ::= CODE                         // code type
@@ -1180,7 +1178,7 @@ const Init *TGParser::ParseIDValue(Record *CurRec, const StringInit *Name,
   return nullptr;
 }
 
-/// ParseOperation - Parse an operator.  This returns null on error.
+/// ParseOperation - Parse an operator. This returns null on error.
 ///
 /// Operation ::= XOperator ['<' Type '>'] '(' Args ')'
 ///
@@ -2253,7 +2251,7 @@ const Init *TGParser::ParseOperation(Record *CurRec, const RecTy *ItemType) {
   }
 }
 
-/// ParseOperatorType - Parse a type for an operator.  This returns
+/// ParseOperatorType - Parse a type for an operator. This returns
 /// null on error.
 ///
 /// OperatorType ::= '<' Type '>'
@@ -2670,7 +2668,7 @@ const Init *TGParser::ParseOperationCond(Record *CurRec,
   return CondOpInit::get(Case, Val, Type)->Fold(CurRec);
 }
 
-/// ParseSimpleValue - Parse a tblgen value.  This returns null on error.
+/// ParseSimpleValue - Parse a tblgen value. This returns null on error.
 ///
 ///   SimpleValue ::= IDValue
 ///   SimpleValue ::= INTVAL
@@ -2803,7 +2801,7 @@ const Init *TGParser::ParseSimpleValue(Record *CurRec, const RecTy *ItemType,
     SmallVector<const Init *, 16> NewBits;
 
     // As we parse { a, b, ... }, 'a' is the highest bit, but we parse it
-    // first.  We'll first read everything in to a vector, then we can reverse
+    // first. We'll first read everything in to a vector, then we can reverse
     // it to get the bits in the correct order for the BitsInit value.
     for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
       // FIXME: The following two loops would not be duplicated
@@ -3319,10 +3317,10 @@ bool TGParser::ParseTemplateArgValueList(
 }
 
 /// ParseDeclaration - Read a declaration, returning the name of field ID, or an
-/// empty string on error.  This can happen in a number of different contexts,
+/// empty string on error. This can happen in a number of different contexts,
 /// including within a def or in the template args for a class (in which case
 /// CurRec will be non-null) and within the template args for a multiclass (in
-/// which case CurRec will be null, but CurMultiClass will be set).  This can
+/// which case CurRec will be null, but CurMultiClass will be set). This can
 /// also happen within a def that is within a multiclass, which will set both
 /// CurRec and CurMultiClass.
 ///
@@ -3384,7 +3382,7 @@ const Init *TGParser::ParseDeclaration(Record *CurRec,
     if (!Val ||
         SetValue(CurRec, ValLoc, DeclName, {}, Val,
                  /*AllowSelfAssignment=*/false, /*OverrideDefLoc=*/false)) {
-      // Return the name, even if an error is thrown.  This is so that we can
+      // Return the name, even if an error is thrown. This is so that we can
       // continue to make some progress, even without the value having been
       // initialized.
       return DeclName;
@@ -3395,7 +3393,7 @@ const Init *TGParser::ParseDeclaration(Record *CurRec,
 }
 
 /// ParseForeachDeclaration - Read a foreach declaration, returning
-/// the name of the declared object or a NULL Init on error.  Return
+/// the name of the declared object or a NULL Init on error. Return
 /// the name of the parsed initializer list through ForeachListName.
 ///
 ///  ForeachDeclaration ::= ID '=' '{' RangeList '}'
@@ -3477,7 +3475,7 @@ TGParser::ParseForeachDeclaration(const Init *&ForeachListValue) {
 }
 
 /// ParseTemplateArgList - Read a template argument list, which is a non-empty
-/// sequence of template-declarations in <>'s.  If CurRec is non-null, these are
+/// sequence of template-declarations in <>'s. If CurRec is non-null, these are
 /// template args for a class. If null, these are the template args for a
 /// multiclass.
 ///
@@ -3578,7 +3576,7 @@ bool TGParser::ParseBodyItem(Record *CurRec) {
   return SetValue(CurRec, IdLoc, FieldName, BitList, Val);
 }
 
-/// ParseBody - Read the body of a class or def.  Return true on error, false on
+/// ParseBody - Read the body of a class or def. Return true on error, false on
 /// success.
 ///
 ///   Body     ::= ';'
@@ -3641,8 +3639,8 @@ bool TGParser::ApplyLetStack(RecordsEntry &Entry) {
   return false;
 }
 
-/// ParseObjectBody - Parse the body of a def or class.  This consists of an
-/// optional ClassList followed by a Body.  CurRec is the current def or class
+/// ParseObjectBody - Parse the body of a def or class. This consists of an
+/// optional ClassList followed by a Body. CurRec is the current def or class
 /// that is being parsed.
 ///
 ///   ObjectBody      ::= BaseClassList Body
@@ -3841,8 +3839,8 @@ bool TGParser::ParseDefvar(Record *CurRec) {
   return false;
 }
 
-/// ParseForeach - Parse a for statement.  Return the record corresponding
-/// to it.  This returns true on error.
+/// ParseForeach - Parse a for statement. Return the record corresponding
+/// to it. This returns true on error.
 ///
 ///   Foreach ::= FOREACH Declaration IN '{ ObjectList '}'
 ///   Foreach ::= FOREACH Declaration IN Object
@@ -4129,7 +4127,7 @@ void TGParser::ParseLetList(SmallVectorImpl<LetRecord> &Result) {
   } while (consume(tgtok::comma));
 }
 
-/// ParseTopLevelLet - Parse a 'let' at top level.  This can be a couple of
+/// ParseTopLevelLet - Parse a 'let' at top level. This can be a couple of
 /// different related productions. This works inside multiclasses too.
 ///
 ///   Object ::= LET LetList IN '{' ObjectList '}'
diff --git a/llvm/lib/TableGen/TGParser.h b/llvm/lib/TableGen/TGParser.h
index 017cc5fff683..2a5a1925343c 100644
--- a/llvm/lib/TableGen/TGParser.h
+++ b/llvm/lib/TableGen/TGParser.h
@@ -161,10 +161,10 @@ class TGParser {
   // Record tracker
   RecordKeeper &Records;
 
-  // A "named boolean" indicating how to parse identifiers.  Usually
+  // A "named boolean" indicating how to parse identifiers. Usually
   // identifiers map to some existing object but in special cases
   // (e.g. parsing def names) no such object exists yet because we are
-  // in the middle of creating in.  For those situations, allow the
+  // in the middle of creating in. For those situations, allow the
   // parser to ignore missing object errors.
   enum IDParseMode {
     ParseValueMode,   // We are parsing a value we expect to look up.
@@ -183,7 +183,7 @@ class TGParser {
         NoWarnOnUnusedTemplateArgs(NoWarnOnUnusedTemplateArgs),
         TrackReferenceLocs(TrackReferenceLocs) {}
 
-  /// ParseFile - Main entrypoint for parsing a tblgen file.  These parser
+  /// ParseFile - Main entrypoint for parsing a tblgen file. These parser
   /// routines return true on error, or false on success.
   bool ParseFile();
 
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index eb5a5199b895..25cdcae4e41d 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -58,12 +58,12 @@ include "AArch64SystemOperands.td"
 
 class AArch64Unsupported { list<Predicate> F; }
 
-let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_SME2p1] in
+let F = [HasSVE2p1, HasSVE2p1_or_SME2, HasSVE2p1_or_StreamingSME2, HasSVE2p1_or_SME2p1] in
 def SVE2p1Unsupported : AArch64Unsupported;
 
 def SVE2Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
-                       HasSMEF8F32, HasSVEAES, HasSVE2SHA3, HasSVE2SM4, HasSVEBitPerm,
+  let F = !listconcat([HasSVE2, HasSVE2_or_SME, HasNonStreamingSVE2_or_SME2, HasSSVE_FP8FMA, HasSMEF8F16,
+                       HasSMEF8F32, HasSVEAES, HasSVESHA3, HasSVE2SM4, HasSVEBitPerm,
                        HasSVEB16B16],
                        SVE2p1Unsupported.F);
 }
@@ -80,14 +80,14 @@ def SME2p2Unsupported : AArch64Unsupported;
 def SME2p1Unsupported : AArch64Unsupported {
   let F = !listconcat([HasSME2p1, HasSVE2p1_or_SME2p1,
                        HasSME_MOP4, HasSME_TMOP, HasNonStreamingSVE_or_SSVE_FEXPA, 
-                       HasNonStreamingSVE2_or_SSVE_BitPerm],
+                       HasNonStreamingSVE_or_SSVE_BitPerm],
                        SME2p2Unsupported.F);
 }
 
 def SME2Unsupported : AArch64Unsupported {
-  let F = !listconcat([HasSME2, HasSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
+  let F = !listconcat([HasSME2, HasNonStreamingSVE2_or_SME2, HasSVE2p1_or_SME2, HasSSVE_FP8FMA,
                       HasSMEF8F16, HasSMEF8F32, HasSMEF16F16_or_SMEF8F16, HasSMEB16B16,
-                      HasNonStreamingSVE2_or_SSVE_AES],
+                      HasNonStreamingSVE_or_SSVE_AES, HasSVE2p1_or_StreamingSME2],
                       SME2p1Unsupported.F);
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index a16c104d8bef..dd1005059219 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -19,7 +19,7 @@
 #include "AArch64TargetObjectFile.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64InstPrinter.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "MCTargetDesc/AArch64TargetStreamer.h"
 #include "TargetInfo/AArch64TargetInfo.h"
@@ -910,15 +910,15 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
       // have a chance to save them.
       EmitToStreamer(MCInstBuilder(AArch64::ADRP)
                          .addReg(AArch64::X16)
-                         .addExpr(MCSpecifierExpr::create(
-                             HwasanTagMismatchRef, AArch64MCExpr::VK_GOT_PAGE,
-                             OutContext)));
+                         .addExpr(MCSpecifierExpr::create(HwasanTagMismatchRef,
+                                                          AArch64::S_GOT_PAGE,
+                                                          OutContext)));
       EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
                          .addReg(AArch64::X16)
                          .addReg(AArch64::X16)
-                         .addExpr(MCSpecifierExpr::create(
-                             HwasanTagMismatchRef, AArch64MCExpr::VK_GOT_LO12,
-                             OutContext)));
+                         .addExpr(MCSpecifierExpr::create(HwasanTagMismatchRef,
+                                                          AArch64::S_GOT_LO12,
+                                                          OutContext)));
       EmitToStreamer(MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 469c76752c78..05562516e519 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -373,8 +373,11 @@ def FeatureAliasSVE2AES : ExtensionWithMArch<"sve2-aes", "SVE2AES",
 def FeatureSVE2SM4 : ExtensionWithMArch<"sve2-sm4", "SVE2SM4", "FEAT_SVE_SM4",
   "Enable SM4 SVE2 instructions", [FeatureSVE2, FeatureSM4]>;
 
-def FeatureSVE2SHA3 : ExtensionWithMArch<"sve2-sha3", "SVE2SHA3", "FEAT_SVE_SHA3",
-  "Enable SHA3 SVE2 instructions", [FeatureSVE2, FeatureSHA3]>;
+def FeatureSVESHA3 : ExtensionWithMArch<"sve-sha3", "SVESHA3", "FEAT_SVE_SHA3",
+  "Enable SVE SHA3 instructions", [FeatureSHA3]>;
+
+def FeatureAliasSVE2SHA3 : ExtensionWithMArch<"sve2-sha3", "SVE2SHA3",
+  "", "Shorthand for +sve2+sve-sha3", [FeatureSVE2, FeatureSVESHA3]>;
 
 def FeatureSVEBitPerm : ExtensionWithMArch<"sve-bitperm", "SVEBitPerm",
   "FEAT_SVE_BitPerm",  "Enable bit permutation SVE2 instructions">;
@@ -609,8 +612,17 @@ def FeatureExperimentalZeroingPseudos
 def FeatureNoSVEFPLD1R : SubtargetFeature<"no-sve-fp-ld1r",
   "NoSVEFPLD1R", "true", "Avoid using LD1RX instructions for FP">;
 
-def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true",
-                                        "Has zero-cycle register moves">;
+def FeatureZCRegMoveGPR64 : SubtargetFeature<"zcm-gpr64", "HasZeroCycleRegMoveGPR64", "true",
+                                        "Has zero-cycle register moves for GPR64 registers">;
+
+def FeatureZCRegMoveGPR32 : SubtargetFeature<"zcm-gpr32", "HasZeroCycleRegMoveGPR32", "true",
+                                        "Has zero-cycle register moves for GPR32 registers">;
+
+def FeatureZCRegMoveFPR64 : SubtargetFeature<"zcm-fpr64", "HasZeroCycleRegMoveFPR64", "true",
+                                        "Has zero-cycle register moves for FPR64 registers">;
+
+def FeatureZCRegMoveFPR32 : SubtargetFeature<"zcm-fpr32", "HasZeroCycleRegMoveFPR32", "true",
+                                        "Has zero-cycle register moves for FPR32 registers">;
 
 def FeatureZCZeroingGP : SubtargetFeature<"zcz-gp", "HasZeroCycleZeroingGP", "true",
                                         "Has zero-cycle zeroing instructions for generic registers">;
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index a71668e71c23..6f1ce5bdbe28 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -3934,6 +3934,11 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
       CSStackSize += SpillSize;
   }
 
+  // Save number of saved regs, so we can easily update CSStackSize later to
+  // account for any additional 64-bit GPR saves. Note: After this point
+  // only 64-bit GPRs can be added to SavedRegs.
+  unsigned NumSavedRegs = SavedRegs.count();
+
   // Increase the callee-saved stack size if the function has streaming mode
   // changes, as we will need to spill the value of the VG register.
   // For locally streaming functions, we spill both the streaming and
@@ -3952,8 +3957,9 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
   if (AFI->hasStackHazardSlotIndex())
     CSStackSize += getStackHazardSize(MF);
 
-  // Save number of saved regs, so we can easily update CSStackSize later.
-  unsigned NumSavedRegs = SavedRegs.count();
+  // If we must call __arm_get_current_vg in the prologue preserve the LR.
+  if (requiresSaveVG(MF) && !Subtarget.hasSVE())
+    SavedRegs.set(AArch64::LR);
 
   // The frame record needs to be created by saving the appropriate registers
   uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fdebfd8b698e..85a9ab3e892f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3409,7 +3409,7 @@ static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
           (isSignedIntSetCC(CC) && isSafeSignedCMN(Op, DAG)));
 }
 
-static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
+static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &DL,
                                       SelectionDAG &DAG, SDValue Chain,
                                       bool IsSignaling) {
   EVT VT = LHS.getValueType();
@@ -3418,29 +3418,29 @@ static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
   const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
   if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
-    LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other},
+    LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                       {Chain, LHS});
-    RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other},
+    RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                       {LHS.getValue(1), RHS});
     Chain = RHS.getValue(1);
   }
   unsigned Opcode =
       IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
-  return DAG.getNode(Opcode, dl, {MVT::i32, MVT::Other}, {Chain, LHS, RHS});
+  return DAG.getNode(Opcode, DL, {MVT::i32, MVT::Other}, {Chain, LHS, RHS});
 }
 
 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
-                              const SDLoc &dl, SelectionDAG &DAG) {
+                              const SDLoc &DL, SelectionDAG &DAG) {
   EVT VT = LHS.getValueType();
   const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
   if (VT.isFloatingPoint()) {
     assert(VT != MVT::f128);
     if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
-      LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
-      RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
+      LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
+      RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
     }
-    return DAG.getNode(AArch64ISD::FCMP, dl, MVT::i32, LHS, RHS);
+    return DAG.getNode(AArch64ISD::FCMP, DL, MVT::i32, LHS, RHS);
   }
 
   // The CMP instruction is just an alias for SUBS, and representing it as
@@ -3464,10 +3464,9 @@ static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
       // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
       // of the signed comparisons.
-      const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
-                                           DAG.getVTList(VT, MVT_CC),
-                                           LHS.getOperand(0),
-                                           LHS.getOperand(1));
+      const SDValue ANDSNode =
+          DAG.getNode(AArch64ISD::ANDS, DL, DAG.getVTList(VT, MVT_CC),
+                      LHS.getOperand(0), LHS.getOperand(1));
       // Replace all users of (and X, Y) with newly generated (ands X, Y)
       DAG.ReplaceAllUsesWith(LHS, ANDSNode);
       return ANDSNode.getValue(1);
@@ -3477,7 +3476,7 @@ static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     }
   }
 
-  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
+  return DAG.getNode(Opcode, DL, DAG.getVTList(VT, MVT_CC), LHS, RHS)
       .getValue(1);
 }
 
@@ -3835,7 +3834,7 @@ static bool shouldBeAdjustedToZero(SDValue LHS, APInt C, ISD::CondCode &CC) {
 
 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                              SDValue &AArch64cc, SelectionDAG &DAG,
-                             const SDLoc &dl) {
+                             const SDLoc &DL) {
   if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
     EVT VT = RHS.getValueType();
     APInt C = RHSC->getAPIntValue();
@@ -3844,7 +3843,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     if (shouldBeAdjustedToZero(LHS, C, CC)) {
       // Adjust the constant to zero.
       // CC has already been adjusted.
-      RHS = DAG.getConstant(0, dl, VT);
+      RHS = DAG.getConstant(0, DL, VT);
     } else if (!isLegalCmpImmed(C)) {
       // Constant does not fit, try adjusting it by one?
       switch (CC) {
@@ -3856,7 +3855,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
           APInt CMinusOne = C - 1;
           if (isLegalCmpImmed(CMinusOne)) {
             CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
-            RHS = DAG.getConstant(CMinusOne, dl, VT);
+            RHS = DAG.getConstant(CMinusOne, DL, VT);
           }
         }
         break;
@@ -3866,7 +3865,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
           APInt CMinusOne = C - 1;
           if (isLegalCmpImmed(CMinusOne)) {
             CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
-            RHS = DAG.getConstant(CMinusOne, dl, VT);
+            RHS = DAG.getConstant(CMinusOne, DL, VT);
           }
         }
         break;
@@ -3876,7 +3875,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
           APInt CPlusOne = C + 1;
           if (isLegalCmpImmed(CPlusOne)) {
             CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
-            RHS = DAG.getConstant(CPlusOne, dl, VT);
+            RHS = DAG.getConstant(CPlusOne, DL, VT);
           }
         }
         break;
@@ -3886,7 +3885,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
           APInt CPlusOne = C + 1;
           if (isLegalCmpImmed(CPlusOne)) {
             CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
-            RHS = DAG.getConstant(CPlusOne, dl, VT);
+            RHS = DAG.getConstant(CPlusOne, DL, VT);
           }
         }
         break;
@@ -3945,11 +3944,11 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       int16_t ValueofRHS = RHS->getAsZExtVal();
       if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
         SDValue SExt =
-            DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
+            DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, LHS.getValueType(), LHS,
                         DAG.getValueType(MVT::i16));
         Cmp = emitComparison(
-            SExt, DAG.getSignedConstant(ValueofRHS, dl, RHS.getValueType()), CC,
-            dl, DAG);
+            SExt, DAG.getSignedConstant(ValueofRHS, DL, RHS.getValueType()), CC,
+            DL, DAG);
         AArch64CC = changeIntCCToAArch64CC(CC);
       }
     }
@@ -3963,10 +3962,10 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
   }
 
   if (!Cmp) {
-    Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
+    Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
     AArch64CC = changeIntCCToAArch64CC(CC);
   }
-  AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
+  AArch64cc = DAG.getConstant(AArch64CC, DL, MVT_CC);
   return Cmp;
 }
 
@@ -4067,7 +4066,7 @@ SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue Sel = Op.getOperand(0);
   SDValue Other = Op.getOperand(1);
-  SDLoc dl(Sel);
+  SDLoc DL(Sel);
 
   // If the operand is an overflow checking operation, invert the condition
   // code and kill the Not operation. I.e., transform:
@@ -4081,13 +4080,13 @@ SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
     if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
       return SDValue();
 
-    SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
-    SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
+    SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
+    SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
     AArch64CC::CondCode CC;
     SDValue Value, Overflow;
     std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
-    SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
-    return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
+    SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), DL, MVT::i32);
+    return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
                        CCVal, Overflow);
   }
   // If neither operand is a SELECT_CC, give up.
@@ -4131,13 +4130,13 @@ SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
   // If the constants line up, perform the transform!
   if (CTVal->isZero() && CFVal->isAllOnes()) {
     SDValue CCVal;
-    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
+    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, DL);
 
     FVal = Other;
-    TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
-                       DAG.getAllOnesConstant(dl, Other.getValueType()));
+    TVal = DAG.getNode(ISD::XOR, DL, Other.getValueType(), Other,
+                       DAG.getAllOnesConstant(DL, Other.getValueType()));
 
-    return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
+    return DAG.getNode(AArch64ISD::CSEL, DL, Sel.getValueType(), FVal, TVal,
                        CCVal, Cmp);
   }
 
@@ -4213,25 +4212,25 @@ static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
   if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
     return SDValue();
 
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   AArch64CC::CondCode CC;
   // The actual operation that sets the overflow or carry flag.
   SDValue Value, Overflow;
   std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
 
   // We use 0 and 1 as false and true values.
-  SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
-  SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
+  SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
+  SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
 
   // We use an inverted condition, because the conditional select is inverted
   // too. This will allow it to be selected to a single instruction:
   // CSINC Wd, WZR, WZR, invert(cond).
-  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
-  Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
-                         CCVal, Overflow);
+  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), DL, MVT::i32);
+  Overflow =
+      DAG.getNode(AArch64ISD::CSEL, DL, MVT::i32, FVal, TVal, CCVal, Overflow);
 
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
-  return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
+  return DAG.getNode(ISD::MERGE_VALUES, DL, VTs, Value, Overflow);
 }
 
 // Prefetch operands are:
@@ -4270,7 +4269,7 @@ static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
 // a power of 2. This is then lowered to ANDS X (Y & ~(Z - 1)) instead of SUBS
 // (AND X Y) Z which produces a better opt with EmitComparison
 static void simplifySetCCIntoEq(ISD::CondCode &CC, SDValue &LHS, SDValue &RHS,
-                                SelectionDAG &DAG, const SDLoc dl) {
+                                SelectionDAG &DAG, const SDLoc DL) {
   if (CC == ISD::SETULT && LHS.getOpcode() == ISD::AND && LHS->hasOneUse()) {
     ConstantSDNode *LHSConstOp = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
     ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
@@ -4280,9 +4279,9 @@ static void simplifySetCCIntoEq(ISD::CondCode &CC, SDValue &LHS, SDValue &RHS,
       if (isPowerOf2_64(RHSConstant)) {
         uint64_t NewMaskValue = LHSConstValue & ~(RHSConstant - 1);
         LHS =
-            DAG.getNode(ISD::AND, dl, LHS.getValueType(), LHS.getOperand(0),
-                        DAG.getConstant(NewMaskValue, dl, LHS.getValueType()));
-        RHS = DAG.getConstant(0, dl, RHS.getValueType());
+            DAG.getNode(ISD::AND, DL, LHS.getValueType(), LHS.getOperand(0),
+                        DAG.getConstant(NewMaskValue, DL, LHS.getValueType()));
+        RHS = DAG.getConstant(0, DL, RHS.getValueType());
         CC = ISD::SETEQ;
       }
     }
@@ -4413,55 +4412,55 @@ SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
   if (VT.getScalarType() == MVT::bf16 &&
       !((Subtarget->hasNEON() || Subtarget->hasSME()) &&
         Subtarget->hasBF16())) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     SDValue Narrow = SrcVal;
     SDValue NaN;
     EVT I32 = SrcVT.changeElementType(MVT::i32);
     EVT F32 = SrcVT.changeElementType(MVT::f32);
     if (SrcVT.getScalarType() == MVT::f32) {
       bool NeverSNaN = DAG.isKnownNeverSNaN(Narrow);
-      Narrow = DAG.getNode(ISD::BITCAST, dl, I32, Narrow);
+      Narrow = DAG.getNode(ISD::BITCAST, DL, I32, Narrow);
       if (!NeverSNaN) {
         // Set the quiet bit.
-        NaN = DAG.getNode(ISD::OR, dl, I32, Narrow,
-                          DAG.getConstant(0x400000, dl, I32));
+        NaN = DAG.getNode(ISD::OR, DL, I32, Narrow,
+                          DAG.getConstant(0x400000, DL, I32));
       }
     } else if (SrcVT.getScalarType() == MVT::f64) {
-      Narrow = DAG.getNode(AArch64ISD::FCVTXN, dl, F32, Narrow);
-      Narrow = DAG.getNode(ISD::BITCAST, dl, I32, Narrow);
+      Narrow = DAG.getNode(AArch64ISD::FCVTXN, DL, F32, Narrow);
+      Narrow = DAG.getNode(ISD::BITCAST, DL, I32, Narrow);
     } else {
       return SDValue();
     }
     if (!Trunc) {
-      SDValue One = DAG.getConstant(1, dl, I32);
-      SDValue Lsb = DAG.getNode(ISD::SRL, dl, I32, Narrow,
-                                DAG.getShiftAmountConstant(16, I32, dl));
-      Lsb = DAG.getNode(ISD::AND, dl, I32, Lsb, One);
+      SDValue One = DAG.getConstant(1, DL, I32);
+      SDValue Lsb = DAG.getNode(ISD::SRL, DL, I32, Narrow,
+                                DAG.getShiftAmountConstant(16, I32, DL));
+      Lsb = DAG.getNode(ISD::AND, DL, I32, Lsb, One);
       SDValue RoundingBias =
-          DAG.getNode(ISD::ADD, dl, I32, DAG.getConstant(0x7fff, dl, I32), Lsb);
-      Narrow = DAG.getNode(ISD::ADD, dl, I32, Narrow, RoundingBias);
+          DAG.getNode(ISD::ADD, DL, I32, DAG.getConstant(0x7fff, DL, I32), Lsb);
+      Narrow = DAG.getNode(ISD::ADD, DL, I32, Narrow, RoundingBias);
     }
 
     // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
     // 0x80000000.
     if (NaN) {
       SDValue IsNaN = DAG.getSetCC(
-          dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT),
+          DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT),
           SrcVal, SrcVal, ISD::SETUO);
-      Narrow = DAG.getSelect(dl, I32, IsNaN, NaN, Narrow);
+      Narrow = DAG.getSelect(DL, I32, IsNaN, NaN, Narrow);
     }
 
     // Now that we have rounded, shift the bits into position.
-    Narrow = DAG.getNode(ISD::SRL, dl, I32, Narrow,
-                         DAG.getShiftAmountConstant(16, I32, dl));
+    Narrow = DAG.getNode(ISD::SRL, DL, I32, Narrow,
+                         DAG.getShiftAmountConstant(16, I32, DL));
     if (VT.isVector()) {
       EVT I16 = I32.changeVectorElementType(MVT::i16);
-      Narrow = DAG.getNode(ISD::TRUNCATE, dl, I16, Narrow);
-      return DAG.getNode(ISD::BITCAST, dl, VT, Narrow);
+      Narrow = DAG.getNode(ISD::TRUNCATE, DL, I16, Narrow);
+      return DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
     }
-    Narrow = DAG.getNode(ISD::BITCAST, dl, F32, Narrow);
-    SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, dl, VT, Narrow);
-    return IsStrict ? DAG.getMergeValues({Result, Op.getOperand(0)}, dl)
+    Narrow = DAG.getNode(ISD::BITCAST, DL, F32, Narrow);
+    SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Narrow);
+    return IsStrict ? DAG.getMergeValues({Result, Op.getOperand(0)}, DL)
                     : Result;
   }
 
@@ -4493,16 +4492,16 @@ SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
   if ((InVT.getVectorElementType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
       InVT.getVectorElementType() == MVT::bf16) {
     EVT NewVT = VT.changeElementType(MVT::f32);
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     if (IsStrict) {
-      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NewVT, MVT::Other},
+      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVT, MVT::Other},
                                 {Op.getOperand(0), Op.getOperand(1)});
-      return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
+      return DAG.getNode(Op.getOpcode(), DL, {VT, MVT::Other},
                          {Ext.getValue(1), Ext.getValue(0)});
     }
     return DAG.getNode(
-        Op.getOpcode(), dl, Op.getValueType(),
-        DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
+        Op.getOpcode(), DL, Op.getValueType(),
+        DAG.getNode(ISD::FP_EXTEND, DL, NewVT, Op.getOperand(0)));
   }
 
   if (VT.isScalableVector()) {
@@ -4531,47 +4530,47 @@ SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
   uint64_t VTSize = VT.getFixedSizeInBits();
   uint64_t InVTSize = InVT.getFixedSizeInBits();
   if (VTSize < InVTSize) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     if (IsStrict) {
       InVT = InVT.changeVectorElementTypeToInteger();
-      SDValue Cv = DAG.getNode(Op.getOpcode(), dl, {InVT, MVT::Other},
+      SDValue Cv = DAG.getNode(Op.getOpcode(), DL, {InVT, MVT::Other},
                                {Op.getOperand(0), Op.getOperand(1)});
-      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
-      return DAG.getMergeValues({Trunc, Cv.getValue(1)}, dl);
+      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Cv);
+      return DAG.getMergeValues({Trunc, Cv.getValue(1)}, DL);
     }
     SDValue Cv =
-        DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
+        DAG.getNode(Op.getOpcode(), DL, InVT.changeVectorElementTypeToInteger(),
                     Op.getOperand(0));
-    return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
+    return DAG.getNode(ISD::TRUNCATE, DL, VT, Cv);
   }
 
   if (VTSize > InVTSize) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     MVT ExtVT =
         MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
                          VT.getVectorNumElements());
     if (IsStrict) {
-      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {ExtVT, MVT::Other},
+      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {ExtVT, MVT::Other},
                                 {Op.getOperand(0), Op.getOperand(1)});
-      return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},
+      return DAG.getNode(Op.getOpcode(), DL, {VT, MVT::Other},
                          {Ext.getValue(1), Ext.getValue(0)});
     }
-    SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
-    return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
+    SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, ExtVT, Op.getOperand(0));
+    return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
   }
 
   // Use a scalar operation for conversions between single-element vectors of
   // the same size.
   if (InVT.getVectorNumElements() == 1) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     SDValue Extract = DAG.getNode(
-        ISD::EXTRACT_VECTOR_ELT, dl, InVT.getScalarType(),
-        Op.getOperand(IsStrict ? 1 : 0), DAG.getConstant(0, dl, MVT::i64));
+        ISD::EXTRACT_VECTOR_ELT, DL, InVT.getScalarType(),
+        Op.getOperand(IsStrict ? 1 : 0), DAG.getConstant(0, DL, MVT::i64));
     EVT ScalarVT = VT.getScalarType();
     if (IsStrict)
-      return DAG.getNode(Op.getOpcode(), dl, {ScalarVT, MVT::Other},
+      return DAG.getNode(Op.getOpcode(), DL, {ScalarVT, MVT::Other},
                          {Op.getOperand(0), Extract});
-    return DAG.getNode(Op.getOpcode(), dl, ScalarVT, Extract);
+    return DAG.getNode(Op.getOpcode(), DL, ScalarVT, Extract);
   }
 
   // Type changing conversions are illegal.
@@ -4589,17 +4588,16 @@ SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
   // f16 conversions are promoted to f32 when full fp16 is not supported.
   if ((SrcVal.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
       SrcVal.getValueType() == MVT::bf16) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     if (IsStrict) {
       SDValue Ext =
-          DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other},
+          DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                       {Op.getOperand(0), SrcVal});
-      return DAG.getNode(Op.getOpcode(), dl, {Op.getValueType(), MVT::Other},
+      return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
                          {Ext.getValue(1), Ext.getValue(0)});
     }
-    return DAG.getNode(
-        Op.getOpcode(), dl, Op.getValueType(),
-        DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal));
+    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
+                       DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
   }
 
   if (SrcVal.getValueType() != MVT::f128) {
@@ -4799,7 +4797,7 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
   // in the cost tables.
   bool IsStrict = Op->isStrictFPOpcode();
   EVT VT = Op.getValueType();
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   SDValue In = Op.getOperand(IsStrict ? 1 : 0);
   EVT InVT = In.getValueType();
   unsigned Opc = Op.getOpcode();
@@ -4810,26 +4808,26 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
 
   // NOTE: i1->bf16 does not require promotion to f32.
   if (VT.isScalableVector() && InVT.getVectorElementType() == MVT::i1) {
-    SDValue FalseVal = DAG.getConstantFP(0.0, dl, VT);
-    SDValue TrueVal = IsSigned ? DAG.getConstantFP(-1.0, dl, VT)
-                               : DAG.getConstantFP(1.0, dl, VT);
-    return DAG.getNode(ISD::VSELECT, dl, VT, In, TrueVal, FalseVal);
+    SDValue FalseVal = DAG.getConstantFP(0.0, DL, VT);
+    SDValue TrueVal = IsSigned ? DAG.getConstantFP(-1.0, DL, VT)
+                               : DAG.getConstantFP(1.0, DL, VT);
+    return DAG.getNode(ISD::VSELECT, DL, VT, In, TrueVal, FalseVal);
   }
 
   // Promote bf16 conversions to f32.
   if (VT.getVectorElementType() == MVT::bf16) {
     EVT F32 = VT.changeElementType(MVT::f32);
     if (IsStrict) {
-      SDValue Val = DAG.getNode(Op.getOpcode(), dl, {F32, MVT::Other},
+      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {F32, MVT::Other},
                                 {Op.getOperand(0), In});
-      return DAG.getNode(ISD::STRICT_FP_ROUND, dl,
+      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                          {Op.getValueType(), MVT::Other},
                          {Val.getValue(1), Val.getValue(0),
-                          DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});
+                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
     }
-    return DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(),
-                       DAG.getNode(Op.getOpcode(), dl, F32, In),
-                       DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+    return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
+                       DAG.getNode(Op.getOpcode(), DL, F32, In),
+                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
   }
 
   if (VT.isScalableVector()) {
@@ -4876,37 +4874,36 @@ SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
         MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
                          InVT.getVectorNumElements());
     if (IsStrict) {
-      In = DAG.getNode(Opc, dl, {CastVT, MVT::Other},
-                       {Op.getOperand(0), In});
-      return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},
+      In = DAG.getNode(Opc, DL, {CastVT, MVT::Other}, {Op.getOperand(0), In});
+      return DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
                          {In.getValue(1), In.getValue(0),
-                          DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});
+                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
     }
-    In = DAG.getNode(Opc, dl, CastVT, In);
-    return DAG.getNode(ISD::FP_ROUND, dl, VT, In,
-                       DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+    In = DAG.getNode(Opc, DL, CastVT, In);
+    return DAG.getNode(ISD::FP_ROUND, DL, VT, In,
+                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
   }
 
   if (VTSize > InVTSize) {
     unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
     EVT CastVT = VT.changeVectorElementTypeToInteger();
-    In = DAG.getNode(CastOpc, dl, CastVT, In);
+    In = DAG.getNode(CastOpc, DL, CastVT, In);
     if (IsStrict)
-      return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op.getOperand(0), In});
-    return DAG.getNode(Opc, dl, VT, In);
+      return DAG.getNode(Opc, DL, {VT, MVT::Other}, {Op.getOperand(0), In});
+    return DAG.getNode(Opc, DL, VT, In);
   }
 
   // Use a scalar operation for conversions between single-element vectors of
   // the same size.
   if (VT.getVectorNumElements() == 1) {
-    SDValue Extract = DAG.getNode(
-        ISD::EXTRACT_VECTOR_ELT, dl, InVT.getScalarType(),
-        In, DAG.getConstant(0, dl, MVT::i64));
+    SDValue Extract =
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InVT.getScalarType(), In,
+                    DAG.getConstant(0, DL, MVT::i64));
     EVT ScalarVT = VT.getScalarType();
     if (IsStrict)
-      return DAG.getNode(Op.getOpcode(), dl, {ScalarVT, MVT::Other},
+      return DAG.getNode(Op.getOpcode(), DL, {ScalarVT, MVT::Other},
                          {Op.getOperand(0), Extract});
-    return DAG.getNode(Op.getOpcode(), dl, ScalarVT, Extract);
+    return DAG.getNode(Op.getOpcode(), DL, ScalarVT, Extract);
   }
 
   return Op;
@@ -4924,18 +4921,18 @@ SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
                   Op->getOpcode() == ISD::SINT_TO_FP;
 
   auto IntToFpViaPromotion = [&](EVT PromoteVT) {
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     if (IsStrict) {
-      SDValue Val = DAG.getNode(Op.getOpcode(), dl, {PromoteVT, MVT::Other},
+      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {PromoteVT, MVT::Other},
                                 {Op.getOperand(0), SrcVal});
-      return DAG.getNode(ISD::STRICT_FP_ROUND, dl,
+      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                          {Op.getValueType(), MVT::Other},
                          {Val.getValue(1), Val.getValue(0),
-                          DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)});
+                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
     }
-    return DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(),
-                       DAG.getNode(Op.getOpcode(), dl, PromoteVT, SrcVal),
-                       DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
+    return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
+                       DAG.getNode(Op.getOpcode(), DL, PromoteVT, SrcVal),
+                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
   };
 
   if (Op.getValueType() == MVT::bf16) {
@@ -5057,7 +5054,7 @@ SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
                                             SelectionDAG &DAG) const {
   // For iOS, we want to call an alternative entry point: __sincos_stret,
   // which returns the values in two S / D registers.
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   SDValue Arg = Op.getOperand(0);
   EVT ArgVT = Arg.getValueType();
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
@@ -5080,7 +5077,7 @@ SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
   StructType *RetTy = StructType::get(ArgTy, ArgTy);
   TargetLowering::CallLoweringInfo CLI(DAG);
   CallingConv::ID CC = getLibcallCallingConv(LC);
-  CLI.setDebugLoc(dl)
+  CLI.setDebugLoc(DL)
       .setChain(DAG.getEntryNode())
       .setLibCallee(CC, RetTy, Callee, std::move(Args));
 
@@ -5238,21 +5235,21 @@ SDValue AArch64TargetLowering::LowerGET_ROUNDING(SDValue Op,
   // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
   // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
   // so that the shift + and get folded into a bitfield extract.
-  SDLoc dl(Op);
+  SDLoc DL(Op);
 
   SDValue Chain = Op.getOperand(0);
   SDValue FPCR_64 = DAG.getNode(
-      ISD::INTRINSIC_W_CHAIN, dl, {MVT::i64, MVT::Other},
-      {Chain, DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl, MVT::i64)});
+      ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other},
+      {Chain, DAG.getConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)});
   Chain = FPCR_64.getValue(1);
-  SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
-  SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
-                                  DAG.getConstant(1U << 22, dl, MVT::i32));
-  SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
-                              DAG.getConstant(22, dl, MVT::i32));
-  SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
-                            DAG.getConstant(3, dl, MVT::i32));
-  return DAG.getMergeValues({AND, Chain}, dl);
+  SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPCR_64);
+  SDValue FltRounds = DAG.getNode(ISD::ADD, DL, MVT::i32, FPCR_32,
+                                  DAG.getConstant(1U << 22, DL, MVT::i32));
+  SDValue RMODE = DAG.getNode(ISD::SRL, DL, MVT::i32, FltRounds,
+                              DAG.getConstant(22, DL, MVT::i32));
+  SDValue AND = DAG.getNode(ISD::AND, DL, MVT::i32, RMODE,
+                            DAG.getConstant(3, DL, MVT::i32));
+  return DAG.getMergeValues({AND, Chain}, DL);
 }
 
 SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
@@ -5509,14 +5506,14 @@ static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
   if (!isa<ConstantSDNode>(N->getOperand(Op1)))
     return SDValue();
 
-  SDLoc dl(N);
+  SDLoc DL(N);
   APInt Y = N->getConstantOperandAPInt(Op1);
 
   // When the second operand is the maximum value, comparisons that include
   // equality can never fail and thus we can return an all active predicate.
   if (IsEqual)
     if (IsSigned ? Y.isMaxSignedValue() : Y.isMaxValue())
-      return DAG.getConstant(1, dl, N->getValueType(0));
+      return DAG.getConstant(1, DL, N->getValueType(0));
 
   if (!isa<ConstantSDNode>(N->getOperand(Op0)))
     return SDValue();
@@ -5545,7 +5542,7 @@ static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
   unsigned ElementSize = 128 / N->getValueType(0).getVectorMinNumElements();
   if (PredPattern != std::nullopt &&
       NumActiveElems.getZExtValue() <= (MinSVEVectorSize / ElementSize))
-    return getPTrue(DAG, dl, N->getValueType(0), *PredPattern);
+    return getPTrue(DAG, DL, N->getValueType(0), *PredPattern);
 
   return SDValue();
 }
@@ -5699,9 +5696,9 @@ SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
 }
 
 SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   SDValue ID =
-      DAG.getTargetConstant(Intrinsic::aarch64_sve_match, dl, MVT::i64);
+      DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
 
   auto Op1 = Op.getOperand(1);
   auto Op2 = Op.getOperand(2);
@@ -5727,41 +5724,41 @@ SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
     Op2 = convertToScalableVector(DAG, OpContainerVT, Op2);
     // Further, if the result is scalable, broadcast Op2 to a full SVE register.
     if (ResVT.isScalableVector())
-      Op2 = DAG.getNode(AArch64ISD::DUPLANE128, dl, OpContainerVT, Op2,
-                        DAG.getTargetConstant(0, dl, MVT::i64));
+      Op2 = DAG.getNode(AArch64ISD::DUPLANE128, DL, OpContainerVT, Op2,
+                        DAG.getTargetConstant(0, DL, MVT::i64));
   } else {
     // If Op2 is not a full 128-bit vector, we always need to broadcast it.
     unsigned Op2BitWidth = Op2VT.getFixedSizeInBits();
     MVT Op2IntVT = MVT::getIntegerVT(Op2BitWidth);
     EVT Op2PromotedVT = getPackedSVEVectorVT(Op2IntVT);
     Op2 = DAG.getBitcast(MVT::getVectorVT(Op2IntVT, 1), Op2);
-    Op2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op2IntVT, Op2,
-                      DAG.getConstant(0, dl, MVT::i64));
-    Op2 = DAG.getSplatVector(Op2PromotedVT, dl, Op2);
+    Op2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op2IntVT, Op2,
+                      DAG.getConstant(0, DL, MVT::i64));
+    Op2 = DAG.getSplatVector(Op2PromotedVT, DL, Op2);
     Op2 = DAG.getBitcast(OpContainerVT, Op2);
   }
 
   // If the result is scalable, we just need to carry out the MATCH.
   if (ResVT.isScalableVector())
-    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResVT, ID, Mask, Op1, Op2);
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResVT, ID, Mask, Op1, Op2);
 
   // If the result is fixed, we can still use MATCH but we need to wrap the
   // first operand and the mask in scalable vectors before doing so.
 
   // Wrap the operands.
   Op1 = convertToScalableVector(DAG, OpContainerVT, Op1);
-  Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, Op1VT, Mask);
+  Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, Op1VT, Mask);
   Mask = convertFixedMaskToScalableVector(Mask, DAG);
 
   // Carry out the match.
-  SDValue Match = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, Mask.getValueType(),
+  SDValue Match = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Mask.getValueType(),
                               ID, Mask, Op1, Op2);
 
   // Extract and promote the match result (nxv16i1/nxv8i1) to ResVT
   // (v16i8/v8i8).
-  Match = DAG.getNode(ISD::SIGN_EXTEND, dl, OpContainerVT, Match);
+  Match = DAG.getNode(ISD::SIGN_EXTEND, DL, OpContainerVT, Match);
   Match = convertFromScalableVector(DAG, Op1VT, Match);
-  return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Match);
+  return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Match);
 }
 
 SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
@@ -5840,22 +5837,22 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                      SelectionDAG &DAG) const {
   unsigned IntNo = Op.getConstantOperandVal(0);
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   switch (IntNo) {
   default: return SDValue();    // Don't custom lower most intrinsics.
   case Intrinsic::thread_pointer: {
     EVT PtrVT = getPointerTy(DAG.getDataLayout());
-    return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
+    return DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
   }
   case Intrinsic::aarch64_neon_abs: {
     EVT Ty = Op.getValueType();
     if (Ty == MVT::i64) {
-      SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
-                                   Op.getOperand(1));
-      Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
-      return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
+      SDValue Result =
+          DAG.getNode(ISD::BITCAST, DL, MVT::v1i64, Op.getOperand(1));
+      Result = DAG.getNode(ISD::ABS, DL, MVT::v1i64, Result);
+      return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Result);
     } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
-      return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
+      return DAG.getNode(ISD::ABS, DL, Ty, Op.getOperand(1));
     } else {
       report_fatal_error("Unexpected type for AArch64 NEON intrinsic");
     }
@@ -5878,14 +5875,14 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     // registers as opposed to a GPR load followed by a fmov.
     auto TryVectorizeOperand = [](SDValue N, std::optional<uint64_t> NLane,
                                   std::optional<uint64_t> OtherLane,
-                                  const SDLoc &dl,
+                                  const SDLoc &DL,
                                   SelectionDAG &DAG) -> SDValue {
       // If the operand is an higher half itself, rewrite it to
       // extract_high_v2i64; this way aarch64_neon_pmull64 could
       // re-use the dag-combiner function with aarch64_neon_{pmull,smull,umull}.
       if (NLane == 1)
-        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i64,
-                           N.getOperand(0), DAG.getConstant(1, dl, MVT::i64));
+        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v1i64,
+                           N.getOperand(0), DAG.getConstant(1, DL, MVT::i64));
 
       // Operand N is not a higher half but the other operand is.
       if (OtherLane == 1) {
@@ -5896,123 +5893,121 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
         //   mov x8, v0.d[1]
         //   fmov d0, x8
         if (NLane == 0)
-          return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i64,
-                             DAG.getNode(AArch64ISD::DUPLANE64, dl, MVT::v2i64,
+          return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v1i64,
+                             DAG.getNode(AArch64ISD::DUPLANE64, DL, MVT::v2i64,
                                          N.getOperand(0),
-                                         DAG.getConstant(0, dl, MVT::i64)),
-                             DAG.getConstant(1, dl, MVT::i64));
+                                         DAG.getConstant(0, DL, MVT::i64)),
+                             DAG.getConstant(1, DL, MVT::i64));
 
         // Otherwise just dup from main to all lanes.
-        return DAG.getNode(AArch64ISD::DUP, dl, MVT::v1i64, N);
+        return DAG.getNode(AArch64ISD::DUP, DL, MVT::v1i64, N);
       }
 
       // Neither operand is an extract of higher half, so codegen may just use
       // the non-high version of PMULL instruction. Use v1i64 to represent i64.
       assert(N.getValueType() == MVT::i64 &&
              "Intrinsic aarch64_neon_pmull64 requires i64 parameters");
-      return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, N);
+      return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i64, N);
     };
 
-    LHS = TryVectorizeOperand(LHS, LHSLane, RHSLane, dl, DAG);
-    RHS = TryVectorizeOperand(RHS, RHSLane, LHSLane, dl, DAG);
+    LHS = TryVectorizeOperand(LHS, LHSLane, RHSLane, DL, DAG);
+    RHS = TryVectorizeOperand(RHS, RHSLane, LHSLane, DL, DAG);
 
-    return DAG.getNode(AArch64ISD::PMULL, dl, Op.getValueType(), LHS, RHS);
+    return DAG.getNode(AArch64ISD::PMULL, DL, Op.getValueType(), LHS, RHS);
   }
   case Intrinsic::aarch64_neon_smax:
-    return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(2));
+    return DAG.getNode(ISD::SMAX, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
   case Intrinsic::aarch64_neon_umax:
-    return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(2));
+    return DAG.getNode(ISD::UMAX, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
   case Intrinsic::aarch64_neon_smin:
-    return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(2));
+    return DAG.getNode(ISD::SMIN, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
   case Intrinsic::aarch64_neon_umin:
-    return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(2));
+    return DAG.getNode(ISD::UMIN, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
   case Intrinsic::aarch64_neon_scalar_sqxtn:
   case Intrinsic::aarch64_neon_scalar_sqxtun:
   case Intrinsic::aarch64_neon_scalar_uqxtn: {
     assert(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::f32);
     if (Op.getValueType() == MVT::i32)
-      return DAG.getNode(ISD::BITCAST, dl, MVT::i32,
-                         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::f32,
+      return DAG.getNode(ISD::BITCAST, DL, MVT::i32,
+                         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::f32,
                                      Op.getOperand(0),
-                                     DAG.getNode(ISD::BITCAST, dl, MVT::f64,
+                                     DAG.getNode(ISD::BITCAST, DL, MVT::f64,
                                                  Op.getOperand(1))));
     return SDValue();
   }
   case Intrinsic::aarch64_neon_sqxtn:
-    return DAG.getNode(ISD::TRUNCATE_SSAT_S, dl, Op.getValueType(),
+    return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_neon_sqxtun:
-    return DAG.getNode(ISD::TRUNCATE_SSAT_U, dl, Op.getValueType(),
+    return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_neon_uqxtn:
-    return DAG.getNode(ISD::TRUNCATE_USAT_U, dl, Op.getValueType(),
+    return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_neon_sqshrn:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::TRUNCATE_SSAT_S, dl, Op.getValueType(),
-                         DAG.getNode(AArch64ISD::VASHR, dl,
+      return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::VASHR, DL,
                                      Op.getOperand(1).getValueType(),
                                      Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_sqshrun:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::TRUNCATE_SSAT_U, dl, Op.getValueType(),
-                         DAG.getNode(AArch64ISD::VASHR, dl,
+      return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::VASHR, DL,
                                      Op.getOperand(1).getValueType(),
                                      Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_uqshrn:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::TRUNCATE_USAT_U, dl, Op.getValueType(),
-                         DAG.getNode(AArch64ISD::VLSHR, dl,
+      return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::VLSHR, DL,
                                      Op.getOperand(1).getValueType(),
                                      Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_sqrshrn:
     if (Op.getValueType().isVector())
-      return DAG.getNode(
-          ISD::TRUNCATE_SSAT_S, dl, Op.getValueType(),
-          DAG.getNode(
-              AArch64ISD::SRSHR_I, dl, Op.getOperand(1).getValueType(),
-              Op.getOperand(1), Op.getOperand(2)));
+      return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::SRSHR_I, DL,
+                                     Op.getOperand(1).getValueType(),
+                                     Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_sqrshrun:
     if (Op.getValueType().isVector())
-      return DAG.getNode(
-          ISD::TRUNCATE_SSAT_U, dl, Op.getValueType(),
-          DAG.getNode(
-              AArch64ISD::SRSHR_I, dl, Op.getOperand(1).getValueType(),
-              Op.getOperand(1), Op.getOperand(2)));
+      return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::SRSHR_I, DL,
+                                     Op.getOperand(1).getValueType(),
+                                     Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_uqrshrn:
     if (Op.getValueType().isVector())
-      return DAG.getNode(
-          ISD::TRUNCATE_USAT_U, dl, Op.getValueType(),
-          DAG.getNode(
-              AArch64ISD::URSHR_I, dl, Op.getOperand(1).getValueType(), Op.getOperand(1), Op.getOperand(2)));
+      return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
+                         DAG.getNode(AArch64ISD::URSHR_I, DL,
+                                     Op.getOperand(1).getValueType(),
+                                     Op.getOperand(1), Op.getOperand(2)));
     return SDValue();
   case Intrinsic::aarch64_neon_sqadd:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::SADDSAT, dl, Op.getValueType(), Op.getOperand(1),
+      return DAG.getNode(ISD::SADDSAT, DL, Op.getValueType(), Op.getOperand(1),
                          Op.getOperand(2));
     return SDValue();
   case Intrinsic::aarch64_neon_sqsub:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::SSUBSAT, dl, Op.getValueType(), Op.getOperand(1),
+      return DAG.getNode(ISD::SSUBSAT, DL, Op.getValueType(), Op.getOperand(1),
                          Op.getOperand(2));
     return SDValue();
   case Intrinsic::aarch64_neon_uqadd:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::UADDSAT, dl, Op.getValueType(), Op.getOperand(1),
+      return DAG.getNode(ISD::UADDSAT, DL, Op.getValueType(), Op.getOperand(1),
                          Op.getOperand(2));
     return SDValue();
   case Intrinsic::aarch64_neon_uqsub:
     if (Op.getValueType().isVector())
-      return DAG.getNode(ISD::USUBSAT, dl, Op.getValueType(), Op.getOperand(1),
+      return DAG.getNode(ISD::USUBSAT, DL, Op.getValueType(), Op.getOperand(1),
                          Op.getOperand(2));
     return SDValue();
   case Intrinsic::aarch64_sve_whilelt:
@@ -6025,224 +6020,224 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
                                      /*IsEqual=*/true);
   case Intrinsic::aarch64_sve_sunpkhi:
-    return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::SUNPKHI, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_sunpklo:
-    return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::SUNPKLO, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_uunpkhi:
-    return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::UUNPKHI, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_uunpklo:
-    return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::UUNPKLO, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_clasta_n:
-    return DAG.getNode(AArch64ISD::CLASTA_N, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::CLASTA_N, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
   case Intrinsic::aarch64_sve_clastb_n:
-    return DAG.getNode(AArch64ISD::CLASTB_N, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::CLASTB_N, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
   case Intrinsic::aarch64_sve_lasta:
-    return DAG.getNode(AArch64ISD::LASTA, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::LASTA, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_lastb:
-    return DAG.getNode(AArch64ISD::LASTB, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::LASTB, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_rev:
-    return DAG.getNode(ISD::VECTOR_REVERSE, dl, Op.getValueType(),
+    return DAG.getNode(ISD::VECTOR_REVERSE, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_tbl:
-    return DAG.getNode(AArch64ISD::TBL, dl, Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(2));
+    return DAG.getNode(AArch64ISD::TBL, DL, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2));
   case Intrinsic::aarch64_sve_trn1:
-    return DAG.getNode(AArch64ISD::TRN1, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::TRN1, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_trn2:
-    return DAG.getNode(AArch64ISD::TRN2, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::TRN2, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_uzp1:
-    return DAG.getNode(AArch64ISD::UZP1, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::UZP1, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_uzp2:
-    return DAG.getNode(AArch64ISD::UZP2, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::UZP2, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_zip1:
-    return DAG.getNode(AArch64ISD::ZIP1, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::ZIP1, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_zip2:
-    return DAG.getNode(AArch64ISD::ZIP2, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::ZIP2, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_splice:
-    return DAG.getNode(AArch64ISD::SPLICE, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::SPLICE, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
   case Intrinsic::aarch64_sve_ptrue:
-    return getPTrue(DAG, dl, Op.getValueType(), Op.getConstantOperandVal(1));
+    return getPTrue(DAG, DL, Op.getValueType(), Op.getConstantOperandVal(1));
   case Intrinsic::aarch64_sve_clz:
-    return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sme_cntsb:
-    return DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
-                       DAG.getConstant(1, dl, MVT::i32));
+    return DAG.getNode(AArch64ISD::RDSVL, DL, Op.getValueType(),
+                       DAG.getConstant(1, DL, MVT::i32));
   case Intrinsic::aarch64_sme_cntsh: {
-    SDValue One = DAG.getConstant(1, dl, MVT::i32);
-    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(), One);
-    return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, One);
+    SDValue One = DAG.getConstant(1, DL, MVT::i32);
+    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, DL, Op.getValueType(), One);
+    return DAG.getNode(ISD::SRL, DL, Op.getValueType(), Bytes, One);
   }
   case Intrinsic::aarch64_sme_cntsw: {
-    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
-                                DAG.getConstant(1, dl, MVT::i32));
-    return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes,
-                       DAG.getConstant(2, dl, MVT::i32));
+    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, DL, Op.getValueType(),
+                                DAG.getConstant(1, DL, MVT::i32));
+    return DAG.getNode(ISD::SRL, DL, Op.getValueType(), Bytes,
+                       DAG.getConstant(2, DL, MVT::i32));
   }
   case Intrinsic::aarch64_sme_cntsd: {
-    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, dl, Op.getValueType(),
-                                DAG.getConstant(1, dl, MVT::i32));
-    return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes,
-                       DAG.getConstant(3, dl, MVT::i32));
+    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, DL, Op.getValueType(),
+                                DAG.getConstant(1, DL, MVT::i32));
+    return DAG.getNode(ISD::SRL, DL, Op.getValueType(), Bytes,
+                       DAG.getConstant(3, DL, MVT::i32));
   }
   case Intrinsic::aarch64_sve_cnt: {
     SDValue Data = Op.getOperand(3);
     // CTPOP only supports integer operands.
     if (Data.getValueType().isFloatingPoint())
-      Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data);
-    return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, dl, Op.getValueType(),
+      Data = DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Data);
+    return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Data, Op.getOperand(1));
   }
   case Intrinsic::aarch64_sve_dupq_lane:
     return LowerDUPQLane(Op, DAG);
   case Intrinsic::aarch64_sve_convert_from_svbool:
     if (Op.getValueType() == MVT::aarch64svcount)
-      return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Op.getOperand(1));
+      return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Op.getOperand(1));
     return getSVEPredicateBitCast(Op.getValueType(), Op.getOperand(1), DAG);
   case Intrinsic::aarch64_sve_convert_to_svbool:
     if (Op.getOperand(1).getValueType() == MVT::aarch64svcount)
-      return DAG.getNode(ISD::BITCAST, dl, MVT::nxv16i1, Op.getOperand(1));
+      return DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i1, Op.getOperand(1));
     return getSVEPredicateBitCast(MVT::nxv16i1, Op.getOperand(1), DAG);
   case Intrinsic::aarch64_sve_fneg:
-    return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintp:
-    return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintm:
-    return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frinti:
-    return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, dl, Op.getValueType(),
-                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+    return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, DL,
+                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
+                       Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintx:
-    return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frinta:
-    return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintn:
-    return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, dl, Op.getValueType(),
-                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
+    return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, DL,
+                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
+                       Op.getOperand(1));
   case Intrinsic::aarch64_sve_frintz:
-    return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_ucvtf:
-    return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, dl,
+    return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, DL,
                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_scvtf:
-    return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, dl,
+    return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, DL,
                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_fcvtzu:
-    return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, dl,
-                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
-                       Op.getOperand(1));
+    return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, DL, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_fcvtzs:
-    return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, dl,
-                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
-                       Op.getOperand(1));
+    return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, DL, Op.getValueType(),
+                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_fsqrt:
-    return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frecpx:
-    return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_frecpe_x:
-    return DAG.getNode(AArch64ISD::FRECPE, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRECPE, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_frecps_x:
-    return DAG.getNode(AArch64ISD::FRECPS, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRECPS, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_frsqrte_x:
-    return DAG.getNode(AArch64ISD::FRSQRTE, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRSQRTE, DL, Op.getValueType(),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_frsqrts_x:
-    return DAG.getNode(AArch64ISD::FRSQRTS, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FRSQRTS, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   case Intrinsic::aarch64_sve_fabs:
-    return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_abs:
-    return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_neg:
-    return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_insr: {
     SDValue Scalar = Op.getOperand(2);
     EVT ScalarTy = Scalar.getValueType();
     if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
-      Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
+      Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Scalar);
 
-    return DAG.getNode(AArch64ISD::INSR, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::INSR, DL, Op.getValueType(),
                        Op.getOperand(1), Scalar);
   }
   case Intrinsic::aarch64_sve_rbit:
-    return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, dl,
+    return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, DL,
                        Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                        Op.getOperand(1));
   case Intrinsic::aarch64_sve_revb:
-    return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_revh:
-    return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_revw:
-    return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_revd:
-    return DAG.getNode(AArch64ISD::REVD_MERGE_PASSTHRU, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::REVD_MERGE_PASSTHRU, DL, Op.getValueType(),
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
   case Intrinsic::aarch64_sve_sxtb:
     return DAG.getNode(
-        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
         Op.getOperand(1));
   case Intrinsic::aarch64_sve_sxth:
     return DAG.getNode(
-        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
         Op.getOperand(1));
   case Intrinsic::aarch64_sve_sxtw:
     return DAG.getNode(
-        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
         Op.getOperand(1));
   case Intrinsic::aarch64_sve_uxtb:
     return DAG.getNode(
-        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i8)),
         Op.getOperand(1));
   case Intrinsic::aarch64_sve_uxth:
     return DAG.getNode(
-        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i16)),
         Op.getOperand(1));
   case Intrinsic::aarch64_sve_uxtw:
     return DAG.getNode(
-        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, dl, Op.getValueType(),
+        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
         Op.getOperand(2), Op.getOperand(3),
         DAG.getValueType(Op.getValueType().changeVectorElementType(MVT::i32)),
         Op.getOperand(1));
@@ -6250,7 +6245,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     const auto &MF = DAG.getMachineFunction();
     const auto *RegInfo = Subtarget->getRegisterInfo();
     unsigned Reg = RegInfo->getLocalAddressRegister(MF);
-    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
+    return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg,
                               Op.getSimpleValueType());
   }
 
@@ -6282,7 +6277,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     bool IsShiftRight = IntNo == Intrinsic::aarch64_neon_vsri ||
                         IntNo == Intrinsic::aarch64_sve_sri;
     unsigned Opcode = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
-    return DAG.getNode(Opcode, dl, Ty, Op.getOperand(1), Op.getOperand(2),
+    return DAG.getNode(Opcode, DL, Ty, Op.getOperand(1), Op.getOperand(2),
                        Op.getOperand(3));
   }
 
@@ -6297,7 +6292,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     unsigned Opcode = IsSignedAdd
                           ? (IsRoundingAdd ? ISD::AVGCEILS : ISD::AVGFLOORS)
                           : (IsRoundingAdd ? ISD::AVGCEILU : ISD::AVGFLOORU);
-    return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
+    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1),
                        Op.getOperand(2));
   }
   case Intrinsic::aarch64_neon_saddlp:
@@ -6305,7 +6300,7 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     unsigned Opcode = IntNo == Intrinsic::aarch64_neon_uaddlp
                           ? AArch64ISD::UADDLP
                           : AArch64ISD::SADDLP;
-    return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1));
+    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1));
   }
   case Intrinsic::aarch64_neon_sdot:
   case Intrinsic::aarch64_neon_udot:
@@ -6315,12 +6310,12 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                        IntNo == Intrinsic::aarch64_sve_udot)
                           ? AArch64ISD::UDOT
                           : AArch64ISD::SDOT;
-    return DAG.getNode(Opcode, dl, Op.getValueType(), Op.getOperand(1),
+    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1),
                        Op.getOperand(2), Op.getOperand(3));
   }
   case Intrinsic::aarch64_neon_usdot:
   case Intrinsic::aarch64_sve_usdot: {
-    return DAG.getNode(AArch64ISD::USDOT, dl, Op.getValueType(),
+    return DAG.getNode(AArch64ISD::USDOT, DL, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
   }
   case Intrinsic::aarch64_neon_saddlv:
@@ -6337,10 +6332,10 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     SDValue ADDLV = DAG.getNode(
         IntNo == Intrinsic::aarch64_neon_uaddlv ? AArch64ISD::UADDLV
                                                 : AArch64ISD::SADDLV,
-        dl, ResVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64, Op.getOperand(1));
+        DL, ResVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64, Op.getOperand(1));
     SDValue EXTRACT_VEC_ELT = DAG.getNode(
-        ISD::EXTRACT_VECTOR_ELT, dl, ResVT == MVT::i32 ? MVT::i32 : MVT::i64,
-        ADDLV, DAG.getConstant(0, dl, MVT::i64));
+        ISD::EXTRACT_VECTOR_ELT, DL, ResVT == MVT::i32 ? MVT::i32 : MVT::i64,
+        ADDLV, DAG.getConstant(0, DL, MVT::i64));
     return EXTRACT_VEC_ELT;
   }
   case Intrinsic::experimental_cttz_elts: {
@@ -6352,13 +6347,13 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       // We can use SVE instructions to lower this intrinsic by first creating
       // an SVE predicate register mask from the fixed-width vector.
       EVT NewVT = getTypeToTransformTo(*DAG.getContext(), VT);
-      SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, NewVT, CttzOp);
+      SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, CttzOp);
       CttzOp = convertFixedMaskToScalableVector(Mask, DAG);
     }
 
     SDValue NewCttzElts =
-        DAG.getNode(AArch64ISD::CTTZ_ELTS, dl, MVT::i64, CttzOp);
-    return DAG.getZExtOrTrunc(NewCttzElts, dl, Op.getValueType());
+        DAG.getNode(AArch64ISD::CTTZ_ELTS, DL, MVT::i64, CttzOp);
+    return DAG.getZExtOrTrunc(NewCttzElts, DL, Op.getValueType());
   }
   case Intrinsic::experimental_vector_match: {
     return LowerVectorMatch(Op, DAG);
@@ -6705,7 +6700,7 @@ static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
 }
 
 static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   SDValue Src = Op.getOperand(0);
   MVT DestVT = Op.getSimpleValueType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -6721,15 +6716,15 @@ static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
   (void)TLI;
 
   if (SrcAS == ARM64AS::PTR32_SPTR) {
-    return DAG.getNode(ISD::SIGN_EXTEND, dl, DestVT, Src,
-                       DAG.getTargetConstant(0, dl, DestVT));
+    return DAG.getNode(ISD::SIGN_EXTEND, DL, DestVT, Src,
+                       DAG.getTargetConstant(0, DL, DestVT));
   } else if (SrcAS == ARM64AS::PTR32_UPTR) {
-    return DAG.getNode(ISD::ZERO_EXTEND, dl, DestVT, Src,
-                       DAG.getTargetConstant(0, dl, DestVT));
+    return DAG.getNode(ISD::ZERO_EXTEND, DL, DestVT, Src,
+                       DAG.getTargetConstant(0, DL, DestVT));
   } else if ((DestAS == ARM64AS::PTR32_SPTR) ||
              (DestAS == ARM64AS::PTR32_UPTR)) {
-    SDValue Ext = DAG.getAnyExtOrTrunc(Src, dl, DestVT);
-    SDValue Trunc = DAG.getZeroExtendInReg(Ext, dl, DestVT);
+    SDValue Ext = DAG.getAnyExtOrTrunc(Src, DL, DestVT);
+    SDValue Trunc = DAG.getZeroExtendInReg(Ext, DL, DestVT);
     return Trunc;
   } else {
     return Src;
@@ -7029,9 +7024,9 @@ static SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) {
 
   AArch64CC::CondCode CC;
   if (SDValue Cmp = emitConjunction(DAG, Cond, CC)) {
-    SDLoc dl(Op);
-    SDValue CCVal = DAG.getConstant(CC, dl, MVT::i32);
-    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
+    SDLoc DL(Op);
+    SDValue CCVal = DAG.getConstant(CC, DL, MVT::i32);
+    return DAG.getNode(AArch64ISD::BRCOND, DL, MVT::Other, Chain, Dest, CCVal,
                        Cmp);
   }
 
@@ -7161,40 +7156,40 @@ SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
 
   SDValue Addr = Trmp;
 
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   OutChains[0] = DAG.getStore(
-      Chain, dl, DAG.getConstant(0x58000080u | NestReg, dl, MVT::i32), Addr,
+      Chain, DL, DAG.getConstant(0x58000080u | NestReg, DL, MVT::i32), Addr,
       MachinePointerInfo(TrmpAddr));
 
-  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                     DAG.getConstant(4, dl, MVT::i64));
+  Addr = DAG.getNode(ISD::ADD, DL, MVT::i64, Trmp,
+                     DAG.getConstant(4, DL, MVT::i64));
   OutChains[1] = DAG.getStore(
-      Chain, dl, DAG.getConstant(0x580000b0u | FptrReg, dl, MVT::i32), Addr,
+      Chain, DL, DAG.getConstant(0x580000b0u | FptrReg, DL, MVT::i32), Addr,
       MachinePointerInfo(TrmpAddr, 4));
 
-  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                     DAG.getConstant(8, dl, MVT::i64));
+  Addr = DAG.getNode(ISD::ADD, DL, MVT::i64, Trmp,
+                     DAG.getConstant(8, DL, MVT::i64));
   OutChains[2] =
-      DAG.getStore(Chain, dl, DAG.getConstant(0xd61f0220u, dl, MVT::i32), Addr,
+      DAG.getStore(Chain, DL, DAG.getConstant(0xd61f0220u, DL, MVT::i32), Addr,
                    MachinePointerInfo(TrmpAddr, 8));
 
-  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                     DAG.getConstant(16, dl, MVT::i64));
+  Addr = DAG.getNode(ISD::ADD, DL, MVT::i64, Trmp,
+                     DAG.getConstant(16, DL, MVT::i64));
   OutChains[3] =
-      DAG.getStore(Chain, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 16));
+      DAG.getStore(Chain, DL, Nest, Addr, MachinePointerInfo(TrmpAddr, 16));
 
-  Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                     DAG.getConstant(24, dl, MVT::i64));
+  Addr = DAG.getNode(ISD::ADD, DL, MVT::i64, Trmp,
+                     DAG.getConstant(24, DL, MVT::i64));
   OutChains[4] =
-      DAG.getStore(Chain, dl, FPtr, Addr, MachinePointerInfo(TrmpAddr, 24));
+      DAG.getStore(Chain, DL, FPtr, Addr, MachinePointerInfo(TrmpAddr, 24));
 
-  SDValue StoreToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
+  SDValue StoreToken = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
 
-  SDValue EndOfTrmp = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
-                                  DAG.getConstant(12, dl, MVT::i64));
+  SDValue EndOfTrmp = DAG.getNode(ISD::ADD, DL, MVT::i64, Trmp,
+                                  DAG.getConstant(12, DL, MVT::i64));
 
   // Call clear cache on the trampoline instructions.
-  return DAG.getNode(ISD::CLEAR_CACHE, dl, MVT::Other, StoreToken, Trmp,
+  return DAG.getNode(ISD::CLEAR_CACHE, DL, MVT::Other, StoreToken, Trmp,
                      EndOfTrmp);
 }
 
@@ -10427,7 +10422,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue LHS = Op.getOperand(2);
   SDValue RHS = Op.getOperand(3);
   SDValue Dest = Op.getOperand(4);
-  SDLoc dl(Op);
+  SDLoc DL(Op);
 
   MachineFunction &MF = DAG.getMachineFunction();
   // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
@@ -10440,12 +10435,12 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   // value of a libcall against zero, which is just what the rest of LowerBR_CC
   // is expecting to deal with.
   if (LHS.getValueType() == MVT::f128) {
-    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
+    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, DL, LHS, RHS);
 
     // If softenSetCCOperands returned a scalar, we need to compare the result
     // against zero to select between true and false values.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, dl, LHS.getValueType());
+      RHS = DAG.getConstant(0, DL, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -10465,9 +10460,9 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
 
     if (CC == ISD::SETNE)
       OFCC = getInvertedCondCode(OFCC);
-    SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
+    SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
 
-    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
+    return DAG.getNode(AArch64ISD::BRCOND, DL, MVT::Other, Chain, Dest, CCVal,
                        Overflow);
   }
 
@@ -10489,12 +10484,12 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
           SDValue Test = LHS.getOperand(0);
           uint64_t Mask = LHS.getConstantOperandVal(1);
-          return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
-                             DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
+          return DAG.getNode(AArch64ISD::TBZ, DL, MVT::Other, Chain, Test,
+                             DAG.getConstant(Log2_64(Mask), DL, MVT::i64),
                              Dest);
         }
 
-        return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
+        return DAG.getNode(AArch64ISD::CBZ, DL, MVT::Other, Chain, LHS, Dest);
       } else if (CC == ISD::SETNE) {
         // See if we can use a TBZ to fold in an AND as well.
         // TBZ has a smaller branch displacement than CBZ.  If the offset is
@@ -10505,20 +10500,20 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
             isPowerOf2_64(LHS.getConstantOperandVal(1))) {
           SDValue Test = LHS.getOperand(0);
           uint64_t Mask = LHS.getConstantOperandVal(1);
-          return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
-                             DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
+          return DAG.getNode(AArch64ISD::TBNZ, DL, MVT::Other, Chain, Test,
+                             DAG.getConstant(Log2_64(Mask), DL, MVT::i64),
                              Dest);
         }
 
-        return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
+        return DAG.getNode(AArch64ISD::CBNZ, DL, MVT::Other, Chain, LHS, Dest);
       } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
         // Don't combine AND since emitComparison converts the AND to an ANDS
         // (a.k.a. TST) and the test in the test bit and branch instruction
         // becomes redundant.  This would also increase register pressure.
         uint64_t SignBitPos;
         std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
-        return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
-                           DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
+        return DAG.getNode(AArch64ISD::TBNZ, DL, MVT::Other, Chain, LHS,
+                           DAG.getConstant(SignBitPos, DL, MVT::i64), Dest);
       }
     }
     if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
@@ -10528,8 +10523,8 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
       // becomes redundant.  This would also increase register pressure.
       uint64_t SignBitPos;
       std::tie(LHS, SignBitPos) = lookThroughSignExtension(LHS);
-      return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
-                         DAG.getConstant(SignBitPos, dl, MVT::i64), Dest);
+      return DAG.getNode(AArch64ISD::TBZ, DL, MVT::Other, Chain, LHS,
+                         DAG.getConstant(SignBitPos, DL, MVT::i64), Dest);
     }
 
     // Try to emit Armv9.6 CB instructions. We prefer tb{n}z/cb{n}z due to their
@@ -10538,14 +10533,14 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
         AArch64CC::isValidCBCond(changeIntCCToAArch64CC(CC)) &&
         ProduceNonFlagSettingCondBr) {
       SDValue Cond =
-          DAG.getTargetConstant(changeIntCCToAArch64CC(CC), dl, MVT::i32);
-      return DAG.getNode(AArch64ISD::CB, dl, MVT::Other, Chain, Cond, LHS, RHS,
+          DAG.getTargetConstant(changeIntCCToAArch64CC(CC), DL, MVT::i32);
+      return DAG.getNode(AArch64ISD::CB, DL, MVT::Other, Chain, Cond, LHS, RHS,
                          Dest);
     }
 
     SDValue CCVal;
-    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
-    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
+    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, DL);
+    return DAG.getNode(AArch64ISD::BRCOND, DL, MVT::Other, Chain, Dest, CCVal,
                        Cmp);
   }
 
@@ -10554,15 +10549,15 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
 
   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
   // clean.  Some of them require two branches to implement.
-  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
+  SDValue Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
   AArch64CC::CondCode CC1, CC2;
   changeFPCCToAArch64CC(CC, CC1, CC2);
-  SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
+  SDValue CC1Val = DAG.getConstant(CC1, DL, MVT::i32);
   SDValue BR1 =
-      DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
+      DAG.getNode(AArch64ISD::BRCOND, DL, MVT::Other, Chain, Dest, CC1Val, Cmp);
   if (CC2 != AArch64CC::AL) {
-    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
-    return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
+    SDValue CC2Val = DAG.getConstant(CC2, DL, MVT::i32);
+    return DAG.getNode(AArch64ISD::BRCOND, DL, MVT::Other, BR1, Dest, CC2Val,
                        Cmp);
   }
 
@@ -10689,12 +10684,9 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
 
   EVT VT = Op.getValueType();
   if (VT.isScalableVector() ||
-      useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
+      useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::CTPOP_MERGE_PASSTHRU);
 
-  if (!Subtarget->isNeonAvailable())
-    return SDValue();
-
   bool IsParity = Op.getOpcode() == ISD::PARITY;
   SDValue Val = Op.getOperand(0);
   SDLoc DL(Op);
@@ -10704,6 +10696,36 @@ SDValue AArch64TargetLowering::LowerCTPOP_PARITY(SDValue Op,
   if (VT == MVT::i32 && IsParity)
     return SDValue();
 
+  if (Subtarget->isSVEorStreamingSVEAvailable()) {
+    if (VT == MVT::i32 || VT == MVT::i64) {
+      EVT ContainerVT = VT == MVT::i32 ? MVT::nxv4i32 : MVT::nxv2i64;
+      Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT,
+                        DAG.getUNDEF(ContainerVT), Val,
+                        DAG.getVectorIdxConstant(0, DL));
+      Val = DAG.getNode(ISD::CTPOP, DL, ContainerVT, Val);
+      Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Val,
+                        DAG.getVectorIdxConstant(0, DL));
+      if (IsParity)
+        Val = DAG.getNode(ISD::AND, DL, VT, Val, DAG.getConstant(1, DL, VT));
+      return Val;
+    }
+
+    if (VT == MVT::i128) {
+      Val = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Val);
+      Val = convertToScalableVector(DAG, MVT::nxv2i64, Val);
+      Val = DAG.getNode(ISD::CTPOP, DL, MVT::nxv2i64, Val);
+      Val = convertFromScalableVector(DAG, MVT::v2i64, Val);
+      Val = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i64, Val);
+      Val = DAG.getZExtOrTrunc(Val, DL, VT);
+      if (IsParity)
+        Val = DAG.getNode(ISD::AND, DL, VT, Val, DAG.getConstant(1, DL, VT));
+      return Val;
+    }
+  }
+
+  if (!Subtarget->isNeonAvailable())
+    return SDValue();
+
   // If there is no CNT instruction available, GPR popcount can
   // be more efficiently lowered to the following sequence that uses
   // AdvSIMD registers/instructions as long as the copies to/from
@@ -10969,40 +10991,40 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   SDValue LHS = Op.getOperand(OpNo + 0);
   SDValue RHS = Op.getOperand(OpNo + 1);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(OpNo + 2))->get();
-  SDLoc dl(Op);
+  SDLoc DL(Op);
 
   // We chose ZeroOrOneBooleanContents, so use zero and one.
   EVT VT = Op.getValueType();
-  SDValue TVal = DAG.getConstant(1, dl, VT);
-  SDValue FVal = DAG.getConstant(0, dl, VT);
+  SDValue TVal = DAG.getConstant(1, DL, VT);
+  SDValue FVal = DAG.getConstant(0, DL, VT);
 
   // Handle f128 first, since one possible outcome is a normal integer
   // comparison which gets picked up by the next if statement.
   if (LHS.getValueType() == MVT::f128) {
-    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS, Chain,
+    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, DL, LHS, RHS, Chain,
                         IsSignaling);
 
     // If softenSetCCOperands returned a scalar, use it.
     if (!RHS.getNode()) {
       assert(LHS.getValueType() == Op.getValueType() &&
              "Unexpected setcc expansion!");
-      return IsStrict ? DAG.getMergeValues({LHS, Chain}, dl) : LHS;
+      return IsStrict ? DAG.getMergeValues({LHS, Chain}, DL) : LHS;
     }
   }
 
   if (LHS.getValueType().isInteger()) {
 
-    simplifySetCCIntoEq(CC, LHS, RHS, DAG, dl);
+    simplifySetCCIntoEq(CC, LHS, RHS, DAG, DL);
 
     SDValue CCVal;
     SDValue Cmp = getAArch64Cmp(
-        LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, dl);
+        LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, DL);
 
     // Note that we inverted the condition above, so we reverse the order of
     // the true and false operands here.  This will allow the setcc to be
     // matched to a single CSINC instruction.
-    SDValue Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
-    return IsStrict ? DAG.getMergeValues({Res, Chain}, dl) : Res;
+    SDValue Res = DAG.getNode(AArch64ISD::CSEL, DL, VT, FVal, TVal, CCVal, Cmp);
+    return IsStrict ? DAG.getMergeValues({Res, Chain}, DL) : Res;
   }
 
   // Now we know we're dealing with FP values.
@@ -11013,9 +11035,9 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   // and do the comparison.
   SDValue Cmp;
   if (IsStrict)
-    Cmp = emitStrictFPComparison(LHS, RHS, dl, DAG, Chain, IsSignaling);
+    Cmp = emitStrictFPComparison(LHS, RHS, DL, DAG, Chain, IsSignaling);
   else
-    Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
+    Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
 
   AArch64CC::CondCode CC1, CC2;
   changeFPCCToAArch64CC(CC, CC1, CC2);
@@ -11023,12 +11045,12 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
   if (CC2 == AArch64CC::AL) {
     changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, LHS.getValueType()), CC1,
                           CC2);
-    SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
+    SDValue CC1Val = DAG.getConstant(CC1, DL, MVT::i32);
 
     // Note that we inverted the condition above, so we reverse the order of
     // the true and false operands here.  This will allow the setcc to be
     // matched to a single CSINC instruction.
-    Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
+    Res = DAG.getNode(AArch64ISD::CSEL, DL, VT, FVal, TVal, CC1Val, Cmp);
   } else {
     // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
     // totally clean.  Some of them require two CSELs to implement.  As is in
@@ -11036,14 +11058,14 @@ SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
     // of the first as the RHS.  We're effectively OR'ing the two CC's together.
 
     // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
-    SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
+    SDValue CC1Val = DAG.getConstant(CC1, DL, MVT::i32);
     SDValue CS1 =
-        DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
+        DAG.getNode(AArch64ISD::CSEL, DL, VT, TVal, FVal, CC1Val, Cmp);
 
-    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
-    Res = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
+    SDValue CC2Val = DAG.getConstant(CC2, DL, MVT::i32);
+    Res = DAG.getNode(AArch64ISD::CSEL, DL, VT, TVal, CS1, CC2Val, Cmp);
   }
-  return IsStrict ? DAG.getMergeValues({Res, Cmp.getValue(1)}, dl) : Res;
+  return IsStrict ? DAG.getMergeValues({Res, Cmp.getValue(1)}, DL) : Res;
 }
 
 SDValue AArch64TargetLowering::LowerSETCCCARRY(SDValue Op,
@@ -11194,16 +11216,16 @@ static SDValue emitFloatCompareMask(SDValue LHS, SDValue RHS, SDValue TVal,
 SDValue AArch64TargetLowering::LowerSELECT_CC(
     ISD::CondCode CC, SDValue LHS, SDValue RHS, SDValue TVal, SDValue FVal,
     iterator_range<SDNode::user_iterator> Users, bool HasNoNaNs,
-    const SDLoc &dl, SelectionDAG &DAG) const {
+    const SDLoc &DL, SelectionDAG &DAG) const {
   // Handle f128 first, because it will result in a comparison of some RTLIB
   // call result against zero.
   if (LHS.getValueType() == MVT::f128) {
-    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
+    softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, DL, LHS, RHS);
 
     // If softenSetCCOperands returned a scalar, we need to compare the result
     // against zero to select between true and false values.
     if (!RHS.getNode()) {
-      RHS = DAG.getConstant(0, dl, LHS.getValueType());
+      RHS = DAG.getConstant(0, DL, LHS.getValueType());
       CC = ISD::SETNE;
     }
   }
@@ -11211,8 +11233,8 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
   // Also handle f16, for which we need to do a f32 comparison.
   if ((LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
       LHS.getValueType() == MVT::bf16) {
-    LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
-    RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
+    LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
+    RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
   }
 
   // Next, handle integers.
@@ -11231,9 +11253,9 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
         LHS.getValueType() == TVal.getValueType()) {
       EVT VT = LHS.getValueType();
       SDValue Shift =
-          DAG.getNode(ISD::SRA, dl, VT, LHS,
-                      DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
-      return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
+          DAG.getNode(ISD::SRA, DL, VT, LHS,
+                      DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
+      return DAG.getNode(ISD::OR, DL, VT, Shift, DAG.getConstant(1, DL, VT));
     }
 
     // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
@@ -11245,13 +11267,13 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
         LHS.getValueType() == RHS.getValueType()) {
       EVT VT = LHS.getValueType();
       SDValue Shift =
-          DAG.getNode(ISD::SRA, dl, VT, LHS,
-                      DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
+          DAG.getNode(ISD::SRA, DL, VT, LHS,
+                      DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
 
       if (CC == ISD::SETGT)
-        Shift = DAG.getNOT(dl, Shift, VT);
+        Shift = DAG.getNOT(DL, Shift, VT);
 
-      return DAG.getNode(ISD::AND, dl, VT, LHS, Shift);
+      return DAG.getNode(ISD::AND, DL, VT, LHS, Shift);
     }
 
     unsigned Opcode = AArch64ISD::CSEL;
@@ -11362,14 +11384,14 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
       if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
         Opcode = AArch64ISD::CSINV;
         TVal = LHS;
-        FVal = DAG.getConstant(0, dl, FVal.getValueType());
+        FVal = DAG.getConstant(0, DL, FVal.getValueType());
       }
     }
 
     SDValue CCVal;
-    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
+    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, DL);
     EVT VT = TVal.getValueType();
-    return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
+    return DAG.getNode(Opcode, DL, VT, TVal, FVal, CCVal, Cmp);
   }
 
   // Now we know we're dealing with FP values.
@@ -11393,12 +11415,12 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
       })) {
     bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath || HasNoNaNs;
     SDValue VectorCmp =
-        emitFloatCompareMask(LHS, RHS, TVal, FVal, CC, NoNaNs, dl, DAG);
+        emitFloatCompareMask(LHS, RHS, TVal, FVal, CC, NoNaNs, DL, DAG);
     if (VectorCmp)
       return VectorCmp;
   }
 
-  SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
+  SDValue Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
 
   // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
   // clean.  Some of them require two CSELs to implement.
@@ -11424,14 +11446,14 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(
   }
 
   // Emit first, and possibly only, CSEL.
-  SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
-  SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
+  SDValue CC1Val = DAG.getConstant(CC1, DL, MVT::i32);
+  SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, DL, VT, TVal, FVal, CC1Val, Cmp);
 
   // If we need a second CSEL, emit it, using the output of the first as the
   // RHS.  We're effectively OR'ing the two CC's together.
   if (CC2 != AArch64CC::AL) {
-    SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
-    return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
+    SDValue CC2Val = DAG.getConstant(CC2, DL, MVT::i32);
+    return DAG.getNode(AArch64ISD::CSEL, DL, VT, TVal, CS1, CC2Val, Cmp);
   }
 
   // Otherwise, return the output of the first CSEL.
@@ -12782,7 +12804,7 @@ static unsigned getExtFactor(SDValue &V) {
 // within range. This can be reconstructed using AND and NEON's TBL1.
 SDValue ReconstructShuffleWithRuntimeMask(SDValue Op, SelectionDAG &DAG) {
   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   assert(!VT.isScalableVector() &&
          "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
@@ -12855,17 +12877,17 @@ SDValue ReconstructShuffleWithRuntimeMask(SDValue Op, SelectionDAG &DAG) {
   // insert, we know that the index in the mask must be smaller than the number
   // of elements in the source, or we would have an out-of-bounds access.
   if (NumElts == 8)
-    SourceVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, SourceVec,
+    SourceVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, SourceVec,
                             DAG.getUNDEF(VT));
 
   // Preconditions met, so we can use a vector (AND +) TBL to build this vector.
   if (!AndMaskConstants.empty())
-    MaskSourceVec = DAG.getNode(ISD::AND, dl, VT, MaskSourceVec,
-                                DAG.getBuildVector(VT, dl, AndMaskConstants));
+    MaskSourceVec = DAG.getNode(ISD::AND, DL, VT, MaskSourceVec,
+                                DAG.getBuildVector(VT, DL, AndMaskConstants));
 
   return DAG.getNode(
-      ISD::INTRINSIC_WO_CHAIN, dl, VT,
-      DAG.getConstant(Intrinsic::aarch64_neon_tbl1, dl, MVT::i32), SourceVec,
+      ISD::INTRINSIC_WO_CHAIN, DL, VT,
+      DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), SourceVec,
       MaskSourceVec);
 }
 
@@ -12875,7 +12897,7 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
                                                   SelectionDAG &DAG) const {
   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
   LLVM_DEBUG(dbgs() << "AArch64TargetLowering::ReconstructShuffle\n");
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   assert(!VT.isScalableVector() &&
          "Scalable vectors cannot be used with ISD::BUILD_VECTOR");
@@ -12968,7 +12990,7 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
     TBLOperands.push_back(DAG.getConstant(Sources.size() == 3
                                               ? Intrinsic::aarch64_neon_tbl3
                                               : Intrinsic::aarch64_neon_tbl4,
-                                          dl, MVT::i32));
+                                          DL, MVT::i32));
     for (unsigned i = 0; i < Sources.size(); i++) {
       SDValue Src = Sources[i].Vec;
       EVT SrcVT = Src.getValueType();
@@ -12976,21 +12998,21 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
       assert((SrcVT.is64BitVector() || SrcVT.is128BitVector()) &&
              "Expected a legally typed vector");
       if (SrcVT.is64BitVector())
-        Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, Src,
+        Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Src,
                           DAG.getUNDEF(MVT::v8i8));
       TBLOperands.push_back(Src);
     }
 
     SmallVector<SDValue, 16> TBLMask;
     for (unsigned i = 0; i < Mask.size(); i++)
-      TBLMask.push_back(DAG.getConstant(Mask[i], dl, MVT::i32));
+      TBLMask.push_back(DAG.getConstant(Mask[i], DL, MVT::i32));
     assert((Mask.size() == 8 || Mask.size() == 16) &&
            "Expected a v8i8 or v16i8 Mask");
-    TBLOperands.push_back(
-        DAG.getBuildVector(Mask.size() == 8 ? MVT::v8i8 : MVT::v16i8, dl, TBLMask));
+    TBLOperands.push_back(DAG.getBuildVector(
+        Mask.size() == 8 ? MVT::v8i8 : MVT::v16i8, DL, TBLMask));
 
     SDValue Shuffle =
-        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
+        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL,
                     Mask.size() == 8 ? MVT::v8i8 : MVT::v16i8, TBLOperands);
     return DAG.getBitcast(VT, Shuffle);
   }
@@ -13038,7 +13060,7 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
       // We can pad out the smaller vector for free, so if it's part of a
       // shuffle...
       Src.ShuffleVec =
-          DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
+          DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Src.ShuffleVec,
                       DAG.getUNDEF(Src.ShuffleVec.getValueType()));
       continue;
     }
@@ -13058,22 +13080,22 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
     if (Src.MinElt >= NumSrcElts) {
       // The extraction can just take the second half
       Src.ShuffleVec =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(NumSrcElts, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, DestVT, Src.ShuffleVec,
+                      DAG.getConstant(NumSrcElts, DL, MVT::i64));
       Src.WindowBase = -NumSrcElts;
     } else if (Src.MaxElt < NumSrcElts) {
       // The extraction can just take the first half
       Src.ShuffleVec =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(0, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, DestVT, Src.ShuffleVec,
+                      DAG.getConstant(0, DL, MVT::i64));
     } else {
       // An actual VEXT is needed
       SDValue VEXTSrc1 =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(0, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, DestVT, Src.ShuffleVec,
+                      DAG.getConstant(0, DL, MVT::i64));
       SDValue VEXTSrc2 =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
-                      DAG.getConstant(NumSrcElts, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, DestVT, Src.ShuffleVec,
+                      DAG.getConstant(NumSrcElts, DL, MVT::i64));
       unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1);
 
       if (!SrcVT.is64BitVector()) {
@@ -13083,9 +13105,9 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
         return SDValue();
       }
 
-      Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1,
-                                   VEXTSrc2,
-                                   DAG.getConstant(Imm, dl, MVT::i32));
+      Src.ShuffleVec =
+          DAG.getNode(AArch64ISD::EXT, DL, DestVT, VEXTSrc1, VEXTSrc2,
+                      DAG.getConstant(Imm, DL, MVT::i32));
       Src.WindowBase = -Src.MinElt;
     }
   }
@@ -13100,9 +13122,9 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
     assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
     if (DAG.getDataLayout().isBigEndian()) {
       Src.ShuffleVec =
-          DAG.getNode(AArch64ISD::NVCAST, dl, ShuffleVT, Src.ShuffleVec);
+          DAG.getNode(AArch64ISD::NVCAST, DL, ShuffleVT, Src.ShuffleVec);
     } else {
-      Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
+      Src.ShuffleVec = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Src.ShuffleVec);
     }
     Src.WindowScale =
         SrcEltTy.getFixedSizeInBits() / SmallestEltTy.getFixedSizeInBits();
@@ -13154,13 +13176,13 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
   for (unsigned i = 0; i < Sources.size(); ++i)
     ShuffleOps[i] = Sources[i].ShuffleVec;
 
-  SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
-                                         ShuffleOps[1], Mask);
+  SDValue Shuffle =
+      DAG.getVectorShuffle(ShuffleVT, DL, ShuffleOps[0], ShuffleOps[1], Mask);
   SDValue V;
   if (DAG.getDataLayout().isBigEndian()) {
-    V = DAG.getNode(AArch64ISD::NVCAST, dl, VT, Shuffle);
+    V = DAG.getNode(AArch64ISD::NVCAST, DL, VT, Shuffle);
   } else {
-    V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
+    V = DAG.getNode(ISD::BITCAST, DL, VT, Shuffle);
   }
 
   LLVM_DEBUG(dbgs() << "Reshuffle, creating node: "; Shuffle.dump();
@@ -13405,30 +13427,6 @@ static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   return true;
 }
 
-/// isDUPQMask - matches a splat of equivalent lanes within 128b segments in
-/// the first vector operand.
-static std::optional<unsigned> isDUPQMask(ArrayRef<int> M, EVT VT) {
-  assert(VT.getFixedSizeInBits() % 128 == 0 && "Unsupported SVE vector size");
-  unsigned Lane = (unsigned)M[0];
-  unsigned Segments = VT.getFixedSizeInBits() / 128;
-  unsigned SegmentElts = VT.getVectorNumElements() / Segments;
-
-  // Make sure there's no size changes.
-  if (SegmentElts * Segments != M.size())
-    return std::nullopt;
-
-  // Check the first index corresponds to one of the lanes in the first segment.
-  if (Lane >= SegmentElts)
-    return std::nullopt;
-
-  // Check that all lanes match the first, adjusted for segment.
-  for (unsigned I = 0; I < M.size(); ++I)
-    if ((unsigned)M[I] != (Lane + ((I / SegmentElts) * SegmentElts)))
-      return std::nullopt;
-
-  return Lane;
-}
-
 /// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
@@ -13537,10 +13535,10 @@ static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
 //ID, V1 and V2 are the original shuffle inputs. PFEntry is the Perfect shuffle
 //table entry and LHS/RHS are the immediate inputs for this stage of the
 //shuffle.
-static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
-                                      SDValue V2, unsigned PFEntry, SDValue LHS,
+static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1, SDValue V2,
+                                      unsigned PFEntry, SDValue LHS,
                                       SDValue RHS, SelectionDAG &DAG,
-                                      const SDLoc &dl) {
+                                      const SDLoc &DL) {
   unsigned OpNum = (PFEntry >> 26) & 0x0F;
   unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1);
   unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1);
@@ -13587,7 +13585,7 @@ static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
     // get the lane to move from the PFID, which is always from the
     // original vectors (V1 or V2).
     SDValue OpLHS = GeneratePerfectShuffle(
-        LHSID, V1, V2, PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
+        LHSID, V1, V2, PerfectShuffleTable[LHSID], LHS, RHS, DAG, DL);
     EVT VT = OpLHS.getValueType();
     assert(RHSID < 8 && "Expected a lane index for RHSID!");
     unsigned ExtLane = 0;
@@ -13622,20 +13620,20 @@ static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
         OpLHS = DAG.getBitcast(MVT::v4f16, OpLHS);
       }
     }
-    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
+    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
                               Input.getValueType().getVectorElementType(),
-                              Input, DAG.getVectorIdxConstant(ExtLane, dl));
+                              Input, DAG.getVectorIdxConstant(ExtLane, DL));
     SDValue Ins =
-        DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Input.getValueType(), OpLHS,
-                    Ext, DAG.getVectorIdxConstant(RHSID & 0x3, dl));
+        DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, Input.getValueType(), OpLHS,
+                    Ext, DAG.getVectorIdxConstant(RHSID & 0x3, DL));
     return DAG.getBitcast(VT, Ins);
   }
 
   SDValue OpLHS, OpRHS;
   OpLHS = GeneratePerfectShuffle(LHSID, V1, V2, PerfectShuffleTable[LHSID], LHS,
-                                 RHS, DAG, dl);
+                                 RHS, DAG, DL);
   OpRHS = GeneratePerfectShuffle(RHSID, V1, V2, PerfectShuffleTable[RHSID], LHS,
-                                 RHS, DAG, dl);
+                                 RHS, DAG, DL);
   EVT VT = OpLHS.getValueType();
 
   switch (OpNum) {
@@ -13645,15 +13643,15 @@ static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
     // VREV divides the vector in half and swaps within the half.
     if (VT.getVectorElementType() == MVT::i32 ||
         VT.getVectorElementType() == MVT::f32)
-      return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
+      return DAG.getNode(AArch64ISD::REV64, DL, VT, OpLHS);
     // vrev <4 x i16> -> REV32
     if (VT.getVectorElementType() == MVT::i16 ||
         VT.getVectorElementType() == MVT::f16 ||
         VT.getVectorElementType() == MVT::bf16)
-      return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
+      return DAG.getNode(AArch64ISD::REV32, DL, VT, OpLHS);
     // vrev <4 x i8> -> REV16
     assert(VT.getVectorElementType() == MVT::i8);
-    return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
+    return DAG.getNode(AArch64ISD::REV16, DL, VT, OpLHS);
   case OP_VDUP0:
   case OP_VDUP1:
   case OP_VDUP2:
@@ -13673,28 +13671,28 @@ static SDValue GeneratePerfectShuffle(unsigned ID, SDValue V1,
 
     if (VT.getSizeInBits() == 64)
       OpLHS = WidenVector(OpLHS, DAG);
-    SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
-    return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
+    SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, DL, MVT::i64);
+    return DAG.getNode(Opcode, DL, VT, OpLHS, Lane);
   }
   case OP_VEXT1:
   case OP_VEXT2:
   case OP_VEXT3: {
     unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS);
-    return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
-                       DAG.getConstant(Imm, dl, MVT::i32));
+    return DAG.getNode(AArch64ISD::EXT, DL, VT, OpLHS, OpRHS,
+                       DAG.getConstant(Imm, DL, MVT::i32));
   }
   case OP_VUZPL:
-    return DAG.getNode(AArch64ISD::UZP1, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::UZP1, DL, VT, OpLHS, OpRHS);
   case OP_VUZPR:
-    return DAG.getNode(AArch64ISD::UZP2, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::UZP2, DL, VT, OpLHS, OpRHS);
   case OP_VZIPL:
-    return DAG.getNode(AArch64ISD::ZIP1, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::ZIP1, DL, VT, OpLHS, OpRHS);
   case OP_VZIPR:
-    return DAG.getNode(AArch64ISD::ZIP2, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::ZIP2, DL, VT, OpLHS, OpRHS);
   case OP_VTRNL:
-    return DAG.getNode(AArch64ISD::TRN1, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::TRN1, DL, VT, OpLHS, OpRHS);
   case OP_VTRNR:
-    return DAG.getNode(AArch64ISD::TRN2, dl, VT, OpLHS, OpRHS);
+    return DAG.getNode(AArch64ISD::TRN2, DL, VT, OpLHS, OpRHS);
   }
 }
 
@@ -13785,7 +13783,7 @@ static unsigned getDUPLANEOp(EVT EltType) {
   llvm_unreachable("Invalid vector element type?");
 }
 
-static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
+static SDValue constructDup(SDValue V, int Lane, SDLoc DL, EVT VT,
                             unsigned Opcode, SelectionDAG &DAG) {
   // Try to eliminate a bitcasted extract subvector before a DUPLANE.
   auto getScaledOffsetDup = [](SDValue BitCast, int &LaneC, MVT &CastVT) {
@@ -13841,7 +13839,7 @@ static SDValue constructDup(SDValue V, int Lane, SDLoc dl, EVT VT,
     // Widen the operand to 128-bit register with undef.
     V = WidenVector(V, DAG);
   }
-  return DAG.getNode(Opcode, dl, VT, V, DAG.getConstant(Lane, dl, MVT::i64));
+  return DAG.getNode(Opcode, DL, VT, V, DAG.getConstant(Lane, DL, MVT::i64));
 }
 
 // Try to widen element type to get a new mask value for a better permutation
@@ -13892,9 +13890,9 @@ static SDValue tryToConvertShuffleOfTbl2ToTbl4(SDValue Op,
                                                SelectionDAG &DAG) {
   SDValue Tbl1 = Op->getOperand(0);
   SDValue Tbl2 = Op->getOperand(1);
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   SDValue Tbl2ID =
-      DAG.getTargetConstant(Intrinsic::aarch64_neon_tbl2, dl, MVT::i64);
+      DAG.getTargetConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i64);
 
   EVT VT = Op.getValueType();
   if (Tbl1.getOpcode() != ISD::INTRINSIC_WO_CHAIN ||
@@ -13920,15 +13918,15 @@ static SDValue tryToConvertShuffleOfTbl2ToTbl4(SDValue Op,
       auto *C = dyn_cast<ConstantSDNode>(Mask2.getOperand(ShuffleMask[I] - 16));
       if (!C)
         return SDValue();
-      TBLMaskParts[I] = DAG.getConstant(C->getSExtValue() + 32, dl, MVT::i32);
+      TBLMaskParts[I] = DAG.getConstant(C->getSExtValue() + 32, DL, MVT::i32);
     }
   }
 
-  SDValue TBLMask = DAG.getBuildVector(VT, dl, TBLMaskParts);
+  SDValue TBLMask = DAG.getBuildVector(VT, DL, TBLMaskParts);
   SDValue ID =
-      DAG.getTargetConstant(Intrinsic::aarch64_neon_tbl4, dl, MVT::i64);
+      DAG.getTargetConstant(Intrinsic::aarch64_neon_tbl4, DL, MVT::i64);
 
-  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v16i8,
+  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::v16i8,
                      {ID, Tbl1->getOperand(1), Tbl1->getOperand(2),
                       Tbl2->getOperand(1), Tbl2->getOperand(2), TBLMask});
 }
@@ -13939,7 +13937,7 @@ static SDValue tryToConvertShuffleOfTbl2ToTbl4(SDValue Op,
 SDValue
 AArch64TargetLowering::LowerZERO_EXTEND_VECTOR_INREG(SDValue Op,
                                                      SelectionDAG &DAG) const {
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   SDValue SrcOp = Op.getOperand(0);
   EVT SrcVT = SrcOp.getValueType();
@@ -13949,14 +13947,14 @@ AArch64TargetLowering::LowerZERO_EXTEND_VECTOR_INREG(SDValue Op,
   // FIXME: support multi-step zipping?
   if (Scale != 2)
     return SDValue();
-  SDValue Zeros = DAG.getConstant(0, dl, SrcVT);
+  SDValue Zeros = DAG.getConstant(0, DL, SrcVT);
   return DAG.getBitcast(VT,
-                        DAG.getNode(AArch64ISD::ZIP1, dl, SrcVT, SrcOp, Zeros));
+                        DAG.getNode(AArch64ISD::ZIP1, DL, SrcVT, SrcOp, Zeros));
 }
 
 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                                                    SelectionDAG &DAG) const {
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
@@ -13987,17 +13985,17 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       Lane = 0;
 
     if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR)
-      return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(),
+      return DAG.getNode(AArch64ISD::DUP, DL, V1.getValueType(),
                          V1.getOperand(0));
     // Test if V1 is a BUILD_VECTOR and the lane being referenced is a non-
     // constant. If so, we can just reference the lane's definition directly.
     if (V1.getOpcode() == ISD::BUILD_VECTOR &&
         !isa<ConstantSDNode>(V1.getOperand(Lane)))
-      return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
+      return DAG.getNode(AArch64ISD::DUP, DL, VT, V1.getOperand(Lane));
 
     // Otherwise, duplicate from the lane of the input vector.
     unsigned Opcode = getDUPLANEOp(V1.getValueType().getVectorElementType());
-    return constructDup(V1, Lane, dl, VT, Opcode, DAG);
+    return constructDup(V1, Lane, DL, VT, Opcode, DAG);
   }
 
   // Check if the mask matches a DUP for a wider element
@@ -14013,7 +14011,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       MVT NewVecTy = MVT::getVectorVT(NewEltTy, NewEltCount);
       V1 = DAG.getBitcast(NewVecTy, V1);
       // Construct the DUP instruction
-      V1 = constructDup(V1, Lane, dl, NewVecTy, Opcode, DAG);
+      V1 = constructDup(V1, Lane, DL, NewVecTy, Opcode, DAG);
       // Cast back to the original type
       return DAG.getBitcast(VT, V1);
     }
@@ -14022,17 +14020,17 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   unsigned NumElts = VT.getVectorNumElements();
   unsigned EltSize = VT.getScalarSizeInBits();
   if (isREVMask(ShuffleMask, EltSize, NumElts, 64))
-    return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1);
+    return DAG.getNode(AArch64ISD::REV64, DL, V1.getValueType(), V1);
   if (isREVMask(ShuffleMask, EltSize, NumElts, 32))
-    return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1);
+    return DAG.getNode(AArch64ISD::REV32, DL, V1.getValueType(), V1);
   if (isREVMask(ShuffleMask, EltSize, NumElts, 16))
-    return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1);
+    return DAG.getNode(AArch64ISD::REV16, DL, V1.getValueType(), V1);
 
   if (((NumElts == 8 && EltSize == 16) || (NumElts == 16 && EltSize == 8)) &&
       ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) {
-    SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1);
-    return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev,
-                       DAG.getConstant(8, dl, MVT::i32));
+    SDValue Rev = DAG.getNode(AArch64ISD::REV64, DL, VT, V1);
+    return DAG.getNode(AArch64ISD::EXT, DL, VT, Rev, Rev,
+                       DAG.getConstant(8, DL, MVT::i32));
   }
 
   bool ReverseEXT = false;
@@ -14041,39 +14039,39 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     if (ReverseEXT)
       std::swap(V1, V2);
     Imm *= getExtFactor(V1);
-    return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2,
-                       DAG.getConstant(Imm, dl, MVT::i32));
+    return DAG.getNode(AArch64ISD::EXT, DL, V1.getValueType(), V1, V2,
+                       DAG.getConstant(Imm, DL, MVT::i32));
   } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
     Imm *= getExtFactor(V1);
-    return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1,
-                       DAG.getConstant(Imm, dl, MVT::i32));
+    return DAG.getNode(AArch64ISD::EXT, DL, V1.getValueType(), V1, V1,
+                       DAG.getConstant(Imm, DL, MVT::i32));
   }
 
   unsigned WhichResult;
   if (isZIPMask(ShuffleMask, NumElts, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V2);
   }
   if (isUZPMask(ShuffleMask, NumElts, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V2);
   }
   if (isTRNMask(ShuffleMask, NumElts, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V2);
   }
 
   if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
   }
   if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
   }
   if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
-    return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
+    return DAG.getNode(Opc, DL, V1.getValueType(), V1, V1);
   }
 
   if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
@@ -14084,7 +14082,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   int NumInputElements = V1.getValueType().getVectorNumElements();
   if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
     SDValue DstVec = DstIsLeft ? V1 : V2;
-    SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
+    SDValue DstLaneV = DAG.getConstant(Anomaly, DL, MVT::i64);
 
     SDValue SrcVec = V1;
     int SrcLane = ShuffleMask[Anomaly];
@@ -14092,7 +14090,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       SrcVec = V2;
       SrcLane -= NumElts;
     }
-    SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
+    SDValue SrcLaneV = DAG.getConstant(SrcLane, DL, MVT::i64);
 
     EVT ScalarVT = VT.getVectorElementType();
 
@@ -14100,8 +14098,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
       ScalarVT = MVT::i32;
 
     return DAG.getNode(
-        ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
-        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV),
+        ISD::INSERT_VECTOR_ELT, DL, VT, DstVec,
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SrcVec, SrcLaneV),
         DstLaneV);
   }
 
@@ -14124,7 +14122,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                             PFIndexes[2] * 9 + PFIndexes[3];
     unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
     return GeneratePerfectShuffle(PFTableIndex, V1, V2, PFEntry, V1, V2, DAG,
-                                  dl);
+                                  DL);
   }
 
   // Check for a "select shuffle", generating a BSL to pick between lanes in
@@ -14135,10 +14133,10 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
     SmallVector<SDValue> MaskElts;
     for (int M : ShuffleMask)
       MaskElts.push_back(DAG.getConstant(
-          M >= static_cast<int>(NumElts) ? 0 : 0xffffffff, dl, MVT::i32));
+          M >= static_cast<int>(NumElts) ? 0 : 0xffffffff, DL, MVT::i32));
     EVT IVT = VT.changeVectorElementTypeToInteger();
-    SDValue MaskConst = DAG.getBuildVector(IVT, dl, MaskElts);
-    return DAG.getBitcast(VT, DAG.getNode(AArch64ISD::BSP, dl, IVT, MaskConst,
+    SDValue MaskConst = DAG.getBuildVector(IVT, DL, MaskElts);
+    return DAG.getBitcast(VT, DAG.getNode(AArch64ISD::BSP, DL, IVT, MaskConst,
                                           DAG.getBitcast(IVT, V1),
                                           DAG.getBitcast(IVT, V2)));
   }
@@ -14257,10 +14255,10 @@ static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
     if (AArch64_AM::isAdvSIMDModImmType10(Value)) {
       Value = AArch64_AM::encodeAdvSIMDModImmType10(Value);
 
-      SDLoc dl(Op);
-      SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
-                                DAG.getConstant(Value, dl, MVT::i32));
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      SDLoc DL(Op);
+      SDValue Mov =
+          DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32));
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14300,20 +14298,20 @@ static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
     }
 
     if (isAdvSIMDModImm) {
-      SDLoc dl(Op);
+      SDLoc DL(Op);
       SDValue Mov;
 
       if (LHS)
-        Mov = DAG.getNode(NewOp, dl, MovTy,
-                          DAG.getNode(AArch64ISD::NVCAST, dl, MovTy, *LHS),
-                          DAG.getConstant(Value, dl, MVT::i32),
-                          DAG.getConstant(Shift, dl, MVT::i32));
+        Mov = DAG.getNode(NewOp, DL, MovTy,
+                          DAG.getNode(AArch64ISD::NVCAST, DL, MovTy, *LHS),
+                          DAG.getConstant(Value, DL, MVT::i32),
+                          DAG.getConstant(Shift, DL, MVT::i32));
       else
-        Mov = DAG.getNode(NewOp, dl, MovTy,
-                          DAG.getConstant(Value, dl, MVT::i32),
-                          DAG.getConstant(Shift, dl, MVT::i32));
+        Mov =
+            DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32),
+                        DAG.getConstant(Shift, DL, MVT::i32));
 
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14345,20 +14343,20 @@ static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
     }
 
     if (isAdvSIMDModImm) {
-      SDLoc dl(Op);
+      SDLoc DL(Op);
       SDValue Mov;
 
       if (LHS)
-        Mov = DAG.getNode(NewOp, dl, MovTy,
-                          DAG.getNode(AArch64ISD::NVCAST, dl, MovTy, *LHS),
-                          DAG.getConstant(Value, dl, MVT::i32),
-                          DAG.getConstant(Shift, dl, MVT::i32));
+        Mov = DAG.getNode(NewOp, DL, MovTy,
+                          DAG.getNode(AArch64ISD::NVCAST, DL, MovTy, *LHS),
+                          DAG.getConstant(Value, DL, MVT::i32),
+                          DAG.getConstant(Shift, DL, MVT::i32));
       else
-        Mov = DAG.getNode(NewOp, dl, MovTy,
-                          DAG.getConstant(Value, dl, MVT::i32),
-                          DAG.getConstant(Shift, dl, MVT::i32));
+        Mov =
+            DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32),
+                        DAG.getConstant(Shift, DL, MVT::i32));
 
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14385,11 +14383,11 @@ static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
     }
 
     if (isAdvSIMDModImm) {
-      SDLoc dl(Op);
-      SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
-                                DAG.getConstant(Value, dl, MVT::i32),
-                                DAG.getConstant(Shift, dl, MVT::i32));
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      SDLoc DL(Op);
+      SDValue Mov =
+          DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32),
+                      DAG.getConstant(Shift, DL, MVT::i32));
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14407,10 +14405,10 @@ static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
     if (AArch64_AM::isAdvSIMDModImmType9(Value)) {
       Value = AArch64_AM::encodeAdvSIMDModImmType9(Value);
 
-      SDLoc dl(Op);
-      SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
-                                DAG.getConstant(Value, dl, MVT::i32));
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      SDLoc DL(Op);
+      SDValue Mov =
+          DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32));
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14438,10 +14436,10 @@ static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
     }
 
     if (isAdvSIMDModImm) {
-      SDLoc dl(Op);
-      SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
-                                DAG.getConstant(Value, dl, MVT::i32));
-      return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
+      SDLoc DL(Op);
+      SDValue Mov =
+          DAG.getNode(NewOp, DL, MovTy, DAG.getConstant(Value, DL, MVT::i32));
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Mov);
     }
   }
 
@@ -14668,7 +14666,7 @@ SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
 static SDValue NormalizeBuildVector(SDValue Op,
                                     SelectionDAG &DAG) {
   assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   EVT EltTy= VT.getVectorElementType();
 
@@ -14684,7 +14682,7 @@ static SDValue NormalizeBuildVector(SDValue Op,
     if (auto *CstLane = dyn_cast<ConstantSDNode>(Lane)) {
       Lane = DAG.getConstant(
           CstLane->getAPIntValue().trunc(EltTy.getSizeInBits()).getZExtValue(),
-          dl, MVT::i32);
+          DL, MVT::i32);
     } else if (Lane.getNode()->isUndef()) {
       Lane = DAG.getUNDEF(MVT::i32);
     } else {
@@ -14693,7 +14691,7 @@ static SDValue NormalizeBuildVector(SDValue Op,
     }
     Ops.push_back(Lane);
   }
-  return DAG.getBuildVector(VT, dl, Ops);
+  return DAG.getBuildVector(VT, DL, Ops);
 }
 
 static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG,
@@ -14877,7 +14875,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
   //             lanes such that we can directly materialize the vector
   //             some other way (MOVI, e.g.), we can be sneaky.
   //   5) if all operands are EXTRACT_VECTOR_ELT, check for VUZP.
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   unsigned NumElts = VT.getVectorNumElements();
   bool isOnlyLowElement = true;
   bool usesOnlyOneValue = true;
@@ -14951,7 +14949,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
   if (isOnlyLowElement && !(NumElts == 1 && isIntOrFPConstant(Value))) {
     LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: only low element used, creating 1 "
                          "SCALAR_TO_VECTOR node\n");
-    return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
+    return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
   }
 
   if (AllLanesExtractElt) {
@@ -15003,16 +15001,16 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
     }
     if (Even || Odd) {
       SDValue LHS =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
-                      DAG.getConstant(0, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(Vector, 0),
+                      DAG.getConstant(0, DL, MVT::i64));
       SDValue RHS =
-          DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
-                      DAG.getConstant(NumElts, dl, MVT::i64));
+          DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(Vector, 0),
+                      DAG.getConstant(NumElts, DL, MVT::i64));
 
       if (Even && !Odd)
-        return DAG.getNode(AArch64ISD::UZP1, dl, VT, LHS, RHS);
+        return DAG.getNode(AArch64ISD::UZP1, DL, VT, LHS, RHS);
       if (Odd && !Even)
-        return DAG.getNode(AArch64ISD::UZP2, dl, VT, LHS, RHS);
+        return DAG.getNode(AArch64ISD::UZP2, DL, VT, LHS, RHS);
     }
   }
 
@@ -15024,7 +15022,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
           Value.getValueType() != VT) {
         LLVM_DEBUG(
             dbgs() << "LowerBUILD_VECTOR: use DUP for non-constant splats\n");
-        return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
+        return DAG.getNode(AArch64ISD::DUP, DL, VT, Value);
       }
 
       // This is actually a DUPLANExx operation, which keeps everything vectory.
@@ -15039,7 +15037,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
       }
 
       unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
-      return DAG.getNode(Opcode, dl, VT, Value, Lane);
+      return DAG.getNode(Opcode, DL, VT, Value, Lane);
     }
 
     if (VT.getVectorElementType().isFloatingPoint()) {
@@ -15052,14 +15050,14 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
                     "BITCASTS, and try again\n");
       MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits());
       for (unsigned i = 0; i < NumElts; ++i)
-        Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i)));
+        Ops.push_back(DAG.getNode(ISD::BITCAST, DL, NewType, Op.getOperand(i)));
       EVT VecVT = EVT::getVectorVT(*DAG.getContext(), NewType, NumElts);
-      SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
+      SDValue Val = DAG.getBuildVector(VecVT, DL, Ops);
       LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: trying to lower new vector: ";
                  Val.dump(););
       Val = LowerBUILD_VECTOR(Val, DAG);
       if (Val.getNode())
-        return DAG.getNode(ISD::BITCAST, dl, VT, Val);
+        return DAG.getNode(ISD::BITCAST, DL, VT, Val);
     }
   }
 
@@ -15078,7 +15076,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
   // for each lane.
   if (!PreferDUPAndInsert && NumConstantLanes > 0 && usesOnlyOneConstantValue) {
     // Firstly, try to materialize the splat constant.
-    SDValue Val = DAG.getSplatBuildVector(VT, dl, ConstantValue);
+    SDValue Val = DAG.getSplatBuildVector(VT, DL, ConstantValue);
     unsigned BitSize = VT.getScalarSizeInBits();
     APInt ConstantValueAPInt(1, 0);
     if (auto *C = dyn_cast<ConstantSDNode>(ConstantValue))
@@ -15088,17 +15086,17 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
       Val = ConstantBuildVector(Val, DAG, Subtarget);
       if (!Val)
         // Otherwise, materialize the constant and splat it.
-        Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
+        Val = DAG.getNode(AArch64ISD::DUP, DL, VT, ConstantValue);
     }
 
     // Now insert the non-constant lanes.
     for (unsigned i = 0; i < NumElts; ++i) {
       SDValue V = Op.getOperand(i);
-      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
+      SDValue LaneIdx = DAG.getConstant(i, DL, MVT::i64);
       if (!isIntOrFPConstant(V))
         // Note that type legalization likely mucked about with the VT of the
         // source operand, so we may have to convert it here before inserting.
-        Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
+        Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Val, V, LaneIdx);
     }
     return Val;
   }
@@ -15129,13 +15127,13 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
   if (PreferDUPAndInsert) {
     // First, build a constant vector with the common element.
     SmallVector<SDValue, 8> Ops(NumElts, Value);
-    SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, dl, Ops), DAG);
+    SDValue NewVector = LowerBUILD_VECTOR(DAG.getBuildVector(VT, DL, Ops), DAG);
     // Next, insert the elements that do not match the common value.
     for (unsigned I = 0; I < NumElts; ++I)
       if (Op.getOperand(I) != Value)
         NewVector =
-            DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector,
-                        Op.getOperand(I), DAG.getConstant(I, dl, MVT::i64));
+            DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NewVector,
+                        Op.getOperand(I), DAG.getConstant(I, DL, MVT::i64));
 
     return NewVector;
   }
@@ -15178,11 +15176,11 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
         SmallVector<SDValue, 8> Ops1(NumElts / 2, Vals[0]);
         SmallVector<SDValue, 8> Ops2(NumElts / 2, Vals[1]);
         SDValue DUP1 =
-            LowerBUILD_VECTOR(DAG.getBuildVector(SubVT, dl, Ops1), DAG);
+            LowerBUILD_VECTOR(DAG.getBuildVector(SubVT, DL, Ops1), DAG);
         SDValue DUP2 =
-            LowerBUILD_VECTOR(DAG.getBuildVector(SubVT, dl, Ops2), DAG);
+            LowerBUILD_VECTOR(DAG.getBuildVector(SubVT, DL, Ops2), DAG);
         SDValue CONCAT_VECTORS =
-            DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, DUP1, DUP2);
+            DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, DUP1, DUP2);
         return CONCAT_VECTORS;
       }
     }
@@ -15208,10 +15206,10 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
 
       SmallVector<SDValue, 8> Ops1(NumElts, Vals[0]);
       SmallVector<SDValue, 8> Ops2(NumElts, Vals[1]);
-      SDValue VEC1 = DAG.getBuildVector(VT, dl, Ops1);
-      SDValue VEC2 = DAG.getBuildVector(VT, dl, Ops2);
+      SDValue VEC1 = DAG.getBuildVector(VT, DL, Ops1);
+      SDValue VEC2 = DAG.getBuildVector(VT, DL, Ops2);
       SDValue VECTOR_SHUFFLE =
-          DAG.getVectorShuffle(VT, dl, VEC1, VEC2, MaskVec);
+          DAG.getVectorShuffle(VT, DL, VEC1, VEC2, MaskVec);
       return VECTOR_SHUFFLE;
     }
   }
@@ -15243,7 +15241,7 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // the upper bits of the lowest lane (e.g. v8i8, v4i16).
     if (!Op0.isUndef()) {
       LLVM_DEBUG(dbgs() << "Creating node for op0, it is not undefined:\n");
-      Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0);
+      Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Op0);
       ++i;
     }
     LLVM_DEBUG({
@@ -15254,8 +15252,8 @@ SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
       SDValue V = Op.getOperand(i);
       if (V.isUndef())
         continue;
-      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
-      Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
+      SDValue LaneIdx = DAG.getConstant(i, DL, MVT::i64);
+      Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V, LaneIdx);
     }
     return Vec;
   }
@@ -15572,7 +15570,7 @@ static bool isPow2Splat(SDValue Op, uint64_t &SplatVal, bool &Negated) {
 
 SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
-  SDLoc dl(Op);
+  SDLoc DL(Op);
 
   if (useSVEForFixedLengthVectorVT(VT, /*OverrideNEON=*/true))
     return LowerFixedLengthVectorIntDivideToSVE(Op, DAG);
@@ -15585,12 +15583,12 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
   bool Negated;
   uint64_t SplatVal;
   if (Signed && isPow2Splat(Op.getOperand(1), SplatVal, Negated)) {
-    SDValue Pg = getPredicateForScalableVector(DAG, dl, VT);
+    SDValue Pg = getPredicateForScalableVector(DAG, DL, VT);
     SDValue Res =
-        DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, VT, Pg, Op->getOperand(0),
-                    DAG.getTargetConstant(Log2_64(SplatVal), dl, MVT::i32));
+        DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, DL, VT, Pg, Op->getOperand(0),
+                    DAG.getTargetConstant(Log2_64(SplatVal), DL, MVT::i32));
     if (Negated)
-      Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res);
+      Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
 
     return Res;
   }
@@ -15610,15 +15608,15 @@ SDValue AArch64TargetLowering::LowerDIV(SDValue Op, SelectionDAG &DAG) const {
 
   unsigned UnpkLo = Signed ? AArch64ISD::SUNPKLO : AArch64ISD::UUNPKLO;
   unsigned UnpkHi = Signed ? AArch64ISD::SUNPKHI : AArch64ISD::UUNPKHI;
-  SDValue Op0Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(0));
-  SDValue Op1Lo = DAG.getNode(UnpkLo, dl, WidenedVT, Op.getOperand(1));
-  SDValue Op0Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(0));
-  SDValue Op1Hi = DAG.getNode(UnpkHi, dl, WidenedVT, Op.getOperand(1));
-  SDValue ResultLo = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Lo, Op1Lo);
-  SDValue ResultHi = DAG.getNode(Op.getOpcode(), dl, WidenedVT, Op0Hi, Op1Hi);
-  SDValue ResultLoCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultLo);
-  SDValue ResultHiCast = DAG.getNode(AArch64ISD::NVCAST, dl, VT, ResultHi);
-  return DAG.getNode(AArch64ISD::UZP1, dl, VT, ResultLoCast, ResultHiCast);
+  SDValue Op0Lo = DAG.getNode(UnpkLo, DL, WidenedVT, Op.getOperand(0));
+  SDValue Op1Lo = DAG.getNode(UnpkLo, DL, WidenedVT, Op.getOperand(1));
+  SDValue Op0Hi = DAG.getNode(UnpkHi, DL, WidenedVT, Op.getOperand(0));
+  SDValue Op1Hi = DAG.getNode(UnpkHi, DL, WidenedVT, Op.getOperand(1));
+  SDValue ResultLo = DAG.getNode(Op.getOpcode(), DL, WidenedVT, Op0Lo, Op1Lo);
+  SDValue ResultHi = DAG.getNode(Op.getOpcode(), DL, WidenedVT, Op0Hi, Op1Hi);
+  SDValue ResultLoCast = DAG.getNode(AArch64ISD::NVCAST, DL, VT, ResultLo);
+  SDValue ResultHiCast = DAG.getNode(AArch64ISD::NVCAST, DL, VT, ResultHi);
+  return DAG.getNode(AArch64ISD::UZP1, DL, VT, ResultLoCast, ResultHiCast);
 }
 
 bool AArch64TargetLowering::shouldExpandBuildVectorWithShuffles(
@@ -15715,12 +15713,12 @@ SDValue AArch64TargetLowering::LowerTRUNCATE(SDValue Op,
 
   if (VT.getScalarType() == MVT::i1) {
     // Lower i1 truncate to `(x & 1) != 0`.
-    SDLoc dl(Op);
+    SDLoc DL(Op);
     EVT OpVT = Op.getOperand(0).getValueType();
-    SDValue Zero = DAG.getConstant(0, dl, OpVT);
-    SDValue One = DAG.getConstant(1, dl, OpVT);
-    SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One);
-    return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE);
+    SDValue Zero = DAG.getConstant(0, DL, OpVT);
+    SDValue One = DAG.getConstant(1, DL, OpVT);
+    SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Op.getOperand(0), One);
+    return DAG.getSetCC(DL, VT, And, Zero, ISD::SETNE);
   }
 
   if (!VT.isVector() || VT.isScalableVector())
@@ -15860,7 +15858,7 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
-  SDLoc dl(Op);
+  SDLoc DL(Op);
 
   if (LHS.getValueType().getVectorElementType().isInteger())
     return Op;
@@ -15897,22 +15895,22 @@ SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
 
   bool NoNaNs =
       getTargetMachine().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs();
-  SDValue Cmp = emitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
+  SDValue Cmp = emitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, DL, DAG);
   if (!Cmp.getNode())
     return SDValue();
 
   if (CC2 != AArch64CC::AL) {
-    SDValue Cmp2 = emitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
+    SDValue Cmp2 = emitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, DL, DAG);
     if (!Cmp2.getNode())
       return SDValue();
 
-    Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2);
+    Cmp = DAG.getNode(ISD::OR, DL, CmpVT, Cmp, Cmp2);
   }
 
-  Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
+  Cmp = DAG.getSExtOrTrunc(Cmp, DL, Op.getValueType());
 
   if (ShouldInvert)
-    Cmp = DAG.getNOT(dl, Cmp, Cmp.getValueType());
+    Cmp = DAG.getNOT(DL, Cmp, Cmp.getValueType());
 
   return Cmp;
 }
@@ -16093,23 +16091,23 @@ SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
   }
 
   // Lower NEON reductions.
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   switch (Op.getOpcode()) {
   case ISD::VECREDUCE_AND:
   case ISD::VECREDUCE_OR:
   case ISD::VECREDUCE_XOR:
     return getVectorBitwiseReduce(Op.getOpcode(), Op.getOperand(0),
-                                  Op.getValueType(), dl, DAG);
+                                  Op.getValueType(), DL, DAG);
   case ISD::VECREDUCE_ADD:
-    return getReductionSDNode(AArch64ISD::UADDV, dl, Op, DAG);
+    return getReductionSDNode(AArch64ISD::UADDV, DL, Op, DAG);
   case ISD::VECREDUCE_SMAX:
-    return getReductionSDNode(AArch64ISD::SMAXV, dl, Op, DAG);
+    return getReductionSDNode(AArch64ISD::SMAXV, DL, Op, DAG);
   case ISD::VECREDUCE_SMIN:
-    return getReductionSDNode(AArch64ISD::SMINV, dl, Op, DAG);
+    return getReductionSDNode(AArch64ISD::SMINV, DL, Op, DAG);
   case ISD::VECREDUCE_UMAX:
-    return getReductionSDNode(AArch64ISD::UMAXV, dl, Op, DAG);
+    return getReductionSDNode(AArch64ISD::UMAXV, DL, Op, DAG);
   case ISD::VECREDUCE_UMIN:
-    return getReductionSDNode(AArch64ISD::UMINV, dl, Op, DAG);
+    return getReductionSDNode(AArch64ISD::UMINV, DL, Op, DAG);
   default:
     llvm_unreachable("Unhandled reduction");
   }
@@ -16123,13 +16121,13 @@ SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
     return SDValue();
 
   // LSE has an atomic load-clear instruction, but not a load-and.
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   MVT VT = Op.getSimpleValueType();
   assert(VT != MVT::i128 && "Handled elsewhere, code replicated.");
   SDValue RHS = Op.getOperand(2);
   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
-  RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getAllOnesConstant(dl, VT), RHS);
-  return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
+  RHS = DAG.getNode(ISD::XOR, DL, VT, DAG.getAllOnesConstant(DL, VT), RHS);
+  return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, DL, AN->getMemoryVT(),
                        Op.getOperand(0), Op.getOperand(1), RHS,
                        AN->getMemOperand());
 }
@@ -16138,7 +16136,7 @@ SDValue
 AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
                                                       SelectionDAG &DAG) const {
 
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   // Get the inputs.
   SDNode *Node = Op.getNode();
   SDValue Chain = Op.getOperand(0);
@@ -16149,19 +16147,19 @@ AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
 
   if (DAG.getMachineFunction().getFunction().hasFnAttribute(
           "no-stack-arg-probe")) {
-    SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
+    SDValue SP = DAG.getCopyFromReg(Chain, DL, AArch64::SP, MVT::i64);
     Chain = SP.getValue(1);
-    SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
+    SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
     if (Align)
       SP =
-          DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                      DAG.getSignedConstant(-(uint64_t)Align->value(), dl, VT));
-    Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
+          DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
+                      DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
+    Chain = DAG.getCopyToReg(Chain, DL, AArch64::SP, SP);
     SDValue Ops[2] = {SP, Chain};
-    return DAG.getMergeValues(Ops, dl);
+    return DAG.getMergeValues(Ops, DL);
   }
 
-  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
+  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
 
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Callee = DAG.getTargetExternalSymbol(Subtarget->getChkStkName(),
@@ -16172,11 +16170,11 @@ AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
   if (Subtarget->hasCustomCallingConv())
     TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
 
-  Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size,
-                     DAG.getConstant(4, dl, MVT::i64));
-  Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
+  Size = DAG.getNode(ISD::SRL, DL, MVT::i64, Size,
+                     DAG.getConstant(4, DL, MVT::i64));
+  Chain = DAG.getCopyToReg(Chain, DL, AArch64::X15, Size, SDValue());
   Chain =
-      DAG.getNode(AArch64ISD::CALL, dl, DAG.getVTList(MVT::Other, MVT::Glue),
+      DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
                   Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
                   DAG.getRegisterMask(Mask), Chain.getValue(1));
   // To match the actual intent better, we should read the output from X15 here
@@ -16184,21 +16182,21 @@ AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
   // from X15 here doesn't work at -O0, since it thinks that X15 is undefined
   // here.
 
-  Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size,
-                     DAG.getConstant(4, dl, MVT::i64));
+  Size = DAG.getNode(ISD::SHL, DL, MVT::i64, Size,
+                     DAG.getConstant(4, DL, MVT::i64));
 
-  SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
+  SDValue SP = DAG.getCopyFromReg(Chain, DL, AArch64::SP, MVT::i64);
   Chain = SP.getValue(1);
-  SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
+  SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
   if (Align)
-    SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                     DAG.getSignedConstant(-(uint64_t)Align->value(), dl, VT));
-  Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
+    SP = DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
+                     DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
+  Chain = DAG.getCopyToReg(Chain, DL, AArch64::SP, SP);
 
-  Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
+  Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), DL);
 
   SDValue Ops[2] = {SP, Chain};
-  return DAG.getMergeValues(Ops, dl);
+  return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue
@@ -16211,21 +16209,21 @@ AArch64TargetLowering::LowerInlineDYNAMIC_STACKALLOC(SDValue Op,
 
   MaybeAlign Align =
       cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Node->getValueType(0);
 
   // Construct the new SP value in a GPR.
-  SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
+  SDValue SP = DAG.getCopyFromReg(Chain, DL, AArch64::SP, MVT::i64);
   Chain = SP.getValue(1);
-  SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
+  SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
   if (Align)
-    SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                     DAG.getSignedConstant(-(uint64_t)Align->value(), dl, VT));
+    SP = DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
+                     DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
 
   // Set the real SP to the new value with a probing loop.
-  Chain = DAG.getNode(AArch64ISD::PROBED_ALLOCA, dl, MVT::Other, Chain, SP);
+  Chain = DAG.getNode(AArch64ISD::PROBED_ALLOCA, DL, MVT::Other, Chain, SP);
   SDValue Ops[2] = {SP, Chain};
-  return DAG.getMergeValues(Ops, dl);
+  return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue
@@ -19911,7 +19909,7 @@ performExtractVectorEltCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
 static SDValue performConcatVectorsCombine(SDNode *N,
                                            TargetLowering::DAGCombinerInfo &DCI,
                                            SelectionDAG &DAG) {
-  SDLoc dl(N);
+  SDLoc DL(N);
   EVT VT = N->getValueType(0);
   SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
   unsigned N0Opc = N0->getOpcode(), N1Opc = N1->getOpcode();
@@ -19944,11 +19942,11 @@ static SDValue performConcatVectorsCombine(SDNode *N,
       SmallVector<int, 8> Mask(MidVT.getVectorNumElements());
       for (size_t i = 0; i < Mask.size(); ++i)
         Mask[i] = i * 2;
-      return DAG.getNode(ISD::TRUNCATE, dl, VT,
+      return DAG.getNode(ISD::TRUNCATE, DL, VT,
                          DAG.getVectorShuffle(
-                             MidVT, dl,
-                             DAG.getNode(ISD::BITCAST, dl, MidVT, N00),
-                             DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask));
+                             MidVT, DL,
+                             DAG.getNode(ISD::BITCAST, DL, MidVT, N00),
+                             DAG.getNode(ISD::BITCAST, DL, MidVT, N10), Mask));
     }
 
     // Optimize two large shifts and a combine into a single combine and shift
@@ -19973,13 +19971,13 @@ static SDValue performConcatVectorsCombine(SDNode *N,
                NScalarSize = N->getValueType(0).getScalarSizeInBits();
 
       if (N001ConstVal == N101ConstVal && N001ConstVal > NScalarSize) {
-        N000 = DAG.getNode(AArch64ISD::NVCAST, dl, VT, N000);
-        N100 = DAG.getNode(AArch64ISD::NVCAST, dl, VT, N100);
-        SDValue Uzp = DAG.getNode(AArch64ISD::UZP2, dl, VT, N000, N100);
+        N000 = DAG.getNode(AArch64ISD::NVCAST, DL, VT, N000);
+        N100 = DAG.getNode(AArch64ISD::NVCAST, DL, VT, N100);
+        SDValue Uzp = DAG.getNode(AArch64ISD::UZP2, DL, VT, N000, N100);
         SDValue NewShiftConstant =
-            DAG.getConstant(N001ConstVal - NScalarSize, dl, MVT::i32);
+            DAG.getConstant(N001ConstVal - NScalarSize, DL, MVT::i32);
 
-        return DAG.getNode(AArch64ISD::VLSHR, dl, VT, Uzp, NewShiftConstant);
+        return DAG.getNode(AArch64ISD::VLSHR, DL, VT, Uzp, NewShiftConstant);
       }
     }
   }
@@ -20011,14 +20009,14 @@ static SDValue performConcatVectorsCombine(SDNode *N,
           Ops.push_back(DAG.getUNDEF(FVT));
         else {
           LoadSDNode *LD = cast<LoadSDNode>(V);
-          SDValue NewLoad = DAG.getLoad(FVT, dl, LD->getChain(),
+          SDValue NewLoad = DAG.getLoad(FVT, DL, LD->getChain(),
                                         LD->getBasePtr(), LD->getMemOperand());
           DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLoad.getValue(1));
           Ops.push_back(NewLoad);
         }
       }
       return DAG.getBitcast(N->getValueType(0),
-                            DAG.getBuildVector(NVT, dl, Ops));
+                            DAG.getBuildVector(NVT, DL, Ops));
     }
   }
 
@@ -20042,11 +20040,11 @@ static SDValue performConcatVectorsCombine(SDNode *N,
     if (isBitwiseVectorNegate(N00) && N0->isOnlyUserOf(N00.getNode()) &&
         isBitwiseVectorNegate(N10) && N1->isOnlyUserOf(N10.getNode())) {
       return DAG.getNOT(
-          dl,
-          DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
-                      DAG.getNode(ISD::TRUNCATE, dl, N0.getValueType(),
+          DL,
+          DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
+                      DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(),
                                   N00->getOperand(0)),
-                      DAG.getNode(ISD::TRUNCATE, dl, N1.getValueType(),
+                      DAG.getNode(ISD::TRUNCATE, DL, N1.getValueType(),
                                   N10->getOperand(0))),
           VT);
     }
@@ -20069,9 +20067,9 @@ static SDValue performConcatVectorsCombine(SDNode *N,
     SDValue N11 = N1->getOperand(1);
 
     if (!N00.isUndef() && !N01.isUndef() && !N10.isUndef() && !N11.isUndef()) {
-      SDValue Concat0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N00, N10);
-      SDValue Concat1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N01, N11);
-      return DAG.getNode(N0Opc, dl, VT, Concat0, Concat1);
+      SDValue Concat0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N00, N10);
+      SDValue Concat1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N01, N11);
+      return DAG.getNode(N0Opc, DL, VT, Concat0, Concat1);
     }
   }
 
@@ -20111,12 +20109,12 @@ static SDValue performConcatVectorsCombine(SDNode *N,
                              : N1.getOperand(0).getOperand(0);
     EVT BVT =
         X.getValueType().getDoubleNumVectorElementsVT(*DCI.DAG.getContext());
-    SDValue CC = DAG.getNode(ISD::CONCAT_VECTORS, dl, BVT, X, Y);
+    SDValue CC = DAG.getNode(ISD::CONCAT_VECTORS, DL, BVT, X, Y);
     SDValue Add = DAG.getNode(
-        ISD::ADD, dl, BVT, CC,
-        DAG.getConstant(1ULL << (N0.getConstantOperandVal(1) - 1), dl, BVT));
+        ISD::ADD, DL, BVT, CC,
+        DAG.getConstant(1ULL << (N0.getConstantOperandVal(1) - 1), DL, BVT));
     SDValue Shr =
-        DAG.getNode(AArch64ISD::VLSHR, dl, BVT, Add, N0.getOperand(1));
+        DAG.getNode(AArch64ISD::VLSHR, DL, BVT, Add, N0.getOperand(1));
     return Shr;
   }
 
@@ -20124,11 +20122,11 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   if (N->getNumOperands() == 2 && N0Opc == AArch64ISD::ZIP1 &&
       N1Opc == AArch64ISD::ZIP2 && N0.getOperand(0) == N1.getOperand(0) &&
       N0.getOperand(1) == N1.getOperand(1)) {
-    SDValue E0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(0),
+    SDValue E0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
                              DAG.getUNDEF(N0.getValueType()));
-    SDValue E1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, N0.getOperand(1),
+    SDValue E1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(1),
                              DAG.getUNDEF(N0.getValueType()));
-    return DAG.getNode(AArch64ISD::ZIP1, dl, VT, E0, E1);
+    return DAG.getNode(AArch64ISD::ZIP1, DL, VT, E0, E1);
   }
 
   // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector
@@ -20136,8 +20134,8 @@ static SDValue performConcatVectorsCombine(SDNode *N,
   // canonicalise to that.
   if (N->getNumOperands() == 2 && N0 == N1 && VT.getVectorNumElements() == 2) {
     assert(VT.getScalarSizeInBits() == 64);
-    return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
-                       DAG.getConstant(0, dl, MVT::i64));
+    return DAG.getNode(AArch64ISD::DUPLANE64, DL, VT, WidenVector(N0, DAG),
+                       DAG.getConstant(0, DL, MVT::i64));
   }
 
   // Canonicalise concat_vectors so that the right-hand vector has as few
@@ -20162,9 +20160,9 @@ static SDValue performConcatVectorsCombine(SDNode *N,
 
   MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(),
                                   RHSTy.getVectorNumElements() * 2);
-  return DAG.getNode(ISD::BITCAST, dl, VT,
-                     DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy,
-                                 DAG.getNode(ISD::BITCAST, dl, RHSTy, N0),
+  return DAG.getNode(ISD::BITCAST, DL, VT,
+                     DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatTy,
+                                 DAG.getNode(ISD::BITCAST, DL, RHSTy, N0),
                                  RHS));
 }
 
@@ -20472,21 +20470,21 @@ static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
 
   SDValue CCVal;
   SDValue Cmp;
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   if (InfoAndKind.IsAArch64) {
     CCVal = DAG.getConstant(
-        AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
+        AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), DL,
         MVT::i32);
     Cmp = *InfoAndKind.Info.AArch64.Cmp;
   } else
     Cmp = getAArch64Cmp(
         *InfoAndKind.Info.Generic.Opnd0, *InfoAndKind.Info.Generic.Opnd1,
         ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, CmpVT), CCVal, DAG,
-        dl);
+        DL);
 
   EVT VT = Op->getValueType(0);
-  LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
-  return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
+  LHS = DAG.getNode(ISD::ADD, DL, VT, RHS, DAG.getConstant(1, DL, VT));
+  return DAG.getNode(AArch64ISD::CSEL, DL, VT, RHS, LHS, CCVal, Cmp);
 }
 
 // ADD(UADDV a, UADDV b) -->  UADDV(ADD a, b)
@@ -21559,25 +21557,25 @@ static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
 
   EVT VT = N->getValueType(0);
   SDValue Op = N->getOperand(1);
-  SDLoc dl(N);
+  SDLoc DL(N);
   if (VT == MVT::i64) {
-    Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op);
+    Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i64, Op);
     VT = MVT::v1i64;
   }
 
   if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
-    Op = DAG.getNode(Opcode, dl, VT, Op,
-                     DAG.getSignedConstant(-ShiftAmount, dl, MVT::i32));
+    Op = DAG.getNode(Opcode, DL, VT, Op,
+                     DAG.getSignedConstant(-ShiftAmount, DL, MVT::i32));
     if (N->getValueType(0) == MVT::i64)
-      Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op,
-                       DAG.getConstant(0, dl, MVT::i64));
+      Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Op,
+                       DAG.getConstant(0, DL, MVT::i64));
     return Op;
   } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
-    Op = DAG.getNode(Opcode, dl, VT, Op,
-                     DAG.getConstant(ShiftAmount, dl, MVT::i32));
+    Op = DAG.getNode(Opcode, DL, VT, Op,
+                     DAG.getConstant(ShiftAmount, DL, MVT::i32));
     if (N->getValueType(0) == MVT::i64)
-      Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op,
-                       DAG.getConstant(0, dl, MVT::i64));
+      Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Op,
+                       DAG.getConstant(0, DL, MVT::i64));
     return Op;
   }
 
@@ -21602,12 +21600,11 @@ static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
 
 static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
                                            SelectionDAG &DAG) {
-  SDLoc dl(N);
-  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0),
-                     DAG.getNode(Opc, dl,
-                                 N->getOperand(1).getSimpleValueType(),
+  SDLoc DL(N);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0),
+                     DAG.getNode(Opc, DL, N->getOperand(1).getSimpleValueType(),
                                  N->getOperand(1)),
-                     DAG.getConstant(0, dl, MVT::i64));
+                     DAG.getConstant(0, DL, MVT::i64));
 }
 
 static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
@@ -21627,21 +21624,21 @@ static SDValue LowerSVEIntrinsicIndex(SDNode *N, SelectionDAG &DAG) {
 }
 
 static SDValue LowerSVEIntrinsicDUP(SDNode *N, SelectionDAG &DAG) {
-  SDLoc dl(N);
+  SDLoc DL(N);
   SDValue Scalar = N->getOperand(3);
   EVT ScalarTy = Scalar.getValueType();
 
   if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
-    Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar);
+    Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Scalar);
 
   SDValue Passthru = N->getOperand(1);
   SDValue Pred = N->getOperand(2);
-  return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, dl, N->getValueType(0),
+  return DAG.getNode(AArch64ISD::DUP_MERGE_PASSTHRU, DL, N->getValueType(0),
                      Pred, Scalar, Passthru);
 }
 
 static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
-  SDLoc dl(N);
+  SDLoc DL(N);
   LLVMContext &Ctx = *DAG.getContext();
   EVT VT = N->getValueType(0);
 
@@ -21657,13 +21654,13 @@ static SDValue LowerSVEIntrinsicEXT(SDNode *N, SelectionDAG &DAG) {
       EVT::getVectorVT(Ctx, MVT::i8, ElementCount::getScalable(ByteSize));
 
   // Convert everything to the domain of EXT (i.e bytes).
-  SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1));
-  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2));
-  SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3),
-                            DAG.getConstant(ElemSize, dl, MVT::i32));
+  SDValue Op0 = DAG.getNode(ISD::BITCAST, DL, ByteVT, N->getOperand(1));
+  SDValue Op1 = DAG.getNode(ISD::BITCAST, DL, ByteVT, N->getOperand(2));
+  SDValue Op2 = DAG.getNode(ISD::MUL, DL, MVT::i32, N->getOperand(3),
+                            DAG.getConstant(ElemSize, DL, MVT::i32));
 
-  SDValue EXT = DAG.getNode(AArch64ISD::EXT, dl, ByteVT, Op0, Op1, Op2);
-  return DAG.getNode(ISD::BITCAST, dl, VT, EXT);
+  SDValue EXT = DAG.getNode(AArch64ISD::EXT, DL, ByteVT, Op0, Op1, Op2);
+  return DAG.getNode(ISD::BITCAST, DL, VT, EXT);
 }
 
 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
@@ -23420,6 +23417,8 @@ static SDValue performPostLD1Combine(SDNode *N,
     auto *LaneC = dyn_cast<ConstantSDNode>(Lane);
     if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements())
       return SDValue();
+    if (LaneC->getZExtValue() == 0 && isNullOrNullSplat(N->getOperand(0)))
+      return SDValue();
   }
 
   LoadSDNode *LoadSDN = cast<LoadSDNode>(LD);
@@ -27299,11 +27298,11 @@ static void ReplaceReductionResults(SDNode *N,
                                     unsigned AcrossOp) {
   EVT LoVT, HiVT;
   SDValue Lo, Hi;
-  SDLoc dl(N);
+  SDLoc DL(N);
   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
   std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
-  SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
-  SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
+  SDValue InterVal = DAG.getNode(InterOp, DL, LoVT, Lo, Hi);
+  SDValue SplitVal = DAG.getNode(AcrossOp, DL, LoVT, InterVal);
   Results.push_back(SplitVal);
 }
 
@@ -27374,17 +27373,17 @@ void AArch64TargetLowering::ReplaceGetActiveLaneMaskResults(
 
 // Create an even/odd pair of X registers holding integer value V.
 static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
-  SDLoc dl(V.getNode());
-  auto [VLo, VHi] = DAG.SplitScalar(V, dl, MVT::i64, MVT::i64);
+  SDLoc DL(V.getNode());
+  auto [VLo, VHi] = DAG.SplitScalar(V, DL, MVT::i64, MVT::i64);
   if (DAG.getDataLayout().isBigEndian())
     std::swap (VLo, VHi);
   SDValue RegClass =
-      DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
-  SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
-  SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
+      DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, DL, MVT::i32);
+  SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, DL, MVT::i32);
+  SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, DL, MVT::i32);
   const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
   return SDValue(
-      DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
+      DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops), 0);
 }
 
 static void ReplaceCMP_SWAP_128Results(SDNode *N,
@@ -27580,13 +27579,13 @@ static void ReplaceATOMIC_LOAD_128Results(SDNode *N,
       getAtomicLoad128Opcode(ISDOpcode, MemOp->getMergedOrdering());
 
   if (ISDOpcode == ISD::ATOMIC_LOAD_AND) {
-    SDLoc dl(Val128);
+    SDLoc DL(Val128);
     Val2x64.first =
-        DAG.getNode(ISD::XOR, dl, MVT::i64,
-                    DAG.getAllOnesConstant(dl, MVT::i64), Val2x64.first);
+        DAG.getNode(ISD::XOR, DL, MVT::i64,
+                    DAG.getAllOnesConstant(DL, MVT::i64), Val2x64.first);
     Val2x64.second =
-        DAG.getNode(ISD::XOR, dl, MVT::i64,
-                    DAG.getAllOnesConstant(dl, MVT::i64), Val2x64.second);
+        DAG.getNode(ISD::XOR, DL, MVT::i64,
+                    DAG.getAllOnesConstant(DL, MVT::i64), Val2x64.second);
   }
 
   SDValue Ops[] = {Val2x64.first, Val2x64.second, Ptr, Chain};
@@ -28918,7 +28917,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorMStoreToSVE(
 
 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
     SDValue Op, SelectionDAG &DAG) const {
-  SDLoc dl(Op);
+  SDLoc DL(Op);
   EVT VT = Op.getValueType();
   EVT EltVT = VT.getVectorElementType();
 
@@ -28930,14 +28929,14 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
   if (Signed && isPow2Splat(Op.getOperand(1), SplatVal, Negated)) {
     EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
     SDValue Op1 = convertToScalableVector(DAG, ContainerVT, Op.getOperand(0));
-    SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), dl, MVT::i32);
+    SDValue Op2 = DAG.getTargetConstant(Log2_64(SplatVal), DL, MVT::i32);
 
-    SDValue Pg = getPredicateForFixedLengthVector(DAG, dl, VT);
+    SDValue Pg = getPredicateForFixedLengthVector(DAG, DL, VT);
     SDValue Res =
-        DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, dl, ContainerVT, Pg, Op1, Op2);
+        DAG.getNode(AArch64ISD::SRAD_MERGE_OP1, DL, ContainerVT, Pg, Op1, Op2);
     if (Negated)
-      Res = DAG.getNode(ISD::SUB, dl, ContainerVT,
-                        DAG.getConstant(0, dl, ContainerVT), Res);
+      Res = DAG.getNode(ISD::SUB, DL, ContainerVT,
+                        DAG.getConstant(0, DL, ContainerVT), Res);
 
     return convertFromScalableVector(DAG, VT, Res);
   }
@@ -28954,32 +28953,32 @@ SDValue AArch64TargetLowering::LowerFixedLengthVectorIntDivideToSVE(
   // If the wider type is legal: extend, op, and truncate.
   EVT WideVT = VT.widenIntegerVectorElementType(*DAG.getContext());
   if (DAG.getTargetLoweringInfo().isTypeLegal(WideVT)) {
-    SDValue Op0 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(0));
-    SDValue Op1 = DAG.getNode(ExtendOpcode, dl, WideVT, Op.getOperand(1));
-    SDValue Div = DAG.getNode(Op.getOpcode(), dl, WideVT, Op0, Op1);
-    return DAG.getNode(ISD::TRUNCATE, dl, VT, Div);
+    SDValue Op0 = DAG.getNode(ExtendOpcode, DL, WideVT, Op.getOperand(0));
+    SDValue Op1 = DAG.getNode(ExtendOpcode, DL, WideVT, Op.getOperand(1));
+    SDValue Div = DAG.getNode(Op.getOpcode(), DL, WideVT, Op0, Op1);
+    return DAG.getNode(ISD::TRUNCATE, DL, VT, Div);
   }
 
-  auto HalveAndExtendVector = [&DAG, &dl, &HalfVT, &PromVT,
+  auto HalveAndExtendVector = [&DAG, &DL, &HalfVT, &PromVT,
                                &ExtendOpcode](SDValue Op) {
-    SDValue IdxZero = DAG.getConstant(0, dl, MVT::i64);
+    SDValue IdxZero = DAG.getConstant(0, DL, MVT::i64);
     SDValue IdxHalf =
-        DAG.getConstant(HalfVT.getVectorNumElements(), dl, MVT::i64);
-    SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxZero);
-    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HalfVT, Op, IdxHalf);
+        DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64);
+    SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Op, IdxZero);
+    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Op, IdxHalf);
     return std::pair<SDValue, SDValue>(
-        {DAG.getNode(ExtendOpcode, dl, PromVT, Lo),
-         DAG.getNode(ExtendOpcode, dl, PromVT, Hi)});
+        {DAG.getNode(ExtendOpcode, DL, PromVT, Lo),
+         DAG.getNode(ExtendOpcode, DL, PromVT, Hi)});
   };
 
   // If wider type is not legal: split, extend, op, trunc and concat.
   auto [Op0LoExt, Op0HiExt] = HalveAndExtendVector(Op.getOperand(0));
   auto [Op1LoExt, Op1HiExt] = HalveAndExtendVector(Op.getOperand(1));
-  SDValue Lo = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0LoExt, Op1LoExt);
-  SDValue Hi = DAG.getNode(Op.getOpcode(), dl, PromVT, Op0HiExt, Op1HiExt);
-  SDValue LoTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Lo);
-  SDValue HiTrunc = DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Hi);
-  return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, {LoTrunc, HiTrunc});
+  SDValue Lo = DAG.getNode(Op.getOpcode(), DL, PromVT, Op0LoExt, Op1LoExt);
+  SDValue Hi = DAG.getNode(Op.getOpcode(), DL, PromVT, Op0HiExt, Op1HiExt);
+  SDValue LoTrunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, Lo);
+  SDValue HiTrunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, Hi);
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoTrunc, HiTrunc});
 }
 
 SDValue AArch64TargetLowering::LowerFixedLengthVectorIntExtendToSVE(
@@ -30030,8 +30029,15 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
           DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
     }
 
-    if (Subtarget->hasSVE2p1()) {
-      if (std::optional<unsigned> Lane = isDUPQMask(ShuffleMask, VT)) {
+    if ((Subtarget->hasSVE2p1() || Subtarget->hasSME2p1()) &&
+        Subtarget->isSVEorStreamingSVEAvailable()) {
+      assert(VT.getFixedSizeInBits() % AArch64::SVEBitsPerBlock == 0 &&
+             "Unsupported SVE vector size");
+
+      unsigned Segments = VT.getFixedSizeInBits() / AArch64::SVEBitsPerBlock;
+      unsigned SegmentElts = VT.getVectorNumElements() / Segments;
+      if (std::optional<unsigned> Lane =
+              isDUPQMask(ShuffleMask, Segments, SegmentElts)) {
         SDValue IID =
             DAG.getConstant(Intrinsic::aarch64_sve_dup_laneq, DL, MVT::i64);
         return convertFromScalableVector(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 951cb93ea8f8..8847c6269071 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -5037,7 +5037,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 
     if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
       // If either operand is WSP, expand to ADD #0.
-      if (Subtarget.hasZeroCycleRegMove()) {
+      if (Subtarget.hasZeroCycleRegMoveGPR64() &&
+          !Subtarget.hasZeroCycleRegMoveGPR32()) {
         // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
         MCRegister DestRegX = TRI->getMatchingSuperReg(
             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
@@ -5063,7 +5064,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
           .addImm(0)
           .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
     } else {
-      if (Subtarget.hasZeroCycleRegMove()) {
+      if (Subtarget.hasZeroCycleRegMoveGPR64() &&
+          !Subtarget.hasZeroCycleRegMoveGPR32()) {
         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
         MCRegister DestRegX = TRI->getMatchingSuperReg(
             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
@@ -5302,30 +5304,78 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 
   if (AArch64::FPR32RegClass.contains(DestReg) &&
       AArch64::FPR32RegClass.contains(SrcReg)) {
-    BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+    if (Subtarget.hasZeroCycleRegMoveFPR64() &&
+        !Subtarget.hasZeroCycleRegMoveFPR32()) {
+      const TargetRegisterInfo *TRI = &getRegisterInfo();
+      MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::ssub,
+                                                     &AArch64::FPR64RegClass);
+      MCRegister SrcRegD = TRI->getMatchingSuperReg(SrcReg, AArch64::ssub,
+                                                    &AArch64::FPR64RegClass);
+      // This instruction is reading and writing D registers. This may upset
+      // the register scavenger and machine verifier, so we need to indicate
+      // that we are reading an undefined value from SrcRegD, but a proper
+      // value from SrcReg.
+      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
+          .addReg(SrcRegD, RegState::Undef)
+          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
+    } else {
+      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc));
+    }
     return;
   }
 
   if (AArch64::FPR16RegClass.contains(DestReg) &&
       AArch64::FPR16RegClass.contains(SrcReg)) {
-    DestReg =
-        RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
-    SrcReg =
-        RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
-    BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+    if (Subtarget.hasZeroCycleRegMoveFPR64() &&
+        !Subtarget.hasZeroCycleRegMoveFPR32()) {
+      const TargetRegisterInfo *TRI = &getRegisterInfo();
+      MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::hsub,
+                                                     &AArch64::FPR64RegClass);
+      MCRegister SrcRegD = TRI->getMatchingSuperReg(SrcReg, AArch64::hsub,
+                                                    &AArch64::FPR64RegClass);
+      // This instruction is reading and writing D registers. This may upset
+      // the register scavenger and machine verifier, so we need to indicate
+      // that we are reading an undefined value from SrcRegD, but a proper
+      // value from SrcReg.
+      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
+          .addReg(SrcRegD, RegState::Undef)
+          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
+    } else {
+      DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
+                                       &AArch64::FPR32RegClass);
+      SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
+                                      &AArch64::FPR32RegClass);
+      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc));
+    }
     return;
   }
 
   if (AArch64::FPR8RegClass.contains(DestReg) &&
       AArch64::FPR8RegClass.contains(SrcReg)) {
-    DestReg =
-        RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
-    SrcReg =
-        RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
-    BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+    if (Subtarget.hasZeroCycleRegMoveFPR64() &&
+        !Subtarget.hasZeroCycleRegMoveFPR32()) {
+      const TargetRegisterInfo *TRI = &getRegisterInfo();
+      MCRegister DestRegD = TRI->getMatchingSuperReg(DestReg, AArch64::bsub,
+                                                     &AArch64::FPR64RegClass);
+      MCRegister SrcRegD = TRI->getMatchingSuperReg(SrcReg, AArch64::bsub,
+                                                    &AArch64::FPR64RegClass);
+      // This instruction is reading and writing D registers. This may upset
+      // the register scavenger and machine verifier, so we need to indicate
+      // that we are reading an undefined value from SrcRegD, but a proper
+      // value from SrcReg.
+      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
+          .addReg(SrcRegD, RegState::Undef)
+          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
+    } else {
+      DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
+                                       &AArch64::FPR32RegClass);
+      SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
+                                      &AArch64::FPR32RegClass);
+      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
+          .addReg(SrcReg, getKillRegState(KillSrc));
+    }
     return;
   }
 
@@ -9646,9 +9696,6 @@ AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
                                        MachineBasicBlock::iterator &MIT,
                                        unsigned Flags) const {
   MachineInstr &MI = *MIT;
-  MachineBasicBlock *MBB = MI.getParent();
-  MachineFunction *MF = MBB->getParent();
-  AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
 
   // Don't outline anything used for return address signing. The outlined
   // function will get signed later if needed
@@ -9676,10 +9723,6 @@ AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
     return outliner::InstrType::Illegal;
   }
 
-  // Don't outline LOHs.
-  if (FuncInfo->getLOHRelated().count(&MI))
-    return outliner::InstrType::Illegal;
-
   // We can only outline these if we will tail call the outlined function, or
   // fix up the CFI offsets. Currently, CFI instructions are outlined only if
   // in a tail call.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 0f3f24f0853c..0cb7b02d84a6 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -153,8 +153,8 @@ def HasSVEAES       : Predicate<"Subtarget->hasSVEAES()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVEAES), "sve-aes">;
 def HasSVE2SM4       : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SM4()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVE2SM4), "sve2-sm4">;
-def HasSVE2SHA3      : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SHA3()">,
-                                 AssemblerPredicateWithAll<(all_of FeatureSVE2SHA3), "sve2-sha3">;
+def HasSVESHA3      : Predicate<"Subtarget->hasSVESHA3()">,
+                                 AssemblerPredicateWithAll<(all_of FeatureSVESHA3), "sve-sha3">;
 def HasSVEBitPerm   : Predicate<"Subtarget->hasSVEBitPerm()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVEBitPerm), "sve-bitperm">;
 def HasSMEandIsNonStreamingSafe
@@ -248,43 +248,59 @@ def HasSVE_or_SME
     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable()">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
                 "sve or sme">;
+def HasNonStreamingSVE_or_SME2p1
+    : Predicate<"Subtarget->isSVEAvailable() ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p1())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p1),
+                "sve or sme2p1">;
 def HasNonStreamingSVE_or_SME2p2
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+    : Predicate<"Subtarget->isSVEAvailable() ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p2),
                 "sve or sme2p2">;
+def HasNonStreamingSVE_or_SSVE_AES
+    : Predicate<"Subtarget->isSVEAvailable() ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_AES())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_AES),
+                "sve or ssve-aes">;
+def HasNonStreamingSVE_or_SSVE_BitPerm
+    : Predicate<"Subtarget->isSVEAvailable() ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_BitPerm())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_BitPerm),
+                "sve or ssve-bitperm">;
 def HasNonStreamingSVE_or_SSVE_FEXPA
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+    : Predicate<"Subtarget->isSVEAvailable() ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA),
                 "sve or ssve-fexpa">;
 
 def HasSVE2_or_SME
-    : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME),
                 "sve2 or sme">;
-def HasSVE2_or_SME2
-    : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
+def HasNonStreamingSVE2_or_SME2
+    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME2),
                 "sve2 or sme2">;
-def HasNonStreamingSVE2_or_SSVE_AES
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
-                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_AES())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_AES),
-                "sve2 or ssve-aes">;
 
 def HasSVE2p1_or_SME
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1),
                 "sme or sve2p1">;
 def HasSVE2p1_or_SME2
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
                 "sme2 or sve2p1">;
 def HasSVE2p1_or_SME2p1
-    : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p1() || Subtarget->hasSME2p1())">,
                 AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1),
                 "sme2p1 or sve2p1">;
+def HasSVE2p1_or_StreamingSME2
+    : Predicate<"(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVE2p1()) ||"
+                "(Subtarget->isStreaming() && Subtarget->hasSME2())">,
+                AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
+                "sme2 or sve2p1">;
 
 def HasSVE2p2_or_SME2p2
     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2())">,
@@ -300,11 +316,6 @@ def HasSMEF16F16_or_SMEF8F16
     : Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
                 AssemblerPredicateWithAll<(any_of FeatureSMEF16F16, FeatureSMEF8F16),
                 "sme-f16f16 or sme-f8f16">;
-def HasNonStreamingSVE2_or_SSVE_BitPerm
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
-                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_BitPerm())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_BitPerm),
-                "sve2 or ssve-bitperm">;
 
 // A subset of NEON instructions are legal in Streaming SVE execution mode,
 // so don't need the additional check for 'isNeonAvailable'.
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index f51f0d11ef9d..dde4e7ab0e89 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -1081,8 +1081,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
           LLVM_DEBUG(dbgs() << "Renamed " << MI);
           return true;
         };
-    forAllMIsUntilDef(MergeForward ? *I : *std::prev(Paired), RegToRename, TRI,
-                      UINT32_MAX, UpdateMIs);
+    forAllMIsUntilDef(MergeForward ? *I : *Paired->getPrevNode(), RegToRename,
+                      TRI, UINT32_MAX, UpdateMIs);
 
 #if !defined(NDEBUG)
     // For forward merging store:
diff --git a/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp b/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
index fd3ce6c72e50..39946633603f 100644
--- a/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
@@ -13,7 +13,7 @@
 
 #include "AArch64MCInstLower.h"
 #include "AArch64MachineFunctionInfo.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -147,29 +147,29 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandMachO(const MachineOperand &MO,
                                                       MCSymbol *Sym) const {
   // FIXME: We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing.
-  auto Spec = AArch64MCExpr::None;
+  auto Spec = AArch64::S_None;
   if ((MO.getTargetFlags() & AArch64II::MO_GOT) != 0) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
-      Spec = AArch64MCExpr::M_GOTPAGE;
+      Spec = AArch64::S_MACHO_GOTPAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
-      Spec = AArch64MCExpr::M_GOTPAGEOFF;
+      Spec = AArch64::S_MACHO_GOTPAGEOFF;
     else
       llvm_unreachable("Unexpected target flags with MO_GOT on GV operand");
   } else if ((MO.getTargetFlags() & AArch64II::MO_TLS) != 0) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
-      Spec = AArch64MCExpr::M_TLVPPAGE;
+      Spec = AArch64::S_MACHO_TLVPPAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
-      Spec = AArch64MCExpr::M_TLVPPAGEOFF;
+      Spec = AArch64::S_MACHO_TLVPPAGEOFF;
     else
       llvm_unreachable("Unexpected target flags with MO_TLS on GV operand");
   } else {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
-      Spec = AArch64MCExpr::M_PAGE;
+      Spec = AArch64::S_MACHO_PAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
-      Spec = AArch64MCExpr::M_PAGEOFF;
+      Spec = AArch64::S_MACHO_PAGEOFF;
   }
   // TODO: Migrate to MCSpecifierExpr::create like ELF.
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Spec, Ctx);
@@ -186,8 +186,8 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
   if (MO.getTargetFlags() & AArch64II::MO_GOT) {
     const MachineFunction *MF = MO.getParent()->getParent()->getParent();
     RefFlags |= (MF->getInfo<AArch64FunctionInfo>()->hasELFSignedGOT()
-                     ? AArch64MCExpr::VK_GOT_AUTH
-                     : AArch64MCExpr::VK_GOT);
+                     ? AArch64::S_GOT_AUTH
+                     : AArch64::S_GOT);
   } else if (MO.getTargetFlags() & AArch64II::MO_TLS) {
     TLSModel::Model Model;
     if (MO.isGlobal()) {
@@ -211,13 +211,13 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
     }
     switch (Model) {
     case TLSModel::InitialExec:
-      RefFlags |= AArch64MCExpr::VK_GOTTPREL;
+      RefFlags |= AArch64::S_GOTTPREL;
       break;
     case TLSModel::LocalExec:
-      RefFlags |= AArch64MCExpr::VK_TPREL;
+      RefFlags |= AArch64::S_TPREL;
       break;
     case TLSModel::LocalDynamic:
-      RefFlags |= AArch64MCExpr::VK_DTPREL;
+      RefFlags |= AArch64::S_DTPREL;
       break;
     case TLSModel::GeneralDynamic: {
       // TODO: it's probably better to introduce MO_TLS_AUTH or smth and avoid
@@ -226,47 +226,44 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
       // making the field wider breaks static assertions.
       const MachineFunction *MF = MO.getParent()->getParent()->getParent();
       RefFlags |= MF->getInfo<AArch64FunctionInfo>()->hasELFSignedGOT()
-                      ? AArch64MCExpr::VK_TLSDESC_AUTH
-                      : AArch64MCExpr::VK_TLSDESC;
+                      ? AArch64::S_TLSDESC_AUTH
+                      : AArch64::S_TLSDESC;
       break;
     }
     }
   } else if (MO.getTargetFlags() & AArch64II::MO_PREL) {
-    RefFlags |= AArch64MCExpr::VK_PREL;
+    RefFlags |= AArch64::S_PREL;
   } else {
     // No modifier means this is a generic reference, classified as absolute for
     // the cases where it matters (:abs_g0: etc).
-    RefFlags |= AArch64MCExpr::VK_ABS;
+    RefFlags |= AArch64::S_ABS;
   }
 
   if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
-    RefFlags |= AArch64MCExpr::VK_PAGE;
+    RefFlags |= AArch64::S_PAGE;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
            AArch64II::MO_PAGEOFF)
-    RefFlags |= AArch64MCExpr::VK_PAGEOFF;
+    RefFlags |= AArch64::S_PAGEOFF;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G3)
-    RefFlags |= AArch64MCExpr::VK_G3;
+    RefFlags |= AArch64::S_G3;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G2)
-    RefFlags |= AArch64MCExpr::VK_G2;
+    RefFlags |= AArch64::S_G2;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G1)
-    RefFlags |= AArch64MCExpr::VK_G1;
+    RefFlags |= AArch64::S_G1;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G0)
-    RefFlags |= AArch64MCExpr::VK_G0;
+    RefFlags |= AArch64::S_G0;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_HI12)
-    RefFlags |= AArch64MCExpr::VK_HI12;
+    RefFlags |= AArch64::S_HI12;
 
   if (MO.getTargetFlags() & AArch64II::MO_NC)
-    RefFlags |= AArch64MCExpr::VK_NC;
+    RefFlags |= AArch64::S_NC;
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 
-  AArch64MCExpr::Specifier RefKind;
-  RefKind = static_cast<AArch64MCExpr::Specifier>(RefFlags);
-  Expr = MCSpecifierExpr::create(Expr, RefKind, Ctx);
-
+  Expr = MCSpecifierExpr::create(Expr, RefFlags, Ctx);
   return MCOperand::createExpr(Expr);
 }
 
@@ -276,31 +273,31 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,
 
   if (MO.getTargetFlags() & AArch64II::MO_TLS) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGEOFF)
-      RefFlags |= AArch64MCExpr::VK_SECREL_LO12;
+      RefFlags |= AArch64::S_SECREL_LO12;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_HI12)
-      RefFlags |= AArch64MCExpr::VK_SECREL_HI12;
+      RefFlags |= AArch64::S_SECREL_HI12;
 
   } else if (MO.getTargetFlags() & AArch64II::MO_S) {
-    RefFlags |= AArch64MCExpr::VK_SABS;
+    RefFlags |= AArch64::S_SABS;
   } else {
-    RefFlags |= AArch64MCExpr::VK_ABS;
+    RefFlags |= AArch64::S_ABS;
 
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGE)
-      RefFlags |= AArch64MCExpr::VK_PAGE;
+      RefFlags |= AArch64::S_PAGE;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_PAGEOFF)
-      RefFlags |= AArch64MCExpr::VK_PAGEOFF | AArch64MCExpr::VK_NC;
+      RefFlags |= AArch64::S_PAGEOFF | AArch64::S_NC;
   }
 
   if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G3)
-    RefFlags |= AArch64MCExpr::VK_G3;
+    RefFlags |= AArch64::S_G3;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G2)
-    RefFlags |= AArch64MCExpr::VK_G2;
+    RefFlags |= AArch64::S_G2;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G1)
-    RefFlags |= AArch64MCExpr::VK_G1;
+    RefFlags |= AArch64::S_G1;
   else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G0)
-    RefFlags |= AArch64MCExpr::VK_G0;
+    RefFlags |= AArch64::S_G0;
 
   // FIXME: Currently we only set VK_NC for MO_G3/MO_G2/MO_G1/MO_G0. This is
   // because setting VK_NC for others would mean setting their respective
@@ -309,7 +306,7 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,
     auto MOFrag = (MO.getTargetFlags() & AArch64II::MO_FRAGMENT);
     if (MOFrag == AArch64II::MO_G3 || MOFrag == AArch64II::MO_G2 ||
         MOFrag == AArch64II::MO_G1 || MOFrag == AArch64II::MO_G0)
-      RefFlags |= AArch64MCExpr::VK_NC;
+      RefFlags |= AArch64::S_NC;
   }
 
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);
@@ -317,11 +314,7 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 
-  auto RefKind = static_cast<AArch64MCExpr::Specifier>(RefFlags);
-  assert(RefKind != AArch64MCExpr::VK_INVALID &&
-         "Invalid relocation requested");
-  Expr = MCSpecifierExpr::create(Expr, RefKind, Ctx);
-
+  Expr = MCSpecifierExpr::create(Expr, RefFlags, Ctx);
   return MCOperand::createExpr(Expr);
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 361d5ec3f2b2..e61f2280865b 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -504,6 +504,20 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
     LOHRelated.insert_range(Args);
   }
 
+  size_t
+  clearLinkerOptimizationHints(const SmallPtrSetImpl<MachineInstr *> &MIs) {
+    size_t InitialSize = LOHContainerSet.size();
+    erase_if(LOHContainerSet, [&](const auto &D) {
+      return any_of(D.getArgs(), [&](auto *Arg) { return MIs.contains(Arg); });
+    });
+    // In theory there could be an LOH with one label in MIs and another label
+    // outside MIs, however we don't know if the label outside MIs is used in
+    // any other LOHs, so we can't remove them from LOHRelated. In that case, we
+    // might produce a few extra labels, but it won't break anything.
+    LOHRelated.remove_if([&](auto *MI) { return MIs.contains(MI); });
+    return InitialSize - LOHContainerSet.size();
+  };
+
   SmallVectorImpl<ForwardedRegister> &getForwardedMustTailRegParms() {
     return ForwardedMustTailRegParms;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
index 7b044cf7c238..f7beca1b8b77 100644
--- a/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
+++ b/llvm/lib/Target/AArch64/AArch64PerfectShuffle.h
@@ -15,6 +15,7 @@
 #define LLVM_LIB_TARGET_AARCH64_AARCH64PERFECTSHUFFLE_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace llvm {
 
@@ -6723,6 +6724,47 @@ inline bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
   return true;
 }
 
+/// isDUPQMask - matches a splat of equivalent lanes within segments of a given
+///              number of elements.
+inline std::optional<unsigned> isDUPQMask(ArrayRef<int> Mask, unsigned Segments,
+                                          unsigned SegmentSize) {
+  unsigned Lane = unsigned(Mask[0]);
+
+  // Make sure there's no size changes.
+  if (SegmentSize * Segments != Mask.size())
+    return std::nullopt;
+
+  // Check the first index corresponds to one of the lanes in the first segment.
+  if (Lane >= SegmentSize)
+    return std::nullopt;
+
+  // Check that all lanes match the first, adjusted for segment.
+  // Undef/poison lanes (<0) are also accepted.
+  if (all_of(enumerate(Mask), [&](auto P) {
+        const unsigned SegmentIndex = P.index() / SegmentSize;
+        return P.value() < 0 ||
+               unsigned(P.value()) == Lane + SegmentIndex * SegmentSize;
+      }))
+    return Lane;
+
+  return std::nullopt;
+}
+
+/// isDUPFirstSegmentMask - matches a splat of the first 128b segment.
+inline bool isDUPFirstSegmentMask(ArrayRef<int> Mask, unsigned Segments,
+                                  unsigned SegmentSize) {
+  // Make sure there's no size changes.
+  if (SegmentSize * Segments != Mask.size())
+    return false;
+
+  // Check that all lanes refer to the equivalent lane in the first segment.
+  // Undef/poison lanes (<0) are also accepted.
+  return all_of(enumerate(Mask), [&](auto P) {
+    const unsigned IndexWithinSegment = P.index() % SegmentSize;
+    return P.value() < 0 || unsigned(P.value()) == IndexWithinSegment;
+  });
+}
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index e1b82953aad8..448495182133 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -311,7 +311,8 @@ def TuneAppleA7  : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7",
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES, FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing,
                                     FeatureZCZeroingFPWorkaround]>;
 
@@ -324,7 +325,8 @@ def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
@@ -336,7 +338,8 @@ def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
@@ -348,7 +351,8 @@ def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
@@ -360,7 +364,8 @@ def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
@@ -377,7 +382,8 @@ def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15",
@@ -394,7 +400,8 @@ def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16",
@@ -411,7 +418,8 @@ def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17",
@@ -428,7 +436,8 @@ def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17",
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
-                                    FeatureZCRegMove,
+                                    FeatureZCRegMoveGPR64,
+                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4",
@@ -444,7 +453,8 @@ def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4",
                                      FeatureFuseCCSelect,
                                      FeatureFuseCryptoEOR,
                                      FeatureFuseLiterals,
-                                     FeatureZCRegMove,
+                                     FeatureZCRegMoveGPR64,
+                                     FeatureZCRegMoveFPR64,
                                      FeatureZCZeroing
                                      ]>;
 
@@ -897,7 +907,7 @@ def ProcessorFeatures {
                                    FeatureFPAC, FeatureFP16FML, FeatureRandGen,
                                    FeatureSSBS, FeatureLS64, FeatureCLRBHB,
                                    FeatureSPECRES2, FeatureSVEAES, FeatureSVE2SM4,
-                                   FeatureSVE2SHA3, FeatureSVE2, FeatureSVEBitPerm, FeatureETE,
+                                   FeatureSVESHA3, FeatureSVE2, FeatureSVEBitPerm, FeatureETE,
                                    FeatureMEC, FeatureFAMINMAX, FeatureFP8DOT2, FeatureFP8DOT4,
                                    FeatureFP8FMA, FeatureLUT];
   list<SubtargetFeature> Carmel   = [HasV8_2aOps, FeatureNEON, FeatureSHA2, FeatureAES,
@@ -910,7 +920,7 @@ def ProcessorFeatures {
                                     FeatureLS64, FeatureLUT, FeatureMEC,
                                     FeatureMTE, FeaturePerfMon, FeatureRandGen,
                                     FeatureSPE, FeatureSPE_EEF, FeatureSSBS,
-                                    FeatureSVEBitPerm, FeatureSVE2SHA3,
+                                    FeatureSVEBitPerm, FeatureSVESHA3,
                                     FeatureSVE2SM4, FeatureSVEAES];
   list<SubtargetFeature> AppleA7  = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON,FeaturePerfMon];
@@ -1107,7 +1117,8 @@ def ProcessorFeatures {
                                      FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureSSBS, FeatureCCIDX,
                                      FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
-  list<SubtargetFeature> Grace = !listconcat(NeoverseV2, [FeatureSVE2SM4, FeatureSVEAES, FeatureSVE2SHA3]);
+  list<SubtargetFeature> GB10 = !listconcat(X925, [FeatureSVEAES, FeatureSVESHA3, FeatureSVE2SM4]);
+  list<SubtargetFeature> Grace = !listconcat(NeoverseV2, [FeatureSVE2SM4, FeatureSVEAES, FeatureSVESHA3]);
 
   // ETE and TRBE are future architecture extensions. We temporarily enable them
   // by default for users targeting generic AArch64. The extensions do not
@@ -1193,6 +1204,8 @@ def : ProcessorModel<"cortex-x4", NeoverseV2Model, ProcessorFeatures.X4,
                      [TuneX4]>;
 def : ProcessorModel<"cortex-x925", NeoverseV2Model, ProcessorFeatures.X925,
                      [TuneX925]>;
+def : ProcessorModel<"gb10", NeoverseV2Model, ProcessorFeatures.GB10,
+                     [TuneX925]>;
 def : ProcessorModel<"grace", NeoverseV2Model, ProcessorFeatures.Grace,
                      [TuneNeoverseV2]>;
 def : ProcessorModel<"neoverse-e1", CortexA53Model,
diff --git a/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp b/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
index 8edf1d0f9296..3f45d55063b5 100644
--- a/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PromoteConstant.cpp
@@ -345,6 +345,10 @@ static bool shouldConvertImpl(const Constant *Cst) {
   if (Cst->isZeroValue())
     return false;
 
+  // Globals cannot be or contain scalable vectors.
+  if (Cst->getType()->isScalableTy())
+    return false;
+
   if (Stress)
     return true;
 
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index 61abc3c3c479..db27ca978980 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -915,7 +915,9 @@ defm FADD_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fadd", 0b0100, MatrixOp16
 defm FADD_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fadd", 0b0100, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_add_za16_vg1x4>;
 defm FSUB_VG2_M2Z_H : sme2_multivec_accum_add_sub_vg2<"fsub", 0b0101, MatrixOp16, ZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x2>;
 defm FSUB_VG4_M4Z_H : sme2_multivec_accum_add_sub_vg4<"fsub", 0b0101, MatrixOp16, ZZZZ_h_mul_r, nxv8f16,  int_aarch64_sme_sub_za16_vg1x4>;
+}
 
+let Predicates = [HasSMEF16F16] in {
 defm FMLA_VG2_M2ZZI_H : sme2p1_multi_vec_array_vg2_index_16b<"fmla", 0b00, 0b100, ZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x2>;
 defm FMLA_VG4_M4ZZI_H : sme2p1_multi_vec_array_vg4_index_16b<"fmla", 0b000, ZZZZ_h_mul_r, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_lane_vg1x4>;
 defm FMLA_VG2_M2ZZ_H :  sme2_dot_mla_add_sub_array_vg2_single<"fmla", 0b0011100, MatrixOp16, ZZ_h, ZPR4b16, nxv8f16, int_aarch64_sme_fmla_single_vg1x2>;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 2360e30de63b..91a9d21fa7b2 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -436,13 +436,23 @@ def AArch64fabd_p : PatFrags<(ops node:$pg, node:$op1, node:$op2),
                               (AArch64fabs_mt node:$pg, (AArch64fsub_p node:$pg, node:$op1, node:$op2), undef)]>;
 
 def AArch64fmla_p : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
-                             [(AArch64fma_p node:$pg, node:$zn, node:$zm, node:$za)]>;
+                             [(AArch64fma_p node:$pg, node:$zn, node:$zm, node:$za),
+                              (AArch64fma_p node:$pg, node:$zm, node:$zn, node:$za)]>;
+
+def AArch64fmlaidx : PatFrags<(ops node:$acc, node:$op1, node:$op2, node:$idx),
+                              [(AArch64fmla_p (SVEAllActive), node:$acc, node:$op1, (int_aarch64_sve_dup_laneq node:$op2, node:$idx)),
+                               (int_aarch64_sve_fmla_lane node:$acc, node:$op1, node:$op2, node:$idx)]>;
 
 def AArch64fmls_p : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
                              [(int_aarch64_sve_fmls_u node:$pg, node:$za, node:$zn, node:$zm),
                               (AArch64fma_p node:$pg, (AArch64fneg_mt node:$pg, node:$zn, (undef)), node:$zm, node:$za),
                               (AArch64fma_p node:$pg, node:$zm, (AArch64fneg_mt node:$pg, node:$zn, (undef)), node:$za)]>;
 
+def AArch64fmlsidx : PatFrags<(ops node:$acc, node:$op1, node:$op2, node:$idx),
+                              [(AArch64fmla_p (SVEAllActive), node:$acc, (AArch64fneg_mt(SVEAllActive), node:$op1, (undef)), (int_aarch64_sve_dup_laneq node:$op2, node:$idx)),
+                               (int_aarch64_sve_fmls_lane node:$acc, node:$op1, node:$op2, node:$idx)]>;
+
+
 def AArch64fnmla_p : PatFrags<(ops node:$pg, node:$za, node:$zn, node:$zm),
                               [(int_aarch64_sve_fnmla_u node:$pg, node:$za, node:$zn, node:$zm),
                                (AArch64fma_p node:$pg, (AArch64fneg_mt node:$pg, node:$zn, (undef)), node:$zm, (AArch64fneg_mt node:$pg, node:$za, (undef))),
@@ -560,7 +570,12 @@ def AArch64fadd : PatFrags<(ops node:$op1, node:$op2),
 
 def AArch64fmul : PatFrags<(ops node:$op1, node:$op2),
                             [(fmul node:$op1, node:$op2),
-                             (AArch64fmul_p (SVEAllActive), node:$op1, node:$op2)]>;
+                             (AArch64fmul_p (SVEAllActive), node:$op1, node:$op2),
+                             (AArch64fmul_p (SVEAllActive), node:$op2, node:$op1)]>;
+
+def AArch64fmulidx : PatFrags<(ops node:$op1, node:$op2, node:$idx),
+                              [(AArch64fmul node:$op1, (int_aarch64_sve_dup_laneq node:$op2, node:$idx)),
+                               (int_aarch64_sve_fmul_lane node:$op1, node:$op2, node:$idx)]>;
 
 def AArch64fsub : PatFrags<(ops node:$op1, node:$op2),
                             [(fsub node:$op1, node:$op2),
@@ -872,11 +887,25 @@ let Predicates = [HasSVE] in {
 } // End HasSVE
 
 let Predicates = [HasSVE_or_SME] in {
-  defm FMLA_ZZZI : sve_fp_fma_by_indexed_elem<0b00, "fmla", int_aarch64_sve_fmla_lane>;
-  defm FMLS_ZZZI : sve_fp_fma_by_indexed_elem<0b01, "fmls", int_aarch64_sve_fmls_lane>;
+  defm FMLA_ZZZI : sve_fp_fma_by_indexed_elem<0b00, "fmla", AArch64fmlaidx>;
+  defm FMLS_ZZZI : sve_fp_fma_by_indexed_elem<0b01, "fmls", AArch64fmlsidx>;
 
   defm FCMLA_ZZZI : sve_fp_fcmla_by_indexed_elem<"fcmla", int_aarch64_sve_fcmla_lane>;
-  defm FMUL_ZZZI   : sve_fp_fmul_by_indexed_elem<"fmul", int_aarch64_sve_fmul_lane>;
+  defm FMUL_ZZZI   : sve_fp_fmul_by_indexed_elem<"fmul", AArch64fmulidx>;
+
+  // 64B segmented lane splats currently end up as trn instructions instead.
+  def : Pat<(nxv2f64 (AArch64fmul nxv2f64:$L, (AArch64trn1 nxv2f64:$R, nxv2f64:$R))),
+            (FMUL_ZZZI_D $L, $R, 0)>;
+  def : Pat<(nxv2f64 (AArch64fmul nxv2f64:$L, (AArch64trn2 nxv2f64:$R, nxv2f64:$R))),
+            (FMUL_ZZZI_D $L, $R, 1)>;
+  def : Pat<(nxv2f64 (AArch64fmla_p (SVEAllActive), nxv2f64:$Acc, nxv2f64:$L, (AArch64trn1 nxv2f64:$R, nxv2f64:$R))),
+            (FMLA_ZZZI_D $Acc, $L, $R, 0)>;
+  def : Pat<(nxv2f64 (AArch64fmla_p (SVEAllActive), nxv2f64:$Acc, nxv2f64:$L, (AArch64trn2 nxv2f64:$R, nxv2f64:$R))),
+            (FMLA_ZZZI_D $Acc, $L, $R, 1)>;
+  def : Pat<(nxv2f64 (AArch64fmls_p (SVEAllActive), nxv2f64:$Acc, nxv2f64:$L, (AArch64trn1 nxv2f64:$R, nxv2f64:$R))),
+            (FMLS_ZZZI_D $Acc, $L, $R, 0)>;
+  def : Pat<(nxv2f64 (AArch64fmls_p (SVEAllActive), nxv2f64:$Acc, nxv2f64:$L, (AArch64trn2 nxv2f64:$R, nxv2f64:$R))),
+            (FMLS_ZZZI_D $Acc, $L, $R, 1)>;
 } // End HasSVE_or_SME
 
 let Predicates = [HasSVE] in {
@@ -4062,7 +4091,7 @@ let Predicates = [HasSVE2_or_SME] in {
   defm WHILERW_PXX : sve2_int_while_rr<0b1, "whilerw", "int_aarch64_sve_whilerw">;
 } // End HasSVE2_or_SME
 
-let Predicates = [HasSVEAES, HasNonStreamingSVE2_or_SSVE_AES] in {
+let Predicates = [HasSVEAES, HasNonStreamingSVE_or_SSVE_AES] in {
   // SVE2 crypto destructive binary operations
   let isCommutable = 1 in {
   def AESE_ZZZ_B : sve2_crypto_des_bin_op<0b00, "aese", ZPR8, int_aarch64_sve_aese, nxv16i8>;
@@ -4087,19 +4116,19 @@ let Predicates = [HasSVE2SM4] in {
   def SM4E_ZZZ_S : sve2_crypto_des_bin_op<0b10, "sm4e", ZPR32, int_aarch64_sve_sm4e, nxv4i32>;
 } // End HasSVE2SM4
 
-let Predicates = [HasSVE2SHA3] in {
+let Predicates = [HasSVESHA3, HasNonStreamingSVE_or_SME2p1] in {
   // SVE2 crypto constructive binary operations
   defm RAX1_ZZZ_D : sve2_crypto_cons_bin_op<0b1, "rax1", ZPR64, int_aarch64_sve_rax1, nxv2i64>;
-} // End HasSVE2SHA3
+} // End HasSVESHA3, HasNonStreamingSVE_or_SME2p1
 
-let Predicates = [HasSVEBitPerm, HasNonStreamingSVE2_or_SSVE_BitPerm] in {
+let Predicates = [HasSVEBitPerm, HasNonStreamingSVE_or_SSVE_BitPerm] in {
   // SVE2 bitwise permute
   defm BEXT_ZZZ : sve2_misc_bitwise<0b1100, "bext", int_aarch64_sve_bext_x>;
   defm BDEP_ZZZ : sve2_misc_bitwise<0b1101, "bdep", int_aarch64_sve_bdep_x>;
   defm BGRP_ZZZ : sve2_misc_bitwise<0b1110, "bgrp", int_aarch64_sve_bgrp_x>;
 }
 
-let Predicates = [HasSVEAES2, HasNonStreamingSVE2_or_SSVE_AES] in {
+let Predicates = [HasSVEAES2, HasNonStreamingSVE_or_SSVE_AES] in {
   // SVE_AES2 multi-vector instructions (x2)
   def AESE_2ZZI_B    : sve_crypto_binary_multi2<0b000, "aese">;
   def AESD_2ZZI_B    : sve_crypto_binary_multi2<0b010, "aesd">;
@@ -4154,11 +4183,6 @@ defm UDOT_ZZZ_HtoS  : sve2p1_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2
 defm SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0, int_aarch64_sve_sdot_lane_x2>;
 defm UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1, int_aarch64_sve_udot_lane_x2>;
 
-defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
-defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
-defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
-defm PTRUE_C  : sve2p1_ptrue_pn<"ptrue">;
-
 defm SQCVTN_Z2Z_StoH  : sve2p1_multi_vec_extract_narrow<"sqcvtn", 0b00, int_aarch64_sve_sqcvtn_x2>;
 defm UQCVTN_Z2Z_StoH  : sve2p1_multi_vec_extract_narrow<"uqcvtn", 0b01, int_aarch64_sve_uqcvtn_x2>;
 defm SQCVTUN_Z2Z_StoH : sve2p1_multi_vec_extract_narrow<"sqcvtun", 0b10, int_aarch64_sve_sqcvtun_x2>;
@@ -4166,6 +4190,22 @@ defm SQRSHRN_Z2ZI_StoH  : sve2p1_multi_vec_shift_narrow<"sqrshrn", 0b101, int_aa
 defm UQRSHRN_Z2ZI_StoH  : sve2p1_multi_vec_shift_narrow<"uqrshrn", 0b111, int_aarch64_sve_uqrshrn_x2>;
 defm SQRSHRUN_Z2ZI_StoH : sve2p1_multi_vec_shift_narrow<"sqrshrun", 0b001, int_aarch64_sve_sqrshrun_x2>;
 
+defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
+defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
+defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
+defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
+defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
+defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
+defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
+defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
+} // End HasSVE2p1_or_SME2
+
+let Predicates = [HasSVE2p1_or_StreamingSME2] in {
+defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
+defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext", int_aarch64_sve_pext>;
+defm PEXT_2PCI : sve2p1_pred_as_ctr_to_mask_pair<"pext">;
+defm PTRUE_C  : sve2p1_ptrue_pn<"ptrue">;
+
 // Load to two registers
 defm LD1B_2Z       : sve2p1_mem_cld_ss_2z<"ld1b", 0b00, 0b0, ZZ_b_mul_r, GPR64shifted8, ZZ_b_strided_and_contiguous>;
 defm LD1H_2Z       : sve2p1_mem_cld_ss_2z<"ld1h", 0b01, 0b0, ZZ_h_mul_r, GPR64shifted16, ZZ_h_strided_and_contiguous>;
@@ -4289,14 +4329,6 @@ defm : store_pn_x4<nxv8bf16, int_aarch64_sve_stnt1_pn_x4, STNT1H_4Z_IMM>;
 defm : store_pn_x4<nxv4f32, int_aarch64_sve_stnt1_pn_x4, STNT1W_4Z_IMM>;
 defm : store_pn_x4<nxv2f64, int_aarch64_sve_stnt1_pn_x4, STNT1D_4Z_IMM>;
 
-defm WHILEGE_2PXX : sve2p1_int_while_rr_pair<"whilege", 0b000>;
-defm WHILEGT_2PXX : sve2p1_int_while_rr_pair<"whilegt", 0b001>;
-defm WHILELT_2PXX : sve2p1_int_while_rr_pair<"whilelt", 0b010>;
-defm WHILELE_2PXX : sve2p1_int_while_rr_pair<"whilele", 0b011>;
-defm WHILEHS_2PXX : sve2p1_int_while_rr_pair<"whilehs", 0b100>;
-defm WHILEHI_2PXX : sve2p1_int_while_rr_pair<"whilehi", 0b101>;
-defm WHILELO_2PXX : sve2p1_int_while_rr_pair<"whilelo", 0b110>;
-defm WHILELS_2PXX : sve2p1_int_while_rr_pair<"whilels", 0b111>;
 defm WHILEGE_CXX  : sve2p1_int_while_rr_pn<"whilege", 0b000>;
 defm WHILEGT_CXX  : sve2p1_int_while_rr_pn<"whilegt", 0b001>;
 defm WHILELT_CXX  : sve2p1_int_while_rr_pn<"whilelt", 0b010>;
@@ -4305,7 +4337,7 @@ defm WHILEHS_CXX  : sve2p1_int_while_rr_pn<"whilehs", 0b100>;
 defm WHILEHI_CXX  : sve2p1_int_while_rr_pn<"whilehi", 0b101>;
 defm WHILELO_CXX  : sve2p1_int_while_rr_pn<"whilelo", 0b110>;
 defm WHILELS_CXX  : sve2p1_int_while_rr_pn<"whilels", 0b111>;
-} // End HasSVE2p1_or_SME2
+} // End HasSVE2p1_or_StreamingSME2
 
 let Predicates = [HasSVE_or_SME] in {
 
@@ -4349,10 +4381,10 @@ defm BFMLS_ZPmZZ : sve_fp_3op_p_zds_a_bfloat<0b01, "bfmls", "BFMLS_ZPZZZ", AArch
 defm BFMLA_ZPZZZ : sve_fp_3op_pred_bfloat<AArch64fmla_p>;
 defm BFMLS_ZPZZZ : sve_fp_3op_pred_bfloat<AArch64fmls_p>;
 
-defm BFMLA_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmla", 0b10, int_aarch64_sve_fmla_lane>;
-defm BFMLS_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmls", 0b11, int_aarch64_sve_fmls_lane>;
+defm BFMLA_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmla", 0b10, AArch64fmlaidx>;
+defm BFMLS_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmls", 0b11, AArch64fmlsidx>;
 
-defm BFMUL_ZZZI : sve_fp_fmul_by_indexed_elem_bfloat<"bfmul", int_aarch64_sve_fmul_lane>;
+defm BFMUL_ZZZI : sve_fp_fmul_by_indexed_elem_bfloat<"bfmul", AArch64fmulidx>;
 
 defm BFCLAMP_ZZZ : sve_fp_clamp_bfloat<"bfclamp", AArch64fclamp>;
 } // End HasSVEB16B16
@@ -4510,7 +4542,7 @@ let Predicates = [HasNonStreamingSVE2p2_or_SME2p2] in {
 //===----------------------------------------------------------------------===//
 // SVE2 FP8 instructions
 //===----------------------------------------------------------------------===//
-let Predicates = [HasSVE2_or_SME2, HasFP8] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasFP8] in {
 // FP8 upconvert
 defm F1CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b00, "f1cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt1>;
 defm F2CVT_ZZ     : sve2_fp8_cvt_single<0b0, 0b01, "f2cvt",    nxv8f16,  int_aarch64_sve_fp8_cvt2>;
@@ -4527,15 +4559,15 @@ defm FCVTNB_Z2Z_StoB : sve2_fp8_down_cvt_single<0b01, "fcvtnb", ZZ_s_mul_r, nxv4
 defm BFCVTN_Z2Z_HtoB : sve2_fp8_down_cvt_single<0b10, "bfcvtn", ZZ_h_mul_r, nxv8bf16, int_aarch64_sve_fp8_cvtn>;
 
 defm FCVTNT_Z2Z_StoB : sve2_fp8_down_cvt_single_top<0b11, "fcvtnt", ZZ_s_mul_r, nxv4f32,  int_aarch64_sve_fp8_cvtnt>;
-} // End HasSVE2_or_SME2, HasFP8
+} // End HasNonStreamingSVE2_or_SME2, HasFP8
 
-let Predicates = [HasSVE2_or_SME2, HasFAMINMAX] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasFAMINMAX] in {
 defm FAMIN_ZPmZ : sve_fp_2op_p_zds<0b1111, "famin", "FAMIN_ZPZZ", int_aarch64_sve_famin, DestructiveBinaryComm>;
 defm FAMAX_ZPmZ : sve_fp_2op_p_zds<0b1110, "famax", "FAMAX_ZPZZ", int_aarch64_sve_famax, DestructiveBinaryComm>;
 
 defm FAMAX_ZPZZ : sve_fp_bin_pred_hfd<AArch64famax_p>;
 defm FAMIN_ZPZZ : sve_fp_bin_pred_hfd<AArch64famin_p>;
-} // End HasSVE2_or_SME2, HasFAMINMAX
+} // End HasNonStreamingSVE2_or_SME2, HasFAMINMAX
 
 let Predicates = [HasSSVE_FP8FMA] in {
 // FP8 Widening Multiply-Add Long - Indexed Group
@@ -4579,14 +4611,14 @@ defm FDOT_ZZZI_BtoS : sve2_fp8_dot_indexed_s<"fdot", int_aarch64_sve_fp8_fdot_la
 defm FDOT_ZZZ_BtoS : sve_fp8_dot<0b1, ZPR32, "fdot", nxv4f32, int_aarch64_sve_fp8_fdot>;
 }
 
-let Predicates = [HasSVE2_or_SME2, HasLUT] in {
+let Predicates = [HasNonStreamingSVE2_or_SME2, HasLUT] in {
 // LUTI2
   defm LUTI2_ZZZI : sve2_luti2_vector_index<"luti2">;
 // LUTI4
   defm LUTI4_ZZZI   : sve2_luti4_vector_index<"luti4">;
 // LUTI4 (two contiguous registers)
   defm LUTI4_Z2ZZI  : sve2_luti4_vector_vg2_index<"luti4">;
-} // End HasSVE2_or_SME2, HasLUT
+} // End HasNonStreamingSVE2_or_SME2, HasLUT
 
 //===----------------------------------------------------------------------===//
 // Checked Pointer Arithmetic (FEAT_CPA)
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
index e23daec97bd2..91a707910a7f 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 def NeoverseN2Model : SchedMachineModel {
-  let IssueWidth            =  10; // Micro-ops dispatched at a time.
+  let IssueWidth            =   5; // Micro-ops dispatched at a time.
   let MicroOpBufferSize     = 160; // Entries in micro-op re-order buffer.
   let LoadLatency           =   4; // Optimistic load latency.
   let MispredictPenalty     =  10; // Extra cycles for mispredicted branch.
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 90f6fc2ea664..bafb8d077323 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -12,7 +12,6 @@
 
 #include "AArch64SelectionDAGInfo.h"
 #include "AArch64MachineFunctionInfo.h"
-#include "AArch64TargetMachine.h"
 
 #define GET_SDNODE_DESC
 #include "AArch64GenSDNodeInfo.inc"
@@ -164,35 +163,34 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
   const AArch64Subtarget &STI =
       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
   const AArch64TargetLowering *TLI = STI.getTargetLowering();
-  SDValue Symbol;
   TargetLowering::ArgListEntry DstEntry;
   DstEntry.Ty = PointerType::getUnqual(*DAG.getContext());
   DstEntry.Node = Dst;
   TargetLowering::ArgListTy Args;
   Args.push_back(DstEntry);
-  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
 
+  RTLIB::Libcall NewLC;
   switch (LC) {
   case RTLIB::MEMCPY: {
+    NewLC = RTLIB::SC_MEMCPY;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memcpy", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMMOVE: {
+    NewLC = RTLIB::SC_MEMMOVE;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memmove", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMSET: {
+    NewLC = RTLIB::SC_MEMSET;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = Type::getInt32Ty(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memset", PointerVT);
     Src = DAG.getZExtOrTrunc(Src, DL, MVT::i32);
     Entry.Node = Src;
     Args.push_back(Entry);
@@ -202,17 +200,17 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
     return SDValue();
   }
 
+  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
+  SDValue Symbol = DAG.getExternalSymbol(TLI->getLibcallName(NewLC), PointerVT);
   TargetLowering::ArgListEntry SizeEntry;
   SizeEntry.Node = Size;
   SizeEntry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
   Args.push_back(SizeEntry);
-  assert(Symbol->getOpcode() == ISD::ExternalSymbol &&
-         "Function name is not set");
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   PointerType *RetTy = PointerType::getUnqual(*DAG.getContext());
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
-      TLI->getLibcallCallingConv(LC), RetTy, Symbol, std::move(Args));
+      TLI->getLibcallCallingConv(NewLC), RetTy, Symbol, std::move(Args));
   return TLI->LowerCallTo(CLI).second;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 8150e91c8ba5..95eab16511e5 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -481,7 +481,8 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const {
         MaxSVEVectorSize, IsStreaming, IsStreamingCompatible, HasMinSize);
   }
 
-  assert((!IsStreaming || I->hasSME()) && "Expected SME to be available");
+  if (IsStreaming && !I->hasSME())
+    reportFatalUsageError("streaming SVE functions require SME");
 
   return I.get();
 }
@@ -511,6 +512,15 @@ AArch64TargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
   return DAG;
 }
 
+size_t AArch64TargetMachine::clearLinkerOptimizationHints(
+    const SmallPtrSetImpl<MachineInstr *> &MIs) const {
+  if (MIs.empty())
+    return 0;
+  auto *MI = *MIs.begin();
+  auto *FuncInfo = MI->getMF()->getInfo<AArch64FunctionInfo>();
+  return FuncInfo->clearLinkerOptimizationHints(MIs);
+}
+
 void AArch64leTargetMachine::anchor() { }
 
 AArch64leTargetMachine::AArch64leTargetMachine(
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index f8ba41f21543..b9e522dd6f22 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -76,6 +76,9 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl {
   ScheduleDAGInstrs *
   createPostMachineScheduler(MachineSchedContext *C) const override;
 
+  size_t clearLinkerOptimizationHints(
+      const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
+
 private:
   bool isLittle;
 };
diff --git a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
index b9ba2a41877e..c218831ce040 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetObjectFile.cpp
@@ -8,7 +8,7 @@
 
 #include "AArch64TargetObjectFile.h"
 #include "AArch64TargetMachine.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64TargetStreamer.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
@@ -25,7 +25,7 @@ using namespace dwarf;
 void AArch64_ELFTargetObjectFile::Initialize(MCContext &Ctx,
                                              const TargetMachine &TM) {
   TargetLoweringObjectFileELF::Initialize(Ctx, TM);
-  PLTRelativeSpecifier = AArch64MCExpr::VK_PLT;
+  PLTRelativeSpecifier = AArch64::S_PLT;
   SupportIndirectSymViaGOTPCRel = true;
 
   // AARCH64 ELF ABI does not define static relocation type for TLS offset
@@ -61,7 +61,7 @@ const MCExpr *AArch64_ELFTargetObjectFile::getIndirectSymViaGOTPCRel(
     int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const {
   int64_t FinalOffset = Offset + MV.getConstant();
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, AArch64MCExpr::VK_GOTPCREL, getContext());
+      MCSymbolRefExpr::create(Sym, AArch64::S_GOTPCREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOffset, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
@@ -80,7 +80,7 @@ const MCExpr *AArch64_MachoTargetObjectFile::getTTypeGlobalReference(
   if (Encoding & (DW_EH_PE_indirect | DW_EH_PE_pcrel)) {
     const MCSymbol *Sym = TM.getSymbol(GV);
     const MCExpr *Res =
-        MCSymbolRefExpr::create(Sym, AArch64MCExpr::M_GOT, getContext());
+        MCSymbolRefExpr::create(Sym, AArch64::S_MACHO_GOT, getContext());
     MCSymbol *PCSym = getContext().createTempSymbol();
     Streamer.emitLabel(PCSym);
     const MCExpr *PC = MCSymbolRefExpr::create(PCSym, getContext());
@@ -105,7 +105,7 @@ const MCExpr *AArch64_MachoTargetObjectFile::getIndirectSymViaGOTPCRel(
   // On ARM64 Darwin, we can reference symbols with foo@GOT-., which
   // is an indirect pc-relative reference.
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, AArch64MCExpr::M_GOT, getContext());
+      MCSymbolRefExpr::create(Sym, AArch64::S_MACHO_GOT, getContext());
   MCSymbol *PCSym = getContext().createTempSymbol();
   Streamer.emitLabel(PCSym);
   const MCExpr *PC = MCSymbolRefExpr::create(PCSym, getContext());
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 15e38e6cb240..380faa6cf693 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4855,7 +4855,7 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
   auto *Latch = L->getLoopLatch();
   SmallVector<BasicBlock *> Preds(predecessors(Latch));
   if (!Term || !Term->isConditional() || Preds.size() == 1 ||
-      none_of(Preds, [Header](BasicBlock *Pred) { return Header == Pred; }) ||
+      !llvm::is_contained(Preds, Header) ||
       none_of(Preds, [L](BasicBlock *Pred) { return L->contains(Pred); }))
     return;
 
@@ -5405,11 +5405,21 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
 
   // Sub opcodes currently only occur in chained cases.
   // Independent partial reduction subtractions are still costed as an add
-  if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
+  if ((Opcode != Instruction::Add && Opcode != Instruction::Sub) ||
+      OpAExtend == TTI::PR_None)
     return Invalid;
 
-  if (InputTypeA != InputTypeB)
+  // We only support multiply binary operations for now, and for muls we
+  // require the types being extended to be the same.
+  // NOTE: For muls AArch64 supports lowering mixed extensions to a usdot but
+  // only if the i8mm or sve/streaming features are available.
+  if (BinOp && (*BinOp != Instruction::Mul || InputTypeA != InputTypeB ||
+                OpBExtend == TTI::PR_None ||
+                (OpAExtend != OpBExtend && !ST->hasMatMulInt8() &&
+                 !ST->isSVEorStreamingSVEAvailable())))
     return Invalid;
+  assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) &&
+         "Unexpected values for OpBExtend or InputTypeB");
 
   EVT InputEVT = EVT::getEVT(InputTypeA);
   EVT AccumEVT = EVT::getEVT(AccumType);
@@ -5456,15 +5466,6 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
   } else
     return Invalid;
 
-  // AArch64 supports lowering mixed fixed-width extensions to a usdot but only
-  // if the i8mm feature is available.
-  if (OpAExtend == TTI::PR_None || OpBExtend == TTI::PR_None ||
-      (OpAExtend != OpBExtend && !ST->hasMatMulInt8()))
-    return Invalid;
-
-  if (!BinOp || *BinOp != Instruction::Mul)
-    return Invalid;
-
   return Cost;
 }
 
@@ -5599,6 +5600,29 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
     SrcTy = DstTy;
   }
 
+  // Segmented shuffle matching.
+  if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(SrcTy) &&
+      !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() &&
+      SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf(
+          AArch64::SVEBitsPerBlock)) {
+
+    FixedVectorType *VTy = cast<FixedVectorType>(SrcTy);
+    unsigned Segments =
+        VTy->getPrimitiveSizeInBits() / AArch64::SVEBitsPerBlock;
+    unsigned SegmentElts = VTy->getNumElements() / Segments;
+
+    // dupq zd.t, zn.t[idx]
+    if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
+        ST->isSVEorStreamingSVEAvailable() &&
+        isDUPQMask(Mask, Segments, SegmentElts))
+      return LT.first;
+
+    // mov zd.q, vn
+    if (ST->isSVEorStreamingSVEAvailable() &&
+        isDUPFirstSegmentMask(Mask, Segments, SegmentElts))
+      return LT.first;
+  }
+
   // Check for broadcast loads, which are supported by the LD1R instruction.
   // In terms of code-size, the shuffle vector is free when a load + dup get
   // folded into a LD1R. That's what we check and return here. For performance
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index faa82abbd898..6712681d8c23 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -9,7 +9,7 @@
 #include "AArch64InstrInfo.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64InstPrinter.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "MCTargetDesc/AArch64TargetStreamer.h"
 #include "TargetInfo/AArch64TargetInfo.h"
@@ -340,9 +340,8 @@ class AArch64AsmParser : public MCTargetAsmParser {
   unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                       unsigned Kind) override;
 
-  static bool classifySymbolRef(const MCExpr *Expr,
-                                AArch64MCExpr::Specifier &ELFSpec,
-                                AArch64MCExpr::Specifier &DarwinSpec,
+  static bool classifySymbolRef(const MCExpr *Expr, AArch64::Specifier &ELFSpec,
+                                AArch64::Specifier &DarwinSpec,
                                 int64_t &Addend);
 };
 
@@ -891,8 +890,8 @@ class AArch64Operand : public MCParsedAsmOperand {
   }
 
   bool isSymbolicUImm12Offset(const MCExpr *Expr) const {
-    AArch64MCExpr::Specifier ELFSpec;
-    AArch64MCExpr::Specifier DarwinSpec;
+    AArch64::Specifier ELFSpec;
+    AArch64::Specifier DarwinSpec;
     int64_t Addend;
     if (!AArch64AsmParser::classifySymbolRef(Expr, ELFSpec, DarwinSpec,
                                              Addend)) {
@@ -901,22 +900,21 @@ class AArch64Operand : public MCParsedAsmOperand {
       return true;
     }
 
-    if (DarwinSpec == AArch64MCExpr::M_PAGEOFF ||
+    if (DarwinSpec == AArch64::S_MACHO_PAGEOFF ||
         llvm::is_contained(
-            {AArch64MCExpr::VK_LO12, AArch64MCExpr::VK_GOT_LO12,
-             AArch64MCExpr::VK_GOT_AUTH_LO12, AArch64MCExpr::VK_DTPREL_LO12,
-             AArch64MCExpr::VK_DTPREL_LO12_NC, AArch64MCExpr::VK_TPREL_LO12,
-             AArch64MCExpr::VK_TPREL_LO12_NC,
-             AArch64MCExpr::VK_GOTTPREL_LO12_NC, AArch64MCExpr::VK_TLSDESC_LO12,
-             AArch64MCExpr::VK_TLSDESC_AUTH_LO12, AArch64MCExpr::VK_SECREL_LO12,
-             AArch64MCExpr::VK_SECREL_HI12, AArch64MCExpr::VK_GOT_PAGE_LO15},
+            {AArch64::S_LO12, AArch64::S_GOT_LO12, AArch64::S_GOT_AUTH_LO12,
+             AArch64::S_DTPREL_LO12, AArch64::S_DTPREL_LO12_NC,
+             AArch64::S_TPREL_LO12, AArch64::S_TPREL_LO12_NC,
+             AArch64::S_GOTTPREL_LO12_NC, AArch64::S_TLSDESC_LO12,
+             AArch64::S_TLSDESC_AUTH_LO12, AArch64::S_SECREL_LO12,
+             AArch64::S_SECREL_HI12, AArch64::S_GOT_PAGE_LO15},
             ELFSpec)) {
       // Note that we don't range-check the addend. It's adjusted modulo page
       // size when converted, so there is no "out of range" condition when using
       // @pageoff.
       return true;
-    } else if (DarwinSpec == AArch64MCExpr::M_GOTPAGEOFF ||
-               DarwinSpec == AArch64MCExpr::M_TLVPPAGEOFF) {
+    } else if (DarwinSpec == AArch64::S_MACHO_GOTPAGEOFF ||
+               DarwinSpec == AArch64::S_MACHO_TLVPPAGEOFF) {
       // @gotpageoff/@tlvppageoff can only be used directly, not with an addend.
       return Addend == 0;
     }
@@ -1008,23 +1006,21 @@ class AArch64Operand : public MCParsedAsmOperand {
       Expr = getImm();
     }
 
-    AArch64MCExpr::Specifier ELFSpec;
-    AArch64MCExpr::Specifier DarwinSpec;
+    AArch64::Specifier ELFSpec;
+    AArch64::Specifier DarwinSpec;
     int64_t Addend;
     if (AArch64AsmParser::classifySymbolRef(Expr, ELFSpec, DarwinSpec,
                                             Addend)) {
-      return DarwinSpec == AArch64MCExpr::M_PAGEOFF ||
-             DarwinSpec == AArch64MCExpr::M_TLVPPAGEOFF ||
-             (DarwinSpec == AArch64MCExpr::M_GOTPAGEOFF && Addend == 0) ||
+      return DarwinSpec == AArch64::S_MACHO_PAGEOFF ||
+             DarwinSpec == AArch64::S_MACHO_TLVPPAGEOFF ||
+             (DarwinSpec == AArch64::S_MACHO_GOTPAGEOFF && Addend == 0) ||
              llvm::is_contained(
-                 {AArch64MCExpr::VK_LO12, AArch64MCExpr::VK_GOT_AUTH_LO12,
-                  AArch64MCExpr::VK_DTPREL_HI12, AArch64MCExpr::VK_DTPREL_LO12,
-                  AArch64MCExpr::VK_DTPREL_LO12_NC,
-                  AArch64MCExpr::VK_TPREL_HI12, AArch64MCExpr::VK_TPREL_LO12,
-                  AArch64MCExpr::VK_TPREL_LO12_NC,
-                  AArch64MCExpr::VK_TLSDESC_LO12,
-                  AArch64MCExpr::VK_TLSDESC_AUTH_LO12,
-                  AArch64MCExpr::VK_SECREL_HI12, AArch64MCExpr::VK_SECREL_LO12},
+                 {AArch64::S_LO12, AArch64::S_GOT_AUTH_LO12,
+                  AArch64::S_DTPREL_HI12, AArch64::S_DTPREL_LO12,
+                  AArch64::S_DTPREL_LO12_NC, AArch64::S_TPREL_HI12,
+                  AArch64::S_TPREL_LO12, AArch64::S_TPREL_LO12_NC,
+                  AArch64::S_TLSDESC_LO12, AArch64::S_TLSDESC_AUTH_LO12,
+                  AArch64::S_SECREL_HI12, AArch64::S_SECREL_LO12},
                  ELFSpec);
     }
 
@@ -1118,51 +1114,48 @@ class AArch64Operand : public MCParsedAsmOperand {
     return (Val >= -((1<<(N-1)) << 2) && Val <= (((1<<(N-1))-1) << 2));
   }
 
-  bool isMovWSymbol(ArrayRef<AArch64MCExpr::Specifier> AllowedModifiers) const {
+  bool isMovWSymbol(ArrayRef<AArch64::Specifier> AllowedModifiers) const {
     if (!isImm())
       return false;
 
-    AArch64MCExpr::Specifier ELFSpec;
-    AArch64MCExpr::Specifier DarwinSpec;
+    AArch64::Specifier ELFSpec;
+    AArch64::Specifier DarwinSpec;
     int64_t Addend;
     if (!AArch64AsmParser::classifySymbolRef(getImm(), ELFSpec, DarwinSpec,
                                              Addend)) {
       return false;
     }
-    if (DarwinSpec != AArch64MCExpr::None)
+    if (DarwinSpec != AArch64::S_None)
       return false;
 
     return llvm::is_contained(AllowedModifiers, ELFSpec);
   }
 
   bool isMovWSymbolG3() const {
-    return isMovWSymbol({AArch64MCExpr::VK_ABS_G3, AArch64MCExpr::VK_PREL_G3});
+    return isMovWSymbol({AArch64::S_ABS_G3, AArch64::S_PREL_G3});
   }
 
   bool isMovWSymbolG2() const {
-    return isMovWSymbol(
-        {AArch64MCExpr::VK_ABS_G2, AArch64MCExpr::VK_ABS_G2_S,
-         AArch64MCExpr::VK_ABS_G2_NC, AArch64MCExpr::VK_PREL_G2,
-         AArch64MCExpr::VK_PREL_G2_NC, AArch64MCExpr::VK_TPREL_G2,
-         AArch64MCExpr::VK_DTPREL_G2});
+    return isMovWSymbol({AArch64::S_ABS_G2, AArch64::S_ABS_G2_S,
+                         AArch64::S_ABS_G2_NC, AArch64::S_PREL_G2,
+                         AArch64::S_PREL_G2_NC, AArch64::S_TPREL_G2,
+                         AArch64::S_DTPREL_G2});
   }
 
   bool isMovWSymbolG1() const {
-    return isMovWSymbol(
-        {AArch64MCExpr::VK_ABS_G1, AArch64MCExpr::VK_ABS_G1_S,
-         AArch64MCExpr::VK_ABS_G1_NC, AArch64MCExpr::VK_PREL_G1,
-         AArch64MCExpr::VK_PREL_G1_NC, AArch64MCExpr::VK_GOTTPREL_G1,
-         AArch64MCExpr::VK_TPREL_G1, AArch64MCExpr::VK_TPREL_G1_NC,
-         AArch64MCExpr::VK_DTPREL_G1, AArch64MCExpr::VK_DTPREL_G1_NC});
+    return isMovWSymbol({AArch64::S_ABS_G1, AArch64::S_ABS_G1_S,
+                         AArch64::S_ABS_G1_NC, AArch64::S_PREL_G1,
+                         AArch64::S_PREL_G1_NC, AArch64::S_GOTTPREL_G1,
+                         AArch64::S_TPREL_G1, AArch64::S_TPREL_G1_NC,
+                         AArch64::S_DTPREL_G1, AArch64::S_DTPREL_G1_NC});
   }
 
   bool isMovWSymbolG0() const {
-    return isMovWSymbol(
-        {AArch64MCExpr::VK_ABS_G0, AArch64MCExpr::VK_ABS_G0_S,
-         AArch64MCExpr::VK_ABS_G0_NC, AArch64MCExpr::VK_PREL_G0,
-         AArch64MCExpr::VK_PREL_G0_NC, AArch64MCExpr::VK_GOTTPREL_G0_NC,
-         AArch64MCExpr::VK_TPREL_G0, AArch64MCExpr::VK_TPREL_G0_NC,
-         AArch64MCExpr::VK_DTPREL_G0, AArch64MCExpr::VK_DTPREL_G0_NC});
+    return isMovWSymbol({AArch64::S_ABS_G0, AArch64::S_ABS_G0_S,
+                         AArch64::S_ABS_G0_NC, AArch64::S_PREL_G0,
+                         AArch64::S_PREL_G0_NC, AArch64::S_GOTTPREL_G0_NC,
+                         AArch64::S_TPREL_G0, AArch64::S_TPREL_G0_NC,
+                         AArch64::S_DTPREL_G0, AArch64::S_DTPREL_G0_NC});
   }
 
   template<int RegWidth, int Shift>
@@ -2301,7 +2294,7 @@ class AArch64Operand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createImm((MCE->getValue() - 90) / 180));
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<AArch64Operand>
   CreateToken(StringRef Str, SMLoc S, MCContext &Ctx, bool IsSuffix = false) {
@@ -2591,7 +2584,7 @@ class AArch64Operand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void AArch64Operand::print(raw_ostream &OS) const {
+void AArch64Operand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case k_FPImm:
     OS << "<fpimm " << getFPImm().bitcastToAPInt().getZExtValue();
@@ -2608,12 +2601,12 @@ void AArch64Operand::print(raw_ostream &OS) const {
     break;
   }
   case k_Immediate:
-    OS << *getImm();
+    MAI.printExpr(OS, *getImm());
     break;
   case k_ShiftedImm: {
     unsigned Shift = getShiftedImmShift();
     OS << "<shiftedimm ";
-    OS << *getShiftedImmVal();
+    MAI.printExpr(OS, *getShiftedImmVal());
     OS << ", lsl #" << AArch64_AM::getShiftValue(Shift) << ">";
     break;
   }
@@ -3299,30 +3292,29 @@ ParseStatus AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
   if (parseSymbolicImmVal(Expr))
     return ParseStatus::Failure;
 
-  AArch64MCExpr::Specifier ELFSpec;
-  AArch64MCExpr::Specifier DarwinSpec;
+  AArch64::Specifier ELFSpec;
+  AArch64::Specifier DarwinSpec;
   int64_t Addend;
   if (classifySymbolRef(Expr, ELFSpec, DarwinSpec, Addend)) {
-    if (DarwinSpec == AArch64MCExpr::None &&
-        ELFSpec == AArch64MCExpr::VK_INVALID) {
+    if (DarwinSpec == AArch64::S_None && ELFSpec == AArch64::S_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADRP relocation (unfortunately).
-      Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE,
-                                     getContext());
-    } else if ((DarwinSpec == AArch64MCExpr::M_GOTPAGE ||
-                DarwinSpec == AArch64MCExpr::M_TLVPPAGE) &&
+      Expr =
+          MCSpecifierExpr::create(Expr, AArch64::S_ABS_PAGE, getContext(), S);
+    } else if ((DarwinSpec == AArch64::S_MACHO_GOTPAGE ||
+                DarwinSpec == AArch64::S_MACHO_TLVPPAGE) &&
                Addend != 0) {
       return Error(S, "gotpage label reference not allowed an addend");
-    } else if (DarwinSpec != AArch64MCExpr::M_PAGE &&
-               DarwinSpec != AArch64MCExpr::M_GOTPAGE &&
-               DarwinSpec != AArch64MCExpr::M_TLVPPAGE &&
-               ELFSpec != AArch64MCExpr::VK_ABS_PAGE_NC &&
-               ELFSpec != AArch64MCExpr::VK_GOT_PAGE &&
-               ELFSpec != AArch64MCExpr::VK_GOT_AUTH_PAGE &&
-               ELFSpec != AArch64MCExpr::VK_GOT_PAGE_LO15 &&
-               ELFSpec != AArch64MCExpr::VK_GOTTPREL_PAGE &&
-               ELFSpec != AArch64MCExpr::VK_TLSDESC_PAGE &&
-               ELFSpec != AArch64MCExpr::VK_TLSDESC_AUTH_PAGE) {
+    } else if (DarwinSpec != AArch64::S_MACHO_PAGE &&
+               DarwinSpec != AArch64::S_MACHO_GOTPAGE &&
+               DarwinSpec != AArch64::S_MACHO_TLVPPAGE &&
+               ELFSpec != AArch64::S_ABS_PAGE_NC &&
+               ELFSpec != AArch64::S_GOT_PAGE &&
+               ELFSpec != AArch64::S_GOT_AUTH_PAGE &&
+               ELFSpec != AArch64::S_GOT_PAGE_LO15 &&
+               ELFSpec != AArch64::S_GOTTPREL_PAGE &&
+               ELFSpec != AArch64::S_TLSDESC_PAGE &&
+               ELFSpec != AArch64::S_TLSDESC_AUTH_PAGE) {
       // The operand must be an @page or @gotpage qualified symbolref.
       return Error(S, "page or gotpage label reference expected");
     }
@@ -3353,16 +3345,15 @@ ParseStatus AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
   if (parseSymbolicImmVal(Expr))
     return ParseStatus::Failure;
 
-  AArch64MCExpr::Specifier ELFSpec;
-  AArch64MCExpr::Specifier DarwinSpec;
+  AArch64::Specifier ELFSpec;
+  AArch64::Specifier DarwinSpec;
   int64_t Addend;
   if (classifySymbolRef(Expr, ELFSpec, DarwinSpec, Addend)) {
-    if (DarwinSpec == AArch64MCExpr::None &&
-        ELFSpec == AArch64MCExpr::VK_INVALID) {
+    if (DarwinSpec == AArch64::S_None && ELFSpec == AArch64::S_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADR relocation (unfortunately).
-      Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS, getContext());
-    } else if (ELFSpec != AArch64MCExpr::VK_GOT_AUTH_PAGE) {
+      Expr = MCSpecifierExpr::create(Expr, AArch64::S_ABS, getContext(), S);
+    } else if (ELFSpec != AArch64::S_GOT_AUTH_PAGE) {
       // For tiny code model, we use :got_auth: operator to fill 21-bit imm of
       // adr. It's not actually GOT entry page address but the GOT address
       // itself - we just share the same variant kind with :got_auth: operator
@@ -3752,7 +3743,8 @@ static const struct Extension {
     {"sve-aes", {AArch64::FeatureSVEAES}},
     {"sve2-aes", {AArch64::FeatureAliasSVE2AES, AArch64::FeatureSVEAES}},
     {"sve2-sm4", {AArch64::FeatureSVE2SM4}},
-    {"sve2-sha3", {AArch64::FeatureSVE2SHA3}},
+    {"sve-sha3", {AArch64::FeatureSVESHA3}},
+    {"sve2-sha3", {AArch64::FeatureAliasSVE2SHA3, AArch64::FeatureSVESHA3}},
     {"sve-bitperm", {AArch64::FeatureSVEBitPerm}},
     {"sve2-bitperm",
      {AArch64::FeatureAliasSVE2BitPerm, AArch64::FeatureSVEBitPerm,
@@ -4402,8 +4394,8 @@ bool AArch64AsmParser::parseRegister(OperandVector &Operands) {
 
 bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
   bool HasELFModifier = false;
-  AArch64MCExpr::Specifier RefKind;
-
+  AArch64::Specifier RefKind;
+  SMLoc Loc = getLexer().getLoc();
   if (parseOptionalToken(AsmToken::Colon)) {
     HasELFModifier = true;
 
@@ -4411,61 +4403,60 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
       return TokError("expect relocation specifier in operand after ':'");
 
     std::string LowerCase = getTok().getIdentifier().lower();
-    RefKind =
-        StringSwitch<AArch64MCExpr::Specifier>(LowerCase)
-            .Case("lo12", AArch64MCExpr::VK_LO12)
-            .Case("abs_g3", AArch64MCExpr::VK_ABS_G3)
-            .Case("abs_g2", AArch64MCExpr::VK_ABS_G2)
-            .Case("abs_g2_s", AArch64MCExpr::VK_ABS_G2_S)
-            .Case("abs_g2_nc", AArch64MCExpr::VK_ABS_G2_NC)
-            .Case("abs_g1", AArch64MCExpr::VK_ABS_G1)
-            .Case("abs_g1_s", AArch64MCExpr::VK_ABS_G1_S)
-            .Case("abs_g1_nc", AArch64MCExpr::VK_ABS_G1_NC)
-            .Case("abs_g0", AArch64MCExpr::VK_ABS_G0)
-            .Case("abs_g0_s", AArch64MCExpr::VK_ABS_G0_S)
-            .Case("abs_g0_nc", AArch64MCExpr::VK_ABS_G0_NC)
-            .Case("prel_g3", AArch64MCExpr::VK_PREL_G3)
-            .Case("prel_g2", AArch64MCExpr::VK_PREL_G2)
-            .Case("prel_g2_nc", AArch64MCExpr::VK_PREL_G2_NC)
-            .Case("prel_g1", AArch64MCExpr::VK_PREL_G1)
-            .Case("prel_g1_nc", AArch64MCExpr::VK_PREL_G1_NC)
-            .Case("prel_g0", AArch64MCExpr::VK_PREL_G0)
-            .Case("prel_g0_nc", AArch64MCExpr::VK_PREL_G0_NC)
-            .Case("dtprel_g2", AArch64MCExpr::VK_DTPREL_G2)
-            .Case("dtprel_g1", AArch64MCExpr::VK_DTPREL_G1)
-            .Case("dtprel_g1_nc", AArch64MCExpr::VK_DTPREL_G1_NC)
-            .Case("dtprel_g0", AArch64MCExpr::VK_DTPREL_G0)
-            .Case("dtprel_g0_nc", AArch64MCExpr::VK_DTPREL_G0_NC)
-            .Case("dtprel_hi12", AArch64MCExpr::VK_DTPREL_HI12)
-            .Case("dtprel_lo12", AArch64MCExpr::VK_DTPREL_LO12)
-            .Case("dtprel_lo12_nc", AArch64MCExpr::VK_DTPREL_LO12_NC)
-            .Case("pg_hi21_nc", AArch64MCExpr::VK_ABS_PAGE_NC)
-            .Case("tprel_g2", AArch64MCExpr::VK_TPREL_G2)
-            .Case("tprel_g1", AArch64MCExpr::VK_TPREL_G1)
-            .Case("tprel_g1_nc", AArch64MCExpr::VK_TPREL_G1_NC)
-            .Case("tprel_g0", AArch64MCExpr::VK_TPREL_G0)
-            .Case("tprel_g0_nc", AArch64MCExpr::VK_TPREL_G0_NC)
-            .Case("tprel_hi12", AArch64MCExpr::VK_TPREL_HI12)
-            .Case("tprel_lo12", AArch64MCExpr::VK_TPREL_LO12)
-            .Case("tprel_lo12_nc", AArch64MCExpr::VK_TPREL_LO12_NC)
-            .Case("tlsdesc_lo12", AArch64MCExpr::VK_TLSDESC_LO12)
-            .Case("tlsdesc_auth_lo12", AArch64MCExpr::VK_TLSDESC_AUTH_LO12)
-            .Case("got", AArch64MCExpr::VK_GOT_PAGE)
-            .Case("gotpage_lo15", AArch64MCExpr::VK_GOT_PAGE_LO15)
-            .Case("got_lo12", AArch64MCExpr::VK_GOT_LO12)
-            .Case("got_auth", AArch64MCExpr::VK_GOT_AUTH_PAGE)
-            .Case("got_auth_lo12", AArch64MCExpr::VK_GOT_AUTH_LO12)
-            .Case("gottprel", AArch64MCExpr::VK_GOTTPREL_PAGE)
-            .Case("gottprel_lo12", AArch64MCExpr::VK_GOTTPREL_LO12_NC)
-            .Case("gottprel_g1", AArch64MCExpr::VK_GOTTPREL_G1)
-            .Case("gottprel_g0_nc", AArch64MCExpr::VK_GOTTPREL_G0_NC)
-            .Case("tlsdesc", AArch64MCExpr::VK_TLSDESC_PAGE)
-            .Case("tlsdesc_auth", AArch64MCExpr::VK_TLSDESC_AUTH_PAGE)
-            .Case("secrel_lo12", AArch64MCExpr::VK_SECREL_LO12)
-            .Case("secrel_hi12", AArch64MCExpr::VK_SECREL_HI12)
-            .Default(AArch64MCExpr::VK_INVALID);
-
-    if (RefKind == AArch64MCExpr::VK_INVALID)
+    RefKind = StringSwitch<AArch64::Specifier>(LowerCase)
+                  .Case("lo12", AArch64::S_LO12)
+                  .Case("abs_g3", AArch64::S_ABS_G3)
+                  .Case("abs_g2", AArch64::S_ABS_G2)
+                  .Case("abs_g2_s", AArch64::S_ABS_G2_S)
+                  .Case("abs_g2_nc", AArch64::S_ABS_G2_NC)
+                  .Case("abs_g1", AArch64::S_ABS_G1)
+                  .Case("abs_g1_s", AArch64::S_ABS_G1_S)
+                  .Case("abs_g1_nc", AArch64::S_ABS_G1_NC)
+                  .Case("abs_g0", AArch64::S_ABS_G0)
+                  .Case("abs_g0_s", AArch64::S_ABS_G0_S)
+                  .Case("abs_g0_nc", AArch64::S_ABS_G0_NC)
+                  .Case("prel_g3", AArch64::S_PREL_G3)
+                  .Case("prel_g2", AArch64::S_PREL_G2)
+                  .Case("prel_g2_nc", AArch64::S_PREL_G2_NC)
+                  .Case("prel_g1", AArch64::S_PREL_G1)
+                  .Case("prel_g1_nc", AArch64::S_PREL_G1_NC)
+                  .Case("prel_g0", AArch64::S_PREL_G0)
+                  .Case("prel_g0_nc", AArch64::S_PREL_G0_NC)
+                  .Case("dtprel_g2", AArch64::S_DTPREL_G2)
+                  .Case("dtprel_g1", AArch64::S_DTPREL_G1)
+                  .Case("dtprel_g1_nc", AArch64::S_DTPREL_G1_NC)
+                  .Case("dtprel_g0", AArch64::S_DTPREL_G0)
+                  .Case("dtprel_g0_nc", AArch64::S_DTPREL_G0_NC)
+                  .Case("dtprel_hi12", AArch64::S_DTPREL_HI12)
+                  .Case("dtprel_lo12", AArch64::S_DTPREL_LO12)
+                  .Case("dtprel_lo12_nc", AArch64::S_DTPREL_LO12_NC)
+                  .Case("pg_hi21_nc", AArch64::S_ABS_PAGE_NC)
+                  .Case("tprel_g2", AArch64::S_TPREL_G2)
+                  .Case("tprel_g1", AArch64::S_TPREL_G1)
+                  .Case("tprel_g1_nc", AArch64::S_TPREL_G1_NC)
+                  .Case("tprel_g0", AArch64::S_TPREL_G0)
+                  .Case("tprel_g0_nc", AArch64::S_TPREL_G0_NC)
+                  .Case("tprel_hi12", AArch64::S_TPREL_HI12)
+                  .Case("tprel_lo12", AArch64::S_TPREL_LO12)
+                  .Case("tprel_lo12_nc", AArch64::S_TPREL_LO12_NC)
+                  .Case("tlsdesc_lo12", AArch64::S_TLSDESC_LO12)
+                  .Case("tlsdesc_auth_lo12", AArch64::S_TLSDESC_AUTH_LO12)
+                  .Case("got", AArch64::S_GOT_PAGE)
+                  .Case("gotpage_lo15", AArch64::S_GOT_PAGE_LO15)
+                  .Case("got_lo12", AArch64::S_GOT_LO12)
+                  .Case("got_auth", AArch64::S_GOT_AUTH_PAGE)
+                  .Case("got_auth_lo12", AArch64::S_GOT_AUTH_LO12)
+                  .Case("gottprel", AArch64::S_GOTTPREL_PAGE)
+                  .Case("gottprel_lo12", AArch64::S_GOTTPREL_LO12_NC)
+                  .Case("gottprel_g1", AArch64::S_GOTTPREL_G1)
+                  .Case("gottprel_g0_nc", AArch64::S_GOTTPREL_G0_NC)
+                  .Case("tlsdesc", AArch64::S_TLSDESC_PAGE)
+                  .Case("tlsdesc_auth", AArch64::S_TLSDESC_AUTH_PAGE)
+                  .Case("secrel_lo12", AArch64::S_SECREL_LO12)
+                  .Case("secrel_hi12", AArch64::S_SECREL_HI12)
+                  .Default(AArch64::S_INVALID);
+
+    if (RefKind == AArch64::S_INVALID)
       return TokError("expect relocation specifier in operand after ':'");
 
     Lex(); // Eat identifier
@@ -4478,7 +4469,7 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
     return true;
 
   if (HasELFModifier)
-    ImmVal = MCSpecifierExpr::create(ImmVal, RefKind, getContext());
+    ImmVal = MCSpecifierExpr::create(ImmVal, RefKind, getContext(), Loc);
 
   SMLoc EndLoc;
   if (getContext().getAsmInfo()->hasSubsectionsViaSymbols()) {
@@ -5846,26 +5837,25 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, SMLoc &IDLoc,
     // some slight duplication here.
     if (Inst.getOperand(2).isExpr()) {
       const MCExpr *Expr = Inst.getOperand(2).getExpr();
-      AArch64MCExpr::Specifier ELFSpec;
-      AArch64MCExpr::Specifier DarwinSpec;
+      AArch64::Specifier ELFSpec;
+      AArch64::Specifier DarwinSpec;
       int64_t Addend;
       if (classifySymbolRef(Expr, ELFSpec, DarwinSpec, Addend)) {
 
         // Only allow these with ADDXri.
-        if ((DarwinSpec == AArch64MCExpr::M_PAGEOFF ||
-             DarwinSpec == AArch64MCExpr::M_TLVPPAGEOFF) &&
+        if ((DarwinSpec == AArch64::S_MACHO_PAGEOFF ||
+             DarwinSpec == AArch64::S_MACHO_TLVPPAGEOFF) &&
             Inst.getOpcode() == AArch64::ADDXri)
           return false;
 
         // Only allow these with ADDXri/ADDWri
         if (llvm::is_contained(
-                {AArch64MCExpr::VK_LO12, AArch64MCExpr::VK_GOT_AUTH_LO12,
-                 AArch64MCExpr::VK_DTPREL_HI12, AArch64MCExpr::VK_DTPREL_LO12,
-                 AArch64MCExpr::VK_DTPREL_LO12_NC, AArch64MCExpr::VK_TPREL_HI12,
-                 AArch64MCExpr::VK_TPREL_LO12, AArch64MCExpr::VK_TPREL_LO12_NC,
-                 AArch64MCExpr::VK_TLSDESC_LO12,
-                 AArch64MCExpr::VK_TLSDESC_AUTH_LO12,
-                 AArch64MCExpr::VK_SECREL_LO12, AArch64MCExpr::VK_SECREL_HI12},
+                {AArch64::S_LO12, AArch64::S_GOT_AUTH_LO12,
+                 AArch64::S_DTPREL_HI12, AArch64::S_DTPREL_LO12,
+                 AArch64::S_DTPREL_LO12_NC, AArch64::S_TPREL_HI12,
+                 AArch64::S_TPREL_LO12, AArch64::S_TPREL_LO12_NC,
+                 AArch64::S_TLSDESC_LO12, AArch64::S_TLSDESC_AUTH_LO12,
+                 AArch64::S_SECREL_LO12, AArch64::S_SECREL_HI12},
                 ELFSpec) &&
             (Inst.getOpcode() == AArch64::ADDXri ||
              Inst.getOpcode() == AArch64::ADDWri))
@@ -7360,7 +7350,7 @@ bool AArch64AsmParser::parseDirectiveTLSDescCall(SMLoc L) {
 
   MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
-  Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_TLSDESC, getContext());
+  Expr = MCSpecifierExpr::create(Expr, AArch64::S_TLSDESC, getContext());
 
   MCInst Inst;
   Inst.setOpcode(AArch64::TLSDESCCALL);
@@ -8192,18 +8182,18 @@ bool AArch64AsmParser::parseDataExpr(const MCExpr *&Res) {
   if (Identifier == "auth")
     return parseAuthExpr(Res, EndLoc);
 
-  auto Spec = AArch64MCExpr::None;
+  auto Spec = AArch64::S_None;
   if (STI->getTargetTriple().isOSBinFormatMachO()) {
     if (Identifier == "got")
-      Spec = AArch64MCExpr::M_GOT;
+      Spec = AArch64::S_MACHO_GOT;
   } else {
     // Unofficial, experimental syntax that will be changed.
     if (Identifier == "gotpcrel")
-      Spec = AArch64MCExpr::VK_GOTPCREL;
+      Spec = AArch64::S_GOTPCREL;
     else if (Identifier == "plt")
-      Spec = AArch64MCExpr::VK_PLT;
+      Spec = AArch64::S_PLT;
   }
-  if (Spec == AArch64MCExpr::None)
+  if (Spec == AArch64::S_None)
     return Error(Loc, "invalid relocation specifier");
   if (auto *SRE = dyn_cast<MCSymbolRefExpr>(Res))
     Res = MCSymbolRefExpr::create(&SRE->getSymbol(), Spec, getContext(),
@@ -8222,7 +8212,7 @@ bool AArch64AsmParser::parseDataExpr(const MCExpr *&Res) {
     const MCExpr *Term;
     if (getParser().parsePrimaryExpr(Term, EndLoc, nullptr))
       return true;
-    Res = MCBinaryExpr::create(*Opcode, Res, Term, getContext());
+    Res = MCBinaryExpr::create(*Opcode, Res, Term, getContext(), Res->getLoc());
   }
   return false;
 }
@@ -8276,16 +8266,16 @@ bool AArch64AsmParser::parseAuthExpr(const MCExpr *&Res, SMLoc &EndLoc) {
     return true;
 
   Res = AArch64AuthMCExpr::create(Res, Discriminator, *KeyIDOrNone,
-                                  UseAddressDiversity, Ctx);
+                                  UseAddressDiversity, Ctx, Res->getLoc());
   return false;
 }
 
 bool AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
-                                         AArch64MCExpr::Specifier &ELFSpec,
-                                         AArch64MCExpr::Specifier &DarwinSpec,
+                                         AArch64::Specifier &ELFSpec,
+                                         AArch64::Specifier &DarwinSpec,
                                          int64_t &Addend) {
-  ELFSpec = AArch64MCExpr::VK_INVALID;
-  DarwinSpec = AArch64MCExpr::None;
+  ELFSpec = AArch64::S_INVALID;
+  DarwinSpec = AArch64::S_None;
   Addend = 0;
 
   if (auto *AE = dyn_cast<MCSpecifierExpr>(Expr)) {
@@ -8296,7 +8286,7 @@ bool AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
   const MCSymbolRefExpr *SE = dyn_cast<MCSymbolRefExpr>(Expr);
   if (SE) {
     // It's a simple symbol reference with no addend.
-    DarwinSpec = AArch64MCExpr::Specifier(SE->getKind());
+    DarwinSpec = AArch64::Specifier(SE->getKind());
     return true;
   }
 
@@ -8308,17 +8298,16 @@ bool AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
 
   // Treat expressions with an ELFSpec (like ":abs_g1:3", or
   // ":abs_g1:x" where x is constant) as symbolic even if there is no symbol.
-  if (!Res.getAddSym() && ELFSpec == AArch64MCExpr::VK_INVALID)
+  if (!Res.getAddSym() && ELFSpec == AArch64::S_INVALID)
     return false;
 
   if (Res.getAddSym())
-    DarwinSpec = AArch64MCExpr::Specifier(Res.getSpecifier());
+    DarwinSpec = AArch64::Specifier(Res.getSpecifier());
   Addend = Res.getConstant();
 
   // It's some symbol reference + a constant addend, but really
   // shouldn't use both Darwin and ELF syntax.
-  return ELFSpec == AArch64MCExpr::VK_INVALID ||
-         DarwinSpec == AArch64MCExpr::None;
+  return ELFSpec == AArch64::S_INVALID || DarwinSpec == AArch64::S_None;
 }
 
 /// Force static initialization.
diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
index f2528bc12104..3a8a0ef46b03 100644
--- a/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
+++ b/llvm/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64ExternalSymbolizer.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
@@ -19,23 +19,23 @@ using namespace llvm;
 
 #define DEBUG_TYPE "aarch64-disassembler"
 
-static AArch64MCExpr::Specifier
+static AArch64::Specifier
 getMachOSpecifier(uint64_t LLVMDisassembler_VariantKind) {
   switch (LLVMDisassembler_VariantKind) {
   case LLVMDisassembler_VariantKind_None:
-    return AArch64MCExpr::None;
+    return AArch64::S_None;
   case LLVMDisassembler_VariantKind_ARM64_PAGE:
-    return AArch64MCExpr::M_PAGE;
+    return AArch64::S_MACHO_PAGE;
   case LLVMDisassembler_VariantKind_ARM64_PAGEOFF:
-    return AArch64MCExpr::M_PAGEOFF;
+    return AArch64::S_MACHO_PAGEOFF;
   case LLVMDisassembler_VariantKind_ARM64_GOTPAGE:
-    return AArch64MCExpr::M_GOTPAGE;
+    return AArch64::S_MACHO_GOTPAGE;
   case LLVMDisassembler_VariantKind_ARM64_GOTPAGEOFF:
-    return AArch64MCExpr::M_GOTPAGEOFF;
+    return AArch64::S_MACHO_GOTPAGEOFF;
   case LLVMDisassembler_VariantKind_ARM64_TLVP:
-    return AArch64MCExpr::M_TLVPPAGE;
+    return AArch64::S_MACHO_TLVPPAGE;
   case LLVMDisassembler_VariantKind_ARM64_TLVOFF:
-    return AArch64MCExpr::M_TLVPPAGEOFF;
+    return AArch64::S_MACHO_TLVPPAGEOFF;
   default:
     llvm_unreachable("bad LLVMDisassembler_VariantKind");
   }
@@ -171,7 +171,7 @@ bool AArch64ExternalSymbolizer::tryAddingSymbolicOperand(
       StringRef Name(SymbolicOp.AddSymbol.Name);
       MCSymbol *Sym = Ctx.getOrCreateSymbol(Name);
       auto Spec = getMachOSpecifier(SymbolicOp.VariantKind);
-      if (Spec != AArch64MCExpr::None)
+      if (Spec != AArch64::S_None)
         Add = MCSymbolRefExpr::create(Sym, Spec, Ctx);
       else
         Add = MCSymbolRefExpr::create(Sym, Ctx);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 12843e16d0da..75f9a44f61e6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -923,16 +923,28 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .moreElementsToNextPow2(1)
       .widenScalarOrEltToNextPow2OrMinSize(1)
       .minScalar(1, s32)
+      .lowerIf([](const LegalityQuery &Query) {
+        return Query.Types[1].isVector() &&
+               Query.Types[1].getScalarSizeInBits() == 64 &&
+               Query.Types[0].getScalarSizeInBits() == 16;
+      })
       .widenScalarOrEltToNextPow2OrMinSize(0, /*MinSize=*/HasFP16 ? 16 : 32)
+      .scalarizeIf(
+          // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
+          [](const LegalityQuery &Query) {
+            return Query.Types[0].getScalarSizeInBits() == 32 &&
+                   Query.Types[1].getScalarSizeInBits() == 64;
+          },
+          0)
       .widenScalarIf(
-          [=](const LegalityQuery &Query) {
+          [](const LegalityQuery &Query) {
             return Query.Types[1].getScalarSizeInBits() <= 64 &&
                    Query.Types[0].getScalarSizeInBits() <
                        Query.Types[1].getScalarSizeInBits();
           },
           LegalizeMutations::changeElementSizeTo(0, 1))
       .widenScalarIf(
-          [=](const LegalityQuery &Query) {
+          [](const LegalityQuery &Query) {
             return Query.Types[0].getScalarSizeInBits() <= 64 &&
                    Query.Types[0].getScalarSizeInBits() >
                        Query.Types[1].getScalarSizeInBits();
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 88ba2ef3fe1f..84884d98e6f9 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -30,8 +29,6 @@ using namespace llvm;
 namespace {
 
 class AArch64AsmBackend : public MCAsmBackend {
-  static const unsigned PCRelFlagVal =
-      MCFixupKindInfo::FKF_IsAlignedDownTo32Bits | MCFixupKindInfo::FKF_IsPCRel;
 protected:
   Triple TheTriple;
 
@@ -50,27 +47,27 @@ class AArch64AsmBackend : public MCAsmBackend {
         // in AArch64FixupKinds.h.
         //
         // Name                           Offset (bits) Size (bits)     Flags
-        {"fixup_aarch64_pcrel_adr_imm21", 0, 32, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_adrp_imm21", 0, 32, PCRelFlagVal},
+        {"fixup_aarch64_pcrel_adr_imm21", 0, 32, 0},
+        {"fixup_aarch64_pcrel_adrp_imm21", 0, 32, 0},
         {"fixup_aarch64_add_imm12", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale1", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale2", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale4", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale8", 10, 12, 0},
         {"fixup_aarch64_ldst_imm12_scale16", 10, 12, 0},
-        {"fixup_aarch64_ldr_pcrel_imm19", 5, 19, PCRelFlagVal},
+        {"fixup_aarch64_ldr_pcrel_imm19", 5, 19, 0},
         {"fixup_aarch64_movw", 5, 16, 0},
-        {"fixup_aarch64_pcrel_branch9", 5, 9,  PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch14", 5, 14, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch16", 5, 16, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch19", 5, 19, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_branch26", 0, 26, PCRelFlagVal},
-        {"fixup_aarch64_pcrel_call26", 0, 26, PCRelFlagVal}};
+        {"fixup_aarch64_pcrel_branch9", 5, 9, 0},
+        {"fixup_aarch64_pcrel_branch14", 5, 14, 0},
+        {"fixup_aarch64_pcrel_branch16", 5, 16, 0},
+        {"fixup_aarch64_pcrel_branch19", 5, 19, 0},
+        {"fixup_aarch64_pcrel_branch26", 0, 26, 0},
+        {"fixup_aarch64_pcrel_call26", 0, 26, 0}};
 
     // Fixup kinds from raw relocation types and .reloc directives force
     // relocations and do not need these fields.
     if (mc::isRelocation(Kind))
-      return MCAsmBackend::getFixupKindInfo(FK_NONE);
+      return {};
 
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
@@ -87,15 +84,10 @@ class AArch64AsmBackend : public MCAsmBackend {
 
   bool fixupNeedsRelaxation(const MCFixup &Fixup,
                             uint64_t Value) const override;
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override;
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
   unsigned getFixupKindContainereSizeInBytes(unsigned Kind) const;
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
 };
 
 } // end anonymous namespace
@@ -219,10 +211,10 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       Ctx.reportError(Fixup.getLoc(), "fixup must be 16-byte aligned");
     return Value >> 4;
   case AArch64::fixup_aarch64_movw: {
-    AArch64MCExpr::Specifier RefKind =
-        static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-    if (AArch64::getSymbolLoc(RefKind) != AArch64MCExpr::VK_ABS &&
-        AArch64::getSymbolLoc(RefKind) != AArch64MCExpr::VK_SABS) {
+    AArch64::Specifier RefKind =
+        static_cast<AArch64::Specifier>(Target.getSpecifier());
+    if (AArch64::getSymbolLoc(RefKind) != AArch64::S_ABS &&
+        AArch64::getSymbolLoc(RefKind) != AArch64::S_SABS) {
       if (!RefKind) {
         // The fixup is an expression
         if (SignedValue > 0xFFFF || SignedValue < -0xFFFF)
@@ -250,17 +242,17 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       return Value;
     }
 
-    if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
+    if (AArch64::getSymbolLoc(RefKind) == AArch64::S_SABS) {
       switch (AArch64::getAddressFrag(RefKind)) {
-      case AArch64MCExpr::VK_G0:
+      case AArch64::S_G0:
         break;
-      case AArch64MCExpr::VK_G1:
+      case AArch64::S_G1:
         SignedValue = SignedValue >> 16;
         break;
-      case AArch64MCExpr::VK_G2:
+      case AArch64::S_G2:
         SignedValue = SignedValue >> 32;
         break;
-      case AArch64MCExpr::VK_G3:
+      case AArch64::S_G3:
         SignedValue = SignedValue >> 48;
         break;
       default:
@@ -269,15 +261,15 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
 
     } else {
       switch (AArch64::getAddressFrag(RefKind)) {
-      case AArch64MCExpr::VK_G0:
+      case AArch64::S_G0:
         break;
-      case AArch64MCExpr::VK_G1:
+      case AArch64::S_G1:
         Value = Value >> 16;
         break;
-      case AArch64MCExpr::VK_G2:
+      case AArch64::S_G2:
         Value = Value >> 32;
         break;
-      case AArch64MCExpr::VK_G3:
+      case AArch64::S_G3:
         Value = Value >> 48;
         break;
       default:
@@ -285,9 +277,9 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       }
     }
 
-    if (RefKind & AArch64MCExpr::VK_NC) {
+    if (RefKind & AArch64::S_NC) {
       Value &= 0xFFFF;
-    } else if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
+    } else if (AArch64::getSymbolLoc(RefKind) == AArch64::S_SABS) {
       if (SignedValue > 0xFFFF || SignedValue < -0xFFFF)
         Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
 
@@ -412,19 +404,37 @@ unsigned AArch64AsmBackend::getFixupKindContainereSizeInBytes(unsigned Kind) con
   }
 }
 
-void AArch64AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+static bool shouldForceRelocation(const MCFixup &Fixup) {
+  // The ADRP instruction adds some multiple of 0x1000 to the current PC &
+  // ~0xfff. This means that the required offset to reach a symbol can vary by
+  // up to one step depending on where the ADRP is in memory. For example:
+  //
+  //     ADRP x0, there
+  //  there:
+  //
+  // If the ADRP occurs at address 0xffc then "there" will be at 0x1000 and
+  // we'll need that as an offset. At any other address "there" will be in the
+  // same page as the ADRP and the instruction should encode 0x0. Assuming the
+  // section isn't 0x1000-aligned, we therefore need to delegate this decision
+  // to the linker -- a relocation!
+  return Fixup.getTargetKind() == AArch64::fixup_aarch64_pcrel_adrp_imm21;
+}
+
+void AArch64AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                    const MCValue &Target,
                                    MutableArrayRef<char> Data, uint64_t Value,
                                    bool IsResolved) {
+  if (shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
 
   if (Fixup.getTargetKind() == FK_Data_8 && TheTriple.isOSBinFormatELF()) {
-    auto RefKind = static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-    AArch64MCExpr::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
-    if (SymLoc == AArch64MCExpr::VK_AUTH ||
-        SymLoc == AArch64MCExpr::VK_AUTHADDR) {
+    auto RefKind = static_cast<AArch64::Specifier>(Target.getSpecifier());
+    AArch64::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
+    if (SymLoc == AArch64::S_AUTH || SymLoc == AArch64::S_AUTHADDR) {
       const auto *Expr = dyn_cast<AArch64AuthMCExpr>(Fixup.getValue());
       if (!Expr) {
         getContext().reportError(Fixup.getValue()->getLoc(),
@@ -475,9 +485,9 @@ void AArch64AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
 
   // FIXME: getFixupKindInfo() and getFixupKindNumBytes() could be fixed to
   // handle this more cleanly. This may affect the output of -show-mc-encoding.
-  AArch64MCExpr::Specifier RefKind =
-      static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-  if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS ||
+  AArch64::Specifier RefKind =
+      static_cast<AArch64::Specifier>(Target.getSpecifier());
+  if (AArch64::getSymbolLoc(RefKind) == AArch64::S_SABS ||
       (!RefKind && Fixup.getTargetKind() == AArch64::fixup_aarch64_movw)) {
     // If the immediate is negative, generate MOVN else MOVZ.
     // (Bit 30 = 0) ==> MOVN, (Bit 30 = 1) ==> MOVZ.
@@ -497,11 +507,6 @@ bool AArch64AsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
   return int64_t(Value) != int64_t(int8_t(Value));
 }
 
-void AArch64AsmBackend::relaxInstruction(MCInst &Inst,
-                                         const MCSubtargetInfo &STI) const {
-  llvm_unreachable("AArch64AsmBackend::relaxInstruction() unimplemented");
-}
-
 bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
                                      const MCSubtargetInfo *STI) const {
   // If the count is not 4-byte aligned, we must be writing data into the text
@@ -516,23 +521,6 @@ bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-bool AArch64AsmBackend::shouldForceRelocation(const MCFixup &Fixup,
-                                              const MCValue &Target) {
-  // The ADRP instruction adds some multiple of 0x1000 to the current PC &
-  // ~0xfff. This means that the required offset to reach a symbol can vary by
-  // up to one step depending on where the ADRP is in memory. For example:
-  //
-  //     ADRP x0, there
-  //  there:
-  //
-  // If the ADRP occurs at address 0xffc then "there" will be at 0x1000 and
-  // we'll need that as an offset. At any other address "there" will be in the
-  // same page as the ADRP and the instruction should encode 0x0. Assuming the
-  // section isn't 0x1000-aligned, we therefore need to delegate this decision
-  // to the linker -- a relocation!
-  return Fixup.getTargetKind() == AArch64::fixup_aarch64_pcrel_adrp_imm21;
-}
-
 namespace {
 
 namespace CU {
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 28aa4644a55f..c3881fc79ba6 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -39,8 +39,7 @@ class AArch64ELFObjectWriter : public MCELFObjectTargetWriter {
   unsigned getRelocType(const MCFixup &, const MCValue &,
                         bool IsPCRel) const override;
   bool needsRelocateWithSymbol(const MCValue &, unsigned Type) const override;
-  bool isNonILP32reloc(const MCFixup &Fixup,
-                       AArch64MCExpr::Specifier RefKind) const;
+  bool isNonILP32reloc(const MCFixup &Fixup, AArch64::Specifier RefKind) const;
 
   bool IsILP32;
 };
@@ -56,23 +55,23 @@ AArch64ELFObjectWriter::AArch64ELFObjectWriter(uint8_t OSABI, bool IsILP32)
   IsILP32 ? ELF::R_AARCH64_P32_##rtype : ELF::R_AARCH64_##rtype
 
 // assumes IsILP32 is true
-bool AArch64ELFObjectWriter::isNonILP32reloc(
-    const MCFixup &Fixup, AArch64MCExpr::Specifier RefKind) const {
+bool AArch64ELFObjectWriter::isNonILP32reloc(const MCFixup &Fixup,
+                                             AArch64::Specifier RefKind) const {
   if (Fixup.getTargetKind() != AArch64::fixup_aarch64_movw)
     return false;
   switch (RefKind) {
-  case AArch64MCExpr::VK_ABS_G3:
-  case AArch64MCExpr::VK_ABS_G2:
-  case AArch64MCExpr::VK_ABS_G2_S:
-  case AArch64MCExpr::VK_ABS_G2_NC:
-  case AArch64MCExpr::VK_ABS_G1_S:
-  case AArch64MCExpr::VK_ABS_G1_NC:
-  case AArch64MCExpr::VK_DTPREL_G2:
-  case AArch64MCExpr::VK_DTPREL_G1_NC:
-  case AArch64MCExpr::VK_TPREL_G2:
-  case AArch64MCExpr::VK_TPREL_G1_NC:
-  case AArch64MCExpr::VK_GOTTPREL_G1:
-  case AArch64MCExpr::VK_GOTTPREL_G0_NC:
+  case AArch64::S_ABS_G3:
+  case AArch64::S_ABS_G2:
+  case AArch64::S_ABS_G2_S:
+  case AArch64::S_ABS_G2_NC:
+  case AArch64::S_ABS_G1_S:
+  case AArch64::S_ABS_G1_NC:
+  case AArch64::S_DTPREL_G2:
+  case AArch64::S_DTPREL_G1_NC:
+  case AArch64::S_TPREL_G2:
+  case AArch64::S_TPREL_G1_NC:
+  case AArch64::S_GOTTPREL_G1:
+  case AArch64::S_GOTTPREL_G0_NC:
     reportError(Fixup.getLoc(),
                 "absolute MOV relocation is not supported in ILP32");
     return true;
@@ -86,17 +85,17 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                               const MCValue &Target,
                                               bool IsPCRel) const {
   unsigned Kind = Fixup.getTargetKind();
-  AArch64MCExpr::Specifier RefKind =
-      static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-  AArch64MCExpr::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
+  AArch64::Specifier RefKind =
+      static_cast<AArch64::Specifier>(Target.getSpecifier());
+  AArch64::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
   bool IsNC = AArch64::isNotChecked(RefKind);
 
   switch (SymLoc) {
-  case AArch64MCExpr::VK_DTPREL:
-  case AArch64MCExpr::VK_GOTTPREL:
-  case AArch64MCExpr::VK_TPREL:
-  case AArch64MCExpr::VK_TLSDESC:
-  case AArch64MCExpr::VK_TLSDESC_AUTH:
+  case AArch64::S_DTPREL:
+  case AArch64::S_GOTTPREL:
+  case AArch64::S_TPREL:
+  case AArch64::S_TLSDESC:
+  case AArch64::S_TLSDESC_AUTH:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -117,10 +116,8 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case FK_Data_2:
       return R_CLS(PREL16);
     case FK_Data_4: {
-      return AArch64MCExpr::Specifier(Target.getSpecifier()) ==
-                     AArch64MCExpr::VK_PLT
-                 ? R_CLS(PLT32)
-                 : R_CLS(PREL32);
+      return Target.getSpecifier() == AArch64::S_PLT ? R_CLS(PLT32)
+                                                     : R_CLS(PREL32);
     }
     case FK_Data_8:
       if (IsILP32) {
@@ -130,7 +127,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
       }
       return ELF::R_AARCH64_PREL64;
     case AArch64::fixup_aarch64_pcrel_adr_imm21:
-      if (SymLoc == AArch64MCExpr::VK_GOT_AUTH) {
+      if (SymLoc == AArch64::S_GOT_AUTH) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "ADR AUTH relocation is not supported in ILP32");
@@ -138,13 +135,13 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_AUTH_GOT_ADR_PREL_LO21;
       }
-      if (SymLoc != AArch64MCExpr::VK_ABS)
+      if (SymLoc != AArch64::S_ABS)
         reportError(Fixup.getLoc(), "invalid symbol kind for ADR relocation");
       return R_CLS(ADR_PREL_LO21);
     case AArch64::fixup_aarch64_pcrel_adrp_imm21:
-      if (SymLoc == AArch64MCExpr::VK_ABS && !IsNC)
+      if (SymLoc == AArch64::S_ABS && !IsNC)
         return R_CLS(ADR_PREL_PG_HI21);
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) {
+      if (SymLoc == AArch64::S_ABS && IsNC) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "invalid fixup for 32-bit pcrel ADRP instruction "
@@ -153,9 +150,9 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_ADR_PREL_PG_HI21_NC;
       }
-      if (SymLoc == AArch64MCExpr::VK_GOT && !IsNC)
+      if (SymLoc == AArch64::S_GOT && !IsNC)
         return R_CLS(ADR_GOT_PAGE);
-      if (SymLoc == AArch64MCExpr::VK_GOT_AUTH && !IsNC) {
+      if (SymLoc == AArch64::S_GOT_AUTH && !IsNC) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "ADRP AUTH relocation is not supported in ILP32");
@@ -163,11 +160,11 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_AUTH_ADR_GOT_PAGE;
       }
-      if (SymLoc == AArch64MCExpr::VK_GOTTPREL && !IsNC)
+      if (SymLoc == AArch64::S_GOTTPREL && !IsNC)
         return R_CLS(TLSIE_ADR_GOTTPREL_PAGE21);
-      if (SymLoc == AArch64MCExpr::VK_TLSDESC && !IsNC)
+      if (SymLoc == AArch64::S_TLSDESC && !IsNC)
         return R_CLS(TLSDESC_ADR_PAGE21);
-      if (SymLoc == AArch64MCExpr::VK_TLSDESC_AUTH && !IsNC) {
+      if (SymLoc == AArch64::S_TLSDESC_AUTH && !IsNC) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "ADRP AUTH relocation is not supported in ILP32");
@@ -182,11 +179,11 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case AArch64::fixup_aarch64_pcrel_call26:
       return R_CLS(CALL26);
     case AArch64::fixup_aarch64_ldr_pcrel_imm19:
-      if (SymLoc == AArch64MCExpr::VK_GOTTPREL)
+      if (SymLoc == AArch64::S_GOTTPREL)
         return R_CLS(TLSIE_LD_GOTTPREL_PREL19);
-      if (SymLoc == AArch64MCExpr::VK_GOT)
+      if (SymLoc == AArch64::S_GOT)
         return R_CLS(GOT_LD_PREL19);
-      if (SymLoc == AArch64MCExpr::VK_GOT_AUTH) {
+      if (SymLoc == AArch64::S_GOT_AUTH) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "LDR AUTH relocation is not supported in ILP32");
@@ -222,8 +219,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case FK_Data_2:
       return R_CLS(ABS16);
     case FK_Data_4:
-      return (!IsILP32 && AArch64MCExpr::Specifier(Target.getSpecifier()) ==
-                              AArch64MCExpr::VK_GOTPCREL)
+      return (!IsILP32 && Target.getSpecifier() == AArch64::S_GOTPCREL)
                  ? ELF::R_AARCH64_GOTPCREL32
                  : R_CLS(ABS32);
     case FK_Data_8: {
@@ -233,27 +229,26 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
             "8 byte absolute data relocation is not supported in ILP32");
         return ELF::R_AARCH64_NONE;
       }
-      if (RefKind == AArch64MCExpr::VK_AUTH ||
-          RefKind == AArch64MCExpr::VK_AUTHADDR)
+      if (RefKind == AArch64::S_AUTH || RefKind == AArch64::S_AUTHADDR)
         return ELF::R_AARCH64_AUTH_ABS64;
       return ELF::R_AARCH64_ABS64;
     }
     case AArch64::fixup_aarch64_add_imm12:
-      if (RefKind == AArch64MCExpr::VK_DTPREL_HI12)
+      if (RefKind == AArch64::S_DTPREL_HI12)
         return R_CLS(TLSLD_ADD_DTPREL_HI12);
-      if (RefKind == AArch64MCExpr::VK_TPREL_HI12)
+      if (RefKind == AArch64::S_TPREL_HI12)
         return R_CLS(TLSLE_ADD_TPREL_HI12);
-      if (RefKind == AArch64MCExpr::VK_DTPREL_LO12_NC)
+      if (RefKind == AArch64::S_DTPREL_LO12_NC)
         return R_CLS(TLSLD_ADD_DTPREL_LO12_NC);
-      if (RefKind == AArch64MCExpr::VK_DTPREL_LO12)
+      if (RefKind == AArch64::S_DTPREL_LO12)
         return R_CLS(TLSLD_ADD_DTPREL_LO12);
-      if (RefKind == AArch64MCExpr::VK_TPREL_LO12_NC)
+      if (RefKind == AArch64::S_TPREL_LO12_NC)
         return R_CLS(TLSLE_ADD_TPREL_LO12_NC);
-      if (RefKind == AArch64MCExpr::VK_TPREL_LO12)
+      if (RefKind == AArch64::S_TPREL_LO12)
         return R_CLS(TLSLE_ADD_TPREL_LO12);
-      if (RefKind == AArch64MCExpr::VK_TLSDESC_LO12)
+      if (RefKind == AArch64::S_TLSDESC_LO12)
         return R_CLS(TLSDESC_ADD_LO12);
-      if (RefKind == AArch64MCExpr::VK_TLSDESC_AUTH_LO12) {
+      if (RefKind == AArch64::S_TLSDESC_AUTH_LO12) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "ADD AUTH relocation is not supported in ILP32");
@@ -261,7 +256,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_AUTH_TLSDESC_ADD_LO12;
       }
-      if (RefKind == AArch64MCExpr::VK_GOT_AUTH_LO12 && IsNC) {
+      if (RefKind == AArch64::S_GOT_AUTH_LO12 && IsNC) {
         if (IsILP32) {
           reportError(Fixup.getLoc(),
                       "ADD AUTH relocation is not supported in ILP32");
@@ -269,60 +264,60 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_AUTH_GOT_ADD_LO12_NC;
       }
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(ADD_ABS_LO12_NC);
 
       reportError(Fixup.getLoc(), "invalid fixup for add (uimm12) instruction");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_ldst_imm12_scale1:
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(LDST8_ABS_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
+      if (SymLoc == AArch64::S_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST8_DTPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
+      if (SymLoc == AArch64::S_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST8_DTPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
+      if (SymLoc == AArch64::S_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST8_TPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
+      if (SymLoc == AArch64::S_TPREL && IsNC)
         return R_CLS(TLSLE_LDST8_TPREL_LO12_NC);
 
       reportError(Fixup.getLoc(),
                   "invalid fixup for 8-bit load/store instruction");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_ldst_imm12_scale2:
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(LDST16_ABS_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
+      if (SymLoc == AArch64::S_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST16_DTPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
+      if (SymLoc == AArch64::S_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST16_DTPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
+      if (SymLoc == AArch64::S_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST16_TPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
+      if (SymLoc == AArch64::S_TPREL && IsNC)
         return R_CLS(TLSLE_LDST16_TPREL_LO12_NC);
 
       reportError(Fixup.getLoc(),
                   "invalid fixup for 16-bit load/store instruction");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_ldst_imm12_scale4:
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(LDST32_ABS_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
+      if (SymLoc == AArch64::S_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST32_DTPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
+      if (SymLoc == AArch64::S_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST32_DTPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
+      if (SymLoc == AArch64::S_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST32_TPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
+      if (SymLoc == AArch64::S_TPREL && IsNC)
         return R_CLS(TLSLE_LDST32_TPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_GOT && IsNC) {
+      if (SymLoc == AArch64::S_GOT && IsNC) {
         if (IsILP32)
           return ELF::R_AARCH64_P32_LD32_GOT_LO12_NC;
         reportError(Fixup.getLoc(), "4 byte unchecked GOT load/store "
                                     "relocation is not supported in LP64");
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_GOT && !IsNC) {
+      if (SymLoc == AArch64::S_GOT && !IsNC) {
         if (IsILP32) {
           reportError(
               Fixup.getLoc(),
@@ -330,14 +325,14 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
         }
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_GOTTPREL && IsNC) {
+      if (SymLoc == AArch64::S_GOTTPREL && IsNC) {
         if (IsILP32)
           return ELF::R_AARCH64_P32_TLSIE_LD32_GOTTPREL_LO12_NC;
         reportError(Fixup.getLoc(), "32-bit load/store "
                                     "relocation is not supported in LP64");
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_TLSDESC && !IsNC) {
+      if (SymLoc == AArch64::S_TLSDESC && !IsNC) {
         if (IsILP32)
           return ELF::R_AARCH64_P32_TLSDESC_LD32_LO12;
         reportError(
@@ -351,15 +346,13 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                   "fixup_aarch64_ldst_imm12_scale4");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_ldst_imm12_scale8:
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(LDST64_ABS_LO12_NC);
-      if ((SymLoc == AArch64MCExpr::VK_GOT ||
-           SymLoc == AArch64MCExpr::VK_GOT_AUTH) &&
-          IsNC) {
-        AArch64MCExpr::Specifier AddressLoc = AArch64::getAddressFrag(RefKind);
-        bool IsAuth = (SymLoc == AArch64MCExpr::VK_GOT_AUTH);
+      if ((SymLoc == AArch64::S_GOT || SymLoc == AArch64::S_GOT_AUTH) && IsNC) {
+        AArch64::Specifier AddressLoc = AArch64::getAddressFrag(RefKind);
+        bool IsAuth = (SymLoc == AArch64::S_GOT_AUTH);
         if (!IsILP32) {
-          if (AddressLoc == AArch64MCExpr::VK_LO15)
+          if (AddressLoc == AArch64::S_LO15)
             return ELF::R_AARCH64_LD64_GOTPAGE_LO15;
           return (IsAuth ? ELF::R_AARCH64_AUTH_LD64_GOT_LO12_NC
                          : ELF::R_AARCH64_LD64_GOT_LO12_NC);
@@ -368,29 +361,29 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                     "64-bit load/store relocation is not supported in ILP32");
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
+      if (SymLoc == AArch64::S_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST64_DTPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
+      if (SymLoc == AArch64::S_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST64_DTPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
+      if (SymLoc == AArch64::S_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST64_TPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
+      if (SymLoc == AArch64::S_TPREL && IsNC)
         return R_CLS(TLSLE_LDST64_TPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_GOTTPREL && IsNC) {
+      if (SymLoc == AArch64::S_GOTTPREL && IsNC) {
         if (!IsILP32)
           return ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC;
         reportError(Fixup.getLoc(),
                     "64-bit load/store relocation is not supported in ILP32");
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_TLSDESC) {
+      if (SymLoc == AArch64::S_TLSDESC) {
         if (!IsILP32)
           return ELF::R_AARCH64_TLSDESC_LD64_LO12;
         reportError(Fixup.getLoc(),
                     "64-bit load/store relocation is not supported in ILP32");
         return ELF::R_AARCH64_NONE;
       }
-      if (SymLoc == AArch64MCExpr::VK_TLSDESC_AUTH) {
+      if (SymLoc == AArch64::S_TLSDESC_AUTH) {
         if (!IsILP32)
           return ELF::R_AARCH64_AUTH_TLSDESC_LD64_LO12;
         reportError(
@@ -402,15 +395,15 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                   "invalid fixup for 64-bit load/store instruction");
       return ELF::R_AARCH64_NONE;
     case AArch64::fixup_aarch64_ldst_imm12_scale16:
-      if (SymLoc == AArch64MCExpr::VK_ABS && IsNC)
+      if (SymLoc == AArch64::S_ABS && IsNC)
         return R_CLS(LDST128_ABS_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC)
+      if (SymLoc == AArch64::S_DTPREL && !IsNC)
         return R_CLS(TLSLD_LDST128_DTPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC)
+      if (SymLoc == AArch64::S_DTPREL && IsNC)
         return R_CLS(TLSLD_LDST128_DTPREL_LO12_NC);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC)
+      if (SymLoc == AArch64::S_TPREL && !IsNC)
         return R_CLS(TLSLE_LDST128_TPREL_LO12);
-      if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC)
+      if (SymLoc == AArch64::S_TPREL && IsNC)
         return R_CLS(TLSLE_LDST128_TPREL_LO12_NC);
 
       reportError(Fixup.getLoc(),
@@ -418,63 +411,63 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_AARCH64_NONE;
     // ILP32 case not reached here, tested with isNonILP32reloc
     case AArch64::fixup_aarch64_movw:
-      if (RefKind == AArch64MCExpr::VK_ABS_G3)
+      if (RefKind == AArch64::S_ABS_G3)
         return ELF::R_AARCH64_MOVW_UABS_G3;
-      if (RefKind == AArch64MCExpr::VK_ABS_G2)
+      if (RefKind == AArch64::S_ABS_G2)
         return ELF::R_AARCH64_MOVW_UABS_G2;
-      if (RefKind == AArch64MCExpr::VK_ABS_G2_S)
+      if (RefKind == AArch64::S_ABS_G2_S)
         return ELF::R_AARCH64_MOVW_SABS_G2;
-      if (RefKind == AArch64MCExpr::VK_ABS_G2_NC)
+      if (RefKind == AArch64::S_ABS_G2_NC)
         return ELF::R_AARCH64_MOVW_UABS_G2_NC;
-      if (RefKind == AArch64MCExpr::VK_ABS_G1)
+      if (RefKind == AArch64::S_ABS_G1)
         return R_CLS(MOVW_UABS_G1);
-      if (RefKind == AArch64MCExpr::VK_ABS_G1_S)
+      if (RefKind == AArch64::S_ABS_G1_S)
         return ELF::R_AARCH64_MOVW_SABS_G1;
-      if (RefKind == AArch64MCExpr::VK_ABS_G1_NC)
+      if (RefKind == AArch64::S_ABS_G1_NC)
         return ELF::R_AARCH64_MOVW_UABS_G1_NC;
-      if (RefKind == AArch64MCExpr::VK_ABS_G0)
+      if (RefKind == AArch64::S_ABS_G0)
         return R_CLS(MOVW_UABS_G0);
-      if (RefKind == AArch64MCExpr::VK_ABS_G0_S)
+      if (RefKind == AArch64::S_ABS_G0_S)
         return R_CLS(MOVW_SABS_G0);
-      if (RefKind == AArch64MCExpr::VK_ABS_G0_NC)
+      if (RefKind == AArch64::S_ABS_G0_NC)
         return R_CLS(MOVW_UABS_G0_NC);
-      if (RefKind == AArch64MCExpr::VK_PREL_G3)
+      if (RefKind == AArch64::S_PREL_G3)
         return ELF::R_AARCH64_MOVW_PREL_G3;
-      if (RefKind == AArch64MCExpr::VK_PREL_G2)
+      if (RefKind == AArch64::S_PREL_G2)
         return ELF::R_AARCH64_MOVW_PREL_G2;
-      if (RefKind == AArch64MCExpr::VK_PREL_G2_NC)
+      if (RefKind == AArch64::S_PREL_G2_NC)
         return ELF::R_AARCH64_MOVW_PREL_G2_NC;
-      if (RefKind == AArch64MCExpr::VK_PREL_G1)
+      if (RefKind == AArch64::S_PREL_G1)
         return R_CLS(MOVW_PREL_G1);
-      if (RefKind == AArch64MCExpr::VK_PREL_G1_NC)
+      if (RefKind == AArch64::S_PREL_G1_NC)
         return ELF::R_AARCH64_MOVW_PREL_G1_NC;
-      if (RefKind == AArch64MCExpr::VK_PREL_G0)
+      if (RefKind == AArch64::S_PREL_G0)
         return R_CLS(MOVW_PREL_G0);
-      if (RefKind == AArch64MCExpr::VK_PREL_G0_NC)
+      if (RefKind == AArch64::S_PREL_G0_NC)
         return R_CLS(MOVW_PREL_G0_NC);
-      if (RefKind == AArch64MCExpr::VK_DTPREL_G2)
+      if (RefKind == AArch64::S_DTPREL_G2)
         return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G2;
-      if (RefKind == AArch64MCExpr::VK_DTPREL_G1)
+      if (RefKind == AArch64::S_DTPREL_G1)
         return R_CLS(TLSLD_MOVW_DTPREL_G1);
-      if (RefKind == AArch64MCExpr::VK_DTPREL_G1_NC)
+      if (RefKind == AArch64::S_DTPREL_G1_NC)
         return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC;
-      if (RefKind == AArch64MCExpr::VK_DTPREL_G0)
+      if (RefKind == AArch64::S_DTPREL_G0)
         return R_CLS(TLSLD_MOVW_DTPREL_G0);
-      if (RefKind == AArch64MCExpr::VK_DTPREL_G0_NC)
+      if (RefKind == AArch64::S_DTPREL_G0_NC)
         return R_CLS(TLSLD_MOVW_DTPREL_G0_NC);
-      if (RefKind == AArch64MCExpr::VK_TPREL_G2)
+      if (RefKind == AArch64::S_TPREL_G2)
         return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G2;
-      if (RefKind == AArch64MCExpr::VK_TPREL_G1)
+      if (RefKind == AArch64::S_TPREL_G1)
         return R_CLS(TLSLE_MOVW_TPREL_G1);
-      if (RefKind == AArch64MCExpr::VK_TPREL_G1_NC)
+      if (RefKind == AArch64::S_TPREL_G1_NC)
         return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G1_NC;
-      if (RefKind == AArch64MCExpr::VK_TPREL_G0)
+      if (RefKind == AArch64::S_TPREL_G0)
         return R_CLS(TLSLE_MOVW_TPREL_G0);
-      if (RefKind == AArch64MCExpr::VK_TPREL_G0_NC)
+      if (RefKind == AArch64::S_TPREL_G0_NC)
         return R_CLS(TLSLE_MOVW_TPREL_G0_NC);
-      if (RefKind == AArch64MCExpr::VK_GOTTPREL_G1)
+      if (RefKind == AArch64::S_GOTTPREL_G1)
         return ELF::R_AARCH64_TLSIE_MOVW_GOTTPREL_G1;
-      if (RefKind == AArch64MCExpr::VK_GOTTPREL_G0_NC)
+      if (RefKind == AArch64::S_GOTTPREL_G0_NC)
         return ELF::R_AARCH64_TLSIE_MOVW_GOTTPREL_G0_NC;
       reportError(Fixup.getLoc(), "invalid fixup for movz/movk instruction");
       return ELF::R_AARCH64_NONE;
@@ -498,10 +491,10 @@ bool AArch64ELFObjectWriter::needsRelocateWithSymbol(const MCValue &Val,
   if (Val.getAddSym() && cast<MCSymbolELF>(Val.getAddSym())->isMemtag())
     return true;
 
-  if ((Val.getSpecifier() & AArch64MCExpr::VK_GOT) == AArch64MCExpr::VK_GOT)
+  if ((Val.getSpecifier() & AArch64::S_GOT) == AArch64::S_GOT)
     return true;
-  return is_contained({AArch64MCExpr::VK_GOTPCREL, AArch64MCExpr::VK_PLT},
-                      AArch64MCExpr::Specifier(Val.getSpecifier()));
+  return is_contained({AArch64::S_GOTPCREL, AArch64::S_PLT},
+                      Val.getSpecifier());
 }
 
 std::unique_ptr<MCObjectTargetWriter>
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index bbe83821eca8..3c8b5712c1f0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -1784,6 +1784,10 @@ void AArch64InstPrinter::printAlignedLabel(const MCInst *MI, uint64_t Address,
                                            unsigned OpNum,
                                            const MCSubtargetInfo &STI,
                                            raw_ostream &O) {
+  // Do not print the numeric target address when symbolizing.
+  if (SymbolizeOperands)
+    return;
+
   const MCOperand &Op = MI->getOperand(OpNum);
 
   // If the label has already been resolved to an immediate offset (say, when
@@ -1813,6 +1817,12 @@ void AArch64InstPrinter::printAdrAdrpLabel(const MCInst *MI, uint64_t Address,
                                            unsigned OpNum,
                                            const MCSubtargetInfo &STI,
                                            raw_ostream &O) {
+  // Do not print the numeric target address when symbolizing.
+  // However, do print for ADRP, as this is typically used together with an ADD
+  // or an immediate-offset ldr/str and the label is likely at the wrong point.
+  if (SymbolizeOperands && MI->getOpcode() != AArch64::ADRP)
+    return;
+
   const MCOperand &Op = MI->getOperand(OpNum);
 
   // If the label has already been resolved to an immediate offset (say, when
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
index b2cd1d0f4156..828c5c546240 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64MCAsmInfo.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
@@ -34,86 +33,86 @@ static cl::opt<AsmWriterVariantTy> AsmWriterVariant(
 
 const MCAsmInfo::AtSpecifier COFFAtSpecifiers[] = {
     {MCSymbolRefExpr::VK_COFF_IMGREL32, "IMGREL"},
-    {AArch64MCExpr::M_PAGEOFF, "PAGEOFF"},
+    {AArch64::S_MACHO_PAGEOFF, "PAGEOFF"},
 };
 
 const MCAsmInfo::AtSpecifier ELFAtSpecifiers[] = {
-    {AArch64MCExpr::VK_GOT, "GOT"},
-    {AArch64MCExpr::VK_GOTPCREL, "GOTPCREL"},
-    {AArch64MCExpr::VK_PLT, "PLT"},
+    {AArch64::S_GOT, "GOT"},
+    {AArch64::S_GOTPCREL, "GOTPCREL"},
+    {AArch64::S_PLT, "PLT"},
 };
 
 const MCAsmInfo::AtSpecifier MachOAtSpecifiers[] = {
-    {AArch64MCExpr::M_GOT, "GOT"},
-    {AArch64MCExpr::M_GOTPAGE, "GOTPAGE"},
-    {AArch64MCExpr::M_GOTPAGEOFF, "GOTPAGEOFF"},
-    {AArch64MCExpr::M_PAGE, "PAGE"},
-    {AArch64MCExpr::M_PAGEOFF, "PAGEOFF"},
-    {AArch64MCExpr::M_TLVP, "TLVP"},
-    {AArch64MCExpr::M_TLVPPAGE, "TLVPPAGE"},
-    {AArch64MCExpr::M_TLVPPAGEOFF, "TLVPPAGEOFF"},
+    {AArch64::S_MACHO_GOT, "GOT"},
+    {AArch64::S_MACHO_GOTPAGE, "GOTPAGE"},
+    {AArch64::S_MACHO_GOTPAGEOFF, "GOTPAGEOFF"},
+    {AArch64::S_MACHO_PAGE, "PAGE"},
+    {AArch64::S_MACHO_PAGEOFF, "PAGEOFF"},
+    {AArch64::S_MACHO_TLVP, "TLVP"},
+    {AArch64::S_MACHO_TLVPPAGE, "TLVPPAGE"},
+    {AArch64::S_MACHO_TLVPPAGEOFF, "TLVPPAGEOFF"},
 };
 
 StringRef AArch64::getSpecifierName(const MCSpecifierExpr &Expr) {
   // clang-format off
   switch (static_cast<uint32_t>(Expr.getSpecifier())) {
-  case AArch64MCExpr::VK_CALL:                return "";
-  case AArch64MCExpr::VK_LO12:                return ":lo12:";
-  case AArch64MCExpr::VK_ABS_G3:              return ":abs_g3:";
-  case AArch64MCExpr::VK_ABS_G2:              return ":abs_g2:";
-  case AArch64MCExpr::VK_ABS_G2_S:            return ":abs_g2_s:";
-  case AArch64MCExpr::VK_ABS_G2_NC:           return ":abs_g2_nc:";
-  case AArch64MCExpr::VK_ABS_G1:              return ":abs_g1:";
-  case AArch64MCExpr::VK_ABS_G1_S:            return ":abs_g1_s:";
-  case AArch64MCExpr::VK_ABS_G1_NC:           return ":abs_g1_nc:";
-  case AArch64MCExpr::VK_ABS_G0:              return ":abs_g0:";
-  case AArch64MCExpr::VK_ABS_G0_S:            return ":abs_g0_s:";
-  case AArch64MCExpr::VK_ABS_G0_NC:           return ":abs_g0_nc:";
-  case AArch64MCExpr::VK_PREL_G3:             return ":prel_g3:";
-  case AArch64MCExpr::VK_PREL_G2:             return ":prel_g2:";
-  case AArch64MCExpr::VK_PREL_G2_NC:          return ":prel_g2_nc:";
-  case AArch64MCExpr::VK_PREL_G1:             return ":prel_g1:";
-  case AArch64MCExpr::VK_PREL_G1_NC:          return ":prel_g1_nc:";
-  case AArch64MCExpr::VK_PREL_G0:             return ":prel_g0:";
-  case AArch64MCExpr::VK_PREL_G0_NC:          return ":prel_g0_nc:";
-  case AArch64MCExpr::VK_DTPREL_G2:           return ":dtprel_g2:";
-  case AArch64MCExpr::VK_DTPREL_G1:           return ":dtprel_g1:";
-  case AArch64MCExpr::VK_DTPREL_G1_NC:        return ":dtprel_g1_nc:";
-  case AArch64MCExpr::VK_DTPREL_G0:           return ":dtprel_g0:";
-  case AArch64MCExpr::VK_DTPREL_G0_NC:        return ":dtprel_g0_nc:";
-  case AArch64MCExpr::VK_DTPREL_HI12:         return ":dtprel_hi12:";
-  case AArch64MCExpr::VK_DTPREL_LO12:         return ":dtprel_lo12:";
-  case AArch64MCExpr::VK_DTPREL_LO12_NC:      return ":dtprel_lo12_nc:";
-  case AArch64MCExpr::VK_TPREL_G2:            return ":tprel_g2:";
-  case AArch64MCExpr::VK_TPREL_G1:            return ":tprel_g1:";
-  case AArch64MCExpr::VK_TPREL_G1_NC:         return ":tprel_g1_nc:";
-  case AArch64MCExpr::VK_TPREL_G0:            return ":tprel_g0:";
-  case AArch64MCExpr::VK_TPREL_G0_NC:         return ":tprel_g0_nc:";
-  case AArch64MCExpr::VK_TPREL_HI12:          return ":tprel_hi12:";
-  case AArch64MCExpr::VK_TPREL_LO12:          return ":tprel_lo12:";
-  case AArch64MCExpr::VK_TPREL_LO12_NC:       return ":tprel_lo12_nc:";
-  case AArch64MCExpr::VK_TLSDESC_LO12:        return ":tlsdesc_lo12:";
-  case AArch64MCExpr::VK_TLSDESC_AUTH_LO12:   return ":tlsdesc_auth_lo12:";
-  case AArch64MCExpr::VK_ABS_PAGE:            return "";
-  case AArch64MCExpr::VK_ABS_PAGE_NC:         return ":pg_hi21_nc:";
-  case AArch64MCExpr::VK_GOT:                 return ":got:";
-  case AArch64MCExpr::VK_GOT_PAGE:            return ":got:";
-  case AArch64MCExpr::VK_GOT_PAGE_LO15:       return ":gotpage_lo15:";
-  case AArch64MCExpr::VK_GOT_LO12:            return ":got_lo12:";
-  case AArch64MCExpr::VK_GOTTPREL:            return ":gottprel:";
-  case AArch64MCExpr::VK_GOTTPREL_PAGE:       return ":gottprel:";
-  case AArch64MCExpr::VK_GOTTPREL_LO12_NC:    return ":gottprel_lo12:";
-  case AArch64MCExpr::VK_GOTTPREL_G1:         return ":gottprel_g1:";
-  case AArch64MCExpr::VK_GOTTPREL_G0_NC:      return ":gottprel_g0_nc:";
-  case AArch64MCExpr::VK_TLSDESC:             return "";
-  case AArch64MCExpr::VK_TLSDESC_PAGE:        return ":tlsdesc:";
-  case AArch64MCExpr::VK_TLSDESC_AUTH:        return "";
-  case AArch64MCExpr::VK_TLSDESC_AUTH_PAGE:   return ":tlsdesc_auth:";
-  case AArch64MCExpr::VK_SECREL_LO12:         return ":secrel_lo12:";
-  case AArch64MCExpr::VK_SECREL_HI12:         return ":secrel_hi12:";
-  case AArch64MCExpr::VK_GOT_AUTH:            return ":got_auth:";
-  case AArch64MCExpr::VK_GOT_AUTH_PAGE:       return ":got_auth:";
-  case AArch64MCExpr::VK_GOT_AUTH_LO12:       return ":got_auth_lo12:";
+  case AArch64::S_CALL:                return "";
+  case AArch64::S_LO12:                return ":lo12:";
+  case AArch64::S_ABS_G3:              return ":abs_g3:";
+  case AArch64::S_ABS_G2:              return ":abs_g2:";
+  case AArch64::S_ABS_G2_S:            return ":abs_g2_s:";
+  case AArch64::S_ABS_G2_NC:           return ":abs_g2_nc:";
+  case AArch64::S_ABS_G1:              return ":abs_g1:";
+  case AArch64::S_ABS_G1_S:            return ":abs_g1_s:";
+  case AArch64::S_ABS_G1_NC:           return ":abs_g1_nc:";
+  case AArch64::S_ABS_G0:              return ":abs_g0:";
+  case AArch64::S_ABS_G0_S:            return ":abs_g0_s:";
+  case AArch64::S_ABS_G0_NC:           return ":abs_g0_nc:";
+  case AArch64::S_PREL_G3:             return ":prel_g3:";
+  case AArch64::S_PREL_G2:             return ":prel_g2:";
+  case AArch64::S_PREL_G2_NC:          return ":prel_g2_nc:";
+  case AArch64::S_PREL_G1:             return ":prel_g1:";
+  case AArch64::S_PREL_G1_NC:          return ":prel_g1_nc:";
+  case AArch64::S_PREL_G0:             return ":prel_g0:";
+  case AArch64::S_PREL_G0_NC:          return ":prel_g0_nc:";
+  case AArch64::S_DTPREL_G2:           return ":dtprel_g2:";
+  case AArch64::S_DTPREL_G1:           return ":dtprel_g1:";
+  case AArch64::S_DTPREL_G1_NC:        return ":dtprel_g1_nc:";
+  case AArch64::S_DTPREL_G0:           return ":dtprel_g0:";
+  case AArch64::S_DTPREL_G0_NC:        return ":dtprel_g0_nc:";
+  case AArch64::S_DTPREL_HI12:         return ":dtprel_hi12:";
+  case AArch64::S_DTPREL_LO12:         return ":dtprel_lo12:";
+  case AArch64::S_DTPREL_LO12_NC:      return ":dtprel_lo12_nc:";
+  case AArch64::S_TPREL_G2:            return ":tprel_g2:";
+  case AArch64::S_TPREL_G1:            return ":tprel_g1:";
+  case AArch64::S_TPREL_G1_NC:         return ":tprel_g1_nc:";
+  case AArch64::S_TPREL_G0:            return ":tprel_g0:";
+  case AArch64::S_TPREL_G0_NC:         return ":tprel_g0_nc:";
+  case AArch64::S_TPREL_HI12:          return ":tprel_hi12:";
+  case AArch64::S_TPREL_LO12:          return ":tprel_lo12:";
+  case AArch64::S_TPREL_LO12_NC:       return ":tprel_lo12_nc:";
+  case AArch64::S_TLSDESC_LO12:        return ":tlsdesc_lo12:";
+  case AArch64::S_TLSDESC_AUTH_LO12:   return ":tlsdesc_auth_lo12:";
+  case AArch64::S_ABS_PAGE:            return "";
+  case AArch64::S_ABS_PAGE_NC:         return ":pg_hi21_nc:";
+  case AArch64::S_GOT:                 return ":got:";
+  case AArch64::S_GOT_PAGE:            return ":got:";
+  case AArch64::S_GOT_PAGE_LO15:       return ":gotpage_lo15:";
+  case AArch64::S_GOT_LO12:            return ":got_lo12:";
+  case AArch64::S_GOTTPREL:            return ":gottprel:";
+  case AArch64::S_GOTTPREL_PAGE:       return ":gottprel:";
+  case AArch64::S_GOTTPREL_LO12_NC:    return ":gottprel_lo12:";
+  case AArch64::S_GOTTPREL_G1:         return ":gottprel_g1:";
+  case AArch64::S_GOTTPREL_G0_NC:      return ":gottprel_g0_nc:";
+  case AArch64::S_TLSDESC:             return "";
+  case AArch64::S_TLSDESC_PAGE:        return ":tlsdesc:";
+  case AArch64::S_TLSDESC_AUTH:        return "";
+  case AArch64::S_TLSDESC_AUTH_PAGE:   return ":tlsdesc_auth:";
+  case AArch64::S_SECREL_LO12:         return ":secrel_lo12:";
+  case AArch64::S_SECREL_HI12:         return ":secrel_hi12:";
+  case AArch64::S_GOT_AUTH:            return ":got_auth:";
+  case AArch64::S_GOT_AUTH_PAGE:       return ":got_auth:";
+  case AArch64::S_GOT_AUTH_LO12:       return ":got_auth_lo12:";
   default:
     llvm_unreachable("Invalid relocation specifier");
   }
@@ -148,7 +147,7 @@ AArch64MCAsmInfoDarwin::AArch64MCAsmInfoDarwin(bool IsILP32) {
 
   ExceptionsType = ExceptionHandling::DwarfCFI;
 
-  initializeVariantKinds(MachOAtSpecifiers);
+  initializeAtSpecifiers(MachOAtSpecifiers);
 }
 
 const MCExpr *AArch64MCAsmInfoDarwin::getExprForPersonalitySymbol(
@@ -159,7 +158,7 @@ const MCExpr *AArch64MCAsmInfoDarwin::getExprForPersonalitySymbol(
   // version.
   MCContext &Context = Streamer.getContext();
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, AArch64MCExpr::M_GOT, Context);
+      MCSymbolRefExpr::create(Sym, AArch64::S_MACHO_GOT, Context);
   MCSymbol *PCSym = Context.createTempSymbol();
   Streamer.emitLabel(PCSym);
   const MCExpr *PC = MCSymbolRefExpr::create(PCSym, Context);
@@ -170,7 +169,7 @@ void AArch64AuthMCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   bool WrapSubExprInParens = !isa<MCSymbolRefExpr>(getSubExpr());
   if (WrapSubExprInParens)
     OS << '(';
-  getSubExpr()->print(OS, MAI);
+  MAI->printExpr(OS, *getSubExpr());
   if (WrapSubExprInParens)
     OS << ')';
 
@@ -226,7 +225,7 @@ AArch64MCAsmInfoELF::AArch64MCAsmInfoELF(const Triple &T) {
 
   HasIdentDirective = true;
 
-  initializeVariantKinds(ELFAtSpecifiers);
+  initializeAtSpecifiers(ELFAtSpecifiers);
 }
 
 void AArch64MCAsmInfoELF::printSpecifierExpr(
@@ -258,7 +257,7 @@ AArch64MCAsmInfoMicrosoftCOFF::AArch64MCAsmInfoMicrosoftCOFF() {
   ExceptionsType = ExceptionHandling::WinEH;
   WinEHEncodingType = WinEH::EncodingType::Itanium;
 
-  initializeVariantKinds(COFFAtSpecifiers);
+  initializeAtSpecifiers(COFFAtSpecifiers);
 }
 
 void AArch64MCAsmInfoMicrosoftCOFF::printSpecifierExpr(
@@ -288,7 +287,7 @@ AArch64MCAsmInfoGNUCOFF::AArch64MCAsmInfoGNUCOFF() {
   ExceptionsType = ExceptionHandling::WinEH;
   WinEHEncodingType = WinEH::EncodingType::Itanium;
 
-  initializeVariantKinds(COFFAtSpecifiers);
+  initializeAtSpecifiers(COFFAtSpecifiers);
 }
 
 void AArch64MCAsmInfoGNUCOFF::printSpecifierExpr(
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
index 58e87f8a208f..c28e925d77e2 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
@@ -13,13 +13,16 @@
 #ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCASMINFO_H
 #define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCASMINFO_H
 
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/MC/MCAsmInfoCOFF.h"
 #include "llvm/MC/MCAsmInfoDarwin.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/Support/Casting.h"
 
 namespace llvm {
 class MCStreamer;
+class MCValue;
 class Triple;
 
 struct AArch64MCAsmInfoDarwin : public MCAsmInfoDarwin {
@@ -60,21 +63,169 @@ struct AArch64MCAsmInfoGNUCOFF : public MCAsmInfoGNUCOFF {
 namespace AArch64 {
 using Specifier = uint16_t;
 
+enum {
+  // clang-format off
+  S_None         = 0,
+  // Symbol locations specifying (roughly speaking) what calculation should be
+  // performed to construct the final address for the relocated
+  // symbol. E.g. direct, via the GOT, ...
+  S_ABS          = 0x001,
+  S_SABS         = 0x002,
+  S_PREL         = 0x003,
+  S_GOT          = 0x004,
+  S_DTPREL       = 0x005,
+  S_GOTTPREL     = 0x006,
+  S_TPREL        = 0x007,
+  S_TLSDESC      = 0x008,
+  S_SECREL       = 0x009,
+  S_AUTH         = 0x00a,
+  S_AUTHADDR     = 0x00b,
+  S_GOT_AUTH     = 0x00c,
+  S_TLSDESC_AUTH = 0x00d,
+  S_SymLocBits   = 0x00f,
+
+  // Variants specifying which part of the final address calculation is
+  // used. E.g. the low 12 bits for an ADD/LDR, the middle 16 bits for a
+  // MOVZ/MOVK.
+  S_PAGE         = 0x010,
+  S_PAGEOFF      = 0x020,
+  S_HI12         = 0x030,
+  S_G0           = 0x040,
+  S_G1           = 0x050,
+  S_G2           = 0x060,
+  S_G3           = 0x070,
+  S_LO15         = 0x080,
+  S_AddressFragBits = 0x0f0,
+
+  // Whether the final relocation is a checked one (where a linker should
+  // perform a range-check on the final address) or not. Note that this field
+  // is unfortunately sometimes omitted from the assembly syntax. E.g. :lo12:
+  // on its own is a non-checked relocation. We side with ELF on being
+  // explicit about this!
+  S_NC           = 0x100,
+
+  // Convenience definitions for referring to specific textual representations
+  // of relocation specifiers. Note that this means the "_NC" is sometimes
+  // omitted in line with assembly syntax here (S_LO12 rather than VK_LO12_NC
+  // since a user would write ":lo12:").
+  S_CALL              = S_ABS,
+  S_ABS_PAGE          = S_ABS          | S_PAGE,
+  S_ABS_PAGE_NC       = S_ABS          | S_PAGE    | S_NC,
+  S_ABS_G3            = S_ABS          | S_G3,
+  S_ABS_G2            = S_ABS          | S_G2,
+  S_ABS_G2_S          = S_SABS         | S_G2,
+  S_ABS_G2_NC         = S_ABS          | S_G2      | S_NC,
+  S_ABS_G1            = S_ABS          | S_G1,
+  S_ABS_G1_S          = S_SABS         | S_G1,
+  S_ABS_G1_NC         = S_ABS          | S_G1      | S_NC,
+  S_ABS_G0            = S_ABS          | S_G0,
+  S_ABS_G0_S          = S_SABS         | S_G0,
+  S_ABS_G0_NC         = S_ABS          | S_G0      | S_NC,
+  S_LO12              = S_ABS          | S_PAGEOFF | S_NC,
+  S_PREL_G3           = S_PREL         | S_G3,
+  S_PREL_G2           = S_PREL         | S_G2,
+  S_PREL_G2_NC        = S_PREL         | S_G2      | S_NC,
+  S_PREL_G1           = S_PREL         | S_G1,
+  S_PREL_G1_NC        = S_PREL         | S_G1      | S_NC,
+  S_PREL_G0           = S_PREL         | S_G0,
+  S_PREL_G0_NC        = S_PREL         | S_G0      | S_NC,
+  S_GOT_LO12          = S_GOT          | S_PAGEOFF | S_NC,
+  S_GOT_PAGE          = S_GOT          | S_PAGE,
+  S_GOT_PAGE_LO15     = S_GOT          | S_LO15    | S_NC,
+  S_GOT_AUTH_LO12     = S_GOT_AUTH     | S_PAGEOFF | S_NC,
+  S_GOT_AUTH_PAGE     = S_GOT_AUTH     | S_PAGE,
+  S_DTPREL_G2         = S_DTPREL       | S_G2,
+  S_DTPREL_G1         = S_DTPREL       | S_G1,
+  S_DTPREL_G1_NC      = S_DTPREL       | S_G1      | S_NC,
+  S_DTPREL_G0         = S_DTPREL       | S_G0,
+  S_DTPREL_G0_NC      = S_DTPREL       | S_G0      | S_NC,
+  S_DTPREL_HI12       = S_DTPREL       | S_HI12,
+  S_DTPREL_LO12       = S_DTPREL       | S_PAGEOFF,
+  S_DTPREL_LO12_NC    = S_DTPREL       | S_PAGEOFF | S_NC,
+  S_GOTTPREL_PAGE     = S_GOTTPREL     | S_PAGE,
+  S_GOTTPREL_LO12_NC  = S_GOTTPREL     | S_PAGEOFF | S_NC,
+  S_GOTTPREL_G1       = S_GOTTPREL     | S_G1,
+  S_GOTTPREL_G0_NC    = S_GOTTPREL     | S_G0      | S_NC,
+  S_TPREL_G2          = S_TPREL        | S_G2,
+  S_TPREL_G1          = S_TPREL        | S_G1,
+  S_TPREL_G1_NC       = S_TPREL        | S_G1      | S_NC,
+  S_TPREL_G0          = S_TPREL        | S_G0,
+  S_TPREL_G0_NC       = S_TPREL        | S_G0      | S_NC,
+  S_TPREL_HI12        = S_TPREL        | S_HI12,
+  S_TPREL_LO12        = S_TPREL        | S_PAGEOFF,
+  S_TPREL_LO12_NC     = S_TPREL        | S_PAGEOFF | S_NC,
+  S_TLSDESC_LO12      = S_TLSDESC      | S_PAGEOFF,
+  S_TLSDESC_PAGE      = S_TLSDESC      | S_PAGE,
+  S_TLSDESC_AUTH_LO12 = S_TLSDESC_AUTH | S_PAGEOFF,
+  S_TLSDESC_AUTH_PAGE = S_TLSDESC_AUTH | S_PAGE,
+  S_SECREL_LO12       = S_SECREL       | S_PAGEOFF,
+  S_SECREL_HI12       = S_SECREL       | S_HI12,
+
+  // ELF relocation specifiers in data directives:
+  S_PLT          = 0x400,
+  S_GOTPCREL,
+
+  // Mach-O @ relocation specifiers:
+  S_MACHO_GOT,
+  S_MACHO_GOTPAGE,
+  S_MACHO_GOTPAGEOFF,
+  S_MACHO_PAGE,
+  S_MACHO_PAGEOFF,
+  S_MACHO_TLVP,
+  S_MACHO_TLVPPAGE,
+  S_MACHO_TLVPPAGEOFF,
+
+  S_INVALID  = 0xfff
+  // clang-format on
+};
+
 /// Return the string representation of the ELF relocation specifier
 /// (e.g. ":got:", ":lo12:").
 StringRef getSpecifierName(const MCSpecifierExpr &Expr);
 
 inline Specifier getSymbolLoc(Specifier S) {
-  return static_cast<Specifier>(S & AArch64MCExpr::VK_SymLocBits);
+  return static_cast<Specifier>(S & AArch64::S_SymLocBits);
 }
 
 inline Specifier getAddressFrag(Specifier S) {
-  return static_cast<Specifier>(S & AArch64MCExpr::VK_AddressFragBits);
+  return static_cast<Specifier>(S & AArch64::S_AddressFragBits);
 }
 
-inline bool isNotChecked(Specifier S) { return S & AArch64MCExpr::VK_NC; }
+inline bool isNotChecked(Specifier S) { return S & AArch64::S_NC; }
 } // namespace AArch64
 
+class AArch64AuthMCExpr final : public MCSpecifierExpr {
+  uint16_t Discriminator;
+  AArch64PACKey::ID Key;
+
+  explicit AArch64AuthMCExpr(const MCExpr *Expr, uint16_t Discriminator,
+                             AArch64PACKey::ID Key, bool HasAddressDiversity,
+                             SMLoc Loc)
+      : MCSpecifierExpr(
+            Expr, HasAddressDiversity ? AArch64::S_AUTHADDR : AArch64::S_AUTH,
+            Loc),
+        Discriminator(Discriminator), Key(Key) {}
+
+public:
+  static const AArch64AuthMCExpr *
+  create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key,
+         bool HasAddressDiversity, MCContext &Ctx, SMLoc Loc = SMLoc());
+
+  AArch64PACKey::ID getKey() const { return Key; }
+  uint16_t getDiscriminator() const { return Discriminator; }
+  bool hasAddressDiversity() const {
+    return getSpecifier() == AArch64::S_AUTHADDR;
+  }
+
+  void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
+
+  static bool classof(const MCExpr *E) {
+    auto *SE = dyn_cast<MCSpecifierExpr>(E);
+    return SE && (SE->getSpecifier() == AArch64::S_AUTH ||
+                  SE->getSpecifier() == AArch64::S_AUTHADDR);
+  }
+};
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index 6db0d7de45e5..6e5a69030dbc 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64FixupKinds.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -223,6 +223,11 @@ class AArch64MCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// getMachineOpValue - Return binary encoding of operand. If the machine
 /// operand requires relocation, record the relocation and return zero.
 unsigned
@@ -248,7 +253,7 @@ AArch64MCCodeEmitter::getLdStUImm12OpValue(const MCInst &MI, unsigned OpIdx,
   else {
     assert(MO.isExpr() && "unable to encode load/store imm operand");
     MCFixupKind Kind = MCFixupKind(FixupKind);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
     ++MCNumFixups;
   }
 
@@ -269,14 +274,11 @@ AArch64MCCodeEmitter::getAdrLabelOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected target type!");
   const MCExpr *Expr = MO.getExpr();
 
-  MCFixupKind Kind = MI.getOpcode() == AArch64::ADR
-                         ? MCFixupKind(AArch64::fixup_aarch64_pcrel_adr_imm21)
-                         : MCFixupKind(AArch64::fixup_aarch64_pcrel_adrp_imm21);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-
+  unsigned Kind = MI.getOpcode() == AArch64::ADR
+                      ? AArch64::fixup_aarch64_pcrel_adr_imm21
+                      : AArch64::fixup_aarch64_pcrel_adrp_imm21;
+  addFixup(Fixups, 0, Expr, Kind, true);
   MCNumFixups += 1;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -301,18 +303,17 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx,
   const MCExpr *Expr = MO.getExpr();
 
   // Encode the 12 bits of the fixup.
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_add_imm12);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+  MCFixupKind Kind = AArch64::fixup_aarch64_add_imm12;
+  addFixup(Fixups, 0, Expr, Kind);
 
   ++MCNumFixups;
 
   // Set the shift bit of the add instruction for relocation types
   // R_AARCH64_TLSLE_ADD_TPREL_HI12 and R_AARCH64_TLSLD_ADD_DTPREL_HI12.
   if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
-    AArch64MCExpr::Specifier RefKind = A64E->getSpecifier();
-    if (RefKind == AArch64MCExpr::VK_TPREL_HI12 ||
-        RefKind == AArch64MCExpr::VK_DTPREL_HI12 ||
-        RefKind == AArch64MCExpr::VK_SECREL_HI12)
+    AArch64::Specifier RefKind = A64E->getSpecifier();
+    if (RefKind == AArch64::S_TPREL_HI12 || RefKind == AArch64::S_DTPREL_HI12 ||
+        RefKind == AArch64::S_SECREL_HI12)
       ShiftVal = 12;
   }
   return ShiftVal == 0 ? 0 : (1 << ShiftVal);
@@ -330,12 +331,9 @@ uint32_t AArch64MCCodeEmitter::getCondBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch19);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch19,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -351,12 +349,9 @@ uint32_t AArch64MCCodeEmitter::getCondCompBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch9);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
+                                   AArch64::fixup_aarch64_pcrel_branch9, true));
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -374,12 +369,9 @@ AArch64MCCodeEmitter::getPAuthPCRelOpValue(const MCInst &MI, unsigned OpIdx,
     return -(MO.getImm());
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch16);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch16,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -396,12 +388,9 @@ AArch64MCCodeEmitter::getLoadLiteralOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_ldr_pcrel_imm19);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_ldr_pcrel_imm19,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -424,8 +413,8 @@ AArch64MCCodeEmitter::getMoveWideImmOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected movz/movk immediate");
 
-  Fixups.push_back(MCFixup::create(
-      0, MO.getExpr(), MCFixupKind(AArch64::fixup_aarch64_movw), MI.getLoc()));
+  Fixups.push_back(
+      MCFixup::create(0, MO.getExpr(), AArch64::fixup_aarch64_movw));
 
   ++MCNumFixups;
 
@@ -444,12 +433,9 @@ uint32_t AArch64MCCodeEmitter::getTestBranchTargetOpValue(
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected ADR target type!");
 
-  MCFixupKind Kind = MCFixupKind(AArch64::fixup_aarch64_pcrel_branch14);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-
+  addFixup(Fixups, 0, MO.getExpr(), AArch64::fixup_aarch64_pcrel_branch14,
+           true);
   ++MCNumFixups;
-
-  // All of the information is in the fixup.
   return 0;
 }
 
@@ -466,10 +452,10 @@ AArch64MCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
     return MO.getImm();
   assert(MO.isExpr() && "Unexpected ADR target type!");
 
-  MCFixupKind Kind = MI.getOpcode() == AArch64::BL
-                         ? MCFixupKind(AArch64::fixup_aarch64_pcrel_call26)
-                         : MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+  unsigned Kind = MI.getOpcode() == AArch64::BL
+                      ? AArch64::fixup_aarch64_pcrel_call26
+                      : AArch64::fixup_aarch64_pcrel_branch26;
+  addFixup(Fixups, 0, MO.getExpr(), Kind, true);
 
   ++MCNumFixups;
 
@@ -720,13 +706,13 @@ unsigned AArch64MCCodeEmitter::fixMOVZ(const MCInst &MI, unsigned EncodedValue,
   const MCExpr *E = UImm16MO.getExpr();
   if (auto *A64E = dyn_cast<MCSpecifierExpr>(E)) {
     switch (A64E->getSpecifier()) {
-    case AArch64MCExpr::VK_DTPREL_G2:
-    case AArch64MCExpr::VK_DTPREL_G1:
-    case AArch64MCExpr::VK_DTPREL_G0:
-    case AArch64MCExpr::VK_GOTTPREL_G1:
-    case AArch64MCExpr::VK_TPREL_G2:
-    case AArch64MCExpr::VK_TPREL_G1:
-    case AArch64MCExpr::VK_TPREL_G0:
+    case AArch64::S_DTPREL_G2:
+    case AArch64::S_DTPREL_G1:
+    case AArch64::S_DTPREL_G0:
+    case AArch64::S_GOTTPREL_G1:
+    case AArch64::S_TPREL_G2:
+    case AArch64::S_TPREL_G1:
+    case AArch64::S_TPREL_G0:
       return EncodedValue & ~(1u << 30);
     default:
       // Nothing to do for an unsigned fixup.
@@ -749,7 +735,7 @@ void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI,
     auto Reloc = STI.getTargetTriple().getEnvironment() == Triple::GNUILP32
                      ? ELF::R_AARCH64_P32_TLSDESC_CALL
                      : ELF::R_AARCH64_TLSDESC_CALL;
-    Fixups.push_back(MCFixup::create(0, MI.getOperand(0).getExpr(), Reloc));
+    addFixup(Fixups, 0, MI.getOperand(0).getExpr(), Reloc);
     return;
   }
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index a3f58ca4ee14..3d4a14b162bf 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "AArch64MCExpr.h"
 #include "AArch64MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
@@ -19,7 +18,7 @@ const AArch64AuthMCExpr *AArch64AuthMCExpr::create(const MCExpr *Expr,
                                                    uint16_t Discriminator,
                                                    AArch64PACKey::ID Key,
                                                    bool HasAddressDiversity,
-                                                   MCContext &Ctx) {
+                                                   MCContext &Ctx, SMLoc Loc) {
   return new (Ctx)
-      AArch64AuthMCExpr(Expr, Discriminator, Key, HasAddressDiversity);
+      AArch64AuthMCExpr(Expr, Discriminator, Key, HasAddressDiversity, Loc);
 }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
deleted file mode 100644
index 8ffd14e338e1..000000000000
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
+++ /dev/null
@@ -1,174 +0,0 @@
-//=--- AArch64MCExpr.h - AArch64 specific MC expression classes ---*- C++ -*-=//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes AArch64-specific MCExprs, used for modifiers like
-// ":lo12:" or ":gottprel_g1:".
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCEXPR_H
-#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCEXPR_H
-
-#include "Utils/AArch64BaseInfo.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/ErrorHandling.h"
-
-namespace llvm {
-
-namespace AArch64MCExpr {
-using Specifier = uint16_t;
-enum {
-  // clang-format off
-    None          = 0,
-    // Symbol locations specifying (roughly speaking) what calculation should be
-    // performed to construct the final address for the relocated
-    // symbol. E.g. direct, via the GOT, ...
-    VK_ABS          = 0x001,
-    VK_SABS         = 0x002,
-    VK_PREL         = 0x003,
-    VK_GOT          = 0x004,
-    VK_DTPREL       = 0x005,
-    VK_GOTTPREL     = 0x006,
-    VK_TPREL        = 0x007,
-    VK_TLSDESC      = 0x008,
-    VK_SECREL       = 0x009,
-    VK_AUTH         = 0x00a,
-    VK_AUTHADDR     = 0x00b,
-    VK_GOT_AUTH     = 0x00c,
-    VK_TLSDESC_AUTH = 0x00d,
-    VK_SymLocBits   = 0x00f,
-
-    // Variants specifying which part of the final address calculation is
-    // used. E.g. the low 12 bits for an ADD/LDR, the middle 16 bits for a
-    // MOVZ/MOVK.
-    VK_PAGE     = 0x010,
-    VK_PAGEOFF  = 0x020,
-    VK_HI12     = 0x030,
-    VK_G0       = 0x040,
-    VK_G1       = 0x050,
-    VK_G2       = 0x060,
-    VK_G3       = 0x070,
-    VK_LO15     = 0x080,
-    VK_AddressFragBits = 0x0f0,
-
-    // Whether the final relocation is a checked one (where a linker should
-    // perform a range-check on the final address) or not. Note that this field
-    // is unfortunately sometimes omitted from the assembly syntax. E.g. :lo12:
-    // on its own is a non-checked relocation. We side with ELF on being
-    // explicit about this!
-    VK_NC       = 0x100,
-
-    // Convenience definitions for referring to specific textual representations
-    // of relocation specifiers. Note that this means the "_NC" is sometimes
-    // omitted in line with assembly syntax here (VK_LO12 rather than VK_LO12_NC
-    // since a user would write ":lo12:").
-    VK_CALL              = VK_ABS,
-    VK_ABS_PAGE          = VK_ABS          | VK_PAGE,
-    VK_ABS_PAGE_NC       = VK_ABS          | VK_PAGE    | VK_NC,
-    VK_ABS_G3            = VK_ABS          | VK_G3,
-    VK_ABS_G2            = VK_ABS          | VK_G2,
-    VK_ABS_G2_S          = VK_SABS         | VK_G2,
-    VK_ABS_G2_NC         = VK_ABS          | VK_G2      | VK_NC,
-    VK_ABS_G1            = VK_ABS          | VK_G1,
-    VK_ABS_G1_S          = VK_SABS         | VK_G1,
-    VK_ABS_G1_NC         = VK_ABS          | VK_G1      | VK_NC,
-    VK_ABS_G0            = VK_ABS          | VK_G0,
-    VK_ABS_G0_S          = VK_SABS         | VK_G0,
-    VK_ABS_G0_NC         = VK_ABS          | VK_G0      | VK_NC,
-    VK_LO12              = VK_ABS          | VK_PAGEOFF | VK_NC,
-    VK_PREL_G3           = VK_PREL         | VK_G3,
-    VK_PREL_G2           = VK_PREL         | VK_G2,
-    VK_PREL_G2_NC        = VK_PREL         | VK_G2      | VK_NC,
-    VK_PREL_G1           = VK_PREL         | VK_G1,
-    VK_PREL_G1_NC        = VK_PREL         | VK_G1      | VK_NC,
-    VK_PREL_G0           = VK_PREL         | VK_G0,
-    VK_PREL_G0_NC        = VK_PREL         | VK_G0      | VK_NC,
-    VK_GOT_LO12          = VK_GOT          | VK_PAGEOFF | VK_NC,
-    VK_GOT_PAGE          = VK_GOT          | VK_PAGE,
-    VK_GOT_PAGE_LO15     = VK_GOT          | VK_LO15    | VK_NC,
-    VK_GOT_AUTH_LO12     = VK_GOT_AUTH     | VK_PAGEOFF | VK_NC,
-    VK_GOT_AUTH_PAGE     = VK_GOT_AUTH     | VK_PAGE,
-    VK_DTPREL_G2         = VK_DTPREL       | VK_G2,
-    VK_DTPREL_G1         = VK_DTPREL       | VK_G1,
-    VK_DTPREL_G1_NC      = VK_DTPREL       | VK_G1      | VK_NC,
-    VK_DTPREL_G0         = VK_DTPREL       | VK_G0,
-    VK_DTPREL_G0_NC      = VK_DTPREL       | VK_G0      | VK_NC,
-    VK_DTPREL_HI12       = VK_DTPREL       | VK_HI12,
-    VK_DTPREL_LO12       = VK_DTPREL       | VK_PAGEOFF,
-    VK_DTPREL_LO12_NC    = VK_DTPREL       | VK_PAGEOFF | VK_NC,
-    VK_GOTTPREL_PAGE     = VK_GOTTPREL     | VK_PAGE,
-    VK_GOTTPREL_LO12_NC  = VK_GOTTPREL     | VK_PAGEOFF | VK_NC,
-    VK_GOTTPREL_G1       = VK_GOTTPREL     | VK_G1,
-    VK_GOTTPREL_G0_NC    = VK_GOTTPREL     | VK_G0      | VK_NC,
-    VK_TPREL_G2          = VK_TPREL        | VK_G2,
-    VK_TPREL_G1          = VK_TPREL        | VK_G1,
-    VK_TPREL_G1_NC       = VK_TPREL        | VK_G1      | VK_NC,
-    VK_TPREL_G0          = VK_TPREL        | VK_G0,
-    VK_TPREL_G0_NC       = VK_TPREL        | VK_G0      | VK_NC,
-    VK_TPREL_HI12        = VK_TPREL        | VK_HI12,
-    VK_TPREL_LO12        = VK_TPREL        | VK_PAGEOFF,
-    VK_TPREL_LO12_NC     = VK_TPREL        | VK_PAGEOFF | VK_NC,
-    VK_TLSDESC_LO12      = VK_TLSDESC      | VK_PAGEOFF,
-    VK_TLSDESC_PAGE      = VK_TLSDESC      | VK_PAGE,
-    VK_TLSDESC_AUTH_LO12 = VK_TLSDESC_AUTH | VK_PAGEOFF,
-    VK_TLSDESC_AUTH_PAGE = VK_TLSDESC_AUTH | VK_PAGE,
-    VK_SECREL_LO12       = VK_SECREL       | VK_PAGEOFF,
-    VK_SECREL_HI12       = VK_SECREL       | VK_HI12,
-
-    // ELF relocation specifiers in data directives:
-    VK_PLT          = 0x400,
-    VK_GOTPCREL,
-
-    // Mach-O @ relocation specifiers:
-    M_GOT,
-    M_GOTPAGE,
-    M_GOTPAGEOFF,
-    M_PAGE,
-    M_PAGEOFF,
-    M_TLVP,
-    M_TLVPPAGE,
-    M_TLVPPAGEOFF,
-
-    VK_INVALID  = 0xfff
-  // clang-format on
-};
-} // namespace AArch64MCExpr
-
-class AArch64AuthMCExpr final : public MCSpecifierExpr {
-  uint16_t Discriminator;
-  AArch64PACKey::ID Key;
-
-  explicit AArch64AuthMCExpr(const MCExpr *Expr, uint16_t Discriminator,
-                             AArch64PACKey::ID Key, bool HasAddressDiversity)
-      : MCSpecifierExpr(Expr, HasAddressDiversity ? AArch64MCExpr::VK_AUTHADDR
-                                                  : AArch64MCExpr::VK_AUTH),
-        Discriminator(Discriminator), Key(Key) {}
-
-public:
-  static const AArch64AuthMCExpr *
-  create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key,
-         bool HasAddressDiversity, MCContext &Ctx);
-
-  AArch64PACKey::ID getKey() const { return Key; }
-  uint16_t getDiscriminator() const { return Discriminator; }
-  bool hasAddressDiversity() const {
-    return getSpecifier() == AArch64MCExpr::VK_AUTHADDR;
-  }
-
-  void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
-
-  static bool classof(const MCExpr *E) {
-    auto *SE = dyn_cast<MCSpecifierExpr>(E);
-    return SE && (SE->getSpecifier() == AArch64MCExpr::VK_AUTH ||
-                  SE->getSpecifier() == AArch64MCExpr::VK_AUTHADDR);
-  }
-};
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index bce99b10f23b..61458d7c24be 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/AArch64FixupKinds.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -34,8 +33,8 @@ namespace {
 
 class AArch64MachObjectWriter : public MCMachObjectTargetWriter {
   bool getAArch64FixupKindMachOInfo(const MCFixup &Fixup, unsigned &RelocType,
-                                    AArch64MCExpr::Specifier Spec,
-                                    unsigned &Log2Size, const MCAssembler &Asm);
+                                    AArch64::Specifier Spec, unsigned &Log2Size,
+                                    const MCAssembler &Asm);
 
 public:
   AArch64MachObjectWriter(uint32_t CPUType, uint32_t CPUSubtype, bool IsILP32)
@@ -49,7 +48,7 @@ class AArch64MachObjectWriter : public MCMachObjectTargetWriter {
 } // end anonymous namespace
 
 bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo(
-    const MCFixup &Fixup, unsigned &RelocType, AArch64MCExpr::Specifier Spec,
+    const MCFixup &Fixup, unsigned &RelocType, AArch64::Specifier Spec,
     unsigned &Log2Size, const MCAssembler &Asm) {
   RelocType = unsigned(MachO::ARM64_RELOC_UNSIGNED);
   Log2Size = ~0U;
@@ -66,12 +65,12 @@ bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo(
     return true;
   case FK_Data_4:
     Log2Size = Log2_32(4);
-    if (Spec == AArch64MCExpr::M_GOT)
+    if (Spec == AArch64::S_MACHO_GOT)
       RelocType = unsigned(MachO::ARM64_RELOC_POINTER_TO_GOT);
     return true;
   case FK_Data_8:
     Log2Size = Log2_32(8);
-    if (Spec == AArch64MCExpr::M_GOT)
+    if (Spec == AArch64::S_MACHO_GOT)
       RelocType = unsigned(MachO::ARM64_RELOC_POINTER_TO_GOT);
     return true;
   case AArch64::fixup_aarch64_add_imm12:
@@ -84,13 +83,13 @@ bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo(
     switch (Spec) {
     default:
       return false;
-    case AArch64MCExpr::M_PAGEOFF:
+    case AArch64::S_MACHO_PAGEOFF:
       RelocType = unsigned(MachO::ARM64_RELOC_PAGEOFF12);
       return true;
-    case AArch64MCExpr::M_GOTPAGEOFF:
+    case AArch64::S_MACHO_GOTPAGEOFF:
       RelocType = unsigned(MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12);
       return true;
-    case AArch64MCExpr::M_TLVPPAGEOFF:
+    case AArch64::S_MACHO_TLVPPAGEOFF:
       RelocType = unsigned(MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12);
       return true;
     }
@@ -101,13 +100,13 @@ bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo(
     default:
       reportError(Fixup.getLoc(), "ADR/ADRP relocations must be GOT relative");
       return false;
-    case AArch64MCExpr::M_PAGE:
+    case AArch64::S_MACHO_PAGE:
       RelocType = unsigned(MachO::ARM64_RELOC_PAGE21);
       return true;
-    case AArch64MCExpr::M_GOTPAGE:
+    case AArch64::S_MACHO_GOTPAGE:
       RelocType = unsigned(MachO::ARM64_RELOC_GOT_LOAD_PAGE21);
       return true;
-    case AArch64MCExpr::M_TLVPPAGE:
+    case AArch64::S_MACHO_TLVPPAGE:
       RelocType = unsigned(MachO::ARM64_RELOC_TLVP_LOAD_PAGE21);
       return true;
     }
@@ -148,7 +147,7 @@ static bool canUseLocalRelocation(const MCSectionMachO &Section,
 void AArch64MachObjectWriter::recordRelocation(
     MachObjectWriter *Writer, MCAssembler &Asm, const MCFragment *Fragment,
     const MCFixup &Fixup, MCValue Target, uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
 
   // See <reloc.h>.
   uint32_t FixupOffset = Asm.getFragmentOffset(*Fragment);
@@ -189,9 +188,8 @@ void AArch64MachObjectWriter::recordRelocation(
     return;
   }
 
-  if (!getAArch64FixupKindMachOInfo(
-          Fixup, Type, AArch64MCExpr::Specifier(Target.getSpecifier()),
-          Log2Size, Asm)) {
+  if (!getAArch64FixupKindMachOInfo(Fixup, Type, Target.getSpecifier(),
+                                    Log2Size, Asm)) {
     reportError(Fixup.getLoc(), "unknown AArch64 fixup kind!");
     return;
   }
@@ -218,7 +216,7 @@ void AArch64MachObjectWriter::recordRelocation(
     // Check for "_foo@got - .", which comes through here as:
     // Ltmp0:
     //    ... _foo@got - Ltmp0
-    if (Target.getSpecifier() == AArch64MCExpr::M_GOT &&
+    if (Target.getSpecifier() == AArch64::S_MACHO_GOT &&
         Asm.getSymbolOffset(*B) ==
             Asm.getFragmentOffset(*Fragment) + Fixup.getOffset()) {
       // SymB is the PC, so use a PC-rel pointer-to-GOT relocation.
@@ -229,7 +227,7 @@ void AArch64MachObjectWriter::recordRelocation(
       MRE.r_word1 = (IsPCRel << 24) | (Log2Size << 25) | (Type << 28);
       Writer->addRelocation(A_Base, Fragment->getParent(), MRE);
       return;
-    } else if (Target.getSpecifier() != AArch64MCExpr::None) {
+    } else if (Target.getSpecifier() != AArch64::S_None) {
       // Otherwise, neither symbol can be modified.
       reportError(Fixup.getLoc(), "unsupported relocation of modified symbol");
       return;
@@ -382,8 +380,8 @@ void AArch64MachObjectWriter::recordRelocation(
     Value = 0;
   }
 
-  if (Target.getSpecifier() == AArch64MCExpr::VK_AUTH ||
-      Target.getSpecifier() == AArch64MCExpr::VK_AUTHADDR) {
+  if (Target.getSpecifier() == AArch64::S_AUTH ||
+      Target.getSpecifier() == AArch64::S_AUTHADDR) {
     auto *Expr = cast<AArch64AuthMCExpr>(Fixup.getValue());
 
     assert(Type == MachO::ARM64_RELOC_UNSIGNED);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
index d878f1e044b8..f840058cd759 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64TARGETSTREAMER_H
 #define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64TARGETSTREAMER_H
 
-#include "AArch64MCExpr.h"
+#include "AArch64MCAsmInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/MC/MCELFStreamer.h"
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
index 70df95029c33..a53b676142a0 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWinCOFFObjectWriter.h"
@@ -49,26 +48,28 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
     MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup,
     bool IsCrossSection, const MCAsmBackend &MAB) const {
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = Fixup.isPCRel();
   if (IsCrossSection) {
     // IMAGE_REL_ARM64_REL64 does not exist. We treat FK_Data_8 as FK_PCRel_4 so
     // that .xword a-b can lower to IMAGE_REL_ARM64_REL32. This allows generic
     // instrumentation to not bother with the COFF limitation. A negative value
     // needs attention.
-    if (FixupKind != FK_Data_4 && FixupKind != FK_Data_8) {
+    if (PCRel || (FixupKind != FK_Data_4 && FixupKind != FK_Data_8)) {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_ARM64_ADDR32;
     }
-    FixupKind = FK_PCRel_4;
+    FixupKind = FK_Data_4;
+    PCRel = true;
   }
 
   auto Spec = Target.getSpecifier();
   const MCExpr *Expr = Fixup.getValue();
 
   if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
-    AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
+    AArch64::Specifier Spec = A64E->getSpecifier();
     switch (AArch64::getSymbolLoc(Spec)) {
-    case AArch64MCExpr::VK_ABS:
-    case AArch64MCExpr::VK_SECREL:
+    case AArch64::S_ABS:
+    case AArch64::S_SECREL:
       // Supported
       break;
     default:
@@ -93,17 +94,14 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
     return COFF::IMAGE_REL_ARM64_ABSOLUTE; // Dummy return value
   }
 
-  case FK_PCRel_4:
-    return COFF::IMAGE_REL_ARM64_REL32;
-
   case FK_Data_4:
+    if (PCRel)
+      return COFF::IMAGE_REL_ARM64_REL32;
     switch (Spec) {
     default:
       return COFF::IMAGE_REL_ARM64_ADDR32;
     case MCSymbolRefExpr::VK_COFF_IMGREL32:
       return COFF::IMAGE_REL_ARM64_ADDR32NB;
-    case MCSymbolRefExpr::VK_SECREL:
-      return COFF::IMAGE_REL_ARM64_SECREL;
     }
 
   case FK_Data_8:
@@ -117,10 +115,10 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
 
   case AArch64::fixup_aarch64_add_imm12:
     if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
-      AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
-      if (Spec == AArch64MCExpr::VK_SECREL_LO12)
+      AArch64::Specifier Spec = A64E->getSpecifier();
+      if (Spec == AArch64::S_SECREL_LO12)
         return COFF::IMAGE_REL_ARM64_SECREL_LOW12A;
-      if (Spec == AArch64MCExpr::VK_SECREL_HI12)
+      if (Spec == AArch64::S_SECREL_HI12)
         return COFF::IMAGE_REL_ARM64_SECREL_HIGH12A;
     }
     return COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A;
@@ -131,8 +129,8 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
   case AArch64::fixup_aarch64_ldst_imm12_scale8:
   case AArch64::fixup_aarch64_ldst_imm12_scale16:
     if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
-      AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
-      if (Spec == AArch64MCExpr::VK_SECREL_LO12)
+      AArch64::Specifier Spec = A64E->getSpecifier();
+      if (Spec == AArch64::S_SECREL_LO12)
         return COFF::IMAGE_REL_ARM64_SECREL_LOW12L;
     }
     return COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index fe033218e228..7293f3ac2d4f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -547,6 +547,17 @@ extern char &GCNRewritePartialRegUsesID;
 void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
 extern char &AMDGPUWaitSGPRHazardsLegacyID;
 
+class AMDGPURewriteAGPRCopyMFMAPass
+    : public PassInfoMixin<AMDGPURewriteAGPRCopyMFMAPass> {
+public:
+  AMDGPURewriteAGPRCopyMFMAPass() = default;
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM);
+};
+
+void initializeAMDGPURewriteAGPRCopyMFMALegacyPass(PassRegistry &);
+extern char &AMDGPURewriteAGPRCopyMFMALegacyID;
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 3adab343d493..07b88c60acf3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1094,6 +1094,12 @@ def FeatureBitOp3Insts : SubtargetFeature<"bitop3-insts",
   "Has v_bitop3_b32/v_bitop3_b16 instructions"
 >;
 
+def FeatureTransposeLoadF4F6Insts : SubtargetFeature<"transpose-load-f4f6-insts",
+  "HasTransposeLoadF4F6Insts",
+  "true",
+  "Has ds_load_tr4/tr6 and global_load_tr4/tr6 instructions"
+>;
+
 def FeaturePrngInst : SubtargetFeature<"prng-inst",
   "HasPrngInst",
   "true",
@@ -1313,6 +1319,12 @@ def FeatureMemToLDSLoad : SubtargetFeature<"vmem-to-lds-load-insts",
   "The platform has memory to lds instructions (global_load w/lds bit set, buffer_load w/lds bit set or global_load_lds. This does not include scratch_load_lds."
 >;
 
+def FeatureLdsBarrierArriveAtomic : SubtargetFeature< "lds-barrier-arrive-atomic",
+  "HasLdsBarrierArriveAtomic",
+  "true",
+  "Has LDS barrier-arrive atomic instructions"
+>;
+
 // Dummy feature used to disable assembler instructions.
 def FeatureDisable : SubtargetFeature<"",
   "FeatureDisable","true",
@@ -1932,6 +1944,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureScalarDwordx3Loads,
    FeatureDPPSrc1SGPR,
    FeatureBitOp3Insts,
+   FeatureTransposeLoadF4F6Insts,
    FeatureBF16ConversionInsts,
    FeatureCvtPkF16F32Inst,
    FeatureMinimum3Maximum3PKF16,
@@ -1947,6 +1960,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureMemoryAtomicFAddF32DenormalSupport,
    FeatureKernargPreload,
    FeatureLshlAddU64Inst,
+   FeatureLdsBarrierArriveAtomic,
    FeatureSetPrioIncWgInst,
 ]>;
 
@@ -2353,11 +2367,17 @@ def D16PreservesUnusedBits :
 def LDSRequiresM0Init : Predicate<"Subtarget->ldsRequiresM0Init()">;
 def NotLDSRequiresM0Init : Predicate<"!Subtarget->ldsRequiresM0Init()">;
 
+def HasMTBUFInsts : Predicate<"Subtarget->hasMTBUFInsts()">,
+  AssemblerPredicate<(all_of (not FeatureGFX1250Insts))>;
+
+def HasFormattedMUBUFInsts : Predicate<"Subtarget->hasFormattedMUBUFInsts()">,
+  AssemblerPredicate<(all_of (not FeatureGFX1250Insts))>;
+
 def HasExportInsts : Predicate<"Subtarget->hasExportInsts()">,
-  AssemblerPredicate<(all_of (not FeatureGFX90AInsts))>;
+  AssemblerPredicate<(all_of (not FeatureGFX90AInsts), (not FeatureGFX1250Insts))>;
 
 def HasVINTERPEncoding : Predicate<"Subtarget->hasVINTERPEncoding()">,
-  AssemblerPredicate<(all_of FeatureGFX11Insts)>;
+  AssemblerPredicate<(all_of FeatureGFX11Insts, (not FeatureGFX1250Insts))>;
 
 def HasDSAddTid : Predicate<"Subtarget->getGeneration() >= AMDGPUSubtarget::GFX9">,
   AssemblerPredicate<(all_of FeatureGFX9Insts)>;
@@ -2620,6 +2640,9 @@ def HasPseudoScalarTrans : Predicate<"Subtarget->hasPseudoScalarTrans()">,
 def HasBitOp3Insts : Predicate<"Subtarget->hasBitOp3Insts()">,
   AssemblerPredicate<(all_of FeatureBitOp3Insts)>;
 
+def HasTransposeLoadF4F6Insts : Predicate<"Subtarget->hasTransposeLoadF4F6Insts()">,
+  AssemblerPredicate<(all_of FeatureTransposeLoadF4F6Insts)>;
+
 def HasPrngInst : Predicate<"Subtarget->hasPrngInst()">,
   AssemblerPredicate<(all_of FeaturePrngInst)>;
 
@@ -2670,6 +2693,9 @@ def HasAshrPkInsts : Predicate<"Subtarget->hasAshrPkInsts()">,
 def HasLshlAddU64Inst : Predicate<"Subtarget->hasLshlAddU64Inst()">,
                         AssemblerPredicate<(all_of FeatureLshlAddU64Inst)>;
 
+def HasLdsBarrierArriveAtomic : Predicate<"Subtarget->hasLdsBarrierArriveAtomic()">,
+  AssemblerPredicate<(all_of FeatureLdsBarrierArriveAtomic)>;
+
 def HasSetPrioIncWgInst : Predicate<"Subtarget->hasSetPrioIncWgInst()">,
  AssemblerPredicate<(all_of FeatureSetPrioIncWgInst)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUExpandFeaturePredicates.cpp b/llvm/lib/Target/AMDGPU/AMDGPUExpandFeaturePredicates.cpp
index f1c73e86fb4a..cd9e29a4e7d6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUExpandFeaturePredicates.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUExpandFeaturePredicates.cpp
@@ -13,12 +13,10 @@
 // (AMDGCNSPIRV). These placeholder globals are used to guide target specific
 // lowering, once the concrete target is known, by way of constant folding their
 // value all the way into a terminator (i.e. a controlled block) or into a no
-// live use scenario. The pass makes a best effort attempt to look through
-// calls, i.e. a constant evaluatable passthrough of a predicate value will
-// generally work, however we hard fail if the folding fails, to avoid obtuse
-// BE errors or opaque run time errors. This pass should run as early as
-// possible / immediately after Clang CodeGen, so that the optimisation pipeline
-// and the BE operate with concrete target data.
+// live use scenario. We hard fail if the folding fails, to avoid obtuse BE
+// errors or opaque run time errors. This pass should run as early as possible /
+// immediately after Clang CodeGen, so that the optimisation pipeline and the BE
+// operate with concrete target data.
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
@@ -50,13 +48,13 @@ template <typename C> void collectUsers(Value *V, C &Container) {
 }
 
 inline void setPredicate(const GCNSubtarget &ST, GlobalVariable *P) {
-  const auto IsFeature = P->getName().starts_with("llvm.amdgcn.has");
-  const auto Offset =
+  const bool IsFeature = P->getName().starts_with("llvm.amdgcn.has");
+  const size_t Offset =
       IsFeature ? sizeof("llvm.amdgcn.has") : sizeof("llvm.amdgcn.is");
 
-  auto PV = P->getName().substr(Offset).str();
+  std::string PV = P->getName().substr(Offset).str();
   if (IsFeature) {
-    auto Dx = PV.find(',');
+    size_t Dx = PV.find(',');
     while (Dx != std::string::npos) {
       PV.insert(++Dx, {'+'});
 
@@ -65,7 +63,7 @@ inline void setPredicate(const GCNSubtarget &ST, GlobalVariable *P) {
     PV.insert(PV.cbegin(), '+');
   }
 
-  auto *PTy = P->getValueType();
+  Type *PTy = P->getValueType();
   P->setLinkage(GlobalValue::PrivateLinkage);
   P->setExternallyInitialized(false);
 
@@ -89,8 +87,9 @@ unfoldableFound(Function *Caller, GlobalVariable *P, Instruction *NoFold) {
   return {PreservedAnalyses::none(), false};
 }
 
-std::pair<PreservedAnalyses, bool> handlePredicate(const GCNSubtarget &ST,
-                                                   GlobalVariable *P) {
+std::pair<PreservedAnalyses, bool>
+handlePredicate(const GCNSubtarget &ST, FunctionAnalysisManager &FAM,
+                SmallPtrSet<Function *, 32> &Predicated, GlobalVariable *P) {
   setPredicate(ST, P);
 
   SmallPtrSet<Instruction *, 32> ToFold;
@@ -100,18 +99,25 @@ std::pair<PreservedAnalyses, bool> handlePredicate(const GCNSubtarget &ST,
     return {PreservedAnalyses::all(), true};
 
   do {
-    auto *I = *ToFold.begin();
+    Instruction *I = *ToFold.begin();
     ToFold.erase(I);
 
+    I->dropDroppableUses();
+
+    Function *F = I->getParent()->getParent();
+    auto &DT = FAM.getResult<DominatorTreeAnalysis>(*F);
+    DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
     if (auto *C = ConstantFoldInstruction(I, P->getDataLayout())) {
       collectUsers(I, ToFold);
       I->replaceAllUsesWith(C);
       I->eraseFromParent();
       continue;
-    } else if (I->isTerminator() && ConstantFoldTerminator(I->getParent())) {
-      continue;
-    } else if (I->users().empty()) {
-      continue;
+    } else if (I->isTerminator() &&
+               ConstantFoldTerminator(I->getParent(), true, nullptr, &DTU)) {
+        Predicated.insert(F);
+
+        continue;
     }
 
     return unfoldableFound(I->getParent()->getParent(), P, I);
@@ -140,9 +146,11 @@ AMDGPUExpandFeaturePredicatesPass::run(Module &M, ModuleAnalysisManager &MAM) {
   const auto &ST = TM.getSubtarget<GCNSubtarget>(
       *find_if(M, [](auto &&F) { return !F.isIntrinsic(); }));
 
+  auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+  SmallPtrSet<Function *, 32> Predicated;
   auto Ret = PreservedAnalyses::all();
   for (auto &&P : Predicates) {
-    auto R = handlePredicate(ST, P);
+    auto R = handlePredicate(ST, FAM, Predicated, P);
 
     if (!R.second)
       break;
@@ -152,6 +160,8 @@ AMDGPUExpandFeaturePredicatesPass::run(Module &M, ModuleAnalysisManager &MAM) {
 
   for (auto &&P : Predicates)
     P->eraseFromParent();
+  for (auto &&F : Predicated)
+    removeUnreachableBlocks(*F);
 
   return Ret;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 3db2b3bff2d3..e32accaba85f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -3853,7 +3853,7 @@ bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
   return true;
 }
 
-// Replace load of an illegal type with a store of a bitcast to a friendlier
+// Replace load of an illegal type with a bitcast from a load of a friendlier
 // type.
 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
@@ -4151,32 +4151,96 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
 
 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
-  if (N->getValueType(0) != MVT::i64)
+  SDValue RHS = N->getOperand(1);
+  ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
+  EVT VT = N->getValueType(0);
+  SDValue LHS = N->getOperand(0);
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc SL(N);
+
+  if (VT.getScalarType() != MVT::i64)
     return SDValue();
 
-  const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
-  if (!RHS)
+  // For C >= 32
+  // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
+
+  // On some subtargets, 64-bit shift is a quarter rate instruction. In the
+  // common case, splitting this into a move and a 32-bit shift is faster and
+  // the same code size.
+  KnownBits Known = DAG.computeKnownBits(RHS);
+
+  EVT ElementType = VT.getScalarType();
+  EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
+  EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType)
+                                 : TargetScalarType;
+
+  if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
     return SDValue();
 
-  SelectionDAG &DAG = DCI.DAG;
-  SDLoc SL(N);
-  unsigned RHSVal = RHS->getZExtValue();
+  SDValue ShiftFullAmt =
+      DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+  SDValue ShiftAmt;
+  if (CRHS) {
+    unsigned RHSVal = CRHS->getZExtValue();
+    ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
+                               TargetType);
+  } else if (Known.getMinValue().getZExtValue() ==
+             (ElementType.getSizeInBits() - 1)) {
+    ShiftAmt = ShiftFullAmt;
+  } else {
+    SDValue truncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+    const SDValue ShiftMask =
+        DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+    // This AND instruction will clamp out of bounds shift values.
+    // It will also be removed during later instruction selection.
+    ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, truncShiftAmt, ShiftMask);
+  }
 
-  // For C >= 32
-  // (sra i64:x, C) -> build_pair (sra hi_32(x), C - 32), (sra hi_32(x), 31)
-  if (RHSVal >= 32) {
-    SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
-    Hi = DAG.getFreeze(Hi);
-    SDValue HiShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
-                                  DAG.getConstant(31, SL, MVT::i32));
-    SDValue LoShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
-                                  DAG.getConstant(RHSVal - 32, SL, MVT::i32));
+  EVT ConcatType;
+  SDValue Hi;
+  SDLoc LHSSL(LHS);
+  // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
+  if (VT.isVector()) {
+    unsigned NElts = TargetType.getVectorNumElements();
+    ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
+    SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+    SmallVector<SDValue, 8> HiOps(NElts);
+    SmallVector<SDValue, 16> HiAndLoOps;
 
-    SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {LoShift, HiShift});
-    return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
+    DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, 0, NElts * 2);
+    for (unsigned I = 0; I != NElts; ++I) {
+      HiOps[I] = HiAndLoOps[2 * I + 1];
+    }
+    Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
+  } else {
+    const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
+    ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
+    SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+    Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
   }
+  Hi = DAG.getFreeze(Hi);
 
-  return SDValue();
+  SDValue HiShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftFullAmt);
+  SDValue NewShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt);
+
+  SDValue Vec;
+  if (VT.isVector()) {
+    unsigned NElts = TargetType.getVectorNumElements();
+    SmallVector<SDValue, 8> HiOps;
+    SmallVector<SDValue, 8> LoOps;
+    SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
+
+    DAG.ExtractVectorElements(HiShift, HiOps, 0, NElts);
+    DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
+    for (unsigned I = 0; I != NElts; ++I) {
+      HiAndLoOps[2 * I + 1] = HiOps[I];
+      HiAndLoOps[2 * I] = LoOps[I];
+    }
+    Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
+  } else {
+    Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, HiShift});
+  }
+  return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
 }
 
 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
@@ -4213,7 +4277,7 @@ SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
     return SDValue();
 
   // for C >= 32
-  // i64 (srl x, C) -> (build_pair (srl hi_32(x), C -32), 0)
+  // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
 
   // On some subtargets, 64-bit shift is a quarter rate instruction. In the
   // common case, splitting this into a move and a 32-bit shift is faster and
@@ -4264,7 +4328,8 @@ SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
     Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
   }
 
-  SDValue NewShift = DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt);
+  SDValue NewShift =
+      DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt, N->getFlags());
 
   SDValue Vec;
   if (VT.isVector()) {
@@ -5265,10 +5330,11 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
     break;
   }
   case ISD::SHL:
+  case ISD::SRA:
   case ISD::SRL: {
     // Range metadata can be invalidated when loads are converted to legal types
     // (e.g. v2i64 -> v4i32).
-    // Try to convert vector shl/srl before type legalization so that range
+    // Try to convert vector shl/sra/srl before type legalization so that range
     // metadata can be utilized.
     if (!(N->getValueType(0).isVector() &&
           DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
@@ -5276,14 +5342,10 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
       break;
     if (N->getOpcode() == ISD::SHL)
       return performShlCombine(N, DCI);
+    if (N->getOpcode() == ISD::SRA)
+      return performSraCombine(N, DCI);
     return performSrlCombine(N, DCI);
   }
-  case ISD::SRA: {
-    if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
-      break;
-
-    return performSraCombine(N, DCI);
-  }
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
   case ISD::MUL:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
index 10efbab80567..44eaebffb70d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInsertDelayAlu.cpp
@@ -49,9 +49,17 @@ class AMDGPUInsertDelayAlu {
 
   static bool instructionWaitsForSGPRWrites(const MachineInstr &MI) {
     // These instruction types wait for VA_SDST==0 before issuing.
-    const uint64_t VA_SDST_0 = SIInstrFlags::SALU | SIInstrFlags::SMRD;
+    uint64_t MIFlags = MI.getDesc().TSFlags;
+    if (MIFlags & SIInstrFlags::SMRD)
+      return true;
 
-    return MI.getDesc().TSFlags & VA_SDST_0;
+    if (MIFlags & SIInstrFlags::SALU) {
+      for (auto &Op : MI.operands()) {
+        if (Op.isReg())
+          return true;
+      }
+    }
+    return false;
   }
 
   // Types of delay that can be encoded in an s_delay_alu instruction.
@@ -148,10 +156,9 @@ class AMDGPUInsertDelayAlu {
       SALUCycles = std::max(SALUCycles, RHS.SALUCycles);
     }
 
-    // Update this DelayInfo after issuing an instruction. IsVALU should be 1
-    // when issuing a (non-TRANS) VALU, else 0. IsTRANS should be 1 when issuing
-    // a TRANS, else 0. Cycles is the number of cycles it takes to issue the
-    // instruction.  Return true if there is no longer any useful delay info.
+    // Update this DelayInfo after issuing an instruction of the specified type.
+    // Cycles is the number of cycles it takes to issue the instruction.  Return
+    // true if there is no longer any useful delay info.
     bool advance(DelayType Type, unsigned Cycles) {
       bool Erase = true;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 5477c5eae939..b8996fb97f1c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1031,6 +1031,14 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     // s1 _nan: min(s0, s2)
     // s2 _nan: min(s0, s1)
 
+    // med3 behavior with infinity
+    // s0 +inf: max(s1, s2)
+    // s1 +inf: max(s0, s2)
+    // s2 +inf: max(s0, s1)
+    // s0 -inf: min(s1, s2)
+    // s1 -inf: min(s0, s2)
+    // s2 -inf: min(s0, s1)
+
     // Checking for NaN before canonicalization provides better fidelity when
     // mapping other operations onto fmed3 since the order of operands is
     // unchanged.
@@ -1039,51 +1047,64 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
     const APFloat *ConstSrc1 = nullptr;
     const APFloat *ConstSrc2 = nullptr;
 
-    // TODO: Also can fold to 2 operands with infinities.
-    if ((match(Src0, m_APFloat(ConstSrc0)) && ConstSrc0->isNaN()) ||
+    if ((match(Src0, m_APFloat(ConstSrc0)) &&
+         (ConstSrc0->isNaN() || ConstSrc0->isInfinity())) ||
         isa<UndefValue>(Src0)) {
+      const bool IsPosInfinity = ConstSrc0 && ConstSrc0->isPosInfinity();
       switch (fpenvIEEEMode(II)) {
       case KnownIEEEMode::On:
         // TODO: If Src2 is snan, does it need quieting?
-        if (ConstSrc0 && ConstSrc0->isSignaling())
+        if (ConstSrc0 && ConstSrc0->isNaN() && ConstSrc0->isSignaling())
           return IC.replaceInstUsesWith(II, Src2);
-        V = IC.Builder.CreateMinNum(Src1, Src2);
+
+        V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src1, Src2)
+                          : IC.Builder.CreateMinNum(Src1, Src2);
         break;
       case KnownIEEEMode::Off:
-        V = IC.Builder.CreateMinimumNum(Src1, Src2);
+        V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src1, Src2)
+                          : IC.Builder.CreateMinimumNum(Src1, Src2);
         break;
       case KnownIEEEMode::Unknown:
         break;
       }
-    } else if ((match(Src1, m_APFloat(ConstSrc1)) && ConstSrc1->isNaN()) ||
+    } else if ((match(Src1, m_APFloat(ConstSrc1)) &&
+                (ConstSrc1->isNaN() || ConstSrc1->isInfinity())) ||
                isa<UndefValue>(Src1)) {
+      const bool IsPosInfinity = ConstSrc1 && ConstSrc1->isPosInfinity();
       switch (fpenvIEEEMode(II)) {
       case KnownIEEEMode::On:
         // TODO: If Src2 is snan, does it need quieting?
-        if (ConstSrc1 && ConstSrc1->isSignaling())
+        if (ConstSrc1 && ConstSrc1->isNaN() && ConstSrc1->isSignaling())
           return IC.replaceInstUsesWith(II, Src2);
 
-        V = IC.Builder.CreateMinNum(Src0, Src2);
+        V = IsPosInfinity ? IC.Builder.CreateMaxNum(Src0, Src2)
+                          : IC.Builder.CreateMinNum(Src0, Src2);
         break;
       case KnownIEEEMode::Off:
-        V = IC.Builder.CreateMinimumNum(Src0, Src2);
+        V = IsPosInfinity ? IC.Builder.CreateMaximumNum(Src0, Src2)
+                          : IC.Builder.CreateMinimumNum(Src0, Src2);
         break;
       case KnownIEEEMode::Unknown:
         break;
       }
-    } else if ((match(Src2, m_APFloat(ConstSrc2)) && ConstSrc2->isNaN()) ||
+    } else if ((match(Src2, m_APFloat(ConstSrc2)) &&
+                (ConstSrc2->isNaN() || ConstSrc2->isInfinity())) ||
                isa<UndefValue>(Src2)) {
       switch (fpenvIEEEMode(II)) {
       case KnownIEEEMode::On:
-        if (ConstSrc2 && ConstSrc2->isSignaling()) {
+        if (ConstSrc2 && ConstSrc2->isNaN() && ConstSrc2->isSignaling()) {
           auto *Quieted = ConstantFP::get(II.getType(), ConstSrc2->makeQuiet());
           return IC.replaceInstUsesWith(II, Quieted);
         }
 
-        V = IC.Builder.CreateMinNum(Src0, Src1);
+        V = (ConstSrc2 && ConstSrc2->isPosInfinity())
+                ? IC.Builder.CreateMaxNum(Src0, Src1)
+                : IC.Builder.CreateMinNum(Src0, Src1);
         break;
       case KnownIEEEMode::Off:
-        V = IC.Builder.CreateMaximumNum(Src0, Src1);
+        V = (ConstSrc2 && ConstSrc2->isNegInfinity())
+                ? IC.Builder.CreateMinimumNum(Src0, Src1)
+                : IC.Builder.CreateMaximumNum(Src0, Src1);
         break;
       case KnownIEEEMode::Unknown:
         break;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 5c92c4eb411e..ac66d7db3193 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -16,6 +16,8 @@
 #ifndef MODULE_PASS
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
+MODULE_PASS("amdgpu-expand-feature-predicates",
+            AMDGPUExpandFeaturePredicatesPass(*this))
 MODULE_PASS("amdgpu-always-inline", AMDGPUAlwaysInlinePass())
 MODULE_PASS("amdgpu-export-kernel-runtime-handles", AMDGPUExportKernelRuntimeHandlesPass())
 MODULE_PASS("amdgpu-lower-buffer-fat-pointers",
@@ -104,6 +106,7 @@ MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
 MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-long-branch-reg", GCNPreRALongBranchRegPass())
 MACHINE_FUNCTION_PASS("amdgpu-reserve-wwm-regs", AMDGPUReserveWWMRegsPass())
+MACHINE_FUNCTION_PASS("amdgpu-rewrite-agpr-copy-mfma", AMDGPURewriteAGPRCopyMFMAPass())
 MACHINE_FUNCTION_PASS("amdgpu-rewrite-partial-reg-uses", GCNRewritePartialRegUsesPass())
 MACHINE_FUNCTION_PASS("amdgpu-set-wave-priority", AMDGPUSetWavePriorityPass())
 MACHINE_FUNCTION_PASS("amdgpu-pre-ra-optimizations", GCNPreRAOptimizationsPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index b2ddc6e88966..6a59a28b1d32 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -171,8 +171,7 @@ void RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
   MI.eraseFromParent();
 }
 
-const std::pair<Register, Register>
-RegBankLegalizeHelper::unpackZExt(Register Reg) {
+std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
   auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
   auto Mask = B.buildConstant(SgprRB_S32, 0x0000ffff);
   auto Lo = B.buildAnd(SgprRB_S32, PackedS32, Mask);
@@ -180,16 +179,14 @@ RegBankLegalizeHelper::unpackZExt(Register Reg) {
   return {Lo.getReg(0), Hi.getReg(0)};
 }
 
-const std::pair<Register, Register>
-RegBankLegalizeHelper::unpackSExt(Register Reg) {
+std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
   auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
   auto Lo = B.buildSExtInReg(SgprRB_S32, PackedS32, 16);
   auto Hi = B.buildAShr(SgprRB_S32, PackedS32, B.buildConstant(SgprRB_S32, 16));
   return {Lo.getReg(0), Hi.getReg(0)};
 }
 
-const std::pair<Register, Register>
-RegBankLegalizeHelper::unpackAExt(Register Reg) {
+std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
   auto PackedS32 = B.buildBitcast(SgprRB_S32, Reg);
   auto Lo = PackedS32;
   auto Hi = B.buildLShr(SgprRB_S32, PackedS32, B.buildConstant(SgprRB_S32, 16));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
index 50bd86dc15a1..08cc7d43bd78 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h
@@ -111,9 +111,9 @@ class RegBankLegalizeHelper {
              SmallSet<Register, 4> &SgprWaterfallOperandRegs);
 
   void lowerVccExtToSel(MachineInstr &MI);
-  const std::pair<Register, Register> unpackZExt(Register Reg);
-  const std::pair<Register, Register> unpackSExt(Register Reg);
-  const std::pair<Register, Register> unpackAExt(Register Reg);
+  std::pair<Register, Register> unpackZExt(Register Reg);
+  std::pair<Register, Register> unpackSExt(Register Reg);
+  std::pair<Register, Register> unpackAExt(Register Reg);
   void lowerUnpackBitShift(MachineInstr &MI);
   void lowerV_BFE(MachineInstr &MI);
   void lowerS_BFE(MachineInstr &MI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index a7b08794fdf1..353fb23fa152 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3348,6 +3348,20 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
         MI.eraseFromParent();
       return;
     }
+    case Intrinsic::amdgcn_tensor_load_to_lds:
+    case Intrinsic::amdgcn_tensor_store_from_lds: {
+      constrainOpWithReadfirstlane(B, MI, 1);
+      constrainOpWithReadfirstlane(B, MI, 2);
+      constrainOpWithReadfirstlane(B, MI, 3);
+      constrainOpWithReadfirstlane(B, MI, 4);
+      return;
+    }
+    case Intrinsic::amdgcn_tensor_load_to_lds_d2:
+    case Intrinsic::amdgcn_tensor_store_from_lds_d2: {
+      constrainOpWithReadfirstlane(B, MI, 1);
+      constrainOpWithReadfirstlane(B, MI, 2);
+      return;
+    }
     default: {
       if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
               AMDGPU::lookupRsrcIntrinsic(IntrID)) {
@@ -4541,6 +4555,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_cvt_pknorm_u16:
     case Intrinsic::amdgcn_cvt_pk_i16:
     case Intrinsic::amdgcn_cvt_pk_u16:
+    case Intrinsic::amdgcn_cvt_pk_f16_fp8:
+    case Intrinsic::amdgcn_cvt_pk_f16_bf8:
     case Intrinsic::amdgcn_fmed3:
     case Intrinsic::amdgcn_cubeid:
     case Intrinsic::amdgcn_cubema:
@@ -4594,6 +4610,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_cvt_sr_bf8_f32:
     case Intrinsic::amdgcn_cvt_sr_bf16_f32:
     case Intrinsic::amdgcn_cvt_sr_f16_f32:
+    case Intrinsic::amdgcn_cvt_f16_fp8:
+    case Intrinsic::amdgcn_cvt_f16_bf8:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
     case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
@@ -5103,10 +5121,18 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
     case Intrinsic::amdgcn_global_load_tr_b64:
     case Intrinsic::amdgcn_global_load_tr_b128:
+    case Intrinsic::amdgcn_global_load_tr4_b64:
+    case Intrinsic::amdgcn_global_load_tr6_b96:
+    case Intrinsic::amdgcn_ds_load_tr8_b64:
+    case Intrinsic::amdgcn_ds_load_tr16_b128:
+    case Intrinsic::amdgcn_ds_load_tr4_b64:
+    case Intrinsic::amdgcn_ds_load_tr6_b96:
     case Intrinsic::amdgcn_ds_read_tr4_b64:
     case Intrinsic::amdgcn_ds_read_tr6_b96:
     case Intrinsic::amdgcn_ds_read_tr8_b64:
     case Intrinsic::amdgcn_ds_read_tr16_b64:
+    case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+    case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_ds_ordered_add:
     case Intrinsic::amdgcn_ds_ordered_swap: {
@@ -5342,6 +5368,22 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     }
     case Intrinsic::amdgcn_pops_exiting_wave_id:
       return getDefaultMappingSOP(MI);
+    case Intrinsic::amdgcn_tensor_load_to_lds_d2:
+    case Intrinsic::amdgcn_tensor_store_from_lds_d2:
+    case Intrinsic::amdgcn_tensor_load_to_lds:
+    case Intrinsic::amdgcn_tensor_store_from_lds: {
+      // Lie and claim everything is legal, even all operands need to be
+      // SGPRs. applyMapping will have to deal with it with readfirstlane.
+      for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
+        if (MI.getOperand(I).isReg()) {
+          Register Reg = MI.getOperand(I).getReg();
+          auto OpBank = getRegBankID(Reg, MRI);
+          unsigned Size = getSizeInBits(Reg, MRI, *TRI);
+          OpdsMapping[I] = AMDGPU::getValueMapping(OpBank, Size);
+        }
+      }
+      break;
+    }
     case Intrinsic::amdgcn_s_prefetch_data: {
       OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
new file mode 100644
index 000000000000..a8e1967116d1
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -0,0 +1,304 @@
+//===-- AMDGPURewriteAGPRCopyMFMA.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file \brief Try to replace MFMA instructions using VGPRs with MFMA
+/// instructions using AGPRs. We expect MFMAs to be selected using VGPRs, and
+/// only use AGPRs if it helps avoid spilling. In this case, the MFMA will have
+/// copies between AGPRs and VGPRs and the AGPR variant of an MFMA pseudo. This
+/// pass will attempt to delete the cross register bank copy and replace the
+/// MFMA opcode.
+///
+/// TODO:
+///  - Handle non-tied dst+src2 cases. We need to try to find a copy from an
+///    AGPR from src2, or reassign src2 to an available AGPR (which should work
+///    in the common case of a load).
+///
+///  - Handle multiple MFMA uses of the same register. e.g. chained MFMAs that
+///    can be rewritten as a set
+///
+///  - Update LiveIntervals incrementally instead of recomputing from scratch
+///
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIMachineFunctionInfo.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-rewrite-agpr-copy-mfma"
+
+namespace {
+
+class AMDGPURewriteAGPRCopyMFMAImpl {
+  const GCNSubtarget &ST;
+  const SIInstrInfo &TII;
+  const SIRegisterInfo &TRI;
+  MachineRegisterInfo &MRI;
+  VirtRegMap &VRM;
+  LiveRegMatrix &LRM;
+  LiveIntervals &LIS;
+
+public:
+  AMDGPURewriteAGPRCopyMFMAImpl(MachineFunction &MF, VirtRegMap &VRM,
+                                LiveRegMatrix &LRM, LiveIntervals &LIS)
+      : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
+        TRI(*ST.getRegisterInfo()), MRI(MF.getRegInfo()), VRM(VRM), LRM(LRM),
+        LIS(LIS) {}
+
+  /// Compute the register class constraints based on the uses of \p Reg,
+  /// excluding uses from \p ExceptMI. This should be nearly identical to
+  /// MachineRegisterInfo::recomputeRegClass.
+  const TargetRegisterClass *
+  recomputeRegClassExcept(Register Reg, const TargetRegisterClass *OldRC,
+                          const TargetRegisterClass *NewRC,
+                          const MachineInstr *ExceptMI) const;
+
+  bool run(MachineFunction &MF) const;
+};
+
+const TargetRegisterClass *
+AMDGPURewriteAGPRCopyMFMAImpl::recomputeRegClassExcept(
+    Register Reg, const TargetRegisterClass *OldRC,
+    const TargetRegisterClass *NewRC, const MachineInstr *ExceptMI) const {
+
+  // Accumulate constraints from all uses.
+  for (MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {
+    // Apply the effect of the given operand to NewRC.
+    MachineInstr *MI = MO.getParent();
+    if (MI == ExceptMI)
+      continue;
+
+    unsigned OpNo = &MO - &MI->getOperand(0);
+    NewRC = MI->getRegClassConstraintEffect(OpNo, NewRC, &TII, &TRI);
+    if (!NewRC || NewRC == OldRC)
+      return nullptr;
+  }
+
+  return NewRC;
+}
+
+bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
+  // This only applies on subtargets that have a configurable AGPR vs. VGPR
+  // allocation.
+  if (!ST.hasGFX90AInsts())
+    return false;
+
+  // Early exit if no AGPRs were assigned.
+  if (!LRM.isPhysRegUsed(AMDGPU::AGPR0))
+    return false;
+
+  bool MadeChange = false;
+
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
+    Register VReg = Register::index2VirtReg(I);
+    Register PhysReg = VRM.getPhys(VReg);
+    if (!PhysReg)
+      continue;
+
+    // Find AV_* registers assigned to AGPRs.
+    const TargetRegisterClass *VirtRegRC = MRI.getRegClass(VReg);
+    if (!TRI.isVectorSuperClass(VirtRegRC))
+      continue;
+
+    const TargetRegisterClass *AssignedRC = TRI.getPhysRegBaseClass(PhysReg);
+    if (!TRI.isAGPRClass(AssignedRC))
+      continue;
+
+    LiveInterval &LI = LIS.getInterval(VReg);
+
+    // TODO: Test multiple uses
+    for (VNInfo *VNI : LI.vnis()) {
+      MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def);
+
+      // TODO: Handle SplitKit produced copy bundles for partially defined
+      // registers.
+      if (!DefMI || !DefMI->isFullCopy())
+        continue;
+
+      Register CopySrcReg = DefMI->getOperand(1).getReg();
+      if (!CopySrcReg.isVirtual())
+        continue;
+
+      LiveInterval &CopySrcLI = LIS.getInterval(CopySrcReg);
+      LiveQueryResult LRQ = CopySrcLI.Query(VNI->def.getRegSlot());
+      MachineInstr *CopySrcMI = LIS.getInstructionFromIndex(LRQ.valueIn()->def);
+      if (!CopySrcMI)
+        continue;
+
+      int AGPROp = AMDGPU::getMFMASrcCVDstAGPROp(CopySrcMI->getOpcode());
+      if (AGPROp == -1)
+        continue;
+
+      MachineOperand *Src2 =
+          TII.getNamedOperand(*CopySrcMI, AMDGPU::OpName::src2);
+
+      // FIXME: getMinimalPhysRegClass returns a nonsense AV_* subclass instead
+      // of an AGPR or VGPR subclass, so we can't simply use the result on the
+      // assignment.
+
+      LLVM_DEBUG({
+        Register Src2PhysReg = VRM.getPhys(Src2->getReg());
+        dbgs() << "Attempting to replace VGPR MFMA with AGPR version:"
+               << " Dst=[" << printReg(VReg) << " => "
+               << printReg(PhysReg, &TRI) << "], Src2=["
+               << printReg(Src2->getReg(), &TRI) << " => "
+               << printReg(Src2PhysReg, &TRI) << "]: " << *CopySrcMI;
+      });
+
+      // If the inputs are tied and the same register, we can shortcut and
+      // directly replace the register.
+      if (Src2->getReg() != CopySrcReg) {
+        LLVM_DEBUG(
+            dbgs()
+            << "Replacing untied VGPR MFMAs with AGPR form not yet handled\n");
+        // TODO: Only handles the tied case for now. If the input operand is a
+        // different register, we need to also reassign it (either by looking
+        // for a compatible copy-from-AGPR, or by seeing if an available AGPR is
+        // compatible with all other uses.
+
+        // If we can't reassign it, we'd need to introduce a different copy
+        // which is likely worse than the copy we'd be saving.
+        continue;
+      }
+
+      const TargetRegisterClass *Src2VirtRegRC =
+          MRI.getRegClass(Src2->getReg());
+
+      // We've found av = COPY (MFMA), and need to verify that we can trivially
+      // rewrite src2 to use the new AGPR. If we can't trivially replace it,
+      // we're going to induce as many copies as we would have emitted in the
+      // first place, as well as need to assign another register, and need to
+      // figure out where to put them. The live range splitting is smarter than
+      // anything we're doing here, so trust it did something reasonable.
+      const TargetRegisterClass *Src2ExceptRC = recomputeRegClassExcept(
+          Src2->getReg(), Src2VirtRegRC, VirtRegRC, CopySrcMI);
+      if (!Src2ExceptRC)
+        continue;
+
+      const TargetRegisterClass *NewSrc2ConstraintRC =
+          TII.getRegClass(TII.get(AGPROp), Src2->getOperandNo(), &TRI, MF);
+
+      // Try to constrain src2 to the replacement instruction candidate's
+      // register class.
+      const TargetRegisterClass *NewSrc2RC =
+          TRI.getCommonSubClass(Src2ExceptRC, NewSrc2ConstraintRC);
+      if (!NewSrc2RC) {
+        // TODO: This is ignoring ther rewritable uses. e.g. a rewritable MFMA
+        // using a rewritable MFMA can be rewritten as a pair.
+        LLVM_DEBUG(dbgs() << "Other uses of " << printReg(Src2->getReg(), &TRI)
+                          << " are incompatible with replacement class\n");
+        continue;
+      }
+
+      MRI.setRegClass(VReg, AssignedRC);
+      MRI.setRegClass(Src2->getReg(), NewSrc2RC);
+
+      CopySrcMI->setDesc(TII.get(AGPROp));
+
+      // TODO: Is replacing too aggressive, fixup these instructions only?
+      MRI.replaceRegWith(CopySrcReg, VReg);
+
+      LLVM_DEBUG(dbgs() << "Replaced VGPR MFMA with AGPR: " << *CopySrcMI);
+
+      // We left behind an identity copy, so delete it.
+      LIS.RemoveMachineInstrFromMaps(*DefMI);
+      DefMI->eraseFromParent();
+
+      LRM.unassign(CopySrcLI);
+
+      // We don't need the liveness information anymore, so don't bother
+      // updating the intervals. Just delete the stale information.
+      // TODO: Is it worth preserving these?
+      LIS.removeInterval(CopySrcReg);
+      LIS.removeInterval(VReg);
+      LIS.createAndComputeVirtRegInterval(VReg);
+
+      MadeChange = true;
+    }
+  }
+
+  return MadeChange;
+}
+
+class AMDGPURewriteAGPRCopyMFMALegacy : public MachineFunctionPass {
+public:
+  static char ID;
+
+  AMDGPURewriteAGPRCopyMFMALegacy() : MachineFunctionPass(ID) {
+    initializeAMDGPURewriteAGPRCopyMFMALegacyPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return "AMDGPU Rewrite AGPR-Copy-MFMA";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<LiveIntervalsWrapperPass>();
+    AU.addRequired<VirtRegMapWrapperLegacy>();
+    AU.addRequired<LiveRegMatrixWrapperLegacy>();
+
+    AU.addPreserved<LiveIntervalsWrapperPass>();
+    AU.addPreserved<VirtRegMapWrapperLegacy>();
+    AU.addPreserved<LiveRegMatrixWrapperLegacy>();
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPURewriteAGPRCopyMFMALegacy, DEBUG_TYPE,
+                      "AMDGPU Rewrite AGPR-Copy-MFMA", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
+INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
+INITIALIZE_PASS_END(AMDGPURewriteAGPRCopyMFMALegacy, DEBUG_TYPE,
+                    "AMDGPU Rewrite AGPR-Copy-MFMA", false, false)
+
+char AMDGPURewriteAGPRCopyMFMALegacy::ID = 0;
+
+char &llvm::AMDGPURewriteAGPRCopyMFMALegacyID =
+    AMDGPURewriteAGPRCopyMFMALegacy::ID;
+
+bool AMDGPURewriteAGPRCopyMFMALegacy::runOnMachineFunction(
+    MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  auto &VRM = getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
+  auto &LRM = getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
+  auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+
+  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS);
+  return Impl.run(MF);
+}
+
+PreservedAnalyses
+AMDGPURewriteAGPRCopyMFMAPass::run(MachineFunction &MF,
+                                   MachineFunctionAnalysisManager &MFAM) {
+  VirtRegMap &VRM = MFAM.getResult<VirtRegMapAnalysis>(MF);
+  LiveRegMatrix &LRM = MFAM.getResult<LiveRegMatrixAnalysis>(MF);
+  LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+
+  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS);
+  if (!Impl.run(MF))
+    return PreservedAnalyses::all();
+  auto PA = getMachineFunctionPassPreservedAnalyses();
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 3b62dcf3c92c..1f6002a3c6a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -344,6 +344,12 @@ def : SourceOfDivergence<intr>;
 
 def : SourceOfDivergence<int_amdgcn_global_load_tr_b64>;
 def : SourceOfDivergence<int_amdgcn_global_load_tr_b128>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr4_b64>;
+def : SourceOfDivergence<int_amdgcn_global_load_tr6_b96>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr8_b64>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr16_b128>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr4_b64>;
+def : SourceOfDivergence<int_amdgcn_ds_load_tr6_b96>;
 
 def : SourceOfDivergence<int_amdgcn_ds_read_tr4_b64>;
 def : SourceOfDivergence<int_amdgcn_ds_read_tr6_b96>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index 23826cc22250..7c24f428d78e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUSUBTARGET_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b3b995ac0190..0321e515f931 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -535,6 +535,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPULowerModuleLDSLegacyPass(*PR);
   initializeAMDGPULowerBufferFatPointersPass(*PR);
   initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
+  initializeAMDGPURewriteAGPRCopyMFMALegacyPass(*PR);
   initializeAMDGPURewriteOutArgumentsPass(*PR);
   initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
   initializeSIAnnotateControlFlowLegacyPass(*PR);
@@ -843,8 +844,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
           // When we are not using -fgpu-rdc, we can run accelerator code
           // selection relatively early, but still after linking to prevent
           // eager removal of potentially reachable symbols.
-          if (EnableHipStdPar)
+          if (EnableHipStdPar) {
+            PM.addPass(HipStdParMathFixupPass());
             PM.addPass(HipStdParAcceleratorCodeSelectionPass());
+          }
           PM.addPass(AMDGPUPrintfRuntimeBindingPass());
         }
 
@@ -930,8 +933,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         // selection after linking to prevent, otherwise we end up removing
         // potentially reachable symbols that were exported as external in other
         // modules.
-        if (EnableHipStdPar)
+        if (EnableHipStdPar) {
+          PM.addPass(HipStdParMathFixupPass());
           PM.addPass(HipStdParAcceleratorCodeSelectionPass());
+        }
         // We want to support the -lto-partitions=N option as "best effort".
         // For that, we need to lower LDS earlier in the pipeline before the
         // module is partitioned for codegen.
@@ -1627,6 +1632,8 @@ void GCNPassConfig::addOptimizedRegAlloc() {
 bool GCNPassConfig::addPreRewrite() {
   if (EnableRegReassign)
     addPass(&GCNNSAReassignID);
+
+  addPass(&AMDGPURewriteAGPRCopyMFMALegacyID);
   return true;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index f7cf051a9ff9..f69358092951 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -313,24 +313,6 @@ bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
   return !F || !ST->isSingleLaneExecution(*F);
 }
 
-unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
-  // For certain 8 bit ops, we can pack a v4i8 into a single part
-  // (e.g. v4i8 shufflevectors -> v_perm v4i8, v4i8). Thus, we
-  // do not limit the numberOfParts for 8 bit vectors to the
-  // legalization costs of such. It is left up to other target
-  // queries (e.g. get*InstrCost) to decide the proper handling
-  // of 8 bit vectors.
-  if (FixedVectorType *VTy = dyn_cast<FixedVectorType>(Tp)) {
-    if (ST->shouldCoerceIllegalTypes() &&
-        DL.getTypeSizeInBits(VTy->getElementType()) == 8) {
-      unsigned ElCount = VTy->getElementCount().getFixedValue();
-      return std::max(UINT64_C(1), PowerOf2Ceil(ElCount / 4));
-    }
-  }
-
-  return BaseT::getNumberOfParts(Tp);
-}
-
 unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
   // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
   // registers. See getRegisterClassForType for the implementation.
@@ -362,11 +344,12 @@ unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
 unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
   if (Opcode == Instruction::Load || Opcode == Instruction::Store)
     return 32 * 4 / ElemWidth;
-
-  return (ST->shouldCoerceIllegalTypes() && ElemWidth == 8) ? 4
-         : (ElemWidth == 16)                                ? 2
-         : (ElemWidth == 32 && ST->hasPackedFP32Ops())      ? 2
-                                                            : 1;
+  // For a given width return the max 0number of elements that can be combined
+  // into a wider bit value:
+  return (ElemWidth == 8 && ST->has16BitInsts())       ? 4
+         : (ElemWidth == 16 && ST->has16BitInsts())    ? 2
+         : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
+                                                       : 1;
 }
 
 unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
@@ -1217,8 +1200,7 @@ InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
 
   unsigned ScalarSize = DL.getTypeSizeInBits(SrcTy->getElementType());
   if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
-      (ScalarSize == 16 ||
-       (ScalarSize == 8 && ST->shouldCoerceIllegalTypes()))) {
+      (ScalarSize == 16 || ScalarSize == 8)) {
     // Larger vector widths may require additional instructions, but are
     // typically cheaper than scalarized versions.
     unsigned NumVectorElts = cast<FixedVectorType>(SrcTy)->getNumElements();
@@ -1527,3 +1509,30 @@ GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
   return AMDGPU::isShader(F->getCallingConv()) ? KnownIEEEMode::Off
                                                : KnownIEEEMode::On;
 }
+
+InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+                                            Align Alignment,
+                                            unsigned AddressSpace,
+                                            TTI::TargetCostKind CostKind,
+                                            TTI::OperandValueInfo OpInfo,
+                                            const Instruction *I) const {
+  if (VectorType *VecTy = dyn_cast<VectorType>(Src)) {
+    if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
+        VecTy->getElementType()->isIntegerTy(8)) {
+      return divideCeil(DL.getTypeSizeInBits(VecTy) - 1,
+                        getLoadStoreVecRegBitWidth(AddressSpace));
+    }
+  }
+  return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
+                                OpInfo, I);
+}
+
+unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
+  if (VectorType *VecTy = dyn_cast<VectorType>(Tp)) {
+    if (VecTy->getElementType()->isIntegerTy(8)) {
+      unsigned ElementCount = VecTy->getElementCount().getFixedValue();
+      return divideCeil(ElementCount - 1, 4);
+    }
+  }
+  return BaseT::getNumberOfParts(Tp);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f9fd648c13b5..20da8344c9d3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -118,7 +118,6 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
     return TTI::PSK_FastHardware;
   }
 
-  unsigned getNumberOfParts(Type *Tp) const override;
   unsigned getNumberOfRegisters(unsigned RCID) const override;
   TypeSize
   getRegisterBitWidth(TargetTransformInfo::RegisterKind Vector) const override;
@@ -289,6 +288,20 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   /// "amdgpu-ieee"="true" and KnownIEEEMode::Off if we can assume
   /// "amdgpu-ieee"="false".
   KnownIEEEMode fpenvIEEEMode(const Instruction &I) const;
+
+  /// Account for loads of i8 vector types to have reduced cost. For
+  /// example the cost of load 4 i8s values is one is the cost of loading
+  /// a single i32 value.
+  InstructionCost getMemoryOpCost(
+      unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+      TTI::TargetCostKind CostKind,
+      TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+      const Instruction *I = nullptr) const override;
+
+  /// When counting parts on AMD GPUs, account for i8s being grouped
+  /// together under a single i32 value. Otherwise fall back to base
+  /// implementation.
+  unsigned getNumberOfParts(Type *Tp) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 30dcd6d81f16..628f1a69865e 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "AMDKernelCodeT.h"
 #include "MCTargetDesc/AMDGPUInstPrinter.h"
+#include "MCTargetDesc/AMDGPUMCAsmInfo.h"
 #include "MCTargetDesc/AMDGPUMCExpr.h"
 #include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
@@ -1151,7 +1152,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     // clang-format on
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Register:
       OS << "<register " << AMDGPUInstPrinter::getRegisterName(getReg())
@@ -1168,7 +1169,9 @@ class AMDGPUOperand : public MCParsedAsmOperand {
       OS << '\'' << getToken() << '\'';
       break;
     case Expression:
-      OS << "<expr " << *Expr << '>';
+      OS << "<expr ";
+      MAI.printExpr(OS, *Expr);
+      OS << '>';
       break;
     }
   }
@@ -1781,6 +1784,7 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool validateMIMGAddrSize(const MCInst &Inst, const SMLoc &IDLoc);
   bool validateMIMGD16(const MCInst &Inst);
   bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands);
+  bool validateTensorR128(const MCInst &Inst);
   bool validateMIMGMSAA(const MCInst &Inst);
   bool validateOpSel(const MCInst &Inst);
   bool validateTrue16OpSel(const MCInst &Inst);
@@ -4277,6 +4281,18 @@ bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
   return true;
 }
 
+bool AMDGPUAsmParser::validateTensorR128(const MCInst &Inst) {
+  const unsigned Opc = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opc);
+
+  if ((Desc.TSFlags & SIInstrFlags::TENSOR_CNT) == 0)
+    return true;
+
+  int R128Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::r128);
+
+  return R128Idx < 0 || !Inst.getOperand(R128Idx).getImm();
+}
+
 static bool IsRevOpcode(const unsigned Opcode)
 {
   switch (Opcode) {
@@ -5110,14 +5126,11 @@ bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst,
       return PrintError("scope and th combination is not valid");
   }
 
-  bool IsStore = TID.mayStore();
-  bool IsAtomic =
-      TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet);
-
-  if (IsAtomic) {
+  unsigned THType = AMDGPU::getTemporalHintType(TID);
+  if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
     if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC))
       return PrintError("invalid th value for atomic instructions");
-  } else if (IsStore) {
+  } else if (THType == AMDGPU::CPol::TH_TYPE_STORE) {
     if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE))
       return PrintError("invalid th value for store instructions");
   } else {
@@ -5202,6 +5215,11 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
     Error(IDLoc, "missing dim operand");
     return false;
   }
+  if (!validateTensorR128(Inst)) {
+    Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
+          "instruction must set modifier r128=0");
+    return false;
+  }
   if (!validateMIMGMSAA(Inst)) {
     Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
           "invalid dim; must be MSAA type");
@@ -6451,26 +6469,26 @@ StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
   setForcedDPP(false);
   setForcedSDWA(false);
 
-  if (Name.ends_with("_e64_dpp")) {
+  if (Name.consume_back("_e64_dpp")) {
     setForcedDPP(true);
     setForcedEncodingSize(64);
-    return Name.substr(0, Name.size() - 8);
+    return Name;
   }
-  if (Name.ends_with("_e64")) {
+  if (Name.consume_back("_e64")) {
     setForcedEncodingSize(64);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_e32")) {
+  if (Name.consume_back("_e32")) {
     setForcedEncodingSize(32);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_dpp")) {
+  if (Name.consume_back("_dpp")) {
     setForcedDPP(true);
-    return Name.substr(0, Name.size() - 4);
+    return Name;
   }
-  if (Name.ends_with("_sdwa")) {
+  if (Name.consume_back("_sdwa")) {
     setForcedSDWA(true);
-    return Name.substr(0, Name.size() - 5);
+    return Name;
   }
   return Name;
 }
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index efcc81716a0f..0caabe41e9b7 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -845,9 +845,12 @@ multiclass MUBUF_Pseudo_Atomics <string opName,
 // MUBUF Instructions
 //===----------------------------------------------------------------------===//
 
+let OtherPredicates = [HasFormattedMUBUFInsts] in
 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
   "buffer_load_format_x", f32
 >;
+
+let SubtargetPredicate = HasFormattedMUBUFInsts in {
 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
   "buffer_load_format_xy", v2f32
 >;
@@ -927,6 +930,7 @@ let TiedSourceNotRead = 1 in {
     "buffer_store_format_d16_xyzw", v4f16
   >;
 } // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1.
+} // End SubtargetPredicate = HasFormattedMUBUFInsts.
 
 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
   "buffer_load_ubyte", i32
@@ -1172,6 +1176,7 @@ defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
   "buffer_load_short_d16_hi", i32, 1
 >;
 
+let OtherPredicates = [HasFormattedMUBUFInsts] in
 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
   "buffer_load_format_d16_hi_x", i32
 >;
@@ -1185,6 +1190,7 @@ defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
   "buffer_store_short_d16_hi", i32
 >;
 
+let OtherPredicates = [HasFormattedMUBUFInsts] in
 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
   "buffer_store_format_d16_hi_x", i32
 >;
@@ -1232,7 +1238,7 @@ defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics <
 //===----------------------------------------------------------------------===//
 // MTBUF Instructions
 //===----------------------------------------------------------------------===//
-
+let OtherPredicates = [HasMTBUFInsts] in {
 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
@@ -1267,6 +1273,7 @@ let TiedSourceNotRead = 1 in {
   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
 } // End HasPackedD16VMem.
+} // End HasMTBUFInsts.
 
 let SubtargetPredicate = isGFX7Plus in {
 
@@ -1365,6 +1372,7 @@ multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
   defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
 }
 
+let OtherPredicates = [HasFormattedMUBUFInsts] in {
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
@@ -1378,17 +1386,18 @@ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORM
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">;
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">;
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">;
+} // End OtherPredicates = [HasFormattedMUBUFInsts].
 
-let OtherPredicates = [HasUnpackedD16VMem] in {
+let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
   defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
-} // End HasUnpackedD16VMem.
+} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
 
-let OtherPredicates = [HasPackedD16VMem] in {
+let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">;
@@ -1398,7 +1407,7 @@ let OtherPredicates = [HasPackedD16VMem] in {
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
-} // End HasPackedD16VMem.
+} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
 
 foreach vt = Reg32Types.types in {
 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORD">;
@@ -1474,6 +1483,7 @@ multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
   defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
 }
 
+let OtherPredicates = [HasFormattedMUBUFInsts] in {
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
@@ -1483,17 +1493,18 @@ defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMA
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
+} // End OtherPredicates = [HasFormattedMUBUFInsts].
 
-let OtherPredicates = [HasUnpackedD16VMem] in {
+let OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts] in {
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
   defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
-} // End HasUnpackedD16VMem.
+} // End OtherPredicates = [HasUnpackedD16VMem, HasFormattedMUBUFInsts].
 
-let OtherPredicates = [HasPackedD16VMem] in {
+let OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts] in {
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">;
@@ -1503,7 +1514,7 @@ let OtherPredicates = [HasPackedD16VMem] in {
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>;
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
-} // End HasPackedD16VMem.
+} // End OtherPredicates = [HasPackedD16VMem, HasFormattedMUBUFInsts].
 
 foreach vt = Reg32Types.types in {
 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORD">;
@@ -2188,12 +2199,13 @@ multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
 
 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
                                   string opcode, ValueType memoryVt = vt> {
-  let OtherPredicates = [HasUnrestrictedSOffset] in {
+  let SubtargetPredicate = HasUnrestrictedSOffset in {
     defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>;
   }
   defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
 }
 
+let OtherPredicates = [HasMTBUFInsts] in {
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
@@ -2202,22 +2214,23 @@ defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
+} // End HasMTBUFInsts.
 
-let SubtargetPredicate = HasUnpackedD16VMem in {
+let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
   defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
   defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
   defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
   defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
   defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
-} // End HasUnpackedD16VMem.
+} // End HasUnpackedD16VMem,HasMTBUFInsts.
 
-let SubtargetPredicate = HasPackedD16VMem in {
+let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X">;
   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
-} // End HasPackedD16VMem.
+} // End HasPackedD16VMem,HasMTBUFInsts.
 
 multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
                                         string opcode, ValueType memoryVt = vt> {
@@ -2260,12 +2273,13 @@ multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt,
 
 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
                                   string opcode, ValueType memoryVt = vt> {
-  let OtherPredicates = [HasUnrestrictedSOffset] in {
+  let SubtargetPredicate = HasUnrestrictedSOffset in {
     defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>;
   }
   defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>;
 }
 
+let OtherPredicates = [HasMTBUFInsts] in {
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
@@ -2274,22 +2288,23 @@ defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
+} // End HasMTBUFInsts.
 
-let SubtargetPredicate = HasUnpackedD16VMem in {
+let OtherPredicates = [HasUnpackedD16VMem,HasMTBUFInsts] in {
   defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
   defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
   defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
   defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
-} // End HasUnpackedD16VMem.
+} // End HasUnpackedD16VMem,HasMTBUFInsts.
 
-let SubtargetPredicate = HasPackedD16VMem in {
+let OtherPredicates = [HasPackedD16VMem,HasMTBUFInsts] in {
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
-} // End HasPackedD16VMem.
+} // End HasPackedD16VMem,HasMTBUFInsts.
 
 //===----------------------------------------------------------------------===//
 // Target-specific instruction encodings.
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 909187dfee9e..26b6e4f18ad3 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -102,6 +102,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPURemoveIncompatibleFunctions.cpp
   AMDGPUReserveWWMRegs.cpp
   AMDGPUResourceUsageAnalysis.cpp
+  AMDGPURewriteAGPRCopyMFMA.cpp
   AMDGPURewriteOutArguments.cpp
   AMDGPURewriteUndefForPHI.cpp
   AMDGPUSelectionDAGInfo.cpp
diff --git a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
index 383e3371993d..d9d7a650dfc2 100644
--- a/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSDIRInstructions.td
@@ -141,7 +141,7 @@ def : GCNPat <
 
 } // End SubtargetPredicate = isGFX11Only
 
-let SubtargetPredicate = isGFX12Plus in {
+let SubtargetPredicate = isGFX12PlusNot12_50 in {
 
 def DS_DIRECT_LOAD : DSDIR_Pseudo<"ds_direct_load", VDSDIR_getIns<1>.ret, 1>;
 def DS_PARAM_LOAD : DSDIR_Pseudo<"ds_param_load", VDSDIR_getIns<0>.ret, 0>;
@@ -156,7 +156,7 @@ def : GCNPat <
   (DS_PARAM_LOAD timm:$attr, timm:$attrchan, 0, 1)
 >;
 
-} // End SubtargetPredicate = isGFX12Only
+} // End SubtargetPredicate = isGFX12PlusNot12_50.
 
 //===----------------------------------------------------------------------===//
 // GFX11
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 604eb7f2c387..e219fe05f881 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -354,6 +354,20 @@ class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
   let gdsValue = 1;
 }
 
+class DS_1A_Off16_NORET <string opName>
+: DS_Pseudo<opName,
+  (outs),
+  (ins VGPR_32:$addr, Offset:$offset, gds:$gds),
+  " $addr$offset$gds"> {
+
+  let has_vdst = 0;
+  let has_offset = 1;
+  let has_data0 = 0;
+  let has_data1 = 0;
+  let has_m0_read = 0;
+  let IsAtomicNoRet = 1;
+}
+
 class DS_0A_RET <string opName> : DS_Pseudo<opName,
   (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
   (ins Offset:$offset, gds:$gds),
@@ -783,6 +797,37 @@ multiclass DSAtomicRetNoRetPatIntrinsic_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
 defm : DSAtomicRetNoRetPatIntrinsic_mc<DS_COND_SUB_RTN_U32, DS_COND_SUB_U32, i32, "int_amdgcn_atomic_cond_sub_u32">;
 } // let SubtargetPredicate = isGFX12Plus
 
+let SubtargetPredicate = isGFX1250Plus in {
+
+let WaveSizePredicate = isWave32, mayStore = 0 in {
+let OtherPredicates = [HasTransposeLoadF4F6Insts] in {
+defm DS_LOAD_TR4_B64   : DS_1A_RET_NoM0<"ds_load_tr4_b64",   VReg_64>;
+defm DS_LOAD_TR6_B96   : DS_1A_RET_NoM0<"ds_load_tr6_b96",   VReg_96>;
+} // End OtherPredicates = [HasTransposeLoadF4F6Insts]
+defm DS_LOAD_TR8_B64   : DS_1A_RET_NoM0<"ds_load_tr8_b64",   VReg_64>;
+defm DS_LOAD_TR16_B128 : DS_1A_RET_NoM0<"ds_load_tr16_b128", VReg_128>;
+} // End WaveSizePredicate = isWave32, mayStore = 0
+
+let OtherPredicates = [HasLdsBarrierArriveAtomic] in {
+let ASYNC_CNT = 1, LGKM_CNT = 0, Uses = [EXEC, ASYNCcnt], Defs = [ASYNCcnt] in {
+def DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_1A_Off16_NORET<"ds_atomic_async_barrier_arrive_b64">;
+}
+
+def : GCNPat <
+  (int_amdgcn_ds_atomic_async_barrier_arrive_b64 (DS1Addr1Offset i32:$ptr, i32:$offset)),
+  (DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 VGPR_32:$ptr, Offset:$offset, (i1 0))
+>;
+
+defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 : DS_1A1D_RET_mc_gfx9<"ds_atomic_barrier_arrive_rtn_b64", VReg_64>;
+
+def : GCNPat<
+  (i64 (int_amdgcn_ds_atomic_barrier_arrive_rtn_b64 (DS1Addr1Offset i32:$ptr, i32:$offset), i64:$data)),
+  (DS_ATOMIC_BARRIER_ARRIVE_RTN_B64 $ptr, $data, Offset:$offset, (i1 0))
+>;
+} // End OtherPredicates = [HasLdsBarrierArriveAtomic]
+
+} // End SubtargetPredicate = isGFX1250Plus
+
 let WaveSizePredicate = isWave64, SubtargetPredicate = HasGFX950Insts, mayStore = 0 in {
   defm DS_READ_B64_TR_B4  : DS_1A_RET_NoM0<"ds_read_b64_tr_b4", VReg_64>;
   defm DS_READ_B64_TR_B8  : DS_1A_RET_NoM0<"ds_read_b64_tr_b8", VReg_64>;
@@ -1263,6 +1308,17 @@ class DSLoadTrPat <DS_Pseudo inst, ValueType vt, SDPatternOperator node> : GCNPa
   (inst $ptr, Offset:$offset, (i1 0))
 >;
 
+let WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus in {
+let OtherPredicates = [HasTransposeLoadF4F6Insts] in {
+  def : DSLoadTrPat <DS_LOAD_TR4_B64, v2i32, int_amdgcn_ds_load_tr4_b64>;
+  def : DSLoadTrPat <DS_LOAD_TR6_B96, v3i32, int_amdgcn_ds_load_tr6_b96>;
+} // End OtherPredicates = [HasTransposeLoadF4F6Insts]
+
+  def : DSLoadTrPat <DS_LOAD_TR8_B64, v2i32, int_amdgcn_ds_load_tr8_b64>;
+  foreach vt = [v8i16, v8f16, v8bf16] in
+    def : DSLoadTrPat <DS_LOAD_TR16_B128, vt, int_amdgcn_ds_load_tr16_b128>;
+} // End WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus
+
 let SubtargetPredicate = HasGFX950Insts in {
   def : DSLoadTrPat <DS_READ_B64_TR_B4,  v2i32, int_amdgcn_ds_read_tr4_b64>;
   def : DSLoadTrPat <DS_READ_B64_TR_B8,  v2i32, int_amdgcn_ds_read_tr8_b64>;
@@ -1302,14 +1358,14 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef,
 // GFX12.
 //===----------------------------------------------------------------------===//
 
-multiclass DS_Real_gfx12<bits<8> op, string name = !tolower(NAME), bit needAlias = true> {
+multiclass DS_Real_gfx12<bits<8> op, string name = !tolower(NAME)> {
   defvar ps = !cast<DS_Pseudo>(NAME);
   let AssemblerPredicate = isGFX12Plus in {
     let DecoderNamespace = "GFX12" in
       def _gfx12 :
         Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,
                                                name, /*hasGDS=*/false>;
-    if !and(needAlias, !ne(ps.Mnemonic, name)) then
+    if !ne(ps.Mnemonic, name) then
       def : AMDGPUMnemonicAlias<ps.Mnemonic, name>;
   } // End AssemblerPredicate
 }
@@ -1332,11 +1388,21 @@ defm DS_PK_ADD_BF16       : DS_Real_gfx12<0x09b>;
 defm DS_PK_ADD_RTN_BF16   : DS_Real_gfx12<0x0ab>;
 defm DS_BPERMUTE_FI_B32   : DS_Real_gfx12<0x0cd>;
 
+defm DS_LOAD_TR4_B64      : DS_Real_gfx12<0x0fa>;
+defm DS_LOAD_TR6_B96      : DS_Real_gfx12<0x0fb>;
+defm DS_LOAD_TR16_B128    : DS_Real_gfx12<0x0fc>;
+defm DS_LOAD_TR8_B64      : DS_Real_gfx12<0x0fd>;
+
 defm DS_BVH_STACK_RTN_B32             : DS_Real_gfx12<0x0e0,
-  "ds_bvh_stack_push4_pop1_rtn_b32", true>;
+  "ds_bvh_stack_push4_pop1_rtn_b32">;
 defm DS_BVH_STACK_PUSH8_POP1_RTN_B32  : DS_Real_gfx12<0x0e1>;
 defm DS_BVH_STACK_PUSH8_POP2_RTN_B64  : DS_Real_gfx12<0x0e2>;
 
+let AssemblerPredicate = HasLdsBarrierArriveAtomic in {
+defm DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 : DS_Real_gfx12<0x056>;
+defm DS_ATOMIC_BARRIER_ARRIVE_RTN_B64   : DS_Real_gfx12<0x075>;
+}
+
 // New aliases added in GFX12 without renaming the instructions.
 let AssemblerPredicate = isGFX12Plus in {
   def : AMDGPUMnemonicAlias<"ds_subrev_u32", "ds_rsub_u32">;
@@ -1345,6 +1411,10 @@ let AssemblerPredicate = isGFX12Plus in {
   def : AMDGPUMnemonicAlias<"ds_subrev_rtn_u64", "ds_rsub_rtn_u64">;
 }
 
+// Aliases that have existed since these instructions were introduced.
+def : MnemonicAlias<"ds_load_tr_b64", "ds_load_tr8_b64">, Requires<[isGFX1250Plus]>;
+def : MnemonicAlias<"ds_load_tr_b128", "ds_load_tr16_b128">, Requires<[isGFX1250Plus]>;
+
 //===----------------------------------------------------------------------===//
 // GFX11.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index 07a4292ef28b..59c72fcbff18 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -599,6 +599,11 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                         DecW, Address, CS))
         break;
 
+      if (isGFX1250() &&
+          tryDecodeInst(DecoderTableGFX125096, DecoderTableGFX1250_FAKE1696, MI,
+                        DecW, Address, CS))
+        break;
+
       if (isGFX12() &&
           tryDecodeInst(DecoderTableGFX1296, DecoderTableGFX12_FAKE1696, MI,
                         DecW, Address, CS))
@@ -661,6 +666,11 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
       if (isGFX10() && tryDecodeInst(DecoderTableGFX1064, MI, QW, Address, CS))
         break;
 
+      if (isGFX1250() &&
+          tryDecodeInst(DecoderTableGFX125064, DecoderTableGFX1250_FAKE1664, MI,
+                        QW, Address, CS))
+        break;
+
       if (isGFX12() &&
           tryDecodeInst(DecoderTableGFX1264, DecoderTableGFX12_FAKE1664, MI, QW,
                         Address, CS))
@@ -717,10 +727,9 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                         Address, CS))
         break;
 
-      // FIXME: Should use DecoderTableGFX1250_FAKE1632, but it is not generated
-      //        yet.
       if (isGFX1250() &&
-          tryDecodeInst(DecoderTableGFX125032, MI, DW, Address, CS))
+          tryDecodeInst(DecoderTableGFX125032, DecoderTableGFX1250_FAKE1632, MI,
+                        DW, Address, CS))
         break;
 
       if (isGFX12() &&
diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td
index 5e426b5acd50..807b69dc85eb 100644
--- a/llvm/lib/Target/AMDGPU/EXPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td
@@ -119,6 +119,7 @@ multiclass VEXPORT_Real_gfx12 {
   def _gfx12 : EXP_Real_Row<ps, SIEncodingFamily.GFX12, "export">,
     EXPe_Row {
     let AssemblerPredicate = isGFX12Only;
+    let OtherPredicates = [HasExportInsts];
     let DecoderNamespace = "GFX12";
     let row = ps.row;
     let done = ps.done;
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index 88b694862d37..3625db9a4791 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -208,21 +208,26 @@ class GlobalSaddrTable <bit is_saddr, string Name = ""> {
 // TODO: Is exec allowed for saddr? The disabled value 0x7f is the
 // same encoding value as exec_hi, so it isn't possible to use that if
 // saddr is 32-bit (which isn't handled here yet).
-class FLAT_Load_Pseudo <string opName, RegisterClass regClass,
-  bit HasTiedOutput = 0,
-  bit HasSaddr = 0, bit EnableSaddr = 0,
-  RegisterOperand vdata_op = getLdStRegisterOperand<regClass>.ret> : FLAT_Pseudo<
-  opName,
-  (outs vdata_op:$vdst),
-  !con(
+class FLAT_Load_Pseudo<
+    string opName, RegisterClass regClass, bit HasTiedOutput = 0,
+    bit HasSaddr = 0, bit EnableSaddr = 0>
+    : FLAT_Pseudo<opName, (outs), (ins), ""> {
+
+  defvar vdata_op = getLdStRegisterOperand<regClass>.ret;
+  let OutOperandList = (outs vdata_op:$vdst);
+  let InOperandList = !con(
     !if(EnableSaddr,
-      (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
-      (ins VReg_64:$vaddr)),
-      (ins flat_offset:$offset),
-      // FIXME: Operands with default values do not work with following non-optional operands.
-      !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
-                         (ins CPol_0:$cpol))),
-  " $vdst, $vaddr"#!if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")#"$offset$cpol"> {
+        (ins SReg_64_XEXEC_XNULL:$saddr, VGPR_32:$vaddr),
+        (ins VReg_64:$vaddr)),
+    (ins flat_offset:$offset),
+    // FIXME: Operands with default values do not work with following
+    // non-optional operands.
+    !if(HasTiedOutput, (ins CPol:$cpol, vdata_op:$vdst_in),
+                       (ins CPol_0:$cpol)));
+  let AsmOperands = " $vdst, $vaddr"
+    # !if(HasSaddr, !if(EnableSaddr, ", $saddr", ", off"), "")
+    # "$offset$cpol";
+
   let has_data = 0;
   let mayLoad = 1;
   let has_saddr = HasSaddr;
@@ -387,6 +392,23 @@ multiclass FLAT_Global_Store_AddTid_Pseudo<string opName, RegisterClass regClass
     GlobalSaddrTable<1, opName>;
 }
 
+class FLAT_Global_Tensor_Pseudo<string opName, bit EnableSaddr = 0> : FLAT_Pseudo<
+  opName,
+  (outs ),
+  !con(!if(EnableSaddr, (ins SReg_64:$saddr, flat_offset:$offset), (ins )), (ins CPol_0:$cpol)),
+  !if(EnableSaddr, " $saddr$offset", " ")#"$cpol"> {
+
+  let is_flat_global = 1;
+  let has_vdst = 0;
+  let has_data = 0;
+  let has_vaddr = 0;
+  let mayLoad = 0;
+  let mayStore = 1;
+  let has_saddr = 1;
+  let enabled_saddr = EnableSaddr;
+  let has_offset = EnableSaddr;
+}
+
 class FLAT_Global_Invalidate_Writeback<string opName, SDPatternOperator node = null_frag> :
   FLAT_Pseudo<opName, (outs), (ins CPol:$cpol), "$cpol", [(node)]> {
 
@@ -1036,15 +1058,23 @@ defm GLOBAL_LOAD_LDS_DWORDX3 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwo
 defm GLOBAL_LOAD_LDS_DWORDX4 : FLAT_Global_Load_LDS_Pseudo <"global_load_lds_dwordx4">;
 }
 
+let SubtargetPredicate = isGFX12PlusNot12_50 in
+  defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : FLAT_Global_Atomic_Pseudo <"global_atomic_ordered_add_b64", VReg_64, i64>;
+
 let SubtargetPredicate = isGFX12Plus in {
   defm GLOBAL_ATOMIC_COND_SUB_U32    : FLAT_Global_Atomic_Pseudo <"global_atomic_cond_sub_u32", VGPR_32, i32>;
-  defm GLOBAL_ATOMIC_ORDERED_ADD_B64 : FLAT_Global_Atomic_Pseudo <"global_atomic_ordered_add_b64", VReg_64, i64>;
 
   def GLOBAL_INV    : FLAT_Global_Invalidate_Writeback<"global_inv">;
   def GLOBAL_WB     : FLAT_Global_Invalidate_Writeback<"global_wb">;
   def GLOBAL_WBINV  : FLAT_Global_Invalidate_Writeback<"global_wbinv">;
 } // End SubtargetPredicate = isGFX12Plus
 
+let SubtargetPredicate = isGFX1250Plus in {
+
+def TENSOR_SAVE : FLAT_Global_Tensor_Pseudo<"tensor_save", 1>;
+def TENSOR_STOP : FLAT_Global_Tensor_Pseudo<"tensor_stop">;
+} // End SubtargetPredicate = isGFX1250Plus
+
 defm SCRATCH_LOAD_UBYTE    : FLAT_Scratch_Load_Pseudo <"scratch_load_ubyte", VGPR_32>;
 defm SCRATCH_LOAD_SBYTE    : FLAT_Scratch_Load_Pseudo <"scratch_load_sbyte", VGPR_32>;
 defm SCRATCH_LOAD_USHORT   : FLAT_Scratch_Load_Pseudo <"scratch_load_ushort", VGPR_32>;
@@ -1090,19 +1120,23 @@ let SubtargetPredicate = isGFX12Plus in {
   }
 
   let WaveSizePredicate = isWave32 in {
-    let Mnemonic = "global_load_tr_b128" in
-    defm GLOBAL_LOAD_TR_B128_w32  : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w32", VReg_128>;
-    let Mnemonic = "global_load_tr_b64" in
-    defm GLOBAL_LOAD_TR_B64_w32   : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w32", VReg_64>;
-  }
-  let WaveSizePredicate = isWave64 in {
-    let Mnemonic = "global_load_tr_b128" in
-    defm GLOBAL_LOAD_TR_B128_w64  : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w64", VReg_64>;
-    let Mnemonic = "global_load_tr_b64" in
-    defm GLOBAL_LOAD_TR_B64_w64   : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPR_32>;
+    defm GLOBAL_LOAD_TR_B128_w32  : FLAT_Global_Load_Pseudo <"global_load_tr_b128", VReg_128>;
+    defm GLOBAL_LOAD_TR_B64_w32   : FLAT_Global_Load_Pseudo <"global_load_tr_b64", VReg_64>;
   }
 } // End SubtargetPredicate = isGFX12Plus
 
+let WaveSizePredicate = isWave64, SubtargetPredicate = isGFX12PlusNot12_50 in {
+  let Mnemonic = "global_load_tr_b128" in
+  defm GLOBAL_LOAD_TR_B128_w64  : FLAT_Global_Load_Pseudo <"global_load_tr_b128_w64", VReg_64>;
+  let Mnemonic = "global_load_tr_b64" in
+  defm GLOBAL_LOAD_TR_B64_w64   : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPR_32>;
+}
+
+let WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus in {
+  defm GLOBAL_LOAD_TR6_B96 : FLAT_Global_Load_Pseudo <"global_load_tr6_b96", VReg_96>;
+  defm GLOBAL_LOAD_TR4_B64 : FLAT_Global_Load_Pseudo <"global_load_tr4_b64", VReg_64>;
+}
+
 let SubtargetPredicate = isGFX10Plus in {
   defm GLOBAL_ATOMIC_FCMPSWAP :
     FLAT_Global_Atomic_Pseudo<"global_atomic_fcmpswap", VGPR_32, f32, v2f32, VReg_64>;
@@ -1827,19 +1861,24 @@ let SubtargetPredicate = isGFX12Plus in {
     defm : GlobalFLATAtomicPatsNoRtnWithAddrSpace  <"GLOBAL_ATOMIC_COND_SUB_U32", "int_amdgcn_atomic_cond_sub_u32", "global_addrspace",  i32>;
 }
 
-let OtherPredicates = [isGFX12Plus] in {
+let OtherPredicates = [isGFX12PlusNot12_50] in
   defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_ORDERED_ADD_B64", "int_amdgcn_global_atomic_ordered_add_b64", i64, i64, /* isIntr */ 1>;
 
-  let WaveSizePredicate = isWave32 in {
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;
-    foreach vt = [v8i16, v8f16, v8bf16] in
-      defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, vt>;
-  }
-  let WaveSizePredicate = isWave64 in {
-    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;
-    foreach vt = [v4i16, v4f16, v4bf16] in
-      defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>;
-  }
+let WaveSizePredicate = isWave32, OtherPredicates = [isGFX12Plus] in {
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w32, int_amdgcn_global_load_tr_b64, v2i32>;
+  foreach vt = [v8i16, v8f16, v8bf16] in
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w32, int_amdgcn_global_load_tr_b128, vt>;
+}
+
+let WaveSizePredicate = isWave64, OtherPredicates = [isGFX12PlusNot12_50] in {
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B64_w64, int_amdgcn_global_load_tr_b64, i32>;
+  foreach vt = [v4i16, v4f16, v4bf16] in
+    defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR_B128_w64, int_amdgcn_global_load_tr_b128, vt>;
+}
+
+let WaveSizePredicate = isWave32,  OtherPredicates = [HasTransposeLoadF4F6Insts] in {
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR4_B64, int_amdgcn_global_load_tr4_b64, v2i32>;
+  defm : GlobalFLATLoadPats <GLOBAL_LOAD_TR6_B96, int_amdgcn_global_load_tr6_b96, v3i32>;
 }
 
 let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts, OtherPredicates = [HasFlatGlobalInsts] in {
@@ -2807,6 +2846,13 @@ multiclass VGLOBAL_Real_AllAddr_gfx12<bits<8> op,
   defm _SADDR : VFLAT_Real_gfx12<op, name>;
 }
 
+multiclass VGLOBAL_Real_AllAddr_gfx1200<bits<8> op> {
+  let AssemblerPredicate = isGFX12Not12_50 in {
+    defm "" : VFLAT_Real_gfx12<op>;
+    defm _SADDR : VFLAT_Real_gfx12<op>;
+  }
+}
+
 multiclass VGLOBAL_Real_AllAddr_gfx12_w64<bits<8> op,
                                        string name = get_FLAT_ps<NAME>.Mnemonic> :
   VFLAT_Aliases_gfx12<name> {
@@ -2949,8 +2995,8 @@ defm GLOBAL_ATOMIC_FMIN            : VGLOBAL_Real_Atomics_gfx12<0x051, "global_a
 defm GLOBAL_ATOMIC_FMAX            : VGLOBAL_Real_Atomics_gfx12<0x052, "global_atomic_max_num_f32", "global_atomic_max_f32">;
 defm GLOBAL_ATOMIC_ADD_F32         : VGLOBAL_Real_Atomics_gfx12<0x056>;
 
-defm GLOBAL_LOAD_TR_B128_w32       : VGLOBAL_Real_AllAddr_gfx12<0x057>;
-defm GLOBAL_LOAD_TR_B64_w32        : VGLOBAL_Real_AllAddr_gfx12<0x058>;
+defm GLOBAL_LOAD_TR_B128_w32       : VGLOBAL_Real_AllAddr_gfx1200<0x057>;
+defm GLOBAL_LOAD_TR_B64_w32        : VGLOBAL_Real_AllAddr_gfx1200<0x058>;
 
 defm GLOBAL_LOAD_TR_B128_w64       : VGLOBAL_Real_AllAddr_gfx12_w64<0x057>;
 defm GLOBAL_LOAD_TR_B64_w64        : VGLOBAL_Real_AllAddr_gfx12_w64<0x058>;
@@ -2990,6 +3036,63 @@ defm SCRATCH_STORE_SHORT_D16_HI    : VSCRATCH_Real_AllAddr_gfx12<0x25, "scratch_
 defm SCRATCH_LOAD_BLOCK            : VSCRATCH_Real_AllAddr_gfx12<0x53>;
 defm SCRATCH_STORE_BLOCK           : VSCRATCH_Real_AllAddr_gfx12<0x54>;
 
+//===----------------------------------------------------------------------===//
+// GFX1250
+//===----------------------------------------------------------------------===//
+
+multiclass VFLAT_Real_gfx1250<bits<8> op,
+                              string name = get_FLAT_ps<NAME>.Mnemonic> {
+  defvar ps = !cast<FLAT_Pseudo>(NAME);
+  def _gfx1250 : VFLAT_Real<op, ps, name>,
+                 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX1250> {
+    let AssemblerPredicate = isGFX125xOnly;
+    let DecoderNamespace = "GFX1250";
+
+    let Inst{25-24} = {ps.is_flat_global, ps.is_flat_scratch};
+  }
+}
+
+multiclass VFLAT_Aliases_gfx1250<string name> {
+  defvar ps = get_FLAT_ps<NAME>;
+  if !ne(ps.Mnemonic, name) then
+    def : MnemonicAlias<ps.Mnemonic, name>, Requires<[isGFX125xOnly]>;
+}
+
+multiclass VFLAT_Real_Base_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :
+  VFLAT_Aliases_gfx1250<name> {
+  defm "" : VFLAT_Real_gfx1250<op, name>;
+}
+
+multiclass VFLAT_Real_RTN_gfx1250<bits<8> op, string name> {
+  defm _RTN : VFLAT_Real_gfx1250<op, name>;
+}
+
+multiclass VFLAT_Real_SADDR_gfx1250<bits<8> op, string name> {
+  defm _SADDR : VFLAT_Real_gfx1250<op, name>;
+}
+
+multiclass VFLAT_Real_SADDR_RTN_gfx1250<bits<8> op, string name> {
+  defm _SADDR_RTN : VFLAT_Real_gfx1250<op, name>;
+}
+
+multiclass VFLAT_Real_AllAddr_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :
+  VFLAT_Real_Base_gfx1250<op, name>,
+  VFLAT_Real_SADDR_gfx1250<op, name>;
+
+multiclass VFLAT_Real_Atomics_gfx1250<bits<8> op, string name = get_FLAT_ps<NAME>.Mnemonic> :
+  VFLAT_Real_AllAddr_gfx1250<op, name>,
+  VFLAT_Real_RTN_gfx1250<op, name>,
+  VFLAT_Real_SADDR_RTN_gfx1250<op, name>;
+
+defm TENSOR_SAVE                      : VFLAT_Real_gfx1250<0x06e>;
+defm TENSOR_STOP                      : VFLAT_Real_gfx1250<0x06f>;
+
+defm GLOBAL_LOAD_TR_B128_w32          : VFLAT_Real_AllAddr_gfx1250<0x057, "global_load_tr16_b128">;
+defm GLOBAL_LOAD_TR_B64_w32           : VFLAT_Real_AllAddr_gfx1250<0x058, "global_load_tr8_b64">;
+
+defm GLOBAL_LOAD_TR4_B64              : VFLAT_Real_AllAddr_gfx1250<0x073>;
+defm GLOBAL_LOAD_TR6_B96              : VFLAT_Real_AllAddr_gfx1250<0x074>;
+
 def True16D16Table : GenericTable {
   let FilterClass = "True16D16Table";
   let CppTypeName = "True16D16Info";
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index bc95d3f040e1..0976fccf78d8 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -44,6 +44,11 @@ static cl::opt<unsigned, false, MFMAPaddingRatioParser>
                      cl::desc("Fill a percentage of the latency between "
                               "neighboring MFMA with s_nops."));
 
+// This is intended for debugging purposes only.
+static cl::opt<unsigned>
+    NopPadding("amdgpu-snop-padding", cl::init(0), cl::Hidden,
+               cl::desc("Insert a s_nop x before every instruction"));
+
 //===----------------------------------------------------------------------===//
 // Hazard Recognizer Implementation
 //===----------------------------------------------------------------------===//
@@ -300,7 +305,7 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
   unsigned W = PreEmitNoopsCommon(MI);
   fixHazards(MI);
   CurrCycleInstr = nullptr;
-  return W;
+  return std::max(W, NopPadding.getValue());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index eed3fb20f5be..7d6723a6108b 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -361,6 +361,69 @@ static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
   return LastUseMask;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// GCNRPTarget
+
+GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP,
+                         bool CombineVGPRSavings)
+    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
+  const Function &F = MF.getFunction();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  setRegLimits(ST.getMaxNumSGPRs(F), ST.getMaxNumVGPRs(F), MF);
+}
+
+GCNRPTarget::GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs,
+                         const MachineFunction &MF, const GCNRegPressure &RP,
+                         bool CombineVGPRSavings)
+    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
+  setRegLimits(NumSGPRs, NumVGPRs, MF);
+}
+
+GCNRPTarget::GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
+                         const GCNRegPressure &RP, bool CombineVGPRSavings)
+    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  unsigned DynamicVGPRBlockSize =
+      MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
+  setRegLimits(ST.getMaxNumSGPRs(Occupancy, /*Addressable=*/false),
+               ST.getMaxNumVGPRs(Occupancy, DynamicVGPRBlockSize), MF);
+}
+
+void GCNRPTarget::setRegLimits(unsigned NumSGPRs, unsigned NumVGPRs,
+                               const MachineFunction &MF) {
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  unsigned DynamicVGPRBlockSize =
+      MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
+  MaxSGPRs = std::min(ST.getAddressableNumSGPRs(), NumSGPRs);
+  MaxVGPRs = std::min(ST.getAddressableNumArchVGPRs(), NumVGPRs);
+  MaxUnifiedVGPRs =
+      ST.hasGFX90AInsts()
+          ? std::min(ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), NumVGPRs)
+          : 0;
+}
+
+bool GCNRPTarget::isSaveBeneficial(Register Reg,
+                                   const MachineRegisterInfo &MRI) const {
+  const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+  const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
+  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
+
+  if (SRI->isSGPRClass(RC))
+    return RP.getSGPRNum() > MaxSGPRs;
+  unsigned NumVGPRs =
+      SRI->isAGPRClass(RC) ? RP.getAGPRNum() : RP.getArchVGPRNum();
+  return isVGPRBankSaveBeneficial(NumVGPRs);
+}
+
+bool GCNRPTarget::satisfied() const {
+  if (RP.getSGPRNum() > MaxSGPRs)
+    return false;
+  if (RP.getVGPRNum(false) > MaxVGPRs &&
+      (!CombineVGPRSavings || !satisifiesVGPRBanksTarget()))
+    return false;
+  return satisfiesUnifiedTarget();
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // GCNRPTracker
 
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index 397e891c8d80..3749b6d1efc6 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -162,6 +162,101 @@ inline GCNRegPressure operator-(const GCNRegPressure &P1,
   return Diff;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// GCNRPTarget
+
+/// Models a register pressure target, allowing to evaluate and track register
+/// savings against that target from a starting \ref GCNRegPressure.
+class GCNRPTarget {
+public:
+  /// Sets up the target such that the register pressure starting at \p RP does
+  /// not show register spilling on function \p MF (w.r.t. the function's
+  /// mininum target occupancy).
+  GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP,
+              bool CombineVGPRSavings = false);
+
+  /// Sets up the target such that the register pressure starting at \p RP does
+  /// not use more than \p NumSGPRs SGPRs and \p NumVGPRs VGPRs on function \p
+  /// MF.
+  GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs, const MachineFunction &MF,
+              const GCNRegPressure &RP, bool CombineVGPRSavings = false);
+
+  /// Sets up the target such that the register pressure starting at \p RP does
+  /// not prevent achieving an occupancy of at least \p Occupancy on function
+  /// \p MF.
+  GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
+              const GCNRegPressure &RP, bool CombineVGPRSavings = false);
+
+  const GCNRegPressure &getCurrentRP() const { return RP; }
+
+  void setRP(const GCNRegPressure &NewRP) { RP = NewRP; }
+
+  /// Determines whether saving virtual register \p Reg will be beneficial
+  /// towards achieving the RP target.
+  bool isSaveBeneficial(Register Reg, const MachineRegisterInfo &MRI) const;
+
+  /// Saves virtual register \p Reg with lanemask \p Mask.
+  void saveReg(Register Reg, LaneBitmask Mask, const MachineRegisterInfo &MRI) {
+    RP.inc(Reg, Mask, LaneBitmask::getNone(), MRI);
+  }
+
+  /// Whether the current RP is at or below the defined pressure target.
+  bool satisfied() const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  friend raw_ostream &operator<<(raw_ostream &OS, const GCNRPTarget &Target) {
+    OS << "Actual/Target: " << Target.RP.getSGPRNum() << '/' << Target.MaxSGPRs
+       << " SGPRs, " << Target.RP.getArchVGPRNum() << '/' << Target.MaxVGPRs
+       << " ArchVGPRs, " << Target.RP.getAGPRNum() << '/' << Target.MaxVGPRs
+       << " AGPRs";
+
+    if (Target.MaxUnifiedVGPRs) {
+      OS << ", " << Target.RP.getVGPRNum(true) << '/' << Target.MaxUnifiedVGPRs
+         << " VGPRs (unified)";
+    } else if (Target.CombineVGPRSavings) {
+      OS << ", " << Target.RP.getArchVGPRNum() + Target.RP.getAGPRNum() << '/'
+         << 2 * Target.MaxVGPRs << " VGPRs (combined target)";
+    }
+    return OS;
+  }
+#endif
+
+private:
+  /// Current register pressure.
+  GCNRegPressure RP;
+
+  /// Target number of SGPRs.
+  unsigned MaxSGPRs;
+  /// Target number of ArchVGPRs and AGPRs.
+  unsigned MaxVGPRs;
+  /// Target number of overall VGPRs for subtargets with unified RFs. Always 0
+  /// for subtargets with non-unified RFs.
+  unsigned MaxUnifiedVGPRs;
+  /// Whether we consider that the register allocator will be able to swap
+  /// between ArchVGPRs and AGPRs by copying them to a super register class.
+  /// Concretely, this allows savings in one of the VGPR banks to help toward
+  /// savings in the other VGPR bank.
+  bool CombineVGPRSavings;
+
+  inline bool satisifiesVGPRBanksTarget() const {
+    assert(CombineVGPRSavings && "only makes sense with combined savings");
+    return RP.getArchVGPRNum() + RP.getAGPRNum() <= 2 * MaxVGPRs;
+  }
+
+  /// Always satisified when the subtarget doesn't have a unified RF.
+  inline bool satisfiesUnifiedTarget() const {
+    return !MaxUnifiedVGPRs || RP.getVGPRNum(true) <= MaxUnifiedVGPRs;
+  }
+
+  inline bool isVGPRBankSaveBeneficial(unsigned NumVGPRs) const {
+    return NumVGPRs > MaxVGPRs || !satisfiesUnifiedTarget() ||
+           (CombineVGPRSavings && !satisifiesVGPRBanksTarget());
+  }
+
+  void setRegLimits(unsigned MaxSGPRs, unsigned MaxVGPRs,
+                    const MachineFunction &MF);
+};
+
 ///////////////////////////////////////////////////////////////////////////////
 // GCNRPTracker
 
@@ -370,7 +465,7 @@ getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {
     if (!LI.hasSubRanges()) {
       for (auto SI : LiveIdxs)
         LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =
-          MRI.getMaxLaneMaskForVReg(Reg);
+            MRI.getMaxLaneMaskForVReg(Reg);
     } else
       for (const auto &S : LI.subranges()) {
         // constrain search for subranges by indexes live at main range
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index e15c0f7f1ab7..fce8f36d4596 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1699,163 +1699,17 @@ bool PreRARematStage::allUsesAvailableAt(const MachineInstr *InstToRemat,
   return true;
 }
 
-namespace {
-/// Models excess register pressure in a region and tracks our progress as we
-/// identify rematerialization opportunities.
-struct ExcessRP {
-  /// Number of excess ArchVGPRs.
-  unsigned ArchVGPRs = 0;
-  /// Number of excess AGPRs.
-  unsigned AGPRs = 0;
-  /// For unified register files, number of excess VGPRs.
-  unsigned VGPRs = 0;
-  /// For unified register files with AGPR usage, number of excess ArchVGPRs to
-  /// save before we are able to save a whole allocation granule.
-  unsigned ArchVGPRsToAlignment = 0;
-  /// Whether the region uses AGPRs.
-  bool HasAGPRs = false;
-  /// Whether the subtarget has a unified RF.
-  bool UnifiedRF;
-
-  /// Constructs the excess RP model; determines the excess pressure w.r.t. a
-  /// maximum number of allowed VGPRs.
-  ExcessRP(const GCNSubtarget &ST, const GCNRegPressure &RP, unsigned MaxVGPRs);
-
-  /// Accounts for \p NumRegs saved ArchVGPRs in the model. If \p
-  /// UseArchVGPRForAGPRSpill is true, saved ArchVGPRs are used to save excess
-  /// AGPRs once excess ArchVGPR pressure has been eliminated. Returns whether
-  /// saving these ArchVGPRs helped reduce excess pressure.
-  bool saveArchVGPRs(unsigned NumRegs, bool UseArchVGPRForAGPRSpill);
-
-  /// Accounts for \p NumRegs saved AGPRS in the model. Returns whether saving
-  /// these ArchVGPRs helped reduce excess pressure.
-  bool saveAGPRs(unsigned NumRegs);
-
-  /// Returns whether there is any excess register pressure.
-  operator bool() const { return ArchVGPRs != 0 || AGPRs != 0 || VGPRs != 0; }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  friend raw_ostream &operator<<(raw_ostream &OS, const ExcessRP &Excess) {
-    OS << Excess.ArchVGPRs << " ArchVGPRs, " << Excess.AGPRs << " AGPRs, and "
-       << Excess.VGPRs << " VGPRs (next ArchVGPR aligment in "
-       << Excess.ArchVGPRsToAlignment << " registers)\n";
-    return OS;
-  }
-#endif
-
-private:
-  static inline bool saveRegs(unsigned &LeftToSave, unsigned &NumRegs) {
-    unsigned NumSaved = std::min(LeftToSave, NumRegs);
-    NumRegs -= NumSaved;
-    LeftToSave -= NumSaved;
-    return NumSaved;
-  }
-};
-} // namespace
-
-ExcessRP::ExcessRP(const GCNSubtarget &ST, const GCNRegPressure &RP,
-                   unsigned MaxVGPRs)
-    : UnifiedRF(ST.hasGFX90AInsts()) {
-  unsigned NumArchVGPRs = RP.getArchVGPRNum();
-  unsigned NumAGPRs = RP.getAGPRNum();
-  HasAGPRs = NumAGPRs;
-
-  if (!UnifiedRF) {
-    // Non-unified RF. Account for excess pressure for ArchVGPRs and AGPRs
-    // independently.
-    if (NumArchVGPRs > MaxVGPRs)
-      ArchVGPRs = NumArchVGPRs - MaxVGPRs;
-    if (NumAGPRs > MaxVGPRs)
-      AGPRs = NumAGPRs - MaxVGPRs;
-    return;
-  }
-
-  // Independently of whether overall VGPR pressure is under the limit, we still
-  // have to check whether ArchVGPR pressure or AGPR pressure alone exceeds the
-  // number of addressable registers in each category.
-  const unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();
-  if (NumArchVGPRs > MaxArchVGPRs) {
-    ArchVGPRs = NumArchVGPRs - MaxArchVGPRs;
-    NumArchVGPRs = MaxArchVGPRs;
-  }
-  if (NumAGPRs > MaxArchVGPRs) {
-    AGPRs = NumAGPRs - MaxArchVGPRs;
-    NumAGPRs = MaxArchVGPRs;
-  }
-
-  // Check overall VGPR usage against the limit; any excess above addressable
-  // register limits has already been accounted for.
-  const unsigned Granule = AMDGPU::IsaInfo::getArchVGPRAllocGranule();
-  unsigned NumVGPRs = GCNRegPressure::getUnifiedVGPRNum(NumArchVGPRs, NumAGPRs);
-  if (NumVGPRs > MaxVGPRs) {
-    VGPRs = NumVGPRs - MaxVGPRs;
-    ArchVGPRsToAlignment = NumArchVGPRs - alignDown(NumArchVGPRs, Granule);
-    if (!ArchVGPRsToAlignment)
-      ArchVGPRsToAlignment = Granule;
-  }
-}
-
-bool ExcessRP::saveArchVGPRs(unsigned NumRegs, bool UseArchVGPRForAGPRSpill) {
-  bool Progress = saveRegs(ArchVGPRs, NumRegs);
-  if (!NumRegs)
-    return Progress;
-
-  if (!UnifiedRF) {
-    if (UseArchVGPRForAGPRSpill)
-      Progress |= saveRegs(AGPRs, NumRegs);
-  } else if (HasAGPRs && (VGPRs || (UseArchVGPRForAGPRSpill && AGPRs))) {
-    // There is progress as long as there are VGPRs left to save, even if the
-    // save induced by this particular call does not cross an ArchVGPR alignment
-    // barrier.
-    Progress = true;
-
-    // ArchVGPRs can only be allocated as a multiple of a granule in unified RF.
-    unsigned NumSavedRegs = 0;
-
-    // Count the number of whole ArchVGPR allocation granules we can save.
-    const unsigned Granule = AMDGPU::IsaInfo::getArchVGPRAllocGranule();
-    if (unsigned NumGranules = NumRegs / Granule; NumGranules) {
-      NumSavedRegs = NumGranules * Granule;
-      NumRegs -= NumSavedRegs;
-    }
-
-    // We may be able to save one more whole ArchVGPR allocation granule.
-    if (NumRegs >= ArchVGPRsToAlignment) {
-      NumSavedRegs += Granule;
-      ArchVGPRsToAlignment = Granule - (NumRegs - ArchVGPRsToAlignment);
-    } else {
-      ArchVGPRsToAlignment -= NumRegs;
-    }
-
-    // Prioritize saving generic VGPRs, then AGPRs if we allow AGPR-to-ArchVGPR
-    // spilling and have some free ArchVGPR slots.
-    saveRegs(VGPRs, NumSavedRegs);
-    if (UseArchVGPRForAGPRSpill)
-      saveRegs(AGPRs, NumSavedRegs);
-  } else {
-    // No AGPR usage in the region i.e., no allocation granule to worry about.
-    Progress |= saveRegs(VGPRs, NumRegs);
-  }
-
-  return Progress;
-}
-
-bool ExcessRP::saveAGPRs(unsigned NumRegs) {
-  return saveRegs(AGPRs, NumRegs) || saveRegs(VGPRs, NumRegs);
-}
-
 bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
-  const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(DAG.TRI);
-
   REMAT_DEBUG({
     dbgs() << "Collecting rematerializable instructions in ";
     MF.getFunction().printAsOperand(dbgs(), false);
     dbgs() << '\n';
   });
 
-  // Maps optimizable regions (i.e., regions at minimum and VGPR-limited
-  // occupancy, or regions with VGPR spilling) to a model of their excess RP.
-  DenseMap<unsigned, ExcessRP> OptRegions;
+  // Maps optimizable regions (i.e., regions at minimum and register-limited
+  // occupancy, or regions with spilling) to the target RP we would like to
+  // reach.
+  DenseMap<unsigned, GCNRPTarget> OptRegions;
   const Function &F = MF.getFunction();
   unsigned DynamicVGPRBlockSize =
       MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
@@ -1869,83 +1723,68 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
       ST.getMaxNumVGPRs(DAG.MinOccupancy + 1, DynamicVGPRBlockSize);
   IncreaseOccupancy = WavesPerEU.second > DAG.MinOccupancy;
 
-  auto ClearOptRegionsIf = [&](bool Cond) -> bool {
-    if (Cond) {
-      // We won't try to increase occupancy.
-      IncreaseOccupancy = false;
-      OptRegions.clear();
-    }
-    return Cond;
-  };
-
   // Collect optimizable regions. If there is spilling in any region we will
-  // just try to reduce ArchVGPR spilling. Otherwise we will try to increase
-  // occupancy by one in the whole function.
+  // just try to reduce spilling. Otherwise we will try to increase occupancy by
+  // one in the whole function.
   for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
     GCNRegPressure &RP = DAG.Pressure[I];
-
-    // Check whether SGPR pressures prevents us from eliminating spilling.
-    unsigned NumSGPRs = RP.getSGPRNum();
-    if (NumSGPRs > MaxSGPRsNoSpill)
-      ClearOptRegionsIf(IncreaseOccupancy);
-
-    ExcessRP Excess(ST, RP, MaxVGPRsNoSpill);
-    if (Excess) {
-      ClearOptRegionsIf(IncreaseOccupancy);
+    // We allow ArchVGPR or AGPR savings to count as savings of the other kind
+    // of VGPR only when trying to eliminate spilling. We cannot do this when
+    // trying to increase occupancy since VGPR class swaps only occur later in
+    // the register allocator i.e., the scheduler will not be able to reason
+    // about these savings and will not report an increase in the achievable
+    // occupancy, triggering rollbacks.
+    GCNRPTarget Target(MaxSGPRsNoSpill, MaxVGPRsNoSpill, MF, RP,
+                       /*CombineVGPRSavings=*/true);
+    if (!Target.satisfied() && IncreaseOccupancy) {
+      // There is spilling in the region and we were so far trying to increase
+      // occupancy. Strop trying that and focus on reducing spilling.
+      IncreaseOccupancy = false;
+      OptRegions.clear();
     } else if (IncreaseOccupancy) {
-      // Check whether SGPR pressure prevents us from increasing occupancy.
-      if (ClearOptRegionsIf(NumSGPRs > MaxSGPRsIncOcc)) {
-        if (DAG.MinOccupancy >= WavesPerEU.first)
-          return false;
-        continue;
-      }
-      if ((Excess = ExcessRP(ST, RP, MaxVGPRsIncOcc))) {
-        // We can only rematerialize ArchVGPRs at this point.
-        unsigned NumArchVGPRsToRemat = Excess.ArchVGPRs + Excess.VGPRs;
-        bool NotEnoughArchVGPRs = NumArchVGPRsToRemat > RP.getArchVGPRNum();
-        if (ClearOptRegionsIf(Excess.AGPRs || NotEnoughArchVGPRs)) {
-          if (DAG.MinOccupancy >= WavesPerEU.first)
-            return false;
-          continue;
-        }
-      }
+      // There is no spilling in the region, try to increase occupancy.
+      Target = GCNRPTarget(MaxSGPRsIncOcc, MaxVGPRsIncOcc, MF, RP,
+                           /*CombineVGPRSavings=*/false);
     }
-    if (Excess)
-      OptRegions.insert({I, Excess});
+    if (!Target.satisfied())
+      OptRegions.insert({I, Target});
   }
   if (OptRegions.empty())
     return false;
 
 #ifndef NDEBUG
-  if (IncreaseOccupancy)
-    REMAT_DEBUG(dbgs() << "Occupancy minimal in regions:\n");
-  else
-    REMAT_DEBUG(dbgs() << "Spilling in regions:\n");
+  if (IncreaseOccupancy) {
+    REMAT_DEBUG(dbgs() << "Occupancy minimal (" << DAG.MinOccupancy
+                       << ") in regions:\n");
+  } else {
+    REMAT_DEBUG(dbgs() << "Spilling w.r.t. minimum target occupancy ("
+                       << WavesPerEU.first << ") in regions:\n");
+  }
   for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
     if (auto OptIt = OptRegions.find(I); OptIt != OptRegions.end())
-      REMAT_DEBUG(dbgs() << "  " << I << ": " << OptIt->getSecond() << '\n');
+      REMAT_DEBUG(dbgs() << "  [" << I << "] " << OptIt->getSecond() << '\n');
   }
 #endif
 
   // When we are reducing spilling, the target is the minimum target number of
-  // waves/EU determined by the subtarget.
-  TargetOcc = IncreaseOccupancy ? DAG.MinOccupancy + 1 : WavesPerEU.first;
+  // waves/EU determined by the subtarget. In cases where either one of
+  // "amdgpu-num-sgpr" or "amdgpu-num-vgpr" are set on the function, the current
+  // minimum region occupancy may be higher than the latter.
+  TargetOcc = IncreaseOccupancy ? DAG.MinOccupancy + 1
+                                : std::max(DAG.MinOccupancy, WavesPerEU.first);
 
   // Accounts for a reduction in RP in an optimizable region. Returns whether we
   // estimate that we have identified enough rematerialization opportunities to
   // achieve our goal, and sets Progress to true when this particular reduction
   // in pressure was helpful toward that goal.
-  auto ReduceRPInRegion = [&](auto OptIt, LaneBitmask Mask,
+  auto ReduceRPInRegion = [&](auto OptIt, Register Reg, LaneBitmask Mask,
                               bool &Progress) -> bool {
-    ExcessRP &Excess = OptIt->getSecond();
-    // We allow saved ArchVGPRs to be considered as free spill slots for AGPRs
-    // only when we are just trying to eliminate spilling to memory. At this
-    // point we err on the conservative side and do not increase
-    // register-to-register spilling for the sake of increasing occupancy.
-    Progress |=
-        Excess.saveArchVGPRs(SIRegisterInfo::getNumCoveredRegs(Mask),
-                             /*UseArchVGPRForAGPRSpill=*/!IncreaseOccupancy);
-    if (!Excess)
+    GCNRPTarget &Target = OptIt->getSecond();
+    if (!Target.isSaveBeneficial(Reg, DAG.MRI))
+      return false;
+    Progress = true;
+    Target.saveReg(Reg, Mask, DAG.MRI);
+    if (Target.satisfied())
       OptRegions.erase(OptIt->getFirst());
     return OptRegions.empty();
   };
@@ -1966,10 +1805,9 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
       if (!isTriviallyReMaterializable(DefMI))
         continue;
 
-      // We only support rematerializing virtual VGPRs with one definition.
+      // We only support rematerializing virtual registers with one definition.
       Register Reg = DefMI.getOperand(0).getReg();
-      if (!Reg.isVirtual() || !SRI->isVGPRClass(DAG.MRI.getRegClass(Reg)) ||
-          !DAG.MRI.hasOneDef(Reg))
+      if (!Reg.isVirtual() || !DAG.MRI.hasOneDef(Reg))
         continue;
 
       // We only care to rematerialize the instruction if it has a single
@@ -2014,7 +1852,7 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
         // instruction and the end of the region.
         REMAT_DEBUG(dbgs() << "  Defining region is optimizable\n");
         LaneBitmask Mask = DAG.RegionLiveOuts.getLiveRegsForRegionIdx(I)[Reg];
-        if (ReduceRPInRegion(It, Mask, RematUseful))
+        if (ReduceRPInRegion(It, Reg, Mask, RematUseful))
           return true;
       }
 
@@ -2034,7 +1872,8 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
         // instruction's use.
         if (auto It = OptRegions.find(LIRegion); It != OptRegions.end()) {
           REMAT_DEBUG(dbgs() << "  Live-in in region " << LIRegion << '\n');
-          if (ReduceRPInRegion(It, DAG.LiveIns[LIRegion][Reg], RematUseful))
+          if (ReduceRPInRegion(It, Reg, DAG.LiveIns[LIRegion][Reg],
+                               RematUseful))
             return true;
         }
       }
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index aa48c7c9eaed..94cd795bbc8f 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -440,8 +440,6 @@ class ClusteredLowOccStage : public GCNSchedStage {
 /// estimates reducing spilling or increasing occupancy is possible, as few
 /// instructions as possible are rematerialized to reduce potential negative
 /// effects on function latency.
-///
-/// TODO: We should extend this to work on SGPRs and AGPRs as well.
 class PreRARematStage : public GCNSchedStage {
 private:
   /// Useful information about a rematerializable instruction.
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index dcc920186ea4..7b8f0f44cbe2 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -52,11 +52,6 @@ static cl::opt<unsigned>
                  cl::desc("Number of addresses from which to enable MIMG NSA."),
                  cl::init(2), cl::Hidden);
 
-static cl::opt<bool>
-    CoerceIllegal("amdgpu-coerce-illegal-types",
-                  cl::desc("Whether or not to coerce illegal types"),
-                  cl::ReallyHidden, cl::init(false));
-
 GCNSubtarget::~GCNSubtarget() = default;
 
 GCNSubtarget &GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
@@ -196,8 +191,6 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
   RegBankInfo = std::make_unique<AMDGPURegisterBankInfo>(*this);
   InstSelector =
       std::make_unique<AMDGPUInstructionSelector>(*this, *RegBankInfo, TM);
-
-  ShouldCoerceIllegalTypes = CoerceIllegal;
 }
 
 const SelectionDAGTargetInfo *GCNSubtarget::getSelectionDAGInfo() const {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 8325e68e3f50..45721422edcf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -231,6 +231,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasPseudoScalarTrans = false;
   bool HasRestrictedSOffset = false;
   bool HasBitOp3Insts = false;
+  bool HasTransposeLoadF4F6Insts = false;
   bool HasPrngInst = false;
   bool HasBVHDualAndBVH8Insts = false;
   bool HasPermlane16Swap = false;
@@ -262,6 +263,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasMinimum3Maximum3PKF16 = false;
   bool HasLshlAddU64Inst = false;
   bool HasPointSampleAccel = false;
+  bool HasLdsBarrierArriveAtomic = false;
   bool HasSetPrioIncWgInst = false;
 
   bool RequiresCOV6 = false;
@@ -270,8 +272,6 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   // Dummy feature to use for assembler in tablegen.
   bool FeatureDisable = false;
 
-  bool ShouldCoerceIllegalTypes = false;
-
 private:
   SIInstrInfo InstrInfo;
   SITargetLowering TLInfo;
@@ -691,13 +691,15 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return GFX10_BEncoding;
   }
 
+  bool hasMTBUFInsts() const { return !hasGFX1250Insts(); }
+
+  bool hasFormattedMUBUFInsts() const { return !hasGFX1250Insts(); }
+
   bool hasExportInsts() const {
-    return !hasGFX940Insts();
+    return !hasGFX940Insts() && !hasGFX1250Insts();
   }
 
-  bool hasVINTERPEncoding() const {
-    return GFX11Insts;
-  }
+  bool hasVINTERPEncoding() const { return GFX11Insts && !hasGFX1250Insts(); }
 
   // DS_ADD_F64/DS_ADD_RTN_F64
   bool hasLdsAtomicAddF64() const { return hasGFX90AInsts(); }
@@ -1290,7 +1292,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool hasVALUReadSGPRHazard() const { return getGeneration() == GFX12; }
 
   /// Return if operations acting on VGPR tuples require even alignment.
-  bool needsAlignedVGPRs() const { return GFX90AInsts; }
+  bool needsAlignedVGPRs() const { return GFX90AInsts || GFX1250Insts; }
 
   /// Return true if the target has the S_PACK_HL_B32_B16 instruction.
   bool hasSPackHL() const { return GFX11Insts; }
@@ -1372,12 +1374,16 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return HasMinimum3Maximum3PKF16;
   }
 
+  bool hasTransposeLoadF4F6Insts() const { return HasTransposeLoadF4F6Insts; }
+
   /// \returns true if the target has s_wait_xcnt insertion. Supported for
   /// GFX1250.
   bool hasWaitXCnt() const { return HasWaitXcnt; }
 
   bool hasPointSampleAccel() const { return HasPointSampleAccel; }
 
+  bool hasLdsBarrierArriveAtomic() const { return HasLdsBarrierArriveAtomic; }
+
   /// \returns The maximum number of instructions that can be enclosed in an
   /// S_CLAUSE on the given subtarget, or 0 for targets that do not support that
   /// instruction.
@@ -1478,10 +1484,6 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   // of sign-extending.
   bool hasGetPCZeroExtension() const { return GFX12Insts; }
 
-  /// \returns whether or not we should coerce illegal types into vectors of
-  // legal types for values that span basic blocks.
-  bool shouldCoerceIllegalTypes() const { return ShouldCoerceIllegalTypes; }
-
   /// \returns SGPR allocation granularity supported by the subtarget.
   unsigned getSGPRAllocGranule() const {
     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index 78bd8de07306..b333c97ecc37 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -15,7 +15,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -50,7 +49,6 @@ class AMDGPUAsmBackend : public MCAsmBackend {
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
 };
 
 } //End anonymous namespace
@@ -95,7 +93,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
     return 2;
   case FK_SecRel_4:
   case FK_Data_4:
-  case FK_PCRel_4:
     return 4;
   case FK_SecRel_8:
   case FK_Data_8:
@@ -122,7 +119,6 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   case FK_Data_2:
   case FK_Data_4:
   case FK_Data_8:
-  case FK_PCRel_4:
   case FK_SecRel_4:
     return Value;
   default:
@@ -130,10 +126,13 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
   }
 }
 
-void AMDGPUAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void AMDGPUAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (mc::isRelocation(Fixup.getKind()))
     return;
 
@@ -173,12 +172,12 @@ AMDGPUAsmBackend::getFixupKind(StringRef Name) const {
 
 MCFixupKindInfo AMDGPUAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[AMDGPU::NumTargetFixupKinds] = {
-    // name                   offset bits  flags
-    { "fixup_si_sopp_br",     0,     16,   MCFixupKindInfo::FKF_IsPCRel },
+      // name                   offset bits  flags
+      {"fixup_si_sopp_br", 0, 16, 0},
   };
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -188,11 +187,6 @@ MCFixupKindInfo AMDGPUAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return Infos[Kind - FirstTargetFixupKind];
 }
 
-bool AMDGPUAsmBackend::shouldForceRelocation(const MCFixup &,
-                                             const MCValue &Target) {
-  return Target.getSpecifier();
-}
-
 unsigned AMDGPUAsmBackend::getMinimumNopSize() const {
   return 4;
 }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
index 9a3f418e0d52..22ae5f4e7191 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
@@ -69,8 +69,6 @@ unsigned AMDGPUELFObjectWriter::getRelocType(const MCFixup &Fixup,
   MCFixupKind Kind = Fixup.getKind();
   switch (Kind) {
   default: break;
-  case FK_PCRel_4:
-    return ELF::R_AMDGPU_REL32;
   case FK_Data_4:
   case FK_SecRel_4:
     return IsPCRel ? ELF::R_AMDGPU_REL32 : ELF::R_AMDGPU_ABS32;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a6ce42dca92b..fa1474d15383 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -173,13 +173,12 @@ void AMDGPUInstPrinter::printTH(const MCInst *MI, int64_t TH, int64_t Scope,
 
   const unsigned Opcode = MI->getOpcode();
   const MCInstrDesc &TID = MII.get(Opcode);
-  bool IsStore = TID.mayStore();
-  bool IsAtomic =
-      TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet);
+  unsigned THType = AMDGPU::getTemporalHintType(TID);
+  bool IsStore = (THType == AMDGPU::CPol::TH_TYPE_STORE);
 
   O << " th:";
 
-  if (IsAtomic) {
+  if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
     O << "TH_ATOMIC_";
     if (TH & AMDGPU::CPol::TH_ATOMIC_CASCADE) {
       if (Scope >= AMDGPU::CPol::SCOPE_DEV)
@@ -196,9 +195,6 @@ void AMDGPUInstPrinter::printTH(const MCInst *MI, int64_t TH, int64_t Scope,
     if (!IsStore && TH == AMDGPU::CPol::TH_RESERVED)
       O << formatHex(TH);
     else {
-      // This will default to printing load variants when neither MayStore nor
-      // MayLoad flag is present which is the case with instructions like
-      // image_get_resinfo.
       O << (IsStore ? "TH_STORE_" : "TH_LOAD_");
       switch (TH) {
       case AMDGPU::CPol::TH_NT:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 5032b7a9ca87..8e1dadd41558 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -16,7 +16,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {AMDGPUMCExpr::S_GOTPCREL, "gotpcrel"},
     {AMDGPUMCExpr::S_GOTPCREL32_LO, "gotpcrel32@lo"},
     {AMDGPUMCExpr::S_GOTPCREL32_HI, "gotpcrel32@hi"},
@@ -57,7 +57,7 @@ AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT,
   SupportsHeterogeneousDebuggingExtensions = true;
 
   UseIntegratedAssembler = false;
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 bool AMDGPUMCAsmInfo::shouldOmitSectionDirective(StringRef SectionName) const {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index ab7e51ac2832..086f980f5fe2 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -103,6 +103,11 @@ MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII,
   return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo());
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // Returns the encoding value to use if the given integer is an integer inline
 // immediate value, or 0 if it is not.
 template <typename IntTy>
@@ -255,13 +260,9 @@ AMDGPUMCCodeEmitter::getLitEncoding(const MCOperand &MO,
                                     const MCSubtargetInfo &STI) const {
   int64_t Imm;
   if (MO.isExpr()) {
-    const auto *C = dyn_cast<MCConstantExpr>(MO.getExpr());
-    if (!C)
+    if (!MO.getExpr()->evaluateAsAbsolute(Imm))
       return 255;
-
-    Imm = C->getValue();
   } else {
-
     assert(!MO.isDFPImm());
 
     if (!MO.isImm())
@@ -449,8 +450,7 @@ void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
 
   if (MO.isExpr()) {
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = (MCFixupKind)AMDGPU::fixup_si_sopp_br;
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    addFixup(Fixups, 0, Expr, AMDGPU::fixup_si_sopp_br, true);
     Op = APInt::getZero(96);
   } else {
     getMachineOpValue(MI, MO, Op, Fixups, STI);
@@ -656,17 +656,11 @@ void AMDGPUMCCodeEmitter::getMachineOpValueCommon(
     //
     // .Ltmp1:
     //   s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1
-    MCFixupKind Kind;
-    if (needsPCRel(MO.getExpr()))
-      Kind = FK_PCRel_4;
-    else
-      Kind = FK_Data_4;
-
+    bool PCRel = needsPCRel(MO.getExpr());
     const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
     uint32_t Offset = Desc.getSize();
     assert(Offset == 4 || Offset == 8);
-
-    Fixups.push_back(MCFixup::create(Offset, MO.getExpr(), Kind, MI.getLoc()));
+    addFixup(Fixups, Offset, MO.getExpr(), FK_Data_4, PCRel);
   }
 
   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
index 8195c93d847b..02a2b06d0951 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
@@ -166,7 +166,7 @@ uint64_t R600MCCodeEmitter::getMachineOpValue(const MCInst &MI,
     // We can't easily get the order of the current one, so compare against
     // the first one and adjust offset.
     const unsigned offset = (&MO == &MI.getOperand(0)) ? 0 : 4;
-    Fixups.push_back(MCFixup::create(offset, MO.getExpr(), FK_SecRel_4, MI.getLoc()));
+    Fixups.push_back(MCFixup::create(offset, MO.getExpr(), FK_SecRel_4));
     return 0;
   }
 
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 8d94d73bc1aa..531fae3ceff5 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -2019,3 +2019,97 @@ def : MIMGG16Mapping<IMAGE_SAMPLE_CD_O_nortn, IMAGE_SAMPLE_CD_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_CD_CL_O_nortn, IMAGE_SAMPLE_CD_CL_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_O_nortn, IMAGE_SAMPLE_C_CD_O_G16_nortn>;
 def : MIMGG16Mapping<IMAGE_SAMPLE_C_CD_CL_O_nortn, IMAGE_SAMPLE_C_CD_CL_O_G16_nortn>;
+
+//===----------------------------------------------------------------------===//
+// VIMAGE Tensor Instructions
+//===----------------------------------------------------------------------===//
+
+class VIMAGE_TENSOR_Pseudo<string opName, bit _UpTo2D = 0> :
+  InstSI<(outs ), (ins ), "", []>,
+  SIMCInstr<opName#!if(_UpTo2D, "_D2", ""), SIEncodingFamily.NONE> {
+
+  let isPseudo = 1;
+  let isCodeGenOnly = 1;
+  string Mnemonic = opName;
+
+  let VALU = 1;
+  let maybeAtomic = 0;
+  let TENSOR_CNT = 1;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let Uses = [EXEC, TENSORcnt];
+  let Defs = [TENSORcnt];
+  let SchedRW = [WriteVMEM, WriteLDS];
+  let UseNamedOperandTable = 1;
+  let hasSideEffects = 0;
+
+  bit UpTo2D = _UpTo2D;
+  let InOperandList = !if(UpTo2D, (ins SReg_128:$vaddr0, SReg_256:$vaddr1, R128A16:$r128, CPol:$cpol),
+                                      (ins SReg_128:$vaddr0, SReg_256:$vaddr1, SReg_128:$vaddr2,
+                                       SReg_128:$vaddr3, R128A16:$r128, CPol:$cpol));
+  string AsmOperands = " $vaddr0, $vaddr1"#!if(UpTo2D, "", ", $vaddr2, $vaddr3")#"$r128$cpol";
+}
+
+let SubtargetPredicate = isGFX1250Plus in {
+def TENSOR_LOAD_TO_LDS    : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds">;
+def TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds">;
+def TENSOR_LOAD_TO_LDS_D2    : VIMAGE_TENSOR_Pseudo<"tensor_load_to_lds", 1>;
+def TENSOR_STORE_FROM_LDS_D2 : VIMAGE_TENSOR_Pseudo<"tensor_store_from_lds", 1>;
+} // End SubtargetPredicate = isGFX1250Plus.
+
+class TensorPat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <
+  (node v4i32:$vaddr0, v8i32:$vaddr1, v4i32:$vaddr2, v4i32:$vaddr3, (i32 timm:$cpol)),
+  (inst $vaddr0, $vaddr1, $vaddr2, $vaddr3, 0, $cpol)
+>;
+
+class TensorD2Pat <VIMAGE_TENSOR_Pseudo inst, SDPatternOperator node> : GCNPat <
+  (node v4i32:$vaddr0, v8i32:$vaddr1, (i32 timm:$cpol)),
+  (inst $vaddr0, $vaddr1, 0, $cpol)
+>;
+
+let SubtargetPredicate = isGFX1250Plus in {
+def : TensorPat   <TENSOR_LOAD_TO_LDS, int_amdgcn_tensor_load_to_lds>;
+def : TensorPat   <TENSOR_STORE_FROM_LDS, int_amdgcn_tensor_store_from_lds>;
+def : TensorD2Pat <TENSOR_LOAD_TO_LDS_D2, int_amdgcn_tensor_load_to_lds_d2>;
+def : TensorD2Pat <TENSOR_STORE_FROM_LDS_D2, int_amdgcn_tensor_store_from_lds_d2>;
+}
+
+class VIMAGE_TENSOR_Real <bits<8> op, VIMAGE_TENSOR_Pseudo ps, string opName = ps.Mnemonic> :
+  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,
+  VIMAGEe<op> {
+
+  // copy relevant pseudo op flags
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let TSFlags              = ps.TSFlags;
+  let mayLoad              = ps.mayLoad;
+  let mayStore             = ps.mayStore;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW              = ps.SchedRW;
+
+  // D# group 2 and 3 set to NULL for 2D or less.
+  let vaddr2 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);
+  let vaddr3 = !if(ps.UpTo2D, !cast<int>(SGPR_NULL_gfx11plus.HWEncoding), ?);
+
+  // set to 0 based on SPG.
+  let vaddr4 = 0;
+  let rsrc = 0;
+  let vdata = 0;
+  let d16 = 0;
+  let a16 = 0;
+  let tfe = 0;
+  let dmask = 1; // sp3
+  let dim = 1;   // sp3
+}
+
+multiclass VIMAGE_TENSOR_Real_gfx1250<bits<8> op> {
+  let AssemblerPredicate = isGFX1250Plus, DecoderNamespace = "GFX1250" in {
+    foreach DSuffix = ["_D2", ""] in {
+      defvar ps = !cast<VIMAGE_TENSOR_Pseudo>(NAME # DSuffix);
+      def DSuffix # _gfx1250 : VIMAGE_TENSOR_Real<op, ps, ps.Mnemonic>,
+                SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX1250>;
+    }
+  }
+}
+
+defm TENSOR_LOAD_TO_LDS    : VIMAGE_TENSOR_Real_gfx1250<0xc4>;
+defm TENSOR_STORE_FROM_LDS : VIMAGE_TENSOR_Real_gfx1250<0xc5>;
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index baf74dbdde20..76e29e439320 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -107,14 +107,12 @@ enum : uint64_t {
   DisableWQM = UINT64_C(1) << 36,
   Gather4 = UINT64_C(1) << 37,
 
-  // Reserved, must be 0.
-  Reserved0 = UINT64_C(1) << 38,
+  TENSOR_CNT = UINT64_C(1) << 38,
 
   SCALAR_STORE = UINT64_C(1) << 39,
   FIXED_SIZE = UINT64_C(1) << 40,
 
-  // Reserved, must be 0.
-  Reserved1 = UINT64_C(1) << 41,
+  ASYNC_CNT = UINT64_C(1) << 41,
 
   VOP3_OPSEL = UINT64_C(1) << 42,
   maybeAtomic = UINT64_C(1) << 43,
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 4715906fc8e0..29467bb566d0 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -30,52 +30,148 @@ static cl::opt<int> SIFoldOperandsPreheaderThreshold(
 
 namespace {
 
-struct FoldCandidate {
-  MachineInstr *UseMI;
+/// Track a value we may want to fold into downstream users, applying
+/// subregister extracts along the way.
+struct FoldableDef {
   union {
-    MachineOperand *OpToFold;
+    MachineOperand *OpToFold = nullptr;
     uint64_t ImmToFold;
     int FrameIndexToFold;
   };
-  int ShrinkOpcode;
-  unsigned UseOpNo;
+
+  /// Register class of the originally defined value.
+  const TargetRegisterClass *DefRC = nullptr;
+
+  /// Track the original defining instruction for the value.
+  const MachineInstr *DefMI = nullptr;
+
+  /// Subregister to apply to the value at the use point.
+  unsigned DefSubReg = AMDGPU::NoSubRegister;
+
+  /// Kind of value stored in the union.
   MachineOperand::MachineOperandType Kind;
-  bool Commuted;
 
-  FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp,
-                bool Commuted_ = false,
-                int ShrinkOp = -1) :
-    UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
-    Kind(FoldOp->getType()),
-    Commuted(Commuted_) {
-    if (FoldOp->isImm()) {
-      ImmToFold = FoldOp->getImm();
-    } else if (FoldOp->isFI()) {
-      FrameIndexToFold = FoldOp->getIndex();
+  FoldableDef() = delete;
+  FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
+              unsigned DefSubReg = AMDGPU::NoSubRegister)
+      : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
+
+    if (FoldOp.isImm()) {
+      ImmToFold = FoldOp.getImm();
+    } else if (FoldOp.isFI()) {
+      FrameIndexToFold = FoldOp.getIndex();
     } else {
-      assert(FoldOp->isReg() || FoldOp->isGlobal());
-      OpToFold = FoldOp;
+      assert(FoldOp.isReg() || FoldOp.isGlobal());
+      OpToFold = &FoldOp;
     }
+
+    DefMI = FoldOp.getParent();
+  }
+
+  FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
+              unsigned DefSubReg = AMDGPU::NoSubRegister)
+      : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
+        Kind(MachineOperand::MO_Immediate) {}
+
+  /// Copy the current def and apply \p SubReg to the value.
+  FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
+    FoldableDef Copy(*this);
+    Copy.DefSubReg = TRI.composeSubRegIndices(DefSubReg, SubReg);
+    return Copy;
+  }
+
+  bool isReg() const { return Kind == MachineOperand::MO_Register; }
+
+  Register getReg() const {
+    assert(isReg());
+    return OpToFold->getReg();
+  }
+
+  unsigned getSubReg() const {
+    assert(isReg());
+    return OpToFold->getSubReg();
   }
 
-  FoldCandidate(MachineInstr *MI, unsigned OpNo, int64_t FoldImm,
-                bool Commuted_ = false, int ShrinkOp = -1)
-      : UseMI(MI), ImmToFold(FoldImm), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
-        Kind(MachineOperand::MO_Immediate), Commuted(Commuted_) {}
+  bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
 
   bool isFI() const {
     return Kind == MachineOperand::MO_FrameIndex;
   }
 
-  bool isImm() const {
-    return Kind == MachineOperand::MO_Immediate;
+  int getFI() const {
+    assert(isFI());
+    return FrameIndexToFold;
   }
 
-  bool isReg() const {
-    return Kind == MachineOperand::MO_Register;
+  bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
+
+  /// Return the effective immediate value defined by this instruction, after
+  /// application of any subregister extracts which may exist between the use
+  /// and def instruction.
+  std::optional<int64_t> getEffectiveImmVal() const {
+    assert(isImm());
+    return SIInstrInfo::extractSubregFromImm(ImmToFold, DefSubReg);
+  }
+
+  /// Check if it is legal to fold this effective value into \p MI's \p OpNo
+  /// operand.
+  bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
+                      unsigned OpIdx) const {
+    switch (Kind) {
+    case MachineOperand::MO_Immediate: {
+      std::optional<int64_t> ImmToFold = getEffectiveImmVal();
+      if (!ImmToFold)
+        return false;
+
+      // TODO: Should verify the subregister index is supported by the class
+      // TODO: Avoid the temporary MachineOperand
+      MachineOperand TmpOp = MachineOperand::CreateImm(*ImmToFold);
+      return TII.isOperandLegal(MI, OpIdx, &TmpOp);
+    }
+    case MachineOperand::MO_FrameIndex: {
+      if (DefSubReg != AMDGPU::NoSubRegister)
+        return false;
+      MachineOperand TmpOp = MachineOperand::CreateFI(FrameIndexToFold);
+      return TII.isOperandLegal(MI, OpIdx, &TmpOp);
+    }
+    default:
+      // TODO: Try to apply DefSubReg, for global address we can extract
+      // low/high.
+      if (DefSubReg != AMDGPU::NoSubRegister)
+        return false;
+      return TII.isOperandLegal(MI, OpIdx, OpToFold);
+    }
+
+    llvm_unreachable("covered MachineOperand kind switch");
   }
+};
 
-  bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
+struct FoldCandidate {
+  MachineInstr *UseMI;
+  FoldableDef Def;
+  int ShrinkOpcode;
+  unsigned UseOpNo;
+  bool Commuted;
+
+  FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
+                bool Commuted = false, int ShrinkOp = -1)
+      : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
+        Commuted(Commuted) {}
+
+  bool isFI() const { return Def.isFI(); }
+
+  int getFI() const {
+    assert(isFI());
+    return Def.FrameIndexToFold;
+  }
+
+  bool isImm() const { return Def.isImm(); }
+
+  bool isReg() const { return Def.isReg(); }
+
+  Register getReg() const { return Def.getReg(); }
+
+  bool isGlobal() const { return Def.isGlobal(); }
 
   bool needsShrink() const { return ShrinkOpcode != -1; }
 };
@@ -89,7 +185,7 @@ class SIFoldOperandsImpl {
   const SIMachineFunctionInfo *MFI;
 
   bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
-                         const MachineOperand &OpToFold) const;
+                         const FoldableDef &OpToFold) const;
 
   // TODO: Just use TII::getVALUOp
   unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
@@ -115,13 +211,16 @@ class SIFoldOperandsImpl {
 
   bool updateOperand(FoldCandidate &Fold) const;
 
-  bool canUseImmWithOpSel(FoldCandidate &Fold) const;
+  bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
+                          int64_t ImmVal) const;
 
-  bool tryFoldImmWithOpSel(FoldCandidate &Fold) const;
+  /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
+  bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
+                           int64_t ImmVal) const;
 
   bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
                         MachineInstr *MI, unsigned OpNo,
-                        MachineOperand *OpToFold) const;
+                        const FoldableDef &OpToFold) const;
   bool isUseSafeToFold(const MachineInstr &MI,
                        const MachineOperand &UseMO) const;
 
@@ -133,19 +232,17 @@ class SIFoldOperandsImpl {
   getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
                 Register UseReg) const;
 
-  std::pair<MachineOperand *, const TargetRegisterClass *>
+  std::pair<int64_t, const TargetRegisterClass *>
   isRegSeqSplat(MachineInstr &RegSeg) const;
 
-  MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
-                                     MachineOperand *SplatVal,
-                                     const TargetRegisterClass *SplatRC) const;
+  bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+                          int64_t SplatVal,
+                          const TargetRegisterClass *SplatRC) const;
 
-  bool tryToFoldACImm(MachineOperand &OpToFold, MachineInstr *UseMI,
+  bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
                       unsigned UseOpIdx,
                       SmallVectorImpl<FoldCandidate> &FoldList) const;
-  void foldOperand(MachineOperand &OpToFold,
-                   MachineInstr *UseMI,
-                   int UseOpIdx,
+  void foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
                    SmallVectorImpl<FoldCandidate> &FoldList,
                    SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
 
@@ -153,7 +250,7 @@ class SIFoldOperandsImpl {
   bool tryConstantFoldOp(MachineInstr *MI) const;
   bool tryFoldCndMask(MachineInstr &MI) const;
   bool tryFoldZeroHighBits(MachineInstr &MI) const;
-  bool foldInstOperand(MachineInstr &MI, MachineOperand &OpToFold) const;
+  bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
 
   bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
   bool tryFoldFoldableCopy(MachineInstr &MI,
@@ -244,8 +341,8 @@ static unsigned macToMad(unsigned Opc) {
 
 // TODO: Add heuristic that the frame index might not fit in the addressing mode
 // immediate offset to avoid materializing in loops.
-bool SIFoldOperandsImpl::frameIndexMayFold(
-    const MachineInstr &UseMI, int OpNo, const MachineOperand &OpToFold) const {
+bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
+                                           const FoldableDef &OpToFold) const {
   if (!OpToFold.isFI())
     return false;
 
@@ -355,20 +452,20 @@ FunctionPass *llvm::createSIFoldOperandsLegacyPass() {
   return new SIFoldOperandsLegacy();
 }
 
-bool SIFoldOperandsImpl::canUseImmWithOpSel(FoldCandidate &Fold) const {
-  MachineInstr *MI = Fold.UseMI;
-  MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
+bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
+                                            unsigned UseOpNo,
+                                            int64_t ImmVal) const {
   const uint64_t TSFlags = MI->getDesc().TSFlags;
 
-  assert(Old.isReg() && Fold.isImm());
-
   if (!(TSFlags & SIInstrFlags::IsPacked) || (TSFlags & SIInstrFlags::IsMAI) ||
       (TSFlags & SIInstrFlags::IsWMMA) || (TSFlags & SIInstrFlags::IsSWMMAC) ||
       (ST->hasDOTOpSelHazard() && (TSFlags & SIInstrFlags::IsDOT)))
     return false;
 
-  unsigned Opcode = MI->getOpcode();
+  const MachineOperand &Old = MI->getOperand(UseOpNo);
   int OpNo = MI->getOperandNo(&Old);
+
+  unsigned Opcode = MI->getOpcode();
   uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
   switch (OpType) {
   default:
@@ -382,8 +479,7 @@ bool SIFoldOperandsImpl::canUseImmWithOpSel(FoldCandidate &Fold) const {
     // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
     // two different constants.
     if ((TSFlags & SIInstrFlags::VOP3) && !(TSFlags & SIInstrFlags::VOP3P) &&
-        static_cast<uint16_t>(Fold.ImmToFold) !=
-            static_cast<uint16_t>(Fold.ImmToFold >> 16))
+        static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
       return false;
     break;
   }
@@ -391,9 +487,9 @@ bool SIFoldOperandsImpl::canUseImmWithOpSel(FoldCandidate &Fold) const {
   return true;
 }
 
-bool SIFoldOperandsImpl::tryFoldImmWithOpSel(FoldCandidate &Fold) const {
-  MachineInstr *MI = Fold.UseMI;
-  MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
+bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
+                                             int64_t ImmVal) const {
+  MachineOperand &Old = MI->getOperand(UseOpNo);
   unsigned Opcode = MI->getOpcode();
   int OpNo = MI->getOperandNo(&Old);
   uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
@@ -401,8 +497,8 @@ bool SIFoldOperandsImpl::tryFoldImmWithOpSel(FoldCandidate &Fold) const {
   // If the literal can be inlined as-is, apply it and short-circuit the
   // tests below. The main motivation for this is to avoid unintuitive
   // uses of opsel.
-  if (AMDGPU::isInlinableLiteralV216(Fold.ImmToFold, OpType)) {
-    Old.ChangeToImmediate(Fold.ImmToFold);
+  if (AMDGPU::isInlinableLiteralV216(ImmVal, OpType)) {
+    Old.ChangeToImmediate(ImmVal);
     return true;
   }
 
@@ -425,10 +521,10 @@ bool SIFoldOperandsImpl::tryFoldImmWithOpSel(FoldCandidate &Fold) const {
   MachineOperand &Mod = MI->getOperand(ModIdx);
   unsigned ModVal = Mod.getImm();
 
-  uint16_t ImmLo = static_cast<uint16_t>(
-      Fold.ImmToFold >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
-  uint16_t ImmHi = static_cast<uint16_t>(
-      Fold.ImmToFold >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
+  uint16_t ImmLo =
+      static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
+  uint16_t ImmHi =
+      static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
   uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
   unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
 
@@ -519,17 +615,21 @@ bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
   MachineOperand &Old = MI->getOperand(Fold.UseOpNo);
   assert(Old.isReg());
 
-  if (Fold.isImm() && canUseImmWithOpSel(Fold)) {
-    if (tryFoldImmWithOpSel(Fold))
+  std::optional<int64_t> ImmVal;
+  if (Fold.isImm())
+    ImmVal = Fold.Def.getEffectiveImmVal();
+
+  if (ImmVal && canUseImmWithOpSel(Fold.UseMI, Fold.UseOpNo, *ImmVal)) {
+    if (tryFoldImmWithOpSel(Fold.UseMI, Fold.UseOpNo, *ImmVal))
       return true;
 
     // We can't represent the candidate as an inline constant. Try as a literal
     // with the original opsel, checking constant bus limitations.
-    MachineOperand New = MachineOperand::CreateImm(Fold.ImmToFold);
+    MachineOperand New = MachineOperand::CreateImm(*ImmVal);
     int OpNo = MI->getOperandNo(&Old);
     if (!TII->isOperandLegal(*MI, OpNo, &New))
       return false;
-    Old.ChangeToImmediate(Fold.ImmToFold);
+    Old.ChangeToImmediate(*ImmVal);
     return true;
   }
 
@@ -577,7 +677,7 @@ bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
 
   assert(!Fold.needsShrink() && "not handled");
 
-  if (Fold.isImm()) {
+  if (ImmVal) {
     if (Old.isTied()) {
       int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(MI->getOpcode());
       if (NewMFMAOpc == -1)
@@ -585,22 +685,30 @@ bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
       MI->setDesc(TII->get(NewMFMAOpc));
       MI->untieRegOperand(0);
     }
-    Old.ChangeToImmediate(Fold.ImmToFold);
+
+    // TODO: Should we try to avoid adding this to the candidate list?
+    MachineOperand New = MachineOperand::CreateImm(*ImmVal);
+    int OpNo = MI->getOperandNo(&Old);
+    if (!TII->isOperandLegal(*MI, OpNo, &New))
+      return false;
+
+    Old.ChangeToImmediate(*ImmVal);
     return true;
   }
 
   if (Fold.isGlobal()) {
-    Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(),
-                   Fold.OpToFold->getTargetFlags());
+    Old.ChangeToGA(Fold.Def.OpToFold->getGlobal(),
+                   Fold.Def.OpToFold->getOffset(),
+                   Fold.Def.OpToFold->getTargetFlags());
     return true;
   }
 
   if (Fold.isFI()) {
-    Old.ChangeToFrameIndex(Fold.FrameIndexToFold);
+    Old.ChangeToFrameIndex(Fold.getFI());
     return true;
   }
 
-  MachineOperand *New = Fold.OpToFold;
+  MachineOperand *New = Fold.Def.OpToFold;
   // Rework once the VS_16 register class is updated to include proper
   // 16-bit SGPRs instead of 32-bit ones.
   if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(*MRI, New->getReg()))
@@ -623,26 +731,19 @@ static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
 
 static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
                                 MachineInstr *MI, unsigned OpNo,
-                                MachineOperand *FoldOp, bool Commuted = false,
-                                int ShrinkOp = -1) {
-  appendFoldCandidate(FoldList,
-                      FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
-}
-
-static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
-                                MachineInstr *MI, unsigned OpNo, int64_t ImmVal,
+                                const FoldableDef &FoldOp,
                                 bool Commuted = false, int ShrinkOp = -1) {
   appendFoldCandidate(FoldList,
-                      FoldCandidate(MI, OpNo, ImmVal, Commuted, ShrinkOp));
+                      FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
 }
 
 bool SIFoldOperandsImpl::tryAddToFoldList(
     SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
-    MachineOperand *OpToFold) const {
+    const FoldableDef &OpToFold) const {
   const unsigned Opc = MI->getOpcode();
 
   auto tryToFoldAsFMAAKorMK = [&]() {
-    if (!OpToFold->isImm())
+    if (!OpToFold.isImm())
       return false;
 
     const bool TryAK = OpNo == 3;
@@ -675,10 +776,10 @@ bool SIFoldOperandsImpl::tryAddToFoldList(
     return false;
   };
 
-  bool IsLegal = TII->isOperandLegal(*MI, OpNo, OpToFold);
-  if (!IsLegal && OpToFold->isImm()) {
-    FoldCandidate Fold(MI, OpNo, OpToFold);
-    IsLegal = canUseImmWithOpSel(Fold);
+  bool IsLegal = OpToFold.isOperandLegal(*TII, *MI, OpNo);
+  if (!IsLegal && OpToFold.isImm()) {
+    if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
+      IsLegal = canUseImmWithOpSel(MI, OpNo, *ImmVal);
   }
 
   if (!IsLegal) {
@@ -710,7 +811,7 @@ bool SIFoldOperandsImpl::tryAddToFoldList(
     }
 
     // Special case for s_setreg_b32
-    if (OpToFold->isImm()) {
+    if (OpToFold.isImm()) {
       unsigned ImmOpc = 0;
       if (Opc == AMDGPU::S_SETREG_B32)
         ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
@@ -751,10 +852,10 @@ bool SIFoldOperandsImpl::tryAddToFoldList(
       return false;
 
     int Op32 = -1;
-    if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) {
+    if (!OpToFold.isOperandLegal(*TII, *MI, CommuteOpNo)) {
       if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
            Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
-          (!OpToFold->isImm() && !OpToFold->isFI() && !OpToFold->isGlobal())) {
+          (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
         TII->commuteInstruction(*MI, false, OpNo, CommuteOpNo);
         return false;
       }
@@ -773,7 +874,8 @@ bool SIFoldOperandsImpl::tryAddToFoldList(
       Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc);
     }
 
-    appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32);
+    appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, /*Commuted=*/true,
+                        Op32);
     return true;
   }
 
@@ -869,15 +971,15 @@ const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
   return getRegSeqInit(*Def, Defs);
 }
 
-std::pair<MachineOperand *, const TargetRegisterClass *>
+std::pair<int64_t, const TargetRegisterClass *>
 SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
   SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
   const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
   if (!SrcRC)
     return {};
 
-  // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
-  // every other other element is 0 for 64-bit immediates)
+  bool TryToMatchSplat64 = false;
+
   int64_t Imm;
   for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
     const MachineOperand *Op = Defs[I].first;
@@ -889,38 +991,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
       Imm = SubImm;
       continue;
     }
-    if (Imm != SubImm)
+
+    if (Imm != SubImm) {
+      if (I == 1 && (E & 1) == 0) {
+        // If we have an even number of inputs, there's a chance this is a
+        // 64-bit element splat broken into 32-bit pieces.
+        TryToMatchSplat64 = true;
+        break;
+      }
+
       return {}; // Can only fold splat constants
+    }
   }
 
-  return {Defs[0].first, SrcRC};
+  if (!TryToMatchSplat64)
+    return {Defs[0].first->getImm(), SrcRC};
+
+  // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+  // (i.e. recognize every other other element is 0 for 64-bit immediates)
+  int64_t SplatVal64;
+  for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+    const MachineOperand *Op0 = Defs[I].first;
+    const MachineOperand *Op1 = Defs[I + 1].first;
+
+    if (!Op0->isImm() || !Op1->isImm())
+      return {};
+
+    unsigned SubReg0 = Defs[I].second;
+    unsigned SubReg1 = Defs[I + 1].second;
+
+    // Assume we're going to generally encounter reg_sequences with sorted
+    // subreg indexes, so reject any that aren't consecutive.
+    if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+        TRI->getChannelFromSubReg(SubReg1))
+      return {};
+
+    int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+    if (I == 0)
+      SplatVal64 = MergedVal;
+    else if (SplatVal64 != MergedVal)
+      return {};
+  }
+
+  const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+      MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+  return {SplatVal64, RC64};
 }
 
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-    MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+    MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
     const TargetRegisterClass *SplatRC) const {
   const MCInstrDesc &Desc = UseMI->getDesc();
   if (UseOpIdx >= Desc.getNumOperands())
-    return nullptr;
+    return false;
 
   // Filter out unhandled pseudos.
   if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-    return nullptr;
+    return false;
 
   int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
   if (RCID == -1)
-    return nullptr;
+    return false;
+
+  const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
 
   // Special case 0/-1, since when interpreted as a 64-bit element both halves
-  // have the same bits. Effectively this code does not handle 64-bit element
-  // operands correctly, as the incoming 64-bit constants are already split into
-  // 32-bit sequence elements.
-  //
-  // TODO: We should try to figure out how to interpret the reg_sequence as a
-  // split 64-bit splat constant, or use 64-bit pseudos for materializing f64
-  // constants.
-  if (SplatVal->getImm() != 0 && SplatVal->getImm() != -1) {
-    const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
+  // have the same bits. These are the only cases where a splat has the same
+  // interpretation for 32-bit and 64-bit splats.
+  if (SplatVal != 0 && SplatVal != -1) {
     // We need to figure out the scalar type read by the operand. e.g. the MFMA
     // operand will be AReg_128, and we want to check if it's compatible with an
     // AReg_32 constant.
@@ -934,21 +1073,22 @@ MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
       OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
       break;
     default:
-      return nullptr;
+      return false;
     }
 
     if (!TRI->getCommonSubClass(OpRC, SplatRC))
-      return nullptr;
+      return false;
   }
 
-  if (!TII->isOperandLegal(*UseMI, UseOpIdx, SplatVal))
-    return nullptr;
+  MachineOperand TmpOp = MachineOperand::CreateImm(SplatVal);
+  if (!TII->isOperandLegal(*UseMI, UseOpIdx, &TmpOp))
+    return false;
 
-  return SplatVal;
+  return true;
 }
 
 bool SIFoldOperandsImpl::tryToFoldACImm(
-    MachineOperand &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
+    const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
     SmallVectorImpl<FoldCandidate> &FoldList) const {
   const MCInstrDesc &Desc = UseMI->getDesc();
   if (UseOpIdx >= Desc.getNumOperands())
@@ -958,27 +1098,9 @@ bool SIFoldOperandsImpl::tryToFoldACImm(
     return false;
 
   MachineOperand &UseOp = UseMI->getOperand(UseOpIdx);
-  if (OpToFold.isImm()) {
-    if (unsigned UseSubReg = UseOp.getSubReg()) {
-      std::optional<int64_t> SubImm =
-          SIInstrInfo::extractSubregFromImm(OpToFold.getImm(), UseSubReg);
-      if (!SubImm)
-        return false;
-
-      // TODO: Avoid the temporary MachineOperand
-      MachineOperand TmpOp = MachineOperand::CreateImm(*SubImm);
-      if (TII->isOperandLegal(*UseMI, UseOpIdx, &TmpOp)) {
-        appendFoldCandidate(FoldList, UseMI, UseOpIdx, *SubImm);
-        return true;
-      }
-
-      return false;
-    }
-
-    if (TII->isOperandLegal(*UseMI, UseOpIdx, &OpToFold)) {
-      appendFoldCandidate(FoldList, UseMI, UseOpIdx, &OpToFold);
-      return true;
-    }
+  if (OpToFold.isImm() && OpToFold.isOperandLegal(*TII, *UseMI, UseOpIdx)) {
+    appendFoldCandidate(FoldList, UseMI, UseOpIdx, OpToFold);
+    return true;
   }
 
   // TODO: Verify the following code handles subregisters correctly.
@@ -994,11 +1116,17 @@ bool SIFoldOperandsImpl::tryToFoldACImm(
     return false;
 
   // Maybe it is just a COPY of an immediate itself.
+
+  // FIXME: Remove this handling. There is already special case folding of
+  // immediate into copy in foldOperand. This is looking for the def of the
+  // value the folding started from in the first place.
   MachineInstr *Def = MRI->getVRegDef(UseReg);
   if (Def && TII->isFoldableCopy(*Def)) {
     MachineOperand &DefOp = Def->getOperand(1);
     if (DefOp.isImm() && TII->isOperandLegal(*UseMI, UseOpIdx, &DefOp)) {
-      appendFoldCandidate(FoldList, UseMI, UseOpIdx, &DefOp);
+      FoldableDef FoldableImm(DefOp.getImm(), OpToFold.DefRC,
+                              OpToFold.DefSubReg);
+      appendFoldCandidate(FoldList, UseMI, UseOpIdx, FoldableImm);
       return true;
     }
   }
@@ -1007,7 +1135,7 @@ bool SIFoldOperandsImpl::tryToFoldACImm(
 }
 
 void SIFoldOperandsImpl::foldOperand(
-    MachineOperand &OpToFold, MachineInstr *UseMI, int UseOpIdx,
+    FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
     SmallVectorImpl<FoldCandidate> &FoldList,
     SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
   const MachineOperand *UseOp = &UseMI->getOperand(UseOpIdx);
@@ -1033,7 +1161,7 @@ void SIFoldOperandsImpl::foldOperand(
     Register RegSeqDstReg = UseMI->getOperand(0).getReg();
     unsigned RegSeqDstSubReg = UseMI->getOperand(UseOpIdx + 1).getImm();
 
-    MachineOperand *SplatVal;
+    int64_t SplatVal;
     const TargetRegisterClass *SplatRC;
     std::tie(SplatVal, SplatRC) = isRegSeqSplat(*UseMI);
 
@@ -1044,21 +1172,22 @@ void SIFoldOperandsImpl::foldOperand(
       MachineInstr *RSUseMI = RSUse->getParent();
       unsigned OpNo = RSUseMI->getOperandNo(RSUse);
 
-      if (SplatVal) {
-        if (MachineOperand *Foldable =
-                tryFoldRegSeqSplat(RSUseMI, OpNo, SplatVal, SplatRC)) {
-          appendFoldCandidate(FoldList, RSUseMI, OpNo, Foldable);
+      if (SplatRC) {
+        if (tryFoldRegSeqSplat(RSUseMI, OpNo, SplatVal, SplatRC)) {
+          FoldableDef SplatDef(SplatVal, SplatRC);
+          appendFoldCandidate(FoldList, RSUseMI, OpNo, SplatDef);
           continue;
         }
       }
 
+      // TODO: Handle general compose
       if (RSUse->getSubReg() != RegSeqDstSubReg)
         continue;
 
-      if (tryToFoldACImm(UseMI->getOperand(0), RSUseMI, OpNo, FoldList))
-        continue;
-
-      foldOperand(OpToFold, RSUseMI, OpNo, FoldList, CopiesToReplace);
+      // FIXME: We should avoid recursing here. There should be a cleaner split
+      // between the in-place mutations and adding to the fold list.
+      foldOperand(OpToFold, RSUseMI, RSUseMI->getOperandNo(RSUse), FoldList,
+                  CopiesToReplace);
     }
 
     return;
@@ -1086,7 +1215,7 @@ void SIFoldOperandsImpl::foldOperand(
 
     // A frame index will resolve to a positive constant, so it should always be
     // safe to fold the addressing mode, even pre-GFX9.
-    UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex());
+    UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getFI());
 
     const unsigned Opc = UseMI->getOpcode();
     if (TII->isFLATScratch(*UseMI) &&
@@ -1116,11 +1245,12 @@ void SIFoldOperandsImpl::foldOperand(
       return;
 
     const TargetRegisterClass *DestRC = TRI->getRegClassForReg(*MRI, DestReg);
-    if (!DestReg.isPhysical()) {
-      if (DestRC == &AMDGPU::AGPR_32RegClass &&
-          TII->isInlineConstant(OpToFold, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
+    if (!DestReg.isPhysical() && DestRC == &AMDGPU::AGPR_32RegClass) {
+      std::optional<int64_t> UseImmVal = OpToFold.getEffectiveImmVal();
+      if (UseImmVal && TII->isInlineConstant(
+                           *UseImmVal, AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
         UseMI->setDesc(TII->get(AMDGPU::V_ACCVGPR_WRITE_B32_e64));
-        UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
+        UseMI->getOperand(1).ChangeToImmediate(*UseImmVal);
         CopiesToReplace.push_back(UseMI);
         return;
       }
@@ -1176,8 +1306,9 @@ void SIFoldOperandsImpl::foldOperand(
     if (UseMI->isCopy() && OpToFold.isReg() &&
         UseMI->getOperand(0).getReg().isVirtual() &&
         !UseMI->getOperand(1).getSubReg() &&
-        OpToFold.getParent()->implicit_operands().empty()) {
-      LLVM_DEBUG(dbgs() << "Folding " << OpToFold << "\n into " << *UseMI);
+        OpToFold.DefMI->implicit_operands().empty()) {
+      LLVM_DEBUG(dbgs() << "Folding " << OpToFold.OpToFold << "\n into "
+                        << *UseMI);
       unsigned Size = TII->getOpSize(*UseMI, 1);
       Register UseReg = OpToFold.getReg();
       UseMI->getOperand(1).setReg(UseReg);
@@ -1217,7 +1348,7 @@ void SIFoldOperandsImpl::foldOperand(
       UseMI->getOperand(1).setSubReg(SubRegIdx);
       UseMI->getOperand(1).setIsKill(false);
       CopiesToReplace.push_back(UseMI);
-      OpToFold.setIsKill(false);
+      OpToFold.OpToFold->setIsKill(false);
 
       // Remove kill flags as kills may now be out of order with uses.
       MRI->clearKillFlags(UseReg);
@@ -1237,21 +1368,21 @@ void SIFoldOperandsImpl::foldOperand(
       if (FoldingImmLike) {
         if (execMayBeModifiedBeforeUse(*MRI,
                                        UseMI->getOperand(UseOpIdx).getReg(),
-                                       *OpToFold.getParent(),
-                                       *UseMI))
+                                       *OpToFold.DefMI, *UseMI))
           return;
 
         UseMI->setDesc(TII->get(AMDGPU::S_MOV_B32));
 
-        if (OpToFold.isImm())
-          UseMI->getOperand(1).ChangeToImmediate(OpToFold.getImm());
-        else if (OpToFold.isFI())
-          UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getIndex());
+        if (OpToFold.isImm()) {
+          UseMI->getOperand(1).ChangeToImmediate(
+              *OpToFold.getEffectiveImmVal());
+        } else if (OpToFold.isFI())
+          UseMI->getOperand(1).ChangeToFrameIndex(OpToFold.getFI());
         else {
           assert(OpToFold.isGlobal());
-          UseMI->getOperand(1).ChangeToGA(OpToFold.getGlobal(),
-                                          OpToFold.getOffset(),
-                                          OpToFold.getTargetFlags());
+          UseMI->getOperand(1).ChangeToGA(OpToFold.OpToFold->getGlobal(),
+                                          OpToFold.OpToFold->getOffset(),
+                                          OpToFold.OpToFold->getTargetFlags());
         }
         UseMI->removeOperand(2); // Remove exec read (or src1 for readlane)
         return;
@@ -1260,7 +1391,7 @@ void SIFoldOperandsImpl::foldOperand(
       if (OpToFold.isReg() && TRI->isSGPRReg(*MRI, OpToFold.getReg())) {
         if (checkIfExecMayBeModifiedBeforeUseAcrossBB(
                 *MRI, UseMI->getOperand(UseOpIdx).getReg(),
-                *OpToFold.getParent(), *UseMI, SIFoldOperandsPreheaderThreshold))
+                *OpToFold.DefMI, *UseMI, SIFoldOperandsPreheaderThreshold))
           return;
 
         // %vgpr = COPY %sgpr0
@@ -1302,7 +1433,7 @@ void SIFoldOperandsImpl::foldOperand(
         return;
     }
 
-    tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
+    tryAddToFoldList(FoldList, UseMI, UseOpIdx, OpToFold);
 
     // FIXME: We could try to change the instruction from 64-bit to 32-bit
     // to enable more folding opportunities.  The shrink operands pass
@@ -1310,32 +1441,7 @@ void SIFoldOperandsImpl::foldOperand(
     return;
   }
 
-
-  const MCInstrDesc &FoldDesc = OpToFold.getParent()->getDesc();
-  const TargetRegisterClass *FoldRC =
-      TRI->getRegClass(FoldDesc.operands()[0].RegClass);
-
-  // Split 64-bit constants into 32-bits for folding.
-  if (UseOp->getSubReg() && AMDGPU::getRegBitWidth(*FoldRC) == 64) {
-    Register UseReg = UseOp->getReg();
-    const TargetRegisterClass *UseRC = MRI->getRegClass(UseReg);
-    if (AMDGPU::getRegBitWidth(*UseRC) != 64)
-      return;
-
-    APInt Imm(64, OpToFold.getImm());
-    if (UseOp->getSubReg() == AMDGPU::sub0) {
-      Imm = Imm.getLoBits(32);
-    } else {
-      assert(UseOp->getSubReg() == AMDGPU::sub1);
-      Imm = Imm.getHiBits(32);
-    }
-
-    MachineOperand ImmOp = MachineOperand::CreateImm(Imm.getSExtValue());
-    tryAddToFoldList(FoldList, UseMI, UseOpIdx, &ImmOp);
-    return;
-  }
-
-  tryAddToFoldList(FoldList, UseMI, UseOpIdx, &OpToFold);
+  tryAddToFoldList(FoldList, UseMI, UseOpIdx, OpToFold);
 }
 
 static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
@@ -1614,7 +1720,7 @@ bool SIFoldOperandsImpl::tryFoldZeroHighBits(MachineInstr &MI) const {
 }
 
 bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
-                                         MachineOperand &OpToFold) const {
+                                         const FoldableDef &OpToFold) const {
   // We need mutate the operands of new mov instructions to add implicit
   // uses of EXEC, but adding them invalidates the use_iterator, so defer
   // this.
@@ -1645,7 +1751,9 @@ bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
       llvm::make_pointer_range(MRI->use_nodbg_operands(Dst.getReg())));
   for (auto *U : UsesToProcess) {
     MachineInstr *UseMI = U->getParent();
-    foldOperand(OpToFold, UseMI, UseMI->getOperandNo(U), FoldList,
+
+    FoldableDef SubOpToFold = OpToFold.getWithSubReg(*TRI, U->getSubReg());
+    foldOperand(SubOpToFold, UseMI, UseMI->getOperandNo(U), FoldList,
                 CopiesToReplace);
   }
 
@@ -1658,10 +1766,10 @@ bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
     Copy->addImplicitDefUseOperands(*MF);
 
   for (FoldCandidate &Fold : FoldList) {
-    assert(!Fold.isReg() || Fold.OpToFold);
-    if (Fold.isReg() && Fold.OpToFold->getReg().isVirtual()) {
-      Register Reg = Fold.OpToFold->getReg();
-      MachineInstr *DefMI = Fold.OpToFold->getParent();
+    assert(!Fold.isReg() || Fold.Def.OpToFold);
+    if (Fold.isReg() && Fold.getReg().isVirtual()) {
+      Register Reg = Fold.getReg();
+      const MachineInstr *DefMI = Fold.Def.DefMI;
       if (DefMI->readsRegister(AMDGPU::EXEC, TRI) &&
           execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI))
         continue;
@@ -1669,11 +1777,11 @@ bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
     if (updateOperand(Fold)) {
       // Clear kill flags.
       if (Fold.isReg()) {
-        assert(Fold.OpToFold && Fold.OpToFold->isReg());
+        assert(Fold.Def.OpToFold && Fold.isReg());
         // FIXME: Probably shouldn't bother trying to fold if not an
         // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
         // copies.
-        MRI->clearKillFlags(Fold.OpToFold->getReg());
+        MRI->clearKillFlags(Fold.getReg());
       }
       LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
                         << static_cast<int>(Fold.UseOpNo) << " of "
@@ -1861,6 +1969,18 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
   if (OpToFold.isReg() && !OpToFold.getReg().isVirtual())
     return false;
 
+  // Prevent folding operands backwards in the function. For example,
+  // the COPY opcode must not be replaced by 1 in this example:
+  //
+  //    %3 = COPY %vgpr0; VGPR_32:%3
+  //    ...
+  //    %vgpr0 = V_MOV_B32_e32 1, implicit %exec
+  if (!DstReg.isVirtual())
+    return false;
+
+  const TargetRegisterClass *DstRC =
+      MRI->getRegClass(MI.getOperand(0).getReg());
+
   // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
   // Can remove this code if proper 16-bit SGPRs are implemented
   // Example: Pre-peephole-opt
@@ -1877,8 +1997,6 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
   // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
   if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
       OpToFold.getSubReg()) {
-    const TargetRegisterClass *DstRC =
-        MRI->getRegClass(MI.getOperand(0).getReg());
     if (DstRC == &AMDGPU::SReg_32RegClass &&
         DstRC == MRI->getRegClass(OpToFold.getReg())) {
       assert(OpToFold.getSubReg() == AMDGPU::lo16);
@@ -1886,15 +2004,6 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
     }
   }
 
-  // Prevent folding operands backwards in the function. For example,
-  // the COPY opcode must not be replaced by 1 in this example:
-  //
-  //    %3 = COPY %vgpr0; VGPR_32:%3
-  //    ...
-  //    %vgpr0 = V_MOV_B32_e32 1, implicit %exec
-  if (!DstReg.isVirtual())
-    return false;
-
   // Fold copy to AGPR through reg_sequence
   // TODO: Handle with subregister extract
   if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(1).getSubReg()) {
@@ -1902,7 +2011,8 @@ bool SIFoldOperandsImpl::tryFoldFoldableCopy(
       return true;
   }
 
-  bool Changed = foldInstOperand(MI, OpToFold);
+  FoldableDef Def(OpToFold, DstRC);
+  bool Changed = foldInstOperand(MI, Def);
 
   // If we managed to fold all uses of this copy then we might as well
   // delete it now.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 17c7fb7bb153..b083a9014737 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -945,6 +945,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   }
 
   setTargetDAGCombine({ISD::ADD,
+                       ISD::PTRADD,
                        ISD::UADDO_CARRY,
                        ISD::SUB,
                        ISD::USUBO_CARRY,
@@ -1403,6 +1404,19 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
     return true;
   }
+  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
+    Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
+                   ? ISD::INTRINSIC_W_CHAIN
+                   : ISD::INTRINSIC_VOID;
+    Info.memVT = MVT::getVT(CI.getType());
+    Info.ptrVal = CI.getOperand(0);
+    Info.memVT = MVT::i64;
+    Info.size = 8;
+    Info.align.reset();
+    Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+    return true;
+  }
   case Intrinsic::amdgcn_global_atomic_csub: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::getVT(CI.getType());
@@ -1443,6 +1457,12 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                   MachineMemOperand::MOVolatile;
     return true;
   }
+  case Intrinsic::amdgcn_ds_load_tr6_b96:
+  case Intrinsic::amdgcn_ds_load_tr4_b64:
+  case Intrinsic::amdgcn_ds_load_tr8_b64:
+  case Intrinsic::amdgcn_ds_load_tr16_b128:
+  case Intrinsic::amdgcn_global_load_tr6_b96:
+  case Intrinsic::amdgcn_global_load_tr4_b64:
   case Intrinsic::amdgcn_global_load_tr_b64:
   case Intrinsic::amdgcn_global_load_tr_b128:
   case Intrinsic::amdgcn_ds_read_tr4_b64:
@@ -1547,12 +1567,18 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
   case Intrinsic::amdgcn_atomic_cond_sub_u32:
   case Intrinsic::amdgcn_ds_append:
   case Intrinsic::amdgcn_ds_consume:
+  case Intrinsic::amdgcn_ds_load_tr8_b64:
+  case Intrinsic::amdgcn_ds_load_tr16_b128:
+  case Intrinsic::amdgcn_ds_load_tr4_b64:
+  case Intrinsic::amdgcn_ds_load_tr6_b96:
   case Intrinsic::amdgcn_ds_read_tr4_b64:
   case Intrinsic::amdgcn_ds_read_tr6_b96:
   case Intrinsic::amdgcn_ds_read_tr8_b64:
   case Intrinsic::amdgcn_ds_read_tr16_b64:
   case Intrinsic::amdgcn_ds_ordered_add:
   case Intrinsic::amdgcn_ds_ordered_swap:
+  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
+  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
   case Intrinsic::amdgcn_flat_atomic_fmax_num:
   case Intrinsic::amdgcn_flat_atomic_fmin_num:
   case Intrinsic::amdgcn_global_atomic_csub:
@@ -1561,6 +1587,8 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
   case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
   case Intrinsic::amdgcn_global_load_tr_b64:
   case Intrinsic::amdgcn_global_load_tr_b128:
+  case Intrinsic::amdgcn_global_load_tr4_b64:
+  case Intrinsic::amdgcn_global_load_tr6_b96:
     Ptr = II->getArgOperand(0);
     break;
   case Intrinsic::amdgcn_load_to_lds:
@@ -5959,35 +5987,6 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
   return false;
 }
 
-// Refer to comments added to the MIR variant of isFMAFasterThanFMulAndFAdd for
-// specific details.
-bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
-                                                  Type *Ty) const {
-  switch (Ty->getScalarSizeInBits()) {
-  case 16: {
-    SIModeRegisterDefaults Mode = SIModeRegisterDefaults(F, *Subtarget);
-    return Subtarget->has16BitInsts() &&
-           Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();
-  }
-  case 32: {
-    if (!Subtarget->hasMadMacF32Insts())
-      return Subtarget->hasFastFMAF32();
-
-    SIModeRegisterDefaults Mode = SIModeRegisterDefaults(F, *Subtarget);
-    if (Mode.FP32Denormals != DenormalMode::getPreserveSign())
-      return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
-
-    return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
-  }
-  case 64:
-    return true;
-  default:
-    break;
-  }
-
-  return false;
-}
-
 bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
   if (!Ty.isScalar())
     return false;
@@ -8765,11 +8764,15 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
                     : False);
   if (IsGFX10Plus)
     Ops.push_back(IsA16 ? True : False);
-  if (!Subtarget->hasGFX90AInsts()) {
+
+  if (!Subtarget->hasGFX90AInsts())
     Ops.push_back(TFE); // tfe
-  } else if (TFE->getAsZExtVal()) {
-    report_fatal_error("TFE is not supported on this GPU");
+  else if (TFE->getAsZExtVal()) {
+    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+        DAG.getMachineFunction().getFunction(),
+        "TFE is not supported on this GPU", DL.getDebugLoc()));
   }
+
   if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
     Ops.push_back(LWE); // lwe
   if (!IsGFX10Plus)
@@ -8800,9 +8803,23 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     if (Subtarget->hasGFX90AInsts()) {
       Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a,
                                      NumVDataDwords, NumVAddrDwords);
-      if (Opcode == -1)
-        report_fatal_error(
-            "requested image instruction is not supported on this GPU");
+      if (Opcode == -1) {
+        DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+            DAG.getMachineFunction().getFunction(),
+            "requested image instruction is not supported on this GPU",
+            DL.getDebugLoc()));
+
+        unsigned Idx = 0;
+        SmallVector<SDValue, 3> RetValues(OrigResultTypes.size());
+        for (EVT VT : OrigResultTypes) {
+          if (VT == MVT::Other)
+            RetValues[Idx++] = Op.getOperand(0); // Chain
+          else
+            RetValues[Idx++] = DAG.getPOISON(VT);
+        }
+
+        return DAG.getMergeValues(RetValues, DL);
+      }
     }
     if (Opcode == -1 &&
         Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
@@ -15095,6 +15112,49 @@ SDValue SITargetLowering::performAddCombine(SDNode *N,
   return SDValue();
 }
 
+SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
+                                               DAGCombinerInfo &DCI) const {
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  if (N1.getOpcode() == ISD::ADD) {
+    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
+    //    y is not, and (add y, z) is used only once.
+    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
+    //    z is not, and (add y, z) is used only once.
+    // The goal is to move constant offsets to the outermost ptradd, to create
+    // more opportunities to fold offsets into memory instructions.
+    // Together with the generic combines in DAGCombiner.cpp, this also
+    // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
+    //
+    // This transform is here instead of in the general DAGCombiner as it can
+    // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for
+    // AArch64's CPA.
+    SDValue X = N0;
+    SDValue Y = N1.getOperand(0);
+    SDValue Z = N1.getOperand(1);
+    if (N1.hasOneUse()) {
+      bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
+      bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
+      if (ZIsConstant != YIsConstant) {
+        // If both additions in the original were NUW, the new ones are as well.
+        SDNodeFlags Flags =
+            (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
+        if (YIsConstant)
+          std::swap(Y, Z);
+
+        SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags);
+        DCI.AddToWorklist(Inner.getNode());
+        return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags);
+      }
+    }
+  }
+
+  return SDValue();
+}
+
 SDValue SITargetLowering::performSubCombine(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -15633,6 +15693,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   switch (N->getOpcode()) {
   case ISD::ADD:
     return performAddCombine(N, DCI);
+  case ISD::PTRADD:
+    return performPtrAddCombine(N, DCI);
   case ISD::SUB:
     return performSubCombine(N, DCI);
   case ISD::UADDO_CARRY:
@@ -17621,33 +17683,6 @@ bool SITargetLowering::checkForPhysRegDependency(
   return false;
 }
 
-/// Check if it is profitable to hoist instruction in then/else to if.
-bool SITargetLowering::isProfitableToHoist(Instruction *I) const {
-  if (!I->hasOneUse())
-    return true;
-
-  Instruction *User = I->user_back();
-  // TODO: Add more patterns that are not profitable to hoist and
-  // handle modifiers such as fabs and fneg
-  switch (I->getOpcode()) {
-  case Instruction::FMul: {
-    if (User->getOpcode() != Instruction::FSub &&
-        User->getOpcode() != Instruction::FAdd)
-      return true;
-
-    const TargetOptions &Options = getTargetMachine().Options;
-
-    return ((!I->hasAllowContract() || !User->hasAllowContract()) &&
-            Options.AllowFPOpFusion != FPOpFusion::Fast &&
-            !Options.UnsafeFPMath) ||
-           !isFMAFasterThanFMulAndFAdd(*I->getFunction(), User->getType());
-  }
-  default:
-    return true;
-  }
-  return true;
-}
-
 void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
     Instruction *AI) const {
   // Given: atomicrmw fadd ptr %addr, float %val ordering
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 89fb12b52c3e..c66f300ec4cb 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -220,6 +220,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                           DAGCombinerInfo &DCI) const;
 
   SDValue performAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue performPtrAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performAddCarrySubCarryCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue performFAddCombine(SDNode *N, DAGCombinerInfo &DCI) const;
@@ -468,7 +469,6 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                   EVT VT) const override;
   bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                   const LLT Ty) const override;
-  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
   bool isFMADLegal(const SelectionDAG &DAG, const SDNode *N) const override;
   bool isFMADLegal(const MachineInstr &MI, const LLT Ty) const override;
 
@@ -549,8 +549,6 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                  const TargetInstrInfo *TII,
                                  MCRegister &PhysReg, int &Cost) const override;
 
-  bool isProfitableToHoist(Instruction *I) const override;
-
   bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
                                     const SelectionDAG &DAG, bool SNaN = false,
                                     unsigned Depth = 0) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 9a7dd3c31e49..7ce1359f03da 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -73,6 +73,7 @@ enum InstCounterType {
   SAMPLE_CNT = NUM_NORMAL_INST_CNTS, // gfx12+ only.
   BVH_CNT,                           // gfx12+ only.
   KM_CNT,                            // gfx12+ only.
+  X_CNT,                             // gfx1250.
   NUM_EXTENDED_INST_CNTS,
   NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
 };
@@ -102,6 +103,7 @@ struct HardwareLimits {
   unsigned SamplecntMax; // gfx12+ only.
   unsigned BvhcntMax;    // gfx12+ only.
   unsigned KmcntMax;     // gfx12+ only.
+  unsigned XcntMax;      // gfx1250.
 };
 
 #define AMDGPU_DECLARE_WAIT_EVENTS(DECL)                                       \
@@ -111,10 +113,12 @@ struct HardwareLimits {
   DECL(VMEM_BVH_READ_ACCESS)     /* vmem BVH read (gfx12+ only) */             \
   DECL(VMEM_WRITE_ACCESS)        /* vmem write that is not scratch */          \
   DECL(SCRATCH_WRITE_ACCESS)     /* vmem write that may be scratch */          \
+  DECL(VMEM_GROUP)               /* vmem group */                              \
   DECL(LDS_ACCESS)               /* lds read & write */                        \
   DECL(GDS_ACCESS)               /* gds read & write */                        \
   DECL(SQ_MESSAGE)               /* send message */                            \
   DECL(SMEM_ACCESS)              /* scalar-memory read & write */              \
+  DECL(SMEM_GROUP)               /* scalar-memory group */                     \
   DECL(EXP_GPR_LOCK)             /* export holding on its data src */          \
   DECL(GDS_GPR_LOCK)             /* GDS holding on its data and addr src */    \
   DECL(EXP_POS_ACCESS)           /* write to export position */                \
@@ -147,14 +151,14 @@ enum RegisterMapping {
   SQ_MAX_PGM_VGPRS = 1024, // Maximum programmable VGPRs across all targets.
   AGPR_OFFSET = 512,       // Maximum programmable ArchVGPRs across all targets.
   SQ_MAX_PGM_SGPRS = 128,  // Maximum programmable SGPRs across all targets.
-  NUM_EXTRA_VGPRS = 9,     // Reserved slots for DS.
   // Artificial register slots to track LDS writes into specific LDS locations
   // if a location is known. When slots are exhausted or location is
   // unknown use the first slot. The first slot is also always updated in
   // addition to known location's slot to properly generate waits if dependent
   // instruction's location is unknown.
-  EXTRA_VGPR_LDS = 0,
-  NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS, // Where SGPR starts.
+  FIRST_LDS_VGPR = SQ_MAX_PGM_VGPRS, // Extra slots for LDS stores.
+  NUM_LDS_VGPRS = 9,                 // One more than the stores we track.
+  NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_LDS_VGPRS, // Where SGPRs start.
 };
 
 // Enumerate different types of result-returning VMEM operations. Although
@@ -178,7 +182,7 @@ enum VmemType {
 static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
     AMDGPU::S_WAIT_LOADCNT,  AMDGPU::S_WAIT_DSCNT,     AMDGPU::S_WAIT_EXPCNT,
     AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
-    AMDGPU::S_WAIT_KMCNT};
+    AMDGPU::S_WAIT_KMCNT,    AMDGPU::S_WAIT_XCNT};
 
 static bool updateVMCntOnly(const MachineInstr &Inst) {
   return (SIInstrInfo::isVMEM(Inst) && !SIInstrInfo::isFLAT(Inst)) ||
@@ -193,18 +197,22 @@ static bool isNormalMode(InstCounterType MaxCounter) {
 
 VmemType getVmemType(const MachineInstr &Inst) {
   assert(updateVMCntOnly(Inst));
-  if (!SIInstrInfo::isMIMG(Inst) && !SIInstrInfo::isVIMAGE(Inst) &&
-      !SIInstrInfo::isVSAMPLE(Inst))
+  if (!SIInstrInfo::isImage(Inst))
     return VMEM_NOSAMPLER;
   const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Inst.getOpcode());
   const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
       AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
+
+  if (BaseInfo->BVH)
+    return VMEM_BVH;
+
   // We have to make an additional check for isVSAMPLE here since some
   // instructions don't have a sampler, but are still classified as sampler
   // instructions for the purposes of e.g. waitcnt.
-  bool HasSampler =
-      BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(Inst);
-  return BaseInfo->BVH ? VMEM_BVH : HasSampler ? VMEM_SAMPLER : VMEM_NOSAMPLER;
+  if (BaseInfo->Sampler || BaseInfo->MSAA || SIInstrInfo::isVSAMPLE(Inst))
+    return VMEM_SAMPLER;
+
+  return VMEM_NOSAMPLER;
 }
 
 unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) {
@@ -223,6 +231,8 @@ unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) {
     return Wait.BvhCnt;
   case KM_CNT:
     return Wait.KmCnt;
+  case X_CNT:
+    return Wait.XCnt;
   default:
     llvm_unreachable("bad InstCounterType");
   }
@@ -283,12 +293,23 @@ class WaitcntBrackets {
       return Limits.BvhcntMax;
     case KM_CNT:
       return Limits.KmcntMax;
+    case X_CNT:
+      return Limits.XcntMax;
     default:
       break;
     }
     return 0;
   }
 
+  bool isSmemCounter(InstCounterType T) const {
+    return T == SmemAccessCounter || T == X_CNT;
+  }
+
+  unsigned getSgprScoresIdx(InstCounterType T) const {
+    assert(isSmemCounter(T) && "Invalid SMEM counter");
+    return T == X_CNT ? 1 : 0;
+  }
+
   unsigned getScoreLB(InstCounterType T) const {
     assert(T < NUM_INST_CNTS);
     return ScoreLBs[T];
@@ -304,11 +325,9 @@ class WaitcntBrackets {
   }
 
   unsigned getRegScore(int GprNo, InstCounterType T) const {
-    if (GprNo < NUM_ALL_VGPRS) {
+    if (GprNo < NUM_ALL_VGPRS)
       return VgprScores[T][GprNo];
-    }
-    assert(T == SmemAccessCounter);
-    return SgprScores[GprNo - NUM_ALL_VGPRS];
+    return SgprScores[getSgprScoresIdx(T)][GprNo - NUM_ALL_VGPRS];
   }
 
   bool merge(const WaitcntBrackets &Other);
@@ -331,6 +350,7 @@ class WaitcntBrackets {
 
   void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
   void applyWaitcnt(InstCounterType T, unsigned Count);
+  void applyXcnt(const AMDGPU::Waitcnt &Wait);
   void updateByEvent(const SIInstrInfo *TII, const SIRegisterInfo *TRI,
                      const MachineRegisterInfo *MRI, WaitEventType E,
                      MachineInstr &MI);
@@ -462,15 +482,17 @@ class WaitcntBrackets {
   int VgprUB = -1;
   int SgprUB = -1;
   unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
-  // Wait cnt scores for every sgpr, only DS_CNT (corresponding to LGKMcnt
-  // pre-gfx12) or KM_CNT (gfx12+ only) are relevant.
-  unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
+  // Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
+  // pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
+  // Row 0 represents the score for either DS_CNT or KM_CNT and row 1 keeps the
+  // X_CNT score.
+  unsigned SgprScores[2][SQ_MAX_PGM_SGPRS] = {{0}};
   // Bitmask of the VmemTypes of VMEM instructions that might have a pending
   // write to each vgpr.
   unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
   // Store representative LDS DMA operations. The only useful info here is
   // alias info. One store is kept per unique AAInfo.
-  SmallVector<const MachineInstr *, NUM_EXTRA_VGPRS - 1> LDSDMAStores;
+  SmallVector<const MachineInstr *, NUM_LDS_VGPRS - 1> LDSDMAStores;
 };
 
 // This abstracts the logic for generating and updating S_WAIT* instructions
@@ -572,6 +594,7 @@ class WaitcntGeneratorPreGFX12 : public WaitcntGenerator {
         eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
         0,
         0,
+        0,
         0};
 
     return WaitEventMaskForInstPreGFX12;
@@ -607,7 +630,8 @@ class WaitcntGeneratorGFX12Plus : public WaitcntGenerator {
         eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
         eventMask({VMEM_SAMPLER_READ_ACCESS}),
         eventMask({VMEM_BVH_READ_ACCESS}),
-        eventMask({SMEM_ACCESS, SQ_MESSAGE})};
+        eventMask({SMEM_ACCESS, SQ_MESSAGE}),
+        eventMask({VMEM_GROUP, SMEM_GROUP})};
 
     return WaitEventMaskForInstGFX12Plus;
   }
@@ -743,9 +767,12 @@ class SIInsertWaitcnts {
     return VmemReadMapping[getVmemType(Inst)];
   }
 
+  bool hasXcnt() const { return ST->hasWaitXCnt(); }
+
   bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
   bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
   bool mayAccessScratchThroughFlat(const MachineInstr &MI) const;
+  bool isVmemAccess(const MachineInstr &MI) const;
   bool generateWaitcntInstBefore(MachineInstr &MI,
                                  WaitcntBrackets &ScoreBrackets,
                                  MachineInstr *OldWaitcntInstr,
@@ -837,9 +864,8 @@ void WaitcntBrackets::setScoreByInterval(RegInterval Interval,
       VgprUB = std::max(VgprUB, RegNo);
       VgprScores[CntTy][RegNo] = Score;
     } else {
-      assert(CntTy == SmemAccessCounter);
       SgprUB = std::max(SgprUB, RegNo - NUM_ALL_VGPRS);
-      SgprScores[RegNo - NUM_ALL_VGPRS] = Score;
+      SgprScores[getSgprScoresIdx(CntTy)][RegNo - NUM_ALL_VGPRS] = Score;
     }
   }
 }
@@ -976,6 +1002,9 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
           setScoreByOperand(&Inst, TRI, MRI, Op, EXP_CNT, CurrScore);
       }
     }
+  } else if (T == X_CNT) {
+    for (const MachineOperand &Op : Inst.all_uses())
+      setScoreByOperand(&Inst, TRI, MRI, Op, T, CurrScore);
   } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
     // Match the score to the destination registers.
     //
@@ -1037,15 +1066,15 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
             }
           }
         }
-        if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
+        if (Slot || LDSDMAStores.size() == NUM_LDS_VGPRS - 1)
           break;
         LDSDMAStores.push_back(&Inst);
         Slot = LDSDMAStores.size();
         break;
       }
-      setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot, T, CurrScore);
+      setRegScore(FIRST_LDS_VGPR + Slot, T, CurrScore);
       if (Slot)
-        setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS, T, CurrScore);
+        setRegScore(FIRST_LDS_VGPR, T, CurrScore);
     }
   }
 }
@@ -1080,6 +1109,9 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
     case KM_CNT:
       OS << "    KM_CNT(" << SR << "): ";
       break;
+    case X_CNT:
+      OS << "    X_CNT(" << SR << "): ";
+      break;
     default:
       OS << "    UNKNOWN(" << SR << "): ";
       break;
@@ -1094,14 +1126,14 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
         if (RegScore <= LB)
           continue;
         unsigned RelScore = RegScore - LB - 1;
-        if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
+        if (J < FIRST_LDS_VGPR) {
           OS << RelScore << ":v" << J << " ";
         } else {
           OS << RelScore << ":ds ";
         }
       }
-      // Also need to print sgpr scores for lgkm_cnt.
-      if (T == SmemAccessCounter) {
+      // Also need to print sgpr scores for lgkm_cnt or xcnt.
+      if (isSmemCounter(T)) {
         for (int J = 0; J <= SgprUB; J++) {
           unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS, T);
           if (RegScore <= LB)
@@ -1140,6 +1172,7 @@ void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
   simplifyWaitcnt(SAMPLE_CNT, Wait.SampleCnt);
   simplifyWaitcnt(BVH_CNT, Wait.BvhCnt);
   simplifyWaitcnt(KM_CNT, Wait.KmCnt);
+  simplifyWaitcnt(X_CNT, Wait.XCnt);
 }
 
 void WaitcntBrackets::simplifyWaitcnt(InstCounterType T,
@@ -1191,6 +1224,7 @@ void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
   applyWaitcnt(SAMPLE_CNT, Wait.SampleCnt);
   applyWaitcnt(BVH_CNT, Wait.BvhCnt);
   applyWaitcnt(KM_CNT, Wait.KmCnt);
+  applyXcnt(Wait);
 }
 
 void WaitcntBrackets::applyWaitcnt(InstCounterType T, unsigned Count) {
@@ -1207,11 +1241,29 @@ void WaitcntBrackets::applyWaitcnt(InstCounterType T, unsigned Count) {
   }
 }
 
+void WaitcntBrackets::applyXcnt(const AMDGPU::Waitcnt &Wait) {
+  // Wait on XCNT is redundant if we are already waiting for a load to complete.
+  // SMEM can return out of order, so only omit XCNT wait if we are waiting till
+  // zero.
+  if (Wait.KmCnt == 0 && hasPendingEvent(SMEM_GROUP))
+    return applyWaitcnt(X_CNT, 0);
+
+  // If we have pending store we cannot optimize XCnt because we do not wait for
+  // stores. VMEM loads retun in order, so if we only have loads XCnt is
+  // decremented to the same number as LOADCnt.
+  if (Wait.LoadCnt != ~0u && hasPendingEvent(VMEM_GROUP) &&
+      !hasPendingEvent(STORE_CNT))
+    return applyWaitcnt(X_CNT, std::min(Wait.XCnt, Wait.LoadCnt));
+
+  applyWaitcnt(X_CNT, Wait.XCnt);
+}
+
 // Where there are multiple types of event in the bracket of a counter,
 // the decrement may go out of order.
 bool WaitcntBrackets::counterOutOfOrder(InstCounterType T) const {
   // Scalar memory read always can go out of order.
-  if (T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
+  if ((T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS)) ||
+      (T == X_CNT && hasPendingEvent(SMEM_GROUP)))
     return true;
   return hasMixedPendingEvents(T);
 }
@@ -1263,6 +1315,8 @@ static std::optional<InstCounterType> counterTypeForInstr(unsigned Opcode) {
     return DS_CNT;
   case AMDGPU::S_WAIT_KMCNT:
     return KM_CNT;
+  case AMDGPU::S_WAIT_XCNT:
+    return X_CNT;
   default:
     return {};
   }
@@ -1292,7 +1346,13 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
   MachineInstr *WaitcntInstr = nullptr;
   MachineInstr *WaitcntVsCntInstr = nullptr;
 
-  LLVM_DEBUG(dbgs() << "PreGFX12::applyPreexistingWaitcnt at: " << *It);
+  LLVM_DEBUG({
+    dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
+    if (It == OldWaitcntInstr.getParent()->instr_end())
+      dbgs() << "end of block\n";
+    else
+      dbgs() << *It;
+  });
 
   for (auto &II :
        make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
@@ -1427,7 +1487,8 @@ WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
 
 AMDGPU::Waitcnt
 WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
-  return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0);
+  return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
+                         ~0u /* XCNT */);
 }
 
 /// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
@@ -1445,7 +1506,13 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
   MachineInstr *CombinedStoreDsCntInstr = nullptr;
   MachineInstr *WaitInstrs[NUM_EXTENDED_INST_CNTS] = {};
 
-  LLVM_DEBUG(dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: " << *It);
+  LLVM_DEBUG({
+    dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
+    if (It == OldWaitcntInstr.getParent()->instr_end())
+      dbgs() << "end of block\n";
+    else
+      dbgs() << *It;
+  });
 
   for (auto &II :
        make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
@@ -1723,8 +1790,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
                                                  bool FlushVmCnt) {
   setForceEmitWaitcnt();
 
-  if (MI.isMetaInstruction())
-    return false;
+  assert(!MI.isMetaInstruction());
 
   AMDGPU::Waitcnt Wait;
 
@@ -1851,7 +1917,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
           continue;
 
         // LOAD_CNT is only relevant to vgpr or LDS.
-        unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
+        unsigned RegNo = FIRST_LDS_VGPR;
         // Only objects with alias scope info were added to LDSDMAScopes array.
         // In the absense of the scope info we will not be able to disambiguate
         // aliasing here. There is no need to try searching for a corresponding
@@ -1909,6 +1975,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
             ScoreBrackets.determineWait(BVH_CNT, Interval, Wait);
             ScoreBrackets.clearVgprVmemTypes(Interval);
           }
+
           if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
             ScoreBrackets.determineWait(EXP_CNT, Interval, Wait);
           }
@@ -1916,6 +1983,9 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
         } else {
           ScoreBrackets.determineWait(SmemAccessCounter, Interval, Wait);
         }
+
+        if (hasXcnt() && Op.isDef())
+          ScoreBrackets.determineWait(X_CNT, Interval, Wait);
       }
     }
   }
@@ -1958,6 +2028,8 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
     Wait.BvhCnt = 0;
   if (ForceEmitWaitcnt[KM_CNT])
     Wait.KmCnt = 0;
+  if (ForceEmitWaitcnt[X_CNT])
+    Wait.XCnt = 0;
 
   if (FlushVmCnt) {
     if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
@@ -2007,6 +2079,21 @@ bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
                       << "Update Instr: " << *It);
   }
 
+  // XCnt may be already consumed by a load wait.
+  if (Wait.KmCnt == 0 && Wait.XCnt != ~0u &&
+      !ScoreBrackets.hasPendingEvent(SMEM_GROUP))
+    Wait.XCnt = ~0u;
+
+  if (Wait.LoadCnt == 0 && Wait.XCnt != ~0u &&
+      !ScoreBrackets.hasPendingEvent(VMEM_GROUP))
+    Wait.XCnt = ~0u;
+
+  // Since the translation for VMEM addresses occur in-order, we can skip the
+  // XCnt if the current instruction is of VMEM type and has a memory dependency
+  // with another VMEM instruction in flight.
+  if (Wait.XCnt != ~0u && isVmemAccess(*It))
+    Wait.XCnt = ~0u;
+
   if (WCG->createNewWaitcnt(Block, It, Wait))
     Modified = true;
 
@@ -2096,6 +2183,11 @@ bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
   });
 }
 
+bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
+  return (TII->isFLAT(MI) && mayAccessVMEMThroughFlat(MI)) ||
+         (TII->isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(MI.getOpcode()));
+}
+
 static bool isGFX12CacheInvOrWBInst(MachineInstr &Inst) {
   auto Opc = Inst.getOpcode();
   return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
@@ -2167,6 +2259,8 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
   // bracket and the destination operand scores.
   // TODO: Use the (TSFlags & SIInstrFlags::DS_CNT) property everywhere.
 
+  bool IsVMEMAccess = false;
+  bool IsSMEMAccess = false;
   if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
     if (TII->isAlwaysGDS(Inst.getOpcode()) ||
         TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
@@ -2189,6 +2283,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
 
     if (mayAccessVMEMThroughFlat(Inst)) {
       ++FlatASCount;
+      IsVMEMAccess = true;
       ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst),
                                    Inst);
     }
@@ -2208,6 +2303,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
       ScoreBrackets->setPendingFlat();
   } else if (SIInstrInfo::isVMEM(Inst) &&
              !llvm::AMDGPU::getMUBUFIsBufferInv(Inst.getOpcode())) {
+    IsVMEMAccess = true;
     ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst),
                                  Inst);
 
@@ -2216,6 +2312,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
       ScoreBrackets->updateByEvent(TII, TRI, MRI, VMW_GPR_LOCK, Inst);
     }
   } else if (TII->isSMRD(Inst)) {
+    IsSMEMAccess = true;
     ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
   } else if (Inst.isCall()) {
     if (callWaitsOnFunctionReturn(Inst)) {
@@ -2258,6 +2355,15 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
       break;
     }
   }
+
+  if (!hasXcnt())
+    return;
+
+  if (IsVMEMAccess)
+    ScoreBrackets->updateByEvent(TII, TRI, MRI, VMEM_GROUP, Inst);
+
+  if (IsSMEMAccess)
+    ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_GROUP, Inst);
 }
 
 bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
@@ -2311,9 +2417,11 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
     for (int J = 0; J <= VgprUB; J++)
       StrictDom |= mergeScore(M, VgprScores[T][J], Other.VgprScores[T][J]);
 
-    if (T == SmemAccessCounter) {
+    if (isSmemCounter(T)) {
+      unsigned Idx = getSgprScoresIdx(T);
       for (int J = 0; J <= SgprUB; J++)
-        StrictDom |= mergeScore(M, SgprScores[J], Other.SgprScores[J]);
+        StrictDom |=
+            mergeScore(M, SgprScores[Idx][J], Other.SgprScores[Idx][J]);
     }
   }
 
@@ -2369,6 +2477,10 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
                                          E = Block.instr_end();
        Iter != E;) {
     MachineInstr &Inst = *Iter;
+    if (Inst.isMetaInstruction()) {
+      ++Iter;
+      continue;
+    }
 
     // Track pre-existing waitcnts that were added in earlier iterations or by
     // the memory legalizer.
@@ -2651,6 +2763,7 @@ bool SIInsertWaitcnts::run(MachineFunction &MF) {
   Limits.SamplecntMax = AMDGPU::getSamplecntBitMask(IV);
   Limits.BvhcntMax = AMDGPU::getBvhcntBitMask(IV);
   Limits.KmcntMax = AMDGPU::getKmcntBitMask(IV);
+  Limits.XcntMax = AMDGPU::getXcntBitMask(IV);
 
   [[maybe_unused]] unsigned NumVGPRsMax =
       ST->getAddressableNumVGPRs(MFI->getDynamicVGPRBlockSize());
@@ -2679,7 +2792,7 @@ bool SIInsertWaitcnts::run(MachineFunction &MF) {
       BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT))
           .addImm(0);
       for (auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
-        if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
+        if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT || CT == X_CNT)
           continue;
 
         if (!ST->hasImageInsts() &&
diff --git a/llvm/lib/Target/AMDGPU/SIInstrFormats.td b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
index 267c9a94b909..c27d4e0df6fc 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
@@ -68,6 +68,9 @@ class InstSI <dag outs, dag ins, string asm = "",
 
   field bit Gather4 = 0;
 
+  // wait count to manage tensor loads/stores.
+  field bit TENSOR_CNT = 0;
+
   // This is an s_store_dword* instruction that requires a cache flush
   // on wave termination. It is necessary to distinguish from mayStore
   // SMEM instructions like the cache flush ones.
@@ -77,6 +80,9 @@ class InstSI <dag outs, dag ins, string asm = "",
   // instruction size.
   field bit FixedSize = 0;
 
+  // wait count to manage asynchronous loads/stores.
+  field bit ASYNC_CNT = 0;
+
   // This bit indicates that this is a VOP3 opcode which supports op_sel
   // modifier.
   field bit VOP3_OPSEL = 0;
@@ -198,14 +204,12 @@ class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{36} = DisableWQM;
   let TSFlags{37} = Gather4;
 
-  // Reserved, must be 0.
-  let TSFlags{38} = 0;
+  let TSFlags{38} = TENSOR_CNT;
 
   let TSFlags{39} = ScalarStore;
   let TSFlags{40} = FixedSize;
 
-  // Reserved, must be 0.
-  let TSFlags{41} = 0;
+  let TSFlags{41} = ASYNC_CNT;
 
   let TSFlags{42} = VOP3_OPSEL;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index ad63f9ab2c50..68310ef56cc2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -387,7 +387,10 @@ bool SIInstrInfo::getMemOperandsWithOffsetWidth(
       DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst);
       if (DataOpIdx == -1)
         DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
-      Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
+      if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+        Width = LocationSize::precise(64);
+      else
+        Width = LocationSize::precise(getOpSize(LdSt, DataOpIdx));
     } else {
       // The 2 offset instructions use offset0 and offset1 instead. We can treat
       // these as a load with a single offset if the 2 offsets are consecutive.
@@ -7107,6 +7110,19 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
     return nullptr;
   }
 
+  // Legalize TENSOR_LOAD_TO_LDS, TENSOR_LOAD_TO_LDS_D2, TENSOR_STORE_FROM_LDS,
+  // TENSOR_STORE_FROM_LDS_D2. All their operands are scalar.
+  if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS ||
+      MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_D2 ||
+      MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS ||
+      MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_D2) {
+    for (MachineOperand &Src : MI.explicit_operands()) {
+      if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
+        Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
+    }
+    return CreatedBB;
+  }
+
   // Legalize MUBUF instructions.
   bool isSoffsetLegal = true;
   int SoffsetIdx =
@@ -8776,9 +8792,8 @@ void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
 void SIInstrInfo::addUsersToMoveToVALUWorklist(
     Register DstReg, MachineRegisterInfo &MRI,
     SIInstrWorklist &Worklist) const {
-  for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
-         E = MRI.use_end(); I != E;) {
-    MachineInstr &UseMI = *I->getParent();
+  for (MachineOperand &MO : make_early_inc_range(MRI.use_operands(DstReg))) {
+    MachineInstr &UseMI = *MO.getParent();
 
     unsigned OpNo = 0;
 
@@ -8793,21 +8808,15 @@ void SIInstrInfo::addUsersToMoveToVALUWorklist(
     case AMDGPU::INSERT_SUBREG:
       break;
     default:
-      OpNo = I.getOperandNo();
+      OpNo = MO.getOperandNo();
       break;
     }
 
-    if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) {
+    if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo)))
       Worklist.insert(&UseMI);
-
-      do {
-        ++I;
-      } while (I != E && I->getParent() == &UseMI);
-    } else {
+    else
+      // Legalization could change user list.
       legalizeOperandsVALUt16(UseMI, OpNo, MRI);
-
-      ++I;
-    }
   }
 }
 
@@ -10233,8 +10242,12 @@ unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
     return 3;
   case CallingConv::AMDGPU_HS:
   case CallingConv::AMDGPU_LS:
-  case CallingConv::AMDGPU_ES:
-    report_fatal_error("ds_ordered_count unsupported for this calling conv");
+  case CallingConv::AMDGPU_ES: {
+    const Function &F = MF.getFunction();
+    F.getContext().diagnose(DiagnosticInfoUnsupported(
+        F, "ds_ordered_count unsupported for this calling conv"));
+    [[fallthrough]];
+  }
   case CallingConv::AMDGPU_CS:
   case CallingConv::AMDGPU_KERNEL:
   case CallingConv::C:
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index eb1867778080..d0cba30a442b 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -735,7 +735,9 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
   case AMDGPU::V_ASHRREV_I16_e32:
   case AMDGPU::V_LSHLREV_B16_e32:
   case AMDGPU::V_LSHRREV_B16_e64:
+  case AMDGPU::V_LSHRREV_B16_opsel_e64:
   case AMDGPU::V_ASHRREV_I16_e64:
+  case AMDGPU::V_LSHLREV_B16_opsel_e64:
   case AMDGPU::V_LSHLREV_B16_e64: {
     // from: v_lshrrev_b16_e32 v1, 8, v0
     // to SDWA src:v0 src_sel:BYTE_1
@@ -758,11 +760,13 @@ SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
       break;
 
     if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
+        Opcode == AMDGPU::V_LSHLREV_B16_opsel_e64 ||
         Opcode == AMDGPU::V_LSHLREV_B16_e64)
       return std::make_unique<SDWADstOperand>(Dst, Src1, BYTE_1, UNUSED_PAD);
     return std::make_unique<SDWASrcOperand>(
         Src1, Dst, BYTE_1, false, false,
         Opcode != AMDGPU::V_LSHRREV_B16_e32 &&
+            Opcode != AMDGPU::V_LSHRREV_B16_opsel_e64 &&
             Opcode != AMDGPU::V_LSHRREV_B16_e64);
     break;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index f59ab167e468..f02ffb550ae1 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -665,6 +665,10 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_BASE);
   reserveRegisterTuples(Reserved, AMDGPU::SRC_PRIVATE_LIMIT);
 
+  // Reserve async counters pseudo registers
+  reserveRegisterTuples(Reserved, AMDGPU::ASYNCcnt);
+  reserveRegisterTuples(Reserved, AMDGPU::TENSORcnt);
+
   // Reserve src_pops_exiting_wave_id - support is not implemented in Codegen.
   reserveRegisterTuples(Reserved, AMDGPU::SRC_POPS_EXITING_WAVE_ID);
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index f601b54715cd..0e6ff2d6b2c8 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -249,6 +249,10 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
     return isSGPRClass(getPhysRegBaseClass(Reg));
   }
 
+  bool isVGPRPhysReg(Register Reg) const {
+    return isVGPRClass(getPhysRegBaseClass(Reg));
+  }
+
   /// \returns true if this class contains only VGPR registers
   static bool isVGPRClass(const TargetRegisterClass *RC) {
     return hasVGPRs(RC) && !hasAGPRs(RC) && !hasSGPRs(RC);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index d595163f820c..a6c7b164c8b2 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -234,6 +234,10 @@ defm SRC_POPS_EXITING_WAVE_ID : SIRegLoHi16<"src_pops_exiting_wave_id", 239>;
 // Not addressable
 def MODE : SIReg <"mode", 0>;
 
+// Not addressable, used to model dependencies.
+def ASYNCcnt : SIReg <"ASYNCcnt", 0>;
+def TENSORcnt : SIReg <"TENSORcnt", 0>;
+
 def LDS_DIRECT : SIReg <"src_lds_direct", 254> {
   // There is no physical register corresponding to this. This is an
   // encoding value in a source field, which will ultimately trigger a
diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 0eb55daf3208..fd39b8a1350c 100644
--- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -842,10 +842,7 @@ bool SIShrinkInstructions::run(MachineFunction &MF) {
 
   unsigned VCCReg = ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC;
 
-  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
-                                                  BI != BE; ++BI) {
-
-    MachineBasicBlock &MBB = *BI;
+  for (MachineBasicBlock &MBB : MF) {
     MachineBasicBlock::iterator I, Next;
     for (I = MBB.begin(); I != MBB.end(); I = Next) {
       Next = std::next(I);
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 376c6eb135b1..c7c4276e0e25 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1589,8 +1589,10 @@ def S_WAKEUP : SOPP_Pseudo <"s_wakeup", (ins) > {
   let mayStore = 1;
 }
 
+let SubtargetPredicate = isNotGFX1250Plus in {
 def S_WAITCNT : SOPP_Pseudo <"s_waitcnt" , (ins SWaitCnt:$simm16), "$simm16",
     [(int_amdgcn_s_waitcnt timm:$simm16)]>;
+}
 
 // "_soft" waitcnts are waitcnts that are either relaxed into their non-soft
 // counterpart, or completely removed.
@@ -1678,6 +1680,12 @@ def S_SET_GPR_IDX_OFF : SOPP_Pseudo<"s_set_gpr_idx_off", (ins) > {
   let Uses = [MODE];
 }
 }
+
+def S_MONITOR_SLEEP : SOPP_Pseudo <"s_monitor_sleep", (ins i16imm:$simm16), "$simm16",
+  [(int_amdgcn_s_monitor_sleep timm:$simm16)]> {
+  let SubtargetPredicate = isGFX1250Plus;
+}
+
 } // End hasSideEffects
 
 let SubtargetPredicate = HasVGPRIndexMode in {
@@ -2690,6 +2698,12 @@ defm S_ICACHE_INV                 : SOPP_Real_32_gfx11_gfx12<0x03c>;
 
 defm S_BARRIER                    : SOPP_Real_32_gfx11<0x03d>;
 
+//===----------------------------------------------------------------------===//
+// SOPP - GFX1250.
+//===----------------------------------------------------------------------===//
+
+defm S_MONITOR_SLEEP              : SOPP_Real_32_gfx12<0x004>;
+
 //===----------------------------------------------------------------------===//
 // SOPP - GFX6, GFX7, GFX8, GFX9, GFX10
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 0e5493259edb..5b21f4a6083a 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -129,6 +129,11 @@ unsigned getKmcntBitWidth(unsigned VersionMajor) {
   return VersionMajor >= 12 ? 5 : 0;
 }
 
+/// \returns Xcnt bit width.
+unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
+  return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
+}
+
 /// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
 unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
   return VersionMajor >= 12 ? 8 : 0;
@@ -679,6 +684,30 @@ bool isGenericAtomic(unsigned Opc) {
          Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
 }
 
+bool isAsyncStore(unsigned Opc) {
+  return false; // placeholder before async store implementation.
+}
+
+bool isTensorStore(unsigned Opc) {
+  return Opc == TENSOR_STORE_FROM_LDS_gfx1250 ||
+         Opc == TENSOR_STORE_FROM_LDS_D2_gfx1250;
+}
+
+unsigned getTemporalHintType(const MCInstrDesc TID) {
+  if (TID.TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet))
+    return CPol::TH_TYPE_ATOMIC;
+  unsigned Opc = TID.getOpcode();
+  // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
+  if (TID.mayStore() &&
+      (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
+    return CPol::TH_TYPE_STORE;
+
+  // This will default to returning TH_TYPE_LOAD when neither MayStore nor
+  // MayLoad flag is present which is the case with instructions like
+  // image_get_resinfo.
+  return CPol::TH_TYPE_LOAD;
+}
+
 bool isTrue16Inst(unsigned Opc) {
   const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opc);
   return Info && Info->IsTrue16;
@@ -1493,6 +1522,10 @@ unsigned getKmcntBitMask(const IsaVersion &Version) {
   return (1 << getKmcntBitWidth(Version.Major)) - 1;
 }
 
+unsigned getXcntBitMask(const IsaVersion &Version) {
+  return (1 << getXcntBitWidth(Version.Major, Version.Minor)) - 1;
+}
+
 unsigned getStorecntBitMask(const IsaVersion &Version) {
   return (1 << getStorecntBitWidth(Version.Major)) - 1;
 }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index ac7c5100be3d..4f7d18170d58 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -891,6 +891,13 @@ LLVM_READONLY
 VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
                                const MCInstrInfo *InstrInfo);
 
+LLVM_READONLY
+bool isAsyncStore(unsigned Opc);
+LLVM_READONLY
+bool isTensorStore(unsigned Opc);
+LLVM_READONLY
+unsigned getTemporalHintType(const MCInstrDesc TID);
+
 LLVM_READONLY
 bool isTrue16Inst(unsigned Opc);
 
@@ -980,6 +987,7 @@ struct Waitcnt {
   unsigned SampleCnt = ~0u; // gfx12+ only.
   unsigned BvhCnt = ~0u;    // gfx12+ only.
   unsigned KmCnt = ~0u;     // gfx12+ only.
+  unsigned XCnt = ~0u;      // gfx1250.
 
   Waitcnt() = default;
   // Pre-gfx12 constructor.
@@ -988,15 +996,15 @@ struct Waitcnt {
 
   // gfx12+ constructor.
   Waitcnt(unsigned LoadCnt, unsigned ExpCnt, unsigned DsCnt, unsigned StoreCnt,
-          unsigned SampleCnt, unsigned BvhCnt, unsigned KmCnt)
+          unsigned SampleCnt, unsigned BvhCnt, unsigned KmCnt, unsigned XCnt)
       : LoadCnt(LoadCnt), ExpCnt(ExpCnt), DsCnt(DsCnt), StoreCnt(StoreCnt),
-        SampleCnt(SampleCnt), BvhCnt(BvhCnt), KmCnt(KmCnt) {}
+        SampleCnt(SampleCnt), BvhCnt(BvhCnt), KmCnt(KmCnt), XCnt(XCnt) {}
 
   bool hasWait() const { return StoreCnt != ~0u || hasWaitExceptStoreCnt(); }
 
   bool hasWaitExceptStoreCnt() const {
     return LoadCnt != ~0u || ExpCnt != ~0u || DsCnt != ~0u ||
-           SampleCnt != ~0u || BvhCnt != ~0u || KmCnt != ~0u;
+           SampleCnt != ~0u || BvhCnt != ~0u || KmCnt != ~0u || XCnt != ~0u;
   }
 
   bool hasWaitStoreCnt() const { return StoreCnt != ~0u; }
@@ -1008,7 +1016,7 @@ struct Waitcnt {
         std::min(LoadCnt, Other.LoadCnt), std::min(ExpCnt, Other.ExpCnt),
         std::min(DsCnt, Other.DsCnt), std::min(StoreCnt, Other.StoreCnt),
         std::min(SampleCnt, Other.SampleCnt), std::min(BvhCnt, Other.BvhCnt),
-        std::min(KmCnt, Other.KmCnt));
+        std::min(KmCnt, Other.KmCnt), std::min(XCnt, Other.XCnt));
   }
 };
 
@@ -1114,6 +1122,10 @@ unsigned getDscntBitMask(const IsaVersion &Version);
 /// Returns 0 for versions that do not support KMcnt
 unsigned getKmcntBitMask(const IsaVersion &Version);
 
+/// \returns Xcnt bit mask for given isa \p Version.
+/// Returns 0 for versions that do not support Xcnt.
+unsigned getXcntBitMask(const IsaVersion &Version);
+
 /// \return STOREcnt or VScnt bit mask for given isa \p Version.
 /// returns 0 for versions that do not support STOREcnt or VScnt.
 /// STOREcnt and VScnt are the same counter, the name used
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index 79152e707a25..e464470143e5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -898,7 +898,7 @@ bool AMDGPUPALMetadata::setFromString(StringRef S) {
         errs() << "Unrecognized PAL metadata register key '" << S << "'\n";
         continue;
       }
-      Key = MsgPackDoc.getNode(uint64_t(Val));
+      Key = MsgPackDoc.getNode(Val);
     }
     Registers.getMap()[Key] = I.second;
   }
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 02b912bcfb9e..0dacd9df7130 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -259,6 +259,12 @@ foreach vt = Reg32Types.types in {
   >;
 }
 
+let HasOMod = 0, HasClamp = 0 in {
+  def VOPProfile_CVT_F32_BF16_gfx1250_t16 : VOPProfile_True16 <VOP_F32_BF16>;
+  let HasOpSel = 1, EmitDstSel = 0 in
+  def VOPProfile_CVT_F32_BF16_gfx1250_fake16 : VOPProfile_Fake16 <VOP_F32_BF16>;
+} // End HasOMod = 0, HasClamp = 0
+
 let isReMaterializable = 1 in {
 let SchedRW = [WriteDoubleCvt] in {
 // OMod clears exceptions when set in this instruction
@@ -309,8 +315,14 @@ let OtherPredicates = [UseRealTrue16Insts] in
 let OtherPredicates = [UseFakeTrue16Insts] in
   defm V_CVT_F32_F16_fake16 : VOP1Inst <"v_cvt_f32_f16_fake16", VOPProfile_Fake16<VOP_F32_F16>, any_fpextend>;
 
-let SubtargetPredicate = HasBF16ConversionInsts in
-defm V_CVT_F32_BF16 : VOP1Inst_t16 <"v_cvt_f32_bf16", VOP_F32_BF16>;
+let SubtargetPredicate = HasGFX950Insts, OtherPredicates = [HasBF16ConversionInsts] in {
+  defm V_CVT_F32_BF16 : VOP1Inst_t16 <"v_cvt_f32_bf16", VOP_F32_BF16>;
+}
+let SubtargetPredicate = isGFX1250Plus, OtherPredicates = [HasBF16ConversionInsts] in {
+  defm V_CVT_F32_BF16_gfx1250 : VOP1Inst_t16_with_profiles <"v_cvt_f32_bf16_gfx1250", VOP_F32_BF16,
+                                                            VOPProfile_CVT_F32_BF16_gfx1250_t16,
+                                                            VOPProfile_CVT_F32_BF16_gfx1250_fake16>;
+}
 
 let ReadsModeReg = 0, mayRaiseFPException = 0 in {
 defm V_CVT_RPI_I32_F32 : VOP1Inst <"v_cvt_rpi_i32_f32", VOP_I32_F32, cvt_rpi_i32_f32>;
@@ -389,8 +401,8 @@ def VOP_PERMLANE_SWAP : VOPProfile<[i32, i32, untyped, untyped]> {
   let HasExtDPP = 0;
   let HasExtSDWA = 0;
 
-  let Ins32 = (ins Src0RC64:$vdst_in, Src0RC32:$src0);
-  let Ins64 = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
+  let Ins32 = (ins DstRC:$vdst_in, Src0RC32:$src0);
+  let Ins64 = (ins DstRC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let InsVOP3OpSel = (ins Src0RC64:$vdst_in, Src0RC64:$src0, Dpp16FI:$fi, DppBoundCtrl:$bound_ctrl);
   let Asm64 = "$vdst, $src0$bound_ctrl$fi";
 }
@@ -675,6 +687,12 @@ class VOPProfile_Base_CVT_F_F8_ByteSel<ValueType DstVT> : VOPProfile<[DstVT, i32
   let HasModifiers = 0;
 }
 
+let IsSingle = 0, HasOpSel = 1, HasModifiers = 1 in {
+def V_CVT_F16_F8_Profile : VOPProfile_Base_CVT_F_F8_ByteSel<f16>;
+def V_CVT_F16_F8_True16_Profile : VOP3_Profile_True16<V_CVT_F16_F8_Profile>;
+def V_CVT_F16_F8_Fake16_Profile : VOP3_Profile_Fake16<V_CVT_F16_F8_Profile>;
+}
+
 let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts],
     mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
   defm V_CVT_F32_FP8_OP_SEL    : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
@@ -690,9 +708,10 @@ let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts],
   }
 }
 
-class Cvt_F_F8_Pat_ByteSel<SDPatternOperator node, VOP3_Pseudo inst> : GCNPat<
+class Cvt_F_F8_Pat_ByteSel<SDPatternOperator node, VOP3_Pseudo inst, bit HasOpSel = 0> : GCNPat<
   (node i32:$src0, timm:$byte_sel),
-  (inst $src0, (as_i32timm $byte_sel))
+  !if(HasOpSel, (inst 0, $src0, (as_i32timm $byte_sel)),
+                (inst $src0, (as_i32timm $byte_sel)))
 >;
 
 let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts] in {
@@ -717,6 +736,37 @@ let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts]
   }
 }
 
+// FIXME-TRUE16: True16 versions of these instructions are untested.
+let HasExtSDWA = 0, HasOpSel = 1, EmitDstSel = 0, HasOMod = 0, HasModifiers = 1 in {
+def VOPProfile_CVT_PK_F16_F8 : VOPProfile<[v2f16, i16, untyped, untyped]>;
+def VOPProfile_CVT_PK_F16_F8_true16 : VOP3_Profile_True16<VOPProfile_CVT_PK_F16_F8>;
+def VOPProfile_CVT_PK_F16_F8_fake16 : VOP3_Profile_Fake16<VOPProfile_CVT_PK_F16_F8>;
+}
+
+let SubtargetPredicate = isGFX1250Plus in {
+  let mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
+    defm V_CVT_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_fp8",
+      V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
+    defm V_CVT_F16_BF8 : VOP1Inst_t16_with_profiles<"v_cvt_f16_bf8",
+      V_CVT_F16_F8_Profile, V_CVT_F16_F8_True16_Profile, V_CVT_F16_F8_Fake16_Profile>;
+    defm V_CVT_PK_F16_FP8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_fp8",
+      VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
+      int_amdgcn_cvt_pk_f16_fp8>;
+    defm V_CVT_PK_F16_BF8 : VOP1Inst_t16_with_profiles<"v_cvt_pk_f16_bf8",
+      VOPProfile_CVT_PK_F16_F8, VOPProfile_CVT_PK_F16_F8_true16, VOPProfile_CVT_PK_F16_F8_fake16,
+      int_amdgcn_cvt_pk_f16_bf8>;
+  }
+
+  let True16Predicate = UseRealTrue16Insts in {
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_fp8, V_CVT_F16_FP8_t16_e64, 1>;
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_t16_e64, 1>;
+  }
+  let True16Predicate = UseFakeTrue16Insts in {
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_fp8, V_CVT_F16_FP8_fake16_e64, 1>;
+    def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f16_bf8, V_CVT_F16_BF8_fake16_e64, 1>;
+  }
+} // End SubtargetPredicate = isGFX1250Plus
+
 let SubtargetPredicate = isGFX10Plus in {
   defm V_PIPEFLUSH        : VOP1Inst<"v_pipeflush", VOP_NO_EXT<VOP_NONE>>;
 
@@ -980,6 +1030,13 @@ multiclass VOP1_Real_NO_DPP_OP_SEL_with_name<GFXGen Gen, bits<9> op,
   VOP1_Real_e32_with_name<Gen, op, opName, asmName>,
   VOP3_Real_with_name<Gen, {0, 1, 1, op{6-0}}, opName, asmName>;
 
+multiclass VOP1_Real_FULL_t16_and_fake16_gfx1250<
+    bits<9> op, string asmName = !tolower(NAME), string opName = NAME> {
+  defm opName#"_t16" :
+       VOP1_Real_FULL_with_name<GFX1250Gen, op, opName#"_t16", asmName>;
+  defm opName#"_fake16":
+       VOP1_Real_FULL_with_name<GFX1250Gen, op, opName#"_fake16", asmName>;
+}
 
 defm V_CVT_F32_FP8      : VOP1_Real_FULL_with_name<GFX12Gen, 0x06c, "V_CVT_F32_FP8_OP_SEL", "v_cvt_f32_fp8">;
 defm V_CVT_F32_BF8      : VOP1_Real_FULL_with_name<GFX12Gen, 0x06d, "V_CVT_F32_BF8_OP_SEL", "v_cvt_f32_bf8">;
@@ -1042,6 +1099,12 @@ defm V_CVT_NORM_U16_F16      : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x064>;
 defm V_CVT_F16_F32           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00a>;
 defm V_CVT_F32_F16           : VOP1_Real_FULL_t16_and_fake16_gfx11_gfx12<0x00b>;
 
+defm V_CVT_F32_BF16          : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x072, "v_cvt_f32_bf16", "V_CVT_F32_BF16_gfx1250">;
+defm V_CVT_PK_F16_FP8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x075>;
+defm V_CVT_PK_F16_BF8        : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x076>;
+defm V_CVT_F16_FP8           : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x077>;
+defm V_CVT_F16_BF8           : VOP1_Real_FULL_t16_and_fake16_gfx1250<0x078>;
+
 //===----------------------------------------------------------------------===//
 // GFX10.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 0c7e20fc1ebf..65d4189f9949 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -211,6 +211,11 @@ multiclass VOP2Inst_e64_t16<string opName,
                         string revOp = opName> {
   let OtherPredicates = [Has16BitInsts], True16Predicate = NotHasTrue16BitInsts in {
     defm NAME : VOP2Inst<opName, P, node, revOp>;
+    let SubtargetPredicate = isGFX10Only in {
+      // V_MAX_I16 etc use VOP3 encoding and allow OP_SEL
+      def _opsel_e64 : VOP3InstBase <opName#"_vop3", VOP3_Profile<P, VOP3_OPSEL>, node, 1>,
+                       Commutable_REV<revOp#"_vop3_e64", !eq(revOp, opName)>;
+    }
   }
   let SubtargetPredicate = UseRealTrue16Insts in {
     defm _t16 : VOP2Inst_e64<opName#"_t16", VOPProfile_True16<P>, node, revOp#"_t16">;
@@ -1945,6 +1950,14 @@ let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
     if !cast<VOP2_Pseudo>(NAME#"_e32").Pfl.HasExt32BitDPP then
     def _dpp8_gfx10 : VOP2_DPP8<op, !cast<VOP2_Pseudo>(NAME#"_e32")>;
   }
+  multiclass VOP2Only_Real_e32_gfx10<bits<6> op> {
+    let IsSingle = 1 in
+      defm NAME: VOP2_Real_e32_gfx10<op>;
+  }
+  multiclass VOP2_Real_e32_dpp_dpp8_gfx10<bits<6> op> :
+    VOP2Only_Real_e32_gfx10<op>,
+    VOP2_Real_dpp_gfx10<op>,
+    VOP2_Real_dpp8_gfx10<op>;
 
   //===------------------------- VOP2 (with name) -------------------------===//
   multiclass VOP2_Real_e32_gfx10_with_name<bits<6> op, string opName,
@@ -2168,10 +2181,7 @@ defm V_FMAAK_F16       : VOP2Only_Real_MADK_gfx10<0x038>;
 defm V_MAX_F16         : VOP2_Real_gfx10<0x039>;
 defm V_MIN_F16         : VOP2_Real_gfx10<0x03a>;
 defm V_LDEXP_F16       : VOP2_Real_gfx10<0x03b>;
-
-let IsSingle = 1 in {
-  defm V_PK_FMAC_F16     : VOP2_Real_e32_gfx10<0x03c>;
-}
+defm V_PK_FMAC_F16     : VOP2_Real_e32_dpp_dpp8_gfx10<0x03c>;
 
 // VOP2 no carry-in, carry-out.
 defm V_ADD_NC_U32 :
@@ -2560,6 +2570,7 @@ defm V_SUBBREV_CO_U32     : VOP2be_Real_e32e64_gfx9 <0x1e, "V_SUBBREV_U32", "v_s
 defm V_ADD_U32            : VOP2_Real_e32e64_gfx9 <0x34>;
 defm V_SUB_U32            : VOP2_Real_e32e64_gfx9 <0x35>;
 defm V_SUBREV_U32         : VOP2_Real_e32e64_gfx9 <0x36>;
+defm V_PK_FMAC_F16        : VOP2_Real_e32e64_gfx9<0x03c>;
 } // End AssemblerPredicate = isGFX9Only
 
 defm V_BFM_B32            : VOP2_Real_e64only_vi <0x293>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 89a9ecc27c6e..279de32a9cee 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -32,6 +32,7 @@ class VOP3b_Profile<ValueType vt> : VOPProfile<[vt, vt, vt, vt]> {
   let HasExtDPP = 0;
 }
 
+let HasExt64BitDPP = 1 in {
 def VOP3b_F32_I1_F32_F32_F32 : VOP3b_Profile<f32>;
 def VOP3b_F64_I1_F64_F64_F64 : VOP3b_Profile<f64>;
 
@@ -48,10 +49,13 @@ class V_MUL_PROF<VOPProfile P> : VOP3_Profile<P> {
   let HasExtDPP = 0;
 }
 
+def V_LSHL_ADD_U64_PROF : VOP3_Profile<VOP_I64_I64_I32_I64>;
+
 def DIV_FIXUP_F32_PROF : VOP3_Profile<VOP_F32_F32_F32_F32> {
   let HasExtVOP3DPP = 0;
   let HasExtDPP = 0;
 }
+} // End HasExt64BitDPP = 1;
 
 //===----------------------------------------------------------------------===//
 // VOP3 INTERP
@@ -722,7 +726,7 @@ defm V_LSHL_OR_B32 : VOP3Inst <"v_lshl_or_b32", VOP3_Profile<VOP_I32_I32_I32_I32
 // V_LSHL_ADD_U64: D0.u64 = (S0.u64 << S1.u[2:0]) + S2.u64
 // src0 is shifted left by 0-4 (use “0” to get ADD_U64).
 let SubtargetPredicate = HasLshlAddU64Inst in
-defm V_LSHL_ADD_U64 : VOP3Inst <"v_lshl_add_u64", VOP3_Profile<VOP_I64_I64_I32_I64>>;
+defm V_LSHL_ADD_U64 : VOP3Inst <"v_lshl_add_u64", V_LSHL_ADD_U64_PROF>;
 
 let OtherPredicates = [HasFP8ConversionInsts], mayRaiseFPException = 0,
     SchedRW = [WriteFloatCvt] in {
@@ -1889,6 +1893,9 @@ let AssemblerPredicate = isGFX11Plus in {
   def : AMDGPUMnemonicAlias<"v_xor_add_u32", "v_xad_u32">;
 }
 
+// These instructions differ from GFX12 variant by supporting DPP:
+defm V_LSHL_ADD_U64                  : VOP3Only_Realtriple_gfx1250<0x252>;
+
 //===----------------------------------------------------------------------===//
 // GFX10.
 //===----------------------------------------------------------------------===//
@@ -2009,16 +2016,14 @@ defm V_DIV_FIXUP_F16 :
 defm V_ADD_NC_U16      : VOP3OpSel_Real_gfx10<0x303>;
 defm V_SUB_NC_U16      : VOP3OpSel_Real_gfx10<0x304>;
 
-// FIXME-GFX10-OPSEL: Need to add "selective" opsel support to some of these
-// (they do not support SDWA or DPP).
-defm V_MUL_LO_U16      : VOP3_Real_gfx10_with_name<0x305, "V_MUL_LO_U16", "v_mul_lo_u16">;
-defm V_LSHRREV_B16     : VOP3_Real_gfx10_with_name<0x307, "V_LSHRREV_B16", "v_lshrrev_b16">;
-defm V_ASHRREV_I16     : VOP3_Real_gfx10_with_name<0x308, "V_ASHRREV_I16", "v_ashrrev_i16">;
-defm V_MAX_U16         : VOP3_Real_gfx10_with_name<0x309, "V_MAX_U16", "v_max_u16">;
-defm V_MAX_I16         : VOP3_Real_gfx10_with_name<0x30a, "V_MAX_I16", "v_max_i16">;
-defm V_MIN_U16         : VOP3_Real_gfx10_with_name<0x30b, "V_MIN_U16", "v_min_u16">;
-defm V_MIN_I16         : VOP3_Real_gfx10_with_name<0x30c, "V_MIN_I16", "v_min_i16">;
-defm V_LSHLREV_B16     : VOP3_Real_gfx10_with_name<0x314, "V_LSHLREV_B16", "v_lshlrev_b16">;
+defm V_MUL_LO_U16      : VOP3OpSel_Real_gfx10_with_name<0x305, "V_MUL_LO_U16_opsel", "v_mul_lo_u16">;
+defm V_LSHRREV_B16     : VOP3OpSel_Real_gfx10_with_name<0x307, "V_LSHRREV_B16_opsel", "v_lshrrev_b16">;
+defm V_ASHRREV_I16     : VOP3OpSel_Real_gfx10_with_name<0x308, "V_ASHRREV_I16_opsel", "v_ashrrev_i16">;
+defm V_MAX_U16         : VOP3OpSel_Real_gfx10_with_name<0x309, "V_MAX_U16_opsel", "v_max_u16">;
+defm V_MAX_I16         : VOP3OpSel_Real_gfx10_with_name<0x30a, "V_MAX_I16_opsel", "v_max_i16">;
+defm V_MIN_U16         : VOP3OpSel_Real_gfx10_with_name<0x30b, "V_MIN_U16_opsel", "v_min_u16">;
+defm V_MIN_I16         : VOP3OpSel_Real_gfx10_with_name<0x30c, "V_MIN_I16_opsel", "v_min_i16">;
+defm V_LSHLREV_B16     : VOP3OpSel_Real_gfx10_with_name<0x314, "V_LSHLREV_B16_opsel", "v_lshlrev_b16">;
 defm V_PERMLANE16_B32  : VOP3OpSel_Real_gfx10<0x377>;
 defm V_PERMLANEX16_B32 : VOP3OpSel_Real_gfx10<0x378>;
 
diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
index ca5ed5cd2460..9e84f6aed017 100644
--- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td
@@ -846,7 +846,7 @@ defm V_CMP_T_U32 : VOPC_I32 <"v_cmp_t_u32">;
 defm V_CMPX_F_U32 : VOPCX_I32 <"v_cmpx_f_u32">;
 defm V_CMPX_LT_U32 : VOPCX_I32 <"v_cmpx_lt_u32", "v_cmpx_gt_u32">;
 defm V_CMPX_EQ_U32 : VOPCX_I32 <"v_cmpx_eq_u32">;
-defm V_CMPX_LE_U32 : VOPCX_I32 <"v_cmpx_le_u32", "v_cmpx_le_u32">;
+defm V_CMPX_LE_U32 : VOPCX_I32 <"v_cmpx_le_u32", "v_cmpx_ge_u32">;
 defm V_CMPX_GT_U32 : VOPCX_I32 <"v_cmpx_gt_u32">;
 defm V_CMPX_NE_U32 : VOPCX_I32 <"v_cmpx_ne_u32">;
 defm V_CMPX_GE_U32 : VOPCX_I32 <"v_cmpx_ge_u32">;
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 6a69cec7de68..f1224a0dc789 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -348,7 +348,8 @@ class VOP3OpSel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3OpSel_gfx10<op, p>;
 class VOP3FP8OpSel_src_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
   bits<2> byte_sel;
   let Inst{11-12} = byte_sel; // NB: bit order is intentionally reversed!
-  let Inst{14-13} = 0;  // op_sel2/3
+  let Inst{13} = !if(!and(p.HasOpSel, p.HasSrc2), src2_modifiers{2}, 0);
+  let Inst{14} = !if(!and(p.HasOpSel, p.HasDst), src0_modifiers{3}, 0);
 }
 
  class VOP3FP8OpSel_dst_bytesel_gfx11_gfx12<bits<10> op, VOPProfile p> : VOP3e_gfx10<op, p> {
@@ -526,7 +527,7 @@ class VOP3PXe <bits<7> op, VOPProfile MFMAPfl, bit acc_cd = 0> : Enc128, VOP3Pe_
   bits<9> scale_src1;
 
   //MFMALdScaleModifierOp transforms 2 bit opsel input to 4 bit value
-  //where opsel and opselHi are in 3rd and 4th bit. 
+  //where opsel and opselHi are in 3rd and 4th bit.
   bits<4> src0_modifiers;
   bits<4> src1_modifiers;
 
@@ -869,14 +870,14 @@ class VOP3_DPPe_Common_Base<bits<10> op, VOPProfile P> : Enc96 {
   let Inst{9}     = !if(P.HasSrc1Mods, src1_modifiers{1}, 0);
   let Inst{10}    = !if(P.HasSrc2Mods, src2_modifiers{1}, 0);
   // 16-bit select fields which can be interpreted as OpSel or hi/lo suffix
-  let Inst{11} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0),
-                                 !if(P.HasFP8SrcByteSel, byte_sel{1}, ?));
-  let Inst{12} = !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0),
-                                 !if(P.HasFP8SrcByteSel, byte_sel{0}, ?));
-  let Inst{13} = !if(P.HasOpSel, !if(P.HasSrc2Mods, src2_modifiers{2}, 0),
-                                 !if(P.HasFP8DstByteSel, byte_sel{0}, ?));
-  let Inst{14} = !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{3}, 0),
-                                 !if(P.HasFP8DstByteSel, byte_sel{1}, ?));
+  let Inst{11} = !if(P.HasFP8SrcByteSel, byte_sel{1},
+                 !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{2}, 0), ?));
+  let Inst{12} = !if(P.HasFP8SrcByteSel, byte_sel{0},
+                 !if(P.HasOpSel, !if(P.HasSrc1Mods, src1_modifiers{2}, 0), ?));
+  let Inst{13} = !if(P.HasFP8DstByteSel, byte_sel{0},
+                 !if(P.HasOpSel, !if(P.HasSrc2Mods, src2_modifiers{2}, 0), ?));
+  let Inst{14} = !if(P.HasFP8DstByteSel, byte_sel{1},
+                 !if(P.HasOpSel, !if(P.HasSrc0Mods, src0_modifiers{3}, 0), ?));
   let Inst{15}    = !if(P.HasClamp, clamp, 0);
   let Inst{25-16} = op;
   let Inst{31-26} = 0x35;
@@ -1551,12 +1552,17 @@ class VOP3InstBase<string OpName, VOPProfile P, SDPatternOperator node = null_fr
             ""));
 }
 
-multiclass VOP3Inst<string OpName, VOPProfile P, SDPatternOperator node = null_frag> {
+multiclass VOP3Inst<string OpName, VOPProfile P, SDPatternOperator node = null_frag,
+                    list<Predicate> predicates = []> {
   def _e64 : VOP3InstBase<OpName, P, node>;
-  let SubtargetPredicate = isGFX11Plus in {
-    if P.HasExtVOP3DPP then
-      def _e64_dpp : VOP3_DPP_Pseudo <OpName, P>;
-  } // end SubtargetPredicate = isGFX11Plus
+  if P.HasExtVOP3DPP then
+    def _e64_dpp  : VOP3_DPP_Pseudo <OpName, P> {
+      let SubtargetPredicate = isGFX11Plus;
+    }
+  else if P.HasExt64BitDPP then
+    def _e64_dpp  : VOP3_DPP_Pseudo <OpName, P> {
+      let OtherPredicates = !listconcat(predicates, [HasDPALU_DPP]);
+    }
 }
 
 class UniformUnaryFragOrOp<SDPatternOperator Op> {
@@ -1673,6 +1679,7 @@ class Base_VOP3_DPP8_t16<bits<10> op, VOP_Pseudo ps, string opName = ps.OpName>
   let SchedRW = ps.SchedRW;
   let Uses = ps.Uses;
 
+  let SubtargetPredicate = ps.SubtargetPredicate;
   let OtherPredicates = ps.OtherPredicates;
   let True16Predicate = ps.True16Predicate;
 }
@@ -1960,6 +1967,17 @@ multiclass VOP3Only_Realtriple_gfx12<bits<10> op, bit isSingle = 0> :
 multiclass VOP3Only_Real_Base_gfx12<bits<10> op> :
   VOP3_Real_Base<GFX12Gen, op, NAME, 1/*IsSingle*/>;
 
+multiclass VOP3Only_Realtriple_with_name_gfx12_not_gfx1250<bits<10> op, string opName,
+                                                           string asmName, string pseudo_mnemonic = "",
+                                                           bit isSingle = 0> :
+  VOP3_Realtriple_with_name<GFX12Not12_50Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
+
+multiclass VOP3Only_Real_Base_gfx1250<bits<10> op> :
+  VOP3_Real_Base<GFX1250Gen, op, NAME, 1/*IsSingle*/>;
+
+multiclass VOP3Only_Realtriple_gfx1250<bits<10> op, bit isSingle = 0> :
+  VOP3_Realtriple<GFX1250Gen, op, isSingle>;
+
 multiclass VOP3_Realtriple_t16_gfx12<bits<10> op, string asmName, string opName = NAME,
                                      string pseudo_mnemonic = "", bit isSingle = 0> :
   VOP3_Realtriple_with_name<GFX12Gen, op, opName, asmName, pseudo_mnemonic, isSingle>;
diff --git a/llvm/lib/Target/ARC/ARCMCInstLower.cpp b/llvm/lib/Target/ARC/ARCMCInstLower.cpp
index 50ba9fe75232..76087a2466ce 100644
--- a/llvm/lib/Target/ARC/ARCMCInstLower.cpp
+++ b/llvm/lib/Target/ARC/ARCMCInstLower.cpp
@@ -29,7 +29,6 @@ ARCMCInstLower::ARCMCInstLower(MCContext *C, AsmPrinter &AsmPrinter)
 MCOperand ARCMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                              MachineOperandType MOTy,
                                              unsigned Offset) const {
-  MCSymbolRefExpr::VariantKind Kind = MCSymbolRefExpr::VK_None;
   const MCSymbol *Symbol;
 
   switch (MOTy) {
@@ -60,7 +59,7 @@ MCOperand ARCMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   }
 
   assert(Symbol && "Symbol creation failed.\n");
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, Kind, *Ctx);
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, *Ctx);
 
   if (!Offset)
     return MCOperand::createExpr(MCSym);
diff --git a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
index e669b9479369..c54f33cbcb99 100644
--- a/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
+++ b/llvm/lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp
@@ -124,7 +124,7 @@ static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
     SRE = dyn_cast<MCSymbolRefExpr>(Expr);
     assert(SRE && "Unexpected MCExpr type.");
   }
-  assert(SRE->getKind() == MCSymbolRefExpr::VK_None);
+  assert(SRE->getSpecifier() == 0);
 
   // Symbols are prefixed with '@'
   OS << '@';
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 2fd784373f4a..850b00406f09 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -859,7 +859,7 @@ static uint8_t getModifierSpecifier(ARMCP::ARMCPModifier Modifier) {
   case ARMCP::GOT_PREL:
     return ARM::S_GOT_PREL;
   case ARMCP::SECREL:
-    return MCSymbolRefExpr::VK_SECREL;
+    return ARM::S_COFF_SECREL;
   }
   llvm_unreachable("Invalid ARMCPModifier!");
 }
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 8f17c62d4b90..50217c3a047d 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -3126,8 +3126,8 @@ bool ARMBaseInstrInfo::optimizeCompareInstr(
   // Modify the condition code of operands in OperandsToUpdate.
   // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
   // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
-  for (unsigned i = 0, e = OperandsToUpdate.size(); i < e; i++)
-    OperandsToUpdate[i].first->setImm(OperandsToUpdate[i].second);
+  for (auto &[MO, Cond] : OperandsToUpdate)
+    MO->setImm(Cond);
 
   MI->clearRegisterDeads(ARM::CPSR);
 
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index ca3dc15ff3ad..ef690838f0f3 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -476,8 +476,10 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
 
     LLVM_DEBUG(dbgs() << "Beginning BR iteration #" << NoBRIters << '\n');
     bool BRChange = false;
-    for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i)
+    for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i) {
+      // Note: fixupImmediateBr can append to ImmBranches.
       BRChange |= fixupImmediateBr(ImmBranches[i]);
+    }
     if (BRChange && ++NoBRIters > 30)
       report_fatal_error("Branch Fix Up pass failed to converge!");
     LLVM_DEBUG(dumpBBs());
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index 80805ff34e1b..50d8eee8644c 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -1701,8 +1701,8 @@ void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB,
                                     .addReg(ARM::SP)
                                     .setMIFlags(MachineInstr::FrameSetup)
                                     .add(predOps(ARMCC::AL));
-      for (unsigned i = 0, e = Regs.size(); i < e; ++i)
-        MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second));
+      for (const auto &[Reg, Kill] : Regs)
+        MIB.addReg(Reg, getKillRegState(Kill));
     } else if (Regs.size() == 1) {
       BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP)
           .addReg(Regs[0].first, getKillRegState(Regs[0].second))
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index b67161b06063..5378ca2a485d 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -522,65 +522,65 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
       // clang-format off
       static const struct {
         const RTLIB::Libcall Op;
-        const char * const Name;
+        const RTLIB::LibcallImpl Impl;
         const CmpInst::Predicate Cond;
       } LibraryCalls[] = {
         // Single-precision floating-point arithmetic.
-        { RTLIB::ADD_F32, "__addsf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::SUB_F32, "__subsf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::MUL_F32, "__mulsf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::DIV_F32, "__divsf3vfp", CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::ADD_F32, RTLIB::__addsf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::SUB_F32, RTLIB::__subsf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::MUL_F32, RTLIB::__mulsf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::DIV_F32, RTLIB::__divsf3vfp, CmpInst::BAD_ICMP_PREDICATE },
 
         // Double-precision floating-point arithmetic.
-        { RTLIB::ADD_F64, "__adddf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::SUB_F64, "__subdf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::MUL_F64, "__muldf3vfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::DIV_F64, "__divdf3vfp", CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::ADD_F64, RTLIB::__adddf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::SUB_F64, RTLIB::__subdf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::MUL_F64, RTLIB::__muldf3vfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::DIV_F64, RTLIB::__divdf3vfp, CmpInst::BAD_ICMP_PREDICATE },
 
         // Single-precision comparisons.
-        { RTLIB::OEQ_F32, "__eqsf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::UNE_F32, "__nesf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OLT_F32, "__ltsf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OLE_F32, "__lesf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OGE_F32, "__gesf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OGT_F32, "__gtsf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::UO_F32,  "__unordsf2vfp", CmpInst::ICMP_NE },
+        { RTLIB::OEQ_F32, RTLIB::__eqsf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::UNE_F32, RTLIB::__nesf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OLT_F32, RTLIB::__ltsf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OLE_F32, RTLIB::__lesf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OGE_F32, RTLIB::__gesf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OGT_F32, RTLIB::__gtsf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::UO_F32,  RTLIB::__unordsf2vfp, CmpInst::ICMP_NE },
 
         // Double-precision comparisons.
-        { RTLIB::OEQ_F64, "__eqdf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::UNE_F64, "__nedf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OLT_F64, "__ltdf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OLE_F64, "__ledf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OGE_F64, "__gedf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::OGT_F64, "__gtdf2vfp",    CmpInst::ICMP_NE },
-        { RTLIB::UO_F64,  "__unorddf2vfp", CmpInst::ICMP_NE },
+        { RTLIB::OEQ_F64, RTLIB::__eqdf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::UNE_F64, RTLIB::__nedf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OLT_F64, RTLIB::__ltdf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OLE_F64, RTLIB::__ledf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OGE_F64, RTLIB::__gedf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::OGT_F64, RTLIB::__gtdf2vfp,    CmpInst::ICMP_NE },
+        { RTLIB::UO_F64,  RTLIB::__unorddf2vfp, CmpInst::ICMP_NE },
 
         // Floating-point to integer conversions.
         // i64 conversions are done via library routines even when generating VFP
         // instructions, so use the same ones.
-        { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp",    CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp",    CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPTOSINT_F64_I32, RTLIB::__fixdfsivfp,    CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPTOUINT_F64_I32, RTLIB::__fixunsdfsivfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPTOSINT_F32_I32, RTLIB::__fixsfsivfp,    CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPTOUINT_F32_I32, RTLIB::__fixunssfsivfp, CmpInst::BAD_ICMP_PREDICATE },
 
         // Conversions between floating types.
-        { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp",  CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::FPEXT_F32_F64,   "__extendsfdf2vfp", CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPROUND_F64_F32, RTLIB::__truncdfsf2vfp,  CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::FPEXT_F32_F64,   RTLIB::__extendsfdf2vfp, CmpInst::BAD_ICMP_PREDICATE },
 
         // Integer to floating-point conversions.
         // i64 conversions are done via library routines even when generating VFP
         // instructions, so use the same ones.
         // FIXME: There appears to be some naming inconsistency in ARM libgcc:
         // e.g., __floatunsidf vs. __floatunssidfvfp.
-        { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp",    CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp",    CmpInst::BAD_ICMP_PREDICATE },
-        { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::SINTTOFP_I32_F64, RTLIB::__floatsidfvfp,    CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::UINTTOFP_I32_F64, RTLIB::__floatunssidfvfp, CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::SINTTOFP_I32_F32, RTLIB::__floatsisfvfp,    CmpInst::BAD_ICMP_PREDICATE },
+        { RTLIB::UINTTOFP_I32_F32, RTLIB::__floatunssisfvfp, CmpInst::BAD_ICMP_PREDICATE },
       };
       // clang-format on
 
       for (const auto &LC : LibraryCalls) {
-        setLibcallName(LC.Op, LC.Name);
+        setLibcallImpl(LC.Op, LC.Impl);
         if (LC.Cond != CmpInst::BAD_ICMP_PREDICATE)
           setCmpLibcallCC(LC.Op, LC.Cond);
       }
@@ -590,97 +590,100 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   // RTLIB
   if (TM.isAAPCS_ABI() && (TT.isTargetAEABI() || TT.isTargetGNUAEABI() ||
                            TT.isTargetMuslAEABI() || TT.isAndroid())) {
+    // FIXME: This does not depend on the subtarget and should go directly into
+    // RuntimeLibcalls. This is only here because of missing support for setting
+    // the calling convention of an implementation.
     // clang-format off
     static const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
       const CallingConv::ID CC;
       const CmpInst::Predicate Cond;
     } LibraryCalls[] = {
       // Double-precision floating-point arithmetic helper functions
       // RTABI chapter 4.1.2, Table 2
-      { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::ADD_F64, RTLIB::__aeabi_dadd, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::DIV_F64, RTLIB::__aeabi_ddiv, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::MUL_F64, RTLIB::__aeabi_dmul, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SUB_F64, RTLIB::__aeabi_dsub, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Double-precision floating-point comparison helper functions
       // RTABI chapter 4.1.2, Table 3
-      { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ },
-      { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::UO_F64,  "__aeabi_dcmpun", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::OEQ_F64, RTLIB::__aeabi_dcmpeq__ne, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::UNE_F64, RTLIB::__aeabi_dcmpeq__eq, CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ },
+      { RTLIB::OLT_F64, RTLIB::__aeabi_dcmplt, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OLE_F64, RTLIB::__aeabi_dcmple, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OGE_F64, RTLIB::__aeabi_dcmpge, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OGT_F64, RTLIB::__aeabi_dcmpgt, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::UO_F64,  RTLIB::__aeabi_dcmpun, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Single-precision floating-point arithmetic helper functions
       // RTABI chapter 4.1.2, Table 4
-      { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::ADD_F32, RTLIB::__aeabi_fadd, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::DIV_F32, RTLIB::__aeabi_fdiv, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::MUL_F32, RTLIB::__aeabi_fmul, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SUB_F32, RTLIB::__aeabi_fsub, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Single-precision floating-point comparison helper functions
       // RTABI chapter 4.1.2, Table 5
-      { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ },
-      { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
-      { RTLIB::UO_F32,  "__aeabi_fcmpun", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::OEQ_F32, RTLIB::__aeabi_fcmpeq__ne, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::UNE_F32, RTLIB::__aeabi_fcmpeq__eq, CallingConv::ARM_AAPCS, CmpInst::ICMP_EQ },
+      { RTLIB::OLT_F32, RTLIB::__aeabi_fcmplt, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OLE_F32, RTLIB::__aeabi_fcmple, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OGE_F32, RTLIB::__aeabi_fcmpge, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::OGT_F32, RTLIB::__aeabi_fcmpgt, CallingConv::ARM_AAPCS, CmpInst::ICMP_NE },
+      { RTLIB::UO_F32,  RTLIB::__aeabi_fcmpun, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Floating-point to integer conversions.
       // RTABI chapter 4.1.2, Table 6
-      { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOSINT_F64_I32, RTLIB::__aeabi_d2iz,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOUINT_F64_I32, RTLIB::__aeabi_d2uiz, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOSINT_F64_I64, RTLIB::__aeabi_d2lz,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOUINT_F64_I64, RTLIB::__aeabi_d2ulz, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOSINT_F32_I32, RTLIB::__aeabi_f2iz,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOUINT_F32_I32, RTLIB::__aeabi_f2uiz, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOSINT_F32_I64, RTLIB::__aeabi_f2lz,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPTOUINT_F32_I64, RTLIB::__aeabi_f2ulz, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Conversions between floating types.
       // RTABI chapter 4.1.2, Table 7
-      { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::FPEXT_F32_F64,   "__aeabi_f2d", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPROUND_F64_F32, RTLIB::__aeabi_d2f, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPROUND_F64_F16, RTLIB::__aeabi_d2h, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::FPEXT_F32_F64,   RTLIB::__aeabi_f2d, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Integer to floating-point conversions.
       // RTABI chapter 4.1.2, Table 8
-      { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SINTTOFP_I32_F64, RTLIB::__aeabi_i2d,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UINTTOFP_I32_F64, RTLIB::__aeabi_ui2d, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SINTTOFP_I64_F64, RTLIB::__aeabi_l2d,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UINTTOFP_I64_F64, RTLIB::__aeabi_ul2d, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SINTTOFP_I32_F32, RTLIB::__aeabi_i2f,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UINTTOFP_I32_F32, RTLIB::__aeabi_ui2f, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SINTTOFP_I64_F32, RTLIB::__aeabi_l2f,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UINTTOFP_I64_F32, RTLIB::__aeabi_ul2f, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Long long helper functions
       // RTABI chapter 4.2, Table 9
-      { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::MUL_I64, RTLIB::__aeabi_lmul, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SHL_I64, RTLIB::__aeabi_llsl, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SRL_I64, RTLIB::__aeabi_llsr, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SRA_I64, RTLIB::__aeabi_lasr, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
 
       // Integer division functions
       // RTABI chapter 4.3.1
-      { RTLIB::SDIV_I8,  "__aeabi_idiv",     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SDIV_I16, "__aeabi_idiv",     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SDIV_I32, "__aeabi_idiv",     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::SDIV_I64, "__aeabi_ldivmod",  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UDIV_I8,  "__aeabi_uidiv",    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UDIV_I16, "__aeabi_uidiv",    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UDIV_I32, "__aeabi_uidiv",    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
-      { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SDIV_I8,  RTLIB::__aeabi_idiv__i8,     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SDIV_I16, RTLIB::__aeabi_idiv__i16,     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SDIV_I32, RTLIB::__aeabi_idiv__i32,     CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::SDIV_I64, RTLIB::__aeabi_ldivmod,  CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UDIV_I8,  RTLIB::__aeabi_uidiv__i8,    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UDIV_I16, RTLIB::__aeabi_uidiv__i16,    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UDIV_I32, RTLIB::__aeabi_uidiv__i32,    CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
+      { RTLIB::UDIV_I64, RTLIB::__aeabi_uldivmod, CallingConv::ARM_AAPCS, CmpInst::BAD_ICMP_PREDICATE },
     };
     // clang-format on
 
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
       setLibcallCallingConv(LC.Op, LC.CC);
       if (LC.Cond != CmpInst::BAD_ICMP_PREDICATE)
         setCmpLibcallCC(LC.Op, LC.Cond);
@@ -691,18 +694,35 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
         TM.Options.EABIVersion == EABI::EABI5) {
       static const struct {
         const RTLIB::Libcall Op;
-        const char *const Name;
+        const RTLIB::LibcallImpl Impl;
         const CallingConv::ID CC;
       } MemOpsLibraryCalls[] = {
           // Memory operations
           // RTABI chapter 4.3.4
-          {RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS},
-          {RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS},
-          {RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS},
+          {RTLIB::MEMCPY, RTLIB::__aeabi_memcpy, CallingConv::ARM_AAPCS},
+          {RTLIB::MEMMOVE, RTLIB::__aeabi_memmove, CallingConv::ARM_AAPCS},
+          {RTLIB::MEMSET, RTLIB::__aeabi_memset, CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMCPY4, RTLIB::__aeabi_memcpy4,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMCPY8, RTLIB::__aeabi_memcpy8,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMMOVE4, RTLIB::__aeabi_memmove4,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMMOVE8, RTLIB::__aeabi_memmove8,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMSET4, RTLIB::__aeabi_memset4,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMSET8, RTLIB::__aeabi_memset8,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMCLR, RTLIB::__aeabi_memclr, CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMCLR4, RTLIB::__aeabi_memclr4,
+           CallingConv::ARM_AAPCS},
+          {RTLIB::AEABI_MEMCLR8, RTLIB::__aeabi_memclr8,
+           CallingConv::ARM_AAPCS},
       };
 
       for (const auto &LC : MemOpsLibraryCalls) {
-        setLibcallName(LC.Op, LC.Name);
+        setLibcallImpl(LC.Op, LC.Impl);
         setLibcallCallingConv(LC.Op, LC.CC);
       }
     }
@@ -726,23 +746,26 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
   // a __gnu_ prefix (which is the default).
   if (TT.isTargetAEABI()) {
+    // FIXME: This does not depend on the subtarget and should go directly into
+    // RuntimeLibcalls. This is only here because of missing support for setting
+    // the calling convention of an implementation.
     static const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
       const CallingConv::ID CC;
     } LibraryCalls[] = {
-      { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS },
-      { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS },
-      { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS },
+        {RTLIB::FPROUND_F32_F16, RTLIB::__aeabi_f2h, CallingConv::ARM_AAPCS},
+        {RTLIB::FPROUND_F64_F16, RTLIB::__aeabi_d2h, CallingConv::ARM_AAPCS},
+        {RTLIB::FPEXT_F16_F32, RTLIB::__aeabi_h2f, CallingConv::ARM_AAPCS},
     };
 
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
       setLibcallCallingConv(LC.Op, LC.CC);
     }
   } else if (!TT.isOSBinFormatMachO()) {
-    setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
-    setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
+    setLibcallImpl(RTLIB::FPROUND_F32_F16, RTLIB::__gnu_f2h_ieee);
+    setLibcallImpl(RTLIB::FPEXT_F16_F32, RTLIB::__gnu_h2f_ieee);
   }
 
   if (Subtarget->isThumb1Only())
@@ -1378,7 +1401,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
   if (Subtarget->useSjLjEH())
-    setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
+    setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
 
   setOperationAction(ISD::SETCC,     MVT::i32, Expand);
   setOperationAction(ISD::SETCC,     MVT::f32, Expand);
@@ -1520,6 +1543,21 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
     setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
 
+    if (Subtarget->hasV8Ops()) {
+      setOperationAction(ISD::FFLOOR, MVT::v2f32, Legal);
+      setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
+      setOperationAction(ISD::FROUND, MVT::v2f32, Legal);
+      setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v2f32, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v4f32, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v2f32, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v2f32, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
+      setOperationAction(ISD::FRINT, MVT::v2f32, Legal);
+      setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
+    }
+
     if (Subtarget->hasFullFP16()) {
       setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
       setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
@@ -1530,6 +1568,19 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
       setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
       setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
       setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
+
+      setOperationAction(ISD::FFLOOR, MVT::v4f16, Legal);
+      setOperationAction(ISD::FFLOOR, MVT::v8f16, Legal);
+      setOperationAction(ISD::FROUND, MVT::v4f16, Legal);
+      setOperationAction(ISD::FROUND, MVT::v8f16, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v4f16, Legal);
+      setOperationAction(ISD::FROUNDEVEN, MVT::v8f16, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v4f16, Legal);
+      setOperationAction(ISD::FCEIL, MVT::v8f16, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v4f16, Legal);
+      setOperationAction(ISD::FTRUNC, MVT::v8f16, Legal);
+      setOperationAction(ISD::FRINT, MVT::v4f16, Legal);
+      setOperationAction(ISD::FRINT, MVT::v8f16, Legal);
     }
   }
 
@@ -2743,9 +2794,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -2929,9 +2979,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const uint32_t *Mask;
@@ -5477,6 +5526,21 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue FalseVal = Op.getOperand(3);
   ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
   ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
+  ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
+  if (Op.getValueType().isInteger()) {
+    // Check for sign pattern (SELECT_CC setgt, iN lhs, -1, 1, -1) and transform
+    // into (OR (ASR lhs, N-1), 1), which requires less instructions for the
+    // supported types.
+    if (CC == ISD::SETGT && RHSC && RHSC->isAllOnes() && CTVal && CFVal &&
+        CTVal->isOne() && CFVal->isAllOnes() &&
+        LHS.getValueType() == TrueVal.getValueType()) {
+      EVT VT = LHS.getValueType();
+      SDValue Shift =
+          DAG.getNode(ISD::SRA, dl, VT, LHS,
+                      DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
+      return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT));
+    }
+  }
 
   if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
       LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
@@ -6123,9 +6187,6 @@ SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -6186,7 +6247,7 @@ static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
 
   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
                     Read.getValue(1)));
-  Results.push_back(Read.getOperand(0));
+  Results.push_back(Read.getValue(2)); // Chain
 }
 
 /// \p BC is a bitcast that is about to be turned into a VMOVDRR.
@@ -12298,6 +12359,11 @@ ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     Fn->insert(BBI, RSBBB);
     Fn->insert(BBI, SinkBB);
 
+    // Set the call frame size on entry to the new basic blocks.
+    unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
+    RSBBB->setCallFrameSize(CallFrameSize);
+    SinkBB->setCallFrameSize(CallFrameSize);
+
     Register ABSSrcReg = MI.getOperand(1).getReg();
     Register ABSDstReg = MI.getOperand(0).getReg();
     bool ABSSrcKIll = MI.getOperand(1).isKill();
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 9c330e60a7d5..604910e04d4c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -609,6 +609,11 @@ class VectorType;
 
     bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
 
+    bool hasAndNotCompare(SDValue V) const override {
+      // We can use bics for any scalar.
+      return V.getValueType().isScalarInteger();
+    }
+
     bool
     isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
     bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td
index 3335f52f1555..7485ef569445 100644
--- a/llvm/lib/Target/ARM/ARMInstrNEON.td
+++ b/llvm/lib/Target/ARM/ARMInstrNEON.td
@@ -7312,12 +7312,12 @@ multiclass VRINT_FPI<string op, bits<3> op9_7, SDPatternOperator Int> {
   }
 }
 
-defm VRINTNN : VRINT_FPI<"n", 0b000, int_arm_neon_vrintn>;
-defm VRINTXN : VRINT_FPI<"x", 0b001, int_arm_neon_vrintx>;
-defm VRINTAN : VRINT_FPI<"a", 0b010, int_arm_neon_vrinta>;
-defm VRINTZN : VRINT_FPI<"z", 0b011, int_arm_neon_vrintz>;
-defm VRINTMN : VRINT_FPI<"m", 0b101, int_arm_neon_vrintm>;
-defm VRINTPN : VRINT_FPI<"p", 0b111, int_arm_neon_vrintp>;
+defm VRINTNN : VRINT_FPI<"n", 0b000, froundeven>;
+defm VRINTXN : VRINT_FPI<"x", 0b001, frint>;
+defm VRINTAN : VRINT_FPI<"a", 0b010, fround>;
+defm VRINTZN : VRINT_FPI<"z", 0b011, ftrunc>;
+defm VRINTMN : VRINT_FPI<"m", 0b101, ffloor>;
+defm VRINTPN : VRINT_FPI<"p", 0b111, fceil>;
 
 // Cryptography instructions
 let PostEncoderMethod = "NEONThumb2DataIPostEncoder",
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 1d5c12fabf09..31650e0137be 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -1135,13 +1135,8 @@ multiclass vrint_inst_anpm<string opc, bits<2> rm,
         Requires<[HasFPARMv8,HasDPVFP]>;
 }
 
-// Match either froundeven or int_arm_neon_vrintn
-def vrintn_or_froundeven : PatFrags<(ops node:$src),
-                                    [(int_arm_neon_vrintn node:$src),
-                                     (froundeven node:$src)]>;
-
 defm VRINTA : vrint_inst_anpm<"a", 0b00, fround>;
-defm VRINTN : vrint_inst_anpm<"n", 0b01, vrintn_or_froundeven>;
+defm VRINTN : vrint_inst_anpm<"n", 0b01, froundeven>;
 defm VRINTP : vrint_inst_anpm<"p", 0b10, fceil>;
 defm VRINTM : vrint_inst_anpm<"m", 0b11, ffloor>;
 
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index 77f4782699c9..b4677a8bfb03 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -121,18 +121,20 @@ SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
     Args.push_back(Entry);
   }
 
-  char const *FunctionNames[4][3] = {
-    { "__aeabi_memcpy",  "__aeabi_memcpy4",  "__aeabi_memcpy8"  },
-    { "__aeabi_memmove", "__aeabi_memmove4", "__aeabi_memmove8" },
-    { "__aeabi_memset",  "__aeabi_memset4",  "__aeabi_memset8"  },
-    { "__aeabi_memclr",  "__aeabi_memclr4",  "__aeabi_memclr8"  }
-  };
+  static const RTLIB::Libcall FunctionImpls[4][3] = {
+      {RTLIB::MEMCPY, RTLIB::AEABI_MEMCPY4, RTLIB::AEABI_MEMCPY8},
+      {RTLIB::MEMMOVE, RTLIB::AEABI_MEMMOVE4, RTLIB::AEABI_MEMMOVE8},
+      {RTLIB::MEMSET, RTLIB::AEABI_MEMSET4, RTLIB::AEABI_MEMSET8},
+      {RTLIB::AEABI_MEMCLR, RTLIB::AEABI_MEMCLR4, RTLIB::AEABI_MEMCLR8}};
+
+  RTLIB::Libcall NewLC = FunctionImpls[AEABILibcall][AlignVariant];
+
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl)
       .setChain(Chain)
       .setLibCallee(
-          TLI->getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()),
-          DAG.getExternalSymbol(FunctionNames[AEABILibcall][AlignVariant],
+          TLI->getLibcallCallingConv(NewLC), Type::getVoidTy(*DAG.getContext()),
+          DAG.getExternalSymbol(TLI->getLibcallName(NewLC),
                                 TLI->getPointerTy(DAG.getDataLayout())),
           std::move(Args))
       .setDiscardResult();
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 25f027301337..ec6f4e2ae216 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -3629,7 +3629,7 @@ class ARMOperand : public MCParsedAsmOperand {
     Inst.addOperand(MCOperand::createImm(Imm == 48 ? 1 : 0));
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<ARMOperand> CreateITMask(unsigned Mask, SMLoc S,
                                                   ARMAsmParser &Parser) {
@@ -3979,7 +3979,7 @@ class ARMOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void ARMOperand::print(raw_ostream &OS) const {
+void ARMOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   auto RegName = [](MCRegister Reg) {
     if (Reg)
       return ARMInstPrinter::getRegisterName(Reg);
@@ -4024,7 +4024,7 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << "<banked reg: " << getBankedReg() << ">";
     break;
   case k_Immediate:
-    OS << *getImm();
+    MAI.printExpr(OS, *getImm());
     break;
   case k_MemBarrierOpt:
     OS << "<ARM_MB::" << MemBOptToString(getMemBarrierOpt(), false) << ">";
@@ -4039,8 +4039,10 @@ void ARMOperand::print(raw_ostream &OS) const {
     OS << "<memory";
     if (Memory.BaseRegNum)
       OS << " base:" << RegName(Memory.BaseRegNum);
-    if (Memory.OffsetImm)
-      OS << " offset-imm:" << *Memory.OffsetImm;
+    if (Memory.OffsetImm) {
+      OS << " offset-imm:";
+      MAI.printExpr(OS, *Memory.OffsetImm);
+    }
     if (Memory.OffsetRegNum)
       OS << " offset-reg:" << (Memory.isNegative ? "-" : "")
          << RegName(Memory.OffsetRegNum);
@@ -4094,7 +4096,8 @@ void ARMOperand::print(raw_ostream &OS) const {
        <<  ModImm.Rot << ")>";
     break;
   case k_ConstantPoolImmediate:
-    OS << "<constant_pool_imm #" << *getConstantPoolImm();
+    OS << "<constant_pool_imm #";
+    MAI.printExpr(OS, *getConstantPoolImm());
     break;
   case k_BitfieldDescriptor:
     OS << "<bitfield " << "lsb: " << Bitfield.LSB
@@ -6433,7 +6436,7 @@ bool ARMAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic) {
       return true;
 
     const auto *ExprVal =
-        MCSpecifierExpr::create(SubExprVal, Spec, getContext());
+        MCSpecifierExpr::create(SubExprVal, Spec, getContext(), S);
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ExprVal, S, E, *this));
     return false;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index f43fdae554b8..fc9a32072a62 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -70,44 +69,30 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // ARMFixupKinds.h.
       //
       // Name                      Offset (bits) Size (bits)     Flags
-      {"fixup_arm_ldst_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_ldst_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_10_unscaled", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_10", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_9", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_9", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
+      {"fixup_arm_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_t2_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_arm_pcrel_10_unscaled", 0, 32, 0},
+      {"fixup_arm_pcrel_10", 0, 32, 0},
+      {"fixup_t2_pcrel_10", 0, 32, 0},
+      {"fixup_arm_pcrel_9", 0, 32, 0},
+      {"fixup_t2_pcrel_9", 0, 32, 0},
       {"fixup_arm_ldst_abs_12", 0, 32, 0},
-      {"fixup_thumb_adr_pcrel_10", 0, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_adr_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_adr_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_condbranch", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbranch", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_condbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_uncondbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_br", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbl", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_condbl", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_blx", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_bl", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_blx", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_cb", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_cp", 0, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_bcc", 0, 8, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_thumb_adr_pcrel_10", 0, 8, 0},
+      {"fixup_arm_adr_pcrel_12", 0, 32, 0},
+      {"fixup_t2_adr_pcrel_12", 0, 32, 0},
+      {"fixup_arm_condbranch", 0, 24, 0},
+      {"fixup_arm_uncondbranch", 0, 24, 0},
+      {"fixup_t2_condbranch", 0, 32, 0},
+      {"fixup_t2_uncondbranch", 0, 32, 0},
+      {"fixup_arm_thumb_br", 0, 16, 0},
+      {"fixup_arm_uncondbl", 0, 24, 0},
+      {"fixup_arm_condbl", 0, 24, 0},
+      {"fixup_arm_blx", 0, 24, 0},
+      {"fixup_arm_thumb_bl", 0, 32, 0},
+      {"fixup_arm_thumb_blx", 0, 32, 0},
+      {"fixup_arm_thumb_cb", 0, 16, 0},
+      {"fixup_arm_thumb_cp", 0, 8, 0},
+      {"fixup_arm_thumb_bcc", 0, 8, 0},
       // movw / movt: 16-bits immediate but scattered into two chunks 0 - 12, 16
       // - 19.
       {"fixup_arm_movt_hi16", 0, 20, 0},
@@ -120,56 +105,43 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_arm_thumb_lower_0_7", 0, 8, 0},
       {"fixup_arm_mod_imm", 0, 12, 0},
       {"fixup_t2_so_imm", 0, 26, 0},
-      {"fixup_bf_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bf_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfl_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfc_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_bf_branch", 0, 32, 0},
+      {"fixup_bf_target", 0, 32, 0},
+      {"fixup_bfl_target", 0, 32, 0},
+      {"fixup_bfc_target", 0, 32, 0},
       {"fixup_bfcsel_else_target", 0, 32, 0},
-      {"fixup_wls", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_le", 0, 32, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_wls", 0, 32, 0},
+      {"fixup_le", 0, 32, 0},
+  };
   const static MCFixupKindInfo InfosBE[ARM::NumTargetFixupKinds] = {
       // This table *must* be in the order that the fixup_* kinds are defined in
       // ARMFixupKinds.h.
       //
       // Name                      Offset (bits) Size (bits)     Flags
-      {"fixup_arm_ldst_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_ldst_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_10_unscaled", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_pcrel_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_10", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_pcrel_9", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_pcrel_9", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
+      {"fixup_arm_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_t2_ldst_pcrel_12", 0, 32, 0},
+      {"fixup_arm_pcrel_10_unscaled", 0, 32, 0},
+      {"fixup_arm_pcrel_10", 0, 32, 0},
+      {"fixup_t2_pcrel_10", 0, 32, 0},
+      {"fixup_arm_pcrel_9", 0, 32, 0},
+      {"fixup_t2_pcrel_9", 0, 32, 0},
       {"fixup_arm_ldst_abs_12", 0, 32, 0},
-      {"fixup_thumb_adr_pcrel_10", 8, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_adr_pcrel_12", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_adr_pcrel_12", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_condbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbranch", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_condbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_t2_uncondbranch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_br", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_uncondbl", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_condbl", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_blx", 8, 24, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_bl", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_blx", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_cb", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_arm_thumb_cp", 8, 8,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_arm_thumb_bcc", 8, 8, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_thumb_adr_pcrel_10", 8, 8, 0},
+      {"fixup_arm_adr_pcrel_12", 0, 32, 0},
+      {"fixup_t2_adr_pcrel_12", 0, 32, 0},
+      {"fixup_arm_condbranch", 8, 24, 0},
+      {"fixup_arm_uncondbranch", 8, 24, 0},
+      {"fixup_t2_condbranch", 0, 32, 0},
+      {"fixup_t2_uncondbranch", 0, 32, 0},
+      {"fixup_arm_thumb_br", 0, 16, 0},
+      {"fixup_arm_uncondbl", 8, 24, 0},
+      {"fixup_arm_condbl", 8, 24, 0},
+      {"fixup_arm_blx", 8, 24, 0},
+      {"fixup_arm_thumb_bl", 0, 32, 0},
+      {"fixup_arm_thumb_blx", 0, 32, 0},
+      {"fixup_arm_thumb_cb", 0, 16, 0},
+      {"fixup_arm_thumb_cp", 8, 8, 0},
+      {"fixup_arm_thumb_bcc", 8, 8, 0},
       // movw / movt: 16-bits immediate but scattered into two chunks 0 - 12, 16
       // - 19.
       {"fixup_arm_movt_hi16", 12, 20, 0},
@@ -182,18 +154,19 @@ MCFixupKindInfo ARMAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_arm_thumb_lower_0_7", 24, 8, 0},
       {"fixup_arm_mod_imm", 20, 12, 0},
       {"fixup_t2_so_imm", 26, 6, 0},
-      {"fixup_bf_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bf_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfl_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_bfc_target", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_bf_branch", 0, 32, 0},
+      {"fixup_bf_target", 0, 32, 0},
+      {"fixup_bfl_target", 0, 32, 0},
+      {"fixup_bfc_target", 0, 32, 0},
       {"fixup_bfcsel_else_target", 0, 32, 0},
-      {"fixup_wls", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_le", 0, 32, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_wls", 0, 32, 0},
+      {"fixup_le", 0, 32, 0},
+  };
 
   // Fixup kinds from .reloc directive are like R_ARM_NONE. They do not require
   // any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -356,15 +329,7 @@ bool ARMAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
 void ARMAsmBackend::relaxInstruction(MCInst &Inst,
                                      const MCSubtargetInfo &STI) const {
   unsigned RelaxedOp = getRelaxedOpcode(Inst.getOpcode(), STI);
-
-  // Return a diagnostic if we get here w/ a bogus instruction.
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
+  assert(RelaxedOp != Inst.getOpcode());
 
   // If we are changing Thumb CBZ or CBNZ instruction to a NOP, aka tHINT, we
   // have to change the operands too.
@@ -1124,10 +1089,32 @@ static unsigned getFixupKindContainerSizeBytes(unsigned Kind) {
   }
 }
 
+std::optional<bool> ARMAsmBackend::evaluateFixup(const MCFragment &F,
+                                                 MCFixup &Fixup, MCValue &,
+                                                 uint64_t &Value) {
+  // For a few PC-relative fixups in Thumb mode, offsets need to be aligned
+  // down. We compensate here because the default handler's `Value` decrement
+  // doesn't account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case ARM::fixup_t2_ldst_pcrel_12:
+  case ARM::fixup_t2_pcrel_10:
+  case ARM::fixup_t2_pcrel_9:
+  case ARM::fixup_thumb_adr_pcrel_10:
+  case ARM::fixup_t2_adr_pcrel_12:
+  case ARM::fixup_arm_thumb_blx:
+  case ARM::fixup_arm_thumb_cp:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
 void ARMAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   auto Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 5efc969641b9..52279a3b6936 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -30,14 +30,15 @@ class ARMAsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   unsigned adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup,
                             const MCValue &Target, uint64_t Value,
                             bool IsResolved, MCContext &Ctx,
                             const MCSubtargetInfo *STI) const;
 
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index 73ad62ed7953..c61e405bd3a0 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -32,7 +32,6 @@
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
@@ -1210,8 +1209,7 @@ inline void ARMELFStreamer::SwitchToExIdxSection(const MCSymbol &FnStart) {
 
 void ARMELFStreamer::EmitFixup(const MCExpr *Expr, MCFixupKind Kind) {
   MCDataFragment *Frag = getOrCreateDataFragment();
-  Frag->getFixups().push_back(MCFixup::create(Frag->getContents().size(), Expr,
-                                              Kind));
+  Frag->addFixup(MCFixup::create(Frag->getContents().size(), Expr, Kind));
 }
 
 void ARMELFStreamer::EHReset() {
@@ -1299,7 +1297,7 @@ void ARMELFStreamer::EmitPersonalityFixup(StringRef Name) {
 
   visitUsedExpr(*PersonalityRef);
   MCDataFragment *DF = getOrCreateDataFragment();
-  DF->getFixups().push_back(
+  DF->addFixup(
       MCFixup::create(DF->getContents().size(), PersonalityRef, FK_Data_4));
 }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index a3d86f690e4a..e9f4b72f4e52 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -17,7 +17,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {ARM::S_GOT_PREL, "GOT_PREL"},
     {ARM::S_ARM_NONE, "none"},
     {ARM::S_PREL31, "prel31"},
@@ -34,7 +34,7 @@ const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
     {ARM::S_GOTTPOFF, "GOTTPOFF"},
     {ARM::S_GOTTPOFF_FDPIC, "gottpoff_fdpic"},
     {ARM::S_PLT, "PLT"},
-    {MCSymbolRefExpr::VK_SECREL, "SECREL32"},
+    {ARM::S_COFF_SECREL, "SECREL32"},
     {ARM::S_TLSCALL, "tlscall"},
     {ARM::S_TLSDESC, "tlsdesc"},
     {ARM::S_TLSGD, "TLSGD"},
@@ -65,7 +65,7 @@ ARMMCAsmInfoDarwin::ARMMCAsmInfoDarwin(const Triple &TheTriple) {
                        ? ExceptionHandling::SjLj
                        : ExceptionHandling::DwarfCFI;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void ARMELFMCAsmInfo::anchor() { }
@@ -100,7 +100,7 @@ ARMELFMCAsmInfo::ARMELFMCAsmInfo(const Triple &TheTriple) {
   UseAtForSpecifier = false;
   UseParensForSpecifier = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void ARMELFMCAsmInfo::setUseIntegratedAssembler(bool Value) {
@@ -127,7 +127,7 @@ ARMCOFFMCAsmInfoMicrosoft::ARMCOFFMCAsmInfoMicrosoft() {
   // Conditional Thumb 4-byte instructions can have an implicit IT.
   MaxInstLength = 6;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void ARMCOFFMCAsmInfoGNU::anchor() { }
@@ -151,7 +151,7 @@ ARMCOFFMCAsmInfoGNU::ARMCOFFMCAsmInfoGNU() {
   // Conditional Thumb 4-byte instructions can have an implicit IT.
   MaxInstLength = 6;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void ARM::printSpecifierExpr(const MCAsmInfo &MAI, raw_ostream &OS,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
index f3f075e99d96..8b24e7b5d801 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
@@ -92,6 +92,8 @@ namespace ARM {
 using Specifier = uint16_t;
 enum {
   S_None,
+  S_COFF_SECREL,
+
   S_HI16 =
       MCSymbolRefExpr::FirstTargetSpecifier, // The R_ARM_MOVT_ABS relocation
                                              // (:upper16: in the .s file)
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index fba32eae4dfa..c56b58951953 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -313,41 +313,12 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
 
   unsigned getModImmOpValue(const MCInst &MI, unsigned Op,
                             SmallVectorImpl<MCFixup> &Fixups,
-                            const MCSubtargetInfo &ST) const {
-    const MCOperand &MO = MI.getOperand(Op);
-
-    // Support for fixups (MCFixup)
-    if (MO.isExpr()) {
-      const MCExpr *Expr = MO.getExpr();
-      // Fixups resolve to plain values that need to be encoded.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_mod_imm);
-      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-      return 0;
-    }
-
-    // Immediate is already in its encoded format
-    return MO.getImm();
-  }
+                            const MCSubtargetInfo &ST) const;
 
   /// getT2SOImmOpValue - Return an encoded 12-bit shifted-immediate value.
   unsigned getT2SOImmOpValue(const MCInst &MI, unsigned Op,
-                           SmallVectorImpl<MCFixup> &Fixups,
-                           const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Op);
-
-    // Support for fixups (MCFixup)
-    if (MO.isExpr()) {
-      const MCExpr *Expr = MO.getExpr();
-      // Fixups resolve to plain values that need to be encoded.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_t2_so_imm);
-      Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
-      return 0;
-    }
-    unsigned SoImm = MO.getImm();
-    unsigned Encoded =  ARM_AM::getT2SOImmVal(SoImm);
-    assert(Encoded != ~0U && "Not a Thumb2 so_imm value?");
-    return Encoded;
-  }
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
 
   unsigned getT2AddrModeSORegOpValue(const MCInst &MI, unsigned OpNum,
     SmallVectorImpl<MCFixup> &Fixups,
@@ -462,6 +433,44 @@ class ARMMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ARM::fixup_arm_ldst_pcrel_12:
+  case ARM::fixup_t2_ldst_pcrel_12:
+  case ARM::fixup_arm_pcrel_10_unscaled:
+  case ARM::fixup_arm_pcrel_10:
+  case ARM::fixup_t2_pcrel_10:
+  case ARM::fixup_arm_pcrel_9:
+  case ARM::fixup_t2_pcrel_9:
+  case ARM::fixup_thumb_adr_pcrel_10:
+  case ARM::fixup_arm_adr_pcrel_12:
+  case ARM::fixup_t2_adr_pcrel_12:
+  case ARM::fixup_arm_condbranch:
+  case ARM::fixup_arm_uncondbranch:
+  case ARM::fixup_t2_condbranch:
+  case ARM::fixup_t2_uncondbranch:
+  case ARM::fixup_arm_thumb_br:
+  case ARM::fixup_arm_uncondbl:
+  case ARM::fixup_arm_condbl:
+  case ARM::fixup_arm_blx:
+  case ARM::fixup_arm_thumb_bl:
+  case ARM::fixup_arm_thumb_blx:
+  case ARM::fixup_arm_thumb_cb:
+  case ARM::fixup_arm_thumb_cp:
+  case ARM::fixup_arm_thumb_bcc:
+  case ARM::fixup_bf_branch:
+  case ARM::fixup_bf_target:
+  case ARM::fixup_bfl_target:
+  case ARM::fixup_bfc_target:
+  case ARM::fixup_wls:
+  case ARM::fixup_le:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// NEONThumb2DataIPostEncoder - Post-process encoded NEON data-processing
 /// instructions, and rewrite them to their Thumb2 form if we are currently in
 /// Thumb2 mode.
@@ -616,7 +625,7 @@ static uint32_t getBranchTargetOpValue(const MCInst &MI, unsigned OpIdx,
   assert(MO.isExpr() && "Unexpected branch target type!");
   const MCExpr *Expr = MO.getExpr();
   MCFixupKind Kind = MCFixupKind(FixupKind);
-  Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Kind);
 
   // All of the information is in the fixup.
   return 0;
@@ -978,18 +987,18 @@ getAddrModeImm12OpValue(const MCInst &MI, unsigned OpIdx,
              "Thumb mode requires different encoding");
       Reg = CTX.getRegisterInfo()->getEncodingValue(MO.getReg());
       isAdd = false; // 'U' bit is set as part of the fixup.
-      MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_ldst_abs_12);
-      Fixups.push_back(MCFixup::create(0, MO1.getExpr(), Kind, MI.getLoc()));
+      MCFixupKind Kind = ARM::fixup_arm_ldst_abs_12;
+      addFixup(Fixups, 0, MO1.getExpr(), Kind);
     }
   } else if (MO.isExpr()) {
     Reg = CTX.getRegisterInfo()->getEncodingValue(ARM::PC); // Rn is PC.
     isAdd = false; // 'U' bit is set as part of the fixup.
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_ldst_pcrel_12);
+      Kind = ARM::fixup_t2_ldst_pcrel_12;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_ldst_pcrel_12);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_ldst_pcrel_12;
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1113,8 +1122,8 @@ getT2AddrModeImm8s4OpValue(const MCInst &MI, unsigned OpIdx,
 
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_t2_pcrel_10;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else
@@ -1232,26 +1241,26 @@ uint32_t ARMMCCodeEmitter::getHiLoImmOpValue(const MCInst &MI, unsigned OpIdx,
     case ARM::S_HI_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":upper_8_15: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_8_15);
+      Kind = ARM::fixup_arm_thumb_upper_8_15;
       break;
     case ARM::S_HI_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":upper_0_7: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_upper_0_7);
+      Kind = ARM::fixup_arm_thumb_upper_0_7;
       break;
     case ARM::S_LO_8_15:
       if (!isThumb(STI))
         llvm_unreachable(":lower_8_15: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_8_15);
+      Kind = ARM::fixup_arm_thumb_lower_8_15;
       break;
     case ARM::S_LO_0_7:
       if (!isThumb(STI))
         llvm_unreachable(":lower_0_7: not supported in Arm state");
-      Kind = MCFixupKind(ARM::fixup_arm_thumb_lower_0_7);
+      Kind = ARM::fixup_arm_thumb_lower_0_7;
       break;
     }
 
-    Fixups.push_back(MCFixup::create(0, E, Kind, MI.getLoc()));
+    addFixup(Fixups, 0, E, Kind);
     return 0;
   }
   // If the expression doesn't have :upper16:, :lower16: on it, it's just a
@@ -1372,8 +1381,8 @@ getAddrMode3OpValue(const MCInst &MI, unsigned OpIdx,
 
     assert(MO.isExpr() && "Unexpected machine operand type!");
     const MCExpr *Expr = MO.getExpr();
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_arm_pcrel_10_unscaled);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_arm_pcrel_10_unscaled;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
     return (Rn << 9) | (1 << 13);
@@ -1452,10 +1461,10 @@ getAddrMode5OpValue(const MCInst &MI, unsigned OpIdx,
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_pcrel_10);
+      Kind = ARM::fixup_t2_pcrel_10;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_pcrel_10);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_pcrel_10;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1492,10 +1501,10 @@ getAddrMode5FP16OpValue(const MCInst &MI, unsigned OpIdx,
     const MCExpr *Expr = MO.getExpr();
     MCFixupKind Kind;
     if (isThumb2(STI))
-      Kind = MCFixupKind(ARM::fixup_t2_pcrel_9);
+      Kind = ARM::fixup_t2_pcrel_9;
     else
-      Kind = MCFixupKind(ARM::fixup_arm_pcrel_9);
-    Fixups.push_back(MCFixup::create(0, Expr, Kind, MI.getLoc()));
+      Kind = ARM::fixup_arm_pcrel_9;
+    addFixup(Fixups, 0, Expr, Kind);
 
     ++MCNumCPRelocations;
   } else {
@@ -1511,6 +1520,43 @@ getAddrMode5FP16OpValue(const MCInst &MI, unsigned OpIdx,
   return Binary;
 }
 
+unsigned ARMMCCodeEmitter::getModImmOpValue(const MCInst &MI, unsigned Op,
+                                            SmallVectorImpl<MCFixup> &Fixups,
+                                            const MCSubtargetInfo &ST) const {
+  const MCOperand &MO = MI.getOperand(Op);
+
+  // Support for fixups (MCFixup)
+  if (MO.isExpr()) {
+    const MCExpr *Expr = MO.getExpr();
+    // Fixups resolve to plain values that need to be encoded.
+    MCFixupKind Kind = ARM::fixup_arm_mod_imm;
+    addFixup(Fixups, 0, Expr, Kind);
+    return 0;
+  }
+
+  // Immediate is already in its encoded format
+  return MO.getImm();
+}
+
+unsigned ARMMCCodeEmitter::getT2SOImmOpValue(const MCInst &MI, unsigned Op,
+                                             SmallVectorImpl<MCFixup> &Fixups,
+                                             const MCSubtargetInfo &STI) const {
+  const MCOperand &MO = MI.getOperand(Op);
+
+  // Support for fixups (MCFixup)
+  if (MO.isExpr()) {
+    const MCExpr *Expr = MO.getExpr();
+    // Fixups resolve to plain values that need to be encoded.
+    MCFixupKind Kind = ARM::fixup_t2_so_imm;
+    addFixup(Fixups, 0, Expr, Kind);
+    return 0;
+  }
+  unsigned SoImm = MO.getImm();
+  unsigned Encoded = ARM_AM::getT2SOImmVal(SoImm);
+  assert(Encoded != ~0U && "Not a Thumb2 so_imm value?");
+  return Encoded;
+}
+
 unsigned ARMMCCodeEmitter::
 getSORegRegOpValue(const MCInst &MI, unsigned OpIdx,
                 SmallVectorImpl<MCFixup> &Fixups,
@@ -1949,8 +1995,8 @@ ARMMCCodeEmitter::getBFAfterTargetOpValue(const MCInst &MI, unsigned OpIdx,
     assert(BranchMO.isExpr());
     const MCExpr *DiffExpr = MCBinaryExpr::createSub(
         MO.getExpr(), BranchMO.getExpr(), CTX);
-    MCFixupKind Kind = MCFixupKind(ARM::fixup_bfcsel_else_target);
-    Fixups.push_back(llvm::MCFixup::create(0, DiffExpr, Kind, MI.getLoc()));
+    MCFixupKind Kind = ARM::fixup_bfcsel_else_target;
+    addFixup(Fixups, 0, DiffExpr, Kind);
     return 0;
   }
 
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index f6108ede4270..c0c40ade5810 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -144,7 +144,7 @@ void ARMMachObjectWriter::recordARMScatteredHalfRelocation(
     return;
   }
 
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Type = MachO::ARM_RELOC_HALF;
 
   // See <reloc.h>.
@@ -251,7 +251,7 @@ void ARMMachObjectWriter::recordARMScatteredRelocation(
     return;
   }
 
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
 
   // See <reloc.h>.
   const MCSymbol *A = Target.getAddSym();
@@ -358,7 +358,7 @@ void ARMMachObjectWriter::recordRelocation(MachObjectWriter *Writer,
                                            const MCFragment *Fragment,
                                            const MCFixup &Fixup, MCValue Target,
                                            uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Log2Size;
   unsigned RelocType = MachO::ARM_RELOC_VANILLA;
   if (!getARMFixupKindMachOInfo(Fixup.getKind(), RelocType, Log2Size)) {
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
index bfcd5e8a9b3b..5fef2c0a0bdc 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/ARMFixupKinds.h"
+#include "MCTargetDesc/ARMMCAsmInfo.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/MC/MCAsmBackend.h"
@@ -46,12 +47,14 @@ unsigned ARMWinCOFFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCAsmBackend &MAB) const {
   auto Spec = Target.getSpecifier();
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = false;
   if (IsCrossSection) {
-    if (FixupKind != FK_Data_4) {
+    if (PCRel || FixupKind != FK_Data_4) {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_ARM_ADDR32;
     }
-    FixupKind = FK_PCRel_4;
+    FixupKind = FK_Data_4;
+    PCRel = true;
   }
 
 
@@ -61,16 +64,16 @@ unsigned ARMWinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     return COFF::IMAGE_REL_ARM_ABSOLUTE;
   }
   case FK_Data_4:
+    if (PCRel)
+      return COFF::IMAGE_REL_ARM_REL32;
     switch (Spec) {
     case MCSymbolRefExpr::VK_COFF_IMGREL32:
       return COFF::IMAGE_REL_ARM_ADDR32NB;
-    case MCSymbolRefExpr::VK_SECREL:
+    case ARM::S_COFF_SECREL:
       return COFF::IMAGE_REL_ARM_SECREL;
     default:
       return COFF::IMAGE_REL_ARM_ADDR32;
     }
-  case FK_PCRel_4:
-    return COFF::IMAGE_REL_ARM_REL32;
   case FK_SecRel_2:
     return COFF::IMAGE_REL_ARM_SECTION;
   case FK_SecRel_4:
diff --git a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index 012cf2c70e2e..4e00b192b875 100644
--- a/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/llvm/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -247,7 +247,7 @@ class AVROperand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &O) const override {
+  void print(raw_ostream &O, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:
       O << "Token: \"" << getToken() << "\"";
@@ -256,12 +256,16 @@ class AVROperand : public MCParsedAsmOperand {
       O << "Register: " << getReg();
       break;
     case k_Immediate:
-      O << "Immediate: \"" << *getImm() << "\"";
+      O << "Immediate: \"";
+      MAI.printExpr(O, *getImm());
+      O << "\"";
       break;
     case k_Memri: {
       // only manually print the size for non-negative values,
       // as the sign is inserted automatically.
-      O << "Memri: \"" << getReg() << '+' << *getImm() << "\"";
+      O << "Memri: \"" << getReg() << '+';
+      MAI.printExpr(O, *getImm());
+      O << "\"";
       break;
     }
     }
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index 41341387b42c..db09738bfd22 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -19,7 +19,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -368,26 +367,21 @@ AVRAsmBackend::createObjectTargetWriter() const {
   return createAVRELFObjectWriter(MCELFObjectTargetWriter::getOSABI(OSType));
 }
 
-bool AVRAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
-                             const MCValue &Target, uint64_t &FixedValue,
-                             bool IsResolved) {
+void AVRAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &Target,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
   // AVR sets the fixup value to bypass the assembly time overflow with a
   // relocation.
   if (IsResolved) {
-    auto TargetVal = MCValue::get(Target.getAddSym(), Target.getSubSym(),
-                                  FixedValue, Target.getSpecifier());
+    auto TargetVal = MCValue::get(Target.getAddSym(), Target.getSubSym(), Value,
+                                  Target.getSpecifier());
     if (forceRelocation(F, Fixup, TargetVal))
       IsResolved = false;
   }
   if (!IsResolved)
-    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
-  return IsResolved;
-}
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
 
-void AVRAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
-                               const MCValue &Target,
-                               MutableArrayRef<char> Data, uint64_t Value,
-                               bool IsResolved) {
   if (mc::isRelocation(Fixup.getKind()))
     return;
   adjustFixupValue(Fixup, Target, Value, &getContext());
@@ -439,8 +433,8 @@ MCFixupKindInfo AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // name                    offset  bits  flags
       {"fixup_32", 0, 32, 0},
 
-      {"fixup_7_pcrel", 3, 7, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_13_pcrel", 0, 12, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_7_pcrel", 3, 7, 0},
+      {"fixup_13_pcrel", 0, 12, 0},
 
       {"fixup_16", 0, 16, 0},
       {"fixup_16_pm", 0, 16, 0},
@@ -491,7 +485,7 @@ MCFixupKindInfo AVRAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_AVR_NONE. They do not require
   // any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index 283f937e1ad8..68c839ec8432 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -37,9 +37,6 @@ class AVRAsmBackend : public MCAsmBackend {
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
 
-  bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
-
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
index 1e4b2e27a183..481219164a0f 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp
@@ -166,7 +166,7 @@ void AVRInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
     O << Imm;
   } else {
     assert(Op.isExpr() && "Unknown pcrel immediate operand");
-    O << *Op.getExpr();
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -189,7 +189,7 @@ void AVRInstPrinter::printMemri(const MCInst *MI, unsigned OpNo,
 
     O << Offset;
   } else if (OffsetOp.isExpr()) {
-    O << *OffsetOp.getExpr();
+    MAI.printExpr(O, *OffsetOp.getExpr());
   } else {
     llvm_unreachable("unknown type for offset");
   }
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
index cfd7dc582262..2ba8383e48e8 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
@@ -195,7 +195,7 @@ bool AVRMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
       return false;
 
     auto Spec = AVR::S_None;
-    if (Value.getSpecifier() != MCSymbolRefExpr::VK_None)
+    if (Value.getSpecifier())
       return false;
     assert(!Value.getSubSym());
     if (E.getSpecifier() == AVR::S_PM)
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index 4934e1c71bc0..4bb16e237db4 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -35,6 +35,17 @@
 
 namespace llvm {
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case AVR::fixup_7_pcrel:
+  case AVR::fixup_13_pcrel:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 /// Performs a post-encoding step on a `LD` or `ST` instruction.
 ///
 /// The encoding of the LD/ST family of instructions is inconsistent w.r.t
@@ -110,8 +121,7 @@ AVRMCCodeEmitter::encodeRelCondBrTarget(const MCInst &MI, unsigned OpNo,
   const MCOperand &MO = MI.getOperand(OpNo);
 
   if (MO.isExpr()) {
-    Fixups.push_back(
-        MCFixup::create(0, MO.getExpr(), MCFixupKind(Fixup), MI.getLoc()));
+    addFixup(Fixups, 0, MO.getExpr(), MCFixupKind(Fixup));
     return 0;
   }
 
@@ -156,8 +166,7 @@ unsigned AVRMCCodeEmitter::encodeMemri(const MCInst &MI, unsigned OpNo,
     OffsetBits = OffsetOp.getImm();
   } else if (OffsetOp.isExpr()) {
     OffsetBits = 0;
-    Fixups.push_back(MCFixup::create(0, OffsetOp.getExpr(),
-                                     MCFixupKind(AVR::fixup_6), MI.getLoc()));
+    addFixup(Fixups, 0, OffsetOp.getExpr(), AVR::fixup_6);
   } else {
     llvm_unreachable("Invalid value for offset");
   }
@@ -191,8 +200,7 @@ unsigned AVRMCCodeEmitter::encodeImm(const MCInst &MI, unsigned OpNo,
     }
 
     MCFixupKind FixupKind = static_cast<MCFixupKind>(Fixup);
-    Fixups.push_back(
-        MCFixup::create(Offset, MO.getExpr(), FixupKind, MI.getLoc()));
+    addFixup(Fixups, Offset, MO.getExpr(), FixupKind);
 
     return 0;
   }
@@ -207,8 +215,8 @@ unsigned AVRMCCodeEmitter::encodeCallTarget(const MCInst &MI, unsigned OpNo,
   auto MO = MI.getOperand(OpNo);
 
   if (MO.isExpr()) {
-    MCFixupKind FixupKind = static_cast<MCFixupKind>(AVR::fixup_call);
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), FixupKind, MI.getLoc()));
+    MCFixupKind FixupKind = AVR::fixup_call;
+    addFixup(Fixups, 0, MO.getExpr(), FixupKind);
     return 0;
   }
 
@@ -238,7 +246,7 @@ unsigned AVRMCCodeEmitter::getExprOpValue(const MCExpr *Expr,
     }
 
     MCFixupKind FixupKind = static_cast<MCFixupKind>(AVRExpr->getFixupKind());
-    Fixups.push_back(MCFixup::create(0, AVRExpr, FixupKind));
+    addFixup(Fixups, 0, AVRExpr, FixupKind);
     return 0;
   }
 
diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
index 0644f422b328..1f3a75c1492a 100644
--- a/llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
+++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
@@ -36,10 +36,8 @@ void AVRMCELFStreamer::emitValueForModiferKind(
     Kind = AVR::S_HI8;
   else if (ModifierKind == AVR::S_HH8)
     Kind = AVR::S_HH8;
-  MCELFStreamer::emitValue(
-      MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VariantKind(Kind),
-                              getContext()),
-      SizeInBytes, Loc);
+  MCELFStreamer::emitValue(MCSymbolRefExpr::create(Sym, Kind, getContext()),
+                           SizeInBytes, Loc);
 }
 
 namespace llvm {
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index b49e8fd96c66..a347794a9a30 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/BPFMCAsmInfo.h"
 #include "MCTargetDesc/BPFMCTargetDesc.h"
 #include "TargetInfo/BPFTargetInfo.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -164,10 +165,10 @@ struct BPFOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case Register:
       OS << "<register x";
@@ -347,6 +348,9 @@ bool BPFAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSImm16:
     return Error(Operands[ErrorInfo]->getStartLoc(),
                  "operand is not a 16-bit signed integer");
+  case Match_InvalidTiedOperand:
+    return Error(Operands[ErrorInfo]->getStartLoc(),
+                 "operand is not the same as the dst register");
   }
 
   llvm_unreachable("Unknown match type detected!");
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 1704fecaff03..958790d49d08 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -12,7 +12,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/Support/EndianStream.h"
 #include <cassert>
@@ -44,7 +43,7 @@ class BPFAsmBackend : public MCAsmBackend {
 
 MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[BPF::NumTargetFixupKinds] = {
-    { "FK_BPF_PCRel_4",  0, 32, MCFixupKindInfo::FKF_IsPCRel },
+      {"FK_BPF_PCRel_4", 0, 32, 0},
   };
 
   if (Kind < FirstTargetFixupKind)
@@ -66,10 +65,11 @@ bool BPFAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void BPFAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target,
                                MutableArrayRef<char> Data, uint64_t Value,
                                bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (Fixup.getKind() == FK_SecRel_8) {
     // The Value is 0 for global variables, and the in-section offset
     // for static variables. Write to the immediate field of the inst.
@@ -77,11 +77,11 @@ void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
     support::endian::write<uint32_t>(&Data[Fixup.getOffset() + 4],
                                      static_cast<uint32_t>(Value),
                                      Endian);
-  } else if (Fixup.getKind() == FK_Data_4) {
+  } else if (Fixup.getKind() == FK_Data_4 && !Fixup.isPCRel()) {
     support::endian::write<uint32_t>(&Data[Fixup.getOffset()], Value, Endian);
   } else if (Fixup.getKind() == FK_Data_8) {
     support::endian::write<uint64_t>(&Data[Fixup.getOffset()], Value, Endian);
-  } else if (Fixup.getKind() == FK_PCRel_4) {
+  } else if (Fixup.getKind() == FK_Data_4 && Fixup.isPCRel()) {
     Value = (uint32_t)((Value - 8) / 8);
     if (Endian == llvm::endianness::little) {
       Data[Fixup.getOffset() + 1] = 0x10;
@@ -96,7 +96,7 @@ void BPFAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
     support::endian::write<uint32_t>(&Data[Fixup.getOffset() + 4], Value,
                                      Endian);
   } else {
-    assert(Fixup.getKind() == FK_PCRel_2);
+    assert(Fixup.getKind() == FK_Data_2 && Fixup.isPCRel());
 
     int64_t ByteOff = (int64_t)Value - 8;
     if (ByteOff > INT16_MAX * 8 || ByteOff < INT16_MIN * 8)
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
index 5d653eed1755..827e928b9b66 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
@@ -45,12 +45,11 @@ unsigned BPFELFObjectWriter::getRelocType(const MCFixup &Fixup,
   case FK_SecRel_8:
     // LD_imm64 instruction.
     return ELF::R_BPF_64_64;
-  case FK_PCRel_4:
-    // CALL instruction.
-    return ELF::R_BPF_64_32;
   case FK_Data_8:
     return ELF::R_BPF_64_ABS64;
   case FK_Data_4:
+    if (Fixup.isPCRel()) // CALL instruction
+      return ELF::R_BPF_64_32;
     if (const auto *A = Target.getAddSym()) {
       const MCSymbol &Sym = *A;
 
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
index 89893b10f384..a0011e81ddd4 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp
@@ -32,24 +32,6 @@ void BPFInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   printAnnotation(O, Annot);
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &O) {
-  const MCSymbolRefExpr *SRE;
-
-  if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr))
-    SRE = dyn_cast<MCSymbolRefExpr>(BE->getLHS());
-  else
-    SRE = dyn_cast<MCSymbolRefExpr>(Expr);
-  if (!SRE)
-    report_fatal_error("Unexpected MCExpr type.");
-
-#ifndef NDEBUG
-  MCSymbolRefExpr::VariantKind Kind = SRE->getKind();
-
-  assert(Kind == MCSymbolRefExpr::VK_None);
-#endif
-  O << *Expr;
-}
-
 void BPFInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                   raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
@@ -59,7 +41,7 @@ void BPFInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     O << formatImm((int32_t)Op.getImm());
   } else {
     assert(Op.isExpr() && "Expected an expression");
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   }
 }
 
@@ -90,7 +72,7 @@ void BPFInstPrinter::printImm64Operand(const MCInst *MI, unsigned OpNo,
   if (Op.isImm())
     O << formatImm(Op.getImm());
   else if (Op.isExpr())
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   else
     O << Op;
 }
@@ -107,7 +89,7 @@ void BPFInstPrinter::printBrTargetOperand(const MCInst *MI, unsigned OpNo,
       O << ((Imm >= 0) ? "+" : "") << formatImm(Imm);
     }
   } else if (Op.isExpr()) {
-    printExpr(Op.getExpr(), O);
+    MAI.printExpr(O, *Op.getExpr());
   } else {
     O << Op;
   }
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index bd9d2de58c8b..cd32e7d16fca 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -77,6 +77,11 @@ MCCodeEmitter *llvm::createBPFbeMCCodeEmitter(const MCInstrInfo &MCII,
   return new BPFMCCodeEmitter(MCII, *Ctx.getRegisterInfo(), false, Ctx);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
                                              const MCOperand &MO,
                                              SmallVectorImpl<MCFixup> &Fixups,
@@ -102,14 +107,14 @@ unsigned BPFMCCodeEmitter::getMachineOpValue(const MCInst &MI,
 
   if (MI.getOpcode() == BPF::JAL)
     // func call name
-    Fixups.push_back(MCFixup::create(0, Expr, FK_PCRel_4));
+    addFixup(Fixups, 0, Expr, FK_Data_4, true);
   else if (MI.getOpcode() == BPF::LD_imm64)
-    Fixups.push_back(MCFixup::create(0, Expr, FK_SecRel_8));
+    addFixup(Fixups, 0, Expr, FK_SecRel_8);
   else if (MI.getOpcode() == BPF::JMPL)
-    Fixups.push_back(MCFixup::create(0, Expr, (MCFixupKind)BPF::FK_BPF_PCRel_4));
+    addFixup(Fixups, 0, Expr, BPF::FK_BPF_PCRel_4, true);
   else
     // bb label
-    Fixups.push_back(MCFixup::create(0, Expr, FK_PCRel_2));
+    addFixup(Fixups, 0, Expr, FK_Data_2, true);
 
   return 0;
 }
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index 9472288229ca..f5f6f5386f89 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -22,7 +22,7 @@ add_llvm_component_library(LLVMTarget
 # When building shared objects for each target there are some internal APIs
 # that are used across shared objects which we can't hide.
 if (NOT BUILD_SHARED_LIBS AND NOT APPLE AND
-    (NOT (WIN32 OR CYGWIN) OR (MINGW AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
+    (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
     NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
     NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
   # Set default visibility to hidden, so we don't export all the Target classes
diff --git a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
index 2ac1212e4bfd..5f7e1497a1eb 100644
--- a/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
+++ b/llvm/lib/Target/CSKY/AsmParser/CSKYAsmParser.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/CSKYInstPrinter.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYMCTargetDesc.h"
 #include "MCTargetDesc/CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
@@ -430,7 +430,7 @@ struct CSKYOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return CSKYInstPrinter::getRegisterName(Reg);
@@ -440,10 +440,10 @@ struct CSKYOperand : public MCParsedAsmOperand {
 
     switch (Kind) {
     case CPOP:
-      OS << *getConstpoolOp();
+      MAI.printExpr(OS, *getConstpoolOp());
       break;
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case KindTy::Register:
       OS << "<register " << RegName(getReg()) << ">";
@@ -848,11 +848,11 @@ bool CSKYAsmParser::processLRW(MCInst &Inst, SMLoc IDLoc, MCStreamer &Out) {
     }
   } else {
     const MCExpr *AdjustExpr = nullptr;
-    if (const CSKYMCExpr *CSKYExpr =
-            dyn_cast<CSKYMCExpr>(Inst.getOperand(1).getExpr())) {
-      if (CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSGD ||
-          CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSIE ||
-          CSKYExpr->getSpecifier() == CSKYMCExpr::VK_TLSLDM) {
+    if (const auto *CSKYExpr =
+            dyn_cast<MCSpecifierExpr>(Inst.getOperand(1).getExpr())) {
+      if (CSKYExpr->getSpecifier() == CSKY::S_TLSGD ||
+          CSKYExpr->getSpecifier() == CSKY::S_TLSIE ||
+          CSKYExpr->getSpecifier() == CSKY::S_TLSLDM) {
         MCSymbol *Dot = getContext().createNamedTempSymbol();
         Out.emitLabel(Dot);
         AdjustExpr = MCSymbolRefExpr::create(Dot, getContext());
@@ -1172,25 +1172,25 @@ ParseStatus CSKYAsmParser::parseCSKYSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return Error(getLoc(), "unknown identifier");
 
-  CSKYMCExpr::Specifier Kind = CSKYMCExpr::VK_None;
+  CSKY::Specifier Kind = CSKY::S_None;
   if (Identifier.consume_back("@GOT"))
-    Kind = CSKYMCExpr::VK_GOT;
+    Kind = CSKY::S_GOT;
   else if (Identifier.consume_back("@GOTOFF"))
-    Kind = CSKYMCExpr::VK_GOTOFF;
+    Kind = CSKY::S_GOTOFF;
   else if (Identifier.consume_back("@PLT"))
-    Kind = CSKYMCExpr::VK_PLT;
+    Kind = CSKY::S_PLT;
   else if (Identifier.consume_back("@GOTPC"))
-    Kind = CSKYMCExpr::VK_GOTPC;
+    Kind = CSKY::S_GOTPC;
   else if (Identifier.consume_back("@TLSGD32"))
-    Kind = CSKYMCExpr::VK_TLSGD;
+    Kind = CSKY::S_TLSGD;
   else if (Identifier.consume_back("@GOTTPOFF"))
-    Kind = CSKYMCExpr::VK_TLSIE;
+    Kind = CSKY::S_TLSIE;
   else if (Identifier.consume_back("@TPOFF"))
-    Kind = CSKYMCExpr::VK_TLSLE;
+    Kind = CSKY::S_TLSLE;
   else if (Identifier.consume_back("@TLSLDM32"))
-    Kind = CSKYMCExpr::VK_TLSLDM;
+    Kind = CSKY::S_TLSLDM;
   else if (Identifier.consume_back("@TLSLDO32"))
-    Kind = CSKYMCExpr::VK_TLSLDO;
+    Kind = CSKY::S_TLSLDO;
 
   MCSymbol *Sym = getContext().getInlineAsmLabel(Identifier);
 
@@ -1210,8 +1210,8 @@ ParseStatus CSKYAsmParser::parseCSKYSymbol(OperandVector &Operands) {
   MCBinaryExpr::Opcode Opcode;
   switch (getLexer().getKind()) {
   default:
-    if (Kind != CSKYMCExpr::VK_None)
-      Res = CSKYMCExpr::create(Res, Kind, getContext());
+    if (Kind != CSKY::S_None)
+      Res = MCSpecifierExpr::create(Res, Kind, getContext());
 
     Operands.push_back(CSKYOperand::createImm(Res, S, E));
     return ParseStatus::Success;
@@ -1258,11 +1258,11 @@ ParseStatus CSKYAsmParser::parseDataSymbol(OperandVector &Operands) {
   if (getParser().parseIdentifier(Identifier))
     return Error(getLoc(), "unknown identifier " + Identifier);
 
-  CSKYMCExpr::Specifier Kind = CSKYMCExpr::VK_None;
+  CSKY::Specifier Kind = CSKY::S_None;
   if (Identifier.consume_back("@GOT"))
-    Kind = CSKYMCExpr::VK_GOT_IMM18_BY4;
+    Kind = CSKY::S_GOT_IMM18_BY4;
   else if (Identifier.consume_back("@PLT"))
-    Kind = CSKYMCExpr::VK_PLT_IMM18_BY4;
+    Kind = CSKY::S_PLT_IMM18_BY4;
 
   MCSymbol *Sym = getContext().getInlineAsmLabel(Identifier);
 
@@ -1288,8 +1288,8 @@ ParseStatus CSKYAsmParser::parseDataSymbol(OperandVector &Operands) {
 
     getLexer().Lex(); // Eat ']'.
 
-    if (Kind != CSKYMCExpr::VK_None)
-      Res = CSKYMCExpr::create(Res, Kind, getContext());
+    if (Kind != CSKY::S_None)
+      Res = MCSpecifierExpr::create(Res, Kind, getContext());
 
     Operands.push_back(CSKYOperand::createConstpoolOp(Res, S, E));
     return ParseStatus::Success;
diff --git a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
index 9a259eac52f4..7e1eb9d9ec43 100644
--- a/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
+++ b/llvm/lib/Target/CSKY/CSKYAsmPrinter.cpp
@@ -15,7 +15,7 @@
 #include "CSKYConstantPoolValue.h"
 #include "CSKYTargetMachine.h"
 #include "MCTargetDesc/CSKYInstPrinter.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/ADT/Statistic.h"
@@ -168,25 +168,24 @@ void CSKYAsmPrinter::emitInstruction(const MachineInstr *MI) {
 
 // Convert a CSKY-specific constant pool modifier into the associated
 // MCSymbolRefExpr variant kind.
-static CSKYMCExpr::Specifier
-getModifierVariantKind(CSKYCP::CSKYCPModifier Modifier) {
+static CSKY::Specifier getModifierVariantKind(CSKYCP::CSKYCPModifier Modifier) {
   switch (Modifier) {
   case CSKYCP::NO_MOD:
-    return CSKYMCExpr::VK_None;
+    return CSKY::S_None;
   case CSKYCP::ADDR:
-    return CSKYMCExpr::VK_ADDR;
+    return CSKY::S_ADDR;
   case CSKYCP::GOT:
-    return CSKYMCExpr::VK_GOT;
+    return CSKY::S_GOT;
   case CSKYCP::GOTOFF:
-    return CSKYMCExpr::VK_GOTOFF;
+    return CSKY::S_GOTOFF;
   case CSKYCP::PLT:
-    return CSKYMCExpr::VK_PLT;
+    return CSKY::S_PLT;
   case CSKYCP::TLSGD:
-    return CSKYMCExpr::VK_TLSGD;
+    return CSKY::S_TLSGD;
   case CSKYCP::TLSLE:
-    return CSKYMCExpr::VK_TLSLE;
+    return CSKY::S_TLSLE;
   case CSKYCP::TLSIE:
-    return CSKYMCExpr::VK_TLSIE;
+    return CSKY::S_TLSIE;
   }
   llvm_unreachable("Invalid CSKYCPModifier!");
 }
@@ -240,8 +239,8 @@ void CSKYAsmPrinter::emitMachineConstantPoolValue(
   }
 
   // Create an MCSymbol for the reference.
-  Expr = CSKYMCExpr::create(Expr, getModifierVariantKind(CCPV->getModifier()),
-                            OutContext);
+  Expr = MCSpecifierExpr::create(
+      Expr, getModifierVariantKind(CCPV->getModifier()), OutContext);
 
   OutStreamer->emitValue(Expr, Size);
 }
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 34221cc69958..707017106e61 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -1207,9 +1207,6 @@ SDValue CSKYTargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
diff --git a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
index a91f94bb2357..db38c622dcd2 100644
--- a/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
+++ b/llvm/lib/Target/CSKY/CSKYMCInstLower.cpp
@@ -13,7 +13,7 @@
 
 #include "CSKYMCInstLower.h"
 #include "MCTargetDesc/CSKYBaseInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/MC/MCExpr.h"
 
@@ -36,38 +36,38 @@ void CSKYMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
 
 MCOperand CSKYMCInstLower::lowerSymbolOperand(const MachineOperand &MO,
                                               MCSymbol *Sym) const {
-  CSKYMCExpr::Specifier Spec;
+  CSKY::Specifier Spec;
   MCContext &Ctx = Printer.OutContext;
 
   switch (MO.getTargetFlags()) {
   default:
     llvm_unreachable("Unknown target flag.");
   case CSKYII::MO_None:
-    Spec = CSKYMCExpr::VK_None;
+    Spec = CSKY::S_None;
     break;
   case CSKYII::MO_GOT32:
-    Spec = CSKYMCExpr::VK_GOT;
+    Spec = CSKY::S_GOT;
     break;
   case CSKYII::MO_GOTOFF:
-    Spec = CSKYMCExpr::VK_GOTOFF;
+    Spec = CSKY::S_GOTOFF;
     break;
   case CSKYII::MO_ADDR32:
-    Spec = CSKYMCExpr::VK_ADDR;
+    Spec = CSKY::S_ADDR;
     break;
   case CSKYII::MO_PLT32:
-    Spec = CSKYMCExpr::VK_PLT;
+    Spec = CSKY::S_PLT;
     break;
   case CSKYII::MO_ADDR_HI16:
-    Spec = CSKYMCExpr::VK_ADDR_HI16;
+    Spec = CSKY::S_ADDR_HI16;
     break;
   case CSKYII::MO_ADDR_LO16:
-    Spec = CSKYMCExpr::VK_ADDR_LO16;
+    Spec = CSKY::S_ADDR_LO16;
     break;
   }
   const MCExpr *ME = MCSymbolRefExpr::create(Sym, Ctx);
 
-  if (Spec != CSKYMCExpr::VK_None)
-    ME = CSKYMCExpr::create(ME, Spec, Ctx);
+  if (Spec != CSKY::S_None)
+    ME = MCSpecifierExpr::create(ME, Spec, Ctx);
 
   return MCOperand::createExpr(ME);
 }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
index 73b4429349ea..e0932808bb7a 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMCSKYDesc
   CSKYELFStreamer.cpp
   CSKYInstPrinter.cpp
   CSKYMCAsmInfo.cpp
-  CSKYMCExpr.cpp
   CSKYMCTargetDesc.cpp
   CSKYMCCodeEmitter.cpp
   CSKYTargetStreamer.cpp
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
index 47d272891570..0922d037149e 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp
@@ -11,7 +11,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
@@ -32,34 +31,27 @@ MCFixupKindInfo CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {CSKY::Fixups::fixup_csky_addr_hi16, {"fixup_csky_addr_hi16", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_addr_lo16, {"fixup_csky_addr_lo16", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm16_scale2,
-       {"fixup_csky_pcrel_imm16_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm16_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm16_scale4,
-       {"fixup_csky_pcrel_uimm16_scale4", 0, 32,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm16_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm8_scale4,
-       {"fixup_csky_pcrel_uimm8_scale4", 0, 32,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm8_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm26_scale2,
-       {"fixup_csky_pcrel_imm26_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm26_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm18_scale2,
-       {"fixup_csky_pcrel_imm18_scale2", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm18_scale2", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_got32, {"fixup_csky_got32", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_got_imm18_scale4,
        {"fixup_csky_got_imm18_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_gotoff, {"fixup_csky_gotoff", 0, 32, 0}},
-      {CSKY::Fixups::fixup_csky_gotpc,
-       {"fixup_csky_gotpc", 0, 32, MCFixupKindInfo::FKF_IsPCRel}},
+      {CSKY::Fixups::fixup_csky_gotpc, {"fixup_csky_gotpc", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_plt32, {"fixup_csky_plt32", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_plt_imm18_scale4,
        {"fixup_csky_plt_imm18_scale4", 0, 32, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_imm10_scale2,
-       {"fixup_csky_pcrel_imm10_scale2", 0, 16, MCFixupKindInfo::FKF_IsPCRel}},
+       {"fixup_csky_pcrel_imm10_scale2", 0, 16, 0}},
       {CSKY::Fixups::fixup_csky_pcrel_uimm7_scale4,
-       {"fixup_csky_pcrel_uimm7_scale4", 0, 16,
-        MCFixupKindInfo::FKF_IsPCRel |
-            MCFixupKindInfo::FKF_IsAlignedDownTo32Bits}},
+       {"fixup_csky_pcrel_uimm7_scale4", 0, 16, 0}},
       {CSKY::Fixups::fixup_csky_doffset_imm18,
        {"fixup_csky_doffset_imm18", 0, 18, 0}},
       {CSKY::Fixups::fixup_csky_doffset_imm18_scale2,
@@ -71,7 +63,7 @@ MCFixupKindInfo CSKYAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
          "Not all fixup kinds added to Infos array");
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
   return Infos[Kind];
@@ -188,10 +180,29 @@ bool CSKYAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
   }
 }
 
-void CSKYAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+std::optional<bool> CSKYAsmBackend::evaluateFixup(const MCFragment &F,
+                                                  MCFixup &Fixup, MCValue &,
+                                                  uint64_t &Value) {
+  // For a few PC-relative fixups, offsets need to be aligned down. We
+  // compensate here because the default handler's `Value` decrement doesn't
+  // account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case CSKY::fixup_csky_pcrel_uimm16_scale4:
+  case CSKY::fixup_csky_pcrel_uimm8_scale4:
+  case CSKY::fixup_csky_pcrel_uimm7_scale4:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
+void CSKYAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                 const MCValue &Target,
                                 MutableArrayRef<char> Data, uint64_t Value,
                                 bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
+
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
index 099aaa6e67cd..142c667a6175 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h
@@ -22,6 +22,8 @@ class CSKYAsmBackend : public MCAsmBackend {
   CSKYAsmBackend(const MCSubtargetInfo &STI, const MCTargetOptions &OP)
       : MCAsmBackend(llvm::endianness::little) {}
 
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
@@ -43,8 +45,7 @@ class CSKYAsmBackend : public MCAsmBackend {
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
index 0a2087db3fee..1de82e6cc6ce 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYELFObjectWriter.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYFixupKinds.h"
-#include "CSKYMCExpr.h"
+#include "CSKYMCAsmInfo.h"
 #include "CSKYMCTargetDesc.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -43,11 +43,11 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
   uint8_t Modifier = Target.getSpecifier();
 
   switch (Target.getSpecifier()) {
-  case CSKYMCExpr::VK_TLSIE:
-  case CSKYMCExpr::VK_TLSLE:
-  case CSKYMCExpr::VK_TLSGD:
-  case CSKYMCExpr::VK_TLSLDM:
-  case CSKYMCExpr::VK_TLSLDO:
+  case CSKY::S_TLSIE:
+  case CSKY::S_TLSLE:
+  case CSKY::S_TLSGD:
+  case CSKY::S_TLSLDM:
+  case CSKY::S_TLSLDO:
     if (auto *SA = Target.getAddSym())
       cast<MCSymbolELF>(SA)->setType(ELF::STT_TLS);
     break;
@@ -62,7 +62,6 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
       reportError(Fixup.getLoc(), "Unsupported relocation type");
       return ELF::R_CKCORE_NONE;
     case FK_Data_4:
-    case FK_PCRel_4:
       return ELF::R_CKCORE_PCREL32;
     case CSKY::fixup_csky_pcrel_uimm16_scale4:
       return ELF::R_CKCORE_PCREL_IMM16_4;
@@ -93,29 +92,29 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
     reportError(Fixup.getLoc(), "2-byte data relocations not supported");
     return ELF::R_CKCORE_NONE;
   case FK_Data_4:
-    if (Expr->getKind() == MCExpr::Target) {
-      auto TK = cast<CSKYMCExpr>(Expr)->getSpecifier();
-      if (TK == CSKYMCExpr::VK_ADDR)
+    if (Expr->getKind() == MCExpr::Specifier) {
+      auto TK = cast<MCSpecifierExpr>(Expr)->getSpecifier();
+      if (TK == CSKY::S_ADDR)
         return ELF::R_CKCORE_ADDR32;
-      if (TK == CSKYMCExpr::VK_GOT)
+      if (TK == CSKY::S_GOT)
         return ELF::R_CKCORE_GOT32;
-      if (TK == CSKYMCExpr::VK_GOTOFF)
+      if (TK == CSKY::S_GOTOFF)
         return ELF::R_CKCORE_GOTOFF;
-      if (TK == CSKYMCExpr::VK_PLT)
+      if (TK == CSKY::S_PLT)
         return ELF::R_CKCORE_PLT32;
-      if (TK == CSKYMCExpr::VK_TLSIE)
+      if (TK == CSKY::S_TLSIE)
         return ELF::R_CKCORE_TLS_IE32;
-      if (TK == CSKYMCExpr::VK_TLSLE)
+      if (TK == CSKY::S_TLSLE)
         return ELF::R_CKCORE_TLS_LE32;
-      if (TK == CSKYMCExpr::VK_TLSGD)
+      if (TK == CSKY::S_TLSGD)
         return ELF::R_CKCORE_TLS_GD32;
-      if (TK == CSKYMCExpr::VK_TLSLDM)
+      if (TK == CSKY::S_TLSLDM)
         return ELF::R_CKCORE_TLS_LDM32;
-      if (TK == CSKYMCExpr::VK_TLSLDO)
+      if (TK == CSKY::S_TLSLDO)
         return ELF::R_CKCORE_TLS_LDO32;
-      if (TK == CSKYMCExpr::VK_GOTPC)
+      if (TK == CSKY::S_GOTPC)
         return ELF::R_CKCORE_GOTPC;
-      if (TK == CSKYMCExpr::VK_None)
+      if (TK == CSKY::S_None)
         return ELF::R_CKCORE_ADDR32;
 
       LLVM_DEBUG(dbgs() << "Unknown FK_Data_4 TK  = " << TK);
@@ -125,19 +124,19 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
       default:
         reportError(Fixup.getLoc(), "invalid fixup for 4-byte data relocation");
         return ELF::R_CKCORE_NONE;
-      case CSKYMCExpr::VK_GOT:
+      case CSKY::S_GOT:
         return ELF::R_CKCORE_GOT32;
-      case CSKYMCExpr::VK_GOTOFF:
+      case CSKY::S_GOTOFF:
         return ELF::R_CKCORE_GOTOFF;
-      case CSKYMCExpr::VK_PLT:
+      case CSKY::S_PLT:
         return ELF::R_CKCORE_PLT32;
-      case CSKYMCExpr::VK_TLSGD:
+      case CSKY::S_TLSGD:
         return ELF::R_CKCORE_TLS_GD32;
-      case CSKYMCExpr::VK_TLSLDM:
+      case CSKY::S_TLSLDM:
         return ELF::R_CKCORE_TLS_LDM32;
-      case CSKYMCExpr::VK_TPOFF:
+      case CSKY::S_TPOFF:
         return ELF::R_CKCORE_TLS_LE32;
-      case CSKYMCExpr::VK_None:
+      case CSKY::S_None:
         return ELF::R_CKCORE_ADDR32;
       }
     }
@@ -167,8 +166,8 @@ unsigned CSKYELFObjectWriter::getRelocType(const MCFixup &Fixup,
 bool CSKYELFObjectWriter::needsRelocateWithSymbol(const MCValue &V,
                                                   unsigned Type) const {
   switch (V.getSpecifier()) {
-  case CSKYMCExpr::VK_PLT:
-  case CSKYMCExpr::VK_GOT:
+  case CSKY::S_PLT:
+  case CSKY::S_GOT:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
index 47c05d67296e..90428e34cefa 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYInstPrinter.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 #include "CSKYInstPrinter.h"
 #include "MCTargetDesc/CSKYBaseInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -138,7 +138,7 @@ void CSKYInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 
   assert(MO.isExpr() && "Unknown operand kind in printOperand");
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
 }
 
 void CSKYInstPrinter::printDataSymbol(const MCInst *MI, unsigned OpNo,
@@ -150,7 +150,7 @@ void CSKYInstPrinter::printDataSymbol(const MCInst *MI, unsigned OpNo,
   if (MO.isImm())
     O << MO.getImm();
   else
-    MO.getExpr()->print(O, &MAI);
+    MAI.printExpr(O, *MO.getExpr());
   O << "]";
 }
 
@@ -173,7 +173,7 @@ void CSKYInstPrinter::printConstpool(const MCInst *MI, uint64_t Address,
   assert(MO.isExpr() && "Unknown operand kind in printConstpool");
 
   O << "[";
-  MO.getExpr()->print(O, &MAI);
+  MAI.printExpr(O, *MO.getExpr());
   O << "]";
 }
 
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
index 796bac33419b..803b75332f7b 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.cpp
@@ -11,17 +11,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYMCAsmInfo.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {CSKYMCExpr::VK_GOT, "GOT"},       {CSKYMCExpr::VK_GOTOFF, "GOTOFF"},
-    {CSKYMCExpr::VK_PLT, "PLT"},       {CSKYMCExpr::VK_TLSGD, "TLSGD"},
-    {CSKYMCExpr::VK_TLSLDM, "TLSLDM"}, {CSKYMCExpr::VK_TPOFF, "TPOFF"},
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
+    {CSKY::S_GOT, "GOT"},       {CSKY::S_GOTOFF, "GOTOFF"},
+    {CSKY::S_PLT, "PLT"},       {CSKY::S_TLSGD, "TLSGD"},
+    {CSKY::S_TLSLDM, "TLSLDM"}, {CSKY::S_TPOFF, "TPOFF"},
 };
 
 void CSKYMCAsmInfo::anchor() {}
@@ -36,5 +36,46 @@ CSKYMCAsmInfo::CSKYMCAsmInfo(const Triple &TargetTriple) {
 
   ExceptionsType = ExceptionHandling::DwarfCFI;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
+}
+
+static StringRef getVariantKindName(uint8_t Kind) {
+  using namespace CSKY;
+  switch (Kind) {
+  default:
+    llvm_unreachable("Invalid ELF symbol kind");
+  case S_None:
+  case S_ADDR:
+    return "";
+  case S_ADDR_HI16:
+    return "@HI16";
+  case S_ADDR_LO16:
+    return "@LO16";
+  case S_GOT_IMM18_BY4:
+  case S_GOT:
+    return "@GOT";
+  case S_GOTPC:
+    return "@GOTPC";
+  case S_GOTOFF:
+    return "@GOTOFF";
+  case S_PLT_IMM18_BY4:
+  case S_PLT:
+    return "@PLT";
+  case S_TLSLE:
+    return "@TPOFF";
+  case S_TLSIE:
+    return "@GOTTPOFF";
+  case S_TLSGD:
+    return "@TLSGD32";
+  case S_TLSLDO:
+    return "@TLSLDO32";
+  case S_TLSLDM:
+    return "@TLSLDM32";
+  }
+}
+
+void CSKYMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
+                                       const MCSpecifierExpr &Expr) const {
+  printExpr(OS, *Expr.getSubExpr());
+  OS << getVariantKindName(Expr.getSpecifier());
 }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
index 3e0609f19531..aa4ba147c00a 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCAsmInfo.h
@@ -14,6 +14,8 @@
 #define LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCASMINFO_H
 
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 namespace llvm {
 class Triple;
@@ -23,7 +25,33 @@ class CSKYMCAsmInfo : public MCAsmInfoELF {
 
 public:
   explicit CSKYMCAsmInfo(const Triple &TargetTriple);
+  void printSpecifierExpr(raw_ostream &OS,
+                          const MCSpecifierExpr &Expr) const override;
 };
+
+namespace CSKY {
+using Specifier = uint8_t;
+enum {
+  S_None,
+  S_ADDR,
+  S_ADDR_HI16,
+  S_ADDR_LO16,
+  S_PCREL,
+  S_GOT,
+  S_GOT_IMM18_BY4,
+  S_GOTPC,
+  S_GOTOFF,
+  S_PLT,
+  S_PLT_IMM18_BY4,
+  S_TLSIE,
+  S_TLSLE,
+  S_TLSGD,
+  S_TLSLDO,
+  S_TLSLDM,
+  S_TPOFF,
+  S_Invalid
+};
+} // namespace CSKY
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCASMINFO_H
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
index 70a797ddd331..5f9f55eb606c 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.cpp
@@ -10,10 +10,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CSKYMCCodeEmitter.h"
-#include "CSKYMCExpr.h"
+#include "CSKYMCAsmInfo.h"
+#include "MCTargetDesc/CSKYFixupKinds.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "MCTargetDesc/CSKYMCTargetDesc.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
@@ -27,6 +30,186 @@ using namespace llvm;
 
 STATISTIC(MCNumEmitted, "Number of MC instructions emitted");
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case CSKY::Fixups::fixup_csky_pcrel_imm16_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm16_scale4:
+  case CSKY::Fixups::fixup_csky_pcrel_imm26_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_imm18_scale2:
+  case CSKY::Fixups::fixup_csky_gotpc:
+  case CSKY::Fixups::fixup_csky_pcrel_imm10_scale2:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm7_scale4:
+  case CSKY::Fixups::fixup_csky_pcrel_uimm8_scale4:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
+namespace {
+class CSKYMCCodeEmitter : public MCCodeEmitter {
+  MCContext &Ctx;
+  const MCInstrInfo &MII;
+
+public:
+  CSKYMCCodeEmitter(MCContext &Ctx, const MCInstrInfo &MII)
+      : Ctx(Ctx), MII(MII) {}
+
+  ~CSKYMCCodeEmitter() {}
+
+  void encodeInstruction(const MCInst &Inst, SmallVectorImpl<char> &CB,
+                         SmallVectorImpl<MCFixup> &Fixups,
+                         const MCSubtargetInfo &STI) const override;
+
+  // Generated by tablegen.
+  uint64_t getBinaryCodeForInstr(const MCInst &MI,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
+
+  // Default encoding method used by tablegen.
+  unsigned getMachineOpValue(const MCInst &MI, const MCOperand &MO,
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
+
+  template <int shift = 0>
+  unsigned getImmOpValue(const MCInst &MI, unsigned Idx,
+                         SmallVectorImpl<MCFixup> &Fixups,
+                         const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    if (MO.isImm())
+      return (MO.getImm() >> shift);
+
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = getTargetFixup(MO.getExpr());
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getRegSeqImmOpValue(const MCInst &MI, unsigned Idx,
+                               SmallVectorImpl<MCFixup> &Fixups,
+                               const MCSubtargetInfo &STI) const;
+
+  unsigned getRegisterSeqOpValue(const MCInst &MI, unsigned Op,
+                                 SmallVectorImpl<MCFixup> &Fixups,
+                                 const MCSubtargetInfo &STI) const;
+
+  unsigned getOImmOpValue(const MCInst &MI, unsigned Idx,
+                          SmallVectorImpl<MCFixup> &Fixups,
+                          const MCSubtargetInfo &STI) const;
+
+  unsigned getImmOpValueIDLY(const MCInst &MI, unsigned Idx,
+                             SmallVectorImpl<MCFixup> &Fixups,
+                             const MCSubtargetInfo &STI) const;
+
+  unsigned getImmJMPIX(const MCInst &MI, unsigned Idx,
+                       SmallVectorImpl<MCFixup> &Fixups,
+                       const MCSubtargetInfo &STI) const;
+
+  unsigned getImmOpValueMSBSize(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const;
+
+  unsigned getImmShiftOpValue(const MCInst &MI, unsigned Idx,
+                              SmallVectorImpl<MCFixup> &Fixups,
+                              const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isImm() && "Unexpected MO type.");
+    return 1 << MO.getImm();
+  }
+
+  MCFixupKind getTargetFixup(const MCExpr *Expr) const;
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getBranchSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                  SmallVectorImpl<MCFixup> &Fixups,
+                                  const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+
+    if (MO.isImm())
+      return MO.getImm() >> 1;
+
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getConstpoolSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                     SmallVectorImpl<MCFixup> &Fixups,
+                                     const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  template <llvm::CSKY::Fixups FIXUP>
+  unsigned getDataSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = MCFixupKind(FIXUP);
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getCallSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = CSKY::fixup_csky_pcrel_imm26_scale2;
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  unsigned getBareSymbolOpValue(const MCInst &MI, unsigned Idx,
+                                SmallVectorImpl<MCFixup> &Fixups,
+                                const MCSubtargetInfo &STI) const {
+    const MCOperand &MO = MI.getOperand(Idx);
+    assert(MO.isExpr() && "Unexpected MO type.");
+
+    MCFixupKind Kind = CSKY::fixup_csky_pcrel_imm18_scale2;
+    if (MO.getExpr()->getKind() == MCExpr::Specifier)
+      Kind = getTargetFixup(MO.getExpr());
+
+    addFixup(Fixups, 0, MO.getExpr(), Kind);
+    return 0;
+  }
+
+  void expandJBTF(const MCInst &MI, SmallVectorImpl<char> &CB,
+                  SmallVectorImpl<MCFixup> &Fixups,
+                  const MCSubtargetInfo &STI) const;
+  void expandNEG(const MCInst &MI, SmallVectorImpl<char> &CB,
+                 SmallVectorImpl<MCFixup> &Fixups,
+                 const MCSubtargetInfo &STI) const;
+  void expandRSUBI(const MCInst &MI, SmallVectorImpl<char> &CB,
+                   SmallVectorImpl<MCFixup> &Fixups,
+                   const MCSubtargetInfo &STI) const;
+};
+} // namespace
+
 unsigned CSKYMCCodeEmitter::getOImmOpValue(const MCInst &MI, unsigned Idx,
                                            SmallVectorImpl<MCFixup> &Fixups,
                                            const MCSubtargetInfo &STI) const {
@@ -293,28 +476,28 @@ unsigned CSKYMCCodeEmitter::getImmJMPIX(const MCInst &MI, unsigned Idx,
 }
 
 MCFixupKind CSKYMCCodeEmitter::getTargetFixup(const MCExpr *Expr) const {
-  const CSKYMCExpr *CSKYExpr = cast<CSKYMCExpr>(Expr);
+  const auto *CSKYExpr = cast<MCSpecifierExpr>(Expr);
   switch (CSKYExpr->getSpecifier()) {
   default:
     llvm_unreachable("Unhandled fixup kind!");
-  case CSKYMCExpr::VK_ADDR:
-    return MCFixupKind(CSKY::fixup_csky_addr32);
-  case CSKYMCExpr::VK_ADDR_HI16:
-    return MCFixupKind(CSKY::fixup_csky_addr_hi16);
-  case CSKYMCExpr::VK_ADDR_LO16:
-    return MCFixupKind(CSKY::fixup_csky_addr_lo16);
-  case CSKYMCExpr::VK_GOT:
-    return MCFixupKind(CSKY::fixup_csky_got32);
-  case CSKYMCExpr::VK_GOTPC:
-    return MCFixupKind(CSKY::fixup_csky_gotpc);
-  case CSKYMCExpr::VK_GOTOFF:
-    return MCFixupKind(CSKY::fixup_csky_gotoff);
-  case CSKYMCExpr::VK_PLT:
-    return MCFixupKind(CSKY::fixup_csky_plt32);
-  case CSKYMCExpr::VK_PLT_IMM18_BY4:
-    return MCFixupKind(CSKY::fixup_csky_plt_imm18_scale4);
-  case CSKYMCExpr::VK_GOT_IMM18_BY4:
-    return MCFixupKind(CSKY::fixup_csky_got_imm18_scale4);
+  case CSKY::S_ADDR:
+    return CSKY::fixup_csky_addr32;
+  case CSKY::S_ADDR_HI16:
+    return CSKY::fixup_csky_addr_hi16;
+  case CSKY::S_ADDR_LO16:
+    return CSKY::fixup_csky_addr_lo16;
+  case CSKY::S_GOT:
+    return CSKY::fixup_csky_got32;
+  case CSKY::S_GOTPC:
+    return CSKY::fixup_csky_gotpc;
+  case CSKY::S_GOTOFF:
+    return CSKY::fixup_csky_gotoff;
+  case CSKY::S_PLT:
+    return CSKY::fixup_csky_plt32;
+  case CSKY::S_PLT_IMM18_BY4:
+    return CSKY::fixup_csky_plt_imm18_scale4;
+  case CSKY::S_GOT_IMM18_BY4:
+    return CSKY::fixup_csky_got_imm18_scale4;
   }
 }
 
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
deleted file mode 100644
index 0e47d259d43f..000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCCodeEmitter.h
+++ /dev/null
@@ -1,188 +0,0 @@
-//===-- CSKYMCCodeEmitter.cpp - CSKY Code Emitter interface ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the CSKYMCCodeEmitter class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
-#define LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
-
-#include "MCTargetDesc/CSKYFixupKinds.h"
-#include "MCTargetDesc/CSKYMCExpr.h"
-#include "llvm/MC/MCCodeEmitter.h"
-#include "llvm/MC/MCContext.h"
-
-namespace llvm {
-
-class MCInstrInfo;
-
-class CSKYMCCodeEmitter : public MCCodeEmitter {
-  MCContext &Ctx;
-  const MCInstrInfo &MII;
-
-public:
-  CSKYMCCodeEmitter(MCContext &Ctx, const MCInstrInfo &MII)
-      : Ctx(Ctx), MII(MII) {}
-
-  ~CSKYMCCodeEmitter() {}
-
-  void encodeInstruction(const MCInst &Inst, SmallVectorImpl<char> &CB,
-                         SmallVectorImpl<MCFixup> &Fixups,
-                         const MCSubtargetInfo &STI) const override;
-
-  // Generated by tablegen.
-  uint64_t getBinaryCodeForInstr(const MCInst &MI,
-                                 SmallVectorImpl<MCFixup> &Fixups,
-                                 const MCSubtargetInfo &STI) const;
-
-  // Default encoding method used by tablegen.
-  unsigned getMachineOpValue(const MCInst &MI, const MCOperand &MO,
-                             SmallVectorImpl<MCFixup> &Fixups,
-                             const MCSubtargetInfo &STI) const;
-
-  template <int shift = 0>
-  unsigned getImmOpValue(const MCInst &MI, unsigned Idx,
-                         SmallVectorImpl<MCFixup> &Fixups,
-                         const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    if (MO.isImm())
-      return (MO.getImm() >> shift);
-
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = getTargetFixup(MO.getExpr());
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getRegSeqImmOpValue(const MCInst &MI, unsigned Idx,
-                               SmallVectorImpl<MCFixup> &Fixups,
-                               const MCSubtargetInfo &STI) const;
-
-  unsigned getRegisterSeqOpValue(const MCInst &MI, unsigned Op,
-                                 SmallVectorImpl<MCFixup> &Fixups,
-                                 const MCSubtargetInfo &STI) const;
-
-  unsigned getOImmOpValue(const MCInst &MI, unsigned Idx,
-                          SmallVectorImpl<MCFixup> &Fixups,
-                          const MCSubtargetInfo &STI) const;
-
-  unsigned getImmOpValueIDLY(const MCInst &MI, unsigned Idx,
-                             SmallVectorImpl<MCFixup> &Fixups,
-                             const MCSubtargetInfo &STI) const;
-
-  unsigned getImmJMPIX(const MCInst &MI, unsigned Idx,
-                       SmallVectorImpl<MCFixup> &Fixups,
-                       const MCSubtargetInfo &STI) const;
-
-  unsigned getImmOpValueMSBSize(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const;
-
-  unsigned getImmShiftOpValue(const MCInst &MI, unsigned Idx,
-                              SmallVectorImpl<MCFixup> &Fixups,
-                              const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isImm() && "Unexpected MO type.");
-    return 1 << MO.getImm();
-  }
-
-  MCFixupKind getTargetFixup(const MCExpr *Expr) const;
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getBranchSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                  SmallVectorImpl<MCFixup> &Fixups,
-                                  const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-
-    if (MO.isImm())
-      return MO.getImm() >> 1;
-
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getConstpoolSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                     SmallVectorImpl<MCFixup> &Fixups,
-                                     const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  template <llvm::CSKY::Fixups FIXUP>
-  unsigned getDataSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(FIXUP);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getCallSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(CSKY::fixup_csky_pcrel_imm26_scale2);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  unsigned getBareSymbolOpValue(const MCInst &MI, unsigned Idx,
-                                SmallVectorImpl<MCFixup> &Fixups,
-                                const MCSubtargetInfo &STI) const {
-    const MCOperand &MO = MI.getOperand(Idx);
-    assert(MO.isExpr() && "Unexpected MO type.");
-
-    MCFixupKind Kind = MCFixupKind(CSKY::fixup_csky_pcrel_imm18_scale2);
-    if (MO.getExpr()->getKind() == MCExpr::Target)
-      Kind = getTargetFixup(MO.getExpr());
-
-    Fixups.push_back(MCFixup::create(0, MO.getExpr(), Kind, MI.getLoc()));
-    return 0;
-  }
-
-  void expandJBTF(const MCInst &MI, SmallVectorImpl<char> &CB,
-                  SmallVectorImpl<MCFixup> &Fixups,
-                  const MCSubtargetInfo &STI) const;
-  void expandNEG(const MCInst &MI, SmallVectorImpl<char> &CB,
-                 SmallVectorImpl<MCFixup> &Fixups,
-                 const MCSubtargetInfo &STI) const;
-  void expandRSUBI(const MCInst &MI, SmallVectorImpl<char> &CB,
-                   SmallVectorImpl<MCFixup> &Fixups,
-                   const MCSubtargetInfo &STI) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_CSKY_MCTARGETDESC_CSKYMCCODEEMITTER_H
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
deleted file mode 100644
index 696cd1daa035..000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- CSKYMCExpr.cpp - CSKY specific MC expression classes -*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "CSKYMCExpr.h"
-#include "CSKYFixupKinds.h"
-#include "llvm/BinaryFormat/ELF.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSymbolELF.h"
-#include "llvm/Support/Casting.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "csky-mc-expr"
-
-const CSKYMCExpr *CSKYMCExpr::create(const MCExpr *Expr, Specifier Kind,
-                                     MCContext &Ctx) {
-  return new (Ctx) CSKYMCExpr(Expr, Kind);
-}
-
-StringRef CSKYMCExpr::getVariantKindName(Specifier Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Invalid ELF symbol kind");
-  case VK_None:
-  case VK_ADDR:
-    return "";
-  case VK_ADDR_HI16:
-    return "@HI16";
-  case VK_ADDR_LO16:
-    return "@LO16";
-  case VK_GOT_IMM18_BY4:
-  case VK_GOT:
-    return "@GOT";
-  case VK_GOTPC:
-    return "@GOTPC";
-  case VK_GOTOFF:
-    return "@GOTOFF";
-  case VK_PLT_IMM18_BY4:
-  case VK_PLT:
-    return "@PLT";
-  case VK_TLSLE:
-    return "@TPOFF";
-  case VK_TLSIE:
-    return "@GOTTPOFF";
-  case VK_TLSGD:
-    return "@TLSGD32";
-  case VK_TLSLDO:
-    return "@TLSLDO32";
-  case VK_TLSLDM:
-    return "@TLSLDM32";
-  }
-}
-
-void CSKYMCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-void CSKYMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-  Expr->print(OS, MAI);
-  OS << getVariantKindName(getSpecifier());
-}
-
-bool CSKYMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                           const MCAssembler *Asm) const {
-  if (!getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-  Res.setSpecifier(specifier);
-  return !Res.getSubSym();
-}
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
deleted file mode 100644
index 14fe56f496a0..000000000000
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCExpr.h
+++ /dev/null
@@ -1,75 +0,0 @@
-//===-- CSKYMCExpr.h - CSKY specific MC expression classes -*- C++ -*----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-#define LLVM_LIB_TARGET_LANAI_MCTARGETDESC_LANAIMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCValue.h"
-
-namespace llvm {
-
-class CSKYMCExpr : public MCTargetExpr {
-public:
-  enum Specifier : uint8_t {
-    VK_None,
-    VK_ADDR = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_ADDR_HI16,
-    VK_ADDR_LO16,
-    VK_PCREL,
-    VK_GOT,
-    VK_GOT_IMM18_BY4,
-    VK_GOTPC,
-    VK_GOTOFF,
-    VK_PLT,
-    VK_PLT_IMM18_BY4,
-    VK_TLSIE,
-    VK_TLSLE,
-    VK_TLSGD,
-    VK_TLSLDO,
-    VK_TLSLDM,
-    VK_TPOFF,
-    VK_Invalid
-  };
-
-private:
-  const MCExpr *Expr;
-  const Specifier specifier;
-
-  explicit CSKYMCExpr(const MCExpr *Expr, Specifier S)
-      : Expr(Expr), specifier(S) {}
-
-public:
-  static const CSKYMCExpr *create(const MCExpr *Expr, Specifier Kind,
-                                  MCContext &Ctx);
-
-  // Returns the kind of this expression.
-  Specifier getSpecifier() const { return specifier; }
-
-  // Returns the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-
-  bool evaluateAsRelocatableImpl(MCValue &Res,
-                                 const MCAssembler *Asm) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  static bool classof(const MCExpr *E) {
-    return E->getKind() == MCExpr::Target;
-  }
-
-  static StringRef getVariantKindName(Specifier Kind);
-};
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
index bb122f683644..faaeeb8ed4dc 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYMCTargetDesc.cpp
@@ -15,7 +15,6 @@
 #include "CSKYELFStreamer.h"
 #include "CSKYInstPrinter.h"
 #include "CSKYMCAsmInfo.h"
-#include "CSKYMCCodeEmitter.h"
 #include "CSKYTargetStreamer.h"
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/MC/MCAssembler.h"
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
index 42876b389127..c2a389515b85 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CSKYTargetStreamer.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -57,11 +58,11 @@ const MCExpr *CSKYConstantPool::addEntry(MCStreamer &Streamer,
   const auto SymRef = MCSymbolRefExpr::create(CPEntryLabel, Context);
 
   if (AdjustExpr) {
-    const CSKYMCExpr *CSKYExpr = cast<CSKYMCExpr>(Value);
+    auto *CSKYExpr = cast<MCSpecifierExpr>(Value);
 
     Value = MCBinaryExpr::createSub(AdjustExpr, SymRef, Context);
     Value = MCBinaryExpr::createSub(CSKYExpr->getSubExpr(), Value, Context);
-    Value = CSKYMCExpr::create(Value, CSKYExpr->getSpecifier(), Context);
+    Value = MCSpecifierExpr::create(Value, CSKYExpr->getSpecifier(), Context);
   }
 
   Entries.push_back(ConstantPoolEntry(CPEntryLabel, Value, Size, Loc));
@@ -84,12 +85,12 @@ CSKYTargetStreamer::CSKYTargetStreamer(MCStreamer &S)
 const MCExpr *
 CSKYTargetStreamer::addConstantPoolEntry(const MCExpr *Expr, SMLoc Loc,
                                          const MCExpr *AdjustExpr) {
-  auto ELFRefKind = CSKYMCExpr::VK_Invalid;
+  uint8_t ELFRefKind = CSKY::S_Invalid;
   ConstantCounter++;
 
   const MCExpr *OrigExpr = Expr;
 
-  if (const CSKYMCExpr *CE = dyn_cast<CSKYMCExpr>(Expr)) {
+  if (auto *CE = dyn_cast<MCSpecifierExpr>(Expr)) {
     Expr = CE->getSubExpr();
     ELFRefKind = CE->getSpecifier();
   }
diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
index 03d5a7b0e08c..7b428502c34f 100644
--- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
+++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYTargetStreamer.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_CSKY_CSKYTARGETSTREAMER_H
 #define LLVM_LIB_TARGET_CSKY_CSKYTARGETSTREAMER_H
 
-#include "MCTargetDesc/CSKYMCExpr.h"
+#include "MCTargetDesc/CSKYMCAsmInfo.h"
 #include "llvm/MC/ConstantPools.h"
 #include "llvm/MC/MCStreamer.h"
 
@@ -46,7 +46,7 @@ class CSKYTargetStreamer : public MCTargetStreamer {
 public:
   typedef struct {
     const MCSymbol *sym;
-    CSKYMCExpr::Specifier kind;
+    CSKY::Specifier kind;
   } SymbolIndex;
 
 protected:
@@ -76,10 +76,10 @@ class CSKYTargetStreamer : public MCTargetStreamer {
 
 template <> struct DenseMapInfo<CSKYTargetStreamer::SymbolIndex> {
   static inline CSKYTargetStreamer::SymbolIndex getEmptyKey() {
-    return {nullptr, CSKYMCExpr::VK_Invalid};
+    return {nullptr, CSKY::S_Invalid};
   }
   static inline CSKYTargetStreamer::SymbolIndex getTombstoneKey() {
-    return {nullptr, CSKYMCExpr::VK_Invalid};
+    return {nullptr, CSKY::S_Invalid};
   }
   static unsigned getHashValue(const CSKYTargetStreamer::SymbolIndex &V) {
     return hash_combine(DenseMapInfo<const MCSymbol *>::getHashValue(V.sym),
diff --git a/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
index 7559f61b4cfb..44277971acd6 100644
--- a/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
+++ b/llvm/lib/Target/DirectX/DXILCBufferAccess.cpp
@@ -11,9 +11,11 @@
 #include "llvm/Frontend/HLSL/CBuffer.h"
 #include "llvm/Frontend/HLSL/HLSLResource.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 #define DEBUG_TYPE "dxil-cbuffer-access"
@@ -54,71 +56,106 @@ struct CBufferRowIntrin {
     }
   }
 };
-} // namespace
 
-static size_t getOffsetForCBufferGEP(GEPOperator *GEP, GlobalVariable *Global,
-                                     const DataLayout &DL) {
-  // Since we should always have a constant offset, we should only ever have a
-  // single GEP of indirection from the Global.
-  assert(GEP->getPointerOperand() == Global &&
-         "Indirect access to resource handle");
+// Helper for creating CBuffer handles and loading data from them
+struct CBufferResource {
+  GlobalVariable *GVHandle;
+  GlobalVariable *Member;
+  size_t MemberOffset;
 
-  APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
-  bool Success = GEP->accumulateConstantOffset(DL, ConstantOffset);
-  (void)Success;
-  assert(Success && "Offsets into cbuffer globals must be constant");
+  LoadInst *Handle;
 
-  if (auto *ATy = dyn_cast<ArrayType>(Global->getValueType()))
-    ConstantOffset = hlsl::translateCBufArrayOffset(DL, ConstantOffset, ATy);
+  CBufferResource(GlobalVariable *GVHandle, GlobalVariable *Member,
+                  size_t MemberOffset)
+      : GVHandle(GVHandle), Member(Member), MemberOffset(MemberOffset) {}
 
-  return ConstantOffset.getZExtValue();
-}
+  const DataLayout &getDataLayout() { return GVHandle->getDataLayout(); }
+  Type *getValueType() { return Member->getValueType(); }
+  iterator_range<ConstantDataSequential::user_iterator> users() {
+    return Member->users();
+  }
+
+  /// Get the byte offset of a Pointer-typed Value * `Val` relative to Member.
+  /// `Val` can either be Member itself, or a GEP of a constant offset from
+  /// Member
+  size_t getOffsetForCBufferGEP(Value *Val) {
+    assert(isa<PointerType>(Val->getType()) &&
+           "Expected a pointer-typed value");
+
+    if (Val == Member)
+      return 0;
+
+    if (auto *GEP = dyn_cast<GEPOperator>(Val)) {
+      // Since we should always have a constant offset, we should only ever have
+      // a single GEP of indirection from the Global.
+      assert(GEP->getPointerOperand() == Member &&
+             "Indirect access to resource handle");
+
+      const DataLayout &DL = getDataLayout();
+      APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
+      bool Success = GEP->accumulateConstantOffset(DL, ConstantOffset);
+      (void)Success;
+      assert(Success && "Offsets into cbuffer globals must be constant");
+
+      if (auto *ATy = dyn_cast<ArrayType>(Member->getValueType()))
+        ConstantOffset =
+            hlsl::translateCBufArrayOffset(DL, ConstantOffset, ATy);
+
+      return ConstantOffset.getZExtValue();
+    }
 
-/// Replace access via cbuffer global with a load from the cbuffer handle
-/// itself.
-static void replaceAccess(LoadInst *LI, GlobalVariable *Global,
-                          GlobalVariable *HandleGV, size_t BaseOffset,
-                          SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
-  const DataLayout &DL = HandleGV->getDataLayout();
+    llvm_unreachable("Expected Val to be a GlobalVariable or GEP");
+  }
 
-  size_t Offset = BaseOffset;
-  if (auto *GEP = dyn_cast<GEPOperator>(LI->getPointerOperand()))
-    Offset += getOffsetForCBufferGEP(GEP, Global, DL);
-  else if (LI->getPointerOperand() != Global)
-    llvm_unreachable("Load instruction doesn't reference cbuffer global");
+  /// Create a handle for this cbuffer resource using the IRBuilder `Builder`
+  /// and sets the handle as the current one to use for subsequent calls to
+  /// `loadValue`
+  void createAndSetCurrentHandle(IRBuilder<> &Builder) {
+    Handle = Builder.CreateLoad(GVHandle->getValueType(), GVHandle,
+                                GVHandle->getName());
+  }
 
-  IRBuilder<> Builder(LI);
-  auto *Handle = Builder.CreateLoad(HandleGV->getValueType(), HandleGV,
-                                    HandleGV->getName());
-
-  Type *Ty = LI->getType();
-  CBufferRowIntrin Intrin(DL, Ty->getScalarType());
-  // The cbuffer consists of some number of 16-byte rows.
-  unsigned int CurrentRow = Offset / hlsl::CBufferRowSizeInBytes;
-  unsigned int CurrentIndex =
-      (Offset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
-
-  auto *CBufLoad = Builder.CreateIntrinsic(
-      Intrin.RetTy, Intrin.IID,
-      {Handle, ConstantInt::get(Builder.getInt32Ty(), CurrentRow)}, nullptr,
-      LI->getName());
-  auto *Elt =
-      Builder.CreateExtractValue(CBufLoad, {CurrentIndex++}, LI->getName());
-
-  Value *Result = nullptr;
-  unsigned int Remaining =
-      ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
-  if (Remaining == 0) {
-    // We only have a single element, so we're done.
-    Result = Elt;
-
-    // However, if we loaded a <1 x T>, then we need to adjust the type here.
-    if (auto *VT = dyn_cast<FixedVectorType>(LI->getType())) {
-      assert(VT->getNumElements() == 1 && "Can't have multiple elements here");
-      Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,
-                                           Builder.getInt32(0));
+  /// Load a value of type `Ty` at offset `Offset` using the handle from the
+  /// last call to `createAndSetCurrentHandle`
+  Value *loadValue(IRBuilder<> &Builder, Type *Ty, size_t Offset,
+                   const Twine &Name = "") {
+    assert(Handle &&
+           "Expected a handle for this cbuffer global resource to be created "
+           "before loading a value from it");
+    const DataLayout &DL = getDataLayout();
+
+    size_t TargetOffset = MemberOffset + Offset;
+    CBufferRowIntrin Intrin(DL, Ty->getScalarType());
+    // The cbuffer consists of some number of 16-byte rows.
+    unsigned int CurrentRow = TargetOffset / hlsl::CBufferRowSizeInBytes;
+    unsigned int CurrentIndex =
+        (TargetOffset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;
+
+    auto *CBufLoad = Builder.CreateIntrinsic(
+        Intrin.RetTy, Intrin.IID,
+        {Handle, ConstantInt::get(Builder.getInt32Ty(), CurrentRow)}, nullptr,
+        Name + ".load");
+    auto *Elt = Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
+                                           Name + ".extract");
+
+    Value *Result = nullptr;
+    unsigned int Remaining =
+        ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;
+
+    if (Remaining == 0) {
+      // We only have a single element, so we're done.
+      Result = Elt;
+
+      // However, if we loaded a <1 x T>, then we need to adjust the type here.
+      if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
+        assert(VT->getNumElements() == 1 &&
+               "Can't have multiple elements here");
+        Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,
+                                             Builder.getInt32(0), Name);
+      }
+      return Result;
     }
-  } else {
+
     // Walk each element and extract it, wrapping to new rows as needed.
     SmallVector<Value *> Extracts{Elt};
     while (Remaining--) {
@@ -128,40 +165,138 @@ static void replaceAccess(LoadInst *LI, GlobalVariable *Global,
         CBufLoad = Builder.CreateIntrinsic(
             Intrin.RetTy, Intrin.IID,
             {Handle, ConstantInt::get(Builder.getInt32Ty(), ++CurrentRow)},
-            nullptr, LI->getName());
+            nullptr, Name + ".load");
 
       Extracts.push_back(Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},
-                                                    LI->getName()));
+                                                    Name + ".extract"));
     }
 
     // Finally, we build up the original loaded value.
     Result = PoisonValue::get(Ty);
     for (int I = 0, E = Extracts.size(); I < E; ++I)
       Result =
-          Builder.CreateInsertElement(Result, Extracts[I], Builder.getInt32(I));
+          Builder.CreateInsertElement(Result, Extracts[I], Builder.getInt32(I),
+                                      Name + formatv(".upto{}", I));
+    return Result;
   }
+};
 
+} // namespace
+
+/// Replace load via cbuffer global with a load from the cbuffer handle itself.
+static void replaceLoad(LoadInst *LI, CBufferResource &CBR,
+                        SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
+  size_t Offset = CBR.getOffsetForCBufferGEP(LI->getPointerOperand());
+  IRBuilder<> Builder(LI);
+  CBR.createAndSetCurrentHandle(Builder);
+  Value *Result = CBR.loadValue(Builder, LI->getType(), Offset, LI->getName());
   LI->replaceAllUsesWith(Result);
   DeadInsts.push_back(LI);
 }
 
-static void replaceAccessesWithHandle(GlobalVariable *Global,
-                                      GlobalVariable *HandleGV,
-                                      size_t BaseOffset) {
+/// This function recursively copies N array elements from the cbuffer resource
+/// CBR to the MemCpy Destination. Recursion is used to unravel multidimensional
+/// arrays into a sequence of scalar/vector extracts and stores.
+static void copyArrayElemsForMemCpy(IRBuilder<> &Builder, MemCpyInst *MCI,
+                                    CBufferResource &CBR, ArrayType *ArrTy,
+                                    size_t ArrOffset, size_t N,
+                                    const Twine &Name = "") {
+  const DataLayout &DL = MCI->getDataLayout();
+  Type *ElemTy = ArrTy->getElementType();
+  size_t ElemTySize = DL.getTypeAllocSize(ElemTy);
+  for (unsigned I = 0; I < N; ++I) {
+    size_t Offset = ArrOffset + I * ElemTySize;
+
+    // Recursively copy nested arrays
+    if (ArrayType *ElemArrTy = dyn_cast<ArrayType>(ElemTy)) {
+      copyArrayElemsForMemCpy(Builder, MCI, CBR, ElemArrTy, Offset,
+                              ElemArrTy->getNumElements(), Name);
+      continue;
+    }
+
+    // Load CBuffer value and store it in Dest
+    APInt CBufArrayOffset(
+        DL.getIndexTypeSizeInBits(MCI->getSource()->getType()), Offset);
+    CBufArrayOffset =
+        hlsl::translateCBufArrayOffset(DL, CBufArrayOffset, ArrTy);
+    Value *CBufferVal =
+        CBR.loadValue(Builder, ElemTy, CBufArrayOffset.getZExtValue(), Name);
+    Value *GEP =
+        Builder.CreateInBoundsGEP(Builder.getInt8Ty(), MCI->getDest(),
+                                  {Builder.getInt32(Offset)}, Name + ".dest");
+    Builder.CreateStore(CBufferVal, GEP, MCI->isVolatile());
+  }
+}
+
+/// Replace memcpy from a cbuffer global with a memcpy from the cbuffer handle
+/// itself. Assumes the cbuffer global is an array, and the length of bytes to
+/// copy is divisible by array element allocation size.
+/// The memcpy source must also be a direct cbuffer global reference, not a GEP.
+static void replaceMemCpy(MemCpyInst *MCI, CBufferResource &CBR) {
+
+  ArrayType *ArrTy = dyn_cast<ArrayType>(CBR.getValueType());
+  assert(ArrTy && "MemCpy lowering is only supported for array types");
+
+  // This assumption vastly simplifies the implementation
+  if (MCI->getSource() != CBR.Member)
+    reportFatalUsageError(
+        "Expected MemCpy source to be a cbuffer global variable");
+
+  ConstantInt *Length = dyn_cast<ConstantInt>(MCI->getLength());
+  uint64_t ByteLength = Length->getZExtValue();
+
+  // If length to copy is zero, no memcpy is needed
+  if (ByteLength == 0) {
+    MCI->eraseFromParent();
+    return;
+  }
+
+  const DataLayout &DL = CBR.getDataLayout();
+
+  Type *ElemTy = ArrTy->getElementType();
+  size_t ElemSize = DL.getTypeAllocSize(ElemTy);
+  assert(ByteLength % ElemSize == 0 &&
+         "Length of bytes to MemCpy must be divisible by allocation size of "
+         "source/destination array elements");
+  size_t ElemsToCpy = ByteLength / ElemSize;
+
+  IRBuilder<> Builder(MCI);
+  CBR.createAndSetCurrentHandle(Builder);
+
+  copyArrayElemsForMemCpy(Builder, MCI, CBR, ArrTy, 0, ElemsToCpy,
+                          "memcpy." + MCI->getDest()->getName() + "." +
+                              MCI->getSource()->getName());
+
+  MCI->eraseFromParent();
+}
+
+static void replaceAccessesWithHandle(CBufferResource &CBR) {
   SmallVector<WeakTrackingVH> DeadInsts;
 
-  SmallVector<User *> ToProcess{Global->users()};
+  SmallVector<User *> ToProcess{CBR.users()};
   while (!ToProcess.empty()) {
     User *Cur = ToProcess.pop_back_val();
 
     // If we have a load instruction, replace the access.
     if (auto *LI = dyn_cast<LoadInst>(Cur)) {
-      replaceAccess(LI, Global, HandleGV, BaseOffset, DeadInsts);
+      replaceLoad(LI, CBR, DeadInsts);
+      continue;
+    }
+
+    // If we have a memcpy instruction, replace it with multiple accesses and
+    // subsequent stores to the destination
+    if (auto *MCI = dyn_cast<MemCpyInst>(Cur)) {
+      replaceMemCpy(MCI, CBR);
       continue;
     }
 
     // Otherwise, walk users looking for a load...
-    ToProcess.append(Cur->user_begin(), Cur->user_end());
+    if (isa<GetElementPtrInst>(Cur) || isa<GEPOperator>(Cur)) {
+      ToProcess.append(Cur->user_begin(), Cur->user_end());
+      continue;
+    }
+
+    llvm_unreachable("Unexpected user of Global");
   }
   RecursivelyDeleteTriviallyDeadInstructions(DeadInsts);
 }
@@ -173,7 +308,8 @@ static bool replaceCBufferAccesses(Module &M) {
 
   for (const hlsl::CBufferMapping &Mapping : *CBufMD)
     for (const hlsl::CBufferMember &Member : Mapping.Members) {
-      replaceAccessesWithHandle(Member.GV, Mapping.Handle, Member.Offset);
+      CBufferResource CBR(Mapping.Handle, Member.GV, Member.Offset);
+      replaceAccessesWithHandle(CBR);
       Member.GV->removeFromParent();
     }
 
diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
index 0a9b2bb99f7e..c97c604fdbf7 100644
--- a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -117,19 +117,23 @@ DataScalarizerVisitor::lookupReplacementGlobal(Value *CurrOperand) {
   return nullptr; // Not found
 }
 
-static bool isArrayOfVectors(Type *T) {
+// Helper function to check if a type is a vector or an array of vectors
+static bool isVectorOrArrayOfVectors(Type *T) {
+  if (isa<VectorType>(T))
+    return true;
   if (ArrayType *ArrType = dyn_cast<ArrayType>(T))
-    return isa<VectorType>(ArrType->getElementType());
+    return isa<VectorType>(ArrType->getElementType()) ||
+           isVectorOrArrayOfVectors(ArrType->getElementType());
   return false;
 }
 
 bool DataScalarizerVisitor::visitAllocaInst(AllocaInst &AI) {
-  if (!isArrayOfVectors(AI.getAllocatedType()))
+  Type *AllocatedType = AI.getAllocatedType();
+  if (!isVectorOrArrayOfVectors(AllocatedType))
     return false;
 
-  ArrayType *ArrType = cast<ArrayType>(AI.getAllocatedType());
   IRBuilder<> Builder(&AI);
-  Type *NewType = equivalentArrayTypeFromVector(ArrType);
+  Type *NewType = equivalentArrayTypeFromVector(AllocatedType);
   AllocaInst *ArrAlloca =
       Builder.CreateAlloca(NewType, nullptr, AI.getName() + ".scalarize");
   ArrAlloca->setAlignment(AI.getAlign());
@@ -308,9 +312,8 @@ bool DataScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
     NeedsTransform = true;
   } else if (AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrOperand)) {
     Type *AllocatedType = Alloca->getAllocatedType();
-    // OrigGEPType might just be a pointer lets make sure
-    // to add the allocated type so we have a size
-    if (AllocatedType != OrigGEPType) {
+    // Only transform if the allocated type is an array
+    if (AllocatedType != OrigGEPType && isa<ArrayType>(AllocatedType)) {
       NewGEPType = AllocatedType;
       NeedsTransform = true;
     }
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index f99e8e7ccdc5..435b80ecaec6 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -25,6 +25,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 
@@ -70,15 +71,17 @@ static bool isIntrinsicExpansion(Function &F) {
   case Intrinsic::vector_reduce_add:
   case Intrinsic::vector_reduce_fadd:
     return true;
-  case Intrinsic::dx_resource_load_typedbuffer:
-    // We need to handle doubles and vector of doubles.
-    return F.getReturnType()
-        ->getStructElementType(0)
-        ->getScalarType()
-        ->isDoubleTy();
-  case Intrinsic::dx_resource_store_typedbuffer:
-    // We need to handle doubles and vector of doubles.
-    return F.getFunctionType()->getParamType(2)->getScalarType()->isDoubleTy();
+  case Intrinsic::dx_resource_load_typedbuffer: {
+    // We need to handle i64, doubles, and vectors of them.
+    Type *ScalarTy =
+        F.getReturnType()->getStructElementType(0)->getScalarType();
+    return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(64);
+  }
+  case Intrinsic::dx_resource_store_typedbuffer: {
+    // We need to handle i64 and doubles and vectors of i64 and doubles.
+    Type *ScalarTy = F.getFunctionType()->getParamType(2)->getScalarType();
+    return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(64);
+  }
   }
   return false;
 }
@@ -545,13 +548,15 @@ static bool expandTypedBufferLoadIntrinsic(CallInst *Orig) {
   IRBuilder<> Builder(Orig);
 
   Type *BufferTy = Orig->getType()->getStructElementType(0);
-  assert(BufferTy->getScalarType()->isDoubleTy() &&
-         "Only expand double or double2");
+  Type *ScalarTy = BufferTy->getScalarType();
+  bool IsDouble = ScalarTy->isDoubleTy();
+  assert(IsDouble || ScalarTy->isIntegerTy(64) &&
+                         "Only expand double or int64 scalars or vectors");
 
   unsigned ExtractNum = 2;
   if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
     assert(VT->getNumElements() == 2 &&
-           "TypedBufferLoad double vector has wrong size");
+           "TypedBufferLoad vector must be size 2");
     ExtractNum = 4;
   }
 
@@ -570,22 +575,42 @@ static bool expandTypedBufferLoadIntrinsic(CallInst *Orig) {
     ExtractElements.push_back(
         Builder.CreateExtractElement(Extract, Builder.getInt32(I)));
 
-  // combine into double(s)
+  // combine into double(s) or int64(s)
   Value *Result = PoisonValue::get(BufferTy);
   for (unsigned I = 0; I < ExtractNum; I += 2) {
-    Value *Dbl =
-        Builder.CreateIntrinsic(Builder.getDoubleTy(), Intrinsic::dx_asdouble,
-                                {ExtractElements[I], ExtractElements[I + 1]});
+    Value *Combined = nullptr;
+    if (IsDouble)
+      // For doubles, use dx_asdouble intrinsic
+      Combined =
+          Builder.CreateIntrinsic(Builder.getDoubleTy(), Intrinsic::dx_asdouble,
+                                  {ExtractElements[I], ExtractElements[I + 1]});
+    else {
+      // For int64, manually combine two int32s
+      // First, zero-extend both values to i64
+      Value *Lo = Builder.CreateZExt(ExtractElements[I], Builder.getInt64Ty());
+      Value *Hi =
+          Builder.CreateZExt(ExtractElements[I + 1], Builder.getInt64Ty());
+      // Shift the high bits left by 32 bits
+      Value *ShiftedHi = Builder.CreateShl(Hi, Builder.getInt64(32));
+      // OR the high and low bits together
+      Combined = Builder.CreateOr(Lo, ShiftedHi);
+    }
+
     if (ExtractNum == 4)
-      Result =
-          Builder.CreateInsertElement(Result, Dbl, Builder.getInt32(I / 2));
+      Result = Builder.CreateInsertElement(Result, Combined,
+                                           Builder.getInt32(I / 2));
     else
-      Result = Dbl;
+      Result = Combined;
   }
 
   Value *CheckBit = nullptr;
   for (User *U : make_early_inc_range(Orig->users())) {
-    auto *EVI = cast<ExtractValueInst>(U);
+    // If it's not a ExtractValueInst, we don't know how to
+    // handle it
+    auto *EVI = dyn_cast<ExtractValueInst>(U);
+    if (!EVI)
+      llvm_unreachable("Unexpected user of typedbufferload");
+
     ArrayRef<unsigned> Indices = EVI->getIndices();
     assert(Indices.size() == 1);
 
@@ -609,38 +634,61 @@ static bool expandTypedBufferStoreIntrinsic(CallInst *Orig) {
   IRBuilder<> Builder(Orig);
 
   Type *BufferTy = Orig->getFunctionType()->getParamType(2);
-  assert(BufferTy->getScalarType()->isDoubleTy() &&
-         "Only expand double or double2");
-
-  unsigned ExtractNum = 2;
-  if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {
-    assert(VT->getNumElements() == 2 &&
-           "TypedBufferStore double vector has wrong size");
-    ExtractNum = 4;
+  Type *ScalarTy = BufferTy->getScalarType();
+  bool IsDouble = ScalarTy->isDoubleTy();
+  assert((IsDouble || ScalarTy->isIntegerTy(64)) &&
+         "Only expand double or int64 scalars or vectors");
+
+  // Determine if we're dealing with a vector or scalar
+  bool IsVector = isa<FixedVectorType>(BufferTy);
+  if (IsVector) {
+    assert(cast<FixedVectorType>(BufferTy)->getNumElements() == 2 &&
+           "TypedBufferStore vector must be size 2");
   }
 
-  Type *SplitElementTy = Builder.getInt32Ty();
-  if (ExtractNum == 4)
+  // Create the appropriate vector type for the result
+  Type *Int32Ty = Builder.getInt32Ty();
+  Type *ResultTy = VectorType::get(Int32Ty, IsVector ? 4 : 2, false);
+  Value *Val = PoisonValue::get(ResultTy);
+
+  Type *SplitElementTy = Int32Ty;
+  if (IsVector)
     SplitElementTy = VectorType::get(SplitElementTy, 2, false);
 
-  // split our double(s)
-  auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);
-  Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
-                                         Orig->getOperand(2));
-  // create our vector
-  Value *LowBits = Builder.CreateExtractValue(Split, 0);
-  Value *HighBits = Builder.CreateExtractValue(Split, 1);
-  Value *Val;
-  if (ExtractNum == 2) {
-    Val = PoisonValue::get(VectorType::get(SplitElementTy, 2, false));
+  Value *LowBits = nullptr;
+  Value *HighBits = nullptr;
+  // Split the 64-bit values into 32-bit components
+  if (IsDouble) {
+    auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);
+    Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
+                                           {Orig->getOperand(2)});
+    LowBits = Builder.CreateExtractValue(Split, 0);
+    HighBits = Builder.CreateExtractValue(Split, 1);
+  } else {
+    // Handle int64 type(s)
+    Value *InputVal = Orig->getOperand(2);
+    Constant *ShiftAmt = Builder.getInt64(32);
+    if (IsVector)
+      ShiftAmt = ConstantVector::getSplat(ElementCount::getFixed(2), ShiftAmt);
+
+    // Split into low and high 32-bit parts
+    LowBits = Builder.CreateTrunc(InputVal, SplitElementTy);
+    Value *ShiftedVal = Builder.CreateLShr(InputVal, ShiftAmt);
+    HighBits = Builder.CreateTrunc(ShiftedVal, SplitElementTy);
+  }
+
+  if (IsVector) {
+    Val = Builder.CreateShuffleVector(LowBits, HighBits, {0, 2, 1, 3});
+  } else {
     Val = Builder.CreateInsertElement(Val, LowBits, Builder.getInt32(0));
     Val = Builder.CreateInsertElement(Val, HighBits, Builder.getInt32(1));
-  } else
-    Val = Builder.CreateShuffleVector(LowBits, HighBits, {0, 2, 1, 3});
+  }
 
+  // Create the final intrinsic call
   Builder.CreateIntrinsic(Builder.getVoidTy(),
                           Intrinsic::dx_resource_store_typedbuffer,
                           {Orig->getOperand(0), Orig->getOperand(1), Val});
+
   Orig->eraseFromParent();
   return true;
 }
diff --git a/llvm/lib/Target/DirectX/DXILPrepare.cpp b/llvm/lib/Target/DirectX/DXILPrepare.cpp
index cb58f4833631..c8866bfefdfc 100644
--- a/llvm/lib/Target/DirectX/DXILPrepare.cpp
+++ b/llvm/lib/Target/DirectX/DXILPrepare.cpp
@@ -11,6 +11,7 @@
 /// Language (DXIL).
 //===----------------------------------------------------------------------===//
 
+#include "DXILRootSignature.h"
 #include "DXILShaderFlags.h"
 #include "DirectX.h"
 #include "DirectXIRPasses/PointerTypeAnalysis.h"
@@ -286,12 +287,21 @@ class DXILPrepareModule : public ModulePass {
     }
     // Remove flags not for DXIL.
     cleanModuleFlags(M);
+
+    // dx.rootsignatures will have been parsed from its metadata form as its
+    // binary form as part of the RootSignatureAnalysisWrapper, so safely
+    // remove it as it is not recognized in DXIL
+    if (NamedMDNode *RootSignature = M.getNamedMetadata("dx.rootsignatures"))
+      RootSignature->eraseFromParent();
+
     return true;
   }
 
   DXILPrepareModule() : ModulePass(ID) {}
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<DXILMetadataAnalysisWrapperPass>();
+    AU.addRequired<RootSignatureAnalysisWrapper>();
+    AU.addPreserved<RootSignatureAnalysisWrapper>();
     AU.addPreserved<ShaderFlagsAnalysisWrapper>();
     AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
     AU.addPreserved<DXILResourceWrapperPass>();
@@ -305,6 +315,7 @@ char DXILPrepareModule::ID = 0;
 INITIALIZE_PASS_BEGIN(DXILPrepareModule, DEBUG_TYPE, "DXIL Prepare Module",
                       false, false)
 INITIALIZE_PASS_DEPENDENCY(DXILMetadataAnalysisWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(RootSignatureAnalysisWrapper)
 INITIALIZE_PASS_END(DXILPrepareModule, DEBUG_TYPE, "DXIL Prepare Module", false,
                     false)
 
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 29e78fcce526..5e0975736f90 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cmath>
 #include <cstdint>
 #include <optional>
 #include <utility>
@@ -55,6 +56,13 @@ static std::optional<uint32_t> extractMdIntValue(MDNode *Node,
   return std::nullopt;
 }
 
+static std::optional<float> extractMdFloatValue(MDNode *Node,
+                                                unsigned int OpId) {
+  if (auto *CI = mdconst::dyn_extract<ConstantFP>(Node->getOperand(OpId).get()))
+    return CI->getValueAPF().convertToFloat();
+  return std::nullopt;
+}
+
 static std::optional<StringRef> extractMdStringValue(MDNode *Node,
                                                      unsigned int OpId) {
   MDString *NodeText = dyn_cast<MDString>(Node->getOperand(OpId));
@@ -261,6 +269,81 @@ static bool parseDescriptorTable(LLVMContext *Ctx,
   return false;
 }
 
+static bool parseStaticSampler(LLVMContext *Ctx, mcdxbc::RootSignatureDesc &RSD,
+                               MDNode *StaticSamplerNode) {
+  if (StaticSamplerNode->getNumOperands() != 14)
+    return reportError(Ctx, "Invalid format for Static Sampler");
+
+  dxbc::RTS0::v1::StaticSampler Sampler;
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 1))
+    Sampler.Filter = *Val;
+  else
+    return reportError(Ctx, "Invalid value for Filter");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 2))
+    Sampler.AddressU = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressU");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 3))
+    Sampler.AddressV = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressV");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 4))
+    Sampler.AddressW = *Val;
+  else
+    return reportError(Ctx, "Invalid value for AddressW");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 5))
+    Sampler.MipLODBias = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MipLODBias");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 6))
+    Sampler.MaxAnisotropy = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MaxAnisotropy");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 7))
+    Sampler.ComparisonFunc = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ComparisonFunc ");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 8))
+    Sampler.BorderColor = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ComparisonFunc ");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 9))
+    Sampler.MinLOD = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MinLOD");
+
+  if (std::optional<float> Val = extractMdFloatValue(StaticSamplerNode, 10))
+    Sampler.MaxLOD = *Val;
+  else
+    return reportError(Ctx, "Invalid value for MaxLOD");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 11))
+    Sampler.ShaderRegister = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderRegister");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 12))
+    Sampler.RegisterSpace = *Val;
+  else
+    return reportError(Ctx, "Invalid value for RegisterSpace");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(StaticSamplerNode, 13))
+    Sampler.ShaderVisibility = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderVisibility");
+
+  RSD.StaticSamplers.push_back(Sampler);
+  return false;
+}
+
 static bool parseRootSignatureElement(LLVMContext *Ctx,
                                       mcdxbc::RootSignatureDesc &RSD,
                                       MDNode *Element) {
@@ -276,6 +359,7 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
           .Case("RootSRV", RootSignatureElementKind::SRV)
           .Case("RootUAV", RootSignatureElementKind::UAV)
           .Case("DescriptorTable", RootSignatureElementKind::DescriptorTable)
+          .Case("StaticSampler", RootSignatureElementKind::StaticSamplers)
           .Default(RootSignatureElementKind::Error);
 
   switch (ElementKind) {
@@ -290,6 +374,8 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
     return parseRootDescriptors(Ctx, RSD, Element, ElementKind);
   case RootSignatureElementKind::DescriptorTable:
     return parseDescriptorTable(Ctx, RSD, Element);
+  case RootSignatureElementKind::StaticSamplers:
+    return parseStaticSampler(Ctx, RSD, Element);
   case RootSignatureElementKind::Error:
     return reportError(Ctx, "Invalid Root Signature Element: " + *ElementText);
   }
@@ -345,7 +431,7 @@ static bool verifyRangeType(uint32_t Type) {
 
 static bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
                                       uint32_t FlagsVal) {
-  using FlagT = dxbc::DescriptorRangeFlag;
+  using FlagT = dxbc::DescriptorRangeFlags;
   FlagT Flags = FlagT(FlagsVal);
 
   const bool IsSampler =
@@ -355,57 +441,107 @@ static bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
     // Since the metadata is unversioned, we expect to explicitly see the values
     // that map to the version 1 behaviour here.
     if (IsSampler)
-      return Flags == FlagT::DESCRIPTORS_VOLATILE;
-    return Flags == (FlagT::DATA_VOLATILE | FlagT::DESCRIPTORS_VOLATILE);
+      return Flags == FlagT::DescriptorsVolatile;
+    return Flags == (FlagT::DataVolatile | FlagT::DescriptorsVolatile);
   }
 
   // The data-specific flags are mutually exclusive.
-  FlagT DataFlags = FlagT::DATA_VOLATILE | FlagT::DATA_STATIC |
-                    FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+  FlagT DataFlags = FlagT::DataVolatile | FlagT::DataStatic |
+                    FlagT::DataStaticWhileSetAtExecute;
 
   if (popcount(llvm::to_underlying(Flags & DataFlags)) > 1)
     return false;
 
   // The descriptor-specific flags are mutually exclusive.
-  FlagT DescriptorFlags =
-      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS |
-      FlagT::DESCRIPTORS_VOLATILE;
+  FlagT DescriptorFlags = FlagT::DescriptorsStaticKeepingBufferBoundsChecks |
+                          FlagT::DescriptorsVolatile;
   if (popcount(llvm::to_underlying(Flags & DescriptorFlags)) > 1)
     return false;
 
   // For volatile descriptors, DATA_STATIC is never valid.
-  if ((Flags & FlagT::DESCRIPTORS_VOLATILE) == FlagT::DESCRIPTORS_VOLATILE) {
-    FlagT Mask = FlagT::DESCRIPTORS_VOLATILE;
+  if ((Flags & FlagT::DescriptorsVolatile) == FlagT::DescriptorsVolatile) {
+    FlagT Mask = FlagT::DescriptorsVolatile;
     if (!IsSampler) {
-      Mask |= FlagT::DATA_VOLATILE;
-      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+      Mask |= FlagT::DataVolatile;
+      Mask |= FlagT::DataStaticWhileSetAtExecute;
     }
-    return (Flags & ~Mask) == FlagT::NONE;
+    return (Flags & ~Mask) == FlagT::None;
   }
 
   // For "STATIC_KEEPING_BUFFER_BOUNDS_CHECKS" descriptors,
   // the other data-specific flags may all be set.
-  if ((Flags & FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) ==
-      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) {
-    FlagT Mask = FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS;
+  if ((Flags & FlagT::DescriptorsStaticKeepingBufferBoundsChecks) ==
+      FlagT::DescriptorsStaticKeepingBufferBoundsChecks) {
+    FlagT Mask = FlagT::DescriptorsStaticKeepingBufferBoundsChecks;
     if (!IsSampler) {
-      Mask |= FlagT::DATA_VOLATILE;
-      Mask |= FlagT::DATA_STATIC;
-      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+      Mask |= FlagT::DataVolatile;
+      Mask |= FlagT::DataStatic;
+      Mask |= FlagT::DataStaticWhileSetAtExecute;
     }
-    return (Flags & ~Mask) == FlagT::NONE;
+    return (Flags & ~Mask) == FlagT::None;
   }
 
   // When no descriptor flag is set, any data flag is allowed.
-  FlagT Mask = FlagT::NONE;
+  FlagT Mask = FlagT::None;
   if (!IsSampler) {
-    Mask |= FlagT::DATA_VOLATILE;
-    Mask |= FlagT::DATA_STATIC;
-    Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+    Mask |= FlagT::DataVolatile;
+    Mask |= FlagT::DataStaticWhileSetAtExecute;
+    Mask |= FlagT::DataStatic;
   }
-  return (Flags & ~Mask) == FlagT::NONE;
+  return (Flags & ~Mask) == FlagT::None;
 }
 
+static bool verifySamplerFilter(uint32_t Value) {
+  switch (Value) {
+#define FILTER(Num, Val) case llvm::to_underlying(dxbc::SamplerFilter::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+// Values allowed here:
+// https://learn.microsoft.com/en-us/windows/win32/api/d3d12/ne-d3d12-d3d12_texture_address_mode#syntax
+static bool verifyAddress(uint32_t Address) {
+  switch (Address) {
+#define TEXTURE_ADDRESS_MODE(Num, Val)                                         \
+  case llvm::to_underlying(dxbc::TextureAddressMode::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+static bool verifyMipLODBias(float MipLODBias) {
+  return MipLODBias >= -16.f && MipLODBias <= 15.99f;
+}
+
+static bool verifyMaxAnisotropy(uint32_t MaxAnisotropy) {
+  return MaxAnisotropy <= 16u;
+}
+
+static bool verifyComparisonFunc(uint32_t ComparisonFunc) {
+  switch (ComparisonFunc) {
+#define COMPARISON_FUNC(Num, Val)                                              \
+  case llvm::to_underlying(dxbc::ComparisonFunc::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+static bool verifyBorderColor(uint32_t BorderColor) {
+  switch (BorderColor) {
+#define STATIC_BORDER_COLOR(Num, Val)                                          \
+  case llvm::to_underlying(dxbc::StaticBorderColor::Val):
+#include "llvm/BinaryFormat/DXContainerConstants.def"
+    return true;
+  }
+  return false;
+}
+
+static bool verifyLOD(float LOD) { return !std::isnan(LOD); }
+
 static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
   if (!verifyVersion(RSD.Version)) {
@@ -463,6 +599,48 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
     }
   }
 
+  for (const dxbc::RTS0::v1::StaticSampler &Sampler : RSD.StaticSamplers) {
+    if (!verifySamplerFilter(Sampler.Filter))
+      return reportValueError(Ctx, "Filter", Sampler.Filter);
+
+    if (!verifyAddress(Sampler.AddressU))
+      return reportValueError(Ctx, "AddressU", Sampler.AddressU);
+
+    if (!verifyAddress(Sampler.AddressV))
+      return reportValueError(Ctx, "AddressV", Sampler.AddressV);
+
+    if (!verifyAddress(Sampler.AddressW))
+      return reportValueError(Ctx, "AddressW", Sampler.AddressW);
+
+    if (!verifyMipLODBias(Sampler.MipLODBias))
+      return reportValueError(Ctx, "MipLODBias", Sampler.MipLODBias);
+
+    if (!verifyMaxAnisotropy(Sampler.MaxAnisotropy))
+      return reportValueError(Ctx, "MaxAnisotropy", Sampler.MaxAnisotropy);
+
+    if (!verifyComparisonFunc(Sampler.ComparisonFunc))
+      return reportValueError(Ctx, "ComparisonFunc", Sampler.ComparisonFunc);
+
+    if (!verifyBorderColor(Sampler.BorderColor))
+      return reportValueError(Ctx, "BorderColor", Sampler.BorderColor);
+
+    if (!verifyLOD(Sampler.MinLOD))
+      return reportValueError(Ctx, "MinLOD", Sampler.MinLOD);
+
+    if (!verifyLOD(Sampler.MaxLOD))
+      return reportValueError(Ctx, "MaxLOD", Sampler.MaxLOD);
+
+    if (!verifyRegisterValue(Sampler.ShaderRegister))
+      return reportValueError(Ctx, "ShaderRegister", Sampler.ShaderRegister);
+
+    if (!verifyRegisterSpace(Sampler.RegisterSpace))
+      return reportValueError(Ctx, "RegisterSpace", Sampler.RegisterSpace);
+
+    if (!dxbc::isValidShaderVisibility(Sampler.ShaderVisibility))
+      return reportValueError(Ctx, "ShaderVisibility",
+                              Sampler.ShaderVisibility);
+  }
+
   return false;
 }
 
@@ -542,6 +720,9 @@ analyzeModule(Module &M) {
     // offset will always equal to the header size.
     RSD.RootParameterOffset = sizeof(dxbc::RTS0::v1::RootSignatureHeader);
 
+    // static sampler offset is calculated when writting dxcontainer.
+    RSD.StaticSamplersOffset = 0u;
+
     if (parse(Ctx, RSD, RootElementListNode) || validate(Ctx, RSD)) {
       return RSDMap;
     }
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.h b/llvm/lib/Target/DirectX/DXILRootSignature.h
index b45cebc15fd3..be5cc78bc6bd 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.h
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.h
@@ -32,6 +32,7 @@ enum class RootSignatureElementKind {
   UAV = 4,
   CBV = 5,
   DescriptorTable = 6,
+  StaticSamplers = 7
 };
 class RootSignatureAnalysis : public AnalysisInfoMixin<RootSignatureAnalysis> {
   friend AnalysisInfoMixin<RootSignatureAnalysis>;
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index 84b2603b4c57..ffd900c68893 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -31,6 +31,8 @@ bool DirectXTTIImpl::isTargetIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
   case Intrinsic::dx_asdouble:
   case Intrinsic::dx_isinf:
   case Intrinsic::dx_firstbitlow:
+  case Intrinsic::dx_firstbituhigh:
+  case Intrinsic::dx_firstbitshigh:
     return OpdIdx == 0;
   default:
     return OpdIdx == -1;
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index c423dca90a4a..b94b1484205a 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -423,7 +423,7 @@ struct HexagonOperand : public MCParsedAsmOperand {
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<HexagonOperand> CreateToken(MCContext &Context,
                                                      StringRef Str, SMLoc S) {
@@ -456,7 +456,7 @@ struct HexagonOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace
 
-void HexagonOperand::print(raw_ostream &OS) const {
+void HexagonOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case Immediate:
     HexagonMCAsmInfo(Triple()).printExpr(OS, *getImm());
diff --git a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
index 5a3b6433fba7..e152dbf764c9 100644
--- a/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
+++ b/llvm/lib/Target/Hexagon/HexagonBlockRanges.h
@@ -37,7 +37,7 @@ struct HexagonBlockRanges {
     unsigned Sub;
 
     bool operator<(RegisterRef R) const {
-      return Reg < R.Reg || (Reg == R.Reg && Sub < R.Sub);
+      return std::tie(Reg, Sub) < std::tie(R.Reg, R.Sub);
     }
   };
   using RegisterSet = std::set<RegisterRef>;
diff --git a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
index 740479174c81..bb7814c5226f 100644
--- a/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonConstPropagation.cpp
@@ -105,8 +105,7 @@ namespace {
     };
 
     LatticeCell() : Kind(Top), Size(0), IsSpecial(false) {
-      for (const Constant *&Value : Values)
-        Value = nullptr;
+      llvm::fill(Values, nullptr);
     }
 
     bool meet(const LatticeCell &L);
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index b75c8863b699..b3c61e1829bf 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -2196,10 +2196,7 @@ void HexagonFrameLowering::optimizeSpillSlots(MachineFunction &MF,
   // and collect relevant information.
   for (auto &B : MF) {
     std::map<int,IndexType> LastStore, LastLoad;
-    // Emplace appears not to be supported in gcc 4.7.2-4.
-    //auto P = BlockIndexes.emplace(&B, HexagonBlockRanges::InstrIndexMap(B));
-    auto P = BlockIndexes.insert(
-                std::make_pair(&B, HexagonBlockRanges::InstrIndexMap(B)));
+    auto P = BlockIndexes.emplace(&B, HexagonBlockRanges::InstrIndexMap(B));
     auto &IndexMap = P.first->second;
     LLVM_DEBUG(dbgs() << "Index map for " << printMBBReference(B) << "\n"
                       << IndexMap << '\n');
diff --git a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
index fcd0429eae8e..1322c627ecfb 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenExtract.cpp
@@ -44,9 +44,6 @@ static cl::opt<unsigned> ExtractCutoff("extract-cutoff", cl::init(~0U),
 static cl::opt<bool> NoSR0("extract-nosr0", cl::init(true), cl::Hidden,
   cl::desc("No extract instruction with offset 0"));
 
-static cl::opt<bool> NeedAnd("extract-needand", cl::init(true), cl::Hidden,
-  cl::desc("Require & in extract patterns"));
-
 namespace {
 
   class HexagonGenExtract : public FunctionPass {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 6534ae938fed..00925ed42fcd 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -102,6 +102,10 @@ static cl::opt<int>
     MaxStoresPerMemsetOptSizeCL("max-store-memset-Os", cl::Hidden, cl::init(4),
                                 cl::desc("Max #stores to inline memset"));
 
+static cl::opt<bool>
+    ConstantLoadsToImm("constant-loads-to-imm", cl::Hidden, cl::init(true),
+                       cl::desc("Convert constant loads to immediate values."));
+
 static cl::opt<bool> AlignLoads("hexagon-align-loads",
   cl::Hidden, cl::init(false),
   cl::desc("Rewrite unaligned loads as a pair of aligned loads"));
@@ -1179,9 +1183,6 @@ HexagonTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -3607,6 +3608,18 @@ bool HexagonTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
   return true;
 }
 
+/// Returns true if it is beneficial to convert a load of a constant
+/// to just the constant itself.
+bool HexagonTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
+                                                              Type *Ty) const {
+  if (!ConstantLoadsToImm)
+    return false;
+
+  assert(Ty->isIntegerTy());
+  unsigned BitSize = Ty->getPrimitiveSizeInBits();
+  return (BitSize > 0 && BitSize <= 64);
+}
+
 /// isLegalAddressingMode - Return true if the addressing mode represented by
 /// AM is legal for this target, for a load/store of the specified type.
 bool HexagonTargetLowering::isLegalAddressingMode(const DataLayout &DL,
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 1321bee44a29..a2c9b57d04ca 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -342,6 +342,11 @@ class HexagonTargetLowering : public TargetLowering {
   SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG)
                                    const override;
 
+  /// Returns true if it is beneficial to convert a load of a constant
+  /// to just the constant itself.
+  bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
+                                         Type *Ty) const override;
+
   bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
                              std::optional<unsigned> ByteOffset) const override;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index fbbcacf0d713..0e13dd3214da 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -1687,28 +1687,27 @@ HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
     // all operations are expected to be type-legalized, and i16 is not
     // a legal type. If any of the extracted elements is not of a valid
     // type, sign-extend it to a valid one.
-    for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
-      SDValue V = Elems[i];
+    for (SDValue &V : Elems) {
       MVT Ty = ty(V);
       if (!isTypeLegal(Ty)) {
         MVT NTy = typeLegalize(Ty, DAG);
         if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
-          Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
-                                 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
-                                             V.getOperand(0), V.getOperand(1)),
-                                 DAG.getValueType(Ty));
+          V = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
+                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
+                                      V.getOperand(0), V.getOperand(1)),
+                          DAG.getValueType(Ty));
           continue;
         }
         // A few less complicated cases.
         switch (V.getOpcode()) {
           case ISD::Constant:
-            Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
+            V = DAG.getSExtOrTrunc(V, dl, NTy);
             break;
           case ISD::UNDEF:
-            Elems[i] = DAG.getUNDEF(NTy);
+            V = DAG.getUNDEF(NTy);
             break;
           case ISD::TRUNCATE:
-            Elems[i] = V.getOperand(0);
+            V = V.getOperand(0);
             break;
           default:
             llvm_unreachable("Unexpected vector element");
diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index 6ea3f96cffd2..6dd83c1d820f 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -448,9 +448,7 @@ bool HexagonOptAddrMode::usedInLoadStore(NodeAddr<StmtNode *> CurrentInstSN,
 
   getAllRealUses(CurrentInstSN, LoadStoreUseList);
   bool FoundLoadStoreUse = false;
-  for (auto I = LoadStoreUseList.begin(), E = LoadStoreUseList.end(); I != E;
-       ++I) {
-    NodeAddr<UseNode *> UN = *I;
+  for (NodeAddr<UseNode *> UN : LoadStoreUseList) {
     NodeAddr<StmtNode *> SN = UN.Addr->getOwner(*DFG);
     MachineInstr *LoadStoreMI = SN.Addr->getCode();
     const MCInstrDesc &MID = LoadStoreMI->getDesc();
@@ -579,8 +577,7 @@ bool HexagonOptAddrMode::processAddBases(NodeAddr<StmtNode *> AddSN,
   // Find all Addi instructions that share the same base register and add them
   // to the AddiList
   getAllRealUses(ReachingDefStmt, AddiUseList);
-  for (auto I = AddiUseList.begin(), E = AddiUseList.end(); I != E; ++I) {
-    NodeAddr<UseNode *> UN = *I;
+  for (NodeAddr<UseNode *> UN : AddiUseList) {
     NodeAddr<StmtNode *> SN = UN.Addr->getOwner(*DFG);
     MachineInstr *MI = SN.Addr->getCode();
 
diff --git a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
index 1b724e8fcae9..610a81fe45c2 100644
--- a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
@@ -41,18 +41,20 @@ SDValue HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(
   Entry.Node = Size;
   Args.push_back(Entry);
 
-  const char *SpecialMemcpyName =
-      "__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes";
+  const char *SpecialMemcpyName = TLI.getLibcallName(
+      RTLIB::HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES);
   const MachineFunction &MF = DAG.getMachineFunction();
   bool LongCalls = MF.getSubtarget<HexagonSubtarget>().useLongCalls();
   unsigned Flags = LongCalls ? HexagonII::HMOTF_ConstExtended : 0;
 
+  CallingConv::ID CC = TLI.getLibcallCallingConv(
+      RTLIB::HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES);
+
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl)
       .setChain(Chain)
       .setLibCallee(
-          TLI.getLibcallCallingConv(RTLIB::MEMCPY),
-          Type::getVoidTy(*DAG.getContext()),
+          CC, Type::getVoidTy(*DAG.getContext()),
           DAG.getTargetExternalSymbol(
               SpecialMemcpyName, TLI.getPointerTy(DAG.getDataLayout()), Flags),
           std::move(Args))
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index 45bb82c945a7..847895f84ca9 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -84,14 +83,15 @@ class HexagonAsmBackend : public MCAsmBackend {
   }
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
+    // clang-format off
     const static MCFixupKindInfo Infos[Hexagon::NumTargetFixupKinds] = {
       // This table *must* be in same the order of fixup_* kinds in
       // HexagonFixupKinds.h.
       //
       // namei                          offset  bits    flags
-      { "fixup_Hexagon_B22_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B15_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B7_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B22_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B15_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B7_PCREL",       0,      32,     0 },
       { "fixup_Hexagon_LO16",           0,      32,     0 },
       { "fixup_Hexagon_HI16",           0,      32,     0 },
       { "fixup_Hexagon_32",             0,      32,     0 },
@@ -102,15 +102,15 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_GPREL16_2",      0,      32,     0 },
       { "fixup_Hexagon_GPREL16_3",      0,      32,     0 },
       { "fixup_Hexagon_HL16",           0,      32,     0 },
-      { "fixup_Hexagon_B13_PCREL",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B9_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B32_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B13_PCREL",      0,      32,     0 },
+      { "fixup_Hexagon_B9_PCREL",       0,      32,     0 },
+      { "fixup_Hexagon_B32_PCREL_X",    0,      32,     0 },
       { "fixup_Hexagon_32_6_X",         0,      32,     0 },
-      { "fixup_Hexagon_B22_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B15_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B13_PCREL_X",    0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B9_PCREL_X",     0,      32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_B7_PCREL_X",     0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_B22_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B15_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B13_PCREL_X",    0,      32,     0 },
+      { "fixup_Hexagon_B9_PCREL_X",     0,      32,     0 },
+      { "fixup_Hexagon_B7_PCREL_X",     0,      32,     0 },
       { "fixup_Hexagon_16_X",           0,      32,     0 },
       { "fixup_Hexagon_12_X",           0,      32,     0 },
       { "fixup_Hexagon_11_X",           0,      32,     0 },
@@ -119,12 +119,12 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_8_X",            0,      32,     0 },
       { "fixup_Hexagon_7_X",            0,      32,     0 },
       { "fixup_Hexagon_6_X",            0,      32,     0 },
-      { "fixup_Hexagon_32_PCREL",       0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_32_PCREL",       0,      32,     0 },
       { "fixup_Hexagon_COPY",           0,      32,     0 },
       { "fixup_Hexagon_GLOB_DAT",       0,      32,     0 },
       { "fixup_Hexagon_JMP_SLOT",       0,      32,     0 },
       { "fixup_Hexagon_RELATIVE",       0,      32,     0 },
-      { "fixup_Hexagon_PLT_B22_PCREL",  0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_PLT_B22_PCREL",  0,      32,     0 },
       { "fixup_Hexagon_GOTREL_LO16",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_HI16",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_32",      0,      32,     0 },
@@ -137,8 +137,8 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_DTPREL_HI16",    0,      32,     0 },
       { "fixup_Hexagon_DTPREL_32",      0,      32,     0 },
       { "fixup_Hexagon_DTPREL_16",      0,      32,     0 },
-      { "fixup_Hexagon_GD_PLT_B22_PCREL",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B22_PCREL",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_GD_PLT_B22_PCREL",0,     32,     0 },
+      { "fixup_Hexagon_LD_PLT_B22_PCREL",0,     32,     0 },
       { "fixup_Hexagon_GD_GOT_LO16",    0,      32,     0 },
       { "fixup_Hexagon_GD_GOT_HI16",    0,      32,     0 },
       { "fixup_Hexagon_GD_GOT_32",      0,      32,     0 },
@@ -159,7 +159,7 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_TPREL_HI16",     0,      32,     0 },
       { "fixup_Hexagon_TPREL_32",       0,      32,     0 },
       { "fixup_Hexagon_TPREL_16",       0,      32,     0 },
-      { "fixup_Hexagon_6_PCREL_X",      0,      32,     MCFixupKindInfo::FKF_IsPCRel },
+      { "fixup_Hexagon_6_PCREL_X",      0,      32,     0 },
       { "fixup_Hexagon_GOTREL_32_6_X",  0,      32,     0 },
       { "fixup_Hexagon_GOTREL_16_X",    0,      32,     0 },
       { "fixup_Hexagon_GOTREL_11_X",    0,      32,     0 },
@@ -183,11 +183,12 @@ class HexagonAsmBackend : public MCAsmBackend {
       { "fixup_Hexagon_TPREL_32_6_X",   0,      32,     0 },
       { "fixup_Hexagon_TPREL_16_X",     0,      32,     0 },
       { "fixup_Hexagon_TPREL_11_X",     0,      32,     0 },
-      { "fixup_Hexagon_GD_PLT_B22_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_GD_PLT_B32_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B22_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_Hexagon_LD_PLT_B32_PCREL_X",0,     32,     MCFixupKindInfo::FKF_IsPCRel }
+      { "fixup_Hexagon_GD_PLT_B22_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_GD_PLT_B32_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_LD_PLT_B22_PCREL_X", 0,  32,     0 },
+      { "fixup_Hexagon_LD_PLT_B32_PCREL_X", 0,  32,     0 },
     };
+    // clang-format on
 
     if (Kind < FirstTargetFixupKind)
       return MCAsmBackend::getFixupKindInfo(Kind);
@@ -198,8 +199,7 @@ class HexagonAsmBackend : public MCAsmBackend {
     return Infos[Kind - FirstTargetFixupKind];
   }
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
+  bool shouldForceRelocation(const MCFixup &Fixup) {
     switch(Fixup.getTargetKind()) {
       default:
         llvm_unreachable("Unknown Fixup Kind!");
@@ -316,7 +316,6 @@ class HexagonAsmBackend : public MCAsmBackend {
       case FK_Data_1:
       case FK_Data_2:
       case FK_Data_4:
-      case FK_PCRel_4:
       case fixup_Hexagon_32:
         // Leave these relocations alone as they are used for EH.
         return false;
@@ -334,8 +333,7 @@ class HexagonAsmBackend : public MCAsmBackend {
         return 1;
       case FK_Data_2:
         return 2;
-      case FK_Data_4:         // this later gets mapped to R_HEX_32
-      case FK_PCRel_4:        // this later gets mapped to R_HEX_32_PCREL
+      case FK_Data_4: // this later gets mapped to R_HEX_32 or R_HEX_32_PCREL
       case fixup_Hexagon_32:
       case fixup_Hexagon_B32_PCREL_X:
       case fixup_Hexagon_B22_PCREL:
@@ -404,131 +402,9 @@ class HexagonAsmBackend : public MCAsmBackend {
     llvm_unreachable(errStr.str().c_str());
   }
 
-  /// ApplyFixup - Apply the \arg Value for given \arg Fixup into the provided
-  /// data fragment, at the offset specified by the fixup and following the
-  /// fixup kind as appropriate.
-  void applyFixup(const MCFragment &, const MCFixup &Fixup,
-                  const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t FixupValue, bool IsResolved) override {
-
-    // When FixupValue is 0 the relocation is external and there
-    // is nothing for us to do.
-    if (!FixupValue) return;
-
-    MCFixupKind Kind = Fixup.getKind();
-    uint64_t Value;
-    uint32_t InstMask;
-    uint32_t Reloc;
-
-    // LLVM gives us an encoded value, we have to convert it back
-    // to a real offset before we can use it.
-    uint32_t Offset = Fixup.getOffset();
-    unsigned NumBytes = getFixupKindNumBytes(Kind);
-    assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
-    char *InstAddr = Data.data() + Offset;
-
-    Value = adjustFixupValue(Kind, FixupValue);
-    if(!Value)
-      return;
-    int sValue = (int)Value;
-
-    switch((unsigned)Kind) {
-      default:
-        return;
-
-      case fixup_Hexagon_B7_PCREL:
-        if (!(isIntN(7, sValue)))
-          HandleFixupError(7, 2, (int64_t)FixupValue, "B7_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B7_PCREL_X:
-        InstMask = 0x00001f18;  // Word32_B7
-        Reloc = (((Value >> 2) & 0x1f) << 8) |    // Value 6-2 = Target 12-8
-                ((Value & 0x3) << 3);             // Value 1-0 = Target 4-3
-        break;
-
-      case fixup_Hexagon_B9_PCREL:
-        if (!(isIntN(9, sValue)))
-          HandleFixupError(9, 2, (int64_t)FixupValue, "B9_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B9_PCREL_X:
-        InstMask = 0x003000fe;  // Word32_B9
-        Reloc = (((Value >> 7) & 0x3) << 20) |    // Value 8-7 = Target 21-20
-                ((Value & 0x7f) << 1);            // Value 6-0 = Target 7-1
-        break;
-
-        // Since the existing branches that use this relocation cannot be
-        // extended, they should only be fixed up if the target is within range.
-      case fixup_Hexagon_B13_PCREL:
-        if (!(isIntN(13, sValue)))
-          HandleFixupError(13, 2, (int64_t)FixupValue, "B13_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B13_PCREL_X:
-        InstMask = 0x00202ffe;  // Word32_B13
-        Reloc = (((Value >> 12) & 0x1) << 21) |    // Value 12   = Target 21
-                (((Value >> 11) & 0x1) << 13) |    // Value 11   = Target 13
-                ((Value & 0x7ff) << 1);            // Value 10-0 = Target 11-1
-        break;
-
-      case fixup_Hexagon_B15_PCREL:
-        if (!(isIntN(15, sValue)))
-          HandleFixupError(15, 2, (int64_t)FixupValue, "B15_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B15_PCREL_X:
-        InstMask = 0x00df20fe;  // Word32_B15
-        Reloc = (((Value >> 13) & 0x3) << 22) |    // Value 14-13 = Target 23-22
-                (((Value >> 8) & 0x1f) << 16) |    // Value 12-8  = Target 20-16
-                (((Value >> 7) & 0x1)  << 13) |    // Value 7     = Target 13
-                ((Value & 0x7f) << 1);             // Value 6-0   = Target 7-1
-        break;
-
-      case fixup_Hexagon_B22_PCREL:
-        if (!(isIntN(22, sValue)))
-          HandleFixupError(22, 2, (int64_t)FixupValue, "B22_PCREL");
-        [[fallthrough]];
-      case fixup_Hexagon_B22_PCREL_X:
-        InstMask = 0x01ff3ffe;  // Word32_B22
-        Reloc = (((Value >> 13) & 0x1ff) << 16) |  // Value 21-13 = Target 24-16
-                ((Value & 0x1fff) << 1);           // Value 12-0  = Target 13-1
-        break;
-
-      case fixup_Hexagon_B32_PCREL_X:
-        InstMask = 0x0fff3fff;  // Word32_X26
-        Reloc = (((Value >> 14) & 0xfff) << 16) |  // Value 25-14 = Target 27-16
-                (Value & 0x3fff);                  // Value 13-0  = Target 13-0
-        break;
-
-      case FK_Data_1:
-      case FK_Data_2:
-      case FK_Data_4:
-      case fixup_Hexagon_32:
-        InstMask = 0xffffffff;  // Word32
-        Reloc = Value;
-        break;
-    }
-
-    LLVM_DEBUG(dbgs() << "Name=" << getFixupKindInfo(Kind).Name << "("
-                      << (unsigned)Kind << ")\n");
-    LLVM_DEBUG(
-        uint32_t OldData = 0; for (unsigned i = 0; i < NumBytes; i++) OldData |=
-                              (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
-        dbgs() << "\tBValue=0x"; dbgs().write_hex(Value) << ": AValue=0x";
-        dbgs().write_hex(FixupValue)
-        << ": Offset=" << Offset << ": Size=" << Data.size() << ": OInst=0x";
-        dbgs().write_hex(OldData) << ": Reloc=0x"; dbgs().write_hex(Reloc););
-
-    // For each byte of the fragment that the fixup touches, mask in the
-    // bits from the fixup value. The Value has been "split up" into the
-    // appropriate bitfields above.
-    for (unsigned i = 0; i < NumBytes; i++){
-      InstAddr[i] &= uint8_t(~InstMask >> (i * 8)) & 0xff; // Clear reloc bits
-      InstAddr[i] |= uint8_t(Reloc >> (i * 8)) & 0xff;     // Apply new reloc
-    }
-
-    LLVM_DEBUG(uint32_t NewData = 0;
-               for (unsigned i = 0; i < NumBytes; i++) NewData |=
-               (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
-               dbgs() << ": NInst=0x"; dbgs().write_hex(NewData) << "\n";);
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char> Data, uint64_t FixupValue,
+                  bool IsResolved) override;
 
   bool isInstRelaxable(MCInst const &HMI) const {
     const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(*MCII, HMI);
@@ -775,6 +651,133 @@ class HexagonAsmBackend : public MCAsmBackend {
 
 } // namespace
 
+void HexagonAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                                   const MCValue &Target,
+                                   MutableArrayRef<char> Data,
+                                   uint64_t FixupValue, bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, FixupValue, IsResolved);
+  // When FixupValue is 0 the relocation is external and there
+  // is nothing for us to do.
+  if (!FixupValue)
+    return;
+
+  MCFixupKind Kind = Fixup.getKind();
+  uint64_t Value;
+  uint32_t InstMask;
+  uint32_t Reloc;
+
+  // LLVM gives us an encoded value, we have to convert it back
+  // to a real offset before we can use it.
+  uint32_t Offset = Fixup.getOffset();
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
+  assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
+  char *InstAddr = Data.data() + Offset;
+
+  Value = adjustFixupValue(Kind, FixupValue);
+  if (!Value)
+    return;
+  int sValue = (int)Value;
+
+  switch ((unsigned)Kind) {
+  default:
+    return;
+
+  case fixup_Hexagon_B7_PCREL:
+    if (!(isIntN(7, sValue)))
+      HandleFixupError(7, 2, (int64_t)FixupValue, "B7_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B7_PCREL_X:
+    InstMask = 0x00001f18;                 // Word32_B7
+    Reloc = (((Value >> 2) & 0x1f) << 8) | // Value 6-2 = Target 12-8
+            ((Value & 0x3) << 3);          // Value 1-0 = Target 4-3
+    break;
+
+  case fixup_Hexagon_B9_PCREL:
+    if (!(isIntN(9, sValue)))
+      HandleFixupError(9, 2, (int64_t)FixupValue, "B9_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B9_PCREL_X:
+    InstMask = 0x003000fe;                 // Word32_B9
+    Reloc = (((Value >> 7) & 0x3) << 20) | // Value 8-7 = Target 21-20
+            ((Value & 0x7f) << 1);         // Value 6-0 = Target 7-1
+    break;
+
+    // Since the existing branches that use this relocation cannot be
+    // extended, they should only be fixed up if the target is within range.
+  case fixup_Hexagon_B13_PCREL:
+    if (!(isIntN(13, sValue)))
+      HandleFixupError(13, 2, (int64_t)FixupValue, "B13_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B13_PCREL_X:
+    InstMask = 0x00202ffe;                  // Word32_B13
+    Reloc = (((Value >> 12) & 0x1) << 21) | // Value 12   = Target 21
+            (((Value >> 11) & 0x1) << 13) | // Value 11   = Target 13
+            ((Value & 0x7ff) << 1);         // Value 10-0 = Target 11-1
+    break;
+
+  case fixup_Hexagon_B15_PCREL:
+    if (!(isIntN(15, sValue)))
+      HandleFixupError(15, 2, (int64_t)FixupValue, "B15_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B15_PCREL_X:
+    InstMask = 0x00df20fe;                  // Word32_B15
+    Reloc = (((Value >> 13) & 0x3) << 22) | // Value 14-13 = Target 23-22
+            (((Value >> 8) & 0x1f) << 16) | // Value 12-8  = Target 20-16
+            (((Value >> 7) & 0x1) << 13) |  // Value 7     = Target 13
+            ((Value & 0x7f) << 1);          // Value 6-0   = Target 7-1
+    break;
+
+  case fixup_Hexagon_B22_PCREL:
+    if (!(isIntN(22, sValue)))
+      HandleFixupError(22, 2, (int64_t)FixupValue, "B22_PCREL");
+    [[fallthrough]];
+  case fixup_Hexagon_B22_PCREL_X:
+    InstMask = 0x01ff3ffe;                    // Word32_B22
+    Reloc = (((Value >> 13) & 0x1ff) << 16) | // Value 21-13 = Target 24-16
+            ((Value & 0x1fff) << 1);          // Value 12-0  = Target 13-1
+    break;
+
+  case fixup_Hexagon_B32_PCREL_X:
+    InstMask = 0x0fff3fff;                    // Word32_X26
+    Reloc = (((Value >> 14) & 0xfff) << 16) | // Value 25-14 = Target 27-16
+            (Value & 0x3fff);                 // Value 13-0  = Target 13-0
+    break;
+
+  case FK_Data_1:
+  case FK_Data_2:
+  case FK_Data_4:
+  case fixup_Hexagon_32:
+    InstMask = 0xffffffff; // Word32
+    Reloc = Value;
+    break;
+  }
+
+  LLVM_DEBUG(dbgs() << "Name=" << getFixupKindInfo(Kind).Name << "("
+                    << (unsigned)Kind << ")\n");
+  LLVM_DEBUG(
+      uint32_t OldData = 0; for (unsigned i = 0; i < NumBytes; i++) OldData |=
+                            (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
+      dbgs() << "\tBValue=0x"; dbgs().write_hex(Value) << ": AValue=0x";
+      dbgs().write_hex(FixupValue)
+      << ": Offset=" << Offset << ": Size=" << Data.size() << ": OInst=0x";
+      dbgs().write_hex(OldData) << ": Reloc=0x"; dbgs().write_hex(Reloc););
+
+  // For each byte of the fragment that the fixup touches, mask in the
+  // bits from the fixup value. The Value has been "split up" into the
+  // appropriate bitfields above.
+  for (unsigned i = 0; i < NumBytes; i++) {
+    InstAddr[i] &= uint8_t(~InstMask >> (i * 8)) & 0xff; // Clear reloc bits
+    InstAddr[i] |= uint8_t(Reloc >> (i * 8)) & 0xff;     // Apply new reloc
+  }
+
+  LLVM_DEBUG(uint32_t NewData = 0;
+             for (unsigned i = 0; i < NumBytes; i++) NewData |=
+             (InstAddr[i] << (i * 8)) & (0xff << (i * 8));
+             dbgs() << ": NInst=0x"; dbgs().write_hex(NewData) << "\n";);
+}
+
 // MCAsmBackend
 MCAsmBackend *llvm::createHexagonAsmBackend(Target const &T,
                                             const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
index b0d79357557f..ed381c33225d 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
@@ -85,8 +85,6 @@ unsigned HexagonELFObjectWriter::getRelocType(const MCFixup &Fixup,
     default:
       report_fatal_error("Unrecognized variant type");
     };
-  case FK_PCRel_4:
-    return ELF::R_HEX_32_PCREL;
   case FK_Data_2:
     switch(Variant) {
     case HexagonMCExpr::VK_DTPREL:
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
index ba8faaeb74a0..8f23585df7e1 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
@@ -16,7 +16,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {HexagonMCExpr::VK_DTPREL, "DTPREL"}, {HexagonMCExpr::VK_GD_GOT, "GDGOT"},
     {HexagonMCExpr::VK_GD_PLT, "GDPLT"},  {HexagonMCExpr::VK_GOT, "GOT"},
     {HexagonMCExpr::VK_GOTREL, "GOTREL"}, {HexagonMCExpr::VK_IE, "IE"},
@@ -47,5 +47,5 @@ HexagonMCAsmInfo::HexagonMCAsmInfo(const Triple &TT) {
   ExceptionsType = ExceptionHandling::DwarfCFI;
   UseLogicalShr = false;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
index 9211671bb630..dfe0fa973c9b 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
@@ -337,6 +337,35 @@ static const std::map<unsigned, std::vector<unsigned>> StdFixups = {
 #undef P
 #undef _
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Hexagon::fixup_Hexagon_B22_PCREL:
+  case Hexagon::fixup_Hexagon_B15_PCREL:
+  case Hexagon::fixup_Hexagon_B7_PCREL:
+  case Hexagon::fixup_Hexagon_B13_PCREL:
+  case Hexagon::fixup_Hexagon_B9_PCREL:
+  case Hexagon::fixup_Hexagon_B32_PCREL_X:
+  case Hexagon::fixup_Hexagon_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_B15_PCREL_X:
+  case Hexagon::fixup_Hexagon_B13_PCREL_X:
+  case Hexagon::fixup_Hexagon_B9_PCREL_X:
+  case Hexagon::fixup_Hexagon_B7_PCREL_X:
+  case Hexagon::fixup_Hexagon_32_PCREL:
+  case Hexagon::fixup_Hexagon_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_GD_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_LD_PLT_B22_PCREL:
+  case Hexagon::fixup_Hexagon_6_PCREL_X:
+  case Hexagon::fixup_Hexagon_GD_PLT_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_GD_PLT_B32_PCREL_X:
+  case Hexagon::fixup_Hexagon_LD_PLT_B22_PCREL_X:
+  case Hexagon::fixup_Hexagon_LD_PLT_B32_PCREL_X:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 uint32_t HexagonMCCodeEmitter::parseBits(size_t Last, MCInst const &MCB,
                                          MCInst const &MCI) const {
   bool Duplex = HexagonMCInstrInfo::isDuplex(MCII, MCI);
@@ -617,7 +646,7 @@ unsigned HexagonMCCodeEmitter::getExprOpValue(const MCInst &MI,
   const MCInstrDesc &MCID = HexagonMCInstrInfo::getDesc(MCII, MI);
   unsigned FixupWidth = HexagonMCInstrInfo::getExtentBits(MCII, MI) -
                         HexagonMCInstrInfo::getExtentAlignment(MCII, MI);
-  HexagonMCExpr::VariantKind VarKind = getVariantKind(MCSRE);
+  auto VarKind = HexagonMCExpr::VariantKind(MCSRE->getSpecifier());
   unsigned Opc = MCID.getOpcode();
   unsigned IType = HexagonMCInstrInfo::getType(MCII, MI);
 
@@ -698,10 +727,7 @@ unsigned HexagonMCCodeEmitter::getExprOpValue(const MCInst &MI,
     FixupExpr = MCBinaryExpr::createAdd(FixupExpr, C, MCT);
   }
 
-  MCFixup Fixup = MCFixup::create(State.Addend, FixupExpr,
-                                  MCFixupKind(FixupKind), MI.getLoc());
-  Fixups.push_back(Fixup);
-  // All of the information is in the fixup.
+  addFixup(Fixups, State.Addend, FixupExpr, FixupKind);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
index b22960efc372..a30d81536941 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
@@ -59,11 +59,6 @@ class HexagonMCExpr : public MCTargetExpr {
   bool S27_2_reloc;
   bool SignMismatch;
 };
-
-static inline HexagonMCExpr::VariantKind
-getVariantKind(const MCSymbolRefExpr *SRE) {
-  return HexagonMCExpr::VariantKind(SRE->getKind());
-}
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_HEXAGON_HEXAGONMCEXPR_H
diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index 6a74686a239d..cef77f1c512f 100644
--- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -344,7 +344,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
       return SymbolRefExpr->getSpecifier() == Lanai::S_None;
     if (const MCSymbolRefExpr *SymbolRefExpr =
             dyn_cast<MCSymbolRefExpr>(Imm.Value)) {
-      return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
+      return SymbolRefExpr->getSpecifier() == 0;
     }
 
     // Binary expression
@@ -354,7 +354,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
         return SymbolRefExpr->getSpecifier() == Lanai::S_None;
       if (const MCSymbolRefExpr *SymbolRefExpr =
               dyn_cast<MCSymbolRefExpr>(BinaryExpr->getLHS()))
-        return SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None;
+        return SymbolRefExpr->getSpecifier() == 0;
     }
 
     return false;
@@ -535,8 +535,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
 #ifndef NDEBUG
       const MCSymbolRefExpr *SymbolRefExpr =
           dyn_cast<MCSymbolRefExpr>(getImm());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == MCSymbolRefExpr::VK_None);
+      assert(SymbolRefExpr && SymbolRefExpr->getSpecifier() == 0);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
@@ -551,7 +550,7 @@ struct LanaiOperand : public MCParsedAsmOperand {
       assert(false && "Operand type not supported.");
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case IMMEDIATE:
       OS << "Imm: " << getImm() << "\n";
@@ -563,10 +562,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
       OS << "Reg: %r" << getReg() << "\n";
       break;
     case MEMORY_IMM:
-      OS << "MemImm: " << *getMemOffset() << "\n";
+      OS << "MemImm: ";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << '\n';
       break;
     case MEMORY_REG_IMM:
-      OS << "MemRegImm: " << getMemBaseReg() << "+" << *getMemOffset() << "\n";
+      OS << "MemRegImm: " << getMemBaseReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << '\n';
       break;
     case MEMORY_REG_REG:
       assert(getMemOffset() == nullptr);
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 16ea2c8461fa..9e5fa2263c1d 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -712,9 +712,8 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
   // Build a sequence of copy-to-reg nodes chained together with token chain and
   // flag operands which copy the outgoing args into registers.  The InGlue in
   // necessary since all emitted instructions must be stuck together.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
-                             RegsToPass[I].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -745,9 +744,8 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
-    Ops.push_back(DAG.getRegister(RegsToPass[I].first,
-                                  RegsToPass[I].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index a0bc4f17ad88..83d16971e35c 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -11,9 +11,9 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -71,13 +71,15 @@ bool LanaiAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void LanaiAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void LanaiAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
-                                 bool) {
+                                 bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
   MCFixupKind Kind = Fixup.getKind();
   Value = adjustFixupValue(static_cast<unsigned>(Kind), Value);
-
   if (!Value)
     return; // This value doesn't change the encoding
 
diff --git a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
index d1b2da40446a..84dc4b831237 100644
--- a/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
@@ -290,8 +290,7 @@ unsigned LanaiMCCodeEmitter::getBranchTargetOpValue(
   if (MCOp.isReg() || MCOp.isImm())
     return getMachineOpValue(Inst, MCOp, Fixups, SubtargetInfo);
 
-  Fixups.push_back(MCFixup::create(
-      0, MCOp.getExpr(), static_cast<MCFixupKind>(Lanai::FIXUP_LANAI_25)));
+  Fixups.push_back(MCFixup::create(0, MCOp.getExpr(), Lanai::FIXUP_LANAI_25));
 
   return 0;
 }
diff --git a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
index a8fed951b0cf..5be4713b349e 100644
--- a/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
+++ b/llvm/lib/Target/LoongArch/AsmParser/LoongArchAsmParser.cpp
@@ -9,7 +9,6 @@
 #include "MCTargetDesc/LoongArchBaseInfo.h"
 #include "MCTargetDesc/LoongArchInstPrinter.h"
 #include "MCTargetDesc/LoongArchMCAsmInfo.h"
-#include "MCTargetDesc/LoongArchMCExpr.h"
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "MCTargetDesc/LoongArchMatInt.h"
 #include "MCTargetDesc/LoongArchTargetStreamer.h"
@@ -556,7 +555,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return LoongArchInstPrinter::getRegisterName(Reg);
@@ -566,7 +565,7 @@ class LoongArchOperand : public MCParsedAsmOperand {
 
     switch (Kind) {
     case KindTy::Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case KindTy::Register:
       OS << "<register " << RegName(getReg()) << ">";
diff --git a/llvm/lib/Target/LoongArch/LoongArchFloat64InstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchFloat64InstrInfo.td
index c5d176596d8c..616640152c8d 100644
--- a/llvm/lib/Target/LoongArch/LoongArchFloat64InstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchFloat64InstrInfo.td
@@ -319,3 +319,19 @@ def : Pat<(bitconvert FPR64:$src), (MOVFR2GR_D FPR64:$src)>;
 let Predicates = [HasBasicD, IsLA64] in {
 def : PatFpr<frint, FRINT_D, FPR64>;
 } // Predicates = [HasBasicD, IsLA64]
+
+/// Pseudo-instructions needed for the soft-float ABI with LA32D
+
+let Predicates = [HasBasicD, IsLA32] in {
+// Moves two GPRs to an FPR.
+let usesCustomInserter = 1 in
+def BuildPairF64Pseudo
+    : Pseudo<(outs FPR64:$dst), (ins GPR:$src1, GPR:$src2),
+             [(set FPR64:$dst, (loongarch_build_pair_f64 GPR:$src1, GPR:$src2))]>;
+
+// Moves an FPR to two GPRs.
+let usesCustomInserter = 1 in
+def SplitPairF64Pseudo
+    : Pseudo<(outs GPR:$dst1, GPR:$dst2), (ins FPR64:$src),
+             [(set GPR:$dst1, GPR:$dst2, (loongarch_split_pair_f64 FPR64:$src))]>;
+} // Predicates = [HasBasicD, IsLA32]
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 6946ed554a7e..602ce240814e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -169,6 +169,8 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::INTRINSIC_VOID, MVT::i64, Custom);
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
+    if (Subtarget.hasBasicD())
+      setOperationAction(ISD::BITCAST, MVT::i64, Custom);
   }
 
   setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
@@ -2007,7 +2009,7 @@ static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,
     if (*it < 0) // UNDEF
       MaskAlloc.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
     else if ((*it >= 0 && *it < HalfSize) ||
-             (*it >= MaskSize && *it <= MaskSize + HalfSize)) {
+             (*it >= MaskSize && *it < MaskSize + HalfSize)) {
       int M = *it < HalfSize ? *it : *it - HalfSize;
       MaskAlloc.push_back(DAG.getTargetConstant(M, DL, MVT::i64));
     } else
@@ -2605,9 +2607,6 @@ SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
 
 SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
                                                  SelectionDAG &DAG) const {
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   // Currently only support lowering return address for current frame.
   if (Op.getConstantOperandVal(0) != 0) {
     DAG.getContext()->emitError(
@@ -2713,13 +2712,20 @@ SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
                                               SelectionDAG &DAG) const {
 
   SDLoc DL(Op);
+  EVT VT = Op.getValueType();
   SDValue Op0 = Op.getOperand(0);
+  EVT Op0VT = Op0.getValueType();
 
-  if (Op.getValueType() == MVT::f32 && Op0.getValueType() == MVT::i32 &&
+  if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
       Subtarget.is64Bit() && Subtarget.hasBasicF()) {
     SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
     return DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, NewOp0);
   }
+  if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
+    SDValue Lo, Hi;
+    std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
+    return DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64, Lo, Hi);
+  }
   return Op;
 }
 
@@ -4006,6 +4012,12 @@ void LoongArchTargetLowering::ReplaceNodeResults(
       SDValue Dst =
           DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Src);
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Dst));
+    } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
+      SDValue NewReg = DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
+                                   DAG.getVTList(MVT::i32, MVT::i32), Src);
+      SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
+                                   NewReg.getValue(0), NewReg.getValue(1));
+      Results.push_back(RetReg);
     }
     break;
   }
@@ -5649,6 +5661,37 @@ static SDValue performVMSKLTZCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue
+performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG,
+                             TargetLowering::DAGCombinerInfo &DCI,
+                             const LoongArchSubtarget &Subtarget) {
+  SDValue Op0 = N->getOperand(0);
+  SDLoc DL(N);
+
+  // If the input to SplitPairF64 is just BuildPairF64 then the operation is
+  // redundant. Instead, use BuildPairF64's operands directly.
+  if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
+    return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
+
+  if (Op0->isUndef()) {
+    SDValue Lo = DAG.getUNDEF(MVT::i32);
+    SDValue Hi = DAG.getUNDEF(MVT::i32);
+    return DCI.CombineTo(N, Lo, Hi);
+  }
+
+  // It's cheaper to materialise two 32-bit integers than to load a double
+  // from the constant pool and transfer it to integer registers through the
+  // stack.
+  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
+    APInt V = C->getValueAPF().bitcastToAPInt();
+    SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
+    SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
+    return DCI.CombineTo(N, Lo, Hi);
+  }
+
+  return SDValue();
+}
+
 SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
                                                    DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -5676,6 +5719,8 @@ SDValue LoongArchTargetLowering::PerformDAGCombine(SDNode *N,
   case LoongArchISD::VMSKLTZ:
   case LoongArchISD::XVMSKLTZ:
     return performVMSKLTZCombine(N, DAG, DCI, Subtarget);
+  case LoongArchISD::SPLIT_PAIR_F64:
+    return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
   }
   return SDValue();
 }
@@ -6072,6 +6117,50 @@ emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB,
   return BB;
 }
 
+static MachineBasicBlock *
+emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
+                       const LoongArchSubtarget &Subtarget) {
+  assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
+         "Unexpected instruction");
+
+  MachineFunction &MF = *BB->getParent();
+  DebugLoc DL = MI.getDebugLoc();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+  Register LoReg = MI.getOperand(0).getReg();
+  Register HiReg = MI.getOperand(1).getReg();
+  Register SrcReg = MI.getOperand(2).getReg();
+
+  BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVFR2GR_S_64), LoReg).addReg(SrcReg);
+  BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVFRH2GR_S), HiReg)
+      .addReg(SrcReg, getKillRegState(MI.getOperand(2).isKill()));
+  MI.eraseFromParent(); // The pseudo instruction is gone now.
+  return BB;
+}
+
+static MachineBasicBlock *
+emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB,
+                       const LoongArchSubtarget &Subtarget) {
+  assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
+         "Unexpected instruction");
+
+  MachineFunction &MF = *BB->getParent();
+  DebugLoc DL = MI.getDebugLoc();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+  Register TmpReg = MRI.createVirtualRegister(&LoongArch::FPR64RegClass);
+  Register DstReg = MI.getOperand(0).getReg();
+  Register LoReg = MI.getOperand(1).getReg();
+  Register HiReg = MI.getOperand(2).getReg();
+
+  BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVGR2FR_W_64), TmpReg)
+      .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()));
+  BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVGR2FRH_W), DstReg)
+      .addReg(TmpReg, RegState::Kill)
+      .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()));
+  MI.eraseFromParent(); // The pseudo instruction is gone now.
+  return BB;
+}
+
 static bool isSelectPseudo(MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
@@ -6252,6 +6341,10 @@ MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
   }
   case LoongArch::Select_GPR_Using_CC_GPR:
     return emitSelectPseudo(MI, BB, Subtarget);
+  case LoongArch::BuildPairF64Pseudo:
+    return emitBuildPairF64Pseudo(MI, BB, Subtarget);
+  case LoongArch::SplitPairF64Pseudo:
+    return emitSplitPairF64Pseudo(MI, BB, Subtarget);
   case LoongArch::PseudoVBZ:
   case LoongArch::PseudoVBZ_B:
   case LoongArch::PseudoVBZ_H:
@@ -6348,6 +6441,8 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
     NODE_NAME_CASE(MOVGR2FR_W_LA64)
     NODE_NAME_CASE(MOVFR2GR_S_LA64)
     NODE_NAME_CASE(FTINT)
+    NODE_NAME_CASE(BUILD_PAIR_F64)
+    NODE_NAME_CASE(SPLIT_PAIR_F64)
     NODE_NAME_CASE(REVB_2H)
     NODE_NAME_CASE(REVB_2W)
     NODE_NAME_CASE(BITREV_4B)
@@ -6527,21 +6622,6 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
     break;
   }
 
-  // FPR32 and FPR64 alias each other.
-  if (State.getFirstUnallocated(ArgFPR32s) == std::size(ArgFPR32s))
-    UseGPRForFloat = true;
-
-  if (UseGPRForFloat && ValVT == MVT::f32) {
-    LocVT = GRLenVT;
-    LocInfo = CCValAssign::BCvt;
-  } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
-    LocVT = MVT::i64;
-    LocInfo = CCValAssign::BCvt;
-  } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
-    // TODO: Handle passing f64 on LA32 with D feature.
-    report_fatal_error("Passing f64 with GPR on LA32 is undefined");
-  }
-
   // If this is a variadic argument, the LoongArch calling convention requires
   // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
   // byte alignment. An aligned register should be used regardless of whether
@@ -6564,6 +6644,45 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
   assert(PendingLocs.size() == PendingArgFlags.size() &&
          "PendingLocs and PendingArgFlags out of sync");
 
+  // FPR32 and FPR64 alias each other.
+  if (State.getFirstUnallocated(ArgFPR32s) == std::size(ArgFPR32s))
+    UseGPRForFloat = true;
+
+  if (UseGPRForFloat && ValVT == MVT::f32) {
+    LocVT = GRLenVT;
+    LocInfo = CCValAssign::BCvt;
+  } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
+    LocVT = MVT::i64;
+    LocInfo = CCValAssign::BCvt;
+  } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
+    // Handle passing f64 on LA32D with a soft float ABI or when floating point
+    // registers are exhausted.
+    assert(PendingLocs.empty() && "Can't lower f64 if it is split");
+    // Depending on available argument GPRS, f64 may be passed in a pair of
+    // GPRs, split between a GPR and the stack, or passed completely on the
+    // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
+    // cases.
+    MCRegister Reg = State.AllocateReg(ArgGPRs);
+    if (!Reg) {
+      int64_t StackOffset = State.AllocateStack(8, Align(8));
+      State.addLoc(
+          CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
+      return false;
+    }
+    LocVT = MVT::i32;
+    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+    MCRegister HiReg = State.AllocateReg(ArgGPRs);
+    if (HiReg) {
+      State.addLoc(
+          CCValAssign::getCustomReg(ValNo, ValVT, HiReg, LocVT, LocInfo));
+    } else {
+      int64_t StackOffset = State.AllocateStack(4, Align(4));
+      State.addLoc(
+          CCValAssign::getCustomMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
+    }
+    return false;
+  }
+
   // Split arguments might be passed indirectly, so keep track of the pending
   // values.
   if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
@@ -6764,6 +6883,38 @@ static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
       MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
 }
 
+static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain,
+                                       const CCValAssign &VA,
+                                       const CCValAssign &HiVA,
+                                       const SDLoc &DL) {
+  assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
+         "Unexpected VA");
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  MachineRegisterInfo &RegInfo = MF.getRegInfo();
+
+  assert(VA.isRegLoc() && "Expected register VA assignment");
+
+  Register LoVReg = RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
+  RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
+  SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
+  SDValue Hi;
+  if (HiVA.isMemLoc()) {
+    // Second half of f64 is passed on the stack.
+    int FI = MFI.CreateFixedObject(4, HiVA.getLocMemOffset(),
+                                   /*IsImmutable=*/true);
+    SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
+    Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
+                     MachinePointerInfo::getFixedStack(MF, FI));
+  } else {
+    // Second half of f64 is passed in another GPR.
+    Register HiVReg = RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
+    RegInfo.addLiveIn(HiVA.getLocReg(), HiVReg);
+    Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
+  }
+  return DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64, Lo, Hi);
+}
+
 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
                                    const CCValAssign &VA, const SDLoc &DL) {
   EVT LocVT = VA.getLocVT();
@@ -6861,11 +7012,16 @@ SDValue LoongArchTargetLowering::LowerFormalArguments(
   else
     analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, CC_LoongArch);
 
-  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
+  for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
     CCValAssign &VA = ArgLocs[i];
     SDValue ArgValue;
-    if (VA.isRegLoc())
-      ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[i], *this);
+    // Passing f64 on LA32D with a soft float ABI must be handled as a special
+    // case.
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      assert(VA.needsCustom());
+      ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL);
+    } else if (VA.isRegLoc())
+      ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this);
     else
       ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
     if (VA.getLocInfo() == CCValAssign::Indirect) {
@@ -6873,17 +7029,18 @@ SDValue LoongArchTargetLowering::LowerFormalArguments(
       // load all parts of it here (using the same address).
       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
                                    MachinePointerInfo()));
-      unsigned ArgIndex = Ins[i].OrigArgIndex;
-      unsigned ArgPartOffset = Ins[i].PartOffset;
+      unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
+      unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
       assert(ArgPartOffset == 0);
-      while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
+      while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
         CCValAssign &PartVA = ArgLocs[i + 1];
-        unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
+        unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
                                      MachinePointerInfo()));
         ++i;
+        ++InsIdx;
       }
       continue;
     }
@@ -7112,31 +7269,67 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
   SmallVector<std::pair<Register, SDValue>> RegsToPass;
   SmallVector<SDValue> MemOpChains;
   SDValue StackPtr;
-  for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
+  for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
+       ++i, ++OutIdx) {
     CCValAssign &VA = ArgLocs[i];
-    SDValue ArgValue = OutVals[i];
-    ISD::ArgFlagsTy Flags = Outs[i].Flags;
+    SDValue ArgValue = OutVals[OutIdx];
+    ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
+
+    // Handle passing f64 on LA32D with a soft float ABI as a special case.
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      assert(VA.isRegLoc() && "Expected register VA assignment");
+      assert(VA.needsCustom());
+      SDValue SplitF64 =
+          DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
+                      DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
+      SDValue Lo = SplitF64.getValue(0);
+      SDValue Hi = SplitF64.getValue(1);
+
+      Register RegLo = VA.getLocReg();
+      RegsToPass.push_back(std::make_pair(RegLo, Lo));
+
+      // Get the CCValAssign for the Hi part.
+      CCValAssign &HiVA = ArgLocs[++i];
+
+      if (HiVA.isMemLoc()) {
+        // Second half of f64 is passed on the stack.
+        if (!StackPtr.getNode())
+          StackPtr = DAG.getCopyFromReg(Chain, DL, LoongArch::R3, PtrVT);
+        SDValue Address =
+            DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
+                        DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL));
+        // Emit the store.
+        MemOpChains.push_back(DAG.getStore(
+            Chain, DL, Hi, Address,
+            MachinePointerInfo::getStack(MF, HiVA.getLocMemOffset())));
+      } else {
+        // Second half of f64 is passed in another GPR.
+        Register RegHigh = HiVA.getLocReg();
+        RegsToPass.push_back(std::make_pair(RegHigh, Hi));
+      }
+      continue;
+    }
 
     // Promote the value if needed.
     // For now, only handle fully promoted and indirect arguments.
     if (VA.getLocInfo() == CCValAssign::Indirect) {
       // Store the argument in a stack slot and pass its address.
       Align StackAlign =
-          std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
+          std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG),
                    getPrefTypeAlign(ArgValue.getValueType(), DAG));
       TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
       // If the original argument was split and passed by reference, we need to
       // store the required parts of it here (and pass just one address).
-      unsigned ArgIndex = Outs[i].OrigArgIndex;
-      unsigned ArgPartOffset = Outs[i].PartOffset;
+      unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
+      unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
       assert(ArgPartOffset == 0);
       // Calculate the total size to store. We don't have access to what we're
       // actually storing other than performing the loop and collecting the
       // info.
       SmallVector<std::pair<SDValue, SDValue>> Parts;
-      while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
-        SDValue PartValue = OutVals[i + 1];
-        unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
+      while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
+        SDValue PartValue = OutVals[OutIdx + 1];
+        unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
         SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
         EVT PartVT = PartValue.getValueType();
 
@@ -7144,6 +7337,7 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
         StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
         Parts.push_back(std::make_pair(PartValue, Offset));
         ++i;
+        ++OutIdx;
       }
       SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
@@ -7279,7 +7473,8 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
   analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_LoongArch);
 
   // Copy all of the result registers out of their specified physreg.
-  for (auto &VA : RVLocs) {
+  for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
+    auto &VA = RVLocs[i];
     // Copy the value out.
     SDValue RetValue =
         DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
@@ -7287,7 +7482,16 @@ LoongArchTargetLowering::LowerCall(CallLoweringInfo &CLI,
     Chain = RetValue.getValue(1);
     Glue = RetValue.getValue(2);
 
-    RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      assert(VA.needsCustom());
+      SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(),
+                                             MVT::i32, Glue);
+      Chain = RetValue2.getValue(1);
+      Glue = RetValue2.getValue(2);
+      RetValue = DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64,
+                             RetValue, RetValue2);
+    } else
+      RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
 
     InVals.push_back(RetValue);
   }
@@ -7333,17 +7537,37 @@ SDValue LoongArchTargetLowering::LowerReturn(
   SmallVector<SDValue, 4> RetOps(1, Chain);
 
   // Copy the result values into the output registers.
-  for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
+    SDValue Val = OutVals[OutIdx];
     CCValAssign &VA = RVLocs[i];
     assert(VA.isRegLoc() && "Can only return in registers!");
 
-    // Handle a 'normal' return.
-    SDValue Val = convertValVTToLocVT(DAG, OutVals[i], VA, DL);
-    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
+    if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
+      // Handle returning f64 on LA32D with a soft float ABI.
+      assert(VA.isRegLoc() && "Expected return via registers");
+      assert(VA.needsCustom());
+      SDValue SplitF64 = DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
+                                     DAG.getVTList(MVT::i32, MVT::i32), Val);
+      SDValue Lo = SplitF64.getValue(0);
+      SDValue Hi = SplitF64.getValue(1);
+      Register RegLo = VA.getLocReg();
+      Register RegHi = RVLocs[++i].getLocReg();
+
+      Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
+      Glue = Chain.getValue(1);
+      RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
+      Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
+      Glue = Chain.getValue(1);
+      RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
+    } else {
+      // Handle a 'normal' return.
+      Val = convertValVTToLocVT(DAG, Val, VA, DL);
+      Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
 
-    // Guarantee that all emitted copies are stuck together.
-    Glue = Chain.getValue(1);
-    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
+      // Guarantee that all emitted copies are stuck together.
+      Glue = Chain.getValue(1);
+      RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
+    }
   }
 
   RetOps[0] = Chain; // Update chain.
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 79aa89726191..60dc2b385a75 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -60,6 +60,10 @@ enum NodeType : unsigned {
 
   FTINT,
 
+  // Build and split F64 pair
+  BUILD_PAIR_F64,
+  SPLIT_PAIR_F64,
+
   // Bit counting operations
   CLZ_W,
   CTZ_W,
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index b6552ed33f5b..2b94e65cac0e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -58,6 +58,13 @@ def SDT_LoongArchMovgr2fcsr : SDTypeProfile<0, 2, [SDTCisVT<0, GRLenVT>,
 def SDT_LoongArchMovfcsr2gr : SDTypeProfile<1, 1, [SDTCisVT<0, GRLenVT>,
                                                    SDTCisSameAs<0, 1>]>;
 
+def SDT_LoongArchBuildPairF64 : SDTypeProfile<1, 2, [SDTCisVT<0, f64>,
+                                                     SDTCisVT<1, i32>,
+                                                     SDTCisSameAs<1, 2>]>;
+def SDT_LoongArchSplitPairF64 : SDTypeProfile<2, 1, [SDTCisVT<0, i32>,
+                                                     SDTCisVT<1, i32>,
+                                                     SDTCisVT<2, f64>]>;
+
 // TODO: Add LoongArch specific DAG Nodes
 // Target-independent nodes, but with target-specific formats.
 def callseq_start : SDNode<"ISD::CALLSEQ_START", SDT_CallSeqStart,
@@ -165,6 +172,11 @@ def loongarch_iocsrwr_d : SDNode<"LoongArchISD::IOCSRWR_D",
 def loongarch_cpucfg : SDNode<"LoongArchISD::CPUCFG", SDTUnaryOp,
                                [SDNPHasChain]>;
 
+def loongarch_build_pair_f64 : SDNode<"LoongArchISD::BUILD_PAIR_F64",
+                                      SDT_LoongArchBuildPairF64>;
+def loongarch_split_pair_f64 : SDNode<"LoongArchISD::SPLIT_PAIR_F64",
+                                      SDT_LoongArchSplitPairF64>;
+
 def to_fclass_mask: SDNodeXForm<timm, [{
   uint64_t Check = N->getZExtValue();
   unsigned Mask = 0;
diff --git a/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp b/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp
index 24bf5a18cfb0..455e6ad7c3ac 100644
--- a/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchMCInstLower.cpp
@@ -13,7 +13,7 @@
 
 #include "LoongArch.h"
 #include "MCTargetDesc/LoongArchBaseInfo.h"
-#include "MCTargetDesc/LoongArchMCExpr.h"
+#include "MCTargetDesc/LoongArchMCAsmInfo.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/LoongArch/MCTargetDesc/CMakeLists.txt
index 796e50b5c491..39b4423a6362 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/CMakeLists.txt
@@ -6,7 +6,6 @@ add_llvm_component_library(LLVMLoongArchDesc
   LoongArchInstPrinter.cpp
   LoongArchMCAsmInfo.cpp
   LoongArchMCCodeEmitter.cpp
-  LoongArchMCExpr.cpp
   LoongArchMCTargetDesc.cpp
   LoongArchMatInt.cpp
   LoongArchTargetStreamer.cpp
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index b1491b75ac5b..1b8893029bb3 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -57,9 +57,9 @@ MCFixupKindInfo LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       // LoongArchFixupKinds.h.
       //
       // {name, offset, bits, flags}
-      {"fixup_loongarch_b16", 10, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_loongarch_b21", 0, 26, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_loongarch_b26", 0, 26, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_loongarch_b16", 10, 16, 0},
+      {"fixup_loongarch_b21", 0, 26, 0},
+      {"fixup_loongarch_b26", 0, 26, 0},
       {"fixup_loongarch_abs_hi20", 5, 20, 0},
       {"fixup_loongarch_abs_lo12", 10, 12, 0},
       {"fixup_loongarch_abs64_lo20", 5, 20, 0},
@@ -72,7 +72,7 @@ MCFixupKindInfo LoongArchAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_LARCH_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -140,10 +140,13 @@ static void fixupLeb128(MCContext &Ctx, const MCFixup &Fixup,
     Ctx.reportError(Fixup.getLoc(), "Invalid uleb128 value!");
 }
 
-void LoongArchAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void LoongArchAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                      const MCValue &Target,
                                      MutableArrayRef<char> Data, uint64_t Value,
                                      bool IsResolved) {
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  IsResolved = addReloc(F, Fixup, Target, Value, IsResolved);
   if (!Value)
     return; // Doesn't change encoding.
 
@@ -262,23 +265,17 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 6:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD6),
-                          MCFixupKind(ELF::R_LARCH_SUB6));
+    return std::make_pair(ELF::R_LARCH_ADD6, ELF::R_LARCH_SUB6);
   case 8:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD8),
-                          MCFixupKind(ELF::R_LARCH_SUB8));
+    return std::make_pair(ELF::R_LARCH_ADD8, ELF::R_LARCH_SUB8);
   case 16:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD16),
-                          MCFixupKind(ELF::R_LARCH_SUB16));
+    return std::make_pair(ELF::R_LARCH_ADD16, ELF::R_LARCH_SUB16);
   case 32:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD32),
-                          MCFixupKind(ELF::R_LARCH_SUB32));
+    return std::make_pair(ELF::R_LARCH_ADD32, ELF::R_LARCH_SUB32);
   case 64:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD64),
-                          MCFixupKind(ELF::R_LARCH_SUB64));
+    return std::make_pair(ELF::R_LARCH_ADD64, ELF::R_LARCH_SUB64);
   case 128:
-    return std::make_pair(MCFixupKind(ELF::R_LARCH_ADD_ULEB128),
-                          MCFixupKind(ELF::R_LARCH_SUB_ULEB128));
+    return std::make_pair(ELF::R_LARCH_ADD_ULEB128, ELF::R_LARCH_SUB_ULEB128);
   }
 }
 
@@ -287,8 +284,7 @@ std::pair<bool, bool> LoongArchAsmBackend::relaxLEB128(MCLEBFragment &LF,
   const MCExpr &Expr = LF.getValue();
   if (LF.isSigned() || !Expr.evaluateKnownAbsolute(Value, *Asm))
     return std::make_pair(false, false);
-  LF.getFixups().push_back(
-      MCFixup::create(0, &Expr, FK_Data_leb128, Expr.getLoc()));
+  LF.addFixup(MCFixup::create(0, &Expr, FK_Data_leb128));
   return std::make_pair(true, true);
 }
 
@@ -298,9 +294,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 
   int64_t LineDelta = DF.getLineDelta();
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 1> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -309,8 +304,7 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
   assert(IsAbsolute && "CFA with invalid expression");
   (void)IsAbsolute;
 
-  Data.clear();
-  Fixups.clear();
+  SmallVector<char> Data;
   raw_svector_ostream OS(Data);
 
   // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -355,6 +349,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
     OS << uint8_t(dwarf::DW_LNS_copy);
   }
 
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
   WasRelaxed = OldSize != Data.size();
   return true;
 }
@@ -362,9 +358,8 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
                                         bool &WasRelaxed) const {
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 2> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -373,14 +368,12 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   assert(IsAbsolute && "CFA with invalid expression");
   (void)IsAbsolute;
 
-  Data.clear();
-  Fixups.clear();
-  raw_svector_ostream OS(Data);
-
   assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
          "expected 1-byte alignment");
   if (Value == 0) {
-    WasRelaxed = OldSize != Data.size();
+    DF.clearContents();
+    DF.clearFixups();
+    WasRelaxed = OldSize != DF.getContents().size();
     return true;
   }
 
@@ -392,6 +385,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
     Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(FK)));
   };
 
+  SmallVector<char, 8> Data;
+  raw_svector_ostream OS(Data);
   if (isUIntN(6, Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc);
     AddFixups(0, getRelocPairForSize(6));
@@ -410,6 +405,8 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   } else {
     llvm_unreachable("unsupported CFA encoding");
   }
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
 
   WasRelaxed = OldSize != Data.size();
   return true;
@@ -453,7 +450,8 @@ bool LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
                                    const MCValue &Target, uint64_t &FixedValue,
                                    bool IsResolved) {
   auto Fallback = [&]() {
-    return MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
+    MCAsmBackend::maybeAddReloc(F, Fixup, Target, FixedValue, IsResolved);
+    return true;
   };
   uint64_t FixedValueA, FixedValueB;
   if (Target.getSubSym()) {
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index 56554c5c664e..4446cadf11e2 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -18,7 +18,6 @@
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 
@@ -40,7 +39,7 @@ class LoongArchAsmBackend : public MCAsmBackend {
                       const MCTargetOptions &Options);
 
   bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
+                uint64_t &FixedValue, bool IsResolved);
 
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
@@ -54,16 +53,12 @@ class LoongArchAsmBackend : public MCAsmBackend {
   bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
                                      MCAlignFragment &AF) override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
 
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {}
-
   bool relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
                           bool &WasRelaxed) const override;
   bool relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
index dc55ceab2dd3..8dc7c88f6e78 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.cpp
@@ -11,14 +11,189 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoongArchMCAsmInfo.h"
-#include "LoongArchMCExpr.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
 
+const LoongArchMCExpr *LoongArchMCExpr::create(const MCExpr *Expr, uint16_t S,
+                                               MCContext &Ctx, bool Hint) {
+  return new (Ctx) LoongArchMCExpr(Expr, S, Hint);
+}
+
+static StringRef getLoongArchSpecifierName(uint16_t S) {
+  switch (S) {
+  default:
+    llvm_unreachable("Invalid ELF symbol kind");
+  case ELF::R_LARCH_B16:
+    return "b16";
+  case ELF::R_LARCH_B21:
+    return "b21";
+  case ELF::R_LARCH_ABS_HI20:
+    return "abs_hi20";
+  case ELF::R_LARCH_ABS_LO12:
+    return "abs_lo12";
+  case ELF::R_LARCH_ABS64_LO20:
+    return "abs64_lo20";
+  case ELF::R_LARCH_ABS64_HI12:
+    return "abs64_hi12";
+  case ELF::R_LARCH_PCALA_HI20:
+    return "pc_hi20";
+  case ELF::R_LARCH_PCALA_LO12:
+    return "pc_lo12";
+  case ELF::R_LARCH_PCALA64_LO20:
+    return "pc64_lo20";
+  case ELF::R_LARCH_PCALA64_HI12:
+    return "pc64_hi12";
+  case ELF::R_LARCH_GOT_PC_HI20:
+    return "got_pc_hi20";
+  case ELF::R_LARCH_GOT_PC_LO12:
+    return "got_pc_lo12";
+  case ELF::R_LARCH_GOT64_PC_LO20:
+    return "got64_pc_lo20";
+  case ELF::R_LARCH_GOT64_PC_HI12:
+    return "got64_pc_hi12";
+  case ELF::R_LARCH_GOT_HI20:
+    return "got_hi20";
+  case ELF::R_LARCH_GOT_LO12:
+    return "got_lo12";
+  case ELF::R_LARCH_GOT64_LO20:
+    return "got64_lo20";
+  case ELF::R_LARCH_GOT64_HI12:
+    return "got64_hi12";
+  case ELF::R_LARCH_TLS_LE_HI20:
+    return "le_hi20";
+  case ELF::R_LARCH_TLS_LE_LO12:
+    return "le_lo12";
+  case ELF::R_LARCH_TLS_LE64_LO20:
+    return "le64_lo20";
+  case ELF::R_LARCH_TLS_LE64_HI12:
+    return "le64_hi12";
+  case ELF::R_LARCH_TLS_IE_PC_HI20:
+    return "ie_pc_hi20";
+  case ELF::R_LARCH_TLS_IE_PC_LO12:
+    return "ie_pc_lo12";
+  case ELF::R_LARCH_TLS_IE64_PC_LO20:
+    return "ie64_pc_lo20";
+  case ELF::R_LARCH_TLS_IE64_PC_HI12:
+    return "ie64_pc_hi12";
+  case ELF::R_LARCH_TLS_IE_HI20:
+    return "ie_hi20";
+  case ELF::R_LARCH_TLS_IE_LO12:
+    return "ie_lo12";
+  case ELF::R_LARCH_TLS_IE64_LO20:
+    return "ie64_lo20";
+  case ELF::R_LARCH_TLS_IE64_HI12:
+    return "ie64_hi12";
+  case ELF::R_LARCH_TLS_LD_PC_HI20:
+    return "ld_pc_hi20";
+  case ELF::R_LARCH_TLS_LD_HI20:
+    return "ld_hi20";
+  case ELF::R_LARCH_TLS_GD_PC_HI20:
+    return "gd_pc_hi20";
+  case ELF::R_LARCH_TLS_GD_HI20:
+    return "gd_hi20";
+  case ELF::R_LARCH_CALL36:
+    return "call36";
+  case ELF::R_LARCH_TLS_DESC_PC_HI20:
+    return "desc_pc_hi20";
+  case ELF::R_LARCH_TLS_DESC_PC_LO12:
+    return "desc_pc_lo12";
+  case ELF::R_LARCH_TLS_DESC64_PC_LO20:
+    return "desc64_pc_lo20";
+  case ELF::R_LARCH_TLS_DESC64_PC_HI12:
+    return "desc64_pc_hi12";
+  case ELF::R_LARCH_TLS_DESC_HI20:
+    return "desc_hi20";
+  case ELF::R_LARCH_TLS_DESC_LO12:
+    return "desc_lo12";
+  case ELF::R_LARCH_TLS_DESC64_LO20:
+    return "desc64_lo20";
+  case ELF::R_LARCH_TLS_DESC64_HI12:
+    return "desc64_hi12";
+  case ELF::R_LARCH_TLS_DESC_LD:
+    return "desc_ld";
+  case ELF::R_LARCH_TLS_DESC_CALL:
+    return "desc_call";
+  case ELF::R_LARCH_TLS_LE_HI20_R:
+    return "le_hi20_r";
+  case ELF::R_LARCH_TLS_LE_ADD_R:
+    return "le_add_r";
+  case ELF::R_LARCH_TLS_LE_LO12_R:
+    return "le_lo12_r";
+  case ELF::R_LARCH_PCREL20_S2:
+    return "pcrel_20";
+  case ELF::R_LARCH_TLS_LD_PCREL20_S2:
+    return "ld_pcrel_20";
+  case ELF::R_LARCH_TLS_GD_PCREL20_S2:
+    return "gd_pcrel_20";
+  case ELF::R_LARCH_TLS_DESC_PCREL20_S2:
+    return "desc_pcrel_20";
+  }
+}
+
+LoongArchMCExpr::Specifier LoongArch::parseSpecifier(StringRef name) {
+  return StringSwitch<LoongArchMCExpr::Specifier>(name)
+      .Case("plt", ELF::R_LARCH_B26)
+      .Case("b16", ELF::R_LARCH_B16)
+      .Case("b21", ELF::R_LARCH_B21)
+      .Case("b26", ELF::R_LARCH_B26)
+      .Case("abs_hi20", ELF::R_LARCH_ABS_HI20)
+      .Case("abs_lo12", ELF::R_LARCH_ABS_LO12)
+      .Case("abs64_lo20", ELF::R_LARCH_ABS64_LO20)
+      .Case("abs64_hi12", ELF::R_LARCH_ABS64_HI12)
+      .Case("pc_hi20", ELF::R_LARCH_PCALA_HI20)
+      .Case("pc_lo12", ELF::R_LARCH_PCALA_LO12)
+      .Case("pc64_lo20", ELF::R_LARCH_PCALA64_LO20)
+      .Case("pc64_hi12", ELF::R_LARCH_PCALA64_HI12)
+      .Case("got_pc_hi20", ELF::R_LARCH_GOT_PC_HI20)
+      .Case("got_pc_lo12", ELF::R_LARCH_GOT_PC_LO12)
+      .Case("got64_pc_lo20", ELF::R_LARCH_GOT64_PC_LO20)
+      .Case("got64_pc_hi12", ELF::R_LARCH_GOT64_PC_HI12)
+      .Case("got_hi20", ELF::R_LARCH_GOT_HI20)
+      .Case("got_lo12", ELF::R_LARCH_GOT_LO12)
+      .Case("got64_lo20", ELF::R_LARCH_GOT64_LO20)
+      .Case("got64_hi12", ELF::R_LARCH_GOT64_HI12)
+      .Case("le_hi20", ELF::R_LARCH_TLS_LE_HI20)
+      .Case("le_lo12", ELF::R_LARCH_TLS_LE_LO12)
+      .Case("le64_lo20", ELF::R_LARCH_TLS_LE64_LO20)
+      .Case("le64_hi12", ELF::R_LARCH_TLS_LE64_HI12)
+      .Case("ie_pc_hi20", ELF::R_LARCH_TLS_IE_PC_HI20)
+      .Case("ie_pc_lo12", ELF::R_LARCH_TLS_IE_PC_LO12)
+      .Case("ie64_pc_lo20", ELF::R_LARCH_TLS_IE64_PC_LO20)
+      .Case("ie64_pc_hi12", ELF::R_LARCH_TLS_IE64_PC_HI12)
+      .Case("ie_hi20", ELF::R_LARCH_TLS_IE_HI20)
+      .Case("ie_lo12", ELF::R_LARCH_TLS_IE_LO12)
+      .Case("ie64_lo20", ELF::R_LARCH_TLS_IE64_LO20)
+      .Case("ie64_hi12", ELF::R_LARCH_TLS_IE64_HI12)
+      .Case("ld_pc_hi20", ELF::R_LARCH_TLS_LD_PC_HI20)
+      .Case("ld_hi20", ELF::R_LARCH_TLS_LD_HI20)
+      .Case("gd_pc_hi20", ELF::R_LARCH_TLS_GD_PC_HI20)
+      .Case("gd_hi20", ELF::R_LARCH_TLS_GD_HI20)
+      .Case("call36", ELF::R_LARCH_CALL36)
+      .Case("desc_pc_hi20", ELF::R_LARCH_TLS_DESC_PC_HI20)
+      .Case("desc_pc_lo12", ELF::R_LARCH_TLS_DESC_PC_LO12)
+      .Case("desc64_pc_lo20", ELF::R_LARCH_TLS_DESC64_PC_LO20)
+      .Case("desc64_pc_hi12", ELF::R_LARCH_TLS_DESC64_PC_HI12)
+      .Case("desc_hi20", ELF::R_LARCH_TLS_DESC_HI20)
+      .Case("desc_lo12", ELF::R_LARCH_TLS_DESC_LO12)
+      .Case("desc64_lo20", ELF::R_LARCH_TLS_DESC64_LO20)
+      .Case("desc64_hi12", ELF::R_LARCH_TLS_DESC64_HI12)
+      .Case("desc_ld", ELF::R_LARCH_TLS_DESC_LD)
+      .Case("desc_call", ELF::R_LARCH_TLS_DESC_CALL)
+      .Case("le_hi20_r", ELF::R_LARCH_TLS_LE_HI20_R)
+      .Case("le_add_r", ELF::R_LARCH_TLS_LE_ADD_R)
+      .Case("le_lo12_r", ELF::R_LARCH_TLS_LE_LO12_R)
+      .Case("pcrel_20", ELF::R_LARCH_PCREL20_S2)
+      .Case("ld_pcrel_20", ELF::R_LARCH_TLS_LD_PCREL20_S2)
+      .Case("gd_pcrel_20", ELF::R_LARCH_TLS_GD_PCREL20_S2)
+      .Case("desc_pcrel_20", ELF::R_LARCH_TLS_DESC_PCREL20_S2)
+      .Default(0);
+}
+
 void LoongArchMCAsmInfo::anchor() {}
 
 LoongArchMCAsmInfo::LoongArchMCAsmInfo(const Triple &TT) {
@@ -40,7 +215,7 @@ void LoongArchMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
   auto S = Expr.getSpecifier();
   bool HasSpecifier = S != 0 && S != ELF::R_LARCH_B26;
   if (HasSpecifier)
-    OS << '%' << LoongArch::getSpecifierName(S) << '(';
+    OS << '%' << getLoongArchSpecifierName(S) << '(';
   printExpr(OS, *Expr.getSubExpr());
   if (HasSpecifier)
     OS << ')';
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
index 58ffb723d62c..e4b29b2741fe 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCAsmInfo.h
@@ -14,9 +14,29 @@
 #define LLVM_LIB_TARGET_LOONGARCH_MCTARGETDESC_LOONGARCHMCASMINFO_H
 
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 class Triple;
+class StringRef;
+
+class LoongArchMCExpr : public MCSpecifierExpr {
+public:
+  using Specifier = uint16_t;
+  enum { VK_None };
+
+private:
+  const bool RelaxHint;
+
+  explicit LoongArchMCExpr(const MCExpr *Expr, Specifier S, bool Hint)
+      : MCSpecifierExpr(Expr, S), RelaxHint(Hint) {}
+
+public:
+  static const LoongArchMCExpr *create(const MCExpr *Expr, uint16_t S,
+                                       MCContext &Ctx, bool Hint = false);
+
+  bool getRelaxHint() const { return RelaxHint; }
+};
 
 class LoongArchMCAsmInfo : public MCAsmInfoELF {
   void anchor() override;
@@ -28,7 +48,6 @@ class LoongArchMCAsmInfo : public MCAsmInfoELF {
 };
 
 namespace LoongArch {
-StringRef getSpecifierName(uint16_t S);
 uint16_t parseSpecifier(StringRef name);
 } // namespace LoongArch
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
index d4ac1cafd42d..b7ead5e61ab8 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCCodeEmitter.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LoongArchFixupKinds.h"
-#include "MCTargetDesc/LoongArchMCExpr.h"
+#include "MCTargetDesc/LoongArchMCAsmInfo.h"
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCCodeEmitter.h"
@@ -97,6 +97,18 @@ class LoongArchMCCodeEmitter : public MCCodeEmitter {
 };
 } // end namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case LoongArch::fixup_loongarch_b16:
+  case LoongArch::fixup_loongarch_b21:
+  case LoongArch::fixup_loongarch_b26:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned
 LoongArchMCCodeEmitter::getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                                           SmallVectorImpl<MCFixup> &Fixups,
@@ -167,9 +179,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
       RelaxCandidate = true;
       break;
     }
-  } else if (Kind == MCExpr::SymbolRef &&
-             cast<MCSymbolRefExpr>(Expr)->getKind() ==
-                 MCSymbolRefExpr::VK_None) {
+  } else if (Kind == MCExpr::SymbolRef) {
     switch (MI.getOpcode()) {
     default:
       break;
@@ -197,8 +207,7 @@ LoongArchMCCodeEmitter::getExprOpValue(const MCInst &MI, const MCOperand &MO,
   assert(FixupKind != LoongArch::fixup_loongarch_invalid &&
          "Unhandled expression!");
 
-  Fixups.push_back(
-      MCFixup::create(0, Expr, MCFixupKind(FixupKind), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, FixupKind);
   // If linker relaxation is enabled and supported by this relocation, set
   // a bit so that if fixup is unresolved, a R_LARCH_RELAX relocation will be
   // appended.
@@ -251,8 +260,7 @@ void LoongArchMCCodeEmitter::expandAddTPRel(const MCInst &MI,
          "Expected %le_add_r relocation on TP-relative symbol");
 
   // Emit the correct %le_add_r relocation for the symbol.
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_LARCH_TLS_LE_ADD_R, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_LARCH_TLS_LE_ADD_R);
   if (STI.hasFeature(LoongArch::FeatureRelax))
     Fixups.back().setLinkerRelaxable();
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
deleted file mode 100644
index c763aaa7276f..000000000000
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.cpp
+++ /dev/null
@@ -1,199 +0,0 @@
-//===-- LoongArchMCExpr.cpp - LoongArch specific MC expression classes ----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the implementation of the assembly expression modifiers
-// accepted by the LoongArch architecture.
-//
-//===----------------------------------------------------------------------===//
-
-#include "LoongArchMCExpr.h"
-#include "LoongArchMCAsmInfo.h"
-#include "llvm/BinaryFormat/ELF.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCValue.h"
-#include "llvm/Support/ErrorHandling.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "loongarch-mcexpr"
-
-const LoongArchMCExpr *LoongArchMCExpr::create(const MCExpr *Expr, uint16_t S,
-                                               MCContext &Ctx, bool Hint) {
-  return new (Ctx) LoongArchMCExpr(Expr, Specifier(S), Hint);
-}
-
-StringRef LoongArch::getSpecifierName(uint16_t S) {
-  switch (S) {
-  default:
-    llvm_unreachable("Invalid ELF symbol kind");
-  case ELF::R_LARCH_B16:
-    return "b16";
-  case ELF::R_LARCH_B21:
-    return "b21";
-  case ELF::R_LARCH_ABS_HI20:
-    return "abs_hi20";
-  case ELF::R_LARCH_ABS_LO12:
-    return "abs_lo12";
-  case ELF::R_LARCH_ABS64_LO20:
-    return "abs64_lo20";
-  case ELF::R_LARCH_ABS64_HI12:
-    return "abs64_hi12";
-  case ELF::R_LARCH_PCALA_HI20:
-    return "pc_hi20";
-  case ELF::R_LARCH_PCALA_LO12:
-    return "pc_lo12";
-  case ELF::R_LARCH_PCALA64_LO20:
-    return "pc64_lo20";
-  case ELF::R_LARCH_PCALA64_HI12:
-    return "pc64_hi12";
-  case ELF::R_LARCH_GOT_PC_HI20:
-    return "got_pc_hi20";
-  case ELF::R_LARCH_GOT_PC_LO12:
-    return "got_pc_lo12";
-  case ELF::R_LARCH_GOT64_PC_LO20:
-    return "got64_pc_lo20";
-  case ELF::R_LARCH_GOT64_PC_HI12:
-    return "got64_pc_hi12";
-  case ELF::R_LARCH_GOT_HI20:
-    return "got_hi20";
-  case ELF::R_LARCH_GOT_LO12:
-    return "got_lo12";
-  case ELF::R_LARCH_GOT64_LO20:
-    return "got64_lo20";
-  case ELF::R_LARCH_GOT64_HI12:
-    return "got64_hi12";
-  case ELF::R_LARCH_TLS_LE_HI20:
-    return "le_hi20";
-  case ELF::R_LARCH_TLS_LE_LO12:
-    return "le_lo12";
-  case ELF::R_LARCH_TLS_LE64_LO20:
-    return "le64_lo20";
-  case ELF::R_LARCH_TLS_LE64_HI12:
-    return "le64_hi12";
-  case ELF::R_LARCH_TLS_IE_PC_HI20:
-    return "ie_pc_hi20";
-  case ELF::R_LARCH_TLS_IE_PC_LO12:
-    return "ie_pc_lo12";
-  case ELF::R_LARCH_TLS_IE64_PC_LO20:
-    return "ie64_pc_lo20";
-  case ELF::R_LARCH_TLS_IE64_PC_HI12:
-    return "ie64_pc_hi12";
-  case ELF::R_LARCH_TLS_IE_HI20:
-    return "ie_hi20";
-  case ELF::R_LARCH_TLS_IE_LO12:
-    return "ie_lo12";
-  case ELF::R_LARCH_TLS_IE64_LO20:
-    return "ie64_lo20";
-  case ELF::R_LARCH_TLS_IE64_HI12:
-    return "ie64_hi12";
-  case ELF::R_LARCH_TLS_LD_PC_HI20:
-    return "ld_pc_hi20";
-  case ELF::R_LARCH_TLS_LD_HI20:
-    return "ld_hi20";
-  case ELF::R_LARCH_TLS_GD_PC_HI20:
-    return "gd_pc_hi20";
-  case ELF::R_LARCH_TLS_GD_HI20:
-    return "gd_hi20";
-  case ELF::R_LARCH_CALL36:
-    return "call36";
-  case ELF::R_LARCH_TLS_DESC_PC_HI20:
-    return "desc_pc_hi20";
-  case ELF::R_LARCH_TLS_DESC_PC_LO12:
-    return "desc_pc_lo12";
-  case ELF::R_LARCH_TLS_DESC64_PC_LO20:
-    return "desc64_pc_lo20";
-  case ELF::R_LARCH_TLS_DESC64_PC_HI12:
-    return "desc64_pc_hi12";
-  case ELF::R_LARCH_TLS_DESC_HI20:
-    return "desc_hi20";
-  case ELF::R_LARCH_TLS_DESC_LO12:
-    return "desc_lo12";
-  case ELF::R_LARCH_TLS_DESC64_LO20:
-    return "desc64_lo20";
-  case ELF::R_LARCH_TLS_DESC64_HI12:
-    return "desc64_hi12";
-  case ELF::R_LARCH_TLS_DESC_LD:
-    return "desc_ld";
-  case ELF::R_LARCH_TLS_DESC_CALL:
-    return "desc_call";
-  case ELF::R_LARCH_TLS_LE_HI20_R:
-    return "le_hi20_r";
-  case ELF::R_LARCH_TLS_LE_ADD_R:
-    return "le_add_r";
-  case ELF::R_LARCH_TLS_LE_LO12_R:
-    return "le_lo12_r";
-  case ELF::R_LARCH_PCREL20_S2:
-    return "pcrel_20";
-  case ELF::R_LARCH_TLS_LD_PCREL20_S2:
-    return "ld_pcrel_20";
-  case ELF::R_LARCH_TLS_GD_PCREL20_S2:
-    return "gd_pcrel_20";
-  case ELF::R_LARCH_TLS_DESC_PCREL20_S2:
-    return "desc_pcrel_20";
-  }
-}
-
-LoongArchMCExpr::Specifier LoongArch::parseSpecifier(StringRef name) {
-  return StringSwitch<LoongArchMCExpr::Specifier>(name)
-      .Case("plt", ELF::R_LARCH_B26)
-      .Case("b16", ELF::R_LARCH_B16)
-      .Case("b21", ELF::R_LARCH_B21)
-      .Case("b26", ELF::R_LARCH_B26)
-      .Case("abs_hi20", ELF::R_LARCH_ABS_HI20)
-      .Case("abs_lo12", ELF::R_LARCH_ABS_LO12)
-      .Case("abs64_lo20", ELF::R_LARCH_ABS64_LO20)
-      .Case("abs64_hi12", ELF::R_LARCH_ABS64_HI12)
-      .Case("pc_hi20", ELF::R_LARCH_PCALA_HI20)
-      .Case("pc_lo12", ELF::R_LARCH_PCALA_LO12)
-      .Case("pc64_lo20", ELF::R_LARCH_PCALA64_LO20)
-      .Case("pc64_hi12", ELF::R_LARCH_PCALA64_HI12)
-      .Case("got_pc_hi20", ELF::R_LARCH_GOT_PC_HI20)
-      .Case("got_pc_lo12", ELF::R_LARCH_GOT_PC_LO12)
-      .Case("got64_pc_lo20", ELF::R_LARCH_GOT64_PC_LO20)
-      .Case("got64_pc_hi12", ELF::R_LARCH_GOT64_PC_HI12)
-      .Case("got_hi20", ELF::R_LARCH_GOT_HI20)
-      .Case("got_lo12", ELF::R_LARCH_GOT_LO12)
-      .Case("got64_lo20", ELF::R_LARCH_GOT64_LO20)
-      .Case("got64_hi12", ELF::R_LARCH_GOT64_HI12)
-      .Case("le_hi20", ELF::R_LARCH_TLS_LE_HI20)
-      .Case("le_lo12", ELF::R_LARCH_TLS_LE_LO12)
-      .Case("le64_lo20", ELF::R_LARCH_TLS_LE64_LO20)
-      .Case("le64_hi12", ELF::R_LARCH_TLS_LE64_HI12)
-      .Case("ie_pc_hi20", ELF::R_LARCH_TLS_IE_PC_HI20)
-      .Case("ie_pc_lo12", ELF::R_LARCH_TLS_IE_PC_LO12)
-      .Case("ie64_pc_lo20", ELF::R_LARCH_TLS_IE64_PC_LO20)
-      .Case("ie64_pc_hi12", ELF::R_LARCH_TLS_IE64_PC_HI12)
-      .Case("ie_hi20", ELF::R_LARCH_TLS_IE_HI20)
-      .Case("ie_lo12", ELF::R_LARCH_TLS_IE_LO12)
-      .Case("ie64_lo20", ELF::R_LARCH_TLS_IE64_LO20)
-      .Case("ie64_hi12", ELF::R_LARCH_TLS_IE64_HI12)
-      .Case("ld_pc_hi20", ELF::R_LARCH_TLS_LD_PC_HI20)
-      .Case("ld_hi20", ELF::R_LARCH_TLS_LD_HI20)
-      .Case("gd_pc_hi20", ELF::R_LARCH_TLS_GD_PC_HI20)
-      .Case("gd_hi20", ELF::R_LARCH_TLS_GD_HI20)
-      .Case("call36", ELF::R_LARCH_CALL36)
-      .Case("desc_pc_hi20", ELF::R_LARCH_TLS_DESC_PC_HI20)
-      .Case("desc_pc_lo12", ELF::R_LARCH_TLS_DESC_PC_LO12)
-      .Case("desc64_pc_lo20", ELF::R_LARCH_TLS_DESC64_PC_LO20)
-      .Case("desc64_pc_hi12", ELF::R_LARCH_TLS_DESC64_PC_HI12)
-      .Case("desc_hi20", ELF::R_LARCH_TLS_DESC_HI20)
-      .Case("desc_lo12", ELF::R_LARCH_TLS_DESC_LO12)
-      .Case("desc64_lo20", ELF::R_LARCH_TLS_DESC64_LO20)
-      .Case("desc64_hi12", ELF::R_LARCH_TLS_DESC64_HI12)
-      .Case("desc_ld", ELF::R_LARCH_TLS_DESC_LD)
-      .Case("desc_call", ELF::R_LARCH_TLS_DESC_CALL)
-      .Case("le_hi20_r", ELF::R_LARCH_TLS_LE_HI20_R)
-      .Case("le_add_r", ELF::R_LARCH_TLS_LE_ADD_R)
-      .Case("le_lo12_r", ELF::R_LARCH_TLS_LE_LO12_R)
-      .Case("pcrel_20", ELF::R_LARCH_PCREL20_S2)
-      .Case("ld_pcrel_20", ELF::R_LARCH_TLS_LD_PCREL20_S2)
-      .Case("gd_pcrel_20", ELF::R_LARCH_TLS_GD_PCREL20_S2)
-      .Case("desc_pcrel_20", ELF::R_LARCH_TLS_DESC_PCREL20_S2)
-      .Default(0);
-}
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
deleted file mode 100644
index 36563d8a6b60..000000000000
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCExpr.h
+++ /dev/null
@@ -1,43 +0,0 @@
-//= LoongArchMCExpr.h - LoongArch specific MC expression classes -*- C++ -*-==//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes LoongArch-specific MCExprs, used for modifiers like
-// "%pc_hi20" or "%pc_lo12" etc.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_LOONGARCH_MCTARGETDESC_LOONGARCHMCEXPR_H
-#define LLVM_LIB_TARGET_LOONGARCH_MCTARGETDESC_LOONGARCHMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-
-class StringRef;
-
-class LoongArchMCExpr : public MCSpecifierExpr {
-public:
-  using Specifier = uint16_t;
-  enum { VK_None };
-
-private:
-  const bool RelaxHint;
-
-  explicit LoongArchMCExpr(const MCExpr *Expr, Specifier S, bool Hint)
-      : MCSpecifierExpr(Expr, S), RelaxHint(Hint) {}
-
-public:
-  static const LoongArchMCExpr *create(const MCExpr *Expr, uint16_t S,
-                                       MCContext &Ctx, bool Hint = false);
-
-  bool getRelaxHint() const { return RelaxHint; }
-};
-
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
index e264d4e521cc..16f97d971840 100644
--- a/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
+++ b/llvm/lib/Target/M68k/AsmParser/M68kAsmParser.cpp
@@ -8,6 +8,7 @@
 
 #include "M68kInstrInfo.h"
 #include "M68kRegisterInfo.h"
+#include "MCTargetDesc/M68kMCAsmInfo.h"
 #include "TargetInfo/M68kTargetInfo.h"
 
 #include "llvm/MC/MCContext.h"
@@ -144,7 +145,7 @@ class M68kOperand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   bool isMem() const override { return false; }
   bool isMemOp() const { return Kind == KindTy::MemOp; }
@@ -1051,7 +1052,7 @@ bool M68kAsmParser::matchAndEmitInstruction(SMLoc Loc, unsigned &Opcode,
   }
 }
 
-void M68kOperand::print(raw_ostream &OS) const {
+void M68kOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case KindTy::Invalid:
     OS << "invalid";
diff --git a/llvm/lib/Target/M68k/M68kInstrInfo.cpp b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
index 568938f1f50b..21e9319aaf0b 100644
--- a/llvm/lib/Target/M68k/M68kInstrInfo.cpp
+++ b/llvm/lib/Target/M68k/M68kInstrInfo.cpp
@@ -95,8 +95,8 @@ bool M68kInstrInfo::AnalyzeBranchImpl(MachineBasicBlock &MBB,
   // Erase any instructions if allowed at the end of the scope.
   std::vector<std::reference_wrapper<llvm::MachineInstr>> EraseList;
   auto FinalizeOnReturn = llvm::make_scope_exit([&EraseList] {
-    std::for_each(EraseList.begin(), EraseList.end(),
-                  [](auto &ref) { ref.get().eraseFromParent(); });
+    for (auto &Ref : EraseList)
+      Ref.get().eraseFromParent();
   });
 
   // Start from the bottom of the block and work up, examining the
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
index 20a37302a5e1..f0c3728ec0ee 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kAsmBackend.cpp
@@ -18,9 +18,9 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCAsmBackend.h"
+#include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -52,23 +52,9 @@ class M68kAsmBackend : public MCAsmBackend {
                               .CasesLower("m68020", "m68030", "m68040", true)
                               .Default(false)) {}
 
-  void applyFixup(const MCFragment &, const MCFixup &Fixup, const MCValue &,
-                  MutableArrayRef<char> Data, uint64_t Value, bool) override {
-    unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind());
-
-    assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
-    // Check that uppper bits are either all zeros or all ones.
-    // Specifically ignore overflow/underflow as long as the leakage is
-    // limited to the lower bits. This is to remain compatible with
-    // other assemblers.
-    assert(isIntN(Size * 8 + 1, static_cast<int64_t>(Value)) &&
-           "Value does not fit in the Fixup field");
-
-    // Write in Big Endian
-    for (unsigned i = 0; i != Size; ++i)
-      Data[Fixup.getOffset() + i] =
-          uint8_t(static_cast<int64_t>(Value) >> ((Size - i - 1) * 8));
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char> Data, uint64_t Value,
+                  bool IsResolved) override;
 
   bool mayNeedRelaxation(const MCInst &Inst,
                          const MCSubtargetInfo &STI) const override;
@@ -91,6 +77,28 @@ class M68kAsmBackend : public MCAsmBackend {
 };
 } // end anonymous namespace
 
+void M68kAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                                const MCValue &Target,
+                                MutableArrayRef<char> Data, uint64_t Value,
+                                bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
+  unsigned Size = 1 << getFixupKindLog2Size(Fixup.getKind());
+  assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
+  // Check that uppper bits are either all zeros or all ones.
+  // Specifically ignore overflow/underflow as long as the leakage is
+  // limited to the lower bits. This is to remain compatible with
+  // other assemblers.
+  assert(isIntN(Size * 8 + 1, static_cast<int64_t>(Value)) &&
+         "Value does not fit in the Fixup field");
+
+  // Write in Big Endian
+  for (unsigned i = 0; i != Size; ++i)
+    Data[Fixup.getOffset() + i] =
+        uint8_t(static_cast<int64_t>(Value) >> ((Size - i - 1) * 8));
+}
+
 /// cc—Carry clear      GE—Greater than or equal
 /// LS—Lower or same    PL—Plus
 /// CS—Carry set        GT—Greater than
@@ -99,8 +107,7 @@ class M68kAsmBackend : public MCAsmBackend {
 /// MI—Minus            VC—Overflow clear
 ///                     LE—Less than or equal
 /// NE—Not equal        VS—Overflow set
-static unsigned getRelaxedOpcodeBranch(const MCInst &Inst) {
-  unsigned Op = Inst.getOpcode();
+static unsigned getRelaxedOpcodeBranch(unsigned Op) {
   switch (Op) {
   default:
     return Op;
@@ -171,37 +178,17 @@ static unsigned getRelaxedOpcodeBranch(const MCInst &Inst) {
   }
 }
 
-static unsigned getRelaxedOpcodeArith(const MCInst &Inst) {
-  unsigned Op = Inst.getOpcode();
+static unsigned getRelaxedOpcode(unsigned Opcode) {
   // NOTE there will be some relaxations for PCD and ARD mem for x20
-  return Op;
-}
-
-static unsigned getRelaxedOpcode(const MCInst &Inst) {
-  unsigned R = getRelaxedOpcodeArith(Inst);
-  if (R != Inst.getOpcode())
-    return R;
-  return getRelaxedOpcodeBranch(Inst);
+  return getRelaxedOpcodeBranch(Opcode);
 }
 
 bool M68kAsmBackend::mayNeedRelaxation(const MCInst &Inst,
                                        const MCSubtargetInfo &STI) const {
   // Branches can always be relaxed in either mode.
-  if (getRelaxedOpcodeBranch(Inst) != Inst.getOpcode())
-    return true;
+  return getRelaxedOpcode(Inst.getOpcode()) != Inst.getOpcode();
 
-  // Check if this instruction is ever relaxable.
-  if (getRelaxedOpcodeArith(Inst) == Inst.getOpcode())
-    return false;
-
-  // Check if the relaxable operand has an expression. For the current set of
-  // relaxable instructions, the relaxable operand is always the last operand.
   // NOTE will change for x20 mem
-  unsigned RelaxableOp = Inst.getNumOperands() - 1;
-  if (Inst.getOperand(RelaxableOp).isExpr())
-    return true;
-
-  return false;
 }
 
 bool M68kAsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
@@ -234,16 +221,8 @@ bool M68kAsmBackend::fixupNeedsRelaxation(const MCFixup &Fixup,
 // we can relax?
 void M68kAsmBackend::relaxInstruction(MCInst &Inst,
                                       const MCSubtargetInfo &STI) const {
-  unsigned RelaxedOp = getRelaxedOpcode(Inst);
-
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
-
+  unsigned RelaxedOp = getRelaxedOpcode(Inst.getOpcode());
+  assert(RelaxedOp != Inst.getOpcode());
   Inst.setOpcode(RelaxedOp);
 }
 
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
index 03416df639cf..ca94a475f783 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kELFObjectWriter.cpp
@@ -49,12 +49,9 @@ static M68kRelType getType(unsigned Kind, M68k::Specifier &Modifier,
                            bool &IsPCRel) {
   switch (Kind) {
   case FK_Data_4:
-  case FK_PCRel_4:
     return RT_32;
-  case FK_PCRel_2:
   case FK_Data_2:
     return RT_16;
-  case FK_PCRel_1:
   case FK_Data_1:
     return RT_8;
   }
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h b/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
index 54a0e98fea6e..0245654d554f 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kFixupKinds.h
@@ -19,36 +19,15 @@
 namespace llvm {
 static inline unsigned getFixupKindLog2Size(unsigned Kind) {
   switch (Kind) {
-  case FK_PCRel_1:
-  case FK_SecRel_1:
   case FK_Data_1:
     return 0;
-  case FK_PCRel_2:
-  case FK_SecRel_2:
   case FK_Data_2:
     return 1;
-  case FK_PCRel_4:
-  case FK_SecRel_4:
   case FK_Data_4:
     return 2;
   }
   llvm_unreachable("invalid fixup kind!");
 }
-
-static inline MCFixupKind getFixupForSize(unsigned Size, bool isPCRel) {
-  switch (Size) {
-  case 8:
-    return isPCRel ? FK_PCRel_1 : FK_Data_1;
-  case 16:
-    return isPCRel ? FK_PCRel_2 : FK_Data_2;
-  case 32:
-    return isPCRel ? FK_PCRel_4 : FK_Data_4;
-  case 64:
-    return isPCRel ? FK_PCRel_8 : FK_Data_8;
-  }
-  llvm_unreachable("Invalid generic fixup size!");
-}
-
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_M68k_MCTARGETDESC_M68kFIXUPKINDS_H
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
index b0a19309b50f..21c73872e3bb 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCAsmInfo.cpp
@@ -18,7 +18,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {M68k::S_GOTOFF, "GOTOFF"},     {M68k::S_GOTPCREL, "GOTPCREL"},
     {M68k::S_GOTTPOFF, "GOTTPOFF"}, {M68k::S_PLT, "PLT"},
     {M68k::S_TLSGD, "TLSGD"},       {M68k::S_TLSLD, "TLSLD"},
@@ -42,5 +42,5 @@ M68kELFMCAsmInfo::M68kELFMCAsmInfo(const Triple &T) {
   UseMotorolaIntegers = true;
   CommentString = ";";
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
diff --git a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
index 3bd1d0d7dcae..9b755e021c5f 100644
--- a/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
+++ b/llvm/lib/Target/M68k/MCTargetDesc/M68kMCCodeEmitter.cpp
@@ -103,6 +103,11 @@ template <unsigned Size> static unsigned getBytePosition(unsigned BitPos) {
   }
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // We need special handlings for relocatable & pc-relative operands that are
 // larger than a word.
 // A M68k instruction is aligned by word (16 bits). That means, 32-bit
@@ -139,9 +144,7 @@ void M68kMCCodeEmitter::encodeRelocImm(const MCInst &MI, unsigned OpIdx,
 
     // Relocatable address
     unsigned InsertByte = getBytePosition<Size>(InsertPos);
-    Fixups.push_back(MCFixup::create(InsertByte, Expr,
-                                     getFixupForSize(Size, /*IsPCRel=*/false),
-                                     MI.getLoc()));
+    addFixup(Fixups, InsertByte, Expr, MCFixup::getDataKindForSize(Size / 8));
   }
 }
 
@@ -175,9 +178,8 @@ void M68kMCCodeEmitter::encodePCRelImm(const MCInst &MI, unsigned OpIdx,
             Expr, MCConstantExpr::create(LabelOffset, Ctx), Ctx);
     }
 
-    Fixups.push_back(MCFixup::create(InsertByte, Expr,
-                                     getFixupForSize(Size, /*IsPCRel=*/true),
-                                     MI.getLoc()));
+    addFixup(Fixups, InsertByte, Expr, MCFixup::getDataKindForSize(Size / 8),
+             true);
   }
 }
 
diff --git a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 5a4121f7cafd..38d22eda5f17 100644
--- a/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/llvm/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/MSP430MCAsmInfo.h"
+#include "MCTargetDesc/MSP430MCTargetDesc.h"
 #include "MSP430.h"
 #include "MSP430RegisterInfo.h"
-#include "MCTargetDesc/MSP430MCTargetDesc.h"
 #include "TargetInfo/MSP430TargetInfo.h"
 
 #include "llvm/ADT/APInt.h"
@@ -224,7 +225,7 @@ class MSP430Operand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const override { return Start; }
   SMLoc getEndLoc() const override { return End; }
 
-  void print(raw_ostream &O) const override {
+  void print(raw_ostream &O, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Tok:
       O << "Token " << Tok;
@@ -233,11 +234,12 @@ class MSP430Operand : public MCParsedAsmOperand {
       O << "Register " << Reg;
       break;
     case k_Imm:
-      O << "Immediate " << *Imm;
+      O << "Immediate ";
+      MAI.printExpr(O, *Imm);
       break;
     case k_Mem:
       O << "Memory ";
-      O << *Mem.Offset << "(" << Reg << ")";
+      MAI.printExpr(O, *Mem.Offset);
       break;
     case k_IndReg:
       O << "RegInd " << Reg;
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index 54fd008790f9..29e5bfabe8c9 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
@@ -46,21 +45,23 @@ class MSP430AsmBackend : public MCAsmBackend {
   }
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
+    // clang-format off
     const static MCFixupKindInfo Infos[MSP430::NumTargetFixupKinds] = {
       // This table must be in the same order of enum in MSP430FixupKinds.h.
       //
       // name            offset bits flags
       {"fixup_32",            0, 32, 0},
-      {"fixup_10_pcrel",      0, 10, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_10_pcrel",      0, 10, 0},
       {"fixup_16",            0, 16, 0},
-      {"fixup_16_pcrel",      0, 16, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_16_pcrel",      0, 16, 0},
       {"fixup_16_byte",       0, 16, 0},
-      {"fixup_16_pcrel_byte", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_2x_pcrel",      0, 10, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_rl_pcrel",      0, 16, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_16_pcrel_byte", 0, 16, 0},
+      {"fixup_2x_pcrel",      0, 10, 0},
+      {"fixup_rl_pcrel",      0, 16, 0},
       {"fixup_8",             0,  8, 0},
       {"fixup_sym_diff",      0, 32, 0},
     };
+    // clang-format on
     static_assert((std::size(Infos)) == MSP430::NumTargetFixupKinds,
                   "Not all fixup kinds added to Infos array");
 
@@ -103,10 +104,11 @@ uint64_t MSP430AsmBackend::adjustFixupValue(const MCFixup &Fixup,
   }
 }
 
-void MSP430AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void MSP430AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   Value = adjustFixupValue(Fixup, Value, getContext());
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   if (!Value)
diff --git a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
index 35a7da4fbe5a..f018bdbeb0bf 100644
--- a/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
+++ b/llvm/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
@@ -77,6 +77,20 @@ class MSP430MCCodeEmitter : public MCCodeEmitter {
                          const MCSubtargetInfo &STI) const override;
 };
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case MSP430::fixup_10_pcrel:
+  case MSP430::fixup_16_pcrel:
+  case MSP430::fixup_16_pcrel_byte:
+  case MSP430::fixup_2x_pcrel:
+  case MSP430::fixup_rl_pcrel:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void MSP430MCCodeEmitter::encodeInstruction(const MCInst &MI,
                                             SmallVectorImpl<char> &CB,
                                             SmallVectorImpl<MCFixup> &Fixups,
@@ -111,8 +125,7 @@ unsigned MSP430MCCodeEmitter::getMachineOpValue(const MCInst &MI,
   }
 
   assert(MO.isExpr() && "Expected expr operand");
-  Fixups.push_back(MCFixup::create(Offset, MO.getExpr(),
-      static_cast<MCFixupKind>(MSP430::fixup_16_byte), MI.getLoc()));
+  addFixup(Fixups, Offset, MO.getExpr(), MSP430::fixup_16_byte);
   Offset += 2;
   return 0;
 }
@@ -143,8 +156,7 @@ unsigned MSP430MCCodeEmitter::getMemOpValue(const MCInst &MI, unsigned Op,
     FixupKind = MSP430::fixup_16_byte;
     break;
   }
-  Fixups.push_back(MCFixup::create(Offset, MO2.getExpr(),
-    static_cast<MCFixupKind>(FixupKind), MI.getLoc()));
+  addFixup(Fixups, Offset, MO2.getExpr(), FixupKind);
   Offset += 2;
   return Reg;
 }
@@ -157,8 +169,7 @@ unsigned MSP430MCCodeEmitter::getPCRelImmOpValue(const MCInst &MI, unsigned Op,
     return MO.getImm();
 
   assert(MO.isExpr() && "Expr operand expected");
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-    static_cast<MCFixupKind>(MSP430::fixup_10_pcrel), MI.getLoc()));
+  addFixup(Fixups, 0, MO.getExpr(), MSP430::fixup_10_pcrel);
   return 0;
 }
 
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index c2946de838d7..32b759ac88f5 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -151,62 +151,62 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
   if (STI.hasHWMult16()) {
     const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
     } LibraryCalls[] = {
-      // Integer Multiply - EABI Table 9
-      { RTLIB::MUL_I16,   "__mspabi_mpyi_hw" },
-      { RTLIB::MUL_I32,   "__mspabi_mpyl_hw" },
-      { RTLIB::MUL_I64,   "__mspabi_mpyll_hw" },
-      // TODO The __mspabi_mpysl*_hw functions ARE implemented in libgcc
-      // TODO The __mspabi_mpyul*_hw functions ARE implemented in libgcc
+        // Integer Multiply - EABI Table 9
+        {RTLIB::MUL_I16, RTLIB::__mspabi_mpyi_hw},
+        {RTLIB::MUL_I32, RTLIB::__mspabi_mpyl_hw},
+        {RTLIB::MUL_I64, RTLIB::__mspabi_mpyll_hw},
+        // TODO The __mspabi_mpysl*_hw functions ARE implemented in libgcc
+        // TODO The __mspabi_mpyul*_hw functions ARE implemented in libgcc
     };
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
     }
   } else if (STI.hasHWMult32()) {
     const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
     } LibraryCalls[] = {
-      // Integer Multiply - EABI Table 9
-      { RTLIB::MUL_I16,   "__mspabi_mpyi_hw" },
-      { RTLIB::MUL_I32,   "__mspabi_mpyl_hw32" },
-      { RTLIB::MUL_I64,   "__mspabi_mpyll_hw32" },
-      // TODO The __mspabi_mpysl*_hw32 functions ARE implemented in libgcc
-      // TODO The __mspabi_mpyul*_hw32 functions ARE implemented in libgcc
+        // Integer Multiply - EABI Table 9
+        {RTLIB::MUL_I16, RTLIB::__mspabi_mpyi_hw},
+        {RTLIB::MUL_I32, RTLIB::__mspabi_mpyl_hw32},
+        {RTLIB::MUL_I64, RTLIB::__mspabi_mpyll_hw32},
+        // TODO The __mspabi_mpysl*_hw32 functions ARE implemented in libgcc
+        // TODO The __mspabi_mpyul*_hw32 functions ARE implemented in libgcc
     };
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
     }
   } else if (STI.hasHWMultF5()) {
     const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
     } LibraryCalls[] = {
-      // Integer Multiply - EABI Table 9
-      { RTLIB::MUL_I16,   "__mspabi_mpyi_f5hw" },
-      { RTLIB::MUL_I32,   "__mspabi_mpyl_f5hw" },
-      { RTLIB::MUL_I64,   "__mspabi_mpyll_f5hw" },
-      // TODO The __mspabi_mpysl*_f5hw functions ARE implemented in libgcc
-      // TODO The __mspabi_mpyul*_f5hw functions ARE implemented in libgcc
+        // Integer Multiply - EABI Table 9
+        {RTLIB::MUL_I16, RTLIB::__mspabi_mpyi_f5hw},
+        {RTLIB::MUL_I32, RTLIB::__mspabi_mpyl_f5hw},
+        {RTLIB::MUL_I64, RTLIB::__mspabi_mpyll_f5hw},
+        // TODO The __mspabi_mpysl*_f5hw functions ARE implemented in libgcc
+        // TODO The __mspabi_mpyul*_f5hw functions ARE implemented in libgcc
     };
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
     }
   } else { // NoHWMult
     const struct {
       const RTLIB::Libcall Op;
-      const char * const Name;
+      const RTLIB::LibcallImpl Impl;
     } LibraryCalls[] = {
-      // Integer Multiply - EABI Table 9
-      { RTLIB::MUL_I16,   "__mspabi_mpyi" },
-      { RTLIB::MUL_I32,   "__mspabi_mpyl" },
-      { RTLIB::MUL_I64,   "__mspabi_mpyll" },
-      // The __mspabi_mpysl* functions are NOT implemented in libgcc
-      // The __mspabi_mpyul* functions are NOT implemented in libgcc
+        // Integer Multiply - EABI Table 9
+        {RTLIB::MUL_I16, RTLIB::__mspabi_mpyi},
+        {RTLIB::MUL_I32, RTLIB::__mspabi_mpyl},
+        {RTLIB::MUL_I64, RTLIB::__mspabi_mpyll},
+        // The __mspabi_mpysl* functions are NOT implemented in libgcc
+        // The __mspabi_mpyul* functions are NOT implemented in libgcc
     };
     for (const auto &LC : LibraryCalls) {
-      setLibcallName(LC.Op, LC.Name);
+      setLibcallImpl(LC.Op, LC.Impl);
     }
     setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::MSP430_BUILTIN);
   }
@@ -768,9 +768,8 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
   // flag operands which copy the outgoing args into registers.  The InGlue in
   // necessary since all emitted instructions must be stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -790,9 +789,8 @@ SDValue MSP430TargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
@@ -1144,9 +1142,6 @@ SDValue MSP430TargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   SDLoc dl(Op);
   EVT PtrVT = Op.getValueType();
diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index b559a8b896e0..01e4d17f6236 100644
--- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -1660,18 +1660,18 @@ class MipsOperand : public MCParsedAsmOperand {
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Immediate:
       OS << "Imm<";
-      OS << *Imm.Val;
+      MAI.printExpr(OS, *Imm.Val);
       OS << ">";
       break;
     case k_Memory:
       OS << "Mem<";
-      Mem.Base->print(OS);
+      Mem.Base->print(OS, MAI);
       OS << ", ";
-      OS << *Mem.Off;
+      MAI.printExpr(OS, *Mem.Off);
       OS << ">";
       break;
     case k_RegisterIndex:
@@ -1774,7 +1774,7 @@ static bool isEvaluated(const MCExpr *Expr) {
   case MCExpr::Constant:
     return true;
   case MCExpr::SymbolRef:
-    return (cast<MCSymbolRefExpr>(Expr)->getKind() != MCSymbolRefExpr::VK_None);
+    return (cast<MCSymbolRefExpr>(Expr)->getSpecifier());
   case MCExpr::Binary: {
     const MCBinaryExpr *BE = cast<MCBinaryExpr>(Expr);
     if (!isEvaluated(BE->getLHS()))
@@ -1817,7 +1817,7 @@ static bool needsExpandMemInst(MCInst &Inst, const MCInstrDesc &MCID) {
 
     // Expand symbol.
     const MCSymbolRefExpr *SR = static_cast<const MCSymbolRefExpr *>(Expr);
-    return SR->getKind() == MCSymbolRefExpr::VK_None;
+    return SR->getSpecifier() == 0;
   }
 
   return false;
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 39d20f0e3c15..c2169beea275 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -20,7 +20,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
@@ -239,13 +238,57 @@ static unsigned calculateMMLEIndex(unsigned i) {
   return (1 - i / 2) * 2 + i % 2;
 }
 
+static bool shouldForceRelocation(const MCFixup &Fixup) {
+  const unsigned FixupKind = Fixup.getKind();
+  switch (FixupKind) {
+  default:
+    return false;
+  // All these relocations require special processing
+  // at linking time. Delegate this work to a linker.
+  case Mips::fixup_Mips_CALL_HI16:
+  case Mips::fixup_Mips_CALL_LO16:
+  case Mips::fixup_Mips_CALL16:
+  case Mips::fixup_Mips_GOT:
+  case Mips::fixup_Mips_GOT_PAGE:
+  case Mips::fixup_Mips_GOT_OFST:
+  case Mips::fixup_Mips_GOT_DISP:
+  case Mips::fixup_Mips_GOT_HI16:
+  case Mips::fixup_Mips_GOT_LO16:
+  case Mips::fixup_Mips_GOTTPREL:
+  case Mips::fixup_Mips_DTPREL_HI:
+  case Mips::fixup_Mips_DTPREL_LO:
+  case Mips::fixup_Mips_TLSGD:
+  case Mips::fixup_Mips_TLSLDM:
+  case Mips::fixup_Mips_TPREL_HI:
+  case Mips::fixup_Mips_TPREL_LO:
+  case Mips::fixup_Mips_JALR:
+  case Mips::fixup_MICROMIPS_CALL16:
+  case Mips::fixup_MICROMIPS_GOT_DISP:
+  case Mips::fixup_MICROMIPS_GOT_PAGE:
+  case Mips::fixup_MICROMIPS_GOT_OFST:
+  case Mips::fixup_MICROMIPS_GOT16:
+  case Mips::fixup_MICROMIPS_GOTTPREL:
+  case Mips::fixup_MICROMIPS_TLS_DTPREL_HI16:
+  case Mips::fixup_MICROMIPS_TLS_DTPREL_LO16:
+  case Mips::fixup_MICROMIPS_TLS_GD:
+  case Mips::fixup_MICROMIPS_TLS_LDM:
+  case Mips::fixup_MICROMIPS_TLS_TPREL_HI16:
+  case Mips::fixup_MICROMIPS_TLS_TPREL_LO16:
+  case Mips::fixup_MICROMIPS_JALR:
+    return true;
+  }
+}
+
 /// ApplyFixup - Apply the \p Value for given \p Fixup into the provided
 /// data fragment, at the offset specified by the fixup and following the
 /// fixup kind as appropriate.
-void MipsAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void MipsAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                 const MCValue &Target,
                                 MutableArrayRef<char> Data, uint64_t Value,
                                 bool IsResolved) {
+  if (shouldForceRelocation(Fixup))
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   MCContext &Ctx = getContext();
   Value = adjustFixupValue(Fixup, Value, Ctx);
@@ -314,41 +357,38 @@ std::optional<MCFixupKind> MipsAsmBackend::getFixupKind(StringRef Name) const {
   return StringSwitch<std::optional<MCFixupKind>>(Name)
       .Case("R_MIPS_NONE", FK_NONE)
       .Case("R_MIPS_32", FK_Data_4)
-      .Case("R_MIPS_CALL_HI16", (MCFixupKind)Mips::fixup_Mips_CALL_HI16)
-      .Case("R_MIPS_CALL_LO16", (MCFixupKind)Mips::fixup_Mips_CALL_LO16)
-      .Case("R_MIPS_CALL16", (MCFixupKind)Mips::fixup_Mips_CALL16)
-      .Case("R_MIPS_GOT16", (MCFixupKind)Mips::fixup_Mips_GOT)
-      .Case("R_MIPS_GOT_PAGE", (MCFixupKind)Mips::fixup_Mips_GOT_PAGE)
-      .Case("R_MIPS_GOT_OFST", (MCFixupKind)Mips::fixup_Mips_GOT_OFST)
-      .Case("R_MIPS_GOT_DISP", (MCFixupKind)Mips::fixup_Mips_GOT_DISP)
-      .Case("R_MIPS_GOT_HI16", (MCFixupKind)Mips::fixup_Mips_GOT_HI16)
-      .Case("R_MIPS_GOT_LO16", (MCFixupKind)Mips::fixup_Mips_GOT_LO16)
-      .Case("R_MIPS_TLS_GOTTPREL", (MCFixupKind)Mips::fixup_Mips_GOTTPREL)
-      .Case("R_MIPS_TLS_DTPREL_HI16", (MCFixupKind)Mips::fixup_Mips_DTPREL_HI)
-      .Case("R_MIPS_TLS_DTPREL_LO16", (MCFixupKind)Mips::fixup_Mips_DTPREL_LO)
-      .Case("R_MIPS_TLS_GD", (MCFixupKind)Mips::fixup_Mips_TLSGD)
-      .Case("R_MIPS_TLS_LDM", (MCFixupKind)Mips::fixup_Mips_TLSLDM)
-      .Case("R_MIPS_TLS_TPREL_HI16", (MCFixupKind)Mips::fixup_Mips_TPREL_HI)
-      .Case("R_MIPS_TLS_TPREL_LO16", (MCFixupKind)Mips::fixup_Mips_TPREL_LO)
-      .Case("R_MICROMIPS_CALL16", (MCFixupKind)Mips::fixup_MICROMIPS_CALL16)
-      .Case("R_MICROMIPS_GOT_DISP", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_DISP)
-      .Case("R_MICROMIPS_GOT_PAGE", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_PAGE)
-      .Case("R_MICROMIPS_GOT_OFST", (MCFixupKind)Mips::fixup_MICROMIPS_GOT_OFST)
-      .Case("R_MICROMIPS_GOT16", (MCFixupKind)Mips::fixup_MICROMIPS_GOT16)
-      .Case("R_MICROMIPS_TLS_GOTTPREL",
-            (MCFixupKind)Mips::fixup_MICROMIPS_GOTTPREL)
+      .Case("R_MIPS_CALL_HI16", Mips::fixup_Mips_CALL_HI16)
+      .Case("R_MIPS_CALL_LO16", Mips::fixup_Mips_CALL_LO16)
+      .Case("R_MIPS_CALL16", Mips::fixup_Mips_CALL16)
+      .Case("R_MIPS_GOT16", Mips::fixup_Mips_GOT)
+      .Case("R_MIPS_GOT_PAGE", Mips::fixup_Mips_GOT_PAGE)
+      .Case("R_MIPS_GOT_OFST", Mips::fixup_Mips_GOT_OFST)
+      .Case("R_MIPS_GOT_DISP", Mips::fixup_Mips_GOT_DISP)
+      .Case("R_MIPS_GOT_HI16", Mips::fixup_Mips_GOT_HI16)
+      .Case("R_MIPS_GOT_LO16", Mips::fixup_Mips_GOT_LO16)
+      .Case("R_MIPS_TLS_GOTTPREL", Mips::fixup_Mips_GOTTPREL)
+      .Case("R_MIPS_TLS_DTPREL_HI16", Mips::fixup_Mips_DTPREL_HI)
+      .Case("R_MIPS_TLS_DTPREL_LO16", Mips::fixup_Mips_DTPREL_LO)
+      .Case("R_MIPS_TLS_GD", Mips::fixup_Mips_TLSGD)
+      .Case("R_MIPS_TLS_LDM", Mips::fixup_Mips_TLSLDM)
+      .Case("R_MIPS_TLS_TPREL_HI16", Mips::fixup_Mips_TPREL_HI)
+      .Case("R_MIPS_TLS_TPREL_LO16", Mips::fixup_Mips_TPREL_LO)
+      .Case("R_MICROMIPS_CALL16", Mips::fixup_MICROMIPS_CALL16)
+      .Case("R_MICROMIPS_GOT_DISP", Mips::fixup_MICROMIPS_GOT_DISP)
+      .Case("R_MICROMIPS_GOT_PAGE", Mips::fixup_MICROMIPS_GOT_PAGE)
+      .Case("R_MICROMIPS_GOT_OFST", Mips::fixup_MICROMIPS_GOT_OFST)
+      .Case("R_MICROMIPS_GOT16", Mips::fixup_MICROMIPS_GOT16)
+      .Case("R_MICROMIPS_TLS_GOTTPREL", Mips::fixup_MICROMIPS_GOTTPREL)
       .Case("R_MICROMIPS_TLS_DTPREL_HI16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_DTPREL_HI16)
+            Mips::fixup_MICROMIPS_TLS_DTPREL_HI16)
       .Case("R_MICROMIPS_TLS_DTPREL_LO16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_DTPREL_LO16)
-      .Case("R_MICROMIPS_TLS_GD", (MCFixupKind)Mips::fixup_MICROMIPS_TLS_GD)
-      .Case("R_MICROMIPS_TLS_LDM", (MCFixupKind)Mips::fixup_MICROMIPS_TLS_LDM)
-      .Case("R_MICROMIPS_TLS_TPREL_HI16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_TPREL_HI16)
-      .Case("R_MICROMIPS_TLS_TPREL_LO16",
-            (MCFixupKind)Mips::fixup_MICROMIPS_TLS_TPREL_LO16)
-      .Case("R_MIPS_JALR", (MCFixupKind)Mips::fixup_Mips_JALR)
-      .Case("R_MICROMIPS_JALR", (MCFixupKind)Mips::fixup_MICROMIPS_JALR)
+            Mips::fixup_MICROMIPS_TLS_DTPREL_LO16)
+      .Case("R_MICROMIPS_TLS_GD", Mips::fixup_MICROMIPS_TLS_GD)
+      .Case("R_MICROMIPS_TLS_LDM", Mips::fixup_MICROMIPS_TLS_LDM)
+      .Case("R_MICROMIPS_TLS_TPREL_HI16", Mips::fixup_MICROMIPS_TLS_TPREL_HI16)
+      .Case("R_MICROMIPS_TLS_TPREL_LO16", Mips::fixup_MICROMIPS_TLS_TPREL_LO16)
+      .Case("R_MIPS_JALR", Mips::fixup_Mips_JALR)
+      .Case("R_MICROMIPS_JALR", Mips::fixup_MICROMIPS_JALR)
       .Default(MCAsmBackend::getFixupKind(Name));
 }
 
@@ -374,7 +414,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GPREL16",      0,     16,   0 },
     { "fixup_Mips_LITERAL",      0,     16,   0 },
     { "fixup_Mips_GOT",          0,     16,   0 },
-    { "fixup_Mips_PC16",         0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC16",         0,     16,   0 },
     { "fixup_Mips_CALL16",       0,     16,   0 },
     { "fixup_Mips_SHIFT5",       6,      5,   0 },
     { "fixup_Mips_SHIFT6",       6,      5,   0 },
@@ -386,7 +426,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_TLSLDM",       0,     16,   0 },
     { "fixup_Mips_DTPREL_HI",    0,     16,   0 },
     { "fixup_Mips_DTPREL_LO",    0,     16,   0 },
-    { "fixup_Mips_Branch_PCRel", 0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_Branch_PCRel", 0,     16,   0 },
     { "fixup_Mips_GPOFF_HI",     0,     16,   0 },
     { "fixup_MICROMIPS_GPOFF_HI",0,     16,   0 },
     { "fixup_Mips_GPOFF_LO",     0,     16,   0 },
@@ -402,23 +442,23 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GOT_LO16",     0,     16,   0 },
     { "fixup_Mips_CALL_HI16",    0,     16,   0 },
     { "fixup_Mips_CALL_LO16",    0,     16,   0 },
-    { "fixup_Mips_PC18_S3",      0,     18,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC19_S2",      0,     19,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC21_S2",      0,     21,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC26_S2",      0,     26,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCHI16",       0,     16,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCLO16",       0,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC18_S3",      0,     18,   0 },
+    { "fixup_MIPS_PC19_S2",      0,     19,   0 },
+    { "fixup_MIPS_PC21_S2",      0,     21,   0 },
+    { "fixup_MIPS_PC26_S2",      0,     26,   0 },
+    { "fixup_MIPS_PCHI16",       0,     16,   0 },
+    { "fixup_MIPS_PCLO16",       0,     16,   0 },
     { "fixup_MICROMIPS_26_S1",   0,     26,   0 },
     { "fixup_MICROMIPS_HI16",    0,     16,   0 },
     { "fixup_MICROMIPS_LO16",    0,     16,   0 },
     { "fixup_MICROMIPS_GOT16",   0,     16,   0 },
-    { "fixup_MICROMIPS_PC7_S1",  0,      7,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC10_S1", 0,     10,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC16_S1", 0,     16,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC26_S1", 0,     26,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC19_S2", 0,     19,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC18_S3", 0,     18,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC21_S1", 0,     21,   MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_MICROMIPS_PC7_S1",  0,      7,   0 },
+    { "fixup_MICROMIPS_PC10_S1", 0,     10,   0 },
+    { "fixup_MICROMIPS_PC16_S1", 0,     16,   0 },
+    { "fixup_MICROMIPS_PC26_S1", 0,     26,   0 },
+    { "fixup_MICROMIPS_PC19_S2", 0,     19,   0 },
+    { "fixup_MICROMIPS_PC18_S3", 0,     18,   0 },
+    { "fixup_MICROMIPS_PC21_S1", 0,     21,   0 },
     { "fixup_MICROMIPS_CALL16",  0,     16,   0 },
     { "fixup_MICROMIPS_GOT_DISP",        0,     16,   0 },
     { "fixup_MICROMIPS_GOT_PAGE",        0,     16,   0 },
@@ -460,7 +500,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GPREL16",     16,     16,   0 },
     { "fixup_Mips_LITERAL",     16,     16,   0 },
     { "fixup_Mips_GOT",         16,     16,   0 },
-    { "fixup_Mips_PC16",        16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC16",        16,     16,   0 },
     { "fixup_Mips_CALL16",      16,     16,   0 },
     { "fixup_Mips_SHIFT5",      21,      5,   0 },
     { "fixup_Mips_SHIFT6",      21,      5,   0 },
@@ -472,11 +512,11 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_TLSLDM",      16,     16,   0 },
     { "fixup_Mips_DTPREL_HI",   16,     16,   0 },
     { "fixup_Mips_DTPREL_LO",   16,     16,   0 },
-    { "fixup_Mips_Branch_PCRel",16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_Branch_PCRel",16,     16,   0 },
     { "fixup_Mips_GPOFF_HI",    16,     16,   0 },
-    { "fixup_MICROMIPS_GPOFF_HI", 16,     16,   0 },
+    { "fixup_MICROMIPS_GPOFF_HI", 16,   16,   0 },
     { "fixup_Mips_GPOFF_LO",    16,     16,   0 },
-    { "fixup_MICROMIPS_GPOFF_LO", 16,     16,   0 },
+    { "fixup_MICROMIPS_GPOFF_LO", 16,   16,   0 },
     { "fixup_Mips_GOT_PAGE",    16,     16,   0 },
     { "fixup_Mips_GOT_OFST",    16,     16,   0 },
     { "fixup_Mips_GOT_DISP",    16,     16,   0 },
@@ -488,23 +528,23 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
     { "fixup_Mips_GOT_LO16",    16,     16,   0 },
     { "fixup_Mips_CALL_HI16",   16,     16,   0 },
     { "fixup_Mips_CALL_LO16",   16,     16,   0 },
-    { "fixup_Mips_PC18_S3",     14,     18,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC19_S2",     13,     19,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC21_S2",     11,     21,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PC26_S2",      6,     26,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCHI16",      16,     16,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MIPS_PCLO16",      16,     16,  MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_Mips_PC18_S3",     14,     18,   0 },
+    { "fixup_MIPS_PC19_S2",     13,     19,   0 },
+    { "fixup_MIPS_PC21_S2",     11,     21,   0 },
+    { "fixup_MIPS_PC26_S2",      6,     26,   0 },
+    { "fixup_MIPS_PCHI16",      16,     16,   0 },
+    { "fixup_MIPS_PCLO16",      16,     16,   0 },
     { "fixup_MICROMIPS_26_S1",   6,     26,   0 },
     { "fixup_MICROMIPS_HI16",   16,     16,   0 },
     { "fixup_MICROMIPS_LO16",   16,     16,   0 },
     { "fixup_MICROMIPS_GOT16",  16,     16,   0 },
-    { "fixup_MICROMIPS_PC7_S1",  9,      7,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC10_S1", 6,     10,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC16_S1",16,     16,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC26_S1", 6,     26,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC19_S2",13,     19,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC18_S3",14,     18,   MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_MICROMIPS_PC21_S1",11,     21,   MCFixupKindInfo::FKF_IsPCRel },
+    { "fixup_MICROMIPS_PC7_S1",  9,      7,   0 },
+    { "fixup_MICROMIPS_PC10_S1", 6,     10,   0 },
+    { "fixup_MICROMIPS_PC16_S1",16,     16,   0 },
+    { "fixup_MICROMIPS_PC26_S1", 6,     26,   0 },
+    { "fixup_MICROMIPS_PC19_S2",13,     19,   0 },
+    { "fixup_MICROMIPS_PC18_S3",14,     18,   0 },
+    { "fixup_MICROMIPS_PC21_S1",11,     21,   0 },
     { "fixup_MICROMIPS_CALL16", 16,     16,   0 },
     { "fixup_MICROMIPS_GOT_DISP",        16,     16,   0 },
     { "fixup_MICROMIPS_GOT_PAGE",        16,     16,   0 },
@@ -526,7 +566,7 @@ MCFixupKindInfo MipsAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
                 "Not all MIPS big endian fixup kinds added!");
 
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
 
@@ -556,48 +596,6 @@ bool MipsAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-bool MipsAsmBackend::shouldForceRelocation(const MCFixup &Fixup,
-                                           const MCValue &Target) {
-  const unsigned FixupKind = Fixup.getKind();
-  switch (FixupKind) {
-  default:
-    return false;
-  // All these relocations require special processing
-  // at linking time. Delegate this work to a linker.
-  case Mips::fixup_Mips_CALL_HI16:
-  case Mips::fixup_Mips_CALL_LO16:
-  case Mips::fixup_Mips_CALL16:
-  case Mips::fixup_Mips_GOT:
-  case Mips::fixup_Mips_GOT_PAGE:
-  case Mips::fixup_Mips_GOT_OFST:
-  case Mips::fixup_Mips_GOT_DISP:
-  case Mips::fixup_Mips_GOT_HI16:
-  case Mips::fixup_Mips_GOT_LO16:
-  case Mips::fixup_Mips_GOTTPREL:
-  case Mips::fixup_Mips_DTPREL_HI:
-  case Mips::fixup_Mips_DTPREL_LO:
-  case Mips::fixup_Mips_TLSGD:
-  case Mips::fixup_Mips_TLSLDM:
-  case Mips::fixup_Mips_TPREL_HI:
-  case Mips::fixup_Mips_TPREL_LO:
-  case Mips::fixup_Mips_JALR:
-  case Mips::fixup_MICROMIPS_CALL16:
-  case Mips::fixup_MICROMIPS_GOT_DISP:
-  case Mips::fixup_MICROMIPS_GOT_PAGE:
-  case Mips::fixup_MICROMIPS_GOT_OFST:
-  case Mips::fixup_MICROMIPS_GOT16:
-  case Mips::fixup_MICROMIPS_GOTTPREL:
-  case Mips::fixup_MICROMIPS_TLS_DTPREL_HI16:
-  case Mips::fixup_MICROMIPS_TLS_DTPREL_LO16:
-  case Mips::fixup_MICROMIPS_TLS_GD:
-  case Mips::fixup_MICROMIPS_TLS_LDM:
-  case Mips::fixup_MICROMIPS_TLS_TPREL_HI16:
-  case Mips::fixup_MICROMIPS_TLS_TPREL_LO16:
-  case Mips::fixup_MICROMIPS_JALR:
-    return true;
-  }
-}
-
 namespace {
 
 class WindowsMipsAsmBackend : public MipsAsmBackend {
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 8661339d3b4a..816626da723a 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -48,9 +48,6 @@ class MipsAsmBackend : public MCAsmBackend {
 
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override;
 }; // class MipsAsmBackend
 
 } // namespace
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index 35d4e0db35c3..a8369f2b28fb 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -55,6 +55,30 @@ MCCodeEmitter *createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
 
 } // end namespace llvm
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Mips::fixup_Mips_PC16:
+  case Mips::fixup_Mips_Branch_PCRel:
+  case Mips::fixup_MIPS_PC18_S3:
+  case Mips::fixup_MIPS_PC19_S2:
+  case Mips::fixup_MIPS_PC21_S2:
+  case Mips::fixup_MIPS_PC26_S2:
+  case Mips::fixup_MIPS_PCHI16:
+  case Mips::fixup_MIPS_PCLO16:
+  case Mips::fixup_MICROMIPS_PC7_S1:
+  case Mips::fixup_MICROMIPS_PC10_S1:
+  case Mips::fixup_MICROMIPS_PC16_S1:
+  case Mips::fixup_MICROMIPS_PC26_S1:
+  case Mips::fixup_MICROMIPS_PC19_S2:
+  case Mips::fixup_MICROMIPS_PC18_S3:
+  case Mips::fixup_MICROMIPS_PC21_S1:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // If the D<shift> instruction has a shift amount that is greater
 // than 31 (checked in calling routine), lower it to a D<shift>32 instruction
 static void LowerLargeShift(MCInst& Inst) {
@@ -236,8 +260,7 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -258,8 +281,7 @@ getBranchTargetOpValue1SImm16(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -281,8 +303,7 @@ getBranchTargetOpValueMMR6(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-2, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -304,8 +325,7 @@ getBranchTargetOpValueLsl2MMR6(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_Mips_PC16)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_Mips_PC16);
   return 0;
 }
 
@@ -325,8 +345,7 @@ getBranchTarget7OpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC7_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC7_S1);
   return 0;
 }
 
@@ -346,8 +365,7 @@ getBranchTargetOpValueMMPC10(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValuePC10 expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                   MCFixupKind(Mips::fixup_MICROMIPS_PC10_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC10_S1);
   return 0;
 }
 
@@ -367,9 +385,7 @@ getBranchTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getBranchTargetOpValueMM expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                   MCFixupKind(Mips::
-                               fixup_MICROMIPS_PC16_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_PC16_S1);
   return 0;
 }
 
@@ -390,8 +406,7 @@ getBranchTarget21OpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MIPS_PC21_S2)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MIPS_PC21_S2);
   return 0;
 }
 
@@ -412,8 +427,7 @@ getBranchTarget21OpValueMM(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC21_S1)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MICROMIPS_PC21_S1);
   return 0;
 }
 
@@ -434,8 +448,7 @@ getBranchTarget26OpValue(const MCInst &MI, unsigned OpNo,
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MIPS_PC26_S2)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MIPS_PC26_S2);
   return 0;
 }
 
@@ -456,8 +469,7 @@ unsigned MipsMCCodeEmitter::getBranchTarget26OpValueMM(
 
   const MCExpr *FixupExpression = MCBinaryExpr::createAdd(
       MO.getExpr(), MCConstantExpr::create(-4, Ctx), Ctx);
-  Fixups.push_back(MCFixup::create(0, FixupExpression,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_PC26_S1)));
+  addFixup(Fixups, 0, FixupExpression, Mips::fixup_MICROMIPS_PC26_S1);
   return 0;
 }
 
@@ -478,7 +490,7 @@ getJumpOffset16OpValue(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind =
       isMicroMips(STI) ? Mips::fixup_MICROMIPS_LO16 : Mips::fixup_Mips_LO16;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
@@ -497,8 +509,7 @@ getJumpTargetOpValue(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValue expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_Mips_26)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_Mips_26);
   return 0;
 }
 
@@ -514,8 +525,7 @@ getJumpTargetOpValueMM(const MCInst &MI, unsigned OpNo,
          "getJumpTargetOpValueMM expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(0, Expr,
-                                   MCFixupKind(Mips::fixup_MICROMIPS_26_S1)));
+  addFixup(Fixups, 0, Expr, Mips::fixup_MICROMIPS_26_S1);
   return 0;
 }
 
@@ -693,7 +703,7 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
           isMicroMips(STI) ? Mips::fixup_MICROMIPS_SUB : Mips::fixup_Mips_SUB;
       break;
     }
-    Fixups.push_back(MCFixup::create(0, MipsExpr, MCFixupKind(FixupKind)));
+    addFixup(Fixups, 0, MipsExpr, MCFixupKind(FixupKind));
     return 0;
   }
 
@@ -734,8 +744,7 @@ unsigned MipsMCCodeEmitter::getImmOpValue(const MCInst &MI, const MCOperand &MO,
     return Res;
   unsigned MIFrm = MipsII::getFormat(MCII.get(MI.getOpcode()).TSFlags);
   if (!isa<MCSpecifierExpr>(Expr) && MIFrm == MipsII::FrmI) {
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Mips::fixup_Mips_AnyImm16), Expr->getLoc()));
+    addFixup(Fixups, 0, Expr, Mips::fixup_Mips_AnyImm16);
     return 0;
   }
   return getExprOpValue(Expr, Fixups, STI);
@@ -964,7 +973,7 @@ MipsMCCodeEmitter::getSimm19Lsl2Encoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_PC19_S2
                                             : Mips::fixup_MIPS_PC19_S2;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
@@ -986,7 +995,7 @@ MipsMCCodeEmitter::getSimm18Lsl3Encoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   Mips::Fixups FixupKind = isMicroMips(STI) ? Mips::fixup_MICROMIPS_PC18_S3
                                             : Mips::fixup_MIPS_PC18_S3;
-  Fixups.push_back(MCFixup::create(0, Expr, MCFixupKind(FixupKind)));
+  addFixup(Fixups, 0, Expr, MCFixupKind(FixupKind));
   return 0;
 }
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index 29f61ed9b2b8..ab1eda0f48e1 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -174,7 +174,7 @@ static MCAsmInfo *createMipsMCAsmInfo(const MCRegisterInfo &MRI,
                                       const MCTargetOptions &Options) {
   MCAsmInfo *MAI;
 
-  if (TT.isOSWindows())
+  if (TT.isOSBinFormatCOFF())
     MAI = new MipsCOFFMCAsmInfo();
   else
     MAI = new MipsELFMCAsmInfo(TT, Options);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index 6097ad801784..c69fc68ab5af 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -1034,43 +1034,43 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() {
 
 void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_GPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_GPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_GPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_GPREL32));
   DF->appendContents(8, 0);
 }
 
 void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_DTPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_DTPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_DTPREL64)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_DTPREL64));
   DF->appendContents(8, 0);
 }
 
 void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_TPREL32)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_TPREL32));
   DF->appendContents(4, 0);
 }
 
 void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) {
   MCDataFragment *DF = getStreamer().getOrCreateDataFragment();
-  DF->getFixups().push_back(MCFixup::create(
-      DF->getContents().size(), Value, MCFixupKind(Mips::fixup_Mips_TPREL64)));
+  DF->addFixup(MCFixup::create(DF->getContents().size(), Value,
+                               Mips::fixup_Mips_TPREL64));
   DF->appendContents(8, 0);
 }
 
diff --git a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
index 4fca1f71fdb5..330cb4e0e206 100644
--- a/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
+++ b/llvm/lib/Target/Mips/Mips16ISelLowering.cpp
@@ -33,7 +33,8 @@ static cl::opt<bool> DontExpandCondPseudos16(
 namespace {
 struct Mips16Libcall {
   RTLIB::Libcall Libcall;
-  const char *Name;
+  RTLIB::LibcallImpl Impl;
+  const char *Name; // FIXME: Remove this
 
   bool operator<(const Mips16Libcall &RHS) const {
     return std::strcmp(Name, RHS.Name) < 0;
@@ -55,41 +56,48 @@ struct Mips16IntrinsicHelperType{
 
 // Libcalls for which no helper is generated. Sorted by name for binary search.
 static const Mips16Libcall HardFloatLibCalls[] = {
-  { RTLIB::ADD_F64, "__mips16_adddf3" },
-  { RTLIB::ADD_F32, "__mips16_addsf3" },
-  { RTLIB::DIV_F64, "__mips16_divdf3" },
-  { RTLIB::DIV_F32, "__mips16_divsf3" },
-  { RTLIB::OEQ_F64, "__mips16_eqdf2" },
-  { RTLIB::OEQ_F32, "__mips16_eqsf2" },
-  { RTLIB::FPEXT_F32_F64, "__mips16_extendsfdf2" },
-  { RTLIB::FPTOSINT_F64_I32, "__mips16_fix_truncdfsi" },
-  { RTLIB::FPTOSINT_F32_I32, "__mips16_fix_truncsfsi" },
-  { RTLIB::SINTTOFP_I32_F64, "__mips16_floatsidf" },
-  { RTLIB::SINTTOFP_I32_F32, "__mips16_floatsisf" },
-  { RTLIB::UINTTOFP_I32_F64, "__mips16_floatunsidf" },
-  { RTLIB::UINTTOFP_I32_F32, "__mips16_floatunsisf" },
-  { RTLIB::OGE_F64, "__mips16_gedf2" },
-  { RTLIB::OGE_F32, "__mips16_gesf2" },
-  { RTLIB::OGT_F64, "__mips16_gtdf2" },
-  { RTLIB::OGT_F32, "__mips16_gtsf2" },
-  { RTLIB::OLE_F64, "__mips16_ledf2" },
-  { RTLIB::OLE_F32, "__mips16_lesf2" },
-  { RTLIB::OLT_F64, "__mips16_ltdf2" },
-  { RTLIB::OLT_F32, "__mips16_ltsf2" },
-  { RTLIB::MUL_F64, "__mips16_muldf3" },
-  { RTLIB::MUL_F32, "__mips16_mulsf3" },
-  { RTLIB::UNE_F64, "__mips16_nedf2" },
-  { RTLIB::UNE_F32, "__mips16_nesf2" },
-  { RTLIB::UNKNOWN_LIBCALL, "__mips16_ret_dc" }, // No associated libcall.
-  { RTLIB::UNKNOWN_LIBCALL, "__mips16_ret_df" }, // No associated libcall.
-  { RTLIB::UNKNOWN_LIBCALL, "__mips16_ret_sc" }, // No associated libcall.
-  { RTLIB::UNKNOWN_LIBCALL, "__mips16_ret_sf" }, // No associated libcall.
-  { RTLIB::SUB_F64, "__mips16_subdf3" },
-  { RTLIB::SUB_F32, "__mips16_subsf3" },
-  { RTLIB::FPROUND_F64_F32, "__mips16_truncdfsf2" },
-  { RTLIB::UO_F64, "__mips16_unorddf2" },
-  { RTLIB::UO_F32, "__mips16_unordsf2" }
-};
+    {RTLIB::ADD_F64, RTLIB::__mips16_adddf3, "__mips16_adddf3"},
+    {RTLIB::ADD_F32, RTLIB::__mips16_addsf3, "__mips16_addsf3"},
+    {RTLIB::DIV_F64, RTLIB::__mips16_divdf3, "__mips16_divdf3"},
+    {RTLIB::DIV_F32, RTLIB::__mips16_divsf3, "__mips16_divsf3"},
+    {RTLIB::OEQ_F64, RTLIB::__mips16_eqdf2, "__mips16_eqdf2"},
+    {RTLIB::OEQ_F32, RTLIB::__mips16_eqsf2, "__mips16_eqsf2"},
+    {RTLIB::FPEXT_F32_F64, RTLIB::__mips16_extendsfdf2, "__mips16_extendsfdf2"},
+    {RTLIB::FPTOSINT_F64_I32, RTLIB::__mips16_fix_truncdfsi,
+     "__mips16_fix_truncdfsi"},
+    {RTLIB::FPTOSINT_F32_I32, RTLIB::__mips16_fix_truncsfsi,
+     "__mips16_fix_truncsfsi"},
+    {RTLIB::SINTTOFP_I32_F64, RTLIB::__mips16_floatsidf, "__mips16_floatsidf"},
+    {RTLIB::SINTTOFP_I32_F32, RTLIB::__mips16_floatsisf, "__mips16_floatsisf"},
+    {RTLIB::UINTTOFP_I32_F64, RTLIB::__mips16_floatunsidf,
+     "__mips16_floatunsidf"},
+    {RTLIB::UINTTOFP_I32_F32, RTLIB::__mips16_floatunsisf,
+     "__mips16_floatunsisf"},
+    {RTLIB::OGE_F64, RTLIB::__mips16_gedf2, "__mips16_gedf2"},
+    {RTLIB::OGE_F32, RTLIB::__mips16_gesf2, "__mips16_gesf2"},
+    {RTLIB::OGT_F64, RTLIB::__mips16_gtdf2, "__mips16_gtdf2"},
+    {RTLIB::OGT_F32, RTLIB::__mips16_gtsf2, "__mips16_gtsf2"},
+    {RTLIB::OLE_F64, RTLIB::__mips16_ledf2, "__mips16_ledf2"},
+    {RTLIB::OLE_F32, RTLIB::__mips16_lesf2, "__mips16_lesf2"},
+    {RTLIB::OLT_F64, RTLIB::__mips16_ltdf2, "__mips16_ltdf2"},
+    {RTLIB::OLT_F32, RTLIB::__mips16_ltsf2, "__mips16_ltsf2"},
+    {RTLIB::MUL_F64, RTLIB::__mips16_muldf3, "__mips16_muldf3"},
+    {RTLIB::MUL_F32, RTLIB::__mips16_mulsf3, "__mips16_mulsf3"},
+    {RTLIB::UNE_F64, RTLIB::__mips16_nedf2, "__mips16_nedf2"},
+    {RTLIB::UNE_F32, RTLIB::__mips16_nesf2, "__mips16_nesf2"},
+    {RTLIB::UNKNOWN_LIBCALL, RTLIB::__mips16_ret_dc,
+     "__mips16_ret_dc"}, // No associated libcall.
+    {RTLIB::UNKNOWN_LIBCALL, RTLIB::__mips16_ret_df,
+     "__mips16_ret_df"}, // No associated libcall.
+    {RTLIB::UNKNOWN_LIBCALL, RTLIB::__mips16_ret_sc,
+     "__mips16_ret_sc"}, // No associated libcall.
+    {RTLIB::UNKNOWN_LIBCALL, RTLIB::__mips16_ret_sf,
+     "__mips16_ret_sf"}, // No associated libcall.
+    {RTLIB::SUB_F64, RTLIB::__mips16_subdf3, "__mips16_subdf3"},
+    {RTLIB::SUB_F32, RTLIB::__mips16_subsf3, "__mips16_subsf3"},
+    {RTLIB::FPROUND_F64_F32, RTLIB::__mips16_truncdfsf2, "__mips16_truncdfsf2"},
+    {RTLIB::UO_F64, RTLIB::__mips16_unorddf2, "__mips16_unorddf2"},
+    {RTLIB::UO_F32, RTLIB::__mips16_unordsf2, "__mips16_unordsf2"}};
 
 static const Mips16IntrinsicHelperType Mips16IntrinsicHelper[] = {
   {"__fixunsdfsi", "__mips16_call_stub_2" },
@@ -250,7 +258,7 @@ void Mips16TargetLowering::setMips16HardFloatLibCalls() {
     assert((I == 0 || HardFloatLibCalls[I - 1] < HardFloatLibCalls[I]) &&
            "Array not sorted!");
     if (HardFloatLibCalls[I].Libcall != RTLIB::UNKNOWN_LIBCALL)
-      setLibcallName(HardFloatLibCalls[I].Libcall, HardFloatLibCalls[I].Name);
+      setLibcallImpl(HardFloatLibCalls[I].Libcall, HardFloatLibCalls[I].Impl);
   }
 }
 
@@ -425,7 +433,8 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
     //
     bool LookupHelper = true;
     if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(CLI.Callee)) {
-      Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL, S->getSymbol() };
+      Mips16Libcall Find = {RTLIB::UNKNOWN_LIBCALL, RTLIB::Unsupported,
+                            S->getSymbol()};
 
       if (llvm::binary_search(HardFloatLibCalls, Find))
         LookupHelper = false;
@@ -465,8 +474,8 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
       }
     } else if (GlobalAddressSDNode *G =
                    dyn_cast<GlobalAddressSDNode>(CLI.Callee)) {
-      Mips16Libcall Find = { RTLIB::UNKNOWN_LIBCALL,
-                             G->getGlobal()->getName().data() };
+      Mips16Libcall Find = {RTLIB::UNKNOWN_LIBCALL, RTLIB::Unsupported,
+                            G->getGlobal()->getName().data()};
 
       if (llvm::binary_search(HardFloatLibCalls, Find))
         LookupHelper = false;
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 9b4b33672723..5a15b7d9849d 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -2698,9 +2698,6 @@ lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue MipsTargetLowering::lowerRETURNADDR(SDValue Op,
                                             SelectionDAG &DAG) const {
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   // check the depth
   if (Op.getConstantOperandVal(0) != 0) {
     DAG.getContext()->emitError(
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
index cc79257fb9c8..28f6968ee6ca 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp
@@ -457,3 +457,25 @@ void NVPTXInstPrinter::printCTAGroup(const MCInst *MI, int OpNum,
   }
   llvm_unreachable("Invalid cta_group in printCTAGroup");
 }
+
+void NVPTXInstPrinter::printCallOperand(const MCInst *MI, int OpNum,
+                                        raw_ostream &O, StringRef Modifier) {
+  const MCOperand &MO = MI->getOperand(OpNum);
+  assert(MO.isImm() && "Invalid operand");
+  const auto Imm = MO.getImm();
+
+  if (Modifier == "RetList") {
+    assert((Imm == 1 || Imm == 0) && "Invalid return list");
+    if (Imm)
+      O << " (retval0),";
+    return;
+  }
+
+  if (Modifier == "ParamList") {
+    assert(Imm >= 0 && "Invalid parameter list");
+    interleaveComma(llvm::seq(Imm), O,
+                    [&](const auto &I) { O << "param" << I; });
+    return;
+  }
+  llvm_unreachable("Invalid modifier");
+}
diff --git a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
index f73af7a3f2c6..6189284e8a58 100644
--- a/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
+++ b/llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h
@@ -52,6 +52,8 @@ class NVPTXInstPrinter : public MCInstPrinter {
   void printPrmtMode(const MCInst *MI, int OpNum, raw_ostream &O);
   void printTmaReductionMode(const MCInst *MI, int OpNum, raw_ostream &O);
   void printCTAGroup(const MCInst *MI, int OpNum, raw_ostream &O);
+  void printCallOperand(const MCInst *MI, int OpNum, raw_ostream &O,
+                        StringRef Modifier = {});
 };
 
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index b7c5a0a5c998..b7fd7090299a 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -76,6 +76,7 @@ void initializeNVPTXAAWrapperPassPass(PassRegistry &);
 void initializeNVPTXExternalAAWrapperPass(PassRegistry &);
 void initializeNVPTXPeepholePass(PassRegistry &);
 void initializeNVPTXTagInvariantLoadLegacyPassPass(PassRegistry &);
+void initializeNVPTXPrologEpilogPassPass(PassRegistry &);
 
 struct NVVMIntrRangePass : PassInfoMixin<NVVMIntrRangePass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ff10eea37104..5631342ecc13 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -151,24 +151,12 @@ void NVPTXDAGToDAGISel::Select(SDNode *N) {
     if (tryLoadParam(N))
       return;
     break;
-  case NVPTXISD::StoreRetval:
-  case NVPTXISD::StoreRetvalV2:
-  case NVPTXISD::StoreRetvalV4:
-    if (tryStoreRetval(N))
-      return;
-    break;
   case NVPTXISD::StoreParam:
   case NVPTXISD::StoreParamV2:
   case NVPTXISD::StoreParamV4:
-  case NVPTXISD::StoreParamS32:
-  case NVPTXISD::StoreParamU32:
     if (tryStoreParam(N))
       return;
     break;
-  case ISD::INTRINSIC_WO_CHAIN:
-    if (tryIntrinsicNoChain(N))
-      return;
-    break;
   case ISD::INTRINSIC_W_CHAIN:
     if (tryIntrinsicChain(N))
       return;
@@ -904,25 +892,6 @@ NVPTXDAGToDAGISel::insertMemoryInstructionFence(SDLoc DL, SDValue &Chain,
   return {InstructionOrdering, Scope};
 }
 
-bool NVPTXDAGToDAGISel::tryIntrinsicNoChain(SDNode *N) {
-  unsigned IID = N->getConstantOperandVal(0);
-  switch (IID) {
-  default:
-    return false;
-  case Intrinsic::nvvm_texsurf_handle_internal:
-    SelectTexSurfHandle(N);
-    return true;
-  }
-}
-
-void NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
-  // Op 0 is the intrinsic ID
-  SDValue Wrapper = N->getOperand(1);
-  SDValue GlobalVal = Wrapper.getOperand(0);
-  ReplaceNode(N, CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N),
-                                        MVT::i64, GlobalVal));
-}
-
 void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
   SDValue Src = N->getOperand(0);
   AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
@@ -1364,20 +1333,18 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
   SDValue Offset, Base;
   SelectADDR(ST->getBasePtr(), Base, Offset);
 
-  SDValue Ops[] = {Value,
+  SDValue Ops[] = {selectPossiblyImm(Value),
                    getI32Imm(Ordering, DL),
                    getI32Imm(Scope, DL),
                    getI32Imm(CodeAddrSpace, DL),
-                   getI32Imm(NVPTX::PTXLdStInstCode::Untyped, DL),
                    getI32Imm(ToTypeWidth, DL),
                    Base,
                    Offset,
                    Chain};
 
-  const MVT::SimpleValueType SourceVT =
-      Value.getNode()->getSimpleValueType(0).SimpleTy;
-  const std::optional<unsigned> Opcode = pickOpcodeForVT(
-      SourceVT, NVPTX::ST_i8, NVPTX::ST_i16, NVPTX::ST_i32, NVPTX::ST_i64);
+  const std::optional<unsigned> Opcode =
+      pickOpcodeForVT(Value.getSimpleValueType().SimpleTy, NVPTX::ST_i8,
+                      NVPTX::ST_i16, NVPTX::ST_i32, NVPTX::ST_i64);
   if (!Opcode)
     return false;
 
@@ -1414,7 +1381,9 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
 
   const unsigned NumElts = getLoadStoreVectorNumElts(ST);
 
-  SmallVector<SDValue, 16> Ops(ST->ops().slice(1, NumElts));
+  SmallVector<SDValue, 16> Ops;
+  for (auto &V : ST->ops().slice(1, NumElts))
+    Ops.push_back(selectPossiblyImm(V));
   SDValue Addr = N->getOperand(NumElts + 1);
   const unsigned ToTypeWidth = TotalWidth / NumElts;
 
@@ -1425,9 +1394,8 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
   SelectADDR(Addr, Base, Offset);
 
   Ops.append({getI32Imm(Ordering, DL), getI32Imm(Scope, DL),
-              getI32Imm(CodeAddrSpace, DL),
-              getI32Imm(NVPTX::PTXLdStInstCode::Untyped, DL),
-              getI32Imm(ToTypeWidth, DL), Base, Offset, Chain});
+              getI32Imm(CodeAddrSpace, DL), getI32Imm(ToTypeWidth, DL), Base,
+              Offset, Chain});
 
   const MVT::SimpleValueType EltVT =
       ST->getOperand(1).getSimpleValueType().SimpleTy;
@@ -1530,84 +1498,6 @@ bool NVPTXDAGToDAGISel::tryLoadParam(SDNode *Node) {
   return true;
 }
 
-bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) {
-  SDLoc DL(N);
-  SDValue Chain = N->getOperand(0);
-  SDValue Offset = N->getOperand(1);
-  unsigned OffsetVal = Offset->getAsZExtVal();
-  MemSDNode *Mem = cast<MemSDNode>(N);
-
-  // How many elements do we have?
-  unsigned NumElts = 1;
-  switch (N->getOpcode()) {
-  default:
-    return false;
-  case NVPTXISD::StoreRetval:
-    NumElts = 1;
-    break;
-  case NVPTXISD::StoreRetvalV2:
-    NumElts = 2;
-    break;
-  case NVPTXISD::StoreRetvalV4:
-    NumElts = 4;
-    break;
-  }
-
-  // Build vector of operands
-  SmallVector<SDValue, 6> Ops;
-  for (unsigned i = 0; i < NumElts; ++i)
-    Ops.push_back(N->getOperand(i + 2));
-  Ops.append({CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32), Chain});
-
-  // Determine target opcode
-  // If we have an i1, use an 8-bit store. The lowering code in
-  // NVPTXISelLowering will have already emitted an upcast.
-  std::optional<unsigned> Opcode = 0;
-  switch (NumElts) {
-  default:
-    return false;
-  case 1:
-    Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                             NVPTX::StoreRetvalI8, NVPTX::StoreRetvalI16,
-                             NVPTX::StoreRetvalI32, NVPTX::StoreRetvalI64);
-    if (Opcode == NVPTX::StoreRetvalI8) {
-      // Fine tune the opcode depending on the size of the operand.
-      // This helps to avoid creating redundant COPY instructions in
-      // InstrEmitter::AddRegisterOperand().
-      switch (Ops[0].getSimpleValueType().SimpleTy) {
-      default:
-        break;
-      case MVT::i32:
-        Opcode = NVPTX::StoreRetvalI8TruncI32;
-        break;
-      case MVT::i64:
-        Opcode = NVPTX::StoreRetvalI8TruncI64;
-        break;
-      }
-    }
-    break;
-  case 2:
-    Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                             NVPTX::StoreRetvalV2I8, NVPTX::StoreRetvalV2I16,
-                             NVPTX::StoreRetvalV2I32, NVPTX::StoreRetvalV2I64);
-    break;
-  case 4:
-    Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                             NVPTX::StoreRetvalV4I8, NVPTX::StoreRetvalV4I16,
-                             NVPTX::StoreRetvalV4I32, {/* no v4i64 */});
-    break;
-  }
-  if (!Opcode)
-    return false;
-
-  SDNode *Ret = CurDAG->getMachineNode(*Opcode, DL, MVT::Other, Ops);
-  MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand();
-  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ret), {MemRef});
-
-  ReplaceNode(N, Ret);
-  return true;
-}
-
 // Helpers for constructing opcode (ex: NVPTX::StoreParamV4F32_iiri)
 #define getOpcV2H(ty, opKind0, opKind1)                                        \
   NVPTX::StoreParamV2##ty##_##opKind0##opKind1
@@ -1717,8 +1607,6 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Unexpected opcode");
-  case NVPTXISD::StoreParamU32:
-  case NVPTXISD::StoreParamS32:
   case NVPTXISD::StoreParam:
     NumElts = 1;
     break;
@@ -1741,80 +1629,54 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
   // If we have an i1, use an 8-bit store. The lowering code in
   // NVPTXISelLowering will have already emitted an upcast.
   std::optional<unsigned> Opcode;
-  switch (N->getOpcode()) {
+  switch (NumElts) {
   default:
-    switch (NumElts) {
-    default:
-      llvm_unreachable("Unexpected NumElts");
-    case 1: {
-      MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
-      SDValue Imm = Ops[0];
-      if (MemTy != MVT::f16 && MemTy != MVT::bf16 &&
-          (isa<ConstantSDNode>(Imm) || isa<ConstantFPSDNode>(Imm))) {
-        // Convert immediate to target constant
-        if (MemTy == MVT::f32 || MemTy == MVT::f64) {
-          const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
-          const ConstantFP *CF = ConstImm->getConstantFPValue();
-          Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
-        } else {
-          const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
-          const ConstantInt *CI = ConstImm->getConstantIntValue();
-          Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
-        }
-        Ops[0] = Imm;
-        // Use immediate version of store param
-        Opcode = pickOpcodeForVT(MemTy, NVPTX::StoreParamI8_i,
-                                 NVPTX::StoreParamI16_i, NVPTX::StoreParamI32_i,
-                                 NVPTX::StoreParamI64_i);
-      } else
-        Opcode =
-            pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
-                            NVPTX::StoreParamI8_r, NVPTX::StoreParamI16_r,
-                            NVPTX::StoreParamI32_r, NVPTX::StoreParamI64_r);
-      if (Opcode == NVPTX::StoreParamI8_r) {
-        // Fine tune the opcode depending on the size of the operand.
-        // This helps to avoid creating redundant COPY instructions in
-        // InstrEmitter::AddRegisterOperand().
-        switch (Ops[0].getSimpleValueType().SimpleTy) {
-        default:
-          break;
-        case MVT::i32:
-          Opcode = NVPTX::StoreParamI8TruncI32_r;
-          break;
-        case MVT::i64:
-          Opcode = NVPTX::StoreParamI8TruncI64_r;
-          break;
-        }
+    llvm_unreachable("Unexpected NumElts");
+  case 1: {
+    MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
+    SDValue Imm = Ops[0];
+    if (MemTy != MVT::f16 && MemTy != MVT::bf16 &&
+        (isa<ConstantSDNode>(Imm) || isa<ConstantFPSDNode>(Imm))) {
+      // Convert immediate to target constant
+      if (MemTy == MVT::f32 || MemTy == MVT::f64) {
+        const ConstantFPSDNode *ConstImm = cast<ConstantFPSDNode>(Imm);
+        const ConstantFP *CF = ConstImm->getConstantFPValue();
+        Imm = CurDAG->getTargetConstantFP(*CF, DL, Imm->getValueType(0));
+      } else {
+        const ConstantSDNode *ConstImm = cast<ConstantSDNode>(Imm);
+        const ConstantInt *CI = ConstImm->getConstantIntValue();
+        Imm = CurDAG->getTargetConstant(*CI, DL, Imm->getValueType(0));
+      }
+      Ops[0] = Imm;
+      // Use immediate version of store param
+      Opcode =
+          pickOpcodeForVT(MemTy, NVPTX::StoreParamI8_i, NVPTX::StoreParamI16_i,
+                          NVPTX::StoreParamI32_i, NVPTX::StoreParamI64_i);
+    } else
+      Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
+                               NVPTX::StoreParamI8_r, NVPTX::StoreParamI16_r,
+                               NVPTX::StoreParamI32_r, NVPTX::StoreParamI64_r);
+    if (Opcode == NVPTX::StoreParamI8_r) {
+      // Fine tune the opcode depending on the size of the operand.
+      // This helps to avoid creating redundant COPY instructions in
+      // InstrEmitter::AddRegisterOperand().
+      switch (Ops[0].getSimpleValueType().SimpleTy) {
+      default:
+        break;
+      case MVT::i32:
+        Opcode = NVPTX::StoreParamI8TruncI32_r;
+        break;
+      case MVT::i64:
+        Opcode = NVPTX::StoreParamI8TruncI64_r;
+        break;
       }
-      break;
-    }
-    case 2:
-    case 4: {
-      MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
-      Opcode = pickOpcodeForVectorStParam(Ops, NumElts, MemTy, CurDAG, DL);
-      break;
-    }
     }
     break;
-  // Special case: if we have a sign-extend/zero-extend node, insert the
-  // conversion instruction first, and use that as the value operand to
-  // the selected StoreParam node.
-  case NVPTXISD::StoreParamU32: {
-    Opcode = NVPTX::StoreParamI32_r;
-    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
-                                                MVT::i32);
-    SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
-                                         MVT::i32, Ops[0], CvtNone);
-    Ops[0] = SDValue(Cvt, 0);
-    break;
   }
-  case NVPTXISD::StoreParamS32: {
-    Opcode = NVPTX::StoreParamI32_r;
-    SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
-                                                MVT::i32);
-    SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
-                                         MVT::i32, Ops[0], CvtNone);
-    Ops[0] = SDValue(Cvt, 0);
+  case 2:
+  case 4: {
+    MVT::SimpleValueType MemTy = Mem->getMemoryVT().getSimpleVT().SimpleTy;
+    Opcode = pickOpcodeForVectorStParam(Ops, NumElts, MemTy, CurDAG, DL);
     break;
   }
   }
@@ -2105,22 +1967,14 @@ static inline bool isAddLike(const SDValue V) {
 // selectBaseADDR - Match a dag node which will serve as the base address for an
 // ADDR operand pair.
 static SDValue selectBaseADDR(SDValue N, SelectionDAG *DAG) {
-  // Return true if TGA or ES.
-  if (N.getOpcode() == ISD::TargetGlobalAddress ||
-      N.getOpcode() == ISD::TargetExternalSymbol)
-    return N;
-
-  if (N.getOpcode() == NVPTXISD::Wrapper)
-    return N.getOperand(0);
-
-  // addrspacecast(Wrapper(arg_symbol) to addrspace(PARAM)) -> arg_symbol
-  if (AddrSpaceCastSDNode *CastN = dyn_cast<AddrSpaceCastSDNode>(N))
-    if (CastN->getSrcAddressSpace() == ADDRESS_SPACE_GENERIC &&
-        CastN->getDestAddressSpace() == ADDRESS_SPACE_PARAM &&
-        CastN->getOperand(0).getOpcode() == NVPTXISD::Wrapper)
-      return selectBaseADDR(CastN->getOperand(0).getOperand(0), DAG);
-
-  if (auto *FIN = dyn_cast<FrameIndexSDNode>(N))
+  if (const auto *GA = dyn_cast<GlobalAddressSDNode>(N))
+    return DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N),
+                                       GA->getValueType(0), GA->getOffset(),
+                                       GA->getTargetFlags());
+  if (const auto *ES = dyn_cast<ExternalSymbolSDNode>(N))
+    return DAG->getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0),
+                                        ES->getTargetFlags());
+  if (const auto *FIN = dyn_cast<FrameIndexSDNode>(N))
     return DAG->getTargetFrameIndex(FIN->getIndex(), FIN->getValueType(0));
 
   return N;
@@ -2158,6 +2012,19 @@ bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
   return true;
 }
 
+SDValue NVPTXDAGToDAGISel::selectPossiblyImm(SDValue V) {
+  if (V.getOpcode() == ISD::BITCAST)
+    V = V.getOperand(0);
+
+  if (auto *CN = dyn_cast<ConstantSDNode>(V))
+    return CurDAG->getTargetConstant(CN->getAPIntValue(), SDLoc(V),
+                                     V.getValueType());
+  if (auto *CN = dyn_cast<ConstantFPSDNode>(V))
+    return CurDAG->getTargetConstantFP(CN->getValueAPF(), SDLoc(V),
+                                       V.getValueType());
+  return V;
+}
+
 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
                                                  unsigned int spN) const {
   const Value *Src = nullptr;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index ff58e4486a22..0e4dec1adca6 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -69,7 +69,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
 #include "NVPTXGenDAGISel.inc"
 
   void Select(SDNode *N) override;
-  bool tryIntrinsicNoChain(SDNode *N);
   bool tryIntrinsicChain(SDNode *N);
   bool tryIntrinsicVoid(SDNode *N);
   void SelectTexSurfHandle(SDNode *N);
@@ -80,7 +79,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   bool tryStore(SDNode *N);
   bool tryStoreVector(SDNode *N);
   bool tryLoadParam(SDNode *N);
-  bool tryStoreRetval(SDNode *N);
   bool tryStoreParam(SDNode *N);
   bool tryFence(SDNode *N);
   void SelectAddrSpaceCast(SDNode *N);
@@ -106,6 +104,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   }
 
   bool SelectADDR(SDValue Addr, SDValue &Base, SDValue &Offset);
+  SDValue selectPossiblyImm(SDValue V);
 
   bool ChkMemSDNodeAddressSpace(SDNode *N, unsigned int spN) const;
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 676654d6d33e..bb0aeb493ed4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -28,6 +28,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -238,18 +239,11 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
       return std::nullopt;
     LLVM_FALLTHROUGH;
   case MVT::v2i8:
-  case MVT::v2i16:
   case MVT::v2i32:
   case MVT::v2i64:
-  case MVT::v2f16:
-  case MVT::v2bf16:
   case MVT::v2f32:
   case MVT::v2f64:
-  case MVT::v4i8:
-  case MVT::v4i16:
   case MVT::v4i32:
-  case MVT::v4f16:
-  case MVT::v4bf16:
   case MVT::v4f32:
     // This is a "native" vector type
     return std::pair(NumElts, EltVT);
@@ -262,6 +256,13 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
     if (!CanLowerTo256Bit)
       return std::nullopt;
     LLVM_FALLTHROUGH;
+  case MVT::v2i16:  // <1 x i16x2>
+  case MVT::v2f16:  // <1 x f16x2>
+  case MVT::v2bf16: // <1 x bf16x2>
+  case MVT::v4i8:   // <1 x i8x4>
+  case MVT::v4i16:  // <2 x i16x2>
+  case MVT::v4f16:  // <2 x f16x2>
+  case MVT::v4bf16: // <2 x bf16x2>
   case MVT::v8i8:   // <2 x i8x4>
   case MVT::v8f16:  // <4 x f16x2>
   case MVT::v8bf16: // <4 x bf16x2>
@@ -370,7 +371,7 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
       } else if (EltVT.getSimpleVT() == MVT::i8 && NumElts == 2) {
         // v2i8 is promoted to v2i16
         NumElts = 1;
-        EltVT = MVT::v2i16;
+        EltVT = MVT::v2i8;
       }
       for (unsigned j = 0; j != NumElts; ++j) {
         ValueVTs.push_back(EltVT);
@@ -390,35 +391,27 @@ static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
 /// and promote them to a larger size if they're not.
 ///
 /// The promoted type is placed in \p PromoteVT if the function returns true.
-static std::optional<MVT> PromoteScalarIntegerPTX(const EVT &VT) {
+static EVT promoteScalarIntegerPTX(const EVT VT) {
   if (VT.isScalarInteger()) {
-    MVT PromotedVT;
     switch (PowerOf2Ceil(VT.getFixedSizeInBits())) {
     default:
       llvm_unreachable(
           "Promotion is not suitable for scalars of size larger than 64-bits");
     case 1:
-      PromotedVT = MVT::i1;
-      break;
+      return MVT::i1;
     case 2:
     case 4:
     case 8:
-      PromotedVT = MVT::i8;
-      break;
+      return MVT::i8;
     case 16:
-      PromotedVT = MVT::i16;
-      break;
+      return MVT::i16;
     case 32:
-      PromotedVT = MVT::i32;
-      break;
+      return MVT::i32;
     case 64:
-      PromotedVT = MVT::i64;
-      break;
+      return MVT::i64;
     }
-    if (VT != PromotedVT)
-      return PromotedVT;
   }
-  return std::nullopt;
+  return VT;
 }
 
 // Check whether we can merge loads/stores of some of the pieces of a
@@ -702,9 +695,6 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   setOperationAction(ISD::BR_JT, MVT::Other, Custom);
   setOperationAction(ISD::BRIND, MVT::Other, Expand);
 
-  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
-  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
-
   // We want to legalize constant related memmove and memcopy
   // intrinsics.
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
@@ -845,7 +835,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   // We have some custom DAG combine patterns for these nodes
   setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT, ISD::FADD,
                        ISD::MUL, ISD::SHL, ISD::SREM, ISD::UREM, ISD::VSELECT,
-                       ISD::BUILD_VECTOR, ISD::ADDRSPACECAST});
+                       ISD::BUILD_VECTOR, ISD::ADDRSPACECAST, ISD::LOAD,
+                       ISD::STORE});
 
   // setcc for f16x2 and bf16x2 needs special handling to prevent
   // legalizer's attempt to scalarize it due to v2i1 not being legal.
@@ -1054,45 +1045,19 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case NVPTXISD::FIRST_NUMBER:
     break;
 
-    MAKE_CASE(NVPTXISD::CALL)
     MAKE_CASE(NVPTXISD::RET_GLUE)
-    MAKE_CASE(NVPTXISD::LOAD_PARAM)
-    MAKE_CASE(NVPTXISD::Wrapper)
-    MAKE_CASE(NVPTXISD::DeclareParam)
+    MAKE_CASE(NVPTXISD::DeclareArrayParam)
     MAKE_CASE(NVPTXISD::DeclareScalarParam)
-    MAKE_CASE(NVPTXISD::DeclareRet)
-    MAKE_CASE(NVPTXISD::DeclareScalarRet)
-    MAKE_CASE(NVPTXISD::DeclareRetParam)
-    MAKE_CASE(NVPTXISD::PrintCall)
-    MAKE_CASE(NVPTXISD::PrintConvergentCall)
-    MAKE_CASE(NVPTXISD::PrintCallUni)
-    MAKE_CASE(NVPTXISD::PrintConvergentCallUni)
+    MAKE_CASE(NVPTXISD::CALL)
     MAKE_CASE(NVPTXISD::LoadParam)
     MAKE_CASE(NVPTXISD::LoadParamV2)
     MAKE_CASE(NVPTXISD::LoadParamV4)
     MAKE_CASE(NVPTXISD::StoreParam)
     MAKE_CASE(NVPTXISD::StoreParamV2)
     MAKE_CASE(NVPTXISD::StoreParamV4)
-    MAKE_CASE(NVPTXISD::StoreParamS32)
-    MAKE_CASE(NVPTXISD::StoreParamU32)
-    MAKE_CASE(NVPTXISD::CallArgBegin)
-    MAKE_CASE(NVPTXISD::CallArg)
-    MAKE_CASE(NVPTXISD::LastCallArg)
-    MAKE_CASE(NVPTXISD::CallArgEnd)
-    MAKE_CASE(NVPTXISD::CallVoid)
-    MAKE_CASE(NVPTXISD::CallVal)
-    MAKE_CASE(NVPTXISD::CallSymbol)
-    MAKE_CASE(NVPTXISD::Prototype)
     MAKE_CASE(NVPTXISD::MoveParam)
-    MAKE_CASE(NVPTXISD::StoreRetval)
-    MAKE_CASE(NVPTXISD::StoreRetvalV2)
-    MAKE_CASE(NVPTXISD::StoreRetvalV4)
-    MAKE_CASE(NVPTXISD::PseudoUseParam)
     MAKE_CASE(NVPTXISD::UNPACK_VECTOR)
     MAKE_CASE(NVPTXISD::BUILD_VECTOR)
-    MAKE_CASE(NVPTXISD::RETURN)
-    MAKE_CASE(NVPTXISD::CallSeqBegin)
-    MAKE_CASE(NVPTXISD::CallSeqEnd)
     MAKE_CASE(NVPTXISD::CallPrototype)
     MAKE_CASE(NVPTXISD::ProxyReg)
     MAKE_CASE(NVPTXISD::LoadV2)
@@ -1114,7 +1079,6 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(NVPTXISD::STACKSAVE)
     MAKE_CASE(NVPTXISD::SETP_F16X2)
     MAKE_CASE(NVPTXISD::SETP_BF16X2)
-    MAKE_CASE(NVPTXISD::Dummy)
     MAKE_CASE(NVPTXISD::MUL_WIDE_SIGNED)
     MAKE_CASE(NVPTXISD::MUL_WIDE_UNSIGNED)
     MAKE_CASE(NVPTXISD::BrxEnd)
@@ -1188,18 +1152,9 @@ SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
   }
 }
 
-SDValue
-NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
-  SDLoc dl(Op);
-  const GlobalAddressSDNode *GAN = cast<GlobalAddressSDNode>(Op);
-  auto PtrVT = getPointerTy(DAG.getDataLayout(), GAN->getAddressSpace());
-  Op = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, PtrVT);
-  return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op);
-}
-
 std::string NVPTXTargetLowering::getPrototype(
-    const DataLayout &DL, Type *retTy, const ArgListTy &Args,
-    const SmallVectorImpl<ISD::OutputArg> &Outs, MaybeAlign RetAlign,
+    const DataLayout &DL, Type *RetTy, const ArgListTy &Args,
+    const SmallVectorImpl<ISD::OutputArg> &Outs,
     std::optional<unsigned> FirstVAArg, const CallBase &CB,
     unsigned UniqueCallSite) const {
   auto PtrVT = getPointerTy(DL);
@@ -1208,22 +1163,22 @@ std::string NVPTXTargetLowering::getPrototype(
   raw_string_ostream O(Prototype);
   O << "prototype_" << UniqueCallSite << " : .callprototype ";
 
-  if (retTy->isVoidTy()) {
+  if (RetTy->isVoidTy()) {
     O << "()";
   } else {
     O << "(";
-    if (shouldPassAsArray(retTy)) {
-      assert(RetAlign && "RetAlign must be set for non-void return types");
-      O << ".param .align " << RetAlign->value() << " .b8 _["
-        << DL.getTypeAllocSize(retTy) << "]";
-    } else if (retTy->isFloatingPointTy() || retTy->isIntegerTy()) {
+    if (shouldPassAsArray(RetTy)) {
+      const Align RetAlign = getArgumentAlignment(&CB, RetTy, 0, DL);
+      O << ".param .align " << RetAlign.value() << " .b8 _["
+        << DL.getTypeAllocSize(RetTy) << "]";
+    } else if (RetTy->isFloatingPointTy() || RetTy->isIntegerTy()) {
       unsigned size = 0;
-      if (auto *ITy = dyn_cast<IntegerType>(retTy)) {
+      if (auto *ITy = dyn_cast<IntegerType>(RetTy)) {
         size = ITy->getBitWidth();
       } else {
-        assert(retTy->isFloatingPointTy() &&
+        assert(RetTy->isFloatingPointTy() &&
                "Floating point type expected here");
-        size = retTy->getPrimitiveSizeInBits();
+        size = RetTy->getPrimitiveSizeInBits();
       }
       // PTX ABI requires all scalar return values to be at least 32
       // bits in size.  fp16 normally uses .b16 as its storage type in
@@ -1231,7 +1186,7 @@ std::string NVPTXTargetLowering::getPrototype(
       size = promoteScalarArgumentSize(size);
 
       O << ".param .b" << size << " _";
-    } else if (isa<PointerType>(retTy)) {
+    } else if (isa<PointerType>(RetTy)) {
       O << ".param .b" << PtrVT.getSizeInBits() << " _";
     } else {
       llvm_unreachable("Unknown return type");
@@ -1292,7 +1247,7 @@ std::string NVPTXTargetLowering::getPrototype(
 
   if (FirstVAArg)
     O << (first ? "" : ",") << " .param .align "
-      << STI.getMaxRequiredAlignment() << " .b8 _[]\n";
+      << STI.getMaxRequiredAlignment() << " .b8 _[]";
   O << ")";
   if (shouldEmitPTXNoReturn(&CB, *nvTM))
     O << " .noreturn";
@@ -1471,7 +1426,26 @@ static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG,
 }
 
 static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags) {
-  return Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+  if (Flags.isSExt())
+    return ISD::SIGN_EXTEND;
+  if (Flags.isZExt())
+    return ISD::ZERO_EXTEND;
+  return ISD::ANY_EXTEND;
+}
+
+static SDValue correctParamType(SDValue V, EVT ExpectedVT,
+                                ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
+                                SDLoc dl) {
+  const EVT ActualVT = V.getValueType();
+  assert((ActualVT == ExpectedVT ||
+          (ExpectedVT.isInteger() && ActualVT.isInteger())) &&
+         "Non-integer argument type size mismatch");
+  if (ExpectedVT.bitsGT(ActualVT))
+    return DAG.getNode(getExtOpcode(Flags), dl, ExpectedVT, V);
+  if (ExpectedVT.bitsLT(ActualVT))
+    return DAG.getNode(ISD::TRUNCATE, dl, ExpectedVT, V);
+
+  return V;
 }
 
 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
@@ -1537,9 +1511,11 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
          "Outs and OutVals must be the same size");
   // Declare the .params or .reg need to pass values
   // to the function
-  for (const auto [ArgI, Arg] : llvm::enumerate(Args)) {
-    const auto ArgOuts = AllOuts.take_while(
-        [ArgI = ArgI](auto O) { return O.OrigArgIndex == ArgI; });
+  for (const auto E : llvm::enumerate(Args)) {
+    const auto ArgI = E.index();
+    const auto Arg = E.value();
+    const auto ArgOuts =
+        AllOuts.take_while([&](auto O) { return O.OrigArgIndex == ArgI; });
     const auto ArgOutVals = AllOutVals.take_front(ArgOuts.size());
     AllOuts = AllOuts.drop_front(ArgOuts.size());
     AllOutVals = AllOutVals.drop_front(ArgOuts.size());
@@ -1547,6 +1523,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const bool IsVAArg = (ArgI >= FirstVAArg);
     const bool IsByVal = Arg.IsByVal;
 
+    const SDValue ParamSymbol =
+        getCallParamSymbol(DAG, IsVAArg ? FirstVAArg : ArgI, MVT::i32);
+
     SmallVector<EVT, 16> VTs;
     SmallVector<uint64_t, 16> Offsets;
 
@@ -1557,38 +1536,43 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     assert(VTs.size() == Offsets.size() && "Size mismatch");
     assert((IsByVal || VTs.size() == ArgOuts.size()) && "Size mismatch");
 
-    Align ArgAlign;
-    if (IsByVal) {
-      // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
-      // so we don't need to worry whether it's naturally aligned or not.
-      // See TargetLowering::LowerCallTo().
-      Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
-      ArgAlign = getFunctionByValParamAlign(CB->getCalledFunction(), ETy,
-                                            InitialAlign, DL);
-      if (IsVAArg)
-        VAOffset = alignTo(VAOffset, ArgAlign);
-    } else {
-      ArgAlign = getArgumentAlignment(CB, Arg.Ty, ArgI + 1, DL);
-    }
+    const Align ArgAlign = [&]() {
+      if (IsByVal) {
+        // The ByValAlign in the Outs[OIdx].Flags is always set at this point,
+        // so we don't need to worry whether it's naturally aligned or not.
+        // See TargetLowering::LowerCallTo().
+        const Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
+        const Align ByValAlign = getFunctionByValParamAlign(
+            CB->getCalledFunction(), ETy, InitialAlign, DL);
+        if (IsVAArg)
+          VAOffset = alignTo(VAOffset, ByValAlign);
+        return ByValAlign;
+      }
+      return getArgumentAlignment(CB, Arg.Ty, ArgI + 1, DL);
+    }();
 
     const unsigned TypeSize = DL.getTypeAllocSize(ETy);
     assert((!IsByVal || TypeSize == ArgOuts[0].Flags.getByValSize()) &&
            "type size mismatch");
 
-    const bool PassAsArray = IsByVal || shouldPassAsArray(Arg.Ty);
-    if (IsVAArg) {
-      if (ArgI == FirstVAArg) {
-        VADeclareParam = Chain =
-            DAG.getNode(NVPTXISD::DeclareParam, dl, {MVT::Other, MVT::Glue},
-                        {Chain, GetI32(STI.getMaxRequiredAlignment()),
-                         GetI32(ArgI), GetI32(1), InGlue});
+    const std::optional<SDValue> ArgDeclare = [&]() -> std::optional<SDValue> {
+      if (IsVAArg) {
+        if (ArgI == FirstVAArg) {
+          VADeclareParam = DAG.getNode(
+              NVPTXISD::DeclareArrayParam, dl, {MVT::Other, MVT::Glue},
+              {Chain, ParamSymbol, GetI32(STI.getMaxRequiredAlignment()),
+               GetI32(0), InGlue});
+          return VADeclareParam;
+        }
+        return std::nullopt;
+      }
+      if (IsByVal || shouldPassAsArray(Arg.Ty)) {
+        // declare .param .align <align> .b8 .param<n>[<size>];
+        return DAG.getNode(NVPTXISD::DeclareArrayParam, dl,
+                           {MVT::Other, MVT::Glue},
+                           {Chain, ParamSymbol, GetI32(ArgAlign.value()),
+                            GetI32(TypeSize), InGlue});
       }
-    } else if (PassAsArray) {
-      // declare .param .align <align> .b8 .param<n>[<size>];
-      Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, {MVT::Other, MVT::Glue},
-                          {Chain, GetI32(ArgAlign.value()), GetI32(ArgI),
-                           GetI32(TypeSize), InGlue});
-    } else {
       assert(ArgOuts.size() == 1 && "We must pass only one value as non-array");
       // declare .param .b<size> .param<n>;
 
@@ -1600,11 +1584,14 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
               ? promoteScalarArgumentSize(TypeSize * 8)
               : TypeSize * 8;
 
-      Chain = DAG.getNode(
-          NVPTXISD::DeclareScalarParam, dl, {MVT::Other, MVT::Glue},
-          {Chain, GetI32(ArgI), GetI32(PromotedSize), GetI32(0), InGlue});
+      return DAG.getNode(NVPTXISD::DeclareScalarParam, dl,
+                         {MVT::Other, MVT::Glue},
+                         {Chain, ParamSymbol, GetI32(PromotedSize), InGlue});
+    }();
+    if (ArgDeclare) {
+      Chain = ArgDeclare->getValue(0);
+      InGlue = ArgDeclare->getValue(1);
     }
-    InGlue = Chain.getValue(1);
 
     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter
     // than 32-bits are sign extended or zero extended, depending on
@@ -1626,8 +1613,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       } else {
         StVal = ArgOutVals[I];
 
-        if (auto PromotedVT = PromoteScalarIntegerPTX(StVal.getValueType())) {
-          StVal = DAG.getNode(getExtOpcode(ArgOuts[I].Flags), dl, *PromotedVT,
+        auto PromotedVT = promoteScalarIntegerPTX(StVal.getValueType());
+        if (PromotedVT != StVal.getValueType()) {
+          StVal = DAG.getNode(getExtOpcode(ArgOuts[I].Flags), dl, PromotedVT,
                               StVal);
         }
       }
@@ -1651,12 +1639,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     unsigned J = 0;
     for (const unsigned NumElts : VectorInfo) {
       const int CurOffset = Offsets[J];
-      EVT EltVT = VTs[J];
+      EVT EltVT = promoteScalarIntegerPTX(VTs[J]);
       const Align PartAlign = commonAlignment(ArgAlign, CurOffset);
 
-      if (auto PromotedVT = PromoteScalarIntegerPTX(EltVT))
-        EltVT = *PromotedVT;
-
       // If we have a PVF_SCALAR entry, it may not be sufficiently aligned for a
       // scalar store. In such cases, fall back to byte stores.
       if (NumElts == 1 && !IsVAArg && PartAlign < DAG.getEVTAlign(EltVT)) {
@@ -1727,30 +1712,26 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
-  MaybeAlign RetAlign = std::nullopt;
 
   // Handle Result
   if (!Ins.empty()) {
-    RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
-
-    // Declare
-    //  .param .align N .b8 retval0[<size-in-bytes>], or
-    //  .param .b<size-in-bits> retval0
-    const unsigned ResultSize = DL.getTypeAllocSizeInBits(RetTy);
-    if (!shouldPassAsArray(RetTy)) {
-      const unsigned PromotedResultSize = promoteScalarArgumentSize(ResultSize);
-      SDValue DeclareRetOps[] = {Chain, GetI32(1), GetI32(PromotedResultSize),
-                                 GetI32(0), InGlue};
-      Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, {MVT::Other, MVT::Glue},
-                          DeclareRetOps);
-      InGlue = Chain.getValue(1);
-    } else {
-      SDValue DeclareRetOps[] = {Chain, GetI32(RetAlign->value()),
-                                 GetI32(ResultSize / 8), GetI32(0), InGlue};
-      Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl,
-                          {MVT::Other, MVT::Glue}, DeclareRetOps);
-      InGlue = Chain.getValue(1);
-    }
+    const SDValue RetDeclare = [&]() {
+      const SDValue RetSymbol = DAG.getExternalSymbol("retval0", MVT::i32);
+      const unsigned ResultSize = DL.getTypeAllocSizeInBits(RetTy);
+      if (shouldPassAsArray(RetTy)) {
+        const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
+        return DAG.getNode(NVPTXISD::DeclareArrayParam, dl,
+                           {MVT::Other, MVT::Glue},
+                           {Chain, RetSymbol, GetI32(RetAlign.value()),
+                            GetI32(ResultSize / 8), InGlue});
+      }
+      const auto PromotedResultSize = promoteScalarArgumentSize(ResultSize);
+      return DAG.getNode(
+          NVPTXISD::DeclareScalarParam, dl, {MVT::Other, MVT::Glue},
+          {Chain, RetSymbol, GetI32(PromotedResultSize), InGlue});
+    }();
+    Chain = RetDeclare.getValue(0);
+    InGlue = RetDeclare.getValue(1);
   }
 
   const bool HasVAArgs = CLI.IsVarArg && (CLI.Args.size() > CLI.NumFixedArgs);
@@ -1795,74 +1776,38 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // The prototype is embedded in a string and put as the operand for a
     // CallPrototype SDNode which will print out to the value of the string.
     std::string Proto =
-        getPrototype(DL, RetTy, Args, CLI.Outs, RetAlign,
+        getPrototype(DL, RetTy, Args, CLI.Outs,
                      HasVAArgs ? std::optional(FirstVAArg) : std::nullopt, *CB,
                      UniqueCallSite);
     const char *ProtoStr = nvTM->getStrPool().save(Proto).data();
-    SDValue ProtoOps[] = {
-        Chain,
-        DAG.getTargetExternalSymbol(ProtoStr, MVT::i32),
-        InGlue,
-    };
-    Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, {MVT::Other, MVT::Glue},
-                        ProtoOps);
+    Chain = DAG.getNode(
+        NVPTXISD::CallPrototype, dl, {MVT::Other, MVT::Glue},
+        {Chain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32), InGlue});
     InGlue = Chain.getValue(1);
   }
-  // Op to just print "call"
-  SDValue PrintCallOps[] = {Chain, GetI32(Ins.empty() ? 0 : 1), InGlue};
-  // We model convergent calls as separate opcodes.
-  unsigned Opcode =
-      IsIndirectCall ? NVPTXISD::PrintCall : NVPTXISD::PrintCallUni;
-  if (CLI.IsConvergent)
-    Opcode = Opcode == NVPTXISD::PrintCallUni ? NVPTXISD::PrintConvergentCallUni
-                                              : NVPTXISD::PrintConvergentCall;
-  Chain = DAG.getNode(Opcode, dl, {MVT::Other, MVT::Glue}, PrintCallOps);
-  InGlue = Chain.getValue(1);
 
   if (ConvertToIndirectCall) {
     // Copy the function ptr to a ptx register and use the register to call the
     // function.
-    EVT DestVT = Callee.getValueType();
-    MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
+    const MVT DestVT = Callee.getValueType().getSimpleVT();
+    MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-    unsigned DestReg =
-        RegInfo.createVirtualRegister(TLI.getRegClassFor(DestVT.getSimpleVT()));
+    Register DestReg = MRI.createVirtualRegister(TLI.getRegClassFor(DestVT));
     auto RegCopy = DAG.getCopyToReg(DAG.getEntryNode(), dl, DestReg, Callee);
     Callee = DAG.getCopyFromReg(RegCopy, dl, DestReg, DestVT);
   }
 
-  // Ops to print out the function name
-  SDValue CallVoidOps[] = { Chain, Callee, InGlue };
-  Chain =
-      DAG.getNode(NVPTXISD::CallVoid, dl, {MVT::Other, MVT::Glue}, CallVoidOps);
-  InGlue = Chain.getValue(1);
-
-  // Ops to print out the param list
-  SDValue CallArgBeginOps[] = { Chain, InGlue };
-  Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, {MVT::Other, MVT::Glue},
-                      CallArgBeginOps);
-  InGlue = Chain.getValue(1);
-
-  const unsigned E = std::min<unsigned>(CLI.NumFixedArgs + 1, Args.size());
-  for (const unsigned I : llvm::seq(E)) {
-    const unsigned Opcode =
-        I == (E - 1) ? NVPTXISD::LastCallArg : NVPTXISD::CallArg;
-    SDValue CallArgOps[] = {Chain, GetI32(1), GetI32(I), InGlue};
-    Chain = DAG.getNode(Opcode, dl, {MVT::Other, MVT::Glue}, CallArgOps);
-    InGlue = Chain.getValue(1);
-  }
-  SDValue CallArgEndOps[] = {Chain, GetI32(IsIndirectCall ? 0 : 1), InGlue};
-  Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, {MVT::Other, MVT::Glue},
-                      CallArgEndOps);
+  const unsigned Proto = IsIndirectCall ? UniqueCallSite : 0;
+  const unsigned NumArgs =
+      std::min<unsigned>(CLI.NumFixedArgs + 1, Args.size());
+  /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
+  ///      NumParams, Callee, Proto, InGlue)
+  Chain = DAG.getNode(NVPTXISD::CALL, dl, {MVT::Other, MVT::Glue},
+                      {Chain, GetI32(CLI.IsConvergent), GetI32(IsIndirectCall),
+                       GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee,
+                       GetI32(Proto), InGlue});
   InGlue = Chain.getValue(1);
 
-  if (IsIndirectCall) {
-    SDValue PrototypeOps[] = {Chain, GetI32(UniqueCallSite), InGlue};
-    Chain = DAG.getNode(NVPTXISD::Prototype, dl, {MVT::Other, MVT::Glue},
-                        PrototypeOps);
-    InGlue = Chain.getValue(1);
-  }
-
   SmallVector<SDValue, 16> ProxyRegOps;
   // An item of the vector is filled if the element does not need a ProxyReg
   // operation on it and should be added to InVals as is. ProxyRegOps and
@@ -1880,7 +1825,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0);
     assert(VTs.size() == Ins.size() && "Bad value decomposition");
 
-    assert(RetAlign && "RetAlign is guaranteed to be set");
+    const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
 
     // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
     // 32-bits are sign extended or zero extended, depending on whether
@@ -1888,17 +1833,15 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     const bool ExtendIntegerRetVal =
         RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32;
 
-    const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, *RetAlign);
+    const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
     unsigned I = 0;
     for (const unsigned VectorizedSize : VectorInfo) {
-      EVT TheLoadType = VTs[I];
+      EVT TheLoadType = promoteScalarIntegerPTX(VTs[I]);
       EVT EltType = Ins[I].VT;
-      const Align EltAlign = commonAlignment(*RetAlign, Offsets[I]);
+      const Align EltAlign = commonAlignment(RetAlign, Offsets[I]);
 
-      if (auto PromotedVT = PromoteScalarIntegerPTX(TheLoadType)) {
-        TheLoadType = *PromotedVT;
-        EltType = *PromotedVT;
-      }
+      if (TheLoadType != VTs[I])
+        EltType = TheLoadType;
 
       if (ExtendIntegerRetVal) {
         TheLoadType = MVT::i32;
@@ -1968,13 +1911,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       continue;
     }
 
-    SDValue Ret = DAG.getNode(
-        NVPTXISD::ProxyReg, dl,
-        {ProxyRegOps[I].getSimpleValueType(), MVT::Other, MVT::Glue},
-        {Chain, ProxyRegOps[I], InGlue});
-
-    Chain = Ret.getValue(1);
-    InGlue = Ret.getValue(2);
+    SDValue Ret =
+        DAG.getNode(NVPTXISD::ProxyReg, dl, ProxyRegOps[I].getSimpleValueType(),
+                    {Chain, ProxyRegOps[I]});
 
     const EVT ExpectedVT = Ins[I].VT;
     if (!Ret.getValueType().bitsEq(ExpectedVT)) {
@@ -1984,14 +1923,10 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   for (SDValue &T : TempProxyRegOps) {
-    SDValue Repl = DAG.getNode(NVPTXISD::ProxyReg, dl,
-                               {T.getSimpleValueType(), MVT::Other, MVT::Glue},
-                               {Chain, T.getOperand(0), InGlue});
+    SDValue Repl = DAG.getNode(NVPTXISD::ProxyReg, dl, T.getSimpleValueType(),
+                               {Chain, T.getOperand(0)});
     DAG.ReplaceAllUsesWith(T, Repl);
     DAG.RemoveDeadNode(T.getNode());
-
-    Chain = Repl.getValue(1);
-    InGlue = Repl.getValue(2);
   }
 
   // set isTailCall to false for now, until we figure out how to express
@@ -2856,15 +2791,16 @@ static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG,
   SDValue X = Op->getOperand(0);
   SDValue Y = Op->getOperand(1);
   EVT Ty = Op.getValueType();
+  SDNodeFlags Flags = Op->getFlags();
 
-  SDValue Div = DAG.getNode(ISD::FDIV, DL, Ty, X, Y);
-  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, Ty, Div);
-  SDValue Mul =
-      DAG.getNode(ISD::FMUL, DL, Ty, Trunc, Y, SDNodeFlags::AllowContract);
-  SDValue Sub =
-      DAG.getNode(ISD::FSUB, DL, Ty, X, Mul, SDNodeFlags::AllowContract);
+  SDValue Div = DAG.getNode(ISD::FDIV, DL, Ty, X, Y, Flags);
+  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, Ty, Div, Flags);
+  SDValue Mul = DAG.getNode(ISD::FMUL, DL, Ty, Trunc, Y,
+                            Flags | SDNodeFlags::AllowContract);
+  SDValue Sub = DAG.getNode(ISD::FSUB, DL, Ty, X, Mul,
+                            Flags | SDNodeFlags::AllowContract);
 
-  if (AllowUnsafeFPMath || Op->getFlags().hasNoInfs())
+  if (AllowUnsafeFPMath || Flags.hasNoInfs())
     return Sub;
 
   // If Y is infinite, return X
@@ -2918,8 +2854,6 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return SDValue();
   case ISD::ADDRSPACECAST:
     return LowerADDRSPACECAST(Op, DAG);
-  case ISD::GlobalAddress:
-    return LowerGlobalAddress(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:
     return Op;
   case ISD::INTRINSIC_WO_CHAIN:
@@ -3128,8 +3062,7 @@ SDValue NVPTXTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   EVT PtrVT = TLI->getPointerTy(DAG.getDataLayout());
 
   // Store the address of unsized array <function>_vararg[] in the ap object.
-  SDValue Arg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);
-  SDValue VAReg = DAG.getNode(NVPTXISD::Wrapper, DL, PtrVT, Arg);
+  SDValue VAReg = getParamSymbol(DAG, /* vararg */ -1, PtrVT);
 
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
   return DAG.getStore(Op.getOperand(0), DL, VAReg, Op.getOperand(1),
@@ -3365,11 +3298,17 @@ bool NVPTXTargetLowering::splitValueIntoRegisterParts(
 // Name of the symbol is composed from its index and the function name.
 // Negative index corresponds to special parameter (unsized array) used for
 // passing variable arguments.
-SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx,
-                                            EVT v) const {
+SDValue NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int I,
+                                            EVT T) const {
   StringRef SavedStr = nvTM->getStrPool().save(
-      getParamName(&DAG.getMachineFunction().getFunction(), idx));
-  return DAG.getTargetExternalSymbol(SavedStr.data(), v);
+      getParamName(&DAG.getMachineFunction().getFunction(), I));
+  return DAG.getExternalSymbol(SavedStr.data(), T);
+}
+
+SDValue NVPTXTargetLowering::getCallParamSymbol(SelectionDAG &DAG, int I,
+                                                EVT T) const {
+  const StringRef SavedStr = nvTM->getStrPool().save("param" + Twine(I));
+  return DAG.getExternalSymbol(SavedStr.data(), T);
 }
 
 SDValue NVPTXTargetLowering::LowerFormalArguments(
@@ -3437,7 +3376,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
 
       SDValue P;
       if (isKernelFunction(*F)) {
-        P = DAG.getNode(NVPTXISD::Wrapper, dl, ByvalIn.VT, ArgSymbol);
+        P = ArgSymbol;
         P.getNode()->setIROrder(Arg.getArgNo() + 1);
       } else {
         P = DAG.getNode(NVPTXISD::MoveParam, dl, ByvalIn.VT, ArgSymbol);
@@ -3447,10 +3386,6 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       }
       InVals.push_back(P);
     } else {
-      bool aggregateIsPacked = false;
-      if (StructType *STy = dyn_cast<StructType>(Ty))
-        aggregateIsPacked = STy->isPacked();
-
       SmallVector<EVT, 16> VTs;
       SmallVector<uint64_t, 16> Offsets;
       ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0);
@@ -3463,31 +3398,23 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign);
       unsigned I = 0;
       for (const unsigned NumElts : VectorInfo) {
-        const EVT EltVT = VTs[I];
-        const EVT LoadVT = [&]() -> EVT {
-          // i1 is loaded/stored as i8.
-          if (EltVT == MVT::i1)
-            return MVT::i8;
-          // getLoad needs a vector type, but it can't handle
-          // vectors which contain v2f16 or v2bf16 elements. So we must load
-          // using i32 here and then bitcast back.
-          if (EltVT.isVector())
-            return MVT::getIntegerVT(EltVT.getFixedSizeInBits());
-          return EltVT;
-        }();
-
-        const EVT VecVT = EVT::getVectorVT(F->getContext(), LoadVT, NumElts);
+        // i1 is loaded/stored as i8
+        const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
+        // If the element is a packed type (ex. v2f16, v4i8, etc) holding
+        // multiple elements.
+        const unsigned PackingAmt =
+            LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
+
+        const EVT VecVT =
+            NumElts == 1
+                ? LoadVT
+                : EVT::getVectorVT(F->getContext(), LoadVT.getScalarType(),
+                                   NumElts * PackingAmt);
+
         SDValue VecAddr = DAG.getObjectPtrOffset(
             dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));
 
-        const MaybeAlign PartAlign = [&]() -> MaybeAlign {
-          if (aggregateIsPacked)
-            return Align(1);
-          if (NumElts != 1)
-            return std::nullopt;
-          Align PartAlign = DAG.getEVTAlign(EltVT);
-          return commonAlignment(PartAlign, Offsets[I]);
-        }();
+        const MaybeAlign PartAlign = commonAlignment(ArgAlign, Offsets[I]);
         SDValue P =
             DAG.getLoad(VecVT, dl, Root, VecAddr,
                         MachinePointerInfo(ADDRESS_SPACE_PARAM), PartAlign,
@@ -3496,25 +3423,16 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
         if (P.getNode())
           P.getNode()->setIROrder(Arg.getArgNo() + 1);
         for (const unsigned J : llvm::seq(NumElts)) {
-          SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P,
-                                    DAG.getIntPtrConstant(J, dl));
-
-          // Extend or truncate the element if necessary (e.g. an i8 is loaded
-          // into an i16 register)
-          const EVT ExpactedVT = ArgIns[I + J].VT;
-          assert((Elt.getValueType().bitsEq(ExpactedVT) ||
-                  (ExpactedVT.isScalarInteger() &&
-                   Elt.getValueType().isScalarInteger())) &&
-                 "Non-integer argument type size mismatch");
-          if (ExpactedVT.bitsGT(Elt.getValueType())) {
-            Elt = DAG.getNode(getExtOpcode(ArgIns[I + J].Flags), dl, ExpactedVT,
-                              Elt);
-          } else if (ExpactedVT.bitsLT(Elt.getValueType())) {
-            Elt = DAG.getNode(ISD::TRUNCATE, dl, ExpactedVT, Elt);
-          } else {
-            // v2f16 was loaded as an i32. Now we must bitcast it back.
-            Elt = DAG.getBitcast(EltVT, Elt);
-          }
+          SDValue Elt =
+              NumElts == 1
+                  ? P
+                  : DAG.getNode(LoadVT.isVector() ? ISD::EXTRACT_SUBVECTOR
+                                                  : ISD::EXTRACT_VECTOR_ELT,
+                                dl, LoadVT, P,
+                                DAG.getVectorIdxConstant(J * PackingAmt, dl));
+
+          Elt = correctParamType(Elt, ArgIns[I + J].VT, ArgIns[I + J].Flags,
+                                 DAG, dl);
           InVals.push_back(Elt);
         }
         I += NumElts;
@@ -3528,33 +3446,6 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
   return Chain;
 }
 
-// Use byte-store when the param adress of the return value is unaligned.
-// This may happen when the return value is a field of a packed structure.
-static SDValue LowerUnalignedStoreRet(SelectionDAG &DAG, SDValue Chain,
-                                      uint64_t Offset, EVT ElementType,
-                                      SDValue RetVal, const SDLoc &dl) {
-  // Bit logic only works on integer types
-  if (adjustElementType(ElementType))
-    RetVal = DAG.getNode(ISD::BITCAST, dl, ElementType, RetVal);
-
-  // Store each byte
-  for (unsigned i = 0, n = ElementType.getSizeInBits() / 8; i < n; i++) {
-    // Shift the byte to the last byte position
-    SDValue ShiftVal = DAG.getNode(ISD::SRL, dl, ElementType, RetVal,
-                                   DAG.getConstant(i * 8, dl, MVT::i32));
-    SDValue StoreOperands[] = {Chain, DAG.getConstant(Offset + i, dl, MVT::i32),
-                               ShiftVal};
-    // Trunc store only the last byte by using
-    //     st.param.b8
-    // The register type can be larger than b8.
-    Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
-                                    DAG.getVTList(MVT::Other), StoreOperands,
-                                    MVT::i8, MachinePointerInfo(), std::nullopt,
-                                    MachineMemOperand::MOStore);
-  }
-  return Chain;
-}
-
 SDValue
 NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                                  bool isVarArg,
@@ -3576,10 +3467,6 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets);
   assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
 
-  for (const unsigned I : llvm::seq(VTs.size()))
-    if (const auto PromotedVT = PromoteScalarIntegerPTX(VTs[I]))
-      VTs[I] = *PromotedVT;
-
   // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than
   // 32-bits are sign extended or zero extended, depending on whether
   // they are signed or unsigned types.
@@ -3588,62 +3475,48 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
   const auto GetRetVal = [&](unsigned I) -> SDValue {
     SDValue RetVal = OutVals[I];
-    assert(!PromoteScalarIntegerPTX(RetVal.getValueType()) &&
+    assert(promoteScalarIntegerPTX(RetVal.getValueType()) ==
+               RetVal.getValueType() &&
            "OutVal type should always be legal");
 
-    if (ExtendIntegerRetVal) {
-      RetVal = DAG.getNode(getExtOpcode(Outs[I].Flags), dl, MVT::i32, RetVal);
-    } else if (RetVal.getValueSizeInBits() < 16) {
-      // Use 16-bit registers for small load-stores as it's the
-      // smallest general purpose register size supported by NVPTX.
-      RetVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, RetVal);
-    }
-    return RetVal;
+    const EVT VTI = promoteScalarIntegerPTX(VTs[I]);
+    const EVT StoreVT =
+        ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
+    return correctParamType(RetVal, StoreVT, Outs[I].Flags, DAG, dl);
   };
 
   const auto RetAlign = getFunctionParamOptimizedAlign(&F, RetTy, DL);
   const auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign);
   unsigned I = 0;
   for (const unsigned NumElts : VectorInfo) {
-    const Align CurrentAlign = commonAlignment(RetAlign, Offsets[I]);
-    if (NumElts == 1 && RetTy->isAggregateType() &&
-        CurrentAlign < DAG.getEVTAlign(VTs[I])) {
-      Chain = LowerUnalignedStoreRet(DAG, Chain, Offsets[I], VTs[I],
-                                     GetRetVal(I), dl);
-
-      // The call to LowerUnalignedStoreRet inserted the necessary SDAG nodes
-      // into the graph, so just move on to the next element.
-      I++;
-      continue;
-    }
-
-    SmallVector<SDValue, 6> StoreOperands{
-        Chain, DAG.getConstant(Offsets[I], dl, MVT::i32)};
+    const MaybeAlign CurrentAlign = ExtendIntegerRetVal
+                                        ? MaybeAlign(std::nullopt)
+                                        : commonAlignment(RetAlign, Offsets[I]);
 
-    for (const unsigned J : llvm::seq(NumElts))
-      StoreOperands.push_back(GetRetVal(I + J));
+    SDValue Val;
+    if (NumElts == 1) {
+      Val = GetRetVal(I);
+    } else {
+      SmallVector<SDValue, 4> StoreVals;
+      for (const unsigned J : llvm::seq(NumElts)) {
+        SDValue ValJ = GetRetVal(I + J);
+        if (ValJ.getValueType().isVector())
+          DAG.ExtractVectorElements(ValJ, StoreVals);
+        else
+          StoreVals.push_back(ValJ);
+      }
 
-    NVPTXISD::NodeType Op;
-    switch (NumElts) {
-    case 1:
-      Op = NVPTXISD::StoreRetval;
-      break;
-    case 2:
-      Op = NVPTXISD::StoreRetvalV2;
-      break;
-    case 4:
-      Op = NVPTXISD::StoreRetvalV4;
-      break;
-    default:
-      llvm_unreachable("Invalid vector info.");
+      EVT VT = EVT::getVectorVT(F.getContext(), StoreVals[0].getValueType(),
+                                StoreVals.size());
+      Val = DAG.getBuildVector(VT, dl, StoreVals);
     }
 
-    // Adjust type of load/store op if we've extended the scalar
-    // return value.
-    EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[I];
-    Chain = DAG.getMemIntrinsicNode(
-        Op, dl, DAG.getVTList(MVT::Other), StoreOperands, TheStoreType,
-        MachinePointerInfo(), CurrentAlign, MachineMemOperand::MOStore);
+    const SDValue RetSymbol = DAG.getExternalSymbol("func_retval0", MVT::i32);
+    SDValue Ptr =
+        DAG.getObjectPtrOffset(dl, RetSymbol, TypeSize::getFixed(Offsets[I]));
+
+    Chain = DAG.getStore(Chain, dl, Val, Ptr,
+                         MachinePointerInfo(ADDRESS_SPACE_PARAM), CurrentAlign);
 
     I += NumElts;
   }
@@ -5047,26 +4920,230 @@ PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
   return SDValue();
 }
 
-static SDValue PerformStoreCombineHelper(SDNode *N, std::size_t Front,
-                                         std::size_t Back) {
+/// Fold extractelts into a load by increasing the number of return values.
+///
+/// ex:
+/// L: v2f16,ch = load <p>
+/// a: f16 = extractelt L:0, 0
+/// b: f16 = extractelt L:0, 1
+/// use(a, b)
+///
+/// ...is turned into...
+/// L: f16,f16,ch = LoadV2 <p>
+/// use(L:0, L:1)
+static SDValue
+combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+  // Don't run this optimization before the legalizer
+  if (!DCI.isAfterLegalizeDAG())
+    return SDValue();
+
+  EVT ElemVT = N->getValueType(0);
+  if (!Isv2x16VT(ElemVT))
+    return SDValue();
+
+  // Check whether all outputs are either used by an extractelt or are
+  // glue/chain nodes
+  if (!all_of(N->uses(), [&](SDUse &U) {
+        // Skip glue, chain nodes
+        if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
+          return true;
+        if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+          if (N->getOpcode() != ISD::LOAD)
+            return true;
+          // Since this is an ISD::LOAD, check all extractelts are used. If
+          // any are not used, we don't want to defeat another optimization that
+          // will narrow the load.
+          //
+          // For example:
+          //
+          // L: v2f16,ch = load <p>
+          // e0: f16 = extractelt L:0, 0
+          // e1: f16 = extractelt L:0, 1        <-- unused
+          // store e0
+          //
+          // Can be optimized by DAGCombiner to:
+          //
+          // L: f16,ch = load <p>
+          // store L:0
+          return !U.getUser()->use_empty();
+        }
+
+        // Otherwise, this use prevents us from splitting a value.
+        return false;
+      }))
+    return SDValue();
+
+  auto *LD = cast<MemSDNode>(N);
+  EVT MemVT = LD->getMemoryVT();
+  SDLoc DL(LD);
+
+  // the new opcode after we double the number of operands
+  NVPTXISD::NodeType Opcode;
+  SmallVector<SDValue> Operands(LD->ops());
+  unsigned OldNumOutputs; // non-glue, non-chain outputs
+  switch (LD->getOpcode()) {
+  case ISD::LOAD:
+    OldNumOutputs = 1;
+    // Any packed type is legal, so the legalizer will not have lowered
+    // ISD::LOAD -> NVPTXISD::Load (unless it's under-aligned). We have to do it
+    // here.
+    Opcode = NVPTXISD::LoadV2;
+    Operands.push_back(DCI.DAG.getIntPtrConstant(
+        cast<LoadSDNode>(LD)->getExtensionType(), DL));
+    break;
+  case NVPTXISD::LoadParamV2:
+    OldNumOutputs = 2;
+    Opcode = NVPTXISD::LoadParamV4;
+    break;
+  case NVPTXISD::LoadV2:
+    OldNumOutputs = 2;
+    Opcode = NVPTXISD::LoadV4;
+    break;
+  case NVPTXISD::LoadV4:
+  case NVPTXISD::LoadV8:
+    // PTX doesn't support the next doubling of outputs
+    return SDValue();
+  }
+
+  // the non-glue, non-chain outputs in the new load
+  const unsigned NewNumOutputs = OldNumOutputs * 2;
+  SmallVector<EVT> NewVTs(NewNumOutputs, ElemVT.getVectorElementType());
+  // add remaining chain and glue values
+  NewVTs.append(LD->value_begin() + OldNumOutputs, LD->value_end());
+
+  // Create the new load
+  SDValue NewLoad =
+      DCI.DAG.getMemIntrinsicNode(Opcode, DL, DCI.DAG.getVTList(NewVTs),
+                                  Operands, MemVT, LD->getMemOperand());
+
+  // Now we use a combination of BUILD_VECTORs and a MERGE_VALUES node to keep
+  // the outputs the same. These nodes will be optimized away in later
+  // DAGCombiner iterations.
+  SmallVector<SDValue> Results;
+  for (unsigned I : seq(OldNumOutputs))
+    Results.push_back(DCI.DAG.getBuildVector(
+        ElemVT, DL, {NewLoad.getValue(I * 2), NewLoad.getValue(I * 2 + 1)}));
+  // Add remaining chain and glue nodes
+  for (unsigned I : seq(NewLoad->getNumValues() - NewNumOutputs))
+    Results.push_back(NewLoad.getValue(NewNumOutputs + I));
+
+  return DCI.DAG.getMergeValues(Results, DL);
+}
+
+/// Fold a packing mov into a store.
+///
+/// ex:
+/// v: v2f16 = BUILD_VECTOR a:f16, b:f16
+/// StoreRetval v
+///
+/// ...is turned into...
+///
+/// StoreRetvalV2 a:f16, b:f16
+static SDValue combinePackingMovIntoStore(SDNode *N,
+                                          TargetLowering::DAGCombinerInfo &DCI,
+                                          unsigned Front, unsigned Back) {
+  // We want to run this as late as possible since other optimizations may
+  // eliminate the BUILD_VECTORs.
+  if (!DCI.isAfterLegalizeDAG())
+    return SDValue();
+
+  // Get the type of the operands being stored.
+  EVT ElementVT = N->getOperand(Front).getValueType();
+
+  if (!Isv2x16VT(ElementVT))
+    return SDValue();
+
+  auto *ST = cast<MemSDNode>(N);
+  EVT MemVT = ElementVT.getVectorElementType();
+
+  // The new opcode after we double the number of operands.
+  NVPTXISD::NodeType Opcode;
+  switch (N->getOpcode()) {
+  case ISD::STORE:
+    // Any packed type is legal, so the legalizer will not have lowered
+    // ISD::STORE -> NVPTXISD::Store (unless it's under-aligned). We have to do
+    // it here.
+    MemVT = ST->getMemoryVT();
+    Opcode = NVPTXISD::StoreV2;
+    break;
+  case NVPTXISD::StoreParam:
+    Opcode = NVPTXISD::StoreParamV2;
+    break;
+  case NVPTXISD::StoreParamV2:
+    Opcode = NVPTXISD::StoreParamV4;
+    break;
+  case NVPTXISD::StoreV2:
+    MemVT = ST->getMemoryVT();
+    Opcode = NVPTXISD::StoreV4;
+    break;
+  case NVPTXISD::StoreV4:
+  case NVPTXISD::StoreParamV4:
+  case NVPTXISD::StoreV8:
+    // PTX doesn't support the next doubling of operands
+    return SDValue();
+  default:
+    llvm_unreachable("Unhandled store opcode");
+  }
+
+  // Scan the operands and if they're all BUILD_VECTORs, we'll have gathered
+  // their elements.
+  SmallVector<SDValue, 4> Operands(N->ops().take_front(Front));
+  for (SDValue BV : N->ops().drop_front(Front).drop_back(Back)) {
+    if (BV.getOpcode() != ISD::BUILD_VECTOR)
+      return SDValue();
+
+    // If the operand has multiple uses, this optimization can increase register
+    // pressure.
+    if (!BV.hasOneUse())
+      return SDValue();
+
+    // DAGCombiner visits nodes bottom-up. Check the BUILD_VECTOR operands for
+    // any signs they may be folded by some other pattern or rule.
+    for (SDValue Op : BV->ops()) {
+      // Peek through bitcasts
+      if (Op.getOpcode() == ISD::BITCAST)
+        Op = Op.getOperand(0);
+
+      // This may be folded into a PRMT.
+      if (Op.getValueType() == MVT::i16 && Op.getOpcode() == ISD::TRUNCATE &&
+          Op->getOperand(0).getValueType() == MVT::i32)
+        return SDValue();
+
+      // This may be folded into cvt.bf16x2
+      if (Op.getOpcode() == ISD::FP_ROUND)
+        return SDValue();
+    }
+    Operands.append({BV.getOperand(0), BV.getOperand(1)});
+  }
+  Operands.append(N->op_end() - Back, N->op_end());
+
+  // Now we replace the store
+  return DCI.DAG.getMemIntrinsicNode(Opcode, SDLoc(N), N->getVTList(), Operands,
+                                     MemVT, ST->getMemOperand());
+}
+
+static SDValue PerformStoreCombineHelper(SDNode *N,
+                                         TargetLowering::DAGCombinerInfo &DCI,
+                                         unsigned Front, unsigned Back) {
   if (all_of(N->ops().drop_front(Front).drop_back(Back),
              [](const SDUse &U) { return U.get()->isUndef(); }))
     // Operand 0 is the previous value in the chain. Cannot return EntryToken
     // as the previous value will become unused and eliminated later.
     return N->getOperand(0);
 
-  return SDValue();
+  return combinePackingMovIntoStore(N, DCI, Front, Back);
 }
 
-static SDValue PerformStoreParamCombine(SDNode *N) {
-  // Operands from the 3rd to the 2nd last one are the values to be stored.
-  //   {Chain, ArgID, Offset, Val, Glue}
-  return PerformStoreCombineHelper(N, 3, 1);
+static SDValue PerformStoreCombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI) {
+  return combinePackingMovIntoStore(N, DCI, 1, 2);
 }
 
-static SDValue PerformStoreRetvalCombine(SDNode *N) {
-  // Operands from the 2nd to the last one are the values to be stored
-  return PerformStoreCombineHelper(N, 2, 0);
+static SDValue PerformStoreParamCombine(SDNode *N,
+                                        TargetLowering::DAGCombinerInfo &DCI) {
+  // Operands from the 3rd to the 2nd last one are the values to be stored.
+  //   {Chain, ArgID, Offset, Val, Glue}
+  return PerformStoreCombineHelper(N, DCI, 3, 1);
 }
 
 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
@@ -5697,14 +5774,19 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
       return PerformREMCombine(N, DCI, OptLevel);
     case ISD::SETCC:
       return PerformSETCCCombine(N, DCI, STI.getSmVersion());
-    case NVPTXISD::StoreRetval:
-    case NVPTXISD::StoreRetvalV2:
-    case NVPTXISD::StoreRetvalV4:
-      return PerformStoreRetvalCombine(N);
+    case ISD::LOAD:
+    case NVPTXISD::LoadParamV2:
+    case NVPTXISD::LoadV2:
+    case NVPTXISD::LoadV4:
+      return combineUnpackingMovIntoLoad(N, DCI);
     case NVPTXISD::StoreParam:
     case NVPTXISD::StoreParamV2:
     case NVPTXISD::StoreParamV4:
-      return PerformStoreParamCombine(N);
+      return PerformStoreParamCombine(N, DCI);
+    case ISD::STORE:
+    case NVPTXISD::StoreV2:
+    case NVPTXISD::StoreV4:
+      return PerformStoreCombine(N, DCI);
     case ISD::EXTRACT_VECTOR_ELT:
       return PerformEXTRACTCombine(N, DCI);
     case ISD::VSELECT:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 0a54a8fd71f3..2477e1fb6159 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -24,32 +24,24 @@ namespace NVPTXISD {
 enum NodeType : unsigned {
   // Start the numbering from where ISD NodeType finishes.
   FIRST_NUMBER = ISD::BUILTIN_OP_END,
-  Wrapper,
-  CALL,
   RET_GLUE,
-  LOAD_PARAM,
-  DeclareParam,
+
+  /// These nodes represent a parameter declaration. In PTX this will look like:
+  ///   .param .align 16 .b8 param0[1024];
+  ///   .param .b32 retval0;
+  ///
+  /// DeclareArrayParam(Chain, Externalsym, Align, Size, Glue)
+  /// DeclareScalarParam(Chain, Externalsym, Size, Glue)
   DeclareScalarParam,
-  DeclareRetParam,
-  DeclareRet,
-  DeclareScalarRet,
-  PrintCall,
-  PrintConvergentCall,
-  PrintCallUni,
-  PrintConvergentCallUni,
-  CallArgBegin,
-  CallArg,
-  LastCallArg,
-  CallArgEnd,
-  CallVoid,
-  CallVal,
-  CallSymbol,
-  Prototype,
+  DeclareArrayParam,
+
+  /// This node represents a PTX call instruction. It's operands are as follows:
+  ///
+  /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
+  ///      NumParams, Callee, Proto, InGlue)
+  CALL,
+
   MoveParam,
-  PseudoUseParam,
-  RETURN,
-  CallSeqBegin,
-  CallSeqEnd,
   CallPrototype,
   ProxyReg,
   FSHL_CLAMP,
@@ -83,7 +75,6 @@ enum NodeType : unsigned {
   CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X,
   CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y,
   CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z,
-  Dummy,
 
   FIRST_MEMORY_OPCODE,
   LoadV2 = FIRST_MEMORY_OPCODE,
@@ -100,12 +91,7 @@ enum NodeType : unsigned {
   StoreParam,
   StoreParamV2,
   StoreParamV4,
-  StoreParamS32, // to sext and store a <32bit value, not used currently
-  StoreParamU32, // to zext and store a <32bit value, not used currently
-  StoreRetval,
-  StoreRetvalV2,
-  StoreRetvalV4,
-  LAST_MEMORY_OPCODE = StoreRetvalV4,
+  LAST_MEMORY_OPCODE = StoreParamV4,
 };
 }
 
@@ -120,8 +106,6 @@ class NVPTXTargetLowering : public TargetLowering {
                                const NVPTXSubtarget &STI);
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
 
-  SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
-
   const char *getTargetNodeName(unsigned Opcode) const override;
 
   bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
@@ -195,7 +179,6 @@ class NVPTXTargetLowering : public TargetLowering {
 
   std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
                            const SmallVectorImpl<ISD::OutputArg> &,
-                           MaybeAlign RetAlign,
                            std::optional<unsigned> FirstVAArg,
                            const CallBase &CB, unsigned UniqueCallSite) const;
 
@@ -293,8 +276,8 @@ class NVPTXTargetLowering : public TargetLowering {
   const NVPTXSubtarget &STI; // cache the subtarget here
   mutable unsigned GlobalUniqueCallSite;
 
-  SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
-
+  SDValue getParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
+  SDValue getCallParamSymbol(SelectionDAG &DAG, int I, EVT T) const;
   SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index bf84d1dca4ed..e218ef17bb09 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -190,22 +190,4 @@ unsigned NVPTXInstrInfo::insertBranch(MachineBasicBlock &MBB,
   BuildMI(&MBB, DL, get(NVPTX::CBranch)).add(Cond[0]).addMBB(TBB);
   BuildMI(&MBB, DL, get(NVPTX::GOTO)).addMBB(FBB);
   return 2;
-}
-
-bool NVPTXInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
-                                          const MachineBasicBlock *MBB,
-                                          const MachineFunction &MF) const {
-  // Prevent the scheduler from reordering & splitting up MachineInstrs
-  // which must stick together (in initially set order) to
-  // comprise a valid PTX function call sequence.
-  switch (MI.getOpcode()) {
-  case NVPTX::CallUniPrintCallRetInst1:
-  case NVPTX::CallArgBeginInst:
-  case NVPTX::CallArgParam:
-  case NVPTX::LastCallArgParam:
-  case NVPTX::CallArgEndInst1:
-    return true;
-  }
-
-  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
-}
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
index 95464dbbd176..4e9dc9d3b468 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
@@ -66,9 +66,6 @@ class NVPTXInstrInfo : public NVPTXGenInstrInfo {
                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
-  bool isSchedulingBoundary(const MachineInstr &MI,
-                            const MachineBasicBlock *MBB,
-                            const MachineFunction &MF) const override;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 597905476464..441ddeeb7d66 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -184,6 +184,18 @@ class OneUse2<SDPatternOperator operator>
 class fpimm_pos_inf<ValueType vt>
     : FPImmLeaf<vt, [{ return Imm.isPosInfinity(); }]>;
 
+
+
+// Operands which can hold a Register or an Immediate.
+//
+// Unfortunately, since most register classes can hold multiple types, we must
+// use the 'Any' type for these.
+
+def RI1  : Operand<i1>;
+def RI16 : Operand<Any>;
+def RI32 : Operand<Any>;
+def RI64 : Operand<Any>;
+
 // Utility class to wrap up information about a register and DAG type for more
 // convenient iteration and parameterization
 class RegTyInfo<ValueType ty, NVPTXRegClass rc, Operand imm, SDNode imm_node,
@@ -1700,17 +1712,6 @@ def Offseti32imm : Operand<i32> {
   let PrintMethod = "printOffseti32imm";
 }
 
-def SDTWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisPtrTy<0>]>;
-def Wrapper    : SDNode<"NVPTXISD::Wrapper", SDTWrapper>;
-
-// Load a memory address into a u32 or u64 register.
-def MOV_ADDR : BasicNVPTXInst<(outs B32:$dst), (ins ADDR_base:$a),
-                         "mov.b32",
-                         [(set i32:$dst, (Wrapper tglobaladdr:$a))]>;
-def MOV_ADDR64 : BasicNVPTXInst<(outs B64:$dst), (ins ADDR_base:$a),
-                           "mov.b64",
-                           [(set i64:$dst, (Wrapper tglobaladdr:$a))]>;
-
 // Get pointer to local stack.
 let hasSideEffects = false in {
   def MOV_DEPOT_ADDR :    NVPTXInst<(outs B32:$d), (ins i32imm:$num),
@@ -1750,8 +1751,27 @@ def BFMOV16i : MOVi<B16, "b16", bf16, bf16imm, fpimm>;
 def FMOV32i : MOVi<B32, "b32", f32, f32imm, fpimm>;
 def FMOV64i : MOVi<B64, "b64", f64, f64imm, fpimm>;
 
-def : Pat<(i32 (Wrapper texternalsym:$dst)), (IMOV32i texternalsym:$dst)>;
-def : Pat<(i64 (Wrapper texternalsym:$dst)), (IMOV64i texternalsym:$dst)>;
+
+def to_tglobaladdr : SDNodeXForm<globaladdr, [{
+  return CurDAG->getTargetGlobalAddress(N->getGlobal(), SDLoc(N),
+                                        N->getValueType(0), N->getOffset(),
+                                        N->getTargetFlags());
+}]>;
+
+def to_texternsym : SDNodeXForm<externalsym, [{
+  return CurDAG->getTargetExternalSymbol(N->getSymbol(), N->getValueType(0),
+                                         N->getTargetFlags());
+}]>;
+
+def to_tframeindex : SDNodeXForm<frameindex, [{
+  return CurDAG->getTargetFrameIndex(N->getIndex(), N->getValueType(0));
+}]>;
+
+def : Pat<(i32 globaladdr:$dst), (IMOV32i (to_tglobaladdr $dst))>;
+def : Pat<(i64 globaladdr:$dst), (IMOV64i (to_tglobaladdr $dst))>;
+
+def : Pat<(i32 externalsym:$dst), (IMOV32i (to_texternsym $dst))>;
+def : Pat<(i64 externalsym:$dst), (IMOV64i (to_texternsym $dst))>;
 
 //---- Copy Frame Index ----
 def LEA_ADDRi :   NVPTXInst<(outs B32:$dst), (ins ADDR:$addr),
@@ -1759,10 +1779,6 @@ def LEA_ADDRi :   NVPTXInst<(outs B32:$dst), (ins ADDR:$addr),
 def LEA_ADDRi64 : NVPTXInst<(outs B64:$dst), (ins ADDR:$addr),
                             "add.u64 \t$dst, ${addr:add};", []>;
 
-def to_tframeindex : SDNodeXForm<frameindex, [{
-  return CurDAG->getTargetFrameIndex(N->getIndex(), N->getValueType(0));
-}]>;
-
 def : Pat<(i32 frameindex:$fi), (LEA_ADDRi (to_tframeindex $fi), 0)>;
 def : Pat<(i64 frameindex:$fi), (LEA_ADDRi64 (to_tframeindex $fi), 0)>;
 
@@ -1974,41 +1990,28 @@ defm FSetNE : FSET_FORMAT<setne, CmpNE, CmpNE_FTZ>;
 defm FSetNUM : FSET_FORMAT<seto, CmpNUM, CmpNUM_FTZ>;
 defm FSetNAN : FSET_FORMAT<setuo, CmpNAN, CmpNAN_FTZ>;
 
-def SDTDeclareParamProfile :
-  SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>]>;
-def SDTDeclareScalarParamProfile :
-  SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>]>;
+def SDTDeclareArrayParam :
+  SDTypeProfile<0, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i32>]>;
+def SDTDeclareScalarParam :
+  SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
 def SDTLoadParamProfile : SDTypeProfile<1, 2, [SDTCisInt<1>, SDTCisInt<2>]>;
 def SDTLoadParamV2Profile : SDTypeProfile<2, 2, [SDTCisSameAs<0, 1>, SDTCisInt<2>, SDTCisInt<3>]>;
 def SDTLoadParamV4Profile : SDTypeProfile<4, 2, [SDTCisInt<4>, SDTCisInt<5>]>;
-def SDTPrintCallProfile : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
 def SDTStoreParamProfile : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>]>;
 def SDTStoreParamV2Profile : SDTypeProfile<0, 4, [SDTCisInt<0>, SDTCisInt<1>]>;
 def SDTStoreParamV4Profile : SDTypeProfile<0, 6, [SDTCisInt<0>, SDTCisInt<1>]>;
-def SDTStoreParam32Profile : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>]>;
-def SDTCallArgProfile : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
-def SDTCallArgMarkProfile : SDTypeProfile<0, 0, []>;
-def SDTCallVoidProfile : SDTypeProfile<0, 1, []>;
-def SDTCallValProfile : SDTypeProfile<1, 0, []>;
-def SDTMoveParamProfile : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisInt<1>]>;
-def SDTStoreRetvalProfile : SDTypeProfile<0, 2, [SDTCisInt<0>]>;
-def SDTStoreRetvalV2Profile : SDTypeProfile<0, 3, [SDTCisInt<0>]>;
-def SDTStoreRetvalV4Profile : SDTypeProfile<0, 5, [SDTCisInt<0>]>;
-def SDTPseudoUseParamProfile : SDTypeProfile<0, 1, []>;
-def SDTProxyRegProfile : SDTypeProfile<1, 1, []>;
-
-def DeclareParam :
-  SDNode<"NVPTXISD::DeclareParam", SDTDeclareParamProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareScalarParam :
-  SDNode<"NVPTXISD::DeclareScalarParam", SDTDeclareScalarParamProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareRetParam :
-  SDNode<"NVPTXISD::DeclareRetParam", SDTDeclareParamProfile,
+def SDTMoveParamProfile : SDTypeProfile<1, 1, [SDTCisInt<0>, SDTCisSameAs<0, 1>]>;
+
+def SDTProxyReg : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>]>;
+
+
+def declare_array_param :
+  SDNode<"NVPTXISD::DeclareArrayParam", SDTDeclareArrayParam,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def DeclareRet :
-  SDNode<"NVPTXISD::DeclareRet", SDTDeclareScalarParamProfile,
+def declare_scalar_param :
+  SDNode<"NVPTXISD::DeclareScalarParam", SDTDeclareScalarParam,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
+
 def LoadParam :
   SDNode<"NVPTXISD::LoadParam", SDTLoadParamProfile,
          [SDNPHasChain, SDNPMayLoad, SDNPOutGlue, SDNPInGlue]>;
@@ -2018,18 +2021,6 @@ def LoadParamV2 :
 def LoadParamV4 :
   SDNode<"NVPTXISD::LoadParamV4", SDTLoadParamV4Profile,
          [SDNPHasChain, SDNPMayLoad, SDNPOutGlue, SDNPInGlue]>;
-def PrintCall :
-  SDNode<"NVPTXISD::PrintCall", SDTPrintCallProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def PrintConvergentCall :
-  SDNode<"NVPTXISD::PrintConvergentCall", SDTPrintCallProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def PrintCallUni :
-  SDNode<"NVPTXISD::PrintCallUni", SDTPrintCallProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def PrintConvergentCallUni :
-  SDNode<"NVPTXISD::PrintConvergentCallUni", SDTPrintCallProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
 def StoreParam :
   SDNode<"NVPTXISD::StoreParam", SDTStoreParamProfile,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
@@ -2039,52 +2030,18 @@ def StoreParamV2 :
 def StoreParamV4 :
   SDNode<"NVPTXISD::StoreParamV4", SDTStoreParamV4Profile,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def StoreParamU32 :
-  SDNode<"NVPTXISD::StoreParamU32", SDTStoreParam32Profile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def StoreParamS32 :
-  SDNode<"NVPTXISD::StoreParamS32", SDTStoreParam32Profile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def CallArgBegin :
-  SDNode<"NVPTXISD::CallArgBegin", SDTCallArgMarkProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def CallArg :
-  SDNode<"NVPTXISD::CallArg", SDTCallArgProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def LastCallArg :
-  SDNode<"NVPTXISD::LastCallArg", SDTCallArgProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def CallArgEnd :
-  SDNode<"NVPTXISD::CallArgEnd", SDTCallVoidProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def CallVoid :
-  SDNode<"NVPTXISD::CallVoid", SDTCallVoidProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def Prototype :
-  SDNode<"NVPTXISD::Prototype", SDTCallVoidProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def CallVal :
-  SDNode<"NVPTXISD::CallVal", SDTCallValProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
 def MoveParam :
   SDNode<"NVPTXISD::MoveParam", SDTMoveParamProfile, []>;
-def StoreRetval :
-  SDNode<"NVPTXISD::StoreRetval", SDTStoreRetvalProfile,
-         [SDNPHasChain, SDNPSideEffect]>;
-def StoreRetvalV2 :
-  SDNode<"NVPTXISD::StoreRetvalV2", SDTStoreRetvalV2Profile,
-         [SDNPHasChain, SDNPSideEffect]>;
-def StoreRetvalV4 :
-  SDNode<"NVPTXISD::StoreRetvalV4", SDTStoreRetvalV4Profile,
-         [SDNPHasChain, SDNPSideEffect]>;
-def PseudoUseParam :
-  SDNode<"NVPTXISD::PseudoUseParam", SDTPseudoUseParamProfile,
-         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
-def RETURNNode :
-  SDNode<"NVPTXISD::RETURN", SDTCallArgMarkProfile,
-         [SDNPHasChain, SDNPSideEffect]>;
-def ProxyReg :
-  SDNode<"NVPTXISD::ProxyReg", SDTProxyRegProfile,
+def proxy_reg :
+  SDNode<"NVPTXISD::ProxyReg", SDTProxyReg, [SDNPHasChain]>;
+
+  /// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
+  ///      NumParams, Callee, Proto, InGlue)
+def SDTCallProfile : SDTypeProfile<0, 6,
+                       [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i32>,
+                        SDTCisVT<3, i32>, SDTCisVT<5, i32>]>;
+def call :
+  SDNode<"NVPTXISD::CALL", SDTCallProfile,
          [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
 
 let mayLoad = true in {
@@ -2107,11 +2064,6 @@ let mayLoad = true in {
                   []>;
 }
 
-class LoadParamRegInst<NVPTXRegClass regclass, string opstr> :
-      NVPTXInst<(outs regclass:$dst), (ins i32imm:$b),
-                !strconcat("mov", opstr, " \t$dst, retval$b;"),
-                [(set regclass:$dst, (LoadParam (i32 0), (i32 imm:$b)))]>;
-
 let mayStore = true in {
 
   multiclass StoreParamInst<NVPTXRegClass regclass, Operand IMMType, string opstr, bit support_imm = true> {
@@ -2153,44 +2105,44 @@ let mayStore = true in {
                           " \t[param$a$b], {{$val1, $val2, $val3, $val4}};",
                           []>;
   }
+}
 
-  class StoreRetvalInst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs), (ins regclass:$val, Offseti32imm:$a),
-                  !strconcat("st.param", opstr, " \t[func_retval0$a], $val;"),
-                  []>;
+/// CALL(Chain, IsConvergent, IsIndirectCall/IsUniform, NumReturns,
+///      NumParams, Callee, Proto, InGlue)
 
-  class StoreRetvalV2Inst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs), (ins regclass:$val, regclass:$val2, Offseti32imm:$a),
-                  !strconcat("st.param.v2", opstr,
-                             " \t[func_retval0$a], {{$val, $val2}};"),
-                  []>;
+def CallOperand : Operand<i32> { let PrintMethod = "printCallOperand"; }
 
-  class StoreRetvalV4Inst<NVPTXRegClass regclass, string opstr> :
-        NVPTXInst<(outs),
-                  (ins regclass:$val, regclass:$val2, regclass:$val3,
-                       regclass:$val4, Offseti32imm:$a),
-                  !strconcat("st.param.v4", opstr,
-                             " \t[func_retval0$a], {{$val, $val2, $val3, $val4}};"),
-                  []>;
-}
+foreach is_convergent = [0, 1] in {
+  defvar convergent_suffix = !if(is_convergent, "_conv", "");
+
+  let isCall = 1, isConvergent = is_convergent in {
+    def CALL # convergent_suffix :
+      NVPTXInst<(outs),
+                (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params, 
+                     i32imm:$proto),
+                "call${rets:RetList} $addr, (${params:ParamList}), prototype_$proto;", []>;
 
-let isCall=1 in {
-  multiclass CALL<string OpcStr, SDNode OpNode> {
-     def PrintCallNoRetInst : NVPTXInst<(outs), (ins),
-       OpcStr # " ", [(OpNode 0)]>;
-     def PrintCallRetInst1 : NVPTXInst<(outs), (ins),
-       OpcStr # " (retval0), ", [(OpNode 1)]>;
+    def CALL_UNI # convergent_suffix :
+      NVPTXInst<(outs),
+                (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params),
+                "call.uni${rets:RetList} $addr, (${params:ParamList});", []>;
   }
-}
 
-defm Call : CALL<"call", PrintCall>;
-defm CallUni : CALL<"call.uni", PrintCallUni>;
+  defvar call_inst = !cast<NVPTXInst>("CALL" # convergent_suffix);
+  def : Pat<(call is_convergent, 1, imm:$rets, imm:$params, globaladdr:$addr, imm:$proto),
+            (call_inst (to_tglobaladdr $addr), imm:$rets, imm:$params, imm:$proto)>;
+  def : Pat<(call is_convergent, 1, imm:$rets, imm:$params, i32:$addr, imm:$proto),
+            (call_inst $addr, imm:$rets, imm:$params, imm:$proto)>;
+  def : Pat<(call is_convergent, 1, imm:$rets, imm:$params, i64:$addr, imm:$proto),
+            (call_inst $addr, imm:$rets, imm:$params, imm:$proto)>;
 
-// Convergent call instructions.  These are identical to regular calls, except
-// they have the isConvergent bit set.
-let isConvergent=1 in {
-  defm ConvergentCall : CALL<"call", PrintConvergentCall>;
-  defm ConvergentCallUni : CALL<"call.uni", PrintConvergentCallUni>;
+  defvar call_uni_inst = !cast<NVPTXInst>("CALL_UNI" # convergent_suffix);
+  def : Pat<(call is_convergent, 0, imm:$rets, imm:$params, globaladdr:$addr, 0),
+            (call_uni_inst (to_tglobaladdr $addr), imm:$rets, imm:$params)>;
+  def : Pat<(call is_convergent, 0, imm:$rets, imm:$params, i32:$addr, 0),
+            (call_uni_inst $addr, imm:$rets, imm:$params)>;
+  def : Pat<(call is_convergent, 0, imm:$rets, imm:$params, i64:$addr, 0),
+            (call_uni_inst $addr, imm:$rets, imm:$params)>;
 }
 
 def LoadParamMemI64    : LoadParamMemInst<B64, ".b64">;
@@ -2230,89 +2182,30 @@ defm StoreParamV2F64  : StoreParamV2Inst<B64, f64imm, ".b64">;
 
 defm StoreParamV4F32  : StoreParamV4Inst<B32, f32imm, ".b32">;
 
-def StoreRetvalI64    : StoreRetvalInst<B64, ".b64">;
-def StoreRetvalI32    : StoreRetvalInst<B32, ".b32">;
-def StoreRetvalI16    : StoreRetvalInst<B16, ".b16">;
-def StoreRetvalI8     : StoreRetvalInst<B16, ".b8">;
-def StoreRetvalI8TruncI32 : StoreRetvalInst<B32, ".b8">;
-def StoreRetvalI8TruncI64 : StoreRetvalInst<B64, ".b8">;
-def StoreRetvalV2I64  : StoreRetvalV2Inst<B64, ".b64">;
-def StoreRetvalV2I32  : StoreRetvalV2Inst<B32, ".b32">;
-def StoreRetvalV2I16  : StoreRetvalV2Inst<B16, ".b16">;
-def StoreRetvalV2I8   : StoreRetvalV2Inst<B16, ".b8">;
-def StoreRetvalV4I32  : StoreRetvalV4Inst<B32, ".b32">;
-def StoreRetvalV4I16  : StoreRetvalV4Inst<B16, ".b16">;
-def StoreRetvalV4I8   : StoreRetvalV4Inst<B16, ".b8">;
-
-def CallArgBeginInst : NVPTXInst<(outs), (ins), "(", [(CallArgBegin)]>;
-def CallArgEndInst1  : NVPTXInst<(outs), (ins), ");", [(CallArgEnd (i32 1))]>;
-def CallArgEndInst0  : NVPTXInst<(outs), (ins), ")", [(CallArgEnd (i32 0))]>;
-def RETURNInst       : NVPTXInst<(outs), (ins), "ret;", [(RETURNNode)]>;
-
-def CallArgParam : NVPTXInst<(outs), (ins i32imm:$a), "param$a, ",
-                             [(CallArg 1, imm:$a)]>;
-def LastCallArgParam : NVPTXInst<(outs), (ins i32imm:$a), "param$a",
-                                 [(LastCallArg 1, imm:$a)]>;
-
-def CallVoidInst :      NVPTXInst<(outs), (ins ADDR_base:$addr), "$addr, ",
-                                  [(CallVoid (Wrapper tglobaladdr:$addr))]>;
-def CallVoidInstReg :   NVPTXInst<(outs), (ins B32:$addr), "$addr, ",
-                                  [(CallVoid i32:$addr)]>;
-def CallVoidInstReg64 : NVPTXInst<(outs), (ins B64:$addr), "$addr, ",
-                                  [(CallVoid i64:$addr)]>;
-def PrototypeInst :     NVPTXInst<(outs), (ins i32imm:$val), ", prototype_$val;",
-                                  [(Prototype (i32 imm:$val))]>;
-
-def DeclareRetMemInst :
-  NVPTXInst<(outs), (ins i32imm:$align, i32imm:$size, i32imm:$num),
-            ".param .align $align .b8 retval$num[$size];",
-            [(DeclareRetParam (i32 imm:$align), (i32 imm:$size), (i32 imm:$num))]>;
-def DeclareRetScalarInst :
-  NVPTXInst<(outs), (ins i32imm:$size, i32imm:$num),
-            ".param .b$size retval$num;",
-            [(DeclareRet (i32 1), (i32 imm:$size), (i32 imm:$num))]>;
-def DeclareRetRegInst :
-  NVPTXInst<(outs), (ins i32imm:$size, i32imm:$num),
-            ".reg .b$size retval$num;",
-            [(DeclareRet (i32 2), (i32 imm:$size), (i32 imm:$num))]>;
-
-def DeclareParamInst :
-  NVPTXInst<(outs), (ins i32imm:$align, i32imm:$a, i32imm:$size),
-            ".param .align $align .b8 param$a[$size];",
-            [(DeclareParam (i32 imm:$align), (i32 imm:$a), (i32 imm:$size))]>;
-def DeclareScalarParamInst :
-  NVPTXInst<(outs), (ins i32imm:$a, i32imm:$size),
-            ".param .b$size param$a;",
-            [(DeclareScalarParam (i32 imm:$a), (i32 imm:$size), (i32 0))]>;
-def DeclareScalarRegInst :
+def DECLARE_PARAM_array :
+  NVPTXInst<(outs), (ins i32imm:$a, i32imm:$align, i32imm:$size),
+            ".param .align $align .b8 \t$a[$size];", []>;
+def DECLARE_PARAM_scalar :
   NVPTXInst<(outs), (ins i32imm:$a, i32imm:$size),
-            ".reg .b$size param$a;",
-            [(DeclareScalarParam (i32 imm:$a), (i32 imm:$size), (i32 1))]>;
+            ".param .b$size \t$a;", []>;
 
-class MoveParamSymbolInst<RegTyInfo t> :
-  BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src),
-            "mov.b" # t.Size,
-            [(set t.Ty:$dst, (MoveParam texternalsym:$src))]>;
+def : Pat<(declare_array_param externalsym:$a, imm:$align, imm:$size),
+          (DECLARE_PARAM_array (to_texternsym $a), imm:$align, imm:$size)>;
+def : Pat<(declare_scalar_param externalsym:$a, imm:$size),
+          (DECLARE_PARAM_scalar (to_texternsym $a), imm:$size)>;
 
-def MOV64_PARAM : MoveParamSymbolInst<I64RT>;
-def MOV32_PARAM : MoveParamSymbolInst<I32RT>;
-
-class PseudoUseParamInst<NVPTXRegClass regclass, ValueType vt> :
-  NVPTXInst<(outs), (ins regclass:$src),
-            "// Pseudo use of $src",
-            [(PseudoUseParam vt:$src)]>;
-
-def PseudoUseParamI64 : PseudoUseParamInst<B64, i64>;
-def PseudoUseParamI32 : PseudoUseParamInst<B32, i32>;
-def PseudoUseParamI16 : PseudoUseParamInst<B16, i16>;
-def PseudoUseParamF64 : PseudoUseParamInst<B64, f64>;
-def PseudoUseParamF32 : PseudoUseParamInst<B32, f32>;
+foreach t = [I32RT, I64RT] in {
+  defvar inst_name = "MOV" # t.Size # "_PARAM";
+  def inst_name : BasicNVPTXInst<(outs t.RC:$dst), (ins t.RC:$src), "mov.b" # t.Size>;
+  def : Pat<(MoveParam (t.Ty externalsym:$src)),
+            (!cast<NVPTXInst>(inst_name) (t.Ty (to_texternsym $src)))>;
+}
 
 multiclass ProxyRegInst<string SzStr, NVPTXRegClass rc> {
   def NAME : BasicNVPTXInst<(outs rc:$dst), (ins rc:$src),
                  "mov." # SzStr>;
   foreach vt = rc.RegTypes in
-    def : Pat<(vt (ProxyReg vt:$src)), (!cast<NVPTXInst>(NAME) $src)>;
+    def : Pat<(vt (proxy_reg vt:$src)), (!cast<NVPTXInst>(NAME) $src)>;
 }
 
 defm ProxyRegB1  : ProxyRegInst<"pred", B1>;
@@ -2338,19 +2231,20 @@ let mayLoad=1, hasSideEffects=0 in {
   def LD_i64 : LD<B64>;
 }
 
-class ST<NVPTXRegClass regclass>
+class ST<DAGOperand O>
   : NVPTXInst<
     (outs),
-    (ins regclass:$src, LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp,
-         LdStCode:$Sign, i32imm:$toWidth, ADDR:$addr),
-    "st${sem:sem}${scope:scope}${addsp:addsp}.${Sign:sign}$toWidth"
+    (ins O:$src,
+         LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp, i32imm:$toWidth,
+         ADDR:$addr),
+    "st${sem:sem}${scope:scope}${addsp:addsp}.b$toWidth"
     " \t[$addr], $src;", []>;
 
 let mayStore=1, hasSideEffects=0 in {
-  def ST_i8  : ST<B16>;
-  def ST_i16 : ST<B16>;
-  def ST_i32 : ST<B32>;
-  def ST_i64 : ST<B64>;
+  def ST_i8  : ST<RI16>;
+  def ST_i16 : ST<RI16>;
+  def ST_i32 : ST<RI32>;
+  def ST_i64 : ST<RI64>;
 }
 
 // The following is used only in and after vector elementizations.  Vector
@@ -2386,38 +2280,38 @@ let mayLoad=1, hasSideEffects=0 in {
   defm LDV_i64 : LD_VEC<B64>;
 }
 
-multiclass ST_VEC<NVPTXRegClass regclass, bit support_v8 = false> {
+multiclass ST_VEC<DAGOperand O, bit support_v8 = false> {
   def _v2 : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, LdStCode:$sem, LdStCode:$scope,
-         LdStCode:$addsp, LdStCode:$Sign, i32imm:$fromWidth,
+    (ins O:$src1, O:$src2,
+         LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp, i32imm:$fromWidth,
          ADDR:$addr),
-    "st${sem:sem}${scope:scope}${addsp:addsp}.v2.${Sign:sign}$fromWidth "
+    "st${sem:sem}${scope:scope}${addsp:addsp}.v2.b$fromWidth "
     "\t[$addr], {{$src1, $src2}};", []>;
   def _v4 : NVPTXInst<
     (outs),
-    (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-         LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp,
-         LdStCode:$Sign, i32imm:$fromWidth, ADDR:$addr),
-    "st${sem:sem}${scope:scope}${addsp:addsp}.v4.${Sign:sign}$fromWidth "
+    (ins O:$src1, O:$src2, O:$src3, O:$src4,
+         LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp, i32imm:$fromWidth,
+         ADDR:$addr),
+    "st${sem:sem}${scope:scope}${addsp:addsp}.v4.b$fromWidth "
     "\t[$addr], {{$src1, $src2, $src3, $src4}};", []>;
   if support_v8 then
     def _v8 : NVPTXInst<
       (outs),
-      (ins regclass:$src1, regclass:$src2, regclass:$src3, regclass:$src4,
-           regclass:$src5, regclass:$src6, regclass:$src7, regclass:$src8,
-           LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp, LdStCode:$Sign,
-           i32imm:$fromWidth, ADDR:$addr),
-      "st${sem:sem}${scope:scope}${addsp:addsp}.v8.${Sign:sign}$fromWidth "
+      (ins O:$src1, O:$src2, O:$src3, O:$src4,
+           O:$src5, O:$src6, O:$src7, O:$src8,
+           LdStCode:$sem, LdStCode:$scope, LdStCode:$addsp, i32imm:$fromWidth, 
+           ADDR:$addr),
+      "st${sem:sem}${scope:scope}${addsp:addsp}.v8.b$fromWidth "
       "\t[$addr], "
       "{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, $src8}};", []>;
 }
 
 let mayStore=1, hasSideEffects=0 in {
-  defm STV_i8  : ST_VEC<B16>;
-  defm STV_i16 : ST_VEC<B16>;
-  defm STV_i32 : ST_VEC<B32, support_v8 = true>;
-  defm STV_i64 : ST_VEC<B64>;
+  defm STV_i8  : ST_VEC<RI16>;
+  defm STV_i16 : ST_VEC<RI16>;
+  defm STV_i32 : ST_VEC<RI32, support_v8 = true>;
+  defm STV_i64 : ST_VEC<RI64>;
 }
 
 //---- Conversion ----
@@ -2861,21 +2755,6 @@ def callseq_end   : SDNode<"ISD::CALLSEQ_END", SDT_NVPTXCallSeqEnd,
                            [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                             SDNPSideEffect]>;
 
-def SDT_NVPTXCall : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
-def call          : SDNode<"NVPTXISD::CALL", SDT_NVPTXCall,
-                           [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
-def calltarget : Operand<i32>;
-let isCall=1 in {
-   def CALL : NVPTXInst<(outs), (ins calltarget:$dst), "call \t$dst, (1);", []>;
-}
-
-def : Pat<(call tglobaladdr:$dst), (CALL tglobaladdr:$dst)>;
-def : Pat<(call texternalsym:$dst), (CALL texternalsym:$dst)>;
-
-// Pseudo instructions.
-class Pseudo<dag outs, dag ins, string asmstr, list<dag> pattern>
-   : NVPTXInst<outs, ins, asmstr, pattern>;
-
 def Callseq_Start :
   NVPTXInst<(outs), (ins i32imm:$amt1, i32imm:$amt2),
             "\\{ // callseq $amt1, $amt2",
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 10d7f04d8d93..cc1fd027d851 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2224,6 +2224,8 @@ def nvvm_move_sym64 : NVPTXInst<(outs B64:$r), (ins ADDR_base:$s),
 
 def texsurf_handles
   : BasicNVPTXInst<(outs B64:$result), (ins ADDR_base:$src), "mov.u64">;
+def : Pat<(int_nvvm_texsurf_handle_internal globaladdr:$src),
+          (texsurf_handles (to_tglobaladdr $src))>;
 
 //-----------------------------------
 // Compiler Error Warn
diff --git a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
index 7929bd2e0df0..ababc66cc5d2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
@@ -41,7 +41,7 @@ class NVPTXPrologEpilogPass : public MachineFunctionPass {
 private:
   void calculateFrameObjectOffsets(MachineFunction &Fn);
 };
-}
+} // end anonymous namespace
 
 MachineFunctionPass *llvm::createNVPTXPrologEpilogPass() {
   return new NVPTXPrologEpilogPass();
@@ -49,6 +49,44 @@ MachineFunctionPass *llvm::createNVPTXPrologEpilogPass() {
 
 char NVPTXPrologEpilogPass::ID = 0;
 
+INITIALIZE_PASS(NVPTXPrologEpilogPass, DEBUG_TYPE,
+                "NVPTX Prologue/Epilogue Insertion", false, false)
+
+static bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
+                                        unsigned OpIdx) {
+  const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
+  const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+  if (MI.isDebugValue()) {
+
+    MachineOperand &Op = MI.getOperand(OpIdx);
+    assert(MI.isDebugOperand(&Op) &&
+           "Frame indices can only appear as a debug operand in a DBG_VALUE*"
+           " machine instruction");
+    Register Reg;
+    unsigned FrameIdx = Op.getIndex();
+
+    StackOffset Offset = TFI->getFrameIndexReference(MF, FrameIdx, Reg);
+    Op.ChangeToRegister(Reg, false /*isDef*/);
+
+    const DIExpression *DIExpr = MI.getDebugExpression();
+    if (MI.isNonListDebugValue()) {
+      DIExpr = TRI.prependOffsetExpression(MI.getDebugExpression(),
+                                           DIExpression::ApplyOffset, Offset);
+    } else {
+      // The debug operand at DebugOpIndex was a frame index at offset
+      // `Offset`; now the operand has been replaced with the frame
+      // register, we must add Offset with `register x, plus Offset`.
+      unsigned DebugOpIndex = MI.getDebugOperandIndex(&Op);
+      SmallVector<uint64_t, 3> Ops;
+      TRI.getOffsetOpcodes(Offset, Ops);
+      DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, DebugOpIndex);
+    }
+    MI.getDebugExpressionOp().setMetadata(DIExpr);
+    return true;
+  }
+  return false;
+}
+
 bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {
   const TargetSubtargetInfo &STI = MF.getSubtarget();
   const TargetFrameLowering &TFI = *STI.getFrameLowering();
@@ -57,41 +95,27 @@ bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {
 
   calculateFrameObjectOffsets(MF);
 
-  for (MachineBasicBlock &MBB : MF) {
-    for (MachineInstr &MI : MBB) {
-      for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
-        if (!MI.getOperand(i).isFI())
+  for (MachineBasicBlock &BB : MF) {
+    for (MachineBasicBlock::iterator I = BB.end(); I != BB.begin();) {
+      MachineInstr &MI = *std::prev(I);
+
+      bool RemovedMI = false;
+      for (const auto &[Idx, Op] : enumerate(MI.operands())) {
+        if (!Op.isFI())
           continue;
 
-        // Frame indices in debug values are encoded in a target independent
-        // way with simply the frame index and offset rather than any
-        // target-specific addressing mode.
-        if (MI.isDebugValue()) {
-          MachineOperand &Op = MI.getOperand(i);
-          assert(
-              MI.isDebugOperand(&Op) &&
-              "Frame indices can only appear as a debug operand in a DBG_VALUE*"
-              " machine instruction");
-          Register Reg;
-          auto Offset =
-              TFI.getFrameIndexReference(MF, Op.getIndex(), Reg);
-          Op.ChangeToRegister(Reg, /*isDef=*/false);
-          const DIExpression *DIExpr = MI.getDebugExpression();
-          if (MI.isNonListDebugValue()) {
-            DIExpr = TRI.prependOffsetExpression(MI.getDebugExpression(), DIExpression::ApplyOffset, Offset);
-          } else {
-	    SmallVector<uint64_t, 3> Ops;
-            TRI.getOffsetOpcodes(Offset, Ops);
-            unsigned OpIdx = MI.getDebugOperandIndex(&Op);
-            DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, OpIdx);
-          }
-          MI.getDebugExpressionOp().setMetadata(DIExpr);
+        if (replaceFrameIndexDebugInstr(MF, MI, Idx))
           continue;
-        }
 
-        TRI.eliminateFrameIndex(MI, 0, i, nullptr);
+        // Eliminate this FrameIndex operand.
+        RemovedMI = TRI.eliminateFrameIndex(MI, 0, Idx, nullptr);
         Modified = true;
+        if (RemovedMI)
+          break;
       }
+
+      if (!RemovedMI)
+        --I;
     }
   }
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
index aa07d510b3a1..646b554878c7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
@@ -103,15 +103,20 @@ BitVector NVPTXRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
 bool NVPTXRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                             int SPAdj, unsigned FIOperandNum,
-                                            RegScavenger *RS) const {
+                                            RegScavenger *) const {
   assert(SPAdj == 0 && "Unexpected");
 
   MachineInstr &MI = *II;
-  int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
+  if (MI.isLifetimeMarker()) {
+    MI.eraseFromParent();
+    return true;
+  }
+
+  const int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
-  MachineFunction &MF = *MI.getParent()->getParent();
-  int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex) +
-               MI.getOperand(FIOperandNum + 1).getImm();
+  const MachineFunction &MF = *MI.getParent()->getParent();
+  const int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex) +
+                     MI.getOperand(FIOperandNum + 1).getImm();
 
   // Using I0 as the frame pointer
   MI.getOperand(FIOperandNum).ChangeToRegister(getFrameRegister(MF), false);
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index ef310e5828f2..0603994606d7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -115,6 +115,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() {
   initializeNVPTXExternalAAWrapperPass(PR);
   initializeNVPTXPeepholePass(PR);
   initializeNVPTXTagInvariantLoadLegacyPassPass(PR);
+  initializeNVPTXPrologEpilogPassPass(PR);
 }
 
 static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) {
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 2b3727be644d..58766b16e86f 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "MCTargetDesc/PPCTargetStreamer.h"
 #include "PPCInstrInfo.h"
@@ -666,7 +666,7 @@ struct PPCOperand : public MCParsedAsmOperand {
     return StringRef(Tok.Data, Tok.Length);
   }
 
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   static std::unique_ptr<PPCOperand> CreateToken(StringRef Str, SMLoc S,
                                                  bool IsPPC64) {
@@ -781,7 +781,7 @@ struct PPCOperand : public MCParsedAsmOperand {
 
 } // end anonymous namespace.
 
-void PPCOperand::print(raw_ostream &OS) const {
+void PPCOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case Token:
     OS << "'" << getToken() << "'";
@@ -791,10 +791,10 @@ void PPCOperand::print(raw_ostream &OS) const {
     OS << getImm();
     break;
   case Expression:
-    OS << *getExpr();
+    MAI.printExpr(OS, *getExpr());
     break;
   case TLSRegister:
-    OS << *getTLSReg();
+    MAI.printExpr(OS, *getTLSReg());
     break;
   }
 }
@@ -1514,8 +1514,7 @@ bool PPCAsmParser::parseOperand(OperandVector &Operands) {
             Tok.getString().compare_insensitive("plt") == 0))
         return Error(Tok.getLoc(), "expected 'plt'");
       EVal = MCSymbolRefExpr::create(getContext().getOrCreateSymbol(TlsGetAddr),
-                                     MCSymbolRefExpr::VariantKind(PPC::S_PLT),
-                                     getContext());
+                                     PPC::S_PLT, getContext());
       if (parseOptionalToken(AsmToken::Plus)) {
         const MCExpr *Addend = nullptr;
         SMLoc EndLoc;
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
index 94b3c2f9abb1..8eb013d18616 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMPowerPCDesc
   PPCMCTargetDesc.cpp
   PPCMCAsmInfo.cpp
   PPCMCCodeEmitter.cpp
-  PPCMCExpr.cpp
   PPCPredicates.cpp
   PPCELFObjectWriter.cpp
   PPCXCOFFObjectWriter.cpp
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index d4b86d5e2811..5779d4e8ca8c 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -7,14 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/PPCFixupKinds.h"
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -91,84 +90,13 @@ class PPCAsmBackend : public MCAsmBackend {
                                          : llvm::endianness::big),
         TT(TT) {}
 
-  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
-    const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = {
-      // name                    offset  bits  flags
-      { "fixup_ppc_br24",        6,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24_notoc",  6,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_brcond14",    16,     14,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24abs",     6,      24,   0 },
-      { "fixup_ppc_brcond14abs", 16,     14,   0 },
-      { "fixup_ppc_half16",       0,     16,   0 },
-      { "fixup_ppc_half16ds",     0,     14,   0 },
-      { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_imm34",       0,      34,   0 },
-      { "fixup_ppc_nofixup",      0,      0,   0 }
-    };
-    const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = {
-      // name                    offset  bits  flags
-      { "fixup_ppc_br24",        2,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24_notoc",  2,      24,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_brcond14",    2,      14,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_br24abs",     2,      24,   0 },
-      { "fixup_ppc_brcond14abs", 2,      14,   0 },
-      { "fixup_ppc_half16",      0,      16,   0 },
-      { "fixup_ppc_half16ds",    2,      14,   0 },
-      { "fixup_ppc_pcrel34",     0,      34,   MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_ppc_imm34",       0,      34,   0 },
-      { "fixup_ppc_nofixup",     0,       0,   0 }
-    };
-
-    // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
-    // do not require any extra processing.
-    if (mc::isRelocation(Kind))
-      return MCAsmBackend::getFixupKindInfo(FK_NONE);
-
-    if (Kind < FirstTargetFixupKind)
-      return MCAsmBackend::getFixupKindInfo(Kind);
-
-    assert(Kind - FirstTargetFixupKind < PPC::NumTargetFixupKinds &&
-           "Invalid kind!");
-    return (Endian == llvm::endianness::little
-                ? InfosLE
-                : InfosBE)[Kind - FirstTargetFixupKind];
-  }
-
-  bool addReloc(const MCFragment &F, const MCFixup &Fixup,
-                const MCValue &TargetVal, uint64_t &FixedValue,
-                bool IsResolved) override {
-    // In PPC64 ELFv1, .quad .TOC.@tocbase in the .opd section is expected to
-    // reference the null symbol.
-    auto Target = TargetVal;
-    if (Target.getSpecifier() == PPC::S_TOCBASE)
-      Target.setAddSym(nullptr);
-    return MCAsmBackend::addReloc(F, Fixup, Target, FixedValue, IsResolved);
-  }
+  MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void applyFixup(const MCFragment &, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t Value, bool IsResolved) override {
-    MCFixupKind Kind = Fixup.getKind();
-    if (mc::isRelocation(Kind))
-      return;
-    Value = adjustFixupValue(Kind, Value);
-    if (!Value) return;           // Doesn't change encoding.
+                  uint64_t Value, bool IsResolved) override;
 
-    unsigned Offset = Fixup.getOffset();
-    unsigned NumBytes = getFixupKindNumBytes(Kind);
-
-    // For each byte of the fragment that the fixup touches, mask in the bits
-    // from the fixup value. The Value has been "split up" into the appropriate
-    // bitfields above.
-    for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx =
-          Endian == llvm::endianness::little ? i : (NumBytes - 1 - i);
-      Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff);
-    }
-  }
-
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
+  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target) {
     // If there is a @ specifier, unless it is optimized out (e.g. constant @l),
     // force a relocation.
     if (Target.getSpecifier())
@@ -200,12 +128,6 @@ class PPCAsmBackend : public MCAsmBackend {
     }
   }
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {
-    // FIXME.
-    llvm_unreachable("relaxInstruction() unimplemented");
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override {
     uint64_t NumNops = Count / 4;
@@ -219,6 +141,81 @@ class PPCAsmBackend : public MCAsmBackend {
 };
 } // end anonymous namespace
 
+MCFixupKindInfo PPCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
+  // clang-format off
+  const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = {
+      // name                    offset  bits  flags
+      {"fixup_ppc_br24", 6, 24, 0},
+      {"fixup_ppc_br24_notoc", 6, 24, 0},
+      {"fixup_ppc_brcond14", 16, 14, 0},
+      {"fixup_ppc_br24abs", 6, 24, 0},
+      {"fixup_ppc_brcond14abs", 16, 14, 0},
+      {"fixup_ppc_half16", 0, 16, 0},
+      {"fixup_ppc_half16ds", 0, 14, 0},
+      {"fixup_ppc_pcrel34", 0, 34, 0},
+      {"fixup_ppc_imm34", 0, 34, 0},
+      {"fixup_ppc_nofixup", 0, 0, 0}};
+  const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = {
+      // name                    offset  bits  flags
+      {"fixup_ppc_br24", 2, 24, 0},
+      {"fixup_ppc_br24_notoc", 2, 24, 0},
+      {"fixup_ppc_brcond14", 2, 14, 0},
+      {"fixup_ppc_br24abs", 2, 24, 0},
+      {"fixup_ppc_brcond14abs", 2, 14, 0},
+      {"fixup_ppc_half16", 0, 16, 0},
+      {"fixup_ppc_half16ds", 2, 14, 0},
+      {"fixup_ppc_pcrel34", 0, 34, 0},
+      {"fixup_ppc_imm34", 0, 34, 0},
+      {"fixup_ppc_nofixup", 0, 0, 0}};
+  // clang-format on
+
+  // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They
+  // do not require any extra processing.
+  if (mc::isRelocation(Kind))
+    return {};
+
+  if (Kind < FirstTargetFixupKind)
+    return MCAsmBackend::getFixupKindInfo(Kind);
+
+  assert(Kind - FirstTargetFixupKind < PPC::NumTargetFixupKinds &&
+         "Invalid kind!");
+  return (Endian == llvm::endianness::little
+              ? InfosLE
+              : InfosBE)[Kind - FirstTargetFixupKind];
+}
+
+void PPCAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &TargetVal,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
+  // In PPC64 ELFv1, .quad .TOC.@tocbase in the .opd section is expected to
+  // reference the null symbol.
+  auto Target = TargetVal;
+  if (Target.getSpecifier() == PPC::S_TOCBASE)
+    Target.setAddSym(nullptr);
+  if (IsResolved && shouldForceRelocation(Fixup, Target))
+    IsResolved = false;
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
+  MCFixupKind Kind = Fixup.getKind();
+  if (mc::isRelocation(Kind))
+    return;
+  Value = adjustFixupValue(Kind, Value);
+  if (!Value)
+    return; // Doesn't change encoding.
+
+  unsigned Offset = Fixup.getOffset();
+  unsigned NumBytes = getFixupKindNumBytes(Kind);
+
+  // For each byte of the fragment that the fixup touches, mask in the bits
+  // from the fixup value. The Value has been "split up" into the appropriate
+  // bitfields above.
+  for (unsigned i = 0; i != NumBytes; ++i) {
+    unsigned Idx = Endian == llvm::endianness::little ? i : (NumBytes - 1 - i);
+    Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff);
+  }
+}
 
 // FIXME: This should be in a separate file.
 namespace {
@@ -285,7 +282,7 @@ ELFPPCAsmBackend::getFixupKind(StringRef Name) const {
 std::optional<MCFixupKind>
 XCOFFPPCAsmBackend::getFixupKind(StringRef Name) const {
   return StringSwitch<std::optional<MCFixupKind>>(Name)
-      .Case("R_REF", (MCFixupKind)PPC::fixup_ppc_nofixup)
+      .Case("R_REF", PPC::fixup_ppc_nofixup)
       .Default(std::nullopt);
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index 8e885c3d86a0..53312e36fb9d 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/PPCFixupKinds.h"
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
index 78065541f0d0..ee99cfc7d655 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.cpp
@@ -18,8 +18,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "PPCELFStreamer.h"
+#include "PPCMCAsmInfo.h"
 #include "PPCMCCodeEmitter.h"
-#include "PPCMCExpr.h"
 #include "PPCMCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmBackend.h"
@@ -159,9 +159,7 @@ void PPCELFStreamer::emitGOTToPCRelReloc(const MCInst &Inst) {
   assert(DF && "Expecting a valid data fragment.");
   MCFixupKind FixupKind = static_cast<MCFixupKind>(FirstLiteralRelocationKind +
                                                    ELF::R_PPC64_PCREL_OPT);
-  DF->getFixups().push_back(
-      MCFixup::create(LabelSym->getOffset() - 8, SubExpr2,
-                      FixupKind, Inst.getLoc()));
+  DF->addFixup(MCFixup::create(LabelSym->getOffset() - 8, SubExpr2, FixupKind));
   emitLabel(CurrentLocation, Inst.getLoc());
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 2c232f450e1a..d1bc9e5b0c2d 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -13,7 +13,7 @@
 #include "MCTargetDesc/PPCInstPrinter.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "MCTargetDesc/PPCPredicates.h"
-#include "PPCMCExpr.h"
+#include "PPCMCAsmInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
index 971b592643dc..b2a1696fb927 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "PPCMCAsmInfo.h"
-#include "PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
@@ -20,7 +19,7 @@ using namespace llvm;
 
 void PPCELFMCAsmInfo::anchor() { }
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier elfAtSpecifiers[] = {
     {PPC::S_DTPREL, "DTPREL"},
     {PPC::S_GOT, "GOT"},
     {PPC::S_GOT_HA, "got@ha"},
@@ -35,7 +34,6 @@ const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
     {PPC::S_HIGHEST, "highest"},
     {PPC::S_HIGHESTA, "highesta"},
     {PPC::S_LO, "l"},
-    {PPC::S_L, "l"}, // FIXME: share the name with VK_LO
     {PPC::S_PCREL, "PCREL"},
     {PPC::S_PLT, "PLT"},
     {PPC::S_TLSGD, "tlsgd"},
@@ -96,9 +94,81 @@ const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
     {PPC::S_TPREL_HIGHEST, "tprel@highest"},
     {PPC::S_TPREL_HIGHESTA, "tprel@highesta"},
     {PPC::S_TPREL_LO, "tprel@l"},
+};
+
+const MCAsmInfo::AtSpecifier xcoffAtSpecifiers[] = {
+    // clang-format off
+    {PPC::S_AIX_TLSGD, "gd"},
+    {PPC::S_AIX_TLSGDM, "m"},
+    {PPC::S_AIX_TLSIE, "ie"},
+    {PPC::S_AIX_TLSLD, "ld"},
+    {PPC::S_AIX_TLSLE, "le"},
+    {PPC::S_AIX_TLSML, "ml"},
+    {PPC::S_L, "l"},
     {PPC::S_U, "u"},
+    // clang-format on
 };
 
+static std::optional<int64_t> evaluateAsInt64(uint16_t specifier,
+                                              int64_t Value) {
+  switch (specifier) {
+  case PPC::S_LO:
+    return Value & 0xffff;
+  case PPC::S_HI:
+    return (Value >> 16) & 0xffff;
+  case PPC::S_HA:
+    return ((Value + 0x8000) >> 16) & 0xffff;
+  case PPC::S_HIGH:
+    return (Value >> 16) & 0xffff;
+  case PPC::S_HIGHA:
+    return ((Value + 0x8000) >> 16) & 0xffff;
+  case PPC::S_HIGHER:
+    return (Value >> 32) & 0xffff;
+  case PPC::S_HIGHERA:
+    return ((Value + 0x8000) >> 32) & 0xffff;
+  case PPC::S_HIGHEST:
+    return (Value >> 48) & 0xffff;
+  case PPC::S_HIGHESTA:
+    return ((Value + 0x8000) >> 48) & 0xffff;
+  default:
+    return {};
+  }
+}
+
+bool PPC::evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res) {
+  MCValue Value;
+
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Value, nullptr))
+    return false;
+
+  if (!Value.isAbsolute())
+    return false;
+  auto Tmp = evaluateAsInt64(Expr.getSpecifier(), Value.getConstant());
+  if (!Tmp)
+    return false;
+  Res = *Tmp;
+  return true;
+}
+
+static bool evaluateAsRelocatable(const MCSpecifierExpr &Expr, MCValue &Res,
+                                  const MCAssembler *Asm) {
+  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
+    return false;
+
+  // The signedness of the result is dependent on the instruction operand. E.g.
+  // in addis 3,3,65535@l, 65535@l is signed. In the absence of information at
+  // parse time (!Asm), disable the folding.
+  std::optional<int64_t> MaybeInt =
+      evaluateAsInt64(Expr.getSpecifier(), Res.getConstant());
+  if (Res.isAbsolute() && MaybeInt) {
+    Res = MCValue::get(*MaybeInt);
+  } else {
+    Res.setSpecifier(Expr.getSpecifier());
+  }
+
+  return true;
+}
+
 PPCELFMCAsmInfo::PPCELFMCAsmInfo(bool is64Bit, const Triple& T) {
   // FIXME: This is not always needed. For example, it is not needed in the
   // v2 abi.
@@ -134,7 +204,7 @@ PPCELFMCAsmInfo::PPCELFMCAsmInfo(bool is64Bit, const Triple& T) {
   AssemblerDialect = 1;           // New-Style mnemonics.
   LCOMMDirectiveAlignmentType = LCOMM::ByteAlignment;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(elfAtSpecifiers);
 }
 
 void PPCELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
@@ -146,7 +216,7 @@ void PPCELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
 bool PPCELFMCAsmInfo::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr,
                                                 MCValue &Res,
                                                 const MCAssembler *Asm) const {
-  return PPC::evaluateAsRelocatableImpl(Expr, Res, Asm);
+  return evaluateAsRelocatable(Expr, Res, Asm);
 }
 
 void PPCXCOFFMCAsmInfo::anchor() {}
@@ -170,7 +240,7 @@ PPCXCOFFMCAsmInfo::PPCXCOFFMCAsmInfo(bool Is64Bit, const Triple &T) {
 
   UsesSetToEquateSymbol = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(xcoffAtSpecifiers);
 }
 
 void PPCXCOFFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
@@ -181,5 +251,5 @@ void PPCXCOFFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
 
 bool PPCXCOFFMCAsmInfo::evaluateAsRelocatableImpl(
     const MCSpecifierExpr &Expr, MCValue &Res, const MCAssembler *Asm) const {
-  return PPC::evaluateAsRelocatableImpl(Expr, Res, Asm);
+  return evaluateAsRelocatable(Expr, Res, Asm);
 }
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
index 172fe81c2bce..0f945b31d439 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
@@ -15,6 +15,8 @@
 
 #include "llvm/MC/MCAsmInfoELF.h"
 #include "llvm/MC/MCAsmInfoXCOFF.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 namespace llvm {
 class Triple;
@@ -123,8 +125,17 @@ enum Specifier {
   S_TPREL_LO,       // symbol@tprel@l
   S_U,              // symbol@u
 };
+
+bool evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res);
+}
+
+namespace PPCMCExpr {
+using Specifier = uint16_t;
 }
 
+static inline uint16_t getSpecifier(const MCSymbolRefExpr *SRE) {
+  return SRE->getKind();
+}
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index c395397c88ab..b7c5ae4e267b 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "PPCMCCodeEmitter.h"
 #include "MCTargetDesc/PPCFixupKinds.h"
-#include "PPCMCExpr.h"
+#include "PPCMCAsmInfo.h"
 #include "PPCMCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -39,6 +39,19 @@ MCCodeEmitter *llvm::createPPCMCCodeEmitter(const MCInstrInfo &MCII,
   return new PPCMCCodeEmitter(MCII, Ctx);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case PPC::fixup_ppc_br24:
+  case PPC::fixup_ppc_br24_notoc:
+  case PPC::fixup_ppc_brcond14:
+  case PPC::fixup_ppc_pcrel34:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 unsigned PPCMCCodeEmitter::
 getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
                     SmallVectorImpl<MCFixup> &Fixups,
@@ -49,10 +62,9 @@ getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (isNoTOCCallInstr(MI)
-                                        ? (MCFixupKind)PPC::fixup_ppc_br24_notoc
-                                        : (MCFixupKind)PPC::fixup_ppc_br24)));
+  addFixup(
+      Fixups, 0, MO.getExpr(),
+      (isNoTOCCallInstr(MI) ? PPC::fixup_ppc_br24_notoc : PPC::fixup_ppc_br24));
   return 0;
 }
 
@@ -157,8 +169,7 @@ unsigned PPCMCCodeEmitter::getCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_brcond14));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_brcond14);
   return 0;
 }
 
@@ -170,8 +181,7 @@ getAbsDirectBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_br24abs));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_br24abs);
   return 0;
 }
 
@@ -183,8 +193,7 @@ getAbsCondBrEncoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_brcond14abs));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_brcond14abs);
   return 0;
 }
 
@@ -205,8 +214,7 @@ unsigned PPCMCCodeEmitter::getImm16Encoding(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(), PPC::fixup_ppc_half16);
   return 0;
 }
 
@@ -220,7 +228,7 @@ uint64_t PPCMCCodeEmitter::getImm34Encoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI);
 
   // Add a fixup for the immediate field.
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Fixup));
+  addFixup(Fixups, 0, MO.getExpr(), Fixup);
   return 0;
 }
 
@@ -228,16 +236,14 @@ uint64_t
 PPCMCCodeEmitter::getImm34EncodingNoPCRel(const MCInst &MI, unsigned OpNo,
                                           SmallVectorImpl<MCFixup> &Fixups,
                                           const MCSubtargetInfo &STI) const {
-  return getImm34Encoding(MI, OpNo, Fixups, STI,
-                          (MCFixupKind)PPC::fixup_ppc_imm34);
+  return getImm34Encoding(MI, OpNo, Fixups, STI, PPC::fixup_ppc_imm34);
 }
 
 uint64_t
 PPCMCCodeEmitter::getImm34EncodingPCRel(const MCInst &MI, unsigned OpNo,
                                         SmallVectorImpl<MCFixup> &Fixups,
                                         const MCSubtargetInfo &STI) const {
-  return getImm34Encoding(MI, OpNo, Fixups, STI,
-                          (MCFixupKind)PPC::fixup_ppc_pcrel34);
+  return getImm34Encoding(MI, OpNo, Fixups, STI, PPC::fixup_ppc_pcrel34);
 }
 
 unsigned PPCMCCodeEmitter::getDispRIEncoding(const MCInst &MI, unsigned OpNo,
@@ -248,8 +254,7 @@ unsigned PPCMCCodeEmitter::getDispRIEncoding(const MCInst &MI, unsigned OpNo,
     return getMachineOpValue(MI, MO, Fixups, STI) & 0xFFFF;
 
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(), PPC::fixup_ppc_half16);
   return 0;
 }
 
@@ -262,8 +267,8 @@ PPCMCCodeEmitter::getDispRIXEncoding(const MCInst &MI, unsigned OpNo,
     return ((getMachineOpValue(MI, MO, Fixups, STI) >> 2) & 0x3FFF);
 
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16ds));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(),
+           PPC::fixup_ppc_half16ds);
   return 0;
 }
 
@@ -279,8 +284,8 @@ PPCMCCodeEmitter::getDispRIX16Encoding(const MCInst &MI, unsigned OpNo,
   }
 
   // Otherwise add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::create(IsLittleEndian ? 0 : 2, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_half16dq));
+  addFixup(Fixups, IsLittleEndian ? 0 : 2, MO.getExpr(),
+           PPC::fixup_ppc_half16dq);
   return 0;
 }
 
@@ -329,18 +334,13 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
     const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(Expr);
     (void)SRE;
     // Currently these are the only valid PCRelative Relocations.
-    assert((getSpecifier(SRE) == PPC::S_PCREL ||
-            getSpecifier(SRE) == PPC::S_GOT_PCREL ||
-            getSpecifier(SRE) == PPC::S_GOT_TLSGD_PCREL ||
-            getSpecifier(SRE) == PPC::S_GOT_TLSLD_PCREL ||
-            getSpecifier(SRE) == PPC::S_GOT_TPREL_PCREL) &&
-           "VariantKind must be VK_PCREL or VK_GOT_PCREL or "
-           "VK_GOT_TLSGD_PCREL or VK_GOT_TLSLD_PCREL or "
-           "VK_GOT_TPREL_PCREL.");
+    assert(is_contained({PPC::S_PCREL, PPC::S_GOT_PCREL, PPC::S_GOT_TLSGD_PCREL,
+                         PPC::S_GOT_TLSLD_PCREL, PPC::S_GOT_TPREL_PCREL},
+                        SRE->getSpecifier()) &&
+           "specifier must be S_PCREL, S_GOT_PCREL, S_GOT_TLSGD_PCREL, "
+           "S_GOT_TLSLD_PCREL, or S_GOT_TPREL_PCREL");
     // Generate the fixup for the relocation.
-    Fixups.push_back(
-        MCFixup::create(0, Expr,
-                        static_cast<MCFixupKind>(PPC::fixup_ppc_pcrel34)));
+    addFixup(Fixups, 0, Expr, PPC::fixup_ppc_pcrel34);
     // Put zero in the location of the immediate. The linker will fill in the
     // correct value based on the relocation.
     return 0;
@@ -372,9 +372,7 @@ PPCMCCodeEmitter::getDispRI34PCRelEncoding(const MCInst &MI, unsigned OpNo,
             getSpecifier(SRE) == PPC::S_GOT_PCREL) &&
            "VariantKind must be VK_PCREL or VK_GOT_PCREL");
     // Generate the fixup for the relocation.
-    Fixups.push_back(
-        MCFixup::create(0, Expr,
-                        static_cast<MCFixupKind>(PPC::fixup_ppc_pcrel34)));
+    addFixup(Fixups, 0, Expr, PPC::fixup_ppc_pcrel34);
     // Put zero in the location of the immediate. The linker will fill in the
     // correct value based on the relocation.
     return 0;
@@ -434,8 +432,7 @@ unsigned PPCMCCodeEmitter::getTLSRegEncoding(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   const MCSymbolRefExpr *SRE = cast<MCSymbolRefExpr>(Expr);
   bool IsPCRel = getSpecifier(SRE) == PPC::S_TLS_PCREL;
-  Fixups.push_back(MCFixup::create(IsPCRel ? 1 : 0, Expr,
-                                   (MCFixupKind)PPC::fixup_ppc_nofixup));
+  addFixup(Fixups, IsPCRel ? 1 : 0, Expr, PPC::fixup_ppc_nofixup);
   const Triple &TT = STI.getTargetTriple();
   bool isPPC64 = TT.isPPC64();
   return CTX.getRegisterInfo()->getEncodingValue(isPPC64 ? PPC::X13 : PPC::R2);
@@ -448,8 +445,7 @@ unsigned PPCMCCodeEmitter::getTLSCallEncoding(const MCInst &MI, unsigned OpNo,
   // (__tls_get_addr), which we create via getDirectBrEncoding as usual,
   // and one for the TLSGD or TLSLD symbol, which is emitted here.
   const MCOperand &MO = MI.getOperand(OpNo+1);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_nofixup));
+  addFixup(Fixups, 0, MO.getExpr(), PPC::fixup_ppc_nofixup);
   return getDirectBrEncoding(MI, OpNo, Fixups, STI);
 }
 
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
deleted file mode 100644
index 8d9c0892ae16..000000000000
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===-- PPCMCExpr.cpp - PPC specific MC expression classes ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "PPCMCExpr.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCObjectStreamer.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "ppcmcexpr"
-
-static std::optional<int64_t> evaluateAsInt64(uint16_t specifier,
-                                              int64_t Value) {
-  switch (specifier) {
-  case PPC::S_LO:
-    return Value & 0xffff;
-  case PPC::S_HI:
-    return (Value >> 16) & 0xffff;
-  case PPC::S_HA:
-    return ((Value + 0x8000) >> 16) & 0xffff;
-  case PPC::S_HIGH:
-    return (Value >> 16) & 0xffff;
-  case PPC::S_HIGHA:
-    return ((Value + 0x8000) >> 16) & 0xffff;
-  case PPC::S_HIGHER:
-    return (Value >> 32) & 0xffff;
-  case PPC::S_HIGHERA:
-    return ((Value + 0x8000) >> 32) & 0xffff;
-  case PPC::S_HIGHEST:
-    return (Value >> 48) & 0xffff;
-  case PPC::S_HIGHESTA:
-    return ((Value + 0x8000) >> 48) & 0xffff;
-  default:
-    return {};
-  }
-}
-
-bool PPC::evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res) {
-  MCValue Value;
-
-  if (!Expr.getSubExpr()->evaluateAsRelocatable(Value, nullptr))
-    return false;
-
-  if (!Value.isAbsolute())
-    return false;
-  auto Tmp = evaluateAsInt64(Expr.getSpecifier(), Value.getConstant());
-  if (!Tmp)
-    return false;
-  Res = *Tmp;
-  return true;
-}
-
-bool PPC::evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
-                                    const MCAssembler *Asm) {
-  if (!Expr.getSubExpr()->evaluateAsRelocatable(Res, Asm))
-    return false;
-
-  // The signedness of the result is dependent on the instruction operand. E.g.
-  // in addis 3,3,65535@l, 65535@l is signed. In the absence of information at
-  // parse time (!Asm), disable the folding.
-  std::optional<int64_t> MaybeInt =
-      evaluateAsInt64(Expr.getSpecifier(), Res.getConstant());
-  if (Res.isAbsolute() && MaybeInt) {
-    Res = MCValue::get(*MaybeInt);
-  } else {
-    Res.setSpecifier(Expr.getSpecifier());
-  }
-
-  return true;
-}
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
deleted file mode 100644
index d97a1204efbc..000000000000
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===-- PPCMCExpr.h - PPC specific MC expression classes --------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCEXPR_H
-#define LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCEXPR_H
-
-#include "PPCMCAsmInfo.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCValue.h"
-#include <optional>
-
-namespace llvm {
-
-namespace PPCMCExpr {
-using Specifier = uint16_t;
-}
-
-static inline PPCMCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
-  return PPCMCExpr::Specifier(SRE->getKind());
-}
-
-namespace PPC {
-bool evaluateAsConstant(const MCSpecifierExpr &Expr, int64_t &Res);
-bool evaluateAsRelocatableImpl(const MCSpecifierExpr &Expr, MCValue &Res,
-                               const MCAssembler *Asm);
-} // namespace PPC
-
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 96cb2928de4a..f65778566694 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -14,7 +14,7 @@
 #include "MCTargetDesc/PPCInstPrinter.h"
 #include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "PPCELFStreamer.h"
-#include "PPCMCExpr.h"
+#include "PPCMCAsmInfo.h"
 #include "PPCTargetStreamer.h"
 #include "PPCXCOFFStreamer.h"
 #include "TargetInfo/PowerPCTargetInfo.h"
@@ -399,10 +399,8 @@ class PPCTargetXCOFFStreamer : public PPCTargetStreamer {
     const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo();
     const unsigned PointerSize = MAI->getCodePointerSize();
     Streamer.emitValueToAlignment(Align(PointerSize));
-    Streamer.emitValue(
-        MCSymbolRefExpr::create(&S, MCSymbolRefExpr::VariantKind(Kind),
-                                Streamer.getContext()),
-        PointerSize);
+    Streamer.emitValue(MCSymbolRefExpr::create(&S, Kind, Streamer.getContext()),
+                       PointerSize);
   }
 
   void emitMachine(StringRef CPU) override {
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCTargetStreamer.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCTargetStreamer.h
index d1d3d9dde935..3da034d4909f 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCTargetStreamer.h
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCTargetStreamer.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIB_TARGET_POWERPC_PPCTARGETSTREAMER_H
 #define LLVM_LIB_TARGET_POWERPC_PPCTARGETSTREAMER_H
 
-#include "PPCMCExpr.h"
+#include "PPCMCAsmInfo.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
index 8532f537e2d6..f75ab62fe95c 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
@@ -9,7 +9,7 @@
 
 #include "MCTargetDesc/PPCFixupKinds.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
-#include "PPCMCExpr.h"
+#include "PPCMCAsmInfo.h"
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCValue.h"
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 954f14c9c7f3..1ee7f835e19e 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -16,7 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/PPCInstPrinter.h"
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "MCTargetDesc/PPCPredicates.h"
 #include "MCTargetDesc/PPCTargetStreamer.h"
@@ -144,12 +144,12 @@ enum {
 class PPCAsmPrinter : public AsmPrinter {
 protected:
   // For TLS on AIX, we need to be able to identify TOC entries of specific
-  // VariantKind so we can add the right relocations when we generate the
+  // specifier so we can add the right relocations when we generate the
   // entries. So each entry is represented by a pair of MCSymbol and
   // VariantKind. For example, we need to be able to identify the following
   // entry as a TLSGD entry so we can add the @m relocation:
   //   .tc .i[TC],i[TL]@m
-  // By default, VK_None is used for the VariantKind.
+  // By default, 0 is used for the specifier.
   MapVector<std::pair<const MCSymbol *, PPCMCExpr::Specifier>, MCSymbol *> TOC;
   const PPCSubtarget *Subtarget = nullptr;
 
@@ -732,8 +732,7 @@ void PPCAsmPrinter::emitTlsCall(const MachineInstr *MI,
   if (Subtarget->is32BitELFABI() && isPositionIndependent())
     Kind = PPC::S_PLT;
 
-  const MCExpr *TlsRef = MCSymbolRefExpr::create(
-      TlsGetAddr, MCSymbolRefExpr::VariantKind(Kind), OutContext);
+  const MCExpr *TlsRef = MCSymbolRefExpr::create(TlsGetAddr, Kind, OutContext);
 
   // Add 32768 offset to the symbol so we follow up the latest GOT/PLT ABI.
   if (Kind == PPC::S_PLT && Subtarget->isSecurePlt() &&
@@ -743,8 +742,7 @@ void PPCAsmPrinter::emitTlsCall(const MachineInstr *MI,
   const MachineOperand &MO = MI->getOperand(2);
   const GlobalValue *GValue = MO.getGlobal();
   MCSymbol *MOSymbol = getSymbol(GValue);
-  const MCExpr *SymVar = MCSymbolRefExpr::create(
-      MOSymbol, MCSymbolRefExpr::VariantKind(VK), OutContext);
+  const MCExpr *SymVar = MCSymbolRefExpr::create(MOSymbol, VK, OutContext);
   EmitToStreamer(*OutStreamer,
                  MCInstBuilder(Subtarget->isPPC64() ? Opcode
                                                     : (unsigned)PPC::BL_TLS)
@@ -801,8 +799,7 @@ getTOCEntryTypeForMO(const MachineOperand &MO) {
 
 const MCExpr *PPCAsmPrinter::symbolWithSpecifier(const MCSymbol *S,
                                                  PPCMCExpr::Specifier Spec) {
-  return MCSymbolRefExpr::create(S, MCSymbolRefExpr::VariantKind(Spec),
-                                 OutContext);
+  return MCSymbolRefExpr::create(S, Spec, OutContext);
 }
 
 /// EmitInstruction -- Print out a single PowerPC MI in Darwin syntax to
@@ -954,8 +951,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     MCSymbol *GOTSymbol =
       OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_"));
     const MCExpr *OffsExpr = MCBinaryExpr::createSub(
-        MCSymbolRefExpr::create(
-            GOTSymbol, MCSymbolRefExpr::VariantKind(PPC::S_LOCAL), OutContext),
+        MCSymbolRefExpr::create(GOTSymbol, PPC::S_LOCAL, OutContext),
         MCConstantExpr::create(4, OutContext), OutContext);
 
     // Emit the 'bl'.
@@ -1331,9 +1327,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     const MCSymbol *const MOSymbol = getMCSymbolForTOCPseudoMO(MO, *this);
 
     const MCExpr *Exp = MCSymbolRefExpr::create(
-        MOSymbol,
-        MCSymbolRefExpr::VariantKind(IsAIX ? PPC::S_L : PPC::S_TOC_LO),
-        OutContext);
+        MOSymbol, IsAIX ? PPC::S_L : PPC::S_TOC_LO, OutContext);
 
     TmpInst.getOperand(2) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
@@ -1728,8 +1722,7 @@ PPCAsmPrinter::getAdjustedFasterLocalExpr(const MachineOperand &MO,
   // assume that the address of extern TLS variables are zero.
   const MCExpr *Expr = MCSymbolRefExpr::create(
       getSymbol(GValue),
-      MCSymbolRefExpr::VariantKind(
-          Model == TLSModel::LocalExec ? PPC::S_AIX_TLSLE : PPC::S_AIX_TLSLD),
+      (Model == TLSModel::LocalExec ? PPC::S_AIX_TLSLE : PPC::S_AIX_TLSLD),
       OutContext);
   Expr = MCBinaryExpr::createAdd(
       Expr, MCConstantExpr::create(Offset, OutContext), OutContext);
@@ -2017,9 +2010,7 @@ void PPCLinuxAsmPrinter::emitFunctionEntryLabel() {
   MCSymbol *Symbol2 = OutContext.getOrCreateSymbol(StringRef(".TOC."));
   // Generates a R_PPC64_TOC relocation for TOC base insertion.
   OutStreamer->emitValue(
-      MCSymbolRefExpr::create(
-          Symbol2, MCSymbolRefExpr::VariantKind(PPC::S_TOCBASE), OutContext),
-      8 /*size*/);
+      MCSymbolRefExpr::create(Symbol2, PPC::S_TOCBASE, OutContext), 8 /*size*/);
   // Emit a null environment pointer.
   OutStreamer->emitIntValue(0, 8 /* size */);
   OutStreamer->switchSection(Current.first, Current.second);
diff --git a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
index e6e8a881e610..6fe7688224d3 100644
--- a/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBoolRetToInt.cpp
@@ -45,7 +45,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/User.h"
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 4ce1507b2325..6a1d3ef927d5 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -2170,8 +2170,8 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
   // The Floating-point register save area is right below the back chain word
   // of the previous stack frame.
   if (HasFPSaveArea) {
-    for (unsigned i = 0, e = FPRegs.size(); i != e; ++i) {
-      int FI = FPRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &FPReg : FPRegs) {
+      int FI = FPReg.getFrameIdx();
 
       MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
     }
@@ -2219,18 +2219,18 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
   if (HasGPSaveArea || HasG8SaveArea) {
     // Move general register save area spill slots down, taking into account
     // the size of the Floating-point register save area.
-    for (unsigned i = 0, e = GPRegs.size(); i != e; ++i) {
-      if (!GPRegs[i].isSpilledToReg()) {
-        int FI = GPRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &GPReg : GPRegs) {
+      if (!GPReg.isSpilledToReg()) {
+        int FI = GPReg.getFrameIdx();
         MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
       }
     }
 
     // Move general register save area spill slots down, taking into account
     // the size of the Floating-point register save area.
-    for (unsigned i = 0, e = G8Regs.size(); i != e; ++i) {
-      if (!G8Regs[i].isSpilledToReg()) {
-        int FI = G8Regs[i].getFrameIdx();
+    for (const CalleeSavedInfo &G8Reg : G8Regs) {
+      if (!G8Reg.isSpilledToReg()) {
+        int FI = G8Reg.getFrameIdx();
         MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
       }
     }
@@ -2272,8 +2272,8 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
     assert(LowerBound <= 0 && "Expect LowerBound have a non-positive value!");
     LowerBound &= ~(15);
 
-    for (unsigned i = 0, e = VRegs.size(); i != e; ++i) {
-      int FI = VRegs[i].getFrameIdx();
+    for (const CalleeSavedInfo &VReg : VRegs) {
+      int FI = VReg.getFrameIdx();
 
       MFI.setObjectOffset(FI, LowerBound + MFI.getObjectOffset(FI));
     }
diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index f921032356d6..415164fc9e2c 100644
--- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -4961,6 +4961,21 @@ bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) {
   // If this is just a masked value where the input is not handled, and
   // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm
   if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) {
+    // The result of LBARX/LHARX do not need to be cleared as the instructions
+    // implicitly clear the upper bits.
+    unsigned AlreadyCleared = 0;
+    if (Val.getOpcode() == ISD::INTRINSIC_W_CHAIN) {
+      auto IntrinsicID = Val.getConstantOperandVal(1);
+      if (IntrinsicID == Intrinsic::ppc_lbarx)
+        AlreadyCleared = 24;
+      else if (IntrinsicID == Intrinsic::ppc_lharx)
+        AlreadyCleared = 16;
+      if (AlreadyCleared != 0 && AlreadyCleared == MB && ME == 31) {
+        ReplaceUses(SDValue(N, 0), N->getOperand(0));
+        return true;
+      }
+    }
+
     SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl),
                      getI32Imm(ME, dl)};
     CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 88c6fe632d26..3616b86a7fd3 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -4445,11 +4445,11 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
     // The fixed integer arguments of a variadic function are stored to the
     // VarArgsFrameIndex on the stack so that they may be loaded by
     // dereferencing the result of va_next.
-    for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) {
+    for (MCPhysReg GPArgReg : GPArgRegs) {
       // Get an existing live-in vreg, or add a new one.
-      Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]);
+      Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgReg);
       if (!VReg)
-        VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass);
+        VReg = MF.addLiveIn(GPArgReg, &PPC::GPRCRegClass);
 
       SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
       SDValue Store =
@@ -4549,13 +4549,13 @@ SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
   unsigned NumBytes = LinkageSize;
   unsigned AvailableFPRs = Num_FPR_Regs;
   unsigned AvailableVRs = Num_VR_Regs;
-  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
-    if (Ins[i].Flags.isNest())
+  for (const ISD::InputArg &In : Ins) {
+    if (In.Flags.isNest())
       continue;
 
-    if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags,
-                               PtrByteSize, LinkageSize, ParamAreaSize,
-                               NumBytes, AvailableFPRs, AvailableVRs))
+    if (CalculateStackSlotUsed(In.VT, In.ArgVT, In.Flags, PtrByteSize,
+                               LinkageSize, ParamAreaSize, NumBytes,
+                               AvailableFPRs, AvailableVRs))
       HasParameterArea = true;
   }
 
@@ -5766,9 +5766,8 @@ buildCallOperands(SmallVectorImpl<SDValue> &Ops,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
   // no way to mark dependencies as implicit here.
@@ -6191,9 +6190,8 @@ SDValue PPCTargetLowering::LowerCall_32SVR4(
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -6805,9 +6803,8 @@ SDValue PPCTargetLowering::LowerCall_64SVR4(
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and flag operands which copy the outgoing args into the appropriate regs.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -9587,6 +9584,36 @@ static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
   return false;
 }
 
+bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) {
+  assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
+
+  BitMask.clearAllBits();
+  EVT VT = BVN.getValueType(0);
+  APInt ConstValue(VT.getSizeInBits(), 0);
+
+  unsigned EltWidth = VT.getScalarSizeInBits();
+
+  unsigned BitPos = 0;
+  for (auto OpVal : BVN.op_values()) {
+    auto *CN = dyn_cast<ConstantSDNode>(OpVal);
+
+    if (!CN)
+      return false;
+
+    ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
+    BitPos += EltWidth;
+  }
+
+  for (unsigned J = 0; J < 16; ++J) {
+    APInt ExtractValue = ConstValue.extractBits(8, J * 8);
+    if (ExtractValue != 0x00 && ExtractValue != 0xFF)
+      return false;
+    if (ExtractValue == 0xFF)
+      BitMask.setBit(J);
+  }
+  return true;
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
@@ -9598,6 +9625,25 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
   assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
 
+  if (Subtarget.hasP10Vector()) {
+    APInt BitMask(32, 0);
+    // If the value of the vector is all zeros or all ones,
+    // we do not convert it to MTVSRBMI.
+    // The xxleqv instruction sets a vector with all ones.
+    // The xxlxor instruction sets a vector with all zeros.
+    if (isValidMtVsrBmi(BitMask, *BVN) && BitMask != 0 && BitMask != 0xffff) {
+      SDValue SDConstant = DAG.getTargetConstant(BitMask, dl, MVT::i32);
+      MachineSDNode *MSDNode =
+          DAG.getMachineNode(PPC::MTVSRBMI, dl, MVT::v16i8, SDConstant);
+      SDValue SDV = SDValue(MSDNode, 0);
+      EVT DVT = BVN->getValueType(0);
+      EVT SVT = SDV.getValueType();
+      if (SVT != DVT) {
+        SDV = DAG.getNode(ISD::BITCAST, dl, DVT, SDV);
+      }
+      return SDV;
+    }
+  }
   // Check if this is a splat of a constant value.
   APInt APSplatBits, APSplatUndef;
   unsigned SplatBitSize;
@@ -17915,9 +17961,6 @@ SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDLoc dl(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
index 386c94a32499..24287a95ecb0 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -1617,10 +1617,14 @@ class VX_VT5_EO5_VB5_XO9_o<bits<5> eo, bits<9> xo, string opc,
 }
 
 // Decimal Convert From/to National/Zoned/Signed-QWord
-def BCDCFN_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<7, 385, "bcdcfn." , []>;
-def BCDCFZ_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<6, 385, "bcdcfz." , []>;
-def BCDCTN_rec  : VX_VT5_EO5_VB5_XO9_o    <5, 385, "bcdctn." , []>;
-def BCDCTZ_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<4, 385, "bcdctz." , []>;
+def BCDCFN_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<7, 385, "bcdcfn." ,
+                  [(set v16i8:$VD, (int_ppc_national2packed v16i8:$VB, timm:$PS))]>;
+def BCDCFZ_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<6, 385, "bcdcfz." , 
+                  [(set v16i8:$VD, (int_ppc_zoned2packed v16i8:$VB, timm:$PS))]>;
+def BCDCTN_rec  : VX_VT5_EO5_VB5_XO9_o    <5, 385, "bcdctn." , 
+                  [(set v16i8:$VD, (int_ppc_packed2national v16i8:$VB))]>;
+def BCDCTZ_rec  : VX_VT5_EO5_VB5_PS1_XO9_o<4, 385, "bcdctz." , 
+                  [(set v16i8:$VD, (int_ppc_packed2zoned v16i8:$VB, timm:$PS))]>;
 def BCDCFSQ_rec : VX_VT5_EO5_VB5_PS1_XO9_o<2, 385, "bcdcfsq.", []>;
 def BCDCTSQ_rec : VX_VT5_EO5_VB5_XO9_o    <0, 385, "bcdctsq.", []>;
 
diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
index cbd53651bbbf..75440a0b779a 100644
--- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/PPCMCExpr.h"
+#include "MCTargetDesc/PPCMCAsmInfo.h"
 #include "PPC.h"
 #include "PPCMachineFunctionInfo.h"
 #include "PPCSubtarget.h"
@@ -136,8 +136,7 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
       RefKind = PPC::S_PCREL_OPT;
   }
 
-  const MCExpr *Expr = MCSymbolRefExpr::create(
-      Symbol, MCSymbolRefExpr::VariantKind(RefKind), Ctx);
+  const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, RefKind, Ctx);
   // If -msecure-plt -fPIC, add 32768 to symbol.
   if (Subtarget->isSecurePlt() && TM.isPositionIndependent() &&
       M->getPICLevel() == PICLevel::BigPIC &&
diff --git a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
index ec20ad2565e6..0ffd35dfa279 100644
--- a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
+++ b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
@@ -108,6 +108,8 @@ struct BlockSplitInfo {
   MachineInstr *OrigBranch;
   MachineInstr *SplitBefore;
   MachineInstr *SplitCond;
+  unsigned OrigSubreg;
+  unsigned SplitCondSubreg;
   bool InvertNewBranch;
   bool InvertOrigBranch;
   bool BranchToFallThrough;
@@ -220,7 +222,7 @@ static bool splitMBB(BlockSplitInfo &BSI) {
   // Add the branches to ThisMBB.
   BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
           TII->get(NewBROpcode))
-      .addReg(BSI.SplitCond->getOperand(0).getReg())
+      .addReg(BSI.SplitCond->getOperand(0).getReg(), 0, BSI.SplitCondSubreg)
       .addMBB(NewBRTarget);
   BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
           TII->get(PPC::B))
@@ -234,6 +236,7 @@ static bool splitMBB(BlockSplitInfo &BSI) {
     assert(FirstTerminator->getOperand(0).isReg() &&
            "Can't update condition of unconditional branch.");
     FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg());
+    FirstTerminator->getOperand(0).setSubReg(BSI.OrigSubreg);
   }
   if (BSI.InvertOrigBranch)
     FirstTerminator->setDesc(TII->get(InvertedOpcode));
@@ -471,6 +474,7 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
   } else {
     MachineInstr *Def1 = lookThroughCRCopy(MIParam.getOperand(1).getReg(),
                                            Ret.SubregDef1, Ret.CopyDefs.first);
+    Ret.SubregDef1 = MIParam.getOperand(1).getSubReg();
     assert(Def1 && "Must be able to find a definition of operand 1.");
     Ret.DefsSingleUse &=
       MRI->hasOneNonDBGUse(Def1->getOperand(0).getReg());
@@ -481,6 +485,7 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
       MachineInstr *Def2 = lookThroughCRCopy(MIParam.getOperand(2).getReg(),
                                              Ret.SubregDef2,
                                              Ret.CopyDefs.second);
+      Ret.SubregDef2 = MIParam.getOperand(2).getSubReg();
       assert(Def2 && "Must be able to find a definition of operand 2.");
       Ret.DefsSingleUse &=
         MRI->hasOneNonDBGUse(Def2->getOperand(0).getReg());
@@ -535,7 +540,6 @@ PPCReduceCRLogicals::createCRLogicalOpInfo(MachineInstr &MIParam) {
 MachineInstr *PPCReduceCRLogicals::lookThroughCRCopy(unsigned Reg,
                                                      unsigned &Subreg,
                                                      MachineInstr *&CpDef) {
-  Subreg = -1;
   if (!Register::isVirtualRegister(Reg))
     return nullptr;
   MachineInstr *Copy = MRI->getVRegDef(Reg);
@@ -543,18 +547,8 @@ MachineInstr *PPCReduceCRLogicals::lookThroughCRCopy(unsigned Reg,
   if (!Copy->isCopy())
     return Copy;
   Register CopySrc = Copy->getOperand(1).getReg();
-  Subreg = Copy->getOperand(1).getSubReg();
   if (!CopySrc.isVirtual()) {
     const TargetRegisterInfo *TRI = &TII->getRegisterInfo();
-    // Set the Subreg
-    if (CopySrc == PPC::CR0EQ || CopySrc == PPC::CR6EQ)
-      Subreg = PPC::sub_eq;
-    if (CopySrc == PPC::CR0LT || CopySrc == PPC::CR6LT)
-      Subreg = PPC::sub_lt;
-    if (CopySrc == PPC::CR0GT || CopySrc == PPC::CR6GT)
-      Subreg = PPC::sub_gt;
-    if (CopySrc == PPC::CR0UN || CopySrc == PPC::CR6UN)
-      Subreg = PPC::sub_un;
     // Loop backwards and return the first MI that modifies the physical CR Reg.
     MachineBasicBlock::iterator Me = Copy, B = Copy->getParent()->begin();
     while (Me != B)
@@ -682,16 +676,21 @@ bool PPCReduceCRLogicals::splitBlockOnBinaryCROp(CRLogicalOpInfo &CRI) {
   computeBranchTargetAndInversion(Opc, Branch->getOpcode(), UsingDef1,
                                   InvertNewBranch, InvertOrigBranch,
                                   TargetIsFallThrough);
-  MachineInstr *SplitCond =
-    UsingDef1 ? CRI.CopyDefs.second : CRI.CopyDefs.first;
+  MachineInstr *NewCond = CRI.CopyDefs.first;
+  MachineInstr *SplitCond = CRI.CopyDefs.second;
+  if (!UsingDef1) {
+    std::swap(NewCond, SplitCond);
+    std::swap(CRI.SubregDef1, CRI.SubregDef2);
+  }
   LLVM_DEBUG(dbgs() << "We will " << (InvertNewBranch ? "invert" : "copy"));
   LLVM_DEBUG(dbgs() << " the original branch and the target is the "
                     << (TargetIsFallThrough ? "fallthrough block\n"
                                             : "orig. target block\n"));
   LLVM_DEBUG(dbgs() << "Original branch instruction: "; Branch->dump());
-  BlockSplitInfo BSI { Branch, SplitBefore, SplitCond, InvertNewBranch,
-    InvertOrigBranch, TargetIsFallThrough, MBPI, CRI.MI,
-    UsingDef1 ? CRI.CopyDefs.first : CRI.CopyDefs.second };
+  BlockSplitInfo BSI{
+      Branch,         SplitBefore,     SplitCond,        CRI.SubregDef1,
+      CRI.SubregDef2, InvertNewBranch, InvertOrigBranch, TargetIsFallThrough,
+      MBPI,           CRI.MI,          NewCond};
   bool Changed = splitMBB(BSI);
   // If we've split on a CR logical that is fed by a CR logical,
   // recompute the source CR logical as it may be usable for splitting.
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index e5d8ab07891a..66f4aade380f 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -732,6 +732,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
   bool isUImm6() const { return isUImm<6>(); }
   bool isUImm7() const { return isUImm<7>(); }
   bool isUImm8() const { return isUImm<8>(); }
+  bool isUImm9() const { return isUImm<9>(); }
   bool isUImm10() const { return isUImm<10>(); }
   bool isUImm11() const { return isUImm<11>(); }
   bool isUImm16() const { return isUImm<16>(); }
@@ -807,6 +808,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
   bool isSImm5() const { return isSImm<5>(); }
   bool isSImm6() const { return isSImm<6>(); }
+  bool isSImm10() const { return isSImm<10>(); }
   bool isSImm11() const { return isSImm<11>(); }
   bool isSImm16() const { return isSImm<16>(); }
   bool isSImm26() const { return isSImm<26>(); }
@@ -901,6 +903,8 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            VK == RISCV::S_QC_ABS20;
   }
 
+  bool isSImm10Unsigned() const { return isSImm<10>() || isUImm<10>(); }
+
   bool isUImm20LUI() const {
     if (!isImm())
       return false;
@@ -1016,7 +1020,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     return Fence.Val;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     auto RegName = [](MCRegister Reg) {
       if (Reg)
         return RISCVInstPrinter::getRegisterName(Reg);
@@ -1026,8 +1030,9 @@ struct RISCVOperand final : public MCParsedAsmOperand {
 
     switch (Kind) {
     case KindTy::Immediate:
-      OS << "<imm: " << *Imm.Val << " " << (Imm.IsRV64 ? "rv64" : "rv32")
-         << ">";
+      OS << "<imm: ";
+      MAI.printExpr(OS, *Imm.Val);
+      OS << ' ' << (Imm.IsRV64 ? "rv64" : "rv32") << '>';
       break;
     case KindTy::FPImmediate:
       OS << "<fpimm: " << FPImm.Val << ">";
@@ -1523,6 +1528,9 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 0, (1 << 8) - 8,
         "immediate must be a multiple of 8 bytes in the range");
+  case Match_InvalidUImm9:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 9) - 1,
+                                      "immediate offset must be in the range");
   case Match_InvalidBareSImm9Lsb0:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, -(1 << 8), (1 << 8) - 2,
@@ -1531,6 +1539,12 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 0, (1 << 9) - 8,
         "immediate must be a multiple of 8 bytes in the range");
+  case Match_InvalidSImm10:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 9),
+                                      (1 << 9) - 1);
+  case Match_InvalidSImm10Unsigned:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 9),
+                                      (1 << 10) - 1);
   case Match_InvalidUImm10Lsb00NonZero:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 4, (1 << 10) - 4,
@@ -2078,12 +2092,13 @@ ParseStatus RISCVAsmParser::parseOperandWithSpecifier(OperandVector &Operands) {
 }
 
 bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
+  SMLoc Loc = getLoc();
   if (getLexer().getKind() != AsmToken::Identifier)
-    return Error(getLoc(), "expected '%' relocation specifier");
+    return TokError("expected '%' relocation specifier");
   StringRef Identifier = getParser().getTok().getIdentifier();
   auto Spec = RISCV::parseSpecifierName(Identifier);
   if (!Spec)
-    return Error(getLoc(), "invalid relocation specifier");
+    return TokError("invalid relocation specifier");
 
   getParser().Lex(); // Eat the identifier
   if (parseToken(AsmToken::LParen, "expected '('"))
@@ -2093,7 +2108,7 @@ bool RISCVAsmParser::parseExprWithSpecifier(const MCExpr *&Res, SMLoc &E) {
   if (getParser().parseParenExpression(SubExpr, E))
     return true;
 
-  Res = MCSpecifierExpr::create(SubExpr, Spec, getContext());
+  Res = MCSpecifierExpr::create(SubExpr, Spec, getContext(), Loc);
   return false;
 }
 
@@ -3749,9 +3764,9 @@ bool RISCVAsmParser::validateInstruction(MCInst &Inst,
   if (!(MCID.TSFlags & RISCVII::ConstraintMask))
     return false;
 
-  if (Opcode == RISCV::VC_V_XVW || Opcode == RISCV::VC_V_IVW ||
-      Opcode == RISCV::VC_V_FVW || Opcode == RISCV::VC_V_VVW) {
-    // Operands Opcode, Dst, uimm, Dst, Rs2, Rs1 for VC_V_XVW.
+  if (Opcode == RISCV::SF_VC_V_XVW || Opcode == RISCV::SF_VC_V_IVW ||
+      Opcode == RISCV::SF_VC_V_FVW || Opcode == RISCV::SF_VC_V_VVW) {
+    // Operands Opcode, Dst, uimm, Dst, Rs2, Rs1 for SF_VC_V_XVW.
     MCRegister VCIXDst = Inst.getOperand(0).getReg();
     SMLoc VCIXDstLoc = Operands[2]->getStartLoc();
     if (MCID.TSFlags & RISCVII::VS1Constraint) {
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 27e04c0cb1f8..b723958a6ff2 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -775,7 +775,8 @@ static constexpr FeatureBitset XTHeadGroup = {
 
 static constexpr FeatureBitset XAndesGroup = {
     RISCV::FeatureVendorXAndesPerf, RISCV::FeatureVendorXAndesVBFHCvt,
-    RISCV::FeatureVendorXAndesVPackFPH, RISCV::FeatureVendorXAndesVDot};
+    RISCV::FeatureVendorXAndesVSIntLoad, RISCV::FeatureVendorXAndesVPackFPH,
+    RISCV::FeatureVendorXAndesVDot};
 
 static constexpr DecoderListEntry DecoderList32[]{
     // Vendor Extensions
@@ -790,6 +791,9 @@ static constexpr DecoderListEntry DecoderList32[]{
     {DecoderTableXmipscmov32,
      {RISCV::FeatureVendorXMIPSCMov},
      "MIPS mips.ccmov"},
+    {DecoderTableXmipscbop32,
+     {RISCV::FeatureVendorXMIPSCBOP},
+     "MIPS mips.pref"},
     {DecoderTableXAndes32, XAndesGroup, "Andes extensions"},
     // Standard Extensions
     {DecoderTableXCV32, XCVFeatureGroup, "CORE-V extensions"},
@@ -811,7 +815,9 @@ DecodeStatus RISCVDisassembler::getInstruction32(MCInst &MI, uint64_t &Size,
   }
   Size = 4;
 
-  uint32_t Insn = support::endian::read32le(Bytes.data());
+  // Use uint64_t to match getInstruction48. decodeInstruction is templated
+  // on the Insn type.
+  uint64_t Insn = support::endian::read32le(Bytes.data());
 
   for (const DecoderListEntry &Entry : DecoderList32) {
     if (!Entry.haveContainedFeatures(STI.getFeatureBits()))
@@ -857,7 +863,9 @@ DecodeStatus RISCVDisassembler::getInstruction16(MCInst &MI, uint64_t &Size,
   }
   Size = 2;
 
-  uint32_t Insn = support::endian::read16le(Bytes.data());
+  // Use uint64_t to match getInstruction48. decodeInstruction is templated
+  // on the Insn type.
+  uint64_t Insn = support::endian::read16le(Bytes.data());
 
   for (const DecoderListEntry &Entry : DecoderList16) {
     if (!Entry.haveContainedFeatures(STI.getFeatureBits()))
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index a6c2d2fc2911..f88e4d8fc8c7 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -14,7 +14,6 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
@@ -70,25 +69,23 @@ MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_riscv_lo12_i", 20, 12, 0},
       {"fixup_riscv_12_i", 20, 12, 0},
       {"fixup_riscv_lo12_s", 0, 32, 0},
-      {"fixup_riscv_pcrel_hi20", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_pcrel_lo12_i", 20, 12,
-       MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_pcrel_lo12_s", 0, 32,
-       MCFixupKindInfo::FKF_IsPCRel | MCFixupKindInfo::FKF_IsTarget},
-      {"fixup_riscv_jal", 12, 20, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_branch", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_rvc_jump", 2, 11, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_rvc_branch", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_call", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_riscv_call_plt", 0, 64, MCFixupKindInfo::FKF_IsPCRel},
-
-      {"fixup_riscv_qc_e_branch", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_pcrel_hi20", 12, 20, 0},
+      {"fixup_riscv_pcrel_lo12_i", 20, 12, 0},
+      {"fixup_riscv_pcrel_lo12_s", 0, 32, 0},
+      {"fixup_riscv_jal", 12, 20, 0},
+      {"fixup_riscv_branch", 0, 32, 0},
+      {"fixup_riscv_rvc_jump", 2, 11, 0},
+      {"fixup_riscv_rvc_branch", 0, 16, 0},
+      {"fixup_riscv_call", 0, 64, 0},
+      {"fixup_riscv_call_plt", 0, 64, 0},
+
+      {"fixup_riscv_qc_e_branch", 0, 48, 0},
       {"fixup_riscv_qc_e_32", 16, 32, 0},
       {"fixup_riscv_qc_abs20_u", 12, 20, 0},
-      {"fixup_riscv_qc_e_call_plt", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_qc_e_call_plt", 0, 48, 0},
 
       // Andes fixups
-      {"fixup_riscv_nds_branch_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"fixup_riscv_nds_branch_10", 0, 32, 0},
   };
   static_assert((std::size(Infos)) == RISCV::NumTargetFixupKinds,
                 "Not all fixup kinds added to Infos array");
@@ -96,7 +93,7 @@ MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from raw relocation types and .reloc directives force
   // relocations and do not use these fields.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -275,17 +272,16 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 
   int64_t LineDelta = DF.getLineDelta();
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 1> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   [[maybe_unused]] bool IsAbsolute =
       AddrDelta.evaluateKnownAbsolute(Value, *Asm);
   assert(IsAbsolute && "CFA with invalid expression");
 
-  Data.clear();
   Fixups.clear();
+  SmallVector<char> Data;
   raw_svector_ostream OS(Data);
 
   // INT64_MAX is a signal that this is actually a DW_LNE_end_sequence.
@@ -330,6 +326,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
     OS << uint8_t(dwarf::DW_LNS_copy);
   }
 
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
   WasRelaxed = OldSize != Data.size();
   return true;
 }
@@ -337,9 +335,8 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCDwarfLineAddrFragment &DF,
 bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
                                     bool &WasRelaxed) const {
   const MCExpr &AddrDelta = DF.getAddrDelta();
-  SmallVectorImpl<char> &Data = DF.getContents();
-  SmallVectorImpl<MCFixup> &Fixups = DF.getFixups();
-  size_t OldSize = Data.size();
+  SmallVector<MCFixup, 2> Fixups;
+  size_t OldSize = DF.getContents().size();
 
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
@@ -348,14 +345,12 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
       AddrDelta.evaluateKnownAbsolute(Value, *Asm);
   assert(IsAbsolute && "CFA with invalid expression");
 
-  Data.clear();
-  Fixups.clear();
-  raw_svector_ostream OS(Data);
-
   assert(getContext().getAsmInfo()->getMinInstAlignment() == 1 &&
          "expected 1-byte alignment");
   if (Value == 0) {
-    WasRelaxed = OldSize != Data.size();
+    DF.clearContents();
+    DF.clearFixups();
+    WasRelaxed = OldSize != DF.getContents().size();
     return true;
   }
 
@@ -366,6 +361,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
     Fixups.push_back(MCFixup::create(Offset, MBE.getRHS(), std::get<1>(Fixup)));
   };
 
+  SmallVector<char, 8> Data;
+  raw_svector_ostream OS(Data);
   if (isUIntN(6, Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc);
     AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6});
@@ -384,6 +381,8 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCDwarfCallFrameFragment &DF,
   } else {
     llvm_unreachable("unsupported CFA encoding");
   }
+  DF.setContents(Data);
+  DF.setFixups(Fixups);
 
   WasRelaxed = OldSize != Data.size();
   return true;
@@ -395,8 +394,7 @@ std::pair<bool, bool> RISCVAsmBackend::relaxLEB128(MCLEBFragment &LF,
     return std::make_pair(false, false);
   const MCExpr &Expr = LF.getValue();
   if (ULEB128Reloc) {
-    LF.getFixups().push_back(
-        MCFixup::create(0, &Expr, FK_Data_leb128, Expr.getLoc()));
+    LF.addFixup(MCFixup::create(0, &Expr, FK_Data_leb128));
   }
   return std::make_pair(Expr.evaluateKnownAbsolute(Value, *Asm), false);
 }
@@ -657,15 +655,17 @@ static const MCFixup *getPCRelHiFixup(const MCSpecifierExpr &Expr,
   return nullptr;
 }
 
-bool RISCVAsmBackend::evaluateTargetFixup(const MCFixup &Fixup,
-                                          const MCValue &Target,
-                                          uint64_t &Value) {
+std::optional<bool> RISCVAsmBackend::evaluateFixup(const MCFragment &,
+                                                   MCFixup &Fixup,
+                                                   MCValue &Target,
+                                                   uint64_t &Value) {
   const MCFixup *AUIPCFixup;
   const MCFragment *AUIPCDF;
   MCValue AUIPCTarget;
   switch (Fixup.getTargetKind()) {
   default:
-    llvm_unreachable("Unexpected fixup kind!");
+    // Use default handling for `Value` and `IsResolved`.
+    return {};
   case RISCV::fixup_riscv_pcrel_lo12_i:
   case RISCV::fixup_riscv_pcrel_lo12_s: {
     AUIPCFixup =
@@ -794,8 +794,7 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
   // generate a relocation and then append a RELAX.
   if (Fixup.isLinkerRelaxable())
     IsResolved = false;
-  if (IsResolved &&
-      (getFixupKindInfo(Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel))
+  if (IsResolved && Fixup.isPCRel())
     IsResolved = isPCRelFixupResolved(Target.getAddSym(), F);
 
   if (!IsResolved) {
@@ -815,10 +814,11 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
   return false;
 }
 
-void RISCVAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void RISCVAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
                                  bool IsResolved) {
+  IsResolved = addReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -885,8 +885,7 @@ bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
 
   MCContext &Ctx = getContext();
   const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
-  // Create fixup_riscv_align fixup.
-  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_RISCV_ALIGN, SMLoc());
+  MCFixup Fixup = MCFixup::create(0, Dummy, ELF::R_RISCV_ALIGN);
 
   uint64_t FixedValue = 0;
   MCValue NopBytes = MCValue::get(Count);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 91efd4454750..305240af88ec 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -14,7 +14,6 @@
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/MC/MCAsmBackend.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 
 namespace llvm {
@@ -47,11 +46,10 @@ class RISCVAsmBackend : public MCAsmBackend {
   bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
                                      MCAlignFragment &AF) override;
 
-  bool evaluateTargetFixup(const MCFixup &Fixup, const MCValue &Target,
-                           uint64_t &Value) override;
-
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   bool addReloc(const MCFragment &, const MCFixup &, const MCValue &,
-                uint64_t &FixedValue, bool IsResolved) override;
+                uint64_t &FixedValue, bool IsResolved);
 
   void maybeAddVendorReloc(const MCFragment &, const MCFixup &);
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 3d304842fac1..e123ef45fa50 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -316,6 +316,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM8_LSB000,
   OPERAND_UIMM8_GE32,
   OPERAND_UIMM9_LSB000,
+  OPERAND_UIMM9,
   OPERAND_UIMM10,
   OPERAND_UIMM10_LSB00_NONZERO,
   OPERAND_UIMM11,
@@ -337,6 +338,8 @@ enum OperandType : unsigned {
   OPERAND_SIMM5_PLUS1,
   OPERAND_SIMM6,
   OPERAND_SIMM6_NONZERO,
+  OPERAND_SIMM10,
+  OPERAND_SIMM10_UNSIGNED,
   OPERAND_SIMM10_LSB0000_NONZERO,
   OPERAND_SIMM11,
   OPERAND_SIMM12,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index b5c23772e6d8..c1cdf511fae5 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -71,21 +71,17 @@ getRelocPairForSize(unsigned Size) {
   default:
     llvm_unreachable("unsupported fixup size");
   case 1:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD8),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB8));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD8,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB8);
   case 2:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD16),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB16));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD16,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB16);
   case 4:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD32),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB32));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD32,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB32);
   case 8:
-    return std::make_pair(
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_ADD64),
-        MCFixupKind(FirstLiteralRelocationKind + ELF::R_RISCV_SUB64));
+    return std::make_pair(FirstLiteralRelocationKind + ELF::R_RISCV_ADD64,
+                          FirstLiteralRelocationKind + ELF::R_RISCV_SUB64);
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index ce0fbc0ac065..2ed7cd9f008a 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -120,6 +120,28 @@ MCCodeEmitter *llvm::createRISCVMCCodeEmitter(const MCInstrInfo &MCII,
   return new RISCVMCCodeEmitter(Ctx, MCII);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ELF::R_RISCV_CALL_PLT:
+  case RISCV::fixup_riscv_pcrel_hi20:
+  case RISCV::fixup_riscv_pcrel_lo12_i:
+  case RISCV::fixup_riscv_pcrel_lo12_s:
+  case RISCV::fixup_riscv_jal:
+  case RISCV::fixup_riscv_branch:
+  case RISCV::fixup_riscv_rvc_jump:
+  case RISCV::fixup_riscv_rvc_branch:
+  case RISCV::fixup_riscv_call:
+  case RISCV::fixup_riscv_call_plt:
+  case RISCV::fixup_riscv_qc_e_branch:
+  case RISCV::fixup_riscv_qc_e_call_plt:
+  case RISCV::fixup_riscv_nds_branch_10:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 // Expand PseudoCALL(Reg), PseudoTAIL and PseudoJump to AUIPC and JALR with
 // relocation types. We expand those pseudo-instructions while encoding them,
 // meaning AUIPC and JALR won't go through RISC-V MC to MC compressed
@@ -181,8 +203,7 @@ void RISCVMCCodeEmitter::expandTLSDESCCall(const MCInst &MI,
   MCRegister Link = MI.getOperand(0).getReg();
   MCRegister Dest = MI.getOperand(1).getReg();
   int64_t Imm = MI.getOperand(2).getImm();
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_RISCV_TLSDESC_CALL, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_RISCV_TLSDESC_CALL);
   MCInst Call =
       MCInstBuilder(RISCV::JALR).addReg(Link).addReg(Dest).addImm(Imm);
 
@@ -209,8 +230,7 @@ void RISCVMCCodeEmitter::expandAddTPRel(const MCInst &MI,
   assert(Expr && Expr->getSpecifier() == ELF::R_RISCV_TPREL_ADD &&
          "Expected tprel_add relocation on TP-relative symbol");
 
-  Fixups.push_back(
-      MCFixup::create(0, Expr, ELF::R_RISCV_TPREL_ADD, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, ELF::R_RISCV_TPREL_ADD);
   if (STI.hasFeature(RISCV::FeatureRelax))
     Fixups.back().setLinkerRelaxable();
 
@@ -320,11 +340,8 @@ void RISCVMCCodeEmitter::expandLongCondBr(const MCInst &MI,
   // Drop any fixup added so we can add the correct one.
   Fixups.resize(FixupStartIndex);
 
-  if (SrcSymbol.isExpr()) {
-    Fixups.push_back(MCFixup::create(Offset, SrcSymbol.getExpr(),
-                                     MCFixupKind(RISCV::fixup_riscv_jal),
-                                     MI.getLoc()));
-  }
+  if (SrcSymbol.isExpr())
+    addFixup(Fixups, Offset, SrcSymbol.getExpr(), RISCV::fixup_riscv_jal);
 }
 
 // Expand PseudoLongQC_(E_)Bxxx to an inverted conditional branch and an
@@ -371,11 +388,8 @@ void RISCVMCCodeEmitter::expandQCLongCondBrImm(const MCInst &MI,
   support::endian::write(CB, JBinary, llvm::endianness::little);
   // Drop any fixup added so we can add the correct one.
   Fixups.resize(FixupStartIndex);
-  if (SrcSymbol.isExpr()) {
-    Fixups.push_back(MCFixup::create(Offset, SrcSymbol.getExpr(),
-                                     MCFixupKind(RISCV::fixup_riscv_jal),
-                                     MI.getLoc()));
-  }
+  if (SrcSymbol.isExpr())
+    addFixup(Fixups, Offset, SrcSymbol.getExpr(), RISCV::fixup_riscv_jal);
 }
 
 void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI,
@@ -653,7 +667,7 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
 
   assert(FixupKind != RISCV::fixup_riscv_invalid && "Unhandled expression!");
 
-  Fixups.push_back(MCFixup::create(0, Expr, FixupKind, MI.getLoc()));
+  addFixup(Fixups, 0, Expr, FixupKind);
   // If linker relaxation is enabled and supported by this relocation, set
   // a bit so that if fixup is unresolved, a R_RISCV_RELAX relocation will be
   // appended.
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 0e59861b8a78..d4f5d8fcad6b 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -292,7 +292,7 @@ void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
     NontemporalMode += 0b10;
 
   MCInst Hint;
-  if (STI->hasStdExtZca() && STI->enableRVCHintInstrs())
+  if (STI->hasStdExtZca())
     Hint.setOpcode(RISCV::C_ADD_HINT);
   else
     Hint.setOpcode(RISCV::ADD);
@@ -305,8 +305,7 @@ void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
 }
 
 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
-  RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
-                                        getSubtargetInfo().getFeatureBits());
+  RISCV_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
 
   emitNTLHint(MI);
 
@@ -517,12 +516,9 @@ void RISCVAsmPrinter::emitSled(const MachineInstr *MI, SledKind Kind) {
   // Assuming we're using JAL to jump to .tmpN, then we only need
   // (68 - 4)/2 = 32 NOPs for RV64 and (44 - 4)/2 = 20 for RV32. However, there
   // is a chance that we'll use C.JAL instead, so an additional NOP is needed.
-  const uint8_t NoopsInSledCount =
-      MI->getParent()->getParent()->getSubtarget<RISCVSubtarget>().is64Bit()
-          ? 33
-          : 21;
+  const uint8_t NoopsInSledCount = STI->is64Bit() ? 33 : 21;
 
-  OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
+  OutStreamer->emitCodeAlignment(Align(4), STI);
   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
   OutStreamer->emitLabel(CurSled);
   auto Target = OutContext.createTempSymbol();
@@ -1070,7 +1066,8 @@ bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
 }
 
 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
-                                            MCInst &OutMI) {
+                                            MCInst &OutMI,
+                                            const RISCVSubtarget *STI) {
   const RISCVVPseudosTable::PseudoInfo *RVV =
       RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
   if (!RVV)
@@ -1078,14 +1075,8 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
 
   OutMI.setOpcode(RVV->BaseInstr);
 
-  const MachineBasicBlock *MBB = MI->getParent();
-  assert(MBB && "MI expected to be in a basic block");
-  const MachineFunction *MF = MBB->getParent();
-  assert(MF && "MBB expected to be in a machine function");
-
-  const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
-  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
-  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
+  const TargetInstrInfo *TII = STI->getInstrInfo();
+  const TargetRegisterInfo *TRI = STI->getRegisterInfo();
   assert(TRI && "TargetRegisterInfo expected");
 
   const MCInstrDesc &MCID = MI->getDesc();
@@ -1182,7 +1173,7 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
 }
 
 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
-  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
+  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI, STI))
     return false;
 
   OutMI.setOpcode(MI->getOpcode());
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index d3dce4edb1e7..cb57c4377779 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -147,6 +147,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   case RISCV::PseudoCCANDN:
   case RISCV::PseudoCCORN:
   case RISCV::PseudoCCXNOR:
+  case RISCV::PseudoCCNDS_BFOS:
+  case RISCV::PseudoCCNDS_BFOZ:
     return expandCCOp(MBB, MBBI, NextMBBI);
   case RISCV::PseudoVMCLR_M_B1:
   case RISCV::PseudoVMCLR_M_B2:
@@ -194,7 +196,7 @@ bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,
   CC = RISCVCC::getOppositeBranchCondition(CC);
 
   // Insert branch instruction.
-  BuildMI(MBB, MBBI, DL, TII->getBrCond(CC))
+  BuildMI(MBB, MBBI, DL, TII->get(RISCVCC::getBrCond(CC)))
       .addReg(MI.getOperand(1).getReg())
       .addReg(MI.getOperand(2).getReg())
       .addMBB(MergeBB);
@@ -240,10 +242,20 @@ bool RISCVExpandPseudo::expandCCOp(MachineBasicBlock &MBB,
     case RISCV::PseudoCCANDN:  NewOpc = RISCV::ANDN;  break;
     case RISCV::PseudoCCORN:   NewOpc = RISCV::ORN;   break;
     case RISCV::PseudoCCXNOR:  NewOpc = RISCV::XNOR;  break;
+    case RISCV::PseudoCCNDS_BFOS: NewOpc = RISCV::NDS_BFOS; break;
+    case RISCV::PseudoCCNDS_BFOZ: NewOpc = RISCV::NDS_BFOZ; break;
+    }
+
+    if (NewOpc == RISCV::NDS_BFOZ || NewOpc == RISCV::NDS_BFOS) {
+      BuildMI(TrueBB, DL, TII->get(NewOpc), DestReg)
+          .add(MI.getOperand(5))
+          .add(MI.getOperand(6))
+          .add(MI.getOperand(7));
+    } else {
+      BuildMI(TrueBB, DL, TII->get(NewOpc), DestReg)
+          .add(MI.getOperand(5))
+          .add(MI.getOperand(6));
     }
-    BuildMI(TrueBB, DL, TII->get(NewOpc), DestReg)
-        .add(MI.getOperand(5))
-        .add(MI.getOperand(6));
   }
 
   TrueBB->addSuccessor(MergeBB);
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 36b3aff51cda..addeab8d9aba 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -373,13 +373,6 @@ def HasStdExtZhinx : Predicate<"Subtarget->hasStdExtZhinx()">,
 def NoStdExtZhinx : Predicate<"!Subtarget->hasStdExtZhinx()">;
 
 // Compressed Extensions
-def FeatureNoRVCHints
-    : SubtargetFeature<"no-rvc-hints", "EnableRVCHintInstrs", "false",
-                       "Disable RVC Hint Instructions.">;
-def HasRVCHints : Predicate<"Subtarget->enableRVCHintInstrs()">,
-                  AssemblerPredicate<(all_of(not FeatureNoRVCHints)),
-                                     "RVC Hint Instructions">;
-
 def FeatureStdExtZca
     : RISCVExtension<1, 0,
                      "part of the C extension, excluding compressed "
@@ -1298,12 +1291,12 @@ def HasVendorXSfvqmaccqoq
 
 def FeatureVendorXSfvfwmaccqqq
     : RISCVExtension<1, 0,
-                     "SiFive Matrix Multiply Accumulate Instruction and 4-by-4)",
+                     "SiFive Matrix Multiply Accumulate Instruction (4-by-4)",
                      [FeatureStdExtZvfbfmin]>;
 def HasVendorXSfvfwmaccqqq
     : Predicate<"Subtarget->hasVendorXSfvfwmaccqqq()">,
       AssemblerPredicate<(all_of FeatureVendorXSfvfwmaccqqq),
-                         "'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))">;
+                         "'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4))">;
 
 def FeatureVendorXSfvfnrclipxfqf
     : RISCVExtension<1, 0,
@@ -1411,6 +1404,13 @@ def HasVendorXMIPSLSP
     : Predicate<"Subtarget->hasVendorXMIPSLSP()">,
       AssemblerPredicate<(all_of FeatureVendorXMIPSLSP),
                          "'Xmipslsp' (load and store pair instructions)">;
+def FeatureVendorXMIPSCBOP
+    : RISCVExtension<1, 0, "MIPS Software Prefetch">;
+def HasVendorXMIPSCBOP
+    : Predicate<"Subtarget->hasVendorXMIPSCBOP()">,
+      AssemblerPredicate<(all_of FeatureVendorXMIPSCBOP),
+                         "'Xmipscbop' (MIPS hardware prefetch)">;
+def NotHasVendorXMIPSCBOP : Predicate<"!Subtarget->hasVendorXMIPSCBOP()">;
 
 // WCH / Nanjing Qinheng Microelectronics Extension(s)
 
@@ -1607,10 +1607,17 @@ def HasVendorXAndesVBFHCvt
       AssemblerPredicate<(all_of FeatureVendorXAndesVBFHCvt),
                          "'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension)">;
 
+def FeatureVendorXAndesVSIntLoad
+    : RISCVExtension<5, 0, "Andes Vector INT4 Load Extension",
+                     [FeatureStdExtZve32x]>;
+def HasVendorXAndesVSIntLoad
+    : Predicate<"Subtarget->hasVendorXAndesVSIntLoad()">,
+      AssemblerPredicate<(all_of FeatureVendorXAndesVSIntLoad),
+                         "'XAndesVSIntLoad' (Andes Vector INT4 Load Extension)">;
 
 def FeatureVendorXAndesVPackFPH
-    : RISCVExtension<5, 0, "Andes Vector Packed FP16 Extension",
-                     [FeatureStdExtZvfhmin]>;
+    : RISCVExtension<5, 0,
+                     "Andes Vector Packed FP16 Extension", [FeatureStdExtF]>;
 def HasVendorXAndesVPackFPH
     : Predicate<"Subtarget->hasVendorXAndesVPackFPH()">,
       AssemblerPredicate<(all_of FeatureVendorXAndesVPackFPH),
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 4539efd591c8..6298c7d5e9ef 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -163,8 +163,6 @@ void RISCVDAGToDAGISel::PostprocessISelDAG() {
 
   CurDAG->setRoot(Dummy.getValue());
 
-  MadeChange |= doPeepholeMergeVVMFold();
-
   // After we're done with everything else, convert IMPLICIT_DEF
   // passthru operands to NoRegister.  This is required to workaround
   // an optimization deficiency in MachineCSE.  This really should
@@ -879,32 +877,32 @@ void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
   auto *LMulSDNode = cast<ConstantSDNode>(Node->getOperand(7));
   switch (LMulSDNode->getSExtValue()) {
   case 5:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF8
-                                                  : RISCV::PseudoVC_I_SE_MF8;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
+                                                  : RISCV::PseudoSF_VC_I_SE_MF8;
     break;
   case 6:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF4
-                                                  : RISCV::PseudoVC_I_SE_MF4;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
+                                                  : RISCV::PseudoSF_VC_I_SE_MF4;
     break;
   case 7:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_MF2
-                                                  : RISCV::PseudoVC_I_SE_MF2;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
+                                                  : RISCV::PseudoSF_VC_I_SE_MF2;
     break;
   case 0:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M1
-                                                  : RISCV::PseudoVC_I_SE_M1;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
+                                                  : RISCV::PseudoSF_VC_I_SE_M1;
     break;
   case 1:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M2
-                                                  : RISCV::PseudoVC_I_SE_M2;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
+                                                  : RISCV::PseudoSF_VC_I_SE_M2;
     break;
   case 2:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M4
-                                                  : RISCV::PseudoVC_I_SE_M4;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
+                                                  : RISCV::PseudoSF_VC_I_SE_M4;
     break;
   case 3:
-    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoVC_X_SE_M8
-                                                  : RISCV::PseudoVC_I_SE_M8;
+    Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
+                                                  : RISCV::PseudoSF_VC_I_SE_M8;
     break;
   }
 
@@ -2824,10 +2822,6 @@ bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
       Base = Addr.getOperand(0);
       return true;
     }
-  } else if (UnwrapShl(Addr, Index, Scale)) {
-    EVT VT = Addr.getValueType();
-    Base = CurDAG->getRegister(RISCV::X0, VT);
-    return true;
   }
 
   return false;
@@ -2925,6 +2919,33 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
   return true;
 }
 
+/// Similar to SelectAddrRegImm, except that the offset restricted for
+/// unsinged nine bits.
+bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
+                                          SDValue &Offset) {
+  if (SelectAddrFrameIndex(Addr, Base, Offset))
+    return true;
+
+  SDLoc DL(Addr);
+  MVT VT = Addr.getSimpleValueType();
+
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
+    if (isUInt<9>(CVal)) {
+      Base = Addr.getOperand(0);
+
+      if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
+        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
+      Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
+      return true;
+    }
+  }
+
+  Base = Addr;
+  Offset = CurDAG->getTargetConstant(0, DL, VT);
+  return true;
+}
+
 /// Similar to SelectAddrRegImm, except that the least significant 5 bits of
 /// Offset should be all zeros.
 bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
@@ -4069,218 +4090,6 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
   return true;
 }
 
-static bool IsVMerge(SDNode *N) {
-  return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
-}
-
-// Try to fold away VMERGE_VVM instructions into their true operands:
-//
-// %true = PseudoVADD_VV ...
-// %x = PseudoVMERGE_VVM %false, %false, %true, %mask
-// ->
-// %x = PseudoVADD_VV_MASK %false, ..., %mask
-//
-// We can only fold if vmerge's passthru operand, vmerge's false operand and
-// %true's passthru operand (if it has one) are the same. This is because we
-// have to consolidate them into one passthru operand in the result.
-//
-// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
-// mask is all ones.
-//
-// The resulting VL is the minimum of the two VLs.
-//
-// The resulting policy is the effective policy the vmerge would have had,
-// i.e. whether or not it's passthru operand was implicit-def.
-bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
-  SDValue Passthru, False, True, VL, Mask;
-  assert(IsVMerge(N));
-  Passthru = N->getOperand(0);
-  False = N->getOperand(1);
-  True = N->getOperand(2);
-  Mask = N->getOperand(3);
-  VL = N->getOperand(4);
-
-  // If the EEW of True is different from vmerge's SEW, then we can't fold.
-  if (True.getSimpleValueType() != N->getSimpleValueType(0))
-    return false;
-
-  // We require that either passthru and false are the same, or that passthru
-  // is undefined.
-  if (Passthru != False && !isImplicitDef(Passthru))
-    return false;
-
-  assert(True.getResNo() == 0 &&
-         "Expect True is the first output of an instruction.");
-
-  // Need N is the exactly one using True.
-  if (!True.hasOneUse())
-    return false;
-
-  if (!True.isMachineOpcode())
-    return false;
-
-  unsigned TrueOpc = True.getMachineOpcode();
-  const MCInstrDesc &TrueMCID = TII->get(TrueOpc);
-  uint64_t TrueTSFlags = TrueMCID.TSFlags;
-  bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(TrueMCID);
-
-  const RISCV::RISCVMaskedPseudoInfo *Info =
-      RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc);
-  if (!Info)
-    return false;
-
-  // If True has a passthru operand then it needs to be the same as vmerge's
-  // False, since False will be used for the result's passthru operand.
-  if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
-    SDValue PassthruOpTrue = True->getOperand(0);
-    if (False != PassthruOpTrue)
-      return false;
-  }
-
-  // Skip if True has side effect.
-  if (TII->get(TrueOpc).hasUnmodeledSideEffects())
-    return false;
-
-  unsigned TrueChainOpIdx = True.getNumOperands() - 1;
-  bool HasChainOp =
-      True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other;
-
-  if (HasChainOp) {
-    // Avoid creating cycles in the DAG. We must ensure that none of the other
-    // operands depend on True through it's Chain.
-    SmallVector<const SDNode *, 4> LoopWorklist;
-    SmallPtrSet<const SDNode *, 16> Visited;
-    LoopWorklist.push_back(False.getNode());
-    LoopWorklist.push_back(Mask.getNode());
-    LoopWorklist.push_back(VL.getNode());
-    if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
-      return false;
-  }
-
-  // The vector policy operand may be present for masked intrinsics
-  bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags);
-  unsigned TrueVLIndex =
-      True.getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
-  SDValue TrueVL = True.getOperand(TrueVLIndex);
-  SDValue SEW = True.getOperand(TrueVLIndex + 1);
-
-  auto GetMinVL = [](SDValue LHS, SDValue RHS) {
-    if (LHS == RHS)
-      return LHS;
-    if (isAllOnesConstant(LHS))
-      return RHS;
-    if (isAllOnesConstant(RHS))
-      return LHS;
-    auto *CLHS = dyn_cast<ConstantSDNode>(LHS);
-    auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
-    if (!CLHS || !CRHS)
-      return SDValue();
-    return CLHS->getZExtValue() <= CRHS->getZExtValue() ? LHS : RHS;
-  };
-
-  // Because N and True must have the same passthru operand (or True's operand
-  // is implicit_def), the "effective" body is the minimum of their VLs.
-  SDValue OrigVL = VL;
-  VL = GetMinVL(TrueVL, VL);
-  if (!VL)
-    return false;
-
-  // Some operations produce different elementwise results depending on the
-  // active elements, like viota.m or vredsum. This transformation is illegal
-  // for these if we change the active elements (i.e. mask or VL).
-  const MCInstrDesc &TrueBaseMCID = TII->get(RISCV::getRVVMCOpcode(TrueOpc));
-  if (RISCVII::elementsDependOnVL(TrueBaseMCID.TSFlags) && (TrueVL != VL))
-    return false;
-  if (RISCVII::elementsDependOnMask(TrueBaseMCID.TSFlags) &&
-      (Mask && !usesAllOnesMask(Mask)))
-    return false;
-
-  // Make sure it doesn't raise any observable fp exceptions, since changing the
-  // active elements will affect how fflags is set.
-  if (mayRaiseFPException(True.getNode()) && !True->getFlags().hasNoFPExcept())
-    return false;
-
-  SDLoc DL(N);
-
-  unsigned MaskedOpc = Info->MaskedPseudo;
-#ifndef NDEBUG
-  const MCInstrDesc &MaskedMCID = TII->get(MaskedOpc);
-  assert(RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) &&
-         "Expected instructions with mask have policy operand.");
-  assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
-                                         MCOI::TIED_TO) == 0 &&
-         "Expected instructions with mask have a tied dest.");
-#endif
-
-  // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
-  // operand is undefined.
-  //
-  // However, if the VL became smaller than what the vmerge had originally, then
-  // elements past VL that were previously in the vmerge's body will have moved
-  // to the tail. In that case we always need to use tail undisturbed to
-  // preserve them.
-  bool MergeVLShrunk = VL != OrigVL;
-  uint64_t Policy = (isImplicitDef(Passthru) && !MergeVLShrunk)
-                        ? RISCVVType::TAIL_AGNOSTIC
-                        : /*TUMU*/ 0;
-  SDValue PolicyOp =
-    CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());
-
-
-  SmallVector<SDValue, 8> Ops;
-  Ops.push_back(False);
-
-  const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
-  const unsigned NormalOpsEnd = TrueVLIndex - HasRoundingMode;
-  Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + NormalOpsEnd);
-
-  Ops.push_back(Mask);
-
-  // For unmasked "VOp" with rounding mode operand, that is interfaces like
-  // (..., rm, vl) or (..., rm, vl, policy).
-  // Its masked version is (..., vm, rm, vl, policy).
-  // Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td
-  if (HasRoundingMode)
-    Ops.push_back(True->getOperand(TrueVLIndex - 1));
-
-  Ops.append({VL, SEW, PolicyOp});
-
-  // Result node should have chain operand of True.
-  if (HasChainOp)
-    Ops.push_back(True.getOperand(TrueChainOpIdx));
-
-  MachineSDNode *Result =
-      CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
-  Result->setFlags(True->getFlags());
-
-  if (!cast<MachineSDNode>(True)->memoperands_empty())
-    CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(True)->memoperands());
-
-  // Replace vmerge.vvm node by Result.
-  ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
-
-  // Replace another value of True. E.g. chain and VL.
-  for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
-    ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
-
-  return true;
-}
-
-bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
-  bool MadeChange = false;
-  SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
-
-  while (Position != CurDAG->allnodes_begin()) {
-    SDNode *N = &*--Position;
-    if (N->use_empty() || !N->isMachineOpcode())
-      continue;
-
-    if (IsVMerge(N))
-      MadeChange |= performCombineVMergeAndVOps(N);
-  }
-  return MadeChange;
-}
-
 /// If our passthru is an implicit_def, use noreg instead.  This side
 /// steps issues with MachineCSE not being able to CSE expressions with
 /// IMPLICIT_DEF operands while preserving the semantic intent. See
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index cb63c21fd8fc..66d878f03744 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -47,6 +47,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 
   bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset);
+  bool SelectAddrRegImm9(SDValue Addr, SDValue &Base, SDValue &Offset);
   bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset);
 
   bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount,
@@ -199,7 +200,6 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
 private:
   bool doPeepholeSExtW(SDNode *Node);
   bool doPeepholeMaskedRVV(MachineSDNode *Node);
-  bool doPeepholeMergeVVMFold();
   bool doPeepholeNoRegPassThru();
   bool performCombineVMergeAndVOps(SDNode *N);
   bool selectImm64IfCheaper(int64_t Imm, int64_t OrigImm, SDValue N,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 13ee3ee63d1a..d2694815be37 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -211,7 +211,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       addRegClassForRVV(VT);
     }
 
-    if (Subtarget.hasVInstructionsF16Minimal())
+    if (Subtarget.hasVInstructionsF16Minimal() ||
+        Subtarget.hasVendorXAndesVPackFPH())
       for (MVT VT : F16VecVTs)
         addRegClassForRVV(VT);
 
@@ -436,7 +437,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ABS, MVT::i32, Custom);
   }
 
-  if (!Subtarget.useCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
+  if (!Subtarget.useCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov() &&
+      !Subtarget.hasVendorXqcicm())
     setOperationAction(ISD::SELECT, XLenVT, Custom);
 
   if (Subtarget.hasVendorXqcia() && !Subtarget.is64Bit()) {
@@ -682,9 +684,10 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.is64Bit())
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
 
-  if (Subtarget.hasStdExtZicbop()) {
+  if (Subtarget.hasVendorXMIPSCBOP())
+    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
+  else if (Subtarget.hasStdExtZicbop())
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
-  }
 
   if (Subtarget.hasStdExtA()) {
     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
@@ -756,8 +759,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         ISD::VP_FROUNDEVEN,  ISD::VP_FCOPYSIGN,   ISD::VP_FROUNDTOZERO,
         ISD::VP_FRINT,       ISD::VP_FNEARBYINT,  ISD::VP_IS_FPCLASS,
         ISD::VP_FMINIMUM,    ISD::VP_FMAXIMUM,    ISD::VP_LRINT,
-        ISD::VP_LLRINT,      ISD::EXPERIMENTAL_VP_REVERSE,
-        ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
+        ISD::VP_LLRINT,       ISD::VP_REDUCE_FMINIMUM,
         ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
 
     static const unsigned IntegerVecReduceOps[] = {
@@ -1112,6 +1114,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
       setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom);
+      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
+      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
       setOperationAction(FloatingPointVPOps, VT, Custom);
 
@@ -1156,6 +1160,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                           ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE,
                           ISD::VECTOR_COMPRESS},
                          VT, Custom);
+      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
+      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
       MVT EltVT = VT.getVectorElementType();
       if (isTypeLegal(EltVT))
         setOperationAction({ISD::SPLAT_VECTOR, ISD::EXPERIMENTAL_VP_SPLAT,
@@ -1418,6 +1424,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                             ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
                             ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
                            VT, Custom);
+        setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
+        setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
         setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
                            VT, Custom);
@@ -1499,7 +1507,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                            VT, Custom);
 
         setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
-                            ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT},
+                            ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
+                            ISD::LLRINT, ISD::FNEARBYINT},
                            VT, Custom);
 
         setCondCodeAction(VFPCCToExpand, VT, Expand);
@@ -2409,12 +2418,15 @@ unsigned RISCVTargetLowering::getVectorTypeBreakdownForCallingConv(
 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
 // with 1/-1.
 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
-                                    ISD::CondCode &CC, SelectionDAG &DAG) {
+                                    ISD::CondCode &CC, SelectionDAG &DAG,
+                                    const RISCVSubtarget &Subtarget) {
   // If this is a single bit test that can't be handled by ANDI, shift the
   // bit to be tested to the MSB and perform a signed compare with 0.
   if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
       LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
-      isa<ConstantSDNode>(LHS.getOperand(1))) {
+      isa<ConstantSDNode>(LHS.getOperand(1)) &&
+      // XAndesPerf supports branch on test bit.
+      !Subtarget.hasVendorXAndesPerf()) {
     uint64_t Mask = LHS.getConstantOperandVal(1);
     if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
       unsigned ShAmt = 0;
@@ -2435,8 +2447,6 @@ static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
 
   if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
     int64_t C = RHSC->getSExtValue();
-    const RISCVSubtarget &Subtarget =
-        DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
     switch (CC) {
     default: break;
     case ISD::SETGT:
@@ -3202,7 +3212,14 @@ static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
   case ISD::VP_FROUND:
     return RISCVFPRndMode::RMM;
   case ISD::FRINT:
+  case ISD::LRINT:
+  case ISD::LLRINT:
+  case ISD::STRICT_FRINT:
+  case ISD::STRICT_LRINT:
+  case ISD::STRICT_LLRINT:
   case ISD::VP_FRINT:
+  case ISD::VP_LRINT:
+  case ISD::VP_LLRINT:
     return RISCVFPRndMode::DYN;
   }
 
@@ -3223,6 +3240,9 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
 
   SDValue Src = Op.getOperand(0);
 
+  // Freeze the source since we are increasing the number of uses.
+  Src = DAG.getFreeze(Src);
+
   MVT ContainerVT = VT;
   if (VT.isFixedLengthVector()) {
     ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
@@ -3240,9 +3260,6 @@ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
     std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
   }
 
-  // Freeze the source since we are increasing the number of uses.
-  Src = DAG.getFreeze(Src);
-
   // We do the conversion on the absolute value and fix the sign at the end.
   SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
@@ -3452,28 +3469,34 @@ lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
 // Expand vector LRINT and LLRINT by converting to the integer domain.
 static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
-  MVT VT = Op.getSimpleValueType();
-  assert(VT.isVector() && "Unexpected type");
-
   SDLoc DL(Op);
+  MVT DstVT = Op.getSimpleValueType();
   SDValue Src = Op.getOperand(0);
-  MVT ContainerVT = VT;
+  MVT SrcVT = Src.getSimpleValueType();
+  assert(SrcVT.isVector() && DstVT.isVector() &&
+         !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
+         "Unexpected type");
 
-  if (VT.isFixedLengthVector()) {
-    ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
-    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
+  MVT DstContainerVT = DstVT;
+  MVT SrcContainerVT = SrcVT;
+
+  if (DstVT.isFixedLengthVector()) {
+    DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget);
+    SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
+    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
   }
 
-  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
-  SDValue Truncated = DAG.getNode(
-      RISCVISD::VFCVT_RM_X_F_VL, DL, ContainerVT, Src, Mask,
-      DAG.getTargetConstant(RISCVFPRndMode::DYN, DL, Subtarget.getXLenVT()),
-      VL);
+  auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
+  SDValue Res =
+      DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
+                  DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
+                                        Subtarget.getXLenVT()),
+                  VL);
 
-  if (!VT.isFixedLengthVector())
-    return Truncated;
+  if (!DstVT.isFixedLengthVector())
+    return Res;
 
-  return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
+  return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
 }
 
 static SDValue
@@ -6590,6 +6613,17 @@ SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
   return DAG.getNode(ISD::FNEG, DL, VT, Const);
 }
 
+static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
+                             SelectionDAG &DAG) {
+
+  unsigned IsData = Op.getConstantOperandVal(4);
+
+  // mips-p8700  we support data prefetch for now.
+  if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
+    return Op.getOperand(0);
+  return Op;
+}
+
 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
   SDLoc dl(Op);
@@ -6842,32 +6876,32 @@ static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
   SDLoc DL(Op);
 
   // Get sign bit into an integer value.
-  SDValue SignAsInt;
   unsigned SignSize = Sign.getValueSizeInBits();
-  if (SignSize == Subtarget.getXLen()) {
-    SignAsInt = DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
-  } else if (SignSize == 16) {
-    SignAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
-  } else if (SignSize == 32) {
-    SignAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
-  } else if (SignSize == 64) {
-    assert(XLenVT == MVT::i32 && "Unexpected type");
-    // Copy the upper word to integer.
-    SignAsInt = DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
-                    .getValue(1);
-    SignSize = 32;
-  } else
-    llvm_unreachable("Unexpected sign size");
+  SDValue SignAsInt = [&]() {
+    if (SignSize == Subtarget.getXLen())
+      return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
+    switch (SignSize) {
+    case 16:
+      return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
+    case 32:
+      return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
+    case 64: {
+      assert(XLenVT == MVT::i32 && "Unexpected type");
+      // Copy the upper word to integer.
+      SignSize = 32;
+      return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
+          .getValue(1);
+    }
+    default:
+      llvm_unreachable("Unexpected sign size");
+    }
+  }();
 
   // Get the signbit at the right position for MagAsInt.
-  int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits();
-  if (ShiftAmount > 0) {
-    SignAsInt = DAG.getNode(ISD::SRL, DL, XLenVT, SignAsInt,
-                            DAG.getConstant(ShiftAmount, DL, XLenVT));
-  } else if (ShiftAmount < 0) {
-    SignAsInt = DAG.getNode(ISD::SHL, DL, XLenVT, SignAsInt,
-                            DAG.getConstant(-ShiftAmount, DL, XLenVT));
-  }
+  if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
+    SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
+                            SignAsInt,
+                            DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
 
   // Mask the sign bit and any bits above it. The extra bits will be dropped
   // when we convert back to FP.
@@ -7038,13 +7072,10 @@ static unsigned getRISCVVLOp(SDValue Op) {
 
 static bool isPromotedOpNeedingSplit(SDValue Op,
                                      const RISCVSubtarget &Subtarget) {
-  if (Op.getValueType() == MVT::nxv32f16 &&
-      (Subtarget.hasVInstructionsF16Minimal() &&
-       !Subtarget.hasVInstructionsF16()))
-    return true;
-  if (Op.getValueType() == MVT::nxv32bf16)
-    return true;
-  return false;
+  return (Op.getValueType() == MVT::nxv32f16 &&
+          (Subtarget.hasVInstructionsF16Minimal() &&
+           !Subtarget.hasVInstructionsF16())) ||
+         Op.getValueType() == MVT::nxv32bf16;
 }
 
 static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
@@ -7158,6 +7189,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   switch (Op.getOpcode()) {
   default:
     report_fatal_error("unimplemented operand");
+  case ISD::PREFETCH:
+    return LowerPREFETCH(Op, Subtarget, DAG);
   case ISD::ATOMIC_FENCE:
     return LowerATOMIC_FENCE(Op, DAG, Subtarget);
   case ISD::GlobalAddress:
@@ -9204,7 +9237,7 @@ SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
       return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
   }
 
-  translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+  translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
   // 1 < x ? x : 1 -> 0 < x ? x : 1
   if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
       RHS == TrueV && LHS == FalseV) {
@@ -9246,7 +9279,7 @@ SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     SDValue RHS = CondV.getOperand(1);
     ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
-    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
 
     SDValue TargetCC = DAG.getCondCode(CCVal);
     return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
@@ -9306,9 +9339,6 @@ SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
   MVT XLenVT = Subtarget.getXLenVT();
   int XLenInBytes = Subtarget.getXLen() / 8;
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
@@ -13239,6 +13269,8 @@ SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
 SDValue
 RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
                                                SelectionDAG &DAG) const {
+  using namespace SDPatternMatch;
+
   SDLoc DL(Op);
 
   SDValue Op1 = Op.getOperand(0);
@@ -13283,6 +13315,42 @@ RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
                       SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2);
   }
 
+  auto getVectorFirstEle = [](SDValue Vec) {
+    SDValue FirstEle;
+    if (sd_match(Vec, m_InsertElt(m_Value(), m_Value(FirstEle), m_Zero())))
+      return FirstEle;
+
+    if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
+        Vec.getOpcode() == ISD::BUILD_VECTOR)
+      return Vec.getOperand(0);
+
+    return SDValue();
+  };
+
+  if (!IsMaskVector && isNullConstant(Offset) && isOneConstant(EVL1))
+    if (auto FirstEle = getVectorFirstEle(Op->getOperand(0))) {
+      MVT EltVT = ContainerVT.getVectorElementType();
+      SDValue Result;
+      if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
+          EltVT == MVT::bf16) {
+        EltVT = EltVT.changeTypeToInteger();
+        ContainerVT = ContainerVT.changeVectorElementType(EltVT);
+        Op2 = DAG.getBitcast(ContainerVT, Op2);
+        FirstEle =
+            DAG.getAnyExtOrTrunc(DAG.getBitcast(EltVT, FirstEle), DL, XLenVT);
+      }
+      Result = DAG.getNode(EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
+                                                   : RISCVISD::VSLIDE1UP_VL,
+                           DL, ContainerVT, DAG.getUNDEF(ContainerVT), Op2,
+                           FirstEle, Mask, EVL2);
+      Result = DAG.getBitcast(
+          ContainerVT.changeVectorElementType(VT.getVectorElementType()),
+          Result);
+      return VT.isFixedLengthVector()
+                 ? convertFromScalableVector(VT, Result, DAG, Subtarget)
+                 : Result;
+    }
+
   int64_t ImmValue = cast<ConstantSDNode>(Offset)->getSExtValue();
   SDValue DownOffset, UpOffset;
   if (ImmValue >= 0) {
@@ -13297,10 +13365,11 @@ RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
     DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
   }
 
-  SDValue SlideDown =
-      getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
-                    Op1, DownOffset, Mask, UpOffset);
-  SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, SlideDown, Op2,
+  if (ImmValue != 0)
+    Op1 = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
+                        DAG.getUNDEF(ContainerVT), Op1, DownOffset, Mask,
+                        UpOffset);
+  SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, Op1, Op2,
                                UpOffset, Mask, EVL2, RISCVVType::TAIL_AGNOSTIC);
 
   if (IsMaskVector) {
@@ -18151,7 +18220,7 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
 
     RHS = LHS.getOperand(1);
     LHS = LHS.getOperand(0);
-    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
+    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
 
     CC = DAG.getCondCode(CCVal);
     return true;
@@ -18217,6 +18286,14 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
       uint64_t Mask = LHS0.getConstantOperandVal(1);
       uint64_t ShAmt = LHS.getConstantOperandVal(1);
       if (isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
+        // XAndesPerf supports branch on test bit.
+        if (Subtarget.hasVendorXAndesPerf()) {
+          LHS =
+              DAG.getNode(ISD::AND, DL, LHS.getValueType(), LHS0.getOperand(0),
+                          DAG.getConstant(Mask, DL, LHS.getValueType()));
+          return true;
+        }
+
         CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
         CC = DAG.getCondCode(CCVal);
 
@@ -18258,6 +18335,8 @@ static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
 // (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
 // (select C, (or Y, X), Y)  -> (or Y, (select C, X, 0)).
 // (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
+// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
+// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
 static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
                                    SDValue TrueVal, SDValue FalseVal,
                                    bool Swapped) {
@@ -18270,6 +18349,8 @@ static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SUB:
+  case ISD::ROTL:
+  case ISD::ROTR:
     Commutative = false;
     break;
   case ISD::ADD:
@@ -21332,7 +21413,7 @@ EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second,
   Register FLHS = First.getOperand(1).getReg();
   Register FRHS = First.getOperand(2).getReg();
   // Insert appropriate branch.
-  BuildMI(FirstMBB, DL, TII.getBrCond(FirstCC))
+  BuildMI(FirstMBB, DL, TII.get(RISCVCC::getBrCond(FirstCC, First.getOpcode())))
       .addReg(FLHS)
       .addReg(FRHS)
       .addMBB(SinkMBB);
@@ -21344,7 +21425,8 @@ EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second,
 
   auto SecondCC = static_cast<RISCVCC::CondCode>(Second.getOperand(3).getImm());
   // Insert appropriate branch.
-  BuildMI(ThisMBB, DL, TII.getBrCond(SecondCC))
+  BuildMI(ThisMBB, DL,
+          TII.get(RISCVCC::getBrCond(SecondCC, Second.getOpcode())))
       .addReg(SLHS)
       .addReg(SRHS)
       .addMBB(SinkMBB);
@@ -21399,12 +21481,8 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   // EmitLoweredCascadedSelect.
 
   auto Next = next_nodbg(MI.getIterator(), BB->instr_end());
-  if ((MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
-       MI.getOpcode() != RISCV::Select_GPR_Using_CC_SImm5_CV &&
-       MI.getOpcode() != RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC &&
-       MI.getOpcode() != RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC &&
-       MI.getOpcode() != RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC &&
-       MI.getOpcode() != RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC) &&
+  if (MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
+      MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
       Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
       Next->getOperand(5).getReg() == MI.getOperand(0).getReg() &&
       Next->getOperand(5).isKill())
@@ -21483,12 +21561,12 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
 
   // Insert appropriate branch.
   if (MI.getOperand(2).isImm())
-    BuildMI(HeadMBB, DL, TII.getBrCond(CC))
+    BuildMI(HeadMBB, DL, TII.get(RISCVCC::getBrCond(CC, MI.getOpcode())))
         .addReg(LHS)
         .addImm(MI.getOperand(2).getImm())
         .addMBB(TailMBB);
   else
-    BuildMI(HeadMBB, DL, TII.getBrCond(CC))
+    BuildMI(HeadMBB, DL, TII.get(RISCVCC::getBrCond(CC, MI.getOpcode())))
         .addReg(LHS)
         .addReg(RHS)
         .addMBB(TailMBB);
@@ -21742,6 +21820,8 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
   case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
   case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
+  case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
+  case RISCV::Select_GPR_Using_CC_UImm7_NDS:
   case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR16INX_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 78d64ea67324..dd002473a06e 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1768,8 +1768,9 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
           SlotIndex NewDefSI =
               LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();
           LiveInterval &DefLI = LIS->getInterval(VLOutput);
-          VNInfo *DefVNI = DefLI.getVNInfoAt(DefLI.beginIndex());
-          DefLI.removeSegment(DefLI.beginIndex(), NewDefSI);
+          LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(NewDefSI);
+          VNInfo *DefVNI = DefLI.getVNInfoAt(DefSeg->start);
+          DefLI.removeSegment(DefSeg->start, NewDefSI);
           DefVNI->def = NewDefSI;
         }
       }
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 6e30ffce99c4..2723229859a5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -513,26 +513,24 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                  Register SrcReg, bool KillSrc,
                                  bool RenamableDest, bool RenamableSrc) const {
   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+  unsigned KillFlag = getKillRegState(KillSrc);
 
   if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc))
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc))
         .addImm(0);
     return;
   }
 
   if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
     return;
   }
 
   if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)
-        .addReg(SrcReg,
-                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
     return;
   }
 
@@ -547,11 +545,11 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     // Emit an ADDI for both parts of GPRPair.
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
             TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
-        .addReg(EvenReg, getKillRegState(KillSrc))
+        .addReg(EvenReg, KillFlag)
         .addImm(0);
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
             TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
-        .addReg(OddReg, getKillRegState(KillSrc))
+        .addReg(OddReg, KillFlag)
         .addImm(0);
     return;
   }
@@ -581,36 +579,36 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       Opc = RISCV::FSGNJ_S;
     }
     BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag)
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::FPR32RegClass.contains(DstReg) &&
       RISCV::GPRRegClass.contains(SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
   if (RISCV::GPRRegClass.contains(DstReg) &&
       RISCV::FPR32RegClass.contains(SrcReg)) {
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -618,7 +616,7 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       RISCV::GPRRegClass.contains(SrcReg)) {
     assert(STI.getXLen() == 64 && "Unexpected GPR size");
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -626,7 +624,7 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       RISCV::FPR64RegClass.contains(SrcReg)) {
     assert(STI.getXLen() == 64 && "Unexpected GPR size");
     BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
+        .addReg(SrcReg, KillFlag);
     return;
   }
 
@@ -958,54 +956,45 @@ void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
   }
 }
 
-static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc) {
+RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
   switch (Opc) {
   default:
     return RISCVCC::COND_INVALID;
   case RISCV::BEQ:
-    return RISCVCC::COND_EQ;
-  case RISCV::BNE:
-    return RISCVCC::COND_NE;
-  case RISCV::BLT:
-    return RISCVCC::COND_LT;
-  case RISCV::BGE:
-    return RISCVCC::COND_GE;
-  case RISCV::BLTU:
-    return RISCVCC::COND_LTU;
-  case RISCV::BGEU:
-    return RISCVCC::COND_GEU;
   case RISCV::CV_BEQIMM:
-    return RISCVCC::COND_CV_BEQIMM;
-  case RISCV::CV_BNEIMM:
-    return RISCVCC::COND_CV_BNEIMM;
   case RISCV::QC_BEQI:
-    return RISCVCC::COND_QC_BEQI;
   case RISCV::QC_E_BEQI:
-    return RISCVCC::COND_QC_E_BEQI;
+  case RISCV::NDS_BBC:
+  case RISCV::NDS_BEQC:
+    return RISCVCC::COND_EQ;
+  case RISCV::BNE:
   case RISCV::QC_BNEI:
-    return RISCVCC::COND_QC_BNEI;
   case RISCV::QC_E_BNEI:
-    return RISCVCC::COND_QC_E_BNEI;
+  case RISCV::CV_BNEIMM:
+  case RISCV::NDS_BBS:
+  case RISCV::NDS_BNEC:
+    return RISCVCC::COND_NE;
+  case RISCV::BLT:
   case RISCV::QC_BLTI:
-    return RISCVCC::COND_QC_BLTI;
   case RISCV::QC_E_BLTI:
-    return RISCVCC::COND_QC_E_BLTI;
+    return RISCVCC::COND_LT;
+  case RISCV::BGE:
   case RISCV::QC_BGEI:
-    return RISCVCC::COND_QC_BGEI;
   case RISCV::QC_E_BGEI:
-    return RISCVCC::COND_QC_E_BGEI;
+    return RISCVCC::COND_GE;
+  case RISCV::BLTU:
   case RISCV::QC_BLTUI:
-    return RISCVCC::COND_QC_BLTUI;
   case RISCV::QC_E_BLTUI:
-    return RISCVCC::COND_QC_E_BLTUI;
+    return RISCVCC::COND_LTU;
+  case RISCV::BGEU:
   case RISCV::QC_BGEUI:
-    return RISCVCC::COND_QC_BGEUI;
   case RISCV::QC_E_BGEUI:
-    return RISCVCC::COND_QC_E_BGEUI;
+    return RISCVCC::COND_GEU;
   }
 }
 
-bool RISCVInstrInfo::evaluateCondBranch(unsigned CC, int64_t C0, int64_t C1) {
+bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
+                                        int64_t C1) {
   switch (CC) {
   default:
     llvm_unreachable("Unexpected CC");
@@ -1033,63 +1022,112 @@ static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
   assert(LastInst.getDesc().isConditionalBranch() &&
          "Unknown conditional branch");
   Target = LastInst.getOperand(2).getMBB();
-  unsigned CC = getCondFromBranchOpc(LastInst.getOpcode());
-  Cond.push_back(MachineOperand::CreateImm(CC));
+  Cond.push_back(MachineOperand::CreateImm(LastInst.getOpcode()));
   Cond.push_back(LastInst.getOperand(0));
   Cond.push_back(LastInst.getOperand(1));
 }
 
-unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC) {
-  switch (CC) {
+unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
+  switch (SelectOpc) {
   default:
-    llvm_unreachable("Unknown condition code!");
-  case RISCVCC::COND_EQ:
-    return RISCV::BEQ;
-  case RISCVCC::COND_NE:
-    return RISCV::BNE;
-  case RISCVCC::COND_LT:
-    return RISCV::BLT;
-  case RISCVCC::COND_GE:
-    return RISCV::BGE;
-  case RISCVCC::COND_LTU:
-    return RISCV::BLTU;
-  case RISCVCC::COND_GEU:
-    return RISCV::BGEU;
-  case RISCVCC::COND_CV_BEQIMM:
-    return RISCV::CV_BEQIMM;
-  case RISCVCC::COND_CV_BNEIMM:
-    return RISCV::CV_BNEIMM;
-  case RISCVCC::COND_QC_BEQI:
-    return RISCV::QC_BEQI;
-  case RISCVCC::COND_QC_E_BEQI:
-    return RISCV::QC_E_BEQI;
-  case RISCVCC::COND_QC_BNEI:
-    return RISCV::QC_BNEI;
-  case RISCVCC::COND_QC_E_BNEI:
-    return RISCV::QC_E_BNEI;
-  case RISCVCC::COND_QC_BLTI:
-    return RISCV::QC_BLTI;
-  case RISCVCC::COND_QC_E_BLTI:
-    return RISCV::QC_E_BLTI;
-  case RISCVCC::COND_QC_BGEI:
-    return RISCV::QC_BGEI;
-  case RISCVCC::COND_QC_E_BGEI:
-    return RISCV::QC_E_BGEI;
-  case RISCVCC::COND_QC_BLTUI:
-    return RISCV::QC_BLTUI;
-  case RISCVCC::COND_QC_E_BLTUI:
-    return RISCV::QC_E_BLTUI;
-  case RISCVCC::COND_QC_BGEUI:
-    return RISCV::QC_BGEUI;
-  case RISCVCC::COND_QC_E_BGEUI:
-    return RISCV::QC_E_BGEUI;
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::BEQ;
+    case RISCVCC::COND_NE:
+      return RISCV::BNE;
+    case RISCVCC::COND_LT:
+      return RISCV::BLT;
+    case RISCVCC::COND_GE:
+      return RISCV::BGE;
+    case RISCVCC::COND_LTU:
+      return RISCV::BLTU;
+    case RISCVCC::COND_GEU:
+      return RISCV::BGEU;
+    }
+    break;
+  case RISCV::Select_GPR_Using_CC_SImm5_CV:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::CV_BEQIMM;
+    case RISCVCC::COND_NE:
+      return RISCV::CV_BNEIMM;
+    }
+    break;
+  case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::QC_BEQI;
+    case RISCVCC::COND_NE:
+      return RISCV::QC_BNEI;
+    case RISCVCC::COND_LT:
+      return RISCV::QC_BLTI;
+    case RISCVCC::COND_GE:
+      return RISCV::QC_BGEI;
+    }
+    break;
+  case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_LTU:
+      return RISCV::QC_BLTUI;
+    case RISCVCC::COND_GEU:
+      return RISCV::QC_BGEUI;
+    }
+    break;
+  case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::QC_E_BEQI;
+    case RISCVCC::COND_NE:
+      return RISCV::QC_E_BNEI;
+    case RISCVCC::COND_LT:
+      return RISCV::QC_E_BLTI;
+    case RISCVCC::COND_GE:
+      return RISCV::QC_E_BGEI;
+    }
+    break;
+  case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_LTU:
+      return RISCV::QC_E_BLTUI;
+    case RISCVCC::COND_GEU:
+      return RISCV::QC_E_BGEUI;
+    }
+    break;
+  case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::NDS_BBC;
+    case RISCVCC::COND_NE:
+      return RISCV::NDS_BBS;
+    }
+    break;
+  case RISCV::Select_GPR_Using_CC_UImm7_NDS:
+    switch (CC) {
+    default:
+      llvm_unreachable("Unexpected condition code!");
+    case RISCVCC::COND_EQ:
+      return RISCV::NDS_BEQC;
+    case RISCVCC::COND_NE:
+      return RISCV::NDS_BNEC;
+    }
+    break;
   }
 }
 
-const MCInstrDesc &RISCVInstrInfo::getBrCond(RISCVCC::CondCode CC) const {
-  return get(RISCVCC::getBrCond(CC));
-}
-
 RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
   switch (CC) {
   default:
@@ -1106,34 +1144,6 @@ RISCVCC::CondCode RISCVCC::getOppositeBranchCondition(RISCVCC::CondCode CC) {
     return RISCVCC::COND_GEU;
   case RISCVCC::COND_GEU:
     return RISCVCC::COND_LTU;
-  case RISCVCC::COND_CV_BEQIMM:
-    return RISCVCC::COND_CV_BNEIMM;
-  case RISCVCC::COND_CV_BNEIMM:
-    return RISCVCC::COND_CV_BEQIMM;
-  case RISCVCC::COND_QC_BEQI:
-    return RISCVCC::COND_QC_BNEI;
-  case RISCVCC::COND_QC_E_BEQI:
-    return RISCVCC::COND_QC_E_BNEI;
-  case RISCVCC::COND_QC_BNEI:
-    return RISCVCC::COND_QC_BEQI;
-  case RISCVCC::COND_QC_E_BNEI:
-    return RISCVCC::COND_QC_E_BEQI;
-  case RISCVCC::COND_QC_BLTI:
-    return RISCVCC::COND_QC_BGEI;
-  case RISCVCC::COND_QC_E_BLTI:
-    return RISCVCC::COND_QC_E_BGEI;
-  case RISCVCC::COND_QC_BGEI:
-    return RISCVCC::COND_QC_BLTI;
-  case RISCVCC::COND_QC_E_BGEI:
-    return RISCVCC::COND_QC_E_BLTI;
-  case RISCVCC::COND_QC_BLTUI:
-    return RISCVCC::COND_QC_BGEUI;
-  case RISCVCC::COND_QC_E_BLTUI:
-    return RISCVCC::COND_QC_E_BGEUI;
-  case RISCVCC::COND_QC_BGEUI:
-    return RISCVCC::COND_QC_BLTUI;
-  case RISCVCC::COND_QC_E_BGEUI:
-    return RISCVCC::COND_QC_E_BLTUI;
   }
 }
 
@@ -1263,9 +1273,10 @@ unsigned RISCVInstrInfo::insertBranch(
   }
 
   // Either a one or two-way conditional branch.
-  auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  MachineInstr &CondMI =
-      *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
+  MachineInstr &CondMI = *BuildMI(&MBB, DL, get(Cond[0].getImm()))
+                              .add(Cond[1])
+                              .add(Cond[2])
+                              .addMBB(TBB);
   if (BytesAdded)
     *BytesAdded += getInstSizeInBytes(CondMI);
 
@@ -1348,8 +1359,83 @@ void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
 bool RISCVInstrInfo::reverseBranchCondition(
     SmallVectorImpl<MachineOperand> &Cond) const {
   assert((Cond.size() == 3) && "Invalid branch condition!");
-  auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  Cond[0].setImm(getOppositeBranchCondition(CC));
+  switch (Cond[0].getImm()) {
+  default:
+    llvm_unreachable("Unknown conditional branch!");
+  case RISCV::BEQ:
+    Cond[0].setImm(RISCV::BNE);
+    break;
+  case RISCV::BNE:
+    Cond[0].setImm(RISCV::BEQ);
+    break;
+  case RISCV::BLT:
+    Cond[0].setImm(RISCV::BGE);
+    break;
+  case RISCV::BGE:
+    Cond[0].setImm(RISCV::BLT);
+    break;
+  case RISCV::BLTU:
+    Cond[0].setImm(RISCV::BGEU);
+    break;
+  case RISCV::BGEU:
+    Cond[0].setImm(RISCV::BLTU);
+    break;
+  case RISCV::CV_BEQIMM:
+    Cond[0].setImm(RISCV::CV_BNEIMM);
+    break;
+  case RISCV::CV_BNEIMM:
+    Cond[0].setImm(RISCV::CV_BEQIMM);
+    break;
+  case RISCV::QC_BEQI:
+    Cond[0].setImm(RISCV::QC_BNEI);
+    break;
+  case RISCV::QC_BNEI:
+    Cond[0].setImm(RISCV::QC_BEQI);
+    break;
+  case RISCV::QC_BGEI:
+    Cond[0].setImm(RISCV::QC_BLTI);
+    break;
+  case RISCV::QC_BLTI:
+    Cond[0].setImm(RISCV::QC_BGEI);
+    break;
+  case RISCV::QC_BGEUI:
+    Cond[0].setImm(RISCV::QC_BLTUI);
+    break;
+  case RISCV::QC_BLTUI:
+    Cond[0].setImm(RISCV::QC_BGEUI);
+    break;
+  case RISCV::QC_E_BEQI:
+    Cond[0].setImm(RISCV::QC_E_BNEI);
+    break;
+  case RISCV::QC_E_BNEI:
+    Cond[0].setImm(RISCV::QC_E_BEQI);
+    break;
+  case RISCV::QC_E_BGEI:
+    Cond[0].setImm(RISCV::QC_E_BLTI);
+    break;
+  case RISCV::QC_E_BLTI:
+    Cond[0].setImm(RISCV::QC_E_BGEI);
+    break;
+  case RISCV::QC_E_BGEUI:
+    Cond[0].setImm(RISCV::QC_E_BLTUI);
+    break;
+  case RISCV::QC_E_BLTUI:
+    Cond[0].setImm(RISCV::QC_E_BGEUI);
+    break;
+  case RISCV::NDS_BBC:
+    Cond[0].setImm(RISCV::NDS_BBS);
+    break;
+  case RISCV::NDS_BBS:
+    Cond[0].setImm(RISCV::NDS_BBC);
+    break;
+  case RISCV::NDS_BEQC:
+    Cond[0].setImm(RISCV::NDS_BNEC);
+    break;
+  case RISCV::NDS_BNEC:
+    Cond[0].setImm(RISCV::NDS_BEQC);
+    break;
+  }
+
   return false;
 }
 
@@ -1379,40 +1465,49 @@ bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
 }
 
 bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
+  bool IsSigned = false;
+  bool IsEquality = false;
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  case RISCV::BEQ:
+  case RISCV::BNE:
+    IsEquality = true;
+    break;
+  case RISCV::BGE:
+  case RISCV::BLT:
+    IsSigned = true;
+    break;
+  case RISCV::BGEU:
+  case RISCV::BLTU:
+    break;
+  }
+
   MachineBasicBlock *MBB = MI.getParent();
   MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
 
-  MachineBasicBlock *TBB, *FBB;
-  SmallVector<MachineOperand, 3> Cond;
-  if (analyzeBranch(*MBB, TBB, FBB, Cond, /*AllowModify=*/false))
-    return false;
+  const MachineOperand &LHS = MI.getOperand(0);
+  const MachineOperand &RHS = MI.getOperand(1);
+  MachineBasicBlock *TBB = MI.getOperand(2).getMBB();
 
-  RISCVCC::CondCode CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
+  RISCVCC::CondCode CC = getCondFromBranchOpc(MI.getOpcode());
   assert(CC != RISCVCC::COND_INVALID);
 
-  auto modifyBranch = [&]() {
-    // Build the new branch and remove the old one.
-    BuildMI(*MBB, MI, MI.getDebugLoc(),
-            getBrCond(static_cast<RISCVCC::CondCode>(Cond[0].getImm())))
-        .add(Cond[1])
-        .add(Cond[2])
-        .addMBB(TBB);
-    MI.eraseFromParent();
-  };
-
   // Canonicalize conditional branches which can be constant folded into
   // beqz or bnez.  We can't modify the CFG here.
   int64_t C0, C1;
-  if (isFromLoadImm(MRI, Cond[1], C0) && isFromLoadImm(MRI, Cond[2], C1)) {
-    unsigned NewCC =
-        evaluateCondBranch(CC, C0, C1) ? RISCVCC::COND_EQ : RISCVCC::COND_NE;
-    Cond[0] = MachineOperand::CreateImm(NewCC);
-    Cond[1] = Cond[2] = MachineOperand::CreateReg(RISCV::X0, /*isDef=*/false);
-    modifyBranch();
+  if (isFromLoadImm(MRI, LHS, C0) && isFromLoadImm(MRI, RHS, C1)) {
+    unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
+    // Build the new branch and remove the old one.
+    BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
+        .addReg(RISCV::X0)
+        .addReg(RISCV::X0)
+        .addMBB(TBB);
+    MI.eraseFromParent();
     return true;
   }
 
-  if (CC == RISCVCC::COND_EQ || CC == RISCVCC::COND_NE)
+  if (IsEquality)
     return false;
 
   // For two constants C0 and C1 from
@@ -1432,8 +1527,6 @@ bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
   //
   // To make sure this optimization is really beneficial, we only
   // optimize for cases where Y had only one use (i.e. only used by the branch).
-  MachineOperand &LHS = MI.getOperand(0);
-  MachineOperand &RHS = MI.getOperand(1);
   // Try to find the register for constant Z; return
   // invalid register otherwise.
   auto searchConst = [&](int64_t C1) -> Register {
@@ -1449,36 +1542,48 @@ bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
     return Register();
   };
 
-  if (isFromLoadImm(MRI, LHS, C0) && MRI.hasOneUse(LHS.getReg())) {
-    // Might be case 1.
-    // Signed integer overflow is UB. (UINT64_MAX is bigger so we don't need
-    // to worry about unsigned overflow here)
-    if (C0 < INT64_MAX)
-      if (Register RegZ = searchConst(C0 + 1)) {
-        reverseBranchCondition(Cond);
-        Cond[1] = MachineOperand::CreateReg(RHS.getReg(), /*isDef=*/false);
-        Cond[2] = MachineOperand::CreateReg(RegZ, /*isDef=*/false);
-        // We might extend the live range of Z, clear its kill flag to
-        // account for this.
-        MRI.clearKillFlags(RegZ);
-        modifyBranch();
-        return true;
-      }
-  } else if (isFromLoadImm(MRI, RHS, C0) && MRI.hasOneUse(RHS.getReg())) {
-    // Might be case 2.
-    // For unsigned cases, we don't want C1 to wrap back to UINT64_MAX
-    // when C0 is zero.
-    if ((CC == RISCVCC::COND_GE || CC == RISCVCC::COND_LT) || C0)
-      if (Register RegZ = searchConst(C0 - 1)) {
-        reverseBranchCondition(Cond);
-        Cond[1] = MachineOperand::CreateReg(RegZ, /*isDef=*/false);
-        Cond[2] = MachineOperand::CreateReg(LHS.getReg(), /*isDef=*/false);
-        // We might extend the live range of Z, clear its kill flag to
-        // account for this.
-        MRI.clearKillFlags(RegZ);
-        modifyBranch();
-        return true;
-      }
+  unsigned NewOpc = RISCVCC::getBrCond(getOppositeBranchCondition(CC));
+
+  // Might be case 1.
+  // Don't change 0 to 1 since we can use x0.
+  // For unsigned cases changing -1U to 0 would be incorrect.
+  // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
+  // return that.
+  if (isFromLoadImm(MRI, LHS, C0) && C0 != 0 && LHS.getReg().isVirtual() &&
+      MRI.hasOneUse(LHS.getReg()) && (IsSigned || C0 != -1)) {
+    assert(isInt<12>(C0) && "Unexpected immediate");
+    if (Register RegZ = searchConst(C0 + 1)) {
+      BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
+          .add(RHS)
+          .addReg(RegZ)
+          .addMBB(TBB);
+      // We might extend the live range of Z, clear its kill flag to
+      // account for this.
+      MRI.clearKillFlags(RegZ);
+      MI.eraseFromParent();
+      return true;
+    }
+  }
+
+  // Might be case 2.
+  // For signed cases we don't want to change 0 since we can use x0.
+  // For unsigned cases changing 0 to -1U would be incorrect.
+  // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
+  // return that.
+  if (isFromLoadImm(MRI, RHS, C0) && C0 != 0 && RHS.getReg().isVirtual() &&
+      MRI.hasOneUse(RHS.getReg())) {
+    assert(isInt<12>(C0) && "Unexpected immediate");
+    if (Register RegZ = searchConst(C0 - 1)) {
+      BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
+          .addReg(RegZ)
+          .add(LHS)
+          .addMBB(TBB);
+      // We might extend the live range of Z, clear its kill flag to
+      // account for this.
+      MRI.clearKillFlags(RegZ);
+      MI.eraseFromParent();
+      return true;
+    }
   }
 
   return false;
@@ -1501,6 +1606,11 @@ bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   switch (BranchOp) {
   default:
     llvm_unreachable("Unexpected opcode!");
+  case RISCV::NDS_BBC:
+  case RISCV::NDS_BBS:
+  case RISCV::NDS_BEQC:
+  case RISCV::NDS_BNEC:
+    return isInt<11>(BrOffset);
   case RISCV::BEQ:
   case RISCV::BNE:
   case RISCV::BLT:
@@ -1521,12 +1631,12 @@ bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   case RISCV::QC_E_BLTI:
   case RISCV::QC_E_BLTUI:
   case RISCV::QC_E_BGEUI:
-    return isIntN(13, BrOffset);
+    return isInt<13>(BrOffset);
   case RISCV::JAL:
   case RISCV::PseudoBR:
-    return isIntN(21, BrOffset);
+    return isInt<21>(BrOffset);
   case RISCV::PseudoJump:
-    return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
+    return isInt<32>(SignExtend64(BrOffset + 0x800, XLen));
   }
 }
 
@@ -1566,6 +1676,9 @@ unsigned getPredicatedOpcode(unsigned Opcode) {
   case RISCV::ANDN:  return RISCV::PseudoCCANDN;  break;
   case RISCV::ORN:   return RISCV::PseudoCCORN;   break;
   case RISCV::XNOR:  return RISCV::PseudoCCXNOR;  break;
+
+  case RISCV::NDS_BFOS:  return RISCV::PseudoCCNDS_BFOS;  break;
+  case RISCV::NDS_BFOZ:  return RISCV::PseudoCCNDS_BFOZ;  break;
   }
 
   return RISCV::INSTRUCTION_LIST_END;
@@ -1717,7 +1830,7 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   if (!MI.memoperands_empty()) {
     MachineMemOperand *MMO = *(MI.memoperands_begin());
     if (STI.hasStdExtZihintntl() && MMO->isNonTemporal()) {
-      if (STI.hasStdExtZca() && STI.enableRVCHintInstrs()) {
+      if (STI.hasStdExtZca()) {
         if (isCompressibleInst(MI, STI))
           return 4; // c.ntl.all + c.load/c.store
         return 6;   // c.ntl.all + load/store
@@ -2692,7 +2805,8 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         CASE_OPERAND_UIMM(6)
         CASE_OPERAND_UIMM(7)
         CASE_OPERAND_UIMM(8)
-        CASE_OPERAND_UIMM(10)
+        CASE_OPERAND_UIMM(9)
+	CASE_OPERAND_UIMM(10)
         CASE_OPERAND_UIMM(12)
         CASE_OPERAND_UIMM(16)
         CASE_OPERAND_UIMM(20)
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.h b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
index 020be91e90e0..785c8352d4a5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.h
@@ -41,25 +41,11 @@ enum CondCode {
   COND_GE,
   COND_LTU,
   COND_GEU,
-  COND_CV_BEQIMM,
-  COND_CV_BNEIMM,
-  COND_QC_BEQI,
-  COND_QC_BNEI,
-  COND_QC_BLTI,
-  COND_QC_BGEI,
-  COND_QC_BLTUI,
-  COND_QC_BGEUI,
-  COND_QC_E_BEQI,
-  COND_QC_E_BNEI,
-  COND_QC_E_BLTI,
-  COND_QC_E_BGEI,
-  COND_QC_E_BLTUI,
-  COND_QC_E_BGEUI,
   COND_INVALID
 };
 
 CondCode getOppositeBranchCondition(CondCode);
-unsigned getBrCond(CondCode CC);
+unsigned getBrCond(CondCode CC, unsigned SelectOpc = 0);
 
 } // end of namespace RISCVCC
 
@@ -79,7 +65,6 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
   explicit RISCVInstrInfo(RISCVSubtarget &STI);
 
   MCInst getNop() const override;
-  const MCInstrDesc &getBrCond(RISCVCC::CondCode CC) const;
 
   Register isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
@@ -325,9 +310,11 @@ class RISCVInstrInfo : public RISCVGenInstrInfo {
 #define GET_INSTRINFO_HELPER_DECLS
 #include "RISCVGenInstrInfo.inc"
 
+  static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc);
+
   /// Return the result of the evaluation of C0 CC C1, where CC is a
   /// RISCVCC::CondCode.
-  static bool evaluateCondBranch(unsigned CC, int64_t C0, int64_t C1);
+  static bool evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0, int64_t C1);
 
   /// Return true if the operand is a load immediate instruction and
   /// sets Imm to the immediate value.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index aa1ebba56782..8f8fb6eba9a6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -321,7 +321,7 @@ def uimm6 : RISCVUImmLeafOp<6>;
 def uimm7_opcode : RISCVUImmOp<7> {
   let ParserMatchClass = InsnDirectiveOpcode;
 }
-def uimm7 : RISCVUImmOp<7>;
+def uimm7 : RISCVUImmLeafOp<7>;
 def uimm8 : RISCVUImmOp<8>;
 def uimm16 : RISCVUImmOp<16>;
 def uimm32 : RISCVUImmOp<32>;
@@ -502,6 +502,12 @@ def uimm6gt32 : ImmLeaf<XLenVT, [{
 // Addressing modes.
 def AddrRegImm : ComplexPattern<iPTR, 2, "SelectAddrRegImm">;
 
+class AddrRegRegScale<int N>
+    : ComplexPattern<iPTR, 3, "SelectAddrRegRegScale<"#N#">">;
+class AddrRegZextRegScale<int N>
+    : ComplexPattern<i64, 3, "SelectAddrRegZextRegScale<"#N#", 32>",
+                     [], [], 10>;
+
 // Return the negation of an immediate value.
 def NegImm : SDNodeXForm<imm, [{
   return CurDAG->getSignedTargetConstant(-N->getSExtValue(), SDLoc(N),
@@ -1609,6 +1615,10 @@ def riscv_seteq : ComplexPattern<XLenVT, 1, "selectSETEQ", [setcc]>;
 def : Pat<(riscv_seteq (XLenVT GPR:$rs1)), (SLTIU GPR:$rs1, 1)>;
 def : Pat<(riscv_setne (XLenVT GPR:$rs1)), (SLTU (XLenVT X0), GPR:$rs1)>;
 def : Pat<(XLenVT (setne (XLenVT GPR:$rs1), -1)), (SLTIU GPR:$rs1, -1)>;
+def : Pat<(XLenVT (seteq (XLenVT (and GPR:$rs, immop_oneuse<TrailingOnesMask>:$mask)), 0)),
+          (SLTIU (XLenVT (SLLI GPR:$rs, (XLenSubTrailingOnes imm:$mask))), 1)>;
+def : Pat<(XLenVT (setne (XLenVT (and GPR:$rs, immop_oneuse<TrailingOnesMask>:$mask)), 0)),
+          (SLTU (XLenVT X0), (XLenVT (SLLI GPR:$rs, (XLenSubTrailingOnes imm:$mask))))>;
 
 def IntCCtoRISCVCC : SDNodeXForm<riscv_selectcc, [{
   ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
@@ -2267,6 +2277,9 @@ include "RISCVInstrInfoV.td"
 include "RISCVInstrInfoZvk.td"
 include "RISCVInstrInfoZvqdotq.td"
 
+// Packed SIMD
+include "RISCVInstrInfoP.td"
+
 // Compressed
 include "RISCVInstrInfoC.td"
 include "RISCVInstrInfoZc.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index 8b1eca45d82d..8252a9b170eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -608,7 +608,7 @@ def C_UNIMP : RVInst16<(outs), (ins), "c.unimp", "", [], InstFormatOther>,
 // HINT Instructions
 //===----------------------------------------------------------------------===//
 
-let Predicates = [HasStdExtZca, HasRVCHints], hasSideEffects = 0, mayLoad = 0,
+let Predicates = [HasStdExtZca], hasSideEffects = 0, mayLoad = 0,
     mayStore = 0 in {
 
 def C_NOP_HINT : RVInst16CI<0b000, 0b01, (outs), (ins simm6nonzero:$imm),
@@ -691,24 +691,24 @@ def C_SRAI64_HINT : RVInst16CB<0b100, 0b01, (outs GPRC:$rd),
   let Inst{12} = 0;
 }
 
-} // Predicates = [HasStdExtZca, HasRVCHints], hasSideEffects = 0, mayLoad = 0,
+} // Predicates = [HasStdExtZca], hasSideEffects = 0, mayLoad = 0,
   // mayStore = 0
 
 //===----------------------------------------------------------------------===//
 // Assembler Pseudo Instructions
 //===----------------------------------------------------------------------===//
 
-let Predicates = [HasStdExtZca, HasRVCHints] in {
+let Predicates = [HasStdExtZca] in {
 // Just a different syntax for the c.nop hint: c.addi x0, simm6 vs c.nop simm6.
 def : InstAlias<"c.addi x0, $imm", (C_NOP_HINT simm6nonzero:$imm), 0>;
 }
 
-let Predicates = [HasStdExtC, HasRVCHints, HasStdExtZihintntl] in {
+let Predicates = [HasStdExtC, HasStdExtZihintntl] in {
 def : InstAlias<"c.ntl.p1", (C_ADD_HINT X0, X2)>;
 def : InstAlias<"c.ntl.pall", (C_ADD_HINT X0, X3)>;
 def : InstAlias<"c.ntl.s1", (C_ADD_HINT X0, X4)>;
 def : InstAlias<"c.ntl.all", (C_ADD_HINT X0, X5)>;
-} // Predicates = [HasStdExtC, HasRVCHints, HasStdExtZihintntl]
+} // Predicates = [HasStdExtC, HasStdExtZihintntl]
 
 let EmitPriority = 0 in {
 let Predicates = [HasStdExtZca] in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
new file mode 100644
index 000000000000..aa9e7b5635de
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -0,0 +1,163 @@
+//===-- RISCVInstrInfoP.td - RISC-V 'P' instructions -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the RISC-V instructions from the standard 'Base P'
+// Packed SIMD instruction set extension.
+//
+//  This version is still experimental as the 'P' extension hasn't been
+//  ratified yet.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand and SDNode transformation definitions.
+//===----------------------------------------------------------------------===//
+
+def simm10 : RISCVSImmLeafOp<10>;
+
+// A 10-bit signed immediate allowing range [-512, 1023]
+// but will decode to [-512, 511].
+def simm10_unsigned : RISCVOp {
+  let ParserMatchClass = SImmAsmOperand<10, "Unsigned">;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<10>";
+  let OperandType = "OPERAND_SIMM10_UNSIGNED";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return isInt<10>(Imm) || isUInt<10>(Imm);
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Instruction class templates
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryImm10<bits<7> funct7, string opcodestr,
+                    DAGOperand TyImm10 = simm10>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins TyImm10:$imm10),
+                  opcodestr, "$rd, $imm10"> {
+  bits<10> imm10;
+
+  let Inst{31-25} = funct7;
+  let Inst{24-16} = imm10{8-0};
+  let Inst{15}    = imm10{9};
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryImm8<bits<8> funct8, string opcodestr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins uimm8:$uimm8),
+                  opcodestr, "$rd, $uimm8"> {
+  bits<8> uimm8;
+
+  let Inst{31-24} = funct8;
+  let Inst{23-16} = uimm8;
+  let Inst{15}    = 0b0;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnary<bits<3> f, string opcodestr, dag operands, string argstr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), operands, opcodestr, argstr> {
+  bits<5> imm;
+  bits<5> rs1;
+
+  let Inst{31}    = 0b1;
+  let Inst{30-28} = f;
+  let Inst{27}    = 0b0;
+  let Inst{19-15} = rs1;
+}
+
+class RVPUnaryImm5<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm5:$uimm5), "$rd, $rs1, $uimm5"> {
+  bits<5> uimm5;
+
+  let imm = uimm5;
+  let Inst{26-25} = 0b01;
+  let Inst{24-20} = uimm5;
+}
+
+class RVPUnaryImm4<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm4:$uimm4), "$rd, $rs1, $uimm4"> {
+  bits<4> uimm4;
+
+  let Inst{26-24} = 0b001;
+  let Inst{23-20} = uimm4;
+}
+
+class RVPUnaryImm3<bits<3> f, string opcodestr>
+    : RVPUnary<f, opcodestr, (ins GPR:$rs1, uimm3:$uimm3), "$rd, $rs1, $uimm3"> {
+  bits<3> uimm3;
+
+  let Inst{26-23} = 0b0001;
+  let Inst{22-20} = uimm3;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPUnaryWUF<bits<2> w, bits<5> uf, string opcodestr>
+    : RVInstIBase<0b010, OPC_OP_IMM_32, (outs GPR:$rd), (ins GPR:$rs1),
+                  opcodestr, "$rd, $rs1">  {
+  let Inst{31-27} = 0b11100;
+  let Inst{26-25} = w;
+  let Inst{24-20} = uf;
+}
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtP] in {
+def CLS    : Unary_r<0b011000000011, 0b001, "cls">;
+def ABS    : Unary_r<0b011000000111, 0b001, "abs">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32] in
+def REV_RV32  : Unary_r<0b011010011111, 0b101, "rev">;
+
+let Predicates = [HasStdExtP, IsRV64] in {
+def REV16      : Unary_r<0b011010110000, 0b101, "rev16">;
+def REV_RV64   : Unary_r<0b011010111111, 0b101, "rev">;
+
+def CLSW  : UnaryW_r<0b011000000011, 0b001, "clsw">;
+def ABSW  : UnaryW_r<0b011000000111, 0b001, "absw">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+def PSLLI_B  : RVPUnaryImm3<0b000, "pslli.b">;
+def PSLLI_H  : RVPUnaryImm4<0b000, "pslli.h">;
+def PSSLAI_H : RVPUnaryImm4<0b101, "psslai.h">;
+} // Predicates = [HasStdExtP]
+let DecoderNamespace = "RV32Only",
+    Predicates = [HasStdExtP, IsRV32] in
+def SSLAI    : RVPUnaryImm5<0b101, "sslai">;
+let Predicates = [HasStdExtP, IsRV64] in {
+def PSLLI_W  : RVPUnaryImm5<0b000, "pslli.w">;
+def PSSLAI_W : RVPUnaryImm5<0b101, "psslai.w">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in
+def PLI_H : RVPUnaryImm10<0b1011000, "pli.h">;
+let Predicates = [HasStdExtP, IsRV64] in
+def PLI_W : RVPUnaryImm10<0b1011001, "pli.w">;
+let Predicates = [HasStdExtP] in
+def PLI_B : RVPUnaryImm8<0b10110100, "pli.b">;
+
+let Predicates = [HasStdExtP] in {
+def PSEXT_H_B : RVPUnaryWUF<0b00, 0b00100, "psext.h.b">;
+def PSABS_H   : RVPUnaryWUF<0b00, 0b00111, "psabs.h">;
+def PSABS_B   : RVPUnaryWUF<0b10, 0b00111, "psabs.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV64] in {
+def PSEXT_W_B      : RVPUnaryWUF<0b01, 0b00100, "psext.w.b">;
+def PSEXT_W_H      : RVPUnaryWUF<0b01, 0b00101, "psext.w.h">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in
+def PLUI_H : RVPUnaryImm10<0b1111000, "plui.h", simm10_unsigned>;
+let Predicates = [HasStdExtP, IsRV64] in
+def PLUI_W : RVPUnaryImm10<0b1111001, "plui.w", simm10_unsigned>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 9c03c7c83af0..de9e55beb6a5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -351,10 +351,13 @@ defset list<VTypeInfo> AllVectors = {
         def VF16MF4: VTypeInfo<vfloat16mf4_t, vbool64_t, 16, V_MF4, f16, FPR16>;
         def VF16MF2: VTypeInfo<vfloat16mf2_t, vbool32_t, 16, V_MF2, f16, FPR16>;
         def VF32MF2: VTypeInfo<vfloat32mf2_t, vbool64_t, 32, V_MF2, f32, FPR32>;
+        def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
+        def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
       }
       def VF16M1: VTypeInfo<vfloat16m1_t, vbool16_t, 16, V_M1, f16, FPR16>;
       def VF32M1: VTypeInfo<vfloat32m1_t, vbool32_t, 32, V_M1, f32, FPR32>;
       def VF64M1: VTypeInfo<vfloat64m1_t, vbool64_t, 64, V_M1, f64, FPR64>;
+      def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
     }
 
     defset list<GroupVTypeInfo> GroupFloatVectors = {
@@ -378,19 +381,7 @@ defset list<VTypeInfo> AllVectors = {
                                  V_M4, f64, FPR64>;
       def VF64M8: GroupVTypeInfo<vfloat64m8_t, vfloat64m1_t, vbool8_t,  64,
                                  V_M8, f64, FPR64>;
-    }
-  }
 
-  defset list<VTypeInfo> AllBFloatVectors = {
-    defset list<VTypeInfo> NoGroupBFloatVectors = {
-      defset list<VTypeInfo> FractionalGroupBFloatVectors = {
-        def VBF16MF4: VTypeInfo<vbfloat16mf4_t, vbool64_t, 16, V_MF4, bf16, FPR16>;
-        def VBF16MF2: VTypeInfo<vbfloat16mf2_t, vbool32_t, 16, V_MF2, bf16, FPR16>;
-      }
-      def VBF16M1:  VTypeInfo<vbfloat16m1_t, vbool16_t, 16, V_M1, bf16, FPR16>;
-    }
-  
-    defset list<GroupVTypeInfo> GroupBFloatVectors = {
       def VBF16M2: GroupVTypeInfo<vbfloat16m2_t, vbfloat16m1_t, vbool8_t, 16,
                                   V_M2, bf16, FPR16>;
       def VBF16M4: GroupVTypeInfo<vbfloat16m4_t, vbfloat16m1_t, vbool4_t, 16,
@@ -401,7 +392,6 @@ defset list<VTypeInfo> AllVectors = {
   }
 }
 
-defvar AllFloatVectorsExceptFP16 = !filter(vti, AllFloatVectors, !ne(vti.Scalar, f16));
 defvar AllFP16Vectors = !filter(vti, AllFloatVectors, !eq(vti.Scalar, f16));
 
 // This functor is used to obtain the int vector type that has the same SEW and
@@ -4577,7 +4567,8 @@ multiclass VPatUnaryS_M<string intrinsic_name,
 multiclass VPatUnaryV_V_AnyMask<string intrinsic, string instruction,
                                 list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     def : VPatUnaryAnyMask<intrinsic, instruction, "VM",
                            vti.Vector, vti.Vector, vti.Mask,
                            vti.Log2SEW, vti.LMul, vti.RegClass, vti.RegClass>;
@@ -4896,7 +4887,7 @@ multiclass VPatBinaryV_VV_INT<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
     defvar ivti = GetIntVTypeInfo<vti>.Vti;
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic,
                       instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW,
                       vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
@@ -4913,15 +4904,16 @@ multiclass VPatBinaryV_VV_INT_EEW<string intrinsic, string instruction,
     defvar octuple_lmul = vlmul.octuple;
     defvar octuple_emul = !srl(!mul(octuple_lmul, eew), vti.Log2SEW);
     if !and(!ge(octuple_emul, 1), !le(octuple_emul, 64)) then {
+      defvar ivti = GetIntVTypeInfo<vti>.Vti;
       defvar emul_str = octuple_to_str<octuple_emul>.ret;
-      defvar ivti = !cast<VTypeInfo>("VI" # eew # emul_str);
       defvar inst = instruction # "_VV_" # vti.LMul.MX # "_E" # vti.SEW # "_" # emul_str;
-      let Predicates = !listconcat(GetVTypeMinimalPredicates<vti>.Predicates,
-                                   GetVTypeMinimalPredicates<ivti>.Predicates) in
+      defvar eivti = !cast<VTypeInfo>("VI" # eew # emul_str);
+      let Predicates = !listconcat(GetVTypePredicates<ivti>.Predicates,
+                                   GetVTypePredicates<eivti>.Predicates) in
       defm : VPatBinary<intrinsic, inst,
-                        vti.Vector, vti.Vector, ivti.Vector, vti.Mask,
+                        vti.Vector, vti.Vector, eivti.Vector, vti.Mask,
                         vti.Log2SEW, vti.RegClass,
-                        vti.RegClass, ivti.RegClass>;
+                        vti.RegClass, eivti.RegClass>;
     }
   }
 }
@@ -4958,12 +4950,14 @@ multiclass VPatBinaryV_VX_RM<string intrinsic, string instruction,
 
 multiclass VPatBinaryV_VX_INT<string intrinsic, string instruction,
                           list<VTypeInfo> vtilist> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic, instruction # "_VX_" # vti.LMul.MX,
                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
                       vti.Log2SEW, vti.RegClass,
                       vti.RegClass, GPR>;
+  }
 }
 
 multiclass VPatBinaryV_VI<string intrinsic, string instruction,
@@ -4990,12 +4984,14 @@ multiclass VPatBinaryV_VI_RM<string intrinsic, string instruction,
 
 multiclass VPatBinaryV_VI_INT<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist, Operand imm_type> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatBinary<intrinsic, instruction # "_VI_" # vti.LMul.MX,
                       vti.Vector, vti.Vector, XLenVT, vti.Mask,
                       vti.Log2SEW, vti.RegClass,
                       vti.RegClass, imm_type>;
+  }
 }
 
 multiclass VPatBinaryM_MM<string intrinsic, string instruction> {
@@ -5583,12 +5579,14 @@ multiclass VPatTernaryV_VV_AAXA_RM<string intrinsic, string instruction,
 
 multiclass VPatTernaryV_VX<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VX",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
                                  vti.RegClass, GPR>;
+  }
 }
 
 multiclass VPatTernaryV_VX_AAXA<string intrinsic, string instruction,
@@ -5615,12 +5613,14 @@ multiclass VPatTernaryV_VX_AAXA_RM<string intrinsic, string instruction,
 
 multiclass VPatTernaryV_VI<string intrinsic, string instruction,
                            list<VTypeInfo> vtilist, Operand Imm_type> {
-  foreach vti = vtilist in
-    let Predicates = GetVTypeMinimalPredicates<vti>.Predicates in
+  foreach vti = vtilist in {
+    defvar ivti = GetIntVTypeInfo<vti>.Vti;
+    let Predicates = GetVTypePredicates<ivti>.Predicates in
     defm : VPatTernaryWithPolicy<intrinsic, instruction, "VI",
                                  vti.Vector, vti.Vector, XLenVT, vti.Mask,
                                  vti.Log2SEW, vti.LMul, vti.RegClass,
                                  vti.RegClass, Imm_type>;
+  }
 }
 
 multiclass VPatTernaryW_VV<string intrinsic, string instruction,
@@ -7228,14 +7228,6 @@ foreach vti = AllFloatVectors in {
                                  vti.RegClass, vti.ScalarRegClass>;
 }
 
-foreach vti = AllBFloatVectors in
-  let Predicates = [HasVInstructionsBF16Minimal] in
-    defm : VPatBinaryCarryInTAIL<"int_riscv_vmerge", "PseudoVMERGE", "VVM",
-                                 vti.Vector,
-                                 vti.Vector, vti.Vector, vti.Mask,
-                                 vti.Log2SEW, vti.LMul, vti.RegClass,
-                                 vti.RegClass, vti.RegClass>;
-
 foreach fvti = AllFloatVectors in {
   defvar instr = !cast<Instruction>("PseudoVMERGE_VIM_"#fvti.LMul.MX);
   let Predicates = GetVTypePredicates<fvti>.Predicates in
@@ -7429,8 +7421,6 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
 
 defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
                                 AllFloatVectors, uimm5>;
-defm : VPatBinaryV_VV_VX_VI_INT<"int_riscv_vrgather", "PseudoVRGATHER",
-                                AllBFloatVectors, uimm5>;
 defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
                               eew=16, vtilist=AllFloatVectors>;
 //===----------------------------------------------------------------------===//
@@ -7438,7 +7428,6 @@ defm : VPatBinaryV_VV_INT_EEW<"int_riscv_vrgatherei16_vv", "PseudoVRGATHEREI16",
 //===----------------------------------------------------------------------===//
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllIntegerVectors>;
 defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllFloatVectors>;
-defm : VPatUnaryV_V_AnyMask<"int_riscv_vcompress", "PseudoVCOMPRESS", AllBFloatVectors>;
 
 // Include the non-intrinsic ISel patterns
 include "RISCVInstrInfoVVLPatterns.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 520959b0896f..b9b7ee4674ae 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1390,7 +1390,7 @@ defm : VPatFPSetCCSDNode_VV_VF_FV<SETOLE, "PseudoVMFLE", "PseudoVMFGE">;
 // Floating-point vselects:
 // 11.15. Vector Integer Merge Instructions
 // 13.15. Vector Floating-Point Merge Instruction
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<fvti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), fvti.RegClass:$rs1,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 6328e6c860f7..1f598863c8d0 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2580,7 +2580,7 @@ foreach vti = AllFloatVectors in {
   }
 }
 
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   // Floating-point vselects:
   // 11.15. Vector Integer Merge Instructions
   // 13.15. Vector Floating-Point Merge Instruction
@@ -2630,7 +2630,7 @@ foreach fvti = AllFloatVectors in {
   }
 }
 
-foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach fvti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<fvti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     // 13.16. Vector Floating-Point Move Instruction
@@ -2954,7 +2954,7 @@ foreach vti = NoGroupFloatVectors in {
   }
 }
 
-foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in {
+foreach vti = AllFloatVectors in {
   defvar ivti = GetIntVTypeInfo<vti>.Vti;
   let Predicates = GetVTypePredicates<ivti>.Predicates in {
     def : Pat<(vti.Vector
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
index 0e2fcd233615..e46b0d409eb5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
@@ -53,12 +53,24 @@ def simm20_lsb000 : Operand<XLenVT> {
   let DecoderMethod = "decodeSImmOperandAndLslN<20, 3>";
 }
 
+// Predicate: True if immediate is a power of 2.
+def PowerOf2 : PatLeaf<(imm), [{
+  return isPowerOf2_64(N->getZExtValue());
+}]>;
+
+// Transformation function: Get log2 of immediate.
+def Log2 : SDNodeXForm<imm, [{
+  uint64_t Imm = Log2_64(N->getZExtValue());
+  return CurDAG->getTargetConstant(Imm, SDLoc(N), N->getValueType(0));
+}]>;
+
 //===----------------------------------------------------------------------===//
 // Instruction Class Templates
 //===----------------------------------------------------------------------===//
 
 class NDSRVInstBB<bit cs, string opcodestr>
-    : RVInst<(outs), (ins GPR:$rs1, uimmlog2xlen:$cimm, bare_simm11_lsb0:$imm10),
+    : RVInst<(outs),
+             (ins GPR:$rs1, uimmlog2xlen:$cimm, bare_simm11_lsb0:$imm10),
              opcodestr, "$rs1, $cimm, $imm10", [], InstFormatNDS_BRANCH_10>,
       Sched<[WriteJmp, ReadIALU]> {
   bits<10> imm10;
@@ -106,7 +118,8 @@ class NDSRVInstBC<bits<3> funct3, string opcodestr>
 }
 
 class NDSRVInstBFO<bits<3> funct3, string opcodestr>
-    : RVInst<(outs GPR:$rd), (ins GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb),
+    : RVInst<(outs GPR:$rd),
+             (ins GPR:$rs1, uimmlog2xlen:$msb, uimmlog2xlen:$lsb),
              opcodestr, "$rd, $rs1, $msb, $lsb", [], InstFormatOther>,
       Sched<[WriteIALU, ReadIALU]> {
   bits<5> rd;
@@ -361,13 +374,13 @@ class NDSRVInstVD4DOT<bits<6> funct6, string opcodestr>
   let RVVConstraint = VMConstraint;
 }
 
-class NDSRVInstVBFHCvt<bits<7> funct7, bits<5> vs1, string opcodestr>
+class NDSRVInstVBFHCvt<bits<5> vs1, string opcodestr>
     : RVInst<(outs VR:$vd), (ins VR:$vs2, VMaskOp:$vm),
              opcodestr, "$vd, $vs2", [], InstFormatR> {
   bits<5> vs2;
   bits<5> vd;
 
-  let Inst{31-25} = funct7;
+  let Inst{31-25} = 0b0000000;
   let Inst{24-20} = vs2;
   let Inst{19-15} = vs1;
   let Inst{14-12} = 0b100;
@@ -380,6 +393,29 @@ class NDSRVInstVBFHCvt<bits<7> funct7, bits<5> vs1, string opcodestr>
   let Uses = [VL, VTYPE];
 }
 
+class NDSRVInstVLN<bits<5> funct5, string opcodestr>
+    : RVInst<(outs VR:$vd), (ins GPRMemZeroOffset:$rs1, VMaskOp:$vm),
+             opcodestr, "$vd, ${rs1}$vm", [], InstFormatR>,
+      VLESchedMC {
+  bits<5> rs1;
+  bits<5> vd;
+  bit vm;
+
+  let Inst{31-26} = 0b000001;
+  let Inst{25} = vm;
+  let Inst{24-20} = funct5;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b100;
+  let Inst{11-7} = vd;
+  let Inst{6-0} = OPC_CUSTOM_2.Value;
+  let hasSideEffects = 0;
+  let mayLoad = 1;
+  let mayStore = 0;
+
+  let Uses = [VTYPE, VL];
+  let RVVConstraint = VMConstraint;
+}
+
 //===----------------------------------------------------------------------===//
 // Multiclass
 //===----------------------------------------------------------------------===//
@@ -398,7 +434,8 @@ multiclass VPseudoVNCVT_BF16_S {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
     let VLMul = m.value, SEW=16 in
-    def "_" # m.MX : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.wvrclass, constraint>,
+    def "_" # m.MX : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.wvrclass,
+                                                    constraint>,
                      SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, 16,
                                 forcePassthruRead=true>;
   }
@@ -409,8 +446,10 @@ multiclass VPatConversionS_BF16<string intrinsic, string instruction> {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
     let Predicates = [HasVendorXAndesVBFHCvt] in
-    def : VPatUnaryNoMask<intrinsic, instruction, "BF16", fwti.Vector, fvti.Vector,
-                          fvti.Log2SEW, fvti.LMul, fwti.RegClass, fvti.RegClass>;
+    def : VPatUnaryNoMask<intrinsic, instruction, "BF16",
+                          fwti.Vector, fvti.Vector,
+                          fvti.Log2SEW, fvti.LMul,
+                          fwti.RegClass, fvti.RegClass>;
   }
 }
 
@@ -419,8 +458,10 @@ multiclass VPatConversionBF16_S<string intrinsic, string instruction> {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
     let Predicates = [HasVendorXAndesVBFHCvt] in
-    def : VPatUnaryNoMaskRoundingMode<intrinsic, instruction, "S", fvti.Vector, fwti.Vector,
-                                      fvti.Log2SEW, fvti.LMul, fvti.RegClass, fwti.RegClass>;
+    def : VPatUnaryNoMaskRoundingMode<intrinsic, instruction, "S",
+                                      fvti.Vector, fwti.Vector,
+                                      fvti.Log2SEW, fvti.LMul,
+                                      fvti.RegClass, fwti.RegClass>;
   }
 }
 
@@ -440,7 +481,8 @@ multiclass VPatVFPMADBinaryV_VX_RM<string intrinsic, string instruction,
                                    list<VTypeInfo> vtilist> {
   foreach vti = vtilist in {
     defvar kind = "V"#vti.ScalarSuffix;
-    defm : VPatBinaryRoundingMode<intrinsic, instruction#"_"#kind#"_"#vti.LMul.MX,
+    defm : VPatBinaryRoundingMode<intrinsic,
+                                  instruction#"_"#kind#"_"#vti.LMul.MX,
                                   vti.Vector, vti.Vector, f32, vti.Mask,
                                   vti.Log2SEW, vti.RegClass,
                                   vti.RegClass, FPR32>;
@@ -526,9 +568,18 @@ def NDS_SDGP  : NDSRVInstSDGP<0b111, "nds.sdgp">;
 let Predicates = [HasVendorXAndesVBFHCvt], Constraints = "@earlyclobber $vd",
     mayRaiseFPException = true in {
 let RVVConstraint = VS2Constraint, DestEEW = EEWSEWx2 in
-def NDS_VFWCVT_S_BF16 : NDSRVInstVBFHCvt<0b0000000, 0b00000, "nds.vfwcvt.s.bf16">;
+def NDS_VFWCVT_S_BF16 : NDSRVInstVBFHCvt<0b00000, "nds.vfwcvt.s.bf16">;
 let Uses = [FRM, VL, VTYPE] in
-def NDS_VFNCVT_BF16_S : NDSRVInstVBFHCvt<0b0000000, 0b00001, "nds.vfncvt.bf16.s">;
+def NDS_VFNCVT_BF16_S : NDSRVInstVBFHCvt<0b00001, "nds.vfncvt.bf16.s">;
+}
+
+//===----------------------------------------------------------------------===//
+// XAndesVSIntLoad
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasVendorXAndesVSIntLoad] in {
+def NDS_VLN8_V  : NDSRVInstVLN<0b00010, "nds.vln8.v">;
+def NDS_VLNU8_V : NDSRVInstVLN<0b00011, "nds.vlnu8.v">;
 }
 
 //===----------------------------------------------------------------------===//
@@ -556,8 +607,44 @@ def NDS_VD4DOTSU_VV : NDSRVInstVD4DOT<0b000101, "nds.vd4dotsu">;
 // Pseudo-instructions and codegen patterns
 //===----------------------------------------------------------------------===//
 
+class NDS_BBPat<CondCode Cond, NDSRVInstBB Inst>
+    : Pat<(riscv_brcc (and(XLenVT GPR:$rs1), PowerOf2:$mask), 0, Cond,
+              bb:$imm10),
+          (Inst GPR:$rs1, (Log2 PowerOf2:$mask), bare_simm11_lsb0:$imm10)>;
+
+class NDS_BCPat<CondCode Cond, NDSRVInstBC Inst>
+    : Pat<(riscv_brcc (XLenVT GPR:$rs1), uimm7:$cimm, Cond, bb:$imm10),
+          (Inst GPR:$rs1, uimm7:$cimm, bare_simm11_lsb0:$imm10)>;
+
+defm CC_UImmLog2XLen_NDS : SelectCC_GPR_riirr<GPR, uimmlog2xlen>;
+defm CC_UImm7_NDS        : SelectCC_GPR_riirr<GPR, uimm7>;
+
+class SelectNDS_BB<CondCode Cond>
+    : Pat<(riscv_selectcc_frag:$cc (and(XLenVT GPR:$lhs), PowerOf2:$mask), 0,
+              Cond, (XLenVT GPR:$truev), GPR:$falsev),
+          (Select_GPR_Using_CC_UImmLog2XLen_NDS GPR:$lhs, (Log2 PowerOf2:$mask),
+              (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+
+class SelectNDS_BC<CondCode Cond>
+    : Pat<(riscv_selectcc_frag:$cc (XLenVT GPR:$lhs), uimm7:$cimm, Cond,
+              (XLenVT GPR:$truev), GPR:$falsev),
+          (Select_GPR_Using_CC_UImm7_NDS GPR:$lhs, uimm7:$cimm,
+              (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
+
 let Predicates = [HasVendorXAndesPerf] in {
 
+def : NDS_BBPat<SETEQ, NDS_BBC>;
+def : NDS_BBPat<SETNE, NDS_BBS>;
+
+def : SelectNDS_BB<SETEQ>;
+def : SelectNDS_BB<SETNE>;
+
+def : NDS_BCPat<SETEQ, NDS_BEQC>;
+def : NDS_BCPat<SETNE, NDS_BNEC>;
+
+def : SelectNDS_BC<SETEQ>;
+def : SelectNDS_BC<SETNE>;
+
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i16), (NDS_BFOS GPR:$rs1, 15, 0)>;
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i8), (NDS_BFOS GPR:$rs1, 7, 0)>;
 def : Pat<(sext_inreg (XLenVT GPR:$rs1), i1), (NDS_BFOS GPR:$rs1, 0, 0)>;
@@ -592,8 +679,10 @@ defm PseudoNDS_VFWCVT_S_BF16 : VPseudoVWCVT_S_BF16;
 defm PseudoNDS_VFNCVT_BF16_S : VPseudoVNCVT_BF16_S;
 } // Predicates = [HasVendorXAndesVBFHCvt]
 
-defm : VPatConversionS_BF16<"int_riscv_nds_vfwcvt_s_bf16", "PseudoNDS_VFWCVT_S">;
-defm : VPatConversionBF16_S<"int_riscv_nds_vfncvt_bf16_s", "PseudoNDS_VFNCVT_BF16">;
+defm : VPatConversionS_BF16<"int_riscv_nds_vfwcvt_s_bf16",
+                            "PseudoNDS_VFWCVT_S">;
+defm : VPatConversionBF16_S<"int_riscv_nds_vfncvt_bf16_s",
+                            "PseudoNDS_VFNCVT_BF16">;
 
 let Predicates = [HasVendorXAndesVPackFPH],
     mayRaiseFPException = true in {
@@ -601,8 +690,10 @@ defm PseudoNDS_VFPMADT : VPseudoVFPMAD_VF_RM;
 defm PseudoNDS_VFPMADB : VPseudoVFPMAD_VF_RM;
 } // Predicates = [HasVendorXAndesVPackFPH]
 
-defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadt", "PseudoNDS_VFPMADT", AllFP16Vectors>;
-defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadb", "PseudoNDS_VFPMADB", AllFP16Vectors>;
+defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadt", "PseudoNDS_VFPMADT",
+                               AllFP16Vectors>;
+defm : VPatVFPMADBinaryV_VX_RM<"int_riscv_nds_vfpmadb", "PseudoNDS_VFPMADB",
+                               AllFP16Vectors>;
 
 let Predicates = [HasVendorXAndesVDot] in {
 defm PseudoNDS_VD4DOTS  : VPseudoVD4DOT_VV;
@@ -622,6 +713,29 @@ defset list<VTypeInfoToWide> AllQuadWidenableVD4DOTVectors = {
   def : VTypeInfoToWide<VI16M8,  VI64M8>;
 }
 
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dots", "PseudoNDS_VD4DOTS", AllQuadWidenableVD4DOTVectors>;
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotu", "PseudoNDS_VD4DOTU", AllQuadWidenableVD4DOTVectors>;
-defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotsu", "PseudoNDS_VD4DOTSU", AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dots", "PseudoNDS_VD4DOTS",
+                            AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotu", "PseudoNDS_VD4DOTU",
+                            AllQuadWidenableVD4DOTVectors>;
+defm : VPatTernaryVD4DOT_VV<"int_riscv_nds_vd4dotsu", "PseudoNDS_VD4DOTSU",
+                            AllQuadWidenableVD4DOTVectors>;
+
+//===----------------------------------------------------------------------===//
+// Pseudo-instructions for SFB (Short Forward Branch)
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasShortForwardBranchOpt], hasSideEffects = 0,
+    mayLoad = 0, mayStore = 0, Size = 8, Constraints = "$dst = $falsev" in {
+def PseudoCCNDS_BFOS : Pseudo<(outs GPR:$dst),
+                              (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                               GPR:$falsev, GPR:$rs1,
+                               uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
+                       Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
+                              ReadSFBALU]>;
+def PseudoCCNDS_BFOZ : Pseudo<(outs GPR:$dst),
+                              (ins GPR:$lhs, GPR:$rhs, ixlenimm:$cc,
+                               GPR:$falsev, GPR:$rs1,
+                               uimmlog2xlen:$msb, uimmlog2xlen:$lsb), []>,
+                       Sched<[WriteSFB, ReadSFBJmp, ReadSFBJmp, ReadSFBALU,
+                              ReadSFBALU]>;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
index 942dde4bab31..996e08bd0a27 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -789,13 +789,6 @@ let Predicates = [HasVendorXCValu, IsRV32], AddedComplexity = 1 in {
 // Patterns for immediate branching operations
 //===----------------------------------------------------------------------===//
 
-def IntCCtoRISCVCCCV : SDNodeXForm<riscv_selectcc, [{
-  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
-  assert(CC == ISD::SETEQ || CC == ISD::SETNE);
-  RISCVCC::CondCode BrCC = CC == ISD::SETEQ ? RISCVCC::COND_CV_BEQIMM : RISCVCC::COND_CV_BNEIMM;
-  return CurDAG->getTargetConstant(BrCC, SDLoc(N), Subtarget->getXLenVT());
-}]>;
-
 let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
   def : Pat<(riscv_brcc GPR:$rs1, simm5:$imm5, SETEQ, bb:$imm12),
             (CV_BEQIMM GPR:$rs1, simm5:$imm5, bare_simm13_lsb0_bb:$imm12)>;
@@ -808,7 +801,7 @@ let Predicates = [HasVendorXCVbi, IsRV32], AddedComplexity = 2 in {
       : Pat<(riscv_selectcc_frag:$cc (i32 GPR:$lhs), simm5:$Constant, Cond,
                                      (i32 GPR:$truev), GPR:$falsev),
             (Select_GPR_Using_CC_SImm5_CV GPR:$lhs, simm5:$Constant,
-             (IntCCtoRISCVCCCV $cc), GPR:$truev, GPR:$falsev)>;
+             (IntCCtoRISCVCC $cc), GPR:$truev, GPR:$falsev)>;
 
   def : Selectbi<SETEQ>;
   def : Selectbi<SETNE>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
index ff751994b89b..0c8487c2f5db 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
@@ -29,6 +29,12 @@ def uimm7_lsb000 : RISCVOp,
   }];
 }
 
+// A 9-bit unsigned offset
+def uimm9 : RISCVUImmOp<9>;
+
+// Custom prefetch ADDR selector
+def AddrRegImm9 : ComplexPattern<iPTR, 2, "SelectAddrRegImm9">;
+
 //===----------------------------------------------------------------------===//
 // MIPS custom instruction formats
 //===----------------------------------------------------------------------===//
@@ -103,9 +109,41 @@ class SWPFormat<dag outs, dag ins, string opcodestr, string argstr>
   let Inst{6-0} = OPC_CUSTOM_0.Value;
 }
 
+// Prefetch format.
+let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in
+class Mips_prefetch_ri<dag outs, dag ins, string opcodestr, string argstr>
+    : RVInst<outs, ins, opcodestr, argstr, [], InstFormatI> {
+  bits<9> imm9;
+  bits<5> rs1;
+  bits<5> hint;
+
+  let Inst{31-29} = 0b000;
+  let Inst{28-20} = imm9;
+  let Inst{19-15} = rs1;
+  let Inst{14-12} = 0b000;
+  let Inst{11-7} = hint;
+  let Inst{6-0} = OPC_CUSTOM_0.Value;
+}
+
 //===----------------------------------------------------------------------===//
 // MIPS extensions
 //===----------------------------------------------------------------------===//
+let Predicates = [HasVendorXMIPSCBOP] ,DecoderNamespace = "Xmipscbop" in {
+  def MIPS_PREFETCH : Mips_prefetch_ri<(outs), (ins GPR:$rs1, uimm9:$imm9, uimm5:$hint),
+                                        "mips.pref", "$hint, ${imm9}(${rs1})">,
+                       Sched<[]>;
+}
+
+let Predicates = [HasVendorXMIPSCBOP] in {
+  // Prefetch Data Write.
+  def : Pat<(prefetch (AddrRegImm9  (XLenVT GPR:$rs1), uimm9:$imm9),
+             (i32 1), timm, (i32 1)),
+            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 9)>;
+  // Prefetch Data Read.
+  def : Pat<(prefetch (AddrRegImm9 (XLenVT GPR:$rs1), uimm9:$imm9),
+             (i32 0), timm, (i32 1)),
+            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 8)>;
+}
 
 let Predicates = [HasVendorXMIPSCMov], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
                  DecoderNamespace = "Xmipscmov" in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index affe3a847675..6fe747800b5a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -160,10 +160,10 @@ multiclass CustomSiFiveVCIX<string suffix, VCIXType type,
                             DAGOperand InTyRd, DAGOperand InTyRs2,
                             DAGOperand InTyRs1> {
   let vm = 1 in
-  defm VC_ # NAME   : CustomSiFiveVCIXorVCIF<suffix, type, InTyRd, InTyRs2,
+  defm SF_VC_ # NAME   : CustomSiFiveVCIXorVCIF<suffix, type, InTyRd, InTyRs2,
                                              InTyRs1, 0>;
   let vm = 0 in
-  defm VC_V_ # NAME : CustomSiFiveVCIXorVCIF<suffix, type, VR, InTyRs2,
+  defm SF_VC_V_ # NAME : CustomSiFiveVCIXorVCIF<suffix, type, VR, InTyRs2,
                                              InTyRs1, 1>;
 }
 
@@ -201,29 +201,29 @@ let Predicates = [HasVendorXSfvcp], mayLoad = 0, mayStore = 0,
 
 let Predicates = [HasVendorXSfvqmaccdod], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx4, RVVConstraint=VS2Constraint in {
-  def VQMACCU_2x8x2  : CustomSiFiveVMACC<0b101100, OPMVV, "sf.vqmaccu.2x8x2">;
-  def VQMACC_2x8x2   : CustomSiFiveVMACC<0b101101, OPMVV, "sf.vqmacc.2x8x2">;
-  def VQMACCUS_2x8x2 : CustomSiFiveVMACC<0b101110, OPMVV, "sf.vqmaccus.2x8x2">;
-  def VQMACCSU_2x8x2 : CustomSiFiveVMACC<0b101111, OPMVV, "sf.vqmaccsu.2x8x2">;
+  def SF_VQMACCU_2x8x2  : CustomSiFiveVMACC<0b101100, OPMVV, "sf.vqmaccu.2x8x2">;
+  def SF_VQMACC_2x8x2   : CustomSiFiveVMACC<0b101101, OPMVV, "sf.vqmacc.2x8x2">;
+  def SF_VQMACCUS_2x8x2 : CustomSiFiveVMACC<0b101110, OPMVV, "sf.vqmaccus.2x8x2">;
+  def SF_VQMACCSU_2x8x2 : CustomSiFiveVMACC<0b101111, OPMVV, "sf.vqmaccsu.2x8x2">;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx4, RVVConstraint=WidenVNoMask in {
-  def VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
-  def VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
-  def VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
-  def VQMACCSU_4x8x4 : CustomSiFiveVMACC<0b111111, OPMVV, "sf.vqmaccsu.4x8x4">;
+  def SF_VQMACCU_4x8x4  : CustomSiFiveVMACC<0b111100, OPMVV, "sf.vqmaccu.4x8x4">;
+  def SF_VQMACC_4x8x4   : CustomSiFiveVMACC<0b111101, OPMVV, "sf.vqmacc.4x8x4">;
+  def SF_VQMACCUS_4x8x4 : CustomSiFiveVMACC<0b111110, OPMVV, "sf.vqmaccus.4x8x4">;
+  def SF_VQMACCSU_4x8x4 : CustomSiFiveVMACC<0b111111, OPMVV, "sf.vqmaccsu.4x8x4">;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq], DecoderNamespace = "XSfvector",
     DestEEW = EEWSEWx2, RVVConstraint=WidenVNoMask in {
-  def VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
+  def SF_VFWMACC_4x4x4 : CustomSiFiveVMACC<0b111100, OPFVV, "sf.vfwmacc.4x4x4">;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf], DecoderNamespace = "XSfvector",
     Uses = [FRM, VL, VTYPE] in {
-  def VFNRCLIP_XU_F_QF : CustomSiFiveVFNRCLIP<0b100010, OPFVF, "sf.vfnrclip.xu.f.qf">;
-  def VFNRCLIP_X_F_QF : CustomSiFiveVFNRCLIP<0b100011, OPFVF, "sf.vfnrclip.x.f.qf">;
+  def SF_VFNRCLIP_XU_F_QF : CustomSiFiveVFNRCLIP<0b100010, OPFVF, "sf.vfnrclip.xu.f.qf">;
+  def SF_VFNRCLIP_X_F_QF : CustomSiFiveVFNRCLIP<0b100011, OPFVF, "sf.vfnrclip.x.f.qf">;
 }
 
 class VPseudoVC_X<Operand OpClass, DAGOperand RS1Class> :
@@ -306,14 +306,14 @@ multiclass VPseudoVC_X<LMULInfo m, DAGOperand RS1Class,
                        Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_X<OpClass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -323,14 +323,14 @@ multiclass VPseudoVC_XV<LMULInfo m, DAGOperand RS1Class,
                         Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_XV<OpClass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -340,14 +340,14 @@ multiclass VPseudoVC_XVV<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
         : VPseudoVC_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX
+    def "PseudoSF_VC_V_" # NAME # "_" # m.MX
       : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
@@ -357,15 +357,15 @@ multiclass VPseudoVC_XVW<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in
-    def "PseudoVC_" # NAME # "_SE_" # m.MX
+    def "PseudoSF_VC_" # NAME # "_SE_" # m.MX
       : VPseudoVC_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
         Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
     let Constraints = "@earlyclobber $rd, $rd = $rs3" in {
       let Defs = [SF_VCIX_STATE], Uses = [SF_VCIX_STATE] in
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_SE_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
-      def "PseudoVC_V_" # NAME # "_" # m.MX
+      def "PseudoSF_VC_V_" # NAME # "_" # m.MX
         : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
           Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
@@ -391,7 +391,7 @@ multiclass VPseudoSiFiveVQMACCQOQ {
 }
 
 multiclass VPseudoSiFiveVFWMACC {
-  foreach m = MxListVF2 in
+  foreach m = MxListFW in
     let VLMul = m.value in
     defm NAME : VPseudoSiFiveVMACC<m.MX, m.wvrclass, m.vrclass>;
 }
@@ -435,26 +435,26 @@ let Predicates = [HasVendorXSfvcp] in {
 }
 
 let Predicates = [HasVendorXSfvqmaccdod] in {
-  defm VQMACCU_2x8x2  : VPseudoSiFiveVQMACCDOD;
-  defm VQMACC_2x8x2   : VPseudoSiFiveVQMACCDOD;
-  defm VQMACCUS_2x8x2 : VPseudoSiFiveVQMACCDOD;
-  defm VQMACCSU_2x8x2 : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCU_2x8x2  : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACC_2x8x2   : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCUS_2x8x2 : VPseudoSiFiveVQMACCDOD;
+  defm SF_VQMACCSU_2x8x2 : VPseudoSiFiveVQMACCDOD;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq] in {
-  defm VQMACCU_4x8x4  : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACC_4x8x4   : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACCUS_4x8x4 : VPseudoSiFiveVQMACCQOQ;
-  defm VQMACCSU_4x8x4 : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCU_4x8x4  : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACC_4x8x4   : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCUS_4x8x4 : VPseudoSiFiveVQMACCQOQ;
+  defm SF_VQMACCSU_4x8x4 : VPseudoSiFiveVQMACCQOQ;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq] in {
-  defm VFWMACC_4x4x4 : VPseudoSiFiveVFWMACC;
+  defm SF_VFWMACC_4x4x4 : VPseudoSiFiveVFWMACC;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf] in {
-  defm VFNRCLIP_XU_F_QF : VPseudoSiFiveVFNRCLIP;
-  defm VFNRCLIP_X_F_QF : VPseudoSiFiveVFNRCLIP;
+  defm SF_VFNRCLIP_XU_F_QF : VPseudoSiFiveVFNRCLIP;
+  defm SF_VFNRCLIP_X_F_QF : VPseudoSiFiveVFNRCLIP;
 }
 
 // SDNode
@@ -660,11 +660,11 @@ class VPatVC_V_OP3<string intrinsic_name,
 multiclass VPatVC_X<string intrinsic_suffix, string instruction_suffix,
                     VTypeInfo vti, ValueType type, DAGOperand kind> {
   def : VPatVC_V_OP3_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                         "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                         "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                          vti.Vector, XLenVT, type, vti.Log2SEW,
                          payload5, kind>;
   def : VPatVC_V_OP3<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      vti.Vector, XLenVT, type, vti.Log2SEW,
                      payload5, kind>;
 }
@@ -673,15 +673,15 @@ multiclass VPatVC_XV<string intrinsic_suffix, string instruction_suffix,
                      VTypeInfo vti, ValueType type, DAGOperand kind,
                      Operand op1_kind = payload2> {
   def : VPatVC_OP4_ISD<!cast<SDPatternOperator>("sf_vc_" # intrinsic_suffix # "_se"),
-                   "PseudoVC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                   "PseudoSF_VC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                    XLenVT, vti.Vector, type, vti.Log2SEW,
                    payload5, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP3_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                         "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                         "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                          vti.Vector, vti.Vector, type, vti.Log2SEW,
                          vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP3<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      vti.Vector, vti.Vector, type, vti.Log2SEW,
                      vti.RegClass, kind, op1_kind>;
 }
@@ -690,15 +690,15 @@ multiclass VPatVC_XVV<string intrinsic_suffix, string instruction_suffix,
                       VTypeInfo wti, VTypeInfo vti, ValueType type, DAGOperand kind,
                       Operand op1_kind = payload2> {
   def : VPatVC_OP4_ISD<!cast<SDPatternOperator>("sf_vc_" # intrinsic_suffix # "_se"),
-                   "PseudoVC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                   "PseudoSF_VC_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                    wti.Vector, vti.Vector, type, vti.Log2SEW,
                    wti.RegClass, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP4_ISD<!cast<SDPatternOperator>("sf_vc_v_" # intrinsic_suffix # "_se"),
-                     "PseudoVC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_SE_" # vti.LMul.MX,
                      wti.Vector, wti.Vector, vti.Vector, type, vti.Log2SEW,
                      wti.RegClass, vti.RegClass, kind, op1_kind>;
   def : VPatVC_V_OP4<"int_riscv_sf_vc_v_" # intrinsic_suffix,
-                     "PseudoVC_V_" # instruction_suffix # "_" # vti.LMul.MX,
+                     "PseudoSF_VC_V_" # instruction_suffix # "_" # vti.LMul.MX,
                      wti.Vector, wti.Vector, vti.Vector, type, vti.Log2SEW,
                      wti.RegClass, vti.RegClass, kind, op1_kind>;
 }
@@ -810,26 +810,26 @@ let Predicates = [HasVendorXSfvcp] in {
 }
 
 let Predicates = [HasVendorXSfvqmaccdod] in {
-  defm : VPatVQMACCDOD<"vqmaccu_2x8x2", "VQMACCU", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmacc_2x8x2", "VQMACC", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmaccus_2x8x2", "VQMACCUS", "2x8x2">;
-  defm : VPatVQMACCDOD<"vqmaccsu_2x8x2", "VQMACCSU", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccu_2x8x2", "SF_VQMACCU", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmacc_2x8x2", "SF_VQMACC", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccus_2x8x2", "SF_VQMACCUS", "2x8x2">;
+  defm : VPatVQMACCDOD<"vqmaccsu_2x8x2", "SF_VQMACCSU", "2x8x2">;
 }
 
 let Predicates = [HasVendorXSfvqmaccqoq] in {
-  defm : VPatVQMACCQOQ<"vqmaccu_4x8x4", "VQMACCU", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmacc_4x8x4", "VQMACC", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmaccus_4x8x4", "VQMACCUS", "4x8x4">;
-  defm : VPatVQMACCQOQ<"vqmaccsu_4x8x4", "VQMACCSU", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccu_4x8x4", "SF_VQMACCU", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmacc_4x8x4", "SF_VQMACC", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccus_4x8x4", "SF_VQMACCUS", "4x8x4">;
+  defm : VPatVQMACCQOQ<"vqmaccsu_4x8x4", "SF_VQMACCSU", "4x8x4">;
 }
 
 let Predicates = [HasVendorXSfvfwmaccqqq] in {
-  defm : VPatVFWMACC<"vfwmacc_4x4x4", "VFWMACC", "4x4x4">;
+  defm : VPatVFWMACC<"vfwmacc_4x4x4", "SF_VFWMACC", "4x4x4">;
 }
 
 let Predicates = [HasVendorXSfvfnrclipxfqf] in {
-  defm : VPatVFNRCLIP<"vfnrclip_xu_f_qf", "VFNRCLIP_XU_F_QF">;
-  defm : VPatVFNRCLIP<"vfnrclip_x_f_qf", "VFNRCLIP_X_F_QF">;
+  defm : VPatVFNRCLIP<"vfnrclip_xu_f_qf", "SF_VFNRCLIP_XU_F_QF">;
+  defm : VPatVFNRCLIP<"vfnrclip_x_f_qf", "SF_VFNRCLIP_X_F_QF">;
 }
 
 let Predicates = [HasVendorXSiFivecdiscarddlone] in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
index 89441444a994..c0f7ab127aeb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXTHead.td
@@ -476,10 +476,10 @@ def TH_FSURD : THStoreIndexed<FPR64, 0b01110, "th.fsurd">,
 } // DecoderNamespace = "XTHead"
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm THVdotVMAQA      : THVdotVMAQA<"th.vmaqa",     0b100000>;
-defm THVdotVMAQAU     : THVdotVMAQA<"th.vmaqau",    0b100010>;
-defm THVdotVMAQASU    : THVdotVMAQA<"th.vmaqasu",   0b100100>;
-defm THVdotVMAQAUS    : THVdotVMAQA_VX<"th.vmaqaus",0b100110>;
+defm TH_VMAQA      : THVdotVMAQA<"th.vmaqa",     0b100000>;
+defm TH_VMAQAU     : THVdotVMAQA<"th.vmaqau",    0b100010>;
+defm TH_VMAQASU    : THVdotVMAQA<"th.vmaqasu",   0b100100>;
+defm TH_VMAQAUS    : THVdotVMAQA_VX<"th.vmaqaus",0b100110>;
 }
 
 // Associate LMUL with tablegen records of register classes.
@@ -661,20 +661,20 @@ def : Pat<(i32 (sub GPR:$rd, (mul (sexti16 (i32 GPR:$rs1)),
 } // Predicates = [HasVendorXTHeadMac, IsRV32]
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm PseudoTHVdotVMAQA      : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQAU     : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQASU    : VPseudoVMAQA_VV_VX;
-defm PseudoTHVdotVMAQAUS    : VPseudoVMAQA_VX;
+defm PseudoTH_VMAQA      : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQAU     : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQASU    : VPseudoVMAQA_VV_VX;
+defm PseudoTH_VMAQAUS    : VPseudoVMAQA_VX;
 }
 
 let Predicates = [HasVendorXTHeadVdot] in {
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqa",  "PseudoTHVdotVMAQA",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqa",  "PseudoTH_VMAQA",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqau", "PseudoTHVdotVMAQAU",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqau", "PseudoTH_VMAQAU",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqasu","PseudoTHVdotVMAQASU",
+defm : VPatTernaryVMAQA_VV_VX<"int_riscv_th_vmaqasu","PseudoTH_VMAQASU",
                               AllQuadWidenableInt8NoVLMulVectors>;
-defm : VPatTernaryVMAQA_VX<"int_riscv_th_vmaqaus",   "PseudoTHVdotVMAQAUS",
+defm : VPatTernaryVMAQA_VX<"int_riscv_th_vmaqaus",   "PseudoTH_VMAQAUS",
                            AllQuadWidenableInt8NoVLMulVectors>;
 }
 
@@ -743,32 +743,30 @@ def TH_SYNC_I      : THCacheInst_void<0b11010, "th.sync.i">;
 def TH_SYNC_IS     : THCacheInst_void<0b11011, "th.sync.is">;
 }
 
-def AddrRegRegScale : ComplexPattern<iPTR, 3, "SelectAddrRegRegScale<3>">;
-def AddrRegZextRegScale
-    : ComplexPattern<i64, 3, "SelectAddrRegZextRegScale<3, 32>",
-                     [], [], 10>;
+def AddrRegRegScale3 : AddrRegRegScale<3>;
+def AddrRegZextRegScale3 : AddrRegZextRegScale<3>;
 
 multiclass LdIdxPat<PatFrag LoadOp, RVInst Inst, ValueType vt = XLenVT> {
-def : Pat<(vt (LoadOp (AddrRegRegScale (XLenVT GPR:$rs1), (XLenVT GPR:$rs2), uimm2:$uimm2))),
+def : Pat<(vt (LoadOp (AddrRegRegScale3 (XLenVT GPR:$rs1), (XLenVT GPR:$rs2), uimm2:$uimm2))),
           (Inst GPR:$rs1, GPR:$rs2, uimm2:$uimm2)>;
 }
 
 multiclass LdZextIdxPat<PatFrag LoadOp, RVInst Inst, ValueType vt = i64> {
-def : Pat<(vt (LoadOp (AddrRegZextRegScale (i64 GPR:$rs1), (i64 GPR:$rs2), uimm2:$uimm2))),
+def : Pat<(vt (LoadOp (AddrRegZextRegScale3 (i64 GPR:$rs1), (i64 GPR:$rs2), uimm2:$uimm2))),
           (Inst GPR:$rs1, GPR:$rs2, uimm2:$uimm2)>;
 }
 
 multiclass StIdxPat<PatFrag StoreOp, RVInst Inst, RegisterClass StTy,
                     ValueType vt = XLenVT> {
 def : Pat<(StoreOp (vt StTy:$rd),
-            (AddrRegRegScale (XLenVT GPR:$rs1), (XLenVT GPR:$rs2), uimm2:$uimm2)),
+            (AddrRegRegScale3 (XLenVT GPR:$rs1), (XLenVT GPR:$rs2), uimm2:$uimm2)),
           (Inst StTy:$rd, GPR:$rs1, GPR:$rs2, uimm2:$uimm2)>;
 }
 
 multiclass StZextIdxPat<PatFrag StoreOp, RVInst Inst, RegisterClass StTy,
                         ValueType vt = i64> {
 def : Pat<(StoreOp (vt StTy:$rd),
-            (AddrRegZextRegScale (i64 GPR:$rs1), (i64 GPR:$rs2), uimm2:$uimm2)),
+            (AddrRegZextRegScale3 (i64 GPR:$rs1), (i64 GPR:$rs2), uimm2:$uimm2)),
           (Inst StTy:$rd, GPR:$rs1, GPR:$rs2, uimm2:$uimm2)>;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index b52798edbe14..e8dd16471487 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -198,39 +198,6 @@ def AddLike: PatFrags<(ops node:$A, node:$B),
     return CurDAG->isBaseWithConstantOffset(SDValue(N, 0));
 }]>;
 
-def AddShl : PatFrag<(ops node:$Ra, node:$Rb, node:$SH3),
-                     (add node:$Ra, (shl node:$Rb, node:$SH3))>;
-
-def IntCCtoQCRISCVCC : SDNodeXForm<riscv_selectcc, [{
-  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
-  int64_t Imm = cast<ConstantSDNode>(N->getOperand(1))->getSExtValue();
-  RISCVCC::CondCode BrCC;
-  switch (CC) {
-  default:
-   report_fatal_error("Unexpected CondCode for Xqcibi branch instructions");
-  case ISD::SETEQ:
-    BrCC = isInt<5>(Imm) ? RISCVCC::COND_QC_BEQI :  RISCVCC::COND_QC_E_BEQI;
-    break;
-  case ISD::SETNE:
-    BrCC = isInt<5>(Imm) ? RISCVCC::COND_QC_BNEI :  RISCVCC::COND_QC_E_BNEI;
-    break;
-  case ISD::SETLT:
-    BrCC = isInt<5>(Imm) ? RISCVCC::COND_QC_BLTI :  RISCVCC::COND_QC_E_BLTI;
-    break;
-  case ISD::SETGE:
-    BrCC = isInt<5>(Imm) ? RISCVCC::COND_QC_BGEI :  RISCVCC::COND_QC_E_BGEI;
-    break;
-  case ISD::SETULT:
-    BrCC = isUInt<5>(Imm) ? RISCVCC::COND_QC_BLTUI :  RISCVCC::COND_QC_E_BLTUI;
-    break;
-  case ISD::SETUGE:
-    BrCC = isUInt<5>(Imm) ? RISCVCC::COND_QC_BGEUI :  RISCVCC::COND_QC_E_BGEUI;
-    break;
-  }
-  return CurDAG->getTargetConstant(BrCC, SDLoc(N), Subtarget->getXLenVT());
-}]>;
-
-
 //===----------------------------------------------------------------------===//
 // Instruction Formats
 //===----------------------------------------------------------------------===//
@@ -596,17 +563,20 @@ class QCILICC<bits<3> funct3, bits<2> funct2, DAGOperand InTyRs2, string opcodes
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class QCIMVCC<bits<3> funct3, string opcodestr>
-    : RVInstR4<0b00, funct3, OPC_CUSTOM_2, (outs GPRNoX0:$rd),
-               (ins GPRNoX0:$rs1, GPRNoX0:$rs2, GPRNoX0:$rs3),
-               opcodestr, "$rd, $rs1, $rs2, $rs3">;
+    : RVInstR4<0b00, funct3, OPC_CUSTOM_2, (outs GPRNoX0:$rd_wb),
+               (ins GPRNoX0:$rd, GPRNoX0:$rs1, GPRNoX0:$rs2, GPRNoX0:$rs3),
+               opcodestr, "$rd, $rs1, $rs2, $rs3"> {
+  let Constraints = "$rd = $rd_wb";
+}
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
 class QCIMVCCI<bits<3> funct3, string opcodestr, DAGOperand immType>
-    : RVInstR4<0b10, funct3, OPC_CUSTOM_2, (outs GPRNoX0:$rd),
-               (ins GPRNoX0:$rs1, immType:$imm, GPRNoX0:$rs3),
+    : RVInstR4<0b10, funct3, OPC_CUSTOM_2, (outs GPRNoX0:$rd_wb),
+               (ins GPRNoX0:$rd, GPRNoX0:$rs1, immType:$imm, GPRNoX0:$rs3),
                opcodestr, "$rd, $rs1, $imm, $rs3"> {
   bits<5> imm;
-
+  
+  let Constraints = "$rd = $rd_wb";
   let rs2 = imm;
 }
 
@@ -1324,14 +1294,24 @@ class QC48StPat<PatFrag StoreOp, RVInst48 Inst>
     : Pat<(StoreOp (i32 GPR:$rs2), (AddLike (i32 GPR:$rs1), simm26_nosimm12:$imm26)),
           (Inst GPR:$rs2, GPR:$rs1, simm26_nosimm12:$imm26)>;
 
+def AddrRegRegScale7 : AddrRegRegScale<7>;
+
 class QCScaledLdPat<PatFrag LoadOp, RVInst Inst>
-    : Pat<(i32 (LoadOp (AddShl (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt))),
+    : Pat<(i32 (LoadOp (AddrRegRegScale7 (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt))),
           (Inst GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt)>;
 
 class QCScaledStPat<PatFrag StoreOp, RVInst Inst>
-    : Pat<(StoreOp (i32 GPR:$rd), (AddShl (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt)),
+    : Pat<(StoreOp (i32 GPR:$rd), (AddrRegRegScale7 (i32 GPRMem:$rs1), (i32 GPRNoX0:$rs2), uimm3:$shamt)),
           (Inst GPR:$rd, GPRMem:$rs1, GPRNoX0:$rs2, uimm3:$shamt)>;
 
+class QCIMVCCPat<CondCode Cond, QCIMVCC Inst>
+    : Pat<(select (XLenVT (setcc (XLenVT GPRNoX0:$rs1), (XLenVT GPRNoX0:$rs2), Cond)), (XLenVT GPRNoX0:$rs3), (XLenVT GPRNoX0:$rd)),
+          (Inst GPRNoX0:$rd, GPRNoX0:$rs1, GPRNoX0:$rs2, GPRNoX0:$rs3)>;
+
+class QCIMVCCIPat<CondCode Cond, QCIMVCCI Inst>
+    : Pat<(select (XLenVT (setcc (XLenVT GPRNoX0:$rs1), simm5:$imm, Cond)), (XLenVT GPRNoX0:$rs3), (XLenVT GPRNoX0:$rd)),
+          (Inst GPRNoX0:$rd, GPRNoX0:$rs1, simm5:$imm, GPRNoX0:$rs3)>;
+
 // Match `riscv_brcc` and lower to the appropriate XQCIBI branch instruction.
 class BcciPat<CondCode Cond, QCIBranchInst_rii Inst, DAGOperand InTyImm>
     : Pat<(riscv_brcc (XLenVT GPRNoX0:$rs1), InTyImm:$rs2, Cond, bb:$imm12),
@@ -1350,7 +1330,7 @@ class SelectQCbi<CondCode Cond, DAGOperand InTyImm, Pseudo OpNode >
     : Pat<(riscv_selectcc_frag:$cc (i32 GPRNoX0:$lhs), InTyImm:$Constant, Cond,
                                    (i32 GPRNoX0:$truev), GPRNoX0:$falsev),
           (OpNode GPRNoX0:$lhs, InTyImm:$Constant,
-           (IntCCtoQCRISCVCC $cc), GPRNoX0:$truev, GPRNoX0:$falsev)>;
+           (IntCCtoRISCVCC $cc), GPRNoX0:$truev, GPRNoX0:$falsev)>;
 
 /// Simple arithmetic operations
 
@@ -1466,6 +1446,20 @@ def: Pat<(i32 (ctlz (not (i32 GPR:$rs1)))), (QC_CLO GPR:$rs1)>;
 let Predicates = [HasVendorXqciint, IsRV32] in
 def : Pat<(riscv_mileaveret_glue), (QC_C_MILEAVERET)>;
 
+let Predicates = [HasVendorXqcicm, IsRV32] in {
+def : Pat<(select (XLenVT GPRNoX0:$rs1), (XLenVT GPRNoX0:$rd),(XLenVT GPRNoX0:$rs3)),
+          (QC_MVEQI GPRNoX0:$rd, GPRNoX0:$rs1, (XLenVT 0), GPRNoX0:$rs3)>;
+
+def : QCIMVCCPat <SETEQ,  QC_MVEQ>;
+def : QCIMVCCPat <SETNE,  QC_MVNE>;
+def : QCIMVCCPat <SETLT,  QC_MVLT>;
+def : QCIMVCCPat <SETULT, QC_MVLTU>;
+
+def : QCIMVCCIPat <SETEQ,  QC_MVEQI>;
+def : QCIMVCCIPat <SETNE,  QC_MVNEI>;
+def : QCIMVCCIPat <SETLT,  QC_MVLTI>;
+def : QCIMVCCIPat <SETULT, QC_MVLTUI>;
+}
 
 //===----------------------------------------------------------------------===/i
 // Compress Instruction tablegen backend.
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
index e44bdcb4e2f0..878b85b14157 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
@@ -57,7 +57,7 @@ let Predicates = [HasStdExtZicboz] in {
 def CBO_ZERO : CBO_r<0b000000000100, "cbo.zero">, Sched<[]>;
 } // Predicates = [HasStdExtZicboz]
 
-let Predicates = [HasStdExtZicbop] in {
+let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
 def PREFETCH_I : Prefetch_ri<0b00000, "prefetch.i">, Sched<[]>;
 def PREFETCH_R : Prefetch_ri<0b00001, "prefetch.r">, Sched<[]>;
 def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
@@ -69,7 +69,7 @@ def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
 
 def AddrRegImmLsb00000 : ComplexPattern<iPTR, 2, "SelectAddrRegImmLsb00000">;
 
-let Predicates = [HasStdExtZicbop] in {
+let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
   def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
                       timm, timm, (i32 0)),
             (PREFETCH_I GPR:$rs1, simm12_lsb00000:$imm12)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
index 1057eeee31d6..4abe62f4e874 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrPredicates.td
@@ -54,6 +54,8 @@ def isSelectPseudo
                        Select_GPRNoX0_Using_CC_UImm5NonZero_QC,
                        Select_GPRNoX0_Using_CC_SImm16NonZero_QC,
                        Select_GPRNoX0_Using_CC_UImm16NonZero_QC,
+                       Select_GPR_Using_CC_UImmLog2XLen_NDS,
+                       Select_GPR_Using_CC_UImm7_NDS,
                        Select_FPR16_Using_CC_GPR,
                        Select_FPR16INX_Using_CC_GPR,
                        Select_FPR32_Using_CC_GPR,
diff --git a/llvm/lib/Target/RISCV/RISCVLateBranchOpt.cpp b/llvm/lib/Target/RISCV/RISCVLateBranchOpt.cpp
index d304d86fd7fa..28fb0b2da09d 100644
--- a/llvm/lib/Target/RISCV/RISCVLateBranchOpt.cpp
+++ b/llvm/lib/Target/RISCV/RISCVLateBranchOpt.cpp
@@ -54,7 +54,7 @@ bool RISCVLateBranchOpt::runOnBasicBlock(MachineBasicBlock &MBB) const {
   if (!TBB || Cond.size() != 3)
     return false;
 
-  RISCVCC::CondCode CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
+  RISCVCC::CondCode CC = RISCVInstrInfo::getCondFromBranchOpc(Cond[0].getImm());
   assert(CC != RISCVCC::COND_INVALID);
 
   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 57b415dc713a..03214fe8239c 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -120,7 +120,8 @@ def MIPS_P8700 : RISCVProcessorModel<"mips-p8700",
                                       FeatureStdExtZifencei,
                                       FeatureStdExtZicsr,
                                       FeatureVendorXMIPSCMov,
-                                      FeatureVendorXMIPSLSP],
+                                      FeatureVendorXMIPSLSP,
+                                      FeatureVendorXMIPSCBOP],
                                      [TuneMIPSP8700]>;
 
 def ROCKET_RV32 : RISCVProcessorModel<"rocket-rv32",
diff --git a/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp b/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
index 811db793ba0e..32d678c817d2 100644
--- a/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRedundantCopyElimination.cpp
@@ -72,12 +72,12 @@ guaranteesZeroRegInBlock(MachineBasicBlock &MBB,
                          MachineBasicBlock *TBB) {
   assert(Cond.size() == 3 && "Unexpected number of operands");
   assert(TBB != nullptr && "Expected branch target basic block");
-  auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
-  if (CC == RISCVCC::COND_EQ && Cond[2].isReg() &&
-      Cond[2].getReg() == RISCV::X0 && TBB == &MBB)
+  auto Opc = Cond[0].getImm();
+  if (Opc == RISCV::BEQ && Cond[2].isReg() && Cond[2].getReg() == RISCV::X0 &&
+      TBB == &MBB)
     return true;
-  if (CC == RISCVCC::COND_NE && Cond[2].isReg() &&
-      Cond[2].getReg() == RISCV::X0 && TBB != &MBB)
+  if (Opc == RISCV::BNE && Cond[2].isReg() && Cond[2].getReg() == RISCV::X0 &&
+      TBB != &MBB)
     return true;
   return false;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 78a176fcf18d..6600a00d4e09 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -1201,7 +1201,7 @@ multiclass SiFive7ReadAdvance {
   def : ReadAdvance<ReadVST4R, 0>;
   def : ReadAdvance<ReadVST8R, 0>;
 
-  // 12. Vector Integer Arithmetic Instructions
+  // 11. Vector Integer Arithmetic Instructions
   defm : LMULReadAdvance<"ReadVIALUV", 0>;
   defm : LMULReadAdvance<"ReadVIALUX", 0>;
   defm : LMULReadAdvanceW<"ReadVIWALUV", 0>;
@@ -1232,7 +1232,7 @@ multiclass SiFive7ReadAdvance {
   defm : LMULReadAdvance<"ReadVIMovV", 0>;
   defm : LMULReadAdvance<"ReadVIMovX", 0>;
 
-  // 13. Vector Fixed-Point Arithmetic Instructions
+  // 12. Vector Fixed-Point Arithmetic Instructions
   defm : LMULReadAdvance<"ReadVSALUV", 0>;
   defm : LMULReadAdvance<"ReadVSALUX", 0>;
   defm : LMULReadAdvance<"ReadVAALUV", 0>;
@@ -1244,7 +1244,7 @@ multiclass SiFive7ReadAdvance {
   defm : LMULReadAdvanceW<"ReadVNClipV", 0>;
   defm : LMULReadAdvanceW<"ReadVNClipX", 0>;
 
-  // 14. Vector Floating-Point Instructions
+  // 13. Vector Floating-Point Instructions
   defm : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
   defm : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
   defm : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
@@ -1280,7 +1280,7 @@ multiclass SiFive7ReadAdvance {
   defm : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
   defm : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
 
-  // 15. Vector Reduction Operations
+  // 14. Vector Reduction Operations
   def : ReadAdvance<ReadVIRedV, 0>;
   def : ReadAdvance<ReadVIRedV0, 0>;
   def : ReadAdvance<ReadVIWRedV, 0>;
@@ -1294,14 +1294,14 @@ multiclass SiFive7ReadAdvance {
   def : ReadAdvance<ReadVFWRedOV, 0>;
   def : ReadAdvance<ReadVFWRedOV0, 0>;
 
-  // 16. Vector Mask Instructions
+  // 15. Vector Mask Instructions
   defm : LMULReadAdvance<"ReadVMALUV", 0>;
   defm : LMULReadAdvance<"ReadVMPopV", 0>;
   defm : LMULReadAdvance<"ReadVMFFSV", 0>;
   defm : LMULReadAdvance<"ReadVMSFSV", 0>;
   defm : LMULReadAdvance<"ReadVIotaV", 0>;
 
-  // 17. Vector Permutation Instructions
+  // 16. Vector Permutation Instructions
   def : ReadAdvance<ReadVMovXS, 0>;
   def : ReadAdvance<ReadVMovSX_V, 0>;
   def : ReadAdvance<ReadVMovSX_X, 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index d7aa85048eed..2d9f38221d42 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -517,6 +517,8 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   case RISCV::VWSUB_VV:
   case RISCV::VWSUB_VX:
   case RISCV::VWSLL_VI:
+  case RISCV::VWSLL_VX:
+  case RISCV::VWSLL_VV:
   // Vector Widening Integer Multiply Instructions
   // Destination EEW=2*SEW. Source EEW=SEW.
   case RISCV::VWMUL_VV:
@@ -1019,6 +1021,8 @@ static bool isSupportedInstr(const MachineInstr &MI) {
 
   // Vector Crypto
   case RISCV::VWSLL_VI:
+  case RISCV::VWSLL_VX:
+  case RISCV::VWSLL_VV:
 
   // Vector Mask Instructions
   // Vector Mask-Register Logical Instructions
@@ -1040,6 +1044,13 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VMSOF_M:
   case RISCV::VIOTA_M:
   case RISCV::VID_V:
+  // Vector Slide Instructions
+  case RISCV::VSLIDEUP_VX:
+  case RISCV::VSLIDEUP_VI:
+  case RISCV::VSLIDEDOWN_VX:
+  case RISCV::VSLIDEDOWN_VI:
+  case RISCV::VSLIDE1UP_VX:
+  case RISCV::VFSLIDE1UP_VF:
   // Vector Single-Width Floating-Point Add/Subtract Instructions
   case RISCV::VFADD_VF:
   case RISCV::VFADD_VV:
@@ -1096,6 +1107,8 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VFSQRT_V:
   // Vector Floating-Point Reciprocal Square-Root Estimate Instruction
   case RISCV::VFRSQRT7_V:
+  // Vector Floating-Point Reciprocal Estimate Instruction
+  case RISCV::VFREC7_V:
   // Vector Floating-Point MIN/MAX Instructions
   case RISCV::VFMIN_VF:
   case RISCV::VFMIN_VV:
@@ -1119,6 +1132,10 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VMFLE_VV:
   case RISCV::VMFGT_VF:
   case RISCV::VMFGE_VF:
+  // Vector Floating-Point Merge Instruction
+  case RISCV::VFMERGE_VFM:
+  // Vector Floating-Point Move Instruction
+  case RISCV::VFMV_V_F:
   // Single-Width Floating-Point/Integer Type-Convert Instructions
   case RISCV::VFCVT_XU_F_V:
   case RISCV::VFCVT_X_F_V:
@@ -1252,6 +1269,9 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
     return false;
   }
 
+  assert(!RISCVII::elementsDependOnVL(RISCV::getRVVMCOpcode(MI.getOpcode())) &&
+         "Instruction shouldn't be supported if elements depend on VL");
+
   assert(MI.getOperand(0).isReg() &&
          isVectorRegClass(MI.getOperand(0).getReg(), MRI) &&
          "All supported instructions produce a vector register result");
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index f7acd676461f..07907298386c 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -67,12 +67,13 @@ class RISCVVectorPeephole : public MachineFunctionPass {
   bool convertSameMaskVMergeToVMv(MachineInstr &MI);
   bool foldUndefPassthruVMV_V_V(MachineInstr &MI);
   bool foldVMV_V_V(MachineInstr &MI);
+  bool foldVMergeToMask(MachineInstr &MI) const;
 
   bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;
   bool isAllOnesMask(const MachineInstr *MaskDef) const;
   std::optional<unsigned> getConstant(const MachineOperand &VL) const;
   bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const;
-  bool isKnownSameDefs(const MachineOperand &A, const MachineOperand &B) const;
+  bool isKnownSameDefs(Register A, Register B) const;
 };
 
 } // namespace
@@ -386,13 +387,23 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const {
   return true;
 }
 
-bool RISCVVectorPeephole::isKnownSameDefs(const MachineOperand &A,
-                                          const MachineOperand &B) const {
-  if (A.getReg().isPhysical() || B.getReg().isPhysical())
+bool RISCVVectorPeephole::isKnownSameDefs(Register A, Register B) const {
+  if (A.isPhysical() || B.isPhysical())
     return false;
 
-  return TRI->lookThruCopyLike(A.getReg(), MRI) ==
-         TRI->lookThruCopyLike(B.getReg(), MRI);
+  auto LookThruVirtRegCopies = [this](Register Reg) {
+    while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {
+      if (!Def->isFullCopy())
+        break;
+      Register Src = Def->getOperand(1).getReg();
+      if (!Src.isVirtual())
+        break;
+      Reg = Src;
+    }
+    return Reg;
+  };
+
+  return LookThruVirtRegCopies(A) == LookThruVirtRegCopies(B);
 }
 
 /// If a PseudoVMERGE_VVM's true operand is a masked pseudo and both have the
@@ -420,7 +431,7 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) {
   const MachineOperand &TrueMask =
       True->getOperand(TrueMaskedInfo->MaskOpIdx + True->getNumExplicitDefs());
   const MachineOperand &MIMask = MI.getOperand(4);
-  if (!isKnownSameDefs(TrueMask, MIMask))
+  if (!isKnownSameDefs(TrueMask.getReg(), MIMask.getReg()))
     return false;
 
   // True's passthru needs to be equivalent to False
@@ -669,6 +680,133 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
   return true;
 }
 
+/// Try to fold away VMERGE_VVM instructions into their operands:
+///
+/// %true = PseudoVADD_VV ...
+/// %x = PseudoVMERGE_VVM_M1 %false, %false, %true, %mask
+/// ->
+/// %x = PseudoVADD_VV_M1_MASK %false, ..., %mask
+///
+/// We can only fold if vmerge's passthru operand, vmerge's false operand and
+/// %true's passthru operand (if it has one) are the same. This is because we
+/// have to consolidate them into one passthru operand in the result.
+///
+/// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
+/// mask is all ones.
+///
+/// The resulting VL is the minimum of the two VLs.
+///
+/// The resulting policy is the effective policy the vmerge would have had,
+/// i.e. whether or not it's passthru operand was implicit-def.
+bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
+  if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMERGE_VVM)
+    return false;
+
+  Register PassthruReg = MI.getOperand(1).getReg();
+  Register FalseReg = MI.getOperand(2).getReg();
+  Register TrueReg = MI.getOperand(3).getReg();
+  if (!TrueReg.isVirtual() || !MRI->hasOneUse(TrueReg))
+    return false;
+  MachineInstr &True = *MRI->getUniqueVRegDef(TrueReg);
+  if (True.getParent() != MI.getParent())
+    return false;
+  const MachineOperand &MaskOp = MI.getOperand(4);
+  MachineInstr *Mask = MRI->getUniqueVRegDef(MaskOp.getReg());
+  assert(Mask);
+
+  const RISCV::RISCVMaskedPseudoInfo *Info =
+      RISCV::lookupMaskedIntrinsicByUnmasked(True.getOpcode());
+  if (!Info)
+    return false;
+
+  // If the EEW of True is different from vmerge's SEW, then we can't fold.
+  if (!hasSameEEW(MI, True))
+    return false;
+
+  // We require that either passthru and false are the same, or that passthru
+  // is undefined.
+  if (PassthruReg && !isKnownSameDefs(PassthruReg, FalseReg))
+    return false;
+
+  // If True has a passthru operand then it needs to be the same as vmerge's
+  // False, since False will be used for the result's passthru operand.
+  Register TruePassthru = True.getOperand(True.getNumExplicitDefs()).getReg();
+  if (RISCVII::isFirstDefTiedToFirstUse(True.getDesc()) && TruePassthru &&
+      !isKnownSameDefs(TruePassthru, FalseReg))
+    return false;
+
+  // Make sure it doesn't raise any observable fp exceptions, since changing the
+  // active elements will affect how fflags is set.
+  if (True.hasUnmodeledSideEffects() || True.mayRaiseFPException())
+    return false;
+
+  const MachineOperand &VMergeVL =
+      MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
+  const MachineOperand &TrueVL =
+      True.getOperand(RISCVII::getVLOpNum(True.getDesc()));
+
+  MachineOperand MinVL = MachineOperand::CreateImm(0);
+  if (RISCV::isVLKnownLE(TrueVL, VMergeVL))
+    MinVL = TrueVL;
+  else if (RISCV::isVLKnownLE(VMergeVL, TrueVL))
+    MinVL = VMergeVL;
+  else
+    return false;
+
+  unsigned RVVTSFlags =
+      TII->get(RISCV::getRVVMCOpcode(True.getOpcode())).TSFlags;
+  if (RISCVII::elementsDependOnVL(RVVTSFlags) && !TrueVL.isIdenticalTo(MinVL))
+    return false;
+  if (RISCVII::elementsDependOnMask(RVVTSFlags) && !isAllOnesMask(Mask))
+    return false;
+
+  // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
+  // operand is undefined.
+  //
+  // However, if the VL became smaller than what the vmerge had originally, then
+  // elements past VL that were previously in the vmerge's body will have moved
+  // to the tail. In that case we always need to use tail undisturbed to
+  // preserve them.
+  uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
+  if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL))
+    Policy |= RISCVVType::TAIL_AGNOSTIC;
+
+  assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
+         "Foldable unmasked pseudo should have a policy op already");
+
+  // Make sure the mask dominates True, otherwise move down True so it does.
+  // VL will always dominate since if it's a register they need to be the same.
+  if (!ensureDominates(MaskOp, True))
+    return false;
+
+  True.setDesc(TII->get(Info->MaskedPseudo));
+
+  // Insert the mask operand.
+  // TODO: Increment MaskOpIdx by number of explicit defs?
+  True.insert(True.operands_begin() + Info->MaskOpIdx +
+                  True.getNumExplicitDefs(),
+              MachineOperand::CreateReg(MaskOp.getReg(), false));
+
+  // Update the passthru, AVL and policy.
+  True.getOperand(True.getNumExplicitDefs()).setReg(FalseReg);
+  True.removeOperand(RISCVII::getVLOpNum(True.getDesc()));
+  True.insert(True.operands_begin() + RISCVII::getVLOpNum(True.getDesc()),
+              MinVL);
+  True.getOperand(RISCVII::getVecPolicyOpNum(True.getDesc())).setImm(Policy);
+
+  MRI->replaceRegWith(True.getOperand(0).getReg(), MI.getOperand(0).getReg());
+  // Now that True is masked, constrain its operands from vr -> vrnov0.
+  for (MachineOperand &MO : True.explicit_operands()) {
+    if (!MO.isReg() || !MO.getReg().isVirtual())
+      continue;
+    MRI->constrainRegClass(
+        MO.getReg(), True.getRegClassConstraint(MO.getOperandNo(), TII, TRI));
+  }
+  MI.eraseFromParent();
+
+  return true;
+}
+
 bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -685,6 +823,9 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   bool Changed = false;
 
   for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : make_early_inc_range(MBB))
+      Changed |= foldVMergeToMask(MI);
+
     for (MachineInstr &MI : make_early_inc_range(MBB)) {
       Changed |= convertToVLMAX(MI);
       Changed |= tryToReduceVL(MI);
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index 566b71dc367c..2a424e673ddf 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -369,22 +369,6 @@ void SPIRVInstPrinter::printUnknownType(const MCInst *MI, raw_ostream &O) {
   printRemainingVariableOps(MI, NumFixedOps, O, true);
 }
 
-static void printExpr(const MCExpr *Expr, raw_ostream &O) {
-#ifndef NDEBUG
-  const MCSymbolRefExpr *SRE;
-
-  if (const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr))
-    SRE = cast<MCSymbolRefExpr>(BE->getLHS());
-  else
-    SRE = cast<MCSymbolRefExpr>(Expr);
-
-  MCSymbolRefExpr::VariantKind Kind = SRE->getKind();
-
-  assert(Kind == MCSymbolRefExpr::VK_None);
-#endif
-  O << *Expr;
-}
-
 void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                     raw_ostream &O) {
   if (OpNo < MI->getNumOperands()) {
@@ -396,7 +380,7 @@ void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     else if (Op.isDFPImm())
       O << formatImm((double)Op.getDFPImm());
     else if (Op.isExpr())
-      printExpr(Op.getExpr(), O);
+      MAI.printExpr(O, *Op.getExpr());
     else
       llvm_unreachable("Unexpected operand type");
   }
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index f73a39c6ee9d..6ec7544767c5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -3086,43 +3086,11 @@ static SPIRVType *getCoopMatrType(const TargetExtType *ExtensionType,
       ExtensionType->getIntParameter(3), true);
 }
 
-static SPIRVType *
-getImageType(const TargetExtType *ExtensionType,
-             const SPIRV::AccessQualifier::AccessQualifier Qualifier,
-             MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
-  assert(ExtensionType->getNumTypeParameters() == 1 &&
-         "SPIR-V image builtin type must have sampled type parameter!");
-  const SPIRVType *SampledType =
-      GR->getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,
-                               SPIRV::AccessQualifier::ReadWrite, true);
-  assert((ExtensionType->getNumIntParameters() == 7 ||
-          ExtensionType->getNumIntParameters() == 6) &&
-         "Invalid number of parameters for SPIR-V image builtin!");
-
-  SPIRV::AccessQualifier::AccessQualifier accessQualifier =
-      SPIRV::AccessQualifier::None;
-  if (ExtensionType->getNumIntParameters() == 7) {
-    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
-                          ? SPIRV::AccessQualifier::WriteOnly
-                          : SPIRV::AccessQualifier::AccessQualifier(
-                                ExtensionType->getIntParameter(6));
-  }
-
-  // Create or get an existing type from GlobalRegistry.
-  return GR->getOrCreateOpTypeImage(
-      MIRBuilder, SampledType,
-      SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),
-      ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
-      ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
-      SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
-      accessQualifier);
-}
-
 static SPIRVType *getSampledImageType(const TargetExtType *OpaqueType,
                                       MachineIRBuilder &MIRBuilder,
                                       SPIRVGlobalRegistry *GR) {
-  SPIRVType *OpaqueImageType = getImageType(
-      OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder, GR);
+  SPIRVType *OpaqueImageType = GR->getImageType(
+      OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder);
   // Create or get an existing type from GlobalRegistry.
   return GR->getOrCreateOpTypeSampledImage(OpaqueImageType, MIRBuilder);
 }
@@ -3293,7 +3261,7 @@ SPIRVType *lowerBuiltinType(const Type *OpaqueType,
 
     switch (TypeRecord->Opcode) {
     case SPIRV::OpTypeImage:
-      TargetType = getImageType(BuiltinType, AccessQual, MIRBuilder, GR);
+      TargetType = GR->getImageType(BuiltinType, AccessQual, MIRBuilder);
       break;
     case SPIRV::OpTypePipe:
       TargetType = getPipeType(BuiltinType, MIRBuilder, GR);
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 401a762cd62a..6897865eb4e1 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -1634,6 +1634,7 @@ def : BuiltinType<"spirv.Event", OpTypeEvent>;
 def : BuiltinType<"spirv.Sampler", OpTypeSampler>;
 def : BuiltinType<"spirv.DeviceEvent", OpTypeDeviceEvent>;
 def : BuiltinType<"spirv.Image", OpTypeImage>;
+def : BuiltinType<"spirv.SignedImage", OpTypeImage>;
 def : BuiltinType<"spirv.SampledImage", OpTypeSampledImage>;
 def : BuiltinType<"spirv.Pipe", OpTypePipe>;
 def : BuiltinType<"spirv.CooperativeMatrixKHR", OpTypeCooperativeMatrixKHR>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index fbaca4e0e4d8..2726203d253a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -100,7 +100,9 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
          SPIRV::Extension::Extension::SPV_INTEL_ternary_bitwise_function},
         {"SPV_INTEL_2d_block_io",
          SPIRV::Extension::Extension::SPV_INTEL_2d_block_io},
-        {"SPV_INTEL_int4", SPIRV::Extension::Extension::SPV_INTEL_int4}};
+        {"SPV_INTEL_int4", SPIRV::Extension::Extension::SPV_INTEL_int4},
+        {"SPV_KHR_float_controls2",
+         SPIRV::Extension::Extension::SPV_KHR_float_controls2}};
 
 bool SPIRVExtensionsParser::parse(cl::Option &O, StringRef ArgName,
                                   StringRef ArgValue,
@@ -119,6 +121,13 @@ bool SPIRVExtensionsParser::parse(cl::Option &O, StringRef ArgName,
       continue;
     }
 
+    if (Token.size() == 3 && Token.upper() == "KHR") {
+      for (const auto &[ExtensionName, ExtensionEnum] : SPIRVExtensionMap)
+        if (StringRef(ExtensionName).starts_with("SPV_KHR_"))
+          EnabledExtensions.insert(ExtensionEnum);
+      continue;
+    }
+
     if (Token.empty() || (!Token.starts_with("+") && !Token.starts_with("-")))
       return O.error("Invalid extension list format: " + Token.str());
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index cc95fde6a516..b90e1aadbb5a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -663,7 +663,8 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
     auto *II = dyn_cast<IntrinsicInst>(I);
     if (II && II->getIntrinsicID() == Intrinsic::spv_resource_getpointer) {
       auto *HandleType = cast<TargetExtType>(II->getOperand(0)->getType());
-      if (HandleType->getTargetExtName() == "spirv.Image") {
+      if (HandleType->getTargetExtName() == "spirv.Image" ||
+          HandleType->getTargetExtName() == "spirv.SignedImage") {
         if (II->hasOneUse()) {
           auto *U = *II->users().begin();
           Ty = cast<Instruction>(U)->getAccessType();
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 292b83e05b56..83fccdc2bdba 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -1406,6 +1406,40 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateLayoutType(
   return SPIRVStructType;
 }
 
+SPIRVType *SPIRVGlobalRegistry::getImageType(
+    const TargetExtType *ExtensionType,
+    const SPIRV::AccessQualifier::AccessQualifier Qualifier,
+    MachineIRBuilder &MIRBuilder) {
+  assert(ExtensionType->getNumTypeParameters() == 1 &&
+         "SPIR-V image builtin type must have sampled type parameter!");
+  const SPIRVType *SampledType =
+      getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,
+                           SPIRV::AccessQualifier::ReadWrite, true);
+  assert((ExtensionType->getNumIntParameters() == 7 ||
+          ExtensionType->getNumIntParameters() == 6) &&
+         "Invalid number of parameters for SPIR-V image builtin!");
+
+  SPIRV::AccessQualifier::AccessQualifier accessQualifier =
+      SPIRV::AccessQualifier::None;
+  if (ExtensionType->getNumIntParameters() == 7) {
+    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly
+                          ? SPIRV::AccessQualifier::WriteOnly
+                          : SPIRV::AccessQualifier::AccessQualifier(
+                                ExtensionType->getIntParameter(6));
+  }
+
+  // Create or get an existing type from GlobalRegistry.
+  SPIRVType *R = getOrCreateOpTypeImage(
+      MIRBuilder, SampledType,
+      SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),
+      ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),
+      ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),
+      SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),
+      accessQualifier);
+  SPIRVToLLVMType[R] = ExtensionType;
+  return R;
+}
+
 SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage(
     MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim,
     uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled,
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 35f616a1981d..7ef812828b7c 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -501,6 +501,13 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                  MachineIRBuilder &MIRBuilder);
   bool hasBlockDecoration(SPIRVType *Type) const;
 
+  SPIRVType *
+  getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType,
+                         SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
+                         uint32_t Multisampled, uint32_t Sampled,
+                         SPIRV::ImageFormat::ImageFormat ImageFormat,
+                         SPIRV::AccessQualifier::AccessQualifier AccQual);
+
 public:
   Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder,
                             SPIRVType *SpvType, bool EmitIR,
@@ -607,11 +614,9 @@ class SPIRVGlobalRegistry : public SPIRVIRMapping {
                                    const TargetExtType *T, bool EmitIr = false);
 
   SPIRVType *
-  getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType,
-                         SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
-                         uint32_t Multisampled, uint32_t Sampled,
-                         SPIRV::ImageFormat::ImageFormat ImageFormat,
-                         SPIRV::AccessQualifier::AccessQualifier AccQual);
+  getImageType(const TargetExtType *ExtensionType,
+               const SPIRV::AccessQualifier::AccessQualifier Qualifier,
+               MachineIRBuilder &MIRBuilder);
 
   SPIRVType *getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder);
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 851e0c6b81fc..40a0bd97adaf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -341,6 +341,14 @@ class SPIRVInstructionSelector : public InstructionSelector {
                                 GIntrinsic &HandleDef, MachineInstr &Pos) const;
 };
 
+bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) {
+  const TargetExtType *TET = cast<TargetExtType>(HandleType);
+  if (TET->getTargetExtName() == "spirv.Image") {
+    return false;
+  }
+  assert(TET->getTargetExtName() == "spirv.SignedImage");
+  return TET->getTypeParameter(0)->isIntegerTy();
+}
 } // end anonymous namespace
 
 #define GET_GLOBALISEL_IMPL
@@ -1195,12 +1203,17 @@ bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
 
     Register IdxReg = IntPtrDef->getOperand(3).getReg();
     if (HandleType->getOpcode() == SPIRV::OpTypeImage) {
-      return BuildMI(*I.getParent(), I, I.getDebugLoc(),
-                     TII.get(SPIRV::OpImageWrite))
-          .addUse(NewHandleReg)
-          .addUse(IdxReg)
-          .addUse(StoreVal)
-          .constrainAllUses(TII, TRI, RBI);
+      auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                         TII.get(SPIRV::OpImageWrite))
+                     .addUse(NewHandleReg)
+                     .addUse(IdxReg)
+                     .addUse(StoreVal);
+
+      const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType);
+      if (sampledTypeIsSignedInteger(LLVMHandleType))
+        BMI.addImm(0x1000); // SignExtend
+
+      return BMI.constrainAllUses(TII, TRI, RBI);
     }
   }
 
@@ -3246,25 +3259,35 @@ bool SPIRVInstructionSelector::generateImageRead(Register &ResVReg,
                                                  Register ImageReg,
                                                  Register IdxReg, DebugLoc Loc,
                                                  MachineInstr &Pos) const {
+  SPIRVType *ImageType = GR.getSPIRVTypeForVReg(ImageReg);
+  assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage &&
+         "ImageReg is not an image type.");
+  bool IsSignedInteger =
+      sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType));
+
   uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType);
   if (ResultSize == 4) {
-    return BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
-        .addDef(ResVReg)
-        .addUse(GR.getSPIRVTypeID(ResType))
-        .addUse(ImageReg)
-        .addUse(IdxReg)
-        .constrainAllUses(TII, TRI, RBI);
+    auto BMI = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
+                   .addDef(ResVReg)
+                   .addUse(GR.getSPIRVTypeID(ResType))
+                   .addUse(ImageReg)
+                   .addUse(IdxReg);
+
+    if (IsSignedInteger)
+      BMI.addImm(0x1000); // SignExtend
+    return BMI.constrainAllUses(TII, TRI, RBI);
   }
 
   SPIRVType *ReadType = widenTypeToVec4(ResType, Pos);
   Register ReadReg = MRI->createVirtualRegister(GR.getRegClass(ReadType));
-  bool Succeed =
-      BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
-          .addDef(ReadReg)
-          .addUse(GR.getSPIRVTypeID(ReadType))
-          .addUse(ImageReg)
-          .addUse(IdxReg)
-          .constrainAllUses(TII, TRI, RBI);
+  auto BMI = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpImageRead))
+                 .addDef(ReadReg)
+                 .addUse(GR.getSPIRVTypeID(ReadType))
+                 .addUse(ImageReg)
+                 .addUse(IdxReg);
+  if (IsSignedInteger)
+    BMI.addImm(0x1000); // SignExtend
+  bool Succeed = BMI.constrainAllUses(TII, TRI, RBI);
   if (!Succeed)
     return false;
 
@@ -4106,6 +4129,7 @@ bool SPIRVInstructionSelector::loadHandleBeforePosition(
   // handle is the image object. So images get an extra load.
   uint32_t LoadOpcode =
       IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad;
+  GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF());
   return BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(),
                  TII.get(LoadOpcode))
       .addDef(HandleReg)
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index b71a9dd68dd4..ad976e528892 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -70,6 +70,8 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
   AvoidCapabilitiesSet AvoidCaps;
   if (!ST.isShader())
     AvoidCaps.S.insert(SPIRV::Capability::Shader);
+  else
+    AvoidCaps.S.insert(SPIRV::Capability::Kernel);
 
   VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);
   VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, i);
@@ -88,8 +90,11 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
   } else if (MinVerOK && MaxVerOK) {
     if (ReqCaps.size() == 1) {
       auto Cap = ReqCaps[0];
-      if (Reqs.isCapabilityAvailable(Cap))
+      if (Reqs.isCapabilityAvailable(Cap)) {
+        ReqExts.append(getSymbolicOperandExtensions(
+            SPIRV::OperandCategory::CapabilityOperand, Cap));
         return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+      }
     } else {
       // By SPIR-V specification: "If an instruction, enumerant, or other
       // feature specifies multiple enabling capabilities, only one such
@@ -103,8 +108,11 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
           UseCaps.push_back(Cap);
       for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
         auto Cap = UseCaps[i];
-        if (i == Sz - 1 || !AvoidCaps.S.contains(Cap))
+        if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) {
+          ReqExts.append(getSymbolicOperandExtensions(
+              SPIRV::OperandCategory::CapabilityOperand, Cap));
           return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+        }
       }
     }
   }
@@ -1975,6 +1983,14 @@ static unsigned getFastMathFlags(const MachineInstr &I) {
   return Flags;
 }
 
+static bool isFastMathMathModeAvailable(const SPIRVSubtarget &ST) {
+  if (ST.isKernel())
+    return true;
+  if (ST.getSPIRVVersion() < VersionTuple(1, 2))
+    return false;
+  return ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
+}
+
 static void handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST,
                                    const SPIRVInstrInfo &TII,
                                    SPIRV::RequirementHandler &Reqs) {
@@ -1998,8 +2014,12 @@ static void handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST,
   unsigned FMFlags = getFastMathFlags(I);
   if (FMFlags == SPIRV::FPFastMathMode::None)
     return;
-  Register DstReg = I.getOperand(0).getReg();
-  buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode, {FMFlags});
+
+  if (isFastMathMathModeAvailable(ST)) {
+    Register DstReg = I.getOperand(0).getReg();
+    buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode,
+                    {FMFlags});
+  }
 }
 
 // Walk all functions and add decorations related to MI flags.
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index f1aae42ea2be..548e9b717c16 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -319,6 +319,7 @@ defm SPV_INTEL_ternary_bitwise_function : ExtensionOperand<120>;
 defm SPV_INTEL_subgroup_matrix_multiply_accumulate : ExtensionOperand<121>;
 defm SPV_INTEL_2d_block_io : ExtensionOperand<122>;
 defm SPV_INTEL_int4 : ExtensionOperand<123>;
+defm SPV_KHR_float_controls2 : ExtensionOperand<124>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Capabilities enum values and at the same time
@@ -489,6 +490,8 @@ defm DotProductInput4x8Bit : CapabilityOperand<6017, 0x10600, 0, [SPV_KHR_intege
 defm DotProductInput4x8BitPacked : CapabilityOperand<6018, 0x10600, 0, [SPV_KHR_integer_dot_product], []>;
 defm DotProduct : CapabilityOperand<6019, 0x10600, 0, [SPV_KHR_integer_dot_product], []>;
 defm GroupNonUniformRotateKHR : CapabilityOperand<6026, 0, 0, [SPV_KHR_subgroup_rotate], [GroupNonUniform]>;
+defm FloatControls2
+    : CapabilityOperand<6029, 0x10200, 0, [SPV_KHR_float_controls2], []>;
 defm AtomicFloat32AddEXT : CapabilityOperand<6033, 0, 0, [SPV_EXT_shader_atomic_float_add], []>;
 defm AtomicFloat64AddEXT : CapabilityOperand<6034, 0, 0, [SPV_EXT_shader_atomic_float_add], []>;
 defm AtomicFloat16AddEXT : CapabilityOperand<6095, 0, 0, [SPV_EXT_shader_atomic_float16_add], []>;
@@ -1239,7 +1242,7 @@ defm XfbBuffer : DecorationOperand<36, 0, 0, [], [TransformFeedback]>;
 defm XfbStride : DecorationOperand<37, 0, 0, [], [TransformFeedback]>;
 defm FuncParamAttr : DecorationOperand<38, 0, 0, [], [Kernel]>;
 defm FPRoundingMode : DecorationOperand<39, 0, 0, [], []>;
-defm FPFastMathMode : DecorationOperand<40, 0, 0, [], [Kernel]>;
+defm FPFastMathMode : DecorationOperand<40, 0, 0, [], [Kernel, FloatControls2]>;
 defm LinkageAttributes : DecorationOperand<41, 0, 0, [], [Linkage]>;
 defm NoContraction : DecorationOperand<42, 0, 0, [], [Shader]>;
 defm InputAttachmentIndex : DecorationOperand<43, 0, 0, [], [InputAttachment]>;
diff --git a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index f1009999dc1b..7c1db3cfcd6b 100644
--- a/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/llvm/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -373,7 +373,7 @@ class SparcOperand : public MCParsedAsmOperand {
     return EndLoc;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:     OS << "Token: " << getToken() << "\n"; break;
     case k_Register:  OS << "Reg: #" << getReg() << "\n"; break;
@@ -381,9 +381,10 @@ class SparcOperand : public MCParsedAsmOperand {
     case k_MemoryReg: OS << "Mem: " << getMemBase() << "+"
                          << getMemOffsetReg() << "\n"; break;
     case k_MemoryImm: assert(getMemOff() != nullptr);
-      OS << "Mem: " << getMemBase()
-         << "+" << *getMemOff()
-         << "\n"; break;
+      OS << "Mem: " << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemOff());
+      OS << "\n";
+      break;
     case k_ASITag:
       OS << "ASI tag: " << getASITag() << "\n";
       break;
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index cc1b7e93b31c..ba023af06292 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -12,7 +12,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -182,15 +181,15 @@ std::optional<MCFixupKind> SparcAsmBackend::getFixupKind(StringRef Name) const {
 MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // clang-format off
   const static MCFixupKindInfo InfosBE[Sparc::NumTargetFixupKinds] = {
-    // name                    offset bits  flags
-    { "fixup_sparc_call30",     2,     30,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_sparc_13",        19,     13,  0 },
+      // name                    offset bits flags
+      {"fixup_sparc_call30",     2,     30,  0},
+      {"fixup_sparc_13",        19,     13,  0},
   };
 
   const static MCFixupKindInfo InfosLE[Sparc::NumTargetFixupKinds] = {
-    // name                    offset bits  flags
-    { "fixup_sparc_call30",     0,     30,  MCFixupKindInfo::FKF_IsPCRel },
-    { "fixup_sparc_13",         0,     13,  0 },
+      // name                    offset bits flags
+      {"fixup_sparc_call30",     0,     30,  0},
+      {"fixup_sparc_13",         0,     13,  0},
   };
   // clang-format on
 
@@ -208,22 +207,22 @@ MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   MCFixupKindInfo Info{};
   switch (uint16_t(Kind)) {
   case ELF::R_SPARC_PC10:
-    Info = {"", 22, 10, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 22, 10, 0};
     break;
   case ELF::R_SPARC_PC22:
-    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 10, 22, 0};
     break;
   case ELF::R_SPARC_WDISP10:
-    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 0, 32, 0};
     break;
   case ELF::R_SPARC_WDISP16:
-    Info = {"", 0, 32, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 0, 32, 0};
     break;
   case ELF::R_SPARC_WDISP19:
-    Info = {"", 13, 19, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 13, 19, 0};
     break;
   case ELF::R_SPARC_WDISP22:
-    Info = {"", 10, 22, MCFixupKindInfo::FKF_IsPCRel};
+    Info = {"", 10, 22, 0};
     break;
 
   case ELF::R_SPARC_HI22:
@@ -253,10 +252,11 @@ MCFixupKindInfo SparcAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return Info;
 }
 
-void SparcAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void SparcAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target,
                                  MutableArrayRef<char> Data, uint64_t Value,
                                  bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   if (!IsResolved)
     return;
   Value = adjustFixupValue(Fixup.getKind(), Value);
diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index 8ba99719946a..233585346946 100644
--- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -91,6 +91,22 @@ class SparcMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case ELF::R_SPARC_PC10:
+  case ELF::R_SPARC_PC22:
+  case ELF::R_SPARC_WDISP10:
+  case ELF::R_SPARC_WDISP16:
+  case ELF::R_SPARC_WDISP19:
+  case ELF::R_SPARC_WDISP22:
+  case Sparc::fixup_sparc_call30:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void SparcMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                            SmallVectorImpl<char> &CB,
                                            SmallVectorImpl<MCFixup> &Fixups,
@@ -135,7 +151,7 @@ getMachineOpValue(const MCInst &MI, const MCOperand &MO,
   assert(MO.isExpr());
   const MCExpr *Expr = MO.getExpr();
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
 
@@ -165,10 +181,10 @@ unsigned SparcMCCodeEmitter::getSImm5OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
-  Fixups.push_back(MCFixup::create(0, Expr, ELF::R_SPARC_5));
+  addFixup(Fixups, 0, Expr, ELF::R_SPARC_5);
   return 0;
 }
 
@@ -191,10 +207,10 @@ SparcMCCodeEmitter::getSImm13OpValue(const MCInst &MI, unsigned OpNo,
     return CE->getValue();
 
   if (auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
-    Fixups.push_back(MCFixup::create(0, Expr, SExpr->getSpecifier()));
+    addFixup(Fixups, 0, Expr, SExpr->getSpecifier());
     return 0;
   }
-  Fixups.push_back(MCFixup::create(0, Expr, Sparc::fixup_sparc_13));
+  addFixup(Fixups, 0, Expr, Sparc::fixup_sparc_13);
   return 0;
 }
 
@@ -209,7 +225,7 @@ getCallTargetOpValue(const MCInst &MI, unsigned OpNo,
   }
 
   const MCOperand &MO = MI.getOperand(OpNo);
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), Sparc::fixup_sparc_call30));
+  addFixup(Fixups, 0, MO.getExpr(), Sparc::fixup_sparc_call30);
   return 0;
 }
 
@@ -221,7 +237,7 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP22));
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP22);
   return 0;
 }
 
@@ -232,7 +248,7 @@ unsigned SparcMCCodeEmitter::getBranchPredTargetOpValue(
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP19));
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP19);
   return 0;
 }
 
@@ -243,8 +259,7 @@ unsigned SparcMCCodeEmitter::getBranchOnRegTargetOpValue(
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP16));
-
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP16);
   return 0;
 }
 
@@ -255,8 +270,7 @@ unsigned SparcMCCodeEmitter::getCompareAndBranchTargetOpValue(
   if (MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(MCFixup::create(0, MO.getExpr(), ELF::R_SPARC_WDISP10));
-
+  addFixup(Fixups, 0, MO.getExpr(), ELF::R_SPARC_WDISP10);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 7abe84a2451b..21dbe8f585b3 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "SparcISelLowering.h"
-#include "MCTargetDesc/SparcMCAsmInfo.h"
 #include "MCTargetDesc/SparcMCTargetDesc.h"
 #include "SparcMachineFunctionInfo.h"
 #include "SparcRegisterInfo.h"
@@ -1045,11 +1044,9 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   // The InGlue in necessary since all emitted instructions must be
   // stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Register Reg = RegsToPass[i].first;
-    if (!isTailCall)
-      Reg = toCallerWindow(Reg);
-    Chain = DAG.getCopyToReg(Chain, dl, Reg, RegsToPass[i].second, InGlue);
+  for (const auto &[OrigReg, N] : RegsToPass) {
+    Register Reg = isTailCall ? OrigReg : toCallerWindow(OrigReg);
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1070,11 +1067,9 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
   if (hasStructRetAttr)
     Ops.push_back(DAG.getTargetConstant(SRetArgSize, dl, MVT::i32));
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Register Reg = RegsToPass[i].first;
-    if (!isTailCall)
-      Reg = toCallerWindow(Reg);
-    Ops.push_back(DAG.getRegister(Reg, RegsToPass[i].second.getValueType()));
+  for (const auto &[OrigReg, N] : RegsToPass) {
+    Register Reg = isTailCall ? OrigReg : toCallerWindow(OrigReg);
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
   }
 
   // Add a register mask operand representing the call-preserved registers.
@@ -1376,9 +1371,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   // necessary since all emitted instructions must be stuck together in order
   // to pass the live physical registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, DL,
-                             RegsToPass[i].first, RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1396,9 +1390,8 @@ SparcTargetLowering::LowerCall_64(TargetLowering::CallLoweringInfo &CLI,
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
   Ops.push_back(Callee);
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const SparcRegisterInfo *TRI = Subtarget->getRegisterInfo();
@@ -1835,16 +1828,16 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
     // .umul works for both signed and unsigned
     setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
     setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
-    setLibcallName(RTLIB::MUL_I32, ".umul");
+    setLibcallImpl(RTLIB::MUL_I32, RTLIB::sparc_umul);
 
     setOperationAction(ISD::SDIV, MVT::i32, Expand);
-    setLibcallName(RTLIB::SDIV_I32, ".div");
+    setLibcallImpl(RTLIB::SDIV_I32, RTLIB::sparc_div);
 
     setOperationAction(ISD::UDIV, MVT::i32, Expand);
-    setLibcallName(RTLIB::UDIV_I32, ".udiv");
+    setLibcallImpl(RTLIB::UDIV_I32, RTLIB::sparc_udiv);
 
-    setLibcallName(RTLIB::SREM_I32, ".rem");
-    setLibcallName(RTLIB::UREM_I32, ".urem");
+    setLibcallImpl(RTLIB::SREM_I32, RTLIB::sparc_rem);
+    setLibcallImpl(RTLIB::UREM_I32, RTLIB::sparc_urem);
   }
 
   if (Subtarget->is64Bit()) {
@@ -1905,10 +1898,10 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
     }
 
     if (!Subtarget->is64Bit()) {
-      setLibcallName(RTLIB::FPTOSINT_F128_I64, "_Q_qtoll");
-      setLibcallName(RTLIB::FPTOUINT_F128_I64, "_Q_qtoull");
-      setLibcallName(RTLIB::SINTTOFP_I64_F128, "_Q_lltoq");
-      setLibcallName(RTLIB::UINTTOFP_I64_F128, "_Q_ulltoq");
+      setLibcallImpl(RTLIB::FPTOSINT_F128_I64, RTLIB::_Q_qtoll);
+      setLibcallImpl(RTLIB::FPTOUINT_F128_I64, RTLIB::_Q_qtoull);
+      setLibcallImpl(RTLIB::SINTTOFP_I64_F128, RTLIB::_Q_lltoq);
+      setLibcallImpl(RTLIB::UINTTOFP_I64_F128, RTLIB::_Q_ulltoq);
     }
 
   } else {
@@ -1928,41 +1921,41 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
 
     // Setup Runtime library names.
     if (Subtarget->is64Bit() && !Subtarget->useSoftFloat()) {
-      setLibcallName(RTLIB::ADD_F128,  "_Qp_add");
-      setLibcallName(RTLIB::SUB_F128,  "_Qp_sub");
-      setLibcallName(RTLIB::MUL_F128,  "_Qp_mul");
-      setLibcallName(RTLIB::DIV_F128,  "_Qp_div");
-      setLibcallName(RTLIB::SQRT_F128, "_Qp_sqrt");
-      setLibcallName(RTLIB::FPTOSINT_F128_I32, "_Qp_qtoi");
-      setLibcallName(RTLIB::FPTOUINT_F128_I32, "_Qp_qtoui");
-      setLibcallName(RTLIB::SINTTOFP_I32_F128, "_Qp_itoq");
-      setLibcallName(RTLIB::UINTTOFP_I32_F128, "_Qp_uitoq");
-      setLibcallName(RTLIB::FPTOSINT_F128_I64, "_Qp_qtox");
-      setLibcallName(RTLIB::FPTOUINT_F128_I64, "_Qp_qtoux");
-      setLibcallName(RTLIB::SINTTOFP_I64_F128, "_Qp_xtoq");
-      setLibcallName(RTLIB::UINTTOFP_I64_F128, "_Qp_uxtoq");
-      setLibcallName(RTLIB::FPEXT_F32_F128, "_Qp_stoq");
-      setLibcallName(RTLIB::FPEXT_F64_F128, "_Qp_dtoq");
-      setLibcallName(RTLIB::FPROUND_F128_F32, "_Qp_qtos");
-      setLibcallName(RTLIB::FPROUND_F128_F64, "_Qp_qtod");
+      setLibcallImpl(RTLIB::ADD_F128, RTLIB::_Qp_add);
+      setLibcallImpl(RTLIB::SUB_F128, RTLIB::_Qp_sub);
+      setLibcallImpl(RTLIB::MUL_F128, RTLIB::_Qp_mul);
+      setLibcallImpl(RTLIB::DIV_F128, RTLIB::_Qp_div);
+      setLibcallImpl(RTLIB::SQRT_F128, RTLIB::_Qp_sqrt);
+      setLibcallImpl(RTLIB::FPTOSINT_F128_I32, RTLIB::_Qp_qtoi);
+      setLibcallImpl(RTLIB::FPTOUINT_F128_I32, RTLIB::_Qp_qtoui);
+      setLibcallImpl(RTLIB::SINTTOFP_I32_F128, RTLIB::_Qp_itoq);
+      setLibcallImpl(RTLIB::UINTTOFP_I32_F128, RTLIB::_Qp_uitoq);
+      setLibcallImpl(RTLIB::FPTOSINT_F128_I64, RTLIB::_Qp_qtox);
+      setLibcallImpl(RTLIB::FPTOUINT_F128_I64, RTLIB::_Qp_qtoux);
+      setLibcallImpl(RTLIB::SINTTOFP_I64_F128, RTLIB::_Qp_xtoq);
+      setLibcallImpl(RTLIB::UINTTOFP_I64_F128, RTLIB::_Qp_uxtoq);
+      setLibcallImpl(RTLIB::FPEXT_F32_F128, RTLIB::_Qp_stoq);
+      setLibcallImpl(RTLIB::FPEXT_F64_F128, RTLIB::_Qp_dtoq);
+      setLibcallImpl(RTLIB::FPROUND_F128_F32, RTLIB::_Qp_qtos);
+      setLibcallImpl(RTLIB::FPROUND_F128_F64, RTLIB::_Qp_qtod);
     } else if (!Subtarget->useSoftFloat()) {
-      setLibcallName(RTLIB::ADD_F128,  "_Q_add");
-      setLibcallName(RTLIB::SUB_F128,  "_Q_sub");
-      setLibcallName(RTLIB::MUL_F128,  "_Q_mul");
-      setLibcallName(RTLIB::DIV_F128,  "_Q_div");
-      setLibcallName(RTLIB::SQRT_F128, "_Q_sqrt");
-      setLibcallName(RTLIB::FPTOSINT_F128_I32, "_Q_qtoi");
-      setLibcallName(RTLIB::FPTOUINT_F128_I32, "_Q_qtou");
-      setLibcallName(RTLIB::SINTTOFP_I32_F128, "_Q_itoq");
-      setLibcallName(RTLIB::UINTTOFP_I32_F128, "_Q_utoq");
-      setLibcallName(RTLIB::FPTOSINT_F128_I64, "_Q_qtoll");
-      setLibcallName(RTLIB::FPTOUINT_F128_I64, "_Q_qtoull");
-      setLibcallName(RTLIB::SINTTOFP_I64_F128, "_Q_lltoq");
-      setLibcallName(RTLIB::UINTTOFP_I64_F128, "_Q_ulltoq");
-      setLibcallName(RTLIB::FPEXT_F32_F128, "_Q_stoq");
-      setLibcallName(RTLIB::FPEXT_F64_F128, "_Q_dtoq");
-      setLibcallName(RTLIB::FPROUND_F128_F32, "_Q_qtos");
-      setLibcallName(RTLIB::FPROUND_F128_F64, "_Q_qtod");
+      setLibcallImpl(RTLIB::ADD_F128, RTLIB::_Q_add);
+      setLibcallImpl(RTLIB::SUB_F128, RTLIB::_Q_sub);
+      setLibcallImpl(RTLIB::MUL_F128, RTLIB::_Q_mul);
+      setLibcallImpl(RTLIB::DIV_F128, RTLIB::_Q_div);
+      setLibcallImpl(RTLIB::SQRT_F128, RTLIB::_Q_sqrt);
+      setLibcallImpl(RTLIB::FPTOSINT_F128_I32, RTLIB::_Q_qtoi);
+      setLibcallImpl(RTLIB::FPTOUINT_F128_I32, RTLIB::_Q_qtou);
+      setLibcallImpl(RTLIB::SINTTOFP_I32_F128, RTLIB::_Q_itoq);
+      setLibcallImpl(RTLIB::UINTTOFP_I32_F128, RTLIB::_Q_utoq);
+      setLibcallImpl(RTLIB::FPTOSINT_F128_I64, RTLIB::_Q_qtoll);
+      setLibcallImpl(RTLIB::FPTOUINT_F128_I64, RTLIB::_Q_qtoull);
+      setLibcallImpl(RTLIB::SINTTOFP_I64_F128, RTLIB::_Q_lltoq);
+      setLibcallImpl(RTLIB::UINTTOFP_I64_F128, RTLIB::_Q_ulltoq);
+      setLibcallImpl(RTLIB::FPEXT_F32_F128, RTLIB::_Q_stoq);
+      setLibcallImpl(RTLIB::FPEXT_F64_F128, RTLIB::_Q_dtoq);
+      setLibcallImpl(RTLIB::FPROUND_F128_F32, RTLIB::_Q_qtos);
+      setLibcallImpl(RTLIB::FPROUND_F128_F64, RTLIB::_Q_qtod);
     }
   }
 
@@ -2881,9 +2874,6 @@ static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (TLI.verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   EVT VT = Op.getValueType();
   SDLoc dl(Op);
   uint64_t depth = Op.getConstantOperandVal(0);
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 04a4c3610924..597ce8eabfeb 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -294,7 +294,7 @@ class SystemZOperand : public MCParsedAsmOperand {
   // Override MCParsedAsmOperand.
   SMLoc getStartLoc() const override { return StartLoc; }
   SMLoc getEndLoc() const override { return EndLoc; }
-  void print(raw_ostream &OS) const override;
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override;
 
   /// getLocRange - Get the range between the first and last token of this
   /// operand.
@@ -729,22 +729,7 @@ static struct InsnMatchEntry InsnMatchTable[] = {
     { MCK_U48Imm, MCK_VR128, MCK_BDAddr64Disp12, MCK_U8Imm } }
 };
 
-static void printMCExpr(const MCExpr *E, raw_ostream &OS) {
-  if (!E)
-    return;
-  if (auto *CE = dyn_cast<MCConstantExpr>(E))
-    OS << *CE;
-  else if (auto *UE = dyn_cast<MCUnaryExpr>(E))
-    OS << *UE;
-  else if (auto *BE = dyn_cast<MCBinaryExpr>(E))
-    OS << *BE;
-  else if (auto *SRE = dyn_cast<MCSymbolRefExpr>(E))
-    OS << *SRE;
-  else
-    OS << *E;
-}
-
-void SystemZOperand::print(raw_ostream &OS) const {
+void SystemZOperand::print(raw_ostream &OS, const MCAsmInfo &MAI) const {
   switch (Kind) {
   case KindToken:
     OS << "Token:" << getToken();
@@ -754,24 +739,26 @@ void SystemZOperand::print(raw_ostream &OS) const {
     break;
   case KindImm:
     OS << "Imm:";
-    printMCExpr(getImm(), OS);
+    MAI.printExpr(OS, *getImm());
     break;
   case KindImmTLS:
     OS << "ImmTLS:";
-    printMCExpr(getImmTLS().Imm, OS);
+    MAI.printExpr(OS, *getImmTLS().Imm);
     if (getImmTLS().Sym) {
       OS << ", ";
-      printMCExpr(getImmTLS().Sym, OS);
+      MAI.printExpr(OS, *getImmTLS().Sym);
     }
     break;
   case KindMem: {
     const MemOp &Op = getMem();
-    OS << "Mem:" << *cast<MCConstantExpr>(Op.Disp);
+    OS << "Mem:";
+    MAI.printExpr(OS, *cast<MCConstantExpr>(Op.Disp));
     if (Op.Base) {
       OS << "(";
-      if (Op.MemKind == BDLMem)
-        OS << *cast<MCConstantExpr>(Op.Length.Imm) << ",";
-      else if (Op.MemKind == BDRMem)
+      if (Op.MemKind == BDLMem) {
+        MAI.printExpr(OS, *cast<MCConstantExpr>(Op.Length.Imm));
+        OS << ',';
+      } else if (Op.MemKind == BDRMem)
         OS << SystemZGNUInstPrinter::getRegisterName(Op.Length.Reg) << ",";
       if (Op.Index)
         OS << SystemZGNUInstPrinter::getRegisterName(Op.Index) << ",";
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
index ec8c81080930..9121f0d44936 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.cpp
@@ -12,7 +12,7 @@
 #include "llvm/Support/Signals.h"
 #include <sstream>
 
-#include <cmath>
+using namespace llvm;
 
 void SystemZHLASMAsmStreamer::EmitEOL() {
   // Comments are emitted on a new line before the instruction.
@@ -280,3 +280,8 @@ void SystemZHLASMAsmStreamer::emitValueImpl(const MCExpr *Value, unsigned Size,
   emitHLASMValueImpl(Value, Size, true);
   EmitEOL();
 }
+
+void SystemZHLASMAsmStreamer::emitEnd() {
+  OS << " END";
+  EmitEOL();
+}
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
index edaaf984df65..c5275339ce01 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZHLASMAsmStreamer.h
@@ -10,6 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
+#define LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
+
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCAsmBackend.h"
@@ -24,7 +27,7 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/FormattedStream.h"
 
-using namespace llvm;
+namespace llvm {
 
 class SystemZHLASMAsmStreamer final : public MCStreamer {
   constexpr static size_t InstLimit = 80;
@@ -119,4 +122,9 @@ class SystemZHLASMAsmStreamer final : public MCStreamer {
   void emitHLASMValueImpl(const MCExpr *Value, unsigned Size,
                           bool Parens = false);
   /// @}
+
+  void emitEnd();
 };
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZHLASMASMSTREAMER_H
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
index 297fdc832592..af79070db7b9 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinterCommon.cpp
@@ -67,7 +67,7 @@ void SystemZInstPrinterCommon::printUImmOperand(const MCInst *MI, int OpNum,
                                                 raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   if (MO.isExpr()) {
-    O << *MO.getExpr();
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
   uint64_t Value = static_cast<uint64_t>(MO.getImm());
@@ -80,7 +80,7 @@ void SystemZInstPrinterCommon::printSImmOperand(const MCInst *MI, int OpNum,
                                                 raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   if (MO.isExpr()) {
-    O << *MO.getExpr();
+    MAI.printExpr(O, *MO.getExpr());
     return;
   }
   int64_t Value = MI->getOperand(OpNum).getImm();
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 4ec988338c1a..d5f849235a84 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -13,7 +13,6 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -113,7 +112,6 @@ class SystemZMCAsmBackend : public MCAsmBackend {
   // Override MCAsmBackend
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
@@ -143,7 +141,7 @@ MCFixupKindInfo SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   // Fixup kinds from .reloc directive are like R_390_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -153,15 +151,13 @@ MCFixupKindInfo SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   return SystemZ::MCFixupKindInfos[Kind - FirstTargetFixupKind];
 }
 
-bool SystemZMCAsmBackend::shouldForceRelocation(const MCFixup &,
-                                                const MCValue &Target) {
-  return Target.getSpecifier();
-}
-
-void SystemZMCAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void SystemZMCAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                      const MCValue &Target,
                                      MutableArrayRef<char> Data, uint64_t Value,
                                      bool IsResolved) {
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCFixupKind Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
index 052875bf0d3f..2933b4becba0 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
@@ -13,7 +13,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {SystemZ::S_DTPOFF, "DTPOFF"}, {SystemZ::S_GOT, "GOT"},
     {SystemZ::S_GOTENT, "GOTENT"}, {SystemZ::S_INDNTPOFF, "INDNTPOFF"},
     {SystemZ::S_NTPOFF, "NTPOFF"}, {SystemZ::S_PLT, "PLT"},
@@ -33,7 +33,7 @@ SystemZMCAsmInfoELF::SystemZMCAsmInfoELF(const Triple &TT) {
   UsesELFSectionDirectiveForBSS = true;
   ZeroDirective = "\t.space\t";
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 SystemZMCAsmInfoGOFF::SystemZMCAsmInfoGOFF(const Triple &TT) {
@@ -52,7 +52,7 @@ SystemZMCAsmInfoGOFF::SystemZMCAsmInfoGOFF(const Triple &TT) {
   MaxInstLength = 6;
   SupportsDebugInformation = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 bool SystemZMCAsmInfoGOFF::isAcceptableChar(char C) const {
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index b161eed95d6e..59d1db784c68 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -129,6 +129,11 @@ class SystemZMCCodeEmitter : public MCCodeEmitter {
 };
 
 } // end anonymous namespace
+  //
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
 
 void SystemZMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                              SmallVectorImpl<char> &CB,
@@ -170,8 +175,7 @@ uint64_t SystemZMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNum,
     unsigned OpBitSize =
         SystemZ::MCFixupKindInfos[Kind - FirstTargetFixupKind].TargetSize;
     uint32_t BitOffset = MIBitSize - RawBitOffset - OpBitSize;
-    Fixups.push_back(MCFixup::create(BitOffset >> 3, MO.getExpr(),
-                                     (MCFixupKind)Kind, MI.getLoc()));
+    addFixup(Fixups, BitOffset >> 3, MO.getExpr(), Kind);
     return 0;
   }
   llvm_unreachable("Unexpected operand type!");
@@ -203,16 +207,15 @@ SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
       // into MI.  Add Offset to the relocation value to cancel out
       // this difference.
       const MCExpr *OffsetExpr = MCConstantExpr::create(Offset, Ctx);
-      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
+      Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx, Loc);
     }
   }
-  Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind, Loc));
+  addFixup(Fixups, Offset, Expr, Kind, true);
 
   // Output the fixup for the TLS marker if present.
   if (AllowTLS && OpNum + 1 < MI.getNumOperands()) {
     const MCOperand &MOTLS = MI.getOperand(OpNum + 1);
-    Fixups.push_back(MCFixup::create(
-        0, MOTLS.getExpr(), (MCFixupKind)SystemZ::FK_390_TLS_CALL, Loc));
+    addFixup(Fixups, 0, MOTLS.getExpr(), SystemZ::FK_390_TLS_CALL);
   }
   return 0;
 }
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
index 512e51c0f933..1ff4217018f1 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
@@ -9,8 +9,8 @@
 #ifndef LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCFIXUPS_H
 #define LLVM_LIB_TARGET_SYSTEMZ_MCTARGETDESC_SYSTEMZMCFIXUPS_H
 
+#include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCFixup.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 
 namespace llvm {
 namespace SystemZ {
@@ -41,26 +41,28 @@ enum FixupKind {
   NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
 };
 
+// clang-format off
 const static MCFixupKindInfo MCFixupKindInfos[SystemZ::NumTargetFixupKinds] = {
-    {"FK_390_PC12DBL", 4, 12, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC16DBL", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC24DBL", 0, 24, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_PC32DBL", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-    {"FK_390_TLS_CALL", 0, 0, 0},
-    {"FK_390_S8Imm", 0, 8, 0},
-    {"FK_390_S16Imm", 0, 16, 0},
-    {"FK_390_S20Imm", 4, 20, 0},
-    {"FK_390_S32Imm", 0, 32, 0},
-    {"FK_390_U1Imm", 0, 1, 0},
-    {"FK_390_U2Imm", 0, 2, 0},
-    {"FK_390_U3Imm", 0, 3, 0},
-    {"FK_390_U4Imm", 0, 4, 0},
-    {"FK_390_U8Imm", 0, 8, 0},
-    {"FK_390_U12Imm", 4, 12, 0},
-    {"FK_390_U16Imm", 0, 16, 0},
-    {"FK_390_U32Imm", 0, 32, 0},
-    {"FK_390_U48Imm", 0, 48, 0},
+    {"FK_390_PC12DBL", 4, 12, 0},
+    {"FK_390_PC16DBL", 0, 16, 0},
+    {"FK_390_PC24DBL", 0, 24, 0},
+    {"FK_390_PC32DBL", 0, 32, 0},
+    {"FK_390_TLS_CALL",0,  0, 0},
+    {"FK_390_S8Imm",   0,  8, 0},
+    {"FK_390_S16Imm",  0, 16, 0},
+    {"FK_390_S20Imm",  4, 20, 0},
+    {"FK_390_S32Imm",  0, 32, 0},
+    {"FK_390_U1Imm",   0,  1, 0},
+    {"FK_390_U2Imm",   0,  2, 0},
+    {"FK_390_U3Imm",   0,  3, 0},
+    {"FK_390_U4Imm",   0,  4, 0},
+    {"FK_390_U8Imm",   0,  8, 0},
+    {"FK_390_U12Imm",  4, 12, 0},
+    {"FK_390_U16Imm",  0, 16, 0},
+    {"FK_390_U32Imm",  0, 32, 0},
+    {"FK_390_U48Imm",  0, 48, 0},
 };
+// clang-format on
 } // end namespace SystemZ
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 86e340b7ff1b..faef5bb92678 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -203,7 +203,7 @@ static MCInstPrinter *createSystemZMCInstPrinter(const Triple &T,
 static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S,
                                                  formatted_raw_ostream &OS,
                                                  MCInstPrinter *InstPrint) {
-  if (S.getContext().getTargetTriple().isOSzOS())
+  if (S.getContext().getTargetTriple().isOSzOS() && !GNUAsOnzOSCL)
     return new SystemZTargetHLASMStreamer(S, OS);
   else
     return new SystemZTargetGNUStreamer(S, OS);
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
index 772067809744..1a3e373f2537 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetStreamer.h"
+#include "SystemZHLASMAsmStreamer.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 
@@ -33,10 +34,16 @@ void SystemZTargetStreamer::emitConstantPools() {
   EXRLTargets2Sym.clear();
 }
 
+SystemZHLASMAsmStreamer &SystemZTargetHLASMStreamer::getHLASMStreamer() {
+  return static_cast<SystemZHLASMAsmStreamer &>(getStreamer());
+}
+
 void SystemZTargetHLASMStreamer::emitExtern(StringRef Sym) {
   getStreamer().emitRawText(Twine(" EXTRN ") + Twine(Sym));
 }
 
+void SystemZTargetHLASMStreamer::emitEnd() { getHLASMStreamer().emitEnd(); }
+
 // HLASM statements can only perform a single operation at a time
 const MCExpr *SystemZTargetHLASMStreamer::createWordDiffExpr(
     MCContext &Ctx, const MCSymbol *Hi, const MCSymbol *Lo) {
diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
index 7e3c5f00525f..3fc09bc8c683 100644
--- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
+++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZTargetStreamer.h
@@ -20,6 +20,7 @@
 #include <utility>
 
 namespace llvm {
+class SystemZHLASMAsmStreamer;
 
 class SystemZTargetStreamer : public MCTargetStreamer {
 public:
@@ -57,6 +58,7 @@ class SystemZTargetStreamer : public MCTargetStreamer {
   virtual void emitMachine(StringRef CPUOrCommand) {};
 
   virtual void emitExtern(StringRef Str) {};
+  virtual void emitEnd() {};
 
   virtual const MCExpr *createWordDiffExpr(MCContext &Ctx, const MCSymbol *Hi,
                                            const MCSymbol *Lo) {
@@ -77,7 +79,9 @@ class SystemZTargetHLASMStreamer : public SystemZTargetStreamer {
 public:
   SystemZTargetHLASMStreamer(MCStreamer &S, formatted_raw_ostream &OS)
       : SystemZTargetStreamer(S), OS(OS) {}
+  SystemZHLASMAsmStreamer &getHLASMStreamer();
   void emitExtern(StringRef Sym) override;
+  void emitEnd() override;
   const MCExpr *createWordDiffExpr(MCContext &Ctx, const MCSymbol *Hi,
                                    const MCSymbol *Lo) override;
 };
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index 6f9d25c050b7..e31d7c6a8647 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -21,6 +21,7 @@
 #include "TargetInfo/SystemZTargetInfo.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/GOFF.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/IR/Mangler.h"
@@ -1113,6 +1114,9 @@ void SystemZAsmPrinter::emitEndOfAsmFile(Module &M) {
     emitIDRLSection(M);
   }
   emitAttributes(M);
+  // Emit the END instruction in case of HLASM output. This must be the last
+  // instruction in the source file.
+  getTargetStreamer()->emitEnd();
 }
 
 void SystemZAsmPrinter::emitADASection() {
@@ -1254,7 +1258,8 @@ void SystemZAsmPrinter::emitFunctionBodyEnd() {
     OutStreamer->emitLabel(FnEndSym);
 
     OutStreamer->pushSection();
-    OutStreamer->switchSection(getObjFileLowering().getPPA1Section());
+    OutStreamer->switchSection(getObjFileLowering().getTextSection(),
+                               GOFF::SK_PPA1);
     emitPPA1(FnEndSym);
     OutStreamer->popSection();
 
@@ -1559,7 +1564,8 @@ void SystemZAsmPrinter::emitStartOfAsmFile(Module &M) {
 
 void SystemZAsmPrinter::emitPPA2(Module &M) {
   OutStreamer->pushSection();
-  OutStreamer->switchSection(getObjFileLowering().getPPA2Section());
+  OutStreamer->switchSection(getObjFileLowering().getTextSection(),
+                             GOFF::SK_PPA2);
   MCContext &OutContext = OutStreamer->getContext();
   // Make CELQSTRT symbol.
   const char *StartSymbolName = "CELQSTRT";
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 831618c4eff5..262b607b953c 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -2387,9 +2387,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
   }
 
   // Build a sequence of copy-to-reg nodes, chained and glued together.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
-                             RegsToPass[I].second, Glue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, Glue);
     Glue = Chain.getValue(1);
   }
 
@@ -2400,9 +2399,8 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
-    Ops.push_back(DAG.getRegister(RegsToPass[I].first,
-                                  RegsToPass[I].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
@@ -4254,9 +4252,6 @@ SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDLoc DL(Op);
   unsigned Depth = Op.getConstantOperandVal(0);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
diff --git a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
index 7987950a2a0a..20f561a8dac3 100644
--- a/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
+++ b/llvm/lib/Target/VE/AsmParser/VEAsmParser.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/VEMCExpr.h"
+#include "MCTargetDesc/VEMCAsmInfo.h"
 #include "MCTargetDesc/VEMCTargetDesc.h"
 #include "TargetInfo/VETargetInfo.h"
 #include "VE.h"
@@ -409,7 +409,7 @@ class VEOperand : public MCParsedAsmOperand {
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const override { return EndLoc; }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case k_Token:
       OS << "Token: " << getToken() << "\n";
@@ -422,29 +422,43 @@ class VEOperand : public MCParsedAsmOperand {
       break;
     case k_MemoryRegRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+#" << getMemIndexReg() << "+"
-         << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+#" << getMemIndexReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryRegImmImm:
       assert(getMemIndex() != nullptr && getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+" << *getMemIndex() << "+"
-         << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemIndex());
+      OS << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: 0+#" << getMemIndexReg() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+#" << getMemIndexReg() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroImmImm:
       assert(getMemIndex() != nullptr && getMemOffset() != nullptr);
-      OS << "Mem: 0+" << *getMemIndex() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+";
+      MAI.printExpr(OS, *getMemIndex());
+      OS << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryRegImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: #" << getMemBase() << "+" << *getMemOffset() << "\n";
+      OS << "Mem: #" << getMemBase() << "+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_MemoryZeroImm:
       assert(getMemOffset() != nullptr);
-      OS << "Mem: 0+" << *getMemOffset() << "\n";
+      OS << "Mem: 0+";
+      MAI.printExpr(OS, *getMemOffset());
+      OS << "\n";
       break;
     case k_CCOp:
       OS << "CCOp: " << getCCVal() << "\n";
diff --git a/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
index 34a9e2044f30..359344346762 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/VE/MCTargetDesc/CMakeLists.txt
@@ -4,7 +4,6 @@ add_llvm_component_library(LLVMVEDesc
   VEInstPrinter.cpp
   VEMCAsmInfo.cpp
   VEMCCodeEmitter.cpp
-  VEMCExpr.cpp
   VEMCTargetDesc.cpp
   VETargetStreamer.cpp
 
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
index 9a23a0e09f16..270c1e08397b 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp
@@ -11,7 +11,6 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCValue.h"
@@ -59,7 +58,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
     return 1;
   case FK_Data_2:
     return 2;
-    return 4;
   case FK_Data_4:
   case VE::fixup_ve_reflong:
   case VE::fixup_ve_srel32:
@@ -79,7 +77,6 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   case VE::fixup_ve_tpoff_lo32:
     return 4;
   case FK_Data_8:
-  case FK_PCRel_8:
     return 8;
   }
 }
@@ -96,22 +93,14 @@ class VEAsmBackend : public MCAsmBackend {
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override {
     const static MCFixupKindInfo Infos[VE::NumTargetFixupKinds] = {
         // name, offset, bits, flags
-        {"fixup_ve_reflong", 0, 32, 0},
-        {"fixup_ve_srel32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_hi32", 0, 32, 0},
-        {"fixup_ve_lo32", 0, 32, 0},
-        {"fixup_ve_pc_hi32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_pc_lo32", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-        {"fixup_ve_got_hi32", 0, 32, 0},
-        {"fixup_ve_got_lo32", 0, 32, 0},
-        {"fixup_ve_gotoff_hi32", 0, 32, 0},
-        {"fixup_ve_gotoff_lo32", 0, 32, 0},
-        {"fixup_ve_plt_hi32", 0, 32, 0},
-        {"fixup_ve_plt_lo32", 0, 32, 0},
-        {"fixup_ve_tls_gd_hi32", 0, 32, 0},
-        {"fixup_ve_tls_gd_lo32", 0, 32, 0},
-        {"fixup_ve_tpoff_hi32", 0, 32, 0},
-        {"fixup_ve_tpoff_lo32", 0, 32, 0},
+        {"fixup_ve_reflong", 0, 32, 0},     {"fixup_ve_srel32", 0, 32, 0},
+        {"fixup_ve_hi32", 0, 32, 0},        {"fixup_ve_lo32", 0, 32, 0},
+        {"fixup_ve_pc_hi32", 0, 32, 0},     {"fixup_ve_pc_lo32", 0, 32, 0},
+        {"fixup_ve_got_hi32", 0, 32, 0},    {"fixup_ve_got_lo32", 0, 32, 0},
+        {"fixup_ve_gotoff_hi32", 0, 32, 0}, {"fixup_ve_gotoff_lo32", 0, 32, 0},
+        {"fixup_ve_plt_hi32", 0, 32, 0},    {"fixup_ve_plt_lo32", 0, 32, 0},
+        {"fixup_ve_tls_gd_hi32", 0, 32, 0}, {"fixup_ve_tls_gd_lo32", 0, 32, 0},
+        {"fixup_ve_tpoff_hi32", 0, 32, 0},  {"fixup_ve_tpoff_lo32", 0, 32, 0},
     };
 
     if (Kind < FirstTargetFixupKind)
@@ -122,18 +111,9 @@ class VEAsmBackend : public MCAsmBackend {
     return Infos[Kind - FirstTargetFixupKind];
   }
 
-  bool shouldForceRelocation(const MCFixup &Fixup,
-                             const MCValue &Target) override {
-    switch ((VE::Fixups)Fixup.getKind()) {
-    default:
-      return false;
-    case VE::fixup_ve_tls_gd_hi32:
-    case VE::fixup_ve_tls_gd_lo32:
-    case VE::fixup_ve_tpoff_hi32:
-    case VE::fixup_ve_tpoff_lo32:
-      return true;
-    }
-  }
+  void applyFixup(const MCFragment &, const MCFixup &, const MCValue &,
+                  MutableArrayRef<char>, uint64_t Value,
+                  bool IsResolved) override;
 
   bool mayNeedRelaxation(const MCInst &Inst,
                          const MCSubtargetInfo &STI) const override {
@@ -143,12 +123,6 @@ class VEAsmBackend : public MCAsmBackend {
     return false;
   }
 
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override {
-    // Aurora VE doesn't support relaxInstruction yet.
-    llvm_unreachable("relaxInstruction() should not be called");
-  }
-
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override {
     if ((Count % 8) != 0)
@@ -169,31 +143,6 @@ class ELFVEAsmBackend : public VEAsmBackend {
   ELFVEAsmBackend(const Target &T, Triple::OSType OSType)
       : VEAsmBackend(T), OSType(OSType) {}
 
-  void applyFixup(const MCFragment &, const MCFixup &Fixup,
-                  const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t Value, bool IsResolved) override {
-    Value = adjustFixupValue(Fixup.getKind(), Value);
-    if (!Value)
-      return; // Doesn't change encoding.
-
-    MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
-
-    // Shift the value into position.
-    Value <<= Info.TargetOffset;
-
-    unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
-    unsigned Offset = Fixup.getOffset();
-    assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
-    // For each byte of the fragment that the fixup touches, mask in the bits
-    // from the fixup value. The Value has been "split up" into the
-    // appropriate bitfields above.
-    for (unsigned i = 0; i != NumBytes; ++i) {
-      unsigned Idx =
-          Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
-      Data[Offset + Idx] |= static_cast<uint8_t>((Value >> (i * 8)) & 0xff);
-    }
-  }
-
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override {
     uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(OSType);
@@ -202,6 +151,39 @@ class ELFVEAsmBackend : public VEAsmBackend {
 };
 } // end anonymous namespace
 
+void VEAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                              const MCValue &Target, MutableArrayRef<char> Data,
+                              uint64_t Value, bool IsResolved) {
+  switch (Fixup.getTargetKind()) {
+  case VE::fixup_ve_tls_gd_hi32:
+  case VE::fixup_ve_tls_gd_lo32:
+  case VE::fixup_ve_tpoff_hi32:
+  case VE::fixup_ve_tpoff_lo32:
+    IsResolved = false;
+    break;
+  }
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
+  Value = adjustFixupValue(Fixup.getKind(), Value);
+  if (!Value)
+    return; // Doesn't change encoding.
+
+  MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
+
+  // Shift the value into position.
+  Value <<= Info.TargetOffset;
+
+  unsigned NumBytes = getFixupKindNumBytes(Fixup.getKind());
+  unsigned Offset = Fixup.getOffset();
+  assert(Offset + NumBytes <= Data.size() && "Invalid fixup offset!");
+  // For each byte of the fragment that the fixup touches, mask in the bits
+  // from the fixup value. The Value has been "split up" into the
+  // appropriate bitfields above.
+  for (unsigned i = 0; i != NumBytes; ++i) {
+    unsigned Idx = Endian == llvm::endianness::little ? i : (NumBytes - 1) - i;
+    Data[Offset + Idx] |= static_cast<uint8_t>((Value >> (i * 8)) & 0xff);
+  }
+}
+
 MCAsmBackend *llvm::createVEAsmBackend(const Target &T,
                                        const MCSubtargetInfo &STI,
                                        const MCRegisterInfo &MRI,
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
index 0e3f5d18de07..1597e7d080f0 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEELFObjectWriter.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/VEMCExpr.h"
+#include "MCTargetDesc/VEMCAsmInfo.h"
 #include "VEFixupKinds.h"
-#include "VEMCExpr.h"
+#include "VEMCAsmInfo.h"
 #include "VEMCTargetDesc.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
@@ -61,7 +61,6 @@ unsigned VEELFObjectWriter::getRelocType(const MCFixup &Fixup,
       reportError(Fixup.getLoc(), "Unsupported pc-relative fixup kind");
       return ELF::R_VE_NONE;
     case FK_Data_1:
-    case FK_PCRel_1:
       reportError(Fixup.getLoc(),
                   "1-byte pc-relative data relocation is not supported");
       return ELF::R_VE_NONE;
@@ -72,7 +71,6 @@ unsigned VEELFObjectWriter::getRelocType(const MCFixup &Fixup,
     case FK_Data_4:
       return ELF::R_VE_SREL32;
     case FK_Data_8:
-    case FK_PCRel_8:
       reportError(Fixup.getLoc(),
                   "8-byte pc-relative data relocation is not supported");
       return ELF::R_VE_NONE;
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
index 8eb3aedd668e..952838155c95 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "VEMCAsmInfo.h"
-#include "VEMCExpr.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCValue.h"
@@ -19,7 +18,7 @@
 
 using namespace llvm;
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
     {VE::S_HI32, "hi"},
     {VE::S_LO32, "lo"},
     {VE::S_PC_HI32, "pc_hi"},
@@ -36,6 +35,43 @@ const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
     {VE::S_TPOFF_LO32, "tpoff_lo"},
 };
 
+VE::Fixups VE::getFixupKind(uint8_t S) {
+  switch (S) {
+  default:
+    llvm_unreachable("Unhandled VEMCExpr::Specifier");
+  case VE::S_REFLONG:
+    return VE::fixup_ve_reflong;
+  case VE::S_HI32:
+    return VE::fixup_ve_hi32;
+  case VE::S_LO32:
+    return VE::fixup_ve_lo32;
+  case VE::S_PC_HI32:
+    return VE::fixup_ve_pc_hi32;
+  case VE::S_PC_LO32:
+    return VE::fixup_ve_pc_lo32;
+  case VE::S_GOT_HI32:
+    return VE::fixup_ve_got_hi32;
+  case VE::S_GOT_LO32:
+    return VE::fixup_ve_got_lo32;
+  case VE::S_GOTOFF_HI32:
+    return VE::fixup_ve_gotoff_hi32;
+  case VE::S_GOTOFF_LO32:
+    return VE::fixup_ve_gotoff_lo32;
+  case VE::S_PLT_HI32:
+    return VE::fixup_ve_plt_hi32;
+  case VE::S_PLT_LO32:
+    return VE::fixup_ve_plt_lo32;
+  case VE::S_TLS_GD_HI32:
+    return VE::fixup_ve_tls_gd_hi32;
+  case VE::S_TLS_GD_LO32:
+    return VE::fixup_ve_tls_gd_lo32;
+  case VE::S_TPOFF_HI32:
+    return VE::fixup_ve_tpoff_hi32;
+  case VE::S_TPOFF_LO32:
+    return VE::fixup_ve_tpoff_lo32;
+  }
+}
+
 void VEELFMCAsmInfo::anchor() {}
 
 VEELFMCAsmInfo::VEELFMCAsmInfo(const Triple &TheTriple) {
@@ -55,7 +91,7 @@ VEELFMCAsmInfo::VEELFMCAsmInfo(const Triple &TheTriple) {
 
   SupportsDebugInformation = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void VEELFMCAsmInfo::printSpecifierExpr(raw_ostream &OS,
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h b/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
index 2d73c94e2113..d66c02304d26 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCAsmInfo.h
@@ -13,7 +13,9 @@
 #ifndef LLVM_LIB_TARGET_VE_MCTARGETDESC_VEMCASMINFO_H
 #define LLVM_LIB_TARGET_VE_MCTARGETDESC_VEMCASMINFO_H
 
+#include "VEFixupKinds.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 
@@ -30,6 +32,29 @@ class VEELFMCAsmInfo : public MCAsmInfoELF {
                                  const MCAssembler *Asm) const override;
 };
 
+namespace VE {
+enum Specifier {
+  S_None,
+
+  S_REFLONG = MCSymbolRefExpr::FirstTargetSpecifier,
+  S_HI32,        // @hi
+  S_LO32,        // @lo
+  S_PC_HI32,     // @pc_hi
+  S_PC_LO32,     // @pc_lo
+  S_GOT_HI32,    // @got_hi
+  S_GOT_LO32,    // @got_lo
+  S_GOTOFF_HI32, // @gotoff_hi
+  S_GOTOFF_LO32, // @gotoff_lo
+  S_PLT_HI32,    // @plt_hi
+  S_PLT_LO32,    // @plt_lo
+  S_TLS_GD_HI32, // @tls_gd_hi
+  S_TLS_GD_LO32, // @tls_gd_lo
+  S_TPOFF_HI32,  // @tpoff_hi
+  S_TPOFF_LO32,  // @tpoff_lo
+};
+
+VE::Fixups getFixupKind(uint8_t S);
+} // namespace VE
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_VE_MCTARGETDESC_VEMCASMINFO_H
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
index 712de5accce5..711937c48827 100644
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
+++ b/llvm/lib/Target/VE/MCTargetDesc/VEMCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "MCTargetDesc/VEFixupKinds.h"
 #include "VE.h"
-#include "VEMCExpr.h"
+#include "VEMCAsmInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/MC/MCCodeEmitter.h"
@@ -76,6 +76,18 @@ class VEMCCodeEmitter : public MCCodeEmitter {
 
 } // end anonymous namespace
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case VE::fixup_ve_srel32:
+  case VE::fixup_ve_pc_hi32:
+  case VE::fixup_ve_pc_lo32:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void VEMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                         SmallVectorImpl<char> &CB,
                                         SmallVectorImpl<MCFixup> &Fixups,
@@ -100,7 +112,7 @@ unsigned VEMCCodeEmitter::getMachineOpValue(const MCInst &MI,
   const MCExpr *Expr = MO.getExpr();
   if (const auto *SExpr = dyn_cast<MCSpecifierExpr>(Expr)) {
     auto Kind = VE::getFixupKind(SExpr->getSpecifier());
-    Fixups.push_back(MCFixup::create(0, Expr, Kind));
+    addFixup(Fixups, 0, Expr, Kind);
     return 0;
   }
 
@@ -120,8 +132,7 @@ VEMCCodeEmitter::getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
   if (MO.isReg() || MO.isImm())
     return getMachineOpValue(MI, MO, Fixups, STI);
 
-  Fixups.push_back(
-      MCFixup::create(0, MO.getExpr(), (MCFixupKind)VE::fixup_ve_srel32));
+  addFixup(Fixups, 0, MO.getExpr(), VE::fixup_ve_srel32);
   return 0;
 }
 
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
deleted file mode 100644
index ca13aba095e2..000000000000
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- VEMCExpr.cpp - VE specific MC expression classes ------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the implementation of the assembly expression modifiers
-// accepted by the VE architecture (e.g. "%hi", "%lo", ...).
-//
-//===----------------------------------------------------------------------===//
-
-#include "VEMCExpr.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCObjectStreamer.h"
-#include "llvm/MC/MCValue.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "vemcexpr"
-
-VE::Fixups VE::getFixupKind(uint8_t S) {
-  switch (S) {
-  default:
-    llvm_unreachable("Unhandled VEMCExpr::Specifier");
-  case VE::S_REFLONG:
-    return VE::fixup_ve_reflong;
-  case VE::S_HI32:
-    return VE::fixup_ve_hi32;
-  case VE::S_LO32:
-    return VE::fixup_ve_lo32;
-  case VE::S_PC_HI32:
-    return VE::fixup_ve_pc_hi32;
-  case VE::S_PC_LO32:
-    return VE::fixup_ve_pc_lo32;
-  case VE::S_GOT_HI32:
-    return VE::fixup_ve_got_hi32;
-  case VE::S_GOT_LO32:
-    return VE::fixup_ve_got_lo32;
-  case VE::S_GOTOFF_HI32:
-    return VE::fixup_ve_gotoff_hi32;
-  case VE::S_GOTOFF_LO32:
-    return VE::fixup_ve_gotoff_lo32;
-  case VE::S_PLT_HI32:
-    return VE::fixup_ve_plt_hi32;
-  case VE::S_PLT_LO32:
-    return VE::fixup_ve_plt_lo32;
-  case VE::S_TLS_GD_HI32:
-    return VE::fixup_ve_tls_gd_hi32;
-  case VE::S_TLS_GD_LO32:
-    return VE::fixup_ve_tls_gd_lo32;
-  case VE::S_TPOFF_HI32:
-    return VE::fixup_ve_tpoff_hi32;
-  case VE::S_TPOFF_LO32:
-    return VE::fixup_ve_tpoff_lo32;
-  }
-}
diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h b/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
deleted file mode 100644
index b7913513bd51..000000000000
--- a/llvm/lib/Target/VE/MCTargetDesc/VEMCExpr.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//====- VEMCExpr.h - VE specific MC expression classes --------*- C++ -*-=====//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes VE-specific MCExprs, used for modifiers like
-// "%hi" or "%lo" etc.,
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_VE_MCTARGETDESC_VEMCEXPR_H
-#define LLVM_LIB_TARGET_VE_MCTARGETDESC_VEMCEXPR_H
-
-#include "VEFixupKinds.h"
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-
-class StringRef;
-
-namespace VE {
-enum Specifier {
-  S_None,
-
-  S_REFLONG = MCSymbolRefExpr::FirstTargetSpecifier,
-  S_HI32,        // @hi
-  S_LO32,        // @lo
-  S_PC_HI32,     // @pc_hi
-  S_PC_LO32,     // @pc_lo
-  S_GOT_HI32,    // @got_hi
-  S_GOT_LO32,    // @got_lo
-  S_GOTOFF_HI32, // @gotoff_hi
-  S_GOTOFF_LO32, // @gotoff_lo
-  S_PLT_HI32,    // @plt_hi
-  S_PLT_LO32,    // @plt_lo
-  S_TLS_GD_HI32, // @tls_gd_hi
-  S_TLS_GD_LO32, // @tls_gd_lo
-  S_TPOFF_HI32,  // @tpoff_hi
-  S_TPOFF_LO32,  // @tpoff_lo
-};
-
-VE::Fixups getFixupKind(uint8_t S);
-} // namespace VE
-
-} // namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/VE/VEAsmPrinter.cpp b/llvm/lib/Target/VE/VEAsmPrinter.cpp
index f7d770c18f88..ba8cf78cf2a5 100644
--- a/llvm/lib/Target/VE/VEAsmPrinter.cpp
+++ b/llvm/lib/Target/VE/VEAsmPrinter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/VEInstPrinter.h"
-#include "MCTargetDesc/VEMCExpr.h"
+#include "MCTargetDesc/VEMCAsmInfo.h"
 #include "MCTargetDesc/VETargetStreamer.h"
 #include "TargetInfo/VETargetInfo.h"
 #include "VE.h"
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index a0389aef6dce..9e8f40025619 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "VEISelLowering.h"
-#include "MCTargetDesc/VEMCExpr.h"
+#include "MCTargetDesc/VEMCAsmInfo.h"
 #include "VECustomDAG.h"
 #include "VEInstrBuilder.h"
 #include "VEMachineFunctionInfo.h"
@@ -738,18 +738,16 @@ SDValue VETargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // necessary since all emitted instructions must be stuck together in order
   // to pass the live physical registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, DL, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
   // Build the operands for the call instruction itself.
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const VERegisterInfo *TRI = Subtarget->getRegisterInfo();
@@ -1741,9 +1739,6 @@ static SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (TLI.verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   SDValue FrameAddr = lowerFRAMEADDR(Op, DAG, TLI, Subtarget);
 
   SDLoc DL(Op);
diff --git a/llvm/lib/Target/VE/VEMCInstLower.cpp b/llvm/lib/Target/VE/VEMCInstLower.cpp
index a438d8740cd0..f6b325ee8f3e 100644
--- a/llvm/lib/Target/VE/VEMCInstLower.cpp
+++ b/llvm/lib/Target/VE/VEMCInstLower.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/VEMCExpr.h"
+#include "MCTargetDesc/VEMCAsmInfo.h"
 #include "VE.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineFunction.h"
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index e4140755edf4..9cebffe993da 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -14,7 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AsmParser/WebAssemblyAsmTypeCheck.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
@@ -180,7 +180,7 @@ struct WebAssemblyOperand : public MCParsedAsmOperand {
     }
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Token:
       OS << "Tok:" << Tok.Tok;
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
index c0729cb4f3e9..4a305ab4d68e 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
@@ -14,7 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AsmParser/WebAssemblyAsmTypeCheck.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 8a29a5902ce2..0f7b27b26798 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -14,7 +14,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "TargetInfo/WebAssemblyTargetInfo.h"
 #include "llvm/BinaryFormat/Wasm.h"
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 7bc672c06947..837fd8eeac58 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -16,10 +16,10 @@
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWasmObjectWriter.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -78,10 +78,14 @@ bool WebAssemblyAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
   return true;
 }
 
-void WebAssemblyAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+void WebAssemblyAsmBackend::applyFixup(const MCFragment &F,
+                                       const MCFixup &Fixup,
                                        const MCValue &Target,
                                        MutableArrayRef<char> Data,
-                                       uint64_t Value, bool) {
+                                       uint64_t Value, bool IsResolved) {
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, Value);
+
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   assert(Info.Flags == 0 && "WebAssembly does not use MCFixupKindInfo flags");
 
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
index 321aee472082..2e97215f53ec 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyInstPrinter.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "llvm/ADT/APFloat.h"
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
index a9793e04c4b6..a783e5fd8521 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyMCAsmInfo.h"
-#include "WebAssemblyMCExpr.h"
 #include "WebAssemblyMCTargetDesc.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/TargetParser/Triple.h"
@@ -56,14 +55,7 @@ WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T,
   LCOMMDirectiveAlignmentType = LCOMM::Log2Alignment;
 
   SupportsDebugInformation = true;
+  ExceptionsType = ExceptionHandling::None;
 
-  // When compilation is done on a cpp file by clang, the exception model info
-  // is stored in LangOptions, which is later used to set the info in
-  // TargetOptions and then MCAsmInfo in CodeGenTargetMachine::initAsmInfo().
-  // But this process does not happen when compiling bitcode directly with
-  // clang, so we make sure this info is set correctly.
-  if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj)
-    ExceptionsType = ExceptionHandling::Wasm;
-
-  initializeVariantKinds(atSpecifiers);
+  initializeAtSpecifiers(atSpecifiers);
 }
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
index 5ba4dcf8c4b3..0b3778b36d6a 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
@@ -27,6 +27,18 @@ class WebAssemblyMCAsmInfo final : public MCAsmInfoWasm {
   ~WebAssemblyMCAsmInfo() override;
 };
 
+namespace WebAssembly {
+enum Specifier {
+  S_None,
+  S_FUNCINDEX, // Wasm function index
+  S_GOT,
+  S_GOT_TLS,   // Wasm global index of TLS symbol
+  S_MBREL,     // Memory address relative to __memory_base
+  S_TBREL,     // Table index relative to __table_base
+  S_TLSREL,    // Memory address relative to __tls_base
+  S_TYPEINDEX, // Reference to a symbol's type (signature)
+};
+}
 } // end namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 807fd42b8cf6..cbaf10fc18b8 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -156,10 +156,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
         const MCOperandInfo &Info = Desc.operands()[I];
         switch (Info.OperandType) {
         case WebAssembly::OPERAND_I32IMM:
-          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i32);
+          FixupKind = WebAssembly::fixup_sleb128_i32;
           break;
         case WebAssembly::OPERAND_I64IMM:
-          FixupKind = MCFixupKind(WebAssembly::fixup_sleb128_i64);
+          FixupKind = WebAssembly::fixup_sleb128_i64;
           PaddedSize = 10;
           break;
         case WebAssembly::OPERAND_FUNCTION32:
@@ -169,10 +169,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
         case WebAssembly::OPERAND_TYPEINDEX:
         case WebAssembly::OPERAND_GLOBAL:
         case WebAssembly::OPERAND_TAG:
-          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
+          FixupKind = WebAssembly::fixup_uleb128_i32;
           break;
         case WebAssembly::OPERAND_OFFSET64:
-          FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i64);
+          FixupKind = WebAssembly::fixup_uleb128_i64;
           PaddedSize = 10;
           break;
         default:
@@ -181,10 +181,10 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
       } else {
         // Variadic expr operands are try_table's catch/catch_ref clauses' tags.
         assert(Opcode == WebAssembly::TRY_TABLE_S);
-        FixupKind = MCFixupKind(WebAssembly::fixup_uleb128_i32);
+        FixupKind = WebAssembly::fixup_uleb128_i32;
       }
-      Fixups.push_back(MCFixup::create(OS.tell() - Start, MO.getExpr(),
-                                       FixupKind, MI.getLoc()));
+      Fixups.push_back(
+          MCFixup::create(OS.tell() - Start, MO.getExpr(), FixupKind));
       ++MCNumFixups;
       encodeULEB128(0, OS, PaddedSize);
     } else {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCExpr.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCExpr.h
deleted file mode 100644
index f74af06fb84f..000000000000
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCExpr.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===- WebAssembly specific MC expression classes ---------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Defines WebAssembly-specific relocation specifiers.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCEXPR_H
-#define LLVM_LIB_TARGET_WEBASSEMBLY_MCTARGETDESC_WEBASSEMBLYMCEXPR_H
-
-namespace llvm::WebAssembly {
-enum Specifier {
-  S_None,
-  S_FUNCINDEX, // Wasm function index
-  S_GOT,
-  S_GOT_TLS,   // Wasm global index of TLS symbol
-  S_MBREL,     // Memory address relative to __memory_base
-  S_TBREL,     // Table index relative to __table_base
-  S_TLSREL,    // Memory address relative to __tls_base
-  S_TYPEINDEX, // Reference to a symbol's type (signature)
-};
-} // namespace llvm::WebAssembly
-
-#endif
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index 6c0031f429c6..8019ae165ec8 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -36,35 +36,6 @@ using namespace llvm;
 #define GET_REGINFO_MC_DESC
 #include "WebAssemblyGenRegisterInfo.inc"
 
-// Exception handling & setjmp-longjmp handling related options.
-
-// Emscripten's asm.js-style exception handling
-cl::opt<bool> WebAssembly::WasmEnableEmEH(
-    "enable-emscripten-cxx-exceptions",
-    cl::desc("WebAssembly Emscripten-style exception handling"),
-    cl::init(false));
-// Emscripten's asm.js-style setjmp/longjmp handling
-cl::opt<bool> WebAssembly::WasmEnableEmSjLj(
-    "enable-emscripten-sjlj",
-    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
-    cl::init(false));
-// Exception handling using wasm EH instructions
-cl::opt<bool>
-    WebAssembly::WasmEnableEH("wasm-enable-eh",
-                              cl::desc("WebAssembly exception handling"));
-// setjmp/longjmp handling using wasm EH instrutions
-cl::opt<bool> WebAssembly::WasmEnableSjLj(
-    "wasm-enable-sjlj", cl::desc("WebAssembly setjmp/longjmp handling"));
-// If true, use the legacy Wasm EH proposal:
-// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/legacy/Exceptions.md
-// And if false, use the standardized Wasm EH proposal:
-// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/Exceptions.md
-// Currently set to true by default because not all major web browsers turn on
-// the new standard proposal by default, but will later change to false.
-cl::opt<bool> WebAssembly::WasmUseLegacyEH(
-    "wasm-use-legacy-eh", cl::desc("WebAssembly exception handling (legacy)"),
-    cl::init(true));
-
 static MCAsmInfo *createMCAsmInfo(const MCRegisterInfo & /*MRI*/,
                                   const Triple &TT,
                                   const MCTargetOptions &Options) {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index d6a2fe4c7839..fe9a4bada243 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -39,13 +39,6 @@ createWebAssemblyWasmObjectWriter(bool Is64Bit, bool IsEmscripten);
 
 namespace WebAssembly {
 
-// Exception handling / setjmp-longjmp handling command-line options
-extern cl::opt<bool> WasmEnableEmEH;   // asm.js-style EH
-extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ
-extern cl::opt<bool> WasmEnableEH;     // EH using Wasm EH instructions
-extern cl::opt<bool> WasmEnableSjLj;   // SjLj using Wasm EH instructions
-extern cl::opt<bool> WasmUseLegacyEH;  // Legacy Wasm EH
-
 enum OperandType {
   /// Basic block label in a branch construct.
   OPERAND_BASIC_BLOCK = MCOI::OPERAND_FIRST_TARGET,
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
index 1a9aa1e14f71..c18de5fc1939 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTypeUtilities.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionWasm.h"
@@ -112,7 +113,9 @@ void WebAssemblyTargetAsmStreamer::emitExportName(const MCSymbolWasm *Sym,
 }
 
 void WebAssemblyTargetAsmStreamer::emitIndIdx(const MCExpr *Value) {
-  OS << "\t.indidx  \t" << *Value << '\n';
+  OS << "\t.indidx\t";
+  getContext().getAsmInfo()->printExpr(OS, *Value);
+  OS << '\n';
 }
 
 void WebAssemblyTargetWasmStreamer::emitLocal(ArrayRef<wasm::ValType> Types) {
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
index 33cf12e59870..2cf4bec07738 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/WebAssemblyFixupKinds.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/MC/MCAsmBackend.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssembly.h b/llvm/lib/Target/WebAssembly/WebAssembly.h
index 17481d77c120..2dbd597f01cc 100644
--- a/llvm/lib/Target/WebAssembly/WebAssembly.h
+++ b/llvm/lib/Target/WebAssembly/WebAssembly.h
@@ -44,7 +44,7 @@ FunctionPass *createWebAssemblyReplacePhysRegs();
 FunctionPass *createWebAssemblyNullifyDebugValueLists();
 FunctionPass *createWebAssemblyOptimizeLiveIntervals();
 FunctionPass *createWebAssemblyMemIntrinsicResults();
-FunctionPass *createWebAssemblyRegStackify();
+FunctionPass *createWebAssemblyRegStackify(CodeGenOptLevel OptLevel);
 FunctionPass *createWebAssemblyRegColoring();
 FunctionPass *createWebAssemblyFixBrTableDefaults();
 FunctionPass *createWebAssemblyFixIrreducibleControlFlow();
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
index b43b7dbfc36b..1bf070e9ec9c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -14,7 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyAsmPrinter.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
 #include "TargetInfo/WebAssemblyTargetInfo.h"
@@ -24,6 +24,7 @@
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblyRegisterInfo.h"
 #include "WebAssemblyRuntimeLibcallSignatures.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallSet.h"
@@ -155,9 +156,11 @@ static std::string getEmscriptenInvokeSymbolName(wasm::WasmSignature *Sig) {
 //===----------------------------------------------------------------------===//
 
 MCSymbolWasm *WebAssemblyAsmPrinter::getMCSymbolForFunction(
-    const Function *F, bool EnableEmEH, wasm::WasmSignature *Sig,
-    bool &InvokeDetected) {
+    const Function *F, wasm::WasmSignature *Sig, bool &InvokeDetected) {
   MCSymbolWasm *WasmSym = nullptr;
+
+  const bool EnableEmEH =
+      WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj;
   if (EnableEmEH && isEmscriptenInvokeName(F->getName())) {
     assert(Sig);
     InvokeDetected = true;
@@ -344,9 +347,7 @@ void WebAssemblyAsmPrinter::emitDecls(const Module &M) {
     // will discard it later if it turns out not to be necessary.
     auto Signature = signatureFromMVTs(OutContext, Results, Params);
     bool InvokeDetected = false;
-    auto *Sym = getMCSymbolForFunction(
-        &F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
-        Signature, InvokeDetected);
+    auto *Sym = getMCSymbolForFunction(&F, Signature, InvokeDetected);
 
     // Multiple functions can be mapped to the same invoke symbol. For
     // example, two IR functions '__invoke_void_i8*' and '__invoke_void_i32'
@@ -404,7 +405,7 @@ void WebAssemblyAsmPrinter::emitEndOfAsmFile(Module &M) {
     if (!F.isIntrinsic() && F.hasAddressTaken()) {
       MCSymbolWasm *FunctionTable =
           WebAssembly::getOrCreateFunctionTableSymbol(OutContext, Subtarget);
-      OutStreamer->emitSymbolAttribute(FunctionTable, MCSA_NoDeadStrip);    
+      OutStreamer->emitSymbolAttribute(FunctionTable, MCSA_NoDeadStrip);
       break;
     }
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
index 46063bbe0fba..aef562e1dd68 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
@@ -73,7 +73,7 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyAsmPrinter final : public AsmPrinter {
   MVT getRegType(unsigned RegNo) const;
   std::string regToString(const MachineOperand &MO);
   WebAssemblyTargetStreamer *getTargetStreamer();
-  MCSymbolWasm *getMCSymbolForFunction(const Function *F, bool EnableEmEH,
+  MCSymbolWasm *getMCSymbolForFunction(const Function *F,
                                        wasm::WasmSignature *Sig,
                                        bool &InvokeDetected);
   MCSymbol *getOrCreateWasmSymbol(StringRef Name);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 640be5fe8e8c..acb889f00a48 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -21,13 +21,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "Utils/WebAssemblyTypeUtilities.h"
 #include "WebAssembly.h"
 #include "WebAssemblyExceptionInfo.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblySortRegion.h"
 #include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/BinaryFormat/Wasm.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index 48c0b7e50f08..ac819cf5c180 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -343,17 +343,29 @@ bool WebAssemblyDAGToDAGISel::SelectAddrAddOperands(MVT OffsetType, SDValue N,
   if (N.getOpcode() == ISD::ADD && !N.getNode()->getFlags().hasNoUnsignedWrap())
     return false;
 
-  // Folds constants in an add into the offset.
   for (size_t i = 0; i < 2; ++i) {
     SDValue Op = N.getOperand(i);
     SDValue OtherOp = N.getOperand(i == 0 ? 1 : 0);
 
+    // Folds constants in an add into the offset.
     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
       Offset =
           CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), OffsetType);
       Addr = OtherOp;
       return true;
     }
+
+    // Fold target global addresses into the offset.
+    if (!TM.isPositionIndependent()) {
+      if (Op.getOpcode() == WebAssemblyISD::Wrapper)
+        Op = Op.getOperand(0);
+
+      if (Op.getOpcode() == ISD::TargetGlobalAddress) {
+        Addr = OtherOp;
+        Offset = Op;
+        return true;
+      }
+    }
   }
   return false;
 }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 3cd923c0ba05..bf2e04caa0a6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -1909,9 +1910,6 @@ SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
     return SDValue();
   }
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   MakeLibCallOptions CallOptions;
   return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
@@ -3214,20 +3212,23 @@ static SDValue performTruncateCombine(SDNode *N,
 
 static SDValue performBitcastCombine(SDNode *N,
                                      TargetLowering::DAGCombinerInfo &DCI) {
+  using namespace llvm::SDPatternMatch;
   auto &DAG = DCI.DAG;
   SDLoc DL(N);
   SDValue Src = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT SrcVT = Src.getValueType();
 
-  // bitcast <N x i1> to iN
+  if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
+        SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1))
+    return SDValue();
+
+  unsigned NumElts = SrcVT.getVectorNumElements();
+  EVT Width = MVT::getIntegerVT(128 / NumElts);
+
+  // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
   //   ==> bitmask
-  if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
-      SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1) {
-    unsigned NumElts = SrcVT.getVectorNumElements();
-    if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
-      return SDValue();
-    EVT Width = MVT::getIntegerVT(128 / NumElts);
+  if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
     return DAG.getZExtOrTrunc(
         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                     {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
@@ -3236,55 +3237,178 @@ static SDValue performBitcastCombine(SDNode *N,
         DL, VT);
   }
 
+  // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
+  if (NumElts == 32 || NumElts == 64) {
+    // Strategy: We will setcc them seperately in v16i8 -> v16i1
+    // Bitcast them to i16, extend them to either i32 or i64.
+    // Add them together, shifting left 1 by 1.
+    SDValue Concat, SetCCVector;
+    ISD::CondCode SetCond;
+
+    if (!sd_match(N, m_BitCast(m_c_SetCC(m_Value(Concat), m_Value(SetCCVector),
+                                         m_CondCode(SetCond)))))
+      return SDValue();
+    if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
+      return SDValue();
+
+    uint64_t ElementWidth =
+        SetCCVector.getValueType().getVectorElementType().getFixedSizeInBits();
+
+    SmallVector<SDValue> VectorsToShuffle;
+    for (size_t I = 0; I < Concat->ops().size(); I++) {
+      VectorsToShuffle.push_back(DAG.getBitcast(
+          MVT::i16,
+          DAG.getSetCC(DL, MVT::v16i1, Concat->ops()[I],
+                       extractSubVector(SetCCVector, I * (128 / ElementWidth),
+                                        DAG, DL, 128),
+                       SetCond)));
+    }
+
+    MVT ReturnType = VectorsToShuffle.size() == 2 ? MVT::i32 : MVT::i64;
+    SDValue ReturningInteger = DAG.getConstant(0, DL, ReturnType);
+
+    for (SDValue V : VectorsToShuffle) {
+      ReturningInteger = DAG.getNode(
+          ISD::SHL, DL, ReturnType,
+          {DAG.getShiftAmountConstant(16, ReturnType, DL), ReturningInteger});
+
+      SDValue ExtendedV = DAG.getZExtOrTrunc(V, DL, ReturnType);
+      ReturningInteger =
+          DAG.getNode(ISD::ADD, DL, ReturnType, {ReturningInteger, ExtendedV});
+    }
+
+    return ReturningInteger;
+  }
+
   return SDValue();
 }
 
-static SDValue performSETCCCombine(SDNode *N,
-                                   TargetLowering::DAGCombinerInfo &DCI) {
-  auto &DAG = DCI.DAG;
+static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {
+  // any_true (setcc <X>, 0, eq) => (not (all_true X))
+  // all_true (setcc <X>, 0, eq) => (not (any_true X))
+  // any_true (setcc <X>, 0, ne) => (any_true X)
+  // all_true (setcc <X>, 0, ne) => (all_true X)
+  assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
+  using namespace llvm::SDPatternMatch;
 
+  SDValue LHS;
+  if (!sd_match(N->getOperand(1),
+                m_c_SetCC(m_Value(LHS), m_Zero(), m_CondCode())))
+    return SDValue();
+  EVT LT = LHS.getValueType();
+  if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
+    return SDValue();
+
+  auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
+                                 ISD::CondCode SetType,
+                                 Intrinsic::WASMIntrinsics InPost) {
+    if (N->getConstantOperandVal(0) != InPre)
+      return SDValue();
+
+    SDValue LHS;
+    if (!sd_match(N->getOperand(1), m_c_SetCC(m_Value(LHS), m_Zero(),
+                                              m_SpecificCondCode(SetType))))
+      return SDValue();
+
+    SDLoc DL(N);
+    SDValue Ret = DAG.getZExtOrTrunc(
+        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
+                    {DAG.getConstant(InPost, DL, MVT::i32), LHS}),
+        DL, MVT::i1);
+    if (SetType == ISD::SETEQ)
+      Ret = DAG.getNOT(DL, Ret, MVT::i1);
+    return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
+  };
+
+  if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
+                                       Intrinsic::wasm_alltrue))
+    return AnyTrueEQ;
+  if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
+                                       Intrinsic::wasm_anytrue))
+    return AllTrueEQ;
+  if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
+                                       Intrinsic::wasm_anytrue))
+    return AnyTrueNE;
+  if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
+                                       Intrinsic::wasm_alltrue))
+    return AllTrueNE;
+
+  return SDValue();
+}
+
+template <int MatchRHS, ISD::CondCode MatchCond, bool RequiresNegate,
+          Intrinsic::ID Intrin>
+static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG) {
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
-  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
+  SDValue Cond = N->getOperand(2);
+  if (MatchCond != cast<CondCodeSDNode>(Cond)->get())
+    return SDValue();
+
+  if (MatchRHS != cast<ConstantSDNode>(RHS)->getSExtValue())
+    return SDValue();
+
   SDLoc DL(N);
+  SDValue Ret = DAG.getZExtOrTrunc(
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
+                  {DAG.getConstant(Intrin, DL, MVT::i32),
+                   DAG.getSExtOrTrunc(LHS->getOperand(0), DL, VecVT)}),
+      DL, MVT::i1);
+  if (RequiresNegate)
+    Ret = DAG.getNOT(DL, Ret, MVT::i1);
+  return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
+}
+
+static SDValue performSETCCCombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI) {
+  if (!DCI.isBeforeLegalize())
+    return SDValue();
+
   EVT VT = N->getValueType(0);
+  if (!VT.isScalarInteger())
+    return SDValue();
 
+  SDValue LHS = N->getOperand(0);
+  if (LHS->getOpcode() != ISD::BITCAST)
+    return SDValue();
+
+  EVT FromVT = LHS->getOperand(0).getValueType();
+  if (!FromVT.isFixedLengthVector() || FromVT.getVectorElementType() != MVT::i1)
+    return SDValue();
+
+  unsigned NumElts = FromVT.getVectorNumElements();
+  if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
+    return SDValue();
+
+  if (!cast<ConstantSDNode>(N->getOperand(1)))
+    return SDValue();
+
+  EVT VecVT = FromVT.changeVectorElementType(MVT::getIntegerVT(128 / NumElts));
+  auto &DAG = DCI.DAG;
   // setcc (iN (bitcast (vNi1 X))), 0, ne
   //   ==> any_true (vNi1 X)
+  if (auto Match = TryMatchTrue<0, ISD::SETNE, false, Intrinsic::wasm_anytrue>(
+          N, VecVT, DAG)) {
+    return Match;
+  }
   // setcc (iN (bitcast (vNi1 X))), 0, eq
   //   ==> xor (any_true (vNi1 X)), -1
+  if (auto Match = TryMatchTrue<0, ISD::SETEQ, true, Intrinsic::wasm_anytrue>(
+          N, VecVT, DAG)) {
+    return Match;
+  }
   // setcc (iN (bitcast (vNi1 X))), -1, eq
   //   ==> all_true (vNi1 X)
+  if (auto Match = TryMatchTrue<-1, ISD::SETEQ, false, Intrinsic::wasm_alltrue>(
+          N, VecVT, DAG)) {
+    return Match;
+  }
   // setcc (iN (bitcast (vNi1 X))), -1, ne
   //   ==> xor (all_true (vNi1 X)), -1
-  if (DCI.isBeforeLegalize() && VT.isScalarInteger() &&
-      (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
-      (isNullConstant(RHS) || isAllOnesConstant(RHS)) &&
-      LHS->getOpcode() == ISD::BITCAST) {
-    EVT FromVT = LHS->getOperand(0).getValueType();
-    if (FromVT.isFixedLengthVector() &&
-        FromVT.getVectorElementType() == MVT::i1) {
-      int Intrin = isNullConstant(RHS) ? Intrinsic::wasm_anytrue
-                                       : Intrinsic::wasm_alltrue;
-      unsigned NumElts = FromVT.getVectorNumElements();
-      if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
-        return SDValue();
-      EVT Width = MVT::getIntegerVT(128 / NumElts);
-      SDValue Ret = DAG.getZExtOrTrunc(
-          DAG.getNode(
-              ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
-              {DAG.getConstant(Intrin, DL, MVT::i32),
-               DAG.getSExtOrTrunc(LHS->getOperand(0), DL,
-                                  FromVT.changeVectorElementType(Width))}),
-          DL, MVT::i1);
-      if ((isNullConstant(RHS) && (Cond == ISD::SETEQ)) ||
-          (isAllOnesConstant(RHS) && (Cond == ISD::SETNE))) {
-        Ret = DAG.getNOT(DL, Ret, MVT::i1);
-      }
-      return DAG.getZExtOrTrunc(Ret, DL, VT);
-    }
+  if (auto Match = TryMatchTrue<-1, ISD::SETNE, true, Intrinsic::wasm_alltrue>(
+          N, VecVT, DAG)) {
+    return Match;
   }
-
   return SDValue();
 }
 
@@ -3400,8 +3524,11 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
     return performVectorTruncZeroCombine(N, DCI);
   case ISD::TRUNCATE:
     return performTruncateCombine(N, DCI);
-  case ISD::INTRINSIC_WO_CHAIN:
+  case ISD::INTRINSIC_WO_CHAIN: {
+    if (auto AnyAllCombine = performAnyAllCombine(N, DCI.DAG))
+      return AnyAllCombine;
     return performLowerPartialReduction(N, DCI.DAG);
+  }
   case ISD::MUL:
     return performMulCombine(N, DCI.DAG);
   }
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
index 254ad5c4f2be..8ac32f939c5f 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
@@ -11,9 +11,9 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "WebAssembly.h"
 #include "WebAssemblySubtarget.h"
+#include "WebAssemblyTargetMachine.h"
 #include "WebAssemblyUtilities.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index 8c8629203bca..cc36244e63ff 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "WebAssemblyMCInstLower.h"
-#include "MCTargetDesc/WebAssemblyMCExpr.h"
+#include "MCTargetDesc/WebAssemblyMCAsmInfo.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "TargetInfo/WebAssemblyTargetInfo.h"
 #include "Utils/WebAssemblyTypeUtilities.h"
@@ -77,9 +77,7 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
   auto Signature = signatureFromMVTs(Ctx, ResultMVTs, ParamMVTs);
 
   bool InvokeDetected = false;
-  auto *WasmSym = Printer.getMCSymbolForFunction(
-      F, WebAssembly::WasmEnableEmEH || WebAssembly::WasmEnableEmSjLj,
-      Signature, InvokeDetected);
+  auto *WasmSym = Printer.getMCSymbolForFunction(F, Signature, InvokeDetected);
   WasmSym->setSignature(Signature);
   WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
   return WasmSym;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
index 428d573668fb..bc91c6424b63 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
@@ -41,14 +41,18 @@ using namespace llvm;
 
 namespace {
 class WebAssemblyRegStackify final : public MachineFunctionPass {
+  bool Optimize;
+
   StringRef getPassName() const override {
     return "WebAssembly Register Stackify";
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
-    AU.addRequired<MachineDominatorTreeWrapperPass>();
-    AU.addRequired<LiveIntervalsWrapperPass>();
+    if (Optimize) {
+      AU.addRequired<LiveIntervalsWrapperPass>();
+      AU.addRequired<MachineDominatorTreeWrapperPass>();
+    }
     AU.addPreserved<MachineBlockFrequencyInfoWrapperPass>();
     AU.addPreserved<SlotIndexesWrapperPass>();
     AU.addPreserved<LiveIntervalsWrapperPass>();
@@ -61,7 +65,9 @@ class WebAssemblyRegStackify final : public MachineFunctionPass {
 
 public:
   static char ID; // Pass identification, replacement for typeid
-  WebAssemblyRegStackify() : MachineFunctionPass(ID) {}
+  WebAssemblyRegStackify(CodeGenOptLevel OptLevel)
+      : MachineFunctionPass(ID), Optimize(OptLevel != CodeGenOptLevel::None) {}
+  WebAssemblyRegStackify() : WebAssemblyRegStackify(CodeGenOptLevel::Default) {}
 };
 } // end anonymous namespace
 
@@ -70,8 +76,8 @@ INITIALIZE_PASS(WebAssemblyRegStackify, DEBUG_TYPE,
                 "Reorder instructions to use the WebAssembly value stack",
                 false, false)
 
-FunctionPass *llvm::createWebAssemblyRegStackify() {
-  return new WebAssemblyRegStackify();
+FunctionPass *llvm::createWebAssemblyRegStackify(CodeGenOptLevel OptLevel) {
+  return new WebAssemblyRegStackify(OptLevel);
 }
 
 // Decorate the given instruction with implicit operands that enforce the
@@ -96,8 +102,7 @@ static void imposeStackOrdering(MachineInstr *MI) {
 static void convertImplicitDefToConstZero(MachineInstr *MI,
                                           MachineRegisterInfo &MRI,
                                           const TargetInstrInfo *TII,
-                                          MachineFunction &MF,
-                                          LiveIntervals &LIS) {
+                                          MachineFunction &MF) {
   assert(MI->getOpcode() == TargetOpcode::IMPLICIT_DEF);
 
   const auto *RegClass = MRI.getRegClass(MI->getOperand(0).getReg());
@@ -262,36 +267,53 @@ static bool shouldRematerialize(const MachineInstr &Def,
 // LiveIntervals to handle complex cases.
 static MachineInstr *getVRegDef(unsigned Reg, const MachineInstr *Insert,
                                 const MachineRegisterInfo &MRI,
-                                const LiveIntervals &LIS) {
+                                const LiveIntervals *LIS) {
   // Most registers are in SSA form here so we try a quick MRI query first.
   if (MachineInstr *Def = MRI.getUniqueVRegDef(Reg))
     return Def;
 
   // MRI doesn't know what the Def is. Try asking LIS.
-  if (const VNInfo *ValNo = LIS.getInterval(Reg).getVNInfoBefore(
-          LIS.getInstructionIndex(*Insert)))
-    return LIS.getInstructionFromIndex(ValNo->def);
+  if (LIS != nullptr) {
+    SlotIndex InstIndex = LIS->getInstructionIndex(*Insert);
+    if (const VNInfo *ValNo = LIS->getInterval(Reg).getVNInfoBefore(InstIndex))
+      return LIS->getInstructionFromIndex(ValNo->def);
+  }
 
   return nullptr;
 }
 
 // Test whether Reg, as defined at Def, has exactly one use. This is a
 // generalization of MachineRegisterInfo::hasOneNonDBGUse that uses
-// LiveIntervals to handle complex cases.
-static bool hasOneNonDBGUse(unsigned Reg, MachineInstr *Def,
-                            MachineRegisterInfo &MRI, MachineDominatorTree &MDT,
-                            LiveIntervals &LIS) {
+// LiveIntervals to handle complex cases in optimized code.
+static bool hasSingleUse(unsigned Reg, MachineRegisterInfo &MRI,
+                         WebAssemblyFunctionInfo &MFI, bool Optimize,
+                         MachineInstr *Def, LiveIntervals *LIS) {
+  if (!Optimize) {
+    // Using "hasOneUse" instead of "hasOneNonDBGUse" here because we don't
+    // want to stackify DBG_VALUE operands - WASM stack locations are less
+    // useful and less widely supported than WASM local locations.
+    if (!MRI.hasOneUse(Reg))
+      return false;
+    // The frame base always has an implicit DBG use as DW_AT_frame_base.
+    if (MFI.isFrameBaseVirtual() && MFI.getFrameBaseVreg() == Reg)
+      return false;
+    return true;
+  }
+
   // Most registers are in SSA form here so we try a quick MRI query first.
   if (MRI.hasOneNonDBGUse(Reg))
     return true;
 
+  if (LIS == nullptr)
+    return false;
+
   bool HasOne = false;
-  const LiveInterval &LI = LIS.getInterval(Reg);
+  const LiveInterval &LI = LIS->getInterval(Reg);
   const VNInfo *DefVNI =
-      LI.getVNInfoAt(LIS.getInstructionIndex(*Def).getRegSlot());
+      LI.getVNInfoAt(LIS->getInstructionIndex(*Def).getRegSlot());
   assert(DefVNI);
   for (auto &I : MRI.use_nodbg_operands(Reg)) {
-    const auto &Result = LI.Query(LIS.getInstructionIndex(*I.getParent()));
+    const auto &Result = LI.Query(LIS->getInstructionIndex(*I.getParent()));
     if (Result.valueIn() == DefVNI) {
       if (!Result.isKill())
         return false;
@@ -311,7 +333,7 @@ static bool hasOneNonDBGUse(unsigned Reg, MachineInstr *Def,
 static bool isSafeToMove(const MachineOperand *Def, const MachineOperand *Use,
                          const MachineInstr *Insert,
                          const WebAssemblyFunctionInfo &MFI,
-                         const MachineRegisterInfo &MRI) {
+                         const MachineRegisterInfo &MRI, bool Optimize) {
   const MachineInstr *DefI = Def->getParent();
   const MachineInstr *UseI = Use->getParent();
   assert(DefI->getParent() == Insert->getParent());
@@ -357,6 +379,12 @@ static bool isSafeToMove(const MachineOperand *Def, const MachineOperand *Use,
   if (NextI == Insert)
     return true;
 
+  // When not optimizing, we only handle the trivial case above
+  // to guarantee no impact to debugging and to avoid spending
+  // compile time.
+  if (!Optimize)
+    return false;
+
   // 'catch' and 'catch_all' should be the first instruction of a BB and cannot
   // move.
   if (WebAssembly::isCatch(DefI->getOpcode()))
@@ -520,14 +548,15 @@ static void shrinkToUses(LiveInterval &LI, LiveIntervals &LIS) {
 /// dependencies; move the def down and nest it with the current instruction.
 static MachineInstr *moveForSingleUse(unsigned Reg, MachineOperand &Op,
                                       MachineInstr *Def, MachineBasicBlock &MBB,
-                                      MachineInstr *Insert, LiveIntervals &LIS,
+                                      MachineInstr *Insert, LiveIntervals *LIS,
                                       WebAssemblyFunctionInfo &MFI,
                                       MachineRegisterInfo &MRI) {
   LLVM_DEBUG(dbgs() << "Move for single use: "; Def->dump());
 
   WebAssemblyDebugValueManager DefDIs(Def);
   DefDIs.sink(Insert);
-  LIS.handleMove(*Def);
+  if (LIS != nullptr)
+    LIS->handleMove(*Def);
 
   if (MRI.hasOneDef(Reg) && MRI.hasOneNonDBGUse(Reg)) {
     // No one else is using this register for anything so we can just stackify
@@ -540,17 +569,18 @@ static MachineInstr *moveForSingleUse(unsigned Reg, MachineOperand &Op,
     Op.setReg(NewReg);
     DefDIs.updateReg(NewReg);
 
-    // Tell LiveIntervals about the new register.
-    LIS.createAndComputeVirtRegInterval(NewReg);
+    if (LIS != nullptr) {
+      // Tell LiveIntervals about the new register.
+      LIS->createAndComputeVirtRegInterval(NewReg);
 
-    // Tell LiveIntervals about the changes to the old register.
-    LiveInterval &LI = LIS.getInterval(Reg);
-    LI.removeSegment(LIS.getInstructionIndex(*Def).getRegSlot(),
-                     LIS.getInstructionIndex(*Op.getParent()).getRegSlot(),
-                     /*RemoveDeadValNo=*/true);
+      // Tell LiveIntervals about the changes to the old register.
+      LiveInterval &LI = LIS->getInterval(Reg);
+      LI.removeSegment(LIS->getInstructionIndex(*Def).getRegSlot(),
+                       LIS->getInstructionIndex(*Op.getParent()).getRegSlot(),
+                       /*RemoveDeadValNo=*/true);
+    }
 
     MFI.stackifyVReg(MRI, NewReg);
-
     LLVM_DEBUG(dbgs() << " - Replaced register: "; Def->dump());
   }
 
@@ -567,11 +597,12 @@ static MachineInstr *getPrevNonDebugInst(MachineInstr *MI) {
 
 /// A trivially cloneable instruction; clone it and nest the new copy with the
 /// current instruction.
-static MachineInstr *rematerializeCheapDef(
-    unsigned Reg, MachineOperand &Op, MachineInstr &Def, MachineBasicBlock &MBB,
-    MachineBasicBlock::instr_iterator Insert, LiveIntervals &LIS,
-    WebAssemblyFunctionInfo &MFI, MachineRegisterInfo &MRI,
-    const WebAssemblyInstrInfo *TII, const WebAssemblyRegisterInfo *TRI) {
+static MachineInstr *
+rematerializeCheapDef(unsigned Reg, MachineOperand &Op, MachineInstr &Def,
+                      MachineBasicBlock::instr_iterator Insert,
+                      LiveIntervals &LIS, WebAssemblyFunctionInfo &MFI,
+                      MachineRegisterInfo &MRI,
+                      const WebAssemblyInstrInfo *TII) {
   LLVM_DEBUG(dbgs() << "Rematerializing cheap def: "; Def.dump());
   LLVM_DEBUG(dbgs() << " - for use in "; Op.getParent()->dump());
 
@@ -811,9 +842,12 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
   MachineRegisterInfo &MRI = MF.getRegInfo();
   WebAssemblyFunctionInfo &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-  const auto *TRI = MF.getSubtarget<WebAssemblySubtarget>().getRegisterInfo();
-  auto &MDT = getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
-  auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  MachineDominatorTree *MDT = nullptr;
+  LiveIntervals *LIS = nullptr;
+  if (Optimize) {
+    MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+    LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+  }
 
   // Walk the instructions from the bottom up. Currently we don't look past
   // block boundaries, and the blocks aren't ordered so the block visitation
@@ -876,23 +910,28 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         // supports intra-block moves) and it's MachineSink's job to catch all
         // the sinking opportunities anyway.
         bool SameBlock = DefI->getParent() == &MBB;
-        bool CanMove = SameBlock && isSafeToMove(Def, &Use, Insert, MFI, MRI) &&
+        bool CanMove = SameBlock &&
+                       isSafeToMove(Def, &Use, Insert, MFI, MRI, Optimize) &&
                        !TreeWalker.isOnStack(Reg);
-        if (CanMove && hasOneNonDBGUse(Reg, DefI, MRI, MDT, LIS)) {
+        if (CanMove && hasSingleUse(Reg, MRI, MFI, Optimize, DefI, LIS)) {
           Insert = moveForSingleUse(Reg, Use, DefI, MBB, Insert, LIS, MFI, MRI);
 
           // If we are removing the frame base reg completely, remove the debug
           // info as well.
           // TODO: Encode this properly as a stackified value.
-          if (MFI.isFrameBaseVirtual() && MFI.getFrameBaseVreg() == Reg)
+          if (MFI.isFrameBaseVirtual() && MFI.getFrameBaseVreg() == Reg) {
+            assert(
+                Optimize &&
+                "Stackifying away frame base in unoptimized code not expected");
             MFI.clearFrameBaseVreg();
-        } else if (shouldRematerialize(*DefI, TII)) {
-          Insert =
-              rematerializeCheapDef(Reg, Use, *DefI, MBB, Insert->getIterator(),
-                                    LIS, MFI, MRI, TII, TRI);
-        } else if (CanMove && oneUseDominatesOtherUses(Reg, Use, MBB, MRI, MDT,
-                                                       LIS, MFI)) {
-          Insert = moveAndTeeForMultiUse(Reg, Use, DefI, MBB, Insert, LIS, MFI,
+          }
+        } else if (Optimize && shouldRematerialize(*DefI, TII)) {
+          Insert = rematerializeCheapDef(Reg, Use, *DefI, Insert->getIterator(),
+                                         *LIS, MFI, MRI, TII);
+        } else if (Optimize && CanMove &&
+                   oneUseDominatesOtherUses(Reg, Use, MBB, MRI, *MDT, *LIS,
+                                            MFI)) {
+          Insert = moveAndTeeForMultiUse(Reg, Use, DefI, MBB, Insert, *LIS, MFI,
                                          MRI, TII);
         } else {
           // We failed to stackify the operand. If the problem was ordering
@@ -915,7 +954,8 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
           Register DefReg = SubsequentDef->getReg();
           Register UseReg = SubsequentUse->getReg();
           // TODO: This single-use restriction could be relaxed by using tees
-          if (DefReg != UseReg || !MRI.hasOneNonDBGUse(DefReg))
+          if (DefReg != UseReg ||
+              !hasSingleUse(DefReg, MRI, MFI, Optimize, nullptr, nullptr))
             break;
           MFI.stackifyVReg(MRI, DefReg);
           ++SubsequentDef;
@@ -926,7 +966,7 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         // to a constant 0 so that the def is explicit, and the push/pop
         // correspondence is maintained.
         if (Insert->getOpcode() == TargetOpcode::IMPLICIT_DEF)
-          convertImplicitDefToConstZero(Insert, MRI, TII, MF, LIS);
+          convertImplicitDefToConstZero(Insert, MRI, TII, MF);
 
         // We stackified an operand. Add the defining instruction's operands to
         // the worklist stack now to continue to build an ever deeper tree.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
index 17889dacc868..31a33c1e7365 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
@@ -64,8 +64,8 @@ def I32 : WebAssemblyRegClass<[i32], 32, (add FP32, SP32, I32_0)>;
 def I64 : WebAssemblyRegClass<[i64], 64, (add FP64, SP64, I64_0)>;
 def F32 : WebAssemblyRegClass<[f32], 32, (add F32_0)>;
 def F64 : WebAssemblyRegClass<[f64], 64, (add F64_0)>;
-def V128 : WebAssemblyRegClass<[v8f16, v4f32, v2f64, v2i64, v4i32, v16i8,
-                                v8i16],
+def V128 : WebAssemblyRegClass<[v2i64, v4i32, v16i8, v8i16,
+                                v8f16, v4f32, v2f64],
                                128, (add V128_0)>;
 def FUNCREF : WebAssemblyRegClass<[funcref], 0, (add FUNCREF_0)>;
 def EXTERNREF : WebAssemblyRegClass<[externref], 0, (add EXTERNREF_0)>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index 6e551e5c8ee4..ad47cb8ea2fe 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -54,6 +54,35 @@ static cl::opt<bool> WasmDisableFixIrreducibleControlFlowPass(
              " irreducible control flow optimization pass"),
     cl::init(false));
 
+// Exception handling & setjmp-longjmp handling related options.
+
+// Emscripten's asm.js-style exception handling
+cl::opt<bool> WebAssembly::WasmEnableEmEH(
+    "enable-emscripten-cxx-exceptions",
+    cl::desc("WebAssembly Emscripten-style exception handling"),
+    cl::init(false));
+// Emscripten's asm.js-style setjmp/longjmp handling
+cl::opt<bool> WebAssembly::WasmEnableEmSjLj(
+    "enable-emscripten-sjlj",
+    cl::desc("WebAssembly Emscripten-style setjmp/longjmp handling"),
+    cl::init(false));
+// Exception handling using wasm EH instructions
+cl::opt<bool>
+    WebAssembly::WasmEnableEH("wasm-enable-eh",
+                              cl::desc("WebAssembly exception handling"));
+// setjmp/longjmp handling using wasm EH instrutions
+cl::opt<bool> WebAssembly::WasmEnableSjLj(
+    "wasm-enable-sjlj", cl::desc("WebAssembly setjmp/longjmp handling"));
+// If true, use the legacy Wasm EH proposal:
+// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/legacy/Exceptions.md
+// And if false, use the standardized Wasm EH proposal:
+// https://github.com/WebAssembly/exception-handling/blob/main/proposals/exception-handling/Exceptions.md
+// Currently set to true by default because not all major web browsers turn on
+// the new standard proposal by default, but will later change to false.
+cl::opt<bool> WebAssembly::WasmUseLegacyEH(
+    "wasm-use-legacy-eh", cl::desc("WebAssembly exception handling (legacy)"),
+    cl::init(true));
+
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeWebAssemblyTarget() {
   // Register the target.
@@ -111,6 +140,57 @@ static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM,
   return *RM;
 }
 
+using WebAssembly::WasmEnableEH;
+using WebAssembly::WasmEnableEmEH;
+using WebAssembly::WasmEnableEmSjLj;
+using WebAssembly::WasmEnableSjLj;
+
+static void basicCheckForEHAndSjLj(TargetMachine *TM) {
+
+  // You can't enable two modes of EH at the same time
+  if (WasmEnableEmEH && WasmEnableEH)
+    report_fatal_error(
+        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-eh");
+  // You can't enable two modes of SjLj at the same time
+  if (WasmEnableEmSjLj && WasmEnableSjLj)
+    report_fatal_error(
+        "-enable-emscripten-sjlj not allowed with -wasm-enable-sjlj");
+  // You can't mix Emscripten EH with Wasm SjLj.
+  if (WasmEnableEmEH && WasmEnableSjLj)
+    report_fatal_error(
+        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-sjlj");
+
+  if (TM->Options.ExceptionModel == ExceptionHandling::None) {
+    // FIXME: These flags should be removed in favor of directly using the
+    // generically configured ExceptionsType
+    if (WebAssembly::WasmEnableEH || WebAssembly::WasmEnableSjLj)
+      TM->Options.ExceptionModel = ExceptionHandling::Wasm;
+  }
+
+  // Basic Correctness checking related to -exception-model
+  if (TM->Options.ExceptionModel != ExceptionHandling::None &&
+      TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error("-exception-model should be either 'none' or 'wasm'");
+  if (WasmEnableEmEH && TM->Options.ExceptionModel == ExceptionHandling::Wasm)
+    report_fatal_error("-exception-model=wasm not allowed with "
+                       "-enable-emscripten-cxx-exceptions");
+  if (WasmEnableEH && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-wasm-enable-eh only allowed with -exception-model=wasm");
+  if (WasmEnableSjLj && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-wasm-enable-sjlj only allowed with -exception-model=wasm");
+  if ((!WasmEnableEH && !WasmEnableSjLj) &&
+      TM->Options.ExceptionModel == ExceptionHandling::Wasm)
+    report_fatal_error(
+        "-exception-model=wasm only allowed with at least one of "
+        "-wasm-enable-eh or -wasm-enable-sjlj");
+
+  // Currently it is allowed to mix Wasm EH with Emscripten SjLj as an interim
+  // measure, but some code will error out at compile time in this combination.
+  // See WebAssemblyLowerEmscriptenEHSjLj pass for details.
+}
+
 /// Create an WebAssembly architecture model.
 ///
 WebAssemblyTargetMachine::WebAssemblyTargetMachine(
@@ -149,7 +229,7 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
   this->Options.UniqueSectionNames = true;
 
   initAsmInfo();
-
+  basicCheckForEHAndSjLj(this);
   // Note that we don't use setRequiresStructuredCFG(true). It disables
   // optimizations than we're ok with, and want, such as critical edge
   // splitting and tail merging.
@@ -400,61 +480,6 @@ FunctionPass *WebAssemblyPassConfig::createTargetRegisterAllocator(bool) {
   return nullptr; // No reg alloc
 }
 
-using WebAssembly::WasmEnableEH;
-using WebAssembly::WasmEnableEmEH;
-using WebAssembly::WasmEnableEmSjLj;
-using WebAssembly::WasmEnableSjLj;
-
-static void basicCheckForEHAndSjLj(TargetMachine *TM) {
-
-  // You can't enable two modes of EH at the same time
-  if (WasmEnableEmEH && WasmEnableEH)
-    report_fatal_error(
-        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-eh");
-  // You can't enable two modes of SjLj at the same time
-  if (WasmEnableEmSjLj && WasmEnableSjLj)
-    report_fatal_error(
-        "-enable-emscripten-sjlj not allowed with -wasm-enable-sjlj");
-  // You can't mix Emscripten EH with Wasm SjLj.
-  if (WasmEnableEmEH && WasmEnableSjLj)
-    report_fatal_error(
-        "-enable-emscripten-cxx-exceptions not allowed with -wasm-enable-sjlj");
-
-  // Here we make sure TargetOptions.ExceptionModel is the same as
-  // MCAsmInfo.ExceptionsType. Normally these have to be the same, because clang
-  // stores the exception model info in LangOptions, which is later transferred
-  // to TargetOptions and MCAsmInfo. But when clang compiles bitcode directly,
-  // clang's LangOptions is not used and thus the exception model info is not
-  // correctly transferred to TargetOptions and MCAsmInfo, so we make sure we
-  // have the correct exception model in WebAssemblyMCAsmInfo constructor. But
-  // in this case TargetOptions is still not updated, so we make sure they are
-  // the same.
-  TM->Options.ExceptionModel = TM->getMCAsmInfo()->getExceptionHandlingType();
-
-  // Basic Correctness checking related to -exception-model
-  if (TM->Options.ExceptionModel != ExceptionHandling::None &&
-      TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error("-exception-model should be either 'none' or 'wasm'");
-  if (WasmEnableEmEH && TM->Options.ExceptionModel == ExceptionHandling::Wasm)
-    report_fatal_error("-exception-model=wasm not allowed with "
-                       "-enable-emscripten-cxx-exceptions");
-  if (WasmEnableEH && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-wasm-enable-eh only allowed with -exception-model=wasm");
-  if (WasmEnableSjLj && TM->Options.ExceptionModel != ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-wasm-enable-sjlj only allowed with -exception-model=wasm");
-  if ((!WasmEnableEH && !WasmEnableSjLj) &&
-      TM->Options.ExceptionModel == ExceptionHandling::Wasm)
-    report_fatal_error(
-        "-exception-model=wasm only allowed with at least one of "
-        "-wasm-enable-eh or -wasm-enable-sjlj");
-
-  // Currently it is allowed to mix Wasm EH with Emscripten SjLj as an interim
-  // measure, but some code will error out at compile time in this combination.
-  // See WebAssemblyLowerEmscriptenEHSjLj pass for details.
-}
-
 //===----------------------------------------------------------------------===//
 // The following functions are called from lib/CodeGen/Passes.cpp to modify
 // the CodeGen pass sequence.
@@ -475,8 +500,6 @@ void WebAssemblyPassConfig::addIRPasses() {
   if (getOptLevel() != CodeGenOptLevel::None)
     addPass(createWebAssemblyOptimizeReturned());
 
-  basicCheckForEHAndSjLj(TM);
-
   // If exception handling is not enabled and setjmp/longjmp handling is
   // enabled, we lower invokes into calls and delete unreachable landingpad
   // blocks. Lowering invokes when there is no EH support is done in
@@ -602,14 +625,16 @@ void WebAssemblyPassConfig::addPreEmitPass() {
 
     // Prepare memory intrinsic calls for register stackifying.
     addPass(createWebAssemblyMemIntrinsicResults());
+  }
 
-    // Mark registers as representing wasm's value stack. This is a key
-    // code-compression technique in WebAssembly. We run this pass (and
-    // MemIntrinsicResults above) very late, so that it sees as much code as
-    // possible, including code emitted by PEI and expanded by late tail
-    // duplication.
-    addPass(createWebAssemblyRegStackify());
+  // Mark registers as representing wasm's value stack. This is a key
+  // code-compression technique in WebAssembly. We run this pass (and
+  // MemIntrinsicResults above) very late, so that it sees as much code as
+  // possible, including code emitted by PEI and expanded by late tail
+  // duplication.
+  addPass(createWebAssemblyRegStackify(getOptLevel()));
 
+  if (getOptLevel() != CodeGenOptLevel::None) {
     // Run the register coloring pass to reduce the total number of registers.
     // This runs after stackification so that it doesn't consider registers
     // that become stackified.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
index 1d8197d02588..6f97f1302189 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
@@ -21,6 +21,15 @@
 
 namespace llvm {
 
+namespace WebAssembly {
+// Exception handling / setjmp-longjmp handling command-line options
+extern cl::opt<bool> WasmEnableEmEH;   // asm.js-style EH
+extern cl::opt<bool> WasmEnableEmSjLj; // asm.js-style SjLJ
+extern cl::opt<bool> WasmEnableEH;     // EH using Wasm EH instructions
+extern cl::opt<bool> WasmEnableSjLj;   // SjLj using Wasm EH instructions
+extern cl::opt<bool> WasmUseLegacyEH;  // Legacy Wasm EH
+} // namespace WebAssembly
+
 class WebAssemblyTargetMachine final : public CodeGenTargetMachineImpl {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   mutable StringMap<std::unique_ptr<WebAssemblySubtarget>> SubtargetMap;
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index f26b46f0d56c..0c8ae14944fd 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -9,6 +9,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86IntelInstPrinter.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCExpr.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "MCTargetDesc/X86TargetStreamer.h"
@@ -2118,7 +2119,7 @@ bool X86AsmParser::ParseIntelExpression(IntelExprStateMachine &SM, SMLoc &End) {
         if (IDVal == "f" || IDVal == "b") {
           MCSymbol *Sym =
               getContext().getDirectionalLocalSymbol(IntVal, IDVal == "b");
-          auto Variant = X86MCExpr::VK_None;
+          auto Variant = X86::S_None;
           const MCExpr *Val =
               MCSymbolRefExpr::create(Sym, Variant, getContext());
           if (IDVal == "b" && Sym->isUndefined())
@@ -2265,7 +2266,7 @@ bool X86AsmParser::ParseIntelInlineAsmIdentifier(
     return false;
   // Create the symbol reference.
   MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-  auto Variant = X86MCExpr::VK_None;
+  auto Variant = X86::S_None;
   Val = MCSymbolRefExpr::create(Sym, Variant, getParser().getContext());
   return false;
 }
diff --git a/llvm/lib/Target/X86/AsmParser/X86Operand.h b/llvm/lib/Target/X86/AsmParser/X86Operand.h
index c143ebd6eae5..89ac53e0ecac 100644
--- a/llvm/lib/Target/X86/AsmParser/X86Operand.h
+++ b/llvm/lib/Target/X86/AsmParser/X86Operand.h
@@ -107,8 +107,7 @@ struct X86Operand final : public MCParsedAsmOperand {
   /// getOffsetOfLoc - Get the location of the offset operator.
   SMLoc getOffsetOfLoc() const override { return OffsetOfLoc; }
 
-  void print(raw_ostream &OS) const override {
-
+  void print(raw_ostream &OS, const MCAsmInfo &) const override {
     auto PrintImmValue = [&](const MCExpr *Val, const char *VName) {
       if (Val->getKind() == MCExpr::Constant) {
         if (auto Imm = cast<MCConstantExpr>(Val)->getValue())
diff --git a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
index f21a7c81459f..8e304c07ed5c 100644
--- a/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
@@ -97,10 +97,10 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
       .widenScalarToNextPow2(0, /*Min=*/8)
       .clampScalar(0, s8, sMaxScalar);
 
-  getActionDefinitionsBuilder({G_LROUND, G_LLROUND, G_FCOS,  G_FCOSH,  G_FACOS,
-                               G_FSIN,   G_FSINH,   G_FASIN, G_FTAN,   G_FTANH,
-                               G_FATAN,  G_FATAN2,  G_FPOW,  G_FEXP,   G_FEXP2,
-                               G_FEXP10, G_FLOG,    G_FLOG2, G_FLOG10, G_FPOWI})
+  getActionDefinitionsBuilder(
+      {G_LROUND, G_LLROUND, G_FCOS,  G_FCOSH, G_FACOS,  G_FSIN,  G_FSINH,
+       G_FASIN,  G_FTAN,    G_FTANH, G_FATAN, G_FATAN2, G_FPOW,  G_FEXP,
+       G_FEXP2,  G_FEXP10,  G_FLOG,  G_FLOG2, G_FLOG10, G_FPOWI, G_FSINCOS})
       .libcall();
 
   getActionDefinitionsBuilder(G_FSQRT)
@@ -418,6 +418,11 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
       .legalFor(HasAVX512, {v16s32, v8s64})
       .legalFor(UseX87, {s80});
 
+  getActionDefinitionsBuilder(G_FABS)
+      .legalFor(UseX87, {s80})
+      .legalFor(UseX87 && !Is64Bit, {s64})
+      .lower();
+
   // fp comparison
   getActionDefinitionsBuilder(G_FCMP)
       .legalFor(HasSSE1 || UseX87, {s8, s32})
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index 74fe5fcaab6c..b23d79150172 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -341,6 +341,7 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                     /* Predicate */ PMI_None, FpRegBank, FpRegBank};
     break;
   }
+  case TargetOpcode::G_FABS:
   case TargetOpcode::G_TRUNC:
   case TargetOpcode::G_ANYEXT: {
     auto &Op0 = MI.getOperand(0);
@@ -354,9 +355,9 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
         Ty0.getSizeInBits() == 128 &&
         (Ty1.getSizeInBits() == 32 || Ty1.getSizeInBits() == 64) &&
         Opc == TargetOpcode::G_ANYEXT;
-
-    getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt,
-                               OpRegBankIdx);
+    bool isFAbs = (Opc == TargetOpcode::G_FABS);
+    getInstrPartialMappingIdxs(
+        MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt || isFAbs, OpRegBankIdx);
     break;
   }
   case TargetOpcode::G_LOAD: {
diff --git a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
index f2e7d43fc17f..bf741506a544 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
+++ b/llvm/lib/Target/X86/MCTargetDesc/CMakeLists.txt
@@ -28,3 +28,4 @@ add_llvm_component_library(LLVMX86Desc
   ADD_TO_COMPONENT
   X86
   )
+set_source_files_properties(X86MCCodeEmitter.cpp PROPERTIES COMPILE_FLAGS "-O2")
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 2d0f021eac5a..f669e6010d20 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -9,7 +9,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCELFStreamer.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectStreamer.h"
@@ -169,7 +168,7 @@ class X86AsmBackend : public MCAsmBackend {
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
-  bool shouldForceRelocation(const MCFixup &, const MCValue &) override;
+  bool shouldForceRelocation(const MCFixup &, const MCValue &);
 
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
@@ -359,7 +358,7 @@ static bool hasVariantSymbol(const MCInst &MI) {
       continue;
     const MCExpr &Expr = *Operand.getExpr();
     if (Expr.getKind() == MCExpr::SymbolRef &&
-        getSpecifier(cast<MCSymbolRefExpr>(&Expr)) != X86MCExpr::VK_None)
+        cast<MCSymbolRefExpr>(&Expr)->getSpecifier())
       return true;
   }
   return false;
@@ -423,14 +422,10 @@ static size_t getSizeForInstFragment(const MCFragment *F) {
   if (!F || !F->hasInstructions())
     return 0;
   // MCEncodedFragmentWithContents being templated makes this tricky.
-  switch (F->getKind()) {
-  default:
+  if (auto *DF = dyn_cast<MCEncodedFragment>(F))
+    return DF->getContents().size();
+  else
     llvm_unreachable("Unknown fragment with instructions!");
-  case MCFragment::FT_Data:
-    return cast<MCDataFragment>(*F).getContents().size();
-  case MCFragment::FT_Relaxable:
-    return cast<MCRelaxableFragment>(*F).getContents().size();
-  }
 }
 
 /// Return true if we can insert NOP or prefixes automatically before the
@@ -623,24 +618,24 @@ std::optional<MCFixupKind> X86AsmBackend::getFixupKind(StringRef Name) const {
 MCFixupKindInfo X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[X86::NumTargetFixupKinds] = {
       // clang-format off
-      {"reloc_riprel_4byte", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_movq_load", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_movq_load_rex2", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_rex", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_rex2", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
-      {"reloc_riprel_4byte_relax_evex", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"reloc_riprel_4byte", 0, 32, 0},
+      {"reloc_riprel_4byte_movq_load", 0, 32, 0},
+      {"reloc_riprel_4byte_movq_load_rex2", 0, 32, 0},
+      {"reloc_riprel_4byte_relax", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_rex", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_rex2", 0, 32, 0},
+      {"reloc_riprel_4byte_relax_evex", 0, 32, 0},
       {"reloc_signed_4byte", 0, 32, 0},
       {"reloc_signed_4byte_relax", 0, 32, 0},
       {"reloc_global_offset_table", 0, 32, 0},
-      {"reloc_branch_4byte_pcrel", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
+      {"reloc_branch_4byte_pcrel", 0, 32, 0},
       // clang-format on
   };
 
   // Fixup kinds from .reloc directive are like R_386_NONE/R_X86_64_NONE. They
   // do not require any extra processing.
   if (mc::isRelocation(Kind))
-    return MCAsmBackend::getFixupKindInfo(FK_NONE);
+    return {};
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -657,15 +652,12 @@ static unsigned getFixupKindSize(unsigned Kind) {
     llvm_unreachable("invalid fixup kind!");
   case FK_NONE:
     return 0;
-  case FK_PCRel_1:
   case FK_SecRel_1:
   case FK_Data_1:
     return 1;
-  case FK_PCRel_2:
   case FK_SecRel_2:
   case FK_Data_2:
     return 2;
-  case FK_PCRel_4:
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
   case X86::reloc_riprel_4byte_relax_rex:
@@ -680,23 +672,22 @@ static unsigned getFixupKindSize(unsigned Kind) {
   case FK_SecRel_4:
   case FK_Data_4:
     return 4;
-  case FK_PCRel_8:
   case FK_SecRel_8:
   case FK_Data_8:
     return 8;
   }
 }
 
-// Force relocation when there is a specifier. This might be too conservative -
-// GAS doesn't emit a relocation for call local@plt; local:.
-bool X86AsmBackend::shouldForceRelocation(const MCFixup &,
-                                          const MCValue &Target) {
-  return Target.getSpecifier();
-}
+void X86AsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
+                               const MCValue &Target,
+                               MutableArrayRef<char> Data, uint64_t Value,
+                               bool IsResolved) {
+  // Force relocation when there is a specifier. This might be too conservative
+  // - GAS doesn't emit a relocation for call local@plt; local:.
+  if (Target.getSpecifier())
+    IsResolved = false;
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
 
-void X86AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
-                               const MCValue &, MutableArrayRef<char> Data,
-                               uint64_t Value, bool IsResolved) {
   auto Kind = Fixup.getKind();
   if (mc::isRelocation(Kind))
     return;
@@ -705,8 +696,7 @@ void X86AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
   assert(Fixup.getOffset() + Size <= Data.size() && "Invalid fixup offset!");
 
   int64_t SignedValue = static_cast<int64_t>(Value);
-  if (IsResolved &&
-      getFixupKindInfo(Fixup.getKind()).Flags & MCFixupKindInfo::FKF_IsPCRel) {
+  if (IsResolved && Fixup.isPCRel()) {
     // check that PC relative fixup fits into the fixup size.
     if (Size > 0 && !isIntN(Size * 8, SignedValue))
       getContext().reportError(Fixup.getLoc(),
@@ -748,7 +738,7 @@ bool X86AsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
 
   // Otherwise, relax unless there is a @ABS8 specifier.
   if (Fixup.getKind() == FK_Data_1 && Target.getAddSym() &&
-      Target.getSpecifier() == X86MCExpr::VK_ABS8)
+      Target.getSpecifier() == X86::S_ABS8)
     return false;
   return true;
 }
@@ -760,15 +750,7 @@ void X86AsmBackend::relaxInstruction(MCInst &Inst,
   // The only relaxations X86 does is from a 1byte pcrel to a 4byte pcrel.
   bool Is16BitMode = STI.hasFeature(X86::Is16Bit);
   unsigned RelaxedOp = getRelaxedOpcode(Inst, Is16BitMode);
-
-  if (RelaxedOp == Inst.getOpcode()) {
-    SmallString<256> Tmp;
-    raw_svector_ostream OS(Tmp);
-    Inst.dump_pretty(OS);
-    OS << "\n";
-    report_fatal_error("unexpected instruction to relax: " + OS.str());
-  }
-
+  assert(RelaxedOp != Inst.getOpcode());
   Inst.setOpcode(RelaxedOp);
 }
 
@@ -847,7 +829,7 @@ bool X86AsmBackend::padInstructionViaRelaxation(MCRelaxableFragment &RF,
     return false;
   RF.setInst(Relaxed);
   RF.setContents(Code);
-  RF.getFixups() = Fixups;
+  RF.setFixups(Fixups);
   RemainingSize -= Delta;
   return true;
 }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index 13f2a76051df..3323b387368d 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -39,10 +39,10 @@ class X86ELFObjectWriter : public MCELFObjectTargetWriter {
 
   void checkIs32(SMLoc Loc, X86_64RelType Type) const;
   void checkIs64(SMLoc Loc, X86_64RelType Type) const;
-  unsigned getRelocType32(SMLoc Loc, X86MCExpr::Specifier Specifier,
+  unsigned getRelocType32(SMLoc Loc, X86::Specifier Specifier,
                           X86_32RelType Type, bool IsPCRel,
                           MCFixupKind Kind) const;
-  unsigned getRelocType64(SMLoc Loc, X86MCExpr::Specifier Specifier,
+  unsigned getRelocType64(SMLoc Loc, X86::Specifier Specifier,
                           X86_64RelType Type, bool IsPCRel,
                           MCFixupKind Kind) const;
 };
@@ -57,8 +57,8 @@ X86ELFObjectWriter::X86ELFObjectWriter(bool IsELF64, uint8_t OSABI,
                               (EMachine != ELF::EM_386) &&
                                   (EMachine != ELF::EM_IAMCU)) {}
 
-static X86_64RelType getType64(MCFixupKind Kind,
-                               X86MCExpr::Specifier &Specifier, bool &IsPCRel) {
+static X86_64RelType getType64(MCFixupKind Kind, X86::Specifier &Specifier,
+                               bool &IsPCRel) {
   switch (unsigned(Kind)) {
   default:
     llvm_unreachable("Unimplemented");
@@ -68,15 +68,14 @@ static X86_64RelType getType64(MCFixupKind Kind,
     return RT64_64;
   case X86::reloc_signed_4byte:
   case X86::reloc_signed_4byte_relax:
-    if (Specifier == X86MCExpr::VK_None && !IsPCRel)
+    if (Specifier == X86::S_None && !IsPCRel)
       return RT64_32S;
     return RT64_32;
   case X86::reloc_global_offset_table:
-    Specifier = X86MCExpr::VK_GOT;
+    Specifier = X86::S_GOT;
     IsPCRel = true;
     return RT64_32;
   case FK_Data_4:
-  case FK_PCRel_4:
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
   case X86::reloc_riprel_4byte_relax_rex:
@@ -86,12 +85,10 @@ static X86_64RelType getType64(MCFixupKind Kind,
   case X86::reloc_riprel_4byte_relax_evex:
     return RT64_32;
   case X86::reloc_branch_4byte_pcrel:
-    Specifier = X86MCExpr::VK_PLT;
+    Specifier = X86::S_PLT;
     return RT64_32;
-  case FK_PCRel_2:
   case FK_Data_2:
     return RT64_16;
-  case FK_PCRel_1:
   case FK_Data_1:
     return RT64_8;
   }
@@ -107,18 +104,17 @@ void X86ELFObjectWriter::checkIs64(SMLoc Loc, X86_64RelType Type) const {
     reportError(Loc, "64 bit reloc applied to a field with a different size");
 }
 
-unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
-                                            X86MCExpr::Specifier Specifier,
+unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc, X86::Specifier Specifier,
                                             X86_64RelType Type, bool IsPCRel,
                                             MCFixupKind Kind) const {
   switch (Specifier) {
   default:
     llvm_unreachable("Unimplemented");
-  case X86MCExpr::VK_None:
-  case X86MCExpr::VK_ABS8:
+  case X86::S_None:
+  case X86::S_ABS8:
     switch (Type) {
     case RT64_NONE:
-      if (Specifier == X86MCExpr::VK_None)
+      if (Specifier == X86::S_None)
         return ELF::R_X86_64_NONE;
       llvm_unreachable("Unimplemented");
     case RT64_64:
@@ -133,7 +129,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       return IsPCRel ? ELF::R_X86_64_PC8 : ELF::R_X86_64_8;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOT:
+  case X86::S_GOT:
     switch (Type) {
     case RT64_64:
       return IsPCRel ? ELF::R_X86_64_GOTPC64 : ELF::R_X86_64_GOT64;
@@ -146,12 +142,12 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOTOFF:
+  case X86::S_GOTOFF:
     assert(!IsPCRel);
     if (Type != RT64_64)
       reportError(Loc, "unsupported relocation type");
     return ELF::R_X86_64_GOTOFF64;
-  case X86MCExpr::VK_TPOFF:
+  case X86::S_TPOFF:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -165,7 +161,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_DTPOFF:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -179,7 +175,7 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_SIZE:
+  case X86::S_SIZE:
     assert(!IsPCRel);
     switch (Type) {
     case RT64_64:
@@ -193,16 +189,16 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       llvm_unreachable("Unimplemented");
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_TLSCALL:
+  case X86::S_TLSCALL:
     return ELF::R_X86_64_TLSDESC_CALL;
-  case X86MCExpr::VK_TLSDESC:
+  case X86::S_TLSDESC:
     return ((unsigned)Kind == X86::reloc_riprel_4byte_relax_rex2)
                ? ELF::R_X86_64_CODE_4_GOTPC32_TLSDESC
                : ELF::R_X86_64_GOTPC32_TLSDESC;
-  case X86MCExpr::VK_TLSGD:
+  case X86::S_TLSGD:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_TLSGD;
-  case X86MCExpr::VK_GOTTPOFF:
+  case X86::S_GOTTPOFF:
     checkIs32(Loc, Type);
     if ((unsigned)Kind == X86::reloc_riprel_4byte_movq_load_rex2 ||
         (unsigned)Kind == X86::reloc_riprel_4byte_relax_rex2)
@@ -210,13 +206,13 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
     else if ((unsigned)Kind == X86::reloc_riprel_4byte_relax_evex)
       return ELF::R_X86_64_CODE_6_GOTTPOFF;
     return ELF::R_X86_64_GOTTPOFF;
-  case X86MCExpr::VK_TLSLD:
+  case X86::S_TLSLD:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_TLSLD;
-  case X86MCExpr::VK_PLT:
+  case X86::S_PLT:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_PLT32;
-  case X86MCExpr::VK_GOTPCREL:
+  case X86::S_GOTPCREL:
     checkIs32(Loc, Type);
     // Older versions of ld.bfd/ld.gold/lld
     // do not support GOTPCRELX/REX_GOTPCRELX/CODE_4_GOTPCRELX,
@@ -236,27 +232,26 @@ unsigned X86ELFObjectWriter::getRelocType64(SMLoc Loc,
       return ELF::R_X86_64_CODE_4_GOTPCRELX;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOTPCREL_NORELAX:
+  case X86::S_GOTPCREL_NORELAX:
     checkIs32(Loc, Type);
     return ELF::R_X86_64_GOTPCREL;
-  case X86MCExpr::VK_PLTOFF:
+  case X86::S_PLTOFF:
     checkIs64(Loc, Type);
     return ELF::R_X86_64_PLTOFF64;
   }
 }
 
-unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
-                                            X86MCExpr::Specifier Specifier,
+unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc, X86::Specifier Specifier,
                                             X86_32RelType Type, bool IsPCRel,
                                             MCFixupKind Kind) const {
   switch (Specifier) {
   default:
     llvm_unreachable("Unimplemented");
-  case X86MCExpr::VK_None:
-  case X86MCExpr::VK_ABS8:
+  case X86::S_None:
+  case X86::S_ABS8:
     switch (Type) {
     case RT32_NONE:
-      if (Specifier == X86MCExpr::VK_None)
+      if (Specifier == X86::S_None)
         return ELF::R_386_NONE;
       llvm_unreachable("Unimplemented");
     case RT32_32:
@@ -267,7 +262,7 @@ unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
       return IsPCRel ? ELF::R_386_PC8 : ELF::R_386_8;
     }
     llvm_unreachable("unexpected relocation type!");
-  case X86MCExpr::VK_GOT:
+  case X86::S_GOT:
     if (Type != RT32_32)
       break;
     if (IsPCRel)
@@ -277,58 +272,57 @@ unsigned X86ELFObjectWriter::getRelocType32(SMLoc Loc,
     if (!getContext().getTargetOptions()->X86RelaxRelocations)
       return ELF::R_386_GOT32;
 
-    return Kind == MCFixupKind(X86::reloc_signed_4byte_relax)
-               ? ELF::R_386_GOT32X
-               : ELF::R_386_GOT32;
-  case X86MCExpr::VK_GOTOFF:
+    return Kind == X86::reloc_signed_4byte_relax ? ELF::R_386_GOT32X
+                                                 : ELF::R_386_GOT32;
+  case X86::S_GOTOFF:
     assert(!IsPCRel);
     if (Type != RT32_32)
       break;
     return ELF::R_386_GOTOFF;
-  case X86MCExpr::VK_TLSCALL:
+  case X86::S_TLSCALL:
     return ELF::R_386_TLS_DESC_CALL;
-  case X86MCExpr::VK_TLSDESC:
+  case X86::S_TLSDESC:
     return ELF::R_386_TLS_GOTDESC;
-  case X86MCExpr::VK_TPOFF:
+  case X86::S_TPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LE_32;
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_DTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LDO_32;
-  case X86MCExpr::VK_TLSGD:
+  case X86::S_TLSGD:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_GD;
-  case X86MCExpr::VK_GOTTPOFF:
+  case X86::S_GOTTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_IE_32;
-  case X86MCExpr::VK_PLT:
+  case X86::S_PLT:
     if (Type != RT32_32)
       break;
     return ELF::R_386_PLT32;
-  case X86MCExpr::VK_INDNTPOFF:
+  case X86::S_INDNTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_IE;
-  case X86MCExpr::VK_NTPOFF:
+  case X86::S_NTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_LE;
-  case X86MCExpr::VK_GOTNTPOFF:
+  case X86::S_GOTNTPOFF:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
     return ELF::R_386_TLS_GOTIE;
-  case X86MCExpr::VK_TLSLDM:
+  case X86::S_TLSLDM:
     if (Type != RT32_32)
       break;
     assert(!IsPCRel);
@@ -342,19 +336,19 @@ unsigned X86ELFObjectWriter::getRelocType(const MCFixup &Fixup,
                                           const MCValue &Target,
                                           bool IsPCRel) const {
   MCFixupKind Kind = Fixup.getKind();
-  auto Specifier = X86MCExpr::Specifier(Target.getSpecifier());
+  auto Specifier = X86::Specifier(Target.getSpecifier());
   switch (Specifier) {
-  case X86MCExpr::VK_GOTTPOFF:
-  case X86MCExpr::VK_INDNTPOFF:
-  case X86MCExpr::VK_NTPOFF:
-  case X86MCExpr::VK_GOTNTPOFF:
-  case X86MCExpr::VK_TLSCALL:
-  case X86MCExpr::VK_TLSDESC:
-  case X86MCExpr::VK_TLSGD:
-  case X86MCExpr::VK_TLSLD:
-  case X86MCExpr::VK_TLSLDM:
-  case X86MCExpr::VK_TPOFF:
-  case X86MCExpr::VK_DTPOFF:
+  case X86::S_GOTTPOFF:
+  case X86::S_INDNTPOFF:
+  case X86::S_NTPOFF:
+  case X86::S_GOTNTPOFF:
+  case X86::S_TLSCALL:
+  case X86::S_TLSDESC:
+  case X86::S_TLSGD:
+  case X86::S_TLSLD:
+  case X86::S_TLSLDM:
+  case X86::S_TPOFF:
+  case X86::S_DTPOFF:
     if (auto *S = Target.getAddSym())
       cast<MCSymbolELF>(S)->setType(ELF::STT_TLS);
     break;
@@ -393,10 +387,10 @@ unsigned X86ELFObjectWriter::getRelocType(const MCFixup &Fixup,
 bool X86ELFObjectWriter::needsRelocateWithSymbol(const MCValue &V,
                                                  unsigned Type) const {
   switch (V.getSpecifier()) {
-  case X86MCExpr::VK_GOT:
-  case X86MCExpr::VK_PLT:
-  case X86MCExpr::VK_GOTPCREL:
-  case X86MCExpr::VK_GOTPCREL_NORELAX:
+  case X86::S_GOT:
+  case X86::S_PLT:
+  case X86::S_GOTPCREL:
+  case X86::S_GOTPCREL_NORELAX:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
index e4fdd85929a4..58c5e329a38f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86EncodingOptimization.cpp
@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86EncodingOptimization.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "X86BaseInfo.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
@@ -375,7 +375,7 @@ bool X86::optimizeMOV(MCInst &MI, bool In64BitMode) {
   if (MI.getOperand(AddrOp).isExpr()) {
     const MCExpr *MCE = MI.getOperand(AddrOp).getExpr();
     if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
-      if (getSpecifier(SRE) == X86MCExpr::VK_TLVP)
+      if (SRE->getSpecifier() == X86::S_TLVP)
         Absolute = false;
   }
   if (Absolute && (MI.getOperand(AddrBase + X86::AddrBaseReg).getReg() ||
@@ -486,7 +486,7 @@ static bool optimizeToShortImmediateForm(MCInst &MI) {
   MCOperand &LastOp = MI.getOperand(MI.getNumOperands() - 1 - SkipOperands);
   if (LastOp.isExpr()) {
     const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(LastOp.getExpr());
-    if (!SRE || getSpecifier(SRE) != X86MCExpr::VK_ABS8)
+    if (!SRE || SRE->getSpecifier() != X86::S_ABS8)
       return false;
   } else if (LastOp.isImm()) {
     if (!isInt<8>(LastOp.getImm()))
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index 8702638564bc..a769570c4e69 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -35,35 +35,35 @@ MarkedJTDataRegions("mark-data-regions", cl::init(true),
   cl::desc("Mark code section jump table data regions."),
   cl::Hidden);
 
-const MCAsmInfo::VariantKindDesc variantKindDescs[] = {
-    {X86MCExpr::VK_ABS8, "ABS8"},
-    {X86MCExpr::VK_DTPOFF, "DTPOFF"},
-    {X86MCExpr::VK_DTPREL, "DTPREL"},
-    {X86MCExpr::VK_GOT, "GOT"},
-    {X86MCExpr::VK_GOTENT, "GOTENT"},
-    {X86MCExpr::VK_GOTNTPOFF, "GOTNTPOFF"},
-    {X86MCExpr::VK_GOTOFF, "GOTOFF"},
-    {X86MCExpr::VK_GOTPCREL, "GOTPCREL"},
-    {X86MCExpr::VK_GOTPCREL_NORELAX, "GOTPCREL_NORELAX"},
-    {X86MCExpr::VK_GOTREL, "GOTREL"},
-    {X86MCExpr::VK_GOTTPOFF, "GOTTPOFF"},
-    {X86MCExpr::VK_INDNTPOFF, "INDNTPOFF"},
+const MCAsmInfo::AtSpecifier atSpecifiers[] = {
+    {X86::S_ABS8, "ABS8"},
+    {X86::S_DTPOFF, "DTPOFF"},
+    {X86::S_DTPREL, "DTPREL"},
+    {X86::S_GOT, "GOT"},
+    {X86::S_GOTENT, "GOTENT"},
+    {X86::S_GOTNTPOFF, "GOTNTPOFF"},
+    {X86::S_GOTOFF, "GOTOFF"},
+    {X86::S_GOTPCREL, "GOTPCREL"},
+    {X86::S_GOTPCREL_NORELAX, "GOTPCREL_NORELAX"},
+    {X86::S_GOTREL, "GOTREL"},
+    {X86::S_GOTTPOFF, "GOTTPOFF"},
+    {X86::S_INDNTPOFF, "INDNTPOFF"},
     {MCSymbolRefExpr::VK_COFF_IMGREL32, "IMGREL"},
-    {X86MCExpr::VK_NTPOFF, "NTPOFF"},
-    {X86MCExpr::VK_PCREL, "PCREL"},
-    {X86MCExpr::VK_PLT, "PLT"},
-    {X86MCExpr::VK_PLTOFF, "PLTOFF"},
-    {MCSymbolRefExpr::VK_SECREL, "SECREL32"},
-    {X86MCExpr::VK_SIZE, "SIZE"},
-    {X86MCExpr::VK_TLSCALL, "tlscall"},
-    {X86MCExpr::VK_TLSDESC, "tlsdesc"},
-    {X86MCExpr::VK_TLSGD, "TLSGD"},
-    {X86MCExpr::VK_TLSLD, "TLSLD"},
-    {X86MCExpr::VK_TLSLDM, "TLSLDM"},
-    {X86MCExpr::VK_TLVP, "TLVP"},
-    {X86MCExpr::VK_TLVPPAGE, "TLVPPAGE"},
-    {X86MCExpr::VK_TLVPPAGEOFF, "TLVPPAGEOFF"},
-    {X86MCExpr::VK_TPOFF, "TPOFF"},
+    {X86::S_NTPOFF, "NTPOFF"},
+    {X86::S_PCREL, "PCREL"},
+    {X86::S_PLT, "PLT"},
+    {X86::S_PLTOFF, "PLTOFF"},
+    {X86::S_COFF_SECREL, "SECREL32"},
+    {X86::S_SIZE, "SIZE"},
+    {X86::S_TLSCALL, "tlscall"},
+    {X86::S_TLSDESC, "tlsdesc"},
+    {X86::S_TLSGD, "TLSGD"},
+    {X86::S_TLSLD, "TLSLD"},
+    {X86::S_TLSLDM, "TLSLDM"},
+    {X86::S_TLVP, "TLVP"},
+    {X86::S_TLVPPAGE, "TLVPPAGE"},
+    {X86::S_TLVPPAGEOFF, "TLVPPAGEOFF"},
+    {X86::S_TPOFF, "TPOFF"},
 };
 
 void X86MCAsmInfoDarwin::anchor() { }
@@ -102,7 +102,7 @@ X86MCAsmInfoDarwin::X86MCAsmInfoDarwin(const Triple &T) {
   // overwhelm ld64's tiny little mind and it fails).
   DwarfFDESymbolsUseAbsDiff = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 X86_64MCAsmInfoDarwin::X86_64MCAsmInfoDarwin(const Triple &Triple)
@@ -131,7 +131,7 @@ X86ELFMCAsmInfo::X86ELFMCAsmInfo(const Triple &T) {
   // Exceptions handling
   ExceptionsType = ExceptionHandling::DwarfCFI;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 const MCExpr *
@@ -139,8 +139,7 @@ X86_64MCAsmInfoDarwin::getExprForPersonalitySymbol(const MCSymbol *Sym,
                                                    unsigned Encoding,
                                                    MCStreamer &Streamer) const {
   MCContext &Context = Streamer.getContext();
-  const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, Context);
+  const MCExpr *Res = MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, Context);
   const MCExpr *Four = MCConstantExpr::create(4, Context);
   return MCBinaryExpr::createAdd(Res, Four, Context);
 }
@@ -166,7 +165,7 @@ X86MCAsmInfoMicrosoft::X86MCAsmInfoMicrosoft(const Triple &Triple) {
 
   AllowAtInName = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
 
 void X86MCAsmInfoMicrosoftMASM::anchor() { }
@@ -201,5 +200,5 @@ X86MCAsmInfoGNUCOFF::X86MCAsmInfoGNUCOFF(const Triple &Triple) {
 
   AllowAtInName = true;
 
-  initializeVariantKinds(variantKindDescs);
+  initializeAtSpecifiers(atSpecifiers);
 }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
index ce8e84fb96b9..9533bfa31cb1 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
@@ -13,9 +13,11 @@
 #ifndef LLVM_LIB_TARGET_X86_MCTARGETDESC_X86MCASMINFO_H
 #define LLVM_LIB_TARGET_X86_MCTARGETDESC_X86MCASMINFO_H
 
+#include "MCTargetDesc/X86MCExpr.h"
 #include "llvm/MC/MCAsmInfoCOFF.h"
 #include "llvm/MC/MCAsmInfoDarwin.h"
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 class Triple;
@@ -61,6 +63,42 @@ class X86MCAsmInfoGNUCOFF : public MCAsmInfoGNUCOFF {
 public:
   explicit X86MCAsmInfoGNUCOFF(const Triple &Triple);
 };
+
+namespace X86 {
+using Specifier = uint16_t;
+
+enum {
+  S_None,
+  S_COFF_SECREL,
+
+  S_ABS8 = MCSymbolRefExpr::FirstTargetSpecifier,
+  S_DTPOFF,
+  S_DTPREL,
+  S_GOT,
+  S_GOTENT,
+  S_GOTNTPOFF,
+  S_GOTOFF,
+  S_GOTPCREL,
+  S_GOTPCREL_NORELAX,
+  S_GOTREL,
+  S_GOTTPOFF,
+  S_INDNTPOFF,
+  S_NTPOFF,
+  S_PCREL,
+  S_PLT,
+  S_PLTOFF,
+  S_SIZE,
+  S_TLSCALL,
+  S_TLSDESC,
+  S_TLSGD,
+  S_TLSLD,
+  S_TLSLDM,
+  S_TLVP,
+  S_TLVPPAGE,
+  S_TLVPPAGEOFF,
+  S_TPOFF,
+};
+} // namespace X86
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 6e28c058b702..8412de32fcb9 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -12,7 +12,7 @@
 
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -358,8 +358,8 @@ class X86MCCodeEmitter : public MCCodeEmitter {
 
   unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
 
-  void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
-                     MCFixupKind FixupKind, uint64_t StartByte,
+  void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned FixupKind,
+                     bool IsPCRel, uint64_t StartByte,
                      SmallVectorImpl<char> &CB,
                      SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
 
@@ -441,27 +441,25 @@ static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
 /// instruction with the specified TSFlags.
 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
   unsigned Size = X86II::getSizeOfImm(TSFlags);
-  bool isPCRel = X86II::isImmPCRel(TSFlags);
-
   if (X86II::isImmSigned(TSFlags)) {
     switch (Size) {
     default:
       llvm_unreachable("Unsupported signed fixup size!");
     case 4:
-      return MCFixupKind(X86::reloc_signed_4byte);
+      return X86::reloc_signed_4byte;
     }
   }
   switch (Size) {
   default:
     llvm_unreachable("Invalid generic fixup size!");
   case 1:
-    return isPCRel ? FK_PCRel_1 : FK_Data_1;
+    return FK_Data_1;
   case 2:
-    return isPCRel ? FK_PCRel_2 : FK_Data_2;
+    return FK_Data_2;
   case 4:
-    return isPCRel ? FK_PCRel_4 : FK_Data_4;
+    return FK_Data_4;
   case 8:
-    return isPCRel ? FK_PCRel_8 : FK_Data_8;
+    return FK_Data_8;
   }
 }
 
@@ -496,7 +494,7 @@ startsWithGlobalOffsetTable(const MCExpr *Expr) {
 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
   if (Expr->getKind() == MCExpr::SymbolRef) {
     auto *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
-    return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
+    return Ref->getSpecifier() == X86::S_COFF_SECREL;
   }
   return false;
 }
@@ -506,7 +504,8 @@ static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
   const MCInstrDesc &Desc = MCII.get(Opcode);
   if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
        Opcode != X86::JCC_4) ||
-      getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
+      !(getImmFixupKind(Desc.TSFlags) == FK_Data_4 &&
+        X86II::isImmPCRel(Desc.TSFlags)))
     return false;
 
   unsigned CurOp = X86II::getOperandBias(Desc);
@@ -515,7 +514,7 @@ static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
     return false;
 
   auto *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
-  return Ref && getSpecifier(Ref) == X86MCExpr::VK_None;
+  return Ref && Ref->getSpecifier() == X86::S_None;
 }
 
 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
@@ -528,17 +527,29 @@ unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
 }
 
 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
-                                     unsigned Size, MCFixupKind FixupKind,
+                                     unsigned FixupKind, bool PCRel,
                                      uint64_t StartByte,
                                      SmallVectorImpl<char> &CB,
                                      SmallVectorImpl<MCFixup> &Fixups,
                                      int ImmOffset) const {
+  unsigned Size = 4;
+  switch (FixupKind) {
+  case FK_Data_1:
+    Size = 1;
+    break;
+  case FK_Data_2:
+    Size = 2;
+    break;
+  case FK_Data_8:
+    Size = 8;
+    break;
+  }
   const MCExpr *Expr = nullptr;
   if (DispOp.isImm()) {
     // If this is a simple integer displacement that doesn't require a
     // relocation, emit it now.
-    if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
-        FixupKind != FK_PCRel_4) {
+    if (!(is_contained({FK_Data_1, FK_Data_2, FK_Data_4}, FixupKind) &&
+          PCRel)) {
       emitConstant(DispOp.getImm() + ImmOffset, Size, CB);
       return;
     }
@@ -549,65 +560,51 @@ void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
 
   // If we have an immoffset, add it to the expression.
   if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
-       FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
+       FixupKind == X86::reloc_signed_4byte)) {
     GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
     if (Kind != GOT_None) {
       assert(ImmOffset == 0);
 
       if (Size == 8) {
-        FixupKind =
-            MCFixupKind(FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64);
+        FixupKind = FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64;
       } else {
         assert(Size == 4);
-        FixupKind = MCFixupKind(X86::reloc_global_offset_table);
+        FixupKind = X86::reloc_global_offset_table;
       }
 
       if (Kind == GOT_Normal)
         ImmOffset = static_cast<int>(CB.size() - StartByte);
     } else if (Expr->getKind() == MCExpr::SymbolRef) {
       if (hasSecRelSymbolRef(Expr)) {
-        FixupKind = MCFixupKind(FK_SecRel_4);
+        FixupKind = FK_SecRel_4;
       }
     } else if (Expr->getKind() == MCExpr::Binary) {
       const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
       if (hasSecRelSymbolRef(Bin->getLHS()) ||
           hasSecRelSymbolRef(Bin->getRHS())) {
-        FixupKind = MCFixupKind(FK_SecRel_4);
+        FixupKind = FK_SecRel_4;
       }
     }
   }
 
   // If the fixup is pc-relative, we need to bias the value to be relative to
   // the start of the field, not the end of the field.
-  if (FixupKind == FK_PCRel_4 ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load_rex2) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex2) ||
-      FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel) ||
-      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_evex)) {
-    ImmOffset -= 4;
+  if (PCRel) {
+    ImmOffset -= Size;
     // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
     // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
     // this needs to be a GOTPC32 relocation.
-    if (startsWithGlobalOffsetTable(Expr) != GOT_None)
-      FixupKind = MCFixupKind(X86::reloc_global_offset_table);
+    if (Size == 4 && startsWithGlobalOffsetTable(Expr) != GOT_None)
+      FixupKind = X86::reloc_global_offset_table;
   }
 
-  if (FixupKind == FK_PCRel_2)
-    ImmOffset -= 2;
-  if (FixupKind == FK_PCRel_1)
-    ImmOffset -= 1;
-
   if (ImmOffset)
     Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
-                                   Ctx);
+                                   Ctx, Expr->getLoc());
 
   // Emit a symbolic constant as a fixup and 4 zeros.
   Fixups.push_back(MCFixup::create(static_cast<uint32_t>(CB.size() - StartByte),
-                                   Expr, FixupKind, Loc));
+                                   Expr, FixupKind, PCRel));
   emitConstant(0, Size, CB);
 }
 
@@ -706,8 +703,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
                       ? X86II::getSizeOfImm(TSFlags)
                       : 0;
 
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
-                  Fixups, -ImmSize);
+    emitImmediate(Disp, MI.getLoc(), FixupKind, true, StartByte, CB, Fixups,
+                  -ImmSize);
     return;
   }
 
@@ -761,7 +758,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
         }
         // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
         emitByte(modRMByte(1, RegOpcodeField, RMfield), CB);
-        emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups);
+        emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,
+                      Fixups);
         return;
       }
       // This is the [REG]+disp16 case.
@@ -773,7 +771,7 @@ void X86MCCodeEmitter::emitMemModRMByte(
     }
 
     // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
-    emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, CB, Fixups);
+    emitImmediate(Disp, MI.getLoc(), FK_Data_2, false, StartByte, CB, Fixups);
     return;
   }
 
@@ -791,7 +789,7 @@ void X86MCCodeEmitter::emitMemModRMByte(
                                    STI.hasFeature(X86::Is64Bit))) {
     if (!BaseReg) { // [disp32]     in X86-32 mode
       emitByte(modRMByte(0, RegOpcodeField, 5), CB);
-      emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, CB, Fixups);
+      emitImmediate(Disp, MI.getLoc(), FK_Data_4, false, StartByte, CB, Fixups);
       return;
     }
 
@@ -809,10 +807,10 @@ void X86MCCodeEmitter::emitMemModRMByte(
       // If the displacement is @tlscall, treat it as a zero.
       if (Disp.isExpr()) {
         auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
-        if (Sym && getSpecifier(Sym) == X86MCExpr::VK_TLSCALL) {
+        if (Sym && Sym->getSpecifier() == X86::S_TLSCALL) {
           // This is exclusively used by call *a@tlscall(base). The relocation
           // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
-          Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
+          Fixups.push_back(MCFixup::create(0, Sym, FK_NONE));
           emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), CB);
           return;
         }
@@ -827,8 +825,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
       int ImmOffset = 0;
       if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
         emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), CB);
-        emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
-                      ImmOffset);
+        emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB,
+                      Fixups, ImmOffset);
         return;
       }
     }
@@ -840,8 +838,8 @@ void X86MCCodeEmitter::emitMemModRMByte(
     unsigned Opcode = MI.getOpcode();
     unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
                                                 : X86::reloc_signed_4byte;
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, CB,
-                  Fixups);
+    emitImmediate(Disp, MI.getLoc(), MCFixupKind(FixupKind), false, StartByte,
+                  CB, Fixups);
     return;
   }
 
@@ -889,11 +887,11 @@ void X86MCCodeEmitter::emitMemModRMByte(
 
   // Do we need to output a displacement?
   if (ForceDisp8)
-    emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, CB, Fixups,
+    emitImmediate(Disp, MI.getLoc(), FK_Data_1, false, StartByte, CB, Fixups,
                   ImmOffset);
   else if (ForceDisp32)
-    emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
-                  StartByte, CB, Fixups);
+    emitImmediate(Disp, MI.getLoc(), X86::reloc_signed_4byte, false, StartByte,
+                  CB, Fixups);
 }
 
 /// Emit all instruction prefixes.
@@ -1385,8 +1383,8 @@ PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
       // handled as a special case here so that it also works for hand-written
       // assembly without the user needing to write REX, as with GNU as.
       const auto *Ref = dyn_cast<MCSymbolRefExpr>(MO.getExpr());
-      if (Ref && (getSpecifier(Ref) == X86MCExpr::VK_GOTTPOFF ||
-                  getSpecifier(Ref) == X86MCExpr::VK_TLSDESC)) {
+      if (Ref && (Ref->getSpecifier() == X86::S_GOTTPOFF ||
+                  Ref->getSpecifier() == X86::S_TLSDESC)) {
         Prefix.setLowerBound(REX);
       }
     }
@@ -1628,33 +1626,29 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
       break;
 
     const MCOperand &Op = MI.getOperand(CurOp++);
-    emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
-                  MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, CB,
-                  Fixups);
+    emitImmediate(Op, MI.getLoc(), X86::reloc_branch_4byte_pcrel, true,
+                  StartByte, CB, Fixups);
     break;
   }
   case X86II::RawFrmMemOffs:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
-                  StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
     ++CurOp; // skip segment operand
     break;
   case X86II::RawFrmImm8:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_1, false,
                   StartByte, CB, Fixups);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
-                  CB, Fixups);
     break;
   case X86II::RawFrmImm16:
     emitByte(BaseOpcode, CB);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), getImmFixupKind(TSFlags),
+                  X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
+    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), FK_Data_2, false,
                   StartByte, CB, Fixups);
-    emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
-                  CB, Fixups);
     break;
 
   case X86II::AddRegFrm:
@@ -2007,7 +2001,7 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
       assert(Val < 16 && "Immediate operand value out of range");
       I8RegNum |= Val;
     }
-    emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
+    emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), FK_Data_1, false,
                   StartByte, CB, Fixups);
   } else {
     // If there is a remaining operand, it must be a trailing immediate. Emit it
@@ -2015,12 +2009,12 @@ void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
     // (SSE4a extrq and insertq) have two trailing immediates.
 
     // Skip two trainling conditional operands encoded in EVEX prefix
-    unsigned RemaningOps = NumOps - CurOp - 2 * HasTwoConditionalOps;
-    while (RemaningOps) {
+    unsigned RemainingOps = NumOps - CurOp - 2 * HasTwoConditionalOps;
+    while (RemainingOps) {
       emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
-                    X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
-                    StartByte, CB, Fixups);
-      --RemaningOps;
+                    getImmFixupKind(Desc.TSFlags),
+                    X86II::isImmPCRel(Desc.TSFlags), StartByte, CB, Fixups);
+      --RemainingOps;
     }
     CurOp += 2 * HasTwoConditionalOps;
   }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h b/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
index 1d3adaf867f6..e1ec3fa1205f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCExpr.h
@@ -31,54 +31,13 @@ class X86MCExpr : public MCTargetExpr {
   explicit X86MCExpr(MCRegister R) : Reg(R) {}
 
 public:
-  // Relocation specifier. Named VK_ for legacy reasons.
-  enum Specifier {
-    VK_None,
-
-    VK_ABS8 = MCSymbolRefExpr::FirstTargetSpecifier,
-    VK_DTPOFF,
-    VK_DTPREL,
-    VK_GOT,
-    VK_GOTENT,
-    VK_GOTNTPOFF,
-    VK_GOTOFF,
-    VK_GOTPCREL,
-    VK_GOTPCREL_NORELAX,
-    VK_GOTREL,
-    VK_GOTTPOFF,
-    VK_INDNTPOFF,
-    VK_NTPOFF,
-    VK_PCREL,
-    VK_PLT,
-    VK_PLTOFF,
-    VK_SIZE,
-    VK_TLSCALL,
-    VK_TLSDESC,
-    VK_TLSGD,
-    VK_TLSLD,
-    VK_TLSLDM,
-    VK_TLVP,
-    VK_TLVPPAGE,
-    VK_TLVPPAGEOFF,
-    VK_TPOFF,
-  };
-
-  /// @name Construction
-  /// @{
-
   static const X86MCExpr *create(MCRegister Reg, MCContext &Ctx) {
     return new (Ctx) X86MCExpr(Reg);
   }
 
-  /// @}
-  /// @name Accessors
-  /// @{
-
   /// getSubExpr - Get the child of this expression.
   MCRegister getReg() const { return Reg; }
 
-  /// @}
-
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override {
     if (!MAI || MAI->getAssemblerDialect() == 0)
       OS << '%';
@@ -103,10 +62,6 @@ class X86MCExpr : public MCTargetExpr {
     return E->getKind() == MCExpr::Target;
   }
 };
-
-static inline X86MCExpr::Specifier getSpecifier(const MCSymbolRefExpr *SRE) {
-  return X86MCExpr::Specifier(SRE->getKind());
-}
 } // end namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index b61dbfd968bd..c34425f6661b 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -78,11 +78,8 @@ static unsigned getFixupKindLog2Size(unsigned Kind) {
   switch (Kind) {
   default:
     llvm_unreachable("invalid fixup kind!");
-  case FK_PCRel_1:
   case FK_Data_1: return 0;
-  case FK_PCRel_2:
   case FK_Data_2: return 1;
-  case FK_PCRel_4:
     // FIXME: Remove these!!!
   case X86::reloc_riprel_4byte:
   case X86::reloc_riprel_4byte_relax:
@@ -102,7 +99,7 @@ static unsigned getFixupKindLog2Size(unsigned Kind) {
 void X86MachObjectWriter::RecordX86_64Relocation(
     MachObjectWriter *Writer, MCAssembler &Asm, const MCFragment *Fragment,
     const MCFixup &Fixup, MCValue Target, uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned IsRIPRel = isFixupKindRIPRel(Fixup.getKind());
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
 
@@ -257,7 +254,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
     auto Specifier = Target.getSpecifier();
     if (IsPCRel) {
       if (IsRIPRel) {
-        if (Specifier == X86MCExpr::VK_GOTPCREL) {
+        if (Specifier == X86::S_GOTPCREL) {
           // x86_64 distinguishes movq foo@GOTPCREL so that the linker can
           // rewrite the movq to an leaq at link time if the symbol ends up in
           // the same linkage unit.
@@ -265,7 +262,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(
             Type = MachO::X86_64_RELOC_GOT_LOAD;
           else
             Type = MachO::X86_64_RELOC_GOT;
-        } else if (Specifier == X86MCExpr::VK_TLVP) {
+        } else if (Specifier == X86::S_TLVP) {
           Type = MachO::X86_64_RELOC_TLV;
         } else if (Specifier) {
           reportError(Fixup.getLoc(),
@@ -304,16 +301,16 @@ void X86MachObjectWriter::RecordX86_64Relocation(
         Type = MachO::X86_64_RELOC_BRANCH;
       }
     } else {
-      if (Specifier == X86MCExpr::VK_GOT) {
+      if (Specifier == X86::S_GOT) {
         Type = MachO::X86_64_RELOC_GOT;
-      } else if (Specifier == X86MCExpr::VK_GOTPCREL) {
+      } else if (Specifier == X86::S_GOTPCREL) {
         // GOTPCREL is allowed as a modifier on non-PCrel instructions, in which
         // case all we do is set the PCrel bit in the relocation entry; this is
         // used with exception handling, for example. The source is required to
         // include any necessary offset directly.
         Type = MachO::X86_64_RELOC_GOT;
         IsPCRel = 1;
-      } else if (Specifier == X86MCExpr::VK_TLVP) {
+      } else if (Specifier == X86::S_TLVP) {
         reportError(Fixup.getLoc(),
                     "TLVP symbol modifier should have been rip-rel");
         return;
@@ -353,7 +350,7 @@ bool X86MachObjectWriter::recordScatteredRelocation(MachObjectWriter *Writer,
                                                     uint64_t &FixedValue) {
   uint64_t OriginalFixedValue = FixedValue;
   uint32_t FixupOffset = Asm.getFragmentOffset(*Fragment) + Fixup.getOffset();
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Type = MachO::GENERIC_RELOC_VANILLA;
 
   // See <reloc.h>.
@@ -446,7 +443,7 @@ void X86MachObjectWriter::recordTLVPRelocation(MachObjectWriter *Writer,
                                                MCValue Target,
                                                uint64_t &FixedValue) {
   const MCSymbol *SymA = Target.getAddSym();
-  assert(Target.getSpecifier() == X86MCExpr::VK_TLVP && !is64Bit() &&
+  assert(Target.getSpecifier() == X86::S_TLVP && !is64Bit() &&
          "Should only be called with a 32-bit TLVP relocation!");
 
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
@@ -483,12 +480,12 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
                                               const MCFixup &Fixup,
                                               MCValue Target,
                                               uint64_t &FixedValue) {
-  unsigned IsPCRel = Writer->isFixupKindPCRel(Asm, Fixup.getKind());
+  unsigned IsPCRel = Fixup.isPCRel();
   unsigned Log2Size = getFixupKindLog2Size(Fixup.getKind());
   const MCSymbol *A = Target.getAddSym();
 
   // If this is a 32-bit TLVP reloc it's handled a bit differently.
-  if (A && Target.getSpecifier() == X86MCExpr::VK_TLVP) {
+  if (A && Target.getSpecifier() == X86::S_TLVP) {
     recordTLVPRelocation(Writer, Asm, Fragment, Fixup, Target, FixedValue);
     return;
   }
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
index bc8bf3868fc3..0bfc0e071a66 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/X86FixupKinds.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/MC/MCContext.h"
@@ -44,14 +45,17 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
                                               const MCAsmBackend &MAB) const {
   const bool Is64Bit = getMachine() == COFF::IMAGE_FILE_MACHINE_AMD64;
   unsigned FixupKind = Fixup.getKind();
+  bool PCRel = Fixup.isPCRel();
   if (IsCrossSection) {
     // IMAGE_REL_AMD64_REL64 does not exist. We treat FK_Data_8 as FK_PCRel_4 so
     // that .quad a-b can lower to IMAGE_REL_AMD64_REL32. This allows generic
     // instrumentation to not bother with the COFF limitation. A negative value
     // needs attention.
-    if (FixupKind == FK_Data_4 || FixupKind == llvm::X86::reloc_signed_4byte ||
-        (FixupKind == FK_Data_8 && Is64Bit)) {
-      FixupKind = FK_PCRel_4;
+    if (!PCRel &&
+        (FixupKind == FK_Data_4 || FixupKind == llvm::X86::reloc_signed_4byte ||
+         (FixupKind == FK_Data_8 && Is64Bit))) {
+      FixupKind = FK_Data_4;
+      PCRel = true;
     } else {
       Ctx.reportError(Fixup.getLoc(), "Cannot represent this expression");
       return COFF::IMAGE_REL_AMD64_ADDR32;
@@ -61,7 +65,6 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
   auto Spec = Target.getSpecifier();
   if (Is64Bit) {
     switch (FixupKind) {
-    case FK_PCRel_4:
     case X86::reloc_riprel_4byte:
     case X86::reloc_riprel_4byte_movq_load:
     case X86::reloc_riprel_4byte_movq_load_rex2:
@@ -72,11 +75,14 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     case X86::reloc_branch_4byte_pcrel:
       return COFF::IMAGE_REL_AMD64_REL32;
     case FK_Data_4:
+      if (PCRel)
+        return COFF::IMAGE_REL_AMD64_REL32;
+      [[fallthrough]];
     case X86::reloc_signed_4byte:
     case X86::reloc_signed_4byte_relax:
       if (Spec == MCSymbolRefExpr::VK_COFF_IMGREL32)
         return COFF::IMAGE_REL_AMD64_ADDR32NB;
-      if (Spec == MCSymbolRefExpr::VK_SECREL)
+      if (Spec == X86::S_COFF_SECREL)
         return COFF::IMAGE_REL_AMD64_SECREL;
       return COFF::IMAGE_REL_AMD64_ADDR32;
     case FK_Data_8:
@@ -91,16 +97,18 @@ unsigned X86WinCOFFObjectWriter::getRelocType(MCContext &Ctx,
     }
   } else if (getMachine() == COFF::IMAGE_FILE_MACHINE_I386) {
     switch (FixupKind) {
-    case FK_PCRel_4:
     case X86::reloc_riprel_4byte:
     case X86::reloc_riprel_4byte_movq_load:
       return COFF::IMAGE_REL_I386_REL32;
     case FK_Data_4:
+      if (PCRel)
+        return COFF::IMAGE_REL_I386_REL32;
+      [[fallthrough]];
     case X86::reloc_signed_4byte:
     case X86::reloc_signed_4byte_relax:
       if (Spec == MCSymbolRefExpr::VK_COFF_IMGREL32)
         return COFF::IMAGE_REL_I386_DIR32NB;
-      if (Spec == MCSymbolRefExpr::VK_SECREL)
+      if (Spec == X86::S_COFF_SECREL)
         return COFF::IMAGE_REL_I386_SECREL;
       return COFF::IMAGE_REL_I386_DIR32;
     case FK_SecRel_2:
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 4d0bfbef794c..772e48efb860 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -811,6 +811,7 @@ include "X86SchedSkylakeClient.td"
 include "X86SchedSkylakeServer.td"
 include "X86SchedIceLake.td"
 include "X86SchedAlderlakeP.td"
+include "X86SchedLunarlakeP.td"
 include "X86SchedSapphireRapids.td"
 
 //===----------------------------------------------------------------------===//
@@ -1573,7 +1574,7 @@ def ProcessorFeatures {
                                                   FeatureVPCLMULQDQ];
   list<SubtargetFeature> ZN3AdditionalTuning = [TuningMacroFusion];
   list<SubtargetFeature> ZN3Tuning =
-    !listconcat(ZN2Tuning, ZN3AdditionalTuning);
+    !listremove(!listconcat(ZN2Tuning, ZN3AdditionalTuning), [TuningSlowSHLD]);
   list<SubtargetFeature> ZN3Features =
     !listconcat(ZN2Features, ZN3AdditionalFeatures);
 
@@ -1870,10 +1871,12 @@ def : ProcModel<"meteorlake", AlderlakePModel,
                 ProcessorFeatures.ADLFeatures, ProcessorFeatures.ADLTuning>;
 def : ProcModel<"arrowlake", AlderlakePModel,
                 ProcessorFeatures.ARLFeatures, ProcessorFeatures.ADLTuning>;
-foreach P = ["arrowlake-s", "arrowlake_s", "lunarlake"] in {
+foreach P = ["arrowlake-s", "arrowlake_s"] in {
 def : ProcModel<P, AlderlakePModel,
                 ProcessorFeatures.ARLSFeatures, ProcessorFeatures.ADLTuning>;
 }
+def : ProcModel<"lunarlake", LunarlakePModel, ProcessorFeatures.ARLSFeatures,
+                ProcessorFeatures.ADLTuning>;
 def : ProcModel<"pantherlake", AlderlakePModel,
                 ProcessorFeatures.PTLFeatures, ProcessorFeatures.ADLTuning>;
 def : ProcModel<"clearwaterforest", AlderlakePModel,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d5837ab938d4..9662b18213a6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10312,6 +10312,11 @@ static bool isNonZeroElementsInOrder(const APInt &Zeroable,
   return true;
 }
 
+static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
+                                      ArrayRef<SDValue> Ops, SelectionDAG &DAG,
+                                      const X86Subtarget &Subtarget,
+                                      unsigned Depth = 0);
+
 /// Try to lower a shuffle with a single PSHUFB of V1 or V2.
 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
                                       ArrayRef<int> Mask, SDValue V1,
@@ -10692,7 +10697,8 @@ static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
                                     SelectionDAG &DAG) {
   assert((VT.is128BitVector() || VT.is256BitVector()) &&
          "Unexpected VTRUNC type");
-  if (!Subtarget.hasAVX512())
+  if (!Subtarget.hasAVX512() ||
+      (VT.is256BitVector() && !Subtarget.useAVX512Regs()))
     return SDValue();
 
   unsigned NumElts = VT.getVectorNumElements();
@@ -10721,30 +10727,19 @@ static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,
       bool UndefUppers =
           UpperElts > 0 && isUndefInRange(Mask, NumSrcElts, UpperElts);
 
+      // As we're using both sources then we need to concat them together
+      // and truncate from the double-sized src.
+      MVT ConcatVT = VT.getDoubleNumVectorElementsVT();
+
       // For offset truncations, ensure that the concat is cheap.
-      if (Offset) {
-        auto IsCheapConcat = [&](SDValue Lo, SDValue Hi) {
-          if (Lo.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
-              Hi.getOpcode() == ISD::EXTRACT_SUBVECTOR)
-            return Lo.getOperand(0) == Hi.getOperand(0);
-          if (ISD::isNormalLoad(Lo.getNode()) &&
-              ISD::isNormalLoad(Hi.getNode())) {
-            auto *LDLo = cast<LoadSDNode>(Lo);
-            auto *LDHi = cast<LoadSDNode>(Hi);
-            return DAG.areNonVolatileConsecutiveLoads(
-                LDHi, LDLo, Lo.getValueType().getStoreSize(), 1);
-          }
-          return false;
-        };
-        if (!IsCheapConcat(peekThroughBitcasts(V1), peekThroughBitcasts(V2)))
+      SDValue Src =
+          combineConcatVectorOps(DL, ConcatVT, {V1, V2}, DAG, Subtarget);
+      if (!Src) {
+        if (Offset)
           continue;
+        Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
       }
 
-      // As we're using both sources then we need to concat them together
-      // and truncate from the double-sized src.
-      MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);
-      SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, V1, V2);
-
       MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);
       MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);
       Src = DAG.getBitcast(SrcVT, Src);
@@ -19211,7 +19206,7 @@ SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
   // global base reg.
   unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
 
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
   SDLoc DL(JT);
   Result =
@@ -19239,7 +19234,7 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
   int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
   SDLoc dl(Op);
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
   Result =
       DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, PtrVT, Result);
@@ -19282,7 +19277,7 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
   bool NeedsLoad = isGlobalStubReference(OpFlags);
 
   CodeModel::Model M = DAG.getTarget().getCodeModel();
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   SDValue Result;
 
   if (GV) {
@@ -19541,7 +19536,7 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     return LowerToTLSEmulatedModel(GA, DAG);
 
   const GlobalValue *GV = GA->getGlobal();
-  auto PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
   bool PositionIndependent = isPositionIndependent();
 
   if (Subtarget.isTargetELF()) {
@@ -25787,7 +25782,7 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
   Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
 
   bool Is64Bit = Subtarget.is64Bit();
-  MVT SPTy = getPointerTy(DAG.getDataLayout());
+  MVT SPTy = Op.getValueType().getSimpleVT();
 
   SDValue Result;
   if (!Lower) {
@@ -25855,7 +25850,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
 
 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
-  auto PtrVT = getPointerTy(MF.getDataLayout());
+  SDValue Ptr = Op.getOperand(1);
+  EVT PtrVT = Ptr.getValueType();
+
   X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
 
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
@@ -25866,8 +25863,7 @@ SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
     // vastart just stores the address of the VarArgsFrameIndex slot into the
     // memory location argument.
     SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
-    return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
-                        MachinePointerInfo(SV));
+    return DAG.getStore(Op.getOperand(0), DL, FR, Ptr, MachinePointerInfo(SV));
   }
 
   // __va_list_tag:
@@ -25956,7 +25952,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
                        DAG.getTargetConstant(ArgSize, dl, MVT::i32),
                        DAG.getTargetConstant(ArgMode, dl, MVT::i8),
                        DAG.getTargetConstant(Align, dl, MVT::i32)};
-  SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
+  SDVTList VTs = DAG.getVTList(SrcPtr.getValueType(), MVT::Other);
   SDValue VAARG = DAG.getMemIntrinsicNode(
       Subtarget.isTarget64BitLP64() ? X86ISD::VAARG_64 : X86ISD::VAARG_X32, dl,
       VTs, InstOps, MVT::i64, MachinePointerInfo(SV),
@@ -26277,9 +26273,6 @@ static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
   MachineFunction &MF = DAG.getMachineFunction();
   SDLoc dl;
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
-
   // It's possible that the parent function no longer has a personality function
   // if the exceptional code was optimized away, in which case we just return
   // the incoming EBP.
@@ -26290,6 +26283,7 @@ static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
   // registration, or the .set_setframe offset.
   MCSymbol *OffsetSym = MF.getContext().getOrCreateParentFrameOffsetSymbol(
       GlobalValue::dropLLVMManglingEscape(Fn->getName()));
+  MVT PtrVT = EntryEBP.getValueType().getSimpleVT();
   SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
   SDValue ParentFrameOffset =
       DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
@@ -27350,7 +27344,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::thread_pointer: {
     if (Subtarget.isTargetELF()) {
       SDLoc dl(Op);
-      EVT PtrVT = getPointerTy(DAG.getDataLayout());
+      EVT PtrVT = Op.getValueType();
       // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
       Value *Ptr = Constant::getNullValue(PointerType::get(
           *DAG.getContext(), Subtarget.is64Bit() ? X86AS::FS : X86AS::GS));
@@ -28217,12 +28211,9 @@ SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
-  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
-    return SDValue();
-
   unsigned Depth = Op.getConstantOperandVal(0);
   SDLoc dl(Op);
-  EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  EVT PtrVT = Op.getValueType();
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
@@ -29428,16 +29419,22 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
                    Op->getFlags().hasNoNaNs() || (IsXNeverNaN && IsYNeverNaN);
 
   // If we did no ordering operands for signed zero handling and we need
-  // to process NaN and we know that the second operand is not NaN then put
-  // it in first operand and we will not need to post handle NaN after max/min.
-  if (IgnoreSignedZero && !IgnoreNaN && DAG.isKnownNeverNaN(NewY))
+  // to process NaN and we know that one of the operands is not NaN then:
+  //  - For minimum/maximum, put it in the first operand,
+  //  - For minimumnum/maximumnum, put it in the second operand,
+  // and we will not need to post handle NaN after max/min.
+  if (IgnoreSignedZero && !IgnoreNaN &&
+      DAG.isKnownNeverNaN(IsNum ? NewX : NewY))
     std::swap(NewX, NewY);
 
   SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
 
-  if (IgnoreNaN || DAG.isKnownNeverNaN(NewX))
+  if (IgnoreNaN || DAG.isKnownNeverNaN(IsNum ? NewY : NewX))
     return MinMax;
 
+  if (DAG.isKnownNeverNaN(NewX))
+    NewX = NewY;
+
   SDValue IsNaN =
       DAG.getSetCC(DL, SetCCType, NewX, NewX, IsNum ? ISD::SETO : ISD::SETUO);
 
@@ -34156,7 +34153,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
-    if (VT.isVector() && Subtarget.hasFP16() &&
+    if (VT.isVector() && Subtarget.hasFP16() && Subtarget.hasVLX() &&
         SrcVT.getVectorElementType() == MVT::f16) {
       EVT EleVT = VT.getVectorElementType();
       EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;
@@ -34170,10 +34167,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       }
 
       if (IsStrict) {
+        Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
         Res =
             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});
         Chain = Res.getValue(1);
       } else {
+        Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         Res = DAG.getNode(Opc, dl, ResVT, Src);
       }
 
@@ -42183,11 +42182,6 @@ static SDValue canonicalizeLaneShuffleWithRepeatedOps(SDValue V,
   return SDValue();
 }
 
-static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
-                                      ArrayRef<SDValue> Ops, SelectionDAG &DAG,
-                                      const X86Subtarget &Subtarget,
-                                      unsigned Depth = 0);
-
 /// Try to combine x86 target specific shuffles.
 static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
                                     SelectionDAG &DAG,
@@ -45053,7 +45047,10 @@ bool X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
   unsigned NumElts = DemandedElts.getBitWidth();
 
   switch (Op.getOpcode()) {
+  case X86ISD::BLENDI:
   case X86ISD::PSHUFD:
+  case X86ISD::UNPCKL:
+  case X86ISD::UNPCKH:
   case X86ISD::VPERMILPI:
   case X86ISD::VPERMV3: {
     SmallVector<int, 8> Mask;
@@ -45097,15 +45094,22 @@ bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
   case X86ISD::VSRLI:
   case X86ISD::VSRAI:
     return false;
+    // SSE blends.
+  case X86ISD::BLENDI:
+  case X86ISD::BLENDV:
+    return false;
+    // SSE target shuffles.
   case X86ISD::PSHUFD:
+  case X86ISD::UNPCKL:
+  case X86ISD::UNPCKH:
   case X86ISD::VPERMILPI:
   case X86ISD::VPERMV3:
-  case X86ISD::UNPCKH:
-  case X86ISD::UNPCKL:
     return false;
-    // SSE comparisons handle all fcmp cases.
-    // TODO: Add PCMPEQ/GT and CMPM/MM with test coverage.
+    // SSE comparisons handle all icmp/fcmp cases.
+    // TODO: Add CMPM/MM with test coverage.
   case X86ISD::CMPP:
+  case X86ISD::PCMPEQ:
+  case X86ISD::PCMPGT:
     return false;
   case ISD::INTRINSIC_WO_CHAIN:
     switch (Op->getConstantOperandVal(0)) {
@@ -46006,12 +46010,11 @@ static bool detectZextAbsDiff(SDValue Abs, SDValue &Op0, SDValue &Op1) {
   using namespace SDPatternMatch;
 
   // Check if the operands of the sub are zero-extended from vectors of i8.
-  EVT SrcVT0, SrcVT1;
-  return sd_match(Abs,
-                  m_Abs(m_Sub(m_AllOf(m_Value(Op0), m_ZExt(m_VT(SrcVT0))),
-                              m_AllOf(m_Value(Op1), m_ZExt(m_VT(SrcVT1)))))) &&
-         SrcVT0.getVectorElementType() == MVT::i8 &&
-         SrcVT1.getVectorElementType() == MVT::i8;
+  return sd_match(
+      Abs,
+      m_Abs(m_Sub(
+          m_AllOf(m_Value(Op0), m_ZExt(m_SpecificVectorElementVT(MVT::i8))),
+          m_AllOf(m_Value(Op1), m_ZExt(m_SpecificVectorElementVT(MVT::i8))))));
 }
 
 static SDValue createVPDPBUSD(SelectionDAG &DAG, SDValue LHS, SDValue RHS,
@@ -47264,13 +47267,14 @@ static SDValue combineToExtendBoolVectorInReg(
                      DAG.getConstant(EltSizeInBits - 1, DL, VT));
 }
 
-/// If a vector select has an operand that is -1 or 0, try to simplify the
+/// If a vector select has an left operand that is 0, try to simplify the
 /// select to a bitwise logic operation.
-/// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
-static SDValue
-combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
-                                 TargetLowering::DAGCombinerInfo &DCI,
-                                 const X86Subtarget &Subtarget) {
+/// TODO: Move to DAGCombiner.combineVSelectWithAllOnesOrZeros, possibly using
+/// TargetLowering::hasAndNot()?
+static SDValue combineVSelectWithLastZeros(SDNode *N, SelectionDAG &DAG,
+                                           const SDLoc &DL,
+                                           TargetLowering::DAGCombinerInfo &DCI,
+                                           const X86Subtarget &Subtarget) {
   SDValue Cond = N->getOperand(0);
   SDValue LHS = N->getOperand(1);
   SDValue RHS = N->getOperand(2);
@@ -47283,20 +47287,6 @@ combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
 
   assert(CondVT.isVector() && "Vector select expects a vector selector!");
 
-  // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
-  // TODO: Can we assert that both operands are not zeros (because that should
-  //       get simplified at node creation time)?
-  bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
-  bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
-
-  // If both inputs are 0/undef, create a complete zero vector.
-  // FIXME: As noted above this should be handled by DAGCombiner/getNode.
-  if (TValIsAllZeros && FValIsAllZeros) {
-    if (VT.isFloatingPoint())
-      return DAG.getConstantFP(0.0, DL, VT);
-    return DAG.getConstant(0, DL, VT);
-  }
-
   // To use the condition operand as a bitwise mask, it must have elements that
   // are the same size as the select elements. Ie, the condition operand must
   // have already been promoted from the IR select condition type <N x i1>.
@@ -47305,56 +47295,15 @@ combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
   if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
     return SDValue();
 
-  // Try to invert the condition if true value is not all 1s and false value is
-  // not all 0s. Only do this if the condition has one use.
-  bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
-  if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
-      // Check if the selector will be produced by CMPP*/PCMP*.
-      Cond.getOpcode() == ISD::SETCC &&
-      // Check if SETCC has already been promoted.
-      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
-          CondVT) {
-    bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
-
-    if (TValIsAllZeros || FValIsAllOnes) {
-      SDValue CC = Cond.getOperand(2);
-      ISD::CondCode NewCC = ISD::getSetCCInverse(
-          cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
-      Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
-                          NewCC);
-      std::swap(LHS, RHS);
-      TValIsAllOnes = FValIsAllOnes;
-      FValIsAllZeros = TValIsAllZeros;
-    }
-  }
-
   // Cond value must be 'sign splat' to be converted to a logical op.
   if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
     return SDValue();
 
-  // vselect Cond, 111..., 000... -> Cond
-  if (TValIsAllOnes && FValIsAllZeros)
-    return DAG.getBitcast(VT, Cond);
-
   if (!TLI.isTypeLegal(CondVT))
     return SDValue();
 
-  // vselect Cond, 111..., X -> or Cond, X
-  if (TValIsAllOnes) {
-    SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
-    SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
-    return DAG.getBitcast(VT, Or);
-  }
-
-  // vselect Cond, X, 000... -> and Cond, X
-  if (FValIsAllZeros) {
-    SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
-    SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
-    return DAG.getBitcast(VT, And);
-  }
-
   // vselect Cond, 000..., X -> andn Cond, X
-  if (TValIsAllZeros) {
+  if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
     SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
     SDValue AndN;
     // The canonical form differs for i1 vectors - x86andnp is not used
@@ -47624,31 +47573,26 @@ static SDValue combineLogicBlendIntoConditionalNegate(
 
 static SDValue commuteSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL,
                              const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
   if (!Subtarget.hasAVX512())
     return SDValue();
-  if (N->getOpcode() != ISD::VSELECT)
-    return SDValue();
-
-  SDValue Cond = N->getOperand(0);
-  SDValue LHS = N->getOperand(1);
-  SDValue RHS = N->getOperand(2);
 
-  if (canCombineAsMaskOperation(LHS, Subtarget))
+  ISD::CondCode CC;
+  SDValue Cond, X, Y, LHS, RHS;
+  if (!sd_match(N, m_VSelect(m_AllOf(m_Value(Cond),
+                                     m_OneUse(m_SetCC(m_Value(X), m_Value(Y),
+                                                      m_CondCode(CC)))),
+                             m_Value(LHS), m_Value(RHS))))
     return SDValue();
 
-  if (!canCombineAsMaskOperation(RHS, Subtarget))
-    return SDValue();
-
-  if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
+  if (canCombineAsMaskOperation(LHS, Subtarget) ||
+      !canCombineAsMaskOperation(RHS, Subtarget))
     return SDValue();
 
   // Commute LHS and RHS to create opportunity to select mask instruction.
   // (vselect M, L, R) -> (vselect ~M, R, L)
-  ISD::CondCode NewCC =
-      ISD::getSetCCInverse(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
-                           Cond.getOperand(0).getValueType());
-  Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), Cond.getOperand(0),
-                      Cond.getOperand(1), NewCC);
+  ISD::CondCode NewCC = ISD::getSetCCInverse(CC, X.getValueType());
+  Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(), X, Y, NewCC);
   return DAG.getSelect(DL, LHS.getValueType(), Cond, RHS, LHS);
 }
 
@@ -48115,7 +48059,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget))
     return SDValue();
 
-  if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DL, DCI, Subtarget))
+  if (SDValue V = combineVSelectWithLastZeros(N, DAG, DL, DCI, Subtarget))
     return V;
 
   if (SDValue V = combineVSelectToBLENDV(N, DAG, DL, DCI, Subtarget))
@@ -49899,31 +49843,24 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
 static SDValue combineShiftToPMULH(SDNode *N, SelectionDAG &DAG,
                                    const SDLoc &DL,
                                    const X86Subtarget &Subtarget) {
+  using namespace SDPatternMatch;
   assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
-           "SRL or SRA node is required here!");
+         "SRL or SRA node is required here!");
 
   if (!Subtarget.hasSSE2())
     return SDValue();
 
-  // The operation feeding into the shift must be a multiply.
-  SDValue ShiftOperand = N->getOperand(0);
-  if (ShiftOperand.getOpcode() != ISD::MUL || !ShiftOperand.hasOneUse())
-    return SDValue();
-
   // Input type should be at least vXi32.
   EVT VT = N->getValueType(0);
   if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32)
     return SDValue();
 
-  // Need a shift by 16.
-  APInt ShiftAmt;
-  if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), ShiftAmt) ||
-      ShiftAmt != 16)
+  // The operation must be a multiply shifted right by 16.
+  SDValue LHS, RHS;
+  if (!sd_match(N->getOperand(1), m_SpecificInt(16)) ||
+      !sd_match(N->getOperand(0), m_OneUse(m_Mul(m_Value(LHS), m_Value(RHS)))))
     return SDValue();
 
-  SDValue LHS = ShiftOperand.getOperand(0);
-  SDValue RHS = ShiftOperand.getOperand(1);
-
   unsigned ExtOpc = LHS.getOpcode();
   if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
       RHS.getOpcode() != ExtOpc)
@@ -52657,7 +52594,8 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
 ///   XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
 /// into:
 ///   SETGT(X, -1)
-static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldXorTruncShiftIntoCmp(SDNode *N, const SDLoc &DL,
+                                        SelectionDAG &DAG) {
   // This is only worth doing if the output type is i8 or i1.
   EVT ResultType = N->getValueType(0);
   if (ResultType != MVT::i8 && ResultType != MVT::i1)
@@ -52692,7 +52630,6 @@ static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
   // Create a greater-than comparison against -1.
   // N.B. Using SETGE against 0 works but we want a canonical looking
   // comparison, using SETGT matches up with what TranslateX86CC.
-  SDLoc DL(N);
   SDValue ShiftOp = Shift.getOperand(0);
   EVT ShiftOpTy = ShiftOp.getValueType();
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -54862,11 +54799,7 @@ static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   SDLoc dl(N);
-
-  unsigned IntBits = VT.getScalarSizeInBits();
-  MVT IntSVT = MVT::getIntegerVT(IntBits);
-  MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
-
+  MVT IntVT = VT.changeVectorElementTypeToInteger();
   SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
   SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
   unsigned IntOpcode;
@@ -54883,9 +54816,8 @@ static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
   return DAG.getBitcast(VT, IntOp);
 }
 
-
 /// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
-static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
+static SDValue foldXor1SetCC(SDNode *N, const SDLoc &DL, SelectionDAG &DAG) {
   if (N->getOpcode() != ISD::XOR)
     return SDValue();
 
@@ -54895,7 +54827,6 @@ static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
 
   X86::CondCode NewCC = X86::GetOppositeBranchCondition(
       X86::CondCode(LHS->getConstantOperandVal(0)));
-  SDLoc DL(N);
   return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
 }
 
@@ -54993,13 +54924,13 @@ static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  if (SDValue SetCC = foldXor1SetCC(N, DAG))
+  if (SDValue SetCC = foldXor1SetCC(N, DL, DAG))
     return SetCC;
 
   if (SDValue R = combineOrXorWithSETCC(N->getOpcode(), DL, VT, N0, N1, DAG))
     return R;
 
-  if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
+  if (SDValue RV = foldXorTruncShiftIntoCmp(N, DL, DAG))
     return RV;
 
   // Fold not(iX bitcast(vXi1)) -> (iX bitcast(not(vec))) for legal boolvecs.
@@ -57508,7 +57439,7 @@ static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
   bool IsSub = X86ISD::SUB == N->getOpcode();
   unsigned GenericOpc = IsSub ? ISD::SUB : ISD::ADD;
 
-  if (IsSub && isOneConstant(N->getOperand(1)) && !N->hasAnyUseOfValue(0))
+  if (IsSub && isOneConstant(RHS) && !N->hasAnyUseOfValue(0))
     if (SDValue CMP = combineX86SubCmpForFlags(N, SDValue(N, 1), DAG, DCI, ST))
       return CMP;
 
@@ -59577,7 +59508,6 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
   MVT VT = N->getSimpleValueType(0);
   SDValue InVec = N->getOperand(0);
   unsigned IdxVal = N->getConstantOperandVal(1);
-  SDValue InVecBC = peekThroughBitcasts(InVec);
   EVT InVecVT = InVec.getValueType();
   unsigned SizeInBits = VT.getSizeInBits();
   unsigned InSizeInBits = InVecVT.getSizeInBits();
@@ -59594,7 +59524,7 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
   // This pattern emerges during AVX1 legalization. We handle it before lowering
   // to avoid complications like splitting constant vector loads.
   if (Subtarget.hasAVX() && !Subtarget.hasAVX2() && TLI.isTypeLegal(InVecVT) &&
-      InSizeInBits == 256 && InVecBC.getOpcode() == ISD::AND) {
+      InSizeInBits == 256 && InVec.getOpcode() == ISD::AND) {
     auto isConcatenatedNot = [](SDValue V) {
       V = peekThroughBitcasts(V);
       if (!isBitwiseNot(V))
@@ -59602,12 +59532,12 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
       SDValue NotOp = V->getOperand(0);
       return peekThroughBitcasts(NotOp).getOpcode() == ISD::CONCAT_VECTORS;
     };
-    if (isConcatenatedNot(InVecBC.getOperand(0)) ||
-        isConcatenatedNot(InVecBC.getOperand(1))) {
+    if (isConcatenatedNot(InVec.getOperand(0)) ||
+        isConcatenatedNot(InVec.getOperand(1))) {
       // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1
-      SDValue Concat = splitVectorIntBinary(InVecBC, DAG, SDLoc(InVecBC));
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT,
-                         DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
+                         splitVectorIntBinary(InVec, DAG, DL),
+                         N->getOperand(1));
     }
   }
 
@@ -59691,7 +59621,7 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
     SmallVector<SDValue, 2> ShuffleInputs;
     unsigned NumSubVecs = InSizeInBits / SizeInBits;
     // Decode the shuffle mask and scale it so its shuffling subvectors.
-    if (getTargetShuffleInputs(InVecBC, ShuffleInputs, ShuffleMask, DAG) &&
+    if (getTargetShuffleInputs(InVec, ShuffleInputs, ShuffleMask, DAG) &&
         scaleShuffleElements(ShuffleMask, NumSubVecs, ScaledMask)) {
       unsigned SubVecIdx = IdxVal / NumSubElts;
       if (ScaledMask[SubVecIdx] == SM_SentinelUndef)
@@ -61746,6 +61676,9 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           return std::make_pair(0U, &X86::VR128XRegClass);
         return std::make_pair(0U, &X86::VR128RegClass);
       case MVT::f128:
+        if (!Subtarget.is64Bit())
+          break;
+        [[fallthrough]];
       case MVT::v16i8:
       case MVT::v8i16:
       case MVT::v4i32:
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 1aa00d4f09f7..cb38a39ff991 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "X86.h"
 #include "X86CallingConv.h"
 #include "X86FrameLowering.h"
@@ -472,7 +472,7 @@ X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
   assert(isPositionIndependent() && Subtarget.isPICStyleGOT());
   // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
   // entries.
-  return MCSymbolRefExpr::create(MBB->getSymbol(), X86MCExpr::VK_GOTOFF, Ctx);
+  return MCSymbolRefExpr::create(MBB->getSymbol(), X86::S_GOTOFF, Ctx);
 }
 
 /// Returns relocation base for the given PIC jumptable.
@@ -1929,9 +1929,9 @@ SDValue X86TargetLowering::LowerFormalArguments(
   }
 
   if (CallingConv::PreserveNone == CallConv)
-    for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
-      if (Ins[I].Flags.isSwiftSelf() || Ins[I].Flags.isSwiftAsync() ||
-          Ins[I].Flags.isSwiftError()) {
+    for (const ISD::InputArg &In : Ins) {
+      if (In.Flags.isSwiftSelf() || In.Flags.isSwiftAsync() ||
+          In.Flags.isSwiftError()) {
         errorUnsupported(DAG, dl,
                          "Swift attributes can't be used with preserve_none");
         break;
@@ -2421,9 +2421,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // Build a sequence of copy-to-reg nodes chained together with token chain
   // and glue operands which copy the outgoing args into registers.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -2462,9 +2461,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const uint32_t *Mask = [&]() {
@@ -2615,9 +2613,9 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   if (CallingConv::PreserveNone == CallConv)
-    for (unsigned I = 0, E = Outs.size(); I != E; ++I) {
-      if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftAsync() ||
-          Outs[I].Flags.isSwiftError()) {
+    for (const ISD::OutputArg &Out : Outs) {
+      if (Out.Flags.isSwiftSelf() || Out.Flags.isSwiftAsync() ||
+          Out.Flags.isSwiftError()) {
         errorUnsupported(DAG, dl,
                          "Swift attributes can't be used with preserve_none");
         break;
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 55d57d15f8d4..45d596bb498f 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -15,7 +15,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86EncodingOptimization.h"
 #include "MCTargetDesc/X86InstComments.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "MCTargetDesc/X86ShuffleDecode.h"
 #include "MCTargetDesc/X86TargetStreamer.h"
 #include "X86AsmPrinter.h"
@@ -234,7 +234,7 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   // FIXME: We would like an efficient form for this, so we don't have to do a
   // lot of extra uniquing.
   const MCExpr *Expr = nullptr;
-  auto Specifier = X86MCExpr::VK_None;
+  uint16_t Specifier = X86::S_None;
 
   switch (MO.getTargetFlags()) {
   default:
@@ -247,61 +247,61 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
     break;
 
   case X86II::MO_TLVP:
-    Specifier = X86MCExpr::VK_TLVP;
+    Specifier = X86::S_TLVP;
     break;
   case X86II::MO_TLVP_PIC_BASE:
-    Expr = MCSymbolRefExpr::create(Sym, X86MCExpr::VK_TLVP, Ctx);
+    Expr = MCSymbolRefExpr::create(Sym, X86::S_TLVP, Ctx);
     // Subtract the pic base.
     Expr = MCBinaryExpr::createSub(
         Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
     break;
   case X86II::MO_SECREL:
-    Specifier = X86MCExpr::Specifier(MCSymbolRefExpr::VK_SECREL);
+    Specifier = uint16_t(X86::S_COFF_SECREL);
     break;
   case X86II::MO_TLSGD:
-    Specifier = X86MCExpr::VK_TLSGD;
+    Specifier = X86::S_TLSGD;
     break;
   case X86II::MO_TLSLD:
-    Specifier = X86MCExpr::VK_TLSLD;
+    Specifier = X86::S_TLSLD;
     break;
   case X86II::MO_TLSLDM:
-    Specifier = X86MCExpr::VK_TLSLDM;
+    Specifier = X86::S_TLSLDM;
     break;
   case X86II::MO_GOTTPOFF:
-    Specifier = X86MCExpr::VK_GOTTPOFF;
+    Specifier = X86::S_GOTTPOFF;
     break;
   case X86II::MO_INDNTPOFF:
-    Specifier = X86MCExpr::VK_INDNTPOFF;
+    Specifier = X86::S_INDNTPOFF;
     break;
   case X86II::MO_TPOFF:
-    Specifier = X86MCExpr::VK_TPOFF;
+    Specifier = X86::S_TPOFF;
     break;
   case X86II::MO_DTPOFF:
-    Specifier = X86MCExpr::VK_DTPOFF;
+    Specifier = X86::S_DTPOFF;
     break;
   case X86II::MO_NTPOFF:
-    Specifier = X86MCExpr::VK_NTPOFF;
+    Specifier = X86::S_NTPOFF;
     break;
   case X86II::MO_GOTNTPOFF:
-    Specifier = X86MCExpr::VK_GOTNTPOFF;
+    Specifier = X86::S_GOTNTPOFF;
     break;
   case X86II::MO_GOTPCREL:
-    Specifier = X86MCExpr::VK_GOTPCREL;
+    Specifier = X86::S_GOTPCREL;
     break;
   case X86II::MO_GOTPCREL_NORELAX:
-    Specifier = X86MCExpr::VK_GOTPCREL_NORELAX;
+    Specifier = X86::S_GOTPCREL_NORELAX;
     break;
   case X86II::MO_GOT:
-    Specifier = X86MCExpr::VK_GOT;
+    Specifier = X86::S_GOT;
     break;
   case X86II::MO_GOTOFF:
-    Specifier = X86MCExpr::VK_GOTOFF;
+    Specifier = X86::S_GOTOFF;
     break;
   case X86II::MO_PLT:
-    Specifier = X86MCExpr::VK_PLT;
+    Specifier = X86::S_PLT;
     break;
   case X86II::MO_ABS8:
-    Specifier = X86MCExpr::VK_ABS8;
+    Specifier = X86::S_ABS8;
     break;
   case X86II::MO_PIC_BASE_OFFSET:
   case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
@@ -533,23 +533,23 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
   MCContext &Ctx = OutStreamer->getContext();
 
-  X86MCExpr::Specifier Specifier;
+  X86::Specifier Specifier;
   switch (MI.getOpcode()) {
   case X86::TLS_addr32:
   case X86::TLS_addr64:
   case X86::TLS_addrX32:
-    Specifier = X86MCExpr::VK_TLSGD;
+    Specifier = X86::S_TLSGD;
     break;
   case X86::TLS_base_addr32:
-    Specifier = X86MCExpr::VK_TLSLDM;
+    Specifier = X86::S_TLSLDM;
     break;
   case X86::TLS_base_addr64:
   case X86::TLS_base_addrX32:
-    Specifier = X86MCExpr::VK_TLSLD;
+    Specifier = X86::S_TLSLD;
     break;
   case X86::TLS_desc32:
   case X86::TLS_desc64:
-    Specifier = X86MCExpr::VK_TLSDESC;
+    Specifier = X86::S_TLSDESC;
     break;
   default:
     llvm_unreachable("unexpected opcode");
@@ -566,10 +566,10 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
   bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
                 Ctx.getTargetOptions()->X86RelaxRelocations;
 
-  if (Specifier == X86MCExpr::VK_TLSDESC) {
+  if (Specifier == X86::S_TLSDESC) {
     const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
-        MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)),
-        X86MCExpr::VK_TLSCALL, Ctx);
+        MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), X86::S_TLSCALL,
+        Ctx);
     EmitAndCountInstruction(
         MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
             .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)
@@ -586,7 +586,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
             .addExpr(Expr)
             .addReg(0));
   } else if (Is64Bits) {
-    bool NeedsPadding = Specifier == X86MCExpr::VK_TLSGD;
+    bool NeedsPadding = Specifier == X86::S_TLSGD;
     if (NeedsPadding && Is64BitsLP64)
       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
@@ -605,7 +605,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
     }
     if (UseGot) {
       const MCExpr *Expr =
-          MCSymbolRefExpr::create(TlsGetAddr, X86MCExpr::VK_GOTPCREL, Ctx);
+          MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOTPCREL, Ctx);
       EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
                                   .addReg(X86::RIP)
                                   .addImm(1)
@@ -613,12 +613,12 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
                                   .addExpr(Expr)
                                   .addReg(0));
     } else {
-      EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
-                                  .addExpr(MCSymbolRefExpr::create(
-                                      TlsGetAddr, X86MCExpr::VK_PLT, Ctx)));
+      EmitAndCountInstruction(
+          MCInstBuilder(X86::CALL64pcrel32)
+              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));
     }
   } else {
-    if (Specifier == X86MCExpr::VK_TLSGD && !UseGot) {
+    if (Specifier == X86::S_TLSGD && !UseGot) {
       EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
                                   .addReg(X86::EAX)
                                   .addReg(0)
@@ -638,8 +638,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 
     const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
     if (UseGot) {
-      const MCExpr *Expr =
-          MCSymbolRefExpr::create(TlsGetAddr, X86MCExpr::VK_GOT, Ctx);
+      const MCExpr *Expr = MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOT, Ctx);
       EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
                                   .addReg(X86::EBX)
                                   .addImm(1)
@@ -647,9 +646,9 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
                                   .addExpr(Expr)
                                   .addReg(0));
     } else {
-      EmitAndCountInstruction(MCInstBuilder(X86::CALLpcrel32)
-                                  .addExpr(MCSymbolRefExpr::create(
-                                      TlsGetAddr, X86MCExpr::VK_PLT, Ctx)));
+      EmitAndCountInstruction(
+          MCInstBuilder(X86::CALLpcrel32)
+              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));
     }
   }
 }
diff --git a/llvm/lib/Target/X86/X86PfmCounters.td b/llvm/lib/Target/X86/X86PfmCounters.td
index b31ed81160a2..b29963344641 100644
--- a/llvm/lib/Target/X86/X86PfmCounters.td
+++ b/llvm/lib/Target/X86/X86PfmCounters.td
@@ -236,6 +236,26 @@ def SapphireRapidsPfmCounters : ProcPfmCounters {
 }
 def : PfmCountersBinding<"sapphirerapids", SapphireRapidsPfmCounters>;
 
+def LunarLakePfmCounters : ProcPfmCounters {
+  let CycleCounter = UnhaltedCoreCyclesPfmCounter;
+  let UopsCounter = UopsIssuedPfmCounter;
+  let IssueCounters = [
+    // Refer: https://perfmon-events.intel.com/ section Lunar Lake Hybrid Event
+    // ALU Dispatch - Any of ALUs with latency 1 cycle that is not jmp or Shift.
+    PfmIssueCounter<"LNLPVPort02_03", "uops_dispatched:alu">,
+    PfmIssueCounter<"LNLPPort00_01_02_03_04_05", "uops_dispatched:int_eu_all">,
+    PfmIssueCounter<"LNLPPort00_02_04", "uops_dispatched:jmp">,
+    PfmIssueCounter<"LNLPPort20_21_22", "uops_dispatched:load">,
+    PfmIssueCounter<"LNLPPort01_03_05", "uops_dispatched:shift">,
+    // Slow Dispatch - If uops latency > 1, counted as slow. TBD
+    // PfmIssueCounter<"LNLPPort01_03_05", "uops_dispatched:slow">,
+    PfmIssueCounter<"LNLPPort25_26_27", "uops_dispatched:sta">,
+    PfmIssueCounter<"LNLPPort10_11", "uops_dispatched:std">
+  ];
+  let ValidationCounters = DefaultIntelPfmValidationCounters;
+}
+def : PfmCountersBinding<"lunarlake", LunarLakePfmCounters>;
+
 // AMD X86 Counters.
 defvar DefaultAMDPfmValidationCounters = [
   PfmValidationCounter<InstructionRetired, "RETIRED_INSTRUCTIONS">,
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index f3800d360fdd..3b4e531f2538 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -80,12 +80,12 @@ struct MIRef {
   bool operator<(const MIRef &RHS) const {
     // Comparison between different BBs happens when inserting a MIRef into set.
     // So we compare MBB first to make the insertion happy.
-    return MBB < RHS.MBB || (MBB == RHS.MBB && Pos < RHS.Pos);
+    return std::tie(MBB, Pos) < std::tie(RHS.MBB, RHS.Pos);
   }
   bool operator>(const MIRef &RHS) const {
     // Comparison between different BBs happens when inserting a MIRef into set.
     // So we compare MBB first to make the insertion happy.
-    return MBB > RHS.MBB || (MBB == RHS.MBB && Pos > RHS.Pos);
+    return std::tie(MBB, Pos) > std::tie(RHS.MBB, RHS.Pos);
   }
 };
 
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 9adac067aace..83b11eede829 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -562,7 +562,7 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(SubReg);
 
   // Set the frame-pointer register and its aliases as reserved if needed.
-  if (TFI->hasFP(MF)) {
+  if (TFI->hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF)) {
     if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
       MF.getContext().reportError(
           SMLoc(),
diff --git a/llvm/lib/Target/X86/X86SchedLunarlakeP.td b/llvm/lib/Target/X86/X86SchedLunarlakeP.td
new file mode 100644
index 000000000000..32979c637c12
--- /dev/null
+++ b/llvm/lib/Target/X86/X86SchedLunarlakeP.td
@@ -0,0 +1,2409 @@
+//=----- X86SchedLunarlakeP.td - X86 LunarlakeP Scheduling *- tablegen -----*=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the machine model for LunarlakeP to support instruction
+// scheduling and other instruction cost heuristics.
+//
+//===----------------------------------------------------------------------===//
+def LunarlakePModel : SchedMachineModel {
+  // LunarlakeP can allocate 8 uops per cycle.
+  // Max micro-ops that may be scheduled per cycle.
+  // Based on Allocator Width
+  let IssueWidth = 8;  
+  // Max micro-ops that can be buffered. 
+  // Based on size of ROB
+  let MicroOpBufferSize = 792;
+  // INT LOAD takes 4 cycles
+  let LoadLatency = 4;
+  let MispredictPenalty = 14;
+  // Latency for microcoded instructions or instructions without latency info.
+  int MaxLatency = 100;
+  // Based on the LSD (loop-stream detector) queue size (ST).
+  // LSD is 200 uops per logical processor in single threaded mode
+  // For SMT 100 uops/thread, LionCove removed SMT in HW.
+  let LoopMicroOpBufferSize = 200;
+  // This flag is set to allow the scheduler to assign a default model to
+  // unrecognized opcodes.
+  let CompleteModel = 0;
+}
+
+let SchedModel = LunarlakePModel in {
+
+// LunarlakeP can issue micro-ops to 18 different ports in one cycle.
+// Lion Cove architectural spec uses port naming that is not sequential
+// for better comprehension we opt for sequential naming since this ports
+// serve logical information for schedule only.
+// 6 INT ALU Ports {P0 to P5}
+def LNLPPort00 : ProcResource<1>;
+def LNLPPort01 : ProcResource<1>;
+def LNLPPort02 : ProcResource<1>;
+def LNLPPort03 : ProcResource<1>;
+def LNLPPort04 : ProcResource<1>;
+def LNLPPort05 : ProcResource<1>;
+// 4 VEC ALU Ports {V0 to V3}
+def LNLPVPort00 : ProcResource<1>;
+def LNLPVPort01 : ProcResource<1>;
+def LNLPVPort02 : ProcResource<1>;
+def LNLPVPort03 : ProcResource<1>;
+// 2 Store Data Ports {P10 to P11}
+def LNLPPort10 : ProcResource<1>;
+def LNLPPort11 : ProcResource<1>;
+// 6 MEM Ports 6 AGU shared with 3 LD, 3 ST
+// AGU LD {P20 to P22}
+def LNLPPort20 : ProcResource<1>;
+def LNLPPort21 : ProcResource<1>;
+def LNLPPort22 : ProcResource<1>;
+// AGU ST {P25 to P27}
+def LNLPPort25 : ProcResource<1>;
+def LNLPPort26 : ProcResource<1>;
+def LNLPPort27 : ProcResource<1>;
+
+// Workaround to represent invalid ports. WriteRes shouldn't use this resource.
+def LNLPPortInvalid :ProcResource<1>;
+
+// Many micro-ops are capable of issuing on multiple ports.
+def LNLPVPort00_01          : ProcResGroup<[LNLPVPort00, LNLPVPort01]>;
+def LNLPVPort02_03          : ProcResGroup<[LNLPVPort02, LNLPVPort03]>;
+def LNLPPort00_02_04        : ProcResGroup<[LNLPPort00, LNLPPort02, LNLPPort04]>;
+def LNLPPort01_03_05        : ProcResGroup<[LNLPPort01, LNLPPort03, LNLPPort05]>;
+def LNLPPort20_21_22        : ProcResGroup<[LNLPPort20, LNLPPort21, LNLPPort22]>;
+def LNLPPort25_26_27        : ProcResGroup<[LNLPPort25, LNLPPort26, LNLPPort27]>;
+
+// INT EU has 112 reservation stations.
+def LNLPPort00_01_02_03_04_05 : ProcResGroup<[LNLPPort00, LNLPPort01, LNLPPort02,
+                                             LNLPPort03, LNLPPort04, LNLPPort05]>{ 
+  let BufferSize = 110;  // Reduced from 128 in GLC
+}
+
+// VEC EU has 180 reservation stations.
+def LNLPVPort00_01_02_03    : ProcResGroup<[LNLPVPort00, LNLPVPort01, LNLPVPort02,
+                                          LNLPVPort03]>{
+  let BufferSize = 180; // EU for INT and VEC are seperated 
+                        // VEC QUEUE SIZE = 60 + VEC EU RS (60+60)
+}
+// STD has 48 reservation stations.
+def LNLPPort10_11          : ProcResGroup<[LNLPPort10, LNLPPort11]> {
+  let BufferSize = 48;
+}
+
+// MEM has 72 reservation stations.
+def LNLPPort20_21_22_25_26_27 : ProcResGroup<[LNLPPort20, LNLPPort21, LNLPPort22,
+                                            LNLPPort25, LNLPPort26, LNLPPort27]> {
+  let BufferSize = 72;
+}
+
+def LNLPPortAny : ProcResGroup<[LNLPPort00, LNLPPort01, LNLPPort02, LNLPPort03,
+                               LNLPPort04, LNLPPort05, LNLPVPort00, LNLPVPort01,
+                               LNLPVPort02, LNLPVPort03, LNLPPort10, LNLPPort11,
+                               LNLPPort20, LNLPPort21, LNLPPort22, LNLPPort25,
+                               LNLPPort26, LNLPPort27]>;
+
+// Integer loads are 4 cycles, so ReadAfterLd registers needn't be available
+// until 4 cycles after the memory operand.
+def : ReadAdvance<ReadAfterLd, 4>;
+
+// TODO: 6 Cycle latency for Vec load comes from ADL
+// Vector loads are 6 cycles, so ReadAfterVec*Ld registers needn't be available
+// until 6 cycles after the memory operand.
+def : ReadAdvance<ReadAfterVecLd, 6>;
+def : ReadAdvance<ReadAfterVecXLd, 6>;
+def : ReadAdvance<ReadAfterVecYLd, 6>;
+
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
+// Many SchedWrites are defined in pairs with and without a folded load.
+// Instructions with folded loads are usually micro-fused, so they only appear
+// as two micro-ops when queued in the reservation station.
+// This multiclass defines the resource usage for variants with and without
+// folded loads.
+multiclass LNLPWriteResPair<X86FoldableSchedWrite SchedRW,
+                           list<ProcResourceKind> ExePorts,
+                           int Lat, list<int> Res = [1], int UOps = 1,
+                           int LoadLat = 4, int LoadUOps = 1> {
+  // Register variant is using a single cycle on ExePort.
+  def : WriteRes<SchedRW, ExePorts> {
+    let Latency = Lat;
+    let ReleaseAtCycles = Res;
+    let NumMicroOps = UOps;
+  }
+
+  // Memory variant also uses a cycle on port 20/21/22 and adds LoadLat cycles to
+  // the latency (default = 4).
+  def : WriteRes<SchedRW.Folded, !listconcat([LNLPPort20_21_22], ExePorts)> {
+    let Latency = !add(Lat, LoadLat);
+    let ReleaseAtCycles = !listconcat([1], Res);
+    let NumMicroOps = !add(UOps, LoadUOps);
+  }
+}
+
+defm : X86WriteResUnsupported<WriteBEXTRLd>;
+
+//===----------------------------------------------------------------------===//
+// The following definitons are infered by smg.
+//===----------------------------------------------------------------------===//
+
+def : WriteRes<WriteADC, [LNLPPort00_02_04]>;
+defm : X86WriteRes<WriteADCLd, [LNLPPort00_01_02_03_04_05, LNLPPort00_02_04], 11, [1, 1], 2>;
+def : WriteRes<WriteAESDecEnc, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 3;
+}
+defm : X86WriteRes<WriteAESDecEncLd, [LNLPVPort00_01, LNLPPort20_21_22], 11, [4, 7], 2>;
+defm : X86WriteResPairUnsupported<WriteAESIMC>;
+defm : X86WriteResPairUnsupported<WriteAESKeyGen>;
+def  : WriteRes<WriteALU, [LNLPPort01_03_05]>;
+def  : WriteRes<WriteALULd, [LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let Latency = 11;
+}
+defm : X86WriteRes<WriteBEXTR, [LNLPPort01_03_05], 6, [2], 2>;
+def  : WriteRes<WriteBLS, [LNLPPort01_03_05]>;
+defm : X86WriteRes<WriteBLSLd, [LNLPPort01_03_05, LNLPPort20_21_22], 5, [1, 4], 2>;
+defm : LNLPWriteResPair<WriteBSF, [LNLPPort01_03_05], 3, [1]>;
+defm : LNLPWriteResPair<WriteBSR, [LNLPPort01_03_05], 3, [1]>;
+def  : WriteRes<WriteBSWAP32, [LNLPPort01_03_05]>;
+defm : X86WriteRes<WriteBSWAP64, [LNLPPort01_03_05, LNLPPort01_03_05], 2, [1, 1], 2>;
+defm : LNLPWriteResPair<WriteBZHI, [LNLPPort01_03_05], 3, [1]>;
+def : WriteRes<WriteBitTestSet, [LNLPPort01_03_05]>;
+def : WriteRes<WriteBitTestSetImmLd, [LNLPPort01_03_05]> {
+  let Latency = 11;
+}
+defm : X86WriteRes<WriteBitTestSetRegLd, [LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort00_02_04, LNLPPort01_03_05], 17, [3, 2, 1, 2], 8>;
+def : WriteRes<WriteBitTest, [LNLPPort01_03_05]>;
+defm : X86WriteRes<WriteBitTestImmLd, [LNLPPort01_03_05, LNLPPort20_21_22], 6, [1, 1], 2>;
+defm : X86WriteRes<WriteBitTestRegLd, [LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22], 11, [4, 2, 1, 2, 1], 10>;
+
+def  : WriteRes<WriteBlend, [LNLPVPort00_01_02_03]>;
+defm : X86WriteRes<WriteBlendLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 7, [1, 6], 2>;
+def  : WriteRes<WriteBlendY, [LNLPVPort00_01_02_03]>;
+defm : X86WriteRes<WriteBlendYLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 8, [1, 7], 2>;
+
+defm : LNLPWriteResPair<WriteCLMul, [LNLPVPort02_03], 3, [1], 1, 7>;
+def : WriteRes<WriteCMOV, [LNLPPort00_01_02_03_04_05]>;
+def : WriteRes<WriteCMOVLd, [LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 4];
+  let Latency = 5;
+}
+defm : X86WriteRes<WriteCMPXCHG, [LNLPPort00_01_02_03_04_05, LNLPPort00_02_04], 3, [3, 2], 5>;
+defm : X86WriteRes<WriteCMPXCHGRMW, [LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort20_21_22], 12, [1, 2, 1, 1, 1], 6>;
+
+def  : WriteRes<WriteCRC32, [LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def  : WriteRes<WriteCRC32Ld, [LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 4];
+  let Latency = 7;
+}
+
+defm : X86WriteRes<WriteCvtI2PD, [LNLPVPort00_01, LNLPVPort02_03], 5, [1, 1], 2>;
+defm : X86WriteRes<WriteCvtI2PDLd, [LNLPVPort00_01, LNLPPort20_21_22], 11, [1,1], 2>; 
+defm : X86WriteRes<WriteCvtI2PDY, [LNLPVPort00_01, LNLPVPort02_03], 7, [4, 3], 2>;
+def  : WriteRes<WriteCvtI2PDYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteCvtI2PDZ>;
+def  : WriteRes<WriteCvtI2PS, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteCvtI2PSLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+defm : LNLPWriteResPair<WriteCvtI2PSY, [LNLPVPort00_01], 4, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteCvtI2PSZ>;
+defm : X86WriteRes<WriteCvtI2SD, [LNLPPort01_03_05, LNLPVPort00_01], 8, [4, 4], 2>;
+def  : WriteRes<WriteCvtI2SDLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+defm : X86WriteRes<WriteCvtI2SS, [LNLPPort01_03_05, LNLPVPort00_01], 8, [4, 4], 2>;
+def  : WriteRes<WriteCvtI2SSLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+
+defm : LNLPWriteResPair<WriteCvtPD2I, [LNLPVPort00_01, LNLPVPort02_03], 5, [1, 1], 2, 7>;
+defm : LNLPWriteResPair<WriteCvtPD2IY, [LNLPVPort00_01, LNLPVPort02_03], 7, [1, 1], 2, 8>;
+defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>;
+defm : LNLPWriteResPair<WriteCvtPD2PS, [LNLPVPort00_01, LNLPVPort02_03], 5, [1, 1], 2, 7>;
+defm : LNLPWriteResPair<WriteCvtPD2PSY, [LNLPVPort00_01, LNLPVPort02_03], 7, [1, 1], 2, 8>;
+defm : X86WriteResPairUnsupported<WriteCvtPD2PSZ>;
+defm : X86WriteRes<WriteCvtPH2PS, [LNLPVPort00_01, LNLPVPort02_03], 5, [1, 1], 2>;
+defm : X86WriteRes<WriteCvtPH2PSLd, [LNLPVPort00_01, LNLPPort20_21_22], 12, [1, 1], 2>;
+defm : X86WriteRes<WriteCvtPH2PSY, [LNLPVPort00_01, LNLPVPort02_03], 7, [4, 3], 2>;
+def  : WriteRes<WriteCvtPH2PSYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteCvtPH2PSZ>;
+def  : WriteRes<WriteCvtPS2I, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+defm : X86WriteRes<WriteCvtPS2ILd, [LNLPVPort00_01, LNLPPort20_21_22], 10, [4, 6], 2>;
+def   : WriteRes<WriteCvtPS2IY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteCvtPS2IYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteCvtPS2IZ>;
+defm : X86WriteRes<WriteCvtPS2PD, [LNLPVPort00_01, LNLPVPort02_03], 5, [4, 1], 2>;
+def  : WriteRes<WriteCvtPS2PDLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+defm : X86WriteRes<WriteCvtPS2PDY, [LNLPVPort00_01, LNLPVPort02_03], 7, [4, 3], 2>;
+def  : WriteRes<WriteCvtPS2PDYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteCvtPS2PDZ>;
+defm : X86WriteRes<WriteCvtPS2PH, [LNLPVPort00_01, LNLPVPort02_03], 5, [4, 1], 2>;
+defm : X86WriteRes<WriteCvtPS2PHSt, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 5, [4, 1, 1], 2>;
+defm : X86WriteRes<WriteCvtPS2PHY, [LNLPVPort00_01, LNLPVPort02_03], 7, [4, 3], 2>;
+defm : X86WriteRes<WriteCvtPS2PHYSt, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 5, [4, 1, 1], 3>;
+defm : X86WriteResUnsupported<WriteCvtPS2PHZ>;
+defm : X86WriteResUnsupported<WriteCvtPS2PHZSt>;
+defm : X86WriteRes<WriteCvtSD2I, [LNLPVPort00_01], 7, [7], 2>;
+defm : X86WriteRes<WriteCvtSD2ILd, [LNLPVPort00_01, LNLPPort20_21_22], 11, [7, 4], 3>;
+defm : X86WriteRes<WriteCvtSD2SS, [LNLPVPort00_01, LNLPVPort02_03], 5, [1, 1], 2>;
+defm : X86WriteRes<WriteCvtSD2SSLd, [LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22], 11, [4, 1, 6], 3>;
+defm : X86WriteRes<WriteCvtSS2I, [LNLPVPort00_01], 7, [7], 2>;
+defm : X86WriteRes<WriteCvtSS2ILd, [LNLPVPort00_01, LNLPPort20_21_22], 11, [7, 4], 2>;
+defm : X86WriteRes<WriteCvtSS2SD, [LNLPVPort00_01, LNLPVPort02_03], 5, [4, 1], 2>;
+def  : WriteRes<WriteCvtSS2SDLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+defm : X86WriteRes<WriteDPPD, [LNLPVPort00_01, LNLPVPort02_03], 8, [2, 1], 3>;
+defm : X86WriteRes<WriteDPPDLd, [LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22], 13, [2, 1, 1], 4>;
+//defm : X86WriteRes<WriteDPPS, [LNLPVPort00_01, LNLPVPort02_03], 99, [7, 5], 5>;
+// FIXME: Incompleted schedwrite.
+//defm : X86WriteResUnsupported<WriteDPPSLd>;
+defm : LNLPWriteResPair<WriteDPPS, [LNLPVPort00_01, LNLPVPort02_03, LNLPVPort02_03, LNLPVPort02_03, LNLPVPort00_01], 11, [2, 1, 1, 1, 1], 5, 7>;
+//defm : X86WriteRes<WriteDPPSY, [LNLPVPort00_01, LNLPVPort02_03], 12, [7, 5], 5>;
+// FIXME: Incompleted schedwrite.
+//defm : X86WriteResUnsupported<WriteDPPSYLd>;
+defm : LNLPWriteResPair<WriteDPPSY, [LNLPVPort00_01, LNLPVPort02_03, LNLPVPort02_03, LNLPVPort02_03, LNLPVPort00_01], 12, [2, 1, 1, 1, 1], 5, 7>;
+
+defm : LNLPWriteResPair<WriteDiv16, [LNLPPort00_01_02_03_04_05, LNLPPort01], 16, [1, 3], 4, 4>;
+defm : LNLPWriteResPair<WriteDiv32, [LNLPPort00_01_02_03_04_05, LNLPPort01], 15, [1, 3], 4, 4>;
+defm : LNLPWriteResPair<WriteDiv64, [LNLPPort01], 18, [3], 3>;
+defm : X86WriteRes<WriteDiv8, [LNLPPort01], 17, [3], 3>;
+defm : X86WriteRes<WriteDiv8Ld, [LNLPPort01], 22, [3], 3>;
+defm : X86WriteRes<WriteEMMS, [LNLPPort01, LNLPVPort00_01, LNLPVPort02_03], 10, [1, 8, 1], 10>;
+def  : WriteRes<WriteFAdd, [LNLPVPort02_03]> {
+  let Latency = 2;
+}
+// FIXME: Latency
+defm : X86WriteRes<WriteFAddLd, [LNLPVPort02_03, LNLPPort20_21_22], 10, [1,1], 2>;  // 8
+defm : LNLPWriteResPair<WriteFAdd64, [LNLPVPort02_03], 3, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFAdd64X, [LNLPVPort02_03], 3, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFAdd64Y, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteFAdd64Z>;
+defm : LNLPWriteResPair<WriteFAddX, [LNLPVPort02_03], 3, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFAddY, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteFAddZ>;
+def  : WriteRes<WriteFBlend, [LNLPVPort00_01_02_03]>;
+defm : X86WriteRes<WriteFBlendLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 7, [1, 6], 2>;
+def  : WriteRes<WriteFBlendY, [LNLPVPort00_01_02_03]>;
+defm : X86WriteRes<WriteFBlendYLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 8, [1, 7], 2>;
+def : WriteRes<WriteFCMOV, [LNLPVPort00_01]> {
+  let Latency = 3;
+}
+def : WriteRes<WriteFCmp, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmpLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFCmp64, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmp64Ld, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFCmp64X, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmp64XLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFCmp64Y, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmp64YLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteFCmp64Z>;
+def : WriteRes<WriteFCmpX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmpXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFCmpY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFCmpYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteFCmpZ>;
+def : WriteRes<WriteFCom, [LNLPVPort00_01]>;
+defm : X86WriteRes<WriteFComLd, [LNLPVPort00_01, LNLPPort20_21_22], 8, [1, 1], 2>;
+def  : WriteRes<WriteFComX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+defm : X86WriteRes<WriteFComXLd, [LNLPVPort00_01, LNLPPort20_21_22], 9, [3, 6], 2>;
+def  : WriteRes<WriteFDiv, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [7];
+  let Latency = 7;
+}
+def : WriteRes<WriteFDivLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7, 6];
+  let Latency = 13;
+}
+def : WriteRes<WriteFDiv64, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFDiv64Ld, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 6];
+  let Latency = 16;
+}
+def : WriteRes<WriteFDiv64X, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFDiv64XLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 6];
+  let Latency = 16;
+}
+def : WriteRes<WriteFDiv64Y, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFDiv64YLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 7];
+  let Latency = 17;
+}
+defm : X86WriteResPairUnsupported<WriteFDiv64Z>;
+def : WriteRes<WriteFDivX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [7];
+  let Latency = 7;
+}
+def : WriteRes<WriteFDivXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7, 6];
+  let Latency = 13;
+}
+def : WriteRes<WriteFDivY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [7];
+  let Latency = 7;
+}
+def : WriteRes<WriteFDivYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7, 7];
+  let Latency = 14;
+}
+defm : X86WriteResPairUnsupported<WriteFDivZ>;
+defm : X86WriteRes<WriteFHAdd, [LNLPVPort02_03], 4, [5], 3>;
+defm : X86WriteRes<WriteFHAddLd, [LNLPVPort02_03, LNLPPort20_21_22], 10, [5, 6], 4>;
+defm : X86WriteRes<WriteFHAddY, [LNLPVPort02_03], 4, [5], 3>;
+defm : X86WriteRes<WriteFHAddYLd, [LNLPVPort02_03, LNLPPort20_21_22], 11, [5, 7], 4>;
+def : WriteRes<WriteFLD0, [LNLPVPort02_03]>;
+defm : X86WriteRes<WriteFLD1, [LNLPVPort02_03], 1, [2], 2>;
+defm : X86WriteRes<WriteFLDC, [LNLPVPort02_03], 1, [2], 2>;
+
+def : WriteRes<WriteFLoad, [LNLPPort20_21_22]> {
+  let Latency = 7;
+}
+def : WriteRes<WriteFLoadX, [LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [6];
+  let Latency = 7;
+}
+def : WriteRes<WriteFLoadY, [LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7];
+  let Latency = 8;
+}
+def : WriteRes<WriteFLogic, [LNLPVPort00_01_02_03]>;
+def : WriteRes<WriteFLogicLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+}
+def : WriteRes<WriteFLogicY, [LNLPVPort00_01_02_03]>;
+def : WriteRes<WriteFLogicYLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+defm : X86WriteResPairUnsupported<WriteFLogicZ>;
+def : WriteRes<WriteFMA, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFMALd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFMAX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFMAXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFMAY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFMAYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteFMAZ>;
+def : WriteRes<WriteFMOVMSK, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+
+defm : X86WriteRes<WriteFMaskedLoad, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 8, [1, 1], 2>;
+defm : X86WriteRes<WriteFMaskedLoadY, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 9, [1, 1], 2>;
+defm : X86WriteRes<WriteFMaskedStore32, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteFMaskedStore32Y, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteFMaskedStore64, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteFMaskedStore64Y, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteFMoveX, [], 1, [], 0>;
+defm : X86WriteRes<WriteFMoveY, [], 1, [], 0>;
+defm : X86WriteResUnsupported<WriteFMoveZ>;
+def : WriteRes<WriteFMul, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMulLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+}
+def : WriteRes<WriteFMul64, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMul64Ld, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+}
+def : WriteRes<WriteFMul64X, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMul64XLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+}
+def : WriteRes<WriteFMul64Y, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMul64YLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+}
+defm : X86WriteResPairUnsupported<WriteFMul64Z>;
+def : WriteRes<WriteFMulX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMulXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+}
+def : WriteRes<WriteFMulY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteFMulYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+}
+defm : X86WriteResPairUnsupported<WriteFMulZ>;
+def : WriteRes<WriteFRcp, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFRcpLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFRcpX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFRcpXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteFRcpY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteFRcpYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteFRcpZ>;
+defm : X86WriteRes<WriteFRnd, [LNLPVPort00_01], 8, [8], 2>;
+defm : X86WriteRes<WriteFRndLd, [LNLPVPort00_01, LNLPPort20_21_22], 14, [8, 6], 3>;
+defm : X86WriteRes<WriteFRndY, [LNLPVPort00_01], 8, [8], 2>;
+defm : X86WriteRes<WriteFRndYLd, [LNLPVPort00_01, LNLPPort20_21_22], 15, [8, 7], 3>;
+defm : X86WriteResPairUnsupported<WriteFRndZ>;
+def  : WriteRes<WriteFRsqrt, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def  : WriteRes<WriteFRsqrtLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def  : WriteRes<WriteFRsqrtX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def  : WriteRes<WriteFRsqrtXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def  : WriteRes<WriteFRsqrtY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def  : WriteRes<WriteFRsqrtYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteFRsqrtZ>;
+
+defm : LNLPWriteResPair<WriteFShuffle, [LNLPVPort02_03], 1, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFShuffle256, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : LNLPWriteResPair<WriteFShuffleY, [LNLPVPort02_03], 1, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteFShuffleZ>;
+def : WriteRes<WriteFSign, [LNLPVPort00_01]>;
+def : WriteRes<WriteFSqrt, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFSqrtLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 6];
+  let Latency = 16;
+}
+def : WriteRes<WriteFSqrt64, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [15];
+  let Latency = 15;
+}
+def : WriteRes<WriteFSqrt64Ld, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [15, 6];
+  let Latency = 21;
+}
+def : WriteRes<WriteFSqrt64X, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [15];
+  let Latency = 15;
+}
+def : WriteRes<WriteFSqrt64XLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [15, 6];
+  let Latency = 21;
+}
+def : WriteRes<WriteFSqrt64Y, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [15];
+  let Latency = 15;
+}
+def : WriteRes<WriteFSqrt64YLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [15, 7];
+  let Latency = 22;
+}
+defm : X86WriteResPairUnsupported<WriteFSqrt64Z>;
+def : WriteRes<WriteFSqrt80, [LNLPPortInvalid, LNLPVPort00_01]> {
+  let ReleaseAtCycles = [7, 1];
+  let Latency = 21;
+}
+def : WriteRes<WriteFSqrtX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFSqrtXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 6];
+  let Latency = 16;
+}
+def : WriteRes<WriteFSqrtY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [10];
+  let Latency = 10;
+}
+def : WriteRes<WriteFSqrtYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [10, 7];
+  let Latency = 17;
+}
+defm : X86WriteResPairUnsupported<WriteFSqrtZ>;
+defm : X86WriteRes<WriteFStore, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : X86WriteResUnsupported<WriteFStoreNT>;
+defm : X86WriteRes<WriteFStoreNTX, [LNLPPort10_11, LNLPPort25_26_27], 518, [1, 1], 2>;
+defm : X86WriteRes<WriteFStoreNTY, [LNLPPort10_11, LNLPPort25_26_27], 542, [1, 1], 2>;
+defm : X86WriteRes<WriteFStoreX, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : X86WriteRes<WriteFStoreY, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : LNLPWriteResPair<WriteFTest, [LNLPVPort00_01], 3, [1]>;
+defm : LNLPWriteResPair<WriteFTestY, [LNLPVPort00_01], 5, [1], 1, 6>;
+defm : LNLPWriteResPair<WriteFVarBlend, [LNLPVPort00_01_02_03], 1, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFVarBlendY, [LNLPVPort00_01_02_03], 3, [3], 3, 7>;
+defm : X86WriteResPairUnsupported<WriteFVarBlendZ>;
+defm : LNLPWriteResPair<WriteFVarShuffle, [LNLPVPort02_03], 1, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteFVarShuffle256, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : LNLPWriteResPair<WriteFVarShuffleY, [LNLPVPort02_03], 1, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteFVarShuffleZ>;
+defm : X86WriteRes<WriteFence, [LNLPPort10_11, LNLPPort25_26_27], 2, [1, 1], 2>;
+defm : LNLPWriteResPair<WriteIDiv16, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05], 16, [1, 3], 4, 4>;
+defm : LNLPWriteResPair<WriteIDiv32, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05], 15, [1, 3], 4, 4>;
+defm : LNLPWriteResPair<WriteIDiv64, [LNLPPort01_03_05], 18, [3], 3>;
+defm : X86WriteRes<WriteIDiv8, [LNLPPort01_03_05], 17, [3], 3>;
+defm : X86WriteRes<WriteIDiv8Ld, [LNLPPort01_03_05], 22, [3], 3>;
+defm : LNLPWriteResPair<WriteIMul16, [LNLPPort01_03_05, LNLPPort00_02_04], 5, [2, 1], 4>;
+defm : LNLPWriteResPair<WriteIMul16Imm, [LNLPPort01_03_05, LNLPPort00_01_02_03_04_05], 4, [1, 1], 2>;
+defm : LNLPWriteResPair<WriteIMul16Reg, [LNLPPort01_03_05], 3, [1]>;
+defm : LNLPWriteResPair<WriteIMul32, [LNLPPort01_03_05, LNLPPort00_02_04], 4, [1, 1], 3>;
+defm : LNLPWriteResPair<WriteIMul32Imm, [LNLPPort01_03_05, LNLPPort00_01_02_03_04_05], 2, [1, 1]>;
+defm : LNLPWriteResPair<WriteIMul32Reg, [LNLPPort01_03_05], 3, [1]>;
+defm : LNLPWriteResPair<WriteIMul64, [LNLPPort01_03_05, LNLPPort00_02_04], 4, [1, 1], 2>;
+defm : LNLPWriteResPair<WriteIMul64Imm, [LNLPPort01_03_05, LNLPPort00_01_02_03_04_05], 2, [1, 1]>;
+defm : LNLPWriteResPair<WriteIMul64Reg, [LNLPPort01_03_05], 3, [1]>;
+defm : LNLPWriteResPair<WriteIMul8, [LNLPPort01_03_05], 3, [1]>;
+def : WriteRes<WriteIMulH, []> {
+  let Latency = 3;  // 4
+}
+def : WriteRes<WriteIMulHLd, []> {
+  let Latency = 3;
+}
+def  : WriteRes<WriteJump, [LNLPPort00_02_04]>;
+defm : X86WriteRes<WriteJumpLd, [LNLPPort00_02_04, LNLPPort20_21_22], 6, [1, 1], 2>;
+defm : X86WriteResUnsupported<WriteLAHFSAHF>;
+defm : X86WriteRes<WriteLDMXCSR, [LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPVPort02_03, LNLPPort25_26_27], 7, [1, 1, 1, 1], 4>;
+def  : WriteRes<WriteLEA, [LNLPPort01]>;
+defm : LNLPWriteResPair<WriteLZCNT, [LNLPPort01], 3, [1]>;
+def  : WriteRes<WriteLoad, [LNLPPort20_21_22]> {
+  let Latency = 4;
+}
+def  : WriteRes<WriteMMXMOVMSK, [LNLPVPort00_01]> {
+  let Latency = 3 ;
+}
+defm : LNLPWriteResPair<WriteMPSAD, [LNLPVPort02_03, LNLPVPort02_03], 4, [1, 1], 2, 7>;
+defm : LNLPWriteResPair<WriteMPSADY, [LNLPVPort02_03, LNLPVPort02_03], 4, [1, 1], 2, 8>;
+defm : LNLPWriteResPair<WriteMULX32, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05], 4, [1, 1, 1], 2>;
+defm : LNLPWriteResPair<WriteMULX64, [LNLPPort01_03_05, LNLPPort01_03_05], 4, [1, 1]>;
+// FIXME: Incompleted schedwrite.
+def  : WriteRes<WriteMicrocoded, [LNLPPort00_01_02_03_04_05]> {
+  let Latency = LunarlakePModel.MaxLatency;
+}
+def  : WriteRes<WriteMove, [LNLPPort00_01_02_03_04_05]>;
+defm : X86WriteRes<WriteNop, [], 1, [], 0>;
+defm : X86WriteRes<WritePCmpEStrI, [LNLPPort00, LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort01_03_05], 16, [3, 2, 1, 1, 1], 8>;
+defm : X86WriteRes<WritePCmpEStrILd, [LNLPPort00, LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort01_03_05], 31, [3, 1, 1, 1, 1, 1], 8>;
+defm : X86WriteRes<WritePCmpEStrM, [LNLPPort00, LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05, LNLPPort01_03_05], 16, [3, 3, 1, 1, 1], 9>;
+defm : X86WriteRes<WritePCmpEStrMLd, [LNLPPort00, LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort01_03_05], 17, [3, 2, 1, 1, 1, 1], 9>;
+defm : LNLPWriteResPair<WritePCmpIStrI, [LNLPPort00], 11, [3], 3, 20>;
+defm : LNLPWriteResPair<WritePCmpIStrM, [LNLPPort00], 11, [3], 3>;
+defm : LNLPWriteResPair<WritePHAdd, [LNLPVPort00_01_02_03, LNLPVPort02_03], 3, [1, 2], 3, 8>;
+defm : LNLPWriteResPair<WritePHAddX, [LNLPVPort00_01_02_03, LNLPVPort02_03], 2, [1, 2], 3, 7>;
+defm : LNLPWriteResPair<WritePHAddY, [LNLPVPort00_01_02_03, LNLPVPort02_03], 2, [1, 2], 3, 8>;
+def : WriteRes<WritePHMINPOS, [LNLPVPort00_01]> {
+  let Latency = 5;
+}
+def : WriteRes<WritePHMINPOSLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let Latency = 11;
+}
+
+// FIXME : uops info is incorrect 
+defm : LNLPWriteResPair<WritePMULLD, [LNLPVPort00_01_02_03], 10, [2], 2, 8>;
+defm : LNLPWriteResPair<WritePMULLDY, [LNLPVPort00_01_02_03], 10, [2], 2, 8>; 
+defm : X86WriteResPairUnsupported<WritePMULLDZ>;
+defm : LNLPWriteResPair<WritePOPCNT, [LNLPPort01_03_05], 3, [1]>;
+defm : LNLPWriteResPair<WritePSADBW, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : LNLPWriteResPair<WritePSADBWX, [LNLPVPort02_03], 3, [1], 1, 7>;
+defm : LNLPWriteResPair<WritePSADBWY, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WritePSADBWZ>;
+defm : X86WriteRes<WriteRMW, [LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27], 1, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteRotate, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05], 2, [1, 2], 3>;
+defm : X86WriteRes<WriteRotateLd, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05], 12, [1, 2], 3>;
+defm : X86WriteRes<WriteRotateCL, [LNLPPort01_03_05], 2, [2], 2>;
+defm : X86WriteRes<WriteRotateCLLd, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05], 19, [2, 3, 2], 7>;
+defm : X86WriteRes<WriteSETCC, [LNLPPort01_03_05], 2, [2], 2>;
+defm : X86WriteRes<WriteSETCCStore, [LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27], 13, [2, 1, 1], 4>;
+defm : X86WriteRes<WriteSHDmrcl, [LNLPPort00_01_02_03_04_05,LNLPPort01_03_05 , LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1, 1, 1, 1, 1], 6>;
+defm : X86WriteRes<WriteSHDmri, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1, 1, 1, 1], 5>;
+defm : X86WriteRes<WriteSHDrrcl, [LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort01_03_05], 5, [1, 1, 1], 3>;
+def  : WriteRes<WriteSHDrri, [LNLPPort01_03_05]> {
+  let Latency = 3;
+}
+defm : X86WriteRes<WriteSTMXCSR, [LNLPPort00_02_04, LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], LunarlakePModel.MaxLatency, [1, 6, 1, 1], 4>;
+def  : WriteRes<WriteShift, [LNLPPort01_03_05]>;
+defm : X86WriteRes<WriteShiftLd, [LNLPPort01_03_05, LNLPPort20_21_22], 5, [1, 4], 2>;
+defm : X86WriteRes<WriteShiftCL, [LNLPPort00_02_04], 2, [2], 2>;
+defm : X86WriteRes<WriteShiftCLLd, [LNLPPort00_02_04], 12, [2], 2>;
+defm : LNLPWriteResPair<WriteShuffle, [LNLPVPort02_03], 1, [1], 1, 8>;
+defm : LNLPWriteResPair<WriteShuffle256, [LNLPVPort02_03], 3, [1], 1, 8>;
+defm : LNLPWriteResPair<WriteShuffleX, [LNLPVPort02_03], 1, [1], 1, 7>;
+defm : LNLPWriteResPair<WriteShuffleY, [LNLPVPort02_03], 1, [1], 1, 8>;
+defm : X86WriteResPairUnsupported<WriteShuffleZ>;
+defm : X86WriteRes<WriteStore, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : X86WriteRes<WriteStoreNT, [LNLPPort10_11, LNLPPort25_26_27], 512, [1, 1], 2>;
+def : WriteRes<WriteSystem, [LNLPPort00_01_02_03_04_05]> {
+  let Latency = LunarlakePModel.MaxLatency;
+}
+def : WriteRes<WriteTZCNT, [LNLPPort01_03_05]> {
+  let Latency = 3;
+}
+def : WriteRes<WriteTZCNTLd, [LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 4];
+  let Latency = 7;
+}
+def : WriteRes<WriteVPMOV256, [LNLPVPort02_03]> {
+  let Latency = 3;
+}
+defm : X86WriteRes<WriteVPMOV256Ld, [LNLPVPort02_03, LNLPPort20_21_22], 10, [3, 7], 2>;
+defm : X86WriteRes<WriteVarBlend, [LNLPVPort00_01_02_03], 3, [3], 3>;
+defm : X86WriteRes<WriteVarBlendLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 8, [3, 6], 4>;
+defm : X86WriteRes<WriteVarBlendY, [LNLPVPort00_01_02_03], 3, [3], 3>;
+defm : X86WriteRes<WriteVarBlendYLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 9, [3, 7], 4>;
+defm : X86WriteResPairUnsupported<WriteVarBlendZ>;
+defm : LNLPWriteResPair<WriteVarShuffle, [LNLPVPort02_03, LNLPVPort00_01], 3, [1, 1], 2, 8>;
+def : WriteRes<WriteVarShuffle256, [LNLPVPort02_03]> {
+  let Latency = 3;  
+}
+def : WriteRes<WriteVarShuffle256Ld, [LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+}
+def : WriteRes<WriteVarShuffleX, [LNLPVPort02_03]>;
+defm : X86WriteRes<WriteVarShuffleXLd, [LNLPVPort02_03, LNLPPort20_21_22], 7, [1, 6], 2>;
+def : WriteRes<WriteVarShuffleY, [LNLPVPort02_03]>;
+defm : X86WriteRes<WriteVarShuffleYLd, [LNLPVPort02_03, LNLPPort20_21_22], 8, [1, 7], 2>;
+defm : X86WriteResPairUnsupported<WriteVarShuffleZ>;
+
+def : WriteRes<WriteVarVecShift, [LNLPVPort00_01]>;
+def : WriteRes<WriteVarVecShiftLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+}
+def : WriteRes<WriteVarVecShiftY, [LNLPVPort00_01]>;
+def : WriteRes<WriteVarVecShiftYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+defm : X86WriteResPairUnsupported<WriteVarVecShiftZ>;
+defm : LNLPWriteResPair<WriteVecALU, [LNLPVPort00], 1, [1], 1, 8>;  // 4 ports ?
+def : WriteRes<WriteVecALUX, [LNLPVPort00_01]>;
+def : WriteRes<WriteVecALUXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+}
+def : WriteRes<WriteVecALUY, [LNLPVPort00_01]>;
+def : WriteRes<WriteVecALUYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+defm : X86WriteResPairUnsupported<WriteVecALUZ>;
+defm : X86WriteRes<WriteVecExtract, [LNLPVPort00_01, LNLPVPort02_03], 4, [3, 1], 2>;
+defm : X86WriteRes<WriteVecExtractSt, [LNLPVPort02_03, LNLPPort10_11, LNLPPort25_26_27], 2, [1, 1, 1], 2>;
+defm : LNLPWriteResPair<WriteVecIMul, [LNLPVPort00_01], 5, [1], 1, 8>;
+def : WriteRes<WriteVecIMulX, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteVecIMulXLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : WriteRes<WriteVecIMulY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+}
+def : WriteRes<WriteVecIMulYLd, [LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+}
+defm : X86WriteResPairUnsupported<WriteVecIMulZ>;
+defm : X86WriteRes<WriteVecInsert, [LNLPPort01_03_05, LNLPVPort02_03], 5, [4, 1], 2>;
+defm : X86WriteRes<WriteVecInsertLd, [LNLPVPort02_03, LNLPPort20_21_22], 7, [1, 6], 2>;
+def  : WriteRes<WriteVecLoad, [LNLPPort20_21_22]> {
+  let Latency = 6;
+}
+// FIXME: Incompleted schedwrite.
+def : WriteRes<WriteVecLoadNT, [LNLPPort20_21_22]> {
+  let Latency = 7;
+}
+// FIXME: Incompleted schedwrite.
+def : WriteRes<WriteVecLoadNTY, [LNLPPort20_21_22]> {
+  let Latency = 8;
+}
+
+def : WriteRes<WriteVecLoadX, [LNLPPort20_21_22]> {
+  let Latency = 6;
+}
+def : WriteRes<WriteVecLoadY, [LNLPPort20_21_22]> {
+  let Latency = 7;
+}
+defm : LNLPWriteResPair<WriteVecLogic, [LNLPVPort00_01_02_03], 1, [1], 1, 8>;
+def : WriteRes<WriteVecLogicX, [LNLPVPort00_01_02_03]>;
+def : WriteRes<WriteVecLogicXLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+}
+def : WriteRes<WriteVecLogicY, [LNLPVPort00_01_02_03]>;
+def : WriteRes<WriteVecLogicYLd, [LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+defm : X86WriteResPairUnsupported<WriteVecLogicZ>;
+def : WriteRes<WriteVecMOVMSK, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : WriteRes<WriteVecMOVMSKY, [LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;  // Tool added Max
+}
+defm : X86WriteRes<WriteVecMaskedGatherWriteback, [], 5, [], 0>;
+defm : X86WriteRes<WriteVecMaskedLoad, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 7, [1, 6], 2>;
+defm : X86WriteRes<WriteVecMaskedLoadY, [LNLPVPort00_01_02_03, LNLPPort20_21_22], 8, [1, 7], 2>;
+defm : X86WriteRes<WriteVecMaskedStore32, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;  // updated lat from 3 to 14
+defm : X86WriteRes<WriteVecMaskedStore32Y, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteVecMaskedStore64, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+defm : X86WriteRes<WriteVecMaskedStore64Y, [LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27], 14, [1, 1, 1], 3>;
+def : WriteRes<WriteVecMove, [LNLPVPort00_01_02_03]>;
+def : WriteRes<WriteVecMoveFromGpr, [LNLPPort01_03_05]> {
+  let Latency = 3;  // Originally 4
+}
+def : WriteRes<WriteVecMoveToGpr, [LNLPVPort00_01]> {
+  let Latency = 3;
+}
+defm : X86WriteRes<WriteVecMoveX, [LNLPVPort00_01_02_03], 1, [2], 2>;
+defm : X86WriteRes<WriteVecMoveY, [], 1, [], 0>;
+defm : X86WriteResUnsupported<WriteVecMoveZ>;
+defm : LNLPWriteResPair<WriteVecShift, [LNLPVPort00_01], 1, [1], 1, 8>;
+def  : WriteRes<WriteVecShiftImm, [LNLPVPort00_01]>;
+def  : WriteRes<WriteVecShiftImmX, [LNLPVPort00_01]>;
+defm : X86WriteResUnsupported<WriteVecShiftImmXLd>;
+def  : WriteRes<WriteVecShiftImmY, [LNLPVPort00_01]>;
+defm : X86WriteResUnsupported<WriteVecShiftImmYLd>;
+defm : X86WriteResPairUnsupported<WriteVecShiftImmZ>;
+defm : X86WriteRes<WriteVecShiftX, [LNLPVPort00_01, LNLPVPort02_03], 2, [1, 1], 2>;
+defm : X86WriteRes<WriteVecShiftXLd, [LNLPVPort00_01, LNLPPort20_21_22], 8, [1, 1], 2>; // 7
+defm : X86WriteRes<WriteVecShiftY, [LNLPVPort00_01, LNLPVPort02_03], 4, [1, 1], 2>;
+defm : X86WriteRes<WriteVecShiftYLd, [LNLPVPort00_01, LNLPPort20_21_22], 9, [1, 7], 2>; // 8
+defm : X86WriteResPairUnsupported<WriteVecShiftZ>;
+defm : X86WriteRes<WriteVecStore, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>; 
+defm : X86WriteRes<WriteVecStoreNT, [LNLPPort10_11, LNLPPort25_26_27], 511, [1, 1], 2>;  // historic value
+defm : X86WriteRes<WriteVecStoreNTY, [LNLPPort10_11, LNLPPort25_26_27], 507, [1, 1], 2>;
+defm : X86WriteRes<WriteVecStoreX, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : X86WriteRes<WriteVecStoreY, [LNLPPort10_11, LNLPPort25_26_27], 12, [1, 1], 2>;
+defm : X86WriteRes<WriteVecTest, [LNLPVPort00_01, LNLPVPort02_03], 4, [1, 1], 2>;
+defm : X86WriteRes<WriteVecTestLd, [LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22], 10, [3, 1, 6], 3>;
+defm : X86WriteRes<WriteVecTestY, [LNLPVPort00_01, LNLPVPort02_03], LunarlakePModel.MaxLatency, [3, 1], 2>;
+defm : X86WriteRes<WriteVecTestYLd, [LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22], 11, [3, 1, 6], 3>;
+defm : X86WriteRes<WriteXCHG, [LNLPPort00_01_02_03_04_05], 2, [3], 3>;
+def : WriteRes<WriteZero, []>;
+
+// Manual Regressive SchedWriteRes and InstRW Definition. Suffix with "_X"
+// All _X(N) prefix sequence are defs used from prev. generation to bypass incomplete data.
+def LNLPWriteResGroupX0 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22, LNLPPort20_21_22, LNLPPort10_11]> {
+  let Latency = 7;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX0], (instregex "^AA(D|N)D64mr$",
+                                              "^A(X?)OR64mr$")>;
+def LNLPWriteResGroupX1 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 1, 1, 1, 1];
+  let Latency = 12;
+  let NumMicroOps = 6;
+}
+def : InstRW<[LNLPWriteResGroupX1, ReadAfterLd, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^(ADC|SBB)(16|32|64)mr$")>;
+def LNLPWriteResGroupX2 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX2], (instregex "^JMP(16|32|64)m((_NT)?)$",
+                                              "^RET(16|32)$",
+                                              "^RORX(32|64)mi$")>;
+def : InstRW<[LNLPWriteResGroupX2, ReadAfterLd, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^(ADC|SBB)(8|16|32|64)rm$",
+                                                                                                                                         "^AD(C|O)X(32|64)rm$")>;
+def LNLPWriteResGroupX5 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX5], (instregex "^CMP(8|16|32)mi$",
+                                              "^CMP(8|16|32|64)mi8$",
+                                              "^MOV(8|16)rm$",
+                                              "^POP(16|32)r((mr)?)$")>;
+def : InstRW<[LNLPWriteResGroupX5], (instrs CMP64mi32,
+                                           MOV8rm_NOREX,
+                                           MOVZX16rm8)>;
+def : InstRW<[LNLPWriteResGroupX5, ReadAfterLd], (instregex "^(ADD|CMP|SUB)(8|16|32|64)rm$",
+                                                           "^AND(8|16|32)rm$",
+                                                           "^(X?)OR(8|16|32)rm$")>;
+def : InstRW<[LNLPWriteResGroupX5, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^CMP(8|16|32|64)mr$")>;
+def LNLPWriteResGroupX6 : SchedWriteRes<[]> {
+  let NumMicroOps = 0;
+}
+def : InstRW<[LNLPWriteResGroupX6], (instregex "^(ADD|SUB)64ri8$",
+                                              "^(DE|IN)C64r$",
+                                              "^MOV64rr((_REV)?)$")>;
+def : InstRW<[LNLPWriteResGroupX6], (instrs CLC,
+                                           JMP_2)>;
+def LNLPWriteResGroupX7 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 13;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX7], (instregex "^A(D|N)D8mi(8?)$",
+                                              "^(DE|IN)C8m$",
+                                              "^N(EG|OT)8m$",
+                                              "^(X?)OR8mi(8?)$",
+                                              "^SUB8mi(8?)$")>;
+def : InstRW<[LNLPWriteResGroupX7, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^A(D|N)D8mr$",
+                                                                                                                            "^(X?)OR8mr$")>;
+def : InstRW<[LNLPWriteResGroupX7, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instrs SUB8mr)>;
+def LNLPWriteResGroupX8 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroupX8], (instregex "^(V?)(ADD|SUB)SSrr((_Int)?)$")>;
+def LNLPWriteResGroupX9 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX9], (instregex "^ADD_F(32|64)m$",
+                                              "^ILD_F(16|32|64)m$",
+                                              "^SUB(R?)_F(32|64)m$")>;
+def LNLPWriteResGroupX10 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 13;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX10], (instregex "^ADD_FI(16|32)m$",
+                                               "^SUB(R?)_FI(16|32)m$")>;
+
+def LNLPWriteResGroupX11 : SchedWriteRes<[LNLPPort00_01_02_03_04_05]> {
+  let Latency = 2;
+}
+def : InstRW<[LNLPWriteResGroupX11], (instregex "^AND(8|16|32|64)r(r|i8)$",
+                                               "^AND(8|16|32|64)rr_REV$",
+                                               "^(AND|TEST)(32|64)i32$",
+                                               "^(AND|TEST)(8|32)ri$",
+                                               "^(AND|TEST)64ri32$",
+                                               "^(AND|TEST)8i8$",
+                                               "^(X?)OR(8|16|32|64)r(r|i8)$",
+                                               "^(X?)OR(8|16|32|64)rr_REV$",
+                                               "^(X?)OR(32|64)i32$",
+                                               "^(X?)OR(8|32)ri$",
+                                               "^(X?)OR64ri32$",
+                                               "^(X?)OR8i8$",
+                                               "^TEST(8|16|32|64)rr$")>;
+def : InstRW<[LNLPWriteResGroupX11], (instrs XOR8rr_NOREX)>;
+def LNLPWriteResGroupX12 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX12], (instregex "^TEST(8|16|32)mi$")>;
+def : InstRW<[LNLPWriteResGroupX12], (instrs TEST64mi32)>;
+def : InstRW<[LNLPWriteResGroupX12, ReadAfterLd], (instregex "^(X?)OR64rm$")>;
+def : InstRW<[LNLPWriteResGroupX12, ReadAfterLd], (instrs AND64rm)>;
+def : InstRW<[LNLPWriteResGroupX12, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^TEST(8|16|32|64)mr$")>;
+def LNLPWriteResGroupX13 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX13, ReadAfterLd], (instregex "^ANDN(32|64)rm$")>;
+def LNLPWriteResGroupX14 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let Latency = 2;
+}
+def : InstRW<[LNLPWriteResGroupX14], (instregex "^ANDN(32|64)rr$")>;
+def LNLPWriteResGroupX15 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [5, 2, 1, 1];
+  let Latency = 10;
+  let NumMicroOps = 9;
+}
+def : InstRW<[LNLPWriteResGroupX15], (instrs BT64mr)>;
+def LNLPWriteResGroupX16 : SchedWriteRes<[LNLPPort01]> {
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroupX16], (instregex "^BT((C|R|S)?)64rr$")>;
+def LNLPWriteResGroupX17 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [4, 2, 1, 1, 1, 1];
+  let Latency = 17;
+  let NumMicroOps = 10;
+}
+def : InstRW<[LNLPWriteResGroupX17], (instregex "^BT(C|R|S)64mr$")>;
+def LNLPWriteResGroupX18 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 7;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX18], (instregex "^CALL(16|32|64)m((_NT)?)$")>;
+
+def LNLPWriteResGroupX19 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 3;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX19], (instregex "^CALL(16|32|64)r((_NT)?)$")>;
+
+def LNLPWriteResGroupX20 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX20], (instrs CALL64pcrel32,
+                                            MFENCE)>;
+def LNLPWriteResGroupX21 : SchedWriteRes<[LNLPVPort02_03]>;
+def : InstRW<[LNLPWriteResGroupX21], (instregex "^C(DQ|WD)E$",
+                                               "^(V?)MOVS(H|L)DUPrr$",
+                                               "^(V?)SHUFP(D|S)rri$",
+                                               "^VMOVS(H|L)DUPYrr$",
+                                               "^VSHUFP(D|S)Yrri$")>;
+def : InstRW<[LNLPWriteResGroupX21], (instrs CBW)>;
+
+def LNLPWriteResGroupX22 : SchedWriteRes<[LNLPPort00_02_04]>;
+def : InstRW<[LNLPWriteResGroupX22], (instregex "^C(DQ|QO)$",
+                                               "^(CL|ST)AC$")>;
+def LNLPWriteResGroupX23 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX23], (instrs CLD)>;
+
+def LNLPWriteResGroupX24 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 3;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX24], (instrs CLDEMOTE)>;
+
+def LNLPWriteResGroupX25 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 2;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX25], (instrs CLFLUSH)>;
+
+def LNLPWriteResGroupX26 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX26], (instrs CLFLUSHOPT)>;
+
+def LNLPWriteResGroupX27 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort01]> {
+  let ReleaseAtCycles = [2, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX27], (instrs CLI)>;
+
+def LNLPWriteResGroupX28 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort01, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [6, 1, 3];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 10;
+}
+def : InstRW<[LNLPWriteResGroupX28], (instrs CLTS)>;
+
+def LNLPWriteResGroupX29 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 5;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX29], (instregex "^MOV16o(16|32|64)a$")>;
+def : InstRW<[LNLPWriteResGroupX29], (instrs CLWB)>;
+
+def LNLPWriteResGroupX30 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [5, 2];
+  let Latency = 6;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX30], (instregex "^CMPS(B|L|Q|W)$")>;
+
+def LNLPWriteResGroupX31 : SchedWriteRes<[LNLPPort00, LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 7, 6, 2, 1, 1, 2, 1];
+  let Latency = 32;
+  let NumMicroOps = 22;
+}
+def : InstRW<[LNLPWriteResGroupX31], (instrs CMPXCHG16B)>;
+
+def LNLPWriteResGroupX32 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [4, 7, 2, 1, 1, 1];
+  let Latency = 25;
+  let NumMicroOps = 16;
+}
+def : InstRW<[LNLPWriteResGroupX32], (instrs CMPXCHG8B)>;
+
+def LNLPWriteResGroupX33 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [1, 2, 1, 1, 1];
+  let Latency = 13;
+  let NumMicroOps = 6;
+}
+def : InstRW<[LNLPWriteResGroupX33], (instrs CMPXCHG8rm)>;
+
+def LNLPWriteResGroupX34 : SchedWriteRes<[LNLPPort00, LNLPVPort00_01, LNLPPort00_02_04, LNLPPort01, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 1, 10, 6, 1, 5, 1];
+  let Latency = 18;
+  let NumMicroOps = 26;
+}
+def : InstRW<[LNLPWriteResGroupX34], (instrs CPUID)>;
+
+def LNLPWriteResGroupX35 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01, LNLPPort20_21_22]> {
+  let Latency = 26;
+  let NumMicroOps = 3;
+}
+def LNLPWriteResGroupX36 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX36], (instrs CVTSI642SSrm)>;
+def : InstRW<[LNLPWriteResGroupX36, ReadAfterVecLd], (instrs VCVTSI642SSrm)>;
+def LNLPWriteResGroupX37 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 8;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX37, ReadInt2Fpu], (instrs CVTSI642SSrr)>;
+def : InstRW<[LNLPWriteResGroupX37, ReadDefault, ReadInt2Fpu], (instrs VCVTSI642SSrr)>;
+def LNLPWriteResGroupX38 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01, LNLPPort01_03_05]> {
+  let Latency = 8;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX38, ReadDefault], (instregex "^(V?)CVT(T?)SS2SI64rr$")>;
+def LNLPWriteResGroupX39 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX39], (instregex "^J(E|R)CXZ$")>;
+def : InstRW<[LNLPWriteResGroupX39], (instrs CWD)>;
+
+def LNLPWriteResGroupX40 : SchedWriteRes<[LNLPPort00_01_02_03_04_05]>;
+def : InstRW<[LNLPWriteResGroupX40], (instregex "^(LD|ST)_Frr$",
+                                               "^MOV16s(m|r)$",
+                                               "^MOV(32|64)sr$")>;
+def : InstRW<[LNLPWriteResGroupX40], (instrs DEC16r_alt,
+                                            SALC,
+                                            ST_FPrr,
+                                            SYSCALL)>;
+
+def LNLPWriteResGroupX41 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 7;
+}
+def : InstRW<[LNLPWriteResGroupX41], (instrs DEC32r_alt)>;
+def LNLPWriteResGroupX42 : SchedWriteRes<[LNLPPort00, LNLPPort20_21_22]> {
+  let Latency = 27;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX42], (instregex "^DIVR_F(32|64)m$")>;
+def LNLPWriteResGroupX43 : SchedWriteRes<[LNLPPort00, LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let Latency = 30;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX43], (instregex "^DIVR_FI(16|32)m$")>;
+
+def LNLPWriteResGroupX44 : SchedWriteRes<[LNLPPort00]> {
+  let Latency = 15;
+}
+def : InstRW<[LNLPWriteResGroupX44], (instregex "^DIVR_F(P?)rST0$")>;
+def : InstRW<[LNLPWriteResGroupX44], (instrs DIVR_FST0r)>;
+def LNLPWriteResGroupX45 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let Latency = 20;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX45, ReadAfterVecLd], (instregex "^(V?)DIVSDrm_Int$")>;
+def LNLPWriteResGroupX46 : SchedWriteRes<[LNLPPort00, LNLPPort20_21_22]> {
+  let Latency = 22;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX46], (instregex "^DIV_F(32|64)m$")>;
+def LNLPWriteResGroupX47 : SchedWriteRes<[LNLPPort00, LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let Latency = 25;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX47], (instregex "^DIV_FI(16|32)m$")>;
+
+def LNLPWriteResGroupX48 : SchedWriteRes<[LNLPPort00]> {
+  let Latency = 20;
+}
+def : InstRW<[LNLPWriteResGroupX48], (instregex "^DIV_F(P?)rST0$")>;
+def : InstRW<[LNLPWriteResGroupX48], (instrs DIV_FST0r)>;
+
+def LNLPWriteResGroupX49 : SchedWriteRes<[LNLPPort00, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 21, 2, 14, 4, 9, 5];
+  let Latency = 126;
+  let NumMicroOps = 57;
+}
+def : InstRW<[LNLPWriteResGroupX49], (instrs ENTER)>;
+
+def LNLPWriteResGroupX50 : SchedWriteRes<[LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX50], (instregex "^(V?)EXTRACTPSmri$")>;
+def : InstRW<[LNLPWriteResGroupX50], (instrs SMSW16m)>;
+
+def LNLPWriteResGroupX51 : SchedWriteRes<[LNLPVPort02_03, LNLPPort01_03_05]> {
+  let Latency = 4;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX51], (instregex "^(V?)EXTRACTPSrri$")>;
+def : InstRW<[LNLPWriteResGroupX51], (instrs MMX_PEXTRWrri)>;
+
+def LNLPWriteResGroupX52 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22, LNLPPort20_21_22, LNLPPort10_11, LNLPVPort02_03]> {
+  let Latency = 7;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX52], (instrs FARCALL64m)>;
+
+def LNLPWriteResGroupX53 : SchedWriteRes<[LNLPPort20_21_22, LNLPVPort02_03]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX53], (instrs FARJMP64m,
+                                            JMP64m_REX)>;
+
+def LNLPWriteResGroupX54 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort10_11]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX54], (instregex "^(V?)MASKMOVDQU((64)?)$",
+                                               "^ST_FP(32|64|80)m$")>;
+def : InstRW<[LNLPWriteResGroupX54], (instrs FBSTPm,
+                                            VMPTRSTm)>;
+
+def LNLPWriteResGroupX55 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX55], (instrs FDECSTP)>;
+
+def LNLPWriteResGroupX56 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 11;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX56], (instregex "^FICOM(P?)(16|32)m$")>;
+
+def LNLPWriteResGroupX57 : SchedWriteRes<[LNLPPort01_03_05]>;
+def : InstRW<[LNLPWriteResGroupX57], (instregex "^MMX_P(ADD|SUB)(B|D|Q|W)rr$")>;
+def : InstRW<[LNLPWriteResGroupX57], (instrs FINCSTP,
+                                            FNOP)>;
+
+def LNLPWriteResGroupX58 : SchedWriteRes<[LNLPVPort02_03, LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let Latency = 7;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX58], (instrs FLDCW16m)>;
+
+def LNLPWriteResGroupX59 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 39, 5, 10, 8];
+  let Latency = 62;
+  let NumMicroOps = 64;
+}
+def : InstRW<[LNLPWriteResGroupX59], (instrs FLDENVm)>;
+
+def LNLPWriteResGroupX60 : SchedWriteRes<[LNLPVPort00_01_02_03]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 4;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX60], (instrs FNCLEX)>;
+
+def LNLPWriteResGroupX61 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [6, 3, 6];
+  let Latency = 75;
+  let NumMicroOps = 15;
+}
+def : InstRW<[LNLPWriteResGroupX61], (instrs FNINIT)>;
+
+def LNLPWriteResGroupX62 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort10_11, LNLPVPort02_03]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX62], (instrs FNSTCW16m)>;
+
+def LNLPWriteResGroupX63 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01_02_03]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX63], (instrs FNSTSW16r)>;
+
+def LNLPWriteResGroupX64 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11]> {
+  let Latency = 3;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX64], (instrs FNSTSWm)>;
+
+def LNLPWriteResGroupX65 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [9, 30, 21, 1, 11, 11, 16, 1];
+  let Latency = 106;
+  let NumMicroOps = 100;
+}
+def : InstRW<[LNLPWriteResGroupX65], (instrs FSTENVm)>;
+
+def LNLPWriteResGroupX66 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort00_02_04, LNLPVPort02_03, LNLPPort20_21_22, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [4, 47, 1, 2, 1, 33, 2];
+  let Latency = 63;
+  let NumMicroOps = 90;
+}
+def : InstRW<[LNLPWriteResGroupX66], (instrs FXRSTOR)>;
+
+def LNLPWriteResGroupX67 : SchedWriteRes<[LNLPVPort02_03, LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort00_02_04, LNLPVPort02_03, LNLPPort20_21_22, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [4, 45, 1, 2, 1, 31, 4];
+  let Latency = 63;
+  let NumMicroOps = 88;
+}
+def : InstRW<[LNLPWriteResGroupX67], (instrs FXRSTOR64)>;
+
+def LNLPWriteResGroupX68 : SchedWriteRes<[LNLPVPort02_03, LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 5, 10, 10, 2, 38, 5, 38];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 110;
+}
+def : InstRW<[LNLPWriteResGroupX68], (instregex "^FXSAVE((64)?)$")>;
+
+def LNLPWriteResGroupX69 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let Latency = 12;
+  let NumMicroOps = 2;
+}
+
+def LNLPWriteResGroupX77 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX77], (instrs INC16r_alt)>;
+
+def LNLPWriteResGroupX78 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 7;
+}
+def : InstRW<[LNLPWriteResGroupX78], (instrs INC32r_alt)>;
+
+def LNLPWriteResGroupX85 : SchedWriteRes<[LNLPPort00_02_04]>;
+def : InstRW<[LNLPWriteResGroupX85], (instrs JMP64r_REX)>;
+def LNLPWriteResGroupX86 : SchedWriteRes<[]> {
+  let Latency = 0;
+  let NumMicroOps = 0;
+}
+def : InstRW<[LNLPWriteResGroupX86], (instregex "^JMP_(1|4)$")>;
+def : InstRW<[LNLPWriteResGroupX86], (instrs VZEROUPPER)>;
+
+def LNLPWriteResGroupX93 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX93], (instrs LEA16r)>;
+
+def LNLPWriteResGroupX104 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [4, 6, 1];
+  let Latency = 3;
+  let NumMicroOps = 11;
+}
+def : InstRW<[LNLPWriteResGroupX104], (instrs LOOPE)>;
+
+def LNLPWriteResGroupX105 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [4, 6, 1];
+  let Latency = 2;
+  let NumMicroOps = 11;
+}
+def : InstRW<[LNLPWriteResGroupX105], (instrs LOOPNE)>;
+
+def LNLPWriteResGroupX115 : SchedWriteRes<[LNLPVPort00_01, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 1, 1];
+  let Latency = 12;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX115], (instregex "^MMX_MASKMOVQ((64)?)$")>;
+def LNLPWriteResGroupX118 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPVPort00_01]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX118], (instregex "^MMX_MOV(DQ|FR64)2Qrr$")>;
+def LNLPWriteResGroupX122 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let Latency = 9;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX122, ReadAfterVecLd], (instregex "^MMX_P(ADD|SUB)(B|D|Q|W)rm$")>;
+def LNLPWriteResGroupX123 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 1, 2];
+  let Latency = 11;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX123, ReadAfterVecLd], (instregex "^MMX_PH(ADD|SUB)SWrm$")>;
+def LNLPWriteResGroupX124 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 3;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX124], (instregex "^MMX_PH(ADD|SUB)SWrr$")>;
+def LNLPWriteResGroupX116 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 18;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX116], (instrs MMX_MOVD64mr)>;
+def LNLPWriteResGroupX117 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 8;
+}
+def : InstRW<[LNLPWriteResGroupX117], (instregex "^MMX_MOV(D|Q)64rm$",
+                                                "^VBROADCASTI128rm$")>;
+def : InstRW<[LNLPWriteResGroupX117], (instrs MMX_MOVD64to64rm)>;
+def LNLPWriteResGroupX120 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX120, ReadAfterVecLd], (instregex "^MMX_PACKSS(DW|WB)rm$")>;
+def : InstRW<[LNLPWriteResGroupX120, ReadAfterVecLd], (instrs MMX_PACKUSWBrm)>;
+def LNLPWriteResGroupX121 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 4;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX121], (instregex "^MMX_PACKSS(DW|WB)rr$")>;
+def : InstRW<[LNLPWriteResGroupX121], (instrs MMX_PACKUSWBrr)>;
+def : InstRW<[LNLPWriteResGroupX121, ReadDefault, ReadInt2Fpu], (instrs MMX_PINSRWrri)>;
+def LNLPWriteResGroupX125 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort01_03_05]> {
+  let Latency = 9;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX125, ReadAfterLd], (instrs MMX_PINSRWrmi)>;
+def LNLPWriteResGroupX126 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX126], (instregex "^MOV16ao(16|32|64)$")>;
+def LNLPWriteResGroupX127 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX127], (instregex "^PUSH(F|G)S(16|32)$")>;
+def : InstRW<[LNLPWriteResGroupX127], (instrs MOV16ms,
+                                             MOVBE32mr)>;
+def LNLPWriteResGroupX128 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX128], (instregex "^MOV(16|32|64)rs$",
+                                                "^S(TR|LDT)16r$")>;
+def LNLPWriteResGroupX129 : SchedWriteRes<[LNLPPort20_21_22]>;
+def : InstRW<[LNLPWriteResGroupX129], (instregex "^MOV32ao(16|32|64)$")>;
+def : InstRW<[LNLPWriteResGroupX129], (instrs MOV64ao64)>;
+def LNLPWriteResGroupX130 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX130], (instregex "^MOV(8|32)o(16|32)a$",
+                                                "^MOV(8|32|64)o64a$")>;
+def LNLPWriteResGroupX131 : SchedWriteRes<[LNLPPort00_01_02_03_04_05]> {
+  let Latency = 0;
+}
+def : InstRW<[LNLPWriteResGroupX131], (instregex "^MOV32rr((_REV)?)$",
+                                                "^MOVZX(32|64)rr8$")>;
+def : InstRW<[LNLPWriteResGroupX131], (instrs MOVZX32rr8_NOREX)>;
+
+def LNLPWriteResGroupX132 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 5;
+}
+def : InstRW<[LNLPWriteResGroupX132], (instrs MOV64ao32)>;
+def LNLPWriteResGroupX134 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX134], (instrs MOV64o32a)>;
+def LNLPWriteResGroupX135 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX135], (instrs MOV64rc)>;
+def LNLPWriteResGroupX137 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX137], (instregex "^MOV8ao(16|32|64)$")>;
+def LNLPWriteResGroupX138 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 13;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX138], (instregex "^MOV8m(i|r)$")>;
+def : InstRW<[LNLPWriteResGroupX138], (instrs MOV8mr_NOREX)>;
+def LNLPWriteResGroupX139 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX139], (instrs MOVBE16mr)>;
+def LNLPWriteResGroupX142 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX142], (instrs MOVBE64mr,
+                                             PUSHF16,
+                                             SLDT16m,
+                                             STRm)>;
+
+def LNLPWriteResGroupX144 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX144], (instregex "^MOVDIR64B(16|32|64)$")>;
+def LNLPWriteResGroupX147 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX147, ReadAfterVecXLd], (instregex "^(V?)MOVLP(D|S)rm$")>;
+def LNLPWriteResGroupX148 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 512;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX148], (instrs MOVNTDQmr)>;
+def LNLPWriteResGroupX149 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 518;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX149], (instrs MOVNTImr)>;
+def LNLPWriteResGroupX150 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [4, 1, 1, 1];
+  let Latency = 8;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX150], (instrs MOVSB)>;
+def LNLPWriteResGroupX151 : SchedWriteRes<[LNLPVPort00_01_02_03]>;
+def : InstRW<[LNLPWriteResGroupX151], (instrs VPBLENDDrri)>;
+def LNLPWriteResGroupX152 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [4, 1, 1, 1];
+  let Latency = 7;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX152], (instregex "^MOVS(L|Q|W)$")>;
+def LNLPWriteResGroupX153 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 6;
+}
+def : InstRW<[LNLPWriteResGroupX153], (instregex "^MOVSX(16|32|64)rm(16|32)$",
+                                                "^MOVSX(32|64)rm8$")>;
+def : InstRW<[LNLPWriteResGroupX153], (instrs MOVSX32rm8_NOREX)>;
+def LNLPWriteResGroupX154 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX154], (instrs MOVSX16rm8)>;
+def LNLPWriteResGroupX156 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPVPort02_03]> {
+  let Latency = 11;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX156], (instregex "^MUL_F(32|64)m$")>;
+def LNLPWriteResGroupX157 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPVPort02_03, LNLPPort01_03_05]> {
+  let Latency = 14;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX157], (instregex "^MUL_FI(16|32)m$")>;
+def LNLPWriteResGroupX158 : SchedWriteRes<[LNLPVPort00_01_02_03]> {
+  let Latency = 4;
+}
+def : InstRW<[LNLPWriteResGroupX158], (instregex "^MUL_F(P?)rST0$")>;
+def : InstRW<[LNLPWriteResGroupX158], (instrs MUL_FST0r)>;
+def LNLPWriteResGroupX155 : SchedWriteRes<[LNLPPort01_03_05]>;
+def : InstRW<[LNLPWriteResGroupX155], (instregex "^MOVSX(16|32|64)rr(8|16|32)$")>;
+def : InstRW<[LNLPWriteResGroupX155], (instrs MOVSX32rr8_NOREX)>;
+def LNLPWriteResGroupX173 : SchedWriteRes<[LNLPPort01_03_05]>;
+def : InstRW<[LNLPWriteResGroupX173], (instregex "^(V?)PALIGNRrri$",
+                                                "^VPBROADCAST(B|D|Q|W)rr$")>;
+def : InstRW<[LNLPWriteResGroupX173], (instrs VPALIGNRYrri)>;
+def LNLPWriteResGroupX176 : SchedWriteRes<[LNLPVPort02_03, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX176], (instregex "^(V?)PEXTR(D|Q)mri$")>;
+def LNLPWriteResGroupX179 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX179], (instregex "^POP(16|32|64)rmm$",
+                                                "^PUSH(16|32)rmm$")>;
+def LNLPWriteResGroupX180 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 5;
+}
+def : InstRW<[LNLPWriteResGroupX180], (instregex "^POPA(16|32)$",
+                                                "^PREFETCHIT(0|1)$")>;
+def : InstRW<[LNLPWriteResGroupX180], (instrs POPF32)>;
+def LNLPWriteResGroupX181 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [6, 2, 1, 1];
+  let Latency = 5;
+  let NumMicroOps = 10;
+}
+def : InstRW<[LNLPWriteResGroupX181], (instrs POPF16)>;
+def LNLPWriteResGroupX182 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 1, 1];
+  let Latency = 5;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX182], (instrs POPF64)>;
+def LNLPWriteResGroupX183 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let Latency = 0;
+}
+def : InstRW<[LNLPWriteResGroupX183], (instregex "^PREFETCHT(0|1|2)$")>;
+def : InstRW<[LNLPWriteResGroupX183], (instrs PREFETCHNTA)>;
+def LNLPWriteResGroupX187 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX187], (instregex "^PUSH64r((mr)?)$")>;
+def LNLPWriteResGroupX188 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX188], (instrs PUSH64rmm)>;
+def LNLPWriteResGroupX189 : SchedWriteRes<[LNLPPort20_21_22, LNLPPort10_11]>;
+def : InstRW<[LNLPWriteResGroupX189], (instregex "^PUSHA(16|32)$",
+                                                "^ST_F(32|64)m$")>;
+def : InstRW<[LNLPWriteResGroupX189], (instrs PUSHF32)>;
+def LNLPWriteResGroupX190 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 4;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX190], (instrs PUSHF64)>;
+def LNLPWriteResGroupX191 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX191], (instregex "^PUSH(F|G)S64$")>;
+def LNLPWriteResGroupX192 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2, 3, 2];
+  let Latency = 8;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX192], (instregex "^RC(L|R)(16|32|64)rCL$")>;
+def LNLPWriteResGroupX193 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 13;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX193, WriteRMW], (instregex "^RC(L|R)8m(1|i)$")>;
+def LNLPWriteResGroupX194 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 5, 2];
+  let Latency = 20;
+  let NumMicroOps = 8;
+}
+def : InstRW<[LNLPWriteResGroupX194, WriteRMW], (instrs RCL8mCL)>;
+def LNLPWriteResGroupX195 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2, 5, 2];
+  let Latency = 7;
+  let NumMicroOps = 9;
+}
+def : InstRW<[LNLPWriteResGroupX195], (instrs RCL8rCL)>;
+def LNLPWriteResGroupX196 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2, 4, 3];
+  let Latency = 20;
+  let NumMicroOps = 9;
+}
+def : InstRW<[LNLPWriteResGroupX196, WriteRMW], (instrs RCR8mCL)>;
+def LNLPWriteResGroupX197 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [3, 4, 3];
+  let Latency = 9;
+  let NumMicroOps = 10;
+}
+def : InstRW<[LNLPWriteResGroupX197], (instrs RCR8rCL)>;
+def LNLPWriteResGroupX206 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [5, 6, 3, 1];
+  let Latency = 18;
+  let NumMicroOps = 15;
+}
+def : InstRW<[LNLPWriteResGroupX206], (instrs RDTSC)>;
+def LNLPWriteResGroupX208 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX208], (instrs RET64)>;
+def LNLPWriteResGroupX209 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 1];
+  let Latency = 6;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX209], (instregex "^RETI(16|32|64)$")>;
+def LNLPWriteResGroupX210 : SchedWriteRes<[]>;
+def : InstRW<[LNLPWriteResGroupX210], (instrs REX64_PREFIX)>;
+def LNLPWriteResGroupX211 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 12;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX211, WriteRMW], (instregex "^RO(L|R)(16|32|64)m(1|i|CL)$")>;
+def LNLPWriteResGroupX212 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [2];
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX212], (instregex "^RO(L|R)(8|16|32|64)r(1|i)$")>;
+def LNLPWriteResGroupX213 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 13;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX213, WriteRMW], (instregex "^RO(L|R)8m(1|i)$",
+                                                          "^(RO|SH)L8mCL$",
+                                                          "^(RO|SA|SH)R8mCL$")>;
+def LNLPWriteResGroupX214 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 4;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX214], (instrs SAHF)>;
+def LNLPWriteResGroupX215 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let Latency = 13;
+}
+def : InstRW<[LNLPWriteResGroupX215, WriteRMW], (instregex "^S(A|H)R8m(1|i)$",
+                                                          "^SHL8m(1|i)$")>;
+def LNLPWriteResGroupX216 : SchedWriteRes<[LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX216, ReadAfterLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^S(A|H)RX(32|64)rm$",
+                                                                                                                              "^SHLX(32|64)rm$")>;
+def LNLPWriteResGroupX217 : SchedWriteRes<[LNLPPort00_02_04]> {
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroupX217], (instregex "^S(A|H)RX(32|64)rr$",
+                                                "^SHLX(32|64)rr$")>;
+def LNLPWriteResGroupX218 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 2, 1, 1, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX218], (instrs SERIALIZE)>;
+def LNLPWriteResGroupX219 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX219], (instrs SFENCE)>;
+def LNLPWriteResGroupX220 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [1, 2, 2, 2];
+  let Latency = 21;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX220], (instregex "^S(G|I)DT64m$")>;
+def LNLPWriteResGroupX223 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPPort00_02_04, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 2, 1, 2, 1];
+  let Latency = 13;
+  let NumMicroOps = 8;
+}
+def : InstRW<[LNLPWriteResGroupX223, ReadAfterVecXLd], (instrs SHA1MSG2rm)>;
+def LNLPWriteResGroupX224 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPPort00_02_04, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [2, 2, 1, 2];
+  let Latency = 6;
+  let NumMicroOps = 7;
+}
+def : InstRW<[LNLPWriteResGroupX224], (instrs SHA1MSG2rr)>;
+def LNLPWriteResGroupX233 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 13;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX233], (instrs SHRD16mri8)>;
+def LNLPWriteResGroupX234 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX234], (instregex "^SLDT(32|64)r$")>;
+def LNLPWriteResGroupX235 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX235], (instrs SMSW16r)>;
+def LNLPWriteResGroupX236 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX236], (instregex "^SMSW(32|64)r$")>;
+def LNLPWriteResGroupX237 : SchedWriteRes<[LNLPPort00, LNLPPort20_21_22]> {
+  let Latency = 24;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX237, ReadAfterVecLd], (instregex "^(V?)SQRTSDm_Int$")>;
+def LNLPWriteResGroupX238 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX238], (instrs STD)>;
+def LNLPWriteResGroupX239 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 4, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 6;
+}
+def : InstRW<[LNLPWriteResGroupX239], (instrs STI)>;
+def LNLPWriteResGroupX240 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 1, 1];
+  let Latency = 8;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX240], (instrs STOSB)>;
+def LNLPWriteResGroupX241 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort10_11, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [2, 1, 1];
+  let Latency = 7;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX241], (instregex "^STOS(L|Q|W)$")>;
+def LNLPWriteResGroupX242 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort01_03_05]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX242], (instregex "^STR(32|64)r$")>;
+def LNLPWriteResGroupX243 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let Latency = 2;
+}
+def : InstRW<[LNLPWriteResGroupX243], (instregex "^(TST|XAM)_F$")>;
+def : InstRW<[LNLPWriteResGroupX243], (instrs UCOM_FPPr)>;
+def LNLPWriteResGroupX244 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 1];
+  let Latency = 9;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroupX244, ReadAfterVecXLd, ReadAfterVecXLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instregex "^VBLENDVP(D|S)rmr$")>;
+def : InstRW<[LNLPWriteResGroupX244, ReadAfterVecXLd, ReadAfterVecXLd, ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault], (instrs VPBLENDVBrmr)>;
+def LNLPWriteResGroupX245 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX245], (instregex "^VBLENDVP(D|S)rrr$")>;
+def : InstRW<[LNLPWriteResGroupX245], (instrs VPBLENDVBrrr)>;
+def LNLPWriteResGroupX250 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 2, 4];
+  let Latency = 29;
+  let NumMicroOps = 8;
+}
+def : InstRW<[LNLPWriteResGroupX250, WriteVecMaskedGatherWriteback], (instregex "^VGATHER(D|Q)PDYrm$",
+                                                                               "^VPGATHER(D|Q)QYrm$")>;
+def : InstRW<[LNLPWriteResGroupX250, WriteVecMaskedGatherWriteback], (instrs VGATHERQPSYrm,
+                                                                            VPGATHERQDYrm)>;
+def LNLPWriteResGroupX251 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 1, 2];
+  let Latency = 20;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX251, WriteVecMaskedGatherWriteback], (instregex "^VGATHER(D|Q)PDrm$",
+                                                                               "^VPGATHER(D|Q)Qrm$")>;
+def : InstRW<[LNLPWriteResGroupX251, WriteVecMaskedGatherWriteback], (instrs VGATHERQPSrm,
+                                                                            VPGATHERQDrm)>;
+def LNLPWriteResGroupX252 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 2, 8];
+  let Latency = 30;
+  let NumMicroOps = 12;
+}
+def : InstRW<[LNLPWriteResGroupX252, WriteVecMaskedGatherWriteback], (instrs VGATHERDPSYrm,
+                                                                            VPGATHERDDYrm)>;
+def LNLPWriteResGroupX253 : SchedWriteRes<[LNLPVPort00_01, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 2, 4];
+  let Latency = 28;
+  let NumMicroOps = 8;
+}
+def : InstRW<[LNLPWriteResGroupX253, WriteVecMaskedGatherWriteback], (instrs VGATHERDPSrm,
+                                                                            VPGATHERDDrm)>;
+def LNLPWriteResGroupX254 : SchedWriteRes<[LNLPVPort02_03, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 5;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX254], (instregex "^VH(ADD|SUB)P(D|S)rr$")>;
+def LNLPWriteResGroupX256 : SchedWriteRes<[LNLPVPort00_01, LNLPPort00_02_04, LNLPPort20_21_22]> {
+  let Latency = 7;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX256], (instrs VLDMXCSR)>;
+def LNLPWriteResGroupX257 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [8, 1, 1, 1, 1, 1, 2, 3];
+  let Latency = 40;
+  let NumMicroOps = 18;
+}
+def : InstRW<[LNLPWriteResGroupX257], (instrs VMCLEARm)>;
+def LNLPWriteResGroupX259 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 521;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX259], (instrs VMOVNTDQmr)>;
+def LNLPWriteResGroupX260 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 473;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX260], (instrs VMOVNTPDmr)>;
+def LNLPWriteResGroupX261 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 494;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX261], (instrs VMOVNTPSYmr)>;
+def LNLPWriteResGroupX262 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = 470;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX262], (instrs VMOVNTPSmr)>;
+def LNLPWriteResGroupX264 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let Latency = 9;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX264, ReadAfterVecYLd], (instregex "^VSHUFP(D|S)Yrmi$")>;
+def LNLPWriteResGroupX267 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [1, 2, 3, 3, 1];
+  let Latency = 16;
+  let NumMicroOps = 10;
+}
+def : InstRW<[LNLPWriteResGroupX267], (instrs VZEROALL)>;
+def LNLPWriteResGroupX268 : SchedWriteRes<[LNLPVPort00_01_02_03]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX268], (instrs WAIT)>;
+def LNLPWriteResGroupX269 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [8, 6, 19, 63, 21, 15, 1, 10, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 144;
+}
+def : InstRW<[LNLPWriteResGroupX269], (instrs WRMSR)>;
+def LNLPWriteResGroupX270 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [2, 1, 4, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 8;
+}
+def : InstRW<[LNLPWriteResGroupX270], (instrs WRPKRUr)>;
+def LNLPWriteResGroupX271 : SchedWriteRes<[LNLPPort00_01_02_03_04_05]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 12;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX271, WriteRMW], (instregex "^XADD(16|32|64)rm$")>;
+def LNLPWriteResGroupX272 : SchedWriteRes<[LNLPPort00_01_02_03_04_05]> {
+  let ReleaseAtCycles = [2];
+  let Latency = 13;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroupX272, WriteRMW], (instrs XADD8rm)>;
+def LNLPWriteResGroupX273 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [4, 1];
+  let Latency = 39;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX273, WriteRMW], (instregex "^XCHG(16|32)rm$")>;
+def LNLPWriteResGroupX274 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [5, 1];
+  let Latency = 39;
+  let NumMicroOps = 6;
+}
+def : InstRW<[LNLPWriteResGroupX274, WriteRMW], (instrs XCHG64rm)>;
+def LNLPWriteResGroupX275 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [4, 1];
+  let Latency = 40;
+  let NumMicroOps = 5;
+}
+def : InstRW<[LNLPWriteResGroupX275, WriteRMW], (instrs XCHG8rm)>;
+def LNLPWriteResGroupX276 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort01_03_05, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [2, 4, 2, 1, 2, 4];
+  let Latency = 17;
+  let NumMicroOps = 15;
+}
+def : InstRW<[LNLPWriteResGroupX276], (instrs XCH_F)>;
+def LNLPWriteResGroupX277 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [7, 3, 8, 5];
+  let Latency = 4;
+  let NumMicroOps = 23;
+}
+def : InstRW<[LNLPWriteResGroupX277], (instrs XGETBV)>;
+def LNLPWriteResGroupX278 : SchedWriteRes<[LNLPPort00_01_02_03_04_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 1];
+  let Latency = 7;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroupX278], (instrs XLAT)>;
+def LNLPWriteResGroupX279 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort00_02_04]> {
+  let ReleaseAtCycles = [21, 1, 1, 8];
+  let Latency = 37;
+  let NumMicroOps = 31;
+}
+def : InstRW<[LNLPWriteResGroupX279], (instregex "^XRSTOR((S|64)?)$")>;
+def : InstRW<[LNLPWriteResGroupX279], (instrs XRSTORS64)>;
+def LNLPWriteResGroupX280 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [14, 25, 44, 21, 21, 4, 1, 9, 1];
+  let Latency = 42;
+  let NumMicroOps = 140;
+}
+def : InstRW<[LNLPWriteResGroupX280], (instrs XSAVE)>;
+def LNLPWriteResGroupX281 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [14, 25, 44, 21, 21, 4, 1, 9, 1];
+  let Latency = 41;
+  let NumMicroOps = 140;
+}
+def : InstRW<[LNLPWriteResGroupX281], (instrs XSAVE64)>;
+def LNLPWriteResGroupX282 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [1, 19, 36, 52, 23, 4, 2, 12, 2];
+  let Latency = 42;
+  let NumMicroOps = 151;
+}
+def : InstRW<[LNLPWriteResGroupX282], (instrs XSAVEC)>;
+def LNLPWriteResGroupX283 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [1, 19, 36, 53, 23, 4, 2, 12, 2];
+  let Latency = 42;
+  let NumMicroOps = 152;
+}
+def : InstRW<[LNLPWriteResGroupX283], (instrs XSAVEC64)>;
+def LNLPWriteResGroupX284 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [25, 35, 52, 27, 4, 1, 10, 1];
+  let Latency = 46;
+  let NumMicroOps = 155;
+}
+def : InstRW<[LNLPWriteResGroupX284], (instrs XSAVEOPT)>;
+def LNLPWriteResGroupX285 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [25, 35, 53, 27, 4, 1, 10, 1];
+  let Latency = 46;
+  let NumMicroOps = 156;
+}
+def : InstRW<[LNLPWriteResGroupX285], (instrs XSAVEOPT64)>;
+def LNLPWriteResGroupX286 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [23, 32, 53, 29, 30, 4, 2, 9, 2];
+  let Latency = 42;
+  let NumMicroOps = 184;
+}
+def : InstRW<[LNLPWriteResGroupX286], (instrs XSAVES)>;
+def LNLPWriteResGroupX287 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPVPort02_03, LNLPPort20_21_22, LNLPPort10_11, LNLPPort01_03_05, LNLPPort25_26_27]> {
+  let ReleaseAtCycles = [23, 33, 53, 29, 32, 4, 2, 8, 2];
+  let Latency = 42;
+  let NumMicroOps = 186;
+}
+def : InstRW<[LNLPWriteResGroupX287], (instrs XSAVES64)>;
+def LNLPWriteResGroupX288 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort00_01_02_03_04_05, LNLPVPort00_01_02_03, LNLPPort00_02_04, LNLPPort01_03_05, LNLPPort01_03_05, LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [4, 23, 2, 14, 8, 1, 2];
+  let Latency = 5;
+  let NumMicroOps = 54;
+}
+def : InstRW<[LNLPWriteResGroupX288], (instrs XSETBV)>;
+
+// SchedWriteRes and InstRW definition
+// Following defs are based on data shared by arch team
+def LNLPWriteResGroup0 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup0, ReadAfterVecXLd], (instregex "^(V?)CMPP(D|S)rmi$",
+                                                               "^GF2P8AFFINE((INV)?)QBrmi$")>;
+def : InstRW<[LNLPWriteResGroup0, ReadAfterVecXLd], (instrs GF2P8MULBrm)>;
+def : InstRW<[LNLPWriteResGroup0, ReadAfterVecLd], (instregex "^(V?)CMPS(D|S)rmi((_Int)?)$")>;
+def LNLPWriteResGroup1 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 1, 6];
+  let Latency = 11;
+  let NumMicroOps = 2;
+}
+def LNLPWriteResGroup2 : SchedWriteRes<[LNLPPort01_03_05, LNLPVPort00_01, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [4, 4, 1];
+  let Latency = 9;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup2, ReadDefault, ReadInt2Fpu], (instregex "^(V?)CVTSI642SSrr_Int$")>;
+def LNLPWriteResGroup3 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7, 4];
+  let Latency = 11;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup3], (instregex "^CVT(T?)SS2SI((64)?)rm_Int$",
+                                              "^CVT(T?)SS2SIrm$",
+                                              "^VCVTTSD2SI((64)?)rm_Int$")>;
+def LNLPWriteResGroup4 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [7, 1];
+  let Latency = 8;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup4], (instregex "^(V?)CVT(T?)SS2SI64rr_Int$")>;
+def LNLPWriteResGroup5 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [4];
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroup5], (instregex "^GF2P8AFFINE((INV)?)QBrri$")>;
+def : InstRW<[LNLPWriteResGroup5], (instrs GF2P8MULBrr)>;
+def LNLPWriteResGroup6 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 1, 8];
+  let Latency = 13;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup6], (instregex "^MMX_CVT(T?)PD2PIrm$")>;
+def LNLPWriteResGroup7 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [4, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup7], (instregex "^MMX_CVT(T?)PD2PIrr$")>;
+def : InstRW<[LNLPWriteResGroup7], (instrs MMX_CVTPI2PDrr)>;
+def LNLPWriteResGroup8 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [5];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup8], (instrs MMX_CVTPI2PSrr)>;
+def LNLPWriteResGroup9 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 8];
+  let Latency = 13;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup9], (instregex "^MMX_CVT(T?)PS2PIrm$")>;
+def LNLPWriteResGroup10 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03]> {
+  let ReleaseAtCycles = [4, 1];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup10], (instrs MMX_CVTPS2PIrr)>;
+def LNLPWriteResGroup11 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03]> {
+  let ReleaseAtCycles = [4, 1];
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup11], (instrs MMX_CVTTPS2PIrr)>;
+def LNLPWriteResGroup12 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03]> {
+  let Latency = 3;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup12], (instregex "^MMX_MOVQ2(DQ|FR64)rr$")>;
+def LNLPWriteResGroup13 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [2, 4];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup13], (instregex "^MOVBE(16|32|64)rm$")>;
+def LNLPWriteResGroup14 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [6];
+  let Latency = 6;
+}
+def : InstRW<[LNLPWriteResGroup14], (instregex "^(V?)MOV(D|SH|SL)DUPrm$",
+                                               "^VPBROADCAST(D|Q|W)rm$")>;
+def : InstRW<[LNLPWriteResGroup14], (instrs VBROADCASTSSrm)>;
+def LNLPWriteResGroup15 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup15], (instregex "^MOVDIRI(32|64)$")>;
+def LNLPWriteResGroup16 : SchedWriteRes<[LNLPVPort00_01_02_03]>;
+def : InstRW<[LNLPWriteResGroup16], (instregex "^(V?)MOVS(D|S)rr((_REV)?)$",
+                                               "^(V?)P(ADD|SUB)(B|D|Q|W)rr$",
+                                               "^VP(ADD|SUB)(B|D|Q|W)Yrr$")>;
+def LNLPWriteResGroup17 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;  // originally 10
+}
+def : InstRW<[LNLPWriteResGroup17, ReadAfterVecXLd], (instregex "^(V?)PACK(S|U)S(DW|WB)rm$",
+                                                                "^(V?)PCMPGTQrm$")>;
+def LNLPWriteResGroup18 : SchedWriteRes<[LNLPVPort02_03]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroup18], (instregex "^(V?)PACK(S|U)S(DW|WB)rr$",
+                                               "^(V?)PACKUSWBrr$",
+                                               "^(V?)PCMPGTQrr$",
+                                               "^VPACK(S|U)S(DW|WB)Yrr$",
+                                               "^VSHA512MSG(1|2)rr$")>;
+def : InstRW<[LNLPWriteResGroup18], (instrs VPCMPGTQYrr)>;
+def LNLPWriteResGroup19 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+}
+def : InstRW<[LNLPWriteResGroup19, ReadAfterVecXLd], (instregex "^(V?)P(ADD|SUB)(B|D|Q|W)rm$")>;
+def : InstRW<[LNLPWriteResGroup19, ReadAfterVecXLd], (instrs VPBLENDDrmi)>;
+def LNLPWriteResGroup20 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 6];
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup20], (instregex "^(V?)PSHUF(D|HW|LW)mi$",
+                                               "^VPERMILP(D|S)mi$")>;
+def : InstRW<[LNLPWriteResGroup20, ReadAfterVecXLd], (instregex "^(V?)PALIGNRrmi$",
+                                                                "^(V?)SHUFP(D|S)rmi$")>;
+def : InstRW<[LNLPWriteResGroup20, ReadAfterVecXLd], (instrs VINSERTPSrmi,
+                                                             VPBLENDWrmi)>;
+def LNLPWriteResGroup21 : SchedWriteRes<[LNLPPort01_03_05, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 4];
+  let Latency = 7;
+}
+def : InstRW<[LNLPWriteResGroup21, ReadAfterLd], (instregex "^P(DEP|EXT)(32|64)rm$")>;
+def LNLPWriteResGroup22 : SchedWriteRes<[LNLPPort01_03_05]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroup22], (instregex "^P(DEP|EXT)(32|64)rr$")>;
+def LNLPWriteResGroup23 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 2, 1];
+  let Latency = 9; // originally LunarlakePModel.MaxLatency;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroup23, ReadAfterVecXLd], (instregex "^(V?)PH(ADD|SUB)SWrm$")>;
+def LNLPWriteResGroup24 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03]> {
+  let ReleaseAtCycles = [1, 2];
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup24], (instregex "^(V?)PH(ADD|SUB)SWrr$",
+                                               "^VPH(ADD|SUB)SWYrr$")>;
+def LNLPWriteResGroup25 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+}
+def : InstRW<[LNLPWriteResGroup25, ReadAfterVecXLd], (instregex "^(V?)PMULUDQrm$")>;
+def : InstRW<[LNLPWriteResGroup25, ReadAfterVecXLd], (instrs VPMULDQrm)>;
+def LNLPWriteResGroup26 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 3;
+}
+def : InstRW<[LNLPWriteResGroup26], (instregex "^(V?)PMULUDQrr$",
+                                               "^VPMUL(U?)DQYrr$")>;
+def : InstRW<[LNLPWriteResGroup26], (instrs VMOVSDto64Zrr,
+                                            VPMULDQrr)>;
+
+def LNLPWriteResGroup27 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 6];
+  let Latency = 8;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup27, ReadAfterVecXLd], (instrs SHA1MSG1rm)>;
+def LNLPWriteResGroup28 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPVPort02_03]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup28], (instrs SHA1MSG1rr)>;
+def LNLPWriteResGroup29 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 1, 1, 6];
+  let Latency = 7;  // Originally 8
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroup29, ReadAfterVecXLd], (instrs SHA1NEXTErm)>;
+def LNLPWriteResGroup30 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort00_01_02_03, LNLPVPort02_03]> {
+  let Latency = 3;  // originally LunarlakePModel.MaxLatency;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup30], (instrs SHA1NEXTErr)>;
+def LNLPWriteResGroup31 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup31, ReadAfterVecXLd], (instrs SHA1RNDS4rmi)>;
+def LNLPWriteResGroup32 : SchedWriteRes<[LNLPVPort02_03]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 2;
+}
+def : InstRW<[LNLPWriteResGroup32], (instrs SHA1RNDS4rri,
+                                            SHA256RNDS2rr)>;
+def LNLPWriteResGroup33 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 12 ; // orignally  99; diff lat for 1 & 2 version
+}
+def : InstRW<[LNLPWriteResGroup33, ReadAfterVecXLd], (instregex "^SHA256MSG(1|2)rm$")>;
+
+def LNLPWriteResGroup34 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 5;  //  orignally 99; Diff lat for 1 & 2 version
+}
+def : InstRW<[LNLPWriteResGroup34], (instregex "^SHA256MSG(1|2)rr$")>;
+def LNLPWriteResGroup35 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = 9;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup35, ReadAfterVecXLd], (instrs SHA256RNDS2rm)>;
+def LNLPWriteResGroup36 : SchedWriteRes<[LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [7];
+  let Latency = 7;
+}
+def : InstRW<[LNLPWriteResGroup36], (instregex "^VBROADCASTS(D|S)Yrm$",
+                                               "^VMOV(D|SH|SL)DUPYrm$",
+                                               "^VPBROADCAST(D|Q|W)Yrm$")>;
+def : InstRW<[LNLPWriteResGroup36], (instrs VBROADCASTF128rm)>;
+def LNLPWriteResGroup37 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 7];
+  let Latency = 11;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup37, ReadAfterVecYLd], (instregex "^VCMPP(D|S)Yrmi$")>;
+def LNLPWriteResGroup38 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 6];
+  let Latency = 10;
+}
+def : InstRW<[LNLPWriteResGroup38], (instregex "^VCVT(T?)PS2DQrm$")>;
+def LNLPWriteResGroup39 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [4, 1, 6];
+  let Latency = 11;
+  let NumMicroOps = 3;
+}
+def : InstRW<[LNLPWriteResGroup39, ReadAfterVecLd], (instrs VCVTSI642SSrm_Int)>;
+def LNLPWriteResGroup40 : SchedWriteRes<[LNLPPort10_11, LNLPPort25_26_27]> {
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup40], (instregex "^VEXTRACT(F|I)128mri$")>;
+def LNLPWriteResGroup41 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup41, ReadAfterVecYLd], (instregex "^VINSERT(F|I)128rmi$")>;
+def LNLPWriteResGroup42 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = 4;
+}
+def : InstRW<[LNLPWriteResGroup42], (instregex "^VMOVMSKP(D|S)Yrr$")>;
+def LNLPWriteResGroup43 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+}
+def : InstRW<[LNLPWriteResGroup43, ReadAfterVecYLd], (instregex "^VPACK(S|U)S(DW|WB)Yrm$")>;
+def : InstRW<[LNLPWriteResGroup43, ReadAfterVecYLd], (instrs VPCMPGTQYrm)>;
+def LNLPWriteResGroup44 : SchedWriteRes<[LNLPVPort00_01_02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+def : InstRW<[LNLPWriteResGroup44, ReadAfterVecYLd], (instregex "^VP(ADD|SUB)(B|D|Q|W)Yrm$")>;
+def LNLPWriteResGroup45 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup45], (instregex "^VPSHUF(D|HW|LW)Ymi$")>;
+def : InstRW<[LNLPWriteResGroup45, ReadAfterVecYLd], (instrs VPALIGNRYrmi,
+                                                             VPBLENDWYrmi)>;
+def LNLPWriteResGroup46 : SchedWriteRes<[LNLPVPort02_03]>;
+def : InstRW<[LNLPWriteResGroup46], (instregex "^VPBLENDW(Y?)rri$")>;
+def LNLPWriteResGroup47 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 7];
+  let Latency = 8;
+}
+def : InstRW<[LNLPWriteResGroup47], (instrs VPBROADCASTBYrm)>;
+def : InstRW<[LNLPWriteResGroup47, ReadAfterVecYLd], (instregex "^VUNPCK(H|L)P(D|S)Yrm$")>;
+def LNLPWriteResGroup48 : SchedWriteRes<[LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup48, ReadAfterVecXLd], (instrs VPCLMULQDQYrmi)>;
+def LNLPWriteResGroup49 : SchedWriteRes<[LNLPVPort00_01, LNLPVPort02_03, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [1, 2, 7];
+  let Latency = 9;
+  let NumMicroOps = 4;
+}
+def : InstRW<[LNLPWriteResGroup49, ReadAfterVecYLd], (instregex "^VPH(ADD|SUB)SWYrm$")>;
+def LNLPWriteResGroup50 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 7];
+  let Latency = 10;
+}
+def : InstRW<[LNLPWriteResGroup50, ReadAfterVecYLd], (instregex "^VPMUL(U?)DQYrm$")>;
+def LNLPWriteResGroup51 : SchedWriteRes<[LNLPVPort02_03]> {
+  let ReleaseAtCycles = [3];
+  let Latency = LunarlakePModel.MaxLatency;
+}
+def : InstRW<[LNLPWriteResGroup51], (instrs VSHA512RNDS2rr)>;
+def LNLPWriteResGroup52 : SchedWriteRes<[LNLPVPort00_01, LNLPPort20_21_22]> {
+  let ReleaseAtCycles = [3, 6];
+  let Latency = LunarlakePModel.MaxLatency;
+  let NumMicroOps = 2;
+}
+def : InstRW<[LNLPWriteResGroup52], (instregex "^VSM3MSG(1|2)rm$")>;
+def LNLPWriteResGroup53 : SchedWriteRes<[LNLPVPort00_01]> {
+  let ReleaseAtCycles = [3];
+  let Latency = LunarlakePModel.MaxLatency;
+}
+def : InstRW<[LNLPWriteResGroup53], (instregex "^VSM3MSG(1|2)rr$")>;
+def : InstRW<[LNLPWriteResGroup53], (instrs VTESTPSYrr)>;
+}
diff --git a/llvm/lib/Target/X86/X86TargetObjectFile.cpp b/llvm/lib/Target/X86/X86TargetObjectFile.cpp
index 4be0111a2195..fc3c06fcc1c9 100644
--- a/llvm/lib/Target/X86/X86TargetObjectFile.cpp
+++ b/llvm/lib/Target/X86/X86TargetObjectFile.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86TargetObjectFile.h"
-#include "MCTargetDesc/X86MCExpr.h"
+#include "MCTargetDesc/X86MCAsmInfo.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCValue.h"
@@ -25,7 +25,7 @@ const MCExpr *X86_64MachoTargetObjectFile::getTTypeGlobalReference(
   if ((Encoding & DW_EH_PE_indirect) && (Encoding & DW_EH_PE_pcrel)) {
     const MCSymbol *Sym = TM.getSymbol(GV);
     const MCExpr *Res =
-        MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+        MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
     const MCExpr *Four = MCConstantExpr::create(4, getContext());
     return MCBinaryExpr::createAdd(Res, Four, getContext());
   }
@@ -48,18 +48,18 @@ const MCExpr *X86_64MachoTargetObjectFile::getIndirectSymViaGOTPCRel(
   // foo@GOTPCREL+4+<offset>.
   unsigned FinalOff = Offset+MV.getConstant()+4;
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+      MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOff, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
 
 X86ELFTargetObjectFile::X86ELFTargetObjectFile() {
-  PLTRelativeSpecifier = X86MCExpr::VK_PLT;
+  PLTRelativeSpecifier = X86::S_PLT;
 }
 
 const MCExpr *X86ELFTargetObjectFile::getDebugThreadLocalSymbol(
     const MCSymbol *Sym) const {
-  return MCSymbolRefExpr::create(Sym, X86MCExpr::VK_DTPOFF, getContext());
+  return MCSymbolRefExpr::create(Sym, X86::S_DTPOFF, getContext());
 }
 
 const MCExpr *X86_64ELFTargetObjectFile::getIndirectSymViaGOTPCRel(
@@ -67,7 +67,7 @@ const MCExpr *X86_64ELFTargetObjectFile::getIndirectSymViaGOTPCRel(
     int64_t Offset, MachineModuleInfo *MMI, MCStreamer &Streamer) const {
   int64_t FinalOffset = Offset + MV.getConstant();
   const MCExpr *Res =
-      MCSymbolRefExpr::create(Sym, X86MCExpr::VK_GOTPCREL, getContext());
+      MCSymbolRefExpr::create(Sym, X86::S_GOTPCREL, getContext());
   const MCExpr *Off = MCConstantExpr::create(FinalOffset, getContext());
   return MCBinaryExpr::createAdd(Res, Off, getContext());
 }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0d60be108ccf..8573738bf5dc 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1676,8 +1676,10 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
         SubLT.second == MVT::f32 && (Index == 0 || ST->hasSSE41()))
       return 1;
 
-    // If the insertion isn't aligned, treat it like a 2-op shuffle.
-    Kind = TTI::SK_PermuteTwoSrc;
+    // If the insertion is the lowest subvector then it will be blended
+    // otherwise treat it like a 2-op shuffle.
+    Kind =
+        (Index == 0 && LT.first == 1) ? TTI::SK_Select : TTI::SK_PermuteTwoSrc;
   }
 
   // Handle some common (illegal) sub-vector types as they are often very cheap
@@ -5446,7 +5448,8 @@ X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
 
   else if (LT.first * Ty.getVectorNumElements() > NumElem) {
     auto *NewMaskTy = FixedVectorType::get(MaskTy->getElementType(),
-                                           Ty.getVectorNumElements());
+                                           (unsigned)LT.first.getValue() *
+                                               Ty.getVectorNumElements());
     // Expanding requires fill mask with zeroes
     Cost += getShuffleCost(TTI::SK_InsertSubvector, NewMaskTy, NewMaskTy, {},
                            CostKind, 0, MaskTy);
diff --git a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
index 707c4a790872..dfd138b99ce0 100644
--- a/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
+++ b/llvm/lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp
@@ -62,7 +62,7 @@ static void printExpr(const MCExpr *Expr, const MCAsmInfo *MAI,
     SRE = dyn_cast<MCSymbolRefExpr>(Expr);
     assert(SRE && "Unexpected MCExpr type.");
   }
-  assert(SRE->getKind() == MCSymbolRefExpr::VK_None);
+  assert(SRE->getSpecifier() == 0);
 
   SRE->getSymbol().print(OS, MAI);
 
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index 1c6e294597c3..ef4cfcd69415 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1064,9 +1064,8 @@ SDValue XCoreTargetLowering::LowerCCCCallTo(
   // The InGlue in necessary since all emitted instructions must be
   // stuck together.
   SDValue InGlue;
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
-    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
-                             RegsToPass[i].second, InGlue);
+  for (const auto &[Reg, N] : RegsToPass) {
+    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
     InGlue = Chain.getValue(1);
   }
 
@@ -1089,9 +1088,8 @@ SDValue XCoreTargetLowering::LowerCCCCallTo(
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
-  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
-    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
-                                  RegsToPass[i].second.getValueType()));
+  for (const auto &[Reg, N] : RegsToPass)
+    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
   if (InGlue.getNode())
     Ops.push_back(InGlue);
diff --git a/llvm/lib/Target/XCore/XCoreMCInstLower.cpp b/llvm/lib/Target/XCore/XCoreMCInstLower.cpp
index 6f5dcb291e6e..8f5096117d94 100644
--- a/llvm/lib/Target/XCore/XCoreMCInstLower.cpp
+++ b/llvm/lib/Target/XCore/XCoreMCInstLower.cpp
@@ -31,7 +31,6 @@ void XCoreMCInstLower::Initialize(MCContext *C) { Ctx = C; }
 MCOperand XCoreMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
                                                MachineOperandType MOTy,
                                                unsigned Offset) const {
-  MCSymbolRefExpr::VariantKind Kind = MCSymbolRefExpr::VK_None;
   const MCSymbol *Symbol;
 
   switch (MOTy) {
@@ -61,8 +60,7 @@ MCOperand XCoreMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
       llvm_unreachable("<unknown operand type>");
   }
 
-  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, Kind, *Ctx);
-
+  const MCSymbolRefExpr *MCSym = MCSymbolRefExpr::create(Symbol, *Ctx);
   if (!Offset)
     return MCOperand::createExpr(MCSym);
 
diff --git a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
index bc34ab431969..1bd92a2b4947 100644
--- a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
@@ -39,14 +39,17 @@ SDValue XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(
     Entry.Node = Src; Args.push_back(Entry);
     Entry.Node = Size; Args.push_back(Entry);
 
+    const char *MemcpyAlign4Name = TLI.getLibcallName(RTLIB::MEMCPY_ALIGN_4);
+    CallingConv::ID CC = TLI.getLibcallCallingConv(RTLIB::MEMCPY_ALIGN_4);
+
     TargetLowering::CallLoweringInfo CLI(DAG);
     CLI.setDebugLoc(dl)
         .setChain(Chain)
-        .setLibCallee(TLI.getLibcallCallingConv(RTLIB::MEMCPY),
-                      Type::getVoidTy(*DAG.getContext()),
-                      DAG.getExternalSymbol(
-                          "__memcpy_4", TLI.getPointerTy(DAG.getDataLayout())),
-                      std::move(Args))
+        .setLibCallee(
+            CC, Type::getVoidTy(*DAG.getContext()),
+            DAG.getExternalSymbol(MemcpyAlign4Name,
+                                  TLI.getPointerTy(DAG.getDataLayout())),
+            std::move(Args))
         .setDiscardResult();
 
     std::pair<SDValue,SDValue> CallResult = TLI.LowerCallTo(CLI);
diff --git a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
index 4103ff6b877a..4001b2cfa2a6 100644
--- a/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
+++ b/llvm/lib/Target/Xtensa/AsmParser/XtensaAsmParser.cpp
@@ -8,7 +8,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/XtensaMCExpr.h"
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "MCTargetDesc/XtensaTargetStreamer.h"
 #include "TargetInfo/XtensaTargetInfo.h"
@@ -302,10 +302,10 @@ struct XtensaOperand : public MCParsedAsmOperand {
     return Tok;
   }
 
-  void print(raw_ostream &OS) const override {
+  void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
     switch (Kind) {
     case Immediate:
-      OS << *getImm();
+      MAI.printExpr(OS, *getImm());
       break;
     case Register:
       OS << "<register x";
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
index deefb22e47a0..8ae48d51891e 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaAsmBackend.cpp
@@ -12,10 +12,10 @@
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -34,13 +34,13 @@ class XtensaAsmBackend : public MCAsmBackend {
         IsLittleEndian(isLE) {}
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
+  std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
+                                    uint64_t &) override;
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   MutableArrayRef<char> Data, uint64_t Value,
                   bool IsResolved) override;
   bool mayNeedRelaxation(const MCInst &Inst,
                          const MCSubtargetInfo &STI) const override;
-  void relaxInstruction(MCInst &Inst,
-                        const MCSubtargetInfo &STI) const override;
   bool writeNopData(raw_ostream &OS, uint64_t Count,
                     const MCSubtargetInfo *STI) const override;
 
@@ -53,17 +53,14 @@ class XtensaAsmBackend : public MCAsmBackend {
 MCFixupKindInfo XtensaAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[Xtensa::NumTargetFixupKinds] = {
       // name                     offset bits  flags
-      {"fixup_xtensa_branch_6", 0, 16, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_branch_8", 16, 8, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_branch_12", 12, 12, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_jump_18", 6, 18, MCFixupKindInfo::FKF_IsPCRel},
-      {"fixup_xtensa_call_18", 6, 18,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_xtensa_l32r_16", 8, 16,
-       MCFixupKindInfo::FKF_IsPCRel |
-           MCFixupKindInfo::FKF_IsAlignedDownTo32Bits},
-      {"fixup_xtensa_loop_8", 16, 8, MCFixupKindInfo::FKF_IsPCRel}};
+      {"fixup_xtensa_branch_6", 0, 16, 0},
+      {"fixup_xtensa_branch_8", 16, 8, 0},
+      {"fixup_xtensa_branch_12", 12, 12, 0},
+      {"fixup_xtensa_jump_18", 6, 18, 0},
+      {"fixup_xtensa_call_18", 6, 18, 0},
+      {"fixup_xtensa_l32r_16", 8, 16, 0},
+      {"fixup_xtensa_loop_8", 16, 8, 0},
+  };
 
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
@@ -136,17 +133,32 @@ static unsigned getSize(unsigned Kind) {
   switch (Kind) {
   default:
     return 3;
-  case MCFixupKind::FK_Data_4:
+  case FK_Data_4:
     return 4;
   case Xtensa::fixup_xtensa_branch_6:
     return 2;
   }
 }
 
-void XtensaAsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
+std::optional<bool> XtensaAsmBackend::evaluateFixup(const MCFragment &F,
+                                                    MCFixup &Fixup, MCValue &,
+                                                    uint64_t &Value) {
+  // For a few PC-relative fixups, offsets need to be aligned down. We
+  // compensate here because the default handler's `Value` decrement doesn't
+  // account for this alignment.
+  switch (Fixup.getTargetKind()) {
+  case Xtensa::fixup_xtensa_call_18:
+  case Xtensa::fixup_xtensa_l32r_16:
+    Value = (Asm->getFragmentOffset(F) + Fixup.getOffset()) % 4;
+  }
+  return {};
+}
+
+void XtensaAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                   const MCValue &Target,
                                   MutableArrayRef<char> Data, uint64_t Value,
                                   bool IsResolved) {
+  maybeAddReloc(F, Fixup, Target, Value, IsResolved);
   MCContext &Ctx = getContext();
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
 
@@ -171,9 +183,6 @@ bool XtensaAsmBackend::mayNeedRelaxation(const MCInst &Inst,
   return false;
 }
 
-void XtensaAsmBackend::relaxInstruction(MCInst &Inst,
-                                        const MCSubtargetInfo &STI) const {}
-
 bool XtensaAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
                                     const MCSubtargetInfo *STI) const {
   uint64_t NumNops24b = Count / 3;
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
index f2c391b03bd4..e2b1c5683d50 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaELFObjectWriter.cpp
@@ -8,6 +8,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
@@ -45,10 +46,12 @@ XtensaObjectWriter::~XtensaObjectWriter() {}
 unsigned XtensaObjectWriter::getRelocType(const MCFixup &Fixup,
                                           const MCValue &Target,
                                           bool IsPCRel) const {
+  uint8_t Specifier = Target.getSpecifier();
 
   switch ((unsigned)Fixup.getKind()) {
   case FK_Data_4:
-    return ELF::R_XTENSA_32;
+    return Specifier == Xtensa::S_TPOFF ? ELF::R_XTENSA_TLS_TPOFF
+                                        : ELF::R_XTENSA_32;
   default:
     return ELF::R_XTENSA_SLOT0_OP;
   }
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
index 6f9f29765452..f1c11d47bc8b 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaInstPrinter.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "XtensaInstPrinter.h"
-#include "MCTargetDesc/XtensaMCExpr.h"
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
index 0b20f2e14a84..258f5cd7156a 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "XtensaMCAsmInfo.h"
-#include "XtensaMCExpr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
index 6f6f4bcb7047..3d681c14cc3c 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCAsmInfo.h
@@ -16,9 +16,11 @@
 #define LLVM_LIB_TARGET_XTENSA_MCTARGETDESC_XTENSATARGETASMINFO_H
 
 #include "llvm/MC/MCAsmInfoELF.h"
+#include "llvm/MC/MCExpr.h"
 
 namespace llvm {
 class Triple;
+class StringRef;
 
 class XtensaMCAsmInfo : public MCAsmInfoELF {
 public:
@@ -28,6 +30,13 @@ class XtensaMCAsmInfo : public MCAsmInfoELF {
                           const MCSpecifierExpr &Expr) const override;
 };
 
+namespace Xtensa {
+enum Specifier { S_None, S_TPOFF };
+
+uint8_t parseSpecifier(StringRef name);
+StringRef getSpecifierName(uint8_t S);
+} // namespace Xtensa
+
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_XTENSA_MCTARGETDESC_XTENSATARGETASMINFO_H
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
index 03b3ed0c121b..bd4d4ebd2a72 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCCodeEmitter.cpp
@@ -13,7 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/XtensaFixupKinds.h"
-#include "MCTargetDesc/XtensaMCExpr.h"
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
@@ -150,6 +150,22 @@ MCCodeEmitter *llvm::createXtensaMCCodeEmitter(const MCInstrInfo &MCII,
   return new XtensaMCCodeEmitter(MCII, Ctx, true);
 }
 
+static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
+                     const MCExpr *Value, uint16_t Kind) {
+  bool PCRel = false;
+  switch (Kind) {
+  case Xtensa::fixup_xtensa_branch_6:
+  case Xtensa::fixup_xtensa_branch_8:
+  case Xtensa::fixup_xtensa_branch_12:
+  case Xtensa::fixup_xtensa_jump_18:
+  case Xtensa::fixup_xtensa_call_18:
+  case Xtensa::fixup_xtensa_l32r_16:
+  case Xtensa::fixup_xtensa_loop_8:
+    PCRel = true;
+  }
+  Fixups.push_back(MCFixup::create(Offset, Value, Kind, PCRel));
+}
+
 void XtensaMCCodeEmitter::encodeInstruction(const MCInst &MI,
                                             SmallVectorImpl<char> &CB,
                                             SmallVectorImpl<MCFixup> &Fixups,
@@ -195,8 +211,7 @@ XtensaMCCodeEmitter::getJumpTargetEncoding(const MCInst &MI, unsigned int OpNum,
     return MO.getImm();
 
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_jump_18), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_jump_18);
   return 0;
 }
 
@@ -213,17 +228,14 @@ uint32_t XtensaMCCodeEmitter::getBranchTargetEncoding(
   case Xtensa::BGEZ:
   case Xtensa::BLTZ:
   case Xtensa::BNEZ:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_12), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_12);
     return 0;
   case Xtensa::BEQZ_N:
   case Xtensa::BNEZ_N:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_6), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_6);
     return 0;
   default:
-    Fixups.push_back(MCFixup::create(
-        0, Expr, MCFixupKind(Xtensa::fixup_xtensa_branch_8), MI.getLoc()));
+    addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_branch_8);
     return 0;
   }
 }
@@ -240,8 +252,7 @@ XtensaMCCodeEmitter::getLoopTargetEncoding(const MCInst &MI, unsigned int OpNum,
 
   const MCExpr *Expr = MO.getExpr();
 
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_loop_8), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_loop_8);
   return 0;
 }
 
@@ -261,8 +272,7 @@ XtensaMCCodeEmitter::getCallEncoding(const MCInst &MI, unsigned int OpNum,
 
   assert((MO.isExpr()) && "Unexpected operand value!");
   const MCExpr *Expr = MO.getExpr();
-  Fixups.push_back(MCFixup::create(
-      0, Expr, MCFixupKind(Xtensa::fixup_xtensa_call_18), MI.getLoc()));
+  addFixup(Fixups, 0, Expr, Xtensa::fixup_xtensa_call_18);
   return 0;
 }
 
@@ -281,8 +291,7 @@ XtensaMCCodeEmitter::getL32RTargetEncoding(const MCInst &MI, unsigned OpNum,
 
   assert((MO.isExpr()) && "Unexpected operand value!");
 
-  Fixups.push_back(MCFixup::create(
-      0, MO.getExpr(), MCFixupKind(Xtensa::fixup_xtensa_l32r_16), MI.getLoc()));
+  addFixup(Fixups, 0, MO.getExpr(), Xtensa::fixup_xtensa_l32r_16);
   return 0;
 }
 
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
deleted file mode 100644
index 5a7b1ee9880f..000000000000
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCExpr.h
+++ /dev/null
@@ -1,33 +0,0 @@
-//===-- XtensaMCExpr.h - Xtensa specific MC expression classes --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file describes Xtensa-specific MCExprs
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_XTENSA_MCTARGETDESC_XTENSAMCEXPR_H
-#define LLVM_LIB_TARGET_XTENSA_MCTARGETDESC_XTENSAMCEXPR_H
-
-#include "llvm/MC/MCExpr.h"
-
-namespace llvm {
-
-class StringRef;
-
-namespace Xtensa {
-enum Specifier { S_None, S_TPOFF };
-
-uint8_t parseSpecifier(StringRef name);
-StringRef getSpecifierName(uint8_t S);
-} // namespace Xtensa
-
-} // end namespace llvm.
-
-#endif // LLVM_LIB_TARGET_XTENSA_MCTARGETDESC_XTENSAMCEXPR_H
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
index 9a55635674d7..821cba0fc25c 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
@@ -186,11 +186,17 @@ bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits,
     return FeatureBits[Xtensa::FeatureMiscSR];
   case Xtensa::PRID:
     return RAType == Xtensa::REGISTER_READ && FeatureBits[Xtensa::FeaturePRID];
+  case Xtensa::THREADPTR:
+    return FeatureBits[FeatureTHREADPTR];
   case Xtensa::VECBASE:
     return FeatureBits[Xtensa::FeatureRelocatableVector];
   case Xtensa::FCR:
   case Xtensa::FSR:
     return FeatureBits[FeatureSingleFloat];
+  case Xtensa::F64R_LO:
+  case Xtensa::F64R_HI:
+  case Xtensa::F64S:
+    return FeatureBits[FeatureDFPAccel];
   case Xtensa::WINDOWBASE:
   case Xtensa::WINDOWSTART:
     return FeatureBits[Xtensa::FeatureWindowed];
@@ -203,12 +209,23 @@ bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits,
 
 // Get Xtensa User Register by encoding value.
 MCRegister Xtensa::getUserRegister(unsigned Code, const MCRegisterInfo &MRI) {
+  MCRegister UserReg = Xtensa::NoRegister;
+
   if (MRI.getEncodingValue(Xtensa::FCR) == Code) {
-    return Xtensa::FCR;
+    UserReg = Xtensa::FCR;
   } else if (MRI.getEncodingValue(Xtensa::FSR) == Code) {
-    return Xtensa::FSR;
+    UserReg = Xtensa::FSR;
+  } else if (MRI.getEncodingValue(Xtensa::F64R_LO) == Code) {
+    UserReg = Xtensa::F64R_LO;
+  } else if (MRI.getEncodingValue(Xtensa::F64R_HI) == Code) {
+    UserReg = Xtensa::F64R_HI;
+  } else if (MRI.getEncodingValue(Xtensa::F64S) == Code) {
+    UserReg = Xtensa::F64S;
+  } else if (MRI.getEncodingValue(Xtensa::THREADPTR) == Code) {
+    UserReg = Xtensa::THREADPTR;
   }
-  return Xtensa::NoRegister;
+
+  return UserReg;
 }
 
 static MCAsmInfo *createXtensaMCAsmInfo(const MCRegisterInfo &MRI,
diff --git a/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp b/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
index d31a256dd00f..4d07400e996b 100644
--- a/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaAsmPrinter.cpp
@@ -13,7 +13,7 @@
 
 #include "XtensaAsmPrinter.h"
 #include "MCTargetDesc/XtensaInstPrinter.h"
-#include "MCTargetDesc/XtensaMCExpr.h"
+#include "MCTargetDesc/XtensaMCAsmInfo.h"
 #include "MCTargetDesc/XtensaTargetStreamer.h"
 #include "TargetInfo/XtensaTargetInfo.h"
 #include "XtensaConstantPoolValue.h"
diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
index d48f494388a2..97d5472f3e96 100644
--- a/llvm/lib/Target/Xtensa/XtensaFeatures.td
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -98,6 +98,11 @@ def FeatureDataCache : SubtargetFeature<"dcache", "HasDataCache", "true",
 def HasDataCache : Predicate<"Subtarget->hasDataCache()">,
                    AssemblerPredicate<(all_of FeatureDataCache)>;
 
+def FeatureTHREADPTR : SubtargetFeature<"threadptr", "HasTHREADPTR", "true",
+                                        "Enable Xtensa THREADPTR option">;
+def HasTHREADPTR : Predicate<"Subtarget->hasTHREADPTR()">,
+                   AssemblerPredicate<(all_of FeatureTHREADPTR)>;
+
 // Xtensa Interrupts Options.
 def FeatureHighPriInterrupts : SubtargetFeature<"highpriinterrupts",
                                                 "HasHighPriInterrupts", "true",
@@ -137,3 +142,8 @@ def FeatureCoprocessor : SubtargetFeature<"coprocessor", "HasCoprocessor", "true
                                           "Enable Xtensa Coprocessor option">;
 def HasCoprocessor : Predicate<"Subtarget->hasCoprocessor()">,
                                 AssemblerPredicate<(all_of FeatureCoprocessor)>;
+
+def FeatureDFPAccel : SubtargetFeature<"dfpaccel", "HasDFPAccel", "true",
+                                       "Enable Xtensa Double Precision FP acceleration">;
+def HasDFPAccel : Predicate<"Subtarget->hasDFPAccel()">,
+                  AssemblerPredicate<(all_of FeatureDFPAccel)>;
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index d51c573282da..fd42fd2e010b 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -101,6 +101,7 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::ConstantPool, PtrVT, Custom);
   setOperationAction(ISD::GlobalAddress, PtrVT, Custom);
+  setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
   setOperationAction(ISD::BlockAddress, PtrVT, Custom);
   setOperationAction(ISD::JumpTable, PtrVT, Custom);
 
@@ -110,12 +111,18 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::BR_CC, MVT::i32, Legal);
   setOperationAction(ISD::BR_CC, MVT::i64, Expand);
-  setOperationAction(ISD::BR_CC, MVT::f32, Expand);
 
   setOperationAction(ISD::SELECT, MVT::i32, Expand);
   setOperationAction(ISD::SELECT, MVT::f32, Expand);
   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
-  setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
+
+  if (Subtarget.hasSingleFloat()) {
+    setOperationAction(ISD::BR_CC, MVT::f32, Legal);
+    setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
+  } else {
+    setOperationAction(ISD::BR_CC, MVT::f32, Expand);
+    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
+  }
 
   setOperationAction(ISD::SETCC, MVT::i32, Expand);
   setOperationAction(ISD::SETCC, MVT::f32, Expand);
@@ -841,21 +848,68 @@ static unsigned getBranchOpcode(ISD::CondCode Cond) {
   }
 }
 
+static std::pair<unsigned, unsigned> getFPBranchKind(ISD::CondCode Cond) {
+  switch (Cond) {
+  case ISD::SETUNE:
+    return std::make_pair(Xtensa::BF, Xtensa::OEQ_S);
+  case ISD::SETUO:
+    return std::make_pair(Xtensa::BT, Xtensa::UN_S);
+  case ISD::SETO:
+    return std::make_pair(Xtensa::BF, Xtensa::UN_S);
+  case ISD::SETUEQ:
+    return std::make_pair(Xtensa::BT, Xtensa::UEQ_S);
+  case ISD::SETULE:
+    return std::make_pair(Xtensa::BT, Xtensa::ULE_S);
+  case ISD::SETULT:
+    return std::make_pair(Xtensa::BT, Xtensa::ULT_S);
+  case ISD::SETEQ:
+  case ISD::SETOEQ:
+    return std::make_pair(Xtensa::BT, Xtensa::OEQ_S);
+  case ISD::SETNE:
+    return std::make_pair(Xtensa::BF, Xtensa::OEQ_S);
+  case ISD::SETLE:
+  case ISD::SETOLE:
+    return std::make_pair(Xtensa::BT, Xtensa::OLE_S);
+  case ISD::SETLT:
+  case ISD::SETOLT:
+    return std::make_pair(Xtensa::BT, Xtensa::OLT_S);
+  case ISD::SETGE:
+    return std::make_pair(Xtensa::BF, Xtensa::OLT_S);
+  case ISD::SETGT:
+    return std::make_pair(Xtensa::BF, Xtensa::OLE_S);
+  default:
+    llvm_unreachable("Invalid condition!");
+  }
+}
+
 SDValue XtensaTargetLowering::LowerSELECT_CC(SDValue Op,
                                              SelectionDAG &DAG) const {
   SDLoc DL(Op);
-  EVT Ty = Op.getOperand(0).getValueType();
+  EVT Ty = Op.getValueType();
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
   SDValue TrueValue = Op.getOperand(2);
   SDValue FalseValue = Op.getOperand(3);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op->getOperand(4))->get();
 
-  unsigned BrOpcode = getBranchOpcode(CC);
-  SDValue TargetCC = DAG.getConstant(BrOpcode, DL, MVT::i32);
+  if (LHS.getValueType() == MVT::i32) {
+    unsigned BrOpcode = getBranchOpcode(CC);
+    SDValue TargetCC = DAG.getConstant(BrOpcode, DL, MVT::i32);
 
-  return DAG.getNode(XtensaISD::SELECT_CC, DL, Ty, LHS, RHS, TrueValue,
-                     FalseValue, TargetCC);
+    SDValue Res = DAG.getNode(XtensaISD::SELECT_CC, DL, Ty, LHS, RHS, TrueValue,
+                              FalseValue, TargetCC, Op->getFlags());
+    return Res;
+  }
+  assert(LHS.getValueType() == MVT::f32 &&
+         "We expect MVT::f32 type of the LHS Operand in SELECT_CC");
+  unsigned BrOpcode;
+  unsigned CmpOpCode;
+  std::tie(BrOpcode, CmpOpCode) = getFPBranchKind(CC);
+  SDValue TargetCC = DAG.getConstant(CmpOpCode, DL, MVT::i32);
+  SDValue TargetBC = DAG.getConstant(BrOpcode, DL, MVT::i32);
+  return DAG.getNode(XtensaISD::SELECT_CC_FP, DL, Ty,
+                     {LHS, RHS, TrueValue, FalseValue, TargetCC, TargetBC},
+                     Op->getFlags());
 }
 
 SDValue XtensaTargetLowering::LowerRETURNADDR(SDValue Op,
@@ -919,6 +973,58 @@ SDValue XtensaTargetLowering::LowerGlobalAddress(SDValue Op,
   return Res;
 }
 
+SDValue XtensaTargetLowering::LowerGlobalTLSAddress(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  const GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
+  SDLoc DL(Op);
+  EVT PtrVT = Op.getValueType();
+  const GlobalValue *GV = G->getGlobal();
+
+  if (DAG.getTarget().useEmulatedTLS())
+    return LowerToTLSEmulatedModel(G, DAG);
+
+  TLSModel::Model model = getTargetMachine().getTLSModel(GV);
+
+  if (!Subtarget.hasTHREADPTR()) {
+    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+        DAG.getMachineFunction().getFunction(), "only emulated TLS supported",
+        DL.getDebugLoc()));
+    return DAG.getPOISON(Op->getValueType(0));
+  }
+
+  if (model == TLSModel::LocalExec || model == TLSModel::InitialExec) {
+    bool Priv = GV->isPrivateLinkage(GV->getLinkage());
+    MachineFunction &MF = DAG.getMachineFunction();
+    XtensaMachineFunctionInfo *XtensaFI =
+        MF.getInfo<XtensaMachineFunctionInfo>();
+    unsigned LabelId = XtensaFI->createCPLabelId();
+
+    // Create a constant pool entry for the callee address
+    XtensaConstantPoolValue *CPV = XtensaConstantPoolSymbol::Create(
+        *DAG.getContext(), GV->getName().str().c_str(), LabelId, Priv,
+        XtensaCP::TPOFF);
+
+    // Get the address of the callee into a register
+    SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
+    SDValue CPWrap = getAddrPCRel(CPAddr, DAG);
+    SDValue Addr = DAG.getLoad(
+        PtrVT, DL, DAG.getEntryNode(), CPWrap,
+        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
+
+    SDValue TPRegister = DAG.getRegister(Xtensa::THREADPTR, MVT::i32);
+    SDValue ThreadPointer =
+        DAG.getNode(XtensaISD::RUR, DL, MVT::i32, TPRegister);
+
+    return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadPointer, Addr);
+  }
+
+  DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+      DAG.getMachineFunction().getFunction(),
+      "only local-exec and initial-exec TLS mode supported", DL.getDebugLoc()));
+
+  return DAG.getPOISON(Op->getValueType(0));
+}
+
 SDValue XtensaTargetLowering::LowerBlockAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
   BlockAddressSDNode *Node = cast<BlockAddressSDNode>(Op);
@@ -1353,6 +1459,8 @@ SDValue XtensaTargetLowering::LowerOperation(SDValue Op,
     return LowerRETURNADDR(Op, DAG);
   case ISD::GlobalAddress:
     return LowerGlobalAddress(Op, DAG);
+  case ISD::GlobalTLSAddress:
+    return LowerGlobalTLSAddress(Op, DAG);
   case ISD::BlockAddress:
     return LowerBlockAddress(Op, DAG);
   case ISD::JumpTable:
@@ -1406,8 +1514,12 @@ const char *XtensaTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "XtensaISD::RET";
   case XtensaISD::RETW:
     return "XtensaISD::RETW";
+  case XtensaISD::RUR:
+    return "XtensaISD::RUR";
   case XtensaISD::SELECT_CC:
     return "XtensaISD::SELECT_CC";
+  case XtensaISD::SELECT_CC_FP:
+    return "XtensaISD::SELECT_CC_FP";
   case XtensaISD::SRCL:
     return "XtensaISD::SRCL";
   case XtensaISD::SRCR:
@@ -1450,7 +1562,6 @@ XtensaTargetLowering::emitSelectCC(MachineInstr &MI,
   MachineOperand &RHS = MI.getOperand(2);
   MachineOperand &TrueValue = MI.getOperand(3);
   MachineOperand &FalseValue = MI.getOperand(4);
-  unsigned BrKind = MI.getOperand(5).getImm();
 
   // To "insert" a SELECT_CC instruction, we actually have to insert
   // CopyMBB and SinkMBB  blocks and add branch to MBB. We build phi
@@ -1482,10 +1593,25 @@ XtensaTargetLowering::emitSelectCC(MachineInstr &MI,
   MBB->addSuccessor(CopyMBB);
   MBB->addSuccessor(SinkMBB);
 
-  BuildMI(MBB, DL, TII.get(BrKind))
-      .addReg(LHS.getReg())
-      .addReg(RHS.getReg())
-      .addMBB(SinkMBB);
+  if (MI.getOpcode() == Xtensa::SELECT_CC_FP_FP ||
+      MI.getOpcode() == Xtensa::SELECT_CC_FP_INT) {
+    unsigned CmpKind = MI.getOperand(5).getImm();
+    unsigned BrKind = MI.getOperand(6).getImm();
+    MCPhysReg BReg = Xtensa::B0;
+
+    BuildMI(MBB, DL, TII.get(CmpKind), BReg)
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg());
+    BuildMI(MBB, DL, TII.get(BrKind))
+        .addReg(BReg, RegState::Kill)
+        .addMBB(SinkMBB);
+  } else {
+    unsigned BrKind = MI.getOperand(5).getImm();
+    BuildMI(MBB, DL, TII.get(BrKind))
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg())
+        .addMBB(SinkMBB);
+  }
 
   CopyMBB->addSuccessor(SinkMBB);
 
@@ -1510,6 +1636,30 @@ MachineBasicBlock *XtensaTargetLowering::EmitInstrWithCustomInserter(
   const XtensaInstrInfo &TII = *Subtarget.getInstrInfo();
 
   switch (MI.getOpcode()) {
+  case Xtensa::BRCC_FP: {
+    MachineOperand &Cond = MI.getOperand(0);
+    MachineOperand &LHS = MI.getOperand(1);
+    MachineOperand &RHS = MI.getOperand(2);
+    MachineBasicBlock *TargetBB = MI.getOperand(3).getMBB();
+    unsigned BrKind = 0;
+    unsigned CmpKind = 0;
+    ISD::CondCode CondCode = (ISD::CondCode)Cond.getImm();
+    MCPhysReg BReg = Xtensa::B0;
+
+    std::tie(BrKind, CmpKind) = getFPBranchKind(CondCode);
+    BuildMI(*MBB, MI, DL, TII.get(CmpKind), BReg)
+        .addReg(LHS.getReg())
+        .addReg(RHS.getReg());
+    BuildMI(*MBB, MI, DL, TII.get(BrKind))
+        .addReg(BReg, RegState::Kill)
+        .addMBB(TargetBB);
+
+    MI.eraseFromParent();
+    return MBB;
+  }
+  case Xtensa::SELECT_CC_FP_FP:
+  case Xtensa::SELECT_CC_FP_INT:
+  case Xtensa::SELECT_CC_INT_FP:
   case Xtensa::SELECT:
     return emitSelectCC(MI, MBB);
   case Xtensa::S8I:
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.h b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
index b6f2ebe21c94..e6ddf9864932 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.h
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
@@ -45,11 +45,18 @@ enum {
   RET,
   RETW,
 
+  RUR,
+
   // Select with condition operator - This selects between a true value and
   // a false value (ops #2 and #3) based on the boolean result of comparing
   // the lhs and rhs (ops #0 and #1) of a conditional expression with the
   // condition code in op #4
   SELECT_CC,
+  // Select with condition operator - This selects between a true value and
+  // a false value (ops #2 and #3) based on the boolean result of comparing
+  // f32 operands lhs and rhs (ops #0 and #1) of a conditional expression
+  // with the condition code in op #4 and boolean branch kind in op #5
+  SELECT_CC_FP,
 
   // SRCL(R) performs shift left(right) of the concatenation of 2 registers
   // and returns high(low) 32-bit part of 64-bit result
@@ -156,6 +163,8 @@ class XtensaTargetLowering : public TargetLowering {
 
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
 
+  SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
+
   SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
index 896e2f8f1c01..55c0729a0c9e 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.cpp
@@ -261,6 +261,12 @@ bool XtensaInstrInfo::reverseBranchCondition(
   case Xtensa::BGEZ:
     Cond[0].setImm(Xtensa::BLTZ);
     return false;
+  case Xtensa::BF:
+    Cond[0].setImm(Xtensa::BT);
+    return false;
+  case Xtensa::BT:
+    Cond[0].setImm(Xtensa::BF);
+    return false;
   default:
     report_fatal_error("Invalid branch condition!");
   }
@@ -294,6 +300,9 @@ XtensaInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
   case Xtensa::BLTZ:
   case Xtensa::BGEZ:
     return MI.getOperand(1).getMBB();
+  case Xtensa::BT:
+  case Xtensa::BF:
+    return MI.getOperand(1).getMBB();
   default:
     llvm_unreachable("Unknown branch opcode");
   }
@@ -329,6 +338,10 @@ bool XtensaInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
   case Xtensa::BGEZ:
     BrOffset -= 4;
     return isIntN(12, BrOffset);
+  case Xtensa::BT:
+  case Xtensa::BF:
+    BrOffset -= 4;
+    return isIntN(8, BrOffset);
   default:
     llvm_unreachable("Unknown branch opcode");
   }
@@ -581,6 +594,10 @@ unsigned XtensaInstrInfo::insertConstBranchAtInst(
   case Xtensa::BGEZ:
     MI = BuildMI(MBB, I, DL, get(BR_C)).addImm(offset).addReg(Cond[1].getReg());
     break;
+  case Xtensa::BT:
+  case Xtensa::BF:
+    MI = BuildMI(MBB, I, DL, get(BR_C)).addImm(offset).addReg(Cond[1].getReg());
+    break;
   default:
     llvm_unreachable("Invalid branch type!");
   }
@@ -641,6 +658,10 @@ unsigned XtensaInstrInfo::insertBranchAtInst(MachineBasicBlock &MBB,
   case Xtensa::BGEZ:
     MI = BuildMI(MBB, I, DL, get(BR_C)).addReg(Cond[1].getReg()).addMBB(TBB);
     break;
+  case Xtensa::BT:
+  case Xtensa::BF:
+    MI = BuildMI(MBB, I, DL, get(BR_C)).addReg(Cond[1].getReg()).addMBB(TBB);
+    break;
   default:
     report_fatal_error("Invalid branch type!");
   }
@@ -689,6 +710,12 @@ bool XtensaInstrInfo::isBranch(const MachineBasicBlock::iterator &MI,
     Target = &MI->getOperand(1);
     return true;
 
+  case Xtensa::BT:
+  case Xtensa::BF:
+    Cond[0].setImm(OpCode);
+    Target = &MI->getOperand(1);
+    return true;
+
   default:
     assert(!MI->getDesc().isBranch() && "Unknown branch opcode");
     return false;
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 1335c6faff6b..31608f465936 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -416,7 +416,6 @@ def BBSI : RRI8_Inst<0x07, (outs),
 }
 
 def : Pat<(brcond AR:$s, bb:$target), (BNEZ AR:$s, bb:$target)>;
-
 //===----------------------------------------------------------------------===//
 // Call and jump instructions
 //===----------------------------------------------------------------------===//
@@ -576,7 +575,7 @@ def WUR : RRR_Inst<0x00, 0x03, 0x0F, (outs UR:$ur), (ins AR:$t),
 }
 
 def RUR : RRR_Inst<0x00, 0x03, 0x0E, (outs AR:$r), (ins UR:$ur),
-                  "rur\t$r, $ur", []> {
+                  "rur\t$r, $ur", [(set AR:$r, (Xtensa_rur UR:$ur))]> {
   bits<8> ur;
 
   let s = ur{7-4};
@@ -1310,6 +1309,34 @@ let AddedComplexity = 10 in
 def : Pat<(f32 (load (Xtensa_pcrel_wrapper tconstpool:$in))),
           (WFR (L32R tconstpool:$in))>;
 
+//===----------------------------------------------------------------------===//
+// SelectCC and BranchCC instructions with FP operands
+//===----------------------------------------------------------------------===//
+
+let usesCustomInserter = 1, Predicates = [HasSingleFloat] in {
+  def SELECT_CC_INT_FP : Pseudo<(outs FPR:$dst), (ins AR:$lhs, AR:$rhs, FPR:$t, FPR:$f, i32imm:$cond),
+                                "!select_cc_int_fp $dst, $lhs, $rhs, $t, $f, $cond",
+                                [(set FPR:$dst, (Xtensa_select_cc AR:$lhs, AR:$rhs, FPR:$t, FPR:$f, imm:$cond))]>;
+  def SELECT_CC_FP_INT : Pseudo<(outs AR:$dst), (ins FPR:$lhs, FPR:$rhs, AR:$t, AR:$f, i32imm:$cond, i32imm:$brkind),
+                                "!select_cc_fp_int $dst, $lhs, $rhs, $t, $f, $cond, $brkind",
+                                [(set AR:$dst, (Xtensa_select_cc_fp FPR:$lhs, FPR:$rhs, AR:$t, AR:$f, imm:$cond, imm:$brkind))]>;
+  def SELECT_CC_FP_FP : Pseudo<(outs FPR:$dst), (ins FPR:$lhs, FPR:$rhs, FPR:$t, FPR:$f, i32imm:$cond, i32imm:$brkind),
+                               "!select_cc_fp_fp $dst, $lhs, $rhs, $t, $f, $cond, $brkind",
+                               [(set FPR:$dst, (Xtensa_select_cc_fp FPR:$lhs, FPR:$rhs, FPR:$t, FPR:$f, imm:$cond, imm:$brkind))]>;
+}
+
+let usesCustomInserter = 1, isBranch = 1, isTerminator = 1, isBarrier = 1, Predicates = [HasSingleFloat] in {
+  def BRCC_FP : Pseudo<(outs), (ins i32imm:$cond, FPR:$lhs, FPR:$rhs, brtarget:$target),
+                       "!brcc_fp $cond, $lhs, $rhs, $target", []>;
+}
+
+def cond_as_i32imm : SDNodeXForm<cond, [{
+  return CurDAG->getTargetConstant(N->get(), SDLoc(N), MVT::i32);
+}]>;
+
+def : Pat<(brcc cond:$cond, FPR:$s, FPR:$t,  bb:$target),
+          (BRCC_FP (cond_as_i32imm $cond), FPR:$s, FPR:$t, bb:$target)>;
+
 //===----------------------------------------------------------------------===//
 // Region Protection feature instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaOperators.td b/llvm/lib/Target/Xtensa/XtensaOperators.td
index fea13c2298d9..56be3d51505e 100644
--- a/llvm/lib/Target/Xtensa/XtensaOperators.td
+++ b/llvm/lib/Target/Xtensa/XtensaOperators.td
@@ -21,14 +21,15 @@ def SDT_XtensaBrJT                : SDTypeProfile<0, 2,
                                                  [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
 
 def SDT_XtensaSelectCC            : SDTypeProfile<1, 5,
-                                                 [SDTCisSameAs<0, 1>,
-                                                  SDTCisSameAs<2, 3>,
+                                                 [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>,
+                                                  SDTCisSameAs<3, 4>,
                                                   SDTCisVT<5, i32>]>;
 
 def SDT_XtensaCmp                 : SDTypeProfile<1, 2, [SDTCisVT<0, v1i1>, SDTCisVT<1, f32>, SDTCisVT<2, f32>]>;
 def SDT_XtensaMADD                : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisVT<0, f32>]>;
 def SDT_XtensaMOVS                : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisVT<0, f32>]>;
-def SDT_XtensaSelectCCFP          : SDTypeProfile<1, 5, [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>, SDTCisVT<5, i32>]>;
+def SDT_XtensaSelectCCFP          : SDTypeProfile<1, 6, [SDTCisSameAs<0, 3>, SDTCisSameAs<1, 2>, SDTCisSameAs<3, 4>,
+                                                         SDTCisVT<5, i32>, SDTCisVT<6, i32>]>;
 
 def SDT_XtensaSRC                 : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                                          SDTCisVT<2, i32>, SDTCisVT<3, i32>]>;
@@ -38,6 +39,8 @@ def SDT_XtensaEXTUI               : SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCi
 
 def SDT_XtensaMOVSP               : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
 
+def SDT_XtensaRUR                : SDTypeProfile<1, 1, [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
+
 //===----------------------------------------------------------------------===//
 // Node definitions
 //===----------------------------------------------------------------------===//
@@ -76,6 +79,9 @@ def Xtensa_extui: SDNode<"XtensaISD::EXTUI", SDT_XtensaEXTUI>;
 def Xtensa_movsp: SDNode<"XtensaISD::MOVSP", SDT_XtensaMOVSP,
                         [SDNPHasChain, SDNPSideEffect, SDNPInGlue]>;
 
+def Xtensa_rur: SDNode<"XtensaISD::RUR", SDT_XtensaRUR,
+                      [SDNPInGlue]>;
+
 def Xtensa_cmpoeq     : SDNode<"XtensaISD::CMPOEQ", SDT_XtensaCmp, [SDNPOutGlue]>;
 def Xtensa_cmpolt     : SDNode<"XtensaISD::CMPOLT", SDT_XtensaCmp, [SDNPOutGlue]>;
 def Xtensa_cmpole     : SDNode<"XtensaISD::CMPOLE", SDT_XtensaCmp, [SDNPOutGlue]>;
@@ -87,3 +93,6 @@ def Xtensa_cmpuo      : SDNode<"XtensaISD::CMPUO", SDT_XtensaCmp, [SDNPOutGlue]>
 def Xtensa_madd: SDNode<"XtensaISD::MADD", SDT_XtensaMADD, [SDNPInGlue]>;
 def Xtensa_msub: SDNode<"XtensaISD::MSUB", SDT_XtensaMADD, [SDNPInGlue]>;
 def Xtensa_movs: SDNode<"XtensaISD::MOVS", SDT_XtensaMOVS, [SDNPInGlue]>;
+
+def Xtensa_select_cc_fp: SDNode<"XtensaISD::SELECT_CC_FP", SDT_XtensaSelectCCFP,
+                               [SDNPInGlue]>;
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
index 74633050861c..ed76431bf549 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.cpp
@@ -53,7 +53,10 @@ BitVector XtensaRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     // Reserve frame pointer.
     Reserved.set(getFrameRegister(MF));
   }
-
+  if (Subtarget.hasTHREADPTR()) {
+    // Reserve frame pointer.
+    Reserved.set(Xtensa::THREADPTR);
+  }
   // Reserve stack pointer.
   Reserved.set(Xtensa::SP);
   return Reserved;
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
index 644faee51f51..596c4105c111 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
@@ -234,10 +234,19 @@ class URReg<bits<8> num, string n, list<string> alt = []> : XtensaReg<n> {
   let AltNames = alt;
 }
 
+// Thread Pointer register
+def THREADPTR : URReg<231, "threadptr", ["THREADPTR"]>;
+
 def FCR : URReg<232, "fcr", ["FCR"]>;
 def FSR : URReg<233, "fsr", ["FSR"]>;
 
-def UR :  RegisterClass<"Xtensa", [i32], 32, (add FCR, FSR)>;
+// DFPAccel registers
+def F64R_LO : URReg<234, "f64r_lo", ["F64R_LO"]>;
+def F64R_HI : URReg<235, "f64r_hi", ["F64R_HI"]>;
+def F64S : URReg<236, "f64s", ["F64S"]>;
+
+def UR :  RegisterClass<"Xtensa", [i32], 32, (add
+  THREADPTR, FCR, FSR, F64R_LO, F64R_HI, F64S)>;
 
 //===----------------------------------------------------------------------===//
 // Floating-Point registers
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index bc32541750ec..fd677a451f3f 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -91,7 +91,7 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
   bool hasHighPriInterruptsLevel7() const { return HasHighPriInterruptsLevel7; }
   bool hasInterrupt() const { return HasInterrupt; }
   bool hasException() const { return HasException; }
-
+  bool hasTHREADPTR() const { return HasTHREADPTR; }
   bool isWindowedABI() const { return hasWindowed(); }
 
   // Automatically generated by tblgen.
diff --git a/llvm/lib/TargetParser/AArch64TargetParser.cpp b/llvm/lib/TargetParser/AArch64TargetParser.cpp
index 4a2523440f0f..c17fa729bac4 100644
--- a/llvm/lib/TargetParser/AArch64TargetParser.cpp
+++ b/llvm/lib/TargetParser/AArch64TargetParser.cpp
@@ -50,7 +50,7 @@ std::optional<AArch64::ArchInfo> AArch64::ArchInfo::findBySubArch(StringRef SubA
 
 std::optional<AArch64::FMVInfo> lookupFMVByID(AArch64::ArchExtKind ExtID) {
   for (const AArch64::FMVInfo &Info : AArch64::getFMVInfo())
-    if (Info.ID && *Info.ID == ExtID)
+    if (Info.ID == ExtID)
       return Info;
   return {};
 }
@@ -298,6 +298,13 @@ void AArch64::ExtensionSet::disable(ArchExtKind E) {
   if (E == AEK_SVE2AES)
     disable(AEK_SVEAES);
 
+  // sve2-sha3 was historically associated with both FEAT_SVE2 and
+  // FEAT_SVE_SHA3, the latter is now associated with sve-sha3 and sve2-sha3 has
+  // become shorthand for +sve2+sve-sha3. For backwards compatibility, when we
+  // disable sve2-sha3 we must also disable sve-sha3.
+  if (E == AEK_SVE2SHA3)
+    disable(AEK_SVESHA3);
+
   if (E == AEK_SVE2BITPERM){
     disable(AEK_SVEBITPERM);
     disable(AEK_SVE2);
diff --git a/llvm/lib/TargetParser/CSKYTargetParser.cpp b/llvm/lib/TargetParser/CSKYTargetParser.cpp
index 006d2bb342ac..5fea08e38e2d 100644
--- a/llvm/lib/TargetParser/CSKYTargetParser.cpp
+++ b/llvm/lib/TargetParser/CSKYTargetParser.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TargetParser/CSKYTargetParser.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index b96a06544888..91304fd57c07 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -176,25 +176,43 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
   SmallVector<StringRef, 32> Lines;
   ProcCpuinfoContent.split(Lines, '\n');
 
-  // Look for the CPU implementer line.
+  // Look for the CPU implementer and hardware lines, and store the CPU part
+  // numbers found.
   StringRef Implementer;
   StringRef Hardware;
-  StringRef Part;
-  for (unsigned I = 0, E = Lines.size(); I != E; ++I) {
-    if (Lines[I].starts_with("CPU implementer"))
-      Implementer = Lines[I].substr(15).ltrim("\t :");
-    if (Lines[I].starts_with("Hardware"))
-      Hardware = Lines[I].substr(8).ltrim("\t :");
-    if (Lines[I].starts_with("CPU part"))
-      Part = Lines[I].substr(8).ltrim("\t :");
+  SmallVector<StringRef, 32> Parts;
+  for (StringRef Line : Lines) {
+    if (Line.consume_front("CPU implementer"))
+      Implementer = Line.ltrim("\t :");
+    else if (Line.consume_front("Hardware"))
+      Hardware = Line.ltrim("\t :");
+    else if (Line.consume_front("CPU part"))
+      Parts.emplace_back(Line.ltrim("\t :"));
   }
 
+  // Last `Part' seen, in case we don't analyse all `Parts' parsed.
+  StringRef Part = Parts.empty() ? StringRef() : Parts.back();
+
+  // Remove duplicate `Parts'.
+  llvm::sort(Parts);
+  Parts.erase(llvm::unique(Parts), Parts.end());
+
+  auto MatchBigLittle = [](auto const &Parts, StringRef Big, StringRef Little) {
+    if (Parts.size() == 2)
+      return (Parts[0] == Big && Parts[1] == Little) ||
+             (Parts[1] == Big && Parts[0] == Little);
+    return false;
+  };
+
   if (Implementer == "0x41") { // ARM Ltd.
     // MSM8992/8994 may give cpu part for the core that the kernel is running on,
     // which is undeterministic and wrong. Always return cortex-a53 for these SoC.
     if (Hardware.ends_with("MSM8994") || Hardware.ends_with("MSM8996"))
       return "cortex-a53";
 
+    // Detect big.LITTLE systems.
+    if (MatchBigLittle(Parts, "0xd85", "0xd87"))
+      return "cortex-x925";
 
     // The CPU part is a 3 digit hexadecimal number with a 0x prefix. The
     // values correspond to the "Part number" in the CP15/c0 register. The
diff --git a/llvm/lib/TargetParser/LoongArchTargetParser.cpp b/llvm/lib/TargetParser/LoongArchTargetParser.cpp
index db68498609b5..572a278a39fc 100644
--- a/llvm/lib/TargetParser/LoongArchTargetParser.cpp
+++ b/llvm/lib/TargetParser/LoongArchTargetParser.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TargetParser/LoongArchTargetParser.h"
+#include "llvm/ADT/SmallVector.h"
 
 using namespace llvm;
 using namespace llvm::LoongArch;
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 49442c30eb44..cae12f9a4ed3 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -443,6 +443,7 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
       Features["gfx1250-insts"] = true;
       Features["bitop3-insts"] = true;
       Features["prng-inst"] = true;
+      Features["transpose-load-f4f6-insts"] = true;
       Features["fp8-conversion-insts"] = true;
       Features["permlane16-swap"] = true;
       Features["ashr-pk-insts"] = true;
diff --git a/llvm/lib/TargetParser/Triple.cpp b/llvm/lib/TargetParser/Triple.cpp
index 5718ae385bac..45ef8c132975 100644
--- a/llvm/lib/TargetParser/Triple.cpp
+++ b/llvm/lib/TargetParser/Triple.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/VersionTuple.h"
@@ -1018,10 +1019,7 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) {
 ///
 /// This stores the string representation and parses the various pieces into
 /// enum members.
-Triple::Triple(const Twine &Str)
-    : Data(Str.str()), Arch(UnknownArch), SubArch(NoSubArch),
-      Vendor(UnknownVendor), OS(UnknownOS), Environment(UnknownEnvironment),
-      ObjectFormat(UnknownObjectFormat) {
+Triple::Triple(std::string &&Str) : Data(std::move(Str)) {
   // Do minimal parsing by hand here.
   SmallVector<StringRef, 4> Components;
   StringRef(Data).split(Components, '-', /*MaxSplit*/ 3);
@@ -1052,6 +1050,8 @@ Triple::Triple(const Twine &Str)
     ObjectFormat = getDefaultFormat(*this);
 }
 
+Triple::Triple(const Twine &Str) : Triple(Str.str()) {}
+
 /// Construct a triple from string representations of the architecture,
 /// vendor, and OS.
 ///
@@ -1636,14 +1636,7 @@ void Triple::setObjectFormat(ObjectFormatType Kind) {
 }
 
 void Triple::setArchName(StringRef Str) {
-  // Work around a miscompilation bug for Twines in gcc 4.0.3.
-  SmallString<64> Triple;
-  Triple += Str;
-  Triple += "-";
-  Triple += getVendorName();
-  Triple += "-";
-  Triple += getOSAndEnvironmentName();
-  setTriple(Triple);
+  setTriple(Str + "-" + getVendorName() + "-" + getOSAndEnvironmentName());
 }
 
 void Triple::setVendorName(StringRef Str) {
@@ -2263,6 +2256,55 @@ bool Triple::isValidVersionForOS(OSType OSKind, const VersionTuple &Version) {
   llvm_unreachable("unexpected or invalid os version");
 }
 
+ExceptionHandling Triple::getDefaultExceptionHandling() const {
+  if (isOSBinFormatCOFF()) {
+    if (getArch() == Triple::x86 &&
+        (isOSCygMing() || isWindowsItaniumEnvironment()))
+      return ExceptionHandling::DwarfCFI;
+    return ExceptionHandling::WinEH;
+  }
+
+  if (isOSBinFormatXCOFF())
+    return ExceptionHandling::AIX;
+  if (isOSBinFormatGOFF())
+    return ExceptionHandling::ZOS;
+
+  if (isARM() || isThumb()) {
+    if (isOSBinFormatELF()) {
+      return getOS() == Triple::NetBSD ? ExceptionHandling::DwarfCFI
+                                       : ExceptionHandling::ARM;
+    }
+
+    return isOSDarwin() && !isWatchABI() ? ExceptionHandling::SjLj
+                                         : ExceptionHandling::DwarfCFI;
+  }
+
+  if (isAArch64() || isX86() || isPPC() || isMIPS() || isSPARC() || isBPF() ||
+      isRISCV() || isLoongArch())
+    return ExceptionHandling::DwarfCFI;
+
+  switch (getArch()) {
+  case Triple::arc:
+  case Triple::csky:
+  case Triple::hexagon:
+  case Triple::lanai:
+  case Triple::msp430:
+  case Triple::systemz:
+  case Triple::xcore:
+  case Triple::xtensa:
+    return ExceptionHandling::DwarfCFI;
+  default:
+    break;
+  }
+
+  // Explicitly none targets.
+  if (isWasm() || isAMDGPU() || isNVPTX() || isSPIROrSPIRV())
+    return ExceptionHandling::None;
+
+  // Default to none.
+  return ExceptionHandling::None;
+}
+
 // HLSL triple environment orders are relied on in the front end
 static_assert(Triple::Vertex - Triple::Pixel == 1,
               "incorrect HLSL stage order");
diff --git a/llvm/lib/TargetParser/Unix/Host.inc b/llvm/lib/TargetParser/Unix/Host.inc
index a33fe6fff893..ef9e288ee3a0 100644
--- a/llvm/lib/TargetParser/Unix/Host.inc
+++ b/llvm/lib/TargetParser/Unix/Host.inc
@@ -55,7 +55,7 @@ static std::string updateTripleOSVersion(std::string TargetTripleString) {
   // On AIX, the AIX version and release should be that of the current host
   // unless if the version has already been specified.
   if (Triple(LLVM_HOST_TRIPLE).getOS() == Triple::AIX) {
-    Triple TT(TargetTripleString);
+    Triple TT(std::move(TargetTripleString));
     if (TT.getOS() == Triple::AIX && !TT.getOSMajorVersion()) {
       struct utsname name;
       if (uname(&name) != -1) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
index a0a26827aa09..c00e9c7bbee0 100644
--- a/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -110,11 +110,11 @@ bool Lowerer::lower(Function &F) {
 
 static bool declaresCoroCleanupIntrinsics(const Module &M) {
   return coro::declaresIntrinsics(
-      M, {"llvm.coro.alloc", "llvm.coro.begin", "llvm.coro.subfn.addr",
-          "llvm.coro.free", "llvm.coro.id", "llvm.coro.id.retcon",
-          "llvm.coro.id.async", "llvm.coro.id.retcon.once",
-          "llvm.coro.async.size.replace", "llvm.coro.async.resume",
-          "llvm.coro.begin.custom.abi"});
+      M, {Intrinsic::coro_alloc, Intrinsic::coro_begin,
+          Intrinsic::coro_subfn_addr, Intrinsic::coro_free, Intrinsic::coro_id,
+          Intrinsic::coro_id_retcon, Intrinsic::coro_id_async,
+          Intrinsic::coro_id_retcon_once, Intrinsic::coro_async_size_replace,
+          Intrinsic::coro_async_resume, Intrinsic::coro_begin_custom_abi});
 }
 
 PreservedAnalyses CoroCleanupPass::run(Module &M,
diff --git a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
index 9ac145b10fe4..e279fec18bdb 100644
--- a/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -279,12 +279,13 @@ void Lowerer::lowerEarlyIntrinsics(Function &F) {
 }
 
 static bool declaresCoroEarlyIntrinsics(const Module &M) {
+  // coro_suspend omitted as it is overloaded.
   return coro::declaresIntrinsics(
-      M, {"llvm.coro.id", "llvm.coro.id.retcon", "llvm.coro.id.retcon.once",
-          "llvm.coro.id.async", "llvm.coro.destroy", "llvm.coro.done",
-          "llvm.coro.end", "llvm.coro.end.async", "llvm.coro.noop",
-          "llvm.coro.free", "llvm.coro.promise", "llvm.coro.resume",
-          "llvm.coro.suspend"});
+      M, {Intrinsic::coro_id, Intrinsic::coro_id_retcon,
+          Intrinsic::coro_id_retcon_once, Intrinsic::coro_id_async,
+          Intrinsic::coro_destroy, Intrinsic::coro_done, Intrinsic::coro_end,
+          Intrinsic::coro_end_async, Intrinsic::coro_noop, Intrinsic::coro_free,
+          Intrinsic::coro_promise, Intrinsic::coro_resume});
 }
 
 PreservedAnalyses CoroEarlyPass::run(Module &M, ModuleAnalysisManager &) {
diff --git a/llvm/lib/Transforms/Coroutines/CoroElide.cpp b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
index c5131aa0787e..1c8d4a8592d6 100644
--- a/llvm/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroElide.cpp
@@ -450,7 +450,7 @@ bool CoroIdElider::attemptElide() {
 
 PreservedAnalyses CoroElidePass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &M = *F.getParent();
-  if (!coro::declaresIntrinsics(M, {"llvm.coro.id"}))
+  if (!coro::declaresIntrinsics(M, Intrinsic::coro_id))
     return PreservedAnalyses::all();
 
   FunctionElideInfo FEI{&F};
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 9210c81cb835..9042d4d94e5f 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -688,9 +688,8 @@ static DIType *solveDIType(DIBuilder &Builder, Type *Ty,
 static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
                                 FrameDataInfo &FrameData) {
   DISubprogram *DIS = F.getSubprogram();
-  // If there is no DISubprogram for F, it implies the Function are not compiled
-  // with debug info. So we also don't need to generate debug info for the frame
-  // neither.
+  // If there is no DISubprogram for F, it implies the function is compiled
+  // without debug info. So we also don't generate debug info for the frame.
   if (!DIS || !DIS->getUnit() ||
       !dwarf::isCPlusPlus(
           (dwarf::SourceLanguage)DIS->getUnit()->getSourceLanguage()) ||
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index a0b52063aca1..b53c5a48eb10 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -24,8 +24,7 @@ namespace coro {
 
 bool isSuspendBlock(BasicBlock *BB);
 bool declaresAnyIntrinsic(const Module &M);
-bool declaresIntrinsics(const Module &M,
-                        const std::initializer_list<StringRef>);
+bool declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List);
 void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
 
 /// Replaces all @llvm.coro.alloc intrinsics calls associated with a given
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 8813f91e9060..077dbe697a87 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -42,6 +42,7 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -302,7 +303,7 @@ static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
 }
 
 // Mark a coroutine as done, which implies that the coroutine is finished and
-// never get resumed.
+// never gets resumed.
 //
 // In resume-switched ABI, the done state is represented by storing zero in
 // ResumeFnAddr.
@@ -1478,6 +1479,18 @@ struct SwitchCoroutineSplitter {
   static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
     LLVMContext &C = F.getContext();
 
+    DIBuilder DBuilder(*F.getParent(), /*AllowUnresolved*/ false);
+    DISubprogram *DIS = F.getSubprogram();
+    // If there is no DISubprogram for F, it implies the function is compiled
+    // without debug info. So we also don't generate debug info for the
+    // suspension points.
+    bool AddDebugLabels =
+        (DIS && DIS->getUnit() &&
+         (DIS->getUnit()->getEmissionKind() ==
+              DICompileUnit::DebugEmissionKind::FullDebug ||
+          DIS->getUnit()->getEmissionKind() ==
+              DICompileUnit::DebugEmissionKind::LineTablesOnly));
+
     // resume.entry:
     //  %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32
     //  0, i32 2 % index = load i32, i32* %index.addr switch i32 %index, label
@@ -1500,6 +1513,7 @@ struct SwitchCoroutineSplitter {
         Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
     Shape.SwitchLowering.ResumeSwitch = Switch;
 
+    // Split all coro.suspend calls
     size_t SuspendIndex = 0;
     for (auto *AnyS : Shape.CoroSuspends) {
       auto *S = cast<CoroSuspendInst>(AnyS);
@@ -1538,6 +1552,7 @@ struct SwitchCoroutineSplitter {
       //     br label %resume.0.landing
       //
       //  resume.0: ; <--- jump from the switch in the resume.entry
+      //        #dbg_label(...)  ; <--- artificial label for debuggers
       //     %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
       //     br label %resume.0.landing
       //
@@ -1560,11 +1575,27 @@ struct SwitchCoroutineSplitter {
       PN->addIncoming(Builder.getInt8(-1), SuspendBB);
       PN->addIncoming(S, ResumeBB);
 
+      if (AddDebugLabels) {
+        if (DebugLoc SuspendLoc = S->getDebugLoc()) {
+          std::string LabelName =
+              ("__coro_resume_" + Twine(SuspendIndex)).str();
+          DILocation &DILoc = *SuspendLoc.get();
+          DILabel *ResumeLabel =
+              DBuilder.createLabel(DIS, LabelName, DILoc.getFile(),
+                                   SuspendLoc.getLine(), SuspendLoc.getCol(),
+                                   /*IsArtificial=*/true,
+                                   /*CoroSuspendIdx=*/SuspendIndex,
+                                   /*AlwaysPreserve=*/false);
+          DBuilder.insertLabel(ResumeLabel, &DILoc, ResumeBB->begin());
+        }
+      }
+
       ++SuspendIndex;
     }
 
     Builder.SetInsertPoint(UnreachBB);
     Builder.CreateUnreachable();
+    DBuilder.finalize();
 
     Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
   }
diff --git a/llvm/lib/Transforms/Coroutines/Coroutines.cpp b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
index 02500ff778b8..59ae057cae79 100644
--- a/llvm/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/llvm/lib/Transforms/Coroutines/Coroutines.cpp
@@ -61,71 +61,56 @@ CallInst *coro::LowererBase::makeSubFnCall(Value *Arg, int Index,
   return CallInst::Create(Fn, {Arg, IndexVal}, "", InsertPt->getIterator());
 }
 
-// NOTE: Must be sorted!
-static const char *const CoroIntrinsics[] = {
-    "llvm.coro.align",
-    "llvm.coro.alloc",
-    "llvm.coro.async.context.alloc",
-    "llvm.coro.async.context.dealloc",
-    "llvm.coro.async.resume",
-    "llvm.coro.async.size.replace",
-    "llvm.coro.await.suspend.bool",
-    "llvm.coro.await.suspend.handle",
-    "llvm.coro.await.suspend.void",
-    "llvm.coro.begin",
-    "llvm.coro.begin.custom.abi",
-    "llvm.coro.destroy",
-    "llvm.coro.done",
-    "llvm.coro.end",
-    "llvm.coro.end.async",
-    "llvm.coro.frame",
-    "llvm.coro.free",
-    "llvm.coro.id",
-    "llvm.coro.id.async",
-    "llvm.coro.id.retcon",
-    "llvm.coro.id.retcon.once",
-    "llvm.coro.noop",
-    "llvm.coro.prepare.async",
-    "llvm.coro.prepare.retcon",
-    "llvm.coro.promise",
-    "llvm.coro.resume",
-    "llvm.coro.save",
-    "llvm.coro.size",
-    "llvm.coro.subfn.addr",
-    "llvm.coro.suspend",
-    "llvm.coro.suspend.async",
-    "llvm.coro.suspend.retcon",
+// We can only efficiently check for non-overloaded intrinsics.
+// The following intrinsics are absent for that reason:
+// coro_align, coro_size, coro_suspend_async, coro_suspend_retcon
+static Intrinsic::ID NonOverloadedCoroIntrinsics[] = {
+    Intrinsic::coro_alloc,
+    Intrinsic::coro_async_context_alloc,
+    Intrinsic::coro_async_context_dealloc,
+    Intrinsic::coro_async_resume,
+    Intrinsic::coro_async_size_replace,
+    Intrinsic::coro_await_suspend_bool,
+    Intrinsic::coro_await_suspend_handle,
+    Intrinsic::coro_await_suspend_void,
+    Intrinsic::coro_begin,
+    Intrinsic::coro_begin_custom_abi,
+    Intrinsic::coro_destroy,
+    Intrinsic::coro_done,
+    Intrinsic::coro_end,
+    Intrinsic::coro_end_async,
+    Intrinsic::coro_frame,
+    Intrinsic::coro_free,
+    Intrinsic::coro_id,
+    Intrinsic::coro_id_async,
+    Intrinsic::coro_id_retcon,
+    Intrinsic::coro_id_retcon_once,
+    Intrinsic::coro_promise,
+    Intrinsic::coro_resume,
+    Intrinsic::coro_save,
+    Intrinsic::coro_subfn_addr,
+    Intrinsic::coro_suspend,
 };
 
-#ifndef NDEBUG
-static bool isCoroutineIntrinsicName(StringRef Name) {
-  return llvm::binary_search(CoroIntrinsics, Name);
-}
-#endif
-
 bool coro::isSuspendBlock(BasicBlock *BB) {
   return isa<AnyCoroSuspendInst>(BB->front());
 }
 
 bool coro::declaresAnyIntrinsic(const Module &M) {
-  for (StringRef Name : CoroIntrinsics) {
-    if (M.getNamedValue(Name))
-      return true;
-  }
-
-  return false;
+  return declaresIntrinsics(M, NonOverloadedCoroIntrinsics);
 }
 
-// Verifies if a module has named values listed. Also, in debug mode verifies
-// that names are intrinsic names.
-bool coro::declaresIntrinsics(const Module &M,
-                              const std::initializer_list<StringRef> List) {
-  for (StringRef Name : List) {
-    assert(isCoroutineIntrinsicName(Name) && "not a coroutine intrinsic");
-    if (M.getNamedValue(Name))
-      return true;
-  }
+// Checks whether the module declares any of the listed intrinsics.
+bool coro::declaresIntrinsics(const Module &M, ArrayRef<Intrinsic::ID> List) {
+#ifndef NDEBUG
+  for (Intrinsic::ID ID : List)
+    assert(!Intrinsic::isOverloaded(ID) &&
+           "Only non-overloaded intrinsics supported");
+#endif
 
+  for (Intrinsic::ID ID : List)
+    if (Intrinsic::getDeclarationIfExists(&M, ID))
+      return true;
   return false;
 }
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index b3e5b7fa6e0b..8017db1cfe14 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -552,10 +552,10 @@ void sinkSpillUsesAfterCoroBegin(const DominatorTree &Dom,
         Worklist.push_back(Inst);
     }
   };
-  std::for_each(Spills.begin(), Spills.end(),
-                [&](auto &I) { collectUsers(I.first); });
-  std::for_each(Allocas.begin(), Allocas.end(),
-                [&](auto &I) { collectUsers(I.Alloca); });
+  for (auto &I : Spills)
+    collectUsers(I.first);
+  for (auto &I : Allocas)
+    collectUsers(I.Alloca);
 
   // Recursively collect users before coro.begin.
   while (!Worklist.empty()) {
diff --git a/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
index 5a87cf8c83d7..b131f34b06d6 100644
--- a/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
+++ b/llvm/lib/Transforms/HipStdPar/HipStdPar.cpp
@@ -37,6 +37,16 @@
 //    memory that ends up in one of the runtime equivalents, since this can
 //    happen if e.g. a library that was compiled without interposition returns
 //    an allocation that can be validly passed to `free`.
+//
+// 3. MathFixup (required): Some accelerators might have an incomplete
+//    implementation for the intrinsics used to implement some of the math
+//    functions in <cmath> / their corresponding libcall lowerings. Since this
+//    can vary quite significantly between accelerators, we replace calls to a
+//    set of intrinsics / lib functions known to be problematic with calls to a
+//    HIPSTDPAR specific forwarding layer, which gives an uniform interface for
+//    accelerators to implement in their own runtime components. This pass
+//    should run before AcceleratorCodeSelection so as to prevent the spurious
+//    removal of the HIPSTDPAR specific forwarding functions.
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/HipStdPar/HipStdPar.h"
@@ -48,6 +58,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
@@ -321,3 +332,110 @@ HipStdParAllocationInterpositionPass::run(Module &M, ModuleAnalysisManager&) {
 
   return PreservedAnalyses::none();
 }
+
+static constexpr std::pair<StringLiteral, StringLiteral> MathLibToHipStdPar[]{
+    {"acosh", "__hipstdpar_acosh_f64"},
+    {"acoshf", "__hipstdpar_acosh_f32"},
+    {"asinh", "__hipstdpar_asinh_f64"},
+    {"asinhf", "__hipstdpar_asinh_f32"},
+    {"atanh", "__hipstdpar_atanh_f64"},
+    {"atanhf", "__hipstdpar_atanh_f32"},
+    {"cbrt", "__hipstdpar_cbrt_f64"},
+    {"cbrtf", "__hipstdpar_cbrt_f32"},
+    {"erf", "__hipstdpar_erf_f64"},
+    {"erff", "__hipstdpar_erf_f32"},
+    {"erfc", "__hipstdpar_erfc_f64"},
+    {"erfcf", "__hipstdpar_erfc_f32"},
+    {"fdim", "__hipstdpar_fdim_f64"},
+    {"fdimf", "__hipstdpar_fdim_f32"},
+    {"expm1", "__hipstdpar_expm1_f64"},
+    {"expm1f", "__hipstdpar_expm1_f32"},
+    {"hypot", "__hipstdpar_hypot_f64"},
+    {"hypotf", "__hipstdpar_hypot_f32"},
+    {"ilogb", "__hipstdpar_ilogb_f64"},
+    {"ilogbf", "__hipstdpar_ilogb_f32"},
+    {"lgamma", "__hipstdpar_lgamma_f64"},
+    {"lgammaf", "__hipstdpar_lgamma_f32"},
+    {"log1p", "__hipstdpar_log1p_f64"},
+    {"log1pf", "__hipstdpar_log1p_f32"},
+    {"logb", "__hipstdpar_logb_f64"},
+    {"logbf", "__hipstdpar_logb_f32"},
+    {"nextafter", "__hipstdpar_nextafter_f64"},
+    {"nextafterf", "__hipstdpar_nextafter_f32"},
+    {"nexttoward", "__hipstdpar_nexttoward_f64"},
+    {"nexttowardf", "__hipstdpar_nexttoward_f32"},
+    {"remainder", "__hipstdpar_remainder_f64"},
+    {"remainderf", "__hipstdpar_remainder_f32"},
+    {"remquo", "__hipstdpar_remquo_f64"},
+    {"remquof", "__hipstdpar_remquo_f32"},
+    {"scalbln", "__hipstdpar_scalbln_f64"},
+    {"scalblnf", "__hipstdpar_scalbln_f32"},
+    {"scalbn", "__hipstdpar_scalbn_f64"},
+    {"scalbnf", "__hipstdpar_scalbn_f32"},
+    {"tgamma", "__hipstdpar_tgamma_f64"},
+    {"tgammaf", "__hipstdpar_tgamma_f32"}};
+
+PreservedAnalyses HipStdParMathFixupPass::run(Module &M,
+                                              ModuleAnalysisManager &) {
+  if (M.empty())
+    return PreservedAnalyses::all();
+
+  SmallVector<std::pair<Function *, std::string>> ToReplace;
+  for (auto &&F : M) {
+    if (!F.hasName())
+      continue;
+
+    StringRef N = F.getName();
+    Intrinsic::ID ID = F.getIntrinsicID();
+
+    switch (ID) {
+    case Intrinsic::not_intrinsic: {
+      auto It =
+          find_if(MathLibToHipStdPar, [&](auto &&M) { return M.first == N; });
+      if (It == std::cend(MathLibToHipStdPar))
+        continue;
+      ToReplace.emplace_back(&F, It->second);
+      break;
+    }
+    case Intrinsic::acos:
+    case Intrinsic::asin:
+    case Intrinsic::atan:
+    case Intrinsic::atan2:
+    case Intrinsic::cosh:
+    case Intrinsic::modf:
+    case Intrinsic::sinh:
+    case Intrinsic::tan:
+    case Intrinsic::tanh:
+      break;
+    default: {
+      if (F.getReturnType()->isDoubleTy()) {
+        switch (ID) {
+        case Intrinsic::cos:
+        case Intrinsic::exp:
+        case Intrinsic::exp2:
+        case Intrinsic::log:
+        case Intrinsic::log10:
+        case Intrinsic::log2:
+        case Intrinsic::pow:
+        case Intrinsic::sin:
+          break;
+        default:
+          continue;
+        }
+        break;
+      }
+      continue;
+    }
+    }
+
+    ToReplace.emplace_back(&F, N);
+    llvm::replace(ToReplace.back().second, '.', '_');
+    StringRef Prefix = "llvm";
+    ToReplace.back().second.replace(0, Prefix.size(), "__hipstdpar");
+  }
+  for (auto &&[F, NewF] : ToReplace)
+    F->replaceAllUsesWith(
+        M.getOrInsertFunction(NewF, F->getFunctionType()).getCallee());
+
+  return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index dac1f7a30c37..448e1b4982b7 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -990,6 +990,13 @@ static bool addIfNotExistent(LLVMContext &Ctx, const Attribute &Attr,
     AB.addAttribute(Attr);
     return true;
   }
+  if (Attr.isConstantRangeAttribute()) {
+    Attribute::AttrKind Kind = Attr.getKindAsEnum();
+    if (!ForceReplace && AttrSet.hasAttribute(Kind))
+      return false;
+    AB.addAttribute(Attr);
+    return true;
+  }
 
   llvm_unreachable("Expected enum or string attribute!");
 }
@@ -3612,6 +3619,8 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
       if (SimplifyAllLoads)
         getAssumedSimplified(IRPosition::value(I), nullptr,
                              UsedAssumedInformation, AA::Intraprocedural);
+      getOrCreateAAFor<AAInvariantLoadPointer>(
+          IRPosition::value(*LI->getPointerOperand()));
       getOrCreateAAFor<AAAddressSpace>(
           IRPosition::value(*LI->getPointerOperand()));
     } else {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index bf1a48936e7d..38f393c5de28 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -191,6 +191,7 @@ PIPE_OPERATOR(AAInterFnReachability)
 PIPE_OPERATOR(AAPointerInfo)
 PIPE_OPERATOR(AAAssumptionInfo)
 PIPE_OPERATOR(AAUnderlyingObjects)
+PIPE_OPERATOR(AAInvariantLoadPointer)
 PIPE_OPERATOR(AAAddressSpace)
 PIPE_OPERATOR(AAAllocationInfo)
 PIPE_OPERATOR(AAIndirectCallInfo)
@@ -9171,44 +9172,58 @@ struct AAValueConstantRangeImpl : AAValueConstantRange {
     return MDNode::get(Ctx, LowAndHigh);
   }
 
-  /// Return true if \p Assumed is included in \p KnownRanges.
-  static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
-
+  /// Return true if \p Assumed is included in ranges from instruction \p I.
+  static bool isBetterRange(const ConstantRange &Assumed,
+                            const Instruction &I) {
     if (Assumed.isFullSet())
       return false;
 
-    if (!KnownRanges)
-      return true;
-
-    // If multiple ranges are annotated in IR, we give up to annotate assumed
-    // range for now.
+    std::optional<ConstantRange> Known;
 
-    // TODO:  If there exists a known range which containts assumed range, we
-    // can say assumed range is better.
-    if (KnownRanges->getNumOperands() > 2)
-      return false;
+    if (const auto *CB = dyn_cast<CallBase>(&I)) {
+      Known = CB->getRange();
+    } else if (MDNode *KnownRanges = I.getMetadata(LLVMContext::MD_range)) {
+      // If multiple ranges are annotated in IR, we give up to annotate assumed
+      // range for now.
+
+      // TODO:  If there exists a known range which containts assumed range, we
+      // can say assumed range is better.
+      if (KnownRanges->getNumOperands() > 2)
+        return false;
 
-    ConstantInt *Lower =
-        mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
-    ConstantInt *Upper =
-        mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
+      ConstantInt *Lower =
+          mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
+      ConstantInt *Upper =
+          mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
 
-    ConstantRange Known(Lower->getValue(), Upper->getValue());
-    return Known.contains(Assumed) && Known != Assumed;
+      Known.emplace(Lower->getValue(), Upper->getValue());
+    }
+    return !Known || (*Known != Assumed && Known->contains(Assumed));
   }
 
   /// Helper function to set range metadata.
   static bool
   setRangeMetadataIfisBetterRange(Instruction *I,
                                   const ConstantRange &AssumedConstantRange) {
-    auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
-    if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
-      if (!AssumedConstantRange.isEmptySet()) {
-        I->setMetadata(LLVMContext::MD_range,
-                       getMDNodeForConstantRange(I->getType(), I->getContext(),
-                                                 AssumedConstantRange));
-        return true;
-      }
+    if (isBetterRange(AssumedConstantRange, *I)) {
+      I->setMetadata(LLVMContext::MD_range,
+                     getMDNodeForConstantRange(I->getType(), I->getContext(),
+                                               AssumedConstantRange));
+      return true;
+    }
+    return false;
+  }
+  /// Helper function to set range return attribute.
+  static bool
+  setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
+                                 Instruction *I,
+                                 const ConstantRange &AssumedConstantRange) {
+    if (isBetterRange(AssumedConstantRange, *I)) {
+      A.manifestAttrs(IRP,
+                      Attribute::get(I->getContext(), Attribute::Range,
+                                     AssumedConstantRange),
+                      /*ForceReplace*/ true);
+      return true;
     }
     return false;
   }
@@ -9225,9 +9240,13 @@ struct AAValueConstantRangeImpl : AAValueConstantRange {
       if (Instruction *I = dyn_cast<Instruction>(&V)) {
         assert(I == getCtxI() && "Should not annotate an instruction which is "
                                  "not the context instruction");
-        if (isa<CallInst>(I) || isa<LoadInst>(I))
+        if (isa<LoadInst>(I))
           if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
             Changed = ChangeStatus::CHANGED;
+        if (isa<CallInst>(I))
+          if (setRangeRetAttrIfisBetterRange(A, getIRPosition(), I,
+                                             AssumedConstantRange))
+            Changed = ChangeStatus::CHANGED;
       }
     }
 
@@ -9623,10 +9642,11 @@ struct AAValueConstantRangeCallSiteReturned
 
   /// See AbstractAttribute::initialize(...).
   void initialize(Attributor &A) override {
-    // If it is a load instruction with range metadata, use the metadata.
-    if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
-      if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
-        intersectKnown(getConstantRangeFromMetadata(*RangeMD));
+    // If it is a call instruction with range attribute, use the range.
+    if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue())) {
+      if (std::optional<ConstantRange> Range = CI->getRange())
+        intersectKnown(*Range);
+    }
 
     AAValueConstantRangeImpl::initialize(A);
   }
@@ -12543,6 +12563,346 @@ struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
 };
 } // namespace
 
+/// --------------------- Invariant Load Pointer -------------------------------
+namespace {
+
+struct AAInvariantLoadPointerImpl
+    : public StateWrapper<BitIntegerState<uint8_t, 15>,
+                          AAInvariantLoadPointer> {
+
+  enum {
+    // pointer does not alias within the bounds of the function
+    IS_NOALIAS = 1 << 0,
+    // pointer is not involved in any effectful instructions within the bounds
+    // of the function
+    IS_NOEFFECT = 1 << 1,
+    // loads are invariant within the bounds of the function
+    IS_LOCALLY_INVARIANT = 1 << 2,
+    // memory lifetime is constrained within the bounds of the function
+    IS_LOCALLY_CONSTRAINED = 1 << 3,
+
+    IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
+                    IS_LOCALLY_CONSTRAINED,
+  };
+  static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
+
+  using Base =
+      StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
+
+  // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
+  // pessimistic about IS_KNOWN_INVARIANT
+  AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
+      : Base(IRP) {}
+
+  bool isKnownInvariant() const final {
+    return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);
+  }
+
+  bool isKnownLocallyInvariant() const final {
+    if (isKnown(IS_LOCALLY_INVARIANT))
+      return true;
+    return isKnown(IS_NOALIAS | IS_NOEFFECT);
+  }
+
+  bool isAssumedInvariant() const final {
+    return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);
+  }
+
+  bool isAssumedLocallyInvariant() const final {
+    if (isAssumed(IS_LOCALLY_INVARIANT))
+      return true;
+    return isAssumed(IS_NOALIAS | IS_NOEFFECT);
+  }
+
+  ChangeStatus updateImpl(Attributor &A) override {
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+
+    Changed |= updateNoAlias(A);
+    if (requiresNoAlias() && !isAssumed(IS_NOALIAS))
+      return indicatePessimisticFixpoint();
+
+    Changed |= updateNoEffect(A);
+
+    Changed |= updateLocalInvariance(A);
+
+    return Changed;
+  }
+
+  ChangeStatus manifest(Attributor &A) override {
+    if (!isKnownInvariant())
+      return ChangeStatus::UNCHANGED;
+
+    ChangeStatus Changed = ChangeStatus::UNCHANGED;
+    const Value *Ptr = &getAssociatedValue();
+    const auto TagInvariantLoads = [&](const Use &U, bool &) {
+      if (U.get() != Ptr)
+        return true;
+      auto *I = dyn_cast<Instruction>(U.getUser());
+      if (!I)
+        return true;
+
+      // Ensure that we are only changing uses from the corresponding callgraph
+      // SSC in the case that the AA isn't run on the entire module
+      if (!A.isRunOn(I->getFunction()))
+        return true;
+
+      if (I->hasMetadata(LLVMContext::MD_invariant_load))
+        return true;
+
+      if (auto *LI = dyn_cast<LoadInst>(I)) {
+        LI->setMetadata(LLVMContext::MD_invariant_load,
+                        MDNode::get(LI->getContext(), {}));
+        Changed = ChangeStatus::CHANGED;
+      }
+      return true;
+    };
+
+    (void)A.checkForAllUses(TagInvariantLoads, *this, *Ptr);
+    return Changed;
+  }
+
+  /// See AbstractAttribute::getAsStr().
+  const std::string getAsStr(Attributor *) const override {
+    if (isKnownInvariant())
+      return "load-invariant pointer";
+    return "non-invariant pointer";
+  }
+
+  /// See AbstractAttribute::trackStatistics().
+  void trackStatistics() const override {}
+
+private:
+  /// Indicate that noalias is required for the pointer to be invariant.
+  bool requiresNoAlias() const {
+    switch (getPositionKind()) {
+    default:
+      // Conservatively default to require noalias.
+      return true;
+    case IRP_FLOAT:
+    case IRP_RETURNED:
+    case IRP_CALL_SITE:
+      return false;
+    case IRP_CALL_SITE_RETURNED: {
+      const auto &CB = cast<CallBase>(getAnchorValue());
+      return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+          &CB, /*MustPreserveNullness=*/false);
+    }
+    case IRP_ARGUMENT: {
+      const Function *F = getAssociatedFunction();
+      assert(F && "no associated function for argument");
+      return !isCallableCC(F->getCallingConv());
+    }
+    }
+  }
+
+  bool isExternal() const {
+    const Function *F = getAssociatedFunction();
+    if (!F)
+      return true;
+    return isCallableCC(F->getCallingConv()) &&
+           getPositionKind() != IRP_CALL_SITE_RETURNED;
+  }
+
+  ChangeStatus updateNoAlias(Attributor &A) {
+    if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))
+      return ChangeStatus::UNCHANGED;
+
+    // Try to use AANoAlias.
+    if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
+            getIRPosition(), this, DepClassTy::REQUIRED)) {
+      if (ANoAlias->isKnownNoAlias()) {
+        addKnownBits(IS_NOALIAS);
+        return ChangeStatus::CHANGED;
+      }
+
+      if (!ANoAlias->isAssumedNoAlias()) {
+        removeAssumedBits(IS_NOALIAS);
+        return ChangeStatus::CHANGED;
+      }
+
+      return ChangeStatus::UNCHANGED;
+    }
+
+    // Try to infer noalias from argument attribute, since it is applicable for
+    // the duration of the function.
+    if (const Argument *Arg = getAssociatedArgument()) {
+      if (Arg->hasNoAliasAttr()) {
+        addKnownBits(IS_NOALIAS);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      // Noalias information is not provided, and cannot be inferred,
+      // so we conservatively assume the pointer aliases.
+      removeAssumedBits(IS_NOALIAS);
+      return ChangeStatus::CHANGED;
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+
+  ChangeStatus updateNoEffect(Attributor &A) {
+    if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))
+      return ChangeStatus::UNCHANGED;
+
+    if (!getAssociatedFunction())
+      return indicatePessimisticFixpoint();
+
+    if (isa<AllocaInst>(&getAssociatedValue()))
+      return indicatePessimisticFixpoint();
+
+    const auto HasNoEffectLoads = [&](const Use &U, bool &) {
+      const auto *LI = dyn_cast<LoadInst>(U.getUser());
+      return !LI || !LI->mayHaveSideEffects();
+    };
+    if (!A.checkForAllUses(HasNoEffectLoads, *this, getAssociatedValue()))
+      return indicatePessimisticFixpoint();
+
+    if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
+            getIRPosition(), this, DepClassTy::REQUIRED)) {
+      // For non-instructions, try to use AAMemoryBehavior to infer the readonly
+      // attribute
+      if (!AMemoryBehavior->isAssumedReadOnly())
+        return indicatePessimisticFixpoint();
+
+      if (AMemoryBehavior->isKnownReadOnly()) {
+        addKnownBits(IS_NOEFFECT);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      return ChangeStatus::UNCHANGED;
+    }
+
+    if (const Argument *Arg = getAssociatedArgument()) {
+      if (Arg->onlyReadsMemory()) {
+        addKnownBits(IS_NOEFFECT);
+        return ChangeStatus::UNCHANGED;
+      }
+
+      // Readonly information is not provided, and cannot be inferred from
+      // AAMemoryBehavior.
+      return indicatePessimisticFixpoint();
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+
+  ChangeStatus updateLocalInvariance(Attributor &A) {
+    if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))
+      return ChangeStatus::UNCHANGED;
+
+    // try to infer invariance from underlying objects
+    const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
+        getIRPosition(), this, DepClassTy::REQUIRED);
+    if (!AUO)
+      return ChangeStatus::UNCHANGED;
+
+    bool UsedAssumedInformation = false;
+    const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
+      if (!V.getType()->isPointerTy())
+        return true;
+      const auto *IsInvariantLoadPointer =
+          A.getOrCreateAAFor<AAInvariantLoadPointer>(IRPosition::value(V), this,
+                                                     DepClassTy::REQUIRED);
+      // Conservatively fail if invariance cannot be inferred.
+      if (!IsInvariantLoadPointer)
+        return false;
+
+      if (IsInvariantLoadPointer->isKnownLocallyInvariant())
+        return true;
+      if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
+        return false;
+
+      UsedAssumedInformation = true;
+      return true;
+    };
+    if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))
+      return indicatePessimisticFixpoint();
+
+    if (const auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
+      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+              CB, /*MustPreserveNullness=*/false)) {
+        for (const Value *Arg : CB->args()) {
+          if (!IsLocallyInvariantLoadIfPointer(*Arg))
+            return indicatePessimisticFixpoint();
+        }
+      }
+    }
+
+    if (!UsedAssumedInformation) {
+      // Pointer is known and not just assumed to be locally invariant.
+      addKnownBits(IS_LOCALLY_INVARIANT);
+      return ChangeStatus::CHANGED;
+    }
+
+    return ChangeStatus::UNCHANGED;
+  }
+};
+
+struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+};
+
+struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &) override {
+    removeAssumedBits(IS_LOCALLY_CONSTRAINED);
+  }
+};
+
+struct AAInvariantLoadPointerCallSiteReturned final
+    : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &A) override {
+    const Function *F = getAssociatedFunction();
+    assert(F && "no associated function for return from call");
+
+    if (!F->isDeclaration() && !F->isIntrinsic())
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    const auto &CB = cast<CallBase>(getAnchorValue());
+    if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
+            &CB, /*MustPreserveNullness=*/false))
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    if (F->onlyReadsMemory() && F->hasNoSync())
+      return AAInvariantLoadPointerImpl::initialize(A);
+
+    // At this point, the function is opaque, so we conservatively assume
+    // non-invariance.
+    indicatePessimisticFixpoint();
+  }
+};
+
+struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+
+  void initialize(Attributor &) override {
+    const Function *F = getAssociatedFunction();
+    assert(F && "no associated function for argument");
+
+    if (!isCallableCC(F->getCallingConv())) {
+      addKnownBits(IS_LOCALLY_CONSTRAINED);
+      return;
+    }
+
+    if (!F->hasLocalLinkage())
+      removeAssumedBits(IS_LOCALLY_CONSTRAINED);
+  }
+};
+
+struct AAInvariantLoadPointerCallSiteArgument final
+    : AAInvariantLoadPointerImpl {
+  AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
+      : AAInvariantLoadPointerImpl(IRP, A) {}
+};
+} // namespace
+
 /// ------------------------ Address Space  ------------------------------------
 namespace {
 
@@ -13048,6 +13408,7 @@ const char AAInterFnReachability::ID = 0;
 const char AAPointerInfo::ID = 0;
 const char AAAssumptionInfo::ID = 0;
 const char AAUnderlyingObjects::ID = 0;
+const char AAInvariantLoadPointer::ID = 0;
 const char AAAddressSpace::ID = 0;
 const char AAAllocationInfo::ID = 0;
 const char AAIndirectCallInfo::ID = 0;
@@ -13182,6 +13543,7 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
 
diff --git a/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp b/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
index 5e8b2a4e3d84..f02256f8a83b 100644
--- a/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
+++ b/llvm/lib/Transforms/IPO/EmbedBitcodePass.cpp
@@ -16,7 +16,6 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
-#include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
 #include <string>
@@ -34,16 +33,13 @@ PreservedAnalyses EmbedBitcodePass::run(Module &M, ModuleAnalysisManager &AM) {
 
   std::string Data;
   raw_string_ostream OS(Data);
-  // Clone the module with Thin LTO, since ThinLTOBitcodeWriterPass changes
-  // vtable linkage that would break the non-lto object code for FatLTO.
   if (IsThinLTO)
-    ThinLTOBitcodeWriterPass(OS, /*ThinLinkOS=*/nullptr)
-        .run(*llvm::CloneModule(M), AM);
+    ThinLTOBitcodeWriterPass(OS, /*ThinLinkOS=*/nullptr).run(M, AM);
   else
     BitcodeWriterPass(OS, /*ShouldPreserveUseListOrder=*/false, EmitLTOSummary)
         .run(M, AM);
 
   embedBufferInModule(M, MemoryBufferRef(Data, "ModuleData"), ".llvm.lto");
 
-  return PreservedAnalyses::all();
+  return PreservedAnalyses::none();
 }
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index f918d7e059b6..f43202eea630 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -675,14 +675,24 @@ ArgumentAccessInfo getArgumentAccessInfo(const Instruction *I,
       [](Value *Length,
          std::optional<int64_t> Offset) -> std::optional<ConstantRange> {
     auto *ConstantLength = dyn_cast<ConstantInt>(Length);
-    if (ConstantLength && Offset &&
-        ConstantLength->getValue().isStrictlyPositive()) {
-      return ConstantRange(
-          APInt(64, *Offset, true),
-          APInt(64, *Offset + ConstantLength->getSExtValue(), true));
+    if (ConstantLength && Offset) {
+      int64_t Len = ConstantLength->getSExtValue();
+
+      // Reject zero or negative lengths
+      if (Len <= 0)
+        return std::nullopt;
+
+      APInt Low(64, *Offset, true);
+      bool Overflow;
+      APInt High = Low.sadd_ov(APInt(64, Len, true), Overflow);
+      if (Overflow)
+        return std::nullopt;
+
+      return ConstantRange(Low, High);
     }
     return std::nullopt;
   };
+
   if (auto *SI = dyn_cast<StoreInst>(I)) {
     if (SI->isSimple() && &SI->getOperandUse(1) == ArgUse.U) {
       // Get the fixed type size of "SI". Since the access range of a write
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 4a06e0fa619c..b1c0e9910f37 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -92,8 +92,6 @@ STATISTIC(NumInternalFunc, "Number of internal functions");
 STATISTIC(NumColdCC, "Number of functions marked coldcc");
 STATISTIC(NumIFuncsResolved, "Number of statically resolved IFuncs");
 STATISTIC(NumIFuncsDeleted, "Number of IFuncs removed");
-STATISTIC(NumGlobalArraysPadded,
-          "Number of global arrays padded to alignment boundary");
 
 static cl::opt<bool>
     OptimizeNonFMVCallers("optimize-non-fmv-callers",
@@ -250,10 +248,10 @@ CleanupPointerRootUsers(GlobalVariable *GV,
     }
   }
 
-  for (int i = 0, e = Dead.size(); i != e; ++i) {
-    if (IsSafeComputationToRemove(Dead[i].first, GetTLI)) {
-      Dead[i].second->eraseFromParent();
-      Instruction *I = Dead[i].first;
+  for (const auto &[Inst, Store] : Dead) {
+    if (IsSafeComputationToRemove(Inst, GetTLI)) {
+      Store->eraseFromParent();
+      Instruction *I = Inst;
       do {
         if (isAllocationFn(I, GetTLI))
           break;
@@ -2037,165 +2035,6 @@ OptimizeFunctions(Module &M,
   return Changed;
 }
 
-static bool callInstIsMemcpy(CallInst *CI) {
-  if (!CI)
-    return false;
-
-  Function *F = CI->getCalledFunction();
-  if (!F || !F->isIntrinsic() || F->getIntrinsicID() != Intrinsic::memcpy)
-    return false;
-
-  return true;
-}
-
-static bool destArrayCanBeWidened(CallInst *CI) {
-  auto *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
-  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
-
-  if (!Alloca || !IsVolatile || IsVolatile->isOne())
-    return false;
-
-  if (!Alloca->isStaticAlloca())
-    return false;
-
-  if (!Alloca->getAllocatedType()->isArrayTy())
-    return false;
-
-  return true;
-}
-
-static GlobalVariable *widenGlobalVariable(GlobalVariable *OldVar,
-                                           unsigned NumBytesToPad,
-                                           unsigned NumBytesToCopy) {
-  if (!OldVar->hasInitializer())
-    return nullptr;
-
-  ConstantDataArray *DataArray =
-      dyn_cast<ConstantDataArray>(OldVar->getInitializer());
-  if (!DataArray)
-    return nullptr;
-
-  // Update to be word aligned (memcpy(...,X,...))
-  // create replacement with padded null bytes.
-  StringRef Data = DataArray->getRawDataValues();
-  std::vector<uint8_t> StrData(Data.begin(), Data.end());
-  for (unsigned int p = 0; p < NumBytesToPad; p++)
-    StrData.push_back('\0');
-  auto Arr = ArrayRef(StrData.data(), NumBytesToCopy + NumBytesToPad);
-  // Create new padded version of global variable.
-  Constant *SourceReplace = ConstantDataArray::get(OldVar->getContext(), Arr);
-  GlobalVariable *NewGV = new GlobalVariable(
-      *(OldVar->getParent()), SourceReplace->getType(), true,
-      OldVar->getLinkage(), SourceReplace, SourceReplace->getName());
-  // Copy any other attributes from original global variable
-  // e.g. unamed_addr
-  NewGV->copyAttributesFrom(OldVar);
-  NewGV->takeName(OldVar);
-  return NewGV;
-}
-
-static void widenDestArray(CallInst *CI, const unsigned NumBytesToPad,
-                           const unsigned NumBytesToCopy,
-                           ConstantDataArray *SourceDataArray) {
-
-  auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
-  if (Alloca) {
-    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
-    unsigned int TotalBytes = NumBytesToCopy + NumBytesToPad;
-    unsigned NumElementsToCopy = divideCeil(TotalBytes, ElementByteWidth);
-    // Update destination array to be word aligned (memcpy(X,...,...))
-    IRBuilder<> BuildAlloca(Alloca);
-    AllocaInst *NewAlloca = BuildAlloca.CreateAlloca(ArrayType::get(
-        Alloca->getAllocatedType()->getArrayElementType(), NumElementsToCopy));
-    NewAlloca->takeName(Alloca);
-    NewAlloca->setAlignment(Alloca->getAlign());
-    Alloca->replaceAllUsesWith(NewAlloca);
-    Alloca->eraseFromParent();
-  }
-}
-
-static bool tryWidenGlobalArrayAndDests(GlobalVariable *SourceVar,
-                                        const unsigned NumBytesToPad,
-                                        const unsigned NumBytesToCopy,
-                                        ConstantInt *BytesToCopyOp,
-                                        ConstantDataArray *SourceDataArray) {
-  auto *NewSourceGV =
-      widenGlobalVariable(SourceVar, NumBytesToPad, NumBytesToCopy);
-  if (!NewSourceGV)
-    return false;
-
-  // Update arguments of remaining uses  that
-  // are memcpys.
-  for (auto *User : SourceVar->users()) {
-    auto *CI = dyn_cast<CallInst>(User);
-    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
-      continue;
-
-    if (CI->getArgOperand(1) != SourceVar)
-      continue;
-
-    widenDestArray(CI, NumBytesToPad, NumBytesToCopy, SourceDataArray);
-
-    CI->setArgOperand(2, ConstantInt::get(BytesToCopyOp->getType(),
-                                          NumBytesToCopy + NumBytesToPad));
-  }
-  SourceVar->replaceAllUsesWith(NewSourceGV);
-
-  NumGlobalArraysPadded++;
-  return true;
-}
-
-static bool tryWidenGlobalArraysUsedByMemcpy(
-    GlobalVariable *GV,
-    function_ref<TargetTransformInfo &(Function &)> GetTTI) {
-
-  if (!GV->hasInitializer() || !GV->isConstant() || !GV->hasLocalLinkage() ||
-      !GV->hasGlobalUnnamedAddr())
-    return false;
-
-  for (auto *User : GV->users()) {
-    CallInst *CI = dyn_cast<CallInst>(User);
-    if (!callInstIsMemcpy(CI) || !destArrayCanBeWidened(CI))
-      continue;
-
-    auto *BytesToCopyOp = dyn_cast<ConstantInt>(CI->getArgOperand(2));
-    if (!BytesToCopyOp)
-      continue;
-
-    ConstantDataArray *SourceDataArray =
-        dyn_cast<ConstantDataArray>(GV->getInitializer());
-    if (!SourceDataArray)
-      continue;
-
-    unsigned NumBytesToCopy = BytesToCopyOp->getZExtValue();
-
-    auto *Alloca = cast<AllocaInst>(CI->getArgOperand(0));
-    uint64_t DZSize = Alloca->getAllocatedType()->getArrayNumElements();
-    uint64_t SZSize = SourceDataArray->getType()->getNumElements();
-    unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
-    // Calculate the number of elements to copy while avoiding floored
-    // division of integers returning wrong values i.e. copying one byte
-    // from an array of i16 would yield 0 elements to copy as supposed to 1.
-    unsigned NumElementsToCopy = divideCeil(NumBytesToCopy, ElementByteWidth);
-
-    // For safety purposes lets add a constraint and only pad when
-    // NumElementsToCopy == destination array size ==
-    // source which is a constant
-    if (NumElementsToCopy != DZSize || DZSize != SZSize)
-      continue;
-
-    unsigned NumBytesToPad =
-        GetTTI(*CI->getFunction())
-            .getNumBytesToPadGlobalArray(NumBytesToCopy,
-                                         SourceDataArray->getType());
-    if (NumBytesToPad) {
-      return tryWidenGlobalArrayAndDests(GV, NumBytesToPad, NumBytesToCopy,
-                                         BytesToCopyOp, SourceDataArray);
-    }
-  }
-  return false;
-}
-
 static bool
 OptimizeGlobalVars(Module &M,
                    function_ref<TargetTransformInfo &(Function &)> GetTTI,
@@ -2225,10 +2064,6 @@ OptimizeGlobalVars(Module &M,
       continue;
     }
 
-    // For global variable arrays called in a memcpy
-    // we try to pad to nearest valid alignment boundary
-    Changed |= tryWidenGlobalArraysUsedByMemcpy(&GV, GetTTI);
-
     Changed |= processGlobal(GV, GetTTI, GetTLI, LookupDomTree);
   }
   return Changed;
diff --git a/llvm/lib/Transforms/IPO/StripSymbols.cpp b/llvm/lib/Transforms/IPO/StripSymbols.cpp
index ecd0ade13146..ec701b6d0037 100644
--- a/llvm/lib/Transforms/IPO/StripSymbols.cpp
+++ b/llvm/lib/Transforms/IPO/StripSymbols.cpp
@@ -309,8 +309,7 @@ PreservedAnalyses StripDeadCGProfilePass::run(Module &M,
   SmallVector<Metadata *, 16> ValidCGEdges;
   for (Metadata *Edge : CGProf->operands()) {
     if (auto *EdgeAsNode = dyn_cast_or_null<MDNode>(Edge))
-      if (llvm::all_of(EdgeAsNode->operands(),
-                       [](const Metadata *V) { return V != nullptr; }))
+      if (!llvm::is_contained(EdgeAsNode->operands(), nullptr))
         ValidCGEdges.push_back(Edge);
   }
   M.setModuleFlag(Module::Append, "CG Profile",
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 418302d1edb3..1ba548b6ff06 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -2164,10 +2164,14 @@ Value *InstCombinerImpl::OptimizePointerDifference(Value *LHS, Value *RHS,
 
   // If this is a single inbounds GEP and the original sub was nuw,
   // then the final multiplication is also nuw.
-  if (auto *I = dyn_cast<Instruction>(Result))
+  if (auto *I = dyn_cast<OverflowingBinaryOperator>(Result))
     if (IsNUW && match(Offset2, m_Zero()) && Base.LHSNW.isInBounds() &&
-        I->getOpcode() == Instruction::Mul)
-      I->setHasNoUnsignedWrap();
+        (I->use_empty() || I->hasOneUse()) && I->hasNoSignedWrap() &&
+        !I->hasNoUnsignedWrap() &&
+        ((I->getOpcode() == Instruction::Mul &&
+          match(I->getOperand(1), m_NonNegative())) ||
+         I->getOpcode() == Instruction::Shl))
+      cast<Instruction>(I)->setHasNoUnsignedWrap();
 
   // If we have a 2nd GEP of the same base pointer, subtract the offsets.
   // If both GEPs are inbounds, then the subtract does not have signed overflow.
@@ -3044,6 +3048,17 @@ Instruction *InstCombinerImpl::visitFNeg(UnaryOperator &I) {
     return replaceInstUsesWith(I, NewCopySign);
   }
 
+  // fneg (shuffle x, Mask) --> shuffle (fneg x), Mask
+  ArrayRef<int> Mask;
+  if (match(OneUse, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))
+    return new ShuffleVectorInst(Builder.CreateFNegFMF(X, &I), Mask);
+
+  // fneg (reverse x) --> reverse (fneg x)
+  if (match(OneUse, m_VecReverse(m_Value(X)))) {
+    Value *Reverse = Builder.CreateVectorReverse(Builder.CreateFNegFMF(X, &I));
+    return replaceInstUsesWith(I, Reverse);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index dce695a03600..dd16cfaeecd4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -681,7 +681,7 @@ static Value *foldLogOpOfMaskedICmps(Value *LHS, Value *RHS, bool IsAnd,
         }
         Value *NewAnd = Builder.CreateAnd(A, BD);
         Value *CEVal = ConstantInt::get(A->getType(), CE);
-        return Builder.CreateICmp(CC, CEVal, NewAnd);
+        return Builder.CreateICmp(CC, NewAnd, CEVal);
       };
 
       if (Mask & BMask_Mixed)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b6ed1dc4331d..e521c9d7001a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1404,11 +1404,6 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
       !II->getCalledFunction()->isSpeculatable())
     return nullptr;
 
-  // fabs is canonicalized to fabs (shuffle ...) in foldShuffleOfUnaryOps, so
-  // avoid undoing it.
-  if (match(II, m_FAbs(m_Value())))
-    return nullptr;
-
   Value *X;
   Constant *C;
   ArrayRef<int> Mask;
@@ -1461,6 +1456,43 @@ InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {
   return new ShuffleVectorInst(NewIntrinsic, Mask);
 }
 
+/// If all arguments of the intrinsic are reverses, try to pull the reverse
+/// after the intrinsic.
+Value *InstCombinerImpl::foldReversedIntrinsicOperands(IntrinsicInst *II) {
+  if (!isTriviallyVectorizable(II->getIntrinsicID()))
+    return nullptr;
+
+  // At least 1 operand must be a reverse with 1 use because we are creating 2
+  // instructions.
+  if (none_of(II->args(), [](Value *V) {
+        return match(V, m_OneUse(m_VecReverse(m_Value())));
+      }))
+    return nullptr;
+
+  Value *X;
+  Constant *C;
+  SmallVector<Value *> NewArgs;
+  for (Use &Arg : II->args()) {
+    if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                           Arg.getOperandNo(), nullptr))
+      NewArgs.push_back(Arg);
+    else if (match(&Arg, m_VecReverse(m_Value(X))))
+      NewArgs.push_back(X);
+    else if (isSplatValue(Arg))
+      NewArgs.push_back(Arg);
+    else if (match(&Arg, m_ImmConstant(C)))
+      NewArgs.push_back(Builder.CreateVectorReverse(C));
+    else
+      return nullptr;
+  }
+
+  // intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)
+  Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;
+  Instruction *NewIntrinsic = Builder.CreateIntrinsic(
+      II->getType(), II->getIntrinsicID(), NewArgs, FPI);
+  return Builder.CreateVectorReverse(NewIntrinsic);
+}
+
 /// Fold the following cases and accepts bswap and bitreverse intrinsics:
 ///   bswap(logic_op(bswap(x), y)) --> logic_op(x, bswap(y))
 ///   bswap(logic_op(bswap(x), bswap(y))) --> logic_op(x, y) (ignores multiuse)
@@ -1972,6 +2004,21 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       return BinaryOperator::CreateOr(I0, I1);
     }
 
+    // smin(smax(X, -1), 1) -> scmp(X, 0)
+    // smax(smin(X, 1), -1) -> scmp(X, 0)
+    // At this point, smax(smin(X, 1), -1) is changed to smin(smax(X, -1)
+    // And i1's have been changed to and/ors
+    // So we only need to check for smin
+    if (IID == Intrinsic::smin) {
+      if (match(I0, m_OneUse(m_SMax(m_Value(X), m_AllOnes()))) &&
+          match(I1, m_One())) {
+        Value *Zero = ConstantInt::get(X->getType(), 0);
+        return replaceInstUsesWith(
+            CI,
+            Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {X, Zero}));
+      }
+    }
+
     if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
       // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)
       // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)
@@ -3554,23 +3601,6 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     }
     break;
   }
-  case Intrinsic::vector_reverse: {
-    Value *Vec = II->getArgOperand(0);
-    // Note: We canonicalize reverse after binops, so we don't need a
-    // corresponding binop case here. TODO: Consider canonicalizing
-    // reverse after fneg?
-
-    // rev(unop rev(X)) --> unop X
-    Value *X;
-    if (match(Vec, m_OneUse(m_UnOp(m_VecReverse(m_Value(X)))))) {
-      auto *OldUnOp = cast<UnaryOperator>(Vec);
-      auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags(
-          OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),
-          II->getIterator());
-      return replaceInstUsesWith(CI, NewUnOp);
-    }
-    break;
-  }
   case Intrinsic::experimental_vp_reverse: {
     Value *X;
     Value *Vec = II->getArgOperand(0);
@@ -3578,6 +3608,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     if (!match(Mask, m_AllOnes()))
       break;
     Value *EVL = II->getArgOperand(2);
+    // TODO: Canonicalize experimental.vp.reverse after unop/binops?
     // rev(unop rev(X)) --> unop X
     if (match(Vec,
               m_OneUse(m_UnOp(m_Intrinsic<Intrinsic::experimental_vp_reverse>(
@@ -3872,6 +3903,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   if (Instruction *Shuf = foldShuffledIntrinsicOperands(II))
     return Shuf;
 
+  if (Value *Reverse = foldReversedIntrinsicOperands(II))
+    return replaceInstUsesWith(*II, Reverse);
+
   // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
   // context, so it is handled in visitCallBase and we should trigger it.
   return visitCallBase(*II);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 0894ca92086f..c6f317a668cf 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -708,11 +708,13 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
     return I;
   };
 
-  Value *PtrBase = GEPLHS->getOperand(0);
-  if (PtrBase == RHS && CanFold(GEPLHS->getNoWrapFlags())) {
+  CommonPointerBase Base = CommonPointerBase::compute(GEPLHS, RHS);
+  if (Base.Ptr == RHS && CanFold(Base.LHSNW)) {
     // ((gep Ptr, OFFSET) cmp Ptr)   ---> (OFFSET cmp 0).
-    Value *Offset = EmitGEPOffset(GEPLHS);
-    return NewICmp(GEPLHS->getNoWrapFlags(), Offset,
+    Type *IdxTy = DL.getIndexType(GEPLHS->getType());
+    Value *Offset =
+        EmitGEPOffsets(Base.LHSGEPs, Base.LHSNW, IdxTy, /*RewriteGEPs=*/true);
+    return NewICmp(Base.LHSNW, Offset,
                    Constant::getNullValue(Offset->getType()));
   }
 
@@ -750,6 +752,7 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
 
     // If the base pointers are different, but the indices are the same, just
     // compare the base pointer.
+    Value *PtrBase = GEPLHS->getOperand(0);
     if (PtrBase != GEPRHS->getOperand(0)) {
       bool IndicesTheSame =
           GEPLHS->getNumOperands() == GEPRHS->getNumOperands() &&
@@ -5117,6 +5120,18 @@ static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
   return nullptr;
 }
 
+/// Return true if X is a multiple of C.
+/// TODO: Handle non-power-of-2 factors.
+static bool isMultipleOf(Value *X, const APInt &C, const SimplifyQuery &Q) {
+  if (C.isOne())
+    return true;
+
+  if (!C.isPowerOf2())
+    return false;
+
+  return MaskedValueIsZero(X, C - 1, Q);
+}
+
 /// Try to fold icmp (binop), X or icmp X, (binop).
 /// TODO: A large part of this logic is duplicated in InstSimplify's
 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
@@ -5275,66 +5290,62 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
     return new ICmpInst(Pred, Y, Z);
   }
 
-  // icmp slt (A + -1), Op1 -> icmp sle A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
-      match(B, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
-
-  // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
-      match(B, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
-
-  // icmp sle (A + 1), Op1 -> icmp slt A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
-
-  // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
+  if (ICmpInst::isRelational(Pred)) {
+    // Return if both X and Y is divisible by Z/-Z.
+    // TODO: Generalize to check if (X - Y) is divisible by Z/-Z.
+    auto ShareCommonDivisor = [&Q](Value *X, Value *Y, Value *Z,
+                                   bool IsNegative) -> bool {
+      const APInt *OffsetC;
+      if (!match(Z, m_APInt(OffsetC)))
+        return false;
 
-  // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
-      match(D, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
+      // Fast path for Z == 1/-1.
+      if (IsNegative ? OffsetC->isAllOnes() : OffsetC->isOne())
+        return true;
 
-  // icmp sle Op0, (C + -1) -> icmp slt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
-      match(D, m_AllOnes()))
-    return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
+      APInt C = *OffsetC;
+      if (IsNegative)
+        C.negate();
+      // Note: -INT_MIN is also negative.
+      if (!C.isStrictlyPositive())
+        return false;
 
-  // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
+      return isMultipleOf(X, C, Q) && isMultipleOf(Y, C, Q);
+    };
 
-  // icmp slt Op0, (C + 1) -> icmp sle Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
+    // TODO: The subtraction-related identities shown below also hold, but
+    // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
+    // wouldn't happen even if they were implemented.
+    //
+    // icmp ult (A - 1), Op1 -> icmp ule A, Op1
+    // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
+    // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
+    // icmp ule Op0, (C - 1) -> icmp ult Op0, C
+
+    // icmp slt (A + -1), Op1 -> icmp sle A, Op1
+    // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
+    // icmp sle (A + 1), Op1 -> icmp slt A, Op1
+    // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
+    // icmp ule (A + 1), Op0 -> icmp ult A, Op1
+    // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
+    if (A && NoOp0WrapProblem &&
+        ShareCommonDivisor(A, Op1, B,
+                           ICmpInst::isLT(Pred) || ICmpInst::isGE(Pred)))
+      return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), A,
+                          Op1);
 
-  // TODO: The subtraction-related identities shown below also hold, but
-  // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
-  // wouldn't happen even if they were implemented.
-  //
-  // icmp ult (A - 1), Op1 -> icmp ule A, Op1
-  // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
-  // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
-  // icmp ule Op0, (C - 1) -> icmp ult Op0, C
-
-  // icmp ule (A + 1), Op0 -> icmp ult A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
-
-  // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
-  if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
-    return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
-
-  // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
-
-  // icmp ult Op0, (C + 1) -> icmp ule Op0, C
-  if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
-    return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
+    // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
+    // icmp sle Op0, (C + -1) -> icmp slt Op0, C
+    // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
+    // icmp slt Op0, (C + 1) -> icmp sle Op0, C
+    // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
+    // icmp ult Op0, (C + 1) -> icmp ule Op0, C
+    if (C && NoOp1WrapProblem &&
+        ShareCommonDivisor(Op0, C, D,
+                           ICmpInst::isGT(Pred) || ICmpInst::isLE(Pred)))
+      return new ICmpInst(ICmpInst::getFlippedStrictnessPredicate(Pred), Op0,
+                          C);
+  }
 
   // if C1 has greater magnitude than C2:
   //  icmp (A + C1), (C + C2) -> icmp (A + C3), C
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 8c9de862fe8f..9adde8094d44 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -148,6 +148,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldItoFPtoI(CastInst &FI);
   Instruction *visitSelectInst(SelectInst &SI);
   Instruction *foldShuffledIntrinsicOperands(IntrinsicInst *II);
+  Value *foldReversedIntrinsicOperands(IntrinsicInst *II);
   Instruction *visitCallInst(CallInst &CI);
   Instruction *visitInvokeInst(InvokeInst &II);
   Instruction *visitCallBrInst(CallBrInst &CBI);
@@ -620,6 +621,20 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN,
                              bool AllowMultipleUses = false);
 
+  /// Try to fold binary operators whose operands are simple interleaved
+  /// recurrences to a single recurrence. This is a common pattern in reduction
+  /// operations.
+  /// Example:
+  ///   %phi1 = phi [init1, %BB1], [%op1, %BB2]
+  ///   %phi2 = phi [init2, %BB1], [%op2, %BB2]
+  ///   %op1 = binop %phi1, constant1
+  ///   %op2 = binop %phi2, constant2
+  ///   %rdx = binop %op1, %op2
+  /// -->
+  ///   %phi_combined = phi [init_combined, %BB1], [%op_combined, %BB2]
+  ///   %rdx_combined = binop %phi_combined, constant_combined
+  Instruction *foldBinopWithRecurrence(BinaryOperator &BO);
+
   /// For a binary operator with 2 phi operands, try to hoist the binary
   /// operation before the phi. This can result in fewer instructions in
   /// patterns where at least one set of phi operands simplifies.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index f946c3856948..1a8d4215b5b7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -542,27 +542,39 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
         }
       }
     } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
-      // If this is extracting an element from a shufflevector, figure out where
-      // it came from and extract from the appropriate input element instead.
-      // Restrict the following transformation to fixed-length vector.
-      if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(Index)) {
-        int SrcIdx =
-            SVI->getMaskValue(cast<ConstantInt>(Index)->getZExtValue());
-        Value *Src;
-        unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
-                                ->getNumElements();
-
-        if (SrcIdx < 0)
-          return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
-        if (SrcIdx < (int)LHSWidth)
-          Src = SVI->getOperand(0);
-        else {
-          SrcIdx -= LHSWidth;
-          Src = SVI->getOperand(1);
+      int SplatIndex = getSplatIndex(SVI->getShuffleMask());
+      // We know the all-0 splat must be reading from the first operand, even
+      // in the case of scalable vectors (vscale is always > 0).
+      if (SplatIndex == 0)
+        return ExtractElementInst::Create(SVI->getOperand(0),
+                                          Builder.getInt64(0));
+
+      if (isa<FixedVectorType>(SVI->getType())) {
+        std::optional<int> SrcIdx;
+        // getSplatIndex returns -1 to mean not-found.
+        if (SplatIndex != -1)
+          SrcIdx = SplatIndex;
+        else if (ConstantInt *CI = dyn_cast<ConstantInt>(Index))
+          SrcIdx = SVI->getMaskValue(CI->getZExtValue());
+
+        if (SrcIdx) {
+          Value *Src;
+          unsigned LHSWidth =
+              cast<FixedVectorType>(SVI->getOperand(0)->getType())
+                  ->getNumElements();
+
+          if (*SrcIdx < 0)
+            return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
+          if (*SrcIdx < (int)LHSWidth)
+            Src = SVI->getOperand(0);
+          else {
+            *SrcIdx -= LHSWidth;
+            Src = SVI->getOperand(1);
+          }
+          Type *Int64Ty = Type::getInt64Ty(EI.getContext());
+          return ExtractElementInst::Create(
+              Src, ConstantInt::get(Int64Ty, *SrcIdx, false));
         }
-        Type *Int64Ty = Type::getInt64Ty(EI.getContext());
-        return ExtractElementInst::Create(
-            Src, ConstantInt::get(Int64Ty, SrcIdx, false));
       }
     } else if (auto *CI = dyn_cast<CastInst>(I)) {
       // Canonicalize extractelement(cast) -> cast(extractelement).
@@ -2533,20 +2545,6 @@ static Instruction *foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf,
 
   bool IsFNeg = S0->getOpcode() == Instruction::FNeg;
 
-  // Match 1-input (unary) shuffle.
-  // shuffle (fneg/fabs X), Mask --> fneg/fabs (shuffle X, Mask)
-  if (S0->hasOneUse() && match(Shuf.getOperand(1), m_Poison())) {
-    Value *NewShuf = Builder.CreateShuffleVector(X, Shuf.getShuffleMask());
-    if (IsFNeg)
-      return UnaryOperator::CreateFNegFMF(NewShuf, S0);
-
-    Function *FAbs = Intrinsic::getOrInsertDeclaration(
-        Shuf.getModule(), Intrinsic::fabs, Shuf.getType());
-    CallInst *NewF = CallInst::Create(FAbs, {NewShuf});
-    NewF->setFastMathFlags(S0->getFastMathFlags());
-    return NewF;
-  }
-
   // Match 2-input (binary) shuffle.
   auto *S1 = dyn_cast<Instruction>(Shuf.getOperand(1));
   Value *Y;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index ce4202926135..91a1b61ddc48 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -205,9 +205,8 @@ Value *InstCombinerImpl::EmitGEPOffset(GEPOperator *GEP, bool RewriteGEP) {
     Builder.SetInsertPoint(Inst);
 
   Value *Offset = EmitGEPOffset(GEP);
-  // If a non-trivial GEP has other uses, rewrite it to avoid duplicating
-  // the offset arithmetic.
-  if (Inst && !GEP->hasOneUse() && !GEP->hasAllConstantIndices() &&
+  // Rewrite non-trivial GEPs to avoid duplicating the offset arithmetic.
+  if (Inst && !GEP->hasAllConstantIndices() &&
       !GEP->getSourceElementType()->isIntegerTy(8)) {
     replaceInstUsesWith(
         *Inst, Builder.CreateGEP(Builder.getInt8Ty(), GEP->getPointerOperand(),
@@ -1989,7 +1988,114 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN,
   return replaceInstUsesWith(I, NewPN);
 }
 
+Instruction *InstCombinerImpl::foldBinopWithRecurrence(BinaryOperator &BO) {
+  if (!BO.isAssociative())
+    return nullptr;
+
+  // Find the interleaved binary ops.
+  auto Opc = BO.getOpcode();
+  auto *BO0 = dyn_cast<BinaryOperator>(BO.getOperand(0));
+  auto *BO1 = dyn_cast<BinaryOperator>(BO.getOperand(1));
+  if (!BO0 || !BO1 || !BO0->hasNUses(2) || !BO1->hasNUses(2) ||
+      BO0->getOpcode() != Opc || BO1->getOpcode() != Opc ||
+      !BO0->isAssociative() || !BO1->isAssociative() ||
+      BO0->getParent() != BO1->getParent())
+    return nullptr;
+
+  assert(BO.isCommutative() && BO0->isCommutative() && BO1->isCommutative() &&
+         "Expected commutative instructions!");
+
+  // Find the matching phis, forming the recurrences.
+  PHINode *PN0, *PN1;
+  Value *Start0, *Step0, *Start1, *Step1;
+  if (!matchSimpleRecurrence(BO0, PN0, Start0, Step0) || !PN0->hasOneUse() ||
+      !matchSimpleRecurrence(BO1, PN1, Start1, Step1) || !PN1->hasOneUse() ||
+      PN0->getParent() != PN1->getParent())
+    return nullptr;
+
+  assert(PN0->getNumIncomingValues() == 2 && PN1->getNumIncomingValues() == 2 &&
+         "Expected PHIs with two incoming values!");
+
+  // Convert the start and step values to constants.
+  auto *Init0 = dyn_cast<Constant>(Start0);
+  auto *Init1 = dyn_cast<Constant>(Start1);
+  auto *C0 = dyn_cast<Constant>(Step0);
+  auto *C1 = dyn_cast<Constant>(Step1);
+  if (!Init0 || !Init1 || !C0 || !C1)
+    return nullptr;
+
+  // Fold the recurrence constants.
+  auto *Init = ConstantFoldBinaryInstruction(Opc, Init0, Init1);
+  auto *C = ConstantFoldBinaryInstruction(Opc, C0, C1);
+  if (!Init || !C)
+    return nullptr;
+
+  // Create the reduced PHI.
+  auto *NewPN = PHINode::Create(PN0->getType(), PN0->getNumIncomingValues(),
+                                "reduced.phi");
+
+  // Create the new binary op.
+  auto *NewBO = BinaryOperator::Create(Opc, NewPN, C);
+  if (Opc == Instruction::FAdd || Opc == Instruction::FMul) {
+    // Intersect FMF flags for FADD and FMUL.
+    FastMathFlags Intersect = BO0->getFastMathFlags() &
+                              BO1->getFastMathFlags() & BO.getFastMathFlags();
+    NewBO->setFastMathFlags(Intersect);
+  } else {
+    OverflowTracking Flags;
+    Flags.AllKnownNonNegative = false;
+    Flags.AllKnownNonZero = false;
+    Flags.mergeFlags(*BO0);
+    Flags.mergeFlags(*BO1);
+    Flags.mergeFlags(BO);
+    Flags.applyFlags(*NewBO);
+  }
+  NewBO->takeName(&BO);
+
+  for (unsigned I = 0, E = PN0->getNumIncomingValues(); I != E; ++I) {
+    auto *V = PN0->getIncomingValue(I);
+    auto *BB = PN0->getIncomingBlock(I);
+    if (V == Init0) {
+      assert(((PN1->getIncomingValue(0) == Init1 &&
+               PN1->getIncomingBlock(0) == BB) ||
+              (PN1->getIncomingValue(1) == Init1 &&
+               PN1->getIncomingBlock(1) == BB)) &&
+             "Invalid incoming block!");
+      NewPN->addIncoming(Init, BB);
+    } else if (V == BO0) {
+      assert(((PN1->getIncomingValue(0) == BO1 &&
+               PN1->getIncomingBlock(0) == BB) ||
+              (PN1->getIncomingValue(1) == BO1 &&
+               PN1->getIncomingBlock(1) == BB)) &&
+             "Invalid incoming block!");
+      NewPN->addIncoming(NewBO, BB);
+    } else
+      llvm_unreachable("Unexpected incoming value!");
+  }
+
+  LLVM_DEBUG(dbgs() << "  Combined " << *PN0 << "\n           " << *BO0
+                    << "\n      with " << *PN1 << "\n           " << *BO1
+                    << '\n');
+
+  // Insert the new recurrence and remove the old (dead) ones.
+  InsertNewInstWith(NewPN, PN0->getIterator());
+  InsertNewInstWith(NewBO, BO0->getIterator());
+
+  eraseInstFromFunction(
+      *replaceInstUsesWith(*BO0, PoisonValue::get(BO0->getType())));
+  eraseInstFromFunction(
+      *replaceInstUsesWith(*BO1, PoisonValue::get(BO1->getType())));
+  eraseInstFromFunction(*PN0);
+  eraseInstFromFunction(*PN1);
+
+  return replaceInstUsesWith(BO, NewBO);
+}
+
 Instruction *InstCombinerImpl::foldBinopWithPhiOperands(BinaryOperator &BO) {
+  // Attempt to fold binary operators whose operands are simple recurrences.
+  if (auto *NewBO = foldBinopWithRecurrence(BO))
+    return NewBO;
+
   // TODO: This should be similar to the incoming values check in foldOpIntoPhi:
   //       we are guarding against replicating the binop in >1 predecessor.
   //       This could miss matching a phi with 2 constant incoming values.
@@ -2993,10 +3099,6 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         return replaceInstUsesWith(GEP, V);
       return &GEP;
     }
-
-    // TODO: 1) Scalarize splat operands, 2) scalarize entire instruction if
-    // possible (decide on canonical form for pointer broadcast), 3) exploit
-    // undef elements to decrease demanded bits
   }
 
   // Eliminate unneeded casts for indices, and replace indices which displace
@@ -3058,6 +3160,32 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     return replaceInstUsesWith(GEP, NewGEP);
   }
 
+  // Scalarize vector operands; prefer splat-of-gep.as canonical form.
+  // Note that this looses information about undef lanes; we run it after
+  // demanded bits to partially mitigate that loss.
+  if (GEPType->isVectorTy() && llvm::any_of(GEP.operands(), [](Value *Op) {
+        return Op->getType()->isVectorTy() && getSplatValue(Op);
+      })) {
+    SmallVector<Value *> NewOps;
+    for (auto &Op : GEP.operands()) {
+      if (Op->getType()->isVectorTy())
+        if (Value *Scalar = getSplatValue(Op)) {
+          NewOps.push_back(Scalar);
+          continue;
+        }
+      NewOps.push_back(Op);
+    }
+
+    Value *Res = Builder.CreateGEP(GEP.getSourceElementType(), NewOps[0],
+                                   ArrayRef(NewOps).drop_front(), GEP.getName(),
+                                   GEP.getNoWrapFlags());
+    if (!Res->getType()->isVectorTy()) {
+      ElementCount EC = cast<VectorType>(GEPType)->getElementCount();
+      Res = Builder.CreateVectorSplat(EC, Res);
+    }
+    return replaceInstUsesWith(GEP, Res);
+  }
+
   // Check to see if the inputs to the PHI node are getelementptr instructions.
   if (auto *PN = dyn_cast<PHINode>(PtrOp)) {
     if (Value *NewPtrOp = foldGEPOfPhi(GEP, PN, Builder))
@@ -3472,13 +3600,13 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
   auto Removable =
       isAllocSiteRemovable(&MI, Users, TLI, KnowInitZero | KnowInitUndef);
   if (Removable) {
-    for (unsigned i = 0, e = Users.size(); i != e; ++i) {
+    for (WeakTrackingVH &User : Users) {
       // Lowering all @llvm.objectsize and MTI calls first because they may use
       // a bitcast/GEP of the alloca we are removing.
-      if (!Users[i])
-       continue;
+      if (!User)
+        continue;
 
-      Instruction *I = cast<Instruction>(&*Users[i]);
+      Instruction *I = cast<Instruction>(&*User);
 
       if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
         if (II->getIntrinsicID() == Intrinsic::objectsize) {
@@ -3489,7 +3617,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
             Worklist.add(Inserted);
           replaceInstUsesWith(*I, Result);
           eraseInstFromFunction(*I);
-          Users[i] = nullptr; // Skip examining in the next loop.
+          User = nullptr; // Skip examining in the next loop.
           continue;
         }
         if (auto *MTI = dyn_cast<MemTransferInst>(I)) {
@@ -3505,11 +3633,11 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
         }
       }
     }
-    for (unsigned i = 0, e = Users.size(); i != e; ++i) {
-      if (!Users[i])
+    for (WeakTrackingVH &User : Users) {
+      if (!User)
         continue;
 
-      Instruction *I = cast<Instruction>(&*Users[i]);
+      Instruction *I = cast<Instruction>(&*User);
 
       if (ICmpInst *C = dyn_cast<ICmpInst>(I)) {
         replaceInstUsesWith(*C,
diff --git a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
index 10e908ef73ce..e7a6fa48e900 100644
--- a/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
+++ b/llvm/lib/Transforms/Instrumentation/LowerAllowCheckPass.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
@@ -73,7 +74,7 @@ static void emitRemark(IntrinsicInst *II, OptimizationRemarkEmitter &ORE,
 static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
                              const ProfileSummaryInfo *PSI,
                              OptimizationRemarkEmitter &ORE,
-                             const std::vector<unsigned int> &cutoffs) {
+                             const LowerAllowCheckPass::Options &Opts) {
   SmallVector<std::pair<IntrinsicInst *, bool>, 16> ReplaceWithValue;
   std::unique_ptr<RandomNumberGenerator> Rng;
 
@@ -88,8 +89,10 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
       return HotPercentileCutoff;
     else if (II->getIntrinsicID() == Intrinsic::allow_ubsan_check) {
       auto *Kind = cast<ConstantInt>(II->getArgOperand(0));
-      if (Kind->getZExtValue() < cutoffs.size())
-        return cutoffs[Kind->getZExtValue()];
+      if (Kind->getZExtValue() < Opts.cutoffs.size())
+        return Opts.cutoffs[Kind->getZExtValue()];
+    } else if (II->getIntrinsicID() == Intrinsic::allow_runtime_check) {
+      return Opts.runtime_check;
     }
 
     return 0;
@@ -111,31 +114,29 @@ static bool removeUbsanTraps(Function &F, const BlockFrequencyInfo &BFI,
     return ShouldRemoveRandom() || ShouldRemoveHot(*(II->getParent()), cutoff);
   };
 
-  for (BasicBlock &BB : F) {
-    for (Instruction &I : BB) {
-      IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
-      if (!II)
-        continue;
-      auto ID = II->getIntrinsicID();
-      switch (ID) {
-      case Intrinsic::allow_ubsan_check:
-      case Intrinsic::allow_runtime_check: {
-        ++NumChecksTotal;
-
-        bool ToRemove = ShouldRemove(II);
-
-        ReplaceWithValue.push_back({
-            II,
-            ToRemove,
-        });
-        if (ToRemove)
-          ++NumChecksRemoved;
-        emitRemark(II, ORE, ToRemove);
-        break;
-      }
-      default:
-        break;
-      }
+  for (Instruction &I : instructions(F)) {
+    IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
+    if (!II)
+      continue;
+    auto ID = II->getIntrinsicID();
+    switch (ID) {
+    case Intrinsic::allow_ubsan_check:
+    case Intrinsic::allow_runtime_check: {
+      ++NumChecksTotal;
+
+      bool ToRemove = ShouldRemove(II);
+
+      ReplaceWithValue.push_back({
+          II,
+          ToRemove,
+      });
+      if (ToRemove)
+        ++NumChecksRemoved;
+      emitRemark(II, ORE, ToRemove);
+      break;
+    }
+    default:
+      break;
     }
   }
 
@@ -158,8 +159,10 @@ PreservedAnalyses LowerAllowCheckPass::run(Function &F,
   OptimizationRemarkEmitter &ORE =
       AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
 
-  return removeUbsanTraps(F, BFI, PSI, ORE, Opts.cutoffs)
-             ? PreservedAnalyses::none()
+  return removeUbsanTraps(F, BFI, PSI, ORE, Opts)
+             // We do not change the CFG, we only replace the intrinsics with
+             // true or false.
+             ? PreservedAnalyses::none().preserveSet<CFGAnalyses>()
              : PreservedAnalyses::all();
 }
 
@@ -181,14 +184,22 @@ void LowerAllowCheckPass::printPipeline(
   // correctness.
   // TODO: print shorter output by combining adjacent runs, etc.
   int i = 0;
+  bool printed = false;
   for (unsigned int cutoff : Opts.cutoffs) {
     if (cutoff > 0) {
-      if (i > 0)
+      if (printed)
         OS << ";";
       OS << "cutoffs[" << i << "]=" << cutoff;
+      printed = true;
     }
 
     i++;
   }
+  if (Opts.runtime_check) {
+    if (printed)
+      OS << ";";
+    OS << "runtime_check=" << Opts.runtime_check;
+  }
+
   OS << '>';
 }
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 3941bed37eba..82bafa39a805 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -282,6 +282,12 @@ static cl::opt<bool> ClPoisonUndefVectors(
              "unaffected by this flag (see -msan-poison-undef)."),
     cl::Hidden, cl::init(false));
 
+static cl::opt<bool> ClPreciseDisjointOr(
+    "msan-precise-disjoint-or",
+    cl::desc("Precisely poison disjoint OR. If false (legacy behavior), "
+             "disjointedness is ignored (i.e., 1|1 is initialized)."),
+    cl::Hidden, cl::init(false));
+
 static cl::opt<bool>
     ClHandleICmp("msan-handle-icmp",
                  cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
@@ -665,6 +671,7 @@ class MemorySanitizer {
 
   // These arrays are indexed by log2(AccessSize).
   FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
+  FunctionCallee MaybeWarningVarSizeFn;
   FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
 
   /// Run-time helper that generates a new origin value for a stack
@@ -939,7 +946,9 @@ void MemorySanitizer::createUserspaceApi(Module &M,
     MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
         FunctionName, TLI.getAttrList(C, {0, 1}, /*Signed=*/false),
         IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
-
+    MaybeWarningVarSizeFn = M.getOrInsertFunction(
+        "__msan_maybe_warning_N", TLI.getAttrList(C, {}, /*Signed=*/false),
+        IRB.getVoidTy(), PtrTy, IRB.getInt64Ty(), IRB.getInt32Ty());
     FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
     MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
         FunctionName, TLI.getAttrList(C, {0, 2}, /*Signed=*/false),
@@ -1248,7 +1257,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // Constants likely will be eliminated by follow-up passes.
     if (isa<Constant>(V))
       return false;
-
     ++SplittableBlocksCount;
     return ClInstrumentationWithCallThreshold >= 0 &&
            SplittableBlocksCount > ClInstrumentationWithCallThreshold;
@@ -1384,9 +1392,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       std::tie(ShadowPtr, OriginPtr) =
           getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
 
-      StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
+      [[maybe_unused]] StoreInst *NewSI =
+          IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
       LLVM_DEBUG(dbgs() << "  STORE: " << *NewSI << "\n");
-      (void)NewSI;
 
       if (SI->isAtomic())
         SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
@@ -1447,18 +1455,32 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const DataLayout &DL = F.getDataLayout();
     TypeSize TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
     unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
-    if (instrumentWithCalls(ConvertedShadow) &&
-        SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
-      FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
+    if (instrumentWithCalls(ConvertedShadow) && !MS.CompileKernel) {
       // ZExt cannot convert between vector and scalar
       ConvertedShadow = convertShadowToScalar(ConvertedShadow, IRB);
       Value *ConvertedShadow2 =
           IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
-      CallBase *CB = IRB.CreateCall(
-          Fn, {ConvertedShadow2,
-               MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
-      CB->addParamAttr(0, Attribute::ZExt);
-      CB->addParamAttr(1, Attribute::ZExt);
+
+      if (SizeIndex < kNumberOfAccessSizes) {
+        FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
+        CallBase *CB = IRB.CreateCall(
+            Fn,
+            {ConvertedShadow2,
+             MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
+        CB->addParamAttr(0, Attribute::ZExt);
+        CB->addParamAttr(1, Attribute::ZExt);
+      } else {
+        FunctionCallee Fn = MS.MaybeWarningVarSizeFn;
+        Value *ShadowAlloca = IRB.CreateAlloca(ConvertedShadow2->getType(), 0u);
+        IRB.CreateStore(ConvertedShadow2, ShadowAlloca);
+        unsigned ShadowSize = DL.getTypeAllocSize(ConvertedShadow2->getType());
+        CallBase *CB = IRB.CreateCall(
+            Fn,
+            {ShadowAlloca, ConstantInt::get(IRB.getInt64Ty(), ShadowSize),
+             MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
+        CB->addParamAttr(1, Attribute::ZExt);
+        CB->addParamAttr(2, Attribute::ZExt);
+      }
     } else {
       Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
       Instruction *CheckTerm = SplitBlockAndInsertIfThen(
@@ -1999,18 +2021,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       Value *Shadow = ShadowMap[V];
       if (!Shadow) {
         LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
-        (void)I;
         assert(Shadow && "No shadow for a value");
       }
       return Shadow;
     }
     // Handle fully undefined values
     // (partially undefined constant vectors are handled later)
-    if (UndefValue *U = dyn_cast<UndefValue>(V)) {
+    if ([[maybe_unused]] UndefValue *U = dyn_cast<UndefValue>(V)) {
       Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
                                                         : getCleanShadow(V);
       LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
-      (void)U;
       return AllOnes;
     }
     if (Argument *A = dyn_cast<Argument>(V)) {
@@ -2059,10 +2079,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
             } else {
               Value *Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
               const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
-              Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
-                                                 CopyAlign, Size);
+              [[maybe_unused]] Value *Cpy = EntryIRB.CreateMemCpy(
+                  CpShadowPtr, CopyAlign, Base, CopyAlign, Size);
               LLVM_DEBUG(dbgs() << "  ByValCpy: " << *Cpy << "\n");
-              (void)Cpy;
 
               if (MS.TrackOrigins) {
                 Value *OriginPtr = getOriginPtrForArgument(EntryIRB, ArgOffset);
@@ -2481,23 +2500,40 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
   void visitOr(BinaryOperator &I) {
     IRBuilder<> IRB(&I);
-    //  "Or" of 1 and a poisoned value results in unpoisoned value.
-    //  1|1 => 1;     0|1 => 1;     p|1 => 1;
-    //  1|0 => 1;     0|0 => 0;     p|0 => p;
-    //  1|p => 1;     0|p => p;     p|p => p;
-    //  S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
+    //  "Or" of 1 and a poisoned value results in unpoisoned value:
+    //    1|1 => 1;     0|1 => 1;     p|1 => 1;
+    //    1|0 => 1;     0|0 => 0;     p|0 => p;
+    //    1|p => 1;     0|p => p;     p|p => p;
+    //
+    //    S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
+    //
+    //  Addendum if the "Or" is "disjoint":
+    //    1|1 => p;
+    //    S = S | (V1 & V2)
     Value *S1 = getShadow(&I, 0);
     Value *S2 = getShadow(&I, 1);
-    Value *V1 = IRB.CreateNot(I.getOperand(0));
-    Value *V2 = IRB.CreateNot(I.getOperand(1));
+    Value *V1 = I.getOperand(0);
+    Value *V2 = I.getOperand(1);
     if (V1->getType() != S1->getType()) {
       V1 = IRB.CreateIntCast(V1, S1->getType(), false);
       V2 = IRB.CreateIntCast(V2, S2->getType(), false);
     }
+
+    Value *NotV1 = IRB.CreateNot(V1);
+    Value *NotV2 = IRB.CreateNot(V2);
+
     Value *S1S2 = IRB.CreateAnd(S1, S2);
-    Value *V1S2 = IRB.CreateAnd(V1, S2);
-    Value *S1V2 = IRB.CreateAnd(S1, V2);
-    setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
+    Value *S2NotV1 = IRB.CreateAnd(NotV1, S2);
+    Value *S1NotV2 = IRB.CreateAnd(S1, NotV2);
+
+    Value *S = IRB.CreateOr({S1S2, S2NotV1, S1NotV2});
+
+    if (ClPreciseDisjointOr && cast<PossiblyDisjointInst>(&I)->isDisjoint()) {
+      Value *V1V2 = IRB.CreateAnd(V1, V2);
+      S = IRB.CreateOr(S, V1V2, "_ms_disjoint");
+    }
+
+    setShadow(&I, S);
     setOriginForNaryOp(I);
   }
 
@@ -3278,22 +3314,51 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOrigin(&I, getOrigin(Op));
   }
 
-  void handleCountZeroes(IntrinsicInst &I) {
+  // Uninitialized bits are ok if they appear after the leading/trailing 0's
+  // and a 1. If the input is all zero, it is fully initialized iff
+  // !is_zero_poison.
+  //
+  // e.g., for ctlz, with little-endian, if 0/1 are initialized bits with
+  // concrete value 0/1, and ? is an uninitialized bit:
+  //       - 0001 0??? is fully initialized
+  //       - 000? ???? is fully uninitialized (*)
+  //       - ???? ???? is fully uninitialized
+  //       - 0000 0000 is fully uninitialized if is_zero_poison,
+  //                      fully initialized   otherwise
+  //
+  // (*) TODO: arguably, since the number of zeros is in the range [3, 8], we
+  //     only need to poison 4 bits.
+  //
+  // OutputShadow =
+  //      ((ConcreteZerosCount >= ShadowZerosCount) && !AllZeroShadow)
+  //   || (is_zero_poison && AllZeroSrc)
+  void handleCountLeadingTrailingZeros(IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
     Value *Src = I.getArgOperand(0);
+    Value *SrcShadow = getShadow(Src);
 
-    // Set the Output shadow based on input Shadow
-    Value *BoolShadow = IRB.CreateIsNotNull(getShadow(Src), "_mscz_bs");
+    Value *False = IRB.getInt1(false);
+    Value *ConcreteZerosCount = IRB.CreateIntrinsic(
+        I.getType(), I.getIntrinsicID(), {Src, /*is_zero_poison=*/False});
+    Value *ShadowZerosCount = IRB.CreateIntrinsic(
+        I.getType(), I.getIntrinsicID(), {SrcShadow, /*is_zero_poison=*/False});
+
+    Value *CompareConcreteZeros = IRB.CreateICmpUGE(
+        ConcreteZerosCount, ShadowZerosCount, "_mscz_cmp_zeros");
+
+    Value *NotAllZeroShadow =
+        IRB.CreateIsNotNull(SrcShadow, "_mscz_shadow_not_null");
+    Value *OutputShadow =
+        IRB.CreateAnd(CompareConcreteZeros, NotAllZeroShadow, "_mscz_main");
 
     // If zero poison is requested, mix in with the shadow
     Constant *IsZeroPoison = cast<Constant>(I.getOperand(1));
     if (!IsZeroPoison->isZeroValue()) {
       Value *BoolZeroPoison = IRB.CreateIsNull(Src, "_mscz_bzp");
-      BoolShadow = IRB.CreateOr(BoolShadow, BoolZeroPoison, "_mscz_bs");
+      OutputShadow = IRB.CreateOr(OutputShadow, BoolZeroPoison, "_mscz_bs");
     }
 
-    Value *OutputShadow =
-        IRB.CreateSExt(BoolShadow, getShadowTy(Src), "_mscz_os");
+    OutputShadow = IRB.CreateSExt(OutputShadow, getShadowTy(Src), "_mscz_os");
 
     setShadow(&I, OutputShadow);
     setOriginForNaryOp(I);
@@ -4710,7 +4775,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     case Intrinsic::ctlz:
     case Intrinsic::cttz:
-      handleCountZeroes(I);
+      handleCountLeadingTrailingZeros(I);
       break;
     case Intrinsic::masked_compressstore:
       handleMaskedCompressStore(I);
@@ -5503,7 +5568,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         insertShadowCheck(A, &CB);
         Size = DL.getTypeAllocSize(A->getType());
       } else {
-        Value *Store = nullptr;
+        [[maybe_unused]] Value *Store = nullptr;
         // Compute the Shadow for arg even if it is ByVal, because
         // in that case getShadow() will copy the actual arg shadow to
         // __msan_param_tls.
@@ -5560,7 +5625,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                             getOriginPtrForArgument(IRB, ArgOffset));
           }
         }
-        (void)Store;
         assert(Store != nullptr);
         LLVM_DEBUG(dbgs() << "  Param:" << *Store << "\n");
       }
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 9c92a7feb6e6..6f06a260e238 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1176,15 +1176,20 @@ class PGOUseFunc {
 
   void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);
 
+  /// Get the profile record, assign it to \p ProfileRecord, handle errors if
+  /// necessary, and assign \p ProgramMaxCount. \returns true if there are no
+  /// errors.
+  bool getRecord(IndexedInstrProfReader *PGOReader);
+
   // Read counts for the instrumented BB from profile.
-  bool readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
+  bool readCounters(bool &AllZeros,
                     InstrProfRecord::CountPseudoKind &PseudoKind);
 
   // Populate the counts for all BBs.
   void populateCounters();
 
   // Set block coverage based on profile coverage values.
-  void populateCoverage(IndexedInstrProfReader *PGOReader);
+  void populateCoverage();
 
   // Set the branch weights based on the count values.
   void setBranchWeights();
@@ -1208,7 +1213,7 @@ class PGOUseFunc {
   uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }
 
   // Return the profile record for this function;
-  InstrProfRecord &getProfileRecord() { return ProfileRecord; }
+  NamedInstrProfRecord &getProfileRecord() { return ProfileRecord; }
 
   // Return the auxiliary BB information.
   PGOUseBBInfo &getBBInfo(const BasicBlock *BB) const {
@@ -1246,7 +1251,7 @@ class PGOUseFunc {
   uint32_t ProfileCountSize = 0;
 
   // ProfileRecord for this function.
-  InstrProfRecord ProfileRecord;
+  NamedInstrProfRecord ProfileRecord;
 
   // Function hotness info derived from profile.
   FuncFreqAttr FreqAttr;
@@ -1441,14 +1446,9 @@ void PGOUseFunc::handleInstrProfError(Error Err, uint64_t MismatchedFuncSum) {
   });
 }
 
-// Read the profile from ProfileFileName and assign the value to the
-// instrumented BB and the edges. This function also updates ProgramMaxCount.
-// Return true if the profile are successfully read, and false on errors.
-bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
-                              InstrProfRecord::CountPseudoKind &PseudoKind) {
-  auto &Ctx = M->getContext();
+bool PGOUseFunc::getRecord(IndexedInstrProfReader *PGOReader) {
   uint64_t MismatchedFuncSum = 0;
-  Expected<InstrProfRecord> Result = PGOReader->getInstrProfRecord(
+  auto Result = PGOReader->getInstrProfRecord(
       FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
       &MismatchedFuncSum);
   if (Error E = Result.takeError()) {
@@ -1456,6 +1456,16 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
     return false;
   }
   ProfileRecord = std::move(Result.get());
+  ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
+  return true;
+}
+
+// Read the profile from ProfileFileName and assign the value to the
+// instrumented BB and the edges. Return true if the profile are successfully
+// read, and false on errors.
+bool PGOUseFunc::readCounters(bool &AllZeros,
+                              InstrProfRecord::CountPseudoKind &PseudoKind) {
+  auto &Ctx = M->getContext();
   PseudoKind = ProfileRecord.getCountPseudoKind();
   if (PseudoKind != InstrProfRecord::NotPseudo) {
     return true;
@@ -1488,22 +1498,13 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros,
         DS_Warning));
     return false;
   }
-  ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
   return true;
 }
 
-void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
-  uint64_t MismatchedFuncSum = 0;
-  Expected<InstrProfRecord> Result = PGOReader->getInstrProfRecord(
-      FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,
-      &MismatchedFuncSum);
-  if (auto Err = Result.takeError()) {
-    handleInstrProfError(std::move(Err), MismatchedFuncSum);
-    return;
-  }
+void PGOUseFunc::populateCoverage() {
   IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;
 
-  std::vector<uint64_t> &CountsFromProfile = Result.get().Counts;
+  ArrayRef<uint64_t> CountsFromProfile = ProfileRecord.Counts;
   DenseMap<const BasicBlock *, bool> Coverage;
   unsigned Index = 0;
   for (auto &BB : F)
@@ -2243,8 +2244,10 @@ static bool annotateAllFunctions(
     PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, LI, PSI, IsCS,
                     InstrumentFuncEntry, InstrumentLoopEntries,
                     HasSingleByteCoverage);
+    if (!Func.getRecord(PGOReader.get()))
+      continue;
     if (HasSingleByteCoverage) {
-      Func.populateCoverage(PGOReader.get());
+      Func.populateCoverage();
       continue;
     }
     // When PseudoKind is set to a vaule other than InstrProfRecord::NotPseudo,
@@ -2253,7 +2256,7 @@ static bool annotateAllFunctions(
     // attribute and drop all the profile counters.
     InstrProfRecord::CountPseudoKind PseudoKind = InstrProfRecord::NotPseudo;
     bool AllZeros = false;
-    if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))
+    if (!Func.readCounters(AllZeros, PseudoKind))
       continue;
     if (AllZeros) {
       F.setEntryCount(ProfileCount(0, Function::PCT_Real));
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 4a2eb9284a6e..85dd9a1bf716 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1017,10 +1017,10 @@ struct DSEState {
       }
     }
 
-    // Treat byval or inalloca arguments the same as Allocas, stores to them are
-    // dead at the end of the function.
+    // Treat byval, inalloca or dead on return arguments the same as Allocas,
+    // stores to them are dead at the end of the function.
     for (Argument &AI : F.args())
-      if (AI.hasPassPointeeByValueCopyAttr())
+      if (AI.hasPassPointeeByValueCopyAttr() || AI.hasDeadOnReturnAttr())
         InvisibleToCallerAfterRet.insert({&AI, true});
 
     // Collect whether there is any irreducible control flow in the function.
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 381de60fcfac..b6cb987c0423 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -493,7 +493,7 @@ struct CallValue {
 
   static bool canHandle(Instruction *Inst) {
     CallInst *CI = dyn_cast<CallInst>(Inst);
-    if (!CI || !CI->onlyReadsMemory() ||
+    if (!CI || (!CI->onlyReadsMemory() && !CI->onlyWritesMemory()) ||
         // FIXME: Currently the calls which may access the thread id may
         // be considered as not accessing the memory. But this is
         // problematic for coroutines, since coroutines may resume in a
@@ -1626,14 +1626,19 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         !(MemInst.isValid() && !MemInst.mayReadFromMemory()))
       LastStore = nullptr;
 
-    // If this is a read-only call, process it.
-    if (CallValue::canHandle(&Inst)) {
+    // If this is a read-only or write-only call, process it. Skip store
+    // MemInsts, as they will be more precisely handled later on. Also skip
+    // memsets, as DSE may be able to optimize them better by removing the
+    // earlier rather than later store.
+    if (CallValue::canHandle(&Inst) &&
+        (!MemInst.isValid() || !MemInst.isStore()) && !isa<MemSetInst>(&Inst)) {
       // If we have an available version of this call, and if it is the right
       // generation, replace this instruction.
       std::pair<Instruction *, unsigned> InVal = AvailableCalls.lookup(&Inst);
       if (InVal.first != nullptr &&
           isSameMemGeneration(InVal.second, CurrentGeneration, InVal.first,
-                              &Inst)) {
+                              &Inst) &&
+          InVal.first->mayReadFromMemory() == Inst.mayReadFromMemory()) {
         LLVM_DEBUG(dbgs() << "EarlyCSE CSE CALL: " << Inst
                           << "  to: " << *InVal.first << '\n');
         if (!DebugCounter::shouldExecute(CSECounter)) {
@@ -1651,6 +1656,11 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
         continue;
       }
 
+      // Increase memory generation for writes. Do this before inserting
+      // the call, so it has the generation after the write occurred.
+      if (Inst.mayWriteToMemory())
+        ++CurrentGeneration;
+
       // Otherwise, remember that we have this instruction.
       AvailableCalls.insert(&Inst, std::make_pair(&Inst, CurrentGeneration));
       continue;
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index cf84366c4200..68094c354cf4 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2490,8 +2490,12 @@ static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo,
   NewCond->takeName(&I);
   I.replaceAllUsesWith(NewCond);
   eraseInstruction(I, SafetyInfo, MSSAU);
-  eraseInstruction(*cast<Instruction>(Cond1), SafetyInfo, MSSAU);
-  eraseInstruction(*cast<Instruction>(Cond2), SafetyInfo, MSSAU);
+  Instruction &CondI1 = *cast<Instruction>(Cond1);
+  Instruction &CondI2 = *cast<Instruction>(Cond2);
+  salvageDebugInfo(CondI1);
+  salvageDebugInfo(CondI2);
+  eraseInstruction(CondI1, SafetyInfo, MSSAU);
+  eraseInstruction(CondI2, SafetyInfo, MSSAU);
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
index e5355d23b5d6..0664eed072a9 100644
--- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -467,8 +467,7 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
     SE.forgetLoop(L);
     // Set incoming value to poison for phi nodes in the exit block.
     for (PHINode &P : ExitBlock->phis()) {
-      std::fill(P.incoming_values().begin(), P.incoming_values().end(),
-                PoisonValue::get(P.getType()));
+      llvm::fill(P.incoming_values(), PoisonValue::get(P.getType()));
     }
     ORE.emit([&]() {
       return OptimizationRemark(DEBUG_TYPE, "NeverExecutes", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 1e16677e5a56..5bb5f749d9f1 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -235,8 +235,8 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
 // matrix by exchanging the two columns.
 static void interChangeDependencies(CharMatrix &DepMatrix, unsigned FromIndx,
                                     unsigned ToIndx) {
-  for (unsigned I = 0, E = DepMatrix.size(); I < E; ++I)
-    std::swap(DepMatrix[I][ToIndx], DepMatrix[I][FromIndx]);
+  for (auto &Row : DepMatrix)
+    std::swap(Row[ToIndx], Row[FromIndx]);
 }
 
 // Check if a direction vector is lexicographically positive. Return true if it
@@ -244,11 +244,9 @@ static void interChangeDependencies(CharMatrix &DepMatrix, unsigned FromIndx,
 // [Theorem] A permutation of the loops in a perfect nest is legal if and only
 // if the direction matrix, after the same permutation is applied to its
 // columns, has no ">" direction as the leftmost non-"=" direction in any row.
-static std::optional<bool> isLexicographicallyPositive(std::vector<char> &DV,
-                                                       unsigned Begin,
-                                                       unsigned End) {
-  ArrayRef<char> DVRef(DV);
-  for (unsigned char Direction : DVRef.slice(Begin, End - Begin)) {
+static std::optional<bool>
+isLexicographicallyPositive(ArrayRef<char> DV, unsigned Begin, unsigned End) {
+  for (unsigned char Direction : DV.slice(Begin, End - Begin)) {
     if (Direction == '<')
       return true;
     if (Direction == '>' || Direction == '*')
@@ -309,18 +307,18 @@ static void populateWorklist(Loop &L, LoopVector &LoopList) {
   LoopList.push_back(CurrentLoop);
 }
 
-static bool hasSupportedLoopDepth(SmallVectorImpl<Loop *> &LoopList,
+static bool hasSupportedLoopDepth(ArrayRef<Loop *> LoopList,
                                   OptimizationRemarkEmitter &ORE) {
   unsigned LoopNestDepth = LoopList.size();
   if (LoopNestDepth < MinLoopNestDepth || LoopNestDepth > MaxLoopNestDepth) {
     LLVM_DEBUG(dbgs() << "Unsupported depth of loop nest " << LoopNestDepth
                       << ", the supported range is [" << MinLoopNestDepth
                       << ", " << MaxLoopNestDepth << "].\n");
-    Loop **OuterLoop = LoopList.begin();
+    Loop *OuterLoop = LoopList.front();
     ORE.emit([&]() {
       return OptimizationRemarkMissed(DEBUG_TYPE, "UnsupportedLoopNestDepth",
-                                      (*OuterLoop)->getStartLoc(),
-                                      (*OuterLoop)->getHeader())
+                                      OuterLoop->getStartLoc(),
+                                      OuterLoop->getHeader())
              << "Unsupported depth of loop nest, the supported range is ["
              << std::to_string(MinLoopNestDepth) << ", "
              << std::to_string(MaxLoopNestDepth) << "].\n";
@@ -377,7 +375,7 @@ class LoopInterchangeLegality {
     return OuterInnerReductions;
   }
 
-  const SmallVectorImpl<PHINode *> &getInnerLoopInductions() const {
+  const ArrayRef<PHINode *> getInnerLoopInductions() const {
     return InnerLoopInductions;
   }
 
@@ -552,10 +550,8 @@ struct LoopInterchange {
     // For the old pass manager CacheCost would be null.
     DenseMap<const Loop *, unsigned> CostMap;
     if (CC != nullptr) {
-      const auto &LoopCosts = CC->getLoopCosts();
-      for (unsigned i = 0; i < LoopCosts.size(); i++) {
-        CostMap[LoopCosts[i].first] = i;
-      }
+      for (const auto &[Idx, Cost] : enumerate(CC->getLoopCosts()))
+        CostMap[Cost.first] = Idx;
     }
     // We try to achieve the globally optimal memory access for the loopnest,
     // and do interchange based on a bubble-sort fasion. We start from
@@ -972,8 +968,8 @@ areInnerLoopExitPHIsSupported(Loop *InnerL, Loop *OuterL,
 static bool areOuterLoopExitPHIsSupported(Loop *OuterLoop, Loop *InnerLoop) {
   BasicBlock *LoopNestExit = OuterLoop->getUniqueExitBlock();
   for (PHINode &PHI : LoopNestExit->phis()) {
-    for (unsigned i = 0; i < PHI.getNumIncomingValues(); i++) {
-      Instruction *IncomingI = dyn_cast<Instruction>(PHI.getIncomingValue(i));
+    for (Value *Incoming : PHI.incoming_values()) {
+      Instruction *IncomingI = dyn_cast<Instruction>(Incoming);
       if (!IncomingI || IncomingI->getParent() != OuterLoop->getLoopLatch())
         continue;
 
@@ -1132,15 +1128,14 @@ int LoopInterchangeProfitability::getInstrOrderCost() {
   for (BasicBlock *BB : InnerLoop->blocks()) {
     for (Instruction &Ins : *BB) {
       if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&Ins)) {
-        unsigned NumOp = GEP->getNumOperands();
         bool FoundInnerInduction = false;
         bool FoundOuterInduction = false;
-        for (unsigned i = 0; i < NumOp; ++i) {
+        for (Value *Op : GEP->operands()) {
           // Skip operands that are not SCEV-able.
-          if (!SE->isSCEVable(GEP->getOperand(i)->getType()))
+          if (!SE->isSCEVable(Op->getType()))
             continue;
 
-          const SCEV *OperandVal = SE->getSCEV(GEP->getOperand(i));
+          const SCEV *OperandVal = SE->getSCEV(Op);
           const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(OperandVal);
           if (!AR)
             continue;
@@ -1220,8 +1215,8 @@ LoopInterchangeProfitability::isProfitablePerInstrOrderCost() {
 
 /// Return true if we can vectorize the loop specified by \p LoopId.
 static bool canVectorize(const CharMatrix &DepMatrix, unsigned LoopId) {
-  for (unsigned I = 0; I != DepMatrix.size(); I++) {
-    char Dir = DepMatrix[I][LoopId];
+  for (const auto &Dep : DepMatrix) {
+    char Dir = Dep[LoopId];
     if (Dir != 'I' && Dir != '=')
       return false;
   }
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 4ba69034d644..845afa6d4228 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -923,10 +923,11 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
 /// If S involves the addition of a constant integer value, return that integer
 /// value, and mutate S to point to a new SCEV with that value excluded.
 static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
-  if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
-    if (C->getAPInt().getSignificantBits() <= 64) {
-      S = SE.getConstant(C->getType(), 0);
-      return Immediate::getFixed(C->getValue()->getSExtValue());
+  const APInt *C;
+  if (match(S, m_scev_APInt(C))) {
+    if (C->getSignificantBits() <= 64) {
+      S = SE.getConstant(S->getType(), 0);
+      return Immediate::getFixed(C->getSExtValue());
     }
   } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
     SmallVector<const SCEV *, 8> NewOps(Add->operands());
@@ -942,14 +943,10 @@ static Immediate ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) {
                            // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
                            SCEV::FlagAnyWrap);
     return Result;
-  } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
-    if (EnableVScaleImmediates && M->getNumOperands() == 2) {
-      if (const SCEVConstant *C = dyn_cast<SCEVConstant>(M->getOperand(0)))
-        if (isa<SCEVVScale>(M->getOperand(1))) {
-          S = SE.getConstant(M->getType(), 0);
-          return Immediate::getScalable(C->getValue()->getSExtValue());
-        }
-    }
+  } else if (EnableVScaleImmediates &&
+             match(S, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale()))) {
+    S = SE.getConstant(S->getType(), 0);
+    return Immediate::getScalable(C->getSExtValue());
   }
   return Immediate::getZero();
 }
@@ -1133,23 +1130,22 @@ static bool isHighCostExpansion(const SCEV *S,
     return false;
   }
 
-  if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
-    if (Mul->getNumOperands() == 2) {
-      // Multiplication by a constant is ok
-      if (isa<SCEVConstant>(Mul->getOperand(0)))
-        return isHighCostExpansion(Mul->getOperand(1), Processed, SE);
-
-      // If we have the value of one operand, check if an existing
-      // multiplication already generates this expression.
-      if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Mul->getOperand(1))) {
-        Value *UVal = U->getValue();
-        for (User *UR : UVal->users()) {
-          // If U is a constant, it may be used by a ConstantExpr.
-          Instruction *UI = dyn_cast<Instruction>(UR);
-          if (UI && UI->getOpcode() == Instruction::Mul &&
-              SE.isSCEVable(UI->getType())) {
-            return SE.getSCEV(UI) == Mul;
-          }
+  const SCEV *Op0, *Op1;
+  if (match(S, m_scev_Mul(m_SCEV(Op0), m_SCEV(Op1)))) {
+    // Multiplication by a constant is ok
+    if (isa<SCEVConstant>(Op0))
+      return isHighCostExpansion(Op1, Processed, SE);
+
+    // If we have the value of one operand, check if an existing
+    // multiplication already generates this expression.
+    if (const auto *U = dyn_cast<SCEVUnknown>(Op1)) {
+      Value *UVal = U->getValue();
+      for (User *UR : UVal->users()) {
+        // If U is a constant, it may be used by a ConstantExpr.
+        Instruction *UI = dyn_cast<Instruction>(UR);
+        if (UI && UI->getOpcode() == Instruction::Mul &&
+            SE.isSCEVable(UI->getType())) {
+          return SE.getSCEV(UI) == S;
         }
       }
     }
@@ -3333,14 +3329,11 @@ static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst,
     IncOffset = Immediate::getFixed(IncConst->getValue()->getSExtValue());
   } else {
     // Look for mul(vscale, constant), to detect a scalable offset.
-    auto *IncVScale = dyn_cast<SCEVMulExpr>(IncExpr);
-    if (!IncVScale || IncVScale->getNumOperands() != 2 ||
-        !isa<SCEVVScale>(IncVScale->getOperand(1)))
-      return false;
-    auto *Scale = dyn_cast<SCEVConstant>(IncVScale->getOperand(0));
-    if (!Scale || Scale->getType()->getScalarSizeInBits() > 64)
+    const APInt *C;
+    if (!match(IncExpr, m_scev_Mul(m_scev_APInt(C), m_SCEVVScale())) ||
+        C->getSignificantBits() > 64)
       return false;
-    IncOffset = Immediate::getScalable(Scale->getValue()->getSExtValue());
+    IncOffset = Immediate::getScalable(C->getSExtValue());
   }
 
   if (!isAddressUse(TTI, UserInst, Operand))
@@ -3818,6 +3811,8 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
     return nullptr;
   }
   const SCEV *Start, *Step;
+  const SCEVConstant *Op0;
+  const SCEV *Op1;
   if (match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step)))) {
     // Split a non-zero base out of an addrec.
     if (Start->isZero())
@@ -3839,19 +3834,13 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
                               // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
                               SCEV::FlagAnyWrap);
     }
-  } else if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(S)) {
+  } else if (match(S, m_scev_Mul(m_SCEVConstant(Op0), m_SCEV(Op1)))) {
     // Break (C * (a + b + c)) into C*a + C*b + C*c.
-    if (Mul->getNumOperands() != 2)
-      return S;
-    if (const SCEVConstant *Op0 =
-        dyn_cast<SCEVConstant>(Mul->getOperand(0))) {
-      C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
-      const SCEV *Remainder =
-        CollectSubexprs(Mul->getOperand(1), C, Ops, L, SE, Depth+1);
-      if (Remainder)
-        Ops.push_back(SE.getMulExpr(C, Remainder));
-      return nullptr;
-    }
+    C = C ? cast<SCEVConstant>(SE.getMulExpr(C, Op0)) : Op0;
+    const SCEV *Remainder = CollectSubexprs(Op1, C, Ops, L, SE, Depth + 1);
+    if (Remainder)
+      Ops.push_back(SE.getMulExpr(C, Remainder));
+    return nullptr;
   }
   return S;
 }
@@ -6479,10 +6468,6 @@ struct SCEVDbgValueBuilder {
   /// Chain (non-affine) SCEVs are not supported.
   bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
     assert(SAR.isAffine() && "Expected affine SCEV");
-    // TODO: Is this check needed?
-    if (isa<SCEVAddRecExpr>(SAR.getStart()))
-      return false;
-
     const SCEV *Start = SAR.getStart();
     const SCEV *Stride = SAR.getStepRecurrence(SE);
 
@@ -6550,11 +6535,6 @@ struct SCEVDbgValueBuilder {
   bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
                            ScalarEvolution &SE) {
     assert(SAR.isAffine() && "Expected affine SCEV");
-    if (isa<SCEVAddRecExpr>(SAR.getStart())) {
-      LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: "
-                        << SAR << '\n');
-      return false;
-    }
     const SCEV *Start = SAR.getStart();
     const SCEV *Stride = SAR.getStepRecurrence(SE);
 
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index a712b4632e9a..b3bffeb7ea41 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -2382,7 +2382,7 @@ class LowerMatrixIntrinsics {
       llvm::copy(C.vectors(), std::back_inserter(CondV));
     } else {
       CondV.resize(A.getNumVectors());
-      std::fill(CondV.begin(), CondV.end(), Cond);
+      llvm::fill(CondV, Cond);
     }
 
     for (auto [CV, AV, BV] : llvm::zip_equal(CondV, A.vectors(), B.vectors()))
diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
index 4291f3aee0cd..a83cbd17a70d 100644
--- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp
+++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
@@ -479,6 +479,10 @@ BCECmpChain::BCECmpChain(const std::vector<BasicBlock *> &Blocks, PHINode &Phi,
   BaseIdentifier BaseId;
   for (BasicBlock *const Block : Blocks) {
     assert(Block && "invalid block");
+    if (Block->hasAddressTaken()) {
+      LLVM_DEBUG(dbgs() << "cannot merge blocks with blockaddress\n");
+      return;
+    }
     std::optional<BCECmpBlock> Comparison = visitCmpBlock(
         Phi.getIncomingValueForBlock(Block), Block, Phi.getParent(), BaseId);
     if (!Comparison) {
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 584cb0e89ef6..7eeaaa0d9960 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -505,13 +505,14 @@ class NewGVN {
   MemorySSAWalker *MSSAWalker = nullptr;
   AssumptionCache *AC = nullptr;
   const DataLayout &DL;
-  std::unique_ptr<PredicateInfo> PredInfo;
 
   // These are the only two things the create* functions should have
   // side-effects on due to allocating memory.
   mutable BumpPtrAllocator ExpressionAllocator;
   mutable ArrayRecycler<Value *> ArgRecycler;
   mutable TarjanSCC SCCFinder;
+
+  std::unique_ptr<PredicateInfo> PredInfo;
   const SimplifyQuery SQ;
 
   // Number of function arguments, used by ranking
@@ -673,7 +674,9 @@ class NewGVN {
          TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
          const DataLayout &DL)
       : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), AC(AC), DL(DL),
-        PredInfo(std::make_unique<PredicateInfo>(F, *DT, *AC)),
+        // Reuse ExpressionAllocator for PredicateInfo as well.
+        PredInfo(
+            std::make_unique<PredicateInfo>(F, *DT, *AC, ExpressionAllocator)),
         SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false,
            /*CanUseUndef=*/false) {}
 
diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp
index 778a6a012556..343da5b2e470 100644
--- a/llvm/lib/Transforms/Scalar/Reassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp
@@ -83,10 +83,10 @@ static void PrintOps(Instruction *I, const SmallVectorImpl<ValueEntry> &Ops) {
   Module *M = I->getModule();
   dbgs() << Instruction::getOpcodeName(I->getOpcode()) << " "
        << *Ops[0].Op->getType() << '\t';
-  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
+  for (const ValueEntry &Op : Ops) {
     dbgs() << "[ ";
-    Ops[i].Op->printAsOperand(dbgs(), false, M);
-    dbgs() << ", #" << Ops[i].Rank << "] ";
+    Op.Op->printAsOperand(dbgs(), false, M);
+    dbgs() << ", #" << Op.Rank << "] ";
   }
 }
 #endif
@@ -1585,9 +1585,9 @@ Value *ReassociatePass::OptimizeAdd(Instruction *I,
   // where they are actually the same multiply.
   unsigned MaxOcc = 0;
   Value *MaxOccVal = nullptr;
-  for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
+  for (const ValueEntry &Op : Ops) {
     BinaryOperator *BOp =
-        isReassociableOp(Ops[i].Op, Instruction::Mul, Instruction::FMul);
+        isReassociableOp(Op.Op, Instruction::Mul, Instruction::FMul);
     if (!BOp)
       continue;
 
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index b6e7f4aebeb8..becda960a16f 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -4070,18 +4070,27 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
   //   => select cond, gep(ptr1, idx), gep(ptr2, idx)
   // and  gep ptr, (select cond, idx1, idx2)
   //   => select cond, gep(ptr, idx1), gep(ptr, idx2)
+  // We also allow for i1 zext indices, which are equivalent to selects.
   bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
     // Check whether the GEP has exactly one select operand and all indices
     // will become constant after the transform.
-    SelectInst *Sel = dyn_cast<SelectInst>(GEPI.getPointerOperand());
+    Instruction *Sel = dyn_cast<SelectInst>(GEPI.getPointerOperand());
     for (Value *Op : GEPI.indices()) {
       if (auto *SI = dyn_cast<SelectInst>(Op)) {
         if (Sel)
           return false;
 
         Sel = SI;
-        if (!isa<ConstantInt>(Sel->getTrueValue()) ||
-            !isa<ConstantInt>(Sel->getFalseValue()))
+        if (!isa<ConstantInt>(SI->getTrueValue()) ||
+            !isa<ConstantInt>(SI->getFalseValue()))
+          return false;
+        continue;
+      }
+      if (auto *ZI = dyn_cast<ZExtInst>(Op)) {
+        if (Sel)
+          return false;
+        Sel = ZI;
+        if (!ZI->getSrcTy()->isIntegerTy(1))
           return false;
         continue;
       }
@@ -4107,8 +4116,16 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
       return NewOps;
     };
 
-    Value *True = Sel->getTrueValue();
-    Value *False = Sel->getFalseValue();
+    Value *Cond, *True, *False;
+    if (auto *SI = dyn_cast<SelectInst>(Sel)) {
+      Cond = SI->getCondition();
+      True = SI->getTrueValue();
+      False = SI->getFalseValue();
+    } else {
+      Cond = Sel->getOperand(0);
+      True = ConstantInt::get(Sel->getType(), 1);
+      False = ConstantInt::get(Sel->getType(), 0);
+    }
     SmallVector<Value *> TrueOps = GetNewOps(True);
     SmallVector<Value *> FalseOps = GetNewOps(False);
 
@@ -4123,8 +4140,8 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
         IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(),
                       False->getName() + ".sroa.gep", NW);
 
-    Value *NSel = IRB.CreateSelect(Sel->getCondition(), NTrue, NFalse,
-                                   Sel->getName() + ".sroa.sel");
+    Value *NSel =
+        IRB.CreateSelect(Cond, NTrue, NFalse, Sel->getName() + ".sroa.sel");
     Visited.erase(&GEPI);
     GEPI.replaceAllUsesWith(NSel);
     GEPI.eraseFromParent();
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 98c65ae11b1c..c8255742c41b 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -377,33 +377,14 @@ bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
 ///
 /// Possible improvements:
 /// - Check fully overlapping fragments and not only identical fragments.
-/// - Support dbg.declare. dbg.label, and possibly other meta instructions being
-///   part of the sequence of consecutive instructions.
 static bool
 DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
   SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
   SmallDenseSet<DebugVariable> VariableSet;
   for (auto &I : reverse(*BB)) {
-    for (DbgRecord &DR : reverse(I.getDbgRecordRange())) {
-      if (isa<DbgLabelRecord>(DR)) {
-        // Emulate existing behaviour (see comment below for dbg.declares).
-        // FIXME: Don't do this.
-        VariableSet.clear();
-        continue;
-      }
-
+    for (DbgVariableRecord &DR :
+         reverse(filterDbgVars(I.getDbgRecordRange()))) {
       DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
-      // Skip declare-type records, as the debug intrinsic method only works
-      // on dbg.value intrinsics.
-      if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
-        // The debug intrinsic method treats dbg.declares are "non-debug"
-        // instructions (i.e., a break in a consecutive range of debug
-        // intrinsics). Emulate that to create identical outputs. See
-        // "Possible improvements" above.
-        // FIXME: Delete the line below.
-        VariableSet.clear();
-        continue;
-      }
 
       DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
                         DVR.getDebugLoc()->getInlinedAt());
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 79c7245fcf91..d9cd24c4a7ca 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -471,10 +471,6 @@ CodeExtractor::getLifetimeMarkers(const CodeExtractorAnalysisCache &CEAC,
         Info.LifeEnd = IntrInst;
         continue;
       }
-      // At this point, permit debug uses outside of the region.
-      // This is fixed in a later call to fixupDebugInfoPostExtraction().
-      if (isa<DbgInfoIntrinsic>(IntrInst))
-        continue;
     }
     // Find untracked uses of the address, bail.
     if (!definedInRegion(Blocks, U))
@@ -1018,13 +1014,13 @@ Function *CodeExtractor::constructFunctionDeclaration(
       case Attribute::DeadOnUnwind:
       case Attribute::Range:
       case Attribute::Initializes:
-      case Attribute::SanitizedPaddedGlobal:
       case Attribute::NoExt:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
       case Attribute::EmptyKey:
       case Attribute::TombstoneKey:
+      case Attribute::DeadOnReturn:
         llvm_unreachable("Not a function attribute");
       }
 
@@ -1078,10 +1074,6 @@ static void applyFirstDebugLoc(Function *oldFunction,
       return any_of(*BB, [&BranchI](const Instruction &I) {
         if (!I.getDebugLoc())
           return false;
-        // Don't use source locations attached to debug-intrinsics: they could
-        // be from completely unrelated scopes.
-        if (isa<DbgInfoIntrinsic>(I))
-          return false;
         BranchI->setDebugLoc(I.getDebugLoc());
         return true;
       });
@@ -1330,7 +1322,6 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
   //  2) They need to point to fresh metadata, e.g. because they currently
   //     point to a variable in the wrong scope.
   SmallDenseMap<DINode *, DINode *> RemappedMetadata;
-  SmallVector<Instruction *, 4> DebugIntrinsicsToDelete;
   SmallVector<DbgVariableRecord *, 4> DVRsToDelete;
   DenseMap<const MDNode *, MDNode *> Cache;
 
@@ -1358,8 +1349,10 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     if (!NewLabel) {
       DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
           *OldLabel->getScope(), *NewSP, Ctx, Cache);
-      NewLabel = DILabel::get(Ctx, NewScope, OldLabel->getName(),
-                              OldLabel->getFile(), OldLabel->getLine());
+      NewLabel =
+          DILabel::get(Ctx, NewScope, OldLabel->getName(), OldLabel->getFile(),
+                       OldLabel->getLine(), OldLabel->getColumn(),
+                       OldLabel->isArtificial(), OldLabel->getCoroSuspendIdx());
     }
     LabelRecord->setLabel(cast<DILabel>(NewLabel));
   };
@@ -1372,66 +1365,29 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
       }
 
       DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
-      // Apply the two updates that dbg.values get: invalid operands, and
-      // variable metadata fixup.
+      // If any of the used locations are invalid, delete the record.
       if (any_of(DVR.location_ops(), IsInvalidLocation)) {
         DVRsToDelete.push_back(&DVR);
         continue;
       }
+
+      // DbgAssign intrinsics have an extra Value argument:
       if (DVR.isDbgAssign() && IsInvalidLocation(DVR.getAddress())) {
         DVRsToDelete.push_back(&DVR);
         continue;
       }
+
+      // If the variable was in the scope of the old function, i.e. it was not
+      // inlined, point the intrinsic to a fresh variable within the new
+      // function.
       if (!DVR.getDebugLoc().getInlinedAt())
         DVR.setVariable(GetUpdatedDIVariable(DVR.getVariable()));
     }
   };
 
-  for (Instruction &I : instructions(NewFunc)) {
+  for (Instruction &I : instructions(NewFunc))
     UpdateDbgRecordsOnInst(I);
 
-    auto *DII = dyn_cast<DbgInfoIntrinsic>(&I);
-    if (!DII)
-      continue;
-
-    // Point the intrinsic to a fresh label within the new function if the
-    // intrinsic was not inlined from some other function.
-    if (auto *DLI = dyn_cast<DbgLabelInst>(&I)) {
-      UpdateDbgLabel(DLI);
-      continue;
-    }
-
-    auto *DVI = cast<DbgVariableIntrinsic>(DII);
-    // If any of the used locations are invalid, delete the intrinsic.
-    if (any_of(DVI->location_ops(), IsInvalidLocation)) {
-      DebugIntrinsicsToDelete.push_back(DVI);
-      continue;
-    }
-    // DbgAssign intrinsics have an extra Value argument:
-    if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
-        DAI && IsInvalidLocation(DAI->getAddress())) {
-      DebugIntrinsicsToDelete.push_back(DVI);
-      continue;
-    }
-    // If the variable was in the scope of the old function, i.e. it was not
-    // inlined, point the intrinsic to a fresh variable within the new function.
-    if (!DVI->getDebugLoc().getInlinedAt()) {
-      DILocalVariable *OldVar = DVI->getVariable();
-      DINode *&NewVar = RemappedMetadata[OldVar];
-      if (!NewVar) {
-        DILocalScope *NewScope = DILocalScope::cloneScopeForSubprogram(
-            *OldVar->getScope(), *NewSP, Ctx, Cache);
-        NewVar = DIB.createAutoVariable(
-            NewScope, OldVar->getName(), OldVar->getFile(), OldVar->getLine(),
-            OldVar->getType(), /*AlwaysPreserve=*/false, DINode::FlagZero,
-            OldVar->getDWARFMemorySpace(), OldVar->getAlignInBits());
-      }
-      DVI->setVariable(GetUpdatedDIVariable(DVI->getVariable()));
-    }
-  }
-
-  for (auto *DII : DebugIntrinsicsToDelete)
-    DII->eraseFromParent();
   for (auto *DVR : DVRsToDelete)
     DVR->getMarker()->MarkedInstr->dropOneDbgRecord(DVR);
   DIB.finalizeSubprogram(NewSP);
@@ -1576,12 +1532,10 @@ CodeExtractor::extractCodeRegion(const CodeExtractorAnalysisCache &CEAC,
   fixupDebugInfoPostExtraction(*oldFunction, *newFunction, *TheCall, inputs,
                                NewValues);
 
-  LLVM_DEBUG(if (verifyFunction(*newFunction, &errs())) {
-    newFunction->dump();
-    report_fatal_error("verification of newFunction failed!");
-  });
-  LLVM_DEBUG(if (verifyFunction(*oldFunction))
-                 report_fatal_error("verification of oldFunction failed!"));
+  LLVM_DEBUG(llvm::dbgs() << "After extractCodeRegion - newFunction:\n");
+  LLVM_DEBUG(newFunction->dump());
+  LLVM_DEBUG(llvm::dbgs() << "After extractCodeRegion - oldFunction:\n");
+  LLVM_DEBUG(oldFunction->dump());
   LLVM_DEBUG(if (AC && verifyAssumptionCache(*oldFunction, *newFunction, AC))
                  report_fatal_error("Stale Asumption cache for old Function!"));
   return newFunction;
@@ -1881,6 +1835,9 @@ CallInst *CodeExtractor::emitReplacerCall(
   // This takes place of the original loop
   BasicBlock *codeReplacer =
       BasicBlock::Create(Context, "codeRepl", oldFunction, ReplIP);
+  if (AllocationBlock)
+    assert(AllocationBlock->getParent() == oldFunction &&
+           "AllocationBlock is not in the same function");
   BasicBlock *AllocaBlock =
       AllocationBlock ? AllocationBlock : &oldFunction->getEntryBlock();
 
@@ -1911,7 +1868,7 @@ CallInst *CodeExtractor::emitReplacerCall(
     ReloadOutputs.push_back(alloca);
   }
 
-  Instruction *Struct = nullptr;
+  AllocaInst *Struct = nullptr;
   if (!StructValues.empty()) {
     Struct = new AllocaInst(StructArgTy, DL.getAllocaAddrSpace(), nullptr,
                             "structArg", AllocaBlock->getFirstInsertionPt());
@@ -1919,11 +1876,11 @@ CallInst *CodeExtractor::emitReplacerCall(
       auto *StructSpaceCast = new AddrSpaceCastInst(
           Struct, PointerType ::get(Context, 0), "structArg.ascast");
       StructSpaceCast->insertAfter(Struct->getIterator());
-      Struct = StructSpaceCast;
+      params.push_back(StructSpaceCast);
+    } else {
+      params.push_back(Struct);
     }
 
-    params.push_back(Struct);
-
     unsigned AggIdx = 0;
     for (Value *input : inputs) {
       if (!StructValues.contains(input))
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index 00c59dff9499..0abb50a7f1cf 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -16,8 +16,10 @@
 #include "llvm/Transforms/Utils/Debugify.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -28,6 +30,11 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/JSON.h"
 #include <optional>
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+// We need the Signals header to operate on stacktraces if we're using DebugLoc
+// origin-tracking.
+#include "llvm/Support/Signals.h"
+#endif
 
 #define DEBUG_TYPE "debugify"
 
@@ -63,6 +70,54 @@ cl::opt<bool> DebugifyDIOpDIExprs(
 
 raw_ostream &dbg() { return Quiet ? nulls() : errs(); }
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+// These maps refer to addresses in the current LLVM process, so we can reuse
+// them everywhere - therefore, we store them at file scope.
+static SymbolizedAddressMap SymbolizedAddrs;
+static AddressSet UnsymbolizedAddrs;
+
+std::string symbolizeStackTrace(const Instruction *I) {
+  // We flush the set of unsymbolized addresses at the latest possible moment,
+  // i.e. now.
+  if (!UnsymbolizedAddrs.empty()) {
+    sys::symbolizeAddresses(UnsymbolizedAddrs, SymbolizedAddrs);
+    UnsymbolizedAddrs.clear();
+  }
+  const DbgLocOrigin::StackTracesTy &OriginStackTraces =
+      I->getDebugLoc().getOriginStackTraces();
+  std::string Result;
+  raw_string_ostream OS(Result);
+  for (size_t TraceIdx = 0; TraceIdx < OriginStackTraces.size(); ++TraceIdx) {
+    if (TraceIdx != 0)
+      OS << "========================================\n";
+    auto &[Depth, StackTrace] = OriginStackTraces[TraceIdx];
+    unsigned VirtualFrameNo = 0;
+    for (int Frame = 0; Frame < Depth; ++Frame) {
+      assert(SymbolizedAddrs.contains(StackTrace[Frame]) &&
+             "Expected each address to have been symbolized.");
+      for (std::string &SymbolizedFrame : SymbolizedAddrs[StackTrace[Frame]]) {
+        OS << right_justify(formatv("#{0}", VirtualFrameNo++).str(),
+                            std::log10(Depth) + 2)
+           << ' ' << SymbolizedFrame << '\n';
+      }
+    }
+  }
+  return Result;
+}
+void collectStackAddresses(Instruction &I) {
+  auto &OriginStackTraces = I.getDebugLoc().getOriginStackTraces();
+  for (auto &[Depth, StackTrace] : OriginStackTraces) {
+    for (int Frame = 0; Frame < Depth; ++Frame) {
+      void *Addr = StackTrace[Frame];
+      if (!SymbolizedAddrs.contains(Addr))
+        UnsymbolizedAddrs.insert(Addr);
+    }
+  }
+}
+#else
+void collectStackAddresses(Instruction &I) {}
+#endif // LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+
 uint64_t getAllocSizeInBits(Module &M, Type *Ty) {
   return Ty->isSized() ? M.getDataLayout().getTypeAllocSizeInBits(Ty) : 0;
 }
@@ -127,7 +182,9 @@ bool llvm::applyDebugifyMetadata(
     if (F.hasPrivateLinkage() || F.hasInternalLinkage())
       SPFlags |= DISubprogram::SPFlagLocalToUnit;
     auto SP = DIB.createFunction(CU, F.getName(), F.getName(), File, NextLine,
-                                 SPType, NextLine, DINode::FlagZero, SPFlags);
+                                 SPType, NextLine, DINode::FlagZero, SPFlags,
+                                 nullptr, nullptr, nullptr, nullptr, "",
+                                 /*UseKeyInstructions*/ ApplyAtomGroups);
     F.setSubprogram(SP);
 
     // Helper that inserts a dbg.value before \p InsertBefore, copying the
@@ -395,6 +452,8 @@ bool llvm::collectDebugInfoMetadata(Module &M,
         LLVM_DEBUG(dbgs() << "  Collecting info for inst: " << I << '\n');
         DebugInfoBeforePass.InstToDelete.insert({&I, &I});
 
+        // Track the addresses to symbolize, if the feature is enabled.
+        collectStackAddresses(I);
         DebugInfoBeforePass.DILocations.insert({&I, hasLoc(I)});
       }
     }
@@ -470,14 +529,23 @@ static bool checkInstructions(const DebugInstMap &DILocsBefore,
     auto BBName = BB->hasName() ? BB->getName() : "no-name";
     auto InstName = Instruction::getOpcodeName(Instr->getOpcode());
 
+    auto CreateJSONBugEntry = [&](const char *Action) {
+      Bugs.push_back(llvm::json::Object({
+          {"metadata", "DILocation"},
+          {"fn-name", FnName.str()},
+          {"bb-name", BBName.str()},
+          {"instr", InstName},
+          {"action", Action},
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN
+          {"origin", symbolizeStackTrace(Instr)},
+#endif
+      }));
+    };
+
     auto InstrIt = DILocsBefore.find(Instr);
     if (InstrIt == DILocsBefore.end()) {
       if (ShouldWriteIntoJSON)
-        Bugs.push_back(llvm::json::Object({{"metadata", "DILocation"},
-                                           {"fn-name", FnName.str()},
-                                           {"bb-name", BBName.str()},
-                                           {"instr", InstName},
-                                           {"action", "not-generate"}}));
+        CreateJSONBugEntry("not-generate");
       else
         dbg() << "WARNING: " << NameOfWrappedPass
               << " did not generate DILocation for " << *Instr
@@ -490,11 +558,7 @@ static bool checkInstructions(const DebugInstMap &DILocsBefore,
       // If the instr had the !dbg attached before the pass, consider it as
       // a debug info issue.
       if (ShouldWriteIntoJSON)
-        Bugs.push_back(llvm::json::Object({{"metadata", "DILocation"},
-                                           {"fn-name", FnName.str()},
-                                           {"bb-name", BBName.str()},
-                                           {"instr", InstName},
-                                           {"action", "drop"}}));
+        CreateJSONBugEntry("drop");
       else
         dbg() << "WARNING: " << NameOfWrappedPass << " dropped DILocation of "
               << *Instr << " (BB: " << BBName << ", Fn: " << FnName
@@ -632,6 +696,8 @@ bool llvm::checkDebugInfoMetadata(Module &M,
 
         LLVM_DEBUG(dbgs() << "  Collecting info for inst: " << I << '\n');
 
+        // Track the addresses to symbolize, if the feature is enabled.
+        collectStackAddresses(I);
         DebugInfoAfterPass.DILocations.insert({&I, hasLoc(I)});
       }
     }
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 7df5e9958182..6929d14bc56e 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -2233,7 +2233,7 @@ inlineRetainOrClaimRVCalls(CallBase &CB, objcarc::ARCInstKind RVCallKind,
 // profile. Note: we only update the "name" and "index" operands in the
 // instrumentation intrinsics, we leave the hash and total nr of indices as-is,
 // it's not worth updating those.
-static const std::pair<std::vector<int64_t>, std::vector<int64_t>>
+static std::pair<std::vector<int64_t>, std::vector<int64_t>>
 remapIndices(Function &Caller, BasicBlock *StartBB,
              PGOContextualProfile &CtxProf, uint32_t CalleeCounters,
              uint32_t CalleeCallsites) {
@@ -2336,16 +2336,14 @@ remapIndices(Function &Caller, BasicBlock *StartBB,
           Worklist.push_back(Succ);
   }
 
-  assert(
-      llvm::all_of(CalleeCounterMap, [&](const auto &V) { return V != 0; }) &&
-      "Counter index mapping should be either to -1 or to non-zero index, "
-      "because the 0 "
-      "index corresponds to the entry BB of the caller");
-  assert(
-      llvm::all_of(CalleeCallsiteMap, [&](const auto &V) { return V != 0; }) &&
-      "Callsite index mapping should be either to -1 or to non-zero index, "
-      "because there should have been at least a callsite - the inlined one "
-      "- which would have had a 0 index.");
+  assert(!llvm::is_contained(CalleeCounterMap, 0) &&
+         "Counter index mapping should be either to -1 or to non-zero index, "
+         "because the 0 "
+         "index corresponds to the entry BB of the caller");
+  assert(!llvm::is_contained(CalleeCallsiteMap, 0) &&
+         "Callsite index mapping should be either to -1 or to non-zero index, "
+         "because there should have been at least a callsite - the inlined one "
+         "- which would have had a 0 index.");
 
   return {std::move(CalleeCounterMap), std::move(CalleeCallsiteMap)};
 }
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index b1f66dca3b8f..cbcd20ef431d 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -2436,6 +2436,11 @@ getNewSalvageOpsForBinOp(BinaryOperator *BI, uint64_t CurrentLocOps,
   return BI->getOperand(0);
 }
 
+static bool getNewDIConversionOps(const DataLayout &DL, Type *SourceTy,
+                                  Type *DestTy,
+                                  std::optional<DIBasicType::Signedness> Sign,
+                                  SmallVectorImpl<DIOp::Variant> &Ops);
+
 /// This is a port of getSalvageOpsForGEP() to DIOp-based DIExpressions.
 static Value *
 getNewSalvageOpsForGEP(GetElementPtrInst *GEP, const DataLayout &DL,
@@ -2459,12 +2464,17 @@ getNewSalvageOpsForGEP(GetElementPtrInst *GEP, const DataLayout &DL,
     AdditionalValues.push_back(Offset.first);
     assert(Offset.second.isStrictlyPositive() &&
            "Expected strictly positive multiplier for offset.");
+    Ops.push_back(DIOp::Arg(CurrentLocOps++, Offset.first->getType()));
+    // Add a conversion operation if the gep offset operand has a different
+    // integer width than the pointer size.
+    if (!getNewDIConversionOps(DL, Offset.first->getType(), IntPtrTy,
+                               DIBasicType::Signedness::Signed, Ops))
+      return nullptr;
     ConstantInt *ConstOffset =
         ConstantInt::get(IntPtrTy, Offset.second.getZExtValue());
-    DIOp::Variant NewOps[] = {
-        DIOp::Arg(CurrentLocOps++, ConstOffset->getType()),
-        DIOp::Constant(ConstOffset), DIOp::Mul(), DIOp::Add()};
-    Ops.append(std::begin(NewOps), std::end(NewOps));
+    Ops.push_back(DIOp::Constant(ConstOffset));
+    Ops.push_back(DIOp::Mul());
+    Ops.push_back(DIOp::Add());
   }
 
   Ops.emplace_back(DIOp::Constant(
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index c50bb4a497c6..200d1fb85415 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1171,7 +1171,7 @@ Value *llvm::getShuffleReduction(IRBuilderBase &Builder, Value *Src,
     SmallVector<int, 32> ShuffleMask(VF);
     for (unsigned stride = 1; stride < VF; stride <<= 1) {
       // Initialise the mask with undef.
-      std::fill(ShuffleMask.begin(), ShuffleMask.end(), -1);
+      llvm::fill(ShuffleMask, -1);
       for (unsigned j = 0; j < VF; j += stride << 1) {
         ShuffleMask[j] = j + stride;
       }
@@ -1227,8 +1227,10 @@ Value *llvm::createFindLastIVReduction(IRBuilderBase &Builder, Value *Src,
                                        RecurKind RdxKind, Value *Start,
                                        Value *Sentinel) {
   bool IsSigned = RecurrenceDescriptor::isSignedRecurrenceKind(RdxKind);
+  bool IsMaxRdx = RecurrenceDescriptor::isFindLastIVRecurrenceKind(RdxKind);
   Value *MaxRdx = Src->getType()->isVectorTy()
-                      ? Builder.CreateIntMaxReduce(Src, IsSigned)
+                      ? (IsMaxRdx ? Builder.CreateIntMaxReduce(Src, IsSigned)
+                                  : Builder.CreateIntMinReduce(Src, IsSigned))
                       : Src;
   // Correct the final reduction result back to the start value if the maximum
   // reduction is sentinel value.
@@ -1324,8 +1326,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
 Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
                                    RecurKind Kind, Value *Mask, Value *EVL) {
   assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
-         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
-         "AnyOf or FindLastIV reductions are not supported.");
+         !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+         "AnyOf and FindIV reductions are not supported.");
   Intrinsic::ID Id = getReductionIntrinsicID(Kind);
   auto VPID = VPIntrinsic::getForIntrinsic(Id);
   assert(VPReductionIntrinsic::isVPReduction(VPID) &&
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index d26707a1361b..ac413c9b5815 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -214,6 +214,8 @@ class PredicateInfoBuilder {
   // whether it returned a valid result.
   DenseMap<Value *, unsigned int> ValueInfoNums;
 
+  BumpPtrAllocator &Allocator;
+
   ValueInfo &getOrCreateValueInfo(Value *);
   const ValueInfo &getValueInfo(Value *) const;
 
@@ -242,8 +244,8 @@ class PredicateInfoBuilder {
 
 public:
   PredicateInfoBuilder(PredicateInfo &PI, Function &F, DominatorTree &DT,
-                       AssumptionCache &AC)
-      : PI(PI), F(F), DT(DT), AC(AC) {
+                       AssumptionCache &AC, BumpPtrAllocator &Allocator)
+      : PI(PI), F(F), DT(DT), AC(AC), Allocator(Allocator) {
     // Push an empty operand info so that we can detect 0 as not finding one
     ValueInfos.resize(1);
   }
@@ -341,7 +343,6 @@ void PredicateInfoBuilder::addInfoFor(SmallVectorImpl<Value *> &OpsToRename,
   auto &OperandInfo = getOrCreateValueInfo(Op);
   if (OperandInfo.Infos.empty())
     OpsToRename.push_back(Op);
-  PI.AllInfos.push_back(PB);
   OperandInfo.Infos.push_back(PB);
 }
 
@@ -373,7 +374,7 @@ void PredicateInfoBuilder::processAssume(
 
     for (Value *V : Values) {
       if (shouldRename(V)) {
-        auto *PA = new PredicateAssume(V, II, Cond);
+        auto *PA = new (Allocator) PredicateAssume(V, II, Cond);
         addInfoFor(OpsToRename, V, PA);
       }
     }
@@ -419,8 +420,8 @@ void PredicateInfoBuilder::processBranch(
 
       for (Value *V : Values) {
         if (shouldRename(V)) {
-          PredicateBase *PB =
-              new PredicateBranch(V, BranchBB, Succ, Cond, TakenEdge);
+          PredicateBase *PB = new (Allocator)
+              PredicateBranch(V, BranchBB, Succ, Cond, TakenEdge);
           addInfoFor(OpsToRename, V, PB);
         }
       }
@@ -445,7 +446,7 @@ void PredicateInfoBuilder::processSwitch(
   for (auto C : SI->cases()) {
     BasicBlock *TargetBlock = C.getCaseSuccessor();
     if (SwitchEdges.lookup(TargetBlock) == 1) {
-      PredicateSwitch *PS = new PredicateSwitch(
+      PredicateSwitch *PS = new (Allocator) PredicateSwitch(
           Op, SI->getParent(), TargetBlock, C.getCaseValue(), SI);
       addInfoFor(OpsToRename, Op, PS);
     }
@@ -704,9 +705,9 @@ PredicateInfoBuilder::getValueInfo(Value *Operand) const {
 }
 
 PredicateInfo::PredicateInfo(Function &F, DominatorTree &DT,
-                             AssumptionCache &AC)
+                             AssumptionCache &AC, BumpPtrAllocator &Allocator)
     : F(F) {
-  PredicateInfoBuilder Builder(*this, F, DT, AC);
+  PredicateInfoBuilder Builder(*this, F, DT, AC, Allocator);
   Builder.buildPredicateInfo();
 }
 
@@ -797,7 +798,8 @@ PreservedAnalyses PredicateInfoPrinterPass::run(Function &F,
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
   auto &AC = AM.getResult<AssumptionAnalysis>(F);
   OS << "PredicateInfo for function: " << F.getName() << "\n";
-  auto PredInfo = std::make_unique<PredicateInfo>(F, DT, AC);
+  BumpPtrAllocator Allocator;
+  auto PredInfo = std::make_unique<PredicateInfo>(F, DT, AC, Allocator);
   PredInfo->print(OS);
 
   replaceCreatedSSACopys(*PredInfo, F);
@@ -859,7 +861,8 @@ PreservedAnalyses PredicateInfoVerifierPass::run(Function &F,
                                                  FunctionAnalysisManager &AM) {
   auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
   auto &AC = AM.getResult<AssumptionAnalysis>(F);
-  std::make_unique<PredicateInfo>(F, DT, AC)->verifyPredicateInfo();
+  BumpPtrAllocator Allocator;
+  std::make_unique<PredicateInfo>(F, DT, AC, Allocator)->verifyPredicateInfo();
 
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
index af0f516f7b00..d53a3144bf57 100644
--- a/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
+++ b/llvm/lib/Transforms/Utils/RelLookupTableConverter.cpp
@@ -66,6 +66,20 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
   if (!ElemType->isPointerTy() || DL.getPointerTypeSizeInBits(ElemType) != 64)
     return false;
 
+  SmallVector<GlobalVariable *, 4> GVOps;
+  Triple TT = M.getTargetTriple();
+  // FIXME: This should be removed in the future.
+  bool ShouldDropUnnamedAddr =
+      // Drop unnamed_addr to avoid matching pattern in
+      // `handleIndirectSymViaGOTPCRel`, which generates GOTPCREL relocations
+      // not supported by the GNU linker and LLD versions below 18 on aarch64.
+      TT.isAArch64()
+      // Apple's ld64 (and ld-prime on Xcode 15.2) miscompile something on
+      // x86_64-apple-darwin. See
+      // https://github.com/rust-lang/rust/issues/140686 and
+      // https://github.com/rust-lang/rust/issues/141306.
+      || (TT.isX86() && TT.isOSDarwin());
+
   for (const Use &Op : Array->operands()) {
     Constant *ConstOp = cast<Constant>(&Op);
     GlobalValue *GVOp;
@@ -85,8 +99,15 @@ static bool shouldConvertToRelLookupTable(Module &M, GlobalVariable &GV) {
         !GlovalVarOp->isDSOLocal() ||
         !GlovalVarOp->isImplicitDSOLocal())
       return false;
+
+    if (ShouldDropUnnamedAddr)
+      GVOps.push_back(GlovalVarOp);
   }
 
+  if (ShouldDropUnnamedAddr)
+    for (auto *GVOp : GVOps)
+      GVOp->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
+
   return true;
 }
 
@@ -108,24 +129,8 @@ static GlobalVariable *createRelLookupTable(Function &Func,
   uint64_t Idx = 0;
   SmallVector<Constant *, 64> RelLookupTableContents(NumElts);
 
-  Triple TT = M.getTargetTriple();
-  // FIXME: This should be removed in the future.
-  bool ShouldDropUnnamedAddr =
-      // Drop unnamed_addr to avoid matching pattern in
-      // `handleIndirectSymViaGOTPCRel`, which generates GOTPCREL relocations
-      // not supported by the GNU linker and LLD versions below 18 on aarch64.
-      TT.isAArch64()
-      // Apple's ld64 (and ld-prime on Xcode 15.2) miscompile something on
-      // x86_64-apple-darwin. See
-      // https://github.com/rust-lang/rust/issues/140686 and
-      // https://github.com/rust-lang/rust/issues/141306.
-      || (TT.isX86() && TT.isOSDarwin());
-
   for (Use &Operand : LookupTableArr->operands()) {
     Constant *Element = cast<Constant>(Operand);
-    if (ShouldDropUnnamedAddr)
-      if (auto *GlobalElement = dyn_cast<GlobalValue>(Element))
-        GlobalElement->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
     Type *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
     Constant *Base = llvm::ConstantExpr::getPtrToInt(RelLookupTable, IntPtrTy);
     Constant *Target = llvm::ConstantExpr::getPtrToInt(Element, IntPtrTy);
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index e1b1cf68cd9f..586874f57ae7 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -426,7 +426,10 @@ void SCCPSolver::inferArgAttributes() const {
 class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   const DataLayout &DL;
   std::function<const TargetLibraryInfo &(Function &)> GetTLI;
-  SmallPtrSet<BasicBlock *, 8> BBExecutable; // The BBs that are executable.
+  /// Basic blocks that are executable (but may not have been visited yet).
+  SmallPtrSet<BasicBlock *, 8> BBExecutable;
+  /// Basic blocks that are executable and have been visited at least once.
+  SmallPtrSet<BasicBlock *, 8> BBVisited;
   DenseMap<Value *, ValueLatticeElement>
       ValueState; // The state each value is in.
 
@@ -466,15 +469,14 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   /// constants.
   SmallPtrSet<Function *, 16> TrackingIncomingArguments;
 
-  /// The reason for two worklists is that overdefined is the lowest state
-  /// on the lattice, and moving things to overdefined as fast as possible
-  /// makes SCCP converge much faster.
-  ///
-  /// By having a separate worklist, we accomplish this because everything
-  /// possibly overdefined will become overdefined at the soonest possible
-  /// point.
-  SmallVector<Value *, 64> OverdefinedInstWorkList;
-  SmallVector<Value *, 64> InstWorkList;
+  /// Worklist of instructions to re-visit. This only includes instructions
+  /// in blocks that have already been visited at least once.
+  SmallSetVector<Instruction *, 16> InstWorkList;
+
+  /// Current instruction while visiting a block for the first time, used to
+  /// avoid unnecessary instruction worklist insertions. Null if an instruction
+  /// is visited outside a whole-block visitation.
+  Instruction *CurI = nullptr;
 
   // The BasicBlock work list
   SmallVector<BasicBlock *, 64> BBWorkList;
@@ -490,17 +492,22 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
 
   LLVMContext &Ctx;
 
+  BumpPtrAllocator PredicateInfoAllocator;
+
 private:
   ConstantInt *getConstantInt(const ValueLatticeElement &IV, Type *Ty) const {
     return dyn_cast_or_null<ConstantInt>(getConstant(IV, Ty));
   }
 
-  // pushToWorkList - Helper for markConstant/markOverdefined
-  void pushToWorkList(ValueLatticeElement &IV, Value *V);
+  /// Push instruction \p I to the worklist.
+  void pushToWorkList(Instruction *I);
+
+  /// Push users of value \p V to the worklist.
+  void pushUsersToWorkList(Value *V);
 
-  // Helper to push \p V to the worklist, after updating it to \p IV. Also
-  // prints a debug message with the updated value.
-  void pushToWorkListMsg(ValueLatticeElement &IV, Value *V);
+  /// Like pushUsersToWorkList(), but also prints a debug message with the
+  /// updated value.
+  void pushUsersToWorkListMsg(ValueLatticeElement &IV, Value *V);
 
   // markConstant - Make a value be marked as "constant".  If the value
   // is not already a constant, add it to the instruction work list so that
@@ -659,46 +666,9 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   // successors are reachable from a given terminator instruction.
   void getFeasibleSuccessors(Instruction &TI, SmallVectorImpl<bool> &Succs);
 
-  // OperandChangedState - This method is invoked on all of the users of an
-  // instruction that was just changed state somehow.  Based on this
-  // information, we need to update the specified user of this instruction.
-  void operandChangedState(Instruction *I) {
-    if (BBExecutable.count(I->getParent())) // Inst is executable?
-      visit(*I);
-  }
-
   // Add U as additional user of V.
   void addAdditionalUser(Value *V, User *U) { AdditionalUsers[V].insert(U); }
 
-  // Mark I's users as changed, including AdditionalUsers.
-  void markUsersAsChanged(Value *I) {
-    // Functions include their arguments in the use-list. Changed function
-    // values mean that the result of the function changed. We only need to
-    // update the call sites with the new function result and do not have to
-    // propagate the call arguments.
-    if (isa<Function>(I)) {
-      for (User *U : I->users()) {
-        if (auto *CB = dyn_cast<CallBase>(U))
-          handleCallResult(*CB);
-      }
-    } else {
-      for (User *U : I->users())
-        if (auto *UI = dyn_cast<Instruction>(U))
-          operandChangedState(UI);
-    }
-
-    auto Iter = AdditionalUsers.find(I);
-    if (Iter != AdditionalUsers.end()) {
-      // Copy additional users before notifying them of changes, because new
-      // users may be added, potentially invalidating the iterator.
-      SmallVector<Instruction *, 2> ToNotify;
-      for (User *U : Iter->second)
-        if (auto *UI = dyn_cast<Instruction>(U))
-          ToNotify.push_back(UI);
-      for (Instruction *UI : ToNotify)
-        operandChangedState(UI);
-    }
-  }
   void handleCallOverdefined(CallBase &CB);
   void handleCallResult(CallBase &CB);
   void handleCallArguments(CallBase &CB);
@@ -761,7 +731,8 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
 
 public:
   void addPredicateInfo(Function &F, DominatorTree &DT, AssumptionCache &AC) {
-    FnPredicateInfo.insert({&F, std::make_unique<PredicateInfo>(F, DT, AC)});
+    FnPredicateInfo.insert({&F, std::make_unique<PredicateInfo>(
+                                    F, DT, AC, PredicateInfoAllocator)});
   }
 
   void removeSSACopies(Function &F) {
@@ -982,19 +953,41 @@ bool SCCPInstVisitor::markBlockExecutable(BasicBlock *BB) {
   return true;
 }
 
-void SCCPInstVisitor::pushToWorkList(ValueLatticeElement &IV, Value *V) {
-  if (IV.isOverdefined()) {
-    if (OverdefinedInstWorkList.empty() || OverdefinedInstWorkList.back() != V)
-      OverdefinedInstWorkList.push_back(V);
+void SCCPInstVisitor::pushToWorkList(Instruction *I) {
+  // If we're currently visiting a block, do not push any instructions in the
+  // same blocks that are after the current one, as they will be visited
+  // anyway. We do have to push updates to earlier instructions (e.g. phi
+  // nodes or loads of tracked globals).
+  if (CurI && I->getParent() == CurI->getParent() && !I->comesBefore(CurI))
     return;
+  // Only push instructions in already visited blocks. Otherwise we'll handle
+  // it when we visit the block for the first time.
+  if (BBVisited.contains(I->getParent()))
+    InstWorkList.insert(I);
+}
+
+void SCCPInstVisitor::pushUsersToWorkList(Value *V) {
+  for (User *U : V->users())
+    if (auto *UI = dyn_cast<Instruction>(U))
+      pushToWorkList(UI);
+
+  auto Iter = AdditionalUsers.find(V);
+  if (Iter != AdditionalUsers.end()) {
+    // Copy additional users before notifying them of changes, because new
+    // users may be added, potentially invalidating the iterator.
+    SmallVector<Instruction *, 2> ToNotify;
+    for (User *U : Iter->second)
+      if (auto *UI = dyn_cast<Instruction>(U))
+        ToNotify.push_back(UI);
+    for (Instruction *UI : ToNotify)
+      pushToWorkList(UI);
   }
-  if (InstWorkList.empty() || InstWorkList.back() != V)
-    InstWorkList.push_back(V);
 }
 
-void SCCPInstVisitor::pushToWorkListMsg(ValueLatticeElement &IV, Value *V) {
+void SCCPInstVisitor::pushUsersToWorkListMsg(ValueLatticeElement &IV,
+                                             Value *V) {
   LLVM_DEBUG(dbgs() << "updated " << IV << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
 }
 
 bool SCCPInstVisitor::markConstant(ValueLatticeElement &IV, Value *V,
@@ -1002,7 +995,7 @@ bool SCCPInstVisitor::markConstant(ValueLatticeElement &IV, Value *V,
   if (!IV.markConstant(C, MayIncludeUndef))
     return false;
   LLVM_DEBUG(dbgs() << "markConstant: " << *C << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1011,7 +1004,7 @@ bool SCCPInstVisitor::markNotConstant(ValueLatticeElement &IV, Value *V,
   if (!IV.markNotConstant(C))
     return false;
   LLVM_DEBUG(dbgs() << "markNotConstant: " << *C << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1020,7 +1013,7 @@ bool SCCPInstVisitor::markConstantRange(ValueLatticeElement &IV, Value *V,
   if (!IV.markConstantRange(CR))
     return false;
   LLVM_DEBUG(dbgs() << "markConstantRange: " << CR << ": " << *V << '\n');
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1033,7 +1026,7 @@ bool SCCPInstVisitor::markOverdefined(ValueLatticeElement &IV, Value *V) {
              << "Function '" << F->getName() << "'\n";
              else dbgs() << *V << '\n');
   // Only instructions go on the work list
-  pushToWorkList(IV, V);
+  pushUsersToWorkList(V);
   return true;
 }
 
@@ -1141,7 +1134,7 @@ bool SCCPInstVisitor::mergeInValue(ValueLatticeElement &IV, Value *V,
                                    ValueLatticeElement MergeWithV,
                                    ValueLatticeElement::MergeOptions Opts) {
   if (IV.mergeIn(MergeWithV, Opts)) {
-    pushToWorkList(IV, V);
+    pushUsersToWorkList(V);
     LLVM_DEBUG(dbgs() << "Merged " << MergeWithV << " into " << *V << " : "
                       << IV << "\n");
     return true;
@@ -1161,7 +1154,7 @@ bool SCCPInstVisitor::markEdgeExecutable(BasicBlock *Source, BasicBlock *Dest) {
                       << " -> " << Dest->getName() << '\n');
 
     for (PHINode &PN : Dest->phis())
-      visitPHINode(PN);
+      pushToWorkList(&PN);
   }
   return true;
 }
@@ -1537,7 +1530,7 @@ void SCCPInstVisitor::visitSelectInst(SelectInst &I) {
   bool Changed = State.mergeIn(TVal);
   Changed |= State.mergeIn(FVal);
   if (Changed)
-    pushToWorkListMsg(State, &I);
+    pushUsersToWorkListMsg(State, &I);
 }
 
 // Handle Unary Operators.
@@ -2036,53 +2029,28 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
 
 void SCCPInstVisitor::solve() {
   // Process the work lists until they are empty!
-  while (!BBWorkList.empty() || !InstWorkList.empty() ||
-         !OverdefinedInstWorkList.empty()) {
-    // Process the overdefined instruction's work list first, which drives other
-    // things to overdefined more quickly.
-    while (!OverdefinedInstWorkList.empty()) {
-      Value *I = OverdefinedInstWorkList.pop_back_val();
-      Invalidated.erase(I);
-
-      LLVM_DEBUG(dbgs() << "\nPopped off OI-WL: " << *I << '\n');
-
-      // "I" got into the work list because it either made the transition from
-      // bottom to constant, or to overdefined.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined
-      // Update all of the users of this instruction's value.
-      //
-      markUsersAsChanged(I);
-    }
-
+  while (!BBWorkList.empty() || !InstWorkList.empty()) {
     // Process the instruction work list.
     while (!InstWorkList.empty()) {
-      Value *I = InstWorkList.pop_back_val();
+      Instruction *I = InstWorkList.pop_back_val();
       Invalidated.erase(I);
 
       LLVM_DEBUG(dbgs() << "\nPopped off I-WL: " << *I << '\n');
 
-      // "I" got into the work list because it made the transition from undef to
-      // constant.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined.
-      // Update all of the users of this instruction's value.
-      //
-      if (I->getType()->isStructTy() || !getValueState(I).isOverdefined())
-        markUsersAsChanged(I);
+      visit(I);
     }
 
     // Process the basic block work list.
     while (!BBWorkList.empty()) {
       BasicBlock *BB = BBWorkList.pop_back_val();
+      BBVisited.insert(BB);
 
       LLVM_DEBUG(dbgs() << "\nPopped off BBWL: " << *BB << '\n');
-
-      // Notify all instructions in this basic block that they are newly
-      // executable.
-      visit(BB);
+      for (Instruction &I : *BB) {
+        CurI = &I;
+        visit(I);
+      }
+      CurI = nullptr;
     }
   }
 }
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index e205551658aa..a75f29000ca1 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6198,7 +6198,7 @@ static bool initializeUniqueCases(SwitchInst *SI, PHINode *&PHI,
 // TODO: Handle switches with more than 2 cases that map to the same result.
 static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector,
                                  Constant *DefaultResult, Value *Condition,
-                                 IRBuilder<> &Builder) {
+                                 IRBuilder<> &Builder, const DataLayout &DL) {
   // If we are selecting between only two cases transform into a simple
   // select or a two-way select if default is possible.
   // Example:
@@ -6234,10 +6234,33 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector,
     // case 0,2,8,10 -> Cond & 0b1..0101 == 0 ? result : default
     if (isPowerOf2_32(CaseCount)) {
       ConstantInt *MinCaseVal = CaseValues[0];
-      // Find mininal value.
-      for (auto *Case : CaseValues)
+      // If there are bits that are set exclusively by CaseValues, we
+      // can transform the switch into a select if the conjunction of
+      // all the values uniquely identify CaseValues.
+      APInt AndMask = APInt::getAllOnes(MinCaseVal->getBitWidth());
+
+      // Find the minimum value and compute the and of all the case values.
+      for (auto *Case : CaseValues) {
         if (Case->getValue().slt(MinCaseVal->getValue()))
           MinCaseVal = Case;
+        AndMask &= Case->getValue();
+      }
+      KnownBits Known = computeKnownBits(Condition, DL);
+
+      if (!AndMask.isZero() && Known.getMaxValue().uge(AndMask)) {
+        // Compute the number of bits that are free to vary.
+        unsigned FreeBits = Known.countMaxActiveBits() - AndMask.popcount();
+
+        // Check if the number of values covered by the mask is equal
+        // to the number of cases.
+        if (FreeBits == Log2_32(CaseCount)) {
+          Value *And = Builder.CreateAnd(Condition, AndMask);
+          Value *Cmp = Builder.CreateICmpEQ(
+              And, Constant::getIntegerValue(And->getType(), AndMask));
+          return Builder.CreateSelect(Cmp, ResultVector[0].first,
+                                      DefaultResult);
+        }
+      }
 
       // Mark the bits case number touched.
       APInt BitMask = APInt::getZero(MinCaseVal->getBitWidth());
@@ -6325,7 +6348,7 @@ static bool trySwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder,
   assert(PHI != nullptr && "PHI for value select not found");
   Builder.SetInsertPoint(SI);
   Value *SelectValue =
-      foldSwitchToSelect(UniqueResults, DefaultResult, Cond, Builder);
+      foldSwitchToSelect(UniqueResults, DefaultResult, Cond, Builder, DL);
   if (!SelectValue)
     return false;
 
@@ -6347,7 +6370,7 @@ class SwitchLookupTable {
 
   /// Build instructions with Builder to retrieve the value at
   /// the position given by Index in the lookup table.
-  Value *buildLookup(Value *Index, IRBuilder<> &Builder);
+  Value *buildLookup(Value *Index, IRBuilder<> &Builder, const DataLayout &DL);
 
   /// Return true if a table with TableSize elements of
   /// type ElementType would fit in a target-legal register.
@@ -6533,7 +6556,8 @@ SwitchLookupTable::SwitchLookupTable(
   Kind = ArrayKind;
 }
 
-Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder) {
+Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder,
+                                      const DataLayout &DL) {
   switch (Kind) {
   case SingleValueKind:
     return SingleValue;
@@ -6575,16 +6599,12 @@ Value *SwitchLookupTable::buildLookup(Value *Index, IRBuilder<> &Builder) {
     return Builder.CreateTrunc(DownShifted, BitMapElementTy, "switch.masked");
   }
   case ArrayKind: {
-    // Make sure the table index will not overflow when treated as signed.
-    IntegerType *IT = cast<IntegerType>(Index->getType());
-    uint64_t TableSize =
-        Array->getInitializer()->getType()->getArrayNumElements();
-    if (TableSize > (1ULL << std::min(IT->getBitWidth() - 1, 63u)))
-      Index = Builder.CreateZExt(
-          Index, IntegerType::get(IT->getContext(), IT->getBitWidth() + 1),
-          "switch.tableidx.zext");
-
-    Value *GEPIndices[] = {Builder.getInt32(0), Index};
+    Type *IndexTy = DL.getIndexType(Array->getType());
+
+    if (Index->getType() != IndexTy)
+      Index = Builder.CreateZExtOrTrunc(Index, IndexTy);
+
+    Value *GEPIndices[] = {ConstantInt::get(IndexTy, 0), Index};
     Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
                                            GEPIndices, "switch.gep");
     return Builder.CreateLoad(
@@ -7064,7 +7084,7 @@ static bool switchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     SwitchLookupTable Table(Mod, TableSize, TableIndexOffset, ResultList, DV,
                             DL, FuncName);
 
-    Value *Result = Table.buildLookup(TableIndex, Builder);
+    Value *Result = Table.buildLookup(TableIndex, Builder, DL);
 
     // Do a small peephole optimization: re-use the switch table compare if
     // possible.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f4259d3d6988..e7bae17dd2ce 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -395,7 +395,7 @@ static cl::opt<bool> UseWiderVFIfCallVariantsPresent(
     cl::desc("Try wider VFs if they enable the use of vector variants"));
 
 static cl::opt<bool> EnableEarlyExitVectorization(
-    "enable-early-exit-vectorization", cl::init(false), cl::Hidden,
+    "enable-early-exit-vectorization", cl::init(true), cl::Hidden,
     cl::desc(
         "Enable vectorization of early exit loops with uncountable exits."));
 
@@ -1768,20 +1768,16 @@ class GeneratedRTChecks {
   BasicBlock *SCEVCheckBlock = nullptr;
 
   /// The value representing the result of the generated SCEV checks. If it is
-  /// nullptr, either no SCEV checks have been generated or they have been used.
+  /// nullptr no SCEV checks have been generated.
   Value *SCEVCheckCond = nullptr;
 
   /// Basic block which contains the generated memory runtime checks, if any.
   BasicBlock *MemCheckBlock = nullptr;
 
   /// The value representing the result of the generated memory runtime checks.
-  /// If it is nullptr, either no memory runtime checks have been generated or
-  /// they have been used.
+  /// If it is nullptr no memory runtime checks have been generated.
   Value *MemRuntimeCheckCond = nullptr;
 
-  /// True if any checks have been added.
-  bool AddedAnyChecks = false;
-
   DominatorTree *DT;
   LoopInfo *LI;
   TargetTransformInfo *TTI;
@@ -1996,13 +1992,14 @@ class GeneratedRTChecks {
   ~GeneratedRTChecks() {
     SCEVExpanderCleaner SCEVCleaner(SCEVExp);
     SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
-    if (!SCEVCheckCond)
+    bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(SCEVCheckBlock);
+    bool MemChecksUsed = !MemCheckBlock || !pred_empty(MemCheckBlock);
+    if (SCEVChecksUsed)
       SCEVCleaner.markResultUsed();
 
-    if (!MemRuntimeCheckCond)
+    if (MemChecksUsed) {
       MemCheckCleaner.markResultUsed();
-
-    if (MemRuntimeCheckCond) {
+    } else {
       auto &SE = *MemCheckExp.getSE();
       // Memory runtime check generation creates compares that use expanded
       // values. Remove them before running the SCEVExpanderCleaners.
@@ -2016,9 +2013,9 @@ class GeneratedRTChecks {
     MemCheckCleaner.cleanup();
     SCEVCleaner.cleanup();
 
-    if (SCEVCheckCond)
+    if (!SCEVChecksUsed)
       SCEVCheckBlock->eraseFromParent();
-    if (MemRuntimeCheckCond)
+    if (!MemChecksUsed)
       MemCheckBlock->eraseFromParent();
   }
 
@@ -2044,9 +2041,6 @@ class GeneratedRTChecks {
     if (AddBranchWeights)
       setBranchWeights(BI, SCEVCheckBypassWeights, /*IsExpected=*/false);
     ReplaceInstWithInst(SCEVCheckBlock->getTerminator(), &BI);
-    // Mark the check as used, to prevent it from being removed during cleanup.
-    SCEVCheckCond = nullptr;
-    AddedAnyChecks = true;
     return SCEVCheckBlock;
   }
 
@@ -2074,14 +2068,15 @@ class GeneratedRTChecks {
     MemCheckBlock->getTerminator()->setDebugLoc(
         Pred->getTerminator()->getDebugLoc());
 
-    // Mark the check as used, to prevent it from being removed during cleanup.
-    MemRuntimeCheckCond = nullptr;
-    AddedAnyChecks = true;
     return MemCheckBlock;
   }
 
   /// Return true if any runtime checks have been added
-  bool hasChecks() const { return AddedAnyChecks; }
+  bool hasChecks() const {
+    using namespace llvm::PatternMatch;
+    return (SCEVCheckCond && !match(SCEVCheckCond, m_ZeroInt())) ||
+           MemRuntimeCheckCond;
+  }
 };
 } // namespace
 
@@ -4681,7 +4676,7 @@ void LoopVectorizationCostModel::collectElementTypesForWidening() {
         if (!Legal->isReductionVariable(PN))
           continue;
         const RecurrenceDescriptor &RdxDesc =
-            Legal->getReductionVars().find(PN)->second;
+            Legal->getRecurrenceDescriptor(PN);
         if (PreferInLoopReductions || useOrderedReductions(RdxDesc) ||
             TTI.preferInLoopReduction(RdxDesc.getRecurrenceKind(),
                                       RdxDesc.getRecurrenceType()))
@@ -4926,7 +4921,7 @@ LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
           const RecurrenceDescriptor &RdxDesc = Reduction.second;
           RecurKind RK = RdxDesc.getRecurrenceKind();
           return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
-                 RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK);
+                 RecurrenceDescriptor::isFindIVRecurrenceKind(RK);
         });
     if (HasSelectCmpReductions) {
       LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
@@ -5481,7 +5476,7 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
     ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
 
   const RecurrenceDescriptor &RdxDesc =
-      Legal->getReductionVars().find(cast<PHINode>(ReductionPhi))->second;
+      Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
 
   InstructionCost BaseCost;
   RecurKind RK = RdxDesc.getRecurrenceKind();
@@ -6069,24 +6064,23 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
   auto *SE = PSE.getSE();
 
-  auto HasSingleCopyAfterVectorization = [this](Instruction *I,
-                                                ElementCount VF) -> bool {
-    if (VF.isScalar())
-      return true;
-
-    auto Scalarized = InstsToScalarize.find(VF);
-    assert(Scalarized != InstsToScalarize.end() &&
-           "VF not yet analyzed for scalarization profitability");
-    return !Scalarized->second.count(I) &&
-           llvm::all_of(I->users(), [&](User *U) {
-             auto *UI = cast<Instruction>(U);
-             return !Scalarized->second.count(UI);
-           });
-  };
-  (void)HasSingleCopyAfterVectorization;
-
   Type *VectorTy;
   if (isScalarAfterVectorization(I, VF)) {
+    [[maybe_unused]] auto HasSingleCopyAfterVectorization =
+        [this](Instruction *I, ElementCount VF) -> bool {
+      if (VF.isScalar())
+        return true;
+
+      auto Scalarized = InstsToScalarize.find(VF);
+      assert(Scalarized != InstsToScalarize.end() &&
+             "VF not yet analyzed for scalarization profitability");
+      return !Scalarized->second.count(I) &&
+             llvm::all_of(I->users(), [&](User *U) {
+               auto *UI = cast<Instruction>(U);
+               return !Scalarized->second.count(UI);
+             });
+    };
+
     // With the exception of GEPs and PHIs, after scalarization there should
     // only be one copy of the instruction generated in the loop. This is
     // because the VF is either 1, or any instructions that need scalarizing
@@ -6346,8 +6340,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     Type *ValTy = I->getOperand(0)->getType();
 
     if (canTruncateToMinimalBitwidth(I, VF)) {
-      Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
-      (void)Op0AsInstruction;
+      [[maybe_unused]] Instruction *Op0AsInstruction =
+          dyn_cast<Instruction>(I->getOperand(0));
       assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
               MinBWs[I] == MinBWs[Op0AsInstruction]) &&
              "if both the operand and the compare are marked for "
@@ -7245,8 +7239,8 @@ static void addRuntimeUnrollDisableMetaData(Loop *L) {
 
 static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
   using namespace VPlanPatternMatch;
-  assert(RdxResult->getOpcode() == VPInstruction::ComputeFindLastIVResult &&
-         "RdxResult must be ComputeFindLastIVResult");
+  assert(RdxResult->getOpcode() == VPInstruction::ComputeFindIVResult &&
+         "RdxResult must be ComputeFindIVResult");
   VPValue *StartVPV = RdxResult->getOperand(1);
   match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
   return StartVPV->getLiveInIRValue();
@@ -7260,17 +7254,19 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
   // Get the VPInstruction computing the reduction result in the middle block.
   // The first operand may not be from the middle block if it is not connected
   // to the scalar preheader. In that case, there's nothing to fix.
-  auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
+  VPValue *Incoming = EpiResumePhiR->getOperand(0);
+  match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
+                      VPlanPatternMatch::m_VPValue(Incoming)));
+  auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
   if (!EpiRedResult ||
       (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
        EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
-       EpiRedResult->getOpcode() != VPInstruction::ComputeFindLastIVResult))
+       EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
     return;
 
   auto *EpiRedHeaderPhi =
       cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
-  const RecurrenceDescriptor &RdxDesc =
-      EpiRedHeaderPhi->getRecurrenceDescriptor();
+  RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
   Value *MainResumeValue;
   if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
     assert((VPI->getOpcode() == VPInstruction::Broadcast ||
@@ -7279,10 +7275,9 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
     MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
   } else
     MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
-  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-          RdxDesc.getRecurrenceKind())) {
-    Value *StartV = EpiRedResult->getOperand(1)->getLiveInIRValue();
-    (void)StartV;
+  if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) {
+    [[maybe_unused]] Value *StartV =
+        EpiRedResult->getOperand(1)->getLiveInIRValue();
     auto *Cmp = cast<ICmpInst>(MainResumeValue);
     assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
            "AnyOf expected to start with ICMP_NE");
@@ -7290,13 +7285,12 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
            "AnyOf expected to start by comparing main resume value to original "
            "start value");
     MainResumeValue = Cmp->getOperand(0);
-  } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
-                 RdxDesc.getRecurrenceKind())) {
+  } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
     Value *StartV = getStartValueFromReductionResult(EpiRedResult);
     Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
     using namespace llvm::PatternMatch;
     Value *Cmp, *OrigResumeV, *CmpOp;
-    bool IsExpectedPattern =
+    [[maybe_unused]] bool IsExpectedPattern =
         match(MainResumeValue,
               m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
                        m_Value(OrigResumeV))) &&
@@ -7304,7 +7298,6 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
                                    m_Value(CmpOp))) &&
          ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
     assert(IsExpectedPattern && "Unexpected reduction resume pattern");
-    (void)IsExpectedPattern;
     MainResumeValue = OrigResumeV;
   }
   PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
@@ -7328,10 +7321,13 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // cost model is complete for better cost estimates.
   VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
                            OrigLoop->getHeader()->getContext());
+  VPlanTransforms::runPass(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
   VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
-  if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
+  if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
+    std::optional<unsigned> VScale = CM.getVScaleForTuning();
     VPlanTransforms::runPass(VPlanTransforms::addBranchWeightToMiddleTerminator,
-                             BestVPlan, BestVF);
+                             BestVPlan, BestVF, VScale);
+  }
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
   VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
   VPlanTransforms::narrowInterleaveGroups(
@@ -7769,8 +7765,9 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
           (CM.foldTailByMasking() || !GEP || !GEP->isInBounds())
               ? GEPNoWrapFlags::none()
               : GEPNoWrapFlags::inBounds();
-      VectorPtr = new VPVectorEndPointerRecipe(
-          Ptr, &Plan.getVF(), getLoadStoreType(I), Flags, I->getDebugLoc());
+      VectorPtr =
+          new VPVectorEndPointerRecipe(Ptr, &Plan.getVF(), getLoadStoreType(I),
+                                       /*Stride*/ -1, Flags, I->getDebugLoc());
     } else {
       VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
                                             GEP ? GEP->getNoWrapFlags()
@@ -8148,15 +8145,15 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   // something that isn't another partial reduction. This is because the
   // extends are intended to be lowered along with the reduction itself.
 
-  // Build up a set of partial reduction bin ops for efficient use checking.
-  SmallSet<User *, 4> PartialReductionBinOps;
+  // Build up a set of partial reduction ops for efficient use checking.
+  SmallSet<User *, 4> PartialReductionOps;
   for (const auto &[PartialRdx, _] : PartialReductionChains)
-    PartialReductionBinOps.insert(PartialRdx.BinOp);
+    PartialReductionOps.insert(PartialRdx.ExtendUser);
 
   auto ExtendIsOnlyUsedByPartialReductions =
-      [&PartialReductionBinOps](Instruction *Extend) {
+      [&PartialReductionOps](Instruction *Extend) {
         return all_of(Extend->users(), [&](const User *U) {
-          return PartialReductionBinOps.contains(U);
+          return PartialReductionOps.contains(U);
         });
       };
 
@@ -8165,7 +8162,7 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   for (auto Pair : PartialReductionChains) {
     PartialReductionChain Chain = Pair.first;
     if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
-        ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB))
+        (!Chain.ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
       ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
   }
 }
@@ -8173,7 +8170,6 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
 bool VPRecipeBuilder::getScaledReductions(
     Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
     SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
-
   if (!CM.TheLoop->contains(RdxExitInstr))
     return false;
 
@@ -8202,43 +8198,71 @@ bool VPRecipeBuilder::getScaledReductions(
   if (PhiOp != PHI)
     return false;
 
-  auto *BinOp = dyn_cast<BinaryOperator>(Op);
-  if (!BinOp || !BinOp->hasOneUse())
-    return false;
-
   using namespace llvm::PatternMatch;
-  // Use the side-effect of match to replace BinOp only if the pattern is
-  // matched, we don't care at this point whether it actually matched.
-  match(BinOp, m_Neg(m_BinOp(BinOp)));
 
-  Value *A, *B;
-  if (!match(BinOp->getOperand(0), m_ZExtOrSExt(m_Value(A))) ||
-      !match(BinOp->getOperand(1), m_ZExtOrSExt(m_Value(B))))
-    return false;
+  // If the update is a binary operator, check both of its operands to see if
+  // they are extends. Otherwise, see if the update comes directly from an
+  // extend.
+  Instruction *Exts[2] = {nullptr};
+  BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
+  std::optional<unsigned> BinOpc;
+  Type *ExtOpTypes[2] = {nullptr};
+
+  auto CollectExtInfo = [&Exts,
+                         &ExtOpTypes](SmallVectorImpl<Value *> &Ops) -> bool {
+    unsigned I = 0;
+    for (Value *OpI : Ops) {
+      Value *ExtOp;
+      if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))))
+        return false;
+      Exts[I] = cast<Instruction>(OpI);
+      ExtOpTypes[I] = ExtOp->getType();
+      I++;
+    }
+    return true;
+  };
+
+  if (ExtendUser) {
+    if (!ExtendUser->hasOneUse())
+      return false;
 
-  Instruction *ExtA = cast<Instruction>(BinOp->getOperand(0));
-  Instruction *ExtB = cast<Instruction>(BinOp->getOperand(1));
+    // Use the side-effect of match to replace BinOp only if the pattern is
+    // matched, we don't care at this point whether it actually matched.
+    match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
+
+    SmallVector<Value *> Ops(ExtendUser->operands());
+    if (!CollectExtInfo(Ops))
+      return false;
+
+    BinOpc = std::make_optional(ExtendUser->getOpcode());
+  } else if (match(Update, m_Add(m_Value(), m_Value()))) {
+    // We already know the operands for Update are Op and PhiOp.
+    SmallVector<Value *> Ops({Op});
+    if (!CollectExtInfo(Ops))
+      return false;
+
+    ExtendUser = Update;
+    BinOpc = std::nullopt;
+  } else
+    return false;
 
   TTI::PartialReductionExtendKind OpAExtend =
-      TargetTransformInfo::getPartialReductionExtendKind(ExtA);
+      TTI::getPartialReductionExtendKind(Exts[0]);
   TTI::PartialReductionExtendKind OpBExtend =
-      TargetTransformInfo::getPartialReductionExtendKind(ExtB);
-
-  PartialReductionChain Chain(RdxExitInstr, ExtA, ExtB, BinOp);
+      Exts[1] ? TTI::getPartialReductionExtendKind(Exts[1]) : TTI::PR_None;
+  PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
 
   TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
-  TypeSize ASize = A->getType()->getPrimitiveSizeInBits();
-
+  TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
   if (!PHISize.hasKnownScalarFactor(ASize))
     return false;
-
   unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
 
   if (LoopVectorizationPlanner::getDecisionAndClampRange(
           [&](ElementCount VF) {
             InstructionCost Cost = TTI->getPartialReductionCost(
-                Update->getOpcode(), A->getType(), B->getType(), PHI->getType(),
-                VF, OpAExtend, OpBExtend, BinOp->getOpcode(), CM.CostKind);
+                Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
+                PHI->getType(), VF, OpAExtend, OpBExtend, BinOpc, CM.CostKind);
             return Cost.isValid();
           },
           Range)) {
@@ -8258,10 +8282,10 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
   SmallVector<VPValue *, 4> Operands(R->operands());
   if (auto *PhiR = dyn_cast<VPWidenPHIRecipe>(R)) {
     VPBasicBlock *Parent = PhiR->getParent();
-    VPRegionBlock *LoopRegionOf = Parent->getEnclosingLoopRegion();
+    [[maybe_unused]] VPRegionBlock *LoopRegionOf =
+        Parent->getEnclosingLoopRegion();
     assert(LoopRegionOf && LoopRegionOf->getEntry() == Parent &&
            "Non-header phis should have been handled during predication");
-    (void)LoopRegionOf;
     auto *Phi = cast<PHINode>(R->getUnderlyingInstr());
     assert(Operands.size() == 2 && "Must have 2 operands for header phis");
     if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
@@ -8273,8 +8297,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
            "can only widen reductions and fixed-order recurrences here");
     VPValue *StartV = Operands[0];
     if (Legal->isReductionVariable(Phi)) {
-      const RecurrenceDescriptor &RdxDesc =
-          Legal->getReductionVars().find(Phi)->second;
+      const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
       assert(RdxDesc.getRecurrenceStartValue() ==
              Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
 
@@ -8282,7 +8305,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
       unsigned ScaleFactor =
           getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
       PhiRecipe = new VPReductionPHIRecipe(
-          Phi, RdxDesc, *StartV, CM.isInLoopReduction(Phi),
+          Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
           CM.useOrderedReductions(RdxDesc), ScaleFactor);
     } else {
       // TODO: Currently fixed-order recurrences are modeled as chains of
@@ -9042,12 +9065,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind Kind = RdxDesc.getRecurrenceKind();
+    RecurKind Kind = PhiR->getRecurrenceKind();
     assert(
         !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
-        !RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
-        "AnyOf and FindLast reductions are not allowed for in-loop reductions");
+        !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
+        "AnyOf and FindIV reductions are not allowed for in-loop reductions");
 
     // Collect the chain of "link" recipes for the reduction starting at PhiR.
     SetVector<VPSingleDefRecipe *> Worklist;
@@ -9149,13 +9171,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
         CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
 
+      // TODO: Retrieve FMFs from recipes directly.
+      RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
+          cast<PHINode>(PhiR->getUnderlyingInstr()));
       // Non-FP RdxDescs will have all fast math flags set, so clear them.
       FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
                                ? RdxDesc.getFastMathFlags()
                                : FastMathFlags();
       auto *RedRecipe = new VPReductionRecipe(
           Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
-          CM.useOrderedReductions(RdxDesc), CurrentLinkI->getDebugLoc());
+          PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
       // Append the recipe to the end of the VPBasicBlock because we need to
       // ensure that it comes after all of it's inputs, including CondOp.
       // Delete CurrentLink as it will be invalid if its operand is replaced
@@ -9179,7 +9204,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     if (!PhiR)
       continue;
 
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+    const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
+        cast<PHINode>(PhiR->getUnderlyingInstr()));
     Type *PhiTy = PhiR->getUnderlyingValue()->getType();
     // If tail is folded by masking, introduce selects between the phi
     // and the users outside the vector region of each reduction, at the
@@ -9205,34 +9231,12 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
                 cast<VPInstruction>(&U)->getOpcode() ==
                     VPInstruction::ComputeReductionResult ||
                 cast<VPInstruction>(&U)->getOpcode() ==
-                    VPInstruction::ComputeFindLastIVResult);
+                    VPInstruction::ComputeFindIVResult);
       });
       if (CM.usePredicatedReductionSelect())
         PhiR->setOperand(1, NewExitingVPV);
     }
 
-    // If the vector reduction can be performed in a smaller type, we truncate
-    // then extend the loop exit value to enable InstCombine to evaluate the
-    // entire expression in the smaller type.
-    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
-        !RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
-      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
-      Type *RdxTy = RdxDesc.getRecurrenceType();
-      auto *Trunc =
-          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
-      auto *Extnd =
-          RdxDesc.isSigned()
-              ? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
-              : new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
-
-      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
-      Extnd->insertAfter(Trunc);
-      if (PhiR->getOperand(1) == NewExitingVPV)
-        PhiR->setOperand(1, Extnd->getVPSingleValue());
-      NewExitingVPV = Extnd;
-    }
-
     // We want code in the middle block to appear to execute on the location of
     // the scalar loop's latch terminator because: (a) it is all compiler
     // generated, (b) these instructions are always executed after evaluating
@@ -9249,41 +9253,70 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
     VPInstruction *FinalReductionResult;
     VPBuilder::InsertPointGuard Guard(Builder);
     Builder.setInsertPoint(MiddleVPBB, IP);
-    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
+    RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
+    if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind)) {
       VPValue *Start = PhiR->getStartValue();
       VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
       FinalReductionResult =
-          Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
+          Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
                                {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
-    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-                   RdxDesc.getRecurrenceKind())) {
+    } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
       VPValue *Start = PhiR->getStartValue();
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
                                {PhiR, Start, NewExitingVPV}, ExitDL);
     } else {
-      VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind(
-                            RdxDesc.getRecurrenceKind())
-                            ? VPIRFlags(RdxDesc.getFastMathFlags())
-                            : VPIRFlags();
+      VPIRFlags Flags =
+          RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind)
+              ? VPIRFlags(RdxDesc.getFastMathFlags())
+              : VPIRFlags();
       FinalReductionResult =
           Builder.createNaryOp(VPInstruction::ComputeReductionResult,
                                {PhiR, NewExitingVPV}, Flags, ExitDL);
     }
-    // Update all users outside the vector region.
-    OrigExitingVPV->replaceUsesWithIf(
-        FinalReductionResult, [FinalReductionResult](VPUser &User, unsigned) {
-          auto *Parent = cast<VPRecipeBase>(&User)->getParent();
-          return FinalReductionResult != &User && !Parent->getParent();
-        });
+    // If the vector reduction can be performed in a smaller type, we truncate
+    // then extend the loop exit value to enable InstCombine to evaluate the
+    // entire expression in the smaller type.
+    if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
+        !RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
+      assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
+      assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
+             "Unexpected truncated min-max recurrence!");
+      Type *RdxTy = RdxDesc.getRecurrenceType();
+      auto *Trunc =
+          new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
+      Instruction::CastOps ExtendOpc =
+          RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
+      auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
+      Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
+      Extnd->insertAfter(Trunc);
+      if (PhiR->getOperand(1) == NewExitingVPV)
+        PhiR->setOperand(1, Extnd->getVPSingleValue());
+
+      // Update ComputeReductionResult with the truncated exiting value and
+      // extend its result.
+      FinalReductionResult->setOperand(1, Trunc);
+      FinalReductionResult =
+          Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
+    }
+
+    // Update all users outside the vector region. Also replace redundant
+    // ExtractLastElement.
+    for (auto *U : to_vector(OrigExitingVPV->users())) {
+      auto *Parent = cast<VPRecipeBase>(U)->getParent();
+      if (FinalReductionResult == U || Parent->getParent())
+        continue;
+      U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
+      if (match(U, m_VPInstruction<VPInstruction::ExtractLastElement>(
+                       m_VPValue())))
+        cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
+    }
 
     // Adjust AnyOf reductions; replace the reduction phi for the selected value
     // with a boolean reduction phi node to check if the condition is true in
     // any iteration. The final value is selected by the final
     // ComputeReductionResult.
-    if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
-            RdxDesc.getRecurrenceKind())) {
+    if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
       auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
         return isa<VPWidenSelectRecipe>(U) ||
                (isa<VPReplicateRecipe>(U) &&
@@ -9312,16 +9345,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       continue;
     }
 
-    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
+    if (RecurrenceDescriptor::isFindIVRecurrenceKind(
             RdxDesc.getRecurrenceKind())) {
-      // Adjust the start value for FindLastIV recurrences to use the sentinel
-      // value after generating the ResumePhi recipe, which uses the original
-      // start value.
+      // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
+      // the sentinel value after generating the ResumePhi recipe, which uses
+      // the original start value.
       PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
     }
     RecurKind RK = RdxDesc.getRecurrenceKind();
     if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
-         !RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+         !RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
          !RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))) {
       VPBuilder PHBuilder(Plan->getVectorPreheader());
       VPValue *Iden = Plan->getOrAddLiveIn(
@@ -9704,18 +9737,18 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
   VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
 
   using namespace VPlanPatternMatch;
-  // When vectorizing the epilogue, FindLastIV reductions can introduce multiple
-  // uses of undef/poison. If the reduction start value may be undef or poison
-  // it needs to be frozen and the frozen start has to be used when computing
-  // the reduction result. We also need to use the frozen value in the resume
-  // phi generated by the main vector loop, as this is also used to compute the
-  // reduction result after the epilogue vector loop.
+  // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
+  // introduce multiple uses of undef/poison. If the reduction start value may
+  // be undef or poison it needs to be frozen and the frozen start has to be
+  // used when computing the reduction result. We also need to use the frozen
+  // value in the resume phi generated by the main vector loop, as this is also
+  // used to compute the reduction result after the epilogue vector loop.
   auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
                                              bool UpdateResumePhis) {
     VPBuilder Builder(Plan.getEntry());
     for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
       auto *VPI = dyn_cast<VPInstruction>(&R);
-      if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindLastIVResult)
+      if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
         continue;
       VPValue *OrigStart = VPI->getOperand(1);
       if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
@@ -9810,38 +9843,30 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
             return VPI &&
                    (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
                     VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
-                    VPI->getOpcode() == VPInstruction::ComputeFindLastIVResult);
+                    VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
           }));
       ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
                     ->getIncomingValueForBlock(L->getLoopPreheader());
-      const RecurrenceDescriptor &RdxDesc =
-          ReductionPhi->getRecurrenceDescriptor();
-      RecurKind RK = RdxDesc.getRecurrenceKind();
+      RecurKind RK = ReductionPhi->getRecurrenceKind();
       if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
         Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
-        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
-               "start value from ComputeAnyOfResult must match");
-
         // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
         // start value; compare the final value from the main vector loop
         // to the start value.
         BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
         ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
-      } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
+      } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
         Value *StartV = getStartValueFromReductionResult(RdxResult);
-        assert(RdxDesc.getRecurrenceStartValue() == StartV &&
-               "start value from ComputeFindLastIVResult must match");
-
         ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
             EPI.MainLoopIterationCountCheck);
 
-        // VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
-        // to the resume value. The resume value is adjusted to the sentinel
-        // value when the final value from the main vector loop equals the start
-        // value. This ensures correctness when the start value might not be
-        // less than the minimum value of a monotonically increasing induction
-        // variable.
+        // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
+        // an adjustment to the resume value. The resume value is adjusted to
+        // the sentinel value when the final value from the main vector loop
+        // equals the start value. This ensures correctness when the start value
+        // might not be less than the minimum value of a monotonically
+        // increasing induction variable.
         BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
         IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
         Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
@@ -10101,6 +10126,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   // Get user vectorization factor and interleave count.
   ElementCount UserVF = Hints.getWidth();
   unsigned UserIC = Hints.getInterleave();
+  if (LVL.hasUncountableEarlyExit() && UserIC != 1 &&
+      !VectorizerParams::isInterleaveForced()) {
+    UserIC = 1;
+    reportVectorizationInfo("Interleaving not supported for loops "
+                            "with uncountable early exits",
+                            "InterleaveEarlyExitDisabled", ORE, L);
+  }
 
   // Plan how to best vectorize.
   LVP.plan(UserVF, UserIC);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 27a7538ecd93..c93af749507f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1511,8 +1511,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
 /// \returns true if all of the values in \p VL have the same type or false
 /// otherwise.
 static bool allSameType(ArrayRef<Value *> VL) {
-  Type *Ty = VL.front()->getType();
-  return all_of(VL.drop_front(), [&](Value *V) { return V->getType() == Ty; });
+  Type *Ty = VL.consume_front()->getType();
+  return all_of(VL, [&](Value *V) { return V->getType() == Ty; });
 }
 
 /// \returns True if in-tree use also needs extract. This refers to
@@ -5443,7 +5443,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       MutableArrayRef<unsigned> Slice = CurrentOrder.slice(I * PartSz, Limit);
       // Shuffle of at least 2 vectors - ignore.
       if (any_of(Slice, [&](unsigned I) { return I != NumScalars; })) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5471,7 +5471,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       FirstMin = (FirstMin / PartSz) * PartSz;
       // Shuffle of at least 2 vectors - ignore.
       if (SecondVecFound) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5492,7 +5492,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
       }
       // Shuffle of at least 2 vectors - ignore.
       if (SecondVecFound) {
-        std::fill(Slice.begin(), Slice.end(), NumScalars);
+        llvm::fill(Slice, NumScalars);
         ShuffledSubMasks.set(I);
         continue;
       }
@@ -5567,8 +5567,8 @@ static bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
 /// Calculates minimal alignment as a common alignment.
 template <typename T>
 static Align computeCommonAlignment(ArrayRef<Value *> VL) {
-  Align CommonAlignment = cast<T>(VL.front())->getAlign();
-  for (Value *V : VL.drop_front())
+  Align CommonAlignment = cast<T>(VL.consume_front())->getAlign();
+  for (Value *V : VL)
     CommonAlignment = std::min(CommonAlignment, cast<T>(V)->getAlign());
   return CommonAlignment;
 }
@@ -9483,8 +9483,8 @@ class PHIHandler {
       ArrayRef<unsigned> IncomingValues = P.second;
       if (IncomingValues.size() <= 1)
         continue;
-      unsigned BasicI = IncomingValues.front();
-      for (unsigned I : IncomingValues.drop_front()) {
+      unsigned BasicI = IncomingValues.consume_front();
+      for (unsigned I : IncomingValues) {
         assert(all_of(enumerate(Operands[I]),
                       [&](const auto &Data) {
                         return !Data.value() ||
@@ -23180,6 +23180,7 @@ class HorizontalReduction {
         case RecurKind::FMul:
         case RecurKind::FMulAdd:
         case RecurKind::AnyOf:
+        case RecurKind::FindFirstIVSMin:
         case RecurKind::FindLastIVSMax:
         case RecurKind::FindLastIVUMax:
         case RecurKind::FMaximumNum:
@@ -23315,6 +23316,7 @@ class HorizontalReduction {
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
     case RecurKind::AnyOf:
+    case RecurKind::FindFirstIVSMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
     case RecurKind::FMaximumNum:
@@ -23415,6 +23417,7 @@ class HorizontalReduction {
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
     case RecurKind::AnyOf:
+    case RecurKind::FindFirstIVSMin:
     case RecurKind::FindLastIVSMax:
     case RecurKind::FindLastIVUMax:
     case RecurKind::FMaximumNum:
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
index f3b62e36e511..ddb4a1e154a1 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/SeedCollection.cpp
@@ -24,6 +24,13 @@ static cl::opt<bool>
     AllowNonPow2("sbvec-allow-non-pow2", cl::init(false), cl::Hidden,
                  cl::desc("Allow non-power-of-2 vectorization."));
 
+#define LoadSeedsDef "loads"
+#define StoreSeedsDef "stores"
+cl::opt<std::string> CollectSeeds(
+    "sbvec-collect-seeds", cl::init(StoreSeedsDef), cl::Hidden,
+    cl::desc("Collect these seeds. Use empty for none or a comma-separated "
+             "list of '" StoreSeedsDef "' and '" LoadSeedsDef "'."));
+
 namespace sandboxir {
 SeedCollection::SeedCollection(StringRef Pipeline)
     : FunctionPass("seed-collection"),
@@ -38,10 +45,12 @@ bool SeedCollection::runOnFunction(Function &F, const Analyses &A) {
           : A.getTTI()
                 .getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
                 .getFixedValue();
+  bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
+  bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
 
   // TODO: Start from innermost BBs first
   for (auto &BB : F) {
-    SeedCollector SC(&BB, A.getScalarEvolution());
+    SeedCollector SC(&BB, A.getScalarEvolution(), CollectStores, CollectLoads);
     for (SeedBundle &Seeds : SC.getStoreSeeds()) {
       unsigned ElmBits =
           Utils::getNumBits(VecUtils::getElementType(Utils::getExpectedType(
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 008976ac9749..1934326866cd 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -21,12 +21,7 @@ namespace llvm::sandboxir {
 static cl::opt<unsigned> SeedBundleSizeLimit(
     "sbvec-seed-bundle-size-limit", cl::init(32), cl::Hidden,
     cl::desc("Limit the size of the seed bundle to cap compilation time."));
-#define LoadSeedsDef "loads"
-#define StoreSeedsDef "stores"
-static cl::opt<std::string> CollectSeeds(
-    "sbvec-collect-seeds", cl::init(LoadSeedsDef "," StoreSeedsDef), cl::Hidden,
-    cl::desc("Collect these seeds. Use empty for none or a comma-separated "
-             "list of '" LoadSeedsDef "' and '" StoreSeedsDef "'."));
+
 static cl::opt<unsigned> SeedGroupsLimit(
     "sbvec-seed-groups-limit", cl::init(256), cl::Hidden,
     cl::desc("Limit the number of collected seeds groups in a BB to "
@@ -162,11 +157,10 @@ template <typename LoadOrStoreT> static bool isValidMemSeed(LoadOrStoreT *LSI) {
 template bool isValidMemSeed<LoadInst>(LoadInst *LSI);
 template bool isValidMemSeed<StoreInst>(StoreInst *LSI);
 
-SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE)
+SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
+                             bool CollectStores, bool CollectLoads)
     : StoreSeeds(SE), LoadSeeds(SE), Ctx(BB->getContext()) {
 
-  bool CollectStores = CollectSeeds.find(StoreSeedsDef) != std::string::npos;
-  bool CollectLoads = CollectSeeds.find(LoadSeedsDef) != std::string::npos;
   if (!CollectStores && !CollectLoads)
     return;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 8369c78a2d78..41878e3c648e 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -24,13 +24,14 @@ struct HistogramInfo;
 struct VFRange;
 
 /// A chain of instructions that form a partial reduction.
-/// Designed to match: reduction_bin_op (bin_op (extend (A), (extend (B))),
-/// accumulator).
+/// Designed to match either:
+///   reduction_bin_op (extend (A), accumulator), or
+///   reduction_bin_op (bin_op (extend (A), (extend (B))), accumulator).
 struct PartialReductionChain {
   PartialReductionChain(Instruction *Reduction, Instruction *ExtendA,
-                        Instruction *ExtendB, Instruction *BinOp)
-      : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB), BinOp(BinOp) {
-  }
+                        Instruction *ExtendB, Instruction *ExtendUser)
+      : Reduction(Reduction), ExtendA(ExtendA), ExtendB(ExtendB),
+        ExtendUser(ExtendUser) {}
   /// The top-level binary operation that forms the reduction to a scalar
   /// after the loop body.
   Instruction *Reduction;
@@ -38,8 +39,8 @@ struct PartialReductionChain {
   Instruction *ExtendA;
   Instruction *ExtendB;
 
-  /// The binary operation using the extends that is then reduced.
-  Instruction *BinOp;
+  /// The user of the extends that is then reduced.
+  Instruction *ExtendUser;
 };
 
 /// Helper class to create VPRecipies from IR instructions.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a3f39f5ad7a2..ffa247afa236 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -261,6 +261,13 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
     return Data.VPV2Scalars[Def][0];
   }
 
+  // Look through BuildVector to avoid redundant extracts.
+  // TODO: Remove once replicate regions are unrolled explicitly.
+  if (Lane.getKind() == VPLane::Kind::First && match(Def, m_BuildVector())) {
+    auto *BuildVector = cast<VPInstruction>(Def);
+    return get(BuildVector->getOperand(Lane.getKnownLane()), true);
+  }
+
   assert(hasVectorValue(Def));
   auto *VecPart = Data.VPV2Vector[Def];
   if (!VecPart->getType()->isVectorTy()) {
@@ -465,7 +472,7 @@ void VPBasicBlock::connectToPredecessors(VPTransformState &State) {
            "Predecessor basic-block not found building successor.");
     BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
     auto *PredBBTerminator = PredBB->getTerminator();
-    LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
+    LLVM_DEBUG(dbgs() << "LV: draw edge from " << PredBB->getName() << '\n');
 
     auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
     if (isa<UnreachableInst>(PredBBTerminator)) {
@@ -579,8 +586,8 @@ VPBasicBlock *VPBasicBlock::clone() {
 }
 
 void VPBasicBlock::executeRecipes(VPTransformState *State, BasicBlock *BB) {
-  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
-                    << " in BB:" << BB->getName() << '\n');
+  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB: " << getName()
+                    << " in BB: " << BB->getName() << '\n');
 
   State->CFG.PrevVPBB = this;
 
@@ -589,7 +596,7 @@ void VPBasicBlock::executeRecipes(VPTransformState *State, BasicBlock *BB) {
     Recipe.execute(*State);
   }
 
-  LLVM_DEBUG(dbgs() << "LV: filled BB:" << *BB);
+  LLVM_DEBUG(dbgs() << "LV: filled BB: " << *BB);
 }
 
 VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
@@ -1501,10 +1508,9 @@ void VPSlotTracker::assignName(const VPValue *V) {
   // Use the name of the underlying Value, wrapped in "ir<>", and versioned by
   // appending ".Number" to the name if there are multiple uses.
   std::string Name;
-  if (UV) {
-    raw_string_ostream S(Name);
-    UV->printAsOperand(S, false);
-  } else
+  if (UV)
+    Name = getName(UV);
+  else
     Name = VPI->getName();
 
   assert(!Name.empty() && "Name cannot be empty.");
@@ -1551,6 +1557,30 @@ void VPSlotTracker::assignNames(const VPBasicBlock *VPBB) {
       assignName(Def);
 }
 
+std::string VPSlotTracker::getName(const Value *V) {
+  std::string Name;
+  raw_string_ostream S(Name);
+  if (V->hasName() || !isa<Instruction>(V)) {
+    V->printAsOperand(S, false);
+    return Name;
+  }
+
+  if (!MST) {
+    // Lazily create the ModuleSlotTracker when we first hit an unnamed
+    // instruction.
+    auto *I = cast<Instruction>(V);
+    // This check is required to support unit tests with incomplete IR.
+    if (I->getParent()) {
+      MST = std::make_unique<ModuleSlotTracker>(I->getModule());
+      MST->incorporateFunction(*I->getFunction());
+    } else {
+      MST = std::make_unique<ModuleSlotTracker>(nullptr);
+    }
+  }
+  V->printAsOperand(S, false, *MST);
+  return Name;
+}
+
 std::string VPSlotTracker::getOrCreateName(const VPValue *V) const {
   std::string Name = VPValue2Name.lookup(V);
   if (!Name.empty())
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f4163b0743a9..356af4a0e74e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -528,11 +528,10 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
     case VPRecipeBase::VPDerivedIVSC:
     case VPRecipeBase::VPEVLBasedIVPHISC:
     case VPRecipeBase::VPExpandSCEVSC:
+    case VPRecipeBase::VPExpressionSC:
     case VPRecipeBase::VPInstructionSC:
     case VPRecipeBase::VPReductionEVLSC:
     case VPRecipeBase::VPReductionSC:
-    case VPRecipeBase::VPMulAccumulateReductionSC:
-    case VPRecipeBase::VPExtendedReductionSC:
     case VPRecipeBase::VPReplicateSC:
     case VPRecipeBase::VPScalarIVStepsSC:
     case VPRecipeBase::VPVectorPointerSC:
@@ -848,13 +847,12 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
            R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
            R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
+           R->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
            R->getVPDefID() == VPRecipeBase::VPReductionSC ||
            R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
            R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
            R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
-           R->getVPDefID() == VPRecipeBase::VPVectorPointerSC ||
-           R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
+           R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
   }
 
   static inline bool classof(const VPUser *U) {
@@ -936,8 +934,17 @@ class VPInstruction : public VPRecipeWithIRFlags,
     BranchOnCount,
     BranchOnCond,
     Broadcast,
+    /// Given operands of (the same) struct type, creates a struct of fixed-
+    /// width vectors each containing a struct field of all operands. The
+    /// number of operands matches the element count of every vector.
+    BuildStructVector,
+    /// Creates a fixed-width vector containing all operands. The number of
+    /// operands matches the vector element count.
+    BuildVector,
+    /// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
+    /// where one of (x,y) is loop invariant, and both x and y are integer type.
     ComputeAnyOfResult,
-    ComputeFindLastIVResult,
+    ComputeFindIVResult,
     ComputeReductionResult,
     // Extracts the last lane from its operand if it is a vector, or the last
     // part if scalar. In the latter case, the recipe will be removed during
@@ -952,10 +959,14 @@ class VPInstruction : public VPRecipeWithIRFlags,
     // operand). Only generates scalar values (either for the first lane only or
     // for all lanes, depending on its uses).
     PtrAdd,
-    // Returns a scalar boolean value, which is true if any lane of its (only
-    // boolean) vector operand is true.
+    // Returns a scalar boolean value, which is true if any lane of its
+    // (boolean) vector operands is true. It produces the reduced value across
+    // all unrolled iterations. Unrolling will add all copies of its original
+    // operand as additional operands.
     AnyOf,
-    // Calculates the first active lane index of the vector predicate operand.
+    // Calculates the first active lane index of the vector predicate operands.
+    // It produces the lane index across all unrolled iterations. Unrolling will
+    // add all copies of its original operand as additional operands.
     FirstActiveLane,
 
     // The opcodes below are used for VPInstructionWithType.
@@ -1001,6 +1012,13 @@ class VPInstruction : public VPRecipeWithIRFlags,
   /// value for lane \p Lane.
   Value *generatePerLane(VPTransformState &State, const VPLane &Lane);
 
+#if !defined(NDEBUG)
+  /// Return the number of operands determined by the opcode of the
+  /// VPInstruction. Returns -1u if the number of operands cannot be determined
+  /// directly by the opcode.
+  static unsigned getNumOperandsForOpcode(unsigned Opcode);
+#endif
+
 public:
   VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL = {},
                 const Twine &Name = "")
@@ -1686,17 +1704,23 @@ class VPWidenGEPRecipe : public VPRecipeWithIRFlags {
 
 /// A recipe to compute a pointer to the last element of each part of a widened
 /// memory access for widened memory accesses of IndexedTy. Used for
-/// VPWidenMemoryRecipes that are reversed.
+/// VPWidenMemoryRecipes or VPInterleaveRecipes that are reversed.
 class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
                                  public VPUnrollPartAccessor<2> {
   Type *IndexedTy;
 
+  /// The constant stride of the pointer computed by this recipe, expressed in
+  /// units of IndexedTy.
+  int64_t Stride;
+
 public:
   VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy,
-                           GEPNoWrapFlags GEPFlags, DebugLoc DL)
+                           int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
       : VPRecipeWithIRFlags(VPDef::VPVectorEndPointerSC,
                             ArrayRef<VPValue *>({Ptr, VF}), GEPFlags, DL),
-        IndexedTy(IndexedTy) {}
+        IndexedTy(IndexedTy), Stride(Stride) {
+    assert(Stride < 0 && "Stride must be negative");
+  }
 
   VP_CLASSOF_IMPL(VPDef::VPVectorEndPointerSC)
 
@@ -1728,7 +1752,8 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
 
   VPVectorEndPointerRecipe *clone() override {
     return new VPVectorEndPointerRecipe(getOperand(0), getVFValue(), IndexedTy,
-                                        getGEPNoWrapFlags(), getDebugLoc());
+                                        Stride, getGEPNoWrapFlags(),
+                                        getDebugLoc());
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2166,8 +2191,8 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
 /// operand.
 class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
                              public VPUnrollPartAccessor<2> {
-  /// Descriptor for the reduction.
-  const RecurrenceDescriptor &RdxDesc;
+  /// The recurrence kind of the reduction.
+  const RecurKind Kind;
 
   /// The phi is part of an in-loop reduction.
   bool IsInLoop;
@@ -2180,23 +2205,21 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
   unsigned VFScaleFactor = 1;
 
 public:
-  /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
-  /// RdxDesc.
-  VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
-                       VPValue &Start, bool IsInLoop = false,
-                       bool IsOrdered = false, unsigned VFScaleFactor = 1)
-      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
-        RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered),
-        VFScaleFactor(VFScaleFactor) {
+  /// Create a new VPReductionPHIRecipe for the reduction \p Phi.
+  VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
+                       bool IsInLoop = false, bool IsOrdered = false,
+                       unsigned VFScaleFactor = 1)
+      : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind),
+        IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) {
     assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
   }
 
   ~VPReductionPHIRecipe() override = default;
 
   VPReductionPHIRecipe *clone() override {
-    auto *R = new VPReductionPHIRecipe(cast<PHINode>(getUnderlyingInstr()),
-                                       RdxDesc, *getOperand(0), IsInLoop,
-                                       IsOrdered, VFScaleFactor);
+    auto *R = new VPReductionPHIRecipe(
+        dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
+        *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor);
     R->addOperand(getBackedgeValue());
     return R;
   }
@@ -2215,9 +2238,8 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
              VPSlotTracker &SlotTracker) const override;
 #endif
 
-  const RecurrenceDescriptor &getRecurrenceDescriptor() const {
-    return RdxDesc;
-  }
+  /// Returns the recurrence kind of the reduction.
+  RecurKind getRecurrenceKind() const { return Kind; }
 
   /// Returns true, if the phi is part of an ordered reduction.
   bool isOrdered() const { return IsOrdered; }
@@ -2321,6 +2343,9 @@ class VPInterleaveRecipe : public VPRecipeBase {
                      DL),
 
         IG(IG), NeedsMaskForGaps(NeedsMaskForGaps) {
+    // TODO: extend the masked interleaved-group support to reversed access.
+    assert((!Mask || !IG->isReverse()) &&
+           "Reversed masked interleave-group not supported.");
     for (unsigned i = 0; i < IG->getFactor(); ++i)
       if (Instruction *I = IG->getMember(i)) {
         if (I->getType()->isVoidTy())
@@ -2420,28 +2445,6 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
     setUnderlyingValue(I);
   }
 
-  /// For VPExtendedReductionRecipe.
-  /// Note that the debug location is from the extend.
-  VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
-                    ArrayRef<VPValue *> Operands, VPValue *CondOp,
-                    bool IsOrdered, DebugLoc DL)
-      : VPRecipeWithIRFlags(SC, Operands, DL), RdxKind(RdxKind),
-        IsOrdered(IsOrdered), IsConditional(CondOp) {
-    if (CondOp)
-      addOperand(CondOp);
-  }
-
-  /// For VPMulAccumulateReductionRecipe.
-  /// Note that the NUW/NSW flags and the debug location are from the Mul.
-  VPReductionRecipe(const unsigned char SC, const RecurKind RdxKind,
-                    ArrayRef<VPValue *> Operands, VPValue *CondOp,
-                    bool IsOrdered, WrapFlagsTy WrapFlags, DebugLoc DL)
-      : VPRecipeWithIRFlags(SC, Operands, WrapFlags, DL), RdxKind(RdxKind),
-        IsOrdered(IsOrdered), IsConditional(CondOp) {
-    if (CondOp)
-      addOperand(CondOp);
-  }
-
 public:
   VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I,
                     VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
@@ -2467,9 +2470,7 @@ class VPReductionRecipe : public VPRecipeWithIRFlags {
 
   static inline bool classof(const VPRecipeBase *R) {
     return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
-           R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC ||
-           R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC;
+           R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
   }
 
   static inline bool classof(const VPUser *U) {
@@ -2608,190 +2609,6 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
   }
 };
 
-/// A recipe to represent inloop extended reduction operations, performing a
-/// reduction on a extended vector operand into a scalar value, and adding the
-/// result to a chain. This recipe is abstract and needs to be lowered to
-/// concrete recipes before codegen. The operands are {ChainOp, VecOp,
-/// [Condition]}.
-class VPExtendedReductionRecipe : public VPReductionRecipe {
-  /// Opcode of the extend for VecOp.
-  Instruction::CastOps ExtOp;
-
-  /// The scalar type after extending.
-  Type *ResultTy;
-
-  /// For cloning VPExtendedReductionRecipe.
-  VPExtendedReductionRecipe(VPExtendedReductionRecipe *ExtRed)
-      : VPReductionRecipe(
-            VPDef::VPExtendedReductionSC, ExtRed->getRecurrenceKind(),
-            {ExtRed->getChainOp(), ExtRed->getVecOp()}, ExtRed->getCondOp(),
-            ExtRed->isOrdered(), ExtRed->getDebugLoc()),
-        ExtOp(ExtRed->getExtOpcode()), ResultTy(ExtRed->getResultType()) {
-    transferFlags(*ExtRed);
-    setUnderlyingValue(ExtRed->getUnderlyingValue());
-  }
-
-public:
-  VPExtendedReductionRecipe(VPReductionRecipe *R, VPWidenCastRecipe *Ext)
-      : VPReductionRecipe(VPDef::VPExtendedReductionSC, R->getRecurrenceKind(),
-                          {R->getChainOp(), Ext->getOperand(0)}, R->getCondOp(),
-                          R->isOrdered(), Ext->getDebugLoc()),
-        ExtOp(Ext->getOpcode()), ResultTy(Ext->getResultType()) {
-    assert((ExtOp == Instruction::CastOps::ZExt ||
-            ExtOp == Instruction::CastOps::SExt) &&
-           "VPExtendedReductionRecipe only supports zext and sext.");
-
-    transferFlags(*Ext);
-    setUnderlyingValue(R->getUnderlyingValue());
-  }
-
-  ~VPExtendedReductionRecipe() override = default;
-
-  VPExtendedReductionRecipe *clone() override {
-    return new VPExtendedReductionRecipe(this);
-  }
-
-  VP_CLASSOF_IMPL(VPDef::VPExtendedReductionSC);
-
-  void execute(VPTransformState &State) override {
-    llvm_unreachable("VPExtendedReductionRecipe should be transform to "
-                     "VPExtendedRecipe + VPReductionRecipe before execution.");
-  };
-
-  /// Return the cost of VPExtendedReductionRecipe.
-  InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print the recipe.
-  void print(raw_ostream &O, const Twine &Indent,
-             VPSlotTracker &SlotTracker) const override;
-#endif
-
-  /// The scalar type after extending.
-  Type *getResultType() const { return ResultTy; }
-
-  /// Is the extend ZExt?
-  bool isZExt() const { return getExtOpcode() == Instruction::ZExt; }
-
-  /// Get the opcode of the extend for VecOp.
-  Instruction::CastOps getExtOpcode() const { return ExtOp; }
-};
-
-/// A recipe to represent inloop MulAccumulateReduction operations,  multiplying
-/// the vector operands (which may be extended), performing a reduction.add on
-/// the result, and adding the scalar result to a chain. This recipe is abstract
-/// and needs to be lowered to concrete recipes before codegen. The operands are
-/// {ChainOp, VecOp1, VecOp2, [Condition]}.
-class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
-  /// Opcode of the extend for VecOp1 and VecOp2.
-  Instruction::CastOps ExtOp;
-
-  /// Non-neg flag of the extend recipe.
-  bool IsNonNeg = false;
-
-  /// The scalar type after extending.
-  Type *ResultTy = nullptr;
-
-  /// For cloning VPMulAccumulateReductionRecipe.
-  VPMulAccumulateReductionRecipe(VPMulAccumulateReductionRecipe *MulAcc)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, MulAcc->getRecurrenceKind(),
-            {MulAcc->getChainOp(), MulAcc->getVecOp0(), MulAcc->getVecOp1()},
-            MulAcc->getCondOp(), MulAcc->isOrdered(),
-            WrapFlagsTy(MulAcc->hasNoUnsignedWrap(), MulAcc->hasNoSignedWrap()),
-            MulAcc->getDebugLoc()),
-        ExtOp(MulAcc->getExtOpcode()), IsNonNeg(MulAcc->isNonNeg()),
-        ResultTy(MulAcc->getResultType()) {
-    transferFlags(*MulAcc);
-    setUnderlyingValue(MulAcc->getUnderlyingValue());
-  }
-
-public:
-  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul,
-                                 VPWidenCastRecipe *Ext0,
-                                 VPWidenCastRecipe *Ext1, Type *ResultTy)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceKind(),
-            {R->getChainOp(), Ext0->getOperand(0), Ext1->getOperand(0)},
-            R->getCondOp(), R->isOrdered(),
-            WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
-            R->getDebugLoc()),
-        ExtOp(Ext0->getOpcode()), ResultTy(ResultTy) {
-    assert(RecurrenceDescriptor::getOpcode(getRecurrenceKind()) ==
-               Instruction::Add &&
-           "The reduction instruction in MulAccumulateteReductionRecipe must "
-           "be Add");
-    assert((ExtOp == Instruction::CastOps::ZExt ||
-            ExtOp == Instruction::CastOps::SExt) &&
-           "VPMulAccumulateReductionRecipe only supports zext and sext.");
-    setUnderlyingValue(R->getUnderlyingValue());
-    // Only set the non-negative flag if the original recipe contains.
-    if (Ext0->hasNonNegFlag())
-      IsNonNeg = Ext0->isNonNeg();
-  }
-
-  VPMulAccumulateReductionRecipe(VPReductionRecipe *R, VPWidenRecipe *Mul,
-                                 Type *ResultTy)
-      : VPReductionRecipe(
-            VPDef::VPMulAccumulateReductionSC, R->getRecurrenceKind(),
-            {R->getChainOp(), Mul->getOperand(0), Mul->getOperand(1)},
-            R->getCondOp(), R->isOrdered(),
-            WrapFlagsTy(Mul->hasNoUnsignedWrap(), Mul->hasNoSignedWrap()),
-            R->getDebugLoc()),
-        ExtOp(Instruction::CastOps::CastOpsEnd), ResultTy(ResultTy) {
-    assert(RecurrenceDescriptor::getOpcode(getRecurrenceKind()) ==
-               Instruction::Add &&
-           "The reduction instruction in MulAccumulateReductionRecipe must be "
-           "Add");
-    setUnderlyingValue(R->getUnderlyingValue());
-  }
-
-  ~VPMulAccumulateReductionRecipe() override = default;
-
-  VPMulAccumulateReductionRecipe *clone() override {
-    return new VPMulAccumulateReductionRecipe(this);
-  }
-
-  VP_CLASSOF_IMPL(VPDef::VPMulAccumulateReductionSC);
-
-  void execute(VPTransformState &State) override {
-    llvm_unreachable("VPMulAccumulateReductionRecipe should transform to "
-                     "VPWidenCastRecipe + "
-                     "VPWidenRecipe + VPReductionRecipe before execution");
-  }
-
-  /// Return the cost of VPMulAccumulateReductionRecipe.
-  InstructionCost computeCost(ElementCount VF,
-                              VPCostContext &Ctx) const override;
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  /// Print the recipe.
-  void print(raw_ostream &O, const Twine &Indent,
-             VPSlotTracker &SlotTracker) const override;
-#endif
-
-  Type *getResultType() const { return ResultTy; }
-
-  /// The first vector value to be extended and reduced.
-  VPValue *getVecOp0() const { return getOperand(1); }
-
-  /// The second vector value to be extended and reduced.
-  VPValue *getVecOp1() const { return getOperand(2); }
-
-  /// Return true if this recipe contains extended operands.
-  bool isExtended() const { return ExtOp != Instruction::CastOps::CastOpsEnd; }
-
-  /// Return the opcode of the extends for the operands.
-  Instruction::CastOps getExtOpcode() const { return ExtOp; }
-
-  /// Return if the operands are zero-extended.
-  bool isZExt() const { return ExtOp == Instruction::CastOps::ZExt; }
-
-  /// Return true if the operand extends have the non-negative flag.
-  bool isNonNeg() const { return IsNonNeg; }
-};
-
 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
 /// copies of the original scalar type, one per lane, instead of producing a
 /// single copy of widened type for all lanes. If the instruction is known to be
@@ -2910,6 +2727,122 @@ class VPBranchOnMaskRecipe : public VPRecipeBase {
   }
 };
 
+/// A recipe to combine multiple recipes into a single 'expression' recipe,
+/// which should be considered a single entity for cost-modeling and transforms.
+/// The recipe needs to be 'decomposed', i.e. replaced by its individual
+/// expression recipes, before execute. The individual expression recipes are
+/// completely disconnected from the def-use graph of other recipes not part of
+/// the expression. Def-use edges between pairs of expression recipes remain
+/// intact, whereas every edge between an expression recipe and a recipe outside
+/// the expression is elevated to connect the non-expression recipe with the
+/// VPExpressionRecipe itself.
+class VPExpressionRecipe : public VPSingleDefRecipe {
+  /// Recipes included in this VPExpressionRecipe.
+  SmallVector<VPSingleDefRecipe *> ExpressionRecipes;
+
+  /// Temporary VPValues used for external operands of the expression, i.e.
+  /// operands not defined by recipes in the expression.
+  SmallVector<VPValue *> LiveInPlaceholders;
+
+  enum class ExpressionTypes {
+    /// Represents an inloop extended reduction operation, performing a
+    /// reduction on an extended vector operand into a scalar value, and adding
+    /// the result to a chain.
+    ExtendedReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// extended vector operands, performing a reduction.add on the result, and
+    /// adding the scalar result to a chain.
+    ExtMulAccReduction,
+    /// Represent an inloop multiply-accumulate reduction, multiplying the
+    /// vector operands, performing a reduction.add on the result, and adding
+    /// the scalar result to a chain.
+    MulAccReduction,
+  };
+
+  /// Type of the expression.
+  ExpressionTypes ExpressionType;
+
+  /// Construct a new VPExpressionRecipe by internalizing recipes in \p
+  /// ExpressionRecipes. External operands (i.e. not defined by another recipe
+  /// in the expression) are replaced by temporary VPValues and the original
+  /// operands are transferred to the VPExpressionRecipe itself. Clone recipes
+  /// as needed (excluding last) to ensure they are only used by other recipes
+  /// in the expression.
+  VPExpressionRecipe(ExpressionTypes ExpressionType,
+                     ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
+
+public:
+  VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
+  VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
+  VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
+                     VPWidenRecipe *Mul, VPReductionRecipe *Red)
+      : VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
+                           {Ext0, Ext1, Mul, Red}) {}
+
+  ~VPExpressionRecipe() override {
+    for (auto *R : reverse(ExpressionRecipes))
+      delete R;
+    for (VPValue *T : LiveInPlaceholders)
+      delete T;
+  }
+
+  VP_CLASSOF_IMPL(VPDef::VPExpressionSC)
+
+  VPExpressionRecipe *clone() override {
+    assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
+    SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
+    for (auto *R : ExpressionRecipes)
+      NewExpressiondRecipes.push_back(R->clone());
+    for (auto *New : NewExpressiondRecipes) {
+      for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
+        New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
+      // Update placeholder operands in the cloned recipe to use the external
+      // operands, to be internalized when the cloned expression is constructed.
+      for (const auto &[Placeholder, OutsideOp] :
+           zip(LiveInPlaceholders, operands()))
+        New->replaceUsesOfWith(Placeholder, OutsideOp);
+    }
+    return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
+  }
+
+  /// Return the VPValue to use to infer the result type of the recipe.
+  VPValue *getOperandOfResultType() const {
+    unsigned OpIdx =
+        cast<VPReductionRecipe>(ExpressionRecipes.back())->isConditional() ? 2
+                                                                           : 1;
+    return getOperand(getNumOperands() - OpIdx);
+  }
+
+  /// Insert the recipes of the expression back into the VPlan, directly before
+  /// the current recipe. Leaves the expression recipe empty, which must be
+  /// removed before codegen.
+  void decompose();
+
+  /// Method for generating code, must not be called as this recipe is abstract.
+  void execute(VPTransformState &State) override {
+    llvm_unreachable("recipe must be removed before execute");
+  }
+
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print the recipe.
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+#endif
+
+  /// Returns true if this expression contains recipes that may read from or
+  /// write to memory.
+  bool mayReadOrWriteMemory() const;
+
+  /// Returns true if this expression contains recipes that may have side
+  /// effects.
+  bool mayHaveSideEffects() const;
+};
+
 /// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
 /// control converges back from a Branch-on-Mask. The phi nodes are needed in
 /// order to merge values that are set under such a branch and feed their uses.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 76da5b0314a8..92db9674ef42 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -91,11 +91,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
            "different types inferred for different operands");
     return IntegerType::get(Ctx, 1);
   case VPInstruction::ComputeAnyOfResult:
-  case VPInstruction::ComputeFindLastIVResult:
+    return inferScalarType(R->getOperand(1));
+  case VPInstruction::ComputeFindIVResult:
   case VPInstruction::ComputeReductionResult: {
-    auto *PhiR = cast<VPReductionPHIRecipe>(R->getOperand(0));
-    auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
-    return OrigPhi->getType();
+    return inferScalarType(R->getOperand(0));
   }
   case VPInstruction::ExplicitVectorLength:
     return Type::getIntNTy(Ctx, 32);
@@ -108,6 +107,8 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case VPInstruction::CalculateTripCountMinusVF:
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::AnyOf:
+  case VPInstruction::BuildStructVector:
+  case VPInstruction::BuildVector:
     return SetResultTyFromOp();
   case VPInstruction::FirstActiveLane:
     return Type::getIntNTy(Ctx, 64);
@@ -296,13 +297,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
             // TODO: Use info from interleave group.
             return V->getUnderlyingValue()->getType();
           })
-          .Case<VPExtendedReductionRecipe, VPMulAccumulateReductionRecipe>(
-              [](const auto *R) { return R->getResultType(); })
           .Case<VPExpandSCEVRecipe>([](const VPExpandSCEVRecipe *R) {
             return R->getSCEV()->getType();
           })
           .Case<VPReductionRecipe>([this](const auto *R) {
             return inferScalarType(R->getChainOp());
+          })
+          .Case<VPExpressionRecipe>([this](const auto *R) {
+            return inferScalarType(R->getOperandOfResultType());
           });
 
   assert(ResultTy && "could not infer type for the given VPValue");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index f33f94b7162c..4154720cd260 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -23,6 +23,7 @@
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/ModuleSlotTracker.h"
 #include "llvm/Support/InstructionCost.h"
 
 namespace llvm {
@@ -388,9 +389,14 @@ class VPSlotTracker {
   /// Number to assign to the next VPValue without underlying value.
   unsigned NextSlot = 0;
 
+  /// Lazily created ModuleSlotTracker, used only when unnamed IR instructions
+  /// require slot tracking.
+  std::unique_ptr<ModuleSlotTracker> MST;
+
   void assignName(const VPValue *V);
   void assignNames(const VPlan &Plan);
   void assignNames(const VPBasicBlock *VPBB);
+  std::string getName(const Value *V);
 
 public:
   VPSlotTracker(const VPlan *Plan = nullptr) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index ae91e310f759..efea99f22d08 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -221,6 +221,13 @@ struct Recipe_match {
   }
 
   bool match(const VPRecipeBase *R) const {
+    if (std::tuple_size<Ops_t>::value == 0) {
+      assert(Opcode == VPInstruction::BuildVector &&
+             "can only match BuildVector with empty ops");
+      auto *VPI = dyn_cast<VPInstruction>(R);
+      return VPI && VPI->getOpcode() == VPInstruction::BuildVector;
+    }
+
     if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
       return false;
 
@@ -263,6 +270,10 @@ struct Recipe_match {
   }
 };
 
+template <unsigned Opcode, typename... RecipeTys>
+using ZeroOpRecipe_match =
+    Recipe_match<std::tuple<>, Opcode, false, RecipeTys...>;
+
 template <typename Op0_t, unsigned Opcode, typename... RecipeTys>
 using UnaryRecipe_match =
     Recipe_match<std::tuple<Op0_t>, Opcode, false, RecipeTys...>;
@@ -271,6 +282,9 @@ template <typename Op0_t, unsigned Opcode>
 using UnaryVPInstruction_match =
     UnaryRecipe_match<Op0_t, Opcode, VPInstruction>;
 
+template <unsigned Opcode>
+using ZeroOpVPInstruction_match = ZeroOpRecipe_match<Opcode, VPInstruction>;
+
 template <typename Op0_t, unsigned Opcode>
 using AllUnaryRecipe_match =
     UnaryRecipe_match<Op0_t, Opcode, VPWidenRecipe, VPReplicateRecipe,
@@ -302,6 +316,12 @@ using AllBinaryRecipe_match =
     BinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative, VPWidenRecipe,
                        VPReplicateRecipe, VPWidenCastRecipe, VPInstruction>;
 
+/// BuildVector is matches only its opcode, w/o matching its operands as the
+/// number of operands is not fixed.
+inline ZeroOpVPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
+  return ZeroOpVPInstruction_match<VPInstruction::BuildVector>();
+}
+
 template <unsigned Opcode, typename Op0_t>
 inline UnaryVPInstruction_match<Op0_t, Opcode>
 m_VPInstruction(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 805cd04c5ce3..ccb7512051d7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -49,6 +49,8 @@ using VectorParts = SmallVector<Value *, 2>;
 
 bool VPRecipeBase::mayWriteToMemory() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
   case VPInstructionSC:
     return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
   case VPInterleaveSC:
@@ -73,8 +75,6 @@ bool VPRecipeBase::mayWriteToMemory() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
   case VPWidenCastSC:
@@ -99,6 +99,8 @@ bool VPRecipeBase::mayWriteToMemory() const {
 
 bool VPRecipeBase::mayReadFromMemory() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayReadOrWriteMemory();
   case VPInstructionSC:
     return cast<VPInstruction>(this)->opcodeMayReadOrWriteFromMemory();
   case VPWidenLoadEVLSC:
@@ -123,8 +125,6 @@ bool VPRecipeBase::mayReadFromMemory() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
   case VPWidenCastSC:
@@ -147,6 +147,8 @@ bool VPRecipeBase::mayReadFromMemory() const {
 
 bool VPRecipeBase::mayHaveSideEffects() const {
   switch (getVPDefID()) {
+  case VPExpressionSC:
+    return cast<VPExpressionRecipe>(this)->mayHaveSideEffects();
   case VPDerivedIVSC:
   case VPFirstOrderRecurrencePHISC:
   case VPPredInstPHISC:
@@ -163,8 +165,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPBlendSC:
   case VPReductionEVLSC:
   case VPReductionSC:
-  case VPExtendedReductionSC:
-  case VPMulAccumulateReductionSC:
   case VPScalarIVStepsSC:
   case VPVectorPointerSC:
   case VPWidenCanonicalIVSC:
@@ -296,49 +296,70 @@ bool VPRecipeBase::isScalarCast() const {
 InstructionCost
 VPPartialReductionRecipe::computeCost(ElementCount VF,
                                       VPCostContext &Ctx) const {
-  std::optional<unsigned> Opcode = std::nullopt;
-  VPValue *BinOp = getOperand(1);
+  std::optional<unsigned> Opcode;
+  VPValue *Op = getOperand(0);
+  VPRecipeBase *OpR = Op->getDefiningRecipe();
 
-  // If the partial reduction is predicated, a select will be operand 0 rather
-  // than the binary op
+  // If the partial reduction is predicated, a select will be operand 0
   using namespace llvm::VPlanPatternMatch;
-  if (match(getOperand(1), m_Select(m_VPValue(), m_VPValue(), m_VPValue())))
-    BinOp = BinOp->getDefiningRecipe()->getOperand(1);
-
-  // If BinOp is a negation, use the side effect of match to assign the actual
-  // binary operation to BinOp
-  match(BinOp, m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(BinOp)));
-  VPRecipeBase *BinOpR = BinOp->getDefiningRecipe();
-
-  if (auto *WidenR = dyn_cast<VPWidenRecipe>(BinOpR))
-    Opcode = std::make_optional(WidenR->getOpcode());
-
-  VPRecipeBase *ExtAR = BinOpR->getOperand(0)->getDefiningRecipe();
-  VPRecipeBase *ExtBR = BinOpR->getOperand(1)->getDefiningRecipe();
+  if (match(getOperand(1), m_Select(m_VPValue(), m_VPValue(Op), m_VPValue()))) {
+    OpR = Op->getDefiningRecipe();
+  }
 
-  auto *PhiType = Ctx.Types.inferScalarType(getOperand(1));
-  auto *InputTypeA = Ctx.Types.inferScalarType(ExtAR ? ExtAR->getOperand(0)
-                                                     : BinOpR->getOperand(0));
-  auto *InputTypeB = Ctx.Types.inferScalarType(ExtBR ? ExtBR->getOperand(0)
-                                                     : BinOpR->getOperand(1));
+  Type *InputTypeA = nullptr, *InputTypeB = nullptr;
+  TTI::PartialReductionExtendKind ExtAType = TTI::PR_None,
+                                  ExtBType = TTI::PR_None;
 
   auto GetExtendKind = [](VPRecipeBase *R) {
-    // The extend could come from outside the plan.
     if (!R)
-      return TargetTransformInfo::PR_None;
+      return TTI::PR_None;
     auto *WidenCastR = dyn_cast<VPWidenCastRecipe>(R);
     if (!WidenCastR)
-      return TargetTransformInfo::PR_None;
+      return TTI::PR_None;
     if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
-      return TargetTransformInfo::PR_ZeroExtend;
+      return TTI::PR_ZeroExtend;
     if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)
-      return TargetTransformInfo::PR_SignExtend;
-    return TargetTransformInfo::PR_None;
+      return TTI::PR_SignExtend;
+    return TTI::PR_None;
   };
 
-  return Ctx.TTI.getPartialReductionCost(
-      getOpcode(), InputTypeA, InputTypeB, PhiType, VF, GetExtendKind(ExtAR),
-      GetExtendKind(ExtBR), Opcode, Ctx.CostKind);
+  // Pick out opcode, type/ext information and use sub side effects from a widen
+  // recipe.
+  auto HandleWiden = [&](VPWidenRecipe *Widen) {
+    if (match(Widen,
+              m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(Op)))) {
+      Widen = dyn_cast<VPWidenRecipe>(Op->getDefiningRecipe());
+    }
+    Opcode = Widen->getOpcode();
+    VPRecipeBase *ExtAR = Widen->getOperand(0)->getDefiningRecipe();
+    VPRecipeBase *ExtBR = Widen->getOperand(1)->getDefiningRecipe();
+    InputTypeA = Ctx.Types.inferScalarType(ExtAR ? ExtAR->getOperand(0)
+                                                 : Widen->getOperand(0));
+    InputTypeB = Ctx.Types.inferScalarType(ExtBR ? ExtBR->getOperand(0)
+                                                 : Widen->getOperand(1));
+    ExtAType = GetExtendKind(ExtAR);
+    ExtBType = GetExtendKind(ExtBR);
+  };
+
+  if (isa<VPWidenCastRecipe>(OpR)) {
+    InputTypeA = Ctx.Types.inferScalarType(OpR->getOperand(0));
+    ExtAType = GetExtendKind(OpR);
+  } else if (isa<VPReductionPHIRecipe>(OpR)) {
+    auto RedPhiOp1R = getOperand(1)->getDefiningRecipe();
+    if (isa<VPWidenCastRecipe>(RedPhiOp1R)) {
+      InputTypeA = Ctx.Types.inferScalarType(RedPhiOp1R->getOperand(0));
+      ExtAType = GetExtendKind(RedPhiOp1R);
+    } else if (auto Widen = dyn_cast<VPWidenRecipe>(RedPhiOp1R))
+      HandleWiden(Widen);
+  } else if (auto Widen = dyn_cast<VPWidenRecipe>(OpR)) {
+    HandleWiden(Widen);
+  } else if (auto Reduction = dyn_cast<VPPartialReductionRecipe>(OpR)) {
+    return Reduction->computeCost(VF, Ctx);
+  }
+  auto *PhiType = Ctx.Types.inferScalarType(getOperand(1));
+  return Ctx.TTI.getPartialReductionCost(getOpcode(), InputTypeA, InputTypeB,
+                                         PhiType, VF, ExtAType, ExtBType,
+                                         Opcode, Ctx.CostKind);
 }
 
 void VPPartialReductionRecipe::execute(VPTransformState &State) {
@@ -415,8 +436,64 @@ VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
       VPIRMetadata(), Opcode(Opcode), Name(Name.str()) {
   assert(flagsValidForOpcode(getOpcode()) &&
          "Set flags not supported for the provided opcode");
+  assert((getNumOperandsForOpcode(Opcode) == -1u ||
+          getNumOperandsForOpcode(Opcode) == getNumOperands()) &&
+         "number of operands does not match opcode");
 }
 
+#ifndef NDEBUG
+unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
+  if (Instruction::isUnaryOp(Opcode) || Instruction::isCast(Opcode))
+    return 1;
+
+  if (Instruction::isBinaryOp(Opcode))
+    return 2;
+
+  switch (Opcode) {
+  case VPInstruction::StepVector:
+    return 0;
+  case Instruction::Alloca:
+  case Instruction::ExtractValue:
+  case Instruction::Freeze:
+  case Instruction::Load:
+  case VPInstruction::AnyOf:
+  case VPInstruction::BranchOnCond:
+  case VPInstruction::CalculateTripCountMinusVF:
+  case VPInstruction::CanonicalIVIncrementForPart:
+  case VPInstruction::ExplicitVectorLength:
+  case VPInstruction::ExtractLastElement:
+  case VPInstruction::ExtractPenultimateElement:
+  case VPInstruction::FirstActiveLane:
+  case VPInstruction::Not:
+    return 1;
+  case Instruction::ICmp:
+  case Instruction::FCmp:
+  case Instruction::Store:
+  case VPInstruction::ActiveLaneMask:
+  case VPInstruction::BranchOnCount:
+  case VPInstruction::ComputeReductionResult:
+  case VPInstruction::FirstOrderRecurrenceSplice:
+  case VPInstruction::LogicalAnd:
+  case VPInstruction::PtrAdd:
+  case VPInstruction::WideIVStep:
+    return 2;
+  case Instruction::Select:
+  case VPInstruction::ComputeAnyOfResult:
+  case VPInstruction::ReductionStartVector:
+    return 3;
+  case VPInstruction::ComputeFindIVResult:
+    return 4;
+  case Instruction::Call:
+  case Instruction::GetElementPtr:
+  case Instruction::PHI:
+  case Instruction::Switch:
+    // Cannot determine the number of operands from the opcode.
+    return -1u;
+  }
+  llvm_unreachable("all cases should be handled above");
+}
+#endif
+
 bool VPInstruction::doesGeneratePerAllLanes() const {
   return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
 }
@@ -495,6 +572,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
   }
   case Instruction::ExtractElement: {
     assert(State.VF.isVector() && "Only extract elements from vectors");
+    if (getOperand(1)->isLiveIn()) {
+      unsigned IdxToExtract =
+          cast<ConstantInt>(getOperand(1)->getLiveInIRValue())->getZExtValue();
+      return State.get(getOperand(0), VPLane(IdxToExtract));
+    }
     Value *Vec = State.get(getOperand(0));
     Value *Idx = State.get(getOperand(1), /*IsScalar=*/true);
     return Builder.CreateExtractElement(Vec, Idx, Name);
@@ -608,6 +690,34 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return Builder.CreateVectorSplat(
         State.VF, State.get(getOperand(0), /*IsScalar*/ true), "broadcast");
   }
+  case VPInstruction::BuildStructVector: {
+    // For struct types, we need to build a new 'wide' struct type, where each
+    // element is widened, i.e., we create a struct of vectors.
+    auto *StructTy =
+        cast<StructType>(State.TypeAnalysis.inferScalarType(getOperand(0)));
+    Value *Res = PoisonValue::get(toVectorizedTy(StructTy, State.VF));
+    for (const auto &[LaneIndex, Op] : enumerate(operands())) {
+      for (unsigned FieldIndex = 0; FieldIndex != StructTy->getNumElements();
+           FieldIndex++) {
+        Value *ScalarValue =
+            Builder.CreateExtractValue(State.get(Op, true), FieldIndex);
+        Value *VectorValue = Builder.CreateExtractValue(Res, FieldIndex);
+        VectorValue =
+            Builder.CreateInsertElement(VectorValue, ScalarValue, LaneIndex);
+        Res = Builder.CreateInsertValue(Res, VectorValue, FieldIndex);
+      }
+    }
+    return Res;
+  }
+  case VPInstruction::BuildVector: {
+    auto *ScalarTy = State.TypeAnalysis.inferScalarType(getOperand(0));
+    auto NumOfElements = ElementCount::getFixed(getNumOperands());
+    Value *Res = PoisonValue::get(toVectorizedTy(ScalarTy, NumOfElements));
+    for (const auto &[Idx, Op] : enumerate(operands()))
+      Res = State.Builder.CreateInsertElement(Res, State.get(Op, true),
+                                              State.Builder.getInt32(Idx));
+    return Res;
+  }
   case VPInstruction::ReductionStartVector: {
     if (State.VF.isScalar())
       return State.get(getOperand(0), true);
@@ -636,14 +746,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return createAnyOfReduction(Builder, ReducedPartRdx,
                                 State.get(getOperand(1), VPLane(0)), OrigPhi);
   }
-  case VPInstruction::ComputeFindLastIVResult: {
+  case VPInstruction::ComputeFindIVResult: {
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
     // Get its reduction variable descriptor.
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind RK = RdxDesc.getRecurrenceKind();
-    assert(RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
+    RecurKind RK = PhiR->getRecurrenceKind();
+    assert(RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
            "Unexpected reduction kind");
     assert(!PhiR->isInLoop() &&
            "In-loop FindLastIV reduction is not supported yet");
@@ -652,9 +761,16 @@ Value *VPInstruction::generate(VPTransformState &State) {
     // sentinel value, followed by one operand for each part of the reduction.
     unsigned UF = getNumOperands() - 3;
     Value *ReducedPartRdx = State.get(getOperand(3));
-    RecurKind MinMaxKind = RecurrenceDescriptor::isSignedRecurrenceKind(RK)
-                               ? RecurKind::SMax
-                               : RecurKind::UMax;
+    RecurKind MinMaxKind;
+    bool IsSigned = RecurrenceDescriptor::isSignedRecurrenceKind(RK);
+    if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
+      MinMaxKind = IsSigned ? RecurKind::SMax : RecurKind::UMax;
+    } else {
+      assert(RecurrenceDescriptor::isFindFirstIVRecurrenceKind(RK) &&
+             "Kind must either be FindLastIV or FindFirstIV");
+      assert(IsSigned && "Only FindFirstIV with SMax is currently supported");
+      MinMaxKind = RecurKind::SMin;
+    }
     for (unsigned Part = 1; Part < UF; ++Part)
       ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
                                       State.get(getOperand(3 + Part)));
@@ -669,13 +785,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
     // Get its reduction variable descriptor.
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
 
-    RecurKind RK = RdxDesc.getRecurrenceKind();
-    assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
-           "should be handled by ComputeFindLastIVResult");
+    RecurKind RK = PhiR->getRecurrenceKind();
+    assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
+           "should be handled by ComputeFindIVResult");
 
-    Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
     // The recipe's operands are the reduction phi, followed by one operand for
     // each part of the reduction.
     unsigned UF = getNumOperands() - 1;
@@ -687,15 +801,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
     if (hasFastMathFlags())
       Builder.setFastMathFlags(getFastMathFlags());
 
-    // If the vector reduction can be performed in a smaller type, we truncate
-    // then extend the loop exit value to enable InstCombine to evaluate the
-    // entire expression in the smaller type.
-    // TODO: Handle this in truncateToMinBW.
-    if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
-      Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
-      for (unsigned Part = 0; Part < UF; ++Part)
-        RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
-    }
     // Reduce all of the unrolled parts into a single vector.
     Value *ReducedPartRdx = RdxParts[0];
     if (PhiR->isOrdered()) {
@@ -707,9 +812,9 @@ Value *VPInstruction::generate(VPTransformState &State) {
         if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))
           ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
         else
-          ReducedPartRdx =
-              Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
-                                  RdxPart, ReducedPartRdx, "bin.rdx");
+          ReducedPartRdx = Builder.CreateBinOp(
+              (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),
+              RdxPart, ReducedPartRdx, "bin.rdx");
       }
     }
 
@@ -720,13 +825,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
       // All ops in the reduction inherit fast-math-flags from the recurrence
       // descriptor.
       ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
-
-      // If the reduction can be performed in a smaller type, we need to extend
-      // the reduction to the wider type before we branch to the original loop.
-      if (ResultTy != RdxDesc.getRecurrenceType())
-        ReducedPartRdx = RdxDesc.isSigned()
-                             ? Builder.CreateSExt(ReducedPartRdx, ResultTy)
-                             : Builder.CreateZExt(ReducedPartRdx, ResultTy);
     }
 
     return ReducedPartRdx;
@@ -761,13 +859,38 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
   }
   case VPInstruction::AnyOf: {
-    Value *A = State.get(getOperand(0));
-    return Builder.CreateOrReduce(A);
+    Value *Res = State.get(getOperand(0));
+    for (VPValue *Op : drop_begin(operands()))
+      Res = Builder.CreateOr(Res, State.get(Op));
+    return Builder.CreateOrReduce(Res);
   }
   case VPInstruction::FirstActiveLane: {
-    Value *Mask = State.get(getOperand(0));
-    return Builder.CreateCountTrailingZeroElems(Builder.getInt64Ty(), Mask,
-                                                true, Name);
+    if (getNumOperands() == 1) {
+      Value *Mask = State.get(getOperand(0));
+      return Builder.CreateCountTrailingZeroElems(Builder.getInt64Ty(), Mask,
+                                                  true, Name);
+    }
+    // If there are multiple operands, create a chain of selects to pick the
+    // first operand with an active lane and add the number of lanes of the
+    // preceding operands.
+    Value *RuntimeVF =
+        getRuntimeVF(State.Builder, State.Builder.getInt64Ty(), State.VF);
+    unsigned LastOpIdx = getNumOperands() - 1;
+    Value *Res = nullptr;
+    for (int Idx = LastOpIdx; Idx >= 0; --Idx) {
+      Value *TrailingZeros = Builder.CreateCountTrailingZeroElems(
+          Builder.getInt64Ty(), State.get(getOperand(Idx)), true, Name);
+      Value *Current = Builder.CreateAdd(
+          Builder.CreateMul(RuntimeVF, Builder.getInt64(Idx)), TrailingZeros);
+      if (Res) {
+        Value *Cmp = Builder.CreateICmpNE(TrailingZeros, RuntimeVF);
+        Res = Builder.CreateSelect(Cmp, Current, Res);
+      } else {
+        Res = Current;
+      }
+    }
+
+    return Res;
   }
   default:
     llvm_unreachable("Unsupported opcode for instruction");
@@ -863,7 +986,7 @@ bool VPInstruction::isVectorToScalar() const {
          getOpcode() == Instruction::ExtractElement ||
          getOpcode() == VPInstruction::FirstActiveLane ||
          getOpcode() == VPInstruction::ComputeAnyOfResult ||
-         getOpcode() == VPInstruction::ComputeFindLastIVResult ||
+         getOpcode() == VPInstruction::ComputeFindIVResult ||
          getOpcode() == VPInstruction::ComputeReductionResult ||
          getOpcode() == VPInstruction::AnyOf;
 }
@@ -897,10 +1020,11 @@ void VPInstruction::execute(VPTransformState &State) {
   if (!hasResult())
     return;
   assert(GeneratedValue && "generate must produce a value");
-  assert(
-      (GeneratedValue->getType()->isVectorTy() == !GeneratesPerFirstLaneOnly ||
-       State.VF.isScalar()) &&
-      "scalar value but not only first lane defined");
+  assert((((GeneratedValue->getType()->isVectorTy() ||
+            GeneratedValue->getType()->isStructTy()) ==
+           !GeneratesPerFirstLaneOnly) ||
+          State.VF.isScalar()) &&
+         "scalar value but not only first lane defined");
   State.set(this, GeneratedValue,
             /*IsScalar*/ GeneratesPerFirstLaneOnly);
 }
@@ -914,6 +1038,8 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case Instruction::ICmp:
   case Instruction::Select:
   case VPInstruction::AnyOf:
+  case VPInstruction::BuildStructVector:
+  case VPInstruction::BuildVector:
   case VPInstruction::CalculateTripCountMinusVF:
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::ExtractLastElement:
@@ -962,7 +1088,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   case VPInstruction::PtrAdd:
     return Op == getOperand(0) || vputils::onlyFirstLaneUsed(this);
   case VPInstruction::ComputeAnyOfResult:
-  case VPInstruction::ComputeFindLastIVResult:
+  case VPInstruction::ComputeFindIVResult:
     return Op == getOperand(1);
   };
   llvm_unreachable("switch should return");
@@ -1036,6 +1162,12 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::Broadcast:
     O << "broadcast";
     break;
+  case VPInstruction::BuildStructVector:
+    O << "buildstructvector";
+    break;
+  case VPInstruction::BuildVector:
+    O << "buildvector";
+    break;
   case VPInstruction::ExtractLastElement:
     O << "extract-last-element";
     break;
@@ -1045,8 +1177,8 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ComputeAnyOfResult:
     O << "compute-anyof-result";
     break;
-  case VPInstruction::ComputeFindLastIVResult:
-    O << "compute-find-last-iv-result";
+  case VPInstruction::ComputeFindIVResult:
+    O << "compute-find-iv-result";
     break;
   case VPInstruction::ComputeReductionResult:
     O << "compute-reduction-result";
@@ -2196,12 +2328,12 @@ void VPWidenGEPRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
-static Type *getGEPIndexTy(bool IsScalable, bool IsReverse,
+static Type *getGEPIndexTy(bool IsScalable, bool IsReverse, bool IsUnitStride,
                            unsigned CurrentPart, IRBuilderBase &Builder) {
   // Use i32 for the gep index type when the value is constant,
   // or query DataLayout for a more suitable index type otherwise.
   const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();
-  return IsScalable && (IsReverse || CurrentPart > 0)
+  return !IsUnitStride || (IsScalable && (IsReverse || CurrentPart > 0))
              ? DL.getIndexType(Builder.getPtrTy(0))
              : Builder.getInt32Ty();
 }
@@ -2209,18 +2341,21 @@ static Type *getGEPIndexTy(bool IsScalable, bool IsReverse,
 void VPVectorEndPointerRecipe::execute(VPTransformState &State) {
   auto &Builder = State.Builder;
   unsigned CurrentPart = getUnrollPart(*this);
+  bool IsUnitStride = Stride == 1 || Stride == -1;
   Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ true,
-                                CurrentPart, Builder);
+                                IsUnitStride, CurrentPart, Builder);
 
   // The wide store needs to start at the last vector element.
   Value *RunTimeVF = State.get(getVFValue(), VPLane(0));
   if (IndexTy != RunTimeVF->getType())
     RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);
-  // NumElt = -CurrentPart * RunTimeVF
+  // NumElt = Stride * CurrentPart * RunTimeVF
   Value *NumElt = Builder.CreateMul(
-      ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
-  // LastLane = 1 - RunTimeVF
-  Value *LastLane = Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
+      ConstantInt::get(IndexTy, Stride * (int64_t)CurrentPart), RunTimeVF);
+  // LastLane = Stride * (RunTimeVF - 1)
+  Value *LastLane = Builder.CreateSub(RunTimeVF, ConstantInt::get(IndexTy, 1));
+  if (Stride != 1)
+    LastLane = Builder.CreateMul(ConstantInt::get(IndexTy, Stride), LastLane);
   Value *Ptr = State.get(getOperand(0), VPLane(0));
   Value *ResultPtr =
       Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", getGEPNoWrapFlags());
@@ -2245,7 +2380,7 @@ void VPVectorPointerRecipe::execute(VPTransformState &State) {
   auto &Builder = State.Builder;
   unsigned CurrentPart = getUnrollPart(*this);
   Type *IndexTy = getGEPIndexTy(State.VF.isScalable(), /*IsReverse*/ false,
-                                CurrentPart, Builder);
+                                /*IsUnitStride*/ true, CurrentPart, Builder);
   Value *Ptr = State.get(getOperand(0), VPLane(0));
 
   Value *Increment = createStepForVF(Builder, IndexTy, State.VF, CurrentPart);
@@ -2450,30 +2585,183 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
                                             Ctx.CostKind);
 }
 
-InstructionCost
-VPExtendedReductionRecipe::computeCost(ElementCount VF,
-                                       VPCostContext &Ctx) const {
-  unsigned Opcode = RecurrenceDescriptor::getOpcode(getRecurrenceKind());
+VPExpressionRecipe::VPExpressionRecipe(
+    ExpressionTypes ExpressionType,
+    ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
+    : VPSingleDefRecipe(VPDef::VPExpressionSC, {}, {}),
+      ExpressionRecipes(SetVector<VPSingleDefRecipe *>(
+                            ExpressionRecipes.begin(), ExpressionRecipes.end())
+                            .takeVector()),
+      ExpressionType(ExpressionType) {
+  assert(!ExpressionRecipes.empty() && "Nothing to combine?");
+  assert(
+      none_of(ExpressionRecipes,
+              [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&
+      "expression cannot contain recipes with side-effects");
+
+  // Maintain a copy of the expression recipes as a set of users.
+  SmallPtrSet<VPUser *, 4> ExpressionRecipesAsSetOfUsers;
+  for (auto *R : ExpressionRecipes)
+    ExpressionRecipesAsSetOfUsers.insert(R);
+
+  // Recipes in the expression, except the last one, must only be used by
+  // (other) recipes inside the expression. If there are other users, external
+  // to the expression, use a clone of the recipe for external users.
+  for (VPSingleDefRecipe *R : ExpressionRecipes) {
+    if (R != ExpressionRecipes.back() &&
+        any_of(R->users(), [&ExpressionRecipesAsSetOfUsers](VPUser *U) {
+          return !ExpressionRecipesAsSetOfUsers.contains(U);
+        })) {
+      // There are users outside of the expression. Clone the recipe and use the
+      // clone those external users.
+      VPSingleDefRecipe *CopyForExtUsers = R->clone();
+      R->replaceUsesWithIf(CopyForExtUsers, [&ExpressionRecipesAsSetOfUsers](
+                                                VPUser &U, unsigned) {
+        return !ExpressionRecipesAsSetOfUsers.contains(&U);
+      });
+      CopyForExtUsers->insertBefore(R);
+    }
+    if (R->getParent())
+      R->removeFromParent();
+  }
+
+  // Internalize all external operands to the expression recipes. To do so,
+  // create new temporary VPValues for all operands defined by a recipe outside
+  // the expression. The original operands are added as operands of the
+  // VPExpressionRecipe itself.
+  for (auto *R : ExpressionRecipes) {
+    for (const auto &[Idx, Op] : enumerate(R->operands())) {
+      auto *Def = Op->getDefiningRecipe();
+      if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
+        continue;
+      addOperand(Op);
+      LiveInPlaceholders.push_back(new VPValue());
+      R->setOperand(Idx, LiveInPlaceholders.back());
+    }
+  }
+}
+
+void VPExpressionRecipe::decompose() {
+  for (auto *R : ExpressionRecipes)
+    R->insertBefore(this);
+
+  for (const auto &[Idx, Op] : enumerate(operands()))
+    LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);
+
+  replaceAllUsesWith(ExpressionRecipes.back());
+  ExpressionRecipes.clear();
+}
+
+InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
+                                                VPCostContext &Ctx) const {
   Type *RedTy = Ctx.Types.inferScalarType(this);
-  auto *SrcVecTy =
-      cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp()), VF));
+  auto *SrcVecTy = cast<VectorType>(
+      toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF));
   assert(RedTy->isIntegerTy() &&
-         "ExtendedReduction only support integer type currently.");
-  return Ctx.TTI.getExtendedReductionCost(Opcode, isZExt(), RedTy, SrcVecTy,
-                                          std::nullopt, Ctx.CostKind);
+         "VPExpressionRecipe only supports integer types currently.");
+  switch (ExpressionType) {
+  case ExpressionTypes::ExtendedReduction: {
+    unsigned Opcode = RecurrenceDescriptor::getOpcode(
+        cast<VPReductionRecipe>(ExpressionRecipes[1])->getRecurrenceKind());
+    return Ctx.TTI.getExtendedReductionCost(
+        Opcode,
+        cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
+            Instruction::ZExt,
+        RedTy, SrcVecTy, std::nullopt, Ctx.CostKind);
+  }
+  case ExpressionTypes::MulAccReduction:
+    return Ctx.TTI.getMulAccReductionCost(false, RedTy, SrcVecTy, Ctx.CostKind);
+
+  case ExpressionTypes::ExtMulAccReduction:
+    return Ctx.TTI.getMulAccReductionCost(
+        cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
+            Instruction::ZExt,
+        RedTy, SrcVecTy, Ctx.CostKind);
+  }
+  llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
+}
+
+bool VPExpressionRecipe::mayReadOrWriteMemory() const {
+  return any_of(ExpressionRecipes, [](VPSingleDefRecipe *R) {
+    return R->mayReadFromMemory() || R->mayWriteToMemory();
+  });
 }
 
-InstructionCost
-VPMulAccumulateReductionRecipe::computeCost(ElementCount VF,
-                                            VPCostContext &Ctx) const {
-  Type *RedTy = Ctx.Types.inferScalarType(this);
-  auto *SrcVecTy =
-      cast<VectorType>(toVectorTy(Ctx.Types.inferScalarType(getVecOp0()), VF));
-  return Ctx.TTI.getMulAccReductionCost(isZExt(), RedTy, SrcVecTy,
-                                        Ctx.CostKind);
+bool VPExpressionRecipe::mayHaveSideEffects() const {
+  assert(
+      none_of(ExpressionRecipes,
+              [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&
+      "expression cannot contain recipes with side-effects");
+  return false;
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+void VPExpressionRecipe::print(raw_ostream &O, const Twine &Indent,
+                               VPSlotTracker &SlotTracker) const {
+  O << Indent << "EXPRESSION ";
+  printAsOperand(O, SlotTracker);
+  O << " = ";
+  auto *Red = cast<VPReductionRecipe>(ExpressionRecipes.back());
+  unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
+
+  switch (ExpressionType) {
+  case ExpressionTypes::ExtendedReduction: {
+    getOperand(1)->printAsOperand(O, SlotTracker);
+    O << " +";
+    O << " reduce." << Instruction::getOpcodeName(Opcode) << " (";
+    getOperand(0)->printAsOperand(O, SlotTracker);
+    Red->printFlags(O);
+
+    auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
+    O << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
+      << *Ext0->getResultType();
+    if (Red->isConditional()) {
+      O << ", ";
+      Red->getCondOp()->printAsOperand(O, SlotTracker);
+    }
+    O << ")";
+    break;
+  }
+  case ExpressionTypes::MulAccReduction:
+  case ExpressionTypes::ExtMulAccReduction: {
+    getOperand(getNumOperands() - 1)->printAsOperand(O, SlotTracker);
+    O << " + ";
+    O << "reduce."
+      << Instruction::getOpcodeName(
+             RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind()))
+      << " (";
+    O << "mul";
+    bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;
+    auto *Mul = cast<VPWidenRecipe>(IsExtended ? ExpressionRecipes[2]
+                                               : ExpressionRecipes[0]);
+    Mul->printFlags(O);
+    if (IsExtended)
+      O << "(";
+    getOperand(0)->printAsOperand(O, SlotTracker);
+    if (IsExtended) {
+      auto *Ext0 = cast<VPWidenCastRecipe>(ExpressionRecipes[0]);
+      O << " " << Instruction::getOpcodeName(Ext0->getOpcode()) << " to "
+        << *Ext0->getResultType() << "), (";
+    } else {
+      O << ", ";
+    }
+    getOperand(1)->printAsOperand(O, SlotTracker);
+    if (IsExtended) {
+      auto *Ext1 = cast<VPWidenCastRecipe>(ExpressionRecipes[1]);
+      O << " " << Instruction::getOpcodeName(Ext1->getOpcode()) << " to "
+        << *Ext1->getResultType() << ")";
+    }
+    if (Red->isConditional()) {
+      O << ", ";
+      Red->getCondOp()->printAsOperand(O, SlotTracker);
+    }
+    O << ")";
+    break;
+  }
+  }
+}
+
 void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
                               VPSlotTracker &SlotTracker) const {
   O << Indent << "REDUCE ";
@@ -2516,58 +2804,6 @@ void VPReductionEVLRecipe::print(raw_ostream &O, const Twine &Indent,
   O << ")";
 }
 
-void VPExtendedReductionRecipe::print(raw_ostream &O, const Twine &Indent,
-                                      VPSlotTracker &SlotTracker) const {
-  O << Indent << "EXTENDED-REDUCE ";
-  printAsOperand(O, SlotTracker);
-  O << " = ";
-  getChainOp()->printAsOperand(O, SlotTracker);
-  O << " +";
-  O << " reduce."
-    << Instruction::getOpcodeName(
-           RecurrenceDescriptor::getOpcode(getRecurrenceKind()))
-    << " (";
-  getVecOp()->printAsOperand(O, SlotTracker);
-  printFlags(O);
-  O << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType();
-  if (isConditional()) {
-    O << ", ";
-    getCondOp()->printAsOperand(O, SlotTracker);
-  }
-  O << ")";
-}
-
-void VPMulAccumulateReductionRecipe::print(raw_ostream &O, const Twine &Indent,
-                                           VPSlotTracker &SlotTracker) const {
-  O << Indent << "MULACC-REDUCE ";
-  printAsOperand(O, SlotTracker);
-  O << " = ";
-  getChainOp()->printAsOperand(O, SlotTracker);
-  O << " + ";
-  O << "reduce."
-    << Instruction::getOpcodeName(
-           RecurrenceDescriptor::getOpcode(getRecurrenceKind()))
-    << " (";
-  O << "mul";
-  printFlags(O);
-  if (isExtended())
-    O << "(";
-  getVecOp0()->printAsOperand(O, SlotTracker);
-  if (isExtended())
-    O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
-      << "), (";
-  else
-    O << ", ";
-  getVecOp1()->printAsOperand(O, SlotTracker);
-  if (isExtended())
-    O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
-      << ")";
-  if (isConditional()) {
-    O << ", ";
-    getCondOp()->printAsOperand(O, SlotTracker);
-  }
-  O << ")";
-}
 #endif
 
 /// A helper function to scalarize a single Instruction in the innermost loop.
@@ -2630,45 +2866,27 @@ static void scalarizeInstruction(const Instruction *Instr,
 
 void VPReplicateRecipe::execute(VPTransformState &State) {
   Instruction *UI = getUnderlyingInstr();
-  if (State.Lane) { // Generate a single instance.
-    assert((State.VF.isScalar() || !isSingleScalar()) &&
-           "uniform recipe shouldn't be predicated");
-    assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
-    scalarizeInstruction(UI, this, *State.Lane, State);
-    // Insert scalar instance packing it into a vector.
-    if (State.VF.isVector() && shouldPack()) {
-      Value *WideValue;
-      // If we're constructing lane 0, initialize to start from poison.
-      if (State.Lane->isFirstLane()) {
-        assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
-        WideValue = PoisonValue::get(VectorType::get(UI->getType(), State.VF));
-      } else {
-        WideValue = State.get(this);
-      }
-      State.set(this, State.packScalarIntoVectorizedValue(this, WideValue,
-                                                          *State.Lane));
-    }
-    return;
-  }
 
-  if (IsSingleScalar) {
-    // Uniform within VL means we need to generate lane 0.
+  if (!State.Lane) {
+    assert(IsSingleScalar && "VPReplicateRecipes outside replicate regions "
+                             "must have already been unrolled");
     scalarizeInstruction(UI, this, VPLane(0), State);
     return;
   }
 
-  // A store of a loop varying value to a uniform address only needs the last
-  // copy of the store.
-  if (isa<StoreInst>(UI) && vputils::isSingleScalar(getOperand(1))) {
-    auto Lane = VPLane::getLastLaneForVF(State.VF);
-    scalarizeInstruction(UI, this, VPLane(Lane), State);
-    return;
+  assert((State.VF.isScalar() || !isSingleScalar()) &&
+         "uniform recipe shouldn't be predicated");
+  assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
+  scalarizeInstruction(UI, this, *State.Lane, State);
+  // Insert scalar instance packing it into a vector.
+  if (State.VF.isVector() && shouldPack()) {
+    Value *WideValue =
+        State.Lane->isFirstLane()
+            ? PoisonValue::get(VectorType::get(UI->getType(), State.VF))
+            : State.get(this);
+    State.set(this, State.packScalarIntoVectorizedValue(this, WideValue,
+                                                        *State.Lane));
   }
-
-  // Generate scalar instances for all VF lanes.
-  const unsigned EndLane = State.VF.getFixedValue();
-  for (unsigned Lane = 0; Lane < EndLane; ++Lane)
-    scalarizeInstruction(UI, this, VPLane(Lane), State);
 }
 
 bool VPReplicateRecipe::shouldPack() const {
@@ -3219,36 +3437,9 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
   unsigned InterleaveFactor = Group->getFactor();
   auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);
 
-  // TODO: extend the masked interleaved-group support to reversed access.
   VPValue *BlockInMask = getMask();
-  assert((!BlockInMask || !Group->isReverse()) &&
-         "Reversed masked interleave-group not supported.");
-
   VPValue *Addr = getAddr();
   Value *ResAddr = State.get(Addr, VPLane(0));
-  if (auto *I = dyn_cast<Instruction>(ResAddr))
-    State.setDebugLocFrom(I->getDebugLoc());
-
-  // If the group is reverse, adjust the index to refer to the last vector lane
-  // instead of the first. We adjust the index from the first vector lane,
-  // rather than directly getting the pointer for lane VF - 1, because the
-  // pointer operand of the interleaved access is supposed to be uniform.
-  if (Group->isReverse()) {
-    Value *RuntimeVF =
-        getRuntimeVF(State.Builder, State.Builder.getInt32Ty(), State.VF);
-    Value *Index =
-        State.Builder.CreateSub(RuntimeVF, State.Builder.getInt32(1));
-    Index = State.Builder.CreateMul(Index,
-                                    State.Builder.getInt32(Group->getFactor()));
-    Index = State.Builder.CreateNeg(Index);
-
-    bool InBounds = false;
-    if (auto *Gep = dyn_cast<GetElementPtrInst>(ResAddr->stripPointerCasts()))
-      InBounds = Gep->isInBounds();
-    ResAddr = State.Builder.CreateGEP(ScalarTy, ResAddr, Index, "", InBounds);
-  }
-
-  State.setDebugLocFrom(getDebugLoc());
   Value *PoisonVec = PoisonValue::get(VecTy);
 
   auto CreateGroupMask = [&BlockInMask, &State,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ac6be09ef271..90137b72c83f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1140,6 +1140,24 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return;
   }
 
+  // Look through ExtractLastElement (BuildVector ....).
+  if (match(&R, m_VPInstruction<VPInstruction::ExtractLastElement>(
+                    m_BuildVector()))) {
+    auto *BuildVector = cast<VPInstruction>(R.getOperand(0));
+    Def->replaceAllUsesWith(
+        BuildVector->getOperand(BuildVector->getNumOperands() - 1));
+    return;
+  }
+
+  // Look through ExtractPenultimateElement (BuildVector ....).
+  if (match(&R, m_VPInstruction<VPInstruction::ExtractPenultimateElement>(
+                    m_BuildVector()))) {
+    auto *BuildVector = cast<VPInstruction>(R.getOperand(0));
+    Def->replaceAllUsesWith(
+        BuildVector->getOperand(BuildVector->getNumOperands() - 2));
+    return;
+  }
+
   // Some simplifications can only be applied after unrolling. Perform them
   // below.
   if (!Plan->isUnrolled())
@@ -1202,9 +1220,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
     for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
-      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
-      if (!RepR && !isa<VPWidenRecipe>(&R))
+      if (!isa<VPWidenRecipe, VPWidenSelectRecipe, VPReplicateRecipe>(&R))
         continue;
+      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
       if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
         continue;
 
@@ -1717,8 +1735,7 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
     auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
     if (!PhiR)
       continue;
-    const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
-    RecurKind RK = RdxDesc.getRecurrenceKind();
+    RecurKind RK = PhiR->getRecurrenceKind();
     if (RK != RecurKind::Add && RK != RecurKind::Mul)
       continue;
 
@@ -2096,21 +2113,19 @@ void VPlanTransforms::addActiveLaneMask(
     HeaderMask->replaceAllUsesWith(LaneMask);
 }
 
-/// Try to convert \p CurRecipe to a corresponding EVL-based recipe. Returns
-/// nullptr if no EVL-based recipe could be created.
+/// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
+/// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
+/// recipe could be created.
 /// \p HeaderMask  Header Mask.
 /// \p CurRecipe   Recipe to be transform.
 /// \p TypeInfo    VPlan-based type analysis.
 /// \p AllOneMask  The vector mask parameter of vector-predication intrinsics.
 /// \p EVL         The explicit vector length parameter of vector-predication
 /// intrinsics.
-/// \p PrevEVL     The explicit vector length of the previous iteration. Only
-/// required if \p CurRecipe is a VPInstruction::FirstOrderRecurrenceSplice.
-static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
-                                     VPRecipeBase &CurRecipe,
-                                     VPTypeAnalysis &TypeInfo,
-                                     VPValue &AllOneMask, VPValue &EVL,
-                                     VPValue *PrevEVL) {
+static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
+                                       VPRecipeBase &CurRecipe,
+                                       VPTypeAnalysis &TypeInfo,
+                                       VPValue &AllOneMask, VPValue &EVL) {
   using namespace llvm::VPlanPatternMatch;
   auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
     assert(OrigMask && "Unmasked recipe when folding tail");
@@ -2134,26 +2149,7 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask,
         VPValue *NewMask = GetNewMask(Red->getCondOp());
         return new VPReductionEVLRecipe(*Red, EVL, NewMask);
       })
-      .Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
-        SmallVector<VPValue *> Ops(Sel->operands());
-        Ops.push_back(&EVL);
-        return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops,
-                                          TypeInfo.inferScalarType(Sel),
-                                          Sel->getDebugLoc());
-      })
       .Case<VPInstruction>([&](VPInstruction *VPI) -> VPRecipeBase * {
-        if (VPI->getOpcode() == VPInstruction::FirstOrderRecurrenceSplice) {
-          assert(PrevEVL && "Fixed-order recurrences require previous EVL");
-          VPValue *MinusOneVPV = VPI->getParent()->getPlan()->getOrAddLiveIn(
-              ConstantInt::getSigned(Type::getInt32Ty(TypeInfo.getContext()),
-                                     -1));
-          SmallVector<VPValue *> Ops(VPI->operands());
-          Ops.append({MinusOneVPV, &AllOneMask, PrevEVL, &EVL});
-          return new VPWidenIntrinsicRecipe(Intrinsic::experimental_vp_splice,
-                                            Ops, TypeInfo.inferScalarType(VPI),
-                                            VPI->getDebugLoc());
-        }
-
         VPValue *LHS, *RHS;
         // Transform select with a header mask condition
         //   select(header_mask, LHS, RHS)
@@ -2180,9 +2176,18 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
 
+  assert(all_of(Plan.getVF().users(),
+                IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+                        VPWidenIntOrFpInductionRecipe>) &&
+         "User of VF that we can't transform to EVL.");
+  Plan.getVF().replaceAllUsesWith(&EVL);
+
+  // Defer erasing recipes till the end so that we don't invalidate the
+  // VPTypeAnalysis cache.
+  SmallVector<VPRecipeBase *> ToErase;
+
   // Create a scalar phi to track the previous EVL if fixed-order recurrence is
   // contained.
-  VPInstruction *PrevEVL = nullptr;
   bool ContainsFORs =
       any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
   if (ContainsFORs) {
@@ -2195,21 +2200,37 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                                              DebugLoc());
 
     Builder.setInsertPoint(Header, Header->getFirstNonPhi());
-    PrevEVL = Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
-  }
-
-  for (VPUser *U : to_vector(Plan.getVF().users())) {
-    if (auto *R = dyn_cast<VPVectorEndPointerRecipe>(U))
-      R->setOperand(1, &EVL);
+    VPValue *PrevEVL =
+        Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
+
+    for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+             vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+      for (VPRecipeBase &R : *VPBB) {
+        using namespace VPlanPatternMatch;
+        VPValue *V1, *V2;
+        if (!match(&R,
+                   m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
+                       m_VPValue(V1), m_VPValue(V2))))
+          continue;
+        VPValue *Imm = Plan.getOrAddLiveIn(
+            ConstantInt::getSigned(Type::getInt32Ty(Ctx), -1));
+        VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
+            Intrinsic::experimental_vp_splice,
+            {V1, V2, Imm, AllOneMask, PrevEVL, &EVL},
+            TypeInfo.inferScalarType(R.getVPSingleValue()), R.getDebugLoc());
+        VPSplice->insertBefore(&R);
+        R.getVPSingleValue()->replaceAllUsesWith(VPSplice);
+        ToErase.push_back(&R);
+      }
+    }
   }
 
-  SmallVector<VPRecipeBase *> ToErase;
-
+  // Try to optimize header mask recipes away to their EVL variants.
   for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
     for (VPUser *U : collectUsersRecursively(HeaderMask)) {
       auto *CurRecipe = cast<VPRecipeBase>(U);
-      VPRecipeBase *EVLRecipe = createEVLRecipe(
-          HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL, PrevEVL);
+      VPRecipeBase *EVLRecipe =
+          optimizeMaskToEVL(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
       if (!EVLRecipe)
         continue;
 
@@ -2225,8 +2246,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
         VPValue *CurVPV = CurRecipe->getVPSingleValue();
         CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
       }
-      // Defer erasing recipes till the end so that we don't invalidate the
-      // VPTypeAnalysis cache.
       ToErase.push_back(CurRecipe);
     }
   }
@@ -2282,12 +2301,11 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
     VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
   VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
   // The transform updates all users of inductions to work based on EVL, instead
-  // of the VF directly. At the moment, widened inductions cannot be updated, so
-  // bail out if the plan contains any.
-  bool ContainsWidenInductions = any_of(
-      Header->phis(),
-      IsaPred<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>);
-  if (ContainsWidenInductions)
+  // of the VF directly. At the moment, widened pointer inductions cannot be
+  // updated, so bail out if the plan contains any.
+  bool ContainsWidenPointerInductions =
+      any_of(Header->phis(), IsaPred<VPWidenPointerInductionRecipe>);
+  if (ContainsWidenPointerInductions)
     return false;
 
   auto *CanonicalIVPHI = Plan.getCanonicalIV();
@@ -2464,23 +2482,23 @@ void VPlanTransforms::createInterleaveGroups(
     auto *InsertPos =
         cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IRInsertPos));
 
+    bool InBounds = false;
+    if (auto *Gep = dyn_cast<GetElementPtrInst>(
+            getLoadStorePointerOperand(IRInsertPos)->stripPointerCasts()))
+      InBounds = Gep->isInBounds();
+
     // Get or create the start address for the interleave group.
     auto *Start =
         cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getMember(0)));
     VPValue *Addr = Start->getAddr();
     VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
     if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPos)) {
-      // TODO: Hoist Addr's defining recipe (and any operands as needed) to
-      // InsertPos or sink loads above zero members to join it.
-      bool InBounds = false;
-      if (auto *Gep = dyn_cast<GetElementPtrInst>(
-              getLoadStorePointerOperand(IRInsertPos)->stripPointerCasts()))
-        InBounds = Gep->isInBounds();
-
       // We cannot re-use the address of member zero because it does not
       // dominate the insert position. Instead, use the address of the insert
       // position and create a PtrAdd adjusting it to the address of member
       // zero.
+      // TODO: Hoist Addr's defining recipe (and any operands as needed) to
+      // InsertPos or sink loads above zero members to join it.
       assert(IG->getIndex(IRInsertPos) != 0 &&
              "index of insert position shouldn't be zero");
       auto &DL = IRInsertPos->getDataLayout();
@@ -2494,6 +2512,19 @@ void VPlanTransforms::createInterleaveGroups(
       Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
                       : B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
     }
+    // If the group is reverse, adjust the index to refer to the last vector
+    // lane instead of the first. We adjust the index from the first vector
+    // lane, rather than directly getting the pointer for lane VF - 1, because
+    // the pointer operand of the interleaved access is supposed to be uniform.
+    if (IG->isReverse()) {
+      auto *ReversePtr = new VPVectorEndPointerRecipe(
+          Addr, &Plan.getVF(), getLoadStoreType(IRInsertPos),
+          -(int64_t)IG->getFactor(),
+          InBounds ? GEPNoWrapFlags::inBounds() : GEPNoWrapFlags::none(),
+          InsertPos->getDebugLoc());
+      ReversePtr->insertBefore(InsertPos);
+      Addr = ReversePtr;
+    }
     auto *VPIG = new VPInterleaveRecipe(IG, Addr, StoredValues,
                                         InsertPos->getMask(), NeedsMaskForGaps, InsertPos->getDebugLoc());
     VPIG->insertBefore(InsertPos);
@@ -2609,6 +2640,8 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
     Inc = SplatVF;
     Prev = WidenIVR->getLastUnrolledPartOperand();
   } else {
+    if (VPRecipeBase *R = VF->getDefiningRecipe())
+      Builder.setInsertPoint(R->getParent(), std::next(R->getIterator()));
     // Multiply the vectorization factor by the step using integer or
     // floating-point arithmetic as appropriate.
     if (StepTy->isFloatingPointTy())
@@ -2645,82 +2678,6 @@ void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) {
     R->dissolveToCFGLoop();
 }
 
-// Expand VPExtendedReductionRecipe to VPWidenCastRecipe + VPReductionRecipe.
-static void expandVPExtendedReduction(VPExtendedReductionRecipe *ExtRed) {
-  VPWidenCastRecipe *Ext;
-  // Only ZExt contains non-neg flags.
-  if (ExtRed->isZExt())
-    Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
-                                ExtRed->getResultType(), *ExtRed,
-                                ExtRed->getDebugLoc());
-  else
-    Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(),
-                                ExtRed->getResultType(), {},
-                                ExtRed->getDebugLoc());
-
-  auto *Red = new VPReductionRecipe(
-      ExtRed->getRecurrenceKind(), FastMathFlags(), ExtRed->getChainOp(), Ext,
-      ExtRed->getCondOp(), ExtRed->isOrdered(), ExtRed->getDebugLoc());
-  Ext->insertBefore(ExtRed);
-  Red->insertBefore(ExtRed);
-  ExtRed->replaceAllUsesWith(Red);
-  ExtRed->eraseFromParent();
-}
-
-// Expand VPMulAccumulateReductionRecipe to VPWidenRecipe (mul) +
-// VPReductionRecipe (reduce.add)
-// + VPWidenCastRecipe (optional).
-static void
-expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
-  // Generate inner VPWidenCastRecipes if necessary.
-  // Note that we will drop the extend after mul which transforms
-  // reduce.add(ext(mul(ext, ext))) to reduce.add(mul(ext, ext)).
-  VPValue *Op0, *Op1;
-  if (MulAcc->isExtended()) {
-    Type *RedTy = MulAcc->getResultType();
-    if (MulAcc->isZExt())
-      Op0 = new VPWidenCastRecipe(
-          MulAcc->getExtOpcode(), MulAcc->getVecOp0(), RedTy,
-          VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()), MulAcc->getDebugLoc());
-    else
-      Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(),
-                                  RedTy, {}, MulAcc->getDebugLoc());
-    Op0->getDefiningRecipe()->insertBefore(MulAcc);
-    // Prevent reduce.add(mul(ext(A), ext(A))) generate duplicate
-    // VPWidenCastRecipe.
-    if (MulAcc->getVecOp0() == MulAcc->getVecOp1()) {
-      Op1 = Op0;
-    } else {
-      if (MulAcc->isZExt())
-        Op1 = new VPWidenCastRecipe(
-            MulAcc->getExtOpcode(), MulAcc->getVecOp1(), RedTy,
-            VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()),
-            MulAcc->getDebugLoc());
-      else
-        Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(),
-                                    RedTy, {}, MulAcc->getDebugLoc());
-      Op1->getDefiningRecipe()->insertBefore(MulAcc);
-    }
-  } else {
-    // No extends in this MulAccRecipe.
-    Op0 = MulAcc->getVecOp0();
-    Op1 = MulAcc->getVecOp1();
-  }
-
-  std::array<VPValue *, 2> MulOps = {Op0, Op1};
-  auto *Mul = new VPWidenRecipe(Instruction::Mul, ArrayRef(MulOps), *MulAcc,
-                                MulAcc->getDebugLoc());
-  Mul->insertBefore(MulAcc);
-
-  auto *Red = new VPReductionRecipe(
-      MulAcc->getRecurrenceKind(), FastMathFlags(), MulAcc->getChainOp(), Mul,
-      MulAcc->getCondOp(), MulAcc->isOrdered(), MulAcc->getDebugLoc());
-  Red->insertBefore(MulAcc);
-
-  MulAcc->replaceAllUsesWith(Red);
-  MulAcc->eraseFromParent();
-}
-
 void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
                                                Type &CanonicalIVTy) {
   using namespace llvm::VPlanPatternMatch;
@@ -2745,6 +2702,11 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
         continue;
       }
 
+      if (auto *Expr = dyn_cast<VPExpressionRecipe>(&R)) {
+        Expr->decompose();
+        ToRemove.push_back(Expr);
+      }
+
       VPValue *VectorStep;
       VPValue *ScalarStep;
       if (!match(&R, m_VPInstruction<VPInstruction::WideIVStep>(
@@ -2785,14 +2747,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
       VPI->replaceAllUsesWith(VectorStep);
       ToRemove.push_back(VPI);
     }
-    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
-      if (auto *ExtRed = dyn_cast<VPExtendedReductionRecipe>(&R)) {
-        expandVPExtendedReduction(ExtRed);
-        continue;
-      }
-      if (auto *MulAcc = dyn_cast<VPMulAccumulateReductionRecipe>(&R))
-        expandVPMulAccumulateReduction(MulAcc);
-    }
   }
 
   for (VPRecipeBase *R : ToRemove)
@@ -2891,10 +2845,10 @@ void VPlanTransforms::handleUncountableEarlyExit(
 }
 
 /// This function tries convert extended in-loop reductions to
-/// VPExtendedReductionRecipe and clamp the \p Range if it is beneficial and
-/// valid. The created recipe must be lowered to concrete
+/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
+/// valid. The created recipe must be decomposed to its constituent
 /// recipes before execution.
-static VPExtendedReductionRecipe *
+static VPExpressionRecipe *
 tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
                                      VFRange &Range) {
   using namespace VPlanPatternMatch;
@@ -2928,19 +2882,20 @@ tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
           cast<VPWidenCastRecipe>(VecOp)->getOpcode() ==
               Instruction::CastOps::ZExt,
           Ctx.Types.inferScalarType(A)))
-    return new VPExtendedReductionRecipe(Red, cast<VPWidenCastRecipe>(VecOp));
+    return new VPExpressionRecipe(cast<VPWidenCastRecipe>(VecOp), Red);
 
   return nullptr;
 }
 
 /// This function tries convert extended in-loop reductions to
-/// VPMulAccumulateReductionRecipe and clamp the \p Range if it is beneficial
-/// and valid. The created VPExtendedReductionRecipe must be lower to concrete
-/// recipes before execution. Patterns of MulAccumulateReduction:
+/// VPExpressionRecipe and clamp the \p Range if it is beneficial
+/// and valid. The created VPExpressionRecipe must be decomposed to its
+/// constituent recipes before execution. Patterns of the
+/// VPExpressionRecipe:
 ///   reduce.add(mul(...)),
 ///   reduce.add(mul(ext(A), ext(B))),
 ///   reduce.add(ext(mul(ext(A), ext(B)))).
-static VPMulAccumulateReductionRecipe *
+static VPExpressionRecipe *
 tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
                                           VPCostContext &Ctx, VFRange &Range) {
   using namespace VPlanPatternMatch;
@@ -2996,12 +2951,12 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
         match(RecipeB, m_ZExtOrSExt(m_VPValue())) &&
         IsMulAccValidAndClampRange(RecipeA->getOpcode() ==
                                        Instruction::CastOps::ZExt,
-                                   Mul, RecipeA, RecipeB, nullptr))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, RecipeA, RecipeB,
-                                                RecipeA->getResultType());
+                                   Mul, RecipeA, RecipeB, nullptr)) {
+      return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
+    }
     // Match reduce.add(mul).
     if (IsMulAccValidAndClampRange(true, Mul, nullptr, nullptr, nullptr))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, RedTy);
+      return new VPExpressionRecipe(Mul, Red);
   }
   // Match reduce.add(ext(mul(ext(A), ext(B)))).
   // All extend recipes must have same opcode or A == B
@@ -3018,9 +2973,24 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
         Ext0->getOpcode() == Ext1->getOpcode() &&
         IsMulAccValidAndClampRange(Ext0->getOpcode() ==
                                        Instruction::CastOps::ZExt,
-                                   Mul, Ext0, Ext1, Ext))
-      return new VPMulAccumulateReductionRecipe(Red, Mul, Ext0, Ext1,
-                                                Ext->getResultType());
+                                   Mul, Ext0, Ext1, Ext)) {
+      auto *NewExt0 = new VPWidenCastRecipe(
+          Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0,
+          Ext0->getDebugLoc());
+      NewExt0->insertBefore(Ext0);
+
+      VPWidenCastRecipe *NewExt1 = NewExt0;
+      if (Ext0 != Ext1) {
+        NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0),
+                                        Ext->getResultType(), *Ext1,
+                                        Ext1->getDebugLoc());
+        NewExt1->insertBefore(Ext1);
+      }
+      Mul->setOperand(0, NewExt0);
+      Mul->setOperand(1, NewExt1);
+      Red->setOperand(1, Mul);
+      return new VPExpressionRecipe(NewExt0, NewExt1, Mul, Red);
+    }
   }
   return nullptr;
 }
@@ -3030,8 +3000,9 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
 static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
                                                VPCostContext &Ctx,
                                                VFRange &Range) {
-  VPReductionRecipe *AbstractR = nullptr;
-
+  VPExpressionRecipe *AbstractR = nullptr;
+  auto IP = std::next(Red->getIterator());
+  auto *VPBB = Red->getParent();
   if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
     AbstractR = MulAcc;
   else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
@@ -3040,7 +3011,7 @@ static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
   if (!AbstractR)
     return;
 
-  AbstractR->insertBefore(Red);
+  AbstractR->insertBefore(*VPBB, IP);
   Red->replaceAllUsesWith(AbstractR);
 }
 
@@ -3048,7 +3019,7 @@ void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
                                                VFRange &Range) {
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
-    for (VPRecipeBase &R : *VPBB) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       if (auto *Red = dyn_cast<VPReductionRecipe>(&R))
         tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
     }
@@ -3159,6 +3130,9 @@ static bool isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
          GroupSize == VectorRegWidth;
 }
 
+/// Returns true if \p VPValue is a narrow VPValue.
+static bool isAlreadyNarrow(VPValue *VPV) { return VPV->isLiveIn(); }
+
 void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
                                              unsigned VectorRegWidth) {
   using namespace llvm::VPlanPatternMatch;
@@ -3210,6 +3184,16 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
     if (InterleaveR->getStoredValues().empty())
       continue;
 
+    // Narrow interleave groups, if all operands are already matching narrow
+    // ops.
+    auto *Member0 = InterleaveR->getStoredValues()[0];
+    if (isAlreadyNarrow(Member0) &&
+        all_of(InterleaveR->getStoredValues(),
+               [Member0](VPValue *VPV) { return Member0 == VPV; })) {
+      StoreGroups.push_back(InterleaveR);
+      continue;
+    }
+
     // For now, we only support full interleave groups storing load interleave
     // groups.
     if (all_of(enumerate(InterleaveR->getStoredValues()), [](auto Op) {
@@ -3233,7 +3217,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
     if (!WideMember0)
       return;
     for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
-      auto *R = dyn_cast<VPWidenRecipe>(V->getDefiningRecipe());
+      auto *R = dyn_cast_or_null<VPWidenRecipe>(V->getDefiningRecipe());
       if (!R || R->getOpcode() != WideMember0->getOpcode() ||
           R->getNumOperands() > 2)
         return;
@@ -3266,6 +3250,12 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
       return L;
     }
 
+    if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
+      assert(RepR->isSingleScalar() &&
+             isa<LoadInst>(RepR->getUnderlyingInstr()) &&
+             "must be a single scalar load");
+      return RepR;
+    }
     auto *WideLoad = cast<VPWidenLoadRecipe>(R);
 
     // Narrow wide load to uniform scalar load, as transformed VPlan will only
@@ -3280,13 +3270,16 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
   // Narrow operation tree rooted at store groups.
   for (auto *StoreGroup : StoreGroups) {
     VPValue *Res = nullptr;
-    if (auto *WideMember0 = dyn_cast<VPWidenRecipe>(
-            StoreGroup->getStoredValues()[0]->getDefiningRecipe())) {
+    VPValue *Member0 = StoreGroup->getStoredValues()[0];
+    if (isAlreadyNarrow(Member0)) {
+      Res = Member0;
+    } else if (auto *WideMember0 =
+                   dyn_cast<VPWidenRecipe>(Member0->getDefiningRecipe())) {
       for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx)
         WideMember0->setOperand(Idx, NarrowOp(WideMember0->getOperand(Idx)));
       Res = WideMember0;
     } else {
-      Res = NarrowOp(StoreGroup->getStoredValues()[0]);
+      Res = NarrowOp(Member0);
     }
 
     auto *S = new VPWidenStoreRecipe(
@@ -3310,8 +3303,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
 
 /// Add branch weight metadata, if the \p Plan's middle block is terminated by a
 /// BranchOnCond recipe.
-void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
-                                                        ElementCount VF) {
+void VPlanTransforms::addBranchWeightToMiddleTerminator(
+    VPlan &Plan, ElementCount VF, std::optional<unsigned> VScaleForTuning) {
   VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
   auto *MiddleTerm =
       dyn_cast_or_null<VPInstruction>(MiddleVPBB->getTerminator());
@@ -3323,6 +3316,8 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(VPlan &Plan,
          "must have a BranchOnCond");
   // Assume that `TripCount % VectorStep ` is equally distributed.
   unsigned VectorStep = Plan.getUF() * VF.getKnownMinValue();
+  if (VF.isScalable() && VScaleForTuning.has_value())
+    VectorStep *= *VScaleForTuning;
   assert(VectorStep > 0 && "trip count should not be zero");
   MDBuilder MDB(Plan.getScalarHeader()->getIRBasicBlock()->getContext());
   MDNode *BranchWeights =
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 7e51c05d1b5b..8d2eded45da2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -99,6 +99,12 @@ struct VPlanTransforms {
   /// Explicitly unroll \p Plan by \p UF.
   static void unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx);
 
+  /// Replace each VPReplicateRecipe outside on any replicate region in \p Plan
+  /// with \p VF single-scalar recipes.
+  /// TODO: Also replicate VPReplicateRecipes inside replicate regions, thereby
+  /// dissolving the latter.
+  static void replicateByVF(VPlan &Plan, ElementCount VF);
+
   /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the
   /// resulting plan to \p BestVF and \p BestUF.
   static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
@@ -232,7 +238,9 @@ struct VPlanTransforms {
 
   /// Add branch weight metadata, if the \p Plan's middle block is terminated by
   /// a BranchOnCond recipe.
-  static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF);
+  static void
+  addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
+                                    std::optional<unsigned> VScaleForTuning);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 0bc683e557e7..2dd43c092ff7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -15,6 +15,7 @@
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
 #include "VPlanCFG.h"
+#include "VPlanHelpers.h"
 #include "VPlanPatternMatch.h"
 #include "VPlanTransforms.h"
 #include "VPlanUtils.h"
@@ -344,14 +345,17 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
     if (ToSkip.contains(&R) || isa<VPIRInstruction>(&R))
       continue;
 
-    // Add all VPValues for all parts to ComputeReductionResult which combines
-    // the parts to compute the final reduction value.
+    // Add all VPValues for all parts to AnyOf, FirstActiveLaneMask and
+    // Compute*Result which combine all parts to compute the final value.
     VPValue *Op1;
-    if (match(&R, m_VPInstruction<VPInstruction::ComputeAnyOfResult>(
+    if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||
+        match(&R, m_VPInstruction<VPInstruction::FirstActiveLane>(
+                      m_VPValue(Op1))) ||
+        match(&R, m_VPInstruction<VPInstruction::ComputeAnyOfResult>(
                       m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
         match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
                       m_VPValue(), m_VPValue(Op1))) ||
-        match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
+        match(&R, m_VPInstruction<VPInstruction::ComputeFindIVResult>(
                       m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
       addUniformForAllParts(cast<VPInstruction>(&R));
       for (unsigned Part = 1; Part != UF; ++Part)
@@ -450,3 +454,95 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) {
 
   VPlanTransforms::removeDeadRecipes(Plan);
 }
+
+/// Create a single-scalar clone of \p RepR for lane \p Lane.
+static VPReplicateRecipe *cloneForLane(VPlan &Plan, VPBuilder &Builder,
+                                       Type *IdxTy, VPReplicateRecipe *RepR,
+                                       VPLane Lane) {
+  // Collect the operands at Lane, creating extracts as needed.
+  SmallVector<VPValue *> NewOps;
+  for (VPValue *Op : RepR->operands()) {
+    if (vputils::isSingleScalar(Op)) {
+      NewOps.push_back(Op);
+      continue;
+    }
+    if (Lane.getKind() == VPLane::Kind::ScalableLast) {
+      NewOps.push_back(
+          Builder.createNaryOp(VPInstruction::ExtractLastElement, {Op}));
+      continue;
+    }
+    // Look through buildvector to avoid unnecessary extracts.
+    if (match(Op, m_BuildVector())) {
+      NewOps.push_back(
+          cast<VPInstruction>(Op)->getOperand(Lane.getKnownLane()));
+      continue;
+    }
+    VPValue *Idx =
+        Plan.getOrAddLiveIn(ConstantInt::get(IdxTy, Lane.getKnownLane()));
+    VPValue *Ext = Builder.createNaryOp(Instruction::ExtractElement, {Op, Idx});
+    NewOps.push_back(Ext);
+  }
+
+  auto *New =
+      new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps,
+                            /*IsSingleScalar=*/true, /*Mask=*/nullptr, *RepR);
+  New->insertBefore(RepR);
+  return New;
+}
+
+void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
+  Type *IdxTy = IntegerType::get(
+      Plan.getScalarHeader()->getIRBasicBlock()->getContext(), 32);
+
+  // Visit all VPBBs outside the loop region and directly inside the top-level
+  // loop region.
+  auto VPBBsOutsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(Plan.getEntry()));
+  auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()));
+  auto VPBBsToUnroll =
+      concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion);
+  for (VPBasicBlock *VPBB : VPBBsToUnroll) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+      if (!RepR || RepR->isSingleScalar())
+        continue;
+
+      VPBuilder Builder(RepR);
+      if (RepR->getNumUsers() == 0) {
+        if (isa<StoreInst>(RepR->getUnderlyingInstr()) &&
+            vputils::isSingleScalar(RepR->getOperand(1))) {
+          // Stores to invariant addresses need to store the last lane only.
+          cloneForLane(Plan, Builder, IdxTy, RepR,
+                       VPLane::getLastLaneForVF(VF));
+        } else {
+          // Create single-scalar version of RepR for all lanes.
+          for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)
+            cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I));
+        }
+        RepR->eraseFromParent();
+        continue;
+      }
+      /// Create single-scalar version of RepR for all lanes.
+      SmallVector<VPValue *> LaneDefs;
+      for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)
+        LaneDefs.push_back(cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I)));
+
+      /// Users that only demand the first lane can use the definition for lane
+      /// 0.
+      RepR->replaceUsesWithIf(LaneDefs[0], [RepR](VPUser &U, unsigned) {
+        return U.onlyFirstLaneUsed(RepR);
+      });
+
+      // If needed, create a Build(Struct)Vector recipe to insert the scalar
+      // lane values into a vector.
+      Type *ResTy = RepR->getUnderlyingInstr()->getType();
+      VPValue *VecRes = Builder.createNaryOp(
+          ResTy->isStructTy() ? VPInstruction::BuildStructVector
+                              : VPInstruction::BuildVector,
+          LaneDefs);
+      RepR->replaceAllUsesWith(VecRes);
+      RepR->eraseFromParent();
+    }
+  }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 8a4e6fda89c1..8dcd57f1b359 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -69,7 +69,8 @@ inline bool isSingleScalar(const VPValue *VPV) {
     return Rep->isSingleScalar() || (PreservesUniformity(Rep->getOpcode()) &&
                                      all_of(Rep->operands(), isSingleScalar));
   }
-  if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe>(VPV))
+  if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe,
+          VPWidenSelectRecipe>(VPV))
     return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
   if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
     return PreservesUniformity(WidenR->getOpcode()) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a0d3dc9b934c..279cdac92d2d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -49,6 +49,7 @@ class VPValue {
   friend struct VPDoubleValueDef;
   friend class VPInterleaveRecipe;
   friend class VPlan;
+  friend class VPExpressionRecipe;
 
   const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
 
@@ -330,13 +331,12 @@ class VPDef {
     VPBranchOnMaskSC,
     VPDerivedIVSC,
     VPExpandSCEVSC,
+    VPExpressionSC,
     VPIRInstructionSC,
     VPInstructionSC,
     VPInterleaveSC,
     VPReductionEVLSC,
     VPReductionSC,
-    VPMulAccumulateReductionSC,
-    VPExtendedReductionSC,
     VPPartialReductionSC,
     VPReplicateSC,
     VPScalarIVStepsSC,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index fba4a68f4a27..38ada33d7ee1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -156,8 +156,16 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
         .Case<VPWidenIntrinsicRecipe>([&](const VPWidenIntrinsicRecipe *S) {
           return VerifyEVLUse(*S, S->getNumOperands() - 1);
         })
-        .Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe>(
+        .Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe,
+              VPWidenIntOrFpInductionRecipe>(
             [&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
+        .Case<VPScalarIVStepsRecipe>([&](auto *R) {
+          if (R->getNumOperands() != 3) {
+            errs() << "Unrolling with EVL tail folding not yet supported\n";
+            return false;
+          }
+          return VerifyEVLUse(*R, 2);
+        })
         .Case<VPWidenLoadEVLRecipe, VPVectorEndPointerRecipe>(
             [&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); })
         .Case<VPInstructionWithType>(
@@ -165,13 +173,27 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
         .Case<VPInstruction>([&](const VPInstruction *I) {
           if (I->getOpcode() == Instruction::PHI)
             return VerifyEVLUse(*I, 1);
-          if (I->getOpcode() != Instruction::Add) {
-            errs() << "EVL is used as an operand in non-VPInstruction::Add\n";
+          switch (I->getOpcode()) {
+          case Instruction::Add:
+            break;
+          case Instruction::UIToFP:
+          case Instruction::Trunc:
+          case Instruction::ZExt:
+          case Instruction::Mul:
+          case Instruction::FMul:
+            // Opcodes above can only use EVL after wide inductions have been
+            // expanded.
+            if (!VerifyLate) {
+              errs() << "EVL used by unexpected VPInstruction\n";
+              return false;
+            }
+            break;
+          default:
+            errs() << "EVL used by unexpected VPInstruction\n";
             return false;
           }
           if (I->getNumUsers() != 1) {
-            errs() << "EVL is used in VPInstruction:Add with multiple "
-                      "users\n";
+            errs() << "EVL is used in VPInstruction with multiple users\n";
             return false;
           }
           if (!VerifyLate && !isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 3e459f5ea4ce..fe8d74c43dfd 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -19,9 +19,10 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
-#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/TargetFolder.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
@@ -72,14 +73,15 @@ class VectorCombine {
                 const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
                 const DataLayout *DL, TTI::TargetCostKind CostKind,
                 bool TryEarlyFoldsOnly)
-      : F(F), Builder(F.getContext()), TTI(TTI), DT(DT), AA(AA), AC(AC), DL(DL),
-        CostKind(CostKind), TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
+      : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
+        DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind),
+        TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
 
   bool run();
 
 private:
   Function &F;
-  IRBuilder<> Builder;
+  IRBuilder<InstSimplifyFolder> Builder;
   const TargetTransformInfo &TTI;
   const DominatorTree &DT;
   AAResults &AA;
@@ -113,6 +115,7 @@ class VectorCombine {
   bool foldInsExtFNeg(Instruction &I);
   bool foldInsExtBinop(Instruction &I);
   bool foldInsExtVectorToShuffle(Instruction &I);
+  bool foldBitOpOfBitcasts(Instruction &I);
   bool foldBitcastShuffle(Instruction &I);
   bool scalarizeOpOrCmp(Instruction &I);
   bool scalarizeVPIntrinsic(Instruction &I);
@@ -120,6 +123,7 @@ class VectorCombine {
   bool foldBinopOfReductions(Instruction &I);
   bool foldSingleElementStore(Instruction &I);
   bool scalarizeLoadExtract(Instruction &I);
+  bool scalarizeExtExtract(Instruction &I);
   bool foldConcatOfBoolMasks(Instruction &I);
   bool foldPermuteOfBinops(Instruction &I);
   bool foldShuffleOfBinops(Instruction &I);
@@ -528,7 +532,7 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
 /// Create a shuffle that translates (shifts) 1 element from the input vector
 /// to a new element location.
 static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
-                                 unsigned NewIndex, IRBuilder<> &Builder) {
+                                 unsigned NewIndex, IRBuilderBase &Builder) {
   // The shuffle mask is poison except for 1 lane that is being translated
   // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
   // ShufMask = { 2, poison, poison, poison }
@@ -544,7 +548,7 @@ static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
 /// unnecessary instructions.
 static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
                                             unsigned NewIndex,
-                                            IRBuilder<> &Builder) {
+                                            IRBuilderBase &Builder) {
   // Shufflevectors can only be created for fixed-width vectors.
   Value *X = ExtElt->getVectorOperand();
   if (!isa<FixedVectorType>(X->getType()))
@@ -559,7 +563,8 @@ static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
 
   Value *Shuf = createShiftShuffle(X, cast<ConstantInt>(C)->getZExtValue(),
                                    NewIndex, Builder);
-  return cast<ExtractElementInst>(Builder.CreateExtractElement(Shuf, NewIndex));
+  return dyn_cast<ExtractElementInst>(
+      Builder.CreateExtractElement(Shuf, NewIndex));
 }
 
 /// Try to reduce extract element costs by converting scalar compares to vector
@@ -803,6 +808,66 @@ bool VectorCombine::foldInsExtBinop(Instruction &I) {
   return true;
 }
 
+bool VectorCombine::foldBitOpOfBitcasts(Instruction &I) {
+  // Match: bitop(bitcast(x), bitcast(y)) -> bitcast(bitop(x, y))
+  Value *LHSSrc, *RHSSrc;
+  if (!match(&I, m_BitwiseLogic(m_BitCast(m_Value(LHSSrc)),
+                                m_BitCast(m_Value(RHSSrc)))))
+    return false;
+
+  // Source types must match
+  if (LHSSrc->getType() != RHSSrc->getType())
+    return false;
+  if (!LHSSrc->getType()->getScalarType()->isIntegerTy())
+    return false;
+
+  // Only handle vector types
+  auto *SrcVecTy = dyn_cast<FixedVectorType>(LHSSrc->getType());
+  auto *DstVecTy = dyn_cast<FixedVectorType>(I.getType());
+  if (!SrcVecTy || !DstVecTy)
+    return false;
+
+  // Same total bit width
+  assert(SrcVecTy->getPrimitiveSizeInBits() ==
+             DstVecTy->getPrimitiveSizeInBits() &&
+         "Bitcast should preserve total bit width");
+
+  // Cost Check :
+  // OldCost = bitlogic + 2*bitcasts
+  // NewCost = bitlogic + bitcast
+  auto *BinOp = cast<BinaryOperator>(&I);
+  InstructionCost OldCost =
+      TTI.getArithmeticInstrCost(BinOp->getOpcode(), DstVecTy) +
+      TTI.getCastInstrCost(Instruction::BitCast, DstVecTy, LHSSrc->getType(),
+                           TTI::CastContextHint::None) +
+      TTI.getCastInstrCost(Instruction::BitCast, DstVecTy, RHSSrc->getType(),
+                           TTI::CastContextHint::None);
+  InstructionCost NewCost =
+      TTI.getArithmeticInstrCost(BinOp->getOpcode(), SrcVecTy) +
+      TTI.getCastInstrCost(Instruction::BitCast, DstVecTy, SrcVecTy,
+                           TTI::CastContextHint::None);
+
+  LLVM_DEBUG(dbgs() << "Found a bitwise logic op of bitcasted values: " << I
+                    << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
+                    << "\n");
+
+  if (NewCost > OldCost)
+    return false;
+
+  // Create the operation on the source type
+  Value *NewOp = Builder.CreateBinOp(BinOp->getOpcode(), LHSSrc, RHSSrc,
+                                     BinOp->getName() + ".inner");
+  if (auto *NewBinOp = dyn_cast<BinaryOperator>(NewOp))
+    NewBinOp->copyIRFlags(BinOp);
+
+  Worklist.pushValue(NewOp);
+
+  // Bitcast the result back
+  Value *Result = Builder.CreateBitCast(NewOp, I.getType());
+  replaceValue(I, *Result);
+  return true;
+}
+
 /// If this is a bitcast of a shuffle, try to bitcast the source vector to the
 /// destination type followed by shuffle. This can enable further transforms by
 /// moving bitcasts or shuffles together.
@@ -1030,12 +1095,14 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
     return false;
 
   // TODO: Allow intrinsics with different argument types
-  // TODO: Allow intrinsics with scalar arguments
-  if (II && (!isTriviallyVectorizable(II->getIntrinsicID()) ||
-             !all_of(II->args(), [&II](Value *Arg) {
-               return Arg->getType() == II->getType();
-             })))
-    return false;
+  if (II) {
+    if (!isTriviallyVectorizable(II->getIntrinsicID()))
+      return false;
+    for (auto [Idx, Arg] : enumerate(II->args()))
+      if (Arg->getType() != II->getType() &&
+          !isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(), Idx, &TTI))
+        return false;
+  }
 
   // Do not convert the vector condition of a vector select into a scalar
   // condition. That may cause problems for codegen because of differences in
@@ -1048,19 +1115,18 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
 
   // Match constant vectors or scalars being inserted into constant vectors:
   // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
-  SmallVector<Constant *> VecCs;
-  SmallVector<Value *> ScalarOps;
+  SmallVector<Value *> VecCs, ScalarOps;
   std::optional<uint64_t> Index;
 
   auto Ops = II ? II->args() : I.operands();
-  for (Value *Op : Ops) {
+  for (auto [OpNum, Op] : enumerate(Ops)) {
     Constant *VecC;
     Value *V;
     uint64_t InsIdx = 0;
-    VectorType *OpTy = cast<VectorType>(Op->getType());
-    if (match(Op, m_InsertElt(m_Constant(VecC), m_Value(V),
-                              m_ConstantInt(InsIdx)))) {
+    if (match(Op.get(), m_InsertElt(m_Constant(VecC), m_Value(V),
+                                    m_ConstantInt(InsIdx)))) {
       // Bail if any inserts are out of bounds.
+      VectorType *OpTy = cast<VectorType>(Op->getType());
       if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
         return false;
       // All inserts must have the same index.
@@ -1071,7 +1137,11 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
         return false;
       VecCs.push_back(VecC);
       ScalarOps.push_back(V);
-    } else if (match(Op, m_Constant(VecC))) {
+    } else if (II && isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),
+                                                        OpNum, &TTI)) {
+      VecCs.push_back(Op.get());
+      ScalarOps.push_back(Op.get());
+    } else if (match(Op.get(), m_Constant(VecC))) {
       VecCs.push_back(VecC);
       ScalarOps.push_back(nullptr);
     } else {
@@ -1115,16 +1185,17 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   // Fold the vector constants in the original vectors into a new base vector to
   // get more accurate cost modelling.
   Value *NewVecC = nullptr;
+  TargetFolder Folder(*DL);
   if (CI)
-    NewVecC = ConstantFoldCompareInstOperands(CI->getPredicate(), VecCs[0],
-                                              VecCs[1], *DL);
+    NewVecC = Folder.FoldCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
   else if (UO)
-    NewVecC = ConstantFoldUnaryOpOperand(Opcode, VecCs[0], *DL);
+    NewVecC =
+        Folder.FoldUnOpFMF(UO->getOpcode(), VecCs[0], UO->getFastMathFlags());
   else if (BO)
-    NewVecC = ConstantFoldBinaryOpOperands(Opcode, VecCs[0], VecCs[1], *DL);
+    NewVecC = Folder.FoldBinOp(BO->getOpcode(), VecCs[0], VecCs[1]);
   else if (II->arg_size() == 2)
-    NewVecC = ConstantFoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0],
-                                          VecCs[1], II->getType(), II);
+    NewVecC = Folder.FoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0],
+                                         VecCs[1], II->getType(), &I);
 
   // Get cost estimate for the insert element. This cost will factor into
   // both sequences.
@@ -1132,8 +1203,9 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   InstructionCost NewCost =
       ScalarOpCost + TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
                                             CostKind, *Index, NewVecC);
-  for (auto [Op, VecC, Scalar] : zip(Ops, VecCs, ScalarOps)) {
-    if (!Scalar)
+  for (auto [Idx, Op, VecC, Scalar] : enumerate(Ops, VecCs, ScalarOps)) {
+    if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
+                              II->getIntrinsicID(), Idx, &TTI)))
       continue;
     InstructionCost InsertCost = TTI.getVectorInstrCost(
         Instruction::InsertElement, VecTy, CostKind, *Index, VecC, Scalar);
@@ -1177,16 +1249,12 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
 
   // Create a new base vector if the constant folding failed.
   if (!NewVecC) {
-    SmallVector<Value *> VecCValues;
-    VecCValues.reserve(VecCs.size());
-    append_range(VecCValues, VecCs);
     if (CI)
       NewVecC = Builder.CreateCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
     else if (UO || BO)
-      NewVecC = Builder.CreateNAryOp(Opcode, VecCValues);
+      NewVecC = Builder.CreateNAryOp(Opcode, VecCs);
     else
-      NewVecC =
-          Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCValues);
+      NewVecC = Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCs);
   }
   Value *Insert = Builder.CreateInsertElement(NewVecC, Scalar, *Index);
   replaceValue(I, *Insert);
@@ -1398,10 +1466,12 @@ bool VectorCombine::foldBinopOfReductions(Instruction &I) {
   LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
                     << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
                     << "\n");
-  Value *VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
-  if (auto *PDInst = dyn_cast<PossiblyDisjointInst>(&I))
-    if (auto *PDVectorBO = dyn_cast<PossiblyDisjointInst>(VectorBO))
-      PDVectorBO->setIsDisjoint(PDInst->isDisjoint());
+  Value *VectorBO;
+  if (BinOpOpc == Instruction::Or)
+    VectorBO = Builder.CreateOr(V0, V1, "",
+                                cast<PossiblyDisjointInst>(I).isDisjoint());
+  else
+    VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
 
   Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
   replaceValue(I, *Rdx);
@@ -1458,7 +1528,7 @@ class ScalarizationResult {
   }
 
   /// Freeze the ToFreeze and update the use in \p User to use it.
-  void freeze(IRBuilder<> &Builder, Instruction &UserI) {
+  void freeze(IRBuilderBase &Builder, Instruction &UserI) {
     assert(isSafeWithFreeze() &&
            "should only be used when freezing is required");
     assert(is_contained(ToFreeze->users(), &UserI) &&
@@ -1709,6 +1779,73 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
   return true;
 }
 
+bool VectorCombine::scalarizeExtExtract(Instruction &I) {
+  auto *Ext = dyn_cast<ZExtInst>(&I);
+  if (!Ext)
+    return false;
+
+  // Try to convert a vector zext feeding only extracts to a set of scalar
+  //   (Src << ExtIdx *Size) & (Size -1)
+  // if profitable   .
+  auto *SrcTy = dyn_cast<FixedVectorType>(Ext->getOperand(0)->getType());
+  if (!SrcTy)
+    return false;
+  auto *DstTy = cast<FixedVectorType>(Ext->getType());
+
+  Type *ScalarDstTy = DstTy->getElementType();
+  if (DL->getTypeSizeInBits(SrcTy) != DL->getTypeSizeInBits(ScalarDstTy))
+    return false;
+
+  InstructionCost VectorCost =
+      TTI.getCastInstrCost(Instruction::ZExt, DstTy, SrcTy,
+                           TTI::CastContextHint::None, CostKind, Ext);
+  unsigned ExtCnt = 0;
+  bool ExtLane0 = false;
+  for (User *U : Ext->users()) {
+    const APInt *Idx;
+    if (!match(U, m_ExtractElt(m_Value(), m_APInt(Idx))))
+      return false;
+    if (cast<Instruction>(U)->use_empty())
+      continue;
+    ExtCnt += 1;
+    ExtLane0 |= Idx->isZero();
+    VectorCost += TTI.getVectorInstrCost(Instruction::ExtractElement, DstTy,
+                                         CostKind, Idx->getZExtValue(), U);
+  }
+
+  InstructionCost ScalarCost =
+      ExtCnt * TTI.getArithmeticInstrCost(
+                   Instruction::And, ScalarDstTy, CostKind,
+                   {TTI::OK_AnyValue, TTI::OP_None},
+                   {TTI::OK_NonUniformConstantValue, TTI::OP_None}) +
+      (ExtCnt - ExtLane0) *
+          TTI.getArithmeticInstrCost(
+              Instruction::LShr, ScalarDstTy, CostKind,
+              {TTI::OK_AnyValue, TTI::OP_None},
+              {TTI::OK_NonUniformConstantValue, TTI::OP_None});
+  if (ScalarCost > VectorCost)
+    return false;
+
+  Value *ScalarV = Ext->getOperand(0);
+  if (!isGuaranteedNotToBePoison(ScalarV, &AC, dyn_cast<Instruction>(ScalarV),
+                                 &DT))
+    ScalarV = Builder.CreateFreeze(ScalarV);
+  ScalarV = Builder.CreateBitCast(
+      ScalarV,
+      IntegerType::get(SrcTy->getContext(), DL->getTypeSizeInBits(SrcTy)));
+  uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(SrcTy->getElementType());
+  uint64_t EltBitMask = (1ull << SrcEltSizeInBits) - 1;
+  for (User *U : Ext->users()) {
+    auto *Extract = cast<ExtractElementInst>(U);
+    uint64_t Idx =
+        cast<ConstantInt>(Extract->getIndexOperand())->getZExtValue();
+    Value *LShr = Builder.CreateLShr(ScalarV, Idx * SrcEltSizeInBits);
+    Value *And = Builder.CreateAnd(LShr, EltBitMask);
+    U->replaceAllUsesWith(And);
+  }
+  return true;
+}
+
 /// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
 /// to "(bitcast (concat X, Y))"
 /// where X/Y are bitcasted from i1 mask vectors.
@@ -2556,7 +2693,7 @@ static Value *generateNewInstTree(ArrayRef<InstLane> Item, FixedVectorType *Ty,
                                   const SmallPtrSet<Use *, 4> &IdentityLeafs,
                                   const SmallPtrSet<Use *, 4> &SplatLeafs,
                                   const SmallPtrSet<Use *, 4> &ConcatLeafs,
-                                  IRBuilder<> &Builder,
+                                  IRBuilderBase &Builder,
                                   const TargetTransformInfo *TTI) {
   auto [FrontU, FrontLane] = Item.front();
 
@@ -2914,17 +3051,20 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
                     << "\n");
   LLVM_DEBUG(dbgs() << "  OldCost: " << OldCost << " vs NewCost: " << NewCost
                     << "\n");
+  bool MadeChanges = false;
   if (NewCost < OldCost) {
     Builder.SetInsertPoint(Shuffle);
     Value *NewShuffle = Builder.CreateShuffleVector(
         Shuffle->getOperand(0), Shuffle->getOperand(1), ConcatMask);
     LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
     replaceValue(*Shuffle, *NewShuffle);
+    MadeChanges = true;
   }
 
   // See if we can re-use foldSelectShuffle, getting it to reduce the size of
   // the shuffle into a nicer order, as it can ignore the order of the shuffles.
-  return foldSelectShuffle(*Shuffle, true);
+  MadeChanges |= foldSelectShuffle(*Shuffle, true);
+  return MadeChanges;
 }
 
 /// Determine if its more efficient to fold:
@@ -3082,10 +3222,10 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
       auto *SSV = cast<ShuffleVectorInst>(SVOp0);
       SVOp0 = SSV->getOperand(0);
       SVOp1 = SSV->getOperand(1);
-      for (unsigned I = 0, E = Mask.size(); I != E; I++) {
-        if (Mask[I] >= static_cast<int>(SSV->getShuffleMask().size()))
+      for (int &Elem : Mask) {
+        if (Elem >= static_cast<int>(SSV->getShuffleMask().size()))
           return false;
-        Mask[I] = Mask[I] < 0 ? Mask[I] : SSV->getMaskValue(Mask[I]);
+        Elem = Elem < 0 ? Elem : SSV->getMaskValue(Elem);
       }
     }
     if (SVOp0 == Op1 && SVOp1 == Op0) {
@@ -3594,6 +3734,7 @@ bool VectorCombine::run() {
     if (IsVectorType) {
       MadeChange |= scalarizeOpOrCmp(I);
       MadeChange |= scalarizeLoadExtract(I);
+      MadeChange |= scalarizeExtExtract(I);
       MadeChange |= scalarizeVPIntrinsic(I);
       MadeChange |= foldInterleaveIntrinsics(I);
     }
@@ -3629,6 +3770,11 @@ bool VectorCombine::run() {
       case Instruction::BitCast:
         MadeChange |= foldBitcastShuffle(I);
         break;
+      case Instruction::And:
+      case Instruction::Or:
+      case Instruction::Xor:
+        MadeChange |= foldBitOpOfBitcasts(I);
+        break;
       default:
         MadeChange |= shrinkType(I);
         break;
diff --git a/llvm/lib/XRay/FDRTraceWriter.cpp b/llvm/lib/XRay/FDRTraceWriter.cpp
index 8e67a8f98433..fb591252ffa3 100644
--- a/llvm/lib/XRay/FDRTraceWriter.cpp
+++ b/llvm/lib/XRay/FDRTraceWriter.cpp
@@ -96,7 +96,7 @@ Error FDRTraceWriter::visit(CustomEventRecord &R) {
   if (auto E = writeMetadata<5u>(OS, R.size(), R.tsc(), R.cpu()))
     return E;
   auto D = R.data();
-  ArrayRef<char> Bytes(D.data(), D.size());
+  ArrayRef<char> Bytes(D);
   OS.write(Bytes);
   return Error::success();
 }
@@ -105,7 +105,7 @@ Error FDRTraceWriter::visit(CustomEventRecordV5 &R) {
   if (auto E = writeMetadata<5u>(OS, R.size(), R.delta()))
     return E;
   auto D = R.data();
-  ArrayRef<char> Bytes(D.data(), D.size());
+  ArrayRef<char> Bytes(D);
   OS.write(Bytes);
   return Error::success();
 }
@@ -114,7 +114,7 @@ Error FDRTraceWriter::visit(TypedEventRecord &R) {
   if (auto E = writeMetadata<8u>(OS, R.size(), R.delta(), R.eventType()))
     return E;
   auto D = R.data();
-  ArrayRef<char> Bytes(D.data(), D.size());
+  ArrayRef<char> Bytes(D);
   OS.write(Bytes);
   return Error::success();
 }
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 9d12634a3a47..894b4a68915e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -510,6 +510,13 @@ if(build_runtimes)
   endif()
 
   # TODO: We need to consider passing it as '-DRUNTIMES_x86_64_LLVM_ENABLE_RUNTIMES'.
+  if("libclc" IN_LIST LLVM_ENABLE_RUNTIMES)
+    foreach(dep clang llvm-as llvm-link opt)
+      if(TARGET ${dep})
+        list(APPEND extra_deps ${dep})
+      endif()
+    endforeach()
+  endif()
   if("openmp" IN_LIST LLVM_ENABLE_RUNTIMES OR "offload" IN_LIST LLVM_ENABLE_RUNTIMES)
     # With ROCm 6.3 the ROCr runtime and the thunk layer share a single repository.
     # No need to provide a separate path for ROCt.
diff --git a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
index 43e7be2ee20e..ac5e0db00111 100644
--- a/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
+++ b/llvm/test/Analysis/BasicAA/cs-cs-arm.ll
@@ -24,7 +24,7 @@ entry:
 ; CHECK: Just Ref:  Ptr: i8* %p        <->  %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %q        <->  %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %p        <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
-; CHECK: Both ModRef:  Ptr: i8* %q     <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
+; CHECK: Just Mod:  Ptr: i8* %q     <->  call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
 ; CHECK: Just Ref:  Ptr: i8* %p        <->  %b = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:  Ptr: i8* %q        <->  %b = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16)
 ; CHECK: NoModRef:   %a = call <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr %p, i32 16) #{{[0-9]+}} <->   call void @llvm.arm.neon.vst1.p0.v8i16(ptr %q, <8 x i16> %y, i32 16)
diff --git a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
index 7772cca2777f..b7c64f18cdfa 100644
--- a/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
+++ b/llvm/test/Analysis/BasicAA/intrinsics-arm.ll
@@ -27,5 +27,5 @@ declare <8 x i16> @llvm.arm.neon.vld1.v8i16.p0(ptr, i32) nounwind readonly
 declare void @llvm.arm.neon.vst1.p0.v8i16(ptr, <8 x i16>, i32) nounwind
 
 ; CHECK: attributes #0 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
-; CHECK: attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK: attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
 ; CHECK: attributes [[ATTR]] = { nounwind }
diff --git a/llvm/test/Analysis/CostModel/AArch64/ldexp.ll b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
new file mode 100644
index 000000000000..8cc865746893
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/ldexp.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @ldexp() {
+; CHECK-BASE-LABEL: 'ldexp'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %1 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %2 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:46 CodeSize:7 Lat:46 SizeLat:46 for: %3 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:92 CodeSize:14 Lat:92 SizeLat:92 for: %4 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:184 CodeSize:28 Lat:184 SizeLat:184 for: %5 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %6 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %7 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:6 Lat:44 SizeLat:44 for: %8 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:88 CodeSize:12 Lat:88 SizeLat:88 for: %9 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:176 CodeSize:24 Lat:176 SizeLat:176 for: %10 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:22 Lat:28 SizeLat:28 for: %4 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:56 CodeSize:44 Lat:56 SizeLat:56 for: %5 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %6 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %7 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:10 Lat:12 SizeLat:12 for: %8 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:20 Lat:24 SizeLat:24 for: %9 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:40 Lat:48 SizeLat:48 for: %10 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+  call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_fp16() {
+; CHECK-BASE-LABEL: 'ldexp_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:3 Lat:4 SizeLat:4 for: %2 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:7 Lat:10 SizeLat:10 for: %3 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:15 Lat:22 SizeLat:22 for: %4 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:44 CodeSize:30 Lat:44 SizeLat:44 for: %5 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp_fp16'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:23 Lat:30 SizeLat:30 for: %4 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:46 Lat:60 SizeLat:60 for: %5 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call half @llvm.ldexp.f16.i32(half poison, i32 poison)
+  call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> poison, <2 x i32> poison)
+  call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> poison, <4 x i32> poison)
+  call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> poison, <8 x i32> poison)
+  call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> poison, <16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_bf16() {
+; CHECK-BASE-LABEL: 'ldexp_bf16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:22 CodeSize:3 Lat:22 SizeLat:22 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:46 CodeSize:7 Lat:46 SizeLat:46 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:94 CodeSize:15 Lat:94 SizeLat:94 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:188 CodeSize:30 Lat:188 SizeLat:188 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-SVE-LABEL: 'ldexp_bf16'
+; CHECK-SVE-NEXT:  Cost Model: Found costs of 2 for: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:5 Lat:6 SizeLat:6 for: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:14 SizeLat:14 for: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:23 Lat:30 SizeLat:30 for: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:46 Lat:60 SizeLat:60 for: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+  call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+  call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+  call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+  call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+  ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AArch64/lrint.ll b/llvm/test/Analysis/CostModel/AArch64/lrint.ll
new file mode 100644
index 000000000000..70fb60adee89
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/lrint.ll
@@ -0,0 +1,254 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 %s | FileCheck --check-prefixes=CHECK,CHECK-NOFP16 %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 -mattr=+fullfp16 %s | FileCheck --check-prefixes=CHECK,CHECK-FP16 %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @lrint() {
+; CHECK-LABEL: 'lrint'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.lrint.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.lrint.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(float poison)
+  call <2 x i64> @llvm.lrint(<2 x float> poison)
+  call <4 x i64> @llvm.lrint(<4 x float> poison)
+  call <8 x i64> @llvm.lrint(<8 x float> poison)
+  call <16 x i64> @llvm.lrint(<16 x float> poison)
+  call i64 @llvm.lrint(double poison)
+  call <2 x i64> @llvm.lrint(<2 x double> poison)
+  call <4 x i64> @llvm.lrint(<4 x double> poison)
+  call <8 x i64> @llvm.lrint(<8 x double> poison)
+  call <16 x i64> @llvm.lrint(<16 x double> poison)
+  ret void
+}
+
+define void @lrint_fp16() {
+; CHECK-LABEL: 'lrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(half poison)
+  call <2 x i64> @llvm.lrint(<2 x half> poison)
+  call <4 x i64> @llvm.lrint(<4 x half> poison)
+  call <8 x i64> @llvm.lrint(<8 x half> poison)
+  call <16 x i64> @llvm.lrint(<16 x half> poison)
+  ret void
+}
+
+define void @lrint_bf16() {
+; CHECK-LABEL: 'lrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lrint.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lrint.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lrint.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lrint.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lrint.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lrint(bfloat poison)
+  call <2 x i64> @llvm.lrint(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lrint(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lrint(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lrint(<16 x bfloat> poison)
+  ret void
+}
+
+define void @llrint() {
+; CHECK-LABEL: 'llrint'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.llrint.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(float poison)
+  call <2 x i64> @llvm.llrint(<2 x float> poison)
+  call <4 x i64> @llvm.llrint(<4 x float> poison)
+  call <8 x i64> @llvm.llrint(<8 x float> poison)
+  call <16 x i64> @llvm.llrint(<16 x float> poison)
+  call i64 @llvm.llrint(double poison)
+  call <2 x i64> @llvm.llrint(<2 x double> poison)
+  call <4 x i64> @llvm.llrint(<4 x double> poison)
+  call <8 x i64> @llvm.llrint(<8 x double> poison)
+  call <16 x i64> @llvm.llrint(<16 x double> poison)
+  ret void
+}
+
+define void @llrint_fp16() {
+; CHECK-LABEL: 'llrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(half poison)
+  call <2 x i64> @llvm.llrint(<2 x half> poison)
+  call <4 x i64> @llvm.llrint(<4 x half> poison)
+  call <8 x i64> @llvm.llrint(<8 x half> poison)
+  call <16 x i64> @llvm.llrint(<16 x half> poison)
+  ret void
+}
+
+define void @llrint_bf16() {
+; CHECK-LABEL: 'llrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llrint.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llrint.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llrint.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llrint.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llrint.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llrint(bfloat poison)
+  call <2 x i64> @llvm.llrint(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llrint(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llrint(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llrint(<16 x bfloat> poison)
+  ret void
+}
+
+
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.lround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(float poison)
+  call <2 x i64> @llvm.lround(<2 x float> poison)
+  call <4 x i64> @llvm.lround(<4 x float> poison)
+  call <8 x i64> @llvm.lround(<8 x float> poison)
+  call <16 x i64> @llvm.lround(<16 x float> poison)
+  call i64 @llvm.lround(double poison)
+  call <2 x i64> @llvm.lround(<2 x double> poison)
+  call <4 x i64> @llvm.lround(<4 x double> poison)
+  call <8 x i64> @llvm.lround(<8 x double> poison)
+  call <16 x i64> @llvm.lround(<16 x double> poison)
+  ret void
+}
+
+define void @lround_fp16() {
+; CHECK-LABEL: 'lround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(half poison)
+  call <2 x i64> @llvm.lround(<2 x half> poison)
+  call <4 x i64> @llvm.lround(<4 x half> poison)
+  call <8 x i64> @llvm.lround(<8 x half> poison)
+  call <16 x i64> @llvm.lround(<16 x half> poison)
+  ret void
+}
+
+define void @lround_bf16() {
+; CHECK-LABEL: 'lround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.lround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.lround(bfloat poison)
+  call <2 x i64> @llvm.lround(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lround(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lround(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lround(<16 x bfloat> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %6 = call i64 @llvm.llround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %7 = call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %8 = call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %9 = call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %10 = call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(float poison)
+  call <2 x i64> @llvm.llround(<2 x float> poison)
+  call <4 x i64> @llvm.llround(<4 x float> poison)
+  call <8 x i64> @llvm.llround(<8 x float> poison)
+  call <16 x i64> @llvm.llround(<16 x float> poison)
+  call i64 @llvm.llround(double poison)
+  call <2 x i64> @llvm.llround(<2 x double> poison)
+  call <4 x i64> @llvm.llround(<4 x double> poison)
+  call <8 x i64> @llvm.llround(<8 x double> poison)
+  call <16 x i64> @llvm.llround(<16 x double> poison)
+  ret void
+}
+
+define void @llround_fp16() {
+; CHECK-LABEL: 'llround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.f16(half poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2f16(<2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4f16(<4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8f16(<8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16f16(<16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(half poison)
+  call <2 x i64> @llvm.llround(<2 x half> poison)
+  call <4 x i64> @llvm.llround(<4 x half> poison)
+  call <8 x i64> @llvm.llround(<8 x half> poison)
+  call <16 x i64> @llvm.llround(<16 x half> poison)
+  ret void
+}
+
+define void @llround_bf16() {
+; CHECK-LABEL: 'llround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %1 = call i64 @llvm.llround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:6 SizeLat:6 for: %2 = call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:8 Lat:12 SizeLat:12 for: %3 = call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:16 Lat:24 SizeLat:24 for: %4 = call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:32 Lat:48 SizeLat:48 for: %5 = call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call i64 @llvm.llround(bfloat poison)
+  call <2 x i64> @llvm.llround(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llround(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llround(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llround(<16 x bfloat> poison)
+  ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-FP16: {{.*}}
+; CHECK-NOFP16: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll b/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
new file mode 100644
index 000000000000..8b94cefbad63
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/segmented-shufflevector-patterns.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+
+;; Broadcast indexed lane within 128b segments (dupq zd.t, zn.t[idx])
+define void @dup_within_each_segment_256b() #0 {
+; CHECK-LABEL: 'dup_within_each_segment_256b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_b11 = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_h2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_s3 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_d0 = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %dupq_512b_d1 = shufflevector <8 x i64> poison, <8 x i64> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_s3_with_poison = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 poison, i32 3, i32 7, i32 poison, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dupq_b11 = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11,
+                                                                            i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27>
+  %dupq_h2  = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                              i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %dupq_s3  = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                           i32 7, i32 7, i32 7, i32 7>
+  %dupq_d0  = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %dupq_512b_d1 = shufflevector <8 x i64> poison, <8 x i64> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+  %dupq_s3_with_poison = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 poison, i32 3,
+                                                                                      i32 7, i32 poison, i32 7, i32 7>
+  ret void
+}
+
+define void @dup_within_each_segment_512b() #1 {
+; CHECK-LABEL: 'dup_within_each_segment_512b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_b11 = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_h2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_s3 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 7, i32 7, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_d0 = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_512b_d1 = shufflevector <8 x i64> poison, <8 x i64> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dupq_s3_with_poison = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 poison, i32 3, i32 7, i32 poison, i32 7, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dupq_b11 = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11, i32 11,
+                                                                            i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27, i32 27>
+  %dupq_h2  = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                              i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %dupq_s3  = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                           i32 7, i32 7, i32 7, i32 7>
+  %dupq_d0  = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %dupq_512b_d1 = shufflevector <8 x i64> poison, <8 x i64> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
+  %dupq_s3_with_poison = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 poison, i32 3,
+                                                                                      i32 7, i32 poison, i32 7, i32 7>
+  ret void
+}
+
+define void @dup_whole_segment_256b() #0 {
+; CHECK-LABEL: 'dup_whole_segment_256b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 poison, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                                                             i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                               i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                            i32 0, i32 1, i32 2, i32 3>
+  %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1,
+                                                                                       i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3,
+                                                                                            i32 poison, i32 1, i32 2, i32 3>
+  ret void
+}
+
+define void @dup_whole_segment_512b() #1 {
+; CHECK-LABEL: 'dup_whole_segment_512b'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 poison, i32 1, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  %dup_seg_b = shufflevector <32 x i8> poison, <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                                                             i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %dup_seg_h = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                               i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %dup_seg_s = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                            i32 0, i32 1, i32 2, i32 3>
+  %dup_seg_d = shufflevector <4 x i64> poison, <4 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_512b_d = shufflevector <8 x double> poison, <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1,
+                                                                                       i32 0, i32 1, i32 0, i32 1>
+  %dup_seg_s_with_poison = shufflevector <8 x float> poison, <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 3,
+                                                                                            i32 poison, i32 1, i32 2, i32 3>
+  ret void
+}
+
+attributes #0 = { noinline vscale_range(2,2) }
+attributes #1 = { noinline vscale_range(4,4) }
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
index 8d68781d0c1b..41c272291d7c 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
@@ -87,21 +87,21 @@ define void @concat() {
 
 define void @insert_subvec() {
 ; CHECK-LABEL: 'insert_subvec'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:6 Lat:12 SizeLat:12 for: %v4i8_2_0 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8_2_0 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i8_2_1 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:14 Lat:28 SizeLat:28 for: %v8i8_2_0 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2_0 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2_1 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2_2 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i8_2_3 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %v8i8_2_05 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found costs of RThru:60 CodeSize:30 Lat:60 SizeLat:60 for: %v16i8_4_0 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_4_0 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_4_1 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_4_2 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i8_4_3 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:7 Lat:14 SizeLat:14 for: %v16i8_4_05 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_2_0 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v4i16_2_1 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:14 Lat:28 SizeLat:28 for: %v8i16_2_0 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2_0 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2_1 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2_2 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i16_2_3 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
@@ -118,7 +118,7 @@ define void @insert_subvec() {
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i32_2_2 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v8i32_2_3 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:3 Lat:6 SizeLat:6 for: %v8i32_2_05 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found costs of 0 for: %v16i32_4_0 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i32_4_0 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i32_4_1 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i32_4_2 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found costs of 1 for: %v16i32_4_3 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll b/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll
new file mode 100644
index 000000000000..037958cf52e2
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-ldexp.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 -mattr=+sve2 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @ldexp() {
+; CHECK-LABEL: 'ldexp'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_fp16() {
+; CHECK-LABEL: 'ldexp_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x half> @llvm.ldexp.nxv2f16.nxv2i32(<vscale x 2 x half> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x half> @llvm.ldexp.nxv4f16.nxv4i32(<vscale x 4 x half> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x half> @llvm.ldexp.nxv8f16.nxv8i32(<vscale x 8 x half> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x half> @llvm.ldexp.nxv16f16.nxv16i32(<vscale x 16 x half> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x half> @llvm.ldexp.nxv2f16.nxv2i32(<vscale x 2 x half> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x half> @llvm.ldexp.nxv4f16.nxv4i32(<vscale x 4 x half> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x half> @llvm.ldexp.nxv8f16.nxv8i32(<vscale x 8 x half> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x half> @llvm.ldexp.nxv16f16.nxv16i32(<vscale x 16 x half> poison, <vscale x 16 x i32> poison)
+  ret void
+}
+
+define void @ldexp_bf16() {
+; CHECK-LABEL: 'ldexp_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll b/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll
new file mode 100644
index 000000000000..540378bbf421
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-lrint.ll
@@ -0,0 +1,218 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64 -mattr=+sve2 %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-linux-gnu"
+
+define void @lrint() {
+; CHECK-LABEL: 'lrint'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @lrint_fp16() {
+; CHECK-LABEL: 'lrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @lrint_bf16() {
+; CHECK-LABEL: 'lrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lrint(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lrint(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lrint(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lrint(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+define void @llrint() {
+; CHECK-LABEL: 'llrint'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llrint_fp16() {
+; CHECK-LABEL: 'llrint_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @llrint_bf16() {
+; CHECK-LABEL: 'llrint_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llrint.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llrint.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llrint(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llrint(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llrint(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llrint(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @lround_fp16() {
+; CHECK-LABEL: 'lround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @lround_bf16() {
+; CHECK-LABEL: 'lround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.lround(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lround(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lround(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lround(<vscale x 16 x bfloat> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %5 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %6 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %7 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %8 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x float> poison)
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llround_fp16() {
+; CHECK-LABEL: 'llround_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f16(<vscale x 2 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f16(<vscale x 4 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f16(<vscale x 8 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f16(<vscale x 16 x half> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x half> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x half> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x half> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x half> poison)
+  ret void
+}
+
+define void @llround_bf16() {
+; CHECK-LABEL: 'llround_bf16'
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %1 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %2 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %3 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of Invalid for: %4 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  call <vscale x 2 x i64> @llvm.llround(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llround(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llround(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llround(<vscale x 16 x bfloat> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll b/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
index 2b1b5906ad01..260cf696270c 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/ldexp.ll
@@ -13,45 +13,45 @@ define void @ldexp_f16_i32() {
 ; GFX7-LABEL: 'ldexp_f16_i32'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_f16_i32'
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX7-SIZE-LABEL: 'ldexp_f16_i32'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_f16_i32'
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i32(<2 x half> undef, <2 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i32(<3 x half> undef, <3 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i32(<4 x half> undef, <4 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i32(<5 x half> undef, <5 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i32(<8 x half> undef, <8 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i32(<16 x half> undef, <16 x i32> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i32(<17 x half> undef, <17 x i32> undef)
 ; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %f16 = call half @llvm.ldexp.f16.i32(half undef, i32 undef)
@@ -69,45 +69,45 @@ define void @ldexp_f16_i16() {
 ; GFX7-LABEL: 'ldexp_f16_i16'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_f16_i16'
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
+; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX8PLUS-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX7-SIZE-LABEL: 'ldexp_f16_i16'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_f16_i16'
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
-; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %v2f16 = call <2 x half> @llvm.ldexp.v2f16.v2i16(<2 x half> undef, <2 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3f16 = call <3 x half> @llvm.ldexp.v3f16.v3i16(<3 x half> undef, <3 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %v4f16 = call <4 x half> @llvm.ldexp.v4f16.v4i16(<4 x half> undef, <4 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %v5f16 = call <5 x half> @llvm.ldexp.v5f16.v5i16(<5 x half> undef, <5 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %v8f16 = call <8 x half> @llvm.ldexp.v8f16.v8i16(<8 x half> undef, <8 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %v16f16 = call <16 x half> @llvm.ldexp.v16f16.v16i16(<16 x half> undef, <16 x i16> undef)
+; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %v17f16 = call <17 x half> @llvm.ldexp.v17f16.v17i16(<17 x half> undef, <17 x i16> undef)
 ; GFX8PLUS-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %f16 = call half @llvm.ldexp.f16.i16(half undef, i16 undef)
@@ -125,12 +125,12 @@ define void @ldexp_bf16() {
 ; GFX7-LABEL: 'ldexp_bf16'
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bf16 = call bfloat @llvm.ldexp.bf16.i32(bfloat undef, i32 undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2bf16 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> undef, <2 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4bf16 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> undef, <4 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8bf16 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> undef, <8 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16bf16 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> undef, <16 x i32> undef)
-; GFX7-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
+; GFX7-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
 ; GFX7-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX8PLUS-LABEL: 'ldexp_bf16'
@@ -147,12 +147,12 @@ define void @ldexp_bf16() {
 ; GFX7-SIZE-LABEL: 'ldexp_bf16'
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %bf16 = call bfloat @llvm.ldexp.bf16.i32(bfloat undef, i32 undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2bf16 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> undef, <2 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v3bf16 = call <3 x bfloat> @llvm.ldexp.v3bf16.v3i32(<3 x bfloat> undef, <3 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4bf16 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> undef, <4 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v5bf16 = call <5 x bfloat> @llvm.ldexp.v5bf16.v5i32(<5 x bfloat> undef, <5 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8bf16 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> undef, <8 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16bf16 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> undef, <16 x i32> undef)
-; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
+; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v17bf16 = call <17 x bfloat> @llvm.ldexp.v17bf16.v17i32(<17 x bfloat> undef, <17 x i32> undef)
 ; GFX7-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; GFX8PLUS-SIZE-LABEL: 'ldexp_bf16'
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index 3f8016178e71..6ec84bd88cd4 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -21,17 +21,17 @@ define void @loads_i1(i32 %arg) {
 define void @loads_i8(i32 %arg) {
 ; GFX90A-LABEL: 'loads_i8'
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = load <2 x i8>, ptr poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = load <3 x i8>, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %7 = load <3 x i8>, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i8>, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %11 = load <3 x i8>, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
   load i8, ptr poison
@@ -154,35 +154,35 @@ define void @loads_addrspace_1(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load <1 x i16>, ptr addrspace(1) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = load <1 x i32>, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = load <2 x i1>, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr addrspace(1) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr addrspace(1) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load <2 x i16>, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = load <2 x i32>, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = load <3 x i1>, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = load <3 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = load <3 x i8>, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i16>, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = load <3 x i32>, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i1>, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = load <4 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = load <4 x i8>, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = load <4 x i16>, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %18 = load <8 x i8>, ptr addrspace(1) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = load <8 x i8>, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %22 = load <16 x i8>, ptr addrspace(1) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = load <16 x i8>, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %26 = load <32 x i8>, ptr addrspace(1) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %26 = load <32 x i8>, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %30 = load <64 x i8>, ptr addrspace(1) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %30 = load <64 x i8>, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(1) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %34 = load <128 x i8>, ptr addrspace(1) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %34 = load <128 x i8>, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(1) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(1) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
@@ -241,35 +241,35 @@ define void @loads_addrspace_3(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load <1 x i16>, ptr addrspace(3) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = load <1 x i32>, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = load <2 x i1>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr addrspace(3) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr addrspace(3) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load <2 x i16>, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = load <2 x i32>, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = load <3 x i1>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = load <3 x i8>, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = load <3 x i8>, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i16>, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = load <3 x i32>, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i1>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = load <4 x i8>, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = load <4 x i8>, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = load <4 x i16>, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %18 = load <8 x i8>, ptr addrspace(3) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %18 = load <8 x i8>, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(3) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %22 = load <16 x i8>, ptr addrspace(3) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = load <16 x i8>, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %26 = load <32 x i8>, ptr addrspace(3) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %26 = load <32 x i8>, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %30 = load <64 x i8>, ptr addrspace(3) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %30 = load <64 x i8>, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(3) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(3) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %34 = load <128 x i8>, ptr addrspace(3) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %34 = load <128 x i8>, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(3) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(3) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
@@ -328,35 +328,35 @@ define void @loads_addrspace_5(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = load <1 x i16>, ptr addrspace(5) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = load <1 x i32>, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = load <2 x i1>, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr addrspace(5) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr addrspace(5) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = load <2 x i16>, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = load <2 x i32>, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %9 = load <3 x i1>, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %10 = load <3 x i8>, ptr addrspace(5) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %10 = load <3 x i8>, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i16>, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %12 = load <3 x i32>, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %13 = load <4 x i1>, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %14 = load <4 x i8>, ptr addrspace(5) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %14 = load <4 x i8>, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %15 = load <4 x i16>, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %18 = load <8 x i8>, ptr addrspace(5) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %18 = load <8 x i8>, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(5) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %22 = load <16 x i8>, ptr addrspace(5) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = load <16 x i8>, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(5) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %26 = load <32 x i8>, ptr addrspace(5) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %26 = load <32 x i8>, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(5) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %30 = load <64 x i8>, ptr addrspace(5) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %30 = load <64 x i8>, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(5) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %34 = load <128 x i8>, ptr addrspace(5) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %34 = load <128 x i8>, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(5) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(5) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
index 1705cd2a2f09..3aa682cd2971 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/shufflevector.ll
@@ -16,11 +16,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf22 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 2>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf33 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 2>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 1>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 3>
@@ -53,11 +53,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 2>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf22_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 2>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf33_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 2>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 3>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 1>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 3>
@@ -93,11 +93,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 2>
-; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
+; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 2>
-; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 1>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 3>
@@ -130,11 +130,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 2>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 2>
-; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
+; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 2>
-; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 3>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 1>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 3>
@@ -170,11 +170,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf22 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 2>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf33 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 2>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 1>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 3>
@@ -207,11 +207,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 2>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf22_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 2>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf33_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 2>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 3>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 1>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 3>
@@ -247,11 +247,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 2>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 2>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 0, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 2>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 1>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 1, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 3, i32 1>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i16> %vec1, <2 x i16> %vec1, <2 x i32> <i32 2, i32 3>
@@ -284,11 +284,11 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf02_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 2>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 2>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 0, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 2>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 1>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf13_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 1, i32 3>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 3, i32 1>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i16> %vec1, <2 x i16> %vec2, <2 x i32> <i32 2, i32 3>
@@ -396,157 +396,157 @@ define amdgpu_kernel void @shufflevector_i16(<2 x i16> %vec1, <2 x i16> %vec2) {
 ; Should not assert
 define amdgpu_kernel void @shufflevector_i8(<2 x i8> %vec1, <2 x i8> %vec2) {
 ; ALL-LABEL: 'shufflevector_i8'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf00 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> zeroinitializer
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf00 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> zeroinitializer
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf01 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf10 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf11 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf10 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf11 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf02 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf20 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf22 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf03 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf30 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf33 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf12 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf21 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf13 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf31 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf32 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf000 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> zeroinitializer
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf001 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf010 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf011 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf100 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf101 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf110 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf002 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf020 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf022 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf200 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf202 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf220 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf222 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf112 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf121 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf122 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf211 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf212 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf221 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf00_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> zeroinitializer
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf32 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> zeroinitializer
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf001 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf010 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf011 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf100 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf101 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf110 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf111 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf002 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf020 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf022 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf200 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf202 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf220 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf112 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf121 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf122 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf211 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf212 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf221 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf00_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> zeroinitializer
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf01_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf10_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf11_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf10_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf11_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf02_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf20_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf22_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf03_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf30_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf33_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf12_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf21_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf13_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf31_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf32_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf000_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> zeroinitializer
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf001_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf010_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf011_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf100_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf101_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf110_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf111_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf002_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf020_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf022_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf200_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf202_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf220_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 0>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf222_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf112_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf121_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf122_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf211_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 1>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf212_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 2>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf221_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf32_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf000_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> zeroinitializer
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf001_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf010_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf011_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf100_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf101_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf110_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf111_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf002_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf020_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf022_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf200_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf202_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf220_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 0>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf222_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf112_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf121_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf122_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf211_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 1>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf212_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 2>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf221_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 1>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; ALL-SIZE-LABEL: 'shufflevector_i8'
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf00 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> zeroinitializer
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf00 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> zeroinitializer
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf01 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf10 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf11 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf10 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf11 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf02 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf20 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf22 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf03 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf30 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf33 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf12 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf21 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 0, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf13 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 1, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf31 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 2, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf32 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf000 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> zeroinitializer
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf001 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf010 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf011 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf100 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf101 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf110 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf111 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf002 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf020 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf022 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf200 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf202 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf220 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf222 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf112 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf121 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf122 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf211 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf212 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf221 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf00_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> zeroinitializer
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf32 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> <i32 3, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf000 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> zeroinitializer
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf001 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf010 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf011 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf100 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf101 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 0, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf110 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf111 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf002 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 0, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf020 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf022 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 0, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf200 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf202 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 0, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf220 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf222 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf112 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf121 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf122 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 1, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf211 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf212 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf221 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <3 x i32> <i32 2, i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf00_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> zeroinitializer
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf01_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf10_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf11_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf10_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf11_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf02_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf20_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf22_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf03_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf30_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %shuf33_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf12_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf21_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %shuf20_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf22_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf03_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 0, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf30_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf33_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf12_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf21_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf13_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 1, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf31_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf31_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %shuf23_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 2, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf32_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf000_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> zeroinitializer
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf001_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf010_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf011_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf100_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf101_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf110_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %shuf111_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf002_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf020_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf022_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf200_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf202_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf220_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 0>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf222_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf112_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf121_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf122_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf211_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 1>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf212_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 2>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %shuf221_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf32_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <2 x i32> <i32 3, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf000_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> zeroinitializer
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf001_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf010_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf011_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf100_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf101_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 0, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf110_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf111_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf002_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 0, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf020_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf022_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 0, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf200_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf202_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 0, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf220_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 0>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf222_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf112_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf121_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf122_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 1, i32 2, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf211_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 1>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf212_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 1, i32 2>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %shuf221_2 = shufflevector <2 x i8> %vec1, <2 x i8> %vec2, <3 x i32> <i32 2, i32 2, i32 1>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %shuf00 = shufflevector <2 x i8> %vec1, <2 x i8> %vec1, <2 x i32> zeroinitializer
@@ -861,22 +861,22 @@ define amdgpu_kernel void @shufflevector_i32(<2 x i32> %vec1, <2 x i32> %vec2) {
 ; Other shuffle cases
 define void @shuffle(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8> %i8v4_2, <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i8> %i8v8, <8 x i8> %i8v8_2, <16 x i8> %i8v16, <16 x i8> %i8v16_2, <2 x i16> %i16v2, <2 x i16> %i16v2_2, <4 x i16> %i16v4, <4 x i16> %i16v4_2, <8 x i16> %i16v8, <8 x i16> %i16v8_2, <2 x i32> %i32v2, <2 x i32> %i32v2_2, <4 x i32> %i32v4, <4 x i32> %i32v4_2, <2 x float> %floatv2, <2 x float> %floatv2_2, <4 x float> %floatv4, <4 x float> %floatv4_2,<2 x i64> %i64v2, <2 x i64> %i64v2_2,<2 x double> %doublev2, <2 x double> %doublev2_2) {
 ; GFX9-10-LABEL: 'shuffle'
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <2 x i32> <i32 1, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <2 x i32> <i32 1, i32 0>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16_4 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
@@ -898,22 +898,22 @@ define void @shuffle(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8>
 ; GFX9-10-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; VI-LABEL: 'shuffle'
-; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <2 x i32> <i32 1, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <2 x i32> <i32 1, i32 0>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16_4 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
@@ -935,22 +935,22 @@ define void @shuffle(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8>
 ; VI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; GFX9-10-SIZE-LABEL: 'shuffle'
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <2 x i32> <i32 1, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v2i16_2_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <2 x i32> <i32 1, i32 0>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16_4 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
@@ -972,22 +972,22 @@ define void @shuffle(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8>
 ; GFX9-10-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; VI-SIZE-LABEL: 'shuffle'
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
-; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <2 x i32> <i32 1, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_2_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <2 x i32> <i32 1, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_4_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_4_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i8_8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v6i8_8_2 = shufflevector <6 x i8> %i8v6, <6 x i8> %i8v6_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8_8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
+; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8_16_2 = shufflevector <16 x i8> %i8v16, <16 x i8> %i8v16_2, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <2 x i32> <i32 1, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_2_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <2 x i32> <i32 1, i32 0>
 ; VI-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16_4 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <4 x i32> <i32 1, i32 3, i32 2, i32 0>
@@ -1047,9 +1047,9 @@ define void @shuffle(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8>
 
 define void @concat(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i8> %i8v8, <8 x i8> %i8v8_2, <2 x half> %halfv2, <2 x half> %halfv2_2, <4 x half> %halfv4, <4 x half> %halfv4_2, <8 x half> %halfv8, <8 x half> %halfv8_2, <2 x i16> %i16v2, <2 x i16> %i16v2_2, <4 x i16> %i16v4, <4 x i16> %i16v4_2, <8 x i16> %i16v8, <8 x i16> %i16v8_2, <2 x i32> %i32v2, <2 x i32> %i32v2_2, <4 x i32> %i32v4, <4 x i32> %i32v4_2, <2 x float> %floatv2, <2 x float> %floatv2_2, <4 x float> %floatv4, <4 x float> %floatv4_2,<2 x i64> %i64v2, <2 x i64> %i64v2_2,<2 x double> %doublev2, <2 x double> %doublev2_2) {
 ; ALL-LABEL: 'concat'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i16 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i16 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16 = shufflevector <8 x i16> %i16v8, <8 x i16> %i16v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -1062,9 +1062,9 @@ define void @concat(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8> %
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32 = shufflevector <2 x float> %floatv2, <2 x float> %floatv2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32 = shufflevector <4 x float> %floatv4, <4 x float> %floatv4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64 = shufflevector <2 x double> %doublev2, <2 x double> %doublev2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; ALL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i16_2 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16_2 = shufflevector <8 x i16> %i16v8, <8 x i16> %i16v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -1080,9 +1080,9 @@ define void @concat(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8> %
 ; ALL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
 ; ALL-SIZE-LABEL: 'concat'
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i8 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i8 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i8 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i16 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i16 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16 = shufflevector <8 x i16> %i16v8, <8 x i16> %i16v8, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -1095,9 +1095,9 @@ define void @concat(<2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i8> %i8v4, <4 x i8> %
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f32 = shufflevector <2 x float> %floatv2, <2 x float> %floatv2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8f32 = shufflevector <4 x float> %floatv4, <4 x float> %floatv4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4f64 = shufflevector <2 x double> %doublev2, <2 x double> %doublev2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i8_2 = shufflevector <2 x i8> %i8v2, <2 x i8> %i8v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i8_2 = shufflevector <4 x i8> %i8v4, <4 x i8> %i8v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i8_2 = shufflevector <8 x i8> %i8v8, <8 x i8> %i8v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v4i16_2 = shufflevector <2 x i16> %i16v2, <2 x i16> %i16v2_2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v8i16_2 = shufflevector <4 x i16> %i16v4, <4 x i16> %i16v4_2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; ALL-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v16i16_2 = shufflevector <8 x i16> %i16v8, <8 x i16> %i16v8_2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
index 9672c3256c75..6dc4befdfbd9 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
@@ -20,17 +20,17 @@ define void @stores_i1(i32 %arg) {
 define void @stores_i8(i32 %arg) {
 ; GFX90A-LABEL: 'stores_i8'
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
 ;
   store i8 poison, ptr poison
@@ -153,35 +153,35 @@ define void @stores_addrspace_1(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i16> poison, ptr addrspace(1) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i32> poison, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i1> poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr addrspace(1) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr addrspace(1) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i1> poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr addrspace(1) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i16> poison, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i1> poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(1) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(1) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i1> poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> poison, ptr addrspace(1) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> poison, ptr addrspace(1) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i1> poison, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(1) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(1) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i16> poison, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i32> poison, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i1> poison, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i8> poison, ptr addrspace(1) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i8> poison, ptr addrspace(1) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i16> poison, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i32> poison, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i1> poison, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i8> poison, ptr addrspace(1) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <64 x i8> poison, ptr addrspace(1) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i16> poison, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i32> poison, ptr addrspace(1) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i1> poison, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i8> poison, ptr addrspace(1) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <128 x i8> poison, ptr addrspace(1) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i16> poison, ptr addrspace(1) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i32> poison, ptr addrspace(1) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
@@ -241,35 +241,35 @@ define void @stores_addrspace_3(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i16> poison, ptr addrspace(3) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i32> poison, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i1> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr addrspace(3) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr addrspace(3) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i1> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i16> poison, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i1> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(3) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i1> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> poison, ptr addrspace(3) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> poison, ptr addrspace(3) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i1> poison, ptr addrspace(3) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(3) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> poison, ptr addrspace(3) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i16> poison, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i32> poison, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i1> poison, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i8> poison, ptr addrspace(3) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <32 x i8> poison, ptr addrspace(3) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i16> poison, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i32> poison, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i1> poison, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i8> poison, ptr addrspace(3) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <64 x i8> poison, ptr addrspace(3) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i16> poison, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i32> poison, ptr addrspace(3) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i1> poison, ptr addrspace(3) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i8> poison, ptr addrspace(3) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <128 x i8> poison, ptr addrspace(3) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i16> poison, ptr addrspace(3) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i32> poison, ptr addrspace(3) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
@@ -329,35 +329,35 @@ define void @stores_addrspace_5(i32 %arg) {
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i16> poison, ptr addrspace(5) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <1 x i32> poison, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i1> poison, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr addrspace(5) poison, align 2
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr addrspace(5) poison, align 2
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i1> poison, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <3 x i8> poison, ptr addrspace(5) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i8> poison, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i16> poison, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i1> poison, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr addrspace(5) poison, align 4
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr addrspace(5) poison, align 4
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i1> poison, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i8> poison, ptr addrspace(5) poison, align 8
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> poison, ptr addrspace(5) poison, align 8
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i16> poison, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> poison, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i1> poison, ptr addrspace(5) poison, align 2
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i8> poison, ptr addrspace(5) poison, align 16
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <16 x i8> poison, ptr addrspace(5) poison, align 16
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i16> poison, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i32> poison, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i1> poison, ptr addrspace(5) poison, align 4
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i8> poison, ptr addrspace(5) poison, align 32
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <32 x i8> poison, ptr addrspace(5) poison, align 32
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i16> poison, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i32> poison, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i1> poison, ptr addrspace(5) poison, align 8
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i8> poison, ptr addrspace(5) poison, align 64
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <64 x i8> poison, ptr addrspace(5) poison, align 64
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i16> poison, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <64 x i32> poison, ptr addrspace(5) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i1> poison, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: store <128 x i8> poison, ptr addrspace(5) poison, align 128
+; GFX90A-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <128 x i8> poison, ptr addrspace(5) poison, align 128
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i16> poison, ptr addrspace(5) poison, align 256
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <128 x i32> poison, ptr addrspace(5) poison, align 512
 ; GFX90A-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: ret void
diff --git a/llvm/test/Analysis/CostModel/ARM/shuffle.ll b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
index 14f16252581a..a17bbaba9d9a 100644
--- a/llvm/test/Analysis/CostModel/ARM/shuffle.ll
+++ b/llvm/test/Analysis/CostModel/ARM/shuffle.ll
@@ -241,7 +241,7 @@ define void @concat() {
 
 define void @select() {
 ; CHECK-MVE-LABEL: 'select'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -266,7 +266,7 @@ define void @select() {
 ; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-NEON-LABEL: 'select'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v2i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> <i32 0, i32 3>
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 8, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
 ; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/Analysis/CostModel/RISCV/exp.ll b/llvm/test/Analysis/CostModel/RISCV/exp.ll
new file mode 100644
index 000000000000..55c3f5554eff
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/RISCV/exp.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -S -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh,+zvfbfmin %s | FileCheck %s
+
+define void @ldexp() {
+; CHECK-LABEL: 'ldexp'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %2 = call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %3 = call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %4 = call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %5 = call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x bfloat> @llvm.ldexp.nxv1bf16.nxv1i32(<vscale x 1 x bfloat> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %11 = call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %12 = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %13 = call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %14 = call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %15 = call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i32(<vscale x 1 x float> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %21 = call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %22 = call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %23 = call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %24 = call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %25 = call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i32(<vscale x 1 x double> poison, <vscale x 1 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %31 = call float @llvm.ldexp.f32.i64(float poison, i64 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %32 = call <2 x float> @llvm.ldexp.v2f32.v2i64(<2 x float> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %33 = call <4 x float> @llvm.ldexp.v4f32.v4i64(<4 x float> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %34 = call <8 x float> @llvm.ldexp.v8f32.v8i64(<8 x float> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %35 = call <16 x float> @llvm.ldexp.v16f32.v16i64(<16 x float> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %36 = call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i64(<vscale x 1 x float> poison, <vscale x 1 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %37 = call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i64(<vscale x 2 x float> poison, <vscale x 2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %38 = call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i64(<vscale x 4 x float> poison, <vscale x 4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %39 = call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i64(<vscale x 8 x float> poison, <vscale x 8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %40 = call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i64(<vscale x 16 x float> poison, <vscale x 16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %41 = call double @llvm.ldexp.f64.i64(double poison, i64 poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %42 = call <2 x double> @llvm.ldexp.v2f64.v2i64(<2 x double> poison, <2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %43 = call <4 x double> @llvm.ldexp.v4f64.v4i64(<4 x double> poison, <4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %44 = call <8 x double> @llvm.ldexp.v8f64.v8i64(<8 x double> poison, <8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %45 = call <16 x double> @llvm.ldexp.v16f64.v16i64(<16 x double> poison, <16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %46 = call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i64(<vscale x 1 x double> poison, <vscale x 1 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %47 = call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i64(<vscale x 2 x double> poison, <vscale x 2 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %48 = call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i64(<vscale x 4 x double> poison, <vscale x 4 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %49 = call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i64(<vscale x 8 x double> poison, <vscale x 8 x i64> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %50 = call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i64(<vscale x 16 x double> poison, <vscale x 16 x i64> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call bfloat @llvm.ldexp.bf16.i32(bfloat poison, i32 poison)
+  call <2 x bfloat> @llvm.ldexp.v2bf16.v2i32(<2 x bfloat> poison, <2 x i32> poison)
+  call <4 x bfloat> @llvm.ldexp.v4bf16.v4i32(<4 x bfloat> poison, <4 x i32> poison)
+  call <8 x bfloat> @llvm.ldexp.v8bf16.v8i32(<8 x bfloat> poison, <8 x i32> poison)
+  call <16 x bfloat> @llvm.ldexp.v16bf16.v16i32(<16 x bfloat> poison, <16 x i32> poison)
+  call <vscale x 1 x bfloat> @llvm.ldexp.nxv1bf16.nxv1i32(<vscale x 1 x bfloat> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x bfloat> @llvm.ldexp.nxv2bf16.nxv2i32(<vscale x 2 x bfloat> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x bfloat> @llvm.ldexp.nxv4bf16.nxv4i32(<vscale x 4 x bfloat> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x bfloat> @llvm.ldexp.nxv8bf16.nxv8i32(<vscale x 8 x bfloat> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x bfloat> @llvm.ldexp.nxv16bf16.nxv16i32(<vscale x 16 x bfloat> poison, <vscale x 16 x i32> poison)
+  call float @llvm.ldexp.f32.i32(float poison, i32 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> poison, <2 x i32> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i32(<4 x float> poison, <4 x i32> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i32(<8 x float> poison, <8 x i32> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i32(<16 x float> poison, <16 x i32> poison)
+  call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i32(<vscale x 1 x float> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i32(<vscale x 2 x float> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i32(<vscale x 4 x float> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i32(<vscale x 8 x float> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i32(<vscale x 16 x float> poison, <vscale x 16 x i32> poison)
+  call double @llvm.ldexp.f64.i32(double poison, i32 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double> poison, <2 x i32> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i32(<4 x double> poison, <4 x i32> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i32(<8 x double> poison, <8 x i32> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i32(<16 x double> poison, <16 x i32> poison)
+  call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i32(<vscale x 1 x double> poison, <vscale x 1 x i32> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i32(<vscale x 2 x double> poison, <vscale x 2 x i32> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i32(<vscale x 4 x double> poison, <vscale x 4 x i32> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i32(<vscale x 8 x double> poison, <vscale x 8 x i32> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i32(<vscale x 16 x double> poison, <vscale x 16 x i32> poison)
+  call float @llvm.ldexp.f32.i64(float poison, i64 poison)
+  call <2 x float> @llvm.ldexp.v2f32.v2i64(<2 x float> poison, <2 x i64> poison)
+  call <4 x float> @llvm.ldexp.v4f32.v4i64(<4 x float> poison, <4 x i64> poison)
+  call <8 x float> @llvm.ldexp.v8f32.v8i64(<8 x float> poison, <8 x i64> poison)
+  call <16 x float> @llvm.ldexp.v16f32.v16i64(<16 x float> poison, <16 x i64> poison)
+  call <vscale x 1 x float> @llvm.ldexp.nxv1f32.nxv1i64(<vscale x 1 x float> poison, <vscale x 1 x i64> poison)
+  call <vscale x 2 x float> @llvm.ldexp.nxv2f32.nxv2i64(<vscale x 2 x float> poison, <vscale x 2 x i64> poison)
+  call <vscale x 4 x float> @llvm.ldexp.nxv4f32.nxv4i64(<vscale x 4 x float> poison, <vscale x 4 x i64> poison)
+  call <vscale x 8 x float> @llvm.ldexp.nxv8f32.nxv8i64(<vscale x 8 x float> poison, <vscale x 8 x i64> poison)
+  call <vscale x 16 x float> @llvm.ldexp.nxv16f32.nxv16i64(<vscale x 16 x float> poison, <vscale x 16 x i64> poison)
+  call double @llvm.ldexp.f64.i64(double poison, i64 poison)
+  call <2 x double> @llvm.ldexp.v2f64.v2i64(<2 x double> poison, <2 x i64> poison)
+  call <4 x double> @llvm.ldexp.v4f64.v4i64(<4 x double> poison, <4 x i64> poison)
+  call <8 x double> @llvm.ldexp.v8f64.v8i64(<8 x double> poison, <8 x i64> poison)
+  call <16 x double> @llvm.ldexp.v16f64.v16i64(<16 x double> poison, <16 x i64> poison)
+  call <vscale x 1 x double> @llvm.ldexp.nxv1f64.nxv1i64(<vscale x 1 x double> poison, <vscale x 1 x i64> poison)
+  call <vscale x 2 x double> @llvm.ldexp.nxv2f64.nxv2i64(<vscale x 2 x double> poison, <vscale x 2 x i64> poison)
+  call <vscale x 4 x double> @llvm.ldexp.nxv4f64.nxv4i64(<vscale x 4 x double> poison, <vscale x 4 x i64> poison)
+  call <vscale x 8 x double> @llvm.ldexp.nxv8f64.nxv8i64(<vscale x 8 x double> poison, <vscale x 8 x i64> poison)
+  call <vscale x 16 x double> @llvm.ldexp.nxv16f64.nxv16i64(<vscale x 16 x double> poison, <vscale x 16 x i64> poison)
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/RISCV/fround.ll b/llvm/test/Analysis/CostModel/RISCV/fround.ll
index a0818d487d15..94c09389cade 100644
--- a/llvm/test/Analysis/CostModel/RISCV/fround.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/fround.ll
@@ -606,6 +606,200 @@ define void @llrint_fp16() {
   ret void
 }
 
+define void @lround() {
+; CHECK-LABEL: 'lround'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i32 @llvm.lround.i32.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call <2 x i32> @llvm.lround.v2i32.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = call <4 x i32> @llvm.lround.v4i32.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %4 = call <8 x i32> @llvm.lround.v8i32.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %5 = call <16 x i32> @llvm.lround.v16i32.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i32 @llvm.lround.i32.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %13 = call <4 x i32> @llvm.lround.v4i32.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %14 = call <8 x i32> @llvm.lround.v8i32.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %15 = call <16 x i32> @llvm.lround.v16i32.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i32 @llvm.lround.i32.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = call <2 x i32> @llvm.lround.v2i32.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %23 = call <4 x i32> @llvm.lround.v4i32.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %24 = call <8 x i32> @llvm.lround.v8i32.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %25 = call <16 x i32> @llvm.lround.v16i32.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %31 = call i64 @llvm.lround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %32 = call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %33 = call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %34 = call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %35 = call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %36 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %37 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %38 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %39 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %40 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %41 = call i64 @llvm.lround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %42 = call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %43 = call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %44 = call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %45 = call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %46 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %47 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %48 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %49 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %50 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %51 = call i64 @llvm.lround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %52 = call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %53 = call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %54 = call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %55 = call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %56 = call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %57 = call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %58 = call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %59 = call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %60 = call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i32 @llvm.lround.i32.bf16(bfloat poison)
+  call <2 x i32> @llvm.lround.v2i32.v2bf16(<2 x bfloat> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4bf16(<4 x bfloat> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8bf16(<8 x bfloat> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i32 @llvm.lround.i32.f32(float poison)
+  call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4f32(<4 x float> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8f32(<8 x float> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16f32(<16 x float> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f32(<vscale x 16 x float> poison)
+  call i32 @llvm.lround.i32.f64(double poison)
+  call <2 x i32> @llvm.lround.v2i32.v2f64(<2 x double> poison)
+  call <4 x i32> @llvm.lround.v4i32.v4f64(<4 x double> poison)
+  call <8 x i32> @llvm.lround.v8i32.v8f64(<8 x double> poison)
+  call <16 x i32> @llvm.lround.v16i32.v16f64(<16 x double> poison)
+  call <vscale x 1 x i32> @llvm.lround.nxv1i32.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i32> @llvm.lround.nxv2i32.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i32> @llvm.lround.nxv4i32.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i32> @llvm.lround.nxv8i32.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i32> @llvm.lround.nxv16i32.nxv16f64(<vscale x 16 x double> poison)
+  call i64 @llvm.lround.i64.bf16(bfloat poison)
+  call <2 x i64> @llvm.lround.v2i64.v2bf16(<2 x bfloat> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4bf16(<4 x bfloat> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8bf16(<8 x bfloat> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i64 @llvm.lround.i64.f32(float poison)
+  call <2 x i64> @llvm.lround.v2i64.v2f32(<2 x float> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4f32(<4 x float> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8f32(<8 x float> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16f32(<16 x float> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+  call i64 @llvm.lround.i64.f64(double poison)
+  call <2 x i64> @llvm.lround.v2i64.v2f64(<2 x double> poison)
+  call <4 x i64> @llvm.lround.v4i64.v4f64(<4 x double> poison)
+  call <8 x i64> @llvm.lround.v8i64.v8f64(<8 x double> poison)
+  call <16 x i64> @llvm.lround.v16i64.v16f64(<16 x double> poison)
+  call <vscale x 1 x i64> @llvm.lround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i64> @llvm.lround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.lround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.lround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.lround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+  ret void
+}
+
+define void @llround() {
+; CHECK-LABEL: 'llround'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = call i64 @llvm.llround.i64.bf16(bfloat poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %3 = call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %4 = call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %5 = call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %6 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %7 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %8 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %9 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %10 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %11 = call i64 @llvm.llround.i64.f32(float poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %12 = call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %13 = call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %14 = call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %15 = call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %16 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %17 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %18 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %19 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %20 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %21 = call i64 @llvm.llround.i64.f64(double poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %22 = call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %23 = call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %24 = call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %25 = call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %26 = call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %27 = call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %28 = call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %29 = call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+; CHECK-NEXT:  Cost Model: Invalid cost for instruction: %30 = call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call i64 @llvm.llround.i64.bf16(bfloat poison)
+  call <2 x i64> @llvm.llround.v2i64.v2bf16(<2 x bfloat> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4bf16(<4 x bfloat> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8bf16(<8 x bfloat> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16bf16(<16 x bfloat> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1bf16(<vscale x 1 x bfloat> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2bf16(<vscale x 2 x bfloat> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4bf16(<vscale x 4 x bfloat> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8bf16(<vscale x 8 x bfloat> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16bf16(<vscale x 16 x bfloat> poison)
+  call i64 @llvm.llround.i64.f32(float poison)
+  call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f32(<vscale x 1 x float> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f32(<vscale x 2 x float> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f32(<vscale x 4 x float> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f32(<vscale x 8 x float> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f32(<vscale x 16 x float> poison)
+  call i64 @llvm.llround.i64.f64(double poison)
+  call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> poison)
+  call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> poison)
+  call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> poison)
+  call <16 x i64> @llvm.llround.v16i64.v16f64(<16 x double> poison)
+  call <vscale x 1 x i64> @llvm.llround.nxv1i64.nxv1f64(<vscale x 1 x double> poison)
+  call <vscale x 2 x i64> @llvm.llround.nxv2i64.nxv2f64(<vscale x 2 x double> poison)
+  call <vscale x 4 x i64> @llvm.llround.nxv4i64.nxv4f64(<vscale x 4 x double> poison)
+  call <vscale x 8 x i64> @llvm.llround.nxv8i64.nxv8f64(<vscale x 8 x double> poison)
+  call <vscale x 16 x i64> @llvm.llround.nxv16i64.nxv16f64(<vscale x 16 x double> poison)
+  ret void
+}
+
 define void @nearbyint() {
 ; CHECK-LABEL: 'nearbyint'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %1 = call bfloat @llvm.nearbyint.bf16(bfloat undef)
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
index b73abbfeba08..6640ff9793a3 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-exact-vlen.ll
@@ -151,21 +151,21 @@ define void @concat() vscale_range(2,2) {
 
 define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-LABEL: 'insert_subvec'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_2_0 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_2_0 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_2_1 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_2_0 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_0 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_1 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_2 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_3 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_05 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_4_0 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_0 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_1 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_2 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_3 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_05 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_2_0 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2_0 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2_1 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i16_2_0 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_0 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_1 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_2 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_3 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
@@ -175,7 +175,7 @@ define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_3 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_05 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_1 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_1 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
@@ -190,21 +190,21 @@ define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'insert_subvec'
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_2_0 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_2_0 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8_2_1 = shufflevector <4 x i8> poison, <4 x i8> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_2_0 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_0 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_1 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_2 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_3 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8_2_05 = shufflevector <8 x i8> poison, <8 x i8> poison, <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16i8_4_0 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_0 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_1 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_2 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_3 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8_4_05 = shufflevector <16 x i8> poison, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_2_0 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2_0 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_2_1 = shufflevector <4 x i16> poison, <4 x i16> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i16_2_0 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_0 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_1 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_2 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16_2_3 = shufflevector <8 x i16> poison, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
@@ -214,7 +214,7 @@ define void @insert_subvec() vscale_range(2,2) {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_2 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_3 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16_4_05 = shufflevector <16 x i16> poison, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 18, i32 19, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_0 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_2_1 = shufflevector <4 x i32> poison, <4 x i32> poison, <4 x i32> <i32 4, i32 5, i32 0, i32 1>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_0 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32_2_1 = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
index ccfb5a9d42e7..f4e29e15111e 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-insert_subvector.ll
@@ -9,13 +9,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -23,13 +23,13 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x double> %src128, <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x double> %src128, <2 x double> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x double> %src256, <4 x double> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x double> %src256, <4 x double> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x double> %src512, <8 x double> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0123 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x double> %src512, <8 x double> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -53,13 +53,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -67,13 +67,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <2 x i64> %src128, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <2 x i64> %src128, <2 x i64> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <4 x i64> %src256, <4 x i64> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <4 x i64> %src256, <4 x i64> %src128_256, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <8 x i64> %src512, <8 x i64> %src128_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0123 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <8 x i64> %src512, <8 x i64> %src256_512, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -100,15 +100,15 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -116,11 +116,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_AB = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -131,15 +131,15 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x float> %src128, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x float> %src128, <4 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x float> %src256, <8 x float> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x float> %src128, <4 x float> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <8 x float> %src256, <8 x float> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x float> %src256, <8 x float> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -147,11 +147,11 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_AB = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_EF = shufflevector <16 x float> %src512, <16 x float> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0123 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01234567 = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x float> %src512, <16 x float> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -197,15 +197,15 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -213,11 +213,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -228,15 +228,15 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -244,11 +244,11 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -298,15 +298,15 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -314,11 +314,11 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -333,15 +333,15 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -349,11 +349,11 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
@@ -413,15 +413,15 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -429,11 +429,11 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -453,15 +453,15 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -469,11 +469,11 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/RISCV/shuffle-select.ll b/llvm/test/Analysis/CostModel/RISCV/shuffle-select.ll
index 0fd1cfc908d9..2e17818c10d0 100644
--- a/llvm/test/Analysis/CostModel/RISCV/shuffle-select.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/shuffle-select.ll
@@ -6,11 +6,11 @@
 
 define <8 x i8> @select_start_v8i8(<8 x i8> %v, <8 x i8> %w) {
 ; CHECK-LABEL: 'select_start_v8i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %res
 ;
 ; CHECK-SIZE-LABEL: 'select_start_v8i8'
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i8> %res
 ;
   %res = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
@@ -32,11 +32,11 @@ define <8 x i8> @select_non_contiguous_v8i8(<8 x i8> %v, <8 x i8> %w) {
 
 define <8 x i64> @select_start_v8i64(<8 x i64> %v, <8 x i64> %w) {
 ; CHECK-LABEL: 'select_start_v8i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = shufflevector <8 x i64> %v, <8 x i64> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = shufflevector <8 x i64> %v, <8 x i64> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %res
 ;
 ; CHECK-SIZE-LABEL: 'select_start_v8i64'
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = shufflevector <8 x i64> %v, <8 x i64> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = shufflevector <8 x i64> %v, <8 x i64> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %res
 ;
   %res = shufflevector <8 x i64> %v, <8 x i64> %w, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll b/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
index 033537d93bf7..777fb5b2f57d 100644
--- a/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/alternate-shuffle-cost.ll
@@ -39,21 +39,9 @@ define <2 x i32> @test_v2i32(<2 x i32> %a, <2 x i32> %b) {
 }
 
 define <2 x float> @test_v2f32(<2 x float> %a, <2 x float> %b) {
-; SSE2-LABEL: 'test_v2f32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; SSSE3-LABEL: 'test_v2f32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; SSE42-LABEL: 'test_v2f32'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; AVX-LABEL: 'test_v2f32'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
+; CHECK-LABEL: 'test_v2f32'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
 ;
   %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 0, i32 3>
   ret <2 x float> %1
@@ -81,21 +69,9 @@ define <2 x i32> @test_v2i32_2(<2 x i32> %a, <2 x i32> %b) {
 }
 
 define <2 x float> @test_v2f32_2(<2 x float> %a, <2 x float> %b) {
-; SSE2-LABEL: 'test_v2f32_2'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; SSSE3-LABEL: 'test_v2f32_2'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; SSE42-LABEL: 'test_v2f32_2'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
-;
-; AVX-LABEL: 'test_v2f32_2'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
+; CHECK-LABEL: 'test_v2f32_2'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %1
 ;
   %1 = shufflevector <2 x float> %a, <2 x float> %b, <2 x i32> <i32 2, i32 1>
   ret <2 x float> %1
@@ -317,9 +293,13 @@ define <4 x double> @test_v4f64_2(<4 x double> %a, <4 x double> %b) {
 }
 
 define <4 x double> @test_v4f64_3(<4 x double> %a, <4 x double> %b) {
-; CHECK-LABEL: 'test_v4f64_3'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %1
+; SSE-LABEL: 'test_v4f64_3'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %1
+;
+; AVX-LABEL: 'test_v4f64_3'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %1
 ;
   %1 = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
   ret <4 x double> %1
@@ -778,3 +758,22 @@ define <32 x i8> @test_v32i8_3(<32 x i8> %a, <32 x i8> %b) {
   %1 = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 36, i32 5, i32 38, i32 7, i32 40, i32 9, i32 42, i32 11, i32 44, i32 13, i32 46, i32 15, i32 48, i32 17, i32 50, i32 19, i32 52, i32 21, i32 54, i32 23, i32 56, i32 25, i32 58, i32 27, i32 60, i32 61, i32 62, i32 63>
   ret <32 x i8> %1
 }
+
+; Treat all undef/poison shuffle masks as free.
+define <2 x i32> @test_v2i32_poison(<2 x i32> %a0, <2 x i32> %a1) {
+; CHECK-LABEL: 'test_v2i32_poison'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s = shufflevector <2 x i32> %a0, <2 x i32> %a1, <2 x i32> poison
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %s
+;
+  %s = shufflevector <2 x i32> %a0, <2 x i32> %a1, <2 x i32> poison
+  ret <2 x i32> %s
+}
+
+define <4 x float> @test_v4f32_v2f32_poison(<2 x float> %a0, <2 x float> %a1) {
+; CHECK-LABEL: 'test_v4f32_v2f32_poison'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %s = shufflevector <2 x float> %a0, <2 x float> %a1, <4 x i32> poison
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %s
+;
+  %s = shufflevector <2 x float> %a0, <2 x float> %a1, <4 x i32> poison
+  ret <4 x float> %s
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
index 62e8e7c3e79c..40cbf26157e5 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-insert_subvector.ll
@@ -292,98 +292,36 @@ define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %sr
 }
 
 define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256, <16 x i32> %src512) {
-; SSE2-LABEL: 'test_vXi32'
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 0 for: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; SSE2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; SSSE3-LABEL: 'test_vXi32'
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; SSSE3-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; SSE42-LABEL: 'test_vXi32'
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 0 for: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-; SSE42-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; SSE-LABEL: 'test_vXi32'
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src64_256 = shufflevector <2 x i32> %src64, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src64_512 = shufflevector <2 x i32> %src64, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <4 x i32> %src128, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <4 x i32> %src128, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <8 x i32> %src256, <8 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V128_23 = shufflevector <4 x i32> %src128, <4 x i32> %src64_128, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V256_23 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V256_45 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V256_67 = shufflevector <8 x i32> %src256, <8 x i32> %src64_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V256_0123 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V256_4567 = shufflevector <8 x i32> %src256, <8 x i32> %src128_256, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V512_01 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V512_23 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V512_45 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V512_67 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V512_89 = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V512_AB = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V512_CD = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V512_EF = shufflevector <16 x i32> %src512, <16 x i32> %src64_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_0123 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_4567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_89AB = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_CDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_01234567 = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 0 for: %V512_89ABCDEF = shufflevector <16 x i32> %src512, <16 x i32> %src128_512, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; AVX1-LABEL: 'test_vXi32'
 ; AVX1-NEXT:  Cost Model: Found costs of 0 for: %src64_128 = shufflevector <2 x i32> %src64, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
@@ -524,13 +462,13 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:8 Lat:8 SizeLat:8 for: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:8 Lat:8 SizeLat:8 for: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:8 Lat:8 SizeLat:8 for: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:8 Lat:8 SizeLat:8 for: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:8 Lat:8 SizeLat:8 for: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -559,13 +497,13 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <8 x i16> %src128, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <8 x i16> %src128, <8 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <16 x i16> %src256, <16 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V64_01 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_23 = shufflevector <4 x i16> %src64, <4 x i16> %src32_64, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_23 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_45 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -637,7 +575,7 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V128_67 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V128_0123 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V128_4567 = shufflevector <8 x i16> %src128, <8 x i16> %src32_128, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V256_01 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 9 for: %V256_23 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 9 for: %V256_45 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 9 for: %V256_67 = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -645,11 +583,11 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; AVX1-NEXT:  Cost Model: Found costs of 15 for: %V256_AB = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 15 for: %V256_CD = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 15 for: %V256_EF = shufflevector <16 x i16> %src256, <16 x i16> %src64_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V256_0123 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 9 for: %V256_4567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V256_89AB = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 15 for: %V256_CDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V256_01234567 = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V256_89ABCDEF = shufflevector <16 x i16> %src256, <16 x i16> %src128_256, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; AVX1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -849,15 +787,15 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSE2-NEXT:  Cost Model: Found costs of 4 for: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of 7 for: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of 7 for: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of 7 for: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:  Cost Model: Found costs of 7 for: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -865,11 +803,11 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:11 CodeSize:13 Lat:13 SizeLat:13 for: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -889,15 +827,15 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_256 = shufflevector <16 x i8> %src128, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src128_512 = shufflevector <16 x i8> %src128, <16 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSSE3-NEXT:  Cost Model: Found costs of 0 for: %src256_512 = shufflevector <32 x i8> %src256, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V32_01 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V32_23 = shufflevector <4 x i8> %src32, <4 x i8> %src16_32, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V64_01 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_23 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_45 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_67 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V64_0123 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64_4567 = shufflevector <8 x i8> %src64, <8 x i8> %src32_64, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128_01 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_23 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 16, i32 17, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_45 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_67 = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -905,11 +843,11 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16x i
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_AB = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 16, i32 17, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_CD = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_EF = shufflevector <16 x i8> %src128, <16 x i8> %src16_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128_0123 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_4567 = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_89AB = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_CDEF = shufflevector <16 x i8> %src128, <16 x i8> %src32_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128_01234567 = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128_89ABCDEF = shufflevector <16 x i8> %src128, <16 x i8> %src64_128, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
 ; SSSE3-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-select.ll b/llvm/test/Analysis/CostModel/X86/shuffle-select.ll
index ba126f9028c7..0d20860d3b65 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-select.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-select.ll
@@ -28,14 +28,14 @@ define void @test_vXf64(<2 x double> %src128, <4 x double> %src256, <8 x double>
 ; XOP-LABEL: 'test_vXf64'
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <2 x double> %src128, <2 x double> %src128_1, <2 x i32> <i32 0, i32 3>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <4 x double> %src256, <4 x double> %src256_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; XOP-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <8 x double> %src512, <8 x double> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; XOP-NEXT:  Cost Model: Found costs of 2 for: %V512 = shufflevector <8 x double> %src512, <8 x double> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <16 x double> %src1024, <16 x double> %src1024_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; AVX-LABEL: 'test_vXf64'
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <2 x double> %src128, <2 x double> %src128_1, <2 x i32> <i32 0, i32 3>
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <4 x double> %src256, <4 x double> %src256_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <8 x double> %src512, <8 x double> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 2 for: %V512 = shufflevector <8 x double> %src512, <8 x double> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <16 x double> %src1024, <16 x double> %src1024_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -64,14 +64,14 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512,
 ; XOP-LABEL: 'test_vXi64'
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <2 x i64> %src128, <2 x i64> %src128_1, <2 x i32> <i32 0, i32 3>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <4 x i64> %src256, <4 x i64> %src256_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; XOP-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <8 x i64> %src512, <8 x i64> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; XOP-NEXT:  Cost Model: Found costs of 2 for: %V512 = shufflevector <8 x i64> %src512, <8 x i64> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <16 x i64> %src1024, <16 x i64> %src1024_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; AVX-LABEL: 'test_vXi64'
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <2 x i64> %src128, <2 x i64> %src128_1, <2 x i32> <i32 0, i32 3>
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <4 x i64> %src256, <4 x i64> %src256_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <8 x i64> %src512, <8 x i64> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 2 for: %V512 = shufflevector <8 x i64> %src512, <8 x i64> %src512_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <16 x i64> %src1024, <16 x i64> %src1024_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
@@ -90,29 +90,13 @@ define void @test_vXi64(<2 x i64> %src128, <4 x i64> %src256, <8 x i64> %src512,
 }
 
 define void @test_vXf32(<2 x float> %src64, <4 x float> %src128, <8 x float> %src256, <16 x float> %src512, <32 x float> %src1024, <2 x float> %src64_1, <4 x float> %src128_1, <8 x float> %src256_1, <16 x float> %src512_1, <32 x float> %src1024_1) {
-; SSE2-LABEL: 'test_vXf32'
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V64 = shufflevector <2 x float> %src64, <2 x float> %src64_1, <2 x i32> <i32 0, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V128 = shufflevector <4 x float> %src128, <4 x float> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x float> %src256, <8 x float> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x float> %src512, <16 x float> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x float> %src1024, <32 x float> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; SSE2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; SSSE3-LABEL: 'test_vXf32'
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V64 = shufflevector <2 x float> %src64, <2 x float> %src64_1, <2 x i32> <i32 0, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V128 = shufflevector <4 x float> %src128, <4 x float> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x float> %src256, <8 x float> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x float> %src512, <16 x float> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x float> %src1024, <32 x float> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; SSSE3-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; SSE42-LABEL: 'test_vXf32'
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <2 x float> %src64, <2 x float> %src64_1, <2 x i32> <i32 0, i32 3>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <4 x float> %src128, <4 x float> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x float> %src256, <8 x float> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x float> %src512, <16 x float> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x float> %src1024, <32 x float> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; SSE42-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; SSE-LABEL: 'test_vXf32'
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <2 x float> %src64, <2 x float> %src64_1, <2 x i32> <i32 0, i32 3>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <4 x float> %src128, <4 x float> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; SSE-NEXT:  Cost Model: Found costs of 2 for: %V256 = shufflevector <8 x float> %src256, <8 x float> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x float> %src512, <16 x float> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x float> %src1024, <32 x float> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; XOP-LABEL: 'test_vXf32'
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <2 x float> %src64, <2 x float> %src64_1, <2 x i32> <i32 0, i32 3>
@@ -150,7 +134,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE2-LABEL: 'test_vXi32'
 ; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V64 = shufflevector <2 x i32> %src64, <2 x i32> %src64_1, <2 x i32> <i32 0, i32 3>
 ; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V128 = shufflevector <4 x i32> %src128, <4 x i32> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 2 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x i32> %src512, <16 x i32> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x i32> %src1024, <32 x i32> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -158,7 +142,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSSE3-LABEL: 'test_vXi32'
 ; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V64 = shufflevector <2 x i32> %src64, <2 x i32> %src64_1, <2 x i32> <i32 0, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V128 = shufflevector <4 x i32> %src128, <4 x i32> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 2 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x i32> %src512, <16 x i32> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x i32> %src1024, <32 x i32> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSSE3-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -166,7 +150,7 @@ define void @test_vXi32(<2 x i32> %src64, <4 x i32> %src128, <8 x i32> %src256,
 ; SSE42-LABEL: 'test_vXi32'
 ; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <2 x i32> %src64, <2 x i32> %src64_1, <2 x i32> <i32 0, i32 3>
 ; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <4 x i32> %src128, <4 x i32> %src128_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSE42-NEXT:  Cost Model: Found costs of 2 for: %V256 = shufflevector <8 x i32> %src256, <8 x i32> %src256_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <16 x i32> %src512, <16 x i32> %src512_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE42-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <32 x i32> %src1024, <32 x i32> %src1024_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE42-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -207,7 +191,7 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSE2-LABEL: 'test_vXi16'
 ; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
 ; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -216,7 +200,7 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; SSSE3-LABEL: 'test_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -235,28 +219,19 @@ define void @test_vXi16(<2 x i16> %src32, <4 x i16> %src64, <8 x i16> %src128, <
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; XOP-NEXT:  Cost Model: Found costs of 3 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; XOP-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; XOP-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; AVX1-LABEL: 'test_vXi16'
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; AVX2-LABEL: 'test_vXi16'
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX-LABEL: 'test_vXi16'
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <4 x i16> %src64, <4 x i16> %src64_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <8 x i16> %src128, <8 x i16> %src128_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <16 x i16> %src256, <16 x i16> %src256_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <32 x i16> %src512, <32 x i16> %src512_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V1024 = shufflevector <64 x i16> %src1024, <64 x i16> %src1024_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; AVX512-LABEL: 'test_vXi16'
 ; AVX512-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <2 x i16> %src32, <2 x i16> %src32_1, <2 x i32> <i32 0, i32 3>
@@ -280,8 +255,8 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 ; SSE2-LABEL: 'test_vXi8'
 ; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
 ; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; SSE2-NEXT:  Cost Model: Found costs of 3 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSE2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; SSE2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -289,8 +264,8 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 ; SSSE3-LABEL: 'test_vXi8'
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
 ; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; SSSE3-NEXT:  Cost Model: Found costs of 3 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; SSSE3-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; SSSE3-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -309,27 +284,18 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; XOP-NEXT:  Cost Model: Found costs of 3 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; XOP-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; XOP-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
 ; XOP-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
-; AVX1-LABEL: 'test_vXi8'
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX1-NEXT:  Cost Model: Found costs of 3 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX1-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; AVX2-LABEL: 'test_vXi8'
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-; AVX2-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
-; AVX2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX-LABEL: 'test_vXi8'
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V32 = shufflevector <4 x i8> %src32, <4 x i8> %src32_1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V64 = shufflevector <8 x i8> %src64, <8 x i8> %src64_1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V128 = shufflevector <16 x i8> %src128, <16 x i8> %src128_1, <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V256 = shufflevector <32 x i8> %src256, <32 x i8> %src256_1, <32 x i32> <i32 32, i32 33, i32 34, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX-NEXT:  Cost Model: Found costs of 1 for: %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; AVX512-LABEL: 'test_vXi8'
 ; AVX512-NEXT:  Cost Model: Found costs of 1 for: %V16 = shufflevector <2 x i8> %src16, <2 x i8> %src16_1, <2 x i32> <i32 0, i32 3>
@@ -348,3 +314,6 @@ define void @test_vXi8(<2 x i8> %src16, <4 x i8> %src32, <8 x i8> %src64, <16 x
   %V512 = shufflevector <64 x i8> %src512, <64 x i8> %src512_1, <64 x i32> <i32 64, i32 65, i32 66, i32 67, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX1: {{.*}}
+; AVX2: {{.*}}
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 0366684a13b5..247a105940e6 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -144,6 +144,54 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
+define i8 @crc8.le.tc16(i16 %msg, i8 %checksum) {
+; CHECK-LABEL: 'crc8.le.tc16'
+; CHECK-NEXT:  Found little-endian CRC-8 loop with trip count 16
+; CHECK-NEXT:    Initial CRC: i8 %checksum
+; CHECK-NEXT:    Generating polynomial: 29
+; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i8 %crc.lshr, i8 %crc.xor
+; CHECK-NEXT:    Auxiliary data: i16 %msg
+; CHECK-NEXT:    Computed CRC lookup table:
+; CHECK-NEXT:  0 9 18 27 31 22 13 4 5 12 23 30 26 19 8 1
+; CHECK-NEXT:  10 3 24 17 21 28 7 14 15 6 29 20 16 25 2 11
+; CHECK-NEXT:  20 29 6 15 11 2 25 16 17 24 3 10 14 7 28 21
+; CHECK-NEXT:  30 23 12 5 1 8 19 26 27 18 9 0 4 13 22 31
+; CHECK-NEXT:  19 26 1 8 12 5 30 23 22 31 4 13 9 0 27 18
+; CHECK-NEXT:  25 16 11 2 6 15 20 29 28 21 14 7 3 10 17 24
+; CHECK-NEXT:  7 14 21 28 24 17 10 3 2 11 16 25 29 20 15 6
+; CHECK-NEXT:  13 4 31 22 18 27 0 9 8 1 26 19 23 30 5 12
+; CHECK-NEXT:  29 20 15 6 2 11 16 25 24 17 10 3 7 14 21 28
+; CHECK-NEXT:  23 30 5 12 8 1 26 19 18 27 0 9 13 4 31 22
+; CHECK-NEXT:  9 0 27 18 22 31 4 13 12 5 30 23 19 26 1 8
+; CHECK-NEXT:  3 10 17 24 28 21 14 7 6 15 20 29 25 16 11 2
+; CHECK-NEXT:  14 7 28 21 17 24 3 10 11 2 25 16 20 29 6 15
+; CHECK-NEXT:  4 13 22 31 27 18 9 0 1 8 19 26 30 23 12 5
+; CHECK-NEXT:  26 19 8 1 5 12 23 30 31 22 13 4 0 9 18 27
+; CHECK-NEXT:  16 25 2 11 15 6 29 20 21 28 7 14 10 3 24 17
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i8 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %data.trunc = trunc i16 %data to i8
+  %xor.crc.data = xor i8 %crc, %data.trunc
+  %and.crc.data = and i8 %xor.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i8 %and.crc.data, 0
+  %crc.lshr = lshr i8 %crc, 1
+  %crc.xor = xor i8 %crc.lshr, 29
+  %crc.next = select i1 %check.sb, i8 %crc.lshr, i8 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i8 %crc.next
+}
+
 define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
 ; CHECK-LABEL: 'crc16.be.tc8.crc.init.li'
 ; CHECK-NEXT:  Found big-endian CRC-16 loop with trip count 8
@@ -601,7 +649,7 @@ exit:                                              ; preds = %loop
 define i16 @not.crc.wrong.sb.check.const(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.wrong.sb.check.const'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Bad RHS of significant-bit-check
 ;
 entry:
   br label %loop
@@ -610,9 +658,8 @@ loop:                                              ; preds = %loop, %entry
   %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
   %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
   %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
-  %crc.lshr = lshr i16 %crc, 8
   %data.ext = zext i8 %data to i16
-  %xor.crc.data = xor i16 %crc.lshr, %data.ext
+  %xor.crc.data = xor i16 %crc, %data.ext
   %check.sb = icmp samesign ult i16 %xor.crc.data, 128
   %crc.shl = shl i16 %crc, 1
   %crc.xor = xor i16 %crc.shl, 258
@@ -649,10 +696,10 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
-define i16 @not.crc.excess.tc(i16 %msg, i16 %checksum) {
+define i16 @not.crc.excess.tc(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.excess.tc'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of result
+; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of data
 ;
 entry:
   br label %loop
@@ -660,16 +707,40 @@ entry:
 loop:                                              ; preds = %loop, %entry
   %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
   %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
-  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
-  %xor.crc.data = xor i16 %crc, %data
-  %and.crc.data = and i16 %xor.crc.data, 1
-  %data.next = lshr i16 %data, 1
-  %check.sb = icmp eq i16 %and.crc.data, 0
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc.trunc = trunc i16 %crc to i8
+  %xor.crc.data = xor i8 %crc.trunc, %data
+  %and.crc.data = and i8 %xor.crc.data, 1
+  %data.next = lshr i8 %data, 1
+  %check.sb = icmp eq i8 %and.crc.data, 0
   %crc.lshr = lshr i16 %crc, 1
   %crc.xor = xor i16 %crc.lshr, -24575
   %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
   %iv.next = add nuw nsw i8 %iv, 1
-  %exit.cond = icmp samesign ult i8 %iv, 31
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.init.arg.excess.tc(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.init.arg.excess.tc'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop iterations exceed bitwidth of data
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign ult i32 %iv, 31
   br i1 %exit.cond, label %loop, label %exit
 
 exit:                                              ; preds = %loop
@@ -838,10 +909,37 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
+define i16 @not.crc.bad.cast(i8 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.bad.cast'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Expected bottom 8 bits zero (????????00001011)
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %data = phi i8 [ %msg, %entry ], [ %data.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data.ext = zext i8 %data to i16
+  %xor.crc.data = xor i16 %crc, %data.ext
+  %check.sb = icmp slt i16 %xor.crc.data, 0
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 29
+  %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
+  %data.next = shl i8 %data, 1
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 7
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
 define i32 @not.crc.dead.msg.bad.use(i32 %checksum, i32 %msg) {
 ; CHECK-LABEL: 'not.crc.dead.msg.bad.use'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences not intertwined with XOR
 ;
 entry:
   br label %loop
@@ -869,7 +967,7 @@ exit:                                              ; preds = %loop
 define i16 @not.crc.dead.msg.no.use(i8 %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.dead.msg.no.use'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences have stray uses
 ;
 entry:
   br label %loop
@@ -898,7 +996,7 @@ exit:                                              ; preds = %loop
 define i32 @not.crc.dead.msg.wrong.op(i32 %checksum, i32 %msg) {
 ; CHECK-LABEL: 'not.crc.dead.msg.wrong.op'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Simple recurrence doesn't use conditional recurrence with XOR
+; CHECK-NEXT:  Reason: Recurrences not intertwined with XOR
 ;
 entry:
   br label %loop
@@ -922,6 +1020,120 @@ exit:                                              ; preds = %loop
   ret i32 %crc.next
 }
 
+define i16 @not.crc.dead.msg.xor.notin.select.chain(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.dead.msg.xor.notin.select.chain'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %or.crc.data = or i16 %crc, %data
+  %and.crc.data = and i16 %or.crc.data, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.bad.xor.crc.data(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.bad.xor.crc.data'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %mul.corrupt = mul i16 %xor.crc.data, 0
+  %xor.crc.data.corrupt = xor i16 %mul.corrupt, %crc
+  %and.crc.data = and i16 %xor.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.dead.msg.or.zero(i16 %msg, i16 %checksum) {
+; CHECK-LABEL: 'not.crc.dead.msg.or.zero'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Recurrences have stray uses
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %mul.corrupt = mul i16 %xor.crc.data, 0
+  %or.crc.data.corrupt = or i16 %mul.corrupt, %crc
+  %and.crc.data = and i16 %or.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.unknown.value(i16 %msg, i16 %checksum, i16 %corrupt) {
+; CHECK-LABEL: 'not.crc.unknown.value'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unknown Value
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %checksum, %entry ], [ %crc.next, %loop ]
+  %data = phi i16 [ %msg, %entry ], [ %data.next, %loop ]
+  %xor.crc.data = xor i16 %crc, %data
+  %xor.crc.data.corrupt = mul i16 %xor.crc.data, %corrupt
+  %and.crc.data = and i16 %xor.crc.data.corrupt, 1
+  %data.next = lshr i16 %data, 1
+  %check.sb = icmp eq i16 %and.crc.data, 0
+  %crc.lshr = lshr i16 %crc, 1
+  %crc.xor = xor i16 %crc.lshr, -24575
+  %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
+  %iv.next = add nuw nsw i8 %iv, 1
+  %exit.cond = icmp samesign ult i8 %iv, 15
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
 define i16 @not.crc.float.simple.recurrence(float %msg, i16 %checksum) {
 ; CHECK-LABEL: 'not.crc.float.simple.recurrence'
 ; CHECK-NEXT:  Did not find a hash algorithm
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
new file mode 100644
index 000000000000..0d0fc93b9e7e
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
@@ -0,0 +1,11 @@
+{
+    "Opcodes": {
+        "dummyOpc": [1, 2]
+    },
+    "Types": {
+        "dummyTy": [1, 2]
+    },
+    "Arguments": {
+        "dummyArg": [1, 2]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
index 0ff5fdd959e9..3b662ab70168 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_3D_vocab.json
@@ -1,7 +1,15 @@
 {
-    "alloca": [1, 2, 3],
-    "load": [4, 5, 6],
-    "store": [7, 8, 9],
-    "add": [10, 11, 12],
-    "mul": [13, 14, 15]
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyTy": [1, 1, 1]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
 }
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
index d6506f06bb32..3345aaf9c982 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_5D_vocab.json
@@ -1,11 +1,17 @@
 {
-    "alloca": [-0.1, -0.2, -0.3, 1, 2],
-    "load": [-0.4, -0.5, -0.6, 4, 5],
-    "store": [-0.7, -0.8, -0.9, 7, 8],
-    "add": [-1.0, -1.1, -1.2, 10, 11],
-    "mul": [12, 13, 14, -1.2, -1.3],
-    "integerTy": [0.2, 0.4, 0.6, 1, 0.5],
-    "pointer": [0, 1, 2, 3, 4],
-    "variable": [2, 4, 6, 8, 10],
-    "ret": [5, 6, 7, 8, 9]
+    "Opcodes": {
+        "alloca": [-0.1, -0.2, -0.3, 1, 2],
+        "load": [-0.4, -0.5, -0.6, 4, 5],
+        "store": [-0.7, -0.8, -0.9, 7, 8],
+        "add": [-1.0, -1.1, -1.2, 10, 11],
+        "mul": [12, 13, 14, -1.2, -1.3],
+        "ret": [5, 6, 7, 8, 9]
+    },
+    "Types": {
+        "integerTy": [0.2, 0.4, 0.6, 1, 0.5]
+    },
+    "Arguments": {
+        "pointer": [0, 1, 2, 3, 4],
+        "variable": [2, 4, 6, 8, 10]
+    }
 }
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json
new file mode 100644
index 000000000000..d6506f06bb32
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab1.json
@@ -0,0 +1,11 @@
+{
+    "alloca": [-0.1, -0.2, -0.3, 1, 2],
+    "load": [-0.4, -0.5, -0.6, 4, 5],
+    "store": [-0.7, -0.8, -0.9, 7, 8],
+    "add": [-1.0, -1.1, -1.2, 10, 11],
+    "mul": [12, 13, 14, -1.2, -1.3],
+    "integerTy": [0.2, 0.4, 0.6, 1, 0.5],
+    "pointer": [0, 1, 2, 3, 4],
+    "variable": [2, 4, 6, 8, 10],
+    "ret": [5, 6, 7, 8, 9]
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json
new file mode 100644
index 000000000000..ae5ab0ffcdd0
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab2.json
@@ -0,0 +1,12 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json
new file mode 100644
index 000000000000..127d0995eb73
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab3.json
@@ -0,0 +1,12 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyType": [1, 1, 1]
+    }
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json
new file mode 100644
index 000000000000..c3e19ec05d6c
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/Inputs/incorrect_vocab4.json
@@ -0,0 +1,16 @@
+{
+    "Opcodes": {
+        "alloca": [1, 2, 3],
+        "load": [4, 5, 6],
+        "store": [7, 8, 9],
+        "add": [10, 11, 12],
+        "mul": [13, 14, 15]
+    },
+    "Types": {
+        "dummyTy": [1, 1, 1, 1, 1, 1]
+    },
+    "Arguments": {
+        "dummyArg": [1, 1, 1]
+    }
+}
+
diff --git a/llvm/test/Analysis/IR2Vec/basic.ll b/llvm/test/Analysis/IR2Vec/basic.ll
index b10e735a5458..ed65ff46dc95 100644
--- a/llvm/test/Analysis/IR2Vec/basic.ll
+++ b/llvm/test/Analysis/IR2Vec/basic.ll
@@ -1,6 +1,10 @@
 ; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=3D-CHECK
 ; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_5D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=5D-CHECK
-
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab1.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB1-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab2.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB2-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab3.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB3-CHECK
+; RUN: not opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/incorrect_vocab4.json %s 2>&1 | FileCheck %s -check-prefix=INCORRECT-VOCAB4-CHECK
+ 
 define dso_local i32 @abc(i32 %0, i32 %1) {
 entry:
   %3 = alloca i32, align 4
@@ -48,3 +52,11 @@ entry:
 ; 5D-CHECK-NEXT: Instruction:   %7 = mul nsw i32 %5, %6 [ 12.90  14.80  16.70  2.50  2.95 ]
 ; 5D-CHECK-NEXT: Instruction:   %8 = add nsw i32 %4, %7 [ -0.10  0.70  1.50  13.70  15.25 ]
 ; 5D-CHECK-NEXT: Instruction:   ret i32 %8 [ 5.40  6.80  8.20  9.60  11.00 ]
+
+; INCORRECT-VOCAB1-CHECK: error: Error reading vocabulary: Missing 'Opcodes' section in vocabulary file
+
+; INCORRECT-VOCAB2-CHECK: error: Error reading vocabulary: Missing 'Types' section in vocabulary file
+
+; INCORRECT-VOCAB3-CHECK: error: Error reading vocabulary: Missing 'Arguments' section in vocabulary file
+
+; INCORRECT-VOCAB4-CHECK: error: Error reading vocabulary: Vocabulary sections have different dimensions
diff --git a/llvm/test/Analysis/IR2Vec/dbg-inst.ll b/llvm/test/Analysis/IR2Vec/dbg-inst.ll
new file mode 100644
index 000000000000..0f486b0ba6a5
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/dbg-inst.ll
@@ -0,0 +1,13 @@
+; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s
+
+define void @bar2(ptr %foo)  {
+  store i32 0, ptr %foo, align 4
+  tail call void @llvm.dbg.value(metadata !{}, i64 0, metadata !{}, metadata !{})
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, i64, metadata, metadata) nounwind readnone
+
+; CHECK: Instruction vectors:
+; CHECK-NEXT: Instruction:   store i32 0, ptr %foo, align 4 [ 7.00  8.00  9.00 ]
+; CHECK-NEXT: Instruction:   ret void [ 0.00  0.00  0.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/unreachable.ll b/llvm/test/Analysis/IR2Vec/unreachable.ll
new file mode 100644
index 000000000000..370fe6881d6c
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/unreachable.ll
@@ -0,0 +1,42 @@
+; RUN: opt -passes='print<ir2vec>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_3D_vocab.json %s 2>&1 | FileCheck %s
+
+define dso_local i32 @abc(i32 noundef %a, i32 noundef %b) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %cmp = icmp sgt i32 %0, %1
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, ptr %b.addr, align 4
+  store i32 %2, ptr %retval, align 4
+  br label %return
+
+if.else:                                          ; preds = %entry
+  %3 = load i32, ptr %a.addr, align 4
+  store i32 %3, ptr %retval, align 4
+  br label %return
+
+unreachable:                                      ; Unreachable
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.else, %if.then
+  %4 = load i32, ptr %retval, align 4
+  ret i32 %4
+}
+
+; CHECK: Basic block vectors:
+; CHECK-NEXT: Basic block: entry:
+; CHECK-NEXT:  [ 25.00 32.00 39.00 ]
+; CHECK-NEXT: Basic block: if.then:
+; CHECK-NEXT:  [ 11.00 13.00 15.00 ]
+; CHECK-NEXT: Basic block: if.else:
+; CHECK-NEXT:  [ 11.00 13.00 15.00 ]
+; CHECK-NEXT: Basic block: return:
+; CHECK-NEXT:  [ 4.00 5.00 6.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/vocab-test.ll b/llvm/test/Analysis/IR2Vec/vocab-test.ll
new file mode 100644
index 000000000000..fbccb504a232
--- /dev/null
+++ b/llvm/test/Analysis/IR2Vec/vocab-test.ll
@@ -0,0 +1,20 @@
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json %s 2>&1 | FileCheck %s -check-prefix=VOCAB-CHECK
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json -ir2vec-opc-weight=0.5 -ir2vec-type-weight=0.5 -ir2vec-arg-weight=0.5 %s 2>&1 | FileCheck %s -check-prefix=WT1-VOCAB-CHECK
+; RUN: opt -passes='print<ir2vec-vocab>' -o /dev/null -ir2vec-vocab-path=%S/Inputs/dummy_2D_vocab.json -ir2vec-opc-weight=0.1 -ir2vec-type-weight=0 -ir2vec-arg-weight=0 %s 2>&1 | FileCheck %s -check-prefix=WT2-VOCAB-CHECK
+ 
+define dso_local void @test() {
+  entry:
+    ret void
+}
+
+; VOCAB-CHECK: Key: dummyArg: [ 0.20 0.40 ]
+; VOCAB-CHECK-NEXT: Key: dummyOpc: [ 1.00 2.00 ]
+; VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.50 1.00 ]
+
+; WT1-VOCAB-CHECK: Key: dummyArg: [ 0.50 1.00 ]
+; WT1-VOCAB-CHECK-NEXT: Key: dummyOpc: [ 0.50 1.00 ]
+; WT1-VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.50 1.00 ]
+
+; WT2-VOCAB-CHECK: Key: dummyArg: [ 0.00 0.00 ]
+; WT2-VOCAB-CHECK-NEXT: Key: dummyOpc: [ 0.10 0.20 ]
+; WT2-VOCAB-CHECK-NEXT: Key: dummyTy: [ 0.00 0.00 ]
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll b/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll
new file mode 100644
index 000000000000..2df451d5df73
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/dependences-i128-inductions.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -disable-output -passes='print<access-info>' %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-p:128:128:128"
+
+; Tests with 128 bit inductions and 128 bit pointer width and index size.
+
+
+define void @backward_i128(ptr %A, i128 %n) {
+; CHECK-LABEL: 'backward_i128'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Backward loop carried data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Backward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  %l = load i32, ptr %gep.A
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  store i32 %l, ptr %gep.A.1
+  %iv.next = add i128 %iv, 1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @forward_i128(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_i128'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A.1, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  %l = load i32, ptr %gep.A.1
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  store i32 %l, ptr %gep.A
+  %iv.next = add i128 %iv, 1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @forward_negative_step(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_negative_step'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 36893488147419103232, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  %l = load i32, ptr %gep.A
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  store i32 %l, ptr %gep.A.1
+  %iv.next = add nsw i128 %iv, -1
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; The induction step is i128 2^63 + 1, which does not fit into a 64 bit signed
+; integer.
+define void @forward_i128_step_63bit_plus_one(ptr %A, i128 %n) {
+; CHECK-LABEL: 'forward_i128_step_63bit_plus_one'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unsafe indirect dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        IndirectUnsafe:
+; CHECK-NEXT:            %l = load i32, ptr %gep.A.1, align 4 ->
+; CHECK-NEXT:            store i32 %l, ptr %gep.A, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i128 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.1 = add nuw nsw i128 %iv, 1
+  %gep.A.1 = getelementptr inbounds i32, ptr %A, i128 %iv.1
+  %l = load i32, ptr %gep.A.1
+  %gep.A = getelementptr inbounds i32, ptr %A, i128 %iv
+  store i32 %l, ptr %gep.A
+  %iv.next = add i128 %iv, 9223372036854775809
+  %ec = icmp eq i128 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
index d4cd61bfde2f..1dc8d4a7e73f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll
@@ -504,3 +504,123 @@ e.0:
 e.1:
   ret i32 1
 }
+
+define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption(ptr %A, ptr %B) {
+; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption'
+; CHECK-NEXT:    loop.header:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %B High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%B,+,4}<nuw><%loop.header>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %A High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 2000) ]
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 2000) ]
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+  %l = load i32, ptr %gep.A, align 4
+  store i32 0, ptr %gep.B, align 4
+  %cntable.c.1 = icmp ult i64 %iv, 1000
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %cntable.c.1, label %b2, label %e.1
+
+b2:
+  %uncntable.c.0 = icmp eq i32 %l, 0
+  br i1 %uncntable.c.0, label %e.2, label %b3
+
+b3:
+  %cntable.c.2 = icmp eq i64 %iv.next, 500
+  br i1 %cntable.c.2, label %cleanup4, label %latch
+
+latch:
+  br label %loop.header
+
+cleanup4:
+  ret void
+
+e.1:
+  ret void
+
+e.2:
+  ret void
+}
+
+define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small(ptr %A, ptr %B) {
+; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small'
+; CHECK-NEXT:    loop.header:
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-NEXT:        Group GRP0:
+; CHECK-NEXT:          (Low: %B High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%B,+,4}<nuw><%loop.header>
+; CHECK-NEXT:        Group GRP1:
+; CHECK-NEXT:          (Low: %A High: inttoptr (i64 -1 to ptr))
+; CHECK-NEXT:            Member: {%A,+,4}<nuw><%loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 1999) ]
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 1999) ]
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv
+  %l = load i32, ptr %gep.A, align 4
+  store i32 0, ptr %gep.B, align 4
+  %cntable.c.1 = icmp ult i64 %iv, 1000
+  %iv.next = add nuw nsw i64 %iv, 1
+  br i1 %cntable.c.1, label %b2, label %e.1
+
+b2:
+  %uncntable.c.0 = icmp eq i32 %l, 0
+  br i1 %uncntable.c.0, label %e.2, label %b3
+
+b3:
+  %cntable.c.2 = icmp eq i64 %iv.next, 500
+  br i1 %cntable.c.2, label %cleanup4, label %latch
+
+latch:
+  br label %loop.header
+
+cleanup4:
+  ret void
+
+e.1:
+  ret void
+
+e.2:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
index 595fb021ec0a..63abd4ef70d6 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll
@@ -320,28 +320,34 @@ exit:
 define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) {
 ; CHECK-LABEL: 'retry_after_dep_check_with_unknown_offset'
 ; CHECK-NEXT:    loop:
-; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
-; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Memory dependences are safe with run-time checks
 ; CHECK-NEXT:      Dependences:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            %l.A = load float, ptr %A, align 4 ->
-; CHECK-NEXT:            store float 0.000000e+00, ptr %A.100.iv.offset.3, align 4
-; CHECK-EMPTY:
-; CHECK-NEXT:        Unknown:
-; CHECK-NEXT:            %l.A = load float, ptr %A, align 4 ->
-; CHECK-NEXT:            store float %l.A, ptr %A.100.iv, align 8
-; CHECK-EMPTY:
 ; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Check 0:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT:        Against group GRP1:
+; CHECK-NEXT:          %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT:      Check 1:
+; CHECK-NEXT:        Comparing group GRP0:
+; CHECK-NEXT:          %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv
+; CHECK-NEXT:        Against group GRP2:
+; CHECK-NEXT:        ptr %A
+; CHECK-NEXT:      Check 2:
+; CHECK-NEXT:        Comparing group GRP1:
+; CHECK-NEXT:          %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3
+; CHECK-NEXT:        Against group GRP2:
+; CHECK-NEXT:        ptr %A
 ; CHECK-NEXT:      Grouped accesses:
 ; CHECK-NEXT:        Group GRP0:
-; CHECK-NEXT:          (Low: %A High: (4 + %A))
-; CHECK-NEXT:            Member: %A
+; CHECK-NEXT:          (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
+; CHECK-NEXT:            Member: {(100 + %A),+,8}<%loop>
 ; CHECK-NEXT:        Group GRP1:
 ; CHECK-NEXT:          (Low: (100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A) High: (96 + (16 * (zext i32 %offset to i64))<nuw><nsw> + %A))
 ; CHECK-NEXT:            Member: {(100 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A),+,8}<%loop>
 ; CHECK-NEXT:        Group GRP2:
-; CHECK-NEXT:          (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64))<nuw><nsw> + %A))
-; CHECK-NEXT:            Member: {(100 + %A),+,8}<%loop>
+; CHECK-NEXT:          (Low: %A High: (4 + %A))
+; CHECK-NEXT:            Member: %A
 ; CHECK-EMPTY:
 ; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
 ; CHECK-NEXT:      SCEV assumptions:
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 1028cc9ebb34..bd7464577b7d 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -261,6 +261,70 @@ bb:
   ret void
 }
 
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr4_b64_v2i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) %addr)
+define amdgpu_kernel void @global_load_tr6_b96_v3i32(ptr addrspace(1) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1) %addr)
+  store <3 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr8_b64_v2i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr4_b64_v2i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3) %addr)
+  store <2 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr6_b96_v3i32(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3) %addr)
+  store <3 x i32> %tmp0, ptr addrspace(1) %out, align 8
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3) %addr)
+  store <8 x i16> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3) %addr)
+  store <8 x half> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
+; CHECK: DIVERGENT: %tmp0 = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) %addr)
+define amdgpu_kernel void @ds_load_tr16_b128_v8bf16(ptr addrspace(3) %addr, ptr addrspace(1) %out) {
+bb:
+  %tmp0 = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3) %addr)
+  store <8 x bfloat> %tmp0, ptr addrspace(1) %out, align 16
+  ret void
+}
+
 declare <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3))
 
 ; CHECK: DIVERGENT: %tmp0 = call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) %gep)
@@ -563,6 +627,14 @@ declare i32 @llvm.amdgcn.global.load.tr.b64.i32(ptr addrspace(1))
 declare <4 x i16> @llvm.amdgcn.global.load.tr.b128.v4i16(ptr addrspace(1))
 declare <4 x half> @llvm.amdgcn.global.load.tr.b128.v4f16(ptr addrspace(1))
 declare <4 x bfloat> @llvm.amdgcn.global.load.tr.b128.v4bf16(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32(ptr addrspace(1))
+declare <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32(ptr addrspace(1))
+declare <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32(ptr addrspace(3))
+declare <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32(ptr addrspace(3))
+declare <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32(ptr addrspace(3))
+declare <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16(ptr addrspace(3))
+declare <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3))
+declare <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3))
 
 declare i32 @llvm.amdgcn.dead.i32()
 
diff --git a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
index 31b7101fba11..33f2758a4b18 100644
--- a/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
+++ b/llvm/test/Assembler/aarch64-intrinsics-attributes.ll
@@ -22,5 +22,15 @@ declare <4 x i32> @llvm.aarch64.neon.shadd.v4i32(<4 x i32>, <4 x i32>)
 ; CHECK: declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_READNONE_WILLRETURN]]
 declare <vscale x 4 x i32> @llvm.aarch64.sve.dup.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, i32)
 
+; CHECK: declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st2.v4i32.p0(<4 x i32>, <4 x i32>, ptr)
+
+; CHECK: declare void @llvm.aarch64.neon.st2lane.v4i32.p0(<4 x i32>, <4 x i32>, i64, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st2lane.v4i32.p0(<4 x i32>, <4 x i32>, i64, ptr)
+
+; CHECK: declare void @llvm.aarch64.neon.st1x2.v8i16.p0(<8 x i16>, <8 x i16>, ptr captures(none)) [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN:#[0-9]+]]
+declare void @llvm.aarch64.neon.st1x2.v8i16.p0(<8 x i16>, <8 x i16>, ptr)
+
 ; CHECK: attributes [[NOFREE_NOUNWIND_WILLRETURN]] = { nofree nounwind willreturn }
 ; CHECK: attributes [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_READNONE_WILLRETURN]] = { nocallback nofree nosync nounwind willreturn memory(none) }
+; CHECK: attributes [[NO_CALLBACK_NOFREE_NOSYNC_NOUNWIND_WRITEONLY_WILLRETURN]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 7dd86a8c0eb1..8c1a76365e1b 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -567,6 +567,11 @@ define void @captures(ptr captures(address) %p) {
   ret void
 }
 
+; CHECK: define void @dead_on_return(ptr dead_on_return %p)
+define void @dead_on_return(ptr dead_on_return %p) {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/Bitcode/debug-loc-again.ll b/llvm/test/Bitcode/debug-loc-again.ll
index 3bfbd1ff30b3..d92720caa926 100644
--- a/llvm/test/Bitcode/debug-loc-again.ll
+++ b/llvm/test/Bitcode/debug-loc-again.ll
@@ -1,9 +1,9 @@
 ; RUN: llvm-as < %s | llvm-bcanalyzer -dump | FileCheck %s -check-prefix=BC
 ; PR23436: Actually emit DEBUG_LOC_AGAIN records.
 
-; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC abbrevid
 ; BC: <DEBUG_LOC_AGAIN/>
-; BC: <DEBUG_LOC op
+; BC: <DEBUG_LOC abbrevid
 ; BC: <DEBUG_LOC_AGAIN/>
 
 ; RUN: llvm-as < %s | llvm-dis | FileCheck %s
diff --git a/llvm/test/Bitcode/function-encoding-rel-operands.ll b/llvm/test/Bitcode/function-encoding-rel-operands.ll
index 84e83c11de36..d107a9c5a313 100644
--- a/llvm/test/Bitcode/function-encoding-rel-operands.ll
+++ b/llvm/test/Bitcode/function-encoding-rel-operands.ll
@@ -48,7 +48,7 @@ define double @test_float_binops(i32 %a) nounwind {
 ; between literals and the formal parameters.
 ; CHECK: INST_GEP {{.*}}
 ; CHECK: INST_LOAD {{.*}}op0=1 {{.*}}
-; CHECK: INST_CMP2 op0=1 {{.*}}
+; CHECK: INST_CMP2 {{.*}}op0=1 {{.*}}
 ; CHECK: INST_RET {{.*}}op0=1
 define i1 @test_load(i32 %a, {i32, i32}* %ptr) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
index a2e20caf43ff..b21046d6049e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
@@ -56,7 +56,7 @@
   }
 
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
   attributes #2 = { optsize }
   attributes #3 = { minsize }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
index 27d09b236256..57d031dcd53c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
@@ -47,7 +47,7 @@
     ret void
   }
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
index 9ed1e2d9eee3..f8d2bf387ffc 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
@@ -50,7 +50,7 @@
 
   declare void @llvm.stackprotector(ptr, ptr) #2
 
-  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-use-vector-truncate.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-use-vector-truncate.mir
new file mode 100644
index 000000000000..3d0d591bccc1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-use-vector-truncate.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
+---
+name:            test_foldable_s32
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_foldable_s32
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[SRC:%src]]:_(<2 x s32>) = COPY $x1
+    ; CHECK-NEXT: [[UNDEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[MID:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[SRC]](<2 x s32>), [[UNDEF]](<2 x s32>)
+    ; CHECK-NEXT: [[DST:%dst]]:_(<4 x s16>) = G_TRUNC [[MID]]
+    ; CHECK-NEXT: $x0 = COPY [[DST]]
+
+    %src:_(<2 x s32>) = COPY $x1
+    %a:_(s32), %b:_(s32) = G_UNMERGE_VALUES %src:_(<2 x s32>)
+    %T_a:_(s16) = G_TRUNC %a:_(s32)
+    %T_b:_(s16) = G_TRUNC %b:_(s32)
+    %Undef:_(s16) = G_IMPLICIT_DEF
+    %dst:_(<4 x s16>) = G_BUILD_VECTOR %T_a:_(s16), %T_b:_(s16), %Undef:_(s16), %Undef:_(s16)
+    $x0 = COPY %dst
+...
+---
+name:            test_unfoldable_s32
+body:             |
+  bb.0:
+    liveins: $w0, $x1
+
+    ; CHECK-LABEL: name: test_unfoldable_s32
+    ; CHECK: liveins: $w0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[SRC:%src]]:_(<2 x s32>) = COPY $x1
+    ; CHECK-NEXT: [[A:%a]]:_(s16), [[B:%b]]:_(s16), [[C:%c]]:_(s16), %d:_(s16) = G_UNMERGE_VALUES [[SRC]](<2 x s32>)
+    ; CHECK-NEXT: [[T_A:%T_a]]:_(s8) = G_TRUNC [[A]](s16)
+    ; CHECK-NEXT: [[T_B:%T_b]]:_(s8) = G_TRUNC [[B]](s16)
+    ; CHECK-NEXT: [[T_C:%T_c]]:_(s8) = G_TRUNC [[C]](s16)
+    ; CHECK-NEXT: [[UNDEF:%Undef]]:_(s8) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: [[DST:%dst]]:_(<4 x s8>) = G_BUILD_VECTOR [[T_A]](s8), [[T_B]](s8), [[T_C]](s8), [[UNDEF]](s8)
+    ; CHECK-NEXT: $w0 = COPY [[DST]](<4 x s8>)
+    %src:_(<2 x s32>) = COPY $x1
+    %a:_(s16), %b:_(s16), %c:_(s16), %d:_(s16) = G_UNMERGE_VALUES %src:_(<2 x s32>)
+    %T_a:_(s8) = G_TRUNC %a:_(s16)
+    %T_b:_(s8) = G_TRUNC %b:_(s16)
+    %T_c:_(s8) = G_TRUNC %c:_(s16)
+    %Undef:_(s8) = G_IMPLICIT_DEF
+    %dst:_(<4 x s8>) = G_BUILD_VECTOR %T_a:_(s8), %T_b:_(s8), %T_c:_(s8), %Undef:_(s8)
+    $w0 = COPY %dst
+
+...
diff --git a/llvm/test/CodeGen/AArch64/PHIElimination-reuse-copy.mir b/llvm/test/CodeGen/AArch64/PHIElimination-reuse-copy.mir
new file mode 100644
index 000000000000..fb32b3189f4d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/PHIElimination-reuse-copy.mir
@@ -0,0 +1,197 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -run-pass=livevars,phi-node-elimination -verify-machineinstrs -mtriple=aarch64-linux-gnu -o - %s | FileCheck %s
+
+# Verify that the original COPY in bb.1 is reappropriated as the PHI source in bb.2,
+# instead of creating a new COPY with the same source register.
+
+---
+name: copy_virtual_reg
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: copy_virtual_reg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $nzcv, $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %a:gpr32 = COPY killed $w0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   Bcc 8, %bb.2, implicit killed $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %b:gpr32 = COPY killed %a
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr32 = COPY killed %b
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   %c:gpr32 = COPY killed [[DEF]]
+  ; CHECK-NEXT:   dead %d:gpr32 = COPY killed %c
+  bb.0:
+    liveins: $nzcv, $w0
+    %a:gpr32 = COPY $w0
+    Bcc 8, %bb.2, implicit $nzcv
+  bb.1:
+    %b:gpr32 = COPY %a:gpr32
+  bb.2:
+    %c:gpr32 = PHI %b:gpr32, %bb.1, undef %undef:gpr32, %bb.0
+    %d:gpr32 = COPY %c:gpr32
+...
+
+---
+name: copy_physical_reg
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: copy_physical_reg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $nzcv, $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   Bcc 8, %bb.2, implicit killed $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead $x0 = IMPLICIT_DEF implicit-def $w0
+  ; CHECK-NEXT:   %a:gpr32 = COPY killed $w0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:gpr32 = COPY killed %a
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   dead %b:gpr32 = COPY killed [[DEF]]
+  bb.0:
+    liveins: $nzcv, $w0
+    Bcc 8, %bb.2, implicit $nzcv
+  bb.1:
+    $x0 = IMPLICIT_DEF
+    %a:gpr32 = COPY $w0
+  bb.2:
+    %b:gpr32 = PHI %a:gpr32, %bb.1, undef %undef:gpr32, %bb.0
+...
+
+---
+name: copy_to_dead
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: copy_to_dead
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $wzr, $xzr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32 = COPY $wzr
+  ; CHECK-NEXT:   dead [[COPY1:%[0-9]+]]:gpr64 = COPY $xzr
+  ; CHECK-NEXT:   TBZW killed [[COPY]], 0, %bb.2
+  ; CHECK-NEXT:   B %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[DEF:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   dead [[DEF1:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[DEF2:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   dead [[DEF3:%[0-9]+]]:gpr64 = IMPLICIT_DEF
+  ; CHECK-NEXT:   B %bb.1
+  bb.0:
+    liveins: $wzr, $xzr
+
+    %9:gpr32 = COPY $wzr
+    dead %5:gpr64 = COPY $xzr
+    TBZW killed %9:gpr32, 0, %bb.2
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000); %bb.2(100.00%)
+
+    dead %1:gpr64 = PHI undef %3:gpr64, %bb.2, undef %5:gpr64, %bb.0
+    dead %2:gpr64 = PHI undef %4:gpr64, %bb.2, undef %5:gpr64, %bb.0
+    B %bb.2
+
+  bb.2:
+    successors: %bb.1(0x80000000); %bb.1(100.00%)
+
+    dead %3:gpr64 = PHI undef %1:gpr64, %bb.1, undef %5:gpr64, %bb.0
+    dead %4:gpr64 = PHI undef %2:gpr64, %bb.1, undef %5:gpr64, %bb.0
+    B %bb.1
+
+...
+
+---
+name: update_livevars
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: update_livevars
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0, $w1, $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32 = COPY killed $w0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32 = COPY killed $w1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY killed [[COPY1]]
+  ; CHECK-NEXT:   B %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[COPY3:%[0-9]+]]:gpr32 = COPY killed [[COPY2]]
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[COPY]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY [[COPY4]]
+  ; CHECK-NEXT:   Bcc 1, %bb.1, implicit $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $nzcv
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY killed [[COPY4]]
+  ; CHECK-NEXT:   B %bb.1
+  bb.0:
+    successors: %bb.1
+    liveins: $w0, $w1, $nzcv
+
+    %0:gpr32 = COPY killed $w0
+    %1:gpr32 = COPY killed $w1
+    B %bb.1
+
+  bb.1:
+    successors: %bb.2, %bb.1
+    liveins: $nzcv
+
+    %2:gpr32 = PHI %3, %bb.2, %1, %bb.0, %3, %bb.1
+    %3:gpr32 = COPY %0
+    Bcc 1, %bb.1, implicit $nzcv
+
+  bb.2:
+    successors: %bb.1
+    liveins: $nzcv
+
+    B %bb.1
+...
+
+---
+name: copy_subreg
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: copy_subreg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY killed $x0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY killed [[COPY]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   dead [[COPY2:%[0-9]+]]:gpr32 = COPY killed [[COPY1]].sub_32
+  bb.0:
+    successors: %bb.1
+    liveins: $x0
+
+    %0:gpr64 = COPY killed $x0
+    %1:gpr64 = COPY killed %0
+
+  bb.1:
+    %2:gpr32 = PHI %1.sub_32, %bb.0
+...
diff --git a/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll b/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll
index 5fbf38b2560d..d6401f347f27 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-bitwisenot-fold.ll
@@ -96,3 +96,51 @@ define i64 @andnot_sub_with_neg_i64(i64 %a0, i64 %a1) {
   %and = and i64 %diff, %a0
   ret i64 %and
 }
+
+define i32 @and_not_select_eq(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: and_not_select_eq:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bics wzr, w1, w0
+; CHECK-NEXT:    csel w0, w0, w2, eq
+; CHECK-NEXT:    ret
+  %or = or i32 %b, %a
+  %cmp = icmp eq i32 %or, %a
+  %a.c = select i1 %cmp, i32 %a, i32 %c
+  ret i32 %a.c
+}
+
+define i32 @and_not_select_ne(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: and_not_select_ne:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bics wzr, w1, w0
+; CHECK-NEXT:    csel w0, w0, w2, ne
+; CHECK-NEXT:    ret
+  %or = or i32 %b, %a
+  %cmp = icmp ne i32 %or, %a
+  %a.c = select i1 %cmp, i32 %a, i32 %c
+  ret i32 %a.c
+}
+
+define i32 @and_not_select_eq_swap(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: and_not_select_eq_swap:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bics wzr, w1, w0
+; CHECK-NEXT:    csel w0, w0, w2, eq
+; CHECK-NEXT:    ret
+  %or = or i32 %b, %a
+  %cmp = icmp eq i32 %a, %or
+  %a.c = select i1 %cmp, i32 %a, i32 %c
+  ret i32 %a.c
+}
+
+define i32 @and_not_select_ne_swap(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: and_not_select_ne_swap:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bics wzr, w1, w0
+; CHECK-NEXT:    csel w0, w0, w2, ne
+; CHECK-NEXT:    ret
+  %or = or i32 %a, %b
+  %cmp = icmp ne i32 %a, %or
+  %a.c = select i1 %cmp, i32 %a, i32 %c
+  ret i32 %a.c
+}
diff --git a/llvm/test/CodeGen/AArch64/abd-combine.ll b/llvm/test/CodeGen/AArch64/abd-combine.ll
index e48680f4be98..d0257890d2c4 100644
--- a/llvm/test/CodeGen/AArch64/abd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/abd-combine.ll
@@ -18,13 +18,11 @@ define <8 x i16> @abdu_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -37,10 +35,10 @@ define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const_lhs:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    usubw v2.4s, v1.4s, v0.4h
-; CHECK-NEXT:    usubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v2.4s
+; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
@@ -53,13 +51,6 @@ define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
 define <8 x i16> @abdu_const_zero(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-NEXT:    usubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    neg v1.4s, v2.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1
@@ -328,13 +319,11 @@ define <8 x i16> @abds_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -347,10 +336,10 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_lhs:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ssubw v2.4s, v1.4s, v0.4h
-; CHECK-NEXT:    ssubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v2.4s
+; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
@@ -363,10 +352,8 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 define <8 x i16> @abds_const_zero(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-NEXT:    ssubw2 v0.4s, v1.4s, v0.8h
-; CHECK-NEXT:    neg v1.4s, v2.4s
+; CHECK-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
 ; CHECK-NEXT:    abs v0.4s, v0.4s
 ; CHECK-NEXT:    abs v1.4s, v1.4s
 ; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 4b1fff642e5f..c838c08cd5e5 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s
+; RUN: opt -passes="require<profile-summary>,function(codegenprepare)" -mtriple=arm64-apple=ios -S -o - %s | FileCheck --check-prefix=OPT %s
 ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck --check-prefix=LLC %s
 
 %struct.X = type { i8, i8, [2 x i8] }
@@ -618,7 +618,7 @@ define void @fct16(ptr nocapture %y, i32 %x) nounwind optsize inlinehint ssp {
 ; LLC-LABEL: fct16:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr w8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and w8, w8, w9
 ; LLC-NEXT:    bfxil w8, w1, #16, #3
@@ -655,7 +655,7 @@ define void @fct16_mask(ptr nocapture %y, i32 %x) nounwind optsize inlinehint ss
 ; LLC-LABEL: fct16_mask:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr w8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and w8, w8, w9
 ; LLC-NEXT:    bfxil w8, w1, #16, #3
@@ -697,7 +697,7 @@ define void @fct17(ptr nocapture %y, i64 %x) nounwind optsize inlinehint ssp {
 ; LLC-LABEL: fct17:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr x8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and x8, x8, x9
 ; LLC-NEXT:    bfxil x8, x1, #16, #3
@@ -734,7 +734,7 @@ define void @fct17_mask(ptr nocapture %y, i64 %x) nounwind optsize inlinehint ss
 ; LLC-LABEL: fct17_mask:
 ; LLC:       // %bb.0: // %entry
 ; LLC-NEXT:    ldr x8, [x0]
-; LLC-NEXT:    mov w9, #33120
+; LLC-NEXT:    mov w9, #33120 // =0x8160
 ; LLC-NEXT:    movk w9, #26, lsl #16
 ; LLC-NEXT:    and x8, x8, x9
 ; LLC-NEXT:    bfxil x8, x1, #16, #3
@@ -819,7 +819,7 @@ define i32 @fct19(i64 %arg1) nounwind readonly ssp  {
 ; LLC-NEXT:    add w0, w8, #32
 ; LLC-NEXT:    ret
 ; LLC-NEXT:  .LBB26_6:
-; LLC-NEXT:    mov w0, #64
+; LLC-NEXT:    mov w0, #64 // =0x40
 ; LLC-NEXT:    ret
 ; OPT-LABEL: @fct19(
 ; OPT-NEXT:  entry:
@@ -916,20 +916,20 @@ return:                                           ; preds = %if.end13, %if.then1
 define i80 @fct20(i128 %a, i128 %b) {
 ; LLC-LABEL: fct20:
 ; LLC:       // %bb.0: // %entry
-; LLC-NEXT:	mov	x12, #11776                     // =0x2e00
-; LLC-NEXT:	lsr	x8, x1, #18
-; LLC-NEXT:	extr	x9, x1, x0, #18
-; LLC-NEXT:	movk	x12, #25856, lsl #16
-; LLC-NEXT:	orr	x10, x2, x3
-; LLC-NEXT:	mov	w11, #26220                     // =0x666c
-; LLC-NEXT:	movk	x12, #11077, lsl #32
-; LLC-NEXT:	and	x11, x8, x11
-; LLC-NEXT:	cmp	x10, #0
-; LLC-NEXT:	movk	x12, #45, lsl #48
-; LLC-NEXT:	csel	x1, x11, x8, eq
-; LLC-NEXT:	and	x12, x9, x12
-; LLC-NEXT:	csel	x0, x12, x9, eq
-; LLC-NEXT:	ret
+; LLC-NEXT:    mov x12, #11776 // =0x2e00
+; LLC-NEXT:    lsr x8, x1, #18
+; LLC-NEXT:    extr x9, x1, x0, #18
+; LLC-NEXT:    movk x12, #25856, lsl #16
+; LLC-NEXT:    orr x10, x2, x3
+; LLC-NEXT:    mov w11, #26220 // =0x666c
+; LLC-NEXT:    movk x12, #11077, lsl #32
+; LLC-NEXT:    and x11, x8, x11
+; LLC-NEXT:    cmp x10, #0
+; LLC-NEXT:    movk x12, #45, lsl #48
+; LLC-NEXT:    csel x1, x11, x8, eq
+; LLC-NEXT:    and x12, x9, x12
+; LLC-NEXT:    csel x0, x12, x9, eq
+; LLC-NEXT:    ret
 ; OPT-LABEL: @fct20(
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    [[SHR:%.*]] = lshr i128 [[A:%.*]], 18
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp-contract-zero.ll b/llvm/test/CodeGen/AArch64/arm64-fp-contract-zero.ll
index 9a753748a29e..53e69b985caf 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp-contract-zero.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp-contract-zero.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=arm64 -fp-contract=fast -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64 -o - %s | FileCheck %s
 
 
 ; Make sure we don't try to fold an fneg into +0.0, creating an illegal constant
@@ -7,12 +7,10 @@
 define double @test_fms_fold(double %a, double %b) {
 ; CHECK-LABEL: test_fms_fold:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi d2, #0000000000000000
-; CHECK-NEXT:    fmul d1, d1, d2
-; CHECK-NEXT:    fnmsub d0, d0, d2, d1
+; CHECK-NEXT:    movi {{d[0-9]+}}, #0000000000000000
 ; CHECK-NEXT:    ret
-  %mul = fmul double %a, 0.000000e+00
-  %mul1 = fmul double %b, 0.000000e+00
+  %mul = fmul fast double %a, 0.000000e+00
+  %mul1 = fmul fast double %b, 0.000000e+00
   %sub = fsub double %mul, %mul1
   ret double %sub
 }
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
index 4d0603722c3a..2af7cd4370b1 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-vector-ldst.ll
@@ -13337,6 +13337,57 @@ define <16 x i8> @test_v16i8_post_reg_ld1lane(ptr %bar, ptr %ptr, i64 %inc, <16
   ret <16 x i8> %tmp2
 }
 
+define <16 x i8> @test_v16i8_post_reg_ld1lane_zero(ptr %bar, ptr %ptr, i64 %inc) {
+; CHECK-SD-LABEL: test_v16i8_post_reg_ld1lane_zero:
+; CHECK-SD:       ; %bb.0:
+; CHECK-SD-NEXT:    ldr b0, [x0]
+; CHECK-SD-NEXT:    add x8, x0, x2
+; CHECK-SD-NEXT:    str x8, [x1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v16i8_post_reg_ld1lane_zero:
+; CHECK-GI:       ; %bb.0:
+; CHECK-GI-NEXT:    ldr b0, [x0]
+; CHECK-GI-NEXT:    mov w8, #0 ; =0x0
+; CHECK-GI-NEXT:    mov.b v0[1], w8
+; CHECK-GI-NEXT:    mov.b v0[2], w8
+; CHECK-GI-NEXT:    mov.b v0[3], w8
+; CHECK-GI-NEXT:    mov.b v0[4], w8
+; CHECK-GI-NEXT:    mov.b v0[5], w8
+; CHECK-GI-NEXT:    mov.b v0[6], w8
+; CHECK-GI-NEXT:    mov.b v0[7], w8
+; CHECK-GI-NEXT:    mov.b v0[8], w8
+; CHECK-GI-NEXT:    mov.b v0[9], w8
+; CHECK-GI-NEXT:    mov.b v0[10], w8
+; CHECK-GI-NEXT:    mov.b v0[11], w8
+; CHECK-GI-NEXT:    mov.b v0[12], w8
+; CHECK-GI-NEXT:    mov.b v0[13], w8
+; CHECK-GI-NEXT:    mov.b v0[14], w8
+; CHECK-GI-NEXT:    mov.b v0[15], w8
+; CHECK-GI-NEXT:    add x8, x0, x2
+; CHECK-GI-NEXT:    str x8, [x1]
+; CHECK-GI-NEXT:    ret
+  %tmp1 = load i8, ptr %bar
+  %tmp2 = insertelement <16 x i8> zeroinitializer, i8 %tmp1, i32 0
+  %tmp3 = getelementptr i8, ptr %bar, i64 %inc
+  store ptr %tmp3, ptr %ptr
+  ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @test_v16i8_post_reg_ld1lane_undef(ptr %bar, ptr %ptr, i64 %inc) {
+; CHECK-LABEL: test_v16i8_post_reg_ld1lane_undef:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldr b0, [x0]
+; CHECK-NEXT:    add x8, x0, x2
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
+  %tmp1 = load i8, ptr %bar
+  %tmp2 = insertelement <16 x i8> poison, i8 %tmp1, i32 0
+  %tmp3 = getelementptr i8, ptr %bar, i64 %inc
+  store ptr %tmp3, ptr %ptr
+  ret <16 x i8> %tmp2
+}
+
 define <8 x i8> @test_v8i8_post_imm_ld1lane(ptr %bar, ptr %ptr, <8 x i8> %A) {
 ; CHECK-SD-LABEL: test_v8i8_post_imm_ld1lane:
 ; CHECK-SD:       ; %bb.0:
@@ -14078,3 +14129,67 @@ define i32 @load_single_extract_variable_index_masked2_i32(ptr %A, i32 %idx) {
   %e = extractelement <4 x i32> %lv, i32 %idx.x
   ret i32 %e
 }
+
+define void @chained_insert_zero(ptr noundef %fenc, ptr noundef %pred, ptr noundef %residual, i32 noundef %stride) {
+; CHECK-SD-LABEL: chained_insert_zero:
+; CHECK-SD:       ; %bb.0: ; %entry
+; CHECK-SD-NEXT:    ldr s0, [x1]
+; CHECK-SD-NEXT:    ldr s1, [x0]
+; CHECK-SD-NEXT:    ; kill: def $w3 killed $w3 def $x3
+; CHECK-SD-NEXT:    sxtw x8, w3
+; CHECK-SD-NEXT:    usubl.8h v0, v1, v0
+; CHECK-SD-NEXT:    str d0, [x2]
+; CHECK-SD-NEXT:    ldr s0, [x0, x8]
+; CHECK-SD-NEXT:    ldr s1, [x1, x8]
+; CHECK-SD-NEXT:    sbfiz x8, x3, #1, #32
+; CHECK-SD-NEXT:    usubl.8h v0, v0, v1
+; CHECK-SD-NEXT:    str d0, [x2, x8]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: chained_insert_zero:
+; CHECK-GI:       ; %bb.0: ; %entry
+; CHECK-GI-NEXT:    ldr s0, [x0]
+; CHECK-GI-NEXT:    ldr s1, [x1]
+; CHECK-GI-NEXT:    ; kill: def $w3 killed $w3 def $x3
+; CHECK-GI-NEXT:    sxtw x8, w3
+; CHECK-GI-NEXT:    mov.s v0[1], wzr
+; CHECK-GI-NEXT:    mov.s v1[1], wzr
+; CHECK-GI-NEXT:    usubl.8h v0, v0, v1
+; CHECK-GI-NEXT:    str d0, [x2]
+; CHECK-GI-NEXT:    ldr s0, [x0, x8]
+; CHECK-GI-NEXT:    ldr s1, [x1, x8]
+; CHECK-GI-NEXT:    lsl x8, x8, #1
+; CHECK-GI-NEXT:    mov.s v0[1], wzr
+; CHECK-GI-NEXT:    mov.s v1[1], wzr
+; CHECK-GI-NEXT:    usubl.8h v0, v0, v1
+; CHECK-GI-NEXT:    str d0, [x2, x8]
+; CHECK-GI-NEXT:    ret
+entry:
+  %idx.ext = sext i32 %stride to i64
+  %0 = load i32, ptr %fenc, align 4
+  %vld1_lane.i = insertelement <2 x i32> <i32 poison, i32 0>, i32 %0, i64 0
+  %1 = bitcast <2 x i32> %vld1_lane.i to <8 x i8>
+  %2 = load i32, ptr %pred, align 4
+  %vld1_lane.i16 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %2, i64 0
+  %3 = bitcast <2 x i32> %vld1_lane.i16 to <8 x i8>
+  %vmovl.i15 = zext <8 x i8> %1 to <8 x i16>
+  %vmovl.i = zext <8 x i8> %3 to <8 x i16>
+  %sub.i = sub nsw <8 x i16> %vmovl.i15, %vmovl.i
+  %shuffle.i = shufflevector <8 x i16> %sub.i, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i16> %shuffle.i, ptr %residual, align 2
+  %add.ptr = getelementptr inbounds i8, ptr %fenc, i64 %idx.ext
+  %add.ptr6 = getelementptr inbounds i8, ptr %pred, i64 %idx.ext
+  %add.ptr8 = getelementptr inbounds i16, ptr %residual, i64 %idx.ext
+  %4 = load i32, ptr %add.ptr, align 4
+  %vld1_lane.i.1 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %4, i64 0
+  %5 = bitcast <2 x i32> %vld1_lane.i.1 to <8 x i8>
+  %6 = load i32, ptr %add.ptr6, align 4
+  %vld1_lane.i16.1 = insertelement <2 x i32> <i32 poison, i32 0>, i32 %6, i64 0
+  %7 = bitcast <2 x i32> %vld1_lane.i16.1 to <8 x i8>
+  %vmovl.i15.1 = zext <8 x i8> %5 to <8 x i16>
+  %vmovl.i.1 = zext <8 x i8> %7 to <8 x i16>
+  %sub.i.1 = sub nsw <8 x i16> %vmovl.i15.1, %vmovl.i.1
+  %shuffle.i.1 = shufflevector <8 x i16> %sub.i.1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i16> %shuffle.i.1, ptr %add.ptr8, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
new file mode 100644
index 000000000000..f422f96f3349
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
@@ -0,0 +1,103 @@
+; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s -check-prefixes=NOTCPU-LINUX --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=generic | FileCheck %s -check-prefixes=NOTCPU-APPLE --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm-fpr64 | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm-fpr64 | FileCheck %s -check-prefixes=ATTR --match-full-lines
+
+define void @zero_cycle_regmov_FPR32(float %a, float %b, float %c, float %d) {
+entry:
+; CHECK-LABEL: t:
+; NOTCPU-LINUX: fmov s0, s2
+; NOTCPU-LINUX: fmov s1, s3
+; NOTCPU-LINUX: fmov [[REG2:s[0-9]+]], s3
+; NOTCPU-LINUX: fmov [[REG1:s[0-9]+]], s2
+; NOTCPU-LINUX-NEXT: bl {{_?foo_float}}
+; NOTCPU-LINUX: fmov s0, [[REG1]]
+; NOTCPU-LINUX: fmov s1, [[REG2]]
+
+; NOTCPU-APPLE: fmov s0, s2
+; NOTCPU-APPLE: fmov s1, s3
+; NOTCPU-APPLE: fmov [[REG2:s[0-9]+]], s3
+; NOTCPU-APPLE: fmov [[REG1:s[0-9]+]], s2
+; NOTCPU-APPLE-NEXT: bl {{_?foo_float}}
+; NOTCPU-APPLE: fmov s0, [[REG1]]
+; NOTCPU-APPLE: fmov s1, [[REG2]]
+
+; CPU: fmov [[REG2:d[0-9]+]], d3
+; CPU: fmov [[REG1:d[0-9]+]], d2
+; CPU: fmov d0, d2
+; CPU: fmov d1, d3
+; CPU-NEXT: bl {{_?foo_float}}
+; CPU: fmov d0, [[REG1]]
+; CPU: fmov d1, [[REG2]]
+
+; NOTATTR: fmov [[REG2:s[0-9]+]], s3
+; NOTATTR: fmov [[REG1:s[0-9]+]], s2
+; NOTATTR: fmov s0, s2
+; NOTATTR: fmov s1, s3
+; NOTATTR-NEXT: bl {{_?foo_float}}
+; NOTATTR: fmov s0, [[REG1]]
+; NOTATTR: fmov s1, [[REG2]]
+
+; ATTR: fmov d0, d2
+; ATTR: fmov d1, d3
+; ATTR: fmov [[REG2:d[0-9]+]], d3
+; ATTR: fmov [[REG1:d[0-9]+]], d2
+; ATTR-NEXT: bl {{_?foo_float}}
+; ATTR: fmov d0, [[REG1]]
+; ATTR: fmov d1, [[REG2]]
+  %call = call float @foo_float(float %c, float %d)
+  %call1 = call float @foo_float(float %c, float %d)
+  unreachable
+}
+
+declare float @foo_float(float, float)
+
+define void @zero_cycle_regmov_FPR16(half %a, half %b, half %c, half %d) {
+entry:
+; CHECK-LABEL: t:
+; NOTCPU-LINUX: fmov s0, s2
+; NOTCPU-LINUX: fmov s1, s3
+; NOTCPU-LINUX: fmov [[REG2:s[0-9]+]], s3
+; NOTCPU-LINUX: fmov [[REG1:s[0-9]+]], s2
+; NOTCPU-LINUX-NEXT: bl {{_?foo_half}}
+; NOTCPU-LINUX: fmov s0, [[REG1]]
+; NOTCPU-LINUX: fmov s1, [[REG2]]
+
+; NOTCPU-APPLE: fmov s0, s2
+; NOTCPU-APPLE: fmov s1, s3
+; NOTCPU-APPLE: fmov [[REG2:s[0-9]+]], s3
+; NOTCPU-APPLE: fmov [[REG1:s[0-9]+]], s2
+; NOTCPU-APPLE-NEXT: bl {{_?foo_half}}
+; NOTCPU-APPLE: fmov s0, [[REG1]]
+; NOTCPU-APPLE: fmov s1, [[REG2]]
+
+; CPU: fmov [[REG2:d[0-9]+]], d3
+; CPU: fmov [[REG1:d[0-9]+]], d2
+; CPU: fmov d0, d2
+; CPU: fmov d1, d3
+; CPU-NEXT: bl {{_?foo_half}}
+; CPU: fmov d0, [[REG1]]
+; CPU: fmov d1, [[REG2]]
+
+; NOTATTR: fmov [[REG2:s[0-9]+]], s3
+; NOTATTR: fmov [[REG1:s[0-9]+]], s2
+; NOTATTR: fmov s0, s2
+; NOTATTR: fmov s1, s3
+; NOTATTR-NEXT: bl {{_?foo_half}}
+; NOTATTR: fmov s0, [[REG1]]
+; NOTATTR: fmov s1, [[REG2]]
+
+; ATTR: fmov d0, d2
+; ATTR: fmov d1, d3
+; ATTR: fmov [[REG2:d[0-9]+]], d3
+; ATTR: fmov [[REG1:d[0-9]+]], d2
+; ATTR-NEXT: bl {{_?foo_half}}
+; ATTR: fmov d0, [[REG1]]
+; ATTR: fmov d1, [[REG2]]
+  %call = call half @foo_half(half %c, half %d)
+  %call1 = call half @foo_half(half %c, half %d)
+  unreachable
+}
+
+declare half @foo_half(half, half)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll
new file mode 100644
index 000000000000..e14e69b5e6a2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr.ll
@@ -0,0 +1,54 @@
+; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s -check-prefixes=NOTCPU-LINUX --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=generic | FileCheck %s -check-prefixes=NOTCPU-APPLE --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm-gpr64 | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm-gpr64 | FileCheck %s -check-prefixes=ATTR --match-full-lines
+
+define void @zero_cycle_regmov_GPR32(i32 %a, i32 %b, i32 %c, i32 %d) {
+entry:
+; CHECK-LABEL: t:
+; NOTCPU-LINUX: mov w0, w2
+; NOTCPU-LINUX: mov w1, w3
+; NOTCPU-LINUX: mov [[REG2:w[0-9]+]], w3
+; NOTCPU-LINUX: mov [[REG1:w[0-9]+]], w2
+; NOTCPU-LINUX-NEXT: bl {{_?foo_i32}}
+; NOTCPU-LINUX: mov w0, [[REG1]]
+; NOTCPU-LINUX: mov w1, [[REG2]]
+
+; NOTCPU-APPLE: mov w0, w2
+; NOTCPU-APPLE: mov w1, w3
+; NOTCPU-APPLE: mov [[REG2:w[0-9]+]], w3
+; NOTCPU-APPLE: mov [[REG1:w[0-9]+]], w2
+; NOTCPU-APPLE-NEXT: bl {{_?foo_i32}}
+; NOTCPU-APPLE: mov w0, [[REG1]]
+; NOTCPU-APPLE: mov w1, [[REG2]]
+
+; CPU: mov [[REG2:x[0-9]+]], x3
+; CPU: mov [[REG1:x[0-9]+]], x2
+; CPU: mov x0, x2
+; CPU: mov x1, x3
+; CPU-NEXT: bl {{_?foo_i32}}
+; CPU: mov x0, [[REG1]]
+; CPU: mov x1, [[REG2]]
+
+; NOTATTR: mov [[REG2:w[0-9]+]], w3
+; NOTATTR: mov [[REG1:w[0-9]+]], w2
+; NOTATTR: mov w0, w2
+; NOTATTR: mov w1, w3
+; NOTATTR-NEXT: bl {{_?foo_i32}}
+; NOTATTR: mov w0, [[REG1]]
+; NOTATTR: mov w1, [[REG2]]
+
+; ATTR: mov x0, x2
+; ATTR: mov x1, x3
+; ATTR: mov [[REG2:x[0-9]+]], x3
+; ATTR: mov [[REG1:x[0-9]+]], x2
+; ATTR-NEXT: bl {{_?foo_i32}}
+; ATTR: mov x0, [[REG1]]
+; ATTR: mov x1, [[REG2]]
+  %call = call i32 @foo_i32(i32 %c, i32 %d)
+  %call1 = call i32 @foo_i32(i32 %c, i32 %d)
+  unreachable
+}
+
+declare i32 @foo_i32(i32, i32)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
deleted file mode 100644
index 5ef6d3e84805..000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; RUN: llc < %s -march=arm64 | FileCheck %s -check-prefixes=NOTCPU --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm | FileCheck %s -check-prefixes=ATTR --match-full-lines
-
-define void @t(i32 %a, i32 %b, i32 %c, i32 %d) {
-entry:
-; CHECK-LABEL: t:
-; NOTCPU: mov w0, w2
-; NOTCPU: mov w1, w3
-; NOTCPU: mov [[REG2:w[0-9]+]], w3
-; NOTCPU: mov [[REG1:w[0-9]+]], w2
-; NOTCPU-NEXT: bl {{_?foo}}
-; NOTCPU: mov w0, [[REG1]]
-; NOTCPU: mov w1, [[REG2]]
-
-; CPU: mov [[REG2:x[0-9]+]], x3
-; CPU: mov [[REG1:x[0-9]+]], x2
-; CPU: mov x0, x2
-; CPU: mov x1, x3
-; CPU-NEXT: bl {{_?foo}}
-; CPU: mov x0, [[REG1]]
-; CPU: mov x1, [[REG2]]
-
-; NOTATTR: mov [[REG2:w[0-9]+]], w3
-; NOTATTR: mov [[REG1:w[0-9]+]], w2
-; NOTATTR: mov w0, w2
-; NOTATTR: mov w1, w3
-; NOTATTR-NEXT: bl {{_?foo}}
-; NOTATTR: mov w0, [[REG1]]
-; NOTATTR: mov w1, [[REG2]]
-
-; ATTR: mov x0, x2
-; ATTR: mov x1, x3
-; ATTR: mov [[REG2:x[0-9]+]], x3
-; ATTR: mov [[REG1:x[0-9]+]], x2
-; ATTR-NEXT: bl {{_?foo}}
-; ATTR: mov x0, [[REG1]]
-; ATTR: mov x1, [[REG2]]
-  %call = call i32 @foo(i32 %c, i32 %d)
-  %call1 = call i32 @foo(i32 %c, i32 %d)
-  unreachable
-}
-
-declare i32 @foo(i32, i32)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zip.ll b/llvm/test/CodeGen/AArch64/arm64-zip.ll
index b24e54a68fb4..20d0c7f1b708 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zip.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zip.ll
@@ -413,7 +413,7 @@ define <4 x float> @shuffle_zip1(<4 x float> %arg) {
 ; CHECK-NEXT:    fmov.4s v1, #1.00000000
 ; CHECK-NEXT:    zip1.4h v0, v0, v0
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    and.16b v0, v1, v0
+; CHECK-NEXT:    and.16b v0, v0, v1
 ; CHECK-NEXT:    ret
 bb:
   %inst = fcmp olt <4 x float> zeroinitializer, %arg
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll b/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
index 38416310b353..911b6fa8eff4 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
@@ -46,27 +46,24 @@ define float @f6(float %val, i32 %a) {
 @dst = global [512 x i8] zeroinitializer, align 1
 @src = global [512 x i8] zeroinitializer, align 1
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memcpy(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memcpy":
-; CHECK: bl __arm_sc_memcpy
+; CHECK: bl "#__arm_sc_memcpy"
 
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memmove(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memmove":
-; CHECK: bl __arm_sc_memmove
+; CHECK: bl "#__arm_sc_memmove"
   tail call void @llvm.memmove.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memset(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memset":
-; CHECK: bl __arm_sc_memset
+; CHECK: bl "#__arm_sc_memset"
   tail call void @llvm.memset.p0.i64(ptr align 1 @dst, i8 2, i64 %n, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/bf16_fast_math.ll b/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
index 871ca12c9de7..e52c76fcc3f2 100644
--- a/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
+++ b/llvm/test/CodeGen/AArch64/bf16_fast_math.ll
@@ -19,13 +19,13 @@ define bfloat @normal_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -65,13 +65,13 @@ define bfloat @fast_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nnan ninf nsz arcp contract afn reassoc nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -111,13 +111,13 @@ define bfloat @ninf_fadd(bfloat %x, bfloat %y) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY3:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY3]], killed [[COPY2]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY4]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY4]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], killed [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr16 = COPY [[COPY5]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY6]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -161,13 +161,13 @@ define bfloat @normal_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -176,12 +176,12 @@ define bfloat @normal_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -232,13 +232,13 @@ define bfloat @nnan_ninf_contract_fadd_sequence(bfloat %x, bfloat %y, bfloat %z)
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = nnan ninf contract nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -247,12 +247,12 @@ define bfloat @nnan_ninf_contract_fadd_sequence(bfloat %x, bfloat %y, bfloat %z)
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = nnan ninf contract nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
@@ -299,13 +299,13 @@ define bfloat @ninf_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_1:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG1]]
   ; CHECK-NOBF16-NEXT:   [[COPY4:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_1]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY4]], killed [[COPY3]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY [[FADDSrr]]
+  ; CHECK-NOBF16-NEXT:   [[COPY5:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY5]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri]], [[COPY5]]
   ; CHECK-NOBF16-NEXT:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 32767
   ; CHECK-NOBF16-NEXT:   [[ADDWrr1:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri1:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr1]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY [[UBFMWri1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY6:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri1]]
   ; CHECK-NOBF16-NEXT:   [[COPY7:%[0-9]+]]:fpr16 = COPY [[COPY6]].hsub
   ; CHECK-NOBF16-NEXT:   [[SUBREG_TO_REG2:%[0-9]+]]:fpr64 = SUBREG_TO_REG 0, killed [[COPY7]], %subreg.hsub
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_2:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG2]]
@@ -314,12 +314,12 @@ define bfloat @ninf_fadd_sequence(bfloat %x, bfloat %y, bfloat %z) {
   ; CHECK-NOBF16-NEXT:   [[SHLLv4i16_3:%[0-9]+]]:fpr128 = SHLLv4i16 killed [[SUBREG_TO_REG3]]
   ; CHECK-NOBF16-NEXT:   [[COPY9:%[0-9]+]]:fpr32 = COPY [[SHLLv4i16_3]].ssub
   ; CHECK-NOBF16-NEXT:   [[FADDSrr1:%[0-9]+]]:fpr32 = ninf nofpexcept FADDSrr killed [[COPY8]], killed [[COPY9]], implicit $fpcr
-  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY [[FADDSrr1]]
+  ; CHECK-NOBF16-NEXT:   [[COPY10:%[0-9]+]]:gpr32 = COPY killed [[FADDSrr1]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri2:%[0-9]+]]:gpr32 = UBFMWri [[COPY10]], 16, 16
   ; CHECK-NOBF16-NEXT:   [[ADDWrr2:%[0-9]+]]:gpr32 = ADDWrr killed [[UBFMWri2]], [[COPY10]]
   ; CHECK-NOBF16-NEXT:   [[ADDWrr3:%[0-9]+]]:gpr32 = ADDWrr killed [[ADDWrr2]], [[MOVi32imm]]
   ; CHECK-NOBF16-NEXT:   [[UBFMWri3:%[0-9]+]]:gpr32 = UBFMWri killed [[ADDWrr3]], 16, 31
-  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY [[UBFMWri3]]
+  ; CHECK-NOBF16-NEXT:   [[COPY11:%[0-9]+]]:fpr32 = COPY killed [[UBFMWri3]]
   ; CHECK-NOBF16-NEXT:   [[COPY12:%[0-9]+]]:fpr16 = COPY [[COPY11]].hsub
   ; CHECK-NOBF16-NEXT:   $h0 = COPY [[COPY12]]
   ; CHECK-NOBF16-NEXT:   RET_ReallyLR implicit $h0
diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
index b4f179e992a0..c2bb032eed78 100644
--- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
+++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll
@@ -197,11 +197,9 @@ define <4 x i32> @not_sign_4xi32(<4 x i32> %a) {
 ; CHECK-LABEL: not_sign_4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI16_0
-; CHECK-NEXT:    movi v2.4s, #1
 ; CHECK-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
-; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    and v1.16b, v0.16b, v2.16b
-; CHECK-NEXT:    orn v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    orr v0.4s, #1
 ; CHECK-NEXT:    ret
   %c = icmp sgt <4 x i32> %a, <i32 1, i32 -1, i32 -1, i32 -1>
   %res = select <4 x i1> %c, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/AArch64/concatbinop.ll b/llvm/test/CodeGen/AArch64/concatbinop.ll
index 828182d18b38..062a5a8c35b2 100644
--- a/llvm/test/CodeGen/AArch64/concatbinop.ll
+++ b/llvm/test/CodeGen/AArch64/concatbinop.ll
@@ -179,7 +179,7 @@ define <16 x i8> @signOf_neon(ptr nocapture noundef readonly %a, ptr nocapture n
 ; CHECK-NEXT:    uzp1 v3.16b, v5.16b, v6.16b
 ; CHECK-NEXT:    uzp1 v1.16b, v1.16b, v2.16b
 ; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
-; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
 entry:
   %0 = load <8 x i16>, ptr %a, align 2
diff --git a/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll b/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll
index 69fd6c4de78b..aeeffde01cb6 100644
--- a/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll
+++ b/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll
@@ -119,23 +119,17 @@ define void @msubpt1(i32 %index, i32 %elem) {
 ; CHECK-CPA-O0:       // %bb.0: // %entry
 ; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
 ; CHECK-CPA-O0-NEXT:    mov w8, w0
-; CHECK-CPA-O0-NEXT:    sxtw x9, w8
-; CHECK-CPA-O0-NEXT:    mov x8, xzr
-; CHECK-CPA-O0-NEXT:    subs x8, x8, x9
-; CHECK-CPA-O0-NEXT:    lsl x8, x8, #1
-; CHECK-CPA-O0-NEXT:    subs x10, x8, x9
+; CHECK-CPA-O0-NEXT:    sxtw x8, w8
+; CHECK-CPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    mneg x8, x8, x9
+; CHECK-CPA-O0-NEXT:    add x8, x8, #288
 ; CHECK-CPA-O0-NEXT:    adrp x9, array2
 ; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array2
-; CHECK-CPA-O0-NEXT:    mov w8, #288 // =0x120
-; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
 ; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
-; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #4
-; CHECK-CPA-O0-NEXT:    mov w10, #96 // =0x60
-; CHECK-CPA-O0-NEXT:    // kill: def $x10 killed $w10
-; CHECK-CPA-O0-NEXT:    addpt x10, x9, x10
-; CHECK-CPA-O0-NEXT:    ldr q1, [x10, #16]
-; CHECK-CPA-O0-NEXT:    ldr q2, [x10, #32]
 ; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #96]
+; CHECK-CPA-O0-NEXT:    ldr q1, [x9, #112]
+; CHECK-CPA-O0-NEXT:    ldr q2, [x9, #128]
 ; CHECK-CPA-O0-NEXT:    str q2, [x8, #32]
 ; CHECK-CPA-O0-NEXT:    str q1, [x8, #16]
 ; CHECK-CPA-O0-NEXT:    str q0, [x8]
@@ -144,21 +138,17 @@ define void @msubpt1(i32 %index, i32 %elem) {
 ; CHECK-CPA-O3-LABEL: msubpt1:
 ; CHECK-CPA-O3:       // %bb.0: // %entry
 ; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-CPA-O3-NEXT:    sxtw x9, w0
-; CHECK-CPA-O3-NEXT:    adrp x8, array2
-; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array2
-; CHECK-CPA-O3-NEXT:    mov w11, #96 // =0x60
-; CHECK-CPA-O3-NEXT:    mov w12, #288 // =0x120
-; CHECK-CPA-O3-NEXT:    ldr q2, [x8, #96]
-; CHECK-CPA-O3-NEXT:    neg x10, x9
-; CHECK-CPA-O3-NEXT:    addpt x11, x8, x11
-; CHECK-CPA-O3-NEXT:    lsl x10, x10, #1
-; CHECK-CPA-O3-NEXT:    ldp q1, q0, [x11, #16]
-; CHECK-CPA-O3-NEXT:    sub x9, x10, x9
-; CHECK-CPA-O3-NEXT:    addpt x10, x8, x12
-; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9, lsl #4
-; CHECK-CPA-O3-NEXT:    stp q1, q0, [x9, #16]
-; CHECK-CPA-O3-NEXT:    str q2, [x9]
+; CHECK-CPA-O3-NEXT:    sxtw x8, w0
+; CHECK-CPA-O3-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    mneg x8, x8, x9
+; CHECK-CPA-O3-NEXT:    adrp x9, array2
+; CHECK-CPA-O3-NEXT:    add x9, x9, :lo12:array2
+; CHECK-CPA-O3-NEXT:    ldp q1, q0, [x9, #112]
+; CHECK-CPA-O3-NEXT:    ldr q2, [x9, #96]
+; CHECK-CPA-O3-NEXT:    add x8, x8, #288
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O3-NEXT:    stp q1, q0, [x8, #16]
+; CHECK-CPA-O3-NEXT:    str q2, [x8]
 ; CHECK-CPA-O3-NEXT:    ret
 ;
 ; CHECK-NOCPA-O0-LABEL: msubpt1:
@@ -205,15 +195,15 @@ entry:
 define void @subpt1(i32 %index, i32 %elem) {
 ; CHECK-CPA-O0-LABEL: subpt1:
 ; CHECK-CPA-O0:       // %bb.0: // %entry
-; CHECK-CPA-O0-NEXT:    adrp x9, array
-; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w0
+; CHECK-CPA-O0-NEXT:    sxtw x9, w8
 ; CHECK-CPA-O0-NEXT:    mov w8, #96 // =0x60
 ; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
+; CHECK-CPA-O0-NEXT:    subs x8, x8, x9, lsl #8
+; CHECK-CPA-O0-NEXT:    adrp x9, array
+; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
 ; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
-; CHECK-CPA-O0-NEXT:    // implicit-def: $x10
-; CHECK-CPA-O0-NEXT:    mov w10, w0
-; CHECK-CPA-O0-NEXT:    sbfiz x10, x10, #8, #32
-; CHECK-CPA-O0-NEXT:    subpt x8, x8, x10
 ; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #32]
 ; CHECK-CPA-O0-NEXT:    str q0, [x8]
 ; CHECK-CPA-O0-NEXT:    ret
@@ -221,13 +211,13 @@ define void @subpt1(i32 %index, i32 %elem) {
 ; CHECK-CPA-O3-LABEL: subpt1:
 ; CHECK-CPA-O3:       // %bb.0: // %entry
 ; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-CPA-O3-NEXT:    adrp x8, array
-; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O3-NEXT:    sxtw x8, w0
 ; CHECK-CPA-O3-NEXT:    mov w9, #96 // =0x60
-; CHECK-CPA-O3-NEXT:    sbfiz x10, x0, #8, #32
-; CHECK-CPA-O3-NEXT:    addpt x9, x8, x9
-; CHECK-CPA-O3-NEXT:    ldr q0, [x8, #32]
-; CHECK-CPA-O3-NEXT:    subpt x8, x9, x10
+; CHECK-CPA-O3-NEXT:    sub x8, x9, x8, lsl #8
+; CHECK-CPA-O3-NEXT:    adrp x9, array
+; CHECK-CPA-O3-NEXT:    add x9, x9, :lo12:array
+; CHECK-CPA-O3-NEXT:    ldr q0, [x9, #32]
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
 ; CHECK-CPA-O3-NEXT:    str q0, [x8]
 ; CHECK-CPA-O3-NEXT:    ret
 ;
@@ -264,28 +254,24 @@ entry:
 define void @subpt2(i32 %index, i32 %elem) {
 ; CHECK-CPA-O0-LABEL: subpt2:
 ; CHECK-CPA-O0:       // %bb.0: // %entry
-; CHECK-CPA-O0-NEXT:    mov x8, xzr
-; CHECK-CPA-O0-NEXT:    subs x10, x8, w0, sxtw
-; CHECK-CPA-O0-NEXT:    adrp x9, array
-; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
 ; CHECK-CPA-O0-NEXT:    mov w8, #96 // =0x60
 ; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
+; CHECK-CPA-O0-NEXT:    subs x8, x8, w0, sxtw #4
+; CHECK-CPA-O0-NEXT:    adrp x9, array
+; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
 ; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
-; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #4
 ; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #32]
 ; CHECK-CPA-O0-NEXT:    str q0, [x8]
 ; CHECK-CPA-O0-NEXT:    ret
 ;
 ; CHECK-CPA-O3-LABEL: subpt2:
 ; CHECK-CPA-O3:       // %bb.0: // %entry
-; CHECK-CPA-O3-NEXT:    mov x8, xzr
-; CHECK-CPA-O3-NEXT:    mov w9, #96 // =0x60
-; CHECK-CPA-O3-NEXT:    adrp x10, array
-; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:array
-; CHECK-CPA-O3-NEXT:    sub x8, x8, w0, sxtw
-; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9
-; CHECK-CPA-O3-NEXT:    ldr q0, [x10, #32]
-; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8, lsl #4
+; CHECK-CPA-O3-NEXT:    mov w8, #96 // =0x60
+; CHECK-CPA-O3-NEXT:    adrp x9, array
+; CHECK-CPA-O3-NEXT:    add x9, x9, :lo12:array
+; CHECK-CPA-O3-NEXT:    sub x8, x8, w0, sxtw #4
+; CHECK-CPA-O3-NEXT:    ldr q0, [x9, #32]
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
 ; CHECK-CPA-O3-NEXT:    str q0, [x8]
 ; CHECK-CPA-O3-NEXT:    ret
 ;
@@ -670,14 +656,13 @@ define hidden void @multidim() {
 ; CHECK-CPA-O0-NEXT:    .cfi_offset w30, -16
 ; CHECK-CPA-O0-NEXT:    adrp x8, b
 ; CHECK-CPA-O0-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
 ; CHECK-CPA-O0-NEXT:    mov w8, w9
-; CHECK-CPA-O0-NEXT:    mov w10, w8
+; CHECK-CPA-O0-NEXT:    ubfiz x8, x8, #1, #32
+; CHECK-CPA-O0-NEXT:    add x10, x8, #2
 ; CHECK-CPA-O0-NEXT:    adrp x8, a
 ; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:a
-; CHECK-CPA-O0-NEXT:    mov w11, #2 // =0x2
-; CHECK-CPA-O0-NEXT:    // kill: def $x11 killed $w11
-; CHECK-CPA-O0-NEXT:    addpt x8, x8, x11
-; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #1
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10
 ; CHECK-CPA-O0-NEXT:    add w9, w9, #1
 ; CHECK-CPA-O0-NEXT:    mov w9, w9
 ; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
@@ -697,13 +682,13 @@ define hidden void @multidim() {
 ; CHECK-CPA-O3-LABEL: multidim:
 ; CHECK-CPA-O3:       // %bb.0: // %entry
 ; CHECK-CPA-O3-NEXT:    adrp x8, b
-; CHECK-CPA-O3-NEXT:    mov w9, #2 // =0x2
 ; CHECK-CPA-O3-NEXT:    adrp x10, a
 ; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:a
 ; CHECK-CPA-O3-NEXT:    ldrh w8, [x8, :lo12:b]
-; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9
-; CHECK-CPA-O3-NEXT:    addpt x9, x9, x8, lsl #1
+; CHECK-CPA-O3-NEXT:    lsl x9, x8, #1
 ; CHECK-CPA-O3-NEXT:    add x8, x8, #1
+; CHECK-CPA-O3-NEXT:    add x9, x9, #2
+; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9
 ; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
 ; CHECK-CPA-O3-NEXT:    ldrb w8, [x8]
 ; CHECK-CPA-O3-NEXT:    cbz w8, .LBB14_2
diff --git a/llvm/test/CodeGen/AArch64/double_reduct.ll b/llvm/test/CodeGen/AArch64/double_reduct.ll
index f30895db2c09..e231b91f1199 100644
--- a/llvm/test/CodeGen/AArch64/double_reduct.ll
+++ b/llvm/test/CodeGen/AArch64/double_reduct.ll
@@ -833,13 +833,11 @@ define i32 @smax_i32_same(<4 x i32> %a, <4 x i32> %b) {
 define float @nested_fadd_f32(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-SD-LABEL: nested_fadd_f32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    faddp v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    fadd s2, s2, s3
 ; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-SD-NEXT:    faddp s1, v1.2s
 ; CHECK-SD-NEXT:    faddp s0, v0.2s
-; CHECK-SD-NEXT:    fadd s1, s1, s3
 ; CHECK-SD-NEXT:    fadd s0, s0, s2
-; CHECK-SD-NEXT:    fadd s0, s0, s1
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_fadd_f32:
@@ -905,15 +903,12 @@ define float @nested_fadd_f32_slow(<4 x float> %a, <4 x float> %b, float %c, flo
 define float @nested_mul_f32(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-SD-LABEL: nested_mul_f32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    fmul v1.2s, v1.2s, v4.2s
-; CHECK-SD-NEXT:    fmul v0.2s, v0.2s, v5.2s
-; CHECK-SD-NEXT:    fmul s1, s1, v1.s[1]
+; CHECK-SD-NEXT:    fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    fmul s2, s2, s3
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    fmul v0.2s, v0.2s, v1.2s
 ; CHECK-SD-NEXT:    fmul s0, s0, v0.s[1]
-; CHECK-SD-NEXT:    fmul s1, s1, s3
 ; CHECK-SD-NEXT:    fmul s0, s0, s2
-; CHECK-SD-NEXT:    fmul s0, s0, s1
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_mul_f32:
@@ -941,12 +936,10 @@ define float @nested_mul_f32(<4 x float> %a, <4 x float> %b, float %c, float %d)
 define i32 @nested_add_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_add_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    addv s1, v1.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    add w8, w0, w1
 ; CHECK-SD-NEXT:    addv s0, v0.4s
-; CHECK-SD-NEXT:    fmov w8, s1
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    add w9, w9, w0
-; CHECK-SD-NEXT:    add w8, w8, w1
 ; CHECK-SD-NEXT:    add w0, w9, w8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -971,12 +964,10 @@ define i32 @nested_add_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_add_c1_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_add_c1_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    addv s1, v1.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    add w8, w0, w1
 ; CHECK-SD-NEXT:    addv s0, v0.4s
-; CHECK-SD-NEXT:    fmov w8, s1
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    add w9, w0, w9
-; CHECK-SD-NEXT:    add w8, w8, w1
 ; CHECK-SD-NEXT:    add w0, w9, w8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1001,12 +992,10 @@ define i32 @nested_add_c1_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_add_c2_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_add_c2_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    addv s1, v1.4s
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    add w8, w0, w1
 ; CHECK-SD-NEXT:    addv s0, v0.4s
-; CHECK-SD-NEXT:    fmov w8, s1
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    add w9, w9, w0
-; CHECK-SD-NEXT:    add w8, w1, w8
 ; CHECK-SD-NEXT:    add w0, w9, w8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1065,19 +1054,14 @@ define i32 @nested_add_manyreduct_i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c,
 define i32 @nested_mul_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_mul_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v3.2s
-; CHECK-SD-NEXT:    mul v1.2s, v1.2s, v2.2s
-; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    mul w8, w0, w1
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    mul v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    mov w9, v0.s[1]
 ; CHECK-SD-NEXT:    fmov w10, s0
-; CHECK-SD-NEXT:    mov w9, v1.s[1]
-; CHECK-SD-NEXT:    mul w8, w10, w8
-; CHECK-SD-NEXT:    fmov w10, s1
 ; CHECK-SD-NEXT:    mul w9, w10, w9
-; CHECK-SD-NEXT:    mul w8, w8, w0
-; CHECK-SD-NEXT:    mul w9, w9, w1
-; CHECK-SD-NEXT:    mul w0, w8, w9
+; CHECK-SD-NEXT:    mul w0, w9, w8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_mul_i32:
@@ -1107,19 +1091,14 @@ define i32 @nested_mul_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_and_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_and_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v3.8b
-; CHECK-SD-NEXT:    fmov x8, d1
+; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    and w8, w0, w1
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    fmov x9, d0
 ; CHECK-SD-NEXT:    lsr x10, x9, #32
-; CHECK-SD-NEXT:    lsr x11, x8, #32
-; CHECK-SD-NEXT:    and w9, w9, w0
-; CHECK-SD-NEXT:    and w8, w8, w1
-; CHECK-SD-NEXT:    and w9, w9, w10
-; CHECK-SD-NEXT:    and w8, w8, w11
-; CHECK-SD-NEXT:    and w0, w9, w8
+; CHECK-SD-NEXT:    and w8, w9, w8
+; CHECK-SD-NEXT:    and w0, w8, w10
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_and_i32:
@@ -1149,19 +1128,14 @@ define i32 @nested_and_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_or_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_or_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    orr v1.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v3.8b
-; CHECK-SD-NEXT:    fmov x8, d1
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    orr w8, w0, w1
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    fmov x9, d0
 ; CHECK-SD-NEXT:    lsr x10, x9, #32
-; CHECK-SD-NEXT:    lsr x11, x8, #32
-; CHECK-SD-NEXT:    orr w9, w9, w0
-; CHECK-SD-NEXT:    orr w8, w8, w1
-; CHECK-SD-NEXT:    orr w9, w9, w10
-; CHECK-SD-NEXT:    orr w8, w8, w11
-; CHECK-SD-NEXT:    orr w0, w9, w8
+; CHECK-SD-NEXT:    orr w8, w9, w8
+; CHECK-SD-NEXT:    orr w0, w8, w10
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_or_i32:
@@ -1191,19 +1165,14 @@ define i32 @nested_or_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_xor_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_xor_i32:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    eor v1.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    eor v0.8b, v0.8b, v3.8b
-; CHECK-SD-NEXT:    fmov x8, d1
+; CHECK-SD-NEXT:    eor v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    eor w8, w0, w1
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    eor v0.8b, v0.8b, v1.8b
 ; CHECK-SD-NEXT:    fmov x9, d0
 ; CHECK-SD-NEXT:    lsr x10, x9, #32
-; CHECK-SD-NEXT:    lsr x11, x8, #32
-; CHECK-SD-NEXT:    eor w9, w9, w0
-; CHECK-SD-NEXT:    eor w8, w8, w1
-; CHECK-SD-NEXT:    eor w9, w9, w10
-; CHECK-SD-NEXT:    eor w8, w8, w11
-; CHECK-SD-NEXT:    eor w0, w9, w8
+; CHECK-SD-NEXT:    eor w8, w9, w8
+; CHECK-SD-NEXT:    eor w0, w8, w10
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_xor_i32:
@@ -1233,14 +1202,11 @@ define i32 @nested_xor_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_smin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_smin_i32:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    cmp w0, w1
+; CHECK-SD-NEXT:    csel w8, w0, w1, lt
 ; CHECK-SD-NEXT:    sminv s0, v0.4s
-; CHECK-SD-NEXT:    sminv s1, v1.4s
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    fmov w8, s1
-; CHECK-SD-NEXT:    cmp w9, w0
-; CHECK-SD-NEXT:    csel w9, w9, w0, lt
-; CHECK-SD-NEXT:    cmp w8, w1
-; CHECK-SD-NEXT:    csel w8, w8, w1, lt
 ; CHECK-SD-NEXT:    cmp w9, w8
 ; CHECK-SD-NEXT:    csel w0, w9, w8, lt
 ; CHECK-SD-NEXT:    ret
@@ -1269,14 +1235,11 @@ define i32 @nested_smin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_smax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_smax_i32:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    smax v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    cmp w0, w1
+; CHECK-SD-NEXT:    csel w8, w0, w1, gt
 ; CHECK-SD-NEXT:    smaxv s0, v0.4s
-; CHECK-SD-NEXT:    smaxv s1, v1.4s
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    fmov w8, s1
-; CHECK-SD-NEXT:    cmp w9, w0
-; CHECK-SD-NEXT:    csel w9, w9, w0, gt
-; CHECK-SD-NEXT:    cmp w8, w1
-; CHECK-SD-NEXT:    csel w8, w8, w1, gt
 ; CHECK-SD-NEXT:    cmp w9, w8
 ; CHECK-SD-NEXT:    csel w0, w9, w8, gt
 ; CHECK-SD-NEXT:    ret
@@ -1305,14 +1268,11 @@ define i32 @nested_smax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_umin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_umin_i32:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    cmp w0, w1
+; CHECK-SD-NEXT:    csel w8, w0, w1, lo
 ; CHECK-SD-NEXT:    uminv s0, v0.4s
-; CHECK-SD-NEXT:    uminv s1, v1.4s
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    fmov w8, s1
-; CHECK-SD-NEXT:    cmp w9, w0
-; CHECK-SD-NEXT:    csel w9, w9, w0, lo
-; CHECK-SD-NEXT:    cmp w8, w1
-; CHECK-SD-NEXT:    csel w8, w8, w1, lo
 ; CHECK-SD-NEXT:    cmp w9, w8
 ; CHECK-SD-NEXT:    csel w0, w9, w8, lo
 ; CHECK-SD-NEXT:    ret
@@ -1341,14 +1301,11 @@ define i32 @nested_umin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_umax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-SD-LABEL: nested_umax_i32:
 ; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    umax v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    cmp w0, w1
+; CHECK-SD-NEXT:    csel w8, w0, w1, hi
 ; CHECK-SD-NEXT:    umaxv s0, v0.4s
-; CHECK-SD-NEXT:    umaxv s1, v1.4s
 ; CHECK-SD-NEXT:    fmov w9, s0
-; CHECK-SD-NEXT:    fmov w8, s1
-; CHECK-SD-NEXT:    cmp w9, w0
-; CHECK-SD-NEXT:    csel w9, w9, w0, hi
-; CHECK-SD-NEXT:    cmp w8, w1
-; CHECK-SD-NEXT:    csel w8, w8, w1, hi
 ; CHECK-SD-NEXT:    cmp w9, w8
 ; CHECK-SD-NEXT:    csel w0, w9, w8, hi
 ; CHECK-SD-NEXT:    ret
@@ -1377,11 +1334,10 @@ define i32 @nested_umax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define float @nested_fmin_float(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-SD-LABEL: nested_fmin_float:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fminnmv s1, v1.4s
+; CHECK-SD-NEXT:    fminnm v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    fminnm s2, s2, s3
 ; CHECK-SD-NEXT:    fminnmv s0, v0.4s
-; CHECK-SD-NEXT:    fminnm s1, s1, s3
 ; CHECK-SD-NEXT:    fminnm s0, s0, s2
-; CHECK-SD-NEXT:    fminnm s0, s0, s1
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_fmin_float:
@@ -1403,11 +1359,10 @@ define float @nested_fmin_float(<4 x float> %a, <4 x float> %b, float %c, float
 define float @nested_fmax_float(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-SD-LABEL: nested_fmax_float:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fmaxnmv s1, v1.4s
+; CHECK-SD-NEXT:    fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    fmaxnm s2, s2, s3
 ; CHECK-SD-NEXT:    fmaxnmv s0, v0.4s
-; CHECK-SD-NEXT:    fmaxnm s1, s1, s3
 ; CHECK-SD-NEXT:    fmaxnm s0, s0, s2
-; CHECK-SD-NEXT:    fmaxnm s0, s0, s1
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: nested_fmax_float:
diff --git a/llvm/test/CodeGen/AArch64/exp10-libcall-names.ll b/llvm/test/CodeGen/AArch64/exp10-libcall-names.ll
index 6e603b7064f8..50358e5f1587 100644
--- a/llvm/test/CodeGen/AArch64/exp10-libcall-names.ll
+++ b/llvm/test/CodeGen/AArch64/exp10-libcall-names.ll
@@ -10,9 +10,9 @@
 ; RUN: llc -mtriple=arm64-apple-driverkit < %s | FileCheck -check-prefix=APPLE %s
 ; RUN: llc -mtriple=arm64-apple-driverkit1.0 < %s | FileCheck -check-prefix=APPLE %s
 ; RUN: llc -mtriple=arm64-apple-driverkit24.0 < %s | FileCheck -check-prefix=APPLE %s
-; RUN: llc -mtriple=arm64-apple-bridgeos < %s | FileCheck -check-prefix=BRIDGEOS %s
-; RUN: llc -mtriple=arm64-apple-bridgeos1.0 < %s | FileCheck -check-prefix=BRIDGEOS %s
-; RUN: llc -mtriple=arm64-apple-bridgeos9.0 < %s | FileCheck -check-prefix=BRIDGEOS %s
+; RUN: llc -mtriple=arm64-apple-bridgeos < %s | FileCheck -check-prefix=APPLE %s
+; RUN: llc -mtriple=arm64-apple-bridgeos1.0 < %s | FileCheck -check-prefix=APPLE %s
+; RUN: llc -mtriple=arm64-apple-bridgeos9.0 < %s | FileCheck -check-prefix=APPLE %s
 
 ; RUN: not llc -mtriple=aarch64-apple-macos10.8 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
 ; RUN: not llc -mtriple=aarch64-apple-ios6.0 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
@@ -29,11 +29,6 @@ define float @test_exp10_f32(float %x) {
 ; APPLE-LABEL: test_exp10_f32:
 ; APPLE:       ; %bb.0:
 ; APPLE-NEXT:    b ___exp10f
-;
-; BRIDGEOS-LABEL: test_exp10_f32:
-; BRIDGEOS:       // %bb.0:
-; BRIDGEOS-NEXT:    b __exp10f
-;
   %ret = call float @llvm.exp10.f32(float %x)
   ret float %ret
 }
@@ -46,11 +41,6 @@ define double @test_exp10_f64(double %x) {
 ; APPLE-LABEL: test_exp10_f64:
 ; APPLE:       ; %bb.0:
 ; APPLE-NEXT:    b ___exp10
-;
-; BRIDGEOS-LABEL: test_exp10_f64:
-; BRIDGEOS:       // %bb.0:
-; BRIDGEOS-NEXT:    b __exp10
-;
   %ret = call double @llvm.exp10.f64(double %x)
   ret double %ret
 }
diff --git a/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
index a1093c28467a..de9811b92424 100644
--- a/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
+++ b/llvm/test/CodeGen/AArch64/fp8-sve-cvt-cvtlt.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll b/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
index 2ffba10e2110..e42f2b1cfba4 100644
--- a/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
+++ b/llvm/test/CodeGen/AArch64/fp8-sve-cvtn.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2,+fp8 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2,+fp8 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2,+fp8 --force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index e8194b9bd9b2..217547c31f7f 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -4309,42 +4309,30 @@ entry:
 }
 
 define <2 x float> @stofp_v2i64_v2f32(<2 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v2i64_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    scvtf s0, x9
-; CHECK-SD-NEXT:    scvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i64_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: stofp_v2i64_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    scvtf s0, x9
+; CHECK-NEXT:    scvtf s1, x8
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %c
 }
 
 define <2 x float> @utofp_v2i64_v2f32(<2 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v2i64_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    ucvtf s0, x9
-; CHECK-SD-NEXT:    ucvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i64_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: utofp_v2i64_v2f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
+; CHECK-NEXT:    ucvtf s0, x9
+; CHECK-NEXT:    ucvtf s1, x8
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <2 x i64> %a to <2 x float>
   ret <2 x float> %c
@@ -4370,16 +4358,13 @@ define <3 x float> @stofp_v3i64_v3f32(<3 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v3i64_v3f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v0.2d
-; CHECK-GI-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s0, x8
+; CHECK-GI-NEXT:    scvtf s1, x9
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    scvtf s2, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -4407,16 +4392,13 @@ define <3 x float> @utofp_v3i64_v3f32(<3 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v3i64_v3f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v0.2d
-; CHECK-GI-NEXT:    mov v0.s[0], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[1]
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s0, x8
+; CHECK-GI-NEXT:    ucvtf s1, x9
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    ucvtf s2, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -4442,10 +4424,17 @@ define <4 x float> @stofp_v4i64_v4f32(<4 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v4i64_v4f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    scvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    scvtf s2, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    scvtf s1, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    scvtf s2, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i64> %a to <4 x float>
@@ -4470,10 +4459,17 @@ define <4 x float> @utofp_v4i64_v4f32(<4 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v4i64_v4f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ucvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    ucvtf s2, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    ucvtf s1, x8
+; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-GI-NEXT:    ucvtf s2, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i64> %a to <4 x float>
@@ -4510,14 +4506,29 @@ define <8 x float> @stofp_v8i64_v8f32(<8 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v8i64_v8f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v4.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    scvtf s0, x10
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    scvtf s5, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    scvtf s2, x10
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    scvtf s1, x9
+; CHECK-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT:    scvtf s3, x10
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-GI-NEXT:    scvtf s5, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x float>
@@ -4554,14 +4565,29 @@ define <8 x float> @utofp_v8i64_v8f32(<8 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v8i64_v8f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v4.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    ucvtf s0, x10
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    ucvtf s5, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    ucvtf s2, x10
+; CHECK-GI-NEXT:    fmov x10, d3
+; CHECK-GI-NEXT:    ucvtf s1, x9
+; CHECK-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-GI-NEXT:    ucvtf s3, x10
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-GI-NEXT:    ucvtf s5, x9
+; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x float>
@@ -4620,22 +4646,50 @@ define <16 x float> @stofp_v16i64_v16f32(<16 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v16i64_v16f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v16.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v7.2d
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x13, d2
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    scvtf s0, x11
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    scvtf s16, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    scvtf s1, x13
+; CHECK-GI-NEXT:    fmov x13, d4
+; CHECK-GI-NEXT:    scvtf s4, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    scvtf s17, x10
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    scvtf s6, x11
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    scvtf s18, x9
+; CHECK-GI-NEXT:    scvtf s2, x13
+; CHECK-GI-NEXT:    fmov x13, d3
+; CHECK-GI-NEXT:    scvtf s3, x12
+; CHECK-GI-NEXT:    mov x9, v5.d[1]
+; CHECK-GI-NEXT:    mov x12, v7.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
+; CHECK-GI-NEXT:    scvtf s5, x11
+; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
+; CHECK-GI-NEXT:    scvtf s19, x13
+; CHECK-GI-NEXT:    fmov x13, d7
+; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
+; CHECK-GI-NEXT:    scvtf s4, x8
+; CHECK-GI-NEXT:    scvtf s6, x10
+; CHECK-GI-NEXT:    scvtf s16, x9
+; CHECK-GI-NEXT:    scvtf s17, x12
+; CHECK-GI-NEXT:    scvtf s7, x13
+; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i64> %a to <16 x float>
@@ -4694,22 +4748,50 @@ define <16 x float> @utofp_v16i64_v16f32(<16 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v16i64_v16f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v16.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v7.2d
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    fmov x13, d2
+; CHECK-GI-NEXT:    fmov x11, d0
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mov x10, v2.d[1]
+; CHECK-GI-NEXT:    ucvtf s0, x11
+; CHECK-GI-NEXT:    mov x11, v6.d[1]
+; CHECK-GI-NEXT:    ucvtf s16, x9
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    ucvtf s1, x13
+; CHECK-GI-NEXT:    fmov x13, d4
+; CHECK-GI-NEXT:    ucvtf s4, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    ucvtf s17, x10
+; CHECK-GI-NEXT:    mov x10, v3.d[1]
+; CHECK-GI-NEXT:    ucvtf s6, x11
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ucvtf s18, x9
+; CHECK-GI-NEXT:    ucvtf s2, x13
+; CHECK-GI-NEXT:    fmov x13, d3
+; CHECK-GI-NEXT:    ucvtf s3, x12
+; CHECK-GI-NEXT:    mov x9, v5.d[1]
+; CHECK-GI-NEXT:    mov x12, v7.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
+; CHECK-GI-NEXT:    ucvtf s5, x11
+; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
+; CHECK-GI-NEXT:    ucvtf s19, x13
+; CHECK-GI-NEXT:    fmov x13, d7
+; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
+; CHECK-GI-NEXT:    ucvtf s4, x8
+; CHECK-GI-NEXT:    ucvtf s6, x10
+; CHECK-GI-NEXT:    ucvtf s16, x9
+; CHECK-GI-NEXT:    ucvtf s17, x12
+; CHECK-GI-NEXT:    ucvtf s7, x13
+; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i64> %a to <16 x float>
@@ -4816,42 +4898,110 @@ define <32 x float> @stofp_v32i64_v32f32(<32 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: stofp_v32i64_v32f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    scvtf v24.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    scvtf v25.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v26.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v27.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v18.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v20.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v22.2d
-; CHECK-GI-NEXT:    scvtf v16.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v19.2d
-; CHECK-GI-NEXT:    scvtf v18.2d, v21.2d
-; CHECK-GI-NEXT:    scvtf v19.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v1.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v5.2s, v5.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v7.2s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v24.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v25.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v26.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v27.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v5.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v18.2d
-; CHECK-GI-NEXT:    fcvtn2 v7.4s, v19.2d
+; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset b8, -8
+; CHECK-GI-NEXT:    .cfi_offset b9, -16
+; CHECK-GI-NEXT:    .cfi_offset b10, -24
+; CHECK-GI-NEXT:    .cfi_offset b11, -32
+; CHECK-GI-NEXT:    .cfi_offset b12, -48
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
+; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
+; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
+; CHECK-GI-NEXT:    scvtf s28, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    scvtf s16, x10
+; CHECK-GI-NEXT:    mov x10, v6.d[1]
+; CHECK-GI-NEXT:    scvtf s9, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    scvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    scvtf s18, x11
+; CHECK-GI-NEXT:    scvtf s17, x8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
+; CHECK-GI-NEXT:    mov x13, v21.d[1]
+; CHECK-GI-NEXT:    scvtf s10, x10
+; CHECK-GI-NEXT:    fmov x10, d21
+; CHECK-GI-NEXT:    fmov x14, d30
+; CHECK-GI-NEXT:    scvtf s1, x8
+; CHECK-GI-NEXT:    fmov x8, d3
+; CHECK-GI-NEXT:    scvtf s3, x12
+; CHECK-GI-NEXT:    mov x12, v30.d[1]
+; CHECK-GI-NEXT:    scvtf s22, x11
+; CHECK-GI-NEXT:    fmov x11, d11
+; CHECK-GI-NEXT:    scvtf s29, x9
+; CHECK-GI-NEXT:    fmov x9, d4
+; CHECK-GI-NEXT:    scvtf s4, x10
+; CHECK-GI-NEXT:    scvtf s19, x8
+; CHECK-GI-NEXT:    mov x8, v5.d[1]
+; CHECK-GI-NEXT:    mov x10, v11.d[1]
+; CHECK-GI-NEXT:    scvtf s5, x11
+; CHECK-GI-NEXT:    mov x11, v31.d[1]
+; CHECK-GI-NEXT:    scvtf s8, x13
+; CHECK-GI-NEXT:    scvtf s11, x12
+; CHECK-GI-NEXT:    fmov x12, d31
+; CHECK-GI-NEXT:    scvtf s2, x9
+; CHECK-GI-NEXT:    mov x9, v7.d[1]
+; CHECK-GI-NEXT:    scvtf s6, x14
+; CHECK-GI-NEXT:    fmov x13, d27
+; CHECK-GI-NEXT:    scvtf s20, x8
+; CHECK-GI-NEXT:    fmov x8, d7
+; CHECK-GI-NEXT:    scvtf s12, x10
+; CHECK-GI-NEXT:    scvtf s7, x12
+; CHECK-GI-NEXT:    fmov x12, d25
+; CHECK-GI-NEXT:    scvtf s30, x11
+; CHECK-GI-NEXT:    mov x10, v25.d[1]
+; CHECK-GI-NEXT:    mov x11, v26.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
+; CHECK-GI-NEXT:    scvtf s21, x9
+; CHECK-GI-NEXT:    fmov x9, d24
+; CHECK-GI-NEXT:    scvtf s23, x8
+; CHECK-GI-NEXT:    scvtf s25, x12
+; CHECK-GI-NEXT:    fmov x12, d26
+; CHECK-GI-NEXT:    mov x8, v24.d[1]
+; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
+; CHECK-GI-NEXT:    scvtf s24, x9
+; CHECK-GI-NEXT:    mov x9, v27.d[1]
+; CHECK-GI-NEXT:    scvtf s27, x13
+; CHECK-GI-NEXT:    scvtf s26, x12
+; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
+; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
+; CHECK-GI-NEXT:    scvtf s28, x8
+; CHECK-GI-NEXT:    scvtf s29, x9
+; CHECK-GI-NEXT:    scvtf s30, x10
+; CHECK-GI-NEXT:    scvtf s31, x11
+; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
+; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
+; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
+; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
+; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
+; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i64> %a to <32 x float>
@@ -4958,42 +5108,110 @@ define <32 x float> @utofp_v32i64_v32f32(<32 x i64> %a) {
 ;
 ; CHECK-GI-LABEL: utofp_v32i64_v32f32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    ucvtf v24.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    ucvtf v25.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v26.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v27.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v18.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v20.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v22.2d
-; CHECK-GI-NEXT:    ucvtf v16.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v19.2d
-; CHECK-GI-NEXT:    ucvtf v18.2d, v21.2d
-; CHECK-GI-NEXT:    ucvtf v19.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v1.2s, v1.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v3.2s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v5.2s, v5.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v7.2s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v24.2d
-; CHECK-GI-NEXT:    fcvtn2 v1.4s, v25.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v26.2d
-; CHECK-GI-NEXT:    fcvtn2 v3.4s, v27.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v16.2d
-; CHECK-GI-NEXT:    fcvtn2 v5.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v18.2d
-; CHECK-GI-NEXT:    fcvtn2 v7.4s, v19.2d
+; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset b8, -8
+; CHECK-GI-NEXT:    .cfi_offset b9, -16
+; CHECK-GI-NEXT:    .cfi_offset b10, -24
+; CHECK-GI-NEXT:    .cfi_offset b11, -32
+; CHECK-GI-NEXT:    .cfi_offset b12, -48
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x10, v1.d[1]
+; CHECK-GI-NEXT:    mov x12, v4.d[1]
+; CHECK-GI-NEXT:    mov x11, v3.d[1]
+; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
+; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
+; CHECK-GI-NEXT:    ucvtf s28, x8
+; CHECK-GI-NEXT:    fmov x8, d1
+; CHECK-GI-NEXT:    ucvtf s16, x10
+; CHECK-GI-NEXT:    mov x10, v6.d[1]
+; CHECK-GI-NEXT:    ucvtf s9, x12
+; CHECK-GI-NEXT:    fmov x12, d6
+; CHECK-GI-NEXT:    ucvtf s0, x9
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ucvtf s18, x11
+; CHECK-GI-NEXT:    ucvtf s17, x8
+; CHECK-GI-NEXT:    fmov x8, d2
+; CHECK-GI-NEXT:    fmov x11, d5
+; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
+; CHECK-GI-NEXT:    mov x13, v21.d[1]
+; CHECK-GI-NEXT:    ucvtf s10, x10
+; CHECK-GI-NEXT:    fmov x10, d21
+; CHECK-GI-NEXT:    fmov x14, d30
+; CHECK-GI-NEXT:    ucvtf s1, x8
+; CHECK-GI-NEXT:    fmov x8, d3
+; CHECK-GI-NEXT:    ucvtf s3, x12
+; CHECK-GI-NEXT:    mov x12, v30.d[1]
+; CHECK-GI-NEXT:    ucvtf s22, x11
+; CHECK-GI-NEXT:    fmov x11, d11
+; CHECK-GI-NEXT:    ucvtf s29, x9
+; CHECK-GI-NEXT:    fmov x9, d4
+; CHECK-GI-NEXT:    ucvtf s4, x10
+; CHECK-GI-NEXT:    ucvtf s19, x8
+; CHECK-GI-NEXT:    mov x8, v5.d[1]
+; CHECK-GI-NEXT:    mov x10, v11.d[1]
+; CHECK-GI-NEXT:    ucvtf s5, x11
+; CHECK-GI-NEXT:    mov x11, v31.d[1]
+; CHECK-GI-NEXT:    ucvtf s8, x13
+; CHECK-GI-NEXT:    ucvtf s11, x12
+; CHECK-GI-NEXT:    fmov x12, d31
+; CHECK-GI-NEXT:    ucvtf s2, x9
+; CHECK-GI-NEXT:    mov x9, v7.d[1]
+; CHECK-GI-NEXT:    ucvtf s6, x14
+; CHECK-GI-NEXT:    fmov x13, d27
+; CHECK-GI-NEXT:    ucvtf s20, x8
+; CHECK-GI-NEXT:    fmov x8, d7
+; CHECK-GI-NEXT:    ucvtf s12, x10
+; CHECK-GI-NEXT:    ucvtf s7, x12
+; CHECK-GI-NEXT:    fmov x12, d25
+; CHECK-GI-NEXT:    ucvtf s30, x11
+; CHECK-GI-NEXT:    mov x10, v25.d[1]
+; CHECK-GI-NEXT:    mov x11, v26.d[1]
+; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
+; CHECK-GI-NEXT:    ucvtf s21, x9
+; CHECK-GI-NEXT:    fmov x9, d24
+; CHECK-GI-NEXT:    ucvtf s23, x8
+; CHECK-GI-NEXT:    ucvtf s25, x12
+; CHECK-GI-NEXT:    fmov x12, d26
+; CHECK-GI-NEXT:    mov x8, v24.d[1]
+; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
+; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
+; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
+; CHECK-GI-NEXT:    ucvtf s24, x9
+; CHECK-GI-NEXT:    mov x9, v27.d[1]
+; CHECK-GI-NEXT:    ucvtf s27, x13
+; CHECK-GI-NEXT:    ucvtf s26, x12
+; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
+; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
+; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
+; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
+; CHECK-GI-NEXT:    ucvtf s28, x8
+; CHECK-GI-NEXT:    ucvtf s29, x9
+; CHECK-GI-NEXT:    ucvtf s30, x10
+; CHECK-GI-NEXT:    ucvtf s31, x11
+; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
+; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
+; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
+; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
+; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
+; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
+; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
+; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
+; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
+; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
+; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
+; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
+; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
+; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i64> %a to <32 x float>
@@ -6389,24 +6607,14 @@ define <2 x half> @stofp_v2i64_v2f16(<2 x i64> %a) {
 ; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v2i64_v2f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
@@ -6435,198 +6643,78 @@ define <2 x half> @utofp_v2i64_v2f16(<2 x i64> %a) {
 ; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v2i64_v2f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <3 x half> @stofp_v3i64_v3f16(<3 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v3i64_v3f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v3i64_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v3i64_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: stofp_v3i64_v3f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <3 x i64> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <3 x half> @utofp_v3i64_v3f16(<3 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v3i64_v3f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v3i64_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v3i64_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: utofp_v3i64_v3f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <3 x i64> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <4 x half> @stofp_v4i64_v4f16(<4 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v4i64_v4f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v4i64_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v4i64_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: stofp_v4i64_v4f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = sitofp <4 x i64> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <4 x half> @utofp_v4i64_v4f16(<4 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v4i64_v4f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v4i64_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v4i64_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d3, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v2.h[0]
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-LABEL: utofp_v4i64_v4f16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
 entry:
   %c = uitofp <4 x i64> %a to <4 x half>
   ret <4 x half> %c
@@ -6647,46 +6735,19 @@ define <8 x half> @stofp_v8i64_v8f16(<8 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v8i64_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v8i64_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d4, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d5, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d5
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v1.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v8i64_v8f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
@@ -6707,46 +6768,19 @@ define <8 x half> @utofp_v8i64_v8f16(<8 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v8i64_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v8i64_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d4, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    mov d5, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d5
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v1.h[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v4.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d1
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v1.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d2
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v1.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v8i64_v8f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
@@ -6777,79 +6811,29 @@ define <16 x half> @stofp_v16i64_v16f16(<16 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v16i64_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v16i64_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    scvtf v18.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d16, v0.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    fcvt h4, d17
-; CHECK-GI-FP16-NEXT:    mov d17, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt h6, d17
-; CHECK-GI-FP16-NEXT:    fcvt h16, d19
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov d5, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v16.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v6.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h7, d17
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov d4, v6.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v16i64_v16f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
@@ -6880,79 +6864,29 @@ define <16 x half> @utofp_v16i64_v16f16(<16 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v16i64_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v16i64_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v18.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d16, v0.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    fcvt h4, d17
-; CHECK-GI-FP16-NEXT:    mov d17, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt h6, d17
-; CHECK-GI-FP16-NEXT:    fcvt h16, d19
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov d5, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov d17, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v16.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v6.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvt h5, d5
-; CHECK-GI-FP16-NEXT:    fcvt h7, d17
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v4.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    mov d4, v6.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v5.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v7.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    fcvt h4, d4
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v4.h[0]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v16i64_v16f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
@@ -7007,155 +6941,53 @@ define <32 x half> @stofp_v32i64_v32f16(<32 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: stofp_v32i64_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NOFP16-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v21.2d, v21.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v22.2d, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    scvtf v23.2d, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v32i64_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ldp q16, q18, [sp]
-; CHECK-GI-FP16-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ldp q17, q19, [sp, #64]
-; CHECK-GI-FP16-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    scvtf v20.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    scvtf v24.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    scvtf v16.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    mov d21, v0.d[1]
-; CHECK-GI-FP16-NEXT:    scvtf v25.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov d22, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d18, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d1
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v20.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    fcvt h4, d20
-; CHECK-GI-FP16-NEXT:    mov d26, v16.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d5
-; CHECK-GI-FP16-NEXT:    fcvt h5, d16
-; CHECK-GI-FP16-NEXT:    fcvt h28, d21
-; CHECK-GI-FP16-NEXT:    fcvt h29, d22
-; CHECK-GI-FP16-NEXT:    fcvt h22, d24
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h19, d19
-; CHECK-GI-FP16-NEXT:    fcvt h27, d23
-; CHECK-GI-FP16-NEXT:    mov d23, v24.d[1]
-; CHECK-GI-FP16-NEXT:    mov d24, v25.d[1]
-; CHECK-GI-FP16-NEXT:    ldp q25, q16, [sp, #32]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v28.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v29.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[1], v27.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v25.2d, v25.2d
-; CHECK-GI-FP16-NEXT:    fcvt h23, d23
-; CHECK-GI-FP16-NEXT:    mov v5.h[1], v26.h[0]
-; CHECK-GI-FP16-NEXT:    ldp q26, q27, [sp, #96]
-; CHECK-GI-FP16-NEXT:    fcvt h24, d24
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v20.h[0]
-; CHECK-GI-FP16-NEXT:    mov d20, v6.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    scvtf v26.2d, v26.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[2], v22.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h22, d25
-; CHECK-GI-FP16-NEXT:    mov v5.h[2], v21.h[0]
-; CHECK-GI-FP16-NEXT:    mov d21, v25.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v19.h[0]
-; CHECK-GI-FP16-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    scvtf v18.2d, v27.2d
-; CHECK-GI-FP16-NEXT:    fcvt h19, d20
-; CHECK-GI-FP16-NEXT:    fcvt h17, d17
-; CHECK-GI-FP16-NEXT:    mov d25, v26.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v4.h[3], v23.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[3], v24.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d21
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    mov v4.h[4], v22.h[0]
-; CHECK-GI-FP16-NEXT:    mov d22, v16.d[1]
-; CHECK-GI-FP16-NEXT:    mov v5.h[4], v26.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    mov d6, v7.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h7, d7
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v19.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h19, d23
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v4.h[5], v20.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d22
-; CHECK-GI-FP16-NEXT:    mov v5.h[5], v21.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v7.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[6], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[6], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[7], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[7], v19.h[0]
-; CHECK-GI-FP16-NEXT:    mov v2.16b, v4.16b
-; CHECK-GI-FP16-NEXT:    mov v3.16b, v5.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: stofp_v32i64_v32f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-GI-NEXT:    scvtf v20.2d, v20.2d
+; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-GI-NEXT:    scvtf v18.2d, v18.2d
+; CHECK-GI-NEXT:    scvtf v21.2d, v21.2d
+; CHECK-GI-NEXT:    scvtf v22.2d, v22.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
+; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
+; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    scvtf v19.2d, v19.2d
+; CHECK-GI-NEXT:    scvtf v23.2d, v23.2d
+; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
+; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
+; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
+; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
+; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
+; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i64> %a to <32 x half>
   ret <32 x half> %c
@@ -7210,155 +7042,53 @@ define <32 x half> @utofp_v32i64_v32f16(<32 x i64> %a) {
 ; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
 ; CHECK-SD-NEXT:    ret
 ;
-; CHECK-GI-NOFP16-LABEL: utofp_v32i64_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v21.2d, v21.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v22.2d, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v19.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    ucvtf v23.2d, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v32i64_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ldp q16, q18, [sp]
-; CHECK-GI-FP16-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ldp q17, q19, [sp, #64]
-; CHECK-GI-FP16-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v20.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v24.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v16.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    mov d21, v0.d[1]
-; CHECK-GI-FP16-NEXT:    ucvtf v25.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov d22, v4.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h0, d0
-; CHECK-GI-FP16-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    mov d18, v1.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d1
-; CHECK-GI-FP16-NEXT:    mov d19, v5.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v20.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h1, d4
-; CHECK-GI-FP16-NEXT:    fcvt h4, d20
-; CHECK-GI-FP16-NEXT:    mov d26, v16.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d5
-; CHECK-GI-FP16-NEXT:    fcvt h5, d16
-; CHECK-GI-FP16-NEXT:    fcvt h28, d21
-; CHECK-GI-FP16-NEXT:    fcvt h29, d22
-; CHECK-GI-FP16-NEXT:    fcvt h22, d24
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h19, d19
-; CHECK-GI-FP16-NEXT:    fcvt h27, d23
-; CHECK-GI-FP16-NEXT:    mov d23, v24.d[1]
-; CHECK-GI-FP16-NEXT:    mov d24, v25.d[1]
-; CHECK-GI-FP16-NEXT:    ldp q25, q16, [sp, #32]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v0.h[1], v28.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v29.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[1], v27.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v25.2d, v25.2d
-; CHECK-GI-FP16-NEXT:    fcvt h23, d23
-; CHECK-GI-FP16-NEXT:    mov v5.h[1], v26.h[0]
-; CHECK-GI-FP16-NEXT:    ldp q26, q27, [sp, #96]
-; CHECK-GI-FP16-NEXT:    fcvt h24, d24
-; CHECK-GI-FP16-NEXT:    mov v0.h[2], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v20.h[0]
-; CHECK-GI-FP16-NEXT:    mov d20, v6.d[1]
-; CHECK-GI-FP16-NEXT:    mov d17, v2.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    ucvtf v26.2d, v26.2d
-; CHECK-GI-FP16-NEXT:    mov v4.h[2], v22.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h22, d25
-; CHECK-GI-FP16-NEXT:    mov v5.h[2], v21.h[0]
-; CHECK-GI-FP16-NEXT:    mov d21, v25.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[3], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[3], v19.h[0]
-; CHECK-GI-FP16-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    ucvtf v18.2d, v27.2d
-; CHECK-GI-FP16-NEXT:    fcvt h19, d20
-; CHECK-GI-FP16-NEXT:    fcvt h17, d17
-; CHECK-GI-FP16-NEXT:    mov d25, v26.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h26, d26
-; CHECK-GI-FP16-NEXT:    mov v4.h[3], v23.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[3], v24.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h20, d21
-; CHECK-GI-FP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[4], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov d2, v3.d[1]
-; CHECK-GI-FP16-NEXT:    mov d23, v18.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h18, d18
-; CHECK-GI-FP16-NEXT:    fcvt h3, d3
-; CHECK-GI-FP16-NEXT:    fcvt h21, d25
-; CHECK-GI-FP16-NEXT:    mov v4.h[4], v22.h[0]
-; CHECK-GI-FP16-NEXT:    mov d22, v16.d[1]
-; CHECK-GI-FP16-NEXT:    mov v5.h[4], v26.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h16, d16
-; CHECK-GI-FP16-NEXT:    mov d6, v7.d[1]
-; CHECK-GI-FP16-NEXT:    fcvt h7, d7
-; CHECK-GI-FP16-NEXT:    mov v0.h[5], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[5], v19.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h19, d23
-; CHECK-GI-FP16-NEXT:    fcvt h2, d2
-; CHECK-GI-FP16-NEXT:    mov v4.h[5], v20.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h17, d22
-; CHECK-GI-FP16-NEXT:    mov v5.h[5], v21.h[0]
-; CHECK-GI-FP16-NEXT:    fcvt h6, d6
-; CHECK-GI-FP16-NEXT:    mov v0.h[6], v3.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[6], v7.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[6], v16.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[6], v18.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.h[7], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[7], v6.h[0]
-; CHECK-GI-FP16-NEXT:    mov v4.h[7], v17.h[0]
-; CHECK-GI-FP16-NEXT:    mov v5.h[7], v19.h[0]
-; CHECK-GI-FP16-NEXT:    mov v2.16b, v4.16b
-; CHECK-GI-FP16-NEXT:    mov v3.16b, v5.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-GI-LABEL: utofp_v32i64_v32f16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-GI-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-GI-NEXT:    ucvtf v20.2d, v20.2d
+; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-GI-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-GI-NEXT:    ucvtf v18.2d, v18.2d
+; CHECK-GI-NEXT:    ucvtf v21.2d, v21.2d
+; CHECK-GI-NEXT:    ucvtf v22.2d, v22.2d
+; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
+; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
+; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-GI-NEXT:    ucvtf v19.2d, v19.2d
+; CHECK-GI-NEXT:    ucvtf v23.2d, v23.2d
+; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
+; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
+; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
+; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
+; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
+; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
+; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
+; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
+; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
+; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
+; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
+; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
+; CHECK-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i64> %a to <32 x half>
   ret <32 x half> %c
diff --git a/llvm/test/CodeGen/AArch64/llvm.frexp.ll b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
index e4cb8ed6eaf9..092fe16cbc2f 100644
--- a/llvm/test/CodeGen/AArch64/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
@@ -1,5 +1,111 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-gnu-linux < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -mtriple=aarch64-windows-pc-msvc < %s | FileCheck -check-prefixes=WINDOWS %s
+define { half, i32 } @test_frexp_f16_i32(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32:
+; WINDOWS:       .seh_proc test_frexp_f16_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  ret { half, i32 } %result
+}
+
+define half @test_frexp_f16_i32_only_use_fract(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f16_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  %result.0 = extractvalue { half, i32 } %result, 0
+  ret half %result.0
+}
+
+define i32 @test_frexp_f16_i32_only_use_exp(half %a) {
+; CHECK-LABEL: test_frexp_f16_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f16_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f16_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  %result.0 = extractvalue { half, i32 } %result, 1
+  ret i32 %result.0
+}
 
 define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; CHECK-LABEL: test_frexp_v2f16_v2i32:
@@ -51,10 +157,240 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x19, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov h1, v0.h[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    fcvt d1, h1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    mov h1, v1.h[2]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v2.h[1], v1.h[0]
+; WINDOWS-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    mov h1, v1.h[3]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v1.h[2], v2.h[0]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    mov v0.h[3], v2.h[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    ldr x19, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
   %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
   ret { <2 x half>, <2 x i32> } %result
 }
 
+define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
+; CHECK-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    mov h1, v0.h[1]
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    mov h1, v1.h[2]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v2.h[1], v1.h[0]
+; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvt h2, s0
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    mov h1, v1.h[3]
+; CHECK-NEXT:    fcvt s0, h1
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v1.h[2], v2.h[0]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    fcvt h1, s0
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.h[3], v1.h[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov h1, v0.h[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h0, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    fcvt d1, h1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #56
+; WINDOWS-NEXT:    mov h1, v1.h[2]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v2.h[1], v1.h[0]
+; WINDOWS-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt h2, d0
+; WINDOWS-NEXT:    add x0, sp, #60
+; WINDOWS-NEXT:    mov h1, v1.h[3]
+; WINDOWS-NEXT:    fcvt d0, h1
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v1.h[2], v2.h[0]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt h1, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.h[3], v1.h[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
+  %result.0 = extractvalue { <2 x half>, <2 x i32> } %result, 0
+  ret <2 x half> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
+; CHECK-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    add x19, sp, #28
+; CHECK-NEXT:    mov h0, v0.h[1]
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #24]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f16_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f16_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    mov h0, v0.h[1]
+; WINDOWS-NEXT:    fcvt d0, h0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #24]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a)
+  %result.1 = extractvalue { <2 x half>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
 define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
 ; CHECK-LABEL: test_frexp_v3f16_v3i32:
 ; CHECK:       // %bb.0:
@@ -96,6 +432,1015 @@ define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
 ; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v3f16_v3i32:
+; WINDOWS:       .seh_proc test_frexp_v3f16_v3i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    stp x19, x20, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    stp x21, x30, [sp, #64] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x20, sp, #40
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    add x21, sp, #44
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    fcvt s2, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.s[3], v2.s[0]
+; WINDOWS-NEXT:    ld1 { v1.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v1.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #64] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    add sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
   %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3float.v3i32(<3 x float> %a)
   ret { <3 x float>, <3 x i32> } %result
 }
+
+define { float, i32 } @test_frexp_f32_i32(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32:
+; WINDOWS:       .seh_proc test_frexp_f32_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+define { float, i32 } @test_frexp_f32_i32_tailcall(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_tailcall:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_tailcall:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_tailcall
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = tail call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+define float @test_frexp_f32_i32_only_use_fract(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  %result.0 = extractvalue { float, i32 } %result, 0
+  ret float %result.0
+}
+
+define i32 @test_frexp_f32_i32_only_use_exp(float %a) {
+; CHECK-LABEL: test_frexp_f32_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f32_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f32_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  %result.0 = extractvalue { float, i32 } %result, 1
+  ret i32 %result.0
+}
+
+define { <2 x float>, <2 x i32> } @test_frexp_v2f32_v2i32(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d1, s1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr s1, [sp, #24]
+; WINDOWS-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    mov v0.s[1], v2.s[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  ret { <2 x float>, <2 x i32> } %result
+}
+
+define <2 x float> @test_frexp_v2f32_v2i32_only_use_fract(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #32
+; WINDOWS-NEXT:    .seh_stackalloc 32
+; WINDOWS-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d1, s1
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fmov d0, d1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 16
+; WINDOWS-NEXT:    add sp, sp, #32
+; WINDOWS-NEXT:    .seh_stackalloc 32
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  %result.0 = extractvalue { <2 x float>, <2 x i32> } %result, 0
+  ret <2 x float> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f32_v2i32_only_use_exp(<2 x float> %a) {
+; CHECK-LABEL: test_frexp_v2f32_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f32_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f32_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x19, sp, #28
+; WINDOWS-NEXT:    mov s0, v0.s[1]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #24]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float> %a)
+  %result.1 = extractvalue { <2 x float>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
+define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    stp x30, x21, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #36
+; CHECK-NEXT:    add x20, sp, #36
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #32
+; CHECK-NEXT:    add x21, sp, #32
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    mov v2.s[3], v0.s[0]
+; CHECK-NEXT:    ld1 { v1.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    ld1 { v1.s }[3], [x21]
+; CHECK-NEXT:    ldp x30, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    stp x19, x20, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    stp x21, x30, [sp, #64] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #36
+; WINDOWS-NEXT:    add x19, sp, #36
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #32
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x20, sp, #40
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    add x21, sp, #44
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #32]
+; WINDOWS-NEXT:    fcvt s2, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.s[3], v2.s[0]
+; WINDOWS-NEXT:    ld1 { v1.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v1.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #64] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 64
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 48
+; WINDOWS-NEXT:    add sp, sp, #80
+; WINDOWS-NEXT:    .seh_stackalloc 80
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  ret { <4 x float>, <4 x i32> } %result
+}
+
+define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    add x0, sp, #56
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #60
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    mov v1.s[2], v0.s[0]
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.s[3], v0.s[0]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    mov s1, v0.s[1]
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    fcvt d0, s1
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #56
+; WINDOWS-NEXT:    mov v0.s[1], v1.s[0]
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s0, d0
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #60
+; WINDOWS-NEXT:    mov v1.s[2], v0.s[0]
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    fcvt s1, d0
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    mov v0.s[3], v1.s[0]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  %result.0 = extractvalue { <4 x float>, <4 x i32> } %result, 0
+  ret <4 x float> %result.0
+}
+
+define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) {
+; CHECK-LABEL: test_frexp_v4f32_v4i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x21, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -24
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov s0, v0.s[1]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #20
+; CHECK-NEXT:    add x20, sp, #20
+; CHECK-NEXT:    mov s0, v0.s[2]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #16
+; CHECK-NEXT:    add x21, sp, #16
+; CHECK-NEXT:    mov s0, v0.s[3]
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x20]
+; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ld1 { v0.s }[3], [x21]
+; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v4f32_v4i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v4f32_v4i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    stp x19, x20, [sp, #32] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_regp x19, 32
+; WINDOWS-NEXT:    stp x21, x30, [sp, #48] // 16-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 48
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    add x0, sp, #16
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #20
+; WINDOWS-NEXT:    add x19, sp, #20
+; WINDOWS-NEXT:    mov s0, v0.s[1]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    add x20, sp, #24
+; WINDOWS-NEXT:    mov s0, v0.s[2]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    add x21, sp, #28
+; WINDOWS-NEXT:    mov s0, v0.s[3]
+; WINDOWS-NEXT:    fcvt d0, s0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #16]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    ld1 { v0.s }[2], [x20]
+; WINDOWS-NEXT:    ld1 { v0.s }[3], [x21]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldp x21, x30, [sp, #48] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_lrpair x21, 48
+; WINDOWS-NEXT:    ldp x19, x20, [sp, #32] // 16-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_regp x19, 32
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float> %a)
+  %result.1 = extractvalue { <4 x float>, <4 x i32> } %result, 1
+  ret <4 x i32> %result.1
+}
+
+define { double, i32 } @test_frexp_f64_i32(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32:
+; WINDOWS:       .seh_proc test_frexp_f64_i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  ret { double, i32 } %result
+}
+
+define double @test_frexp_f64_i32_only_use_fract(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_f64_i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  %result.0 = extractvalue { double, i32 } %result, 0
+  ret double %result.0
+}
+
+define i32 @test_frexp_f64_i32_only_use_exp(double %a) {
+; CHECK-LABEL: test_frexp_f64_i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #12
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr w0, [sp, #12]
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_f64_i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_f64_i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #12
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr w0, [sp, #12]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg_x x30, 16
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  %result.0 = extractvalue { double, i32 } %result, 1
+  ret i32 %result.0
+}
+
+define { <2 x double>, <2 x i32> } @test_frexp_v2f64_v2i32(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    stp x30, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    add x19, sp, #40
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr s1, [sp, #44]
+; CHECK-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    str x19, [sp, #48] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    add x19, sp, #40
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s1, [sp, #44]
+; WINDOWS-NEXT:    ldr q2, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    ld1 { v1.s }[1], [x19]
+; WINDOWS-NEXT:    mov v0.d[1], v2.d[0]
+; WINDOWS-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 56
+; WINDOWS-NEXT:    ldr x19, [sp, #48] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 48
+; WINDOWS-NEXT:    add sp, sp, #64
+; WINDOWS-NEXT:    .seh_stackalloc 64
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  ret { <2 x double>, <2 x i32> } %result
+}
+
+define <2 x double> @test_frexp_v2f64_v2i32_only_use_fract(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32_only_use_fract:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    add x0, sp, #40
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    add x0, sp, #44
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32_only_use_fract:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32_only_use_fract
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 32
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    add x0, sp, #40
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    add x0, sp, #44
+; WINDOWS-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 def $q0
+; WINDOWS-NEXT:    mov v0.d[1], v1.d[0]
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  %result.0 = extractvalue { <2 x double>, <2 x i32> } %result, 0
+  ret <2 x double> %result.0
+}
+
+define <2 x i32> @test_frexp_v2f64_v2i32_only_use_exp(<2 x double> %a) {
+; CHECK-LABEL: test_frexp_v2f64_v2i32_only_use_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    stp x30, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    add x0, sp, #28
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x0, sp, #24
+; CHECK-NEXT:    add x19, sp, #24
+; CHECK-NEXT:    mov d0, v0.d[1]
+; CHECK-NEXT:    bl frexp
+; CHECK-NEXT:    ldr s0, [sp, #28]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+;
+; WINDOWS-LABEL: test_frexp_v2f64_v2i32_only_use_exp:
+; WINDOWS:       .seh_proc test_frexp_v2f64_v2i32_only_use_exp
+; WINDOWS-NEXT:  // %bb.0:
+; WINDOWS-NEXT:    sub sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    str x19, [sp, #32] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    .seh_endprologue
+; WINDOWS-NEXT:    add x0, sp, #28
+; WINDOWS-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; WINDOWS-NEXT:    add x0, sp, #24
+; WINDOWS-NEXT:    add x19, sp, #24
+; WINDOWS-NEXT:    mov d0, v0.d[1]
+; WINDOWS-NEXT:    bl frexp
+; WINDOWS-NEXT:    ldr s0, [sp, #28]
+; WINDOWS-NEXT:    ld1 { v0.s }[1], [x19]
+; WINDOWS-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; WINDOWS-NEXT:    .seh_startepilogue
+; WINDOWS-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x30, 40
+; WINDOWS-NEXT:    ldr x19, [sp, #32] // 8-byte Folded Reload
+; WINDOWS-NEXT:    .seh_save_reg x19, 32
+; WINDOWS-NEXT:    add sp, sp, #48
+; WINDOWS-NEXT:    .seh_stackalloc 48
+; WINDOWS-NEXT:    .seh_endepilogue
+; WINDOWS-NEXT:    ret
+; WINDOWS-NEXT:    .seh_endfunclet
+; WINDOWS-NEXT:    .seh_endproc
+  %result = call { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double> %a)
+  %result.1 = extractvalue { <2 x double>, <2 x i32> } %result, 1
+  ret <2 x i32> %result.1
+}
+
+declare { float, i32 } @llvm.frexp.f32.i32(float) #0
+declare { <2 x float>, <2 x i32> } @llvm.frexp.v2f32.v2i32(<2 x float>) #0
+declare { <4 x float>, <4 x i32> } @llvm.frexp.v4f32.v4i32(<4 x float>) #0
+
+declare { half, i32 } @llvm.frexp.f16.i32(half) #0
+declare { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half>) #0
+
+declare { double, i32 } @llvm.frexp.f64.i32(double) #0
+declare { <2 x double>, <2 x i32> } @llvm.frexp.v2f64.v2i32(<2 x double>) #0
+
+declare { half, i16 } @llvm.frexp.f16.i16(half) #0
+declare { <2 x half>, <2 x i16> } @llvm.frexp.v2f16.v2i16(<2 x half>) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll b/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
index f394bc467af6..29427146e8a4 100644
--- a/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
+++ b/llvm/test/CodeGen/AArch64/lr-reserved-for-ra-live-in.ll
@@ -53,5 +53,5 @@ bb2:                                              ; preds = %bb1, %bb
   ret i32 %phi
 }
 
-attributes #1 = { minsize nounwind "target-features"="+aes,+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+reserve-lr-for-ra,+sha2,+v8.1a,+v8.2a,+v8.3a,+v8a,+zcm,+zcz" }
+attributes #1 = { minsize nounwind "target-features"="+aes,+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+reserve-lr-for-ra,+sha2,+v8.1a,+v8.2a,+v8.3a,+v8a" }
 
diff --git a/llvm/test/CodeGen/AArch64/machine-combiner.ll b/llvm/test/CodeGen/AArch64/machine-combiner.ll
index c8df283aace0..70a638857ce4 100644
--- a/llvm/test/CodeGen/AArch64/machine-combiner.ll
+++ b/llvm/test/CodeGen/AArch64/machine-combiner.ll
@@ -262,8 +262,8 @@ define half @reassociate_adds_half(half %x0, half %x1, half %x2, half %x3) {
 ; CHECK-UNSAFE-LABEL: reassociate_adds_half:
 ; CHECK-UNSAFE:       // %bb.0:
 ; CHECK-UNSAFE-NEXT:    fdiv h0, h0, h1
-; CHECK-UNSAFE-NEXT:    fadd h2, h3, h2
-; CHECK-UNSAFE-NEXT:    fadd h0, h2, h0
+; CHECK-UNSAFE-NEXT:    fadd h1, h3, h2
+; CHECK-UNSAFE-NEXT:    fadd h0, h1, h0
 ; CHECK-UNSAFE-NEXT:    ret
   %t0 = fdiv half %x0, %x1
   %t1 = fadd half %x2, %t0
@@ -284,8 +284,8 @@ define half @reassociate_muls_half(half %x0, half %x1, half %x2, half %x3) {
 ; CHECK-UNSAFE-LABEL: reassociate_muls_half:
 ; CHECK-UNSAFE:       // %bb.0:
 ; CHECK-UNSAFE-NEXT:    fdiv h0, h0, h1
-; CHECK-UNSAFE-NEXT:    fmul h2, h3, h2
-; CHECK-UNSAFE-NEXT:    fmul h0, h2, h0
+; CHECK-UNSAFE-NEXT:    fmul h1, h3, h2
+; CHECK-UNSAFE-NEXT:    fmul h0, h1, h0
 ; CHECK-UNSAFE-NEXT:    ret
   %t0 = fdiv half %x0, %x1
   %t1 = fmul half %x2, %t0
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
new file mode 100644
index 000000000000..196aebffce97
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-loh.ll
@@ -0,0 +1,74 @@
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,LOH
+; RUN: llc -verify-machineinstrs -mtriple=aarch64-apple-darwin -enable-machine-outliner < %s | FileCheck %s --implicit-check-not=.loh --check-prefixes=CHECK,OUTLINE
+
+@A = global i32 0, align 4
+@B = global i32 0, align 4
+
+declare void @foo();
+declare void @bar(ptr %a);
+declare void @goo(ptr %a);
+
+; CHECK-LABEL: _a0:
+define void @a0(i32 %a) {
+
+  ; This becomes AdrpAdd when outlining is disabled, otherwise it is outlined
+  ; and there should be no LOH.
+  %addr = getelementptr inbounds i32, ptr @A, i32 0
+  %res = load i32, ptr %addr, align 4
+  ; LOH:      [[L0:Lloh.+]]:
+  ; LOH-NEXT:   adrp x19, _A@PAGE
+  ; LOH-NEXT: [[L1:Lloh.+]]:
+  ; LOH-NEXT:   add x19, x19, _A@PAGEOFF
+
+  call void @foo()
+  ; OUTLINE:      bl _OUTLINED_FUNCTION_0
+  ; OUTLINE-NEXT: mov x0, x19
+  ; OUTLINE-NEXT: bl _bar
+  call void @bar(ptr %addr)
+
+  ; This becomes AdrpAddStr.
+  %addr2 = getelementptr inbounds i32, ptr @B, i32 4
+  store i32 %res, ptr %addr2, align 4
+  ; CHECK:      [[L2:Lloh.+]]:
+  ; CHECK-NEXT:   adrp x8, _B@PAGE
+  ; CHECK-NEXT: [[L3:Lloh.+]]:
+  ; CHECK-NEXT:   add x8, x8, _B@PAGEOFF
+  ; CHECK-NEXT: [[L4:Lloh.+]]:
+  ; CHECK-NEXT:   str w20, [x8, #16]
+  ret void
+
+  ; LOH-DAG:   .loh AdrpAdd [[L0]], [[L1]]
+  ; CHECK-DAG: .loh AdrpAddStr [[L2]], [[L3]], [[L4]]
+  ; CHECK:     .cfi_endproc
+}
+
+; CHECK-LABEL: _a1:
+define i32 @a1(i32 %a) {
+
+  ; This becomes AdrpAdd when outlining is disabled, otherwise it is outlined
+  ; and there should be no LOH.
+  %addr = getelementptr inbounds i32, ptr @A, i32 0
+  %res = load i32, ptr %addr, align 4
+  ; LOH:      [[L5:Lloh.+]]:
+  ; LOH-NEXT:   adrp x19, _A@PAGE
+  ; LOH-NEXT: [[L6:Lloh.+]]:
+  ; LOH-NEXT:   add x19, x19, _A@PAGEOFF
+
+  call void @foo()
+  ; OUTLINE:      bl _OUTLINED_FUNCTION_0
+  ; OUTLINE-NEXT: mov x0, x19
+  ; OUTLINE-NEXT: bl _goo
+  call void @goo(ptr %addr)
+  ret i32 %res
+
+  ; LOH:   .loh AdrpAdd [[L5]], [[L6]]
+  ; CHECK: .cfi_endproc
+}
+
+; Note: it is not safe to add LOHs to this function as outlined functions do not
+; follow calling convention and thus x19 could be live across the call.
+; OUTLINE: _OUTLINED_FUNCTION_0:
+; OUTLINE:   adrp x19, _A@PAGE
+; OUTLINE:   add x19, x19, _A@PAGEOFF
+; OUTLINE:   ldr w20, [x19]
+; OUTLINE:   b _foo
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir b/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir
new file mode 100644
index 000000000000..c613e316f828
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-loh.mir
@@ -0,0 +1,50 @@
+# RUN: llc %s -mtriple=aarch64-apple-ios -run-pass=aarch64-collect-loh -run-pass=machine-outliner -stats -o - 2>%t | FileCheck %s
+# RUN: FileCheck %s --input-file=%t --check-prefix=STATS
+# REQUIRES: asserts
+
+--- |
+  define void  @func0() noredzone minsize { ret void }
+
+  @g0 = external global i32
+...
+---
+# CHECK-LABEL: name: func0
+name: func0
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: bb.0:
+  bb.0:
+    ; CHECK: BL @OUTLINED_FUNCTION_0
+    $x10 = ADRP target-flags(aarch64-page) @g0
+    $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+
+  ; CHECK-LABEL: bb.1:
+  bb.1:
+    ; CHECK: BL @OUTLINED_FUNCTION_0
+    ; MCLOH_AdrpAdd is not generated because $x11 is still live. If we want to
+    ; outline these instructions, we must remove the MCLOH_AdrpAdd from bb.0.
+    $x10 = ADRP target-flags(aarch64-page) @g0
+    $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    $w0 = ORRWri $wzr, 1
+    ; CHECK: $x12 = ADDXri $x11, target-flags(aarch64-pageoff) @g0, 0
+    $x12 = ADDXri $x11, target-flags(aarch64-pageoff) @g0, 0
+
+...
+
+# CHECK-LABEL: name: OUTLINED_FUNCTION_0
+# CHECK:           $x10 = ADRP target-flags(aarch64-page) @g0
+# CHECK:           $x11 = ADDXri $x10, target-flags(aarch64-pageoff) @g0, 0
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+# CHECK:           $w0 = ORRWri $wzr, 1
+
+# STATS: 1 aarch64-collect-loh - Number of simplifiable ADRP + ADD
+# STATS: 1 machine-outliner    - Total number of Linker Optimization Hints removed
diff --git a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
index 2271190963d1..7dde16802427 100644
--- a/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
+++ b/llvm/test/CodeGen/AArch64/memcpy-scoped-aa.ll
@@ -134,3 +134,5 @@ declare ptr @mempcpy(ptr, ptr, i64)
 !2 = !{!1}
 !3 = distinct !{!3, !0, !"bax: %q"}
 !4 = !{!3}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MIR: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/memsize-remarks.ll b/llvm/test/CodeGen/AArch64/memsize-remarks.ll
index 93e3d6fb0260..f256389c7349 100644
--- a/llvm/test/CodeGen/AArch64/memsize-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/memsize-remarks.ll
@@ -315,8 +315,8 @@ bb:
   ret void
 }
 
-attributes #0 = { noinline nounwind ssp uwtable "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
-attributes #1 = { nounwind "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
+attributes #0 = { noinline nounwind ssp uwtable "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
+attributes #1 = { nounwind "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-a7" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
 attributes #2 = { nofree nosync nounwind readnone speculatable willreturn }
 attributes #3 = { argmemonly nofree nosync nounwind willreturn }
 attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
index a0eba7c747e9..6b2a8c5aaf8b 100644
--- a/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
+++ b/llvm/test/CodeGen/AArch64/merge-scoped-aa-store.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc %s -mtriple=aarch64 -stop-after=finalize-isel -o - | FileCheck --check-prefix=MIR %s
 
 ; Ensure the scoped AA metadata is still retained after store merging.
@@ -13,9 +14,15 @@
 ; MIR-DAG: ![[SET3:[0-9]+]] = !{![[SCOPE3]]}
 
 define void @blam0(ptr %g0, ptr %g1) {
-; MIR-LABEL: name: blam0
-; MIR: LDRDui %0, 0 :: (load (s64) from %ir.g0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]])
-; MIR: STRDui killed %{{[0-9]*}}, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[SET1]], !noalias ![[SET0]])
+  ; MIR-LABEL: name: blam0
+  ; MIR: bb.0 (%ir-block.0):
+  ; MIR-NEXT:   liveins: $x0, $x1
+  ; MIR-NEXT: {{  $}}
+  ; MIR-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; MIR-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+  ; MIR-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY1]], 0 :: (load (s64) from %ir.g0, align 4, !alias.scope !0, !noalias !3)
+  ; MIR-NEXT:   STRDui killed [[LDRDui]], [[COPY]], 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope !3, !noalias !0)
+  ; MIR-NEXT:   RET_ReallyLR
   %tmp4 = getelementptr inbounds <3 x float>, ptr %g1, i64 0, i64 0
   %tmp5 = load <3 x float>, ptr %g0, align 4, !alias.scope !0, !noalias !1
   %tmp6 = extractelement <3 x float> %tmp5, i64 0
@@ -29,13 +36,15 @@ define void @blam0(ptr %g0, ptr %g1) {
 
 ; Ensure new scoped AA metadata are calculated after merging stores.
 define void @blam1(ptr %g0, ptr %g1) {
-; MIR-LABEL: name: blam1
-; MIR: machineMetadataNodes:
-; MIR-DAG: ![[MMSET0:[0-9]+]] = !{![[SCOPE2]], ![[SCOPE1]]}
-; MIR-DAG: ![[MMSET1:[0-9]+]] = !{}
-; MIR: body:
-; MIR: LDRDui %0, 0 :: (load (s64) from %ir.g0, align 4, !alias.scope ![[SET0]], !noalias ![[SET1]])
-; MIR: STRDui killed %{{[0-9]*}}, %1, 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope ![[MMSET0]], !noalias ![[MMSET1]])
+  ; MIR-LABEL: name: blam1
+  ; MIR: bb.0 (%ir-block.0):
+  ; MIR-NEXT:   liveins: $x0, $x1
+  ; MIR-NEXT: {{  $}}
+  ; MIR-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x1
+  ; MIR-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+  ; MIR-NEXT:   [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[COPY1]], 0 :: (load (s64) from %ir.g0, align 4, !alias.scope !0, !noalias !3)
+  ; MIR-NEXT:   STRDui killed [[LDRDui]], [[COPY]], 0 :: (store (s64) into %ir.tmp41, align 4, !alias.scope !9, !noalias !10)
+  ; MIR-NEXT:   RET_ReallyLR
   %tmp4 = getelementptr inbounds <3 x float>, ptr %g1, i64 0, i64 0
   %tmp5 = load <3 x float>, ptr %g0, align 4, !alias.scope !0, !noalias !1
   %tmp6 = extractelement <3 x float> %tmp5, i64 0
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 0db4dc07fbfe..2d81a264e02b 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1255,4 +1255,3 @@ entry:
     %partial.reduce = tail call <2 x i64> @llvm.experimental.vector.partial.reduce.add(<2 x i64> %acc, <8 x i64> %input.wide)
     ret <2 x i64> %partial.reduce
 }
-
diff --git a/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll b/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll
new file mode 100644
index 000000000000..db7d26b6ed13
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no-promote-scalabale-const-to-global.ll
@@ -0,0 +1,20 @@
+; RUN: llc -mtriple=aarch64 -mattr=+sve -stop-after=aarch64-promote-const < %s | FileCheck %s
+
+; Test that the constant inside the `phi` is not promoted to a global
+; CHECK-NOT: _PromotedConst
+define void @f(i1 %c, ptr %p, ptr %q) {
+entry:
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %u = load [2 x <vscale x 4 x float> ], ptr %p
+  br label %exit
+
+if.else:
+  br label %exit
+
+exit:
+  %v = phi [2 x <vscale x 4 x float> ]  [ %u, %if.then], [[<vscale x 4 x float> zeroinitializer, <vscale x 4 x float> poison], %if.else]
+  store [2 x <vscale x 4 x float>] %v, ptr %q
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/popcount.ll b/llvm/test/CodeGen/AArch64/popcount.ll
index c158d8ad93b0..3900910c1031 100644
--- a/llvm/test/CodeGen/AArch64/popcount.ll
+++ b/llvm/test/CodeGen/AArch64/popcount.ll
@@ -23,15 +23,36 @@ define i8 @popcount128(ptr nocapture nonnull readonly %0) {
 ; CHECKO0-NEXT:    fmov w0, s0
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount128:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    ld1 { v0.d }[1], [x8]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount128:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    ldr d0, [x0]
+; NEON-NEXT:    add x8, x0, #8
+; NEON-NEXT:    ld1 { v0.d }[1], [x8]
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    fmov w0, s0
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount128:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    ldr d0, [x0]
+; DOT-NEXT:    add x8, x0, #8
+; DOT-NEXT:    ld1 { v0.d }[1], [x8]
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    fmov w0, s0
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount128:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ldr d0, [x0]
+; SVE-NEXT:    add x8, x0, #8
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    ld1 { v0.d }[1], [x8]
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov w0, s0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount128:
 ; BE:       // %bb.0: // %Entry
@@ -107,22 +128,55 @@ define i16 @popcount256(ptr nocapture nonnull readonly %0) {
 ; CHECKO0-NEXT:    mov w0, w8
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount256:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    ldr d0, [x0, #16]
-; CHECK-NEXT:    ldr d1, [x0]
-; CHECK-NEXT:    add x8, x0, #8
-; CHECK-NEXT:    add x9, x0, #24
-; CHECK-NEXT:    ld1 { v0.d }[1], [x9]
-; CHECK-NEXT:    ld1 { v1.d }[1], [x8]
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    cnt v1.16b, v1.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    addv b1, v1.16b
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    fmov w9, s1
-; CHECK-NEXT:    add w0, w9, w8
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount256:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    ldr d0, [x0, #16]
+; NEON-NEXT:    ldr d1, [x0]
+; NEON-NEXT:    add x8, x0, #8
+; NEON-NEXT:    add x9, x0, #24
+; NEON-NEXT:    ld1 { v0.d }[1], [x9]
+; NEON-NEXT:    ld1 { v1.d }[1], [x8]
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    cnt v1.16b, v1.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    addv b1, v1.16b
+; NEON-NEXT:    fmov w8, s0
+; NEON-NEXT:    fmov w9, s1
+; NEON-NEXT:    add w0, w9, w8
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount256:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    ldr d0, [x0, #16]
+; DOT-NEXT:    ldr d1, [x0]
+; DOT-NEXT:    add x8, x0, #8
+; DOT-NEXT:    add x9, x0, #24
+; DOT-NEXT:    ld1 { v0.d }[1], [x9]
+; DOT-NEXT:    ld1 { v1.d }[1], [x8]
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    cnt v1.16b, v1.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    addv b1, v1.16b
+; DOT-NEXT:    fmov w8, s0
+; DOT-NEXT:    fmov w9, s1
+; DOT-NEXT:    add w0, w9, w8
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount256:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ldr d0, [x0, #16]
+; SVE-NEXT:    ldr d1, [x0]
+; SVE-NEXT:    add x8, x0, #8
+; SVE-NEXT:    add x9, x0, #24
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    ld1 { v0.d }[1], [x9]
+; SVE-NEXT:    ld1 { v1.d }[1], [x8]
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    cnt z1.d, p0/m, z1.d
+; SVE-NEXT:    add v0.2d, v1.2d, v0.2d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov w0, s0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount256:
 ; BE:       // %bb.0: // %Entry
@@ -223,15 +277,36 @@ define <1 x i128> @popcount1x128(<1 x i128> %0) {
 ; CHECKO0-NEXT:    bfi x0, x8, #32, #32
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount1x128:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    fmov d0, x0
-; CHECK-NEXT:    mov v0.d[1], x1
-; CHECK-NEXT:    mov x1, xzr
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    addv b0, v0.16b
-; CHECK-NEXT:    fmov x0, d0
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount1x128:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    fmov d0, x0
+; NEON-NEXT:    mov v0.d[1], x1
+; NEON-NEXT:    mov x1, xzr
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    addv b0, v0.16b
+; NEON-NEXT:    fmov x0, d0
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount1x128:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    fmov d0, x0
+; DOT-NEXT:    mov v0.d[1], x1
+; DOT-NEXT:    mov x1, xzr
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    addv b0, v0.16b
+; DOT-NEXT:    fmov x0, d0
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount1x128:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    fmov d0, x0
+; SVE-NEXT:    ptrue p0.d
+; SVE-NEXT:    mov v0.d[1], x1
+; SVE-NEXT:    mov x1, xzr
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    addp d0, v0.2d
+; SVE-NEXT:    fmov x0, d0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount1x128:
 ; BE:       // %bb.0: // %Entry
@@ -305,10 +380,10 @@ define <2 x i64> @popcount2x64(<2 x i64> %0) {
 ;
 ; SVE-LABEL: popcount2x64:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.16b, v0.16b
-; SVE-NEXT:    uaddlp v0.8h, v0.16b
-; SVE-NEXT:    uaddlp v0.4s, v0.8h
-; SVE-NEXT:    uaddlp v0.2d, v0.4s
+; SVE-NEXT:    ptrue p0.d, vl2
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount2x64:
@@ -374,13 +449,29 @@ define <1 x i64> @popcount1x64(<1 x i64> %0) {
 ; CHECKO0-NEXT:    fmov d0, x8
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount1x64:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount1x64:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.8b, v0.8b
+; NEON-NEXT:    uaddlp v0.4h, v0.8b
+; NEON-NEXT:    uaddlp v0.2s, v0.4h
+; NEON-NEXT:    uaddlp v0.1d, v0.2s
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount1x64:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.8b, v0.8b
+; DOT-NEXT:    uaddlp v0.4h, v0.8b
+; DOT-NEXT:    uaddlp v0.2s, v0.4h
+; DOT-NEXT:    uaddlp v0.1d, v0.2s
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount1x64:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.d, vl1
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.d, p0/m, z0.d
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount1x64:
 ; BE:       // %bb.0: // %Entry
@@ -442,9 +533,10 @@ define <4 x i32> @popcount4x32(<4 x i32> %0) {
 ;
 ; SVE-LABEL: popcount4x32:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.16b, v0.16b
-; SVE-NEXT:    uaddlp v0.8h, v0.16b
-; SVE-NEXT:    uaddlp v0.4s, v0.8h
+; SVE-NEXT:    ptrue p0.s, vl4
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.s, p0/m, z0.s
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount4x32:
@@ -520,9 +612,10 @@ define <2 x i32> @popcount2x32(<2 x i32> %0) {
 ;
 ; SVE-LABEL: popcount2x32:
 ; SVE:       // %bb.0: // %Entry
-; SVE-NEXT:    cnt v0.8b, v0.8b
-; SVE-NEXT:    uaddlp v0.4h, v0.8b
-; SVE-NEXT:    uaddlp v0.2s, v0.4h
+; SVE-NEXT:    ptrue p0.s, vl2
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.s, p0/m, z0.s
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount2x32:
@@ -577,11 +670,25 @@ define <8 x i16> @popcount8x16(<8 x i16> %0) {
 ; CHECKO0-NEXT:    uaddlp v0.8h, v0.16b
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount8x16:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount8x16:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.16b, v0.16b
+; NEON-NEXT:    uaddlp v0.8h, v0.16b
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount8x16:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.16b, v0.16b
+; DOT-NEXT:    uaddlp v0.8h, v0.16b
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount8x16:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.h, vl8
+; SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT:    cnt z0.h, p0/m, z0.h
+; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount8x16:
 ; BE:       // %bb.0: // %Entry
@@ -618,11 +725,25 @@ define <4 x i16> @popcount4x16(<4 x i16> %0) {
 ; CHECKO0-NEXT:    uaddlp v0.4h, v0.8b
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: popcount4x16:
-; CHECK:       // %bb.0: // %Entry
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    ret
+; NEON-LABEL: popcount4x16:
+; NEON:       // %bb.0: // %Entry
+; NEON-NEXT:    cnt v0.8b, v0.8b
+; NEON-NEXT:    uaddlp v0.4h, v0.8b
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: popcount4x16:
+; DOT:       // %bb.0: // %Entry
+; DOT-NEXT:    cnt v0.8b, v0.8b
+; DOT-NEXT:    uaddlp v0.4h, v0.8b
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: popcount4x16:
+; SVE:       // %bb.0: // %Entry
+; SVE-NEXT:    ptrue p0.h, vl4
+; SVE-NEXT:    // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT:    cnt z0.h, p0/m, z0.h
+; SVE-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: popcount4x16:
 ; BE:       // %bb.0: // %Entry
@@ -673,20 +794,49 @@ define i32 @ctpop_into_extract(ptr %p) {
 ; CHECKO0-NEXT:    mov w0, wzr
 ; CHECKO0-NEXT:    ret
 ;
-; CHECK-LABEL: ctpop_into_extract:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    movi v2.2d, #0xffffffffffffffff
-; CHECK-NEXT:    mov x8, x0
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    cnt v1.8b, v1.8b
-; CHECK-NEXT:    addv b1, v1.8b
-; CHECK-NEXT:    mov v2.s[1], v1.s[0]
-; CHECK-NEXT:    sub v0.2s, v0.2s, v2.2s
-; CHECK-NEXT:    str d0, [x8]
-; CHECK-NEXT:    ret
+; NEON-LABEL: ctpop_into_extract:
+; NEON:       // %bb.0:
+; NEON-NEXT:    ldr d0, [x0]
+; NEON-NEXT:    movi v2.2d, #0xffffffffffffffff
+; NEON-NEXT:    mov x8, x0
+; NEON-NEXT:    mov w0, wzr
+; NEON-NEXT:    fmov w9, s0
+; NEON-NEXT:    fmov s1, w9
+; NEON-NEXT:    cnt v1.8b, v1.8b
+; NEON-NEXT:    addv b1, v1.8b
+; NEON-NEXT:    mov v2.s[1], v1.s[0]
+; NEON-NEXT:    sub v0.2s, v0.2s, v2.2s
+; NEON-NEXT:    str d0, [x8]
+; NEON-NEXT:    ret
+;
+; DOT-LABEL: ctpop_into_extract:
+; DOT:       // %bb.0:
+; DOT-NEXT:    ldr d0, [x0]
+; DOT-NEXT:    movi v2.2d, #0xffffffffffffffff
+; DOT-NEXT:    mov x8, x0
+; DOT-NEXT:    mov w0, wzr
+; DOT-NEXT:    fmov w9, s0
+; DOT-NEXT:    fmov s1, w9
+; DOT-NEXT:    cnt v1.8b, v1.8b
+; DOT-NEXT:    addv b1, v1.8b
+; DOT-NEXT:    mov v2.s[1], v1.s[0]
+; DOT-NEXT:    sub v0.2s, v0.2s, v2.2s
+; DOT-NEXT:    str d0, [x8]
+; DOT-NEXT:    ret
+;
+; SVE-LABEL: ctpop_into_extract:
+; SVE:       // %bb.0:
+; SVE-NEXT:    ldr d0, [x0]
+; SVE-NEXT:    ptrue p0.s
+; SVE-NEXT:    movi v2.2d, #0xffffffffffffffff
+; SVE-NEXT:    mov x8, x0
+; SVE-NEXT:    mov w0, wzr
+; SVE-NEXT:    movprfx z1, z0
+; SVE-NEXT:    cnt z1.s, p0/m, z0.s
+; SVE-NEXT:    mov v2.s[1], v1.s[0]
+; SVE-NEXT:    sub v0.2s, v0.2s, v2.2s
+; SVE-NEXT:    str d0, [x8]
+; SVE-NEXT:    ret
 ;
 ; BE-LABEL: ctpop_into_extract:
 ; BE:       // %bb.0:
@@ -752,3 +902,5 @@ define i32 @ctpop_into_extract(ptr %p) {
 }
 
 declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/pr58431.ll b/llvm/test/CodeGen/AArch64/pr58431.ll
index 88bab4af95d6..467ceb062f24 100644
--- a/llvm/test/CodeGen/AArch64/pr58431.ll
+++ b/llvm/test/CodeGen/AArch64/pr58431.ll
@@ -4,10 +4,12 @@
 define i32 @f(i64 %0) {
 ; CHECK-LABEL: f:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #10 // =0xa
+; CHECK-NEXT:    mov x8, #-7378697629483820647 // =0x9999999999999999
 ; CHECK-NEXT:    mov w9, w0
-; CHECK-NEXT:    udiv x10, x9, x8
-; CHECK-NEXT:    msub x0, x10, x8, x9
+; CHECK-NEXT:    mov w10, #10 // =0xa
+; CHECK-NEXT:    eor x8, x8, #0x8000000000000003
+; CHECK-NEXT:    umulh x8, x9, x8
+; CHECK-NEXT:    msub x0, x8, x10, x9
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; CHECK-NEXT:    ret
   %2 = trunc i64 %0 to i32
diff --git a/llvm/test/CodeGen/AArch64/ptradd.ll b/llvm/test/CodeGen/AArch64/ptradd.ll
index 28a8f4303765..fc364357436e 100644
--- a/llvm/test/CodeGen/AArch64/ptradd.ll
+++ b/llvm/test/CodeGen/AArch64/ptradd.ll
@@ -285,19 +285,11 @@ entry:
 }
 
 define <1 x ptr> @vector_gep_v1i64_c10(ptr %b) {
-; CHECK-SD-LABEL: vector_gep_v1i64_c10:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    fmov d1, x8
-; CHECK-SD-NEXT:    add d0, d0, d1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_gep_v1i64_c10:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    add x8, x0, #10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_gep_v1i64_c10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    add x8, x0, #10
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
 entry:
   %g = getelementptr i8, ptr %b, <1 x i64> <i64 10>
   ret <1 x ptr> %g
@@ -306,10 +298,8 @@ entry:
 define <2 x ptr> @vector_gep_v2i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v2i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    dup v1.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    dup v0.2d, x8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v2i64_c10:
@@ -327,15 +317,10 @@ entry:
 define <3 x ptr> @vector_gep_v3i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v3i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    fmov d3, x0
-; CHECK-SD-NEXT:    dup v2.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
-; CHECK-SD-NEXT:    add d2, d3, d2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v3i64_c10:
@@ -356,10 +341,8 @@ entry:
 define <4 x ptr> @vector_gep_v4i64_c10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v4i64_c10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov w8, #10 // =0xa
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    dup v1.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    add x8, x0, #10
+; CHECK-SD-NEXT:    dup v0.2d, x8
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
@@ -377,19 +360,11 @@ entry:
 }
 
 define <1 x ptr> @vector_gep_v1i64_cm10(ptr %b) {
-; CHECK-SD-LABEL: vector_gep_v1i64_cm10:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    fmov d1, x0
-; CHECK-SD-NEXT:    fmov d0, x8
-; CHECK-SD-NEXT:    add d0, d1, d0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: vector_gep_v1i64_cm10:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub x8, x0, #10
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: vector_gep_v1i64_cm10:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub x8, x0, #10
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
 entry:
   %g = getelementptr i8, ptr %b, <1 x i64> <i64 -10>
   ret <1 x ptr> %g
@@ -398,10 +373,8 @@ entry:
 define <2 x ptr> @vector_gep_v2i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v2i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v1.2d, x0
+; CHECK-SD-NEXT:    sub x8, x0, #10
 ; CHECK-SD-NEXT:    dup v0.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v2i64_cm10:
@@ -419,15 +392,10 @@ entry:
 define <3 x ptr> @vector_gep_v3i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v3i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v0.2d, x0
-; CHECK-SD-NEXT:    fmov d3, x0
-; CHECK-SD-NEXT:    dup v2.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v0.2d, v2.2d
-; CHECK-SD-NEXT:    add d2, d3, d2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    sub x8, x0, #10
+; CHECK-SD-NEXT:    fmov d0, x8
+; CHECK-SD-NEXT:    fmov d1, d0
+; CHECK-SD-NEXT:    fmov d2, d0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vector_gep_v3i64_cm10:
@@ -448,10 +416,8 @@ entry:
 define <4 x ptr> @vector_gep_v4i64_cm10(ptr %b) {
 ; CHECK-SD-LABEL: vector_gep_v4i64_cm10:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, #-10 // =0xfffffffffffffff6
-; CHECK-SD-NEXT:    dup v1.2d, x0
+; CHECK-SD-NEXT:    sub x8, x0, #10
 ; CHECK-SD-NEXT:    dup v0.2d, x8
-; CHECK-SD-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-SD-NEXT:    mov v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll b/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll
index 89731e62dcc1..2d8085cf489b 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll
@@ -2,7 +2,7 @@
 ; RUN:   -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \
 ; RUN:   -filetype=obj < %s | llvm-readelf -r -s - | FileCheck --check-prefix=CHECK-OBJ %s
-; RUN: not --crash llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \
+; RUN: not llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \
 ; RUN:   -global-isel=1 < %s 2>&1 | FileCheck --check-prefix=CHECK-ERR %s
 
 @general_dynamic_var = external thread_local global i32
diff --git a/llvm/test/CodeGen/AArch64/rem-by-const.ll b/llvm/test/CodeGen/AArch64/rem-by-const.ll
new file mode 100644
index 000000000000..1376f5d9a380
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/rem-by-const.ll
@@ -0,0 +1,3616 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel=0 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define i8 @si8_7(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: si8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxtb w8, w0
+; CHECK-SD-NEXT:    mov w9, #-109 // =0xffffff93
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    add w8, w0, w8, lsr #8
+; CHECK-SD-NEXT:    sbfx w9, w8, #2, #6
+; CHECK-SD-NEXT:    and w8, w8, #0x80
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #7
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i8 %a, 7
+  ret i8 %s
+}
+
+define i8 @si8_100(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: si8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxtb w8, w0
+; CHECK-SD-NEXT:    mov w9, #41 // =0x29
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #12
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i8 %a, 100
+  ret i8 %s
+}
+
+define i8 @ui8_7(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: ui8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #37 // =0x25
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    lsr w8, w8, #8
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    and w9, w9, #0xfe
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #37 // =0x25
+; CHECK-GI-NEXT:    and w9, w0, #0xff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    lsr w8, w8, #8
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    ubfx w9, w9, #1, #7
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    ubfx w8, w8, #2, #6
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i8 %a, 7
+  ret i8 %s
+}
+
+define i8 @ui8_100(i8 %a, i8 %b) {
+; CHECK-SD-LABEL: ui8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #41 // =0x29
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    lsr w8, w8, #12
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #41 // =0x29
+; CHECK-GI-NEXT:    and w9, w0, #0xff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    lsr w8, w8, #8
+; CHECK-GI-NEXT:    lsr w8, w8, #4
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i8 %a, 100
+  ret i8 %s
+}
+
+define i16 @si16_7(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: si16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxth w8, w0
+; CHECK-SD-NEXT:    mov w9, #18725 // =0x4925
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #17
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i16 %a, 7
+  ret i16 %s
+}
+
+define i16 @si16_100(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: si16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sxth w8, w0
+; CHECK-SD-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    asr w9, w8, #19
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxth w8, w0
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w8, w8, w9
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i16 %a, 100
+  ret i16 %s
+}
+
+define i16 @ui16_7(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: ui16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    and w9, w0, #0xffff
+; CHECK-SD-NEXT:    mul w8, w9, w8
+; CHECK-SD-NEXT:    lsr w8, w8, #16
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    and w9, w9, #0xfffe
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    and w9, w0, #0xffff
+; CHECK-GI-NEXT:    mul w8, w9, w8
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    ubfx w9, w9, #1, #15
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    ubfx w8, w8, #2, #14
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i16 %a, 7
+  ret i16 %s
+}
+
+define i16 @ui16_100(i16 %a, i16 %b) {
+; CHECK-SD-LABEL: ui16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ubfx w8, w0, #2, #14
+; CHECK-SD-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-SD-NEXT:    mul w8, w8, w9
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    lsr w8, w8, #17
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ubfx w8, w0, #2, #14
+; CHECK-GI-NEXT:    mov w9, #5243 // =0x147b
+; CHECK-GI-NEXT:    mul w8, w8, w9
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    lsr w8, w8, #1
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i16 %a, 100
+  ret i16 %s
+}
+
+define i32 @si32_7(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: si32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    smull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w8, w8, w0
+; CHECK-SD-NEXT:    asr w9, w8, #2
+; CHECK-SD-NEXT:    add w8, w9, w8, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w8, w0, w8
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i32 %a, 7
+  ret i32 %s
+}
+
+define i32 @si32_100(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: si32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    smull x8, w0, w8
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w9, w0, w8
+; CHECK-GI-NEXT:    msub w0, w9, w8, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i32 %a, 100
+  ret i32 %s
+}
+
+define i32 @ui32_7(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: ui32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w9, w0, w8
+; CHECK-SD-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w0, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-GI-NEXT:    movk w8, #9362, lsl #16
+; CHECK-GI-NEXT:    umull x8, w0, w8
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    sub w9, w0, w8
+; CHECK-GI-NEXT:    add w8, w8, w9, lsr #1
+; CHECK-GI-NEXT:    lsr w8, w8, #2
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w0, w0, w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i32 %a, 7
+  ret i32 %s
+}
+
+define i32 @ui32_100(i32 %a, i32 %b) {
+; CHECK-SD-LABEL: ui32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    umull x8, w0, w8
+; CHECK-SD-NEXT:    lsr x8, x8, #37
+; CHECK-SD-NEXT:    msub w0, w8, w9, w0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    movk w8, #20971, lsl #16
+; CHECK-GI-NEXT:    umull x8, w0, w8
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    lsr w8, w8, #5
+; CHECK-GI-NEXT:    msub w0, w8, w9, w0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i32 %a, 100
+  ret i32 %s
+}
+
+define i64 @si64_7(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: si64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #16
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #32
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #48
+; CHECK-SD-NEXT:    smulh x8, x0, x8
+; CHECK-SD-NEXT:    asr x9, x8, #1
+; CHECK-SD-NEXT:    add x8, x9, x8, lsr #63
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    add x0, x0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv x8, x0, x8
+; CHECK-GI-NEXT:    lsl x9, x8, #3
+; CHECK-GI-NEXT:    sub x8, x9, x8
+; CHECK-GI-NEXT:    sub x0, x0, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i64 %a, 7
+  ret i64 %s
+}
+
+define i64 @si64_100(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: si64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #55051 // =0xd70b
+; CHECK-SD-NEXT:    movk x8, #28835, lsl #16
+; CHECK-SD-NEXT:    movk x8, #2621, lsl #32
+; CHECK-SD-NEXT:    movk x8, #41943, lsl #48
+; CHECK-SD-NEXT:    smulh x8, x0, x8
+; CHECK-SD-NEXT:    add x8, x8, x0
+; CHECK-SD-NEXT:    asr x9, x8, #6
+; CHECK-SD-NEXT:    add x8, x9, x8, lsr #63
+; CHECK-SD-NEXT:    mov w9, #100 // =0x64
+; CHECK-SD-NEXT:    msub x0, x8, x9, x0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: si64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv x9, x0, x8
+; CHECK-GI-NEXT:    msub x0, x9, x8, x0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem i64 %a, 100
+  ret i64 %s
+}
+
+define i64 @ui64_7(i64 %a, i64 %b) {
+; CHECK-SD-LABEL: ui64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #32
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #48
+; CHECK-SD-NEXT:    umulh x8, x0, x8
+; CHECK-SD-NEXT:    sub x9, x0, x8
+; CHECK-SD-NEXT:    add x8, x8, x9, lsr #1
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    add x0, x0, x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    umulh x8, x0, x8
+; CHECK-GI-NEXT:    sub x9, x0, x8
+; CHECK-GI-NEXT:    add x8, x8, x9, lsr #1
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    lsl x9, x8, #3
+; CHECK-GI-NEXT:    sub x8, x9, x8
+; CHECK-GI-NEXT:    sub x0, x0, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i64 %a, 7
+  ret i64 %s
+}
+
+define i64 @ui64_100(i64 %a, i64 %b) {
+; CHECK-LABEL: ui64_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x9, #62915 // =0xf5c3
+; CHECK-NEXT:    lsr x8, x0, #2
+; CHECK-NEXT:    movk x9, #23592, lsl #16
+; CHECK-NEXT:    movk x9, #49807, lsl #32
+; CHECK-NEXT:    movk x9, #10485, lsl #48
+; CHECK-NEXT:    umulh x8, x8, x9
+; CHECK-NEXT:    mov w9, #100 // =0x64
+; CHECK-NEXT:    lsr x8, x8, #2
+; CHECK-NEXT:    msub x0, x8, x9, x0
+; CHECK-NEXT:    ret
+entry:
+  %s = urem i64 %a, 100
+  ret i64 %s
+}
+
+define i128 @si128_7(i128 %a, i128 %b) {
+; CHECK-LABEL: si128_7:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #7 // =0x7
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    bl __modti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %s = srem i128 %a, 7
+  ret i128 %s
+}
+
+define i128 @si128_100(i128 %a, i128 %b) {
+; CHECK-LABEL: si128_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov w2, #100 // =0x64
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    bl __modti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %s = srem i128 %a, 100
+  ret i128 %s
+}
+
+define i128 @ui128_7(i128 %a, i128 %b) {
+; CHECK-SD-LABEL: ui128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-GI-NEXT:    mov x10, #9362 // =0x2492
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #16
+; CHECK-GI-NEXT:    movk x10, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #32
+; CHECK-GI-NEXT:    movk x10, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #48
+; CHECK-GI-NEXT:    movk x10, #9362, lsl #48
+; CHECK-GI-NEXT:    mul x9, x1, x8
+; CHECK-GI-NEXT:    mul x11, x0, x10
+; CHECK-GI-NEXT:    umulh x12, x0, x8
+; CHECK-GI-NEXT:    mul x13, x1, x10
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    umulh x14, x1, x8
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    cmn x9, x12
+; CHECK-GI-NEXT:    and x9, x11, #0x1
+; CHECK-GI-NEXT:    sub x12, x0, x0
+; CHECK-GI-NEXT:    umulh x15, x0, x10
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    and x11, x11, #0x1
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    and x13, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x8, xzr, x8
+; CHECK-GI-NEXT:    add x9, x9, x11
+; CHECK-GI-NEXT:    and x11, xzr, #0x1
+; CHECK-GI-NEXT:    adds x12, x12, x14
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    umulh x10, x1, x10
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    adds x12, x12, x15
+; CHECK-GI-NEXT:    and x13, x13, #0x1
+; CHECK-GI-NEXT:    umulh x14, x0, xzr
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    adds x9, x12, x9
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    and x12, x15, #0x1
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, x13, #0x1
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    add x10, x11, x12
+; CHECK-GI-NEXT:    add x8, x8, x14
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    subs x10, x0, x9
+; CHECK-GI-NEXT:    sbc x11, x1, x8
+; CHECK-GI-NEXT:    lsl x12, x11, #63
+; CHECK-GI-NEXT:    lsr x11, x11, #1
+; CHECK-GI-NEXT:    orr x10, x12, x10, lsr #1
+; CHECK-GI-NEXT:    adds x9, x10, x9
+; CHECK-GI-NEXT:    adc x8, x11, x8
+; CHECK-GI-NEXT:    lsl x10, x8, #62
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    orr x9, x10, x9, lsr #2
+; CHECK-GI-NEXT:    mov w10, #7 // =0x7
+; CHECK-GI-NEXT:    lsl x12, x8, #3
+; CHECK-GI-NEXT:    umulh x10, x9, x10
+; CHECK-GI-NEXT:    lsl x11, x9, #3
+; CHECK-GI-NEXT:    sub x8, x12, x8
+; CHECK-GI-NEXT:    sub x9, x11, x9
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    sbc x1, x1, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i128 %a, 7
+  ret i128 %s
+}
+
+define i128 @ui128_100(i128 %a, i128 %b) {
+; CHECK-SD-LABEL: ui128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: ui128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #23593 // =0x5c29
+; CHECK-GI-NEXT:    mov x10, #62914 // =0xf5c2
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #16
+; CHECK-GI-NEXT:    movk x10, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #32
+; CHECK-GI-NEXT:    movk x10, #49807, lsl #32
+; CHECK-GI-NEXT:    movk x8, #36700, lsl #48
+; CHECK-GI-NEXT:    movk x10, #10485, lsl #48
+; CHECK-GI-NEXT:    mul x9, x1, x8
+; CHECK-GI-NEXT:    mul x11, x0, x10
+; CHECK-GI-NEXT:    umulh x12, x0, x8
+; CHECK-GI-NEXT:    mul x13, x1, x10
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    umulh x14, x1, x8
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    cmn x9, x12
+; CHECK-GI-NEXT:    and x9, x11, #0x1
+; CHECK-GI-NEXT:    sub x12, x0, x0
+; CHECK-GI-NEXT:    umulh x15, x0, x10
+; CHECK-GI-NEXT:    cset w11, hs
+; CHECK-GI-NEXT:    and x11, x11, #0x1
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    and x13, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x8, xzr, x8
+; CHECK-GI-NEXT:    add x9, x9, x11
+; CHECK-GI-NEXT:    and x11, xzr, #0x1
+; CHECK-GI-NEXT:    adds x12, x12, x14
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    umulh x10, x1, x10
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    adds x12, x12, x15
+; CHECK-GI-NEXT:    and x13, x13, #0x1
+; CHECK-GI-NEXT:    umulh x14, x0, xzr
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    adds x9, x12, x9
+; CHECK-GI-NEXT:    add x11, x11, x13
+; CHECK-GI-NEXT:    and x12, x15, #0x1
+; CHECK-GI-NEXT:    cset w13, hs
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, x13, #0x1
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    add x10, x11, x12
+; CHECK-GI-NEXT:    add x8, x8, x14
+; CHECK-GI-NEXT:    add x8, x8, x10
+; CHECK-GI-NEXT:    lsl x10, x8, #60
+; CHECK-GI-NEXT:    lsr x8, x8, #4
+; CHECK-GI-NEXT:    orr x9, x10, x9, lsr #4
+; CHECK-GI-NEXT:    mov w10, #100 // =0x64
+; CHECK-GI-NEXT:    umulh x11, x9, x10
+; CHECK-GI-NEXT:    mul x9, x9, x10
+; CHECK-GI-NEXT:    madd x8, x8, x10, x11
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    sbc x1, x1, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem i128 %a, 100
+  ret i128 %s
+}
+
+define <2 x i8> @sv2i8_7(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: sv2i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #24
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v1.2s, #24
+; CHECK-SD-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #32
+; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #24
+; CHECK-SD-NEXT:    sshr v1.2s, v2.2s, #2
+; CHECK-SD-NEXT:    usra v1.2s, v2.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    smov w11, v1.h[1]
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    smov w10, v1.h[0]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i8> %d, <i8 7, i8 7>
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @sv2i8_100(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: sv2i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    smov w11, v1.h[1]
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    smov w10, v1.h[0]
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i8> %d, <i8 100, i8 100>
+  ret <2 x i8> %s
+}
+
+define <3 x i8> @sv3i8_7(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: sv3i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxtb x8, w0
+; CHECK-SD-NEXT:    mov x9, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    sxtb x10, w1
+; CHECK-SD-NEXT:    sxtb x11, w2
+; CHECK-SD-NEXT:    movk x9, #37449, lsl #16
+; CHECK-SD-NEXT:    sxtb w12, w1
+; CHECK-SD-NEXT:    smull x8, w8, w9
+; CHECK-SD-NEXT:    sxtb w13, w0
+; CHECK-SD-NEXT:    smull x10, w10, w9
+; CHECK-SD-NEXT:    smull x9, w11, w9
+; CHECK-SD-NEXT:    sxtb w11, w2
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    add w8, w8, w13
+; CHECK-SD-NEXT:    add w10, w10, w12
+; CHECK-SD-NEXT:    asr w14, w8, #2
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    asr w15, w10, #2
+; CHECK-SD-NEXT:    asr w16, w9, #2
+; CHECK-SD-NEXT:    add w8, w14, w8, lsr #31
+; CHECK-SD-NEXT:    add w10, w15, w10, lsr #31
+; CHECK-SD-NEXT:    add w9, w16, w9, lsr #31
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    add w0, w13, w8
+; CHECK-SD-NEXT:    add w1, w12, w10
+; CHECK-SD-NEXT:    add w2, w11, w9
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w11, w1
+; CHECK-GI-NEXT:    sxtb w13, w2
+; CHECK-GI-NEXT:    mov w9, #7 // =0x7
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    sdiv w12, w11, w9
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w0, w8, w10
+; CHECK-GI-NEXT:    sdiv w9, w13, w9
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w12, w15, w12
+; CHECK-GI-NEXT:    sub w1, w11, w12
+; CHECK-GI-NEXT:    lsl w16, w9, #3
+; CHECK-GI-NEXT:    sub w9, w16, w9
+; CHECK-GI-NEXT:    sub w2, w13, w9
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i8> %d, <i8 7, i8 7, i8 7>
+  ret <3 x i8> %s
+}
+
+define <3 x i8> @sv3i8_100(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: sv3i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-SD-NEXT:    sxtb x8, w0
+; CHECK-SD-NEXT:    mov w9, #34079 // =0x851f
+; CHECK-SD-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-SD-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-SD-NEXT:    sxtb x10, w1
+; CHECK-SD-NEXT:    movk w9, #20971, lsl #16
+; CHECK-SD-NEXT:    sxtb x11, w2
+; CHECK-SD-NEXT:    sxtb w12, w0
+; CHECK-SD-NEXT:    smull x8, w8, w9
+; CHECK-SD-NEXT:    smull x10, w10, w9
+; CHECK-SD-NEXT:    smull x9, w11, w9
+; CHECK-SD-NEXT:    mov w11, #100 // =0x64
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    msub w0, w8, w11, w12
+; CHECK-SD-NEXT:    sxtb w8, w1
+; CHECK-SD-NEXT:    msub w1, w10, w11, w8
+; CHECK-SD-NEXT:    sxtb w8, w2
+; CHECK-SD-NEXT:    msub w2, w9, w11, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sxtb w8, w0
+; CHECK-GI-NEXT:    sxtb w11, w1
+; CHECK-GI-NEXT:    sxtb w13, w2
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    sdiv w10, w8, w9
+; CHECK-GI-NEXT:    sdiv w12, w11, w9
+; CHECK-GI-NEXT:    msub w0, w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w14, w13, w9
+; CHECK-GI-NEXT:    msub w1, w12, w9, w11
+; CHECK-GI-NEXT:    msub w2, w14, w9, w13
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i8> %d, <i8 100, i8 100, i8 100>
+  ret <3 x i8> %s
+}
+
+define <4 x i8> @sv4i8_7(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: sv4i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    mov x8, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    smov w11, v0.h[0]
+; CHECK-SD-NEXT:    smov x12, v0.h[2]
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smov x14, v0.h[3]
+; CHECK-SD-NEXT:    smov w16, v0.h[2]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x12, w12, w8
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    smull x8, w14, w8
+; CHECK-SD-NEXT:    smov w14, v0.h[3]
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    lsr x12, x12, #32
+; CHECK-SD-NEXT:    asr w15, w9, #2
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    asr w17, w10, #2
+; CHECK-SD-NEXT:    add w12, w12, w16
+; CHECK-SD-NEXT:    add w9, w15, w9, lsr #31
+; CHECK-SD-NEXT:    asr w15, w12, #2
+; CHECK-SD-NEXT:    add w8, w8, w14
+; CHECK-SD-NEXT:    add w10, w17, w10, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    add w9, w11, w9
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w13, w10
+; CHECK-SD-NEXT:    add w9, w15, w12, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    asr w10, w8, #2
+; CHECK-SD-NEXT:    add w9, w16, w9
+; CHECK-SD-NEXT:    add w8, w10, w8, lsr #31
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w8, w14, w8
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v3.4h, #7
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    mov v3.d[1], v2.d[0]
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w9, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w9
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i8> %d, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %s
+}
+
+define <4 x i8> @sv4i8_100(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: sv4i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    sshr v1.4h, v0.4h, #8
+; CHECK-SD-NEXT:    smov x9, v1.h[0]
+; CHECK-SD-NEXT:    smov x10, v1.h[1]
+; CHECK-SD-NEXT:    smov x11, v1.h[2]
+; CHECK-SD-NEXT:    smov w12, v1.h[0]
+; CHECK-SD-NEXT:    smov x13, v1.h[3]
+; CHECK-SD-NEXT:    smov w15, v1.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x11, w11, w8
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    smull x8, w13, w8
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    asr x11, x11, #37
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    msub w9, w9, w14, w12
+; CHECK-SD-NEXT:    msub w10, w10, w14, w15
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w9, w11, w11, lsr #31
+; CHECK-SD-NEXT:    smov w11, v1.h[2]
+; CHECK-SD-NEXT:    msub w9, w9, w14, w11
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    smov w10, v1.h[3]
+; CHECK-SD-NEXT:    msub w8, w8, w14, w10
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v3.4h, #100
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    mov v3.d[1], v2.d[0]
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w9, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w9
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i8> %d, <i8 100, i8 100, i8 100, i8 100>
+  ret <4 x i8> %s
+}
+
+define <8 x i8> @sv8i8_7(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: sv8i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #147
+; CHECK-SD-NEXT:    movi v2.8b, #7
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    add v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    sshr v1.8b, v1.8b, #2
+; CHECK-SD-NEXT:    usra v1.8b, v1.8b, #7
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v4.8b, #7
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v5.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <8 x i8> %s
+}
+
+define <8 x i8> @sv8i8_100(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: sv8i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #41
+; CHECK-SD-NEXT:    movi v2.8b, #100
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    sshr v1.8b, v1.8b, #4
+; CHECK-SD-NEXT:    usra v1.8b, v1.8b, #7
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v4.8b, #100
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    sshll v4.8h, v4.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v5.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v5.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @sv16i8_7(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-SD-LABEL: sv16i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #147
+; CHECK-SD-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    movi v2.16b, #7
+; CHECK-SD-NEXT:    add v1.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    sshr v1.16b, v1.16b, #2
+; CHECK-SD-NEXT:    usra v1.16b, v1.16b, #7
+; CHECK-SD-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv16i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v16.8b, #7
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s2
+; CHECK-GI-NEXT:    fmov w17, s0
+; CHECK-GI-NEXT:    fmov w2, s3
+; CHECK-GI-NEXT:    mov w14, v2.s[1]
+; CHECK-GI-NEXT:    mov w18, v0.s[1]
+; CHECK-GI-NEXT:    mov w3, v3.s[1]
+; CHECK-GI-NEXT:    mov w15, v2.s[2]
+; CHECK-GI-NEXT:    mov w0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w11, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov w4, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v2.s[3]
+; CHECK-GI-NEXT:    mov w1, v0.s[3]
+; CHECK-GI-NEXT:    mov w5, v3.s[3]
+; CHECK-GI-NEXT:    sshll v17.4s, v16.4h, #0
+; CHECK-GI-NEXT:    sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s4, w11
+; CHECK-GI-NEXT:    sdiv w17, w17, w8
+; CHECK-GI-NEXT:    fmov s5, w13
+; CHECK-GI-NEXT:    sdiv w2, w2, w8
+; CHECK-GI-NEXT:    fmov s6, w17
+; CHECK-GI-NEXT:    sdiv w12, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[2]
+; CHECK-GI-NEXT:    fmov s7, w2
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w12
+; CHECK-GI-NEXT:    sdiv w18, w18, w8
+; CHECK-GI-NEXT:    mov v5.s[1], w14
+; CHECK-GI-NEXT:    sdiv w3, w3, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w18
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[3]
+; CHECK-GI-NEXT:    mov v7.s[1], w3
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    sdiv w0, w0, w8
+; CHECK-GI-NEXT:    mov v5.s[2], w15
+; CHECK-GI-NEXT:    sdiv w4, w4, w8
+; CHECK-GI-NEXT:    mov v6.s[2], w0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    mov v7.s[2], w4
+; CHECK-GI-NEXT:    sdiv w16, w16, w8
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mls v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    sdiv w1, w1, w8
+; CHECK-GI-NEXT:    mov v5.s[3], w16
+; CHECK-GI-NEXT:    mls v2.4s, v5.4s, v16.4s
+; CHECK-GI-NEXT:    sdiv w8, w5, w8
+; CHECK-GI-NEXT:    mov v6.s[3], w1
+; CHECK-GI-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    mls v0.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    mls v3.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <16 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @sv16i8_100(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-SD-LABEL: sv16i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #41
+; CHECK-SD-NEXT:    smull2 v2.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT:    smull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    movi v2.16b, #100
+; CHECK-SD-NEXT:    sshr v1.16b, v1.16b, #4
+; CHECK-SD-NEXT:    usra v1.16b, v1.16b, #7
+; CHECK-SD-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv16i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-GI-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v16.8b, #100
+; CHECK-GI-NEXT:    sshll v1.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT:    sshll v0.4s, v3.4h, #0
+; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-GI-NEXT:    sshll v16.8h, v16.8b, #0
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s2
+; CHECK-GI-NEXT:    fmov w17, s0
+; CHECK-GI-NEXT:    fmov w2, s3
+; CHECK-GI-NEXT:    mov w14, v2.s[1]
+; CHECK-GI-NEXT:    mov w18, v0.s[1]
+; CHECK-GI-NEXT:    mov w3, v3.s[1]
+; CHECK-GI-NEXT:    mov w15, v2.s[2]
+; CHECK-GI-NEXT:    mov w0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w11, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov w4, v3.s[2]
+; CHECK-GI-NEXT:    mov w16, v2.s[3]
+; CHECK-GI-NEXT:    mov w1, v0.s[3]
+; CHECK-GI-NEXT:    mov w5, v3.s[3]
+; CHECK-GI-NEXT:    sshll v17.4s, v16.4h, #0
+; CHECK-GI-NEXT:    sshll2 v16.4s, v16.8h, #0
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s4, w11
+; CHECK-GI-NEXT:    sdiv w17, w17, w8
+; CHECK-GI-NEXT:    fmov s5, w13
+; CHECK-GI-NEXT:    sdiv w2, w2, w8
+; CHECK-GI-NEXT:    fmov s6, w17
+; CHECK-GI-NEXT:    sdiv w12, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[2]
+; CHECK-GI-NEXT:    fmov s7, w2
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w12
+; CHECK-GI-NEXT:    sdiv w18, w18, w8
+; CHECK-GI-NEXT:    mov v5.s[1], w14
+; CHECK-GI-NEXT:    sdiv w3, w3, w8
+; CHECK-GI-NEXT:    mov v6.s[1], w18
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[3]
+; CHECK-GI-NEXT:    mov v7.s[1], w3
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v4.s[2], w10
+; CHECK-GI-NEXT:    sdiv w0, w0, w8
+; CHECK-GI-NEXT:    mov v5.s[2], w15
+; CHECK-GI-NEXT:    sdiv w4, w4, w8
+; CHECK-GI-NEXT:    mov v6.s[2], w0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    mov v7.s[2], w4
+; CHECK-GI-NEXT:    sdiv w16, w16, w8
+; CHECK-GI-NEXT:    mov v4.s[3], w9
+; CHECK-GI-NEXT:    mls v1.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT:    sdiv w1, w1, w8
+; CHECK-GI-NEXT:    mov v5.s[3], w16
+; CHECK-GI-NEXT:    mls v2.4s, v5.4s, v16.4s
+; CHECK-GI-NEXT:    sdiv w8, w5, w8
+; CHECK-GI-NEXT:    mov v6.s[3], w1
+; CHECK-GI-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    mls v0.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT:    mov v7.s[3], w8
+; CHECK-GI-NEXT:    mls v3.4s, v7.4s, v16.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-GI-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <16 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <16 x i8> %s
+}
+
+define <2 x i8> @uv2i8_7(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: uv2i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-GI-NEXT:    movi v2.2s, #37
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    mov w9, v2.s[1]
+; CHECK-GI-NEXT:    mov v2.b[1], w9
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    umov w8, v2.b[0]
+; CHECK-GI-NEXT:    umov w9, v2.b[1]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mov v2.s[1], w9
+; CHECK-GI-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov w9, v1.s[1]
+; CHECK-GI-NEXT:    mov v2.b[1], w8
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    neg v2.8b, v2.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    umov w8, v1.b[0]
+; CHECK-GI-NEXT:    umov w9, v1.b[1]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i8> %d, <i8 7, i8 7>
+  ret <2 x i8> %s
+}
+
+define <2 x i8> @uv2i8_100(<2 x i8> %d, <2 x i8> %e) {
+; CHECK-SD-LABEL: uv2i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-GI-NEXT:    movi v2.2s, #41
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    and v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    umov w8, v1.b[0]
+; CHECK-GI-NEXT:    umov w9, v1.b[1]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i8> %d, <i8 100, i8 100>
+  ret <2 x i8> %s
+}
+
+define <3 x i8> @uv3i8_7(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: uv3i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    and w10, w1, #0xff
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    and w12, w2, #0xff
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    sub w13, w13, w13, lsl #3
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    add w0, w9, w11
+; CHECK-SD-NEXT:    add w1, w10, w13
+; CHECK-SD-NEXT:    add w2, w12, w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xff
+; CHECK-GI-NEXT:    mov w10, #37 // =0x25
+; CHECK-GI-NEXT:    and w9, w1, #0xff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    fmov s0, w0
+; CHECK-GI-NEXT:    mov v1.h[1], w9
+; CHECK-GI-NEXT:    mov v2.h[1], w10
+; CHECK-GI-NEXT:    and w9, w2, #0xff
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov v0.h[1], w1
+; CHECK-GI-NEXT:    mov v1.h[2], w9
+; CHECK-GI-NEXT:    mov v2.h[2], w10
+; CHECK-GI-NEXT:    mov v3.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov v0.h[2], w2
+; CHECK-GI-NEXT:    mul v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    neg v2.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    mov b3, v2.b[1]
+; CHECK-GI-NEXT:    mov b4, v2.b[2]
+; CHECK-GI-NEXT:    fmov w8, s3
+; CHECK-GI-NEXT:    fmov w9, s4
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    add v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    mov b2, v1.b[1]
+; CHECK-GI-NEXT:    mov b3, v1.b[2]
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w9
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s3
+; CHECK-GI-NEXT:    mov v1.h[2], w9
+; CHECK-GI-NEXT:    mov v2.h[2], w8
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    umov w0, v0.h[0]
+; CHECK-GI-NEXT:    umov w1, v0.h[1]
+; CHECK-GI-NEXT:    umov w2, v0.h[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i8> %d, <i8 7, i8 7, i8 7>
+  ret <3 x i8> %s
+}
+
+define <3 x i8> @uv3i8_100(<3 x i8> %d, <3 x i8> %e) {
+; CHECK-SD-LABEL: uv3i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    and w9, w0, #0xff
+; CHECK-SD-NEXT:    and w10, w1, #0xff
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    and w12, w2, #0xff
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w0, w11, w14, w9
+; CHECK-SD-NEXT:    msub w1, w13, w14, w10
+; CHECK-SD-NEXT:    msub w2, w8, w14, w12
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    and w8, w0, #0xff
+; CHECK-GI-NEXT:    mov w10, #41 // =0x29
+; CHECK-GI-NEXT:    and w9, w1, #0xff
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    mov w8, #8 // =0x8
+; CHECK-GI-NEXT:    fmov s1, w10
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
+; CHECK-GI-NEXT:    mov v1.h[1], w10
+; CHECK-GI-NEXT:    and w9, w2, #0xff
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov v0.h[2], w9
+; CHECK-GI-NEXT:    mov v1.h[2], w10
+; CHECK-GI-NEXT:    mov v2.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mul v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    neg v1.4h, v2.4h
+; CHECK-GI-NEXT:    ushl v0.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    neg v1.8b, v3.8b
+; CHECK-GI-NEXT:    fmov s3, w0
+; CHECK-GI-NEXT:    ushl v0.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    mov v3.h[1], w1
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    mov v3.h[2], w2
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v0.h[1], w9
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s2
+; CHECK-GI-NEXT:    mov v0.h[2], w9
+; CHECK-GI-NEXT:    mov v1.h[2], w8
+; CHECK-GI-NEXT:    mls v3.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umov w0, v3.h[0]
+; CHECK-GI-NEXT:    umov w1, v3.h[1]
+; CHECK-GI-NEXT:    umov w2, v3.h[2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i8> %d, <i8 100, i8 100, i8 100>
+  ret <3 x i8> %s
+}
+
+define <4 x i8> @uv4i8_7(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: uv4i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    umov w13, v0.h[2]
+; CHECK-SD-NEXT:    umov w15, v0.h[3]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x12, w10, w8
+; CHECK-SD-NEXT:    umull x14, w13, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    umull x8, w15, w8
+; CHECK-SD-NEXT:    lsr x12, x12, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    sub w12, w12, w12, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w10, w12
+; CHECK-SD-NEXT:    lsr x9, x14, #32
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    add w8, w15, w8
+; CHECK-SD-NEXT:    add w9, w13, w9
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #37 // =0x25
+; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ushr v2.4h, v1.4h, #8
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v2.4h
+; CHECK-GI-NEXT:    neg v3.8b, v3.8b
+; CHECK-GI-NEXT:    mov v4.b[1], w8
+; CHECK-GI-NEXT:    uzp1 v2.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    mov v4.b[2], w8
+; CHECK-GI-NEXT:    ushl v2.8b, v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-GI-NEXT:    mov v4.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    usra v2.4h, v1.4h, #8
+; CHECK-GI-NEXT:    uzp1 v1.8b, v2.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v4.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    dup v2.4h, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i8> %d, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %s
+}
+
+define <4 x i8> @uv4i8_100(<4 x i8> %d, <4 x i8> %e) {
+; CHECK-SD-LABEL: uv4i8_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    umov w15, v0.h[3]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    msub w9, w11, w14, w9
+; CHECK-SD-NEXT:    umull x11, w12, w8
+; CHECK-SD-NEXT:    msub w10, w13, w14, w10
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    umull x8, w15, w8
+; CHECK-SD-NEXT:    lsr x9, x11, #32
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    msub w9, w9, w14, w12
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w8, w8, w14, w15
+; CHECK-SD-NEXT:    mov v0.h[2], w9
+; CHECK-SD-NEXT:    mov v0.h[3], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i8_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #41 // =0x29
+; CHECK-GI-NEXT:    movi d2, #0xff00ff00ff00ff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.b[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #4 // =0x4
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.b[1], w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mul v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.b[2], w8
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #8
+; CHECK-GI-NEXT:    mov v3.b[3], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    neg v2.8b, v3.8b
+; CHECK-GI-NEXT:    ushl v1.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    dup v2.4h, w8
+; CHECK-GI-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i8> %d, <i8 100, i8 100, i8 100, i8 100>
+  ret <4 x i8> %s
+}
+
+define <8 x i8> @uv8i8_7(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-SD-LABEL: uv8i8_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #37
+; CHECK-SD-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-SD-NEXT:    sub v2.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ushll v2.8h, v2.8b, #0
+; CHECK-SD-NEXT:    shrn v2.8b, v2.8h, #1
+; CHECK-SD-NEXT:    add v1.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT:    movi v2.8b, #7
+; CHECK-SD-NEXT:    ushr v1.8b, v1.8b, #2
+; CHECK-SD-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i8_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.8b, #37
+; CHECK-GI-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-GI-NEXT:    sub v2.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    usra v1.8b, v2.8b, #1
+; CHECK-GI-NEXT:    movi v2.8b, #7
+; CHECK-GI-NEXT:    ushr v1.8b, v1.8b, #2
+; CHECK-GI-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <8 x i8> %s
+}
+
+define <8 x i8> @uv8i8_100(<8 x i8> %d, <8 x i8> %e) {
+; CHECK-LABEL: uv8i8_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.8b, #41
+; CHECK-NEXT:    movi v2.8b, #100
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    shrn v1.8b, v1.8h, #8
+; CHECK-NEXT:    ushr v1.8b, v1.8b, #4
+; CHECK-NEXT:    mls v0.8b, v1.8b, v2.8b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <8 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @uv16i8_7(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-LABEL: uv16i8_7:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.16b, #37
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
+; CHECK-NEXT:    usra v1.16b, v2.16b, #1
+; CHECK-NEXT:    movi v2.16b, #7
+; CHECK-NEXT:    ushr v1.16b, v1.16b, #2
+; CHECK-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <16 x i8> %d, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
+  ret <16 x i8> %s
+}
+
+define <16 x i8> @uv16i8_100(<16 x i8> %d, <16 x i8> %e) {
+; CHECK-LABEL: uv16i8_100:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.16b, #41
+; CHECK-NEXT:    umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT:    umull v1.8h, v0.8b, v1.8b
+; CHECK-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    movi v2.16b, #100
+; CHECK-NEXT:    ushr v1.16b, v1.16b, #4
+; CHECK-NEXT:    mls v0.16b, v1.16b, v2.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = urem <16 x i8> %d, <i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100, i8 100>
+  ret <16 x i8> %s
+}
+
+define <2 x i16> @sv2i16_7(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: sv2i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v1.2s, v0.2s, #16
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v1.2s, #16
+; CHECK-SD-NEXT:    smull v2.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #32
+; CHECK-SD-NEXT:    ssra v2.2s, v1.2s, #16
+; CHECK-SD-NEXT:    sshr v1.2s, v2.2s, #2
+; CHECK-SD-NEXT:    usra v1.2s, v2.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i16> %d, <i16 7, i16 7>
+  ret <2 x i16> %s
+}
+
+define <2 x i16> @sv2i16_100(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: sv2i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mls v0.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i16> %d, <i16 100, i16 100>
+  ret <2 x i16> %s
+}
+
+define <3 x i16> @sv3i16_7(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: sv3i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    mov x8, #-56173 // =0xffffffffffff2493
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    smov w12, v0.h[0]
+; CHECK-SD-NEXT:    smov x11, v0.h[2]
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x8, w11, w8
+; CHECK-SD-NEXT:    smov w11, v0.h[2]
+; CHECK-SD-NEXT:    lsr x9, x9, #32
+; CHECK-SD-NEXT:    lsr x10, x10, #32
+; CHECK-SD-NEXT:    add w9, w9, w12
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    asr w14, w9, #2
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    asr w15, w10, #2
+; CHECK-SD-NEXT:    add w8, w8, w11
+; CHECK-SD-NEXT:    add w9, w14, w9, lsr #31
+; CHECK-SD-NEXT:    asr w14, w8, #2
+; CHECK-SD-NEXT:    add w10, w15, w10, lsr #31
+; CHECK-SD-NEXT:    sub w9, w9, w9, lsl #3
+; CHECK-SD-NEXT:    add w8, w14, w8, lsr #31
+; CHECK-SD-NEXT:    sub w10, w10, w10, lsl #3
+; CHECK-SD-NEXT:    add w9, w12, w9
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w13, w10
+; CHECK-SD-NEXT:    add w8, w11, w8
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    smov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    smov w11, v0.h[1]
+; CHECK-GI-NEXT:    smov w13, v0.h[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w9, w9, w10
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w8, w13, w8
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w10, w15, w12
+; CHECK-GI-NEXT:    sub w10, w11, w10
+; CHECK-GI-NEXT:    mov v0.h[1], w10
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w8, w13, w8
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i16> %d, <i16 7, i16 7, i16 7>
+  ret <3 x i16> %s
+}
+
+define <3 x i16> @sv3i16_100(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: sv3i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    smov x9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    smov x10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    smov x11, v0.h[2]
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    smov w13, v0.h[1]
+; CHECK-SD-NEXT:    smull x9, w9, w8
+; CHECK-SD-NEXT:    smull x10, w10, w8
+; CHECK-SD-NEXT:    smull x8, w11, w8
+; CHECK-SD-NEXT:    smov w11, v0.h[0]
+; CHECK-SD-NEXT:    asr x9, x9, #37
+; CHECK-SD-NEXT:    asr x10, x10, #37
+; CHECK-SD-NEXT:    add w9, w9, w9, lsr #31
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w10, w10, w10, lsr #31
+; CHECK-SD-NEXT:    msub w9, w9, w12, w11
+; CHECK-SD-NEXT:    smov w11, v0.h[2]
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    msub w10, w10, w12, w13
+; CHECK-SD-NEXT:    msub w8, w8, w12, w11
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    smov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    smov w11, v0.h[1]
+; CHECK-GI-NEXT:    smov w13, v0.h[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    msub w9, w10, w8, w9
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w14, w13, w8
+; CHECK-GI-NEXT:    msub w10, w12, w8, w11
+; CHECK-GI-NEXT:    mov v0.h[1], w10
+; CHECK-GI-NEXT:    msub w8, w14, w8, w13
+; CHECK-GI-NEXT:    mov v0.h[2], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i16> %d, <i16 100, i16 100, i16 100>
+  ret <3 x i16> %s
+}
+
+define <4 x i16> @sv4i16_7(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: sv4i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movi v2.4h, #7
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #17
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    usra v1.4h, v1.4h, #15
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v2.4h, #7
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i16> %d, <i16 7, i16 7, i16 7, i16 7>
+  ret <4 x i16> %s
+}
+
+define <4 x i16> @sv4i16_100(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: sv4i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    movi v2.4h, #100
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    sshr v1.4s, v1.4s, #19
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    usra v1.4h, v1.4h, #15
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v2.4h, #100
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i16> %d, <i16 100, i16 100, i16 100, i16 100>
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @sv8i16_7(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: sv8i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    movi v2.8h, #7
+; CHECK-SD-NEXT:    sshr v1.8h, v1.8h, #1
+; CHECK-SD-NEXT:    usra v1.8h, v1.8h, #15
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    movi v4.4h, #7
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i16> %d, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @sv8i16_100(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: sv8i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    smull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    smull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    movi v2.8h, #100
+; CHECK-SD-NEXT:    sshr v1.8h, v1.8h, #3
+; CHECK-SD-NEXT:    usra v1.8h, v1.8h, #15
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv8i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    movi v4.4h, #100
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    mov w10, v1.s[1]
+; CHECK-GI-NEXT:    mov w14, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[2]
+; CHECK-GI-NEXT:    mov w15, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v1.s[3]
+; CHECK-GI-NEXT:    mov w16, v0.s[3]
+; CHECK-GI-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w13, w13, w8
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s3, w13
+; CHECK-GI-NEXT:    sdiv w14, w14, w8
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v3.s[1], w14
+; CHECK-GI-NEXT:    sdiv w15, w15, w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    sdiv w12, w12, w8
+; CHECK-GI-NEXT:    mov v3.s[2], w15
+; CHECK-GI-NEXT:    sdiv w8, w16, w8
+; CHECK-GI-NEXT:    mov v2.s[3], w12
+; CHECK-GI-NEXT:    mls v1.4s, v2.4s, v4.4s
+; CHECK-GI-NEXT:    mov v3.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v3.4s, v4.4s
+; CHECK-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <8 x i16> %d, <i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100>
+  ret <8 x i16> %s
+}
+
+define <2 x i16> @uv2i16_7(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: uv2i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    movi d2, #0x00ffff0000ffff
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    and v2.8b, v0.8b, v2.8b
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mul v1.2s, v2.2s, v1.2s
+; CHECK-GI-NEXT:    neg v3.4h, v3.4h
+; CHECK-GI-NEXT:    ushr v2.2s, v1.2s, #16
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v2.2s
+; CHECK-GI-NEXT:    uzp1 v2.4h, v2.4h, v0.4h
+; CHECK-GI-NEXT:    ushl v2.4h, v2.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    usra v2.2s, v1.2s, #16
+; CHECK-GI-NEXT:    uzp1 v1.4h, v2.4h, v0.4h
+; CHECK-GI-NEXT:    neg v2.4h, v3.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    dup v2.2s, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i16> %d, <i16 7, i16 7>
+  ret <2 x i16> %s
+}
+
+define <2 x i16> @uv2i16_100(<2 x i16> %d, <2 x i16> %e) {
+; CHECK-SD-LABEL: uv2i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    dup v2.2s, w8
+; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    uzp1 v2.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    ushl v1.4h, v2.4h, v1.4h
+; CHECK-GI-NEXT:    ushll v2.4s, v3.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mul v1.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    mov v2.h[1], w8
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #16
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    dup v2.2s, w8
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i16> %d, <i16 100, i16 100>
+  ret <2 x i16> %s
+}
+
+define <3 x i16> @uv3i16_7(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: uv3i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    lsr x13, x13, #32
+; CHECK-SD-NEXT:    sub w11, w11, w11, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w13, w13, w13, lsl #3
+; CHECK-SD-NEXT:    add w9, w9, w11
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    add w10, w10, w13
+; CHECK-SD-NEXT:    add w8, w12, w8
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    umov w9, v0.h[0]
+; CHECK-GI-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-GI-NEXT:    umov w10, v0.h[1]
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    umov w11, v0.h[2]
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov w9, #16 // =0x10
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    mov v2.s[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v3.s[2], w9
+; CHECK-GI-NEXT:    mov w9, #2 // =0x2
+; CHECK-GI-NEXT:    mul v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v3.h[1], w8
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    mov v2.h[1], w9
+; CHECK-GI-NEXT:    mov v3.h[2], w8
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    sub v4.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    neg v3.4h, v3.4h
+; CHECK-GI-NEXT:    ushl v3.4h, v4.4h, v3.4h
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    neg v2.4h, v2.4h
+; CHECK-GI-NEXT:    mov v4.h[1], w8
+; CHECK-GI-NEXT:    add v1.4h, v3.4h, v1.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    mov v4.h[2], w8
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v4.4h
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i16> %d, <i16 7, i16 7, i16 7>
+  ret <3 x i16> %s
+}
+
+define <3 x i16> @uv3i16_100(<3 x i16> %d, <3 x i16> %e) {
+; CHECK-SD-LABEL: uv3i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    umov w9, v0.h[0]
+; CHECK-SD-NEXT:    mov w8, #23593 // =0x5c29
+; CHECK-SD-NEXT:    umov w10, v0.h[1]
+; CHECK-SD-NEXT:    movk w8, #655, lsl #16
+; CHECK-SD-NEXT:    umov w12, v0.h[2]
+; CHECK-SD-NEXT:    mov w14, #100 // =0x64
+; CHECK-SD-NEXT:    umull x11, w9, w8
+; CHECK-SD-NEXT:    umull x13, w10, w8
+; CHECK-SD-NEXT:    umull x8, w12, w8
+; CHECK-SD-NEXT:    lsr x11, x11, #32
+; CHECK-SD-NEXT:    msub w9, w11, w14, w9
+; CHECK-SD-NEXT:    lsr x11, x13, #32
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    msub w10, w11, w14, w10
+; CHECK-SD-NEXT:    fmov s0, w9
+; CHECK-SD-NEXT:    msub w8, w8, w14, w12
+; CHECK-SD-NEXT:    mov v0.h[1], w10
+; CHECK-SD-NEXT:    mov v0.h[2], w8
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, #2 // =0x2
+; CHECK-GI-NEXT:    mov w11, #5243 // =0x147b
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fmov s2, w11
+; CHECK-GI-NEXT:    mov v1.h[1], w8
+; CHECK-GI-NEXT:    mov v2.s[1], w11
+; CHECK-GI-NEXT:    mov v1.h[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w11
+; CHECK-GI-NEXT:    neg v1.4h, v1.4h
+; CHECK-GI-NEXT:    ushl v1.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    umov w8, v1.h[0]
+; CHECK-GI-NEXT:    umov w9, v1.h[1]
+; CHECK-GI-NEXT:    umov w10, v1.h[2]
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    mov w8, #16 // =0x10
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    mov w9, #100 // =0x64
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    mov v1.s[2], w10
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    mov w8, #1 // =0x1
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    mul v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    mov v4.h[1], w8
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s2, w9
+; CHECK-GI-NEXT:    mov v4.h[2], w8
+; CHECK-GI-NEXT:    mov v2.h[1], w9
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    neg v3.4h, v4.4h
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    ushl v1.4h, v1.4h, v3.4h
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i16> %d, <i16 100, i16 100, i16 100>
+  ret <3 x i16> %s
+}
+
+define <4 x i16> @uv4i16_7(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: uv4i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-SD-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-SD-NEXT:    shrn v2.4h, v2.4s, #1
+; CHECK-SD-NEXT:    add v1.4h, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.4h, #7
+; CHECK-SD-NEXT:    ushr v1.4h, v1.4h, #2
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI52_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI52_0]
+; CHECK-GI-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-GI-NEXT:    sub v2.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    usra v1.4h, v2.4h, #1
+; CHECK-GI-NEXT:    movi v2.4h, #7
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #2
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i16> %d, <i16 7, i16 7, i16 7, i16 7>
+  ret <4 x i16> %s
+}
+
+define <4 x i16> @uv4i16_100(<4 x i16> %d, <4 x i16> %e) {
+; CHECK-SD-LABEL: uv4i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    ushr v2.4h, v0.4h, #2
+; CHECK-SD-NEXT:    dup v1.4h, w8
+; CHECK-SD-NEXT:    umull v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.4h, #100
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #17
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI53_0
+; CHECK-GI-NEXT:    ushr v1.4h, v0.4h, #2
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI53_0]
+; CHECK-GI-NEXT:    umull v1.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    movi v2.4h, #100
+; CHECK-GI-NEXT:    shrn v1.4h, v1.4s, #16
+; CHECK-GI-NEXT:    ushr v1.4h, v1.4h, #1
+; CHECK-GI-NEXT:    mls v0.4h, v1.4h, v2.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i16> %d, <i16 100, i16 100, i16 100, i16 100>
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @uv8i16_7(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: uv8i16_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    sub v2.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT:    usra v1.8h, v2.8h, #1
+; CHECK-SD-NEXT:    movi v2.8h, #7
+; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #2
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i16_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI54_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI54_0]
+; CHECK-GI-NEXT:    umull2 v2.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT:    umull v1.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT:    uzp2 v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    sub v2.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    usra v1.8h, v2.8h, #1
+; CHECK-GI-NEXT:    movi v2.8h, #7
+; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #2
+; CHECK-GI-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i16> %d, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
+  ret <8 x i16> %s
+}
+
+define <8 x i16> @uv8i16_100(<8 x i16> %d, <8 x i16> %e) {
+; CHECK-SD-LABEL: uv8i16_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #5243 // =0x147b
+; CHECK-SD-NEXT:    ushr v2.8h, v0.8h, #2
+; CHECK-SD-NEXT:    dup v1.8h, w8
+; CHECK-SD-NEXT:    umull2 v3.4s, v2.8h, v1.8h
+; CHECK-SD-NEXT:    umull v1.4s, v2.4h, v1.4h
+; CHECK-SD-NEXT:    movi v2.8h, #100
+; CHECK-SD-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
+; CHECK-SD-NEXT:    ushr v1.8h, v1.8h, #1
+; CHECK-SD-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv8i16_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI55_0
+; CHECK-GI-NEXT:    ushr v1.8h, v0.8h, #2
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI55_0]
+; CHECK-GI-NEXT:    umull2 v3.4s, v1.8h, v2.8h
+; CHECK-GI-NEXT:    umull v1.4s, v1.4h, v2.4h
+; CHECK-GI-NEXT:    movi v2.8h, #100
+; CHECK-GI-NEXT:    uzp2 v1.8h, v1.8h, v3.8h
+; CHECK-GI-NEXT:    ushr v1.8h, v1.8h, #1
+; CHECK-GI-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <8 x i16> %d, <i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100, i16 100>
+  ret <8 x i16> %s
+}
+
+define <2 x i32> @sv2i32_7(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: sv2i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    add v1.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    sshr v2.2s, v1.2s, #2
+; CHECK-SD-NEXT:    usra v2.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v2.2s, v3.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i32> %d, <i32 7, i32 7>
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @sv2i32_100(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: sv2i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w8, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w8
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i32> %d, <i32 100, i32 100>
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @sv3i32_7(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: sv3i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v3.2s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull x8, w9, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    add w8, w8, w9
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    asr w10, w8, #2
+; CHECK-SD-NEXT:    add w8, w10, w8, lsr #31
+; CHECK-SD-NEXT:    add v1.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    sshr v2.2s, v1.2s, #2
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    usra v2.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v2.2s, v3.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov s0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    fmov w11, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    lsl w14, w10, #3
+; CHECK-GI-NEXT:    sub w10, w14, w10
+; CHECK-GI-NEXT:    sub w9, w9, w10
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w8, w13, w8
+; CHECK-GI-NEXT:    lsl w15, w12, #3
+; CHECK-GI-NEXT:    sub w10, w15, w12
+; CHECK-GI-NEXT:    sub w10, w11, w10
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    lsl w9, w8, #3
+; CHECK-GI-NEXT:    sub w8, w9, w8
+; CHECK-GI-NEXT:    sub w8, w13, w8
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i32> %d, <i32 7, i32 7, i32 7>
+  ret <3 x i32> %s
+}
+
+define <3 x i32> @sv3i32_100(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: sv3i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    mov w10, #100 // =0x64
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    smull x8, w9, w8
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    asr x8, x8, #37
+; CHECK-SD-NEXT:    add w8, w8, w8, lsr #31
+; CHECK-SD-NEXT:    sshr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    msub w8, w8, w10, w9
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    usra v1.2s, v1.2s, #31
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv3i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov s0, v0.s[2]
+; CHECK-GI-NEXT:    sdiv w10, w9, w8
+; CHECK-GI-NEXT:    fmov w11, s1
+; CHECK-GI-NEXT:    fmov w13, s0
+; CHECK-GI-NEXT:    sdiv w12, w11, w8
+; CHECK-GI-NEXT:    msub w9, w10, w8, w9
+; CHECK-GI-NEXT:    fmov s0, w9
+; CHECK-GI-NEXT:    sdiv w14, w13, w8
+; CHECK-GI-NEXT:    msub w10, w12, w8, w11
+; CHECK-GI-NEXT:    mov v0.s[1], w10
+; CHECK-GI-NEXT:    msub w8, w14, w8, w13
+; CHECK-GI-NEXT:    mov v0.s[2], w8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <3 x i32> %d, <i32 100, i32 100, i32 100>
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @sv4i32_7(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: sv4i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #9363 // =0x2493
+; CHECK-SD-NEXT:    movi v3.4s, #7
+; CHECK-SD-NEXT:    movk w8, #37449, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    sshr v2.4s, v1.4s, #2
+; CHECK-SD-NEXT:    usra v2.4s, v1.4s, #31
+; CHECK-SD-NEXT:    mls v0.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    movi v2.4s, #7
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i32> %d, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @sv4i32_100(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: sv4i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v3.4s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    smull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    smull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sshr v2.4s, v1.4s, #5
+; CHECK-SD-NEXT:    usra v2.4s, v1.4s, #31
+; CHECK-SD-NEXT:    mls v0.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv4i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov w10, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v0.s[2]
+; CHECK-GI-NEXT:    mov w12, v0.s[3]
+; CHECK-GI-NEXT:    movi v2.4s, #100
+; CHECK-GI-NEXT:    sdiv w9, w9, w8
+; CHECK-GI-NEXT:    sdiv w10, w10, w8
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    sdiv w11, w11, w8
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    sdiv w8, w12, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w11
+; CHECK-GI-NEXT:    mov v1.s[3], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <4 x i32> %d, <i32 100, i32 100, i32 100, i32 100>
+  ret <4 x i32> %s
+}
+
+define <2 x i32> @uv2i32_7(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: uv2i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #1
+; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI62_0
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI62_0]
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-GI-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    usra v1.2s, v2.2s, #1
+; CHECK-GI-NEXT:    movi v2.2s, #7
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i32> %d, <i32 7, i32 7>
+  ret <2 x i32> %s
+}
+
+define <2 x i32> @uv2i32_100(<2 x i32> %d, <2 x i32> %e) {
+; CHECK-SD-LABEL: uv2i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    ushr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI63_0
+; CHECK-GI-NEXT:    movi v2.2s, #100
+; CHECK-GI-NEXT:    ldr d1, [x8, :lo12:.LCPI63_0]
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-GI-NEXT:    ushr v1.2s, v1.2s, #5
+; CHECK-GI-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i32> %d, <i32 100, i32 100>
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @uv3i32_7(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: uv3i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull x8, w9, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #32
+; CHECK-SD-NEXT:    sub w10, w9, w8
+; CHECK-SD-NEXT:    shrn v1.2s, v1.2d, #32
+; CHECK-SD-NEXT:    add w8, w8, w10, lsr #1
+; CHECK-SD-NEXT:    lsr w8, w8, #2
+; CHECK-SD-NEXT:    sub v2.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    sub w8, w8, w8, lsl #3
+; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-SD-NEXT:    add w8, w9, w8
+; CHECK-SD-NEXT:    shrn v2.2s, v2.2d, #1
+; CHECK-SD-NEXT:    add v1.2s, v2.2s, v1.2s
+; CHECK-SD-NEXT:    movi v2.2s, #7
+; CHECK-SD-NEXT:    ushr v1.2s, v1.2s, #2
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI64_0
+; CHECK-GI-NEXT:    mov w9, #18725 // =0x4925
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI64_0]
+; CHECK-GI-NEXT:    mov w8, v0.s[2]
+; CHECK-GI-NEXT:    movk w9, #9362, lsl #16
+; CHECK-GI-NEXT:    mov w10, #1 // =0x1
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    umull x8, w8, w9
+; CHECK-GI-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    lsr x8, x8, #32
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #32
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov s1, w9
+; CHECK-GI-NEXT:    mov w9, #2 // =0x2
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    fmov s3, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w11
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mov v3.s[1], w9
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w10
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov v3.s[2], w9
+; CHECK-GI-NEXT:    sub v4.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    neg v2.4s, v2.4s
+; CHECK-GI-NEXT:    ushl v2.4s, v4.4s, v2.4s
+; CHECK-GI-NEXT:    fmov s4, w8
+; CHECK-GI-NEXT:    mov v4.s[1], w8
+; CHECK-GI-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT:    neg v2.4s, v3.4s
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    mov v4.s[2], w8
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v4.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i32> %d, <i32 7, i32 7, i32 7>
+  ret <3 x i32> %s
+}
+
+define <3 x i32> @uv3i32_100(<3 x i32> %d, <3 x i32> %e) {
+; CHECK-SD-LABEL: uv3i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    mov w9, v0.s[2]
+; CHECK-SD-NEXT:    movi v2.2s, #100
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    mov w10, #100 // =0x64
+; CHECK-SD-NEXT:    dup v1.2s, w8
+; CHECK-SD-NEXT:    umull x8, w9, w8
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    lsr x8, x8, #37
+; CHECK-SD-NEXT:    msub w8, w8, w10, w9
+; CHECK-SD-NEXT:    ushr v1.2d, v1.2d, #37
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    mls v0.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT:    mov v0.s[2], w8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv3i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI65_0
+; CHECK-GI-NEXT:    mov w9, v0.s[2]
+; CHECK-GI-NEXT:    ldr d2, [x8, :lo12:.LCPI65_0]
+; CHECK-GI-NEXT:    mov w8, #5 // =0x5
+; CHECK-GI-NEXT:    mov w10, #34079 // =0x851f
+; CHECK-GI-NEXT:    fmov s3, w8
+; CHECK-GI-NEXT:    movk w10, #20971, lsl #16
+; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-GI-NEXT:    umull x9, w9, w10
+; CHECK-GI-NEXT:    mov v3.s[1], w8
+; CHECK-GI-NEXT:    umull v1.2d, v1.2s, v2.2s
+; CHECK-GI-NEXT:    mov v3.s[2], w8
+; CHECK-GI-NEXT:    lsr x8, x9, #32
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #32
+; CHECK-GI-NEXT:    neg v3.4s, v3.4s
+; CHECK-GI-NEXT:    mov d2, v1.d[1]
+; CHECK-GI-NEXT:    fmov x11, d1
+; CHECK-GI-NEXT:    fmov s1, w11
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    mov v1.s[1], w10
+; CHECK-GI-NEXT:    mov w10, #100 // =0x64
+; CHECK-GI-NEXT:    fmov s2, w10
+; CHECK-GI-NEXT:    mov v2.s[1], w10
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v2.s[2], w10
+; CHECK-GI-NEXT:    ushl v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <3 x i32> %d, <i32 100, i32 100, i32 100>
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @uv4i32_7(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: uv4i32_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #18725 // =0x4925
+; CHECK-SD-NEXT:    movk w8, #9362, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    sub v2.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    usra v1.4s, v2.4s, #1
+; CHECK-SD-NEXT:    movi v2.4s, #7
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-SD-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i32_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI66_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI66_0]
+; CHECK-GI-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    sub v2.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    usra v1.4s, v2.4s, #1
+; CHECK-GI-NEXT:    movi v2.4s, #7
+; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #2
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i32> %d, <i32 7, i32 7, i32 7, i32 7>
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @uv4i32_100(<4 x i32> %d, <4 x i32> %e) {
+; CHECK-SD-LABEL: uv4i32_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #34079 // =0x851f
+; CHECK-SD-NEXT:    movk w8, #20971, lsl #16
+; CHECK-SD-NEXT:    dup v1.4s, w8
+; CHECK-SD-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    movi v2.4s, #100
+; CHECK-SD-NEXT:    ushr v1.4s, v1.4s, #5
+; CHECK-SD-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv4i32_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI67_0
+; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI67_0]
+; CHECK-GI-NEXT:    umull2 v2.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT:    umull v1.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT:    uzp2 v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    movi v2.4s, #100
+; CHECK-GI-NEXT:    ushr v1.4s, v1.4s, #5
+; CHECK-GI-NEXT:    mls v0.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <4 x i32> %d, <i32 100, i32 100, i32 100, i32 100>
+  ret <4 x i32> %s
+}
+
+define <2 x i64> @sv2i64_7(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: sv2i64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #18725 // =0x4925
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #16
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #32
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #48
+; CHECK-SD-NEXT:    smulh x11, x10, x8
+; CHECK-SD-NEXT:    smulh x8, x9, x8
+; CHECK-SD-NEXT:    asr x12, x11, #1
+; CHECK-SD-NEXT:    add x11, x12, x11, lsr #63
+; CHECK-SD-NEXT:    asr x13, x8, #1
+; CHECK-SD-NEXT:    sub x11, x11, x11, lsl #3
+; CHECK-SD-NEXT:    add x8, x13, x8, lsr #63
+; CHECK-SD-NEXT:    add x10, x10, x11
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    add x8, x9, x8
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov w8, #7 // =0x7
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    sdiv x9, x9, x8
+; CHECK-GI-NEXT:    sdiv x8, x10, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI68_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI68_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i64> %d, <i64 7, i64 7>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @sv2i64_100(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: sv2i64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #55051 // =0xd70b
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #28835, lsl #16
+; CHECK-SD-NEXT:    movk x8, #2621, lsl #32
+; CHECK-SD-NEXT:    movk x8, #41943, lsl #48
+; CHECK-SD-NEXT:    smulh x11, x10, x8
+; CHECK-SD-NEXT:    smulh x8, x9, x8
+; CHECK-SD-NEXT:    add x11, x11, x10
+; CHECK-SD-NEXT:    asr x12, x11, #6
+; CHECK-SD-NEXT:    add x8, x8, x9
+; CHECK-SD-NEXT:    add x11, x12, x11, lsr #63
+; CHECK-SD-NEXT:    asr x13, x8, #6
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    msub x10, x11, x12, x10
+; CHECK-SD-NEXT:    add x8, x13, x8, lsr #63
+; CHECK-SD-NEXT:    msub x8, x8, x12, x9
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    mov w8, #100 // =0x64
+; CHECK-GI-NEXT:    mov x10, v0.d[1]
+; CHECK-GI-NEXT:    sdiv x9, x9, x8
+; CHECK-GI-NEXT:    sdiv x8, x10, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI69_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI69_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i64> %d, <i64 100, i64 100>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @uv2i64_7(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: uv2i64_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #37449, lsl #16
+; CHECK-SD-NEXT:    movk x8, #18724, lsl #32
+; CHECK-SD-NEXT:    movk x8, #9362, lsl #48
+; CHECK-SD-NEXT:    umulh x11, x10, x8
+; CHECK-SD-NEXT:    umulh x8, x9, x8
+; CHECK-SD-NEXT:    sub x12, x10, x11
+; CHECK-SD-NEXT:    add x11, x11, x12, lsr #1
+; CHECK-SD-NEXT:    sub x12, x9, x8
+; CHECK-SD-NEXT:    lsr x11, x11, #2
+; CHECK-SD-NEXT:    add x8, x8, x12, lsr #1
+; CHECK-SD-NEXT:    sub x11, x11, x11, lsl #3
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    add x10, x10, x11
+; CHECK-SD-NEXT:    sub x8, x8, x8, lsl #3
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    add x8, x9, x8
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i64_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x8, #9363 // =0x2493
+; CHECK-GI-NEXT:    fmov x10, d0
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    umulh x10, x10, x8
+; CHECK-GI-NEXT:    umulh x8, x9, x8
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI70_0
+; CHECK-GI-NEXT:    sub v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    usra v1.2d, v2.2d, #1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI70_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #2
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i64> %d, <i64 7, i64 7>
+  ret <2 x i64> %s
+}
+
+define <2 x i64> @uv2i64_100(<2 x i64> %d, <2 x i64> %e) {
+; CHECK-SD-LABEL: uv2i64_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    fmov x10, d0
+; CHECK-SD-NEXT:    mov x8, #62915 // =0xf5c3
+; CHECK-SD-NEXT:    mov x9, v0.d[1]
+; CHECK-SD-NEXT:    movk x8, #23592, lsl #16
+; CHECK-SD-NEXT:    movk x8, #49807, lsl #32
+; CHECK-SD-NEXT:    lsr x11, x10, #2
+; CHECK-SD-NEXT:    movk x8, #10485, lsl #48
+; CHECK-SD-NEXT:    lsr x12, x9, #2
+; CHECK-SD-NEXT:    umulh x11, x11, x8
+; CHECK-SD-NEXT:    umulh x8, x12, x8
+; CHECK-SD-NEXT:    mov w12, #100 // =0x64
+; CHECK-SD-NEXT:    lsr x11, x11, #2
+; CHECK-SD-NEXT:    msub x10, x11, x12, x10
+; CHECK-SD-NEXT:    lsr x8, x8, #2
+; CHECK-SD-NEXT:    msub x8, x8, x12, x9
+; CHECK-SD-NEXT:    fmov d0, x10
+; CHECK-SD-NEXT:    mov v0.d[1], x8
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i64_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ushr v1.2d, v0.2d, #2
+; CHECK-GI-NEXT:    mov x8, #62915 // =0xf5c3
+; CHECK-GI-NEXT:    movk x8, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #32
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #48
+; CHECK-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-GI-NEXT:    umulh x10, x10, x8
+; CHECK-GI-NEXT:    umulh x8, x9, x8
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    adrp x8, .LCPI71_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI71_0]
+; CHECK-GI-NEXT:    fmov x11, d2
+; CHECK-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-GI-NEXT:    ushr v1.2d, v1.2d, #2
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    mul x8, x8, x9
+; CHECK-GI-NEXT:    fmov d1, x10
+; CHECK-GI-NEXT:    mov v1.d[1], x8
+; CHECK-GI-NEXT:    sub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i64> %d, <i64 100, i64 100>
+  ret <2 x i64> %s
+}
+
+define <2 x i128> @sv2i128_7(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: sv2i128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-GI-NEXT:    .cfi_offset w30, -48
+; CHECK-GI-NEXT:    mov x19, x2
+; CHECK-GI-NEXT:    mov x20, x3
+; CHECK-GI-NEXT:    mov w2, #7 // =0x7
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x21, x0
+; CHECK-GI-NEXT:    mov x22, x1
+; CHECK-GI-NEXT:    mov x0, x19
+; CHECK-GI-NEXT:    mov x1, x20
+; CHECK-GI-NEXT:    mov w2, #7 // =0x7
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x2, x0
+; CHECK-GI-NEXT:    mov x3, x1
+; CHECK-GI-NEXT:    mov x0, x21
+; CHECK-GI-NEXT:    mov x1, x22
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i128> %d, <i128 7, i128 7>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @sv2i128_100(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: sv2i128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __modti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sv2i128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-GI-NEXT:    .cfi_offset w30, -48
+; CHECK-GI-NEXT:    mov x19, x2
+; CHECK-GI-NEXT:    mov x20, x3
+; CHECK-GI-NEXT:    mov w2, #100 // =0x64
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x21, x0
+; CHECK-GI-NEXT:    mov x22, x1
+; CHECK-GI-NEXT:    mov x0, x19
+; CHECK-GI-NEXT:    mov x1, x20
+; CHECK-GI-NEXT:    mov w2, #100 // =0x64
+; CHECK-GI-NEXT:    mov x3, xzr
+; CHECK-GI-NEXT:    bl __modti3
+; CHECK-GI-NEXT:    mov x2, x0
+; CHECK-GI-NEXT:    mov x3, x1
+; CHECK-GI-NEXT:    mov x0, x21
+; CHECK-GI-NEXT:    mov x1, x22
+; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = srem <2 x i128> %d, <i128 100, i128 100>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @uv2i128_7(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: uv2i128_7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #7 // =0x7
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i128_7:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x10, #18725 // =0x4925
+; CHECK-GI-NEXT:    mov x8, #9362 // =0x2492
+; CHECK-GI-NEXT:    sub x4, x0, x0
+; CHECK-GI-NEXT:    movk x10, #9362, lsl #16
+; CHECK-GI-NEXT:    movk x8, #37449, lsl #16
+; CHECK-GI-NEXT:    umulh x18, x0, xzr
+; CHECK-GI-NEXT:    movk x10, #37449, lsl #32
+; CHECK-GI-NEXT:    movk x8, #18724, lsl #32
+; CHECK-GI-NEXT:    movk x10, #18724, lsl #48
+; CHECK-GI-NEXT:    movk x8, #9362, lsl #48
+; CHECK-GI-NEXT:    mul x11, x1, x10
+; CHECK-GI-NEXT:    mul x12, x0, x8
+; CHECK-GI-NEXT:    umulh x13, x0, x10
+; CHECK-GI-NEXT:    mul x14, x1, x8
+; CHECK-GI-NEXT:    adds x11, x11, x12
+; CHECK-GI-NEXT:    umulh x15, x1, x10
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    cmn x11, x13
+; CHECK-GI-NEXT:    and x11, x12, #0x1
+; CHECK-GI-NEXT:    umulh x16, x0, x8
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    add x14, x14, x4
+; CHECK-GI-NEXT:    and x12, x12, #0x1
+; CHECK-GI-NEXT:    and x4, xzr, #0x1
+; CHECK-GI-NEXT:    mul x13, x3, x10
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, xzr, #0x1
+; CHECK-GI-NEXT:    adds x14, x14, x15
+; CHECK-GI-NEXT:    add x12, x12, x4
+; CHECK-GI-NEXT:    mul x5, x2, x8
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    adds x14, x14, x16
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    umulh x9, xzr, x10
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    adds x11, x14, x11
+; CHECK-GI-NEXT:    add x12, x12, x16
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    umulh x17, x1, x8
+; CHECK-GI-NEXT:    add x12, x12, x16
+; CHECK-GI-NEXT:    adds x13, x13, x5
+; CHECK-GI-NEXT:    umulh x15, x2, x10
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    and x16, x4, #0x1
+; CHECK-GI-NEXT:    mul x6, x3, x8
+; CHECK-GI-NEXT:    umulh x10, x3, x10
+; CHECK-GI-NEXT:    cmn x13, x15
+; CHECK-GI-NEXT:    and x13, x14, #0x1
+; CHECK-GI-NEXT:    add x14, x9, x17
+; CHECK-GI-NEXT:    umulh x15, x2, x8
+; CHECK-GI-NEXT:    add x12, x12, x13
+; CHECK-GI-NEXT:    add x13, x14, x18
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    sub x17, x2, x2
+; CHECK-GI-NEXT:    and x18, xzr, #0x1
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    umulh x8, x3, x8
+; CHECK-GI-NEXT:    add x12, x13, x12
+; CHECK-GI-NEXT:    add x14, x16, x14
+; CHECK-GI-NEXT:    add x16, x6, x17
+; CHECK-GI-NEXT:    and x17, xzr, #0x1
+; CHECK-GI-NEXT:    adds x10, x16, x10
+; CHECK-GI-NEXT:    add x17, x17, x18
+; CHECK-GI-NEXT:    cset w16, hs
+; CHECK-GI-NEXT:    adds x10, x10, x15
+; CHECK-GI-NEXT:    umulh x15, x2, xzr
+; CHECK-GI-NEXT:    and x16, x16, #0x1
+; CHECK-GI-NEXT:    cset w18, hs
+; CHECK-GI-NEXT:    adds x10, x10, x14
+; CHECK-GI-NEXT:    add x16, x17, x16
+; CHECK-GI-NEXT:    and x17, x18, #0x1
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    add x13, x16, x17
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    add x8, x9, x8
+; CHECK-GI-NEXT:    subs x9, x0, x11
+; CHECK-GI-NEXT:    add x13, x13, x14
+; CHECK-GI-NEXT:    add x8, x8, x15
+; CHECK-GI-NEXT:    sbc x14, x1, x12
+; CHECK-GI-NEXT:    add x8, x8, x13
+; CHECK-GI-NEXT:    subs x13, x2, x10
+; CHECK-GI-NEXT:    lsl x15, x14, #63
+; CHECK-GI-NEXT:    sbc x16, x3, x8
+; CHECK-GI-NEXT:    lsr x14, x14, #1
+; CHECK-GI-NEXT:    orr x9, x15, x9, lsr #1
+; CHECK-GI-NEXT:    lsl x15, x16, #63
+; CHECK-GI-NEXT:    orr x13, x15, x13, lsr #1
+; CHECK-GI-NEXT:    adds x9, x9, x11
+; CHECK-GI-NEXT:    lsr x11, x16, #1
+; CHECK-GI-NEXT:    adc x12, x14, x12
+; CHECK-GI-NEXT:    adds x10, x13, x10
+; CHECK-GI-NEXT:    lsl x13, x12, #62
+; CHECK-GI-NEXT:    lsr x12, x12, #2
+; CHECK-GI-NEXT:    adc x8, x11, x8
+; CHECK-GI-NEXT:    lsl x11, x8, #62
+; CHECK-GI-NEXT:    orr x9, x13, x9, lsr #2
+; CHECK-GI-NEXT:    mov w13, #7 // =0x7
+; CHECK-GI-NEXT:    lsr x8, x8, #2
+; CHECK-GI-NEXT:    lsl x14, x12, #3
+; CHECK-GI-NEXT:    orr x10, x11, x10, lsr #2
+; CHECK-GI-NEXT:    umulh x11, x9, x13
+; CHECK-GI-NEXT:    lsl x15, x9, #3
+; CHECK-GI-NEXT:    sub x12, x14, x12
+; CHECK-GI-NEXT:    lsl x16, x8, #3
+; CHECK-GI-NEXT:    umulh x13, x10, x13
+; CHECK-GI-NEXT:    lsl x14, x10, #3
+; CHECK-GI-NEXT:    sub x9, x15, x9
+; CHECK-GI-NEXT:    sub x8, x16, x8
+; CHECK-GI-NEXT:    subs x0, x0, x9
+; CHECK-GI-NEXT:    sub x10, x14, x10
+; CHECK-GI-NEXT:    add x11, x12, x11
+; CHECK-GI-NEXT:    sbc x1, x1, x11
+; CHECK-GI-NEXT:    subs x2, x2, x10
+; CHECK-GI-NEXT:    add x8, x8, x13
+; CHECK-GI-NEXT:    sbc x3, x3, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i128> %d, <i128 7, i128 7>
+  ret <2 x i128> %s
+}
+
+define <2 x i128> @uv2i128_100(<2 x i128> %d, <2 x i128> %e) {
+; CHECK-SD-LABEL: uv2i128_100:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    str x30, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-SD-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-SD-NEXT:    .cfi_offset w21, -24
+; CHECK-SD-NEXT:    .cfi_offset w22, -32
+; CHECK-SD-NEXT:    .cfi_offset w30, -48
+; CHECK-SD-NEXT:    mov x19, x3
+; CHECK-SD-NEXT:    mov x20, x2
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x21, x0
+; CHECK-SD-NEXT:    mov x22, x1
+; CHECK-SD-NEXT:    mov x0, x20
+; CHECK-SD-NEXT:    mov x1, x19
+; CHECK-SD-NEXT:    mov w2, #100 // =0x64
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    bl __umodti3
+; CHECK-SD-NEXT:    mov x2, x0
+; CHECK-SD-NEXT:    mov x3, x1
+; CHECK-SD-NEXT:    mov x0, x21
+; CHECK-SD-NEXT:    mov x1, x22
+; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: uv2i128_100:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov x10, #23593 // =0x5c29
+; CHECK-GI-NEXT:    mov x8, #62914 // =0xf5c2
+; CHECK-GI-NEXT:    sub x18, x0, x0
+; CHECK-GI-NEXT:    movk x10, #49807, lsl #16
+; CHECK-GI-NEXT:    movk x8, #23592, lsl #16
+; CHECK-GI-NEXT:    movk x10, #10485, lsl #32
+; CHECK-GI-NEXT:    movk x8, #49807, lsl #32
+; CHECK-GI-NEXT:    movk x10, #36700, lsl #48
+; CHECK-GI-NEXT:    movk x8, #10485, lsl #48
+; CHECK-GI-NEXT:    mul x11, x1, x10
+; CHECK-GI-NEXT:    mul x12, x0, x8
+; CHECK-GI-NEXT:    umulh x13, x0, x10
+; CHECK-GI-NEXT:    mul x14, x1, x8
+; CHECK-GI-NEXT:    adds x11, x11, x12
+; CHECK-GI-NEXT:    umulh x15, x1, x10
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    cmn x11, x13
+; CHECK-GI-NEXT:    and x11, x12, #0x1
+; CHECK-GI-NEXT:    umulh x16, x0, x8
+; CHECK-GI-NEXT:    cset w12, hs
+; CHECK-GI-NEXT:    and x12, x12, #0x1
+; CHECK-GI-NEXT:    add x14, x14, x18
+; CHECK-GI-NEXT:    add x11, x11, x12
+; CHECK-GI-NEXT:    and x12, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x9, xzr, x10
+; CHECK-GI-NEXT:    adds x14, x14, x15
+; CHECK-GI-NEXT:    and x15, xzr, #0x1
+; CHECK-GI-NEXT:    umulh x17, x1, x8
+; CHECK-GI-NEXT:    cset w4, hs
+; CHECK-GI-NEXT:    add x15, x12, x15
+; CHECK-GI-NEXT:    adds x12, x14, x16
+; CHECK-GI-NEXT:    and x4, x4, #0x1
+; CHECK-GI-NEXT:    mul x18, x3, x10
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    adds x12, x12, x11
+; CHECK-GI-NEXT:    add x11, x15, x4
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    cset w15, hs
+; CHECK-GI-NEXT:    mul x5, x2, x8
+; CHECK-GI-NEXT:    add x11, x11, x14
+; CHECK-GI-NEXT:    and x14, x15, #0x1
+; CHECK-GI-NEXT:    add x17, x9, x17
+; CHECK-GI-NEXT:    add x14, x11, x14
+; CHECK-GI-NEXT:    mov w11, #100 // =0x64
+; CHECK-GI-NEXT:    umulh x13, x0, xzr
+; CHECK-GI-NEXT:    umulh x16, x2, x10
+; CHECK-GI-NEXT:    adds x18, x18, x5
+; CHECK-GI-NEXT:    mul x15, x3, x8
+; CHECK-GI-NEXT:    add x13, x17, x13
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    umulh x10, x3, x10
+; CHECK-GI-NEXT:    add x13, x13, x14
+; CHECK-GI-NEXT:    and x17, x17, #0x1
+; CHECK-GI-NEXT:    cmn x18, x16
+; CHECK-GI-NEXT:    sub x18, x2, x2
+; CHECK-GI-NEXT:    umulh x16, x2, x8
+; CHECK-GI-NEXT:    cset w14, hs
+; CHECK-GI-NEXT:    and x14, x14, #0x1
+; CHECK-GI-NEXT:    add x15, x15, x18
+; CHECK-GI-NEXT:    and x18, xzr, #0x1
+; CHECK-GI-NEXT:    add x14, x17, x14
+; CHECK-GI-NEXT:    umulh x8, x3, x8
+; CHECK-GI-NEXT:    and x17, xzr, #0x1
+; CHECK-GI-NEXT:    adds x10, x15, x10
+; CHECK-GI-NEXT:    add x15, x17, x18
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    umulh x18, x2, xzr
+; CHECK-GI-NEXT:    and x17, x17, #0x1
+; CHECK-GI-NEXT:    adds x10, x10, x16
+; CHECK-GI-NEXT:    lsl x16, x13, #60
+; CHECK-GI-NEXT:    add x15, x15, x17
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    adds x10, x10, x14
+; CHECK-GI-NEXT:    and x14, x17, #0x1
+; CHECK-GI-NEXT:    cset w17, hs
+; CHECK-GI-NEXT:    add x8, x9, x8
+; CHECK-GI-NEXT:    add x14, x15, x14
+; CHECK-GI-NEXT:    and x15, x17, #0x1
+; CHECK-GI-NEXT:    orr x12, x16, x12, lsr #4
+; CHECK-GI-NEXT:    add x9, x14, x15
+; CHECK-GI-NEXT:    add x8, x8, x18
+; CHECK-GI-NEXT:    add x8, x8, x9
+; CHECK-GI-NEXT:    lsr x9, x13, #4
+; CHECK-GI-NEXT:    umulh x14, x12, x11
+; CHECK-GI-NEXT:    lsl x13, x8, #60
+; CHECK-GI-NEXT:    lsr x8, x8, #4
+; CHECK-GI-NEXT:    mul x12, x12, x11
+; CHECK-GI-NEXT:    orr x10, x13, x10, lsr #4
+; CHECK-GI-NEXT:    madd x9, x9, x11, x14
+; CHECK-GI-NEXT:    umulh x13, x10, x11
+; CHECK-GI-NEXT:    subs x0, x0, x12
+; CHECK-GI-NEXT:    mul x10, x10, x11
+; CHECK-GI-NEXT:    sbc x1, x1, x9
+; CHECK-GI-NEXT:    madd x8, x8, x11, x13
+; CHECK-GI-NEXT:    subs x2, x2, x10
+; CHECK-GI-NEXT:    sbc x3, x3, x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = urem <2 x i128> %d, <i128 100, i128 100>
+  ret <2 x i128> %s
+}
diff --git a/llvm/test/CodeGen/AArch64/sat-add.ll b/llvm/test/CodeGen/AArch64/sat-add.ll
index 2deb19be2482..ecd48d6b7c65 100644
--- a/llvm/test/CodeGen/AArch64/sat-add.ll
+++ b/llvm/test/CodeGen/AArch64/sat-add.ll
@@ -530,7 +530,7 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
@@ -570,7 +570,7 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
@@ -610,7 +610,7 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
@@ -651,7 +651,7 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2
 ; CHECK-NEXT:    mvn v2.16b, v1.16b
 ; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
diff --git a/llvm/test/CodeGen/AArch64/select_cc.ll b/llvm/test/CodeGen/AArch64/select_cc.ll
index 73e4d4c7f0ae..483f6c26af8c 100644
--- a/llvm/test/CodeGen/AArch64/select_cc.ll
+++ b/llvm/test/CodeGen/AArch64/select_cc.ll
@@ -88,7 +88,7 @@ define <2 x double> @select_olt_load_cmp(<2 x double> %a, ptr %src) {
 ; CHECK-SD-NEXT:    ldr d1, [x0]
 ; CHECK-SD-NEXT:    fcmgt v1.2s, v1.2s, #0.0
 ; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    and v0.16b, v1.16b, v0.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: select_olt_load_cmp:
diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
index 32fc9c137770..0d4a63644616 100644
--- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
@@ -249,9 +249,6 @@ define <16 x i8> @sel_shift_bool_v16i8(<16 x i1> %t) {
 ; CHECK-SD-LABEL: sel_shift_bool_v16i8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    shl v0.16b, v0.16b, #7
-; CHECK-SD-NEXT:    movi v1.16b, #128
-; CHECK-SD-NEXT:    cmlt v0.16b, v0.16b, #0
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: sel_shift_bool_v16i8:
diff --git a/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll b/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
new file mode 100644
index 000000000000..69f603458670
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -O0 < %s | FileCheck %s
+
+; Example of locally streaming function that (at -O0) must preserve the LR (X30)
+; before calling __arm_get_current_vg.
+define void @foo() "aarch64_pstate_sm_body" {
+; CHECK-LABEL: foo:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    lsr x9, x9, #3
+; CHECK-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
+; CHECK-NEXT:    bl __arm_get_current_vg
+; CHECK-NEXT:    str x0, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
+; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    ret
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fclamp.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fclamp.ll
new file mode 100644
index 000000000000..b7ada9f1d3fa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fclamp.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+define { <vscale x 8 x half>, <vscale x 8 x half> } @test_fclamp_single_x2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d) {
+; CHECK-LABEL: test_fclamp_single_x2_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fclamp { z0.h, z1.h }, z2.h, z3.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x2.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d)
+  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float> } @test_fclamp_single_x2_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d) {
+; CHECK-LABEL: test_fclamp_single_x2_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fclamp { z0.s, z1.s }, z2.s, z3.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x2.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d)
+  ret { <vscale x 4 x float>, <vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double> } @test_fclamp_single_x2_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d) {
+; CHECK-LABEL: test_fclamp_single_x2_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    fclamp { z0.d, z1.d }, z2.d, z3.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x2.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d)
+  ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
+}
+
+
+define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @test_fclamp_single_x4_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f) {
+; CHECK-LABEL: test_fclamp_single_x4_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fclamp { z0.h - z3.h }, z4.h, z5.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x4.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f)
+  ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @test_fclamp_single_x4_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f) {
+; CHECK-LABEL: test_fclamp_single_x4_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fclamp { z0.s - z3.s }, z4.s, z5.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x4.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f)
+  ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @test_fclamp_single_x4_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f) {
+; CHECK-LABEL: test_fclamp_single_x4_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    fclamp { z0.d - z3.d }, z4.d, z5.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x4.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f)
+  ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
+}
+
+declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
+declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
+
+declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x4.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f)
+declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x4.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f)
+declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x4.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f)
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvtn.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvtn.ll
index 25f14d0a6c42..704b63edfd69 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvtn.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-qcvtn.ll
@@ -1,22 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py$
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+bf16 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; SQCVTN
 ;
 
-; x2
-define <vscale x 8 x i16 > @multi_vector_qcvtn_x2_s16_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
-; CHECK-LABEL: multi_vector_qcvtn_x2_s16_s32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z3.d, z2.d
-; CHECK-NEXT:    mov z2.d, z1.d
-; CHECK-NEXT:    sqcvtn z0.h, { z2.s, z3.s }
-; CHECK-NEXT:    ret
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtn.x2.nxv4i32(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)
-  ret <vscale x 8 x i16> %res
-}
-
 ; x4
 define <vscale x 16 x i8 > @multi_vector_qcvtn_x4_s8_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {
 ; CHECK-LABEL: multi_vector_qcvtn_x4_s8_s32:
@@ -48,18 +36,6 @@ define <vscale x 8 x i16> @multi_vector_qcvtn_x4_s16_s64(<vscale x 2 x i64> %unu
 ; UQCVTN
 ;
 
-; x2
-define <vscale x 8 x i16> @multi_vector_qcvtn_x2_u16_u32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1) {
-; CHECK-LABEL: multi_vector_qcvtn_x2_u16_u32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z3.d, z2.d
-; CHECK-NEXT:    mov z2.d, z1.d
-; CHECK-NEXT:    uqcvtn z0.h, { z2.s, z3.s }
-; CHECK-NEXT:    ret
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uqcvtn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)
-  ret<vscale x 8 x i16> %res
-}
-
 ; x4
 define <vscale x 16 x i8> @multi_vector_qcvtn_x4_u8_u32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {
 ; CHECK-LABEL: multi_vector_qcvtn_x4_u8_u32:
@@ -91,17 +67,6 @@ define <vscale x 8 x i16> @multi_vector_qcvtn_x4_u16_u64(<vscale x 2 x i64> %unu
 ; SQCVTUN
 ;
 
-; x2
-define <vscale x 8 x i16 > @multi_vector_qcvtn_x2_s16_u32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
-; CHECK-LABEL: multi_vector_qcvtn_x2_s16_u32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov z3.d, z2.d
-; CHECK-NEXT:    mov z2.d, z1.d
-; CHECK-NEXT:    sqcvtun z0.h, { z2.s, z3.s }
-; CHECK-NEXT:    ret
-  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtun.x2.nxv4i322(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)
-  ret <vscale x 8 x i16> %res
-}
 ; x4
 define <vscale x 16 x i8> @multi_vector_qcvtn_x4_u8_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2, <vscale x 4 x i32> %zn3, <vscale x 4 x i32> %zn4) {
 ; CHECK-LABEL: multi_vector_qcvtn_x4_u8_s32:
@@ -129,9 +94,6 @@ define <vscale x 8 x i16> @multi_vector_qcvtn_x4_u16_s64(<vscale x 2 x i64> %unu
   ret <vscale x 8 x i16> %res
 }
 
-declare <vscale x 8 x i16> @llvm.aarch64.sve.uqcvtn.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtn.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtun.x2.nxv4i322(<vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.sqcvtn.x4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtn.x4.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.uqcvtn.x4.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-sclamp.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sclamp.ll
new file mode 100644
index 000000000000..b9b8469d692b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-sclamp.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_sclamp_single_x2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d) {
+; CHECK-LABEL: test_sclamp_single_x2_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    sclamp { z0.b, z1.b }, z2.b, z3.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x2.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d)
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_sclamp_single_x2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d) {
+; CHECK-LABEL: test_sclamp_single_x2_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    sclamp { z0.h, z1.h }, z2.h, z3.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x  8 x i16>, <vscale x  8 x i16> } @llvm.aarch64.sve.sclamp.single.x2.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d)
+  ret { <vscale x 8 x i16>, <vscale x  8 x i16> } %res
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32> } @test_sclamp_single_x2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) {
+; CHECK-LABEL: test_sclamp_single_x2_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    sclamp { z0.s, z1.s }, z2.s, z3.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x2.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
+  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64> } @test_sclamp_single_x2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d) {
+; CHECK-LABEL: test_sclamp_single_x2_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    sclamp { z0.d, z1.d }, z2.d, z3.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x2.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d)
+  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_sclamp_single_x4_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f) {
+; CHECK-LABEL: test_sclamp_single_x4_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    sclamp { z0.b - z3.b }, z4.b, z5.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_sclamp_single_x4_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f) {
+; CHECK-LABEL: test_sclamp_single_x4_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    sclamp { z0.h - z3.h }, z4.h, z5.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
+  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @test_sclamp_single_x4_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f) {
+; CHECK-LABEL: test_sclamp_single_x4_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    sclamp { z0.s - z3.s }, z4.s, z5.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
+  ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @test_sclamp_single_x4_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f) {
+; CHECK-LABEL: test_sclamp_single_x4_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    sclamp { z0.d - z3.d }, z4.d, z5.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
+  ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
+}
+
+declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
+declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
+declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
+declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-uclamp.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-uclamp.ll
new file mode 100644
index 000000000000..be0391b67e47
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-uclamp.ll
@@ -0,0 +1,110 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux-gnu"
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_uclamp_single_x2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d) {
+; CHECK-LABEL: test_uclamp_single_x2_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    uclamp { z0.b, z1.b }, z2.b, z3.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x2.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d)
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_uclamp_single_x2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d) {
+; CHECK-LABEL: test_uclamp_single_x2_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    uclamp { z0.h, z1.h }, z2.h, z3.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x  8 x i16>, <vscale x  8 x i16> } @llvm.aarch64.sve.uclamp.single.x2.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d)
+  ret { <vscale x 8 x i16>, <vscale x  8 x i16> } %res
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32> } @test_uclamp_single_x2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) {
+; CHECK-LABEL: test_uclamp_single_x2_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    uclamp { z0.s, z1.s }, z2.s, z3.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x2.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
+  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64> } @test_uclamp_single_x2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d) {
+; CHECK-LABEL: test_uclamp_single_x2_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT:    uclamp { z0.d, z1.d }, z2.d, z3.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x2.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d)
+  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
+}
+
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_uclamp_single_x4_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f) {
+; CHECK-LABEL: test_uclamp_single_x4_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    uclamp { z0.b - z3.b }, z4.b, z5.b
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
+  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_uclamp_single_x4_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f) {
+; CHECK-LABEL: test_uclamp_single_x4_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    uclamp { z0.h - z3.h }, z4.h, z5.h
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
+  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @test_uclamp_single_x4_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f) {
+; CHECK-LABEL: test_uclamp_single_x4_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    uclamp { z0.s - z3.s }, z4.s, z5.s
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
+  ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @test_uclamp_single_x4_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f) {
+; CHECK-LABEL: test_uclamp_single_x4_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
+; CHECK-NEXT:    uclamp { z0.d - z3.d }, z4.d, z5.d
+; CHECK-NEXT:    ret
+  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
+  ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
+}
+
+declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
+declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
+declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
+declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
+declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
diff --git a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
index cbb7f09513aa..02fef348592c 100644
--- a/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
+++ b/llvm/test/CodeGen/AArch64/stp-opt-with-renaming.mir
@@ -735,6 +735,35 @@ body:             |
     RET undef $lr
 ...
 ---
+name:            bundled_load_rename
+alignment:       4
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:     1
+  maxCallFrameSize: 0
+machineFunctionInfo:
+  hasRedZone:      false
+body:             |
+  bb.0.entry:
+    liveins: $z3, $z19, $p0, $z16
+    ; CHECK-LABEL: name: bundled_load_rename
+    ; CHECK: liveins: $z3, $z19, $p0, $z16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $q16, $q20 = LDPQi $sp, 0 :: (load (s128)), (load (s128), align 32)
+    ; CHECK-NEXT: BUNDLE implicit-def $z3, implicit-def $q3, implicit-def $d3, implicit-def $s3, implicit-def $h3, implicit-def $b3, implicit $z19, implicit $p0, implicit $z20 {
+    ; CHECK-NEXT:   $z3 = MOVPRFX_ZZ $z19
+    ; CHECK-NEXT:   $z3 = FMUL_ZPmZ_S renamable $p0, killed $z3, $z20
+    ; CHECK-NEXT: }
+    ; CHECK-NEXT: RET undef $lr
+    renamable $q16 = LDRQui $sp, 1 :: (load 16)
+    BUNDLE implicit-def $z3, implicit-def $q3, implicit-def $d3, implicit-def $s3, implicit-def $h3, implicit-def $b3, implicit $z19, implicit $p0, implicit $z16 {
+      $z3 = MOVPRFX_ZZ $z19
+      $z3 = FMUL_ZPmZ_S renamable $p0, killed $z3, renamable $z16
+    }
+    renamable $q16 = LDRQui $sp, 0 :: (load 16, align 32)
+    RET undef $lr
+...
+---
 #
 # CHECK-LABEL: name: bundled_use
 # CHECK: renamable $q0 = LDRQui $sp, 1
diff --git a/llvm/test/CodeGen/AArch64/streaming-func-no-sme.ll b/llvm/test/CodeGen/AArch64/streaming-func-no-sme.ll
new file mode 100644
index 000000000000..9be776f81727
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/streaming-func-no-sme.ll
@@ -0,0 +1,7 @@
+; RUN: not llc -mtriple aarch64-none-linux-gnu -filetype=null %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: streaming SVE functions require SME
+define void @streaming(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-abd.ll b/llvm/test/CodeGen/AArch64/sve-abd.ll
index 72790155d046..92bbd4e7f275 100644
--- a/llvm/test/CodeGen/AArch64/sve-abd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-abd.ll
@@ -283,8 +283,7 @@ define <vscale x 4 x i32> @uabd_non_matching_promotion(<vscale x 4 x i8> %a, <vs
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    and z0.s, z0.s, #0xff
 ; CHECK-NEXT:    sxtb z1.s, p0/m, z1.s
-; CHECK-NEXT:    sub z0.s, z0.s, z1.s
-; CHECK-NEXT:    abs z0.s, p0/m, z0.s
+; CHECK-NEXT:    sabd z0.s, p0/m, z0.s, z1.s
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.zext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
index 1e71c4b66156..b62b85043446 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-bit-counting.ll
@@ -457,23 +457,25 @@ define void @ctpop_v256i8(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <4 x i16> @ctpop_v4i16(<4 x i16> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v4i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
   ret <4 x i16> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <8 x i16> @ctpop_v8i16(<8 x i16> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    ptrue p0.h, vl8
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
   ret <8 x i16> %res
@@ -547,25 +549,25 @@ define void @ctpop_v128i16(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <2 x i32> @ctpop_v2i32(<2 x i32> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
   ret <2 x i32> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <4 x i32> @ctpop_v4i32(<4 x i32> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
   ret <4 x i32> %res
@@ -639,27 +641,25 @@ define void @ctpop_v64i32(ptr %a) vscale_range(16,0) #0 {
   ret void
 }
 
-; Don't use SVE for 64-bit vectors.
 define <1 x i64> @ctpop_v1i64(<1 x i64> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
   ret <1 x i64> %res
 }
 
-; Don't use SVE for 128-bit vectors.
 define <2 x i64> @ctpop_v2i64(<2 x i64> %op) vscale_range(2,0) #0 {
 ; CHECK-LABEL: ctpop_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ret
   %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
   ret <2 x i64> %res
diff --git a/llvm/test/CodeGen/AArch64/sve-indexed-arithmetic.ll b/llvm/test/CodeGen/AArch64/sve-indexed-arithmetic.ll
new file mode 100644
index 000000000000..bcf5063bdda0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-indexed-arithmetic.ll
@@ -0,0 +1,366 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+define void @fmul_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    fmul z0.h, z1.h, z0.h[2]
+; CHECK-NEXT:    str z0, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x half>, ptr %a
+  %ld.b = load <16 x half>, ptr %b
+  %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                  i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %res = fmul <16 x half> %ld.b, %splat.lanes
+  store <16 x half> %res, ptr %c
+  ret void
+}
+
+define void @fmul_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_bf16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEXT:    dup v0.8h, v0.h[2]
+; CHECK-NEXT:    dup v1.8h, v1.h[2]
+; CHECK-NEXT:    shll v4.4s, v2.4h, #16
+; CHECK-NEXT:    shll v6.4s, v3.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT:    shll v5.4s, v0.4h, #16
+; CHECK-NEXT:    shll v7.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT:    fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT:    fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT:    fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    bfcvtn v2.4h, v4.4s
+; CHECK-NEXT:    bfcvtn v3.4h, v5.4s
+; CHECK-NEXT:    bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT:    bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT:    stp q2, q3, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x bfloat>, ptr %a
+  %ld.b = load <16 x bfloat>, ptr %b
+  %splat.lanes = shufflevector <16 x bfloat> %ld.a, <16 x bfloat> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                      i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %res = fmul <16 x bfloat> %ld.b, %splat.lanes
+  store <16 x bfloat> %res, ptr %c
+  ret void
+}
+
+define void @fmul_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f32_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    fmul z0.s, z1.s, z0.s[3]
+; CHECK-NEXT:    str z0, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <8 x float>, ptr %a
+  %ld.b = load <8 x float>, ptr %b
+  %splat.lanes = shufflevector <8 x float> %ld.a, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                                 i32 7, i32 7, i32 7, i32 7>
+  %res = fmul <8 x float> %splat.lanes, %ld.b
+  store <8 x float> %res, ptr %c
+  ret void
+}
+
+define void @fmul_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f64_256b_trn1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    fmul z0.d, z1.d, z0.d[0]
+; CHECK-NEXT:    str z0, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %res = fmul <4 x double> %splat.lanes, %ld.b
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+define void @fmul_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmul_indexed_f64_256b_trn2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    fmul z0.d, z1.d, z0.d[1]
+; CHECK-NEXT:    str z0, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  %res = fmul <4 x double> %ld.b, %splat.lanes
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+define void @fmla_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmla z2.h, z1.h, z0.h[2]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x half>, ptr %a
+  %ld.b = load <16 x half>, ptr %b
+  %ld.c = load <16 x half>, ptr %c
+  %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                  i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %res = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %ld.b, <16 x half> %splat.lanes, <16 x half> %ld.c)
+  store <16 x half> %res, ptr %c
+  ret void
+}
+
+define void @fmla_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_bf16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEXT:    dup v0.8h, v0.h[2]
+; CHECK-NEXT:    dup v1.8h, v1.h[2]
+; CHECK-NEXT:    shll v4.4s, v2.4h, #16
+; CHECK-NEXT:    shll v6.4s, v3.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT:    shll v5.4s, v0.4h, #16
+; CHECK-NEXT:    shll v7.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT:    fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT:    fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT:    fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    bfcvtn v2.4h, v4.4s
+; CHECK-NEXT:    bfcvtn v3.4h, v5.4s
+; CHECK-NEXT:    bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT:    bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT:    ldp q0, q1, [x2]
+; CHECK-NEXT:    shll v4.4s, v0.4h, #16
+; CHECK-NEXT:    shll v5.4s, v2.4h, #16
+; CHECK-NEXT:    shll v6.4s, v1.4h, #16
+; CHECK-NEXT:    shll v7.4s, v3.4h, #16
+; CHECK-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT:    fadd v4.4s, v5.4s, v4.4s
+; CHECK-NEXT:    fadd v5.4s, v7.4s, v6.4s
+; CHECK-NEXT:    fadd v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fadd v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    bfcvtn v2.4h, v4.4s
+; CHECK-NEXT:    bfcvtn v3.4h, v5.4s
+; CHECK-NEXT:    bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT:    bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT:    stp q2, q3, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x bfloat>, ptr %a
+  %ld.b = load <16 x bfloat>, ptr %b
+  %ld.c = load <16 x bfloat>, ptr %c
+  %splat.lanes = shufflevector <16 x bfloat> %ld.a, <16 x bfloat> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                      i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %res = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> %ld.b, <16 x bfloat> %splat.lanes, <16 x bfloat> %ld.c)
+  store <16 x bfloat> %res, ptr %c
+  ret void
+}
+
+define void @fmla_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f32_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmla z2.s, z1.s, z0.s[3]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <8 x float>, ptr %a
+  %ld.b = load <8 x float>, ptr %b
+  %ld.c = load <8 x float>, ptr %c
+  %splat.lanes = shufflevector <8 x float> %ld.a, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                                 i32 7, i32 7, i32 7, i32 7>
+  %res = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %splat.lanes, <8 x float> %ld.b, <8 x float> %ld.c)
+  store <8 x float> %res, ptr %c
+  ret void
+}
+
+define void @fmla_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f64_256b_trn1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmla z2.d, z1.d, z0.d[0]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %ld.c = load <4 x double>, ptr %c
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %splat.lanes, <4 x double> %ld.b, <4 x double> %ld.c)
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+define void @fmla_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmla_indexed_f64_256b_trn2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmla z2.d, z1.d, z0.d[1]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %ld.c = load <4 x double>, ptr %c
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %ld.b, <4 x double> %splat.lanes, <4 x double> %ld.c)
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+define void @fmls_indexed_f16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmls z2.h, z1.h, z0.h[2]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x half>, ptr %a
+  %ld.b = load <16 x half>, ptr %b
+  %ld.c = load <16 x half>, ptr %c
+  %splat.lanes = shufflevector <16 x half> %ld.a, <16 x half> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                  i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %neg.b = fneg <16 x half> %ld.b
+  %res = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> %neg.b, <16 x half> %splat.lanes, <16 x half> %ld.c)
+  store <16 x half> %res, ptr %c
+  ret void
+}
+
+define void @fmls_indexed_bf16_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_bf16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q2, q3, [x1]
+; CHECK-NEXT:    dup v0.8h, v0.h[2]
+; CHECK-NEXT:    dup v1.8h, v1.h[2]
+; CHECK-NEXT:    shll v4.4s, v2.4h, #16
+; CHECK-NEXT:    shll v6.4s, v3.4h, #16
+; CHECK-NEXT:    shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT:    shll v5.4s, v0.4h, #16
+; CHECK-NEXT:    shll v7.4s, v1.4h, #16
+; CHECK-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT:    fmul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT:    fmul v5.4s, v6.4s, v7.4s
+; CHECK-NEXT:    fmul v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    fmul v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    bfcvtn v2.4h, v4.4s
+; CHECK-NEXT:    bfcvtn v3.4h, v5.4s
+; CHECK-NEXT:    bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT:    bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT:    ldp q0, q1, [x2]
+; CHECK-NEXT:    shll v4.4s, v0.4h, #16
+; CHECK-NEXT:    shll v5.4s, v2.4h, #16
+; CHECK-NEXT:    shll v6.4s, v1.4h, #16
+; CHECK-NEXT:    shll v7.4s, v3.4h, #16
+; CHECK-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-NEXT:    shll2 v2.4s, v2.8h, #16
+; CHECK-NEXT:    shll2 v1.4s, v1.8h, #16
+; CHECK-NEXT:    shll2 v3.4s, v3.8h, #16
+; CHECK-NEXT:    fsub v4.4s, v4.4s, v5.4s
+; CHECK-NEXT:    fsub v5.4s, v6.4s, v7.4s
+; CHECK-NEXT:    fsub v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    fsub v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    bfcvtn v2.4h, v4.4s
+; CHECK-NEXT:    bfcvtn v3.4h, v5.4s
+; CHECK-NEXT:    bfcvtn2 v2.8h, v0.4s
+; CHECK-NEXT:    bfcvtn2 v3.8h, v1.4s
+; CHECK-NEXT:    stp q2, q3, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <16 x bfloat>, ptr %a
+  %ld.b = load <16 x bfloat>, ptr %b
+  %ld.c = load <16 x bfloat>, ptr %c
+  %splat.lanes = shufflevector <16 x bfloat> %ld.a, <16 x bfloat> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                      i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  %neg.b = fneg <16 x bfloat> %ld.b
+  %res = call <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> %neg.b, <16 x bfloat> %splat.lanes, <16 x bfloat> %ld.c)
+  store <16 x bfloat> %res, ptr %c
+  ret void
+}
+
+define void @fmls_indexed_f32_256b(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f32_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmls z2.s, z1.s, z0.s[3]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <8 x float>, ptr %a
+  %ld.b = load <8 x float>, ptr %b
+  %ld.c = load <8 x float>, ptr %c
+  %splat.lanes = shufflevector <8 x float> %ld.a, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                                 i32 7, i32 7, i32 7, i32 7>
+  %neg.b = fneg <8 x float> %ld.b
+  %res = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> %splat.lanes, <8 x float> %neg.b, <8 x float> %ld.c)
+  store <8 x float> %res, ptr %c
+  ret void
+}
+
+define void @fmls_indexed_f64_256b_trn1(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f64_256b_trn1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmls z2.d, z1.d, z0.d[0]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %ld.c = load <4 x double>, ptr %c
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  %neg.b = fneg <4 x double> %ld.b
+  %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %splat.lanes, <4 x double> %neg.b, <4 x double> %ld.c)
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+define void @fmls_indexed_f64_256b_trn2(ptr %a, ptr %b, ptr %c) #0 {
+; CHECK-LABEL: fmls_indexed_f64_256b_trn2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ldr z1, [x1]
+; CHECK-NEXT:    ldr z2, [x2]
+; CHECK-NEXT:    fmls z2.d, z1.d, z0.d[1]
+; CHECK-NEXT:    str z2, [x2]
+; CHECK-NEXT:    ret
+  %ld.a = load <4 x double>, ptr %a
+  %ld.b = load <4 x double>, ptr %b
+  %ld.c = load <4 x double>, ptr %c
+  %splat.lanes = shufflevector <4 x double> %ld.a, <4 x double> poison, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
+  %neg.b = fneg <4 x double> %ld.b
+  %res = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> %neg.b, <4 x double> %splat.lanes, <4 x double> %ld.c)
+  store <4 x double> %res, ptr %c
+  ret void
+}
+
+declare <16 x half> @llvm.fmuladd.v16f16(<16 x half>, <16 x half>, <16 x half>);
+declare <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat>, <16 x bfloat>, <16 x bfloat>);
+declare <8 x float> @llvm.fmuladd.v8f32(<8 x float>, <8 x float>, <8 x float>);
+declare <4 x double> @llvm.fmuladd.v4f64(<4 x double>, <4 x double>, <4 x double>);
+
+attributes #0 = { noinline vscale_range(2,2) "target-features"="+sve2p1,+bf16,+sve-b16b16" }
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
index 021d4855905e..fb837c4279f6 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-fexpa.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -force-streaming -mattr=+ssve-fexpa < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+ssve-fexpa -force-streaming < %s | FileCheck %s
 
 define <vscale x 8 x half> @fexpa_h(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: fexpa_h:
@@ -27,4 +27,4 @@ define <vscale x 2 x double> @fexpa_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
 ; CHECK-NEXT:    ret
   %out = call <vscale x 2 x double> @llvm.aarch64.sve.fexpa.x.nxv2f64(<vscale x 2 x i64> %a)
   ret <vscale x 2 x double> %out
-}
\ No newline at end of file
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-merging-unary.ll b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
new file mode 100644
index 000000000000..9e331a69bcf7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
@@ -0,0 +1,1748 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s
+; RUN: llc -mattr=+sme2 -force-streaming < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: abs_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    abs z1.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: abs_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    abs z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %b, i1 0)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: abs_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    abs z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %b, i1 0)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @abs_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: abs_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    abs z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %b, i1 0)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @clz_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: clz_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    clz z1.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @clz_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: clz_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    clz z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @clz_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: clz_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    clz z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @clz_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: clz_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    clz z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @cnt_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cnt_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    cnt z1.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @cnt_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cnt_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    cnt z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @cnt_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cnt_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    cnt z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @cnt_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cnt_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    cnt z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x half> @fabs_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fabs_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fabs z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fabs_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fabs_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fabs z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fabs_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fabs_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    fabs z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fabs_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fabs_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fabs z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fabs_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fabs_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fabs z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fabs_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fabs_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fabs z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fabs_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @fabs_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @fabs_nxv8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fabs_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0x7fff
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %b.op, <vscale x 8 x bfloat> %a
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 2 x float> @fcvt_nxv2f16_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvt_nxv2f16_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.s, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x half> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2f16_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvt_nxv2f16_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.d, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x half> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 4 x float> @fcvt_nxv4f16_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvt_nxv4f16_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvt z1.s, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 4 x half> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f32_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2f32_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.d, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x float> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fcvt_nxv2f32_to_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f32_to_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    bfcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x bfloat>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x half> @fcvt_nxv4f32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_nxv4f32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 4 x float> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x bfloat> @fcvt_nxv4f32_to_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvt_nxv4f32_to_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    bfcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 4 x float> %b to <vscale x 4 x bfloat>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.h, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x double> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x half> @fcvt_nxv2f64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x float> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x bfloat> @fcvt_nxv2f64_to_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x double> %b) "target-features"="+sve2" {
+; CHECK-LABEL: fcvt_nxv2f64_to_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtx z1.s, p1/m, z1.d
+; CHECK-NEXT:    bfcvt z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptrunc <vscale x 2 x double> %b to <vscale x 2 x bfloat>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 2 x float> @fcvt_nxv2bf16_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv2bf16_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @fcvt_nxv2bf16_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv2bf16_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvt z1.d, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 2 x bfloat> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 4 x float> @fcvt_nxv4bf16_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcvt_nxv4bf16_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fpext <vscale x 4 x bfloat> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f16_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f16_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzs z1.d, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x half> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtsu_nxv4f16_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv4f16_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvtzs z1.s, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 4 x half> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @fcvtsu_nxv8f16_to_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtsu_nxv8f16_to_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    fcvtzs z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 8 x half> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f32_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f32_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzs z1.d, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x float> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtsu_nxv4f32_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtsu_nxv4f32_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvtzs z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 4 x float> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @fcvtsu_nxv2f64_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtsu_nxv2f64_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzs z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptosi <vscale x 2 x double> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f16_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f16_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzu z1.d, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x half> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtzu_nxv4f16_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv4f16_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvtzu z1.s, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 4 x half> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i16> @fcvtzu_nxv8f16_to_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fcvtzu_nxv8f16_to_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    fcvtzu z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 8 x half> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f32_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f32_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzu z1.d, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x float> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @fcvtzu_nxv4f32_to_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fcvtzu_nxv4f32_to_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fcvtzu z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 4 x float> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @fcvtzu_nxv2f64_to_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fcvtzu_nxv2f64_to_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fcvtzu z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fptoui <vscale x 2 x double> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x half> @fneg_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fneg_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fneg z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x half> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fneg_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fneg_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fneg z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x half> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fneg_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fneg_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    fneg z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 8 x half> %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fneg_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fneg_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fneg z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x float> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fneg_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fneg_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fneg z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x float> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fneg_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fneg_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fneg z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x double> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x bfloat> @fneg_nxv2bf16(<vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 2 x bfloat> %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x bfloat> %b.op, <vscale x 2 x bfloat> %a
+  ret <vscale x 2 x bfloat> %res
+}
+
+define <vscale x 4 x bfloat> @fneg_nxv4bf16(<vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 4 x bfloat> %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x bfloat> %b.op, <vscale x 4 x bfloat> %a
+  ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @fneg_nxv8bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fneg_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eor z1.h, z1.h, #0x8000
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = fneg <vscale x 8 x bfloat> %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %b.op, <vscale x 8 x bfloat> %a
+  ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 2 x half> @frinta_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frinta_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinta z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.round.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frinta_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frinta_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frinta z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.round.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frinta_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinta_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frinta z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.round.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frinta_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frinta_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinta z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frinta_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinta_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frinta z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frinta_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinta_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinta z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frinti_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frinti_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinti z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frinti_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frinti_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frinti z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frinti_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frinti_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frinti z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frinti_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frinti_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinti z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frinti_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frinti_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frinti z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frinti_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frinti_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frinti z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintm_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintm_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintm z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.floor.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintm_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintm_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintm z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.floor.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintm_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintm_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frintm z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.floor.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintm_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintm_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintm z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintm_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintm_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintm z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintm_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintm_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintm z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintn_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintn_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintn z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.roundeven.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintn_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintn_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintn z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.roundeven.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintn_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintn_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frintn z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.roundeven.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintn_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintn_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintn z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintn_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintn_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintn z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintn_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintn_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintn z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintp_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintp_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintp z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintp_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintp_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintp z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintp_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintp_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frintp z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintp_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintp_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintp z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintp_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintp_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintp z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintp_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintp_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintp z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintx_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintx_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintx z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.rint.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintx_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintx_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintx z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.rint.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintx_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintx_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frintx z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.rint.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintx_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintx_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintx z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintx_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintx_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintx z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintx_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintx_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintx z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @frintz_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: frintz_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintz z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.trunc.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @frintz_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: frintz_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintz z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.trunc.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @frintz_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: frintz_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    frintz z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.trunc.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @frintz_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: frintz_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintz z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @frintz_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: frintz_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    frintz z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @frintz_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: frintz_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    frintz z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 2 x half> @fsqrt_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fsqrt z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x half> @llvm.sqrt.nxv2f16(<vscale x 2 x half> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 4 x half> @fsqrt_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fsqrt z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x half> @llvm.sqrt.nxv4f16(<vscale x 4 x half> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @fsqrt_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: fsqrt_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    fsqrt z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 2 x float> @fsqrt_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: fsqrt_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fsqrt z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @fsqrt_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: fsqrt_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    fsqrt z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x double> @fsqrt_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: fsqrt_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    fsqrt z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 16 x i8> @neg_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: neg_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 16 x i8> zeroinitializer, %b
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @neg_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: neg_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 8 x i16> zeroinitializer, %b
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @neg_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: neg_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 4 x i32> zeroinitializer, %b
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @neg_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: neg_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sub <vscale x 2 x i64> zeroinitializer, %b
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 16 x i8> @rbit_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: rbit_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.b
+; CHECK-NEXT:    rbit z1.b, p1/m, z1.b
+; CHECK-NEXT:    mov z0.b, p0/m, z1.b
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 16 x i8> @llvm.bitreverse.nxv16i8(<vscale x 16 x i8> %b)
+  %res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
+  ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 8 x i16> @rbit_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: rbit_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    rbit z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.bitreverse.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @rbit_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: rbit_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    rbit z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.bitreverse.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @rbit_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: rbit_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    rbit z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.bitreverse.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x i16> @revb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: revb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    revb z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16> %b)
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @revb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: revb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    revb z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32> %b)
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @revb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: revb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    revb z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = call <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64> %b)
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @scvtf_nxv8i16_to_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: scvtf_nxv8i16_to_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    scvtf z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 8 x i16> %b to <vscale x 8 x half>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @scvtf_nxv4i32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_nxv4i32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    scvtf z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 4 x i32> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @scvtf_nxv4i32_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: scvtf_nxv4i32_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    scvtf z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 4 x i32> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @scvtf_nxv2i64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    scvtf z1.h, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x float> @scvtf_nxv2i64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    scvtf z1.s, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @scvtf_nxv2i64_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: scvtf_nxv2i64_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    scvtf z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sitofp <vscale x 2 x i64> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x i16> @sxtb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    sxtb z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @sxtb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    sxtb z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sxtb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i8> %b) {
+; CHECK-LABEL: sxtb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxtb z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i8> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @sxth_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: sxth_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    sxth z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @sxth_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: sxth_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxth z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @sxtw_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: sxtw_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    sxtw z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = sext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 8 x half> @ucvtf_nxv8i16_to_nxv8f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: ucvtf_nxv8i16_to_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.h
+; CHECK-NEXT:    ucvtf z1.h, p1/m, z1.h
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 8 x i16> %b to <vscale x 8 x half>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x half> %b.op, <vscale x 8 x half> %a
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @ucvtf_nxv4i32_to_nxv4f16(<vscale x 4 x i1> %pg, <vscale x 4 x half> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_nxv4i32_to_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    ucvtf z1.h, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 4 x i32> %b to <vscale x 4 x half>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x half> %b.op, <vscale x 4 x half> %a
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 4 x float> @ucvtf_nxv4i32_to_nxv4f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: ucvtf_nxv4i32_to_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.s
+; CHECK-NEXT:    ucvtf z1.s, p1/m, z1.s
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 4 x i32> %b to <vscale x 4 x float>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x float> %b.op, <vscale x 4 x float> %a
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x half> @ucvtf_nxv2i64_to_nxv2f16(<vscale x 2 x i1> %pg, <vscale x 2 x half> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    ucvtf z1.h, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x half>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x half> %b.op, <vscale x 2 x half> %a
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 2 x float> @ucvtf_nxv2i64_to_nxv2f32(<vscale x 2 x i1> %pg, <vscale x 2 x float> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    ucvtf z1.s, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x float>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x float> %b.op, <vscale x 2 x float> %a
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 2 x double> @ucvtf_nxv2i64_to_nxv2f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: ucvtf_nxv2i64_to_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    ucvtf z1.d, p1/m, z1.d
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = uitofp <vscale x 2 x i64> %b to <vscale x 2 x double>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x double> %b.op, <vscale x 2 x double> %a
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 8 x i16> @uxtb_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.h, z1.h, #0xff
+; CHECK-NEXT:    mov z0.h, p0/m, z1.h
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 8 x i8> %b to <vscale x 8 x i16>
+  %res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
+  ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 4 x i32> @uxtb_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.s, z1.s, #0xff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 4 x i8> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uxtb_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i8> %b) {
+; CHECK-LABEL: uxtb_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i8> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i32> @uxth_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i16> %b) {
+; CHECK-LABEL: uxth_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.s, z1.s, #0xffff
+; CHECK-NEXT:    mov z0.s, p0/m, z1.s
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 4 x i16> %b to <vscale x 4 x i32>
+  %res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
+  ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 2 x i64> @uxth_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i16> %b) {
+; CHECK-LABEL: uxth_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xffff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 2 x i64> @uxtw_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i32> %b) {
+; CHECK-LABEL: uxtw_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and z1.d, z1.d, #0xffffffff
+; CHECK-NEXT:    mov z0.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+  %b.op = zext <vscale x 2 x i32> %b to <vscale x 2 x i64>
+  %res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
+  ret <vscale x 2 x i64> %res
+}
+
+
+declare <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8>, i1)
+declare <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16>, i1)
+declare <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32>, i1)
+declare <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64>, i1)
+
+declare <vscale x 16 x i8> @llvm.bitreverse.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.bitreverse.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.bitreverse.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.bitreverse.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 8 x i16> @llvm.bswap.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.bswap.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.bswap.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 2 x half> @llvm.ceil.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.ceil.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.ceil.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.ceil.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.ceil.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.ceil.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 16 x i8> @llvm.ctlz.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctlz.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctlz.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 16 x i8> @llvm.ctpop.nxv16i8(<vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.ctpop.nxv8i16(<vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.ctpop.nxv4i32(<vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ctpop.nxv2i64(<vscale x 2 x i64>)
+
+declare <vscale x 2 x half> @llvm.fabs.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.fabs.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.fabs.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 2 x bfloat> @llvm.fabs.nxv2bf16(<vscale x 2 x bfloat>)
+declare <vscale x 4 x bfloat> @llvm.fabs.nxv4bf16(<vscale x 4 x bfloat>)
+declare <vscale x 8 x bfloat> @llvm.fabs.nxv8bf16(<vscale x 8 x bfloat>)
+
+declare <vscale x 2 x half> @llvm.floor.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.floor.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.floor.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.floor.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.floor.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.floor.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 4 x half> @llvm.nearbyint.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.nearbyint.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.nearbyint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.nearbyint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.nearbyint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.rint.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.rint.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.rint.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.rint.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.rint.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.rint.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.round.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.round.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.round.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.round.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.round.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.round.nxv2f64(<vscale x 2 x double>)
+declare <vscale x 2 x half> @llvm.nearbyint.nxv2f16(<vscale x 2 x half>)
+
+declare <vscale x 2 x half> @llvm.roundeven.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.roundeven.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.roundeven.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.roundeven.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.roundeven.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.roundeven.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.sqrt.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.sqrt.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.sqrt.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.sqrt.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.sqrt.nxv2f64(<vscale x 2 x double>)
+
+declare <vscale x 2 x half> @llvm.trunc.nxv2f16(<vscale x 2 x half>)
+declare <vscale x 4 x half> @llvm.trunc.nxv4f16(<vscale x 4 x half>)
+declare <vscale x 8 x half> @llvm.trunc.nxv8f16(<vscale x 8 x half>)
+declare <vscale x 2 x float> @llvm.trunc.nxv2f32(<vscale x 2 x float>)
+declare <vscale x 4 x float> @llvm.trunc.nxv4f32(<vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.trunc.nxv2f64(<vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-vscale.ll b/llvm/test/CodeGen/AArch64/sve-vscale.ll
index 7214c98cc318..173652630bf1 100644
--- a/llvm/test/CodeGen/AArch64/sve-vscale.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vscale.ll
@@ -1,50 +1,50 @@
 ; RUN: llc -mtriple aarch64 -mattr=+sve -asm-verbose=0 < %s | FileCheck %s
-; RUN: opt -mtriple=aarch64 -passes='require<profile-summary>,function(codegenprepare)' -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s
+; RUN: opt -mtriple=aarch64 -passes="require<profile-summary>,function(codegenprepare)" -S < %s | llc -mtriple=aarch64 -mattr=+sve -asm-verbose=0 | FileCheck %s
 
 ;
 ; RDVL
 ;
 
+define i8 @rdvl_i8() nounwind {
 ; CHECK-LABEL: rdvl_i8:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i8 @rdvl_i8() nounwind {
   %vscale = call i8 @llvm.vscale.i8()
   %1 = mul nsw i8 %vscale, 16
   ret i8 %1
 }
 
+define i16 @rdvl_i16() nounwind {
 ; CHECK-LABEL: rdvl_i16:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i16 @rdvl_i16() nounwind {
   %vscale = call i16 @llvm.vscale.i16()
   %1 = mul nsw i16 %vscale, 16
   ret i16 %1
 }
 
+define i32 @rdvl_i32() nounwind {
 ; CHECK-LABEL: rdvl_i32:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i32 @rdvl_i32() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 16
   ret i32 %1
 }
 
+define i64 @rdvl_i64() nounwind {
 ; CHECK-LABEL: rdvl_i64:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i64 @rdvl_i64() nounwind {
   %vscale = call i64 @llvm.vscale.i64()
   %1 = mul nsw i64 %vscale, 16
   ret i64 %1
 }
 
+define i32 @rdvl_const() nounwind {
 ; CHECK-LABEL: rdvl_const:
 ; CHECK:       rdvl x0, #1
 ; CHECK-NEXT:  ret
-define i32 @rdvl_const() nounwind {
   %vscale.ptr = getelementptr <vscale x 1 x i8>, ptr null, i64 1
   %vscale.int = ptrtoint ptr %vscale.ptr to i32
   %vscale.scaled = mul nsw i32 %vscale.int, 16
@@ -70,32 +70,31 @@ define i32 @vscale_neg1() nounwind {
   ret i32 %neg
 }
 
+define i32 @rdvl_3() nounwind {
 ; CHECK-LABEL: rdvl_3:
 ; CHECK:       rdvl [[VL_B:x[0-9]+]], #1
 ; CHECK-NEXT:  mov  w[[MUL:[0-9]+]], #3
 ; CHECK-NEXT:  lsr  [[VL_Q:x[0-9]+]], [[VL_B]], #4
 ; CHECK-NEXT:  mul  x0, [[VL_Q]], x[[MUL]]
 ; CHECK-NEXT:  ret
-define i32 @rdvl_3() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 3
   ret i32 %1
 }
 
-
+define i32 @rdvl_min() nounwind {
 ; CHECK-LABEL: rdvl_min:
 ; CHECK:       rdvl x0, #-32
 ; CHECK-NEXT:  ret
-define i32 @rdvl_min() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -512
   ret i32 %1
 }
 
+define i32 @rdvl_max() nounwind {
 ; CHECK-LABEL: rdvl_max:
 ; CHECK:       rdvl x0, #31
 ; CHECK-NEXT:  ret
-define i32 @rdvl_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 496
   ret i32 %1
@@ -116,29 +115,29 @@ define i1 @rdvl_i1() {
 ; CNTH
 ;
 
+define i32 @cnth() nounwind {
 ; CHECK-LABEL: cnth:
 ; CHECK:       cnth x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cnth() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 3
   ret i32 %1
 }
 
+define i32 @cnth_max() nounwind {
 ; CHECK-LABEL: cnth_max:
 ; CHECK:       cnth x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cnth_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 120
   ret i32 %1
 }
 
+define i32 @cnth_neg() nounwind {
 ; CHECK-LABEL: cnth_neg:
 ; CHECK:       cnth [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cnth_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -8
   ret i32 %1
@@ -148,29 +147,29 @@ define i32 @cnth_neg() nounwind {
 ; CNTW
 ;
 
+define i32 @cntw() nounwind {
 ; CHECK-LABEL: cntw:
 ; CHECK:       cntw x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cntw() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 2
   ret i32 %1
 }
 
+define i32 @cntw_max() nounwind {
 ; CHECK-LABEL: cntw_max:
 ; CHECK:       cntw x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cntw_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 60
   ret i32 %1
 }
 
+define i32 @cntw_neg() nounwind {
 ; CHECK-LABEL: cntw_neg:
 ; CHECK:       cntw [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cntw_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -4
   ret i32 %1
@@ -180,29 +179,29 @@ define i32 @cntw_neg() nounwind {
 ; CNTD
 ;
 
+define i32 @cntd() nounwind {
 ; CHECK-LABEL: cntd:
 ; CHECK:       cntd x0{{$}}
 ; CHECK-NEXT:  ret
-define i32 @cntd() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = shl nsw i32 %vscale, 1
   ret i32 %1
 }
 
+define i32 @cntd_max() nounwind {
 ; CHECK-LABEL: cntd_max:
 ; CHECK:       cntd x0, all, mul #15
 ; CHECK-NEXT:  ret
-define i32 @cntd_max() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, 30
   ret i32 %1
 }
 
+define i32 @cntd_neg() nounwind {
 ; CHECK-LABEL: cntd_neg:
 ; CHECK:       cntd [[CNT:x[0-9]+]]
 ; CHECK:       neg x0, [[CNT]]
 ; CHECK-NEXT:  ret
-define i32 @cntd_neg() nounwind {
   %vscale = call i32 @llvm.vscale.i32()
   %1 = mul nsw i32 %vscale, -2
   ret i32 %1
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
index e2483cff3d18..1ccb5264aa83 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-add-sub.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ; ADDHNB
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
index 2f7b82751cdc..2b2c9da1a306 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-binary-narrowing-shr.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll
index 5cbe88213680..67e8896cf755 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-bit-permutation.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2,+sve2-bitperm < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-bitperm < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sve-bitperm < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+ssve-bitperm < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+ssve-bitperm -force-streaming < %s | FileCheck %s
 
 ;
 ; BDEP
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
index 8d395adda079..3a2a02f80a58 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-complex-dot.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
index 6005fb69ae1b..27416375ad6a 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-contiguous-conflict-detection.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
index f477fcbe1eb5..1647f0bd299a 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-crypto.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-aes,+sve2-sha3,+sve2-sm4 < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2,+sve-aes,+sve2-sha3,+sve2-sm4 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-aes < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sve-aes < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+ssve-aes < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+ssve-aes -force-streaming < %s | FileCheck %s
 
 ;
 ; AESD
@@ -78,53 +80,7 @@ define <vscale x 16 x i8> @aesmc_i8(<vscale x 16 x i8> %a) {
   ret <vscale x 16 x i8> %out
 }
 
-;
-; RAX1
-;
-
-define <vscale x 2 x i64> @rax1_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
-; CHECK-LABEL: rax1_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    rax1 z0.d, z0.d, z1.d
-; CHECK-NEXT:    ret
-  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.rax1(<vscale x 2 x i64> %a,
-                                                        <vscale x 2 x i64> %b)
-  ret <vscale x 2 x i64> %out
-}
-
-;
-; SM4E
-;
-
-define <vscale x 4 x i32> @sm4e_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: sm4e_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sm4e z0.s, z0.s, z1.s
-; CHECK-NEXT:    ret
-  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sm4e(<vscale x 4 x i32> %a,
-                                                        <vscale x 4 x i32> %b)
-  ret <vscale x 4 x i32> %out
-}
-
-;
-; SM4EKEY
-;
-
-define <vscale x 4 x i32> @sm4ekey_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: sm4ekey_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sm4ekey z0.s, z0.s, z1.s
-; CHECK-NEXT:    ret
-  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sm4ekey(<vscale x 4 x i32> %a,
-                                                           <vscale x 4 x i32> %b)
-  ret <vscale x 4 x i32> %out
-}
-
-
 declare <vscale x 16 x i8> @llvm.aarch64.sve.aesd(<vscale x 16 x i8>, <vscale x 16 x i8>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.aesimc(<vscale x 16 x i8>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.aese(<vscale x 16 x i8>, <vscale x 16 x i8>)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.aesmc(<vscale x 16 x i8>)
-declare <vscale x 2 x i64> @llvm.aarch64.sve.rax1(<vscale x 2 x i64>, <vscale x 2 x i64>)
-declare <vscale x 4 x i32> @llvm.aarch64.sve.sm4e(<vscale x 4 x i32>, <vscale x 4 x i32>)
-declare <vscale x 4 x i32> @llvm.aarch64.sve.sm4ekey(<vscale x 4 x i32>, <vscale x 4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
index 7d16f8383d96..9be3a67f88b0 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-faminmax.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2 < %s | FileCheck %s
 ; RUN: llc -mattr=+sme2 -force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-linux"
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
index 16041766605e..1ce756416799 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-converts.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
index 52c04d614b4e..d9a3a28ff9aa 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-int-binary-logarithm.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
index 6dc2c67b5fd9..70824f88caef 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-fp-widening-mul-acc.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
index c46016e0c40d..c78a872fcfbf 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-int-mul-lane.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
index 5cea7536e1f3..8e53a82401e0 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-luti.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve2,+lut,+bf16 | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve2,+lut | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sve,+sme2,+lut | FileCheck %s
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -mattr=+sme2,+lut --force-streaming | FileCheck %s
 
 define <vscale x 16 x i8> @test_luti2_lane_i8(<vscale x 16 x i8> %table, <vscale x 16 x i8> %indices){
 ; CHECK-LABEL: test_luti2_lane_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
index ab4945f5178d..ab6c02b22db7 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-non-widening-pairwise-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
index 08afb88a2c57..400b2890f497 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-polynomial-arithmetic.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
index ccc7a4d14a77..01f714283191 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-psel.ll
@@ -1,4 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,sme -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 16 x i1> @psel_b(<vscale x 16 x i1> %p1, <vscale x 16 x i1> %p2, i32 %idx) {
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-rax1.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-rax1.ll
new file mode 100644
index 000000000000..8f208955630b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-rax1.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-sha3 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sve-sha3 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1,+sve-sha3 -force-streaming < %s | FileCheck %s
+
+;
+; RAX1
+;
+
+define <vscale x 2 x i64> @rax1_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: rax1_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rax1 z0.d, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.rax1(<vscale x 2 x i64> %a,
+                                                        <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 2 x i64> @llvm.aarch64.sve.rax1(<vscale x 2 x i64>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
index 16ebd594ca21..5e837da8bb9c 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-revd.ll
@@ -1,4 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_revd_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
new file mode 100644
index 000000000000..de5af6ce0ae9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-sm4.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2-sm4 < %s | FileCheck %s
+
+;
+; SM4E
+;
+
+define <vscale x 4 x i32> @sm4e_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sm4e_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sm4e z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sm4e(<vscale x 4 x i32> %a,
+                                                        <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+;
+; SM4EKEY
+;
+
+define <vscale x 4 x i32> @sm4ekey_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sm4ekey_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sm4ekey z0.s, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sm4ekey(<vscale x 4 x i32> %a,
+                                                           <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sm4e(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sm4ekey(<vscale x 4 x i32>, <vscale x 4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
index 6492f094b707..77503a62041b 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-unary-narrowing.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
index 8c0ef68dda5a..63889df5336e 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-uniform-complex-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
index e5750099258d..c702038be31f 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while-reversed.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
index 610333f0ff34..5f09ab2458f5 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
index 1786f5c514fb..8ad983d92e84 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-complex-int-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
index 3d2a6930ca97..c88549e7769e 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-dsp.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
index 6854ad51989a..7e852f7f1c28 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -force-streaming < %s | FileCheck %s
 
 ;
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
index 2b96d452fba0..579bbf52dd7a 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-bfmlsl.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -mattr=+b16b16 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 4 x float> @bfmlslb_f32(<vscale x 4 x float> %zda, <vscale x 8 x bfloat> %zn, <vscale x 8 x bfloat> %zm) {
 ; CHECK-LABEL: bfmlslb_f32:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
index 45446a4127ee..b7c47628a76f 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-cntp.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define i64 @test_svcntp_c8_vlx2(target("aarch64.svcount") %pn) nounwind {
 ; CHECK-LABEL: test_svcntp_c8_vlx2:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
index f4cb0a3f70ee..b280ee69bb85 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 4 x i32> @sdot_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
 ; CHECK-LABEL: sdot_x2:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
index f1a423b02ac2..53e7d684e1f9 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dupq.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_dupq_i8 (<vscale x 16 x i8> %zn) {
 ; CHECK-LABEL: test_dupq_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
index bb4c67fca5dc..ac52e23133b6 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-extq.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i8> @test_extq_i8 (<vscale x 16 x i8> %zn, <vscale x 16 x i8> %zm) {
 ; CHECK-LABEL: test_extq_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
index 0a5a0b7cfcc8..82b272e8d814 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fclamp.ll
@@ -1,9 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme2 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
-define <vscale x 8 x half> @test_fclamp_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) #0 {
+define <vscale x 8 x half> @test_fclamp_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c) {
 ; CHECK-LABEL: test_fclamp_f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fclamp z0.h, z1.h, z2.h
@@ -12,7 +15,7 @@ define <vscale x 8 x half> @test_fclamp_f16(<vscale x 8 x half> %a, <vscale x 8
   ret <vscale x 8 x half> %res
 }
 
-define <vscale x 4 x float> @test_fclamp_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) #0 {
+define <vscale x 4 x float> @test_fclamp_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) {
 ; CHECK-LABEL: test_fclamp_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fclamp z0.s, z1.s, z2.s
@@ -21,7 +24,7 @@ define <vscale x 4 x float> @test_fclamp_f32(<vscale x 4 x float> %a, <vscale x
   ret <vscale x 4 x float> %res
 }
 
-define <vscale x 2 x double> @test_fclamp_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) #0 {
+define <vscale x 2 x double> @test_fclamp_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c) {
 ; CHECK-LABEL: test_fclamp_f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fclamp z0.d, z1.d, z2.d
@@ -30,91 +33,6 @@ define <vscale x 2 x double> @test_fclamp_f64(<vscale x 2 x double> %a, <vscale
   ret <vscale x 2 x double> %res
 }
 
-define { <vscale x 8 x half>, <vscale x 8 x half> } @test_fclamp_single_x2_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d) #1 {
-; CHECK-LABEL: test_fclamp_single_x2_f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    fclamp { z0.h, z1.h }, z2.h, z3.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x2.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d)
-  ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
-}
-
-define { <vscale x 4 x float>, <vscale x 4 x float> } @test_fclamp_single_x2_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d) #1 {
-; CHECK-LABEL: test_fclamp_single_x2_f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    fclamp { z0.s, z1.s }, z2.s, z3.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x2.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d)
-  ret { <vscale x 4 x float>, <vscale x 4 x float> } %res
-}
-
-define { <vscale x 2 x double>, <vscale x 2 x double> } @test_fclamp_single_x2_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d) #1 {
-; CHECK-LABEL: test_fclamp_single_x2_f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    fclamp { z0.d, z1.d }, z2.d, z3.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x2.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d)
-  ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
-}
-
-
-define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @test_fclamp_single_x4_f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f) #1 {
-; CHECK-LABEL: test_fclamp_single_x4_f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    fclamp { z0.h - z3.h }, z4.h, z5.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x4.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f)
-  ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res
-}
-
-define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @test_fclamp_single_x4_f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f) #1 {
-; CHECK-LABEL: test_fclamp_single_x4_f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    fclamp { z0.s - z3.s }, z4.s, z5.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x4.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f)
-  ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res
-}
-
-define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @test_fclamp_single_x4_f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f) #1 {
-; CHECK-LABEL: test_fclamp_single_x4_f64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    fclamp { z0.d - z3.d }, z4.d, z5.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x4.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f)
-  ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res
-}
-
-
-attributes #0 = { "target-features"="+sve2p1" }
-attributes #1 = { "target-features"="+sme2" "aarch64_pstate_sm_enabled" }
-
 declare <vscale x 8 x half> @llvm.aarch64.sve.fclamp.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float> @llvm.aarch64.sve.fclamp.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.fclamp.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
-
-declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
-declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
-declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>)
-
-declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.fclamp.single.x4.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x half> %d, <vscale x 8 x half> %e, <vscale x 8 x half> %f)
-declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.fclamp.single.x4.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x float> %d, <vscale x 4 x float> %e, <vscale x 4 x float> %f)
-declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.fclamp.single.x4.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x double> %d, <vscale x 2 x double> %e, <vscale x 2 x double> %f)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
index 41b4c77178b5..08b62840ed02 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; FMAXNMQV
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
index 94545a3c6442..feb212621493 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; ORQV
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
index c0e0a5834eec..0eacac2ca63f 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-loads.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+bf16 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming  < %s | FileCheck %s
 
 ; == Normal Multi-Vector Consecutive Loads ==
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
index f15aa2ba0223..393cdeb6f1b3 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-loads.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming  < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;;LD2Q
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
index 8fe0694808c8..67e92d59a42a 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-multivec-stores.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; ST2Q
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
index a592dcd4b8ce..a6f6960507b7 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-pred.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i1> @test_pmov_to_pred_i8(<vscale x 16 x i8> %zn) {
 ; CHECK-LABEL: test_pmov_to_pred_i8:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
index b7f36c651023..6b1ec35f7c00 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-pmov-to-vector.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; Merge
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
index 9e033a978aab..ffcc412b50e2 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-predicate-as-counter.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64 -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 define <vscale x 16 x i1> @pext_b(target("aarch64.svcount") %x) nounwind {
 ; CHECK-LABEL: pext_b:
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
new file mode 100644
index 000000000000..0b5469bacc4f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qcvtn.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py$
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
+
+;
+; SQCVTN
+;
+
+; x2
+define <vscale x 8 x i16 > @multi_vector_qcvtn_x2_s16_s32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
+; CHECK-LABEL: multi_vector_qcvtn_x2_s16_s32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z3.d, z2.d
+; CHECK-NEXT:    mov z2.d, z1.d
+; CHECK-NEXT:    sqcvtn z0.h, { z2.s, z3.s }
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtn.x2.nxv4i32(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)
+  ret <vscale x 8 x i16> %res
+}
+
+;
+; UQCVTN
+;
+
+; x2
+define <vscale x 8 x i16> @multi_vector_qcvtn_x2_u16_u32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1) {
+; CHECK-LABEL: multi_vector_qcvtn_x2_u16_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z3.d, z2.d
+; CHECK-NEXT:    mov z2.d, z1.d
+; CHECK-NEXT:    uqcvtn z0.h, { z2.s, z3.s }
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.uqcvtn.x2.nxv4i32(<vscale x 4 x i32> %zn0, <vscale x 4 x i32> %zn1)
+  ret<vscale x 8 x i16> %res
+}
+
+;
+; SQCVTUN
+;
+
+; x2
+define <vscale x 8 x i16 > @multi_vector_qcvtn_x2_s16_u32(<vscale x 4 x i32> %unused, <vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2) {
+; CHECK-LABEL: multi_vector_qcvtn_x2_s16_u32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z3.d, z2.d
+; CHECK-NEXT:    mov z2.d, z1.d
+; CHECK-NEXT:    sqcvtun z0.h, { z2.s, z3.s }
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtun.x2.nxv4i322(<vscale x 4 x i32> %zn1, <vscale x 4 x i32> %zn2)
+  ret <vscale x 8 x i16> %res
+}
+
+declare <vscale x 8 x i16> @llvm.aarch64.sve.uqcvtn.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtn.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.sqcvtun.x2.nxv4i322(<vscale x 4 x i32>, <vscale x 4 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
index eb6e5a07558a..b85b4799d1cc 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-qrshr.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 ;
 ; S/UQRSHRN x2
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
index 0475601f1907..18e276994964 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-sclamp.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
-define <vscale x 16 x i8> @test_sclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) #0 {
+define <vscale x 16 x i8> @test_sclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
 ; CHECK-LABEL: test_sclamp_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sclamp z0.b, z1.b, z2.b
@@ -12,7 +14,7 @@ define <vscale x 16 x i8> @test_sclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @test_sclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) #0 {
+define <vscale x 8 x i16> @test_sclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
 ; CHECK-LABEL: test_sclamp_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sclamp z0.h, z1.h, z2.h
@@ -21,7 +23,7 @@ define <vscale x 8 x i16> @test_sclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @test_sclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) #0 {
+define <vscale x 4 x i32> @test_sclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
 ; CHECK-LABEL: test_sclamp_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sclamp z0.s, z1.s, z2.s
@@ -30,7 +32,7 @@ define <vscale x 4 x i32> @test_sclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @test_sclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) #0 {
+define <vscale x 2 x i64> @test_sclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
 ; CHECK-LABEL: test_sclamp_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sclamp z0.d, z1.d, z2.d
@@ -39,117 +41,7 @@ define <vscale x 2 x i64> @test_sclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x
   ret <vscale x 2 x i64> %res
 }
 
-define { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_sclamp_single_x2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d) #1 {
-; CHECK-LABEL: test_sclamp_single_x2_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    sclamp { z0.b, z1.b }, z2.b, z3.b
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x2.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d)
-  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
-}
-
-define { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_sclamp_single_x2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d) #1 {
-; CHECK-LABEL: test_sclamp_single_x2_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    sclamp { z0.h, z1.h }, z2.h, z3.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x  8 x i16>, <vscale x  8 x i16> } @llvm.aarch64.sve.sclamp.single.x2.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d)
-  ret { <vscale x 8 x i16>, <vscale x  8 x i16> } %res
-}
-
-define { <vscale x 4 x i32>, <vscale x 4 x i32> } @test_sclamp_single_x2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) #1 {
-; CHECK-LABEL: test_sclamp_single_x2_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    sclamp { z0.s, z1.s }, z2.s, z3.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x2.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
-  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
-}
-
-define { <vscale x 2 x i64>, <vscale x 2 x i64> } @test_sclamp_single_x2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d) #1 {
-; CHECK-LABEL: test_sclamp_single_x2_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    sclamp { z0.d, z1.d }, z2.d, z3.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x2.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d)
-  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
-}
-
-define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_sclamp_single_x4_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f) #1 {
-; CHECK-LABEL: test_sclamp_single_x4_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    sclamp { z0.b - z3.b }, z4.b, z5.b
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
-  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
-}
-
-define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_sclamp_single_x4_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f) #1 {
-; CHECK-LABEL: test_sclamp_single_x4_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    sclamp { z0.h - z3.h }, z4.h, z5.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
-  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res
-}
-
-define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @test_sclamp_single_x4_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f) #1 {
-; CHECK-LABEL: test_sclamp_single_x4_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    sclamp { z0.s - z3.s }, z4.s, z5.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
-  ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res
-}
-
-define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @test_sclamp_single_x4_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f) #1 {
-; CHECK-LABEL: test_sclamp_single_x4_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    sclamp { z0.d - z3.d }, z4.d, z5.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
-  ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
-}
-
-
-attributes #0 = { "target-features"="+sve2p1" }
-attributes #1 = { "target-features"="+sme2" "aarch64_pstate_sm_enabled" }
-
 declare <vscale x 16 x i8> @llvm.aarch64.sve.sclamp.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.sclamp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.sclamp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.sclamp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-
-declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
-declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
-declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
-declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-
-declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
-declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
-declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
-declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
index e7d9c3e6839e..0ff2b53a818e 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-stores.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2,+bf16 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; == Normal Multi-Vector Consecutive Stores ==
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
index a39f6cfa3b4c..2c2539dba189 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-uclamp.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sve,+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming -verify-machineinstrs < %s | FileCheck %s
 
 target triple = "aarch64-linux-gnu"
 
-define <vscale x 16 x i8> @test_uclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) #0 {
+define <vscale x 16 x i8> @test_uclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c) {
 ; CHECK-LABEL: test_uclamp_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uclamp z0.b, z1.b, z2.b
@@ -12,7 +14,7 @@ define <vscale x 16 x i8> @test_uclamp_i8(<vscale x 16 x i8> %a, <vscale x 16 x
   ret <vscale x 16 x i8> %res
 }
 
-define <vscale x 8 x i16> @test_uclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) #0 {
+define <vscale x 8 x i16> @test_uclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c) {
 ; CHECK-LABEL: test_uclamp_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uclamp z0.h, z1.h, z2.h
@@ -21,7 +23,7 @@ define <vscale x 8 x i16> @test_uclamp_i16(<vscale x 8 x i16> %a, <vscale x 8 x
   ret <vscale x 8 x i16> %res
 }
 
-define <vscale x 4 x i32> @test_uclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) #0 {
+define <vscale x 4 x i32> @test_uclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
 ; CHECK-LABEL: test_uclamp_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uclamp z0.s, z1.s, z2.s
@@ -30,7 +32,7 @@ define <vscale x 4 x i32> @test_uclamp_i32(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %res
 }
 
-define <vscale x 2 x i64> @test_uclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) #0 {
+define <vscale x 2 x i64> @test_uclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c) {
 ; CHECK-LABEL: test_uclamp_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    uclamp z0.d, z1.d, z2.d
@@ -39,117 +41,7 @@ define <vscale x 2 x i64> @test_uclamp_i64(<vscale x 2 x i64> %a, <vscale x 2 x
   ret <vscale x 2 x i64> %res
 }
 
-define { <vscale x 16 x i8>, <vscale x 16 x i8> } @test_uclamp_single_x2_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d) #1 {
-; CHECK-LABEL: test_uclamp_single_x2_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    uclamp { z0.b, z1.b }, z2.b, z3.b
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x2.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d)
-  ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res
-}
-
-define { <vscale x 8 x i16>, <vscale x 8 x i16> } @test_uclamp_single_x2_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d) #1 {
-; CHECK-LABEL: test_uclamp_single_x2_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    uclamp { z0.h, z1.h }, z2.h, z3.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x  8 x i16>, <vscale x  8 x i16> } @llvm.aarch64.sve.uclamp.single.x2.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d)
-  ret { <vscale x 8 x i16>, <vscale x  8 x i16> } %res
-}
-
-define { <vscale x 4 x i32>, <vscale x 4 x i32> } @test_uclamp_single_x2_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d) #1 {
-; CHECK-LABEL: test_uclamp_single_x2_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    uclamp { z0.s, z1.s }, z2.s, z3.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x2.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
-  ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res
-}
-
-define { <vscale x 2 x i64>, <vscale x 2 x i64> } @test_uclamp_single_x2_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d) #1 {
-; CHECK-LABEL: test_uclamp_single_x2_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT:    uclamp { z0.d, z1.d }, z2.d, z3.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x2.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d)
-  ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res
-}
-
-define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @test_uclamp_single_x4_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f) #1 {
-; CHECK-LABEL: test_uclamp_single_x4_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    uclamp { z0.b - z3.b }, z4.b, z5.b
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
-  ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res
-}
-
-define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @test_uclamp_single_x4_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f) #1 {
-; CHECK-LABEL: test_uclamp_single_x4_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    uclamp { z0.h - z3.h }, z4.h, z5.h
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
-  ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res
-}
-
-define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @test_uclamp_single_x4_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f) #1 {
-; CHECK-LABEL: test_uclamp_single_x4_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    uclamp { z0.s - z3.s }, z4.s, z5.s
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
-  ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res
-}
-
-define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @test_uclamp_single_x4_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f) #1 {
-; CHECK-LABEL: test_uclamp_single_x4_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def $z0_z1_z2_z3
-; CHECK-NEXT:    uclamp { z0.d - z3.d }, z4.d, z5.d
-; CHECK-NEXT:    ret
-  %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
-  ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res
-}
-
-
-attributes #0 = { "target-features"="+sve2p1" }
-attributes #1 = { "target-features"="+sme2" "aarch64_pstate_sm_enabled" }
-
 declare <vscale x 16 x i8> @llvm.aarch64.sve.uclamp.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.uclamp.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
 declare <vscale x 4 x i32> @llvm.aarch64.sve.uclamp.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
 declare <vscale x 2 x i64> @llvm.aarch64.sve.uclamp.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-
-declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x2.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>)
-declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x2.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
-declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x2.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
-declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x2.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
-
-declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.uclamp.single.x4.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %c, <vscale x 16 x i8> %d, <vscale x 16 x i8> %e, <vscale x 16 x i8> %f)
-declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.uclamp.single.x4.nxv8i16(<vscale x  8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %c, <vscale x 8 x i16> %d, <vscale x 8 x i16> %e, <vscale x 8 x i16> %f)
-declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.uclamp.single.x4.nxv4i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, <vscale x 4 x i32> %e, <vscale x 4 x i32> %f)
-declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.uclamp.single.x4.nxv2i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %c, <vscale x 2 x i64> %d, <vscale x 2 x i64> %e, <vscale x 2 x i64> %f)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
index 7a3ffb020ea5..efe1e161c50f 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pn.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming < %s | FileCheck %s
-
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ;
 ; WHILEGE
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
index ab70f57b4887..db7a3fa46946 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-while-pp.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve,+sme2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme2p1 -force-streaming < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sme,+sve2p1 -force-streaming < %s | FileCheck %s
 
 ; == WHILEGE ==
 
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll b/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll
index 40d4d0ff6014..da83b27ce4d5 100644
--- a/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,SVE
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p1,+bf16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,SME
 
 define void @dupq_i8_256b(ptr %addr) #0 {
 ; CHECK-LABEL: dupq_i8_256b:
@@ -71,13 +72,43 @@ define void @dupq_f16_256b(ptr %addr) #0 {
 }
 
 define void @dupq_bf16_256b(ptr %addr) #0 {
-; CHECK-LABEL: dupq_bf16_256b:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
-; CHECK-NEXT:    dup v0.8h, v0.h[2]
-; CHECK-NEXT:    dup v1.8h, v1.h[2]
-; CHECK-NEXT:    stp q0, q1, [x0]
-; CHECK-NEXT:    ret
+; SVE-LABEL: dupq_bf16_256b:
+; SVE:       // %bb.0:
+; SVE-NEXT:    ldp q0, q1, [x0]
+; SVE-NEXT:    dup v0.8h, v0.h[2]
+; SVE-NEXT:    dup v1.8h, v1.h[2]
+; SVE-NEXT:    stp q0, q1, [x0]
+; SVE-NEXT:    ret
+;
+; SME-LABEL: dupq_bf16_256b:
+; SME:       // %bb.0:
+; SME-NEXT:    ldp q1, q0, [x0]
+; SME-NEXT:    str q0, [sp, #-64]!
+; SME-NEXT:    .cfi_def_cfa_offset 64
+; SME-NEXT:    ldr h0, [sp, #4]
+; SME-NEXT:    str q1, [sp, #32]
+; SME-NEXT:    str h0, [sp, #30]
+; SME-NEXT:    str h0, [sp, #28]
+; SME-NEXT:    str h0, [sp, #26]
+; SME-NEXT:    str h0, [sp, #24]
+; SME-NEXT:    str h0, [sp, #22]
+; SME-NEXT:    str h0, [sp, #20]
+; SME-NEXT:    str h0, [sp, #18]
+; SME-NEXT:    str h0, [sp, #16]
+; SME-NEXT:    ldr h0, [sp, #36]
+; SME-NEXT:    ldr q1, [sp, #16]
+; SME-NEXT:    str h0, [sp, #62]
+; SME-NEXT:    str h0, [sp, #60]
+; SME-NEXT:    str h0, [sp, #58]
+; SME-NEXT:    str h0, [sp, #56]
+; SME-NEXT:    str h0, [sp, #54]
+; SME-NEXT:    str h0, [sp, #52]
+; SME-NEXT:    str h0, [sp, #50]
+; SME-NEXT:    str h0, [sp, #48]
+; SME-NEXT:    ldr q0, [sp, #48]
+; SME-NEXT:    stp q0, q1, [x0]
+; SME-NEXT:    add sp, sp, #64
+; SME-NEXT:    ret
   %load = load <16 x bfloat>, ptr %addr
   %splat.lanes = shufflevector <16 x bfloat> %load, <16 x bfloat> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
                                                                                       i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
@@ -112,4 +143,18 @@ define void @dupq_f64_256b(ptr %addr) #0 {
   ret void
 }
 
-attributes #0 = { noinline vscale_range(2,2) "target-features"="+sve2p1,+bf16" }
+define void @dupq_f32_256b_with_poison(ptr %addr) #0 {
+; CHECK-LABEL: dupq_f32_256b_with_poison:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.s, z0.s[3]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <8 x float>, ptr %addr
+  %splat.lanes = shufflevector <8 x float> %load, <8 x float> poison, <8 x i32> <i32 3, i32 poison, i32 3, i32 3,
+                                                                                 i32 7, i32 7, i32 7, i32 poison>
+  store <8 x float> %splat.lanes, ptr %addr
+  ret void
+}
+
+attributes #0 = { noinline vscale_range(2,2) }
diff --git a/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll b/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll
index 8d863dab7b47..6017070b114a 100644
--- a/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/sve2p2-intrinsics.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p2 < %s | FileCheck %s --check-prefixes=CHECK
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK
 
 ;
@@ -26,6 +27,26 @@ define <vscale x 8 x i16> @compact_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
   ret <vscale x 8 x i16> %out
 }
 
+define <vscale x 4 x i32> @compact_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: compact_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.s, p0, z0.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.compact.nxv4i32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x i32> %a)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @compact_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: compact_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.d, p0, z0.d
+; CHECK-NEXT:    ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.compact.nxv2i64(<vscale x 2 x i1> %pg,
+                                                                   <vscale x 2 x i64> %a)
+  ret <vscale x 2 x i64> %out
+}
+
 define <vscale x 8 x half> @compact_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
 ; CHECK-LABEL: compact_f16:
 ; CHECK:       // %bb.0:
@@ -36,6 +57,26 @@ define <vscale x 8 x half> @compact_f16(<vscale x 8 x i1> %pg, <vscale x 8 x hal
   ret <vscale x 8 x half> %out
 }
 
+define <vscale x 4 x float> @compact_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: compact_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.s, p0, z0.s
+; CHECK-NEXT:    ret
+  %out = call <vscale x 4 x float> @llvm.aarch64.sve.compact.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                     <vscale x 4 x float> %a)
+  ret <vscale x 4 x float> %out
+}
+
+define <vscale x 2 x double> @compact_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: compact_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    compact z0.d, p0, z0.d
+; CHECK-NEXT:    ret
+  %out = call <vscale x 2 x double> @llvm.aarch64.sve.compact.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                      <vscale x 2 x double> %a)
+  ret <vscale x 2 x double> %out
+}
+
 define <vscale x 8 x bfloat> @compact_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x bfloat> %a) {
 ; CHECK-LABEL: compact_bf16:
 ; CHECK:       // %bb.0:
@@ -48,5 +89,9 @@ define <vscale x 8 x bfloat> @compact_bf16(<vscale x 8 x i1> %pg, <vscale x 8 x
 
 declare <vscale x 16 x i8> @llvm.aarch64.sve.compact.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
 declare <vscale x 8 x i16> @llvm.aarch64.sve.compact.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.compact.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.compact.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
 declare <vscale x 8 x half> @llvm.aarch64.sve.compact.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x float> @llvm.aarch64.sve.compact.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.aarch64.sve.compact.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
 declare <vscale x 8 x bfloat> @llvm.aarch64.sve.compact.nxv8bf16(<vscale x 8 x i1>, <vscale x 8 x bfloat>)
diff --git a/llvm/test/CodeGen/AArch64/switch-cases-to-branch-and.ll b/llvm/test/CodeGen/AArch64/switch-cases-to-branch-and.ll
index 4d36a003cbfe..e99ba4843c45 100644
--- a/llvm/test/CodeGen/AArch64/switch-cases-to-branch-and.ll
+++ b/llvm/test/CodeGen/AArch64/switch-cases-to-branch-and.ll
@@ -569,8 +569,7 @@ define void @test_successor_with_loop_phi(ptr %A, ptr %B) {
 ; CHECK-NEXT:    ldr w8, [x0]
 ; CHECK-NEXT:    str wzr, [x0]
 ; CHECK-NEXT:    mov x0, x1
-; CHECK-NEXT:    orr w8, w8, #0x4
-; CHECK-NEXT:    cmp w8, #4
+; CHECK-NEXT:    tst w8, #0xfffffffb
 ; CHECK-NEXT:    b.eq LBB7_1
 ; CHECK-NEXT:  ; %bb.2: ; %exit
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index b5d64112db72..aa0a163b96ac 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -31,12 +31,12 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
 ; CHECK-NEXT:    add x13, x13, #32
 ; CHECK-NEXT:    fcmgt v3.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    fcmgt v4.4s, v2.4s, v0.4s
-; CHECK-NEXT:    fcmlt v5.4s, v1.4s, #0.0
-; CHECK-NEXT:    fcmlt v6.4s, v2.4s, #0.0
-; CHECK-NEXT:    bit v1.16b, v0.16b, v3.16b
-; CHECK-NEXT:    bit v2.16b, v0.16b, v4.16b
-; CHECK-NEXT:    bic v1.16b, v1.16b, v5.16b
-; CHECK-NEXT:    bic v2.16b, v2.16b, v6.16b
+; CHECK-NEXT:    bsl v3.16b, v0.16b, v1.16b
+; CHECK-NEXT:    bsl v4.16b, v0.16b, v2.16b
+; CHECK-NEXT:    fcmlt v1.4s, v1.4s, #0.0
+; CHECK-NEXT:    fcmlt v2.4s, v2.4s, #0.0
+; CHECK-NEXT:    bic v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bic v2.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
 ; CHECK-NEXT:    xtn v1.4h, v1.4s
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
index 830f1efbdf67..5bd9ec2afdad 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-propagate-sd-flags.ll
@@ -12,7 +12,7 @@
 ; CHECK: Legally typed node: [[VTWOA]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legalizing node: [[VTWOB:t.*]]: v2f64 = BUILD_VECTOR
 ; CHECK: Legally typed node: [[VTWOB]]: v2f64 = BUILD_VECTOR
-; CHECK: Legalizing node: t30: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
+; CHECK: Legalizing node: t31: v2f64 = fmaxnum nnan reassoc [[VTWOB]], [[VTWOA]]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/vselect-constants.ll b/llvm/test/CodeGen/AArch64/vselect-constants.ll
index a7cf5ece5d27..fe125c9626ea 100644
--- a/llvm/test/CodeGen/AArch64/vselect-constants.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-constants.ll
@@ -146,10 +146,8 @@ define <4 x i32> @cmp_sel_0_or_minus1_vec(<4 x i32> %x, <4 x i32> %y) {
 define <4 x i32> @sel_1_or_0_vec(<4 x i1> %cond) {
 ; CHECK-LABEL: sel_1_or_0_vec:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    shl v0.4s, v0.4s, #31
-; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %add = select <4 x i1> %cond, <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll
index 76b7f3d9dfc0..4f2b9c5a6266 100644
--- a/llvm/test/CodeGen/AArch64/vselect-ext.ll
+++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll
@@ -12,10 +12,10 @@ define <16 x i32> @no_existing_zext(<16 x i8> %a, <16 x i32> %op) {
 ; CHECK-NEXT:    sshll.4s v6, v5, #0
 ; CHECK-NEXT:    sshll.4s v7, v0, #0
 ; CHECK-NEXT:    sshll2.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v4, v4, v16
-; CHECK-NEXT:    and.16b v0, v1, v6
-; CHECK-NEXT:    and.16b v1, v2, v5
-; CHECK-NEXT:    and.16b v2, v3, v7
+; CHECK-NEXT:    and.16b v4, v16, v4
+; CHECK-NEXT:    and.16b v0, v6, v1
+; CHECK-NEXT:    and.16b v1, v5, v2
+; CHECK-NEXT:    and.16b v2, v7, v3
 ; CHECK-NEXT:    mov.16b v3, v4
 ; CHECK-NEXT:    ret
 entry:
@@ -40,10 +40,10 @@ define <16 x i32> @second_compare_operand_not_splat(<16 x i8> %a, <16 x i8> %b)
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -69,10 +69,10 @@ define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select(<16 x i8> %a) {
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -97,10 +97,10 @@ define <8 x i64> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i64(<8 x i8>
 ; CHECK-NEXT:    cmhi.2d v7, v1, v2
 ; CHECK-NEXT:    cmhi.2d v6, v5, v2
 ; CHECK-NEXT:    cmhi.2d v2, v4, v2
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v1, v1, v7
-; CHECK-NEXT:    and.16b v3, v4, v2
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v7, v1
+; CHECK-NEXT:    and.16b v3, v2, v4
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i8> %a to <8 x i64>
   %cmp = icmp ugt <8 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -123,10 +123,10 @@ define <16 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v16i32(<16 x i
 ; CHECK-NEXT:    cmhi.4s v7, v2, v1
 ; CHECK-NEXT:    cmhi.4s v6, v5, v1
 ; CHECK-NEXT:    cmhi.4s v1, v4, v1
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v3, v4, v1
-; CHECK-NEXT:    and.16b v1, v2, v7
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v3, v1, v4
+; CHECK-NEXT:    and.16b v1, v7, v2
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
   %ext = zext <16 x i8> %a to <16 x i32>
   %cmp = icmp ugt <16 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -143,8 +143,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32(<8 x i8>
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i8> %a to <8 x i32>
   %cmp = icmp ugt <8 x i8> %a, <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
@@ -160,8 +160,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32_2(<8 x i1
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i16> %a to <8 x i32>
   %cmp = icmp ugt <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -179,8 +179,8 @@ define <8 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v8i32_from_v8i1
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v3, v0, v1
 ; CHECK-NEXT:    cmhi.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i15> %a to <8 x i32>
   %cmp = icmp ugt <8 x i15> %a, <i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10>
@@ -197,8 +197,8 @@ define <7 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v7i32(<7 x i16>
 ; CHECK-NEXT:    ushll2.4s v0, v0, #0
 ; CHECK-NEXT:    sshll.4s v3, v1, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v2, v2, v3
-; CHECK-NEXT:    and.16b v0, v0, v1
+; CHECK-NEXT:    and.16b v2, v3, v2
+; CHECK-NEXT:    and.16b v0, v1, v0
 ; CHECK-NEXT:    mov.s w1, v2[1]
 ; CHECK-NEXT:    mov.s w2, v2[2]
 ; CHECK-NEXT:    mov.s w3, v2[3]
@@ -244,7 +244,7 @@ define <4 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v4i32(<4 x i16>
 ; CHECK-NEXT:    movi.4s v1, #10
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmhi.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v0, v1
+; CHECK-NEXT:    and.16b v0, v1, v0
 ; CHECK-NEXT:    ret
   %ext = zext <4 x i16> %a to <4 x i32>
   %cmp = icmp ugt <4 x i16> %a, <i16 10, i16 10, i16 10, i16 10>
@@ -259,7 +259,7 @@ define <2 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_v2i32(<2 x i16>
 ; CHECK-NEXT:    movi.2s v2, #10
 ; CHECK-NEXT:    and.8b v0, v0, v1
 ; CHECK-NEXT:    cmhi.2s v1, v0, v2
-; CHECK-NEXT:    and.8b v0, v0, v1
+; CHECK-NEXT:    and.8b v0, v1, v0
 ; CHECK-NEXT:    ret
   %ext = zext <2 x i16> %a to <2 x i32>
   %cmp = icmp ugt <2 x i16> %a, <i16 10, i16 10>
@@ -275,8 +275,8 @@ define <8 x i32> @same_zext_used_in_cmp_eq_and_select_v8i32(<8 x i16> %a) {
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i16> %a to <8 x i32>
   %cmp = icmp eq <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -293,8 +293,8 @@ define <8 x i32> @same_zext_used_in_cmp_eq_and_select_v8i32_from_v8i13(<8 x i13>
 ; CHECK-NEXT:    ushll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = zext <8 x i13> %a to <8 x i32>
   %cmp = icmp eq <8 x i13> %a, <i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10>
@@ -358,16 +358,16 @@ define <16 x i32> @same_zext_used_in_cmp_unsigned_pred_and_select_other_use(<16
 ; CHECK-NEXT:    and.16b v6, v6, v26
 ; CHECK-NEXT:    sshll2.2d v26, v16, #0
 ; CHECK-NEXT:    and.16b v27, v4, v27
-; CHECK-NEXT:    and.16b v4, v0, v18
-; CHECK-NEXT:    and.16b v0, v24, v16
+; CHECK-NEXT:    and.16b v4, v18, v0
+; CHECK-NEXT:    and.16b v0, v16, v24
 ; CHECK-NEXT:    stp q7, q21, [x0, #96]
 ; CHECK-NEXT:    sshll.2d v21, v16, #0
 ; CHECK-NEXT:    and.16b v5, v5, v22
 ; CHECK-NEXT:    and.16b v7, v3, v23
-; CHECK-NEXT:    and.16b v3, v19, v20
+; CHECK-NEXT:    and.16b v3, v20, v19
 ; CHECK-NEXT:    stp q5, q6, [x0, #64]
 ; CHECK-NEXT:    and.16b v6, v2, v26
-; CHECK-NEXT:    and.16b v2, v25, v17
+; CHECK-NEXT:    and.16b v2, v17, v25
 ; CHECK-NEXT:    and.16b v5, v1, v21
 ; CHECK-NEXT:    mov.16b v1, v3
 ; CHECK-NEXT:    mov.16b v3, v4
@@ -397,10 +397,10 @@ define <16 x i32> @same_sext_used_in_cmp_signed_pred_and_select_v16i32(<16 x i8>
 ; CHECK-NEXT:    cmgt.4s v7, v2, v1
 ; CHECK-NEXT:    cmgt.4s v6, v5, v1
 ; CHECK-NEXT:    cmgt.4s v1, v4, v1
-; CHECK-NEXT:    and.16b v0, v3, v0
-; CHECK-NEXT:    and.16b v3, v4, v1
-; CHECK-NEXT:    and.16b v1, v2, v7
-; CHECK-NEXT:    and.16b v2, v5, v6
+; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v3, v1, v4
+; CHECK-NEXT:    and.16b v1, v7, v2
+; CHECK-NEXT:    and.16b v2, v6, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = sext <16 x i8> %a to <16 x i32>
@@ -417,8 +417,8 @@ define <8 x i32> @same_sext_used_in_cmp_eq_and_select_v8i32(<8 x i16> %a) {
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    cmeq.4s v3, v0, v1
 ; CHECK-NEXT:    cmeq.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i16> %a to <8 x i32>
   %cmp = icmp eq <8 x i16> %a, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
@@ -438,8 +438,8 @@ define <8 x i32> @same_sext_used_in_cmp_eq_and_select_v8i32_from_v8i13(<8 x i13>
 ; CHECK-NEXT:    sshr.4s v2, v2, #19
 ; CHECK-NEXT:    cmeq.4s v3, v2, v1
 ; CHECK-NEXT:    cmeq.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v2, v3
+; CHECK-NEXT:    and.16b v1, v1, v0
+; CHECK-NEXT:    and.16b v0, v3, v2
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i13> %a to <8 x i32>
   %cmp = icmp eq <8 x i13> %a, <i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10, i13 10>
@@ -480,8 +480,8 @@ define <8 x i32> @same_sext_used_in_cmp_signed_pred_and_select_v8i32(<8 x i16> %
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    cmgt.4s v3, v0, v1
 ; CHECK-NEXT:    cmgt.4s v1, v2, v1
-; CHECK-NEXT:    and.16b v1, v2, v1
-; CHECK-NEXT:    and.16b v0, v0, v3
+; CHECK-NEXT:    and.16b v1, v1, v2
+; CHECK-NEXT:    and.16b v0, v3, v0
 ; CHECK-NEXT:    ret
 entry:
   %ext = sext <8 x i16> %a to <8 x i32>
@@ -502,8 +502,8 @@ define <8 x i32> @same_sext_used_in_cmp_unsigned_pred_and_select_v8i32_from_v8i1
 ; CHECK-NEXT:    sshr.4s v2, v2, #17
 ; CHECK-NEXT:    cmge.4s v3, v2, v1
 ; CHECK-NEXT:    cmge.4s v1, v0, v1
-; CHECK-NEXT:    and.16b v1, v0, v1
-; CHECK-NEXT:    and.16b v0, v2, v3
+; CHECK-NEXT:    and.16b v1, v1, v0
+; CHECK-NEXT:    and.16b v0, v3, v2
 ; CHECK-NEXT:    ret
   %ext = sext <8 x i15> %a to <8 x i32>
   %cmp = icmp sge <8 x i15> %a, <i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10, i15 10>
@@ -524,11 +524,11 @@ define <16 x i32> @same_sext_used_in_cmp_unsigned_pred_and_select(<16 x i8> %a)
 ; CHECK-NEXT:    ext.16b v5, v0, v0, #8
 ; CHECK-NEXT:    ext.16b v6, v3, v3, #8
 ; CHECK-NEXT:    ext.16b v7, v1, v1, #8
-; CHECK-NEXT:    and.8b v2, v2, v3
-; CHECK-NEXT:    and.8b v1, v0, v1
+; CHECK-NEXT:    and.8b v2, v3, v2
+; CHECK-NEXT:    and.8b v1, v1, v0
 ; CHECK-NEXT:    sshll.4s v0, v2, #0
-; CHECK-NEXT:    and.8b v3, v5, v7
-; CHECK-NEXT:    and.8b v4, v4, v6
+; CHECK-NEXT:    and.8b v3, v7, v5
+; CHECK-NEXT:    and.8b v4, v6, v4
 ; CHECK-NEXT:    sshll.4s v2, v1, #0
 ; CHECK-NEXT:    sshll.4s v3, v3, #0
 ; CHECK-NEXT:    sshll.4s v1, v4, #0
@@ -556,10 +556,10 @@ define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_u
 ; CHECK-NEXT:    sshll.4s v7, v1, #0
 ; CHECK-NEXT:    sshll2.4s v16, v3, #0
 ; CHECK-NEXT:    sshll2.4s v1, v1, #0
-; CHECK-NEXT:    and.16b v0, v4, v0
-; CHECK-NEXT:    and.16b v3, v6, v1
-; CHECK-NEXT:    and.16b v1, v2, v16
-; CHECK-NEXT:    and.16b v2, v5, v7
+; CHECK-NEXT:    and.16b v0, v0, v4
+; CHECK-NEXT:    and.16b v3, v1, v6
+; CHECK-NEXT:    and.16b v1, v16, v2
+; CHECK-NEXT:    and.16b v2, v7, v5
 ; CHECK-NEXT:    ret
 entry:
   %ext = zext <16 x i8> %a to <16 x i32>
@@ -604,10 +604,10 @@ define void @extension_in_loop_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB24_1
@@ -674,10 +674,10 @@ define void @extension_in_loop_as_shuffle_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB25_1
@@ -745,10 +745,10 @@ define void @shuffle_in_loop_is_no_extend_v16i8_to_v16i32(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    sshll.4s v6, v6, #0
 ; CHECK-NEXT:    sshll2.4s v19, v5, #0
 ; CHECK-NEXT:    sshll.4s v5, v5, #0
-; CHECK-NEXT:    and.16b v7, v7, v17
-; CHECK-NEXT:    and.16b v6, v16, v6
-; CHECK-NEXT:    and.16b v16, v18, v19
-; CHECK-NEXT:    and.16b v4, v4, v5
+; CHECK-NEXT:    and.16b v7, v17, v7
+; CHECK-NEXT:    and.16b v6, v6, v16
+; CHECK-NEXT:    and.16b v16, v19, v18
+; CHECK-NEXT:    and.16b v4, v5, v4
 ; CHECK-NEXT:    stp q6, q7, [x1, #32]
 ; CHECK-NEXT:    stp q4, q16, [x1], #64
 ; CHECK-NEXT:    b.ne LBB26_1
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll
index ba8a606b331e..618887f418e7 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-abs-neg.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-counts-not.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-counts-not.ll
index b904634d0b76..c7a928fd3746 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-counts-not.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-counts-not.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+bf16,+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+bf16,+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+bf16,+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+bf16,+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+bf16,+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+bf16,+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll
index 20a5475706c9..d56e5e989e75 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-ext.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll
index 3ce8376394b2..90b9b4fd415f 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvt-bfcvt.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve,+bf16    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2,+bf16 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2,+bf16 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme,+bf16    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2,+bf16 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2,+bf16 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll
index 114f2163d94f..a4b11acb8d4c 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtlt-fcvtx.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2   < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme2   -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll
index e03e7ca14871..55685a5595ab 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-fcvtzsu.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll
index 79d66b0e0482..18d75153d51f 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-flogb.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2   < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme2   -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll
index 25252d222db2..9c8f48dbc10b 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-frint-frecpx-fsqrt.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll
index fb95047a4120..8ca2c8d1ca24 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-rev.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2p1 < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll
index 50c73ae198ab..b2a9aafc30c1 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-urecpe-ursqrte-sqabs-sqneg.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve2   < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
-; RUN: llc -mattr=+sme    --force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
-; RUN: llc -mattr=+sme2p2 --force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
+; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AArch64/zeroing-forms-uscvtf.ll b/llvm/test/CodeGen/AArch64/zeroing-forms-uscvtf.ll
index e5447071cbfd..2064d384b847 100644
--- a/llvm/test/CodeGen/AArch64/zeroing-forms-uscvtf.ll
+++ b/llvm/test/CodeGen/AArch64/zeroing-forms-uscvtf.ll
@@ -1,9 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sve    < %s | FileCheck %s -check-prefixes=CHECK,SVE
 ; RUN: llc -mattr=+sve2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sve,+sme2p2 < %s | FileCheck %s -check-prefix CHECK-2p2
 
 ; RUN: llc -mattr=+sme    -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,STREAMING-SVE
 ; RUN: llc -mattr=+sme2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
+; RUN: llc -mattr=+sme,+sve2p2 -force-streaming < %s | FileCheck %s -check-prefix CHECK-2p2
 
 target triple = "aarch64-linux"
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index cbadd1eb431f..788a4e6fb214 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -3070,6 +3070,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v3
 ; GFX12-NEXT:    s_endpgm
@@ -4161,6 +4162,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.illegal.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.illegal.ll
index 1d65096c2e5c..3390ad2cf2a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.illegal.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.illegal.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -global-isel -global-isel-abort=1 -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -global-isel -global-isel-abort=1 -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERR %s
 
 ; FIXME: Should produce context error for each one
 ; ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(p5) = G_GLOBAL_VALUE @external_private (in function: fn_external_private)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
index 4c3f4d9b06ed..f30b7c368582 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
@@ -100,7 +100,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX11-LABEL: name: ashr_s16_s16_vs
     ; GFX11: liveins: $sgpr0, $vgpr0
@@ -193,7 +193,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX11-LABEL: name: ashr_s16_s16_vv
     ; GFX11: liveins: $vgpr0, $vgpr1
@@ -238,7 +238,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
     ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_]], [[V_ASHRREV_I16_e64_]], implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
@@ -292,7 +292,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GFX10-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
     ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_1]], [[V_ASHRREV_I16_e64_]], implicit $exec
@@ -442,7 +442,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX11-LABEL: name: ashr_s16_s16_sv
     ; GFX11: liveins: $sgpr0, $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
index 4769b5f77e3b..3dd3fab73587 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
@@ -98,7 +98,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX11-LABEL: name: lshr_s16_s16_vs
     ; GFX11: liveins: $sgpr0, $vgpr0
@@ -191,7 +191,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX11-LABEL: name: lshr_s16_s16_vv
     ; GFX11: liveins: $vgpr0, $vgpr1
@@ -236,7 +236,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
     ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_]], [[V_LSHRREV_B16_e64_]], implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
@@ -290,7 +290,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GFX10-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
     ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_1]], [[V_LSHRREV_B16_e64_]], implicit $exec
@@ -440,7 +440,7 @@ body: |
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX11-LABEL: name: lshr_s16_s16_sv
     ; GFX11: liveins: $sgpr0, $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-smed3.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-smed3.s16.mir
index 19143c52b3f4..c60ae60a1af1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-smed3.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-smed3.s16.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX8 %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefixes=GFX11 %s
 
 ---
@@ -34,6 +34,15 @@ body: |
     ; GFX9-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]]
     ;
+    ; GFX10-LABEL: name: smed3_s16_vvv
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]]
+    ;
     ; GFX11-LABEL: name: smed3_s16_vvv
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -88,6 +97,16 @@ body: |
     ; GFX9-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MAX_I16_e64_]]
     ;
+    ; GFX10-LABEL: name: smed3_s16_vvv_multiuse0
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MAX_I16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MAX_I16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MAX_I16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: smed3_s16_vvv_multiuse0
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -143,6 +162,16 @@ body: |
     ; GFX9-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MIN_I16_e64_]]
     ;
+    ; GFX10-LABEL: name: smed3_s16_vvv_multiuse1
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MIN_I16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MIN_I16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MIN_I16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: smed3_s16_vvv_multiuse1
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -199,6 +228,17 @@ body: |
     ; GFX9-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MAX_I16_e64_]]
     ;
+    ; GFX10-LABEL: name: smed3_s16_vvv_multiuse2
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MIN_I16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MIN_I16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MAX_I16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MAX_I16_opsel_e64 0, [[V_MIN_I16_opsel_e64_]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_I16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_I16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_I16_e64_]], implicit [[V_MAX_I16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: smed3_s16_vvv_multiuse2
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-umed3.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-umed3.s16.mir
index b7f48d34b8f9..a51bc9339632 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-umed3.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-umed3.s16.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX8 %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX9 %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX10 %s
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefixes=GFX11 %s
 
 ---
@@ -34,6 +34,15 @@ body: |
     ; GFX9-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]]
     ;
+    ; GFX10-LABEL: name: umed3_s16_vvv
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]]
+    ;
     ; GFX11-LABEL: name: umed3_s16_vvv
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -88,6 +97,16 @@ body: |
     ; GFX9-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MAX_U16_e64_]]
     ;
+    ; GFX10-LABEL: name: umed3_s16_vvv_multiuse0
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MAX_U16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MAX_U16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MAX_U16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: umed3_s16_vvv_multiuse0
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -143,6 +162,16 @@ body: |
     ; GFX9-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MIN_U16_e64_]]
     ;
+    ; GFX10-LABEL: name: umed3_s16_vvv_multiuse1
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MIN_U16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MIN_U16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MIN_U16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: umed3_s16_vvv_multiuse1
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
@@ -199,6 +228,17 @@ body: |
     ; GFX9-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MAX_U16_e64_]]
     ;
+    ; GFX10-LABEL: name: umed3_s16_vvv_multiuse2
+    ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX10-NEXT: {{  $}}
+    ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX10-NEXT: [[V_MIN_U16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MIN_U16_opsel_e64 0, [[COPY]], 0, [[COPY1]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MAX_U16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_MAX_U16_opsel_e64 0, [[V_MIN_U16_opsel_e64_]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: [[V_MED3_U16_e64_:%[0-9]+]]:vgpr_32 = V_MED3_U16_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_MED3_U16_e64_]], implicit [[V_MAX_U16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: umed3_s16_vvv_multiuse2
     ; GFX11: liveins: $vgpr0, $vgpr1, $vgpr2
     ; GFX11-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
index 73f164ed10df..b229981c2484 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
@@ -36,6 +36,7 @@ body: |
     ; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX8-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX9-LABEL: name: shl_s16_s16_ss
     ; GFX9: liveins: $sgpr0, $sgpr1
     ; GFX9-NEXT: {{  $}}
@@ -45,6 +46,7 @@ body: |
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX10-LABEL: name: shl_s16_s16_ss
     ; GFX10: liveins: $sgpr0, $sgpr1
     ; GFX10-NEXT: {{  $}}
@@ -54,6 +56,7 @@ body: |
     ; GFX10-NEXT: [[TRUNC1:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY1]](s32)
     ; GFX10-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX11-LABEL: name: shl_s16_s16_ss
     ; GFX11: liveins: $sgpr0, $sgpr1
     ; GFX11-NEXT: {{  $}}
@@ -86,6 +89,7 @@ body: |
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX8-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX9-LABEL: name: shl_s16_s16_vs
     ; GFX9: liveins: $sgpr0, $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -93,13 +97,15 @@ body: |
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX9-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX10-LABEL: name: shl_s16_s16_vs
     ; GFX10: liveins: $sgpr0, $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
-    ; GFX10-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ; GFX10-NEXT: [[V_LSHLREV_B16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: shl_s16_s16_vs
     ; GFX11: liveins: $sgpr0, $vgpr0
     ; GFX11-NEXT: {{  $}}
@@ -132,6 +138,7 @@ body: |
     ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX9-LABEL: name: shl_s16_s32_vv
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9-NEXT: {{  $}}
@@ -140,6 +147,7 @@ body: |
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX10-LABEL: name: shl_s16_s32_vv
     ; GFX10: liveins: $vgpr0, $vgpr1
     ; GFX10-NEXT: {{  $}}
@@ -148,6 +156,7 @@ body: |
     ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX10-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX11-LABEL: name: shl_s16_s32_vv
     ; GFX11: liveins: $vgpr0, $vgpr1
     ; GFX11-NEXT: {{  $}}
@@ -179,6 +188,7 @@ body: |
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX8-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX9-LABEL: name: shl_s16_s16_vv
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9-NEXT: {{  $}}
@@ -186,13 +196,15 @@ body: |
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX9-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX10-LABEL: name: shl_s16_s16_vv
     ; GFX10: liveins: $vgpr0, $vgpr1
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ; GFX10-NEXT: [[V_LSHLREV_B16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: shl_s16_s16_vv
     ; GFX11: liveins: $vgpr0, $vgpr1
     ; GFX11-NEXT: {{  $}}
@@ -224,6 +236,7 @@ body: |
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX8-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX9-LABEL: name: shl_s16_s16_vv_zext_to_s32
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9-NEXT: {{  $}}
@@ -231,15 +244,17 @@ body: |
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX9-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX10-LABEL: name: shl_s16_s16_vv_zext_to_s32
     ; GFX10: liveins: $vgpr0, $vgpr1
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHLREV_B16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
-    ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_]], [[V_LSHLREV_B16_e64_]], implicit $exec
+    ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_]], [[V_LSHLREV_B16_opsel_e64_]], implicit $exec
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
+    ;
     ; GFX11-LABEL: name: shl_s16_s16_vv_zext_to_s32
     ; GFX11: liveins: $vgpr0, $vgpr1
     ; GFX11-NEXT: {{  $}}
@@ -276,6 +291,7 @@ body: |
     ; GFX8-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX8-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_LSHLREV_B16_e64_]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    ;
     ; GFX9-LABEL: name: shl_s16_vv_zext_to_s64
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9-NEXT: {{  $}}
@@ -285,18 +301,20 @@ body: |
     ; GFX9-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_LSHLREV_B16_e64_]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    ;
     ; GFX10-LABEL: name: shl_s16_vv_zext_to_s64
     ; GFX10: liveins: $vgpr0, $vgpr1
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX10-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
+    ; GFX10-NEXT: [[V_LSHLREV_B16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
     ; GFX10-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GFX10-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
-    ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_1]], [[V_LSHLREV_B16_e64_]], implicit $exec
+    ; GFX10-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[S_MOV_B32_1]], [[V_LSHLREV_B16_opsel_e64_]], implicit $exec
     ; GFX10-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
     ; GFX10-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_AND_B32_e64_]], %subreg.sub0, [[COPY2]], %subreg.sub1
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    ;
     ; GFX11-LABEL: name: shl_s16_vv_zext_to_s64
     ; GFX11: liveins: $vgpr0, $vgpr1
     ; GFX11-NEXT: {{  $}}
@@ -335,6 +353,7 @@ body: |
     ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX9-LABEL: name: shl_s16_s32_ss
     ; GFX9: liveins: $sgpr0, $sgpr1
     ; GFX9-NEXT: {{  $}}
@@ -343,6 +362,7 @@ body: |
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX10-LABEL: name: shl_s16_s32_ss
     ; GFX10: liveins: $sgpr0, $sgpr1
     ; GFX10-NEXT: {{  $}}
@@ -351,6 +371,7 @@ body: |
     ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX10-NEXT: [[SHL:%[0-9]+]]:sgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX11-LABEL: name: shl_s16_s32_ss
     ; GFX11: liveins: $sgpr0, $sgpr1
     ; GFX11-NEXT: {{  $}}
@@ -382,6 +403,7 @@ body: |
     ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX9-LABEL: name: shl_s16_s32_sv
     ; GFX9: liveins: $sgpr0, $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -390,6 +412,7 @@ body: |
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX10-LABEL: name: shl_s16_s32_sv
     ; GFX10: liveins: $sgpr0, $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -398,6 +421,7 @@ body: |
     ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:sgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX10-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX11-LABEL: name: shl_s16_s32_sv
     ; GFX11: liveins: $sgpr0, $vgpr0
     ; GFX11-NEXT: {{  $}}
@@ -428,6 +452,7 @@ body: |
     ; GFX8-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX8-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX9-LABEL: name: shl_s16_s16_sv
     ; GFX9: liveins: $sgpr0, $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -435,13 +460,15 @@ body: |
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ;
     ; GFX10-LABEL: name: shl_s16_s16_sv
     ; GFX10: liveins: $sgpr0, $vgpr0
     ; GFX10-NEXT: {{  $}}
     ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX10-NEXT: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
-    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
+    ; GFX10-NEXT: [[V_LSHLREV_B16_opsel_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_opsel_e64 0, [[COPY1]], 0, [[COPY]], 0, 0, implicit $exec
+    ; GFX10-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B16_opsel_e64_]]
+    ;
     ; GFX11-LABEL: name: shl_s16_s16_sv
     ; GFX11: liveins: $sgpr0, $vgpr0
     ; GFX11-NEXT: {{  $}}
@@ -473,6 +500,7 @@ body: |
     ; GFX8-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX8-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX8-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX9-LABEL: name: shl_s16_s32_vs
     ; GFX9: liveins: $sgpr0, $vgpr0
     ; GFX9-NEXT: {{  $}}
@@ -481,6 +509,7 @@ body: |
     ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX9-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX9-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX10-LABEL: name: shl_s16_s32_vs
     ; GFX10: liveins: $sgpr0, $vgpr0
     ; GFX10-NEXT: {{  $}}
@@ -489,6 +518,7 @@ body: |
     ; GFX10-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
     ; GFX10-NEXT: [[SHL:%[0-9]+]]:vgpr(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
     ; GFX10-NEXT: S_ENDPGM 0, implicit [[SHL]](s16)
+    ;
     ; GFX11-LABEL: name: shl_s16_s32_vs
     ; GFX11: liveins: $sgpr0, $vgpr0
     ; GFX11-NEXT: {{  $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-stacksave-stackrestore.invalid.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-stacksave-stackrestore.invalid.mir
index f2c44afd57fa..b5f17dea5bb6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-stacksave-stackrestore.invalid.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-stacksave-stackrestore.invalid.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass=instruction-select -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERR %s
+# RUN: not llc -mtriple=amdgcn -mcpu=gfx1030 -run-pass=instruction-select -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERR %s
 
 # ERR: LLVM ERROR: cannot select: G_STACKRESTORE %{{[0-9]+}}:vgpr(p5) (in function: stackrestore_waveaddress_vgpr)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir
index e90c3e14158b..743aa503d607 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o - %s 2>&1| FileCheck -check-prefix=ERROR %s
+# RUN: not llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o - %s 2>&1| FileCheck -check-prefix=ERROR %s
 
 # This needs to be expanded into a cmpxchg loop.
 # TODO: Will AtomicExpand still do this?
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir
index 80597c7fcfad..22970d311a34 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -o - %s | FileCheck %s
-# RUN: not --crash llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERROR %s
+# RUN: not llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERROR %s
 
 # ERROR: LLVM ERROR: unable to legalize instruction: %2:_(s32) = G_ATOMICRMW_XCHG %0:_(p0), %1:_ :: (load store seq_cst (s32)) (in function: atomicrmw_xchg_flat_i32)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-jump-table.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-jump-table.mir
index b54c368fbeee..b43c18ee2aa3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-jump-table.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-jump-table.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: not --crash llc -mtriple=amdgcn -run-pass=legalizer -o /dev/null %s 2>&1  | FileCheck %s
+# RUN: not llc -mtriple=amdgcn -run-pass=legalizer -o /dev/null %s 2>&1  | FileCheck %s
 
 # CHECK: LLVM ERROR: unable to legalize instruction: %3:_(p0) = G_JUMP_TABLE %jump-table.0 (in function: jt_test)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
index 7c04fe315918..91b34ae21b48 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -o - %s 2>&1 | FileCheck -check-prefix=GFX11-ERR %s
+; RUN: not llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -o - %s 2>&1 | FileCheck -check-prefix=GFX11-ERR %s
 
 ; GFX11-ERR: LLVM ERROR: cannot select: {{.*}} = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.gather4
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index b905e548846a..0bcf52aedc1f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1030 %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1013 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1013 %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX11 %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s -o /dev/null 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -global-isel -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s -o /dev/null 2>&1 | FileCheck -check-prefix=ERR %s
 
 ; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(uint node_ptr, float ray_extent, float3 ray_origin, float3 ray_dir, float3 ray_inv_dir, uint4 texture_descr)
 ; uint4 llvm.amdgcn.image.bvh.intersect.ray.i32.v4f16(uint node_ptr, float ray_extent, float3 ray_origin, half3 ray_dir, half3 ray_inv_dir, uint4 texture_descr)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
index 328849a9cea2..d3cc70a28e3f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.fadd-with-ret.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
-; RUN: not --crash llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
+; RUN: not llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
 
 declare float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float, <4 x i32>, i32, i32, i32 immarg)
 declare <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
index dadd971396a3..ac23cbfd6e62 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
-; RUN: not --crash llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
+; RUN: not llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
 
 declare float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
 declare <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
index 95aa339a05ec..1b21af854839 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.fadd-with-ret.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
-; RUN: not --crash llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
+; RUN: not llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
 
 ; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32), align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
index d55cd96f6d7c..a71e7ebb13ab 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
-; RUN: not --crash llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
+; RUN: not llc -global-isel < %s -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=GFX908
 
 ; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32) on %ir.rsrc.load, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index dd37e855c4a3..a224c8b39132 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -1025,11 +1025,12 @@ define i128 @v_mul_i128(i128 %num, i128 %den) {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v2, v11
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], vcc_lo, v8, v5, v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s0, v9, v4, v[1:2]
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v7, null, v12, v7, s0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, null, v7, v6, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[5:6], null, v10, v5, v[6:7]
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
@@ -2387,33 +2388,39 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], null, v16, v12, 0
 ; GFX12-NEXT:    v_mul_lo_u32 v30, v17, v14
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v17, v13, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], s0, v17, v11, v[18:19]
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v2, v12, v[0:1]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v22, null, 0, v20, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[20:21], null, v16, v10, 0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v3, v11, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v24, null, 0, v22, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v10, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v26, null, 0, v24, vcc_lo
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v5, v9, v[0:1]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[22:23], null, v6, v8, v[0:1]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v17, v9, v[20:21]
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cndmask_b32_e64 v25, 0, 1, s0
 ; GFX12-NEXT:    v_mov_b32_e32 v20, v22
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[21:22], vcc_lo, v2, v8, v[0:1]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v29, null, 0, v25, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v16, v13, v[19:20]
 ; GFX12-NEXT:    v_mov_b32_e32 v19, v22
 ; GFX12-NEXT:    v_mul_lo_u32 v22, v16, v15
@@ -2434,6 +2441,7 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, null, 0, v6, s2
 ; GFX12-NEXT:    v_dual_mov_b32 v13, v1 :: v_dual_mov_b32 v14, v21
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s2, v2, v9, v[11:12]
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, null, 0, v6, s2
@@ -2447,6 +2455,7 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[5:6], s4, v5, v8, v[10:11]
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], s5, v17, v8, v[12:13]
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, s5, v9, v3, s5
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v4, s5, v29, v4, s5
@@ -2463,9 +2472,10 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v25, s3
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v20, s1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v28, vcc_lo
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v28, vcc_lo
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v27, s0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[7:8], null, v7, v8, v[9:10]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %result = mul i256 %num, %den
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
index 8a26fbeb1d11..ec48fdacad6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-illegal-copy.mir
@@ -1,5 +1,5 @@
-# RUN: not --crash llc -mtriple=amdgcn -run-pass=regbankselect  -regbankselect-fast  %s -o /dev/null 2>&1 | FileCheck %s
-# RUN: not --crash llc -mtriple=amdgcn -run-pass=regbankselect  -regbankselect-greedy  %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgcn -run-pass=regbankselect -regbankselect-fast -filetype=null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgcn -run-pass=regbankselect -regbankselect-greedy -filetype=null %s 2>&1 | FileCheck %s
 
 # Check behavior for illegal copies.
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-load.ll
index ac248f57eb01..dccc55b7d045 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-load.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - < %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - < %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
 
 ; GISEL-ERR: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_LOAD %{{[0-9]+}}:vgpr(p8) :: (load (s32) from %ir.rsrc, addrspace 8)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-ptr-add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-ptr-add.ll
index 316fe65361f4..3b05f6da45bb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-ptr-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/unsupported-ptr-add.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - < %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o - < %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
 
 ; GISEL-ERR: LLVM ERROR: unable to legalize instruction: %{{[0-9]+}}:_(p8) = G_PTR_ADD %{{[0-9]+}}:_, %{{[0-9]+}}:_(s48)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
index 24ec4fa48f77..6ae2f56f6ae6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i32.ll
@@ -211,91 +211,41 @@ define i32 @v_urem_i32_oddk_denom(i32 %num) {
 ; CHECK-LABEL: v_urem_i32_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, 0x4996c7d8
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0xffed2705
-; CHECK-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
-; CHECK-NEXT:    v_mul_f32_e32 v1, 0x4f7ffffe, v1
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v1, v1
-; CHECK-NEXT:    v_mul_lo_u32 v2, v1, v2
-; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0xb2a50881
 ; CHECK-NEXT:    v_mul_hi_u32 v1, v0, v1
-; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v3
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v0, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 1, v2
+; CHECK-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
+; CHECK-NEXT:    v_lshrrev_b32_e32 v1, 20, v1
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v1, v1, v2
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 0xffed2705, v0
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v1, vcc, 0xffed2705, v0
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i32 %num, 1235195
   ret i32 %result
 }
 
 define <2 x i32> @v_urem_v2i32_oddk_denom(<2 x i32> %num) {
-; GISEL-LABEL: v_urem_v2i32_oddk_denom:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v3, 0x12d8fb
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffed2705
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_f32_e32 v3, 0x4f7ffffe, v3
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; GISEL-NEXT:    v_mul_lo_u32 v5, v3, v4
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v0, v3
-; GISEL-NEXT:    v_mul_hi_u32 v3, v1, v3
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v2
-; GISEL-NEXT:    v_mul_lo_u32 v3, v3, v2
-; GISEL-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v0, v4
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, 0xffed2705, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; GISEL-NEXT:    v_add_i32_e32 v3, vcc, v0, v4
-; GISEL-NEXT:    v_add_i32_e32 v4, vcc, 0xffed2705, v1
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v2
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; CGP-LABEL: v_urem_v2i32_oddk_denom:
-; CGP:       ; %bb.0:
-; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v2, 0x4996c7d8
-; CGP-NEXT:    v_mov_b32_e32 v3, 0xffed2705
-; CGP-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; CGP-NEXT:    v_mul_f32_e32 v2, 0x4f7ffffe, v2
-; CGP-NEXT:    v_cvt_u32_f32_e32 v2, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v2, v3
-; CGP-NEXT:    v_mul_hi_u32 v5, v2, v5
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
-; CGP-NEXT:    v_mul_hi_u32 v5, v0, v2
-; CGP-NEXT:    v_mul_hi_u32 v2, v1, v2
-; CGP-NEXT:    v_mul_lo_u32 v5, v5, v4
-; CGP-NEXT:    v_mul_lo_u32 v2, v2, v4
-; CGP-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v0, v3
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, 0xffed2705, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
-; CGP-NEXT:    v_add_i32_e32 v2, vcc, v0, v3
-; CGP-NEXT:    v_add_i32_e32 v3, vcc, 0xffed2705, v1
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v4
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; CGP-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-LABEL: v_urem_v2i32_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xb2a50881
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0x12d8fb
+; CHECK-NEXT:    v_mul_hi_u32 v4, v0, v2
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v4
+; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v1, v2
+; CHECK-NEXT:    v_lshrrev_b32_e32 v5, 1, v5
+; CHECK-NEXT:    v_lshrrev_b32_e32 v6, 1, v6
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT:    v_lshrrev_b32_e32 v4, 20, v4
+; CHECK-NEXT:    v_lshrrev_b32_e32 v2, 20, v2
+; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v3
+; CHECK-NEXT:    v_mul_lo_u32 v2, v2, v3
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
+; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v1, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i32> %num, <i32 1235195, i32 1235195>
   ret <2 x i32> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index f6a228614a27..2a1bf4bf068f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -968,523 +968,106 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
 ; CHECK-LABEL: v_urem_i64_oddk_denom:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_u32_e32 v3, 0x12d8fb
-; CHECK-NEXT:    v_cvt_f32_ubyte0_e32 v4, 0
-; CHECK-NEXT:    v_mov_b32_e32 v5, 0xffed2705
-; CHECK-NEXT:    v_mac_f32_e32 v3, 0x4f800000, v4
-; CHECK-NEXT:    v_rcp_iflag_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v3
-; CHECK-NEXT:    v_mul_f32_e32 v4, 0x2f800000, v3
-; CHECK-NEXT:    v_trunc_f32_e32 v4, v4
-; CHECK-NEXT:    v_mac_f32_e32 v3, 0xcf800000, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v4, v4
-; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
-; CHECK-NEXT:    v_mul_lo_u32 v6, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v8, v3, v5
-; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v6, v3
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v7
-; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v7
-; CHECK-NEXT:    v_mul_hi_u32 v7, v4, v7
-; CHECK-NEXT:    v_mul_lo_u32 v10, v3, v6
-; CHECK-NEXT:    v_mul_lo_u32 v11, v4, v6
-; CHECK-NEXT:    v_mul_hi_u32 v12, v3, v6
-; CHECK-NEXT:    v_mul_hi_u32 v6, v4, v6
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v11, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v8
-; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v11, v9
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
-; CHECK-NEXT:    v_addc_u32_e32 v4, vcc, v4, v6, vcc
-; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_lo_u32 v5, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v6
-; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v6
-; CHECK-NEXT:    v_mul_hi_u32 v6, v4, v6
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v5, v3
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_mul_lo_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_lo_u32 v10, v4, v5
-; CHECK-NEXT:    v_mul_hi_u32 v11, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v5, v4, v5
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v10, v6
-; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v11
-; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_add_i32_e32 v8, vcc, v10, v9
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v6
-; CHECK-NEXT:    v_addc_u32_e32 v4, vcc, v4, v5, vcc
-; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v3
-; CHECK-NEXT:    v_mul_hi_u32 v6, v0, v3
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0x1fb03c31
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0xd9528440
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v5, v1, v2
+; CHECK-NEXT:    v_mul_lo_u32 v6, v0, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v0, v2
+; CHECK-NEXT:    v_mul_lo_u32 v8, v1, v3
+; CHECK-NEXT:    v_mul_hi_u32 v2, v1, v2
+; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v3
 ; CHECK-NEXT:    v_mul_hi_u32 v3, v1, v3
-; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v4
-; CHECK-NEXT:    v_mul_lo_u32 v8, v1, v4
-; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v4
-; CHECK-NEXT:    v_mul_hi_u32 v4, v1, v4
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v8, v3
-; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
-; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v9
 ; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v5
-; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v6
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v8, v2
+; CHECK-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
 ; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v2
-; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
-; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
-; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v2
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
-; CHECK-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v6
-; CHECK-NEXT:    v_subb_u32_e64 v4, vcc, v1, v3, s[4:5]
-; CHECK-NEXT:    v_sub_i32_e32 v1, vcc, v1, v3
-; CHECK-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v2
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
-; CHECK-NEXT:    v_sub_i32_e32 v5, vcc, v0, v2
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[6:7], 0, v4
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, -1, v3, s[6:7]
-; CHECK-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CHECK-NEXT:    v_cmp_ge_u32_e64 s[4:5], v5, v2
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[4:5]
-; CHECK-NEXT:    s_mov_b64 s[4:5], vcc
-; CHECK-NEXT:    v_subrev_i32_e32 v6, vcc, 0x12d8fb, v5
-; CHECK-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CHECK-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, -1, v2, s[4:5]
-; CHECK-NEXT:    v_subbrev_u32_e32 v7, vcc, 0, v1, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
-; CHECK-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
-; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CHECK-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v8, v7
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; CHECK-NEXT:    v_lshr_b64 v[2:3], v[2:3], 20
+; CHECK-NEXT:    v_mul_lo_u32 v5, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v3, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v2, v2, v4
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem i64 %num, 1235195
   ret i64 %result
 }
 
 define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
-; GISEL-LABEL: v_urem_v2i64_oddk_denom:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_u32_e32 v6, 0x12d8fb
-; GISEL-NEXT:    v_cvt_f32_ubyte0_e32 v7, 0
-; GISEL-NEXT:    s_mov_b32 s4, 1
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0xffed2705
-; GISEL-NEXT:    s_mov_b32 s5, 1
-; GISEL-NEXT:    v_mac_f32_e32 v6, 0x4f800000, v7
-; GISEL-NEXT:    v_rcp_iflag_f32_e32 v6, v6
-; GISEL-NEXT:    s_cmp_lg_u32 s4, 0
-; GISEL-NEXT:    s_subb_u32 s6, 0, 0
-; GISEL-NEXT:    v_mul_f32_e32 v6, 0x5f7ffffc, v6
-; GISEL-NEXT:    v_mul_f32_e32 v7, 0x2f800000, v6
-; GISEL-NEXT:    s_cmp_lg_u32 s5, 0
-; GISEL-NEXT:    s_subb_u32 s7, 0, 0
-; GISEL-NEXT:    v_trunc_f32_e32 v7, v7
-; GISEL-NEXT:    v_mac_f32_e32 v6, 0xcf800000, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v7, v7
-; GISEL-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; GISEL-NEXT:    v_mul_lo_u32 v8, v7, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v10, s6, v6
-; GISEL-NEXT:    v_mul_hi_u32 v11, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v12, s7, v6
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v8
-; GISEL-NEXT:    v_mul_lo_u32 v13, v7, v9
-; GISEL-NEXT:    v_mul_hi_u32 v14, v6, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v7, v9
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v11
-; GISEL-NEXT:    v_mul_lo_u32 v11, v6, v10
-; GISEL-NEXT:    v_mul_lo_u32 v12, v7, v10
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v10
-; GISEL-NEXT:    v_mul_hi_u32 v10, v7, v10
-; GISEL-NEXT:    v_mul_lo_u32 v16, v6, v8
-; GISEL-NEXT:    v_mul_lo_u32 v17, v7, v8
-; GISEL-NEXT:    v_mul_hi_u32 v18, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v9, s[4:5], v17, v9
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v11, s[4:5], v11, v14
-; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v12, s[4:5], v12, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v19, v11
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v16, v15
-; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v17, v18
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v13, v12
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v15, v14
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v6, v11
-; GISEL-NEXT:    v_addc_u32_e32 v10, vcc, v7, v10, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v12, v11, v5
-; GISEL-NEXT:    v_mul_lo_u32 v13, s6, v11
-; GISEL-NEXT:    v_mul_hi_u32 v14, v11, v5
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v9
-; GISEL-NEXT:    v_addc_u32_e32 v7, vcc, v7, v8, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v9, s7, v6
-; GISEL-NEXT:    v_mul_hi_u32 v15, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v16, v10, v5
-; GISEL-NEXT:    v_mul_lo_u32 v17, v10, v12
-; GISEL-NEXT:    v_mul_hi_u32 v18, v11, v12
-; GISEL-NEXT:    v_mul_hi_u32 v12, v10, v12
-; GISEL-NEXT:    v_mul_lo_u32 v5, v7, v5
-; GISEL-NEXT:    v_mul_lo_u32 v19, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v16
-; GISEL-NEXT:    v_mul_hi_u32 v16, v6, v8
-; GISEL-NEXT:    v_mul_hi_u32 v8, v7, v8
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v13, v14
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v15
-; GISEL-NEXT:    v_mul_lo_u32 v13, v11, v9
-; GISEL-NEXT:    v_mul_lo_u32 v14, v10, v9
-; GISEL-NEXT:    v_mul_hi_u32 v15, v11, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v10, v9
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v17, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v18
-; GISEL-NEXT:    v_mul_lo_u32 v13, v6, v5
-; GISEL-NEXT:    v_mul_lo_u32 v18, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v19, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v19, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e64 v13, s[4:5], v13, v16
-; GISEL-NEXT:    v_mul_hi_u32 v13, v6, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v7, v5
-; GISEL-NEXT:    v_add_i32_e64 v12, s[6:7], v14, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, s[6:7]
-; GISEL-NEXT:    v_add_i32_e64 v8, s[6:7], v18, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v17, vcc, v17, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v18, 0, 1, s[4:5]
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v15
-; GISEL-NEXT:    v_add_i32_e32 v15, vcc, v19, v18
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v16, v13
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v12, v17
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v8, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v14, vcc, v14, v16
-; GISEL-NEXT:    v_add_i32_e32 v13, vcc, v13, v15
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v13
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v12
-; GISEL-NEXT:    v_addc_u32_e32 v9, vcc, v10, v9, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v10, v1, v11
-; GISEL-NEXT:    v_mul_hi_u32 v12, v0, v11
-; GISEL-NEXT:    v_mul_hi_u32 v11, v1, v11
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
-; GISEL-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; GISEL-NEXT:    v_mul_lo_u32 v7, v3, v6
-; GISEL-NEXT:    v_mul_hi_u32 v8, v2, v6
-; GISEL-NEXT:    v_mul_hi_u32 v6, v3, v6
-; GISEL-NEXT:    v_mul_lo_u32 v13, v0, v9
-; GISEL-NEXT:    v_mul_lo_u32 v14, v1, v9
-; GISEL-NEXT:    v_mul_hi_u32 v15, v0, v9
-; GISEL-NEXT:    v_mul_hi_u32 v9, v1, v9
-; GISEL-NEXT:    v_mul_lo_u32 v16, v2, v5
-; GISEL-NEXT:    v_mul_lo_u32 v17, v3, v5
-; GISEL-NEXT:    v_mul_hi_u32 v18, v2, v5
-; GISEL-NEXT:    v_mul_hi_u32 v5, v3, v5
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v13
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v14, v11
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v16
-; GISEL-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v17, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v11, v15
-; GISEL-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v18
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v13, v10
-; GISEL-NEXT:    v_add_i32_e32 v12, vcc, v14, v12
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v16, v7
-; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v17, v8
-; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; GISEL-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v7
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GISEL-NEXT:    v_add_i32_e32 v11, vcc, v12, v11
-; GISEL-NEXT:    v_mul_lo_u32 v12, v10, v4
-; GISEL-NEXT:    v_mul_hi_u32 v10, v10, v4
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; GISEL-NEXT:    v_mul_lo_u32 v8, v6, v4
-; GISEL-NEXT:    v_mul_hi_u32 v6, v6, v4
-; GISEL-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v7
-; GISEL-NEXT:    v_mul_lo_u32 v7, v9, v4
-; GISEL-NEXT:    v_mul_lo_u32 v5, v5, v4
-; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
-; GISEL-NEXT:    v_add_i32_e32 v5, vcc, v5, v6
-; GISEL-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v12
-; GISEL-NEXT:    v_subb_u32_e64 v6, vcc, v1, v7, s[4:5]
-; GISEL-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; GISEL-NEXT:    v_sub_i32_e64 v2, s[6:7], v2, v8
-; GISEL-NEXT:    v_subb_u32_e64 v8, vcc, v3, v5, s[6:7]
-; GISEL-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
-; GISEL-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; GISEL-NEXT:    v_sub_i32_e32 v9, vcc, v2, v4
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[8:9], 0, v6
-; GISEL-NEXT:    v_cndmask_b32_e64 v7, -1, v7, s[8:9]
-; GISEL-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[4:5]
-; GISEL-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[6:7]
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; GISEL-NEXT:    s_mov_b64 s[4:5], vcc
-; GISEL-NEXT:    v_subrev_i32_e32 v11, vcc, 0x12d8fb, v9
-; GISEL-NEXT:    v_sub_i32_e64 v12, s[6:7], v0, v4
-; GISEL-NEXT:    v_subbrev_u32_e64 v1, s[6:7], 0, v1, s[6:7]
-; GISEL-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v4
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; GISEL-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, -1, v13, s[4:5]
-; GISEL-NEXT:    v_sub_i32_e64 v4, s[4:5], v12, v4
-; GISEL-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v1, s[4:5]
-; GISEL-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
-; GISEL-NEXT:    v_cndmask_b32_e64 v10, -1, v10, s[4:5]
-; GISEL-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v3, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
-; GISEL-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v1, v14, vcc
-; GISEL-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
-; GISEL-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; CGP-LABEL: v_urem_v2i64_oddk_denom:
-; CGP:       ; %bb.0:
-; CGP-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CGP-NEXT:    v_mov_b32_e32 v4, 0x12d8fb
-; CGP-NEXT:    v_cvt_f32_u32_e32 v5, 0x12d8fb
-; CGP-NEXT:    v_cvt_f32_ubyte0_e32 v6, 0
-; CGP-NEXT:    v_mov_b32_e32 v7, 0xffed2705
-; CGP-NEXT:    v_mac_f32_e32 v5, 0x4f800000, v6
-; CGP-NEXT:    v_rcp_iflag_f32_e32 v5, v5
-; CGP-NEXT:    v_mul_f32_e32 v5, 0x5f7ffffc, v5
-; CGP-NEXT:    v_mul_f32_e32 v6, 0x2f800000, v5
-; CGP-NEXT:    v_trunc_f32_e32 v6, v6
-; CGP-NEXT:    v_mac_f32_e32 v5, 0xcf800000, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v6, v6
-; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
-; CGP-NEXT:    v_mul_lo_u32 v8, v6, v7
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v10, v5, v7
-; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v8, v5
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v9
-; CGP-NEXT:    v_mul_hi_u32 v11, v5, v9
-; CGP-NEXT:    v_mul_hi_u32 v9, v6, v9
-; CGP-NEXT:    v_mul_lo_u32 v12, v5, v8
-; CGP-NEXT:    v_mul_lo_u32 v13, v6, v8
-; CGP-NEXT:    v_mul_hi_u32 v14, v5, v8
-; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v12
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v14
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v10
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v13, v11
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
-; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v8, vcc
-; CGP-NEXT:    v_mul_lo_u32 v8, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_lo_u32 v7, v6, v7
-; CGP-NEXT:    v_mul_lo_u32 v10, v6, v8
-; CGP-NEXT:    v_mul_hi_u32 v11, v5, v8
-; CGP-NEXT:    v_mul_hi_u32 v8, v6, v8
-; CGP-NEXT:    v_sub_i32_e32 v7, vcc, v7, v5
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT:    v_mul_lo_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_lo_u32 v12, v6, v7
-; CGP-NEXT:    v_mul_hi_u32 v13, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v7, v6, v7
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v9, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v13
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v12, v11
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v9
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v8
-; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v7, vcc
-; CGP-NEXT:    v_mul_lo_u32 v7, v1, v5
-; CGP-NEXT:    v_mul_hi_u32 v8, v0, v5
-; CGP-NEXT:    v_mul_hi_u32 v9, v1, v5
-; CGP-NEXT:    v_mul_lo_u32 v10, v3, v5
-; CGP-NEXT:    v_mul_hi_u32 v11, v2, v5
-; CGP-NEXT:    v_mul_hi_u32 v5, v3, v5
-; CGP-NEXT:    v_mul_lo_u32 v12, v0, v6
-; CGP-NEXT:    v_mul_lo_u32 v13, v1, v6
-; CGP-NEXT:    v_mul_hi_u32 v14, v0, v6
-; CGP-NEXT:    v_mul_hi_u32 v15, v1, v6
-; CGP-NEXT:    v_mul_lo_u32 v16, v2, v6
-; CGP-NEXT:    v_mul_lo_u32 v17, v3, v6
-; CGP-NEXT:    v_mul_hi_u32 v18, v2, v6
-; CGP-NEXT:    v_mul_hi_u32 v6, v3, v6
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
-; CGP-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v16
-; CGP-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v17, v5
-; CGP-NEXT:    v_cndmask_b32_e64 v17, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v7, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v14
-; CGP-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v10, v11
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v18
-; CGP-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v12, v7
-; CGP-NEXT:    v_add_i32_e32 v9, vcc, v13, v9
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v16, v10
-; CGP-NEXT:    v_add_i32_e32 v11, vcc, v17, v11
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
-; CGP-NEXT:    v_mul_lo_u32 v9, v7, v4
-; CGP-NEXT:    v_mul_hi_u32 v7, v7, v4
-; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
-; CGP-NEXT:    v_mul_lo_u32 v11, v5, v4
-; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
-; CGP-NEXT:    v_add_i32_e32 v8, vcc, v15, v8
-; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
-; CGP-NEXT:    v_mul_lo_u32 v8, v8, v4
-; CGP-NEXT:    v_mul_lo_u32 v6, v6, v4
-; CGP-NEXT:    v_add_i32_e32 v7, vcc, v8, v7
-; CGP-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
-; CGP-NEXT:    v_sub_i32_e64 v0, s[4:5], v0, v9
-; CGP-NEXT:    v_subb_u32_e64 v6, vcc, v1, v7, s[4:5]
-; CGP-NEXT:    v_sub_i32_e32 v1, vcc, v1, v7
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v0, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v7, 0, -1, vcc
-; CGP-NEXT:    v_sub_i32_e64 v2, s[6:7], v2, v11
-; CGP-NEXT:    v_subb_u32_e64 v8, vcc, v3, v5, s[6:7]
-; CGP-NEXT:    v_sub_i32_e32 v3, vcc, v3, v5
-; CGP-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
-; CGP-NEXT:    v_sub_i32_e32 v9, vcc, v2, v4
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[8:9], 0, v6
-; CGP-NEXT:    v_cndmask_b32_e64 v7, -1, v7, s[8:9]
-; CGP-NEXT:    v_subbrev_u32_e64 v1, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v8
-; CGP-NEXT:    v_cndmask_b32_e64 v5, -1, v5, s[4:5]
-; CGP-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[6:7]
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[4:5], v9, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, -1, s[4:5]
-; CGP-NEXT:    s_mov_b64 s[4:5], vcc
-; CGP-NEXT:    v_subrev_i32_e32 v11, vcc, 0x12d8fb, v9
-; CGP-NEXT:    v_sub_i32_e64 v12, s[6:7], v0, v4
-; CGP-NEXT:    v_subbrev_u32_e64 v1, s[6:7], 0, v1, s[6:7]
-; CGP-NEXT:    v_cmp_ge_u32_e64 s[6:7], v12, v4
-; CGP-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[6:7]
-; CGP-NEXT:    v_subbrev_u32_e64 v3, s[4:5], 0, v3, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
-; CGP-NEXT:    v_cndmask_b32_e64 v13, -1, v13, s[4:5]
-; CGP-NEXT:    v_sub_i32_e64 v4, s[4:5], v12, v4
-; CGP-NEXT:    v_subbrev_u32_e64 v14, s[4:5], 0, v1, s[4:5]
-; CGP-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v3
-; CGP-NEXT:    v_cndmask_b32_e64 v10, -1, v10, s[4:5]
-; CGP-NEXT:    v_subbrev_u32_e32 v15, vcc, 0, v3, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
-; CGP-NEXT:    v_cndmask_b32_e32 v4, v12, v4, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v10
-; CGP-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v1, v14, vcc
-; CGP-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v7
-; CGP-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s[4:5]
-; CGP-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v5
-; CGP-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[4:5]
-; CGP-NEXT:    v_cndmask_b32_e32 v1, v6, v1, vcc
-; CGP-NEXT:    v_cndmask_b32_e64 v3, v8, v3, s[4:5]
-; CGP-NEXT:    s_setpc_b64 s[30:31]
+; CHECK-LABEL: v_urem_v2i64_oddk_denom:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x1fb03c31
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0xd9528440
+; CHECK-NEXT:    v_mov_b32_e32 v8, 0x12d8fb
+; CHECK-NEXT:    v_mul_lo_u32 v6, v1, v4
+; CHECK-NEXT:    v_mul_lo_u32 v7, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v9, v0, v4
+; CHECK-NEXT:    v_mul_lo_u32 v10, v1, v5
+; CHECK-NEXT:    v_mul_hi_u32 v11, v1, v4
+; CHECK-NEXT:    v_mul_hi_u32 v12, v0, v5
+; CHECK-NEXT:    v_mul_hi_u32 v13, v1, v5
+; CHECK-NEXT:    v_mul_lo_u32 v14, v3, v4
+; CHECK-NEXT:    v_mul_lo_u32 v15, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v16, v2, v4
+; CHECK-NEXT:    v_mul_lo_u32 v17, v3, v5
+; CHECK-NEXT:    v_mul_hi_u32 v4, v3, v4
+; CHECK-NEXT:    v_mul_hi_u32 v18, v2, v5
+; CHECK-NEXT:    v_mul_hi_u32 v19, v3, v5
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v7
+; CHECK-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v10, v11
+; CHECK-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v14, v15
+; CHECK-NEXT:    v_cndmask_b32_e64 v14, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v17, v4
+; CHECK-NEXT:    v_cndmask_b32_e64 v15, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v7, v12
+; CHECK-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v11, vcc, v11, v16
+; CHECK-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v12, vcc, v4, v18
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
+; CHECK-NEXT:    v_add_i32_e32 v9, vcc, v10, v9
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v14, v11
+; CHECK-NEXT:    v_add_i32_e32 v10, vcc, v15, v4
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v7, v5
+; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v12, v6
+; CHECK-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v9, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v10, v7
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v13, v5
+; CHECK-NEXT:    v_add_i32_e32 v7, vcc, v19, v7
+; CHECK-NEXT:    v_lshr_b64 v[4:5], v[4:5], 20
+; CHECK-NEXT:    v_lshr_b64 v[6:7], v[6:7], 20
+; CHECK-NEXT:    v_mul_lo_u32 v9, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v5, v5, v8
+; CHECK-NEXT:    v_mul_hi_u32 v4, v4, v8
+; CHECK-NEXT:    v_mul_lo_u32 v10, v6, v8
+; CHECK-NEXT:    v_mul_lo_u32 v7, v7, v8
+; CHECK-NEXT:    v_mul_hi_u32 v6, v6, v8
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v7, v6
+; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, v0, v9
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; CHECK-NEXT:    v_sub_i32_e32 v2, vcc, v2, v10
+; CHECK-NEXT:    v_subb_u32_e32 v3, vcc, v3, v5, vcc
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %result = urem <2 x i64> %num, <i64 1235195, i64 1235195>
   ret <2 x i64> %result
 }
diff --git a/llvm/test/CodeGen/AMDGPU/add_i1.ll b/llvm/test/CodeGen/AMDGPU/add_i1.ll
index ff1a3ee38be1..c0d73fc14eb1 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i1.ll
@@ -1,11 +1,56 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
 
-; GCN-LABEL: {{^}}add_var_var_i1:
-; GFX9:  v_xor_b32_e32
-; GFX10: v_xor_b32_e32
 define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; GFX9-LABEL: add_var_var_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[6:7] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_var_var_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v2, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_var_var_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_u8 v2, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
   %add = add i1 %a, %b
@@ -13,21 +58,175 @@ define amdgpu_kernel void @add_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; GCN-LABEL: {{^}}add_var_imm_i1:
-; GFX9:  s_xor_b64
-; GFX10: s_xor_b32
 define amdgpu_kernel void @add_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GFX9-LABEL: add_var_imm_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_xor_b64 s[2:3], vcc, -1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_var_imm_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_var_imm_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in
   %add = add i1 %a, 1
   store i1 %add, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}add_i1_cf:
-; GCN: ; %endif
-; GFX9: s_xor_b64
-; GFX10: s_xor_b32
 define amdgpu_kernel void @add_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX9-LABEL: add_i1_cf:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, 15, v0
+; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %else
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
+; GFX9-NEXT:  .LBB2_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_4
+; GFX9-NEXT:  ; %bb.3: ; %if
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_andn2_b64 s[2:3], s[4:5], exec
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT:    s_or_b64 s[4:5], s[2:3], s[4:5]
+; GFX9-NEXT:  .LBB2_4: ; %endif
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b64 s[2:3], s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: add_i1_cf:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 15, v0
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_2
+; GFX10-NEXT:  ; %bb.1: ; %else
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 1, v0
+; GFX10-NEXT:  .LBB2_2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s5, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_4
+; GFX10-NEXT:  ; %bb.3: ; %if
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s4, s2, s3
+; GFX10-NEXT:  .LBB2_4: ; %endif
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_xor_b32 s2, s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: add_i1_cf:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
+; GFX11-NEXT:    ; implicit-def: $sgpr6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmpx_lt_u32_e32 15, v0
+; GFX11-NEXT:    s_xor_b32 s7, exec_lo, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %else
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v0, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 1, v0
+; GFX11-NEXT:  .LBB2_2: ; %Flow
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s4, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_4
+; GFX11-NEXT:  ; %bb.3: ; %if
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_u8 v0, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_and_not1_b32 s2, s6, exec_lo
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-NEXT:    s_or_b32 s6, s2, s3
+; GFX11-NEXT:  .LBB2_4: ; %endif
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_xor_b32 s2, s6, -1
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %d_cmp = icmp ult i32 %tid, 16
diff --git a/llvm/test/CodeGen/AMDGPU/add_i128.ll b/llvm/test/CodeGen/AMDGPU/add_i128.ll
index d2010568df91..c2c50460a27b 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i128.ll
@@ -1,12 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}test_i128_vreg:
-; GCN: v_add_i32_e32 v[[LO:[0-9]+]], vcc,
-; GCN-NEXT: v_addc_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc
-; GCN-NEXT: v_addc_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc
-; GCN-NEXT: v_addc_u32_e32 v[[HI:[0-9]+]], vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc
-; GCN: buffer_store_dwordx4 v[[[LO]]:[[HI]]],
 define amdgpu_kernel void @test_i128_vreg(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %inA, ptr addrspace(1) noalias %inB) {
+; GCN-LABEL: test_i128_vreg:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s14, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; GCN-NEXT:    v_mov_b32_e32 v5, 0
+; GCN-NEXT:    s_mov_b32 s15, s11
+; GCN-NEXT:    s_mov_b64 s[6:7], s[14:15]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[12:15], 0 addr64
+; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[4:5], s[4:7], 0 addr64
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s8, s0
+; GCN-NEXT:    s_mov_b32 s9, s1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; GCN-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; GCN-NEXT:    v_addc_u32_e32 v2, vcc, v2, v6, vcc
+; GCN-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone
   %a_ptr = getelementptr i128, ptr addrspace(1) %inA, i32 %tid
   %b_ptr = getelementptr i128, ptr addrspace(1) %inB, i32 %tid
@@ -18,36 +37,74 @@ define amdgpu_kernel void @test_i128_vreg(ptr addrspace(1) noalias %out, ptr add
 }
 
 ; Check that the SGPR add operand is correctly moved to a VGPR.
-; GCN-LABEL: {{^}}sgpr_operand:
-; GCN: s_add_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
 define amdgpu_kernel void @sgpr_operand(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i128 %a) {
+; GCN-LABEL: sgpr_operand:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[2:3], 0x0
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_u32 s4, s8, s4
+; GCN-NEXT:    s_addc_u32 s5, s9, s5
+; GCN-NEXT:    s_addc_u32 s6, s10, s6
+; GCN-NEXT:    s_addc_u32 s7, s11, s7
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %foo = load i128, ptr addrspace(1) %in, align 8
   %result = add i128 %foo, %a
   store i128 %result, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}sgpr_operand_reversed:
-; GCN: s_add_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
 define amdgpu_kernel void @sgpr_operand_reversed(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i128 %a) {
+; GCN-LABEL: sgpr_operand_reversed:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[2:3], 0x0
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_u32 s4, s4, s8
+; GCN-NEXT:    s_addc_u32 s5, s5, s9
+; GCN-NEXT:    s_addc_u32 s6, s6, s10
+; GCN-NEXT:    s_addc_u32 s7, s7, s11
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %foo = load i128, ptr addrspace(1) %in, align 8
   %result = add i128 %a, %foo
   store i128 %result, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}test_sreg:
-; GCN: s_add_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
-; GCN: s_addc_u32
 define amdgpu_kernel void @test_sreg(ptr addrspace(1) noalias %out, i128 %a, i128 %b) {
+; GCN-LABEL: test_sreg:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_u32 s4, s8, s12
+; GCN-NEXT:    s_addc_u32 s5, s9, s13
+; GCN-NEXT:    s_addc_u32 s6, s10, s14
+; GCN-NEXT:    s_addc_u32 s7, s11, s15
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    v_mov_b32_e32 v3, s7
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %result = add i128 %a, %b
   store i128 %result, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
new file mode 100644
index 000000000000..b88429d8a63f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx1250.mir
@@ -0,0 +1,148 @@
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -start-before=greedy,0 -stop-after=virtregrewriter,2 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN,GFX1250 %s
+# Using the unaligned vector tuples are OK as long as they aren't used
+# in a real instruction.
+
+---
+# GCN-LABEL: name: alloc_vgpr_64
+# GFX1250: $vgpr4_vgpr5 = GLOBAL_LOAD
+name:            alloc_vgpr_64
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX2 %0, %2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_96
+# GFX1250: $vgpr4_vgpr5_vgpr6 = GLOBAL_LOAD
+name:            alloc_vgpr_96
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_96_align2 = GLOBAL_LOAD_DWORDX3 %0, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX3 %0, %2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_128
+# GFX1250: $vgpr4_vgpr5_vgpr6_vgpr7 = GLOBAL_LOAD
+name:            alloc_vgpr_128
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_160
+# GFX1250: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 = IMAGE_LOAD
+name:            alloc_vgpr_160
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_160_align2 = IMAGE_LOAD_V5_V1 %0.sub0, undef %3:sgpr_256, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
+    GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_256
+# GFX1250: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11 = COPY
+name:            alloc_vgpr_256
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %3:sgpr_256 = IMPLICIT_DEF
+    %2:vreg_256_align2 = COPY %3:sgpr_256
+    %4:vreg_128_align2 = IMAGE_SAMPLE_C_CL_O_V4_V8 %2, %3:sgpr_256, undef %5:sgpr_128, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
+    GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_512
+# GFX1250: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 = IMPLICIT_DEF
+name:            alloc_vgpr_512
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_512_align2 = IMPLICIT_DEF
+    GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub4_sub5_sub6_sub7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub8_sub9_sub10_sub11, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub12_sub13_sub14_sub15, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
+
+---
+# GCN-LABEL: name: alloc_vgpr_1024
+# GFX1250: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = IMPLICIT_DEF
+name:            alloc_vgpr_1024
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0_vgpr1' }
+  - { reg: '$vgpr2' }
+body:             |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+    %1:vgpr_32 = COPY $vgpr2
+    %2:vreg_1024_align2 = IMPLICIT_DEF
+    GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub4_sub5_sub6_sub7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub8_sub9_sub10_sub11, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub12_sub13_sub14_sub15, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub16_sub17_sub18_sub19, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub20_sub21_sub22_sub23, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub24_sub25_sub26_sub27, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %2.sub28_sub29_sub30_sub31, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-expand-feature-predicates.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-expand-feature-predicates.ll
index 277323c35326..a16a7fc31da2 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-expand-feature-predicates.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-expand-feature-predicates.ll
@@ -54,42 +54,23 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX906-NEXT:  [[ENTRY:.*:]]
 ; GFX906-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P_COERCE]] to i64
 ; GFX906-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; GFX906-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS1:.*]]
-; GFX906:       [[IF_GFX1201_OR_GFX12_INSTS1]]:
-; GFX906-NEXT:    br label %[[IF_NOT_GFX906:.*]]
-; GFX906:       [[IF_GFX1201_OR_GFX12_INSTS:.*:]]
-; GFX906-NEXT:    call void @llvm.amdgcn.s.sleep.var(i32 [[X]])
-; GFX906-NEXT:    br label %[[IF_NOT_GFX906]]
-; GFX906:       [[IF_NOT_GFX906]]:
-; GFX906-NEXT:    br label %[[IF_GFX1010_OR_GFX1102:.*]]
-; GFX906:       [[IF_NOT_GFX907:.*:]]
-; GFX906-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX906-NEXT:    br label %[[IF_END6:.*]]
-; GFX906:       [[IF_GFX1010_OR_GFX1102]]:
+; GFX906-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS:.*]]
+; GFX906:       [[IF_GFX1201_OR_GFX12_INSTS]]:
+; GFX906-NEXT:    br label %[[IF_NOT_GFX907:.*]]
+; GFX906:       [[IF_NOT_GFX907]]:
+; GFX906-NEXT:    br label %[[IF_GFX1010_OR_GFX1101:.*]]
+; GFX906:       [[IF_GFX1010_OR_GFX1101]]:
 ; GFX906-NEXT:    br label %[[LOR_NOT_GFX1010:.*]]
 ; GFX906:       [[LOR_NOT_GFX1010]]:
 ; GFX906-NEXT:    br label %[[FOR_COND:.*]]
-; GFX906:       [[IF_GFX1010_OR_GFX1101:.*:]]
-; GFX906-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX906-NEXT:    br label %[[IF_END6]]
-; GFX906:       [[IF_END6]]:
-; GFX906-NEXT:    call void @llvm.assume(i1 true)
-; GFX906-NEXT:    call void @llvm.assume(i1 true)
-; GFX906-NEXT:    br label %[[FOR_COND]]
 ; GFX906:       [[FOR_COND]]:
 ; GFX906-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX906-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED]], [[X]]
 ; GFX906-NEXT:    store i32 [[SUB_PEEL]], ptr [[TMP1]], align 4
-; GFX906-NEXT:    br label %[[IF_GFX10_INSTS1:.*]]
-; GFX906:       [[IF_GFX11_INSTS:.*:]]
-; GFX906-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX906-NEXT:    br label %[[IF_END11:.*]]
-; GFX906:       [[IF_GFX10_INSTS1]]:
-; GFX906-NEXT:    br label %[[IF_END11]]
-; GFX906:       [[IF_GFX10_INSTS:.*:]]
-; GFX906-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX906-NEXT:    br label %[[IF_END11]]
-; GFX906:       [[IF_END11]]:
+; GFX906-NEXT:    br label %[[IF_GFX11_INSTS:.*]]
+; GFX906:       [[IF_GFX11_INSTS]]:
+; GFX906-NEXT:    br label %[[IF_GFX10_INSTS:.*]]
+; GFX906:       [[IF_GFX10_INSTS]]:
 ; GFX906-NEXT:    call void @llvm.assume(i1 true)
 ; GFX906-NEXT:    [[DOTPROMOTED9:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX906-NEXT:    [[SUB13_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED9]], [[X]]
@@ -101,41 +82,28 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1010-NEXT:  [[ENTRY:.*:]]
 ; GFX1010-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P_COERCE]] to i64
 ; GFX1010-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; GFX1010-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS1:.*]]
-; GFX1010:       [[IF_GFX1201_OR_GFX12_INSTS1]]:
-; GFX1010-NEXT:    br label %[[IF_END:.*]]
-; GFX1010:       [[IF_GFX1201_OR_GFX12_INSTS:.*:]]
-; GFX1010-NEXT:    call void @llvm.amdgcn.s.sleep.var(i32 [[X]])
-; GFX1010-NEXT:    br label %[[IF_END]]
-; GFX1010:       [[IF_END]]:
-; GFX1010-NEXT:    br label %[[IF_NOT_GFX907:.*]]
-; GFX1010:       [[IF_NOT_GFX907]]:
+; GFX1010-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS:.*]]
+; GFX1010:       [[IF_GFX1201_OR_GFX12_INSTS]]:
+; GFX1010-NEXT:    br label %[[IF_NOT_GFX906:.*]]
+; GFX1010:       [[IF_NOT_GFX906]]:
+; GFX1010-NEXT:    br label %[[LOR_NOT_GFX1010:.*]]
+; GFX1010:       [[LOR_NOT_GFX1010]]:
 ; GFX1010-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
 ; GFX1010-NEXT:    br label %[[IF_END6:.*]]
-; GFX1010:       [[IF_NOT_GFX906:.*:]]
-; GFX1010-NEXT:    br label %[[IF_GFX1010_OR_GFX1101:.*]]
-; GFX1010:       [[LOR_NOT_GFX1010:.*:]]
-; GFX1010-NEXT:    br label %[[FOR_COND:.*]]
-; GFX1010:       [[IF_GFX1010_OR_GFX1101]]:
-; GFX1010-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1010-NEXT:    br label %[[IF_END6]]
 ; GFX1010:       [[IF_END6]]:
 ; GFX1010-NEXT:    call void @llvm.assume(i1 true)
-; GFX1010-NEXT:    call void @llvm.assume(i1 false)
-; GFX1010-NEXT:    br label %[[FOR_COND]]
+; GFX1010-NEXT:    call void @llvm.assume(i1 true)
+; GFX1010-NEXT:    br label %[[FOR_COND:.*]]
 ; GFX1010:       [[FOR_COND]]:
 ; GFX1010-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1010-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED]], [[X]]
 ; GFX1010-NEXT:    store i32 [[SUB_PEEL]], ptr [[TMP1]], align 4
-; GFX1010-NEXT:    br label %[[IF_ELSE8:.*]]
-; GFX1010:       [[IF_GFX11_INSTS:.*:]]
-; GFX1010-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX1010-NEXT:    br label %[[IF_END11:.*]]
-; GFX1010:       [[IF_ELSE8]]:
+; GFX1010-NEXT:    br label %[[IF_GFX11_INSTS:.*]]
+; GFX1010:       [[IF_GFX11_INSTS]]:
 ; GFX1010-NEXT:    br label %[[IF_GFX10_INSTS:.*]]
 ; GFX1010:       [[IF_GFX10_INSTS]]:
 ; GFX1010-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1010-NEXT:    br label %[[IF_END11]]
+; GFX1010-NEXT:    br label %[[IF_END11:.*]]
 ; GFX1010:       [[IF_END11]]:
 ; GFX1010-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1010-NEXT:    [[DOTPROMOTED9:%.*]] = load i32, ptr [[TMP1]], align 4
@@ -148,26 +116,16 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1101-NEXT:  [[ENTRY:.*:]]
 ; GFX1101-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P_COERCE]] to i64
 ; GFX1101-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; GFX1101-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS1:.*]]
-; GFX1101:       [[IF_GFX1201_OR_GFX12_INSTS1]]:
+; GFX1101-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS:.*]]
+; GFX1101:       [[IF_GFX1201_OR_GFX12_INSTS]]:
 ; GFX1101-NEXT:    br label %[[IF_END:.*]]
-; GFX1101:       [[IF_GFX1201_OR_GFX12_INSTS:.*:]]
-; GFX1101-NEXT:    call void @llvm.amdgcn.s.sleep.var(i32 [[X]])
-; GFX1101-NEXT:    br label %[[IF_END]]
 ; GFX1101:       [[IF_END]]:
 ; GFX1101-NEXT:    br label %[[IF_NOT_GFX907:.*]]
 ; GFX1101:       [[IF_NOT_GFX907]]:
 ; GFX1101-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX1101-NEXT:    br label %[[IF_END6:.*]]
-; GFX1101:       [[IF_NOT_GFX906:.*:]]
-; GFX1101-NEXT:    br label %[[LOR_NOT_GFX1010:.*]]
-; GFX1101:       [[LOR_NOT_GFX1010]]:
-; GFX1101-NEXT:    br label %[[IF_GFX1010_OR_GFX1101:.*]]
-; GFX1101:       [[IF_GFX1010_OR_GFX1101]]:
-; GFX1101-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1101-NEXT:    br label %[[IF_END6]]
-; GFX1101:       [[IF_END6]]:
-; GFX1101-NEXT:    call void @llvm.assume(i1 false)
+; GFX1101-NEXT:    br label %[[IF_NOT_GFX906:.*]]
+; GFX1101:       [[IF_NOT_GFX906]]:
+; GFX1101-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1101-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1101-NEXT:    br label %[[FOR_COND:.*]]
 ; GFX1101:       [[FOR_COND]]:
@@ -177,13 +135,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1101-NEXT:    br label %[[IF_GFX11_INSTS:.*]]
 ; GFX1101:       [[IF_GFX11_INSTS]]:
 ; GFX1101-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX1101-NEXT:    br label %[[IF_END11:.*]]
-; GFX1101:       [[IF_ELSE8:.*:]]
-; GFX1101-NEXT:    br label %[[IF_GFX10_INSTS:.*]]
-; GFX1101:       [[IF_GFX10_INSTS]]:
-; GFX1101-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1101-NEXT:    br label %[[IF_END11]]
-; GFX1101:       [[IF_END11]]:
+; GFX1101-NEXT:    br label %[[IF_ELSE8:.*]]
+; GFX1101:       [[IF_ELSE8]]:
 ; GFX1101-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1101-NEXT:    [[DOTPROMOTED9:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1101-NEXT:    [[SUB13_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED9]], [[X]]
@@ -195,28 +148,19 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1201-NEXT:  [[ENTRY:.*:]]
 ; GFX1201-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P_COERCE]] to i64
 ; GFX1201-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; GFX1201-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS:.*]]
-; GFX1201:       [[LOR_NOT_GFX1201:.*:]]
-; GFX1201-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS]]
-; GFX1201:       [[IF_GFX1201_OR_GFX12_INSTS]]:
+; GFX1201-NEXT:    br label %[[LOR_NOT_GFX1201:.*]]
+; GFX1201:       [[LOR_NOT_GFX1201]]:
 ; GFX1201-NEXT:    call void @llvm.amdgcn.s.sleep.var(i32 [[X]])
-; GFX1201-NEXT:    br label %[[IF_END:.*]]
-; GFX1201:       [[IF_END]]:
-; GFX1201-NEXT:    br label %[[IF_NOT_GFX907:.*]]
-; GFX1201:       [[IF_NOT_GFX907]]:
+; GFX1201-NEXT:    br label %[[IF_NOT_GFX906:.*]]
+; GFX1201:       [[IF_NOT_GFX906]]:
+; GFX1201-NEXT:    br label %[[IF_GFX1010_OR_GFX1101:.*]]
+; GFX1201:       [[IF_GFX1010_OR_GFX1101]]:
 ; GFX1201-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
 ; GFX1201-NEXT:    br label %[[IF_END6:.*]]
-; GFX1201:       [[IF_NOT_GFX906:.*:]]
-; GFX1201-NEXT:    br label %[[IF_GFX1010_OR_GFX1102:.*]]
-; GFX1201:       [[IF_GFX1010_OR_GFX1102]]:
-; GFX1201-NEXT:    br label %[[FOR_COND:.*]]
-; GFX1201:       [[IF_GFX1010_OR_GFX1101:.*:]]
-; GFX1201-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1201-NEXT:    br label %[[IF_END6]]
 ; GFX1201:       [[IF_END6]]:
 ; GFX1201-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1201-NEXT:    call void @llvm.assume(i1 true)
-; GFX1201-NEXT:    br label %[[FOR_COND]]
+; GFX1201-NEXT:    br label %[[FOR_COND:.*]]
 ; GFX1201:       [[FOR_COND]]:
 ; GFX1201-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1201-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED]], [[X]]
@@ -224,13 +168,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1201-NEXT:    br label %[[IF_GFX11_INSTS:.*]]
 ; GFX1201:       [[IF_GFX11_INSTS]]:
 ; GFX1201-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX1201-NEXT:    br label %[[IF_END11:.*]]
-; GFX1201:       [[IF_ELSE8:.*:]]
-; GFX1201-NEXT:    br label %[[IF_GFX10_INSTS:.*]]
-; GFX1201:       [[IF_GFX10_INSTS]]:
-; GFX1201-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1201-NEXT:    br label %[[IF_END11]]
-; GFX1201:       [[IF_END11]]:
+; GFX1201-NEXT:    br label %[[IF_ELSE8:.*]]
+; GFX1201:       [[IF_ELSE8]]:
 ; GFX1201-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1201-NEXT:    [[DOTPROMOTED9:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1201-NEXT:    [[SUB13_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED9]], [[X]]
@@ -242,28 +181,19 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1201-W64-NEXT:  [[ENTRY:.*:]]
 ; GFX1201-W64-NEXT:    [[TMP0:%.*]] = ptrtoint ptr addrspace(1) [[P_COERCE]] to i64
 ; GFX1201-W64-NEXT:    [[TMP1:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; GFX1201-W64-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS:.*]]
-; GFX1201-W64:       [[LOR_NOT_GFX1201:.*:]]
-; GFX1201-W64-NEXT:    br label %[[IF_GFX1201_OR_GFX12_INSTS]]
-; GFX1201-W64:       [[IF_GFX1201_OR_GFX12_INSTS]]:
+; GFX1201-W64-NEXT:    br label %[[LOR_NOT_GFX1201:.*]]
+; GFX1201-W64:       [[LOR_NOT_GFX1201]]:
 ; GFX1201-W64-NEXT:    call void @llvm.amdgcn.s.sleep.var(i32 [[X]])
-; GFX1201-W64-NEXT:    br label %[[IF_END:.*]]
-; GFX1201-W64:       [[IF_END]]:
-; GFX1201-W64-NEXT:    br label %[[IF_NOT_GFX907:.*]]
-; GFX1201-W64:       [[IF_NOT_GFX907]]:
+; GFX1201-W64-NEXT:    br label %[[IF_NOT_GFX906:.*]]
+; GFX1201-W64:       [[IF_NOT_GFX906]]:
+; GFX1201-W64-NEXT:    br label %[[IF_GFX1010_OR_GFX1101:.*]]
+; GFX1201-W64:       [[IF_GFX1010_OR_GFX1101]]:
 ; GFX1201-W64-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
 ; GFX1201-W64-NEXT:    br label %[[IF_END6:.*]]
-; GFX1201-W64:       [[IF_NOT_GFX906:.*:]]
-; GFX1201-W64-NEXT:    br label %[[IF_GFX1010_OR_GFX1102:.*]]
-; GFX1201-W64:       [[IF_GFX1010_OR_GFX1102]]:
-; GFX1201-W64-NEXT:    br label %[[FOR_COND:.*]]
-; GFX1201-W64:       [[IF_GFX1010_OR_GFX1101:.*:]]
-; GFX1201-W64-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1201-W64-NEXT:    br label %[[IF_END6]]
 ; GFX1201-W64:       [[IF_END6]]:
 ; GFX1201-W64-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1201-W64-NEXT:    call void @llvm.assume(i1 true)
-; GFX1201-W64-NEXT:    br label %[[FOR_COND]]
+; GFX1201-W64-NEXT:    br label %[[FOR_COND:.*]]
 ; GFX1201-W64:       [[FOR_COND]]:
 ; GFX1201-W64-NEXT:    [[DOTPROMOTED:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1201-W64-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED]], [[X]]
@@ -271,14 +201,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %p.coerce, i32 %x) {
 ; GFX1201-W64-NEXT:    br label %[[IF_GFX11_INSTS:.*]]
 ; GFX1201-W64:       [[IF_GFX11_INSTS]]:
 ; GFX1201-W64-NEXT:    call void @llvm.amdgcn.s.wait.event.export.ready()
-; GFX1201-W64-NEXT:    br label %[[IF_END11:.*]]
-; GFX1201-W64:       [[IF_ELSE8:.*:]]
-; GFX1201-W64-NEXT:    br label %[[IF_GFX10_INSTS:.*]]
-; GFX1201-W64:       [[IF_GFX10_INSTS]]:
-; GFX1201-W64-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
-; GFX1201-W64-NEXT:    br label %[[IF_END11]]
-; GFX1201-W64:       [[IF_END11]]:
-; GFX1201-W64-NEXT:    call void @llvm.assume(i1 false)
+; GFX1201-W64-NEXT:    br label %[[IF_ELSE8:.*]]
+; GFX1201-W64:       [[IF_ELSE8]]:
+; GFX1201-W64-NEXT:    call void @llvm.assume(i1 true)
 ; GFX1201-W64-NEXT:    [[DOTPROMOTED9:%.*]] = load i32, ptr [[TMP1]], align 4
 ; GFX1201-W64-NEXT:    [[SUB13_PEEL:%.*]] = sub nsw i32 [[DOTPROMOTED9]], [[X]]
 ; GFX1201-W64-NEXT:    store i32 [[SUB13_PEEL]], ptr [[TMP1]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir
new file mode 100644
index 000000000000..22c913496b73
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-snop-padding.mir
@@ -0,0 +1,132 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-snop-padding=8 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN8 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -amdgpu-snop-padding=16 -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN16 %s
+
+---
+name:            test_snop_padding
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+stack:
+  - { id: 0, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+  isEntryFunction: false
+  scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
+  stackPtrOffsetReg: '$sgpr32'
+  frameOffsetReg: '$sgpr33'
+  hasSpilledSGPRs: true
+body:             |
+  ; GCN8-LABEL: name: test_snop_padding
+  ; GCN8: bb.0:
+  ; GCN8-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_BRANCH %bb.1
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.1:
+  ; GCN8-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_CBRANCH_EXECZ %bb.3, implicit $exec
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.2:
+  ; GCN8-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN8-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_NOP 0
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_BRANCH %bb.3
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT: bb.3:
+  ; GCN8-NEXT:   liveins: $sgpr10_sgpr11
+  ; GCN8-NEXT: {{  $}}
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   $sgpr5 = V_READFIRSTLANE_B32 [[V_MOV_B32_e32_]], implicit $exec
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+  ; GCN8-NEXT:   S_NOP 7
+  ; GCN8-NEXT:   SI_RETURN
+  ;
+  ; GCN16-LABEL: name: test_snop_padding
+  ; GCN16: bb.0:
+  ; GCN16-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_BRANCH %bb.1
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.1:
+  ; GCN16-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_CBRANCH_EXECZ %bb.3, implicit $exec
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.2:
+  ; GCN16-NEXT:   successors: %bb.3(0x80000000)
+  ; GCN16-NEXT:   liveins: $sgpr6, $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 0
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_BRANCH %bb.3
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT: bb.3:
+  ; GCN16-NEXT:   liveins: $sgpr10_sgpr11
+  ; GCN16-NEXT: {{  $}}
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   $sgpr5 = V_READFIRSTLANE_B32 [[V_MOV_B32_e32_]], implicit $exec
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   S_NOP 7
+  ; GCN16-NEXT:   SI_RETURN
+  bb.0:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    S_BRANCH %bb.1
+  bb.1:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    %0:vgpr_32 = V_MOV_B32_e32 10, implicit $exec
+    S_CBRANCH_EXECZ %bb.3, implicit $exec
+  bb.2:
+    liveins: $sgpr6, $sgpr10_sgpr11
+    SI_SPILL_S32_SAVE killed $sgpr6, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    S_NOP 0
+    renamable $sgpr6 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+    %0:vgpr_32 = V_MOV_B32_e32 20, implicit $exec
+    S_BRANCH %bb.3
+  bb.3:
+    liveins: $sgpr10_sgpr11
+    $sgpr5 = V_READFIRSTLANE_B32 %0:vgpr_32, implicit $exec
+    S_STORE_DWORD_IMM $sgpr5, $sgpr10_sgpr11, 0, 0
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll b/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll
index 0718a17dfd06..78942bfc68d6 100644
--- a/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/ashr64_reduce.ll
@@ -17,9 +17,11 @@ define i64 @ashr_metadata(i64 %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr i64 %arg0, %shift.amt
@@ -29,9 +31,11 @@ define i64 @ashr_metadata(i64 %arg0, ptr %arg1.ptr) {
 define amdgpu_ps i64 @ashr_metadata_sgpr_return(i64 inreg %arg0, ptr addrspace(1) inreg %arg1.ptr) {
 ; CHECK-LABEL: ashr_metadata_sgpr_return:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_load_dword s2, s[2:3], 0x0
+; CHECK-NEXT:    s_load_dword s0, s[2:3], 0x0
+; CHECK-NEXT:    s_ashr_i32 s2, s1, 31
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_ashr_i64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    s_ashr_i32 s0, s1, s0
+; CHECK-NEXT:    s_mov_b32 s1, s2
 ; CHECK-NEXT:    ; return to shader part epilog
   %shift.amt = load i64, ptr addrspace(1) %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr i64 %arg0, %shift.amt
@@ -43,9 +47,11 @@ define i64 @ashr_exact_metadata(i64 %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_exact_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load i64, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr exact i64 %arg0, %shift.amt
@@ -56,9 +62,11 @@ define i64 @ashr_metadata_two_ranges(i64 %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_metadata_two_ranges:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dword v2, v[2:3]
+; CHECK-NEXT:    flat_load_dword v0, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, v2
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load i64, ptr %arg1.ptr, !range !1, !noundef !{}
   %ashr = ashr i64 %arg0, %shift.amt
@@ -100,25 +108,47 @@ define <2 x i64> @ashr_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_v2_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    flat_load_dwordx4 v[6:9], v[4:5]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v6, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v8, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    v_mov_b32_e32 v3, v4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr <2 x i64> %arg0, %shift.amt
   ret <2 x i64> %ashr
 }
 
+define <2 x i64> @ashr_v2_metadata_63(<2 x i64> %arg0, ptr %arg1.ptr) {
+; CHECK-LABEL: ashr_v2_metadata_63:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, 31, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, v0
+; CHECK-NEXT:    v_mov_b32_e32 v3, v2
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !4, !noundef !{}
+  %ashr = ashr <2 x i64> %arg0, %shift.amt
+  ret <2 x i64> %ashr
+}
+
 ; Exact attribute does not inhibit reduction
 define <2 x i64> @ashr_exact_v2_metadata(<2 x i64> %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_exact_v2_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT:    flat_load_dwordx4 v[6:9], v[4:5]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v5, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v6, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v6, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v8, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, v5
+; CHECK-NEXT:    v_mov_b32_e32 v3, v4
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load <2 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr exact <2 x i64> %arg0, %shift.amt
@@ -129,12 +159,18 @@ define <3 x i64> @ashr_v3_metadata(<3 x i64> %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_v3_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dword v12, v[6:7] offset:16
-; CHECK-NEXT:    flat_load_dwordx4 v[8:11], v[6:7]
+; CHECK-NEXT:    flat_load_dword v0, v[6:7] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[9:12], v[6:7]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v7, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v8, 31, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v12, v[4:5]
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, v0, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v9, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v11, v3
+; CHECK-NEXT:    v_mov_b32_e32 v1, v7
+; CHECK-NEXT:    v_mov_b32_e32 v3, v8
+; CHECK-NEXT:    v_mov_b32_e32 v5, v6
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load <3 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr <3 x i64> %arg0, %shift.amt
@@ -145,15 +181,23 @@ define <4 x i64> @ashr_v4_metadata(<4 x i64> %arg0, ptr %arg1.ptr) {
 ; CHECK-LABEL: ashr_v4_metadata:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dwordx4 v[10:13], v[8:9]
+; CHECK-NEXT:    flat_load_dwordx4 v[12:15], v[8:9]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    flat_load_dwordx4 v[13:16], v[8:9] offset:16
+; CHECK-NEXT:    flat_load_dwordx4 v[15:18], v[8:9] offset:16
 ; CHECK-NEXT:    ; kill: killed $vgpr8 killed $vgpr9
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v10, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v12, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v11, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v9, 31, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v10, 31, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v12, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v14, v3
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v13, v[4:5]
-; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v15, v[6:7]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, v15, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v6, v17, v7
+; CHECK-NEXT:    v_mov_b32_e32 v1, v11
+; CHECK-NEXT:    v_mov_b32_e32 v3, v9
+; CHECK-NEXT:    v_mov_b32_e32 v5, v10
+; CHECK-NEXT:    v_mov_b32_e32 v7, v8
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %shift.amt = load <4 x i64>, ptr %arg1.ptr, !range !0, !noundef !{}
   %ashr = ashr <4 x i64> %arg0, %shift.amt
@@ -164,6 +208,7 @@ define <4 x i64> @ashr_v4_metadata(<4 x i64> %arg0, ptr %arg1.ptr) {
 !1 = !{i64 32, i64 38, i64 42, i64 48}
 !2 = !{i64 31, i64 38, i64 42, i64 48}
 !3 = !{i64 32, i64 38, i64 2147483680, i64 2147483681}
+!4 = !{i64 63, i64 64}
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; Test range with an "or X, 16"
@@ -333,8 +378,8 @@ define i64 @ashr_or32(i64 %arg0, i64 %shift_amt) {
 ; CHECK-LABEL: ashr_or32:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v2, 32, v2
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v2, v[0:1]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v2, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v1
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %or = or i64 %shift_amt, 32
   %ashr = ashr i64 %arg0, %or
@@ -345,10 +390,10 @@ define <2 x i64> @ashr_v2_or32(<2 x i64> %arg0, <2 x i64> %shift_amt) {
 ; CHECK-LABEL: ashr_v2_or32:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v5, 32, v6
-; CHECK-NEXT:    v_or_b32_e32 v4, 32, v4
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v4, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v5, v[2:3]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v4, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v6, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %or = or <2 x i64> %shift_amt, splat (i64 32)
   %ashr = ashr <2 x i64> %arg0, %or
@@ -359,12 +404,12 @@ define <3 x i64> @ashr_v3_or32(<3 x i64> %arg0, <3 x i64> %shift_amt) {
 ; CHECK-LABEL: ashr_v3_or32:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v7, 32, v10
-; CHECK-NEXT:    v_or_b32_e32 v8, 32, v8
-; CHECK-NEXT:    v_or_b32_e32 v6, 32, v6
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v6, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v8, v[2:3]
-; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v7, v[4:5]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v6, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v8, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, v10, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v5, 31, v5
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %or = or <3 x i64> %shift_amt, splat (i64 32)
   %ashr = ashr <3 x i64> %arg0, %or
@@ -375,14 +420,14 @@ define <4 x i64> @ashr_v4_or32(<4 x i64> %arg0, <4 x i64> %shift_amt) {
 ; CHECK-LABEL: ashr_v4_or32:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v9, 32, v14
-; CHECK-NEXT:    v_or_b32_e32 v11, 32, v12
-; CHECK-NEXT:    v_or_b32_e32 v10, 32, v10
-; CHECK-NEXT:    v_or_b32_e32 v8, 32, v8
-; CHECK-NEXT:    v_ashrrev_i64 v[0:1], v8, v[0:1]
-; CHECK-NEXT:    v_ashrrev_i64 v[2:3], v10, v[2:3]
-; CHECK-NEXT:    v_ashrrev_i64 v[4:5], v11, v[4:5]
-; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v9, v[6:7]
+; CHECK-NEXT:    v_ashrrev_i32_e32 v0, v8, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v1, 31, v1
+; CHECK-NEXT:    v_ashrrev_i32_e32 v2, v10, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v3, 31, v3
+; CHECK-NEXT:    v_ashrrev_i32_e32 v4, v12, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v5, 31, v5
+; CHECK-NEXT:    v_ashrrev_i32_e32 v6, v14, v7
+; CHECK-NEXT:    v_ashrrev_i32_e32 v7, 31, v7
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %or = or <4 x i64> %shift_amt, splat (i64 32)
   %ashr = ashr <4 x i64> %arg0, %or
@@ -395,8 +440,8 @@ define i64 @ashr_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
 ; CHECK-LABEL: ashr_or32_sgpr:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_or_b32 s4, s18, 32
-; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
+; CHECK-NEXT:    s_ashr_i32 s4, s17, s18
+; CHECK-NEXT:    s_ashr_i32 s5, s17, 31
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s5
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
@@ -408,8 +453,8 @@ define i64 @ashr_or32_sgpr(i64 inreg %arg0, i64 inreg %shift_amt) {
 define amdgpu_ps i64 @ashr_or32_sgpr_return(i64 inreg %arg0, i64 inreg %shift_amt) {
 ; CHECK-LABEL: ashr_or32_sgpr_return:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_or_b32 s2, s2, 32
-; CHECK-NEXT:    s_ashr_i64 s[0:1], s[0:1], s2
+; CHECK-NEXT:    s_ashr_i32 s0, s1, s2
+; CHECK-NEXT:    s_ashr_i32 s1, s1, 31
 ; CHECK-NEXT:    ; return to shader part epilog
   %or = or i64 %shift_amt, 32
   %ashr = ashr i64 %arg0, %or
@@ -420,10 +465,10 @@ define <2 x i64> @ashr_v2_or32_sgpr(<2 x i64> inreg %arg0, <2 x i64> inreg %shif
 ; CHECK-LABEL: ashr_v2_or32_sgpr:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_or_b32 s6, s22, 32
-; CHECK-NEXT:    s_or_b32 s4, s20, 32
-; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
-; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
+; CHECK-NEXT:    s_ashr_i32 s4, s17, s20
+; CHECK-NEXT:    s_ashr_i32 s5, s17, 31
+; CHECK-NEXT:    s_ashr_i32 s6, s19, s22
+; CHECK-NEXT:    s_ashr_i32 s7, s19, 31
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s5
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s6
@@ -438,12 +483,12 @@ define <3 x i64> @ashr_v3_or32_sgpr(<3 x i64> inreg %arg0, <3 x i64> inreg %shif
 ; CHECK-LABEL: ashr_v3_or32_sgpr:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    s_or_b32 s8, s26, 32
-; CHECK-NEXT:    s_or_b32 s6, s24, 32
-; CHECK-NEXT:    s_or_b32 s4, s22, 32
-; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
-; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
-; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
+; CHECK-NEXT:    s_ashr_i32 s4, s17, s22
+; CHECK-NEXT:    s_ashr_i32 s5, s17, 31
+; CHECK-NEXT:    s_ashr_i32 s6, s19, s24
+; CHECK-NEXT:    s_ashr_i32 s7, s19, 31
+; CHECK-NEXT:    s_ashr_i32 s8, s21, s26
+; CHECK-NEXT:    s_ashr_i32 s9, s21, 31
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s5
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s6
@@ -460,20 +505,21 @@ define <4 x i64> @ashr_v4_or32_sgpr(<4 x i64> inreg %arg0, <4 x i64> inreg %shif
 ; CHECK-LABEL: ashr_v4_or32_sgpr:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_or_b32_e32 v0, 32, v0
-; CHECK-NEXT:    s_or_b32 s8, s28, 32
-; CHECK-NEXT:    s_or_b32 s6, s26, 32
-; CHECK-NEXT:    s_or_b32 s4, s24, 32
-; CHECK-NEXT:    s_ashr_i64 s[4:5], s[16:17], s4
-; CHECK-NEXT:    s_ashr_i64 s[6:7], s[18:19], s6
-; CHECK-NEXT:    s_ashr_i64 s[8:9], s[20:21], s8
-; CHECK-NEXT:    v_ashrrev_i64 v[6:7], v0, s[22:23]
+; CHECK-NEXT:    s_ashr_i32 s4, s17, s24
+; CHECK-NEXT:    s_ashr_i32 s5, s17, 31
+; CHECK-NEXT:    s_ashr_i32 s6, s19, s26
+; CHECK-NEXT:    s_ashr_i32 s7, s19, 31
+; CHECK-NEXT:    s_ashr_i32 s8, s21, s28
+; CHECK-NEXT:    s_ashr_i32 s9, s21, 31
+; CHECK-NEXT:    s_ashr_i32 s10, s23, 31
+; CHECK-NEXT:    v_ashrrev_i32_e64 v6, v0, s23
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s4
 ; CHECK-NEXT:    v_mov_b32_e32 v1, s5
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s6
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s7
 ; CHECK-NEXT:    v_mov_b32_e32 v4, s8
 ; CHECK-NEXT:    v_mov_b32_e32 v5, s9
+; CHECK-NEXT:    v_mov_b32_e32 v7, s10
 ; CHECK-NEXT:    s_setpc_b64 s[30:31]
   %or = or <4 x i64> %shift_amt, splat (i64 32)
   %ashr = ashr <4 x i64> %arg0, %or
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index 8319e112f526..4b68f8a4bd19 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -240,6 +240,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -453,6 +454,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W64-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
 ; GFX11W64-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -482,6 +484,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W32-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]
 ; GFX11W32-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W32-NEXT:    global_store_b32 v0, v1, s[2:3]
@@ -514,6 +517,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -544,6 +548,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -882,6 +887,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -926,6 +932,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1285,6 +1292,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1331,6 +1339,7 @@ define amdgpu_kernel void @struct_add_i32_varying_vdata(ptr addrspace(1) %out, p
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1968,6 +1977,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12W64-NEXT:    s_endpgm
@@ -2000,6 +2010,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12W32-NEXT:    s_endpgm
@@ -2338,6 +2349,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -2383,6 +2395,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index f979d01e495b..70211c302829 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -3390,6 +3390,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_readlane_b32 s2, v2, 31
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
@@ -3445,6 +3446,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_add_co_u32 v6, vcc, s2, v8
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
@@ -6954,6 +6956,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_readlane_b32 s2, v2, 31
 ; GFX1264_DPP-NEXT:    v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1264_DPP-NEXT:    v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
@@ -7009,6 +7012,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1264_DPP-NEXT:    v_mov_b32_e32 v9, v5
 ; GFX1264_DPP-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xf1ff
+; GFX1264_DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1264_DPP-NEXT:    v_sub_co_u32 v6, vcc, s2, v8
 ; GFX1264_DPP-NEXT:    s_mov_b32 s2, s6
 ; GFX1264_DPP-NEXT:    s_wait_alu 0xfffd
@@ -8233,6 +8237,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_mad_u16 v0.l, s10, v4.l, s2
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-TRUE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
@@ -8298,6 +8303,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
 ; GFX1264-FAKE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-FAKE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-FAKE16-NEXT:    v_mad_u16 v0, s10, v4, s2
 ; GFX1264-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-FAKE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
@@ -8364,6 +8370,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_mad_u16 v0.l, s8, v4.l, s2
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-TRUE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
@@ -8429,6 +8436,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-FAKE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-FAKE16-NEXT:    v_mad_u16 v0, s8, v4, s2
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
@@ -8818,7 +8826,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT:                       ; implicit-def: $vgpr0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB15_2
 ; GFX7LESS-NEXT:  ; %bb.1:
@@ -9328,7 +9336,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s6, 0
 ; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v4, s7, v0
 ; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
-; GFX7LESS-NEXT:                               ; implicit-def: $vgpr0
+; GFX7LESS-NEXT:    ; implicit-def: $vgpr0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_4
 ; GFX7LESS-NEXT:  ; %bb.1:
@@ -9931,6 +9939,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_mad_u16 v0.l, s10, v4.l, s2
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -9996,6 +10005,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1264-FAKE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-FAKE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-FAKE16-NEXT:    v_mad_u16 v0, s10, v4, s2
 ; GFX1264-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -10062,6 +10072,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_mad_u16 v0.l, s8, v4.l, s2
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -10127,6 +10138,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-FAKE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-FAKE16-NEXT:    v_mad_u16 v0, s8, v4, s2
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -12703,6 +12715,7 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
 ; GFX1264-FAKE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
 ; GFX1264-FAKE16-NEXT:    v_cmp_u_f32_e64 s[0:1], v0, v0
 ; GFX1264-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1264-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
 ; GFX1264-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX1264-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[0:1]
@@ -12816,6 +12829,7 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
 ; GFX1232-FAKE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
 ; GFX1232-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v0, v0
 ; GFX1232-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1232-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
 ; GFX1232-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX1232-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 8e0b3cb9aa1d..17737cccec7c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -405,6 +405,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
 ; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
 ; GFX1164-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
@@ -1901,6 +1902,7 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
 ; GFX1164-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1164-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1164-NEXT:    s_mov_b32 s2, -1
 ; GFX1164-NEXT:    v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2]
@@ -1937,6 +1939,7 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
 ; GFX1132-NEXT:    v_readfirstlane_b32 s5, v1
 ; GFX1132-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_mad_u64_u32 v[0:1], null, s2, v2, s[4:5]
 ; GFX1132-NEXT:    s_mov_b32 s2, -1
 ; GFX1132-NEXT:    v_mad_u64_u32 v[3:4], null, s3, v2, v[1:2]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 6a82dbeec5e2..0a06fe4ea949 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -239,6 +239,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -452,6 +453,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W64-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
 ; GFX11W64-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -481,6 +483,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W32-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]
 ; GFX11W32-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W32-NEXT:    global_store_b32 v0, v1, s[2:3]
@@ -513,6 +516,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -543,6 +547,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -881,6 +886,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -925,6 +931,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1562,6 +1569,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12W64-NEXT:    s_endpgm
@@ -1594,6 +1602,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12W32-NEXT:    s_endpgm
@@ -1932,6 +1941,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1977,6 +1987,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index dd4c0b0625ea..bc0bec4772e5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -247,6 +247,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -466,6 +467,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W64-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX11W64-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W64-NEXT:    v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
 ; GFX11W64-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W64-NEXT:    global_store_b32 v0, v1, s[0:1]
@@ -496,6 +498,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX11W32-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11W32-NEXT:    v_mad_u64_u32 v[1:2], null, s0, v0, s[4:5]
 ; GFX11W32-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11W32-NEXT:    global_store_b32 v0, v1, s[2:3]
@@ -529,6 +532,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W64-NEXT:    v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -559,6 +563,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s4, v1
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_mad_co_u64_u32 v[0:1], null, s0, v0, s[4:5]
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
@@ -904,6 +909,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_add_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -948,6 +954,7 @@ define amdgpu_kernel void @add_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    s_wait_loadcnt 0x0
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12W32-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -1729,6 +1736,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12W64-NEXT:    s_endpgm
@@ -1761,6 +1769,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s0, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s0, v0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[2:3]
 ; GFX12W32-NEXT:    s_endpgm
@@ -2106,6 +2115,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W64-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W64-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W64-NEXT:    s_wait_alu 0xf1ff
+; GFX12W64-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W64-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W64-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W64-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -2151,6 +2161,7 @@ define amdgpu_kernel void @sub_i32_varying_vdata(ptr addrspace(1) %out, ptr addr
 ; GFX12W32-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12W32-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12W32-NEXT:    s_wait_alu 0xf1ff
+; GFX12W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12W32-NEXT:    v_sub_nc_u32_e32 v0, s2, v0
 ; GFX12W32-NEXT:    s_wait_kmcnt 0x0
 ; GFX12W32-NEXT:    global_store_b32 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index a597faa028f2..5b4866c38679 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -158,7 +158,7 @@ define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) {
 ; GFX-950-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v7
 ; GFX-950-NEXT:    v_cndmask_b32_e64 v2, -1, 1, s[2:3]
 ; GFX-950-NEXT:    v_add_u32_e32 v2, v6, v2
-; GFX-950-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX-950-NEXT:    s_or_b64 vcc, s[0:1], vcc
 ; GFX-950-NEXT:    v_cvt_f32_f64_e32 v5, v[0:1]
 ; GFX-950-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
 ; GFX-950-NEXT:    v_cvt_f64_f32_e32 v[2:3], v5
@@ -168,7 +168,7 @@ define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) {
 ; GFX-950-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v6
 ; GFX-950-NEXT:    v_cndmask_b32_e64 v0, -1, 1, s[2:3]
 ; GFX-950-NEXT:    v_add_u32_e32 v0, v5, v0
-; GFX-950-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX-950-NEXT:    s_or_b64 vcc, s[0:1], vcc
 ; GFX-950-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
 ; GFX-950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v4
 ; GFX-950-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 3eba106b861c..348862d4d8ce 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -387,6 +387,7 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -2381,6 +2382,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -2411,6 +2413,7 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -4307,6 +4310,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -4341,6 +4345,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -4388,6 +4393,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -4422,6 +4428,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6132,6 +6139,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6178,6 +6186,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6225,6 +6234,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6270,6 +6280,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -7485,6 +7496,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -9903,6 +9915,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index c17225594164..ab867b089b87 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -379,6 +379,7 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -1591,6 +1592,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -1623,6 +1625,7 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3415,6 +3418,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3451,6 +3455,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3498,6 +3503,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3535,6 +3541,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5264,6 +5271,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5310,6 +5318,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5357,6 +5366,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5402,6 +5412,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6775,6 +6786,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6807,6 +6819,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -7995,6 +8008,7 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v6, v8, s4
@@ -8441,6 +8455,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8492,6 +8507,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8532,6 +8548,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8582,6 +8599,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 56719dccbd08..1a25904dd553 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -379,6 +379,7 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -1591,6 +1592,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -1623,6 +1625,7 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v7
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v8
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3415,6 +3418,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3451,6 +3455,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3498,6 +3503,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -3535,6 +3541,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5264,6 +5271,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5310,6 +5318,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5357,6 +5366,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -5402,6 +5412,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6775,6 +6786,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -6807,6 +6819,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -7995,6 +8008,7 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v0, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s4, v0, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v6, v8, s4
@@ -8441,6 +8455,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8492,6 +8507,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8532,6 +8548,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -8582,6 +8599,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir b/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
index ae3fa153f381..9ff904cc31de 100644
--- a/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
+++ b/llvm/test/CodeGen/AMDGPU/cmp_shrink.mir
@@ -10,3 +10,16 @@ body:             |
     ; GCN: S_CMP_LT_I32 65, 1, implicit-def $scc
     S_CMP_GT_I32 1, 65, implicit-def $scc
 ...
+
+---
+name:             shrink_icmp
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: shrink_icmp
+    ; GCN: %op0:sgpr_32 = COPY $sgpr0
+    ; GCN-NEXT: %cond:sreg_64 = V_CMPX_GE_U32_e64 5, %op0, implicit-def $exec, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    %op0:sgpr_32 = COPY $sgpr0
+    %cond:sreg_64 = V_CMPX_LE_U32_e64 %op0, 5, implicit-def $exec, implicit $exec
+    S_NOP 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll b/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
index f0e7cba6924d..ce4db2f84774 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-add-zext-xor.ll
@@ -48,15 +48,17 @@ define i32 @combine_add_zext_xor(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_branch .LBB0_2
 ; GFX1100-NEXT:  .LBB0_1: ; %bb9
 ; GFX1100-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    s_xor_b32 s1, s1, -1
 ; GFX1100-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0xfffffbe6, v1
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_add_nc_u32_e32 v2, v1, v0
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB0_4
 ; GFX1100-NEXT:  .LBB0_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr1
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB0_1
@@ -139,15 +141,17 @@ define i32 @combine_sub_zext_xor(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_branch .LBB1_2
 ; GFX1100-NEXT:  .LBB1_1: ; %bb9
 ; GFX1100-NEXT:    ; in Loop: Header=BB1_2 Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    s_xor_b32 s1, s1, -1
 ; GFX1100-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0xfffffbe6, v1
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_sub_nc_u32_e32 v2, v1, v0
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB1_4
 ; GFX1100-NEXT:  .LBB1_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr1
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB1_1
@@ -239,6 +243,7 @@ define i32 @combine_add_zext_or(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB2_4
 ; GFX1100-NEXT:  .LBB2_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr2
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB2_1
@@ -335,6 +340,7 @@ define i32 @combine_sub_zext_or(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB3_4
 ; GFX1100-NEXT:  .LBB3_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr2
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB3_1
@@ -421,15 +427,16 @@ define i32 @combine_add_zext_and(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_branch .LBB4_2
 ; GFX1100-NEXT:  .LBB4_1: ; %bb9
 ; GFX1100-NEXT:    ; in Loop: Header=BB4_2 Depth=1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1100-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0xfffffbe6, v1
 ; GFX1100-NEXT:    s_and_b32 s1, s1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_add_nc_u32_e32 v1, v1, v0
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB4_4
 ; GFX1100-NEXT:  .LBB4_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr1
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB4_1
@@ -512,15 +519,16 @@ define i32 @combine_sub_zext_and(i32 inreg %cond) {
 ; GFX1100-NEXT:    s_branch .LBB5_2
 ; GFX1100-NEXT:  .LBB5_1: ; %bb9
 ; GFX1100-NEXT:    ; in Loop: Header=BB5_2 Depth=1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1100-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 0xfffffbe6, v1
 ; GFX1100-NEXT:    s_and_b32 s1, s1, vcc_lo
+; GFX1100-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1100-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s1
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    v_sub_nc_u32_e32 v1, v1, v0
 ; GFX1100-NEXT:    s_cbranch_vccz .LBB5_4
 ; GFX1100-NEXT:  .LBB5_2: ; %.a
 ; GFX1100-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-NEXT:    s_and_b32 vcc_lo, exec_lo, s0
 ; GFX1100-NEXT:    ; implicit-def: $sgpr1
 ; GFX1100-NEXT:    s_cbranch_vccnz .LBB5_1
diff --git a/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll b/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
index b98c81db5da9..4b0fc9380b29 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-and-sext-bool.ll
@@ -1,13 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}and_i1_sext_bool:
-; GCN: v_cmp_{{gt|le}}_u32_e{{32|64}} [[CC:[^,]+]], v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: v_cndmask_b32_e{{32|64}} [[VAL:v[0-9]+]], 0, v{{[0-9]+}}, [[CC]]
-; GCN: store_dword {{.*}}[[VAL]]
-; GCN-NOT: v_cndmask_b32_e64 v{{[0-9]+}}, {{0|-1}}, {{0|-1}}
-; GCN-NOT: v_and_b32_e32
-
 define amdgpu_kernel void @and_i1_sext_bool(ptr addrspace(1) nocapture %arg) {
+; GCN-LABEL: and_i1_sext_bool:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT:    v_cmp_gt_u32_e32 vcc, v0, v1
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v4, vcc
+; GCN-NEXT:    buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT:    s_endpgm
 bb:
   %x = tail call i32 @llvm.amdgcn.workitem.id.x()
   %y = tail call i32 @llvm.amdgcn.workitem.id.y()
@@ -20,37 +28,40 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_fcmp:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_fcmp(float %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_fcmp:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %cmp = fcmp oeq float %x, 0.0
   %sext = sext i1 %cmp to i32
   %and = and i32 %sext, %y
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_fpclass:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 [[K:v[0-9]+]], 0x7b
-; GCN-NEXT: v_cmp_class_f32_e32 vcc, v0, [[K]]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_fpclass(float %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_fpclass:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x7b
+; GCN-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %class = call i1 @llvm.is.fpclass(float %x, i32 123)
   %sext = sext i1 %class to i32
   %and = and i32 %sext, %y
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_uadd_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_uadd_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_uadd_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -58,12 +69,13 @@ define i32 @and_sext_bool_uadd_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_usub_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_usub_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_usub_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_sub_i32_e32 v0, vcc, v0, v1
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -71,15 +83,16 @@ define i32 @and_sext_bool_usub_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_sadd_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GCN-NEXT: v_add_i32_e64 v2, s[4:5], v0, v1
-; GCN-NEXT: v_cmp_lt_i32_e64 s[4:5], v2, v0
-; GCN-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_sadd_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_sadd_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GCN-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; GCN-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GCN-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -87,15 +100,16 @@ define i32 @and_sext_bool_sadd_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_ssub_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, 0, v1
-; GCN-NEXT: v_add_i32_e64 v2, s[4:5], v0, v1
-; GCN-NEXT: v_cmp_lt_i32_e64 s[4:5], v2, v0
-; GCN-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_ssub_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_ssub_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; GCN-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v1
+; GCN-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
+; GCN-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -103,15 +117,16 @@ define i32 @and_sext_bool_ssub_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_smul_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_hi_i32 v2, v0, v1
-; GCN-NEXT: v_mul_lo_u32 v0, v0, v1
-; GCN-NEXT: v_ashrrev_i32_e32 v0, 31, v0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, v2, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_smul_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_smul_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_i32 v2, v0, v1
+; GCN-NEXT:    v_mul_lo_u32 v0, v0, v1
+; GCN-NEXT:    v_ashrrev_i32_e32 v0, 31, v0
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, v2, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
@@ -119,13 +134,14 @@ define i32 @and_sext_bool_smul_w_overflow(i32 %x, i32 %y) {
   ret i32 %and
 }
 
-; GCN-LABEL: {{^}}and_sext_bool_umul_w_overflow:
-; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mul_hi_u32 v0, v0, v1
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GCN-NEXT: s_setpc_b64
 define i32 @and_sext_bool_umul_w_overflow(i32 %x, i32 %y) {
+; GCN-LABEL: and_sext_bool_umul_w_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_mul_hi_u32 v0, v0, v1
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e32 v0, 0, v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %uadd = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %x, i32 %y)
   %carry = extractvalue { i32, i1 } %uadd, 1
   %sext = sext i1 %carry to i32
diff --git a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
index b7a1749be18b..57ab371d5b6f 100644
--- a/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
+++ b/llvm/test/CodeGen/AMDGPU/convergence-laneops.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -stop-after=amdgpu-isel -mtriple=amdgcn-- -mcpu=gfx1100 -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,ISEL %s
-; RUN: not --crash llc -mtriple=amdgcn--amdhsa -mcpu=1100 -verify-machineinstrs < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
+; RUN: not --crash llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s 2>&1 | FileCheck --check-prefix=CHECK-ERROR %s
 
 ; FIXME: Merge these tests with existing lane op tests (llvm.amdgcn.readlane.ll, llvm.amdgcn.writelane.ll ...) once the crash is fixed.
 
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 51398a45055e..f8e13fcdd227 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -475,21 +475,28 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -500,6 +507,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -1035,10 +1043,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
@@ -2656,21 +2664,28 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -2681,6 +2696,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -3216,10 +3232,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
index 4a3a44d09210..ea28af450838 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-ctpop.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @s_ctpop_i32(ptr addrspace(1) noalias %out, i32 %val)
 
 ; GCN-LABEL: name:            s_ctpop_i64
 ; GCN: %[[BCNT:[0-9]+]]:sreg_32 = S_BCNT1_I32_B64
-; GCN: %[[SREG1:[0-9]+]]:sreg_32 = COPY %[[BCNT]]
+; GCN: %[[SREG1:[0-9]+]]:sreg_32 = COPY killed %[[BCNT]]
 ; GCN: %[[SREG2:[0-9]+]]:sreg_32 = S_MOV_B32 0
 ; GCN: REG_SEQUENCE killed %[[SREG1]], %subreg.sub0, killed %[[SREG2]], %subreg.sub1
 define amdgpu_kernel void @s_ctpop_i64(ptr addrspace(1) noalias %out, i64 %val) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 447e0ca4f58e..bd0b5d423cb9 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -2261,7 +2261,6 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
 ; GFX9-SDAG-NEXT:    s_mov_b32 s32, s34
 ; GFX9-SDAG-NEXT:    s_mov_b32 s34, s12
 ; GFX9-SDAG-NEXT:    s_mov_b32 s33, s11
-; GFX9-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-GISEL-LABEL: test_dynamic_stackalloc_device_control_flow:
@@ -2328,7 +2327,6 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
 ; GFX9-GISEL-NEXT:    s_mov_b32 s32, s34
 ; GFX9-GISEL-NEXT:    s_mov_b32 s34, s12
 ; GFX9-GISEL-NEXT:    s_mov_b32 s33, s11
-; GFX9-GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-GISEL-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-LABEL: test_dynamic_stackalloc_device_control_flow:
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index bb66bb319d48..a07f1d8a0294 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -731,7 +731,6 @@ define <4 x i16> @vec_16xi16_extract_4xi16(ptr addrspace(1) %p0, ptr addrspace(1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX9-NEXT:  .LBB3_3: ; %exit
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v5 op_sel_hi:[0,0]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
@@ -973,7 +972,6 @@ define <4 x i16> @vec_16xi16_extract_4xi16_2(ptr addrspace(1) %p0, ptr addrspace
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; kill: killed $vgpr0 killed $vgpr1
 ; GFX9-NEXT:  .LBB4_3: ; %exit
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_pk_ashrrev_i16 v0, 15, v7 op_sel_hi:[0,1]
 ; GFX9-NEXT:    s_movk_i32 s4, 0x8000
 ; GFX9-NEXT:    v_or_b32_e32 v1, 0xffff8000, v0
@@ -1217,7 +1215,6 @@ define <4 x half> @vec_16xf16_extract_4xf16(ptr addrspace(1) %p0, ptr addrspace(
 ; GFX9-NEXT:  .LBB5_3: ; %exit
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v4
 ; GFX9-NEXT:    v_mov_b32_e32 v3, 0x3800
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v0, v1, vcc
@@ -1595,7 +1592,6 @@ define amdgpu_gfx <8 x i16> @vec_16xi16_extract_8xi16_0(i1 inreg %cond, ptr addr
 ; GFX9-NEXT:    s_movk_i32 s34, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_gt_u16_e32 vcc, s35, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v1, vcc
 ; GFX9-NEXT:    v_cmp_gt_u16_sdwa vcc, v7, s34 src0_sel:WORD_1 src1_sel:DWORD
@@ -1933,7 +1929,6 @@ define amdgpu_gfx <8 x half> @vec_16xf16_extract_8xf16_0(i1 inreg %cond, ptr add
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0x3800
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0x3900
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0x3d00
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_cmp_ge_f16_e32 vcc, 0.5, v7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v1, v2, vcc
 ; GFX9-NEXT:    v_cmp_nle_f16_sdwa vcc, v7, v0 src0_sel:WORD_1 src1_sel:DWORD
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
index 41082821bafe..a8d94146195f 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
@@ -127,7 +127,6 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -138,7 +137,6 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB1_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff8000
 ; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, -1, v1, vcc
@@ -197,7 +195,6 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -208,7 +205,6 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB2_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0xffff8000
 ; GCN-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v0, v1, -1, vcc
@@ -305,7 +301,6 @@ define <8 x i64> @extract_8xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[6:7], 0, v[10:11]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[8:9], 0, v[12:13]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[10:11], 0, v[14:15]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[12:13], 0, v[16:17]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[14:15], 0, v[18:19]
 ; GCN-NEXT:    v_cmp_gt_i64_e64 s[16:17], 0, v[4:5]
@@ -376,7 +371,6 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -387,7 +381,6 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB4_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0xbff00000
 ; GCN-NEXT:    v_cmp_lt_f64_e32 vcc, -1.0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e64 v1, v0, -2.0, vcc
@@ -446,7 +439,6 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_mov_b32 s11, 0xf000
 ; GCN-NEXT:    s_mov_b32 s8, s10
 ; GCN-NEXT:    s_mov_b32 s9, s10
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
@@ -457,7 +449,6 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:  .LBB5_4: ; %exit
 ; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0xbff00000
 ; GCN-NEXT:    v_cmp_nlt_f64_e32 vcc, -1.0, v[4:5]
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, -2.0, v0, vcc
@@ -554,7 +545,6 @@ define <8 x double> @extract_8xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[6:7], -1.0, v[10:11]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[8:9], -1.0, v[12:13]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[10:11], -1.0, v[14:15]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[12:13], -1.0, v[16:17]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[14:15], -1.0, v[18:19]
 ; GCN-NEXT:    v_cmp_nlt_f64_e64 s[16:17], -1.0, v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/fdot2.ll b/llvm/test/CodeGen/AMDGPU/fdot2.ll
index 776816d6aa0e..b61981bbcece 100644
--- a/llvm/test/CodeGen/AMDGPU/fdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdot2.ll
@@ -1,28 +1,53 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX900
-; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX906-DL-UNSAFE
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX900
+; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX906-DL-UNSAFE
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=preserve-sign -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-CONTRACT
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DENORM-CONTRACT
-; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math -mattr="+dot7-insts,-dot10-insts" -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DOT10-DISABLED
+; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -mattr="+dot7-insts,-dot10-insts" -verify-machineinstrs < %s | FileCheck %s  -check-prefixes=GCN,GFX906-DOT10-DISABLED
 ; (fadd (fmul S1.x, S2.x), (fadd (fmul (S1.y, S2.y), z))) -> (fdot2 S1, S2, z)
 
 ; Tests to make sure fdot2 is not generated when vector elements of dot-product expressions
 ; are not converted from f16 to f32.
-; GCN-LABEL: {{^}}dotproduct_f16
+; GCN-LABEL: {{^}}dotproduct_f16_contract
 ; GFX900: v_fma_f16
 ; GFX900: v_fma_f16
 
-; GFX906: v_mul_f16_e32
-; GFX906: v_mul_f16_e32
-
 ; GFX906-DL-UNSAFE:  v_fma_f16
 ; GFX10-CONTRACT: v_fmac_f16
 
 ; GFX906-CONTRACT: v_mac_f16_e32
 ; GFX906-DENORM-CONTRACT: v_fma_f16
 ; GFX906-DOT10-DISABLED: v_fma_f16
+
+define amdgpu_kernel void @dotproduct_f16_contract(ptr addrspace(1) %src1,
+                                                   ptr addrspace(1) %src2,
+                                                   ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <2 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <2 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <2 x half> %src1.vec, i64 0
+  %src2.el1 = extractelement <2 x half> %src2.vec, i64 0
+
+  %src1.el2 = extractelement <2 x half> %src1.vec, i64 1
+  %src2.el2 = extractelement <2 x half> %src2.vec, i64 1
+
+  %mul2 = fmul contract half %src1.el2, %src2.el2
+  %mul1 = fmul contract half %src1.el1, %src2.el1
+  %acc = load half, ptr addrspace(1) %dst, align 2
+  %acc1 = fadd contract half %mul2, %acc
+  %acc2 = fadd contract half %mul1, %acc1
+  store half %acc2, ptr addrspace(1) %dst, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}dotproduct_f16
+
+; GFX906: v_mul_f16_e32
+; GFX906: v_mul_f16_e32
+
 define amdgpu_kernel void @dotproduct_f16(ptr addrspace(1) %src1,
                                           ptr addrspace(1) %src2,
                                           ptr addrspace(1) nocapture %dst) {
@@ -45,18 +70,12 @@ entry:
   ret void
 }
 
-
 ; We only want to generate fdot2 if:
 ; - vector element of dot product is converted from f16 to f32, and
 ; - the vectors are of type <2 x half>, and
 ; - "dot10-insts" is enabled
 
-; GCN-LABEL: {{^}}dotproduct_f16_f32
-; GFX900: v_mad_mix_f32
-; GFX900: v_mad_mix_f32
-
-; GFX906: v_mad_f32
-; GFX906: v_mac_f32_e32
+; GCN-LABEL: {{^}}dotproduct_f16_f32_contract
 
 ; GFX906-DL-UNSAFE: v_dot2_f32_f16
 ; GFX10-DL-UNSAFE: v_dot2c_f32_f16
@@ -65,6 +84,39 @@ entry:
 
 ; GFX906-DENORM-CONTRACT: v_dot2_f32_f16
 ; GFX906-DOT10-DISABLED: v_fma_mix_f32
+define amdgpu_kernel void @dotproduct_f16_f32_contract(ptr addrspace(1) %src1,
+                                                       ptr addrspace(1) %src2,
+                                                       ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <2 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <2 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <2 x half> %src1.vec, i64 0
+  %csrc1.el1 = fpext half %src1.el1 to float
+  %src2.el1 = extractelement <2 x half> %src2.vec, i64 0
+  %csrc2.el1 = fpext half %src2.el1 to float
+
+  %src1.el2 = extractelement <2 x half> %src1.vec, i64 1
+  %csrc1.el2 = fpext half %src1.el2 to float
+  %src2.el2 = extractelement <2 x half> %src2.vec, i64 1
+  %csrc2.el2 = fpext half %src2.el2 to float
+
+  %mul2 = fmul contract float %csrc1.el2, %csrc2.el2
+  %mul1 = fmul contract float %csrc1.el1, %csrc2.el1
+  %acc = load float, ptr addrspace(1) %dst, align 4
+  %acc1 = fadd contract float %mul2, %acc
+  %acc2 = fadd contract float %mul1, %acc1
+  store float %acc2, ptr addrspace(1) %dst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}dotproduct_f16_f32
+; GFX900: v_mad_mix_f32
+; GFX900: v_mad_mix_f32
+
+; GFX906: v_mad_f32
+; GFX906: v_mac_f32_e32
+
 define amdgpu_kernel void @dotproduct_f16_f32(ptr addrspace(1) %src1,
                                               ptr addrspace(1) %src2,
                                               ptr addrspace(1) nocapture %dst) {
@@ -96,6 +148,39 @@ entry:
 ; - the vectors are of type <2 x half>, and
 ; - "dot10-insts" is enabled
 
+; GCN-LABEL: {{^}}dotproduct_diffvecorder_contract
+; GFX906-DL-UNSAFE: v_dot2_f32_f16
+; GFX10-DL-UNSAFE: v_dot2c_f32_f16
+
+; GFX906-CONTRACT: v_dot2_f32_f16
+; GFX906-DENORM-CONTRACT: v_dot2_f32_f16
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
+define amdgpu_kernel void @dotproduct_diffvecorder_contract(ptr addrspace(1) %src1,
+                                                            ptr addrspace(1) %src2,
+                                                            ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <2 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <2 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <2 x half> %src1.vec, i64 0
+  %csrc1.el1 = fpext half %src1.el1 to float
+  %src2.el1 = extractelement <2 x half> %src2.vec, i64 0
+  %csrc2.el1 = fpext half %src2.el1 to float
+
+  %src1.el2 = extractelement <2 x half> %src1.vec, i64 1
+  %csrc1.el2 = fpext half %src1.el2 to float
+  %src2.el2 = extractelement <2 x half> %src2.vec, i64 1
+  %csrc2.el2 = fpext half %src2.el2 to float
+
+  %mul2 = fmul contract float %csrc2.el2, %csrc1.el2
+  %mul1 = fmul contract float %csrc1.el1, %csrc2.el1
+  %acc = load float, ptr addrspace(1) %dst, align 4
+  %acc1 = fadd contract float %mul2, %acc
+  %acc2 = fadd contract float %mul1, %acc1
+  store float %acc2, ptr addrspace(1) %dst, align 4
+  ret void
+}
+
 ; GCN-LABEL: {{^}}dotproduct_diffvecorder
 ; GFX900: v_mad_mix_f32
 ; GFX900: v_mad_mix_f32
@@ -103,12 +188,6 @@ entry:
 ; GFX906: v_mad_f32
 ; GFX906: v_mac_f32_e32
 
-; GFX906-DL-UNSAFE: v_dot2_f32_f16
-; GFX10-DL-UNSAFE: v_dot2c_f32_f16
-
-; GFX906-CONTRACT: v_dot2_f32_f16
-; GFX906-DENORM-CONTRACT: v_dot2_f32_f16
-; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @dotproduct_diffvecorder(ptr addrspace(1) %src1,
                                                    ptr addrspace(1) %src2,
                                                    ptr addrspace(1) nocapture %dst) {
@@ -136,17 +215,45 @@ entry:
 }
 
 ; Tests to make sure dot product is not generated when the vectors are not of <2 x half>.
-; GCN-LABEL: {{^}}dotproduct_v4f16
-; GFX900: v_mad_mix_f32
-
-; GFX906: v_mad_f32
-; GFX906: v_mac_f32_e32
+; GCN-LABEL: {{^}}dotproduct_v4f16_contract
 
 ; GCN-DL-UNSAFE: v_fma_mix_f32
 
 ; GFX906-CONTRACT: v_fma_mix_f32
 ; GFX906-DENORM-CONTRACT: v_fma_mix_f32
 ; GFX906-DOT10-DISABLED: v_fma_mix_f32
+define amdgpu_kernel void @dotproduct_v4f16_contract(ptr addrspace(1) %src1,
+                                                     ptr addrspace(1) %src2,
+                                                     ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <4 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <4 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <4 x half> %src1.vec, i64 0
+  %csrc1.el1 = fpext half %src1.el1 to float
+  %src2.el1 = extractelement <4 x half> %src2.vec, i64 0
+  %csrc2.el1 = fpext half %src2.el1 to float
+
+  %src1.el2 = extractelement <4 x half> %src1.vec, i64 1
+  %csrc1.el2 = fpext half %src1.el2 to float
+  %src2.el2 = extractelement <4 x half> %src2.vec, i64 1
+  %csrc2.el2 = fpext half %src2.el2 to float
+
+  %mul2 = fmul contract float %csrc1.el2, %csrc2.el2
+  %mul1 = fmul float %csrc1.el1, %csrc2.el1
+  %acc = load float, ptr addrspace(1) %dst, align 4
+  %acc1 = fadd contract float %mul2, %acc
+  %acc2 = fadd contract float %mul1, %acc1
+  store float %acc2, ptr addrspace(1) %dst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}dotproduct_v4f16
+; GFX900: v_mad_mix_f32
+
+; GFX906: v_mad_f32
+; GFX906: v_mac_f32_e32
+
 define amdgpu_kernel void @dotproduct_v4f16(ptr addrspace(1) %src1,
                                             ptr addrspace(1) %src2,
                                             ptr addrspace(1) nocapture %dst) {
@@ -173,6 +280,39 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}NotAdotproductContract
+
+; GCN-DL-UNSAFE: v_fma_mix_f32
+
+; GFX906-CONTRACT: v_fma_mix_f32
+; GFX906-DENORM-CONTRACT: v_fma_mix_f32
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
+define amdgpu_kernel void @NotAdotproductContract(ptr addrspace(1) %src1,
+                                                  ptr addrspace(1) %src2,
+                                                  ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <2 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <2 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <2 x half> %src1.vec, i64 0
+  %csrc1.el1 = fpext half %src1.el1 to float
+  %src2.el1 = extractelement <2 x half> %src2.vec, i64 0
+  %csrc2.el1 = fpext half %src2.el1 to float
+
+  %src1.el2 = extractelement <2 x half> %src1.vec, i64 1
+  %csrc1.el2 = fpext half %src1.el2 to float
+  %src2.el2 = extractelement <2 x half> %src2.vec, i64 1
+  %csrc2.el2 = fpext half %src2.el2 to float
+
+  %mul2 = fmul contract float %csrc1.el2, %csrc1.el1
+  %mul1 = fmul contract float %csrc2.el1, %csrc2.el2
+  %acc = load float, ptr addrspace(1) %dst, align 4
+  %acc1 = fadd contract float %mul2, %acc
+  %acc2 = fadd contract float %mul1, %acc1
+  store float %acc2, ptr addrspace(1) %dst, align 4
+  ret void
+}
+
 ; GCN-LABEL: {{^}}NotAdotproduct
 ; GFX900: v_mad_mix_f32
 ; GFX900: v_mad_mix_f32
@@ -180,11 +320,6 @@ entry:
 ; GFX906: v_mad_f32
 ; GFX906: v_mac_f32_e32
 
-; GCN-DL-UNSAFE: v_fma_mix_f32
-
-; GFX906-CONTRACT: v_fma_mix_f32
-; GFX906-DENORM-CONTRACT: v_fma_mix_f32
-; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @NotAdotproduct(ptr addrspace(1) %src1,
                                           ptr addrspace(1) %src2,
                                           ptr addrspace(1) nocapture %dst) {
@@ -211,6 +346,39 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}Diff_Idx_NotAdotproductContract
+
+; GCN-DL-UNSAFE: v_fma_mix_f32
+
+; GFX906-CONTRACT: v_fma_mix_f32
+; GFX906-DENORM-CONTRACT: v_fma_mix_f32
+; GFX906-DOT10-DISABLED: v_fma_mix_f32
+define amdgpu_kernel void @Diff_Idx_NotAdotproductContract(ptr addrspace(1) %src1,
+                                                           ptr addrspace(1) %src2,
+                                                           ptr addrspace(1) nocapture %dst) {
+entry:
+  %src1.vec = load <2 x half>, ptr addrspace(1) %src1
+  %src2.vec = load <2 x half>, ptr addrspace(1) %src2
+
+  %src1.el1 = extractelement <2 x half> %src1.vec, i64 0
+  %csrc1.el1 = fpext half %src1.el1 to float
+  %src2.el1 = extractelement <2 x half> %src2.vec, i64 0
+  %csrc2.el1 = fpext half %src2.el1 to float
+
+  %src1.el2 = extractelement <2 x half> %src1.vec, i64 1
+  %csrc1.el2 = fpext half %src1.el2 to float
+  %src2.el2 = extractelement <2 x half> %src2.vec, i64 1
+  %csrc2.el2 = fpext half %src2.el2 to float
+
+  %mul2 = fmul contract float %csrc1.el2, %csrc2.el1
+  %mul1 = fmul contract float %csrc1.el1, %csrc2.el2
+  %acc = load float, ptr addrspace(1) %dst, align 4
+  %acc1 = fadd contract float %mul2, %acc
+  %acc2 = fadd contract float %mul1, %acc1
+  store float %acc2, ptr addrspace(1) %dst, align 4
+  ret void
+}
+
 ; GCN-LABEL: {{^}}Diff_Idx_NotAdotproduct
 ; GFX900: v_mad_mix_f32
 ; GFX900: v_mad_mix_f32
@@ -218,11 +386,6 @@ entry:
 ; GFX906: v_mad_f32
 ; GFX906: v_mac_f32_e32
 
-; GCN-DL-UNSAFE: v_fma_mix_f32
-
-; GFX906-CONTRACT: v_fma_mix_f32
-; GFX906-DENORM-CONTRACT: v_fma_mix_f32
-; GFX906-DOT10-DISABLED: v_fma_mix_f32
 define amdgpu_kernel void @Diff_Idx_NotAdotproduct(ptr addrspace(1) %src1,
                                                    ptr addrspace(1) %src2,
                                                    ptr addrspace(1) nocapture %dst) {
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
index 9b6a2f3a1aa1..59184499de2f 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-true16.mir
@@ -108,6 +108,32 @@ body:             |
     %4:sreg_32 = S_FMAC_F16 %3:sreg_32, %3:sreg_32, %2:sreg_32, implicit $mode
 ...
 
+---
+name:            legalize_with_multi_user
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: legalize_with_multi_user
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[DEF]], %subreg.lo16, [[DEF1]], %subreg.hi16
+    ; GCN-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_F16_t16_e64 0, [[REG_SEQUENCE]].lo16, 0, 1, 0, 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32768
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_ADD_F16_t16_e64_]], %subreg.lo16, [[DEF3]], %subreg.hi16
+    ; GCN-NEXT: [[V_PK_FMA_F16_:%[0-9]+]]:vgpr_32 = V_PK_FMA_F16 11, [[S_MOV_B32_]], 0, [[REG_SEQUENCE1]], 8, [[DEF2]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF4:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_ADD_F16_t16_e64_]], %subreg.lo16, [[DEF4]], %subreg.hi16
+    ; GCN-NEXT: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[REG_SEQUENCE2]], [[S_MOV_B32_]], implicit $exec
+    %0:vgpr_16 = IMPLICIT_DEF
+    %1:sreg_32 = COPY %0:vgpr_16
+    %2:sreg_32 = S_ADD_F16 %1:sreg_32, 1, implicit $mode
+    %3:sreg_32 = S_MOV_B32 32768
+    %4:vgpr_32 = IMPLICIT_DEF
+    %5:vgpr_32 = V_PK_FMA_F16 11, %3:sreg_32, 0, %2:sreg_32, 8, %4:vgpr_32, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %6:sreg_32 = S_XOR_B32 %2:sreg_32, %3:sreg_32, implicit-def dead $scc
+...
+
 ---
 name:            vgpr16_to_spgr32
 body:             |
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index cfe4d24d427e..8581e4d03026 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -12426,13 +12426,14 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12483,13 +12484,14 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12890,13 +12892,14 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12947,13 +12950,14 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13355,13 +13359,14 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB49_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13410,13 +13415,14 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB49_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13804,13 +13810,14 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB50_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13859,13 +13866,14 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB50_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -15417,13 +15425,14 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB54_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -15475,13 +15484,14 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB54_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -15885,13 +15895,14 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB55_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -15941,13 +15952,14 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB55_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index d2cbc25bf7e0..883063b5471c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -6474,13 +6474,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB27_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6882,13 +6883,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB28_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7671,13 +7673,14 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB30_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -8068,13 +8071,14 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB31_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9068,13 +9072,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9479,13 +9484,14 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB35_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10283,13 +10289,14 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10340,13 +10347,14 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10748,13 +10756,14 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10805,13 +10814,14 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11648,13 +11658,14 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11703,13 +11714,14 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12098,13 +12110,14 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12153,13 +12166,14 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13281,13 +13295,14 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13339,13 +13354,14 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13750,13 +13766,14 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13806,13 +13823,14 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17588,6 +17606,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18024,6 +18043,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18467,6 +18487,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -19393,6 +19414,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 805848fc3e1c..c603421ca15b 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -6474,13 +6474,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB27_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6882,13 +6883,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB28_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7671,13 +7673,14 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB30_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -8068,13 +8071,14 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB31_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9068,13 +9072,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9479,13 +9484,14 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB35_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10283,13 +10289,14 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10340,13 +10347,14 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10748,13 +10756,14 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10805,13 +10814,14 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11648,13 +11658,14 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11703,13 +11714,14 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12098,13 +12110,14 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12153,13 +12166,14 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13281,13 +13295,14 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13339,13 +13354,14 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13750,13 +13766,14 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13806,13 +13823,14 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17588,6 +17606,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18024,6 +18043,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18467,6 +18487,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -19393,6 +19414,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index e0138d58963c..c987effec3be 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9871,13 +9871,14 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB33_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9928,13 +9929,14 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB33_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10335,13 +10337,14 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10392,13 +10395,14 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11233,13 +11237,14 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB36_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11288,13 +11293,14 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB36_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11682,13 +11688,14 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11737,13 +11744,14 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12862,13 +12870,14 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12920,13 +12929,14 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v5, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13330,13 +13340,14 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13386,13 +13397,14 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17023,6 +17035,7 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -17459,6 +17472,7 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -17902,6 +17916,7 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18828,6 +18843,7 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/fma-combine.ll b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
index a96d022b66f1..c79cf87712dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fma-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
@@ -1,11 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-FMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-NOFMA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FMA %s
-
-; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs  -enable-no-infs-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-FMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs  < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs -enable-no-infs-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FMA %s
 
 ; Note: The SI-FMA conversions of type x * (y + 1) --> x * y + x would be
 ; beneficial even without fp32 denormals, but they do require no-infs-fp-math
@@ -65,8 +63,8 @@ define amdgpu_kernel void @combine_to_fma_f64_0(ptr addrspace(1) noalias %out, p
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fadd double %mul, %c
+  %mul = fmul contract double %a, %b
+  %fma = fadd contract double %mul, %c
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -134,9 +132,9 @@ define amdgpu_kernel void @combine_to_fma_f64_0_2use(ptr addrspace(1) noalias %o
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fadd double %mul, %c
-  %fma1 = fadd double %mul, %d
+  %mul = fmul contract double %a, %b
+  %fma0 = fadd contract double %mul, %c
+  %fma1 = fadd contract double %mul, %d
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -190,8 +188,8 @@ define amdgpu_kernel void @combine_to_fma_f64_1(ptr addrspace(1) noalias %out, p
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fadd double %c, %mul
+  %mul = fmul contract double %a, %b
+  %fma = fadd contract double %c, %mul
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -244,8 +242,8 @@ define amdgpu_kernel void @combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fsub double %mul, %c
+  %mul = fmul contract double %a, %b
+  %fma = fsub contract double %mul, %c
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -313,9 +311,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_f64_0_2use(ptr addrspace(1) noali
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fsub double %mul, %c
-  %fma1 = fsub double %mul, %d
+  %mul = fmul contract double %a, %b
+  %fma0 = fsub contract double %mul, %c
+  %fma1 = fsub contract double %mul, %d
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -369,8 +367,8 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %fma = fsub double %c, %mul
+  %mul = fmul contract double %a, %b
+  %fma = fsub contract double %c, %mul
   store double %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -438,9 +436,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64_2use(ptr addrspace(1) noali
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %fma0 = fsub double %c, %mul
-  %fma1 = fsub double %d, %mul
+  %mul = fmul contract double %a, %b
+  %fma0 = fsub contract double %c, %mul
+  %fma1 = fsub contract double %d, %mul
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -494,9 +492,9 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64(ptr addrspace(1) noalias %o
   %b = load volatile double, ptr addrspace(1) %gep.1
   %c = load volatile double, ptr addrspace(1) %gep.2
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma = fsub double %mul.neg, %c
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma = fsub contract double %mul.neg, %c
 
   store double %fma, ptr addrspace(1) %gep.out
   ret void
@@ -565,10 +563,10 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_neg(ptr addrspace(1)
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma0 = fsub double %mul.neg, %c
-  %fma1 = fsub double %mul.neg, %d
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma0 = fsub contract double %mul.neg, %c
+  %fma1 = fsub contract double %mul.neg, %d
 
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
@@ -638,10 +636,10 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1)
   %c = load volatile double, ptr addrspace(1) %gep.2
   %d = load volatile double, ptr addrspace(1) %gep.3
 
-  %mul = fmul double %a, %b
-  %mul.neg = fsub double -0.0, %mul
-  %fma0 = fsub double %mul.neg, %c
-  %fma1 = fsub double %mul, %d
+  %mul = fmul contract double %a, %b
+  %mul.neg = fsub contract double -0.0, %mul
+  %fma0 = fsub contract double %mul.neg, %c
+  %fma1 = fsub contract double %mul, %d
 
   store volatile double %fma0, ptr addrspace(1) %gep.out.0
   store volatile double %fma1, ptr addrspace(1) %gep.out.1
@@ -650,32 +648,6 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1)
 
 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
 define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
-; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
-; SI-NOFMA:       ; %bb.0:
-; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
-; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
-; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
-; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
-; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
-; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
-; SI-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
-; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
-; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
-; SI-NOFMA-NEXT:    s_endpgm
-;
 ; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
 ; SI-FMA:       ; %bb.0:
 ; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
@@ -701,30 +673,6 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1)
 ; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-FMA-NEXT:    s_endpgm
 ;
-; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
-; GFX11-NOFMA:       ; %bb.0:
-; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
-; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
-; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
-; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
-; GFX11-NOFMA-NEXT:    s_endpgm
-;
 ; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
 ; GFX11-FMA:       ; %bb.0:
 ; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -761,18 +709,16 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1)
   %u = load volatile double, ptr addrspace(1) %gep.3
   %v = load volatile double, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul double %u, %v
-  %tmp1 = call double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
-  %tmp2 = fsub double %tmp1, %z
+  %tmp0 = fmul contract fast double %u, %v
+  %tmp1 = call contract fast double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
+  %tmp2 = fsub contract fast double %tmp1, %z
 
   store double %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
 
-; fold (fsub x, (fma y, z, (fmul u, v)))
-;   -> (fma (fneg y), z, (fma (fneg u), v, x))
-define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
-; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+define amdgpu_kernel void @no_aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_0_f64:
 ; SI-NOFMA:       ; %bb.0:
 ; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
@@ -793,11 +739,59 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
 ; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
 ; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
-; SI-NOFMA-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; SI-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
 ; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-NOFMA-NEXT:    s_endpgm
 ;
+; GFX11-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_0_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_endpgm
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr double, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr double, ptr addrspace(1) %gep.0, i32 3
+  %gep.4 = getelementptr double, ptr addrspace(1) %gep.0, i32 4
+  %gep.out = getelementptr double, ptr addrspace(1) %out, i32 %tid
+
+  %x = load volatile double, ptr addrspace(1) %gep.0
+  %y = load volatile double, ptr addrspace(1) %gep.1
+  %z = load volatile double, ptr addrspace(1) %gep.2
+  %u = load volatile double, ptr addrspace(1) %gep.3
+  %v = load volatile double, ptr addrspace(1) %gep.4
+
+  %tmp0 = fmul double %u, %v
+  %tmp1 = call double @llvm.fma.f64(double %x, double %y, double %tmp0) #0
+  %tmp2 = fsub double %tmp1, %z
+
+  store double %tmp2, ptr addrspace(1) %gep.out
+  ret void
+}
+
+; fold (fsub x, (fma y, z, (fmul u, v)))
+;   -> (fma (fneg y), z, (fma (fneg u), v, x))
+define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
 ; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
 ; SI-FMA:       ; %bb.0:
 ; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
@@ -823,30 +817,6 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
 ; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
 ; SI-FMA-NEXT:    s_endpgm
 ;
-; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
-; GFX11-NOFMA:       ; %bb.0:
-; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
-; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
-; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
-; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
-; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
-; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
-; GFX11-NOFMA-NEXT:    s_endpgm
-;
 ; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
 ; GFX11-FMA:       ; %bb.0:
 ; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
@@ -883,6 +853,78 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
   %u = load volatile double, ptr addrspace(1) %gep.3
   %v = load volatile double, ptr addrspace(1) %gep.4
 
+  ; nsz flag is needed since this combine may change sign of zero
+  %tmp0 = fmul contract reassoc nsz double %u, %v
+  %tmp1 = call contract reassoc nsz double @llvm.fma.f64(double %y, double %z, double %tmp0) #0
+  %tmp2 = fsub contract reassoc nsz double %x, %tmp1
+
+  store double %tmp2, ptr addrspace(1) %gep.out
+  ret void
+}
+define amdgpu_kernel void @no_aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_1_f64:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
+; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
+; SI-NOFMA-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: no_aggressive_combine_to_fma_fsub_1_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NOFMA-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_endpgm
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr double, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr double, ptr addrspace(1) %gep.0, i32 3
+  %gep.4 = getelementptr double, ptr addrspace(1) %gep.0, i32 4
+  %gep.out = getelementptr double, ptr addrspace(1) %out, i32 %tid
+
+  %x = load volatile double, ptr addrspace(1) %gep.0
+  %y = load volatile double, ptr addrspace(1) %gep.1
+  %z = load volatile double, ptr addrspace(1) %gep.2
+  %u = load volatile double, ptr addrspace(1) %gep.3
+  %v = load volatile double, ptr addrspace(1) %gep.4
+
   ; nsz flag is needed since this combine may change sign of zero
   %tmp0 = fmul nsz double %u, %v
   %tmp1 = call nsz double @llvm.fma.f64(double %y, double %z, double %tmp0) #0
@@ -979,8 +1021,8 @@ define amdgpu_kernel void @test_f32_mul_add_x_one_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
   %y = load volatile float, ptr addrspace(1) %in2
-  %a = fadd float %x, 1.0
-  %m = fmul float %a, %y
+  %a = fadd contract float %x, 1.0
+  %m = fmul contract float %a, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1068,8 +1110,8 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_one(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
   %y = load volatile float, ptr addrspace(1) %in2
-  %a = fadd float %x, 1.0
-  %m = fmul float %y, %a
+  %a = fadd contract float %x, 1.0
+  %m = fmul contract float %y, %a
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1157,8 +1199,8 @@ define amdgpu_kernel void @test_f32_mul_add_x_negone_y(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %a = fadd float %x, -1.0
-  %m = fmul float %a, %y
+  %a = fadd contract float %x, -1.0
+  %m = fmul contract float %a, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1246,8 +1288,8 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_negone(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %a = fadd float %x, -1.0
-  %m = fmul float %y, %a
+  %a = fadd contract float %x, -1.0
+  %m = fmul contract float %y, %a
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1335,8 +1377,8 @@ define amdgpu_kernel void @test_f32_mul_sub_one_x_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float 1.0, %x
-  %m = fmul float %s, %y
+  %s = fsub contract float 1.0, %x
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1424,8 +1466,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_one_x(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float 1.0, %x
-  %m = fmul float %y, %s
+  %s = fsub contract float 1.0, %x
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1513,8 +1555,8 @@ define amdgpu_kernel void @test_f32_mul_sub_negone_x_y(ptr addrspace(1) %out,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float -1.0, %x
-  %m = fmul float %s, %y
+  %s = fsub contract float -1.0, %x
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1602,8 +1644,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_negone_x(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float -1.0, %x
-  %m = fmul float %y, %s
+  %s = fsub contract float -1.0, %x
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1691,8 +1733,8 @@ define amdgpu_kernel void @test_f32_mul_sub_x_one_y(ptr addrspace(1) %out,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, 1.0
-  %m = fmul float %s, %y
+  %s = fsub contract float %x, 1.0
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1780,8 +1822,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_one(ptr addrspace(1) %out,
                                       ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, 1.0
-  %m = fmul float %y, %s
+  %s = fsub contract float %x, 1.0
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1869,8 +1911,8 @@ define amdgpu_kernel void @test_f32_mul_sub_x_negone_y(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, -1.0
-  %m = fmul float %s, %y
+  %s = fsub contract float %x, -1.0
+  %m = fmul contract float %s, %y
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -1958,8 +2000,8 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_negone(ptr addrspace(1) %out,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
-  %s = fsub float %x, -1.0
-  %m = fmul float %y, %s
+  %s = fsub contract float %x, -1.0
+  %m = fmul contract float %y, %s
   store float %m, ptr addrspace(1) %out
   ret void
 }
@@ -2072,10 +2114,10 @@ define amdgpu_kernel void @test_f32_interp(ptr addrspace(1) %out,
   %x = load float, ptr addrspace(1) %in1
   %y = load float, ptr addrspace(1) %in2
   %t = load float, ptr addrspace(1) %in3
-  %t1 = fsub float 1.0, %t
-  %tx = fmul float %x, %t
-  %ty = fmul float %y, %t1
-  %r = fadd float %tx, %ty
+  %t1 = fsub contract float 1.0, %t
+  %tx = fmul contract float %x, %t
+  %ty = fmul contract float %y, %t1
+  %r = fadd contract float %tx, %ty
   store float %r, ptr addrspace(1) %out
   ret void
 }
@@ -2152,10 +2194,10 @@ define amdgpu_kernel void @test_f64_interp(ptr addrspace(1) %out,
   %x = load double, ptr addrspace(1) %in1
   %y = load double, ptr addrspace(1) %in2
   %t = load double, ptr addrspace(1) %in3
-  %t1 = fsub double 1.0, %t
-  %tx = fmul double %x, %t
-  %ty = fmul double %y, %t1
-  %r = fadd double %tx, %ty
+  %t1 = fsub contract double 1.0, %t
+  %tx = fmul contract double %x, %t
+  %ty = fmul contract double %y, %t1
+  %r = fadd contract double %tx, %ty
   store double %r, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index 8c75b5c7c027..2f08931f2287 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -2744,11 +2744,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
@@ -2760,7 +2760,7 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
@@ -2778,38 +2778,38 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_gt_f32_e64 s0, v5, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
@@ -2821,7 +2821,7 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index fd7c7006b361..969c6c3980fc 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -2749,11 +2749,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v4.l, v3.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v6.l, v1.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v5.l, v3.l, vcc_lo
@@ -2765,7 +2765,7 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v5.l, v0.h, vcc_lo
@@ -2783,38 +2783,38 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v1.l, v1.l, v2.h, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v3.l, v2.h, v1.l, s1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v2.l, v2.l, v0.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s1, 0x8000, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e32 vcc_lo, v4, v6
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_lt_f32_e64 s0, v5, v7
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v1.l, vcc_lo
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v5.l, v2.l, v0.l, s0
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
@@ -2826,7 +2826,7 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v2.l, s2
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index 7c0d3692242a..d4471c85c467 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -73,8 +73,8 @@ define amdgpu_kernel void @multiple_fadd_use_test_f32(ptr addrspace(1) %out, flo
   %a16 = select i1 %a15, float %a12, float %a14
   %a17 = fmul float %a16, 2.0
   %a18 = fmul float %a17, %a17
-  %a19 = fmul float %a18, %a17
-  %a20 = fsub float 1.0, %a19
+  %a19 = fmul contract float %a18, %a17
+  %a20 = fsub contract float 1.0, %a19
   store float %a20, ptr addrspace(1) %out
   ret void
 }
@@ -540,8 +540,8 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
   %a16 = select i1 %a15, half %a12, half %a14
   %a17 = fmul half %a16, 2.0
   %a18 = fmul half %a17, %a17
-  %a19 = fmul half %a18, %a17
-  %a20 = fsub half 1.0, %a19
+  %a19 = fmul contract half %a18, %a17
+  %a20 = fsub contract half 1.0, %a19
   store half %a20, ptr addrspace(1) %out
   ret void
 }
@@ -1166,7 +1166,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x
   ret void
 }
 
-attributes #0 = { nounwind "unsafe-fp-math"="true" }
+attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GFX11-DENORM: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
index f199db3ca12c..462d7748b86c 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.f16.ll
@@ -832,9 +832,9 @@ define amdgpu_ps half @fneg_fadd_0_nsz_f16(half inreg %tmp2, half inreg %tmp6, <
 ; GFX11-NSZ-TRUE16-NEXT:    ; return to shader part epilog
 .entry:
   %tmp7 = fdiv afn half 1.000000e+00, %tmp6
-  %tmp8 = fmul half 0.000000e+00, %tmp7
+  %tmp8 = fmul contract half 0.000000e+00, %tmp7
   %tmp9 = fmul reassoc nnan arcp contract half 0.000000e+00, %tmp8
-  %.i188 = fadd half %tmp9, 0.000000e+00
+  %.i188 = fadd nnan ninf contract half %tmp9, 0.000000e+00
   %tmp10 = fcmp uge half %.i188, %tmp2
   %tmp11 = fneg half %.i188
   %.i092 = select i1 %tmp10, half %tmp2, half %tmp11
@@ -6258,7 +6258,7 @@ declare <4 x half> @llvm.fmuladd.v4f16(<4 x half>, <4 x half>, <4 x half>) #1
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "unsafe-fp-math"="true" }
+attributes #2 = { nounwind }
 attributes #3 = { nounwind "no-signed-zeros-fp-math"="true" }
 attributes #4 = { nounwind "amdgpu-ieee"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
index 86e73ed03f18..e68774546901 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-SAFE,SI,SI-SAFE %s
-; RUN: llc -enable-no-signed-zeros-fp-math -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-NSZ,SI,SI-NSZ %s
+; RUN: llc -enable-no-signed-zeros-fp-math -mtriple=amdgcn -mcpu=hawaii -mattr=+flat-for-global -fp-contract=fast < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-NSZ,SI,SI-NSZ %s
 
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-SAFE,VI,VI-SAFE %s
-; RUN: llc -enable-no-signed-zeros-fp-math -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-NSZ,VI,VI-NSZ %s
+; RUN: llc -enable-no-signed-zeros-fp-math -mtriple=amdgcn -mcpu=fiji -fp-contract=fast < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-NSZ,VI,VI-NSZ %s
 
 ; --------------------------------------------------------------------------------
 ; fadd tests
@@ -289,14 +289,18 @@ define amdgpu_ps float @fneg_fadd_0_f32(float inreg %tmp2, float inreg %tmp6, <4
 ; function attribute unsafe-fp-math automatically. Combine with the previous test
 ; when that is done.
 define amdgpu_ps float @fneg_fadd_0_nsz_f32(float inreg %tmp2, float inreg %tmp6, <4 x i32> %arg) #2 {
-; SI-SAFE-LABEL: fneg_fadd_0_nsz_f32:
-; SI-SAFE:       ; %bb.0: ; %.entry
-; SI-SAFE-NEXT:    v_min_legacy_f32_e64 v0, 0, s0
-; SI-SAFE-NEXT:    s_brev_b32 s0, 1
-; SI-SAFE-NEXT:    v_mov_b32_e32 v1, 0x7fc00000
-; SI-SAFE-NEXT:    v_cmp_ngt_f32_e32 vcc, s0, v0
-; SI-SAFE-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
-; SI-SAFE-NEXT:    ; return to shader part epilog
+; GCN-SAFE-LABEL: fneg_fadd_0_nsz_f32:
+; GCN-SAFE:       ; %bb.0: ; %.entry
+; GCN-SAFE-NEXT:    v_rcp_f32_e32 v0, s1
+; GCN-SAFE-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-SAFE-NEXT:    v_mul_f32_e32 v0, 0, v0
+; GCN-SAFE-NEXT:    v_add_f32_e32 v0, 0, v0
+; GCN-SAFE-NEXT:    v_cmp_ngt_f32_e32 vcc, s0, v0
+; GCN-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; GCN-SAFE-NEXT:    v_mov_b32_e32 v1, 0x7fc00000
+; GCN-SAFE-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
+; GCN-SAFE-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
+; GCN-SAFE-NEXT:    ; return to shader part epilog
 ;
 ; GCN-NSZ-LABEL: fneg_fadd_0_nsz_f32:
 ; GCN-NSZ:       ; %bb.0: ; %.entry
@@ -309,19 +313,6 @@ define amdgpu_ps float @fneg_fadd_0_nsz_f32(float inreg %tmp2, float inreg %tmp6
 ; GCN-NSZ-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
 ; GCN-NSZ-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
 ; GCN-NSZ-NEXT:    ; return to shader part epilog
-;
-; VI-SAFE-LABEL: fneg_fadd_0_nsz_f32:
-; VI-SAFE:       ; %bb.0: ; %.entry
-; VI-SAFE-NEXT:    v_rcp_f32_e32 v0, s1
-; VI-SAFE-NEXT:    v_mov_b32_e32 v1, s0
-; VI-SAFE-NEXT:    v_mul_f32_e32 v0, 0, v0
-; VI-SAFE-NEXT:    v_add_f32_e32 v0, 0, v0
-; VI-SAFE-NEXT:    v_cmp_ngt_f32_e32 vcc, s0, v0
-; VI-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; VI-SAFE-NEXT:    v_mov_b32_e32 v1, 0x7fc00000
-; VI-SAFE-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
-; VI-SAFE-NEXT:    v_cndmask_b32_e64 v0, v1, 0, vcc
-; VI-SAFE-NEXT:    ; return to shader part epilog
 .entry:
   %tmp7 = fdiv afn float 1.000000e+00, %tmp6
   %tmp8 = fmul float 0.000000e+00, %tmp7
@@ -672,17 +663,28 @@ define amdgpu_ps double @fneg_fadd_0_f64(double inreg %tmp2, double inreg %tmp6,
 ; function attribute unsafe-fp-math automatically. Combine with the previous test
 ; when that is done.
 define amdgpu_ps double @fneg_fadd_0_nsz_f64(double inreg %tmp2, double inreg %tmp6, <4 x i32> %arg) #2 {
-; GCN-SAFE-LABEL: fneg_fadd_0_nsz_f64:
-; GCN-SAFE:       ; %bb.0: ; %.entry
-; GCN-SAFE-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[0:1], 0
-; GCN-SAFE-NEXT:    s_and_b64 s[2:3], s[2:3], exec
-; GCN-SAFE-NEXT:    s_cselect_b32 s1, s1, 0x80000000
-; GCN-SAFE-NEXT:    s_cselect_b32 s0, s0, 0
-; GCN-SAFE-NEXT:    v_cmp_ngt_f64_e64 s[0:1], s[0:1], 0
-; GCN-SAFE-NEXT:    s_and_b64 s[0:1], s[0:1], exec
-; GCN-SAFE-NEXT:    s_cselect_b32 s1, 0, 0x7ff80000
-; GCN-SAFE-NEXT:    s_mov_b32 s0, 0
-; GCN-SAFE-NEXT:    ; return to shader part epilog
+; SI-SAFE-LABEL: fneg_fadd_0_nsz_f64:
+; SI-SAFE:       ; %bb.0: ; %.entry
+; SI-SAFE-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; SI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; SI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; SI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; SI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; SI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; SI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; SI-SAFE-NEXT:    v_mov_b32_e32 v2, s1
+; SI-SAFE-NEXT:    v_mul_f64 v[0:1], v[0:1], 0
+; SI-SAFE-NEXT:    v_mov_b32_e32 v3, s0
+; SI-SAFE-NEXT:    v_add_f64 v[0:1], v[0:1], 0
+; SI-SAFE-NEXT:    v_cmp_ngt_f64_e32 vcc, s[0:1], v[0:1]
+; SI-SAFE-NEXT:    v_xor_b32_e32 v4, 0x80000000, v1
+; SI-SAFE-NEXT:    v_cndmask_b32_e32 v1, v4, v2, vcc
+; SI-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; SI-SAFE-NEXT:    v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
+; SI-SAFE-NEXT:    s_and_b64 s[0:1], vcc, exec
+; SI-SAFE-NEXT:    s_cselect_b32 s1, 0, 0x7ff80000
+; SI-SAFE-NEXT:    s_mov_b32 s0, 0
+; SI-SAFE-NEXT:    ; return to shader part epilog
 ;
 ; SI-NSZ-LABEL: fneg_fadd_0_nsz_f64:
 ; SI-NSZ:       ; %bb.0: ; %.entry
@@ -707,6 +709,29 @@ define amdgpu_ps double @fneg_fadd_0_nsz_f64(double inreg %tmp2, double inreg %t
 ; SI-NSZ-NEXT:    s_mov_b32 s0, 0
 ; SI-NSZ-NEXT:    ; return to shader part epilog
 ;
+; VI-SAFE-LABEL: fneg_fadd_0_nsz_f64:
+; VI-SAFE:       ; %bb.0: ; %.entry
+; VI-SAFE-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
+; VI-SAFE-NEXT:    v_mov_b32_e32 v4, s0
+; VI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-SAFE-NEXT:    v_fma_f64 v[2:3], -s[2:3], v[0:1], 1.0
+; VI-SAFE-NEXT:    v_fma_f64 v[0:1], v[2:3], v[0:1], v[0:1]
+; VI-SAFE-NEXT:    v_mov_b32_e32 v2, s1
+; VI-SAFE-NEXT:    v_mul_f64 v[0:1], v[0:1], 0
+; VI-SAFE-NEXT:    v_add_f64 v[0:1], v[0:1], 0
+; VI-SAFE-NEXT:    v_cmp_ngt_f64_e32 vcc, s[0:1], v[0:1]
+; VI-SAFE-NEXT:    v_xor_b32_e32 v3, 0x80000000, v1
+; VI-SAFE-NEXT:    v_cndmask_b32_e32 v1, v3, v2, vcc
+; VI-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; VI-SAFE-NEXT:    v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
+; VI-SAFE-NEXT:    s_and_b64 s[0:1], vcc, exec
+; VI-SAFE-NEXT:    s_cselect_b32 s1, 0, 0x7ff80000
+; VI-SAFE-NEXT:    s_mov_b32 s0, 0
+; VI-SAFE-NEXT:    ; return to shader part epilog
+;
 ; VI-NSZ-LABEL: fneg_fadd_0_nsz_f64:
 ; VI-NSZ:       ; %bb.0: ; %.entry
 ; VI-NSZ-NEXT:    v_rcp_f64_e32 v[0:1], s[2:3]
@@ -4584,6 +4609,6 @@ declare half @llvm.amdgcn.rcp.f16(half) #1
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind "unsafe-fp-math"="true" }
+attributes #2 = { nounwind }
 attributes #3 = { nounwind "no-signed-zeros-fp-math"="true" }
 attributes #4 = { nounwind "amdgpu-ieee"="false" "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
index d431503643d6..230b73a03722 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs-divergence-driven-isel.ll
@@ -398,7 +398,7 @@ define amdgpu_kernel void @uniform_fneg_f64(ptr addrspace(1) %out, ptr addrspace
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
 ; GCN: %[[XOR:[0-9]+]]:sreg_32 = S_XOR_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[XOR_COPY:[0-9]+]]:sreg_32 = COPY %[[XOR]]
+; GCN: %[[XOR_COPY:[0-9]+]]:sreg_32 = COPY killed %[[XOR]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[XOR_COPY]], %subreg.sub1
 
   %in.gep = getelementptr inbounds double, ptr addrspace(1) %in, i64 %idx
@@ -440,7 +440,7 @@ define amdgpu_kernel void @uniform_fabs_f64(ptr addrspace(1) %out, ptr addrspace
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 2147483647
 ; GCN: %[[AND:[0-9]+]]:sreg_32 = S_AND_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[AND_COPY:[0-9]+]]:sreg_32 = COPY %[[AND]]
+; GCN: %[[AND_COPY:[0-9]+]]:sreg_32 = COPY killed %[[AND]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[AND_COPY]], %subreg.sub1
 
 
@@ -484,7 +484,7 @@ define amdgpu_kernel void @uniform_fneg_fabs_f64(ptr addrspace(1) %out, ptr addr
 ; GCN: %[[HI32:[0-9]+]]:sreg_32 = COPY %[[VREG64]].sub1
 ; GCN: %[[SREG_MASK:[0-9]+]]:sreg_32 = S_MOV_B32 -2147483648
 ; GCN: %[[OR:[0-9]+]]:sreg_32 = S_OR_B32 killed %[[HI32]], killed %[[SREG_MASK]]
-; GCN: %[[OR_COPY:[0-9]+]]:sreg_32 = COPY %[[OR]]
+; GCN: %[[OR_COPY:[0-9]+]]:sreg_32 = COPY killed %[[OR]]
 ; GCN: REG_SEQUENCE killed %[[LO32]], %subreg.sub0, killed %[[OR_COPY]], %subreg.sub1
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll
index 726272406491..c3f4ebe30152 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.ll
@@ -3,7 +3,9 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
-; RUN: not llc -mtriple=r600 -mcpu=redwood < %s
+; RUN: not --crash llc -mtriple=r600 -mcpu=redwood < %s 2>&1 | FileCheck -check-prefix=R600-ERR %s
+
+; R600-ERR: LLVM ERROR: unsupported library call operation
 
 define amdgpu_kernel void @s_fneg_f32(ptr addrspace(1) %out, float %in) {
 ; SI-LABEL: s_fneg_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 0959687d3834..3e2680f55832 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -75,6 +75,7 @@ define i32 @flat_offset_maybe_oob(ptr %p, i32 %i) {
 ; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
 ; GFX12-NEXT:    flat_load_b32 v0, v[0:1] offset:12
 ; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-image-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-image-atomics.ll
index 71b0abe18339..d483364aae55 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-image-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-image-atomics.ll
@@ -9,7 +9,7 @@
 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=hawaii -verify-machineinstrs | FileCheck %s  -check-prefix=G_GFX7
 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX10
 ; RUN: llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s -check-prefix=G_GFX1030
-; RUN: not --crash llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=G_GFX11-ERR
+; RUN: not llc < %s -global-isel -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=G_GFX11-ERR
 
 ; GFX11-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.image.atomic.f
 ; G_GFX11-ERR: LLVM ERROR: cannot select: {{.*}} = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.f
diff --git a/llvm/test/CodeGen/AMDGPU/fract-match.ll b/llvm/test/CodeGen/AMDGPU/fract-match.ll
index 4ee48716439b..d97ea042b50f 100644
--- a/llvm/test/CodeGen/AMDGPU/fract-match.ll
+++ b/llvm/test/CodeGen/AMDGPU/fract-match.ll
@@ -118,6 +118,7 @@ define float @safe_math_fract_f32(float %x, ptr addrspace(1) writeonly captures(
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -229,6 +230,7 @@ define float @safe_math_fract_f32_swap(float %x, ptr addrspace(1) writeonly capt
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -2422,6 +2424,7 @@ define double @safe_math_fract_f64(double %x, ptr addrspace(1) writeonly capture
 ; GFX12-NEXT:    v_cmp_neq_f64_e64 vcc_lo, 0x7ff00000, |v[0:1]|
 ; GFX12-NEXT:    v_floor_f64_e32 v[6:7], v[0:1]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v0, 0, v4 :: v_dual_cndmask_b32 v1, 0, v5
 ; GFX12-NEXT:    global_store_b64 v[2:3], v[6:7], off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -2569,6 +2572,7 @@ define half @safe_math_fract_f16(half %x, ptr addrspace(1) writeonly captures(no
 ; GFX12-TRUE16-NEXT:    v_cmp_neq_f16_e64 s0, 0x7c00, |v0.l|
 ; GFX12-TRUE16-NEXT:    v_floor_f16_e32 v3.l, v0.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, 0, v0.h, s0
 ; GFX12-TRUE16-NEXT:    global_store_b16 v[1:2], v3, off
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -2584,6 +2588,7 @@ define half @safe_math_fract_f16(half %x, ptr addrspace(1) writeonly captures(no
 ; GFX12-FAKE16-NEXT:    v_cmp_neq_f16_e64 vcc_lo, 0x7c00, |v0|
 ; GFX12-FAKE16-NEXT:    v_floor_f16_e32 v4, v0
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-FAKE16-NEXT:    global_store_b16 v[1:2], v4, off
 ; GFX12-FAKE16-NEXT:    s_setpc_b64 s[30:31]
@@ -2804,14 +2809,14 @@ define <2 x half> @safe_math_fract_v2f16(<2 x half> %x, ptr addrspace(1) writeon
 ; GFX12-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
 ; GFX12-FAKE16-NEXT:    v_fract_f16_e32 v6, v0
 ; GFX12-FAKE16-NEXT:    v_floor_f16_e32 v5, v0
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_fract_f16_e32 v4, v3
 ; GFX12-FAKE16-NEXT:    v_cmp_class_f16_e64 s0, v3, 0x204
 ; GFX12-FAKE16-NEXT:    v_floor_f16_e32 v7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v3, v4, 0, s0
 ; GFX12-FAKE16-NEXT:    v_cmp_class_f16_e64 s0, v0, 0x204
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_pack_b32_f16 v4, v5, v7
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v6, 0, s0
@@ -3093,6 +3098,7 @@ define float @safe_math_fract_f32_minimum(float %x, ptr addrspace(1) writeonly c
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -3207,6 +3213,7 @@ define float @safe_math_fract_f32_minimum_swap(float %x, ptr addrspace(1) writeo
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -3318,6 +3325,7 @@ define float @safe_math_fract_f32_minimumnum(float %x, ptr addrspace(1) writeonl
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -3429,6 +3437,7 @@ define float @safe_math_fract_f32_minimumnum_swap(float %x, ptr addrspace(1) wri
 ; GFX12-NEXT:    v_cmp_neq_f32_e64 vcc_lo, 0x7f800000, |v0|
 ; GFX12-NEXT:    v_floor_f32_e32 v4, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_cndmask_b32_e32 v0, 0, v3, vcc_lo
 ; GFX12-NEXT:    global_store_b32 v[1:2], v4, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 0ddb181decb9..100a560c1d12 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -166,6 +166,7 @@ define float @global_agent_atomic_fadd_ret_f32__amdgpu_no_fine_grained_memory(pt
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -351,6 +352,7 @@ define float @global_agent_atomic_fadd_ret_f32__offset12b_pos__amdgpu_no_fine_gr
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -542,6 +544,7 @@ define float @global_agent_atomic_fadd_ret_f32__offset12b_neg__amdgpu_no_fine_gr
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -727,6 +730,7 @@ define void @global_agent_atomic_fadd_noret_f32__amdgpu_no_fine_grained_memory(p
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -907,6 +911,7 @@ define void @global_agent_atomic_fadd_noret_f32__offset12b_pos__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1092,6 +1097,7 @@ define void @global_agent_atomic_fadd_noret_f32__offset12b_neg__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1284,6 +1290,7 @@ define float @global_system_atomic_fadd_ret_f32__offset12b_pos__amdgpu_no_fine_g
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1469,6 +1476,7 @@ define void @global_system_atomic_fadd_noret_f32__offset12b_pos__amdgpu_no_fine_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1670,6 +1678,7 @@ define float @global_agent_atomic_fadd_ret_f32_maybe_remote(ptr addrspace(1) %pt
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32_maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1872,6 +1881,7 @@ define float @global_agent_atomic_fadd_ret_f32_maybe_remote__amdgpu_ignore_denor
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32_maybe_remote__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2068,6 +2078,7 @@ define void @global_agent_atomic_fadd_noret_f32_maybe_remote__amdgpu_ignore_deno
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32_maybe_remote__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2253,6 +2264,7 @@ define float @global_agent_atomic_fadd_ret_f32___amdgpu_no_fine_grained_memory(p
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32___amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2425,6 +2437,7 @@ define float @global_agent_atomic_fadd_ret_f32___amdgpu_no_fine_grained_memory__
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32___amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2627,6 +2640,7 @@ define float @global_agent_atomic_fadd_ret_f32_amdgpu_ignore_denormal_mode(ptr a
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32_amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2823,6 +2837,7 @@ define void @global_agent_atomic_fadd_noret_f32_maybe_remote(ptr addrspace(1) %p
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32_maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3004,6 +3019,7 @@ define void @global_agent_atomic_fadd_noret_f32___amdgpu_no_fine_grained_memory(
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32___amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3159,6 +3175,7 @@ define void @global_agent_atomic_fadd_noret_f32___amdgpu_no_fine_grained_memory_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32___amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3354,6 +3371,7 @@ define void @global_agent_atomic_fadd_noret_f32_amdgpu_ignore_denormal_mode(ptr
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32_amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3554,6 +3572,7 @@ define float @global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory(ptr addr
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3747,6 +3766,7 @@ define void @global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory(ptr add
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -3946,6 +3966,7 @@ define float @global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory__amdgpu_
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4139,6 +4160,7 @@ define void @global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory__amdgpu
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4322,6 +4344,7 @@ define float @global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory__amdgpu_
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__amdgpu_no_remote_memory__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4501,6 +4524,7 @@ define void @global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory__amdgpu
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__amdgpu_no_remote_memory__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4674,6 +4698,7 @@ define float @global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_fine_grained_memo
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -4845,6 +4870,7 @@ define float @global_agent_atomic_fadd_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5022,6 +5048,7 @@ define float @global_agent_atomic_fadd_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5181,6 +5208,7 @@ define void @global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_fine_grained_mem
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5335,6 +5363,7 @@ define void @global_agent_atomic_fadd_noret_f32__offset12b_pos__ftz__amdgpu_no_f
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5494,6 +5523,7 @@ define void @global_agent_atomic_fadd_noret_f32__offset12b_neg__ftz__amdgpu_no_f
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5672,6 +5702,7 @@ define float @global_system_atomic_fadd_ret_f32__offset12b_pos__ftz__amdgpu_no_f
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -5831,6 +5862,7 @@ define void @global_system_atomic_fadd_noret_f32__offset12b_pos__ftz__amdgpu_no_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6002,6 +6034,7 @@ define float @global_agent_atomic_fadd_ret_f32__offset12b_pos__ieee__amdgpu_no_f
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__offset12b_pos__ieee__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6158,6 +6191,7 @@ define void @global_agent_atomic_fadd_noret_f32__offset12b_pos__ieee__amdgpu_no_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__offset12b_pos__ieee__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6358,6 +6392,7 @@ define float @global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_remote_memory(ptr
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6551,6 +6586,7 @@ define void @global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_remote_memory(pt
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6720,6 +6756,7 @@ define float @global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_fine_grained_memo
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f32__ftz__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -6873,6 +6910,7 @@ define void @global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_fine_grained_mem
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f32__ftz__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7091,6 +7129,7 @@ define double @global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f64__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7312,6 +7351,7 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7538,6 +7578,7 @@ define double @global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7750,6 +7791,7 @@ define void @global_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory(p
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f64__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -7955,6 +7997,7 @@ define void @global_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -8165,6 +8208,7 @@ define void @global_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -8585,6 +8629,7 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v5, v2
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9019,6 +9064,7 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9457,6 +9503,7 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -9868,6 +9915,7 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -10286,6 +10334,7 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -10707,6 +10756,7 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -11039,6 +11089,7 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -11350,6 +11401,7 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -11780,6 +11832,7 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -12206,6 +12259,7 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -12708,6 +12762,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v5, v2
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -12767,13 +12822,14 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB55_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12824,13 +12880,14 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB55_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13220,6 +13277,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -13282,13 +13340,14 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB56_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13339,13 +13398,14 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB56_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13736,6 +13796,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -14221,6 +14282,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -14279,13 +14341,14 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB58_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -14334,13 +14397,14 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB58_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -14717,6 +14781,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -14777,13 +14842,14 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB59_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -14832,13 +14898,14 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB59_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -15216,6 +15283,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -15633,6 +15701,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v4
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -16029,6 +16098,7 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -16080,13 +16150,14 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB62_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -16138,13 +16209,14 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB62_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -16537,6 +16609,7 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, v6, v4
 ; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -16599,13 +16672,14 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB63_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -16655,13 +16729,14 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB63_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17041,6 +17116,7 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -17269,6 +17345,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__amdgpu_no_fine_grained_me
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -17500,6 +17577,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__offset12b_pos__amdgpu_no_
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -17734,6 +17812,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -17947,6 +18026,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__amdgpu_no_fine_grained_memory
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -18157,6 +18237,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__offset12b_pos__amdgpu_no_fine
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -18372,6 +18453,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -18609,6 +18691,7 @@ define <2 x half> @global_system_atomic_fadd_ret_v2f16__offset12b_pos__amdgpu_no
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -18825,6 +18908,7 @@ define void @global_system_atomic_fadd_noret_v2f16__offset12b_pos__amdgpu_no_fin
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -19068,6 +19152,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__amdgpu_no_remote_memory(p
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -19304,6 +19389,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__amdgpu_no_remote_memory(ptr a
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -19532,6 +19618,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__amdgpu_no_fine_grained_me
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -19742,6 +19829,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__amdgpu_no_fine_grained_memory
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -19984,6 +20072,7 @@ define <2 x half> @global_agent_atomic_fadd_ret_v2f16__maybe_remote(ptr addrspac
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v3
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2f16__maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -20220,6 +20309,7 @@ define void @global_agent_atomic_fadd_noret_v2f16__maybe_remote(ptr addrspace(1)
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2f16__maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -20604,6 +20694,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -20983,6 +21074,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -21365,6 +21457,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -21736,6 +21829,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -22104,6 +22198,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -22477,6 +22572,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -22862,6 +22958,7 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -23236,6 +23333,7 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -23613,6 +23711,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -23981,6 +24080,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -24357,6 +24457,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -24725,6 +24826,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -25101,6 +25203,7 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
 ; GFX7-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX7-NEXT:    v_mov_b32_e32 v1, v2
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_ret_v2bf16__maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -25469,6 +25572,7 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
 ; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX6-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index a24d6c5ff222..faa3ee61427a 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -4948,13 +4948,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB27_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -5408,13 +5409,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB28_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6298,13 +6300,14 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB30_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6745,13 +6748,14 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB31_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7871,13 +7875,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -8334,13 +8339,14 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB35_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9239,13 +9245,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9296,13 +9303,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9757,13 +9765,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9814,13 +9823,14 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10760,13 +10770,14 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10815,13 +10826,14 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11261,13 +11273,14 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11316,13 +11329,14 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12573,13 +12587,14 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12631,13 +12646,14 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13095,13 +13111,14 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13151,13 +13168,14 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17531,6 +17549,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18018,6 +18037,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18508,6 +18528,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -19517,6 +19538,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 5834d4ab4d8e..cb66f85ff3ae 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -4948,13 +4948,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB27_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -5408,13 +5409,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB28_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6298,13 +6300,14 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB30_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -6745,13 +6748,14 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB31_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7871,13 +7875,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -8334,13 +8339,14 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB35_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9239,13 +9245,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9296,13 +9303,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9757,13 +9765,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9814,13 +9823,14 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB38_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10760,13 +10770,14 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10815,13 +10826,14 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11261,13 +11273,14 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11316,13 +11329,14 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12573,13 +12587,14 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -12631,13 +12646,14 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13095,13 +13111,14 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13151,13 +13168,14 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17531,6 +17549,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18018,6 +18037,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18508,6 +18528,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -19517,6 +19538,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 765185327a03..f869b5778bfb 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9768,13 +9768,14 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB33_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -9825,13 +9826,14 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB33_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10284,13 +10286,14 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -10341,13 +10344,14 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB34_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11283,13 +11287,14 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB36_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11338,13 +11343,14 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB36_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11782,13 +11788,14 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -11837,13 +11844,14 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB37_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13088,13 +13096,14 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-TRUE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13146,13 +13155,14 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v3
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v3, 3, v3
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v5, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v4, v4
 ; GFX12-FAKE16-NEXT:  .LBB40_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13608,13 +13618,14 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-TRUE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-TRUE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13664,13 +13675,14 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, -4, v4
 ; GFX12-FAKE16-NEXT:    v_and_b32_e32 v4, 3, v4
 ; GFX12-FAKE16-NEXT:    s_mov_b32 s0, 0
 ; GFX12-FAKE16-NEXT:    global_load_b32 v3, v[0:1], off
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 3, v4
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_not_b32_e32 v5, v5
 ; GFX12-FAKE16-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -17898,6 +17910,7 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18385,6 +18398,7 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -18875,6 +18889,7 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
@@ -19884,6 +19899,7 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v8, v8, v6, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v2, v2
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v6, v8, v10, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v2, v7, v9, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
new file mode 100644
index 000000000000..32663b4b2cc3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -0,0 +1,605 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-SDAG %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL %s
+
+; Test S_WAIT_XCNT insertion for global_load/store instructions.
+; Introduced additional operations in between the clauses to have the register dependency
+; between the operands of VMEM operations and the def ops of VALU instructions that followed.
+
+define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %ptr_b, ptr addrspace(1) %ptr_c, ptr addrspace(1) %ptr_d, ptr addrspace(1) %out) {
+; GCN-SDAG-LABEL: test_i8load_v4i8store:
+; GCN-SDAG:       ; %bb.0:
+; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GCN-SDAG-NEXT:    global_load_u8 v2, v[2:3], off
+; GCN-SDAG-NEXT:    global_load_u8 v3, v[4:5], off
+; GCN-SDAG-NEXT:    global_load_u8 v0, v[0:1], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x2
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    v_lshlrev_b16 v1, 8, v2
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x1
+; GCN-SDAG-NEXT:    v_lshlrev_b16 v2, 8, v3
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GCN-SDAG-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-SDAG-NEXT:    v_or_b32_e32 v1, v3, v2
+; GCN-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GCN-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GCN-SDAG-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-SDAG-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-SDAG-NEXT:    global_store_b32 v[8:9], v0, off
+; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GCN-GISEL-LABEL: test_i8load_v4i8store:
+; GCN-GISEL:       ; %bb.0:
+; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    global_load_u8 v0, v[0:1], off
+; GCN-GISEL-NEXT:    global_load_u8 v1, v[2:3], off
+; GCN-GISEL-NEXT:    global_load_u8 v2, v[4:5], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x1
+; GCN-GISEL-NEXT:    v_lshl_or_b32 v0, v1, 8, v0
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
+; GCN-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 24, v2
+; GCN-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-GISEL-NEXT:    v_or3_b32 v0, v0, v1, v2
+; GCN-GISEL-NEXT:    global_store_b32 v[8:9], v0, off
+; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+  %a = load i8, ptr addrspace(1) %ptr_a
+  %b = load i8, ptr addrspace(1) %ptr_b
+  %c = load i8, ptr addrspace(1) %ptr_c
+  %d = load i8, ptr addrspace(1) %ptr_d
+  %ins_0 = insertelement <4 x i8> poison, i8 %a, i32 0
+  %ins_1 = insertelement <4 x i8> %ins_0, i8 %b, i32 1
+  %ins_2 = insertelement <4 x i8> %ins_1, i8 %c, i32 2
+  %ins_3 = insertelement <4 x i8> %ins_2, i8 %c, i32 3
+  store <4 x i8> %ins_3, ptr addrspace(1) %out
+  ret void
+}
+
+define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2) {
+; GCN-SDAG-LABEL: test_v7i16_load_store:
+; GCN-SDAG:       ; %bb.0:
+; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GCN-SDAG-NEXT:    global_load_b128 v[4:7], v[0:1], off
+; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v[2:3], off
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    v_pk_add_u16 v10, v6, v2
+; GCN-SDAG-NEXT:    v_pk_add_u16 v11, v7, v3
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v2, 12
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v6, 8
+; GCN-SDAG-NEXT:    v_pk_add_u16 v4, v4, v0
+; GCN-SDAG-NEXT:    v_lshrrev_b32_e32 v0, 16, v10
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-SDAG-NEXT:    v_pk_add_u16 v5, v5, v1
+; GCN-SDAG-NEXT:    s_clause 0x2
+; GCN-SDAG-NEXT:    global_store_b16 v[2:3], v11, off
+; GCN-SDAG-NEXT:    global_store_b32 v[6:7], v10, off
+; GCN-SDAG-NEXT:    global_store_b64 v[8:9], v[4:5], off
+; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GCN-GISEL-LABEL: test_v7i16_load_store:
+; GCN-GISEL:       ; %bb.0:
+; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    global_load_b128 v[4:7], v[0:1], off
+; GCN-GISEL-NEXT:    global_load_b128 v[0:3], v[2:3], off
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 2
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 4
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 6
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v16, 8
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v18, 10
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v20, 12 :: v_dual_mov_b32 v19, 0
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v21, 0
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    v_pk_add_u16 v2, v6, v2
+; GCN-GISEL-NEXT:    v_pk_add_u16 v4, v4, v0
+; GCN-GISEL-NEXT:    v_pk_add_u16 v1, v5, v1
+; GCN-GISEL-NEXT:    v_pk_add_u16 v3, v7, v3
+; GCN-GISEL-NEXT:    s_clause 0x6
+; GCN-GISEL-NEXT:    global_store_b16 v[8:9], v4, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[10:11], v4, off
+; GCN-GISEL-NEXT:    global_store_b16 v[12:13], v1, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[14:15], v1, off
+; GCN-GISEL-NEXT:    global_store_b16 v[16:17], v2, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[18:19], v2, off
+; GCN-GISEL-NEXT:    global_store_b16 v[20:21], v3, off
+; GCN-GISEL-NEXT:    v_lshrrev_b32_e32 v0, 16, v2
+; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+  %vec1 = load <7 x i16>, ptr addrspace(1) %ptr1
+  %insert = insertelement <7 x i16> %vec1, i16 20, i32 4
+  %vec2 = load <7 x i16>, ptr addrspace(1) %ptr2
+  %add = add <7 x i16> %vec1, %vec2
+  store <7 x i16> %add, ptr addrspace(1) null
+  %elt = extractelement <7 x i16> %add, i32 5
+  ret i16 %elt
+}
+
+define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspace(1) %out) {
+; GCN-SDAG-LABEL: test_v64i32_load_store:
+; GCN-SDAG:       ; %bb.0:
+; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GCN-SDAG-NEXT:    s_clause 0xd
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:52
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v41, s32 offset:48
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v42, s32 offset:44
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v43, s32 offset:40
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v44, s32 offset:36
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v45, s32 offset:32
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v56, s32 offset:28
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v57, s32 offset:24
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v58, s32 offset:20
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v59, s32 offset:16
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v60, s32 offset:12
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v61, s32 offset:8
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v62, s32 offset:4
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v63, s32
+; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:224
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    scratch_store_b128 off, v[6:9], s32 offset:56 ; 16-byte Folded Spill
+; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:240
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    scratch_store_b128 off, v[6:9], s32 offset:72 ; 16-byte Folded Spill
+; GCN-SDAG-NEXT:    s_clause 0xd
+; GCN-SDAG-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:192
+; GCN-SDAG-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:208
+; GCN-SDAG-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:160
+; GCN-SDAG-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:176
+; GCN-SDAG-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:128
+; GCN-SDAG-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:144
+; GCN-SDAG-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:96
+; GCN-SDAG-NEXT:    global_load_b128 v[48:51], v[0:1], off offset:112
+; GCN-SDAG-NEXT:    global_load_b128 v[52:55], v[0:1], off offset:64
+; GCN-SDAG-NEXT:    global_load_b128 v[38:41], v[0:1], off offset:80
+; GCN-SDAG-NEXT:    global_load_b128 v[42:45], v[0:1], off offset:32
+; GCN-SDAG-NEXT:    global_load_b128 v[56:59], v[0:1], off offset:48
+; GCN-SDAG-NEXT:    global_load_b128 v[60:63], v[0:1], off
+; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:16
+; GCN-SDAG-NEXT:    scratch_load_b128 v[6:9], off, s32 offset:56 th:TH_LOAD_LU ; 16-byte Folded Reload
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:224
+; GCN-SDAG-NEXT:    scratch_load_b128 v[6:9], off, s32 offset:72 th:TH_LOAD_LU ; 16-byte Folded Reload
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    s_clause 0xe
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:240
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[10:13], off offset:192
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:208
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:160
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[22:25], off offset:176
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[26:29], off offset:128
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[30:33], off offset:144
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[34:37], off offset:96
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[48:51], off offset:112
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[52:55], off offset:64
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[38:41], off offset:80
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[42:45], off offset:32
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[56:59], off offset:48
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[60:63], off
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off offset:16
+; GCN-SDAG-NEXT:    s_clause 0xd
+; GCN-SDAG-NEXT:    scratch_load_b32 v63, off, s32
+; GCN-SDAG-NEXT:    scratch_load_b32 v62, off, s32 offset:4
+; GCN-SDAG-NEXT:    scratch_load_b32 v61, off, s32 offset:8
+; GCN-SDAG-NEXT:    scratch_load_b32 v60, off, s32 offset:12
+; GCN-SDAG-NEXT:    scratch_load_b32 v59, off, s32 offset:16
+; GCN-SDAG-NEXT:    scratch_load_b32 v58, off, s32 offset:20
+; GCN-SDAG-NEXT:    scratch_load_b32 v57, off, s32 offset:24
+; GCN-SDAG-NEXT:    scratch_load_b32 v56, off, s32 offset:28
+; GCN-SDAG-NEXT:    scratch_load_b32 v45, off, s32 offset:32
+; GCN-SDAG-NEXT:    scratch_load_b32 v44, off, s32 offset:36
+; GCN-SDAG-NEXT:    scratch_load_b32 v43, off, s32 offset:40
+; GCN-SDAG-NEXT:    scratch_load_b32 v42, off, s32 offset:44
+; GCN-SDAG-NEXT:    scratch_load_b32 v41, off, s32 offset:48
+; GCN-SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:52
+; GCN-SDAG-NEXT:    s_wait_xcnt 0xe
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GCN-GISEL-LABEL: test_v64i32_load_store:
+; GCN-GISEL:       ; %bb.0:
+; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    s_clause 0xf
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:60
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v41, s32 offset:56
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v42, s32 offset:52
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v43, s32 offset:48
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v44, s32 offset:44
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v45, s32 offset:40
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v46, s32 offset:36
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v47, s32 offset:32
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v56, s32 offset:28
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v57, s32 offset:24
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v58, s32 offset:20
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v59, s32 offset:16
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v60, s32 offset:12
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v61, s32 offset:8
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v62, s32 offset:4
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v63, s32
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x8
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v46, v3 :: v_dual_mov_b32 v47, v4
+; GCN-GISEL-NEXT:    global_load_b128 v[2:5], v[0:1], off offset:32
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    scratch_store_b128 off, v[2:5], s32 offset:80 ; 16-byte Folded Spill
+; GCN-GISEL-NEXT:    s_clause 0xe
+; GCN-GISEL-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:48
+; GCN-GISEL-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:64
+; GCN-GISEL-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:80
+; GCN-GISEL-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:96
+; GCN-GISEL-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:112
+; GCN-GISEL-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:128
+; GCN-GISEL-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:144
+; GCN-GISEL-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:160
+; GCN-GISEL-NEXT:    global_load_b128 v[48:51], v[0:1], off offset:176
+; GCN-GISEL-NEXT:    global_load_b128 v[52:55], v[0:1], off offset:192
+; GCN-GISEL-NEXT:    global_load_b128 v[38:41], v[0:1], off offset:208
+; GCN-GISEL-NEXT:    global_load_b128 v[42:45], v[0:1], off offset:224
+; GCN-GISEL-NEXT:    global_load_b128 v[56:59], v[0:1], off
+; GCN-GISEL-NEXT:    global_load_b128 v[60:63], v[0:1], off offset:16
+; GCN-GISEL-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:240
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:64 ; 16-byte Folded Spill
+; GCN-GISEL-NEXT:    scratch_load_b128 v[0:3], off, s32 offset:80 th:TH_LOAD_LU ; 16-byte Folded Reload
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    s_clause 0xe
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[0:3], off offset:32
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[6:9], off offset:48
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[10:13], off offset:64
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[14:17], off offset:80
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[18:21], off offset:96
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[22:25], off offset:112
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[26:29], off offset:128
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[30:33], off offset:144
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[34:37], off offset:160
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[48:51], off offset:176
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[52:55], off offset:192
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[38:41], off offset:208
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[42:45], off offset:224
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[56:59], off
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[60:63], off offset:16
+; GCN-GISEL-NEXT:    scratch_load_b128 v[0:3], off, s32 offset:64 th:TH_LOAD_LU ; 16-byte Folded Reload
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[0:3], off offset:240
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x0
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v0, v62
+; GCN-GISEL-NEXT:    s_clause 0xf
+; GCN-GISEL-NEXT:    scratch_load_b32 v63, off, s32
+; GCN-GISEL-NEXT:    scratch_load_b32 v62, off, s32 offset:4
+; GCN-GISEL-NEXT:    scratch_load_b32 v61, off, s32 offset:8
+; GCN-GISEL-NEXT:    scratch_load_b32 v60, off, s32 offset:12
+; GCN-GISEL-NEXT:    scratch_load_b32 v59, off, s32 offset:16
+; GCN-GISEL-NEXT:    scratch_load_b32 v58, off, s32 offset:20
+; GCN-GISEL-NEXT:    scratch_load_b32 v57, off, s32 offset:24
+; GCN-GISEL-NEXT:    scratch_load_b32 v56, off, s32 offset:28
+; GCN-GISEL-NEXT:    scratch_load_b32 v47, off, s32 offset:32
+; GCN-GISEL-NEXT:    scratch_load_b32 v46, off, s32 offset:36
+; GCN-GISEL-NEXT:    scratch_load_b32 v45, off, s32 offset:40
+; GCN-GISEL-NEXT:    scratch_load_b32 v44, off, s32 offset:44
+; GCN-GISEL-NEXT:    scratch_load_b32 v43, off, s32 offset:48
+; GCN-GISEL-NEXT:    scratch_load_b32 v42, off, s32 offset:52
+; GCN-GISEL-NEXT:    scratch_load_b32 v41, off, s32 offset:56
+; GCN-GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:60
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+  %vec = load <64 x i32>, ptr addrspace(1) %ptr
+  store <64 x i32> %vec, ptr addrspace(1) %out, align 4
+  %elt = extractelement <64 x i32> %vec, i32 6
+  ret i32 %elt
+}
+
+define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %ptr_b, ptr addrspace(1) %out) {
+; GCN-SDAG-LABEL: test_v16i64_load_store:
+; GCN-SDAG:       ; %bb.0:
+; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GCN-SDAG-NEXT:    s_clause 0x3
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:12
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v41, s32 offset:8
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v42, s32 offset:4
+; GCN-SDAG-NEXT:    ; meta instruction
+; GCN-SDAG-NEXT:    scratch_store_b32 off, v43, s32
+; GCN-SDAG-NEXT:    s_clause 0x7
+; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:112
+; GCN-SDAG-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:96
+; GCN-SDAG-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:80
+; GCN-SDAG-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:48
+; GCN-SDAG-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:32
+; GCN-SDAG-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:16
+; GCN-SDAG-NEXT:    global_load_b128 v[26:29], v[0:1], off
+; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:64
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v16, 0x70
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v50, 0x60
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v51, 0 :: v_dual_mov_b32 v52, 48
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v38, 0x50 :: v_dual_mov_b32 v53, 0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v54, 32
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v14, 0xc8 :: v_dual_mov_b32 v15, 0
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v39, 0 :: v_dual_mov_b32 v48, 64
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v55, 0 :: v_dual_mov_b32 v40, 16
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v49, 0
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v41, 0 :: v_dual_mov_b32 v42, 0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v43, 0
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GCN-SDAG-NEXT:    global_store_b128 v[16:17], v[6:9], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x6
+; GCN-SDAG-NEXT:    global_store_b128 v[50:51], v[10:13], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x5
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x1
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v16, v20 :: v_dual_mov_b32 v17, v21
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[12:13], v[12:13], 0, v[12:13]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[10:11], v[10:11], 0, v[10:11]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[8:9], v[8:9], 0, v[8:9]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[6:7], v[6:7], 0, v[6:7]
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x4
+; GCN-SDAG-NEXT:    global_store_b128 v[52:53], v[34:37], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x3
+; GCN-SDAG-NEXT:    global_store_b128 v[54:55], v[30:33], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x2
+; GCN-SDAG-NEXT:    global_store_b128 v[40:41], v[22:25], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x1
+; GCN-SDAG-NEXT:    global_store_b128 v[42:43], v[26:29], off
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x3
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[52:53], v[2:3], 0, v[2:3]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[50:51], v[0:1], 0, v[0:1]
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x1
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[24:25], v[24:25], 0, v[24:25]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[22:23], v[22:23], 0, v[22:23]
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[28:29], v[28:29], 0, v[28:29]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[26:27], v[26:27], 0, v[26:27]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[36:37], v[36:37], 0, v[36:37]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[34:35], v[34:35], 0, v[34:35]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[32:33], v[32:33], 0, 0x64
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[30:31], v[30:31], 0, v[30:31]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[20:21], v[20:21], 0, v[20:21]
+; GCN-SDAG-NEXT:    v_lshl_add_u64 v[18:19], v[18:19], 0, 0xc8
+; GCN-SDAG-NEXT:    s_clause 0x1
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[14:17], off
+; GCN-SDAG-NEXT:    global_store_b128 v[48:49], v[0:3], off
+; GCN-SDAG-NEXT:    s_clause 0x7
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[10:13], off offset:96
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:112
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[50:53], off offset:64
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:80
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[30:33], off offset:32
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[34:37], off offset:48
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[26:29], off
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[22:25], off offset:16
+; GCN-SDAG-NEXT:    s_clause 0x3
+; GCN-SDAG-NEXT:    scratch_load_b32 v43, off, s32
+; GCN-SDAG-NEXT:    scratch_load_b32 v42, off, s32 offset:4
+; GCN-SDAG-NEXT:    scratch_load_b32 v41, off, s32 offset:8
+; GCN-SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:12
+; GCN-SDAG-NEXT:    s_wait_xcnt 0xc
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v0, v28 :: v_dual_mov_b32 v1, v29
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GCN-GISEL-LABEL: test_v16i64_load_store:
+; GCN-GISEL:       ; %bb.0:
+; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    s_clause 0x5
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:20
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v41, s32 offset:16
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v42, s32 offset:12
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v43, s32 offset:8
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v44, s32 offset:4
+; GCN-GISEL-NEXT:    ; meta instruction
+; GCN-GISEL-NEXT:    scratch_store_b32 off, v45, s32
+; GCN-GISEL-NEXT:    s_clause 0x7
+; GCN-GISEL-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:80
+; GCN-GISEL-NEXT:    global_load_b128 v[10:13], v[0:1], off
+; GCN-GISEL-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:16
+; GCN-GISEL-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:32
+; GCN-GISEL-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:48
+; GCN-GISEL-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:96
+; GCN-GISEL-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:112
+; GCN-GISEL-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:64
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v34, 0xc8
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v35, 0 :: v_dual_mov_b32 v38, 0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v39, 0 :: v_dual_mov_b32 v48, 16
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v49, 0 :: v_dual_mov_b32 v50, 32
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v52, 48 :: v_dual_mov_b32 v51, 0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v53, 0 :: v_dual_mov_b32 v54, 64
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v40, 0x50 :: v_dual_mov_b32 v55, 0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v41, 0 :: v_dual_mov_b32 v42, 0x60
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v44, 0x70 :: v_dual_mov_b32 v43, 0
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v45, 0
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x7
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v37, v9 :: v_dual_mov_b32 v36, v8
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[6:7], v[6:7], 0, 0xc8
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[8:9], v[8:9], 0, v[8:9]
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x6
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[10:13], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x5
+; GCN-GISEL-NEXT:    global_store_b128 v[48:49], v[14:17], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x4
+; GCN-GISEL-NEXT:    global_store_b128 v[50:51], v[18:21], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x3
+; GCN-GISEL-NEXT:    global_store_b128 v[52:53], v[22:25], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x2
+; GCN-GISEL-NEXT:    global_store_b128 v[42:43], v[26:29], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x1
+; GCN-GISEL-NEXT:    global_store_b128 v[44:45], v[30:33], off
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x5
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[10:11], v[10:11], 0, v[10:11]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[12:13], v[12:13], 0, v[12:13]
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x4
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[14:15], v[14:15], 0, v[14:15]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[16:17], v[16:17], 0, v[16:17]
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x3
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[18:19], v[18:19], 0, v[18:19]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[20:21], v[20:21], 0, 0x64
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x2
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[22:23], v[22:23], 0, v[22:23]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[24:25], v[24:25], 0, v[24:25]
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[48:49], v[0:1], 0, v[0:1]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[50:51], v[2:3], 0, v[2:3]
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x1
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[26:27], v[26:27], 0, v[26:27]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[28:29], v[28:29], 0, v[28:29]
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x0
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[30:31], v[30:31], 0, v[30:31]
+; GCN-GISEL-NEXT:    v_lshl_add_u64 v[32:33], v[32:33], 0, v[32:33]
+; GCN-GISEL-NEXT:    s_clause 0x1
+; GCN-GISEL-NEXT:    global_store_b128 v[54:55], v[0:3], off
+; GCN-GISEL-NEXT:    global_store_b128 v[40:41], v[34:37], off
+; GCN-GISEL-NEXT:    s_clause 0x7
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[10:13], off
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:16
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:32
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[22:25], off offset:48
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[48:51], off offset:64
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:80
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[26:29], off offset:96
+; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[30:33], off offset:112
+; GCN-GISEL-NEXT:    s_clause 0x5
+; GCN-GISEL-NEXT:    scratch_load_b32 v45, off, s32
+; GCN-GISEL-NEXT:    scratch_load_b32 v44, off, s32 offset:4
+; GCN-GISEL-NEXT:    scratch_load_b32 v43, off, s32 offset:8
+; GCN-GISEL-NEXT:    scratch_load_b32 v42, off, s32 offset:12
+; GCN-GISEL-NEXT:    scratch_load_b32 v41, off, s32 offset:16
+; GCN-GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:20
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v0, v12 :: v_dual_mov_b32 v1, v13
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
+  %a = load <16 x i64>, ptr addrspace(1) %ptr_a, align 4
+  %in_a = insertelement <16 x i64> %a, i64 100, i32 5
+  store <16 x i64> %in_a, ptr addrspace(1) null
+  %b = load <16 x i64>, ptr addrspace(1) %ptr_b, align 4
+  %in_b = insertelement <16 x i64> %a, i64 200, i32 10
+  store <16 x i64> %in_b, ptr addrspace(1) null
+  %add = add <16 x i64> %in_a, %in_b
+  store <16 x i64> %add, ptr addrspace(1) %out, align 4
+  %elt = extractelement <16 x i64> %add, i32 1
+  ret i64 %elt
+}
+
+define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2, ptr addrspace(1) %out) {
+; GCN-SDAG-LABEL: test_v7i16_load_store_kernel:
+; GCN-SDAG:       ; %bb.0:
+; GCN-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GCN-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v8, 12
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 8
+; GCN-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v11, 0 :: v_dual_lshlrev_b32 v4, 4, v0
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GCN-SDAG-NEXT:    s_clause 0x1
+; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v4, s[0:1]
+; GCN-SDAG-NEXT:    global_load_b128 v[4:7], v4, s[2:3]
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    v_pk_add_u16 v3, v3, v7
+; GCN-SDAG-NEXT:    v_pk_add_u16 v2, v2, v6
+; GCN-SDAG-NEXT:    v_pk_add_u16 v1, v1, v5
+; GCN-SDAG-NEXT:    v_pk_add_u16 v0, v0, v4
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-SDAG-NEXT:    s_clause 0x2
+; GCN-SDAG-NEXT:    global_store_b16 v[8:9], v3, off
+; GCN-SDAG-NEXT:    global_store_b32 v[10:11], v2, off
+; GCN-SDAG-NEXT:    global_store_b64 v[12:13], v[0:1], off
+; GCN-SDAG-NEXT:    global_store_d16_hi_b16 v4, v2, s[4:5]
+; GCN-SDAG-NEXT:    s_endpgm
+;
+; GCN-GISEL-LABEL: test_v7i16_load_store_kernel:
+; GCN-GISEL:       ; %bb.0:
+; GCN-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x0
+; GCN-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 2
+; GCN-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v11, 0 :: v_dual_lshlrev_b32 v4, 4, v0
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x0
+; GCN-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x10
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v12, 4
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 6
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v16, 8
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v18, 10
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v20, 12 :: v_dual_mov_b32 v19, 0
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v21, 0
+; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    s_clause 0x1
+; GCN-GISEL-NEXT:    global_load_b128 v[0:3], v4, s[0:1]
+; GCN-GISEL-NEXT:    global_load_b128 v[4:7], v4, s[2:3]
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    v_pk_add_u16 v0, v0, v4
+; GCN-GISEL-NEXT:    v_pk_add_u16 v1, v1, v5
+; GCN-GISEL-NEXT:    v_pk_add_u16 v2, v2, v6
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-GISEL-NEXT:    v_pk_add_u16 v3, v3, v7
+; GCN-GISEL-NEXT:    s_clause 0x6
+; GCN-GISEL-NEXT:    global_store_b16 v[8:9], v0, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[10:11], v0, off
+; GCN-GISEL-NEXT:    global_store_b16 v[12:13], v1, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[14:15], v1, off
+; GCN-GISEL-NEXT:    global_store_b16 v[16:17], v2, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v[18:19], v2, off
+; GCN-GISEL-NEXT:    global_store_b16 v[20:21], v3, off
+; GCN-GISEL-NEXT:    global_store_d16_hi_b16 v4, v2, s[4:5]
+; GCN-GISEL-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep1 = getelementptr inbounds <7 x i16>, ptr addrspace(1) %ptr1, i32 %tid
+  %gep2 = getelementptr inbounds <7 x i16>, ptr addrspace(1) %ptr2, i32 %tid
+  %vec1 = load <7 x i16>, ptr addrspace(1) %gep1
+  %insert = insertelement <7 x i16> %vec1, i16 20, i32 4
+  %vec2 = load <7 x i16>, ptr addrspace(1) %gep2
+  %add = add <7 x i16> %vec1, %vec2
+  store <7 x i16> %add, ptr addrspace(1) null
+  %elt = extractelement <7 x i16> %add, i32 5
+  store i16 %elt, ptr addrspace(1) %out
+  ret void
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 82c58394c03b..37756d15861b 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -5736,966 +5736,469 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[0:1], v[3:4], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_mov_b32 s2, 0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[3:4], v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
-; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -6705,7 +6208,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB10_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -6715,102 +6218,70 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -6823,99 +6294,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -6928,93 +6364,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7023,61 +6423,32 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7085,31 +6456,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7118,7 +6482,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -7133,81 +6497,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7218,522 +6549,358 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
-; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
-; GFX1132-NEXT:  .LBB10_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
+; GFX1132-NEXT:  .LBB10_5:
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -7741,238 +6908,175 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value()
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
@@ -11760,998 +10864,555 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s14, -1
+; GFX1064-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s14, -1
+; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    s_clause 0x1
-; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT:    s_clause 0x1
+; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_clause 0x1
-; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT:    s_clause 0x1
+; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    s_clause 0x1
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_clause 0x1
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_clause 0x1
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT:    s_clause 0x1
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -12761,7 +11422,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB17_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -12771,102 +11432,70 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -12879,99 +11508,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -12984,93 +11578,57 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13079,61 +11637,32 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13141,31 +11670,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13174,7 +11696,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -13189,81 +11711,48 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13274,522 +11763,358 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -13797,238 +12122,175 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value() strictfp
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
@@ -14374,7 +12636,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
 ; GFX1132-DPP-NEXT:    global_atomic_add_f32 v1, v0, s[2:3]
 ; GFX1132-DPP-NEXT:  .LBB18_2:
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0  monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !amdgpu.ignore.denormal.mode !1
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0  monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !amdgpu.ignore.denormal.mode !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index f8f911b693e0..c8b24f71d573 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -3134,932 +3134,377 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-NEXT:  .LBB6_3:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB6_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-NEXT:  .LBB6_3:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB6_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-DPP-NEXT:  .LBB6_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB6_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-DPP-NEXT:  .LBB6_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB6_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -4067,7 +3512,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB7_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -4077,109 +3522,77 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4189,90 +3602,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -4296,93 +3674,42 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1064-NEXT:  .LBB7_5:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB7_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -4402,96 +3729,45 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1032-NEXT:  .LBB7_5:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB7_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -4500,7 +3776,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4511,89 +3787,57 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -4601,811 +3845,557 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    s_nop 1
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1064-DPP-NEXT:  .LBB7_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB7_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1032-DPP-NEXT:  .LBB7_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB7_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
@@ -6680,932 +5670,377 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-NEXT:  .LBB10_3:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-NEXT:  .LBB10_3:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-DPP-NEXT:  .LBB10_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB10_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-DPP-NEXT:  .LBB10_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB10_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -7613,7 +6048,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB11_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7623,109 +6058,77 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -7735,90 +6138,55 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -7842,93 +6210,42 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1064-NEXT:  .LBB11_5:
+; GFX1064-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB11_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -7948,96 +6265,45 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1032-NEXT:  .LBB11_5:
+; GFX1032-NEXT:    global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB11_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -8046,7 +6312,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8057,89 +6323,57 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -8147,811 +6381,557 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1064-DPP-NEXT:  .LBB11_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB11_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1032-DPP-NEXT:  .LBB11_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB11_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 1f76a476107a..4fccfc051d56 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -3134,932 +3134,377 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-NEXT:  .LBB6_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-NEXT:  .LBB6_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-NEXT:  .LBB6_3:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB6_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-NEXT:  .LBB6_3:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB6_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-NEXT:  .LBB6_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-NEXT:  .LBB6_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX7LESS-DPP-NEXT:  .LBB6_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX9-DPP-NEXT:  .LBB6_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1064-DPP-NEXT:  .LBB6_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB6_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB6_2
-; GFX1032-DPP-NEXT:  .LBB6_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB6_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1164-DPP-NEXT:  .LBB6_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB6_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB6_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB6_2
 ; GFX1132-DPP-NEXT:  .LBB6_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -4067,7 +3512,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB7_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -4077,109 +3522,77 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX7LESS-NEXT:  .LBB7_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4189,90 +3602,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX9-NEXT:  .LBB7_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -4296,93 +3674,42 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1064-NEXT:  .LBB7_5:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB7_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -4402,96 +3729,45 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB7_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB7_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB7_4
-; GFX1032-NEXT:  .LBB7_5:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB7_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -4500,7 +3776,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -4511,89 +3787,57 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1164-NEXT:  .LBB7_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB7_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -4601,811 +3845,557 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB7_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB7_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB7_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB7_4
 ; GFX1132-NEXT:  .LBB7_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB7_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB7_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    s_nop 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    s_nop 1
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_min_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX9-DPP-NEXT:  .LBB7_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1064-DPP-NEXT:  .LBB7_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB7_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB7_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB7_2
-; GFX1032-DPP-NEXT:  .LBB7_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB7_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1164-DPP-NEXT:  .LBB7_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB7_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB7_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB7_2
 ; GFX1132-DPP-NEXT:  .LBB7_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
@@ -6680,932 +5670,377 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-NEXT:  .LBB10_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-NEXT:  .LBB10_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-NEXT:  .LBB10_3:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT:  .LBB10_2:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-NEXT:  .LBB10_3:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-NEXT:  .LBB10_2:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-NEXT:  .LBB10_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-NEXT:  .LBB10_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, exec_hi, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[2:3], 4.0
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v5
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX7LESS-DPP-NEXT:  .LBB10_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1064-DPP-NEXT:  .LBB10_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB10_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
-; GFX1032-DPP-NEXT:  .LBB10_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB10_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], 4.0
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX7LESS-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX7LESS-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX7LESS-NEXT:    s_lshl_b64 s[4:5], 1, s4
@@ -7613,7 +6048,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX7LESS-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB11_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7623,109 +6058,77 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX7LESS-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_max_f64 v[2:3], v[0:1], v[0:1]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[2:3], v[41:42]
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7LESS-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX7LESS-NEXT:  .LBB11_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX9-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX9-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX9-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX9-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX9-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX9-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX9-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -7735,90 +6138,55 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX9-NEXT:  .LBB11_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-NEXT:    v_mov_b32_e32 v2, 0
@@ -7842,93 +6210,42 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
-; GFX1064-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1064-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1064-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1064-NEXT:  .LBB11_5:
+; GFX1064-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT:  .LBB11_4:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-NEXT:    v_mov_b32_e32 v2, 0
@@ -7948,96 +6265,45 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
-; GFX1032-NEXT:    s_cbranch_execz .LBB11_5
+; GFX1032-NEXT:    s_cbranch_execz .LBB11_4
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[4:5], v0, s[52:53]
-; GFX1032-NEXT:  .LBB11_4: ; %atomicrmw.start
-; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v4, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v5, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-NEXT:    s_cbranch_execnz .LBB11_4
-; GFX1032-NEXT:  .LBB11_5:
+; GFX1032-NEXT:    global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT:  .LBB11_4:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
@@ -8046,7 +6312,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-NEXT:    s_cmp_lg_u64 s[0:1], 0
-; GFX1164-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1164-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1164-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8057,89 +6323,57 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1164-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1164-NEXT:  .LBB11_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v2, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB11_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-NEXT:    s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[2:3], v[4:5], v[4:5]
 ; GFX1132-NEXT:    v_readlane_b32 s3, v1, s1
 ; GFX1132-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
@@ -8147,811 +6381,557 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
 ; GFX1132-NEXT:    v_max_f64 v[4:5], s[2:3], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX1132-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-NEXT:    v_min_f64 v[4:5], v[2:3], v[4:5]
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB11_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB11_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT:    v_max_f64 v[41:42], v[2:3], v[2:3]
+; GFX1132-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[4:5], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB11_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX1132-NEXT:    scratch_store_b64 off, v[4:5], off
-; GFX1132-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 8
-; GFX1132-NEXT:    scratch_store_b64 off, v[0:1], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v0, 8 :: v_dual_mov_b32 v1, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[4:5], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB11_4
 ; GFX1132-NEXT:  .LBB11_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_max_f64 v[41:42], v[0:1], v[0:1]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[2:3], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:    v_max_f64 v[4:5], v[0:1], v[0:1]
 ; GFX7LESS-DPP-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7LESS-DPP-NEXT:    v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[41:42]
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    v_min_f64 v[0:1], v[0:1], v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB11_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v13, 0x7ff80000
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0x7ff80000
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[12:13], v[12:13], v[12:13]
-; GFX9-DPP-NEXT:    v_min_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[7:8], v[7:8], v[7:8]
+; GFX9-DPP-NEXT:    v_min_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX9-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT:    v_max_f64 v[3:4], s[52:53], s[52:53]
+; GFX9-DPP-NEXT:    v_max_f64 v[1:2], s[0:1], s[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_max_f64 v[9:10], v[11:12], v[11:12]
+; GFX9-DPP-NEXT:    v_min_f64 v[9:10], v[9:10], v[1:2]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX9-DPP-NEXT:  .LBB11_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_max_f64 v[8:9], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT:    v_max_f64 v[10:11], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], s[4:5], s[4:5]
+; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], s[2:3], s[2:3]
+; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1064-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT:    v_max_f64 v[3:4], v[41:42], v[41:42]
-; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_max_f64 v[5:6], v[1:2], v[1:2]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1064-DPP-NEXT:  .LBB11_3:
+; GFX1064-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT:  .LBB11_2:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1032-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[3:4], v[3:4]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0x7ff80000
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_3
+; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB11_2
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
-; GFX1032-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
-; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
-; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB11_2
-; GFX1032-DPP-NEXT:  .LBB11_3:
+; GFX1032-DPP-NEXT:    global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT:  .LBB11_2:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v11, 0x7ff80000
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 0x7ff80000
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1164-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1164-DPP-NEXT:  .LBB11_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_div_value_default_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.double.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.double.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.double.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.double.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, 0x7ff80000 :: v_dual_mov_b32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[8:9], v[8:9], v[8:9]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v11, 0x7ff80000 :: v_dual_mov_b32 v10, 0
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_max_f64 v[10:11], v[10:11], v[10:11]
-; GFX1132-DPP-NEXT:    v_min_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT:    v_min_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB11_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-DPP-NEXT:    v_max_f64 v[41:42], v[3:4], v[3:4]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB11_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_max_f64 v[3:4], v[1:2], v[1:2]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-DPP-NEXT:    v_min_f64 v[3:4], v[3:4], v[41:42]
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT:    v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB11_2
 ; GFX1132-DPP-NEXT:  .LBB11_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.double.value()
-  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 4, !amdgpu.no.fine.grained.memory !1
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %divValue monotonic, align 8, !amdgpu.no.fine.grained.memory !1
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 9db3c37045cc..bb119ebd11da 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -6064,966 +6064,469 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-NEXT:  .LBB9_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[0:1], v[3:4], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-NEXT:  .LBB9_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-NEXT:  .LBB9_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-NEXT:  .LBB9_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-NEXT:  .LBB9_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-NEXT:    s_mov_b32 s2, 0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-NEXT:  .LBB9_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v3, s0, 0
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v3, s1, v3
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[1:2], s2
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v4, v0, v4
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], v[1:2], 4.0
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v4, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX7LESS-DPP-NEXT:  .LBB9_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s0, 0
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s1, v3
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[3:4], -v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v3, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v4, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX9-DPP-NEXT:  .LBB9_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s10, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, s11, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1064-DPP-NEXT:  .LBB9_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s9, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, s9, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    s_mov_b32 s3, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[3:4], s0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s3, s3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], v[3:4], 4.0
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1032-DPP-NEXT:  .LBB9_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s10, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s11, v0
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
 ; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[10:11]
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1164-DPP-NEXT:  .LBB9_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b32 s8, exec_lo
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB9_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s8
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], v[0:1], 4.0
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], v[0:1], 4.0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB9_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB9_2
 ; GFX1132-DPP-NEXT:  .LBB9_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 syncscope("agent") monotonic, align 8
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe(ptr addrspace(1) %ptr) #0 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -7033,7 +6536,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB10_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -7043,102 +6546,70 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX7LESS-NEXT:  .LBB10_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -7151,99 +6622,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX9-NEXT:  .LBB10_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -7256,93 +6692,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1064-NEXT:  .LBB10_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7351,61 +6751,32 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1032-NEXT:  .LBB10_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -7413,31 +6784,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7446,7 +6810,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -7461,81 +6825,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1164-NEXT:  .LBB10_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB10_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -7546,522 +6877,358 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB10_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB10_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB10_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB10_4
 ; GFX1132-NEXT:  .LBB10_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
-; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
+; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
+; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7LESS-DPP-NEXT:  .LBB10_1: ; %atomicrmw.start
+; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], -v[0:1]
+; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB10_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], -s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX9-DPP-NEXT:  .LBB10_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1064-DPP-NEXT:  .LBB10_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1032-DPP-NEXT:  .LBB10_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -8069,238 +7236,175 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1164-DPP-NEXT:  .LBB10_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_agent_scope_align4_unsafe:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB10_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB10_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB10_2
 ; GFX1132-DPP-NEXT:  .LBB10_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value()
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue syncscope("agent") monotonic, align 8
   ret void
 }
 
@@ -12086,998 +11190,555 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-NEXT:  ; %bb.1:
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-NEXT:  .LBB16_3:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-NEXT:  ; %bb.1:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b32 s50, s9
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-NEXT:  .LBB16_3:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s14, -1
+; GFX1064-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-NEXT:  ; %bb.1:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b32 s50, s9
-; GFX1064-NEXT:    s_mov_b32 s51, s8
-; GFX1064-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-NEXT:  .LBB16_3:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT:    s_mov_b32 s54, 0
-; GFX1032-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s14, -1
+; GFX1032-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
+; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-NEXT:  ; %bb.1:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b32 s50, s9
-; GFX1032-NEXT:    s_mov_b32 s51, s8
-; GFX1032-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-NEXT:  .LBB16_3:
 ; GFX1032-NEXT:    s_endpgm
 ;
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    s_clause 0x1
-; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT:    s_clause 0x1
+; GFX1164-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-NEXT:  ; %bb.1:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b32 s50, s9
-; GFX1164-NEXT:    s_mov_b32 s51, s8
-; GFX1164-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-NEXT:  .LBB16_3:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-NEXT:    s_mov_b32 s54, 0
-; GFX1132-NEXT:    s_clause 0x1
-; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-NEXT:    s_mov_b32 s32, 32
+; GFX1132-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT:    s_clause 0x1
+; GFX1132-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-NEXT:  ; %bb.1:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-NEXT:  .LBB16_3:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v5, exec_lo, 0
-; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v5, exec_hi, v5
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX7LESS-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s15, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], exec
+; GFX7LESS-DPP-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
+; GFX7LESS-DPP-NEXT:    v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
 ; GFX7LESS-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX7LESS-DPP-NEXT:  ; %bb.1:
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_bcnt1_i32_b64 s6, s[2:3]
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v3, v0, v1
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX7LESS-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v40, v3, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX7LESS-DPP-NEXT:  .LBB16_3:
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0xc3300000
-; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], s[0:1], v[3:4]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX9-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT:    s_mov_b32 s15, 0xe00000
+; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX9-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-DPP-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0xc3300000
+; GFX9-DPP-NEXT:    s_mov_b32 s1, 0x43300000
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], s[0:1], v[0:1]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DPP-NEXT:    s_mov_b64 s[54:55], 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX9-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX9-DPP-NEXT:  .LBB16_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1064-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1064-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v3, exec_hi, v3
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s15, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX1064-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-DPP-NEXT:    s_mov_b32 s3, 0x43300000
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1064-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1064-DPP-NEXT:  .LBB16_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
-; GFX1032-DPP-NEXT:    s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], 0xc3300000, s[0:1]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
-; GFX1032-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[3:4]
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT:    s_addc_u32 s13, s13, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x24
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v3, 20, v2
-; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v4, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v4, v3
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT:    s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, s[6:7]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[52:53], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX1032-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1032-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1032-DPP-NEXT:  .LBB16_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
 ;
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1164-DPP-NEXT:    s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    s_clause 0x1
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
+; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_clause 0x1
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1164-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1164-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v2, exec_hi, v2
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1164-DPP-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1164-DPP-NEXT:  .LBB16_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
 ; GFX1132-DPP-NEXT:    s_bcnt1_i32_b32 s0, exec_lo
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v40, v0 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, 0x43300000
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1132-DPP-NEXT:    s_clause 0x1
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:20
-; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off offset:16
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off offset:16
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mul_f64 v[41:42], 4.0, v[0:1]
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT:    s_clause 0x1
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v0, off offset:4
+; GFX1132-DPP-NEXT:    scratch_store_b32 off, v1, off
+; GFX1132-DPP-NEXT:    scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v2
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB16_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[4:5], 0x24
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[52:53], 0x0
+; GFX1132-DPP-NEXT:    s_load_b64 s[4:5], s[0:1], 0x0
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT:    v_mul_f64 v[4:5], 4.0, v[0:1]
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
 ; GFX1132-DPP-NEXT:  .LBB16_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB16_2
 ; GFX1132-DPP-NEXT:  .LBB16_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double 4.0 monotonic, align 8
   ret void
 }
 
 define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
 ; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS:       ; %bb.0:
-; GFX7LESS-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX7LESS-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX7LESS-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-NEXT:    s_mov_b32 s39, 0xe8f000
+; GFX7LESS-NEXT:    s_add_u32 s36, s36, s11
+; GFX7LESS-NEXT:    s_addc_u32 s37, s37, 0
+; GFX7LESS-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s8, s34, 44
+; GFX7LESS-NEXT:    s_addc_u32 s9, s35, 0
+; GFX7LESS-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-NEXT:    v_or_b32_e32 v40, v0, v2
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[38:39]
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX7LESS-NEXT:    s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT:    v_mov_b32_e32 v41, 0
-; GFX7LESS-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
+; GFX7LESS-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX7LESS-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_ff1_i32_b64 s4, s[0:1]
@@ -13087,7 +11748,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
 ; GFX7LESS-NEXT:    v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
 ; GFX7LESS-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX7LESS-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX7LESS-NEXT:    s_cbranch_vccnz .LBB17_1
 ; GFX7LESS-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX7LESS-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
@@ -13097,102 +11758,70 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX7LESS-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX7LESS-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX7LESS-NEXT:  ; %bb.3:
-; GFX7LESS-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x9
-; GFX7LESS-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-NEXT:    s_mov_b32 s54, -1
+; GFX7LESS-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7LESS-NEXT:    s_mov_b32 s2, -1
 ; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7LESS-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX7LESS-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT:    v_add_f64 v[2:3], v[0:1], -v[41:42]
-; GFX7LESS-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-NEXT:    v_mov_b32_e32 v31, v40
-; GFX7LESS-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
 ; GFX7LESS-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7LESS-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7LESS-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7LESS-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7LESS-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7LESS-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7LESS-NEXT:    v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT:    v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX7LESS-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX7LESS-NEXT:  .LBB17_5:
 ; GFX7LESS-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s66, -1
-; GFX9-NEXT:    s_mov_b32 s67, 0xe00000
-; GFX9-NEXT:    s_add_u32 s64, s64, s11
-; GFX9-NEXT:    s_addc_u32 s65, s65, 0
-; GFX9-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-NEXT:    s_mov_b32 s51, s8
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_mov_b32 s50, s9
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-NEXT:    s_mov_b32 s38, -1
+; GFX9-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-NEXT:    s_mov_b32 s12, s8
+; GFX9-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-NEXT:    s_mov_b32 s13, s9
+; GFX9-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-NEXT:    s_mov_b32 s14, s10
+; GFX9-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT:    s_mov_b32 s33, s10
-; GFX9-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT:    s_mov_b32 s32, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    v_mov_b32_e32 v41, 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
-; GFX9-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX9-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX9-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX9-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX9-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX9-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX9-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX9-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -13205,99 +11834,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX9-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX9-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX9-NEXT:  ; %bb.3:
-; GFX9-NEXT:    s_load_dwordx2 s[52:53], s[36:37], 0x24
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    s_mov_b64 s[54:55], 0
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v6, 0
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX9-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX9-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX9-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX9-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX9-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-NEXT:    s_mov_b32 s12, s51
-; GFX9-NEXT:    s_mov_b32 s13, s50
-; GFX9-NEXT:    s_mov_b32 s14, s33
-; GFX9-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX9-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-NEXT:    v_mov_b32_e32 v2, s52
-; GFX9-NEXT:    v_mov_b32_e32 v3, s53
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX9-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX9-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX9-NEXT:  .LBB17_5:
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX1064-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064:       ; %bb.0:
-; GFX1064-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT:    s_mov_b32 s66, -1
-; GFX1064-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-NEXT:    s_mov_b32 s38, -1
+; GFX1064-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-NEXT:    s_mov_b32 s51, s8
+; GFX1064-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_mov_b32 s50, s9
+; GFX1064-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-NEXT:    s_mov_b32 s33, s10
-; GFX1064-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-NEXT:    s_mov_b32 s14, s10
+; GFX1064-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1064-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1064-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1064-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1064-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_ff1_i32_b64 s4, s[0:1]
 ; GFX1064-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1064-NEXT:    v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1064-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1064-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1064-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1064-NEXT:    s_cmp_lg_u64 s[0:1], 0
@@ -13310,93 +11904,57 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1064-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1064-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1064-NEXT:  ; %bb.3:
-; GFX1064-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1064-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1064-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1064-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-NEXT:    s_mov_b32 s12, s51
-; GFX1064-NEXT:    s_mov_b32 s13, s50
-; GFX1064-NEXT:    s_mov_b32 s14, s33
-; GFX1064-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT:    s_clause 0x1
-; GFX1064-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1064-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1064-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1064-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1064-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1064-NEXT:  .LBB17_5:
 ; GFX1064-NEXT:    s_endpgm
 ;
 ; GFX1032-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032:       ; %bb.0:
-; GFX1032-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT:    s_mov_b32 s66, -1
-; GFX1032-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-NEXT:    s_mov_b32 s38, -1
+; GFX1032-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-NEXT:    s_mov_b32 s51, s8
+; GFX1032-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_mov_b32 s50, s9
+; GFX1032-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-NEXT:    s_mov_b32 s33, s10
-; GFX1032-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-NEXT:    s_mov_b32 s14, s10
+; GFX1032-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1032-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1032-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1032-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1032-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13405,61 +11963,32 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1032-NEXT:    v_readlane_b32 s2, v0, s1
 ; GFX1032-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1032-NEXT:    s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1032-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1032-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1032-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1032-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT:    s_mov_b32 s54, 0
+; GFX1032-NEXT:    s_mov_b32 s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1032-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1032-NEXT:  ; %bb.3:
-; GFX1032-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-NEXT:    global_load_dwordx2 v[2:3], v6, s[0:1]
 ; GFX1032-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1032-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-NEXT:    s_mov_b32 s12, s51
-; GFX1032-NEXT:    s_mov_b32 s13, s50
-; GFX1032-NEXT:    s_mov_b32 s14, s33
-; GFX1032-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT:    s_clause 0x1
-; GFX1032-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1032-NEXT:    global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1032-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1032-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1032-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1032-NEXT:  .LBB17_5:
 ; GFX1032-NEXT:    s_endpgm
@@ -13467,31 +11996,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164:       ; %bb.0:
 ; GFX1164-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-NEXT:    s_mov_b32 s51, s8
+; GFX1164-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_mov_b32 s50, s9
+; GFX1164-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    s_mov_b32 s33, s10
-; GFX1164-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-NEXT:    s_mov_b32 s14, s10
+; GFX1164-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    s_mov_b32 s32, 32
-; GFX1164-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1164-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1164-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13500,7 +12022,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    v_readlane_b32 s3, v1, s4
 ; GFX1164-NEXT:    v_readlane_b32 s2, v0, s4
 ; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1164-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1164-NEXT:    s_lshl_b64 s[2:3], 1, s4
 ; GFX1164-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
 ; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -13515,81 +12037,48 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
 ; GFX1164-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1164-NEXT:  ; %bb.3:
-; GFX1164-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
+; GFX1164-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-NEXT:    .p2align 6
+; GFX1164-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1164-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1164-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-NEXT:    s_mov_b32 s12, s51
-; GFX1164-NEXT:    s_mov_b32 s13, s50
-; GFX1164-NEXT:    s_mov_b32 s14, s33
-; GFX1164-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1164-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX1164-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-NEXT:    v_mov_b32_e32 v2, v0
+; GFX1164-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1164-NEXT:  .LBB17_5:
-; GFX1164-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-NEXT:    s_endpgm
 ;
 ; GFX1132-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132:       ; %bb.0:
 ; GFX1132-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-NEXT:    s_mov_b32 s50, s14
-; GFX1132-NEXT:    s_mov_b32 s51, s13
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-NEXT:    s_mov_b32 s12, s13
+; GFX1132-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-NEXT:    s_mov_b32 s14, s15
-; GFX1132-NEXT:    s_mov_b32 s32, 32
-; GFX1132-NEXT:    s_mov_b32 s33, s15
-; GFX1132-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    v_mov_b32_e32 v41, 0
-; GFX1132-NEXT:    v_bfrev_b32_e32 v42, 1
+; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-NEXT:    v_bfrev_b32_e32 v5, 1
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:  .LBB17_1: ; %ComputeLoop
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -13600,522 +12089,358 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1132-NEXT:    s_lshl_b32 s1, 1, s1
 ; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX1132-NEXT:    s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT:    v_add_f64 v[41:42], v[41:42], s[2:3]
+; GFX1132-NEXT:    v_add_f64 v[4:5], v[4:5], s[2:3]
 ; GFX1132-NEXT:    s_cmp_lg_u32 s0, 0
 ; GFX1132-NEXT:    s_cbranch_scc1 .LBB17_1
 ; GFX1132-NEXT:  ; %bb.2: ; %ComputeEnd
 ; GFX1132-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT:    s_mov_b32 s54, 0
+; GFX1132-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-NEXT:    v_cmpx_eq_u32_e32 0, v0
 ; GFX1132-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX1132-NEXT:    s_cbranch_execz .LBB17_5
 ; GFX1132-NEXT:  ; %bb.3:
-; GFX1132-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1132-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1132-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT:    v_mov_b32_e32 v6, 0
 ; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-NEXT:    .p2align 6
+; GFX1132-NEXT:    global_load_b64 v[2:3], v6, s[0:1]
 ; GFX1132-NEXT:  .LBB17_4: ; %atomicrmw.start
 ; GFX1132-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-NEXT:    s_mov_b32 s12, s51
-; GFX1132-NEXT:    s_mov_b32 s13, s50
-; GFX1132-NEXT:    s_mov_b32 s14, s33
-; GFX1132-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-NEXT:    v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX1132-NEXT:    global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX1132-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-NEXT:    s_cbranch_execnz .LBB17_4
 ; GFX1132-NEXT:  .LBB17_5:
-; GFX1132-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-NEXT:    s_endpgm
 ;
 ; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX7LESS-DPP:       ; %bb.0:
-; GFX7LESS-DPP-NEXT:    s_movk_i32 s32, 0x800
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s67, 0xe8f000
-; GFX7LESS-DPP-NEXT:    s_add_u32 s64, s64, s11
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[52:53], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s55, 0xf000
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s54, -1
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s32, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s48, SCRATCH_RSRC_DWORD0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s49, SCRATCH_RSRC_DWORD1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s50, -1
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s51, 0xe8f000
+; GFX7LESS-DPP-NEXT:    s_add_u32 s48, s48, s11
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s49, s49, 0
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[36:37], s[4:5], 0x9
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s39, 0xf000
+; GFX7LESS-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s4, 44
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s5, 0
+; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX7LESS-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX7LESS-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX7LESS-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
 ; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v0, v0, v1
-; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v42, v0, v2
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX7LESS-DPP-NEXT:    v_or_b32_e32 v31, v0, v2
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[48:49]
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[50:51]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v41, v1
-; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[0:1], off, s[52:55], 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX7LESS-DPP-NEXT:    buffer_load_dwordx2 v[4:5], off, s[36:39], 0
+; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX7LESS-DPP-NEXT:  .LBB17_1: ; %atomicrmw.start
 ; GFX7LESS-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[0:1], -v[40:41]
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:12
-; GFX7LESS-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:8
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX7LESS-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX7LESS-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX7LESS-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX7LESS-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(2)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX7LESS-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v31, v42
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX7LESS-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
+; GFX7LESS-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], -v[0:1]
 ; GFX7LESS-DPP-NEXT:    s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX7LESS-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-DPP-NEXT:    v_and_b32_e32 v2, 1, v0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v0, off, s[64:67], 0
-; GFX7LESS-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0 offset:4
-; GFX7LESS-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7LESS-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v9, v5
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v8, v4
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[36:39], 0 glc
+; GFX7LESS-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX7LESS-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
+; GFX7LESS-DPP-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v4, v6
+; GFX7LESS-DPP-NEXT:    v_mov_b32_e32 v5, v7
+; GFX7LESS-DPP-NEXT:    s_andn2_b64 exec, exec, s[0:1]
 ; GFX7LESS-DPP-NEXT:    s_cbranch_execnz .LBB17_1
 ; GFX7LESS-DPP-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7LESS-DPP-NEXT:    s_endpgm
 ;
 ; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX9-DPP:       ; %bb.0:
-; GFX9-DPP-NEXT:    s_mov_b32 s68, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT:    s_mov_b32 s69, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT:    s_mov_b32 s70, -1
-; GFX9-DPP-NEXT:    s_mov_b32 s71, 0xe00000
-; GFX9-DPP-NEXT:    s_add_u32 s68, s68, s11
-; GFX9-DPP-NEXT:    s_addc_u32 s69, s69, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[36:37], s[4:5]
-; GFX9-DPP-NEXT:    s_mov_b32 s51, s8
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_mov_b32 s50, s9
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX9-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX9-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX9-DPP-NEXT:    s_mov_b32 s39, 0xe00000
+; GFX9-DPP-NEXT:    s_add_u32 s36, s36, s11
+; GFX9-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
+; GFX9-DPP-NEXT:    s_mov_b32 s12, s8
+; GFX9-DPP-NEXT:    s_add_u32 s8, s34, 44
+; GFX9-DPP-NEXT:    s_mov_b32 s13, s9
+; GFX9-DPP-NEXT:    s_addc_u32 s9, s35, 0
+; GFX9-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX9-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX9-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX9-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
 ; GFX9-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX9-DPP-NEXT:    s_mov_b64 s[34:35], s[6:7]
-; GFX9-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
-; GFX9-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX9-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX9-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v11, v9, v1, s[0:1]
-; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v6, v4, v1, s[0:1]
+; GFX9-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
-; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v13, 1
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
+; GFX9-DPP-NEXT:    v_bfrev_b32_e32 v8, 1
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
 ; GFX9-DPP-NEXT:    s_nop 0
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v13, v11 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v12, v10 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[10:11], v[10:11], v[12:13]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[5:6], v[5:6], v[7:8]
 ; GFX9-DPP-NEXT:    s_nop 1
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
 ; GFX9-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v1
 ; GFX9-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT:    v_readlane_b32 s53, v9, 63
-; GFX9-DPP-NEXT:    v_readlane_b32 s52, v8, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s3, v4, 63
+; GFX9-DPP-NEXT:    v_readlane_b32 s2, v3, 63
 ; GFX9-DPP-NEXT:    s_mov_b64 exec, s[0:1]
+; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[2:3]
 ; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX9-DPP-NEXT:  ; %bb.1:
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[54:55], s[36:37], 0x24
-; GFX9-DPP-NEXT:    s_mov_b64 s[64:65], 0
+; GFX9-DPP-NEXT:    s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[54:55]
+; GFX9-DPP-NEXT:    global_load_dwordx2 v[11:12], v0, s[2:3]
 ; GFX9-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX9-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -s[52:53]
-; GFX9-DPP-NEXT:    s_add_u32 s8, s36, 44
-; GFX9-DPP-NEXT:    s_addc_u32 s9, s37, 0
-; GFX9-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX9-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX9-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX9-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX9-DPP-NEXT:    s_mov_b64 s[0:1], s[68:69]
-; GFX9-DPP-NEXT:    buffer_store_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    buffer_store_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX9-DPP-NEXT:    buffer_store_dword v4, off, s[68:71], 0 offset:12
-; GFX9-DPP-NEXT:    buffer_store_dword v3, off, s[68:71], 0 offset:8
-; GFX9-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX9-DPP-NEXT:    s_mov_b64 s[10:11], s[34:35]
-; GFX9-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX9-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX9-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX9-DPP-NEXT:    s_mov_b64 s[2:3], s[70:71]
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v2, s54
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v3, s55
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX9-DPP-NEXT:    buffer_load_dword v1, off, s[68:71], 0
-; GFX9-DPP-NEXT:    buffer_load_dword v2, off, s[68:71], 0 offset:4
-; GFX9-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX9-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-DPP-NEXT:    s_or_b64 s[64:65], vcc, s[64:65]
-; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[64:65]
+; GFX9-DPP-NEXT:    v_add_f64 v[9:10], v[11:12], -s[0:1]
+; GFX9-DPP-NEXT:    global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-DPP-NEXT:    v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX9-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX9-DPP-NEXT:  .LBB17_3:
 ; GFX9-DPP-NEXT:    s_endpgm
 ;
 ; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1064-DPP:       ; %bb.0:
-; GFX1064-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1064-DPP-NEXT:    s_mov_b32 s67, 0x31e16000
-; GFX1064-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1064-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1064-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1064-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1064-DPP-NEXT:    s_mov_b32 s39, 0x31e16000
+; GFX1064-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1064-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1064-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1064-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1064-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1064-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1064-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1064-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1064-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1064-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1064-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1064-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1064-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1064-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    s_movk_i32 s32, 0x800
+; GFX1064-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1064-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1064-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1064-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v9, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v8, 0
-; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v9, 32
-; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v8, 32
-; GFX1064-DPP-NEXT:    v_add_f64 v[8:9], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1064-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT:    v_readlane_b32 s3, v4, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s2, v3, 0
+; GFX1064-DPP-NEXT:    v_readlane_b32 s5, v4, 32
+; GFX1064-DPP-NEXT:    v_readlane_b32 s4, v3, 32
+; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], s[2:3], s[4:5]
 ; GFX1064-DPP-NEXT:    s_mov_b64 exec, s[0:1]
 ; GFX1064-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1064-DPP-NEXT:    v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v7
 ; GFX1064-DPP-NEXT:    s_and_saveexec_b64 s[0:1], vcc
 ; GFX1064-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1064-DPP-NEXT:  ; %bb.1:
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
-; GFX1064-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1064-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1064-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1064-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1064-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1064-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1064-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1064-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1064-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1064-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1064-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1064-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1064-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1064-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1064-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1064-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1064-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1064-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1064-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1064-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1064-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-DPP-NEXT:    s_clause 0x1
-; GFX1064-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1064-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1064-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1064-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1064-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[54:55]
+; GFX1064-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1064-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1064-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1064-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1064-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1064-DPP-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX1064-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1064-DPP-NEXT:  .LBB17_3:
 ; GFX1064-DPP-NEXT:    s_endpgm
 ;
 ; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1032-DPP:       ; %bb.0:
-; GFX1032-DPP-NEXT:    s_mov_b32 s64, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT:    s_mov_b32 s65, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT:    s_mov_b32 s66, -1
-; GFX1032-DPP-NEXT:    s_mov_b32 s67, 0x31c16000
-; GFX1032-DPP-NEXT:    s_add_u32 s64, s64, s11
+; GFX1032-DPP-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GFX1032-DPP-NEXT:    s_mov_b32 s38, -1
+; GFX1032-DPP-NEXT:    s_mov_b32 s39, 0x31c16000
+; GFX1032-DPP-NEXT:    s_add_u32 s36, s36, s11
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1032-DPP-NEXT:    s_addc_u32 s65, s65, 0
-; GFX1032-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1032-DPP-NEXT:    s_addc_u32 s37, s37, 0
+; GFX1032-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1032-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1032-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1032-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1032-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[4:5], 0x0
 ; GFX1032-DPP-NEXT:    v_lshlrev_b32_e32 v1, 10, v1
-; GFX1032-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1032-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1032-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1032-DPP-NEXT:    v_or3_b32 v40, v0, v1, v2
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    s_movk_i32 s32, 0x400
+; GFX1032-DPP-NEXT:    v_or3_b32 v31, v0, v1, v2
+; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GFX1032-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1032-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1032-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
-; GFX1032-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v4, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v6, 0x80000000, v1, s0
+; GFX1032-DPP-NEXT:    v_cndmask_b32_e64 v5, 0, v0, s0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_bfrev_b32_e32 v6, 1
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v6, v4, 0, 0
+; GFX1032-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[3:4], v[5:6]
 ; GFX1032-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1032-DPP-NEXT:    s_mov_b32 s54, 0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT:    v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, v4
+; GFX1032-DPP-NEXT:    s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v7
 ; GFX1032-DPP-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX1032-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1032-DPP-NEXT:  ; %bb.1:
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[52:53], s[34:35], 0x24
+; GFX1032-DPP-NEXT:    s_load_dwordx2 s[0:1], s[34:35], 0x24
 ; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    global_load_dwordx2 v[1:2], v0, s[52:53]
+; GFX1032-DPP-NEXT:    global_load_dwordx2 v[9:10], v2, s[0:1]
 ; GFX1032-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1032-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1032-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1032-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1032-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1032-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1032-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1032-DPP-NEXT:    buffer_store_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    buffer_store_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    s_load_dwordx2 s[16:17], s[0:1], 0x0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1032-DPP-NEXT:    s_mov_b64 s[0:1], s[64:65]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1032-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1032-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1032-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1032-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1032-DPP-NEXT:    s_mov_b64 s[2:3], s[66:67]
-; GFX1032-DPP-NEXT:    buffer_store_dword v4, off, s[64:67], 0 offset:12
-; GFX1032-DPP-NEXT:    buffer_store_dword v3, off, s[64:67], 0 offset:8
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1032-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1032-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT:    s_clause 0x1
-; GFX1032-DPP-NEXT:    buffer_load_dword v1, off, s[64:67], 0
-; GFX1032-DPP-NEXT:    buffer_load_dword v2, off, s[64:67], 0 offset:4
-; GFX1032-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1032-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1032-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
+; GFX1032-DPP-NEXT:    v_add_f64 v[7:8], v[9:10], -v[0:1]
+; GFX1032-DPP-NEXT:    global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1032-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v10, v8
+; GFX1032-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1032-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1032-DPP-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX1032-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1032-DPP-NEXT:  .LBB17_3:
 ; GFX1032-DPP-NEXT:    s_endpgm
@@ -14123,238 +12448,175 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
 ; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1164-DPP:       ; %bb.0:
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1164-DPP-NEXT:    s_mov_b32 s51, s8
+; GFX1164-DPP-NEXT:    s_mov_b32 s12, s8
 ; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_mov_b32 s50, s9
+; GFX1164-DPP-NEXT:    s_mov_b32 s13, s9
 ; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
+; GFX1164-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1164-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1164-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    s_mov_b32 s33, s10
-; GFX1164-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; GFX1164-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1164-DPP-NEXT:    s_mov_b32 s14, s10
+; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1164-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1164-DPP-NEXT:    s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s[0:1]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1164-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1164-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v11, v9
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v5, v3
 ; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT:    v_permlane64_b32 v10, v8
-; GFX1164-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1164-DPP-NEXT:    v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1164-DPP-NEXT:    s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX1164-DPP-NEXT:    v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v41, v8
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v42, v9
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX1164-DPP-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX1164-DPP-NEXT:    s_waitcnt_depctr 0xfffe
-; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 0
-; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1164-DPP-NEXT:    v_mbcnt_hi_u32_b32 v6, exec_hi, v0
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, v3
+; GFX1164-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1164-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1164-DPP-NEXT:  ; %bb.1:
-; GFX1164-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
-; GFX1164-DPP-NEXT:    s_mov_b64 s[54:55], 0
+; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1164-DPP-NEXT:    .p2align 6
+; GFX1164-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1164-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1164-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1164-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1164-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1164-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1164-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1164-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1164-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v31, v40
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v0, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v2, s52
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v5, 8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1164-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1164-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1164-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1164-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1164-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1164-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v3, s53
-; GFX1164-DPP-NEXT:    v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1164-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1164-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1164-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX1164-DPP-NEXT:    s_or_b64 s[54:55], vcc, s[54:55]
-; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[54:55]
+; GFX1164-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1164-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1164-DPP-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v9, v7
+; GFX1164-DPP-NEXT:    v_mov_b32_e32 v8, v6
+; GFX1164-DPP-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
 ; GFX1164-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1164-DPP-NEXT:  .LBB17_3:
-; GFX1164-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1164-DPP-NEXT:    s_endpgm
 ;
 ; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_div_value_default_scope_strictfp:
 ; GFX1132-DPP:       ; %bb.0:
 ; GFX1132-DPP-NEXT:    s_mov_b64 s[34:35], s[4:5]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[48:49], s[0:1]
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
 ; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
 ; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, div.float.value@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, div.float.value@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v31, v0
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[36:37], s[6:7]
-; GFX1132-DPP-NEXT:    s_mov_b32 s50, s14
-; GFX1132-DPP-NEXT:    s_mov_b32 s51, s13
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; GFX1132-DPP-NEXT:    s_getpc_b64 s[4:5]
+; GFX1132-DPP-NEXT:    s_add_u32 s4, s4, div.float.value@gotpcrel32@lo+4
+; GFX1132-DPP-NEXT:    s_addc_u32 s5, s5, div.float.value@gotpcrel32@hi+12
 ; GFX1132-DPP-NEXT:    s_mov_b32 s12, s13
+; GFX1132-DPP-NEXT:    s_load_b64 s[16:17], s[4:5], 0x0
+; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[2:3]
 ; GFX1132-DPP-NEXT:    s_mov_b32 s13, s14
 ; GFX1132-DPP-NEXT:    s_mov_b32 s14, s15
-; GFX1132-DPP-NEXT:    s_mov_b32 s32, 32
-; GFX1132-DPP-NEXT:    s_mov_b32 s33, s15
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v40, v0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[38:39], s[2:3]
+; GFX1132-DPP-NEXT:    s_mov_b32 s32, 0
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[16:17]
 ; GFX1132-DPP-NEXT:    s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v9, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v8, 0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v11, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v10, 0, v0, s0
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v3, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v5, 0x80000000, v1, s0
+; GFX1132-DPP-NEXT:    v_cndmask_b32_e64 v4, 0, v0, s0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v9, v11 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v8, v10 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[10:11], v[8:9]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[4:5], v[2:3]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
-; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v11, 1
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v10, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v11, v9 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT:    v_bfrev_b32_e32 v5, 1
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_mov_b32_dpp v10, v8 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v11, v9, 0, 0
-; GFX1132-DPP-NEXT:    v_permlanex16_b32 v10, v8, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v5, v3, 0, 0
+; GFX1132-DPP-NEXT:    v_permlanex16_b32 v4, v2, 0, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT:    v_add_f64 v[8:9], v[8:9], v[10:11]
+; GFX1132-DPP-NEXT:    v_add_f64 v[2:3], v[2:3], v[4:5]
 ; GFX1132-DPP-NEXT:    s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v41, v8
-; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1132-DPP-NEXT:    v_mbcnt_lo_u32_b32 v6, exec_lo, 0
 ; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v42, v9
-; GFX1132-DPP-NEXT:    s_mov_b32 s54, 0
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
+; GFX1132-DPP-NEXT:    s_mov_b32 s2, 0
 ; GFX1132-DPP-NEXT:    s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v1
+; GFX1132-DPP-NEXT:    v_cmpx_eq_u32_e32 0, v6
 ; GFX1132-DPP-NEXT:    s_cbranch_execz .LBB17_3
 ; GFX1132-DPP-NEXT:  ; %bb.1:
-; GFX1132-DPP-NEXT:    s_load_b64 s[52:53], s[34:35], 0x24
+; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[34:35], 0x24
 ; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    global_load_b64 v[1:2], v0, s[52:53]
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x1
-; GFX1132-DPP-NEXT:    .p2align 6
+; GFX1132-DPP-NEXT:    global_load_b64 v[8:9], v10, s[0:1]
 ; GFX1132-DPP-NEXT:  .LBB17_2: ; %atomicrmw.start
 ; GFX1132-DPP-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT:    v_add_f64 v[3:4], v[1:2], -v[41:42]
-; GFX1132-DPP-NEXT:    s_add_u32 s8, s34, 44
-; GFX1132-DPP-NEXT:    s_addc_u32 s9, s35, 0
-; GFX1132-DPP-NEXT:    s_getpc_b64 s[0:1]
-; GFX1132-DPP-NEXT:    s_add_u32 s0, s0, __atomic_compare_exchange@gotpcrel32@lo+4
-; GFX1132-DPP-NEXT:    s_addc_u32 s1, s1, __atomic_compare_exchange@gotpcrel32@hi+12
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[1:2], off
-; GFX1132-DPP-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, 8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s52
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v5, 8 :: v_dual_mov_b32 v6, 0
-; GFX1132-DPP-NEXT:    v_mov_b32_e32 v7, 0
-; GFX1132-DPP-NEXT:    s_mov_b64 s[4:5], s[48:49]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[6:7], s[38:39]
-; GFX1132-DPP-NEXT:    s_mov_b64 s[10:11], s[36:37]
-; GFX1132-DPP-NEXT:    s_mov_b32 s12, s51
-; GFX1132-DPP-NEXT:    s_mov_b32 s13, s50
-; GFX1132-DPP-NEXT:    s_mov_b32 s14, s33
-; GFX1132-DPP-NEXT:    scratch_store_b64 off, v[3:4], off offset:8
-; GFX1132-DPP-NEXT:    v_dual_mov_b32 v3, s53 :: v_dual_mov_b32 v4, 0
-; GFX1132-DPP-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX1132-DPP-NEXT:    scratch_load_b64 v[1:2], off, off
-; GFX1132-DPP-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX1132-DPP-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX1132-DPP-NEXT:    s_or_b32 s54, vcc_lo, s54
-; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s54
+; GFX1132-DPP-NEXT:    v_add_f64 v[6:7], v[8:9], -v[0:1]
+; GFX1132-DPP-NEXT:    global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT:    s_waitcnt vmcnt(0)
+; GFX1132-DPP-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
+; GFX1132-DPP-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX1132-DPP-NEXT:    s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
 ; GFX1132-DPP-NEXT:    s_cbranch_execnz .LBB17_2
 ; GFX1132-DPP-NEXT:  .LBB17_3:
-; GFX1132-DPP-NEXT:    s_set_inst_prefetch_distance 0x2
 ; GFX1132-DPP-NEXT:    s_endpgm
   %divValue = call double @div.float.value() strictfp
-  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 4
+  %result = atomicrmw fsub ptr addrspace(1) %ptr, double %divValue monotonic, align 8
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
index 6a4ae7f4e0d7..f9dcd92a3e51 100644
--- a/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/icmp.i16.ll
@@ -1,18 +1,95 @@
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-FAKE16 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s| FileCheck -check-prefixes=GCN,GFX11-TRUE16 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -verify-machineinstrs < %s| FileCheck -check-prefix=GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -verify-machineinstrs < %s| FileCheck -check-prefix=GFX11-TRUE16 %s
 
 ;;;==========================================================================;;;
 ;; 16-bit integer comparisons
 ;;;==========================================================================;;;
-
-; GCN-LABEL: {{^}}i16_eq:
-; VI: v_cmp_eq_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_eq_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_eq(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_eq:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_eq:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_eq:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_eq:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -27,12 +104,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ne:
-; VI: v_cmp_ne_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ne(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ne:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ne_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ne:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ne:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ne:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -47,12 +201,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ugt:
-; VI: v_cmp_gt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ugt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ugt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ugt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ugt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ugt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -67,12 +298,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_uge:
-; VI: v_cmp_ge_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_uge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_uge:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_uge:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_uge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_uge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -87,12 +395,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ult:
-; VI: v_cmp_lt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ult(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ult:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ult:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ult:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ult:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -107,12 +492,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ule:
-; VI: v_cmp_le_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ule(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_ule:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_u16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ule:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_ushort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_u32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ule:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ule:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -128,12 +590,89 @@ entry:
 
 }
 
-; GCN-LABEL: {{^}}i16_sgt:
-; VI: v_cmp_gt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sgt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sgt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sgt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sgt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sgt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -148,12 +687,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sge:
-; VI: v_cmp_ge_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sge(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sge:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sge:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sge:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sge:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -168,12 +784,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_slt:
-; VI: v_cmp_lt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_slt(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_slt:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_slt:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_slt:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_slt:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -188,12 +881,89 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sle:
-; VI: v_cmp_le_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, v{{[0-9]+}}.{{(l|h)}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sle(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) #0 {
+; VI-LABEL: i16_sle:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v3, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v3
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    v_mov_b32_e32 v4, s5
+; VI-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v4, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    flat_load_ushort v3, v[3:4]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_i16_e32 vcc, v2, v3
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sle:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b64 s[6:7], s[10:11]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_sshort v4, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_i32_e32 vcc, v3, v4
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sle:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    global_load_u16 v2, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[4:5]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sle:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-TRUE16-NEXT:    global_load_d16_hi_b16 v0, v2, s[4:5]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -209,12 +979,78 @@ entry:
 }
 
 ; These should be commuted to reduce code size
-; GCN-LABEL: {{^}}i16_eq_v_s:
-; VI: v_cmp_eq_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_eq_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_eq_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_eq_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_eq_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_eq_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_eq_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_eq_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_eq_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -227,12 +1063,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ne_v_s:
-; VI: v_cmp_ne_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ne_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ne_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ne_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ne_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ne_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ne_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ne_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ne_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ne_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -245,12 +1147,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ugt_v_s:
-; VI: v_cmp_lt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ugt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ugt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ugt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ugt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ugt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -263,12 +1231,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_uge_v_s:
-; VI: v_cmp_le_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_uge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_uge_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_uge_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_uge_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_uge_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -281,12 +1315,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ult_v_s:
-; VI: v_cmp_gt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ult_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ult_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ult_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ult_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ult_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -299,12 +1399,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_ule_v_s:
-; VI: v_cmp_ge_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_u16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_ule_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_ule_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_u16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_ule_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_ushort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_and_b32 s4, s8, 0xffff
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_u32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_ule_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_ule_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_u16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -317,12 +1483,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sgt_v_s:
-; VI: v_cmp_lt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_lt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_lt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_lt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sgt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sgt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_lt_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sgt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sgt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sgt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_lt_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -335,12 +1567,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sge_v_s:
-; VI: v_cmp_le_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_le_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_le_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sge_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sge_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_le_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sge_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_le_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sge_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sge_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_le_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -353,12 +1651,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_slt_v_s:
-; VI: v_cmp_gt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_gt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_gt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_gt_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_slt_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_slt_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_gt_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_slt_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_slt_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_slt_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_gt_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -371,12 +1735,78 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}i16_sle_v_s:
-; VI: v_cmp_ge_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; SI: v_cmp_ge_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-FAKE16: v_cmp_ge_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}
-; GFX11-TRUE16: v_cmp_ge_i16_e32 vcc_lo, s{{[0-9]+}}, v{{[0-9]+}}.{{(l|h)}}
 define amdgpu_kernel void @i16_sle_v_s(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, i16 %b) #0 {
+; VI-LABEL: i16_sle_v_s:
+; VI:       ; %bb.0: ; %entry
+; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s4, s[4:5], 0x34
+; VI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, s3
+; VI-NEXT:    v_add_u32_e32 v1, vcc, s2, v1
+; VI-NEXT:    v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; VI-NEXT:    flat_load_ushort v2, v[1:2]
+; VI-NEXT:    v_mov_b32_e32 v1, s1
+; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_ge_i16_e32 vcc, s4, v2
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
+;
+; SI-LABEL: i16_sle_v_s:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s8, s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_sshort v3, v[1:2], s[4:7], 0 addr64
+; SI-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; SI-NEXT:    s_sext_i32_i16 s4, s8
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ge_i32_e32 vcc, s4, v3
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc
+; SI-NEXT:    buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: i16_sle_v_s:
+; GFX11-FAKE16:       ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
+; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v1, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, s4, v1
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e64 v1, 0, -1, vcc_lo
+; GFX11-FAKE16-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-TRUE16-LABEL: i16_sle_v_s:
+; GFX11-TRUE16:       ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 1, v1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_ge_i16_e32 vcc_lo, s4, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e64 v0, 0, -1, vcc_lo
+; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index b5665835eaf7..17a5f520ff41 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -6178,13 +6178,12 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
 ; NOOPT-NEXT:    v_mov_b32_e32 v11, v14
 ; NOOPT-NEXT:    v_mov_b32_e32 v12, v13
 ; NOOPT-NEXT:    buffer_store_dwordx4 v[9:12], off, s[4:7], 0 offset:32
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
+; NOOPT-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; implicit-def: $sgpr1
 ; NOOPT-NEXT:    ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
-; NOOPT-NEXT:    s_waitcnt expcnt(0)
 ; NOOPT-NEXT:    v_mov_b32_e32 v9, v4
 ; NOOPT-NEXT:    v_mov_b32_e32 v10, v3
 ; NOOPT-NEXT:    v_mov_b32_e32 v11, v2
@@ -7297,7 +7296,6 @@ define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) {
 ; NOOPT-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 glc
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7320,7 +7318,6 @@ define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) {
 ; NOOPT-NEXT:    buffer_load_dwordx4 v[0:3], off, s[0:3], 0 glc
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7541,7 +7538,6 @@ define amdgpu_kernel void @insert_adjacent_blocks(i32 %arg, float %val0) {
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
 ; NOOPT-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
@@ -7565,7 +7561,6 @@ define amdgpu_kernel void @insert_adjacent_blocks(i32 %arg, float %val0) {
 ; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ; implicit-def: $sgpr0_sgpr1_sgpr2_sgpr3
 ; NOOPT-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; NOOPT-NEXT:    s_waitcnt vmcnt(0)
 ; NOOPT-NEXT:    ;;#ASMSTART
 ; NOOPT-NEXT:    ; reg use v[0:3]
 ; NOOPT-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
new file mode 100644
index 000000000000..cf1546625b80
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -0,0 +1,313 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -start-before=greedy,2 -stop-after=virtregrewriter,2 -o - %s | FileCheck %s
+
+--- |
+
+  define amdgpu_kernel void @inflate_result_to_agpr__not_mfma() {
+    ret void
+  }
+
+  ; FIXME: Need IR for amdgpu-waves-per-eu
+  define amdgpu_kernel void @av_split_is_phi_def_crash() #0 {
+    ret void
+  }
+
+  define amdgpu_kernel void @copy_physreg_to_av_crash() #0 {
+    ret void
+  }
+
+  attributes #0 = { "amdgpu-wave-limiter"="true" "amdgpu-waves-per-eu"="8,8" }
+...
+
+# Inflate pattern, except the defining instruction isn't an MFMA.
+---
+name:            inflate_result_to_agpr__not_mfma
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__not_mfma
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr2_vgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit renamable $vgpr2_vgpr3, implicit renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %4.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def %0:vreg_512_align2, implicit %1, implicit %4
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+---
+name:            av_split_is_phi_def_crash
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  waveLimiter:     true
+  hasSpilledSGPRs: true
+  scratchRSrcReg:  '$sgpr68_sgpr69_sgpr70_sgpr71'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       8
+  wwmReservedRegs:
+    - '$vgpr31'
+  sgprForEXECCopy: '$sgpr72_sgpr73'
+body:             |
+  ; CHECK-LABEL: name: av_split_is_phi_def_crash
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $vgpr0, $sgpr2_sgpr3, $sgpr8_sgpr9
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr8 = IMPLICIT_DEF
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = V_MOV_B64_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr8, $vgpr0_vgpr1:0x0000000000000003
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = V_MOV_B64_PSEUDO 4607182418800017408, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT:   liveins: $vgpr8, $vgpr31, $sgpr8_sgpr9, $vgpr0_vgpr1:0x0000000000000003, $vgpr2_vgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr1 = COPY renamable $vgpr8
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = COPY killed renamable $vgpr2_vgpr3
+  ; CHECK-NEXT:   renamable $vgpr2 = COPY renamable $vgpr1
+  ; CHECK-NEXT:   renamable $vgpr3 = COPY renamable $vgpr8
+  ; CHECK-NEXT:   renamable $vgpr6 = COPY renamable $vgpr1
+  ; CHECK-NEXT:   dead renamable $vgpr7 = COPY renamable $vgpr8
+  ; CHECK-NEXT:   renamable $vgpr6 = COPY renamable $vgpr1
+  ; CHECK-NEXT:   renamable $vgpr7 = COPY renamable $vgpr8
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   renamable $vgpr10_vgpr11 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr12_vgpr13 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr14_vgpr15 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = V_ADD_F64_e64 0, undef $sgpr2_sgpr3, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr20_vgpr21 = V_ADD_F64_e64 0, undef $sgpr28_sgpr29, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr22_vgpr23 = V_ADD_F64_e64 0, undef $sgpr24_sgpr25, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr24_vgpr25 = V_ADD_F64_e64 0, undef $sgpr30_sgpr31, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr26_vgpr27 = V_ADD_F64_e64 0, undef $sgpr26_sgpr27, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr29 = COPY renamable $vgpr12
+  ; CHECK-NEXT:   renamable $vgpr28 = COPY renamable $vgpr12
+  ; CHECK-NEXT:   renamable $vgpr4_vgpr5 = V_MOV_B64_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr4_vgpr5 = V_ADD_F64_e64 0, killed $vgpr28_vgpr29, 0, killed $vgpr4_vgpr5, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr28_vgpr29 = COPY killed renamable $agpr0_agpr1
+  ; CHECK-NEXT:   renamable $vgpr4_vgpr5 = V_ADD_F64_e64 0, killed $vgpr4_vgpr5, 0, killed $vgpr28_vgpr29, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr11
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr14
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr20
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr24
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr26
+  ; CHECK-NEXT:   renamable $vgpr4 = COPY killed renamable $vgpr6
+  ; CHECK-NEXT:   S_NOP 0
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr16
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr22
+  ; CHECK-NEXT:   dead renamable $vgpr4 = COPY killed renamable $vgpr18
+  ; CHECK-NEXT:   renamable $vgpr2 = COPY killed renamable $vgpr1
+  ; CHECK-NEXT:   dead renamable $vgpr3 = COPY killed renamable $vgpr8
+  ; CHECK-NEXT:   dead renamable $vgpr1 = COPY renamable $vgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    liveins: $vgpr0, $sgpr8_sgpr9, $sgpr2_sgpr3, $sgpr8_sgpr9
+
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vreg_64_align2 = V_MOV_B64_PSEUDO 0, implicit $exec
+    undef %2.sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    %1:vreg_64_align2 = V_MOV_B64_PSEUDO 4607182418800017408, implicit $exec
+
+  bb.2:
+    liveins: $vgpr31, $sgpr8_sgpr9
+
+    %3:vgpr_32 = COPY %0
+    %4:vgpr_32 = COPY %0
+    undef %5.sub0:vreg_64_align2 = COPY %3
+    %5.sub1:vreg_64_align2 = COPY %4
+    undef %6.sub0:vreg_64_align2 = COPY %3
+    dead %6.sub1:vreg_64_align2 = COPY %4
+    undef %7.sub0:vreg_64_align2 = COPY %3
+    %7.sub1:vreg_64_align2 = COPY %4
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    %8:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %9:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %10:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %11:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr6_sgpr7, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %12:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr2_sgpr3, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %13:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr28_sgpr29, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %14:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr24_sgpr25, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %15:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr30_sgpr31, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %16:vreg_64_align2 = V_ADD_F64_e64 0, undef $sgpr26_sgpr27, 0, 0, 0, 0, implicit $mode, implicit $exec
+    undef %17.sub1:vreg_64_align2 = COPY %9.sub0
+    %17.sub0:vreg_64_align2 = COPY %9.sub0
+    %18:vreg_64_align2 = V_MOV_B64_PSEUDO 0, implicit $exec
+    %19:vreg_64_align2 = V_ADD_F64_e64 0, %17, 0, %18, 0, 0, implicit $mode, implicit $exec
+    %20:vreg_64_align2 = V_ADD_F64_e64 0, %19, 0, %1, 0, 0, implicit $mode, implicit $exec
+    dead %21:vgpr_32 = COPY %20.sub0
+    dead %22:vgpr_32 = COPY %8.sub1
+    dead %23:vgpr_32 = COPY %10.sub0
+    dead %24:vgpr_32 = COPY %13.sub0
+    dead %25:vgpr_32 = COPY %15.sub0
+    dead %26:vgpr_32 = COPY %16.sub0
+    %27:vgpr_32 = COPY %7.sub0
+    %28:vgpr_32 = COPY %27
+    S_NOP 0
+    dead %29:vgpr_32 = COPY %11.sub0
+    dead %30:vgpr_32 = COPY %14.sub0
+    dead %31:vgpr_32 = COPY %12.sub0
+    %32:vreg_64_align2 = COPY %5
+    undef %33.sub0:vreg_64_align2 = COPY %3
+    dead %33.sub1:vreg_64_align2 = COPY %4
+    dead undef %34.sub1:vreg_64_align2 = COPY %9.sub1
+    dead %2.sub1:vreg_64_align2 = COPY %2.sub0
+
+  bb.3:
+    S_ENDPGM 0
+
+...
+
+---
+name:            copy_physreg_to_av_crash
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: av_32, preferred-register: '$agpr0' }
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+body:             |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: copy_physreg_to_av_crash
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $agpr0 = COPY $vgpr0
+    %0:av_32 = COPY $vgpr0
+    $agpr0 = COPY %0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
new file mode 100644
index 000000000000..665d63476372
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -0,0 +1,358 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+
+# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
+# split and inflated to AV_512. The VGPR version of the instruction
+# should be replaced with the AGPR version, and the copies eliminated.
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Same, except spam some other uses of the register that we shouldn't
+# break the liveness of
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_multi_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_multi_load
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr2_agpr3 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 8, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr4_agpr5_agpr6 = GLOBAL_LOAD_DWORDX3 undef renamable $vgpr0_vgpr1, 16, 0, implicit $exec :: (load (s96), align 16, addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 28, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR undef renamable $vgpr0, renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0.sub2_sub3:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 8, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0.sub4_sub5_sub6:vreg_512_align2 = GLOBAL_LOAD_DWORDX3 undef %3:vreg_64_align2, 16, 0, implicit $exec :: (load (s96), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 28, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit %0
+    GLOBAL_STORE_DWORDX4_SADDR undef %4:vgpr_32, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Non-mac variant, src2 is other VGPR. We cannot replace this case
+# unless we also try to reassign src2 to an AGPR.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr2_vgpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %4.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %4, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# TODO: The two MFMAs are both rewritable as a set, but if we look at
+# one at a time the register class constraints of the other block the
+# transform.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
new file mode 100644
index 000000000000..33554f852510
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av.mir
@@ -0,0 +1,190 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -start-before=greedy,2 -stop-after=virtregrewriter,2 -verify-regalloc -verify-machineinstrs -o - %s | FileCheck %s
+
+# There aren't enough VGPRs for %0 to stay in a VGPR. %0 should be
+# split and inflated to AV_512. The VGPR version of the instruction
+# can't be replaced with AGPR version since we can't easily replace
+# src2's register class
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $vgpr11 = COPY renamable $vgpr10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000000F0000
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = nofpexcept V_PK_MUL_F32 8, $vgpr10_vgpr11, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub8_sub9, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Non-mac version, src2 is other VGPR
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr26 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr18_vgpr19
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   renamable $vgpr27 = COPY renamable $vgpr26
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000000F000F
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = nofpexcept V_PK_MUL_F32 8, $vgpr26_vgpr27, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr4_vgpr5 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr6_vgpr7 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr8_vgpr9 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr10_vgpr11 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr12_vgpr13 = nofpexcept V_PK_MUL_F32 8, $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr14_vgpr15 = nofpexcept V_PK_MUL_F32 8, killed $vgpr18_vgpr19, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %3.sub0_sub1:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub8_sub9, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub2_sub3:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub4_sub5:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub6_sub7:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub8_sub9:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub10_sub11:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub12_sub13:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %3.sub14_sub15:vreg_512_align2 = nofpexcept V_PK_MUL_F32 8, %0.sub0_sub1, 0, 0, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index 81fc554a8857..a3b0a7768ca6 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -1064,6 +1064,7 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_mad_u32_u24 v0, v0, 5, s2
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-NEXT:    s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 678d06e96927..546144dac647 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -5437,13 +5437,14 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 1
 ; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v7, v4, v3
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v6, v5, v2
-; GFX1200-SDAG-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v4
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v1, v7, v6
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v6, v4, v3
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[3:4], null, v4, v2, 0
 ; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
@@ -5451,16 +5452,16 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v2, v5, v2
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v4, v4, v6, v2
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v2, v3, v1
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v5, v4, v0
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[0:1], null, v3, v0, v[3:4]
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v5, v1, v2
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v2, v0, v4
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v4, v1, v3
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v3, v[0:1]
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v1, v4, v1, v2
 ; GFX1200-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5474,18 +5475,19 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v0, 1
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v4, v2
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v6, v4, v2
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v3, v[0:1]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v6, v4
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v7, v4, v2
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[0:1], null, v5, v2, v[0:1]
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v4, v2
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v5, null, v0, v5, vcc_lo
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v4, v3, v[1:2]
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v4, vcc_lo, v6, 1
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
@@ -5494,15 +5496,16 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) {
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v7, v4
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v5, v2, v[3:4]
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v5, v7, v4
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v7, v6, v[0:1]
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v5, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v1, v4, v[2:3]
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[2:3], null, v5, v3, v[0:1]
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v0, v5, v8
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[1:2], null, v1, v8, v[2:3]
 ; GFX1200-GISEL-NEXT:    s_setpc_b64 s[30:31]
 entry:
@@ -6288,18 +6291,19 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v10, vcc_lo, v2, 1
 ; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, v3, vcc_lo
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v12, v9, v4
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v13, v8, v5
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[0:1], null, v8, v4, 0
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v14, v11, v6
 ; GFX1200-SDAG-NEXT:    v_mul_lo_u32 v15, v10, v7
 ; GFX1200-SDAG-NEXT:    v_mad_co_u64_u32 v[2:3], null, v10, v6, 0
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v12, v1, v13, v12
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v1, vcc_lo, v0, v8
-; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_add3_u32 v13, v3, v15, v14
 ; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
+; GFX1200-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX1200-SDAG-NEXT:    v_add_co_ci_u32_e64 v3, null, v12, v9, vcc_lo
 ; GFX1200-SDAG-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v10
 ; GFX1200-SDAG-NEXT:    s_wait_alu 0xfffd
@@ -6348,6 +6352,7 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v9, s0, v2, 1
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v10, null, 0, v1, vcc_lo
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v0, v8, v4
 ; GFX1200-GISEL-NEXT:    v_mul_hi_u32 v1, v9, v6
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xf1ff
@@ -6368,14 +6373,16 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) {
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v16, null, v1, v10, vcc_lo
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v11, null, v2, v11, s0
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[8:9], null, v14, v5, v[0:1]
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[9:10], null, v15, v7, v[3:4]
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v10, v15, v6
 ; GFX1200-GISEL-NEXT:    v_mul_lo_u32 v7, v14, v4
-; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[3:4], null, v16, v4, v[8:9]
 ; GFX1200-GISEL-NEXT:    v_add_co_u32 v8, vcc_lo, v12, 1
+; GFX1200-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX1200-GISEL-NEXT:    v_mad_co_u64_u32 v[4:5], null, v11, v6, v[9:10]
 ; GFX1200-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX1200-GISEL-NEXT:    v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index dd2ff2e013cc..af3241e95e91 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -376,6 +376,7 @@
 ; GCN-O1-NEXT:        Live Register Matrix
 ; GCN-O1-NEXT:        Greedy Register Allocator
 ; GCN-O1-NEXT:        GCN NSA Reassign
+; GCN-O1-NEXT:        AMDGPU Rewrite AGPR-Copy-MFMA
 ; GCN-O1-NEXT:        Virtual Register Rewriter
 ; GCN-O1-NEXT:        AMDGPU Mark Last Scratch Load
 ; GCN-O1-NEXT:        Stack Slot Coloring
@@ -686,6 +687,7 @@
 ; GCN-O1-OPTS-NEXT:        Live Register Matrix
 ; GCN-O1-OPTS-NEXT:        Greedy Register Allocator
 ; GCN-O1-OPTS-NEXT:        GCN NSA Reassign
+; GCN-O1-OPTS-NEXT:        AMDGPU Rewrite AGPR-Copy-MFMA
 ; GCN-O1-OPTS-NEXT:        Virtual Register Rewriter
 ; GCN-O1-OPTS-NEXT:        AMDGPU Mark Last Scratch Load
 ; GCN-O1-OPTS-NEXT:        Stack Slot Coloring
@@ -1002,6 +1004,7 @@
 ; GCN-O2-NEXT:        Live Register Matrix
 ; GCN-O2-NEXT:        Greedy Register Allocator
 ; GCN-O2-NEXT:        GCN NSA Reassign
+; GCN-O2-NEXT:        AMDGPU Rewrite AGPR-Copy-MFMA
 ; GCN-O2-NEXT:        Virtual Register Rewriter
 ; GCN-O2-NEXT:        AMDGPU Mark Last Scratch Load
 ; GCN-O2-NEXT:        Stack Slot Coloring
@@ -1331,6 +1334,7 @@
 ; GCN-O3-NEXT:        Live Register Matrix
 ; GCN-O3-NEXT:        Greedy Register Allocator
 ; GCN-O3-NEXT:        GCN NSA Reassign
+; GCN-O3-NEXT:        AMDGPU Rewrite AGPR-Copy-MFMA
 ; GCN-O3-NEXT:        Virtual Register Rewriter
 ; GCN-O3-NEXT:        AMDGPU Mark Last Scratch Load
 ; GCN-O3-NEXT:        Stack Slot Coloring
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.lds.err.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.lds.err.ll
index 7679db8d113e..770283563001 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.lds.err.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.lds.err.ll
@@ -1,13 +1,13 @@
 ; RUN: split-file %s %t
 ;
 ; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ll 2>&1 | FileCheck --ignore-case %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ll 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ll 2>&1 | FileCheck --ignore-case %s
 ; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ptr.ll 2>&1 | FileCheck --ignore-case --check-prefix=LEGALIZER-FAIL %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ptr.ll 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/struct.ptr.ll 2>&1 | FileCheck --ignore-case %s
 ; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ll 2>&1 | FileCheck --ignore-case %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ll 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ll 2>&1 | FileCheck --ignore-case %s
 ; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ptr.ll 2>&1 | FileCheck --ignore-case --check-prefix=LEGALIZER-FAIL %s
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ptr.ll 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 %t/raw.ptr.ll 2>&1 | FileCheck --ignore-case %s
 ;
 ; CHECK: LLVM ERROR: Cannot select
 ; LEGALIZER-FAIL: Do not know how to expand this operator's operand!
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
new file mode 100644
index 000000000000..25889ded9168
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-REAL16 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
+
+declare half @llvm.amdgcn.cvt.f16.bf8(i32, i32)
+declare half @llvm.amdgcn.cvt.f16.fp8(i32, i32)
+declare <2 x half> @llvm.amdgcn.cvt.pk.f16.bf8(i16)
+declare <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16)
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte0(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e32 v0.l, v0
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte0:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 0)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte1(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:1
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte1:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:1
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 1)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte2(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:2
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte2:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:2
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 2)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte3(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-SDAG-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f32_f16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 3)
+  %ret = fpext half %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_f16_fp8_byte3_hi(i32 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.h, v0 byte_sel:3
+; GFX1250-SDAG-REAL16-NEXT:    v_mov_b16_e32 v0.l, 0
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_perm_b32 v0, v0, 0, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_f16_fp8_e64 v0.l, v0 byte_sel:3
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_lshl_or_b32 v0, v0, 16, 0
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_f16_fp8_byte3_hi:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_f16_fp8_e64 v0, v0 byte_sel:3
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_lshl_or_b32 v0, v0, 16, 0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call half @llvm.amdgcn.cvt.f16.fp8(i32 %a, i32 3)
+  %ins.0 = insertelement <2 x half> undef, half 0.0, i32 0
+  %ins.1 = insertelement <2 x half> %ins.0, half %cvt, i32 1
+  %ret = bitcast <2 x half> %ins.1 to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_pk_f16_bf8_v(i16 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_bf8_v:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_bf8_v:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_pk_f16_bf8 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_bf8_v:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_pk_f16_bf8 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_bf8_v:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_pk_f16_bf8 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call <2 x half> @llvm.amdgcn.cvt.pk.f16.bf8(i16 %a)
+  %ret = bitcast <2 x half> %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_pk_f16_bf8_s(i16 inreg %a) {
+; GFX1250-LABEL: test_cvt_pk_f16_bf8_s:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_cvt_pk_f16_bf8 v0, s0
+; GFX1250-NEXT:    ; return to shader part epilog
+  %cvt = tail call <2 x half> @llvm.amdgcn.cvt.pk.f16.bf8(i16 %a)
+  %ret = bitcast <2 x half> %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_pk_f16_fp8_v(i16 %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_fp8_v:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_fp8_v:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_fp8_v:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_fp8_v:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %cvt = tail call <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16 %a)
+  %ret = bitcast <2 x half> %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_pk_f16_fp8_s(i16 inreg %a) {
+; GFX1250-LABEL: test_cvt_pk_f16_fp8_s:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_cvt_pk_f16_fp8 v0, s0
+; GFX1250-NEXT:    ; return to shader part epilog
+  %cvt = tail call <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16 %a)
+  %ret = bitcast <2 x half> %cvt to float
+  ret float %ret
+}
+
+define amdgpu_ps float @test_cvt_pk_f16_fp8_v_hi(<2 x i16> %a) {
+; GFX1250-SDAG-REAL16-LABEL: test_cvt_pk_f16_fp8_v_hi:
+; GFX1250-SDAG-REAL16:       ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT:    v_cvt_pk_f16_fp8 v0, v0.h
+; GFX1250-SDAG-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-SDAG-FAKE16-LABEL: test_cvt_pk_f16_fp8_v_hi:
+; GFX1250-SDAG-FAKE16:       ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-SDAG-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT:    v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-SDAG-FAKE16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-REAL16-LABEL: test_cvt_pk_f16_fp8_v_hi:
+; GFX1250-GISEL-REAL16:       ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT:    v_cvt_pk_f16_fp8 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT:    ; return to shader part epilog
+;
+; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_fp8_v_hi:
+; GFX1250-GISEL-FAKE16:       ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT:    v_cvt_pk_f16_fp8 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT:    ; return to shader part epilog
+  %a.1 = extractelement <2 x i16> %a, i32 1
+  %cvt = tail call <2 x half> @llvm.amdgcn.cvt.pk.f16.fp8(i16 %a.1)
+  %ret = bitcast <2 x half> %cvt to float
+  ret float %ret
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f32.fp8.err.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f32.fp8.err.ll
index 29812993d541..d323c7fed9e4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f32.fp8.err.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f32.fp8.err.ll
@@ -11,15 +11,15 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/bf8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE1-ERR %s
 
 
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/fp8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE0-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/fp8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE1-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/bf8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE0-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/bf8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE1-ERR-GISEL %s
-
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/fp8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE0-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/fp8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE1-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/bf8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE0-ERR-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/bf8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE1-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/fp8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE0-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/fp8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE1-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/bf8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE0-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx908 -filetype=null %t/bf8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE1-ERR-GISEL %s
+
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/fp8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE0-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/fp8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-FP8-BYTE1-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/bf8-byte0-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE0-ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -filetype=null %t/bf8-byte1-err.ll 2>&1 | FileCheck -check-prefix=ERR-BF8-BYTE1-ERR-GISEL %s
 
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
new file mode 100644
index 000000000000..9e0bd5e24e7b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.async.barrier.arrive.b64.ll
@@ -0,0 +1,21 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3))
+
+; GCN-LABEL: {{^}}test_ds_atomic_async_barrier_arrive_b64:
+; GCN: ds_atomic_async_barrier_arrive_b64 v0{{$}}
+define void @test_ds_atomic_async_barrier_arrive_b64(ptr addrspace(3) %bar) {
+entry:
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_ds_atomic_async_barrier_arrive_b64_off:
+; GCN: ds_atomic_async_barrier_arrive_b64 v0 offset:8184{{$}}
+define void @test_ds_atomic_async_barrier_arrive_b64_off(ptr addrspace(3) %in) {
+entry:
+  %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
+  call void @llvm.amdgcn.ds.atomic.async.barrier.arrive.b64(ptr addrspace(3) %bar)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
new file mode 100644
index 000000000000..8c7c760aa501
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64.ll
@@ -0,0 +1,27 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3), i64)
+
+; GCN-LABEL: {{^}}test_ds_atomic_barrier_arrive_rtn_b64:
+; GCN: ds_atomic_barrier_arrive_rtn_b64 v[{{[0-9:]+}}], v2, v[0:1]{{$}}
+; GCN: s_wait_dscnt 0x0
+; GCN: flat_store_b64
+define void @test_ds_atomic_barrier_arrive_rtn_b64(i64 %data, ptr addrspace(3) %bar, ptr %out) {
+entry:
+  %ret = call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) %bar, i64 %data)
+  store i64 %ret, ptr %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_ds_atomic_barrier_arrive_rtn_b64_off:
+; GCN: ds_atomic_barrier_arrive_rtn_b64 v[{{[0-9:]+}}], v0, v[{{[0-9:]+}}] offset:8184{{$}}
+; GCN: s_wait_dscnt 0x0
+; GCN: flat_store_b64
+define void @test_ds_atomic_barrier_arrive_rtn_b64_off(ptr addrspace(3) %in, ptr %out) {
+entry:
+  %bar = getelementptr i64, ptr addrspace(3) %in, i32 1023
+  %ret = call i64 @llvm.amdgcn.ds.atomic.barrier.arrive.rtn.b64(ptr addrspace(3) %bar, i64 512)
+  store i64 %ret, ptr %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
index 818e23b5649d..16dce87f05b7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.gws.sema.release.all.ll
@@ -1,5 +1,5 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - < %s 2>&1 | FileCheck -check-prefix=GFX6ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - < %s 2>&1 | FileCheck -check-prefix=GFX6ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - < %s 2>&1 | FileCheck -check-prefix=GFX6ERR-GISEL %s
 
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,LOOP %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -o - -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,LOOP %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add-unsupported.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add-unsupported.ll
new file mode 100644
index 000000000000..20da8d651c0e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add-unsupported.ll
@@ -0,0 +1,22 @@
+; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -filetype=null 2>&1 %s | FileCheck %s
+
+; CHECK: error: <unknown>:0:0: in function ds_ordered_add_amdgpu_hs void (ptr addrspace(2), ptr addrspace(1)): ds_ordered_count unsupported for this calling conv
+define amdgpu_hs void @ds_ordered_add_amdgpu_hs(ptr addrspace(2) inreg %gds, ptr addrspace(1) %out) {
+  %val = call i32 @llvm.amdgcn.ds.ordered.add(ptr addrspace(2) %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function ds_ordered_add_amdgpu_ls void (ptr addrspace(2), ptr addrspace(1)): ds_ordered_count unsupported for this calling conv
+define amdgpu_ls void @ds_ordered_add_amdgpu_ls(ptr addrspace(2) inreg %gds, ptr addrspace(1) %out) {
+  %val = call i32 @llvm.amdgcn.ds.ordered.add(ptr addrspace(2) %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function ds_ordered_add_amdgpu_es void (ptr addrspace(2), ptr addrspace(1)): ds_ordered_count unsupported for this calling conv
+define amdgpu_es void @ds_ordered_add_amdgpu_es(ptr addrspace(2) inreg %gds, ptr addrspace(1) %out) {
+  %val = call i32 @llvm.amdgcn.ds.ordered.add(ptr addrspace(2) %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  store i32 %val, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
index 7e22d60cd710..34b7a231ff21 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dual_intersect_ray.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERR %s
 ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12-SDAG %s
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12-GISEL %s
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.err.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.err.ll
index 3abc5bd33d8a..63ec424deb7d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.err.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.err.ll
@@ -1,7 +1,7 @@
 ; RUN: not --crash llc -filetype=null -mtriple=amdgcn -mcpu=gfx810 %s 2>&1 | FileCheck --ignore-case %s
-; RUN: not --crash llc -filetype=null -global-isel -mtriple=amdgcn -mcpu=gfx810 %s 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -filetype=null -global-isel -mtriple=amdgcn -mcpu=gfx810 %s 2>&1 | FileCheck --ignore-case %s
 ; RUN: not --crash llc -filetype=null -mtriple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --ignore-case %s
-; RUN: not --crash llc -filetype=null -global-isel -mtriple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --ignore-case %s
+; RUN: not llc -filetype=null -global-isel -mtriple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --ignore-case %s
 ;
 ; CHECK: LLVM ERROR: Cannot select
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.gfx950.ll
index d7a9f4d104cb..65fdca00dae6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.load.lds.gfx950.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; ERR-SDAG: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.global.load.lds
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
index a319f1260d87..85dd27520790 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.iglp.AFLCustomIRMutator.opt.ll
@@ -5,18 +5,19 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
 ; GCN-LABEL: test_iglp_opt_rev_mfma_gemm:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    v_mov_b32_e32 v32, 0
+; GCN-NEXT:    ds_read_b128 v[0:3], v32
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
 ; GCN-NEXT:    ds_read_b128 v[28:31], v32 offset:112
 ; GCN-NEXT:    ds_read_b128 v[24:27], v32 offset:96
 ; GCN-NEXT:    ds_read_b128 v[20:23], v32 offset:80
 ; GCN-NEXT:    ds_read_b128 v[16:19], v32 offset:64
-; GCN-NEXT:    ds_read_b128 v[0:3], v32
 ; GCN-NEXT:    ds_read_b128 v[4:7], v32 offset:16
 ; GCN-NEXT:    ds_read_b128 v[8:11], v32 offset:32
 ; GCN-NEXT:    ds_read_b128 v[12:15], v32 offset:48
-; GCN-NEXT:    v_mov_b32_e32 v34, 0
-; GCN-NEXT:    v_mov_b32_e32 v35, v34
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_write_b128 v32, v[0:3]
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
 ; GCN-NEXT:    s_cmp_lg_u64 s[0:1], 0
 ; GCN-NEXT:    ; iglp_opt mask(0x00000001)
 ; GCN-NEXT:    ds_write_b128 v32, v[28:31] offset:112
@@ -26,8 +27,7 @@ define amdgpu_kernel void @test_iglp_opt_rev_mfma_gemm(<1 x i64> %L1) {
 ; GCN-NEXT:    ds_write_b128 v32, v[12:15] offset:48
 ; GCN-NEXT:    ds_write_b128 v32, v[8:11] offset:32
 ; GCN-NEXT:    ds_write_b128 v32, v[4:7] offset:16
-; GCN-NEXT:    ds_write_b128 v32, v[0:3]
-; GCN-NEXT:    ds_write_b64 v32, v[34:35]
+; GCN-NEXT:    ds_write_b64 v32, v[0:1]
 ; GCN-NEXT:    s_endpgm
 entry:
   call void @llvm.amdgcn.iglp.opt(i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
index 38c1f10f2c01..4a2c1fe2cf91 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
@@ -1,18 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs -early-live-intervals < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
-; GCN-LABEL: {{^}}load_1d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_lwe:
-; GCN: image_load v[0:4], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}}
 define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s) {
+; GCN-LABEL: load_1d_lwe:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-NEXT:    v_mov_b32_e32 v6, v0
+; GCN-NEXT:    v_mov_b32_e32 v9, v8
+; GCN-NEXT:    v_mov_b32_e32 v10, v8
+; GCN-NEXT:    v_mov_b32_e32 v11, v8
+; GCN-NEXT:    v_mov_b32_e32 v12, v8
+; GCN-NEXT:    v_mov_b32_e32 v0, v8
+; GCN-NEXT:    v_mov_b32_e32 v1, v9
+; GCN-NEXT:    v_mov_b32_e32 v2, v10
+; GCN-NEXT:    v_mov_b32_e32 v3, v11
+; GCN-NEXT:    v_mov_b32_e32 v4, v12
+; GCN-NEXT:    image_load v[0:4], v6, s[0:7] dmask:0xf unorm lwe
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v8, v4, s[8:9]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 2, i32 0)
   %v.vec = extractvalue {<4 x float>, i32} %v, 0
@@ -21,32 +41,39 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_2d:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
+; GCN-LABEL: load_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_3d:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) {
+; GCN-LABEL: load_3d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_cube:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: load_cube:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_cube_lwe:
-; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
@@ -56,24 +83,28 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_1darray:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %slice) {
+; GCN-LABEL: load_1darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32(i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darray:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: load_2darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darray_lwe:
-; GCN: image_load v[0:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
 define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %t, i32 %slice) {
 main_body:
   %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
@@ -83,184 +114,235 @@ main_body:
   ret <4 x float> %v.vec
 }
 
-; GCN-LABEL: {{^}}load_2dmsaa:
-; GCN: image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
+; GCN-LABEL: load_2dmsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_2darraymsaa:
-; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GCN-LABEL: load_2darraymsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_addr_align:
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_load v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_1d_addr_align(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+; GCN-LABEL: load_1d_addr_align:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %s1 = extractelement <2 x i32> %s, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2d:
-; GCN: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
+; GCN-LABEL: store_2d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_3d:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %r) {
+; GCN-LABEL: store_3d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.3d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_cube:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: store_cube:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.cube.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1darray:
-; GCN: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %slice) {
+; GCN-LABEL: store_1darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2darray:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice) {
+; GCN-LABEL: store_2darray:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2darray.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2dmsaa:
-; GCN: image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %fragid) {
+; GCN-LABEL: store_2dmsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2dmsaa.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_2darraymsaa:
-; GCN: image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+; GCN-LABEL: store_2darraymsaa:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm da
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.2darraymsaa.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}load_1d_V1:
-; GCN: image_load v0, v0, s[0:7] dmask:0x8 unorm{{$}}
 define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_V1:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call float @llvm.amdgcn.image.load.1d.f32.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret float %v
 }
 
-; GCN-LABEL: {{^}}load_1d_V2:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x9 unorm{{$}}
 define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_V2:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32 9, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret <2 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d_V1:
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_store v0, [[VADDR]], s[0:7] dmask:0x2 unorm{{$}}
 define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, i32 %s) {
+; GCN-LABEL: store_1d_V1:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    image_store v0, v2, s[0:7] dmask:0x2 unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.f32.i32(float %vdata, i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_V2:
-; GCN: image_store v[0:1], v2, s[0:7] dmask:0xc unorm{{$}}
 define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_V2:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:1], v2, s[0:7] dmask:0xc unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v2f32.i32(<2 x float> %vdata, i32 12, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}load_1d_glc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc{{$}}
 define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_glc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_slc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm slc{{$}}
 define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm slc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}load_1d_glc_slc:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc slc{{$}}
 define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, i32 %s) {
+; GCN-LABEL: load_1d_glc_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf unorm glc slc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 3)
   ret <4 x float> %v
 }
 
-; GCN-LABEL: {{^}}store_1d_glc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
 define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_glc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_slc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm slc{{$}}
 define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm slc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 2)
   ret void
 }
 
-; GCN-LABEL: {{^}}store_1d_glc_slc:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc slc{{$}}
 define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
+; GCN-LABEL: store_1d_glc_slc:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm glc slc
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %vdata, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 3)
   ret void
 }
 
-; GCN-LABEL: {{^}}image_store_wait:
-; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf
-; SI: s_waitcnt expcnt(0)
-; GCN: image_load v[0:3], v4, s[8:15] dmask:0xf
-; GCN: s_waitcnt vmcnt(0)
-; GCN: image_store v[0:3], v4, s[16:23] dmask:0xf
 define amdgpu_ps void @image_store_wait(<8 x i32> inreg %arg, <8 x i32> inreg %arg1, <8 x i32> inreg %arg2, <4 x float> %arg3, i32 %arg4) #0 {
+; GCN-LABEL: image_store_wait:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    image_load v[0:3], v4, s[8:15] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    image_store v[0:3], v4, s[16:23] dmask:0xf unorm
+; GCN-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %arg3, i32 15, i32 %arg4, <8 x i32> %arg, i32 0, i32 0)
   %data = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %arg4, <8 x i32> %arg1, i32 0, i32 0)
@@ -268,8 +350,6 @@ main_body:
   ret void
 }
 
-; GCN-LABEL: image_load_mmo
-; GCN: image_load v1, v[{{[0-9:]+}}], s[0:7] dmask:0x1 unorm
 define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, ptr addrspace(3) %lds, <2 x i32> %c) #0 {
   store float 0.000000e+00, ptr addrspace(3) %lds
   %c0 = extractelement <2 x i32> %c, i32 0
@@ -280,9 +360,13 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, ptr addrspace(3) %
   ret float %tex
 }
 
-; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
-; GCN: image_get_resinfo v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm
 define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+; GCN-LABEL: getresinfo_1d:
+; GCN:       ; %bb.0: ; %main_body
+; GCN-NEXT:    v_mov_b32_e32 v0, v1
+; GCN-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf unorm
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    ; return to shader part epilog
 main_body:
   %s1 = extractelement <2 x i32> %s, i32 1
   %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
index c2f8c2c44316..2964f07c285f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w32.ll
@@ -121,11 +121,12 @@ define amdgpu_cs_chain void @wwm_in_shader(<3 x i32> inreg %sgpr, ptr inreg %cal
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v10, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v0
 ; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GISEL12-NEXT:    v_dual_mov_b32 v11, v0 :: v_dual_add_nc_u32 v10, 42, v10
 ; GISEL12-NEXT:  ; %bb.2: ; %tail
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
@@ -240,11 +241,12 @@ define amdgpu_cs_chain void @phi_whole_struct(<3 x i32> inreg %sgpr, ptr inreg %
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v12, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v0
 ; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GISEL12-NEXT:    v_dual_mov_b32 v11, v0 :: v_dual_add_nc_u32 v10, 42, v12
 ; GISEL12-NEXT:  ; %bb.2: ; %tail
 ; GISEL12-NEXT:    s_or_b32 exec_lo, exec_lo, s3
@@ -361,12 +363,13 @@ define amdgpu_cs_chain void @control_flow(<3 x i32> inreg %sgpr, ptr inreg %call
 ; GISEL12-NEXT:    s_or_saveexec_b32 s8, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v1, s8
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s9, 0, v0
 ; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s9
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s8
 ; GISEL12-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v13, v1
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GISEL12-NEXT:    v_mov_b32_e32 v11, v0
 ; GISEL12-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
@@ -611,11 +614,12 @@ define amdgpu_cs_chain void @use_v0_7(<3 x i32> inreg %sgpr, ptr inreg %callee,
 ; GISEL12-NEXT:    s_or_saveexec_b32 s4, -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v13, 0x47, v12, s4
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s8, 0, v13
 ; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v13, s8
 ; GISEL12-NEXT:    s_mov_b32 exec_lo, s4
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GISEL12-NEXT:    v_dual_mov_b32 v11, v13 :: v_dual_add_nc_u32 v10, 42, v12
 ; GISEL12-NEXT:    ;;#ASMSTART
 ; GISEL12-NEXT:    ; use v0-7
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
index baa904878310..362b18f65e58 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.whole.wave-w64.ll
@@ -26,10 +26,11 @@ define amdgpu_cs_chain void @basic(<3 x i32> inreg %sgpr, ptr inreg %callee, i64
 ; GISEL12-NEXT:    s_or_saveexec_b64 s[10:11], -1
 ; GISEL12-NEXT:    s_wait_alu 0xfffe
 ; GISEL12-NEXT:    v_cndmask_b32_e64 v0, 0x47, v13, s[10:11]
-; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GISEL12-NEXT:    v_cmp_ne_u32_e64 s[12:13], 0, v0
 ; GISEL12-NEXT:    s_wait_alu 0xf1ff
 ; GISEL12-NEXT:    v_mov_b32_e32 v0, s12
+; GISEL12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GISEL12-NEXT:    v_mov_b32_e32 v1, s13
 ; GISEL12-NEXT:    s_mov_b64 exec, s[10:11]
 ; GISEL12-NEXT:    v_mov_b32_e32 v11, v0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
index d08f826e8062..8e244b522766 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.inverse.ballot.i32.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=1 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,GISEL %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=0 -verify-machineinstrs < %s | FileCheck  -check-prefixes=GFX11,SDAG %s
 
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -global-isel=1 < %s 2>&1 | FileCheck  -check-prefix=GISEL-ERR %s
+; RUN: not llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -global-isel=1 < %s 2>&1 | FileCheck  -check-prefix=GISEL-ERR %s
 ; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -global-isel=0 < %s 2>&1 | FileCheck  -check-prefix=SDAG-ERR %s
 
 ; GISEL-ERR: LLVM ERROR: cannot select: {{.*}}  = G_INTRINSIC intrinsic(@llvm.amdgcn.inverse.ballot)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.to.lds.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.to.lds.gfx950.ll
index 3a97ddf0ffd2..eb99c89e53ed 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.to.lds.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.to.lds.gfx950.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950-GISEL %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; ERR-SDAG: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.load.to.lds
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
new file mode 100644
index 000000000000..d91b03ca4461
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.tr.gfx1250.w32.ll
@@ -0,0 +1,322 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+declare <2 x i32>    @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1))
+declare <2 x i32>    @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1))
+declare <3 x i32>    @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1))
+declare <8 x i16>    @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1))
+declare <8 x half>   @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1))
+declare <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1))
+
+declare <2 x i32>    @llvm.amdgcn.ds.load.tr4.b64.v2i32.p3(ptr addrspace(3))
+declare <2 x i32>    @llvm.amdgcn.ds.load.tr8.b64.v2i32.p3(ptr addrspace(3))
+declare <3 x i32>    @llvm.amdgcn.ds.load.tr6.b96.v3i32.p3(ptr addrspace(3))
+declare <8 x i16>    @llvm.amdgcn.ds.load.tr16.b128.v8i16.p3(ptr addrspace(3))
+declare <8 x half>   @llvm.amdgcn.ds.load.tr16.b128.v8f16.p3(ptr addrspace(3))
+declare <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16.p3(ptr addrspace(3))
+
+
+define amdgpu_ps void @global_load_tr4_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr4_b64_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr4_b64 v[0:1], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr4_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr4_b64_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr4_b64 v[2:3], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr4.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr8_b64_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr8_b64_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr8_b64 v[0:1], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr8_b64_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr8_b64_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr8_b64 v[2:3], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.global.load.tr.b64.v2i32.p1(ptr addrspace(1) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr6_b96_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr6_b96_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr6_b96 v[4:6], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b96 v[2:3], v[4:6], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr6_b96_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr6_b96_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr6_b96 v[2:4], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b96 v[0:1], v[2:4], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.global.load.tr6.b96.v3i32.p1(ptr addrspace(1) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8i16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8i16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8i16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8i16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.global.load.tr.b128.v8i16.p1(ptr addrspace(1) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8f16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8f16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8f16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8f16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.global.load.tr.b128.v8f16.p1(ptr addrspace(1) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8b16_vaddr(ptr addrspace(1) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8b16_vaddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_load_tr16_b128 v[4:7], v[0:1], off offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @global_load_tr16_b128_v8bf16_saddr(ptr addrspace(1) inreg %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: global_load_tr16_b128_v8bf16_saddr:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_load_tr16_b128 v[2:5], v2, s[0:1] offset:32
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[0:1], v[2:5], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(1) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.global.load.tr.b128.v8bf16.p1(ptr addrspace(1) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr4_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr4_b64:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr4_b64 v[0:1], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr4_b64:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr4_b64 v[0:1], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b64 v[4:5], v[0:1], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.ds.load.tr4.b64.v2i32.p3(ptr addrspace(3) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr8_b64(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr8_b64:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr8_b64 v[0:1], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b64 v[2:3], v[0:1], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr8_b64:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr8_b64 v[0:1], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b64 v[4:5], v[0:1], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.ds.load.tr8.b64.v2i32.p3(ptr addrspace(3) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr6_b96(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr6_b96:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr6_b96 v[0:2], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b96 v[4:5], v[0:2], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr6_b96:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr6_b96 v[0:2], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b96 v[4:5], v[0:2], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <3 x i32> @llvm.amdgcn.ds.load.tr6.b96.v3i32.p3(ptr addrspace(3) %gep)
+  store <3 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8i16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8i16:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8i16:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x i16> @llvm.amdgcn.ds.load.tr16.b128.v8i16.p3(ptr addrspace(3) %gep)
+  store <8 x i16> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8f16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-SDAG-LABEL: ds_load_tr16_b128_v8f16:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-SDAG-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-SDAG-NEXT:    s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: ds_load_tr16_b128_v8f16:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-GISEL-NEXT:    s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16.p3(ptr addrspace(3) %gep)
+  store <8 x half> %val, ptr addrspace(1) %use
+  ret void
+}
+
+define amdgpu_ps void @ds_load_tr16_b128_v8bf16(ptr addrspace(3) %addr, ptr addrspace(1) %use) {
+; GFX1250-LABEL: ds_load_tr16_b128_v8bf16:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX1250-NEXT:    ds_load_tr16_b128 v[0:3], v0 offset:32
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b128 v[4:5], v[0:3], off
+; GFX1250-NEXT:    s_endpgm
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16.p3(ptr addrspace(3) %gep)
+  store <8 x bfloat> %val, ptr addrspace(1) %use
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
index 5d5dc01439fe..a9cffd6e1c94 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll
@@ -165,19 +165,9 @@ bb:
 }
 
 ; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_1:
-; GCN: v_mov_b32_e32 [[HIGH_BITS:v[0-9]+]], 0x3ff00000
-; GCN: v_accvgpr_write_b32 a[[A_HIGH_BITS_0:[0-9]+]], [[HIGH_BITS]]
-; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 0{{$}}
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_HIGH_BITS_0]]
-
-; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}}
 ; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3
-; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 1.0{{$}}
 ; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0]
 ; GCN:    global_store_dwordx4
 ; GCN:    global_store_dwordx4
@@ -190,19 +180,9 @@ bb:
 }
 
 ; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_neg1:
-; GCN: v_mov_b32_e32 [[HIGH_BITS:v[0-9]+]], 0xbff00000
-; GCN: v_accvgpr_write_b32 a[[A_HIGH_BITS_0:[0-9]+]], [[HIGH_BITS]]
-; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 0{{$}}
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_HIGH_BITS_0]]
-
-; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}}
 ; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3
-; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], -1.0{{$}}
 ; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0]
 ; GCN:    global_store_dwordx4
 ; GCN:    global_store_dwordx4
@@ -215,18 +195,9 @@ bb:
 }
 
 ; GCN-LABEL: {{^}}test_mfma_f64_16x16x4f64_splat_imm_int_64:
-; GCN: v_accvgpr_write_b32 a[[A_LOW_BITS_0:[0-9]+]], 64{{$}}
-; GCN: v_accvgpr_write_b32 a[[A_HIGH_BITS_0:[0-9]+]], 0
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_HIGH_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a{{[0-9]+}}, a[[A_LOW_BITS_0]]
-; GCN: v_accvgpr_mov_b32 a[[LAST_CONST_REG:[0-9]+]], a[[A_HIGH_BITS_0]]
-
-; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX90A: v_mfma_f64_16x16x4f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}}
 ; GFX90A: v_mfma_f64_16x16x4f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 blgp:3
-; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], a{{\[}}[[A_LOW_BITS_0]]:[[LAST_CONST_REG]]{{\]$}}
+; GFX942: v_mfma_f64_16x16x4_f64 [[M1:a\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], 64{{$}}
 ; GFX942: v_mfma_f64_16x16x4_f64 a[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], [[M1]] cbsz:1 abid:2 neg:[1,1,0]
 ; GCN:    global_store_dwordx4
 ; GCN:    global_store_dwordx4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index d05501fb0c8b..8506e75fe968 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -801,12 +801,13 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -833,13 +834,14 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -856,6 +858,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -976,12 +979,13 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -1008,13 +1012,14 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -1031,6 +1036,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -1167,7 +1173,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -1317,7 +1323,7 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -1470,7 +1476,7 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -1658,7 +1664,7 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -3360,12 +3366,13 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -3392,13 +3399,14 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -3415,6 +3423,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i32(ptr addrspace(1) %out, i32
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -3462,12 +3471,13 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
 ; GFX11-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX11-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX11-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX11-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX11-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX11-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX11-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-SDAG-NEXT:    s_endpgm
@@ -3494,13 +3504,14 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    s_load_b96 s[0:2], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v1, v1, s3, s2
 ; GFX12-SDAG-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-SDAG-NEXT:    s_endpgm
@@ -3517,6 +3528,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s3, v1
 ; GFX12-GISEL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v0, v0, s3, s4
 ; GFX12-GISEL-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-GISEL-NEXT:    s_endpgm
@@ -3726,7 +3738,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -3804,7 +3816,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -4029,7 +4041,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -4111,7 +4123,7 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, floa
 ; GFX12-SDAG-NEXT:    v_bfe_u32 v0, v0, 10, 10
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v1, s2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
@@ -8559,6 +8571,7 @@ define void @v_permlane16_half(ptr addrspace(1) %out, half %src0, i32 %src1, i32
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8597,6 +8610,7 @@ define void @v_permlanex16_half(ptr addrspace(1) %out, half %src0, i32 %src1, i3
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8635,6 +8649,7 @@ define void @v_permlane16_bfloat(ptr addrspace(1) %out, bfloat %src0, i32 %src1,
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8673,6 +8688,7 @@ define void @v_permlanex16_bfloat(ptr addrspace(1) %out, bfloat %src0, i32 %src1
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8711,6 +8727,7 @@ define void @v_permlane16_i16(ptr addrspace(1) %out, i16 %src0, i32 %src1, i32 %
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8749,6 +8766,7 @@ define void @v_permlanex16_i16(ptr addrspace(1) %out, i16 %src0, i32 %src1, i32
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b16 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8787,6 +8805,7 @@ define void @v_permlane16_v2f16(ptr addrspace(1) %out, <2 x half> %src0, i32 %sr
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8825,6 +8844,7 @@ define void @v_permlanex16_v2f16(ptr addrspace(1) %out, <2 x half> %src0, i32 %s
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -8886,6 +8906,7 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -8901,6 +8922,7 @@ define void @v_permlane16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %s
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -8963,6 +8985,7 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -8978,6 +9001,7 @@ define void @v_permlanex16_v2f32(ptr addrspace(1) %out, <2 x float> %src0, i32 %
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -9066,6 +9090,7 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v10
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
@@ -9088,6 +9113,7 @@ define void @v_permlane16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v10
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9183,6 +9209,7 @@ define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v10
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
@@ -9205,6 +9232,7 @@ define void @v_permlanex16_v7i32(ptr addrspace(1) %out, <7 x i32> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v9
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v10
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -9282,6 +9310,7 @@ define void @v_permlane16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -9299,6 +9328,7 @@ define void @v_permlane16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9371,6 +9401,7 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -9388,6 +9419,7 @@ define void @v_permlanex16_v8i16(ptr addrspace(1) %out, <8 x i16> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -9460,6 +9492,7 @@ define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
@@ -9477,6 +9510,7 @@ define void @v_permlane16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9563,6 +9597,7 @@ define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
@@ -9584,6 +9619,7 @@ define void @v_permlane16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %src
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9682,6 +9718,7 @@ define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v9, v9, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
@@ -9705,6 +9742,7 @@ define void @v_permlane16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32 %
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9845,6 +9883,7 @@ define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v17, v17, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v16, v16, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v15, v15, s0, s1
@@ -9878,6 +9917,7 @@ define void @v_permlane16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32 %
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlane16_b32 v4, v4, s0, s1
@@ -9966,6 +10006,7 @@ define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
@@ -9983,6 +10024,7 @@ define void @v_permlanex16_v2i64(ptr addrspace(1) %out, <2 x i64> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v6
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v7
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -10069,6 +10111,7 @@ define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
@@ -10090,6 +10133,7 @@ define void @v_permlanex16_v3i64(ptr addrspace(1) %out, <3 x i64> %src0, i32 %sr
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -10188,6 +10232,7 @@ define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v10
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v11
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v9, v9, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v8, v8, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
@@ -10211,6 +10256,7 @@ define void @v_permlanex16_v4f64(ptr addrspace(1) %out, <4 x double> %src0, i32
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v10
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v11
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
@@ -10351,6 +10397,7 @@ define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v18
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v19
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v17, v17, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v16, v16, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v15, v15, s0, s1
@@ -10384,6 +10431,7 @@ define void @v_permlanex16_v8f64(ptr addrspace(1) %out, <8 x double> %src0, i32
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s0, v18
 ; GFX12-GISEL-NEXT:    v_readfirstlane_b32 s1, v19
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-GISEL-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
index 8eab7e2fc62f..10c000095fe3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ptr.ll
@@ -35,6 +35,7 @@ define void @v_permlane16_p0(ptr addrspace(1) %out, ptr %src0, i32 %src1, i32 %s
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -76,6 +77,7 @@ define void @v_permlanex16_p0(ptr addrspace(1) %out, ptr %src0, i32 %src1, i32 %
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b64 v[0:1], v[2:3], off
@@ -128,6 +130,7 @@ define void @v_permlane16_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0, i32 %src1
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v6, v6, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v5, v5, s0, s1
@@ -186,6 +189,7 @@ define void @v_permlanex16_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0, i32 %src
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v8
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v9
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v7, v7, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v6, v6, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v5, v5, s0, s1
@@ -231,6 +235,7 @@ define void @v_permlane16_p3(ptr addrspace(1) %out, ptr addrspace(3) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -269,6 +274,7 @@ define void @v_permlanex16_p3(ptr addrspace(1) %out, ptr addrspace(3) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -311,6 +317,7 @@ define void @v_permlane16_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3)> %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
@@ -355,6 +362,7 @@ define void @v_permlanex16_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3)> %s
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
@@ -395,6 +403,7 @@ define void @v_permlane16_p5(ptr addrspace(1) %out, ptr addrspace(5) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -433,6 +442,7 @@ define void @v_permlanex16_p5(ptr addrspace(1) %out, ptr addrspace(5) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -475,6 +485,7 @@ define void @v_permlane16_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5)> %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
@@ -519,6 +530,7 @@ define void @v_permlanex16_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5)> %s
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
@@ -559,6 +571,7 @@ define void @v_permlane16_p6(ptr addrspace(1) %out, ptr addrspace(6) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -597,6 +610,7 @@ define void @v_permlanex16_p6(ptr addrspace(1) %out, ptr addrspace(6) %src0, i32
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v3
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
 ; GFX12-SDAG-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -639,6 +653,7 @@ define void @v_permlane16_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6)> %sr
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlane16_b32 v2, v2, s0, s1
@@ -683,6 +698,7 @@ define void @v_permlanex16_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6)> %s
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s0, v5
 ; GFX12-SDAG-NEXT:    v_readfirstlane_b32 s1, v6
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v4, v4, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v3, v3, s0, s1
 ; GFX12-SDAG-NEXT:    v_permlanex16_b32 v2, v2, s0, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll
index cba53175087e..814086685880 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane16.swap.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GCN %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; ERR-SDAG: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.permlane16.swap
 ; ERR-GISEL: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32), %{{[0-9]+}}:vgpr_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.permlane16.swap)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane32.swap.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane32.swap.ll
index 963c73d61535..1ad90013b68d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane32.swap.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane32.swap.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 < %s | FileCheck -check-prefix=GCN %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; ERR-SDAG: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.permlane32.swap
 ; ERR-GISEL: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32), %{{[0-9]+}}:vgpr_32(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.permlane32.swap)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
index a0f03d020b98..0d04a4a9c478 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ptr.buffer.atomic.fadd_rtn_errors.ll
@@ -11,10 +11,10 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-SDAG %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-SDAG %s
 
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-f32-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-F32-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-f32-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-F32-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2F16-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2F16-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-f32-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-F32-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-f32-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-F32-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2F16-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2f16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2F16-GISEL %s
 ; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
 ; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -filetype=null %t/struct-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-STRUCT-V2BF16-GISEL %s
 ; RUN: not --crash llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -filetype=null %t/raw-ret-v2bf16-error.ll 2>&1 | FileCheck -check-prefix=ERR-RAW-V2BF16-GISEL %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 6c032ed06154..8b6ba1a3cc09 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -80,6 +80,7 @@ define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v6
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v6
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.lds.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.lds.gfx950.ll
index 476a9e9c8cd6..c485ffca4b12 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.lds.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.lds.gfx950.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-SDAG %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; FIXME: Not a great error
 ; ERR-SDAG: LLVM ERROR: Do not know how to expand this operator's operand!
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll
new file mode 100644
index 000000000000..706f470a6285
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.monitor.sleep.ll
@@ -0,0 +1,20 @@
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s
+
+declare void @llvm.amdgcn.s.monitor.sleep(i16)
+
+; GCN-LABEL: {{^}}test_monitor_sleep_1:
+; GCN: s_monitor_sleep 1
+define amdgpu_ps void @test_monitor_sleep_1() {
+  call void @llvm.amdgcn.s.monitor.sleep(i16 1)
+  ret void
+}
+
+; FIXME: 0x8000 would look better
+
+; GCN-LABEL: {{^}}test_monitor_sleep_forever:
+; GCN: s_monitor_sleep 0xffff8000
+define amdgpu_ps void @test_monitor_sleep_forever() {
+  call void @llvm.amdgcn.s.monitor.sleep(i16 32768)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
index 701b80d59bcc..74d5274f22e0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.v3f16.ll
@@ -152,6 +152,7 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-TRUE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-TRUE16-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -180,6 +181,7 @@ define amdgpu_gs void @main(<4 x i32> %arg, i32 %arg1) {
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s6, v2
 ; GFX12-FAKE16-NEXT:    v_readfirstlane_b32 s7, v3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]
 ; GFX12-FAKE16-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]
 ; GFX12-FAKE16-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
index 2c9f9a6ca4d5..148a5ba75d98 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -49,6 +49,7 @@ define <2 x bfloat> @struct_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsr
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -88,6 +89,7 @@ define void @struct_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__vgpr_rsrc__vgp
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index a86ad8ede2f2..746b8791c39f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -273,6 +273,7 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -410,6 +411,7 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index a3bdcbe17cc7..71c63bfd6973 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -218,6 +218,7 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
@@ -326,6 +327,7 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX1200-NEXT:    s_wait_alu 0xf1ff
 ; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index db6e0ad67074..e3889ab8f5a2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -456,6 +456,7 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
 ; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
@@ -610,6 +611,7 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index eef6bb7b0788..f001bf97fcd9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -456,6 +456,7 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
 ; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
 ; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
@@ -610,6 +611,7 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
 ; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
 ; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.lds.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.lds.gfx950.ll
index c1b8df089807..1516e2781f7c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.lds.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.lds.gfx950.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950 %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 -filetype=null < %s 2>&1 | FileCheck -check-prefix=ERR-GISEL %s
 
 ; ERR-SDAG: LLVM ERROR: Do not know how to expand this operator's operand!
 ; ERR-GISEL: LLVM ERROR: cannot select: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.ptr.buffer.load.lds),
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
new file mode 100644
index 000000000000..d2712ac8e08a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.tensor.load.store.ll
@@ -0,0 +1,245 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
+
+declare void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 %cpol)
+declare void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 %cpol)
+declare void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 %cpol)
+declare void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 %cpol)
+
+define amdgpu_ps void @tensor_load_to_lds(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
+; GFX1250-LABEL: tensor_load_to_lds:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @tensor_load_to_lds_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
+; GFX1250-SDAG-LABEL: tensor_load_to_lds_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_load_to_lds_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_load_to_lds s[8:11], s[0:7], s[12:15], s[16:19]
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 0)
+  ret void
+}
+
+
+define amdgpu_ps void @tensor_load_to_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
+; GFX1250-LABEL: tensor_load_to_lds_d2:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 27)
+  ret void
+}
+
+define amdgpu_ps void @tensor_load_to_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
+; GFX1250-SDAG-LABEL: tensor_load_to_lds_d2_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_load_to_lds_d2_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_load_to_lds s[8:11], s[0:7] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.load.to.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 27)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds(<4 x i32> inreg %D0, <8 x i32> inreg %D1, <4 x i32> inreg %D2, <4 x i32> inreg %D3) {
+; GFX1250-LABEL: tensor_store_from_lds:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 22)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_vector(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3) {
+; GFX1250-SDAG-LABEL: tensor_store_from_lds_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_store_from_lds_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s14, v14
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s15, v15
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s16, v16
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s17, v17
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s18, v18
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s19, v19
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_store_from_lds s[8:11], s[0:7], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds(<4 x i32> %D0, <8 x i32> %D1, <4 x i32> %D2, <4 x i32> %D3, i32 22)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_d2(<4 x i32> inreg %D0, <8 x i32> inreg %D1) {
+; GFX1250-LABEL: tensor_store_from_lds_d2:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    tensor_store_from_lds s[0:3], s[4:11]
+; GFX1250-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @tensor_store_from_lds_d2_vector(<4 x i32> %D0, <8 x i32> %D1) {
+; GFX1250-SDAG-LABEL: tensor_store_from_lds_d2_vector:
+; GFX1250-SDAG:       ; %bb.0: ; %entry
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-SDAG-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    tensor_store_from_lds s[8:11], s[0:7]
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: tensor_store_from_lds_d2_vector:
+; GFX1250-GISEL:       ; %bb.0: ; %entry
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s9, v1
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s10, v2
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s11, v3
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s2, v6
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-GISEL-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    tensor_store_from_lds s[8:11], s[0:7]
+; GFX1250-GISEL-NEXT:    s_endpgm
+entry:
+  call void @llvm.amdgcn.tensor.store.from.lds.d2(<4 x i32> %D0, <8 x i32> %D1, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
index d44d2a4f7338..b95cf865dc73 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: not --crash llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=0 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GFX9-SDAG-ERR %s
-; RUN: not --crash llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=1 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GFX9-GISEL-ERR %s
+; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=1 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GFX9-GISEL-ERR %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+architected-sgprs -global-isel=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+architected-sgprs -global-isel=1 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -global-isel=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index 2da07e51781e..e6cc8f9f1f3c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -2077,6 +2077,7 @@ define void @test_writelane_half(ptr addrspace(1) %out, half %src, i32 %src1) {
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2112,6 +2113,7 @@ define void @test_writelane_half(ptr addrspace(1) %out, half %src, i32 %src1) {
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-GISEL-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -2153,6 +2155,7 @@ define void @test_writelane_float(ptr addrspace(1) %out, float %src, i32 %src1)
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2188,6 +2191,7 @@ define void @test_writelane_float(ptr addrspace(1) %out, float %src, i32 %src1)
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-GISEL-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -2229,6 +2233,7 @@ define void @test_writelane_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2264,6 +2269,7 @@ define void @test_writelane_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-GISEL-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -2305,6 +2311,7 @@ define void @test_writelane_i16(ptr addrspace(1) %out, i16 %src, i32 %src1) {
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2340,6 +2347,7 @@ define void @test_writelane_i16(ptr addrspace(1) %out, i16 %src, i32 %src1) {
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-GISEL-NEXT:    global_store_b16 v[0:1], v4, off
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -2381,6 +2389,7 @@ define void @test_writelane_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %s
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -2416,6 +2425,7 @@ define void @test_writelane_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %s
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-GISEL-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -2462,6 +2472,7 @@ define void @test_readlane_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %s
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v6, s0, s1
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v5, s2, s1
 ; GFX1100-SDAG-NEXT:    global_store_b64 v[0:1], v[5:6], off
@@ -2503,6 +2514,7 @@ define void @test_readlane_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %s
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX1100-GISEL-NEXT:    v_readfirstlane_b32 s2, v3
 ; GFX1100-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v5, s0, s1
 ; GFX1100-GISEL-NEXT:    v_writelane_b32 v6, s2, s1
 ; GFX1100-GISEL-NEXT:    global_store_b64 v[0:1], v[5:6], off
@@ -2586,6 +2598,7 @@ define void @test_writelane_v7i32(ptr addrspace(1) %out, <7 x i32> %src, i32 %sr
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v7
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v6
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v16, s0, s1
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v13, s4, s1
@@ -2981,6 +2994,7 @@ define void @test_writelane_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %sr
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v7
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v14, s0, s1
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v12, s3, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ptr.ll
index 11cdc625d9ad..40e124382df9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ptr.ll
@@ -40,6 +40,7 @@ define void @test_writelane_p0(ptr addrspace(1) %out, ptr %src, i32 %src1) {
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v4
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v6, s0, s1
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v5, s2, s1
 ; GFX1100-SDAG-NEXT:    global_store_b64 v[0:1], v[5:6], off
@@ -118,6 +119,7 @@ define void @test_writelane_v3p0(ptr addrspace(1) %out, <3 x ptr> %src, i32 %src
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v7
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v6
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(1)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v14, s0, s1
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v12, s3, s1
@@ -167,6 +169,7 @@ define void @test_writelane_p3(ptr addrspace(1) %out, ptr addrspace(3) %src, i32
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -218,8 +221,10 @@ define void @test_writelane_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3)> %
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v3
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v2
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v8, s0, s1
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v7, s2, s1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v6, s3, s1
 ; GFX1100-SDAG-NEXT:    global_store_b96 v[0:1], v[6:8], off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -261,6 +266,7 @@ define void @test_writelane_p5(ptr addrspace(1) %out, ptr addrspace(5) %src, i32
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -312,8 +318,10 @@ define void @test_writelane_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5)> %
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v3
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v2
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v8, s0, s1
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v7, s2, s1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v6, s3, s1
 ; GFX1100-SDAG-NEXT:    global_store_b96 v[0:1], v[6:8], off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -355,6 +363,7 @@ define void @test_writelane_p6(ptr addrspace(1) %out, ptr addrspace(6) %src, i32
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s0, v2
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v4, s0, s1
 ; GFX1100-SDAG-NEXT:    global_store_b32 v[0:1], v4, off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
@@ -406,8 +415,10 @@ define void @test_writelane_v3p6(ptr addrspace(1) %out, <3 x ptr addrspace(6)> %
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s2, v3
 ; GFX1100-SDAG-NEXT:    v_readfirstlane_b32 s3, v2
 ; GFX1100-SDAG-NEXT:    s_waitcnt vmcnt(0)
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v8, s0, s1
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v7, s2, s1
+; GFX1100-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX1100-SDAG-NEXT:    v_writelane_b32 v6, s3, s1
 ; GFX1100-SDAG-NEXT:    global_store_b96 v[0:1], v[6:8], off
 ; GFX1100-SDAG-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.err.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.err.ll
index 21fe1ce4dc1d..d2caadff14b8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.err.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.err.ll
@@ -1,17 +1,17 @@
 ; RUN: split-file %s %t
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f64-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F64-FAIL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f64-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F64-FAIL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f64-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F64-FAIL %s
 
 ; TODO: check for GISEL when bfloat is supported.
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/bf16-f32-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=BF16-F32-FAIL %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/bf16-f64-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=BF16-F64-FAIL %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f32-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F32-TONEARESTAWAY-FAIL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f32-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F32-TONEARESTAWAY-FAIL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f16-f32-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F16-F32-TONEARESTAWAY-FAIL %s
 
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f32-f64-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F32-F64-TONEARESTAWAY-FAIL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f32-f64-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F32-F64-TONEARESTAWAY-FAIL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -filetype=null %t/f32-f64-tonearestaway-err.ll 2>&1 | FileCheck --ignore-case --check-prefix=F32-F64-TONEARESTAWAY-FAIL %s
 
 ;--- f16-f64-err.ll
 define amdgpu_gs void @test_fptrunc_round_f16_f64(double %a, ptr addrspace(1) %out) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
index c8f33f1464d4..66cf8a309ccf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
@@ -110,15 +110,17 @@ define { i64, i1 } @umulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v1, v5
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v6, null, 0, v6, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_add3_u32 v1, v1, v5, v7
 ; GFX12-NEXT:    v_add_co_u32 v4, vcc_lo, v4, v7
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v6, v8, vcc_lo
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
@@ -286,20 +288,22 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v5, v2, 0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[10:11], null, v5, v3, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v1, v6
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v7, null, 0, v7, vcc_lo
 ; GFX12-NEXT:    v_add3_u32 v1, v1, v6, v8
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v10, null, 0, v9, vcc_lo
@@ -311,12 +315,13 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX12-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
 ; GFX12-NEXT:    v_subrev_co_ci_u32_e64 v7, null, 0, v5, vcc_lo
 ; GFX12-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v3
 ; GFX12-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_cndmask_b32 v4, v6, v4 :: v_dual_cndmask_b32 v5, v5, v7
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[2:3]
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index ae4acfe35d08..a42c71c4849b 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -8314,8 +8314,8 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
 ; GFX12-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
@@ -9176,8 +9176,8 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    v_readfirstlane_b32 s2, v1
 ; GFX12-NEXT:    s_wait_alu 0xf1ff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 28504da5a683..8351d2805756 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -7666,6 +7666,7 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
@@ -8128,6 +8129,7 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 48714b7282b1..0c4aca88b378 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -7666,6 +7666,7 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
@@ -8128,6 +8129,7 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 6879a7cfd09c..37310b614c0d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -8438,6 +8438,7 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
@@ -8900,6 +8901,7 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
 ; GFX12-FAKE16-NEXT:    v_add3_u32 v6, v6, v4, 0x7fff
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v4, v4
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v5, v7, v9, vcc_lo
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v4, v6, v8, s0
diff --git a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
index 7f256af6763a..b80c259c255d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
@@ -149,7 +149,6 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) {
 ; MUBUF-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_mov_b32 s33, s5
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: func_local_stack_offset_uses_sp:
@@ -189,7 +188,6 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) {
 ; FLATSCR-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_mov_b32 s33, s2
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %pin.low = alloca i32, align 8192, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
index f5558964d270..23412aaeb2e2 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
@@ -3,7 +3,10 @@
 # RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler -amdgpu-disable-unclustered-high-rp-reschedule -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX90A %s
 
 --- |
-  define void @small_num_vgprs_as_spill() "amdgpu-num-vgpr"="28" {
+  define void @small_num_sgprs_as_spill() "amdgpu-num-sgpr"="85" {
+    ret void
+  }
+  define void @small_num_vgprs_as_spill() "amdgpu-num-vgpr"="14" {
     ret void
   }
   define void @dont_remat_waves_per_eu() "amdgpu-flat-work-group-size"="1024,1024" "amdgpu-waves-per-eu"="7,7" {
@@ -15,13 +18,333 @@
   define void @reduce_arch_and_acc_vgrp_spill() "amdgpu-waves-per-eu"="8,8" {
     ret void
   }
-  define void @reduce_spill_archvgpr_above_addressable_limit() "amdgpu-waves-per-eu"="1,10" {
+  define void @reduce_spill_archvgpr_above_addressable_limit() "amdgpu-flat-work-group-size"="1,64" "amdgpu-waves-per-eu"="1,2" {
     ret void
   }
-  define void @reduce_spill_agpr_above_addressable_limit() "amdgpu-waves-per-eu"="1,10" {
+  define void @reduce_spill_agpr_above_addressable_limit() "amdgpu-flat-work-group-size"="1,64" "amdgpu-waves-per-eu"="1,2" {
     ret void
   }
 ---
+# User-requested maximum number of SGPRs need to be taken into account by
+# the scheduler's rematerialization stage. Register usage above that number
+# is considered like spill.
+name:            small_num_sgprs_as_spill
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; GFX908-LABEL: name: small_num_sgprs_as_spill
+  ; GFX908: bb.0:
+  ; GFX908-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
+  ; GFX908-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
+  ; GFX908-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
+  ; GFX908-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
+  ; GFX908-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
+  ; GFX908-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
+  ; GFX908-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
+  ; GFX908-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
+  ; GFX908-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
+  ; GFX908-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
+  ; GFX908-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
+  ; GFX908-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
+  ; GFX908-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
+  ; GFX908-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
+  ; GFX908-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
+  ; GFX908-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
+  ; GFX908-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
+  ; GFX908-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
+  ; GFX908-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
+  ; GFX908-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
+  ; GFX908-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
+  ; GFX908-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
+  ; GFX908-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
+  ; GFX908-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
+  ; GFX908-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
+  ; GFX908-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
+  ; GFX908-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
+  ; GFX908-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
+  ; GFX908-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
+  ; GFX908-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
+  ; GFX908-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
+  ; GFX908-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
+  ; GFX908-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
+  ; GFX908-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
+  ; GFX908-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
+  ; GFX908-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
+  ; GFX908-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
+  ; GFX908-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
+  ; GFX908-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
+  ; GFX908-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
+  ; GFX908-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
+  ; GFX908-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
+  ; GFX908-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
+  ; GFX908-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
+  ; GFX908-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
+  ; GFX908-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
+  ; GFX908-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
+  ; GFX908-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
+  ; GFX908-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
+  ; GFX908-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
+  ; GFX908-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
+  ; GFX908-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
+  ; GFX908-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
+  ; GFX908-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
+  ; GFX908-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
+  ; GFX908-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
+  ; GFX908-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
+  ; GFX908-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
+  ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
+  ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
+  ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
+  ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
+  ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
+  ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
+  ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
+  ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
+  ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
+  ; GFX908-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
+  ; GFX908-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
+  ; GFX908-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
+  ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
+  ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
+  ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
+  ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
+  ; GFX908-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
+  ; GFX908-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
+  ; GFX908-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
+  ; GFX908-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
+  ; GFX908-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
+  ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT: bb.1:
+  ; GFX908-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_79]], implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_4]], implicit [[S_MOV_B32_5]], implicit [[S_MOV_B32_6]], implicit [[S_MOV_B32_7]], implicit [[S_MOV_B32_8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_9]], implicit [[S_MOV_B32_10]], implicit [[S_MOV_B32_11]], implicit [[S_MOV_B32_12]], implicit [[S_MOV_B32_13]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_14]], implicit [[S_MOV_B32_15]], implicit [[S_MOV_B32_16]], implicit [[S_MOV_B32_17]], implicit [[S_MOV_B32_18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_19]], implicit [[S_MOV_B32_20]], implicit [[S_MOV_B32_21]], implicit [[S_MOV_B32_22]], implicit [[S_MOV_B32_23]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_24]], implicit [[S_MOV_B32_25]], implicit [[S_MOV_B32_26]], implicit [[S_MOV_B32_27]], implicit [[S_MOV_B32_28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_29]], implicit [[S_MOV_B32_30]], implicit [[S_MOV_B32_31]], implicit [[S_MOV_B32_32]], implicit [[S_MOV_B32_33]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_34]], implicit [[S_MOV_B32_35]], implicit [[S_MOV_B32_36]], implicit [[S_MOV_B32_37]], implicit [[S_MOV_B32_38]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_39]], implicit [[S_MOV_B32_40]], implicit [[S_MOV_B32_41]], implicit [[S_MOV_B32_42]], implicit [[S_MOV_B32_43]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_44]], implicit [[S_MOV_B32_45]], implicit [[S_MOV_B32_46]], implicit [[S_MOV_B32_47]], implicit [[S_MOV_B32_48]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_49]], implicit [[S_MOV_B32_50]], implicit [[S_MOV_B32_51]], implicit [[S_MOV_B32_52]], implicit [[S_MOV_B32_53]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_54]], implicit [[S_MOV_B32_55]], implicit [[S_MOV_B32_56]], implicit [[S_MOV_B32_57]], implicit [[S_MOV_B32_58]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_59]], implicit [[S_MOV_B32_60]], implicit [[S_MOV_B32_61]], implicit [[S_MOV_B32_62]], implicit [[S_MOV_B32_63]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_64]], implicit [[S_MOV_B32_65]], implicit [[S_MOV_B32_66]], implicit [[S_MOV_B32_67]], implicit [[S_MOV_B32_68]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_69]], implicit [[S_MOV_B32_70]], implicit [[S_MOV_B32_71]], implicit [[S_MOV_B32_72]], implicit [[S_MOV_B32_73]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_74]], implicit [[S_MOV_B32_75]], implicit [[S_MOV_B32_76]], implicit [[S_MOV_B32_77]], implicit [[S_MOV_B32_78]]
+  ; GFX908-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX90A-LABEL: name: small_num_sgprs_as_spill
+  ; GFX90A: bb.0:
+  ; GFX90A-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
+  ; GFX90A-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
+  ; GFX90A-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
+  ; GFX90A-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
+  ; GFX90A-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
+  ; GFX90A-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
+  ; GFX90A-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
+  ; GFX90A-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
+  ; GFX90A-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
+  ; GFX90A-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
+  ; GFX90A-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
+  ; GFX90A-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
+  ; GFX90A-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
+  ; GFX90A-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
+  ; GFX90A-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
+  ; GFX90A-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
+  ; GFX90A-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
+  ; GFX90A-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
+  ; GFX90A-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
+  ; GFX90A-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
+  ; GFX90A-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
+  ; GFX90A-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
+  ; GFX90A-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
+  ; GFX90A-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
+  ; GFX90A-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
+  ; GFX90A-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
+  ; GFX90A-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
+  ; GFX90A-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
+  ; GFX90A-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
+  ; GFX90A-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
+  ; GFX90A-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
+  ; GFX90A-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
+  ; GFX90A-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
+  ; GFX90A-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
+  ; GFX90A-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
+  ; GFX90A-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
+  ; GFX90A-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
+  ; GFX90A-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
+  ; GFX90A-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
+  ; GFX90A-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
+  ; GFX90A-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
+  ; GFX90A-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
+  ; GFX90A-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
+  ; GFX90A-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
+  ; GFX90A-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
+  ; GFX90A-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
+  ; GFX90A-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
+  ; GFX90A-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
+  ; GFX90A-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
+  ; GFX90A-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
+  ; GFX90A-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
+  ; GFX90A-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
+  ; GFX90A-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
+  ; GFX90A-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
+  ; GFX90A-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
+  ; GFX90A-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
+  ; GFX90A-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
+  ; GFX90A-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
+  ; GFX90A-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
+  ; GFX90A-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
+  ; GFX90A-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
+  ; GFX90A-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
+  ; GFX90A-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
+  ; GFX90A-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
+  ; GFX90A-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
+  ; GFX90A-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
+  ; GFX90A-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
+  ; GFX90A-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
+  ; GFX90A-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
+  ; GFX90A-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
+  ; GFX90A-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
+  ; GFX90A-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
+  ; GFX90A-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
+  ; GFX90A-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
+  ; GFX90A-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
+  ; GFX90A-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
+  ; GFX90A-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
+  ; GFX90A-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
+  ; GFX90A-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
+  ; GFX90A-NEXT: {{  $}}
+  ; GFX90A-NEXT: bb.1:
+  ; GFX90A-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_79]], implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_4]], implicit [[S_MOV_B32_5]], implicit [[S_MOV_B32_6]], implicit [[S_MOV_B32_7]], implicit [[S_MOV_B32_8]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_9]], implicit [[S_MOV_B32_10]], implicit [[S_MOV_B32_11]], implicit [[S_MOV_B32_12]], implicit [[S_MOV_B32_13]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_14]], implicit [[S_MOV_B32_15]], implicit [[S_MOV_B32_16]], implicit [[S_MOV_B32_17]], implicit [[S_MOV_B32_18]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_19]], implicit [[S_MOV_B32_20]], implicit [[S_MOV_B32_21]], implicit [[S_MOV_B32_22]], implicit [[S_MOV_B32_23]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_24]], implicit [[S_MOV_B32_25]], implicit [[S_MOV_B32_26]], implicit [[S_MOV_B32_27]], implicit [[S_MOV_B32_28]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_29]], implicit [[S_MOV_B32_30]], implicit [[S_MOV_B32_31]], implicit [[S_MOV_B32_32]], implicit [[S_MOV_B32_33]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_34]], implicit [[S_MOV_B32_35]], implicit [[S_MOV_B32_36]], implicit [[S_MOV_B32_37]], implicit [[S_MOV_B32_38]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_39]], implicit [[S_MOV_B32_40]], implicit [[S_MOV_B32_41]], implicit [[S_MOV_B32_42]], implicit [[S_MOV_B32_43]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_44]], implicit [[S_MOV_B32_45]], implicit [[S_MOV_B32_46]], implicit [[S_MOV_B32_47]], implicit [[S_MOV_B32_48]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_49]], implicit [[S_MOV_B32_50]], implicit [[S_MOV_B32_51]], implicit [[S_MOV_B32_52]], implicit [[S_MOV_B32_53]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_54]], implicit [[S_MOV_B32_55]], implicit [[S_MOV_B32_56]], implicit [[S_MOV_B32_57]], implicit [[S_MOV_B32_58]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_59]], implicit [[S_MOV_B32_60]], implicit [[S_MOV_B32_61]], implicit [[S_MOV_B32_62]], implicit [[S_MOV_B32_63]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_64]], implicit [[S_MOV_B32_65]], implicit [[S_MOV_B32_66]], implicit [[S_MOV_B32_67]], implicit [[S_MOV_B32_68]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_69]], implicit [[S_MOV_B32_70]], implicit [[S_MOV_B32_71]], implicit [[S_MOV_B32_72]], implicit [[S_MOV_B32_73]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[S_MOV_B32_74]], implicit [[S_MOV_B32_75]], implicit [[S_MOV_B32_76]], implicit [[S_MOV_B32_77]], implicit [[S_MOV_B32_78]]
+  ; GFX90A-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1
+
+    %0:sgpr_32 = S_MOV_B32 0
+    %1:sgpr_32 = S_MOV_B32 1
+    %2:sgpr_32 = S_MOV_B32 2
+    %3:sgpr_32 = S_MOV_B32 3
+    %4:sgpr_32 = S_MOV_B32 4
+    %5:sgpr_32 = S_MOV_B32 5
+    %6:sgpr_32 = S_MOV_B32 6
+    %7:sgpr_32 = S_MOV_B32 7
+    %8:sgpr_32 = S_MOV_B32 8
+    %9:sgpr_32 = S_MOV_B32 9
+    %10:sgpr_32 = S_MOV_B32 10
+    %11:sgpr_32 = S_MOV_B32 11
+    %12:sgpr_32 = S_MOV_B32 12
+    %13:sgpr_32 = S_MOV_B32 13
+    %14:sgpr_32 = S_MOV_B32 14
+    %15:sgpr_32 = S_MOV_B32 15
+    %16:sgpr_32 = S_MOV_B32 16
+    %17:sgpr_32 = S_MOV_B32 17
+    %18:sgpr_32 = S_MOV_B32 18
+    %19:sgpr_32 = S_MOV_B32 19
+    %20:sgpr_32 = S_MOV_B32 20
+    %21:sgpr_32 = S_MOV_B32 21
+    %22:sgpr_32 = S_MOV_B32 22
+    %23:sgpr_32 = S_MOV_B32 23
+    %24:sgpr_32 = S_MOV_B32 24
+    %25:sgpr_32 = S_MOV_B32 25
+    %26:sgpr_32 = S_MOV_B32 26
+    %27:sgpr_32 = S_MOV_B32 27
+    %28:sgpr_32 = S_MOV_B32 28
+    %29:sgpr_32 = S_MOV_B32 29
+    %30:sgpr_32 = S_MOV_B32 30
+    %31:sgpr_32 = S_MOV_B32 31
+    %32:sgpr_32 = S_MOV_B32 32
+    %33:sgpr_32 = S_MOV_B32 33
+    %34:sgpr_32 = S_MOV_B32 34
+    %35:sgpr_32 = S_MOV_B32 35
+    %36:sgpr_32 = S_MOV_B32 36
+    %37:sgpr_32 = S_MOV_B32 37
+    %38:sgpr_32 = S_MOV_B32 38
+    %39:sgpr_32 = S_MOV_B32 39
+    %40:sgpr_32 = S_MOV_B32 40
+    %41:sgpr_32 = S_MOV_B32 41
+    %42:sgpr_32 = S_MOV_B32 42
+    %43:sgpr_32 = S_MOV_B32 43
+    %44:sgpr_32 = S_MOV_B32 44
+    %45:sgpr_32 = S_MOV_B32 45
+    %46:sgpr_32 = S_MOV_B32 46
+    %47:sgpr_32 = S_MOV_B32 47
+    %48:sgpr_32 = S_MOV_B32 48
+    %49:sgpr_32 = S_MOV_B32 49
+    %50:sgpr_32 = S_MOV_B32 50
+    %51:sgpr_32 = S_MOV_B32 51
+    %52:sgpr_32 = S_MOV_B32 52
+    %53:sgpr_32 = S_MOV_B32 53
+    %54:sgpr_32 = S_MOV_B32 54
+    %55:sgpr_32 = S_MOV_B32 55
+    %56:sgpr_32 = S_MOV_B32 56
+    %57:sgpr_32 = S_MOV_B32 57
+    %58:sgpr_32 = S_MOV_B32 58
+    %59:sgpr_32 = S_MOV_B32 59
+    %60:sgpr_32 = S_MOV_B32 60
+    %61:sgpr_32 = S_MOV_B32 61
+    %62:sgpr_32 = S_MOV_B32 62
+    %63:sgpr_32 = S_MOV_B32 63
+    %64:sgpr_32 = S_MOV_B32 64
+    %65:sgpr_32 = S_MOV_B32 65
+    %66:sgpr_32 = S_MOV_B32 66
+    %67:sgpr_32 = S_MOV_B32 67
+    %68:sgpr_32 = S_MOV_B32 68
+    %69:sgpr_32 = S_MOV_B32 69
+    %70:sgpr_32 = S_MOV_B32 70
+    %71:sgpr_32 = S_MOV_B32 71
+    %72:sgpr_32 = S_MOV_B32 72
+    %73:sgpr_32 = S_MOV_B32 73
+    %74:sgpr_32 = S_MOV_B32 74
+    %75:sgpr_32 = S_MOV_B32 75
+    %76:sgpr_32 = S_MOV_B32 76
+    %77:sgpr_32 = S_MOV_B32 77
+    %78:sgpr_32 = S_MOV_B32 78
+    %79:sgpr_32 = S_MOV_B32 79
+
+  bb.1:
+
+    S_NOP 0, implicit %0,   implicit %1,   implicit %2,   implicit %3,   implicit %4
+    S_NOP 0, implicit %5,   implicit %6,   implicit %7,   implicit %8,   implicit %9
+    S_NOP 0, implicit %10,  implicit %11,  implicit %12,  implicit %13,  implicit %14
+    S_NOP 0, implicit %15,  implicit %16,  implicit %17,  implicit %18,  implicit %19
+    S_NOP 0, implicit %20,  implicit %21,  implicit %22,  implicit %23,  implicit %24
+    S_NOP 0, implicit %25,  implicit %26,  implicit %27,  implicit %28,  implicit %29
+    S_NOP 0, implicit %30,  implicit %31,  implicit %32,  implicit %33,  implicit %34
+    S_NOP 0, implicit %35,  implicit %36,  implicit %37,  implicit %38,  implicit %39
+    S_NOP 0, implicit %40,  implicit %41,  implicit %42,  implicit %43,  implicit %44
+    S_NOP 0, implicit %45,  implicit %46,  implicit %47,  implicit %48,  implicit %49
+    S_NOP 0, implicit %50,  implicit %51,  implicit %52,  implicit %53,  implicit %54
+    S_NOP 0, implicit %55,  implicit %56,  implicit %57,  implicit %58,  implicit %59
+    S_NOP 0, implicit %60,  implicit %61,  implicit %62,  implicit %63,  implicit %64
+    S_NOP 0, implicit %65,  implicit %66,  implicit %67,  implicit %68,  implicit %69
+    S_NOP 0, implicit %70,  implicit %71,  implicit %72,  implicit %73,  implicit %74
+    S_NOP 0, implicit %75,  implicit %76,  implicit %77,  implicit %78,  implicit %79
+
+    S_ENDPGM 0
+...
 # User-requested maximum number of VGPRs need to be taken into account by
 # the scheduler's rematerialization stage. Register usage above that number
 # is considered like spill; occupancy is "inadvertently" increased when
@@ -68,14 +391,14 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_33:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_33]], implicit [[V_CVT_I32_F64_e32_27]]
   ; GFX908-NEXT:   S_ENDPGM 0
@@ -111,13 +434,7 @@ body:             |
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_33:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 33, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 33, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
@@ -125,8 +442,14 @@ body:             |
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_27]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_30]], implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_33]]
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]]
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_33:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_33]], implicit [[V_CVT_I32_F64_e32_27]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1
@@ -492,10 +815,9 @@ body:             |
     S_ENDPGM 0
 ...
 # Min/Max waves/EU is 8. For targets with non-unified RF (gfx908) we are able to
-# eliminate both ArchVGPR and AGPR spilling by saving 2 ArchVGPRs (one for
-# spilling AGPR to ArchVGPR). In the unified RF case (gfx90a) the ArchVGPR
-# allocation granule forces us to remat all eligible ArchVGPRs to eliminate
-# spilling.
+# eliminate both ArchVGPR and AGPR spilling by saving 2 VGPRs. In the unified RF
+# case (gfx90a) the ArchVGPR allocation granule forces us to remat more
+# ArchVGPRs to eliminate spilling.
 ---
 name:            reduce_arch_and_acc_vgrp_spill
 tracksRegLiveness: true
@@ -538,8 +860,6 @@ body:             |
   ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX908-NEXT:   [[DEF32:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -566,10 +886,12 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
@@ -577,17 +899,17 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_27]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[V_CVT_I32_F64_e32_30]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF32]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_27]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]]
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF32:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_30]], implicit [[V_CVT_I32_F64_e32_31]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[V_CVT_I32_F64_e32_32]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF30]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: reduce_arch_and_acc_vgrp_spill
@@ -653,8 +975,11 @@ body:             |
   ; GFX90A-NEXT:   [[DEF29:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF30:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF31:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   [[DEF32:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
@@ -662,20 +987,17 @@ body:             |
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[V_CVT_I32_F64_e32_27]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF32]]
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_27]], implicit [[V_CVT_I32_F64_e32_28]]
+  ; GFX90A-NEXT:   [[DEF32:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]], implicit [[DEF32]], implicit [[DEF]], implicit [[DEF1]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[V_CVT_I32_F64_e32_31]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF31]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1
@@ -767,6 +1089,9 @@ body:             |
 
     S_ENDPGM 0
 ...
+# Requested [min,max] occupancy is [1,2]. There are 257 ArchVGPRs in use when
+# only 256 are available. Nothing to do since it should be possible to use an
+# AGPR for one of the ArchVGPRs.
 ---
 name:            reduce_spill_archvgpr_above_addressable_limit
 tracksRegLiveness: true
@@ -1033,6 +1358,8 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_252:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 252, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_253:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 253, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_254:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 254, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]], implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
@@ -1060,8 +1387,6 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_220]], implicit [[V_CVT_I32_F64_e32_221]], implicit [[V_CVT_I32_F64_e32_222]], implicit [[V_CVT_I32_F64_e32_223]], implicit [[V_CVT_I32_F64_e32_224]], implicit [[V_CVT_I32_F64_e32_225]], implicit [[V_CVT_I32_F64_e32_226]], implicit [[V_CVT_I32_F64_e32_227]], implicit [[V_CVT_I32_F64_e32_228]], implicit [[V_CVT_I32_F64_e32_229]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_230]], implicit [[V_CVT_I32_F64_e32_231]], implicit [[V_CVT_I32_F64_e32_232]], implicit [[V_CVT_I32_F64_e32_233]], implicit [[V_CVT_I32_F64_e32_234]], implicit [[V_CVT_I32_F64_e32_235]], implicit [[V_CVT_I32_F64_e32_236]], implicit [[V_CVT_I32_F64_e32_237]], implicit [[V_CVT_I32_F64_e32_238]], implicit [[V_CVT_I32_F64_e32_239]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_240]], implicit [[V_CVT_I32_F64_e32_241]], implicit [[V_CVT_I32_F64_e32_242]], implicit [[V_CVT_I32_F64_e32_243]], implicit [[V_CVT_I32_F64_e32_244]], implicit [[V_CVT_I32_F64_e32_245]], implicit [[V_CVT_I32_F64_e32_246]], implicit [[V_CVT_I32_F64_e32_247]], implicit [[V_CVT_I32_F64_e32_248]], implicit [[V_CVT_I32_F64_e32_249]]
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[DEF]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
@@ -1070,7 +1395,6 @@ body:             |
   ; GFX90A-NEXT:   successors: %bb.1(0x80000000)
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -1325,6 +1649,9 @@ body:             |
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_252:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 252, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_253:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 253, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_254:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 254, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]], implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
@@ -1352,9 +1679,7 @@ body:             |
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_220]], implicit [[V_CVT_I32_F64_e32_221]], implicit [[V_CVT_I32_F64_e32_222]], implicit [[V_CVT_I32_F64_e32_223]], implicit [[V_CVT_I32_F64_e32_224]], implicit [[V_CVT_I32_F64_e32_225]], implicit [[V_CVT_I32_F64_e32_226]], implicit [[V_CVT_I32_F64_e32_227]], implicit [[V_CVT_I32_F64_e32_228]], implicit [[V_CVT_I32_F64_e32_229]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_230]], implicit [[V_CVT_I32_F64_e32_231]], implicit [[V_CVT_I32_F64_e32_232]], implicit [[V_CVT_I32_F64_e32_233]], implicit [[V_CVT_I32_F64_e32_234]], implicit [[V_CVT_I32_F64_e32_235]], implicit [[V_CVT_I32_F64_e32_236]], implicit [[V_CVT_I32_F64_e32_237]], implicit [[V_CVT_I32_F64_e32_238]], implicit [[V_CVT_I32_F64_e32_239]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_240]], implicit [[V_CVT_I32_F64_e32_241]], implicit [[V_CVT_I32_F64_e32_242]], implicit [[V_CVT_I32_F64_e32_243]], implicit [[V_CVT_I32_F64_e32_244]], implicit [[V_CVT_I32_F64_e32_245]], implicit [[V_CVT_I32_F64_e32_246]], implicit [[V_CVT_I32_F64_e32_247]], implicit [[V_CVT_I32_F64_e32_248]], implicit [[V_CVT_I32_F64_e32_249]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[DEF]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[DEF]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1
@@ -1650,6 +1975,9 @@ body:             |
 
     S_ENDPGM 0
 ...
+# Requested [min,max] occupancy is [1,2]. There are 257 AGPRs in use when only
+# 256 are available. Nothing to do since it should be possible to use an
+# ArchVGPR for one of the AGPRs.
 ---
 name:            reduce_spill_agpr_above_addressable_limit
 tracksRegLiveness: true
@@ -1660,6 +1988,8 @@ body:             |
   ; GFX908: bb.0:
   ; GFX908-NEXT:   successors: %bb.1(0x80000000)
   ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF1:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF2:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
@@ -1919,34 +2249,32 @@ body:             |
   ; GFX908-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]], implicit [[DEF129]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]], implicit [[DEF139]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]], implicit [[DEF149]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]], implicit [[DEF159]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]], implicit [[DEF169]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]], implicit [[DEF179]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]], implicit [[DEF189]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]], implicit [[DEF199]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]], implicit [[DEF209]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]], implicit [[DEF219]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]], implicit [[DEF229]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]], implicit [[DEF239]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]], implicit [[DEF249]]
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF128]], implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF138]], implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF148]], implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF158]], implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF168]], implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF178]], implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF188]], implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF198]], implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF208]], implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF218]], implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF228]], implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF238]], implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF248]], implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[DEF]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]], implicit [[DEF40]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]], implicit [[DEF50]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]], implicit [[DEF60]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]], implicit [[DEF70]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]], implicit [[DEF80]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]], implicit [[DEF90]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]], implicit [[DEF100]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]], implicit [[DEF110]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]], implicit [[DEF120]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: reduce_spill_agpr_above_addressable_limit
@@ -2205,6 +2533,8 @@ body:             |
   ; GFX90A-NEXT:   [[DEF249:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF250:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF251:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
   ; GFX90A-NEXT:   [[DEF252:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF253:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF254:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
@@ -2212,34 +2542,32 @@ body:             |
   ; GFX90A-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]], implicit [[DEF129]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]], implicit [[DEF139]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]], implicit [[DEF149]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]], implicit [[DEF159]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]], implicit [[DEF169]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]], implicit [[DEF179]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]], implicit [[DEF189]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]], implicit [[DEF199]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]], implicit [[DEF209]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]], implicit [[DEF219]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]], implicit [[DEF229]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]], implicit [[DEF239]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]], implicit [[DEF249]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]], implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]], implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]], implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]], implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]], implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]], implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]], implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]], implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]], implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]], implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]], implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]], implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]], implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]], implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]], implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]], implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]], implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]], implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]], implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]], implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]], implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]], implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
   ; GFX90A-NEXT:   S_ENDPGM 0
 
   bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index 5bdb7255d262..f69337e67ba8 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -1602,12 +1602,12 @@ body:             |
     S_ENDPGM 0
 ...
 ---
-name:            test_occ_9_no_sink_limited_by_sgprs
+name:            test_occ_9_no_sink_vgprs_and_sgprs
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
 body:             |
-  ; GFX908-LABEL: name: test_occ_9_no_sink_limited_by_sgprs
+  ; GFX908-LABEL: name: test_occ_9_no_sink_vgprs_and_sgprs
   ; GFX908: bb.0:
   ; GFX908-NEXT:   successors: %bb.1(0x80000000)
   ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
@@ -1638,91 +1638,82 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 17, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 18, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 19, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
-  ; GFX908-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
-  ; GFX908-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
-  ; GFX908-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
-  ; GFX908-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
-  ; GFX908-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
-  ; GFX908-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
-  ; GFX908-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
-  ; GFX908-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
-  ; GFX908-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
-  ; GFX908-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
-  ; GFX908-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
-  ; GFX908-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
-  ; GFX908-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
-  ; GFX908-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
-  ; GFX908-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
-  ; GFX908-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
-  ; GFX908-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
-  ; GFX908-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
-  ; GFX908-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
-  ; GFX908-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
-  ; GFX908-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
-  ; GFX908-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
-  ; GFX908-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
-  ; GFX908-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
-  ; GFX908-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
-  ; GFX908-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
-  ; GFX908-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
-  ; GFX908-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
-  ; GFX908-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
-  ; GFX908-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
-  ; GFX908-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
-  ; GFX908-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
-  ; GFX908-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
-  ; GFX908-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
-  ; GFX908-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
-  ; GFX908-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
-  ; GFX908-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
-  ; GFX908-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
-  ; GFX908-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
-  ; GFX908-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
-  ; GFX908-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
-  ; GFX908-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
-  ; GFX908-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
-  ; GFX908-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
-  ; GFX908-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
-  ; GFX908-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
-  ; GFX908-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
-  ; GFX908-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
-  ; GFX908-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
-  ; GFX908-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
-  ; GFX908-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
-  ; GFX908-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
-  ; GFX908-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
-  ; GFX908-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
-  ; GFX908-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
-  ; GFX908-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
-  ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
-  ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
-  ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
-  ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
-  ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
-  ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
-  ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
-  ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
-  ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
-  ; GFX908-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
-  ; GFX908-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
-  ; GFX908-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
-  ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
-  ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
-  ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
+  ; GFX908-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
+  ; GFX908-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
+  ; GFX908-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
+  ; GFX908-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
+  ; GFX908-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
+  ; GFX908-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
+  ; GFX908-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
+  ; GFX908-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
+  ; GFX908-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
+  ; GFX908-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
+  ; GFX908-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
+  ; GFX908-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
+  ; GFX908-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
+  ; GFX908-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
+  ; GFX908-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
+  ; GFX908-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
+  ; GFX908-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
+  ; GFX908-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
+  ; GFX908-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
+  ; GFX908-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
+  ; GFX908-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
+  ; GFX908-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
+  ; GFX908-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
+  ; GFX908-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
+  ; GFX908-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
+  ; GFX908-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
+  ; GFX908-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
+  ; GFX908-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
+  ; GFX908-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
+  ; GFX908-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
+  ; GFX908-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
+  ; GFX908-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
+  ; GFX908-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
+  ; GFX908-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
+  ; GFX908-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
+  ; GFX908-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
+  ; GFX908-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
+  ; GFX908-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
+  ; GFX908-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
+  ; GFX908-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
+  ; GFX908-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
+  ; GFX908-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
+  ; GFX908-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
+  ; GFX908-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
+  ; GFX908-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
+  ; GFX908-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
+  ; GFX908-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
+  ; GFX908-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
+  ; GFX908-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
+  ; GFX908-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
+  ; GFX908-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
+  ; GFX908-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
+  ; GFX908-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
+  ; GFX908-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
+  ; GFX908-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
+  ; GFX908-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
+  ; GFX908-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
+  ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
+  ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
+  ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
+  ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
+  ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
+  ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
+  ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
+  ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
+  ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
+  ; GFX908-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
+  ; GFX908-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
+  ; GFX908-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
+  ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
+  ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
+  ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 80
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
-  ; GFX908-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
-  ; GFX908-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
-  ; GFX908-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
-  ; GFX908-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
-  ; GFX908-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
-  ; GFX908-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
-  ; GFX908-NEXT:   [[S_MOV_B32_80:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
-  ; GFX908-NEXT:   [[S_MOV_B32_81:%[0-9]+]]:sgpr_32 = S_MOV_B32 80
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
@@ -1742,6 +1733,7 @@ body:             |
   ; GFX908-NEXT: bb.3:
   ; GFX908-NEXT:   successors: %bb.5(0x04000000), %bb.4(0x7c000000)
   ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_23]]
   ; GFX908-NEXT:   $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
   ; GFX908-NEXT:   undef [[S_MOV_B32_:%[0-9]+]].sub0:sreg_64 = S_ADD_I32 [[S_MOV_B32_]].sub0, -1, implicit-def dead $scc
@@ -1766,6 +1758,18 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_9]], implicit [[V_CVT_I32_F64_e32_19]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_22]]
+  ; GFX908-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX908-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_74]], implicit [[S_MOV_B32_75]]
+  ; GFX908-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
+  ; GFX908-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_76]], implicit [[S_MOV_B32_77]]
+  ; GFX908-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
+  ; GFX908-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_78]], implicit [[S_MOV_B32_79]]
+  ; GFX908-NEXT:   [[S_MOV_B32_80:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
+  ; GFX908-NEXT:   [[S_MOV_B32_81:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_80]], implicit [[S_MOV_B32_81]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_5]], implicit [[S_MOV_B32_6]]
@@ -1802,11 +1806,7 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_67]], implicit [[S_MOV_B32_68]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_69]], implicit [[S_MOV_B32_70]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_71]], implicit [[S_MOV_B32_72]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_73]], implicit [[S_MOV_B32_74]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_75]], implicit [[S_MOV_B32_76]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_77]], implicit [[S_MOV_B32_78]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_79]], implicit [[S_MOV_B32_80]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_81]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_73]]
   ; GFX908-NEXT:   S_ENDPGM 0
   bb.0:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -2008,12 +2008,12 @@ body:             |
     S_ENDPGM 0
 ...
 ---
-name:            test_occ_8_no_sink_limited_by_sgprs
+name:            test_occ_8_sink_vgprs_and_sgprs
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
 body:             |
-  ; GFX908-LABEL: name: test_occ_8_no_sink_limited_by_sgprs
+  ; GFX908-LABEL: name: test_occ_8_sink_vgprs_and_sgprs
   ; GFX908: bb.0:
   ; GFX908-NEXT:   successors: %bb.1(0x80000000)
   ; GFX908-NEXT:   liveins: $vgpr0, $sgpr0_sgpr1
@@ -2044,99 +2044,105 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 17, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 18, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 19, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
-  ; GFX908-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
-  ; GFX908-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
-  ; GFX908-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
-  ; GFX908-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
-  ; GFX908-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
-  ; GFX908-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
-  ; GFX908-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
-  ; GFX908-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
-  ; GFX908-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
-  ; GFX908-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
-  ; GFX908-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
-  ; GFX908-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
-  ; GFX908-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
-  ; GFX908-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
-  ; GFX908-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
-  ; GFX908-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
-  ; GFX908-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
-  ; GFX908-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
-  ; GFX908-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
-  ; GFX908-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
-  ; GFX908-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
-  ; GFX908-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
-  ; GFX908-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
-  ; GFX908-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
-  ; GFX908-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
-  ; GFX908-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
-  ; GFX908-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
-  ; GFX908-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
-  ; GFX908-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
-  ; GFX908-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
-  ; GFX908-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
-  ; GFX908-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
-  ; GFX908-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
-  ; GFX908-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
-  ; GFX908-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
-  ; GFX908-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
-  ; GFX908-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
-  ; GFX908-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
-  ; GFX908-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
-  ; GFX908-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
-  ; GFX908-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
-  ; GFX908-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
-  ; GFX908-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
-  ; GFX908-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
-  ; GFX908-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
-  ; GFX908-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
-  ; GFX908-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
-  ; GFX908-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
-  ; GFX908-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
-  ; GFX908-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
-  ; GFX908-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
-  ; GFX908-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
-  ; GFX908-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
-  ; GFX908-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
-  ; GFX908-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
-  ; GFX908-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
-  ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
-  ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
-  ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
-  ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
-  ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
-  ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
-  ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
-  ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
-  ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
-  ; GFX908-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
-  ; GFX908-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
-  ; GFX908-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
-  ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
-  ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
-  ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
+  ; GFX908-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 12
+  ; GFX908-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 13
+  ; GFX908-NEXT:   [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 14
+  ; GFX908-NEXT:   [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 15
+  ; GFX908-NEXT:   [[S_MOV_B32_5:%[0-9]+]]:sgpr_32 = S_MOV_B32 16
+  ; GFX908-NEXT:   [[S_MOV_B32_6:%[0-9]+]]:sgpr_32 = S_MOV_B32 17
+  ; GFX908-NEXT:   [[S_MOV_B32_7:%[0-9]+]]:sgpr_32 = S_MOV_B32 18
+  ; GFX908-NEXT:   [[S_MOV_B32_8:%[0-9]+]]:sgpr_32 = S_MOV_B32 19
+  ; GFX908-NEXT:   [[S_MOV_B32_9:%[0-9]+]]:sgpr_32 = S_MOV_B32 20
+  ; GFX908-NEXT:   [[S_MOV_B32_10:%[0-9]+]]:sgpr_32 = S_MOV_B32 21
+  ; GFX908-NEXT:   [[S_MOV_B32_11:%[0-9]+]]:sgpr_32 = S_MOV_B32 22
+  ; GFX908-NEXT:   [[S_MOV_B32_12:%[0-9]+]]:sgpr_32 = S_MOV_B32 23
+  ; GFX908-NEXT:   [[S_MOV_B32_13:%[0-9]+]]:sgpr_32 = S_MOV_B32 24
+  ; GFX908-NEXT:   [[S_MOV_B32_14:%[0-9]+]]:sgpr_32 = S_MOV_B32 25
+  ; GFX908-NEXT:   [[S_MOV_B32_15:%[0-9]+]]:sgpr_32 = S_MOV_B32 26
+  ; GFX908-NEXT:   [[S_MOV_B32_16:%[0-9]+]]:sgpr_32 = S_MOV_B32 27
+  ; GFX908-NEXT:   [[S_MOV_B32_17:%[0-9]+]]:sgpr_32 = S_MOV_B32 28
+  ; GFX908-NEXT:   [[S_MOV_B32_18:%[0-9]+]]:sgpr_32 = S_MOV_B32 29
+  ; GFX908-NEXT:   [[S_MOV_B32_19:%[0-9]+]]:sgpr_32 = S_MOV_B32 30
+  ; GFX908-NEXT:   [[S_MOV_B32_20:%[0-9]+]]:sgpr_32 = S_MOV_B32 31
+  ; GFX908-NEXT:   [[S_MOV_B32_21:%[0-9]+]]:sgpr_32 = S_MOV_B32 32
+  ; GFX908-NEXT:   [[S_MOV_B32_22:%[0-9]+]]:sgpr_32 = S_MOV_B32 33
+  ; GFX908-NEXT:   [[S_MOV_B32_23:%[0-9]+]]:sgpr_32 = S_MOV_B32 34
+  ; GFX908-NEXT:   [[S_MOV_B32_24:%[0-9]+]]:sgpr_32 = S_MOV_B32 35
+  ; GFX908-NEXT:   [[S_MOV_B32_25:%[0-9]+]]:sgpr_32 = S_MOV_B32 36
+  ; GFX908-NEXT:   [[S_MOV_B32_26:%[0-9]+]]:sgpr_32 = S_MOV_B32 37
+  ; GFX908-NEXT:   [[S_MOV_B32_27:%[0-9]+]]:sgpr_32 = S_MOV_B32 38
+  ; GFX908-NEXT:   [[S_MOV_B32_28:%[0-9]+]]:sgpr_32 = S_MOV_B32 39
+  ; GFX908-NEXT:   [[S_MOV_B32_29:%[0-9]+]]:sgpr_32 = S_MOV_B32 40
+  ; GFX908-NEXT:   [[S_MOV_B32_30:%[0-9]+]]:sgpr_32 = S_MOV_B32 41
+  ; GFX908-NEXT:   [[S_MOV_B32_31:%[0-9]+]]:sgpr_32 = S_MOV_B32 42
+  ; GFX908-NEXT:   [[S_MOV_B32_32:%[0-9]+]]:sgpr_32 = S_MOV_B32 43
+  ; GFX908-NEXT:   [[S_MOV_B32_33:%[0-9]+]]:sgpr_32 = S_MOV_B32 44
+  ; GFX908-NEXT:   [[S_MOV_B32_34:%[0-9]+]]:sgpr_32 = S_MOV_B32 45
+  ; GFX908-NEXT:   [[S_MOV_B32_35:%[0-9]+]]:sgpr_32 = S_MOV_B32 46
+  ; GFX908-NEXT:   [[S_MOV_B32_36:%[0-9]+]]:sgpr_32 = S_MOV_B32 47
+  ; GFX908-NEXT:   [[S_MOV_B32_37:%[0-9]+]]:sgpr_32 = S_MOV_B32 48
+  ; GFX908-NEXT:   [[S_MOV_B32_38:%[0-9]+]]:sgpr_32 = S_MOV_B32 49
+  ; GFX908-NEXT:   [[S_MOV_B32_39:%[0-9]+]]:sgpr_32 = S_MOV_B32 50
+  ; GFX908-NEXT:   [[S_MOV_B32_40:%[0-9]+]]:sgpr_32 = S_MOV_B32 51
+  ; GFX908-NEXT:   [[S_MOV_B32_41:%[0-9]+]]:sgpr_32 = S_MOV_B32 52
+  ; GFX908-NEXT:   [[S_MOV_B32_42:%[0-9]+]]:sgpr_32 = S_MOV_B32 53
+  ; GFX908-NEXT:   [[S_MOV_B32_43:%[0-9]+]]:sgpr_32 = S_MOV_B32 54
+  ; GFX908-NEXT:   [[S_MOV_B32_44:%[0-9]+]]:sgpr_32 = S_MOV_B32 55
+  ; GFX908-NEXT:   [[S_MOV_B32_45:%[0-9]+]]:sgpr_32 = S_MOV_B32 56
+  ; GFX908-NEXT:   [[S_MOV_B32_46:%[0-9]+]]:sgpr_32 = S_MOV_B32 57
+  ; GFX908-NEXT:   [[S_MOV_B32_47:%[0-9]+]]:sgpr_32 = S_MOV_B32 58
+  ; GFX908-NEXT:   [[S_MOV_B32_48:%[0-9]+]]:sgpr_32 = S_MOV_B32 59
+  ; GFX908-NEXT:   [[S_MOV_B32_49:%[0-9]+]]:sgpr_32 = S_MOV_B32 60
+  ; GFX908-NEXT:   [[S_MOV_B32_50:%[0-9]+]]:sgpr_32 = S_MOV_B32 61
+  ; GFX908-NEXT:   [[S_MOV_B32_51:%[0-9]+]]:sgpr_32 = S_MOV_B32 62
+  ; GFX908-NEXT:   [[S_MOV_B32_52:%[0-9]+]]:sgpr_32 = S_MOV_B32 63
+  ; GFX908-NEXT:   [[S_MOV_B32_53:%[0-9]+]]:sgpr_32 = S_MOV_B32 64
+  ; GFX908-NEXT:   [[S_MOV_B32_54:%[0-9]+]]:sgpr_32 = S_MOV_B32 65
+  ; GFX908-NEXT:   [[S_MOV_B32_55:%[0-9]+]]:sgpr_32 = S_MOV_B32 66
+  ; GFX908-NEXT:   [[S_MOV_B32_56:%[0-9]+]]:sgpr_32 = S_MOV_B32 67
+  ; GFX908-NEXT:   [[S_MOV_B32_57:%[0-9]+]]:sgpr_32 = S_MOV_B32 68
+  ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
+  ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
+  ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
+  ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
+  ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
+  ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
-  ; GFX908-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
-  ; GFX908-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
-  ; GFX908-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
-  ; GFX908-NEXT:   [[S_MOV_B32_80:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
-  ; GFX908-NEXT:   [[S_MOV_B32_81:%[0-9]+]]:sgpr_32 = S_MOV_B32 80
-  ; GFX908-NEXT:   [[S_MOV_B32_82:%[0-9]+]]:sgpr_32 = S_MOV_B32 81
-  ; GFX908-NEXT:   [[S_MOV_B32_83:%[0-9]+]]:sgpr_32 = S_MOV_B32 82
-  ; GFX908-NEXT:   [[S_MOV_B32_84:%[0-9]+]]:sgpr_32 = S_MOV_B32 83
-  ; GFX908-NEXT:   [[S_MOV_B32_85:%[0-9]+]]:sgpr_32 = S_MOV_B32 84
+  ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
+  ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
+  ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
+  ; GFX908-NEXT:   [[S_MOV_B32_67:%[0-9]+]]:sgpr_32 = S_MOV_B32 78
+  ; GFX908-NEXT:   [[S_MOV_B32_68:%[0-9]+]]:sgpr_32 = S_MOV_B32 79
+  ; GFX908-NEXT:   [[S_MOV_B32_69:%[0-9]+]]:sgpr_32 = S_MOV_B32 80
+  ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 81
+  ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 82
+  ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 83
+  ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 84
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
   ; GFX908-NEXT: {{  $}}
+  ; GFX908-NEXT:   [[S_MOV_B32_74:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+  ; GFX908-NEXT:   [[S_MOV_B32_75:%[0-9]+]]:sgpr_32 = S_MOV_B32 1
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_74]], implicit [[S_MOV_B32_75]]
+  ; GFX908-NEXT:   [[S_MOV_B32_76:%[0-9]+]]:sgpr_32 = S_MOV_B32 2
+  ; GFX908-NEXT:   [[S_MOV_B32_77:%[0-9]+]]:sgpr_32 = S_MOV_B32 3
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_76]], implicit [[S_MOV_B32_77]]
+  ; GFX908-NEXT:   [[S_MOV_B32_78:%[0-9]+]]:sgpr_32 = S_MOV_B32 4
+  ; GFX908-NEXT:   [[S_MOV_B32_79:%[0-9]+]]:sgpr_32 = S_MOV_B32 5
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_78]], implicit [[S_MOV_B32_79]]
+  ; GFX908-NEXT:   [[S_MOV_B32_80:%[0-9]+]]:sgpr_32 = S_MOV_B32 6
+  ; GFX908-NEXT:   [[S_MOV_B32_81:%[0-9]+]]:sgpr_32 = S_MOV_B32 7
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_80]], implicit [[S_MOV_B32_81]]
+  ; GFX908-NEXT:   [[S_MOV_B32_82:%[0-9]+]]:sgpr_32 = S_MOV_B32 8
+  ; GFX908-NEXT:   [[S_MOV_B32_83:%[0-9]+]]:sgpr_32 = S_MOV_B32 9
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_82]], implicit [[S_MOV_B32_83]]
+  ; GFX908-NEXT:   [[S_MOV_B32_84:%[0-9]+]]:sgpr_32 = S_MOV_B32 10
+  ; GFX908-NEXT:   [[S_MOV_B32_85:%[0-9]+]]:sgpr_32 = S_MOV_B32 11
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_84]], implicit [[S_MOV_B32_85]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_5]], implicit [[S_MOV_B32_6]]
@@ -2173,13 +2179,7 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_67]], implicit [[S_MOV_B32_68]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_69]], implicit [[S_MOV_B32_70]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_71]], implicit [[S_MOV_B32_72]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_73]], implicit [[S_MOV_B32_74]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_75]], implicit [[S_MOV_B32_76]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_77]], implicit [[S_MOV_B32_78]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_79]], implicit [[S_MOV_B32_80]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_81]], implicit [[S_MOV_B32_82]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_83]], implicit [[S_MOV_B32_84]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_85]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[S_MOV_B32_73]]
   ; GFX908-NEXT:   [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GFX908-NEXT:   [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_GT_U32_e64_]], implicit-def dead $scc
   ; GFX908-NEXT:   $exec = S_MOV_B64_term [[S_AND_B64_]]
diff --git a/llvm/test/CodeGen/AMDGPU/mad-combine.ll b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
index e94aa4b8ce3d..7b8aa86bee2c 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1,14 +1,11 @@
 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
 
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
-
-; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
 
 ; Make sure we don't form mad with denormals
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=verde -denormal-fp-math-f32=ieee -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare float @llvm.fabs.f32(float) #0
@@ -25,15 +22,41 @@ declare float @llvm.fmuladd.f32(float, float, float) #0
 
 ; SI-DENORM-FASTFMAF: v_fma_f32 [[RESULT:v[0-9]+]], [[A]], [[B]], [[C]]
 
+; SI-DENORM-FASTFMAF: buffer_store_dword [[RESULT]]
+; SI-STD: buffer_store_dword [[C]]
+define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+  %gep.out = getelementptr float, ptr addrspace(1) %out, i32 %tid
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+
+
+  %mul = fmul contract float %a, %b
+  %fma = fadd contract float %mul, %c
+  store float %fma, ptr addrspace(1) %gep.out
+  ret void
+}
+; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0:
+; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
+; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
+; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
+
+; SI-STD: v_mac_f32_e32 [[C]], [[A]], [[B]]
+
 ; SI-DENORM-SLOWFMAF-NOT: v_fma
 ; SI-DENORM-SLOWFMAF-NOT: v_mad
 
 ; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
 ; SI-DENORM-SLOWFMAF: v_add_f32_e32 [[RESULT:v[0-9]+]],  [[TMP]], [[C]]
 
-; SI-DENORM: buffer_store_dword [[RESULT]]
+; SI-DENORM-SLOWFMAF: buffer_store_dword [[RESULT]]
 ; SI-STD: buffer_store_dword [[C]]
-define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @no_combine_to_mad_f32_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -72,7 +95,46 @@ define amdgpu_kernel void @combine_to_mad_f32_0(ptr addrspace(1) noalias %out, p
 ; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+  %gep.3 = getelementptr float, ptr addrspace(1) %gep.0, i32 3
+  %gep.out.0 = getelementptr float, ptr addrspace(1) %out, i32 %tid
+  %gep.out.1 = getelementptr float, ptr addrspace(1) %gep.out.0, i32 1
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+  %d = load volatile float, ptr addrspace(1) %gep.3
+
+  %mul = fmul contract fast float %a, %b
+  %fma0 = fadd contract fast float %mul, %c
+  %fma1 = fadd contract fast float %mul, %d
+  store volatile float %fma0, ptr addrspace(1) %gep.out.0
+  store volatile float %fma1, ptr addrspace(1) %gep.out.1
+  ret void
+}
+; FUNC-LABEL: {{^}}no_combine_to_mad_f32_0_2use:
+; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
+; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
+; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
+; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12 glc{{$}}
+
+; SI-STD-DAG: v_mac_f32_e32 [[C]], [[A]], [[B]]
+; SI-STD-DAG: v_mac_f32_e32 [[D]], [[A]], [[B]]
+
+; SI-DENORM-SLOWFMAF: v_mul_f32_e32 [[TMP:v[0-9]+]], [[A]], [[B]]
+; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT0:v[0-9]+]], [[TMP]], [[C]]
+; SI-DENORM-SLOWFMAF-DAG: v_add_f32_e32 [[RESULT1:v[0-9]+]], [[TMP]], [[D]]
+
+; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-DENORM-SLOWFMAF-DAG: buffer_store_dword [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI-STD-DAG: buffer_store_dword [[C]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; SI-STD-DAG: buffer_store_dword [[D]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
+; SI: s_endpgm
+define amdgpu_kernel void @no_combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_fast) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -89,7 +151,6 @@ define amdgpu_kernel void @combine_to_mad_f32_0_2use(ptr addrspace(1) noalias %o
   %mul = fmul float %a, %b
   %fma0 = fadd float %mul, %c
   %fma1 = fadd float %mul, %d
-
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -120,8 +181,8 @@ define amdgpu_kernel void @combine_to_mad_f32_1(ptr addrspace(1) noalias %out, p
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fadd float %c, %mul
+  %mul = fmul contract float %a, %b
+  %fma = fadd contract float %c, %mul
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -150,8 +211,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fsub float %mul, %c
+  %mul = fmul contract float %a, %b
+  %fma = fsub contract float %mul, %c
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -190,9 +251,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_0_f32_2use(ptr addrspace(1) noali
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %fma0 = fsub float %mul, %c
-  %fma1 = fsub float %mul, %d
+  %mul = fmul contract float %a, %b
+  %fma0 = fsub contract float %mul, %c
+  %fma1 = fsub contract float %mul, %d
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -222,8 +283,8 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %fma = fsub float %c, %mul
+  %mul = fmul contract float %a, %b
+  %fma = fsub contract float %c, %mul
   store float %fma, ptr addrspace(1) %gep.out
   ret void
 }
@@ -262,9 +323,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32_2use(ptr addrspace(1) noali
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %fma0 = fsub float %c, %mul
-  %fma1 = fsub float %d, %mul
+  %mul = fmul contract float %a, %b
+  %fma0 = fsub contract float %c, %mul
+  %fma1 = fsub contract float %d, %mul
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
   ret void
@@ -295,9 +356,9 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %o
   %b = load volatile float, ptr addrspace(1) %gep.1
   %c = load volatile float, ptr addrspace(1) %gep.2
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma = fsub float %mul.neg, %c
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma = fsub contract float %mul.neg, %c
 
   store float %fma, ptr addrspace(1) %gep.out
   ret void
@@ -337,10 +398,10 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(ptr addrspace(1)
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma0 = fsub float %mul.neg, %c
-  %fma1 = fsub float %mul.neg, %d
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma0 = fsub contract float %mul.neg, %c
+  %fma1 = fsub contract float %mul.neg, %d
 
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
@@ -381,10 +442,10 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1)
   %c = load volatile float, ptr addrspace(1) %gep.2
   %d = load volatile float, ptr addrspace(1) %gep.3
 
-  %mul = fmul float %a, %b
-  %mul.neg = fneg float %mul
-  %fma0 = fsub float %mul.neg, %c
-  %fma1 = fsub float %mul, %d
+  %mul = fmul contract float %a, %b
+  %mul.neg = fneg contract float %mul
+  %fma0 = fsub contract float %mul.neg, %c
+  %fma1 = fsub contract float %mul, %d
 
   store volatile float %fma0, ptr addrspace(1) %gep.out.0
   store volatile float %fma1, ptr addrspace(1) %gep.out.1
@@ -412,7 +473,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(ptr addrspace(1)
 ; SI-DENORM: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[C]]
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -427,10 +488,22 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul float %u, %v
-  %tmp1 = call float @llvm.fma.f32(float %x, float %y, float %tmp0) #0
-  %tmp2 = fsub float %tmp1, %z
+  br i1 %is_aggressive, label %aggressive, label %normal
+
+normal:
+  %tmp0_normal = fmul float %u, %v
+  %tmp1_normal = call float @llvm.fma.f32(float %x, float %y, float %tmp0_normal) #0
+  %tmp2_normal = fsub float %tmp1_normal, %z
+  br label %exit
 
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc float %u, %v
+  %tmp1_aggressive = call contract reassoc float @llvm.fma.f32(float %x, float %y, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc float %tmp1_aggressive, %z
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
@@ -505,7 +578,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_1_f32(ptr addrspace(1)
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -520,10 +593,22 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  %tmp0 = fmul float %u, %v
-  %tmp1 = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0) #0
-  %tmp2 = fsub float %tmp1, %z
+  br i1 %is_aggressive, label %aggressive, label %normal
 
+normal:
+  %tmp0_normal = fmul float %u, %v
+  %tmp1_normal = call float @llvm.fmuladd.f32(float %x, float %y, float %tmp0_normal) #0
+  %tmp2_normal = fsub float %tmp1_normal, %z
+  br label %exit
+
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc float %u, %v
+  %tmp1_aggressive = call contract reassoc float @llvm.fmuladd.f32(float %x, float %y, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc float %tmp1_aggressive, %z
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
@@ -556,7 +641,7 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_2_f32(ptr addrspace(1)
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI: s_endpgm
-define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i1 %is_aggressive) #1 {
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -571,11 +656,23 @@ define amdgpu_kernel void @aggressive_combine_to_mad_fsub_3_f32(ptr addrspace(1)
   %u = load volatile float, ptr addrspace(1) %gep.3
   %v = load volatile float, ptr addrspace(1) %gep.4
 
-  ; nsz flag is needed since this combine may change sign of zero
-  %tmp0 = fmul nsz float %u, %v
-  %tmp1 = call nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0) #0
-  %tmp2 = fsub nsz float %x, %tmp1
+  br i1 %is_aggressive, label %aggressive, label %normal
 
+normal:
+  ; nsz flag is needed since this combine may change sign of zero
+  %tmp0_normal = fmul nsz float %u, %v
+  %tmp1_normal = call nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0_normal) #0
+  %tmp2_normal = fsub nsz float %x, %tmp1_normal
+  br label %exit
+
+aggressive:
+  %tmp0_aggressive = fmul contract reassoc nsz float %u, %v
+  %tmp1_aggressive = call contract reassoc nsz float @llvm.fmuladd.f32(float %y, float %z, float %tmp0_aggressive) #0
+  %tmp2_aggressive = fsub contract reassoc nsz float %x, %tmp1_aggressive
+  br label %exit
+
+exit:
+  %tmp2 = phi float [%tmp2_normal, %normal], [%tmp2_aggressive, %aggressive]
   store float %tmp2, ptr addrspace(1) %gep.out
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 117f359be0c3..66df769a6beb 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -382,6 +382,7 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[8:9], null, v13, v14, v[8:9]
 ; GFX12-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v0
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v9, null, v9, v1, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
@@ -1157,7 +1158,7 @@ define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, v1, v3, vcc_lo
@@ -1247,10 +1248,11 @@ define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, 0
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v2, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, v1, v3, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
@@ -1794,9 +1796,11 @@ define i64 @lshr_mad_i64_negative_3(i64 %arg0) #0 {
 ; GFX12-NEXT:    v_and_b32_e32 v2, 0xfffffc00, v2
 ; GFX12-NEXT:    v_sub_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 1
 ; GFX12-NEXT:    s_wait_alu 0xfffd
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %op = add i64 %arg0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 7531777b803a..0c930ab0dbf2 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -772,11 +772,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
@@ -788,7 +788,7 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
@@ -1109,7 +1109,7 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v1.h, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
@@ -13276,6 +13276,7 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
@@ -13296,16 +13297,17 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_gt_f32_e64 s1, v52, v53
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
@@ -13835,9 +13837,9 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
@@ -15089,11 +15091,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0, v0.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
@@ -15105,7 +15107,7 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index 8b600c835a16..74a72e04fa4a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -7367,7 +7367,6 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7918,7 +7917,6 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8215,7 +8213,6 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8505,7 +8502,6 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8777,7 +8773,6 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9052,7 +9047,6 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9349,7 +9343,6 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9646,7 +9639,6 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9943,7 +9935,6 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10240,7 +10231,6 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10533,7 +10523,6 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -10830,7 +10819,6 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -11127,7 +11115,6 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18653,7 +18640,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18928,7 +18914,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19225,7 +19210,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19515,7 +19499,6 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19787,7 +19770,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20062,7 +20044,6 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20359,7 +20340,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20656,7 +20636,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20953,7 +20932,6 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21250,7 +21228,6 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21543,7 +21520,6 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21840,7 +21816,6 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -22137,7 +22112,6 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index 9c11781da56f..be148464c156 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -6365,7 +6365,6 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -6642,7 +6641,6 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -6945,7 +6943,6 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7241,7 +7238,6 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7515,7 +7511,6 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -7792,7 +7787,6 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8095,7 +8089,6 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8398,7 +8391,6 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -8701,7 +8693,6 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9004,7 +8995,6 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9303,7 +9293,6 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9606,7 +9595,6 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -9909,7 +9897,6 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -17555,7 +17542,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18112,7 +18098,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18415,7 +18400,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18711,7 +18695,6 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -18985,7 +18968,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19262,7 +19244,6 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19565,7 +19546,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -19868,7 +19848,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20171,7 +20150,6 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20474,7 +20452,6 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -20773,7 +20750,6 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21076,7 +21052,6 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
@@ -21379,7 +21354,6 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
-; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index cf45a0da5d15..d595d741caf3 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -781,11 +781,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
@@ -797,7 +797,7 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
@@ -1121,7 +1121,7 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v0.h, v1.h, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v5
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.l, vcc_lo
@@ -13331,6 +13331,7 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v69, 16, v132
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s7, 0, v69
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v33.h, v100.l, v33.h, s7
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff0000, v31
@@ -13351,16 +13352,17 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v51.l
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.h, v99.l, v33.l, s6
 ; GFX12-TRUE16-NEXT:    v_cmp_u_f32_e64 s0, v54, v54
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v53, 16, v53
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v31.l, v31.l, v15.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cmp_lt_f32_e64 s1, v52, v53
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v31.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v32.l, v51.l, v50.l, s1
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v38, 16, v52
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
@@ -13890,9 +13892,9 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v128, 16, v34
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e64 s3, v66, v66
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e64 v68, v68, v67, s3
 ; GFX12-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-FAKE16-NEXT:    v_cndmask_b32_e32 v67, v67, v68, vcc_lo
 ; GFX12-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v81, v81
 ; GFX12-FAKE16-NEXT:    v_lshlrev_b32_e32 v65, 16, v68
@@ -15149,11 +15151,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v4.l, v3.l, v2.l, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v5.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_u16_e64 s0, 0x8000, v0.l
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v2.l, vcc_lo
@@ -15165,7 +15167,7 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.l, v1.l, s2
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cmp_eq_f32_e64 s0, 0, v6
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v4.l, v0.h, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
index a8fab161e1ff..944951d3a536 100644
--- a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -mtriple=amdgcn -mcpu=gfx900 -amdgpu-aa -amdgpu-aa-wrapper -amdgpu-annotate-uniform -S < %s | FileCheck %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -check-prefix=GCN %s
 
@@ -7,16 +8,6 @@
 
 @LDS = linkonce_odr hidden local_unnamed_addr addrspace(3) global i32 poison
 
-; GCN-LABEL: {{^}}simple_barrier:
-; GCN: s_load_dword s
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: s_barrier
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: ; wave barrier
-; GCN-NOT: global_load_dword
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @simple_barrier(ptr addrspace(1) %arg) {
 ; CHECK-LABEL: @simple_barrier(
 ; CHECK-NEXT:  bb:
@@ -32,6 +23,21 @@ define amdgpu_kernel void @simple_barrier(ptr addrspace(1) %arg) {
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: simple_barrier:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    ; wave barrier
+; GCN-NEXT:    s_load_dword s3, s[0:1], 0x4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s2, s3, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:8
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   fence syncscope("workgroup") release
@@ -46,15 +52,6 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}memory_phi_no_clobber:
-; GCN: s_load_dword s
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: s_waitcnt lgkmcnt(0)
-; GCN: s_barrier
-; GCN-NOT: global_load_dword
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @memory_phi_no_clobber(ptr addrspace(1) %arg, i1 %cond) {
 ; CHECK-LABEL: @memory_phi_no_clobber(
 ; CHECK-NEXT:  bb:
@@ -74,6 +71,32 @@ define amdgpu_kernel void @memory_phi_no_clobber(ptr addrspace(1) %arg, i1 %cond
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: memory_phi_no_clobber:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp0_b32 s2, 0
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GCN-NEXT:    s_mov_b64 s[2:3], -1
+; GCN-NEXT:    s_cbranch_scc0 .LBB1_2
+; GCN-NEXT:  ; %bb.1: ; %if.else
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[2:3], 0
+; GCN-NEXT:  .LBB1_2: ; %Flow
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; GCN-NEXT:    s_cbranch_vccnz .LBB1_4
+; GCN-NEXT:  ; %bb.3: ; %if.then
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:  .LBB1_4: ; %if.end
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x4
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s2, s2, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:8
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br i1 %cond, label %if.then, label %if.else
@@ -95,12 +118,6 @@ if.end:
   ret void
 }
 
-; GCN-LABEL: {{^}}memory_phi_clobber1:
-; GCN: s_load_dword s
-; GCN: s_barrier
-; GCN: global_store_dword
-; GCN: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @memory_phi_clobber1(ptr addrspace(1) %arg, i1 %cond) {
 ; CHECK-LABEL: @memory_phi_clobber1(
 ; CHECK-NEXT:  bb:
@@ -121,6 +138,33 @@ define amdgpu_kernel void @memory_phi_clobber1(ptr addrspace(1) %arg, i1 %cond)
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: memory_phi_clobber1:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp0_b32 s2, 0
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GCN-NEXT:    s_mov_b64 s[2:3], -1
+; GCN-NEXT:    s_cbranch_scc0 .LBB2_2
+; GCN-NEXT:  ; %bb.1: ; %if.else
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_mov_b64 s[2:3], 0
+; GCN-NEXT:  .LBB2_2: ; %Flow
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; GCN-NEXT:    s_cbranch_vccnz .LBB2_4
+; GCN-NEXT:  ; %bb.3: ; %if.then
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, 1
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:12
+; GCN-NEXT:  .LBB2_4: ; %if.end
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1] offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_u32_e32 v1, s4, v1
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:8
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br i1 %cond, label %if.then, label %if.else
@@ -143,12 +187,6 @@ if.end:
   ret void
 }
 
-; GCN-LABEL: {{^}}memory_phi_clobber2:
-; GCN-DAG: s_load_dword s
-; GCN-DAG: global_store_dword
-; GCN: s_barrier
-; GCN: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @memory_phi_clobber2(ptr addrspace(1) %arg, i1 %cond) {
 ; CHECK-LABEL: @memory_phi_clobber2(
 ; CHECK-NEXT:  bb:
@@ -169,6 +207,33 @@ define amdgpu_kernel void @memory_phi_clobber2(ptr addrspace(1) %arg, i1 %cond)
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: memory_phi_clobber2:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp0_b32 s2, 0
+; GCN-NEXT:    s_mov_b64 s[2:3], -1
+; GCN-NEXT:    s_cbranch_scc0 .LBB3_2
+; GCN-NEXT:  ; %bb.1: ; %if.else
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, 1
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:12
+; GCN-NEXT:    s_mov_b64 s[2:3], 0
+; GCN-NEXT:  .LBB3_2: ; %Flow
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GCN-NEXT:    s_andn2_b64 vcc, exec, s[2:3]
+; GCN-NEXT:    s_cbranch_vccnz .LBB3_4
+; GCN-NEXT:  ; %bb.3: ; %if.then
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:  .LBB3_4: ; %if.end
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1] offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_u32_e32 v1, s4, v1
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:8
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br i1 %cond, label %if.then, label %if.else
@@ -191,11 +256,6 @@ if.end:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_clobbering_loop1:
-; GCN: s_load_dword s
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @no_clobbering_loop1(ptr addrspace(1) %arg, i1 %cc) {
 ; CHECK-LABEL: @no_clobbering_loop1(
 ; CHECK-NEXT:  bb:
@@ -212,6 +272,30 @@ define amdgpu_kernel void @no_clobbering_loop1(ptr addrspace(1) %arg, i1 %cc) {
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_clobbering_loop1:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s0, 0
+; GCN-NEXT:    s_load_dword s4, s[2:3], 0x0
+; GCN-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], -1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; GCN-NEXT:  .LBB4_1: ; %while.cond
+; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT:    s_load_dword s5, s[2:3], 0x4
+; GCN-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s5, s5, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3] offset:8
+; GCN-NEXT:    ; wave barrier
+; GCN-NEXT:    s_cbranch_vccnz .LBB4_1
+; GCN-NEXT:  ; %bb.2: ; %end
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br label %while.cond
@@ -229,11 +313,6 @@ end:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_clobbering_loop2:
-; GCN: s_load_dword s
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @no_clobbering_loop2(ptr addrspace(1) noalias %arg, ptr addrspace(1) noalias %out, i32 %n) {
 ; CHECK-LABEL: @no_clobbering_loop2(
 ; CHECK-NEXT:  bb:
@@ -253,6 +332,28 @@ define amdgpu_kernel void @no_clobbering_loop2(ptr addrspace(1) noalias %arg, pt
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_clobbering_loop2:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GCN-NEXT:  .LBB5_1: ; %while.cond
+; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GCN-NEXT:    ; wave barrier
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_add_i32 s4, s5, s4
+; GCN-NEXT:    s_add_u32 s0, s0, 4
+; GCN-NEXT:    s_addc_u32 s1, s1, 0
+; GCN-NEXT:    s_add_i32 s6, s6, -1
+; GCN-NEXT:    s_cmp_eq_u32 s6, 0
+; GCN-NEXT:    s_cbranch_scc1 .LBB5_1
+; GCN-NEXT:  ; %bb.2: ; %end
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br label %while.cond
@@ -273,10 +374,6 @@ end:
   ret void
 }
 
-; GCN-LABEL: {{^}}clobbering_loop:
-; GCN: s_load_dword s
-; GCN: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @clobbering_loop(ptr addrspace(1) %arg, ptr addrspace(1) %out, i1 %cc) {
 ; CHECK-LABEL: @clobbering_loop(
 ; CHECK-NEXT:  bb:
@@ -293,6 +390,29 @@ define amdgpu_kernel void @clobbering_loop(ptr addrspace(1) %arg, ptr addrspace(
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: clobbering_loop:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dword s0, s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s0, 0
+; GCN-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GCN-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT:    s_xor_b64 s[0:1], s[0:1], -1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GCN-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; GCN-NEXT:  .LBB6_1: ; %while.cond
+; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT:    global_load_dword v1, v0, s[8:9] offset:4
+; GCN-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_add_u32_e32 v1, s2, v1
+; GCN-NEXT:    global_store_dword v0, v1, s[10:11] offset:4
+; GCN-NEXT:    ; wave barrier
+; GCN-NEXT:    s_cbranch_vccnz .LBB6_1
+; GCN-NEXT:  ; %bb.2: ; %end
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   br label %while.cond
@@ -310,11 +430,6 @@ end:
   ret void
 }
 
-; GCN-LABEL: {{^}}clobber_by_atomic_load:
-; GCN: s_load_dword s
-; GCN: global_load_dword {{.*}} glc
-; GCN: global_load_dword
-; GCN: global_store_dword
 define amdgpu_kernel void @clobber_by_atomic_load(ptr addrspace(1) %arg) {
 ; CHECK-LABEL: @clobber_by_atomic_load(
 ; CHECK-NEXT:  bb:
@@ -328,6 +443,21 @@ define amdgpu_kernel void @clobber_by_atomic_load(ptr addrspace(1) %arg) {
 ; CHECK-NEXT:    store i32 [[I3]], ptr addrspace(1) [[I4]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: clobber_by_atomic_load:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1] offset:8 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_wbinvl1_vol
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1] offset:12
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_u32_e32 v1, s2, v1
+; GCN-NEXT:    global_store_dword v0, v1, s[0:1] offset:16
+; GCN-NEXT:    s_endpgm
 bb:
   %i = load i32, ptr addrspace(1) %arg, align 4
   %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 2
@@ -340,12 +470,6 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_store:
-; GCN: ds_write_b32
-; GCN: s_barrier
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @no_alias_store(
 ; CHECK-NEXT:  entry:
@@ -357,6 +481,18 @@ define protected amdgpu_kernel void @no_alias_store(ptr addrspace(1) %in, ptr ad
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_store:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    ds_write_b32 v0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   store i32 0, ptr addrspace(3) @LDS, align 4
   fence syncscope("workgroup") release
@@ -367,11 +503,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}may_alias_store:
-; GCN: global_store_dword
-; GCN: s_barrier
-; GCN: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @may_alias_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @may_alias_store(
 ; CHECK-NEXT:  entry:
@@ -383,6 +514,18 @@ define protected amdgpu_kernel void @may_alias_store(ptr addrspace(1) %in, ptr a
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: may_alias_store:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_dword v0, v0, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   store i32 0, ptr addrspace(1) %out, align 4
   fence syncscope("workgroup") release
@@ -393,12 +536,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_volatile_store:
-; GCN: ds_write_b32
-; GCN: s_barrier
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_volatile_store(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @no_alias_volatile_store(
 ; CHECK-NEXT:  entry:
@@ -410,6 +547,18 @@ define protected amdgpu_kernel void @no_alias_volatile_store(ptr addrspace(1) %i
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_volatile_store:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    ds_write_b32 v0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   store volatile i32 0, ptr addrspace(3) @LDS, align 4
   fence syncscope("workgroup") release
@@ -420,11 +569,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_atomic_rmw_relaxed:
-; GCN: ds_add_u32
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_atomic_rmw_relaxed(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @no_alias_atomic_rmw_relaxed(
 ; CHECK-NEXT:  entry:
@@ -433,6 +577,18 @@ define protected amdgpu_kernel void @no_alias_atomic_rmw_relaxed(ptr addrspace(1
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_atomic_rmw_relaxed:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 5
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    ds_add_u32 v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   %unused = atomicrmw add ptr addrspace(3) @LDS, i32 5 monotonic
   %ld = load i32, ptr addrspace(1) %in, align 4
@@ -440,11 +596,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_atomic_cmpxchg:
-; GCN: ds_cmpst_b32
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_atomic_cmpxchg(ptr addrspace(1) %in, ptr addrspace(1) %out, i32 %swap) {
 ; CHECK-LABEL: @no_alias_atomic_cmpxchg(
 ; CHECK-NEXT:  entry:
@@ -456,6 +607,22 @@ define protected amdgpu_kernel void @no_alias_atomic_cmpxchg(ptr addrspace(1) %i
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_atomic_cmpxchg:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 7
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v2, s6
+; GCN-NEXT:    ds_cmpst_b32 v1, v0, v2
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   %unused = cmpxchg ptr addrspace(3) @LDS, i32 7, i32 %swap seq_cst monotonic
   fence syncscope("workgroup") release
@@ -466,11 +633,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_atomic_rmw:
-; GCN: ds_add_u32
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_atomic_rmw(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @no_alias_atomic_rmw(
 ; CHECK-NEXT:  entry:
@@ -482,6 +644,20 @@ define protected amdgpu_kernel void @no_alias_atomic_rmw(ptr addrspace(1) %in, p
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_atomic_rmw:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 5
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_add_u32 v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   %unused = atomicrmw add ptr addrspace(3) @LDS, i32 5 seq_cst
   fence syncscope("workgroup") release
@@ -492,10 +668,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}may_alias_atomic_cmpxchg:
-; GCN: global_atomic_cmpswap
-; GCN: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @may_alias_atomic_cmpxchg(ptr addrspace(1) %in, ptr addrspace(1) %out, i32 %swap) {
 ; CHECK-LABEL: @may_alias_atomic_cmpxchg(
 ; CHECK-NEXT:  entry:
@@ -507,6 +679,22 @@ define protected amdgpu_kernel void @may_alias_atomic_cmpxchg(ptr addrspace(1) %
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: may_alias_atomic_cmpxchg:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, 7
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_wbinvl1_vol
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    global_load_dword v0, v2, s[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v2, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   %unused = cmpxchg ptr addrspace(1) %out, i32 7, i32 %swap seq_cst monotonic
   fence syncscope("workgroup") release
@@ -517,10 +705,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}may_alias_atomic_rmw:
-; GCN: global_atomic_add
-; GCN: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @may_alias_atomic_rmw(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; CHECK-LABEL: @may_alias_atomic_rmw(
 ; CHECK-NEXT:  entry:
@@ -532,6 +716,20 @@ define protected amdgpu_kernel void @may_alias_atomic_rmw(ptr addrspace(1) %in,
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: may_alias_atomic_rmw:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    v_mov_b32_e32 v1, 5
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_atomic_add v0, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_wbinvl1_vol
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    global_load_dword v1, v0, s[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v0, v1, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   %unused = atomicrmw add ptr addrspace(1) %out, i32 5 syncscope("agent") seq_cst
   fence syncscope("workgroup") release
@@ -542,12 +740,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_atomic_rmw_then_clobber:
-; GCN: global_store_dword
-; GCN: global_store_dword
-; GCN: ds_add_u32
-; GCN: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_atomic_rmw_then_clobber(ptr addrspace(1) %in, ptr addrspace(1) %out, ptr addrspace(1) noalias %noalias) {
 ; CHECK-LABEL: @no_alias_atomic_rmw_then_clobber(
 ; CHECK-NEXT:  entry:
@@ -561,6 +753,24 @@ define protected amdgpu_kernel void @no_alias_atomic_rmw_then_clobber(ptr addrsp
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_atomic_rmw_then_clobber:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN-NEXT:    v_mov_b32_e32 v0, 1
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 2
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    global_store_dword v1, v2, s[6:7]
+; GCN-NEXT:    v_mov_b32_e32 v0, 5
+; GCN-NEXT:    ds_add_u32 v1, v0
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    global_load_dword v0, v1, s[0:1]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   store i32 1, ptr addrspace(1) %out, align 4
   store i32 2, ptr addrspace(1) %noalias, align 4
@@ -573,12 +783,6 @@ entry:
   ret void
 }
 
-; GCN-LABEL: {{^}}no_alias_atomic_rmw_then_no_alias_store:
-; GCN: global_store_dword
-; GCN: ds_add_u32
-; GCN: s_load_dword s
-; GCN-NOT: global_load_dword
-; GCN: global_store_dword
 define protected amdgpu_kernel void @no_alias_atomic_rmw_then_no_alias_store(ptr addrspace(1) %in, ptr addrspace(1) %out, ptr addrspace(1) noalias %noalias) {
 ; CHECK-LABEL: @no_alias_atomic_rmw_then_no_alias_store(
 ; CHECK-NEXT:  entry:
@@ -591,6 +795,23 @@ define protected amdgpu_kernel void @no_alias_atomic_rmw_then_no_alias_store(ptr
 ; CHECK-NEXT:    store i32 [[LD]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
+; GCN-LABEL: no_alias_atomic_rmw_then_no_alias_store:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT:    v_mov_b32_e32 v0, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v2, 5
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_dword v1, v0, s[6:7]
+; GCN-NEXT:    ds_add_u32 v1, v2
+; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_barrier
+; GCN-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    global_store_dword v1, v0, s[2:3]
+; GCN-NEXT:    s_endpgm
 entry:
   store i32 2, ptr addrspace(1) %noalias, align 4
   %unused = atomicrmw add ptr addrspace(3) @LDS, i32 5 seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index e7343c150ea7..b1e304e79348 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -773,7 +773,6 @@ define amdgpu_kernel void @mix_elt_types_op_sel(ptr addrspace(1) %out, ptr addrs
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; kill: killed $vgpr0_vgpr1
 ; GCN-NEXT:    v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0]
-; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-NEXT:    v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
 ; GCN-NEXT:    global_store_dword v3, v0, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/prevent-fmul-hoist-ir.ll b/llvm/test/CodeGen/AMDGPU/prevent-fmul-hoist-ir.ll
index c68cd8254091..ef3e04c0e996 100644
--- a/llvm/test/CodeGen/AMDGPU/prevent-fmul-hoist-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/prevent-fmul-hoist-ir.ll
@@ -11,17 +11,16 @@ define double @is_profitable_f64_contract(ptr dereferenceable(8) %ptr_x, ptr der
 ; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
 ; GFX-NEXT:    [[X:%.*]] = load double, ptr [[PTR_X]], align 8
 ; GFX-NEXT:    [[A_1:%.*]] = load double, ptr [[PTR_A]], align 8
+; GFX-NEXT:    [[MUL:%.*]] = fmul contract double [[X]], [[A_1]]
 ; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
 ; GFX:       [[COMMON_RET:.*]]:
 ; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi double [ [[ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
 ; GFX-NEXT:    ret double [[COMMON_RET_OP]]
 ; GFX:       [[IF_THEN]]:
-; GFX-NEXT:    [[MUL:%.*]] = fmul contract double [[X]], [[A_1]]
 ; GFX-NEXT:    [[ADD]] = fadd contract double 1.000000e+00, [[MUL]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ; GFX:       [[IF_ELSE]]:
-; GFX-NEXT:    [[MUL1:%.*]] = fmul contract double [[X]], [[A_1]]
-; GFX-NEXT:    [[SUB]] = fsub contract double [[MUL1]], [[Y]]
+; GFX-NEXT:    [[SUB]] = fsub contract double [[MUL]], [[Y]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
 entry:
@@ -94,17 +93,16 @@ define float @is_profitable_f32(ptr dereferenceable(8) %ptr_x, ptr dereferenceab
 ; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq float [[Y]], 0.000000e+00
 ; GFX-NEXT:    [[X:%.*]] = load float, ptr [[PTR_X]], align 8
 ; GFX-NEXT:    [[A_1:%.*]] = load float, ptr [[PTR_A]], align 8
+; GFX-NEXT:    [[MUL:%.*]] = fmul contract float [[X]], [[A_1]]
 ; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
 ; GFX:       [[COMMON_RET:.*]]:
-; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[MUL:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
+; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[MUL]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
 ; GFX-NEXT:    ret float [[COMMON_RET_OP]]
 ; GFX:       [[IF_THEN]]:
-; GFX-NEXT:    [[MUL]] = fmul contract float [[X]], [[A_1]]
 ; GFX-NEXT:    [[ADD:%.*]] = fadd contract float 1.000000e+00, [[MUL]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ; GFX:       [[IF_ELSE]]:
-; GFX-NEXT:    [[MUL1:%.*]] = fmul contract float [[X]], [[A_1]]
-; GFX-NEXT:    [[SUB]] = fsub contract float [[MUL1]], [[Y]]
+; GFX-NEXT:    [[SUB]] = fsub contract float [[MUL]], [[Y]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
 entry:
@@ -113,6 +111,7 @@ entry:
   %x = load float, ptr %ptr_x, align 8
   br i1 %cmp, label %if.then, label %if.else
 
+
 if.then:                                          ; preds = %entry
   %a_1 = load float, ptr %ptr_a, align 8
   %mul = fmul contract float %x, %a_1
@@ -173,17 +172,16 @@ define half @is_profitable_f16_ieee(ptr dereferenceable(8) %ptr_x, ptr dereferen
 ; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq half [[Y]], 0xH0000
 ; GFX-NEXT:    [[X:%.*]] = load half, ptr [[PTR_X]], align 8
 ; GFX-NEXT:    [[A_1:%.*]] = load half, ptr [[PTR_A]], align 8
+; GFX-NEXT:    [[MUL:%.*]] = fmul contract half [[X]], [[A_1]]
 ; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
 ; GFX:       [[COMMON_RET:.*]]:
-; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi half [ [[MUL:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
+; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi half [ [[MUL]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
 ; GFX-NEXT:    ret half [[COMMON_RET_OP]]
 ; GFX:       [[IF_THEN]]:
-; GFX-NEXT:    [[MUL]] = fmul contract half [[X]], [[A_1]]
 ; GFX-NEXT:    [[ADD:%.*]] = fadd contract half [[Y]], [[MUL]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ; GFX:       [[IF_ELSE]]:
-; GFX-NEXT:    [[MUL1:%.*]] = fmul contract half [[X]], [[A_1]]
-; GFX-NEXT:    [[SUB]] = fsub contract half [[MUL1]], [[Y]]
+; GFX-NEXT:    [[SUB]] = fsub contract half [[MUL]], [[Y]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
 entry:
@@ -252,17 +250,16 @@ define bfloat @is_profitable_bfloat_ieee(ptr dereferenceable(8) %ptr_x, ptr dere
 ; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq bfloat [[Y]], 0xR0000
 ; GFX-NEXT:    [[X:%.*]] = load bfloat, ptr [[PTR_X]], align 8
 ; GFX-NEXT:    [[A_1:%.*]] = load bfloat, ptr [[PTR_A]], align 8
+; GFX-NEXT:    [[MUL:%.*]] = fmul contract bfloat [[X]], [[A_1]]
 ; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
 ; GFX:       [[COMMON_RET:.*]]:
-; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi bfloat [ [[MUL:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
+; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi bfloat [ [[MUL]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
 ; GFX-NEXT:    ret bfloat [[COMMON_RET_OP]]
 ; GFX:       [[IF_THEN]]:
-; GFX-NEXT:    [[MUL]] = fmul contract bfloat [[X]], [[A_1]]
 ; GFX-NEXT:    [[ADD:%.*]] = fadd contract bfloat 0xR3F80, [[MUL]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ; GFX:       [[IF_ELSE]]:
-; GFX-NEXT:    [[MUL1:%.*]] = fmul contract bfloat [[X]], [[A_1]]
-; GFX-NEXT:    [[SUB]] = fsub contract bfloat [[MUL1]], [[Y]]
+; GFX-NEXT:    [[SUB]] = fsub contract bfloat [[MUL]], [[Y]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
 entry:
@@ -333,17 +330,16 @@ define <8 x half> @is_profitable_vector(ptr dereferenceable(8) %ptr_x, ptr deref
 ; GFX-NEXT:    [[V1:%.*]] = load <8 x half>, ptr addrspace(3) @v1_ptr, align 16
 ; GFX-NEXT:    [[V2:%.*]] = load <8 x half>, ptr addrspace(3) @v2_ptr, align 16
 ; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
+; GFX-NEXT:    [[MUL:%.*]] = fmul contract <8 x half> [[V1]], [[X]]
 ; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
 ; GFX:       [[COMMON_RET:.*]]:
 ; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi <8 x half> [ [[ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
 ; GFX-NEXT:    ret <8 x half> [[COMMON_RET_OP]]
 ; GFX:       [[IF_THEN]]:
-; GFX-NEXT:    [[MUL:%.*]] = fmul contract <8 x half> [[V1]], [[X]]
 ; GFX-NEXT:    [[ADD]] = fadd contract <8 x half> [[V2]], [[MUL]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ; GFX:       [[IF_ELSE]]:
-; GFX-NEXT:    [[MUL1:%.*]] = fmul contract <8 x half> [[V1]], [[X]]
-; GFX-NEXT:    [[SUB]] = fsub contract <8 x half> [[MUL1]], [[V2]]
+; GFX-NEXT:    [[SUB]] = fsub contract <8 x half> [[MUL]], [[V2]]
 ; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
 entry:
@@ -366,61 +362,23 @@ if.else:                                          ; preds = %entry
 }
 
 define double @is_profitable_f64_nocontract(ptr dereferenceable(8) %ptr_x, ptr dereferenceable(8) %ptr_y, ptr dereferenceable(8) %ptr_a) #0 {
-; FP-CONTRACT-FAST-LABEL: define double @is_profitable_f64_nocontract(
-; FP-CONTRACT-FAST-SAME: ptr dereferenceable(8) [[PTR_X:%.*]], ptr dereferenceable(8) [[PTR_Y:%.*]], ptr dereferenceable(8) [[PTR_A:%.*]]) #[[ATTR0]] {
-; FP-CONTRACT-FAST-NEXT:    [[Y:%.*]] = load double, ptr [[PTR_Y]], align 8
-; FP-CONTRACT-FAST-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
-; FP-CONTRACT-FAST-NEXT:    [[X:%.*]] = load double, ptr [[PTR_X]], align 8
-; FP-CONTRACT-FAST-NEXT:    [[A_1:%.*]] = load double, ptr [[PTR_A]], align 8
-; FP-CONTRACT-FAST-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; FP-CONTRACT-FAST:       [[COMMON_RET:.*]]:
-; FP-CONTRACT-FAST-NEXT:    [[COMMON_RET_OP:%.*]] = phi double [ [[PTR_ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
-; FP-CONTRACT-FAST-NEXT:    ret double [[COMMON_RET_OP]]
-; FP-CONTRACT-FAST:       [[IF_THEN]]:
-; FP-CONTRACT-FAST-NEXT:    [[MUL:%.*]] = fmul double [[X]], [[A_1]]
-; FP-CONTRACT-FAST-NEXT:    [[PTR_ADD]] = fadd double 1.000000e+00, [[MUL]]
-; FP-CONTRACT-FAST-NEXT:    br label %[[COMMON_RET]]
-; FP-CONTRACT-FAST:       [[IF_ELSE]]:
-; FP-CONTRACT-FAST-NEXT:    [[MUL1:%.*]] = fmul double [[X]], [[A_1]]
-; FP-CONTRACT-FAST-NEXT:    [[SUB]] = fsub double [[MUL1]], [[Y]]
-; FP-CONTRACT-FAST-NEXT:    br label %[[COMMON_RET]]
-;
-; UNSAFE-FP-MATH-LABEL: define double @is_profitable_f64_nocontract(
-; UNSAFE-FP-MATH-SAME: ptr dereferenceable(8) [[PTR_X:%.*]], ptr dereferenceable(8) [[PTR_Y:%.*]], ptr dereferenceable(8) [[PTR_A:%.*]]) #[[ATTR0]] {
-; UNSAFE-FP-MATH-NEXT:    [[Y:%.*]] = load double, ptr [[PTR_Y]], align 8
-; UNSAFE-FP-MATH-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
-; UNSAFE-FP-MATH-NEXT:    [[X:%.*]] = load double, ptr [[PTR_X]], align 8
-; UNSAFE-FP-MATH-NEXT:    [[A_1:%.*]] = load double, ptr [[PTR_A]], align 8
-; UNSAFE-FP-MATH-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; UNSAFE-FP-MATH:       [[COMMON_RET:.*]]:
-; UNSAFE-FP-MATH-NEXT:    [[COMMON_RET_OP:%.*]] = phi double [ [[PTR_ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
-; UNSAFE-FP-MATH-NEXT:    ret double [[COMMON_RET_OP]]
-; UNSAFE-FP-MATH:       [[IF_THEN]]:
-; UNSAFE-FP-MATH-NEXT:    [[MUL:%.*]] = fmul double [[X]], [[A_1]]
-; UNSAFE-FP-MATH-NEXT:    [[PTR_ADD]] = fadd double 1.000000e+00, [[MUL]]
-; UNSAFE-FP-MATH-NEXT:    br label %[[COMMON_RET]]
-; UNSAFE-FP-MATH:       [[IF_ELSE]]:
-; UNSAFE-FP-MATH-NEXT:    [[MUL1:%.*]] = fmul double [[X]], [[A_1]]
-; UNSAFE-FP-MATH-NEXT:    [[SUB]] = fsub double [[MUL1]], [[Y]]
-; UNSAFE-FP-MATH-NEXT:    br label %[[COMMON_RET]]
-;
-; NO-UNSAFE-FP-MATH-LABEL: define double @is_profitable_f64_nocontract(
-; NO-UNSAFE-FP-MATH-SAME: ptr dereferenceable(8) [[PTR_X:%.*]], ptr dereferenceable(8) [[PTR_Y:%.*]], ptr dereferenceable(8) [[PTR_A:%.*]]) #[[ATTR0]] {
-; NO-UNSAFE-FP-MATH-NEXT:    [[Y:%.*]] = load double, ptr [[PTR_Y]], align 8
-; NO-UNSAFE-FP-MATH-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
-; NO-UNSAFE-FP-MATH-NEXT:    [[X:%.*]] = load double, ptr [[PTR_X]], align 8
-; NO-UNSAFE-FP-MATH-NEXT:    [[A_1:%.*]] = load double, ptr [[PTR_A]], align 8
-; NO-UNSAFE-FP-MATH-NEXT:    [[MUL:%.*]] = fmul double [[X]], [[A_1]]
-; NO-UNSAFE-FP-MATH-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; NO-UNSAFE-FP-MATH:       [[COMMON_RET:.*]]:
-; NO-UNSAFE-FP-MATH-NEXT:    [[COMMON_RET_OP:%.*]] = phi double [ [[PTR_ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
-; NO-UNSAFE-FP-MATH-NEXT:    ret double [[COMMON_RET_OP]]
-; NO-UNSAFE-FP-MATH:       [[IF_THEN]]:
-; NO-UNSAFE-FP-MATH-NEXT:    [[PTR_ADD]] = fadd double 1.000000e+00, [[MUL]]
-; NO-UNSAFE-FP-MATH-NEXT:    br label %[[COMMON_RET]]
-; NO-UNSAFE-FP-MATH:       [[IF_ELSE]]:
-; NO-UNSAFE-FP-MATH-NEXT:    [[SUB]] = fsub double [[MUL]], [[Y]]
-; NO-UNSAFE-FP-MATH-NEXT:    br label %[[COMMON_RET]]
+; GFX-LABEL: define double @is_profitable_f64_nocontract(
+; GFX-SAME: ptr dereferenceable(8) [[PTR_X:%.*]], ptr dereferenceable(8) [[PTR_Y:%.*]], ptr dereferenceable(8) [[PTR_A:%.*]]) #[[ATTR0]] {
+; GFX-NEXT:    [[Y:%.*]] = load double, ptr [[PTR_Y]], align 8
+; GFX-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y]], 0.000000e+00
+; GFX-NEXT:    [[X:%.*]] = load double, ptr [[PTR_X]], align 8
+; GFX-NEXT:    [[A_1:%.*]] = load double, ptr [[PTR_A]], align 8
+; GFX-NEXT:    [[MUL:%.*]] = fmul double [[X]], [[A_1]]
+; GFX-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; GFX:       [[COMMON_RET:.*]]:
+; GFX-NEXT:    [[COMMON_RET_OP:%.*]] = phi double [ [[PTR_ADD:%.*]], %[[IF_THEN]] ], [ [[SUB:%.*]], %[[IF_ELSE]] ]
+; GFX-NEXT:    ret double [[COMMON_RET_OP]]
+; GFX:       [[IF_THEN]]:
+; GFX-NEXT:    [[PTR_ADD]] = fadd double 1.000000e+00, [[MUL]]
+; GFX-NEXT:    br label %[[COMMON_RET]]
+; GFX:       [[IF_ELSE]]:
+; GFX-NEXT:    [[SUB]] = fsub double [[MUL]], [[Y]]
+; GFX-NEXT:    br label %[[COMMON_RET]]
 ;
   %y = load double, ptr %ptr_y, align 8
   %cmp = fcmp oeq double %y, 0.000000e+00
@@ -442,3 +400,7 @@ if.else:                                          ; preds = %entry
 
 attributes #0 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
 attributes #1 = { nounwind "denormal-fp-math"="ieee,ieee" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FP-CONTRACT-FAST: {{.*}}
+; NO-UNSAFE-FP-MATH: {{.*}}
+; UNSAFE-FP-MATH: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index c4842c1f4f52..bbfd5f49981b 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -277,24 +277,27 @@ define amdgpu_kernel void @clmem_read_simplified(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:2048
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v4, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v5, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v10, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v11, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v9, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(3)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v6, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v7, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v12, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v13, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v14, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v15, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
@@ -879,10 +882,11 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    s_cmp_gt_u32 s2, 0x3fffff
 ; GFX11-NEXT:    s_waitcnt vmcnt(10)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v13, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v14, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(9)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v9, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v10, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v7, v2
@@ -892,24 +896,27 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v12, v3, s0
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v19, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v20, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v17, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v18, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v15, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v16, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(3)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v21, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v22, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v23, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v24, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_add_co_u32 v2, s0, v25, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v26, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v27, v2
@@ -1528,10 +1535,11 @@ define amdgpu_kernel void @Offset64(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v6, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, v7, v5, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v5, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
@@ -2317,27 +2325,30 @@ define amdgpu_kernel void @ReverseOrder(ptr addrspace(1) %buffer) {
 ; GFX11-NEXT:    global_load_b64 v[0:1], v[0:1], off offset:2048
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
 ; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v6, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, v7, v5, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v5, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v12, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v13, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v8, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v9, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v10, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v11, v3, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v14, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, v15, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
 ; GFX11-NEXT:    global_store_b64 v16, v[0:1], s[34:35]
 ; GFX11-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index 4a5fa641da03..1ec94162951a 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -8,24 +8,14 @@
 
 ; Tests reassociation (ptradd N0:(ptradd p, c1), z) where N0 has only one use.
 define i64 @global_load_ZTwoUses(ptr addrspace(1) %base, i64 %voffset) {
-; GFX942_PTRADD-LABEL: global_load_ZTwoUses:
-; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 24
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942_LEGACY-LABEL: global_load_ZTwoUses:
-; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_LEGACY-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:24
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0)
-; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+; GFX942-LABEL: global_load_ZTwoUses:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:24
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %base, i64 24
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 %voffset
   %l = load i64, ptr addrspace(1) %gep1, align 8
@@ -37,9 +27,8 @@ define i64 @global_load_gep_add_reassoc(ptr addrspace(1) %base, i64 %voffset) {
 ; GFX942_PTRADD-LABEL: global_load_gep_add_reassoc:
 ; GFX942_PTRADD:       ; %bb.0:
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], v[2:3], 0, 24
 ; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942_PTRADD-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off
+; GFX942_PTRADD-NEXT:    global_load_dwordx2 v[0:1], v[0:1], off offset:24
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -60,143 +49,272 @@ define i64 @global_load_gep_add_reassoc(ptr addrspace(1) %base, i64 %voffset) {
 ; would be folded away in most cases, but the index computation introduced by
 ; the legalization of wide vector stores can for example introduce them.
 define amdgpu_kernel void @store_v16i32(ptr addrspace(1) %out, <16 x i32> %a) {
-; GFX942_PTRADD-LABEL: store_v16i32:
+; GFX942-LABEL: store_v16i32:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v0, s20
+; GFX942-NEXT:    v_mov_b32_e32 v1, s21
+; GFX942-NEXT:    v_mov_b32_e32 v2, s22
+; GFX942-NEXT:    v_mov_b32_e32 v3, s23
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_mov_b32_e32 v0, s16
+; GFX942-NEXT:    v_mov_b32_e32 v1, s17
+; GFX942-NEXT:    v_mov_b32_e32 v2, s18
+; GFX942-NEXT:    v_mov_b32_e32 v3, s19
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_mov_b32_e32 v0, s12
+; GFX942-NEXT:    v_mov_b32_e32 v1, s13
+; GFX942-NEXT:    v_mov_b32_e32 v2, s14
+; GFX942-NEXT:    v_mov_b32_e32 v3, s15
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; GFX942-NEXT:    s_nop 1
+; GFX942-NEXT:    v_mov_b32_e32 v0, s8
+; GFX942-NEXT:    v_mov_b32_e32 v1, s9
+; GFX942-NEXT:    v_mov_b32_e32 v2, s10
+; GFX942-NEXT:    v_mov_b32_e32 v3, s11
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
+entry:
+  store <16 x i32> %a, ptr addrspace(1) %out
+  ret void
+}
+
+
+; Tests the (ptradd 0, x) -> x DAG combine.
+define void @baseptr_null(i64 %offset, i8 %v) {
+; GFX942-LABEL: baseptr_null:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    flat_store_byte v[0:1], v2
+; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr i8, ptr null, i64 %offset
+  store i8 %v, ptr %gep, align 1
+  ret void
+}
+
+; Taken from implicit-kernarg-backend-usage.ll, tests the PTRADD handling in the
+; assertalign DAG combine.
+define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  #0 {
+; GFX942-LABEL: llvm_amdgcn_queue_ptr:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    global_load_ubyte v0, v2, s[2:3] sc0 sc1
+; GFX942-NEXT:    global_load_ubyte v0, v2, s[4:5] offset:8 sc0 sc1
+; GFX942-NEXT:    global_load_ubyte v0, v2, s[0:1] sc0 sc1
+; GFX942-NEXT:    ; kill: killed $sgpr0_sgpr1
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
+; GFX942-NEXT:    ; kill: killed $sgpr2_sgpr3
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_endpgm
+  %queue.ptr = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %dispatch.ptr = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
+  %dispatch.id = call i64 @llvm.amdgcn.dispatch.id()
+  %queue.load = load volatile i8, ptr addrspace(4) %queue.ptr
+  %implicitarg.load = load volatile i8, ptr addrspace(4) %implicitarg.ptr
+  %dispatch.load = load volatile i8, ptr addrspace(4) %dispatch.ptr
+  store volatile i64 %dispatch.id, ptr addrspace(1) %ptr
+  ret void
+}
+
+; Taken from memcpy-param-combinations.ll, tests PTRADD handling in
+; SelectionDAGAddressAnalysis.
+define void @memcpy_p1_p4_sz16_align_1_1(ptr addrspace(1) align 1 %dst, ptr addrspace(4) align 1 readonly %src) {
+; GFX942-LABEL: memcpy_p1_p4_sz16_align_1_1:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noundef nonnull align 1 %dst, ptr addrspace(4) noundef nonnull align 1 %src, i64 16, i1 false)
+  ret void
+}
+
+; Test skipping the lower-32-bit addition if it is unnecessary.
+define ptr @huge_offset_low_32_unused(ptr %p) {
+; GFX942_PTRADD-LABEL: huge_offset_low_32_unused:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    s_mov_b32 s0, 0
+; GFX942_PTRADD-NEXT:    s_mov_b32 s1, 1
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: huge_offset_low_32_unused:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_add_u32_e32 v1, 1, v1
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %gep = getelementptr inbounds i8, ptr %p, i64 u0x100000000
+  ret ptr %gep
+}
+
+; Reassociate address computation if it leads to more scalar operations.
+define amdgpu_kernel void @reassoc_scalar_r(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
+; GFX942_PTRADD-LABEL: reassoc_scalar_r:
 ; GFX942_PTRADD:       ; %bb.0: ; %entry
-; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    s_add_u32 s2, s0, 32
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v0, s20
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, s21
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, s22
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v3, s23
-; GFX942_PTRADD-NEXT:    s_addc_u32 s3, s1, 0
-; GFX942_PTRADD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v0, s16
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, s17
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, s18
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v3, s19
-; GFX942_PTRADD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v0, s12
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, s13
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, s14
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v3, s15
-; GFX942_PTRADD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX942_PTRADD-NEXT:    s_nop 1
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX942_PTRADD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], v[0:1], 0, s[6:7]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
 ; GFX942_PTRADD-NEXT:    s_endpgm
 ;
-; GFX942_LEGACY-LABEL: store_v16i32:
+; GFX942_LEGACY-LABEL: reassoc_scalar_r:
 ; GFX942_LEGACY:       ; %bb.0: ; %entry
-; GFX942_LEGACY-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
-; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v0, s20
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, s21
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, s22
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v3, s23
-; GFX942_LEGACY-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v0, s16
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, s17
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, s18
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v3, s19
-; GFX942_LEGACY-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v0, s12
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, s13
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, s14
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v3, s15
-; GFX942_LEGACY-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX942_LEGACY-NEXT:    s_nop 1
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v0, s8
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, s9
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, s10
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v3, s11
-; GFX942_LEGACY-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
+; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
 ; GFX942_LEGACY-NEXT:    s_endpgm
 entry:
-  store <16 x i32> %a, ptr addrspace(1) %out
+  %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
+  %voffset = zext i32 %voffset32 to i64
+  %offset = add nuw nsw i64 %voffset, %soffset
+  %gep = getelementptr i8, ptr addrspace(1) %p, i64 %offset
+  store ptr addrspace(1) %gep, ptr addrspace(1) %out
   ret void
 }
 
+define amdgpu_kernel void @reassoc_scalar_l(ptr addrspace(1) %out, ptr addrspace(1) %p, i64 %soffset) {
+; GFX942_PTRADD-LABEL: reassoc_scalar_l:
+; GFX942_PTRADD:       ; %bb.0: ; %entry
+; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942_PTRADD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[6:7], 0, v[0:1]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[2:3]
+; GFX942_PTRADD-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942_PTRADD-NEXT:    s_endpgm
+;
+; GFX942_LEGACY-LABEL: reassoc_scalar_l:
+; GFX942_LEGACY:       ; %bb.0: ; %entry
+; GFX942_LEGACY-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    s_add_u32 s2, s2, s6
+; GFX942_LEGACY-NEXT:    s_addc_u32 s3, s3, s7
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]
+; GFX942_LEGACY-NEXT:    global_store_dwordx2 v1, v[2:3], s[0:1]
+; GFX942_LEGACY-NEXT:    s_endpgm
+entry:
+  %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
+  %voffset = zext i32 %voffset32 to i64
+  %offset = add nuw nsw i64 %soffset, %voffset
+  %gep = getelementptr i8, ptr addrspace(1) %p, i64 %offset
+  store ptr addrspace(1) %gep, ptr addrspace(1) %out
+  ret void
+}
 
-; Tests the (ptradd 0, x) -> x DAG combine.
-define void @baseptr_null(i64 %offset, i8 %v) {
-; GFX942_PTRADD-LABEL: baseptr_null:
+; Tests the target-specific (ptradd x, shl(0 - y, k)) -> sub(x, shl(y, k)) fold
+define ptr addrspace(1) @shl_neg_offset(ptr addrspace(1) %p, i64 %noffset, i64 %shift) {
+; GFX942_PTRADD-LABEL: shl_neg_offset:
 ; GFX942_PTRADD:       ; %bb.0:
 ; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], 0, 0, v[0:1]
-; GFX942_PTRADD-NEXT:    flat_store_byte v[0:1], v2
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    v_sub_co_u32_e32 v2, vcc, 0, v2
+; GFX942_PTRADD-NEXT:    s_nop 1
+; GFX942_PTRADD-NEXT:    v_subb_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX942_PTRADD-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
 ; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX942_LEGACY-LABEL: baseptr_null:
+; GFX942_LEGACY-LABEL: shl_neg_offset:
 ; GFX942_LEGACY:       ; %bb.0:
 ; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    flat_store_byte v[0:1], v2
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX942_LEGACY-NEXT:    v_sub_co_u32_e32 v0, vcc, v0, v2
+; GFX942_LEGACY-NEXT:    s_nop 1
+; GFX942_LEGACY-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
 ; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr null, i64 %offset
-  store i8 %v, ptr %gep, align 1
-  ret void
+  %offset = sub i64 0, %noffset
+  %x = shl i64 %offset, %shift
+  %gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 %x
+  ret ptr addrspace(1) %gep
 }
 
-; Taken from implicit-kernarg-backend-usage.ll, tests the PTRADD handling in the
-; assertalign DAG combine.
-define amdgpu_kernel void @llvm_amdgcn_queue_ptr(ptr addrspace(1) %ptr)  #0 {
-; GFX942_PTRADD-LABEL: llvm_amdgcn_queue_ptr:
+%complextype = type { i64, [10 x i8], float }
+
+@v0 = dso_local addrspace(1) global %complextype zeroinitializer
+
+; Check that offsets are folded into global addresses if possible. For example,
+; this is relevant when using --amdgpu-lower-module-lds-strategy=table.
+define ptr addrspace(1) @complextype_global_gep(i64 %offset) {
+; GFX942_PTRADD-LABEL: complextype_global_gep:
+; GFX942_PTRADD:       ; %bb.0:
+; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_PTRADD-NEXT:    s_getpc_b64 s[0:1]
+; GFX942_PTRADD-NEXT:    s_add_u32 s0, s0, v0@rel32@lo+4
+; GFX942_PTRADD-NEXT:    s_addc_u32 s1, s1, v0@rel32@hi+12
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, 10
+; GFX942_PTRADD-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942_LEGACY-LABEL: complextype_global_gep:
+; GFX942_LEGACY:       ; %bb.0:
+; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942_LEGACY-NEXT:    s_getpc_b64 s[0:1]
+; GFX942_LEGACY-NEXT:    s_add_u32 s0, s0, v0@rel32@lo+14
+; GFX942_LEGACY-NEXT:    s_addc_u32 s1, s1, v0@rel32@hi+22
+; GFX942_LEGACY-NEXT:    v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
+; GFX942_LEGACY-NEXT:    s_setpc_b64 s[30:31]
+  %gep0 = getelementptr inbounds %complextype, ptr addrspace(1) @v0, i64 0, i32 1, i64 %offset
+  %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 2
+  ret ptr addrspace(1) %gep1
+}
+
+%S = type <{ float, double }>
+
+; Tests the tryFoldToMad64_32 PTRADD combine.
+define amdgpu_kernel void @fold_mad64(ptr addrspace(1) %p) {
+; GFX942_PTRADD-LABEL: fold_mad64:
 ; GFX942_PTRADD:       ; %bb.0:
-; GFX942_PTRADD-NEXT:    s_add_u32 s8, s4, 8
-; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX942_PTRADD-NEXT:    s_addc_u32 s9, s5, 0
-; GFX942_PTRADD-NEXT:    global_load_ubyte v0, v2, s[2:3] sc0 sc1
-; GFX942_PTRADD-NEXT:    global_load_ubyte v0, v2, s[8:9] sc0 sc1
-; GFX942_PTRADD-NEXT:    global_load_ubyte v0, v2, s[0:1] sc0 sc1
-; GFX942_PTRADD-NEXT:    ; kill: killed $sgpr0_sgpr1
 ; GFX942_PTRADD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0)
-; GFX942_PTRADD-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
-; GFX942_PTRADD-NEXT:    ; kill: killed $sgpr8 killed $sgpr9
-; GFX942_PTRADD-NEXT:    ; kill: killed $sgpr2_sgpr3
+; GFX942_PTRADD-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942_PTRADD-NEXT:    v_mul_hi_u32_u24_e32 v1, 12, v0
+; GFX942_PTRADD-NEXT:    v_mul_u32_u24_e32 v0, 12, v0
+; GFX942_PTRADD-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GFX942_PTRADD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_PTRADD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX942_PTRADD-NEXT:    s_waitcnt vmcnt(0)
+; GFX942_PTRADD-NEXT:    v_lshl_add_u64 v[0:1], s[0:1], 0, v[0:1]
+; GFX942_PTRADD-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX942_PTRADD-NEXT:    s_endpgm
 ;
-; GFX942_LEGACY-LABEL: llvm_amdgcn_queue_ptr:
+; GFX942_LEGACY-LABEL: fold_mad64:
 ; GFX942_LEGACY:       ; %bb.0:
-; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, 0
-; GFX942_LEGACY-NEXT:    global_load_ubyte v0, v2, s[2:3] sc0 sc1
-; GFX942_LEGACY-NEXT:    global_load_ubyte v0, v2, s[4:5] offset:8 sc0 sc1
-; GFX942_LEGACY-NEXT:    global_load_ubyte v0, v2, s[0:1] sc0 sc1
-; GFX942_LEGACY-NEXT:    ; kill: killed $sgpr0_sgpr1
 ; GFX942_LEGACY-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0)
-; GFX942_LEGACY-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]
-; GFX942_LEGACY-NEXT:    ; kill: killed $sgpr2_sgpr3
+; GFX942_LEGACY-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942_LEGACY-NEXT:    v_mov_b32_e32 v2, 1.0
 ; GFX942_LEGACY-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX942_LEGACY-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX942_LEGACY-NEXT:    s_waitcnt vmcnt(0)
+; GFX942_LEGACY-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v0, 12, s[0:1]
+; GFX942_LEGACY-NEXT:    global_store_dword v[0:1], v2, off
 ; GFX942_LEGACY-NEXT:    s_endpgm
-  %queue.ptr = call ptr addrspace(4) @llvm.amdgcn.queue.ptr()
-  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
-  %dispatch.ptr = call ptr addrspace(4) @llvm.amdgcn.dispatch.ptr()
-  %dispatch.id = call i64 @llvm.amdgcn.dispatch.id()
-  %queue.load = load volatile i8, ptr addrspace(4) %queue.ptr
-  %implicitarg.load = load volatile i8, ptr addrspace(4) %implicitarg.ptr
-  %dispatch.load = load volatile i8, ptr addrspace(4) %dispatch.ptr
-  store volatile i64 %dispatch.id, ptr addrspace(1) %ptr
+  %voffset32 = call i32 @llvm.amdgcn.workitem.id.x()
+  %voffset = zext i32 %voffset32 to i64
+  %p1 = getelementptr inbounds %S, ptr addrspace(1) %p, i64 %voffset, i32 0
+  store float 1.0, ptr addrspace(1) %p1
   ret void
 }
-
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX942: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-undef-poison.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-undef-poison.ll
new file mode 100644
index 000000000000..1934ce395e63
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-undef-poison.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -start-before=amdgpu-isel -amdgpu-use-sdag-ptradd=1 < %s | FileCheck --check-prefixes=GFX942,GFX942_PTRADD %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -start-before=amdgpu-isel -amdgpu-use-sdag-ptradd=0 < %s | FileCheck --check-prefixes=GFX942,GFX942_LEGACY %s
+
+; Tests for undef and poison DAG folds for the ISD::PTRADD SelectionDAG opcode.
+; If any additions are generated for these tests, the folds don't work.
+
+define ptr @poison_offset(ptr %p, i64 %offset) {
+; GFX942-LABEL: poison_offset:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %gep1 = getelementptr i8, ptr %p, i64 poison
+  ret ptr %gep1
+}
+
+define ptr @poison_base(ptr %p, i64 %offset) {
+; GFX942-LABEL: poison_base:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %gep1 = getelementptr i8, ptr poison, i64 %offset
+  ret ptr %gep1
+}
+
+define ptr @undef_offset(ptr %p, i64 %offset) {
+; GFX942-LABEL: undef_offset:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %gep1 = getelementptr i8, ptr %p, i64 undef
+  ret ptr %gep1
+}
+
+define ptr @undef_base(ptr %p, i64 %offset) {
+; GFX942-LABEL: undef_base:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %gep1 = getelementptr i8, ptr undef, i64 %offset
+  ret ptr %gep1
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX942_LEGACY: {{.*}}
+; GFX942_PTRADD: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll
index 653d4b85a9a5..32f2395c7b2a 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag.ll
@@ -70,10 +70,11 @@ define ptr @gep_as0(ptr %p, i64 %offset) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
@@ -232,10 +233,11 @@ define ptr @multi_gep_as0(ptr %p, i64 %offset) {
 ; GFX12-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 5
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 6512bee36e88..ba9dd8f7c246 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -513,21 +513,28 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -538,6 +545,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -1073,10 +1081,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
@@ -1889,21 +1897,28 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v5, v8
-; GFX9-O0-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; implicit-def: $sgpr8
 ; GFX9-O0-NEXT:    ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v8, v6
-; GFX9-O0-NEXT:    buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v8
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v7
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_nop 0
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    v_mov_b32_e32 v10, v5
+; GFX9-O0-NEXT:    v_mov_b32_e32 v9, v4
+; GFX9-O0-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
 ; GFX9-O0-NEXT:    s_nop 0
-; GFX9-O0-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
-; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
+; GFX9-O0-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_eq_u64_e64 s[8:9], v[7:8], s[8:9]
 ; GFX9-O0-NEXT:    s_mov_b64 s[12:13], 0x7f
-; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[12:13]
+; GFX9-O0-NEXT:    v_cmp_gt_u64_e64 s[14:15], v[4:5], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v9, 0, 1, s[14:15]
-; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
+; GFX9-O0-NEXT:    s_mov_b64 s[14:15], s[6:7]
+; GFX9-O0-NEXT:    v_cmp_ne_u64_e64 s[14:15], v[7:8], s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[14:15]
 ; GFX9-O0-NEXT:    v_cndmask_b32_e64 v6, v6, v9, s[8:9]
 ; GFX9-O0-NEXT:    v_and_b32_e64 v6, 1, v6
@@ -1914,6 +1929,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    v_mov_b32_e32 v6, v5
 ; GFX9-O0-NEXT:    s_mov_b32 s14, s13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v6, v6, s14
+; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
 ; GFX9-O0-NEXT:    ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
 ; GFX9-O0-NEXT:    v_xor_b32_e64 v4, v4, s12
 ; GFX9-O0-NEXT:    ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
@@ -2449,10 +2465,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
 ; GFX9-O0-NEXT:    buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    buffer_load_dword v11, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GFX9-O0-NEXT:    buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
 ; GFX9-O0-NEXT:    s_mov_b64 s[6:7], 1
 ; GFX9-O0-NEXT:    s_mov_b32 s5, s6
 ; GFX9-O0-NEXT:    s_waitcnt vmcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
new file mode 100644
index 000000000000..0c6339e4f512
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mcpu=gfx90a < %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v0, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v0, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v0, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v0, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v0, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v0, s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v0, s[0:1] offset:16
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v0, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v1
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v2
+; CHECK-NEXT:    v_accvgpr_write_b32 a3, v3
+; CHECK-NEXT:    v_accvgpr_write_b32 a4, v4
+; CHECK-NEXT:    v_accvgpr_write_b32 a5, v5
+; CHECK-NEXT:    v_accvgpr_write_b32 a6, v6
+; CHECK-NEXT:    v_accvgpr_write_b32 a7, v7
+; CHECK-NEXT:    v_accvgpr_write_b32 a8, v8
+; CHECK-NEXT:    v_accvgpr_write_b32 a9, v9
+; CHECK-NEXT:    v_accvgpr_write_b32 a10, v10
+; CHECK-NEXT:    v_accvgpr_write_b32 a11, v11
+; CHECK-NEXT:    v_accvgpr_write_b32 a12, v12
+; CHECK-NEXT:    v_accvgpr_write_b32 a13, v13
+; CHECK-NEXT:    v_accvgpr_write_b32 a14, v14
+; CHECK-NEXT:    v_accvgpr_write_b32 a15, v15
+; CHECK-NEXT:    v_accvgpr_write_b32 a16, v16
+; CHECK-NEXT:    v_accvgpr_write_b32 a17, v17
+; CHECK-NEXT:    v_accvgpr_write_b32 a18, v18
+; CHECK-NEXT:    v_accvgpr_write_b32 a19, v19
+; CHECK-NEXT:    v_accvgpr_write_b32 a20, v20
+; CHECK-NEXT:    v_accvgpr_write_b32 a21, v21
+; CHECK-NEXT:    v_accvgpr_write_b32 a22, v22
+; CHECK-NEXT:    v_accvgpr_write_b32 a23, v23
+; CHECK-NEXT:    v_accvgpr_write_b32 a24, v24
+; CHECK-NEXT:    v_accvgpr_write_b32 a25, v25
+; CHECK-NEXT:    v_accvgpr_write_b32 a26, v26
+; CHECK-NEXT:    v_accvgpr_write_b32 a27, v27
+; CHECK-NEXT:    v_accvgpr_write_b32 a28, v28
+; CHECK-NEXT:    v_accvgpr_write_b32 a29, v29
+; CHECK-NEXT:    v_accvgpr_write_b32 a30, v30
+; CHECK-NEXT:    v_accvgpr_write_b32 a31, v31
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1.0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[32:63], v0, v1, a[0:31]
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 2
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a59
+; CHECK-NEXT:    v_accvgpr_read_b32 v5, a58
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a57
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a56
+; CHECK-NEXT:    v_accvgpr_read_b32 v8, a55
+; CHECK-NEXT:    v_accvgpr_read_b32 v9, a54
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a53
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a52
+; CHECK-NEXT:    v_accvgpr_read_b32 v12, a51
+; CHECK-NEXT:    v_accvgpr_read_b32 v13, a50
+; CHECK-NEXT:    v_accvgpr_read_b32 v14, a49
+; CHECK-NEXT:    v_accvgpr_read_b32 v15, a48
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a47
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a46
+; CHECK-NEXT:    v_accvgpr_read_b32 v18, a45
+; CHECK-NEXT:    v_accvgpr_read_b32 v19, a44
+; CHECK-NEXT:    v_accvgpr_read_b32 v20, a43
+; CHECK-NEXT:    v_accvgpr_read_b32 v21, a42
+; CHECK-NEXT:    v_accvgpr_read_b32 v22, a41
+; CHECK-NEXT:    v_accvgpr_read_b32 v23, a40
+; CHECK-NEXT:    v_accvgpr_read_b32 v24, a39
+; CHECK-NEXT:    v_accvgpr_read_b32 v25, a38
+; CHECK-NEXT:    v_accvgpr_read_b32 v26, a37
+; CHECK-NEXT:    v_accvgpr_read_b32 v27, a36
+; CHECK-NEXT:    v_accvgpr_read_b32 v28, a35
+; CHECK-NEXT:    v_accvgpr_read_b32 v29, a34
+; CHECK-NEXT:    v_accvgpr_mov_b32 a2, a32
+; CHECK-NEXT:    v_accvgpr_mov_b32 a3, a33
+; CHECK-NEXT:    v_accvgpr_write_b32 a4, v29
+; CHECK-NEXT:    v_accvgpr_write_b32 a5, v28
+; CHECK-NEXT:    v_accvgpr_write_b32 a6, v27
+; CHECK-NEXT:    v_accvgpr_write_b32 a7, v26
+; CHECK-NEXT:    v_accvgpr_write_b32 a8, v25
+; CHECK-NEXT:    v_accvgpr_write_b32 a9, v24
+; CHECK-NEXT:    v_accvgpr_write_b32 a10, v23
+; CHECK-NEXT:    v_accvgpr_write_b32 a11, v22
+; CHECK-NEXT:    v_accvgpr_write_b32 a12, v21
+; CHECK-NEXT:    v_accvgpr_write_b32 a13, v20
+; CHECK-NEXT:    v_accvgpr_write_b32 a14, v19
+; CHECK-NEXT:    v_accvgpr_write_b32 a15, v18
+; CHECK-NEXT:    v_accvgpr_write_b32 a16, v17
+; CHECK-NEXT:    v_accvgpr_write_b32 a17, v16
+; CHECK-NEXT:    v_accvgpr_write_b32 a18, v15
+; CHECK-NEXT:    v_accvgpr_write_b32 a19, v14
+; CHECK-NEXT:    v_accvgpr_write_b32 a20, v13
+; CHECK-NEXT:    v_accvgpr_write_b32 a21, v12
+; CHECK-NEXT:    v_accvgpr_write_b32 a22, v11
+; CHECK-NEXT:    v_accvgpr_write_b32 a23, v10
+; CHECK-NEXT:    v_accvgpr_write_b32 a24, v9
+; CHECK-NEXT:    v_accvgpr_write_b32 a25, v8
+; CHECK-NEXT:    v_accvgpr_write_b32 a26, v7
+; CHECK-NEXT:    v_accvgpr_write_b32 a27, v6
+; CHECK-NEXT:    v_accvgpr_write_b32 a28, v5
+; CHECK-NEXT:    v_accvgpr_write_b32 a29, v4
+; CHECK-NEXT:    v_accvgpr_mov_b32 a30, a60
+; CHECK-NEXT:    v_accvgpr_mov_b32 a31, a61
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    global_store_dwordx4 v0, a[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v0, a[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v0, a[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v0, a[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
+; CHECK-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %tmp.1 = shufflevector <32 x float> %mai.2, <32 x float> %mai.1, <32 x i32> <i32 32, i32 33, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29>
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %tmp.1, i32 0, i32 0, i32 0)
+  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  ret void
+}
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v0, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v0, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v0, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v0, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v0, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v0, s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v0, s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v0, s[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1.0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    global_store_dwordx4 v0, a[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v0, a[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v0, a[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v0, a[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
+; CHECK-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.2, i32 0, i32 0, i32 0)
+  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  ret void
+}
+
+declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) #1
+declare noundef i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,4" }
+attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
new file mode 100644
index 000000000000..7f822c135ffb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
@@ -0,0 +1,1956 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,SI-DAZ-SAFE %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,SI-IEEE-SAFE %s
+
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,CI-DAZ-SAFE %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,CI-IEEE-SAFE %s
+
+
+declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+declare float @llvm.sqrt.f32(float) nounwind readnone
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float>) nounwind readnone
+
+define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %sqrt = call contract float @llvm.sqrt.f32(float %val) nounwind readnone
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
+; GCN-DAZ-SAFE-LABEL: rsq_f32_sgpr:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, -1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %sqrt = call contract float @llvm.sqrt.f32(float %val) nounwind readnone
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; Recognize that this is rsqrt(a) * rcp(b) * c,
+; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
+
+; NOTE: c * rcp( sqrt(a) * b ) is generated when we move rcp generation to AMGGPUCogenPrepare.
+define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GCN-DAZ-SAFE-LABEL: rsqrt_fmul:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0
+; GCN-DAZ-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v2, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v5, v7, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v7, v8, v7
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v8, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v7, v7, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v8, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v3, s[4:5], v2, v2, v4
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
+; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
+; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
+; GCN-DAZ-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
+; GCN-DAZ-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; GCN-IEEE-SAFE-LABEL: rsqrt_fmul:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0
+; GCN-IEEE-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
+; GCN-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[0:1], -1, v5
+; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v8, s[0:1], 1, v5
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v9, -v7, v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v10, -v8, v5, v2
+; GCN-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0, v9
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
+; GCN-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v10
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GCN-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v3, s[0:1], v2, v2, v4
+; GCN-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
+; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
+; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
+; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
+; GCN-IEEE-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
+; GCN-IEEE-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
+; GCN-IEEE-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-IEEE-SAFE-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+  %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
+  %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
+  %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
+  %gep.2 = getelementptr float, ptr addrspace(1) %gep.0, i32 2
+
+  %a = load volatile float, ptr addrspace(1) %gep.0
+  %b = load volatile float, ptr addrspace(1) %gep.1
+  %c = load volatile float, ptr addrspace(1) %gep.2
+
+  %x = call contract float @llvm.sqrt.f32(float %a)
+  %y = fmul contract float %x, %b
+  %z = fdiv contract float %c, %y
+  store float %z, ptr addrspace(1) %out.gep
+  ret void
+}
+
+define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: neg_rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: neg_rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: neg_rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %sqrt = call contract float @llvm.sqrt.f32(float %val)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
+; GCN-DAZ-SAFE-LABEL: neg_rsq_neg_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-SAFE-NEXT:    s_endpgm
+;
+; SI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-IEEE-SAFE-NEXT:    s_endpgm
+;
+; CI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
+; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
+; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CI-IEEE-SAFE-NEXT:    s_endpgm
+  %val = load float, ptr addrspace(1) %in, align 4
+  %val.fneg = fneg float %val
+  %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  store float %div, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define float @v_neg_rsq_neg_f32(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val.fneg = fneg float %val
+  %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  ret float %div
+}
+
+define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v3, -v0, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val.fneg = fneg <2 x float> %val
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  ret <2 x float> %div
+}
+
+define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.neg = fneg float %val0
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0.neg)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  %user = fmul contract float %div, %val1
+  ret float %user
+}
+
+define <2 x float> @v_neg_rsq_neg_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v5, -v0, s5
+; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.fneg = fneg <2 x float> %val0
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_neg_rsq_f32(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  ret float %div
+}
+
+define <2 x float> @v_neg_rsq_v2f32(<2 x float> %val) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  ret <2 x float> %div
+}
+
+define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0)
+  %div = fdiv contract float -1.0, %sqrt, !fpmath !0
+  %user = fmul contract float %div, %val1
+  ret float %user
+}
+
+define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
+; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v0
+; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
+; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
+; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
+; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_rsq_f32(float %val) {
+; GCN-DAZ-LABEL: v_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define { float, float } @v_rsq_f32_multi_use(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_multi_use:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v1, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v2, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v2, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %insert.0 = insertvalue { float, float } poison, float %sqrt, 0
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %insert.1 = insertvalue { float, float } %insert.0, float %div, 1
+  ret { float, float } %insert.1
+}
+
+define float @v_rsq_f32_missing_contract0(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define float @v_rsq_f32_missing_contract1(float %val) {
+; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; GCN-DAZ-SAFE:       ; %bb.0:
+; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
+; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; SI-IEEE-SAFE:       ; %bb.0:
+; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
+; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
+; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
+; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+;
+; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
+; CI-IEEE-SAFE:       ; %bb.0:
+; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
+; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
+; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
+; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
+; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
+; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
+; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
+; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
+; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
+; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
+; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
+; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+; Test that we contract into FMA for an fadd user after introducing
+; the fmul.
+define float @v_rsq_f32_contractable_user(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd contract float %div, %val1
+  ret float %add
+}
+
+; Missing contract on the fdiv
+define float @v_rsq_f32_contractable_user_missing_contract0(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd contract float %div, %val1
+  ret float %add
+}
+
+; Missing contract on the fadd
+define float @v_rsq_f32_contractable_user_missing_contract1(float %val0, float %val1) {
+; GCN-DAZ-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
+; GCN-IEEE-SAFE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %add = fadd float %div, %val1
+  ret float %add
+}
+
+define float @v_rsq_f32_known_never_denormal(float nofpclass(sub) %val) {
+; GCN-DAZ-LABEL: v_rsq_f32_known_never_denormal:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_rsq_f32_known_never_denormal:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
+; GCN-DAZ-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-SAFE-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-IEEE-SAFE:       ; %bb.0:
+; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
+; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
+; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
+; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+  %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
+  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  ret float %div
+}
+
+!0 = !{float 2.500000e+00}
+!1 = !{float 1.000000e+00}
+
+attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CI-DAZ-SAFE: {{.*}}
+; SI-DAZ-SAFE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
index f4b947ade8da..f7e038856110 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
@@ -2,175 +2,51 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-UNSAFE,SI-DAZ-UNSAFE %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -enable-unsafe-fp-math < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-UNSAFE,SI-IEEE-UNSAFE %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,SI-DAZ-SAFE %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,SI-IEEE-SAFE %s
-
 
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign -enable-unsafe-fp-math < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-UNSAFE,CI-DAZ-UNSAFE %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee -enable-unsafe-fp-math < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-UNSAFE,CI-IEEE-UNSAFE %s
 
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GCN-DAZ,GCN-DAZ-SAFE,CI-DAZ-SAFE %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=hawaii -denormal-fp-math-f32=ieee < %s          | FileCheck -check-prefixes=GCN-IEEE,GCN-IEEE-SAFE,CI-IEEE-SAFE %s
-
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare float @llvm.sqrt.f32(float) nounwind readnone
 declare <2 x float> @llvm.sqrt.v2f32(<2 x float>) nounwind readnone
 
 define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
 ;
-; CI-IEEE-SAFE-LABEL: rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-IEEE-LABEL: rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: rsq_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -196,131 +72,27 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(
 }
 
 define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
-; GCN-DAZ-UNSAFE-LABEL: rsq_f32_sgpr:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dword s2, s[4:5], 0xb
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsq_f32_sgpr:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dword s2, s[4:5], 0xb
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsq_f32_sgpr:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, -1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
-;
-; CI-IEEE-SAFE-LABEL: rsq_f32_sgpr:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dword s0, s[4:5], 0xb
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, s0, v1
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, s0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v2, vcc, -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v1, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsq_f32_sgpr:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dword s2, s[4:5], 0xb
+; GCN-DAZ-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s2, -1
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: rsq_f32_sgpr:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dword s2, s[4:5], 0xb
+; GCN-IEEE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s2, -1
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: rsq_f32_sgpr:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dword s2, s[0:1], 0xb
@@ -365,154 +137,53 @@ define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; GCN-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
 ; GCN-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
 ; GCN-UNSAFE-NEXT:    s_endpgm
-; GCN-DAZ-UNSAFE-LABEL: rsqrt_fmul:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, 0
-; GCN-DAZ-UNSAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[8:9], s[2:3]
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b64 s[4:5], s[0:1]
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v2, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_rcp_f32_e32 v3, v3
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: rsqrt_fmul:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, 0
-; GCN-IEEE-UNSAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[8:9], s[2:3]
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b64 s[4:5], s[0:1]
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v2, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_rcp_f32_e32 v3, v3
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e32 v2, v4, v2
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: rsqrt_fmul:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0
-; GCN-DAZ-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v2, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v5, v7, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v7, v8, v7
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v8, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v7, v7, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v8, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v3, s[4:5], v2, v2, v4
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
-; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
-; GCN-DAZ-SAFE-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GCN-DAZ-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
-; GCN-DAZ-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-SAFE-LABEL: rsqrt_fmul:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s3, 0xf000
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0
-; GCN-IEEE-SAFE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[8:9], s[6:7]
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[10:11], s[2:3]
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x4f800000, v2
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s0, v2
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v5, vcc
-; GCN-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[0:1], -1, v5
-; GCN-IEEE-SAFE-NEXT:    v_add_i32_e64 v8, s[0:1], 1, v5
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v9, -v7, v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v10, -v8, v5, v2
-; GCN-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0, v9
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v7, s[0:1]
-; GCN-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], 0, v10
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v5, v8, s[0:1]
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, 0x37800000, v5
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; GCN-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v2, v6
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v5, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, v2, v3
-; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v3, s[0:1], v2, v2, v4
-; GCN-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v5, v3
-; GCN-IEEE-SAFE-NEXT:    v_div_scale_f32 v6, vcc, v4, v2, v4
-; GCN-IEEE-SAFE-NEXT:    s_mov_b64 s[0:1], s[4:5]
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v3, v5, 1.0
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v5, v7, v5, v5
-; GCN-IEEE-SAFE-NEXT:    v_mul_f32_e32 v7, v6, v5
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v8, -v3, v7, v6
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v7, v8, v5, v7
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v3, v7, v6
-; GCN-IEEE-SAFE-NEXT:    v_div_fmas_f32 v3, v3, v5, v7
-; GCN-IEEE-SAFE-NEXT:    v_div_fixup_f32 v2, v3, v2, v4
-; GCN-IEEE-SAFE-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; GCN-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: rsqrt_fmul:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, 0
+; GCN-DAZ-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-DAZ-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; GCN-DAZ-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GCN-DAZ-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v2, v2
+; GCN-DAZ-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-DAZ-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-DAZ-NEXT:    v_mul_f32_e32 v2, v4, v2
+; GCN-DAZ-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: rsqrt_fmul:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, 0
+; GCN-IEEE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b64 s[8:9], s[2:3]
+; GCN-IEEE-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GCN-IEEE-NEXT:    buffer_load_dword v2, v[0:1], s[8:11], 0 addr64 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:4 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:8 glc
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v2, v2
+; GCN-IEEE-NEXT:    v_rcp_f32_e32 v3, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v2, v2, v3
+; GCN-IEEE-NEXT:    v_mul_f32_e32 v2, v4, v2
+; GCN-IEEE-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; GCN-IEEE-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %gep.0 = getelementptr float, ptr addrspace(1) %in, i32 %tid
@@ -525,167 +196,49 @@ define amdgpu_kernel void @rsqrt_fmul(ptr addrspace(1) %out, ptr addrspace(1) %i
 
   %x = call contract float @llvm.sqrt.f32(float %a)
   %y = fmul contract float %x, %b
-  %z = fdiv contract float %c, %y
+  %z = fdiv arcp contract float %c, %y
   store float %z, ptr addrspace(1) %out.gep
   ret void
 }
 
 define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: neg_rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: neg_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: neg_rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: neg_rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: neg_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
 ;
-; CI-IEEE-SAFE-LABEL: neg_rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-IEEE-LABEL: neg_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: neg_rsq_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -712,161 +265,43 @@ define amdgpu_kernel void @neg_rsq_f32(ptr addrspace(1) noalias %out, ptr addrsp
 }
 
 define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) {
-; GCN-DAZ-UNSAFE-LABEL: neg_rsq_neg_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-IEEE-UNSAFE-LABEL: neg_rsq_neg_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-IEEE-UNSAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-IEEE-UNSAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-IEEE-UNSAFE-NEXT:    s_endpgm
-;
-; GCN-DAZ-SAFE-LABEL: neg_rsq_neg_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s6, -1
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s10, s6
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s11, s7
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s8, s2
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s9, s3
-; GCN-DAZ-SAFE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s2, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, s0
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, s1
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s2, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v1, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GCN-DAZ-SAFE-NEXT:    s_endpgm
-;
-; SI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; SI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; SI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s2, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], |v0|, s2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[0:1]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-IEEE-SAFE-NEXT:    s_endpgm
-;
-; CI-IEEE-SAFE-LABEL: neg_rsq_neg_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0xf000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, -1
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s2, s6
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s3, s7
-; CI-IEEE-SAFE-NEXT:    s_waitcnt lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, s10
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s1, s11
-; CI-IEEE-SAFE-NEXT:    buffer_load_dword v0, off, s[0:3], 0
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s0, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v1, 0x260
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, s8
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s5, s9
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0)
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[0:1], s0, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v3, vcc, -1, v2
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e32 v4, vcc, 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v3, s[0:1]
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; CI-IEEE-SAFE-NEXT:    s_endpgm
+; GCN-DAZ-LABEL: neg_rsq_neg_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-DAZ-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-DAZ-NEXT:    s_mov_b32 s6, -1
+; GCN-DAZ-NEXT:    s_mov_b32 s10, s6
+; GCN-DAZ-NEXT:    s_mov_b32 s11, s7
+; GCN-DAZ-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT:    s_mov_b32 s8, s2
+; GCN-DAZ-NEXT:    s_mov_b32 s9, s3
+; GCN-DAZ-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-DAZ-NEXT:    s_mov_b32 s4, s0
+; GCN-DAZ-NEXT:    s_mov_b32 s5, s1
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-DAZ-NEXT:    s_endpgm
+;
+; GCN-IEEE-LABEL: neg_rsq_neg_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-IEEE-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT:    s_mov_b32 s6, -1
+; GCN-IEEE-NEXT:    s_mov_b32 s10, s6
+; GCN-IEEE-NEXT:    s_mov_b32 s11, s7
+; GCN-IEEE-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT:    s_mov_b32 s8, s2
+; GCN-IEEE-NEXT:    s_mov_b32 s9, s3
+; GCN-IEEE-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GCN-IEEE-NEXT:    s_mov_b32 s4, s0
+; GCN-IEEE-NEXT:    s_mov_b32 s5, s1
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-IEEE-NEXT:    s_endpgm
 ; GCN-UNSAFE-LABEL: neg_rsq_neg_f32:
 ; GCN-UNSAFE:       ; %bb.0:
 ; GCN-UNSAFE-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
@@ -894,101 +329,19 @@ define amdgpu_kernel void @neg_rsq_neg_f32(ptr addrspace(1) noalias %out, ptr ad
 }
 
 define float @v_neg_rsq_neg_f32(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val.fneg = fneg float %val
   %sqrt = call contract float @llvm.sqrt.f32(float %val.fneg)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
@@ -996,168 +349,23 @@ define float @v_neg_rsq_neg_f32(float %val) {
 }
 
 define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v3, -v0, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v2, -v1, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v0, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_v2f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val.fneg = fneg <2 x float> %val
   %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val.fneg)
   %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
@@ -1165,104 +373,19 @@ define <2 x float> @v_neg_rsq_neg_v2f32(<2 x float> %val) {
 }
 
 define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_neg_f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %val0.neg = fneg float %val0
   %sqrt = call contract float @llvm.sqrt.f32(float %val0.neg)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
@@ -1271,546 +394,86 @@ define float @v_neg_rsq_neg_f32_foldable_user(float %val0, float %val1) {
 }
 
 define <2 x float> @v_neg_rsq_neg_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v0, -v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e64 v1, -v1
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0x8f800000
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s5, 0x4f800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v4, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, 0.5, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v4, v5, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v6, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v6, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v7, v4, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e64 v5, -v0, s5
-; GCN-DAZ-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v5, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v6, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v4, v0, v5
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0.5, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v7, -v5, v4, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v4, v7, v4
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v8, -v4, v4, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, v5, v7, v5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, v8, v5, v4
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v6
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v0, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, -v1, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x8f800000
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s7, 0x4f800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v4, -v1, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v1, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v5, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v5, v4, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v6, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v6, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v5, v6, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v5, 0x37800000, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v5, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e64 v6, -v0, s7
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v0, -v0, v6, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v6, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v5, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v4, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], -1, v6
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v7, -v4, v6, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v7
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v6, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v7, s[4:5], 1, v6
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v6, -v7, v6, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v6
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v4, v4, v7, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v6, 0x37800000, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v4, v4, v6, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v4, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v4, v4
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v4, v1
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-  %val0.fneg = fneg <2 x float> %val0
-  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
-  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
-  %user = fmul contract <2 x float> %div, %val1
-  ret <2 x float> %user
-}
-
-define float @v_neg_rsq_f32(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, 0.5, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, -v1, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v1, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v1, v4, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, -v0, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v1, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_neg_rsq_neg_v2f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v0, -v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e64 v1, -v1
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
+  %val0.fneg = fneg <2 x float> %val0
+  %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val0.fneg)
+  %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
+  %user = fmul contract <2 x float> %div, %val1
+  ret <2 x float> %user
+}
+
+define float @v_neg_rsq_f32(float %val) {
+; GCN-DAZ-LABEL: v_neg_rsq_f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
+;
+; GCN-IEEE-LABEL: v_neg_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
   ret float %div
 }
 
 define <2 x float> @v_neg_rsq_v2f32(<2 x float> %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v1
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v1
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v1, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v1
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v4, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0.5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v2, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v6, -v2, v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v5, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v6, v3, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v4
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v1, -v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v1|, s6
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_v2f32:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-DAZ-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s6, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v1
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v1
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s6, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v4, v0
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e64 s[4:5], v1, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v1, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v5, -v2, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v5
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v4, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v5, s[4:5], 1, v4
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v5, v4, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v4, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v1
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v1, vcc, 0, v1
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v2, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_v2f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
+; GCN-IEEE-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract <2 x float> @llvm.sqrt.v2f32(<2 x float> %val)
   %div = fdiv contract <2 x float> <float -1.0, float -1.0>, %sqrt, !fpmath !0
   ret <2 x float> %div
 }
 
 define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; GCN-DAZ-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rsq_f32_e32 v2, v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, v0, v2
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v2, 0.5, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v4, -v2, v3, 0.5
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v3, v3, v4, v3
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v5, -v3, v3, v0
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v2, v4, v2
-; GCN-DAZ-SAFE-NEXT:    v_fma_f32 v2, v5, v2, v3
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; GCN-DAZ-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; GCN-DAZ-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; GCN-DAZ-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e64 v0, -v0
-; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, -v0, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_neg_rsq_f32_foldable_user:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], -1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v4, -v3, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v4
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v3, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v4, s[4:5], 1, v2
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v2, -v4, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v3, v4, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v3, 0x37800000, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v3, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e64 v2, -v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v2, v2
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v2, v0
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v1
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_neg_rsq_f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0)
   %div = fdiv contract float -1.0, %sqrt, !fpmath !0
   %user = fmul contract float %div, %val1
@@ -1818,24 +481,23 @@ define float @v_neg_rsq_f32_foldable_user(float %val0, float %val1) {
 }
 
 define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float> %val1) {
-; GCN-DAZ-UNSAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v1
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v0, -v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    v_mul_f32_e64 v1, -v1, v3
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-DAZ-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_neg_rsq_v2f32_foldable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v1
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v0, -v0, v2
+; GCN-IEEE-NEXT:    v_mul_f32_e64 v1, -v1, v3
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1876,7 +538,6 @@ define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float>
 ; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
 ; GCN-DAZ-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1932,7 +593,6 @@ define <2 x float> @v_neg_rsq_v2f32_foldable_user(<2 x float> %val0, <2 x float>
 ; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v0, v0, v2
 ; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, v1, v3
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_neg_rsq_v2f32_foldable_user:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1996,12 +656,11 @@ define float @v_rsq_f32(float %val) {
 ; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
+; GCN-IEEE-LABEL: v_rsq_f32:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-IEEE-SAFE-LABEL: v_rsq_f32:
 ; GCN-IEEE-SAFE:       ; %bb.0:
 ; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2019,29 +678,27 @@ define float @v_rsq_f32(float %val) {
 }
 
 define { float, float } @v_rsq_f32_multi_use(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_multi_use:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-DAZ-UNSAFE-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_multi_use:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_sqrt_f32_e32 v2, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v1, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_multi_use:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_sqrt_f32_e32 v2, v0
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-DAZ-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_rsq_f32_multi_use:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_sqrt_f32_e32 v2, v0
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v1, v0
+; GCN-IEEE-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_rsq_f32_multi_use:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v1, v0
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2072,7 +729,6 @@ define { float, float } @v_rsq_f32_multi_use(float %val) {
 ; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v2, vcc, 0, v2
 ; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v1, v1, v2
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_rsq_f32_multi_use:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2102,31 +758,29 @@ define { float, float } @v_rsq_f32_multi_use(float %val) {
 ; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %insert.0 = insertvalue { float, float } poison, float %sqrt, 0
-  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp contract float 1.0, %sqrt, !fpmath !1
   %insert.1 = insertvalue { float, float } %insert.0, float %div, 1
   ret { float, float } %insert.1
 }
 
 define float @v_rsq_f32_missing_contract0(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_missing_contract0:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_missing_contract0:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_missing_contract0:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GCN-IEEE-LABEL: v_rsq_f32_missing_contract0:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
 ; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; GCN-DAZ-SAFE:       ; %bb.0:
 ; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
 ; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; SI-IEEE-SAFE:       ; %bb.0:
 ; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2157,7 +811,6 @@ define float @v_rsq_f32_missing_contract0(float %val) {
 ; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
 ; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
 ; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
 ; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract0:
 ; CI-IEEE-SAFE:       ; %bb.0:
 ; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2186,90 +839,24 @@ define float @v_rsq_f32_missing_contract0(float %val) {
 ; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
 ; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call float @llvm.sqrt.f32(float %val), !fpmath !1
-  %div = fdiv contract float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp contract float 1.0, %sqrt, !fpmath !1
   ret float %div
 }
 
 define float @v_rsq_f32_missing_contract1(float %val) {
-; GCN-DAZ-UNSAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-DAZ-UNSAFE:       ; %bb.0:
-; GCN-DAZ-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-DAZ-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-DAZ-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; GCN-DAZ-SAFE:       ; %bb.0:
-; GCN-DAZ-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-DAZ-SAFE-NEXT:    v_sqrt_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    v_rcp_f32_e32 v0, v0
-; GCN-DAZ-SAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; SI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; SI-IEEE-SAFE:       ; %bb.0:
-; SI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; SI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; SI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; SI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; SI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x7f800000
-; SI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; SI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 vcc, |v0|, s4
-; SI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; SI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; SI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; SI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; SI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-DAZ-LABEL: v_rsq_f32_missing_contract1:
+; GCN-DAZ:       ; %bb.0:
+; GCN-DAZ-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; CI-IEEE-SAFE-LABEL: v_rsq_f32_missing_contract1:
-; CI-IEEE-SAFE:       ; %bb.0:
-; CI-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0xf800000
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v1, 0x4f800000, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
-; CI-IEEE-SAFE-NEXT:    v_sqrt_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v2, s[4:5], -1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v3, -v2, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_ge_f32_e64 s[4:5], 0, v3
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, v1, v2, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_add_i32_e64 v3, s[4:5], 1, v1
-; CI-IEEE-SAFE-NEXT:    v_fma_f32 v1, -v3, v1, v0
-; CI-IEEE-SAFE-NEXT:    v_cmp_lt_f32_e64 s[4:5], 0, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, v2, v3, s[4:5]
-; CI-IEEE-SAFE-NEXT:    v_mul_f32_e32 v2, 0x37800000, v1
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
-; CI-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x260
-; CI-IEEE-SAFE-NEXT:    v_cmp_class_f32_e32 vcc, v0, v2
-; CI-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
-; CI-IEEE-SAFE-NEXT:    v_frexp_mant_f32_e32 v1, v0
-; CI-IEEE-SAFE-NEXT:    v_rcp_f32_e32 v1, v1
-; CI-IEEE-SAFE-NEXT:    v_frexp_exp_i32_f32_e32 v0, v0
-; CI-IEEE-SAFE-NEXT:    v_sub_i32_e32 v0, vcc, 0, v0
-; CI-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v1, v0
-; CI-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_missing_contract1:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
-  %div = fdiv float 1.0, %sqrt, !fpmath !1
+  %div = fdiv arcp float 1.0, %sqrt, !fpmath !1
   ret float %div
 }
 
@@ -2283,25 +870,12 @@ define float @v_rsq_f32_contractable_user(float %val0, float %val1) {
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd contract float %div, %val1
@@ -2317,25 +891,12 @@ define float @v_rsq_f32_contractable_user_missing_contract0(float %val0, float %
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_mov_b32_e32 v2, 0x45800000
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; GCN-IEEE-SAFE-NEXT:    v_fma_f32 v0, v0, v2, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user_missing_contract0:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd contract float %div, %val1
@@ -2351,25 +912,12 @@ define float @v_rsq_f32_contractable_user_missing_contract1(float %val0, float %
 ; GCN-DAZ-NEXT:    v_add_f32_e32 v0, v0, v1
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v2, 0, 12, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v2
-; GCN-IEEE-SAFE-NEXT:    v_add_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_contractable_user_missing_contract1:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    v_add_f32_e32 v0, v0, v1
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val0), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   %add = fadd float %div, %val1
@@ -2400,23 +948,11 @@ define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
 ; GCN-DAZ-NEXT:    v_rsq_f32_e32 v0, v0
 ; GCN-DAZ-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GCN-IEEE-UNSAFE-LABEL: v_rsq_f32_known_never_posdenormal:
-; GCN-IEEE-UNSAFE:       ; %bb.0:
-; GCN-IEEE-UNSAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-UNSAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-UNSAFE-NEXT:    s_setpc_b64 s[30:31]
-;
-; GCN-IEEE-SAFE-LABEL: v_rsq_f32_known_never_posdenormal:
-; GCN-IEEE-SAFE:       ; %bb.0:
-; GCN-IEEE-SAFE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IEEE-SAFE-NEXT:    s_mov_b32 s4, 0x800000
-; GCN-IEEE-SAFE-NEXT:    v_cmp_gt_f32_e32 vcc, s4, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 24, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    v_rsq_f32_e32 v0, v0
-; GCN-IEEE-SAFE-NEXT:    v_cndmask_b32_e64 v1, 0, 12, vcc
-; GCN-IEEE-SAFE-NEXT:    v_ldexp_f32_e32 v0, v0, v1
-; GCN-IEEE-SAFE-NEXT:    s_setpc_b64 s[30:31]
+; GCN-IEEE-LABEL: v_rsq_f32_known_never_posdenormal:
+; GCN-IEEE:       ; %bb.0:
+; GCN-IEEE-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-IEEE-NEXT:    v_rsq_f32_e32 v0, v0
+; GCN-IEEE-NEXT:    s_setpc_b64 s[30:31]
   %sqrt = call contract float @llvm.sqrt.f32(float %val), !fpmath !1
   %div = fdiv contract float 1.0, %sqrt, !fpmath !1
   ret float %div
@@ -2427,9 +963,9 @@ define float @v_rsq_f32_known_never_posdenormal(float nofpclass(psub) %val) {
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CI-DAZ-SAFE: {{.*}}
 ; CI-DAZ-UNSAFE: {{.*}}
 ; CI-IEEE-UNSAFE: {{.*}}
-; SI-DAZ-SAFE: {{.*}}
+; GCN-DAZ-UNSAFE: {{.*}}
+; GCN-IEEE-UNSAFE: {{.*}}
 ; SI-DAZ-UNSAFE: {{.*}}
 ; SI-IEEE-UNSAFE: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
index 39428dc44801..1b7112ef4ddc 100644
--- a/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-flags-to-fmin-fmax.ll
@@ -916,6 +916,7 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_safe(<2 x half> %a, <2 x half> %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -984,6 +985,7 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_nnan_flag(<2 x half> %a, <2 x ha
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1052,6 +1054,7 @@ define <2 x half> @v_test_fmin_legacy_ule_v2f16_nsz_flag(<2 x half> %a, <2 x hal
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1156,6 +1159,7 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_safe(<2 x half> %a, <2 x half> %
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1224,6 +1228,7 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_nnan_flag(<2 x half> %a, <2 x ha
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1292,6 +1297,7 @@ define <2 x half> @v_test_fmax_legacy_uge_v2f16_nsz_flag(<2 x half> %a, <2 x hal
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffd
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xf1ff
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.l, s0
 ; GFX12-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 33585024d81d..ea6449b99b51 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -31,6 +31,7 @@
 ; DEFAULT-NEXT: Live Register Matrix
 ; DEFAULT-NEXT: Greedy Register Allocator
 ; DEFAULT-NEXT: GCN NSA Reassign
+; DEFAULT-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
 ; DEFAULT-NEXT: Virtual Register Rewriter
 ; DEFAULT-NEXT: AMDGPU Mark Last Scratch Load
 ; DEFAULT-NEXT: Stack Slot Coloring
@@ -77,6 +78,7 @@
 ; BASIC-DEFAULT-NEXT: Live Register Matrix
 ; BASIC-DEFAULT-NEXT: Greedy Register Allocator
 ; BASIC-DEFAULT-NEXT: GCN NSA Reassign
+; BASIC-DEFAULT-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
 ; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
 ; BASIC-DEFAULT-NEXT: AMDGPU Mark Last Scratch Load
 ; BASIC-DEFAULT-NEXT: Stack Slot Coloring
@@ -99,6 +101,7 @@
 ; DEFAULT-BASIC-NEXT: Live Register Matrix
 ; DEFAULT-BASIC-NEXT: Basic Register Allocator
 ; DEFAULT-BASIC-NEXT: GCN NSA Reassign
+; DEFAULT-BASIC-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
 ; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
 ; DEFAULT-BASIC-NEXT: AMDGPU Mark Last Scratch Load
 ; DEFAULT-BASIC-NEXT: Stack Slot Coloring
@@ -127,6 +130,7 @@
 ; BASIC-BASIC-NEXT: Live Register Matrix
 ; BASIC-BASIC-NEXT: Basic Register Allocator
 ; BASIC-BASIC-NEXT: GCN NSA Reassign
+; BASIC-BASIC-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
 ; BASIC-BASIC-NEXT: Virtual Register Rewriter
 ; BASIC-BASIC-NEXT: AMDGPU Mark Last Scratch Load
 ; BASIC-BASIC-NEXT: Stack Slot Coloring
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll
index 008e19b62052..5914253b5f58 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v2bf16.ll
@@ -1228,51 +1228,49 @@ define void @v_shuffle_v3bf16_v2bf16__3_u_1(ptr addrspace(1) inreg %ptr) {
 ; GFX900-LABEL: v_shuffle_v3bf16_v2bf16__3_u_1:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v1
 ; GFX900-NEXT:    ;;#ASMEND
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v2
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX900-NEXT:    v_alignbit_b32 v2, s4, v2, 16
-; GFX900-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX900-NEXT:    global_store_short_d16_hi v0, v1, s[16:17] offset:4
 ; GFX900-NEXT:    global_store_dword v0, v2, s[16:17]
-; GFX900-NEXT:    global_store_short v0, v1, s[16:17] offset:4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX90A-LABEL: v_shuffle_v3bf16_v2bf16__3_u_1:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v1
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v2
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    v_alignbit_b32 v2, s4, v2, 16
-; GFX90A-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX90A-NEXT:    global_store_short_d16_hi v0, v1, s[16:17] offset:4
 ; GFX90A-NEXT:    global_store_dword v0, v2, s[16:17]
-; GFX90A-NEXT:    global_store_short v0, v1, s[16:17] offset:4
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: v_shuffle_v3bf16_v2bf16__3_u_1:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v1
 ; GFX942-NEXT:    ;;#ASMEND
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v2
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_nop 0
 ; GFX942-NEXT:    v_alignbit_b32 v2, s0, v2, 16
-; GFX942-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX942-NEXT:    global_store_short_d16_hi v0, v1, s[0:1] offset:4
 ; GFX942-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX942-NEXT:    global_store_short v0, v1, s[0:1] offset:4
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x bfloat> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll
index 99c9480adc41..cd4dbe93e8a1 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3bf16.v3bf16.ll
@@ -1928,48 +1928,45 @@ define void @v_shuffle_v3bf16_v3bf16__5_u_1(ptr addrspace(1) inreg %ptr) {
 ; GFX900-LABEL: v_shuffle_v3bf16_v3bf16__5_u_1:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v[0:1]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_mov_b32_e32 v3, 0
-; GFX900-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v[1:2]
 ; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    global_store_short_d16_hi v3, v0, s[16:17] offset:4
 ; GFX900-NEXT:    global_store_dword v3, v2, s[16:17]
-; GFX900-NEXT:    global_store_short v3, v0, s[16:17] offset:4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX90A-LABEL: v_shuffle_v3bf16_v3bf16__5_u_1:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    global_store_short_d16_hi v4, v0, s[16:17] offset:4
 ; GFX90A-NEXT:    global_store_dword v4, v3, s[16:17]
-; GFX90A-NEXT:    global_store_short v4, v0, s[16:17] offset:4
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: v_shuffle_v3bf16_v3bf16__5_u_1:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v[0:1]
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_mov_b32_e32 v4, 0
-; GFX942-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v[2:3]
 ; GFX942-NEXT:    ;;#ASMEND
+; GFX942-NEXT:    global_store_short_d16_hi v4, v0, s[0:1] offset:4
 ; GFX942-NEXT:    global_store_dword v4, v3, s[0:1]
-; GFX942-NEXT:    global_store_short v4, v0, s[0:1] offset:4
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x bfloat> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll
index e34becc1065f..99cb8a38f57c 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v2f16.ll
@@ -1228,51 +1228,49 @@ define void @v_shuffle_v3f16_v2f16__3_u_1(ptr addrspace(1) inreg %ptr) {
 ; GFX900-LABEL: v_shuffle_v3f16_v2f16__3_u_1:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v1
 ; GFX900-NEXT:    ;;#ASMEND
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v2
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX900-NEXT:    v_alignbit_b32 v2, s4, v2, 16
-; GFX900-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX900-NEXT:    global_store_short_d16_hi v0, v1, s[16:17] offset:4
 ; GFX900-NEXT:    global_store_dword v0, v2, s[16:17]
-; GFX900-NEXT:    global_store_short v0, v1, s[16:17] offset:4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX90A-LABEL: v_shuffle_v3f16_v2f16__3_u_1:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v1
 ; GFX90A-NEXT:    ;;#ASMEND
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v2
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX90A-NEXT:    v_alignbit_b32 v2, s4, v2, 16
-; GFX90A-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX90A-NEXT:    global_store_short_d16_hi v0, v1, s[16:17] offset:4
 ; GFX90A-NEXT:    global_store_dword v0, v2, s[16:17]
-; GFX90A-NEXT:    global_store_short v0, v1, s[16:17] offset:4
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: v_shuffle_v3f16_v2f16__3_u_1:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v1
 ; GFX942-NEXT:    ;;#ASMEND
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v2
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_nop 0
 ; GFX942-NEXT:    v_alignbit_b32 v2, s0, v2, 16
-; GFX942-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
+; GFX942-NEXT:    global_store_short_d16_hi v0, v1, s[0:1] offset:4
 ; GFX942-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX942-NEXT:    global_store_short v0, v1, s[0:1] offset:4
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <2 x half> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll
index 84d42c882494..0854ff2ebfc5 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v3f16.v3f16.ll
@@ -1928,48 +1928,45 @@ define void @v_shuffle_v3f16_v3f16__5_u_1(ptr addrspace(1) inreg %ptr) {
 ; GFX900-LABEL: v_shuffle_v3f16_v3f16__5_u_1:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v[0:1]
 ; GFX900-NEXT:    ;;#ASMEND
-; GFX900-NEXT:    v_mov_b32_e32 v3, 0
-; GFX900-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX900-NEXT:    ;;#ASMSTART
 ; GFX900-NEXT:    ; def v[1:2]
 ; GFX900-NEXT:    ;;#ASMEND
+; GFX900-NEXT:    global_store_short_d16_hi v3, v0, s[16:17] offset:4
 ; GFX900-NEXT:    global_store_dword v3, v2, s[16:17]
-; GFX900-NEXT:    global_store_short v3, v0, s[16:17] offset:4
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
 ; GFX900-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX90A-LABEL: v_shuffle_v3f16_v3f16__5_u_1:
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[0:1]
 ; GFX90A-NEXT:    ;;#ASMEND
-; GFX90A-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90A-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX90A-NEXT:    ;;#ASMSTART
 ; GFX90A-NEXT:    ; def v[2:3]
 ; GFX90A-NEXT:    ;;#ASMEND
+; GFX90A-NEXT:    global_store_short_d16_hi v4, v0, s[16:17] offset:4
 ; GFX90A-NEXT:    global_store_dword v4, v3, s[16:17]
-; GFX90A-NEXT:    global_store_short v4, v0, s[16:17] offset:4
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX942-LABEL: v_shuffle_v3f16_v3f16__5_u_1:
 ; GFX942:       ; %bb.0:
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v[0:1]
 ; GFX942-NEXT:    ;;#ASMEND
-; GFX942-NEXT:    v_mov_b32_e32 v4, 0
-; GFX942-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX942-NEXT:    ;;#ASMSTART
 ; GFX942-NEXT:    ; def v[2:3]
 ; GFX942-NEXT:    ;;#ASMEND
+; GFX942-NEXT:    global_store_short_d16_hi v4, v0, s[0:1] offset:4
 ; GFX942-NEXT:    global_store_dword v4, v3, s[0:1]
-; GFX942-NEXT:    global_store_short v4, v0, s[0:1] offset:4
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %vec0 = call <4 x half> asm "; def $0", "=v"()
diff --git a/llvm/test/CodeGen/AMDGPU/si-fold-operands-subreg-imm.mir b/llvm/test/CodeGen/AMDGPU/si-fold-operands-subreg-imm.mir
index 38b4533a1489..1742349673d7 100644
--- a/llvm/test/CodeGen/AMDGPU/si-fold-operands-subreg-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/si-fold-operands-subreg-imm.mir
@@ -115,8 +115,8 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr8_sgpr9
     ; CHECK-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 8
     ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B64_]].sub1, %subreg.sub0, [[S_MOV_B64_]].sub0, %subreg.sub1
-    ; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]].sub0, 8, implicit-def $scc
-    ; CHECK-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY]].sub1, 0, implicit-def $scc, implicit $scc
+    ; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]].sub0, 0, implicit-def $scc
+    ; CHECK-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[COPY]].sub1, 8, implicit-def $scc, implicit $scc
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_ADDC_U32_]]
     %0:sgpr_64 = COPY $sgpr8_sgpr9
     %1:sreg_64 = S_MOV_B64 8
diff --git a/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll b/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll
new file mode 100644
index 000000000000..ca4b72872a03
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/srl64_reduce_flags.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -stop-after=finalize-isel -o - %s | FileCheck %s
+
+;; Test that reduction of:
+;;
+;;   DST = lshr i64 X, Y
+;;
+;; where Y is in the range [63-32] to:
+;;
+;;   DST = [srl i32 X, (Y & 0x1F), 0]
+;;
+;; preserves flags
+
+define i64 @srl_exact(i64 %arg0, i64 %shift_amt) {
+  ; CHECK-LABEL: name: srl_exact
+  ; CHECK: bb.0 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
+  ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE]].sub1
+  ; CHECK-NEXT:   [[DEF3:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF4:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF5:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[DEF3]]
+  ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub0
+  ; CHECK-NEXT:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = exact V_LSHRREV_B32_e64 killed [[COPY5]], killed [[COPY3]], implicit $exec
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   $vgpr0 = COPY [[V_LSHRREV_B32_e64_]]
+  ; CHECK-NEXT:   $vgpr1 = COPY [[V_MOV_B32_e32_]]
+  ; CHECK-NEXT:   SI_RETURN implicit $vgpr0, implicit $vgpr1
+  %or = or i64 %shift_amt, 32
+  %srl = lshr exact i64 %arg0, %or
+  ret i64 %srl
+}
diff --git a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.invalid.ll b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.invalid.ll
index 13dcaaf35132..ca51568eb0e0 100644
--- a/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.invalid.ll
+++ b/llvm/test/CodeGen/AMDGPU/stacksave_stackrestore.invalid.ll
@@ -2,8 +2,8 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stacksave-error.ll 2>&1 | FileCheck -check-prefix=ERR-SAVE-SDAG %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stackrestore-error.ll 2>&1 | FileCheck -check-prefix=ERR-RESTORE-SDAG %s
 
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stacksave-error.ll 2>&1 | FileCheck -check-prefix=ERR-SAVE-GISEL %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stackrestore-error.ll 2>&1 | FileCheck -check-prefix=ERR-RESTORE-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stacksave-error.ll 2>&1 | FileCheck -check-prefix=ERR-SAVE-GISEL %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -filetype=null %t/stackrestore-error.ll 2>&1 | FileCheck -check-prefix=ERR-RESTORE-GISEL %s
 
 ; Test that an error is produced if stacksave/stackrestore are used
 ; with the wrong (default) address space.
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll
index 92fb155430ba..0bfc45c84b0c 100644
--- a/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll
@@ -1,5 +1,5 @@
-; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL %s
+; RUN: not --crash  llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL %s
 
 ; SDAG: LLVM ERROR: Cannot select: {{[a-z0-9]+}}: ch = store<(store (s32) into %ir.ptr.load, addrspace 4)>
 ; GISEL: LLVM ERROR: cannot select: G_STORE %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr(p4) :: (store (s32) into %ir.ptr.load, addrspace 4) (in function: store_to_constant_i32)
diff --git a/llvm/test/CodeGen/AMDGPU/sub_i1.ll b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
index 19d012fc074f..8e65e6450ec4 100644
--- a/llvm/test/CodeGen/AMDGPU/sub_i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub_i1.ll
@@ -1,12 +1,56 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE64 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE32 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,WAVE32 %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
 
-
-; GCN-LABEL: {{^}}sub_var_var_i1:
-; WAVE32: v_xor_b32_e32
-; WAVE64: v_xor_b32_e32
 define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
+; GFX9-LABEL: sub_var_var_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[6:7] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_var_var_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v2, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_var_var_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_u8 v2, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_xor_b32_e32 v1, v1, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in0
   %b = load volatile i1, ptr addrspace(1) %in1
   %sub = sub i1 %a, %b
@@ -14,21 +58,175 @@ define amdgpu_kernel void @sub_var_var_i1(ptr addrspace(1) %out, ptr addrspace(1
   ret void
 }
 
-; GCN-LABEL: {{^}}sub_var_imm_i1:
-; WAVE32: s_xor_b32
-; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_var_imm_i1(ptr addrspace(1) %out, ptr addrspace(1) %in) {
+; GFX9-LABEL: sub_var_imm_i1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT:    s_xor_b64 s[2:3], vcc, -1
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_var_imm_i1:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_var_imm_i1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v1, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT:    s_xor_b32 s2, vcc_lo, -1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %a = load volatile i1, ptr addrspace(1) %in
   %sub = sub i1 %a, 1
   store i1 %sub, ptr addrspace(1) %out
   ret void
 }
 
-; GCN-LABEL: {{^}}sub_i1_cf:
-; GCN: ; %endif
-; WAVE32: s_xor_b32
-; WAVE64: s_xor_b64
 define amdgpu_kernel void @sub_i1_cf(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX9-LABEL: sub_i1_cf:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, 15, v0
+; GFX9-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_2
+; GFX9-NEXT:  ; %bb.1: ; %else
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[8:9] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v0
+; GFX9-NEXT:  .LBB2_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[6:7], s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB2_4
+; GFX9-NEXT:  ; %bb.3: ; %if
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v0, v0, s[2:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_andn2_b64 s[2:3], s[4:5], exec
+; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX9-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX9-NEXT:    s_or_b64 s[4:5], s[2:3], s[4:5]
+; GFX9-NEXT:  .LBB2_4: ; %endif
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b64 s[2:3], s[4:5], -1
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX9-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX10-LABEL: sub_i1_cf:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 15, v0
+; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_2
+; GFX10-NEXT:  ; %bb.1: ; %else
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[6:7] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 1, v0
+; GFX10-NEXT:  .LBB2_2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s5, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB2_4
+; GFX10-NEXT:  ; %bb.3: ; %if
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    global_load_ubyte v0, v0, s[2:3] glc dlc
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_andn2_b32 s2, s4, exec_lo
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s4, s2, s3
+; GFX10-NEXT:  .LBB2_4: ; %endif
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_xor_b32 s2, s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX10-NEXT:    global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: sub_i1_cf:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s7, exec_lo
+; GFX11-NEXT:    ; implicit-def: $sgpr6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmpx_lt_u32_e32 15, v0
+; GFX11-NEXT:    s_xor_b32 s7, exec_lo, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_2
+; GFX11-NEXT:  ; %bb.1: ; %else
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u8 v0, v0, s[4:5] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 1, v0
+; GFX11-NEXT:  .LBB2_2: ; %Flow
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_and_not1_saveexec_b32 s4, s7
+; GFX11-NEXT:    s_cbranch_execz .LBB2_4
+; GFX11-NEXT:  ; %bb.3: ; %if
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    global_load_u8 v0, v0, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_and_not1_b32 s2, s6, exec_lo
+; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT:    s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-NEXT:    s_or_b32 s6, s2, s3
+; GFX11-NEXT:  .LBB2_4: ; %endif
+; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_xor_b32 s2, s6, -1
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX11-NEXT:    global_store_b8 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %d_cmp = icmp ult i32 %tid, 16
diff --git a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
new file mode 100644
index 000000000000..d0d1ba82dc00
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s
+
+declare ptr @G()
+
+define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x i32> %vec) {
+; CHECK-LABEL: foo:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_add_u32 flat_scratch_lo, s12, s17
+; CHECK-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
+; CHECK-NEXT:    v_mov_b32_e32 v40, v0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
+; CHECK-NEXT:    flat_load_dword v42, v[0:1]
+; CHECK-NEXT:    s_mov_b64 s[38:39], s[6:7]
+; CHECK-NEXT:    s_mov_b64 s[48:49], s[4:5]
+; CHECK-NEXT:    s_load_dwordx4 s[4:7], s[8:9], 0x8
+; CHECK-NEXT:    s_load_dword s64, s[8:9], 0x0
+; CHECK-NEXT:    s_add_u32 s0, s0, s17
+; CHECK-NEXT:    s_addc_u32 s1, s1, 0
+; CHECK-NEXT:    s_mov_b64 s[34:35], s[8:9]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v46, s6
+; CHECK-NEXT:    v_mov_b32_e32 v47, s7
+; CHECK-NEXT:    s_mov_b64 s[6:7], src_private_base
+; CHECK-NEXT:    s_cmp_lg_u32 s64, -1
+; CHECK-NEXT:    s_cselect_b32 s7, s7, 0
+; CHECK-NEXT:    s_cselect_b32 s8, s64, 0
+; CHECK-NEXT:    s_add_u32 s50, s34, 48
+; CHECK-NEXT:    s_addc_u32 s51, s35, 0
+; CHECK-NEXT:    v_pk_mov_b32 v[58:59], s[4:5], s[4:5] op_sel:[0,1]
+; CHECK-NEXT:    s_getpc_b64 s[4:5]
+; CHECK-NEXT:    s_add_u32 s4, s4, G@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s5, s5, G@gotpcrel32@hi+12
+; CHECK-NEXT:    s_load_dwordx2 s[54:55], s[4:5], 0x0
+; CHECK-NEXT:    s_mov_b32 s6, 0
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], 0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v57, s7
+; CHECK-NEXT:    s_mov_b32 s7, s6
+; CHECK-NEXT:    s_mov_b32 s53, s14
+; CHECK-NEXT:    v_accvgpr_write_b32 a33, v1
+; CHECK-NEXT:    v_mov_b32_e32 v56, s8
+; CHECK-NEXT:    v_pk_mov_b32 v[60:61], s[6:7], s[6:7] op_sel:[0,1]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[50:51]
+; CHECK-NEXT:    s_mov_b32 s12, s14
+; CHECK-NEXT:    s_mov_b32 s13, s15
+; CHECK-NEXT:    s_mov_b32 s14, s16
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
+; CHECK-NEXT:    s_mov_b32 s32, 0
+; CHECK-NEXT:    s_mov_b32 s33, s16
+; CHECK-NEXT:    s_mov_b32 s52, s15
+; CHECK-NEXT:    s_mov_b64 s[36:37], s[10:11]
+; CHECK-NEXT:    v_accvgpr_write_b32 a32, v0
+; CHECK-NEXT:    flat_store_dwordx2 v[58:59], v[60:61]
+; CHECK-NEXT:    ; kill: def $sgpr15 killed $sgpr15
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; CHECK-NEXT:    flat_load_dwordx2 v[62:63], v[58:59]
+; CHECK-NEXT:    v_accvgpr_read_b32 v0, a32
+; CHECK-NEXT:    v_mov_b32_e32 v44, 0
+; CHECK-NEXT:    v_mov_b32_e32 v45, 0x3ff00000
+; CHECK-NEXT:    v_accvgpr_read_b32 v1, a33
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[48:49]
+; CHECK-NEXT:    s_mov_b64 s[6:7], s[38:39]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[50:51]
+; CHECK-NEXT:    s_mov_b64 s[10:11], s[36:37]
+; CHECK-NEXT:    s_mov_b32 s12, s53
+; CHECK-NEXT:    s_mov_b32 s13, s52
+; CHECK-NEXT:    s_mov_b32 s14, s33
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
+; CHECK-NEXT:    flat_store_dwordx2 v[0:1], v[44:45]
+; CHECK-NEXT:    flat_store_dwordx2 v[58:59], v[60:61]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ; kill: def $sgpr15 killed $sgpr15
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[54:55]
+; CHECK-NEXT:    flat_load_dwordx2 v[0:1], v[56:57] glc
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s64
+; CHECK-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v42
+; CHECK-NEXT:    flat_store_dwordx2 v[58:59], v[62:63]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_dwordx2 v[58:59], v[46:47]
+; CHECK-NEXT:    buffer_store_dword v47, v0, s[0:3], 0 offen offset:4
+; CHECK-NEXT:    buffer_store_dword v44, v0, s[0:3], 0 offen
+; CHECK-NEXT:    ; implicit-def: $vgpr4
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_execz .LBB0_4
+; CHECK-NEXT:  ; %bb.1: ; %LeafBlock5
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v42
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT:  ; %bb.2: ; %sw.bb17.i.i.i.i
+; CHECK-NEXT:    v_mov_b32_e32 v4, 1
+; CHECK-NEXT:  ; %bb.3: ; %Flow
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:  .LBB0_4: ; %Flow8
+; CHECK-NEXT:    s_or_saveexec_b64 s[4:5], s[4:5]
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], v[42:43], v[42:43] op_sel:[0,1]
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], v[44:45], v[44:45] op_sel:[0,1]
+; CHECK-NEXT:    s_xor_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    s_cbranch_execz .LBB0_8
+; CHECK-NEXT:  ; %bb.5: ; %LeafBlock
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v42
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], v[42:43], v[42:43] op_sel:[0,1]
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], v[44:45], v[44:45] op_sel:[0,1]
+; CHECK-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT:  ; %bb.6: ; %sw.bb.i.i.i.i
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:  ; %bb.7: ; %Flow7
+; CHECK-NEXT:    s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:  .LBB0_8: ; %bb.1
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:    s_cbranch_execz .LBB0_10
+; CHECK-NEXT:  ; %bb.9: ; %sw.bb.i.i.i.i.i
+; CHECK-NEXT:    s_load_dwordx4 s[8:11], s[34:35], 0x20
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[10:11], s[10:11] op_sel:[0,1]
+; CHECK-NEXT:  .LBB0_10: ; %bb.2
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; CHECK-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; CHECK-NEXT:    s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT:    buffer_store_dword v42, off, s[0:3], 0
+; CHECK-NEXT:    s_endpgm
+entry:
+  %load.null = load i32, ptr null, align 8
+  %insert = insertelement <4 x i32> zeroinitializer, i32 %load.null, i64 0
+  %cast = addrspacecast ptr addrspace(5) %ptr5 to ptr
+  store double 0.000000e+00, ptr %p0, align 8
+  %call = tail call ptr @G()
+  store double 1.000000e+00, ptr null, align 8
+  %load.0 = load double, ptr %p0, align 8
+  store volatile double 0.000000e+00, ptr %p0, align 8
+  %call.1 = tail call ptr @G()
+  %load.1 = load volatile double, ptr %cast, align 8
+  store volatile double %load.0, ptr %p0, align 8
+  store double %v0, ptr %p0, align 8
+  %load.2 = load double, ptr %p0, align 8
+  store double %load.2, ptr addrspace(5) %ptr5, align 8
+  store i32 0, ptr addrspace(5) %ptr5, align 4
+  switch i32 %load.null, label %bb.1 [
+    i32 0, label %sw.bb.i.i.i.i
+    i32 1, label %sw.bb17.i.i.i.i
+  ]
+
+sw.bb.i.i.i.i:                                    ; preds = %entry
+  br label %bb.1
+
+sw.bb17.i.i.i.i:                                  ; preds = %entry
+  br label %bb.1
+
+bb.1: ; preds = %sw.bb17.i.i.i.i, %sw.bb.i.i.i.i, %entry
+  %phi.0 = phi i32 [ 0, %entry ], [ 0, %sw.bb.i.i.i.i ], [ 1, %sw.bb17.i.i.i.i ]
+  %phi.1 = phi <4 x i32> [ %insert, %entry ], [ zeroinitializer, %sw.bb.i.i.i.i ], [ %insert, %sw.bb17.i.i.i.i ]
+  switch i32 %phi.0, label %bb.2 [
+    i32 0, label %sw.bb.i.i.i.i.i
+  ]
+
+sw.bb.i.i.i.i.i:                                  ; preds = %bb.1
+  br label %bb.2
+
+bb.2: ; preds = %sw.bb.i.i.i.i.i, %bb.1
+  %phi.2 = phi <4 x i32> [ %phi.1, %bb.1 ], [ %vec, %sw.bb.i.i.i.i.i ]
+  %extract.1 = extractelement <4 x i32> %phi.2, i64 0
+  switch i32 1, label %bb.3 [
+    i32 0, label %sw.bb.i.i5.i.i
+  ]
+
+sw.bb.i.i5.i.i:                                   ; preds = %bb.2
+  br label %bb.3
+
+bb.3: ; preds = %sw.bb.i.i5.i.i, %bb.2
+  %phi.3 = phi <4 x i32> [ zeroinitializer, %sw.bb.i.i5.i.i ], [ %insert, %bb.2 ]
+  switch i32 %extract.1, label %bb.4 [
+    i32 0, label %sw.bb7.i.i.i3.i.i
+  ]
+
+sw.bb7.i.i.i3.i.i:                                ; preds = %bb.3
+  %insert.0 = insertelement <4 x i32> %insert, i32 0, i64 1
+  br label %bb.4
+
+bb.4: ; preds = %sw.bb7.i.i.i3.i.i, %bb.3
+  %phi.4 = phi <4 x i32> [ %phi.3, %bb.3 ], [ %insert.0, %sw.bb7.i.i.i3.i.i ]
+  %extract = extractelement <4 x i32> %phi.4, i64 0
+  store i32 %extract, ptr addrspace(5) null, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-cs-chain.ll b/llvm/test/CodeGen/AMDGPU/unsupported-cs-chain.ll
index 3d6fd0a7b0b8..1cbf904f1c40 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-cs-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-cs-chain.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 -global-isel=1 -mattr=+wavefrontsize64 -verify-machineinstrs=0 < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 -global-isel=1 -mattr=+wavefrontsize64 -verify-machineinstrs=0 < %s 2>&1 | FileCheck %s
 ; RUN: not llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 -global-isel=0 -mattr=+wavefrontsize64 -verify-machineinstrs=0 < %s 2>&1 | FileCheck %s
 
 declare amdgpu_cs_chain void @callee() nounwind
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll b/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
index 95fb8a802277..40f1664fe0c2 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-image-a16.ll
@@ -1,5 +1,5 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG-ERR %s
-; RUN: not --crash llc -global-isel=1 -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
+; RUN: not llc -global-isel=1 -global-isel-abort=1 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
 
 ; Make sure this doesn't assert on targets without the r128-16
 ; feature, and instead generates a selection error.
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll b/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
index 57ba4111b353..df9188768f26 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-image-g16.ll
@@ -1,8 +1,8 @@
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG-ERR %s
 ; RUN: not --crash llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG-ERR %s
 
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
-; RUN: not --crash llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
+; RUN: not llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL-ERR %s
 
 ; Make sure this doesn't assert on targets without the g16 feature, and instead
 ; generates a selection error.
diff --git a/llvm/test/CodeGen/AMDGPU/unsupported-image-sample.ll b/llvm/test/CodeGen/AMDGPU/unsupported-image-sample.ll
index db5f0ad42a67..b3cf3790a59d 100644
--- a/llvm/test/CodeGen/AMDGPU/unsupported-image-sample.ll
+++ b/llvm/test/CodeGen/AMDGPU/unsupported-image-sample.ll
@@ -1,17 +1,16 @@
 ; RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx9-generic --amdhsa-code-object-version=6 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
-; RUN: not --crash llc -O0 -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefixes=GFX90A %s
-; RUN: not --crash llc -O0 -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefixes=GFX942 %s
+; RUN: not llc -O0 -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefixes=GFX90A %s
+; RUN: not llc -O0 -mtriple=amdgcn -mcpu=gfx942 -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefixes=GFX942 %s
 ; RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1030 %s
 ; RUN: llc -O0 -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX1100 %s
 
 ; GFX9-LABEL: image_sample_test:
 ; GFX9: image_sample_lz
 
-; GFX90A: LLVM ERROR: requested image instruction is not supported on this GPU
-
-; GFX942: LLVM ERROR: requested image instruction is not supported on this GPU
+; GFX90A: error: <unknown>:0:0: in function image_sample_test void (ptr addrspace(1), float, float, <8 x i32>, <4 x i32>): requested image instruction is not supported on this GPU
+; GFX942: error: <unknown>:0:0: in function image_sample_test void (ptr addrspace(1), float, float, <8 x i32>, <4 x i32>): requested image instruction is not supported on this GPU
 
 ; GFX1030-LABEL: image_sample_test:
 ; GFX1030: image_sample_lz
@@ -28,3 +27,13 @@ define amdgpu_kernel void @image_sample_test(ptr addrspace(1) %out, float %arg1,
 }
 
 declare <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 immarg, float, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg)
+
+; GFX90A: error: <unknown>:0:0: in function sample_1d_tfe <4 x float> (<8 x i32>, <4 x i32>, ptr addrspace(1), float): TFE is not supported on this GPU
+; GFX942: error: <unknown>:0:0: in function sample_1d_tfe <4 x float> (<8 x i32>, <4 x i32>, ptr addrspace(1), float): TFE is not supported on this GPU
+define <4 x float> @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, ptr addrspace(1) inreg %out, float %s) {
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, ptr addrspace(1) %out, align 4
+  ret <4 x float> %v.vec
+}
diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
index 35cd2663f523..c500565d36b0 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
@@ -1,30 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs -enable-misched -asm-verbose -disable-block-placement -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -check-prefix=SI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
-; SI-LABEL: {{^}}test_if:
 ; Make sure the i1 values created by the cfg structurizer pass are
 ; moved using VALU instructions
 
 
 ; waitcnt should be inserted after exec modification
-; SI:      v_cmp_lt_i32_e32 vcc, 1,
-; SI-NEXT: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0
-; SI-NEXT: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0
-; SI-NEXT: s_and_saveexec_b64 [[SAVE1:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_xor_b64 [[SAVE2:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE1]]
-; SI-NEXT: s_cbranch_execz [[FLOW_BB:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}: ; %LeafBlock3
-; SI:      s_mov_b64 s[{{[0-9]:[0-9]}}], -1
-; SI:      s_and_saveexec_b64
-; SI-NEXT: s_cbranch_execnz
-
 ; v_mov should be after exec modification
-; SI: [[FLOW_BB]]:
-; SI-NEXT: s_andn2_saveexec_b64 [[SAVE2]], [[SAVE2]]
-;
 define amdgpu_kernel void @test_if(i32 %b, ptr addrspace(1) %src, ptr addrspace(1) %dst) #1 {
+; SI-LABEL: test_if:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_load_dword s8, s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 1, v0
+; SI-NEXT:    s_mov_b64 s[10:11], 0
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; SI-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execz .LBB0_3
+; SI-NEXT:  ; %bb.1: ; %LeafBlock3
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
+; SI-NEXT:    s_mov_b64 s[2:3], -1
+; SI-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SI-NEXT:    s_cbranch_execnz .LBB0_9
+; SI-NEXT:  .LBB0_2: ; %Flow7
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT:  .LBB0_3: ; %Flow6
+; SI-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
+; SI-NEXT:    s_cbranch_execz .LBB0_5
+; SI-NEXT:  ; %bb.4: ; %LeafBlock
+; SI-NEXT:    s_mov_b64 s[10:11], exec
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 1, v0
+; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_and_b64 s[6:7], vcc, exec
+; SI-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; SI-NEXT:  .LBB0_5: ; %Flow8
+; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; SI-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
+; SI-NEXT:    s_xor_b64 s[2:3], exec, s[4:5]
+; SI-NEXT:    s_cbranch_execnz .LBB0_10
+; SI-NEXT:  .LBB0_6: ; %Flow9
+; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_and_saveexec_b64 s[2:3], s[10:11]
+; SI-NEXT:    s_cbranch_execz .LBB0_8
+; SI-NEXT:  ; %bb.7: ; %case1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_lshl_b64 s[4:5], s[8:9], 2
+; SI-NEXT:    v_mov_b32_e32 v2, 13
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB0_8: ; %end
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB0_9: ; %case2
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_lshl_b64 s[12:13], s[8:9], 2
+; SI-NEXT:    v_mov_b32_e32 v3, 17
+; SI-NEXT:    v_mov_b32_e32 v1, s12
+; SI-NEXT:    v_mov_b32_e32 v2, s13
+; SI-NEXT:    buffer_store_dword v3, v[1:2], s[0:3], 0 addr64
+; SI-NEXT:    s_xor_b64 s[2:3], exec, -1
+; SI-NEXT:    s_branch .LBB0_2
+; SI-NEXT:  .LBB0_10: ; %default
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_ashr_i32 s9, s8, 31
+; SI-NEXT:    s_lshl_b64 s[4:5], s[8:9], 2
+; SI-NEXT:    s_add_u32 s4, s0, s4
+; SI-NEXT:    s_addc_u32 s5, s1, s5
+; SI-NEXT:    s_and_saveexec_b64 s[6:7], vcc
+; SI-NEXT:    s_xor_b64 s[12:13], exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB0_14
+; SI-NEXT:  .LBB0_11: ; %Flow
+; SI-NEXT:    s_andn2_saveexec_b64 s[12:13], s[12:13]
+; SI-NEXT:    s_cbranch_execz .LBB0_13
+; SI-NEXT:  ; %bb.12: ; %if
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, 19
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:  .LBB0_13: ; %Flow5
+; SI-NEXT:    s_or_b64 exec, exec, s[12:13]
+; SI-NEXT:    s_andn2_b64 s[10:11], s[10:11], exec
+; SI-NEXT:    s_branch .LBB0_6
+; SI-NEXT:  .LBB0_14: ; %else
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    v_mov_b32_e32 v0, 21
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_branch .LBB0_11
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   switch i32 %tid, label %default [
@@ -59,17 +134,23 @@ end:
   ret void
 }
 
-; SI-LABEL: {{^}}simple_test_v_if:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}:
-; SI: buffer_store_dword
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB1_2
+; SI-NEXT:  ; %bb.1: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB1_2: ; %exit
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -84,18 +165,23 @@ exit:
 }
 
 ; FIXME: It would be better to endpgm in the then block.
-
-; SI-LABEL: {{^}}simple_test_v_if_ret_else_ret:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}:
-; SI: buffer_store_dword
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if_ret_else_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if_ret_else_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB2_2
+; SI-NEXT:  ; %bb.1: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB2_2: ; %UnifiedReturnBlock
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -112,27 +198,33 @@ exit:
 ; Final block has more than a ret to execute. This was miscompiled
 ; before function exit blocks were unified since the endpgm would
 ; terminate the then wavefront before reaching the store.
-
-; SI-LABEL: {{^}}simple_test_v_if_ret_else_code_ret:
-; SI: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
-; SI: s_cbranch_execnz [[EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: {{^.LBB[0-9]+_[0-9]+}}: ; %Flow
-; SI-NEXT: s_andn2_saveexec_b64 [[BR_SREG]], [[BR_SREG]]
-; SI-NEXT: s_cbranch_execz [[UNIFIED_RETURN:.LBB[0-9]+_[0-9]+]]
-
-; SI-NEXT: ; %bb.{{[0-9]+}}: ; %then
-; SI: s_waitcnt
-; SI-NEXT: buffer_store_dword
-
-; SI-NEXT: {{^}}[[UNIFIED_RETURN]]: ; %UnifiedReturnBlock
-; SI: s_endpgm
-
-; SI-NEXT: {{^}}[[EXIT]]:
-; SI: ds_write_b32
 define amdgpu_kernel void @simple_test_v_if_ret_else_code_ret(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_if_ret_else_code_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_xor_b64 s[0:1], exec, s[0:1]
+; SI-NEXT:    s_cbranch_execnz .LBB3_4
+; SI-NEXT:  .LBB3_1: ; %Flow
+; SI-NEXT:    s_andn2_saveexec_b64 s[0:1], s[0:1]
+; SI-NEXT:    s_cbranch_execz .LBB3_3
+; SI-NEXT:  ; %bb.2: ; %then
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0x3e7
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:  .LBB3_3: ; %UnifiedReturnBlock
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB3_4: ; %exit
+; SI-NEXT:    v_mov_b32_e32 v0, 7
+; SI-NEXT:    s_mov_b32 m0, -1
+; SI-NEXT:    ds_write_b32 v0, v0
+; SI-NEXT:    ; implicit-def: $vgpr0
+; SI-NEXT:    s_branch .LBB3_1
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %is.0 = icmp ne i32 %tid, 0
   br i1 %is.0, label %then, label %exit
@@ -147,21 +239,38 @@ exit:
   ret void
 }
 
-; SI-LABEL: {{^}}simple_test_v_loop:
-; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
-; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:.LBB[0-9]+_[0-9]+]]
-
-; SI: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0{{$}}
-
-; SI: [[LABEL_LOOP:.LBB[0-9]+_[0-9]+]]:
-; SI: buffer_load_dword
-; SI-DAG: buffer_store_dword
-; SI-DAG: s_cmpk_lg_i32 s{{[0-9]+}}, 0x100
-; SI: s_cbranch_scc1 [[LABEL_LOOP]]
-; SI: [[LABEL_EXIT]]:
-; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_loop(ptr addrspace(1) %dst, ptr addrspace(1) %src) #1 {
+; SI-LABEL: simple_test_v_loop:
+; SI:       ; %bb.0: ; %entry
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB4_3
+; SI-NEXT:  ; %bb.1: ; %loop.preheader
+; SI-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    s_mov_b64 s[0:1], 0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s8, v0
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_mov_b32 s4, s10
+; SI-NEXT:    s_mov_b32 s5, s11
+; SI-NEXT:    s_mov_b32 s7, s3
+; SI-NEXT:  .LBB4_2: ; %loop
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v2, off, s[4:7], 0
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_add_u32 s0, s0, 4
+; SI-NEXT:    s_addc_u32 s1, s1, 0
+; SI-NEXT:    s_cmpk_lg_i32 s0, 0x100
+; SI-NEXT:    s_cbranch_scc1 .LBB4_2
+; SI-NEXT:  .LBB4_3: ; %exit
+; SI-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %is.0 = icmp ne i32 %tid, 0
@@ -182,45 +291,81 @@ exit:
   ret void
 }
 
-; SI-LABEL: {{^}}multi_vcond_loop:
-
 ; Load loop limit from buffer
 ; Branch to exit if uniformly not taken
-; SI: ; %bb.0:
-; SI: buffer_load_dword [[VBOUND:v[0-9]+]]
-; SI: v_cmp_lt_i32_e32 vcc
-; SI: s_and_saveexec_b64 [[OUTER_CMP_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:.LBB[0-9]+_[0-9]+]]
-
 ; Initialize inner condition to false
-; SI: ; %bb.{{[0-9]+}}: ; %bb10.preheader
-; SI: s_mov_b64 [[COND_STATE:s\[[0-9]+:[0-9]+\]]], 0{{$}}
-
 ; Clear exec bits for workitems that load -1s
-; SI: .L[[LABEL_LOOP:BB[0-9]+_[0-9]+]]:
-; SI: buffer_load_dword [[B:v[0-9]+]]
-; SI: buffer_load_dword [[A:v[0-9]+]]
-; SI-DAG: v_cmp_ne_u32_e64 [[NEG1_CHECK_0:s\[[0-9]+:[0-9]+\]]], -1, [[A]]
-; SI-DAG: v_cmp_ne_u32_e32 [[NEG1_CHECK_1:vcc]], -1, [[B]]
-; SI: s_and_b64 [[ORNEG1:s\[[0-9]+:[0-9]+\]]], [[NEG1_CHECK_1]], [[NEG1_CHECK_0]]
-; SI: s_and_saveexec_b64 [[ORNEG2:s\[[0-9]+:[0-9]+\]]], [[ORNEG1]]
-; SI: s_cbranch_execz [[LABEL_FLOW:.LBB[0-9]+_[0-9]+]]
-
-; SI: ; %bb.{{[0-9]+}}: ; %bb20
-; SI: buffer_store_dword
-
-; SI: [[LABEL_FLOW]]:
-; SI-NEXT: ; in Loop: Header=[[LABEL_LOOP]]
-; SI-NEXT: s_or_b64 exec, exec, [[ORNEG2]]
-; SI-NEXT: s_and_b64 [[TMP1:s\[[0-9]+:[0-9]+\]]],
-; SI-NEXT: s_or_b64 [[COND_STATE]], [[TMP1]], [[COND_STATE]]
-; SI-NEXT: s_andn2_b64 exec, exec, [[COND_STATE]]
-; SI-NEXT: s_cbranch_execnz .L[[LABEL_LOOP]]
-
-; SI: [[LABEL_EXIT]]:
-; SI-NOT: [[COND_STATE]]
-; SI: s_endpgm
 define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %arg, ptr addrspace(1) noalias nocapture readonly %arg1, ptr addrspace(1) noalias nocapture readonly %arg2, ptr addrspace(1) noalias nocapture readonly %arg3) #1 {
+; SI-LABEL: multi_vcond_loop:
+; SI:       ; %bb.0: ; %bb
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0xf
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    v_mov_b32_e32 v7, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 2, v0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v0, v[6:7], s[8:11], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v0
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; SI-NEXT:    s_cbranch_execz .LBB5_5
+; SI-NEXT:  ; %bb.1: ; %bb10.preheader
+; SI-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; SI-NEXT:    s_mov_b64 s[2:3], 0
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, s13
+; SI-NEXT:    v_add_i32_e32 v2, vcc, s12, v6
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; SI-NEXT:    v_mov_b32_e32 v5, s1
+; SI-NEXT:    v_add_i32_e32 v4, vcc, s0, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v7, s15
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s14, v6
+; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB5_2: ; %bb10
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    buffer_load_dword v8, v[6:7], s[8:11], 0 addr64
+; SI-NEXT:    buffer_load_dword v9, v[4:5], s[8:11], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_cmp_ne_u32_e32 vcc, -1, v8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], -1, v9
+; SI-NEXT:    s_and_b64 s[12:13], vcc, s[0:1]
+; SI-NEXT:    s_or_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_and_saveexec_b64 s[0:1], s[12:13]
+; SI-NEXT:    s_cbranch_execz .LBB5_4
+; SI-NEXT:  ; %bb.3: ; %bb20
+; SI-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; SI-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
+; SI-NEXT:    s_add_u32 s6, s6, 1
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 4, v4
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; SI-NEXT:    v_add_i32_e32 v6, vcc, 4, v6
+; SI-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; SI-NEXT:    buffer_store_dword v8, v[2:3], s[8:11], 0 addr64
+; SI-NEXT:    s_addc_u32 s7, s7, 0
+; SI-NEXT:    v_add_i32_e32 v2, vcc, 4, v2
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; SI-NEXT:    v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
+; SI-NEXT:    s_andn2_b64 s[4:5], s[4:5], exec
+; SI-NEXT:    s_and_b64 s[12:13], vcc, exec
+; SI-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
+; SI-NEXT:  .LBB5_4: ; %Flow
+; SI-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
+; SI-NEXT:    s_and_b64 s[0:1], exec, s[4:5]
+; SI-NEXT:    s_or_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_cbranch_execnz .LBB5_2
+; SI-NEXT:  .LBB5_5: ; %bb26
+; SI-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %tmp4 = sext i32 %tmp to i64
diff --git a/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll b/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
index 2c66d38a1be6..33ca7180325a 100644
--- a/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
+++ b/llvm/test/CodeGen/AMDGPU/vcmp-saveexec-to-vcmpx.ll
@@ -1,12 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1010 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1030 %s
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_lt:
-; GFX1010: v_cmp_lt_i32_e32 vcc_lo, 15, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_lt_i32_e32 15, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_lt(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_lt:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 15, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_lt:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 15, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp slt i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -19,12 +36,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_gt:
-; GFX1010: v_cmp_gt_i32_e32 vcc_lo, 17, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_gt_i32_e32 17, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_gt(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_gt:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 17, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_gt:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_gt_i32_e32 17, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sgt i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -37,12 +70,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_eq:
-; GFX1010: v_cmp_ne_u32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_ne_u32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_eq(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_eq:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_eq:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_ne_u32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp eq i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -55,12 +104,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_ne:
-; GFX1010: v_cmp_eq_u32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_eq_u32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_ne(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_ne:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_ne:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_eq_u32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp ne i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -73,12 +138,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_le:
-; GFX1010: v_cmp_lt_i32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_lt_i32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_le(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_le:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_lt_i32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_le:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_lt_i32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sle i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -91,12 +172,28 @@ endif:
   ret i32 %x
 }
 
-; GCN-LABEL: {{^}}test_insert_vcmpx_pattern_ge:
-; GFX1010: v_cmp_gt_i32_e32 vcc_lo, 16, v{{.*}}
-; GFX1010-NEXT: s_and_saveexec_b32 s{{.*}}, vcc_lo
-; GFX1030: s_mov_b32 s{{.*}}, exec_lo
-; GFX1030-NEXT: v_cmpx_gt_i32_e32 16, v{{.*}}
 define i32 @test_insert_vcmpx_pattern_ge(i32 %x) {
+; GFX1010-LABEL: test_insert_vcmpx_pattern_ge:
+; GFX1010:       ; %bb.0: ; %entry
+; GFX1010-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1010-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 16, v0
+; GFX1010-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX1010-NEXT:  ; %bb.1: ; %if
+; GFX1010-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1010-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1010-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1010-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1030-LABEL: test_insert_vcmpx_pattern_ge:
+; GFX1030:       ; %bb.0: ; %entry
+; GFX1030-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT:    s_mov_b32 s4, exec_lo
+; GFX1030-NEXT:    v_cmpx_gt_i32_e32 16, v0
+; GFX1030-NEXT:  ; %bb.1: ; %if
+; GFX1030-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT:  ; %bb.2: ; %UnifiedReturnBlock
+; GFX1030-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX1030-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %bc = icmp sge i32 %x, 16
   br i1 %bc, label %endif, label %if
@@ -113,13 +210,32 @@ declare amdgpu_gfx void @check_live_outs_helper(i64) #0
 
 ; In cases where the output operand cannot be safely removed,
 ; don't apply the v_cmpx transformation.
-
-; GCN-LABEL: {{^}}check_live_outs:
-; GFX1010: v_cmp_eq_u32_e64 s{{.*}}, v{{.*}}, v{{.*}}
-; GFX1010: s_and_saveexec_b32 s{{.*}}, s{{.*}}
-; GFX1030: v_cmp_eq_u32_e64 s{{.*}}, v{{.*}}, v{{.*}}
-; GFX1030: s_and_saveexec_b32 s{{.*}}, s{{.*}}
 define amdgpu_cs void @check_live_outs(i32 %a, i32 %b) {
+; GCN-LABEL: check_live_outs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GCN-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s11, 0x31c16000
+; GCN-NEXT:    s_add_u32 s8, s8, s0
+; GCN-NEXT:    v_cmp_eq_u32_e64 s0, v0, v1
+; GCN-NEXT:    s_addc_u32 s9, s9, 0
+; GCN-NEXT:    s_mov_b32 s32, 0
+; GCN-NEXT:    s_and_saveexec_b32 s1, s0
+; GCN-NEXT:    s_cbranch_execz .LBB6_2
+; GCN-NEXT:  ; %bb.1: ; %l1
+; GCN-NEXT:    s_getpc_b64 s[2:3]
+; GCN-NEXT:    s_add_u32 s2, s2, check_live_outs_helper@gotpcrel32@lo+4
+; GCN-NEXT:    s_addc_u32 s3, s3, check_live_outs_helper@gotpcrel32@hi+12
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_mov_b64 s[0:1], s[8:9]
+; GCN-NEXT:    s_mov_b64 s[2:3], s[10:11]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:  .LBB6_2: ; %l2
+; GCN-NEXT:    s_endpgm
   %cond = icmp eq i32 %a, %b
   %result = call i64 @llvm.amdgcn.icmp.i32(i32 %a, i32 %b, i32 32)
   br i1 %cond, label %l1, label %l2
@@ -132,14 +248,27 @@ l2:
 
 ; Omit the transformation if the s_and_saveexec instruction overwrites
 ; any of the v_cmp source operands.
-
-; GCN-LABEL: check_saveexec_overwrites_vcmp_source:
-; GCN:  .LBB7_2: ; %then
-; GFX1010:          v_cmp_eq_u32_e64 s[[C:[0-9]+]], s[[A:[0-9]+]], s[[B:[0-9]+]]
-; GFX1010-NEXT:     s_cmp_ge_i32 s[[C]], s[[B]]
-; GFX1030:          v_cmp_eq_u32_e64 s[[C:[0-9]+]], s[[A:[0-9]+]], s[[B:[0-9]+]]
-; GFX1030-NEXT:     s_cmp_ge_i32 s[[C]], s[[B]]
 define i32 @check_saveexec_overwrites_vcmp_source(i32 inreg %a, i32 inreg %b) {
+; GCN-LABEL: check_saveexec_overwrites_vcmp_source:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_cmp_lt_i32 s16, 0
+; GCN-NEXT:    s_cbranch_scc1 .LBB7_2
+; GCN-NEXT:  ; %bb.1: ; %if
+; GCN-NEXT:    s_lshl_b32 s4, s16, 2
+; GCN-NEXT:    s_or_b32 s4, s4, s17
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_2: ; %then
+; GCN-NEXT:    v_cmp_eq_u32_e64 s4, s16, s17
+; GCN-NEXT:    s_cmp_ge_i32 s4, s17
+; GCN-NEXT:    s_cbranch_scc1 .LBB7_4
+; GCN-NEXT:  ; %bb.3: ; %after
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:  .LBB7_4: ; %end
+; GCN-NEXT:    v_mov_b32_e32 v0, s16
+; GCN-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %0 = icmp sge i32 %a, 0
   br i1 %0, label %if, label %then
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index dfa50ce55a52..30ed6ae5484c 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -2502,6 +2502,7 @@ define i64 @test_vector_reduce_add_v3i64(<3 x i64> %v) {
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
@@ -2639,6 +2640,7 @@ define i64 @test_vector_reduce_add_v4i64(<4 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
@@ -2657,6 +2659,7 @@ define i64 @test_vector_reduce_add_v4i64(<4 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v6
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v7, vcc_lo
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
@@ -2892,6 +2895,7 @@ define i64 @test_vector_reduce_add_v8i64(<8 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
@@ -2922,6 +2926,7 @@ define i64 @test_vector_reduce_add_v8i64(<8 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v6
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v7, vcc_lo
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
@@ -3371,6 +3376,7 @@ define i64 @test_vector_reduce_add_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v5, vcc_lo
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
@@ -3427,6 +3433,7 @@ define i64 @test_vector_reduce_add_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v6
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v3, null, v3, v7, vcc_lo
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 16fbd1eabb30..f0829b53168d 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -3542,9 +3542,9 @@ define i64 @test_vector_reduce_smax_v4i64(<4 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3562,9 +3562,9 @@ define i64 @test_vector_reduce_smax_v4i64(<4 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3873,9 +3873,9 @@ define i64 @test_vector_reduce_smax_v8i64(<8 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3907,9 +3907,9 @@ define i64 @test_vector_reduce_smax_v8i64(<8 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4476,11 +4476,12 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v8, v24, v8, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s3
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e64 s0, v[2:3], v[10:11]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e64 s2, v[0:1], v[8:9]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s2
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -4488,7 +4489,7 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[4:5], v[12:13]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e64 s1, v[6:7], v[14:15]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4
@@ -4499,11 +4500,11 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e64 s0, v[0:1], v[4:5]
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[6:7]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4545,7 +4546,7 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v13, v29, v13, s1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e64 s1, v[4:5], v[12:13]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
@@ -4553,25 +4554,25 @@ define i64 @test_vector_reduce_smax_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e64 s0, v[14:15], v[30:31]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v14, v30, v14, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v15, v31, v15, s0
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e64 s0, v[2:3], v[10:11]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[6:7], v[14:15]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e64 s0, v[2:3], v[6:7]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index bb868621c23d..e67420562e25 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -3542,9 +3542,9 @@ define i64 @test_vector_reduce_smin_v4i64(<4 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3562,9 +3562,9 @@ define i64 @test_vector_reduce_smin_v4i64(<4 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3873,9 +3873,9 @@ define i64 @test_vector_reduce_smin_v8i64(<8 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3907,9 +3907,9 @@ define i64 @test_vector_reduce_smin_v8i64(<8 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4476,11 +4476,12 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v8, v24, v8, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s3
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e64 s0, v[2:3], v[10:11]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e64 s2, v[0:1], v[8:9]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s2
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -4488,7 +4489,7 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[12:13]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e64 s1, v[6:7], v[14:15]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4
@@ -4499,11 +4500,11 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e64 s0, v[0:1], v[4:5]
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[6:7]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4545,7 +4546,7 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v13, v29, v13, s1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e64 s1, v[4:5], v[12:13]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
@@ -4553,25 +4554,25 @@ define i64 @test_vector_reduce_smin_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e64 s0, v[14:15], v[30:31]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v14, v30, v14, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v15, v31, v15, s0
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e64 s0, v[2:3], v[10:11]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[6:7], v[14:15]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e64 s0, v[2:3], v[6:7]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 2eeedd4cfffb..92993d07b4f8 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -3423,9 +3423,9 @@ define i64 @test_vector_reduce_umax_v4i64(<4 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3443,9 +3443,9 @@ define i64 @test_vector_reduce_umax_v4i64(<4 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3754,9 +3754,9 @@ define i64 @test_vector_reduce_umax_v8i64(<8 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3788,9 +3788,9 @@ define i64 @test_vector_reduce_umax_v8i64(<8 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4357,11 +4357,12 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v8, v24, v8, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s3
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e64 s0, v[2:3], v[10:11]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e64 s2, v[0:1], v[8:9]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s2
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -4369,7 +4370,7 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[4:5], v[12:13]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e64 s1, v[6:7], v[14:15]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4
@@ -4380,11 +4381,11 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e64 s0, v[0:1], v[4:5]
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[6:7]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4426,7 +4427,7 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v13, v29, v13, s1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e64 s1, v[4:5], v[12:13]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
@@ -4434,25 +4435,25 @@ define i64 @test_vector_reduce_umax_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e64 s0, v[14:15], v[30:31]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v14, v30, v14, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v15, v31, v15, s0
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e64 s0, v[2:3], v[10:11]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[6:7], v[14:15]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e64 s0, v[2:3], v[6:7]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 6e4a06b3f8f4..2bcee373d924 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -3162,9 +3162,9 @@ define i64 @test_vector_reduce_umin_v4i64(<4 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3182,9 +3182,9 @@ define i64 @test_vector_reduce_umin_v4i64(<4 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3493,9 +3493,9 @@ define i64 @test_vector_reduce_umin_v8i64(<8 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -3527,9 +3527,9 @@ define i64 @test_vector_reduce_umin_v8i64(<8 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4096,11 +4096,12 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v8, v24, v8, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v16, v0, s3
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e64 s0, v[2:3], v[10:11]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e64 s2, v[0:1], v[8:9]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v9, v1, s2
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v8, v0, s2
 ; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
@@ -4108,7 +4109,7 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v15, v31, v15 :: v_dual_cndmask_b32 v14, v30, v14
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[4:5], v[12:13]
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e64 s1, v[6:7], v[14:15]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v5, v13, v5 :: v_dual_cndmask_b32 v4, v12, v4
@@ -4119,11 +4120,11 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) {
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e64 s0, v[0:1], v[4:5]
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[6:7]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xf1ff
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s0
 ; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, v4, v0, s0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v3, v7, v3 :: v_dual_cndmask_b32 v2, v6, v2
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
 ; GFX12-SDAG-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
@@ -4165,7 +4166,7 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v13, v29, v13, s1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v8, v0 :: v_dual_cndmask_b32 v1, v9, v1
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e64 s1, v[4:5], v[12:13]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v4, v12, v4, s1
@@ -4173,25 +4174,25 @@ define i64 @test_vector_reduce_umin_v16i64(<16 x i64> %v) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e64 s0, v[14:15], v[30:31]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v14, v30, v14, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v15, v31, v15, s0
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e64 s0, v[2:3], v[10:11]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[6:7], v[14:15]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v10, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v11, v3, s0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v6, v14, v6 :: v_dual_cndmask_b32 v7, v15, v7
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e64 s0, v[2:3], v[6:7]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v4, v0 :: v_dual_cndmask_b32 v1, v5, v1
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s0
 ; GFX12-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s0
-; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-GISEL-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
 ; GFX12-GISEL-NEXT:    v_dual_cndmask_b32 v0, v2, v0 :: v_dual_cndmask_b32 v1, v3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index 883ec12ebcfe..a401f989a250 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -1,105 +1,40 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -amdgpu-coerce-illegal-types=1 < %s | FileCheck --check-prefix=FEATURE %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefix=DEFAULT %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefix=GFX942 %s
 
 define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v3i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
-; GFX906-NEXT:    v_mov_b32_e32 v3, 8
-; GFX906-NEXT:    s_mov_b32 s4, 0xff0000
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dword v4, v2, s[0:1]
-; GFX906-NEXT:    v_mov_b32_e32 v1, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_lshrrev_b32_sdwa v5, v3, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX906-NEXT:    v_and_or_b32 v4, v4, s4, v5
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB0_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dword v0, v2, s[2:3]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_lshrrev_b32_sdwa v2, v3, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX906-NEXT:    v_and_or_b32 v4, v0, s4, v2
-; GFX906-NEXT:  .LBB0_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    global_store_byte_d16_hi v1, v4, s[6:7] offset:2
-; GFX906-NEXT:    global_store_short v1, v4, s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v3i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v1, 2, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 8
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dword v3, v1, s[0:1]
-; FEATURE-NEXT:    s_mov_b32 s4, 0xff0000
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_lshrrev_b32_sdwa v5, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; FEATURE-NEXT:    s_nop 0
-; FEATURE-NEXT:    v_or_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; FEATURE-NEXT:    v_and_or_b32 v3, v3, s4, v5
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB0_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dword v1, v1, s[2:3]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; FEATURE-NEXT:    s_nop 0
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; FEATURE-NEXT:    v_and_or_b32 v3, v1, s4, v2
-; FEATURE-NEXT:  .LBB0_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    global_store_byte_d16_hi v0, v3, s[6:7] offset:2
-; FEATURE-NEXT:    global_store_short v0, v3, s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v3i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v1, 2, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 8
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dword v3, v1, s[0:1]
-; DEFAULT-NEXT:    s_mov_b32 s4, 0xff0000
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_lshrrev_b32_sdwa v5, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; DEFAULT-NEXT:    s_nop 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; DEFAULT-NEXT:    v_and_or_b32 v3, v3, s4, v5
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB0_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dword v1, v1, s[2:3]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; DEFAULT-NEXT:    s_nop 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; DEFAULT-NEXT:    v_and_or_b32 v3, v1, s4, v2
-; DEFAULT-NEXT:  .LBB0_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    global_store_byte_d16_hi v0, v3, s[6:7] offset:2
-; DEFAULT-NEXT:    global_store_short v0, v3, s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v3i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v1, 2, v4
+; GFX942-NEXT:    v_mov_b32_e32 v2, 8
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dword v3, v1, s[0:1]
+; GFX942-NEXT:    s_mov_b32 s4, 0xff0000
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshrrev_b32_sdwa v5, v2, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_or_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX942-NEXT:    v_and_or_b32 v3, v3, s4, v5
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB0_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dword v1, v1, s[2:3]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX942-NEXT:    v_and_or_b32 v3, v1, s4, v2
+; GFX942-NEXT:  .LBB0_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    global_store_byte_d16_hi v0, v3, s[6:7] offset:2
+; GFX942-NEXT:    global_store_short v0, v3, s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <3 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -118,63 +53,25 @@ bb.2:
 }
 
 define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v4i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
-; GFX906-NEXT:    v_mov_b32_e32 v1, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dword v2, v3, s[0:1]
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB1_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dword v2, v3, s[2:3]
-; GFX906-NEXT:  .LBB1_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dword v1, v2, s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v4i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v3, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v2, 2, v3
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dword v1, v2, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v3
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB1_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dword v1, v2, s[2:3]
-; FEATURE-NEXT:  .LBB1_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dword v0, v1, s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v4i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v3, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v2, 2, v3
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dword v1, v2, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v3
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB1_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dword v1, v2, s[2:3]
-; DEFAULT-NEXT:  .LBB1_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dword v0, v1, s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v4i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v3, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v2, 2, v3
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dword v1, v2, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v3
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB1_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dword v1, v2, s[2:3]
+; GFX942-NEXT:  .LBB1_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -193,75 +90,29 @@ bb.2:
 }
 
 define amdgpu_kernel void @v5i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v5i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
-; GFX906-NEXT:    v_mov_b32_e32 v3, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v4, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB2_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v4, s[2:3]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v2
-; GFX906-NEXT:  .LBB2_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    global_store_byte v3, v2, s[6:7] offset:4
-; GFX906-NEXT:    global_store_dword v3, v1, s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v5i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB2_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; FEATURE-NEXT:  .LBB2_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    global_store_byte v2, v1, s[6:7] offset:4
-; FEATURE-NEXT:    global_store_dword v2, v0, s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v5i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB2_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_and_b32_e32 v1, 0xff, v1
-; DEFAULT-NEXT:  .LBB2_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    global_store_byte v2, v1, s[6:7] offset:4
-; DEFAULT-NEXT:    global_store_dword v2, v0, s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v5i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB2_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_and_b32_e32 v1, 0xff, v1
+; GFX942-NEXT:  .LBB2_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    global_store_byte v2, v1, s[6:7] offset:4
+; GFX942-NEXT:    global_store_dword v2, v0, s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <5 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -280,63 +131,25 @@ bb.2:
 }
 
 define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v8i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
-; GFX906-NEXT:    v_mov_b32_e32 v3, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v4, s[0:1]
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB3_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v4, s[2:3]
-; GFX906-NEXT:  .LBB3_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v3, v[1:2], s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB3_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; FEATURE-NEXT:  .LBB3_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB3_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; DEFAULT-NEXT:  .LBB3_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB3_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
+; GFX942-NEXT:  .LBB3_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -355,63 +168,25 @@ bb.2:
 }
 
 define amdgpu_kernel void @v16i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v16i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v6, 4, v0
-; GFX906-NEXT:    v_mov_b32_e32 v5, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx4 v[1:4], v6, s[0:1]
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB4_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx4 v[1:4], v6, s[2:3]
-; GFX906-NEXT:  .LBB4_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx4 v5, v[1:4], s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v16i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v5, 4, v6
-; FEATURE-NEXT:    v_mov_b32_e32 v4, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx4 v[0:3], v5, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v6
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB4_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx4 v[0:3], v5, s[2:3]
-; FEATURE-NEXT:  .LBB4_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v16i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v5, 4, v6
-; DEFAULT-NEXT:    v_mov_b32_e32 v4, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx4 v[0:3], v5, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v6
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB4_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx4 v[0:3], v5, s[2:3]
-; DEFAULT-NEXT:  .LBB4_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v16i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v5, 4, v6
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[0:3], v5, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v6
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB4_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx4 v[0:3], v5, s[2:3]
+; GFX942-NEXT:  .LBB4_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <16 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -430,75 +205,29 @@ bb.2:
 }
 
 define amdgpu_kernel void @v32i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v32i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v10, 5, v0
-; GFX906-NEXT:    v_mov_b32_e32 v9, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx4 v[5:8], v10, s[0:1] offset:16
-; GFX906-NEXT:    global_load_dwordx4 v[1:4], v10, s[0:1]
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB5_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx4 v[5:8], v10, s[2:3] offset:16
-; GFX906-NEXT:    global_load_dwordx4 v[1:4], v10, s[2:3]
-; GFX906-NEXT:  .LBB5_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(1)
-; GFX906-NEXT:    global_store_dwordx4 v9, v[5:8], s[6:7] offset:16
-; GFX906-NEXT:    s_waitcnt vmcnt(1)
-; GFX906-NEXT:    global_store_dwordx4 v9, v[1:4], s[6:7]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v32i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v10, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v9, 5, v10
-; FEATURE-NEXT:    v_mov_b32_e32 v8, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx4 v[4:7], v9, s[0:1] offset:16
-; FEATURE-NEXT:    global_load_dwordx4 v[0:3], v9, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v10
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB5_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx4 v[4:7], v9, s[2:3] offset:16
-; FEATURE-NEXT:    global_load_dwordx4 v[0:3], v9, s[2:3]
-; FEATURE-NEXT:  .LBB5_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(1)
-; FEATURE-NEXT:    global_store_dwordx4 v8, v[4:7], s[6:7] offset:16
-; FEATURE-NEXT:    s_waitcnt vmcnt(1)
-; FEATURE-NEXT:    global_store_dwordx4 v8, v[0:3], s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v32i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v10, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v9, 5, v10
-; DEFAULT-NEXT:    v_mov_b32_e32 v8, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx4 v[4:7], v9, s[0:1] offset:16
-; DEFAULT-NEXT:    global_load_dwordx4 v[0:3], v9, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v10
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB5_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx4 v[4:7], v9, s[2:3] offset:16
-; DEFAULT-NEXT:    global_load_dwordx4 v[0:3], v9, s[2:3]
-; DEFAULT-NEXT:  .LBB5_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(1)
-; DEFAULT-NEXT:    global_store_dwordx4 v8, v[4:7], s[6:7] offset:16
-; DEFAULT-NEXT:    s_waitcnt vmcnt(1)
-; DEFAULT-NEXT:    global_store_dwordx4 v8, v[0:3], s[6:7]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v32i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v10, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v9, 5, v10
+; GFX942-NEXT:    v_mov_b32_e32 v8, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v9, s[0:1] offset:16
+; GFX942-NEXT:    global_load_dwordx4 v[0:3], v9, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v10
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB5_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v9, s[2:3] offset:16
+; GFX942-NEXT:    global_load_dwordx4 v[0:3], v9, s[2:3]
+; GFX942-NEXT:  .LBB5_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(1)
+; GFX942-NEXT:    global_store_dwordx4 v8, v[4:7], s[6:7] offset:16
+; GFX942-NEXT:    s_waitcnt vmcnt(1)
+; GFX942-NEXT:    global_store_dwordx4 v8, v[0:3], s[6:7]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <32 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -517,244 +246,77 @@ bb.2:
 }
 
 define amdgpu_kernel void @v256i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v256i8_liveout:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX906-NEXT:    v_lshlrev_b32_e32 v61, 3, v0
-; GFX906-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX906-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx4 v[13:16], v61, s[0:1] offset:240
-; GFX906-NEXT:    global_load_dwordx4 v[9:12], v61, s[0:1] offset:224
-; GFX906-NEXT:    global_load_dwordx4 v[5:8], v61, s[0:1] offset:208
-; GFX906-NEXT:    global_load_dwordx4 v[17:20], v61, s[0:1] offset:192
-; GFX906-NEXT:    s_mov_b32 s14, -1
-; GFX906-NEXT:    s_mov_b32 s15, 0xe00000
-; GFX906-NEXT:    s_add_u32 s12, s12, s11
-; GFX906-NEXT:    s_addc_u32 s13, s13, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    v_mov_b32_e32 v4, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    buffer_store_dword v17, off, s[12:15], 0 ; 4-byte Folded Spill
-; GFX906-NEXT:    s_nop 0
-; GFX906-NEXT:    buffer_store_dword v18, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GFX906-NEXT:    buffer_store_dword v19, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
-; GFX906-NEXT:    buffer_store_dword v20, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
-; GFX906-NEXT:    global_load_dwordx4 v[29:32], v61, s[0:1] offset:176
-; GFX906-NEXT:    global_load_dwordx4 v[25:28], v61, s[0:1] offset:160
-; GFX906-NEXT:    global_load_dwordx4 v[21:24], v61, s[0:1] offset:144
-; GFX906-NEXT:    s_nop 0
-; GFX906-NEXT:    global_load_dwordx4 v[17:20], v61, s[0:1] offset:128
-; GFX906-NEXT:    global_load_dwordx4 v[0:3], v61, s[0:1] offset:112
-; GFX906-NEXT:    global_load_dwordx4 v[57:60], v61, s[0:1] offset:96
-; GFX906-NEXT:    global_load_dwordx4 v[53:56], v61, s[0:1] offset:80
-; GFX906-NEXT:    global_load_dwordx4 v[49:52], v61, s[0:1] offset:64
-; GFX906-NEXT:    global_load_dwordx4 v[45:48], v61, s[0:1] offset:48
-; GFX906-NEXT:    global_load_dwordx4 v[41:44], v61, s[0:1] offset:32
-; GFX906-NEXT:    global_load_dwordx4 v[37:40], v61, s[0:1] offset:16
-; GFX906-NEXT:    global_load_dwordx4 v[33:36], v61, s[0:1]
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB6_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx4 v[13:16], v61, s[2:3] offset:240
-; GFX906-NEXT:    global_load_dwordx4 v[9:12], v61, s[2:3] offset:224
-; GFX906-NEXT:    global_load_dwordx4 v[5:8], v61, s[2:3] offset:208
-; GFX906-NEXT:    global_load_dwordx4 v[0:3], v61, s[2:3] offset:192
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
-; GFX906-NEXT:    s_nop 0
-; GFX906-NEXT:    buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GFX906-NEXT:    buffer_store_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
-; GFX906-NEXT:    buffer_store_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
-; GFX906-NEXT:    global_load_dwordx4 v[29:32], v61, s[2:3] offset:176
-; GFX906-NEXT:    global_load_dwordx4 v[25:28], v61, s[2:3] offset:160
-; GFX906-NEXT:    global_load_dwordx4 v[21:24], v61, s[2:3] offset:144
-; GFX906-NEXT:    global_load_dwordx4 v[17:20], v61, s[2:3] offset:128
-; GFX906-NEXT:    s_nop 0
-; GFX906-NEXT:    global_load_dwordx4 v[0:3], v61, s[2:3] offset:112
-; GFX906-NEXT:    global_load_dwordx4 v[57:60], v61, s[2:3] offset:96
-; GFX906-NEXT:    global_load_dwordx4 v[53:56], v61, s[2:3] offset:80
-; GFX906-NEXT:    global_load_dwordx4 v[49:52], v61, s[2:3] offset:64
-; GFX906-NEXT:    global_load_dwordx4 v[45:48], v61, s[2:3] offset:48
-; GFX906-NEXT:    global_load_dwordx4 v[41:44], v61, s[2:3] offset:32
-; GFX906-NEXT:    global_load_dwordx4 v[37:40], v61, s[2:3] offset:16
-; GFX906-NEXT:    global_load_dwordx4 v[33:36], v61, s[2:3]
-; GFX906-NEXT:  .LBB6_2: ; %bb.2
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:112
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[57:60], s[6:7] offset:96
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[53:56], s[6:7] offset:80
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[49:52], s[6:7] offset:64
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[45:48], s[6:7] offset:48
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[41:44], s[6:7] offset:32
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[37:40], s[6:7] offset:16
-; GFX906-NEXT:    s_waitcnt vmcnt(7)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[33:36], s[6:7]
-; GFX906-NEXT:    global_store_dwordx4 v4, v[13:16], s[6:7] offset:240
-; GFX906-NEXT:    global_store_dwordx4 v4, v[9:12], s[6:7] offset:224
-; GFX906-NEXT:    global_store_dwordx4 v4, v[5:8], s[6:7] offset:208
-; GFX906-NEXT:    buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
-; GFX906-NEXT:    buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
-; GFX906-NEXT:    buffer_load_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
-; GFX906-NEXT:    buffer_load_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:192
-; GFX906-NEXT:    global_store_dwordx4 v4, v[29:32], s[6:7] offset:176
-; GFX906-NEXT:    global_store_dwordx4 v4, v[25:28], s[6:7] offset:160
-; GFX906-NEXT:    global_store_dwordx4 v4, v[21:24], s[6:7] offset:144
-; GFX906-NEXT:    global_store_dwordx4 v4, v[17:20], s[6:7] offset:128
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v256i8_liveout:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx4 v[28:31], v1, s[0:1] offset:240
-; FEATURE-NEXT:    global_load_dwordx4 v[24:27], v1, s[0:1] offset:224
-; FEATURE-NEXT:    global_load_dwordx4 v[20:23], v1, s[0:1] offset:208
-; FEATURE-NEXT:    global_load_dwordx4 v[16:19], v1, s[0:1] offset:192
-; FEATURE-NEXT:    global_load_dwordx4 v[12:15], v1, s[0:1] offset:176
-; FEATURE-NEXT:    global_load_dwordx4 v[8:11], v1, s[0:1] offset:160
-; FEATURE-NEXT:    global_load_dwordx4 v[4:7], v1, s[0:1] offset:144
-; FEATURE-NEXT:    global_load_dwordx4 a[0:3], v1, s[0:1] offset:128
-; FEATURE-NEXT:    global_load_dwordx4 v[60:63], v1, s[0:1] offset:112
-; FEATURE-NEXT:    global_load_dwordx4 v[56:59], v1, s[0:1] offset:96
-; FEATURE-NEXT:    global_load_dwordx4 v[52:55], v1, s[0:1] offset:80
-; FEATURE-NEXT:    global_load_dwordx4 v[48:51], v1, s[0:1] offset:64
-; FEATURE-NEXT:    global_load_dwordx4 v[44:47], v1, s[0:1] offset:48
-; FEATURE-NEXT:    global_load_dwordx4 v[40:43], v1, s[0:1] offset:32
-; FEATURE-NEXT:    global_load_dwordx4 v[36:39], v1, s[0:1] offset:16
-; FEATURE-NEXT:    global_load_dwordx4 v[32:35], v1, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB6_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx4 v[28:31], v1, s[2:3] offset:240
-; FEATURE-NEXT:    global_load_dwordx4 v[24:27], v1, s[2:3] offset:224
-; FEATURE-NEXT:    global_load_dwordx4 v[20:23], v1, s[2:3] offset:208
-; FEATURE-NEXT:    global_load_dwordx4 v[16:19], v1, s[2:3] offset:192
-; FEATURE-NEXT:    global_load_dwordx4 v[12:15], v1, s[2:3] offset:176
-; FEATURE-NEXT:    global_load_dwordx4 v[8:11], v1, s[2:3] offset:160
-; FEATURE-NEXT:    global_load_dwordx4 v[4:7], v1, s[2:3] offset:144
-; FEATURE-NEXT:    global_load_dwordx4 a[0:3], v1, s[2:3] offset:128
-; FEATURE-NEXT:    global_load_dwordx4 v[60:63], v1, s[2:3] offset:112
-; FEATURE-NEXT:    global_load_dwordx4 v[56:59], v1, s[2:3] offset:96
-; FEATURE-NEXT:    global_load_dwordx4 v[52:55], v1, s[2:3] offset:80
-; FEATURE-NEXT:    global_load_dwordx4 v[48:51], v1, s[2:3] offset:64
-; FEATURE-NEXT:    global_load_dwordx4 v[44:47], v1, s[2:3] offset:48
-; FEATURE-NEXT:    global_load_dwordx4 v[40:43], v1, s[2:3] offset:32
-; FEATURE-NEXT:    global_load_dwordx4 v[36:39], v1, s[2:3] offset:16
-; FEATURE-NEXT:    global_load_dwordx4 v[32:35], v1, s[2:3]
-; FEATURE-NEXT:  .LBB6_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[60:63], s[6:7] offset:112
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[56:59], s[6:7] offset:96
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[52:55], s[6:7] offset:80
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[48:51], s[6:7] offset:64
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[44:47], s[6:7] offset:48
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[40:43], s[6:7] offset:32
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[36:39], s[6:7] offset:16
-; FEATURE-NEXT:    s_waitcnt vmcnt(7)
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[32:35], s[6:7]
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[28:31], s[6:7] offset:240
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[24:27], s[6:7] offset:224
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[20:23], s[6:7] offset:208
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[16:19], s[6:7] offset:192
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[12:15], s[6:7] offset:176
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[8:11], s[6:7] offset:160
-; FEATURE-NEXT:    global_store_dwordx4 v0, v[4:7], s[6:7] offset:144
-; FEATURE-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7] offset:128
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v256i8_liveout:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx4 v[28:31], v1, s[0:1] offset:240
-; DEFAULT-NEXT:    global_load_dwordx4 v[24:27], v1, s[0:1] offset:224
-; DEFAULT-NEXT:    global_load_dwordx4 v[20:23], v1, s[0:1] offset:208
-; DEFAULT-NEXT:    global_load_dwordx4 v[16:19], v1, s[0:1] offset:192
-; DEFAULT-NEXT:    global_load_dwordx4 v[12:15], v1, s[0:1] offset:176
-; DEFAULT-NEXT:    global_load_dwordx4 v[8:11], v1, s[0:1] offset:160
-; DEFAULT-NEXT:    global_load_dwordx4 v[4:7], v1, s[0:1] offset:144
-; DEFAULT-NEXT:    global_load_dwordx4 a[0:3], v1, s[0:1] offset:128
-; DEFAULT-NEXT:    global_load_dwordx4 v[60:63], v1, s[0:1] offset:112
-; DEFAULT-NEXT:    global_load_dwordx4 v[56:59], v1, s[0:1] offset:96
-; DEFAULT-NEXT:    global_load_dwordx4 v[52:55], v1, s[0:1] offset:80
-; DEFAULT-NEXT:    global_load_dwordx4 v[48:51], v1, s[0:1] offset:64
-; DEFAULT-NEXT:    global_load_dwordx4 v[44:47], v1, s[0:1] offset:48
-; DEFAULT-NEXT:    global_load_dwordx4 v[40:43], v1, s[0:1] offset:32
-; DEFAULT-NEXT:    global_load_dwordx4 v[36:39], v1, s[0:1] offset:16
-; DEFAULT-NEXT:    global_load_dwordx4 v[32:35], v1, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB6_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx4 v[28:31], v1, s[2:3] offset:240
-; DEFAULT-NEXT:    global_load_dwordx4 v[24:27], v1, s[2:3] offset:224
-; DEFAULT-NEXT:    global_load_dwordx4 v[20:23], v1, s[2:3] offset:208
-; DEFAULT-NEXT:    global_load_dwordx4 v[16:19], v1, s[2:3] offset:192
-; DEFAULT-NEXT:    global_load_dwordx4 v[12:15], v1, s[2:3] offset:176
-; DEFAULT-NEXT:    global_load_dwordx4 v[8:11], v1, s[2:3] offset:160
-; DEFAULT-NEXT:    global_load_dwordx4 v[4:7], v1, s[2:3] offset:144
-; DEFAULT-NEXT:    global_load_dwordx4 a[0:3], v1, s[2:3] offset:128
-; DEFAULT-NEXT:    global_load_dwordx4 v[60:63], v1, s[2:3] offset:112
-; DEFAULT-NEXT:    global_load_dwordx4 v[56:59], v1, s[2:3] offset:96
-; DEFAULT-NEXT:    global_load_dwordx4 v[52:55], v1, s[2:3] offset:80
-; DEFAULT-NEXT:    global_load_dwordx4 v[48:51], v1, s[2:3] offset:64
-; DEFAULT-NEXT:    global_load_dwordx4 v[44:47], v1, s[2:3] offset:48
-; DEFAULT-NEXT:    global_load_dwordx4 v[40:43], v1, s[2:3] offset:32
-; DEFAULT-NEXT:    global_load_dwordx4 v[36:39], v1, s[2:3] offset:16
-; DEFAULT-NEXT:    global_load_dwordx4 v[32:35], v1, s[2:3]
-; DEFAULT-NEXT:  .LBB6_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[60:63], s[6:7] offset:112
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[56:59], s[6:7] offset:96
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[52:55], s[6:7] offset:80
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[48:51], s[6:7] offset:64
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[44:47], s[6:7] offset:48
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[40:43], s[6:7] offset:32
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[36:39], s[6:7] offset:16
-; DEFAULT-NEXT:    s_waitcnt vmcnt(7)
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[32:35], s[6:7]
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[28:31], s[6:7] offset:240
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[24:27], s[6:7] offset:224
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[20:23], s[6:7] offset:208
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[16:19], s[6:7] offset:192
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[12:15], s[6:7] offset:176
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[8:11], s[6:7] offset:160
-; DEFAULT-NEXT:    global_store_dwordx4 v0, v[4:7], s[6:7] offset:144
-; DEFAULT-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7] offset:128
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v256i8_liveout:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX942-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v1, 3, v2
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[28:31], v1, s[0:1] offset:240
+; GFX942-NEXT:    global_load_dwordx4 v[24:27], v1, s[0:1] offset:224
+; GFX942-NEXT:    global_load_dwordx4 v[20:23], v1, s[0:1] offset:208
+; GFX942-NEXT:    global_load_dwordx4 v[16:19], v1, s[0:1] offset:192
+; GFX942-NEXT:    global_load_dwordx4 v[12:15], v1, s[0:1] offset:176
+; GFX942-NEXT:    global_load_dwordx4 v[8:11], v1, s[0:1] offset:160
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v1, s[0:1] offset:144
+; GFX942-NEXT:    global_load_dwordx4 a[0:3], v1, s[0:1] offset:128
+; GFX942-NEXT:    global_load_dwordx4 v[60:63], v1, s[0:1] offset:112
+; GFX942-NEXT:    global_load_dwordx4 v[56:59], v1, s[0:1] offset:96
+; GFX942-NEXT:    global_load_dwordx4 v[52:55], v1, s[0:1] offset:80
+; GFX942-NEXT:    global_load_dwordx4 v[48:51], v1, s[0:1] offset:64
+; GFX942-NEXT:    global_load_dwordx4 v[44:47], v1, s[0:1] offset:48
+; GFX942-NEXT:    global_load_dwordx4 v[40:43], v1, s[0:1] offset:32
+; GFX942-NEXT:    global_load_dwordx4 v[36:39], v1, s[0:1] offset:16
+; GFX942-NEXT:    global_load_dwordx4 v[32:35], v1, s[0:1]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB6_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx4 v[28:31], v1, s[2:3] offset:240
+; GFX942-NEXT:    global_load_dwordx4 v[24:27], v1, s[2:3] offset:224
+; GFX942-NEXT:    global_load_dwordx4 v[20:23], v1, s[2:3] offset:208
+; GFX942-NEXT:    global_load_dwordx4 v[16:19], v1, s[2:3] offset:192
+; GFX942-NEXT:    global_load_dwordx4 v[12:15], v1, s[2:3] offset:176
+; GFX942-NEXT:    global_load_dwordx4 v[8:11], v1, s[2:3] offset:160
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v1, s[2:3] offset:144
+; GFX942-NEXT:    global_load_dwordx4 a[0:3], v1, s[2:3] offset:128
+; GFX942-NEXT:    global_load_dwordx4 v[60:63], v1, s[2:3] offset:112
+; GFX942-NEXT:    global_load_dwordx4 v[56:59], v1, s[2:3] offset:96
+; GFX942-NEXT:    global_load_dwordx4 v[52:55], v1, s[2:3] offset:80
+; GFX942-NEXT:    global_load_dwordx4 v[48:51], v1, s[2:3] offset:64
+; GFX942-NEXT:    global_load_dwordx4 v[44:47], v1, s[2:3] offset:48
+; GFX942-NEXT:    global_load_dwordx4 v[40:43], v1, s[2:3] offset:32
+; GFX942-NEXT:    global_load_dwordx4 v[36:39], v1, s[2:3] offset:16
+; GFX942-NEXT:    global_load_dwordx4 v[32:35], v1, s[2:3]
+; GFX942-NEXT:  .LBB6_2: ; %bb.2
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[60:63], s[6:7] offset:112
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[56:59], s[6:7] offset:96
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[52:55], s[6:7] offset:80
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[48:51], s[6:7] offset:64
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[44:47], s[6:7] offset:48
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[40:43], s[6:7] offset:32
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[36:39], s[6:7] offset:16
+; GFX942-NEXT:    s_waitcnt vmcnt(7)
+; GFX942-NEXT:    global_store_dwordx4 v0, v[32:35], s[6:7]
+; GFX942-NEXT:    global_store_dwordx4 v0, v[28:31], s[6:7] offset:240
+; GFX942-NEXT:    global_store_dwordx4 v0, v[24:27], s[6:7] offset:224
+; GFX942-NEXT:    global_store_dwordx4 v0, v[20:23], s[6:7] offset:208
+; GFX942-NEXT:    global_store_dwordx4 v0, v[16:19], s[6:7] offset:192
+; GFX942-NEXT:    global_store_dwordx4 v0, v[12:15], s[6:7] offset:176
+; GFX942-NEXT:    global_store_dwordx4 v0, v[8:11], s[6:7] offset:160
+; GFX942-NEXT:    global_store_dwordx4 v0, v[4:7], s[6:7] offset:144
+; GFX942-NEXT:    global_store_dwordx4 v0, a[0:3], s[6:7] offset:128
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -773,90 +335,34 @@ bb.2:
 }
 
 define amdgpu_kernel void @repeat_successor(i32 %in, ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: repeat_successor:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dword s8, s[4:5], 0x24
-; GFX906-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX906-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    s_cmp_lt_i32 s8, 3
-; GFX906-NEXT:    s_cbranch_scc0 .LBB7_3
-; GFX906-NEXT:  ; %bb.1: ; %LeafBlock
-; GFX906-NEXT:    s_cmp_gt_i32 s8, 0
-; GFX906-NEXT:    s_cbranch_scc0 .LBB7_6
-; GFX906-NEXT:  ; %bb.2:
-; GFX906-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX906-NEXT:    global_load_dword v0, v0, s[0:1]
-; GFX906-NEXT:    s_branch .LBB7_5
-; GFX906-NEXT:  .LBB7_3: ; %LeafBlock5
-; GFX906-NEXT:    s_cmp_eq_u32 s8, 3
-; GFX906-NEXT:    s_cbranch_scc0 .LBB7_6
-; GFX906-NEXT:  ; %bb.4: ; %sw.bb5
-; GFX906-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX906-NEXT:    global_load_dword v0, v0, s[2:3]
-; GFX906-NEXT:  .LBB7_5: ; %return.sink.split
-; GFX906-NEXT:    v_mov_b32_e32 v1, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dword v1, v0, s[6:7]
-; GFX906-NEXT:  .LBB7_6: ; %return
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: repeat_successor:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dword s8, s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; FEATURE-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    s_cmp_lt_i32 s8, 3
-; FEATURE-NEXT:    s_cbranch_scc0 .LBB7_3
-; FEATURE-NEXT:  ; %bb.1: ; %LeafBlock
-; FEATURE-NEXT:    s_cmp_gt_i32 s8, 0
-; FEATURE-NEXT:    s_cbranch_scc0 .LBB7_6
-; FEATURE-NEXT:  ; %bb.2:
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; FEATURE-NEXT:    global_load_dword v0, v0, s[0:1]
-; FEATURE-NEXT:    s_branch .LBB7_5
-; FEATURE-NEXT:  .LBB7_3: ; %LeafBlock5
-; FEATURE-NEXT:    s_cmp_eq_u32 s8, 3
-; FEATURE-NEXT:    s_cbranch_scc0 .LBB7_6
-; FEATURE-NEXT:  ; %bb.4: ; %sw.bb5
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; FEATURE-NEXT:    global_load_dword v0, v0, s[2:3]
-; FEATURE-NEXT:  .LBB7_5: ; %return.sink.split
-; FEATURE-NEXT:    v_mov_b32_e32 v1, 0
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dword v1, v0, s[6:7]
-; FEATURE-NEXT:  .LBB7_6: ; %return
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: repeat_successor:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dword s8, s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; DEFAULT-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    s_cmp_lt_i32 s8, 3
-; DEFAULT-NEXT:    s_cbranch_scc0 .LBB7_3
-; DEFAULT-NEXT:  ; %bb.1: ; %LeafBlock
-; DEFAULT-NEXT:    s_cmp_gt_i32 s8, 0
-; DEFAULT-NEXT:    s_cbranch_scc0 .LBB7_6
-; DEFAULT-NEXT:  ; %bb.2:
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; DEFAULT-NEXT:    global_load_dword v0, v0, s[0:1]
-; DEFAULT-NEXT:    s_branch .LBB7_5
-; DEFAULT-NEXT:  .LBB7_3: ; %LeafBlock5
-; DEFAULT-NEXT:    s_cmp_eq_u32 s8, 3
-; DEFAULT-NEXT:    s_cbranch_scc0 .LBB7_6
-; DEFAULT-NEXT:  ; %bb.4: ; %sw.bb5
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; DEFAULT-NEXT:    global_load_dword v0, v0, s[2:3]
-; DEFAULT-NEXT:  .LBB7_5: ; %return.sink.split
-; DEFAULT-NEXT:    v_mov_b32_e32 v1, 0
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dword v1, v0, s[6:7]
-; DEFAULT-NEXT:  .LBB7_6: ; %return
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: repeat_successor:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3c
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_lt_i32 s8, 3
+; GFX942-NEXT:    s_cbranch_scc0 .LBB7_3
+; GFX942-NEXT:  ; %bb.1: ; %LeafBlock
+; GFX942-NEXT:    s_cmp_gt_i32 s8, 0
+; GFX942-NEXT:    s_cbranch_scc0 .LBB7_6
+; GFX942-NEXT:  ; %bb.2:
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX942-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX942-NEXT:    s_branch .LBB7_5
+; GFX942-NEXT:  .LBB7_3: ; %LeafBlock5
+; GFX942-NEXT:    s_cmp_eq_u32 s8, 3
+; GFX942-NEXT:    s_cbranch_scc0 .LBB7_6
+; GFX942-NEXT:  ; %bb.4: ; %sw.bb5
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX942-NEXT:    global_load_dword v0, v0, s[2:3]
+; GFX942-NEXT:  .LBB7_5: ; %return.sink.split
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dword v1, v0, s[6:7]
+; GFX942-NEXT:  .LBB7_6: ; %return
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -882,99 +388,37 @@ return:
 }
 
 define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
-; GFX906-LABEL: v8i8_phi_chain:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
-; GFX906-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[8:9]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB8_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; GFX906-NEXT:    s_and_b64 s[4:5], vcc, exec
-; GFX906-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX906-NEXT:  .LBB8_2: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; GFX906-NEXT:    s_cbranch_execz .LBB8_4
-; GFX906-NEXT:  ; %bb.3: ; %bb.2
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[12:13]
-; GFX906-NEXT:  .LBB8_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[14:15]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_phi_chain:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v2
-; FEATURE-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v2
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB8_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[10:11]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
-; FEATURE-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; FEATURE-NEXT:    s_and_b64 s[4:5], vcc, exec
-; FEATURE-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; FEATURE-NEXT:  .LBB8_2: ; %Flow
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; FEATURE-NEXT:    s_cbranch_execz .LBB8_4
-; FEATURE-NEXT:  ; %bb.3: ; %bb.2
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
-; FEATURE-NEXT:  .LBB8_4: ; %bb.3
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dwordx2 v2, v[0:1], s[14:15]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_phi_chain:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v2
-; DEFAULT-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v2
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB8_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[10:11]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
-; DEFAULT-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; DEFAULT-NEXT:    s_and_b64 s[4:5], vcc, exec
-; DEFAULT-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; DEFAULT-NEXT:  .LBB8_2: ; %Flow
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; DEFAULT-NEXT:    s_cbranch_execz .LBB8_4
-; DEFAULT-NEXT:  ; %bb.3: ; %bb.2
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
-; DEFAULT-NEXT:  .LBB8_4: ; %bb.3
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dwordx2 v2, v[0:1], s[14:15]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_phi_chain:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v3, 3, v2
+; GFX942-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v2
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB8_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[10:11]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
+; GFX942-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX942-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX942-NEXT:  .LBB8_2: ; %Flow
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB8_4
+; GFX942-NEXT:  ; %bb.3: ; %bb.2
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
+; GFX942-NEXT:  .LBB8_4: ; %bb.3
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[14:15]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1000,115 +444,42 @@ bb.3:
 
 
 define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
-; GFX906-LABEL: v8i8_phi_zeroinit:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
-; GFX906-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    ; implicit-def: $vgpr3_vgpr4
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v5, s[8:9]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB9_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[3:4], v5, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_waitcnt vmcnt(1)
-; GFX906-NEXT:    v_mov_b32_e32 v1, 0
-; GFX906-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; GFX906-NEXT:    s_and_b64 s[4:5], vcc, exec
-; GFX906-NEXT:    v_mov_b32_e32 v2, v1
-; GFX906-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX906-NEXT:  .LBB9_2: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; GFX906-NEXT:    s_cbranch_execz .LBB9_4
-; GFX906-NEXT:  ; %bb.3: ; %bb.2
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_mov_b32_e32 v4, v2
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    v_mov_b32_e32 v3, v1
-; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[12:13]
-; GFX906-NEXT:  .LBB9_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v0, v[3:4], s[14:15]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_phi_zeroinit:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v5, 3, v4
-; FEATURE-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v4
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v5, s[8:9]
-; FEATURE-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB9_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[2:3], v5, s[10:11]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v4
-; FEATURE-NEXT:    s_waitcnt vmcnt(1)
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; FEATURE-NEXT:    s_and_b64 s[4:5], vcc, exec
-; FEATURE-NEXT:    v_mov_b32_e32 v1, v0
-; FEATURE-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; FEATURE-NEXT:  .LBB9_2: ; %Flow
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; FEATURE-NEXT:    s_cbranch_execz .LBB9_4
-; FEATURE-NEXT:  ; %bb.3: ; %bb.2
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
-; FEATURE-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
-; FEATURE-NEXT:  .LBB9_4: ; %bb.3
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    global_store_dwordx2 v0, v[2:3], s[14:15]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_phi_zeroinit:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v5, 3, v4
-; DEFAULT-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v4
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v5, s[8:9]
-; DEFAULT-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB9_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[2:3], v5, s[10:11]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v4
-; DEFAULT-NEXT:    s_waitcnt vmcnt(1)
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; DEFAULT-NEXT:    s_and_b64 s[4:5], vcc, exec
-; DEFAULT-NEXT:    v_mov_b32_e32 v1, v0
-; DEFAULT-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; DEFAULT-NEXT:  .LBB9_2: ; %Flow
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; DEFAULT-NEXT:    s_cbranch_execz .LBB9_4
-; DEFAULT-NEXT:  ; %bb.3: ; %bb.2
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
-; DEFAULT-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
-; DEFAULT-NEXT:  .LBB9_4: ; %bb.3
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    global_store_dwordx2 v0, v[2:3], s[14:15]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_phi_zeroinit:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v5, 3, v4
+; GFX942-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v4
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v5, s[8:9]
+; GFX942-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB9_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[2:3], v5, s[10:11]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v4
+; GFX942-NEXT:    s_waitcnt vmcnt(1)
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX942-NEXT:    v_mov_b32_e32 v1, v0
+; GFX942-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX942-NEXT:  .LBB9_2: ; %Flow
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB9_4
+; GFX942-NEXT:  ; %bb.3: ; %bb.2
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[12:13]
+; GFX942-NEXT:    v_mov_b64_e32 v[2:3], v[0:1]
+; GFX942-NEXT:  .LBB9_4: ; %bb.3
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[14:15]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1133,264 +504,92 @@ bb.3:
 }
 
 define amdgpu_kernel void @v8i8_phi_const(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
-; GFX906-LABEL: v8i8_phi_const:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
-; GFX906-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    ; implicit-def: $vgpr3
-; GFX906-NEXT:    ; implicit-def: $vgpr13
-; GFX906-NEXT:    ; implicit-def: $vgpr11
-; GFX906-NEXT:    ; implicit-def: $vgpr14
-; GFX906-NEXT:    ; implicit-def: $vgpr15
-; GFX906-NEXT:    ; implicit-def: $vgpr12
-; GFX906-NEXT:    ; implicit-def: $vgpr16
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v4, s[8:9]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
-; GFX906-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
-; GFX906-NEXT:    v_lshrrev_b32_e32 v7, 8, v2
-; GFX906-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
-; GFX906-NEXT:    v_lshrrev_b32_e32 v9, 16, v1
-; GFX906-NEXT:    v_lshrrev_b32_e32 v10, 8, v1
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB10_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[3:4], v4, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; GFX906-NEXT:    s_and_b64 s[4:5], vcc, exec
-; GFX906-NEXT:    v_mov_b32_e32 v5, 8
-; GFX906-NEXT:    v_mov_b32_e32 v6, 7
-; GFX906-NEXT:    v_mov_b32_e32 v7, 6
-; GFX906-NEXT:    v_mov_b32_e32 v2, 5
-; GFX906-NEXT:    v_mov_b32_e32 v8, 4
-; GFX906-NEXT:    v_mov_b32_e32 v9, 3
-; GFX906-NEXT:    v_mov_b32_e32 v10, 2
-; GFX906-NEXT:    v_mov_b32_e32 v1, 1
-; GFX906-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_lshrrev_b32_e32 v16, 24, v4
-; GFX906-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
-; GFX906-NEXT:    v_lshrrev_b32_e32 v15, 8, v4
-; GFX906-NEXT:    v_lshrrev_b32_e32 v14, 24, v3
-; GFX906-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; GFX906-NEXT:    v_lshrrev_b32_e32 v13, 8, v3
-; GFX906-NEXT:  .LBB10_2: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; GFX906-NEXT:    s_cbranch_execz .LBB10_4
-; GFX906-NEXT:  ; %bb.3: ; %bb.2
-; GFX906-NEXT:    v_lshlrev_b16_e32 v3, 8, v10
-; GFX906-NEXT:    v_lshlrev_b16_e32 v4, 8, v8
-; GFX906-NEXT:    v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v4, v9, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_lshlrev_b16_e32 v4, 8, v7
-; GFX906-NEXT:    v_lshlrev_b16_e32 v11, 8, v5
-; GFX906-NEXT:    v_or_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v11, v6, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    v_or_b32_sdwa v4, v4, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    global_store_dwordx2 v0, v[3:4], s[12:13]
-; GFX906-NEXT:    v_mov_b32_e32 v3, v1
-; GFX906-NEXT:    v_mov_b32_e32 v13, v10
-; GFX906-NEXT:    v_mov_b32_e32 v11, v9
-; GFX906-NEXT:    v_mov_b32_e32 v14, v8
-; GFX906-NEXT:    v_mov_b32_e32 v4, v2
-; GFX906-NEXT:    v_mov_b32_e32 v15, v7
-; GFX906-NEXT:    v_mov_b32_e32 v12, v6
-; GFX906-NEXT:    v_mov_b32_e32 v16, v5
-; GFX906-NEXT:  .LBB10_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    v_lshlrev_b16_e32 v0, 8, v13
-; GFX906-NEXT:    v_lshlrev_b16_e32 v1, 8, v14
-; GFX906-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v1, v11, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_lshlrev_b16_e32 v1, 8, v15
-; GFX906-NEXT:    v_lshlrev_b16_e32 v3, 8, v16
-; GFX906-NEXT:    v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v3, v12, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_mov_b32_e32 v2, 0
-; GFX906-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    global_store_dwordx2 v2, v[0:1], s[14:15]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_phi_const:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v16, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v16
-; FEATURE-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v16
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v16
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; FEATURE-NEXT:    ; implicit-def: $vgpr2
-; FEATURE-NEXT:    ; implicit-def: $vgpr12
-; FEATURE-NEXT:    ; implicit-def: $vgpr10
-; FEATURE-NEXT:    ; implicit-def: $vgpr13
-; FEATURE-NEXT:    ; implicit-def: $vgpr14
-; FEATURE-NEXT:    ; implicit-def: $vgpr11
-; FEATURE-NEXT:    ; implicit-def: $vgpr15
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v9, 8, v0
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB10_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[2:3], v3, s[10:11]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v16
-; FEATURE-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; FEATURE-NEXT:    s_and_b64 s[4:5], vcc, exec
-; FEATURE-NEXT:    v_mov_b32_e32 v4, 8
-; FEATURE-NEXT:    v_mov_b32_e32 v5, 7
-; FEATURE-NEXT:    v_mov_b32_e32 v6, 6
-; FEATURE-NEXT:    v_mov_b32_e32 v1, 5
-; FEATURE-NEXT:    v_mov_b32_e32 v7, 4
-; FEATURE-NEXT:    v_mov_b32_e32 v8, 3
-; FEATURE-NEXT:    v_mov_b32_e32 v9, 2
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 1
-; FEATURE-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v15, 24, v3
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v14, 8, v3
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v13, 24, v2
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v12, 8, v2
-; FEATURE-NEXT:  .LBB10_2: ; %Flow
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; FEATURE-NEXT:    s_cbranch_execz .LBB10_4
-; FEATURE-NEXT:  ; %bb.3: ; %bb.2
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v2, 8, v9
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v3, 8, v7
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v8, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v11, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_mov_b32_e32 v10, 0
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    global_store_dwordx2 v10, v[2:3], s[12:13]
-; FEATURE-NEXT:    v_mov_b32_e32 v2, v0
-; FEATURE-NEXT:    v_mov_b32_e32 v12, v9
-; FEATURE-NEXT:    v_mov_b32_e32 v10, v8
-; FEATURE-NEXT:    v_mov_b32_e32 v13, v7
-; FEATURE-NEXT:    v_mov_b32_e32 v3, v1
-; FEATURE-NEXT:    v_mov_b32_e32 v14, v6
-; FEATURE-NEXT:    v_mov_b32_e32 v11, v5
-; FEATURE-NEXT:    v_mov_b32_e32 v15, v4
-; FEATURE-NEXT:  .LBB10_4: ; %bb.3
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v1, 8, v13
-; FEATURE-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v2, 8, v15
-; FEATURE-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v1, 8, v14
-; FEATURE-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v11, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_mov_b32_e32 v4, 0
-; FEATURE-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_phi_const:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v16, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v16
-; DEFAULT-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v16
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v16
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; DEFAULT-NEXT:    ; implicit-def: $vgpr2
-; DEFAULT-NEXT:    ; implicit-def: $vgpr12
-; DEFAULT-NEXT:    ; implicit-def: $vgpr10
-; DEFAULT-NEXT:    ; implicit-def: $vgpr13
-; DEFAULT-NEXT:    ; implicit-def: $vgpr14
-; DEFAULT-NEXT:    ; implicit-def: $vgpr11
-; DEFAULT-NEXT:    ; implicit-def: $vgpr15
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v9, 8, v0
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB10_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[2:3], v3, s[10:11]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v16
-; DEFAULT-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; DEFAULT-NEXT:    s_and_b64 s[4:5], vcc, exec
-; DEFAULT-NEXT:    v_mov_b32_e32 v4, 8
-; DEFAULT-NEXT:    v_mov_b32_e32 v5, 7
-; DEFAULT-NEXT:    v_mov_b32_e32 v6, 6
-; DEFAULT-NEXT:    v_mov_b32_e32 v1, 5
-; DEFAULT-NEXT:    v_mov_b32_e32 v7, 4
-; DEFAULT-NEXT:    v_mov_b32_e32 v8, 3
-; DEFAULT-NEXT:    v_mov_b32_e32 v9, 2
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 1
-; DEFAULT-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v15, 24, v3
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v14, 8, v3
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v13, 24, v2
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v12, 8, v2
-; DEFAULT-NEXT:  .LBB10_2: ; %Flow
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; DEFAULT-NEXT:    s_cbranch_execz .LBB10_4
-; DEFAULT-NEXT:  ; %bb.3: ; %bb.2
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v2, 8, v9
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v3, 8, v7
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v8, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v11, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_mov_b32_e32 v10, 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    global_store_dwordx2 v10, v[2:3], s[12:13]
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, v0
-; DEFAULT-NEXT:    v_mov_b32_e32 v12, v9
-; DEFAULT-NEXT:    v_mov_b32_e32 v10, v8
-; DEFAULT-NEXT:    v_mov_b32_e32 v13, v7
-; DEFAULT-NEXT:    v_mov_b32_e32 v3, v1
-; DEFAULT-NEXT:    v_mov_b32_e32 v14, v6
-; DEFAULT-NEXT:    v_mov_b32_e32 v11, v5
-; DEFAULT-NEXT:    v_mov_b32_e32 v15, v4
-; DEFAULT-NEXT:  .LBB10_4: ; %bb.3
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v1, 8, v13
-; DEFAULT-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v2, 8, v15
-; DEFAULT-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v1, 8, v14
-; DEFAULT-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v11, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_mov_b32_e32 v4, 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_phi_const:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v16, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v3, 3, v16
+; GFX942-NEXT:    v_cmp_lt_u32_e64 s[0:1], 14, v16
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v16
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
+; GFX942-NEXT:    ; implicit-def: $vgpr2
+; GFX942-NEXT:    ; implicit-def: $vgpr12
+; GFX942-NEXT:    ; implicit-def: $vgpr10
+; GFX942-NEXT:    ; implicit-def: $vgpr13
+; GFX942-NEXT:    ; implicit-def: $vgpr14
+; GFX942-NEXT:    ; implicit-def: $vgpr11
+; GFX942-NEXT:    ; implicit-def: $vgpr15
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshrrev_b32_e32 v4, 24, v1
+; GFX942-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
+; GFX942-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX942-NEXT:    v_lshrrev_b32_e32 v7, 24, v0
+; GFX942-NEXT:    v_lshrrev_b32_e32 v8, 16, v0
+; GFX942-NEXT:    v_lshrrev_b32_e32 v9, 8, v0
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB10_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[2:3], v3, s[10:11]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v16
+; GFX942-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT:    s_and_b64 s[4:5], vcc, exec
+; GFX942-NEXT:    v_mov_b32_e32 v4, 8
+; GFX942-NEXT:    v_mov_b32_e32 v5, 7
+; GFX942-NEXT:    v_mov_b32_e32 v6, 6
+; GFX942-NEXT:    v_mov_b32_e32 v1, 5
+; GFX942-NEXT:    v_mov_b32_e32 v7, 4
+; GFX942-NEXT:    v_mov_b32_e32 v8, 3
+; GFX942-NEXT:    v_mov_b32_e32 v9, 2
+; GFX942-NEXT:    v_mov_b32_e32 v0, 1
+; GFX942-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshrrev_b32_e32 v15, 24, v3
+; GFX942-NEXT:    v_lshrrev_b32_e32 v11, 16, v3
+; GFX942-NEXT:    v_lshrrev_b32_e32 v14, 8, v3
+; GFX942-NEXT:    v_lshrrev_b32_e32 v13, 24, v2
+; GFX942-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
+; GFX942-NEXT:    v_lshrrev_b32_e32 v12, 8, v2
+; GFX942-NEXT:  .LBB10_2: ; %Flow
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
+; GFX942-NEXT:    s_cbranch_execz .LBB10_4
+; GFX942-NEXT:  ; %bb.3: ; %bb.2
+; GFX942-NEXT:    v_lshlrev_b16_e32 v2, 8, v9
+; GFX942-NEXT:    v_lshlrev_b16_e32 v3, 8, v7
+; GFX942-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v3, v8, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_e32 v11, 8, v4
+; GFX942-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_e32 v3, 8, v6
+; GFX942-NEXT:    v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v11, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_mov_b32_e32 v10, 0
+; GFX942-NEXT:    v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    global_store_dwordx2 v10, v[2:3], s[12:13]
+; GFX942-NEXT:    v_mov_b32_e32 v2, v0
+; GFX942-NEXT:    v_mov_b32_e32 v12, v9
+; GFX942-NEXT:    v_mov_b32_e32 v10, v8
+; GFX942-NEXT:    v_mov_b32_e32 v13, v7
+; GFX942-NEXT:    v_mov_b32_e32 v3, v1
+; GFX942-NEXT:    v_mov_b32_e32 v14, v6
+; GFX942-NEXT:    v_mov_b32_e32 v11, v5
+; GFX942-NEXT:    v_mov_b32_e32 v15, v4
+; GFX942-NEXT:  .LBB10_4: ; %bb.3
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:    v_lshlrev_b16_e32 v0, 8, v12
+; GFX942-NEXT:    v_lshlrev_b16_e32 v1, 8, v13
+; GFX942-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v1, v10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_e32 v2, 8, v15
+; GFX942-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_e32 v1, 8, v14
+; GFX942-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v2, v11, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1415,91 +614,34 @@ bb.3:
 }
 
 define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst0, ptr addrspace(1) nocapture %dst1) {
-; GFX906-LABEL: v8i8_multi_block:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
-; GFX906-NEXT:    v_mov_b32_e32 v5, 0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[3:4], v6, s[8:9]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_mov_b32_e32 v1, v3
-; GFX906-NEXT:    v_mov_b32_e32 v2, v4
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB11_4
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v6, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB11_3
-; GFX906-NEXT:  ; %bb.2: ; %bb.2
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    global_store_dwordx2 v0, v[3:4], s[12:13]
-; GFX906-NEXT:  .LBB11_3: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:  .LBB11_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v5, v[1:2], s[14:15]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_multi_block:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v5, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v6, 3, v5
-; FEATURE-NEXT:    v_mov_b32_e32 v4, 0
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v5
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[2:3], v6, s[8:9]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_mov_b64_e32 v[0:1], v[2:3]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB11_4
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v6, s[10:11]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v5
-; FEATURE-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB11_3
-; FEATURE-NEXT:  ; %bb.2: ; %bb.2
-; FEATURE-NEXT:    v_mov_b32_e32 v5, 0
-; FEATURE-NEXT:    global_store_dwordx2 v5, v[2:3], s[12:13]
-; FEATURE-NEXT:  .LBB11_3: ; %Flow
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[2:3]
-; FEATURE-NEXT:  .LBB11_4: ; %bb.3
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_multi_block:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v5, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v6, 3, v5
-; DEFAULT-NEXT:    v_mov_b32_e32 v4, 0
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v5
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[2:3], v6, s[8:9]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_mov_b64_e32 v[0:1], v[2:3]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB11_4
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v6, s[10:11]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v5
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB11_3
-; DEFAULT-NEXT:  ; %bb.2: ; %bb.2
-; DEFAULT-NEXT:    v_mov_b32_e32 v5, 0
-; DEFAULT-NEXT:    global_store_dwordx2 v5, v[2:3], s[12:13]
-; DEFAULT-NEXT:  .LBB11_3: ; %Flow
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[2:3]
-; DEFAULT-NEXT:  .LBB11_4: ; %bb.3
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_multi_block:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v5, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v6, 3, v5
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v5
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[2:3], v6, s[8:9]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b64_e32 v[0:1], v[2:3]
+; GFX942-NEXT:    s_and_saveexec_b64 s[0:1], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB11_4
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v6, s[10:11]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v5
+; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB11_3
+; GFX942-NEXT:  ; %bb.2: ; %bb.2
+; GFX942-NEXT:    v_mov_b32_e32 v5, 0
+; GFX942-NEXT:    global_store_dwordx2 v5, v[2:3], s[12:13]
+; GFX942-NEXT:  .LBB11_3: ; %Flow
+; GFX942-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX942-NEXT:  .LBB11_4: ; %bb.3
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[14:15]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1523,84 +665,32 @@ bb.3:
 }
 
 define amdgpu_kernel void @v32i8_loop_carried(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
-; GFX906-LABEL: v32i8_loop_carried:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v1, 5, v0
-; GFX906-NEXT:    v_cmp_lt_u32_e32 vcc, 14, v0
-; GFX906-NEXT:    s_mov_b32 s2, 0x2000604
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dword v1, v1, s[0:1]
-; GFX906-NEXT:    s_mov_b64 s[0:1], 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_mov_b32_e32 v0, v1
-; GFX906-NEXT:  .LBB12_1: ; %bb.1
-; GFX906-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX906-NEXT:    s_and_b64 s[6:7], exec, vcc
-; GFX906-NEXT:    s_or_b64 s[0:1], s[6:7], s[0:1]
-; GFX906-NEXT:    v_perm_b32 v0, v1, v0, s2
-; GFX906-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_cbranch_execnz .LBB12_1
-; GFX906-NEXT:  ; %bb.2: ; %bb.2.loopexit
-; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX906-NEXT:    v_mov_b32_e32 v1, 0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_store_dword v1, v0, s[0:1]
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v32i8_loop_carried:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v0, 5, v1
-; FEATURE-NEXT:    v_cmp_lt_u32_e32 vcc, 14, v1
-; FEATURE-NEXT:    s_mov_b32 s2, 0x2000604
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dword v0, v0, s[0:1]
-; FEATURE-NEXT:    s_mov_b64 s[0:1], 0
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_mov_b32_e32 v1, v0
-; FEATURE-NEXT:  .LBB12_1: ; %bb.1
-; FEATURE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; FEATURE-NEXT:    s_and_b64 s[6:7], exec, vcc
-; FEATURE-NEXT:    s_or_b64 s[0:1], s[6:7], s[0:1]
-; FEATURE-NEXT:    v_perm_b32 v1, v0, v1, s2
-; FEATURE-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_cbranch_execnz .LBB12_1
-; FEATURE-NEXT:  ; %bb.2: ; %bb.2.loopexit
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
-; FEATURE-NEXT:    v_mov_b32_e32 v0, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_store_dword v0, v1, s[0:1]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v32i8_loop_carried:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v0, 5, v1
-; DEFAULT-NEXT:    v_cmp_lt_u32_e32 vcc, 14, v1
-; DEFAULT-NEXT:    s_mov_b32 s2, 0x2000604
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dword v0, v0, s[0:1]
-; DEFAULT-NEXT:    s_mov_b64 s[0:1], 0
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_mov_b32_e32 v1, v0
-; DEFAULT-NEXT:  .LBB12_1: ; %bb.1
-; DEFAULT-NEXT:    ; =>This Inner Loop Header: Depth=1
-; DEFAULT-NEXT:    s_and_b64 s[6:7], exec, vcc
-; DEFAULT-NEXT:    s_or_b64 s[0:1], s[6:7], s[0:1]
-; DEFAULT-NEXT:    v_perm_b32 v1, v0, v1, s2
-; DEFAULT-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_cbranch_execnz .LBB12_1
-; DEFAULT-NEXT:  ; %bb.2: ; %bb.2.loopexit
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
-; DEFAULT-NEXT:    v_mov_b32_e32 v0, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_store_dword v0, v1, s[0:1]
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v32i8_loop_carried:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v1, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 5, v1
+; GFX942-NEXT:    v_cmp_lt_u32_e32 vcc, 14, v1
+; GFX942-NEXT:    s_mov_b32 s2, 0x2000604
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dword v0, v0, s[0:1]
+; GFX942-NEXT:    s_mov_b64 s[0:1], 0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v1, v0
+; GFX942-NEXT:  .LBB12_1: ; %bb.1
+; GFX942-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT:    s_and_b64 s[6:7], exec, vcc
+; GFX942-NEXT:    s_or_b64 s[0:1], s[6:7], s[0:1]
+; GFX942-NEXT:    v_perm_b32 v1, v0, v1, s2
+; GFX942-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_cbranch_execnz .LBB12_1
+; GFX942-NEXT:  ; %bb.2: ; %bb.2.loopexit
+; GFX942-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <32 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1622,264 +712,92 @@ bb.2:
 ; Should not have instances of "Instruction does not dominate all uses!"
 
 define amdgpu_kernel void @v8i8_multiuse_multiblock(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst1, ptr addrspace(1) nocapture %dst2, ptr addrspace(1) nocapture %dst3) {
-; GFX906-LABEL: v8i8_multiuse_multiblock:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v1, 3, v0
-; GFX906-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
-; GFX906-NEXT:    v_cmp_lt_u32_e64 s[2:3], 14, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v1, s[8:9]
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX906-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB13_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    s_movk_i32 s6, 0xff00
-; GFX906-NEXT:    v_mov_b32_e32 v5, 8
-; GFX906-NEXT:    v_and_b32_sdwa v6, v1, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX906-NEXT:    s_mov_b32 s6, 0x6070504
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    v_and_b32_e32 v4, 0xffffff00, v1
-; GFX906-NEXT:    v_lshlrev_b16_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX906-NEXT:    v_perm_b32 v7, v1, v1, s6
-; GFX906-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; GFX906-NEXT:    s_and_b64 s[6:7], vcc, exec
-; GFX906-NEXT:    v_mov_b32_e32 v3, 0
-; GFX906-NEXT:    v_or_b32_sdwa v4, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v6, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX906-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
-; GFX906-NEXT:    v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    global_store_dword v3, v1, s[12:13]
-; GFX906-NEXT:    global_store_dword v3, v7, s[12:13] offset:8
-; GFX906-NEXT:    global_store_dword v3, v6, s[12:13] offset:16
-; GFX906-NEXT:    global_store_dword v3, v4, s[12:13] offset:24
-; GFX906-NEXT:  .LBB13_2: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX906-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
-; GFX906-NEXT:    s_cbranch_execz .LBB13_4
-; GFX906-NEXT:  ; %bb.3: ; %bb.2
-; GFX906-NEXT:    v_lshlrev_b16_e32 v3, 8, v2
-; GFX906-NEXT:    v_and_b32_e32 v4, 0xffffff00, v2
-; GFX906-NEXT:    v_and_b32_e32 v5, 0xffffff00, v1
-; GFX906-NEXT:    s_mov_b32 s2, 0xc0c0001
-; GFX906-NEXT:    v_or_b32_sdwa v3, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v5, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_perm_b32 v2, 0, v2, s2
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_perm_b32 v6, 0, v1, s2
-; GFX906-NEXT:    s_mov_b32 s3, 0xffff0000
-; GFX906-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
-; GFX906-NEXT:    v_and_or_b32 v7, v1, s3, v6
-; GFX906-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_e32 v2, v6, v2
-; GFX906-NEXT:    global_store_dword v0, v3, s[14:15]
-; GFX906-NEXT:    global_store_dword v0, v4, s[14:15] offset:8
-; GFX906-NEXT:    global_store_dword v0, v7, s[14:15] offset:16
-; GFX906-NEXT:    global_store_dword v0, v2, s[14:15] offset:24
-; GFX906-NEXT:  .LBB13_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX906-NEXT:    s_movk_i32 s3, 0xff00
-; GFX906-NEXT:    v_mov_b32_e32 v4, 8
-; GFX906-NEXT:    s_movk_i32 s2, 0xff
-; GFX906-NEXT:    v_and_b32_sdwa v2, v1, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX906-NEXT:    v_lshlrev_b16_sdwa v4, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX906-NEXT:    v_or_b32_sdwa v3, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v5, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX906-NEXT:    v_lshlrev_b16_e32 v6, 8, v1
-; GFX906-NEXT:    v_and_b32_sdwa v7, v1, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v7, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v4, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v1, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX906-NEXT:    global_store_dword v0, v3, s[0:1]
-; GFX906-NEXT:    global_store_dword v0, v1, s[0:1] offset:8
-; GFX906-NEXT:    global_store_dword v0, v4, s[0:1] offset:16
-; GFX906-NEXT:    global_store_dword v0, v2, s[0:1] offset:24
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_multiuse_multiblock:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v0, 3, v2
-; FEATURE-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
-; FEATURE-NEXT:    v_cmp_lt_u32_e64 s[2:3], 14, v2
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v0, s[8:9]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; FEATURE-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB13_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    s_movk_i32 s6, 0xff00
-; FEATURE-NEXT:    v_mov_b32_e32 v5, 8
-; FEATURE-NEXT:    v_and_b32_sdwa v6, v0, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; FEATURE-NEXT:    s_mov_b32 s6, 0x6070504
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0xffffff00, v0
-; FEATURE-NEXT:    v_lshlrev_b16_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; FEATURE-NEXT:    v_perm_b32 v7, v0, v0, s6
-; FEATURE-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; FEATURE-NEXT:    s_and_b64 s[6:7], vcc, exec
-; FEATURE-NEXT:    v_mov_b32_e32 v3, 0
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; FEATURE-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
-; FEATURE-NEXT:    v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    global_store_dword v3, v0, s[12:13]
-; FEATURE-NEXT:    global_store_dword v3, v7, s[12:13] offset:8
-; FEATURE-NEXT:    global_store_dword v3, v6, s[12:13] offset:16
-; FEATURE-NEXT:    global_store_dword v3, v4, s[12:13] offset:24
-; FEATURE-NEXT:  .LBB13_2: ; %Flow
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[4:5]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
-; FEATURE-NEXT:    s_cbranch_execz .LBB13_4
-; FEATURE-NEXT:  ; %bb.3: ; %bb.2
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0xffffff00, v1
-; FEATURE-NEXT:    v_and_b32_e32 v5, 0xffffff00, v0
-; FEATURE-NEXT:    s_mov_b32 s2, 0xc0c0001
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v5, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_perm_b32 v1, 0, v1, s2
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_perm_b32 v6, 0, v0, s2
-; FEATURE-NEXT:    s_mov_b32 s3, 0xffff0000
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; FEATURE-NEXT:    v_and_or_b32 v7, v0, s3, v6
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_e32 v1, v6, v1
-; FEATURE-NEXT:    global_store_dword v2, v3, s[14:15]
-; FEATURE-NEXT:    global_store_dword v2, v4, s[14:15] offset:8
-; FEATURE-NEXT:    global_store_dword v2, v7, s[14:15] offset:16
-; FEATURE-NEXT:    global_store_dword v2, v1, s[14:15] offset:24
-; FEATURE-NEXT:  .LBB13_4: ; %bb.3
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[4:5]
-; FEATURE-NEXT:    s_movk_i32 s3, 0xff00
-; FEATURE-NEXT:    v_mov_b32_e32 v4, 8
-; FEATURE-NEXT:    s_movk_i32 s2, 0xff
-; FEATURE-NEXT:    v_and_b32_sdwa v2, v0, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v5, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; FEATURE-NEXT:    v_lshlrev_b16_e32 v6, 8, v0
-; FEATURE-NEXT:    v_and_b32_sdwa v7, v0, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; FEATURE-NEXT:    v_mov_b32_e32 v1, 0
-; FEATURE-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v7, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; FEATURE-NEXT:    global_store_dword v1, v3, s[0:1]
-; FEATURE-NEXT:    global_store_dword v1, v0, s[0:1] offset:8
-; FEATURE-NEXT:    global_store_dword v1, v4, s[0:1] offset:16
-; FEATURE-NEXT:    global_store_dword v1, v2, s[0:1] offset:24
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_multiuse_multiblock:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v0, 3, v2
-; DEFAULT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
-; DEFAULT-NEXT:    v_cmp_lt_u32_e64 s[2:3], 14, v2
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v0, s[8:9]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB13_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    s_movk_i32 s6, 0xff00
-; DEFAULT-NEXT:    v_mov_b32_e32 v5, 8
-; DEFAULT-NEXT:    v_and_b32_sdwa v6, v0, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; DEFAULT-NEXT:    s_mov_b32 s6, 0x6070504
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0xffffff00, v0
-; DEFAULT-NEXT:    v_lshlrev_b16_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; DEFAULT-NEXT:    v_perm_b32 v7, v0, v0, s6
-; DEFAULT-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
-; DEFAULT-NEXT:    s_and_b64 s[6:7], vcc, exec
-; DEFAULT-NEXT:    v_mov_b32_e32 v3, 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; DEFAULT-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
-; DEFAULT-NEXT:    v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    global_store_dword v3, v0, s[12:13]
-; DEFAULT-NEXT:    global_store_dword v3, v7, s[12:13] offset:8
-; DEFAULT-NEXT:    global_store_dword v3, v6, s[12:13] offset:16
-; DEFAULT-NEXT:    global_store_dword v3, v4, s[12:13] offset:24
-; DEFAULT-NEXT:  .LBB13_2: ; %Flow
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[4:5]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
-; DEFAULT-NEXT:    s_cbranch_execz .LBB13_4
-; DEFAULT-NEXT:  ; %bb.3: ; %bb.2
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0xffffff00, v1
-; DEFAULT-NEXT:    v_and_b32_e32 v5, 0xffffff00, v0
-; DEFAULT-NEXT:    s_mov_b32 s2, 0xc0c0001
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v5, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_perm_b32 v1, 0, v1, s2
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_perm_b32 v6, 0, v0, s2
-; DEFAULT-NEXT:    s_mov_b32 s3, 0xffff0000
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; DEFAULT-NEXT:    v_and_or_b32 v7, v0, s3, v6
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_e32 v1, v6, v1
-; DEFAULT-NEXT:    global_store_dword v2, v3, s[14:15]
-; DEFAULT-NEXT:    global_store_dword v2, v4, s[14:15] offset:8
-; DEFAULT-NEXT:    global_store_dword v2, v7, s[14:15] offset:16
-; DEFAULT-NEXT:    global_store_dword v2, v1, s[14:15] offset:24
-; DEFAULT-NEXT:  .LBB13_4: ; %bb.3
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[4:5]
-; DEFAULT-NEXT:    s_movk_i32 s3, 0xff00
-; DEFAULT-NEXT:    v_mov_b32_e32 v4, 8
-; DEFAULT-NEXT:    s_movk_i32 s2, 0xff
-; DEFAULT-NEXT:    v_and_b32_sdwa v2, v0, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v5, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; DEFAULT-NEXT:    v_lshlrev_b16_e32 v6, 8, v0
-; DEFAULT-NEXT:    v_and_b32_sdwa v7, v0, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; DEFAULT-NEXT:    v_mov_b32_e32 v1, 0
-; DEFAULT-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v7, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; DEFAULT-NEXT:    global_store_dword v1, v3, s[0:1]
-; DEFAULT-NEXT:    global_store_dword v1, v0, s[0:1] offset:8
-; DEFAULT-NEXT:    global_store_dword v1, v4, s[0:1] offset:16
-; DEFAULT-NEXT:    global_store_dword v1, v2, s[0:1] offset:24
-; DEFAULT-NEXT:    s_endpgm
+; GFX942-LABEL: v8i8_multiuse_multiblock:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 3, v2
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x44
+; GFX942-NEXT:    v_cmp_lt_u32_e64 s[2:3], 14, v2
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v0, s[8:9]
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v2
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
+; GFX942-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX942-NEXT:    s_cbranch_execz .LBB13_2
+; GFX942-NEXT:  ; %bb.1: ; %bb.1
+; GFX942-NEXT:    s_movk_i32 s6, 0xff00
+; GFX942-NEXT:    v_mov_b32_e32 v5, 8
+; GFX942-NEXT:    v_and_b32_sdwa v6, v0, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX942-NEXT:    s_mov_b32 s6, 0x6070504
+; GFX942-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v2
+; GFX942-NEXT:    v_and_b32_e32 v4, 0xffffff00, v0
+; GFX942-NEXT:    v_lshlrev_b16_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT:    v_perm_b32 v7, v0, v0, s6
+; GFX942-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; GFX942-NEXT:    s_and_b64 s[6:7], vcc, exec
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0
+; GFX942-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v6, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX942-NEXT:    s_or_b64 s[2:3], s[2:3], s[6:7]
+; GFX942-NEXT:    v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    global_store_dword v3, v0, s[12:13]
+; GFX942-NEXT:    global_store_dword v3, v7, s[12:13] offset:8
+; GFX942-NEXT:    global_store_dword v3, v6, s[12:13] offset:16
+; GFX942-NEXT:    global_store_dword v3, v4, s[12:13] offset:24
+; GFX942-NEXT:  .LBB13_2: ; %Flow
+; GFX942-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT:    s_and_saveexec_b64 s[4:5], s[2:3]
+; GFX942-NEXT:    s_cbranch_execz .LBB13_4
+; GFX942-NEXT:  ; %bb.3: ; %bb.2
+; GFX942-NEXT:    v_lshlrev_b16_e32 v3, 8, v1
+; GFX942-NEXT:    v_and_b32_e32 v4, 0xffffff00, v1
+; GFX942-NEXT:    v_and_b32_e32 v5, 0xffffff00, v0
+; GFX942-NEXT:    s_mov_b32 s2, 0xc0c0001
+; GFX942-NEXT:    v_or_b32_sdwa v3, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v5, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_perm_b32 v1, 0, v1, s2
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_perm_b32 v6, 0, v0, s2
+; GFX942-NEXT:    s_mov_b32 s3, 0xffff0000
+; GFX942-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX942-NEXT:    v_and_or_b32 v7, v0, s3, v6
+; GFX942-NEXT:    v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_e32 v1, v6, v1
+; GFX942-NEXT:    global_store_dword v2, v3, s[14:15]
+; GFX942-NEXT:    global_store_dword v2, v4, s[14:15] offset:8
+; GFX942-NEXT:    global_store_dword v2, v7, s[14:15] offset:16
+; GFX942-NEXT:    global_store_dword v2, v1, s[14:15] offset:24
+; GFX942-NEXT:  .LBB13_4: ; %bb.3
+; GFX942-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX942-NEXT:    s_movk_i32 s3, 0xff00
+; GFX942-NEXT:    v_mov_b32_e32 v4, 8
+; GFX942-NEXT:    s_movk_i32 s2, 0xff
+; GFX942-NEXT:    v_and_b32_sdwa v2, v0, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT:    v_or_b32_sdwa v3, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v5, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX942-NEXT:    v_lshlrev_b16_e32 v6, 8, v0
+; GFX942-NEXT:    v_and_b32_sdwa v7, v0, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0
+; GFX942-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v7, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v4, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX942-NEXT:    global_store_dword v1, v3, s[0:1]
+; GFX942-NEXT:    global_store_dword v1, v0, s[0:1] offset:8
+; GFX942-NEXT:    global_store_dword v1, v4, s[0:1] offset:16
+; GFX942-NEXT:    global_store_dword v1, v2, s[0:1] offset:24
+; GFX942-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -1935,138 +853,6 @@ bb.3:
   ret void
 }
 
-; This should not cause Assertion `getType() == V->getType() && "All operands to PHI node must be the same type as the PHI node
-; Note: whether or not the assertion fires depends on the iteration ortder of PhiNodes in AMDGPULateCodeGenPrepare, which
-; is non-deterministic due to iterators over a set of pointers.
-
-define amdgpu_kernel void @MissingInc_PhiChain(i1 %cmp1.i.i.i.i.i.not, <16 x i8> %promotealloca31.i.i.i.i) {
-; GFX906-LABEL: MissingInc_PhiChain:
-; GFX906:       ; %bb.0: ; %entry
-; GFX906-NEXT:    s_load_dword s0, s[0:1], 0x24
-; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    s_bitcmp1_b32 s0, 0
-; GFX906-NEXT:    s_cselect_b64 s[2:3], -1, 0
-; GFX906-NEXT:    s_xor_b64 s[0:1], s[2:3], -1
-; GFX906-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX906-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; GFX906-NEXT:    s_branch .LBB14_2
-; GFX906-NEXT:  .LBB14_1: ; %Flow1
-; GFX906-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; GFX906-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; GFX906-NEXT:    s_cbranch_vccnz .LBB14_6
-; GFX906-NEXT:  .LBB14_2: ; %for.body10.i.i.i.i
-; GFX906-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX906-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; GFX906-NEXT:    s_mov_b64 s[4:5], s[2:3]
-; GFX906-NEXT:    s_cbranch_vccnz .LBB14_4
-; GFX906-NEXT:  ; %bb.3: ; %if.then.i.i.i.i.i
-; GFX906-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; GFX906-NEXT:    s_mov_b64 s[4:5], -1
-; GFX906-NEXT:  .LBB14_4: ; %Flow
-; GFX906-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; GFX906-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; GFX906-NEXT:    s_mov_b64 s[4:5], -1
-; GFX906-NEXT:    s_cbranch_vccnz .LBB14_1
-; GFX906-NEXT:  ; %bb.5: ; %if.end.i.i.i.i.i
-; GFX906-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; GFX906-NEXT:    s_mov_b64 s[4:5], 0
-; GFX906-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; GFX906-NEXT:    s_branch .LBB14_1
-; GFX906-NEXT:  .LBB14_6: ; %DummyReturnBlock
-; GFX906-NEXT:    s_endpgm
-; FEATURE-LABEL: MissingInc_PhiChain:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dword s0, s[4:5], 0x24
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    s_bitcmp1_b32 s0, 0
-; FEATURE-NEXT:    s_cselect_b64 s[2:3], -1, 0
-; FEATURE-NEXT:    s_xor_b64 s[0:1], s[2:3], -1
-; FEATURE-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; FEATURE-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; FEATURE-NEXT:    s_branch .LBB14_2
-; FEATURE-NEXT:  .LBB14_1: ; %Flow1
-; FEATURE-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; FEATURE-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; FEATURE-NEXT:    s_cbranch_vccnz .LBB14_6
-; FEATURE-NEXT:  .LBB14_2: ; %for.body10.i.i.i.i
-; FEATURE-NEXT:    ; =>This Inner Loop Header: Depth=1
-; FEATURE-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; FEATURE-NEXT:    s_mov_b64 s[4:5], s[2:3]
-; FEATURE-NEXT:    s_cbranch_vccnz .LBB14_4
-; FEATURE-NEXT:  ; %bb.3: ; %if.then.i.i.i.i.i
-; FEATURE-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; FEATURE-NEXT:    s_mov_b64 s[4:5], -1
-; FEATURE-NEXT:  .LBB14_4: ; %Flow
-; FEATURE-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; FEATURE-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; FEATURE-NEXT:    s_mov_b64 s[4:5], -1
-; FEATURE-NEXT:    s_cbranch_vccnz .LBB14_1
-; FEATURE-NEXT:  ; %bb.5: ; %if.end.i.i.i.i.i
-; FEATURE-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; FEATURE-NEXT:    s_mov_b64 s[4:5], 0
-; FEATURE-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; FEATURE-NEXT:    s_branch .LBB14_1
-; FEATURE-NEXT:  .LBB14_6: ; %DummyReturnBlock
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: MissingInc_PhiChain:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dword s0, s[4:5], 0x24
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    s_bitcmp1_b32 s0, 0
-; DEFAULT-NEXT:    s_cselect_b64 s[2:3], -1, 0
-; DEFAULT-NEXT:    s_xor_b64 s[0:1], s[2:3], -1
-; DEFAULT-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; DEFAULT-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
-; DEFAULT-NEXT:    s_branch .LBB14_2
-; DEFAULT-NEXT:  .LBB14_1: ; %Flow1
-; DEFAULT-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; DEFAULT-NEXT:    s_and_b64 vcc, exec, s[4:5]
-; DEFAULT-NEXT:    s_cbranch_vccnz .LBB14_6
-; DEFAULT-NEXT:  .LBB14_2: ; %for.body10.i.i.i.i
-; DEFAULT-NEXT:    ; =>This Inner Loop Header: Depth=1
-; DEFAULT-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; DEFAULT-NEXT:    s_mov_b64 s[4:5], s[2:3]
-; DEFAULT-NEXT:    s_cbranch_vccnz .LBB14_4
-; DEFAULT-NEXT:  ; %bb.3: ; %if.then.i.i.i.i.i
-; DEFAULT-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; DEFAULT-NEXT:    s_mov_b64 s[4:5], -1
-; DEFAULT-NEXT:  .LBB14_4: ; %Flow
-; DEFAULT-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; DEFAULT-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
-; DEFAULT-NEXT:    s_mov_b64 s[4:5], -1
-; DEFAULT-NEXT:    s_cbranch_vccnz .LBB14_1
-; DEFAULT-NEXT:  ; %bb.5: ; %if.end.i.i.i.i.i
-; DEFAULT-NEXT:    ; in Loop: Header=BB14_2 Depth=1
-; DEFAULT-NEXT:    s_mov_b64 s[4:5], 0
-; DEFAULT-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; DEFAULT-NEXT:    s_branch .LBB14_1
-; DEFAULT-NEXT:  .LBB14_6: ; %DummyReturnBlock
-; DEFAULT-NEXT:    s_endpgm
-entry:
-  br label %for.body10.i.i.i.i
-
-for.body10.i.i.i.i:                               ; preds = %if.end.1.i.i.i.i.i, %entry
-  %promotealloca3237.i.i.i.i = phi <16 x i8> [ <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, %entry ], [ %1, %if.end.1.i.i.i.i.i ]
-  br i1 %cmp1.i.i.i.i.i.not, label %if.end.i.i.i.i.i, label %if.then.i.i.i.i.i
-
-if.then.i.i.i.i.i:                                ; preds = %for.body10.i.i.i.i
-  %0 = insertelement <16 x i8> %promotealloca3237.i.i.i.i, i8 0, i64 0
-  br label %if.end.i.i.i.i.i
-
-if.end.i.i.i.i.i:                                 ; preds = %if.then.i.i.i.i.i, %for.body10.i.i.i.i
-  %promotealloca31.i.i.i.i3 = phi <16 x i8> [ %0, %if.then.i.i.i.i.i ], [ %promotealloca3237.i.i.i.i, %for.body10.i.i.i.i ]
-  br i1 %cmp1.i.i.i.i.i.not, label %if.end.1.i.i.i.i.i, label %if.then.1.i.i.i.i.i
-
-if.then.1.i.i.i.i.i:                              ; preds = %if.end.i.i.i.i.i
-  br label %if.end.1.i.i.i.i.i
-
-if.end.1.i.i.i.i.i:                               ; preds = %if.then.1.i.i.i.i.i, %if.end.i.i.i.i.i
-  %promotealloca30.i.i.i.i = phi <16 x i8> [ %promotealloca31.i.i.i.i, %if.then.1.i.i.i.i.i ], [ %promotealloca31.i.i.i.i3, %if.end.i.i.i.i.i ]
-  %1 = shufflevector <16 x i8> %promotealloca30.i.i.i.i, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
-  br label %for.body10.i.i.i.i
-}
-
 
 define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst, ptr addrspace(1) %arg) {
 ; GFX942-LABEL: v8i8_mfma_i8:
@@ -2096,61 +882,6 @@ define amdgpu_kernel void @v8i8_mfma_i8(ptr addrspace(1) %src1, ptr addrspace(1)
 ; GFX942-NEXT:    s_nop 6
 ; GFX942-NEXT:    global_store_dwordx4 v2, a[0:3], s[12:13]
 ; GFX942-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_mfma_i8:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB15_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[10:11]
-; FEATURE-NEXT:  .LBB15_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[14:15], 0x0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    v_accvgpr_write_b32 a0, s0
-; FEATURE-NEXT:    v_accvgpr_write_b32 a1, s1
-; FEATURE-NEXT:    v_accvgpr_write_b32 a2, s2
-; FEATURE-NEXT:    v_accvgpr_write_b32 a3, s3
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    s_nop 0
-; FEATURE-NEXT:    v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
-; FEATURE-NEXT:    s_nop 6
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[0:3], s[12:13]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_mfma_i8:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[8:9]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB15_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[10:11]
-; DEFAULT-NEXT:  .LBB15_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[14:15], 0x0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a0, s0
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a1, s1
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a2, s2
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a3, s3
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    s_nop 0
-; DEFAULT-NEXT:    v_mfma_i32_16x16x32_i8 a[0:3], v[0:1], v[0:1], a[0:3] cbsz:1 abid:2 blgp:3
-; DEFAULT-NEXT:    s_nop 6
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[0:3], s[12:13]
-; DEFAULT-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -2239,137 +970,6 @@ define amdgpu_kernel void @v8i8_mfma_half(ptr addrspace(1) %src1, ptr addrspace(
 ; GFX942-NEXT:    global_store_dwordx4 v2, a[4:7], s[40:41] offset:16
 ; GFX942-NEXT:    global_store_dwordx4 v2, a[0:3], s[40:41]
 ; GFX942-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_mfma_half:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx8 s[36:43], s[4:5], 0x24
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[36:37]
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB16_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[38:39]
-; FEATURE-NEXT:  .LBB16_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_load_dwordx16 s[16:31], s[42:43], 0x0
-; FEATURE-NEXT:    s_load_dwordx16 s[0:15], s[42:43], 0x40
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    v_accvgpr_write_b32 a0, s16
-; FEATURE-NEXT:    v_accvgpr_write_b32 a1, s17
-; FEATURE-NEXT:    v_accvgpr_write_b32 a2, s18
-; FEATURE-NEXT:    v_accvgpr_write_b32 a3, s19
-; FEATURE-NEXT:    v_accvgpr_write_b32 a4, s20
-; FEATURE-NEXT:    v_accvgpr_write_b32 a5, s21
-; FEATURE-NEXT:    v_accvgpr_write_b32 a6, s22
-; FEATURE-NEXT:    v_accvgpr_write_b32 a7, s23
-; FEATURE-NEXT:    v_accvgpr_write_b32 a8, s24
-; FEATURE-NEXT:    v_accvgpr_write_b32 a9, s25
-; FEATURE-NEXT:    v_accvgpr_write_b32 a10, s26
-; FEATURE-NEXT:    v_accvgpr_write_b32 a11, s27
-; FEATURE-NEXT:    v_accvgpr_write_b32 a12, s28
-; FEATURE-NEXT:    v_accvgpr_write_b32 a13, s29
-; FEATURE-NEXT:    v_accvgpr_write_b32 a14, s30
-; FEATURE-NEXT:    v_accvgpr_write_b32 a15, s31
-; FEATURE-NEXT:    v_accvgpr_write_b32 a16, s0
-; FEATURE-NEXT:    v_accvgpr_write_b32 a17, s1
-; FEATURE-NEXT:    v_accvgpr_write_b32 a18, s2
-; FEATURE-NEXT:    v_accvgpr_write_b32 a19, s3
-; FEATURE-NEXT:    v_accvgpr_write_b32 a20, s4
-; FEATURE-NEXT:    v_accvgpr_write_b32 a21, s5
-; FEATURE-NEXT:    v_accvgpr_write_b32 a22, s6
-; FEATURE-NEXT:    v_accvgpr_write_b32 a23, s7
-; FEATURE-NEXT:    v_accvgpr_write_b32 a24, s8
-; FEATURE-NEXT:    v_accvgpr_write_b32 a25, s9
-; FEATURE-NEXT:    v_accvgpr_write_b32 a26, s10
-; FEATURE-NEXT:    v_accvgpr_write_b32 a27, s11
-; FEATURE-NEXT:    v_accvgpr_write_b32 a28, s12
-; FEATURE-NEXT:    v_accvgpr_write_b32 a29, s13
-; FEATURE-NEXT:    v_accvgpr_write_b32 a30, s14
-; FEATURE-NEXT:    v_accvgpr_write_b32 a31, s15
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    s_nop 0
-; FEATURE-NEXT:    v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
-; FEATURE-NEXT:    s_nop 7
-; FEATURE-NEXT:    s_nop 7
-; FEATURE-NEXT:    s_nop 2
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[28:31], s[40:41] offset:112
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[24:27], s[40:41] offset:96
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[20:23], s[40:41] offset:80
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[16:19], s[40:41] offset:64
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[12:15], s[40:41] offset:48
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[8:11], s[40:41] offset:32
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[4:7], s[40:41] offset:16
-; FEATURE-NEXT:    global_store_dwordx4 v2, a[0:3], s[40:41]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_mfma_half:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx8 s[36:43], s[4:5], 0x24
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[36:37]
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB16_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[38:39]
-; DEFAULT-NEXT:  .LBB16_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_load_dwordx16 s[16:31], s[42:43], 0x0
-; DEFAULT-NEXT:    s_load_dwordx16 s[0:15], s[42:43], 0x40
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a0, s16
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a1, s17
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a2, s18
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a3, s19
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a4, s20
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a5, s21
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a6, s22
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a7, s23
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a8, s24
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a9, s25
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a10, s26
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a11, s27
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a12, s28
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a13, s29
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a14, s30
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a15, s31
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a16, s0
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a17, s1
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a18, s2
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a19, s3
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a20, s4
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a21, s5
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a22, s6
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a23, s7
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a24, s8
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a25, s9
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a26, s10
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a27, s11
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a28, s12
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a29, s13
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a30, s14
-; DEFAULT-NEXT:    v_accvgpr_write_b32 a31, s15
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    s_nop 0
-; DEFAULT-NEXT:    v_mfma_f32_32x32x4_2b_f16 a[0:31], v[0:1], v[0:1], a[0:31] cbsz:1 abid:2 blgp:3
-; DEFAULT-NEXT:    s_nop 7
-; DEFAULT-NEXT:    s_nop 7
-; DEFAULT-NEXT:    s_nop 2
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[28:31], s[40:41] offset:112
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[24:27], s[40:41] offset:96
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[20:23], s[40:41] offset:80
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[16:19], s[40:41] offset:64
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[12:15], s[40:41] offset:48
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[8:11], s[40:41] offset:32
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[4:7], s[40:41] offset:16
-; DEFAULT-NEXT:    global_store_dwordx4 v2, a[0:3], s[40:41]
-; DEFAULT-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
@@ -2412,47 +1012,6 @@ define amdgpu_kernel void @v8i8_intrinsic(ptr addrspace(1) %src1, ptr addrspace(
 ; GFX942-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[0:1], v[0:1]
 ; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
 ; GFX942-NEXT:    s_endpgm
-; FEATURE-LABEL: v8i8_intrinsic:
-; FEATURE:       ; %bb.0: ; %entry
-; FEATURE-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; FEATURE-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; FEATURE-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; FEATURE-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; FEATURE-NEXT:    v_mov_b32_e32 v2, 0
-; FEATURE-NEXT:    s_waitcnt lgkmcnt(0)
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; FEATURE-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; FEATURE-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; FEATURE-NEXT:    s_cbranch_execz .LBB17_2
-; FEATURE-NEXT:  ; %bb.1: ; %bb.1
-; FEATURE-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; FEATURE-NEXT:  .LBB17_2: ; %bb.2
-; FEATURE-NEXT:    s_or_b64 exec, exec, s[0:1]
-; FEATURE-NEXT:    s_waitcnt vmcnt(0)
-; FEATURE-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[0:1], v[0:1]
-; FEATURE-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
-; FEATURE-NEXT:    s_endpgm
-;
-; DEFAULT-LABEL: v8i8_intrinsic:
-; DEFAULT:       ; %bb.0: ; %entry
-; DEFAULT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; DEFAULT-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; DEFAULT-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
-; DEFAULT-NEXT:    v_lshlrev_b32_e32 v3, 3, v4
-; DEFAULT-NEXT:    v_mov_b32_e32 v2, 0
-; DEFAULT-NEXT:    s_waitcnt lgkmcnt(0)
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[0:1]
-; DEFAULT-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v4
-; DEFAULT-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; DEFAULT-NEXT:    s_cbranch_execz .LBB17_2
-; DEFAULT-NEXT:  ; %bb.1: ; %bb.1
-; DEFAULT-NEXT:    global_load_dwordx2 v[0:1], v3, s[2:3]
-; DEFAULT-NEXT:  .LBB17_2: ; %bb.2
-; DEFAULT-NEXT:    s_or_b64 exec, exec, s[0:1]
-; DEFAULT-NEXT:    s_waitcnt vmcnt(0)
-; DEFAULT-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[0:1], v[0:1]
-; DEFAULT-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
-; DEFAULT-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()
   %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
index 895a03d8bca1..7996a2dd1a4d 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-live-reg-opt.ll
@@ -1,5 +1,4 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-late-codegenprepare -amdgpu-coerce-illegal-types=1 < %s | FileCheck --check-prefix=FEATURE %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-late-codegenprepare < %s | FileCheck --check-prefix=DEFAULT %s
 
 define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
@@ -26,29 +25,6 @@ define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
 ; GFX906-NEXT:    store <3 x i8> [[TMP3]], ptr addrspace(1) [[DST]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v3i8_liveout(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <3 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <3 x i8>, ptr addrspace(1) [[GEP1]], align 4
-; FEATURE-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i8> [[VEC1]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <3 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <3 x i8>, ptr addrspace(1) [[GEP2]], align 4
-; FEATURE-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i8> [[VEC2]], <3 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <4 x i8> [[TMP1]] to i32
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    br label [[BB_2]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi i32 [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ]
-; FEATURE-NEXT:    [[TMP2:%.*]] = trunc i32 [[PHI5_TC]] to i24
-; FEATURE-NEXT:    [[TMP3:%.*]] = bitcast i24 [[TMP2]] to <3 x i8>
-; FEATURE-NEXT:    store <3 x i8> [[TMP3]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v3i8_liveout(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; DEFAULT-NEXT:  entry:
@@ -110,26 +86,6 @@ define amdgpu_kernel void @v4i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
 ; GFX906-NEXT:    store <4 x i8> [[TMP5_TC_BC]], ptr addrspace(1) [[DST]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v4i8_liveout(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP1]], align 4
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <4 x i8> [[VEC1]] to i32
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP2]], align 4
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <4 x i8> [[VEC2]] to i32
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    br label [[BB_2]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi i32 [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ]
-; FEATURE-NEXT:    [[PHI5_TC_BC:%.*]] = bitcast i32 [[PHI5_TC]] to <4 x i8>
-; FEATURE-NEXT:    store <4 x i8> [[PHI5_TC_BC]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v4i8_liveout(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -191,29 +147,6 @@ define amdgpu_kernel void @v5i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
 ; GFX906-NEXT:    store <5 x i8> [[TMP3]], ptr addrspace(1) [[DST]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v5i8_liveout(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <5 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <5 x i8>, ptr addrspace(1) [[GEP1]], align 8
-; FEATURE-NEXT:    [[TMP0:%.*]] = shufflevector <5 x i8> [[VEC1]], <5 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 5, i32 5>
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <8 x i8> [[TMP0]] to <2 x i32>
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <5 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <5 x i8>, ptr addrspace(1) [[GEP2]], align 8
-; FEATURE-NEXT:    [[TMP1:%.*]] = shufflevector <5 x i8> [[VEC2]], <5 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 5, i32 5>
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <8 x i8> [[TMP1]] to <2 x i32>
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    br label [[BB_2]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi <2 x i32> [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ]
-; FEATURE-NEXT:    [[TMP2:%.*]] = bitcast <2 x i32> [[PHI5_TC]] to <8 x i8>
-; FEATURE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP2]], <8 x i8> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
-; FEATURE-NEXT:    store <5 x i8> [[TMP3]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v5i8_liveout(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -275,26 +208,6 @@ define amdgpu_kernel void @v8i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
 ; GFX906-NEXT:    store <8 x i8> [[TMP5_TC_BC]], ptr addrspace(1) [[DST]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v8i8_liveout(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP1]], align 8
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <8 x i8> [[VEC1]] to <2 x i32>
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP2]], align 8
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <8 x i8> [[VEC2]] to <2 x i32>
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    br label [[BB_2]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi <2 x i32> [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ]
-; FEATURE-NEXT:    [[PHI5_TC_BC:%.*]] = bitcast <2 x i32> [[PHI5_TC]] to <8 x i8>
-; FEATURE-NEXT:    store <8 x i8> [[PHI5_TC_BC]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v8i8_liveout(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -358,31 +271,6 @@ define amdgpu_kernel void @repeat_successor(i32 %in, ptr addrspace(1) %src1, ptr
 ; GFX906:       return:
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @repeat_successor(
-; FEATURE-SAME: i32 [[IN:%.*]], ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP1]], align 4
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <4 x i8> [[VEC1]] to i32
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <4 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP2]], align 4
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <4 x i8> [[VEC2]] to i32
-; FEATURE-NEXT:    switch i32 [[IN]], label [[RETURN:%.*]] [
-; FEATURE-NEXT:      i32 1, label [[RETURN_SINK_SPLIT:%.*]]
-; FEATURE-NEXT:      i32 2, label [[RETURN_SINK_SPLIT]]
-; FEATURE-NEXT:      i32 3, label [[SW_BB5:%.*]]
-; FEATURE-NEXT:    ]
-; FEATURE:       sw.bb5:
-; FEATURE-NEXT:    br label [[RETURN_SINK_SPLIT]]
-; FEATURE:       return.sink.split:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi i32 [ [[VEC2_BC]], [[SW_BB5]] ], [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC1_BC]], [[ENTRY]] ]
-; FEATURE-NEXT:    [[PHI5_TC_BC:%.*]] = bitcast i32 [[PHI5_TC]] to <4 x i8>
-; FEATURE-NEXT:    store <4 x i8> [[PHI5_TC_BC]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-; FEATURE:       return:
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @repeat_successor(
 ; DEFAULT-SAME: i32 [[IN:%.*]], ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -459,32 +347,6 @@ define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(
 ; GFX906-NEXT:    store <8 x i8> [[TMP7_TC_BC]], ptr addrspace(1) [[DST1]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v8i8_phi_chain(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP1]], align 8
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <8 x i8> [[VEC1]] to <2 x i32>
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP2]], align 8
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <8 x i8> [[VEC2]] to <2 x i32>
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[IDX]], 7
-; FEATURE-NEXT:    br i1 [[CMP2]], label [[BB_2]], label [[BB_3:%.*]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi <2 x i32> [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ]
-; FEATURE-NEXT:    [[PHI5_TC_BC:%.*]] = bitcast <2 x i32> [[PHI5_TC]] to <8 x i8>
-; FEATURE-NEXT:    store <8 x i8> [[PHI5_TC_BC]], ptr addrspace(1) [[DST0]], align 4
-; FEATURE-NEXT:    br label [[BB_3]]
-; FEATURE:       bb.3:
-; FEATURE-NEXT:    [[PHI7_TC:%.*]] = phi <2 x i32> [ [[VEC2_BC]], [[BB_1]] ], [ [[PHI5_TC]], [[BB_2]] ]
-; FEATURE-NEXT:    [[PHI7_TC_BC:%.*]] = bitcast <2 x i32> [[PHI7_TC]] to <8 x i8>
-; FEATURE-NEXT:    store <8 x i8> [[PHI7_TC_BC]], ptr addrspace(1) [[DST1]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v8i8_phi_chain(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -560,31 +422,6 @@ define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspac
 ; GFX906-NEXT:    store <8 x i8> [[TMP5_TC_BC]], ptr addrspace(1) [[DST1]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v8i8_multi_block(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP1]], align 8
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <8 x i8> [[VEC1]] to <2 x i32>
-; FEATURE-NEXT:    [[GEP2:%.*]] = getelementptr <8 x i8>, ptr addrspace(1) [[SRC2]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC2:%.*]] = load <8 x i8>, ptr addrspace(1) [[GEP2]], align 8
-; FEATURE-NEXT:    [[VEC2_BC:%.*]] = bitcast <8 x i8> [[VEC2]] to <2 x i32>
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1:%.*]], label [[BB_3:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[IDX]], 7
-; FEATURE-NEXT:    br i1 [[CMP2]], label [[BB_2:%.*]], label [[BB_3]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[VEC1_BC_BC:%.*]] = bitcast <2 x i32> [[VEC1_BC]] to <8 x i8>
-; FEATURE-NEXT:    store <8 x i8> [[VEC1_BC_BC]], ptr addrspace(1) [[DST0]], align 4
-; FEATURE-NEXT:    br label [[BB_3]]
-; FEATURE:       bb.3:
-; FEATURE-NEXT:    [[PHI5_TC:%.*]] = phi <2 x i32> [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC]], [[BB_1]] ], [ [[VEC2_BC]], [[BB_2]] ]
-; FEATURE-NEXT:    [[PHI5_TC_BC:%.*]] = bitcast <2 x i32> [[PHI5_TC]] to <8 x i8>
-; FEATURE-NEXT:    store <8 x i8> [[PHI5_TC_BC]], ptr addrspace(1) [[DST1]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v8i8_multi_block(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST0:%.*]], ptr addrspace(1) captures(none) [[DST1:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -656,29 +493,6 @@ define amdgpu_kernel void @v32i8_loop_carried(ptr addrspace(1) %src1, ptr addrsp
 ; GFX906-NEXT:    store <4 x i8> [[VEC2_BC_BC]], ptr addrspace(1) [[DST]], align 4
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @v32i8_loop_carried(
-; FEATURE-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[IDX:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; FEATURE-NEXT:    [[GEP1:%.*]] = getelementptr <32 x i8>, ptr addrspace(1) [[SRC1]], i32 [[IDX]]
-; FEATURE-NEXT:    [[VEC1:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP1]], align 4
-; FEATURE-NEXT:    [[VEC1_BC:%.*]] = bitcast <4 x i8> [[VEC1]] to i32
-; FEATURE-NEXT:    br label [[BB_1:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[TEMP_TC:%.*]] = phi i32 [ [[VEC1_BC]], [[ENTRY:%.*]] ], [ [[VEC2_BC:%.*]], [[BB_1]] ]
-; FEATURE-NEXT:    [[TEMP_TC_BC:%.*]] = bitcast i32 [[TEMP_TC]] to <4 x i8>
-; FEATURE-NEXT:    [[VEC1_BC_BC:%.*]] = bitcast i32 [[VEC1_BC]] to <4 x i8>
-; FEATURE-NEXT:    [[VEC2:%.*]] = shufflevector <4 x i8> [[VEC1_BC_BC]], <4 x i8> [[TEMP_TC_BC]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FEATURE-NEXT:    [[VEC2_BC]] = bitcast <4 x i8> [[VEC2]] to i32
-; FEATURE-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX]], 15
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_1]], label [[BB_2:%.*]]
-; FEATURE:       0:
-; FEATURE-NEXT:    br label [[BB_2]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[VEC2_BC_BC:%.*]] = bitcast i32 [[VEC2_BC]] to <4 x i8>
-; FEATURE-NEXT:    store <4 x i8> [[VEC2_BC_BC]], ptr addrspace(1) [[DST]], align 4
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @v32i8_loop_carried(
 ; DEFAULT-SAME: ptr addrspace(1) [[SRC1:%.*]], ptr addrspace(1) [[SRC2:%.*]], ptr addrspace(1) captures(none) [[DST:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -742,25 +556,6 @@ define void @broken_phi() {
 ; GFX906-NEXT:    [[I8]] = phi <4 x i8> [ zeroinitializer, [[BB5]] ], [ zeroinitializer, [[BB3]] ]
 ; GFX906-NEXT:    br label [[BB1]]
 ;
-; FEATURE-LABEL: define void @broken_phi(
-; FEATURE-SAME: ) #[[ATTR0]] {
-; FEATURE-NEXT:  bb:
-; FEATURE-NEXT:    br label [[BB1:%.*]]
-; FEATURE:       bb1:
-; FEATURE-NEXT:    [[I:%.*]] = phi <4 x i8> [ splat (i8 1), [[BB:%.*]] ], [ [[I8:%.*]], [[BB7:%.*]] ]
-; FEATURE-NEXT:    br i1 false, label [[BB3:%.*]], label [[BB2:%.*]]
-; FEATURE:       bb2:
-; FEATURE-NEXT:    br label [[BB3]]
-; FEATURE:       bb3:
-; FEATURE-NEXT:    [[I4:%.*]] = phi <4 x i8> [ zeroinitializer, [[BB2]] ], [ [[I]], [[BB1]] ]
-; FEATURE-NEXT:    br i1 false, label [[BB7]], label [[BB5:%.*]]
-; FEATURE:       bb5:
-; FEATURE-NEXT:    [[I6:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[I4]], <4 x i8> zeroinitializer)
-; FEATURE-NEXT:    br label [[BB7]]
-; FEATURE:       bb7:
-; FEATURE-NEXT:    [[I8]] = phi <4 x i8> [ zeroinitializer, [[BB5]] ], [ zeroinitializer, [[BB3]] ]
-; FEATURE-NEXT:    br label [[BB1]]
-;
 ; DEFAULT-LABEL: define void @broken_phi(
 ; DEFAULT-SAME: ) #[[ATTR0]] {
 ; DEFAULT-NEXT:  bb:
@@ -815,19 +610,6 @@ define amdgpu_kernel void @reuseOp() {
 ; GFX906-NEXT:    [[VAL:%.*]] = extractelement <16 x i8> [[SEL0]], i64 0
 ; GFX906-NEXT:    ret void
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @reuseOp(
-; FEATURE-SAME: ) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    [[VEC1:%.*]] = insertelement <16 x i8> zeroinitializer, i8 0, i64 0
-; FEATURE-NEXT:    br label [[BB_1:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[SEL0:%.*]] = select i1 false, <16 x i8> zeroinitializer, <16 x i8> zeroinitializer
-; FEATURE-NEXT:    [[SEL1:%.*]] = select i1 false, <16 x i8> [[VEC1]], <16 x i8> [[SEL0]]
-; FEATURE-NEXT:    br label [[BB_2:%.*]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    [[VAL:%.*]] = extractelement <16 x i8> [[SEL0]], i64 0
-; FEATURE-NEXT:    ret void
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @reuseOp(
 ; DEFAULT-SAME: ) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -892,42 +674,6 @@ define amdgpu_kernel void @deletedPHI(i32 %in0, i1 %cmp, <10 x i8> %invec0) {
 ; GFX906-NEXT:    [[VEC1]] = shufflevector <10 x i8> [[PHI6]], <10 x i8> zeroinitializer, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 15, i32 16, i32 17, i32 18, i32 19>
 ; GFX906-NEXT:    br label [[BB_1]]
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @deletedPHI(
-; FEATURE-SAME: i32 [[IN0:%.*]], i1 [[CMP:%.*]], <10 x i8> [[INVEC0:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    br label [[BB_1:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[PHI0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB_11:%.*]] ]
-; FEATURE-NEXT:    [[PHI1:%.*]] = phi <10 x i8> [ splat (i8 1), [[ENTRY]] ], [ [[VEC1:%.*]], [[BB_11]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_3:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    br label [[BB_3]]
-; FEATURE:       bb.3:
-; FEATURE-NEXT:    [[PHI2:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_2]] ], [ [[PHI1]], [[BB_1]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_5:%.*]], label [[BB_4:%.*]]
-; FEATURE:       bb.4:
-; FEATURE-NEXT:    [[VEC0:%.*]] = insertelement <10 x i8> [[PHI2]], i8 0, i64 0
-; FEATURE-NEXT:    br label [[BB_5]]
-; FEATURE:       bb.5:
-; FEATURE-NEXT:    [[PHI3:%.*]] = phi <10 x i8> [ [[VEC0]], [[BB_4]] ], [ [[PHI2]], [[BB_3]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_7:%.*]], label [[BB_6:%.*]]
-; FEATURE:       bb.6:
-; FEATURE-NEXT:    br label [[BB_7]]
-; FEATURE:       bb.7:
-; FEATURE-NEXT:    [[PHI4:%.*]] = phi <10 x i8> [ [[INVEC0]], [[BB_6]] ], [ [[PHI3]], [[BB_5]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_9:%.*]], label [[BB_8:%.*]]
-; FEATURE:       bb.8:
-; FEATURE-NEXT:    br label [[BB_9]]
-; FEATURE:       bb.9:
-; FEATURE-NEXT:    [[PHI5:%.*]] = phi <10 x i8> [ [[INVEC0]], [[BB_8]] ], [ [[PHI4]], [[BB_7]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_11]], label [[BB_10:%.*]]
-; FEATURE:       bb.10:
-; FEATURE-NEXT:    br label [[BB_11]]
-; FEATURE:       bb.11:
-; FEATURE-NEXT:    [[PHI6:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_10]] ], [ [[PHI5]], [[BB_9]] ]
-; FEATURE-NEXT:    [[VEC1]] = shufflevector <10 x i8> [[PHI6]], <10 x i8> zeroinitializer, <10 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 15, i32 16, i32 17, i32 18, i32 19>
-; FEATURE-NEXT:    br label [[BB_1]]
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @deletedPHI(
 ; DEFAULT-SAME: i32 [[IN0:%.*]], i1 [[CMP:%.*]], <10 x i8> [[INVEC0:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
@@ -1036,31 +782,6 @@ define amdgpu_kernel void @multiple_unwind(i1 %cmp, <10 x i8> %invec) {
 ; GFX906:       bb.8:
 ; GFX906-NEXT:    br label [[BB_1]]
 ;
-; FEATURE-LABEL: define amdgpu_kernel void @multiple_unwind(
-; FEATURE-SAME: i1 [[CMP:%.*]], <10 x i8> [[INVEC:%.*]]) #[[ATTR0]] {
-; FEATURE-NEXT:  entry:
-; FEATURE-NEXT:    br label [[BB_1:%.*]]
-; FEATURE:       bb.1:
-; FEATURE-NEXT:    [[PHI0:%.*]] = phi <10 x i8> [ splat (i8 1), [[ENTRY:%.*]] ], [ [[PHI3:%.*]], [[BB_8:%.*]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_3:%.*]], label [[BB_2:%.*]]
-; FEATURE:       bb.2:
-; FEATURE-NEXT:    br label [[BB_3]]
-; FEATURE:       bb.3:
-; FEATURE-NEXT:    [[PHI1:%.*]] = phi <10 x i8> [ zeroinitializer, [[BB_2]] ], [ [[PHI0]], [[BB_1]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_5:%.*]], label [[BB_4:%.*]]
-; FEATURE:       bb.4:
-; FEATURE-NEXT:    br label [[BB_5]]
-; FEATURE:       bb.5:
-; FEATURE-NEXT:    [[PHI2:%.*]] = phi <10 x i8> [ [[PHI0]], [[BB_4]] ], [ [[PHI1]], [[BB_3]] ]
-; FEATURE-NEXT:    br i1 [[CMP]], label [[BB_7:%.*]], label [[BB_6:%.*]]
-; FEATURE:       bb.6:
-; FEATURE-NEXT:    br label [[BB_7]]
-; FEATURE:       bb.7:
-; FEATURE-NEXT:    [[PHI3]] = phi <10 x i8> [ [[INVEC]], [[BB_6]] ], [ [[PHI2]], [[BB_5]] ]
-; FEATURE-NEXT:    br label [[BB_8]]
-; FEATURE:       bb.8:
-; FEATURE-NEXT:    br label [[BB_1]]
-;
 ; DEFAULT-LABEL: define amdgpu_kernel void @multiple_unwind(
 ; DEFAULT-SAME: i1 [[CMP:%.*]], <10 x i8> [[INVEC:%.*]]) #[[ATTR0]] {
 ; DEFAULT-NEXT:  entry:
diff --git a/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
new file mode 100644
index 000000000000..f4b95236b2fa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/wait-xcnt.mir
@@ -0,0 +1,968 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefix=GCN %s
+
+---
+name: vmem_scratch_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: vmem_scratch_load
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr1 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr2, killed $vgpr2, implicit $mode, implicit $exec
+    $vgpr1 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr2, killed $vgpr2, implicit $mode, implicit $exec
+...
+
+---
+name: vmem_buffer_load_dword_offset
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-LABEL: name: vmem_buffer_load_dword_offset
+    ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr2, killed $vgpr2, implicit $mode, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
+    $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr2, killed $vgpr2, implicit $mode, implicit $exec
+...
+
+---
+name: vmem_buffer_load_addr
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-LABEL: name: vmem_buffer_load_addr
+    ; GCN: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_ADDR64 $vgpr0_vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_ADDR64 $vgpr0_vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
+    $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr1 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+...
+
+---
+name: vmem_flat_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vmem_flat_load
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+    $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr0 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+...
+
+---
+name: vmem_global_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vmem_global_load
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr0 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+    $vgpr2 = GLOBAL_LOAD_DWORD killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec:: (load (s32) from `float addrspace(1)* undef`, align 4, addrspace 1)
+    $vgpr3 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr0 = nofpexcept V_ADD_F32_e32 killed $vgpr3, killed $vgpr4, implicit $mode, implicit $exec
+...
+
+---
+name: vmem_global_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: vmem_global_store
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr3, implicit $exec
+...
+
+---
+name: vmem_buffer_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg:  $sgpr32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: vmem_buffer_store
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GCN-NEXT: S_WAIT_KMCNT 0
+    ; GCN-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $vgpr0 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec
+    $vgpr0 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+...
+
+---
+name: vmem_scratch_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  stackPtrOffsetReg: '$sgpr32'
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-LABEL: name: vmem_scratch_store
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAIT_LOADCNT_DSCNT 0
+    ; GCN-NEXT: S_WAIT_KMCNT 0
+    ; GCN-NEXT: SCRATCH_STORE_DWORDX2_SADDR killed $vgpr0_vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+    SCRATCH_STORE_DWORDX2_SADDR killed $vgpr0_vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr2, implicit $exec
+...
+
+---
+name: smem_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $sgpr2_sgpr3
+    ; GCN-LABEL: name: smem_load
+    ; GCN: liveins: $sgpr2_sgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr2_sgpr3, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $sgpr2 = S_MOV_B32 0
+     $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr2_sgpr3, 0, 0 :: (load (s64), addrspace 4)
+     $sgpr2 = S_MOV_B32 0
+...
+
+---
+name: smem_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+     liveins: $sgpr0, $sgpr2, $sgpr3
+    ; GCN-LABEL: name: smem_store
+    ; GCN: liveins: $sgpr0, $sgpr2, $sgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_STORE_DWORD_IMM $sgpr0, $sgpr2_sgpr3, 0, 0
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $sgpr3 = S_MOV_B32 0
+     S_STORE_DWORD_IMM $sgpr0, $sgpr2_sgpr3, 0, 0
+     $sgpr3 = S_MOV_B32 0
+...
+
+# 4 global_load instructions together form a load-group.
+
+---
+name: vmem_load_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-LABEL: name: vmem_load_group
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    ; GCN-NEXT: $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    ; GCN-NEXT: $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 2
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+...
+
+# The contiguous stores form a single group.
+
+---
+name: vmem_store_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-LABEL: name: vmem_store_group
+    ; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+...
+
+---
+name: smem_load_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; GCN-LABEL: name: smem_load_group
+    ; GCN: liveins: $sgpr0_sgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: S_WAIT_KMCNT 0
+    ; GCN-NEXT: $sgpr2 = S_MOV_B32 0
+     $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+     $sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+     $sgpr6_sgpr7 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+     $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+     $sgpr2 = S_MOV_B32 0
+...
+
+---
+name: smem_store_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
+    ; GCN-LABEL: name: smem_store_group
+    ; GCN: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_STORE_DWORD_IMM $sgpr2, $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: S_STORE_DWORD_IMM $sgpr3, $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: S_STORE_DWORD_IMM $sgpr4, $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: S_STORE_DWORD_IMM $sgpr5, $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $sgpr2 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr3 = S_MOV_B32 0
+     S_STORE_DWORD_IMM $sgpr2, $sgpr0_sgpr1, 0, 0
+     S_STORE_DWORD_IMM $sgpr3, $sgpr0_sgpr1, 0, 0
+     S_STORE_DWORD_IMM $sgpr4, $sgpr0_sgpr1, 0, 0
+     S_STORE_DWORD_IMM $sgpr5, $sgpr0_sgpr1, 0, 0
+     $sgpr2 = S_MOV_B32 0
+     $sgpr3 = S_MOV_B32 0
+...
+
+# The four global_load instructions form two separate groups due to the interveing s_nop.
+
+---
+name: vmem_loads_with_an_intervening_nop
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-LABEL: name: vmem_loads_with_an_intervening_nop
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    ; GCN-NEXT: $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 2
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    S_NOP 0
+    $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+...
+
+---
+name: vmem_contiguous_loads_with_an_intervening_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-LABEL: name: vmem_contiguous_loads_with_an_intervening_store
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 1
+    ; GCN-NEXT: GLOBAL_STORE_DWORDX2 $vgpr0_vgpr1, $vgpr2_vgpr3, 32, 0, implicit $exec
+    ; GCN-NEXT: $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    ; GCN-NEXT: $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 2
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    GLOBAL_STORE_DWORDX2 $vgpr0_vgpr1, $vgpr2_vgpr3, 32, 0, implicit $exec
+    $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+...
+
+---
+name: vmem_stores_with_intervening_nop
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-LABEL: name: vmem_stores_with_intervening_nop
+    ; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    S_NOP 0
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+...
+
+# The intervening load breaks the store group and form two distict store groups.
+
+---
+name: vmem_contiguous_stores_with_an_intervening_load
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-LABEL: name: vmem_contiguous_stores_with_an_intervening_load
+    ; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr11 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    $vgpr11 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    $vgpr10 = V_LSHLREV_B32_e64 16, $vgpr6, implicit $exec
+...
+
+# Atomic operations should not form a group. But they are memory instructions and should increment
+# the xcnt counter value as they might cause register dependnecy. This test ensures S_WAIT_XCNT
+# insertion for such cases.
+
+---
+name: atomic_op
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-LABEL: name: atomic_op
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = BUFFER_ATOMIC_ADD_ADDR64_RTN $vgpr2, $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
+    ; GCN-NEXT: GLOBAL_ATOMIC_ADD_F32 $vgpr4_vgpr5, killed renamable $vgpr3, 0, 0, implicit $exec :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr6 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_XCNT 2
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 1
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2 = BUFFER_ATOMIC_ADD_ADDR64_RTN $vgpr2, $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 1, implicit $exec :: (load store (s32), addrspace 1)
+    GLOBAL_ATOMIC_ADD_F32 $vgpr4_vgpr5, killed renamable $vgpr3, 0, 0, implicit $exec :: (load store syncscope("agent-one-as") monotonic monotonic (s32), addrspace 1)
+    $vgpr6 = SCRATCH_LOAD_DWORD $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+    $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+...
+
+# Force insert S_WAIT_XCNT 0 for dependency in SMEM instruction even though
+# there is a pending VMEM dependency.
+
+---
+name: smem_xcnt_insertion_with_pending_vmem_event
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; GCN-LABEL: name: smem_xcnt_insertion_with_pending_vmem_event
+    ; GCN: liveins: $sgpr0_sgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+    ; GCN-NEXT: $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 4, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORDX2 $vgpr0_vgpr1, $vgpr4_vgpr5, 16, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_KMCNT 0
+    ; GCN-NEXT: $sgpr2 = S_ADD_I32 $sgpr0, 100, implicit-def $scc
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 20, implicit $exec
+    $sgpr2_sgpr3 = S_LOAD_DWORDX2_IMM $sgpr0_sgpr1, 0, 0 :: (load (s64), addrspace 4)
+    $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+    $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr5 = V_MOV_B32_e32 4, implicit $exec
+    GLOBAL_STORE_DWORDX2 $vgpr0_vgpr1, $vgpr4_vgpr5, 16, 0, implicit $exec
+    $sgpr2 = S_ADD_I32 $sgpr0, 100, implicit-def $scc
+    $vgpr0 = V_MOV_B32_e32 20, implicit $exec
+...
+
+# The second instruction in the flat_load group has a WAR dependency with a prior
+# memory opeartion (scratch_load instruction).
+
+---
+name: vmem_group_reg_dependency_with_prior_instruction
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr4, $vgpr5
+    ; GCN-LABEL: name: vmem_group_reg_dependency_with_prior_instruction
+    ; GCN: liveins: $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr4 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr4 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr3 = V_MOV_B32_e32 1, implicit $exec
+...
+
+# Two instructions inside the load group have dependencies with prior instructions.
+
+---
+name: multiple_xcnt_insertion_in_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: multiple_xcnt_insertion_in_group
+    ; GCN: liveins: $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD $vgpr3, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr7 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr4 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr5 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    ; GCN-NEXT: $vgpr8 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr4, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    $vgpr2 = SCRATCH_LOAD_DWORD $vgpr3, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr6 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr7 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr4 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr5 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load seq_cst (s32), addrspace 1)
+    $vgpr8 = V_MOV_B32_e32 1, implicit $exec
+...
+
+---
+name: xcnt_event_post_load_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-LABEL: name: xcnt_event_post_load_group
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    ; GCN-NEXT: $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    ; GCN-NEXT: $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 3
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr11 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 2
+    ; GCN-NEXT: $vgpr6 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 1
+    ; GCN-NEXT: $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 8, 0, implicit $exec
+    $vgpr6_vgpr7 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 16, 0, implicit $exec
+    $vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX2 $vgpr0_vgpr1, 24, 0, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    $vgpr11 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    $vgpr6 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr2 = V_LSHLREV_B32_e64 16, $vgpr4, implicit $exec
+...
+
+# The three V_MOV_B32 instructions waiting outside the group needs appropriate wait_xcnt
+# insertion as their dst registers have dependencies with instructions inside the group.
+
+---
+name: xcnt_event_post_store_group
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-LABEL: name: xcnt_event_post_store_group
+    ; GCN: liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr11 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 8
+    ; GCN-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 6
+    ; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 4
+    ; GCN-NEXT: $vgpr7 = V_MOV_B32_e32 2, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr11 = V_LSHLREV_B32_e64 16, $vgpr10, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr4, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr5, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr6, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr7, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr8, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr9, 0, 0, implicit $exec
+    $vgpr11 = SCRATCH_LOAD_DWORD $vgpr10, 0, 0, implicit $exec, implicit $flat_scr
+    GLOBAL_STORE_DWORD killed $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+    $vgpr5 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr7 = V_MOV_B32_e32 2, implicit $exec
+    $vgpr11 = V_LSHLREV_B32_e64 16, $vgpr10, implicit $exec
+...
+
+# This test captures the case that interleaving load store operations form separate groups.
+# The registers in V_MOV_B32 are all have dependency with these independent groups and
+# should have the wait_xcnt insertion with appropriate wait values.
+
+---
+name: load_store_switching
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-LABEL: name: load_store_switching
+    ; GCN: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr0 = SCRATCH_LOAD_DWORD $vgpr5, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr7 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr2_vgpr3, $vgpr4, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr8 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD killed $vgpr2_vgpr3, $vgpr5, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 1
+    ; GCN-NEXT: $vgpr7 = V_MOV_B32_e32 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 2
+    ; GCN-NEXT: $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr8 = V_MOV_B32_e32 2, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 0
+    ; GCN-NEXT: $vgpr5 = V_MOV_B32_e32 3, implicit $exec
+    $vgpr0 = SCRATCH_LOAD_DWORD $vgpr5, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr1 = V_LSHLREV_B32_e64 16, $vgpr1, implicit $exec
+    $vgpr7 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr2_vgpr3, $vgpr4, 0, 0, implicit $exec
+    $vgpr8 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec
+    GLOBAL_STORE_DWORD killed $vgpr2_vgpr3, $vgpr5, 0, 0, implicit $exec
+    $vgpr7 = V_MOV_B32_e32 0, implicit $exec
+    $vgpr4 = V_MOV_B32_e32 1, implicit $exec
+    $vgpr8 = V_MOV_B32_e32 2, implicit $exec
+    $vgpr5 = V_MOV_B32_e32 3, implicit $exec
+...
+
+# V_DUAL_MOV is a single instruction and should emit required xcnt
+# if the destination registers have any memory-op dependency.
+
+---
+name: dual_mov
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+     liveins: $sgpr0, $sgpr1, $vgpr1
+    ; GCN-LABEL: name: dual_mov
+    ; GCN: liveins: $sgpr0, $sgpr1, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = SCRATCH_LOAD_DWORD $vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr1, $vgpr2 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+     $vgpr2 = SCRATCH_LOAD_DWORD $vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+     $vgpr1, $vgpr2 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
+     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
+...
+
+# No xcnt wait insertion for DS load/store operations.
+
+---
+name: ds_load_store
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: ds_load_store
+    ; GCN: $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+    ; GCN-NEXT: $vgpr0 = DS_READ_B32_gfx9 killed $vgpr1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) undef`, addrspace 3)
+    ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+    ; GCN-NEXT: S_WAIT_DSCNT 0
+    ; GCN-NEXT: DS_WRITE_B32_gfx9 killed $vgpr0, killed $vgpr1, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(3) undef`, addrspace 3)
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 20, implicit $exec
+     $vgpr1 = V_MOV_B32_e32 1, implicit $exec
+     $vgpr0 = DS_READ_B32_gfx9 killed $vgpr1, 0, 0, implicit $exec :: (load (s32) from `i32 addrspace(3)* undef`)
+     $vgpr1 = V_MOV_B32_e32 2, implicit $exec
+     DS_WRITE_B32_gfx9 killed $vgpr0, killed $vgpr1, 0, 0, implicit $exec :: (store (s32) into `i32 addrspace(3)* undef`)
+     $vgpr0 = V_MOV_B32_e32 20, implicit $exec
+...
+
+---
+name: xcnt_max
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
+    ; GCN-LABEL: name: xcnt_max
+    ; GCN: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_XCNT 62
+    ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 1, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr3, 0, 0, implicit $exec
+    $vgpr2 = V_MOV_B32_e32 1, implicit $exec
+...
+
+# FIXME: Missing S_WAIT_XCNT before overwriting vgpr0.
+---
+name: wait_kmcnt_with_outstanding_vmem
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; GCN-LABEL: name: wait_kmcnt_with_outstanding_vmem
+    ; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: S_WAIT_KMCNT 0
+    ; GCN-NEXT: $sgpr2 = S_MOV_B32 $sgpr2
+    ; GCN-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    $sgpr2 = S_MOV_B32 $sgpr2
+    $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+...
+
+# FIXME: Missing S_WAIT_XCNT before overwriting sgpr0.
+---
+name: wait_loadcnt_with_outstanding_smem
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; GCN-LABEL: name: wait_loadcnt_with_outstanding_smem
+    ; GCN: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: S_WAIT_LOADCNT 0
+    ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 $vgpr2, implicit $exec
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    $vgpr2 = V_MOV_B32_e32 $vgpr2, implicit $exec
+    $sgpr0 = S_MOV_B32 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir
new file mode 100644
index 000000000000..90faebd6967b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-trailing.mir
@@ -0,0 +1,20 @@
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck %s
+
+# Check that a trivial soft waitcnt at the end of a block is deleted even if it
+# is followed by a meta instruction.
+
+# CHECK-LABEL: name: waitcnt-no-redundant
+# CHECK: S_WAITCNT 0
+# CHECK: S_MOV_B32
+# CHECK-NOT: S_WAITCNT
+
+---
+name: waitcnt-no-redundant
+body: |
+  bb.1:
+    S_WAITCNT_soft 53119
+    $sgpr2 = S_MOV_B32 42
+    S_WAITCNT_soft 53119
+    $vgpr2 = IMPLICIT_DEF
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
index 26be45e45697..ce01a9d66955 100644
--- a/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
+++ b/llvm/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
@@ -1,15 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; Check that DAGTypeLegalizer::WidenVSELECTAndMask doesn't try to
 ; create vselects with i64 condition masks.
 
 ; FIXME: Should be able to avoid intermediate vselect
-; GCN-LABEL: {{^}}widen_vselect_and_mask_v4f64:
-; GCN: v_cmp_u_f64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
-; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
-; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
-; GCN: v_cmp_lt_i64_e32 vcc, -1, v[[[VSEL]]:[[VSEL_EXT]]]
 define amdgpu_kernel void @widen_vselect_and_mask_v4f64(<4 x double> %arg) #0 {
+; GCN-LABEL: widen_vselect_and_mask_v4f64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], 16
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    v_cmp_u_f64_e64 s[2:3], s[0:1], s[0:1]
+; GCN-NEXT:    s_waitcnt expcnt(0)
+; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, -1, s[2:3]
+; GCN-NEXT:    v_cmp_neq_f64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:    v_mov_b32_e32 v2, v1
+; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[1:2]
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], exec
+; GCN-NEXT:    s_cselect_b32 s0, 0x3ff00000, 0
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = extractelement <4 x double> %arg, i64 0
   %tmp1 = fcmp uno double %tmp, 0.000000e+00
@@ -26,12 +48,34 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_vselect_and_mask_v4i64:
-; GCN: v_cmp_eq_u64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
-; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
-; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
-; GCN: v_cmp_lt_i64_e32 vcc, -1, v[[[VSEL]]:[[VSEL_EXT]]]
 define amdgpu_kernel void @widen_vselect_and_mask_v4i64(<4 x i64> %arg) #0 {
+; GCN-LABEL: widen_vselect_and_mask_v4i64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], 0
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_mov_b64 s[8:9], 16
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    v_mov_b32_e32 v1, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, v0
+; GCN-NEXT:    v_mov_b32_e32 v3, v0
+; GCN-NEXT:    v_cmp_eq_u64_e64 s[2:3], s[0:1], 0
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[2:3]
+; GCN-NEXT:    v_cmp_ne_u64_e64 s[0:1], s[0:1], 0
+; GCN-NEXT:    v_mov_b32_e32 v5, v4
+; GCN-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
+; GCN-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GCN-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
+; GCN-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NEXT:    v_mov_b32_e32 v5, v0
+; GCN-NEXT:    v_mov_b32_e32 v6, v0
+; GCN-NEXT:    v_mov_b32_e32 v7, v0
+; GCN-NEXT:    s_mov_b32 s6, s10
+; GCN-NEXT:    s_mov_b32 s7, s11
+; GCN-NEXT:    buffer_store_dwordx4 v[4:7], off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = extractelement <4 x i64> %arg, i64 0
   %tmp1 = icmp eq i64 %tmp, 0
diff --git a/llvm/test/CodeGen/ARM/Windows/frexp.ll b/llvm/test/CodeGen/ARM/Windows/frexp.ll
new file mode 100644
index 000000000000..5646c75a7597
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/Windows/frexp.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mtriple thumbv7--windows-itanium < %s | FileCheck %s
+
+; CHECK-LABEL: test_frexp_f16_i32:
+; CHECK: bl __gnu_h2f_ieee
+; CHECK: vcvt.f64.f32 d0, s0
+; CHECK: bl frexp
+; CHECK: vcvt.f32.f64 s0, d0
+; CHECK: bl __gnu_f2h_ieee
+define { half, i32 } @test_frexp_f16_i32(half %a) {
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  ret { half, i32 } %result
+}
+
+; CHECK-LABEL: test_frexp_f32_i32:
+; CHECK: vcvt.f64.f32
+; CHECK: bl frexp
+; CHECK: vcvt.f32.f64	s0, d0
+define { float, i32 } @test_frexp_f32_i32(float %a) {
+  %result = call { float, i32 } @llvm.frexp.f32.i32(float %a)
+  ret { float, i32 } %result
+}
+
+; CHECK-LABEL: test_frexp_f64_i32:
+; CHECK: bl frexp
+define { double, i32 } @test_frexp_f64_i32(double %a) {
+  %result = call { double, i32 } @llvm.frexp.f64.i32(double %a)
+  ret { double, i32 } %result
+}
diff --git a/llvm/test/CodeGen/ARM/cmp-select-sign.ll b/llvm/test/CodeGen/ARM/cmp-select-sign.ll
new file mode 100644
index 000000000000..298a623ed1d8
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmp-select-sign.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv7a < %s | FileCheck %s --check-prefix=ARM
+; RUN: llc -mtriple=armv6m < %s | FileCheck %s --check-prefix=THUMB
+; RUN: llc -mtriple=armv7m < %s | FileCheck %s --check-prefix=THUMB2
+; RUN: llc -mtriple=thumbv8.1m.main < %s | FileCheck %s --check-prefix=THUMBV8
+
+define i3 @sign_i3(i3 %a) {
+; ARM-LABEL: sign_i3:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #29
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i3:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #29
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i3:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #29
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i3:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #29
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i3 %a, -1
+  %res = select i1 %c, i3 1, i3 -1
+  ret i3 %res
+}
+
+define i4 @sign_i4(i4 %a) {
+; ARM-LABEL: sign_i4:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #28
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i4:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #28
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i4:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #28
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i4:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #28
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i4 %a, -1
+  %res = select i1 %c, i4 1, i4 -1
+  ret i4 %res
+}
+
+define i8 @sign_i8(i8 %a) {
+; ARM-LABEL: sign_i8:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #24
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i8:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #24
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i8:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #24
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i8:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #24
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i8 %a, -1
+  %res = select i1 %c, i8 1, i8 -1
+  ret i8 %res
+}
+
+define i16 @sign_i16(i16 %a) {
+; ARM-LABEL: sign_i16:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    lsl r0, r0, #16
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i16:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    lsls r0, r0, #16
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i16:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    lsls r0, r0, #16
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i16:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    lsls r0, r0, #16
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i16 %a, -1
+  %res = select i1 %c, i16 1, i16 -1
+  ret i16 %res
+}
+
+define i32 @sign_i32(i32 %a) {
+; ARM-LABEL: sign_i32:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    mov r1, #1
+; ARM-NEXT:    orr r0, r1, r0, asr #31
+; ARM-NEXT:    bx lr
+;
+; THUMB-LABEL: sign_i32:
+; THUMB:       @ %bb.0:
+; THUMB-NEXT:    asrs r1, r0, #31
+; THUMB-NEXT:    movs r0, #1
+; THUMB-NEXT:    orrs r0, r1
+; THUMB-NEXT:    bx lr
+;
+; THUMB2-LABEL: sign_i32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    movs r1, #1
+; THUMB2-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMB2-NEXT:    bx lr
+;
+; THUMBV8-LABEL: sign_i32:
+; THUMBV8:       @ %bb.0:
+; THUMBV8-NEXT:    movs r1, #1
+; THUMBV8-NEXT:    orr.w r0, r1, r0, asr #31
+; THUMBV8-NEXT:    bx lr
+  %c = icmp sgt i32 %a, -1
+  %res = select i1 %c, i32 1, i32 -1
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/ARM/fp16_fast_math.ll b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
index 4c2e3ce4efcd..165eb4b8af43 100644
--- a/llvm/test/CodeGen/ARM/fp16_fast_math.ll
+++ b/llvm/test/CodeGen/ARM/fp16_fast_math.ll
@@ -21,7 +21,7 @@ define half @normal_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -55,7 +55,7 @@ define half @fast_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = nnan ninf nsz arcp contract afn reassoc VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -89,7 +89,7 @@ define half @ninf_fadd(half %x, half %y) {
   ; CHECK-CVT-NEXT:   [[VADDS:%[0-9]+]]:spr = ninf VADDS killed [[VCVTBHS1]], killed [[VCVTBHS]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY4]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -129,13 +129,13 @@ define half @normal_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VCVTBHS2:%[0-9]+]]:spr = VCVTBHS killed [[COPY5]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
-  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY [[COPY6]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY killed [[COPY6]]
   ; CHECK-CVT-NEXT:   [[VCVTBHS3:%[0-9]+]]:spr = VCVTBHS killed [[COPY7]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = VADDS killed [[VCVTBHS3]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF1:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH1:%[0-9]+]]:spr = VCVTBSH [[DEF1]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY [[VCVTBSH1]]
+  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY killed [[VCVTBSH1]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY8]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -177,7 +177,7 @@ define half @nnan_ninf_contract_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = nnan ninf contract VADDS killed [[VADDS]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY6]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
@@ -218,13 +218,13 @@ define half @ninf_fadd_sequence(half %x, half %y, half %z) {
   ; CHECK-CVT-NEXT:   [[VCVTBHS2:%[0-9]+]]:spr = ninf VCVTBHS killed [[COPY5]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH:%[0-9]+]]:spr = VCVTBSH [[DEF]], killed [[VADDS]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY [[VCVTBSH]]
-  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY [[COPY6]]
+  ; CHECK-CVT-NEXT:   [[COPY6:%[0-9]+]]:gpr = COPY killed [[VCVTBSH]]
+  ; CHECK-CVT-NEXT:   [[COPY7:%[0-9]+]]:spr = COPY killed [[COPY6]]
   ; CHECK-CVT-NEXT:   [[VCVTBHS3:%[0-9]+]]:spr = ninf VCVTBHS killed [[COPY7]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[VADDS1:%[0-9]+]]:spr = ninf VADDS killed [[VCVTBHS3]], killed [[VCVTBHS2]], 14 /* CC::al */, $noreg
   ; CHECK-CVT-NEXT:   [[DEF1:%[0-9]+]]:spr = IMPLICIT_DEF
   ; CHECK-CVT-NEXT:   [[VCVTBSH1:%[0-9]+]]:spr = VCVTBSH [[DEF1]], killed [[VADDS1]], 14 /* CC::al */, $noreg
-  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY [[VCVTBSH1]]
+  ; CHECK-CVT-NEXT:   [[COPY8:%[0-9]+]]:gpr = COPY killed [[VCVTBSH1]]
   ; CHECK-CVT-NEXT:   $r0 = COPY [[COPY8]]
   ; CHECK-CVT-NEXT:   MOVPCLR 14 /* CC::al */, $noreg, implicit $r0
   ;
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat.ll b/llvm/test/CodeGen/ARM/fpclamptosat.ll
index 478b98dfac80..8ab56b228d2a 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat.ll
@@ -613,14 +613,21 @@ define i16 @ustest_f64i16(double %x) {
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    vmov r0, r1, d0
 ; VFP2-NEXT:    bl __aeabi_d2iz
-; VFP2-NEXT:    usat r0, #16, r0
+; VFP2-NEXT:    movw r1, #65535
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f64i16:
 ; FULL:       @ %bb.0: @ %entry
 ; FULL-NEXT:    vcvt.s32.f64 s0, d0
+; FULL-NEXT:    movw r1, #65535
 ; FULL-NEXT:    vmov r0, s0
-; FULL-NEXT:    usat r0, #16, r0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
 ; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi double %x to i32
@@ -738,12 +745,26 @@ define i16 @ustest_f32i16(float %x) {
 ; SOFT-NEXT:  .LCPI14_0:
 ; SOFT-NEXT:    .long 65535 @ 0xffff
 ;
-; VFP-LABEL: ustest_f32i16:
-; VFP:       @ %bb.0: @ %entry
-; VFP-NEXT:    vcvt.s32.f32 s0, s0
-; VFP-NEXT:    vmov r0, s0
-; VFP-NEXT:    usat r0, #16, r0
-; VFP-NEXT:    bx lr
+; VFP2-LABEL: ustest_f32i16:
+; VFP2:       @ %bb.0: @ %entry
+; VFP2-NEXT:    vcvt.s32.f32 s0, s0
+; VFP2-NEXT:    movw r1, #65535
+; VFP2-NEXT:    vmov r0, s0
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
+; VFP2-NEXT:    bx lr
+;
+; FULL-LABEL: ustest_f32i16:
+; FULL:       @ %bb.0: @ %entry
+; FULL-NEXT:    vcvt.s32.f32 s0, s0
+; FULL-NEXT:    movw r1, #65535
+; FULL-NEXT:    vmov r0, s0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
+; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi float %x to i32
   %0 = icmp slt i32 %conv, 65535
@@ -890,16 +911,23 @@ define i16 @ustest_f16i16(half %x) {
 ; VFP2-NEXT:    vmov r0, s0
 ; VFP2-NEXT:    bl __aeabi_h2f
 ; VFP2-NEXT:    vmov s0, r0
+; VFP2-NEXT:    movw r1, #65535
 ; VFP2-NEXT:    vcvt.s32.f32 s0, s0
 ; VFP2-NEXT:    vmov r0, s0
-; VFP2-NEXT:    usat r0, #16, r0
+; VFP2-NEXT:    cmp r0, r1
+; VFP2-NEXT:    it lt
+; VFP2-NEXT:    movlt r1, r0
+; VFP2-NEXT:    bic.w r0, r1, r1, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f16i16:
 ; FULL:       @ %bb.0: @ %entry
 ; FULL-NEXT:    vcvt.s32.f16 s0, s0
+; FULL-NEXT:    movw r1, #65535
 ; FULL-NEXT:    vmov r0, s0
-; FULL-NEXT:    usat r0, #16, r0
+; FULL-NEXT:    cmp r0, r1
+; FULL-NEXT:    csel r0, r0, r1, lt
+; FULL-NEXT:    bic.w r0, r0, r0, asr #31
 ; FULL-NEXT:    bx lr
 entry:
   %conv = fptosi half %x to i32
@@ -1101,83 +1129,48 @@ entry:
 define i64 @ustest_f64i64(double %x) {
 ; SOFT-LABEL: ustest_f64i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixdfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB20_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB20_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB20_10
+; SOFT-NEXT:    bge .LBB20_6
 ; SOFT-NEXT:  .LBB20_2: @ %entry
-; SOFT-NEXT:    bge .LBB20_11
+; SOFT-NEXT:    blt .LBB20_4
 ; SOFT-NEXT:  .LBB20_3: @ %entry
-; SOFT-NEXT:    blt .LBB20_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB20_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB20_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB20_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB20_13
-; SOFT-NEXT:  .LBB20_7: @ %entry
-; SOFT-NEXT:    beq .LBB20_14
-; SOFT-NEXT:  .LBB20_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB20_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB20_2
-; SOFT-NEXT:  .LBB20_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB20_3
-; SOFT-NEXT:  .LBB20_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB20_4
-; SOFT-NEXT:    b .LBB20_5
-; SOFT-NEXT:  .LBB20_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB20_7
-; SOFT-NEXT:  .LBB20_13: @ %entry
+; SOFT-NEXT:  .LBB20_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB20_8
-; SOFT-NEXT:  .LBB20_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB20_3
+; SOFT-NEXT:    b .LBB20_4
 ;
 ; VFP2-LABEL: ustest_f64i64:
 ; VFP2:       @ %bb.0: @ %entry
 ; VFP2-NEXT:    .save {r7, lr}
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    bl __fixdfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f64i64:
@@ -1185,22 +1178,14 @@ define i64 @ustest_f64i64(double %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixdfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi double %x to i128
@@ -1400,83 +1385,48 @@ entry:
 define i64 @ustest_f32i64(float %x) {
 ; SOFT-LABEL: ustest_f32i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB23_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB23_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB23_10
+; SOFT-NEXT:    bge .LBB23_6
 ; SOFT-NEXT:  .LBB23_2: @ %entry
-; SOFT-NEXT:    bge .LBB23_11
+; SOFT-NEXT:    blt .LBB23_4
 ; SOFT-NEXT:  .LBB23_3: @ %entry
-; SOFT-NEXT:    blt .LBB23_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB23_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB23_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB23_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB23_13
-; SOFT-NEXT:  .LBB23_7: @ %entry
-; SOFT-NEXT:    beq .LBB23_14
-; SOFT-NEXT:  .LBB23_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB23_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB23_2
-; SOFT-NEXT:  .LBB23_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB23_3
-; SOFT-NEXT:  .LBB23_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB23_4
-; SOFT-NEXT:    b .LBB23_5
-; SOFT-NEXT:  .LBB23_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB23_7
-; SOFT-NEXT:  .LBB23_13: @ %entry
+; SOFT-NEXT:  .LBB23_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB23_8
-; SOFT-NEXT:  .LBB23_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB23_3
+; SOFT-NEXT:    b .LBB23_4
 ;
 ; VFP2-LABEL: ustest_f32i64:
 ; VFP2:       @ %bb.0: @ %entry
 ; VFP2-NEXT:    .save {r7, lr}
 ; VFP2-NEXT:    push {r7, lr}
 ; VFP2-NEXT:    bl __fixsfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f32i64:
@@ -1484,22 +1434,14 @@ define i64 @ustest_f32i64(float %x) {
 ; FULL-NEXT:    .save {r7, lr}
 ; FULL-NEXT:    push {r7, lr}
 ; FULL-NEXT:    bl __fixsfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi float %x to i128
@@ -1713,61 +1655,34 @@ entry:
 define i64 @ustest_f16i64(half %x) {
 ; SOFT-LABEL: ustest_f16i64:
 ; SOFT:       @ %bb.0: @ %entry
-; SOFT-NEXT:    .save {r4, r5, r6, lr}
-; SOFT-NEXT:    push {r4, r5, r6, lr}
+; SOFT-NEXT:    .save {r4, lr}
+; SOFT-NEXT:    push {r4, lr}
 ; SOFT-NEXT:    uxth r0, r0
 ; SOFT-NEXT:    bl __aeabi_h2f
 ; SOFT-NEXT:    bl __fixsfti
-; SOFT-NEXT:    movs r4, #1
-; SOFT-NEXT:    movs r5, #0
-; SOFT-NEXT:    subs r6, r2, #1
-; SOFT-NEXT:    mov r6, r3
-; SOFT-NEXT:    sbcs r6, r5
-; SOFT-NEXT:    bge .LBB26_9
+; SOFT-NEXT:    movs r4, #0
+; SOFT-NEXT:    subs r2, r2, #1
+; SOFT-NEXT:    mov r2, r3
+; SOFT-NEXT:    sbcs r2, r4
+; SOFT-NEXT:    bge .LBB26_5
 ; SOFT-NEXT:  @ %bb.1: @ %entry
-; SOFT-NEXT:    bge .LBB26_10
+; SOFT-NEXT:    bge .LBB26_6
 ; SOFT-NEXT:  .LBB26_2: @ %entry
-; SOFT-NEXT:    bge .LBB26_11
+; SOFT-NEXT:    blt .LBB26_4
 ; SOFT-NEXT:  .LBB26_3: @ %entry
-; SOFT-NEXT:    blt .LBB26_5
+; SOFT-NEXT:    mov r3, r4
 ; SOFT-NEXT:  .LBB26_4: @ %entry
-; SOFT-NEXT:    mov r0, r5
+; SOFT-NEXT:    asrs r2, r3, #31
+; SOFT-NEXT:    bics r0, r2
+; SOFT-NEXT:    bics r1, r2
+; SOFT-NEXT:    pop {r4, pc}
 ; SOFT-NEXT:  .LBB26_5: @ %entry
-; SOFT-NEXT:    rsbs r6, r0, #0
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r1
-; SOFT-NEXT:    mov r6, r5
-; SOFT-NEXT:    sbcs r6, r2
-; SOFT-NEXT:    mov r2, r5
-; SOFT-NEXT:    sbcs r2, r3
-; SOFT-NEXT:    bge .LBB26_12
-; SOFT-NEXT:  @ %bb.6: @ %entry
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    beq .LBB26_13
-; SOFT-NEXT:  .LBB26_7: @ %entry
-; SOFT-NEXT:    beq .LBB26_14
-; SOFT-NEXT:  .LBB26_8: @ %entry
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
-; SOFT-NEXT:  .LBB26_9: @ %entry
-; SOFT-NEXT:    mov r3, r5
+; SOFT-NEXT:    mov r1, r4
 ; SOFT-NEXT:    blt .LBB26_2
-; SOFT-NEXT:  .LBB26_10: @ %entry
-; SOFT-NEXT:    mov r2, r4
-; SOFT-NEXT:    blt .LBB26_3
-; SOFT-NEXT:  .LBB26_11: @ %entry
-; SOFT-NEXT:    mov r1, r5
-; SOFT-NEXT:    bge .LBB26_4
-; SOFT-NEXT:    b .LBB26_5
-; SOFT-NEXT:  .LBB26_12: @ %entry
-; SOFT-NEXT:    mov r4, r5
-; SOFT-NEXT:    cmp r4, #0
-; SOFT-NEXT:    bne .LBB26_7
-; SOFT-NEXT:  .LBB26_13: @ %entry
+; SOFT-NEXT:  .LBB26_6: @ %entry
 ; SOFT-NEXT:    mov r0, r4
-; SOFT-NEXT:    bne .LBB26_8
-; SOFT-NEXT:  .LBB26_14: @ %entry
-; SOFT-NEXT:    mov r1, r4
-; SOFT-NEXT:    pop {r4, r5, r6, pc}
+; SOFT-NEXT:    bge .LBB26_3
+; SOFT-NEXT:    b .LBB26_4
 ;
 ; VFP2-LABEL: ustest_f16i64:
 ; VFP2:       @ %bb.0: @ %entry
@@ -1777,24 +1692,16 @@ define i64 @ustest_f16i64(half %x) {
 ; VFP2-NEXT:    bl __aeabi_h2f
 ; VFP2-NEXT:    vmov s0, r0
 ; VFP2-NEXT:    bl __fixsfti
-; VFP2-NEXT:    subs.w lr, r2, #1
+; VFP2-NEXT:    subs r2, #1
 ; VFP2-NEXT:    mov.w r12, #0
-; VFP2-NEXT:    sbcs lr, r3, #0
-; VFP2-NEXT:    itttt ge
+; VFP2-NEXT:    sbcs r2, r3, #0
+; VFP2-NEXT:    itt ge
 ; VFP2-NEXT:    movge r3, r12
-; VFP2-NEXT:    movge r2, #1
-; VFP2-NEXT:    movge r1, r12
 ; VFP2-NEXT:    movge r0, r12
-; VFP2-NEXT:    rsbs.w lr, r0, #0
-; VFP2-NEXT:    sbcs.w lr, r12, r1
-; VFP2-NEXT:    sbcs.w r2, r12, r2
-; VFP2-NEXT:    sbcs.w r2, r12, r3
-; VFP2-NEXT:    it lt
-; VFP2-NEXT:    movlt.w r12, #1
-; VFP2-NEXT:    cmp.w r12, #0
-; VFP2-NEXT:    itt eq
-; VFP2-NEXT:    moveq r0, r12
-; VFP2-NEXT:    moveq r1, r12
+; VFP2-NEXT:    it ge
+; VFP2-NEXT:    movge r1, r12
+; VFP2-NEXT:    bic.w r0, r0, r3, asr #31
+; VFP2-NEXT:    bic.w r1, r1, r3, asr #31
 ; VFP2-NEXT:    pop {r7, pc}
 ;
 ; FULL-LABEL: ustest_f16i64:
@@ -1804,22 +1711,14 @@ define i64 @ustest_f16i64(half %x) {
 ; FULL-NEXT:    vmov.f16 r0, s0
 ; FULL-NEXT:    vmov s0, r0
 ; FULL-NEXT:    bl __fixhfti
-; FULL-NEXT:    subs.w lr, r2, #1
+; FULL-NEXT:    subs r2, #1
 ; FULL-NEXT:    mov.w r12, #0
-; FULL-NEXT:    sbcs lr, r3, #0
-; FULL-NEXT:    it ge
-; FULL-NEXT:    movge r2, #1
+; FULL-NEXT:    sbcs r2, r3, #0
+; FULL-NEXT:    csel r2, r3, r12, lt
 ; FULL-NEXT:    csel r0, r0, r12, lt
-; FULL-NEXT:    csel lr, r3, r12, lt
 ; FULL-NEXT:    csel r1, r1, r12, lt
-; FULL-NEXT:    rsbs r3, r0, #0
-; FULL-NEXT:    sbcs.w r3, r12, r1
-; FULL-NEXT:    sbcs.w r2, r12, r2
-; FULL-NEXT:    sbcs.w r2, r12, lr
-; FULL-NEXT:    cset r2, lt
-; FULL-NEXT:    cmp r2, #0
-; FULL-NEXT:    csel r0, r0, r2, ne
-; FULL-NEXT:    csel r1, r1, r2, ne
+; FULL-NEXT:    bic.w r0, r0, r2, asr #31
+; FULL-NEXT:    bic.w r1, r1, r2, asr #31
 ; FULL-NEXT:    pop {r7, pc}
 entry:
   %conv = fptosi half %x to i128
diff --git a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
index 4d091c230265..96f009a4da02 100644
--- a/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/ARM/fpclamptosat_vec.ll
@@ -174,7 +174,7 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) {
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    mvnne r2, #0
 ; CHECK-NEXT:    vmov.32 d18[0], r2
-; CHECK-NEXT:    vand q8, q8, q9
+; CHECK-NEXT:    vand q8, q9, q8
 ; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    pop {r4, r5, r11, pc}
@@ -483,8 +483,8 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-NEXT:    vmov.32 d20[0], r7
 ; CHECK-NEXT:    mvnne r4, #0
 ; CHECK-NEXT:    vmov.32 d18[0], r4
-; CHECK-NEXT:    vand q10, q4, q10
-; CHECK-NEXT:    vand q8, q8, q9
+; CHECK-NEXT:    vand q10, q10, q4
+; CHECK-NEXT:    vand q8, q9, q8
 ; CHECK-NEXT:    vmovn.i64 d1, q10
 ; CHECK-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
@@ -995,8 +995,8 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-NEON-NEXT:    vmov.32 d20[0], r6
 ; CHECK-NEON-NEXT:    mvnne r7, #0
 ; CHECK-NEON-NEXT:    vmov.32 d18[0], r7
-; CHECK-NEON-NEXT:    vand q10, q4, q10
-; CHECK-NEON-NEXT:    vand q8, q8, q9
+; CHECK-NEON-NEXT:    vand q10, q10, q4
+; CHECK-NEON-NEXT:    vand q8, q9, q8
 ; CHECK-NEON-NEXT:    vmovn.i64 d1, q10
 ; CHECK-NEON-NEXT:    vmovn.i64 d0, q8
 ; CHECK-NEON-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
@@ -1097,8 +1097,8 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-FP16-NEXT:    vmov.32 d20[0], r7
 ; CHECK-FP16-NEXT:    mvnne r6, #0
 ; CHECK-FP16-NEXT:    vmov.32 d18[0], r6
-; CHECK-FP16-NEXT:    vand q10, q4, q10
-; CHECK-FP16-NEXT:    vand q8, q8, q9
+; CHECK-FP16-NEXT:    vand q10, q10, q4
+; CHECK-FP16-NEXT:    vand q8, q9, q8
 ; CHECK-FP16-NEXT:    vmovn.i64 d1, q10
 ; CHECK-FP16-NEXT:    vmovn.i64 d0, q8
 ; CHECK-FP16-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
diff --git a/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll b/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll
new file mode 100644
index 000000000000..7a02dbd9d0d1
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/frexp-soften-libcall-error.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -mtriple arm-linux-gnu -float-abi=soft -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: ffrexp exponent does not match sizeof(int)
+define { float, i64 } @soften_frexp_error(float %x) {
+  %frexp = call { float, i64 } @llvm.frexp.f32.i64(float %x)
+  ret { float, i64 } %frexp
+}
diff --git a/llvm/test/CodeGen/ARM/iabs.ll b/llvm/test/CodeGen/ARM/iabs.ll
index fffa9555b296..758fe7507c0b 100644
--- a/llvm/test/CodeGen/ARM/iabs.ll
+++ b/llvm/test/CodeGen/ARM/iabs.ll
@@ -50,3 +50,28 @@ define i64 @test3(i64 %a) {
   %abs = select i1 %b, i64 %a, i64 %tmp1neg
   ret i64 %abs
 }
+
+declare void @callee(...)
+
+define void @testcallframe(i32 %a) {
+; CHECK-LABEL: testcallframe:
+; CHECK:       @ %bb.0: @ %bb
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, sp, #8
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    mov r1, #0
+; CHECK-NEXT:    rsbmi r0, r0, #0
+; CHECK-NEXT:    mov r2, #0
+; CHECK-NEXT:    mov r3, #0
+; CHECK-NEXT:    str r1, [sp]
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    add sp, sp, #8
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    bx lr
+bb:
+  %i = tail call i32 @llvm.abs.i32(i32 %a, i1 false)
+  tail call void @callee(i32 %i, i32 0, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/ifcvt_unanalyzable_fallthrough.mir b/llvm/test/CodeGen/ARM/ifcvt_unanalyzable_fallthrough.mir
new file mode 100644
index 000000000000..d2673c36f0f4
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ifcvt_unanalyzable_fallthrough.mir
@@ -0,0 +1,114 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=thumbv7-apple-ios -run-pass=if-converter %s -o - | FileCheck %s
+
+# Testcase with unanalyzable branches (that may fallthrough) in the BB
+# following the diamond/triangle.
+
+# Goal here is to showcase a problem seen in the IfConverter when
+# AnalyzeBranch is indicating that the branches couldn't be analyzed. Problem
+# was originally seen for an out-of-tree target, and here we use ARM and a
+# MBB with two conditional jumps to make AnalyzeBranch return false.
+#
+# The problem was that if-converter when analyzing branches was using a
+# variable named HasFallThrough, to remember that an MBB could fallthrough to
+# the textual successor. When HasFallThrough is set we know that there are
+# fallthrough exits, but the opposite is not guaranteed. If
+# HasFallThrough==false there could still be fallthrough exists in situations
+# when analyzeBranch found unanalyzable branches. There were however a couple
+# of places in the code that checked !HasFallThrough assuming that it would
+# imply that there was no fallthrough exit.
+#
+# As a consequence we could end up merging blocks at the end of a converted
+# diamond/triangle and while doing that we messed up when fixing up the CFG
+# related to fallthrough edges. For the test cases below we incorrectly ended
+# up with a fallthrough from the MBBs with two Bcc instructions to the MBB
+# with the STRH after if conversion.
+#
+---
+name:            avoidMergeBlockDiamond
+body:             |
+  ; CHECK-LABEL: name: avoidMergeBlockDiamond
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $sp = tADDspi $sp, 2, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT:   $sp = tADDspi $sp, 1, 0 /* CC::eq */, $cpsr, implicit $sp
+  ; CHECK-NEXT:   $sp = tADDspi $sp, 3, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tBcc %bb.1, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT:   tBcc %bb.1, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   tBX_RET 14 /* CC::al */, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STRH $sp, $sp, $noreg, 0, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tB %bb.2, 14 /* CC::al */, $noreg
+  bb.0:
+    tBcc %bb.2, 1, $cpsr
+
+  bb.1:
+    $sp = tADDspi $sp, 1, 14, _
+    tB %bb.4, 14, $noreg
+
+  bb.2:
+    $sp = tADDspi $sp, 2, 14, _
+    tB %bb.4, 14, $noreg
+
+  bb.3:
+    STRH $sp, $sp, $noreg, 0, 14, $noreg
+    tB %bb.3, 14, $noreg
+
+  bb.4:
+    $sp = tADDspi $sp, 3, 14, _
+    tBcc %bb.5, 1, $cpsr
+    tBcc %bb.5, 1, $cpsr
+
+  bb.5:
+  successors:
+    tBX_RET 14, _
+...
+
+# Similar to the above, but with a triangle.
+---
+name:            avoidMergeBlockTriangle
+body:             |
+  ; CHECK-LABEL: name: avoidMergeBlockTriangle
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $sp = tADDspi $sp, 1, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT:   $sp = tADDspi $sp, 2, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tBcc %bb.1, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT:   tBcc %bb.1, 1 /* CC::ne */, $cpsr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   tBX_RET 14 /* CC::al */, $noreg
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   STRH $sp, $sp, $noreg, 0, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tB %bb.2, 14 /* CC::al */, $noreg
+  bb.0:
+    tBcc %bb.1, 1, $cpsr
+    tB %bb.3, 14, $noreg
+
+  bb.1:
+    $sp = tADDspi $sp, 1, 14, _
+    tB %bb.3, 14, $noreg
+
+  bb.2:
+    STRH $sp, $sp, $noreg, 0, 14, $noreg
+    tB %bb.2, 14, $noreg
+
+  bb.3:
+    $sp = tADDspi $sp, 2, 14, _
+    tBcc %bb.4, 1, $cpsr
+    tBcc %bb.4, 1, $cpsr
+
+  bb.4:
+  successors:
+    tBX_RET 14, _
+...
diff --git a/llvm/test/CodeGen/ARM/ldexp.ll b/llvm/test/CodeGen/ARM/ldexp.ll
index 941390ee94c1..cdf91eb902e0 100644
--- a/llvm/test/CodeGen/ARM/ldexp.ll
+++ b/llvm/test/CodeGen/ARM/ldexp.ll
@@ -5,7 +5,7 @@ define double @testExp(double %val, i32 %a) {
 ; LINUX:    b ldexp{{$}}
 ; WINDOWS:  b.w ldexp{{$}}
 entry:
-  %call = tail call fast double @ldexp(double %val, i32 %a)
+  %call = tail call double @ldexp(double %val, i32 %a)
   ret double %call
 }
 
@@ -15,7 +15,7 @@ define double @testExpIntrinsic(double %val, i32 %a) {
 ; LINUX:    b ldexp{{$}}
 ; WINDOWS:  b.w ldexp{{$}}
 entry:
-  %call = tail call fast double @llvm.ldexp.f64(double %val, i32 %a)
+  %call = tail call double @llvm.ldexp.f64(double %val, i32 %a)
   ret double %call
 }
 
@@ -23,7 +23,7 @@ define float @testExpf(float %val, i32 %a) {
 ; LINUX:    b ldexpf
 ; WINDOWS:  b.w ldexpf
 entry:
-  %call = tail call fast float @ldexpf(float %val, i32 %a)
+  %call = tail call float @ldexpf(float %val, i32 %a)
   ret float %call
 }
 
@@ -31,7 +31,7 @@ define float @testExpfIntrinsic(float %val, i32 %a) {
 ; LINUX:    b ldexpf
 ; WINDOWS:  bl ldexp{{$}}
 entry:
-  %call = tail call fast float @llvm.ldexp.f32(float %val, i32 %a)
+  %call = tail call float @llvm.ldexp.f32(float %val, i32 %a)
   ret float %call
 }
 
@@ -41,7 +41,7 @@ define fp128 @testExpl(fp128 %val, i32 %a) {
 ; LINUX:    bl ldexpl
 ; WINDOWS:    b.w ldexpl
 entry:
-  %call = tail call fast fp128 @ldexpl(fp128 %val, i32 %a)
+  %call = tail call fp128 @ldexpl(fp128 %val, i32 %a)
   ret fp128 %call
 }
 
@@ -51,7 +51,7 @@ define half @testExpf16(half %val, i32 %a) {
 ; LINUX: bl ldexpf
 ; WINDOWS: bl ldexp{{$}}
 entry:
-  %0 = tail call fast half @llvm.ldexp.f16.i32(half %val, i32 %a)
+  %0 = tail call half @llvm.ldexp.f16.i32(half %val, i32 %a)
   ret half %0
 }
 
diff --git a/llvm/test/CodeGen/ARM/llvm.frexp.ll b/llvm/test/CodeGen/ARM/llvm.frexp.ll
index e79ddbe93336..6f21834ea5e3 100644
--- a/llvm/test/CodeGen/ARM/llvm.frexp.ll
+++ b/llvm/test/CodeGen/ARM/llvm.frexp.ll
@@ -141,6 +141,43 @@ define <2 x i32> @test_frexp_v2f16_v2i32_only_use_exp(<2 x half> %a) {
   ret <2 x i32> %result.1
 }
 
+define { <3 x float>, <3 x i32> } @test_frexp_v3f16_v3i32(<3 x float> %a) {
+; CHECK-LABEL: test_frexp_v3f16_v3i32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    mov r8, sp
+; CHECK-NEXT:    vldr d9, [sp, #48]
+; CHECK-NEXT:    mov r4, r0
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    mov r1, r8
+; CHECK-NEXT:    mov r5, r3
+; CHECK-NEXT:    vmov d8, r2, r3
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    add r6, sp, #4
+; CHECK-NEXT:    mov r7, r0
+; CHECK-NEXT:    mov r0, r5
+; CHECK-NEXT:    mov r1, r6
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    mov r5, r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r8:32]
+; CHECK-NEXT:    add.w r1, r4, #16
+; CHECK-NEXT:    vld1.32 {d16[1]}, [r6:32]
+; CHECK-NEXT:    vst1.32 {d16}, [r1:64]!
+; CHECK-NEXT:    bl frexpf
+; CHECK-NEXT:    vmov s1, r5
+; CHECK-NEXT:    vmov s0, r7
+; CHECK-NEXT:    vst1.32 {d0}, [r4:64]!
+; CHECK-NEXT:    str r0, [r4]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+  %result = call { <3 x float>, <3 x i32> } @llvm.frexp.v3float.v3i32(<3 x float> %a)
+  ret { <3 x float>, <3 x i32> } %result
+}
+
 define { float, i32 } @test_frexp_f32_i32(float %a) {
 ; CHECK-LABEL: test_frexp_f32_i32:
 ; CHECK:       @ %bb.0:
diff --git a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
index ec7516524ee6..6706d25ae01d 100644
--- a/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
+++ b/llvm/test/CodeGen/ARM/minnum-maxnum-intrinsics.ll
@@ -1362,7 +1362,7 @@ define void @pr65820(ptr %y, <4 x float> %splat) {
 ; ARMV7-NEXT:    vmov d16, r2, r3
 ; ARMV7-NEXT:    vdup.32 q8, d16[0]
 ; ARMV7-NEXT:    vcgt.f32 q9, q8, #0
-; ARMV7-NEXT:    vand q8, q8, q9
+; ARMV7-NEXT:    vand q8, q9, q8
 ; ARMV7-NEXT:    vst1.32 {d16, d17}, [r0]
 ; ARMV7-NEXT:    bx lr
 ;
diff --git a/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll b/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll
new file mode 100644
index 000000000000..1feb544d42e7
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/powi-soften-libcall-error.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -mtriple=arm-linux-gnu -float-abi=soft -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error(float %x, i64 %n) {
+  %powi = call float @llvm.powi.f32.i64(float %x, i64 %n)
+  ret float %powi
+}
diff --git a/llvm/test/CodeGen/ARM/special-reg.ll b/llvm/test/CodeGen/ARM/special-reg.ll
index e966550e673d..cc95f79d2c73 100644
--- a/llvm/test/CodeGen/ARM/special-reg.ll
+++ b/llvm/test/CodeGen/ARM/special-reg.ll
@@ -25,14 +25,18 @@ entry:
 define i64 @read_volatile_i64_twice() {
 ; ACORE-LABEL: read_volatile_i64_twice:
 ; ACORE:       @ %bb.0: @ %entry
-; ACORE-NEXT:    mov r0, #0
-; ACORE-NEXT:    mov r1, #0
+; ACORE-NEXT:    mrrc p15, #1, r0, r1, c14
+; ACORE-NEXT:    mrrc p15, #1, r2, r3, c14
+; ACORE-NEXT:    eor r0, r2, r0
+; ACORE-NEXT:    eor r1, r3, r1
 ; ACORE-NEXT:    bx lr
 ;
 ; MCORE-LABEL: read_volatile_i64_twice:
 ; MCORE:       @ %bb.0: @ %entry
-; MCORE-NEXT:    movs r0, #0
-; MCORE-NEXT:    movs r1, #0
+; MCORE-NEXT:    mrrc p15, #1, r0, r1, c14
+; MCORE-NEXT:    mrrc p15, #1, r2, r3, c14
+; MCORE-NEXT:    eors r0, r2
+; MCORE-NEXT:    eors r1, r3
 ; MCORE-NEXT:    bx lr
 entry:
   %0 = tail call i64 @llvm.read_volatile_register.i64(metadata !5)
diff --git a/llvm/test/CodeGen/ARM/usat-with-shift.ll b/llvm/test/CodeGen/ARM/usat-with-shift.ll
index cc3de9d6d340..b9c083e498c0 100644
--- a/llvm/test/CodeGen/ARM/usat-with-shift.ll
+++ b/llvm/test/CodeGen/ARM/usat-with-shift.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s
+; RUN: llc -mtriple=armv6-eabi %s -o - | FileCheck %s --check-prefixes=CHECK,ARMV6
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+dsp %s -o - | FileCheck %s --check-prefixes=CHECK,THUMB
 
 define arm_aapcs_vfpcc i32 @usat_lsl(i32 %num){
-; CHECK-LABEL:  usat_lsl
-; CHECK:        @ %bb.0: @ %entry
-; CHECK-NEXT: 	usat	r0, #7, r0, lsl #2
-; CHECK-NEXT:   bx	lr
+; CHECK-LABEL: usat_lsl:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    usat r0, #7, r0, lsl #2
+; CHECK-NEXT:    bx lr
 entry:
   %shl = shl i32 %num, 2
   %0 = tail call i32 @llvm.arm.usat(i32 %shl, i32 7)
@@ -14,10 +14,10 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_asr(i32 %num){
-; CHECK-LABEL:  usat_asr
-; CHECK:        @ %bb.0: @ %entry
-; CHECK-NEXT:   usat	r0, #7, r0, asr #2
-; CHECK-NEXT:   bx	lr
+; CHECK-LABEL: usat_asr:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    usat r0, #7, r0, asr #2
+; CHECK-NEXT:    bx lr
 entry:
   %shr = ashr i32 %num, 2
   %0 = tail call i32 @llvm.arm.usat(i32 %shr, i32 7)
@@ -25,10 +25,24 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_lsl2(i32 %num){
-; CHECK-LABEL: usat_lsl2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    usat r0, #15, r0, lsl #15
-; CHECK-NEXT:    bx lr
+; ARMV6-LABEL: usat_lsl2:
+; ARMV6:       @ %bb.0: @ %entry
+; ARMV6-NEXT:    lsl r0, r0, #15
+; ARMV6-NEXT:    bic r1, r0, r0, asr #31
+; ARMV6-NEXT:    mov r0, #255
+; ARMV6-NEXT:    orr r0, r0, #32512
+; ARMV6-NEXT:    cmp r1, r0
+; ARMV6-NEXT:    movlt r0, r1
+; ARMV6-NEXT:    bx lr
+;
+; THUMB-LABEL: usat_lsl2:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    lsls r0, r0, #15
+; THUMB-NEXT:    movw r1, #32767
+; THUMB-NEXT:    bic.w r0, r0, r0, asr #31
+; THUMB-NEXT:    cmp r0, r1
+; THUMB-NEXT:    csel r0, r0, r1, lt
+; THUMB-NEXT:    bx lr
 entry:
   %shl = shl nsw i32 %num, 15
   %0 = icmp sgt i32 %shl, 0
@@ -39,10 +53,24 @@ entry:
 }
 
 define arm_aapcs_vfpcc i32 @usat_asr2(i32 %num){
-; CHECK-LABEL: usat_asr2:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    usat r0, #15, r0, asr #15
-; CHECK-NEXT:    bx lr
+; ARMV6-LABEL: usat_asr2:
+; ARMV6:       @ %bb.0: @ %entry
+; ARMV6-NEXT:    asr r1, r0, #15
+; ARMV6-NEXT:    bic r1, r1, r0, asr #31
+; ARMV6-NEXT:    mov r0, #255
+; ARMV6-NEXT:    orr r0, r0, #32512
+; ARMV6-NEXT:    cmp r1, r0
+; ARMV6-NEXT:    movlt r0, r1
+; ARMV6-NEXT:    bx lr
+;
+; THUMB-LABEL: usat_asr2:
+; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    asrs r1, r0, #15
+; THUMB-NEXT:    bic.w r0, r1, r0, asr #31
+; THUMB-NEXT:    movw r1, #32767
+; THUMB-NEXT:    cmp r0, r1
+; THUMB-NEXT:    csel r0, r0, r1, lt
+; THUMB-NEXT:    bx lr
 entry:
   %shr = ashr i32 %num, 15
   %0 = icmp sgt i32 %shr, 0
diff --git a/llvm/test/CodeGen/ARM/usat.ll b/llvm/test/CodeGen/ARM/usat.ll
index d01aa1520b32..2e1d0283ebde 100644
--- a/llvm/test/CodeGen/ARM/usat.ll
+++ b/llvm/test/CodeGen/ARM/usat.ll
@@ -32,12 +32,23 @@ define i32 @unsigned_sat_base_32bit(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_base_32bit:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI0_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI0_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_base_32bit:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i32 %x, 8388607
@@ -57,11 +68,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V4T-NEXT:    orr r2, r2, #1792
 ; V4T-NEXT:    asr r1, r1, #16
 ; V4T-NEXT:    cmp r1, r2
-; V4T-NEXT:    movge r0, r2
-; V4T-NEXT:    lsl r1, r0, #16
-; V4T-NEXT:    asr r1, r1, #16
-; V4T-NEXT:    cmp r1, #0
-; V4T-NEXT:    movle r0, #0
+; V4T-NEXT:    movlt r2, r0
+; V4T-NEXT:    lsl r0, r2, #16
+; V4T-NEXT:    bic r0, r2, r0, asr #31
 ; V4T-NEXT:    bx lr
 ;
 ; V6-LABEL: unsigned_sat_base_16bit:
@@ -70,10 +79,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V6-NEXT:    sxth r1, r0
 ; V6-NEXT:    orr r2, r2, #1792
 ; V6-NEXT:    cmp r1, r2
-; V6-NEXT:    movge r0, r2
-; V6-NEXT:    sxth r1, r0
-; V6-NEXT:    cmp r1, #0
-; V6-NEXT:    movle r0, #0
+; V6-NEXT:    movlt r2, r0
+; V6-NEXT:    sxth r0, r2
+; V6-NEXT:    bic r0, r2, r0, asr #15
 ; V6-NEXT:    bx lr
 ;
 ; V6T2-LABEL: unsigned_sat_base_16bit:
@@ -81,10 +89,9 @@ define i16 @unsigned_sat_base_16bit(i16 %x) #0 {
 ; V6T2-NEXT:    sxth r1, r0
 ; V6T2-NEXT:    movw r2, #2047
 ; V6T2-NEXT:    cmp r1, r2
-; V6T2-NEXT:    movge r0, r2
-; V6T2-NEXT:    sxth r1, r0
-; V6T2-NEXT:    cmp r1, #0
-; V6T2-NEXT:    movle r0, #0
+; V6T2-NEXT:    movlt r2, r0
+; V6T2-NEXT:    sxth r0, r2
+; V6T2-NEXT:    bic r0, r2, r0, asr #15
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i16 %x, 2047
@@ -104,9 +111,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V4T-NEXT:    cmp r1, #31
 ; V4T-NEXT:    movge r0, #31
 ; V4T-NEXT:    lsl r1, r0, #24
-; V4T-NEXT:    asr r1, r1, #24
-; V4T-NEXT:    cmp r1, #0
-; V4T-NEXT:    movle r0, #0
+; V4T-NEXT:    bic r0, r0, r1, asr #31
 ; V4T-NEXT:    bx lr
 ;
 ; V6-LABEL: unsigned_sat_base_8bit:
@@ -115,8 +120,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V6-NEXT:    cmp r1, #31
 ; V6-NEXT:    movge r0, #31
 ; V6-NEXT:    sxtb r1, r0
-; V6-NEXT:    cmp r1, #0
-; V6-NEXT:    movle r0, #0
+; V6-NEXT:    bic r0, r0, r1, asr #7
 ; V6-NEXT:    bx lr
 ;
 ; V6T2-LABEL: unsigned_sat_base_8bit:
@@ -125,8 +129,7 @@ define i8 @unsigned_sat_base_8bit(i8 %x) #0 {
 ; V6T2-NEXT:    cmp r1, #31
 ; V6T2-NEXT:    movge r0, #31
 ; V6T2-NEXT:    sxtb r1, r0
-; V6T2-NEXT:    cmp r1, #0
-; V6T2-NEXT:    movle r0, #0
+; V6T2-NEXT:    bic r0, r0, r1, asr #7
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i8 %x, 31
@@ -157,12 +160,23 @@ define i32 @unsigned_sat_lower_upper_1(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_lower_upper_1:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI3_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI3_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_lower_upper_1:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %cmpUp = icmp slt i32 %x, 8388607
@@ -188,12 +202,23 @@ define i32 @unsigned_sat_lower_upper_2(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_lower_upper_2:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    ldr r1, .LCPI4_0
+; V6-NEXT:    cmp r0, r1
+; V6-NEXT:    movlt r1, r0
+; V6-NEXT:    bic r0, r1, r1, asr #31
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI4_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_lower_upper_2:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    movw r1, #65535
+; V6T2-NEXT:    movt r1, #127
+; V6T2-NEXT:    cmp r0, r1
+; V6T2-NEXT:    movlt r1, r0
+; V6T2-NEXT:    bic r0, r1, r1, asr #31
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp slt i32 %x, 8388607
@@ -219,12 +244,23 @@ define i32 @unsigned_sat_upper_lower_1(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_1:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI5_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI5_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_1:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp sgt i32 %x, 0
@@ -250,12 +286,23 @@ define i32 @unsigned_sat_upper_lower_2(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_2:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI6_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI6_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_2:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = icmp sgt i32 %x, 0
@@ -281,12 +328,23 @@ define i32 @unsigned_sat_upper_lower_3(i32 %x) #0 {
 ;
 ; V6-LABEL: unsigned_sat_upper_lower_3:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    usat r0, #23, r0
+; V6-NEXT:    bic r1, r0, r0, asr #31
+; V6-NEXT:    ldr r0, .LCPI7_0
+; V6-NEXT:    cmp r1, r0
+; V6-NEXT:    movlt r0, r1
 ; V6-NEXT:    bx lr
+; V6-NEXT:    .p2align 2
+; V6-NEXT:  @ %bb.1:
+; V6-NEXT:  .LCPI7_0:
+; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: unsigned_sat_upper_lower_3:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    usat r0, #23, r0
+; V6T2-NEXT:    bic r1, r0, r0, asr #31
+; V6T2-NEXT:    movw r0, #65535
+; V6T2-NEXT:    movt r0, #127
+; V6T2-NEXT:    cmp r1, r0
+; V6T2-NEXT:    movlt r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %cmpLow = icmp sgt i32 %x, 0
diff --git a/llvm/test/CodeGen/ARM/vrint.ll b/llvm/test/CodeGen/ARM/vrint.ll
index 2391e9499479..52107aac6518 100644
--- a/llvm/test/CodeGen/ARM/vrint.ll
+++ b/llvm/test/CodeGen/ARM/vrint.ll
@@ -79,21 +79,7 @@ define <4 x half> @frinta_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frinta_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrinta.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrinta.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.round.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -243,35 +229,7 @@ define <8 x half> @frinta_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frinta_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrinta.f16 s4, s1
-; CHECK-FP16-NEXT:    vrinta.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrinta.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.round.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -297,9 +255,7 @@ define <2 x float> @frinta_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frinta_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrinta.f32 s3, s1
-; CHECK-NEXT:    vrinta.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrinta.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.round.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -331,11 +287,7 @@ define <4 x float> @frinta_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frinta_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrinta.f32 s7, s3
-; CHECK-NEXT:    vrinta.f32 s6, s2
-; CHECK-NEXT:    vrinta.f32 s5, s1
-; CHECK-NEXT:    vrinta.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrinta.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.round.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -813,21 +765,7 @@ define <4 x half> @frintm_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintm_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintm.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintm.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.floor.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -977,35 +915,7 @@ define <8 x half> @frintm_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintm_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintm.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintm.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintm.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.floor.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1031,9 +941,7 @@ define <2 x float> @frintm_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintm_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintm.f32 s3, s1
-; CHECK-NEXT:    vrintm.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintm.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.floor.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1065,11 +973,7 @@ define <4 x float> @frintm_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintm_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintm.f32 s7, s3
-; CHECK-NEXT:    vrintm.f32 s6, s2
-; CHECK-NEXT:    vrintm.f32 s5, s1
-; CHECK-NEXT:    vrintm.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintm.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.floor.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1180,21 +1084,7 @@ define <4 x half> @frintn_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintn_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintn.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintn.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.roundeven.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -1344,35 +1234,7 @@ define <8 x half> @frintn_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintn_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintn.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintn.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintn.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.roundeven.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1398,9 +1260,7 @@ define <2 x float> @frintn_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintn_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintn.f32 s3, s1
-; CHECK-NEXT:    vrintn.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintn.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1432,11 +1292,7 @@ define <4 x float> @frintn_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintn_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintn.f32 s7, s3
-; CHECK-NEXT:    vrintn.f32 s6, s2
-; CHECK-NEXT:    vrintn.f32 s5, s1
-; CHECK-NEXT:    vrintn.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintn.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1547,21 +1403,7 @@ define <4 x half> @frintp_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintp_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintp.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintp.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.ceil.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -1711,35 +1553,7 @@ define <8 x half> @frintp_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintp_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintp.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintp.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintp.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.ceil.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -1765,9 +1579,7 @@ define <2 x float> @frintp_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintp_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintp.f32 s3, s1
-; CHECK-NEXT:    vrintp.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintp.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.ceil.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -1799,11 +1611,7 @@ define <4 x float> @frintp_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintp_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintp.f32 s7, s3
-; CHECK-NEXT:    vrintp.f32 s6, s2
-; CHECK-NEXT:    vrintp.f32 s5, s1
-; CHECK-NEXT:    vrintp.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintp.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.ceil.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -1914,21 +1722,7 @@ define <4 x half> @frintx_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintx_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintx.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintx.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.rint.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -2078,35 +1872,7 @@ define <8 x half> @frintx_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintx_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintx.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintx.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintx.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.rint.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -2132,9 +1898,7 @@ define <2 x float> @frintx_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintx_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintx.f32 s3, s1
-; CHECK-NEXT:    vrintx.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintx.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.rint.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -2166,11 +1930,7 @@ define <4 x float> @frintx_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintx_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintx.f32 s7, s3
-; CHECK-NEXT:    vrintx.f32 s6, s2
-; CHECK-NEXT:    vrintx.f32 s5, s1
-; CHECK-NEXT:    vrintx.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintx.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.rint.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
@@ -2281,21 +2041,7 @@ define <4 x half> @frintz_4h(<4 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintz_4h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s2, s0
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s2
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s0
-; CHECK-FP16-NEXT:    vmov r1, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s2, s1
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vrintz.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s2
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr d0, d16, d16
+; CHECK-FP16-NEXT:    vrintz.f16 d0, d0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <4 x half> @llvm.trunc.v4f16(<4 x half> %A)
   ret <4 x half> %tmp3
@@ -2445,35 +2191,7 @@ define <8 x half> @frintz_8h(<8 x half> %A) nounwind {
 ;
 ; CHECK-FP16-LABEL: frintz_8h:
 ; CHECK-FP16:       @ %bb.0:
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s2
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s2
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s3
-; CHECK-FP16-NEXT:    vmov.16 d17[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d17[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s3
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmovx.f16 s4, s0
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s4
-; CHECK-FP16-NEXT:    vmov.16 d17[3], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s0
-; CHECK-FP16-NEXT:    vmovx.f16 s0, s1
-; CHECK-FP16-NEXT:    vmov r1, s4
-; CHECK-FP16-NEXT:    vrintz.f16 s4, s1
-; CHECK-FP16-NEXT:    vrintz.f16 s0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[0], r1
-; CHECK-FP16-NEXT:    vmov.16 d16[1], r0
-; CHECK-FP16-NEXT:    vmov r0, s4
-; CHECK-FP16-NEXT:    vmov.16 d16[2], r0
-; CHECK-FP16-NEXT:    vmov r0, s0
-; CHECK-FP16-NEXT:    vmov.16 d16[3], r0
-; CHECK-FP16-NEXT:    vorr q0, q8, q8
+; CHECK-FP16-NEXT:    vrintz.f16 q0, q0
 ; CHECK-FP16-NEXT:    bx lr
   %tmp3 = call <8 x half> @llvm.trunc.v8f16(<8 x half> %A)
   ret <8 x half> %tmp3
@@ -2499,9 +2217,7 @@ define <2 x float> @frintz_2s(<2 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintz_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintz.f32 s3, s1
-; CHECK-NEXT:    vrintz.f32 s2, s0
-; CHECK-NEXT:    vmov.f64 d0, d1
+; CHECK-NEXT:    vrintz.f32 d0, d0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x float> @llvm.trunc.v2f32(<2 x float> %A)
   ret <2 x float> %tmp3
@@ -2533,11 +2249,7 @@ define <4 x float> @frintz_4s(<4 x float> %A) nounwind {
 ;
 ; CHECK-LABEL: frintz_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vrintz.f32 s7, s3
-; CHECK-NEXT:    vrintz.f32 s6, s2
-; CHECK-NEXT:    vrintz.f32 s5, s1
-; CHECK-NEXT:    vrintz.f32 s4, s0
-; CHECK-NEXT:    vorr q0, q1, q1
+; CHECK-NEXT:    vrintz.f32 q0, q0
 ; CHECK-NEXT:    bx lr
   %tmp3 = call <4 x float> @llvm.trunc.v4f32(<4 x float> %A)
   ret <4 x float> %tmp3
diff --git a/llvm/test/CodeGen/ARM/vrintn.ll b/llvm/test/CodeGen/ARM/vrintn.ll
index 40f806ba55d3..077007b22fc4 100644
--- a/llvm/test/CodeGen/ARM/vrintn.ll
+++ b/llvm/test/CodeGen/ARM/vrintn.ll
@@ -1,12 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=armv8 -mattr=+neon %s -o - | FileCheck %s
 
 declare float @llvm.arm.neon.vrintn.f32(float) nounwind readnone
 declare <2 x float> @llvm.arm.neon.vrintn.v2f32(<2 x float>) nounwind readnone
 declare <4 x float> @llvm.arm.neon.vrintn.v4f32(<4 x float>) nounwind readnone
 
-; CHECK-LABEL: vrintn_f32:
-; CHECK: vrintn.f32
 define float @vrintn_f32(ptr %A) nounwind {
+; CHECK-LABEL: vrintn_f32:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr s0, [r0]
+; CHECK-NEXT:    vrintn.f32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
   %tmp1 = load float, ptr %A
   %tmp2 = call float @llvm.arm.neon.vrintn.f32(float %tmp1)
   ret float %tmp2
@@ -74,10 +79,9 @@ define <4 x half> @roundeven_4h(<4 x half> %A) nounwind {
 define <2 x float> @roundeven_2s(<2 x float> %A) nounwind {
 ; CHECK-LABEL: roundeven_2s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vrintn.f32 s3, s1
-; CHECK-NEXT:    vrintn.f32 s2, s0
-; CHECK-NEXT:    vmov r0, r1, d1
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vrintn.f32 d16, d16
+; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    bx lr
 	%tmp3 = call <2 x float> @llvm.roundeven.v2f32(<2 x float> %A)
 	ret <2 x float> %tmp3
@@ -86,14 +90,11 @@ define <2 x float> @roundeven_2s(<2 x float> %A) nounwind {
 define <4 x float> @roundeven_4s(<4 x float> %A) nounwind {
 ; CHECK-LABEL: roundeven_4s:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmov d1, r2, r3
-; CHECK-NEXT:    vmov d0, r0, r1
-; CHECK-NEXT:    vrintn.f32 s7, s3
-; CHECK-NEXT:    vrintn.f32 s6, s2
-; CHECK-NEXT:    vrintn.f32 s5, s1
-; CHECK-NEXT:    vrintn.f32 s4, s0
-; CHECK-NEXT:    vmov r2, r3, d3
-; CHECK-NEXT:    vmov r0, r1, d2
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vmov d16, r0, r1
+; CHECK-NEXT:    vrintn.f32 q8, q8
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    bx lr
 	%tmp3 = call <4 x float> @llvm.roundeven.v4f32(<4 x float> %A)
 	ret <4 x float> %tmp3
diff --git a/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll b/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
index 80a071a66364..25abf2111060 100644
--- a/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
+++ b/llvm/test/CodeGen/DirectX/BufferLoadDouble.ll
@@ -88,4 +88,4 @@ define void @loadf64WithCheckBit() {
   ; CHECK-NOT: extractvalue { double, i1 }
   %cb = extractvalue {double, i1} %load0, 1
   ret void
-}
\ No newline at end of file
+}
diff --git a/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll b/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll
new file mode 100644
index 000000000000..42c0012ff347
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/BufferLoadInt64.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-intrinsic-expansion %s | FileCheck %s
+
+target triple = "dxil-pc-shadermodel6.2-compute"
+
+define void @loadi64() {
+; CHECK-LABEL: define void @loadi64() {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i32>, i1 } @llvm.dx.resource.load.typedbuffer.v2i32.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) [[BUFFER]], i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i32>, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 32
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %result = call { i64, i1 } @llvm.dx.resource.load.typedbuffer.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) %buffer, i32 0)
+  ret void
+}
+
+define void @loadv2i64() {
+; CHECK-LABEL: define void @loadv2i64() {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, i1 } @llvm.dx.resource.load.typedbuffer.v4i32.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) [[BUFFER]], i32 0)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP8]], 32
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> poison, i64 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[TMP13]], 32
+; CHECK-NEXT:    [[TMP15:%.*]] = or i64 [[TMP12]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP15]], i32 1
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  %result = call { <2 x i64>, i1 } @llvm.dx.resource.load.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) %buffer, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll b/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll
new file mode 100644
index 000000000000..c97a02d1873a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/BufferStoreInt64.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-intrinsic-expansion %s | FileCheck %s
+
+target triple = "dxil-pc-shadermodel6.6-compute"
+
+define void @storei64(i64 %0) {
+; CHECK-LABEL: define void @storei64(
+; CHECK-SAME: i64 [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP0]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP4]], i32 1
+; CHECK-NEXT:    call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_i64_1_0_0t.v2i32(target("dx.TypedBuffer", i64, 1, 0, 0) [[BUFFER]], i32 0, <2 x i32> [[TMP6]])
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", i64, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_i64_1_0_0t(target("dx.TypedBuffer", i64, 1, 0, 0) %buffer, i32 0,i64 %0)
+  ret void
+}
+
+
+define void @storev2i64(<2 x i64> %0) {
+; CHECK-LABEL: define void @storev2i64(
+; CHECK-SAME: <2 x i64> [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[BUFFER:%.*]] = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <2 x i64> [[TMP0]] to <2 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <2 x i64> [[TMP0]], splat (i64 32)
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP4]], <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t.v4i32(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) [[BUFFER]], i32 0, <4 x i32> [[TMP13]])
+; CHECK-NEXT:    ret void
+;
+  %buffer = tail call target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v2i64_1_0_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  call void @llvm.dx.resource.store.typedbuffer.tdx.TypedBuffer_v2i64_1_0_0t(target("dx.TypedBuffer", <2 x i64>, 1, 0, 0) %buffer, i32 0, <2 x i64> %0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll b/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll
new file mode 100644
index 000000000000..001f3320137a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/CBufferAccess/memcpy.ll
@@ -0,0 +1,216 @@
+; RUN: opt -S -dxil-cbuffer-access -mtriple=dxil--shadermodel6.3-library %s | FileCheck %s
+
+; cbuffer CB : register(b0) {
+;   float a1[3];
+;   double3 a2[2];
+;   float16_t a3[2][2];
+;   uint64_t a4[3];
+;   int2 a5[3][2];
+;   uint16_t a6[1];
+;   int64_t a7[2];
+;   bool a8[4];
+; }
+%__cblayout_CB = type <{ [3 x float], [2 x <3 x double>], [2 x [2 x half]], [3 x i64], [3 x [2 x <2 x i32>]], [1 x i16], [2 x i64], [4 x i32] }>
+
+@CB.cb = local_unnamed_addr global target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) poison
+@a1 = external local_unnamed_addr addrspace(2) global [3 x float], align 4
+@a2 = external local_unnamed_addr addrspace(2) global [2 x <3 x double>], align 32
+@a3 = external local_unnamed_addr addrspace(2) global [2 x [2 x half]], align 2
+@a4 = external local_unnamed_addr addrspace(2) global [3 x i64], align 8
+@a5 = external local_unnamed_addr addrspace(2) global [3 x [2 x <2 x i32>]], align 16
+@a6 = external local_unnamed_addr addrspace(2) global [1 x i16], align 2
+@a7 = external local_unnamed_addr addrspace(2) global [2 x i64], align 8
+@a8 = external local_unnamed_addr addrspace(2) global [4 x i32], align 4
+
+; CHECK: define void @f(
+define void @f(ptr %dst) {
+entry:
+  %CB.cb_h.i.i = tail call target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) @llvm.dx.resource.handlefrombinding(i32 0, i32 0, i32 1, i32 0, i1 false, ptr null)
+  store target("dx.CBuffer", target("dx.Layout", %__cblayout_CB, 708, 0, 48, 112, 176, 224, 272, 288, 320)) %CB.cb_h.i.i, ptr @CB.cb, align 4
+
+  %a1.copy = alloca [3 x float], align 4
+  %a2.copy = alloca [2 x <3 x double>], align 32
+  %a3.copy = alloca [2 x [2 x half]], align 2
+  %a4.copy = alloca [3 x i64], align 8
+  %a5.copy = alloca [3 x [2 x <2 x i32>]], align 16
+  %a6.copy = alloca [1 x i16], align 2
+  %a7.copy = alloca [2 x i64], align 8
+  %a8.copy = alloca [4 x i32], align 4
+
+  ; Try copying no elements
+; CHECK-NOT: memcpy
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 0, i1 false)
+
+  ; Try copying only the first element
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+; CHECK:    [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY:%.*]], i32 0
+; CHECK:    store float [[X]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 4, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 0)
+; CHECK:    [[X:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY:%.*]], i32 0
+; CHECK:    store float [[X]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 1)
+; CHECK:    [[Y:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY]], i32 4
+; CHECK:    store float [[Y]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { float, float, float, float } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 2)
+; CHECK:    [[Z:%.*]] = extractvalue { float, float, float, float } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A1_COPY]], i32 8
+; CHECK:    store float [[Z]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a1.copy, ptr addrspace(2) align 4 @a1, i32 12, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 3)
+; CHECK:    [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 4)
+; CHECK:    [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[UPTO0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <3 x double> [[UPTO0]], double [[Y]], i32 1
+; CHECK:    [[UPTO2:%.*]] = insertelement <3 x double> [[UPTO1]], double [[Z]], i32 2
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A2_COPY:%.*]], i32 0
+; CHECK:    store <3 x double> [[UPTO2]], ptr [[DEST]], align 32
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 5)
+; CHECK:    [[X:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { double, double } [[LOAD]], 1
+; CHECK:    [[LOAD:%.*]] = call { double, double } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 6)
+; CHECK:    [[Z:%.*]] = extractvalue { double, double } [[LOAD]], 0
+; CHECK:    [[UPTO0:%.*]] = insertelement <3 x double> poison, double [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <3 x double> [[UPTO0]], double [[Y]], i32 1
+; CHECK:    [[UPTO2:%.*]] = insertelement <3 x double> [[UPTO1]], double [[Z]], i32 2
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A2_COPY]], i32 32
+; CHECK:    store <3 x double> [[UPTO2]], ptr [[DEST]], align 32
+  call void @llvm.memcpy.p0.p2.i32(ptr align 32 %a2.copy, ptr addrspace(2) align 32 @a2, i32 64, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 7)
+; CHECK:    [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY:%.*]], i32 0
+; CHECK:    store half [[X]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 8)
+; CHECK:    [[Y:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 2
+; CHECK:    store half [[Y]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 9)
+; CHECK:    [[X:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 4
+; CHECK:    store half [[X]], ptr [[DEST]], align 2
+; CHECK:    [[LOAD:%.*]] = call { half, half, half, half, half, half, half, half } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 10)
+; CHECK:    [[Y:%.*]] = extractvalue { half, half, half, half, half, half, half, half } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A3_COPY]], i32 6
+; CHECK:    store half [[Y]], ptr [[DEST]], align 2
+  call void @llvm.memcpy.p0.p2.i32(ptr align 2 %a3.copy, ptr addrspace(2) align 2 @a3, i32 8, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 11)
+; CHECK:    [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY:%.*]], i32 0
+; CHECK:    store i64 [[X]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 12)
+; CHECK:    [[Y:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY]], i32 8
+; CHECK:    store i64 [[Y]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 13)
+; CHECK:    [[Z:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A4_COPY]], i32 16
+; CHECK:    store i64 [[Z]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 8 %a4.copy, ptr addrspace(2) align 8 @a4, i32 24, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 14)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY:%.*]], i32 0
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 15)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 8
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 16)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 16
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 17)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 24
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 18)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 32
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 19)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 1
+; CHECK:    [[UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[X]], i32 0
+; CHECK:    [[UPTO1:%.*]] = insertelement <2 x i32> [[UPTO0]], i32 [[Y]], i32 1
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A5_COPY]], i32 40
+; CHECK:    store <2 x i32> [[UPTO1]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 16 %a5.copy, ptr addrspace(2) align 16 @a5, i32 48, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i16, i16, i16, i16, i16, i16, i16, i16 } @llvm.dx.resource.load.cbufferrow.8.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 17)
+; CHECK:    [[X:%.*]] = extractvalue { i16, i16, i16, i16, i16, i16, i16, i16 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A6_COPY:%.*]], i32 0
+; CHECK:    store i16 [[X]], ptr [[DEST]], align 2
+  call void @llvm.memcpy.p0.p2.i32(ptr align 2 %a6.copy, ptr addrspace(2) align 2 @a6, i32 2, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 18)
+; CHECK:    [[X:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A7_COPY:%.*]], i32 0
+; CHECK:    store i64 [[X]], ptr [[DEST]], align 8
+; CHECK:    [[LOAD:%.*]] = call { i64, i64 } @llvm.dx.resource.load.cbufferrow.2.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 19)
+; CHECK:    [[Y:%.*]] = extractvalue { i64, i64 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A7_COPY]], i32 8
+; CHECK:    store i64 [[Y]], ptr [[DEST]], align 8
+  call void @llvm.memcpy.p0.p2.i32(ptr align 8 %a7.copy, ptr addrspace(2) align 8 @a7, i32 16, i1 false)
+
+; CHECK:    [[CB:%.*]] = load target("dx.CBuffer", {{.*}})), ptr @CB.cb, align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 20)
+; CHECK:    [[X:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY:%.*]], i32 0
+; CHECK:    store i32 [[X]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 21)
+; CHECK:    [[Y:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 4
+; CHECK:    store i32 [[Y]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 22)
+; CHECK:    [[Z:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 8
+; CHECK:    store i32 [[Z]], ptr [[DEST]], align 4
+; CHECK:    [[LOAD:%.*]] = call { i32, i32, i32, i32 } @llvm.dx.resource.load.cbufferrow.4.{{.*}}(target("dx.CBuffer", {{.*}})) [[CB]], i32 23)
+; CHECK:    [[W:%.*]] = extractvalue { i32, i32, i32, i32 } [[LOAD]], 0
+; CHECK:    [[DEST:%.*]] = getelementptr inbounds i8, ptr [[A8_COPY]], i32 12
+; CHECK:    store i32 [[W]], ptr [[DEST]], align 4
+  call void @llvm.memcpy.p0.p2.i32(ptr align 4 %a8.copy, ptr addrspace(2) align 4 @a8, i32 16, i1 false)
+
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p2.i32(ptr noalias writeonly captures(none), ptr addrspace(2) noalias readonly captures(none), i32, i1 immarg)
+
+; CHECK-NOT: !hlsl.cbs =
+!hlsl.cbs = !{!0}
+
+!0 = !{ptr @CB.cb, ptr addrspace(2) @a1, ptr addrspace(2) @a2, ptr addrspace(2) @a3, ptr addrspace(2) @a4, ptr addrspace(2) @a5, ptr addrspace(2) @a6, ptr addrspace(2) @a7, ptr addrspace(2) @a8}
+!1 = !{i32 0, i32 2}
+!2 = !{}
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll
new file mode 100644
index 000000000000..288dea00b6e5
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressU.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressU: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 666, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll
new file mode 100644
index 000000000000..e9abcf966999
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressV.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressV: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 666, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll
new file mode 100644
index 000000000000..238f488ee78d
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-AddressW.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for AddressW: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 666, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll
new file mode 100644
index 000000000000..8dc69eb1f9d7
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-BorderColor.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for BorderColor: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 666, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll
new file mode 100644
index 000000000000..b2c8faf8d4a0
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ComparisonFunc.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ComparisonFunc: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 666, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll
new file mode 100644
index 000000000000..758d2629ed78
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-Filter.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for Filter: 45 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 45, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll
new file mode 100644
index 000000000000..47d4b52d72e8
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxAnisotropy.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MaxAnisotropy: 666 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 666, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll
new file mode 100644
index 000000000000..7e7d56eff309
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MaxLod.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MaxLOD: 0
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 0x7FF8000000000000, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll
new file mode 100644
index 000000000000..d958f10d3c1a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLod.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MinLOD: 0
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float 0x7FF8000000000000, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll
new file mode 100644
index 000000000000..34b27eb40f5f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-MinLopBias.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for MipLODBias: 666 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 6.660000e+02, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll
new file mode 100644
index 000000000000..dc6ee4290b53
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-RegisterSpace.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for RegisterSpace: 4294967280
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 4294967280, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll
new file mode 100644
index 000000000000..6cee1dd95fd8
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderRegister.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ShaderRegister: 4294967295
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 4294967295, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll
new file mode 100644
index 000000000000..fa5bf12e2b8c
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers-Invalid-ShaderVisibility.ll
@@ -0,0 +1,19 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for ShaderVisibility: 666
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 666 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll
new file mode 100644
index 000000000000..d9ee39dbb728
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-StaticSamplers.ll
@@ -0,0 +1,42 @@
+; RUN: opt %s -dxil-embed -dxil-globals -S -o - | FileCheck %s
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: @dx.rts0 = private constant [76 x i8]  c"{{.*}}", section "RTS0", align 4
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
+
+; DXC: - Name:            RTS0
+; DXC-NEXT:     Size:            76
+; DXC-NEXT:     RootSignature:
+; DXC-NEXT:       Version:         2
+; DXC-NEXT:       NumRootParameters: 0
+; DXC-NEXT:       RootParametersOffset: 24
+; DXC-NEXT:       NumStaticSamplers: 1
+; DXC-NEXT:       StaticSamplersOffset: 24
+; DXC-NEXT:       Parameters:      []
+; DXC-NEXT:       Samplers:
+; DXC-NEXT:         - Filter:          4
+; DXC-NEXT:           AddressU:        2
+; DXC-NEXT:           AddressV:        3
+; DXC-NEXT:           AddressW:        5
+; DXC-NEXT:           MipLODBias:      1.425
+; DXC-NEXT:           MaxAnisotropy:   9
+; DXC-NEXT:           ComparisonFunc:  3
+; DXC-NEXT:           BorderColor:     2
+; DXC-NEXT:           MinLOD:          -128
+; DXC-NEXT:           MaxLOD:          128
+; DXC-NEXT:           ShaderRegister:  42
+; DXC-NEXT:           RegisterSpace:   0
+; DXC-NEXT:           ShaderVisibility: 0
diff --git a/llvm/test/CodeGen/DirectX/firstbithigh.ll b/llvm/test/CodeGen/DirectX/firstbithigh.ll
index 794b0f20a026..bf9efd30d9b2 100644
--- a/llvm/test/CodeGen/DirectX/firstbithigh.ll
+++ b/llvm/test/CodeGen/DirectX/firstbithigh.ll
@@ -4,42 +4,42 @@
 
 define noundef i32 @test_firstbithigh_ushort(i16 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i16(i32 33, i16 %{{.*}}) #[[#ATTR:]]
+; CHECK: call i32 @dx.op.unaryBits.i16(i32 33, i16 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbituhigh.i16(i16 %a)
   ret i32 %elt.firstbithigh
 }
 
 define noundef i32 @test_firstbithigh_short(i16 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i16(i32 34, i16 %{{.*}}) #[[#ATTR]]
+; CHECK: call i32 @dx.op.unaryBits.i16(i32 34, i16 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbitshigh.i16(i16 %a)
   ret i32 %elt.firstbithigh
 }
 
 define noundef i32 @test_firstbithigh_uint(i32 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i32(i32 33, i32 %{{.*}}) #[[#ATTR]]
+; CHECK: call i32 @dx.op.unaryBits.i32(i32 33, i32 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbituhigh.i32(i32 %a)
   ret i32 %elt.firstbithigh
 }
 
 define noundef i32 @test_firstbithigh_int(i32 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i32(i32 34, i32 %{{.*}}) #[[#ATTR]]
+; CHECK: call i32 @dx.op.unaryBits.i32(i32 34, i32 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbitshigh.i32(i32 %a)
   ret i32 %elt.firstbithigh
 }
 
 define noundef i32 @test_firstbithigh_ulong(i64 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i64(i32 33, i64 %{{.*}}) #[[#ATTR]]
+; CHECK: call i32 @dx.op.unaryBits.i64(i32 33, i64 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbituhigh.i64(i64 %a)
   ret i32 %elt.firstbithigh
 }
 
 define noundef i32 @test_firstbithigh_long(i64 noundef %a) {
 entry:
-; CHECK: call i32 @dx.op.unaryBits.i64(i32 34, i64 %{{.*}}) #[[#ATTR]]
+; CHECK: call i32 @dx.op.unaryBits.i64(i32 34, i64 %{{.*}})
   %elt.firstbithigh = call i32 @llvm.dx.firstbitshigh.i64(i64 %a)
   ret i32 %elt.firstbithigh
 }
@@ -47,17 +47,18 @@ entry:
 define noundef <4 x i32> @test_firstbituhigh_vec4_i32(<4 x i32> noundef %a)  {
 entry:
   ; CHECK: [[ee0:%.*]] = extractelement <4 x i32> %a, i64 0
-  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee0]]) #[[#ATTR]]
+  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee0]])
   ; CHECK: [[ee1:%.*]] = extractelement <4 x i32> %a, i64 1
-  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee1]]) #[[#ATTR]]
+  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee1]])
   ; CHECK: [[ee2:%.*]] = extractelement <4 x i32> %a, i64 2
-  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee2]]) #[[#ATTR]]
+  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee2]])
   ; CHECK: [[ee3:%.*]] = extractelement <4 x i32> %a, i64 3
-  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee3]]) #[[#ATTR]]
-  ; CHECK: insertelement <4 x i32> poison, i32 [[ie0]], i64 0
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie1]], i64 1
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie2]], i64 2
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie3]], i64 3
+  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 33, i32 [[ee3]])
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
   %2 = call <4 x i32> @llvm.dx.firstbituhigh.v4i32(<4 x i32> %a)
   ret <4 x i32> %2
 }
@@ -65,29 +66,56 @@ entry:
 define noundef <4 x i32> @test_firstbitshigh_vec4_i32(<4 x i32> noundef %a)  {
 entry:
   ; CHECK: [[ee0:%.*]] = extractelement <4 x i32> %a, i64 0
-  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee0]]) #[[#ATTR]]
+  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee0]])
   ; CHECK: [[ee1:%.*]] = extractelement <4 x i32> %a, i64 1
-  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee1]]) #[[#ATTR]]
+  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee1]])
   ; CHECK: [[ee2:%.*]] = extractelement <4 x i32> %a, i64 2
-  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee2]]) #[[#ATTR]]
+  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee2]])
   ; CHECK: [[ee3:%.*]] = extractelement <4 x i32> %a, i64 3
-  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee3]]) #[[#ATTR]]
-  ; CHECK: insertelement <4 x i32> poison, i32 [[ie0]], i64 0
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie1]], i64 1
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie2]], i64 2
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie3]], i64 3
+  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 34, i32 [[ee3]])
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
   %2 = call <4 x i32> @llvm.dx.firstbitshigh.v4i32(<4 x i32> %a)
   ret <4 x i32> %2
 }
 
-; CHECK: attributes #[[#ATTR]] = {{{.*}} memory(none) {{.*}}}
-
-declare i32 @llvm.dx.firstbituhigh.i16(i16)
-declare i32 @llvm.dx.firstbituhigh.i32(i32)
-declare i32 @llvm.dx.firstbituhigh.i64(i64)
-declare <4 x i32> @llvm.dx.firstbituhigh.v4i32(<4 x i32>)
+define noundef <4 x i32> @test_firstbituhigh_vec4_i16(<4 x i16> noundef %a)  {
+entry:
+  ; CHECK: [[ee0:%.*]] = extractelement <4 x i16> %a, i64 0
+  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i16(i32 33, i16 [[ee0]])
+  ; CHECK: [[ee1:%.*]] = extractelement <4 x i16> %a, i64 1
+  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i16(i32 33, i16 [[ee1]])
+  ; CHECK: [[ee2:%.*]] = extractelement <4 x i16> %a, i64 2
+  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i16(i32 33, i16 [[ee2]])
+  ; CHECK: [[ee3:%.*]] = extractelement <4 x i16> %a, i64 3
+  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i16(i32 33, i16 [[ee3]])
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
+  %2 = call <4 x i32> @llvm.dx.firstbituhigh.v4i16(<4 x i16> %a)
+  ret <4 x i32> %2
+}
 
-declare i32 @llvm.dx.firstbitshigh.i16(i16)
-declare i32 @llvm.dx.firstbitshigh.i32(i32)
-declare i32 @llvm.dx.firstbitshigh.i64(i64)
-declare <4 x i32> @llvm.dx.firstbitshigh.v4i32(<4 x i32>)
+define noundef <4 x i32> @test_firstbitshigh_vec4_i16(<4 x i16> noundef %a)  {
+entry:
+  ; CHECK: [[ee0:%.*]] = extractelement <4 x i16> %a, i64 0
+  ; CHECK: [[ie0:%.*]] = call i32 @dx.op.unaryBits.i16(i32 34, i16 [[ee0]])
+  ; CHECK: [[ee1:%.*]] = extractelement <4 x i16> %a, i64 1
+  ; CHECK: [[ie1:%.*]] = call i32 @dx.op.unaryBits.i16(i32 34, i16 [[ee1]])
+  ; CHECK: [[ee2:%.*]] = extractelement <4 x i16> %a, i64 2
+  ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i16(i32 34, i16 [[ee2]])
+  ; CHECK: [[ee3:%.*]] = extractelement <4 x i16> %a, i64 3
+  ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i16(i32 34, i16 [[ee3]])
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
+  %2 = call <4 x i32> @llvm.dx.firstbitshigh.v4i16(<4 x i16> %a)
+  ret <4 x i32> %2
+}
diff --git a/llvm/test/CodeGen/DirectX/firstbitlow.ll b/llvm/test/CodeGen/DirectX/firstbitlow.ll
index fe3786d3fbeb..7f5f0b2dd7ed 100644
--- a/llvm/test/CodeGen/DirectX/firstbitlow.ll
+++ b/llvm/test/CodeGen/DirectX/firstbitlow.ll
@@ -33,10 +33,11 @@ entry:
   ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i32(i32 32, i32 [[ee2]])
   ; CHECK: [[ee3:%.*]] = extractelement <4 x i32> %a, i64 3
   ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i32(i32 32, i32 [[ee3]])
-  ; CHECK: insertelement <4 x i32> poison, i32 [[ie0]], i64 0
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie1]], i64 1
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie2]], i64 2
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie3]], i64 3
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
   %2 = call <4 x i32> @llvm.dx.firstbitlow.v4i32(<4 x i32> %a)
   ret <4 x i32> %2
 }
@@ -51,10 +52,11 @@ entry:
   ; CHECK: [[ie2:%.*]] = call i32 @dx.op.unaryBits.i16(i32 32, i16 [[ee2]])
   ; CHECK: [[ee3:%.*]] = extractelement <4 x i16> %a, i64 3
   ; CHECK: [[ie3:%.*]] = call i32 @dx.op.unaryBits.i16(i32 32, i16 [[ee3]])
-  ; CHECK: insertelement <4 x i32> poison, i32 [[ie0]], i64 0
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie1]], i64 1
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie2]], i64 2
-  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 [[ie3]], i64 3
+  ; CHECK: [[rt0:%.*]] = insertelement <4 x i32> poison, i32 [[ie0]], i64 0
+  ; CHECK: [[rt1:%.*]] = insertelement <4 x i32> [[rt0]], i32 [[ie1]], i64 1
+  ; CHECK: [[rt2:%.*]] = insertelement <4 x i32> [[rt1]], i32 [[ie2]], i64 2
+  ; CHECK: [[rt3:%.*]] = insertelement <4 x i32> [[rt2]], i32 [[ie3]], i64 3
+  ; CHECK: ret <4 x i32> [[rt3]]
   %2 = call <4 x i32> @llvm.dx.firstbitlow.v4i16(<4 x i16> %a)
   ret <4 x i32> %2
 }
diff --git a/llvm/test/CodeGen/DirectX/issue-145408-gep-struct-fix.ll b/llvm/test/CodeGen/DirectX/issue-145408-gep-struct-fix.ll
new file mode 100644
index 000000000000..40d222cdf2f8
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/issue-145408-gep-struct-fix.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-data-scalarization -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+%struct.RawStruct8D = type { [8 x i32] }
+
+define void @test_no_transform_of_struct()  {
+; CHECK-LABEL: define void @test_no_transform_of_struct() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OUTPUTSIZESLOCAL_I:%.*]] = alloca [[STRUCT_RAWSTRUCT8D:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT13_I76:%.*]] = getelementptr inbounds nuw [1 x %struct.RawStruct8D], ptr [[OUTPUTSIZESLOCAL_I]], i32 0, i32 0
+; CHECK-NEXT:    ret void
+;
+entry:
+  %outputSizesLocal.i = alloca %struct.RawStruct8D, align 4
+  %arrayinit.element13.i76 = getelementptr inbounds nuw [1 x %struct.RawStruct8D], ptr %outputSizesLocal.i, i32 0, i32 0
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index 2b29fd30a7a5..36fed88fc52d 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -33,13 +33,13 @@
 ; CHECK-NEXT:   DXIL Translate Metadata
 ; CHECK-NEXT:   DXIL Post Optimization Validation
 ; CHECK-NEXT:   DXIL Op Lowering
+; CHECK-NEXT:   DXIL Root Signature Analysis
 ; CHECK-NEXT:   DXIL Prepare Module
 
 ; CHECK-ASM-NEXT: DXIL Metadata Pretty Printer
 ; CHECK-ASM-NEXT: Print Module IR
 
 ; CHECK-OBJ-NEXT: DXIL Embedder
-; CHECK-OBJ-NEXT: DXIL Root Signature Analysis
 ; CHECK-OBJ-NEXT: DXContainer Global Emitter
 ; CHECK-OBJ-NEXT: FunctionPass Manager
 ; CHECK-OBJ-NEXT:   Lazy Machine Block Frequency Analysis
diff --git a/llvm/test/CodeGen/DirectX/scalarize-alloca.ll b/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
index b589136d6965..32e2c3ca2c30 100644
--- a/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
+++ b/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
@@ -2,7 +2,7 @@
 ; RUN: opt -S -passes='dxil-data-scalarization,dxil-flatten-arrays' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=FCHECK,CHECK
 
 ; CHECK-LABEL: alloca_2d__vec_test
-define void @alloca_2d__vec_test() local_unnamed_addr #2 {
+define void @alloca_2d__vec_test() {
   ; SCHECK:  alloca [2 x [4 x i32]], align 16
   ; FCHECK:  alloca [8 x i32], align 16
   ; CHECK: ret void
@@ -10,6 +10,23 @@ define void @alloca_2d__vec_test() local_unnamed_addr #2 {
   ret void
 }
 
+; CHECK-LABEL: alloca_4d__vec_test
+define void @alloca_4d__vec_test() {
+  ; SCHECK:  alloca [2 x [2 x [2 x [2 x i32]]]], align 16
+  ; FCHECK:  alloca [16 x i32], align 16
+  ; CHECK: ret void
+  %1 = alloca [2 x [2 x [2 x <2 x i32>]]], align 16
+  ret void
+}
+
+; CHECK-LABEL: alloca_vec_test
+define void @alloca_vec_test() {
+  ; CHECK:  alloca [4 x i32], align 16
+  ; CHECK: ret void
+  %1 = alloca <4 x i32>, align 16
+  ret void
+}
+
 ; CHECK-LABEL: alloca_2d_gep_test
 define void @alloca_2d_gep_test() {
   ; SCHECK:  [[alloca_val:%.*]] = alloca [2 x [2 x i32]], align 16
diff --git a/llvm/test/CodeGen/DirectX/strip-rootsignatures.ll b/llvm/test/CodeGen/DirectX/strip-rootsignatures.ll
new file mode 100644
index 000000000000..3ac617ae871f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/strip-rootsignatures.ll
@@ -0,0 +1,18 @@
+; RUN: opt -S -dxil-prepare < %s | FileCheck %s
+
+; Ensures that dxil-prepare will remove the dx.rootsignatures metadata
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+define void @main() {
+entry:
+  ret void
+}
+
+; CHECK-NOT: !dx.rootsignatures
+; CHECK-NOT: {{^!}}
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !4 } ; list of root signature elements
+!4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir b/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
index 663c13af5e9d..f96d83fede40 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
@@ -1,5 +1,5 @@
 # RUN: llc -run-pass=mir-debugify -o - %s | FileCheck --check-prefixes=ALL,VALUE %s
-# RUN: llc -run-pass=mir-debugify -debugify-level=locations -o - %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg.value %s
+# RUN: llc -run-pass=mir-debugify -debugify-level=locations -o - %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg_value %s
 # RUN: llc -run-pass=mir-debugify,mir-strip-debug,mir-debugify -o - %s | FileCheck --check-prefixes=ALL,VALUE %s
 # RUN: llc -run-pass=mir-debugify,mir-strip-debug -o - %s | FileCheck --check-prefix=STRIP %s
 
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll b/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
index d56520da78ae..965f3c684963 100644
--- a/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
+++ b/llvm/test/CodeGen/Hexagon/autohvx/build-vector-float-type.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon < %s | FileCheck %s
 
 ; Check that this code does compile.
@@ -6,9 +7,248 @@ target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i
 target triple = "hexagon"
 
 ; Function Attrs: nounwind
-; CHECK-LABEL: f0:
-; CHECK: vinsert
 define <32 x float> @f0(ptr %a0, ptr %a1) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vxor(v0,v0)
+; CHECK-NEXT:     r2 = memw(r0+#0)
+; CHECK-NEXT:     r6 = memw(r0+#8)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = v0
+; CHECK-NEXT:     r3 = memw(r0+#64)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r4 = memw(r0+#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#68)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r7 = memw(r1+r4<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#72)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#12)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#76)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#80)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#20)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#84)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#24)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#88)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#28)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#92)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#32)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#96)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#36)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#100)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#40)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#104)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#44)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#108)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#48)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#112)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#52)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r2 = memw(r0+#116)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#56)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r1+r2<<#2)
+; CHECK-NEXT:     r3 = memw(r0+#120)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r1+r6<<#2)
+; CHECK-NEXT:     r6 = memw(r0+#60)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r2 = #68
+; CHECK-NEXT:     r3 = memw(r1+r3<<#2)
+; CHECK-NEXT:     r5 = memw(r0+#124)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r4 = memw(r1+r6<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r0 = memw(r1+r5<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vror(v1,r2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vor(v1,v0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
 b0:
   %v1 = load i32, ptr %a0, align 4
   %v2 = getelementptr float, ptr %a1, i32 %v1
@@ -173,9 +413,380 @@ b0:
 }
 
 ; Function Attrs: nounwind
-; CHECK-LABEL: f1:
-; CHECK: vinsert
 define <64 x half> @f1(ptr %a0, ptr %a1) #0 {
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vxor(v0,v0)
+; CHECK-NEXT:     r6 = memw(r0+#0)
+; CHECK-NEXT:     r2 = memw(r0+#16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = v0
+; CHECK-NEXT:     r8 = memw(r0+#4)
+; CHECK-NEXT:     r4 = memw(r0+#24)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = memw(r0+#8)
+; CHECK-NEXT:     r3 = memw(r0+#20)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memw(r0+#28)
+; CHECK-NEXT:     r9 = memw(r0+#12)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r8 = memuh(r1+r8<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r8 = combine(r8.l,r6.l)
+; CHECK-NEXT:     r14 = memw(r0+#128)
+; CHECK-NEXT:     r6 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r8)
+; CHECK-NEXT:     r5 = memw(r0+#136)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r2.l)
+; CHECK-NEXT:     r2 = combine(r6.l,r4.l)
+; CHECK-NEXT:     r12 = memw(r0+#132)
+; CHECK-NEXT:     r6 = memw(r0+#140)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r9 = memuh(r1+r9<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r9.l,r7.l)
+; CHECK-NEXT:     r9 = memuh(r1+r14<<#1)
+; CHECK-NEXT:     r12 = memuh(r1+r12<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r7)
+; CHECK-NEXT:     r15 = combine(r12.l,r9.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r15)
+; CHECK-NEXT:     r5 = combine(r6.l,r5.l)
+; CHECK-NEXT:     r6 = memw(r0+#144)
+; CHECK-NEXT:     r4 = memw(r0+#32)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#148)
+; CHECK-NEXT:     r28 = memw(r0+#36)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r10 = memw(r0+#40)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r5)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = combine(r7.l,r6.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#152)
+; CHECK-NEXT:     r9 = memuh(r1+r28<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#156)
+; CHECK-NEXT:     r5 = memw(r0+#44)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r6)
+; CHECK-NEXT:     r4 = combine(r9.l,r4.l)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r11 = memw(r0+#48)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#52)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     r3 = combine(r3.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memw(r0+#160)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r2 = memw(r0+#164)
+; CHECK-NEXT:     r13 = memw(r0+#168)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r8 = memuh(r1+r10<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r7.l)
+; CHECK-NEXT:     r5 = combine(r5.l,r8.l)
+; CHECK-NEXT:     r7 = memw(r0+#172)
+; CHECK-NEXT:     r9 = memuh(r1+r11<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     v1.w = vinsert(r5)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r14 = memw(r0+#176)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r6 = combine(r6.l,r9.l)
+; CHECK-NEXT:     r9 = memuh(r1+r13<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r7.l,r9.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r12 = memw(r0+#56)
+; CHECK-NEXT:     r15 = memw(r0+#180)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r7)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memw(r0+#60)
+; CHECK-NEXT:     r4 = memw(r0+#64)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r6)
+; CHECK-NEXT:     r7 = memuh(r1+r14<<#1)
+; CHECK-NEXT:     r28 = memuh(r1+r15<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = combine(r28.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memw(r0+#68)
+; CHECK-NEXT:     r6 = memw(r0+#184)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r7)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r8 = memuh(r1+r12<<#1)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r8.l)
+; CHECK-NEXT:     r7 = memw(r0+#188)
+; CHECK-NEXT:     r5 = memw(r0+#72)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r4.l)
+; CHECK-NEXT:     r4 = memw(r0+#76)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r6 = combine(r7.l,r6.l)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r7 = memw(r0+#88)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r6)
+; CHECK-NEXT:     r4 = combine(r4.l,r5.l)
+; CHECK-NEXT:     r6 = memw(r0+#192)
+; CHECK-NEXT:     r10 = memw(r0+#80)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r5 = memw(r0+#92)
+; CHECK-NEXT:     r2 = memw(r0+#196)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#84)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r4 = combine(r5.l,r7.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r11 = memw(r0+#200)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r6.l)
+; CHECK-NEXT:     r5 = memw(r0+#204)
+; CHECK-NEXT:     r8 = memuh(r1+r10<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r11<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r8.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#96)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     r5 = combine(r5.l,r7.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memw(r0+#208)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r5)
+; CHECK-NEXT:     r7 = memw(r0+#212)
+; CHECK-NEXT:     r2 = memw(r0+#100)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r5 = memw(r0+#104)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r4 = memw(r0+#108)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r7.l,r3.l)
+; CHECK-NEXT:     r7 = memw(r0+#216)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:     r3 = memw(r0+#220)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r2.l,r6.l)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r4 = combine(r4.l,r5.l)
+; CHECK-NEXT:     r2 = memw(r0+#224)
+; CHECK-NEXT:     r6 = memuh(r1+r7<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:     r5 = memw(r0+#228)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r3.l,r6.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r6 = memw(r0+#112)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     v1.w = vinsert(r4)
+; CHECK-NEXT:     r3 = memw(r0+#116)
+; CHECK-NEXT:     r4 = memw(r0+#232)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:     r7 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = combine(r5.l,r2.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r5 = memw(r0+#236)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r2)
+; CHECK-NEXT:     r2 = combine(r3.l,r7.l)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r3 = memw(r0+#240)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r2)
+; CHECK-NEXT:     r4 = memuh(r1+r4<<#1)
+; CHECK-NEXT:     r5 = memuh(r1+r5<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r4 = combine(r5.l,r4.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r2 = memw(r0+#120)
+; CHECK-NEXT:     r6 = memw(r0+#244)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r4)
+; CHECK-NEXT:     v1 = valign(v1,v1,#4)
+; CHECK-NEXT:     r7 = memw(r0+#124)
+; CHECK-NEXT:     r3 = memuh(r1+r3<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = memuh(r1+r6<<#1)
+; CHECK-NEXT:     r6 = memw(r0+#248)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r5.l,r3.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:     r0 = memw(r0+#252)
+; CHECK-NEXT:     r2 = memuh(r1+r2<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r3)
+; CHECK-NEXT:     r7 = memuh(r1+r7<<#1)
+; CHECK-NEXT:     r6 = memuh(r1+r6<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3 = combine(r7.l,r2.l)
+; CHECK-NEXT:     r2 = #68
+; CHECK-NEXT:     r0 = memuh(r1+r0<<#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1.w = vinsert(r3)
+; CHECK-NEXT:     r0 = combine(r0.l,r6.l)
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vinsert(r0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vror(v1,r2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = valign(v0,v0,#4)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vor(v1,v0)
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
 b0:
   %v1 = load i32, ptr %a0, align 4
   %v2 = getelementptr half, ptr %a1, i32 %v1
diff --git a/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll b/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll
new file mode 100644
index 000000000000..b23366bc11ac
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/hexagon-strcpy.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -march=hexagon -verify-machineinstrs  < %s | FileCheck %s
+
+@.str = private unnamed_addr constant [31 x i8] c"DHRYSTONE PROGRAM, 3'RD STRING\00", align 1
+@.str1 = private unnamed_addr constant [3 x i8] c"%s\00", align 1
+
+; Function Attrs: nounwind
+declare i32 @printf(i8* nocapture readonly, ...)
+
+; Function Attrs: nounwind
+define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    .cfi_def_cfa r30, 8
+; CHECK-NEXT:    .cfi_offset r31, -4
+; CHECK-NEXT:    .cfi_offset r30, -8
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = ##.L.str1
+; CHECK-NEXT:     r3:2 = CONST64(#2325073635944967245)
+; CHECK-NEXT:     allocframe(r29,#40):raw
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r1 = add(r29,#8)
+; CHECK-NEXT:     r7:6 = CONST64(#4706902966564560965)
+; CHECK-NEXT:     r5:4 = CONST64(#5642821575076104260)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memb(r29+#38) = #0
+; CHECK-NEXT:     memw(r29+#0) = r1
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r29+#24) = r3:2
+; CHECK-NEXT:     memd(r29+#16) = r7:6
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memd(r29+#8) = r5:4
+; CHECK-NEXT:     memh(r29+#36) = ##18254
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     call printf
+; CHECK-NEXT:     memw(r29+#32) = ##1230132307
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = #0
+; CHECK-NEXT:     dealloc_return
+; CHECK-NEXT:    }
+entry:
+  %blah = alloca [30 x i8], align 8
+  %arraydecay = getelementptr inbounds [30 x i8], [30 x i8]* %blah, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %arraydecay, i8* getelementptr inbounds ([31 x i8], [31 x i8]* @.str, i32 0, i32 0), i32 31, i32 1, i1 false)
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str1, i32 0, i32 0), i8* %arraydecay)
+  ret i32 0
+}
+
+; Function Attrs: nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture readonly, i32, i32, i1)
diff --git a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
index 568d596266b0..f9092a83a628 100644
--- a/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/vect-vshifts.ll
@@ -1,12 +1,203 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=hexagon -mcpu=hexagonv5 < %s | FileCheck %s
 
-; Check that store is post-incremented.
-; CHECK: r{{[0-9]+:[0-9]+}} = vasrw(r{{[0-9]+:[0-9]+}},r{{[0-9]+}})
-; CHECK: r{{[0-9]+:[0-9]+}} = vaslw(r{{[0-9]+:[0-9]+}},r{{[0-9]+}})
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
 target triple = "hexagon"
 
+; Check that store is post-incremented.
 define void @foo(ptr nocapture %buf, ptr nocapture %dest, i32 %offset, i32 %oddBlock, i32 %gb) #0 {
+; CHECK-LABEL: foo:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = sub(r2,r3)
+; CHECK-NEXT:     r3 = sub(#31,r4)
+; CHECK-NEXT:     p0 = cmp.eq(r3,#0)
+; CHECK-NEXT:     r5 = memw(r0+#0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = asr(r5,r4)
+; CHECK-NEXT:     r29 = add(r29,#-24)
+; CHECK-NEXT:     r9:8 = combine(#-1,#-1)
+; CHECK-NEXT:     memw(r0+#0) = r5.new
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r5 = and(r2,#7)
+; CHECK-NEXT:     r2 = r3
+; CHECK-NEXT:     memd(r29+#8) = r19:18
+; CHECK-NEXT:     memd(r29+#16) = r17:16
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = vaslw(r3:2,#1)
+; CHECK-NEXT:     if (p0) r5 = add(r5,#1)
+; CHECK-NEXT:     memd(r29+#0) = r21:20
+; CHECK-NEXT:    } // 8-byte Folded Spill
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = addasl(r1,r5,#2)
+; CHECK-NEXT:     r7:6 = vaddw(r7:6,r9:8)
+; CHECK-NEXT:     r14 = memw(r1+r5<<#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15 = memw(r0+#256)
+; CHECK-NEXT:     r10 = memw(r0+#512)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vasrw(r15:14,#31)
+; CHECK-NEXT:     r19:18 = vasrw(r15:14,r3)
+; CHECK-NEXT:     r12 = memw(r0+#1024)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r9:8,r19:18)
+; CHECK-NEXT:     r17:16 = xor(r9:8,r7:6)
+; CHECK-NEXT:     r11 = memw(r0+#768)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r15:14 = vmux(p0,r15:14,r17:16)
+; CHECK-NEXT:     r19:18 = vasrw(r11:10,#31)
+; CHECK-NEXT:     r13 = memw(r0+#1280)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r11:10,r3)
+; CHECK-NEXT:     r15:14 = vaslw(r15:14,r4)
+; CHECK-NEXT:     r8 = memw(r0+#1536)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p1 = vcmpw.eq(r19:18,r21:20)
+; CHECK-NEXT:     r19:18 = xor(r19:18,r7:6)
+; CHECK-NEXT:     r9 = memw(r0+#1792)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r13:12,#31)
+; CHECK-NEXT:     r11:10 = vmux(p1,r11:10,r19:18)
+; CHECK-NEXT:     memw(r0+#32) = r14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = vasrw(r13:12,r3)
+; CHECK-NEXT:     r11:10 = vaslw(r11:10,r4)
+; CHECK-NEXT:     r14 = memw(r0+#2048)
+; CHECK-NEXT:     memw(r1+r5<<#2) = r14
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r17:16)
+; CHECK-NEXT:     r19:18 = xor(r21:20,r7:6)
+; CHECK-NEXT:     memw(r0+#288) = r15
+; CHECK-NEXT:     memw(r0+#256) = r15
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = vasrw(r9:8,#31)
+; CHECK-NEXT:     r19:18 = vmux(p0,r13:12,r19:18)
+; CHECK-NEXT:     r15 = memw(r0+#2304)
+; CHECK-NEXT:     memw(r0+#544) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r9:8,r3)
+; CHECK-NEXT:     memw(r0+#800) = r11
+; CHECK-NEXT:     memw(r0+#512) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r13:12 = vasrw(r15:14,#31)
+; CHECK-NEXT:     r11:10 = vasrw(r15:14,r3)
+; CHECK-NEXT:     memw(r0+#768) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r17:16,r21:20)
+; CHECK-NEXT:     p1 = vcmpw.eq(r13:12,r11:10)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = xor(r13:12,r7:6)
+; CHECK-NEXT:     r17:16 = xor(r17:16,r7:6)
+; CHECK-NEXT:     r12 = memw(r0+#2560)
+; CHECK-NEXT:     r13 = memw(r0+#2816)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslw(r19:18,r4)
+; CHECK-NEXT:     r17:16 = vmux(p0,r9:8,r17:16)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r9:8 = vmux(p1,r15:14,r21:20)
+; CHECK-NEXT:     r19:18 = vasrw(r13:12,#31)
+; CHECK-NEXT:     r14 = memw(r0+#3072)
+; CHECK-NEXT:     memw(r0+#1056) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r13:12,r3)
+; CHECK-NEXT:     r9:8 = vaslw(r9:8,r4)
+; CHECK-NEXT:     memw(r0+#1312) = r11
+; CHECK-NEXT:     memw(r0+#1024) = r10
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vaslw(r17:16,r4)
+; CHECK-NEXT:     p0 = vcmpw.eq(r19:18,r21:20)
+; CHECK-NEXT:     r15 = memw(r0+#3328)
+; CHECK-NEXT:     memw(r0+#1280) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r17:16 = xor(r19:18,r7:6)
+; CHECK-NEXT:     r21:20 = vasrw(r15:14,#31)
+; CHECK-NEXT:     memw(r0+#1568) = r10
+; CHECK-NEXT:     memw(r0+#1824) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r11:10 = vasrw(r15:14,r3)
+; CHECK-NEXT:     r13:12 = vmux(p0,r13:12,r17:16)
+; CHECK-NEXT:     memw(r0+#1536) = r10
+; CHECK-NEXT:     memw(r0+#1792) = r11
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r11:10)
+; CHECK-NEXT:     r19:18 = xor(r21:20,r7:6)
+; CHECK-NEXT:     r10 = memw(r0+#3584)
+; CHECK-NEXT:     memw(r0+#2080) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r19:18 = vmux(p0,r15:14,r19:18)
+; CHECK-NEXT:     r13:12 = vaslw(r13:12,r4)
+; CHECK-NEXT:     r11 = memw(r0+#3840)
+; CHECK-NEXT:     memw(r0+#2336) = r9
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r21:20 = vasrw(r11:10,#31)
+; CHECK-NEXT:     r3:2 = vasrw(r11:10,r3)
+; CHECK-NEXT:     memw(r0+#2048) = r8
+; CHECK-NEXT:     memw(r0+#2304) = r9
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7:6 = xor(r21:20,r7:6)
+; CHECK-NEXT:     p0 = vcmpw.eq(r21:20,r3:2)
+; CHECK-NEXT:     r17:16 = memd(r29+#16)
+; CHECK-NEXT:     memw(r0+#2592) = r12
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3:2 = vmux(p0,r11:10,r7:6)
+; CHECK-NEXT:     r9:8 = vaslw(r19:18,r4)
+; CHECK-NEXT:     r19:18 = memd(r29+#8)
+; CHECK-NEXT:     memw(r0+#2848) = r13
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r3:2 = vaslw(r3:2,r4)
+; CHECK-NEXT:     r29 = add(r29,#24)
+; CHECK-NEXT:     r21:20 = memd(r29+#0)
+; CHECK-NEXT:     memw(r0+#2560) = r12
+; CHECK-NEXT:    } // 8-byte Folded Reload
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#2816) = r13
+; CHECK-NEXT:     memw(r0+#3104) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3360) = r9
+; CHECK-NEXT:     memw(r0+#3072) = r8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3328) = r9
+; CHECK-NEXT:     memw(r0+#3616) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     memw(r0+#3872) = r3
+; CHECK-NEXT:     memw(r0+#3584) = r2
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:     memw(r0+#3840) = r3
+; CHECK-NEXT:    }
 entry:
   %0 = load i32, ptr %buf, align 4, !tbaa !0
   %shr = ashr i32 %0, %gb
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-ilp32d.ll b/llvm/test/CodeGen/LoongArch/calling-conv-ilp32d.ll
new file mode 100644
index 000000000000..62c2cc999456
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-ilp32d.ll
@@ -0,0 +1,193 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch32 --mattr=+d --target-abi=ilp32d < %s \
+; RUN:   | FileCheck %s
+
+;; This file contains specific tests for the ilp32d ABI.
+
+;; Check pass floating-point arguments whith FPRs.
+
+define i32 @callee_float_in_fpr(i32 %a, float %b, double %c) nounwind {
+; CHECK-LABEL: callee_float_in_fpr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ftintrz.w.s $fa0, $fa0
+; CHECK-NEXT:    movfr2gr.s $a1, $fa0
+; CHECK-NEXT:    ftintrz.w.d $fa0, $fa1
+; CHECK-NEXT:    movfr2gr.s $a2, $fa0
+; CHECK-NEXT:    add.w $a0, $a0, $a1
+; CHECK-NEXT:    add.w $a0, $a0, $a2
+; CHECK-NEXT:    ret
+  %b_fptosi = fptosi float %b to i32
+  %c_fptosi = fptosi double %c to i32
+  %1 = add i32 %a, %b_fptosi
+  %2 = add i32 %1, %c_fptosi
+  ret i32 %2
+}
+
+define i32 @caller_float_in_fpr() nounwind {
+; CHECK-LABEL: caller_float_in_fpr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.w $sp, $sp, -16
+; CHECK-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; CHECK-NEXT:    movgr2fr.w $fa1, $zero
+; CHECK-NEXT:    movgr2frh.w $fa1, $zero
+; CHECK-NEXT:    movgr2fr.w $fa0, $zero
+; CHECK-NEXT:    ori $a0, $zero, 1
+; CHECK-NEXT:    bl callee_float_in_fpr
+; CHECK-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; CHECK-NEXT:    addi.w $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = call i32 @callee_float_in_fpr(i32 1, float 0.0, double 0.0)
+  ret i32 %1
+}
+
+;; Check that the GPR is used once the FPRs are exhausted.
+
+;; Must keep define on a single line due to an update_llc_test_checks.py limitation.
+define i32 @callee_double_in_gpr_exhausted_fprs(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h, double %i) nounwind {
+; CHECK-LABEL: callee_double_in_gpr_exhausted_fprs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    movgr2frh.w $fa0, $a1
+; CHECK-NEXT:    ftintrz.w.d $fa1, $fa7
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    ftintrz.w.d $fa0, $fa0
+; CHECK-NEXT:    movfr2gr.s $a1, $fa0
+; CHECK-NEXT:    add.w $a0, $a0, $a1
+; CHECK-NEXT:    ret
+  %h_fptosi = fptosi double %h to i32
+  %i_fptosi = fptosi double %i to i32
+  %1 = add i32 %h_fptosi, %i_fptosi
+  ret i32 %1
+}
+
+define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
+; CHECK-LABEL: caller_double_in_gpr_exhausted_fprs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.w $sp, $sp, -16
+; CHECK-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    fld.d $fa1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
+; CHECK-NEXT:    fld.d $fa2, $a0, %pc_lo12(.LCPI3_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_2)
+; CHECK-NEXT:    fld.d $fa3, $a0, %pc_lo12(.LCPI3_2)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_3)
+; CHECK-NEXT:    fld.d $fa4, $a0, %pc_lo12(.LCPI3_3)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_4)
+; CHECK-NEXT:    fld.d $fa5, $a0, %pc_lo12(.LCPI3_4)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_5)
+; CHECK-NEXT:    fld.d $fa6, $a0, %pc_lo12(.LCPI3_5)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_6)
+; CHECK-NEXT:    fld.d $fa7, $a0, %pc_lo12(.LCPI3_6)
+; CHECK-NEXT:    addi.w $a0, $zero, 1
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    ffint.s.w $fa0, $fa0
+; CHECK-NEXT:    fcvt.d.s $fa0, $fa0
+; CHECK-NEXT:    lu12i.w $a1, 262688
+; CHECK-NEXT:    move $a0, $zero
+; CHECK-NEXT:    bl callee_double_in_gpr_exhausted_fprs
+; CHECK-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; CHECK-NEXT:    addi.w $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = call i32 @callee_double_in_gpr_exhausted_fprs(
+      double 1.0, double 2.0, double 3.0, double 4.0, double 5.0, double 6.0,
+      double 7.0, double 8.0, double 9.0)
+  ret i32 %1
+}
+
+;; Check that the stack is used once the FPRs and GPRs are both exhausted.
+
+;; Must keep define on a single line due to an update_llc_test_checks.py limitation.
+define i32 @callee_double_on_stack_exhausted_fprs_gprs(double %a, double %b, double %c, double %d, double %e, double %f, double %g, double %h, double %i, double %j, double %k, double %l, double %m, double %n) nounwind {
+; CHECK-LABEL: callee_double_on_stack_exhausted_fprs_gprs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fld.d $fa0, $sp, 0
+; CHECK-NEXT:    fld.d $fa1, $sp, 8
+; CHECK-NEXT:    ftintrz.w.d $fa0, $fa0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    ftintrz.w.d $fa0, $fa1
+; CHECK-NEXT:    movfr2gr.s $a1, $fa0
+; CHECK-NEXT:    add.w $a0, $a0, $a1
+; CHECK-NEXT:    ret
+  %m_fptosi = fptosi double %m to i32
+  %n_fptosi = fptosi double %n to i32
+  %1 = add i32 %m_fptosi, %n_fptosi
+  ret i32 %1
+}
+
+define i32 @caller_double_on_stack_exhausted_fprs_gprs() nounwind {
+; CHECK-LABEL: caller_double_on_stack_exhausted_fprs_gprs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.w $sp, $sp, -32
+; CHECK-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; CHECK-NEXT:    lu12i.w $a0, 262816
+; CHECK-NEXT:    st.w $a0, $sp, 4
+; CHECK-NEXT:    st.w $zero, $sp, 0
+; CHECK-NEXT:    lu12i.w $a0, 262848
+; CHECK-NEXT:    st.w $a0, $sp, 12
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; CHECK-NEXT:    fld.d $fa1, $a0, %pc_lo12(.LCPI5_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
+; CHECK-NEXT:    fld.d $fa2, $a0, %pc_lo12(.LCPI5_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_2)
+; CHECK-NEXT:    fld.d $fa3, $a0, %pc_lo12(.LCPI5_2)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_3)
+; CHECK-NEXT:    fld.d $fa4, $a0, %pc_lo12(.LCPI5_3)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_4)
+; CHECK-NEXT:    fld.d $fa5, $a0, %pc_lo12(.LCPI5_4)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_5)
+; CHECK-NEXT:    fld.d $fa6, $a0, %pc_lo12(.LCPI5_5)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_6)
+; CHECK-NEXT:    fld.d $fa7, $a0, %pc_lo12(.LCPI5_6)
+; CHECK-NEXT:    addi.w $a0, $zero, 1
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    ffint.s.w $fa0, $fa0
+; CHECK-NEXT:    fcvt.d.s $fa0, $fa0
+; CHECK-NEXT:    lu12i.w $a1, 262688
+; CHECK-NEXT:    lu12i.w $a3, 262720
+; CHECK-NEXT:    lu12i.w $a5, 262752
+; CHECK-NEXT:    lu12i.w $a7, 262784
+; CHECK-NEXT:    st.w $zero, $sp, 8
+; CHECK-NEXT:    move $a0, $zero
+; CHECK-NEXT:    move $a2, $zero
+; CHECK-NEXT:    move $a4, $zero
+; CHECK-NEXT:    move $a6, $zero
+; CHECK-NEXT:    bl callee_double_on_stack_exhausted_fprs_gprs
+; CHECK-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; CHECK-NEXT:    addi.w $sp, $sp, 32
+; CHECK-NEXT:    ret
+  %1 = call i32 @callee_double_on_stack_exhausted_fprs_gprs(
+      double 1.0, double 2.0, double 3.0, double 4.0, double 5.0, double 6.0,
+      double 7.0, double 8.0, double 9.0, double 10.0, double 11.0, double 12.0,
+      double 13.0, double 14.0)
+  ret i32 %1
+}
+
+;; Check returning doubles.
+
+define double @callee_double_ret() nounwind {
+; CHECK-LABEL: callee_double_ret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.w $a0, $zero, 1
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    ffint.s.w $fa0, $fa0
+; CHECK-NEXT:    fcvt.d.s $fa0, $fa0
+; CHECK-NEXT:    ret
+  ret double 1.0
+}
+
+define i64 @caller_double_ret() nounwind {
+; CHECK-LABEL: caller_double_ret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi.w $sp, $sp, -16
+; CHECK-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; CHECK-NEXT:    bl callee_double_ret
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    movfrh2gr.s $a1, $fa0
+; CHECK-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; CHECK-NEXT:    addi.w $sp, $sp, 16
+; CHECK-NEXT:    ret
+  %1 = call double @callee_double_ret()
+  %2 = bitcast double %1 to i64
+  ret i64 %2
+}
diff --git a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-f.ll b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-f.ll
index be9ea29b54c3..c1d75ddd3280 100644
--- a/llvm/test/CodeGen/LoongArch/inline-asm-constraint-f.ll
+++ b/llvm/test/CodeGen/LoongArch/inline-asm-constraint-f.ll
@@ -32,18 +32,14 @@ define double @constraint_f_double(double %a) nounwind {
 define double @constraint_gpr(double %a) {
 ; LA32-LABEL: constraint_gpr:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $sp, $sp, -16
-; LA32-NEXT:    .cfi_def_cfa_offset 16
-; LA32-NEXT:    fst.d $fa0, $sp, 8
-; LA32-NEXT:    ld.w $a7, $sp, 8
-; LA32-NEXT:    ld.w $t0, $sp, 12
+; LA32-NEXT:    .cfi_def_cfa_offset 0
+; LA32-NEXT:    movfr2gr.s $a7, $fa0
+; LA32-NEXT:    movfrh2gr.s $t0, $fa0
 ; LA32-NEXT:    #APP
 ; LA32-NEXT:    move $a6, $a7
 ; LA32-NEXT:    #NO_APP
-; LA32-NEXT:    st.w $a7, $sp, 4
-; LA32-NEXT:    st.w $a6, $sp, 0
-; LA32-NEXT:    fld.d $fa0, $sp, 0
-; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    movgr2fr.w $fa0, $a6
+; LA32-NEXT:    movgr2frh.w $fa0, $a7
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: constraint_gpr:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll
index 2a51fd97feb6..0b82ea220d7f 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/double-convert.ll
@@ -279,11 +279,8 @@ define double @convert_u64_to_double(i64 %a) nounwind {
 define double @bitcast_i64_to_double(i64 %a, i64 %b) nounwind {
 ; LA32-LABEL: bitcast_i64_to_double:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $sp, $sp, -16
-; LA32-NEXT:    st.w $a1, $sp, 12
-; LA32-NEXT:    st.w $a0, $sp, 8
-; LA32-NEXT:    fld.d $fa0, $sp, 8
-; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    movgr2frh.w $fa0, $a1
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: bitcast_i64_to_double:
@@ -297,11 +294,8 @@ define double @bitcast_i64_to_double(i64 %a, i64 %b) nounwind {
 define i64 @bitcast_double_to_i64(double %a) nounwind {
 ; LA32-LABEL: bitcast_double_to_i64:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    addi.w $sp, $sp, -16
-; LA32-NEXT:    fst.d $fa0, $sp, 8
-; LA32-NEXT:    ld.w $a0, $sp, 8
-; LA32-NEXT:    ld.w $a1, $sp, 12
-; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    movfr2gr.s $a0, $fa0
+; LA32-NEXT:    movfrh2gr.s $a1, $fa0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: bitcast_double_to_i64:
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
index 78cabd37c0ad..b6507e87f088 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
@@ -115,9 +115,8 @@ define double @load_acquire_double(ptr %ptr) {
 ; LA32-NEXT:    .cfi_offset 1, -4
 ; LA32-NEXT:    ori $a1, $zero, 2
 ; LA32-NEXT:    bl __atomic_load_8
-; LA32-NEXT:    st.w $a1, $sp, 4
-; LA32-NEXT:    st.w $a0, $sp, 0
-; LA32-NEXT:    fld.d $fa0, $sp, 0
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    movgr2frh.w $fa0, $a1
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -234,9 +233,8 @@ define double @load_unordered_double(ptr %ptr) {
 ; LA32-NEXT:    .cfi_offset 1, -4
 ; LA32-NEXT:    move $a1, $zero
 ; LA32-NEXT:    bl __atomic_load_8
-; LA32-NEXT:    st.w $a1, $sp, 4
-; LA32-NEXT:    st.w $a0, $sp, 0
-; LA32-NEXT:    fld.d $fa0, $sp, 0
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    movgr2frh.w $fa0, $a1
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -352,9 +350,8 @@ define double @load_monotonic_double(ptr %ptr) {
 ; LA32-NEXT:    .cfi_offset 1, -4
 ; LA32-NEXT:    move $a1, $zero
 ; LA32-NEXT:    bl __atomic_load_8
-; LA32-NEXT:    st.w $a1, $sp, 4
-; LA32-NEXT:    st.w $a0, $sp, 0
-; LA32-NEXT:    fld.d $fa0, $sp, 0
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    movgr2frh.w $fa0, $a1
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -481,9 +478,8 @@ define double @load_seq_cst_double(ptr %ptr) {
 ; LA32-NEXT:    .cfi_offset 1, -4
 ; LA32-NEXT:    ori $a1, $zero, 5
 ; LA32-NEXT:    bl __atomic_load_8
-; LA32-NEXT:    st.w $a1, $sp, 4
-; LA32-NEXT:    st.w $a0, $sp, 0
-; LA32-NEXT:    fld.d $fa0, $sp, 0
+; LA32-NEXT:    movgr2fr.w $fa0, $a0
+; LA32-NEXT:    movgr2frh.w $fa0, $a1
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
@@ -605,9 +601,8 @@ define void @store_release_double(ptr %ptr, double %v) {
 ; LA32-NEXT:    .cfi_def_cfa_offset 16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    .cfi_offset 1, -4
-; LA32-NEXT:    fst.d $fa0, $sp, 0
-; LA32-NEXT:    ld.w $a1, $sp, 0
-; LA32-NEXT:    ld.w $a2, $sp, 4
+; LA32-NEXT:    movfr2gr.s $a1, $fa0
+; LA32-NEXT:    movfrh2gr.s $a2, $fa0
 ; LA32-NEXT:    ori $a3, $zero, 3
 ; LA32-NEXT:    bl __atomic_store_8
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -723,9 +718,8 @@ define void @store_unordered_double(ptr %ptr, double %v) {
 ; LA32-NEXT:    .cfi_def_cfa_offset 16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    .cfi_offset 1, -4
-; LA32-NEXT:    fst.d $fa0, $sp, 0
-; LA32-NEXT:    ld.w $a1, $sp, 0
-; LA32-NEXT:    ld.w $a2, $sp, 4
+; LA32-NEXT:    movfr2gr.s $a1, $fa0
+; LA32-NEXT:    movfrh2gr.s $a2, $fa0
 ; LA32-NEXT:    move $a3, $zero
 ; LA32-NEXT:    bl __atomic_store_8
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -841,9 +835,8 @@ define void @store_monotonic_double(ptr %ptr, double %v) {
 ; LA32-NEXT:    .cfi_def_cfa_offset 16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    .cfi_offset 1, -4
-; LA32-NEXT:    fst.d $fa0, $sp, 0
-; LA32-NEXT:    ld.w $a1, $sp, 0
-; LA32-NEXT:    ld.w $a2, $sp, 4
+; LA32-NEXT:    movfr2gr.s $a1, $fa0
+; LA32-NEXT:    movfrh2gr.s $a2, $fa0
 ; LA32-NEXT:    move $a3, $zero
 ; LA32-NEXT:    bl __atomic_store_8
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
@@ -973,9 +966,8 @@ define void @store_seq_cst_double(ptr %ptr, double %v) {
 ; LA32-NEXT:    .cfi_def_cfa_offset 16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    .cfi_offset 1, -4
-; LA32-NEXT:    fst.d $fa0, $sp, 0
-; LA32-NEXT:    ld.w $a1, $sp, 0
-; LA32-NEXT:    ld.w $a2, $sp, 4
+; LA32-NEXT:    movfr2gr.s $a1, $fa0
+; LA32-NEXT:    movfrh2gr.s $a2, $fa0
 ; LA32-NEXT:    ori $a3, $zero, 5
 ; LA32-NEXT:    bl __atomic_store_8
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll
new file mode 100644
index 000000000000..86808c7a8f01
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/bitcast-extract-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define i32 @bitcast_extract_v8f32(<8 x float> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    movgr2fr.w $fa0, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x float> %a, i32 7
+  %c = bitcast float %b to i32
+  ret i32 %c
+}
+
+define i64 @bitcast_extract_v4f64(<4 x double> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa0, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x double> %a, i32 3
+  %c = bitcast double %b to i64
+  ret i64 %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fix-xvshuf.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fix-xvshuf.ll
new file mode 100644
index 000000000000..f3bec11810e9
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fix-xvshuf.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+;; Fix https://github.com/llvm/llvm-project/issues/137000.
+
+define <4 x double> @shufflevector_v4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: shufflevector_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    movgr2fr.d $fa2, $a0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr1, 2
+; CHECK-NEXT:    movgr2fr.d $fa3, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa2
+; CHECK-NEXT:    xvinsgr2vr.d $xr2, $a0, 0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa3
+; CHECK-NEXT:    xvinsgr2vr.d $xr2, $a0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa0, $a0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr1, 3
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa0
+; CHECK-NEXT:    xvinsgr2vr.d $xr2, $a0, 2
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr2, $a0, 3
+; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
+; CHECK-NEXT:    ret
+entry:
+  %c = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 6, i32 3, i32 7>
+  ret <4 x double> %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll
new file mode 100644
index 000000000000..7b2461b11f12
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-bitcast-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define <8 x float> @insert_bitcast_v8f32(<8 x float> %a, i32 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i32 %b to float
+  %d = insertelement <8 x float> %a, float %c, i32 1
+  ret <8 x float> %d
+}
+
+define <4 x double> @insert_bitcast_v4f64(<4 x double> %a, i64 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i64 %b to double
+  %d = insertelement <4 x double> %a, double %c, i32 1
+  ret <4 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
new file mode 100644
index 000000000000..3fdc439e6867
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define <32 x i8> @insert_extract_v32i8(<32 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.b $a0, $sp, 31
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <32 x i8> %a, i32 31
+  %c = insertelement <32 x i8> %a, i8 %b, i32 1
+  ret <32 x i8> %c
+}
+
+define <16 x i16> @insert_extract_v16i16(<16 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.h $a0, $sp, 30
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <16 x i16> %a, i32 15
+  %c = insertelement <16 x i16> %a, i16 %b, i32 1
+  ret <16 x i16> %c
+}
+
+define <8 x i32> @insert_extract_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x i32> %a, i32 7
+  %c = insertelement <8 x i32> %a, i32 %b, i32 1
+  ret <8 x i32> %c
+}
+
+define <8 x float> @insert_extract_v8f32(<8 x float> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x float> %a, i32 7
+  %c = insertelement <8 x float> %a, float %b, i32 1
+  ret <8 x float> %c
+}
+
+define <4 x i64> @insert_extract_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x i64> %a, i32 3
+  %c = insertelement <4 x i64> %a, i64 %b, i32 1
+  ret <4 x i64> %c
+}
+
+define <4 x double> @insert_extract_v4f64(<4 x double> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x double> %a, i32 3
+  %c = insertelement <4 x double> %a, double %b, i32 1
+  ret <4 x double> %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll
new file mode 100644
index 000000000000..88c3e4367ffa
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-pair-elements.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s
+
+define <32 x i8> @insert_extract_v32i8(<32 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v32i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    vpickve2gr.b $a0, $vr0, 15
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.b $a1, $sp, 31
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <32 x i8> %a, i32 15
+  %b_hi = extractelement <32 x i8> %a, i32 31
+  %c = insertelement <32 x i8> %a, i8 %b_lo, i32 1
+  %d = insertelement <32 x i8> %c, i8 %b_hi, i32 17
+  ret <32 x i8> %d
+}
+
+define <16 x i16> @insert_extract_v16i16(<16 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -64
+; CHECK-NEXT:    st.d $ra, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    st.d $fp, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    addi.d $fp, $sp, 64
+; CHECK-NEXT:    bstrins.d $sp, $zero, 4, 0
+; CHECK-NEXT:    vpickve2gr.h $a0, $vr0, 7
+; CHECK-NEXT:    xvst $xr0, $sp, 0
+; CHECK-NEXT:    ld.h $a1, $sp, 30
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    xvori.b $xr1, $xr0, 0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 1
+; CHECK-NEXT:    vinsgr2vr.h $vr1, $a1, 1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    addi.d $sp, $fp, -64
+; CHECK-NEXT:    ld.d $fp, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    ld.d $ra, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 64
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <16 x i16> %a, i32 7
+  %b_hi = extractelement <16 x i16> %a, i32 15
+  %c = insertelement <16 x i16> %a, i16 %b_lo, i32 1
+  %d = insertelement <16 x i16> %c, i16 %b_hi, i32 9
+  ret <16 x i16> %d
+}
+
+define <8 x i32> @insert_extract_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
+; CHECK-NEXT:    xvpickve2gr.w $a1, $xr0, 7
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a1, 5
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <8 x i32> %a, i32 3
+  %b_hi = extractelement <8 x i32> %a, i32 7
+  %c = insertelement <8 x i32> %a, i32 %b_lo, i32 1
+  %d = insertelement <8 x i32> %c, i32 %b_hi, i32 5
+  ret <8 x i32> %d
+}
+
+define <8 x float> @insert_extract_v8f32(<8 x float> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
+; CHECK-NEXT:    movgr2fr.w $fa2, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    movfr2gr.s $a0, $fa2
+; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 5
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <8 x float> %a, i32 3
+  %b_hi = extractelement <8 x float> %a, i32 7
+  %c = insertelement <8 x float> %a, float %b_lo, i32 1
+  %d = insertelement <8 x float> %c, float %b_hi, i32 5
+  ret <8 x float> %d
+}
+
+define <4 x i64> @insert_extract_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    xvpickve2gr.d $a1, $xr0, 3
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a1, 2
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <4 x i64> %a, i32 1
+  %b_hi = extractelement <4 x i64> %a, i32 3
+  %c = insertelement <4 x i64> %a, i64 %b_lo, i32 0
+  %d = insertelement <4 x i64> %c, i64 %b_hi, i32 2
+  ret <4 x i64> %d
+}
+
+define <4 x double> @insert_extract_v4f64(<4 x double> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 1
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
+; CHECK-NEXT:    movgr2fr.d $fa2, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa2
+; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 2
+; CHECK-NEXT:    ret
+entry:
+  %b_lo = extractelement <4 x double> %a, i32 1
+  %b_hi = extractelement <4 x double> %a, i32 3
+  %c = insertelement <4 x double> %a, double %b_lo, i32 0
+  %d = insertelement <4 x double> %c, double %b_hi, i32 2
+  ret <4 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
index 5a861be95977..c61b7841b95a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/xvmskcond.ll
@@ -383,8 +383,7 @@ define i8 @xvmsk_eq_vsel_slt_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2)
 ; CHECK-LABEL: xvmsk_eq_vsel_slt_v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvseq.w $xr0, $xr0, $xr1
-; CHECK-NEXT:    xvrepli.b $xr1, -1
-; CHECK-NEXT:    xvbitsel.v $xr0, $xr2, $xr1, $xr0
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr2
 ; CHECK-NEXT:    xvmskltz.w $xr0, $xr0
 ; CHECK-NEXT:    xvpickve2gr.wu $a0, $xr0, 0
 ; CHECK-NEXT:    xvpickve2gr.wu $a1, $xr0, 4
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll
new file mode 100644
index 000000000000..df4896d7ec93
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/bitcast-extract-element.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
+
+define i32 @bitcast_extract_v4f32(<4 x float> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.w $vr0, $vr0, 3
+; CHECK-NEXT:    movfr2gr.s $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x float> %a, i32 3
+  %c = bitcast float %b to i32
+  ret i32 %c
+}
+
+define i64 @bitcast_extract_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: bitcast_extract_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.d $vr0, $vr0, 1
+; CHECK-NEXT:    movfr2gr.d $a0, $fa0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <2 x double> %a, i32 1
+  %c = bitcast double %b to i64
+  ret i64 %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll
new file mode 100644
index 000000000000..a20d17efdfb1
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-bitcast-element.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
+
+define <4 x float> @insert_bitcast_v4f32(<4 x float> %a, i32 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.w $fa1, $a0
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i32 %b to float
+  %d = insertelement <4 x float> %a, float %c, i32 1
+  ret <4 x float> %d
+}
+
+define <2 x double> @insert_bitcast_v2f64(<2 x double> %a, i64 %b) nounwind {
+; CHECK-LABEL: insert_bitcast_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movgr2fr.d $fa1, $a0
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %c = bitcast i64 %b to double
+  %d = insertelement <2 x double> %a, double %c, i32 1
+  ret <2 x double> %d
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
new file mode 100644
index 000000000000..c7dd1454c7e3
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/insert-extract-element.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s
+
+define <16 x i8> @insert_extract_v16i8(<16 x i8> %a) nounwind {
+; CHECK-LABEL: insert_extract_v16i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.b $a0, $vr0, 15
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <16 x i8> %a, i32 15
+  %c = insertelement <16 x i8> %a, i8 %b, i32 1
+  ret <16 x i8> %c
+}
+
+define <8 x i16> @insert_extract_v8i16(<8 x i16> %a) nounwind {
+; CHECK-LABEL: insert_extract_v8i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.h $a0, $vr0, 7
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x i16> %a, i32 7
+  %c = insertelement <8 x i16> %a, i16 %b, i32 1
+  ret <8 x i16> %c
+}
+
+define <4 x i32> @insert_extract_v4i32(<4 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 3
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x i32> %a, i32 3
+  %c = insertelement <4 x i32> %a, i32 %b, i32 1
+  ret <4 x i32> %c
+}
+
+define <4 x float> @insert_extract_v4f32(<4 x float> %a) nounwind {
+; CHECK-LABEL: insert_extract_v4f32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 3
+; CHECK-NEXT:    movfr2gr.s $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x float> %a, i32 3
+  %c = insertelement <4 x float> %a, float %b, i32 0
+  ret <4 x float> %c
+}
+
+define <2 x i64> @insert_extract_v2i64(<2 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract_v2i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 1
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <2 x i64> %a, i32 1
+  %c = insertelement <2 x i64> %a, i64 %b, i32 0
+  ret <2 x i64> %c
+}
+
+define <2 x double> @insert_extract_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: insert_extract_v2f64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    movfr2gr.d $a0, $fa1
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <2 x double> %a, i32 1
+  %c = insertelement <2 x double> %a, double %b, i32 0
+  ret <2 x double> %c
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
index 1c10e6c3087a..4d2ddeb2889b 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
@@ -6,8 +6,8 @@ define void @select_v16i8_imm(ptr %res, ptr %a0) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vld $vr0, $a1, 0
 ; CHECK-NEXT:    vrepli.h $vr1, -256
-; CHECK-NEXT:    vbitseli.b $vr1, $vr0, 255
-; CHECK-NEXT:    vst $vr1, $a0, 0
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
   %v0 = load <16 x i8>, ptr %a0
   %sel = select <16 x i1> <i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %v0
diff --git a/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir b/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
index bb36b1df115d..61c3ba2ee54a 100644
--- a/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
+++ b/llvm/test/CodeGen/MIR/NVPTX/floating-point-immediate-operands.mir
@@ -50,7 +50,7 @@ body: |
   ; CHECK: %6:b32 = FADD_rnf32ri %5, float 6.250000e+00
     %6 = FADD_rnf32ri %5, float 6.250000e+00
     %7 = FMUL_rnf32rr %6, %4
-    StoreRetvalI32 %7, 0
+    ST_i32 %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
     Return
 ...
 ---
@@ -76,6 +76,6 @@ body: |
   ; CHECK: %6:b32 = FADD_rnf32ri %5, float 0x7FF8000000000000
     %6 = FADD_rnf32ri %5, float 0x7FF8000000000000
     %7 = FMUL_rnf32rr %6, %4
-    StoreRetvalI32 %7, 0
+    ST_i32 %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
     Return
 ...
diff --git a/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll b/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll
new file mode 100644
index 000000000000..b0fbdc671761
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/powi-soften-libcall-error.ll
@@ -0,0 +1,16 @@
+; RUN: not llc -mtriple=msp430 -filetype=null %s 2>&1 | FileCheck %s
+
+; FIXME: This should not fail but isn't implemented
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error(float %x, i32 %n) {
+  %powi = call float @llvm.powi.f32.i32(float %x, i32 %n)
+  ret float %powi
+}
+
+; CHECK: error: powi exponent does not match sizeof(int)
+define float @soften_powi_error_strictfp(float %x, i32 %n) strictfp {
+ %powi = call float @llvm.experimental.constrained.powi.f32.i32(float %x, i32 %n, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  ret float %powi
+}
+
+
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/load.ll b/llvm/test/CodeGen/Mips/llvm-ir/load.ll
index ee858ac94aed..4f29fd848aed 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/load.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/load.ll
@@ -27,48 +27,48 @@ define i8 @f1() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
@@ -76,11 +76,11 @@ define i8 @f1() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -88,7 +88,7 @@ define i8 @f1() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -98,17 +98,17 @@ define i8 @f1() {
 ; MIPS3-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64-LABEL: f1:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -116,7 +116,7 @@ define i8 @f1() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -126,17 +126,17 @@ define i8 @f1() {
 ; MIPS64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -144,7 +144,7 @@ define i8 @f1() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -155,31 +155,31 @@ define i8 @f1() {
 ; MIPS64R6-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR5FP64-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST6:]] LBu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R5FP643-NEXT:    lbu $2, %lo(a)($1) # <MCInst #[[#MCINST3:]] LBu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 entry:
   %0 = load i8, ptr @a
   ret i8 %0
@@ -190,48 +190,48 @@ define i32 @f2() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -239,11 +239,11 @@ define i32 @f2() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -251,7 +251,7 @@ define i32 @f2() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -261,17 +261,17 @@ define i32 @f2() {
 ; MIPS3-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64-LABEL: f2:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -279,7 +279,7 @@ define i32 @f2() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -289,17 +289,17 @@ define i32 @f2() {
 ; MIPS64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -307,7 +307,7 @@ define i32 @f2() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -318,31 +318,31 @@ define i32 @f2() {
 ; MIPS64R6-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST14:]] LB_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lb $2, %lo(a)($1) # <MCInst #[[#MCINST13:]] LB
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 entry:
   %0 = load i8, ptr @a
   %1 = sext i8 %0 to i32
@@ -354,48 +354,48 @@ define i16 @f3() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -403,11 +403,11 @@ define i16 @f3() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -415,7 +415,7 @@ define i16 @f3() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -425,17 +425,17 @@ define i16 @f3() {
 ; MIPS3-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64-LABEL: f3:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -443,7 +443,7 @@ define i16 @f3() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -453,17 +453,17 @@ define i16 @f3() {
 ; MIPS64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -471,7 +471,7 @@ define i16 @f3() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -482,31 +482,31 @@ define i16 @f3() {
 ; MIPS64R6-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST16:]] LHu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lhu $2, %lo(b)($1) # <MCInst #[[#MCINST15:]] LHu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 entry:
   %0 = load i16, ptr @b
   ret i16 %0
@@ -517,48 +517,48 @@ define i32 @f4() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f4:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f4:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f4:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -566,11 +566,11 @@ define i32 @f4() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -578,7 +578,7 @@ define i32 @f4() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -588,17 +588,17 @@ define i32 @f4() {
 ; MIPS3-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64-LABEL: f4:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -606,7 +606,7 @@ define i32 @f4() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -616,17 +616,17 @@ define i32 @f4() {
 ; MIPS64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -634,7 +634,7 @@ define i32 @f4() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -645,31 +645,31 @@ define i32 @f4() {
 ; MIPS64R6-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST18:]] LH_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f4:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lh $2, %lo(b)($1) # <MCInst #[[#MCINST17:]] LH
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 entry:
   %0 = load i16, ptr @b
   %1 = sext i16 %0 to i32
@@ -681,48 +681,48 @@ define i32 @f5() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -730,11 +730,11 @@ define i32 @f5() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -742,7 +742,7 @@ define i32 @f5() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -752,17 +752,17 @@ define i32 @f5() {
 ; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f5:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -770,7 +770,7 @@ define i32 @f5() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -780,17 +780,17 @@ define i32 @f5() {
 ; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -798,7 +798,7 @@ define i32 @f5() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -809,31 +809,31 @@ define i32 @f5() {
 ; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST20:]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST19:]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 entry:
   %0 = load i32, ptr @c
   ret i32 %0
@@ -844,11 +844,11 @@ define i64 @f6() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
@@ -860,7 +860,7 @@ define i64 @f6() {
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Imm:0>>
@@ -869,17 +869,17 @@ define i64 @f6() {
 ; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -892,11 +892,11 @@ define i64 @f6() {
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Imm:0>>
@@ -907,11 +907,11 @@ define i64 @f6() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -919,7 +919,7 @@ define i64 @f6() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -929,17 +929,17 @@ define i64 @f6() {
 ; MIPS3-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f6:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -947,7 +947,7 @@ define i64 @f6() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -957,17 +957,17 @@ define i64 @f6() {
 ; MIPS64-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -975,7 +975,7 @@ define i64 @f6() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -986,13 +986,13 @@ define i64 @f6() {
 ; MIPS64R6-NEXT:    lwu $2, %lo(c)($1) # <MCInst #[[#MCINST23:]] LWu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    li16 $2, 0 # <MCInst #[[#MCINST22:]] LI16_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Imm:0>>
@@ -1001,17 +1001,17 @@ define i64 @f6() {
 ; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    addiu $2, $zero, 0 # <MCInst #[[#MCINST21:]] ADDiu
@@ -1029,11 +1029,11 @@ define i64 @f7() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
@@ -1045,11 +1045,11 @@ define i64 @f7() {
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
@@ -1061,11 +1061,11 @@ define i64 @f7() {
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -1078,11 +1078,11 @@ define i64 @f7() {
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
@@ -1094,11 +1094,11 @@ define i64 @f7() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1106,7 +1106,7 @@ define i64 @f7() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1116,17 +1116,17 @@ define i64 @f7() {
 ; MIPS3-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64-LABEL: f7:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1134,7 +1134,7 @@ define i64 @f7() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1144,17 +1144,17 @@ define i64 @f7() {
 ; MIPS64-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f7:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1162,7 +1162,7 @@ define i64 @f7() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1173,17 +1173,17 @@ define i64 @f7() {
 ; MIPS64R6-NEXT:    lw $2, %lo(c)($1) # <MCInst #[[#MCINST26:]] LW64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG9]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f7:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST20]] LW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST25:]] SRA_MM
@@ -1195,11 +1195,11 @@ define i64 @f7() {
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    lw $3, %lo(c)($1) # <MCInst #[[#MCINST19]] LW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG8]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sra $2, $3, 31 # <MCInst #[[#MCINST24:]] SRA
@@ -1217,48 +1217,48 @@ define float @f8() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR3-LABEL: f8:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R6-LABEL: f8:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR6-LABEL: f8:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -1266,11 +1266,11 @@ define float @f8() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1278,7 +1278,7 @@ define float @f8() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1288,17 +1288,17 @@ define float @f8() {
 ; MIPS3-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64-LABEL: f8:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1306,7 +1306,7 @@ define float @f8() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1316,17 +1316,17 @@ define float @f8() {
 ; MIPS64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64R6-LABEL: f8:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1334,7 +1334,7 @@ define float @f8() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1345,31 +1345,31 @@ define float @f8() {
 ; MIPS64R6-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR5FP64-LABEL: f8:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST28:]] LWC1_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f8:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    lwc1 $f0, %lo(e)($1) # <MCInst #[[#MCINST27:]] LWC1
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 entry:
   %0 = load float, ptr @e
   ret float %0
@@ -1380,48 +1380,48 @@ define double @f9() {
 ; MIPS32:       # %bb.0: # %entry
 ; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST29:]] LDC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR3-LABEL: f9:
 ; MMR3:       # %bb.0: # %entry
 ; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST30:]] LDC1_MM_D32
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R6-LABEL: f9:
 ; MIPS32R6:       # %bb.0: # %entry
 ; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR6-LABEL: f9:
 ; MMR6:       # %bb.0: # %entry
 ; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST32:]] LDC1_D64_MMR6
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -1429,11 +1429,11 @@ define double @f9() {
 ; MIPS3:       # %bb.0: # %entry
 ; MIPS3-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS3-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1441,7 +1441,7 @@ define double @f9() {
 ; MIPS3-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS3-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1451,17 +1451,17 @@ define double @f9() {
 ; MIPS3-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS3-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64-LABEL: f9:
 ; MIPS64:       # %bb.0: # %entry
 ; MIPS64-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1469,7 +1469,7 @@ define double @f9() {
 ; MIPS64-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1479,17 +1479,17 @@ define double @f9() {
 ; MIPS64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS64-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64R6-LABEL: f9:
 ; MIPS64R6:       # %bb.0: # %entry
 ; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1497,7 +1497,7 @@ define double @f9() {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -1508,31 +1508,31 @@ define double @f9() {
 ; MIPS64R6-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR5FP64-LABEL: f9:
 ; MMR5FP64:       # %bb.0: # %entry
 ; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST33:]] LDC1_MM_D64
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f9:
 ; MIPS32R5FP643:       # %bb.0: # %entry
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    ldc1 $f0, %lo(f)($1) # <MCInst #[[#MCINST31:]] LDC164
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 entry:
   %0 = load double, ptr @f
   ret double %0
diff --git a/llvm/test/CodeGen/Mips/llvm-ir/store.ll b/llvm/test/CodeGen/Mips/llvm-ir/store.ll
index 880a0f522574..8b51c0939b8d 100644
--- a/llvm/test/CodeGen/Mips/llvm-ir/store.ll
+++ b/llvm/test/CodeGen/Mips/llvm-ir/store.ll
@@ -26,48 +26,48 @@ define void @f1(i8 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR3-LABEL: f1:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR3-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R6-LABEL: f1:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7:]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR6-LABEL: f1:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8:]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ;
@@ -75,11 +75,11 @@ define void @f1(i8 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -87,7 +87,7 @@ define void @f1(i8 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -97,17 +97,17 @@ define void @f1(i8 %a) {
 ; MIPS4-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS64R6-LABEL: f1:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(a) # <MCInst #[[#MCINST9:]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,a)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(a) # <MCInst #[[#MCINST10:]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11:]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -115,7 +115,7 @@ define void @f1(i8 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(a) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -126,31 +126,31 @@ define void @f1(i8 %a) {
 ; MIPS64R6-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST12:]] SB64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MMR5FP64-LABEL: f1:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST4:]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5:]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MMR5FP64-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST6:]] SB_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
 ;
 ; MIPS32R5FP643-LABEL: f1:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(a) # <MCInst #[[#MCINST1:]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,a)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2:]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>>
 ; MIPS32R5FP643-NEXT:    sb $4, %lo(a)($1) # <MCInst #[[#MCINST3:]] SB
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,a)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,a)>>
   store i8 %a, ptr @a
   ret void
 }
@@ -160,48 +160,48 @@ define void @f2(i16 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR3-LABEL: f2:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R6-LABEL: f2:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR6-LABEL: f2:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -209,11 +209,11 @@ define void @f2(i16 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -221,7 +221,7 @@ define void @f2(i16 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -231,17 +231,17 @@ define void @f2(i16 %a) {
 ; MIPS4-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS64R6-LABEL: f2:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(b) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,b)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -249,7 +249,7 @@ define void @f2(i16 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(b) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -260,31 +260,31 @@ define void @f2(i16 %a) {
 ; MIPS64R6-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST16:]] SH64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MMR5FP64-LABEL: f2:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST15:]] SH_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
 ;
 ; MIPS32R5FP643-LABEL: f2:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(b) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,b)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sh $4, %lo(b)($1) # <MCInst #[[#MCINST14:]] SH
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,b)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,b)>>
   store i16 %a, ptr @b
   ret void
 }
@@ -294,48 +294,48 @@ define void @f3(i32 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR3-LABEL: f3:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R6-LABEL: f3:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR6-LABEL: f3:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -343,11 +343,11 @@ define void @f3(i32 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -355,7 +355,7 @@ define void @f3(i32 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -365,17 +365,17 @@ define void @f3(i32 %a) {
 ; MIPS4-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS64R6-LABEL: f3:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(c) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,c)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -383,7 +383,7 @@ define void @f3(i32 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(c) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -394,31 +394,31 @@ define void @f3(i32 %a) {
 ; MIPS64R6-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST19:]] SW64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MMR5FP64-LABEL: f3:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST18:]] SW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
 ;
 ; MIPS32R5FP643-LABEL: f3:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(c) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,c)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sw $4, %lo(c)($1) # <MCInst #[[#MCINST17:]] SW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,c)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,c)>>
   store i32 %a, ptr @c
   ret void
 }
@@ -428,15 +428,15 @@ define void @f4(i64 %a) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
@@ -448,15 +448,15 @@ define void @f4(i64 %a) {
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR3-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR3-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR3-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -468,15 +468,15 @@ define void @f4(i64 %a) {
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32R6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R6-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
@@ -489,15 +489,15 @@ define void @f4(i64 %a) {
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR6-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR6-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR6-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -509,11 +509,11 @@ define void @f4(i64 %a) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,d)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,d)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -521,7 +521,7 @@ define void @f4(i64 %a) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -531,17 +531,17 @@ define void @f4(i64 %a) {
 ; MIPS4-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ;
 ; MIPS64R6-LABEL: f4:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(d) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,d)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,d)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -549,7 +549,7 @@ define void @f4(i64 %a) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(d) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -560,21 +560,21 @@ define void @f4(i64 %a) {
 ; MIPS64R6-NEXT:    sd $4, %lo(d)($1) # <MCInst #[[#MCINST23:]] SD
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG7]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ;
 ; MMR5FP64-LABEL: f4:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MMR5FP64-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST18]] SW_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR5FP64-NEXT:    addiu $2, $1, %lo(d) # <MCInst #[[#MCINST21:]] ADDiu_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MMR5FP64-NEXT:    sw16 $5, 4($2) # <MCInst #[[#MCINST22:]] SW16_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG9:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG10]]>
@@ -586,15 +586,15 @@ define void @f4(i64 %a) {
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(d) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,d)>>
 ; MIPS32R5FP643-NEXT:    sw $4, %lo(d)($1) # <MCInst #[[#MCINST17]] SW
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG3]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R5FP643-NEXT:    addiu $1, $1, %lo(d) # <MCInst #[[#MCINST20:]] ADDiu
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,d)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,d)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sw $5, 4($1) # <MCInst #[[#MCINST17]] SW
@@ -610,48 +610,48 @@ define void @f5(float %e) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR3-LABEL: f5:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R6-LABEL: f5:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR6-LABEL: f5:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -659,11 +659,11 @@ define void @f5(float %e) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -671,7 +671,7 @@ define void @f5(float %e) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -681,17 +681,17 @@ define void @f5(float %e) {
 ; MIPS4-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS64R6-LABEL: f5:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(e) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,e)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -699,7 +699,7 @@ define void @f5(float %e) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(e) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -710,31 +710,31 @@ define void @f5(float %e) {
 ; MIPS64R6-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MMR5FP64-LABEL: f5:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST25:]] SWC1_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
 ;
 ; MIPS32R5FP643-LABEL: f5:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(e) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,e)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    swc1 $f12, %lo(e)($1) # <MCInst #[[#MCINST24:]] SWC1
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG11:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,e)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,e)>>
   store float %e, ptr @e
   ret void
 }
@@ -744,48 +744,48 @@ define void @f6(double %f) {
 ; MIPS32:       # %bb.0:
 ; MIPS32-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST26:]] SDC1
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MIPS32-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR3-LABEL: f6:
 ; MMR3:       # %bb.0:
 ; MMR3-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR3-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR3-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST27:]] SDC1_MM_D32
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG12:]]>
 ; MMR3-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR3-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR3-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R6-LABEL: f6:
 ; MIPS32R6:       # %bb.0:
 ; MIPS32R6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R6-NEXT:    jr $ra # <MCInst #[[#MCINST7]] JALR
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS32R6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR6-LABEL: f6:
 ; MMR6:       # %bb.0:
 ; MMR6-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST29:]] SDC1_D64_MMR6
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST8]] JRC16_MM
 ; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ;
@@ -793,11 +793,11 @@ define void @f6(double %f) {
 ; MIPS4:       # %bb.0:
 ; MIPS4-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS4-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -805,7 +805,7 @@ define void @f6(double %f) {
 ; MIPS4-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS4-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -815,17 +815,17 @@ define void @f6(double %f) {
 ; MIPS4-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS4-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS4-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS4-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS64R6-LABEL: f6:
 ; MIPS64R6:       # %bb.0:
 ; MIPS64R6-NEXT:    lui $1, %highest(f) # <MCInst #[[#MCINST9]] LUi64
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4030,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4026,f)>>
 ; MIPS64R6-NEXT:    daddiu $1, $1, %higher(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4029,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4025,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -833,7 +833,7 @@ define void @f6(double %f) {
 ; MIPS64R6-NEXT:    daddiu $1, $1, %hi(f) # <MCInst #[[#MCINST10]] DADDiu
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS64R6-NEXT:    dsll $1, $1, 16 # <MCInst #[[#MCINST11]] DSLL
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
@@ -844,31 +844,31 @@ define void @f6(double %f) {
 ; MIPS64R6-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS64R6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>
-; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS64R6-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MMR5FP64-LABEL: f6:
 ; MMR5FP64:       # %bb.0:
 ; MMR5FP64-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST4]] LUi_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MMR5FP64-NEXT:    jr $ra # <MCInst #[[#MCINST5]] JR_MM
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MMR5FP64-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST30:]] SDC1_MM_D64
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MMR5FP64-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MMR5FP64-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
 ;
 ; MIPS32R5FP643-LABEL: f6:
 ; MIPS32R5FP643:       # %bb.0:
 ; MIPS32R5FP643-NEXT:    lui $1, %hi(f) # <MCInst #[[#MCINST1]] LUi
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4028,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4024,f)>>
 ; MIPS32R5FP643-NEXT:    jr $ra # <MCInst #[[#MCINST2]] JR
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
 ; MIPS32R5FP643-NEXT:    sdc1 $f12, %lo(f)($1) # <MCInst #[[#MCINST28:]] SDC164
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG13:]]>
 ; MIPS32R5FP643-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
-; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4031,f)>>
+; MIPS32R5FP643-NEXT:    # <MCOperand Expr:specifier(4027,f)>>
   store double %f, ptr @f
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/msa/compare_float.ll b/llvm/test/CodeGen/Mips/msa/compare_float.ll
index 2656cb839768..d06131bd0962 100644
--- a/llvm/test/CodeGen/Mips/msa/compare_float.ll
+++ b/llvm/test/CodeGen/Mips/msa/compare_float.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=mips-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 ; RUN: llc -mtriple=mipsel-elf -mattr=+msa,+fp64,+mips32r2 < %s | FileCheck %s
 
@@ -6,656 +7,614 @@ declare <2 x double> @llvm.mips.fmax.d(<2 x double>, <2 x double>) nounwind
 declare <4 x float> @llvm.mips.fmin.w(<4 x float>, <4 x float>) nounwind
 declare <2 x double> @llvm.mips.fmin.d(<2 x double>, <2 x double>) nounwind
 
+; (setcc $a, $b, SETFALSE) is always folded, so we won't get fcaf
 define void @false_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: false_v4f32:
-
+; CHECK-LABEL: false_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldi.b $w0, 0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
   %2 = load <4 x float>, ptr %b
   %3 = fcmp false <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
   store <4 x i32> %4, ptr %c
   ret void
-
-  ; (setcc $a, $b, SETFALSE) is always folded, so we won't get fcaf:
-  ; CHECK-DAG: ldi.b [[R1:\$w[0-9]+]], 0
-  ; CHECK-DAG: st.w [[R1]], 0($4)
-  ; CHECK: .size false_v4f32
 }
 
+; (setcc $a, $b, SETFALSE) is always folded
 define void @false_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: false_v2f64:
-
+; CHECK-LABEL: false_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldi.b $w0, 0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <2 x double>, ptr %a
   %2 = load <2 x double>, ptr %b
   %3 = fcmp false <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
   store <2 x i64> %4, ptr %c
   ret void
-
-  ; (setcc $a, $b, SETFALSE) is always folded
-  ; CHECK-DAG: ldi.b [[R1:\$w[0-9]+]], 0
-  ; CHECK-DAG: st.w [[R1]], 0($4)
-  ; CHECK: .size false_v2f64
 }
 
 define void @oeq_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: oeq_v4f32:
-
+; CHECK-LABEL: oeq_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fceq.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp oeq <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fceq.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size oeq_v4f32
 }
 
 define void @oeq_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: oeq_v2f64:
-
+; CHECK-LABEL: oeq_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fceq.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp oeq <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fceq.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size oeq_v2f64
 }
 
 define void @oge_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: oge_v4f32:
-
+; CHECK-LABEL: oge_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fcle.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp oge <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcle.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size oge_v4f32
 }
 
 define void @oge_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: oge_v2f64:
-
+; CHECK-LABEL: oge_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fcle.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp oge <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcle.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size oge_v2f64
 }
 
 define void @ogt_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ogt_v4f32:
-
+; CHECK-LABEL: ogt_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fclt.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ogt <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fclt.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ogt_v4f32
 }
 
 define void @ogt_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ogt_v2f64:
-
+; CHECK-LABEL: ogt_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fclt.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ogt <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fclt.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ogt_v2f64
 }
 
 define void @ole_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ole_v4f32:
-
+; CHECK-LABEL: ole_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcle.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ole <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcle.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ole_v4f32
 }
 
 define void @ole_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ole_v2f64:
-
+; CHECK-LABEL: ole_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcle.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ole <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcle.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ole_v2f64
 }
 
 define void @olt_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: olt_v4f32:
-
+; CHECK-LABEL: olt_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fclt.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp olt <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fclt.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size olt_v4f32
 }
 
 define void @olt_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: olt_v2f64:
-
+; CHECK-LABEL: olt_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fclt.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp olt <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fclt.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size olt_v2f64
 }
 
 define void @one_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: one_v4f32:
-
+; CHECK-LABEL: one_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcne.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp one <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcne.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size one_v4f32
 }
 
 define void @one_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: one_v2f64:
-
+; CHECK-LABEL: one_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcne.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp one <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcne.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size one_v2f64
 }
 
 define void @ord_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ord_v4f32:
-
+; CHECK-LABEL: ord_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcor.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ord <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcor.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ord_v4f32
 }
 
 define void @ord_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ord_v2f64:
-
+; CHECK-LABEL: ord_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcor.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ord <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcor.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ord_v2f64
 }
 
 define void @ueq_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ueq_v4f32:
-
+; CHECK-LABEL: ueq_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcueq.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ueq <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcueq.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ueq_v4f32
 }
 
 define void @ueq_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ueq_v2f64:
-
+; CHECK-LABEL: ueq_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcueq.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ueq <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcueq.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ueq_v2f64
 }
 
 define void @uge_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: uge_v4f32:
-
+; CHECK-LABEL: uge_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fcule.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp uge <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcule.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size uge_v4f32
 }
 
 define void @uge_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: uge_v2f64:
-
+; CHECK-LABEL: uge_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fcule.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp uge <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcule.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size uge_v2f64
 }
 
 define void @ugt_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ugt_v4f32:
-
+; CHECK-LABEL: ugt_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fcult.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ugt <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcult.w [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ugt_v4f32
 }
 
 define void @ugt_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ugt_v2f64:
-
+; CHECK-LABEL: ugt_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fcult.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ugt <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcult.d [[R3:\$w[0-9]+]], [[R2]], [[R1]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ugt_v2f64
 }
 
 define void @ule_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ule_v4f32:
-
+; CHECK-LABEL: ule_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcule.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ule <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcule.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ule_v4f32
 }
 
 define void @ule_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ule_v2f64:
-
+; CHECK-LABEL: ule_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcule.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ule <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcule.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ule_v2f64
 }
 
 define void @ult_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ult_v4f32:
-
+; CHECK-LABEL: ult_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcult.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ult <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcult.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ult_v4f32
 }
 
 define void @ult_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: ult_v2f64:
-
+; CHECK-LABEL: ult_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcult.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ult <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcult.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size ult_v2f64
 }
 
 define void @uno_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: uno_v4f32:
-
+; CHECK-LABEL: uno_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fcun.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp uno <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
-  ; CHECK-DAG: fcun.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x i32> %4, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size uno_v4f32
 }
 
 define void @uno_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: uno_v2f64:
-
+; CHECK-LABEL: uno_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fcun.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp uno <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
-  ; CHECK-DAG: fcun.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x i64> %4, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size uno_v2f64
 }
 
+; (setcc $a, $b, SETTRUE) is always folded, so we won't get fcaf
 define void @true_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: true_v4f32:
-
+; CHECK-LABEL: true_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldi.b $w0, -1
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
   %2 = load <4 x float>, ptr %b
   %3 = fcmp true <4 x float> %1, %2
   %4 = sext <4 x i1> %3 to <4 x i32>
   store <4 x i32> %4, ptr %c
   ret void
-
-  ; (setcc $a, $b, SETTRUE) is always folded, so we won't get fcaf:
-  ; CHECK-DAG: ldi.b [[R1:\$w[0-9]+]], -1
-  ; CHECK-DAG: st.w [[R1]], 0($4)
-  ; CHECK: .size true_v4f32
 }
 
+; (setcc $a, $b, SETTRUE) is always folded.
 define void @true_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: true_v2f64:
-
+; CHECK-LABEL: true_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldi.b $w0, -1
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <2 x double>, ptr %a
   %2 = load <2 x double>, ptr %b
   %3 = fcmp true <2 x double> %1, %2
   %4 = sext <2 x i1> %3 to <2 x i64>
   store <2 x i64> %4, ptr %c
   ret void
-
-  ; (setcc $a, $b, SETTRUE) is always folded.
-  ; CHECK-DAG: ldi.b [[R1:\$w[0-9]+]], -1
-  ; CHECK-DAG: st.w [[R1]], 0($4)
-  ; CHECK: .size true_v2f64
 }
 
-define void @bsel_v4f32(ptr %d, ptr %a, ptr %b,
-                          ptr %c) nounwind {
-  ; CHECK: bsel_v4f32:
-
+; Note that IfSet and IfClr are swapped since the condition is inverted
+define void @bsel_v4f32(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
+; CHECK-LABEL: bsel_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fclt.w $w1, $w1, $w0
+; CHECK-NEXT:    ld.w $w2, 0($7)
+; CHECK-NEXT:    bsel.v $w1, $w2, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w1, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = load <4 x float>, ptr %c
-  ; CHECK-DAG: ld.w [[R3:\$w[0-9]+]], 0($7)
   %4 = fcmp ogt <4 x float> %1, %2
-  ; CHECK-DAG: fclt.w [[R4:\$w[0-9]+]], [[R2]], [[R1]]
   %5 = select <4 x i1> %4, <4 x float> %1, <4 x float> %3
-  ; Note that IfSet and IfClr are swapped since the condition is inverted
-  ; CHECK-DAG: bsel.v [[R4]], [[R3]], [[R1]]
   store <4 x float> %5, ptr %d
-  ; CHECK-DAG: st.w [[R4]], 0($4)
-
   ret void
-  ; CHECK: .size bsel_v4f32
 }
 
-define void @bsel_v2f64(ptr %d, ptr %a, ptr %b,
-                          ptr %c) nounwind {
-  ; CHECK: bsel_v2f64:
-
+; Note that IfSet and IfClr are swapped since the condition is inverted
+define void @bsel_v2f64(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
+; CHECK-LABEL: bsel_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fclt.d $w1, $w1, $w0
+; CHECK-NEXT:    ld.d $w2, 0($7)
+; CHECK-NEXT:    bsel.v $w1, $w2, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w1, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = load <2 x double>, ptr %c
-  ; CHECK-DAG: ld.d [[R3:\$w[0-9]+]], 0($7)
   %4 = fcmp ogt <2 x double> %1, %2
-  ; CHECK-DAG: fclt.d [[R4:\$w[0-9]+]], [[R2]], [[R1]]
   %5 = select <2 x i1> %4, <2 x double> %1, <2 x double> %3
-  ; Note that IfSet and IfClr are swapped since the condition is inverted
-  ; CHECK-DAG: bsel.v [[R4]], [[R3]], [[R1]]
   store <2 x double> %5, ptr %d
-  ; CHECK-DAG: st.d [[R4]], 0($4)
-
   ret void
-  ; CHECK: .size bsel_v2f64
 }
 
-define void @bseli_v4f32(ptr %d, ptr %a, ptr %b,
-                          ptr %c) nounwind {
-  ; CHECK: bseli_v4f32:
-
+; Note that IfSet and IfClr are swapped since the condition is inverted
+define void @bseli_v4f32(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
+; CHECK-LABEL: bseli_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($5)
+; CHECK-NEXT:    ld.w $w1, 0($6)
+; CHECK-NEXT:    fclt.w $w1, $w1, $w0
+; CHECK-NEXT:    and.v $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ogt <4 x float> %1, %2
-  ; CHECK-DAG: fclt.w [[R4:\$w[0-9]+]], [[R2]], [[R1]]
   %4 = select <4 x i1> %3, <4 x float> %1, <4 x float> zeroinitializer
-  ; Note that IfSet and IfClr are swapped since the condition is inverted
-  ; CHECK-DAG: bsel.v [[R4]], [[R3:\$w[0-9]+]], [[R1]]
   store <4 x float> %4, ptr %d
-  ; CHECK-DAG: st.w [[R4]], 0($4)
-
   ret void
-  ; CHECK: .size bseli_v4f32
 }
 
-define void @bseli_v2f64(ptr %d, ptr %a, ptr %b,
-                          ptr %c) nounwind {
-  ; CHECK: bseli_v2f64:
-
+; Note that IfSet and IfClr are swapped since the condition is inverted
+define void @bseli_v2f64(ptr %d, ptr %a, ptr %b, ptr %c) nounwind {
+; CHECK-LABEL: bseli_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($5)
+; CHECK-NEXT:    ld.d $w1, 0($6)
+; CHECK-NEXT:    fclt.d $w1, $w1, $w0
+; CHECK-NEXT:    and.v $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = fcmp ogt <2 x double> %1, %2
-  ; CHECK-DAG: fclt.d [[R4:\$w[0-9]+]], [[R2]], [[R1]]
   %4 = select <2 x i1> %3, <2 x double> %1, <2 x double> zeroinitializer
-  ; Note that IfSet and IfClr are swapped since the condition is inverted
-  ; CHECK-DAG: bsel.v [[R4]], [[R3:\$w[0-9]+]], [[R1]]
   store <2 x double> %4, ptr %d
-  ; CHECK-DAG: st.d [[R4]], 0($4)
-
   ret void
-  ; CHECK: .size bseli_v2f64
 }
 
 define void @max_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: max_v4f32:
-
+; CHECK-LABEL: max_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fmax.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = tail call <4 x float> @llvm.mips.fmax.w(<4 x float> %1, <4 x float> %2)
-  ; CHECK-DAG: fmax.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x float> %3, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size max_v4f32
 }
 
 define void @max_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: max_v2f64:
-
+; CHECK-LABEL: max_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fmax.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = tail call <2 x double> @llvm.mips.fmax.d(<2 x double> %1, <2 x double> %2)
-  ; CHECK-DAG: fmax.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x double> %3, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size max_v2f64
 }
 
 define void @min_v4f32(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: min_v4f32:
-
+; CHECK-LABEL: min_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $w0, 0($6)
+; CHECK-NEXT:    ld.w $w1, 0($5)
+; CHECK-NEXT:    fmin.w $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.w $w0, 0($4)
   %1 = load <4 x float>, ptr %a
-  ; CHECK-DAG: ld.w [[R1:\$w[0-9]+]], 0($5)
   %2 = load <4 x float>, ptr %b
-  ; CHECK-DAG: ld.w [[R2:\$w[0-9]+]], 0($6)
   %3 = tail call <4 x float> @llvm.mips.fmin.w(<4 x float> %1, <4 x float> %2)
-  ; CHECK-DAG: fmin.w [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <4 x float> %3, ptr %c
-  ; CHECK-DAG: st.w [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size min_v4f32
 }
 
 define void @min_v2f64(ptr %c, ptr %a, ptr %b) nounwind {
-  ; CHECK: min_v2f64:
-
+; CHECK-LABEL: min_v2f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $w0, 0($6)
+; CHECK-NEXT:    ld.d $w1, 0($5)
+; CHECK-NEXT:    fmin.d $w0, $w1, $w0
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    st.d $w0, 0($4)
   %1 = load <2 x double>, ptr %a
-  ; CHECK-DAG: ld.d [[R1:\$w[0-9]+]], 0($5)
   %2 = load <2 x double>, ptr %b
-  ; CHECK-DAG: ld.d [[R2:\$w[0-9]+]], 0($6)
   %3 = tail call <2 x double> @llvm.mips.fmin.d(<2 x double> %1, <2 x double> %2)
-  ; CHECK-DAG: fmin.d [[R3:\$w[0-9]+]], [[R1]], [[R2]]
   store <2 x double> %3, ptr %c
-  ; CHECK-DAG: st.d [[R3]], 0($4)
-
   ret void
-  ; CHECK: .size min_v2f64
 }
diff --git a/llvm/test/CodeGen/Mips/setcc-se.ll b/llvm/test/CodeGen/Mips/setcc-se.ll
index 26fd2845f081..c1054607d5e9 100644
--- a/llvm/test/CodeGen/Mips/setcc-se.ll
+++ b/llvm/test/CodeGen/Mips/setcc-se.ll
@@ -1,39 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=mipsel < %s | FileCheck %s
-; RUN: llc  -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
+; RUN: llc -mtriple=mips -mcpu=mips32r6 -mattr=micromips -relocation-model=pic < %s -asm-show-inst | FileCheck %s -check-prefix=MMR6
 
 @g1 = external global i32
 
-; CHECK-LABEL: seteq0:
-; CHECK:  sltiu ${{[0-9]+}}, $4, 1
-; MMR6:   sltiu ${{[0-9]+}}, $4, 1
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-
 define i32 @seteq0(i32 %a) {
+; CHECK-LABEL: seteq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    sltiu $2, $4, 1
+;
+; MMR6-LABEL: seteq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sltiu $2, $4, 1 # <MCInst #[[#MCINST1:]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2:]]>
+; MMR6-NEXT:    # <MCOperand Imm:1>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2:]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3:]]>>
 entry:
   %cmp = icmp eq i32 %a, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
-; CHECK-LABEL: setne0:
-; CHECK:  sltu ${{[0-9]+}}, $zero, $4
-; MMR6:   sltu ${{[0-9]+}}, $zero, $4
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define i32 @setne0(i32 %a) {
+; CHECK-LABEL: setne0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    sltu $2, $zero, $4
+;
+; MMR6-LABEL: setne0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    sltu $2, $zero, $4 # <MCInst #[[#MCINST3:]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ne i32 %a, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
-; CHECK-LABEL: slti_beq0:
-; CHECK:  slti $[[R0:[0-9]+]], $4, -32768
-; MMR6:   slti $[[R0:[0-9]+]], $4, -32768
-; MMR6:   <MCInst #{{[0-9]+}} SLTi_MM
-; CHECK:  beqz $[[R0]]
-
 define void @slti_beq0(i32 %a) {
+; CHECK-LABEL: slti_beq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slti $1, $4, -32768
+; CHECK-NEXT:    beqz $1, $BB2_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB2_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4:]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5:]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6:]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5:]]>>
+; MMR6-NEXT:    slti $1, $4, -32768 # <MCInst #[[#MCINST7:]] SLTi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6:]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:-32768>>
+; MMR6-NEXT:    beqzc $1, $BB2_2 # <MCInst #[[#MCINST8:]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB2_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9:]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10:]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB2_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, -32768
   br i1 %cmp, label %if.then, label %if.end
@@ -46,12 +101,60 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq1:
-; CHECK:  slt ${{[0-9]+}}
-; MMR6:   slt ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLT_MM
-
 define void @slti_beq1(i32 %a) {
+; CHECK-LABEL: slti_beq1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui $1, 65535
+; CHECK-NEXT:    ori $1, $1, 32766
+; CHECK-NEXT:    slt $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB3_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB3_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq1:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    lui $1, 65535 # <MCInst #[[#MCINST11:]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:65535>>
+; MMR6-NEXT:    ori $1, $1, 32766 # <MCInst #[[#MCINST12:]] ORi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:32766>>
+; MMR6-NEXT:    slt $1, $1, $4 # <MCInst #[[#MCINST13:]] SLT_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB3_2 # <MCInst #[[#MCINST14:]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB3_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB3_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, -32769
   br i1 %cmp, label %if.then, label %if.end
@@ -64,13 +167,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq2:
-; CHECK:  slti $[[R0:[0-9]+]], $4, 32767
-; MMR6:   slti $[[R0:[0-9]+]], $4, 32767
-; MMR6:   <MCInst #{{[0-9]+}} SLTi_MM
-; CHECK:  beqz $[[R0]]
-
 define void @slti_beq2(i32 %a) {
+; CHECK-LABEL: slti_beq2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    slti $1, $4, 32767
+; CHECK-NEXT:    beqz $1, $BB4_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB4_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq2:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    slti $1, $4, 32767 # <MCInst #[[#MCINST7]] SLTi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    beqzc $1, $BB4_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB4_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB4_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, 32767
   br i1 %cmp, label %if.then, label %if.end
@@ -83,12 +224,56 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: slti_beq3:
-; CHECK:  slt ${{[0-9]+}}
-; MMR6:   slt ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLT_MM
-
 define void @slti_beq3(i32 %a) {
+; CHECK-LABEL: slti_beq3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $1, $zero, 32767
+; CHECK-NEXT:    slt $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB5_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB5_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: slti_beq3:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    addiu $1, $zero, 32767 # <MCInst #[[#MCINST15:]] ADDiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    slt $1, $1, $4 # <MCInst #[[#MCINST13]] SLT_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB5_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB5_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB5_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp slt i32 %a, 32768
   br i1 %cmp, label %if.then, label %if.end
@@ -101,13 +286,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq0:
-; CHECK:  sltiu $[[R0:[0-9]+]], $4, 32767
-; MMR6:   sltiu $[[R0:[0-9]+]], $4, 32767
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-; CHECK:  beqz $[[R0]]
-
 define void @sltiu_beq0(i32 %a) {
+; CHECK-LABEL: sltiu_beq0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sltiu $1, $4, 32767
+; CHECK-NEXT:    beqz $1, $BB6_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB6_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq0:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    sltiu $1, $4, 32767 # <MCInst #[[#MCINST1]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    beqzc $1, $BB6_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB6_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB6_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, 32767
   br i1 %cmp, label %if.then, label %if.end
@@ -120,12 +343,56 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq1:
-; CHECK:  sltu ${{[0-9]+}}
-; MMR6:   sltu ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define void @sltiu_beq1(i32 %a) {
+; CHECK-LABEL: sltiu_beq1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addiu $1, $zero, 32767
+; CHECK-NEXT:    sltu $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB7_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB7_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq1:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    addiu $1, $zero, 32767 # <MCInst #[[#MCINST15]] ADDiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG4]]>
+; MMR6-NEXT:    # <MCOperand Imm:32767>>
+; MMR6-NEXT:    sltu $1, $1, $4 # <MCInst #[[#MCINST3]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB7_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB7_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB7_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, 32768
   br i1 %cmp, label %if.then, label %if.end
@@ -138,13 +405,51 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq2:
-; CHECK:  sltiu $[[R0:[0-9]+]], $4, -32768
-; MMR6:   sltiu $[[R0:[0-9]+]], $4, -32768
-; MMR6:   <MCInst #{{[0-9]+}} SLTiu_MM
-; CHECK:  beqz $[[R0]]
-
 define void @sltiu_beq2(i32 %a) {
+; CHECK-LABEL: sltiu_beq2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    sltiu $1, $4, -32768
+; CHECK-NEXT:    beqz $1, $BB8_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB8_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq2:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    sltiu $1, $4, -32768 # <MCInst #[[#MCINST1]] SLTiu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Imm:-32768>>
+; MMR6-NEXT:    beqzc $1, $BB8_2 # <MCInst #[[#MCINST8]] BEQZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB8_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB8_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, -32768
   br i1 %cmp, label %if.then, label %if.end
@@ -157,12 +462,60 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: sltiu_beq3:
-; CHECK:  sltu ${{[0-9]+}}
-; MMR6:   sltu ${{[0-9]+}}
-; MMR6:   <MCInst #{{[0-9]+}} SLTu_MM
-
 define void @sltiu_beq3(i32 %a) {
+; CHECK-LABEL: sltiu_beq3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui $1, 65535
+; CHECK-NEXT:    ori $1, $1, 32766
+; CHECK-NEXT:    sltu $1, $1, $4
+; CHECK-NEXT:    bnez $1, $BB9_2
+; CHECK-NEXT:    nop
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    lui $1, %hi(g1)
+; CHECK-NEXT:    sw $4, %lo(g1)($1)
+; CHECK-NEXT:  $BB9_2: # %if.end
+; CHECK-NEXT:    jr $ra
+; CHECK-NEXT:    nop
+;
+; MMR6-LABEL: sltiu_beq3:
+; MMR6:       # %bb.0: # %entry
+; MMR6-NEXT:    lui $2, %hi(_gp_disp) # <MCInst #[[#MCINST4]] LUi
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4024,_gp_disp)>>
+; MMR6-NEXT:    addiu $2, $2, %lo(_gp_disp) # <MCInst #[[#MCINST5]] ADDiu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4027,_gp_disp)>>
+; MMR6-NEXT:    addu $2, $2, $25 # <MCInst #[[#MCINST6]] ADDu
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG5]]>>
+; MMR6-NEXT:    lui $1, 65535 # <MCInst #[[#MCINST11]] LUi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:65535>>
+; MMR6-NEXT:    ori $1, $1, 32766 # <MCInst #[[#MCINST12]] ORi_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Imm:32766>>
+; MMR6-NEXT:    sltu $1, $1, $4 # <MCInst #[[#MCINST3]] SLTu_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>>
+; MMR6-NEXT:    bnezc $1, $BB9_2 # <MCInst #[[#MCINST14]] BNEZC_MMR6
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG6]]>
+; MMR6-NEXT:    # <MCOperand Expr:$BB9_2>>
+; MMR6-NEXT:  # %bb.1: # %if.then
+; MMR6-NEXT:    lw $2, %got(g1)($2) # <MCInst #[[#MCINST9]] LW_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Expr:specifier(4015,g1)>>
+; MMR6-NEXT:    sw16 $4, 0($2) # <MCInst #[[#MCINST10]] SW16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG2]]>
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG1]]>
+; MMR6-NEXT:    # <MCOperand Imm:0>>
+; MMR6-NEXT:  $BB9_2: # %if.end
+; MMR6-NEXT:    jrc $ra # <MCInst #[[#MCINST2]] JRC16_MM
+; MMR6-NEXT:    # <MCOperand Reg:[[#MCREG3]]>>
 entry:
   %cmp = icmp ult i32 %a, -32769
   br i1 %cmp, label %if.then, label %if.end
diff --git a/llvm/test/CodeGen/NVPTX/access-non-generic.ll b/llvm/test/CodeGen/NVPTX/access-non-generic.ll
index 9edd4de017ee..601a35288f54 100644
--- a/llvm/test/CodeGen/NVPTX/access-non-generic.ll
+++ b/llvm/test/CodeGen/NVPTX/access-non-generic.ll
@@ -107,8 +107,7 @@ define void @nested_const_expr() {
 ; PTX-LABEL: nested_const_expr(
   ; store 1 to bitcast(gep(addrspacecast(array), 0, 1))
   store i32 1, ptr getelementptr ([10 x float], ptr addrspacecast (ptr addrspace(3) @array to ptr), i64 0, i64 1), align 4
-; PTX: mov.b32 %r1, 1;
-; PTX-NEXT: st.shared.b32 [array+4], %r1;
+; PTX: st.shared.b32 [array+4], 1;
   ret void
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/alias.ll b/llvm/test/CodeGen/NVPTX/alias.ll
index 8ae29b51290e..01761c21ab10 100644
--- a/llvm/test/CodeGen/NVPTX/alias.ll
+++ b/llvm/test/CodeGen/NVPTX/alias.ll
@@ -56,8 +56,7 @@ attributes #0 = { noreturn }
 ; CHECK-NEXT: .noreturn
 
 ;      CHECK: .visible .func  (.param .b32 func_retval0) z()
-;      CHECK:      call.uni (retval0), 
-; CHECK-NEXT:      b,
+;      CHECK:      call.uni (retval0), b,
 
 
 ; CHECK: .alias b, a;
diff --git a/llvm/test/CodeGen/NVPTX/and-or-setcc.ll b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
index b7e6e8b85298..e75dd20e46a6 100644
--- a/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
+++ b/llvm/test/CodeGen/NVPTX/and-or-setcc.ll
@@ -14,7 +14,7 @@ define i1 @and_ord(float %a, float %b) {
 ; CHECK-NEXT:    ld.param.b32 %r1, [and_ord_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [and_ord_param_1];
 ; CHECK-NEXT:    setp.num.f32 %p1, %r1, %r2;
-; CHECK-NEXT:    selp.b32 %r3, 1, 0, %p1;
+; CHECK-NEXT:    selp.b32 %r3, -1, 0, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %c = fcmp ord float %a, 0.0
@@ -33,7 +33,7 @@ define i1 @or_uno(float %a, float %b) {
 ; CHECK-NEXT:    ld.param.b32 %r1, [or_uno_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [or_uno_param_1];
 ; CHECK-NEXT:    setp.nan.f32 %p1, %r1, %r2;
-; CHECK-NEXT:    selp.b32 %r3, 1, 0, %p1;
+; CHECK-NEXT:    selp.b32 %r3, -1, 0, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %c = fcmp uno float %a, 0.0
diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll
index c8dc34e9de2c..55ce3dfc4653 100644
--- a/llvm/test/CodeGen/NVPTX/atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics.ll
@@ -424,7 +424,7 @@ define half @atomicrmw_add_f16_generic(ptr %addr, half %val) {
 ; CHECK-LABEL: atomicrmw_add_f16_generic(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-NEXT:    .reg .b32 %r<20>;
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
@@ -458,8 +458,7 @@ define half @atomicrmw_add_f16_generic(ptr %addr, half %val) {
 ; CHECK-NEXT:    @%p1 bra $L__BB24_1;
 ; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
 ; CHECK-NEXT:    shr.u32 %r18, %r5, %r1;
-; CHECK-NEXT:    cvt.u16.u32 %rs4, %r18;
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r18;
 ; CHECK-NEXT:    ret;
   %ret = atomicrmw fadd ptr %addr, half %val seq_cst
   ret half %ret
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 32225ed04e2d..f59f51c3c57d 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -17,8 +17,7 @@ define bfloat @test_fadd(bfloat %0, bfloat %1) {
 ; SM70-LABEL: test_fadd(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .b32 %r<12>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %r1, [test_fadd_param_1];
@@ -32,8 +31,8 @@ define bfloat @test_fadd(bfloat %0, bfloat %1) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
 ; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
 ; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r10;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r11, %r10, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r11;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fadd(
@@ -81,8 +80,7 @@ define bfloat @test_fsub(bfloat %0, bfloat %1) {
 ; SM70-LABEL: test_fsub(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .b32 %r<12>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %r1, [test_fsub_param_1];
@@ -96,8 +94,8 @@ define bfloat @test_fsub(bfloat %0, bfloat %1) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
 ; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
 ; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r10;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r11, %r10, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r11;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fsub(
@@ -146,37 +144,35 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<5>;
-; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .b32 %r<22>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_faddx2_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_faddx2_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    add.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT:    add.s32 %r9, %r8, %r7;
-; SM70-NEXT:    add.s32 %r10, %r9, 32767;
-; SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
-; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    add.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    add.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT:    add.s32 %r19, %r18, %r17;
-; SM70-NEXT:    add.s32 %r20, %r19, 32767;
-; SM70-NEXT:    setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
-; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r23;
+; SM70-NEXT:    add.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT:    bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT:    add.s32 %r17, %r16, %r15;
+; SM70-NEXT:    add.s32 %r18, %r17, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT:    or.b32 %r19, %r15, 4194304;
+; SM70-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT:    prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r21;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_faddx2(
@@ -184,31 +180,29 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    .reg .b32 %r<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
 ; SM80-NEXT:    mov.b32 %r3, 1065369472;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT:    fma.rn.bf16x2 %r4, %r1, %r3, %r2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_faddx2(
 ; SM80-FTZ:       {
 ; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT:    .reg .b32 %r<10>;
+; SM80-FTZ-NEXT:    .reg .b32 %r<8>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT:    add.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT:    add.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_faddx2_param_0];
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_faddx2_param_1];
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT:    add.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT:    add.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_faddx2(
@@ -216,9 +210,9 @@ define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_faddx2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_faddx2_param_0];
-; SM90-NEXT:    add.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM90-NEXT:    add.rn.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fadd <2 x bfloat> %a, %b
@@ -230,37 +224,35 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<5>;
-; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .b32 %r<22>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fsubx2_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fsubx2_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    sub.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT:    add.s32 %r9, %r8, %r7;
-; SM70-NEXT:    add.s32 %r10, %r9, 32767;
-; SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
-; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    sub.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    sub.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT:    add.s32 %r19, %r18, %r17;
-; SM70-NEXT:    add.s32 %r20, %r19, 32767;
-; SM70-NEXT:    setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
-; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r23;
+; SM70-NEXT:    sub.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT:    bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT:    add.s32 %r17, %r16, %r15;
+; SM70-NEXT:    add.s32 %r18, %r17, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT:    or.b32 %r19, %r15, 4194304;
+; SM70-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT:    prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r21;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fsubx2(
@@ -269,30 +261,28 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-NEXT:    mov.b32 %r3, -1082081408;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT:    mov.b32 %r2, -1082081408;
+; SM80-NEXT:    ld.param.b32 %r3, [test_fsubx2_param_1];
+; SM80-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fsubx2(
 ; SM80-FTZ:       {
 ; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT:    .reg .b32 %r<10>;
+; SM80-FTZ-NEXT:    .reg .b32 %r<8>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fsubx2_param_0];
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fsubx2_param_1];
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT:    sub.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_fsubx2(
@@ -300,9 +290,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_0];
-; SM90-NEXT:    sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM90-NEXT:    sub.rn.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fsub <2 x bfloat> %a, %b
@@ -314,37 +304,35 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<5>;
-; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .b32 %r<22>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fmulx2_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fmulx2_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    mul.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT:    add.s32 %r9, %r8, %r7;
-; SM70-NEXT:    add.s32 %r10, %r9, 32767;
-; SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
-; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    mul.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    mul.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT:    add.s32 %r19, %r18, %r17;
-; SM70-NEXT:    add.s32 %r20, %r19, 32767;
-; SM70-NEXT:    setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
-; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r23;
+; SM70-NEXT:    mul.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT:    bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT:    add.s32 %r17, %r16, %r15;
+; SM70-NEXT:    add.s32 %r18, %r17, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT:    or.b32 %r19, %r15, 4194304;
+; SM70-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT:    prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r21;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fmulx2(
@@ -352,31 +340,29 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    .reg .b32 %r<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM80-NEXT:    mov.b32 %r3, -2147450880;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r1, %r3;
+; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT:    mov.b32 %r2, -2147450880;
+; SM80-NEXT:    ld.param.b32 %r3, [test_fmulx2_param_1];
+; SM80-NEXT:    fma.rn.bf16x2 %r4, %r1, %r3, %r2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fmulx2(
 ; SM80-FTZ:       {
 ; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT:    .reg .b32 %r<10>;
+; SM80-FTZ-NEXT:    .reg .b32 %r<8>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fmulx2_param_0];
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fmulx2_param_1];
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT:    mul.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_fmulx2(
@@ -384,9 +370,9 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM90-NEXT:    mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM90-NEXT:    mul.rn.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fmul <2 x bfloat> %a, %b
@@ -398,97 +384,89 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<5>;
-; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .b32 %r<22>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    div.rn.f32 %r7, %r6, %r4;
-; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT:    add.s32 %r9, %r8, %r7;
-; SM70-NEXT:    add.s32 %r10, %r9, 32767;
-; SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
-; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    div.rn.f32 %r5, %r4, %r2;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    div.rn.f32 %r17, %r16, %r14;
-; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT:    add.s32 %r19, %r18, %r17;
-; SM70-NEXT:    add.s32 %r20, %r19, 32767;
-; SM70-NEXT:    setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
-; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r23;
+; SM70-NEXT:    div.rn.f32 %r15, %r14, %r12;
+; SM70-NEXT:    bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT:    add.s32 %r17, %r16, %r15;
+; SM70-NEXT:    add.s32 %r18, %r17, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT:    or.b32 %r19, %r15, 4194304;
+; SM70-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT:    prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r21;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fdiv(
 ; SM80:       {
 ; SM80-NEXT:    .reg .b16 %rs<5>;
-; SM80-NEXT:    .reg .b32 %r<10>;
+; SM80-NEXT:    .reg .b32 %r<8>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r3, %rs1;
-; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r4, %rs3;
-; SM80-NEXT:    div.rn.f32 %r5, %r4, %r3;
-; SM80-NEXT:    cvt.f32.bf16 %r6, %rs2;
-; SM80-NEXT:    cvt.f32.bf16 %r7, %rs4;
-; SM80-NEXT:    div.rn.f32 %r8, %r7, %r6;
-; SM80-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM80-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; SM80-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r4, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %r5, %rs2;
+; SM80-NEXT:    div.rn.f32 %r6, %r5, %r4;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_fdiv(
 ; SM80-FTZ:       {
 ; SM80-FTZ-NEXT:    .reg .b16 %rs<5>;
-; SM80-FTZ-NEXT:    .reg .b32 %r<10>;
+; SM80-FTZ-NEXT:    .reg .b32 %r<8>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs3;
-; SM80-FTZ-NEXT:    div.rn.ftz.f32 %r5, %r4, %r3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT:    div.rn.ftz.f32 %r8, %r7, %r6;
-; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM80-FTZ-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; SM80-FTZ-NEXT:    div.rn.ftz.f32 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs2;
+; SM80-FTZ-NEXT:    div.rn.ftz.f32 %r6, %r5, %r4;
+; SM80-FTZ-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_fdiv(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<5>;
-; SM90-NEXT:    .reg .b32 %r<10>;
+; SM90-NEXT:    .reg .b32 %r<8>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    cvt.f32.bf16 %r3, %rs1;
-; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    cvt.f32.bf16 %r4, %rs3;
-; SM90-NEXT:    div.rn.f32 %r5, %r4, %r3;
-; SM90-NEXT:    cvt.f32.bf16 %r6, %rs2;
-; SM90-NEXT:    cvt.f32.bf16 %r7, %rs4;
-; SM90-NEXT:    div.rn.f32 %r8, %r7, %r6;
-; SM90-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; SM90-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; SM90-NEXT:    cvt.f32.bf16 %r1, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; SM90-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; SM90-NEXT:    cvt.f32.bf16 %r4, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %r5, %rs2;
+; SM90-NEXT:    div.rn.f32 %r6, %r5, %r4;
+; SM90-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r7;
 ; SM90-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
@@ -571,8 +549,7 @@ define bfloat @test_fptrunc_float(float %a) #0 {
 ; SM70-LABEL: test_fptrunc_float(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .b32 %r<8>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b32 %r1, [test_fptrunc_float_param_0];
@@ -582,8 +559,8 @@ define bfloat @test_fptrunc_float(float %a) #0 {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r1, %r1;
 ; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
 ; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r6;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r7, %r6, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fptrunc_float(
@@ -626,8 +603,7 @@ define bfloat @test_fadd_imm_1(bfloat %a) #0 {
 ; SM70-LABEL: test_fadd_imm_1(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .b32 %r<10>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %r1, [test_fadd_imm_1_param_0];
@@ -639,8 +615,8 @@ define bfloat @test_fadd_imm_1(bfloat %a) #0 {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r3, %r3;
 ; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
 ; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r8;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r9, %r8, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r9;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_fadd_imm_1(
@@ -712,10 +688,10 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM70-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM70-NEXT:    cvt.u32.u16 %r5, %rs8;
 ; SM70-NEXT:    shl.b32 %r6, %r5, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r7, %rs7;
@@ -732,8 +708,8 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM70-NEXT:    shl.b32 %r18, %r17, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r19, %rs1;
 ; SM70-NEXT:    shl.b32 %r20, %r19, 16;
-; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r20, %r18, %r16, %r14};
-; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r10, %r8, %r6};
+; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r20, %r18, %r16, %r14};
+; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r10, %r8, %r6};
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_extload_bf16x8(
@@ -745,10 +721,10 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM80-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
-; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM80-NEXT:    cvt.f32.bf16 %r5, %rs8;
 ; SM80-NEXT:    cvt.f32.bf16 %r6, %rs7;
 ; SM80-NEXT:    cvt.f32.bf16 %r7, %rs6;
@@ -757,8 +733,8 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-NEXT:    cvt.f32.bf16 %r10, %rs3;
 ; SM80-NEXT:    cvt.f32.bf16 %r11, %rs2;
 ; SM80-NEXT:    cvt.f32.bf16 %r12, %rs1;
-; SM80-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r10, %r9};
-; SM80-NEXT:    st.param.v4.b32 [func_retval0+16], {%r8, %r7, %r6, %r5};
+; SM80-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
+; SM80-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM80-NEXT:    ret;
 ;
 ; SM80-FTZ-LABEL: test_extload_bf16x8(
@@ -770,10 +746,10 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-FTZ-NEXT:  // %bb.0:
 ; SM80-FTZ-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM80-FTZ-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; SM80-FTZ-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
-; SM80-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM80-FTZ-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM80-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs8;
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs7;
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs6;
@@ -782,8 +758,8 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r10, %rs3;
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r11, %rs2;
 ; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r12, %rs1;
-; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r10, %r9};
-; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0+16], {%r8, %r7, %r6, %r5};
+; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
+; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM80-FTZ-NEXT:    ret;
 ;
 ; SM90-LABEL: test_extload_bf16x8(
@@ -795,10 +771,10 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
-; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
 ; SM90-NEXT:    cvt.f32.bf16 %r5, %rs8;
 ; SM90-NEXT:    cvt.f32.bf16 %r6, %rs7;
 ; SM90-NEXT:    cvt.f32.bf16 %r7, %rs6;
@@ -807,8 +783,8 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM90-NEXT:    cvt.f32.bf16 %r10, %rs3;
 ; SM90-NEXT:    cvt.f32.bf16 %r11, %rs2;
 ; SM90-NEXT:    cvt.f32.bf16 %r12, %rs1;
-; SM90-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r10, %r9};
-; SM90-NEXT:    st.param.v4.b32 [func_retval0+16], {%r8, %r7, %r6, %r5};
+; SM90-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
+; SM90-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM90-NEXT:    ret;
   %load = load <8 x bfloat>, ptr addrspace(3) %arg, align 16
   %res = fpext <8 x bfloat> %load to <8 x float>
@@ -929,8 +905,8 @@ define bfloat @test_sitofp_i16(i16 %a) {
 ; SM70-LABEL: test_sitofp_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<3>;
-; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<8>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [test_sitofp_i16_param_0];
@@ -941,8 +917,8 @@ define bfloat @test_sitofp_i16(i16 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r1, %r1;
 ; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
 ; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs2}, %r6;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs2;
+; SM70-NEXT:    shr.u32 %r7, %r6, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_sitofp_i16(
@@ -986,8 +962,8 @@ define bfloat @test_uitofp_i8(i8 %a) {
 ; SM70-LABEL: test_uitofp_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<3>;
-; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<8>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [test_uitofp_i8_param_0];
@@ -998,8 +974,8 @@ define bfloat @test_uitofp_i8(i8 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r1, %r1;
 ; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
 ; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs2}, %r6;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs2;
+; SM70-NEXT:    shr.u32 %r7, %r6, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_uitofp_i8(
@@ -1043,8 +1019,8 @@ define bfloat @test_uitofp_i1(i1 %a) {
 ; SM70-LABEL: test_uitofp_i1(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
-; SM70-NEXT:    .reg .b16 %rs<4>;
-; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [test_uitofp_i1_param_0];
@@ -1058,8 +1034,8 @@ define bfloat @test_uitofp_i1(i1 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p2, %r2, %r2;
 ; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
 ; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p2;
-; SM70-NEXT:    mov.b32 {_, %rs3}, %r7;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs3;
+; SM70-NEXT:    shr.u32 %r8, %r7, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r8;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_uitofp_i1(
@@ -1116,8 +1092,8 @@ define bfloat @test_uitofp_i16(i16 %a) {
 ; SM70-LABEL: test_uitofp_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<3>;
-; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<8>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [test_uitofp_i16_param_0];
@@ -1128,8 +1104,8 @@ define bfloat @test_uitofp_i16(i16 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r1, %r1;
 ; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
 ; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs2}, %r6;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs2;
+; SM70-NEXT:    shr.u32 %r7, %r6, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_uitofp_i16(
@@ -1173,8 +1149,7 @@ define bfloat @test_uitofp_i32(i32 %a) {
 ; SM70-LABEL: test_uitofp_i32(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .b32 %r<9>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b32 %r1, [test_uitofp_i32_param_0];
@@ -1185,8 +1160,8 @@ define bfloat @test_uitofp_i32(i32 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r2, %r2;
 ; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
 ; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r7;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r8, %r7, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r8;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_uitofp_i32(
@@ -1231,8 +1206,7 @@ define bfloat @test_uitofp_i64(i64 %a) {
 ; SM70-LABEL: test_uitofp_i64(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .b32 %r<8>;
 ; SM70-NEXT:    .reg .b64 %rd<2>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
@@ -1244,8 +1218,8 @@ define bfloat @test_uitofp_i64(i64 %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r1, %r1;
 ; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
 ; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r6;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r7, %r6, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r7;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_uitofp_i64(
@@ -1292,8 +1266,7 @@ define bfloat @test_roundeven(bfloat %a) {
 ; SM70-LABEL: test_roundeven(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .b32 %r<10>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %r1, [test_roundeven_param_0];
@@ -1305,8 +1278,8 @@ define bfloat @test_roundeven(bfloat %a) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r3, %r3;
 ; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
 ; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r8;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r9, %r8, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r9;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_roundeven(
@@ -1417,8 +1390,7 @@ define bfloat @test_maxnum(bfloat %a, bfloat %b) {
 ; SM70-LABEL: test_maxnum(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<2>;
-; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .b32 %r<12>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %r1, [test_maxnum_param_1];
@@ -1432,8 +1404,8 @@ define bfloat @test_maxnum(bfloat %a, bfloat %b) {
 ; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
 ; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
 ; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
-; SM70-NEXT:    mov.b32 {_, %rs1}, %r10;
-; SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; SM70-NEXT:    shr.u32 %r11, %r10, 16;
+; SM70-NEXT:    st.param.b16 [func_retval0], %r11;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_maxnum(
@@ -1477,47 +1449,44 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<11>;
 ; SM70-NEXT:    .reg .b16 %rs<15>;
-; SM70-NEXT:    .reg .b32 %r<16>;
+; SM70-NEXT:    .reg .b32 %r<13>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_maximum_v2_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_maximum_v2_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    setp.gt.f32 %p1, %r6, %r4;
-; SM70-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
-; SM70-NEXT:    setp.nan.f32 %p2, %r6, %r4;
+; SM70-NEXT:    setp.gt.f32 %p1, %r4, %r2;
+; SM70-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %r4, %r2;
 ; SM70-NEXT:    selp.b16 %rs6, 0x7FC0, %rs5, %p2;
-; SM70-NEXT:    setp.eq.s16 %p3, %rs4, 0;
-; SM70-NEXT:    selp.b16 %rs7, %rs4, %rs6, %p3;
-; SM70-NEXT:    setp.eq.s16 %p4, %rs2, 0;
-; SM70-NEXT:    selp.b16 %rs8, %rs2, %rs7, %p4;
-; SM70-NEXT:    cvt.u32.u16 %r7, %rs6;
-; SM70-NEXT:    shl.b32 %r8, %r7, 16;
-; SM70-NEXT:    setp.eq.f32 %p5, %r8, 0f00000000;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs2, 0;
+; SM70-NEXT:    selp.b16 %rs7, %rs2, %rs6, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs4, 0;
+; SM70-NEXT:    selp.b16 %rs8, %rs4, %rs7, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs6;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    setp.eq.f32 %p5, %r6, 0f00000000;
 ; SM70-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p5;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs3;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r9, %rs1;
 ; SM70-NEXT:    shl.b32 %r10, %r9, 16;
-; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
-; SM70-NEXT:    shl.b32 %r12, %r11, 16;
-; SM70-NEXT:    setp.gt.f32 %p6, %r12, %r10;
-; SM70-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
-; SM70-NEXT:    setp.nan.f32 %p7, %r12, %r10;
+; SM70-NEXT:    setp.gt.f32 %p6, %r10, %r8;
+; SM70-NEXT:    selp.b16 %rs10, %rs1, %rs3, %p6;
+; SM70-NEXT:    setp.nan.f32 %p7, %r10, %r8;
 ; SM70-NEXT:    selp.b16 %rs11, 0x7FC0, %rs10, %p7;
-; SM70-NEXT:    setp.eq.s16 %p8, %rs3, 0;
-; SM70-NEXT:    selp.b16 %rs12, %rs3, %rs11, %p8;
-; SM70-NEXT:    setp.eq.s16 %p9, %rs1, 0;
-; SM70-NEXT:    selp.b16 %rs13, %rs1, %rs12, %p9;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs11;
-; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    setp.eq.f32 %p10, %r14, 0f00000000;
+; SM70-NEXT:    setp.eq.s16 %p8, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs12, %rs1, %rs11, %p8;
+; SM70-NEXT:    setp.eq.s16 %p9, %rs3, 0;
+; SM70-NEXT:    selp.b16 %rs13, %rs3, %rs12, %p9;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs11;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    setp.eq.f32 %p10, %r12, 0f00000000;
 ; SM70-NEXT:    selp.b16 %rs14, %rs13, %rs11, %p10;
-; SM70-NEXT:    mov.b32 %r15, {%rs14, %rs9};
-; SM70-NEXT:    st.param.b32 [func_retval0], %r15;
+; SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs14, %rs9};
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_maximum_v2(
@@ -1525,9 +1494,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM80-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM80-NEXT:    max.NaN.bf16x2 %r3, %r1, %r2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
@@ -1536,9 +1505,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM80-FTZ-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM80-FTZ-NEXT:    max.NaN.bf16x2 %r3, %r1, %r2;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
@@ -1547,9 +1516,9 @@ define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
-; SM90-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM90-NEXT:    max.NaN.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.maximum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -1561,37 +1530,35 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<5>;
-; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .b32 %r<22>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
-; SM70-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
-; SM70-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_maxnum_v2_param_0];
+; SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_maxnum_v2_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
 ; SM70-NEXT:    shl.b32 %r4, %r3, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs4;
-; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    max.f32 %r7, %r6, %r4;
-; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
-; SM70-NEXT:    add.s32 %r9, %r8, %r7;
-; SM70-NEXT:    add.s32 %r10, %r9, 32767;
-; SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
-; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
-; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    max.f32 %r5, %r4, %r2;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %r5, %r5;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
 ; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
-; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    max.f32 %r17, %r16, %r14;
-; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
-; SM70-NEXT:    add.s32 %r19, %r18, %r17;
-; SM70-NEXT:    add.s32 %r20, %r19, 32767;
-; SM70-NEXT:    setp.nan.f32 %p2, %r17, %r17;
-; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
-; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
-; SM70-NEXT:    prmt.b32 %r23, %r22, %r12, 0x7632U;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r23;
+; SM70-NEXT:    max.f32 %r15, %r14, %r12;
+; SM70-NEXT:    bfe.u32 %r16, %r15, 16, 1;
+; SM70-NEXT:    add.s32 %r17, %r16, %r15;
+; SM70-NEXT:    add.s32 %r18, %r17, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %r15, %r15;
+; SM70-NEXT:    or.b32 %r19, %r15, 4194304;
+; SM70-NEXT:    selp.b32 %r20, %r19, %r18, %p2;
+; SM70-NEXT:    prmt.b32 %r21, %r20, %r10, 0x7632U;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r21;
 ; SM70-NEXT:    ret;
 ;
 ; SM80-LABEL: test_maxnum_v2(
@@ -1599,9 +1566,9 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM80-NEXT:    .reg .b32 %r<4>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM80-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM80-NEXT:    max.bf16x2 %r3, %r1, %r2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-NEXT:    ret;
 ;
@@ -1610,9 +1577,9 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM80-FTZ-NEXT:    .reg .b32 %r<4>;
 ; SM80-FTZ-EMPTY:
 ; SM80-FTZ-NEXT:  // %bb.0:
-; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM80-FTZ-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM80-FTZ-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM80-FTZ-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM80-FTZ-NEXT:    max.bf16x2 %r3, %r1, %r2;
 ; SM80-FTZ-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM80-FTZ-NEXT:    ret;
 ;
@@ -1621,14 +1588,32 @@ define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
-; SM90-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM90-NEXT:    max.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
   ret <2 x bfloat> %r
 }
 
+define void @store_bf16(ptr %p1, ptr %p2, bfloat %v) {
+; CHECK-LABEL: store_bf16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [store_bf16_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs1, [store_bf16_param_2];
+; CHECK-NEXT:    st.b16 [%rd1], %rs1;
+; CHECK-NEXT:    ld.param.b64 %rd2, [store_bf16_param_1];
+; CHECK-NEXT:    st.b16 [%rd2], 0x3F80;
+; CHECK-NEXT:    ret;
+  store bfloat %v, ptr %p1
+  store bfloat 1.0, ptr %p2
+  ret void
+}
+
 declare bfloat @llvm.maximum.bf16(bfloat, bfloat)
 declare <2 x bfloat> @llvm.maximum.v2bf16(<2 x bfloat>, <2 x bfloat>)
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 0e90b254225e..80627a03354a 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -11,17 +11,16 @@ define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
 ; CHECK-LABEL: test_sin(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_sin_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT:    sin.approx.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT:    sin.approx.f32 %r5, %r4;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_sin_param_0];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT:    sin.approx.f32 %r2, %r1;
+; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT:    sin.approx.f32 %r4, %r3;
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -31,17 +30,16 @@ define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 {
 ; CHECK-LABEL: test_cos(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_cos_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT:    cos.approx.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT:    cos.approx.f32 %r5, %r4;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_cos_param_0];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT:    cos.approx.f32 %r2, %r1;
+; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT:    cos.approx.f32 %r4, %r3;
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
index ec993aa15a85..ba5813c86923 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
@@ -9,11 +9,10 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define <2 x bfloat> @test_ret_const() #0 {
 ; CHECK-LABEL: test_ret_const(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 1073758080;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1073758080;
 ; CHECK-NEXT:    ret;
   ret <2 x bfloat> <bfloat 1.0, bfloat 2.0>
 }
@@ -79,9 +78,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
-; SM80-NEXT:    mov.b32 %r3, -1082081408;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r3, %r1;
+; SM80-NEXT:    mov.b32 %r2, -1082081408;
+; SM80-NEXT:    ld.param.b32 %r3, [test_fsubx2_param_1];
+; SM80-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
 ; SM80-NEXT:    ret;
 ;
@@ -90,9 +89,9 @@ define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_0];
-; SM90-NEXT:    sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM90-NEXT:    sub.rn.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fsub <2 x bfloat> %a, %b
@@ -105,10 +104,10 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    .reg .b32 %r<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM80-NEXT:    mov.b32 %r3, -2147450880;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r1, %r3;
+; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT:    mov.b32 %r2, -2147450880;
+; SM80-NEXT:    ld.param.b32 %r3, [test_fmulx2_param_1];
+; SM80-NEXT:    fma.rn.bf16x2 %r4, %r1, %r3, %r2;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r4;
 ; SM80-NEXT:    ret;
 ;
@@ -117,9 +116,9 @@ define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<4>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
-; SM90-NEXT:    mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM90-NEXT:    mul.rn.bf16x2 %r3, %r1, %r2;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r3;
 ; SM90-NEXT:    ret;
   %r = fmul <2 x bfloat> %a, %b
@@ -130,21 +129,19 @@ define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_fdiv(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs1;
-; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs3;
-; CHECK-NEXT:    div.rn.f32 %r5, %r4, %r3;
-; CHECK-NEXT:    cvt.f32.bf16 %r6, %rs2;
-; CHECK-NEXT:    cvt.f32.bf16 %r7, %rs4;
-; CHECK-NEXT:    div.rn.f32 %r8, %r7, %r6;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r9, %r8, %r5;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fdiv_param_0];
+; CHECK-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_fdiv_param_1];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs3;
+; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs4;
+; CHECK-NEXT:    cvt.f32.bf16 %r5, %rs2;
+; CHECK-NEXT:    div.rn.f32 %r6, %r5, %r4;
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r7, %r6, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
@@ -157,7 +154,7 @@ define <2 x bfloat> @test_fneg(<2 x bfloat> %a) #0 {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_fneg_param_0];
-; CHECK-NEXT:    xor.b32 %r2, %r1, -2147450880;
+; CHECK-NEXT:    neg.bf16x2 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %r = fneg <2 x bfloat> %a
@@ -218,12 +215,7 @@ define <2 x bfloat> @test_call(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r2;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -243,9 +235,9 @@ define <2 x bfloat> @test_select(<2 x bfloat> %a, <2 x bfloat> %b, i1 zeroext %c
 ; CHECK-NEXT:    ld.param.b8 %rs1, [test_select_param_2];
 ; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
 ; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT:    ld.param.b32 %r1, [test_select_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_select_param_0];
-; CHECK-NEXT:    selp.b32 %r3, %r2, %r1, %p1;
+; CHECK-NEXT:    ld.param.b32 %r1, [test_select_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [test_select_param_1];
+; CHECK-NEXT:    selp.b32 %r3, %r1, %r2, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %r = select i1 %c, <2 x bfloat> %a, <2 x bfloat> %b
@@ -257,47 +249,39 @@ define <2 x bfloat> @test_select_cc(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloa
 ; SM80:       {
 ; SM80-NEXT:    .reg .pred %p<3>;
 ; SM80-NEXT:    .reg .b16 %rs<11>;
-; SM80-NEXT:    .reg .b32 %r<10>;
+; SM80-NEXT:    .reg .b32 %r<5>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_select_cc_param_0];
-; SM80-NEXT:    ld.param.b32 %r2, [test_select_cc_param_1];
-; SM80-NEXT:    ld.param.b32 %r3, [test_select_cc_param_2];
-; SM80-NEXT:    ld.param.b32 %r4, [test_select_cc_param_3];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; SM80-NEXT:    cvt.f32.bf16 %r5, %rs1;
-; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
-; SM80-NEXT:    cvt.f32.bf16 %r6, %rs3;
-; SM80-NEXT:    setp.neu.f32 %p1, %r6, %r5;
-; SM80-NEXT:    cvt.f32.bf16 %r7, %rs2;
-; SM80-NEXT:    cvt.f32.bf16 %r8, %rs4;
-; SM80-NEXT:    setp.neu.f32 %p2, %r8, %r7;
-; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
-; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
-; SM80-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p2;
-; SM80-NEXT:    selp.b16 %rs10, %rs7, %rs5, %p1;
-; SM80-NEXT:    mov.b32 %r9, {%rs10, %rs9};
-; SM80-NEXT:    st.param.b32 [func_retval0], %r9;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
+; SM80-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_3];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs3;
+; SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [test_select_cc_param_2];
+; SM80-NEXT:    cvt.f32.bf16 %r2, %rs5;
+; SM80-NEXT:    setp.neu.f32 %p1, %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %r4, %rs6;
+; SM80-NEXT:    setp.neu.f32 %p2, %r4, %r3;
+; SM80-NEXT:    ld.param.v2.b16 {%rs7, %rs8}, [test_select_cc_param_1];
+; SM80-NEXT:    selp.b16 %rs9, %rs2, %rs8, %p2;
+; SM80-NEXT:    selp.b16 %rs10, %rs1, %rs7, %p1;
+; SM80-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_select_cc(
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<7>;
-; SM90-NEXT:    .reg .b32 %r<6>;
+; SM90-NEXT:    .reg .b32 %r<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_select_cc_param_0];
-; SM90-NEXT:    ld.param.b32 %r2, [test_select_cc_param_1];
-; SM90-NEXT:    ld.param.b32 %r3, [test_select_cc_param_3];
-; SM90-NEXT:    ld.param.b32 %r4, [test_select_cc_param_2];
-; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r4, %r3;
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
-; SM90-NEXT:    selp.b16 %rs6, %rs3, %rs1, %p1;
-; SM90-NEXT:    mov.b32 %r5, {%rs6, %rs5};
-; SM90-NEXT:    st.param.b32 [func_retval0], %r5;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_param_0];
+; SM90-NEXT:    ld.param.b32 %r1, [test_select_cc_param_2];
+; SM90-NEXT:    ld.param.b32 %r2, [test_select_cc_param_3];
+; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r1, %r2;
+; SM90-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_param_1];
+; SM90-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p2;
+; SM90-NEXT:    selp.b16 %rs6, %rs1, %rs3, %p1;
+; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; SM90-NEXT:    ret;
   %cc = fcmp une <2 x bfloat> %c, %d
   %r = select <2 x i1> %cc, <2 x bfloat> %a, <2 x bfloat> %b
@@ -309,24 +293,22 @@ define <2 x float> @test_select_cc_f32_bf16(<2 x float> %a, <2 x float> %b,
 ; SM80:       {
 ; SM80-NEXT:    .reg .pred %p<3>;
 ; SM80-NEXT:    .reg .b16 %rs<5>;
-; SM80-NEXT:    .reg .b32 %r<13>;
+; SM80-NEXT:    .reg .b32 %r<11>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
-; SM80-NEXT:    ld.param.b32 %r3, [test_select_cc_f32_bf16_param_2];
-; SM80-NEXT:    ld.param.b32 %r4, [test_select_cc_f32_bf16_param_3];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
-; SM80-NEXT:    cvt.f32.bf16 %r5, %rs1;
-; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
-; SM80-NEXT:    cvt.f32.bf16 %r6, %rs3;
-; SM80-NEXT:    setp.neu.f32 %p1, %r6, %r5;
-; SM80-NEXT:    cvt.f32.bf16 %r7, %rs2;
-; SM80-NEXT:    cvt.f32.bf16 %r8, %rs4;
-; SM80-NEXT:    setp.neu.f32 %p2, %r8, %r7;
-; SM80-NEXT:    ld.param.v2.b32 {%r9, %r10}, [test_select_cc_f32_bf16_param_1];
-; SM80-NEXT:    selp.f32 %r11, %r2, %r10, %p2;
-; SM80-NEXT:    selp.f32 %r12, %r1, %r9, %p1;
-; SM80-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r11};
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_f32_bf16_param_3];
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs1;
+; SM80-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_f32_bf16_param_2];
+; SM80-NEXT:    cvt.f32.bf16 %r4, %rs3;
+; SM80-NEXT:    setp.neu.f32 %p1, %r4, %r3;
+; SM80-NEXT:    cvt.f32.bf16 %r5, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %r6, %rs4;
+; SM80-NEXT:    setp.neu.f32 %p2, %r6, %r5;
+; SM80-NEXT:    ld.param.v2.b32 {%r7, %r8}, [test_select_cc_f32_bf16_param_1];
+; SM80-NEXT:    selp.f32 %r9, %r2, %r8, %p2;
+; SM80-NEXT:    selp.f32 %r10, %r1, %r7, %p1;
+; SM80-NEXT:    st.param.v2.b32 [func_retval0], {%r10, %r9};
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_select_cc_f32_bf16(
@@ -336,9 +318,9 @@ define <2 x float> @test_select_cc_f32_bf16(<2 x float> %a, <2 x float> %b,
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_bf16_param_0];
-; SM90-NEXT:    ld.param.b32 %r3, [test_select_cc_f32_bf16_param_3];
-; SM90-NEXT:    ld.param.b32 %r4, [test_select_cc_f32_bf16_param_2];
-; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r4, %r3;
+; SM90-NEXT:    ld.param.b32 %r3, [test_select_cc_f32_bf16_param_2];
+; SM90-NEXT:    ld.param.b32 %r4, [test_select_cc_f32_bf16_param_3];
+; SM90-NEXT:    setp.neu.bf16x2 %p1|%p2, %r3, %r4;
 ; SM90-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f32_bf16_param_1];
 ; SM90-NEXT:    selp.f32 %r7, %r2, %r6, %p2;
 ; SM90-NEXT:    selp.f32 %r8, %r1, %r5, %p1;
@@ -355,21 +337,18 @@ define <2 x bfloat> @test_select_cc_bf16_f32(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_select_cc_bf16_f32_param_0];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_select_cc_bf16_f32_param_1];
-; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_bf16_f32_param_2];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_bf16_f32_param_3];
-; CHECK-NEXT:    setp.neu.f32 %p1, %r3, %r5;
-; CHECK-NEXT:    setp.neu.f32 %p2, %r4, %r6;
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
-; CHECK-NEXT:    selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-NEXT:    mov.b32 %r7, {%rs6, %rs5};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_select_cc_bf16_f32_param_0];
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_bf16_f32_param_2];
+; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_bf16_f32_param_3];
+; CHECK-NEXT:    setp.neu.f32 %p1, %r1, %r3;
+; CHECK-NEXT:    setp.neu.f32 %p2, %r2, %r4;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [test_select_cc_bf16_f32_param_1];
+; CHECK-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p2;
+; CHECK-NEXT:    selp.b16 %rs6, %rs1, %rs3, %p1;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; CHECK-NEXT:    ret;
                                           <2 x float> %c, <2 x float> %d) #0 {
   %cc = fcmp une <2 x float> %c, %d
@@ -395,14 +374,13 @@ define <2 x float> @test_fpext_2xfloat(<2 x bfloat> %a) #0 {
 ; CHECK-LABEL: test_fpext_2xfloat(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_fpext_2xfloat_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs2;
-; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r2};
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_fpext_2xfloat_param_0];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
 ; CHECK-NEXT:    ret;
   %r = fpext <2 x bfloat> %a to <2 x float>
   ret <2 x float> %r
@@ -461,17 +439,16 @@ define <2 x bfloat> @test_sqrt(<2 x bfloat> %a) #0 {
 ; CHECK-LABEL: test_sqrt(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_sqrt_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT:    sqrt.rn.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; CHECK-NEXT:    sqrt.rn.f32 %r5, %r4;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_sqrt_param_0];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT:    sqrt.rn.f32 %r2, %r1;
+; CHECK-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; CHECK-NEXT:    sqrt.rn.f32 %r4, %r3;
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.sqrt.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -483,10 +460,10 @@ define <2 x bfloat> @test_fmuladd(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_fmuladd_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_fmuladd_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_fmuladd_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [test_fmuladd_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [test_fmuladd_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.fmuladd.f16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
@@ -500,7 +477,7 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_fabs_param_0];
-; CHECK-NEXT:    and.b32 %r2, %r1, 2147450879;
+; CHECK-NEXT:    abs.bf16x2 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %a)
@@ -513,12 +490,12 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM80-NEXT:    .reg .b32 %r<7>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_1];
-; SM80-NEXT:    ld.param.b32 %r2, [test_fabs_add_param_0];
-; SM80-NEXT:    mov.b32 %r3, 1065369472;
-; SM80-NEXT:    fma.rn.bf16x2 %r4, %r2, %r3, %r2;
-; SM80-NEXT:    abs.bf16x2 %r5, %r4;
-; SM80-NEXT:    fma.rn.bf16x2 %r6, %r5, %r3, %r1;
+; SM80-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_0];
+; SM80-NEXT:    mov.b32 %r2, 1065369472;
+; SM80-NEXT:    fma.rn.bf16x2 %r3, %r1, %r2, %r1;
+; SM80-NEXT:    ld.param.b32 %r4, [test_fabs_add_param_1];
+; SM80-NEXT:    abs.bf16x2 %r5, %r3;
+; SM80-NEXT:    fma.rn.bf16x2 %r6, %r5, %r2, %r4;
 ; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
 ; SM80-NEXT:    ret;
 ;
@@ -527,11 +504,11 @@ define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; SM90-NEXT:    .reg .b32 %r<6>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_1];
-; SM90-NEXT:    ld.param.b32 %r2, [test_fabs_add_param_0];
-; SM90-NEXT:    add.rn.bf16x2 %r3, %r2, %r2;
-; SM90-NEXT:    abs.bf16x2 %r4, %r3;
-; SM90-NEXT:    add.rn.bf16x2 %r5, %r4, %r1;
+; SM90-NEXT:    ld.param.b32 %r1, [test_fabs_add_param_0];
+; SM90-NEXT:    add.rn.bf16x2 %r2, %r1, %r1;
+; SM90-NEXT:    ld.param.b32 %r3, [test_fabs_add_param_1];
+; SM90-NEXT:    abs.bf16x2 %r4, %r2;
+; SM90-NEXT:    add.rn.bf16x2 %r5, %r4, %r3;
 ; SM90-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM90-NEXT:    ret;
   %s = fadd <2 x bfloat> %a, %a
@@ -546,9 +523,9 @@ define <2 x bfloat> @test_minnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_minnum_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_minnum_param_0];
-; CHECK-NEXT:    min.bf16x2 %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r1, [test_minnum_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [test_minnum_param_1];
+; CHECK-NEXT:    min.bf16x2 %r3, %r1, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.minnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -561,9 +538,9 @@ define <2 x bfloat> @test_maxnum(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_maxnum_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_maxnum_param_0];
-; CHECK-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r1, [test_maxnum_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [test_maxnum_param_1];
+; CHECK-NEXT:    max.bf16x2 %r3, %r1, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.maxnum.f16(<2 x bfloat> %a, <2 x bfloat> %b)
@@ -574,31 +551,27 @@ define <2 x bfloat> @test_floor(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_floor(
 ; SM80:       {
 ; SM80-NEXT:    .reg .b16 %rs<3>;
-; SM80-NEXT:    .reg .b32 %r<7>;
+; SM80-NEXT:    .reg .b32 %r<6>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_floor_param_0];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT:    cvt.rmi.f32.f32 %r3, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT:    cvt.rmi.f32.f32 %r5, %r4;
-; SM80-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT:    cvt.rmi.f32.f32 %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT:    cvt.rmi.f32.f32 %r4, %r3;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_floor(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<5>;
-; SM90-NEXT:    .reg .b32 %r<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_floor_param_0];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_floor_param_0];
 ; SM90-NEXT:    cvt.rmi.bf16.bf16 %rs3, %rs2;
 ; SM90-NEXT:    cvt.rmi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT:    st.param.b32 [func_retval0], %r2;
+; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.floor.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -608,31 +581,27 @@ define <2 x bfloat> @test_ceil(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_ceil(
 ; SM80:       {
 ; SM80-NEXT:    .reg .b16 %rs<3>;
-; SM80-NEXT:    .reg .b32 %r<7>;
+; SM80-NEXT:    .reg .b32 %r<6>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_ceil_param_0];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT:    cvt.rpi.f32.f32 %r3, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT:    cvt.rpi.f32.f32 %r5, %r4;
-; SM80-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT:    cvt.rpi.f32.f32 %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT:    cvt.rpi.f32.f32 %r4, %r3;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_ceil(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<5>;
-; SM90-NEXT:    .reg .b32 %r<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_ceil_param_0];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_ceil_param_0];
 ; SM90-NEXT:    cvt.rpi.bf16.bf16 %rs3, %rs2;
 ; SM90-NEXT:    cvt.rpi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT:    st.param.b32 [func_retval0], %r2;
+; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.ceil.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -642,31 +611,27 @@ define <2 x bfloat> @test_trunc(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_trunc(
 ; SM80:       {
 ; SM80-NEXT:    .reg .b16 %rs<3>;
-; SM80-NEXT:    .reg .b32 %r<7>;
+; SM80-NEXT:    .reg .b32 %r<6>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_trunc_param_0];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT:    cvt.rzi.f32.f32 %r5, %r4;
-; SM80-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT:    cvt.rzi.f32.f32 %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_trunc(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<5>;
-; SM90-NEXT:    .reg .b32 %r<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_trunc_param_0];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_trunc_param_0];
 ; SM90-NEXT:    cvt.rzi.bf16.bf16 %rs3, %rs2;
 ; SM90-NEXT:    cvt.rzi.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT:    st.param.b32 [func_retval0], %r2;
+; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.trunc.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -676,31 +641,27 @@ define <2 x bfloat> @test_rint(<2 x bfloat> %a) #0 {
 ; SM80-LABEL: test_rint(
 ; SM80:       {
 ; SM80-NEXT:    .reg .b16 %rs<3>;
-; SM80-NEXT:    .reg .b32 %r<7>;
+; SM80-NEXT:    .reg .b32 %r<6>;
 ; SM80-EMPTY:
 ; SM80-NEXT:  // %bb.0:
-; SM80-NEXT:    ld.param.b32 %r1, [test_rint_param_0];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; SM80-NEXT:    cvt.rni.f32.f32 %r3, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r4, %rs2;
-; SM80-NEXT:    cvt.rni.f32.f32 %r5, %r4;
-; SM80-NEXT:    cvt.rn.bf16x2.f32 %r6, %r5, %r3;
-; SM80-NEXT:    st.param.b32 [func_retval0], %r6;
+; SM80-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; SM80-NEXT:    cvt.rni.f32.f32 %r2, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; SM80-NEXT:    cvt.rni.f32.f32 %r4, %r3;
+; SM80-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
+; SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; SM80-NEXT:    ret;
 ;
 ; SM90-LABEL: test_rint(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<5>;
-; SM90-NEXT:    .reg .b32 %r<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
-; SM90-NEXT:    ld.param.b32 %r1, [test_rint_param_0];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_rint_param_0];
 ; SM90-NEXT:    cvt.rni.bf16.bf16 %rs3, %rs2;
 ; SM90-NEXT:    cvt.rni.bf16.bf16 %rs4, %rs1;
-; SM90-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; SM90-NEXT:    st.param.b32 [func_retval0], %r2;
+; SM90-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; SM90-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.rint.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -711,35 +672,34 @@ define <2 x bfloat> @test_round(<2 x bfloat> %a) #0 {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<21>;
+; CHECK-NEXT:    .reg .b32 %r<20>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_round_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NEXT:    and.b32 %r3, %r2, -2147483648;
-; CHECK-NEXT:    or.b32 %r4, %r3, 1056964608;
-; CHECK-NEXT:    add.rn.f32 %r5, %r2, %r4;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
-; CHECK-NEXT:    abs.f32 %r7, %r2;
-; CHECK-NEXT:    setp.gt.f32 %p1, %r7, 0f4B000000;
-; CHECK-NEXT:    selp.f32 %r8, %r2, %r6, %p1;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r9, %r2;
-; CHECK-NEXT:    setp.lt.f32 %p2, %r7, 0f3F000000;
-; CHECK-NEXT:    selp.f32 %r10, %r9, %r8, %p2;
-; CHECK-NEXT:    cvt.f32.bf16 %r11, %rs2;
-; CHECK-NEXT:    and.b32 %r12, %r11, -2147483648;
-; CHECK-NEXT:    or.b32 %r13, %r12, 1056964608;
-; CHECK-NEXT:    add.rn.f32 %r14, %r11, %r13;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r15, %r14;
-; CHECK-NEXT:    abs.f32 %r16, %r11;
-; CHECK-NEXT:    setp.gt.f32 %p3, %r16, 0f4B000000;
-; CHECK-NEXT:    selp.f32 %r17, %r11, %r15, %p3;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r18, %r11;
-; CHECK-NEXT:    setp.lt.f32 %p4, %r16, 0f3F000000;
-; CHECK-NEXT:    selp.f32 %r19, %r18, %r17, %p4;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r20, %r19, %r10;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r20;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_round_param_0];
+; CHECK-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NEXT:    and.b32 %r2, %r1, -2147483648;
+; CHECK-NEXT:    or.b32 %r3, %r2, 1056964608;
+; CHECK-NEXT:    add.rn.f32 %r4, %r1, %r3;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r5, %r4;
+; CHECK-NEXT:    abs.f32 %r6, %r1;
+; CHECK-NEXT:    setp.gt.f32 %p1, %r6, 0f4B000000;
+; CHECK-NEXT:    selp.f32 %r7, %r1, %r5, %p1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r8, %r1;
+; CHECK-NEXT:    setp.lt.f32 %p2, %r6, 0f3F000000;
+; CHECK-NEXT:    selp.f32 %r9, %r8, %r7, %p2;
+; CHECK-NEXT:    cvt.f32.bf16 %r10, %rs2;
+; CHECK-NEXT:    and.b32 %r11, %r10, -2147483648;
+; CHECK-NEXT:    or.b32 %r12, %r11, 1056964608;
+; CHECK-NEXT:    add.rn.f32 %r13, %r10, %r12;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r14, %r13;
+; CHECK-NEXT:    abs.f32 %r15, %r10;
+; CHECK-NEXT:    setp.gt.f32 %p3, %r15, 0f4B000000;
+; CHECK-NEXT:    selp.f32 %r16, %r10, %r14, %p3;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r17, %r10;
+; CHECK-NEXT:    setp.lt.f32 %p4, %r15, 0f3F000000;
+; CHECK-NEXT:    selp.f32 %r18, %r17, %r16, %p4;
+; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r19, %r18, %r9;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r19;
 ; CHECK-NEXT:    ret;
   %r = call <2 x bfloat> @llvm.round.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
@@ -762,3 +722,20 @@ define <2 x bfloat> @test_copysign(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
   ret <2 x bfloat> %r
 }
 
+define void @test_store_bf16x2(ptr %p1, ptr %p2, <2 x bfloat> %v) {
+; CHECK-LABEL: test_store_bf16x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_store_bf16x2_param_0];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_store_bf16x2_param_2];
+; CHECK-NEXT:    st.b32 [%rd1], %r1;
+; CHECK-NEXT:    ld.param.b64 %rd2, [test_store_bf16x2_param_1];
+; CHECK-NEXT:    st.b32 [%rd2], 1065369472;
+; CHECK-NEXT:    ret;
+  store <2 x bfloat> %v, ptr %p1
+  store <2 x bfloat> <bfloat 1.0, bfloat 1.0>, ptr %p2
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/byval-const-global.ll b/llvm/test/CodeGen/NVPTX/byval-const-global.ll
index 2af1e6d7e185..ad9e4b089e8d 100644
--- a/llvm/test/CodeGen/NVPTX/byval-const-global.ll
+++ b/llvm/test/CodeGen/NVPTX/byval-const-global.ll
@@ -19,11 +19,7 @@ define void @foo() {
 ; CHECK-NEXT:    .param .align 8 .b8 param0[16];
 ; CHECK-NEXT:    st.param.b64 [param0], %rd1;
 ; CHECK-NEXT:    st.param.b64 [param0+8], %rd2;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    bar,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni bar, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    ret;
   call void @bar(ptr byval(%struct) @G)
diff --git a/llvm/test/CodeGen/NVPTX/call-with-alloca-buffer.ll b/llvm/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
index a2175dd009f5..0cd7058174d6 100644
--- a/llvm/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
+++ b/llvm/test/CodeGen/NVPTX/call-with-alloca-buffer.ll
@@ -48,8 +48,7 @@ entry:
 ; CHECK-NEXT:   st.param.b64  [param0], %rd[[A_REG]]
 ; CHECK-NEXT:   .param .b64 param1;
 ; CHECK-NEXT:   st.param.b64  [param1], %rd[[SP_REG]]
-; CHECK-NEXT:   call.uni
-; CHECK-NEXT:   callee,
+; CHECK-NEXT:   call.uni callee,
 
   call void @callee(ptr %a, ptr %buf) #2
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/chain-different-as.ll b/llvm/test/CodeGen/NVPTX/chain-different-as.ll
index f2d0d9d069ea..a33d286b4738 100644
--- a/llvm/test/CodeGen/NVPTX/chain-different-as.ll
+++ b/llvm/test/CodeGen/NVPTX/chain-different-as.ll
@@ -4,14 +4,13 @@
 define i64 @test() nounwind readnone {
 ; CHECK-LABEL: test(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    mov.b64 %rd1, 1;
-; CHECK-NEXT:    mov.b64 %rd2, 42;
-; CHECK-NEXT:    st.b64 [%rd1], %rd2;
-; CHECK-NEXT:    ld.global.b64 %rd3, [%rd1];
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT:    st.b64 [%rd1], 42;
+; CHECK-NEXT:    ld.global.b64 %rd2, [%rd1];
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
 ; CHECK-NEXT:    ret;
   %addr0 = inttoptr i64 1 to ptr
   %addr1 = inttoptr i64 1 to ptr addrspace(1)
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
index 65a077d67e4b..c99860cc5cc1 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB0_1;
 ; SM60-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB1_1;
 ; SM60-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB2_1;
 ; SM60-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB3_1;
 ; SM60-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB4_1;
 ; SM60-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB5_1;
 ; SM60-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB6_1;
 ; SM60-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB7_1;
 ; SM60-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB8_1;
 ; SM60-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB9_1;
 ; SM60-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB10_1;
 ; SM60-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB11_1;
 ; SM60-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB12_1;
 ; SM60-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB13_1;
 ; SM60-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB14_1;
 ; SM60-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB15_1;
 ; SM60-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB16_1;
 ; SM60-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB17_1;
 ; SM60-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB18_1;
 ; SM60-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB19_1;
 ; SM60-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    mov.b32 %r20, %r8;
 ; SM60-NEXT:    @%p2 bra $L__BB20_1;
 ; SM60-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB21_1;
 ; SM60-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB22_1;
 ; SM60-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB23_1;
 ; SM60-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB24_1;
 ; SM60-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB25_1;
 ; SM60-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB26_1;
 ; SM60-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB27_1;
 ; SM60-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB28_1;
 ; SM60-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB29_1;
 ; SM60-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB30_1;
 ; SM60-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB31_1;
 ; SM60-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB32_1;
 ; SM60-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB33_1;
 ; SM60-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB34_1;
 ; SM60-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB35_1;
 ; SM60-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB36_1;
 ; SM60-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB37_1;
 ; SM60-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM60-NEXT:    @%p2 bra $L__BB38_1;
 ; SM60-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB39_1;
 ; SM60-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB40_1;
 ; SM60-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB41_1;
 ; SM60-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB42_1;
 ; SM60-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB43_1;
 ; SM60-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM60-NEXT:    membar.sys;
+; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM60-NEXT:    and.b32 %r10, %r9, 3;
-; SM60-NEXT:    shl.b32 %r1, %r10, 3;
-; SM60-NEXT:    mov.b32 %r11, 255;
-; SM60-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM60-NEXT:    not.b32 %r2, %r12;
-; SM60-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM60-NEXT:    and.b32 %r14, %r13, 255;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM60-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM60-NEXT:    and.b32 %r11, %r10, 3;
+; SM60-NEXT:    shl.b32 %r1, %r11, 3;
+; SM60-NEXT:    mov.b32 %r12, 255;
+; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM60-NEXT:    not.b32 %r2, %r13;
+; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM60-NEXT:    and.b32 %r15, %r14, 255;
+; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM60-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM60-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM60-NEXT:    @%p2 bra $L__BB44_1;
 ; SM60-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
index 7107fbcf6eb5..68de517f65bb 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB5_1;
 ; SM70-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB6_1;
 ; SM70-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB7_1;
 ; SM70-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB8_1;
 ; SM70-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB9_1;
 ; SM70-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB10_1;
 ; SM70-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB11_1;
 ; SM70-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB12_1;
 ; SM70-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB13_1;
 ; SM70-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB14_1;
 ; SM70-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB15_1;
 ; SM70-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB16_1;
 ; SM70-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB17_1;
 ; SM70-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB18_1;
 ; SM70-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB19_1;
 ; SM70-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB20_1;
 ; SM70-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB21_1;
 ; SM70-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB22_1;
 ; SM70-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB23_1;
 ; SM70-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB24_1;
 ; SM70-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB25_1;
 ; SM70-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB26_1;
 ; SM70-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB27_1;
 ; SM70-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB28_1;
 ; SM70-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB29_1;
 ; SM70-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB30_1;
 ; SM70-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB31_1;
 ; SM70-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB32_1;
 ; SM70-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB33_1;
 ; SM70-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB34_1;
 ; SM70-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB35_1;
 ; SM70-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB36_1;
 ; SM70-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB37_1;
 ; SM70-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM70-NEXT:    @%p2 bra $L__BB38_1;
 ; SM70-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB39_1;
 ; SM70-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB40_1;
 ; SM70-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB41_1;
 ; SM70-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB42_1;
 ; SM70-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB43_1;
 ; SM70-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB44_1;
 ; SM70-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
index f289c3cf3d50..e20f98857728 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
@@ -14,17 +14,17 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -41,7 +41,7 @@ define i8 @monotonic_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB0_1;
 ; SM90-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -59,17 +59,17 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -86,7 +86,7 @@ define i8 @monotonic_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB1_1;
 ; SM90-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -104,17 +104,17 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -131,7 +131,7 @@ define i8 @monotonic_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB2_1;
 ; SM90-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic monotonic
     ret i8 %new
@@ -149,17 +149,17 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -177,7 +177,7 @@ define i8 @monotonic_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB3_1;
 ; SM90-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -195,17 +195,17 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -223,7 +223,7 @@ define i8 @monotonic_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB4_1;
 ; SM90-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -241,17 +241,17 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
@@ -269,7 +269,7 @@ define i8 @monotonic_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB5_1;
 ; SM90-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic acquire
     ret i8 %new
@@ -287,18 +287,18 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
@@ -316,7 +316,7 @@ define i8 @monotonic_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB6_1;
 ; SM90-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -334,18 +334,18 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
@@ -363,7 +363,7 @@ define i8 @monotonic_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB7_1;
 ; SM90-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -381,18 +381,18 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [monotonic_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
@@ -410,7 +410,7 @@ define i8 @monotonic_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB8_1;
 ; SM90-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new monotonic seq_cst
     ret i8 %new
@@ -428,17 +428,17 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
@@ -456,7 +456,7 @@ define i8 @acquire_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB9_1;
 ; SM90-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -474,17 +474,17 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
@@ -502,7 +502,7 @@ define i8 @acquire_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB10_1;
 ; SM90-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -520,17 +520,17 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
@@ -548,7 +548,7 @@ define i8 @acquire_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB11_1;
 ; SM90-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire monotonic
     ret i8 %new
@@ -566,17 +566,17 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_generic_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_generic_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
@@ -594,7 +594,7 @@ define i8 @acquire_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB12_1;
 ; SM90-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -612,17 +612,17 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
@@ -640,7 +640,7 @@ define i8 @acquire_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB13_1;
 ; SM90-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -658,17 +658,17 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_shared_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_shared_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
@@ -686,7 +686,7 @@ define i8 @acquire_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB14_1;
 ; SM90-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire acquire
     ret i8 %new
@@ -704,18 +704,18 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
@@ -733,7 +733,7 @@ define i8 @acquire_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB15_1;
 ; SM90-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -751,18 +751,18 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
@@ -780,7 +780,7 @@ define i8 @acquire_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB16_1;
 ; SM90-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -798,18 +798,18 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acquire_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
@@ -827,7 +827,7 @@ define i8 @acquire_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB17_1;
 ; SM90-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acquire seq_cst
     ret i8 %new
@@ -845,18 +845,18 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
@@ -873,7 +873,7 @@ define i8 @release_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB18_1;
 ; SM90-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -891,18 +891,18 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
@@ -919,7 +919,7 @@ define i8 @release_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB19_1;
 ; SM90-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -937,18 +937,18 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
@@ -965,7 +965,7 @@ define i8 @release_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    mov.b32 %r20, %r8;
 ; SM90-NEXT:    @%p2 bra $L__BB20_1;
 ; SM90-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release monotonic
     ret i8 %new
@@ -983,18 +983,18 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
@@ -1012,7 +1012,7 @@ define i8 @release_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB21_1;
 ; SM90-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1030,18 +1030,18 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
@@ -1059,7 +1059,7 @@ define i8 @release_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB22_1;
 ; SM90-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1077,18 +1077,18 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
@@ -1106,7 +1106,7 @@ define i8 @release_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB23_1;
 ; SM90-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release acquire
     ret i8 %new
@@ -1124,18 +1124,18 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
@@ -1153,7 +1153,7 @@ define i8 @release_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB24_1;
 ; SM90-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1171,18 +1171,18 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
@@ -1200,7 +1200,7 @@ define i8 @release_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB25_1;
 ; SM90-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1218,18 +1218,18 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [release_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
@@ -1247,7 +1247,7 @@ define i8 @release_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB26_1;
 ; SM90-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new release seq_cst
     ret i8 %new
@@ -1265,18 +1265,18 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
@@ -1294,7 +1294,7 @@ define i8 @acq_rel_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB27_1;
 ; SM90-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1312,18 +1312,18 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
@@ -1341,7 +1341,7 @@ define i8 @acq_rel_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB28_1;
 ; SM90-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1359,18 +1359,18 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
@@ -1388,7 +1388,7 @@ define i8 @acq_rel_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB29_1;
 ; SM90-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel monotonic
     ret i8 %new
@@ -1406,18 +1406,18 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB30_1: // %partword.cmpxchg.loop
@@ -1435,7 +1435,7 @@ define i8 @acq_rel_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB30_1;
 ; SM90-NEXT:  $L__BB30_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1453,18 +1453,18 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB31_1: // %partword.cmpxchg.loop
@@ -1482,7 +1482,7 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB31_1;
 ; SM90-NEXT:  $L__BB31_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1500,18 +1500,18 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.release.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB32_1: // %partword.cmpxchg.loop
@@ -1529,7 +1529,7 @@ define i8 @acq_rel_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB32_1;
 ; SM90-NEXT:  $L__BB32_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1547,18 +1547,18 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB33_1: // %partword.cmpxchg.loop
@@ -1576,7 +1576,7 @@ define i8 @acq_rel_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB33_1;
 ; SM90-NEXT:  $L__BB33_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1594,18 +1594,18 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB34_1: // %partword.cmpxchg.loop
@@ -1623,7 +1623,7 @@ define i8 @acq_rel_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB34_1;
 ; SM90-NEXT:  $L__BB34_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1641,18 +1641,18 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [acq_rel_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB35_1: // %partword.cmpxchg.loop
@@ -1670,7 +1670,7 @@ define i8 @acq_rel_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB35_1;
 ; SM90-NEXT:  $L__BB35_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new acq_rel seq_cst
     ret i8 %new
@@ -1688,18 +1688,18 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB36_1: // %partword.cmpxchg.loop
@@ -1717,7 +1717,7 @@ define i8 @seq_cst_monotonic_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB36_1;
 ; SM90-NEXT:  $L__BB36_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1735,18 +1735,18 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB37_1: // %partword.cmpxchg.loop
@@ -1764,7 +1764,7 @@ define i8 @seq_cst_monotonic_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB37_1;
 ; SM90-NEXT:  $L__BB37_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1782,18 +1782,18 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_monotonic_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB38_1: // %partword.cmpxchg.loop
@@ -1811,7 +1811,7 @@ define i8 @seq_cst_monotonic_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new)
 ; SM90-NEXT:    @%p2 bra $L__BB38_1;
 ; SM90-NEXT:  $L__BB38_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst monotonic
     ret i8 %new
@@ -1829,18 +1829,18 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB39_1: // %partword.cmpxchg.loop
@@ -1858,7 +1858,7 @@ define i8 @seq_cst_acquire_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB39_1;
 ; SM90-NEXT:  $L__BB39_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1876,18 +1876,18 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB40_1: // %partword.cmpxchg.loop
@@ -1905,7 +1905,7 @@ define i8 @seq_cst_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB40_1;
 ; SM90-NEXT:  $L__BB40_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1923,18 +1923,18 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_acquire_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB41_1: // %partword.cmpxchg.loop
@@ -1952,7 +1952,7 @@ define i8 @seq_cst_acquire_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB41_1;
 ; SM90-NEXT:  $L__BB41_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst acquire
     ret i8 %new
@@ -1970,18 +1970,18 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_generic_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_generic_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_generic_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_generic_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB42_1: // %partword.cmpxchg.loop
@@ -1999,7 +1999,7 @@ define i8 @seq_cst_seq_cst_i8_generic(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB42_1;
 ; SM90-NEXT:  $L__BB42_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2017,18 +2017,18 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_global_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB43_1: // %partword.cmpxchg.loop
@@ -2046,7 +2046,7 @@ define i8 @seq_cst_seq_cst_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB43_1;
 ; SM90-NEXT:  $L__BB43_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
@@ -2064,18 +2064,18 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_shared_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_shared_param_0];
 ; SM90-NEXT:    fence.sc.sys;
+; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_shared_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM90-NEXT:    and.b32 %r10, %r9, 3;
-; SM90-NEXT:    shl.b32 %r1, %r10, 3;
-; SM90-NEXT:    mov.b32 %r11, 255;
-; SM90-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM90-NEXT:    not.b32 %r2, %r12;
-; SM90-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM90-NEXT:    and.b32 %r14, %r13, 255;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    ld.param.b8 %r15, [seq_cst_seq_cst_i8_shared_param_1];
-; SM90-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM90-NEXT:    and.b32 %r11, %r10, 3;
+; SM90-NEXT:    shl.b32 %r1, %r11, 3;
+; SM90-NEXT:    mov.b32 %r12, 255;
+; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM90-NEXT:    not.b32 %r2, %r13;
+; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM90-NEXT:    and.b32 %r15, %r14, 255;
+; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
 ; SM90-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM90-NEXT:  $L__BB44_1: // %partword.cmpxchg.loop
@@ -2093,7 +2093,7 @@ define i8 @seq_cst_seq_cst_i8_shared(ptr addrspace(3) %addr, i8 %cmp, i8 %new) {
 ; SM90-NEXT:    @%p2 bra $L__BB44_1;
 ; SM90-NEXT:  $L__BB44_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new seq_cst seq_cst
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg.ll b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
index 9eeff9d7c2b7..85414a2ab04e 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
@@ -21,17 +21,17 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [relaxed_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -48,7 +48,7 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    mov.b32 %r20, %r8;
 ; SM30-NEXT:    @%p2 bra $L__BB0_1;
 ; SM30-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: relaxed_sys_i8(
@@ -62,17 +62,17 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [relaxed_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
@@ -89,7 +89,7 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: relaxed_sys_i8(
 ; SM90:       {
@@ -147,17 +147,17 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [acquire_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -175,7 +175,7 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB1_1;
 ; SM30-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acquire_sys_i8(
@@ -189,17 +189,17 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acquire_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
@@ -217,7 +217,7 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acquire_sys_i8(
 ; SM90:       {
@@ -276,18 +276,18 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [release_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -304,7 +304,7 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    mov.b32 %r20, %r8;
 ; SM30-NEXT:    @%p2 bra $L__BB2_1;
 ; SM30-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: release_sys_i8(
@@ -318,18 +318,18 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [release_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
@@ -346,7 +346,7 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    mov.b32 %r20, %r8;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: release_sys_i8(
 ; SM90:       {
@@ -405,18 +405,18 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [acq_rel_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -434,7 +434,7 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB3_1;
 ; SM30-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acq_rel_sys_i8(
@@ -448,18 +448,18 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [acq_rel_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
@@ -477,7 +477,7 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acq_rel_sys_i8(
 ; SM90:       {
@@ -537,18 +537,18 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
+; SM30-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM30-NEXT:    and.b32 %r10, %r9, 3;
-; SM30-NEXT:    shl.b32 %r1, %r10, 3;
-; SM30-NEXT:    mov.b32 %r11, 255;
-; SM30-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM30-NEXT:    not.b32 %r2, %r12;
-; SM30-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM30-NEXT:    and.b32 %r14, %r13, 255;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    ld.param.b8 %r15, [seq_cst_sys_i8_param_1];
-; SM30-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM30-NEXT:    and.b32 %r11, %r10, 3;
+; SM30-NEXT:    shl.b32 %r1, %r11, 3;
+; SM30-NEXT:    mov.b32 %r12, 255;
+; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM30-NEXT:    not.b32 %r2, %r13;
+; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM30-NEXT:    and.b32 %r15, %r14, 255;
+; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM30-NEXT:    ld.b32 %r16, [%rd1];
 ; SM30-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM30-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -566,7 +566,7 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30-NEXT:    @%p2 bra $L__BB4_1;
 ; SM30-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: seq_cst_sys_i8(
@@ -580,18 +580,18 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM70-NEXT:    fence.sc.sys;
+; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r9, %rd2;
-; SM70-NEXT:    and.b32 %r10, %r9, 3;
-; SM70-NEXT:    shl.b32 %r1, %r10, 3;
-; SM70-NEXT:    mov.b32 %r11, 255;
-; SM70-NEXT:    shl.b32 %r12, %r11, %r1;
-; SM70-NEXT:    not.b32 %r2, %r12;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
-; SM70-NEXT:    and.b32 %r14, %r13, 255;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    ld.param.b8 %r15, [seq_cst_sys_i8_param_1];
-; SM70-NEXT:    shl.b32 %r4, %r15, %r1;
+; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
+; SM70-NEXT:    and.b32 %r11, %r10, 3;
+; SM70-NEXT:    shl.b32 %r1, %r11, 3;
+; SM70-NEXT:    mov.b32 %r12, 255;
+; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
+; SM70-NEXT:    not.b32 %r2, %r13;
+; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
+; SM70-NEXT:    and.b32 %r15, %r14, 255;
+; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
 ; SM70-NEXT:    ld.b32 %r16, [%rd1];
 ; SM70-NEXT:    and.b32 %r20, %r16, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
@@ -609,7 +609,7 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r13;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: seq_cst_sys_i8(
 ; SM90:       {
diff --git a/llvm/test/CodeGen/NVPTX/combine-mad.ll b/llvm/test/CodeGen/NVPTX/combine-mad.ll
index dc6d504c2c66..2232810d0212 100644
--- a/llvm/test/CodeGen/NVPTX/combine-mad.ll
+++ b/llvm/test/CodeGen/NVPTX/combine-mad.ll
@@ -203,12 +203,7 @@ define i32 @test_mad_multi_use(i32 %a, i32 %b, i32 %c) {
 ; CHECK-NEXT:    .param .b32 param1;
 ; CHECK-NEXT:    st.param.b32 [param1], %r5;
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    use,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), use, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r6, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
diff --git a/llvm/test/CodeGen/NVPTX/compute-ptx-value-vts.ll b/llvm/test/CodeGen/NVPTX/compute-ptx-value-vts.ll
index aa08b9605790..c5a995ae47bc 100644
--- a/llvm/test/CodeGen/NVPTX/compute-ptx-value-vts.ll
+++ b/llvm/test/CodeGen/NVPTX/compute-ptx-value-vts.ll
@@ -6,12 +6,11 @@ target triple = "nvptx-nvidia-cuda"
 define <6 x half> @half6() {
 ; CHECK-LABEL: half6(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0x0000;
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b16 [func_retval0+8], {%rs1, %rs1};
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <6 x half> zeroinitializer
 }
@@ -19,13 +18,12 @@ define <6 x half> @half6() {
 define <10 x half> @half10() {
 ; CHECK-LABEL: half10(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0x0000;
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b16 [func_retval0+16], {%rs1, %rs1};
+; CHECK-NEXT:    st.param.b32 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0+8], {0, 0};
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <10 x half> zeroinitializer
 }
@@ -33,13 +31,11 @@ define <10 x half> @half10() {
 define <12 x i8> @byte12() {
 ; CHECK-LABEL: byte12(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <12 x i8> zeroinitializer
 }
@@ -47,15 +43,12 @@ define <12 x i8> @byte12() {
 define <20 x i8> @byte20() {
 ; CHECK-LABEL: byte20(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+12], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+16], {%rs1, %rs1, %rs1, %rs1};
+; CHECK-NEXT:    st.param.b32 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <20 x i8> zeroinitializer
 }
diff --git a/llvm/test/CodeGen/NVPTX/convergent-mir-call.ll b/llvm/test/CodeGen/NVPTX/convergent-mir-call.ll
index 5e85bf455454..39a2d7f9e150 100644
--- a/llvm/test/CodeGen/NVPTX/convergent-mir-call.ll
+++ b/llvm/test/CodeGen/NVPTX/convergent-mir-call.ll
@@ -9,18 +9,16 @@ declare void @conv() convergent
 declare void @not_conv()
 
 define void @test(ptr %f) {
-  ; CHECK: ConvergentCallUniPrintCall
-  ; CHECK-NEXT: @conv
+  ; CHECK: CALL_UNI_conv @conv
   call void @conv()
 
-  ; CHECK: CallUniPrintCall
-  ; CHECK-NEXT: @not_conv
+  ; CHECK: CALL_UNI @not_conv
   call void @not_conv()
 
-  ; CHECK: ConvergentCallPrintCall
+  ; CHECK: CALL_conv %{{[0-9]+}}
   call void %f() convergent
 
-  ; CHECK: CallPrintCall
+  ; CHECK: CALL %{{[0-9]+}}
   call void %f()
 
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
index 71a46fa6d482..d1b478d34191 100644
--- a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
@@ -9,12 +9,7 @@ define %struct.64 @test_return_type_mismatch(ptr %p) {
 ; CHECK-LABEL: test_return_type_mismatch(
 ; CHECK:         .param .align 1 .b8 retval0[8];
 ; CHECK-NEXT:    prototype_0 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_0;
+; CHECK-NEXT:    call (retval0), %rd{{[0-9]+}}, (param0), prototype_0;
   %ret = call %struct.64 @callee(ptr %p)
   ret %struct.64 %ret
 }
@@ -23,12 +18,7 @@ define i64 @test_param_type_mismatch(ptr %p) {
 ; CHECK-LABEL: test_param_type_mismatch(
 ; CHECK:         .param .b64 retval0;
 ; CHECK-NEXT:    prototype_1 : .callprototype (.param .b64 _) _ (.param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_1;
+; CHECK-NEXT:    call (retval0), %rd{{[0-9]+}}, (param0), prototype_1;
   %ret = call i64 @callee(i64 7)
   ret i64 %ret
 }
@@ -37,13 +27,7 @@ define i64 @test_param_count_mismatch(ptr %p) {
 ; CHECK-LABEL: test_param_count_mismatch(
 ; CHECK:         .param .b64 retval0;
 ; CHECK-NEXT:    prototype_2 : .callprototype (.param .b64 _) _ (.param .b64 _, .param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_2;
+; CHECK-NEXT:    call (retval0), %rd{{[0-9]+}}, (param0, param1), prototype_2;
   %ret = call i64 @callee(ptr %p, i64 7)
   ret i64 %ret
 }
@@ -52,12 +36,7 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_return_type_mismatch_variadic(
 ; CHECK:         .param .align 1 .b8 retval0[8];
 ; CHECK-NEXT:    prototype_3 : .callprototype (.param .align 1 .b8 _[8]) _ (.param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_3;
+; CHECK-NEXT:    call (retval0), %rd{{[0-9]+}}, (param0), prototype_3;
   %ret = call %struct.64 (ptr, ...) @callee_variadic(ptr %p)
   ret %struct.64 %ret
 }
@@ -65,12 +44,7 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) {
 define i64 @test_param_type_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_param_type_mismatch_variadic(
 ; CHECK:         .param .b64 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee_variadic
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    )
+; CHECK-NEXT:    call.uni (retval0), callee_variadic, (param0, param1);
   %ret = call i64 (ptr, ...) @callee_variadic(ptr %p, i64 7)
   ret i64 %ret
 }
@@ -78,12 +52,7 @@ define i64 @test_param_type_mismatch_variadic(ptr %p) {
 define i64 @test_param_count_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_param_count_mismatch_variadic(
 ; CHECK:         .param .b64 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee_variadic
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    )
+; CHECK-NEXT:    call.uni (retval0), callee_variadic, (param0, param1);
   %ret = call i64 (ptr, ...) @callee_variadic(ptr %p, i64 7)
   ret i64 %ret
 }
diff --git a/llvm/test/CodeGen/NVPTX/convert-fp-i8.ll b/llvm/test/CodeGen/NVPTX/convert-fp-i8.ll
index 5e856112c014..f8bfcd553150 100644
--- a/llvm/test/CodeGen/NVPTX/convert-fp-i8.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-fp-i8.ll
@@ -100,14 +100,13 @@ define i8 @cvt_s8_f32(float %x) {
 ; CHECK-LABEL: cvt_s8_f32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [cvt_s8_f32_param_0];
 ; CHECK-NEXT:    cvt.rzi.s16.f32 %rs1, %r1;
 ; CHECK-NEXT:    cvt.u32.u16 %r2, %rs1;
-; CHECK-NEXT:    and.b32 %r3, %r2, 255;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %a = fptosi float %x to i8
   ret i8 %a
@@ -117,15 +116,14 @@ define i8 @cvt_s8_f64(double %x) {
 ; CHECK-LABEL: cvt_s8_f64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [cvt_s8_f64_param_0];
 ; CHECK-NEXT:    cvt.rzi.s16.f64 %rs1, %rd1;
 ; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
-; CHECK-NEXT:    and.b32 %r2, %r1, 255;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %a = fptosi double %x to i8
   ret i8 %a
diff --git a/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll b/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
index ce6a16d9c040..a2fc8da3f1e6 100644
--- a/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-int-sm20.ll
@@ -1,66 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_20 | FileCheck %s
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
 ; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -mtriple=nvptx -mcpu=sm_20 | %ptxas-verify %}
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
-
 ;; Integer conversions happen inplicitly by loading/storing the proper types
 
-
 ; i16
 
 define i16 @cvt_i16_i32(i32 %x) {
-; CHECK: ld.param.b16 %r[[R0:[0-9]+]], [cvt_i16_i32_param_{{[0-9]+}}]
-; CHECK: st.param.b32 [func_retval{{[0-9]+}}], %r[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i16_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [cvt_i16_i32_param_0];
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   %a = trunc i32 %x to i16
   ret i16 %a
 }
 
 define i16 @cvt_i16_i64(i64 %x) {
-; CHECK: ld.param.b16 %r[[R0:[0-9]+]], [cvt_i16_i64_param_{{[0-9]+}}]
-; CHECK: st.param.b32 [func_retval{{[0-9]+}}], %r[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i16_i64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %r1, [cvt_i16_i64_param_0];
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   %a = trunc i64 %x to i16
   ret i16 %a
 }
 
-
-
 ; i32
 
 define i32 @cvt_i32_i16(i16 %x) {
-; CHECK: ld.param.b16 %r[[R0:[0-9]+]], [cvt_i32_i16_param_{{[0-9]+}}]
-; CHECK: st.param.b32 [func_retval{{[0-9]+}}], %r[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i32_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %r1, [cvt_i32_i16_param_0];
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   %a = zext i16 %x to i32
   ret i32 %a
 }
 
 define i32 @cvt_i32_i64(i64 %x) {
-; CHECK: ld.param.b32 %r[[R0:[0-9]+]], [cvt_i32_i64_param_{{[0-9]+}}]
-; CHECK: st.param.b32 [func_retval{{[0-9]+}}], %r[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i32_i64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [cvt_i32_i64_param_0];
+; CHECK-NEXT:    st.param.b32 [func_retval0], %rd1;
+; CHECK-NEXT:    ret;
   %a = trunc i64 %x to i32
   ret i32 %a
 }
 
-
-
 ; i64
 
 define i64 @cvt_i64_i16(i16 %x) {
-; CHECK: ld.param.b16 %rd[[R0:[0-9]+]], [cvt_i64_i16_param_{{[0-9]+}}]
-; CHECK: st.param.b64 [func_retval{{[0-9]+}}], %rd[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i64_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rd1, [cvt_i64_i16_param_0];
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    ret;
   %a = zext i16 %x to i64
   ret i64 %a
 }
 
 define i64 @cvt_i64_i32(i32 %x) {
-; CHECK: ld.param.b32 %rd[[R0:[0-9]+]], [cvt_i64_i32_param_{{[0-9]+}}]
-; CHECK: st.param.b64 [func_retval{{[0-9]+}}], %rd[[R0]]
-; CHECK: ret
+; CHECK-LABEL: cvt_i64_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %rd1, [cvt_i64_i32_param_0];
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    ret;
   %a = zext i32 %x to i64
   ret i64 %a
 }
diff --git a/llvm/test/CodeGen/NVPTX/demote-vars.ll b/llvm/test/CodeGen/NVPTX/demote-vars.ll
index ab89b62b53d0..e554e4aaea36 100644
--- a/llvm/test/CodeGen/NVPTX/demote-vars.ll
+++ b/llvm/test/CodeGen/NVPTX/demote-vars.ll
@@ -67,8 +67,7 @@ define void @define_private_global(i64 %val) {
 ; Also check that the if-then is still here, otherwise we may not be testing
 ; the "more-than-one-use" part.
 ; CHECK: st.shared.b64   [private_global_used_more_than_once_in_same_fct],
-; CHECK: mov.b64 %[[VAR:.*]], 25
-; CHECK: st.shared.b64   [private_global_used_more_than_once_in_same_fct], %[[VAR]]
+; CHECK: st.shared.b64   [private_global_used_more_than_once_in_same_fct], 25
 define void @define_private_global_more_than_one_use(i64 %val, i1 %cond) {
   store i64 %val, ptr addrspace(3) @private_global_used_more_than_once_in_same_fct
   br i1 %cond, label %then, label %end
diff --git a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
index b73aea76a452..4d2ba7d00f87 100644
--- a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
@@ -24,11 +24,7 @@ define i32 @test_dynamic_stackalloc(i64 %n) {
 ; CHECK-32-NEXT:    .param .b32 param0;
 ; CHECK-32-NEXT:    st.param.b32 [param0], %r5;
 ; CHECK-32-NEXT:    .param .b32 retval0;
-; CHECK-32-NEXT:    call.uni (retval0),
-; CHECK-32-NEXT:    bar,
-; CHECK-32-NEXT:    (
-; CHECK-32-NEXT:    param0
-; CHECK-32-NEXT:    );
+; CHECK-32-NEXT:    call.uni (retval0), bar, (param0);
 ; CHECK-32-NEXT:    ld.param.b32 %r6, [retval0];
 ; CHECK-32-NEXT:    } // callseq 0
 ; CHECK-32-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -49,11 +45,7 @@ define i32 @test_dynamic_stackalloc(i64 %n) {
 ; CHECK-64-NEXT:    .param .b64 param0;
 ; CHECK-64-NEXT:    st.param.b64 [param0], %rd5;
 ; CHECK-64-NEXT:    .param .b32 retval0;
-; CHECK-64-NEXT:    call.uni (retval0),
-; CHECK-64-NEXT:    bar,
-; CHECK-64-NEXT:    (
-; CHECK-64-NEXT:    param0
-; CHECK-64-NEXT:    );
+; CHECK-64-NEXT:    call.uni (retval0), bar, (param0);
 ; CHECK-64-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-64-NEXT:    } // callseq 0
 ; CHECK-64-NEXT:    st.param.b32 [func_retval0], %r1;
diff --git a/llvm/test/CodeGen/NVPTX/elect.ll b/llvm/test/CodeGen/NVPTX/elect.ll
index 93c30a9b0006..b65fa5a6376e 100644
--- a/llvm/test/CodeGen/NVPTX/elect.ll
+++ b/llvm/test/CodeGen/NVPTX/elect.ll
@@ -54,9 +54,9 @@ define {i32, i1} @elect_sync_twice(i32 %mask) {
 ; CHECK-NEXT:    ld.param.b32 %r1, [elect_sync_twice_param_0];
 ; CHECK-NEXT:    elect.sync %r2|%p1, %r1;
 ; CHECK-NEXT:    elect.sync %r3|%p2, %r1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p1;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %val = call {i32, i1} @llvm.nvvm.elect.sync(i32 %mask)
   %val2 = call {i32, i1} @llvm.nvvm.elect.sync(i32 %mask)
diff --git a/llvm/test/CodeGen/NVPTX/extractelement.ll b/llvm/test/CodeGen/NVPTX/extractelement.ll
index 79d80e6f8fa8..f37777ab954e 100644
--- a/llvm/test/CodeGen/NVPTX/extractelement.ll
+++ b/llvm/test/CodeGen/NVPTX/extractelement.ll
@@ -41,7 +41,7 @@ define i1  @test_v2i8_load(ptr %a) {
 ; CHECK-NEXT:    or.b16 %rs5, %rs1, %rs2;
 ; CHECK-NEXT:    and.b16 %rs6, %rs5, 255;
 ; CHECK-NEXT:    setp.eq.s16 %p1, %rs6, 0;
-; CHECK-NEXT:    selp.b32 %r1, 1, 0, %p1;
+; CHECK-NEXT:    selp.b32 %r1, -1, 0, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %v = load <2 x i8>, ptr %a, align 4
@@ -158,27 +158,24 @@ define i16  @test_v8i8(i64 %a) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<16>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v8i8_param_0];
-; CHECK-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r1}, %rd1; }
-; CHECK-NEXT:    cvt.u32.u64 %r2, %rd1;
-; CHECK-NEXT:    bfe.s32 %r3, %r2, 0, 8;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v8i8_param_0];
+; CHECK-NEXT:    bfe.s32 %r3, %r1, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs1, %r3;
-; CHECK-NEXT:    bfe.s32 %r4, %r2, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r4, %r1, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs2, %r4;
-; CHECK-NEXT:    bfe.s32 %r5, %r2, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r5, %r1, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs3, %r5;
-; CHECK-NEXT:    bfe.s32 %r6, %r2, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r6, %r1, 24, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs4, %r6;
-; CHECK-NEXT:    bfe.s32 %r7, %r1, 0, 8;
+; CHECK-NEXT:    bfe.s32 %r7, %r2, 0, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs5, %r7;
-; CHECK-NEXT:    bfe.s32 %r8, %r1, 8, 8;
+; CHECK-NEXT:    bfe.s32 %r8, %r2, 8, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs6, %r8;
-; CHECK-NEXT:    bfe.s32 %r9, %r1, 16, 8;
+; CHECK-NEXT:    bfe.s32 %r9, %r2, 16, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs7, %r9;
-; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
+; CHECK-NEXT:    bfe.s32 %r10, %r2, 24, 8;
 ; CHECK-NEXT:    cvt.s8.s32 %rs8, %r10;
 ; CHECK-NEXT:    add.s16 %rs9, %rs1, %rs2;
 ; CHECK-NEXT:    add.s16 %rs10, %rs3, %rs4;
diff --git a/llvm/test/CodeGen/NVPTX/f16-instructions.ll b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
index c905fc04ce78..13f1c2f30b83 100644
--- a/llvm/test/CodeGen/NVPTX/f16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
@@ -43,8 +43,7 @@
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
 ; CHECK-LABEL: test_ret_const(
-; CHECK:      mov.b16         [[R:%rs[0-9]+]], 0x3C00;
-; CHECK-NEXT: st.param.b16    [func_retval0], [[R]];
+; CHECK: st.param.b16    [func_retval0], 0x3C00;
 ; CHECK-NEXT: ret;
 define half @test_ret_const() #0 {
   ret half 1.0
@@ -263,12 +262,7 @@ declare half @test_callee(half %a, half %b) #0
 ; CHECK-DAG:  st.param.b16    [param0], [[A]];
 ; CHECK-DAG:  st.param.b16    [param1], [[B]];
 ; CHECK-DAG:  .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK-NEXT: (
-; CHECK-NEXT:        param0,
-; CHECK-NEXT:        param1
-; CHECK-NEXT: );
+; CHECK:      call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT: ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK-NEXT: }
 ; CHECK-NEXT: st.param.b16    [func_retval0], [[R]];
@@ -287,12 +281,7 @@ define half @test_call(half %a, half %b) #0 {
 ; CHECK-DAG:  st.param.b16    [param0], [[B]];
 ; CHECK-DAG:  st.param.b16    [param1], [[A]];
 ; CHECK-DAG:  .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK-NEXT: (
-; CHECK-NEXT:        param0,
-; CHECK-NEXT:        param1
-; CHECK-NEXT: );
+; CHECK:      call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT: ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK-NEXT: }
 ; CHECK-NEXT: st.param.b16    [func_retval0], [[R]];
@@ -311,12 +300,7 @@ define half @test_call_flipped(half %a, half %b) #0 {
 ; CHECK-DAG:  st.param.b16    [param0], [[B]];
 ; CHECK-DAG:  st.param.b16    [param1], [[A]];
 ; CHECK-DAG:  .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT:        test_callee,
-; CHECK-NEXT: (
-; CHECK-NEXT:        param0,
-; CHECK-NEXT:        param1
-; CHECK-NEXT: );
+; CHECK:      call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT: ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK-NEXT: }
 ; CHECK-NEXT: st.param.b16    [func_retval0], [[R]];
@@ -399,7 +383,7 @@ define half @test_select_cc_f16_f32(half %a, half %b, float %c, float %d) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.neu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_une(half %a, half %b) #0 {
@@ -415,7 +399,7 @@ define i1 @test_fcmp_une(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.equ.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ueq(half %a, half %b) #0 {
@@ -431,7 +415,7 @@ define i1 @test_fcmp_ueq(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.gtu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ugt(half %a, half %b) #0 {
@@ -447,7 +431,7 @@ define i1 @test_fcmp_ugt(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.geu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_uge(half %a, half %b) #0 {
@@ -463,7 +447,7 @@ define i1 @test_fcmp_uge(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.ltu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ult(half %a, half %b) #0 {
@@ -479,7 +463,7 @@ define i1 @test_fcmp_ult(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.leu.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ule(half %a, half %b) #0 {
@@ -496,7 +480,7 @@ define i1 @test_fcmp_ule(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.nan.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_uno(half %a, half %b) #0 {
@@ -512,7 +496,7 @@ define i1 @test_fcmp_uno(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.ne.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_one(half %a, half %b) #0 {
@@ -528,7 +512,7 @@ define i1 @test_fcmp_one(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.eq.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_oeq(half %a, half %b) #0 {
@@ -544,7 +528,7 @@ define i1 @test_fcmp_oeq(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.gt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ogt(half %a, half %b) #0 {
@@ -560,7 +544,7 @@ define i1 @test_fcmp_ogt(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.ge.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_oge(half %a, half %b) #0 {
@@ -576,7 +560,7 @@ define i1 @test_fcmp_oge(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.lt.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_olt(half %a, half %b) #0 {
@@ -592,7 +576,7 @@ define i1 @test_fcmp_olt(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.le.f32    [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ole(half %a, half %b) #0 {
@@ -608,7 +592,7 @@ define i1 @test_fcmp_ole(half %a, half %b) #0 {
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[AF:%r[0-9]+]], [[A]];
 ; CHECK-NOF16-DAG: cvt.f32.f16 [[BF:%r[0-9]+]], [[B]];
 ; CHECK-NOF16: setp.num.f32   [[PRED:%p[0-9]+]], [[AF]], [[BF]]
-; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], 1, 0, [[PRED]];
+; CHECK-NEXT: selp.b32        [[R:%r[0-9]+]], -1, 0, [[PRED]];
 ; CHECK-NEXT: st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
 define i1 @test_fcmp_ord(half %a, half %b) #0 {
@@ -650,8 +634,7 @@ else:
 ; CHECK:      ld.b16  [[AB:%rs[0-9]+]], [%[[P1]]];
 ; CHECK:      {
 ; CHECK:      st.param.b64    [param0], %[[P1]];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_dummy
+; CHECK:      call.uni (retval0), test_dummy
 ; CHECK:      }
 ; CHECK:      setp.ne.b32     [[PRED:%p[0-9]+]], %r{{[0-9]+}}, 0;
 ; CHECK:      @[[PRED]] bra   [[LOOP]];
diff --git a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
index 8c89f82dbf9c..43a605f2b34d 100644
--- a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
@@ -34,11 +34,10 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define <2 x half> @test_ret_const() #0 {
 ; CHECK-LABEL: test_ret_const(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 1073757184;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1073757184;
 ; CHECK-NEXT:    ret;
   ret <2 x half> <half 1.0, half 2.0>
 }
@@ -420,14 +419,14 @@ define void @test_ldst_v3f16(ptr %a, ptr %b) {
 define void @test_ldst_v4f16(ptr %a, ptr %b) {
 ; CHECK-LABEL: test_ldst_v4f16(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd2, [test_ldst_v4f16_param_1];
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v4f16_param_0];
-; CHECK-NEXT:    ld.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [%rd1];
-; CHECK-NEXT:    st.v4.b16 [%rd2], {%rs1, %rs2, %rs3, %rs4};
+; CHECK-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT:    st.v2.b32 [%rd2], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   %t1 = load <4 x half>, ptr %a
   store <4 x half> %t1, ptr %b, align 16
@@ -467,12 +466,7 @@ define <2 x half> @test_call(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r2;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -495,12 +489,7 @@ define <2 x half> @test_call_flipped(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r1;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -523,12 +512,7 @@ define <2 x half> @test_tailcall_flipped(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r1;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 2
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -562,7 +546,7 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
 ; CHECK-F16:       {
 ; CHECK-F16-NEXT:    .reg .pred %p<3>;
 ; CHECK-F16-NEXT:    .reg .b16 %rs<7>;
-; CHECK-F16-NEXT:    .reg .b32 %r<6>;
+; CHECK-F16-NEXT:    .reg .b32 %r<5>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
 ; CHECK-F16-NEXT:    ld.param.b32 %r4, [test_select_cc_param_3];
@@ -574,15 +558,14 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
 ; CHECK-F16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; CHECK-F16-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
 ; CHECK-F16-NEXT:    selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-F16-NEXT:    mov.b32 %r5, {%rs6, %rs5};
-; CHECK-F16-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-F16-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; CHECK-F16-NEXT:    ret;
 ;
 ; CHECK-NOF16-LABEL: test_select_cc(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .pred %p<3>;
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<11>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<9>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
 ; CHECK-NOF16-NEXT:    ld.param.b32 %r4, [test_select_cc_param_3];
@@ -601,8 +584,7 @@ define <2 x half> @test_select_cc(<2 x half> %a, <2 x half> %b, <2 x half> %c, <
 ; CHECK-NOF16-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p2;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs10, %rs7, %rs5, %p1;
-; CHECK-NOF16-NEXT:    mov.b32 %r9, {%rs10, %rs9};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-NOF16-NEXT:    ret;
   %cc = fcmp une <2 x half> %c, %d
   %r = select <2 x i1> %cc, <2 x half> %a, <2 x half> %b
@@ -660,7 +642,7 @@ define <2 x half> @test_select_cc_f16_f32(<2 x half> %a, <2 x half> %b,
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f16_f32_param_3];
@@ -673,8 +655,7 @@ define <2 x half> @test_select_cc_f16_f32(<2 x half> %a, <2 x half> %b,
 ; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; CHECK-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
 ; CHECK-NEXT:    selp.b16 %rs6, %rs3, %rs1, %p1;
-; CHECK-NEXT:    mov.b32 %r7, {%rs6, %rs5};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; CHECK-NEXT:    ret;
                                           <2 x float> %c, <2 x float> %d) #0 {
   %cc = fcmp une <2 x float> %c, %d
@@ -1359,14 +1340,13 @@ define <2 x half> @test_uitofp_2xi32(<2 x i32> %a) #0 {
 ; CHECK-LABEL: test_uitofp_2xi32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_uitofp_2xi32_param_0];
 ; CHECK-NEXT:    cvt.rn.f16.u32 %rs1, %r2;
 ; CHECK-NEXT:    cvt.rn.f16.u32 %rs2, %r1;
-; CHECK-NEXT:    mov.b32 %r3, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %r = uitofp <2 x i32> %a to <2 x half>
   ret <2 x half> %r
@@ -1376,15 +1356,13 @@ define <2 x half> @test_uitofp_2xi64(<2 x i64> %a) #0 {
 ; CHECK-LABEL: test_uitofp_2xi64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_uitofp_2xi64_param_0];
 ; CHECK-NEXT:    cvt.rn.f16.u64 %rs1, %rd2;
 ; CHECK-NEXT:    cvt.rn.f16.u64 %rs2, %rd1;
-; CHECK-NEXT:    mov.b32 %r1, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %r = uitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %r
@@ -1394,14 +1372,13 @@ define <2 x half> @test_sitofp_2xi32(<2 x i32> %a) #0 {
 ; CHECK-LABEL: test_sitofp_2xi32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_sitofp_2xi32_param_0];
 ; CHECK-NEXT:    cvt.rn.f16.s32 %rs1, %r2;
 ; CHECK-NEXT:    cvt.rn.f16.s32 %rs2, %r1;
-; CHECK-NEXT:    mov.b32 %r3, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %r = sitofp <2 x i32> %a to <2 x half>
   ret <2 x half> %r
@@ -1411,15 +1388,13 @@ define <2 x half> @test_sitofp_2xi64(<2 x i64> %a) #0 {
 ; CHECK-LABEL: test_sitofp_2xi64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_sitofp_2xi64_param_0];
 ; CHECK-NEXT:    cvt.rn.f16.s64 %rs1, %rd2;
 ; CHECK-NEXT:    cvt.rn.f16.s64 %rs2, %rd1;
-; CHECK-NEXT:    mov.b32 %r1, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %r = sitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %r
@@ -1918,7 +1893,7 @@ define <2 x half> @test_copysign(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-NOF16-LABEL: test_copysign(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<11>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
 ; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [test_copysign_param_1];
@@ -1931,8 +1906,7 @@ define <2 x half> @test_copysign(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-NOF16-NEXT:    and.b16 %rs8, %rs1, -32768;
 ; CHECK-NOF16-NEXT:    and.b16 %rs9, %rs4, 32767;
 ; CHECK-NOF16-NEXT:    or.b16 %rs10, %rs9, %rs8;
-; CHECK-NOF16-NEXT:    mov.b32 %r3, {%rs10, %rs7};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs7};
 ; CHECK-NOF16-NEXT:    ret;
   %r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %r
@@ -1959,7 +1933,7 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
 ; CHECK-NOF16-LABEL: test_copysign_f32(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<9>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<6>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
 ; CHECK-NOF16-NEXT:    ld.param.v2.b32 {%r2, %r3}, [test_copysign_f32_param_1];
@@ -1973,8 +1947,7 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
 ; CHECK-NOF16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs6}, %r5; }
 ; CHECK-NOF16-NEXT:    and.b16 %rs7, %rs2, 32767;
 ; CHECK-NOF16-NEXT:    or.b16 %rs8, %rs7, %rs6;
-; CHECK-NOF16-NEXT:    mov.b32 %r6, {%rs8, %rs5};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs8, %rs5};
 ; CHECK-NOF16-NEXT:    ret;
   %tb = fptrunc <2 x float> %b to <2 x half>
   %r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %tb)
@@ -2003,7 +1976,7 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
 ; CHECK-NOF16-LABEL: test_copysign_f64(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<9>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NOF16-NEXT:    .reg .b64 %rd<7>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
@@ -2020,8 +1993,7 @@ define <2 x half> @test_copysign_f64(<2 x half> %a, <2 x double> %b) #0 {
 ; CHECK-NOF16-NEXT:    shr.u64 %rd6, %rd5, 48;
 ; CHECK-NOF16-NEXT:    cvt.u16.u64 %rs7, %rd6;
 ; CHECK-NOF16-NEXT:    or.b16 %rs8, %rs6, %rs7;
-; CHECK-NOF16-NEXT:    mov.b32 %r2, {%rs8, %rs5};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs8, %rs5};
 ; CHECK-NOF16-NEXT:    ret;
   %tb = fptrunc <2 x double> %b to <2 x half>
   %r = call <2 x half> @llvm.copysign.f16(<2 x half> %a, <2 x half> %tb)
@@ -2075,15 +2047,14 @@ define <2 x half> @test_floor(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_floor(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_floor_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rmi.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rmi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.floor.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2093,15 +2064,14 @@ define <2 x half> @test_ceil(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_ceil(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_ceil_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rpi.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rpi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.ceil.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2111,15 +2081,14 @@ define <2 x half> @test_trunc(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_trunc(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_trunc_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rzi.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rzi.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.trunc.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2129,15 +2098,14 @@ define <2 x half> @test_rint(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_rint(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_rint_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.rint.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2147,15 +2115,14 @@ define <2 x half> @test_nearbyint(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_nearbyint(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_nearbyint_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.nearbyint.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2165,15 +2132,14 @@ define <2 x half> @test_roundeven(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_roundeven(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_roundeven_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rni.f16.f16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = call <2 x half> @llvm.roundeven.f16(<2 x half> %a)
   ret <2 x half> %r
@@ -2267,13 +2233,12 @@ define <2 x half> @test_shufflevector(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_shufflevector(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_shufflevector_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %s = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
   ret <2 x half> %s
@@ -2283,14 +2248,13 @@ define <2 x half> @test_insertelement(<2 x half> %a, half %x) #0 {
 ; CHECK-LABEL: test_insertelement(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b16 %rs1, [test_insertelement_param_1];
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_insertelement_param_0];
 ; CHECK-NEXT:    { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
-; CHECK-NEXT:    mov.b32 %r2, {%rs2, %rs1};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; CHECK-NEXT:    ret;
   %i = insertelement <2 x half> %a, half %x, i64 1
   ret <2 x half> %i
@@ -2300,15 +2264,14 @@ define <2 x half> @test_sitofp_2xi16_to_2xhalf(<2 x i16> %a) #0 {
 ; CHECK-LABEL: test_sitofp_2xi16_to_2xhalf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_sitofp_2xi16_to_2xhalf_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rn.f16.s16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rn.f16.s16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = sitofp <2 x i16> %a to <2 x half>
   ret <2 x half> %r
@@ -2318,19 +2281,38 @@ define <2 x half> @test_uitofp_2xi16_to_2xhalf(<2 x i16> %a) #0 {
 ; CHECK-LABEL: test_uitofp_2xi16_to_2xhalf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_uitofp_2xi16_to_2xhalf_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; CHECK-NEXT:    cvt.rn.f16.u16 %rs3, %rs2;
 ; CHECK-NEXT:    cvt.rn.f16.u16 %rs4, %rs1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %r = uitofp <2 x i16> %a to <2 x half>
   ret <2 x half> %r
 }
 
+define void @test_store_2xhalf(ptr %p1, ptr %p2, <2 x half> %v) {
+; CHECK-LABEL: test_store_2xhalf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [test_store_2xhalf_param_2];
+; CHECK-NEXT:    ld.param.b64 %rd2, [test_store_2xhalf_param_1];
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_store_2xhalf_param_0];
+; CHECK-NEXT:    st.b32 [%rd1], %r1;
+; CHECK-NEXT:    st.b32 [%rd2], 1006648320;
+; CHECK-NEXT:    ret;
+  store <2 x half> %v, ptr %p1
+  store <2 x half> <half 1.0, half 1.0>, ptr %p2
+  ret void
+}
+
+
+
 attributes #0 = { nounwind }
 attributes #1 = { "unsafe-fp-math" = "true" }
diff --git a/llvm/test/CodeGen/NVPTX/fexp2.ll b/llvm/test/CodeGen/NVPTX/fexp2.ll
index ef2a788bb826..391aa453f075 100644
--- a/llvm/test/CodeGen/NVPTX/fexp2.ll
+++ b/llvm/test/CodeGen/NVPTX/fexp2.ll
@@ -206,19 +206,18 @@ define <2 x half> @exp2_f16_test_v(<2 x half> %in) {
 ; CHECK-LABEL: exp2_f16_test_v(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
-; CHECK-NEXT:    ld.param.b32 %r1, [exp2_f16_test_v_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT:    ex2.approx.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r3;
-; CHECK-NEXT:    cvt.f32.f16 %r4, %rs1;
-; CHECK-NEXT:    ex2.approx.f32 %r5, %r4;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs4, %r5;
-; CHECK-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [exp2_f16_test_v_param_0];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    ex2.approx.f32 %r2, %r1;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r3, %rs1;
+; CHECK-NEXT:    ex2.approx.f32 %r4, %r3;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs4, %r4;
+; CHECK-NEXT:    mov.b32 %r5, {%rs4, %rs3};
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FP16-LABEL: exp2_f16_test_v(
@@ -253,8 +252,7 @@ define bfloat @exp2_bf16_test(bfloat %in) {
 ; CHECK-LABEL: exp2_bf16_test(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b16 %r1, [exp2_bf16_test_param_0];
@@ -266,15 +264,14 @@ define bfloat @exp2_bf16_test(bfloat %in) {
 ; CHECK-NEXT:    setp.nan.f32 %p1, %r3, %r3;
 ; CHECK-NEXT:    or.b32 %r7, %r3, 4194304;
 ; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    shr.u32 %r9, %r8, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r9;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FP16-LABEL: exp2_bf16_test(
 ; CHECK-FP16:       {
 ; CHECK-FP16-NEXT:    .reg .pred %p<2>;
-; CHECK-FP16-NEXT:    .reg .b16 %rs<2>;
-; CHECK-FP16-NEXT:    .reg .b32 %r<9>;
+; CHECK-FP16-NEXT:    .reg .b32 %r<10>;
 ; CHECK-FP16-EMPTY:
 ; CHECK-FP16-NEXT:  // %bb.0: // %entry
 ; CHECK-FP16-NEXT:    ld.param.b16 %r1, [exp2_bf16_test_param_0];
@@ -286,8 +283,8 @@ define bfloat @exp2_bf16_test(bfloat %in) {
 ; CHECK-FP16-NEXT:    setp.nan.f32 %p1, %r3, %r3;
 ; CHECK-FP16-NEXT:    or.b32 %r7, %r3, 4194304;
 ; CHECK-FP16-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-FP16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
-; CHECK-FP16-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-FP16-NEXT:    shr.u32 %r9, %r8, 16;
+; CHECK-FP16-NEXT:    st.param.b16 [func_retval0], %r9;
 ; CHECK-FP16-NEXT:    ret;
 ;
 ; CHECK-BF16-LABEL: exp2_bf16_test(
@@ -310,62 +307,60 @@ define <2 x bfloat> @exp2_bf16_test_v(<2 x bfloat> %in) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<19>;
+; CHECK-NEXT:    .reg .b32 %r<18>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
-; CHECK-NEXT:    ld.param.b32 %r1, [exp2_bf16_test_v_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.u32.u16 %r2, %rs2;
-; CHECK-NEXT:    shl.b32 %r3, %r2, 16;
-; CHECK-NEXT:    ex2.approx.f32 %r4, %r3;
-; CHECK-NEXT:    bfe.u32 %r5, %r4, 16, 1;
-; CHECK-NEXT:    add.s32 %r6, %r5, %r4;
-; CHECK-NEXT:    add.s32 %r7, %r6, 32767;
-; CHECK-NEXT:    setp.nan.f32 %p1, %r4, %r4;
-; CHECK-NEXT:    or.b32 %r8, %r4, 4194304;
-; CHECK-NEXT:    selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs1;
-; CHECK-NEXT:    shl.b32 %r11, %r10, 16;
-; CHECK-NEXT:    ex2.approx.f32 %r12, %r11;
-; CHECK-NEXT:    bfe.u32 %r13, %r12, 16, 1;
-; CHECK-NEXT:    add.s32 %r14, %r13, %r12;
-; CHECK-NEXT:    add.s32 %r15, %r14, 32767;
-; CHECK-NEXT:    setp.nan.f32 %p2, %r12, %r12;
-; CHECK-NEXT:    or.b32 %r16, %r12, 4194304;
-; CHECK-NEXT:    selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-NEXT:    prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r18;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [exp2_bf16_test_v_param_0];
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-NEXT:    ex2.approx.f32 %r3, %r2;
+; CHECK-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; CHECK-NEXT:    add.s32 %r5, %r4, %r3;
+; CHECK-NEXT:    add.s32 %r6, %r5, 32767;
+; CHECK-NEXT:    setp.nan.f32 %p1, %r3, %r3;
+; CHECK-NEXT:    or.b32 %r7, %r3, 4194304;
+; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    cvt.u32.u16 %r9, %rs1;
+; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
+; CHECK-NEXT:    ex2.approx.f32 %r11, %r10;
+; CHECK-NEXT:    bfe.u32 %r12, %r11, 16, 1;
+; CHECK-NEXT:    add.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    add.s32 %r14, %r13, 32767;
+; CHECK-NEXT:    setp.nan.f32 %p2, %r11, %r11;
+; CHECK-NEXT:    or.b32 %r15, %r11, 4194304;
+; CHECK-NEXT:    selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-NEXT:    prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r17;
 ; CHECK-NEXT:    ret;
 ;
 ; CHECK-FP16-LABEL: exp2_bf16_test_v(
 ; CHECK-FP16:       {
 ; CHECK-FP16-NEXT:    .reg .pred %p<3>;
 ; CHECK-FP16-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FP16-NEXT:    .reg .b32 %r<19>;
+; CHECK-FP16-NEXT:    .reg .b32 %r<18>;
 ; CHECK-FP16-EMPTY:
 ; CHECK-FP16-NEXT:  // %bb.0: // %entry
-; CHECK-FP16-NEXT:    ld.param.b32 %r1, [exp2_bf16_test_v_param_0];
-; CHECK-FP16-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-FP16-NEXT:    cvt.u32.u16 %r2, %rs2;
-; CHECK-FP16-NEXT:    shl.b32 %r3, %r2, 16;
-; CHECK-FP16-NEXT:    ex2.approx.f32 %r4, %r3;
-; CHECK-FP16-NEXT:    bfe.u32 %r5, %r4, 16, 1;
-; CHECK-FP16-NEXT:    add.s32 %r6, %r5, %r4;
-; CHECK-FP16-NEXT:    add.s32 %r7, %r6, 32767;
-; CHECK-FP16-NEXT:    setp.nan.f32 %p1, %r4, %r4;
-; CHECK-FP16-NEXT:    or.b32 %r8, %r4, 4194304;
-; CHECK-FP16-NEXT:    selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-FP16-NEXT:    cvt.u32.u16 %r10, %rs1;
-; CHECK-FP16-NEXT:    shl.b32 %r11, %r10, 16;
-; CHECK-FP16-NEXT:    ex2.approx.f32 %r12, %r11;
-; CHECK-FP16-NEXT:    bfe.u32 %r13, %r12, 16, 1;
-; CHECK-FP16-NEXT:    add.s32 %r14, %r13, %r12;
-; CHECK-FP16-NEXT:    add.s32 %r15, %r14, 32767;
-; CHECK-FP16-NEXT:    setp.nan.f32 %p2, %r12, %r12;
-; CHECK-FP16-NEXT:    or.b32 %r16, %r12, 4194304;
-; CHECK-FP16-NEXT:    selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-FP16-NEXT:    prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-FP16-NEXT:    st.param.b32 [func_retval0], %r18;
+; CHECK-FP16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [exp2_bf16_test_v_param_0];
+; CHECK-FP16-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-FP16-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-FP16-NEXT:    ex2.approx.f32 %r3, %r2;
+; CHECK-FP16-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; CHECK-FP16-NEXT:    add.s32 %r5, %r4, %r3;
+; CHECK-FP16-NEXT:    add.s32 %r6, %r5, 32767;
+; CHECK-FP16-NEXT:    setp.nan.f32 %p1, %r3, %r3;
+; CHECK-FP16-NEXT:    or.b32 %r7, %r3, 4194304;
+; CHECK-FP16-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-FP16-NEXT:    cvt.u32.u16 %r9, %rs1;
+; CHECK-FP16-NEXT:    shl.b32 %r10, %r9, 16;
+; CHECK-FP16-NEXT:    ex2.approx.f32 %r11, %r10;
+; CHECK-FP16-NEXT:    bfe.u32 %r12, %r11, 16, 1;
+; CHECK-FP16-NEXT:    add.s32 %r13, %r12, %r11;
+; CHECK-FP16-NEXT:    add.s32 %r14, %r13, 32767;
+; CHECK-FP16-NEXT:    setp.nan.f32 %p2, %r11, %r11;
+; CHECK-FP16-NEXT:    or.b32 %r15, %r11, 4194304;
+; CHECK-FP16-NEXT:    selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-FP16-NEXT:    prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-FP16-NEXT:    st.param.b32 [func_retval0], %r17;
 ; CHECK-FP16-NEXT:    ret;
 ;
 ; CHECK-BF16-LABEL: exp2_bf16_test_v(
diff --git a/llvm/test/CodeGen/NVPTX/flog2.ll b/llvm/test/CodeGen/NVPTX/flog2.ll
index 7a5b1bb0ddef..acac5a8da4e1 100644
--- a/llvm/test/CodeGen/NVPTX/flog2.ll
+++ b/llvm/test/CodeGen/NVPTX/flog2.ll
@@ -97,19 +97,18 @@ define <2 x half> @log2_f16_test_v(<2 x half> %in) {
 ; CHECK-LABEL: log2_f16_test_v(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
-; CHECK-NEXT:    ld.param.b32 %r1, [log2_f16_test_v_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT:    lg2.approx.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r3;
-; CHECK-NEXT:    cvt.f32.f16 %r4, %rs1;
-; CHECK-NEXT:    lg2.approx.f32 %r5, %r4;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs4, %r5;
-; CHECK-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [log2_f16_test_v_param_0];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    lg2.approx.f32 %r2, %r1;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r3, %rs1;
+; CHECK-NEXT:    lg2.approx.f32 %r4, %r3;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs4, %r4;
+; CHECK-NEXT:    mov.b32 %r5, {%rs4, %rs3};
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
 entry:
   %log2 = call <2 x half> @llvm.log2.v2f16(<2 x half> %in)
@@ -123,8 +122,7 @@ define bfloat @log2_bf16_test(bfloat %in) {
 ; CHECK-LABEL: log2_bf16_test(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b16 %r1, [log2_bf16_test_param_0];
@@ -136,8 +134,8 @@ define bfloat @log2_bf16_test(bfloat %in) {
 ; CHECK-NEXT:    setp.nan.f32 %p1, %r3, %r3;
 ; CHECK-NEXT:    or.b32 %r7, %r3, 4194304;
 ; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    shr.u32 %r9, %r8, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r9;
 ; CHECK-NEXT:    ret;
 entry:
   %log2 = call bfloat @llvm.log2.bf16(bfloat %in)
@@ -149,8 +147,7 @@ define bfloat @log2_bf16_ftz_test(bfloat %in) #0 {
 ; CHECK-LABEL: log2_bf16_ftz_test(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b16 %r1, [log2_bf16_ftz_test_param_0];
@@ -162,8 +159,8 @@ define bfloat @log2_bf16_ftz_test(bfloat %in) #0 {
 ; CHECK-NEXT:    setp.nan.ftz.f32 %p1, %r3, %r3;
 ; CHECK-NEXT:    or.b32 %r7, %r3, 4194304;
 ; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
-; CHECK-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    shr.u32 %r9, %r8, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r9;
 ; CHECK-NEXT:    ret;
 entry:
   %log2 = call bfloat @llvm.log2.bf16(bfloat %in)
@@ -176,31 +173,30 @@ define <2 x bfloat> @log2_bf16_test_v(<2 x bfloat> %in) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<19>;
+; CHECK-NEXT:    .reg .b32 %r<18>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
-; CHECK-NEXT:    ld.param.b32 %r1, [log2_bf16_test_v_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.u32.u16 %r2, %rs2;
-; CHECK-NEXT:    shl.b32 %r3, %r2, 16;
-; CHECK-NEXT:    lg2.approx.f32 %r4, %r3;
-; CHECK-NEXT:    bfe.u32 %r5, %r4, 16, 1;
-; CHECK-NEXT:    add.s32 %r6, %r5, %r4;
-; CHECK-NEXT:    add.s32 %r7, %r6, 32767;
-; CHECK-NEXT:    setp.nan.f32 %p1, %r4, %r4;
-; CHECK-NEXT:    or.b32 %r8, %r4, 4194304;
-; CHECK-NEXT:    selp.b32 %r9, %r8, %r7, %p1;
-; CHECK-NEXT:    cvt.u32.u16 %r10, %rs1;
-; CHECK-NEXT:    shl.b32 %r11, %r10, 16;
-; CHECK-NEXT:    lg2.approx.f32 %r12, %r11;
-; CHECK-NEXT:    bfe.u32 %r13, %r12, 16, 1;
-; CHECK-NEXT:    add.s32 %r14, %r13, %r12;
-; CHECK-NEXT:    add.s32 %r15, %r14, 32767;
-; CHECK-NEXT:    setp.nan.f32 %p2, %r12, %r12;
-; CHECK-NEXT:    or.b32 %r16, %r12, 4194304;
-; CHECK-NEXT:    selp.b32 %r17, %r16, %r15, %p2;
-; CHECK-NEXT:    prmt.b32 %r18, %r17, %r9, 0x7632U;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r18;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [log2_bf16_test_v_param_0];
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-NEXT:    lg2.approx.f32 %r3, %r2;
+; CHECK-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; CHECK-NEXT:    add.s32 %r5, %r4, %r3;
+; CHECK-NEXT:    add.s32 %r6, %r5, 32767;
+; CHECK-NEXT:    setp.nan.f32 %p1, %r3, %r3;
+; CHECK-NEXT:    or.b32 %r7, %r3, 4194304;
+; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    cvt.u32.u16 %r9, %rs1;
+; CHECK-NEXT:    shl.b32 %r10, %r9, 16;
+; CHECK-NEXT:    lg2.approx.f32 %r11, %r10;
+; CHECK-NEXT:    bfe.u32 %r12, %r11, 16, 1;
+; CHECK-NEXT:    add.s32 %r13, %r12, %r11;
+; CHECK-NEXT:    add.s32 %r14, %r13, 32767;
+; CHECK-NEXT:    setp.nan.f32 %p2, %r11, %r11;
+; CHECK-NEXT:    or.b32 %r15, %r11, 4194304;
+; CHECK-NEXT:    selp.b32 %r16, %r15, %r14, %p2;
+; CHECK-NEXT:    prmt.b32 %r17, %r16, %r8, 0x7632U;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r17;
 ; CHECK-NEXT:    ret;
 entry:
   %log2 = call <2 x bfloat> @llvm.log2.v2bf16(<2 x bfloat> %in)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
index 9c11f169a89d..92293ab171a1 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -fp-contract=fast | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -fp-contract=fast | %ptxas-verify -arch=sm_80 %}
 
 ; Using FTZ should emit fma.ftz.relu for f16, not for bf16
-; RUN: llc < %s -denormal-fp-math-f32=preserve-sign -mtriple=nvptx64 -mcpu=sm_80 | FileCheck %s --check-prefixes=CHECK-FTZ
-; RUN: %if ptxas %{ llc < %s -denormal-fp-math-f32=preserve-sign -mtriple=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+; RUN: llc < %s -denormal-fp-math-f32=preserve-sign -mtriple=nvptx64 -mcpu=sm_80 -fp-contract=fast | FileCheck %s --check-prefixes=CHECK-FTZ
+; RUN: %if ptxas %{ llc < %s -denormal-fp-math-f32=preserve-sign -fp-contract=fast -mtriple=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
 
 ; SM < 80 or (which needs PTX version >= 70) should not emit fma{.ftz}.relu
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefixes=CHECK-SM70
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -fp-contract=fast | FileCheck %s --check-prefixes=CHECK-SM70
 
 define half @fma_f16_expanded_no_nans(half %a, half %b, half %c) #0 {
 ; CHECK-LABEL: fma_f16_expanded_no_nans(
@@ -119,7 +119,7 @@ define half @fma_f16_expanded_no_nans_multiple_uses_of_fma(half %a, half %b, hal
   ret half %6
 }
 
-define half @fma_f16_expanded_unsafe_with_nans(half %a, half %b, half %c) #1 {
+define half @fma_f16_expanded_unsafe_with_nans(half %a, half %b, half %c) {
 ; CHECK-LABEL: fma_f16_expanded_unsafe_with_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
@@ -216,7 +216,7 @@ define half @fma_f16_expanded_maxnum_no_nans(half %a, half %b, half %c) #0 {
   ret half %3
 }
 
-define bfloat @fma_bf16_expanded_unsafe_with_nans(bfloat %a, bfloat %b, bfloat %c) #1 {
+define bfloat @fma_bf16_expanded_unsafe_with_nans(bfloat %a, bfloat %b, bfloat %c) {
 ; CHECK-LABEL: fma_bf16_expanded_unsafe_with_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
@@ -384,8 +384,8 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
 ; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<29>;
+; CHECK-SM70-NEXT:    .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<30>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
@@ -422,8 +422,8 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r23, %r23;
 ; CHECK-SM70-NEXT:    or.b32 %r27, %r23, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r28, %r27, %r26, %p4;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r28; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT:    shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r29;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul bfloat %a, %b
   %2 = fadd bfloat %1, %c
@@ -462,8 +462,7 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
 ; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<20>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<21>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
@@ -487,8 +486,8 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r14, %r14;
 ; CHECK-SM70-NEXT:    or.b32 %r18, %r14, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r20, %r19, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r20;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul bfloat %a, %b
   %2 = fadd bfloat %1, %c
@@ -502,10 +501,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -514,10 +513,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -525,20 +524,19 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<7>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<6>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs3, %rs2, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x half> %a, %b
   %2 = fadd <2 x half> %1, %c
@@ -554,10 +552,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 %r7, 1191200512;
@@ -571,10 +569,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<10>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 %r7, 1191200512;
@@ -590,10 +588,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -614,16 +612,16 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
   ret <2 x half> %6
 }
 
-define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #1 {
+define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
 ; CHECK-LABEL: fma_f16x2_expanded_unsafe_with_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -634,10 +632,10 @@ define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half>
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<7>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -647,20 +645,19 @@ define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half>
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<7>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<6>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs3, %rs2, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x half> %a, %b
   %2 = fadd <2 x half> %1, %c
@@ -675,10 +672,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -687,10 +684,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -700,10 +697,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    cvt.f32.f16 %r5, %rs2;
 ; CHECK-SM70-NEXT:    max.f32 %r6, %r5, 0f00000000;
@@ -720,16 +717,16 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
   ret <2 x half> %3
 }
 
-define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #1 {
+define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
 ; CHECK-LABEL: fma_bf16x2_expanded_unsafe_with_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -740,10 +737,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<7>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -753,51 +750,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<31>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x bfloat> %a, %b
   %2 = fadd <2 x bfloat> %1, %c
@@ -812,10 +805,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -824,10 +817,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -835,51 +828,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<31>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x bfloat> %a, %b
   %2 = fadd <2 x bfloat> %1, %c
@@ -895,10 +884,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-NEXT:    .reg .b32 %r<11>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 %r7, 1065369472;
@@ -914,10 +903,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<24>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -948,85 +937,82 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<9>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<61>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<58>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    add.f32 %r30, %r29, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r31, %r30, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, %r30;
-; CHECK-SM70-NEXT:    add.s32 %r33, %r32, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r30, %r30;
-; CHECK-SM70-NEXT:    or.b32 %r34, %r30, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r35, %r34, %r33, %p5;
-; CHECK-SM70-NEXT:    add.f32 %r36, %r28, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p6;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r42, %rs10;
-; CHECK-SM70-NEXT:    shl.b32 %r43, %r42, 16;
-; CHECK-SM70-NEXT:    and.b32 %r44, %r41, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r45, %r43, %r44;
-; CHECK-SM70-NEXT:    bfe.u32 %r46, %r45, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r47, %r46, %r45;
-; CHECK-SM70-NEXT:    add.s32 %r48, %r47, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p7, %r45, %r45;
-; CHECK-SM70-NEXT:    or.b32 %r49, %r45, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r51, %rs9;
-; CHECK-SM70-NEXT:    shl.b32 %r52, %r51, 16;
-; CHECK-SM70-NEXT:    and.b32 %r53, %r35, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r54, %r52, %r53;
-; CHECK-SM70-NEXT:    bfe.u32 %r55, %r54, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r56, %r55, %r54;
-; CHECK-SM70-NEXT:    add.s32 %r57, %r56, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p8, %r54, %r54;
-; CHECK-SM70-NEXT:    or.b32 %r58, %r54, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r59, %r58, %r57, %p8;
-; CHECK-SM70-NEXT:    prmt.b32 %r60, %r59, %r50, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT:    add.f32 %r27, %r26, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r28, %r27, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, %r27;
+; CHECK-SM70-NEXT:    add.s32 %r30, %r29, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r27, %r27;
+; CHECK-SM70-NEXT:    or.b32 %r31, %r27, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r32, %r31, %r30, %p5;
+; CHECK-SM70-NEXT:    add.f32 %r33, %r25, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p6;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r39, %rs10;
+; CHECK-SM70-NEXT:    shl.b32 %r40, %r39, 16;
+; CHECK-SM70-NEXT:    and.b32 %r41, %r38, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r42, %r40, %r41;
+; CHECK-SM70-NEXT:    bfe.u32 %r43, %r42, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r44, %r43, %r42;
+; CHECK-SM70-NEXT:    add.s32 %r45, %r44, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p7, %r42, %r42;
+; CHECK-SM70-NEXT:    or.b32 %r46, %r42, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r47, %r46, %r45, %p7;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r48, %rs9;
+; CHECK-SM70-NEXT:    shl.b32 %r49, %r48, 16;
+; CHECK-SM70-NEXT:    and.b32 %r50, %r32, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r51, %r49, %r50;
+; CHECK-SM70-NEXT:    bfe.u32 %r52, %r51, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r53, %r52, %r51;
+; CHECK-SM70-NEXT:    add.s32 %r54, %r53, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p8, %r51, %r51;
+; CHECK-SM70-NEXT:    or.b32 %r55, %r51, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r56, %r55, %r54, %p8;
+; CHECK-SM70-NEXT:    prmt.b32 %r57, %r56, %r47, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r57;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x bfloat> %a, %b
   %2 = fadd <2 x bfloat> %1, %c
@@ -1043,10 +1029,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -1055,10 +1041,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -1066,59 +1052,56 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<43>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<40>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r39;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul <2 x bfloat> %a, %b
   %2 = fadd <2 x bfloat> %1, %c
@@ -1126,5 +1109,4 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
   ret <2 x bfloat> %3
 }
 
-attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="true" }
-attributes #1 = { "unsafe-fp-math"="true" }
+attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" }
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index c725b797526a..2f1d7d632143 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -253,8 +253,7 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
 ; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<4>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<28>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
@@ -286,8 +285,8 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r21, %r21;
 ; CHECK-SM70-NEXT:    or.b32 %r25, %r21, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r26, %r25, %r24, %p3;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r26; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r27, %r26, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r27;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = fcmp ogt bfloat %1, 0.0
@@ -325,8 +324,7 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<20>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<21>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_maxnum_no_nans_param_2];
@@ -350,8 +348,8 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r14, %r14;
 ; CHECK-SM70-NEXT:    or.b32 %r18, %r14, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r20, %r19, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r20;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = call bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
@@ -364,10 +362,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -376,10 +374,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -387,20 +385,19 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<7>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<6>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs3, %rs2, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
   %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
   %2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
@@ -415,10 +412,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-NEXT:    add.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    add.f16x2 %r7, %r6, %r4;
@@ -430,10 +427,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<8>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-FTZ-NEXT:    add.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    add.ftz.f16x2 %r7, %r6, %r4;
@@ -445,10 +442,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-SM70-NEXT:    .reg .b32 %r<8>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-SM70-NEXT:    add.f16x2 %r6, %r4, %r5;
 ; CHECK-SM70-NEXT:    add.f16x2 %r7, %r6, %r4;
@@ -468,10 +465,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -480,10 +477,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -493,10 +490,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    cvt.f32.f16 %r5, %rs2;
 ; CHECK-SM70-NEXT:    max.f32 %r6, %r5, 0f00000000;
@@ -518,10 +515,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -530,10 +527,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -541,51 +538,47 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<31>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
   %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -600,10 +593,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 1065369472;
 ; CHECK-NEXT:    mov.b32 %r6, 1088438496;
 ; CHECK-NEXT:    fma.rn.bf16x2 %r7, %r4, %r5, %r6;
@@ -617,10 +610,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<18>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r6, %r5, 16;
@@ -644,75 +637,72 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<7>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<57>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<54>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r29, %r28, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r36, %r35, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    and.b32 %r42, %r41, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r43, %r42, %r35;
-; CHECK-SM70-NEXT:    bfe.u32 %r44, %r43, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r45, %r44, %r43;
-; CHECK-SM70-NEXT:    add.s32 %r46, %r45, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r43, %r43;
-; CHECK-SM70-NEXT:    or.b32 %r47, %r43, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r48, %r47, %r46, %p5;
-; CHECK-SM70-NEXT:    and.b32 %r49, %r34, -65536;
-; CHECK-SM70-NEXT:    add.f32 %r50, %r49, %r28;
-; CHECK-SM70-NEXT:    bfe.u32 %r51, %r50, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r52, %r51, %r50;
-; CHECK-SM70-NEXT:    add.s32 %r53, %r52, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r50, %r50;
-; CHECK-SM70-NEXT:    or.b32 %r54, %r50, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r55, %r54, %r53, %p6;
-; CHECK-SM70-NEXT:    prmt.b32 %r56, %r55, %r48, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r26, %r25, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r33, %r32, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    and.b32 %r39, %r38, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r40, %r39, %r32;
+; CHECK-SM70-NEXT:    bfe.u32 %r41, %r40, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r42, %r41, %r40;
+; CHECK-SM70-NEXT:    add.s32 %r43, %r42, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r40, %r40;
+; CHECK-SM70-NEXT:    or.b32 %r44, %r40, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r45, %r44, %r43, %p5;
+; CHECK-SM70-NEXT:    and.b32 %r46, %r31, -65536;
+; CHECK-SM70-NEXT:    add.f32 %r47, %r46, %r25;
+; CHECK-SM70-NEXT:    bfe.u32 %r48, %r47, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r49, %r48, %r47;
+; CHECK-SM70-NEXT:    add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r47, %r47;
+; CHECK-SM70-NEXT:    or.b32 %r51, %r47, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r52, %r51, %r50, %p6;
+; CHECK-SM70-NEXT:    prmt.b32 %r53, %r52, %r45, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r53;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -728,10 +718,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -740,10 +730,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -751,59 +741,56 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<43>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<40>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r39;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = call <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
index 6b462f846859..090dd00f1b93 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -271,8 +271,8 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
 ; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<29>;
+; CHECK-SM70-NEXT:    .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<30>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
@@ -309,8 +309,8 @@ define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r23, %r23;
 ; CHECK-SM70-NEXT:    or.b32 %r27, %r23, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r28, %r27, %r26, %p4;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r28; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT:    shr.u32 %r29, %r28, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r29;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast bfloat %a, %b
   %2 = fadd fast bfloat %1, %c
@@ -351,8 +351,7 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
 ; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<20>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<21>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
@@ -376,8 +375,8 @@ define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r14, %r14;
 ; CHECK-SM70-NEXT:    or.b32 %r18, %r14, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r20, %r19, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r20;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast bfloat %a, %b
   %2 = fadd fast bfloat %1, %c
@@ -393,10 +392,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -405,10 +404,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -416,20 +415,19 @@ define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<7>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<6>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs3, %rs2, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast <2 x half> %a, %b
   %2 = fadd fast <2 x half> %1, %c
@@ -447,10 +445,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 %r7, 1191200512;
@@ -464,10 +462,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<10>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 %r7, 1191200512;
@@ -483,10 +481,10 @@ define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -513,10 +511,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -525,10 +523,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -538,10 +536,10 @@ define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    cvt.f32.f16 %r5, %rs2;
 ; CHECK-SM70-NEXT:    max.f32 %r6, %r5, 0f00000000;
@@ -564,10 +562,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -576,10 +574,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -587,51 +585,47 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<31>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast <2 x bfloat> %a, %b
   %2 = fadd fast <2 x bfloat> %1, %c
@@ -647,10 +641,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-NEXT:    .reg .b32 %r<11>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    mov.b32 %r7, 1065369472;
@@ -666,10 +660,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<24>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.bf16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
@@ -700,85 +694,82 @@ define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloa
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<9>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<61>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<58>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    add.rn.f32 %r30, %r29, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r31, %r30, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, %r30;
-; CHECK-SM70-NEXT:    add.s32 %r33, %r32, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r30, %r30;
-; CHECK-SM70-NEXT:    or.b32 %r34, %r30, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r35, %r34, %r33, %p5;
-; CHECK-SM70-NEXT:    add.rn.f32 %r36, %r28, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p6;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r42, %rs10;
-; CHECK-SM70-NEXT:    shl.b32 %r43, %r42, 16;
-; CHECK-SM70-NEXT:    and.b32 %r44, %r41, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r45, %r43, %r44;
-; CHECK-SM70-NEXT:    bfe.u32 %r46, %r45, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r47, %r46, %r45;
-; CHECK-SM70-NEXT:    add.s32 %r48, %r47, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p7, %r45, %r45;
-; CHECK-SM70-NEXT:    or.b32 %r49, %r45, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r50, %r49, %r48, %p7;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r51, %rs9;
-; CHECK-SM70-NEXT:    shl.b32 %r52, %r51, 16;
-; CHECK-SM70-NEXT:    and.b32 %r53, %r35, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r54, %r52, %r53;
-; CHECK-SM70-NEXT:    bfe.u32 %r55, %r54, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r56, %r55, %r54;
-; CHECK-SM70-NEXT:    add.s32 %r57, %r56, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p8, %r54, %r54;
-; CHECK-SM70-NEXT:    or.b32 %r58, %r54, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r59, %r58, %r57, %p8;
-; CHECK-SM70-NEXT:    prmt.b32 %r60, %r59, %r50, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT:    add.rn.f32 %r27, %r26, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r28, %r27, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, %r27;
+; CHECK-SM70-NEXT:    add.s32 %r30, %r29, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r27, %r27;
+; CHECK-SM70-NEXT:    or.b32 %r31, %r27, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r32, %r31, %r30, %p5;
+; CHECK-SM70-NEXT:    add.rn.f32 %r33, %r25, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p6;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r39, %rs10;
+; CHECK-SM70-NEXT:    shl.b32 %r40, %r39, 16;
+; CHECK-SM70-NEXT:    and.b32 %r41, %r38, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r42, %r40, %r41;
+; CHECK-SM70-NEXT:    bfe.u32 %r43, %r42, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r44, %r43, %r42;
+; CHECK-SM70-NEXT:    add.s32 %r45, %r44, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p7, %r42, %r42;
+; CHECK-SM70-NEXT:    or.b32 %r46, %r42, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r47, %r46, %r45, %p7;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r48, %rs9;
+; CHECK-SM70-NEXT:    shl.b32 %r49, %r48, 16;
+; CHECK-SM70-NEXT:    and.b32 %r50, %r32, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r51, %r49, %r50;
+; CHECK-SM70-NEXT:    bfe.u32 %r52, %r51, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r53, %r52, %r51;
+; CHECK-SM70-NEXT:    add.s32 %r54, %r53, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p8, %r51, %r51;
+; CHECK-SM70-NEXT:    or.b32 %r55, %r51, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r56, %r55, %r54, %p8;
+; CHECK-SM70-NEXT:    prmt.b32 %r57, %r56, %r47, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r57;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast <2 x bfloat> %a, %b
   %2 = fadd fast <2 x bfloat> %1, %c
@@ -795,10 +786,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -807,10 +798,10 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -818,59 +809,56 @@ define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bf
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<43>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<40>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r39;
 ; CHECK-SM70-NEXT:    ret;
   %1 = fmul fast <2 x bfloat> %a, %b
   %2 = fadd fast <2 x bfloat> %1, %c
@@ -1122,8 +1110,7 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
 ; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<4>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<28>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
@@ -1155,8 +1142,8 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r21, %r21;
 ; CHECK-SM70-NEXT:    or.b32 %r25, %r21, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r26, %r25, %r24, %p3;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r26; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r27, %r26, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r27;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = fcmp nsz ogt bfloat %1, 0.0
@@ -1194,8 +1181,7 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)  {
 ; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
-; CHECK-SM70-NEXT:    .reg .b16 %rs<2>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<20>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<21>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
 ; CHECK-SM70-NEXT:    ld.param.b16 %r1, [fma_bf16_maxnum_no_nans_param_2];
@@ -1219,8 +1205,8 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c)  {
 ; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r14, %r14;
 ; CHECK-SM70-NEXT:    or.b32 %r18, %r14, 4194304;
 ; CHECK-SM70-NEXT:    selp.b32 %r19, %r18, %r17, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
-; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT:    shr.u32 %r20, %r19, 16;
+; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r20;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = call nsz bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
@@ -1233,10 +1219,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 0;
 ; CHECK-NEXT:    max.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -1247,10 +1233,10 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<7>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 0;
 ; CHECK-FTZ-NEXT:    max.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r6;
@@ -1260,20 +1246,19 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<3>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<5>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<7>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<6>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 0;
 ; CHECK-SM70-NEXT:    setp.gt.f16x2 %p1|%p2, %r4, %r5;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs3, %rs2, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
-; CHECK-SM70-NEXT:    mov.b32 %r6, {%rs4, %rs3};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
   %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
   %2 = fcmp nsz ogt <2 x half> %1, <half 0.0, half 0.0>
@@ -1288,10 +1273,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-NEXT:    add.rn.f16x2 %r6, %r4, %r5;
 ; CHECK-NEXT:    add.rn.f16x2 %r7, %r6, %r4;
@@ -1303,10 +1288,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<8>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f16x2 %r6, %r4, %r5;
 ; CHECK-FTZ-NEXT:    add.rn.ftz.f16x2 %r7, %r6, %r4;
@@ -1318,10 +1303,10 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
 ; CHECK-SM70-NEXT:    .reg .b32 %r<8>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 %r5, 1191200512;
 ; CHECK-SM70-NEXT:    add.rn.f16x2 %r6, %r4, %r5;
 ; CHECK-SM70-NEXT:    add.rn.f16x2 %r7, %r6, %r4;
@@ -1341,10 +1326,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -1353,10 +1338,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.ftz.relu.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -1366,10 +1351,10 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-SM70-NEXT:    .reg .b32 %r<10>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_0];
 ; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    fma.rn.f16x2 %r4, %r1, %r2, %r3;
 ; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM70-NEXT:    cvt.f32.f16 %r5, %rs2;
 ; CHECK-SM70-NEXT:    max.f32 %r6, %r5, 0f00000000;
@@ -1391,10 +1376,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -1403,10 +1388,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -1414,51 +1399,47 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<11>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<31>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<27>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r15; }
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r27; }
-; CHECK-SM70-NEXT:    and.b32 %r28, %r15, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    and.b32 %r29, %r27, -65536;
-; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r29, 0f00000000;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs8}, %r24; }
+; CHECK-SM70-NEXT:    and.b32 %r25, %r12, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p3, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    and.b32 %r26, %r24, -65536;
+; CHECK-SM70-NEXT:    setp.gt.f32 %p4, %r26, 0f00000000;
 ; CHECK-SM70-NEXT:    selp.b16 %rs9, %rs8, 0x0000, %p4;
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
-; CHECK-SM70-NEXT:    mov.b32 %r30, {%rs10, %rs9};
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
   %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -1473,10 +1454,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    mov.b32 %r5, 1065369472;
 ; CHECK-NEXT:    mov.b32 %r6, 1088438496;
 ; CHECK-NEXT:    fma.rn.bf16x2 %r7, %r4, %r5, %r6;
@@ -1490,10 +1471,10 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<18>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-FTZ-NEXT:    cvt.u32.u16 %r5, %rs2;
 ; CHECK-FTZ-NEXT:    shl.b32 %r6, %r5, 16;
@@ -1517,75 +1498,72 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<7>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<57>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<54>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r29, %r28, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r36, %r35, 0f40E00000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    and.b32 %r42, %r41, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r43, %r42, %r35;
-; CHECK-SM70-NEXT:    bfe.u32 %r44, %r43, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r45, %r44, %r43;
-; CHECK-SM70-NEXT:    add.s32 %r46, %r45, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r43, %r43;
-; CHECK-SM70-NEXT:    or.b32 %r47, %r43, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r48, %r47, %r46, %p5;
-; CHECK-SM70-NEXT:    and.b32 %r49, %r34, -65536;
-; CHECK-SM70-NEXT:    add.rn.f32 %r50, %r49, %r28;
-; CHECK-SM70-NEXT:    bfe.u32 %r51, %r50, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r52, %r51, %r50;
-; CHECK-SM70-NEXT:    add.s32 %r53, %r52, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r50, %r50;
-; CHECK-SM70-NEXT:    or.b32 %r54, %r50, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r55, %r54, %r53, %p6;
-; CHECK-SM70-NEXT:    prmt.b32 %r56, %r55, %r48, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r26, %r25, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r33, %r32, 0f40E00000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    and.b32 %r39, %r38, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r40, %r39, %r32;
+; CHECK-SM70-NEXT:    bfe.u32 %r41, %r40, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r42, %r41, %r40;
+; CHECK-SM70-NEXT:    add.s32 %r43, %r42, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p5, %r40, %r40;
+; CHECK-SM70-NEXT:    or.b32 %r44, %r40, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r45, %r44, %r43, %p5;
+; CHECK-SM70-NEXT:    and.b32 %r46, %r31, -65536;
+; CHECK-SM70-NEXT:    add.rn.f32 %r47, %r46, %r25;
+; CHECK-SM70-NEXT:    bfe.u32 %r48, %r47, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r49, %r48, %r47;
+; CHECK-SM70-NEXT:    add.s32 %r50, %r49, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p6, %r47, %r47;
+; CHECK-SM70-NEXT:    or.b32 %r51, %r47, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r52, %r51, %r50, %p6;
+; CHECK-SM70-NEXT:    prmt.b32 %r53, %r52, %r45, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r53;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
@@ -1601,10 +1579,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
 ;
@@ -1613,10 +1591,10 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
 ; CHECK-FTZ-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT:    fma.rn.relu.bf16x2 %r4, %r1, %r2, %r3;
 ; CHECK-FTZ-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-FTZ-NEXT:    ret;
 ;
@@ -1624,59 +1602,56 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-SM70:       {
 ; CHECK-SM70-NEXT:    .reg .pred %p<5>;
 ; CHECK-SM70-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM70-NEXT:    .reg .b32 %r<43>;
+; CHECK-SM70-NEXT:    .reg .b32 %r<40>;
 ; CHECK-SM70-EMPTY:
 ; CHECK-SM70-NEXT:  // %bb.0:
-; CHECK-SM70-NEXT:    ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
-; CHECK-SM70-NEXT:    ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
-; CHECK-SM70-NEXT:    ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
-; CHECK-SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r4, %rs1;
-; CHECK-SM70-NEXT:    shl.b32 %r5, %r4, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r6, %rs3;
-; CHECK-SM70-NEXT:    shl.b32 %r7, %r6, 16;
-; CHECK-SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r8, %rs5;
-; CHECK-SM70-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r10, %r9, %r7, %r5;
-; CHECK-SM70-NEXT:    bfe.u32 %r11, %r10, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r12, %r11, %r10;
-; CHECK-SM70-NEXT:    add.s32 %r13, %r12, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r10, %r10;
-; CHECK-SM70-NEXT:    or.b32 %r14, %r10, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r15, %r14, %r13, %p1;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r16, %rs2;
-; CHECK-SM70-NEXT:    shl.b32 %r17, %r16, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r18, %rs4;
-; CHECK-SM70-NEXT:    shl.b32 %r19, %r18, 16;
-; CHECK-SM70-NEXT:    cvt.u32.u16 %r20, %rs6;
-; CHECK-SM70-NEXT:    shl.b32 %r21, %r20, 16;
-; CHECK-SM70-NEXT:    fma.rn.f32 %r22, %r21, %r19, %r17;
-; CHECK-SM70-NEXT:    bfe.u32 %r23, %r22, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r24, %r23, %r22;
-; CHECK-SM70-NEXT:    add.s32 %r25, %r24, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r22, %r22;
-; CHECK-SM70-NEXT:    or.b32 %r26, %r22, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r27, %r26, %r25, %p2;
-; CHECK-SM70-NEXT:    and.b32 %r28, %r27, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r29, %r28, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r30, %r29, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r31, %r30, %r29;
-; CHECK-SM70-NEXT:    add.s32 %r32, %r31, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r29, %r29;
-; CHECK-SM70-NEXT:    or.b32 %r33, %r29, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r34, %r33, %r32, %p3;
-; CHECK-SM70-NEXT:    and.b32 %r35, %r15, -65536;
-; CHECK-SM70-NEXT:    max.f32 %r36, %r35, 0f00000000;
-; CHECK-SM70-NEXT:    bfe.u32 %r37, %r36, 16, 1;
-; CHECK-SM70-NEXT:    add.s32 %r38, %r37, %r36;
-; CHECK-SM70-NEXT:    add.s32 %r39, %r38, 32767;
-; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r36, %r36;
-; CHECK-SM70-NEXT:    or.b32 %r40, %r36, 4194304;
-; CHECK-SM70-NEXT:    selp.b32 %r41, %r40, %r39, %p4;
-; CHECK-SM70-NEXT:    prmt.b32 %r42, %r41, %r34, 0x7632U;
-; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; CHECK-SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r3, %rs5;
+; CHECK-SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r5, %rs1;
+; CHECK-SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
+; CHECK-SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p1, %r7, %r7;
+; CHECK-SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; CHECK-SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r15, %rs6;
+; CHECK-SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; CHECK-SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; CHECK-SM70-NEXT:    fma.rn.f32 %r19, %r18, %r16, %r14;
+; CHECK-SM70-NEXT:    bfe.u32 %r20, %r19, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r21, %r20, %r19;
+; CHECK-SM70-NEXT:    add.s32 %r22, %r21, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p2, %r19, %r19;
+; CHECK-SM70-NEXT:    or.b32 %r23, %r19, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r24, %r23, %r22, %p2;
+; CHECK-SM70-NEXT:    and.b32 %r25, %r24, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r26, %r25, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r27, %r26, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r28, %r27, %r26;
+; CHECK-SM70-NEXT:    add.s32 %r29, %r28, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p3, %r26, %r26;
+; CHECK-SM70-NEXT:    or.b32 %r30, %r26, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r31, %r30, %r29, %p3;
+; CHECK-SM70-NEXT:    and.b32 %r32, %r12, -65536;
+; CHECK-SM70-NEXT:    max.f32 %r33, %r32, 0f00000000;
+; CHECK-SM70-NEXT:    bfe.u32 %r34, %r33, 16, 1;
+; CHECK-SM70-NEXT:    add.s32 %r35, %r34, %r33;
+; CHECK-SM70-NEXT:    add.s32 %r36, %r35, 32767;
+; CHECK-SM70-NEXT:    setp.nan.f32 %p4, %r33, %r33;
+; CHECK-SM70-NEXT:    or.b32 %r37, %r33, 4194304;
+; CHECK-SM70-NEXT:    selp.b32 %r38, %r37, %r36, %p4;
+; CHECK-SM70-NEXT:    prmt.b32 %r39, %r38, %r31, 0x7632U;
+; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r39;
 ; CHECK-SM70-NEXT:    ret;
   %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
diff --git a/llvm/test/CodeGen/NVPTX/fma.ll b/llvm/test/CodeGen/NVPTX/fma.ll
index 327851725991..5aa12b08a381 100644
--- a/llvm/test/CodeGen/NVPTX/fma.ll
+++ b/llvm/test/CodeGen/NVPTX/fma.ll
@@ -40,12 +40,7 @@ define ptx_device float @t2_f32(float %x, float %y, float %z, float %w) {
 ; CHECK-NEXT:    .param .b32 param1;
 ; CHECK-NEXT:    st.param.b32 [param1], %r6;
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    dummy_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), dummy_f32, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r7, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
@@ -92,12 +87,7 @@ define ptx_device double @t2_f64(double %x, double %y, double %z, double %w) {
 ; CHECK-NEXT:    .param .b64 param1;
 ; CHECK-NEXT:    st.param.b64 [param1], %rd6;
 ; CHECK-NEXT:    .param .b64 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    dummy_f64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), dummy_f64, (param0, param1);
 ; CHECK-NEXT:    ld.param.b64 %rd7, [retval0];
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd7;
@@ -126,11 +116,10 @@ define ptx_device float @f32_iir(float %x) {
 define ptx_device float @f32_iii(float %x) {
 ; CHECK-LABEL: f32_iii(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0f41200000;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1092616192;
 ; CHECK-NEXT:    ret;
   %r = call float @llvm.fma.f32(float 2.0, float 3.0, float 4.0)
   ret float %r
diff --git a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
index d253df5ed1b9..ed8f6b451107 100644
--- a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
+++ b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
@@ -50,11 +50,7 @@ define void @test_ld_param_escaping(ptr byval(i32) %a) {
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], %rd2;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    escape,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni escape, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    ret;
   call void @escape(ptr %a)
@@ -72,11 +68,7 @@ define void @test_ld_param_byval(ptr byval(i32) %a) {
 ; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.b32 [param0], %r1;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    byval_user,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni byval_user, (param0);
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    ret;
   call void @byval_user(ptr %a)
diff --git a/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
index d40f514acd40..de69d02ded5e 100644
--- a/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
+++ b/llvm/test/CodeGen/NVPTX/fp128-storage-type.ll
@@ -42,11 +42,7 @@ define void @call(fp128 %x) {
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {%rd1, %rd2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    ret;
   call void @call(fp128 %x)
diff --git a/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll b/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
new file mode 100644
index 000000000000..42655538cc7a
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
+
+;; This test is intended to verify that we don't crash when -opt-bisect-limit
+;; is used in conjunction with lifetime markers. Previously, later passes
+;; would not handle these intructions correctly and relied on earlier passes
+;; to remove them.
+
+declare void @bar(ptr)
+
+define void @foo() {
+  %p = alloca i32
+  call void @llvm.lifetime.start(i64 4, ptr %p)
+  call void @bar(ptr %p)
+  call void @llvm.lifetime.end(i64 4, ptr %p)
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/frem.ll b/llvm/test/CodeGen/NVPTX/frem.ll
index 909f2534f821..5805aed1bebe 100644
--- a/llvm/test/CodeGen/NVPTX/frem.ll
+++ b/llvm/test/CodeGen/NVPTX/frem.ll
@@ -222,25 +222,52 @@ define double @frem_f64_ninf(double %a, double %b) {
   ret double %r
 }
 
-define float @frem_f32_imm1(float %a) {
-; FAST-LABEL: frem_f32_imm1(
+define float @frem_f32_imm1_fast(float %a) {
+; FAST-LABEL: frem_f32_imm1_fast(
 ; FAST:       {
 ; FAST-NEXT:    .reg .b32 %r<5>;
 ; FAST-EMPTY:
 ; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_param_0];
+; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
 ; FAST-NEXT:    mul.f32 %r2, %r1, 0f3E124925;
 ; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
 ; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
 ; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
 ; FAST-NEXT:    ret;
 ;
-; NORMAL-LABEL: frem_f32_imm1(
+; NORMAL-LABEL: frem_f32_imm1_fast(
 ; NORMAL:       {
 ; NORMAL-NEXT:    .reg .b32 %r<5>;
 ; NORMAL-EMPTY:
 ; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_param_0];
+; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
+; NORMAL-NEXT:    mul.rn.f32 %r2, %r1, 0f3E124925;
+; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; NORMAL-NEXT:    st.param.b32 [func_retval0], %r4;
+; NORMAL-NEXT:    ret;
+  %r = frem arcp float %a, 7.0
+  ret float %r
+}
+define float @frem_f32_imm1_normal(float %a) {
+; FAST-LABEL: frem_f32_imm1_normal(
+; FAST:       {
+; FAST-NEXT:    .reg .b32 %r<5>;
+; FAST-EMPTY:
+; FAST-NEXT:  // %bb.0:
+; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
+; FAST-NEXT:    div.approx.f32 %r2, %r1, 0f40E00000;
+; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
+; FAST-NEXT:    ret;
+;
+; NORMAL-LABEL: frem_f32_imm1_normal(
+; NORMAL:       {
+; NORMAL-NEXT:    .reg .b32 %r<5>;
+; NORMAL-EMPTY:
+; NORMAL-NEXT:  // %bb.0:
+; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
 ; NORMAL-NEXT:    div.rn.f32 %r2, %r1, 0f40E00000;
 ; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
 ; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
diff --git a/llvm/test/CodeGen/NVPTX/i1-icmp.ll b/llvm/test/CodeGen/NVPTX/i1-icmp.ll
index e43a9da88a50..5d5e861432b2 100644
--- a/llvm/test/CodeGen/NVPTX/i1-icmp.ll
+++ b/llvm/test/CodeGen/NVPTX/i1-icmp.ll
@@ -8,7 +8,7 @@ define i32 @icmp_i1_eq(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_eq(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_eq_param_0];
@@ -18,12 +18,10 @@ define i32 @icmp_i1_eq(i32 %a, i32 %b) {
 ; CHECK-NEXT:    xor.pred %p3, %p1, %p2;
 ; CHECK-NEXT:    @%p3 bra $L__BB0_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB0_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -39,7 +37,7 @@ define i32 @icmp_i1_ne(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_ne(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_ne_param_0];
@@ -50,12 +48,10 @@ define i32 @icmp_i1_ne(i32 %a, i32 %b) {
 ; CHECK-NEXT:    not.pred %p4, %p3;
 ; CHECK-NEXT:    @%p4 bra $L__BB1_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -71,7 +67,7 @@ define i32 @icmp_i1_sgt(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_sgt(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_sgt_param_0];
@@ -81,12 +77,10 @@ define i32 @icmp_i1_sgt(i32 %a, i32 %b) {
 ; CHECK-NEXT:    or.pred %p3, %p1, %p2;
 ; CHECK-NEXT:    @%p3 bra $L__BB2_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB2_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -102,7 +96,7 @@ define i32 @icmp_i1_slt(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_slt(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_slt_param_0];
@@ -112,12 +106,10 @@ define i32 @icmp_i1_slt(i32 %a, i32 %b) {
 ; CHECK-NEXT:    or.pred %p3, %p2, %p1;
 ; CHECK-NEXT:    @%p3 bra $L__BB3_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB3_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -133,7 +125,7 @@ define i32 @icmp_i1_sge(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_sge(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_sge_param_0];
@@ -143,12 +135,10 @@ define i32 @icmp_i1_sge(i32 %a, i32 %b) {
 ; CHECK-NEXT:    and.pred %p3, %p1, %p2;
 ; CHECK-NEXT:    @%p3 bra $L__BB4_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB4_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -164,7 +154,7 @@ define i32 @icmp_i1_sle(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_sle(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_sle_param_0];
@@ -174,12 +164,10 @@ define i32 @icmp_i1_sle(i32 %a, i32 %b) {
 ; CHECK-NEXT:    and.pred %p3, %p2, %p1;
 ; CHECK-NEXT:    @%p3 bra $L__BB5_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB5_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -195,7 +183,7 @@ define i32 @icmp_i1_uge(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_uge(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_uge_param_0];
@@ -205,12 +193,10 @@ define i32 @icmp_i1_uge(i32 %a, i32 %b) {
 ; CHECK-NEXT:    and.pred %p3, %p2, %p1;
 ; CHECK-NEXT:    @%p3 bra $L__BB6_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB6_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -226,7 +212,7 @@ define i32 @icmp_i1_ugt(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_ugt(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_ugt_param_0];
@@ -236,12 +222,10 @@ define i32 @icmp_i1_ugt(i32 %a, i32 %b) {
 ; CHECK-NEXT:    or.pred %p3, %p2, %p1;
 ; CHECK-NEXT:    @%p3 bra $L__BB7_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB7_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -257,7 +241,7 @@ define i32 @icmp_i1_ule(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_ule(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_ule_param_0];
@@ -267,12 +251,10 @@ define i32 @icmp_i1_ule(i32 %a, i32 %b) {
 ; CHECK-NEXT:    and.pred %p3, %p1, %p2;
 ; CHECK-NEXT:    @%p3 bra $L__BB8_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB8_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
@@ -288,7 +270,7 @@ define i32 @icmp_i1_ult(i32 %a, i32 %b) {
 ; CHECK-LABEL: icmp_i1_ult(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [icmp_i1_ult_param_0];
@@ -298,12 +280,10 @@ define i32 @icmp_i1_ult(i32 %a, i32 %b) {
 ; CHECK-NEXT:    or.pred %p3, %p1, %p2;
 ; CHECK-NEXT:    @%p3 bra $L__BB9_2;
 ; CHECK-NEXT:  // %bb.1: // %bb1
-; CHECK-NEXT:    mov.b32 %r4, 1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 1;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB9_2: // %bb2
-; CHECK-NEXT:    mov.b32 %r3, 127;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 127;
 ; CHECK-NEXT:    ret;
   %p1 = icmp sgt i32 %a, 1
   %p2 = icmp sgt i32 %b, 1
diff --git a/llvm/test/CodeGen/NVPTX/i1-load-lower.ll b/llvm/test/CodeGen/NVPTX/i1-load-lower.ll
index 50d39c88a46b..5214d272e161 100644
--- a/llvm/test/CodeGen/NVPTX/i1-load-lower.ll
+++ b/llvm/test/CodeGen/NVPTX/i1-load-lower.ll
@@ -10,14 +10,13 @@ target triple = "nvptx-nvidia-cuda"
 define void @foo() {
 ; CHECK-LABEL: foo(
 ; CHECK:    .reg .pred %p<2>;
-; CHECK:    .reg .b16 %rs<4>;
+; CHECK:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK:    ld.global.b8 %rs1, [i1g];
 ; CHECK:    and.b16 %rs2, %rs1, 1;
 ; CHECK:    setp.ne.b16 %p1, %rs2, 0;
 ; CHECK:    @%p1 bra $L__BB0_2;
-; CHECK:    mov.b16 %rs3, 1;
-; CHECK:    st.global.b8 [i1g], %rs3;
+; CHECK:    st.global.b8 [i1g], 1;
 ; CHECK:    ret;
   %tmp = load i1, ptr addrspace(1) @i1g, align 2
   br i1 %tmp, label %if.end, label %if.then
diff --git a/llvm/test/CodeGen/NVPTX/i1-select.ll b/llvm/test/CodeGen/NVPTX/i1-select.ll
index 6fb5aad4b1eb..562c746200d8 100644
--- a/llvm/test/CodeGen/NVPTX/i1-select.ll
+++ b/llvm/test/CodeGen/NVPTX/i1-select.ll
@@ -94,27 +94,27 @@ define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %fals
 define i32 @test_select_i1_basic_folding(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) {
 ; CHECK-LABEL: test_select_i1_basic_folding(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<13>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .pred %p<12>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_select_i1_basic_folding_param_0];
 ; CHECK-NEXT:    setp.eq.s32 %p1, %r1, 0;
-; CHECK-NEXT:    ld.param.b32 %r2, [test_select_i1_basic_folding_param_1];
-; CHECK-NEXT:    setp.ne.s32 %p2, %r2, 0;
-; CHECK-NEXT:    setp.eq.s32 %p3, %r2, 0;
-; CHECK-NEXT:    ld.param.b32 %r3, [test_select_i1_basic_folding_param_2];
-; CHECK-NEXT:    setp.eq.s32 %p4, %r3, 0;
-; CHECK-NEXT:    ld.param.b32 %r4, [test_select_i1_basic_folding_param_3];
+; CHECK-NEXT:    ld.param.b32 %r3, [test_select_i1_basic_folding_param_1];
+; CHECK-NEXT:    setp.ne.s32 %p2, %r3, 0;
+; CHECK-NEXT:    setp.eq.s32 %p3, %r3, 0;
+; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_folding_param_2];
+; CHECK-NEXT:    setp.eq.s32 %p4, %r5, 0;
+; CHECK-NEXT:    ld.param.b32 %r6, [test_select_i1_basic_folding_param_3];
 ; CHECK-NEXT:    xor.pred %p6, %p1, %p3;
-; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_folding_param_4];
+; CHECK-NEXT:    ld.param.b32 %r7, [test_select_i1_basic_folding_param_4];
 ; CHECK-NEXT:    and.pred %p7, %p6, %p4;
-; CHECK-NEXT:    and.pred %p9, %p2, %p4;
-; CHECK-NEXT:    and.pred %p10, %p3, %p7;
-; CHECK-NEXT:    or.pred %p11, %p10, %p9;
-; CHECK-NEXT:    xor.pred %p12, %p11, %p3;
-; CHECK-NEXT:    selp.b32 %r6, %r4, %r5, %p12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    and.pred %p8, %p2, %p4;
+; CHECK-NEXT:    and.pred %p9, %p3, %p7;
+; CHECK-NEXT:    or.pred %p10, %p9, %p8;
+; CHECK-NEXT:    xor.pred %p11, %p10, %p3;
+; CHECK-NEXT:    selp.b32 %r8, %r6, %r7, %p11;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
   %b1 = icmp eq i32 %v1, 0
   %b2 = icmp eq i32 %v2, 0
diff --git a/llvm/test/CodeGen/NVPTX/i128-ld-st.ll b/llvm/test/CodeGen/NVPTX/i128-ld-st.ll
index 6bf65d4d4ad6..83f1ed43960f 100644
--- a/llvm/test/CodeGen/NVPTX/i128-ld-st.ll
+++ b/llvm/test/CodeGen/NVPTX/i128-ld-st.ll
@@ -7,15 +7,14 @@ target triple = "nvptx64-nvidia-cuda"
 define i128 @foo(ptr %p, ptr %o) {
 ; CHECK-LABEL: foo(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd2, [foo_param_1];
 ; CHECK-NEXT:    ld.param.b64 %rd1, [foo_param_0];
 ; CHECK-NEXT:    ld.b8 %rd3, [%rd1];
-; CHECK-NEXT:    mov.b64 %rd4, 0;
-; CHECK-NEXT:    st.v2.b64 [%rd2], {%rd3, %rd4};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT:    st.v2.b64 [%rd2], {%rd3, 0};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, 0};
 ; CHECK-NEXT:    ret;
   %c = load i8, ptr %p, align 1
   %i = zext i8 %c to i128
diff --git a/llvm/test/CodeGen/NVPTX/i128.ll b/llvm/test/CodeGen/NVPTX/i128.ll
index ecd42fd6ceb3..49dbc5c385dc 100644
--- a/llvm/test/CodeGen/NVPTX/i128.ll
+++ b/llvm/test/CodeGen/NVPTX/i128.ll
@@ -5,9 +5,9 @@
 define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: srem_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .pred %p<22>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<127>;
+; CHECK-NEXT:    .reg .b64 %rd<126>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [srem_i128_param_0];
@@ -42,102 +42,103 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd62, %r4;
 ; CHECK-NEXT:    add.s64 %rd63, %rd62, 64;
 ; CHECK-NEXT:    selp.b64 %rd64, %rd61, %rd63, %p7;
-; CHECK-NEXT:    mov.b64 %rd117, 0;
+; CHECK-NEXT:    mov.b64 %rd116, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd66, %rd60, %rd64;
-; CHECK-NEXT:    subc.cc.s64 %rd67, %rd117, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd66, 127;
-; CHECK-NEXT:    setp.eq.s64 %p9, %rd67, 0;
-; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.s64 %p11, %rd67, 0;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd68, %rd66, 127;
-; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd67;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd69, 0;
-; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p13;
-; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p13;
-; CHECK-NEXT:    or.pred %p15, %p13, %p14;
-; CHECK-NEXT:    @%p15 bra $L__BB0_5;
+; CHECK-NEXT:    subc.cc.s64 %rd8, %rd116, 0;
+; CHECK-NEXT:    setp.ne.s64 %p8, %rd8, 0;
+; CHECK-NEXT:    and.pred %p10, %p8, %p8;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd8, 0;
+; CHECK-NEXT:    setp.gt.u64 %p12, %rd66, 127;
+; CHECK-NEXT:    and.pred %p13, %p11, %p12;
+; CHECK-NEXT:    or.pred %p14, %p13, %p10;
+; CHECK-NEXT:    or.pred %p15, %p5, %p14;
+; CHECK-NEXT:    xor.b64 %rd67, %rd66, 127;
+; CHECK-NEXT:    or.b64 %rd68, %rd67, %rd8;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd68, 0;
+; CHECK-NEXT:    selp.b64 %rd125, 0, %rd4, %p15;
+; CHECK-NEXT:    selp.b64 %rd124, 0, %rd3, %p15;
+; CHECK-NEXT:    or.pred %p17, %p15, %p16;
+; CHECK-NEXT:    @%p17 bra $L__BB0_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd66, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd67, 0;
-; CHECK-NEXT:    or.b64 %rd72, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd72, 0;
+; CHECK-NEXT:    add.cc.s64 %rd118, %rd66, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd119, %rd8, 0;
+; CHECK-NEXT:    or.b64 %rd71, %rd118, %rd119;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd71, 0;
 ; CHECK-NEXT:    cvt.u32.u64 %r5, %rd66;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r6;
+; CHECK-NEXT:    shl.b64 %rd72, %rd4, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r7;
-; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
+; CHECK-NEXT:    shr.u64 %rd73, %rd3, %r7;
+; CHECK-NEXT:    or.b64 %rd74, %rd72, %rd73;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r8;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd124, %rd76, %rd75, %p17;
-; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r6;
-; CHECK-NEXT:    mov.b64 %rd114, %rd117;
-; CHECK-NEXT:    @%p16 bra $L__BB0_4;
+; CHECK-NEXT:    shl.b64 %rd75, %rd3, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p19, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd123, %rd75, %rd74, %p19;
+; CHECK-NEXT:    shl.b64 %rd122, %rd3, %r6;
+; CHECK-NEXT:    mov.b64 %rd113, %rd116;
+; CHECK-NEXT:    @%p18 bra $L__BB0_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd119;
-; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd118;
+; CHECK-NEXT:    shr.u64 %rd78, %rd3, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r10;
-; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
+; CHECK-NEXT:    shl.b64 %rd79, %rd4, %r10;
+; CHECK-NEXT:    or.b64 %rd80, %rd78, %rd79;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r11;
-; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd121, %rd82, %rd81, %p18;
-; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r9;
+; CHECK-NEXT:    shr.u64 %rd81, %rd4, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p20, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd120, %rd81, %rd80, %p20;
+; CHECK-NEXT:    shr.u64 %rd121, %rd4, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd5, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd6, -1;
-; CHECK-NEXT:    mov.b64 %rd114, 0;
-; CHECK-NEXT:    mov.b64 %rd117, %rd114;
+; CHECK-NEXT:    mov.b64 %rd113, 0;
+; CHECK-NEXT:    mov.b64 %rd116, %rd113;
 ; CHECK-NEXT:  $L__BB0_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd83, %rd121, 63;
-; CHECK-NEXT:    shl.b64 %rd84, %rd122, 1;
-; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
-; CHECK-NEXT:    shl.b64 %rd86, %rd121, 1;
-; CHECK-NEXT:    shr.u64 %rd87, %rd124, 63;
-; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
-; CHECK-NEXT:    shr.u64 %rd89, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd123, %rd117, %rd92;
-; CHECK-NEXT:    or.b64 %rd124, %rd114, %rd91;
-; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
-; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
-; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
-; CHECK-NEXT:    and.b64 %rd117, %rd95, 1;
-; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd5;
-; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd6;
-; CHECK-NEXT:    sub.cc.s64 %rd121, %rd88, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd122, %rd85, %rd97;
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd119, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd120, -1;
-; CHECK-NEXT:    or.b64 %rd98, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.s64 %p19, %rd98, 0;
-; CHECK-NEXT:    @%p19 bra $L__BB0_4;
+; CHECK-NEXT:    shr.u64 %rd82, %rd120, 63;
+; CHECK-NEXT:    shl.b64 %rd83, %rd121, 1;
+; CHECK-NEXT:    or.b64 %rd84, %rd83, %rd82;
+; CHECK-NEXT:    shl.b64 %rd85, %rd120, 1;
+; CHECK-NEXT:    shr.u64 %rd86, %rd123, 63;
+; CHECK-NEXT:    or.b64 %rd87, %rd85, %rd86;
+; CHECK-NEXT:    shr.u64 %rd88, %rd122, 63;
+; CHECK-NEXT:    shl.b64 %rd89, %rd123, 1;
+; CHECK-NEXT:    or.b64 %rd90, %rd89, %rd88;
+; CHECK-NEXT:    shl.b64 %rd91, %rd122, 1;
+; CHECK-NEXT:    or.b64 %rd122, %rd116, %rd91;
+; CHECK-NEXT:    or.b64 %rd123, %rd113, %rd90;
+; CHECK-NEXT:    sub.cc.s64 %rd92, %rd35, %rd87;
+; CHECK-NEXT:    subc.cc.s64 %rd93, %rd36, %rd84;
+; CHECK-NEXT:    shr.s64 %rd94, %rd93, 63;
+; CHECK-NEXT:    and.b64 %rd116, %rd94, 1;
+; CHECK-NEXT:    and.b64 %rd95, %rd94, %rd5;
+; CHECK-NEXT:    and.b64 %rd96, %rd94, %rd6;
+; CHECK-NEXT:    sub.cc.s64 %rd120, %rd87, %rd95;
+; CHECK-NEXT:    subc.cc.s64 %rd121, %rd84, %rd96;
+; CHECK-NEXT:    add.cc.s64 %rd118, %rd118, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd119, %rd119, -1;
+; CHECK-NEXT:    or.b64 %rd97, %rd118, %rd119;
+; CHECK-NEXT:    setp.eq.s64 %p21, %rd97, 0;
+; CHECK-NEXT:    @%p21 bra $L__BB0_4;
 ; CHECK-NEXT:    bra.uni $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd99, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd100, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
-; CHECK-NEXT:    shl.b64 %rd102, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd125, %rd117, %rd102;
-; CHECK-NEXT:    or.b64 %rd126, %rd114, %rd101;
+; CHECK-NEXT:    shr.u64 %rd98, %rd122, 63;
+; CHECK-NEXT:    shl.b64 %rd99, %rd123, 1;
+; CHECK-NEXT:    or.b64 %rd100, %rd99, %rd98;
+; CHECK-NEXT:    shl.b64 %rd101, %rd122, 1;
+; CHECK-NEXT:    or.b64 %rd124, %rd116, %rd101;
+; CHECK-NEXT:    or.b64 %rd125, %rd113, %rd100;
 ; CHECK-NEXT:  $L__BB0_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd103, %rd5, %rd125;
-; CHECK-NEXT:    mad.lo.s64 %rd104, %rd5, %rd126, %rd103;
-; CHECK-NEXT:    mad.lo.s64 %rd105, %rd6, %rd125, %rd104;
-; CHECK-NEXT:    mul.lo.s64 %rd106, %rd5, %rd125;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd3, %rd106;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd4, %rd105;
+; CHECK-NEXT:    mul.hi.u64 %rd102, %rd5, %rd124;
+; CHECK-NEXT:    mad.lo.s64 %rd103, %rd5, %rd125, %rd102;
+; CHECK-NEXT:    mad.lo.s64 %rd104, %rd6, %rd124, %rd103;
+; CHECK-NEXT:    mul.lo.s64 %rd105, %rd5, %rd124;
+; CHECK-NEXT:    sub.cc.s64 %rd106, %rd3, %rd105;
+; CHECK-NEXT:    subc.cc.s64 %rd107, %rd4, %rd104;
+; CHECK-NEXT:    xor.b64 %rd108, %rd106, %rd2;
 ; CHECK-NEXT:    xor.b64 %rd109, %rd107, %rd2;
-; CHECK-NEXT:    xor.b64 %rd110, %rd108, %rd2;
-; CHECK-NEXT:    sub.cc.s64 %rd111, %rd109, %rd2;
-; CHECK-NEXT:    subc.cc.s64 %rd112, %rd110, %rd2;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd111, %rd112};
+; CHECK-NEXT:    sub.cc.s64 %rd110, %rd108, %rd2;
+; CHECK-NEXT:    subc.cc.s64 %rd111, %rd109, %rd2;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd110, %rd111};
 ; CHECK-NEXT:    ret;
   %div = srem i128 %lhs, %rhs
   ret i128 %div
@@ -148,7 +149,7 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<113>;
+; CHECK-NEXT:    .reg .b64 %rd<111>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [urem_i128_param_0];
@@ -172,98 +173,98 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd103, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd103, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    mov.b64 %rd101, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd6, %rd101, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd5, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd6, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd6, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
+; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd57, 0;
+; CHECK-NEXT:    selp.b64 %rd110, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd109, 0, %rd41, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB1_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd103, %rd5, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd104, %rd6, 0;
+; CHECK-NEXT:    or.b64 %rd60, %rd103, %rd104;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd60, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd5;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r7;
+; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd110, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd100, %rd103;
+; CHECK-NEXT:    selp.b64 %rd108, %rd64, %rd63, %p15;
+; CHECK-NEXT:    shl.b64 %rd107, %rd41, %r6;
+; CHECK-NEXT:    mov.b64 %rd98, %rd101;
 ; CHECK-NEXT:    @%p14 bra $L__BB1_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd105;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd103;
+; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r10;
+; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd107, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r9;
+; CHECK-NEXT:    selp.b64 %rd105, %rd70, %rd69, %p16;
+; CHECK-NEXT:    shr.u64 %rd106, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd100, 0;
-; CHECK-NEXT:    mov.b64 %rd103, %rd100;
+; CHECK-NEXT:    mov.b64 %rd98, 0;
+; CHECK-NEXT:    mov.b64 %rd101, %rd98;
 ; CHECK-NEXT:  $L__BB1_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd107, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd108, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd107, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd110, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd109, %rd103, %rd82;
-; CHECK-NEXT:    or.b64 %rd110, %rd100, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd103, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd3;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd105, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd106, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd71, %rd105, 63;
+; CHECK-NEXT:    shl.b64 %rd72, %rd106, 1;
+; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
+; CHECK-NEXT:    shl.b64 %rd74, %rd105, 1;
+; CHECK-NEXT:    shr.u64 %rd75, %rd108, 63;
+; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd77, %rd107, 63;
+; CHECK-NEXT:    shl.b64 %rd78, %rd108, 1;
+; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
+; CHECK-NEXT:    shl.b64 %rd80, %rd107, 1;
+; CHECK-NEXT:    or.b64 %rd107, %rd101, %rd80;
+; CHECK-NEXT:    or.b64 %rd108, %rd98, %rd79;
+; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
+; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
+; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
+; CHECK-NEXT:    and.b64 %rd101, %rd83, 1;
+; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd3;
+; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd105, %rd76, %rd84;
+; CHECK-NEXT:    subc.cc.s64 %rd106, %rd73, %rd85;
+; CHECK-NEXT:    add.cc.s64 %rd103, %rd103, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd104, %rd104, -1;
+; CHECK-NEXT:    or.b64 %rd86, %rd103, %rd104;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd86, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB1_4;
 ; CHECK-NEXT:    bra.uni $L__BB1_2;
 ; CHECK-NEXT:  $L__BB1_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd111, %rd103, %rd92;
-; CHECK-NEXT:    or.b64 %rd112, %rd100, %rd91;
+; CHECK-NEXT:    shr.u64 %rd87, %rd107, 63;
+; CHECK-NEXT:    shl.b64 %rd88, %rd108, 1;
+; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
+; CHECK-NEXT:    shl.b64 %rd90, %rd107, 1;
+; CHECK-NEXT:    or.b64 %rd109, %rd101, %rd90;
+; CHECK-NEXT:    or.b64 %rd110, %rd98, %rd89;
 ; CHECK-NEXT:  $L__BB1_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd93, %rd3, %rd111;
-; CHECK-NEXT:    mad.lo.s64 %rd94, %rd3, %rd112, %rd93;
-; CHECK-NEXT:    mad.lo.s64 %rd95, %rd4, %rd111, %rd94;
-; CHECK-NEXT:    mul.lo.s64 %rd96, %rd3, %rd111;
-; CHECK-NEXT:    sub.cc.s64 %rd97, %rd41, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd98, %rd42, %rd95;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd97, %rd98};
+; CHECK-NEXT:    mul.hi.u64 %rd91, %rd3, %rd109;
+; CHECK-NEXT:    mad.lo.s64 %rd92, %rd3, %rd110, %rd91;
+; CHECK-NEXT:    mad.lo.s64 %rd93, %rd4, %rd109, %rd92;
+; CHECK-NEXT:    mul.lo.s64 %rd94, %rd3, %rd109;
+; CHECK-NEXT:    sub.cc.s64 %rd95, %rd41, %rd94;
+; CHECK-NEXT:    subc.cc.s64 %rd96, %rd42, %rd93;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd95, %rd96};
 ; CHECK-NEXT:    ret;
   %div = urem i128 %lhs, %rhs
   ret i128 %div
@@ -292,13 +293,12 @@ define i128 @srem_i128_pow2k(i128 %lhs) {
 define i128 @urem_i128_pow2k(i128 %lhs) {
 ; CHECK-LABEL: urem_i128_pow2k(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [urem_i128_pow2k_param_0];
 ; CHECK-NEXT:    and.b64 %rd3, %rd1, 8589934591;
-; CHECK-NEXT:    mov.b64 %rd4, 0;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd4};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, 0};
 ; CHECK-NEXT:    ret;
   %div = urem i128 %lhs, 8589934592
   ret i128 %div
@@ -307,9 +307,9 @@ define i128 @urem_i128_pow2k(i128 %lhs) {
 define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-LABEL: sdiv_i128(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<20>;
+; CHECK-NEXT:    .reg .pred %p<22>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<122>;
+; CHECK-NEXT:    .reg .b64 %rd<121>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [sdiv_i128_param_0];
@@ -345,96 +345,97 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd63, %r4;
 ; CHECK-NEXT:    add.s64 %rd64, %rd63, 64;
 ; CHECK-NEXT:    selp.b64 %rd65, %rd62, %rd64, %p7;
-; CHECK-NEXT:    mov.b64 %rd112, 0;
+; CHECK-NEXT:    mov.b64 %rd111, 0;
 ; CHECK-NEXT:    sub.cc.s64 %rd67, %rd61, %rd65;
-; CHECK-NEXT:    subc.cc.s64 %rd68, %rd112, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd67, 127;
-; CHECK-NEXT:    setp.eq.s64 %p9, %rd68, 0;
-; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.s64 %p11, %rd68, 0;
-; CHECK-NEXT:    or.pred %p12, %p10, %p11;
-; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd69, %rd67, 127;
-; CHECK-NEXT:    or.b64 %rd70, %rd69, %rd68;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd70, 0;
-; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p13;
-; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p13;
-; CHECK-NEXT:    or.pred %p15, %p13, %p14;
-; CHECK-NEXT:    @%p15 bra $L__BB4_5;
+; CHECK-NEXT:    subc.cc.s64 %rd8, %rd111, 0;
+; CHECK-NEXT:    setp.ne.s64 %p8, %rd8, 0;
+; CHECK-NEXT:    and.pred %p10, %p8, %p8;
+; CHECK-NEXT:    setp.eq.s64 %p11, %rd8, 0;
+; CHECK-NEXT:    setp.gt.u64 %p12, %rd67, 127;
+; CHECK-NEXT:    and.pred %p13, %p11, %p12;
+; CHECK-NEXT:    or.pred %p14, %p13, %p10;
+; CHECK-NEXT:    or.pred %p15, %p5, %p14;
+; CHECK-NEXT:    xor.b64 %rd68, %rd67, 127;
+; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd8;
+; CHECK-NEXT:    setp.eq.s64 %p16, %rd69, 0;
+; CHECK-NEXT:    selp.b64 %rd120, 0, %rd2, %p15;
+; CHECK-NEXT:    selp.b64 %rd119, 0, %rd1, %p15;
+; CHECK-NEXT:    or.pred %p17, %p15, %p16;
+; CHECK-NEXT:    @%p17 bra $L__BB4_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd67, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd68, 0;
-; CHECK-NEXT:    or.b64 %rd73, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p16, %rd73, 0;
+; CHECK-NEXT:    add.cc.s64 %rd113, %rd67, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd114, %rd8, 0;
+; CHECK-NEXT:    or.b64 %rd72, %rd113, %rd114;
+; CHECK-NEXT:    setp.eq.s64 %p18, %rd72, 0;
 ; CHECK-NEXT:    cvt.u32.u64 %r5, %rd67;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r6;
+; CHECK-NEXT:    shl.b64 %rd73, %rd2, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r7;
-; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd74, %rd1, %r7;
+; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r8;
-; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p17;
-; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r6;
-; CHECK-NEXT:    mov.b64 %rd109, %rd112;
-; CHECK-NEXT:    @%p16 bra $L__BB4_4;
+; CHECK-NEXT:    shl.b64 %rd76, %rd1, %r8;
+; CHECK-NEXT:    setp.gt.s32 %p19, %r6, 63;
+; CHECK-NEXT:    selp.b64 %rd118, %rd76, %rd75, %p19;
+; CHECK-NEXT:    shl.b64 %rd117, %rd1, %r6;
+; CHECK-NEXT:    mov.b64 %rd108, %rd111;
+; CHECK-NEXT:    @%p18 bra $L__BB4_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd114;
-; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd113;
+; CHECK-NEXT:    shr.u64 %rd79, %rd1, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r10;
-; CHECK-NEXT:    or.b64 %rd82, %rd80, %rd81;
+; CHECK-NEXT:    shl.b64 %rd80, %rd2, %r10;
+; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r11;
-; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p18;
-; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r9;
+; CHECK-NEXT:    shr.u64 %rd82, %rd2, %r11;
+; CHECK-NEXT:    setp.gt.s32 %p20, %r9, 63;
+; CHECK-NEXT:    selp.b64 %rd115, %rd82, %rd81, %p20;
+; CHECK-NEXT:    shr.u64 %rd116, %rd2, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd35, %rd3, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd36, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd109, 0;
-; CHECK-NEXT:    mov.b64 %rd112, %rd109;
+; CHECK-NEXT:    mov.b64 %rd108, 0;
+; CHECK-NEXT:    mov.b64 %rd111, %rd108;
 ; CHECK-NEXT:  $L__BB4_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd84, %rd116, 63;
-; CHECK-NEXT:    shl.b64 %rd85, %rd117, 1;
-; CHECK-NEXT:    or.b64 %rd86, %rd85, %rd84;
-; CHECK-NEXT:    shl.b64 %rd87, %rd116, 1;
-; CHECK-NEXT:    shr.u64 %rd88, %rd119, 63;
-; CHECK-NEXT:    or.b64 %rd89, %rd87, %rd88;
-; CHECK-NEXT:    shr.u64 %rd90, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd91, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd92, %rd91, %rd90;
-; CHECK-NEXT:    shl.b64 %rd93, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd118, %rd112, %rd93;
-; CHECK-NEXT:    or.b64 %rd119, %rd109, %rd92;
-; CHECK-NEXT:    sub.cc.s64 %rd94, %rd35, %rd89;
-; CHECK-NEXT:    subc.cc.s64 %rd95, %rd36, %rd86;
-; CHECK-NEXT:    shr.s64 %rd96, %rd95, 63;
-; CHECK-NEXT:    and.b64 %rd112, %rd96, 1;
-; CHECK-NEXT:    and.b64 %rd97, %rd96, %rd3;
-; CHECK-NEXT:    and.b64 %rd98, %rd96, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd116, %rd89, %rd97;
-; CHECK-NEXT:    subc.cc.s64 %rd117, %rd86, %rd98;
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd114, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd115, -1;
-; CHECK-NEXT:    or.b64 %rd99, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.s64 %p19, %rd99, 0;
-; CHECK-NEXT:    @%p19 bra $L__BB4_4;
+; CHECK-NEXT:    shr.u64 %rd83, %rd115, 63;
+; CHECK-NEXT:    shl.b64 %rd84, %rd116, 1;
+; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
+; CHECK-NEXT:    shl.b64 %rd86, %rd115, 1;
+; CHECK-NEXT:    shr.u64 %rd87, %rd118, 63;
+; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
+; CHECK-NEXT:    shr.u64 %rd89, %rd117, 63;
+; CHECK-NEXT:    shl.b64 %rd90, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
+; CHECK-NEXT:    shl.b64 %rd92, %rd117, 1;
+; CHECK-NEXT:    or.b64 %rd117, %rd111, %rd92;
+; CHECK-NEXT:    or.b64 %rd118, %rd108, %rd91;
+; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
+; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
+; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
+; CHECK-NEXT:    and.b64 %rd111, %rd95, 1;
+; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd3;
+; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd115, %rd88, %rd96;
+; CHECK-NEXT:    subc.cc.s64 %rd116, %rd85, %rd97;
+; CHECK-NEXT:    add.cc.s64 %rd113, %rd113, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd114, %rd114, -1;
+; CHECK-NEXT:    or.b64 %rd98, %rd113, %rd114;
+; CHECK-NEXT:    setp.eq.s64 %p21, %rd98, 0;
+; CHECK-NEXT:    @%p21 bra $L__BB4_4;
 ; CHECK-NEXT:    bra.uni $L__BB4_2;
 ; CHECK-NEXT:  $L__BB4_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd100, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd101, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd102, %rd101, %rd100;
-; CHECK-NEXT:    shl.b64 %rd103, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd120, %rd112, %rd103;
-; CHECK-NEXT:    or.b64 %rd121, %rd109, %rd102;
+; CHECK-NEXT:    shr.u64 %rd99, %rd117, 63;
+; CHECK-NEXT:    shl.b64 %rd100, %rd118, 1;
+; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
+; CHECK-NEXT:    shl.b64 %rd102, %rd117, 1;
+; CHECK-NEXT:    or.b64 %rd119, %rd111, %rd102;
+; CHECK-NEXT:    or.b64 %rd120, %rd108, %rd101;
 ; CHECK-NEXT:  $L__BB4_5: // %udiv-end
+; CHECK-NEXT:    xor.b64 %rd103, %rd119, %rd5;
 ; CHECK-NEXT:    xor.b64 %rd104, %rd120, %rd5;
-; CHECK-NEXT:    xor.b64 %rd105, %rd121, %rd5;
-; CHECK-NEXT:    sub.cc.s64 %rd106, %rd104, %rd5;
-; CHECK-NEXT:    subc.cc.s64 %rd107, %rd105, %rd5;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd106, %rd107};
+; CHECK-NEXT:    sub.cc.s64 %rd105, %rd103, %rd5;
+; CHECK-NEXT:    subc.cc.s64 %rd106, %rd104, %rd5;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd105, %rd106};
 ; CHECK-NEXT:    ret;
   %div = sdiv i128 %lhs, %rhs
   ret i128 %div
@@ -445,7 +446,7 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<107>;
+; CHECK-NEXT:    .reg .b64 %rd<105>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [udiv_i128_param_0];
@@ -469,92 +470,92 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
 ; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
 ; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd97, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd97, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.s64 %p7, %rd57, 0;
+; CHECK-NEXT:    mov.b64 %rd95, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd5, %rd50, %rd54;
+; CHECK-NEXT:    subc.cc.s64 %rd6, %rd95, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd5, 127;
+; CHECK-NEXT:    setp.eq.s64 %p7, %rd6, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.s64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.s64 %p9, %rd6, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.s64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd56, %rd5, 127;
+; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd6;
+; CHECK-NEXT:    setp.eq.s64 %p12, %rd57, 0;
+; CHECK-NEXT:    selp.b64 %rd104, 0, %rd42, %p11;
+; CHECK-NEXT:    selp.b64 %rd103, 0, %rd41, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB5_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd97, %rd5, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd98, %rd6, 0;
+; CHECK-NEXT:    or.b64 %rd60, %rd97, %rd98;
+; CHECK-NEXT:    setp.eq.s64 %p14, %rd60, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd5;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd61, %rd42, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd62, %rd41, %r7;
+; CHECK-NEXT:    or.b64 %rd63, %rd61, %rd62;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd64, %rd41, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd94, %rd97;
+; CHECK-NEXT:    selp.b64 %rd102, %rd64, %rd63, %p15;
+; CHECK-NEXT:    shl.b64 %rd101, %rd41, %r6;
+; CHECK-NEXT:    mov.b64 %rd92, %rd95;
 ; CHECK-NEXT:    @%p14 bra $L__BB5_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd99;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd97;
+; CHECK-NEXT:    shr.u64 %rd67, %rd41, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd68, %rd42, %r10;
+; CHECK-NEXT:    or.b64 %rd69, %rd67, %rd68;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd70, %rd42, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r9;
+; CHECK-NEXT:    selp.b64 %rd99, %rd70, %rd69, %p16;
+; CHECK-NEXT:    shr.u64 %rd100, %rd42, %r9;
 ; CHECK-NEXT:    add.cc.s64 %rd33, %rd43, -1;
 ; CHECK-NEXT:    addc.cc.s64 %rd34, %rd44, -1;
-; CHECK-NEXT:    mov.b64 %rd94, 0;
-; CHECK-NEXT:    mov.b64 %rd97, %rd94;
+; CHECK-NEXT:    mov.b64 %rd92, 0;
+; CHECK-NEXT:    mov.b64 %rd95, %rd92;
 ; CHECK-NEXT:  $L__BB5_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd101, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd102, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd101, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd104, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd103, %rd97, %rd82;
-; CHECK-NEXT:    or.b64 %rd104, %rd94, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd97, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd43;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd44;
-; CHECK-NEXT:    sub.cc.s64 %rd101, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd102, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd99, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd100, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.s64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd71, %rd99, 63;
+; CHECK-NEXT:    shl.b64 %rd72, %rd100, 1;
+; CHECK-NEXT:    or.b64 %rd73, %rd72, %rd71;
+; CHECK-NEXT:    shl.b64 %rd74, %rd99, 1;
+; CHECK-NEXT:    shr.u64 %rd75, %rd102, 63;
+; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd77, %rd101, 63;
+; CHECK-NEXT:    shl.b64 %rd78, %rd102, 1;
+; CHECK-NEXT:    or.b64 %rd79, %rd78, %rd77;
+; CHECK-NEXT:    shl.b64 %rd80, %rd101, 1;
+; CHECK-NEXT:    or.b64 %rd101, %rd95, %rd80;
+; CHECK-NEXT:    or.b64 %rd102, %rd92, %rd79;
+; CHECK-NEXT:    sub.cc.s64 %rd81, %rd33, %rd76;
+; CHECK-NEXT:    subc.cc.s64 %rd82, %rd34, %rd73;
+; CHECK-NEXT:    shr.s64 %rd83, %rd82, 63;
+; CHECK-NEXT:    and.b64 %rd95, %rd83, 1;
+; CHECK-NEXT:    and.b64 %rd84, %rd83, %rd43;
+; CHECK-NEXT:    and.b64 %rd85, %rd83, %rd44;
+; CHECK-NEXT:    sub.cc.s64 %rd99, %rd76, %rd84;
+; CHECK-NEXT:    subc.cc.s64 %rd100, %rd73, %rd85;
+; CHECK-NEXT:    add.cc.s64 %rd97, %rd97, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd98, %rd98, -1;
+; CHECK-NEXT:    or.b64 %rd86, %rd97, %rd98;
+; CHECK-NEXT:    setp.eq.s64 %p17, %rd86, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB5_4;
 ; CHECK-NEXT:    bra.uni $L__BB5_2;
 ; CHECK-NEXT:  $L__BB5_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd105, %rd97, %rd92;
-; CHECK-NEXT:    or.b64 %rd106, %rd94, %rd91;
+; CHECK-NEXT:    shr.u64 %rd87, %rd101, 63;
+; CHECK-NEXT:    shl.b64 %rd88, %rd102, 1;
+; CHECK-NEXT:    or.b64 %rd89, %rd88, %rd87;
+; CHECK-NEXT:    shl.b64 %rd90, %rd101, 1;
+; CHECK-NEXT:    or.b64 %rd103, %rd95, %rd90;
+; CHECK-NEXT:    or.b64 %rd104, %rd92, %rd89;
 ; CHECK-NEXT:  $L__BB5_5: // %udiv-end
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd105, %rd106};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd103, %rd104};
 ; CHECK-NEXT:    ret;
   %div = udiv i128 %lhs, %rhs
   ret i128 %div
diff --git a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
index 5bfa5b2bc63a..e89ab7a5605c 100644
--- a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
@@ -23,45 +23,66 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define <2 x i16> @test_ret_const() #0 {
 ; COMMON-LABEL: test_ret_const(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b32 %r<2>;
+; COMMON-EMPTY:
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
-; COMMON-NEXT:    mov.b32 %r1, 131073;
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r1;
+; COMMON-NEXT:    st.param.b32 [func_retval0], 131073;
 ; COMMON-NEXT:    ret;
   ret <2 x i16> <i16 1, i16 2>
 }
 
 define i16 @test_extract_0(<2 x i16> %a) #0 {
-; COMMON-LABEL: test_extract_0(
-; COMMON:       {
-; COMMON-NEXT:    .reg .b16 %rs<2>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT:  // %bb.0:
-; COMMON-NEXT:    ld.param.b32 %r1, [test_extract_0_param_0];
+; I16x2-LABEL: test_extract_0(
+; I16x2:       {
+; I16x2-NEXT:    .reg .b16 %rs<2>;
+; I16x2-NEXT:    .reg .b32 %r<3>;
+; I16x2-EMPTY:
+; I16x2-NEXT:  // %bb.0:
+; I16x2-NEXT:    ld.param.b32 %r1, [test_extract_0_param_0];
 ; I16x2-NEXT:    mov.b32 {%rs1, _}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs1, tmp}, %r1; }
-; COMMON-NEXT:    cvt.u32.u16 %r2, %rs1;
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT:    ret;
+; I16x2-NEXT:    cvt.u32.u16 %r2, %rs1;
+; I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; I16x2-NEXT:    ret;
+;
+; NO-I16x2-LABEL: test_extract_0(
+; NO-I16x2:       {
+; NO-I16x2-NEXT:    .reg .b16 %rs<2>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT:  // %bb.0:
+; NO-I16x2-NEXT:    ld.param.b32 %r1, [test_extract_0_param_0];
+; NO-I16x2-NEXT:    { .reg .b16 tmp; mov.b32 {%rs1, tmp}, %r1; }
+; NO-I16x2-NEXT:    cvt.u32.u16 %r2, %rs1;
+; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT:    ret;
   %e = extractelement <2 x i16> %a, i32 0
   ret i16 %e
 }
 
 define i16 @test_extract_1(<2 x i16> %a) #0 {
-; COMMON-LABEL: test_extract_1(
-; COMMON:       {
-; COMMON-NEXT:    .reg .b16 %rs<2>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT:  // %bb.0:
-; COMMON-NEXT:    ld.param.b32 %r1, [test_extract_1_param_0];
+; I16x2-LABEL: test_extract_1(
+; I16x2:       {
+; I16x2-NEXT:    .reg .b16 %rs<2>;
+; I16x2-NEXT:    .reg .b32 %r<3>;
+; I16x2-EMPTY:
+; I16x2-NEXT:  // %bb.0:
+; I16x2-NEXT:    ld.param.b32 %r1, [test_extract_1_param_0];
 ; I16x2-NEXT:    mov.b32 {_, %rs1}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r1; }
-; COMMON-NEXT:    cvt.u32.u16 %r2, %rs1;
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT:    ret;
+; I16x2-NEXT:    cvt.u32.u16 %r2, %rs1;
+; I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; I16x2-NEXT:    ret;
+;
+; NO-I16x2-LABEL: test_extract_1(
+; NO-I16x2:       {
+; NO-I16x2-NEXT:    .reg .b16 %rs<2>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT:  // %bb.0:
+; NO-I16x2-NEXT:    ld.param.b32 %r1, [test_extract_1_param_0];
+; NO-I16x2-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r1; }
+; NO-I16x2-NEXT:    cvt.u32.u16 %r2, %rs1;
+; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT:    ret;
   %e = extractelement <2 x i16> %a, i32 1
   ret i16 %e
 }
@@ -102,7 +123,7 @@ define <2 x i16> @test_add(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-LABEL: test_add(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<7>;
-; NO-I16x2-NEXT:    .reg .b32 %r<4>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r2, [test_add_param_1];
@@ -111,8 +132,7 @@ define <2 x i16> @test_add(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; NO-I16x2-NEXT:    add.s16 %rs5, %rs4, %rs2;
 ; NO-I16x2-NEXT:    add.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; NO-I16x2-NEXT:    ret;
   %r = add <2 x i16> %a, %b
   ret <2 x i16> %r
@@ -134,15 +154,14 @@ define <2 x i16> @test_add_imm_0(<2 x i16> %a) #0 {
 ; NO-I16x2-LABEL: test_add_imm_0(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<5>;
-; NO-I16x2-NEXT:    .reg .b32 %r<3>;
+; NO-I16x2-NEXT:    .reg .b32 %r<2>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r1, [test_add_imm_0_param_0];
 ; NO-I16x2-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; NO-I16x2-NEXT:    add.s16 %rs3, %rs2, 2;
 ; NO-I16x2-NEXT:    add.s16 %rs4, %rs1, 1;
-; NO-I16x2-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; NO-I16x2-NEXT:    ret;
   %r = add <2 x i16> <i16 1, i16 2>, %a
   ret <2 x i16> %r
@@ -163,15 +182,14 @@ define <2 x i16> @test_add_imm_1(<2 x i16> %a) #0 {
 ; NO-I16x2-LABEL: test_add_imm_1(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<5>;
-; NO-I16x2-NEXT:    .reg .b32 %r<3>;
+; NO-I16x2-NEXT:    .reg .b32 %r<2>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r1, [test_add_imm_1_param_0];
 ; NO-I16x2-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; NO-I16x2-NEXT:    add.s16 %rs3, %rs2, 2;
 ; NO-I16x2-NEXT:    add.s16 %rs4, %rs1, 1;
-; NO-I16x2-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r2;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; NO-I16x2-NEXT:    ret;
   %r = add <2 x i16> %a, <i16 1, i16 2>
   ret <2 x i16> %r
@@ -181,7 +199,7 @@ define <2 x i16> @test_sub(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-LABEL: test_sub(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<7>;
-; COMMON-NEXT:    .reg .b32 %r<4>;
+; COMMON-NEXT:    .reg .b32 %r<3>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r2, [test_sub_param_1];
@@ -190,8 +208,7 @@ define <2 x i16> @test_sub(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; COMMON-NEXT:    sub.s16 %rs5, %rs4, %rs2;
 ; COMMON-NEXT:    sub.s16 %rs6, %rs3, %rs1;
-; COMMON-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r3;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; COMMON-NEXT:    ret;
   %r = sub <2 x i16> %a, %b
   ret <2 x i16> %r
@@ -212,7 +229,7 @@ define <2 x i16> @test_smax(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-LABEL: test_smax(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<7>;
-; NO-I16x2-NEXT:    .reg .b32 %r<4>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r2, [test_smax_param_1];
@@ -221,8 +238,7 @@ define <2 x i16> @test_smax(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; NO-I16x2-NEXT:    max.s16 %rs5, %rs4, %rs2;
 ; NO-I16x2-NEXT:    max.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; NO-I16x2-NEXT:    ret;
   %cmp = icmp sgt <2 x i16> %a, %b
   %r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -244,7 +260,7 @@ define <2 x i16> @test_umax(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-LABEL: test_umax(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<7>;
-; NO-I16x2-NEXT:    .reg .b32 %r<4>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r2, [test_umax_param_1];
@@ -253,8 +269,7 @@ define <2 x i16> @test_umax(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; NO-I16x2-NEXT:    max.u16 %rs5, %rs4, %rs2;
 ; NO-I16x2-NEXT:    max.u16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; NO-I16x2-NEXT:    ret;
   %cmp = icmp ugt <2 x i16> %a, %b
   %r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -276,7 +291,7 @@ define <2 x i16> @test_smin(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-LABEL: test_smin(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<7>;
-; NO-I16x2-NEXT:    .reg .b32 %r<4>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r2, [test_smin_param_1];
@@ -285,8 +300,7 @@ define <2 x i16> @test_smin(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; NO-I16x2-NEXT:    min.s16 %rs5, %rs4, %rs2;
 ; NO-I16x2-NEXT:    min.s16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; NO-I16x2-NEXT:    ret;
   %cmp = icmp sle <2 x i16> %a, %b
   %r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -308,7 +322,7 @@ define <2 x i16> @test_umin(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-LABEL: test_umin(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<7>;
-; NO-I16x2-NEXT:    .reg .b32 %r<4>;
+; NO-I16x2-NEXT:    .reg .b32 %r<3>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
 ; NO-I16x2-NEXT:    ld.param.b32 %r2, [test_umin_param_1];
@@ -317,8 +331,7 @@ define <2 x i16> @test_umin(<2 x i16> %a, <2 x i16> %b) #0 {
 ; NO-I16x2-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; NO-I16x2-NEXT:    min.u16 %rs5, %rs4, %rs2;
 ; NO-I16x2-NEXT:    min.u16 %rs6, %rs3, %rs1;
-; NO-I16x2-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; NO-I16x2-NEXT:    ret;
   %cmp = icmp ule <2 x i16> %a, %b
   %r = select <2 x i1> %cmp, <2 x i16> %a, <2 x i16> %b
@@ -329,7 +342,7 @@ define <2 x i16> @test_mul(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-LABEL: test_mul(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<7>;
-; COMMON-NEXT:    .reg .b32 %r<4>;
+; COMMON-NEXT:    .reg .b32 %r<3>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r2, [test_mul_param_1];
@@ -338,8 +351,7 @@ define <2 x i16> @test_mul(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; COMMON-NEXT:    mul.lo.s16 %rs5, %rs4, %rs2;
 ; COMMON-NEXT:    mul.lo.s16 %rs6, %rs3, %rs1;
-; COMMON-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r3;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; COMMON-NEXT:    ret;
   %r = mul <2 x i16> %a, %b
   ret <2 x i16> %r
@@ -587,14 +599,14 @@ define void @test_ldst_v3i16(ptr %a, ptr %b) {
 define void @test_ldst_v4i16(ptr %a, ptr %b) {
 ; COMMON-LABEL: test_ldst_v4i16(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b16 %rs<5>;
+; COMMON-NEXT:    .reg .b32 %r<3>;
 ; COMMON-NEXT:    .reg .b64 %rd<3>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b64 %rd2, [test_ldst_v4i16_param_1];
 ; COMMON-NEXT:    ld.param.b64 %rd1, [test_ldst_v4i16_param_0];
-; COMMON-NEXT:    ld.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [%rd1];
-; COMMON-NEXT:    st.v4.b16 [%rd2], {%rs1, %rs2, %rs3, %rs4};
+; COMMON-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1];
+; COMMON-NEXT:    st.v2.b32 [%rd2], {%r1, %r2};
 ; COMMON-NEXT:    ret;
   %t1 = load <4 x i16>, ptr %a
   store <4 x i16> %t1, ptr %b, align 16
@@ -634,12 +646,7 @@ define <2 x i16> @test_call(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-NEXT:    .param .align 4 .b8 param1[4];
 ; COMMON-NEXT:    st.param.b32 [param1], %r2;
 ; COMMON-NEXT:    .param .align 4 .b8 retval0[4];
-; COMMON-NEXT:    call.uni (retval0),
-; COMMON-NEXT:    test_callee,
-; COMMON-NEXT:    (
-; COMMON-NEXT:    param0,
-; COMMON-NEXT:    param1
-; COMMON-NEXT:    );
+; COMMON-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; COMMON-NEXT:    ld.param.b32 %r3, [retval0];
 ; COMMON-NEXT:    } // callseq 0
 ; COMMON-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -662,12 +669,7 @@ define <2 x i16> @test_call_flipped(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-NEXT:    .param .align 4 .b8 param1[4];
 ; COMMON-NEXT:    st.param.b32 [param1], %r1;
 ; COMMON-NEXT:    .param .align 4 .b8 retval0[4];
-; COMMON-NEXT:    call.uni (retval0),
-; COMMON-NEXT:    test_callee,
-; COMMON-NEXT:    (
-; COMMON-NEXT:    param0,
-; COMMON-NEXT:    param1
-; COMMON-NEXT:    );
+; COMMON-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; COMMON-NEXT:    ld.param.b32 %r3, [retval0];
 ; COMMON-NEXT:    } // callseq 1
 ; COMMON-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -690,12 +692,7 @@ define <2 x i16> @test_tailcall_flipped(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-NEXT:    .param .align 4 .b8 param1[4];
 ; COMMON-NEXT:    st.param.b32 [param1], %r1;
 ; COMMON-NEXT:    .param .align 4 .b8 retval0[4];
-; COMMON-NEXT:    call.uni (retval0),
-; COMMON-NEXT:    test_callee,
-; COMMON-NEXT:    (
-; COMMON-NEXT:    param0,
-; COMMON-NEXT:    param1
-; COMMON-NEXT:    );
+; COMMON-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; COMMON-NEXT:    ld.param.b32 %r3, [retval0];
 ; COMMON-NEXT:    } // callseq 2
 ; COMMON-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -729,7 +726,7 @@ define <2 x i16> @test_select_cc(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .pred %p<3>;
 ; COMMON-NEXT:    .reg .b16 %rs<11>;
-; COMMON-NEXT:    .reg .b32 %r<6>;
+; COMMON-NEXT:    .reg .b32 %r<5>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r4, [test_select_cc_param_3];
@@ -744,8 +741,7 @@ define <2 x i16> @test_select_cc(<2 x i16> %a, <2 x i16> %b, <2 x i16> %c, <2 x
 ; COMMON-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
 ; COMMON-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p2;
 ; COMMON-NEXT:    selp.b16 %rs10, %rs7, %rs5, %p1;
-; COMMON-NEXT:    mov.b32 %r5, {%rs10, %rs9};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r5;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; COMMON-NEXT:    ret;
   %cc = icmp ne <2 x i16> %c, %d
   %r = select <2 x i1> %cc, <2 x i16> %a, <2 x i16> %b
@@ -783,7 +779,7 @@ define <2 x i16> @test_select_cc_i16_i32(<2 x i16> %a, <2 x i16> %b,
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .pred %p<3>;
 ; COMMON-NEXT:    .reg .b16 %rs<7>;
-; COMMON-NEXT:    .reg .b32 %r<8>;
+; COMMON-NEXT:    .reg .b32 %r<7>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_i16_i32_param_3];
@@ -796,8 +792,7 @@ define <2 x i16> @test_select_cc_i16_i32(<2 x i16> %a, <2 x i16> %b,
 ; COMMON-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; COMMON-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
 ; COMMON-NEXT:    selp.b16 %rs6, %rs3, %rs1, %p1;
-; COMMON-NEXT:    mov.b32 %r7, {%rs6, %rs5};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r7;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs6, %rs5};
 ; COMMON-NEXT:    ret;
                                           <2 x i32> %c, <2 x i32> %d) #0 {
   %cc = icmp ne <2 x i32> %c, %d
@@ -839,7 +834,7 @@ define <2 x i16> @test_trunc_2xi32_muliple_use0(<2 x i32> %a, ptr %p) #0 {
 ; NO-I16x2-LABEL: test_trunc_2xi32_muliple_use0(
 ; NO-I16x2:       {
 ; NO-I16x2-NEXT:    .reg .b16 %rs<5>;
-; NO-I16x2-NEXT:    .reg .b32 %r<5>;
+; NO-I16x2-NEXT:    .reg .b32 %r<4>;
 ; NO-I16x2-NEXT:    .reg .b64 %rd<2>;
 ; NO-I16x2-EMPTY:
 ; NO-I16x2-NEXT:  // %bb.0:
@@ -850,8 +845,7 @@ define <2 x i16> @test_trunc_2xi32_muliple_use0(<2 x i32> %a, ptr %p) #0 {
 ; NO-I16x2-NEXT:    mov.b32 %r3, {%rs2, %rs1};
 ; NO-I16x2-NEXT:    add.s16 %rs3, %rs1, 1;
 ; NO-I16x2-NEXT:    add.s16 %rs4, %rs2, 1;
-; NO-I16x2-NEXT:    mov.b32 %r4, {%rs4, %rs3};
-; NO-I16x2-NEXT:    st.b32 [%rd1], %r4;
+; NO-I16x2-NEXT:    st.v2.b16 [%rd1], {%rs4, %rs3};
 ; NO-I16x2-NEXT:    st.param.b32 [func_retval0], %r3;
 ; NO-I16x2-NEXT:    ret;
   %r = trunc <2 x i32> %a to <2 x i16>
@@ -889,15 +883,13 @@ define <2 x i16> @test_trunc_2xi64(<2 x i64> %a) #0 {
 ; COMMON-LABEL: test_trunc_2xi64(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<3>;
-; COMMON-NEXT:    .reg .b32 %r<2>;
 ; COMMON-NEXT:    .reg .b64 %rd<3>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_trunc_2xi64_param_0];
 ; COMMON-NEXT:    cvt.u16.u64 %rs1, %rd2;
 ; COMMON-NEXT:    cvt.u16.u64 %rs2, %rd1;
-; COMMON-NEXT:    mov.b32 %r1, {%rs2, %rs1};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r1;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; COMMON-NEXT:    ret;
   %r = trunc <2 x i64> %a to <2 x i16>
   ret <2 x i16> %r
@@ -912,9 +904,9 @@ define <2 x i32> @test_zext_2xi32(<2 x i16> %a) #0 {
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r1, [test_zext_2xi32_param_0];
 ; COMMON-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; COMMON-NEXT:    cvt.u32.u16 %r2, %rs1;
-; COMMON-NEXT:    cvt.u32.u16 %r3, %rs2;
-; COMMON-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r3};
+; COMMON-NEXT:    cvt.u32.u16 %r2, %rs2;
+; COMMON-NEXT:    cvt.u32.u16 %r3, %rs1;
+; COMMON-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r2};
 ; COMMON-NEXT:    ret;
   %r = zext <2 x i16> %a to <2 x i32>
   ret <2 x i32> %r
@@ -967,14 +959,11 @@ define i32 @test_bitcast_2xi16_to_i32(<2 x i16> %a) #0 {
 define <2 x half> @test_bitcast_2xi16_to_2xhalf(i16 %a) #0 {
 ; COMMON-LABEL: test_bitcast_2xi16_to_2xhalf(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b16 %rs<3>;
-; COMMON-NEXT:    .reg .b32 %r<2>;
+; COMMON-NEXT:    .reg .b16 %rs<2>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b16 %rs1, [test_bitcast_2xi16_to_2xhalf_param_0];
-; COMMON-NEXT:    mov.b16 %rs2, 5;
-; COMMON-NEXT:    mov.b32 %r1, {%rs1, %rs2};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r1;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs1, 5};
 ; COMMON-NEXT:    ret;
   %ins.0 = insertelement <2 x i16> undef, i16 %a, i32 0
   %ins.1 = insertelement <2 x i16> %ins.0, i16 5, i32 1
@@ -987,32 +976,41 @@ define <2 x i16> @test_shufflevector(<2 x i16> %a) #0 {
 ; COMMON-LABEL: test_shufflevector(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<3>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
+; COMMON-NEXT:    .reg .b32 %r<2>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r1, [test_shufflevector_param_0];
 ; COMMON-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; COMMON-NEXT:    mov.b32 %r2, {%rs2, %rs1};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
 ; COMMON-NEXT:    ret;
   %s = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
   ret <2 x i16> %s
 }
 
 define <2 x i16> @test_insertelement(<2 x i16> %a, i16 %x) #0 {
-; COMMON-LABEL: test_insertelement(
-; COMMON:       {
-; COMMON-NEXT:    .reg .b16 %rs<3>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
-; COMMON-EMPTY:
-; COMMON-NEXT:  // %bb.0:
-; COMMON-NEXT:    ld.param.b16 %rs1, [test_insertelement_param_1];
-; COMMON-NEXT:    ld.param.b32 %r1, [test_insertelement_param_0];
+; I16x2-LABEL: test_insertelement(
+; I16x2:       {
+; I16x2-NEXT:    .reg .b16 %rs<3>;
+; I16x2-NEXT:    .reg .b32 %r<2>;
+; I16x2-EMPTY:
+; I16x2-NEXT:  // %bb.0:
+; I16x2-NEXT:    ld.param.b16 %rs1, [test_insertelement_param_1];
+; I16x2-NEXT:    ld.param.b32 %r1, [test_insertelement_param_0];
 ; I16x2-NEXT:    mov.b32 {%rs2, _}, %r1;
-; NO-I16x2-NEXT: { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
-; COMMON-NEXT:    mov.b32 %r2, {%rs2, %rs1};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
-; COMMON-NEXT:    ret;
+; I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
+; I16x2-NEXT:    ret;
+;
+; NO-I16x2-LABEL: test_insertelement(
+; NO-I16x2:       {
+; NO-I16x2-NEXT:    .reg .b16 %rs<3>;
+; NO-I16x2-NEXT:    .reg .b32 %r<2>;
+; NO-I16x2-EMPTY:
+; NO-I16x2-NEXT:  // %bb.0:
+; NO-I16x2-NEXT:    ld.param.b16 %rs1, [test_insertelement_param_1];
+; NO-I16x2-NEXT:    ld.param.b32 %r1, [test_insertelement_param_0];
+; NO-I16x2-NEXT:    { .reg .b16 tmp; mov.b32 {%rs2, tmp}, %r1; }
+; NO-I16x2-NEXT:    st.param.v2.b16 [func_retval0], {%rs2, %rs1};
+; NO-I16x2-NEXT:    ret;
   %i = insertelement <2 x i16> %a, i16 %x, i64 1
   ret <2 x i16> %i
 }
@@ -1021,15 +1019,14 @@ define <2 x i16> @test_fptosi_2xhalf_to_2xi16(<2 x half> %a) #0 {
 ; COMMON-LABEL: test_fptosi_2xhalf_to_2xi16(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<5>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
+; COMMON-NEXT:    .reg .b32 %r<2>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r1, [test_fptosi_2xhalf_to_2xi16_param_0];
 ; COMMON-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; COMMON-NEXT:    cvt.rzi.s16.f16 %rs3, %rs2;
 ; COMMON-NEXT:    cvt.rzi.s16.f16 %rs4, %rs1;
-; COMMON-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; COMMON-NEXT:    ret;
   %r = fptosi <2 x half> %a to <2 x i16>
   ret <2 x i16> %r
@@ -1039,15 +1036,14 @@ define <2 x i16> @test_fptoui_2xhalf_to_2xi16(<2 x half> %a) #0 {
 ; COMMON-LABEL: test_fptoui_2xhalf_to_2xi16(
 ; COMMON:       {
 ; COMMON-NEXT:    .reg .b16 %rs<5>;
-; COMMON-NEXT:    .reg .b32 %r<3>;
+; COMMON-NEXT:    .reg .b32 %r<2>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r1, [test_fptoui_2xhalf_to_2xi16_param_0];
 ; COMMON-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
 ; COMMON-NEXT:    cvt.rzi.u16.f16 %rs3, %rs2;
 ; COMMON-NEXT:    cvt.rzi.u16.f16 %rs4, %rs1;
-; COMMON-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; COMMON-NEXT:    st.param.b32 [func_retval0], %r2;
+; COMMON-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; COMMON-NEXT:    ret;
   %r = fptoui <2 x half> %a to <2 x i16>
   ret <2 x i16> %r
diff --git a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
index fe8113489592..3edd4e4da60e 100644
--- a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
@@ -13,16 +13,15 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define i16 @test_bitcast_2xi8_i16(<2 x i8> %a) {
 ; CHECK-LABEL: test_bitcast_2xi8_i16(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<6>;
+; CHECK-NEXT:    .reg .b16 %rs<5>;
 ; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_bitcast_2xi8_i16_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [test_bitcast_2xi8_i16_param_0];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
 ; CHECK-NEXT:    shl.b16 %rs3, %rs2, 8;
-; CHECK-NEXT:    and.b16 %rs4, %rs1, 255;
-; CHECK-NEXT:    or.b16 %rs5, %rs4, %rs3;
-; CHECK-NEXT:    cvt.u32.u16 %r2, %rs5;
+; CHECK-NEXT:    or.b16 %rs4, %rs1, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r2, %rs4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
   %res = bitcast <2 x i8> %a to i16
@@ -32,14 +31,11 @@ define i16 @test_bitcast_2xi8_i16(<2 x i8> %a) {
 define <2 x i8> @test_bitcast_i16_2xi8(i16 %a) {
 ; CHECK-LABEL: test_bitcast_i16_2xi8(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b16 %rs<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b16 %rs1, [test_bitcast_i16_2xi8_param_0];
-; CHECK-NEXT:    shr.u16 %rs2, %rs1, 8;
-; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
 ; CHECK-NEXT:    ret;
   %res = bitcast i16 %a to <2 x i8>
   ret <2 x i8> %res
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index 4e1d13696edf..fd2e56bb126b 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -14,11 +14,10 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define <4 x i8> @test_ret_const() #0 {
 ; CHECK-LABEL: test_ret_const(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, -66911489;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], -66911489;
 ; CHECK-NEXT:    ret;
   ret <4 x i8> <i8 -1, i8 2, i8 3, i8 -4>
 }
@@ -268,19 +267,19 @@ define <4 x i8> @test_smax(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-NEXT:    bfe.s32 %r9, %r2, 24, 8;
 ; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
 ; CHECK-NEXT:    setp.gt.s32 %p4, %r10, %r9;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r15, %r2, 24, 8;
-; CHECK-NEXT:    selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT:    bfe.u32 %r17, %r2, 16, 8;
-; CHECK-NEXT:    selp.b32 %r18, %r13, %r17, %p3;
+; CHECK-NEXT:    bfe.u32 %r11, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r13, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    selp.b32 %r16, %r15, %r14, %p4;
+; CHECK-NEXT:    bfe.u32 %r17, %r1, 16, 8;
+; CHECK-NEXT:    selp.b32 %r18, %r17, %r13, %p3;
 ; CHECK-NEXT:    prmt.b32 %r19, %r18, %r16, 0x3340U;
-; CHECK-NEXT:    bfe.u32 %r20, %r2, 8, 8;
-; CHECK-NEXT:    selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT:    bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT:    selp.b32 %r23, %r11, %r22, %p1;
+; CHECK-NEXT:    bfe.u32 %r20, %r1, 8, 8;
+; CHECK-NEXT:    selp.b32 %r21, %r20, %r12, %p2;
+; CHECK-NEXT:    bfe.u32 %r22, %r1, 0, 8;
+; CHECK-NEXT:    selp.b32 %r23, %r22, %r11, %p1;
 ; CHECK-NEXT:    prmt.b32 %r24, %r23, %r21, 0x3340U;
 ; CHECK-NEXT:    prmt.b32 %r25, %r24, %r19, 0x5410U;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r25;
@@ -346,19 +345,19 @@ define <4 x i8> @test_smin(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-NEXT:    bfe.s32 %r9, %r2, 24, 8;
 ; CHECK-NEXT:    bfe.s32 %r10, %r1, 24, 8;
 ; CHECK-NEXT:    setp.le.s32 %p4, %r10, %r9;
-; CHECK-NEXT:    bfe.u32 %r11, %r1, 0, 8;
-; CHECK-NEXT:    bfe.u32 %r12, %r1, 8, 8;
-; CHECK-NEXT:    bfe.u32 %r13, %r1, 16, 8;
-; CHECK-NEXT:    bfe.u32 %r14, %r1, 24, 8;
-; CHECK-NEXT:    bfe.u32 %r15, %r2, 24, 8;
-; CHECK-NEXT:    selp.b32 %r16, %r14, %r15, %p4;
-; CHECK-NEXT:    bfe.u32 %r17, %r2, 16, 8;
-; CHECK-NEXT:    selp.b32 %r18, %r13, %r17, %p3;
+; CHECK-NEXT:    bfe.u32 %r11, %r2, 0, 8;
+; CHECK-NEXT:    bfe.u32 %r12, %r2, 8, 8;
+; CHECK-NEXT:    bfe.u32 %r13, %r2, 16, 8;
+; CHECK-NEXT:    bfe.u32 %r14, %r2, 24, 8;
+; CHECK-NEXT:    bfe.u32 %r15, %r1, 24, 8;
+; CHECK-NEXT:    selp.b32 %r16, %r15, %r14, %p4;
+; CHECK-NEXT:    bfe.u32 %r17, %r1, 16, 8;
+; CHECK-NEXT:    selp.b32 %r18, %r17, %r13, %p3;
 ; CHECK-NEXT:    prmt.b32 %r19, %r18, %r16, 0x3340U;
-; CHECK-NEXT:    bfe.u32 %r20, %r2, 8, 8;
-; CHECK-NEXT:    selp.b32 %r21, %r12, %r20, %p2;
-; CHECK-NEXT:    bfe.u32 %r22, %r2, 0, 8;
-; CHECK-NEXT:    selp.b32 %r23, %r11, %r22, %p1;
+; CHECK-NEXT:    bfe.u32 %r20, %r1, 8, 8;
+; CHECK-NEXT:    selp.b32 %r21, %r20, %r12, %p2;
+; CHECK-NEXT:    bfe.u32 %r22, %r1, 0, 8;
+; CHECK-NEXT:    selp.b32 %r23, %r22, %r11, %p1;
 ; CHECK-NEXT:    prmt.b32 %r24, %r23, %r21, 0x3340U;
 ; CHECK-NEXT:    prmt.b32 %r25, %r24, %r19, 0x5410U;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r25;
@@ -833,12 +832,7 @@ define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r2;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -861,12 +855,7 @@ define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r1;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -889,12 +878,7 @@ define <4 x i8> @test_tailcall_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ; CHECK-NEXT:    .param .align 4 .b8 param1[4];
 ; CHECK-NEXT:    st.param.b32 [param1], %r1;
 ; CHECK-NEXT:    .param .align 4 .b8 retval0[4];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    test_callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
 ; CHECK-NEXT:    } // callseq 2
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
diff --git a/llvm/test/CodeGen/NVPTX/idioms.ll b/llvm/test/CodeGen/NVPTX/idioms.ll
index d41b9b997335..be84f9bfb1ae 100644
--- a/llvm/test/CodeGen/NVPTX/idioms.ll
+++ b/llvm/test/CodeGen/NVPTX/idioms.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; Check that various LLVM idioms get lowered to NVPTX as expected.
 
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_20 | FileCheck %s
@@ -8,114 +9,178 @@
 %struct.S16 = type { i16, i16 }
 %struct.S32 = type { i32, i32 }
 
-; CHECK-LABEL: abs_i16(
 define i16 @abs_i16(i16 %a) {
-; CHECK: abs.s16
+; CHECK-LABEL: abs_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [abs_i16_param_0];
+; CHECK-NEXT:    abs.s16 %rs2, %rs1;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   %neg = sub i16 0, %a
   %abs.cond = icmp sge i16 %a, 0
   %abs = select i1 %abs.cond, i16 %a, i16 %neg
   ret i16 %abs
 }
 
-; CHECK-LABEL: abs_i32(
 define i32 @abs_i32(i32 %a) {
-; CHECK: abs.s32
+; CHECK-LABEL: abs_i32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [abs_i32_param_0];
+; CHECK-NEXT:    abs.s32 %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %neg = sub i32 0, %a
   %abs.cond = icmp sge i32 %a, 0
   %abs = select i1 %abs.cond, i32 %a, i32 %neg
   ret i32 %abs
 }
 
-; CHECK-LABEL: abs_i64(
 define i64 @abs_i64(i64 %a) {
-; CHECK: abs.s64
+; CHECK-LABEL: abs_i64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [abs_i64_param_0];
+; CHECK-NEXT:    abs.s64 %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
   %neg = sub i64 0, %a
   %abs.cond = icmp sge i64 %a, 0
   %abs = select i1 %abs.cond, i64 %a, i64 %neg
   ret i64 %abs
 }
 
-; CHECK-LABEL: i32_to_2xi16(
 define %struct.S16 @i32_to_2xi16(i32 noundef %in) {
+; CHECK-LABEL: i32_to_2xi16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [i32_to_2xi16_param_0];
+; CHECK-NEXT:    shr.u32 %r2, %r1, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+2], %r2;
+; CHECK-NEXT:    ret;
   %low = trunc i32 %in to i16
   %high32 = lshr i32 %in, 16
   %high = trunc i32 %high32 to i16
-; CHECK:       ld.param.b32  %[[R32:r[0-9]+]], [i32_to_2xi16_param_0];
-; CHECK-DAG:   cvt.u16.u32   %rs{{[0-9+]}}, %[[R32]];
-; CHECK-DAG    mov.b32       {tmp, %rs{{[0-9+]}}}, %[[R32]];
   %s1 = insertvalue %struct.S16 poison, i16 %low, 0
   %s = insertvalue %struct.S16 %s1, i16 %high, 1
   ret %struct.S16 %s
 }
 
-; CHECK-LABEL: i32_to_2xi16_lh(
 ; Same as above, but with rearranged order of low/high parts.
 define %struct.S16 @i32_to_2xi16_lh(i32 noundef %in) {
+; CHECK-LABEL: i32_to_2xi16_lh(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [i32_to_2xi16_lh_param_0];
+; CHECK-NEXT:    shr.u32 %r2, %r1, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+2], %r2;
+; CHECK-NEXT:    ret;
   %high32 = lshr i32 %in, 16
   %high = trunc i32 %high32 to i16
   %low = trunc i32 %in to i16
-; CHECK:       ld.param.b32  %[[R32:r[0-9]+]], [i32_to_2xi16_lh_param_0];
-; CHECK-DAG:   cvt.u16.u32   %rs{{[0-9+]}}, %[[R32]];
-; CHECK-DAG    mov.b32       {tmp, %rs{{[0-9+]}}}, %[[R32]];
   %s1 = insertvalue %struct.S16 poison, i16 %low, 0
   %s = insertvalue %struct.S16 %s1, i16 %high, 1
   ret %struct.S16 %s
 }
 
 
-; CHECK-LABEL: i32_to_2xi16_not(
 define %struct.S16 @i32_to_2xi16_not(i32 noundef %in) {
+; CHECK-LABEL: i32_to_2xi16_not(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [i32_to_2xi16_not_param_0];
+; CHECK-NEXT:    shr.u32 %r2, %r1, 15;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+2], %r2;
+; CHECK-NEXT:    ret;
   %low = trunc i32 %in to i16
   ;  Shift by any value other than 16 blocks the conversiopn to mov.
   %high32 = lshr i32 %in, 15
   %high = trunc i32 %high32 to i16
-; CHECK:       cvt.u16.u32
-; CHECK:       shr.u32
-; CHECK:       cvt.u16.u32
   %s1 = insertvalue %struct.S16 poison, i16 %low, 0
   %s = insertvalue %struct.S16 %s1, i16 %high, 1
   ret %struct.S16 %s
 }
 
-; CHECK-LABEL: i64_to_2xi32(
 define %struct.S32 @i64_to_2xi32(i64 noundef %in) {
+; CHECK-LABEL: i64_to_2xi32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [i64_to_2xi32_param_0];
+; CHECK-NEXT:    shr.u64 %rd2, %rd1, 32;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %rd1;
+; CHECK-NEXT:    st.param.b32 [func_retval0+4], %rd2;
+; CHECK-NEXT:    ret;
   %low = trunc i64 %in to i32
   %high64 = lshr i64 %in, 32
   %high = trunc i64 %high64 to i32
-; CHECK:       ld.param.b64  %[[R64:rd[0-9]+]], [i64_to_2xi32_param_0];
-; CHECK-DAG:   cvt.u32.u64   %r{{[0-9+]}}, %[[R64]];
-; CHECK-DAG    mov.b64       {tmp, %r{{[0-9+]}}}, %[[R64]];
   %s1 = insertvalue %struct.S32 poison, i32 %low, 0
   %s = insertvalue %struct.S32 %s1, i32 %high, 1
   ret %struct.S32 %s
 }
 
-; CHECK-LABEL: i64_to_2xi32_not(
 define %struct.S32 @i64_to_2xi32_not(i64 noundef %in) {
+; CHECK-LABEL: i64_to_2xi32_not(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [i64_to_2xi32_not_param_0];
+; CHECK-NEXT:    shr.u64 %rd2, %rd1, 31;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %rd1;
+; CHECK-NEXT:    st.param.b32 [func_retval0+4], %rd2;
+; CHECK-NEXT:    ret;
   %low = trunc i64 %in to i32
   ;  Shift by any value other than 32 blocks the conversiopn to mov.
   %high64 = lshr i64 %in, 31
   %high = trunc i64 %high64 to i32
-; CHECK:       cvt.u32.u64
-; CHECK:       shr.u64
-; CHECK:       cvt.u32.u64
   %s1 = insertvalue %struct.S32 poison, i32 %low, 0
   %s = insertvalue %struct.S32 %s1, i32 %high, 1
   ret %struct.S32 %s
 }
 
-; CHECK-LABEL: i32_to_2xi16_shr(
 ; Make sure we do not get confused when our input itself is [al]shr.
 define %struct.S16 @i32_to_2xi16_shr(i32 noundef %i){
+; CHECK-LABEL: i32_to_2xi16_shr(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [i32_to_2xi16_shr_param_0];
+; CHECK-NEXT:    { // callseq 0, 0
+; CHECK-NEXT:    .param .b32 param0;
+; CHECK-NEXT:    st.param.b32 [param0], %r1;
+; CHECK-NEXT:    call.uni escape_int, (param0);
+; CHECK-NEXT:    } // callseq 0
+; CHECK-NEXT:    shr.s32 %r2, %r1, 16;
+; CHECK-NEXT:    shr.u32 %r3, %r2, 16;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.b16 [func_retval0+2], %r3;
+; CHECK-NEXT:    ret;
   call void @escape_int(i32 %i); // Force %i to be loaded completely.
   %i1 = ashr i32 %i, 16
   %l = trunc i32 %i1 to i16
   %h32 = ashr i32 %i1, 16
   %h = trunc i32 %h32 to i16
-; CHECK:      ld.param.b32    %[[R32:r[0-9]+]], [i32_to_2xi16_shr_param_0];
-; CHECK:      shr.s32         %[[R32H:r[0-9]+]], %[[R32]], 16;
-; CHECK-DAG    mov.b32       {tmp, %rs{{[0-9+]}}}, %[[R32]];
-; CHECK-DAG    mov.b32       {tmp, %rs{{[0-9+]}}}, %[[R32H]];
   %s0 = insertvalue %struct.S16 poison, i16 %l, 0
   %s1 = insertvalue %struct.S16 %s0, i16 %h, 1
   ret %struct.S16 %s1
diff --git a/llvm/test/CodeGen/NVPTX/indirect_byval.ll b/llvm/test/CodeGen/NVPTX/indirect_byval.ll
index 1341a04c939c..eae032143394 100644
--- a/llvm/test/CodeGen/NVPTX/indirect_byval.ll
+++ b/llvm/test/CodeGen/NVPTX/indirect_byval.ll
@@ -33,13 +33,7 @@ define internal i32 @foo() {
 ; CHECK-NEXT:    st.param.b64 [param1], %rd4;
 ; CHECK-NEXT:    .param .b32 retval0;
 ; CHECK-NEXT:    prototype_0 : .callprototype (.param .b32 _) _ (.param .align 1 .b8 _[1], .param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd1,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_0;
+; CHECK-NEXT:    call (retval0), %rd1, (param0, param1), prototype_0;
 ; CHECK-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
@@ -76,13 +70,7 @@ define internal i32 @bar() {
 ; CHECK-NEXT:    st.param.b64 [param1], %rd5;
 ; CHECK-NEXT:    .param .b32 retval0;
 ; CHECK-NEXT:    prototype_1 : .callprototype (.param .b32 _) _ (.param .align 8 .b8 _[8], .param .b64 _);
-; CHECK-NEXT:    call (retval0),
-; CHECK-NEXT:    %rd1,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0,
-; CHECK-NEXT:    param1
-; CHECK-NEXT:    )
-; CHECK-NEXT:    , prototype_1;
+; CHECK-NEXT:    call (retval0), %rd1, (param0, param1), prototype_1;
 ; CHECK-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
diff --git a/llvm/test/CodeGen/NVPTX/jump-table.ll b/llvm/test/CodeGen/NVPTX/jump-table.ll
index 0718e6d603b6..955befc624c7 100644
--- a/llvm/test/CodeGen/NVPTX/jump-table.ll
+++ b/llvm/test/CodeGen/NVPTX/jump-table.ll
@@ -10,7 +10,7 @@ define void @foo(i32 %i) {
 ; CHECK-LABEL: foo(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b32 %r2, [foo_param_0];
@@ -24,20 +24,16 @@ define void @foo(i32 %i) {
 ; CHECK-NEXT:     $L__BB0_5;
 ; CHECK-NEXT:    brx.idx %r2, $L_brx_0;
 ; CHECK-NEXT:  $L__BB0_2: // %case0
-; CHECK-NEXT:    mov.b32 %r6, 0;
-; CHECK-NEXT:    st.global.b32 [out], %r6;
+; CHECK-NEXT:    st.global.b32 [out], 0;
 ; CHECK-NEXT:    bra.uni $L__BB0_6;
 ; CHECK-NEXT:  $L__BB0_4: // %case2
-; CHECK-NEXT:    mov.b32 %r4, 2;
-; CHECK-NEXT:    st.global.b32 [out], %r4;
+; CHECK-NEXT:    st.global.b32 [out], 2;
 ; CHECK-NEXT:    bra.uni $L__BB0_6;
 ; CHECK-NEXT:  $L__BB0_5: // %case3
-; CHECK-NEXT:    mov.b32 %r3, 3;
-; CHECK-NEXT:    st.global.b32 [out], %r3;
+; CHECK-NEXT:    st.global.b32 [out], 3;
 ; CHECK-NEXT:    bra.uni $L__BB0_6;
 ; CHECK-NEXT:  $L__BB0_3: // %case1
-; CHECK-NEXT:    mov.b32 %r5, 1;
-; CHECK-NEXT:    st.global.b32 [out], %r5;
+; CHECK-NEXT:    st.global.b32 [out], 1;
 ; CHECK-NEXT:  $L__BB0_6: // %end
 ; CHECK-NEXT:    ret;
 entry:
@@ -73,7 +69,7 @@ define i32 @test2(i32 %tmp158) {
 ; CHECK-LABEL: test2(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<6>;
-; CHECK-NEXT:    .reg .b32 %r<10>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b32 %r1, [test2_param_0];
@@ -100,36 +96,29 @@ define i32 @test2(i32 %tmp158) {
 ; CHECK-NEXT:     $L__BB1_11;
 ; CHECK-NEXT:    brx.idx %r2, $L_brx_0;
 ; CHECK-NEXT:  $L__BB1_7: // %bb339
-; CHECK-NEXT:    mov.b32 %r7, 12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 12;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_5: // %entry
 ; CHECK-NEXT:    setp.eq.s32 %p3, %r1, 1024;
 ; CHECK-NEXT:    @%p3 bra $L__BB1_3;
 ; CHECK-NEXT:    bra.uni $L__BB1_6;
 ; CHECK-NEXT:  $L__BB1_3: // %bb338
-; CHECK-NEXT:    mov.b32 %r8, 11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 11;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_10: // %bb342
-; CHECK-NEXT:    mov.b32 %r4, 15;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 15;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_6: // %bb336
-; CHECK-NEXT:    mov.b32 %r9, 10;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 10;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_8: // %bb340
-; CHECK-NEXT:    mov.b32 %r6, 13;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 13;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_9: // %bb341
-; CHECK-NEXT:    mov.b32 %r5, 14;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 14;
 ; CHECK-NEXT:    ret;
 ; CHECK-NEXT:  $L__BB1_11: // %bb343
-; CHECK-NEXT:    mov.b32 %r3, 18;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 18;
 ; CHECK-NEXT:    ret;
 entry:
   switch i32 %tmp158, label %bb336 [
diff --git a/llvm/test/CodeGen/NVPTX/ldg-invariant-256.ll b/llvm/test/CodeGen/NVPTX/ldg-invariant-256.ll
index b4a74c762f52..d5078f5f19af 100644
--- a/llvm/test/CodeGen/NVPTX/ldg-invariant-256.ll
+++ b/llvm/test/CodeGen/NVPTX/ldg-invariant-256.ll
@@ -11,7 +11,7 @@ define i8 @ld_global_v32i8(ptr addrspace(1) %ptr) {
 ; SM90-LABEL: ld_global_v32i8(
 ; SM90:       {
 ; SM90-NEXT:    .reg .b16 %rs<16>;
-; SM90-NEXT:    .reg .b32 %r<19>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<2>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
@@ -42,14 +42,13 @@ define i8 @ld_global_v32i8(ptr addrspace(1) %ptr) {
 ; SM90-NEXT:    add.s16 %rs14, %rs11, %rs12;
 ; SM90-NEXT:    add.s16 %rs15, %rs13, %rs14;
 ; SM90-NEXT:    cvt.u32.u16 %r17, %rs15;
-; SM90-NEXT:    and.b32 %r18, %r17, 255;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r18;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r17;
 ; SM90-NEXT:    ret;
 ;
 ; SM100-LABEL: ld_global_v32i8(
 ; SM100:       {
 ; SM100-NEXT:    .reg .b16 %rs<16>;
-; SM100-NEXT:    .reg .b32 %r<19>;
+; SM100-NEXT:    .reg .b32 %r<18>;
 ; SM100-NEXT:    .reg .b64 %rd<2>;
 ; SM100-EMPTY:
 ; SM100-NEXT:  // %bb.0:
@@ -79,8 +78,7 @@ define i8 @ld_global_v32i8(ptr addrspace(1) %ptr) {
 ; SM100-NEXT:    add.s16 %rs14, %rs11, %rs12;
 ; SM100-NEXT:    add.s16 %rs15, %rs13, %rs14;
 ; SM100-NEXT:    cvt.u32.u16 %r17, %rs15;
-; SM100-NEXT:    and.b32 %r18, %r17, 255;
-; SM100-NEXT:    st.param.b32 [func_retval0], %r18;
+; SM100-NEXT:    st.param.b32 [func_retval0], %r17;
 ; SM100-NEXT:    ret;
   %a = load <32 x i8>, ptr addrspace(1) %ptr, !invariant.load !0
   %v1 = extractelement <32 x i8> %a, i32 0
diff --git a/llvm/test/CodeGen/NVPTX/ldg-invariant.ll b/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
index c5c5de4c1b85..3bd46000661c 100644
--- a/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
+++ b/llvm/test/CodeGen/NVPTX/ldg-invariant.ll
@@ -26,17 +26,16 @@ define half @ld_global_v2f16(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: ld_global_v2f16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [ld_global_v2f16_param_0];
-; CHECK-NEXT:    ld.global.nc.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    cvt.f32.f16 %r2, %rs2;
-; CHECK-NEXT:    cvt.f32.f16 %r3, %rs1;
-; CHECK-NEXT:    add.rn.f32 %r4, %r3, %r2;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r4;
+; CHECK-NEXT:    ld.global.nc.v2.b16 {%rs1, %rs2}, [%rd1];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT:    add.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r3;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
 ; CHECK-NEXT:    ret;
   %a = load <2 x half>, ptr addrspace(1) %ptr, !invariant.load !0
@@ -129,7 +128,7 @@ define i8 @ld_global_v8i8(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: ld_global_v8i8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<8>;
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -147,8 +146,7 @@ define i8 @ld_global_v8i8(ptr addrspace(1) %ptr) {
 ; CHECK-NEXT:    add.s16 %rs6, %rs2, %rs1;
 ; CHECK-NEXT:    add.s16 %rs7, %rs5, %rs6;
 ; CHECK-NEXT:    cvt.u32.u16 %r7, %rs7;
-; CHECK-NEXT:    and.b32 %r8, %r7, 255;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
   %a = load <8 x i8>, ptr addrspace(1) %ptr, !invariant.load !0
   %v1 = extractelement <8 x i8> %a, i32 0
@@ -165,7 +163,7 @@ define i8 @ld_global_v16i8(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: ld_global_v16i8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<16>;
-; CHECK-NEXT:    .reg .b32 %r<15>;
+; CHECK-NEXT:    .reg .b32 %r<14>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -195,8 +193,7 @@ define i8 @ld_global_v16i8(ptr addrspace(1) %ptr) {
 ; CHECK-NEXT:    add.s16 %rs14, %rs11, %rs12;
 ; CHECK-NEXT:    add.s16 %rs15, %rs13, %rs14;
 ; CHECK-NEXT:    cvt.u32.u16 %r13, %rs15;
-; CHECK-NEXT:    and.b32 %r14, %r13, 255;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
 ; CHECK-NEXT:    ret;
   %a = load <16 x i8>, ptr addrspace(1) %ptr, !invariant.load !0
   %v1 = extractelement <16 x i8> %a, i32 0
diff --git a/llvm/test/CodeGen/NVPTX/ldparam-v4.ll b/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
index 419c780f7d82..9e9705709f2b 100644
--- a/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
+++ b/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
@@ -14,10 +14,7 @@ define void @foo(ptr %ptr) {
 ; CHECK-NEXT:    ld.param.b64 %rd1, [foo_param_0];
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 16 .b8 retval0[16];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    bar,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), bar, ();
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
diff --git a/llvm/test/CodeGen/NVPTX/ldu-i8.ll b/llvm/test/CodeGen/NVPTX/ldu-i8.ll
index 89f23f30f34e..5a0a66b8b93e 100644
--- a/llvm/test/CodeGen/NVPTX/ldu-i8.ll
+++ b/llvm/test/CodeGen/NVPTX/ldu-i8.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -verify-machineinstrs | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
@@ -6,10 +7,18 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 declare i8 @llvm.nvvm.ldu.global.i.i8.p0(ptr, i32)
 
 define i8 @foo(ptr %a) {
-; Ensure we properly truncate off the high-order 24 bits
-; CHECK:        ldu.global.b8
-; CHECK:        cvt.u32.u16
-; CHECK:        and.b32         %r{{[0-9]+}}, %r{{[0-9]+}}, 255
+; CHECK-LABEL: foo(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo_param_0];
+; CHECK-NEXT:    ldu.global.b8 %rs1, [%rd1];
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p0(ptr %a, i32 4)
   ret i8 %val
 }
diff --git a/llvm/test/CodeGen/NVPTX/ldu-ldg.ll b/llvm/test/CodeGen/NVPTX/ldu-ldg.ll
index 7f4b049af84f..977e7353f58e 100644
--- a/llvm/test/CodeGen/NVPTX/ldu-ldg.ll
+++ b/llvm/test/CodeGen/NVPTX/ldu-ldg.ll
@@ -27,15 +27,14 @@ define i8 @test_ldu_i8(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: test_ldu_i8(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldu_i8_param_0];
 ; CHECK-NEXT:    ldu.global.b8 %rs1, [%rd1];
 ; CHECK-NEXT:    cvt.u32.u16 %r1, %rs1;
-; CHECK-NEXT:    and.b32 %r2, %r1, 255;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
   ret i8 %val
diff --git a/llvm/test/CodeGen/NVPTX/load-store-vectors.ll b/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
index 9e7e940ab5a7..3c90323da01d 100644
--- a/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store-vectors.ll
@@ -237,17 +237,14 @@ define void @generic_2xi16(ptr %a) {
 ; CHECK-LABEL: generic_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [generic_2xi16_param_0];
-; CHECK-NEXT:    ld.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load <2 x i16>, ptr %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -657,17 +654,14 @@ define void @generic_volatile_2xi16(ptr %a) {
 ; CHECK-LABEL: generic_volatile_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [generic_volatile_2xi16_param_0];
-; CHECK-NEXT:    ld.volatile.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.volatile.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.volatile.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.volatile.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load volatile <2 x i16>, ptr %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1060,17 +1054,14 @@ define void @global_2xi16(ptr addrspace(1) %a) {
 ; CHECK-LABEL: global_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [global_2xi16_param_0];
-; CHECK-NEXT:    ld.global.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.global.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.global.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.global.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load <2 x i16>, ptr addrspace(1) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1461,17 +1452,14 @@ define void @global_volatile_2xi16(ptr addrspace(1) %a) {
 ; CHECK-LABEL: global_volatile_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [global_volatile_2xi16_param_0];
-; CHECK-NEXT:    ld.volatile.global.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.volatile.global.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.volatile.global.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.volatile.global.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load volatile <2 x i16>, ptr addrspace(1) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -1864,17 +1852,14 @@ define void @shared_2xi16(ptr addrspace(3) %a) {
 ; CHECK-LABEL: shared_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [shared_2xi16_param_0];
-; CHECK-NEXT:    ld.shared.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.shared.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.shared.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.shared.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load <2 x i16>, ptr addrspace(3) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -2265,17 +2250,14 @@ define void @shared_volatile_2xi16(ptr addrspace(3) %a) {
 ; CHECK-LABEL: shared_volatile_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [shared_volatile_2xi16_param_0];
-; CHECK-NEXT:    ld.volatile.shared.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.volatile.shared.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.volatile.shared.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.volatile.shared.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load volatile <2 x i16>, ptr addrspace(3) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -2668,17 +2650,14 @@ define void @local_2xi16(ptr addrspace(5) %a) {
 ; CHECK-LABEL: local_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [local_2xi16_param_0];
-; CHECK-NEXT:    ld.local.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.local.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.local.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.local.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load <2 x i16>, ptr addrspace(5) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
@@ -3069,17 +3048,14 @@ define void @local_volatile_2xi16(ptr addrspace(5) %a) {
 ; CHECK-LABEL: local_volatile_2xi16(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [local_volatile_2xi16_param_0];
-; CHECK-NEXT:    ld.local.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.local.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    add.s16 %rs3, %rs2, 1;
 ; CHECK-NEXT:    add.s16 %rs4, %rs1, 1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.local.b32 [%rd1], %r2;
+; CHECK-NEXT:    st.local.v2.b16 [%rd1], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
   %a.load = load volatile <2 x i16>, ptr addrspace(5) %a
   %a.add = add <2 x i16> %a.load, <i16 1, i16 1>
diff --git a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
index 4d7a4b50e894..d494ee30c282 100644
--- a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
+++ b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -verify-machineinstrs | FileCheck -check-prefix=SM20 %s
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_35 -verify-machineinstrs | FileCheck -check-prefix=SM35 %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
@@ -6,195 +7,642 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
 target triple = "nvptx64-unknown-unknown"
 
-; SM20-LABEL: .visible .entry foo1(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo1(
-; SM35: ld.global.nc.b32
 define ptx_kernel void @foo1(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo1(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo1_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo1_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b32 %r1, [%rd2];
+; SM20-NEXT:    st.global.b32 [%rd4], %r1;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo1(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo1_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo1_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT:    st.global.b32 [%rd4], %r1;
+; SM35-NEXT:    ret;
   %1 = load float, ptr %from
   store float %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo2(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo2(
-; SM35: ld.global.nc.b64
 define ptx_kernel void @foo2(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo2(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo2_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo2_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo2(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo2_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo2_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM35-NEXT:    ret;
   %1 = load double, ptr %from
   store double %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo3(
-; SM20: ld.global.b16
-; SM35-LABEL: .visible .entry foo3(
-; SM35: ld.global.nc.b16
 define ptx_kernel void @foo3(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo3(
+; SM20:       {
+; SM20-NEXT:    .reg .b16 %rs<2>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo3_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo3_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b16 %rs1, [%rd2];
+; SM20-NEXT:    st.global.b16 [%rd4], %rs1;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo3(
+; SM35:       {
+; SM35-NEXT:    .reg .b16 %rs<2>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo3_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo3_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b16 %rs1, [%rd2];
+; SM35-NEXT:    st.global.b16 [%rd4], %rs1;
+; SM35-NEXT:    ret;
   %1 = load i16, ptr %from
   store i16 %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo4(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo4(
-; SM35: ld.global.nc.b32
 define ptx_kernel void @foo4(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo4(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo4_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo4_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b32 %r1, [%rd2];
+; SM20-NEXT:    st.global.b32 [%rd4], %r1;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo4(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo4_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo4_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT:    st.global.b32 [%rd4], %r1;
+; SM35-NEXT:    ret;
   %1 = load i32, ptr %from
   store i32 %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo5(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo5(
-; SM35: ld.global.nc.b64
 define ptx_kernel void @foo5(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo5(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo5_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo5_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo5(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo5_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo5_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM35-NEXT:    ret;
   %1 = load i64, ptr %from
   store i64 %1, ptr %to
   ret void
 }
 
 ; i128 is non standard integer in nvptx64
-; SM20-LABEL: .visible .entry foo6(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo6(
-; SM35: ld.global.nc.v2.b64
 define ptx_kernel void @foo6(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo6(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo6_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo6_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo6(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo6_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo6_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT:    ret;
   %1 = load i128, ptr %from
   store i128 %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo7(
-; SM20: ld.global.v2.b8
-; SM35-LABEL: .visible .entry foo7(
-; SM35: ld.global.nc.v2.b8
 define ptx_kernel void @foo7(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo7(
+; SM20:       {
+; SM20-NEXT:    .reg .b16 %rs<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo7_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo7_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b8 {%rs1, %rs2}, [%rd2];
+; SM20-NEXT:    st.global.v2.b8 [%rd4], {%rs1, %rs2};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo7(
+; SM35:       {
+; SM35-NEXT:    .reg .b16 %rs<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo7_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo7_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b8 {%rs1, %rs2}, [%rd2];
+; SM35-NEXT:    st.global.v2.b8 [%rd4], {%rs1, %rs2};
+; SM35-NEXT:    ret;
   %1 = load <2 x i8>, ptr %from
   store <2 x i8> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo8(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo8(
-; SM35: ld.global.nc.b32
 define ptx_kernel void @foo8(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo8(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo8_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo8_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b32 %r1, [%rd2];
+; SM20-NEXT:    st.global.b32 [%rd4], %r1;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo8(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo8_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo8_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT:    st.global.b32 [%rd4], %r1;
+; SM35-NEXT:    ret;
   %1 = load <2 x i16>, ptr %from
   store <2 x i16> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo9(
-; SM20: ld.global.v2.b32
-; SM35-LABEL: .visible .entry foo9(
-; SM35: ld.global.nc.v2.b32
 define ptx_kernel void @foo9(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo9(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo9_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo9_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo9(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo9_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo9_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT:    ret;
   %1 = load <2 x i32>, ptr %from
   store <2 x i32> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo10(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo10(
-; SM35: ld.global.nc.v2.b64
 define ptx_kernel void @foo10(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo10(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo10_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo10_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo10(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo10_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo10_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT:    ret;
   %1 = load <2 x i64>, ptr %from
   store <2 x i64> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo11(
-; SM20: ld.global.v2.b32
-; SM35-LABEL: .visible .entry foo11(
-; SM35: ld.global.nc.v2.b32
 define ptx_kernel void @foo11(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo11(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo11_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo11_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo11(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo11_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo11_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT:    ret;
   %1 = load <2 x float>, ptr %from
   store <2 x float> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo12(
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo12(
-; SM35: ld.global.nc.v2.b64
 define ptx_kernel void @foo12(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo12(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo12_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo12_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo12(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo12_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo12_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT:    ret;
   %1 = load <2 x double>, ptr %from
   store <2 x double> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo13(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo13(
-; SM35: ld.global.nc.b32
 define ptx_kernel void @foo13(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo13(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo13_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo13_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b32 %r1, [%rd2];
+; SM20-NEXT:    st.global.b32 [%rd4], %r1;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo13(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo13_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo13_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b32 %r1, [%rd2];
+; SM35-NEXT:    st.global.b32 [%rd4], %r1;
+; SM35-NEXT:    ret;
   %1 = load <4 x i8>, ptr %from
   store <4 x i8> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo14(
-; SM20: ld.global.v4.b16
-; SM35-LABEL: .visible .entry foo14(
-; SM35: ld.global.nc.v4.b16
 define ptx_kernel void @foo14(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo14(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo14_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo14_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo14(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo14_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo14_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
+; SM35-NEXT:    ret;
   %1 = load <4 x i16>, ptr %from
   store <4 x i16> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo15(
-; SM20: ld.global.v4.b32
-; SM35-LABEL: .visible .entry foo15(
-; SM35: ld.global.nc.v4.b32
 define ptx_kernel void @foo15(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo15(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo15_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo15_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM20-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo15(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo15_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo15_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM35-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM35-NEXT:    ret;
   %1 = load <4 x i32>, ptr %from
   store <4 x i32> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo16(
-; SM20: ld.global.v4.b32
-; SM35-LABEL: .visible .entry foo16(
-; SM35: ld.global.nc.v4.b32
 define ptx_kernel void @foo16(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo16(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo16_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo16_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM20-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo16(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo16_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo16_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM35-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
+; SM35-NEXT:    ret;
   %1 = load <4 x float>, ptr %from
   store <4 x float> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo17(
-; SM20: ld.global.v2.b64
-; SM20: ld.global.v2.b64
-; SM35-LABEL: .visible .entry foo17(
-; SM35: ld.global.nc.v2.b64
-; SM35: ld.global.nc.v2.b64
 define ptx_kernel void @foo17(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo17(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<9>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo17_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo17_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM20-NEXT:    ld.global.v2.b64 {%rd7, %rd8}, [%rd2+16];
+; SM20-NEXT:    st.global.v2.b64 [%rd4+16], {%rd7, %rd8};
+; SM20-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo17(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<9>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo17_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo17_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
+; SM35-NEXT:    ld.global.nc.v2.b64 {%rd7, %rd8}, [%rd2+16];
+; SM35-NEXT:    st.global.v2.b64 [%rd4+16], {%rd7, %rd8};
+; SM35-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT:    ret;
   %1 = load <4 x double>, ptr %from
   store <4 x double> %1, ptr %to
   ret void
 }
 
-; SM20-LABEL: .visible .entry foo18(
-; SM20: ld.global.b64
-; SM35-LABEL: .visible .entry foo18(
-; SM35: ld.global.nc.b64
 define ptx_kernel void @foo18(ptr noalias readonly %from, ptr %to) {
+; SM20-LABEL: foo18(
+; SM20:       {
+; SM20-NEXT:    .reg .b64 %rd<6>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [foo18_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo18_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM20-NEXT:    ld.global.b64 %rd5, [%rd2];
+; SM20-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo18(
+; SM35:       {
+; SM35-NEXT:    .reg .b64 %rd<6>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [foo18_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo18_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
+; SM35-NEXT:    ld.global.nc.b64 %rd5, [%rd2];
+; SM35-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM35-NEXT:    ret;
   %1 = load ptr, ptr %from
   store ptr %1, ptr %to
   ret void
 }
 
 ; Test that we can infer a cached load for a pointer induction variable.
-; SM20-LABEL: .visible .entry foo19(
-; SM20: ld.global.b32
-; SM35-LABEL: .visible .entry foo19(
-; SM35: ld.global.nc.b32
 define ptx_kernel void @foo19(ptr noalias readonly %from, ptr %to, i32 %n) {
+; SM20-LABEL: foo19(
+; SM20:       {
+; SM20-NEXT:    .reg .pred %p<2>;
+; SM20-NEXT:    .reg .b32 %r<10>;
+; SM20-NEXT:    .reg .b64 %rd<8>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0: // %entry
+; SM20-NEXT:    ld.param.b32 %r8, [foo19_param_2];
+; SM20-NEXT:    ld.param.b64 %rd5, [foo19_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd7, %rd5;
+; SM20-NEXT:    ld.param.b64 %rd6, [foo19_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd6;
+; SM20-NEXT:    mov.b32 %r9, 0f00000000;
+; SM20-NEXT:  $L__BB18_1: // %loop
+; SM20-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM20-NEXT:    ld.global.b32 %r7, [%rd7];
+; SM20-NEXT:    add.rn.f32 %r9, %r7, %r9;
+; SM20-NEXT:    add.s64 %rd7, %rd7, 4;
+; SM20-NEXT:    add.s32 %r8, %r8, -1;
+; SM20-NEXT:    setp.ne.s32 %p1, %r8, 0;
+; SM20-NEXT:    @%p1 bra $L__BB18_1;
+; SM20-NEXT:  // %bb.2: // %exit
+; SM20-NEXT:    st.global.b32 [%rd2], %r9;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: foo19(
+; SM35:       {
+; SM35-NEXT:    .reg .pred %p<2>;
+; SM35-NEXT:    .reg .b32 %r<10>;
+; SM35-NEXT:    .reg .b64 %rd<8>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0: // %entry
+; SM35-NEXT:    ld.param.b32 %r8, [foo19_param_2];
+; SM35-NEXT:    ld.param.b64 %rd5, [foo19_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd7, %rd5;
+; SM35-NEXT:    ld.param.b64 %rd6, [foo19_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd6;
+; SM35-NEXT:    mov.b32 %r9, 0f00000000;
+; SM35-NEXT:  $L__BB18_1: // %loop
+; SM35-NEXT:    // =>This Inner Loop Header: Depth=1
+; SM35-NEXT:    ld.global.nc.b32 %r7, [%rd7];
+; SM35-NEXT:    add.rn.f32 %r9, %r7, %r9;
+; SM35-NEXT:    add.s64 %rd7, %rd7, 4;
+; SM35-NEXT:    add.s32 %r8, %r8, -1;
+; SM35-NEXT:    setp.ne.s32 %p1, %r8, 0;
+; SM35-NEXT:    @%p1 bra $L__BB18_1;
+; SM35-NEXT:  // %bb.2: // %exit
+; SM35-NEXT:    st.global.b32 [%rd2], %r9;
+; SM35-NEXT:    ret;
 entry:
   br label %loop
 
@@ -218,11 +666,30 @@ exit:
 ; know that the parameter is global. We also do not know that the
 ; pointed-to memory is never written to (for the duration of the
 ; kernel). For both reasons, we cannot use a cached load here.
+define void @notkernel(ptr noalias readonly %from, ptr %to) {
 ; SM20-LABEL: notkernel(
-; SM20: ld.b32
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [notkernel_param_0];
+; SM20-NEXT:    ld.b32 %r1, [%rd1];
+; SM20-NEXT:    ld.param.b64 %rd2, [notkernel_param_1];
+; SM20-NEXT:    st.b32 [%rd2], %r1;
+; SM20-NEXT:    ret;
+;
 ; SM35-LABEL: notkernel(
-; SM35: ld.b32
-define void @notkernel(ptr noalias readonly %from, ptr %to) {
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [notkernel_param_0];
+; SM35-NEXT:    ld.b32 %r1, [%rd1];
+; SM35-NEXT:    ld.param.b64 %rd2, [notkernel_param_1];
+; SM35-NEXT:    st.b32 [%rd2], %r1;
+; SM35-NEXT:    ret;
   %1 = load float, ptr %from
   store float %1, ptr %to
   ret void
@@ -232,11 +699,30 @@ define void @notkernel(ptr noalias readonly %from, ptr %to) {
 ; do not know that the parameter is never written to (for the duration of the
 ; kernel). This case does not currently come up normally since we do not infer
 ; that pointers are global interprocedurally as of 2015-08-05.
+define void @notkernel2(ptr addrspace(1) noalias readonly %from, ptr %to) {
 ; SM20-LABEL: notkernel2(
-; SM20: ld.global.b32
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<2>;
+; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.b64 %rd1, [notkernel2_param_0];
+; SM20-NEXT:    ld.global.b32 %r1, [%rd1];
+; SM20-NEXT:    ld.param.b64 %rd2, [notkernel2_param_1];
+; SM20-NEXT:    st.b32 [%rd2], %r1;
+; SM20-NEXT:    ret;
+;
 ; SM35-LABEL: notkernel2(
-; SM35: ld.global.b32
-define void @notkernel2(ptr addrspace(1) noalias readonly %from, ptr %to) {
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<2>;
+; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.b64 %rd1, [notkernel2_param_0];
+; SM35-NEXT:    ld.global.b32 %r1, [%rd1];
+; SM35-NEXT:    ld.param.b64 %rd2, [notkernel2_param_1];
+; SM35-NEXT:    st.b32 [%rd2], %r1;
+; SM35-NEXT:    ret;
   %1 = load float, ptr addrspace(1) %from
   store float %1, ptr %to
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/local-stack-frame.ll b/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
index 2bfd891a04a1..5c3017310d0a 100644
--- a/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
+++ b/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
@@ -58,11 +58,7 @@ define ptx_kernel void @foo2(i32 %a) {
 ; PTX32-NEXT:    { // callseq 0, 0
 ; PTX32-NEXT:    .param .b32 param0;
 ; PTX32-NEXT:    st.param.b32 [param0], %r2;
-; PTX32-NEXT:    call.uni
-; PTX32-NEXT:    bar,
-; PTX32-NEXT:    (
-; PTX32-NEXT:    param0
-; PTX32-NEXT:    );
+; PTX32-NEXT:    call.uni bar, (param0);
 ; PTX32-NEXT:    } // callseq 0
 ; PTX32-NEXT:    ret;
 ;
@@ -84,11 +80,7 @@ define ptx_kernel void @foo2(i32 %a) {
 ; PTX64-NEXT:    { // callseq 0, 0
 ; PTX64-NEXT:    .param .b64 param0;
 ; PTX64-NEXT:    st.param.b64 [param0], %rd1;
-; PTX64-NEXT:    call.uni
-; PTX64-NEXT:    bar,
-; PTX64-NEXT:    (
-; PTX64-NEXT:    param0
-; PTX64-NEXT:    );
+; PTX64-NEXT:    call.uni bar, (param0);
 ; PTX64-NEXT:    } // callseq 0
 ; PTX64-NEXT:    ret;
   %local = alloca i32, align 4
@@ -144,7 +136,7 @@ define void @foo4() {
 ; PTX32-NEXT:    .local .align 4 .b8 __local_depot3[8];
 ; PTX32-NEXT:    .reg .b32 %SP;
 ; PTX32-NEXT:    .reg .b32 %SPL;
-; PTX32-NEXT:    .reg .b32 %r<6>;
+; PTX32-NEXT:    .reg .b32 %r<5>;
 ; PTX32-EMPTY:
 ; PTX32-NEXT:  // %bb.0:
 ; PTX32-NEXT:    mov.b32 %SPL, __local_depot3;
@@ -153,26 +145,17 @@ define void @foo4() {
 ; PTX32-NEXT:    add.u32 %r2, %SPL, 0;
 ; PTX32-NEXT:    add.u32 %r3, %SP, 4;
 ; PTX32-NEXT:    add.u32 %r4, %SPL, 4;
-; PTX32-NEXT:    mov.b32 %r5, 0;
-; PTX32-NEXT:    st.local.b32 [%r2], %r5;
-; PTX32-NEXT:    st.local.b32 [%r4], %r5;
+; PTX32-NEXT:    st.local.b32 [%r2], 0;
+; PTX32-NEXT:    st.local.b32 [%r4], 0;
 ; PTX32-NEXT:    { // callseq 1, 0
 ; PTX32-NEXT:    .param .b32 param0;
 ; PTX32-NEXT:    st.param.b32 [param0], %r1;
-; PTX32-NEXT:    call.uni
-; PTX32-NEXT:    bar,
-; PTX32-NEXT:    (
-; PTX32-NEXT:    param0
-; PTX32-NEXT:    );
+; PTX32-NEXT:    call.uni bar, (param0);
 ; PTX32-NEXT:    } // callseq 1
 ; PTX32-NEXT:    { // callseq 2, 0
 ; PTX32-NEXT:    .param .b32 param0;
 ; PTX32-NEXT:    st.param.b32 [param0], %r3;
-; PTX32-NEXT:    call.uni
-; PTX32-NEXT:    bar,
-; PTX32-NEXT:    (
-; PTX32-NEXT:    param0
-; PTX32-NEXT:    );
+; PTX32-NEXT:    call.uni bar, (param0);
 ; PTX32-NEXT:    } // callseq 2
 ; PTX32-NEXT:    ret;
 ;
@@ -181,7 +164,6 @@ define void @foo4() {
 ; PTX64-NEXT:    .local .align 4 .b8 __local_depot3[8];
 ; PTX64-NEXT:    .reg .b64 %SP;
 ; PTX64-NEXT:    .reg .b64 %SPL;
-; PTX64-NEXT:    .reg .b32 %r<2>;
 ; PTX64-NEXT:    .reg .b64 %rd<5>;
 ; PTX64-EMPTY:
 ; PTX64-NEXT:  // %bb.0:
@@ -191,26 +173,17 @@ define void @foo4() {
 ; PTX64-NEXT:    add.u64 %rd2, %SPL, 0;
 ; PTX64-NEXT:    add.u64 %rd3, %SP, 4;
 ; PTX64-NEXT:    add.u64 %rd4, %SPL, 4;
-; PTX64-NEXT:    mov.b32 %r1, 0;
-; PTX64-NEXT:    st.local.b32 [%rd2], %r1;
-; PTX64-NEXT:    st.local.b32 [%rd4], %r1;
+; PTX64-NEXT:    st.local.b32 [%rd2], 0;
+; PTX64-NEXT:    st.local.b32 [%rd4], 0;
 ; PTX64-NEXT:    { // callseq 1, 0
 ; PTX64-NEXT:    .param .b64 param0;
 ; PTX64-NEXT:    st.param.b64 [param0], %rd1;
-; PTX64-NEXT:    call.uni
-; PTX64-NEXT:    bar,
-; PTX64-NEXT:    (
-; PTX64-NEXT:    param0
-; PTX64-NEXT:    );
+; PTX64-NEXT:    call.uni bar, (param0);
 ; PTX64-NEXT:    } // callseq 1
 ; PTX64-NEXT:    { // callseq 2, 0
 ; PTX64-NEXT:    .param .b64 param0;
 ; PTX64-NEXT:    st.param.b64 [param0], %rd3;
-; PTX64-NEXT:    call.uni
-; PTX64-NEXT:    bar,
-; PTX64-NEXT:    (
-; PTX64-NEXT:    param0
-; PTX64-NEXT:    );
+; PTX64-NEXT:    call.uni bar, (param0);
 ; PTX64-NEXT:    } // callseq 2
 ; PTX64-NEXT:    ret;
   %A = alloca i32
diff --git a/llvm/test/CodeGen/NVPTX/lower-alloca.ll b/llvm/test/CodeGen/NVPTX/lower-alloca.ll
index 489bcf4a7d55..57c1e5826c89 100644
--- a/llvm/test/CodeGen/NVPTX/lower-alloca.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-alloca.ll
@@ -15,7 +15,7 @@ define ptx_kernel void @kernel() {
 ; LOWERALLOCAONLY: [[V1:%.*]] = addrspacecast ptr %A to ptr addrspace(5)
 ; LOWERALLOCAONLY: [[V2:%.*]] = addrspacecast ptr addrspace(5) [[V1]] to ptr
 ; LOWERALLOCAONLY: store i32 0, ptr [[V2]], align 4
-; PTX: st.local.b32 [{{%rd[0-9]+}}], {{%r[0-9]+}}
+; PTX: st.local.b32 [{{%rd[0-9]+}}], 0
   store i32 0, ptr %A
   call void @callee(ptr %A)
   ret void
@@ -26,7 +26,7 @@ define void @alloca_in_explicit_local_as() {
 ; PTX-LABEL: .visible .func alloca_in_explicit_local_as(
   %A = alloca i32, addrspace(5)
 ; CHECK: store i32 0, ptr addrspace(5) {{%.+}}
-; PTX: st.local.b32 [%SP], {{%r[0-9]+}}
+; PTX: st.local.b32 [%SP], 0
 ; LOWERALLOCAONLY: [[V1:%.*]] = addrspacecast ptr addrspace(5) %A to ptr
 ; LOWERALLOCAONLY: store i32 0, ptr [[V1]], align 4
   store i32 0, ptr addrspace(5) %A
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
index c3f94455b303..321a6240df09 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
@@ -121,7 +121,7 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 %input, p
 define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-LABEL: grid_const_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<3>;
+; PTX-NEXT:    .reg .b32 %r<2>;
 ; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -133,12 +133,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-NEXT:    st.param.b64 [param0], %rd3;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_0 : .callprototype (.param .b32 _) _ (.param .b64 _);
-; PTX-NEXT:    call (retval0),
-; PTX-NEXT:    %rd1,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    )
-; PTX-NEXT:    , prototype_0;
+; PTX-NEXT:    call (retval0), %rd1, (param0), prototype_0;
 ; PTX-NEXT:    ld.param.b32 %r1, [retval0];
 ; PTX-NEXT:    } // callseq 0
 ; PTX-NEXT:    ret;
@@ -158,7 +153,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    .local .align 4 .b8 __local_depot4[4];
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
-; PTX-NEXT:    .reg .b32 %r<4>;
+; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<8>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -182,14 +177,7 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    st.param.b64 [param2], %rd4;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_1 : .callprototype (.param .b32 _) _ (.param .b64 _, .param .b64 _, .param .b64 _);
-; PTX-NEXT:    call (retval0),
-; PTX-NEXT:    %rd1,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0,
-; PTX-NEXT:    param1,
-; PTX-NEXT:    param2
-; PTX-NEXT:    )
-; PTX-NEXT:    , prototype_1;
+; PTX-NEXT:    call (retval0), %rd1, (param0, param1, param2), prototype_1;
 ; PTX-NEXT:    ld.param.b32 %r2, [retval0];
 ; PTX-NEXT:    } // callseq 1
 ; PTX-NEXT:    ret;
@@ -267,7 +255,7 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4
 define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %output) {
 ; PTX-LABEL: grid_const_partial_escape(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<5>;
+; PTX-NEXT:    .reg .b32 %r<4>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -284,12 +272,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 ; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_2 : .callprototype (.param .b32 _) _ (.param .b64 _);
-; PTX-NEXT:    call (retval0),
-; PTX-NEXT:    %rd1,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    )
-; PTX-NEXT:    , prototype_2;
+; PTX-NEXT:    call (retval0), %rd1, (param0), prototype_2;
 ; PTX-NEXT:    ld.param.b32 %r3, [retval0];
 ; PTX-NEXT:    } // callseq 2
 ; PTX-NEXT:    ret;
@@ -312,7 +295,7 @@ define ptx_kernel void @grid_const_partial_escape(ptr byval(i32) %input, ptr %ou
 define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input, ptr %output) {
 ; PTX-LABEL: grid_const_partial_escapemem(
 ; PTX:       {
-; PTX-NEXT:    .reg .b32 %r<6>;
+; PTX-NEXT:    .reg .b32 %r<5>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
@@ -330,12 +313,7 @@ define ptx_kernel i32 @grid_const_partial_escapemem(ptr byval(%struct.s) %input,
 ; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b32 retval0;
 ; PTX-NEXT:    prototype_3 : .callprototype (.param .b32 _) _ (.param .b64 _);
-; PTX-NEXT:    call (retval0),
-; PTX-NEXT:    %rd1,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    )
-; PTX-NEXT:    , prototype_3;
+; PTX-NEXT:    call (retval0), %rd1, (param0), prototype_3;
 ; PTX-NEXT:    ld.param.b32 %r4, [retval0];
 ; PTX-NEXT:    } // callseq 3
 ; PTX-NEXT:    st.param.b32 [func_retval0], %r3;
@@ -561,11 +539,7 @@ define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
 ; PTX-NEXT:    { // callseq 4, 0
 ; PTX-NEXT:    .param .align 4 .b8 param0[4];
 ; PTX-NEXT:    st.param.b32 [param0], %r1;
-; PTX-NEXT:    call.uni
-; PTX-NEXT:    device_func,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni device_func, (param0);
 ; PTX-NEXT:    } // callseq 4
 ; PTX-NEXT:    ret;
   call void @device_func(ptr byval(i32) align 4 %input)
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 246408ecf6a3..c165de7ffff0 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -S -nvptx-lower-args --mtriple nvptx64-nvidia-cuda | FileCheck %s --check-prefixes IR,IRC
 ; RUN: opt < %s -S -nvptx-lower-args --mtriple nvptx64-nvidia-nvcl | FileCheck %s --check-prefixes IR,IRO
 ; RUN: llc < %s -mcpu=sm_20 --mtriple nvptx64-nvidia-cuda | FileCheck %s --check-prefixes PTX,PTXC
@@ -31,7 +31,7 @@ define void @load_alignment(ptr nocapture readonly byval(%class.outer) align 8 %
 ; PTX-LABEL: load_alignment(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<4>;
-; PTX-NEXT:    .reg .b64 %rd<8>;
+; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
 ; PTX-NEXT:    mov.b64 %rd1, load_alignment_param_0;
@@ -47,11 +47,7 @@ define void @load_alignment(ptr nocapture readonly byval(%class.outer) align 8 %
 ; PTX-NEXT:    .param .b64 param0;
 ; PTX-NEXT:    st.param.b64 [param0], %rd5;
 ; PTX-NEXT:    .param .b64 retval0;
-; PTX-NEXT:    call.uni (retval0),
-; PTX-NEXT:    escape,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni (retval0), escape, (param0);
 ; PTX-NEXT:    ld.param.b64 %rd6, [retval0];
 ; PTX-NEXT:    } // callseq 0
 ; PTX-NEXT:    ret;
@@ -80,7 +76,7 @@ define void @load_padding(ptr nocapture readonly byval(%class.padded) %arg) {
 ;
 ; PTX-LABEL: load_padding(
 ; PTX:       {
-; PTX-NEXT:    .reg .b64 %rd<5>;
+; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %rd1, load_padding_param_0;
@@ -89,11 +85,7 @@ define void @load_padding(ptr nocapture readonly byval(%class.padded) %arg) {
 ; PTX-NEXT:    .param .b64 param0;
 ; PTX-NEXT:    st.param.b64 [param0], %rd2;
 ; PTX-NEXT:    .param .b64 retval0;
-; PTX-NEXT:    call.uni (retval0),
-; PTX-NEXT:    escape,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni (retval0), escape, (param0);
 ; PTX-NEXT:    ld.param.b64 %rd3, [retval0];
 ; PTX-NEXT:    } // callseq 1
 ; PTX-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 54495cf0d61f..4784d7093a79 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -153,11 +153,7 @@ define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out
 ; PTX-NEXT:    { // callseq 0, 0
 ; PTX-NEXT:    .param .b64 param0;
 ; PTX-NEXT:    st.param.b64 [param0], %rd1;
-; PTX-NEXT:    call.uni
-; PTX-NEXT:    _Z6escapePv,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni _Z6escapePv, (param0);
 ; PTX-NEXT:    } // callseq 0
 ; PTX-NEXT:    ret;
 entry:
@@ -198,11 +194,7 @@ define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone
 ; PTX-NEXT:    { // callseq 1, 0
 ; PTX-NEXT:    .param .b64 param0;
 ; PTX-NEXT:    st.param.b64 [param0], %rd3;
-; PTX-NEXT:    call.uni
-; PTX-NEXT:    _Z6escapePv,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni _Z6escapePv, (param0);
 ; PTX-NEXT:    } // callseq 1
 ; PTX-NEXT:    ret;
 entry:
@@ -658,7 +650,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b16 %rs<3>;
-; PTX-NEXT:    .reg .b32 %r<4>;
+; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<6>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
@@ -674,8 +666,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; PTX-NEXT:    add.u64 %rd2, %SPL, 4;
 ; PTX-NEXT:    add.u64 %rd4, %SPL, 0;
 ; PTX-NEXT:    selp.b64 %rd5, %rd2, %rd4, %p1;
-; PTX-NEXT:    mov.b32 %r3, 1;
-; PTX-NEXT:    st.local.b32 [%rd5], %r3;
+; PTX-NEXT:    st.local.b32 [%rd5], 1;
 ; PTX-NEXT:    ret;
 bb:
   %ptrnew = select i1 %cond, ptr %input1, ptr %input2
@@ -838,7 +829,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b16 %rs<3>;
-; PTX-NEXT:    .reg .b32 %r<4>;
+; PTX-NEXT:    .reg .b32 %r<3>;
 ; PTX-NEXT:    .reg .b64 %rd<7>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
@@ -857,8 +848,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:  // %bb.1: // %second
 ; PTX-NEXT:    mov.b64 %rd6, %rd1;
 ; PTX-NEXT:  $L__BB14_2: // %merge
-; PTX-NEXT:    mov.b32 %r3, 1;
-; PTX-NEXT:    st.local.b32 [%rd6], %r3;
+; PTX-NEXT:    st.local.b32 [%rd6], 1;
 ; PTX-NEXT:    ret;
 bb:
   br i1 %cond, label %first, label %second
@@ -902,11 +892,7 @@ define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
 ; PTX-NEXT:    { // callseq 2, 0
 ; PTX-NEXT:    .param .align 4 .b8 param0[4];
 ; PTX-NEXT:    st.param.b32 [param0], %r1;
-; PTX-NEXT:    call.uni
-; PTX-NEXT:    device_func,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni device_func, (param0);
 ; PTX-NEXT:    } // callseq 2
 ; PTX-NEXT:    ret;
   call void @device_func(ptr byval(i32) align 4 %input)
@@ -929,11 +915,7 @@ define void @device_func(ptr byval(i32) align 4 %input) {
 ; PTX-NEXT:    { // callseq 3, 0
 ; PTX-NEXT:    .param .align 4 .b8 param0[4];
 ; PTX-NEXT:    st.param.b32 [param0], %r1;
-; PTX-NEXT:    call.uni
-; PTX-NEXT:    device_func,
-; PTX-NEXT:    (
-; PTX-NEXT:    param0
-; PTX-NEXT:    );
+; PTX-NEXT:    call.uni device_func, (param0);
 ; PTX-NEXT:    } // callseq 3
 ; PTX-NEXT:    ret;
   call void @device_func(ptr byval(i32) align 4 %input)
diff --git a/llvm/test/CodeGen/NVPTX/math-intrins.ll b/llvm/test/CodeGen/NVPTX/math-intrins.ll
index 71af7a7d475d..dde71b009d56 100644
--- a/llvm/test/CodeGen/NVPTX/math-intrins.ll
+++ b/llvm/test/CodeGen/NVPTX/math-intrins.ll
@@ -552,23 +552,21 @@ define <2 x half> @minnum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-NOF16-LABEL: minnum_v2half(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<7>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<8>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [minnum_v2half_param_0];
-; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [minnum_v2half_param_1];
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT:    min.f32 %r5, %r4, %r3;
-; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT:    min.f32 %r8, %r7, %r6;
-; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-NOF16-NEXT:    mov.b32 %r9, {%rs6, %rs5};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [minnum_v2half_param_0];
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [minnum_v2half_param_1];
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT:    min.f32 %r3, %r2, %r1;
+; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT:    min.f32 %r6, %r5, %r4;
+; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-NOF16-NEXT:    mov.b32 %r7, {%rs6, %rs5};
+; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NOF16-NEXT:    ret;
 ;
 ; CHECK-F16-LABEL: minnum_v2half(
@@ -576,32 +574,30 @@ define <2 x half> @minnum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-F16-NEXT:    .reg .b32 %r<4>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b32 %r1, [minnum_v2half_param_1];
-; CHECK-F16-NEXT:    ld.param.b32 %r2, [minnum_v2half_param_0];
-; CHECK-F16-NEXT:    min.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT:    ld.param.b32 %r1, [minnum_v2half_param_0];
+; CHECK-F16-NEXT:    ld.param.b32 %r2, [minnum_v2half_param_1];
+; CHECK-F16-NEXT:    min.f16x2 %r3, %r1, %r2;
 ; CHECK-F16-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-F16-NEXT:    ret;
 ;
 ; CHECK-SM80-NOF16-LABEL: minnum_v2half(
 ; CHECK-SM80-NOF16:       {
 ; CHECK-SM80-NOF16-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<8>;
 ; CHECK-SM80-NOF16-EMPTY:
 ; CHECK-SM80-NOF16-NEXT:  // %bb.0:
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r1, [minnum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r2, [minnum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT:    min.f32 %r5, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT:    min.f32 %r8, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 %r9, {%rs6, %rs5};
-; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [minnum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [minnum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT:    min.f32 %r3, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT:    min.f32 %r6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-SM80-NOF16-NEXT:    mov.b32 %r7, {%rs6, %rs5};
+; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-SM80-NOF16-NEXT:    ret;
   %x = call <2 x half> @llvm.minnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %x
@@ -877,41 +873,38 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .pred %p<11>;
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<15>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<7>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [minimum_v2half_param_0];
-; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [minimum_v2half_param_1];
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT:    setp.lt.f32 %p1, %r4, %r3;
-; CHECK-NOF16-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-NOF16-NEXT:    setp.nan.f32 %p2, %r4, %r3;
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [minimum_v2half_param_0];
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [minimum_v2half_param_1];
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT:    setp.lt.f32 %p1, %r2, %r1;
+; CHECK-NOF16-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-NOF16-NEXT:    setp.nan.f32 %p2, %r2, %r1;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p3, %rs4, -32768;
-; CHECK-NOF16-NEXT:    selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p4, %rs2, -32768;
-; CHECK-NOF16-NEXT:    selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs6;
-; CHECK-NOF16-NEXT:    setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p3, %rs2, -32768;
+; CHECK-NOF16-NEXT:    selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p4, %rs4, -32768;
+; CHECK-NOF16-NEXT:    selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-NOF16-NEXT:    setp.eq.f32 %p5, %r3, 0f00000000;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT:    setp.lt.f32 %p6, %r7, %r6;
-; CHECK-NOF16-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-NOF16-NEXT:    setp.nan.f32 %p7, %r7, %r6;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT:    setp.lt.f32 %p6, %r5, %r4;
+; CHECK-NOF16-NEXT:    selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-NOF16-NEXT:    setp.nan.f32 %p7, %r5, %r4;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p8, %rs3, -32768;
-; CHECK-NOF16-NEXT:    selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p9, %rs1, -32768;
-; CHECK-NOF16-NEXT:    selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r8, %rs11;
-; CHECK-NOF16-NEXT:    setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p8, %rs1, -32768;
+; CHECK-NOF16-NEXT:    selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p9, %rs3, -32768;
+; CHECK-NOF16-NEXT:    selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs11;
+; CHECK-NOF16-NEXT:    setp.eq.f32 %p10, %r6, 0f00000000;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-NOF16-NEXT:    mov.b32 %r9, {%rs14, %rs9};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs14, %rs9};
 ; CHECK-NOF16-NEXT:    ret;
 ;
 ; CHECK-F16-LABEL: minimum_v2half(
@@ -919,9 +912,9 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-F16-NEXT:    .reg .b32 %r<4>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b32 %r1, [minimum_v2half_param_1];
-; CHECK-F16-NEXT:    ld.param.b32 %r2, [minimum_v2half_param_0];
-; CHECK-F16-NEXT:    min.NaN.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT:    ld.param.b32 %r1, [minimum_v2half_param_0];
+; CHECK-F16-NEXT:    ld.param.b32 %r2, [minimum_v2half_param_1];
+; CHECK-F16-NEXT:    min.NaN.f16x2 %r3, %r1, %r2;
 ; CHECK-F16-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-F16-NEXT:    ret;
 ;
@@ -929,41 +922,38 @@ define <2 x half> @minimum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-SM80-NOF16:       {
 ; CHECK-SM80-NOF16-NEXT:    .reg .pred %p<11>;
 ; CHECK-SM80-NOF16-NEXT:    .reg .b16 %rs<15>;
-; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<7>;
 ; CHECK-SM80-NOF16-EMPTY:
 ; CHECK-SM80-NOF16-NEXT:  // %bb.0:
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r1, [minimum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r2, [minimum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT:    setp.lt.f32 %p1, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p2, %r4, %r3;
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [minimum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [minimum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT:    setp.lt.f32 %p1, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p2, %r2, %r1;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p3, %rs4, -32768;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p4, %rs2, -32768;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs6;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p3, %rs2, -32768;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p4, %rs4, -32768;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p5, %r3, 0f00000000;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT:    setp.lt.f32 %p6, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p7, %r7, %r6;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT:    setp.lt.f32 %p6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p7, %r5, %r4;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p8, %rs3, -32768;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p9, %rs1, -32768;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r8, %rs11;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p8, %rs1, -32768;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p9, %rs3, -32768;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs11;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p10, %r6, 0f00000000;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 %r9, {%rs14, %rs9};
-; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs14, %rs9};
 ; CHECK-SM80-NOF16-NEXT:    ret;
   %x = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %x
@@ -1093,23 +1083,21 @@ define <2 x half> @maxnum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-NOF16-LABEL: maxnum_v2half(
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<7>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<8>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [maxnum_v2half_param_0];
-; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [maxnum_v2half_param_1];
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT:    max.f32 %r5, %r4, %r3;
-; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT:    max.f32 %r8, %r7, %r6;
-; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-NOF16-NEXT:    mov.b32 %r9, {%rs6, %rs5};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [maxnum_v2half_param_0];
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [maxnum_v2half_param_1];
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT:    max.f32 %r3, %r2, %r1;
+; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT:    max.f32 %r6, %r5, %r4;
+; CHECK-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-NOF16-NEXT:    mov.b32 %r7, {%rs6, %rs5};
+; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NOF16-NEXT:    ret;
 ;
 ; CHECK-F16-LABEL: maxnum_v2half(
@@ -1117,32 +1105,30 @@ define <2 x half> @maxnum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-F16-NEXT:    .reg .b32 %r<4>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b32 %r1, [maxnum_v2half_param_1];
-; CHECK-F16-NEXT:    ld.param.b32 %r2, [maxnum_v2half_param_0];
-; CHECK-F16-NEXT:    max.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT:    ld.param.b32 %r1, [maxnum_v2half_param_0];
+; CHECK-F16-NEXT:    ld.param.b32 %r2, [maxnum_v2half_param_1];
+; CHECK-F16-NEXT:    max.f16x2 %r3, %r1, %r2;
 ; CHECK-F16-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-F16-NEXT:    ret;
 ;
 ; CHECK-SM80-NOF16-LABEL: maxnum_v2half(
 ; CHECK-SM80-NOF16:       {
 ; CHECK-SM80-NOF16-NEXT:    .reg .b16 %rs<7>;
-; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<8>;
 ; CHECK-SM80-NOF16-EMPTY:
 ; CHECK-SM80-NOF16-NEXT:  // %bb.0:
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r1, [maxnum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r2, [maxnum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT:    max.f32 %r5, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r5;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT:    max.f32 %r8, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r8;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 %r9, {%rs6, %rs5};
-; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [maxnum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [maxnum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT:    max.f32 %r3, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs5, %r3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT:    max.f32 %r6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT:    cvt.rn.f16.f32 %rs6, %r6;
+; CHECK-SM80-NOF16-NEXT:    mov.b32 %r7, {%rs6, %rs5};
+; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-SM80-NOF16-NEXT:    ret;
   %x = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %x
@@ -1414,41 +1400,38 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .pred %p<11>;
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<15>;
-; CHECK-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-NOF16-NEXT:    .reg .b32 %r<7>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [maximum_v2half_param_0];
-; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [maximum_v2half_param_1];
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-NOF16-NEXT:    setp.gt.f32 %p1, %r4, %r3;
-; CHECK-NOF16-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-NOF16-NEXT:    setp.nan.f32 %p2, %r4, %r3;
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [maximum_v2half_param_0];
+; CHECK-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [maximum_v2half_param_1];
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOF16-NEXT:    setp.gt.f32 %p1, %r2, %r1;
+; CHECK-NOF16-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-NOF16-NEXT:    setp.nan.f32 %p2, %r2, %r1;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p3, %rs4, 0;
-; CHECK-NOF16-NEXT:    selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p4, %rs2, 0;
-; CHECK-NOF16-NEXT:    selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs6;
-; CHECK-NOF16-NEXT:    setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p3, %rs2, 0;
+; CHECK-NOF16-NEXT:    selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p4, %rs4, 0;
+; CHECK-NOF16-NEXT:    selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-NOF16-NEXT:    setp.eq.f32 %p5, %r3, 0f00000000;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-NOF16-NEXT:    setp.gt.f32 %p6, %r7, %r6;
-; CHECK-NOF16-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-NOF16-NEXT:    setp.nan.f32 %p7, %r7, %r6;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-NOF16-NEXT:    setp.gt.f32 %p6, %r5, %r4;
+; CHECK-NOF16-NEXT:    selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-NOF16-NEXT:    setp.nan.f32 %p7, %r5, %r4;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p8, %rs3, 0;
-; CHECK-NOF16-NEXT:    selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-NOF16-NEXT:    setp.eq.s16 %p9, %rs1, 0;
-; CHECK-NOF16-NEXT:    selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r8, %rs11;
-; CHECK-NOF16-NEXT:    setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p8, %rs1, 0;
+; CHECK-NOF16-NEXT:    selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-NOF16-NEXT:    setp.eq.s16 %p9, %rs3, 0;
+; CHECK-NOF16-NEXT:    selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs11;
+; CHECK-NOF16-NEXT:    setp.eq.f32 %p10, %r6, 0f00000000;
 ; CHECK-NOF16-NEXT:    selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-NOF16-NEXT:    mov.b32 %r9, {%rs14, %rs9};
-; CHECK-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs14, %rs9};
 ; CHECK-NOF16-NEXT:    ret;
 ;
 ; CHECK-F16-LABEL: maximum_v2half(
@@ -1456,9 +1439,9 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-F16-NEXT:    .reg .b32 %r<4>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b32 %r1, [maximum_v2half_param_1];
-; CHECK-F16-NEXT:    ld.param.b32 %r2, [maximum_v2half_param_0];
-; CHECK-F16-NEXT:    max.NaN.f16x2 %r3, %r2, %r1;
+; CHECK-F16-NEXT:    ld.param.b32 %r1, [maximum_v2half_param_0];
+; CHECK-F16-NEXT:    ld.param.b32 %r2, [maximum_v2half_param_1];
+; CHECK-F16-NEXT:    max.NaN.f16x2 %r3, %r1, %r2;
 ; CHECK-F16-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-F16-NEXT:    ret;
 ;
@@ -1466,41 +1449,38 @@ define <2 x half> @maximum_v2half(<2 x half> %a, <2 x half> %b) {
 ; CHECK-SM80-NOF16:       {
 ; CHECK-SM80-NOF16-NEXT:    .reg .pred %p<11>;
 ; CHECK-SM80-NOF16-NEXT:    .reg .b16 %rs<15>;
-; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<10>;
+; CHECK-SM80-NOF16-NEXT:    .reg .b32 %r<7>;
 ; CHECK-SM80-NOF16-EMPTY:
 ; CHECK-SM80-NOF16-NEXT:  // %bb.0:
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r1, [maximum_v2half_param_0];
-; CHECK-SM80-NOF16-NEXT:    ld.param.b32 %r2, [maximum_v2half_param_1];
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs2;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs4;
-; CHECK-SM80-NOF16-NEXT:    setp.gt.f32 %p1, %r4, %r3;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p1;
-; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p2, %r4, %r3;
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [maximum_v2half_param_0];
+; CHECK-SM80-NOF16-NEXT:    ld.param.v2.b16 {%rs3, %rs4}, [maximum_v2half_param_1];
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r1, %rs4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-SM80-NOF16-NEXT:    setp.gt.f32 %p1, %r2, %r1;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs5, %rs2, %rs4, %p1;
+; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p2, %r2, %r1;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs6, 0x7E00, %rs5, %p2;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p3, %rs4, 0;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs7, %rs4, %rs6, %p3;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p4, %rs2, 0;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs8, %rs2, %rs7, %p4;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs6;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p5, %r5, 0f00000000;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p3, %rs2, 0;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs7, %rs2, %rs6, %p3;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p4, %rs4, 0;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs8, %rs4, %rs7, %p4;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p5, %r3, 0f00000000;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs9, %rs8, %rs6, %p5;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r7, %rs3;
-; CHECK-SM80-NOF16-NEXT:    setp.gt.f32 %p6, %r7, %r6;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs10, %rs3, %rs1, %p6;
-; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p7, %r7, %r6;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r5, %rs1;
+; CHECK-SM80-NOF16-NEXT:    setp.gt.f32 %p6, %r5, %r4;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs10, %rs1, %rs3, %p6;
+; CHECK-SM80-NOF16-NEXT:    setp.nan.f32 %p7, %r5, %r4;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs11, 0x7E00, %rs10, %p7;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p8, %rs3, 0;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs12, %rs3, %rs11, %p8;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p9, %rs1, 0;
-; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs13, %rs1, %rs12, %p9;
-; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r8, %rs11;
-; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p10, %r8, 0f00000000;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p8, %rs1, 0;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs12, %rs1, %rs11, %p8;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.s16 %p9, %rs3, 0;
+; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs13, %rs3, %rs12, %p9;
+; CHECK-SM80-NOF16-NEXT:    cvt.f32.f16 %r6, %rs11;
+; CHECK-SM80-NOF16-NEXT:    setp.eq.f32 %p10, %r6, 0f00000000;
 ; CHECK-SM80-NOF16-NEXT:    selp.b16 %rs14, %rs13, %rs11, %p10;
-; CHECK-SM80-NOF16-NEXT:    mov.b32 %r9, {%rs14, %rs9};
-; CHECK-SM80-NOF16-NEXT:    st.param.b32 [func_retval0], %r9;
+; CHECK-SM80-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs14, %rs9};
 ; CHECK-SM80-NOF16-NEXT:    ret;
   %x = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %b)
   ret <2 x half> %x
diff --git a/llvm/test/CodeGen/NVPTX/misched_func_call.ll b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
index 7e907990147a..8401f457418d 100644
--- a/llvm/test/CodeGen/NVPTX/misched_func_call.ll
+++ b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
@@ -8,7 +8,7 @@ define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-LABEL: wombat(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<11>;
-; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %bb
 ; CHECK-NEXT:    ld.param.b32 %r4, [wombat_param_2];
@@ -21,21 +21,17 @@ define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], 0d0000000000000000;
 ; CHECK-NEXT:    .param .b64 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    quux,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), quux, (param0);
 ; CHECK-NEXT:    ld.param.b64 %rd1, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    mul.lo.s32 %r7, %r10, %r3;
 ; CHECK-NEXT:    or.b32 %r8, %r4, %r7;
 ; CHECK-NEXT:    mul.lo.s32 %r9, %r2, %r8;
-; CHECK-NEXT:    cvt.rn.f64.s32 %rd3, %r9;
-; CHECK-NEXT:    cvt.rn.f64.u32 %rd4, %r10;
-; CHECK-NEXT:    add.rn.f64 %rd5, %rd4, %rd3;
-; CHECK-NEXT:    mov.b64 %rd6, 0;
-; CHECK-NEXT:    st.global.b64 [%rd6], %rd5;
+; CHECK-NEXT:    cvt.rn.f64.s32 %rd2, %r9;
+; CHECK-NEXT:    cvt.rn.f64.u32 %rd3, %r10;
+; CHECK-NEXT:    add.rn.f64 %rd4, %rd3, %rd2;
+; CHECK-NEXT:    mov.b64 %rd5, 0;
+; CHECK-NEXT:    st.global.b64 [%rd5], %rd4;
 ; CHECK-NEXT:    mov.b32 %r10, 1;
 ; CHECK-NEXT:    bra.uni $L__BB0_1;
 bb:
diff --git a/llvm/test/CodeGen/NVPTX/mulwide.ll b/llvm/test/CodeGen/NVPTX/mulwide.ll
index 7db7dec56199..666c7a160e1f 100644
--- a/llvm/test/CodeGen/NVPTX/mulwide.ll
+++ b/llvm/test/CodeGen/NVPTX/mulwide.ll
@@ -1,111 +1,325 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O3 | FileCheck %s --check-prefix=OPT
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | FileCheck %s --check-prefix=NOOPT
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O3 | %ptxas-verify %}
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 -O0 | %ptxas-verify %}
 
-; OPT-LABEL: @mulwide16
-; NOOPT-LABEL: @mulwide16
 define i32 @mulwide16(i16 %a, i16 %b) {
-; OPT: mul.wide.s16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwide16(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b16 %rs1, [mulwide16_param_0];
+; OPT-NEXT:    ld.param.b16 %rs2, [mulwide16_param_1];
+; OPT-NEXT:    mul.wide.s16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide16(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b16 %rs2, [mulwide16_param_1];
+; NOOPT-NEXT:    ld.param.b16 %rs1, [mulwide16_param_0];
+; NOOPT-NEXT:    cvt.s32.s16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.s32.s16 %r2, %rs2;
+; NOOPT-NEXT:    mul.lo.s32 %r3, %r1, %r2;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r3;
+; NOOPT-NEXT:    ret;
   %val0 = sext i16 %a to i32
   %val1 = sext i16 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwideu16
-; NOOPT-LABEL: @mulwideu16
 define i32 @mulwideu16(i16 %a, i16 %b) {
-; OPT: mul.wide.u16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwideu16(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b16 %rs1, [mulwideu16_param_0];
+; OPT-NEXT:    ld.param.b16 %rs2, [mulwideu16_param_1];
+; OPT-NEXT:    mul.wide.u16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu16(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b16 %rs2, [mulwideu16_param_1];
+; NOOPT-NEXT:    ld.param.b16 %rs1, [mulwideu16_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.u32.u16 %r2, %rs2;
+; NOOPT-NEXT:    mul.lo.s32 %r3, %r1, %r2;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r3;
+; NOOPT-NEXT:    ret;
   %val0 = zext i16 %a to i32
   %val1 = zext i16 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwide8
-; NOOPT-LABEL: @mulwide8
 define i32 @mulwide8(i8 %a, i8 %b) {
-; OPT: mul.wide.s16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwide8(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.s8 %rs1, [mulwide8_param_0];
+; OPT-NEXT:    ld.param.s8 %rs2, [mulwide8_param_1];
+; OPT-NEXT:    mul.wide.s16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide8(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwide8_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwide8_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    cvt.s32.s8 %r2, %r1;
+; NOOPT-NEXT:    cvt.u32.u16 %r3, %rs2;
+; NOOPT-NEXT:    cvt.s32.s8 %r4, %r3;
+; NOOPT-NEXT:    mul.lo.s32 %r5, %r2, %r4;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r5;
+; NOOPT-NEXT:    ret;
   %val0 = sext i8 %a to i32
   %val1 = sext i8 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwideu8
-; NOOPT-LABEL: @mulwideu8
 define i32 @mulwideu8(i8 %a, i8 %b) {
-; OPT: mul.wide.u16
-; NOOPT: mul.lo.s32
+; OPT-LABEL: mulwideu8(
+; OPT:       {
+; OPT-NEXT:    .reg .b16 %rs<3>;
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %rs1, [mulwideu8_param_0];
+; OPT-NEXT:    ld.param.b8 %rs2, [mulwideu8_param_1];
+; OPT-NEXT:    mul.wide.u16 %r1, %rs1, %rs2;
+; OPT-NEXT:    st.param.b32 [func_retval0], %r1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu8(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b32 %r<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwideu8_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwideu8_param_0];
+; NOOPT-NEXT:    cvt.u32.u16 %r1, %rs1;
+; NOOPT-NEXT:    and.b32 %r2, %r1, 255;
+; NOOPT-NEXT:    cvt.u32.u16 %r3, %rs2;
+; NOOPT-NEXT:    and.b32 %r4, %r3, 255;
+; NOOPT-NEXT:    mul.lo.s32 %r5, %r2, %r4;
+; NOOPT-NEXT:    st.param.b32 [func_retval0], %r5;
+; NOOPT-NEXT:    ret;
   %val0 = zext i8 %a to i32
   %val1 = zext i8 %b to i32
   %val2 = mul i32 %val0, %val1
   ret i32 %val2
 }
 
-; OPT-LABEL: @mulwide32
-; NOOPT-LABEL: @mulwide32
 define i64 @mulwide32(i32 %a, i32 %b) {
-; OPT: mul.wide.s32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwide32(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [mulwide32_param_0];
+; OPT-NEXT:    ld.param.b32 %r2, [mulwide32_param_1];
+; OPT-NEXT:    mul.wide.s32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwide32(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r2, [mulwide32_param_1];
+; NOOPT-NEXT:    ld.param.b32 %r1, [mulwide32_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    cvt.s64.s32 %rd2, %r2;
+; NOOPT-NEXT:    mul.lo.s64 %rd3, %rd1, %rd2;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd3;
+; NOOPT-NEXT:    ret;
   %val0 = sext i32 %a to i64
   %val1 = sext i32 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwideu32
-; NOOPT-LABEL: @mulwideu32
 define i64 @mulwideu32(i32 %a, i32 %b) {
-; OPT: mul.wide.u32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwideu32(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [mulwideu32_param_0];
+; OPT-NEXT:    ld.param.b32 %r2, [mulwideu32_param_1];
+; OPT-NEXT:    mul.wide.u32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu32(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<4>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r2, [mulwideu32_param_1];
+; NOOPT-NEXT:    ld.param.b32 %r1, [mulwideu32_param_0];
+; NOOPT-NEXT:    cvt.u64.u32 %rd1, %r1;
+; NOOPT-NEXT:    cvt.u64.u32 %rd2, %r2;
+; NOOPT-NEXT:    mul.lo.s64 %rd3, %rd1, %rd2;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd3;
+; NOOPT-NEXT:    ret;
   %val0 = zext i32 %a to i64
   %val1 = zext i32 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwideu7
-; NOOPT-LABEL: @mulwideu7
 define i64 @mulwideu7(i7 %a, i7 %b) {
-; OPT: mul.wide.u32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwideu7(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %r1, [mulwideu7_param_0];
+; OPT-NEXT:    ld.param.b8 %r2, [mulwideu7_param_1];
+; OPT-NEXT:    mul.wide.u32 %rd1, %r1, %r2;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwideu7(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwideu7_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwideu7_param_0];
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
+; NOOPT-NEXT:    and.b64 %rd2, %rd1, 127;
+; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
+; NOOPT-NEXT:    and.b64 %rd4, %rd3, 127;
+; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
+; NOOPT-NEXT:    ret;
   %val0 = zext i7 %a to i64
   %val1 = zext i7 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @mulwides7
-; NOOPT-LABEL: @mulwides7
 define i64 @mulwides7(i7 %a, i7 %b) {
-; OPT: mul.wide.s32
-; NOOPT: mul.lo.s64
+; OPT-LABEL: mulwides7(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<5>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b8 %r1, [mulwides7_param_0];
+; OPT-NEXT:    bfe.s32 %r2, %r1, 0, 7;
+; OPT-NEXT:    ld.param.b8 %r3, [mulwides7_param_1];
+; OPT-NEXT:    bfe.s32 %r4, %r3, 0, 7;
+; OPT-NEXT:    mul.wide.s32 %rd1, %r2, %r4;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: mulwides7(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b64 %rd<6>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwides7_param_1];
+; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwides7_param_0];
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
+; NOOPT-NEXT:    bfe.s64 %rd2, %rd1, 0, 7;
+; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
+; NOOPT-NEXT:    bfe.s64 %rd4, %rd3, 0, 7;
+; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
+; NOOPT-NEXT:    ret;
   %val0 = sext i7 %a to i64
   %val1 = sext i7 %b to i64
   %val2 = mul i64 %val0, %val1
   ret i64 %val2
 }
 
-; OPT-LABEL: @shl30
-; NOOPT-LABEL: @shl30
 define i64 @shl30(i32 %a) {
-; OPT: mul.wide
-; NOOPT: shl.b64
+; OPT-LABEL: shl30(
+; OPT:       {
+; OPT-NEXT:    .reg .b32 %r<2>;
+; OPT-NEXT:    .reg .b64 %rd<2>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.b32 %r1, [shl30_param_0];
+; OPT-NEXT:    mul.wide.s32 %rd1, %r1, 1073741824;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: shl30(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<2>;
+; NOOPT-NEXT:    .reg .b64 %rd<3>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r1, [shl30_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    shl.b64 %rd2, %rd1, 30;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; NOOPT-NEXT:    ret;
   %conv = sext i32 %a to i64
   %shl = shl i64 %conv, 30
   ret i64 %shl
 }
 
-; OPT-LABEL: @shl31
-; NOOPT-LABEL: @shl31
 define i64 @shl31(i32 %a) {
-; OPT-NOT: mul.wide
-; NOOPT-NOT: mul.wide
+; OPT-LABEL: shl31(
+; OPT:       {
+; OPT-NEXT:    .reg .b64 %rd<3>;
+; OPT-EMPTY:
+; OPT-NEXT:  // %bb.0:
+; OPT-NEXT:    ld.param.s32 %rd1, [shl31_param_0];
+; OPT-NEXT:    shl.b64 %rd2, %rd1, 31;
+; OPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; OPT-NEXT:    ret;
+;
+; NOOPT-LABEL: shl31(
+; NOOPT:       {
+; NOOPT-NEXT:    .reg .b32 %r<2>;
+; NOOPT-NEXT:    .reg .b64 %rd<3>;
+; NOOPT-EMPTY:
+; NOOPT-NEXT:  // %bb.0:
+; NOOPT-NEXT:    ld.param.b32 %r1, [shl31_param_0];
+; NOOPT-NEXT:    cvt.s64.s32 %rd1, %r1;
+; NOOPT-NEXT:    shl.b64 %rd2, %rd1, 31;
+; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd2;
+; NOOPT-NEXT:    ret;
   %conv = sext i32 %a to i64
   %shl = shl i64 %conv, 31
   ret i64 %shl
diff --git a/llvm/test/CodeGen/NVPTX/naked-fn-with-frame-pointer.ll b/llvm/test/CodeGen/NVPTX/naked-fn-with-frame-pointer.ll
index a1f0577c2218..448960181ae4 100644
--- a/llvm/test/CodeGen/NVPTX/naked-fn-with-frame-pointer.ll
+++ b/llvm/test/CodeGen/NVPTX/naked-fn-with-frame-pointer.ll
@@ -11,10 +11,7 @@ define dso_local void @naked() naked "frame-pointer"="all" {
 ; CHECK-32-EMPTY:
 ; CHECK-32-NEXT:  // %bb.0:
 ; CHECK-32-NEXT:    { // callseq 0, 0
-; CHECK-32-NEXT:    call.uni
-; CHECK-32-NEXT:    main,
-; CHECK-32-NEXT:    (
-; CHECK-32-NEXT:    );
+; CHECK-32-NEXT:    call.uni main, ();
 ; CHECK-32-NEXT:    } // callseq 0
 ; CHECK-32-NEXT:    // begin inline asm
 ; CHECK-32-NEXT:    exit;
@@ -26,10 +23,7 @@ define dso_local void @naked() naked "frame-pointer"="all" {
 ; CHECK-64-EMPTY:
 ; CHECK-64-NEXT:  // %bb.0:
 ; CHECK-64-NEXT:    { // callseq 0, 0
-; CHECK-64-NEXT:    call.uni
-; CHECK-64-NEXT:    main,
-; CHECK-64-NEXT:    (
-; CHECK-64-NEXT:    );
+; CHECK-64-NEXT:    call.uni main, ();
 ; CHECK-64-NEXT:    } // callseq 0
 ; CHECK-64-NEXT:    // begin inline asm
 ; CHECK-64-NEXT:    exit;
@@ -45,10 +39,7 @@ define dso_local void @normal() "frame-pointer"="all" {
 ; CHECK-32-EMPTY:
 ; CHECK-32-NEXT:  // %bb.0:
 ; CHECK-32-NEXT:    { // callseq 1, 0
-; CHECK-32-NEXT:    call.uni
-; CHECK-32-NEXT:    main,
-; CHECK-32-NEXT:    (
-; CHECK-32-NEXT:    );
+; CHECK-32-NEXT:    call.uni main, ();
 ; CHECK-32-NEXT:    } // callseq 1
 ; CHECK-32-NEXT:    // begin inline asm
 ; CHECK-32-NEXT:    exit;
@@ -60,10 +51,7 @@ define dso_local void @normal() "frame-pointer"="all" {
 ; CHECK-64-EMPTY:
 ; CHECK-64-NEXT:  // %bb.0:
 ; CHECK-64-NEXT:    { // callseq 1, 0
-; CHECK-64-NEXT:    call.uni
-; CHECK-64-NEXT:    main,
-; CHECK-64-NEXT:    (
-; CHECK-64-NEXT:    );
+; CHECK-64-NEXT:    call.uni main, ();
 ; CHECK-64-NEXT:    } // callseq 1
 ; CHECK-64-NEXT:    // begin inline asm
 ; CHECK-64-NEXT:    exit;
diff --git a/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch-O0.ll b/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch-O0.ll
index ff04e18701a8..a7f3103e5fcb 100644
--- a/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch-O0.ll
+++ b/llvm/test/CodeGen/NVPTX/nvvm-reflect-arch-O0.ll
@@ -1,26 +1,18 @@
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_52 -mattr=+ptx64 -O0 | FileCheck %s --check-prefix=SM_52
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx64 -O0 | FileCheck %s --check-prefix=SM_70
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx72 -O0 | FileCheck %s --check-prefix=SM_90
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_52 -mattr=+ptx64 -O0 | FileCheck %s --check-prefixes=SM_52,COMMON
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 -mattr=+ptx64 -O0 | FileCheck %s --check-prefixes=SM_70,COMMON
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx72 -O0 | FileCheck %s --check-prefixes=SM_90,COMMON
 
 @.str = private unnamed_addr constant [12 x i8] c"__CUDA_ARCH\00"
 @.str1 = constant [11 x i8] c"__CUDA_FTZ\00"
 
 declare i32 @__nvvm_reflect(ptr)
 
-;      SM_52: .visible .func  (.param .b32 func_retval0) foo()
-;      SM_52: mov.b32         %[[REG:.+]], 3;
-; SM_52-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_52-NEXT: ret;
-;
-;      SM_70: .visible .func  (.param .b32 func_retval0) foo()
-;      SM_70: mov.b32         %[[REG:.+]], 2;
-; SM_70-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_70-NEXT: ret;
-;
-;      SM_90: .visible .func  (.param .b32 func_retval0) foo()
-;      SM_90: mov.b32         %[[REG:.+]], 1;
-; SM_90-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_90-NEXT: ret;
+; COMMON-LABEL: .visible .func  (.param .b32 func_retval0) foo()
+; SM_52: st.param.b32    [func_retval0], 3;
+; SM_70: st.param.b32    [func_retval0], 2;
+; SM_90: st.param.b32    [func_retval0], 1;
+; COMMON-NEXT: ret;
+
 define i32 @foo() {
 entry:
   %call = call i32 @__nvvm_reflect(ptr @.str)
@@ -54,20 +46,11 @@ return:
   ret i32 %retval.0
 }
 
-;      SM_52: .visible .func  (.param .b32 func_retval0) bar()
-;      SM_52: mov.b32         %[[REG:.+]], 2;
-; SM_52-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_52-NEXT: ret;
-;
-;      SM_70: .visible .func  (.param .b32 func_retval0) bar()
-;      SM_70: mov.b32         %[[REG:.+]], 1;
-; SM_70-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_70-NEXT: ret;
-;
-;      SM_90: .visible .func  (.param .b32 func_retval0) bar()
-;      SM_90: mov.b32         %[[REG:.+]], 1;
-; SM_90-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_90-NEXT: ret;
+; COMMON-LABEL: .visible .func  (.param .b32 func_retval0) bar()
+; SM_52: st.param.b32    [func_retval0], 2;
+; SM_70: st.param.b32    [func_retval0], 1;
+; SM_90: st.param.b32    [func_retval0], 1;
+; COMMON-NEXT: ret;
 define i32 @bar() {
 entry:
   %call = call i32 @__nvvm_reflect(ptr @.str)
@@ -102,20 +85,11 @@ if.end:
   ret void
 }
 
-;      SM_52: .visible .func  (.param .b32 func_retval0) qux()
-;      SM_52: mov.b32         %[[REG:.+]], 3;
-; SM_52-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_52-NEXT: ret;
-;
-;      SM_70: .visible .func  (.param .b32 func_retval0) qux()
-;      SM_70: mov.b32         %[[REG:.+]], 2;
-; SM_70-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_70-NEXT: ret;
-;
-;      SM_90: .visible .func  (.param .b32 func_retval0) qux()
-;      SM_90: mov.b32         %[[REG:.+]], 1;
-; SM_90-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_90-NEXT: ret;
+; SM_52: .visible .func  (.param .b32 func_retval0) qux()
+; SM_52: st.param.b32    [func_retval0], 3;
+; SM_70: st.param.b32    [func_retval0], 2;
+; SM_90: st.param.b32    [func_retval0], 1;
+; COMMON-NEXT: ret;
 define i32 @qux() {
 entry:
   %call = call i32 @__nvvm_reflect(ptr noundef @.str)
@@ -142,18 +116,9 @@ return:
   ret i32 %retval
 }
 
-;      SM_52: .visible .func  (.param .b32 func_retval0) phi()
-;      SM_52: mov.b32         %[[REG:.+]], 0f00000000;
-; SM_52-NEXT: st.param.b32    [func_retval0], %[[REG]];
-; SM_52-NEXT: ret;
-;      SM_70: .visible .func  (.param .b32 func_retval0) phi()
-;      SM_70: mov.b32         %[[REG:.+]], 0f00000000;
-; SM_70-NEXT: st.param.b32    [func_retval0], %[[REG]];
-; SM_70-NEXT: ret;
-;      SM_90: .visible .func  (.param .b32 func_retval0) phi()
-;      SM_90: mov.b32         %[[REG:.+]], 0f00000000;
-; SM_90-NEXT: st.param.b32    [func_retval0], %[[REG]];
-; SM_90-NEXT: ret;
+; COMMON-LABEL: .visible .func  (.param .b32 func_retval0) phi()
+; COMMON: st.param.b32    [func_retval0], 0;
+; COMMON-NEXT: ret;
 define float @phi() {
 entry:
   %0 = call i32 @__nvvm_reflect(ptr @.str)
@@ -175,20 +140,11 @@ exit:
   ret float 0.000000e+00
 }
 
-;      SM_52: .visible .func  (.param .b32 func_retval0) prop()
-;      SM_52: mov.b32         %[[REG:.+]], 3;
-; SM_52-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_52-NEXT: ret;
-;
-;      SM_70: .visible .func  (.param .b32 func_retval0) prop()
-;      SM_70: mov.b32         %[[REG:.+]], 2;
-; SM_70-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_70-NEXT: ret;
-;
-;      SM_90: .visible .func  (.param .b32 func_retval0) prop()
-;      SM_90: mov.b32         %[[REG:.+]], 1;
-; SM_90-NEXT: st.param.b32    [func_retval0], %[[REG:.+]];
-; SM_90-NEXT: ret;
+; COMMON-LABEL: .visible .func  (.param .b32 func_retval0) prop()
+; SM_52: st.param.b32    [func_retval0], 3;
+; SM_70: st.param.b32    [func_retval0], 2;
+; SM_90: st.param.b32    [func_retval0], 1;
+; COMMON-NEXT: ret;
 define i32 @prop() {
 entry:
   %call = call i32 @__nvvm_reflect(ptr @.str)
diff --git a/llvm/test/CodeGen/NVPTX/param-add.ll b/llvm/test/CodeGen/NVPTX/param-add.ll
index 4fc8786c1e2f..4fa1235633cf 100644
--- a/llvm/test/CodeGen/NVPTX/param-add.ll
+++ b/llvm/test/CodeGen/NVPTX/param-add.ll
@@ -14,37 +14,24 @@ declare i32 @callee(%struct.1float %a)
 define i32 @test(%struct.1float alignstack(32) %data) {
 ; CHECK-LABEL: test(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %r1, [test_param_0+1];
-; CHECK-NEXT:    shl.b32 %r2, %r1, 8;
-; CHECK-NEXT:    ld.param.b8 %r3, [test_param_0];
-; CHECK-NEXT:    or.b32 %r4, %r2, %r3;
-; CHECK-NEXT:    ld.param.b8 %r5, [test_param_0+3];
-; CHECK-NEXT:    shl.b32 %r6, %r5, 8;
-; CHECK-NEXT:    ld.param.b8 %r7, [test_param_0+2];
-; CHECK-NEXT:    or.b32 %r8, %r6, %r7;
-; CHECK-NEXT:    shl.b32 %r9, %r8, 16;
-; CHECK-NEXT:    or.b32 %r10, %r9, %r4;
-; CHECK-NEXT:    shr.u32 %r11, %r10, 8;
-; CHECK-NEXT:    shr.u32 %r12, %r10, 16;
-; CHECK-NEXT:    shr.u32 %r13, %r10, 24;
+; CHECK-NEXT:    ld.param.b32 %r1, [test_param_0];
+; CHECK-NEXT:    shr.u32 %r2, %r1, 8;
+; CHECK-NEXT:    shr.u32 %r3, %r1, 16;
+; CHECK-NEXT:    shr.u32 %r4, %r1, 24;
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 1 .b8 param0[4];
-; CHECK-NEXT:    st.param.b8 [param0], %r10;
-; CHECK-NEXT:    st.param.b8 [param0+1], %r11;
-; CHECK-NEXT:    st.param.b8 [param0+2], %r12;
-; CHECK-NEXT:    st.param.b8 [param0+3], %r13;
+; CHECK-NEXT:    st.param.b8 [param0], %r1;
+; CHECK-NEXT:    st.param.b8 [param0+1], %r2;
+; CHECK-NEXT:    st.param.b8 [param0+2], %r3;
+; CHECK-NEXT:    st.param.b8 [param0+3], %r4;
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
-; CHECK-NEXT:    ld.param.b32 %r14, [retval0];
+; CHECK-NEXT:    call.uni (retval0), callee, (param0);
+; CHECK-NEXT:    ld.param.b32 %r5, [retval0];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
 
   %1 = call i32 @callee(%struct.1float %data)
diff --git a/llvm/test/CodeGen/NVPTX/param-align.ll b/llvm/test/CodeGen/NVPTX/param-align.ll
index 16220fb4d47b..c85080fdf295 100644
--- a/llvm/test/CodeGen/NVPTX/param-align.ll
+++ b/llvm/test/CodeGen/NVPTX/param-align.ll
@@ -73,12 +73,10 @@ define ptx_device void @t6() {
 ; CHECK-LABEL: .func check_ptr_align1(
 ; CHECK: 	ld.param.b64 	%rd1, [check_ptr_align1_param_0];
 ; CHECK-NOT: 	ld.param.b8
-; CHECK: 	mov.b32 	%r1, 0;
-; CHECK: 	st.b8 	[%rd1+3], %r1;
-; CHECK: 	st.b8 	[%rd1+2], %r1;
-; CHECK: 	st.b8 	[%rd1+1], %r1;
-; CHECK: 	mov.b32 	%r2, 1;
-; CHECK: 	st.b8 	[%rd1], %r2;
+; CHECK: 	st.b8 	[%rd1+3], 0;
+; CHECK: 	st.b8 	[%rd1+2], 0;
+; CHECK: 	st.b8 	[%rd1+1], 0;
+; CHECK: 	st.b8 	[%rd1], 1;
 ; CHECK: 	ret;
 define void @check_ptr_align1(ptr align 1 %_arg_ptr) {
 entry:
@@ -89,10 +87,8 @@ entry:
 ; CHECK-LABEL: .func check_ptr_align2(
 ; CHECK: 	ld.param.b64 	%rd1, [check_ptr_align2_param_0];
 ; CHECK-NOT: 	ld.param.b16
-; CHECK: 	mov.b32 	%r1, 0;
-; CHECK: 	st.b16 	[%rd1+2], %r1;
-; CHECK: 	mov.b32 	%r2, 2;
-; CHECK: 	st.b16 	[%rd1], %r2;
+; CHECK: 	st.b16 	[%rd1+2], 0;
+; CHECK: 	st.b16 	[%rd1], 2;
 ; CHECK: 	ret;
 define void @check_ptr_align2(ptr align 2 %_arg_ptr) {
 entry:
@@ -103,8 +99,7 @@ entry:
 ; CHECK-LABEL: .func check_ptr_align4(
 ; CHECK: 	ld.param.b64 	%rd1, [check_ptr_align4_param_0];
 ; CHECK-NOT: 	ld.param.b32
-; CHECK: 	mov.b32 	%r1, 4;
-; CHECK: 	st.b32 	[%rd1], %r1;
+; CHECK: 	st.b32 	[%rd1], 4;
 ; CHECK: 	ret;
 define void @check_ptr_align4(ptr align 4 %_arg_ptr) {
 entry:
@@ -114,8 +109,7 @@ entry:
 
 ; CHECK-LABEL: .func check_ptr_align8(
 ; CHECK: 	ld.param.b64 	%rd1, [check_ptr_align8_param_0];
-; CHECK: 	mov.b32 	%r1, 8;
-; CHECK: 	st.b32 	[%rd1], %r1;
+; CHECK: 	st.b32 	[%rd1], 8;
 ; CHECK: 	ret;
 define void @check_ptr_align8(ptr align 8 %_arg_ptr) {
 entry:
diff --git a/llvm/test/CodeGen/NVPTX/param-load-store.ll b/llvm/test/CodeGen/NVPTX/param-load-store.ll
index ce6707c4564b..6c52bfd6cbfd 100644
--- a/llvm/test/CodeGen/NVPTX/param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/param-load-store.ll
@@ -28,15 +28,12 @@
 ; CHECK:      and.b16 [[A:%rs[0-9]+]], [[A8]], 1;
 ; CHECK:      setp.ne.b16 %p1, [[A]], 0
 ; CHECK:      cvt.u32.u16 [[B:%r[0-9]+]], [[A8]]
-; CHECK:      and.b32 [[C:%r[0-9]+]], [[B]], 1;
 ; CHECK:      .param .b32 param0;
-; CHECK:      st.param.b32    [param0], [[C]]
+; CHECK:      st.param.b32    [param0], [[B]]
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni
-; CHECK-NEXT: test_i1,
+; CHECK:      call.uni (retval0), test_i1,
 ; CHECK:      ld.param.b32    [[R8:%r[0-9]+]], [retval0];
-; CHECK:      and.b32         [[R:%r[0-9]+]], [[R8]], 1;
-; CHECK:      st.param.b32    [func_retval0], [[R]];
+; CHECK:      st.param.b32    [func_retval0], [[R8]];
 ; CHECK:      ret;
 define i1 @test_i1(i1 %a) {
   %r = tail call i1 @test_i1(i1 %a);
@@ -76,8 +73,7 @@ define signext i1 @test_i1s(i1 signext %a) {
 ; CHECK-DAG:  st.param.b8     [param0], [[E0]];
 ; CHECK-DAG:  st.param.b8     [param0+2], [[E2]];
 ; CHECK:      .param .align 1 .b8 retval0[1];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v3i1,
+; CHECK:      call.uni (retval0), test_v3i1,
 ; CHECK-DAG:  ld.param.b8     [[RE0:%rs[0-9]+]], [retval0];
 ; CHECK-DAG:  ld.param.b8     [[RE2:%rs[0-9]+]], [retval0+2];
 ; CHECK-DAG:  st.param.b8     [func_retval0], [[RE0]]
@@ -95,8 +91,7 @@ define <3 x i1> @test_v3i1(<3 x i1> %a) {
 ; CHECK:      .param .align 1 .b8 param0[1];
 ; CHECK:      st.param.b8  [param0], [[E0]];
 ; CHECK:      .param .align 1 .b8 retval0[1];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v4i1,
+; CHECK:      call.uni (retval0), test_v4i1,
 ; CHECK:      ld.param.b8  [[RE0:%rs[0-9]+]], [retval0];
 ; CHECK:      ld.param.b8  [[RE1:%rs[0-9]+]], [retval0+1];
 ; CHECK:      ld.param.b8  [[RE2:%rs[0-9]+]], [retval0+2];
@@ -120,8 +115,7 @@ define <4 x i1> @test_v4i1(<4 x i1> %a) {
 ; CHECK-DAG:  st.param.b8     [param0], [[E0]];
 ; CHECK-DAG:  st.param.b8     [param0+4], [[E4]];
 ; CHECK:      .param .align 1 .b8 retval0[1];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v5i1,
+; CHECK:      call.uni (retval0), test_v5i1,
 ; CHECK-DAG:  ld.param.b8  [[RE0:%rs[0-9]+]], [retval0];
 ; CHECK-DAG:  ld.param.b8     [[RE4:%rs[0-9]+]], [retval0+4];
 ; CHECK-DAG:  st.param.b8  [func_retval0], [[RE0]]
@@ -139,8 +133,7 @@ define <5 x i1> @test_v5i1(<5 x i1> %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK:      test_i2,
+; CHECK:      call.uni (retval0), test_i2,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -156,8 +149,7 @@ define i2 @test_i2(i2 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK:      test_i3,
+; CHECK:      call.uni (retval0), test_i3,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -172,15 +164,12 @@ define i3 @test_i3(i3 %a) {
 ; CHECK-NEXT: .param .b32 test_i8_param_0
 ; CHECK:      ld.param.b8 [[A8:%rs[0-9]+]], [test_i8_param_0];
 ; CHECK:      cvt.u32.u16     [[A32:%r[0-9]+]], [[A8]];
-; CHECK:      and.b32         [[A:%r[0-9]+]], [[A32]], 255;
 ; CHECK:      .param .b32 param0;
-; CHECK:      st.param.b32    [param0], [[A]];
+; CHECK:      st.param.b32    [param0], [[A32]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK:      test_i8,
+; CHECK:      call.uni (retval0), test_i8,
 ; CHECK:      ld.param.b32    [[R32:%r[0-9]+]], [retval0];
-; CHECK:      and.b32         [[R:%r[0-9]+]], [[R32]], 255;
-; CHECK:      st.param.b32    [func_retval0], [[R]];
+; CHECK:      st.param.b32    [func_retval0], [[R32]];
 ; CHECK-NEXT: ret;
 define i8 @test_i8(i8 %a) {
        %r = tail call i8 @test_i8(i8 %a);
@@ -196,8 +185,7 @@ define i8 @test_i8(i8 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], [[A]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK:      test_i8s,
+; CHECK:      call.uni (retval0), test_i8s,
 ; CHECK:      ld.param.b32    [[R32:%r[0-9]+]], [retval0];
 ; -- This is suspicious (though correct) -- why not cvt.u8.u32, cvt.s8.s32 ?
 ; CHECK:      cvt.u16.u32     [[R16:%rs[0-9]+]], [[R32]];
@@ -216,8 +204,7 @@ define signext i8 @test_i8s(i8 signext %a) {
 ; CHECK:      .param .align 4 .b8 param0[4];
 ; CHECK:      st.param.b32  [param0], [[R]]
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v3i8,
+; CHECK:      call.uni (retval0), test_v3i8,
 ; CHECK:      ld.param.b32  [[RE:%r[0-9]+]], [retval0];
 ; v4i8/i32->{v3i8 elements}->v4i8/i32 conversion is messy and not very
 ; interesting here, so it's skipped.
@@ -235,8 +222,7 @@ define <3 x i8> @test_v3i8(<3 x i8> %a) {
 ; CHECK:      .param .align 4 .b8 param0[4];
 ; CHECK:      st.param.b32  [param0], [[R]];
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v4i8,
+; CHECK:      call.uni (retval0), test_v4i8,
 ; CHECK:      ld.param.b32  [[RET:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32  [func_retval0], [[RET]];
 ; CHECK-NEXT: ret;
@@ -254,11 +240,17 @@ define <4 x i8> @test_v4i8(<4 x i8> %a) {
 ; CHECK-DAG:  st.param.v4.b8  [param0], 
 ; CHECK-DAG:  st.param.b8     [param0+4], [[E4]];
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v5i8,
+; CHECK:      call.uni (retval0), test_v5i8,
 ; CHECK-DAG:  ld.param.v4.b8  {[[RE0:%rs[0-9]+]], [[RE1:%rs[0-9]+]], [[RE2:%rs[0-9]+]], [[RE3:%rs[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.b8     [[RE4:%rs[0-9]+]], [retval0+4];
-; CHECK-DAG:  st.param.v4.b8  [func_retval0], {[[RE0]], [[RE1]], [[RE2]], [[RE3]]}
+; CHECK-DAG:  cvt.u32.u16     [[R3:%r[0-9]+]], [[RE3]];
+; CHECK-DAG:  cvt.u32.u16     [[R2:%r[0-9]+]], [[RE2]];
+; CHECK-DAG:  prmt.b32        [[P0:%r[0-9]+]], [[R2]], [[R3]], 0x3340U;
+; CHECK-DAG:  cvt.u32.u16     [[R1:%r[0-9]+]], [[RE1]];
+; CHECK-DAG:  cvt.u32.u16     [[R0:%r[0-9]+]], [[RE0]];
+; CHECK-DAG:  prmt.b32        [[P1:%r[0-9]+]], [[R0]], [[R1]], 0x3340U;
+; CHECK-DAG:  prmt.b32        [[P2:%r[0-9]+]], [[P1]], [[P0]], 0x5410U;
+; CHECK-DAG:  st.param.b32  [func_retval0], [[P2]];
 ; CHECK-DAG:  st.param.b8     [func_retval0+4], [[RE4]];
 ; CHECK-NEXT: ret;
 define <5 x i8> @test_v5i8(<5 x i8> %a) {
@@ -272,8 +264,7 @@ define <5 x i8> @test_v5i8(<5 x i8> %a) {
 ; CHECK:      ld.param.b16    {{%rs[0-9]+}}, [test_i11_param_0];
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i11,
+; CHECK:      call.uni (retval0), test_i11,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -290,11 +281,9 @@ define i11 @test_i11(i11 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], [[E32]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i16,
+; CHECK:      call.uni (retval0), test_i16,
 ; CHECK:      ld.param.b32    [[RE32:%r[0-9]+]], [retval0];
-; CHECK:      and.b32         [[R:%r[0-9]+]], [[RE32]], 65535;
-; CHECK:      st.param.b32    [func_retval0], [[R]];
+; CHECK:      st.param.b32    [func_retval0], [[RE32]];
 ; CHECK-NEXT: ret;
 define i16 @test_i16(i16 %a) {
        %r = tail call i16 @test_i16(i16 %a);
@@ -309,8 +298,7 @@ define i16 @test_i16(i16 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], [[E32]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i16s,
+; CHECK:      call.uni (retval0), test_i16s,
 ; CHECK:      ld.param.b32    [[RE32:%r[0-9]+]], [retval0];
 ; CHECK:      cvt.s32.s16     [[R:%r[0-9]+]], [[RE32]];
 ; CHECK:      st.param.b32    [func_retval0], [[R]];
@@ -323,15 +311,13 @@ define signext i16 @test_i16s(i16 signext %a) {
 ; CHECK: .func  (.param .align 8 .b8 func_retval0[8])
 ; CHECK-LABEL: test_v3i16(
 ; CHECK-NEXT: .param .align 8 .b8 test_v3i16_param_0[8]
-; CHECK-DAG:  ld.param.b16    [[E2:%rs[0-9]+]], [test_v3i16_param_0+4];
-; CHECK-DAG:  ld.param.b32    [[R:%r[0-9]+]], [test_v3i16_param_0];
-; CHECK-DAG:  mov.b32 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[R]];
+; CHECK-DAG:  ld.param.b16      [[E2:%rs[0-9]+]], [test_v3i16_param_0+4];
+; CHECK-DAG:  ld.param.v2.b16   {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [test_v3i16_param_0];
 ; CHECK:      .param .align 8 .b8 param0[8];
 ; CHECK:      st.param.v2.b16 [param0], {[[E0]], [[E1]]};
 ; CHECK:      st.param.b16    [param0+4], [[E2]];
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v3i16,
+; CHECK:      call.uni (retval0), test_v3i16,
 ; CHECK:      ld.param.v2.b16 {[[RE0:%rs[0-9]+]], [[RE1:%rs[0-9]+]]}, [retval0];
 ; CHECK:      ld.param.b16    [[RE2:%rs[0-9]+]], [retval0+4];
 ; CHECK-DAG:  st.param.v2.b16 [func_retval0], {[[RE0]], [[RE1]]};
@@ -349,8 +335,7 @@ define <3 x i16> @test_v3i16(<3 x i16> %a) {
 ; CHECK:      .param .align 8 .b8 param0[8];
 ; CHECK:      st.param.v2.b32 [param0], {[[E0]], [[E1]]};
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v4i16,
+; CHECK:      call.uni (retval0), test_v4i16,
 ; CHECK:      ld.param.v2.b32 {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]]}, [retval0];
 ; CHECK:      st.param.v2.b32 [func_retval0], {[[RE0]], [[RE1]]}
 ; CHECK-NEXT: ret;
@@ -368,8 +353,7 @@ define <4 x i16> @test_v4i16(<4 x i16> %a) {
 ; CHECK-DAG:  st.param.v4.b16 [param0], {[[E0]], [[E1]], [[E2]], [[E3]]};
 ; CHECK-DAG:  st.param.b16    [param0+8], [[E4]];
 ; CHECK:      .param .align 16 .b8 retval0[16];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v5i16,
+; CHECK:      call.uni (retval0), test_v5i16,
 ; CHECK-DAG:  ld.param.v4.b16 {[[RE0:%rs[0-9]+]], [[RE1:%rs[0-9]+]], [[RE2:%rs[0-9]+]], [[RE3:%rs[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.b16    [[RE4:%rs[0-9]+]], [retval0+8];
 ; CHECK-DAG:  st.param.v4.b16 [func_retval0], {[[RE0]], [[RE1]], [[RE2]], [[RE3]]}
@@ -387,8 +371,7 @@ define <5 x i16> @test_v5i16(<5 x i16> %a) {
 ; CHECK:      .param .align 2 .b8 param0[2];
 ; CHECK:      st.param.b16    [param0], [[E]];
 ; CHECK:      .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_f16,
+; CHECK:      call.uni (retval0), test_f16,
 ; CHECK:      ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b16    [func_retval0], [[R]]
 ; CHECK-NEXT: ret;
@@ -404,8 +387,7 @@ define half @test_f16(half %a) {
 ; CHECK:      .param .align 4 .b8 param0[4];
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v2f16,
+; CHECK:      call.uni (retval0), test_v2f16,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]]
 ; CHECK-NEXT: ret;
@@ -421,8 +403,7 @@ define <2 x half> @test_v2f16(<2 x half> %a) {
 ; CHECK:      .param .align 2 .b8 param0[2];
 ; CHECK:      st.param.b16    [param0], [[E]];
 ; CHECK:      .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_bf16,
+; CHECK:      call.uni (retval0), test_bf16,
 ; CHECK:      ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b16    [func_retval0], [[R]]
 ; CHECK-NEXT: ret;
@@ -438,8 +419,7 @@ define bfloat @test_bf16(bfloat %a) {
 ; CHECK:      .param .align 4 .b8 param0[4];
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v2bf16,
+; CHECK:      call.uni (retval0), test_v2bf16,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]]
 ; CHECK-NEXT: ret;
@@ -452,15 +432,13 @@ define <2 x bfloat> @test_v2bf16(<2 x bfloat> %a) {
 ; CHECK:.func  (.param .align 8 .b8 func_retval0[8])
 ; CHECK-LABEL: test_v3f16(
 ; CHECK:      .param .align 8 .b8 test_v3f16_param_0[8]
-; CHECK-DAG:  ld.param.b32    [[HH01:%r[0-9]+]], [test_v3f16_param_0];
-; CHECK-DAG:  mov.b32         {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [[HH01]];
+; CHECK-DAG:  ld.param.v2.b16 {[[E0:%rs[0-9]+]], [[E1:%rs[0-9]+]]}, [test_v3f16_param_0];
 ; CHECK-DAG:  ld.param.b16    [[E2:%rs[0-9]+]], [test_v3f16_param_0+4];
 ; CHECK:      .param .align 8 .b8 param0[8];
 ; CHECK-DAG:  st.param.v2.b16 [param0], {[[E0]], [[E1]]};
 ; CHECK-DAG:  st.param.b16    [param0+4], [[E2]];
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v3f16,
+; CHECK:      call.uni (retval0),      test_v3f16,
 ; CHECK-DAG:  ld.param.v2.b16 {[[R0:%rs[0-9]+]], [[R1:%rs[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.b16    [[R2:%rs[0-9]+]], [retval0+4];
 ; CHECK-DAG:  st.param.v2.b16 [func_retval0], {[[R0]], [[R1]]};
@@ -478,8 +456,7 @@ define <3 x half> @test_v3f16(<3 x half> %a) {
 ; CHECK:      .param .align 8 .b8 param0[8];
 ; CHECK:      st.param.v2.b32 [param0], {[[R01]], [[R23]]};
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v4f16,
+; CHECK:      call.uni (retval0),      test_v4f16,
 ; CHECK:      ld.param.v2.b32 {[[RH01:%r[0-9]+]], [[RH23:%r[0-9]+]]}, [retval0];
 ; CHECK:      st.param.v2.b32 [func_retval0], {[[RH01]], [[RH23]]};
 ; CHECK:      ret;
@@ -497,8 +474,7 @@ define <4 x half> @test_v4f16(<4 x half> %a) {
 ; CHECK-DAG:  st.param.v4.b16 [param0],
 ; CHECK-DAG:  st.param.b16    [param0+8], [[E4]];
 ; CHECK:      .param .align 16 .b8 retval0[16];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v5f16,
+; CHECK:      call.uni (retval0),      test_v5f16,
 ; CHECK-DAG:  ld.param.v4.b16 {[[R0:%rs[0-9]+]], [[R1:%rs[0-9]+]], [[R2:%rs[0-9]+]], [[R3:%rs[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.b16    [[R4:%rs[0-9]+]], [retval0+8];
 ; CHECK-DAG:  st.param.v4.b16 [func_retval0], {[[R0]], [[R1]], [[R2]], [[R3]]};
@@ -516,8 +492,7 @@ define <5 x half> @test_v5f16(<5 x half> %a) {
 ; CHECK:      .param .align 16 .b8 param0[16];
 ; CHECK:      st.param.v4.b32 [param0], {[[R01]], [[R23]], [[R45]], [[R67]]};
 ; CHECK:      .param .align 16 .b8 retval0[16];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v8f16,
+; CHECK:      call.uni (retval0), test_v8f16,
 ; CHECK:      ld.param.v4.b32 {[[RH01:%r[0-9]+]], [[RH23:%r[0-9]+]], [[RH45:%r[0-9]+]], [[RH67:%r[0-9]+]]}, [retval0];
 ; CHECK:      st.param.v4.b32 [func_retval0], {[[RH01]], [[RH23]], [[RH45]], [[RH67]]};
 ; CHECK:      ret;
@@ -537,8 +512,7 @@ define <8 x half> @test_v8f16(<8 x half> %a) {
 ; CHECK-DAG:  st.param.v4.b16 [param0+8],
 ; CHECK-DAG:  st.param.b16    [param0+16], [[E8]];
 ; CHECK:      .param .align 32 .b8 retval0[32];
-; CHECK:      call.uni (retval0),
-; CHECK:      test_v9f16,
+; CHECK:      call.uni (retval0), test_v9f16,
 ; CHECK-DAG:  ld.param.v4.b16 {[[R0:%rs[0-9]+]], [[R1:%rs[0-9]+]], [[R2:%rs[0-9]+]], [[R3:%rs[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.v4.b16 {[[R4:%rs[0-9]+]], [[R5:%rs[0-9]+]], [[R6:%rs[0-9]+]], [[R7:%rs[0-9]+]]}, [retval0+8];
 ; CHECK-DAG:  ld.param.b16    [[R8:%rs[0-9]+]], [retval0+16];
@@ -559,8 +533,7 @@ define <9 x half> @test_v9f16(<9 x half> %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i19,
+; CHECK:      call.uni (retval0), test_i19,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -577,8 +550,7 @@ define i19 @test_i19(i19 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i23,
+; CHECK:      call.uni (retval0), test_i23,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -595,8 +567,7 @@ define i23 @test_i23(i23 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i24,
+; CHECK:      call.uni (retval0), test_i24,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -612,8 +583,7 @@ define i24 @test_i24(i24 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i29,
+; CHECK:      call.uni (retval0), test_i29,
 ; CHECK:      ld.param.b32    {{%r[0-9]+}}, [retval0];
 ; CHECK:      st.param.b32    [func_retval0], {{%r[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -629,8 +599,7 @@ define i29 @test_i29(i29 %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i32,
+; CHECK:      call.uni (retval0), test_i32,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -648,8 +617,7 @@ define i32 @test_i32(i32 %a) {
 ; CHECK:      st.param.v2.b32  [param0], {[[E0]], [[E1]]};
 ; CHECK:      st.param.b32     [param0+8], [[E2]];
 ; CHECK:      .param .align 16 .b8 retval0[16];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v3i32,
+; CHECK:      call.uni (retval0), test_v3i32,
 ; CHECK:      ld.param.v2.b32  {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]]}, [retval0];
 ; CHECK:      ld.param.b32     [[RE2:%r[0-9]+]], [retval0+8];
 ; CHECK-DAG:  st.param.v2.b32  [func_retval0], {[[RE0]], [[RE1]]};
@@ -667,8 +635,7 @@ define <3 x i32> @test_v3i32(<3 x i32> %a) {
 ; CHECK:      .param .align 16 .b8 param0[16];
 ; CHECK:      st.param.v4.b32  [param0], {[[E0]], [[E1]], [[E2]], [[E3]]};
 ; CHECK:      .param .align 16 .b8 retval0[16];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v4i32,
+; CHECK:      call.uni (retval0), test_v4i32,
 ; CHECK:      ld.param.v4.b32  {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]], [[RE2:%r[0-9]+]], [[RE3:%r[0-9]+]]}, [retval0];
 ; CHECK:      st.param.v4.b32  [func_retval0], {[[RE0]], [[RE1]], [[RE2]], [[RE3]]}
 ; CHECK-NEXT: ret;
@@ -686,8 +653,7 @@ define <4 x i32> @test_v4i32(<4 x i32> %a) {
 ; CHECK-DAG:  st.param.v4.b32  [param0], {[[E0]], [[E1]], [[E2]], [[E3]]};
 ; CHECK-DAG:  st.param.b32     [param0+16], [[E4]];
 ; CHECK:      .param .align 32 .b8 retval0[32];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v5i32,
+; CHECK:      call.uni (retval0), test_v5i32,
 ; CHECK-DAG:  ld.param.v4.b32  {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]], [[RE2:%r[0-9]+]], [[RE3:%r[0-9]+]]}, [retval0];
 ; CHECK-DAG:  ld.param.b32     [[RE4:%r[0-9]+]], [retval0+16];
 ; CHECK-DAG:  st.param.v4.b32  [func_retval0], {[[RE0]], [[RE1]], [[RE2]], [[RE3]]}
@@ -705,8 +671,7 @@ define <5 x i32> @test_v5i32(<5 x i32> %a) {
 ; CHECK:      .param .b32 param0;
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .b32 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_f32,
+; CHECK:      call.uni (retval0), test_f32,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -723,8 +688,7 @@ define float @test_f32(float %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i40,
+; CHECK:      call.uni (retval0), test_i40,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -741,8 +705,7 @@ define i40 @test_i40(i40 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i47,
+; CHECK:      call.uni (retval0), test_i47,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -759,8 +722,7 @@ define i47 @test_i47(i47 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i48,
+; CHECK:      call.uni (retval0), test_i48,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -778,8 +740,7 @@ define i48 @test_i48(i48 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i51,
+; CHECK:      call.uni (retval0), test_i51,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -797,8 +758,7 @@ define i51 @test_i51(i51 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i56,
+; CHECK:      call.uni (retval0), test_i56,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -814,8 +774,7 @@ define i56 @test_i56(i56 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i57,
+; CHECK:      call.uni (retval0), test_i57,
 ; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [retval0];
 ; CHECK:      st.param.b64    [func_retval0], {{%rd[0-9]+}};
 ; CHECK-NEXT: ret;
@@ -831,8 +790,7 @@ define i57 @test_i57(i57 %a) {
 ; CHECK:      .param .b64 param0;
 ; CHECK:      st.param.b64    [param0], [[E]];
 ; CHECK:      .param .b64 retval0;
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_i64,
+; CHECK:      call.uni (retval0), test_i64,
 ; CHECK:      ld.param.b64    [[R:%rd[0-9]+]], [retval0];
 ; CHECK:      st.param.b64    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -850,8 +808,7 @@ define i64 @test_i64(i64 %a) {
 ; CHECK:      st.param.v2.b64  [param0], {[[E0]], [[E1]]};
 ; CHECK:      st.param.b64     [param0+16], [[E2]];
 ; CHECK:      .param .align 32 .b8 retval0[32];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v3i64,
+; CHECK:      call.uni (retval0), test_v3i64,
 ; CHECK:      ld.param.v2.b64  {[[RE0:%rd[0-9]+]], [[RE1:%rd[0-9]+]]}, [retval0];
 ; CHECK:      ld.param.b64     [[RE2:%rd[0-9]+]], [retval0+16];
 ; CHECK-DAG:  st.param.v2.b64  [func_retval0], {[[RE0]], [[RE1]]};
@@ -874,8 +831,7 @@ define <3 x i64> @test_v3i64(<3 x i64> %a) {
 ; CHECK:      st.param.v2.b64  [param0], {[[E0]], [[E1]]};
 ; CHECK:      st.param.v2.b64  [param0+16], {[[E2]], [[E3]]};
 ; CHECK:      .param .align 32 .b8 retval0[32];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_v4i64,
+; CHECK:      call.uni (retval0), test_v4i64,
 ; CHECK:      ld.param.v2.b64  {[[RE0:%rd[0-9]+]], [[RE1:%rd[0-9]+]]}, [retval0];
 ; CHECK:      ld.param.v2.b64  {[[RE2:%rd[0-9]+]], [[RE3:%rd[0-9]+]]}, [retval0+16];
 ; CHECK-DAG:  st.param.v2.b64  [func_retval0+16], {[[RE2]], [[RE3]]};
@@ -895,8 +851,7 @@ define <4 x i64> @test_v4i64(<4 x i64> %a) {
 ; CHECK:      .param .align 1 .b8 param0[1];
 ; CHECK:      st.param.b8    [param0], [[A]]
 ; CHECK:      .param .align 1 .b8 retval0[1];
-; CHECK:      call.uni
-; CHECK-NEXT: test_s_i1,
+; CHECK:      call.uni (retval0), test_s_i1,
 ; CHECK:      ld.param.b8    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b8    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -912,8 +867,7 @@ define %s_i1 @test_s_i1(%s_i1 %a) {
 ; CHECK:      .param .align 1 .b8 param0[1];
 ; CHECK:      st.param.b8    [param0], [[A]]
 ; CHECK:      .param .align 1 .b8 retval0[1];
-; CHECK:      call.uni
-; CHECK-NEXT: test_s_i8,
+; CHECK:      call.uni (retval0), test_s_i8,
 ; CHECK:      ld.param.b8    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b8    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -929,8 +883,7 @@ define %s_i8 @test_s_i8(%s_i8 %a) {
 ; CHECK:      .param .align 2 .b8 param0[2];
 ; CHECK:      st.param.b16    [param0], [[A]]
 ; CHECK:      .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni
-; CHECK-NEXT: test_s_i16,
+; CHECK:      call.uni (retval0), test_s_i16,
 ; CHECK:      ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b16    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -946,8 +899,7 @@ define %s_i16 @test_s_i16(%s_i16 %a) {
 ; CHECK:      .param .align 2 .b8 param0[2];
 ; CHECK:      st.param.b16    [param0], [[A]]
 ; CHECK:      .param .align 2 .b8 retval0[2];
-; CHECK:      call.uni
-; CHECK-NEXT: test_s_f16,
+; CHECK:      call.uni (retval0), test_s_f16,
 ; CHECK:      ld.param.b16    [[R:%rs[0-9]+]], [retval0];
 ; CHECK:      st.param.b16    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -963,8 +915,7 @@ define %s_f16 @test_s_f16(%s_f16 %a) {
 ; CHECK:      .param .align 4 .b8 param0[4]
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_s_i32,
+; CHECK:      call.uni (retval0), test_s_i32,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -980,8 +931,7 @@ define %s_i32 @test_s_i32(%s_i32 %a) {
 ; CHECK:      .param .align 4 .b8 param0[4]
 ; CHECK:      st.param.b32    [param0], [[E]];
 ; CHECK:      .param .align 4 .b8 retval0[4];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_s_f32,
+; CHECK:      call.uni (retval0), test_s_f32,
 ; CHECK:      ld.param.b32    [[R:%r[0-9]+]], [retval0];
 ; CHECK:      st.param.b32    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -997,8 +947,7 @@ define %s_f32 @test_s_f32(%s_f32 %a) {
 ; CHECK:      .param .align 8 .b8 param0[8];
 ; CHECK:      st.param.b64    [param0], [[E]];
 ; CHECK:      .param .align 8 .b8 retval0[8];
-; CHECK:      call.uni (retval0),
-; CHECK-NEXT: test_s_i64,
+; CHECK:      call.uni (retval0), test_s_i64,
 ; CHECK:      ld.param.b64    [[R:%rd[0-9]+]], [retval0];
 ; CHECK:      st.param.b64    [func_retval0], [[R]];
 ; CHECK-NEXT: ret;
@@ -1023,8 +972,7 @@ define %s_i64 @test_s_i64(%s_i64 %a) {
 ; CHECK-DAG:    st.param.b32    [param0+12], [[E3]];
 ; CHECK-DAG:    st.param.b64    [param0+16], [[E4]];
 ; CHECK:        .param .align 8 .b8 retval0[24];
-; CHECK:        call.uni (retval0),
-; CHECK-NEXT:   test_s_i32f32,
+; CHECK:        call.uni (retval0), test_s_i32f32,
 ; CHECK-DAG:    ld.param.b32    [[RE0:%r[0-9]+]], [retval0];
 ; CHECK-DAG:    ld.param.b32    [[RE1:%r[0-9]+]], [retval0+4];
 ; CHECK-DAG:    ld.param.b32    [[RE2:%r[0-9]+]], [retval0+8];
@@ -1053,8 +1001,7 @@ define %s_i32f32 @test_s_i32f32(%s_i32f32 %a) {
 ; CHECK:        st.param.v2.b32 [param0+8], {[[E2]], [[E3]]};
 ; CHECK:        st.param.b64    [param0+16], [[E4]];
 ; CHECK:        .param .align 8 .b8 retval0[24];
-; CHECK:        call.uni (retval0),
-; CHECK-NEXT:   test_s_i32x4,
+; CHECK:        call.uni (retval0), test_s_i32x4,
 ; CHECK:        ld.param.v2.b32 {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]]}, [retval0];
 ; CHECK:        ld.param.v2.b32 {[[RE2:%r[0-9]+]], [[RE3:%r[0-9]+]]}, [retval0+8];
 ; CHECK:        ld.param.b64    [[RE4:%rd[0-9]+]], [retval0+16];
@@ -1083,8 +1030,7 @@ define %s_i32x4 @test_s_i32x4(%s_i32x4 %a) {
 ; CHECK:        st.param.b32    [param0+16], [[E4]];
 ; CHECK:        st.param.b64    [param0+24], [[E5]];
 ; CHECK:        .param .align 8 .b8 retval0[32];
-; CHECK:        call.uni (retval0),
-; CHECK:        test_s_i1i32x4,
+; CHECK:        call.uni (retval0), test_s_i1i32x4,
 ; CHECK:        (
 ; CHECK:        param0
 ; CHECK:        );
@@ -1162,8 +1108,7 @@ define %s_i8i32x4 @test_s_i1i32x4(%s_i8i32x4 %a) {
 ; CHECK-DAG:        st.param.b8     [param0+23],
 ; CHECK-DAG:        st.param.b8     [param0+24],
 ; CHECK:            .param .align 1 .b8 retval0[25];
-; CHECK:            call.uni (retval0),
-; CHECK-NEXT:       test_s_i1i32x4p,
+; CHECK:            call.uni (retval0), test_s_i1i32x4p,
 ; CHECK-DAG:        ld.param.b8 %rs{{[0-9]+}}, [retval0];
 ; CHECK-DAG:        ld.param.b8 %rs{{[0-9]+}}, [retval0+1];
 ; CHECK-DAG:        ld.param.b8 %rs{{[0-9]+}}, [retval0+2];
@@ -1239,8 +1184,7 @@ define %s_i8i32x4p @test_s_i1i32x4p(%s_i8i32x4p %a) {
 ; CHECK:        st.param.v4.b32 [param0+48], {[[E11]], [[E12]], [[E13]], [[E14]]};
 ; CHECK:        st.param.b32    [param0+64], [[E15]];
 ; CHECK:        .param .align 16 .b8 retval0[80];
-; CHECK:        call.uni (retval0),
-; CHECK:        test_s_crossfield,
+; CHECK:        call.uni (retval0), test_s_crossfield,
 ; CHECK:        ld.param.v2.b32 {[[RE0:%r[0-9]+]], [[RE1:%r[0-9]+]]}, [retval0];
 ; CHECK:        ld.param.b32    [[RE2:%r[0-9]+]], [retval0+8];
 ; CHECK:        ld.param.v4.b32 {[[RE3:%r[0-9]+]], [[RE4:%r[0-9]+]], [[RE5:%r[0-9]+]], [[RE6:%r[0-9]+]]}, [retval0+16];
diff --git a/llvm/test/CodeGen/NVPTX/param-overalign.ll b/llvm/test/CodeGen/NVPTX/param-overalign.ll
index 22a648c7a978..88ad0b0a9f9d 100644
--- a/llvm/test/CodeGen/NVPTX/param-overalign.ll
+++ b/llvm/test/CodeGen/NVPTX/param-overalign.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx | FileCheck %s
 ; RUN: %if ptxas && !ptxas-12.0 %{ llc < %s -mtriple=nvptx -verify-machineinstrs | %ptxas-verify %}
 
@@ -18,27 +19,23 @@ target triple = "nvptx64-nvidia-cuda"
 ; CHECK-NEXT: ;
 
 define float @caller_md(float %a, float %b) {
-; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) caller_md(
-; CHECK-NEXT:         .param .b32 caller_md_param_0,
-; CHECK-NEXT:         .param .b32 caller_md_param_1
-; CHECK-NEXT: )
-; CHECK-NEXT: {
-
-; CHECK:         ld.param.b32 %r1, [caller_md_param_0];
+; CHECK-LABEL: caller_md(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [caller_md_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [caller_md_param_1];
-; CHECK-NEXT:    {
+; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, %r2};
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee_md,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), callee_md, (param0);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
-; CHECK-NEXT:    }
+; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
+
   %s1 = insertvalue %struct.float2 poison, float %a, 0
   %s2 = insertvalue %struct.float2 %s1, float %b, 1
   %r = call float @callee_md(%struct.float2 %s2)
@@ -46,15 +43,16 @@ define float @caller_md(float %a, float %b) {
 }
 
 define float @callee_md(%struct.float2 alignstack(8) %a) {
-; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee_md(
-; CHECK-NEXT:         .param .align 8 .b8 callee_md_param_0[8]
-; CHECK-NEXT: )
-; CHECK-NEXT: {
-
-; CHECK:         ld.param.v2.b32 {%r1, %r2}, [callee_md_param_0];
+; CHECK-LABEL: callee_md(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [callee_md_param_0];
 ; CHECK-NEXT:    add.rn.f32 %r3, %r1, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
+
   %v0 = extractvalue %struct.float2 %a, 0
   %v1 = extractvalue %struct.float2 %a, 1
   %2 = fadd float %v0, %v1
@@ -62,27 +60,23 @@ define float @callee_md(%struct.float2 alignstack(8) %a) {
 }
 
 define float @caller(float %a, float %b) {
-; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) caller(
-; CHECK-NEXT:         .param .b32 caller_param_0,
-; CHECK-NEXT:         .param .b32 caller_param_1
-; CHECK-NEXT: )
-; CHECK-NEXT: {
-
-; CHECK:         ld.param.b32 %r1, [caller_param_0];
+; CHECK-LABEL: caller(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [caller_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [caller_param_1];
-; CHECK-NEXT:    {
+; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, %r2};
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), callee, (param0);
 ; CHECK-NEXT:    ld.param.b32 %r3, [retval0];
-; CHECK-NEXT:    }
+; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
+
   %s1 = insertvalue %struct.float2 poison, float %a, 0
   %s2 = insertvalue %struct.float2 %s1, float %b, 1
   %r = call float @callee(%struct.float2 %s2)
@@ -90,15 +84,16 @@ define float @caller(float %a, float %b) {
 }
 
 define float @callee(%struct.float2 alignstack(8) %a ) {
-; CHECK-LABEL: .visible .func  (.param .b32 func_retval0) callee(
-; CHECK-NEXT:         .param .align 8 .b8 callee_param_0[8]
-; CHECK-NEXT: )
-; CHECK-NEXT: {
-
-; CHECK:         ld.param.v2.b32 {%r1, %r2}, [callee_param_0];
+; CHECK-LABEL: callee(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [callee_param_0];
 ; CHECK-NEXT:    add.rn.f32 %r3, %r1, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
+
   %v0 = extractvalue %struct.float2 %a, 0
   %v1 = extractvalue %struct.float2 %a, 1
   %2 = fadd float %v0, %v1
@@ -106,9 +101,15 @@ define float @callee(%struct.float2 alignstack(8) %a ) {
 }
 
 define alignstack(8) %struct.float2 @aligned_return(%struct.float2 %a ) {
-; CHECK-LABEL: .visible .func  (.param .align 8 .b8 func_retval0[8]) aligned_return(
-; CHECK-NEXT:         .param .align 4 .b8 aligned_return_param_0[8]
-; CHECK-NEXT: )
-; CHECK-NEXT: {
+; CHECK-LABEL: aligned_return(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [aligned_return_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [aligned_return_param_0+4];
+; CHECK-NEXT:    st.param.b32 [func_retval0+4], %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
   ret %struct.float2 %a
 }
diff --git a/llvm/test/CodeGen/NVPTX/param-vectorize-device.ll b/llvm/test/CodeGen/NVPTX/param-vectorize-device.ll
index abb1aff86775..a480984a538b 100644
--- a/llvm/test/CodeGen/NVPTX/param-vectorize-device.ll
+++ b/llvm/test/CodeGen/NVPTX/param-vectorize-device.ll
@@ -86,11 +86,7 @@ define dso_local void @caller_St4x1(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       .param .b32 param0;
   ; CHECK:       st.param.b32 [param0], {{%r[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[4];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x1,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x1, (param0);
   ; CHECK:       ld.param.b32 {{%r[0-9]+}}, [retval0];
   %1 = load i32, ptr %in, align 4
   %call = tail call fastcc [1 x i32] @callee_St4x1(i32 %1) #2
@@ -118,11 +114,7 @@ define dso_local void @caller_St4x2(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       .param .align 16 .b8 param0[8];
   ; CHECK:       st.param.v2.b32 [param0], {{{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[8];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x2,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x2, (param0);
   ; CHECK:       ld.param.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   %agg.tmp = alloca %struct.St4x2, align 8
   %1 = load i64, ptr %in, align 4
@@ -160,11 +152,7 @@ define dso_local void @caller_St4x3(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v2.b32 [param0], {{{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       st.param.b32    [param0+8], {{%r[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[12];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x3,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x3, (param0);
   ; CHECK:       ld.param.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.b32    {{%r[0-9]+}},  [retval0+8];
   %call = tail call fastcc [3 x i32] @callee_St4x3(ptr noundef nonnull byval(%struct.St4x3) align 4 %in) #2
@@ -185,8 +173,8 @@ define internal fastcc [3 x i32] @callee_St4x3(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St4x3_param_0[12]
   ; CHECK:       ld.param.v2.b32 {[[R1:%r[0-9]+]], [[R2:%r[0-9]+]]}, [callee_St4x3_param_0];
   ; CHECK:       ld.param.b32    [[R3:%r[0-9]+]],  [callee_St4x3_param_0+8];
-  ; CHECK:       st.param.v2.b32 [func_retval0], {[[R1]], [[R2]]};
-  ; CHECK:       st.param.b32    [func_retval0+8], [[R3]];
+  ; CHECK-DAG:   st.param.v2.b32 [func_retval0], {[[R1]], [[R2]]};
+  ; CHECK-DAG:   st.param.b32    [func_retval0+8], [[R3]];
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [3 x i32], ptr %in, i64 0, i64 1
@@ -207,11 +195,7 @@ define dso_local void @caller_St4x4(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       .param .align 16 .b8 param0[16];
   ; CHECK:       st.param.v4.b32 [param0], {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[16];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x4,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x4, (param0);
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   %call = tail call fastcc [4 x i32] @callee_St4x4(ptr noundef nonnull byval(%struct.St4x4) align 4 %in) #2
   %.fca.0.extract = extractvalue [4 x i32] %call, 0
@@ -258,11 +242,7 @@ define dso_local void @caller_St4x5(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v4.b32 [param0],  {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       st.param.b32    [param0+16], {{%r[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[20];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x5,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x5, (param0);
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.b32    {{%r[0-9]+}},  [retval0+16];
   %call = tail call fastcc [5 x i32] @callee_St4x5(ptr noundef nonnull byval(%struct.St4x5) align 4 %in) #2
@@ -289,8 +269,8 @@ define internal fastcc [5 x i32] @callee_St4x5(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St4x5_param_0[20]
   ; CHECK:       ld.param.v4.b32 {[[R1:%r[0-9]+]], [[R2:%r[0-9]+]], [[R3:%r[0-9]+]], [[R4:%r[0-9]+]]}, [callee_St4x5_param_0];
   ; CHECK:       ld.param.b32    [[R5:%r[0-9]+]],   [callee_St4x5_param_0+16];
-  ; CHECK:       st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
-  ; CHECK:       st.param.b32    [func_retval0+16], [[R5]];
+  ; CHECK-DAG:   st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
+  ; CHECK-DAG:   st.param.b32    [func_retval0+16], [[R5]];
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [5 x i32], ptr %in, i64 0, i64 1
@@ -318,11 +298,7 @@ define dso_local void @caller_St4x6(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v4.b32 [param0],  {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       st.param.v2.b32 [param0+16], {{{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[24];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x6,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x6, (param0);
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0+16];
   %call = tail call fastcc [6 x i32] @callee_St4x6(ptr noundef nonnull byval(%struct.St4x6) align 4 %in) #2
@@ -352,8 +328,8 @@ define internal fastcc [6 x i32] @callee_St4x6(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St4x6_param_0[24]
   ; CHECK:       ld.param.v4.b32 {[[R1:%r[0-9]+]], [[R2:%r[0-9]+]], [[R3:%r[0-9]+]], [[R4:%r[0-9]+]]}, [callee_St4x6_param_0];
   ; CHECK:       ld.param.v2.b32 {[[R5:%r[0-9]+]],  [[R6:%r[0-9]+]]}, [callee_St4x6_param_0+16];
-  ; CHECK:       st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
-  ; CHECK:       st.param.v2.b32 [func_retval0+16], {[[R5]], [[R6]]};
+  ; CHECK-DAG:   st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
+  ; CHECK-DAG:   st.param.v2.b32 [func_retval0+16], {[[R5]], [[R6]]};
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [6 x i32], ptr %in, i64 0, i64 1
@@ -385,11 +361,7 @@ define dso_local void @caller_St4x7(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v2.b32 [param0+16], {{{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       st.param.b32    [param0+24], {{%r[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[28];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x7,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x7, (param0);
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.v2.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0+16];
   ; CHECK:       ld.param.b32    {{%r[0-9]+}}, [retval0+24];
@@ -424,9 +396,9 @@ define internal fastcc [7 x i32] @callee_St4x7(ptr nocapture noundef readonly by
   ; CHECK:       ld.param.v4.b32 {[[R1:%r[0-9]+]], [[R2:%r[0-9]+]], [[R3:%r[0-9]+]], [[R4:%r[0-9]+]]}, [callee_St4x7_param_0];
   ; CHECK:       ld.param.v2.b32 {[[R5:%r[0-9]+]],  [[R6:%r[0-9]+]]}, [callee_St4x7_param_0+16];
   ; CHECK:       ld.param.b32    [[R7:%r[0-9]+]],   [callee_St4x7_param_0+24];
-  ; CHECK:       st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
-  ; CHECK:       st.param.v2.b32 [func_retval0+16], {[[R5]], [[R6]]};
-  ; CHECK:       st.param.b32    [func_retval0+24], [[R7]];
+  ; CHECK-DAG:   st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
+  ; CHECK-DAG:   st.param.v2.b32 [func_retval0+16], {[[R5]], [[R6]]};
+  ; CHECK-DAG:   st.param.b32    [func_retval0+24], [[R7]];
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [7 x i32], ptr %in, i64 0, i64 1
@@ -460,11 +432,7 @@ define dso_local void @caller_St4x8(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v4.b32 [param0],  {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       st.param.v4.b32 [param0+16], {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[32];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St4x8,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St4x8, (param0);
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.v4.b32 {{{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}, {{%r[0-9]+}}}, [retval0+16];
   %call = tail call fastcc [8 x i32] @callee_St4x8(ptr noundef nonnull byval(%struct.St4x8) align 4 %in) #2
@@ -500,8 +468,8 @@ define internal fastcc [8 x i32] @callee_St4x8(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St4x8_param_0[32]
   ; CHECK:       ld.param.v4.b32 {[[R1:%r[0-9]+]], [[R2:%r[0-9]+]], [[R3:%r[0-9]+]], [[R4:%r[0-9]+]]}, [callee_St4x8_param_0];
   ; CHECK:       ld.param.v4.b32 {[[R5:%r[0-9]+]], [[R6:%r[0-9]+]], [[R7:%r[0-9]+]], [[R8:%r[0-9]+]]}, [callee_St4x8_param_0+16];
-  ; CHECK:       st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
-  ; CHECK:       st.param.v4.b32 [func_retval0+16], {[[R5]], [[R6]], [[R7]], [[R8]]};
+  ; CHECK-DAG:   st.param.v4.b32 [func_retval0],  {[[R1]], [[R2]], [[R3]], [[R4]]};
+  ; CHECK-DAG:   st.param.v4.b32 [func_retval0+16], {[[R5]], [[R6]], [[R7]], [[R8]]};
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [8 x i32], ptr %in, i64 0, i64 1
@@ -537,11 +505,7 @@ define dso_local void @caller_St8x1(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       .param .b64 param0;
   ; CHECK:       st.param.b64 [param0], {{%rd[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[8];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St8x1,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St8x1, (param0);
   ; CHECK:       ld.param.b64 {{%rd[0-9]+}}, [retval0];
   %1 = load i64, ptr %in, align 8
   %call = tail call fastcc [1 x i64] @callee_St8x1(i64 %1) #2
@@ -569,11 +533,7 @@ define dso_local void @caller_St8x2(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       .param .align 16 .b8 param0[16];
   ; CHECK:       st.param.v2.b64 [param0],  {{{%rd[0-9]+}}, {{%rd[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[16];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St8x2,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St8x2, (param0);
   ; CHECK:       ld.param.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [retval0];
   %call = tail call fastcc [2 x i64] @callee_St8x2(ptr noundef nonnull byval(%struct.St8x2) align 8 %in) #2
   %.fca.0.extract = extractvalue [2 x i64] %call, 0
@@ -608,11 +568,7 @@ define dso_local void @caller_St8x3(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v2.b64 [param0],  {{{%rd[0-9]+}}, {{%rd[0-9]+}}};
   ; CHECK:       st.param.b64    [param0+16], {{%rd[0-9]+}};
   ; CHECK:       .param .align 16 .b8 retval0[24];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St8x3,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St8x3, (param0);
   ; CHECK:       ld.param.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.b64    {{%rd[0-9]+}}, [retval0+16];
   %call = tail call fastcc [3 x i64] @callee_St8x3(ptr noundef nonnull byval(%struct.St8x3) align 8 %in) #2
@@ -633,8 +589,8 @@ define internal fastcc [3 x i64] @callee_St8x3(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St8x3_param_0[24]
   ; CHECK:       ld.param.v2.b64 {[[RD1:%rd[0-9]+]], [[RD2:%rd[0-9]+]]}, [callee_St8x3_param_0];
   ; CHECK:       ld.param.b64    [[RD3:%rd[0-9]+]],  [callee_St8x3_param_0+16];
-  ; CHECK:       st.param.v2.b64 [func_retval0],   {[[RD1]], [[RD2]]};
-  ; CHECK:       st.param.b64    [func_retval0+16],  [[RD3]];
+  ; CHECK-DAG:   st.param.v2.b64 [func_retval0],   {[[RD1]], [[RD2]]};
+  ; CHECK-DAG:   st.param.b64    [func_retval0+16],  [[RD3]];
   ; CHECK-NEXT:  ret;
   %1 = load i64, ptr %in, align 8
   %arrayidx.1 = getelementptr inbounds [3 x i64], ptr %in, i64 0, i64 1
@@ -656,11 +612,7 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
   ; CHECK:       st.param.v2.b64 [param0],  {{{%rd[0-9]+}}, {{%rd[0-9]+}}};
   ; CHECK:       st.param.v2.b64 [param0+16], {{{%rd[0-9]+}}, {{%rd[0-9]+}}};
   ; CHECK:       .param .align 16 .b8 retval0[32];
-  ; CHECK:       call.uni (retval0),
-  ; CHECK-NEXT:  callee_St8x4,
-  ; CHECK-NEXT:  (
-  ; CHECK-NEXT:  param0
-  ; CHECK-NEXT:  );
+  ; CHECK:       call.uni (retval0), callee_St8x4, (param0);
   ; CHECK:       ld.param.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [retval0];
   ; CHECK:       ld.param.v2.b64 {{{%rd[0-9]+}}, {{%rd[0-9]+}}}, [retval0+16];
   %call = tail call fastcc [4 x i64] @callee_St8x4(ptr noundef nonnull byval(%struct.St8x4) align 8 %in) #2
@@ -684,8 +636,8 @@ define internal fastcc [4 x i64] @callee_St8x4(ptr nocapture noundef readonly by
   ; CHECK-NEXT:  .param .align 16 .b8 callee_St8x4_param_0[32]
   ; CHECK:       ld.param.v2.b64 {[[RD1:%rd[0-9]+]], [[RD2:%rd[0-9]+]]}, [callee_St8x4_param_0];
   ; CHECK:       ld.param.v2.b64 {[[RD3:%rd[0-9]+]], [[RD4:%rd[0-9]+]]}, [callee_St8x4_param_0+16];
-  ; CHECK:       st.param.v2.b64 [func_retval0],  {[[RD1]], [[RD2]]};
-  ; CHECK:       st.param.v2.b64 [func_retval0+16], {[[RD3]], [[RD4]]};
+  ; CHECK-DAG:   st.param.v2.b64 [func_retval0],  {[[RD1]], [[RD2]]};
+  ; CHECK-DAG:   st.param.v2.b64 [func_retval0+16], {[[RD3]], [[RD4]]};
   ; CHECK-NEXT:  ret;
   %1 = load i64, ptr %in, align 8
   %arrayidx.1 = getelementptr inbounds [4 x i64], ptr %in, i64 0, i64 1
@@ -735,10 +687,10 @@ define external fastcc [4 x i32] @callee_St4x4_external(ptr nocapture noundef re
   ; CHECK:       ld.param.b32 [[R2:%r[0-9]+]],   [callee_St4x4_external_param_0+4];
   ; CHECK:       ld.param.b32 [[R3:%r[0-9]+]],   [callee_St4x4_external_param_0+8];
   ; CHECK:       ld.param.b32 [[R4:%r[0-9]+]],   [callee_St4x4_external_param_0+12];
-  ; CHECK:       st.param.b32 [func_retval0],  [[R1]];
-  ; CHECK:       st.param.b32 [func_retval0+4],  [[R2]];
-  ; CHECK:       st.param.b32 [func_retval0+8],  [[R3]];
-  ; CHECK:       st.param.b32 [func_retval0+12], [[R4]];
+  ; CHECK-DAG:   st.param.b32 [func_retval0],  [[R1]];
+  ; CHECK-DAG:   st.param.b32 [func_retval0+4],  [[R2]];
+  ; CHECK-DAG:   st.param.b32 [func_retval0+8],  [[R3]];
+  ; CHECK-DAG:   st.param.b32 [func_retval0+12], [[R4]];
   ; CHECK-NEXT:  ret;
   %1 = load i32, ptr %in, align 4
   %arrayidx.1 = getelementptr inbounds [4 x i32], ptr %in, i64 0, i64 1
diff --git a/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll b/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
index 3096b953e8d3..160511387652 100644
--- a/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
+++ b/llvm/test/CodeGen/NVPTX/pow2_mask_cmp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=nvptx64 -verify-machineinstrs < %s | FileCheck %s
 ; RUN: %if ptxas %{ llc -mtriple=nvptx64 -verify-machineinstrs < %s | %ptxas-verify %}
 
@@ -10,9 +11,19 @@
 ; value will be identical regardless of the boolean representation.
 ; Check that the optimization triggers in this case.
 
-; CHECK-LABEL: @pow2_mask_cmp
-; CHECK: bfe.u32 {{%r[0-9]+}}, {{%r[0-9]+}}, 3, 1
 define i32 @pow2_mask_cmp(i32 %x) {
+; CHECK-LABEL: pow2_mask_cmp(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs1, [pow2_mask_cmp_param_0];
+; CHECK-NEXT:    shr.u16 %rs2, %rs1, 3;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT:    and.b32 %r2, %r1, 1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
   %a = and i32 %x, 8
   %cmp = icmp ne i32 %a, 0
   %r = zext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/NVPTX/pr13291-i1-store.ll b/llvm/test/CodeGen/NVPTX/pr13291-i1-store.ll
index b6f1964c54c7..cd2505c20d39 100644
--- a/llvm/test/CodeGen/NVPTX/pr13291-i1-store.ll
+++ b/llvm/test/CodeGen/NVPTX/pr13291-i1-store.ll
@@ -4,10 +4,8 @@
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
 define ptx_kernel void @t1(ptr %a) {
-; PTX32:      mov.b16 %rs{{[0-9]+}}, 0;
-; PTX32-NEXT: st.global.b8 [%r{{[0-9]+}}], %rs{{[0-9]+}};
-; PTX64:      mov.b16 %rs{{[0-9]+}}, 0;
-; PTX64-NEXT: st.global.b8 [%rd{{[0-9]+}}], %rs{{[0-9]+}};
+; PTX32:      st.global.b8 [%r{{[0-9]+}}], 0;
+; PTX64:      st.global.b8 [%rd{{[0-9]+}}], 0;
   store i1 false, ptr %a
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
index 3649ef53b088..74023c4f5631 100644
--- a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
@@ -15,11 +15,9 @@ define i1 @check_i1() {
   ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0];
   ; PTX-DAG: } // callseq {{[0-9]+}}
 
-  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
-  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 1;
-  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 1;
-
-  ; PTX-DAG: st.param.b32 [func_retval0], [[RES]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[RES:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0], [[RES]];
+  ; PTX-WITH-DAG: st.param.b32 [func_retval0], [[LD]];
 
   %ret = call i1 @callee_i1()
   ret i1 %ret
@@ -32,11 +30,9 @@ define  i16 @check_i16() {
   ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0];
   ; PTX-DAG: } // callseq {{[0-9]+}}
 
-  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
-  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 65535;
-  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 65535;
-
-  ; PTX-DAG: st.param.b32 [func_retval0], [[RES]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[RES:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0], [[RES]];
+  ; PTX-WITH-DAG: st.param.b32 [func_retval0], [[LD]];
 
   %ret = call i16 @callee_i16()
   ret i16 %ret
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
index a1d8d0590f16..5d0d6f6ecd5f 100644
--- a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
@@ -83,16 +83,16 @@ body:             |
     %5:b32 = ProxyRegB32 killed %1
     %6:b32 = ProxyRegB32 killed %2
     %7:b32 = ProxyRegB32 killed %3
-    ; CHECK: StoreRetvalV4I32 killed %0, killed %1, killed %2, killed %3
-    StoreRetvalV4I32 killed %4, killed %5, killed %6, killed %7, 0
+    ; CHECK: STV_i32_v4 killed %0, killed %1, killed %2, killed %3
+    STV_i32_v4 killed %4, killed %5, killed %6, killed %7, 0, 0, 101, 32, &func_retval0, 0 :: (store (s128), addrspace 101)
 
     %8:b32 = LoadParamMemI32 0
     ; CHECK-NOT: ProxyReg
     %9:b32 = ProxyRegB32 killed %8
     %10:b32 = ProxyRegB32 killed %9
     %11:b32 = ProxyRegB32 killed %10
-    ; CHECK: StoreRetvalI32 killed %8
-    StoreRetvalI32 killed %11, 0
+    ; CHECK: ST_i32 killed %8
+    ST_i32 killed %11, 0, 0, 101, 32, &func_retval0, 0 :: (store (s32), addrspace 101)
     Return
 
 ...
diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
index d5b451dad7bc..e10949f95fac 100644
--- a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -23,19 +23,19 @@ define half @reduce_fadd_half(<8 x half> %in) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    mov.b16 %rs3, 0x0000;
-; CHECK-NEXT:    add.rn.f16 %rs4, %rs1, %rs3;
-; CHECK-NEXT:    add.rn.f16 %rs5, %rs4, %rs2;
-; CHECK-NEXT:    mov.b32 {%rs6, %rs7}, %r2;
-; CHECK-NEXT:    add.rn.f16 %rs8, %rs5, %rs6;
-; CHECK-NEXT:    add.rn.f16 %rs9, %rs8, %rs7;
-; CHECK-NEXT:    mov.b32 {%rs10, %rs11}, %r3;
-; CHECK-NEXT:    add.rn.f16 %rs12, %rs9, %rs10;
-; CHECK-NEXT:    add.rn.f16 %rs13, %rs12, %rs11;
-; CHECK-NEXT:    mov.b32 {%rs14, %rs15}, %r4;
-; CHECK-NEXT:    add.rn.f16 %rs16, %rs13, %rs14;
-; CHECK-NEXT:    add.rn.f16 %rs17, %rs16, %rs15;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-NEXT:    mov.b16 %rs9, 0x0000;
+; CHECK-NEXT:    add.rn.f16 %rs10, %rs7, %rs9;
+; CHECK-NEXT:    add.rn.f16 %rs11, %rs10, %rs8;
+; CHECK-NEXT:    add.rn.f16 %rs12, %rs11, %rs5;
+; CHECK-NEXT:    add.rn.f16 %rs13, %rs12, %rs6;
+; CHECK-NEXT:    add.rn.f16 %rs14, %rs13, %rs3;
+; CHECK-NEXT:    add.rn.f16 %rs15, %rs14, %rs4;
+; CHECK-NEXT:    add.rn.f16 %rs16, %rs15, %rs1;
+; CHECK-NEXT:    add.rn.f16 %rs17, %rs16, %rs2;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs17;
 ; CHECK-NEXT:    ret;
   %res = call half @llvm.vector.reduce.fadd(half 0.0, <8 x half> %in)
@@ -90,12 +90,10 @@ define half @reduce_fadd_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-LABEL: reduce_fadd_half_reassoc_nonpow2(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<16>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fadd_half_reassoc_nonpow2_param_0+8];
-; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fadd_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_fadd_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fadd_half_reassoc_nonpow2_param_0];
 ; CHECK-NEXT:    mov.b16 %rs8, 0x0000;
 ; CHECK-NEXT:    add.rn.f16 %rs9, %rs1, %rs8;
@@ -187,17 +185,17 @@ define half @reduce_fmul_half(<8 x half> %in) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_param_0];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    mul.rn.f16 %rs3, %rs1, %rs2;
-; CHECK-NEXT:    mov.b32 {%rs4, %rs5}, %r2;
-; CHECK-NEXT:    mul.rn.f16 %rs6, %rs3, %rs4;
-; CHECK-NEXT:    mul.rn.f16 %rs7, %rs6, %rs5;
-; CHECK-NEXT:    mov.b32 {%rs8, %rs9}, %r3;
-; CHECK-NEXT:    mul.rn.f16 %rs10, %rs7, %rs8;
-; CHECK-NEXT:    mul.rn.f16 %rs11, %rs10, %rs9;
-; CHECK-NEXT:    mov.b32 {%rs12, %rs13}, %r4;
-; CHECK-NEXT:    mul.rn.f16 %rs14, %rs11, %rs12;
-; CHECK-NEXT:    mul.rn.f16 %rs15, %rs14, %rs13;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r2;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-NEXT:    mul.rn.f16 %rs9, %rs7, %rs8;
+; CHECK-NEXT:    mul.rn.f16 %rs10, %rs9, %rs5;
+; CHECK-NEXT:    mul.rn.f16 %rs11, %rs10, %rs6;
+; CHECK-NEXT:    mul.rn.f16 %rs12, %rs11, %rs3;
+; CHECK-NEXT:    mul.rn.f16 %rs13, %rs12, %rs4;
+; CHECK-NEXT:    mul.rn.f16 %rs14, %rs13, %rs1;
+; CHECK-NEXT:    mul.rn.f16 %rs15, %rs14, %rs2;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs15;
 ; CHECK-NEXT:    ret;
   %res = call half @llvm.vector.reduce.fmul(half 1.0, <8 x half> %in)
@@ -253,15 +251,15 @@ define half @reduce_fmul_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmul_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmul_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmul_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmul_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    mul.rn.f16x2 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0x3C00;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    mul.rn.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT:    mul.rn.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    mul.rn.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    mul.rn.f16x2 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    mul.rn.f16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
@@ -382,15 +380,15 @@ define half @reduce_fmax_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmax_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmax_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmax_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmax_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    max.f16x2 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0xFE00;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    max.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT:    max.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT:    max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    max.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    max.f16x2 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    max.f16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
@@ -514,15 +512,15 @@ define half @reduce_fmin_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmin_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmin_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmin_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmin_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    min.f16x2 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0x7E00;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    min.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT:    min.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT:    min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    min.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    min.f16x2 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    min.f16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
@@ -646,15 +644,15 @@ define half @reduce_fmaximum_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmaximum_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fmaximum_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmaximum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmaximum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    max.NaN.f16x2 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0xFC00;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    max.NaN.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT:    max.NaN.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT:    max.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    max.NaN.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    max.NaN.f16x2 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    max.NaN.f16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
@@ -778,15 +776,15 @@ define half @reduce_fminimum_half_reassoc_nonpow2(<7 x half> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fminimum_half_reassoc_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_fminimum_half_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fminimum_half_reassoc_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fminimum_half_reassoc_nonpow2_param_0+12];
+; CHECK-NEXT:    min.NaN.f16x2 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0x7C00;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    min.NaN.f16x2 %r5, %r3, %r4;
-; CHECK-NEXT:    min.NaN.f16x2 %r6, %r2, %r1;
-; CHECK-NEXT:    min.NaN.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    min.NaN.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    min.NaN.f16x2 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    min.NaN.f16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
@@ -911,12 +909,11 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-LABEL: reduce_add_i16_nonpow2(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<2>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.b32 %r1, [reduce_add_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-SM80-NEXT:    ld.param.b16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_add_i16_nonpow2_param_0+8];
 ; CHECK-SM80-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
 ; CHECK-SM80-NEXT:    add.s16 %rs8, %rs3, %rs7;
 ; CHECK-SM80-NEXT:    add.s16 %rs9, %rs1, %rs5;
@@ -924,8 +921,8 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-NEXT:    add.s16 %rs11, %rs2, %rs6;
 ; CHECK-SM80-NEXT:    add.s16 %rs12, %rs11, %rs4;
 ; CHECK-SM80-NEXT:    add.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_add_i16_nonpow2(
@@ -936,15 +933,15 @@ define i16 @reduce_add_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.b32 %r1, [reduce_add_i16_nonpow2_param_0+8];
 ; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_add_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_add_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-SM100-NEXT:    ld.param.b16 %rs7, [reduce_add_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    add.s16x2 %r4, %r2, %r1;
 ; CHECK-SM100-NEXT:    mov.b16 %rs8, 0;
-; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT:    add.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT:    add.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT:    add.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    add.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT:    add.s16x2 %r7, %r4, %r6;
 ; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-SM100-NEXT:    add.s16 %rs11, %rs9, %rs10;
 ; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1026,12 +1023,11 @@ define i16 @reduce_mul_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-LABEL: reduce_mul_i16_nonpow2(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<14>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [reduce_mul_i16_nonpow2_param_0+8];
-; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_mul_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_mul_i16_nonpow2_param_0+8];
 ; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_mul_i16_nonpow2_param_0];
 ; CHECK-NEXT:    mul.lo.s16 %rs8, %rs3, %rs7;
 ; CHECK-NEXT:    mul.lo.s16 %rs9, %rs1, %rs5;
@@ -1039,8 +1035,8 @@ define i16 @reduce_mul_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-NEXT:    mul.lo.s16 %rs11, %rs2, %rs6;
 ; CHECK-NEXT:    mul.lo.s16 %rs12, %rs4, %rs11;
 ; CHECK-NEXT:    mul.lo.s16 %rs13, %rs10, %rs12;
-; CHECK-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.mul(<7 x i16> %in)
   ret i16 %res
@@ -1137,12 +1133,11 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-LABEL: reduce_umax_i16_nonpow2(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<2>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.b32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-SM80-NEXT:    ld.param.b16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_umax_i16_nonpow2_param_0+8];
 ; CHECK-SM80-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
 ; CHECK-SM80-NEXT:    max.u16 %rs8, %rs3, %rs7;
 ; CHECK-SM80-NEXT:    max.u16 %rs9, %rs1, %rs5;
@@ -1150,8 +1145,8 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-NEXT:    max.u16 %rs11, %rs2, %rs6;
 ; CHECK-SM80-NEXT:    max.u16 %rs12, %rs4, %rs11;
 ; CHECK-SM80-NEXT:    max.u16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_umax_i16_nonpow2(
@@ -1162,15 +1157,15 @@ define i16 @reduce_umax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.b32 %r1, [reduce_umax_i16_nonpow2_param_0+8];
 ; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umax_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_umax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-SM100-NEXT:    ld.param.b16 %rs7, [reduce_umax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    max.u16x2 %r4, %r2, %r1;
 ; CHECK-SM100-NEXT:    mov.b16 %rs8, 0;
-; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT:    max.u16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT:    max.u16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT:    max.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    max.u16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT:    max.u16x2 %r7, %r4, %r6;
 ; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-SM100-NEXT:    max.u16 %rs11, %rs9, %rs10;
 ; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1271,12 +1266,11 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-LABEL: reduce_umin_i16_nonpow2(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<2>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.b32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-SM80-NEXT:    ld.param.b16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_umin_i16_nonpow2_param_0+8];
 ; CHECK-SM80-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
 ; CHECK-SM80-NEXT:    min.u16 %rs8, %rs3, %rs7;
 ; CHECK-SM80-NEXT:    min.u16 %rs9, %rs1, %rs5;
@@ -1284,8 +1278,8 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-NEXT:    min.u16 %rs11, %rs2, %rs6;
 ; CHECK-SM80-NEXT:    min.u16 %rs12, %rs4, %rs11;
 ; CHECK-SM80-NEXT:    min.u16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_umin_i16_nonpow2(
@@ -1296,15 +1290,15 @@ define i16 @reduce_umin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.b32 %r1, [reduce_umin_i16_nonpow2_param_0+8];
 ; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_umin_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_umin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-SM100-NEXT:    ld.param.b16 %rs7, [reduce_umin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    min.u16x2 %r4, %r2, %r1;
 ; CHECK-SM100-NEXT:    mov.b16 %rs8, -1;
-; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT:    min.u16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT:    min.u16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT:    min.u16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    min.u16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT:    min.u16x2 %r7, %r4, %r6;
 ; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-SM100-NEXT:    min.u16 %rs11, %rs9, %rs10;
 ; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1405,12 +1399,11 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-LABEL: reduce_smax_i16_nonpow2(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<2>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.b32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-SM80-NEXT:    ld.param.b16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_smax_i16_nonpow2_param_0+8];
 ; CHECK-SM80-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
 ; CHECK-SM80-NEXT:    max.s16 %rs8, %rs3, %rs7;
 ; CHECK-SM80-NEXT:    max.s16 %rs9, %rs1, %rs5;
@@ -1418,8 +1411,8 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-NEXT:    max.s16 %rs11, %rs2, %rs6;
 ; CHECK-SM80-NEXT:    max.s16 %rs12, %rs4, %rs11;
 ; CHECK-SM80-NEXT:    max.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_smax_i16_nonpow2(
@@ -1430,15 +1423,15 @@ define i16 @reduce_smax_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.b32 %r1, [reduce_smax_i16_nonpow2_param_0+8];
 ; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smax_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_smax_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-SM100-NEXT:    ld.param.b16 %rs7, [reduce_smax_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    max.s16x2 %r4, %r2, %r1;
 ; CHECK-SM100-NEXT:    mov.b16 %rs8, -32768;
-; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT:    max.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT:    max.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT:    max.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    max.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT:    max.s16x2 %r7, %r4, %r6;
 ; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-SM100-NEXT:    max.s16 %rs11, %rs9, %rs10;
 ; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1539,12 +1532,11 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-LABEL: reduce_smin_i16_nonpow2(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b16 %rs<14>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<3>;
+; CHECK-SM80-NEXT:    .reg .b32 %r<2>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.b32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
-; CHECK-SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
 ; CHECK-SM80-NEXT:    ld.param.b16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM80-NEXT:    ld.param.v2.b16 {%rs5, %rs6}, [reduce_smin_i16_nonpow2_param_0+8];
 ; CHECK-SM80-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
 ; CHECK-SM80-NEXT:    min.s16 %rs8, %rs3, %rs7;
 ; CHECK-SM80-NEXT:    min.s16 %rs9, %rs1, %rs5;
@@ -1552,8 +1544,8 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM80-NEXT:    min.s16 %rs11, %rs2, %rs6;
 ; CHECK-SM80-NEXT:    min.s16 %rs12, %rs4, %rs11;
 ; CHECK-SM80-NEXT:    min.s16 %rs13, %rs10, %rs12;
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r2, %rs13;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-SM80-NEXT:    cvt.u32.u16 %r1, %rs13;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_smin_i16_nonpow2(
@@ -1564,15 +1556,15 @@ define i16 @reduce_smin_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.b32 %r1, [reduce_smin_i16_nonpow2_param_0+8];
 ; CHECK-SM100-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-SM100-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_smin_i16_nonpow2_param_0];
-; CHECK-SM100-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_smin_i16_nonpow2_param_0];
+; CHECK-SM100-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-SM100-NEXT:    ld.param.b16 %rs7, [reduce_smin_i16_nonpow2_param_0+12];
+; CHECK-SM100-NEXT:    min.s16x2 %r4, %r2, %r1;
 ; CHECK-SM100-NEXT:    mov.b16 %rs8, 32767;
-; CHECK-SM100-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-SM100-NEXT:    min.s16x2 %r5, %r3, %r4;
-; CHECK-SM100-NEXT:    min.s16x2 %r6, %r2, %r1;
-; CHECK-SM100-NEXT:    min.s16x2 %r7, %r6, %r5;
+; CHECK-SM100-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-SM100-NEXT:    min.s16x2 %r6, %r3, %r5;
+; CHECK-SM100-NEXT:    min.s16x2 %r7, %r4, %r6;
 ; CHECK-SM100-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-SM100-NEXT:    min.s16 %rs11, %rs9, %rs10;
 ; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1675,15 +1667,15 @@ define i16 @reduce_and_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_and_i16_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_and_i16_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_and_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_and_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    and.b32 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, -1;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    and.b32 %r5, %r3, %r4;
-; CHECK-NEXT:    and.b32 %r6, %r2, %r1;
-; CHECK-NEXT:    and.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    and.b32 %r6, %r3, %r5;
+; CHECK-NEXT:    and.b32 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    and.b16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1786,15 +1778,15 @@ define i16 @reduce_or_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_or_i16_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_or_i16_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_or_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_or_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    or.b32 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    or.b32 %r5, %r3, %r4;
-; CHECK-NEXT:    or.b32 %r6, %r2, %r1;
-; CHECK-NEXT:    or.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    or.b32 %r6, %r3, %r5;
+; CHECK-NEXT:    or.b32 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    or.b16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
@@ -1897,15 +1889,15 @@ define i16 @reduce_xor_i16_nonpow2(<7 x i16> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [reduce_xor_i16_nonpow2_param_0+8];
 ; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
-; CHECK-NEXT:    ld.param.v4.b16 {%rs1, %rs2, %rs3, %rs4}, [reduce_xor_i16_nonpow2_param_0];
-; CHECK-NEXT:    mov.b32 %r2, {%rs1, %rs2};
-; CHECK-NEXT:    mov.b32 %r3, {%rs3, %rs4};
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_xor_i16_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_xor_i16_nonpow2_param_0+12];
+; CHECK-NEXT:    xor.b32 %r4, %r2, %r1;
 ; CHECK-NEXT:    mov.b16 %rs8, 0;
-; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
-; CHECK-NEXT:    xor.b32 %r5, %r3, %r4;
-; CHECK-NEXT:    xor.b32 %r6, %r2, %r1;
-; CHECK-NEXT:    xor.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    xor.b32 %r6, %r3, %r5;
+; CHECK-NEXT:    xor.b32 %r7, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
 ; CHECK-NEXT:    xor.b16 %rs11, %rs9, %rs10;
 ; CHECK-NEXT:    cvt.u32.u16 %r8, %rs11;
diff --git a/llvm/test/CodeGen/NVPTX/reg-types.ll b/llvm/test/CodeGen/NVPTX/reg-types.ll
index fb065e1b01bb..ea45bfdc5e19 100644
--- a/llvm/test/CodeGen/NVPTX/reg-types.ll
+++ b/llvm/test/CodeGen/NVPTX/reg-types.ll
@@ -6,64 +6,47 @@
 ; RUN: %if ptxas && !ptxas-12.0 %{ llc -O0 < %s -mtriple=nvptx -mcpu=sm_20 | %ptxas-verify %}
 ; RUN: %if ptxas %{ llc -O0 < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
-; CHECK-LABEL: .visible .func func()
-; NO8BIT-LABEL: .visible .func func()
-define void @func() {
-entry:
-  %s8 = alloca i8, align 1
-  %u8 = alloca i8, align 1
-  %s16 = alloca i16, align 2
-  %u16 = alloca i16, align 2
-; Both 8- and 16-bit integers are packed into 16-bit registers.
-; CHECK-DAG: .reg .b16 %rs<
-; We should not generate 8-bit registers.
+; CHECK-LABEL: .visible .func func(
+; NO8BIT-LABEL: .visible .func func(
+define void @func(ptr %p, i1 %cond) {
+; Both 8- and 16-bit integers are packed into 16-bit registers. So we should 
+; not generate 8-bit registers.
 ; NO8BIT-NOT: .reg .{{[bsu]}}8
-  %s32 = alloca i32, align 4
-  %u32 = alloca i32, align 4
+
+; CHECK-DAG: .reg .pred %p<
+; CHECK-DAG: .reg .b16 %rs<
 ; CHECK-DAG: .reg .b32 %r<
-  %s64 = alloca i64, align 8
-  %u64 = alloca i64, align 8
 ; CHECK-DAG: .reg .b64 %rd<
-  %f32 = alloca float, align 4
-  %f64 = alloca double, align 8
 
-; Verify that we use correct register types.
-  store i8 1, ptr %s8, align 1
-; CHECK: mov.b16 [[R1:%rs[0-9]]], 1;
-; CHECK-NEXT: st.b8 {{.*}}, [[R1]]
-  store i8 2, ptr %u8, align 1
-; CHECK: mov.b16 [[R2:%rs[0-9]]], 2;
-; CHECK-NEXT: st.b8 {{.*}}, [[R2]]
-  store i16 3, ptr %s16, align 2
-; CHECK: mov.b16 [[R3:%rs[0-9]]], 3;
-; CHECK-NEXT: st.b16 {{.*}}, [[R3]]
-  store i16 4, ptr %u16, align 2
-; CHECK: mov.b16 [[R4:%rs[0-9]]], 4;
-; CHECK-NEXT: st.b16 {{.*}}, [[R4]]
-  store i32 5, ptr %s32, align 4
-; CHECK: mov.b32 [[R5:%r[0-9]]], 5;
-; CHECK-NEXT: st.b32 {{.*}}, [[R5]]
-  store i32 6, ptr %u32, align 4
-; CHECK: mov.b32 [[R6:%r[0-9]]], 6;
-; CHECK-NEXT: st.b32 {{.*}}, [[R6]]
-  store i64 7, ptr %s64, align 8
-; CHECK: mov.b64 [[R7:%rd[0-9]]], 7;
-; CHECK-NEXT: st.b64 {{.*}}, [[R7]]
-  store i64 8, ptr %u64, align 8
-; CHECK: mov.b64 [[R8:%rd[0-9]]], 8;
-; CHECK-NEXT: st.b64 {{.*}}, [[R8]]
+entry:
+  br i1 %cond, label %if, label %join
+if:
+  br label %join
+join:
+  ; CHECK-DAG: mov.pred %p{{[0-9]+}}, %p{{[0-9]+}};
+  ; CHECK-DAG: mov.b16 %rs{{[0-9]+}}, %rs{{[0-9]+}};
+  ; CHECK-DAG: mov.b16 %rs{{[0-9]+}}, %rs{{[0-9]+}};
+  ; CHECK-DAG: mov.b32 %r{{[0-9]+}}, %r{{[0-9]+}};
+  ; CHECK-DAG: mov.b64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+
+  ; CHECK-DAG: mov.b32 %r{{[0-9]+}}, %r{{[0-9]+}};
+  ; CHECK-DAG: mov.b64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+
+  %v1 = phi i1 [ true, %if ], [ false, %entry ]
+  %v8 = phi i8 [ 1, %if ], [ 0, %entry ]
+  %v16 = phi i16 [ 2, %if ], [ 0, %entry ]
+  %v32 = phi i32 [ 3, %if ], [ 0, %entry ]
+  %v64 = phi i64 [ 4, %if ], [ 0, %entry ]
+  %f32 = phi float [ 5.0, %if ], [ 0.0, %entry ]
+  %f64 = phi double [ 6.0, %if ], [ 0.0, %entry ]
 
-; FP constants are stored via integer registers, but that's an
-; implementation detail that's irrelevant here.
-  store float 9.000000e+00, ptr %f32, align 4
-  store double 1.000000e+01, ptr %f64, align 8
-; Instead, we force a load into a register and then verify register type.
-  %f32v = load volatile float, ptr %f32, align 4
-; CHECK: ld.volatile.b32         %r{{[0-9]+}}
-  %f64v = load volatile double, ptr %f64, align 8
-; CHECK: ld.volatile.b64         %rd{{[0-9]+}}
+  store volatile i1 %v1, ptr %p, align 1
+  store volatile i8 %v8, ptr %p, align 1
+  store volatile i16 %v16, ptr %p, align 2
+  store volatile i32 %v32, ptr %p, align 4
+  store volatile i64 %v64, ptr %p, align 8
+  store volatile float %f32, ptr %p, align 4
+  store volatile double %f64, ptr %p, align 8
   ret void
-; CHECK: ret;
-; NO8BIT: ret;
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/sext-setcc.ll b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
index 802954bda6a9..0af8190f20d1 100644
--- a/llvm/test/CodeGen/NVPTX/sext-setcc.ll
+++ b/llvm/test/CodeGen/NVPTX/sext-setcc.ll
@@ -7,19 +7,16 @@ define <2 x i16> @sext_setcc_v2i1_to_v2i16(ptr %p) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b64 %rd1, [sext_setcc_v2i1_to_v2i16_param_0];
-; CHECK-NEXT:    ld.b32 %r1, [%rd1];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; CHECK-NEXT:    ld.v2.b16 {%rs1, %rs2}, [%rd1];
 ; CHECK-NEXT:    setp.eq.s16 %p1, %rs1, 0;
 ; CHECK-NEXT:    setp.eq.s16 %p2, %rs2, 0;
 ; CHECK-NEXT:    selp.b16 %rs3, -1, 0, %p2;
 ; CHECK-NEXT:    selp.b16 %rs4, -1, 0, %p1;
-; CHECK-NEXT:    mov.b32 %r2, {%rs4, %rs3};
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-NEXT:    ret;
 entry:
   %v = load <2 x i16>, ptr %p, align 4
diff --git a/llvm/test/CodeGen/NVPTX/shift-opt.ll b/llvm/test/CodeGen/NVPTX/shift-opt.ll
index 65bcbb8e6715..e7866b01064c 100644
--- a/llvm/test/CodeGen/NVPTX/shift-opt.ll
+++ b/llvm/test/CodeGen/NVPTX/shift-opt.ll
@@ -70,19 +70,19 @@ define i64 @test_and(i64 %x, i32 %y) {
 define <2 x i16> @test_vec(<2 x i16> %x, <2 x i8> %y) {
 ; CHECK-LABEL: test_vec(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<5>;
-; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [test_vec_param_0];
-; CHECK-NEXT:    ld.param.b32 %r2, [test_vec_param_1];
-; CHECK-NEXT:    and.b32 %r3, %r2, 16711935;
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NEXT:    shr.u16 %rs3, %rs2, 5;
-; CHECK-NEXT:    shr.u16 %rs4, %rs1, 5;
-; CHECK-NEXT:    mov.b32 %r4, {%rs4, %rs3};
-; CHECK-NEXT:    or.b32 %r5, %r4, %r3;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_vec_param_0];
+; CHECK-NEXT:    ld.param.v2.b8 {%rs3, %rs4}, [test_vec_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs3, %rs4};
+; CHECK-NEXT:    and.b32 %r2, %r1, 16711935;
+; CHECK-NEXT:    shr.u16 %rs5, %rs2, 5;
+; CHECK-NEXT:    shr.u16 %rs6, %rs1, 5;
+; CHECK-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; CHECK-NEXT:    or.b32 %r4, %r3, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
   %ext = zext <2 x i8> %y to <2 x i16>
   %shl = shl <2 x i16> %ext, splat(i16 5)
@@ -132,11 +132,7 @@ define i64 @test_negative_use_lop(i64 %x, i32 %y) {
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], %rd3;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    use,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni use, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd4;
 ; CHECK-NEXT:    ret;
@@ -165,11 +161,7 @@ define i64 @test_negative_use_shl(i64 %x, i32 %y) {
 ; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], %rd2;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    use,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni use, (param0);
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd4;
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
index a28d264cd8ec..3989c8e32e45 100644
--- a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
@@ -43,7 +43,7 @@ define float @test_rsqrt_ftz(float %a) #0 #1 {
   ret float %ret
 }
 
-define double @test_rsqrt64(double %a) #0 {
+define double @test_rsqrt64(double %a) {
 ; CHECK-LABEL: test_rsqrt64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -54,12 +54,12 @@ define double @test_rsqrt64(double %a) #0 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
 ; There's no rsqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
-define double @test_rsqrt64_ftz(double %a) #0 #1 {
+define double @test_rsqrt64_ftz(double %a) #1 {
 ; CHECK-LABEL: test_rsqrt64_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -70,7 +70,7 @@ define double @test_rsqrt64_ftz(double %a) #0 #1 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd2;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
@@ -229,7 +229,7 @@ define float @test_rsqrt32_refined(float %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %val
+  %ret = fdiv arcp float 1.0, %val
   ret float %ret
 }
 
@@ -284,7 +284,7 @@ define double @test_rsqrt64_refined(double %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
@@ -341,7 +341,7 @@ define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %val
+  %ret = fdiv arcp float 1.0, %val
   ret float %ret
 }
 
@@ -396,7 +396,7 @@ define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv double 1.0, %val
+  %ret = fdiv arcp double 1.0, %val
   ret double %ret
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/st-param-imm.ll b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
index bdab9958fe2b..6aa111932a4a 100644
--- a/llvm/test/CodeGen/NVPTX/st-param-imm.ll
+++ b/llvm/test/CodeGen/NVPTX/st-param-imm.ll
@@ -28,11 +28,7 @@ define void @st_param_i8_i16() {
 ; CHECK-NEXT:    .param .align 2 .b8 param0[4];
 ; CHECK-NEXT:    st.param.b8 [param0], 1;
 ; CHECK-NEXT:    st.param.b16 [param0+2], 2;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_i8_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_i8_i16, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    ret;
   call void @call_i8_i16(%struct.A { i8 1, i16 2 })
@@ -48,11 +44,7 @@ define void @st_param_i32() {
 ; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .b32 param0;
 ; CHECK-NEXT:    st.param.b32 [param0], 3;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_i32, (param0);
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    ret;
   call void @call_i32(i32 3)
@@ -68,11 +60,7 @@ define void @st_param_i64() {
 ; CHECK-NEXT:    { // callseq 2, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], 4;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_i64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_i64, (param0);
 ; CHECK-NEXT:    } // callseq 2
 ; CHECK-NEXT:    ret;
   call void @call_i64(i64 4)
@@ -88,11 +76,7 @@ define void @st_param_f32() {
 ; CHECK-NEXT:    { // callseq 3, 0
 ; CHECK-NEXT:    .param .b32 param0;
 ; CHECK-NEXT:    st.param.b32 [param0], 0f40A00000;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_f32, (param0);
 ; CHECK-NEXT:    } // callseq 3
 ; CHECK-NEXT:    ret;
   call void @call_f32(float 5.0)
@@ -108,11 +92,7 @@ define void @st_param_f64() {
 ; CHECK-NEXT:    { // callseq 4, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], 0d4018000000000000;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_f64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_f64, (param0);
 ; CHECK-NEXT:    } // callseq 4
 ; CHECK-NEXT:    ret;
   call void @call_f64(double 6.0)
@@ -134,11 +114,7 @@ define void @st_param_v2_i8_ii() {
 ; CHECK-NEXT:    { // callseq 5, 0
 ; CHECK-NEXT:    .param .align 2 .b8 param0[2];
 ; CHECK-NEXT:    st.param.v2.b8 [param0], {1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i8, (param0);
 ; CHECK-NEXT:    } // callseq 5
 ; CHECK-NEXT:    ret;
   call void @call_v2_i8(%struct.char2 { i8 1, i8 2 })
@@ -154,11 +130,7 @@ define void @st_param_v2_i8_ir(i8 %val) {
 ; CHECK-NEXT:    { // callseq 6, 0
 ; CHECK-NEXT:    .param .align 2 .b8 param0[2];
 ; CHECK-NEXT:    st.param.v2.b8 [param0], {1, %rs1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i8, (param0);
 ; CHECK-NEXT:    } // callseq 6
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.char2 poison, i8 1, 0
@@ -176,11 +148,7 @@ define void @st_param_v2_i8_ri(i8 %val) {
 ; CHECK-NEXT:    { // callseq 7, 0
 ; CHECK-NEXT:    .param .align 2 .b8 param0[2];
 ; CHECK-NEXT:    st.param.v2.b8 [param0], {%rs1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i8, (param0);
 ; CHECK-NEXT:    } // callseq 7
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.char2 poison, i8 %val, 0
@@ -198,11 +166,7 @@ define void @st_param_v2_i16_ii() {
 ; CHECK-NEXT:    { // callseq 8, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v2.b16 [param0], {1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i16, (param0);
 ; CHECK-NEXT:    } // callseq 8
 ; CHECK-NEXT:    ret;
   call void @call_v2_i16(%struct.short2 { i16 1, i16 2 })
@@ -218,11 +182,7 @@ define void @st_param_v2_i16_ir(i16 %val) {
 ; CHECK-NEXT:    { // callseq 9, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v2.b16 [param0], {1, %rs1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i16, (param0);
 ; CHECK-NEXT:    } // callseq 9
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.short2 poison, i16 1, 0
@@ -240,11 +200,7 @@ define void @st_param_v2_i16_ri(i16 %val) {
 ; CHECK-NEXT:    { // callseq 10, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v2.b16 [param0], {%rs1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i16, (param0);
 ; CHECK-NEXT:    } // callseq 10
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.short2 poison, i16 %val, 0
@@ -262,11 +218,7 @@ define void @st_param_v2_i32_ii() {
 ; CHECK-NEXT:    { // callseq 11, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i32, (param0);
 ; CHECK-NEXT:    } // callseq 11
 ; CHECK-NEXT:    ret;
   call void @call_v2_i32(%struct.int2 { i32 1, i32 2 })
@@ -282,11 +234,7 @@ define void @st_param_v2_i32_ir(i32 %val) {
 ; CHECK-NEXT:    { // callseq 12, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {1, %r1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i32, (param0);
 ; CHECK-NEXT:    } // callseq 12
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.int2 poison, i32 1, 0
@@ -304,11 +252,7 @@ define void @st_param_v2_i32_ri(i32 %val) {
 ; CHECK-NEXT:    { // callseq 13, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i32, (param0);
 ; CHECK-NEXT:    } // callseq 13
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.int2 poison, i32 %val, 0
@@ -326,11 +270,7 @@ define void @st_param_v2_i64_ii() {
 ; CHECK-NEXT:    { // callseq 14, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i64, (param0);
 ; CHECK-NEXT:    } // callseq 14
 ; CHECK-NEXT:    ret;
   call void @call_v2_i64(%struct.longlong2 { i64 1, i64 2 })
@@ -346,11 +286,7 @@ define void @st_param_v2_i64_ir(i64 %val) {
 ; CHECK-NEXT:    { // callseq 15, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {1, %rd1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i64, (param0);
 ; CHECK-NEXT:    } // callseq 15
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.longlong2 poison, i64 1, 0
@@ -368,11 +304,7 @@ define void @st_param_v2_i64_ri(i64 %val) {
 ; CHECK-NEXT:    { // callseq 16, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {%rd1, 2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_i64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_i64, (param0);
 ; CHECK-NEXT:    } // callseq 16
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.longlong2 poison, i64 %val, 0
@@ -390,11 +322,7 @@ define void @st_param_v2_f32_ii(float %val) {
 ; CHECK-NEXT:    { // callseq 17, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {0f3F800000, 0f40000000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f32, (param0);
 ; CHECK-NEXT:    } // callseq 17
 ; CHECK-NEXT:    ret;
   call void @call_v2_f32(%struct.float2 { float 1.0, float 2.0 })
@@ -410,11 +338,7 @@ define void @st_param_v2_f32_ir(float %val) {
 ; CHECK-NEXT:    { // callseq 18, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {0f3F800000, %r1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f32, (param0);
 ; CHECK-NEXT:    } // callseq 18
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.float2 poison, float 1.0, 0
@@ -432,11 +356,7 @@ define void @st_param_v2_f32_ri(float %val) {
 ; CHECK-NEXT:    { // callseq 19, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, 0f40000000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f32, (param0);
 ; CHECK-NEXT:    } // callseq 19
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.float2 poison, float %val, 0
@@ -454,11 +374,7 @@ define void @st_param_v2_f64_ii(double %val) {
 ; CHECK-NEXT:    { // callseq 20, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {0d3FF0000000000000, 0d4000000000000000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f64, (param0);
 ; CHECK-NEXT:    } // callseq 20
 ; CHECK-NEXT:    ret;
   call void @call_v2_f64(%struct.double2 { double 1.0, double 2.0 })
@@ -474,11 +390,7 @@ define void @st_param_v2_f64_ir(double %val) {
 ; CHECK-NEXT:    { // callseq 21, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {0d3FF0000000000000, %rd1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f64, (param0);
 ; CHECK-NEXT:    } // callseq 21
 ; CHECK-NEXT:    ret;
   %struct.ir0 = insertvalue %struct.double2 poison, double 1.0, 0
@@ -496,11 +408,7 @@ define void @st_param_v2_f64_ri(double %val) {
 ; CHECK-NEXT:    { // callseq 22, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {%rd1, 0d4000000000000000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v2_f64,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v2_f64, (param0);
 ; CHECK-NEXT:    } // callseq 22
 ; CHECK-NEXT:    ret;
   %struct.ri0 = insertvalue %struct.double2 poison, double %val, 0
@@ -525,11 +433,7 @@ define void @st_param_v4_i8_iiii() {
 ; CHECK-NEXT:    { // callseq 23, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 23
 ; CHECK-NEXT:    ret;
   call void @call_v4_i8(%struct.char4 { i8 1, i8 2, i8 3, i8 4 })
@@ -541,17 +445,13 @@ define void @st_param_v4_i8_irrr(i8 %b, i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irrr_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irrr_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irrr_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_irrr_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_irrr_param_0];
 ; CHECK-NEXT:    { // callseq 24, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, %rs2, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs3, %rs2, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 24
 ; CHECK-NEXT:    ret;
   %struct.irrr0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -567,17 +467,13 @@ define void @st_param_v4_i8_rirr(i8 %a, i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rirr_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rirr_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rirr_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rirr_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rirr_param_0];
 ; CHECK-NEXT:    { // callseq 25, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, %rs2, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, 2, %rs2, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 25
 ; CHECK-NEXT:    ret;
   %struct.rirr0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -593,17 +489,13 @@ define void @st_param_v4_i8_rrir(i8 %a, i8 %b, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrir_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrir_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrir_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrir_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrir_param_0];
 ; CHECK-NEXT:    { // callseq 26, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, 3, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, %rs2, 3, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 26
 ; CHECK-NEXT:    ret;
   %struct.rrir0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -619,17 +511,13 @@ define void @st_param_v4_i8_rrri(i8 %a, i8 %b, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrri_param_0];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrri_param_2];
 ; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrri_param_1];
-; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrri_param_2];
+; CHECK-NEXT:    ld.param.b8 %rs3, [st_param_v4_i8_rrri_param_0];
 ; CHECK-NEXT:    { // callseq 27, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, %rs3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs3, %rs2, %rs1, 4};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 27
 ; CHECK-NEXT:    ret;
   %struct.rrri0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -645,16 +533,12 @@ define void @st_param_v4_i8_iirr(i8 %c, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_iirr_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_iirr_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_iirr_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_iirr_param_0];
 ; CHECK-NEXT:    { // callseq 28, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, %rs1, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, %rs2, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 28
 ; CHECK-NEXT:    ret;
   %struct.iirr0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -670,16 +554,12 @@ define void @st_param_v4_i8_irir(i8 %b, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irir_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irir_param_0];
 ; CHECK-NEXT:    { // callseq 29, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, 3, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs2, 3, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 29
 ; CHECK-NEXT:    ret;
   %struct.irir0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -695,16 +575,12 @@ define void @st_param_v4_i8_irri(i8 %b, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irri_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_irri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_irri_param_0];
 ; CHECK-NEXT:    { // callseq 30, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, %rs2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs2, %rs1, 4};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 30
 ; CHECK-NEXT:    ret;
   %struct.irri0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -720,16 +596,12 @@ define void @st_param_v4_i8_riir(i8 %a, i8 %d) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riir_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riir_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riir_param_0];
 ; CHECK-NEXT:    { // callseq 31, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, 3, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, 2, 3, %rs1};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 31
 ; CHECK-NEXT:    ret;
   %struct.riir0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -745,16 +617,12 @@ define void @st_param_v4_i8_riri(i8 %a, i8 %c) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riri_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_riri_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_riri_param_0];
 ; CHECK-NEXT:    { // callseq 32, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, %rs2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, 2, %rs1, 4};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 32
 ; CHECK-NEXT:    ret;
   %struct.riri0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -770,16 +638,12 @@ define void @st_param_v4_i8_rrii(i8 %a, i8 %b) {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrii_param_0];
-; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrii_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs1, [st_param_v4_i8_rrii_param_1];
+; CHECK-NEXT:    ld.param.b8 %rs2, [st_param_v4_i8_rrii_param_0];
 ; CHECK-NEXT:    { // callseq 33, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
-; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, %rs2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs2, %rs1, 3, 4};
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 33
 ; CHECK-NEXT:    ret;
   %struct.rrii0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -799,11 +663,7 @@ define void @st_param_v4_i8_iiir(i8 %d) {
 ; CHECK-NEXT:    { // callseq 34, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, 3, %rs1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 34
 ; CHECK-NEXT:    ret;
   %struct.iiir0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -823,11 +683,7 @@ define void @st_param_v4_i8_iiri(i8 %c) {
 ; CHECK-NEXT:    { // callseq 35, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v4.b8 [param0], {1, 2, %rs1, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 35
 ; CHECK-NEXT:    ret;
   %struct.iiri0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -847,11 +703,7 @@ define void @st_param_v4_i8_irii(i8 %b) {
 ; CHECK-NEXT:    { // callseq 36, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v4.b8 [param0], {1, %rs1, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 36
 ; CHECK-NEXT:    ret;
   %struct.irii0 = insertvalue %struct.char4 poison, i8 1, 0
@@ -871,11 +723,7 @@ define void @st_param_v4_i8_riii(i8 %a) {
 ; CHECK-NEXT:    { // callseq 37, 0
 ; CHECK-NEXT:    .param .align 4 .b8 param0[4];
 ; CHECK-NEXT:    st.param.v4.b8 [param0], {%rs1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i8,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i8, (param0);
 ; CHECK-NEXT:    } // callseq 37
 ; CHECK-NEXT:    ret;
   %struct.riii0 = insertvalue %struct.char4 poison, i8 %a, 0
@@ -895,11 +743,7 @@ define void @st_param_v4_i16_iiii() {
 ; CHECK-NEXT:    { // callseq 38, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 38
 ; CHECK-NEXT:    ret;
   call void @call_v4_i16(%struct.short4 { i16 1, i16 2, i16 3, i16 4 })
@@ -917,11 +761,7 @@ define void @st_param_v4_i16_irrr(i16 %b, i16 %c, i16 %d) {
 ; CHECK-NEXT:    { // callseq 39, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, %rs1, %rs2, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 39
 ; CHECK-NEXT:    ret;
   %struct.irrr0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -943,11 +783,7 @@ define void @st_param_v4_i16_rirr(i16 %a, i16 %c, i16 %d) {
 ; CHECK-NEXT:    { // callseq 40, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, 2, %rs2, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 40
 ; CHECK-NEXT:    ret;
   %struct.rirr0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -969,11 +805,7 @@ define void @st_param_v4_i16_rrir(i16 %a, i16 %b, i16 %d) {
 ; CHECK-NEXT:    { // callseq 41, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, %rs2, 3, %rs3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 41
 ; CHECK-NEXT:    ret;
   %struct.rrir0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -995,11 +827,7 @@ define void @st_param_v4_i16_rrri(i16 %a, i16 %b, i16 %c) {
 ; CHECK-NEXT:    { // callseq 42, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, %rs2, %rs3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 42
 ; CHECK-NEXT:    ret;
   %struct.rrri0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -1020,11 +848,7 @@ define void @st_param_v4_i16_iirr(i16 %c, i16 %d) {
 ; CHECK-NEXT:    { // callseq 43, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, 2, %rs1, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 43
 ; CHECK-NEXT:    ret;
   %struct.iirr0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1045,11 +869,7 @@ define void @st_param_v4_i16_irir(i16 %b, i16 %d) {
 ; CHECK-NEXT:    { // callseq 44, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, %rs1, 3, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 44
 ; CHECK-NEXT:    ret;
   %struct.irir0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1070,11 +890,7 @@ define void @st_param_v4_i16_irri(i16 %b, i16 %c) {
 ; CHECK-NEXT:    { // callseq 45, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, %rs1, %rs2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 45
 ; CHECK-NEXT:    ret;
   %struct.irri0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1095,11 +911,7 @@ define void @st_param_v4_i16_riir(i16 %a, i16 %d) {
 ; CHECK-NEXT:    { // callseq 46, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, 2, 3, %rs2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 46
 ; CHECK-NEXT:    ret;
   %struct.riir0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -1120,11 +932,7 @@ define void @st_param_v4_i16_riri(i16 %a, i16 %c) {
 ; CHECK-NEXT:    { // callseq 47, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, 2, %rs2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 47
 ; CHECK-NEXT:    ret;
   %struct.riri0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -1145,11 +953,7 @@ define void @st_param_v4_i16_rrii(i16 %a, i16 %b) {
 ; CHECK-NEXT:    { // callseq 48, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, %rs2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 48
 ; CHECK-NEXT:    ret;
   %struct.rrii0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -1169,11 +973,7 @@ define void @st_param_v4_i16_iiir(i16 %d) {
 ; CHECK-NEXT:    { // callseq 49, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, 2, 3, %rs1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 49
 ; CHECK-NEXT:    ret;
   %struct.iiir0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1193,11 +993,7 @@ define void @st_param_v4_i16_iiri(i16 %c) {
 ; CHECK-NEXT:    { // callseq 50, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, 2, %rs1, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 50
 ; CHECK-NEXT:    ret;
   %struct.iiri0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1217,11 +1013,7 @@ define void @st_param_v4_i16_irii(i16 %b) {
 ; CHECK-NEXT:    { // callseq 51, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {1, %rs1, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 51
 ; CHECK-NEXT:    ret;
   %struct.irii0 = insertvalue %struct.short4 poison, i16 1, 0
@@ -1241,11 +1033,7 @@ define void @st_param_v4_i16_riii(i16 %a) {
 ; CHECK-NEXT:    { // callseq 52, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    st.param.v4.b16 [param0], {%rs1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i16,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i16, (param0);
 ; CHECK-NEXT:    } // callseq 52
 ; CHECK-NEXT:    ret;
   %struct.riii0 = insertvalue %struct.short4 poison, i16 %a, 0
@@ -1265,11 +1053,7 @@ define void @st_param_v4_i32_iiii() {
 ; CHECK-NEXT:    { // callseq 53, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 53
 ; CHECK-NEXT:    ret;
   call void @call_v4_i32(%struct.int4 { i32 1, i32 2, i32 3, i32 4 })
@@ -1287,11 +1071,7 @@ define void @st_param_v4_i32_irrr(i32 %b, i32 %c, i32 %d) {
 ; CHECK-NEXT:    { // callseq 54, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, %r1, %r2, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 54
 ; CHECK-NEXT:    ret;
   %struct.irrr0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1313,11 +1093,7 @@ define void @st_param_v4_i32_rirr(i32 %a, i32 %c, i32 %d) {
 ; CHECK-NEXT:    { // callseq 55, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 2, %r2, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 55
 ; CHECK-NEXT:    ret;
   %struct.rirr0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1339,11 +1115,7 @@ define void @st_param_v4_i32_rrir(i32 %a, i32 %b, i32 %d) {
 ; CHECK-NEXT:    { // callseq 56, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, 3, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 56
 ; CHECK-NEXT:    ret;
   %struct.rrir0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1365,11 +1137,7 @@ define void @st_param_v4_i32_rrri(i32 %a, i32 %b, i32 %c) {
 ; CHECK-NEXT:    { // callseq 57, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, %r3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 57
 ; CHECK-NEXT:    ret;
   %struct.rrri0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1390,11 +1158,7 @@ define void @st_param_v4_i32_iirr(i32 %c, i32 %d) {
 ; CHECK-NEXT:    { // callseq 58, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, 2, %r1, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 58
 ; CHECK-NEXT:    ret;
   %struct.iirr0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1415,11 +1179,7 @@ define void @st_param_v4_i32_irir(i32 %b, i32 %d) {
 ; CHECK-NEXT:    { // callseq 59, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, %r1, 3, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 59
 ; CHECK-NEXT:    ret;
   %struct.irir0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1440,11 +1200,7 @@ define void @st_param_v4_i32_irri(i32 %b, i32 %c) {
 ; CHECK-NEXT:    { // callseq 60, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, %r1, %r2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 60
 ; CHECK-NEXT:    ret;
   %struct.irri0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1465,11 +1221,7 @@ define void @st_param_v4_i32_riir(i32 %a, i32 %d) {
 ; CHECK-NEXT:    { // callseq 61, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 2, 3, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 61
 ; CHECK-NEXT:    ret;
   %struct.riir0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1490,11 +1242,7 @@ define void @st_param_v4_i32_riri(i32 %a, i32 %c) {
 ; CHECK-NEXT:    { // callseq 62, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 2, %r2, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 62
 ; CHECK-NEXT:    ret;
   %struct.riri0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1515,11 +1263,7 @@ define void @st_param_v4_i32_rrii(i32 %a, i32 %b) {
 ; CHECK-NEXT:    { // callseq 63, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 63
 ; CHECK-NEXT:    ret;
   %struct.rrii0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1539,11 +1283,7 @@ define void @st_param_v4_i32_iiir(i32 %d) {
 ; CHECK-NEXT:    { // callseq 64, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, 2, 3, %r1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 64
 ; CHECK-NEXT:    ret;
   %struct.iiir0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1563,11 +1303,7 @@ define void @st_param_v4_i32_iiri(i32 %c) {
 ; CHECK-NEXT:    { // callseq 65, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, 2, %r1, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 65
 ; CHECK-NEXT:    ret;
   %struct.iiri0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1587,11 +1323,7 @@ define void @st_param_v4_i32_irii(i32 %b) {
 ; CHECK-NEXT:    { // callseq 66, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {1, %r1, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 66
 ; CHECK-NEXT:    ret;
   %struct.irii0 = insertvalue %struct.int4 poison, i32 1, 0
@@ -1611,11 +1343,7 @@ define void @st_param_v4_i32_riii(i32 %a) {
 ; CHECK-NEXT:    { // callseq 67, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 2, 3, 4};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_i32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_i32, (param0);
 ; CHECK-NEXT:    } // callseq 67
 ; CHECK-NEXT:    ret;
   %struct.riii0 = insertvalue %struct.int4 poison, i32 %a, 0
@@ -1635,11 +1363,7 @@ define void @st_param_v4_f32_iiii() {
 ; CHECK-NEXT:    { // callseq 68, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, 0f40000000, 0f40400000, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 68
 ; CHECK-NEXT:    ret;
   call void @call_v4_f32(%struct.float4 { float 1.0, float 2.0, float 3.0, float 4.0 })
@@ -1657,11 +1381,7 @@ define void @st_param_v4_f32_irrr(float %b, float %c, float %d) {
 ; CHECK-NEXT:    { // callseq 69, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, %r1, %r2, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 69
 ; CHECK-NEXT:    ret;
   %struct.irrr0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1683,11 +1403,7 @@ define void @st_param_v4_f32_rirr(float %a, float %c, float %d) {
 ; CHECK-NEXT:    { // callseq 70, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 0f40000000, %r2, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 70
 ; CHECK-NEXT:    ret;
   %struct.rirr0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1709,11 +1425,7 @@ define void @st_param_v4_f32_rrir(float %a, float %b, float %d) {
 ; CHECK-NEXT:    { // callseq 71, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, 0f40400000, %r3};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 71
 ; CHECK-NEXT:    ret;
   %struct.rrir0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1735,11 +1447,7 @@ define void @st_param_v4_f32_rrri(float %a, float %b, float %c) {
 ; CHECK-NEXT:    { // callseq 72, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, %r3, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 72
 ; CHECK-NEXT:    ret;
   %struct.rrri0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1760,11 +1468,7 @@ define void @st_param_v4_f32_iirr(float %c, float %d) {
 ; CHECK-NEXT:    { // callseq 73, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, 0f40000000, %r1, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 73
 ; CHECK-NEXT:    ret;
   %struct.iirr0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1785,11 +1489,7 @@ define void @st_param_v4_f32_irir(float %b, float %d) {
 ; CHECK-NEXT:    { // callseq 74, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, %r1, 0f40400000, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 74
 ; CHECK-NEXT:    ret;
   %struct.irir0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1810,11 +1510,7 @@ define void @st_param_v4_f32_irri(float %b, float %c) {
 ; CHECK-NEXT:    { // callseq 75, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, %r1, %r2, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 75
 ; CHECK-NEXT:    ret;
   %struct.irri0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1835,11 +1531,7 @@ define void @st_param_v4_f32_riir(float %a, float %d) {
 ; CHECK-NEXT:    { // callseq 76, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 0f40000000, 0f40400000, %r2};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 76
 ; CHECK-NEXT:    ret;
   %struct.riir0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1860,11 +1552,7 @@ define void @st_param_v4_f32_riri(float %a, float %c) {
 ; CHECK-NEXT:    { // callseq 77, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 0f40000000, %r2, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 77
 ; CHECK-NEXT:    ret;
   %struct.riri0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1885,11 +1573,7 @@ define void @st_param_v4_f32_rrii(float %a, float %b) {
 ; CHECK-NEXT:    { // callseq 78, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, %r2, 0f40400000, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 78
 ; CHECK-NEXT:    ret;
   %struct.rrii0 = insertvalue %struct.float4 poison, float %a, 0
@@ -1909,11 +1593,7 @@ define void @st_param_v4_f32_iiir(float %d) {
 ; CHECK-NEXT:    { // callseq 79, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, 0f40000000, 0f40400000, %r1};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 79
 ; CHECK-NEXT:    ret;
   %struct.iiir0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1933,11 +1613,7 @@ define void @st_param_v4_f32_iiri(float %c) {
 ; CHECK-NEXT:    { // callseq 80, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, 0f40000000, %r1, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 80
 ; CHECK-NEXT:    ret;
   %struct.iiri0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1957,11 +1633,7 @@ define void @st_param_v4_f32_irii(float %b) {
 ; CHECK-NEXT:    { // callseq 81, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {0f3F800000, %r1, 0f40400000, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 81
 ; CHECK-NEXT:    ret;
   %struct.irii0 = insertvalue %struct.float4 poison, float 1.0, 0
@@ -1981,11 +1653,7 @@ define void @st_param_v4_f32_riii(float %a) {
 ; CHECK-NEXT:    { // callseq 82, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    st.param.v4.b32 [param0], {%r1, 0f40000000, 0f40400000, 0f40800000};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_v4_f32,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_v4_f32, (param0);
 ; CHECK-NEXT:    } // callseq 82
 ; CHECK-NEXT:    ret;
   %struct.riii0 = insertvalue %struct.float4 poison, float %a, 0
@@ -2011,11 +1679,7 @@ define void @st_param_bfloat() {
 ; CHECK-NEXT:    { // callseq 83, 0
 ; CHECK-NEXT:    .param .align 2 .b8 param0[2];
 ; CHECK-NEXT:    st.param.b16 [param0], %rs1;
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    call_bfloat,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni call_bfloat, (param0);
 ; CHECK-NEXT:    } // callseq 83
 ; CHECK-NEXT:    ret;
   %five = bitcast i16 16640 to bfloat
diff --git a/llvm/test/CodeGen/NVPTX/store-undef.ll b/llvm/test/CodeGen/NVPTX/store-undef.ll
index 52415b05e03d..5b31b5e24bc6 100644
--- a/llvm/test/CodeGen/NVPTX/store-undef.ll
+++ b/llvm/test/CodeGen/NVPTX/store-undef.ll
@@ -16,11 +16,7 @@ define void @test_store_param_undef() {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[32];
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    test_call,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni test_call, (param0);
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    ret;
   call void @test_call(%struct.T undef)
@@ -41,11 +37,7 @@ define void @test_store_param_def(i64 %param0, i32 %param1) {
 ; CHECK-NEXT:    st.param.b64 [param0], %rd1;
 ; CHECK-NEXT:    st.param.v2.b32 [param0+8], {%r2, %r1};
 ; CHECK-NEXT:    st.param.v4.b32 [param0+16], {%r3, %r1, %r4, %r5};
-; CHECK-NEXT:    call.uni
-; CHECK-NEXT:    test_call,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni test_call, (param0);
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:    ret;
   %V2 = insertelement <2 x i32> undef, i32 %param1, i32 1
diff --git a/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll b/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
index a97a8b5822f9..d6961a954177 100644
--- a/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
+++ b/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
@@ -71,11 +71,7 @@ define ptx_kernel void @baz(ptr %red, i32 %idx) {
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    st.param.b64 [param0], %rd3;
 ; CHECK-NEXT:    .param .b32 retval0;
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    texfunc,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), texfunc, (param0);
 ; CHECK-NEXT:    ld.param.b32 %r6, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    add.rn.f32 %r8, %r2, %r6;
diff --git a/llvm/test/CodeGen/NVPTX/tid-range.ll b/llvm/test/CodeGen/NVPTX/tid-range.ll
index 27bc19bfd6c4..019814e47c2b 100644
--- a/llvm/test/CodeGen/NVPTX/tid-range.ll
+++ b/llvm/test/CodeGen/NVPTX/tid-range.ll
@@ -1,20 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 | %ptxas-verify %}
 
 declare i32 @get_register()
 
 define i1 @test1() {
+; CHECK-LABEL: test1(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    mov.u32 %r1, %tid.x;
+; CHECK-NEXT:    setp.eq.s32 %p1, %r1, 1;
+; CHECK-NEXT:    selp.b32 %r2, -1, 0, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT:    ret;
 entry:
   %call = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !0
   %cmp = icmp eq i32 %call, 1
   ret i1 %cmp
 }
 
-; CHECK-LABEL: test1(
-; CHECK: setp.eq.s32  %p1, %r1, 1;
-; CHECK: selp.b32     %[[R:.+]], 1, 0, %p1;
-; CHECK: st.param.b32 [func_retval0], %[[R]];
-
 declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()
 
 !0 = !{ i32 0, i32 3 }
diff --git a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
index efbac868dba3..87e46b1505e3 100644
--- a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; Verifies correctness of load/store of parameters and return values.
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_35 -O0 -verify-machineinstrs | FileCheck -allow-deprecated-dag-overlap %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_35 -O0 -verify-machineinstrs | %ptxas-verify %}
@@ -19,367 +20,489 @@
 ;   -- Only loading and storing the said fields are checked in the following
 ;      series of tests so that they are more concise.
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[16])
-; CHECK-LABEL: test_s_i8i16p(
-; CHECK:        .param .align 8 .b8 test_s_i8i16p_param_0[16]
-; CHECK-DAG:    ld.param.b16 [[P0:%rs[0-9]+]],   [test_s_i8i16p_param_0];
-; CHECK-DAG:    ld.param.b8 [[P2_0:%rs[0-9]+]],   [test_s_i8i16p_param_0+3];
-; CHECK-DAG:    ld.param.b8 [[P2_1:%rs[0-9]+]],   [test_s_i8i16p_param_0+4];
-; CHECK-DAG:    shl.b16     [[P2_1_shl:%rs[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    or.b16      [[P2_1_or:%rs[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK:        { // callseq
-; CHECK:        .param .align 8 .b8 param0[16];
-; CHECK-DAG:    st.param.b16 [param0], [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+3], [[P2_1_or]];
-; CHECK-DAG:    st.param.b8  [param0+4], [[P2_1]];
-; CHECK:        .param .align 8 .b8 retval0[16];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8i16p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b16 [[R0:%rs[0-9]+]],   [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+3];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+4];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b16 [func_retval0], [[R0]];
-; CHECK-DAG:    shl.b16      [[R2_1_shl:%rs[0-9]+]], [[R2_1]], 8;
-; CHECK-DAG:    and.b16      [[R2_0_and:%rs[0-9]+]], [[R2_0]], 255;
-; CHECK-DAG:    or.b16       [[R2:%rs[0-9]+]], [[R2_0_and]], [[R2_1_shl]];
-; CHECK-DAG:    st.param.b8  [func_retval0+3], [[R2]];
-; CHECK-DAG:    and.b16      [[R2_1_and:%rs[0-9]+]], [[R2_1]], 255;
-; CHECK-DAG:    st.param.b8  [func_retval0+4], [[R2_1_and]];
-; CHECK:        ret;
 
 define %s_i8i16p @test_s_i8i16p(%s_i8i16p %a) {
-       %r = tail call %s_i8i16p @test_s_i8i16p(%s_i8i16p %a)
-       ret %s_i8i16p %r
+; CHECK-LABEL: test_s_i8i16p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<15>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs4, [test_s_i8i16p_param_0+4];
+; CHECK-NEXT:    shl.b16 %rs5, %rs4, 8;
+; CHECK-NEXT:    ld.param.b8 %rs6, [test_s_i8i16p_param_0+3];
+; CHECK-NEXT:    or.b16 %rs3, %rs5, %rs6;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8i16p_param_0+8];
+; CHECK-NEXT:    ld.param.b8 %rs2, [test_s_i8i16p_param_0+2];
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_s_i8i16p_param_0];
+; CHECK-NEXT:    { // callseq 0, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[16];
+; CHECK-NEXT:    st.param.b16 [param0], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+2], %rs2;
+; CHECK-NEXT:    st.param.b8 [param0+3], %rs3;
+; CHECK-NEXT:    st.param.b8 [param0+4], %rs4;
+; CHECK-NEXT:    st.param.b64 [param0+8], %rd1;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[16];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8i16p, (param0);
+; CHECK-NEXT:    ld.param.b16 %rs7, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs8, [retval0+2];
+; CHECK-NEXT:    ld.param.b8 %rs9, [retval0+3];
+; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+4];
+; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+8];
+; CHECK-NEXT:    } // callseq 0
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs7;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs10;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs9;
+; CHECK-NEXT:    st.param.b64 [func_retval0+8], %rd2;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8i16p @test_s_i8i16p(%s_i8i16p %a)
+  ret %s_i8i16p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[24])
-; CHECK-LABEL: test_s_i8i32p(
-; CHECK:        .param .align 8 .b8 test_s_i8i32p_param_0[24]
-; CHECK-DAG:    ld.param.b32 [[P0:%r[0-9]+]],   [test_s_i8i32p_param_0];
-; CHECK-DAG:    ld.param.b8 [[P2_0:%r[0-9]+]],   [test_s_i8i32p_param_0+5];
-; CHECK-DAG:    ld.param.b8 [[P2_1:%r[0-9]+]],   [test_s_i8i32p_param_0+6];
-; CHECK-DAG:    ld.param.b8 [[P2_2:%r[0-9]+]],   [test_s_i8i32p_param_0+7];
-; CHECK-DAG:    ld.param.b8 [[P2_3:%r[0-9]+]],   [test_s_i8i32p_param_0+8];
-; CHECK-DAG:    shl.b32     [[P2_1_shl:%r[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    shl.b32     [[P2_2_shl:%r[0-9]+]], [[P2_2]], 16;
-; CHECK-DAG:    shl.b32     [[P2_3_shl:%r[0-9]+]], [[P2_3]], 24;
-; CHECK-DAG:    or.b32      [[P2_or:%r[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK-DAG:    or.b32      [[P2_or_1:%r[0-9]+]], [[P2_3_shl]], [[P2_2_shl]];
-; CHECK-DAG:    or.b32      [[P2:%r[0-9]+]], [[P2_or_1]], [[P2_or]];
-; CHECK-DAG:    shr.u32     [[P2_1_shr:%r[0-9]+]], [[P2]], 8;
-; CHECK-DAG:    shr.u32     [[P2_2_shr:%r[0-9]+]], [[P2_or_1]], 16;
-; CHECK:        { // callseq
-; CHECK-DAG:    .param .align 8 .b8 param0[24];
-; CHECK-DAG:    st.param.b32 [param0], [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+5], [[P2]];
-; CHECK-DAG:    st.param.b8  [param0+6], [[P2_1_shr]];
-; CHECK-DAG:    st.param.b8  [param0+7], [[P2_2_shr]];
-; CHECK-DAG:    st.param.b8  [param0+8], [[P2_3]];
-; CHECK:        .param .align 8 .b8 retval0[24];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8i32p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b32 [[R0:%r[0-9]+]],   [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+5];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+6];
-; CHECK-DAG:    ld.param.b8  [[R2_2:%rs[0-9]+]], [retval0+7];
-; CHECK-DAG:    ld.param.b8  [[R2_3:%rs[0-9]+]], [retval0+8];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b32 [func_retval0], [[R0]];
-; CHECK-DAG:    st.param.b8  [func_retval0+5],
-; CHECK-DAG:    st.param.b8  [func_retval0+6],
-; CHECK-DAG:    st.param.b8  [func_retval0+7],
-; CHECK-DAG:    st.param.b8  [func_retval0+8],
-; CHECK:        ret;
 
 define %s_i8i32p @test_s_i8i32p(%s_i8i32p %a) {
-       %r = tail call %s_i8i32p @test_s_i8i32p(%s_i8i32p %a)
-       ret %s_i8i32p %r
+; CHECK-LABEL: test_s_i8i32p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<20>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %r3, [test_s_i8i32p_param_0+6];
+; CHECK-NEXT:    shl.b32 %r4, %r3, 8;
+; CHECK-NEXT:    ld.param.b8 %r5, [test_s_i8i32p_param_0+5];
+; CHECK-NEXT:    or.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    ld.param.b8 %r7, [test_s_i8i32p_param_0+7];
+; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
+; CHECK-NEXT:    ld.param.b8 %r9, [test_s_i8i32p_param_0+8];
+; CHECK-NEXT:    shl.b32 %r10, %r9, 24;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r8;
+; CHECK-NEXT:    or.b32 %r2, %r11, %r6;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8i32p_param_0+16];
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_s_i8i32p_param_0+4];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8i32p_param_0];
+; CHECK-NEXT:    shr.u32 %r12, %r2, 8;
+; CHECK-NEXT:    shr.u32 %r13, %r11, 16;
+; CHECK-NEXT:    { // callseq 1, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[24];
+; CHECK-NEXT:    st.param.b32 [param0], %r1;
+; CHECK-NEXT:    st.param.b8 [param0+4], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+5], %r2;
+; CHECK-NEXT:    st.param.b8 [param0+6], %r12;
+; CHECK-NEXT:    st.param.b8 [param0+7], %r13;
+; CHECK-NEXT:    st.param.b8 [param0+8], %r9;
+; CHECK-NEXT:    st.param.b64 [param0+16], %rd1;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[24];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8i32p, (param0);
+; CHECK-NEXT:    ld.param.b32 %r14, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs2, [retval0+4];
+; CHECK-NEXT:    ld.param.b8 %rs3, [retval0+5];
+; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+6];
+; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+7];
+; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
+; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
+; CHECK-NEXT:    } // callseq 1
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
+; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8i32p @test_s_i8i32p(%s_i8i32p %a)
+  ret %s_i8i32p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[32])
-; CHECK-LABEL: test_s_i8i64p(
-; CHECK:        .param .align 8 .b8 test_s_i8i64p_param_0[32]
-; CHECK-DAG:    ld.param.b64 [[P0:%rd[0-9]+]],   [test_s_i8i64p_param_0];
-; CHECK-DAG:    ld.param.b8 [[P2_0:%rd[0-9]+]],   [test_s_i8i64p_param_0+9];
-; CHECK-DAG:    ld.param.b8 [[P2_1:%rd[0-9]+]],   [test_s_i8i64p_param_0+10];
-; CHECK-DAG:    ld.param.b8 [[P2_2:%rd[0-9]+]],   [test_s_i8i64p_param_0+11];
-; CHECK-DAG:    ld.param.b8 [[P2_3:%rd[0-9]+]],   [test_s_i8i64p_param_0+12];
-; CHECK-DAG:    ld.param.b8 [[P2_4:%rd[0-9]+]],   [test_s_i8i64p_param_0+13];
-; CHECK-DAG:    ld.param.b8 [[P2_5:%rd[0-9]+]],   [test_s_i8i64p_param_0+14];
-; CHECK-DAG:    ld.param.b8 [[P2_6:%rd[0-9]+]],   [test_s_i8i64p_param_0+15];
-; CHECK-DAG:    ld.param.b8 [[P2_7:%rd[0-9]+]],   [test_s_i8i64p_param_0+16];
-; CHECK-DAG:    shl.b64      [[P2_1_shl:%rd[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    shl.b64      [[P2_2_shl:%rd[0-9]+]], [[P2_2]], 16;
-; CHECK-DAG:    shl.b64      [[P2_3_shl:%rd[0-9]+]], [[P2_3]], 24;
-; CHECK-DAG:    or.b64       [[P2_or_0:%rd[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK-DAG:    or.b64       [[P2_or_1:%rd[0-9]+]], [[P2_3_shl]], [[P2_2_shl]];
-; CHECK-DAG:    or.b64       [[P2_or_2:%rd[0-9]+]], [[P2_or_1]], [[P2_or_0]];
-; CHECK-DAG:    shl.b64 	 [[P2_5_shl:%rd[0-9]+]], [[P2_5]], 8;
-; CHECK-DAG:    shl.b64      [[P2_6_shl:%rd[0-9]+]], [[P2_6]], 16;
-; CHECK-DAG:    shl.b64      [[P2_7_shl:%rd[0-9]+]], [[P2_7]], 24;
-; CHECK-DAG:    or.b64       [[P2_or_3:%rd[0-9]+]], [[P2_5_shl]], [[P2_4]];
-; CHECK-DAG:    or.b64       [[P2_or_4:%rd[0-9]+]], [[P2_7_shl]], [[P2_6_shl]];
-; CHECK-DAG:    or.b64       [[P2_or_5:%rd[0-9]+]], [[P2_or_4]], [[P2_or_3]];
-; CHECK-DAG:    shl.b64      [[P2_or_shl:%rd[0-9]+]], [[P2_or_5]], 32;
-; CHECK-DAG:    or.b64       [[P2:%rd[0-9]+]], [[P2_or_shl]], [[P2_or_2]];
-; CHECK-DAG:    shr.u64      [[P2_shr_1:%rd[0-9]+]], [[P2]], 8;
-; CHECK-DAG:    shr.u64      [[P2_shr_2:%rd[0-9]+]], [[P2]], 16;
-; CHECK-DAG:    shr.u64      [[P2_shr_3:%rd[0-9]+]], [[P2]], 24;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_4:%rd[0-9]+]], [[P2_or_5]], 8, 24;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_5:%rd[0-9]+]], [[P2_or_5]], 16, 16;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_6:%rd[0-9]+]], [[P2_or_5]], 24, 8;
-; CHECK:        { // callseq
-; CHECK:        .param .align 8 .b8 param0[32];
-; CHECK-DAG:    st.param.b64 [param0],  [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+9],  [[P2]];
-; CHECK-DAG:    st.param.b8  [param0+10], [[P2_shr_1]];
-; CHECK-DAG:    st.param.b8  [param0+11], [[P2_shr_2]];
-; CHECK-DAG:    st.param.b8  [param0+12], [[P2_shr_3]];
-; CHECK-DAG:    st.param.b8  [param0+13], [[P2_or_5]];
-; CHECK-DAG:    st.param.b8  [param0+14], [[P2_bfe_4]];
-; CHECK-DAG:    st.param.b8  [param0+15], [[P2_bfe_5]];
-; CHECK-DAG:    st.param.b8  [param0+16], [[P2_bfe_6]];
-; CHECK:        .param .align 8 .b8 retval0[32];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8i64p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b64 [[R0:%rd[0-9]+]],   [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+9];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+10];
-; CHECK-DAG:    ld.param.b8  [[R2_2:%rs[0-9]+]], [retval0+11];
-; CHECK-DAG:    ld.param.b8  [[R2_3:%rs[0-9]+]], [retval0+12];
-; CHECK-DAG:    ld.param.b8  [[R2_4:%rs[0-9]+]], [retval0+13];
-; CHECK-DAG:    ld.param.b8  [[R2_5:%rs[0-9]+]], [retval0+14];
-; CHECK-DAG:    ld.param.b8  [[R2_6:%rs[0-9]+]], [retval0+15];
-; CHECK-DAG:    ld.param.b8  [[R2_7:%rs[0-9]+]], [retval0+16];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b64 [func_retval0], [[R0]];
-; CHECK-DAG:    st.param.b8  [func_retval0+9],
-; CHECK-DAG:    st.param.b8  [func_retval0+10],
-; CHECK-DAG:    st.param.b8  [func_retval0+11],
-; CHECK-DAG:    st.param.b8  [func_retval0+12],
-; CHECK-DAG:    st.param.b8  [func_retval0+13],
-; CHECK-DAG:    st.param.b8  [func_retval0+14],
-; CHECK-DAG:    st.param.b8  [func_retval0+15],
-; CHECK-DAG:    st.param.b8  [func_retval0+16],
-; CHECK:        ret;
 
 define %s_i8i64p @test_s_i8i64p(%s_i8i64p %a) {
-       %r = tail call %s_i8i64p @test_s_i8i64p(%s_i8i64p %a)
-       ret %s_i8i64p %r
+; CHECK-LABEL: test_s_i8i64p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<20>;
+; CHECK-NEXT:    .reg .b64 %rd<68>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rd4, [test_s_i8i64p_param_0+10];
+; CHECK-NEXT:    shl.b64 %rd5, %rd4, 8;
+; CHECK-NEXT:    ld.param.b8 %rd6, [test_s_i8i64p_param_0+9];
+; CHECK-NEXT:    or.b64 %rd7, %rd5, %rd6;
+; CHECK-NEXT:    ld.param.b8 %rd8, [test_s_i8i64p_param_0+11];
+; CHECK-NEXT:    shl.b64 %rd9, %rd8, 16;
+; CHECK-NEXT:    ld.param.b8 %rd10, [test_s_i8i64p_param_0+12];
+; CHECK-NEXT:    shl.b64 %rd11, %rd10, 24;
+; CHECK-NEXT:    or.b64 %rd12, %rd11, %rd9;
+; CHECK-NEXT:    or.b64 %rd13, %rd12, %rd7;
+; CHECK-NEXT:    ld.param.b8 %rd14, [test_s_i8i64p_param_0+14];
+; CHECK-NEXT:    shl.b64 %rd15, %rd14, 8;
+; CHECK-NEXT:    ld.param.b8 %rd16, [test_s_i8i64p_param_0+13];
+; CHECK-NEXT:    or.b64 %rd17, %rd15, %rd16;
+; CHECK-NEXT:    ld.param.b8 %rd18, [test_s_i8i64p_param_0+15];
+; CHECK-NEXT:    shl.b64 %rd19, %rd18, 16;
+; CHECK-NEXT:    ld.param.b8 %rd20, [test_s_i8i64p_param_0+16];
+; CHECK-NEXT:    shl.b64 %rd21, %rd20, 24;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd19;
+; CHECK-NEXT:    or.b64 %rd23, %rd22, %rd17;
+; CHECK-NEXT:    shl.b64 %rd24, %rd23, 32;
+; CHECK-NEXT:    or.b64 %rd2, %rd24, %rd13;
+; CHECK-NEXT:    ld.param.b64 %rd3, [test_s_i8i64p_param_0+24];
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_s_i8i64p_param_0+8];
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8i64p_param_0];
+; CHECK-NEXT:    shr.u64 %rd25, %rd2, 8;
+; CHECK-NEXT:    shr.u64 %rd26, %rd2, 16;
+; CHECK-NEXT:    shr.u64 %rd27, %rd2, 24;
+; CHECK-NEXT:    bfe.u64 %rd28, %rd23, 8, 24;
+; CHECK-NEXT:    bfe.u64 %rd29, %rd23, 16, 16;
+; CHECK-NEXT:    bfe.u64 %rd30, %rd23, 24, 8;
+; CHECK-NEXT:    { // callseq 2, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[32];
+; CHECK-NEXT:    st.param.b64 [param0], %rd1;
+; CHECK-NEXT:    st.param.b8 [param0+8], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+9], %rd2;
+; CHECK-NEXT:    st.param.b8 [param0+10], %rd25;
+; CHECK-NEXT:    st.param.b8 [param0+11], %rd26;
+; CHECK-NEXT:    st.param.b8 [param0+12], %rd27;
+; CHECK-NEXT:    st.param.b8 [param0+13], %rd23;
+; CHECK-NEXT:    st.param.b8 [param0+14], %rd28;
+; CHECK-NEXT:    st.param.b8 [param0+15], %rd29;
+; CHECK-NEXT:    st.param.b8 [param0+16], %rd30;
+; CHECK-NEXT:    st.param.b64 [param0+24], %rd3;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[32];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8i64p, (param0);
+; CHECK-NEXT:    ld.param.b64 %rd31, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs2, [retval0+8];
+; CHECK-NEXT:    ld.param.b8 %rs3, [retval0+9];
+; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+10];
+; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+11];
+; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+12];
+; CHECK-NEXT:    ld.param.b8 %rs7, [retval0+13];
+; CHECK-NEXT:    ld.param.b8 %rs8, [retval0+14];
+; CHECK-NEXT:    ld.param.b8 %rs9, [retval0+15];
+; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+16];
+; CHECK-NEXT:    ld.param.b64 %rd32, [retval0+24];
+; CHECK-NEXT:    } // callseq 2
+; CHECK-NEXT:    cvt.u64.u16 %rd33, %rs3;
+; CHECK-NEXT:    and.b64 %rd34, %rd33, 255;
+; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs4;
+; CHECK-NEXT:    and.b64 %rd36, %rd35, 255;
+; CHECK-NEXT:    shl.b64 %rd37, %rd36, 8;
+; CHECK-NEXT:    or.b64 %rd38, %rd34, %rd37;
+; CHECK-NEXT:    cvt.u64.u16 %rd39, %rs5;
+; CHECK-NEXT:    and.b64 %rd40, %rd39, 255;
+; CHECK-NEXT:    shl.b64 %rd41, %rd40, 16;
+; CHECK-NEXT:    or.b64 %rd42, %rd38, %rd41;
+; CHECK-NEXT:    cvt.u64.u16 %rd43, %rs6;
+; CHECK-NEXT:    and.b64 %rd44, %rd43, 255;
+; CHECK-NEXT:    shl.b64 %rd45, %rd44, 24;
+; CHECK-NEXT:    or.b64 %rd46, %rd42, %rd45;
+; CHECK-NEXT:    cvt.u64.u16 %rd47, %rs7;
+; CHECK-NEXT:    and.b64 %rd48, %rd47, 255;
+; CHECK-NEXT:    shl.b64 %rd49, %rd48, 32;
+; CHECK-NEXT:    or.b64 %rd50, %rd46, %rd49;
+; CHECK-NEXT:    cvt.u64.u16 %rd51, %rs8;
+; CHECK-NEXT:    and.b64 %rd52, %rd51, 255;
+; CHECK-NEXT:    shl.b64 %rd53, %rd52, 40;
+; CHECK-NEXT:    or.b64 %rd54, %rd50, %rd53;
+; CHECK-NEXT:    cvt.u64.u16 %rd55, %rs9;
+; CHECK-NEXT:    and.b64 %rd56, %rd55, 255;
+; CHECK-NEXT:    shl.b64 %rd57, %rd56, 48;
+; CHECK-NEXT:    or.b64 %rd58, %rd54, %rd57;
+; CHECK-NEXT:    cvt.u64.u16 %rd59, %rs10;
+; CHECK-NEXT:    shl.b64 %rd60, %rd59, 56;
+; CHECK-NEXT:    or.b64 %rd61, %rd58, %rd60;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd31;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd39;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
+; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd33;
+; CHECK-NEXT:    shr.u64 %rd64, %rd50, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd64;
+; CHECK-NEXT:    shr.u64 %rd65, %rd54, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd65;
+; CHECK-NEXT:    shr.u64 %rd66, %rd58, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd66;
+; CHECK-NEXT:    shr.u64 %rd67, %rd61, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd67;
+; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd32;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8i64p @test_s_i8i64p(%s_i8i64p %a)
+  ret %s_i8i64p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[16])
-; CHECK-LABEL: test_s_i8f16p(
-; CHECK:        .param .align 8 .b8 test_s_i8f16p_param_0[16]
-; CHECK-DAG:    ld.param.b16 [[P0:%rs[0-9]+]],     [test_s_i8f16p_param_0];
-; CHECK-DAG:    ld.param.b8  [[P2_0:%rs[0-9]+]],   [test_s_i8f16p_param_0+3];
-; CHECK-DAG:    ld.param.b8  [[P2_1:%rs[0-9]+]],   [test_s_i8f16p_param_0+4];
-; CHECK-DAG:    shl.b16      [[P2_1_shl:%rs[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    or.b16       [[P2_1_or:%rs[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK:        { // callseq
-; CHECK:        .param .align 8 .b8 param0[16];
-; CHECK-DAG:    st.param.b16 [param0], [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+3], [[P2_1_or]];
-; CHECK-DAG:    st.param.b8  [param0+4], [[P2_1]];
-; CHECK:        .param .align 8 .b8 retval0[16];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8f16p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b16 [[R0:%rs[0-9]+]],     [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2I_0:%rs[0-9]+]], [retval0+3];
-; CHECK-DAG:    ld.param.b8  [[R2I_1:%rs[0-9]+]], [retval0+4];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b16 [func_retval0], [[R0]];
-; CHECK-DAG:    shl.b16      [[R2I_1_shl:%rs[0-9]+]], [[R2I_1]], 8;
-; CHECK-DAG:    and.b16      [[R2I_0_and:%rs[0-9]+]], [[R2I_0]], 255;
-; CHECK-DAG:    or.b16       [[R2I:%rs[0-9]+]], [[R2I_0_and]], [[R2I_1_shl]];
-; CHECK-DAG:    st.param.b8  [func_retval0+3],  [[R2I]];
-; CHECK-DAG:    and.b16      [[R2I_1_and:%rs[0-9]+]], [[R2I_1]], 255;
-; CHECK-DAG:    st.param.b8  [func_retval0+4],  [[R2I_1_and]];
-; CHECK:        ret;
 
 define %s_i8f16p @test_s_i8f16p(%s_i8f16p %a) {
-       %r = tail call %s_i8f16p @test_s_i8f16p(%s_i8f16p %a)
-       ret %s_i8f16p %r
+; CHECK-LABEL: test_s_i8f16p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<15>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rs4, [test_s_i8f16p_param_0+4];
+; CHECK-NEXT:    shl.b16 %rs5, %rs4, 8;
+; CHECK-NEXT:    ld.param.b8 %rs6, [test_s_i8f16p_param_0+3];
+; CHECK-NEXT:    or.b16 %rs3, %rs5, %rs6;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8f16p_param_0+8];
+; CHECK-NEXT:    ld.param.b8 %rs2, [test_s_i8f16p_param_0+2];
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_s_i8f16p_param_0];
+; CHECK-NEXT:    { // callseq 3, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[16];
+; CHECK-NEXT:    st.param.b16 [param0], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+2], %rs2;
+; CHECK-NEXT:    st.param.b8 [param0+3], %rs3;
+; CHECK-NEXT:    st.param.b8 [param0+4], %rs4;
+; CHECK-NEXT:    st.param.b64 [param0+8], %rd1;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[16];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8f16p, (param0);
+; CHECK-NEXT:    ld.param.b16 %rs7, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs8, [retval0+2];
+; CHECK-NEXT:    ld.param.b8 %rs9, [retval0+3];
+; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+4];
+; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+8];
+; CHECK-NEXT:    } // callseq 3
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs7;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs10;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs9;
+; CHECK-NEXT:    st.param.b64 [func_retval0+8], %rd2;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8f16p @test_s_i8f16p(%s_i8f16p %a)
+  ret %s_i8f16p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[24])
-; CHECK-LABEL: test_s_i8f16x2p(
-; CHECK:        .param .align 8 .b8 test_s_i8f16x2p_param_0[24]
-; CHECK-DAG:    ld.param.b32 [[P0:%r[0-9]+]],  [test_s_i8f16x2p_param_0];
-; CHECK-DAG:    ld.param.b8  [[P2_0:%r[0-9]+]],   [test_s_i8f16x2p_param_0+5];
-; CHECK-DAG:    ld.param.b8  [[P2_1:%r[0-9]+]],   [test_s_i8f16x2p_param_0+6];
-; CHECK-DAG:    ld.param.b8  [[P2_2:%r[0-9]+]],   [test_s_i8f16x2p_param_0+7];
-; CHECK-DAG:    ld.param.b8  [[P2_3:%r[0-9]+]],   [test_s_i8f16x2p_param_0+8];
-; CHECK-DAG:    shl.b32      [[P2_1_shl:%r[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    shl.b32      [[P2_2_shl:%r[0-9]+]], [[P2_2]], 16;
-; CHECK-DAG:    shl.b32      [[P2_3_shl:%r[0-9]+]], [[P2_3]], 24;
-; CHECK-DAG:    or.b32       [[P2_or:%r[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK-DAG:    or.b32       [[P2_or_1:%r[0-9]+]], [[P2_3_shl]], [[P2_2_shl]];
-; CHECK-DAG:    or.b32       [[P2:%r[0-9]+]], [[P2_or_1]], [[P2_or]];
-; CHECK-DAG:    shr.u32      [[P2_1_shr:%r[0-9]+]], [[P2]], 8;
-; CHECK-DAG:    shr.u32      [[P2_2_shr:%r[0-9]+]], [[P2_or_1]], 16;
-; CHECK:        { // callseq
-; CHECK-DAG:    .param .align 8 .b8 param0[24];
-; CHECK-DAG:    st.param.b32 [param0], [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+5], [[P2]];
-; CHECK-DAG:    st.param.b8  [param0+6], [[P2_1_shr]];
-; CHECK-DAG:    st.param.b8  [param0+7], [[P2_2_shr]];
-; CHECK-DAG:    st.param.b8  [param0+8], [[P2_3]];
-; CHECK:        .param .align 8 .b8 retval0[24];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8f16x2p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b32 [[R0:%r[0-9]+]],   [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+5];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+6];
-; CHECK-DAG:    ld.param.b8  [[R2_2:%rs[0-9]+]], [retval0+7];
-; CHECK-DAG:    ld.param.b8  [[R2_3:%rs[0-9]+]], [retval0+8];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b32 [func_retval0], [[R0]];
-; CHECK-DAG:    st.param.b8  [func_retval0+5],
-; CHECK-DAG:    st.param.b8  [func_retval0+6],
-; CHECK-DAG:    st.param.b8  [func_retval0+7],
-; CHECK-DAG:    st.param.b8  [func_retval0+8],
-; CHECK:        ret;
 
 define %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a) {
-       %r = tail call %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a)
-       ret %s_i8f16x2p %r
+; CHECK-LABEL: test_s_i8f16x2p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<20>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %r3, [test_s_i8f16x2p_param_0+6];
+; CHECK-NEXT:    shl.b32 %r4, %r3, 8;
+; CHECK-NEXT:    ld.param.b8 %r5, [test_s_i8f16x2p_param_0+5];
+; CHECK-NEXT:    or.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    ld.param.b8 %r7, [test_s_i8f16x2p_param_0+7];
+; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
+; CHECK-NEXT:    ld.param.b8 %r9, [test_s_i8f16x2p_param_0+8];
+; CHECK-NEXT:    shl.b32 %r10, %r9, 24;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r8;
+; CHECK-NEXT:    or.b32 %r2, %r11, %r6;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8f16x2p_param_0+16];
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_s_i8f16x2p_param_0+4];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8f16x2p_param_0];
+; CHECK-NEXT:    shr.u32 %r12, %r2, 8;
+; CHECK-NEXT:    shr.u32 %r13, %r11, 16;
+; CHECK-NEXT:    { // callseq 4, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[24];
+; CHECK-NEXT:    st.param.b32 [param0], %r1;
+; CHECK-NEXT:    st.param.b8 [param0+4], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+5], %r2;
+; CHECK-NEXT:    st.param.b8 [param0+6], %r12;
+; CHECK-NEXT:    st.param.b8 [param0+7], %r13;
+; CHECK-NEXT:    st.param.b8 [param0+8], %r9;
+; CHECK-NEXT:    st.param.b64 [param0+16], %rd1;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[24];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8f16x2p, (param0);
+; CHECK-NEXT:    ld.param.b32 %r14, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs2, [retval0+4];
+; CHECK-NEXT:    ld.param.b8 %rs3, [retval0+5];
+; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+6];
+; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+7];
+; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
+; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
+; CHECK-NEXT:    } // callseq 4
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
+; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a)
+  ret %s_i8f16x2p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[24])
-; CHECK-LABEL: test_s_i8f32p(
-; CHECK:        .param .align 8 .b8 test_s_i8f32p_param_0[24]
-; CHECK-DAG:    ld.param.b32 [[P0:%r[0-9]+]],    [test_s_i8f32p_param_0];
-; CHECK-DAG:    ld.param.b8  [[P2_0:%r[0-9]+]],   [test_s_i8f32p_param_0+5];
-; CHECK-DAG:    ld.param.b8  [[P2_1:%r[0-9]+]],   [test_s_i8f32p_param_0+6];
-; CHECK-DAG:    ld.param.b8  [[P2_2:%r[0-9]+]],   [test_s_i8f32p_param_0+7];
-; CHECK-DAG:    ld.param.b8  [[P2_3:%r[0-9]+]],   [test_s_i8f32p_param_0+8];
-; CHECK-DAG:    shl.b32      [[P2_1_shl:%r[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    shl.b32      [[P2_2_shl:%r[0-9]+]], [[P2_2]], 16;
-; CHECK-DAG:    shl.b32      [[P2_3_shl:%r[0-9]+]], [[P2_3]], 24;
-; CHECK-DAG:    or.b32       [[P2_or:%r[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK-DAG:    or.b32       [[P2_or_1:%r[0-9]+]], [[P2_3_shl]], [[P2_2_shl]];
-; CHECK-DAG:    or.b32       [[P2:%r[0-9]+]], [[P2_or_1]], [[P2_or]];
-; CHECK-DAG:    shr.u32      [[P2_1_shr:%r[0-9]+]], [[P2]], 8;
-; CHECK-DAG:    shr.u32      [[P2_2_shr:%r[0-9]+]], [[P2_or_1]], 16;
-; CHECK:        { // callseq
-; CHECK-DAG:    .param .align 8 .b8 param0[24];
-; CHECK-DAG:    st.param.b32 [param0], [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+5], [[P2]];
-; CHECK-DAG:    st.param.b8  [param0+6], [[P2_1_shr]];
-; CHECK-DAG:    st.param.b8  [param0+7], [[P2_2_shr]];
-; CHECK-DAG:    st.param.b8  [param0+8], [[P2_3]];
-; CHECK:        .param .align 8 .b8 retval0[24];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8f32p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b32 [[R0:%r[0-9]+]],    [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+5];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+6];
-; CHECK-DAG:    ld.param.b8  [[R2_2:%rs[0-9]+]], [retval0+7];
-; CHECK-DAG:    ld.param.b8  [[R2_3:%rs[0-9]+]], [retval0+8];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b32 [func_retval0], [[R0]];
-; CHECK-DAG:    st.param.b8  [func_retval0+5],
-; CHECK-DAG:    st.param.b8  [func_retval0+6],
-; CHECK-DAG:    st.param.b8  [func_retval0+7],
-; CHECK-DAG:    st.param.b8  [func_retval0+8],
-; CHECK:        ret;
 
 define %s_i8f32p @test_s_i8f32p(%s_i8f32p %a) {
-       %r = tail call %s_i8f32p @test_s_i8f32p(%s_i8f32p %a)
-       ret %s_i8f32p %r
+; CHECK-LABEL: test_s_i8f32p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<20>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %r3, [test_s_i8f32p_param_0+6];
+; CHECK-NEXT:    shl.b32 %r4, %r3, 8;
+; CHECK-NEXT:    ld.param.b8 %r5, [test_s_i8f32p_param_0+5];
+; CHECK-NEXT:    or.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    ld.param.b8 %r7, [test_s_i8f32p_param_0+7];
+; CHECK-NEXT:    shl.b32 %r8, %r7, 16;
+; CHECK-NEXT:    ld.param.b8 %r9, [test_s_i8f32p_param_0+8];
+; CHECK-NEXT:    shl.b32 %r10, %r9, 24;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r8;
+; CHECK-NEXT:    or.b32 %r2, %r11, %r6;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8f32p_param_0+16];
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_s_i8f32p_param_0+4];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8f32p_param_0];
+; CHECK-NEXT:    shr.u32 %r12, %r2, 8;
+; CHECK-NEXT:    shr.u32 %r13, %r11, 16;
+; CHECK-NEXT:    { // callseq 5, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[24];
+; CHECK-NEXT:    st.param.b32 [param0], %r1;
+; CHECK-NEXT:    st.param.b8 [param0+4], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+5], %r2;
+; CHECK-NEXT:    st.param.b8 [param0+6], %r12;
+; CHECK-NEXT:    st.param.b8 [param0+7], %r13;
+; CHECK-NEXT:    st.param.b8 [param0+8], %r9;
+; CHECK-NEXT:    st.param.b64 [param0+16], %rd1;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[24];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8f32p, (param0);
+; CHECK-NEXT:    ld.param.b32 %r14, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs2, [retval0+4];
+; CHECK-NEXT:    ld.param.b8 %rs3, [retval0+5];
+; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+6];
+; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+7];
+; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+8];
+; CHECK-NEXT:    ld.param.b64 %rd2, [retval0+16];
+; CHECK-NEXT:    } // callseq 5
+; CHECK-NEXT:    cvt.u32.u16 %r15, %rs3;
+; CHECK-NEXT:    cvt.u32.u16 %r16, %rs4;
+; CHECK-NEXT:    cvt.u32.u16 %r17, %rs5;
+; CHECK-NEXT:    cvt.u32.u16 %r18, %rs6;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r14;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r18;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r15;
+; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8f32p @test_s_i8f32p(%s_i8f32p %a)
+  ret %s_i8f32p %r
 }
 
-; CHECK:       .visible .func (.param .align 8 .b8 func_retval0[32])
-; CHECK-LABEL: test_s_i8f64p(
-; CHECK:        .param .align 8 .b8 test_s_i8f64p_param_0[32]
-; CHECK-DAG:    ld.param.b64 [[P0:%rd[0-9]+]],    [test_s_i8f64p_param_0];
-; CHECK-DAG:    ld.param.b8  [[P2_0:%rd[0-9]+]],   [test_s_i8f64p_param_0+9];
-; CHECK-DAG:    ld.param.b8  [[P2_1:%rd[0-9]+]],   [test_s_i8f64p_param_0+10];
-; CHECK-DAG:    ld.param.b8  [[P2_2:%rd[0-9]+]],   [test_s_i8f64p_param_0+11];
-; CHECK-DAG:    ld.param.b8  [[P2_3:%rd[0-9]+]],   [test_s_i8f64p_param_0+12];
-; CHECK-DAG:    ld.param.b8  [[P2_4:%rd[0-9]+]],   [test_s_i8f64p_param_0+13];
-; CHECK-DAG:    ld.param.b8  [[P2_5:%rd[0-9]+]],   [test_s_i8f64p_param_0+14];
-; CHECK-DAG:    ld.param.b8  [[P2_6:%rd[0-9]+]],   [test_s_i8f64p_param_0+15];
-; CHECK-DAG:    ld.param.b8  [[P2_7:%rd[0-9]+]],   [test_s_i8f64p_param_0+16];
-; CHECK-DAG:    shl.b64      [[P2_1_shl:%rd[0-9]+]], [[P2_1]], 8;
-; CHECK-DAG:    shl.b64      [[P2_2_shl:%rd[0-9]+]], [[P2_2]], 16;
-; CHECK-DAG:    shl.b64      [[P2_3_shl:%rd[0-9]+]], [[P2_3]], 24;
-; CHECK-DAG:    or.b64       [[P2_or_0:%rd[0-9]+]], [[P2_1_shl]], [[P2_0]];
-; CHECK-DAG:    or.b64       [[P2_or_1:%rd[0-9]+]], [[P2_3_shl]], [[P2_2_shl]];
-; CHECK-DAG:    or.b64       [[P2_or_2:%rd[0-9]+]], [[P2_or_1]], [[P2_or_0]];
-; CHECK-DAG:    shl.b64 	 [[P2_5_shl:%rd[0-9]+]], [[P2_5]], 8;
-; CHECK-DAG:    shl.b64      [[P2_6_shl:%rd[0-9]+]], [[P2_6]], 16;
-; CHECK-DAG:    shl.b64      [[P2_7_shl:%rd[0-9]+]], [[P2_7]], 24;
-; CHECK-DAG:    or.b64       [[P2_or_3:%rd[0-9]+]], [[P2_5_shl]], [[P2_4]];
-; CHECK-DAG:    or.b64       [[P2_or_4:%rd[0-9]+]], [[P2_7_shl]], [[P2_6_shl]];
-; CHECK-DAG:    or.b64       [[P2_or_5:%rd[0-9]+]], [[P2_or_4]], [[P2_or_3]];
-; CHECK-DAG:    shl.b64      [[P2_or_shl:%rd[0-9]+]], [[P2_or_5]], 32;
-; CHECK-DAG:    or.b64       [[P2:%rd[0-9]+]], [[P2_or_shl]], [[P2_or_2]];
-; CHECK-DAG:    shr.u64      [[P2_shr_1:%rd[0-9]+]], [[P2]], 8;
-; CHECK-DAG:    shr.u64      [[P2_shr_2:%rd[0-9]+]], [[P2]], 16;
-; CHECK-DAG:    shr.u64      [[P2_shr_3:%rd[0-9]+]], [[P2]], 24;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_4:%rd[0-9]+]], [[P2_or_5]], 8, 24;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_5:%rd[0-9]+]], [[P2_or_5]], 16, 16;
-; CHECK-DAG:    bfe.u64      [[P2_bfe_6:%rd[0-9]+]], [[P2_or_5]], 24, 8;
-; CHECK:        { // callseq
-; CHECK:        .param .align 8 .b8 param0[32];
-; CHECK-DAG:    st.param.b64 [param0],  [[P0]];
-; CHECK-DAG:    st.param.b8  [param0+9],  [[P2]];
-; CHECK-DAG:    st.param.b8  [param0+10], [[P2_shr_1]];
-; CHECK-DAG:    st.param.b8  [param0+11], [[P2_shr_2]];
-; CHECK-DAG:    st.param.b8  [param0+12], [[P2_shr_3]];
-; CHECK-DAG:    st.param.b8  [param0+13], [[P2_or_5]];
-; CHECK-DAG:    st.param.b8  [param0+14], [[P2_bfe_4]];
-; CHECK-DAG:    st.param.b8  [param0+15], [[P2_bfe_5]];
-; CHECK-DAG:    st.param.b8  [param0+16], [[P2_bfe_6]];
-; CHECK:        .param .align 8 .b8 retval0[32];
-; CHECK-NEXT:   call.uni (retval0),
-; CHECK-NEXT:   test_s_i8f64p,
-; CHECK-NEXT:   (
-; CHECK-NEXT:   param0
-; CHECK-NEXT:   );
-; CHECK-DAG:    ld.param.b64 [[R0:%rd[0-9]+]],   [retval0];
-; CHECK-DAG:    ld.param.b8  [[R2_0:%rs[0-9]+]], [retval0+9];
-; CHECK-DAG:    ld.param.b8  [[R2_1:%rs[0-9]+]], [retval0+10];
-; CHECK-DAG:    ld.param.b8  [[R2_2:%rs[0-9]+]], [retval0+11];
-; CHECK-DAG:    ld.param.b8  [[R2_3:%rs[0-9]+]], [retval0+12];
-; CHECK-DAG:    ld.param.b8  [[R2_4:%rs[0-9]+]], [retval0+13];
-; CHECK-DAG:    ld.param.b8  [[R2_5:%rs[0-9]+]], [retval0+14];
-; CHECK-DAG:    ld.param.b8  [[R2_6:%rs[0-9]+]], [retval0+15];
-; CHECK-DAG:    ld.param.b8  [[R2_7:%rs[0-9]+]], [retval0+16];
-; CHECK:        } // callseq
-; CHECK-DAG:    st.param.b64 [func_retval0], [[R0]];
-; CHECK-DAG:    st.param.b8  [func_retval0+9],
-; CHECK-DAG:    st.param.b8  [func_retval0+10],
-; CHECK-DAG:    st.param.b8  [func_retval0+11],
-; CHECK-DAG:    st.param.b8  [func_retval0+12],
-; CHECK-DAG:    st.param.b8  [func_retval0+13],
-; CHECK-DAG:    st.param.b8  [func_retval0+14],
-; CHECK-DAG:    st.param.b8  [func_retval0+15],
-; CHECK-DAG:    st.param.b8  [func_retval0+16],
-; CHECK:        ret;
 
 define %s_i8f64p @test_s_i8f64p(%s_i8f64p %a) {
-       %r = tail call %s_i8f64p @test_s_i8f64p(%s_i8f64p %a)
-       ret %s_i8f64p %r
+; CHECK-LABEL: test_s_i8f64p(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<20>;
+; CHECK-NEXT:    .reg .b64 %rd<68>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b8 %rd4, [test_s_i8f64p_param_0+10];
+; CHECK-NEXT:    shl.b64 %rd5, %rd4, 8;
+; CHECK-NEXT:    ld.param.b8 %rd6, [test_s_i8f64p_param_0+9];
+; CHECK-NEXT:    or.b64 %rd7, %rd5, %rd6;
+; CHECK-NEXT:    ld.param.b8 %rd8, [test_s_i8f64p_param_0+11];
+; CHECK-NEXT:    shl.b64 %rd9, %rd8, 16;
+; CHECK-NEXT:    ld.param.b8 %rd10, [test_s_i8f64p_param_0+12];
+; CHECK-NEXT:    shl.b64 %rd11, %rd10, 24;
+; CHECK-NEXT:    or.b64 %rd12, %rd11, %rd9;
+; CHECK-NEXT:    or.b64 %rd13, %rd12, %rd7;
+; CHECK-NEXT:    ld.param.b8 %rd14, [test_s_i8f64p_param_0+14];
+; CHECK-NEXT:    shl.b64 %rd15, %rd14, 8;
+; CHECK-NEXT:    ld.param.b8 %rd16, [test_s_i8f64p_param_0+13];
+; CHECK-NEXT:    or.b64 %rd17, %rd15, %rd16;
+; CHECK-NEXT:    ld.param.b8 %rd18, [test_s_i8f64p_param_0+15];
+; CHECK-NEXT:    shl.b64 %rd19, %rd18, 16;
+; CHECK-NEXT:    ld.param.b8 %rd20, [test_s_i8f64p_param_0+16];
+; CHECK-NEXT:    shl.b64 %rd21, %rd20, 24;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd19;
+; CHECK-NEXT:    or.b64 %rd23, %rd22, %rd17;
+; CHECK-NEXT:    shl.b64 %rd24, %rd23, 32;
+; CHECK-NEXT:    or.b64 %rd2, %rd24, %rd13;
+; CHECK-NEXT:    ld.param.b64 %rd3, [test_s_i8f64p_param_0+24];
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_s_i8f64p_param_0+8];
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8f64p_param_0];
+; CHECK-NEXT:    shr.u64 %rd25, %rd2, 8;
+; CHECK-NEXT:    shr.u64 %rd26, %rd2, 16;
+; CHECK-NEXT:    shr.u64 %rd27, %rd2, 24;
+; CHECK-NEXT:    bfe.u64 %rd28, %rd23, 8, 24;
+; CHECK-NEXT:    bfe.u64 %rd29, %rd23, 16, 16;
+; CHECK-NEXT:    bfe.u64 %rd30, %rd23, 24, 8;
+; CHECK-NEXT:    { // callseq 6, 0
+; CHECK-NEXT:    .param .align 8 .b8 param0[32];
+; CHECK-NEXT:    st.param.b64 [param0], %rd1;
+; CHECK-NEXT:    st.param.b8 [param0+8], %rs1;
+; CHECK-NEXT:    st.param.b8 [param0+9], %rd2;
+; CHECK-NEXT:    st.param.b8 [param0+10], %rd25;
+; CHECK-NEXT:    st.param.b8 [param0+11], %rd26;
+; CHECK-NEXT:    st.param.b8 [param0+12], %rd27;
+; CHECK-NEXT:    st.param.b8 [param0+13], %rd23;
+; CHECK-NEXT:    st.param.b8 [param0+14], %rd28;
+; CHECK-NEXT:    st.param.b8 [param0+15], %rd29;
+; CHECK-NEXT:    st.param.b8 [param0+16], %rd30;
+; CHECK-NEXT:    st.param.b64 [param0+24], %rd3;
+; CHECK-NEXT:    .param .align 8 .b8 retval0[32];
+; CHECK-NEXT:    call.uni (retval0), test_s_i8f64p, (param0);
+; CHECK-NEXT:    ld.param.b64 %rd31, [retval0];
+; CHECK-NEXT:    ld.param.b8 %rs2, [retval0+8];
+; CHECK-NEXT:    ld.param.b8 %rs3, [retval0+9];
+; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+10];
+; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+11];
+; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+12];
+; CHECK-NEXT:    ld.param.b8 %rs7, [retval0+13];
+; CHECK-NEXT:    ld.param.b8 %rs8, [retval0+14];
+; CHECK-NEXT:    ld.param.b8 %rs9, [retval0+15];
+; CHECK-NEXT:    ld.param.b8 %rs10, [retval0+16];
+; CHECK-NEXT:    ld.param.b64 %rd32, [retval0+24];
+; CHECK-NEXT:    } // callseq 6
+; CHECK-NEXT:    cvt.u64.u16 %rd33, %rs3;
+; CHECK-NEXT:    and.b64 %rd34, %rd33, 255;
+; CHECK-NEXT:    cvt.u64.u16 %rd35, %rs4;
+; CHECK-NEXT:    and.b64 %rd36, %rd35, 255;
+; CHECK-NEXT:    shl.b64 %rd37, %rd36, 8;
+; CHECK-NEXT:    or.b64 %rd38, %rd34, %rd37;
+; CHECK-NEXT:    cvt.u64.u16 %rd39, %rs5;
+; CHECK-NEXT:    and.b64 %rd40, %rd39, 255;
+; CHECK-NEXT:    shl.b64 %rd41, %rd40, 16;
+; CHECK-NEXT:    or.b64 %rd42, %rd38, %rd41;
+; CHECK-NEXT:    cvt.u64.u16 %rd43, %rs6;
+; CHECK-NEXT:    and.b64 %rd44, %rd43, 255;
+; CHECK-NEXT:    shl.b64 %rd45, %rd44, 24;
+; CHECK-NEXT:    or.b64 %rd46, %rd42, %rd45;
+; CHECK-NEXT:    cvt.u64.u16 %rd47, %rs7;
+; CHECK-NEXT:    and.b64 %rd48, %rd47, 255;
+; CHECK-NEXT:    shl.b64 %rd49, %rd48, 32;
+; CHECK-NEXT:    or.b64 %rd50, %rd46, %rd49;
+; CHECK-NEXT:    cvt.u64.u16 %rd51, %rs8;
+; CHECK-NEXT:    and.b64 %rd52, %rd51, 255;
+; CHECK-NEXT:    shl.b64 %rd53, %rd52, 40;
+; CHECK-NEXT:    or.b64 %rd54, %rd50, %rd53;
+; CHECK-NEXT:    cvt.u64.u16 %rd55, %rs9;
+; CHECK-NEXT:    and.b64 %rd56, %rd55, 255;
+; CHECK-NEXT:    shl.b64 %rd57, %rd56, 48;
+; CHECK-NEXT:    or.b64 %rd58, %rd54, %rd57;
+; CHECK-NEXT:    cvt.u64.u16 %rd59, %rs10;
+; CHECK-NEXT:    shl.b64 %rd60, %rd59, 56;
+; CHECK-NEXT:    or.b64 %rd61, %rd58, %rd60;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd31;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs2;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd39;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
+; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd33;
+; CHECK-NEXT:    shr.u64 %rd64, %rd50, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd64;
+; CHECK-NEXT:    shr.u64 %rd65, %rd54, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd65;
+; CHECK-NEXT:    shr.u64 %rd66, %rd58, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd66;
+; CHECK-NEXT:    shr.u64 %rd67, %rd61, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd67;
+; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd32;
+; CHECK-NEXT:    ret;
+  %r = tail call %s_i8f64p @test_s_i8f64p(%s_i8f64p %a)
+  ret %s_i8f64p %r
 }
diff --git a/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll b/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
index bac8bbbf0b4d..c9a1072019dc 100644
--- a/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
+++ b/llvm/test/CodeGen/NVPTX/unfold-masked-merge-vector-variablemask.ll
@@ -61,13 +61,13 @@ define <4 x i8> @out_v4i8(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwind {
 ; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [out_v4i8_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [out_v4i8_param_0];
-; CHECK-NEXT:    ld.param.b32 %r3, [out_v4i8_param_2];
-; CHECK-NEXT:    and.b32 %r4, %r2, %r3;
-; CHECK-NEXT:    xor.b32 %r5, %r3, -1;
-; CHECK-NEXT:    and.b32 %r6, %r1, %r5;
-; CHECK-NEXT:    or.b32 %r7, %r4, %r6;
+; CHECK-NEXT:    ld.param.b32 %r1, [out_v4i8_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [out_v4i8_param_2];
+; CHECK-NEXT:    and.b32 %r3, %r1, %r2;
+; CHECK-NEXT:    ld.param.b32 %r4, [out_v4i8_param_1];
+; CHECK-NEXT:    xor.b32 %r5, %r2, -1;
+; CHECK-NEXT:    and.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    or.b32 %r7, %r3, %r6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
   %mx = and <4 x i8> %x, %mask
@@ -83,13 +83,13 @@ define <4 x i8> @out_v4i8_undef(<4 x i8> %x, <4 x i8> %y, <4 x i8> %mask) nounwi
 ; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [out_v4i8_undef_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [out_v4i8_undef_param_0];
-; CHECK-NEXT:    ld.param.b32 %r3, [out_v4i8_undef_param_2];
-; CHECK-NEXT:    and.b32 %r4, %r2, %r3;
-; CHECK-NEXT:    xor.b32 %r5, %r3, -16711681;
-; CHECK-NEXT:    and.b32 %r6, %r1, %r5;
-; CHECK-NEXT:    or.b32 %r7, %r4, %r6;
+; CHECK-NEXT:    ld.param.b32 %r1, [out_v4i8_undef_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [out_v4i8_undef_param_2];
+; CHECK-NEXT:    and.b32 %r3, %r1, %r2;
+; CHECK-NEXT:    ld.param.b32 %r4, [out_v4i8_undef_param_1];
+; CHECK-NEXT:    xor.b32 %r5, %r2, -16711681;
+; CHECK-NEXT:    and.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    or.b32 %r7, %r3, %r6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
   %mx = and <4 x i8> %x, %mask
@@ -105,13 +105,13 @@ define <2 x i16> @out_v2i16(<2 x i16> %x, <2 x i16> %y, <2 x i16> %mask) nounwin
 ; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [out_v2i16_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [out_v2i16_param_0];
-; CHECK-NEXT:    ld.param.b32 %r3, [out_v2i16_param_2];
-; CHECK-NEXT:    and.b32 %r4, %r2, %r3;
-; CHECK-NEXT:    xor.b32 %r5, %r3, -1;
-; CHECK-NEXT:    and.b32 %r6, %r1, %r5;
-; CHECK-NEXT:    or.b32 %r7, %r4, %r6;
+; CHECK-NEXT:    ld.param.b32 %r1, [out_v2i16_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [out_v2i16_param_2];
+; CHECK-NEXT:    and.b32 %r3, %r1, %r2;
+; CHECK-NEXT:    ld.param.b32 %r4, [out_v2i16_param_1];
+; CHECK-NEXT:    xor.b32 %r5, %r2, -1;
+; CHECK-NEXT:    and.b32 %r6, %r4, %r5;
+; CHECK-NEXT:    or.b32 %r7, %r3, %r6;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
   %mx = and <2 x i16> %x, %mask
@@ -551,14 +551,14 @@ define <8 x i8> @in_v8i8(<8 x i8> %x, <8 x i8> %y, <8 x i8> %mask) nounwind {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [in_v8i8_param_0];
 ; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [in_v8i8_param_1];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [in_v8i8_param_2];
-; CHECK-NEXT:    xor.b32 %r7, %r2, %r4;
-; CHECK-NEXT:    and.b32 %r8, %r7, %r6;
-; CHECK-NEXT:    xor.b32 %r9, %r8, %r4;
-; CHECK-NEXT:    xor.b32 %r10, %r1, %r3;
-; CHECK-NEXT:    and.b32 %r11, %r10, %r5;
-; CHECK-NEXT:    xor.b32 %r12, %r11, %r3;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r9};
+; CHECK-NEXT:    xor.b32 %r5, %r2, %r4;
+; CHECK-NEXT:    xor.b32 %r6, %r1, %r3;
+; CHECK-NEXT:    ld.param.v2.b32 {%r7, %r8}, [in_v8i8_param_2];
+; CHECK-NEXT:    and.b32 %r9, %r6, %r7;
+; CHECK-NEXT:    and.b32 %r10, %r5, %r8;
+; CHECK-NEXT:    xor.b32 %r11, %r10, %r4;
+; CHECK-NEXT:    xor.b32 %r12, %r9, %r3;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r11};
 ; CHECK-NEXT:    ret;
   %n0 = xor <8 x i8> %x, %y
   %n1 = and <8 x i8> %n0, %mask
@@ -574,14 +574,14 @@ define <4 x i16> @in_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i16> %mask) nounwind
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [in_v4i16_param_0];
 ; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [in_v4i16_param_1];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [in_v4i16_param_2];
-; CHECK-NEXT:    xor.b32 %r7, %r2, %r4;
-; CHECK-NEXT:    and.b32 %r8, %r7, %r6;
-; CHECK-NEXT:    xor.b32 %r9, %r8, %r4;
-; CHECK-NEXT:    xor.b32 %r10, %r1, %r3;
-; CHECK-NEXT:    and.b32 %r11, %r10, %r5;
-; CHECK-NEXT:    xor.b32 %r12, %r11, %r3;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r9};
+; CHECK-NEXT:    xor.b32 %r5, %r2, %r4;
+; CHECK-NEXT:    xor.b32 %r6, %r1, %r3;
+; CHECK-NEXT:    ld.param.v2.b32 {%r7, %r8}, [in_v4i16_param_2];
+; CHECK-NEXT:    and.b32 %r9, %r6, %r7;
+; CHECK-NEXT:    and.b32 %r10, %r5, %r8;
+; CHECK-NEXT:    xor.b32 %r11, %r10, %r4;
+; CHECK-NEXT:    xor.b32 %r12, %r9, %r3;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r11};
 ; CHECK-NEXT:    ret;
   %n0 = xor <4 x i16> %x, %y
   %n1 = and <4 x i16> %n0, %mask
diff --git a/llvm/test/CodeGen/NVPTX/unreachable.ll b/llvm/test/CodeGen/NVPTX/unreachable.ll
index 80cf938d48b5..618c7ed0c499 100644
--- a/llvm/test/CodeGen/NVPTX/unreachable.ll
+++ b/llvm/test/CodeGen/NVPTX/unreachable.ll
@@ -28,10 +28,7 @@ define ptx_kernel void @kernel_func() {
 ; NO-TRAP-UNREACHABLE-EMPTY:
 ; NO-TRAP-UNREACHABLE-NEXT:  // %bb.0:
 ; NO-TRAP-UNREACHABLE-NEXT:    { // callseq 0, 0
-; NO-TRAP-UNREACHABLE-NEXT:    call.uni
-; NO-TRAP-UNREACHABLE-NEXT:    throw,
-; NO-TRAP-UNREACHABLE-NEXT:    (
-; NO-TRAP-UNREACHABLE-NEXT:    );
+; NO-TRAP-UNREACHABLE-NEXT:    call.uni throw, ();
 ; NO-TRAP-UNREACHABLE-NEXT:    } // callseq 0
 ; NO-TRAP-UNREACHABLE-NEXT:    // begin inline asm
 ; NO-TRAP-UNREACHABLE-NEXT:    exit;
@@ -43,10 +40,7 @@ define ptx_kernel void @kernel_func() {
 ; NO-TRAP-AFTER-NORETURN-EMPTY:
 ; NO-TRAP-AFTER-NORETURN-NEXT:  // %bb.0:
 ; NO-TRAP-AFTER-NORETURN-NEXT:    { // callseq 0, 0
-; NO-TRAP-AFTER-NORETURN-NEXT:    call.uni
-; NO-TRAP-AFTER-NORETURN-NEXT:    throw,
-; NO-TRAP-AFTER-NORETURN-NEXT:    (
-; NO-TRAP-AFTER-NORETURN-NEXT:    );
+; NO-TRAP-AFTER-NORETURN-NEXT:    call.uni throw, ();
 ; NO-TRAP-AFTER-NORETURN-NEXT:    } // callseq 0
 ; NO-TRAP-AFTER-NORETURN-NEXT:    // begin inline asm
 ; NO-TRAP-AFTER-NORETURN-NEXT:    exit;
@@ -59,10 +53,7 @@ define ptx_kernel void @kernel_func() {
 ; TRAP-EMPTY:
 ; TRAP-NEXT:  // %bb.0:
 ; TRAP-NEXT:    { // callseq 0, 0
-; TRAP-NEXT:    call.uni
-; TRAP-NEXT:    throw,
-; TRAP-NEXT:    (
-; TRAP-NEXT:    );
+; TRAP-NEXT:    call.uni throw, ();
 ; TRAP-NEXT:    } // callseq 0
 ; TRAP-NEXT:    trap; exit;
 ;
@@ -72,10 +63,7 @@ define ptx_kernel void @kernel_func() {
 ; BUG-FIXED-EMPTY:
 ; BUG-FIXED-NEXT:  // %bb.0:
 ; BUG-FIXED-NEXT:    { // callseq 0, 0
-; BUG-FIXED-NEXT:    call.uni
-; BUG-FIXED-NEXT:    throw,
-; BUG-FIXED-NEXT:    (
-; BUG-FIXED-NEXT:    );
+; BUG-FIXED-NEXT:    call.uni throw, ();
 ; BUG-FIXED-NEXT:    } // callseq 0
 ; BUG-FIXED-NEXT:    trap;
   call void @throw()
diff --git a/llvm/test/CodeGen/NVPTX/variadics-backend.ll b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
index ddaa9fd831af..ad2e7044e93b 100644
--- a/llvm/test/CodeGen/NVPTX/variadics-backend.ll
+++ b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
@@ -104,37 +104,28 @@ define dso_local i32 @foo() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot1[40];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<4>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot1;
 ; CHECK-PTX-NEXT:    cvta.local.u64 %SP, %SPL;
-; CHECK-PTX-NEXT:    mov.b64 %rd1, 4294967297;
-; CHECK-PTX-NEXT:    st.b64 [%SP], %rd1;
-; CHECK-PTX-NEXT:    mov.b32 %r1, 1;
-; CHECK-PTX-NEXT:    st.b32 [%SP+8], %r1;
-; CHECK-PTX-NEXT:    mov.b64 %rd2, 1;
-; CHECK-PTX-NEXT:    st.b64 [%SP+16], %rd2;
-; CHECK-PTX-NEXT:    mov.b64 %rd3, 4607182418800017408;
-; CHECK-PTX-NEXT:    st.b64 [%SP+24], %rd3;
-; CHECK-PTX-NEXT:    st.b64 [%SP+32], %rd3;
-; CHECK-PTX-NEXT:    add.u64 %rd4, %SP, 0;
+; CHECK-PTX-NEXT:    st.b64 [%SP], 4294967297;
+; CHECK-PTX-NEXT:    st.b32 [%SP+8], 1;
+; CHECK-PTX-NEXT:    st.b64 [%SP+16], 1;
+; CHECK-PTX-NEXT:    st.b64 [%SP+24], 4607182418800017408;
+; CHECK-PTX-NEXT:    st.b64 [%SP+32], 4607182418800017408;
+; CHECK-PTX-NEXT:    add.u64 %rd1, %SP, 0;
 ; CHECK-PTX-NEXT:    { // callseq 0, 0
 ; CHECK-PTX-NEXT:    .param .b32 param0;
 ; CHECK-PTX-NEXT:    st.param.b32 [param0], 1;
 ; CHECK-PTX-NEXT:    .param .b64 param1;
-; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd4;
+; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd1;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    call.uni (retval0),
-; CHECK-PTX-NEXT:    variadics1,
-; CHECK-PTX-NEXT:    (
-; CHECK-PTX-NEXT:    param0,
-; CHECK-PTX-NEXT:    param1
-; CHECK-PTX-NEXT:    );
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [retval0];
+; CHECK-PTX-NEXT:    call.uni (retval0), variadics1, (param0, param1);
+; CHECK-PTX-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-PTX-NEXT:    } // callseq 0
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %conv = sext i8 1 to i32
@@ -151,7 +142,7 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
 ; CHECK-PTX-NEXT:    .reg .b16 %rs<4>;
-; CHECK-PTX-NEXT:    .reg .b32 %r<7>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<6>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<9>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
@@ -174,8 +165,7 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
 ; CHECK-PTX-NEXT:    add.s32 %r5, %r4, %r3;
 ; CHECK-PTX-NEXT:    cvt.u64.u32 %rd7, %r5;
 ; CHECK-PTX-NEXT:    add.s64 %rd8, %rd7, %rd6;
-; CHECK-PTX-NEXT:    cvt.u32.u64 %r6, %rd8;
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd8;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %vlist = alloca ptr, align 8
@@ -211,9 +201,9 @@ define dso_local i32 @bar() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot3[24];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b16 %rs<5>;
-; CHECK-PTX-NEXT:    .reg .b32 %r<4>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<5>;
+; CHECK-PTX-NEXT:    .reg .b16 %rs<4>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot3;
@@ -225,28 +215,20 @@ define dso_local i32 @bar() {
 ; CHECK-PTX-NEXT:    st.local.b8 [%rd2+1], %rs2;
 ; CHECK-PTX-NEXT:    ld.global.nc.b8 %rs3, [__const_$_bar_$_s1+5];
 ; CHECK-PTX-NEXT:    st.local.b8 [%rd2], %rs3;
-; CHECK-PTX-NEXT:    mov.b32 %r1, 1;
-; CHECK-PTX-NEXT:    st.b32 [%SP+8], %r1;
-; CHECK-PTX-NEXT:    mov.b16 %rs4, 1;
-; CHECK-PTX-NEXT:    st.b8 [%SP+12], %rs4;
-; CHECK-PTX-NEXT:    mov.b64 %rd3, 1;
-; CHECK-PTX-NEXT:    st.b64 [%SP+16], %rd3;
-; CHECK-PTX-NEXT:    add.u64 %rd4, %SP, 8;
+; CHECK-PTX-NEXT:    st.b32 [%SP+8], 1;
+; CHECK-PTX-NEXT:    st.b8 [%SP+12], 1;
+; CHECK-PTX-NEXT:    st.b64 [%SP+16], 1;
+; CHECK-PTX-NEXT:    add.u64 %rd3, %SP, 8;
 ; CHECK-PTX-NEXT:    { // callseq 1, 0
 ; CHECK-PTX-NEXT:    .param .b32 param0;
 ; CHECK-PTX-NEXT:    st.param.b32 [param0], 1;
 ; CHECK-PTX-NEXT:    .param .b64 param1;
-; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd4;
+; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd3;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    call.uni (retval0),
-; CHECK-PTX-NEXT:    variadics2,
-; CHECK-PTX-NEXT:    (
-; CHECK-PTX-NEXT:    param0,
-; CHECK-PTX-NEXT:    param1
-; CHECK-PTX-NEXT:    );
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [retval0];
+; CHECK-PTX-NEXT:    call.uni (retval0), variadics2, (param0, param1);
+; CHECK-PTX-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-PTX-NEXT:    } // callseq 1
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %s1.sroa.3 = alloca [3 x i8], align 1
@@ -300,14 +282,13 @@ define dso_local i32 @baz() {
 ; CHECK-PTX-NEXT:    .local .align 16 .b8 __local_depot5[16];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<4>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot5;
 ; CHECK-PTX-NEXT:    cvta.local.u64 %SP, %SPL;
-; CHECK-PTX-NEXT:    mov.b32 %r1, 1;
-; CHECK-PTX-NEXT:    st.v4.b32 [%SP], {%r1, %r1, %r1, %r1};
+; CHECK-PTX-NEXT:    st.v4.b32 [%SP], {1, 1, 1, 1};
 ; CHECK-PTX-NEXT:    add.u64 %rd1, %SP, 0;
 ; CHECK-PTX-NEXT:    { // callseq 2, 0
 ; CHECK-PTX-NEXT:    .param .b32 param0;
@@ -315,15 +296,10 @@ define dso_local i32 @baz() {
 ; CHECK-PTX-NEXT:    .param .b64 param1;
 ; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd1;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    call.uni (retval0),
-; CHECK-PTX-NEXT:    variadics3,
-; CHECK-PTX-NEXT:    (
-; CHECK-PTX-NEXT:    param0,
-; CHECK-PTX-NEXT:    param1
-; CHECK-PTX-NEXT:    );
-; CHECK-PTX-NEXT:    ld.param.b32 %r2, [retval0];
+; CHECK-PTX-NEXT:    call.uni (retval0), variadics3, (param0, param1);
+; CHECK-PTX-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-PTX-NEXT:    } // callseq 2
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r2;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %call = call i32 (i32, ...) @variadics3(i32 noundef 1, <4 x i32> noundef <i32 1, i32 1, i32 1, i32 1>)
@@ -333,7 +309,6 @@ entry:
 define dso_local i32 @variadics4(ptr noundef byval(%struct.S2) align 8 %first, ...) {
 ; CHECK-PTX-LABEL: variadics4(
 ; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<10>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
@@ -345,8 +320,7 @@ define dso_local i32 @variadics4(ptr noundef byval(%struct.S2) align 8 %first, .
 ; CHECK-PTX-NEXT:    ld.param.b64 %rd7, [variadics4_param_0+8];
 ; CHECK-PTX-NEXT:    add.s64 %rd8, %rd6, %rd7;
 ; CHECK-PTX-NEXT:    add.s64 %rd9, %rd8, %rd5;
-; CHECK-PTX-NEXT:    cvt.u32.u64 %r1, %rd9;
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd9;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %vlist = alloca ptr, align 8
@@ -374,8 +348,8 @@ define dso_local void @qux() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot7[24];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<9>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot7;
@@ -385,24 +359,18 @@ define dso_local void @qux() {
 ; CHECK-PTX-NEXT:    st.local.b64 [%rd2+8], %rd3;
 ; CHECK-PTX-NEXT:    ld.global.nc.b64 %rd4, [__const_$_qux_$_s];
 ; CHECK-PTX-NEXT:    st.local.b64 [%rd2], %rd4;
-; CHECK-PTX-NEXT:    mov.b64 %rd5, 1;
-; CHECK-PTX-NEXT:    st.b64 [%SP+16], %rd5;
-; CHECK-PTX-NEXT:    ld.local.b64 %rd6, [%rd2];
-; CHECK-PTX-NEXT:    ld.local.b64 %rd7, [%rd2+8];
-; CHECK-PTX-NEXT:    add.u64 %rd8, %SP, 16;
+; CHECK-PTX-NEXT:    st.b64 [%SP+16], 1;
+; CHECK-PTX-NEXT:    ld.local.b64 %rd5, [%rd2];
+; CHECK-PTX-NEXT:    ld.local.b64 %rd6, [%rd2+8];
+; CHECK-PTX-NEXT:    add.u64 %rd7, %SP, 16;
 ; CHECK-PTX-NEXT:    { // callseq 3, 0
 ; CHECK-PTX-NEXT:    .param .align 8 .b8 param0[16];
-; CHECK-PTX-NEXT:    st.param.b64 [param0], %rd6;
-; CHECK-PTX-NEXT:    st.param.b64 [param0+8], %rd7;
+; CHECK-PTX-NEXT:    st.param.b64 [param0], %rd5;
+; CHECK-PTX-NEXT:    st.param.b64 [param0+8], %rd6;
 ; CHECK-PTX-NEXT:    .param .b64 param1;
-; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd8;
+; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd7;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    call.uni (retval0),
-; CHECK-PTX-NEXT:    variadics4,
-; CHECK-PTX-NEXT:    (
-; CHECK-PTX-NEXT:    param0,
-; CHECK-PTX-NEXT:    param1
-; CHECK-PTX-NEXT:    );
+; CHECK-PTX-NEXT:    call.uni (retval0), variadics4, (param0, param1);
 ; CHECK-PTX-NEXT:    ld.param.b32 %r1, [retval0];
 ; CHECK-PTX-NEXT:    } // callseq 3
 ; CHECK-PTX-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/vector-loads.ll b/llvm/test/CodeGen/NVPTX/vector-loads.ll
index 88ff59407a14..1ae6f6bcd748 100644
--- a/llvm/test/CodeGen/NVPTX/vector-loads.ll
+++ b/llvm/test/CodeGen/NVPTX/vector-loads.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -mtriple=nvptx64 -mcpu=sm_20 | %ptxas-verify %}
 
@@ -8,28 +9,67 @@
 ;
 ; which will load two floats at once into scalar registers.
 
-; CHECK-LABEL: foo
 define void @foo(ptr %a) {
-; CHECK: ld.v2.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo_param_0];
+; CHECK-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT:    mul.rn.f32 %r3, %r2, %r2;
+; CHECK-NEXT:    mul.rn.f32 %r4, %r1, %r1;
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r4, %r3};
+; CHECK-NEXT:    ret;
   %t1 = load <2 x float>, ptr %a
   %t2 = fmul <2 x float> %t1, %t1
   store <2 x float> %t2, ptr %a
   ret void
 }
 
-; CHECK-LABEL: foo2
 define void @foo2(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo2_param_0];
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NEXT:    mul.rn.f32 %r5, %r4, %r4;
+; CHECK-NEXT:    mul.rn.f32 %r6, %r3, %r3;
+; CHECK-NEXT:    mul.rn.f32 %r7, %r2, %r2;
+; CHECK-NEXT:    mul.rn.f32 %r8, %r1, %r1;
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    ret;
   %t1 = load <4 x float>, ptr %a
   %t2 = fmul <4 x float> %t1, %t1
   store <4 x float> %t2, ptr %a
   ret void
 }
 
-; CHECK-LABEL: foo3
 define void @foo3(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
-; CHECK-NEXT: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo3(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<17>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo3_param_0];
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1+16];
+; CHECK-NEXT:    ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1];
+; CHECK-NEXT:    mul.rn.f32 %r9, %r8, %r8;
+; CHECK-NEXT:    mul.rn.f32 %r10, %r7, %r7;
+; CHECK-NEXT:    mul.rn.f32 %r11, %r6, %r6;
+; CHECK-NEXT:    mul.rn.f32 %r12, %r5, %r5;
+; CHECK-NEXT:    mul.rn.f32 %r13, %r4, %r4;
+; CHECK-NEXT:    mul.rn.f32 %r14, %r3, %r3;
+; CHECK-NEXT:    mul.rn.f32 %r15, %r2, %r2;
+; CHECK-NEXT:    mul.rn.f32 %r16, %r1, %r1;
+; CHECK-NEXT:    st.v4.b32 [%rd1+16], {%r16, %r15, %r14, %r13};
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT:    ret;
   %t1 = load <8 x float>, ptr %a
   %t2 = fmul <8 x float> %t1, %t1
   store <8 x float> %t2, ptr %a
@@ -38,28 +78,67 @@ define void @foo3(ptr %a) {
 
 
 
-; CHECK-LABEL: foo4
 define void @foo4(ptr %a) {
-; CHECK: ld.v2.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo4(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo4_param_0];
+; CHECK-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NEXT:    mul.lo.s32 %r3, %r2, %r2;
+; CHECK-NEXT:    mul.lo.s32 %r4, %r1, %r1;
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r4, %r3};
+; CHECK-NEXT:    ret;
   %t1 = load <2 x i32>, ptr %a
   %t2 = mul <2 x i32> %t1, %t1
   store <2 x i32> %t2, ptr %a
   ret void
 }
 
-; CHECK-LABEL: foo5
 define void @foo5(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo5(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo5_param_0];
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NEXT:    mul.lo.s32 %r5, %r4, %r4;
+; CHECK-NEXT:    mul.lo.s32 %r6, %r3, %r3;
+; CHECK-NEXT:    mul.lo.s32 %r7, %r2, %r2;
+; CHECK-NEXT:    mul.lo.s32 %r8, %r1, %r1;
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    ret;
   %t1 = load <4 x i32>, ptr %a
   %t2 = mul <4 x i32> %t1, %t1
   store <4 x i32> %t2, ptr %a
   ret void
 }
 
-; CHECK-LABEL: foo6
 define void @foo6(ptr %a) {
-; CHECK: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
-; CHECK-NEXT: ld.v4.b32 {%r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}, %r{{[0-9]+}}}
+; CHECK-LABEL: foo6(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<17>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo6_param_0];
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1+16];
+; CHECK-NEXT:    ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1];
+; CHECK-NEXT:    mul.lo.s32 %r9, %r8, %r8;
+; CHECK-NEXT:    mul.lo.s32 %r10, %r7, %r7;
+; CHECK-NEXT:    mul.lo.s32 %r11, %r6, %r6;
+; CHECK-NEXT:    mul.lo.s32 %r12, %r5, %r5;
+; CHECK-NEXT:    mul.lo.s32 %r13, %r4, %r4;
+; CHECK-NEXT:    mul.lo.s32 %r14, %r3, %r3;
+; CHECK-NEXT:    mul.lo.s32 %r15, %r2, %r2;
+; CHECK-NEXT:    mul.lo.s32 %r16, %r1, %r1;
+; CHECK-NEXT:    st.v4.b32 [%rd1+16], {%r16, %r15, %r14, %r13};
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT:    ret;
   %t1 = load <8 x i32>, ptr %a
   %t2 = mul <8 x i32> %t1, %t1
   store <8 x i32> %t2, ptr %a
@@ -70,8 +149,30 @@ define void @foo6(ptr %a) {
 ; computation was still too complex when LSV was called.
 declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() #0
 declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() #0
-; CHECK-LABEL: foo_complex
 define void @foo_complex(ptr nocapture readonly align 16 dereferenceable(134217728) %alloc0) {
+; CHECK-LABEL: foo_complex(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo_complex_param_0];
+; CHECK-NEXT:    mov.u32 %r1, %tid.x;
+; CHECK-NEXT:    mov.u32 %r2, %ctaid.x;
+; CHECK-NEXT:    shr.u32 %r3, %r2, 8;
+; CHECK-NEXT:    shl.b32 %r4, %r2, 9;
+; CHECK-NEXT:    and.b32 %r5, %r4, 130560;
+; CHECK-NEXT:    shl.b32 %r6, %r1, 1;
+; CHECK-NEXT:    or.b32 %r7, %r5, %r6;
+; CHECK-NEXT:    cvt.u64.u32 %rd2, %r7;
+; CHECK-NEXT:    mul.wide.u32 %rd3, %r3, 131072;
+; CHECK-NEXT:    add.s64 %rd4, %rd1, %rd3;
+; CHECK-NEXT:    add.s64 %rd5, %rd4, %rd2;
+; CHECK-NEXT:    ld.v2.b8 {%rs1, %rs2}, [%rd5+128];
+; CHECK-NEXT:    max.u16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.b8 [%rd5+129], %rs3;
+; CHECK-NEXT:    ret;
   %t0 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !1
   %t1 = tail call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
   %t2 = lshr i32 %t1, 8
@@ -86,112 +187,138 @@ define void @foo_complex(ptr nocapture readonly align 16 dereferenceable(1342177
   %t11 = zext i32 %t10 to i64
   %t20 = zext i32 %t2 to i64
   %t27 = getelementptr inbounds [1024 x [131072 x i8]], ptr %alloc0, i64 0, i64 %t20, i64 %t9
-; CHECK: ld.v2.b8
   %t28 = load i8, ptr %t27, align 2
   %t31 = getelementptr inbounds [1024 x [131072 x i8]], ptr %alloc0, i64 0, i64 %t20, i64 %t11
   %t32 = load i8, ptr %t31, align 1
   %t33 = icmp ult i8 %t28, %t32
   %t34 = select i1 %t33, i8 %t32, i8 %t28
   store i8 %t34, ptr %t31
-; CHECK: ret
   ret void
 }
 
-; CHECK-LABEL: extv8f16_global_a16(
 define void @extv8f16_global_a16(ptr addrspace(1) noalias readonly align 16 %dst, ptr addrspace(1) noalias readonly align 16 %src) #0 {
-; CHECK: ld.global.v4.b32 {%r
+; CHECK-LABEL: extv8f16_global_a16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<13>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_global_a16_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_global_a16_param_1];
+; CHECK-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r12, %rs1;
+; CHECK-NEXT:    st.global.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT:    st.global.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    ret;
   %v = load <8 x half>, ptr addrspace(1) %src, align 16
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
   %ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.global.v4.b32
-; CHECK: st.global.v4.b32
   store <8 x float> %ext, ptr addrspace(1) %dst, align 16
   ret void
 }
 
-; CHECK-LABEL: extv8f16_global_a4(
 define void @extv8f16_global_a4(ptr addrspace(1) noalias readonly align 16 %dst, ptr addrspace(1) noalias readonly align 16 %src) #0 {
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
-; CHECK: ld.global.b32 %r
+; CHECK-LABEL: extv8f16_global_a4(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_global_a4_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_global_a4_param_1];
+; CHECK-NEXT:    ld.global.v2.b16 {%rs1, %rs2}, [%rd2+8];
+; CHECK-NEXT:    ld.global.v2.b16 {%rs3, %rs4}, [%rd2+12];
+; CHECK-NEXT:    ld.global.v2.b16 {%rs5, %rs6}, [%rd2];
+; CHECK-NEXT:    ld.global.v2.b16 {%rs7, %rs8}, [%rd2+4];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r4, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs1;
+; CHECK-NEXT:    st.global.v4.b32 [%rd1+16], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    st.global.v4.b32 [%rd1], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT:    ret;
   %v = load <8 x half>, ptr addrspace(1) %src, align 4
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
   %ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.global.v4.b32
-; CHECK: st.global.v4.b32
   store <8 x float> %ext, ptr addrspace(1) %dst, align 16
   ret void
 }
 
 
-; CHECK-LABEL: extv8f16_generic_a16(
 define void @extv8f16_generic_a16(ptr noalias readonly align 16 %dst, ptr noalias readonly align 16 %src) #0 {
-; CHECK: ld.v4.b32 {%r
+; CHECK-LABEL: extv8f16_generic_a16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<13>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_generic_a16_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_generic_a16_param_1];
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r12, %rs1;
+; CHECK-NEXT:    st.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    ret;
   %v = load <8 x half>, ptr %src, align 16
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
   %ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.v4.b32
-; CHECK: st.v4.b32
   store <8 x float> %ext, ptr %dst, align 16
   ret void
 }
 
-; CHECK-LABEL: extv8f16_generic_a4(
 define void @extv8f16_generic_a4(ptr noalias readonly align 16 %dst, ptr noalias readonly align 16 %src) #0 {
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
-; CHECK: ld.b32 %r
+; CHECK-LABEL: extv8f16_generic_a4(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_generic_a4_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_generic_a4_param_1];
+; CHECK-NEXT:    ld.v2.b16 {%rs1, %rs2}, [%rd2+8];
+; CHECK-NEXT:    ld.v2.b16 {%rs3, %rs4}, [%rd2+12];
+; CHECK-NEXT:    ld.v2.b16 {%rs5, %rs6}, [%rd2];
+; CHECK-NEXT:    ld.v2.b16 {%rs7, %rs8}, [%rd2+4];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r3, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r4, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs1;
+; CHECK-NEXT:    st.v4.b32 [%rd1+16], {%r8, %r7, %r6, %r5};
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r4, %r3, %r2, %r1};
+; CHECK-NEXT:    ret;
   %v = load <8 x half>, ptr %src, align 4
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: mov.b32 {%rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
-; CHECK: cvt.f32.f16 %r{{.*}}, %rs
   %ext = fpext <8 x half> %v to <8 x float>
-; CHECK: st.v4.b32
-; CHECK: st.v4.b32
   store <8 x float> %ext, ptr %dst, align 16
   ret void
 }
@@ -199,10 +326,18 @@ define void @extv8f16_generic_a4(ptr noalias readonly align 16 %dst, ptr noalias
 
 !1 = !{i32 0, i32 64}
 
-; CHECK-LABEL: bf16_v4_align_load_store
 define dso_local void @bf16_v4_align_load_store(ptr noundef %0, ptr noundef %1) #0 {
-  ; CHECK: ld.v4.b16
-  ; CHECK: st.v4.b16
+; CHECK-LABEL: bf16_v4_align_load_store(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [bf16_v4_align_load_store_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [bf16_v4_align_load_store_param_1];
+; CHECK-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd2];
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r1, %r2};
+; CHECK-NEXT:    ret;
   %3 = load <4 x bfloat>, ptr %1, align 8
   store <4 x bfloat> %3, ptr %0, align 8
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/vector-returns.ll b/llvm/test/CodeGen/NVPTX/vector-returns.ll
index 2001d199ce0a..472cdee11a4b 100644
--- a/llvm/test/CodeGen/NVPTX/vector-returns.ll
+++ b/llvm/test/CodeGen/NVPTX/vector-returns.ll
@@ -6,12 +6,11 @@ target triple = "nvptx-nvidia-cuda"
 define <3 x i64> @long3() {
 ; CHECK-LABEL: long3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b64 %rd1, 0;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd1};
-; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <3 x i64> zeroinitializer
 }
@@ -19,11 +18,10 @@ define <3 x i64> @long3() {
 define <2 x i64> @long2() {
 ; CHECK-LABEL: long2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b64 %rd1, 0;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd1};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <2 x i64> zeroinitializer
 }
@@ -31,11 +29,10 @@ define <2 x i64> @long2() {
 define <1 x i64> @long1() {
 ; CHECK-LABEL: long1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b64 %rd1, 0;
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <1 x i64> zeroinitializer
 }
@@ -43,12 +40,11 @@ define <1 x i64> @long1() {
 define <5 x i32> @int5() {
 ; CHECK-LABEL: int5(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r1, %r1, %r1};
-; CHECK-NEXT:    st.param.b32 [func_retval0+16], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {0, 0, 0, 0};
 ; CHECK-NEXT:    ret;
   ret <5 x i32> zeroinitializer
 }
@@ -56,11 +52,10 @@ define <5 x i32> @int5() {
 define <4 x i32> @int4() {
 ; CHECK-LABEL: int4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r1, %r1, %r1};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {0, 0, 0, 0};
 ; CHECK-NEXT:    ret;
   ret <4 x i32> zeroinitializer
 }
@@ -68,12 +63,11 @@ define <4 x i32> @int4() {
 define <3 x i32> @int3() {
 ; CHECK-LABEL: int3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r1};
-; CHECK-NEXT:    st.param.b32 [func_retval0+8], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <3 x i32> zeroinitializer
 }
@@ -81,11 +75,10 @@ define <3 x i32> @int3() {
 define <2 x i32> @int2() {
 ; CHECK-LABEL: int2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r1};
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <2 x i32> zeroinitializer
 }
@@ -93,11 +86,10 @@ define <2 x i32> @int2() {
 define <1 x i32> @int1() {
 ; CHECK-LABEL: int1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <1 x i32> zeroinitializer
 }
@@ -105,13 +97,12 @@ define <1 x i32> @int1() {
 define <9 x i16> @short9() {
 ; CHECK-LABEL: short9(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b16 [func_retval0+16], %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0+8], {0, 0};
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <9 x i16> zeroinitializer
 }
@@ -119,11 +110,10 @@ define <9 x i16> @short9() {
 define <8 x i16> @short8() {
 ; CHECK-LABEL: short8(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r1, %r1, %r1};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {0, 0, 0, 0};
 ; CHECK-NEXT:    ret;
   ret <8 x i16> zeroinitializer
 }
@@ -131,13 +121,12 @@ define <8 x i16> @short8() {
 define <7 x i16> @short7() {
 ; CHECK-LABEL: short7(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b16 [func_retval0+8], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b16 [func_retval0+12], %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+12], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <7 x i16> zeroinitializer
 }
@@ -145,12 +134,11 @@ define <7 x i16> @short7() {
 define <5 x i16> @short5() {
 ; CHECK-LABEL: short5(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b16 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b16 [func_retval0+8], %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <5 x i16> zeroinitializer
 }
@@ -158,11 +146,10 @@ define <5 x i16> @short5() {
 define <4 x i16> @short4() {
 ; CHECK-LABEL: short4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r1};
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <4 x i16> zeroinitializer
 }
@@ -170,12 +157,11 @@ define <4 x i16> @short4() {
 define <3 x i16> @short3() {
 ; CHECK-LABEL: short3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v2.b16 [func_retval0], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b16 [func_retval0+4], %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <3 x i16> zeroinitializer
 }
@@ -183,11 +169,10 @@ define <3 x i16> @short3() {
 define <2 x i16> @short2() {
 ; CHECK-LABEL: short2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <2 x i16> zeroinitializer
 }
@@ -195,11 +180,10 @@ define <2 x i16> @short2() {
 define <1 x i16> @short1() {
 ; CHECK-LABEL: short1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b16 [func_retval0], %rs1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <1 x i16> zeroinitializer
 }
@@ -207,15 +191,12 @@ define <1 x i16> @short1() {
 define <17 x i8> @byte17() {
 ; CHECK-LABEL: byte17(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+12], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <17 x i8> zeroinitializer
 }
@@ -223,11 +204,10 @@ define <17 x i8> @byte17() {
 define <16 x i8> @byte16() {
 ; CHECK-LABEL: byte16(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r1, %r1, %r1};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {0, 0, 0, 0};
 ; CHECK-NEXT:    ret;
   ret <16 x i8> zeroinitializer
 }
@@ -235,15 +215,13 @@ define <16 x i8> @byte16() {
 define <15 x i8> @byte15() {
 ; CHECK-LABEL: byte15(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], 0;
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {0, 0};
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <15 x i8> zeroinitializer
 }
@@ -251,13 +229,11 @@ define <15 x i8> @byte15() {
 define <9 x i8> @byte9() {
 ; CHECK-LABEL: byte9(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b64 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <9 x i8> zeroinitializer
 }
@@ -265,11 +241,10 @@ define <9 x i8> @byte9() {
 define <8 x i8> @byte8() {
 ; CHECK-LABEL: byte8(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r1};
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <8 x i8> zeroinitializer
 }
@@ -277,13 +252,12 @@ define <8 x i8> @byte8() {
 define <7 x i8> @byte7() {
 ; CHECK-LABEL: byte7(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], 0;
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {0, 0};
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <7 x i8> zeroinitializer
 }
@@ -291,12 +265,11 @@ define <7 x i8> @byte7() {
 define <5 x i8> @byte5() {
 ; CHECK-LABEL: byte5(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <5 x i8> zeroinitializer
 }
@@ -304,11 +277,10 @@ define <5 x i8> @byte5() {
 define <4 x i8> @byte4() {
 ; CHECK-LABEL: byte4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <4 x i8> zeroinitializer
 }
@@ -316,11 +288,10 @@ define <4 x i8> @byte4() {
 define <3 x i8> @byte3() {
 ; CHECK-LABEL: byte3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <3 x i8> zeroinitializer
 }
@@ -328,11 +299,10 @@ define <3 x i8> @byte3() {
 define <2 x i8> @byte2() {
 ; CHECK-LABEL: byte2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b32 %r1, 0;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <2 x i8> zeroinitializer
 }
@@ -340,11 +310,10 @@ define <2 x i8> @byte2() {
 define <1 x i8> @byte1() {
 ; CHECK-LABEL: byte1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <1 x i8> zeroinitializer
 }
@@ -352,15 +321,14 @@ define <1 x i8> @byte1() {
 define <17 x i1> @bit17() {
 ; CHECK-LABEL: bit17(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+4], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+8], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.v4.b8 [func_retval0+12], {%rs1, %rs1, %rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0+12], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b32 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <17 x i1> zeroinitializer
 }
@@ -368,18 +336,17 @@ define <17 x i1> @bit17() {
 define <16 x i1> @bit16() {
 ; CHECK-LABEL: bit16(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+8], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+10], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+14], {%rs1, %rs1};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+14], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+10], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+8], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <16 x i1> zeroinitializer
 }
@@ -387,18 +354,17 @@ define <16 x i1> @bit16() {
 define <15 x i1> @bit15() {
 ; CHECK-LABEL: bit15(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+8], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+10], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], 0;
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+12], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+10], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+8], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <15 x i1> zeroinitializer
 }
@@ -406,15 +372,14 @@ define <15 x i1> @bit15() {
 define <9 x i1> @bit9() {
 ; CHECK-LABEL: bit9(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {%rs1, %rs1};
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], 0;
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+6], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+4], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0+2], {0, 0};
+; CHECK-NEXT:    st.param.v2.b8 [func_retval0], {0, 0};
 ; CHECK-NEXT:    ret;
   ret <9 x i1> zeroinitializer
 }
@@ -422,18 +387,17 @@ define <9 x i1> @bit9() {
 define <8 x i1> @bit8() {
 ; CHECK-LABEL: bit8(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <8 x i1> zeroinitializer
 }
@@ -441,17 +405,16 @@ define <8 x i1> @bit8() {
 define <7 x i1> @bit7() {
 ; CHECK-LABEL: bit7(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <7 x i1> zeroinitializer
 }
@@ -459,15 +422,14 @@ define <7 x i1> @bit7() {
 define <5 x i1> @bit5() {
 ; CHECK-LABEL: bit5(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <5 x i1> zeroinitializer
 }
@@ -475,14 +437,13 @@ define <5 x i1> @bit5() {
 define <4 x i1> @bit4() {
 ; CHECK-LABEL: bit4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <4 x i1> zeroinitializer
 }
@@ -490,13 +451,12 @@ define <4 x i1> @bit4() {
 define <3 x i1> @bit3() {
 ; CHECK-LABEL: bit3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <3 x i1> zeroinitializer
 }
@@ -504,12 +464,11 @@ define <3 x i1> @bit3() {
 define <2 x i1> @bit2() {
 ; CHECK-LABEL: bit2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], 0;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <2 x i1> zeroinitializer
 }
@@ -517,11 +476,10 @@ define <2 x i1> @bit2() {
 define <1 x i1> @bit1() {
 ; CHECK-LABEL: bit1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    mov.b16 %rs1, 0;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NEXT:    st.param.b8 [func_retval0], 0;
 ; CHECK-NEXT:    ret;
   ret <1 x i1> zeroinitializer
 }
diff --git a/llvm/test/CodeGen/PowerPC/GlobalISel/fconstant-unsupported.ll b/llvm/test/CodeGen/PowerPC/GlobalISel/fconstant-unsupported.ll
index f80928231878..eb4c9e3817d5 100644
--- a/llvm/test/CodeGen/PowerPC/GlobalISel/fconstant-unsupported.ll
+++ b/llvm/test/CodeGen/PowerPC/GlobalISel/fconstant-unsupported.ll
@@ -1,6 +1,6 @@
 ; RUN: not --crash llc -global-isel -mtriple=powerpc-unknown-linux-gnu \
 ; RUN:   -o - < %s 2>&1 | FileCheck %s --check-prefix=BE
-; RUN: not --crash llc -global-isel -mtriple=powerpcle-unknown-linux-gnu \
+; RUN: not llc -global-isel -mtriple=powerpcle-unknown-linux-gnu \
 ; RUN:   -o - < %s 2>&1 | FileCheck %s --check-prefix=BIT32
 
 ; BE: LLVM ERROR: unable to translate in big endian mode
diff --git a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
index dac17dc3225e..b7852c3c3e6e 100644
--- a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -18,7 +18,6 @@ define signext i32 @main() nounwind {
 ; CHECK-NEXT:    sth 3, 46(1)
 ; CHECK-NEXT:    addi 3, 1, 46
 ; CHECK-NEXT:    lharx 4, 0, 3
-; CHECK-NEXT:    clrlwi 4, 4, 16
 ; CHECK-NEXT:    cmplwi 4, 33059
 ; CHECK-NEXT:    bne 0, .LBB0_4
 ; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -32,7 +31,6 @@ define signext i32 @main() nounwind {
 ; CHECK-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 5, 0, 3
-; CHECK-NEXT:    clrlwi 5, 5, 16
 ; CHECK-NEXT:    cmplwi 5, 33059
 ; CHECK-NEXT:    beq 0, .LBB0_2
 ; CHECK-NEXT:  .LBB0_4: # %cmpxchg.nostore
diff --git a/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll b/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
index 04f81124b6d8..9e50455a0b42 100644
--- a/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
+++ b/llvm/test/CodeGen/PowerPC/aix32-vector-vararg-caller-split.ll
@@ -18,13 +18,13 @@ declare <4 x i32> @split_spill(double, double, double, ...)
 ; CHECK-DAG: [[ELEMENT1:%[0-9]+]]:gprc = LWZ 48, $r1 :: (load (s32))
 ; CHECK-DAG: [[ELEMENT2:%[0-9]+]]:gprc = LWZ 52, $r1 :: (load (s32))
 ; CHECK:     [[FLOAT1SPLAT:%[0-9]+]]:vrrc = VSPLTISW 1
-; CHECK:     [[FLOAT1COPY:%[0-9]+]]:vsrc = COPY [[FLOAT1SPLAT]]
+; CHECK:     [[FLOAT1COPY:%[0-9]+]]:vsrc = COPY killed [[FLOAT1SPLAT]]
 ; CHECK:     [[DOUBLE1:%[0-9]+]]:vsrc = XVCVSXWDP killed [[FLOAT1COPY]], implicit $rm
-; CHECK:     [[DOUBLE1COPY:%[0-9]+]]:vsfrc = COPY [[DOUBLE1]]
+; CHECK:     [[DOUBLE1COPY:%[0-9]+]]:vsfrc = COPY killed [[DOUBLE1]]
 ; CHECK:     [[FLOAT2SPLAT:%[0-9]+]]:vrrc = VSPLTISW 2
-; CHECK:     [[FLOAT2COPY:%[0-9]+]]:vsrc = COPY [[FLOAT2SPLAT]]
+; CHECK:     [[FLOAT2COPY:%[0-9]+]]:vsrc = COPY killed [[FLOAT2SPLAT]]
 ; CHECK:     [[DOUBLE2:%[0-9]+]]:vsrc = XVCVSXWDP killed [[FLOAT2COPY]], implicit $rm
-; CHECK:     [[DOUBLE2COPY:%[0-9]+]]:vsfrc = COPY [[DOUBLE2]]
+; CHECK:     [[DOUBLE2COPY:%[0-9]+]]:vsfrc = COPY killed [[DOUBLE2]]
 
 ; CHECK:     [[DZERO:%[0-9]+]]:vsfrc = XXLXORdpz
 ; CHECK:     [[DTOI1:%[0-9]+]]:gprc = LIS 16368
diff --git a/llvm/test/CodeGen/PowerPC/all-atomics.ll b/llvm/test/CodeGen/PowerPC/all-atomics.ll
index 5e14fbbb6ad6..07afea75aec6 100644
--- a/llvm/test/CodeGen/PowerPC/all-atomics.ll
+++ b/llvm/test/CodeGen/PowerPC/all-atomics.ll
@@ -4346,8 +4346,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lbz 7, uc@toc@l(3)
 ; CHECK-NEXT:    lbz 8, sc@toc@l(4)
 ; CHECK-NEXT:    lbarx 5, 0, 6
-; CHECK-NEXT:    clrlwi 9, 5, 24
-; CHECK-NEXT:    cmplw 9, 7
+; CHECK-NEXT:    cmplw 5, 7
 ; CHECK-NEXT:    bne 0, .LBB3_4
 ; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore276
 ; CHECK-NEXT:    sync
@@ -4359,8 +4358,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.3: # %cmpxchg.releasedload274
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lbarx 5, 0, 6
-; CHECK-NEXT:    clrlwi 9, 5, 24
-; CHECK-NEXT:    cmplw 9, 7
+; CHECK-NEXT:    cmplw 5, 7
 ; CHECK-NEXT:    beq 0, .LBB3_2
 ; CHECK-NEXT:  .LBB3_4: # %cmpxchg.nostore272
 ; CHECK-NEXT:    addi 7, 3, uc@toc@l
@@ -4368,8 +4366,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    stb 5, sc@toc@l(4)
 ; CHECK-NEXT:    lbz 9, uc@toc@l(3)
 ; CHECK-NEXT:    lbarx 8, 0, 7
-; CHECK-NEXT:    clrlwi 10, 8, 24
-; CHECK-NEXT:    cmplw 10, 9
+; CHECK-NEXT:    cmplw 8, 9
 ; CHECK-NEXT:    bne 0, .LBB3_8
 ; CHECK-NEXT:  # %bb.5: # %cmpxchg.fencedstore257
 ; CHECK-NEXT:    sync
@@ -4382,8 +4379,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.7: # %cmpxchg.releasedload255
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lbarx 8, 0, 7
-; CHECK-NEXT:    clrlwi 10, 8, 24
-; CHECK-NEXT:    cmplw 10, 9
+; CHECK-NEXT:    cmplw 8, 9
 ; CHECK-NEXT:    beq 0, .LBB3_6
 ; CHECK-NEXT:  .LBB3_8: # %cmpxchg.nostore253
 ; CHECK-NEXT:    addis 5, 2, ss@toc@ha
@@ -4393,8 +4389,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lbz 11, sc@toc@l(4)
 ; CHECK-NEXT:    addi 8, 5, ss@toc@l
 ; CHECK-NEXT:    lharx 9, 0, 8
-; CHECK-NEXT:    clrlwi 12, 9, 16
-; CHECK-NEXT:    cmplw 12, 10
+; CHECK-NEXT:    cmplw 9, 10
 ; CHECK-NEXT:    bne 0, .LBB3_12
 ; CHECK-NEXT:  # %bb.9: # %cmpxchg.fencedstore238
 ; CHECK-NEXT:    extsb 11, 11
@@ -4408,8 +4403,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.11: # %cmpxchg.releasedload236
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 9, 0, 8
-; CHECK-NEXT:    clrlwi 12, 9, 16
-; CHECK-NEXT:    cmplw 12, 10
+; CHECK-NEXT:    cmplw 9, 10
 ; CHECK-NEXT:    beq 0, .LBB3_10
 ; CHECK-NEXT:  .LBB3_12: # %cmpxchg.nostore234
 ; CHECK-NEXT:    lwsync
@@ -4419,8 +4413,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lbz 12, sc@toc@l(4)
 ; CHECK-NEXT:    addi 9, 5, us@toc@l
 ; CHECK-NEXT:    lharx 10, 0, 9
-; CHECK-NEXT:    clrlwi 0, 10, 16
-; CHECK-NEXT:    cmplw 0, 11
+; CHECK-NEXT:    cmplw 10, 11
 ; CHECK-NEXT:    bne 0, .LBB3_16
 ; CHECK-NEXT:  # %bb.13: # %cmpxchg.fencedstore219
 ; CHECK-NEXT:    extsb 12, 12
@@ -4434,8 +4427,7 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.15: # %cmpxchg.releasedload217
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 10, 0, 9
-; CHECK-NEXT:    clrlwi 0, 10, 16
-; CHECK-NEXT:    cmplw 0, 11
+; CHECK-NEXT:    cmplw 10, 11
 ; CHECK-NEXT:    beq 0, .LBB3_14
 ; CHECK-NEXT:  .LBB3_16: # %cmpxchg.nostore215
 ; CHECK-NEXT:    lwsync
@@ -4535,7 +4527,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    lbz 30, uc@toc@l(3)
 ; CHECK-NEXT:    lbz 29, sc@toc@l(4)
 ; CHECK-NEXT:    lbarx 28, 0, 6
-; CHECK-NEXT:    clrlwi 28, 28, 24
 ; CHECK-NEXT:    cmplw 28, 30
 ; CHECK-NEXT:    bne 0, .LBB3_36
 ; CHECK-NEXT:  # %bb.33: # %cmpxchg.fencedstore124
@@ -4548,7 +4539,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.35: # %cmpxchg.releasedload122
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lbarx 28, 0, 6
-; CHECK-NEXT:    clrlwi 28, 28, 24
 ; CHECK-NEXT:    cmplw 28, 30
 ; CHECK-NEXT:    beq 0, .LBB3_34
 ; CHECK-NEXT:  .LBB3_36: # %cmpxchg.nostore120
@@ -4566,7 +4556,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    stw 6, ui@toc@l(5)
 ; CHECK-NEXT:    lbz 6, uc@toc@l(3)
 ; CHECK-NEXT:    lbarx 29, 0, 7
-; CHECK-NEXT:    clrlwi 29, 29, 24
 ; CHECK-NEXT:    cmplw 29, 6
 ; CHECK-NEXT:    bne 0, .LBB3_42
 ; CHECK-NEXT:  # %bb.39: # %cmpxchg.fencedstore105
@@ -4579,7 +4568,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.41: # %cmpxchg.releasedload103
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lbarx 29, 0, 7
-; CHECK-NEXT:    clrlwi 29, 29, 24
 ; CHECK-NEXT:    cmplw 29, 6
 ; CHECK-NEXT:    beq 0, .LBB3_40
 ; CHECK-NEXT:  .LBB3_42: # %cmpxchg.nostore101
@@ -4597,7 +4585,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    stw 6, ui@toc@l(5)
 ; CHECK-NEXT:    lbz 6, uc@toc@l(3)
 ; CHECK-NEXT:    lharx 30, 0, 8
-; CHECK-NEXT:    clrlwi 30, 30, 16
 ; CHECK-NEXT:    cmplw 30, 6
 ; CHECK-NEXT:    bne 0, .LBB3_48
 ; CHECK-NEXT:  # %bb.45: # %cmpxchg.fencedstore86
@@ -4612,7 +4599,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.47: # %cmpxchg.releasedload84
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 30, 0, 8
-; CHECK-NEXT:    clrlwi 30, 30, 16
 ; CHECK-NEXT:    cmplw 30, 6
 ; CHECK-NEXT:    beq 0, .LBB3_46
 ; CHECK-NEXT:  .LBB3_48: # %cmpxchg.nostore82
@@ -4630,7 +4616,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:    stw 6, ui@toc@l(5)
 ; CHECK-NEXT:    lbz 6, uc@toc@l(3)
 ; CHECK-NEXT:    lharx 8, 0, 9
-; CHECK-NEXT:    clrlwi 8, 8, 16
 ; CHECK-NEXT:    cmplw 8, 6
 ; CHECK-NEXT:    bne 0, .LBB3_54
 ; CHECK-NEXT:  # %bb.51: # %cmpxchg.fencedstore67
@@ -4645,7 +4630,6 @@ define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-NEXT:  # %bb.53: # %cmpxchg.releasedload65
 ; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 8, 0, 9
-; CHECK-NEXT:    clrlwi 8, 8, 16
 ; CHECK-NEXT:    cmplw 8, 6
 ; CHECK-NEXT:    beq 0, .LBB3_52
 ; CHECK-NEXT:  .LBB3_54: # %cmpxchg.nostore63
diff --git a/llvm/test/CodeGen/PowerPC/anyext_srl.ll b/llvm/test/CodeGen/PowerPC/anyext_srl.ll
index 49b13d70150d..f92e622601b0 100644
--- a/llvm/test/CodeGen/PowerPC/anyext_srl.ll
+++ b/llvm/test/CodeGen/PowerPC/anyext_srl.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:     -mcpu=pwr8 < %s | FileCheck %s
 
@@ -6,6 +7,13 @@
 
 ; Function Attrs: norecurse nounwind readonly
 define zeroext i1 @foo(ptr %s_a, ptr %s_b) local_unnamed_addr {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lbz 3, 0(3)
+; CHECK-NEXT:    lbz 4, 0(4)
+; CHECK-NEXT:    xor 3, 3, 4
+; CHECK-NEXT:    rldicl 3, 3, 61, 63
+; CHECK-NEXT:    blr
 entry:
   %0 = load i32, ptr %s_a, align 8, !tbaa !1
   %and.i = and i32 %0, 8
@@ -15,9 +23,6 @@ entry:
   %cmp.i5 = icmp ne i32 %and.i4, 0
   %cmp = xor i1 %cmp.i, %cmp.i5
   ret i1 %cmp
-; CHECK-LABEL: @foo
-; CHECK: rldicl  {{[0-9]+}}, {{[0-9]+}}, 61, 63
-
 }
 
 !1 = !{!2, !2, i64 0}
diff --git a/llvm/test/CodeGen/PowerPC/atomics-regression.ll b/llvm/test/CodeGen/PowerPC/atomics-regression.ll
index 280c4299c30b..0474a479a1fe 100644
--- a/llvm/test/CodeGen/PowerPC/atomics-regression.ll
+++ b/llvm/test/CodeGen/PowerPC/atomics-regression.ll
@@ -406,7 +406,6 @@ define void @test40(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB40_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -428,7 +427,6 @@ define void @test41(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB41_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -451,7 +449,6 @@ define void @test42(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB42_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB42_3
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -470,7 +467,6 @@ define void @test43(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -484,7 +480,6 @@ define void @test43(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB43_2
 ; PPC64LE-NEXT:    blr
@@ -497,7 +492,6 @@ define void @test44(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB44_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -511,7 +505,6 @@ define void @test44(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB44_2
 ; PPC64LE-NEXT:  .LBB44_4: # %cmpxchg.nostore
@@ -526,7 +519,6 @@ define void @test45(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -540,7 +532,6 @@ define void @test45(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB45_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -557,7 +548,6 @@ define void @test46(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB46_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -571,7 +561,6 @@ define void @test46(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB46_2
 ; PPC64LE-NEXT:  .LBB46_4: # %cmpxchg.nostore
@@ -586,7 +575,6 @@ define void @test47(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -600,7 +588,6 @@ define void @test47(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB47_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -617,7 +604,6 @@ define void @test48(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB48_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -631,7 +617,6 @@ define void @test48(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB48_2
 ; PPC64LE-NEXT:  .LBB48_4: # %cmpxchg.nostore
@@ -646,7 +631,6 @@ define void @test49(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB49_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -660,7 +644,6 @@ define void @test49(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB49_2
 ; PPC64LE-NEXT:  .LBB49_4: # %cmpxchg.nostore
@@ -679,7 +662,6 @@ define void @test50(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB50_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -701,7 +683,6 @@ define void @test51(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB51_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -724,7 +705,6 @@ define void @test52(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB52_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB52_3
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -743,7 +723,6 @@ define void @test53(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -757,7 +736,6 @@ define void @test53(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB53_2
 ; PPC64LE-NEXT:    blr
@@ -770,7 +748,6 @@ define void @test54(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB54_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -784,7 +761,6 @@ define void @test54(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB54_2
 ; PPC64LE-NEXT:  .LBB54_4: # %cmpxchg.nostore
@@ -799,7 +775,6 @@ define void @test55(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -813,7 +788,6 @@ define void @test55(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB55_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -830,7 +804,6 @@ define void @test56(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB56_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -844,7 +817,6 @@ define void @test56(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB56_2
 ; PPC64LE-NEXT:  .LBB56_4: # %cmpxchg.nostore
@@ -859,7 +831,6 @@ define void @test57(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -873,7 +844,6 @@ define void @test57(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB57_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -890,7 +860,6 @@ define void @test58(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB58_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -904,7 +873,6 @@ define void @test58(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB58_2
 ; PPC64LE-NEXT:  .LBB58_4: # %cmpxchg.nostore
@@ -919,7 +887,6 @@ define void @test59(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB59_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -933,7 +900,6 @@ define void @test59(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB59_2
 ; PPC64LE-NEXT:  .LBB59_4: # %cmpxchg.nostore
@@ -1424,7 +1390,6 @@ define void @test80(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB80_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1446,7 +1411,6 @@ define void @test81(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB81_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1469,7 +1433,6 @@ define void @test82(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB82_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB82_3
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1488,7 +1451,6 @@ define void @test83(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1502,7 +1464,6 @@ define void @test83(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB83_2
 ; PPC64LE-NEXT:    blr
@@ -1515,7 +1476,6 @@ define void @test84(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB84_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1529,7 +1489,6 @@ define void @test84(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB84_2
 ; PPC64LE-NEXT:  .LBB84_4: # %cmpxchg.nostore
@@ -1544,7 +1503,6 @@ define void @test85(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1558,7 +1516,6 @@ define void @test85(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB85_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -1575,7 +1532,6 @@ define void @test86(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB86_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1589,7 +1545,6 @@ define void @test86(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB86_2
 ; PPC64LE-NEXT:  .LBB86_4: # %cmpxchg.nostore
@@ -1604,7 +1559,6 @@ define void @test87(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1618,7 +1572,6 @@ define void @test87(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB87_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -1635,7 +1588,6 @@ define void @test88(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB88_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1649,7 +1601,6 @@ define void @test88(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB88_2
 ; PPC64LE-NEXT:  .LBB88_4: # %cmpxchg.nostore
@@ -1664,7 +1615,6 @@ define void @test89(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB89_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1678,7 +1628,6 @@ define void @test89(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB89_2
 ; PPC64LE-NEXT:  .LBB89_4: # %cmpxchg.nostore
@@ -1697,7 +1646,6 @@ define void @test90(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB90_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1719,7 +1667,6 @@ define void @test91(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB91_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1742,7 +1689,6 @@ define void @test92(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  .LBB92_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB92_3
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
@@ -1761,7 +1707,6 @@ define void @test93(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1775,7 +1720,6 @@ define void @test93(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB93_2
 ; PPC64LE-NEXT:    blr
@@ -1788,7 +1732,6 @@ define void @test94(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB94_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1802,7 +1745,6 @@ define void @test94(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB94_2
 ; PPC64LE-NEXT:  .LBB94_4: # %cmpxchg.nostore
@@ -1817,7 +1759,6 @@ define void @test95(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1831,7 +1772,6 @@ define void @test95(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB95_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -1848,7 +1788,6 @@ define void @test96(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB96_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1862,7 +1801,6 @@ define void @test96(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB96_2
 ; PPC64LE-NEXT:  .LBB96_4: # %cmpxchg.nostore
@@ -1877,7 +1815,6 @@ define void @test97(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1891,7 +1828,6 @@ define void @test97(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB97_2
 ; PPC64LE-NEXT:  # %bb.4: # %cmpxchg.end
@@ -1908,7 +1844,6 @@ define void @test98(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB98_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1922,7 +1857,6 @@ define void @test98(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB98_2
 ; PPC64LE-NEXT:  .LBB98_4: # %cmpxchg.nostore
@@ -1937,7 +1871,6 @@ define void @test99(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE:       # %bb.0: # %cmpxchg.start
 ; PPC64LE-NEXT:    lharx 6, 0, 3
 ; PPC64LE-NEXT:    clrlwi 4, 4, 16
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bne 0, .LBB99_4
 ; PPC64LE-NEXT:  # %bb.1: # %cmpxchg.fencedstore
@@ -1951,7 +1884,6 @@ define void @test99(ptr %ptr, i16 %cmp, i16 %val) {
 ; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.releasedload
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lharx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 16
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    beq 0, .LBB99_2
 ; PPC64LE-NEXT:  .LBB99_4: # %cmpxchg.nostore
diff --git a/llvm/test/CodeGen/PowerPC/builtins-bcd-transform.ll b/llvm/test/CodeGen/PowerPC/builtins-bcd-transform.ll
new file mode 100644
index 000000000000..449beeb18c2d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/builtins-bcd-transform.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Testfile that verifies positive case (0 or 1 only) for BCD builtins national2packed, packed2zoned and zoned2packed.
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names  < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names  < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names  < %s | FileCheck %s
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names  < %s | FileCheck %s
+
+declare <16 x i8> @llvm.ppc.national2packed(<16 x i8>, i32 immarg)
+
+define <16 x i8> @tBcd_National2packed_imm0(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_National2packed_imm0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdcfn. v2, v2, 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.national2packed(<16 x i8> %a, i32 0)
+  ret <16 x i8> %0
+}
+
+define <16 x i8> @tBcd_National2packed_imm1(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_National2packed_imm1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdcfn. v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.national2packed(<16 x i8> %a, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.ppc.packed2national(<16 x i8>)
+
+define <16 x i8> @tBcd_Packed2national(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_Packed2national:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdctn. v2, v2
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.packed2national(<16 x i8> %a)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.ppc.packed2zoned(<16 x i8>, i32 immarg)
+
+define <16 x i8> @tBcd_Packed2zoned_imm0(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_Packed2zoned_imm0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdctz. v2, v2, 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.packed2zoned(<16 x i8> %a, i32 0)
+  ret <16 x i8> %0
+}
+
+define <16 x i8> @tBcd_Packed2zoned_imm1(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_Packed2zoned_imm1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdctz. v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.packed2zoned(<16 x i8> %a, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.ppc.zoned2packed(<16 x i8>, i32 immarg)
+
+define <16 x i8> @tBcd_Zoned2packed_imm0(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_Zoned2packed_imm0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdcfz. v2, v2, 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.zoned2packed(<16 x i8> %a, i32 0)
+  ret <16 x i8> %0
+}
+
+define <16 x i8> @tBcd_Zoned2packed_imm1(<16 x i8> %a) {
+; CHECK-LABEL: tBcd_Zoned2packed_imm1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdcfz. v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = call <16 x i8> @llvm.ppc.zoned2packed(<16 x i8> %a, i32 1)
+  ret <16 x i8> %0
+}
diff --git a/llvm/test/CodeGen/PowerPC/crreduce-reg.mir b/llvm/test/CodeGen/PowerPC/crreduce-reg.mir
new file mode 100644
index 000000000000..1b6045d82f9d
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/crreduce-reg.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=powerpc64-unknown-linux-gnu -run-pass=ppc-reduce-cr-ops \
+# RUN:   -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name:            subreg_folding_regression
+tracksRegLiveness: true
+isSSA:           true
+body:             |
+  ; CHECK-LABEL: name: subreg_folding_regression
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+  ; CHECK-NEXT:   [[LD:%[0-9]+]]:g8rc = LD 0, [[COPY]] :: (load (s64))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x20000000), %bb.3(0x60000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[PHI:%[0-9]+]]:g8rc_and_g8rc_nox0 = PHI [[LD]], %bb.0, %3, %bb.3, %4, %bb.2
+  ; CHECK-NEXT:   [[LBZ:%[0-9]+]]:gprc = LBZ 0, [[PHI]] :: (load (s8))
+  ; CHECK-NEXT:   [[CMPWI:%[0-9]+]]:crrc = CMPWI killed [[LBZ]], 0
+  ; CHECK-NEXT:   [[ADDI8_:%[0-9]+]]:g8rc = nuw ADDI8 [[PHI]], 1
+  ; CHECK-NEXT:   STD [[ADDI8_]], 0, [[COPY]] :: (store (s64))
+  ; CHECK-NEXT:   [[LBZ1:%[0-9]+]]:gprc = LBZ 1, [[PHI]] :: (load (s8))
+  ; CHECK-NEXT:   BCn [[CMPWI]].sub_lt, %bb.2
+  ; CHECK-NEXT:   B %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.1(0x55555555), %bb.2(0x2aaaaaab)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[CMPWI1:%[0-9]+]]:crrc = CMPWI killed [[LBZ1]], 0
+  ; CHECK-NEXT:   BC killed [[CMPWI1]].sub_eq, %bb.1
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI8_1:%[0-9]+]]:g8rc = nuw ADDI8 [[PHI]], 2
+  ; CHECK-NEXT:   STD [[ADDI8_1]], 0, [[COPY]] :: (store (s64))
+  ; CHECK-NEXT:   B %bb.1
+  bb.0:
+    liveins: $x3
+
+    %0:g8rc_and_g8rc_nox0 = COPY $x3
+    %1:g8rc = LD 0, %0 :: (load (s64))
+
+  bb.1:
+    %2:g8rc_and_g8rc_nox0 = PHI %1, %bb.0, %3, %bb.1, %4, %bb.2
+    %5:gprc = LBZ 0, %2 :: (load (s8))
+    %6:crrc = CMPWI killed %5, 0
+    %3:g8rc = nuw ADDI8 %2, 1
+    STD %3, 0, %0 :: (store (s64))
+    %7:gprc = LBZ 1, %2 :: (load (s8))
+    %8:crrc = CMPWI killed %7, 0
+    %9:crbitrc = CRAND %8.sub_eq, %6.sub_lt
+    BC killed %9, %bb.1
+    B %bb.2
+
+  bb.2:
+    %4:g8rc = nuw ADDI8 %2, 2
+    STD %4, 0, %0 :: (store (s64))
+    B %bb.1
+
+...
diff --git a/llvm/test/CodeGen/PowerPC/fma-combine.ll b/llvm/test/CodeGen/PowerPC/fma-combine.ll
index 3d45e9a3a509..456f85ad3eef 100644
--- a/llvm/test/CodeGen/PowerPC/fma-combine.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-combine.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
-; RUN:     -enable-unsafe-fp-math  < %s | FileCheck -check-prefix=CHECK-FAST %s
+; RUN:       < %s | FileCheck -check-prefix=CHECK-FAST %s
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -enable-no-signed-zeros-fp-math \
-; RUN:     -enable-unsafe-fp-math -mattr=-vsx < %s | FileCheck -check-prefix=CHECK-FAST-NOVSX %s
+; RUN:      -mattr=-vsx < %s | FileCheck -check-prefix=CHECK-FAST-NOVSX %s
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
 
 define dso_local double @fma_combine1(double %a, double %b, double %c) {
@@ -19,13 +19,12 @@ define dso_local double @fma_combine1(double %a, double %b, double %c) {
 ; CHECK-LABEL: fma_combine1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    blr
 entry:
   %fneg1 = fneg double %c
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -43,13 +42,12 @@ define dso_local double @fma_combine2(double %a, double %b, double %c) {
 ; CHECK-LABEL: fma_combine2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 2, 0
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 2, 0
 ; CHECK-NEXT:    blr
 entry:
   %fneg1 = fneg double %c
-  %mul = fmul double %b, %fneg1
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %b, %fneg1
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -85,17 +83,16 @@ define dso_local double @fma_combine_two_uses(double %a, double %b, double %c) {
 ; CHECK-NEXT:    stfd 0, v@toc@l(3)
 ; CHECK-NEXT:    xsnegdp 0, 3
 ; CHECK-NEXT:    addis 3, 2, z@toc@ha
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    stfd 0, z@toc@l(3)
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
 ; CHECK-NEXT:    blr
 entry:
   %fneg = fneg double %a
   store double %fneg, ptr @v, align 8
   %fneg1 = fneg double %c
   store double %fneg1, ptr @z, align 8
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -122,15 +119,14 @@ define dso_local double @fma_combine_one_use(double %a, double %b, double %c) {
 ; CHECK-NEXT:    addis 3, 2, v@toc@ha
 ; CHECK-NEXT:    stfd 0, v@toc@l(3)
 ; CHECK-NEXT:    xsnegdp 0, 3
-; CHECK-NEXT:    xsmuldp 0, 0, 2
-; CHECK-NEXT:    xssubdp 1, 0, 1
+; CHECK-NEXT:    xsmsubadp 1, 0, 2
 ; CHECK-NEXT:    blr
 entry:
   %fneg = fneg double %a
   store double %fneg, ptr @v, align 8
   %fneg1 = fneg double %c
-  %mul = fmul double %fneg1, %b
-  %add = fsub double %mul, %a
+  %mul = fmul contract double %fneg1, %b
+  %add = fsub contract double %mul, %a
   ret double %add
 }
 
@@ -327,15 +323,12 @@ define dso_local double @fma_combine_const(double %a, double %b) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis 3, 2, .LCPI9_0@toc@ha
 ; CHECK-NEXT:    lfd 0, .LCPI9_0@toc@l(3)
-; CHECK-NEXT:    addis 3, 2, .LCPI9_1@toc@ha
-; CHECK-NEXT:    xsmuldp 0, 1, 0
-; CHECK-NEXT:    lfd 1, .LCPI9_1@toc@l(3)
-; CHECK-NEXT:    xsmaddadp 2, 0, 1
+; CHECK-NEXT:    xsmaddadp 2, 1, 0
 ; CHECK-NEXT:    fmr 1, 2
 ; CHECK-NEXT:    blr
 entry:
-  %0 = fmul double %a, 1.1
-  %1 = call contract double @llvm.fma.f64(double %0, double 2.1, double %b)
+  %0 = fmul reassoc double %a, 1.1
+  %1 = call contract reassoc double @llvm.fma.f64(double %0, double 2.1, double %b)
   ret double %1
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
index 4e72a5ac5ede..e71f59c79ce4 100644
--- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
+++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
@@ -2,8 +2,8 @@
 ; REQUIRES: asserts
 ; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1                        | FileCheck %s --check-prefix=FMFDEBUG
 ; RUN: llc < %s -mtriple=powerpc64le                                                           | FileCheck %s --check-prefix=FMF
-; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1 -enable-unsafe-fp-math -enable-no-nans-fp-math | FileCheck %s --check-prefix=GLOBALDEBUG
-; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math | FileCheck %s --check-prefix=GLOBAL
+; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1 -enable-unsafe-fp-math -fp-contract=fast -enable-no-nans-fp-math | FileCheck %s --check-prefix=GLOBALDEBUG
+; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -fp-contract=fast -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math | FileCheck %s --check-prefix=GLOBAL
 
 ; Test FP transforms using instruction/node-level fast-math-flags.
 ; We're also checking debug output to verify that FMF is propagated to the newly created nodes.
diff --git a/llvm/test/CodeGen/PowerPC/fp-branch.ll b/llvm/test/CodeGen/PowerPC/fp-branch.ll
index ba8052d93cb9..4b67c4d31991 100644
--- a/llvm/test/CodeGen/PowerPC/fp-branch.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-branch.ll
@@ -1,8 +1,21 @@
-; RUN: llc -verify-machineinstrs < %s -mattr=-vsx -mtriple=ppc32-- | grep fcmp | count 1
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs < %s -mattr=-vsx -mtriple=ppc32-- | FileCheck %s
 
 declare i1 @llvm.isunordered.f64(double, double)
 
 define i1 @intcoord_cond_next55(double %tmp48.reload) {
+; CHECK-LABEL: intcoord_cond_next55:
+; CHECK:       # %bb.0: # %newFuncRoot
+; CHECK-NEXT:    lis 3, .LCPI0_0@ha
+; CHECK-NEXT:    lfs 0, .LCPI0_0@l(3)
+; CHECK-NEXT:    fcmpu 0, 1, 0
+; CHECK-NEXT:    blt 0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %bb72.exitStub
+; CHECK-NEXT:    li 3, 1
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB0_2: # %cond_next62.exitStub
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    blr
 newFuncRoot:
         br label %cond_next55
 
diff --git a/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll b/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
index c6e6737211c2..55482a0c5ff2 100644
--- a/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
+++ b/llvm/test/CodeGen/PowerPC/licm-xxsplti.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ;; Test hoisting `xxspltib` out of the loop.
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff --mcpu=pwr10 \
@@ -10,27 +11,166 @@
 ; RUN:   %s -o - 2>&1 | FileCheck --check-prefix=LINUX64LE %s
 
 define void @_Z3fooPfS_Pi(ptr noalias nocapture noundef %_a, ptr noalias nocapture %In_a, ptr noalias nocapture %n) {
+; AIX64-LABEL: _Z3fooPfS_Pi:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    lwz 5, 0(5)
+; AIX64-NEXT:    cmpwi 5, 1
+; AIX64-NEXT:    bltlr 0
+; AIX64-NEXT:  # %bb.1: # %for.body.preheader
+; AIX64-NEXT:    li 6, 0
+; AIX64-NEXT:    cmplwi 5, 1
+; AIX64-NEXT:    beq 0, L..BB0_4
+; AIX64-NEXT:  # %bb.2: # %for.body.preheader.new
+; AIX64-NEXT:    rlwinm 6, 5, 0, 1, 30
+; AIX64-NEXT:    xxspltib 0, 6
+; AIX64-NEXT:    addi 9, 4, -8
+; AIX64-NEXT:    addi 7, 3, -8
+; AIX64-NEXT:    li 8, 8
+; AIX64-NEXT:    li 10, 12
+; AIX64-NEXT:    li 11, 4
+; AIX64-NEXT:    addi 6, 6, -2
+; AIX64-NEXT:    rldicl 6, 6, 63, 1
+; AIX64-NEXT:    addi 6, 6, 1
+; AIX64-NEXT:    mtctr 6
+; AIX64-NEXT:    li 6, 0
+; AIX64-NEXT:    .align 4
+; AIX64-NEXT:  L..BB0_3: # %for.body
+; AIX64-NEXT:    #
+; AIX64-NEXT:    lxvwsx 1, 9, 8
+; AIX64-NEXT:    addi 6, 6, 2
+; AIX64-NEXT:    xxland 1, 1, 0
+; AIX64-NEXT:    xscvspdpn 1, 1
+; AIX64-NEXT:    stfsu 1, 8(7)
+; AIX64-NEXT:    lxvwsx 1, 9, 10
+; AIX64-NEXT:    addi 9, 9, 8
+; AIX64-NEXT:    xxland 1, 1, 0
+; AIX64-NEXT:    xxsldwi 1, 1, 1, 3
+; AIX64-NEXT:    stfiwx 1, 7, 11
+; AIX64-NEXT:    bdnz L..BB0_3
+; AIX64-NEXT:  L..BB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; AIX64-NEXT:    andi. 5, 5, 1
+; AIX64-NEXT:    bclr 4, 1, 0
+; AIX64-NEXT:  # %bb.5: # %for.body.epil
+; AIX64-NEXT:    sldi 5, 6, 2
+; AIX64-NEXT:    xxspltib 1, 6
+; AIX64-NEXT:    lxvwsx 0, 4, 5
+; AIX64-NEXT:    xxland 0, 0, 1
+; AIX64-NEXT:    xxsldwi 0, 0, 0, 3
+; AIX64-NEXT:    stfiwx 0, 3, 5
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: _Z3fooPfS_Pi:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    lwz 5, 0(5)
+; AIX32-NEXT:    cmpwi 5, 1
+; AIX32-NEXT:    bltlr 0
+; AIX32-NEXT:  # %bb.1: # %for.body.preheader
+; AIX32-NEXT:    li 6, 0
+; AIX32-NEXT:    beq 0, L..BB0_4
+; AIX32-NEXT:  # %bb.2: # %for.body.preheader.new
+; AIX32-NEXT:    xxspltib 0, 6
+; AIX32-NEXT:    addi 12, 4, -8
+; AIX32-NEXT:    addi 9, 3, -8
+; AIX32-NEXT:    rlwinm 7, 5, 0, 1, 30
+; AIX32-NEXT:    li 8, 0
+; AIX32-NEXT:    li 10, 8
+; AIX32-NEXT:    li 11, 12
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB0_3: # %for.body
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lxvwsx 1, 12, 10
+; AIX32-NEXT:    addic 6, 6, 2
+; AIX32-NEXT:    addze 8, 8
+; AIX32-NEXT:    xor 0, 6, 7
+; AIX32-NEXT:    or. 0, 0, 8
+; AIX32-NEXT:    xxland 1, 1, 0
+; AIX32-NEXT:    xscvspdpn 1, 1
+; AIX32-NEXT:    stfsu 1, 8(9)
+; AIX32-NEXT:    lxvwsx 1, 12, 11
+; AIX32-NEXT:    addi 12, 12, 8
+; AIX32-NEXT:    xxland 1, 1, 0
+; AIX32-NEXT:    xscvspdpn 1, 1
+; AIX32-NEXT:    stfs 1, 4(9)
+; AIX32-NEXT:    bne 0, L..BB0_3
+; AIX32-NEXT:  L..BB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; AIX32-NEXT:    andi. 5, 5, 1
+; AIX32-NEXT:    bclr 4, 1, 0
+; AIX32-NEXT:  # %bb.5: # %for.body.epil
+; AIX32-NEXT:    slwi 5, 6, 2
+; AIX32-NEXT:    xxspltib 1, 6
+; AIX32-NEXT:    lxvwsx 0, 4, 5
+; AIX32-NEXT:    xxland 0, 0, 1
+; AIX32-NEXT:    xscvspdpn 0, 0
+; AIX32-NEXT:    stfsx 0, 3, 5
+; AIX32-NEXT:    blr
+;
+; LINUX64LE-LABEL: _Z3fooPfS_Pi:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    lwz 5, 0(5)
+; LINUX64LE-NEXT:    cmpwi 5, 1
+; LINUX64LE-NEXT:    bltlr 0
+; LINUX64LE-NEXT:  # %bb.1: # %for.body.preheader
+; LINUX64LE-NEXT:    li 6, 0
+; LINUX64LE-NEXT:    cmplwi 5, 1
+; LINUX64LE-NEXT:    beq 0, .LBB0_4
+; LINUX64LE-NEXT:  # %bb.2: # %for.body.preheader.new
+; LINUX64LE-NEXT:    rlwinm 6, 5, 0, 1, 30
+; LINUX64LE-NEXT:    xxspltib 0, 6
+; LINUX64LE-NEXT:    addi 8, 4, -8
+; LINUX64LE-NEXT:    addi 7, 3, -8
+; LINUX64LE-NEXT:    li 9, 8
+; LINUX64LE-NEXT:    li 10, 12
+; LINUX64LE-NEXT:    li 11, 4
+; LINUX64LE-NEXT:    addi 6, 6, -2
+; LINUX64LE-NEXT:    rldicl 6, 6, 63, 1
+; LINUX64LE-NEXT:    addi 6, 6, 1
+; LINUX64LE-NEXT:    mtctr 6
+; LINUX64LE-NEXT:    li 6, 0
+; LINUX64LE-NEXT:    .p2align 4
+; LINUX64LE-NEXT:  .LBB0_3: # %for.body
+; LINUX64LE-NEXT:    #
+; LINUX64LE-NEXT:    lxvwsx 1, 8, 9
+; LINUX64LE-NEXT:    addi 6, 6, 2
+; LINUX64LE-NEXT:    xxland 1, 1, 0
+; LINUX64LE-NEXT:    xxsldwi 1, 1, 1, 3
+; LINUX64LE-NEXT:    xscvspdpn 1, 1
+; LINUX64LE-NEXT:    stfsu 1, 8(7)
+; LINUX64LE-NEXT:    lxvwsx 1, 8, 10
+; LINUX64LE-NEXT:    addi 8, 8, 8
+; LINUX64LE-NEXT:    xxland 1, 1, 0
+; LINUX64LE-NEXT:    stxvrwx 1, 7, 11
+; LINUX64LE-NEXT:    bdnz .LBB0_3
+; LINUX64LE-NEXT:  .LBB0_4: # %for.cond.cleanup.loopexit.unr-lcssa
+; LINUX64LE-NEXT:    andi. 5, 5, 1
+; LINUX64LE-NEXT:    bclr 4, 1, 0
+; LINUX64LE-NEXT:  # %bb.5: # %for.body.epil
+; LINUX64LE-NEXT:    sldi 5, 6, 2
+; LINUX64LE-NEXT:    xxspltib 1, 6
+; LINUX64LE-NEXT:    lxvwsx 0, 4, 5
+; LINUX64LE-NEXT:    xxland 0, 0, 1
+; LINUX64LE-NEXT:    stxvrwx 0, 3, 5
+; LINUX64LE-NEXT:    blr
 entry:
   %0 = load i32, ptr %n, align 4
   %cmp9 = icmp sgt i32 %0, 0
   br i1 %cmp9, label %for.body.preheader, label %for.cond.cleanup
 
-for.body.preheader:                               
+for.body.preheader:
   %wide.trip.count = zext nneg i32 %0 to i64
   %xtraiter = and i64 %wide.trip.count, 1
   %1 = icmp eq i32 %0, 1
   br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
 
-for.body.preheader.new:                           
+for.body.preheader.new:
   %unroll_iter = and i64 %wide.trip.count, 2147483646
   br label %for.body
 
-for.cond.cleanup.loopexit.unr-lcssa:              
+for.cond.cleanup.loopexit.unr-lcssa:
   %indvars.iv.unr = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next.1, %for.body ]
   %lcmp.mod.not = icmp eq i64 %xtraiter, 0
   br i1 %lcmp.mod.not, label %for.cond.cleanup, label %for.body.epil
 
-for.body.epil:                                    
+for.body.epil:
   %arrayidx.epil = getelementptr inbounds nuw float, ptr %In_a, i64 %indvars.iv.unr
   %2 = load float, ptr %arrayidx.epil, align 4
   %vecins.i.epil = insertelement <4 x float> poison, float %2, i64 0
@@ -42,10 +182,10 @@ for.body.epil:
   store float %vecext.i.epil, ptr %arrayidx5.epil, align 4
   br label %for.cond.cleanup
 
-for.cond.cleanup:                                 
+for.cond.cleanup:
   ret void
 
-for.body:                                         
+for.body:
   %indvars.iv = phi i64 [ 0, %for.body.preheader.new ], [ %indvars.iv.next.1, %for.body ]
   %niter = phi i64 [ 0, %for.body.preheader.new ], [ %niter.next.1, %for.body ]
   %arrayidx = getelementptr inbounds nuw float, ptr %In_a, i64 %indvars.iv
@@ -72,113 +212,3 @@ for.body:
   %niter.ncmp.1 = icmp eq i64 %niter.next.1, %unroll_iter
   br i1 %niter.ncmp.1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
 }
-
-; AIX32:      ._Z3fooPfS_Pi:
-; AIX32-NEXT: # %bb.0:                                # %entry
-; AIX32-NEXT:   lwz 5, 0(5)
-; AIX32-NEXT:   cmpwi   5, 1
-; AIX32-NEXT:   bltlr   0
-; AIX32-NEXT: # %bb.1:                                # %for.body.preheader
-; AIX32-NEXT:   li 6, 0
-; AIX32-NEXT:   beq     0, L..BB0_4
-; AIX32-NEXT: # %bb.2:                                # %for.body.preheader.new
-; AIX32-NEXT:   xxspltib 0, 6
-; AIX32-NEXT:   addi 12, 4, -8
-; AIX32-NEXT:   addi 9, 3, -8
-; AIX32-NEXT:   rlwinm 7, 5, 0, 1, 30
-; AIX32-NEXT:   li 8, 0
-; AIX32-NEXT:   li 10, 8
-; AIX32-NEXT:   li 11, 12
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT: L..BB0_3:                               # %for.body
-; AIX32-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   lxvwsx 1, 12, 10
-; AIX32-NEXT:   addic 6, 6, 2
-; AIX32-NEXT:   addze 8, 8
-; AIX32-NEXT:   xor 0, 6, 7
-; AIX32-NEXT:   or. 0, 0, 8
-; AIX32-NEXT:   xxland 1, 1, 0
-; AIX32-NEXT:   xscvspdpn 1, 1
-; AIX32-NEXT:   stfsu 1, 8(9)
-; AIX32-NEXT:   lxvwsx 1, 12, 11
-; AIX32-NEXT:   addi 12, 12, 8
-; AIX32-NEXT:   xxland 1, 1, 0
-; AIX32-NEXT:   xscvspdpn 1, 1
-; AIX32-NEXT:   stfs 1, 4(9)
-; AIX32-NEXT:   bne     0, L..BB0_3
-
-; AIX64:      ._Z3fooPfS_Pi:
-; AIX64-NEXT: # %bb.0:                                # %entry
-; AIX64-NEXT:   lwz 5, 0(5)
-; AIX64-NEXT:   cmpwi   5, 1
-; AIX64-NEXT:   bltlr   0
-; AIX64-NEXT: # %bb.1:                                # %for.body.preheader
-; AIX64-NEXT:   li 6, 0
-; AIX64-NEXT:   cmplwi  5, 1
-; AIX64-NEXT:   beq     0, L..BB0_4
-; AIX64-NEXT: # %bb.2:                                # %for.body.preheader.new
-; AIX64-NEXT:   rlwinm 6, 5, 0, 1, 30
-; AIX64-NEXT:   xxspltib 0, 6
-; AIX64-NEXT:   addi 9, 4, -8
-; AIX64-NEXT:   addi 7, 3, -8
-; AIX64-NEXT:   li 8, 8
-; AIX64-NEXT:   li 10, 12
-; AIX64-NEXT:   li 11, 4
-; AIX64-NEXT:   addi 6, 6, -2
-; AIX64-NEXT:   rldicl 6, 6, 63, 1
-; AIX64-NEXT:   addi 6, 6, 1
-; AIX64-NEXT:   mtctr 6
-; AIX64-NEXT:   li 6, 0
-; AIX64-NEXT:   .align  4
-; AIX64-NEXT: L..BB0_3:                               # %for.body
-; AIX64-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; AIX64-NEXT:   lxvwsx 1, 9, 8
-; AIX64-NEXT:   addi 6, 6, 2
-; AIX64-NEXT:   xxland 1, 1, 0
-; AIX64-NEXT:   xscvspdpn 1, 1
-; AIX64-NEXT:   stfsu 1, 8(7)
-; AIX64-NEXT:   lxvwsx 1, 9, 10
-; AIX64-NEXT:   addi 9, 9, 8
-; AIX64-NEXT:   xxland 1, 1, 0
-; AIX64-NEXT:   xxsldwi 1, 1, 1, 3
-; AIX64-NEXT:   stfiwx 1, 7, 11
-; AIX64-NEXT:   bdnz L..BB0_3
-
-; LINUX64LE:      _Z3fooPfS_Pi:                           # @_Z3fooPfS_Pi
-; LINUX64LE-NEXT: .Lfunc_begin0:
-; LINUX64LE-NEXT:       .cfi_startproc
-; LINUX64LE-NEXT: # %bb.0:                                # %entry
-; LINUX64LE-NEXT:       lwz 5, 0(5)
-; LINUX64LE-NEXT:       cmpwi   5, 1
-; LINUX64LE-NEXT:       bltlr   0
-; LINUX64LE-NEXT: # %bb.1:                                # %for.body.preheader
-; LINUX64LE-NEXT:       li 6, 0
-; LINUX64LE-NEXT:       cmplwi  5, 1
-; LINUX64LE-NEXT:       beq     0, .LBB0_4
-; LINUX64LE-NEXT: # %bb.2:                                # %for.body.preheader.new
-; LINUX64LE-NEXT:       rlwinm 6, 5, 0, 1, 30
-; LINUX64LE-NEXT:       xxspltib 0, 6
-; LINUX64LE-NEXT:       addi 8, 4, -8
-; LINUX64LE-NEXT:       addi 7, 3, -8
-; LINUX64LE-NEXT:       li 9, 8
-; LINUX64LE-NEXT:       li 10, 12
-; LINUX64LE-NEXT:       li 11, 4
-; LINUX64LE-NEXT:       addi 6, 6, -2
-; LINUX64LE-NEXT:       rldicl 6, 6, 63, 1
-; LINUX64LE-NEXT:       addi 6, 6, 1
-; LINUX64LE-NEXT:       mtctr 6
-; LINUX64LE-NEXT:       li 6, 0
-; LINUX64LE-NEXT:       .p2align        4
-; LINUX64LE-NEXT: .LBB0_3:                                # %for.body
-; LINUX64LE-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; LINUX64LE-NEXT:       lxvwsx 1, 8, 9
-; LINUX64LE-NEXT:       addi 6, 6, 2
-; LINUX64LE-NEXT:       xxland 1, 1, 0
-; LINUX64LE-NEXT:       xxsldwi 1, 1, 1, 3
-; LINUX64LE-NEXT:       xscvspdpn 1, 1
-; LINUX64LE-NEXT:       stfsu 1, 8(7)
-; LINUX64LE-NEXT:       lxvwsx 1, 8, 10
-; LINUX64LE-NEXT:       addi 8, 8, 8
-; LINUX64LE-NEXT:       xxland 1, 1, 0
-; LINUX64LE-NEXT:       stxvrwx 1, 7, 11
-; LINUX64LE-NEXT:       bdnz .LBB0_3
diff --git a/llvm/test/CodeGen/PowerPC/loop-comment.ll b/llvm/test/CodeGen/PowerPC/loop-comment.ll
index 34b29cbe901e..530e67b4804f 100644
--- a/llvm/test/CodeGen/PowerPC/loop-comment.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-comment.ll
@@ -10,7 +10,6 @@ define void @test(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE-NEXT:  .LBB0_1: # %cmpxchg.start
 ; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
 ; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
diff --git a/llvm/test/CodeGen/PowerPC/memintr32.ll b/llvm/test/CodeGen/PowerPC/memintr32.ll
new file mode 100644
index 000000000000..c07a5af17e48
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/memintr32.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names -mtriple=powerpc-ibm-aix < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-AIX-32-P9
+
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names -mtriple=powerpc-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-LINUX32-P9
+
+define i32 @memcmp_test(ptr nocapture noundef readonly %ptr1, ptr nocapture noundef readonly %ptr2, i32 noundef %num) nounwind {
+; CHECK-AIX-32-P9-LABEL: memcmp_test:
+; CHECK-AIX-32-P9:       # %bb.0: # %entry
+; CHECK-AIX-32-P9-NEXT:    mflr r0
+; CHECK-AIX-32-P9-NEXT:    stwu r1, -64(r1)
+; CHECK-AIX-32-P9-NEXT:    stw r0, 72(r1)
+; CHECK-AIX-32-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-32-P9-NEXT:    nop
+; CHECK-AIX-32-P9-NEXT:    addi r1, r1, 64
+; CHECK-AIX-32-P9-NEXT:    lwz r0, 8(r1)
+; CHECK-AIX-32-P9-NEXT:    mtlr r0
+; CHECK-AIX-32-P9-NEXT:    blr
+;
+; CHECK-LINUX32-P9-LABEL: memcmp_test:
+; CHECK-LINUX32-P9:       # %bb.0: # %entry
+; CHECK-LINUX32-P9-NEXT:    mflr r0
+; CHECK-LINUX32-P9-NEXT:    stwu r1, -16(r1)
+; CHECK-LINUX32-P9-NEXT:    stw r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    bl memcmp
+; CHECK-LINUX32-P9-NEXT:    lwz r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    addi r1, r1, 16
+; CHECK-LINUX32-P9-NEXT:    mtlr r0
+; CHECK-LINUX32-P9-NEXT:    blr
+entry:
+  %call = tail call i32 @memcmp(ptr noundef %ptr1, ptr noundef %ptr2, i32 noundef %num)
+    ret i32 %call
+}
+
+declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i32 noundef) nounwind
+
+
+
diff --git a/llvm/test/CodeGen/PowerPC/memintr64.ll b/llvm/test/CodeGen/PowerPC/memintr64.ll
new file mode 100644
index 000000000000..b3a6650b8f6e
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/memintr64.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64le-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-LE-P9
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:   -mtriple=powerpc64-unknown-linux-gnu < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-BE-P9
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:    -mtriple=powerpc64-ibm-aix < %s | \
+; RUN:   FileCheck %s --check-prefix=CHECK-AIX-64-P9
+
+define noundef i32 @_Z11memcmp_testPKvS0_m(ptr noundef readonly captures(none) %ptr1, ptr noundef readonly captures(none) %ptr2, i64 noundef %num) nounwind {
+; CHECK-LE-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-LE-P9:       # %bb.0: # %entry
+; CHECK-LE-P9-NEXT:    mflr r0
+; CHECK-LE-P9-NEXT:    stdu r1, -32(r1)
+; CHECK-LE-P9-NEXT:    std r0, 48(r1)
+; CHECK-LE-P9-NEXT:    bl memcmp
+; CHECK-LE-P9-NEXT:    nop
+; CHECK-LE-P9-NEXT:    addi r1, r1, 32
+; CHECK-LE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-LE-P9-NEXT:    mtlr r0
+; CHECK-LE-P9-NEXT:    blr
+;
+; CHECK-BE-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-BE-P9:       # %bb.0: # %entry
+; CHECK-BE-P9-NEXT:    mflr r0
+; CHECK-BE-P9-NEXT:    stdu r1, -112(r1)
+; CHECK-BE-P9-NEXT:    std r0, 128(r1)
+; CHECK-BE-P9-NEXT:    bl memcmp
+; CHECK-BE-P9-NEXT:    nop
+; CHECK-BE-P9-NEXT:    addi r1, r1, 112
+; CHECK-BE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-BE-P9-NEXT:    mtlr r0
+; CHECK-BE-P9-NEXT:    blr
+;
+; CHECK-AIX-64-P9-LABEL: _Z11memcmp_testPKvS0_m:
+; CHECK-AIX-64-P9:       # %bb.0: # %entry
+; CHECK-AIX-64-P9-NEXT:    mflr r0
+; CHECK-AIX-64-P9-NEXT:    stdu r1, -112(r1)
+; CHECK-AIX-64-P9-NEXT:    std r0, 128(r1)
+; CHECK-AIX-64-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-64-P9-NEXT:    nop
+; CHECK-AIX-64-P9-NEXT:    addi r1, r1, 112
+; CHECK-AIX-64-P9-NEXT:    ld r0, 16(r1)
+; CHECK-AIX-64-P9-NEXT:    mtlr r0
+; CHECK-AIX-64-P9-NEXT:    blr
+entry:
+  %call = tail call i32 @memcmp(ptr noundef %ptr1, ptr noundef %ptr2, i64 noundef %num)
+  ret i32 %call
+}
+
+declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i64 noundef) nounwind
+
+
diff --git a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
index 5486dc02faf9..232014db9a01 100644
--- a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
+++ b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
@@ -10,28 +10,13 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK
 
 define dso_local noundef range(i8 -1, 1) <16 x i8> @_Z5v00FFv() {
-; CHECK:      L..CPI0_0:
-; CHECK-NEXT:   .byte   255                             # 0xff
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
-; CHECK-NEXT:   .byte   0                               # 0x0
+; CHECK-NOT:      L..CPI0_0:
+; CHECK-NOT:   .byte   255                             # 0xff
+; CHECK-NOT:   .byte   0                               # 0x0
 
 ; CHECK-LABEL: _Z5v00FFv:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lwz r3, L..C0(r2) # %const.0
-; CHECK-NEXT:    lxv vs34, 0(r3)
+; CHECK-NEXT:    mtvsrbmi v2, 1
 ; CHECK-NEXT:    blr
 entry:
   ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
diff --git a/llvm/test/CodeGen/PowerPC/nofpexcept.ll b/llvm/test/CodeGen/PowerPC/nofpexcept.ll
index f764dfc2d249..14c6e68fb922 100644
--- a/llvm/test/CodeGen/PowerPC/nofpexcept.ll
+++ b/llvm/test/CodeGen/PowerPC/nofpexcept.ll
@@ -81,35 +81,35 @@ define void @fptoint_nofpexcept(ppc_fp128 %p, fp128 %m, ptr %addr1, ptr %addr2)
   ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:f8rc = COPY $f2
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:f8rc = COPY $f1
   ; CHECK-NEXT:   [[XSCVQPSWZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPSWZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vslrc = COPY [[XSCVQPSWZ]]
-  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vsfrc = COPY [[COPY5]].sub_64
-  ; CHECK-NEXT:   STIWX killed [[COPY6]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSWZ]]
+  ; CHECK-NEXT:   [[COPY6:%[0-9]+]]:vsfrc = COPY killed [[COPY5]].sub_64
+  ; CHECK-NEXT:   STIWX killed [[COPY6]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[XSCVQPUWZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPUWZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vslrc = COPY [[XSCVQPUWZ]]
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vsfrc = COPY [[COPY7]].sub_64
-  ; CHECK-NEXT:   STIWX killed [[COPY8]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUWZ]]
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vsfrc = COPY killed [[COPY7]].sub_64
+  ; CHECK-NEXT:   STIWX killed [[COPY8]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[XSCVQPSDZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPSDZ [[COPY2]]
   ; CHECK-NEXT:   [[MFVRD:%[0-9]+]]:g8rc = nofpexcept MFVRD killed [[XSCVQPSDZ]]
   ; CHECK-NEXT:   [[XSCVQPSDZ1:%[0-9]+]]:vrrc = nofpexcept XSCVQPSDZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vslrc = COPY [[XSCVQPSDZ1]]
-  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vfrc = COPY [[COPY9]].sub_64
-  ; CHECK-NEXT:   STXSD killed [[COPY10]], 0, [[COPY]]
+  ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSDZ1]]
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vfrc = COPY killed [[COPY9]].sub_64
+  ; CHECK-NEXT:   STXSD killed [[COPY10]], 0, [[COPY]] :: (volatile store (s64) into %ir.addr2)
   ; CHECK-NEXT:   [[XSCVQPUDZ:%[0-9]+]]:vrrc = nofpexcept XSCVQPUDZ [[COPY2]]
   ; CHECK-NEXT:   [[MFVRD1:%[0-9]+]]:g8rc = nofpexcept MFVRD killed [[XSCVQPUDZ]]
   ; CHECK-NEXT:   [[XSCVQPUDZ1:%[0-9]+]]:vrrc = nofpexcept XSCVQPUDZ [[COPY2]]
-  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vslrc = COPY [[XSCVQPUDZ1]]
-  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vfrc = COPY [[COPY11]].sub_64
-  ; CHECK-NEXT:   STXSD killed [[COPY12]], 0, [[COPY]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUDZ1]]
+  ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vfrc = COPY killed [[COPY11]].sub_64
+  ; CHECK-NEXT:   STXSD killed [[COPY12]], 0, [[COPY]] :: (volatile store (s64) into %ir.addr2)
   ; CHECK-NEXT:   [[MFFS:%[0-9]+]]:f8rc = MFFS implicit $rm
   ; CHECK-NEXT:   MTFSB1 31, implicit-def $rm, implicit-def $rm
   ; CHECK-NEXT:   MTFSB0 30, implicit-def $rm, implicit-def $rm
   ; CHECK-NEXT:   [[FADD:%[0-9]+]]:f8rc = nofpexcept FADD [[COPY3]], [[COPY4]], implicit $rm
   ; CHECK-NEXT:   MTFSFb 1, [[MFFS]], implicit-def $rm
   ; CHECK-NEXT:   [[XSCVDPSXWS:%[0-9]+]]:vsfrc = nofpexcept XSCVDPSXWS killed [[FADD]], implicit $rm
-  ; CHECK-NEXT:   STIWX killed [[XSCVDPSXWS]], $zero8, [[COPY1]]
+  ; CHECK-NEXT:   STIWX killed [[XSCVDPSXWS]], $zero8, [[COPY1]] :: (volatile store (s32) into %ir.addr1)
   ; CHECK-NEXT:   [[ADDIStocHA8_:%[0-9]+]]:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
   ; CHECK-NEXT:   [[DFLOADf32_:%[0-9]+]]:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0, killed [[ADDIStocHA8_]] :: (load (s32) from constant-pool)
-  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:f8rc = COPY [[DFLOADf32_]]
+  ; CHECK-NEXT:   [[COPY13:%[0-9]+]]:f8rc = COPY killed [[DFLOADf32_]]
   ; CHECK-NEXT:   [[FCMPOD:%[0-9]+]]:crrc = FCMPOD [[COPY4]], [[COPY13]]
   ; CHECK-NEXT:   [[COPY14:%[0-9]+]]:crbitrc = COPY [[FCMPOD]].sub_eq
   ; CHECK-NEXT:   [[XXLXORdpz:%[0-9]+]]:f8rc = XXLXORdpz
@@ -174,8 +174,8 @@ define signext i32 @q_to_i32(fp128 %m) #0 {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vrrc = COPY $v2
   ; CHECK-NEXT:   [[XSCVQPSWZ:%[0-9]+]]:vrrc = XSCVQPSWZ [[COPY]]
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY [[XSCVQPSWZ]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY [[COPY1]].sub_64
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY killed [[XSCVQPSWZ]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY killed [[COPY1]].sub_64
   ; CHECK-NEXT:   [[MFVSRWZ:%[0-9]+]]:gprc = MFVSRWZ killed [[COPY2]]
   ; CHECK-NEXT:   [[EXTSW_32_64_:%[0-9]+]]:g8rc = EXTSW_32_64 killed [[MFVSRWZ]]
   ; CHECK-NEXT:   $x3 = COPY [[EXTSW_32_64_]]
@@ -222,8 +222,8 @@ define zeroext i32 @q_to_u32(fp128 %m) #0 {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vrrc = COPY $v2
   ; CHECK-NEXT:   [[XSCVQPUWZ:%[0-9]+]]:vrrc = XSCVQPUWZ [[COPY]]
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY [[XSCVQPUWZ]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY [[COPY1]].sub_64
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vslrc = COPY killed [[XSCVQPUWZ]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vfrc = COPY killed [[COPY1]].sub_64
   ; CHECK-NEXT:   [[MFVSRWZ:%[0-9]+]]:gprc = MFVSRWZ killed [[COPY2]]
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:g8rc = IMPLICIT_DEF
   ; CHECK-NEXT:   [[INSERT_SUBREG:%[0-9]+]]:g8rc = INSERT_SUBREG [[DEF]], killed [[MFVSRWZ]], %subreg.sub_32
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
index b540948b20f7..821cfd00dcd0 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-vabsd.ll
@@ -764,13 +764,8 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ;
 ; CHECK-PWR7-LABEL: sub_absv_8_ext:
 ; CHECK-PWR7:       # %bb.0: # %entry
-; CHECK-PWR7-NEXT:    stdu r1, -512(r1)
-; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 512
-; CHECK-PWR7-NEXT:    .cfi_offset r14, -144
-; CHECK-PWR7-NEXT:    .cfi_offset r15, -136
-; CHECK-PWR7-NEXT:    .cfi_offset r16, -128
-; CHECK-PWR7-NEXT:    .cfi_offset r17, -120
-; CHECK-PWR7-NEXT:    .cfi_offset r18, -112
+; CHECK-PWR7-NEXT:    stdu r1, -448(r1)
+; CHECK-PWR7-NEXT:    .cfi_def_cfa_offset 448
 ; CHECK-PWR7-NEXT:    .cfi_offset r19, -104
 ; CHECK-PWR7-NEXT:    .cfi_offset r20, -96
 ; CHECK-PWR7-NEXT:    .cfi_offset r21, -88
@@ -783,244 +778,258 @@ define <16 x i8> @sub_absv_8_ext(<16 x i8> %a, <16 x i8> %b) local_unnamed_addr
 ; CHECK-PWR7-NEXT:    .cfi_offset r28, -32
 ; CHECK-PWR7-NEXT:    .cfi_offset r29, -24
 ; CHECK-PWR7-NEXT:    .cfi_offset r30, -16
-; CHECK-PWR7-NEXT:    .cfi_offset r31, -8
-; CHECK-PWR7-NEXT:    .cfi_offset r2, -152
-; CHECK-PWR7-NEXT:    addi r3, r1, 320
-; CHECK-PWR7-NEXT:    std r14, 368(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r15, 376(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r16, 384(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r17, 392(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r18, 400(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r19, 408(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r20, 416(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r21, 424(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r22, 432(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r23, 440(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r24, 448(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r25, 456(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r26, 464(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r27, 472(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r28, 480(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r29, 488(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r30, 496(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r31, 504(r1) # 8-byte Folded Spill
-; CHECK-PWR7-NEXT:    std r2, 360(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    addi r3, r1, 304
+; CHECK-PWR7-NEXT:    std r19, 344(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r20, 352(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r21, 360(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r22, 368(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r23, 376(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r24, 384(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r25, 392(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r26, 400(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r27, 408(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r28, 416(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r29, 424(r1) # 8-byte Folded Spill
+; CHECK-PWR7-NEXT:    std r30, 432(r1) # 8-byte Folded Spill
 ; CHECK-PWR7-NEXT:    stxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    lbz r3, 320(r1)
-; CHECK-PWR7-NEXT:    addi r4, r1, 336
-; CHECK-PWR7-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
-; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r4
-; CHECK-PWR7-NEXT:    lbz r15, 334(r1)
-; CHECK-PWR7-NEXT:    lbz r14, 350(r1)
-; CHECK-PWR7-NEXT:    lbz r31, 335(r1)
-; CHECK-PWR7-NEXT:    lbz r2, 351(r1)
-; CHECK-PWR7-NEXT:    sub r15, r15, r14
-; CHECK-PWR7-NEXT:    sub r14, r31, r2
-; CHECK-PWR7-NEXT:    srawi r2, r14, 31
-; CHECK-PWR7-NEXT:    xor r14, r14, r2
-; CHECK-PWR7-NEXT:    lbz r3, 333(r1)
-; CHECK-PWR7-NEXT:    lbz r19, 331(r1)
-; CHECK-PWR7-NEXT:    lbz r18, 347(r1)
-; CHECK-PWR7-NEXT:    sub r19, r19, r18
-; CHECK-PWR7-NEXT:    lbz r17, 332(r1)
-; CHECK-PWR7-NEXT:    lbz r16, 348(r1)
-; CHECK-PWR7-NEXT:    sub r17, r17, r16
-; CHECK-PWR7-NEXT:    lbz r23, 329(r1)
-; CHECK-PWR7-NEXT:    sub r14, r14, r2
-; CHECK-PWR7-NEXT:    lbz r2, 349(r1)
-; CHECK-PWR7-NEXT:    lbz r22, 345(r1)
-; CHECK-PWR7-NEXT:    lbz r4, 336(r1)
-; CHECK-PWR7-NEXT:    lbz r5, 321(r1)
-; CHECK-PWR7-NEXT:    lbz r6, 337(r1)
-; CHECK-PWR7-NEXT:    lbz r7, 322(r1)
-; CHECK-PWR7-NEXT:    lbz r8, 338(r1)
-; CHECK-PWR7-NEXT:    lbz r9, 323(r1)
-; CHECK-PWR7-NEXT:    lbz r10, 339(r1)
-; CHECK-PWR7-NEXT:    lbz r11, 324(r1)
-; CHECK-PWR7-NEXT:    lbz r12, 340(r1)
-; CHECK-PWR7-NEXT:    lbz r0, 325(r1)
-; CHECK-PWR7-NEXT:    lbz r30, 341(r1)
-; CHECK-PWR7-NEXT:    lbz r29, 326(r1)
-; CHECK-PWR7-NEXT:    lbz r28, 342(r1)
-; CHECK-PWR7-NEXT:    lbz r27, 327(r1)
-; CHECK-PWR7-NEXT:    lbz r26, 343(r1)
-; CHECK-PWR7-NEXT:    sub r3, r3, r2
-; CHECK-PWR7-NEXT:    lbz r25, 328(r1)
-; CHECK-PWR7-NEXT:    lbz r24, 344(r1)
-; CHECK-PWR7-NEXT:    lbz r21, 330(r1)
-; CHECK-PWR7-NEXT:    lbz r20, 346(r1)
+; CHECK-PWR7-NEXT:    addi r3, r1, 320
+; CHECK-PWR7-NEXT:    lbz r7, 304(r1)
+; CHECK-PWR7-NEXT:    stxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    lbz r8, 320(r1)
+; CHECK-PWR7-NEXT:    lbz r9, 305(r1)
+; CHECK-PWR7-NEXT:    lbz r10, 321(r1)
+; CHECK-PWR7-NEXT:    lbz r26, 325(r1)
+; CHECK-PWR7-NEXT:    clrlwi r7, r7, 24
+; CHECK-PWR7-NEXT:    clrlwi r8, r8, 24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r10, r10, 24
+; CHECK-PWR7-NEXT:    lbz r11, 306(r1)
+; CHECK-PWR7-NEXT:    lbz r12, 322(r1)
+; CHECK-PWR7-NEXT:    lbz r23, 314(r1)
+; CHECK-PWR7-NEXT:    clrlwi r22, r26, 24
+; CHECK-PWR7-NEXT:    lbz r26, 330(r1)
+; CHECK-PWR7-NEXT:    sub r8, r7, r8
+; CHECK-PWR7-NEXT:    lbz r7, 315(r1)
+; CHECK-PWR7-NEXT:    sub r20, r9, r10
+; CHECK-PWR7-NEXT:    lbz r9, 331(r1)
+; CHECK-PWR7-NEXT:    lbz r0, 307(r1)
+; CHECK-PWR7-NEXT:    lbz r30, 323(r1)
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r12, r12, 24
+; CHECK-PWR7-NEXT:    clrlwi r23, r23, 24
+; CHECK-PWR7-NEXT:    clrlwi r21, r26, 24
+; CHECK-PWR7-NEXT:    clrlwi r7, r7, 24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r0, r0, 24
+; CHECK-PWR7-NEXT:    clrlwi r30, r30, 24
+; CHECK-PWR7-NEXT:    lbz r29, 308(r1)
+; CHECK-PWR7-NEXT:    lbz r28, 324(r1)
+; CHECK-PWR7-NEXT:    lbz r27, 309(r1)
+; CHECK-PWR7-NEXT:    lbz r25, 310(r1)
+; CHECK-PWR7-NEXT:    lbz r24, 326(r1)
+; CHECK-PWR7-NEXT:    sub r19, r11, r12
+; CHECK-PWR7-NEXT:    sub r11, r23, r21
+; CHECK-PWR7-NEXT:    sub r9, r7, r9
+; CHECK-PWR7-NEXT:    sub r26, r0, r30
+; CHECK-PWR7-NEXT:    srawi r12, r11, 31
+; CHECK-PWR7-NEXT:    srawi r0, r9, 31
+; CHECK-PWR7-NEXT:    lbz r3, 312(r1)
+; CHECK-PWR7-NEXT:    clrlwi r29, r29, 24
+; CHECK-PWR7-NEXT:    clrlwi r28, r28, 24
+; CHECK-PWR7-NEXT:    clrlwi r27, r27, 24
+; CHECK-PWR7-NEXT:    clrlwi r25, r25, 24
+; CHECK-PWR7-NEXT:    clrlwi r24, r24, 24
+; CHECK-PWR7-NEXT:    xor r11, r11, r12
+; CHECK-PWR7-NEXT:    xor r9, r9, r0
+; CHECK-PWR7-NEXT:    sub r28, r29, r28
+; CHECK-PWR7-NEXT:    sub r30, r27, r22
+; CHECK-PWR7-NEXT:    sub r29, r25, r24
+; CHECK-PWR7-NEXT:    sub r27, r11, r12
+; CHECK-PWR7-NEXT:    sub r24, r9, r0
+; CHECK-PWR7-NEXT:    lbz r9, 316(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 332(r1)
+; CHECK-PWR7-NEXT:    lbz r4, 328(r1)
+; CHECK-PWR7-NEXT:    lbz r5, 311(r1)
+; CHECK-PWR7-NEXT:    lbz r6, 327(r1)
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r3, r3, 24
+; CHECK-PWR7-NEXT:    clrlwi r4, r4, 24
+; CHECK-PWR7-NEXT:    clrlwi r5, r5, 24
+; CHECK-PWR7-NEXT:    clrlwi r6, r6, 24
+; CHECK-PWR7-NEXT:    sub r3, r3, r4
 ; CHECK-PWR7-NEXT:    sub r5, r5, r6
-; CHECK-PWR7-NEXT:    srawi r18, r3, 31
-; CHECK-PWR7-NEXT:    sub r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r9, r9, r10
-; CHECK-PWR7-NEXT:    sub r11, r11, r12
-; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    sub r29, r29, r28
-; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    srawi r31, r15, 31
-; CHECK-PWR7-NEXT:    ld r2, 360(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    xor r3, r3, r18
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    srawi r4, r3, 31
 ; CHECK-PWR7-NEXT:    srawi r6, r5, 31
-; CHECK-PWR7-NEXT:    srawi r8, r7, 31
-; CHECK-PWR7-NEXT:    srawi r10, r9, 31
-; CHECK-PWR7-NEXT:    srawi r12, r11, 31
-; CHECK-PWR7-NEXT:    srawi r30, r0, 31
-; CHECK-PWR7-NEXT:    sub r3, r3, r18
-; CHECK-PWR7-NEXT:    srawi r18, r19, 31
-; CHECK-PWR7-NEXT:    srawi r28, r29, 31
-; CHECK-PWR7-NEXT:    ld r16, 384(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    srawi r26, r27, 31
-; CHECK-PWR7-NEXT:    srawi r24, r25, 31
-; CHECK-PWR7-NEXT:    xor r19, r19, r18
-; CHECK-PWR7-NEXT:    xor r15, r15, r31
+; CHECK-PWR7-NEXT:    xor r3, r3, r4
+; CHECK-PWR7-NEXT:    sldi r27, r27, 56
 ; CHECK-PWR7-NEXT:    xor r5, r5, r6
-; CHECK-PWR7-NEXT:    std r3, 272(r1)
-; CHECK-PWR7-NEXT:    std r3, 280(r1)
-; CHECK-PWR7-NEXT:    srawi r3, r17, 31
-; CHECK-PWR7-NEXT:    sub r19, r19, r18
-; CHECK-PWR7-NEXT:    xor r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r15, r15, r31
-; CHECK-PWR7-NEXT:    xor r17, r17, r3
-; CHECK-PWR7-NEXT:    xor r9, r9, r10
-; CHECK-PWR7-NEXT:    xor r11, r11, r12
-; CHECK-PWR7-NEXT:    xor r0, r0, r30
-; CHECK-PWR7-NEXT:    xor r29, r29, r28
-; CHECK-PWR7-NEXT:    xor r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r3, r17, r3
-; CHECK-PWR7-NEXT:    xor r25, r25, r24
-; CHECK-PWR7-NEXT:    sub r25, r25, r24
-; CHECK-PWR7-NEXT:    sub r27, r27, r26
-; CHECK-PWR7-NEXT:    sub r29, r29, r28
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    sub r3, r3, r4
+; CHECK-PWR7-NEXT:    sldi r24, r24, 56
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    sub r0, r0, r30
-; CHECK-PWR7-NEXT:    sub r11, r11, r12
-; CHECK-PWR7-NEXT:    sub r9, r9, r10
-; CHECK-PWR7-NEXT:    sub r7, r7, r8
-; CHECK-PWR7-NEXT:    sub r5, r5, r6
-; CHECK-PWR7-NEXT:    sldi r14, r14, 56
-; CHECK-PWR7-NEXT:    sldi r15, r15, 56
-; CHECK-PWR7-NEXT:    ld r31, 504(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r3, 256(r1)
-; CHECK-PWR7-NEXT:    std r3, 264(r1)
-; CHECK-PWR7-NEXT:    sldi r3, r19, 56
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r27, 208(r1)
+; CHECK-PWR7-NEXT:    sub r4, r5, r6
+; CHECK-PWR7-NEXT:    std r27, 216(r1)
+; CHECK-PWR7-NEXT:    srawi r27, r29, 31
+; CHECK-PWR7-NEXT:    lbz r10, 313(r1)
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    std r24, 224(r1)
+; CHECK-PWR7-NEXT:    lbz r22, 329(r1)
+; CHECK-PWR7-NEXT:    std r24, 232(r1)
+; CHECK-PWR7-NEXT:    srawi r24, r30, 31
+; CHECK-PWR7-NEXT:    ld r21, 360(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r23, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 317(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 333(r1)
+; CHECK-PWR7-NEXT:    xor r29, r29, r27
+; CHECK-PWR7-NEXT:    std r3, 176(r1)
+; CHECK-PWR7-NEXT:    std r3, 184(r1)
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    sldi r23, r23, 56
+; CHECK-PWR7-NEXT:    xor r30, r30, r24
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    sub r4, r30, r24
+; CHECK-PWR7-NEXT:    ld r30, 432(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 160(r1)
+; CHECK-PWR7-NEXT:    std r3, 168(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    sub r3, r29, r27
+; CHECK-PWR7-NEXT:    std r23, 240(r1)
+; CHECK-PWR7-NEXT:    ld r29, 424(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r23, 248(r1)
+; CHECK-PWR7-NEXT:    ld r27, 408(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r23, r28, 31
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    xor r28, r28, r23
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    std r3, 144(r1)
+; CHECK-PWR7-NEXT:    ld r24, 384(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 152(r1)
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    sub r25, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 318(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 334(r1)
+; CHECK-PWR7-NEXT:    std r3, 128(r1)
 ; CHECK-PWR7-NEXT:    sldi r25, r25, 56
-; CHECK-PWR7-NEXT:    sldi r27, r27, 56
-; CHECK-PWR7-NEXT:    std r3, 240(r1)
-; CHECK-PWR7-NEXT:    std r3, 248(r1)
-; CHECK-PWR7-NEXT:    sub r3, r23, r22
-; CHECK-PWR7-NEXT:    srawi r23, r3, 31
-; CHECK-PWR7-NEXT:    sub r22, r21, r20
-; CHECK-PWR7-NEXT:    srawi r21, r22, 31
-; CHECK-PWR7-NEXT:    sldi r29, r29, 56
-; CHECK-PWR7-NEXT:    sldi r0, r0, 56
-; CHECK-PWR7-NEXT:    sldi r11, r11, 56
-; CHECK-PWR7-NEXT:    xor r3, r3, r23
-; CHECK-PWR7-NEXT:    xor r22, r22, r21
-; CHECK-PWR7-NEXT:    sldi r9, r9, 56
-; CHECK-PWR7-NEXT:    sldi r7, r7, 56
-; CHECK-PWR7-NEXT:    sldi r5, r5, 56
-; CHECK-PWR7-NEXT:    ld r30, 496(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r28, 480(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    sub r3, r3, r23
-; CHECK-PWR7-NEXT:    sub r22, r22, r21
-; CHECK-PWR7-NEXT:    std r14, 304(r1)
-; CHECK-PWR7-NEXT:    ld r26, 464(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 136(r1)
+; CHECK-PWR7-NEXT:    sub r3, r28, r23
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    sldi r22, r22, 56
-; CHECK-PWR7-NEXT:    ld r24, 448(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r23, 440(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r14, 312(r1)
-; CHECK-PWR7-NEXT:    std r15, 288(r1)
-; CHECK-PWR7-NEXT:    std r3, 208(r1)
-; CHECK-PWR7-NEXT:    std r3, 216(r1)
-; CHECK-PWR7-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r15, 296(r1)
-; CHECK-PWR7-NEXT:    ld r21, 424(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r20, 416(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r22, 224(r1)
-; CHECK-PWR7-NEXT:    std r22, 232(r1)
-; CHECK-PWR7-NEXT:    sub r4, r3, r4
-; CHECK-PWR7-NEXT:    std r25, 192(r1)
-; CHECK-PWR7-NEXT:    ld r22, 432(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r19, 408(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    srawi r3, r4, 31
-; CHECK-PWR7-NEXT:    std r25, 200(r1)
-; CHECK-PWR7-NEXT:    ld r25, 456(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r27, 176(r1)
-; CHECK-PWR7-NEXT:    std r27, 184(r1)
-; CHECK-PWR7-NEXT:    xor r4, r4, r3
-; CHECK-PWR7-NEXT:    std r29, 160(r1)
-; CHECK-PWR7-NEXT:    ld r27, 472(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r29, 168(r1)
-; CHECK-PWR7-NEXT:    std r0, 144(r1)
-; CHECK-PWR7-NEXT:    sub r3, r4, r3
-; CHECK-PWR7-NEXT:    std r0, 152(r1)
-; CHECK-PWR7-NEXT:    ld r29, 488(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    ld r18, 400(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    std r3, 112(r1)
+; CHECK-PWR7-NEXT:    ld r28, 416(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    clrlwi r10, r10, 24
+; CHECK-PWR7-NEXT:    std r25, 256(r1)
+; CHECK-PWR7-NEXT:    std r25, 264(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    srawi r25, r26, 31
+; CHECK-PWR7-NEXT:    xor r26, r26, r25
+; CHECK-PWR7-NEXT:    ld r23, 376(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r3, 120(r1)
+; CHECK-PWR7-NEXT:    sub r4, r26, r25
+; CHECK-PWR7-NEXT:    clrlwi r22, r22, 24
+; CHECK-PWR7-NEXT:    srawi r7, r8, 31
+; CHECK-PWR7-NEXT:    sub r10, r10, r22
+; CHECK-PWR7-NEXT:    ld r26, 400(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
+; CHECK-PWR7-NEXT:    srawi r22, r10, 31
+; CHECK-PWR7-NEXT:    xor r8, r8, r7
+; CHECK-PWR7-NEXT:    xor r10, r10, r22
+; CHECK-PWR7-NEXT:    sub r10, r10, r22
+; CHECK-PWR7-NEXT:    ld r25, 392(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r12, r9, r11
+; CHECK-PWR7-NEXT:    lbz r9, 319(r1)
+; CHECK-PWR7-NEXT:    lbz r11, 335(r1)
+; CHECK-PWR7-NEXT:    std r3, 96(r1)
+; CHECK-PWR7-NEXT:    sldi r12, r12, 56
+; CHECK-PWR7-NEXT:    std r3, 104(r1)
+; CHECK-PWR7-NEXT:    ld r22, 368(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sldi r10, r10, 56
+; CHECK-PWR7-NEXT:    std r10, 192(r1)
+; CHECK-PWR7-NEXT:    clrlwi r9, r9, 24
+; CHECK-PWR7-NEXT:    clrlwi r11, r11, 24
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    std r12, 272(r1)
+; CHECK-PWR7-NEXT:    std r12, 280(r1)
+; CHECK-PWR7-NEXT:    srawi r12, r19, 31
+; CHECK-PWR7-NEXT:    xor r0, r19, r12
+; CHECK-PWR7-NEXT:    ld r19, 344(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r3, r0, r12
+; CHECK-PWR7-NEXT:    srawi r11, r9, 31
+; CHECK-PWR7-NEXT:    std r10, 200(r1)
+; CHECK-PWR7-NEXT:    xor r9, r9, r11
 ; CHECK-PWR7-NEXT:    sldi r3, r3, 56
-; CHECK-PWR7-NEXT:    std r11, 128(r1)
-; CHECK-PWR7-NEXT:    ld r17, 392(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r11, 136(r1)
-; CHECK-PWR7-NEXT:    std r9, 112(r1)
+; CHECK-PWR7-NEXT:    sub r9, r9, r11
+; CHECK-PWR7-NEXT:    std r3, 80(r1)
+; CHECK-PWR7-NEXT:    std r3, 88(r1)
+; CHECK-PWR7-NEXT:    sldi r9, r9, 56
+; CHECK-PWR7-NEXT:    std r9, 288(r1)
+; CHECK-PWR7-NEXT:    std r9, 296(r1)
+; CHECK-PWR7-NEXT:    srawi r9, r20, 31
+; CHECK-PWR7-NEXT:    xor r11, r20, r9
+; CHECK-PWR7-NEXT:    ld r20, 352(r1) # 8-byte Folded Reload
+; CHECK-PWR7-NEXT:    sub r4, r11, r9
+; CHECK-PWR7-NEXT:    sldi r3, r4, 56
 ; CHECK-PWR7-NEXT:    std r3, 64(r1)
 ; CHECK-PWR7-NEXT:    std r3, 72(r1)
-; CHECK-PWR7-NEXT:    addi r3, r1, 304
-; CHECK-PWR7-NEXT:    std r9, 120(r1)
-; CHECK-PWR7-NEXT:    ld r15, 376(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    std r7, 96(r1)
-; CHECK-PWR7-NEXT:    std r7, 104(r1)
-; CHECK-PWR7-NEXT:    std r5, 80(r1)
-; CHECK-PWR7-NEXT:    std r5, 88(r1)
-; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
+; CHECK-PWR7-NEXT:    sub r3, r8, r7
+; CHECK-PWR7-NEXT:    sldi r3, r3, 56
+; CHECK-PWR7-NEXT:    std r3, 48(r1)
+; CHECK-PWR7-NEXT:    std r3, 56(r1)
 ; CHECK-PWR7-NEXT:    addi r3, r1, 288
-; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 272
-; CHECK-PWR7-NEXT:    ld r14, 368(r1) # 8-byte Folded Reload
-; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 256
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
+; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 240
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 224
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 224
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 208
-; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 192
-; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
+; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 176
+; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 160
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs0, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v2, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 160
-; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 144
-; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 128
+; CHECK-PWR7-NEXT:    vmrghb v2, v3, v2
+; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 112
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 96
 ; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
-; CHECK-PWR7-NEXT:    addi r3, r1, 112
 ; CHECK-PWR7-NEXT:    vmrghh v2, v3, v2
 ; CHECK-PWR7-NEXT:    lxvw4x v3, 0, r3
-; CHECK-PWR7-NEXT:    addi r3, r1, 96
-; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 80
-; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
 ; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
 ; CHECK-PWR7-NEXT:    addi r3, r1, 64
+; CHECK-PWR7-NEXT:    vmrghb v3, v4, v3
+; CHECK-PWR7-NEXT:    lxvw4x v4, 0, r3
+; CHECK-PWR7-NEXT:    addi r3, r1, 48
 ; CHECK-PWR7-NEXT:    lxvw4x v5, 0, r3
 ; CHECK-PWR7-NEXT:    vmrghb v4, v5, v4
 ; CHECK-PWR7-NEXT:    vmrghh v3, v4, v3
 ; CHECK-PWR7-NEXT:    xxmrghw vs1, v3, v2
 ; CHECK-PWR7-NEXT:    xxmrghd v2, vs1, vs0
-; CHECK-PWR7-NEXT:    addi r1, r1, 512
+; CHECK-PWR7-NEXT:    addi r1, r1, 448
 ; CHECK-PWR7-NEXT:    blr
 entry:
   %vecext = extractelement <16 x i8> %a, i32 0
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index 96e2c6c3e081..2598a410b876 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -1024,15 +1024,16 @@ define <4 x float> @hoo3_fmf(<4 x float> %a) #1 {
 ; CHECK-P7-NEXT:    vslw 3, 3, 3
 ; CHECK-P7-NEXT:    lvx 0, 0, 3
 ; CHECK-P7-NEXT:    addis 3, 2, .LCPI25_1@toc@ha
-; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P7-NEXT:    vmaddfp 5, 2, 4, 3
-; CHECK-P7-NEXT:    lvx 1, 0, 3
+; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P7-NEXT:    vmaddfp 4, 5, 4, 0
-; CHECK-P7-NEXT:    vmaddfp 5, 5, 1, 3
+; CHECK-P7-NEXT:    lvx 0, 0, 3
+; CHECK-P7-NEXT:    vmaddfp 5, 5, 0, 3
 ; CHECK-P7-NEXT:    vmaddfp 3, 5, 4, 3
 ; CHECK-P7-NEXT:    vxor 4, 4, 4
 ; CHECK-P7-NEXT:    vcmpeqfp 2, 2, 4
-; CHECK-P7-NEXT:    vsel 2, 3, 4, 2
+; CHECK-P7-NEXT:    vnot 2, 2
+; CHECK-P7-NEXT:    vand 2, 2, 3
 ; CHECK-P7-NEXT:    blr
 ;
 ; CHECK-P8-LABEL: hoo3_fmf:
diff --git a/llvm/test/CodeGen/PowerPC/sat-add.ll b/llvm/test/CodeGen/PowerPC/sat-add.ll
index d9b22bda85e4..34b703a98110 100644
--- a/llvm/test/CodeGen/PowerPC/sat-add.ll
+++ b/llvm/test/CodeGen/PowerPC/sat-add.ll
@@ -536,12 +536,11 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI34_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI34_0@toc@l
 ; CHECK-NEXT:    lxvd2x 35, 0, 3
 ; CHECK-NEXT:    vaddudm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
@@ -553,7 +552,6 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI35_0@toc@l
 ; CHECK-NEXT:    lxvd2x 35, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI35_1@toc@ha
@@ -561,7 +559,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-NEXT:    lxvd2x 36, 0, 3
 ; CHECK-NEXT:    vaddudm 3, 2, 3
 ; CHECK-NEXT:    vcmpgtud 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
@@ -599,9 +597,8 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vaddubm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtub 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   %a = add <16 x i8> %x, %y
@@ -640,9 +637,8 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vadduhm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuh 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %a = add <8 x i16> %x, %y
@@ -681,9 +677,8 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vadduwm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtuw 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = add <4 x i32> %x, %y
@@ -710,9 +705,8 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddudm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
@@ -725,9 +719,8 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xxlnor 36, 35, 35
 ; CHECK-NEXT:    vaddudm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    vcmpgtud 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    xxlor 34, 34, 35
 ; CHECK-NEXT:    blr
   %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
   %a = add <2 x i64> %x, %y
diff --git a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-trivial-copy.ll b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-trivial-copy.ll
index 96aa58000f9b..539b56369172 100644
--- a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-trivial-copy.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-trivial-copy.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -fp-contract=fast < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -31,7 +31,7 @@ declare double @llvm.sqrt.f64(double) #1
 
 declare signext i32 @p_col_helper(...) #2
 
-attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="pwr7" "unsafe-fp-math"="true" }
+attributes #0 = { nounwind "no-infs-fp-math"="true" "no-nans-fp-math"="true" "target-cpu"="pwr7" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind }
 
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
index 2868669c52ce..57d4c48a1aaa 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-and.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, xor(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_xor_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %xor, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, xor(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_xor_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_xor_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %xor, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_and_BC_4x32:
@@ -86,6 +122,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_and_BC_4x32:
@@ -125,6 +199,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B, C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(C), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_C_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_C_and_BC_4x32:
@@ -162,6 +274,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(C), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_C_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %C, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(C), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_C_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_C_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %C, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %and
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(B), and(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_B_and_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_B_and_BC_4x32:
@@ -198,3 +346,39 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %and
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, not(B), and(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_B_and_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %B, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %and
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(B), and(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_B_and_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_B_and_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxland vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %B, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %and
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
index 37a0edb14b78..c366fd5f0a8c 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-b.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -43,6 +43,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %B
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), B) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_B_4x32:
@@ -78,6 +110,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %B
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), B) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_B_4x32:
@@ -113,37 +179,38 @@ entry:
   ret <2 x i64> %res
 }
 
-; Function to test ternary(A, not(C), B) for <4 x i32>
-define <4 x i32> @ternary_A_not_C_B_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
-; CHECK-LABEL: ternary_A_not_C_B_4x32:
+; Function to test ternary(A, eqv(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_16x8:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxleqv v5, v5, v5
-; CHECK-NEXT:    xxlnor vs0, v4, v4
-; CHECK-NEXT:    vslw v2, v2, v5
-; CHECK-NEXT:    vsraw v2, v2, v5
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
 ; CHECK-NEXT:    xxsel v2, v3, vs0, v2
 ; CHECK-NEXT:    blr
 entry:
-  %not = xor <4 x i32> %C, <i32 -1, i32 -1, i32 -1, i32 -1>  ; Vector not operation
-  %res = select <4 x i1> %A, <4 x i32> %not, <4 x i32> %B
-  ret <4 x i32> %res
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %B
+  ret <16 x i8> %res
 }
 
-; Function to test ternary(A, not(C), B) for <2 x i64>
-define <2 x i64> @ternary_A_not_C_B_2x64(<2 x i1> %A, <2 x i64> %B, <2 x i64> %C) {
-; CHECK-LABEL: ternary_A_not_C_B_2x64:
+; Function to test ternary(A, eqv(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_B_8x16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxlxor v5, v5, v5
-; CHECK-NEXT:    xxlnor vs0, v4, v4
-; CHECK-NEXT:    xxsplti32dx v5, 1, 63
-; CHECK-NEXT:    vsld v2, v2, v5
-; CHECK-NEXT:    vsrad v2, v2, v5
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
 ; CHECK-NEXT:    xxsel v2, v3, vs0, v2
 ; CHECK-NEXT:    blr
 entry:
-  %not = xor <2 x i64> %C, <i64 -1, i64 -1>  ; Vector not operation
-  %res = select <2 x i1> %A, <2 x i64> %not, <2 x i64> %B
-  ret <2 x i64> %res
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %B
+  ret <8 x i16> %res
 }
 
 ; Function to test ternary(A, nand(B, C), B) for <4 x i32>
@@ -180,3 +247,37 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %B
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B, C), B) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_B_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %B
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B, C), B) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_B_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_B_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v3, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %B
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
index 411aa27a6186..f70f1d093f06 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-c.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -43,6 +43,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_C_4x32:
@@ -78,6 +110,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, nor(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector NOR operation
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector NOR operation
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, eqv(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_eqv_BC_C_4x32:
@@ -113,6 +179,40 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %C
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nand(B, C), C) for <4 x i32>
 define <4 x i32> @ternary_A_nand_BC_C_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nand_BC_C_4x32:
@@ -147,3 +247,37 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %C
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B, C), C) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_C_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %C
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B, C), C) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_C_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_C_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, v4, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %C
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
index 1ad7e95e3682..861789f4d326 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-or.ll
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, B, or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_B_or_BC_4x32:
@@ -80,6 +116,37 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, B, or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_B_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %B, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, B, or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_B_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_B_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %B, <8 x i16> %or
+  ret <8 x i16> %res
+}
 
 ; Function to test ternary(A, C, or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
@@ -114,6 +181,37 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, C, or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_C_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %C, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, C, or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_C_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_C_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %C, <8 x i16> %or
+  ret <8 x i16> %res
+}
 
 ; Function to test ternary(A, eqv(B,C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_eqv_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
@@ -154,6 +252,44 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_eqv_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %eqv = xor <16 x i8> %xor, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector eqv operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %eqv, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, eqv(B,C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_eqv_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_eqv_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxleqv vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %eqv = xor <8 x i16> %xor, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector eqv operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %eqv, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_C_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_C_or_BC_4x32:
@@ -191,6 +327,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_C_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %C, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_C_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_C_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v4, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %C, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, not(B), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_not_B_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_not_B_or_BC_4x32:
@@ -228,6 +400,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, not(B), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_not_B_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <16 x i8> %B, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector not operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %not, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, not(B), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_not_B_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_not_B_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v3
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %not = xor <8 x i16> %B, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector not operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %not, <8 x i16> %or
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nand(B,C), or(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nand_BC_or_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nand_BC_or_BC_4x32:
@@ -266,3 +474,41 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nand, <2 x i64> %or
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nand_BC_or_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %nand = xor <16 x i8> %and, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>  ; Vector nand operation
+  %or = or <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nand, <16 x i8> %or
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nand(B,C), or(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nand_BC_or_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nand_BC_or_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnand vs0, v3, v4
+; CHECK-NEXT:    xxlor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %nand = xor <8 x i16> %and, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>  ; Vector nand operation
+  %or = or <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nand, <8 x i16> %or
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
index f8a8b6e9a048..a3fdc905cb52 100644
--- a/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
+++ b/llvm/test/CodeGen/PowerPC/xxeval-vselect-x-xor.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; Test file to verify the emission of Vector selection instructions when ternary operators are used.
+; Test file to verify the emission of Vector Selection instructions when ternary operators are used.
 
 ; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
@@ -47,6 +47,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, and(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_and_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <16 x i8> %B, %C
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %and, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, and(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_and_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_and_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxland vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %and = and <8 x i16> %B, %C
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %and, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, B, xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_B_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_B_xor_BC_4x32:
@@ -80,6 +116,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, B, xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_B_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %B, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, B, xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_B_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_B_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v3, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %B, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, C, xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_C_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_C_xor_BC_4x32:
@@ -113,6 +181,38 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, C, xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_C_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %C, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, C, xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_C_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_C_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlxor vs0, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs0, v4, v2
+; CHECK-NEXT:    blr
+entry:
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %C, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, or(B, C), xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_or_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_or_BC_xor_BC_4x32:
@@ -150,6 +250,42 @@ entry:
   ret <2 x i64> %res
 }
 
+; Function to test ternary(A, or(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_or_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %or, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, or(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_or_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_or_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %or, <8 x i16> %xor
+  ret <8 x i16> %res
+}
+
 ; Function to test ternary(A, nor(B, C), xor(B, C)) for <4 x i32>
 define <4 x i32> @ternary_A_nor_BC_xor_BC_4x32(<4 x i1> %A, <4 x i32> %B, <4 x i32> %C) {
 ; CHECK-LABEL: ternary_A_nor_BC_xor_BC_4x32:
@@ -188,3 +324,41 @@ entry:
   %res = select <2 x i1> %A, <2 x i64> %nor, <2 x i64> %xor
   ret <2 x i64> %res
 }
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <16 x i8>
+define <16 x i8> @ternary_A_nor_BC_xor_BC_16x8(<16 x i1> %A, <16 x i8> %B, <16 x i8> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_16x8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltib v5, 7
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslb v2, v2, v5
+; CHECK-NEXT:    vsrab v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <16 x i8> %B, %C
+  %nor = xor <16 x i8> %or, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>    ; vector nor operation
+  %xor = xor <16 x i8> %B, %C
+  %res = select <16 x i1> %A, <16 x i8> %nor, <16 x i8> %xor
+  ret <16 x i8> %res
+}
+
+; Function to test ternary(A, nor(B, C), xor(B, C)) for <8 x i16>
+define <8 x i16> @ternary_A_nor_BC_xor_BC_8x16(<8 x i1> %A, <8 x i16> %B, <8 x i16> %C) {
+; CHECK-LABEL: ternary_A_nor_BC_xor_BC_8x16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxspltiw v5, 983055
+; CHECK-NEXT:    xxlnor vs0, v3, v4
+; CHECK-NEXT:    xxlxor vs1, v3, v4
+; CHECK-NEXT:    vslh v2, v2, v5
+; CHECK-NEXT:    vsrah v2, v2, v5
+; CHECK-NEXT:    xxsel v2, vs1, vs0, v2
+; CHECK-NEXT:    blr
+entry:
+  %or = or <8 x i16> %B, %C
+  %nor = xor <8 x i16> %or, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>    ; vector nor operation
+  %xor = xor <8 x i16> %B, %C
+  %res = select <8 x i1> %A, <8 x i16> %nor, <8 x i16> %xor
+  ret <8 x i16> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
index 88413291c26c..7f387a763b6d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-intrinsics.ll
@@ -972,19 +972,19 @@ define i1 @fpclass(float %x) {
 ; RV32I-NEXT:    slli a2, a0, 1
 ; RV32I-NEXT:    lui a3, 2048
 ; RV32I-NEXT:    lui a4, 1046528
-; RV32I-NEXT:    srli a2, a2, 1
+; RV32I-NEXT:    srli a5, a2, 1
 ; RV32I-NEXT:    addi a3, a3, -1
-; RV32I-NEXT:    addi a5, a2, -1
+; RV32I-NEXT:    xor a0, a0, a5
+; RV32I-NEXT:    xor a6, a5, a1
+; RV32I-NEXT:    sltu a1, a1, a5
+; RV32I-NEXT:    add a4, a5, a4
+; RV32I-NEXT:    addi a5, a5, -1
 ; RV32I-NEXT:    sltu a3, a5, a3
 ; RV32I-NEXT:    lui a5, 520192
-; RV32I-NEXT:    xor a0, a0, a2
-; RV32I-NEXT:    add a4, a2, a4
 ; RV32I-NEXT:    sltu a4, a4, a5
-; RV32I-NEXT:    xor a5, a2, a1
-; RV32I-NEXT:    sltu a1, a1, a2
 ; RV32I-NEXT:    seqz a2, a2
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    seqz a5, a6
 ; RV32I-NEXT:    and a3, a3, a0
 ; RV32I-NEXT:    or a2, a2, a5
 ; RV32I-NEXT:    and a0, a4, a0
@@ -1000,19 +1000,19 @@ define i1 @fpclass(float %x) {
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    li a3, 1
 ; RV64I-NEXT:    lui a4, 2048
-; RV64I-NEXT:    lui a5, 520192
-; RV64I-NEXT:    srli a2, a2, 33
-; RV64I-NEXT:    addi a6, a4, -1
-; RV64I-NEXT:    xor a0, a0, a2
-; RV64I-NEXT:    subw a3, a2, a3
-; RV64I-NEXT:    sltu a3, a3, a6
-; RV64I-NEXT:    xor a6, a2, a1
-; RV64I-NEXT:    sltu a1, a1, a2
-; RV64I-NEXT:    subw a4, a2, a4
+; RV64I-NEXT:    srli a5, a2, 33
+; RV64I-NEXT:    xor a0, a0, a5
+; RV64I-NEXT:    subw a3, a5, a3
+; RV64I-NEXT:    xor a6, a5, a1
+; RV64I-NEXT:    sltu a1, a1, a5
+; RV64I-NEXT:    subw a5, a5, a4
+; RV64I-NEXT:    addi a4, a4, -1
+; RV64I-NEXT:    sltu a3, a3, a4
+; RV64I-NEXT:    lui a4, 520192
 ; RV64I-NEXT:    seqz a2, a2
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    seqz a6, a6
-; RV64I-NEXT:    sltu a4, a4, a5
+; RV64I-NEXT:    sltu a4, a5, a4
 ; RV64I-NEXT:    and a3, a3, a0
 ; RV64I-NEXT:    or a2, a2, a6
 ; RV64I-NEXT:    or a1, a2, a1
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index c5188aa1918b..43e16606aae7 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -107,6 +107,7 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcisync %s -o - | FileCheck --check-prefix=RV32XQCISYNC %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesperf %s -o - | FileCheck --check-prefix=RV32XANDESPERF %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvbfhcvt %s -o - | FileCheck --check-prefix=RV32XANDESVBFHCVT %s
+; RUN: llc -mtriple=riscv32 -mattr=+xandesvsintload %s -o - | FileCheck --check-prefix=RV32XANDESVSINTLOAD %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvdot %s -o - | FileCheck --check-prefix=RV32XANDESVDOT %s
 ; RUN: llc -mtriple=riscv32 -mattr=+xandesvpackfph %s -o - | FileCheck --check-prefix=RV32XANDESVPACKFPH %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zaamo %s -o - | FileCheck --check-prefix=RV32ZAAMO %s
@@ -262,6 +263,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+xtheadvdot %s -o - | FileCheck --check-prefixes=CHECK,RV64XTHEADVDOT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesperf %s -o - | FileCheck --check-prefix=RV64XANDESPERF %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvbfhcvt %s -o - | FileCheck --check-prefix=RV64XANDESVBFHCVT %s
+; RUN: llc -mtriple=riscv64 -mattr=+xandesvsintload %s -o - | FileCheck --check-prefix=RV64XANDESVSINTLOAD %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvdot %s -o - | FileCheck --check-prefix=RV64XANDESVDOT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+xandesvpackfph %s -o - | FileCheck --check-prefix=RV64XANDESVPACKFPH %s
 ; RUN: llc -mtriple=riscv64 -mattr=+za64rs %s -o - | FileCheck --check-prefixes=CHECK,RV64ZA64RS %s
@@ -337,6 +339,7 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+sdext  %s -o - | FileCheck --check-prefix=RV64SDEXT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+sdtrig  %s -o - | FileCheck --check-prefix=RV64SDTRIG %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp %s -o - | FileCheck --check-prefix=RV64XQCCMP %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-p %s -o - | FileCheck --check-prefix=RV64P %s
 
 
 ; Tests for profile features.
@@ -460,8 +463,9 @@
 ; RV32XQCISYNC: attribute 5, "rv32i2p1_zca1p0_xqcisync0p3"
 ; RV32XANDESPERF: .attribute 5, "rv32i2p1_xandesperf5p0"
 ; RV32XANDESVBFHCVT: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvl32b1p0_xandesvbfhcvt5p0"
+; RV32XANDESVSINTLOAD: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvsintload5p0"
 ; RV32XANDESVDOT: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvdot5p0"
-; RV32XANDESVPACKFPH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfhmin1p0_zvl32b1p0_xandesvpackfph5p0"
+; RV32XANDESVPACKFPH: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_xandesvpackfph5p0"
 ; RV32ZAAMO: .attribute 5, "rv32i2p1_zaamo1p0"
 ; RV32ZALRSC: .attribute 5, "rv32i2p1_zalrsc1p0"
 ; RV32ZCA: .attribute 5, "rv32i2p1_zca1p0"
@@ -526,6 +530,7 @@
 ; RV32SUPM: .attribute 5, "rv32i2p1_supm1p0"
 ; RV32SMCTR: .attribute 5, "rv32i2p1_smctr1p0_sscsrind1p0"
 ; RV32SSCTR: .attribute 5, "rv32i2p1_sscsrind1p0_ssctr1p0"
+; RV32P: .attribute 5, "rv32i2p1_p0p14"
 
 ; RV64M: .attribute 5, "rv64i2p1_m2p0_zmmul1p0"
 ; RV64ZMMUL: .attribute 5, "rv64i2p1_zmmul1p0"
@@ -616,8 +621,9 @@
 ; RV64XTHEADVDOT: .attribute 5, "rv64i2p1_f2p2_d2p2_v1p0_zicsr2p0_zve32f1p0_zve32x1p0_zve64d1p0_zve64f1p0_zve64x1p0_zvl128b1p0_zvl32b1p0_zvl64b1p0_xtheadvdot1p0"
 ; RV64XANDESPERF: .attribute 5, "rv64i2p1_xandesperf5p0"
 ; RV64XANDESVBFHCVT: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvl32b1p0_xandesvbfhcvt5p0"
+; RV64XANDESVSINTLOAD: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvsintload5p0"
 ; RV64XANDESVDOT: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvl32b1p0_xandesvdot5p0"
-; RV64XANDESVPACKFPH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zve32f1p0_zve32x1p0_zvfhmin1p0_zvl32b1p0_xandesvpackfph5p0"
+; RV64XANDESVPACKFPH: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_xandesvpackfph5p0"
 ; RV64ZTSO: .attribute 5, "rv64i2p1_ztso1p0"
 ; RV64ZAAMO: .attribute 5, "rv64i2p1_zaamo1p0"
 ; RV64ZALRSC: .attribute 5, "rv64i2p1_zalrsc1p0"
@@ -688,6 +694,7 @@
 ; RV64SDEXT: .attribute 5, "rv64i2p1_sdext1p0"
 ; RV64SDTRIG: .attribute 5, "rv64i2p1_sdtrig1p0"
 ; RV64XQCCMP: .attribute 5, "rv64i2p1_zca1p0_xqccmp0p3"
+; RV64P: .attribute 5, "rv64i2p1_p0p14"
 
 ; RVI20U32: .attribute 5, "rv32i2p1"
 ; RVI20U64: .attribute 5, "rv64i2p1"
diff --git a/llvm/test/CodeGen/RISCV/branch-opt.mir b/llvm/test/CodeGen/RISCV/branch-opt.mir
index ba3a20f2fbfc..76ade18ce516 100644
--- a/llvm/test/CodeGen/RISCV/branch-opt.mir
+++ b/llvm/test/CodeGen/RISCV/branch-opt.mir
@@ -18,6 +18,74 @@
     ret void
   }
 
+  define void @ule_negone(ptr %a, i32 signext %b, ptr %c, ptr %d) {
+    store i32 0, ptr %a
+    %p = icmp ule i32 %b, -1
+    br i1 %p, label %block1, label %block2
+
+  block1:                                           ; preds = %0
+    store i32 %b, ptr %c
+    br label %end_block
+
+  block2:                                           ; preds = %0
+    store i32 87, ptr %d
+    br label %end_block
+
+  end_block:                                        ; preds = %block2, %block1
+    ret void
+  }
+
+  define void @ult_zero(ptr %a, i32 signext %b, ptr %c, ptr %d) {
+    store i32 -1, ptr %a
+    %p = icmp ult i32 %b, 0
+    br i1 %p, label %block1, label %block2
+
+  block1:                                           ; preds = %0
+    store i32 %b, ptr %c
+    br label %end_block
+
+  block2:                                           ; preds = %0
+    store i32 87, ptr %d
+    br label %end_block
+
+  end_block:                                        ; preds = %block2, %block1
+    ret void
+  }
+
+  define void @sle_zero(ptr %a, i32 signext %b, ptr %c, ptr %d) {
+    store i32 1, ptr %a
+    %p = icmp sle i32 %b, 0
+    br i1 %p, label %block1, label %block2
+
+  block1:                                           ; preds = %0
+    store i32 %b, ptr %c
+    br label %end_block
+
+  block2:                                           ; preds = %0
+    store i32 87, ptr %d
+    br label %end_block
+
+  end_block:                                        ; preds = %block2, %block1
+    ret void
+  }
+
+  define void @slt_zero(ptr %a, i32 signext %b, ptr %c, ptr %d) {
+    store i32 -1, ptr %a
+    %p = icmp slt i32 %b, 0
+    br i1 %p, label %block1, label %block2
+
+  block1:                                           ; preds = %0
+    store i32 %b, ptr %c
+    br label %end_block
+
+  block2:                                           ; preds = %0
+    store i32 87, ptr %d
+    br label %end_block
+
+  end_block:                                        ; preds = %block2, %block1
+    ret void
+  }
+
   declare void @bar(...)
 
 ...
@@ -66,3 +134,243 @@ body:             |
     PseudoRET
 
 ...
+---
+name:            ule_negone
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: ule_negone
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10, $x11, $x12, $x13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x13
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x12
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x11
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 0
+  ; CHECK-NEXT:   SW killed [[ADDI]], [[COPY3]], 0 :: (store (s32))
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI $x0, -1
+  ; CHECK-NEXT:   BLTU killed [[ADDI1]], [[COPY2]], %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[COPY2]]
+  ; CHECK-NEXT:   SW [[COPY4]], [[COPY1]], 0 :: (store (s32))
+  ; CHECK-NEXT:   PseudoBR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI2:%[0-9]+]]:gpr = ADDI $x0, 87
+  ; CHECK-NEXT:   SW killed [[ADDI2]], [[COPY]], 0 :: (store (s32))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   PseudoRET
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $x10, $x11, $x12, $x13
+
+    %3:gpr = COPY $x13
+    %2:gpr = COPY $x12
+    %1:gpr = COPY $x11
+    %0:gpr = COPY $x10
+    %5:gpr = ADDI $x0, 0
+    SW killed %5, %0, 0 :: (store (s32))
+    %6:gpr = ADDI $x0, -1
+    BLTU killed %6, %1, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    %4:gpr = COPY %1
+    SW %4, %2, 0 :: (store (s32))
+    PseudoBR %bb.3
+
+  bb.2:
+    %7:gpr = ADDI $x0, 87
+    SW killed %7, %3, 0 :: (store (s32))
+
+  bb.3:
+    PseudoRET
+...
+---
+name:            ult_zero
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: ult_zero
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10, $x11, $x12, $x13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x13
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x12
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x11
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, -1
+  ; CHECK-NEXT:   SW killed [[ADDI]], [[COPY3]], 0 :: (store (s32))
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI $x0, 0
+  ; CHECK-NEXT:   BLTU [[COPY2]], killed [[ADDI1]], %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[COPY2]]
+  ; CHECK-NEXT:   SW [[COPY4]], [[COPY1]], 0 :: (store (s32))
+  ; CHECK-NEXT:   PseudoBR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI2:%[0-9]+]]:gpr = ADDI $x0, 87
+  ; CHECK-NEXT:   SW killed [[ADDI2]], [[COPY]], 0 :: (store (s32))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   PseudoRET
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $x10, $x11, $x12, $x13
+
+    %3:gpr = COPY $x13
+    %2:gpr = COPY $x12
+    %1:gpr = COPY $x11
+    %0:gpr = COPY $x10
+    %5:gpr = ADDI $x0, -1
+    SW killed %5, %0, 0 :: (store (s32))
+    %6:gpr = ADDI $x0, 0
+    BLTU %1, killed %6, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    %4:gpr = COPY %1
+    SW %4, %2, 0 :: (store (s32))
+    PseudoBR %bb.3
+
+  bb.2:
+    %7:gpr = ADDI $x0, 87
+    SW killed %7, %3, 0 :: (store (s32))
+
+  bb.3:
+    PseudoRET
+...
+---
+name:            sle_zero
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: sle_zero
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10, $x11, $x12, $x13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x13
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x12
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x11
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+  ; CHECK-NEXT:   SW killed [[ADDI]], [[COPY3]], 0 :: (store (s32))
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI $x0, 0
+  ; CHECK-NEXT:   BLT killed [[ADDI1]], [[COPY2]], %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[COPY2]]
+  ; CHECK-NEXT:   SW [[COPY4]], [[COPY1]], 0 :: (store (s32))
+  ; CHECK-NEXT:   PseudoBR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI2:%[0-9]+]]:gpr = ADDI $x0, 87
+  ; CHECK-NEXT:   SW killed [[ADDI2]], [[COPY]], 0 :: (store (s32))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   PseudoRET
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $x10, $x11, $x12, $x13
+
+    %3:gpr = COPY $x13
+    %2:gpr = COPY $x12
+    %1:gpr = COPY $x11
+    %0:gpr = COPY $x10
+    %5:gpr = ADDI $x0, 1
+    SW killed %5, %0, 0 :: (store (s32))
+    %6:gpr = ADDI $x0, 0
+    BLT killed %6, %1, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    %4:gpr = COPY %1
+    SW %4, %2, 0 :: (store (s32))
+    PseudoBR %bb.3
+
+  bb.2:
+    %7:gpr = ADDI $x0, 87
+    SW killed %7, %3, 0 :: (store (s32))
+
+  bb.3:
+    PseudoRET
+...
+---
+name:            slt_zero
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: slt_zero
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $x10, $x11, $x12, $x13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr = COPY $x13
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x12
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x11
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr = COPY $x10
+  ; CHECK-NEXT:   [[ADDI:%[0-9]+]]:gpr = ADDI $x0, -1
+  ; CHECK-NEXT:   SW killed [[ADDI]], [[COPY3]], 0 :: (store (s32))
+  ; CHECK-NEXT:   [[ADDI1:%[0-9]+]]:gpr = ADDI $x0, 0
+  ; CHECK-NEXT:   BLT [[COPY2]], killed [[ADDI1]], %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr = COPY [[COPY2]]
+  ; CHECK-NEXT:   SW [[COPY4]], [[COPY1]], 0 :: (store (s32))
+  ; CHECK-NEXT:   PseudoBR %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ADDI2:%[0-9]+]]:gpr = ADDI $x0, 87
+  ; CHECK-NEXT:   SW killed [[ADDI2]], [[COPY]], 0 :: (store (s32))
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   PseudoRET
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $x10, $x11, $x12, $x13
+
+    %3:gpr = COPY $x13
+    %2:gpr = COPY $x12
+    %1:gpr = COPY $x11
+    %0:gpr = COPY $x10
+    %5:gpr = ADDI $x0, -1
+    SW killed %5, %0, 0 :: (store (s32))
+    %6:gpr = ADDI $x0, 0
+    BLT %1, killed %6, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    %4:gpr = COPY %1
+    SW %4, %2, 0 :: (store (s32))
+    PseudoBR %bb.3
+
+  bb.2:
+    %7:gpr = ADDI $x0, 87
+    SW killed %7, %3, 0 :: (store (s32))
+
+  bb.3:
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
index e7ff99141301..1deea55b083c 100644
--- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -638,3 +638,48 @@ define signext i32 @select_fcmp_uge_1_2(double %a, double %b) nounwind {
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
 }
+
+define double @CascadedSelect(double noundef %a) {
+; CHECKRV32ZDINX-LABEL: CascadedSelect:
+; CHECKRV32ZDINX:       # %bb.0: # %entry
+; CHECKRV32ZDINX-NEXT:    lui a3, %hi(.LCPI20_0)
+; CHECKRV32ZDINX-NEXT:    lw a2, %lo(.LCPI20_0)(a3)
+; CHECKRV32ZDINX-NEXT:    addi a3, a3, %lo(.LCPI20_0)
+; CHECKRV32ZDINX-NEXT:    lw a3, 4(a3)
+; CHECKRV32ZDINX-NEXT:    flt.d a4, a2, a0
+; CHECKRV32ZDINX-NEXT:    bnez a4, .LBB20_3
+; CHECKRV32ZDINX-NEXT:  # %bb.1: # %entry
+; CHECKRV32ZDINX-NEXT:    flt.d a4, a0, zero
+; CHECKRV32ZDINX-NEXT:    li a2, 0
+; CHECKRV32ZDINX-NEXT:    li a3, 0
+; CHECKRV32ZDINX-NEXT:    bnez a4, .LBB20_3
+; CHECKRV32ZDINX-NEXT:  # %bb.2: # %entry
+; CHECKRV32ZDINX-NEXT:    mv a2, a0
+; CHECKRV32ZDINX-NEXT:    mv a3, a1
+; CHECKRV32ZDINX-NEXT:  .LBB20_3: # %entry
+; CHECKRV32ZDINX-NEXT:    mv a0, a2
+; CHECKRV32ZDINX-NEXT:    mv a1, a3
+; CHECKRV32ZDINX-NEXT:    ret
+;
+; CHECKRV64ZDINX-LABEL: CascadedSelect:
+; CHECKRV64ZDINX:       # %bb.0: # %entry
+; CHECKRV64ZDINX-NEXT:    li a1, 1023
+; CHECKRV64ZDINX-NEXT:    slli a1, a1, 52
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a1, a0
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB20_3
+; CHECKRV64ZDINX-NEXT:  # %bb.1: # %entry
+; CHECKRV64ZDINX-NEXT:    flt.d a2, a0, zero
+; CHECKRV64ZDINX-NEXT:    li a1, 0
+; CHECKRV64ZDINX-NEXT:    bnez a2, .LBB20_3
+; CHECKRV64ZDINX-NEXT:  # %bb.2: # %entry
+; CHECKRV64ZDINX-NEXT:    mv a1, a0
+; CHECKRV64ZDINX-NEXT:  .LBB20_3: # %entry
+; CHECKRV64ZDINX-NEXT:    mv a0, a1
+; CHECKRV64ZDINX-NEXT:    ret
+entry:
+  %cmp = fcmp ogt double %a, 1.000000e+00
+  %cmp1 = fcmp olt double %a, 0.000000e+00
+  %.a = select i1 %cmp1, double 0.000000e+00, double %a
+  %retval.0 = select i1 %cmp, double 1.000000e+00, double %.a
+  ret double %retval.0
+}
diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll
index 8b931f70aa5c..95d330889d2c 100644
--- a/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/llvm/test/CodeGen/RISCV/features-info.ll
@@ -61,7 +61,6 @@
 ; CHECK-NEXT:   m                                - 'M' (Integer Multiplication and Division).
 ; CHECK-NEXT:   mips-p8700                       - MIPS p8700 processor.
 ; CHECK-NEXT:   no-default-unroll                - Disable default unroll preference..
-; CHECK-NEXT:   no-rvc-hints                     - Disable RVC Hint Instructions..
 ; CHECK-NEXT:   no-sink-splat-operands           - Disable sink splat operands to enable .vx, .vf,.wx, and .wf instructions.
 ; CHECK-NEXT:   no-trailing-seq-cst-fence        - Disable trailing fence for seq-cst store..
 ; CHECK-NEXT:   optimized-nf2-segment-load-store - vlseg2eN.v and vsseg2eN.v are implemented as a wide memory op and shuffle.
@@ -177,6 +176,7 @@
 ; CHECK-NEXT:   xandesvbfhcvt                    - 'XAndesVBFHCvt' (Andes Vector BFLOAT16 Conversion Extension).
 ; CHECK-NEXT:   xandesvdot                       - 'XAndesVDot' (Andes Vector Dot Product Extension).
 ; CHECK-NEXT:   xandesvpackfph                   - 'XAndesVPackFPH' (Andes Vector Packed FP16 Extension).
+; CHECK-NEXT:   xandesvsintload                  - 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension).
 ; CHECK-NEXT:   xcvalu                           - 'XCValu' (CORE-V ALU Operations).
 ; CHECK-NEXT:   xcvbi                            - 'XCVbi' (CORE-V Immediate Branching).
 ; CHECK-NEXT:   xcvbitmanip                      - 'XCVbitmanip' (CORE-V Bit Manipulation).
@@ -184,6 +184,7 @@
 ; CHECK-NEXT:   xcvmac                           - 'XCVmac' (CORE-V Multiply-Accumulate).
 ; CHECK-NEXT:   xcvmem                           - 'XCVmem' (CORE-V Post-incrementing Load & Store).
 ; CHECK-NEXT:   xcvsimd                          - 'XCVsimd' (CORE-V SIMD ALU).
+; CHECK-NEXT:   xmipscbop                        - 'XMIPSCBOP' (MIPS Software Prefetch).
 ; CHECK-NEXT:   xmipscmov                        - 'XMIPSCMov' (MIPS conditional move instruction (mips.ccmov)).
 ; CHECK-NEXT:   xmipslsp                         - 'XMIPSLSP' (MIPS optimization for hardware load-store bonding).
 ; CHECK-NEXT:   xsfcease                         - 'XSfcease' (SiFive sf.cease Instruction).
@@ -199,7 +200,7 @@
 ; CHECK-NEXT:   xsfmmbase                        - 'XSfmmbase' (All non arithmetic instructions for all TEWs and sf.vtzero).
 ; CHECK-NEXT:   xsfvcp                           - 'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions).
 ; CHECK-NEXT:   xsfvfnrclipxfqf                  - 'XSfvfnrclipxfqf' (SiFive FP32-to-int8 Ranged Clip Instructions).
-; CHECK-NEXT:   xsfvfwmaccqqq                    - 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4)).
+; CHECK-NEXT:   xsfvfwmaccqqq                    - 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4)).
 ; CHECK-NEXT:   xsfvqmaccdod                     - 'XSfvqmaccdod' (SiFive Int8 Matrix Multiplication Instructions (2-by-8 and 8-by-2)).
 ; CHECK-NEXT:   xsfvqmaccqoq                     - 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4)).
 ; CHECK-NEXT:   xsifivecdiscarddlone             - 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction).
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 797d59dfcdc8..1cb7b27dd69e 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -643,12 +643,12 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32IF-NEXT:    addi a2, a3, -1
 ; RV32IF-NEXT:  .LBB12_4: # %start
 ; RV32IF-NEXT:    feq.s a3, fs0, fs0
-; RV32IF-NEXT:    neg a4, s0
-; RV32IF-NEXT:    neg a5, a1
+; RV32IF-NEXT:    neg a4, a1
+; RV32IF-NEXT:    neg a1, s0
 ; RV32IF-NEXT:    neg a3, a3
-; RV32IF-NEXT:    and a0, a4, a0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    and a1, a3, a2
-; RV32IF-NEXT:    or a0, a5, a0
+; RV32IF-NEXT:    or a0, a4, a0
 ; RV32IF-NEXT:    and a0, a3, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index be9ddc68ce66..ed50042f54ab 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -1625,18 +1625,18 @@ define i1 @fpclass(float %x) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    slli a1, a0, 1
 ; RV32I-NEXT:    lui a2, 2048
-; RV32I-NEXT:    slti a0, a0, 0
 ; RV32I-NEXT:    lui a3, 522240
 ; RV32I-NEXT:    lui a4, 1046528
-; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    srli a5, a1, 1
 ; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    addi a5, a1, -1
+; RV32I-NEXT:    xor a6, a5, a3
+; RV32I-NEXT:    slt a3, a3, a5
+; RV32I-NEXT:    add a4, a5, a4
+; RV32I-NEXT:    addi a5, a5, -1
 ; RV32I-NEXT:    sltu a2, a5, a2
-; RV32I-NEXT:    xor a5, a1, a3
-; RV32I-NEXT:    slt a3, a3, a1
-; RV32I-NEXT:    add a4, a1, a4
+; RV32I-NEXT:    slti a0, a0, 0
 ; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    seqz a5, a5
+; RV32I-NEXT:    seqz a5, a6
 ; RV32I-NEXT:    srli a4, a4, 24
 ; RV32I-NEXT:    and a2, a2, a0
 ; RV32I-NEXT:    or a1, a1, a5
@@ -1649,29 +1649,29 @@ define i1 @fpclass(float %x) {
 ;
 ; RV64I-LABEL: fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    slli a1, a0, 33
 ; RV64I-NEXT:    lui a2, 2048
 ; RV64I-NEXT:    lui a3, 522240
 ; RV64I-NEXT:    lui a4, 1046528
-; RV64I-NEXT:    srli a0, a0, 33
+; RV64I-NEXT:    srli a5, a1, 33
 ; RV64I-NEXT:    addi a2, a2, -1
-; RV64I-NEXT:    slti a1, a1, 0
-; RV64I-NEXT:    addi a5, a0, -1
+; RV64I-NEXT:    xor a6, a5, a3
+; RV64I-NEXT:    slt a3, a3, a5
+; RV64I-NEXT:    add a4, a5, a4
+; RV64I-NEXT:    addi a5, a5, -1
 ; RV64I-NEXT:    sltu a2, a5, a2
-; RV64I-NEXT:    xor a5, a0, a3
-; RV64I-NEXT:    slt a3, a3, a0
-; RV64I-NEXT:    add a4, a0, a4
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    seqz a5, a5
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    seqz a1, a1
+; RV64I-NEXT:    seqz a5, a6
 ; RV64I-NEXT:    srliw a4, a4, 24
-; RV64I-NEXT:    and a2, a2, a1
-; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    and a2, a2, a0
+; RV64I-NEXT:    or a1, a1, a5
 ; RV64I-NEXT:    sltiu a4, a4, 127
-; RV64I-NEXT:    or a0, a0, a2
-; RV64I-NEXT:    or a0, a0, a3
-; RV64I-NEXT:    and a1, a4, a1
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    or a1, a1, a3
+; RV64I-NEXT:    and a0, a4, a0
+; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
   %cmp = call i1 @llvm.is.fpclass.f32(float %x, i32 639)
   ret i1 %cmp
diff --git a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
index a2ff0d33e2d3..f08777ac3e5d 100644
--- a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -451,3 +451,42 @@ define signext i32 @select_fcmp_uge_1_2(float %a, float %b) nounwind {
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
 }
+
+define float @CascadedSelect(float noundef %a) {
+; CHECK-LABEL: CascadedSelect:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a0, 260096
+; CHECK-NEXT:    fmv.w.x fa5, a0
+; CHECK-NEXT:    flt.s a0, fa5, fa0
+; CHECK-NEXT:    bnez a0, .LBB20_3
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    fmv.w.x fa5, zero
+; CHECK-NEXT:    flt.s a0, fa0, fa5
+; CHECK-NEXT:    bnez a0, .LBB20_3
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    fmv.s fa5, fa0
+; CHECK-NEXT:  .LBB20_3: # %entry
+; CHECK-NEXT:    fmv.s fa0, fa5
+; CHECK-NEXT:    ret
+;
+; CHECKZFINX-LABEL: CascadedSelect:
+; CHECKZFINX:       # %bb.0: # %entry
+; CHECKZFINX-NEXT:    lui a1, 260096
+; CHECKZFINX-NEXT:    flt.s a2, a1, a0
+; CHECKZFINX-NEXT:    bnez a2, .LBB20_3
+; CHECKZFINX-NEXT:  # %bb.1: # %entry
+; CHECKZFINX-NEXT:    flt.s a2, a0, zero
+; CHECKZFINX-NEXT:    li a1, 0
+; CHECKZFINX-NEXT:    bnez a2, .LBB20_3
+; CHECKZFINX-NEXT:  # %bb.2: # %entry
+; CHECKZFINX-NEXT:    mv a1, a0
+; CHECKZFINX-NEXT:  .LBB20_3: # %entry
+; CHECKZFINX-NEXT:    mv a0, a1
+; CHECKZFINX-NEXT:    ret
+entry:
+  %cmp = fcmp ogt float %a, 1.000000e+00
+  %cmp1 = fcmp olt float %a, 0.000000e+00
+  %.a = select i1 %cmp1, float 0.000000e+00, float %a
+  %retval.0 = select i1 %cmp, float 1.000000e+00, float %.a
+  ret float %retval.0
+}
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index 117e3e4aac45..246e6a614d6a 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -3292,30 +3292,30 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    mv a1, a0
 ; RV32IF-NEXT:    addi a0, sp, 8
 ; RV32IF-NEXT:    call __fixdfti
-; RV32IF-NEXT:    lw a0, 20(sp)
-; RV32IF-NEXT:    lw a1, 8(sp)
-; RV32IF-NEXT:    lw a2, 12(sp)
+; RV32IF-NEXT:    lw a0, 8(sp)
+; RV32IF-NEXT:    lw a1, 12(sp)
+; RV32IF-NEXT:    lw a2, 20(sp)
 ; RV32IF-NEXT:    lw a3, 16(sp)
-; RV32IF-NEXT:    beqz a0, .LBB47_2
+; RV32IF-NEXT:    beqz a2, .LBB47_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a4, a0, 0
+; RV32IF-NEXT:    slti a4, a2, 0
 ; RV32IF-NEXT:    j .LBB47_3
 ; RV32IF-NEXT:  .LBB47_2:
 ; RV32IF-NEXT:    seqz a4, a3
 ; RV32IF-NEXT:  .LBB47_3: # %entry
 ; RV32IF-NEXT:    xori a3, a3, 1
-; RV32IF-NEXT:    or a3, a3, a0
+; RV32IF-NEXT:    or a3, a3, a2
 ; RV32IF-NEXT:    seqz a3, a3
 ; RV32IF-NEXT:    addi a3, a3, -1
 ; RV32IF-NEXT:    and a3, a3, a4
 ; RV32IF-NEXT:    neg a3, a3
-; RV32IF-NEXT:    and a2, a3, a2
 ; RV32IF-NEXT:    and a1, a3, a1
 ; RV32IF-NEXT:    and a0, a3, a0
-; RV32IF-NEXT:    slti a0, a0, 0
-; RV32IF-NEXT:    addi a3, a0, -1
-; RV32IF-NEXT:    and a0, a3, a1
-; RV32IF-NEXT:    and a1, a3, a2
+; RV32IF-NEXT:    and a2, a3, a2
+; RV32IF-NEXT:    slti a2, a2, 0
+; RV32IF-NEXT:    addi a2, a2, -1
+; RV32IF-NEXT:    and a0, a2, a0
+; RV32IF-NEXT:    and a1, a2, a1
 ; RV32IF-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    .cfi_restore ra
 ; RV32IF-NEXT:    addi sp, sp, 32
@@ -3354,30 +3354,30 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    .cfi_offset ra, -4
 ; RV32IFD-NEXT:    addi a0, sp, 8
 ; RV32IFD-NEXT:    call __fixdfti
-; RV32IFD-NEXT:    lw a0, 20(sp)
-; RV32IFD-NEXT:    lw a1, 8(sp)
-; RV32IFD-NEXT:    lw a2, 12(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    lw a2, 20(sp)
 ; RV32IFD-NEXT:    lw a3, 16(sp)
-; RV32IFD-NEXT:    beqz a0, .LBB47_2
+; RV32IFD-NEXT:    beqz a2, .LBB47_2
 ; RV32IFD-NEXT:  # %bb.1: # %entry
-; RV32IFD-NEXT:    slti a4, a0, 0
+; RV32IFD-NEXT:    slti a4, a2, 0
 ; RV32IFD-NEXT:    j .LBB47_3
 ; RV32IFD-NEXT:  .LBB47_2:
 ; RV32IFD-NEXT:    seqz a4, a3
 ; RV32IFD-NEXT:  .LBB47_3: # %entry
 ; RV32IFD-NEXT:    xori a3, a3, 1
-; RV32IFD-NEXT:    or a3, a3, a0
+; RV32IFD-NEXT:    or a3, a3, a2
 ; RV32IFD-NEXT:    seqz a3, a3
 ; RV32IFD-NEXT:    addi a3, a3, -1
 ; RV32IFD-NEXT:    and a3, a3, a4
 ; RV32IFD-NEXT:    neg a3, a3
-; RV32IFD-NEXT:    and a2, a3, a2
 ; RV32IFD-NEXT:    and a1, a3, a1
 ; RV32IFD-NEXT:    and a0, a3, a0
-; RV32IFD-NEXT:    slti a0, a0, 0
-; RV32IFD-NEXT:    addi a3, a0, -1
-; RV32IFD-NEXT:    and a0, a3, a1
-; RV32IFD-NEXT:    and a1, a3, a2
+; RV32IFD-NEXT:    and a2, a3, a2
+; RV32IFD-NEXT:    slti a2, a2, 0
+; RV32IFD-NEXT:    addi a2, a2, -1
+; RV32IFD-NEXT:    and a0, a2, a0
+; RV32IFD-NEXT:    and a1, a2, a1
 ; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    .cfi_restore ra
 ; RV32IFD-NEXT:    addi sp, sp, 32
@@ -3530,30 +3530,30 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    call __fixsfti
-; RV32-NEXT:    lw a0, 20(sp)
-; RV32-NEXT:    lw a1, 8(sp)
-; RV32-NEXT:    lw a2, 12(sp)
+; RV32-NEXT:    lw a0, 8(sp)
+; RV32-NEXT:    lw a1, 12(sp)
+; RV32-NEXT:    lw a2, 20(sp)
 ; RV32-NEXT:    lw a3, 16(sp)
-; RV32-NEXT:    beqz a0, .LBB50_2
+; RV32-NEXT:    beqz a2, .LBB50_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a0, 0
+; RV32-NEXT:    slti a4, a2, 0
 ; RV32-NEXT:    j .LBB50_3
 ; RV32-NEXT:  .LBB50_2:
 ; RV32-NEXT:    seqz a4, a3
 ; RV32-NEXT:  .LBB50_3: # %entry
 ; RV32-NEXT:    xori a3, a3, 1
-; RV32-NEXT:    or a3, a3, a0
+; RV32-NEXT:    or a3, a3, a2
 ; RV32-NEXT:    seqz a3, a3
 ; RV32-NEXT:    addi a3, a3, -1
 ; RV32-NEXT:    and a3, a3, a4
 ; RV32-NEXT:    neg a3, a3
-; RV32-NEXT:    and a2, a3, a2
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    addi a3, a0, -1
-; RV32-NEXT:    and a0, a3, a1
-; RV32-NEXT:    and a1, a3, a2
+; RV32-NEXT:    and a2, a3, a2
+; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a0, a2, a0
+; RV32-NEXT:    and a1, a2, a1
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore ra
 ; RV32-NEXT:    addi sp, sp, 32
@@ -3767,30 +3767,30 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; RV32-NEXT:    call __extendhfsf2
 ; RV32-NEXT:    addi a0, sp, 8
 ; RV32-NEXT:    call __fixsfti
-; RV32-NEXT:    lw a0, 20(sp)
-; RV32-NEXT:    lw a1, 8(sp)
-; RV32-NEXT:    lw a2, 12(sp)
+; RV32-NEXT:    lw a0, 8(sp)
+; RV32-NEXT:    lw a1, 12(sp)
+; RV32-NEXT:    lw a2, 20(sp)
 ; RV32-NEXT:    lw a3, 16(sp)
-; RV32-NEXT:    beqz a0, .LBB53_2
+; RV32-NEXT:    beqz a2, .LBB53_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a0, 0
+; RV32-NEXT:    slti a4, a2, 0
 ; RV32-NEXT:    j .LBB53_3
 ; RV32-NEXT:  .LBB53_2:
 ; RV32-NEXT:    seqz a4, a3
 ; RV32-NEXT:  .LBB53_3: # %entry
 ; RV32-NEXT:    xori a3, a3, 1
-; RV32-NEXT:    or a3, a3, a0
+; RV32-NEXT:    or a3, a3, a2
 ; RV32-NEXT:    seqz a3, a3
 ; RV32-NEXT:    addi a3, a3, -1
 ; RV32-NEXT:    and a3, a3, a4
 ; RV32-NEXT:    neg a3, a3
-; RV32-NEXT:    and a2, a3, a2
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
-; RV32-NEXT:    slti a0, a0, 0
-; RV32-NEXT:    addi a3, a0, -1
-; RV32-NEXT:    and a0, a3, a1
-; RV32-NEXT:    and a1, a3, a2
+; RV32-NEXT:    and a2, a3, a2
+; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    addi a2, a2, -1
+; RV32-NEXT:    and a0, a2, a0
+; RV32-NEXT:    and a1, a2, a1
 ; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    .cfi_restore ra
 ; RV32-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/half-select-fcmp.ll b/llvm/test/CodeGen/RISCV/half-select-fcmp.ll
index d92dcb9eac4c..bf535b1cbd08 100644
--- a/llvm/test/CodeGen/RISCV/half-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/half-select-fcmp.ll
@@ -874,3 +874,87 @@ define signext i32 @select_fcmp_uge_1_2(half %a, half %b) nounwind {
   %2 = select i1 %1, i32 1, i32 2
   ret i32 %2
 }
+
+define half @CascadedSelect(half noundef %a) {
+; CHECK-LABEL: CascadedSelect:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECK-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
+; CHECK-NEXT:    flt.h a0, fa5, fa0
+; CHECK-NEXT:    bnez a0, .LBB20_3
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    fmv.h.x fa5, zero
+; CHECK-NEXT:    flt.h a0, fa0, fa5
+; CHECK-NEXT:    bnez a0, .LBB20_3
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    fmv.h fa5, fa0
+; CHECK-NEXT:  .LBB20_3: # %entry
+; CHECK-NEXT:    fmv.h fa0, fa5
+; CHECK-NEXT:    ret
+;
+; CHECKIZHINX-LABEL: CascadedSelect:
+; CHECKIZHINX:       # %bb.0: # %entry
+; CHECKIZHINX-NEXT:    li a1, 15
+; CHECKIZHINX-NEXT:    slli a1, a1, 10
+; CHECKIZHINX-NEXT:    flt.h a2, a1, a0
+; CHECKIZHINX-NEXT:    bnez a2, .LBB20_3
+; CHECKIZHINX-NEXT:  # %bb.1: # %entry
+; CHECKIZHINX-NEXT:    flt.h a2, a0, zero
+; CHECKIZHINX-NEXT:    li a1, 0
+; CHECKIZHINX-NEXT:    bnez a2, .LBB20_3
+; CHECKIZHINX-NEXT:  # %bb.2: # %entry
+; CHECKIZHINX-NEXT:    mv a1, a0
+; CHECKIZHINX-NEXT:  .LBB20_3: # %entry
+; CHECKIZHINX-NEXT:    mv a0, a1
+; CHECKIZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: CascadedSelect:
+; CHECKIZFHMIN:       # %bb.0: # %entry
+; CHECKIZFHMIN-NEXT:    lui a0, %hi(.LCPI20_0)
+; CHECKIZFHMIN-NEXT:    flh fa5, %lo(.LCPI20_0)(a0)
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa3, fa5
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa4, fa0
+; CHECKIZFHMIN-NEXT:    flt.s a0, fa3, fa4
+; CHECKIZFHMIN-NEXT:    bnez a0, .LBB20_3
+; CHECKIZFHMIN-NEXT:  # %bb.1: # %entry
+; CHECKIZFHMIN-NEXT:    fmv.w.x fa5, zero
+; CHECKIZFHMIN-NEXT:    flt.s a0, fa4, fa5
+; CHECKIZFHMIN-NEXT:    bnez a0, .LBB20_4
+; CHECKIZFHMIN-NEXT:  # %bb.2: # %entry
+; CHECKIZFHMIN-NEXT:    fmv.s fa5, fa0
+; CHECKIZFHMIN-NEXT:  .LBB20_3: # %entry
+; CHECKIZFHMIN-NEXT:    fmv.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+; CHECKIZFHMIN-NEXT:  .LBB20_4:
+; CHECKIZFHMIN-NEXT:    fmv.h.x fa0, zero
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: CascadedSelect:
+; CHECKIZHINXMIN:       # %bb.0: # %entry
+; CHECKIZHINXMIN-NEXT:    mv a1, a0
+; CHECKIZHINXMIN-NEXT:    li a0, 0
+; CHECKIZHINXMIN-NEXT:    fcvt.s.h a2, a1
+; CHECKIZHINXMIN-NEXT:    lui a3, 260096
+; CHECKIZHINXMIN-NEXT:    flt.s a4, a2, zero
+; CHECKIZHINXMIN-NEXT:    flt.s a2, a3, a2
+; CHECKIZHINXMIN-NEXT:    beqz a4, .LBB20_3
+; CHECKIZHINXMIN-NEXT:  # %bb.1: # %entry
+; CHECKIZHINXMIN-NEXT:    bnez a2, .LBB20_4
+; CHECKIZHINXMIN-NEXT:  .LBB20_2: # %entry
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
+; CHECKIZHINXMIN-NEXT:    ret
+; CHECKIZHINXMIN-NEXT:  .LBB20_3: # %entry
+; CHECKIZHINXMIN-NEXT:    mv a0, a1
+; CHECKIZHINXMIN-NEXT:    beqz a2, .LBB20_2
+; CHECKIZHINXMIN-NEXT:  .LBB20_4:
+; CHECKIZHINXMIN-NEXT:    li a0, 15
+; CHECKIZHINXMIN-NEXT:    slli a0, a0, 10
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
+; CHECKIZHINXMIN-NEXT:    ret
+entry:
+  %cmp = fcmp ogt half %a, 1.000000e+00
+  %cmp1 = fcmp olt half %a, 0.000000e+00
+  %.a = select i1 %cmp1, half 0.000000e+00, half %a
+  %retval.0 = select i1 %cmp, half 1.000000e+00, half %.a
+  ret half %retval.0
+}
diff --git a/llvm/test/CodeGen/RISCV/i32-icmp.ll b/llvm/test/CodeGen/RISCV/i32-icmp.ll
index 6e3e0fe39cca..53892f9497bb 100644
--- a/llvm/test/CodeGen/RISCV/i32-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/i32-icmp.ll
@@ -1136,3 +1136,57 @@ define i32 @icmp_sle_constant_neg_2050(i32 %a) nounwind {
   %2 = zext i1 %1 to i32
   ret i32 %2
 }
+
+define i32 @mask_test_eq(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_eq:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 12
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_eq:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    slli a0, a0, 12
+; RV32XQCILIA-NEXT:    seqz a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 1048575
+  %cmp = icmp eq i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @mask_test_ne(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_ne:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a0, a0, 12
+; RV32I-NEXT:    snez a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_ne:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    slli a0, a0, 12
+; RV32XQCILIA-NEXT:    snez a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 1048575
+  %cmp = icmp ne i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
+
+define i32 @mask_test_eq_simm12(i32 %x) nounwind {
+; RV32I-LABEL: mask_test_eq_simm12:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, 3
+; RV32I-NEXT:    seqz a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCILIA-LABEL: mask_test_eq_simm12:
+; RV32XQCILIA:       # %bb.0:
+; RV32XQCILIA-NEXT:    andi a0, a0, 3
+; RV32XQCILIA-NEXT:    seqz a0, a0
+; RV32XQCILIA-NEXT:    ret
+  %y = and i32 %x, 3
+  %cmp = icmp eq i32 %y, 0
+  %ext = zext i1 %cmp to i32
+  ret i32 %ext
+}
diff --git a/llvm/test/CodeGen/RISCV/i64-icmp.ll b/llvm/test/CodeGen/RISCV/i64-icmp.ll
index 49103231a075..837987d8b916 100644
--- a/llvm/test/CodeGen/RISCV/i64-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/i64-icmp.ll
@@ -767,4 +767,56 @@ define i64 @icmp_ne_zext_inreg_umin(i64 %a) nounwind {
   %4 = zext i1 %3 to i64
   ret i64 %4
 }
+
+define i64 @mask_test_eq(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_eq:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_ne(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_ne:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    snez a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  %cmp = icmp ne i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_eq_simm12(i64 %x) nounwind {
+; RV64I-LABEL: mask_test_eq_simm12:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a0, a0, 3
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 3
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
+define i64 @mask_test_eq_multiuse(i64 %x, ptr %p) nounwind {
+; RV64I-LABEL: mask_test_eq_multiuse:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    seqz a0, a0
+; RV64I-NEXT:    sd a2, 0(a1)
+; RV64I-NEXT:    ret
+  %y = and i64 %x, 4611686018427387903
+  store i64 %y, ptr %p, align 8
+  %cmp = icmp eq i64 %y, 0
+  %ext = zext i1 %cmp to i64
+  ret i64 %ext
+}
+
 declare i64 @llvm.umin.i64(i64, i64)
diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
index 97d102561129..b1a6d163664e 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
@@ -7,18 +7,18 @@
 define i32 @ctz_nxv4i32(<vscale x 4 x i32> %a) #0 {
 ; RV32-LABEL: ctz_nxv4i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; RV32-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vid.v v10
-; RV32-NEXT:    li a1, -1
+; RV32-NEXT:    vmv.v.i v11, -1
+; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v8, 0
 ; RV32-NEXT:    srli a0, a0, 1
 ; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vmadd.vx v10, a1, v8
-; RV32-NEXT:    vmv.v.i v8, 0
-; RV32-NEXT:    vmerge.vvm v8, v8, v10, v0
+; RV32-NEXT:    vmacc.vv v8, v10, v11
+; RV32-NEXT:    vmv.v.i v9, 0
+; RV32-NEXT:    vmerge.vvm v8, v9, v8, v0
 ; RV32-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV32-NEXT:    vmv.x.s a1, v8
 ; RV32-NEXT:    sub a0, a0, a1
@@ -28,18 +28,18 @@ define i32 @ctz_nxv4i32(<vscale x 4 x i32> %a) #0 {
 ;
 ; RV64-LABEL: ctz_nxv4i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; RV64-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vid.v v10
-; RV64-NEXT:    li a1, -1
+; RV64-NEXT:    vmv.v.i v11, -1
+; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v8, 0
 ; RV64-NEXT:    srli a0, a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vmadd.vx v10, a1, v8
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vvm v8, v8, v10, v0
+; RV64-NEXT:    vmacc.vv v8, v10, v11
+; RV64-NEXT:    vmv.v.i v9, 0
+; RV64-NEXT:    vmerge.vvm v8, v9, v8, v0
 ; RV64-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV64-NEXT:    vmv.x.s a1, v8
 ; RV64-NEXT:    subw a0, a0, a1
@@ -109,17 +109,17 @@ define i64 @ctz_nxv8i1_no_range(<vscale x 8 x i16> %a) {
 ;
 ; RV64-LABEL: ctz_nxv8i1_no_range:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vid.v v16
-; RV64-NEXT:    li a1, -1
+; RV64-NEXT:    vmv.v.i v24, -1
+; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v8, 0
 ; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vmadd.vx v16, a1, v8
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vvm v8, v8, v16, v0
+; RV64-NEXT:    vmacc.vv v8, v16, v24
+; RV64-NEXT:    vmv.v.i v16, 0
+; RV64-NEXT:    vmerge.vvm v8, v16, v8, v0
 ; RV64-NEXT:    vredmaxu.vs v8, v8, v8
 ; RV64-NEXT:    vmv.x.s a1, v8
 ; RV64-NEXT:    sub a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
index 98c897084ab4..47b90a006a24 100644
--- a/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/overflow-intrinsics.ll
@@ -594,7 +594,6 @@ define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, ptr %p) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi a2, a0, 1
 ; RV32-NEXT:    slli a0, a2, 16
-; RV32-NEXT:    srli a0, a0, 16
 ; RV32-NEXT:    seqz a0, a0
 ; RV32-NEXT:    sh a2, 0(a1)
 ; RV32-NEXT:    ret
@@ -603,7 +602,6 @@ define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, ptr %p) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi a2, a0, 1
 ; RV64-NEXT:    slli a0, a2, 48
-; RV64-NEXT:    srli a0, a0, 48
 ; RV64-NEXT:    seqz a0, a0
 ; RV64-NEXT:    sh a2, 0(a1)
 ; RV64-NEXT:    ret
@@ -759,10 +757,9 @@ define i1 @uaddo_i42_increment_illegal_type(i42 %x, ptr %p) {
 ; RV64-LABEL: uaddo_i42_increment_illegal_type:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi a2, a0, 1
-; RV64-NEXT:    slli a0, a2, 22
-; RV64-NEXT:    srli a3, a0, 22
+; RV64-NEXT:    slli a3, a2, 22
 ; RV64-NEXT:    seqz a0, a3
-; RV64-NEXT:    srli a3, a3, 32
+; RV64-NEXT:    srli a3, a3, 54
 ; RV64-NEXT:    sw a2, 0(a1)
 ; RV64-NEXT:    sh a3, 4(a1)
 ; RV64-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/pr145363.ll b/llvm/test/CodeGen/RISCV/pr145363.ll
new file mode 100644
index 000000000000..14bb4e469ddb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr145363.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s
+
+; These two loads will CSE, we need to conservatively combine the range
+; metadata. The final assembly should not contain an OR.
+define i32 @f(ptr %p) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lw a0, 0(a0)
+; CHECK-NEXT:    lui a1, 294471
+; CHECK-NEXT:    addi a1, a1, 1064
+; CHECK-NEXT:    addw a0, a0, a1
+; CHECK-NEXT:    ret
+  %load = load i32, ptr %p, align 4, !range !0
+  %load2 = load i32, ptr %p, align 4
+  %add = add i32 1206154280, %load2
+  ret i32 %add
+}
+
+; The mul and getelementptr will get removed in DAGCombine causing the loads
+; to CSE after they are created.
+define i32 @test(ptr %p, i32 %x, ptr %q) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lw a1, 0(a0)
+; CHECK-NEXT:    lui a0, 294471
+; CHECK-NEXT:    addi a0, a0, 1064
+; CHECK-NEXT:    addw a0, a1, a0
+; CHECK-NEXT:    sw a1, 0(a2)
+; CHECK-NEXT:    ret
+  %load = load i32, ptr %p, align 4, !range !0
+  %mul = mul i32 0, %x
+  %a = getelementptr i32, ptr %p, i32 %mul
+  %load2 = load i32, ptr %a, align 4
+  %add = add i32 1206154280, %load2
+  store i32 %load, ptr %q
+  ret i32 %add
+}
+
+!0 = !{i32 1, i32 2, i32 3, i32 4}
diff --git a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
index 3e7f09f3d6c2..5cabb8c53e26 100644
--- a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
@@ -2,6 +2,170 @@
 ; RUN: llc -O0 -mtriple=riscv32 -mattr=+xandesperf -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
+; NDS.BBC
+
+define i32 @bbc(i32 %a) nounwind {
+; CHECK-LABEL: bbc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bbc a0, 16, .LBB0_2
+; CHECK-NEXT:    j .LBB0_1
+; CHECK-NEXT:  .LBB0_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp eq i32 %and, 0
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bbc(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bbc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bbc a0, 16, .LBB1_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB1_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp eq i32 %and, 0
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BBS
+
+define i32 @bbs(i32 %a) nounwind {
+; CHECK-LABEL: bbs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bbs a0, 16, .LBB2_2
+; CHECK-NEXT:    j .LBB2_1
+; CHECK-NEXT:  .LBB2_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp ne i32 %and, 0
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bbs(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bbs:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bbs a0, 16, .LBB3_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB3_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %and = and i32 %a, 65536
+  %tst = icmp ne i32 %and, 0
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BEQC
+
+define i32 @beqc(i32 %a) nounwind {
+; CHECK-LABEL: beqc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.beqc a0, 5, .LBB4_2
+; CHECK-NEXT:    j .LBB4_1
+; CHECK-NEXT:  .LBB4_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %tst = icmp eq i32 %a, 5
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_beqc(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_beqc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.beqc a0, 5, .LBB5_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB5_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %tst = icmp eq i32 %a, 5
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
+; NDS.BNEC
+
+define i32 @bnec(i32 %a) nounwind {
+; CHECK-LABEL: bnec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nds.bnec a0, 5, .LBB6_2
+; CHECK-NEXT:    j .LBB6_1
+; CHECK-NEXT:  .LBB6_1: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_2: # %t
+; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+  %tst = icmp ne i32 %a, 5
+  br i1 %tst, label %t, label %f
+f:
+  ret i32 0
+t:
+  ret i32 1
+}
+
+define i32 @select_bnec(i32 %a, i32 %b, i32 %c) nounwind {
+; CHECK-LABEL: select_bnec:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    nds.bnec a0, 5, .LBB7_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:  .LBB7_2:
+; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  %tst = icmp ne i32 %a, 5
+  %ret = select i1 %tst, i32 %b, i32 %c
+  ret i32 %ret
+}
+
 ; NDS.BFOZ
 
 ; MSB >= LSB
diff --git a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
index 6aae8984546e..896e371452db 100644
--- a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
@@ -125,26 +125,26 @@ define i128 @fptosi_sat_f32_to_i128(float %a) nounwind {
 ; RV64IF-NEXT:    fmv.w.x fa5, a0
 ; RV64IF-NEXT:    fle.s s0, fa5, fa0
 ; RV64IF-NEXT:    call __fixsfti
-; RV64IF-NEXT:    li a3, -1
+; RV64IF-NEXT:    li a2, -1
 ; RV64IF-NEXT:    bnez s0, .LBB4_2
 ; RV64IF-NEXT:  # %bb.1:
-; RV64IF-NEXT:    slli a1, a3, 63
+; RV64IF-NEXT:    slli a1, a2, 63
 ; RV64IF-NEXT:  .LBB4_2:
-; RV64IF-NEXT:    lui a2, %hi(.LCPI4_0)
-; RV64IF-NEXT:    flw fa5, %lo(.LCPI4_0)(a2)
-; RV64IF-NEXT:    flt.s a2, fa5, fs0
-; RV64IF-NEXT:    beqz a2, .LBB4_4
+; RV64IF-NEXT:    lui a3, %hi(.LCPI4_0)
+; RV64IF-NEXT:    flw fa5, %lo(.LCPI4_0)(a3)
+; RV64IF-NEXT:    flt.s a3, fa5, fs0
+; RV64IF-NEXT:    beqz a3, .LBB4_4
 ; RV64IF-NEXT:  # %bb.3:
-; RV64IF-NEXT:    srli a1, a3, 1
+; RV64IF-NEXT:    srli a1, a2, 1
 ; RV64IF-NEXT:  .LBB4_4:
-; RV64IF-NEXT:    feq.s a3, fs0, fs0
+; RV64IF-NEXT:    feq.s a2, fs0, fs0
+; RV64IF-NEXT:    neg a3, a3
 ; RV64IF-NEXT:    neg a4, s0
 ; RV64IF-NEXT:    neg a2, a2
-; RV64IF-NEXT:    neg a3, a3
 ; RV64IF-NEXT:    and a0, a4, a0
-; RV64IF-NEXT:    and a1, a3, a1
-; RV64IF-NEXT:    or a0, a2, a0
-; RV64IF-NEXT:    and a0, a3, a0
+; RV64IF-NEXT:    and a1, a2, a1
+; RV64IF-NEXT:    or a0, a3, a0
+; RV64IF-NEXT:    and a0, a2, a0
 ; RV64IF-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rvv/abd.ll b/llvm/test/CodeGen/RISCV/rvv/abd.ll
index 583d872238df..be4292c9902e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/abd.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/abd.ll
@@ -316,12 +316,10 @@ define <vscale x 4 x i32> @uabd_non_matching_promotion(<vscale x 4 x i8> %a, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vzext.vf4 v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vsext.vf2 v8, v9
-; CHECK-NEXT:    vwsub.wv v10, v10, v8
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 0
-; CHECK-NEXT:    vmax.vv v8, v10, v8
+; CHECK-NEXT:    vsext.vf4 v12, v9
+; CHECK-NEXT:    vmin.vv v8, v10, v12
+; CHECK-NEXT:    vmax.vv v10, v10, v12
+; CHECK-NEXT:    vsub.vv v8, v10, v8
 ; CHECK-NEXT:    ret
   %a.zext = zext <vscale x 4 x i8> %a to <vscale x 4 x i32>
   %b.zext = sext <vscale x 4 x i8> %b to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll b/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
index 5dc532273b77..0d8aff306252 100644
--- a/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/combine-reduce-add-to-vcpop.ll
@@ -313,12 +313,12 @@ define i32 @test_nxv128i1(<vscale x 128 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v6, a0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v6, v7, a1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v7, a0
 ; CHECK-NEXT:    vslidedown.vx v5, v6, a0
+; CHECK-NEXT:    vslidedown.vx v4, v7, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v4
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v5
 ; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
@@ -364,9 +364,9 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vmv1r.v v7, v9
 ; CHECK-NEXT:    vmv1r.v v5, v8
 ; CHECK-NEXT:    vmv1r.v v4, v0
-; CHECK-NEXT:    vmv.v.i v24, 0
+; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    mv a2, a0
@@ -376,7 +376,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v5
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 5
 ; CHECK-NEXT:    add a0, sp, a0
@@ -389,7 +389,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v2, v5, a0
 ; CHECK-NEXT:    vmv.v.v v0, v3
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    mv a3, a2
@@ -399,41 +399,41 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a2, a2, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v0, v2
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmv8r.v v8, v16
+; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 4
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v3, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
 ; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    slli a2, a2, 3
 ; CHECK-NEXT:    add a2, sp, a2
 ; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v16, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v2, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v16, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v24, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v8, v24, 1, v0
+; CHECK-NEXT:    vmerge.vim v16, v8, 1, v0
 ; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v5, a1
-; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vim v24, v24, 1, v0
-; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v6, a1
 ; CHECK-NEXT:    vslidedown.vx v5, v7, a1
+; CHECK-NEXT:    vslidedown.vx v4, v6, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v5
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v4
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v24
+; CHECK-NEXT:    vmv1r.v v0, v5
+; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    addi a2, sp, 16
 ; CHECK-NEXT:    vs8r.v v8, (a2) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    vsetvli a2, zero, e8, m1, ta, ma
@@ -443,7 +443,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    vslidedown.vx v0, v4, a1
 ; CHECK-NEXT:    vslidedown.vx v3, v5, a1
 ; CHECK-NEXT:    vsetvli a0, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v3
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -451,7 +451,7 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vadd.vv v8, v8, v24
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
@@ -492,16 +492,16 @@ define i32 @test_nxv256i1(<vscale x 256 x i1> %x) {
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vadd.vi v24, v24, 1, v0.t
-; CHECK-NEXT:    vadd.vv v0, v24, v8
+; CHECK-NEXT:    vadd.vv v24, v24, v8
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vadd.vv v8, v8, v24
-; CHECK-NEXT:    vadd.vv v16, v0, v16
+; CHECK-NEXT:    vl8r.v v0, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vadd.vv v8, v8, v0
+; CHECK-NEXT:    vadd.vv v16, v24, v16
 ; CHECK-NEXT:    vadd.vv v8, v16, v8
 ; CHECK-NEXT:    vmv.s.x v16, zero
 ; CHECK-NEXT:    vredsum.vs v8, v8, v16
@@ -537,17 +537,18 @@ entry:
 define i16 @test_narrow_nxv64i1(<vscale x 64 x i1> %x) {
 ; CHECK-LABEL: test_narrow_nxv64i1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v0, a0
+; CHECK-NEXT:    vslidedown.vx v8, v0, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e16, m8, ta, mu
-; CHECK-NEXT:    vadd.vi v8, v8, 1, v0.t
-; CHECK-NEXT:    vmv.s.x v16, zero
-; CHECK-NEXT:    vredsum.vs v8, v8, v16
+; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vadd.vi v16, v16, 1, v0.t
+; CHECK-NEXT:    vmv.s.x v8, zero
+; CHECK-NEXT:    vredsum.vs v8, v16, v8
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
index be73d4808937..31e79e58f44c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir
@@ -47,7 +47,7 @@ body:             |
     %22:vr = PseudoVMSNE_VI_M1 %3, 0, 1, 6 /* e64 */
     %23:vmv0 = COPY %22
     %25:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, %17, -1, %23, 1, 6 /* e64 */
-    %29:vr = PseudoVC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $sf_vcix_state, implicit $sf_vcix_state
+    %29:vr = PseudoSF_VC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $sf_vcix_state, implicit $sf_vcix_state
     %30:vr = PseudoVMV_V_I_M1 undef $noreg, 0, 1, 6 /* e64 */, 0
     BGEU %1, $x0, %bb.2
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
index c2ae1ce49138..3e822d357b66 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll
@@ -293,31 +293,6 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @vector_deinterleave_load_fact
   ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res3
 }
 
-; TODO: Remove once recursive deinterleaving support is removed
-define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @vector_deinterleave_load_factor4_recursive(ptr %p) {
-; CHECK-LABEL: vector_deinterleave_load_factor4_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vlseg4e8.v v8, (a0)
-; CHECK-NEXT:    ret
-  %vec = load <32 x i8>, ptr %p
-  %d0 = call {<16 x i8>, <16 x i8>} @llvm.vector.deinterleave2.v32i8(<32 x i8> %vec)
-  %d0.0 = extractvalue { <16 x i8>, <16 x i8> } %d0, 0
-  %d0.1 = extractvalue { <16 x i8>, <16 x i8> } %d0, 1
-  %d1 = call {<8 x i8>, <8 x i8>} @llvm.vector.deinterleave2.v16i8(<16 x i8> %d0.0)
-  %t0 = extractvalue { <8 x i8>, <8 x i8> } %d1, 0
-  %t2 = extractvalue { <8 x i8>, <8 x i8> } %d1, 1
-  %d2 = call {<8 x i8>, <8 x i8>} @llvm.vector.deinterleave2.v16i8(<16 x i8> %d0.1)
-  %t1 = extractvalue { <8 x i8>, <8 x i8> } %d2, 0
-  %t3 = extractvalue { <8 x i8>, <8 x i8> } %d2, 1
-
-  %res0 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } undef, <8 x i8> %t0, 0
-  %res1 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res0, <8 x i8> %t1, 1
-  %res2 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res1, <8 x i8> %t2, 2
-  %res3 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res2, <8 x i8> %t3, 3
-  ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res3
-}
-
 define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } @vector_deinterleave_load_factor5(ptr %p) {
 ; CHECK-LABEL: vector_deinterleave_load_factor5:
 ; CHECK:       # %bb.0:
@@ -414,45 +389,3 @@ define { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <
   %res7 = insertvalue { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res6, <8 x i8> %t6, 7
   ret { <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8>, <8 x i8> } %res7
 }
-
-; TODO: Remove once recursive deinterleaving support is removed
-define {<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>} @vector_deinterleave_load_factor8_recursive(ptr %ptr) {
-; CHECK-LABEL: vector_deinterleave_load_factor8_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vlseg8e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %vec = load <16 x i32>, ptr %ptr
-  %d0 = call { <8 x i32>, <8 x i32> } @llvm.vector.deinterleave2.v16i32(<16 x i32> %vec)
-  %d0.0 = extractvalue { <8 x i32>, <8 x i32> } %d0, 0
-  %d0.1 = extractvalue { <8 x i32>, <8 x i32> } %d0, 1
-  %d1 = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %d0.0)
-  %d1.0 = extractvalue { <4 x i32>, <4 x i32> } %d1, 0
-  %d1.1 = extractvalue { <4 x i32>, <4 x i32> } %d1, 1
-  %d2 = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %d0.1)
-  %d2.0 = extractvalue { <4 x i32>, <4 x i32> } %d2, 0
-  %d2.1 = extractvalue { <4 x i32>, <4 x i32> } %d2, 1
-
-  %d3 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d1.0)
-  %t0 = extractvalue { <2 x i32>, <2 x i32> } %d3, 0
-  %t4 = extractvalue { <2 x i32>, <2 x i32> } %d3, 1
-  %d4 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d1.1)
-  %t2 = extractvalue { <2 x i32>, <2 x i32> } %d4, 0
-  %t6 = extractvalue { <2 x i32>, <2 x i32> } %d4, 1
-  %d5 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d2.0)
-  %t1 = extractvalue { <2 x i32>, <2 x i32> } %d5, 0
-  %t5 = extractvalue { <2 x i32>, <2 x i32> } %d5, 1
-  %d6 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d2.1)
-  %t3 = extractvalue { <2 x i32>, <2 x i32> } %d6, 0
-  %t7 = extractvalue { <2 x i32>, <2 x i32> } %d6, 1
-
-  %res0 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } undef, <2 x i32> %t0, 0
-  %res1 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res0, <2 x i32> %t1, 1
-  %res2 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res1, <2 x i32> %t2, 2
-  %res3 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res2, <2 x i32> %t3, 3
-  %res4 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res3, <2 x i32> %t4, 4
-  %res5 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res4, <2 x i32> %t5, 5
-  %res6 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res5, <2 x i32> %t6, 6
-  %res7 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res6, <2 x i32> %t7, 7
-  ret { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res7
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index 564e95c43f68..3c3e08d387fa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -1756,16 +1756,16 @@ define <8 x float> @buildvec_v8f32_zvl256(float %e0, float %e1, float %e2, float
 ; CHECK-LABEL: buildvec_v8f32_zvl256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
-; CHECK-NEXT:    vfmv.v.f v8, fa0
-; CHECK-NEXT:    vfmv.v.f v9, fa4
+; CHECK-NEXT:    vfmv.v.f v8, fa4
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa1
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa5
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa2
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa6
-; CHECK-NEXT:    vfslide1down.vf v10, v8, fa3
-; CHECK-NEXT:    vfslide1down.vf v8, v9, fa7
-; CHECK-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa5
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa1
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa6
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa2
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa7
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa3
+; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; CHECK-NEXT:    ret
   %v0 = insertelement <8 x float> poison, float %e0, i64 0
   %v1 = insertelement <8 x float> %v0, float %e1, i64 1
@@ -1807,16 +1807,16 @@ define <8 x double> @buildvec_v8f64_zvl512(double %e0, double %e1, double %e2, d
 ; CHECK-LABEL: buildvec_v8f64_zvl512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e64, m1, ta, mu
-; CHECK-NEXT:    vfmv.v.f v8, fa0
-; CHECK-NEXT:    vfmv.v.f v9, fa4
+; CHECK-NEXT:    vfmv.v.f v8, fa4
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa1
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa5
-; CHECK-NEXT:    vfslide1down.vf v8, v8, fa2
-; CHECK-NEXT:    vfslide1down.vf v9, v9, fa6
-; CHECK-NEXT:    vfslide1down.vf v10, v8, fa3
-; CHECK-NEXT:    vfslide1down.vf v8, v9, fa7
-; CHECK-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa5
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa1
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa6
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa2
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa7
+; CHECK-NEXT:    vfslide1down.vf v9, v9, fa3
+; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; CHECK-NEXT:    ret
   %v0 = insertelement <8 x double> poison, double %e0, i64 0
   %v1 = insertelement <8 x double> %v0, double %e1, i64 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index f235540cc8ff..d9bb007a10f7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -1359,23 +1359,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu s0, 14(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 15(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t2
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, t1
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32-ONLY-NEXT:    .cfi_restore s0
 ; RV32-ONLY-NEXT:    addi sp, sp, 16
@@ -1494,23 +1494,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu s0, 14(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 15(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t2
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, t1
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64V-ONLY-NEXT:    .cfi_restore s0
 ; RV64V-ONLY-NEXT:    addi sp, sp, 16
@@ -1631,23 +1631,23 @@ define <16 x i8> @buildvec_v16i8_loads_contigous(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu s0, 14(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 15(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t2
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t4
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, s0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, t1
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t2
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t3
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, s0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64ZVE32-NEXT:    .cfi_restore s0
 ; RV64ZVE32-NEXT:    addi sp, sp, 16
@@ -1733,23 +1733,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu s0, 124(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 144(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, t2
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, t1
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32-ONLY-NEXT:    .cfi_restore s0
 ; RV32-ONLY-NEXT:    addi sp, sp, 16
@@ -1868,23 +1868,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu s0, 124(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 144(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, s0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t4
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, t2
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, t1
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t3
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, s0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t2
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64V-ONLY-NEXT:    .cfi_restore s0
 ; RV64V-ONLY-NEXT:    addi sp, sp, 16
@@ -2013,23 +2013,23 @@ define <16 x i8> @buildvec_v16i8_loads_gather(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu s0, 124(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 144(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, s0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t4
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, t2
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, t1
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t3
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, s0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t2
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64ZVE32-NEXT:    .cfi_restore s0
 ; RV64ZVE32-NEXT:    addi sp, sp, 16
@@ -2505,17 +2505,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu t0, 105(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 161(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a2
-; RV32-ONLY-NEXT:    vmv.v.x v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 4
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vmv.v.x v8, a6
+; RV32-ONLY-NEXT:    vmv.v.x v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v16i8_undef_edges:
@@ -2592,17 +2592,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu t0, 105(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 161(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a2
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a0
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 4
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a6
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 4
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_undef_edges:
@@ -2679,17 +2679,17 @@ define <16 x i8> @buildvec_v16i8_undef_edges(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu t0, 105(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 161(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a2
-; RV64ZVE32-NEXT:    vmv.v.x v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a5
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 4
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vmv.v.x v8, a6
+; RV64ZVE32-NEXT:    vmv.v.x v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 4
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ret
   %p4 = getelementptr i8, ptr %p, i32 31
   %p5 = getelementptr i8, ptr %p, i32 44
@@ -2740,21 +2740,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV32-ONLY-NEXT:    lbu t1, 144(a0)
 ; RV32-ONLY-NEXT:    lbu a0, 154(a0)
 ; RV32-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a1
-; RV32-ONLY-NEXT:    vmv.v.x v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV32-ONLY-NEXT:    vmv.v.x v8, a6
+; RV32-ONLY-NEXT:    vmv.v.x v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
 ; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 2
 ; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 2
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
 ; RV32-ONLY-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, t1
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, t1
+; RV32-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v16i8_loads_undef_scattered:
@@ -2834,21 +2834,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV64V-ONLY-NEXT:    lbu t1, 144(a0)
 ; RV64V-ONLY-NEXT:    lbu a0, 154(a0)
 ; RV64V-ONLY-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a1
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a7
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a6
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
 ; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 2
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t0
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a4
-; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
 ; RV64V-ONLY-NEXT:    vslidedown.vi v8, v8, 1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, t1
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a0
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a4
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, t1
+; RV64V-ONLY-NEXT:    vslidedown.vi v9, v9, 1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a0
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v16i8_loads_undef_scattered:
@@ -2930,21 +2930,21 @@ define <16 x i8> @buildvec_v16i8_loads_undef_scattered(ptr %p) {
 ; RV64ZVE32-NEXT:    lbu t1, 144(a0)
 ; RV64ZVE32-NEXT:    lbu a0, 154(a0)
 ; RV64ZVE32-NEXT:    vsetivli zero, 16, e8, m1, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32-NEXT:    vmv.v.x v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a7
+; RV64ZVE32-NEXT:    vmv.v.x v8, a6
+; RV64ZVE32-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
 ; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 2
 ; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 2
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t0
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
 ; RV64ZVE32-NEXT:    vslidedown.vi v8, v8, 1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, t1
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 8, v0.t
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, t1
+; RV64ZVE32-NEXT:    vslidedown.vi v9, v9, 1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 8, v0.t
 ; RV64ZVE32-NEXT:    ret
   %p2 = getelementptr i8, ptr %p, i32 1
   %p3 = getelementptr i8, ptr %p, i32 22
@@ -3002,16 +3002,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV32-ONLY-LABEL: buildvec_v8i8_pack:
 ; RV32-ONLY:       # %bb.0:
 ; RV32-ONLY-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV32-ONLY-NEXT:    vmv.v.x v8, a0
-; RV32-ONLY-NEXT:    vmv.v.x v9, a4
+; RV32-ONLY-NEXT:    vmv.v.x v8, a4
+; RV32-ONLY-NEXT:    vmv.v.x v9, a0
 ; RV32-ONLY-NEXT:    vmv.v.i v0, 15
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a5
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a6
-; RV32-ONLY-NEXT:    vslide1down.vx v10, v8, a3
-; RV32-ONLY-NEXT:    vslide1down.vx v8, v9, a7
-; RV32-ONLY-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a6
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV32-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV32-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV32-ONLY-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV32-ONLY-NEXT:    ret
 ;
 ; RV32VB-LABEL: buildvec_v8i8_pack:
@@ -3055,16 +3055,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV64V-ONLY-LABEL: buildvec_v8i8_pack:
 ; RV64V-ONLY:       # %bb.0:
 ; RV64V-ONLY-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV64V-ONLY-NEXT:    vmv.v.x v8, a0
-; RV64V-ONLY-NEXT:    vmv.v.x v9, a4
+; RV64V-ONLY-NEXT:    vmv.v.x v8, a4
+; RV64V-ONLY-NEXT:    vmv.v.x v9, a0
 ; RV64V-ONLY-NEXT:    vmv.v.i v0, 15
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a1
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a5
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a2
-; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a6
-; RV64V-ONLY-NEXT:    vslide1down.vx v10, v8, a3
-; RV64V-ONLY-NEXT:    vslide1down.vx v8, v9, a7
-; RV64V-ONLY-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a5
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a1
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a6
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a2
+; RV64V-ONLY-NEXT:    vslide1down.vx v8, v8, a7
+; RV64V-ONLY-NEXT:    vslide1down.vx v9, v9, a3
+; RV64V-ONLY-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64V-ONLY-NEXT:    ret
 ;
 ; RVA22U64-LABEL: buildvec_v8i8_pack:
@@ -3110,16 +3110,16 @@ define <8 x i8> @buildvec_v8i8_pack(i8 %e1, i8 %e2, i8 %e3, i8 %e4, i8 %e5, i8 %
 ; RV64ZVE32-LABEL: buildvec_v8i8_pack:
 ; RV64ZVE32:       # %bb.0:
 ; RV64ZVE32-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; RV64ZVE32-NEXT:    vmv.v.x v8, a0
-; RV64ZVE32-NEXT:    vmv.v.x v9, a4
+; RV64ZVE32-NEXT:    vmv.v.x v8, a4
+; RV64ZVE32-NEXT:    vmv.v.x v9, a0
 ; RV64ZVE32-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32-NEXT:    vslide1down.vx v10, v8, a3
-; RV64ZVE32-NEXT:    vslide1down.vx v8, v9, a7
-; RV64ZVE32-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a5
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32-NEXT:    ret
   %v1 = insertelement <8 x i8> poison, i8 %e1, i32 0
   %v2 = insertelement <8 x i8> %v1, i8 %e2, i32 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
index c394e7aa2e3e..a49eeed3605c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleave-store.ll
@@ -203,20 +203,6 @@ define void @vector_interleave_store_factor4(<4 x i32> %a, <4 x i32> %b, <4 x i3
   ret void
 }
 
-; TODO: Remove once recursive interleaving support is removed
-define void @vector_interleave_store_factor4_recursive(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, ptr %p) {
-; CHECK-LABEL: vector_interleave_store_factor4_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vsseg4e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %v0 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %a, <4 x i32> %c)
-  %v1 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %b, <4 x i32> %d)
-  %v2 = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
-  store <16 x i32> %v2, ptr %p
-  ret void
-}
-
 define void @vector_interleave_store_factor5(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, ptr %p) {
 ; CHECK-LABEL: vector_interleave_store_factor5:
 ; CHECK:       # %bb.0:
@@ -260,23 +246,3 @@ define void @vector_interleave_store_factor8(<4 x i32> %a, <4 x i32> %b, <4 x i3
   store <32 x i32> %v, ptr %p
   ret void
 }
-
-; TODO: Remove once recursive interleaving support is removed
-define void @vector_interleave_store_factor8_recursive(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32> %d, <4 x i32> %e, <4 x i32> %f, <4 x i32> %g, <4 x i32> %h, ptr %p) {
-; CHECK-LABEL: vector_interleave_store_factor8_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vsseg8e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %v0 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %a, <4 x i32> %e)
-  %v1 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %c, <4 x i32> %g)
-  %v2 = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v0, <8 x i32> %v1)
-
-  %v3 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %b, <4 x i32> %f)
-  %v4 = call <8 x i32> @llvm.vector.interleave2.v8i32(<4 x i32> %d, <4 x i32> %h)
-  %v5 = call <16 x i32> @llvm.vector.interleave2.v16i32(<8 x i32> %v3, <8 x i32> %v4)
-
-  %v6 = call <32 x i32> @llvm.vector.interleave2.v32i32(<16 x i32> %v2, <16 x i32> %v5)
-  store <32 x i32> %v6, ptr %p
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
index 8ac4c7447c7d..5e3ae2faf1a5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
@@ -302,15 +302,11 @@ define {<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>} @vpload_factor4_intrinsics(p
 ; CHECK-NEXT:    vlseg4e32.v v8, (a0)
 ; CHECK-NEXT:    ret
   %wide.masked.load = call <8 x i32> @llvm.vp.load.v8i32.p0(ptr %ptr, <8 x i1> splat (i1 true), i32 8)
-  %d0 = call { <4 x i32>, <4 x i32> } @llvm.vector.deinterleave2.v8i32(<8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <4 x i32>, <4 x i32> } %d0, 0
-  %d0.1 = extractvalue { <4 x i32>, <4 x i32> } %d0, 1
-  %d1 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d0.0)
-  %t0 = extractvalue { <2 x i32>, <2 x i32> } %d1, 0
-  %t2 = extractvalue { <2 x i32>, <2 x i32> } %d1, 1
-  %d2 = call { <2 x i32>, <2 x i32> } @llvm.vector.deinterleave2.v4i32(<4 x i32> %d0.1)
-  %t1 = extractvalue { <2 x i32>, <2 x i32> } %d2, 0
-  %t3 = extractvalue { <2 x i32>, <2 x i32> } %d2, 1
+  %d = call { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } @llvm.vector.deinterleave4.v8i32(<8 x i32> %wide.masked.load)
+  %t0 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %d, 0
+  %t1 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %d, 1
+  %t2 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %d, 2
+  %t3 = extractvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %d, 3
 
   %res0 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> %t0, 0
   %res1 = insertvalue { <2 x i32>, <2 x i32>, <2 x i32>, <2 x i32> } %res0, <2 x i32> %t1, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
index c1252902ede6..0a6f9f5ba092 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llrint.ll
@@ -7,31 +7,16 @@
 define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 ; RV32-LABEL: llrint_v1i64_v1f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a0), zero
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfwcvt.x.f.v v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v1i64_v1f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-NEXT:    vmv1r.v v8, v9
 ; RV64-NEXT:    ret
   %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
   ret <1 x i64> %a
@@ -41,60 +26,16 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; RV32-LABEL: llrint_v2i64_v2f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 16
-; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vfwcvt.x.f.v v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v2i64_v2f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-NEXT:    vmv1r.v v8, v9
 ; RV64-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
   ret <2 x i64> %a
@@ -104,106 +45,16 @@ declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 define <3 x i64> @llrint_v3i64_v3f32(<3 x float> %x) {
 ; RV32-LABEL: llrint_v3i64_v3f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x03, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 3 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v3i64_v3f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v11
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-NEXT:    ret
   %a = call <3 x i64> @llvm.llrint.v3i64.v3f32(<3 x float> %x)
   ret <3 x i64> %a
@@ -213,106 +64,16 @@ declare <3 x i64> @llvm.llrint.v3i64.v3f32(<3 x float>)
 define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; RV32-LABEL: llrint_v4i64_v4f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x03, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 3 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a1, a0, 1
-; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv1r.v v10, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v4i64_v4f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v11
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vmv1r.v v10, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
   ret <4 x i64> %a
@@ -322,149 +83,16 @@ declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; RV32-LABEL: llrint_v8i64_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -208
-; RV32-NEXT:    .cfi_def_cfa_offset 208
-; RV32-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 200(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 208
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 64(sp)
-; RV32-NEXT:    sw a1, 68(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 120(sp)
-; RV32-NEXT:    sw a1, 124(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 6
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 112(sp)
-; RV32-NEXT:    sw a1, 116(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 5
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 104(sp)
-; RV32-NEXT:    sw a1, 108(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 96(sp)
-; RV32-NEXT:    sw a1, 100(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 88(sp)
-; RV32-NEXT:    sw a1, 92(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 80(sp)
-; RV32-NEXT:    sw a1, 84(sp)
-; RV32-NEXT:    addi a0, sp, 192
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 72(sp)
-; RV32-NEXT:    sw a1, 76(sp)
-; RV32-NEXT:    addi a0, sp, 64
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -208
-; RV32-NEXT:    .cfi_def_cfa sp, 208
-; RV32-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 200(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 208
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv2r.v v12, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v8i64_v8f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -128
-; RV64-NEXT:    .cfi_def_cfa_offset 128
-; RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 128
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -64
-; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-NEXT:    fcvt.l.s a2, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 4
-; RV64-NEXT:    fcvt.l.s a3, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 3
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v8, v8, 1
-; RV64-NEXT:    fcvt.l.s a4, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.s a5, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
-; RV64-NEXT:    fcvt.l.s a6, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a4, 32(sp)
-; RV64-NEXT:    sd a3, 40(sp)
-; RV64-NEXT:    sd a2, 48(sp)
-; RV64-NEXT:    sd a1, 56(sp)
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    sd a0, 0(sp)
-; RV64-NEXT:    sd a1, 8(sp)
-; RV64-NEXT:    sd a6, 16(sp)
-; RV64-NEXT:    sd a5, 24(sp)
-; RV64-NEXT:    mv a0, sp
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -128
-; RV64-NEXT:    .cfi_def_cfa sp, 128
-; RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 128
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vmv2r.v v12, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV64-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
   ret <8 x i64> %a
@@ -474,214 +102,16 @@ declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; RV32-LABEL: llrint_v16i64_v16f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -400
-; RV32-NEXT:    .cfi_def_cfa_offset 400
-; RV32-NEXT:    sw ra, 396(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 392(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 400
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -128
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    flw fa0, 124(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 248(sp)
-; RV32-NEXT:    sw a1, 252(sp)
-; RV32-NEXT:    flw fa0, 120(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 240(sp)
-; RV32-NEXT:    sw a1, 244(sp)
-; RV32-NEXT:    flw fa0, 116(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 232(sp)
-; RV32-NEXT:    sw a1, 236(sp)
-; RV32-NEXT:    flw fa0, 112(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 224(sp)
-; RV32-NEXT:    sw a1, 228(sp)
-; RV32-NEXT:    flw fa0, 108(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 216(sp)
-; RV32-NEXT:    sw a1, 220(sp)
-; RV32-NEXT:    flw fa0, 104(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 208(sp)
-; RV32-NEXT:    sw a1, 212(sp)
-; RV32-NEXT:    flw fa0, 100(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 200(sp)
-; RV32-NEXT:    sw a1, 204(sp)
-; RV32-NEXT:    flw fa0, 96(sp)
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 192(sp)
-; RV32-NEXT:    sw a1, 196(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 128(sp)
-; RV32-NEXT:    sw a1, 132(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 152(sp)
-; RV32-NEXT:    sw a1, 156(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 144(sp)
-; RV32-NEXT:    sw a1, 148(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 136(sp)
-; RV32-NEXT:    sw a1, 140(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 184(sp)
-; RV32-NEXT:    sw a1, 188(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 6
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 176(sp)
-; RV32-NEXT:    sw a1, 180(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 5
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 168(sp)
-; RV32-NEXT:    sw a1, 172(sp)
-; RV32-NEXT:    addi a0, sp, 384
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrintf
-; RV32-NEXT:    sw a0, 160(sp)
-; RV32-NEXT:    sw a1, 164(sp)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    addi a1, sp, 128
-; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-NEXT:    vle32.v v8, (a1)
-; RV32-NEXT:    addi sp, s0, -400
-; RV32-NEXT:    .cfi_def_cfa sp, 400
-; RV32-NEXT:    lw ra, 396(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 392(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 400
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vmv4r.v v16, v8
+; RV32-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v16i64_v16f32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -384
-; RV64-NEXT:    .cfi_def_cfa_offset 384
-; RV64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 384
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -128
-; RV64-NEXT:    addi a0, sp, 64
 ; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-NEXT:    vse32.v v8, (a0)
-; RV64-NEXT:    flw fa5, 124(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v8
-; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 248(sp)
-; RV64-NEXT:    flw fa5, 120(sp)
-; RV64-NEXT:    vslidedown.vi v13, v8, 1
-; RV64-NEXT:    fcvt.l.s a0, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.s a1, fa5
-; RV64-NEXT:    sd a1, 240(sp)
-; RV64-NEXT:    flw fa5, 116(sp)
-; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-NEXT:    fcvt.l.s a1, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v12
-; RV64-NEXT:    fcvt.l.s a2, fa5
-; RV64-NEXT:    sd a2, 232(sp)
-; RV64-NEXT:    flw fa5, 112(sp)
-; RV64-NEXT:    fcvt.l.s a2, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v13
-; RV64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-NEXT:    fcvt.l.s a3, fa5
-; RV64-NEXT:    sd a3, 224(sp)
-; RV64-NEXT:    flw fa5, 108(sp)
-; RV64-NEXT:    fcvt.l.s a3, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-NEXT:    fcvt.l.s a4, fa5
-; RV64-NEXT:    sd a4, 216(sp)
-; RV64-NEXT:    flw fa5, 104(sp)
-; RV64-NEXT:    fcvt.l.s a4, fa4
-; RV64-NEXT:    vfmv.f.s fa4, v12
-; RV64-NEXT:    fcvt.l.s a5, fa4
-; RV64-NEXT:    fcvt.l.s a6, fa5
-; RV64-NEXT:    sd a6, 208(sp)
-; RV64-NEXT:    flw fa5, 100(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.s a6, fa4
-; RV64-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-NEXT:    fcvt.l.s a7, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a7, 200(sp)
-; RV64-NEXT:    fcvt.l.s a7, fa5
-; RV64-NEXT:    flw fa5, 96(sp)
-; RV64-NEXT:    sd a0, 128(sp)
-; RV64-NEXT:    sd a3, 136(sp)
-; RV64-NEXT:    sd a2, 144(sp)
-; RV64-NEXT:    sd a1, 152(sp)
-; RV64-NEXT:    sd a7, 160(sp)
-; RV64-NEXT:    sd a6, 168(sp)
-; RV64-NEXT:    sd a5, 176(sp)
-; RV64-NEXT:    sd a4, 184(sp)
-; RV64-NEXT:    fcvt.l.s a0, fa5
-; RV64-NEXT:    sd a0, 192(sp)
-; RV64-NEXT:    addi a0, sp, 128
-; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -384
-; RV64-NEXT:    .cfi_def_cfa sp, 384
-; RV64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 384
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vmv4r.v v16, v8
+; RV64-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV64-NEXT:    ret
   %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
   ret <16 x i64> %a
@@ -691,30 +121,14 @@ declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 ; RV32-LABEL: llrint_v1i64_v1f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 0(sp)
-; RV32-NEXT:    sw a1, 4(sp)
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vlse64.v v8, (a0), zero
-; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v1i64_v1f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
   ret <1 x i64> %a
@@ -724,59 +138,14 @@ declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 ; RV32-LABEL: llrint_v2i64_v2f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 2 * vlenb
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    add a2, sp, a2
-; RV32-NEXT:    addi a2, a2, 16
-; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v2i64_v2f64:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v9
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
   ret <2 x i64> %a
@@ -786,103 +155,14 @@ declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 ; RV32-LABEL: llrint_v4i64_v4f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    .cfi_def_cfa_offset 32
-; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 4 * vlenb
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    addi a2, sp, 16
-; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    .cfi_def_cfa sp, 32
-; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    addi sp, sp, 32
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v4i64_v4f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v12
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v10, a0
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
   ret <4 x i64> %a
@@ -892,137 +172,14 @@ declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; RV32-LABEL: llrint_v8i64_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -272
-; RV32-NEXT:    .cfi_def_cfa_offset 272
-; RV32-NEXT:    sw ra, 268(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 264(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 272
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 2
-; RV32-NEXT:    sub sp, sp, a0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; RV32-NEXT:    addi a0, sp, 64
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vse64.v v8, (a0)
-; RV32-NEXT:    fld fa0, 120(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 184(sp)
-; RV32-NEXT:    sw a1, 188(sp)
-; RV32-NEXT:    fld fa0, 112(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 176(sp)
-; RV32-NEXT:    sw a1, 180(sp)
-; RV32-NEXT:    fld fa0, 104(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 168(sp)
-; RV32-NEXT:    sw a1, 172(sp)
-; RV32-NEXT:    fld fa0, 96(sp)
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 160(sp)
-; RV32-NEXT:    sw a1, 164(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 128(sp)
-; RV32-NEXT:    sw a1, 132(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 1
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 136(sp)
-; RV32-NEXT:    sw a1, 140(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 152(sp)
-; RV32-NEXT:    sw a1, 156(sp)
-; RV32-NEXT:    addi a0, sp, 256
-; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 2
-; RV32-NEXT:    vfmv.f.s fa0, v8
-; RV32-NEXT:    call llrint
-; RV32-NEXT:    sw a0, 144(sp)
-; RV32-NEXT:    sw a1, 148(sp)
-; RV32-NEXT:    addi a0, sp, 128
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -272
-; RV32-NEXT:    .cfi_def_cfa sp, 272
-; RV32-NEXT:    lw ra, 268(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 264(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 272
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: llrint_v8i64_v8f64:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -192
-; RV64-NEXT:    .cfi_def_cfa_offset 192
-; RV64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    .cfi_offset s0, -16
-; RV64-NEXT:    addi s0, sp, 192
-; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    andi sp, sp, -64
-; RV64-NEXT:    mv a0, sp
-; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vse64.v v8, (a0)
-; RV64-NEXT:    fld fa5, 56(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v8
-; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-NEXT:    fcvt.l.d a0, fa4
-; RV64-NEXT:    fcvt.l.d a1, fa5
-; RV64-NEXT:    sd a1, 120(sp)
-; RV64-NEXT:    fld fa5, 48(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-NEXT:    fcvt.l.d a1, fa4
-; RV64-NEXT:    fcvt.l.d a2, fa5
-; RV64-NEXT:    sd a2, 112(sp)
-; RV64-NEXT:    fld fa5, 40(sp)
-; RV64-NEXT:    vfmv.f.s fa4, v10
-; RV64-NEXT:    fcvt.l.d a2, fa4
-; RV64-NEXT:    vslidedown.vi v8, v8, 2
-; RV64-NEXT:    fcvt.l.d a3, fa5
-; RV64-NEXT:    vfmv.f.s fa5, v8
-; RV64-NEXT:    sd a3, 104(sp)
-; RV64-NEXT:    fcvt.l.d a3, fa5
-; RV64-NEXT:    fld fa5, 32(sp)
-; RV64-NEXT:    sd a0, 64(sp)
-; RV64-NEXT:    sd a1, 72(sp)
-; RV64-NEXT:    sd a3, 80(sp)
-; RV64-NEXT:    sd a2, 88(sp)
-; RV64-NEXT:    fcvt.l.d a0, fa5
-; RV64-NEXT:    sd a0, 96(sp)
-; RV64-NEXT:    addi a0, sp, 64
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-NEXT:    vle64.v v8, (a0)
-; RV64-NEXT:    addi sp, s0, -192
-; RV64-NEXT:    .cfi_def_cfa sp, 192
-; RV64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-NEXT:    .cfi_restore ra
-; RV64-NEXT:    .cfi_restore s0
-; RV64-NEXT:    addi sp, sp, 192
-; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-NEXT:    ret
   %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
   ret <8 x i64> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llround.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llround.ll
new file mode 100644
index 000000000000..b8ca7fd71cb9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-llround.ll
@@ -0,0 +1,1674 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v,+f,+d,+zvfh -target-abi=ilp32d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh -target-abi=lp64d \
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefix=RV64
+
+define <1 x i64> @llround_v1f16(<1 x half> %x) nounwind {
+; RV32-LABEL: llround_v1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v8, (a0), zero
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v1f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    ret
+  %a = call <1 x i64> @llvm.llround.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llround.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @llround_v2f16(<2 x half> %x) nounwind {
+; RV32-LABEL: llround_v2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v2f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    ret
+  %a = call <2 x i64> @llvm.llround.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llround.v2i64.v2f16(<2 x half>)
+
+define <3 x i64> @llround_v3f16(<3 x half> %x) nounwind {
+; RV32-LABEL: llround_v3f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v3f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    ret
+  %a = call <3 x i64> @llvm.llround.v3i64.v3f16(<3 x half> %x)
+  ret <3 x i64> %a
+}
+declare <3 x i64> @llvm.llround.v3i64.v3f16(<3 x half>)
+
+define <4 x i64> @llround_v4f16(<4 x half> %x) nounwind {
+; RV32-LABEL: llround_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v4f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    ret
+  %a = call <4 x i64> @llvm.llround.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llround.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @llround_v8f16(<8 x half> %x) nounwind {
+; RV32-LABEL: llround_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -208
+; RV32-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 200(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 208
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 64(sp)
+; RV32-NEXT:    sw a1, 68(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 120(sp)
+; RV32-NEXT:    sw a1, 124(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 6
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 112(sp)
+; RV32-NEXT:    sw a1, 116(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 5
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 104(sp)
+; RV32-NEXT:    sw a1, 108(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 96(sp)
+; RV32-NEXT:    sw a1, 100(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 88(sp)
+; RV32-NEXT:    sw a1, 92(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 80(sp)
+; RV32-NEXT:    sw a1, 84(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 72(sp)
+; RV32-NEXT:    sw a1, 76(sp)
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -208
+; RV32-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 200(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 208
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v8f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -128
+; RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 128
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v9, v8, 7
+; RV64-NEXT:    vslidedown.vi v10, v8, 6
+; RV64-NEXT:    vslidedown.vi v11, v8, 5
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    vslidedown.vi v9, v8, 4
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    vslidedown.vi v11, v8, 2
+; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    sd a4, 32(sp)
+; RV64-NEXT:    sd a3, 40(sp)
+; RV64-NEXT:    sd a2, 48(sp)
+; RV64-NEXT:    sd a1, 56(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    sd a0, 0(sp)
+; RV64-NEXT:    sd a1, 8(sp)
+; RV64-NEXT:    sd a6, 16(sp)
+; RV64-NEXT:    sd a5, 24(sp)
+; RV64-NEXT:    mv a0, sp
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -128
+; RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 128
+; RV64-NEXT:    ret
+  %a = call <8 x i64> @llvm.llround.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llround.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @llround_v16f16(<16 x half> %x) nounwind {
+; RV32-LABEL: llround_v16f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -400
+; RV32-NEXT:    sw ra, 396(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 392(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 400
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -128
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 128(sp)
+; RV32-NEXT:    sw a1, 132(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 15
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 248(sp)
+; RV32-NEXT:    sw a1, 252(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 14
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 240(sp)
+; RV32-NEXT:    sw a1, 244(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 13
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 232(sp)
+; RV32-NEXT:    sw a1, 236(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 12
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 224(sp)
+; RV32-NEXT:    sw a1, 228(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 11
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 216(sp)
+; RV32-NEXT:    sw a1, 220(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 10
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 208(sp)
+; RV32-NEXT:    sw a1, 212(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 9
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 200(sp)
+; RV32-NEXT:    sw a1, 204(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 8
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 192(sp)
+; RV32-NEXT:    sw a1, 196(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 184(sp)
+; RV32-NEXT:    sw a1, 188(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 6
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 176(sp)
+; RV32-NEXT:    sw a1, 180(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 5
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 168(sp)
+; RV32-NEXT:    sw a1, 172(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 160(sp)
+; RV32-NEXT:    sw a1, 164(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 152(sp)
+; RV32-NEXT:    sw a1, 156(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 144(sp)
+; RV32-NEXT:    sw a1, 148(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa0, fa5
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 136(sp)
+; RV32-NEXT:    sw a1, 140(sp)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    addi a1, sp, 128
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    addi sp, s0, -400
+; RV32-NEXT:    lw ra, 396(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 392(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 400
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v16f16:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -256
+; RV64-NEXT:    sd ra, 248(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 240(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 256
+; RV64-NEXT:    andi sp, sp, -128
+; RV64-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 15
+; RV64-NEXT:    vslidedown.vi v12, v8, 14
+; RV64-NEXT:    vslidedown.vi v14, v8, 13
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 12
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v12
+; RV64-NEXT:    vslidedown.vi v12, v8, 11
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v14
+; RV64-NEXT:    vslidedown.vi v14, v8, 10
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v12
+; RV64-NEXT:    vslidedown.vi v12, v8, 8
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v14
+; RV64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 7
+; RV64-NEXT:    vslidedown.vi v11, v8, 6
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-NEXT:    vslidedown.vi v13, v8, 4
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v12
+; RV64-NEXT:    vslidedown.vi v12, v8, 3
+; RV64-NEXT:    vslidedown.vi v14, v8, 2
+; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s t0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s t1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s t2, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s t3, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v13
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    fcvt.l.s t4, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v12
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    sd a5, 96(sp)
+; RV64-NEXT:    sd a3, 104(sp)
+; RV64-NEXT:    sd a2, 112(sp)
+; RV64-NEXT:    sd a1, 120(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v14
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    sd t0, 64(sp)
+; RV64-NEXT:    sd a7, 72(sp)
+; RV64-NEXT:    sd a6, 80(sp)
+; RV64-NEXT:    sd a4, 88(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-NEXT:    sd t4, 32(sp)
+; RV64-NEXT:    sd t3, 40(sp)
+; RV64-NEXT:    sd t2, 48(sp)
+; RV64-NEXT:    sd t1, 56(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-NEXT:    sd a0, 0(sp)
+; RV64-NEXT:    sd a3, 8(sp)
+; RV64-NEXT:    sd a2, 16(sp)
+; RV64-NEXT:    sd a1, 24(sp)
+; RV64-NEXT:    mv a0, sp
+; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -256
+; RV64-NEXT:    ld ra, 248(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 240(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 256
+; RV64-NEXT:    ret
+  %a = call <16 x i64> @llvm.llround.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llround.v16i64.v16f16(<16 x half>)
+
+define <1 x i64> @llround_v1i64_v1f32(<1 x float> %x) nounwind {
+; RV32-LABEL: llround_v1i64_v1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v8, (a0), zero
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v1i64_v1f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    ret
+  %a = call <1 x i64> @llvm.llround.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llround.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llround_v2i64_v2f32(<2 x float> %x) nounwind {
+; RV32-LABEL: llround_v2i64_v2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v2i64_v2f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    ret
+  %a = call <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llround.v2i64.v2f32(<2 x float>)
+
+define <3 x i64> @llround_v3i64_v3f32(<3 x float> %x) nounwind {
+; RV32-LABEL: llround_v3i64_v3f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v3i64_v3f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    ret
+  %a = call <3 x i64> @llvm.llround.v3i64.v3f32(<3 x float> %x)
+  ret <3 x i64> %a
+}
+declare <3 x i64> @llvm.llround.v3i64.v3f32(<3 x float>)
+
+define <4 x i64> @llround_v4i64_v4f32(<4 x float> %x) nounwind {
+; RV32-LABEL: llround_v4i64_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    add a0, a1, a0
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v4i64_v4f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v11
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    ret
+  %a = call <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llround.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llround_v8i64_v8f32(<8 x float> %x) nounwind {
+; RV32-LABEL: llround_v8i64_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -208
+; RV32-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 200(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 208
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 64(sp)
+; RV32-NEXT:    sw a1, 68(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 120(sp)
+; RV32-NEXT:    sw a1, 124(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 6
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 112(sp)
+; RV32-NEXT:    sw a1, 116(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 5
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 104(sp)
+; RV32-NEXT:    sw a1, 108(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 96(sp)
+; RV32-NEXT:    sw a1, 100(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 88(sp)
+; RV32-NEXT:    sw a1, 92(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 80(sp)
+; RV32-NEXT:    sw a1, 84(sp)
+; RV32-NEXT:    addi a0, sp, 192
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 72(sp)
+; RV32-NEXT:    sw a1, 76(sp)
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -208
+; RV32-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 200(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 208
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v8i64_v8f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -128
+; RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 128
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslidedown.vi v10, v8, 7
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 6
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 3
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    sd a4, 32(sp)
+; RV64-NEXT:    sd a3, 40(sp)
+; RV64-NEXT:    sd a2, 48(sp)
+; RV64-NEXT:    sd a1, 56(sp)
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    sd a0, 0(sp)
+; RV64-NEXT:    sd a1, 8(sp)
+; RV64-NEXT:    sd a6, 16(sp)
+; RV64-NEXT:    sd a5, 24(sp)
+; RV64-NEXT:    mv a0, sp
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -128
+; RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 128
+; RV64-NEXT:    ret
+  %a = call <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llround.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llround_v16i64_v16f32(<16 x float> %x) nounwind {
+; RV32-LABEL: llround_v16i64_v16f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -400
+; RV32-NEXT:    sw ra, 396(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 392(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 400
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -128
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    flw fa0, 124(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 248(sp)
+; RV32-NEXT:    sw a1, 252(sp)
+; RV32-NEXT:    flw fa0, 120(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 240(sp)
+; RV32-NEXT:    sw a1, 244(sp)
+; RV32-NEXT:    flw fa0, 116(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 232(sp)
+; RV32-NEXT:    sw a1, 236(sp)
+; RV32-NEXT:    flw fa0, 112(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 224(sp)
+; RV32-NEXT:    sw a1, 228(sp)
+; RV32-NEXT:    flw fa0, 108(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 216(sp)
+; RV32-NEXT:    sw a1, 220(sp)
+; RV32-NEXT:    flw fa0, 104(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 208(sp)
+; RV32-NEXT:    sw a1, 212(sp)
+; RV32-NEXT:    flw fa0, 100(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 200(sp)
+; RV32-NEXT:    sw a1, 204(sp)
+; RV32-NEXT:    flw fa0, 96(sp)
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 192(sp)
+; RV32-NEXT:    sw a1, 196(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 128(sp)
+; RV32-NEXT:    sw a1, 132(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 152(sp)
+; RV32-NEXT:    sw a1, 156(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 144(sp)
+; RV32-NEXT:    sw a1, 148(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 136(sp)
+; RV32-NEXT:    sw a1, 140(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 184(sp)
+; RV32-NEXT:    sw a1, 188(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 6
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 176(sp)
+; RV32-NEXT:    sw a1, 180(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 5
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 168(sp)
+; RV32-NEXT:    sw a1, 172(sp)
+; RV32-NEXT:    addi a0, sp, 384
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llroundf
+; RV32-NEXT:    sw a0, 160(sp)
+; RV32-NEXT:    sw a1, 164(sp)
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    addi a1, sp, 128
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vle32.v v8, (a1)
+; RV32-NEXT:    addi sp, s0, -400
+; RV32-NEXT:    lw ra, 396(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 392(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 400
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v16i64_v16f32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -384
+; RV64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 384
+; RV64-NEXT:    andi sp, sp, -128
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-NEXT:    vse32.v v8, (a0)
+; RV64-NEXT:    flw fa5, 124(sp)
+; RV64-NEXT:    vfmv.f.s fa4, v8
+; RV64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-NEXT:    vslidedown.vi v12, v8, 2
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    sd a0, 248(sp)
+; RV64-NEXT:    flw fa5, 120(sp)
+; RV64-NEXT:    vslidedown.vi v13, v8, 1
+; RV64-NEXT:    fcvt.l.s a0, fa4, rmm
+; RV64-NEXT:    vfmv.f.s fa4, v10
+; RV64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-NEXT:    sd a1, 240(sp)
+; RV64-NEXT:    flw fa5, 116(sp)
+; RV64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 7
+; RV64-NEXT:    fcvt.l.s a1, fa4, rmm
+; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-NEXT:    sd a2, 232(sp)
+; RV64-NEXT:    flw fa5, 112(sp)
+; RV64-NEXT:    fcvt.l.s a2, fa4, rmm
+; RV64-NEXT:    vfmv.f.s fa4, v13
+; RV64-NEXT:    vslidedown.vi v12, v8, 6
+; RV64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-NEXT:    sd a3, 224(sp)
+; RV64-NEXT:    flw fa5, 108(sp)
+; RV64-NEXT:    fcvt.l.s a3, fa4, rmm
+; RV64-NEXT:    vfmv.f.s fa4, v10
+; RV64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-NEXT:    sd a4, 216(sp)
+; RV64-NEXT:    flw fa5, 104(sp)
+; RV64-NEXT:    fcvt.l.s a4, fa4, rmm
+; RV64-NEXT:    vfmv.f.s fa4, v12
+; RV64-NEXT:    fcvt.l.s a5, fa4, rmm
+; RV64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-NEXT:    sd a6, 208(sp)
+; RV64-NEXT:    flw fa5, 100(sp)
+; RV64-NEXT:    vfmv.f.s fa4, v10
+; RV64-NEXT:    fcvt.l.s a6, fa4, rmm
+; RV64-NEXT:    vslidedown.vi v8, v8, 4
+; RV64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    sd a7, 200(sp)
+; RV64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-NEXT:    flw fa5, 96(sp)
+; RV64-NEXT:    sd a0, 128(sp)
+; RV64-NEXT:    sd a3, 136(sp)
+; RV64-NEXT:    sd a2, 144(sp)
+; RV64-NEXT:    sd a1, 152(sp)
+; RV64-NEXT:    sd a7, 160(sp)
+; RV64-NEXT:    sd a6, 168(sp)
+; RV64-NEXT:    sd a5, 176(sp)
+; RV64-NEXT:    sd a4, 184(sp)
+; RV64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-NEXT:    sd a0, 192(sp)
+; RV64-NEXT:    addi a0, sp, 128
+; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -384
+; RV64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 384
+; RV64-NEXT:    ret
+  %a = call <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llround.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @llround_v1i64_v1f64(<1 x double> %x) nounwind {
+; RV32-LABEL: llround_v1i64_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a1, 4(sp)
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vlse64.v v8, (a0), zero
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v1i64_v1f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    vmv.s.x v8, a0
+; RV64-NEXT:    ret
+  %a = call <1 x i64> @llvm.llround.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llround.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llround_v2i64_v2f64(<2 x double> %x) nounwind {
+; RV32-LABEL: llround_v2i64_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    add a2, sp, a2
+; RV32-NEXT:    addi a2, a2, 16
+; RV32-NEXT:    vl1r.v v8, (a2) # vscale x 8-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v2i64_v2f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v9
+; RV64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-NEXT:    vmv.v.x v8, a0
+; RV64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-NEXT:    ret
+  %a = call <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llround.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llround_v4i64_v4f64(<4 x double> %x) nounwind {
+; RV32-LABEL: llround_v4i64_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -32
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add a0, sp, a0
+; RV32-NEXT:    addi a0, a0, 16
+; RV32-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    addi a2, sp, 16
+; RV32-NEXT:    vl2r.v v8, (a2) # vscale x 16-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 32
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v4i64_v4f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v12, v8, 1
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v12
+; RV64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v10
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    vslide1down.vx v8, v10, a1
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-NEXT:    ret
+  %a = call <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llround.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llround_v8i64_v8f64(<8 x double> %x) nounwind {
+; RV32-LABEL: llround_v8i64_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -272
+; RV32-NEXT:    sw ra, 268(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 264(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 272
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    sub sp, sp, a0
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    addi a0, sp, 256
+; RV32-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV32-NEXT:    vse64.v v8, (a0)
+; RV32-NEXT:    fld fa0, 120(sp)
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 184(sp)
+; RV32-NEXT:    sw a1, 188(sp)
+; RV32-NEXT:    fld fa0, 112(sp)
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 176(sp)
+; RV32-NEXT:    sw a1, 180(sp)
+; RV32-NEXT:    fld fa0, 104(sp)
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 168(sp)
+; RV32-NEXT:    sw a1, 172(sp)
+; RV32-NEXT:    fld fa0, 96(sp)
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 160(sp)
+; RV32-NEXT:    sw a1, 164(sp)
+; RV32-NEXT:    addi a0, sp, 256
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 128(sp)
+; RV32-NEXT:    sw a1, 132(sp)
+; RV32-NEXT:    addi a0, sp, 256
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 136(sp)
+; RV32-NEXT:    sw a1, 140(sp)
+; RV32-NEXT:    addi a0, sp, 256
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 152(sp)
+; RV32-NEXT:    sw a1, 156(sp)
+; RV32-NEXT:    addi a0, sp, 256
+; RV32-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 2
+; RV32-NEXT:    vfmv.f.s fa0, v8
+; RV32-NEXT:    call llround
+; RV32-NEXT:    sw a0, 144(sp)
+; RV32-NEXT:    sw a1, 148(sp)
+; RV32-NEXT:    addi a0, sp, 128
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -272
+; RV32-NEXT:    lw ra, 268(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 264(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 272
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: llround_v8i64_v8f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -192
+; RV64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
+; RV64-NEXT:    addi s0, sp, 192
+; RV64-NEXT:    andi sp, sp, -64
+; RV64-NEXT:    mv a0, sp
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-NEXT:    vse64.v v8, (a0)
+; RV64-NEXT:    fld fa5, 56(sp)
+; RV64-NEXT:    vfmv.f.s fa4, v8
+; RV64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-NEXT:    fcvt.l.d a0, fa4, rmm
+; RV64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-NEXT:    sd a1, 120(sp)
+; RV64-NEXT:    fld fa5, 48(sp)
+; RV64-NEXT:    vfmv.f.s fa4, v10
+; RV64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-NEXT:    fcvt.l.d a1, fa4, rmm
+; RV64-NEXT:    fcvt.l.d a2, fa5, rmm
+; RV64-NEXT:    sd a2, 112(sp)
+; RV64-NEXT:    fld fa5, 40(sp)
+; RV64-NEXT:    vfmv.f.s fa4, v10
+; RV64-NEXT:    fcvt.l.d a2, fa4, rmm
+; RV64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-NEXT:    fcvt.l.d a3, fa5, rmm
+; RV64-NEXT:    vfmv.f.s fa5, v8
+; RV64-NEXT:    sd a3, 104(sp)
+; RV64-NEXT:    fcvt.l.d a3, fa5, rmm
+; RV64-NEXT:    fld fa5, 32(sp)
+; RV64-NEXT:    sd a0, 64(sp)
+; RV64-NEXT:    sd a1, 72(sp)
+; RV64-NEXT:    sd a3, 80(sp)
+; RV64-NEXT:    sd a2, 88(sp)
+; RV64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-NEXT:    sd a0, 96(sp)
+; RV64-NEXT:    addi a0, sp, 64
+; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-NEXT:    vle64.v v8, (a0)
+; RV64-NEXT:    addi sp, s0, -192
+; RV64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 192
+; RV64-NEXT:    ret
+  %a = call <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llround.v8i64.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
index 27abd624f776..ef2208666e0b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lrint.ll
@@ -9,27 +9,21 @@
 define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
 ; RV32-LABEL: lrint_v1f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v1f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v1f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-i64-NEXT:    vmv1r.v v8, v9
 ; RV64-i64-NEXT:    ret
   %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
   ret <1 x iXLen> %a
@@ -40,38 +34,20 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 ; RV32-LABEL: lrint_v2f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v2f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v2f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-i64-NEXT:    vfwcvt.x.f.v v9, v8
+; RV64-i64-NEXT:    vmv1r.v v8, v9
 ; RV64-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
   ret <2 x iXLen> %a
@@ -82,64 +58,20 @@ define <3 x iXLen> @lrint_v3f32(<3 x float> %x) {
 ; RV32-LABEL: lrint_v3f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v3f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v3f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v11
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i64-NEXT:    vmv1r.v v10, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-i64-NEXT:    ret
   %a = call <3 x iXLen> @llvm.lrint.v3iXLen.v3f32(<3 x float> %x)
   ret <3 x iXLen> %a
@@ -150,64 +82,20 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 ; RV32-LABEL: lrint_v4f32:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v4f32:
 ; RV64-i32:       # %bb.0:
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v4f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v11
-; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i64-NEXT:    vmv1r.v v10, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v10
 ; RV64-i64-NEXT:    ret
   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
   ret <4 x iXLen> %a
@@ -217,133 +105,21 @@ declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 ; RV32-LABEL: lrint_v8f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslidedown.vi v11, v8, 2
-; RV32-NEXT:    vslidedown.vi v12, v8, 3
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v11
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 4
-; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 5
-; RV32-NEXT:    vslide1down.vx v10, v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslidedown.vi v12, v8, 6
-; RV32-NEXT:    vslidedown.vi v8, v8, 7
-; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vslide1down.vx v10, v10, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vslide1down.vx v8, v10, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v8f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 3
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v11
 ; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 4
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 5
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 7
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v8f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -128
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 128
-; RV64-i64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 128
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -64
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i64-NEXT:    fcvt.l.s a2, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 4
-; RV64-i64-NEXT:    fcvt.l.s a3, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 3
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
-; RV64-i64-NEXT:    fcvt.l.s a4, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.s a5, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
-; RV64-i64-NEXT:    fcvt.l.s a6, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a4, 32(sp)
-; RV64-i64-NEXT:    sd a3, 40(sp)
-; RV64-i64-NEXT:    sd a2, 48(sp)
-; RV64-i64-NEXT:    sd a1, 56(sp)
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    sd a0, 0(sp)
-; RV64-i64-NEXT:    sd a1, 8(sp)
-; RV64-i64-NEXT:    sd a6, 16(sp)
-; RV64-i64-NEXT:    sd a5, 24(sp)
-; RV64-i64-NEXT:    mv a0, sp
-; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -128
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 128
-; RV64-i64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 128
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-i64-NEXT:    vmv2r.v v12, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v12
 ; RV64-i64-NEXT:    ret
   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
   ret <8 x iXLen> %a
@@ -353,254 +129,21 @@ declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 ; RV32-LABEL: lrint_v16f32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -192
-; RV32-NEXT:    .cfi_def_cfa_offset 192
-; RV32-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 184(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 192
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    mv a0, sp
 ; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v8, (a0)
-; RV32-NEXT:    flw fa5, 60(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v8
-; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 3
-; RV32-NEXT:    vslidedown.vi v11, v8, 2
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 124(sp)
-; RV32-NEXT:    flw fa5, 56(sp)
-; RV32-NEXT:    fcvt.w.s a0, fa4
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 1
-; RV32-NEXT:    fcvt.w.s a1, fa5
-; RV32-NEXT:    sw a1, 120(sp)
-; RV32-NEXT:    flw fa5, 52(sp)
-; RV32-NEXT:    fcvt.w.s a1, fa4
-; RV32-NEXT:    vfmv.f.s fa4, v11
-; RV32-NEXT:    fcvt.w.s a2, fa4
-; RV32-NEXT:    fcvt.w.s a3, fa5
-; RV32-NEXT:    sw a3, 116(sp)
-; RV32-NEXT:    flw fa5, 48(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 7
-; RV32-NEXT:    fcvt.w.s a3, fa4
-; RV32-NEXT:    fcvt.w.s a4, fa5
-; RV32-NEXT:    sw a4, 112(sp)
-; RV32-NEXT:    flw fa5, 44(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 6
-; RV32-NEXT:    fcvt.w.s a4, fa4
-; RV32-NEXT:    fcvt.w.s a5, fa5
-; RV32-NEXT:    sw a5, 108(sp)
-; RV32-NEXT:    flw fa5, 40(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    vslidedown.vi v10, v8, 5
-; RV32-NEXT:    fcvt.w.s a5, fa4
-; RV32-NEXT:    fcvt.w.s a6, fa5
-; RV32-NEXT:    sw a6, 104(sp)
-; RV32-NEXT:    flw fa5, 36(sp)
-; RV32-NEXT:    vfmv.f.s fa4, v10
-; RV32-NEXT:    fcvt.w.s a6, fa4
-; RV32-NEXT:    vslidedown.vi v8, v8, 4
-; RV32-NEXT:    fcvt.w.s a7, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    sw a7, 100(sp)
-; RV32-NEXT:    fcvt.w.s a7, fa5
-; RV32-NEXT:    flw fa5, 32(sp)
-; RV32-NEXT:    sw a0, 64(sp)
-; RV32-NEXT:    sw a3, 68(sp)
-; RV32-NEXT:    sw a2, 72(sp)
-; RV32-NEXT:    sw a1, 76(sp)
-; RV32-NEXT:    sw a7, 80(sp)
-; RV32-NEXT:    sw a6, 84(sp)
-; RV32-NEXT:    sw a5, 88(sp)
-; RV32-NEXT:    sw a4, 92(sp)
-; RV32-NEXT:    fcvt.w.s a0, fa5
-; RV32-NEXT:    sw a0, 96(sp)
-; RV32-NEXT:    addi a0, sp, 64
-; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    addi sp, s0, -192
-; RV32-NEXT:    .cfi_def_cfa sp, 192
-; RV32-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 184(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 192
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v16f32:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    addi sp, sp, -192
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 192
-; RV64-i32-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    .cfi_offset ra, -8
-; RV64-i32-NEXT:    .cfi_offset s0, -16
-; RV64-i32-NEXT:    addi s0, sp, 192
-; RV64-i32-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i32-NEXT:    andi sp, sp, -64
-; RV64-i32-NEXT:    mv a0, sp
-; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i32-NEXT:    vse32.v v8, (a0)
-; RV64-i32-NEXT:    flw fa5, 60(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 124(sp)
-; RV64-i32-NEXT:    flw fa5, 56(sp)
-; RV64-i32-NEXT:    fcvt.l.s a0, fa4
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i32-NEXT:    fcvt.l.s a1, fa5
-; RV64-i32-NEXT:    sw a1, 120(sp)
-; RV64-i32-NEXT:    flw fa5, 52(sp)
-; RV64-i32-NEXT:    fcvt.l.s a1, fa4
-; RV64-i32-NEXT:    vfmv.f.s fa4, v11
-; RV64-i32-NEXT:    fcvt.l.s a2, fa4
-; RV64-i32-NEXT:    fcvt.l.s a3, fa5
-; RV64-i32-NEXT:    sw a3, 116(sp)
-; RV64-i32-NEXT:    flw fa5, 48(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i32-NEXT:    fcvt.l.s a3, fa4
-; RV64-i32-NEXT:    fcvt.l.s a4, fa5
-; RV64-i32-NEXT:    sw a4, 112(sp)
-; RV64-i32-NEXT:    flw fa5, 44(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 6
-; RV64-i32-NEXT:    fcvt.l.s a4, fa4
-; RV64-i32-NEXT:    fcvt.l.s a5, fa5
-; RV64-i32-NEXT:    sw a5, 108(sp)
-; RV64-i32-NEXT:    flw fa5, 40(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i32-NEXT:    fcvt.l.s a5, fa4
-; RV64-i32-NEXT:    fcvt.l.s a6, fa5
-; RV64-i32-NEXT:    sw a6, 104(sp)
-; RV64-i32-NEXT:    flw fa5, 36(sp)
-; RV64-i32-NEXT:    vfmv.f.s fa4, v10
-; RV64-i32-NEXT:    fcvt.l.s a6, fa4
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-i32-NEXT:    fcvt.l.s a7, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    sw a7, 100(sp)
-; RV64-i32-NEXT:    fcvt.l.s a7, fa5
-; RV64-i32-NEXT:    flw fa5, 32(sp)
-; RV64-i32-NEXT:    sw a0, 64(sp)
-; RV64-i32-NEXT:    sw a3, 68(sp)
-; RV64-i32-NEXT:    sw a2, 72(sp)
-; RV64-i32-NEXT:    sw a1, 76(sp)
-; RV64-i32-NEXT:    sw a7, 80(sp)
-; RV64-i32-NEXT:    sw a6, 84(sp)
-; RV64-i32-NEXT:    sw a5, 88(sp)
-; RV64-i32-NEXT:    sw a4, 92(sp)
-; RV64-i32-NEXT:    fcvt.l.s a0, fa5
-; RV64-i32-NEXT:    sw a0, 96(sp)
-; RV64-i32-NEXT:    addi a0, sp, 64
 ; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i32-NEXT:    vle32.v v8, (a0)
-; RV64-i32-NEXT:    addi sp, s0, -192
-; RV64-i32-NEXT:    .cfi_def_cfa sp, 192
-; RV64-i32-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    .cfi_restore ra
-; RV64-i32-NEXT:    .cfi_restore s0
-; RV64-i32-NEXT:    addi sp, sp, 192
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i32-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v16f32:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -384
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 384
-; RV64-i64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 384
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -128
-; RV64-i64-NEXT:    addi a0, sp, 64
 ; RV64-i64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV64-i64-NEXT:    vse32.v v8, (a0)
-; RV64-i64-NEXT:    flw fa5, 124(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 248(sp)
-; RV64-i64-NEXT:    flw fa5, 120(sp)
-; RV64-i64-NEXT:    vslidedown.vi v13, v8, 1
-; RV64-i64-NEXT:    fcvt.l.s a0, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.s a1, fa5
-; RV64-i64-NEXT:    sd a1, 240(sp)
-; RV64-i64-NEXT:    flw fa5, 116(sp)
-; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
-; RV64-i64-NEXT:    fcvt.l.s a1, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v12
-; RV64-i64-NEXT:    fcvt.l.s a2, fa5
-; RV64-i64-NEXT:    sd a2, 232(sp)
-; RV64-i64-NEXT:    flw fa5, 112(sp)
-; RV64-i64-NEXT:    fcvt.l.s a2, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v13
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 6
-; RV64-i64-NEXT:    fcvt.l.s a3, fa5
-; RV64-i64-NEXT:    sd a3, 224(sp)
-; RV64-i64-NEXT:    flw fa5, 108(sp)
-; RV64-i64-NEXT:    fcvt.l.s a3, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
-; RV64-i64-NEXT:    fcvt.l.s a4, fa5
-; RV64-i64-NEXT:    sd a4, 216(sp)
-; RV64-i64-NEXT:    flw fa5, 104(sp)
-; RV64-i64-NEXT:    fcvt.l.s a4, fa4
-; RV64-i64-NEXT:    vfmv.f.s fa4, v12
-; RV64-i64-NEXT:    fcvt.l.s a5, fa4
-; RV64-i64-NEXT:    fcvt.l.s a6, fa5
-; RV64-i64-NEXT:    sd a6, 208(sp)
-; RV64-i64-NEXT:    flw fa5, 100(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.s a6, fa4
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 4
-; RV64-i64-NEXT:    fcvt.l.s a7, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a7, 200(sp)
-; RV64-i64-NEXT:    fcvt.l.s a7, fa5
-; RV64-i64-NEXT:    flw fa5, 96(sp)
-; RV64-i64-NEXT:    sd a0, 128(sp)
-; RV64-i64-NEXT:    sd a3, 136(sp)
-; RV64-i64-NEXT:    sd a2, 144(sp)
-; RV64-i64-NEXT:    sd a1, 152(sp)
-; RV64-i64-NEXT:    sd a7, 160(sp)
-; RV64-i64-NEXT:    sd a6, 168(sp)
-; RV64-i64-NEXT:    sd a5, 176(sp)
-; RV64-i64-NEXT:    sd a4, 184(sp)
-; RV64-i64-NEXT:    fcvt.l.s a0, fa5
-; RV64-i64-NEXT:    sd a0, 192(sp)
-; RV64-i64-NEXT:    addi a0, sp, 128
-; RV64-i64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -384
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 384
-; RV64-i64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 384
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vmv4r.v v16, v8
+; RV64-i64-NEXT:    vfwcvt.x.f.v v8, v16
 ; RV64-i64-NEXT:    ret
   %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
   ret <16 x iXLen> %a
@@ -610,26 +153,22 @@ declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
 ; RV32-LABEL: lrint_v1f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV32-NEXT:    vfncvt.x.f.w v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v1f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vfncvt.x.f.w v9, v8
+; RV64-i32-NEXT:    vmv1r.v v8, v9
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v1f64:
 ; RV64-i64:       # %bb.0:
 ; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double> %x)
   ret <1 x iXLen> %a
@@ -639,40 +178,22 @@ declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 ; RV32-LABEL: lrint_v2f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v9, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v9
-; RV32-NEXT:    fcvt.w.d a1, fa5
 ; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vfncvt.x.f.w v9, v8
+; RV32-NEXT:    vmv1r.v v8, v9
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v2f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v9
-; RV64-i32-NEXT:    fcvt.l.d a1, fa5
 ; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vfncvt.x.f.w v9, v8
+; RV64-i32-NEXT:    vmv1r.v v8, v9
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v2f64:
 ; RV64-i64:       # %bb.0:
 ; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v9
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    vmv.v.x v8, a0
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double> %x)
   ret <2 x iXLen> %a
@@ -682,72 +203,22 @@ declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 ; RV32-LABEL: lrint_v4f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v10, v8, 2
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    fcvt.w.d a1, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v10
 ; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV32-NEXT:    vmv.v.x v9, a0
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV32-NEXT:    vslide1down.vx v8, v9, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    vfncvt.x.f.w v10, v8
+; RV32-NEXT:    vmv.v.v v8, v10
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v4f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    fcvt.l.d a1, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v9, a0
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
-; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    vfncvt.x.f.w v10, v8
+; RV64-i32-NEXT:    vmv.v.v v8, v10
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v4f64:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v12, v8, 1
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v12
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v10
 ; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; RV64-i64-NEXT:    vmv.v.x v10, a0
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    vslide1down.vx v8, v10, a1
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double> %x)
   ret <4 x iXLen> %a
@@ -757,168 +228,22 @@ declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
 define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; RV32-LABEL: lrint_v8f64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -128
-; RV32-NEXT:    .cfi_def_cfa_offset 128
-; RV32-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
-; RV32-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
-; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    .cfi_offset s0, -8
-; RV32-NEXT:    addi s0, sp, 128
-; RV32-NEXT:    .cfi_def_cfa s0, 0
-; RV32-NEXT:    andi sp, sp, -64
-; RV32-NEXT:    mv a0, sp
-; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV32-NEXT:    vslidedown.vi v14, v8, 1
-; RV32-NEXT:    vfmv.f.s fa5, v8
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v12, v8, 2
-; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV32-NEXT:    vse64.v v8, (a0)
-; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV32-NEXT:    vslidedown.vi v8, v8, 3
-; RV32-NEXT:    vfmv.f.s fa4, v14
-; RV32-NEXT:    fcvt.w.d a0, fa5
-; RV32-NEXT:    vfmv.f.s fa5, v12
-; RV32-NEXT:    vfmv.f.s fa3, v8
-; RV32-NEXT:    fcvt.w.d a1, fa4
-; RV32-NEXT:    fcvt.w.d a2, fa5
-; RV32-NEXT:    fcvt.w.d a3, fa3
-; RV32-NEXT:    fld fa5, 32(sp)
-; RV32-NEXT:    fld fa4, 40(sp)
-; RV32-NEXT:    fld fa3, 48(sp)
-; RV32-NEXT:    fld fa2, 56(sp)
-; RV32-NEXT:    fcvt.w.d a4, fa5
-; RV32-NEXT:    fcvt.w.d a5, fa4
-; RV32-NEXT:    fcvt.w.d a6, fa3
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
-; RV32-NEXT:    vslide1down.vx v8, v8, a2
-; RV32-NEXT:    vslide1down.vx v8, v8, a3
-; RV32-NEXT:    vslide1down.vx v8, v8, a4
-; RV32-NEXT:    vslide1down.vx v8, v8, a5
-; RV32-NEXT:    vslide1down.vx v8, v8, a6
-; RV32-NEXT:    fcvt.w.d a0, fa2
-; RV32-NEXT:    vslide1down.vx v8, v8, a0
-; RV32-NEXT:    addi sp, s0, -128
-; RV32-NEXT:    .cfi_def_cfa sp, 128
-; RV32-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
-; RV32-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
-; RV32-NEXT:    .cfi_restore ra
-; RV32-NEXT:    .cfi_restore s0
-; RV32-NEXT:    addi sp, sp, 128
-; RV32-NEXT:    .cfi_def_cfa_offset 0
+; RV32-NEXT:    vfncvt.x.f.w v12, v8
+; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
 ; RV64-i32-LABEL: lrint_v8f64:
 ; RV64-i32:       # %bb.0:
-; RV64-i32-NEXT:    addi sp, sp, -128
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 128
-; RV64-i32-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
-; RV64-i32-NEXT:    .cfi_offset ra, -8
-; RV64-i32-NEXT:    .cfi_offset s0, -16
-; RV64-i32-NEXT:    addi s0, sp, 128
-; RV64-i32-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i32-NEXT:    andi sp, sp, -64
-; RV64-i32-NEXT:    mv a0, sp
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v14, v8, 1
-; RV64-i32-NEXT:    vfmv.f.s fa5, v8
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v12, v8, 2
-; RV64-i32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i32-NEXT:    vse64.v v8, (a0)
-; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
-; RV64-i32-NEXT:    vfmv.f.s fa4, v14
-; RV64-i32-NEXT:    fcvt.l.d a0, fa5
-; RV64-i32-NEXT:    vfmv.f.s fa5, v12
-; RV64-i32-NEXT:    vfmv.f.s fa3, v8
-; RV64-i32-NEXT:    fcvt.l.d a1, fa4
-; RV64-i32-NEXT:    fcvt.l.d a2, fa5
-; RV64-i32-NEXT:    fcvt.l.d a3, fa3
-; RV64-i32-NEXT:    fld fa5, 32(sp)
-; RV64-i32-NEXT:    fld fa4, 40(sp)
-; RV64-i32-NEXT:    fld fa3, 48(sp)
-; RV64-i32-NEXT:    fld fa2, 56(sp)
-; RV64-i32-NEXT:    fcvt.l.d a4, fa5
-; RV64-i32-NEXT:    fcvt.l.d a5, fa4
-; RV64-i32-NEXT:    fcvt.l.d a6, fa3
 ; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV64-i32-NEXT:    vmv.v.x v8, a0
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a2
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a3
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a4
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a5
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a6
-; RV64-i32-NEXT:    fcvt.l.d a0, fa2
-; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
-; RV64-i32-NEXT:    addi sp, s0, -128
-; RV64-i32-NEXT:    .cfi_def_cfa sp, 128
-; RV64-i32-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
-; RV64-i32-NEXT:    .cfi_restore ra
-; RV64-i32-NEXT:    .cfi_restore s0
-; RV64-i32-NEXT:    addi sp, sp, 128
-; RV64-i32-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i32-NEXT:    vfncvt.x.f.w v12, v8
+; RV64-i32-NEXT:    vmv.v.v v8, v12
 ; RV64-i32-NEXT:    ret
 ;
 ; RV64-i64-LABEL: lrint_v8f64:
 ; RV64-i64:       # %bb.0:
-; RV64-i64-NEXT:    addi sp, sp, -192
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 192
-; RV64-i64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
-; RV64-i64-NEXT:    .cfi_offset ra, -8
-; RV64-i64-NEXT:    .cfi_offset s0, -16
-; RV64-i64-NEXT:    addi s0, sp, 192
-; RV64-i64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-i64-NEXT:    andi sp, sp, -64
-; RV64-i64-NEXT:    mv a0, sp
-; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vse64.v v8, (a0)
-; RV64-i64-NEXT:    fld fa5, 56(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v8
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 1
-; RV64-i64-NEXT:    fcvt.l.d a0, fa4
-; RV64-i64-NEXT:    fcvt.l.d a1, fa5
-; RV64-i64-NEXT:    sd a1, 120(sp)
-; RV64-i64-NEXT:    fld fa5, 48(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
-; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
-; RV64-i64-NEXT:    fcvt.l.d a1, fa4
-; RV64-i64-NEXT:    fcvt.l.d a2, fa5
-; RV64-i64-NEXT:    sd a2, 112(sp)
-; RV64-i64-NEXT:    fld fa5, 40(sp)
-; RV64-i64-NEXT:    vfmv.f.s fa4, v10
-; RV64-i64-NEXT:    fcvt.l.d a2, fa4
-; RV64-i64-NEXT:    vslidedown.vi v8, v8, 2
-; RV64-i64-NEXT:    fcvt.l.d a3, fa5
-; RV64-i64-NEXT:    vfmv.f.s fa5, v8
-; RV64-i64-NEXT:    sd a3, 104(sp)
-; RV64-i64-NEXT:    fcvt.l.d a3, fa5
-; RV64-i64-NEXT:    fld fa5, 32(sp)
-; RV64-i64-NEXT:    sd a0, 64(sp)
-; RV64-i64-NEXT:    sd a1, 72(sp)
-; RV64-i64-NEXT:    sd a3, 80(sp)
-; RV64-i64-NEXT:    sd a2, 88(sp)
-; RV64-i64-NEXT:    fcvt.l.d a0, fa5
-; RV64-i64-NEXT:    sd a0, 96(sp)
-; RV64-i64-NEXT:    addi a0, sp, 64
 ; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
-; RV64-i64-NEXT:    vle64.v v8, (a0)
-; RV64-i64-NEXT:    addi sp, s0, -192
-; RV64-i64-NEXT:    .cfi_def_cfa sp, 192
-; RV64-i64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
-; RV64-i64-NEXT:    .cfi_restore ra
-; RV64-i64-NEXT:    .cfi_restore s0
-; RV64-i64-NEXT:    addi sp, sp, 192
-; RV64-i64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-i64-NEXT:    vfcvt.x.f.v v8, v8
 ; RV64-i64-NEXT:    ret
   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double> %x)
   ret <8 x iXLen> %a
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lround.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lround.ll
new file mode 100644
index 000000000000..8289a8b8f833
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-lround.ll
@@ -0,0 +1,1594 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+f,+d,+zvfh \
+; RUN:     -target-abi=ilp32d -verify-machineinstrs | FileCheck %s --check-prefix=RV32
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh \
+; RUN:     -target-abi=lp64d -verify-machineinstrs | FileCheck %s --check-prefix=RV64-i32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+f,+d,+zvfh \
+; RUN:     -target-abi=lp64d -verify-machineinstrs | FileCheck %s --check-prefix=RV64-i64
+
+define <1 x iXLen> @lround_v1f16(<1 x half> %x) nounwind {
+; RV32-LABEL: lround_v1f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v1f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v1f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lround.v1iXLen.v1f16(<1 x half> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lround.v1iXLen.v1f16(<1 x half>)
+
+define <2 x iXLen> @lround_v2f16(<2 x half> %x) nounwind {
+; RV32-LABEL: lround_v2f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v2f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v8, a0
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v2f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lround.v2iXLen.v2f16(<2 x half> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lround.v2iXLen.v2f16(<2 x half>)
+
+define <3 x iXLen> @lround_v3f16(<3 x half> %x) nounwind {
+; RV32-LABEL: lround_v3f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v9, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v3f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v9, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v3f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <3 x iXLen> @llvm.lround.v3iXLen.v3f16(<3 x half> %x)
+  ret <3 x iXLen> %a
+}
+declare <3 x iXLen> @llvm.lround.v3iXLen.v3f16(<3 x half>)
+
+define <4 x iXLen> @lround_v4f16(<4 x half> %x) nounwind {
+; RV32-LABEL: lround_v4f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v9, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v4f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v9, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v4f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lround.v4iXLen.v4f16(<4 x half> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lround.v4iXLen.v4f16(<4 x half>)
+
+define <8 x iXLen> @lround_v8f16(<8 x half> %x) nounwind {
+; RV32-LABEL: lround_v8f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v12, v8, 3
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v10, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v9, v8, 4
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    vslidedown.vi v9, v8, 5
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    vslidedown.vi v12, v8, 6
+; RV32-NEXT:    vslidedown.vi v13, v8, 7
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v10, a1
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v13
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v8f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 3
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v10, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 4
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 5
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 6
+; RV64-i32-NEXT:    vslidedown.vi v13, v8, 7
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v10, a1
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v13
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v8f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    addi sp, sp, -128
+; RV64-i64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    addi s0, sp, 128
+; RV64-i64-NEXT:    andi sp, sp, -64
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 7
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 5
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 4
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    sd a4, 32(sp)
+; RV64-i64-NEXT:    sd a3, 40(sp)
+; RV64-i64-NEXT:    sd a2, 48(sp)
+; RV64-i64-NEXT:    sd a1, 56(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 0(sp)
+; RV64-i64-NEXT:    sd a1, 8(sp)
+; RV64-i64-NEXT:    sd a6, 16(sp)
+; RV64-i64-NEXT:    sd a5, 24(sp)
+; RV64-i64-NEXT:    mv a0, sp
+; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-i64-NEXT:    vle64.v v8, (a0)
+; RV64-i64-NEXT:    addi sp, s0, -128
+; RV64-i64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    addi sp, sp, 128
+; RV64-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lround.v8iXLen.v8f16(<8 x half> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lround.v8iXLen.v8f16(<8 x half>)
+
+define <16 x iXLen> @lround_v16f16(<16 x half> %x) nounwind {
+; RV32-LABEL: lround_v16f16:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -128
+; RV32-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 128
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 15
+; RV32-NEXT:    vslidedown.vi v12, v8, 14
+; RV32-NEXT:    vslidedown.vi v14, v8, 13
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 12
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 11
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a2, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v14
+; RV32-NEXT:    vslidedown.vi v14, v8, 10
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a5, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a4, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v14
+; RV32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 7
+; RV32-NEXT:    vslidedown.vi v11, v8, 6
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a6, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 5
+; RV32-NEXT:    vslidedown.vi v13, v8, 4
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 3
+; RV32-NEXT:    vslidedown.vi v14, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 1
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s t0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s t1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v11
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s t2, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s t3, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v13
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    fcvt.w.s t4, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    sw a5, 48(sp)
+; RV32-NEXT:    sw a3, 52(sp)
+; RV32-NEXT:    sw a2, 56(sp)
+; RV32-NEXT:    sw a1, 60(sp)
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v14
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    sw t0, 32(sp)
+; RV32-NEXT:    sw a7, 36(sp)
+; RV32-NEXT:    sw a6, 40(sp)
+; RV32-NEXT:    sw a4, 44(sp)
+; RV32-NEXT:    fcvt.w.s a2, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.s.h fa5, fa5
+; RV32-NEXT:    sw t4, 16(sp)
+; RV32-NEXT:    sw t3, 20(sp)
+; RV32-NEXT:    sw t2, 24(sp)
+; RV32-NEXT:    sw t1, 28(sp)
+; RV32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV32-NEXT:    sw a0, 0(sp)
+; RV32-NEXT:    sw a3, 4(sp)
+; RV32-NEXT:    sw a2, 8(sp)
+; RV32-NEXT:    sw a1, 12(sp)
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -128
+; RV32-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 128
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v16f16:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    addi sp, sp, -128
+; RV64-i32-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    addi s0, sp, 128
+; RV64-i32-NEXT:    andi sp, sp, -64
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 15
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 14
+; RV64-i32-NEXT:    vslidedown.vi v14, v8, 13
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 12
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 11
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a2, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v14
+; RV64-i32-NEXT:    vslidedown.vi v14, v8, 10
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a5, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a4, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v14
+; RV64-i32-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 7
+; RV64-i32-NEXT:    vslidedown.vi v11, v8, 6
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a6, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-i32-NEXT:    vslidedown.vi v13, v8, 4
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 3
+; RV64-i32-NEXT:    vslidedown.vi v14, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s t0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s t1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v11
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s t2, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s t3, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v13
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    fcvt.w.s t4, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    sw a5, 48(sp)
+; RV64-i32-NEXT:    sw a3, 52(sp)
+; RV64-i32-NEXT:    sw a2, 56(sp)
+; RV64-i32-NEXT:    sw a1, 60(sp)
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v14
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    sw t0, 32(sp)
+; RV64-i32-NEXT:    sw a7, 36(sp)
+; RV64-i32-NEXT:    sw a6, 40(sp)
+; RV64-i32-NEXT:    sw a4, 44(sp)
+; RV64-i32-NEXT:    fcvt.w.s a2, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i32-NEXT:    sw t4, 16(sp)
+; RV64-i32-NEXT:    sw t3, 20(sp)
+; RV64-i32-NEXT:    sw t2, 24(sp)
+; RV64-i32-NEXT:    sw t1, 28(sp)
+; RV64-i32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV64-i32-NEXT:    sw a0, 0(sp)
+; RV64-i32-NEXT:    sw a3, 4(sp)
+; RV64-i32-NEXT:    sw a2, 8(sp)
+; RV64-i32-NEXT:    sw a1, 12(sp)
+; RV64-i32-NEXT:    mv a0, sp
+; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-i32-NEXT:    vle32.v v8, (a0)
+; RV64-i32-NEXT:    addi sp, s0, -128
+; RV64-i32-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    addi sp, sp, 128
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v16f16:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    addi sp, sp, -256
+; RV64-i64-NEXT:    sd ra, 248(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    sd s0, 240(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    addi s0, sp, 256
+; RV64-i64-NEXT:    andi sp, sp, -128
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 15
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 14
+; RV64-i64-NEXT:    vslidedown.vi v14, v8, 13
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 12
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v12
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 11
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v14
+; RV64-i64-NEXT:    vslidedown.vi v14, v8, 10
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v12
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 8
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v14
+; RV64-i64-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 7
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 6
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-i64-NEXT:    vslidedown.vi v13, v8, 4
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v12
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 3
+; RV64-i64-NEXT:    vslidedown.vi v14, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s t0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s t1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s t2, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s t3, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v13
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    fcvt.l.s t4, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v12
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    sd a5, 96(sp)
+; RV64-i64-NEXT:    sd a3, 104(sp)
+; RV64-i64-NEXT:    sd a2, 112(sp)
+; RV64-i64-NEXT:    sd a1, 120(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v14
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    sd t0, 64(sp)
+; RV64-i64-NEXT:    sd a7, 72(sp)
+; RV64-i64-NEXT:    sd a6, 80(sp)
+; RV64-i64-NEXT:    sd a4, 88(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.s.h fa5, fa5
+; RV64-i64-NEXT:    sd t4, 32(sp)
+; RV64-i64-NEXT:    sd t3, 40(sp)
+; RV64-i64-NEXT:    sd t2, 48(sp)
+; RV64-i64-NEXT:    sd t1, 56(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 0(sp)
+; RV64-i64-NEXT:    sd a3, 8(sp)
+; RV64-i64-NEXT:    sd a2, 16(sp)
+; RV64-i64-NEXT:    sd a1, 24(sp)
+; RV64-i64-NEXT:    mv a0, sp
+; RV64-i64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-i64-NEXT:    vle64.v v8, (a0)
+; RV64-i64-NEXT:    addi sp, s0, -256
+; RV64-i64-NEXT:    ld ra, 248(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    ld s0, 240(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    addi sp, sp, 256
+; RV64-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lround.v16iXLen.v16f16(<16 x half> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lround.v16iXLen.v16f16(<16 x half>)
+
+define <1 x iXLen> @lround_v1f32(<1 x float> %x) nounwind {
+; RV32-LABEL: lround_v1f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v1f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v1f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lround.v1iXLen.v1f32(<1 x float> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lround.v1iXLen.v1f32(<1 x float>)
+
+define <2 x iXLen> @lround_v2f32(<2 x float> %x) nounwind {
+; RV32-LABEL: lround_v2f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v2f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vmv.v.x v8, a0
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v2f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lround.v2iXLen.v2f32(<2 x float> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lround.v2iXLen.v2f32(<2 x float>)
+
+define <3 x iXLen> @lround_v3f32(<3 x float> %x) nounwind {
+; RV32-LABEL: lround_v3f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vmv.v.x v9, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslide1down.vx v8, v9, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v3f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vmv.v.x v9, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v3f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <3 x iXLen> @llvm.lround.v3iXLen.v3f32(<3 x float> %x)
+  ret <3 x iXLen> %a
+}
+declare <3 x iXLen> @llvm.lround.v3iXLen.v3f32(<3 x float>)
+
+define <4 x iXLen> @lround_v4f32(<4 x float> %x) nounwind {
+; RV32-LABEL: lround_v4f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vmv.v.x v9, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslide1down.vx v8, v9, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v4f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vmv.v.x v9, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v4f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v11, v8, 3
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v11
+; RV64-i64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lround.v4iXLen.v4f32(<4 x float> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lround.v4iXLen.v4f32(<4 x float>)
+
+define <8 x iXLen> @lround_v8f32(<8 x float> %x) nounwind {
+; RV32-LABEL: lround_v8f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslidedown.vi v11, v8, 2
+; RV32-NEXT:    vslidedown.vi v12, v8, 3
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v11
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v10, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 4
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 5
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslidedown.vi v12, v8, 6
+; RV32-NEXT:    vslidedown.vi v8, v8, 7
+; RV32-NEXT:    vslide1down.vx v10, v10, a1
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vslide1down.vx v10, v10, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vslide1down.vx v8, v10, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v8f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 3
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v11
+; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v10, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 4
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 5
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 6
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 7
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a1
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vslide1down.vx v10, v10, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vslide1down.vx v8, v10, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v8f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    addi sp, sp, -128
+; RV64-i64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    addi s0, sp, 128
+; RV64-i64-NEXT:    andi sp, sp, -64
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 6
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 4
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 3
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 1
+; RV64-i64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.l.s a5, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    sd a4, 32(sp)
+; RV64-i64-NEXT:    sd a3, 40(sp)
+; RV64-i64-NEXT:    sd a2, 48(sp)
+; RV64-i64-NEXT:    sd a1, 56(sp)
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 0(sp)
+; RV64-i64-NEXT:    sd a1, 8(sp)
+; RV64-i64-NEXT:    sd a6, 16(sp)
+; RV64-i64-NEXT:    sd a5, 24(sp)
+; RV64-i64-NEXT:    mv a0, sp
+; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-i64-NEXT:    vle64.v v8, (a0)
+; RV64-i64-NEXT:    addi sp, s0, -128
+; RV64-i64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    addi sp, sp, 128
+; RV64-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lround.v8iXLen.v8f32(<8 x float> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lround.v8iXLen.v8f32(<8 x float>)
+
+define <16 x iXLen> @lround_v16f32(<16 x float> %x) nounwind {
+; RV32-LABEL: lround_v16f32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -192
+; RV32-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 184(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 192
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vse32.v v8, (a0)
+; RV32-NEXT:    flw fa5, 60(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v8
+; RV32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 3
+; RV32-NEXT:    vslidedown.vi v11, v8, 2
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    sw a0, 124(sp)
+; RV32-NEXT:    flw fa5, 56(sp)
+; RV32-NEXT:    fcvt.w.s a0, fa4, rmm
+; RV32-NEXT:    vfmv.f.s fa4, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 1
+; RV32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV32-NEXT:    sw a1, 120(sp)
+; RV32-NEXT:    flw fa5, 52(sp)
+; RV32-NEXT:    fcvt.w.s a1, fa4, rmm
+; RV32-NEXT:    vfmv.f.s fa4, v11
+; RV32-NEXT:    fcvt.w.s a2, fa4, rmm
+; RV32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV32-NEXT:    sw a3, 116(sp)
+; RV32-NEXT:    flw fa5, 48(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v10
+; RV32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 7
+; RV32-NEXT:    fcvt.w.s a3, fa4, rmm
+; RV32-NEXT:    fcvt.w.s a4, fa5, rmm
+; RV32-NEXT:    sw a4, 112(sp)
+; RV32-NEXT:    flw fa5, 44(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 6
+; RV32-NEXT:    fcvt.w.s a4, fa4, rmm
+; RV32-NEXT:    fcvt.w.s a5, fa5, rmm
+; RV32-NEXT:    sw a5, 108(sp)
+; RV32-NEXT:    flw fa5, 40(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v10
+; RV32-NEXT:    vslidedown.vi v10, v8, 5
+; RV32-NEXT:    fcvt.w.s a5, fa4, rmm
+; RV32-NEXT:    fcvt.w.s a6, fa5, rmm
+; RV32-NEXT:    sw a6, 104(sp)
+; RV32-NEXT:    flw fa5, 36(sp)
+; RV32-NEXT:    vfmv.f.s fa4, v10
+; RV32-NEXT:    fcvt.w.s a6, fa4, rmm
+; RV32-NEXT:    vslidedown.vi v8, v8, 4
+; RV32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    sw a7, 100(sp)
+; RV32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV32-NEXT:    flw fa5, 32(sp)
+; RV32-NEXT:    sw a0, 64(sp)
+; RV32-NEXT:    sw a3, 68(sp)
+; RV32-NEXT:    sw a2, 72(sp)
+; RV32-NEXT:    sw a1, 76(sp)
+; RV32-NEXT:    sw a7, 80(sp)
+; RV32-NEXT:    sw a6, 84(sp)
+; RV32-NEXT:    sw a5, 88(sp)
+; RV32-NEXT:    sw a4, 92(sp)
+; RV32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV32-NEXT:    sw a0, 96(sp)
+; RV32-NEXT:    addi a0, sp, 64
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v8, (a0)
+; RV32-NEXT:    addi sp, s0, -192
+; RV32-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 184(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 192
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v16f32:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    addi sp, sp, -192
+; RV64-i32-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    addi s0, sp, 192
+; RV64-i32-NEXT:    andi sp, sp, -64
+; RV64-i32-NEXT:    mv a0, sp
+; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-i32-NEXT:    vse32.v v8, (a0)
+; RV64-i32-NEXT:    flw fa5, 60(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v8
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-i32-NEXT:    vslidedown.vi v11, v8, 2
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    sw a0, 124(sp)
+; RV64-i32-NEXT:    flw fa5, 56(sp)
+; RV64-i32-NEXT:    fcvt.w.s a0, fa4, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa4, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-i32-NEXT:    fcvt.w.s a1, fa5, rmm
+; RV64-i32-NEXT:    sw a1, 120(sp)
+; RV64-i32-NEXT:    flw fa5, 52(sp)
+; RV64-i32-NEXT:    fcvt.w.s a1, fa4, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa4, v11
+; RV64-i32-NEXT:    fcvt.w.s a2, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.s a3, fa5, rmm
+; RV64-i32-NEXT:    sw a3, 116(sp)
+; RV64-i32-NEXT:    flw fa5, 48(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v10
+; RV64-i32-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 7
+; RV64-i32-NEXT:    fcvt.w.s a3, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.s a4, fa5, rmm
+; RV64-i32-NEXT:    sw a4, 112(sp)
+; RV64-i32-NEXT:    flw fa5, 44(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 6
+; RV64-i32-NEXT:    fcvt.w.s a4, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.s a5, fa5, rmm
+; RV64-i32-NEXT:    sw a5, 108(sp)
+; RV64-i32-NEXT:    flw fa5, 40(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v10
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-i32-NEXT:    fcvt.w.s a5, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.s a6, fa5, rmm
+; RV64-i32-NEXT:    sw a6, 104(sp)
+; RV64-i32-NEXT:    flw fa5, 36(sp)
+; RV64-i32-NEXT:    vfmv.f.s fa4, v10
+; RV64-i32-NEXT:    fcvt.w.s a6, fa4, rmm
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 4
+; RV64-i32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    sw a7, 100(sp)
+; RV64-i32-NEXT:    fcvt.w.s a7, fa5, rmm
+; RV64-i32-NEXT:    flw fa5, 32(sp)
+; RV64-i32-NEXT:    sw a0, 64(sp)
+; RV64-i32-NEXT:    sw a3, 68(sp)
+; RV64-i32-NEXT:    sw a2, 72(sp)
+; RV64-i32-NEXT:    sw a1, 76(sp)
+; RV64-i32-NEXT:    sw a7, 80(sp)
+; RV64-i32-NEXT:    sw a6, 84(sp)
+; RV64-i32-NEXT:    sw a5, 88(sp)
+; RV64-i32-NEXT:    sw a4, 92(sp)
+; RV64-i32-NEXT:    fcvt.w.s a0, fa5, rmm
+; RV64-i32-NEXT:    sw a0, 96(sp)
+; RV64-i32-NEXT:    addi a0, sp, 64
+; RV64-i32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-i32-NEXT:    vle32.v v8, (a0)
+; RV64-i32-NEXT:    addi sp, s0, -192
+; RV64-i32-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    addi sp, sp, 192
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v16f32:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    addi sp, sp, -384
+; RV64-i64-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    sd s0, 368(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    addi s0, sp, 384
+; RV64-i64-NEXT:    andi sp, sp, -128
+; RV64-i64-NEXT:    addi a0, sp, 64
+; RV64-i64-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV64-i64-NEXT:    vse32.v v8, (a0)
+; RV64-i64-NEXT:    flw fa5, 124(sp)
+; RV64-i64-NEXT:    vfmv.f.s fa4, v8
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 2
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 248(sp)
+; RV64-i64-NEXT:    flw fa5, 120(sp)
+; RV64-i64-NEXT:    vslidedown.vi v13, v8, 1
+; RV64-i64-NEXT:    fcvt.l.s a0, fa4, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa4, v10
+; RV64-i64-NEXT:    fcvt.l.s a1, fa5, rmm
+; RV64-i64-NEXT:    sd a1, 240(sp)
+; RV64-i64-NEXT:    flw fa5, 116(sp)
+; RV64-i64-NEXT:    vsetivli zero, 1, e32, m2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 7
+; RV64-i64-NEXT:    fcvt.l.s a1, fa4, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa4, v12
+; RV64-i64-NEXT:    fcvt.l.s a2, fa5, rmm
+; RV64-i64-NEXT:    sd a2, 232(sp)
+; RV64-i64-NEXT:    flw fa5, 112(sp)
+; RV64-i64-NEXT:    fcvt.l.s a2, fa4, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa4, v13
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 6
+; RV64-i64-NEXT:    fcvt.l.s a3, fa5, rmm
+; RV64-i64-NEXT:    sd a3, 224(sp)
+; RV64-i64-NEXT:    flw fa5, 108(sp)
+; RV64-i64-NEXT:    fcvt.l.s a3, fa4, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa4, v10
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 5
+; RV64-i64-NEXT:    fcvt.l.s a4, fa5, rmm
+; RV64-i64-NEXT:    sd a4, 216(sp)
+; RV64-i64-NEXT:    flw fa5, 104(sp)
+; RV64-i64-NEXT:    fcvt.l.s a4, fa4, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa4, v12
+; RV64-i64-NEXT:    fcvt.l.s a5, fa4, rmm
+; RV64-i64-NEXT:    fcvt.l.s a6, fa5, rmm
+; RV64-i64-NEXT:    sd a6, 208(sp)
+; RV64-i64-NEXT:    flw fa5, 100(sp)
+; RV64-i64-NEXT:    vfmv.f.s fa4, v10
+; RV64-i64-NEXT:    fcvt.l.s a6, fa4, rmm
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 4
+; RV64-i64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    sd a7, 200(sp)
+; RV64-i64-NEXT:    fcvt.l.s a7, fa5, rmm
+; RV64-i64-NEXT:    flw fa5, 96(sp)
+; RV64-i64-NEXT:    sd a0, 128(sp)
+; RV64-i64-NEXT:    sd a3, 136(sp)
+; RV64-i64-NEXT:    sd a2, 144(sp)
+; RV64-i64-NEXT:    sd a1, 152(sp)
+; RV64-i64-NEXT:    sd a7, 160(sp)
+; RV64-i64-NEXT:    sd a6, 168(sp)
+; RV64-i64-NEXT:    sd a5, 176(sp)
+; RV64-i64-NEXT:    sd a4, 184(sp)
+; RV64-i64-NEXT:    fcvt.l.s a0, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 192(sp)
+; RV64-i64-NEXT:    addi a0, sp, 128
+; RV64-i64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; RV64-i64-NEXT:    vle64.v v8, (a0)
+; RV64-i64-NEXT:    addi sp, s0, -384
+; RV64-i64-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    ld s0, 368(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    addi sp, sp, 384
+; RV64-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lround.v16iXLen.v16f32(<16 x float> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lround.v16iXLen.v16f32(<16 x float>)
+
+define <1 x iXLen> @lround_v1f64(<1 x double> %x) nounwind {
+; RV32-LABEL: lround_v1f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vmv.s.x v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v1f64:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vmv.s.x v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v1f64:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    vmv.s.x v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lround.v1iXLen.v1f64(<1 x double> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lround.v1iXLen.v1f64(<1 x double>)
+
+define <2 x iXLen> @lround_v2f64(<2 x double> %x) nounwind {
+; RV32-LABEL: lround_v2f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v9, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v9
+; RV32-NEXT:    fcvt.w.d a1, fa5, rmm
+; RV32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v2f64:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v9
+; RV64-i32-NEXT:    fcvt.w.d a1, fa5, rmm
+; RV64-i32-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v8, a0
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v2f64:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v9, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v9
+; RV64-i64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-i64-NEXT:    vmv.v.x v8, a0
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lround.v2iXLen.v2f64(<2 x double> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lround.v2iXLen.v2f64(<2 x double>)
+
+define <4 x iXLen> @lround_v4f64(<4 x double> %x) nounwind {
+; RV32-LABEL: lround_v4f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v12, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v8, 2
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    fcvt.w.d a1, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v10
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a0
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV32-NEXT:    vslide1down.vx v8, v9, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v4f64:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    fcvt.w.d a1, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v10
+; RV64-i32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v9, a0
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-i32-NEXT:    vslide1down.vx v8, v9, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v4f64:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v12, v8, 1
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 2
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v12
+; RV64-i64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v10
+; RV64-i64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-i64-NEXT:    vmv.v.x v10, a0
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    vslide1down.vx v8, v10, a1
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lround.v4iXLen.v4f64(<4 x double> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lround.v4iXLen.v4f64(<4 x double>)
+
+define <8 x iXLen> @lround_v8f64(<8 x double> %x) nounwind {
+; RV32-LABEL: lround_v8f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -128
+; RV32-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
+; RV32-NEXT:    addi s0, sp, 128
+; RV32-NEXT:    andi sp, sp, -64
+; RV32-NEXT:    mv a0, sp
+; RV32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV32-NEXT:    vslidedown.vi v14, v8, 1
+; RV32-NEXT:    vfmv.f.s fa5, v8
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v12, v8, 2
+; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV32-NEXT:    vse64.v v8, (a0)
+; RV32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV32-NEXT:    vslidedown.vi v8, v8, 3
+; RV32-NEXT:    vfmv.f.s fa4, v14
+; RV32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV32-NEXT:    vfmv.f.s fa5, v12
+; RV32-NEXT:    vfmv.f.s fa3, v8
+; RV32-NEXT:    fcvt.w.d a1, fa4, rmm
+; RV32-NEXT:    fcvt.w.d a2, fa5, rmm
+; RV32-NEXT:    fcvt.w.d a3, fa3, rmm
+; RV32-NEXT:    fld fa5, 32(sp)
+; RV32-NEXT:    fld fa4, 40(sp)
+; RV32-NEXT:    fld fa3, 48(sp)
+; RV32-NEXT:    fld fa2, 56(sp)
+; RV32-NEXT:    fcvt.w.d a4, fa5, rmm
+; RV32-NEXT:    fcvt.w.d a5, fa4, rmm
+; RV32-NEXT:    fcvt.w.d a6, fa3, rmm
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vmv.v.x v8, a0
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
+; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a3
+; RV32-NEXT:    vslide1down.vx v8, v8, a4
+; RV32-NEXT:    vslide1down.vx v8, v8, a5
+; RV32-NEXT:    vslide1down.vx v8, v8, a6
+; RV32-NEXT:    fcvt.w.d a0, fa2, rmm
+; RV32-NEXT:    vslide1down.vx v8, v8, a0
+; RV32-NEXT:    addi sp, s0, -128
+; RV32-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 128
+; RV32-NEXT:    ret
+;
+; RV64-i32-LABEL: lround_v8f64:
+; RV64-i32:       # %bb.0:
+; RV64-i32-NEXT:    addi sp, sp, -128
+; RV64-i32-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; RV64-i32-NEXT:    addi s0, sp, 128
+; RV64-i32-NEXT:    andi sp, sp, -64
+; RV64-i32-NEXT:    mv a0, sp
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v14, v8, 1
+; RV64-i32-NEXT:    vfmv.f.s fa5, v8
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v12, v8, 2
+; RV64-i32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-i32-NEXT:    vse64.v v8, (a0)
+; RV64-i32-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-i32-NEXT:    vslidedown.vi v8, v8, 3
+; RV64-i32-NEXT:    vfmv.f.s fa4, v14
+; RV64-i32-NEXT:    fcvt.w.d a0, fa5, rmm
+; RV64-i32-NEXT:    vfmv.f.s fa5, v12
+; RV64-i32-NEXT:    vfmv.f.s fa3, v8
+; RV64-i32-NEXT:    fcvt.w.d a1, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.d a2, fa5, rmm
+; RV64-i32-NEXT:    fcvt.w.d a3, fa3, rmm
+; RV64-i32-NEXT:    fld fa5, 32(sp)
+; RV64-i32-NEXT:    fld fa4, 40(sp)
+; RV64-i32-NEXT:    fld fa3, 48(sp)
+; RV64-i32-NEXT:    fld fa2, 56(sp)
+; RV64-i32-NEXT:    fcvt.w.d a4, fa5, rmm
+; RV64-i32-NEXT:    fcvt.w.d a5, fa4, rmm
+; RV64-i32-NEXT:    fcvt.w.d a6, fa3, rmm
+; RV64-i32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-i32-NEXT:    vmv.v.x v8, a0
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a1
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a2
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a3
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a4
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a5
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a6
+; RV64-i32-NEXT:    fcvt.w.d a0, fa2, rmm
+; RV64-i32-NEXT:    vslide1down.vx v8, v8, a0
+; RV64-i32-NEXT:    addi sp, s0, -128
+; RV64-i32-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64-i32-NEXT:    addi sp, sp, 128
+; RV64-i32-NEXT:    ret
+;
+; RV64-i64-LABEL: lround_v8f64:
+; RV64-i64:       # %bb.0:
+; RV64-i64-NEXT:    addi sp, sp, -192
+; RV64-i64-NEXT:    sd ra, 184(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    sd s0, 176(sp) # 8-byte Folded Spill
+; RV64-i64-NEXT:    addi s0, sp, 192
+; RV64-i64-NEXT:    andi sp, sp, -64
+; RV64-i64-NEXT:    mv a0, sp
+; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-i64-NEXT:    vse64.v v8, (a0)
+; RV64-i64-NEXT:    fld fa5, 56(sp)
+; RV64-i64-NEXT:    vfmv.f.s fa4, v8
+; RV64-i64-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 1
+; RV64-i64-NEXT:    fcvt.l.d a0, fa4, rmm
+; RV64-i64-NEXT:    fcvt.l.d a1, fa5, rmm
+; RV64-i64-NEXT:    sd a1, 120(sp)
+; RV64-i64-NEXT:    fld fa5, 48(sp)
+; RV64-i64-NEXT:    vfmv.f.s fa4, v10
+; RV64-i64-NEXT:    vsetivli zero, 1, e64, m2, ta, ma
+; RV64-i64-NEXT:    vslidedown.vi v10, v8, 3
+; RV64-i64-NEXT:    fcvt.l.d a1, fa4, rmm
+; RV64-i64-NEXT:    fcvt.l.d a2, fa5, rmm
+; RV64-i64-NEXT:    sd a2, 112(sp)
+; RV64-i64-NEXT:    fld fa5, 40(sp)
+; RV64-i64-NEXT:    vfmv.f.s fa4, v10
+; RV64-i64-NEXT:    fcvt.l.d a2, fa4, rmm
+; RV64-i64-NEXT:    vslidedown.vi v8, v8, 2
+; RV64-i64-NEXT:    fcvt.l.d a3, fa5, rmm
+; RV64-i64-NEXT:    vfmv.f.s fa5, v8
+; RV64-i64-NEXT:    sd a3, 104(sp)
+; RV64-i64-NEXT:    fcvt.l.d a3, fa5, rmm
+; RV64-i64-NEXT:    fld fa5, 32(sp)
+; RV64-i64-NEXT:    sd a0, 64(sp)
+; RV64-i64-NEXT:    sd a1, 72(sp)
+; RV64-i64-NEXT:    sd a3, 80(sp)
+; RV64-i64-NEXT:    sd a2, 88(sp)
+; RV64-i64-NEXT:    fcvt.l.d a0, fa5, rmm
+; RV64-i64-NEXT:    sd a0, 96(sp)
+; RV64-i64-NEXT:    addi a0, sp, 64
+; RV64-i64-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
+; RV64-i64-NEXT:    vle64.v v8, (a0)
+; RV64-i64-NEXT:    addi sp, s0, -192
+; RV64-i64-NEXT:    ld ra, 184(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    ld s0, 176(sp) # 8-byte Folded Reload
+; RV64-i64-NEXT:    addi sp, sp, 192
+; RV64-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lround.v8iXLen.v8f64(<8 x double> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lround.v8iXLen.v8f64(<8 x double>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
index c29ccd45528b..3bfe41337a11 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-buildvec.ll
@@ -280,16 +280,16 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a3
+; CHECK-NEXT:    li a3, 1
 ; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v9, v9, zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
-; CHECK-NEXT:    vslide1down.vx v9, v9, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, zero
-; CHECK-NEXT:    vslide1down.vx v9, v9, a1
-; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslide1down.vx v9, v9, a2
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -297,16 +297,16 @@ define <8 x i1> @buildvec_mask_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %w) {
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    li a0, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a3
+; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, zero
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
@@ -325,16 +325,16 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a3
+; CHECK-NEXT:    li a3, 1
 ; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslide1down.vx v9, v9, zero
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a3
-; CHECK-NEXT:    vslide1down.vx v9, v9, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, zero
-; CHECK-NEXT:    vslide1down.vx v9, v9, a1
-; CHECK-NEXT:    vslide1down.vx v8, v8, a2
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslide1down.vx v9, v9, a2
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -342,16 +342,16 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1_2(i1 %x, i1 %y, i1 %z, i1 %
 ; ZVE32F:       # %bb.0:
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    li a0, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a3
+; ZVE32F-NEXT:    li a3, 1
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, zero
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, zero
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
@@ -371,14 +371,14 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1(i1 %x, i1 %y) optsize {
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; CHECK-NEXT:    vmv.v.x v8, a0
 ; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vslide1down.vx v9, v8, a0
-; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v9, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
 ; CHECK-NEXT:    vslide1down.vx v9, v9, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a1
 ; CHECK-NEXT:    vslide1down.vx v9, v9, a1
 ; CHECK-NEXT:    vslide1down.vx v8, v8, a1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; CHECK-NEXT:    vand.vi v8, v8, 1
+; CHECK-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; CHECK-NEXT:    vand.vi v8, v9, 1
 ; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    ret
 ;
@@ -387,14 +387,14 @@ define <8 x i1> @buildvec_mask_optsize_nonconst_v8i1(i1 %x, i1 %y) optsize {
 ; ZVE32F-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
 ; ZVE32F-NEXT:    vmv.v.x v8, a0
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a0
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
-; ZVE32F-NEXT:    vand.vi v8, v8, 1
+; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
+; ZVE32F-NEXT:    vand.vi v8, v9, 1
 ; ZVE32F-NEXT:    vmsne.vi v0, v8, 0
 ; ZVE32F-NEXT:    ret
   %1 = insertelement <8 x i1> poison, i1 %x, i32 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 533b8b6864eb..67d55366674f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -14055,45 +14055,45 @@ define <8 x i16> @mgather_strided_unaligned(ptr %base) {
 ; RV32-NEXT:    vmv.x.s t0, v8
 ; RV32-NEXT:    lbu t1, 0(a1)
 ; RV32-NEXT:    lbu a1, 1(a1)
-; RV32-NEXT:    lbu t2, 0(a2)
-; RV32-NEXT:    lbu a2, 1(a2)
 ; RV32-NEXT:    slli a0, a0, 8
 ; RV32-NEXT:    or a0, a0, a6
-; RV32-NEXT:    lbu a6, 0(a3)
-; RV32-NEXT:    lbu a3, 1(a3)
+; RV32-NEXT:    lbu a6, 0(a2)
+; RV32-NEXT:    lbu a2, 1(a2)
 ; RV32-NEXT:    slli a1, a1, 8
 ; RV32-NEXT:    or a1, a1, t1
-; RV32-NEXT:    lbu t1, 0(a4)
-; RV32-NEXT:    lbu a4, 1(a4)
+; RV32-NEXT:    lbu t1, 0(a3)
+; RV32-NEXT:    lbu a3, 1(a3)
 ; RV32-NEXT:    slli a2, a2, 8
-; RV32-NEXT:    or a2, a2, t2
-; RV32-NEXT:    lbu t2, 0(a5)
-; RV32-NEXT:    lbu a5, 1(a5)
+; RV32-NEXT:    or a2, a2, a6
+; RV32-NEXT:    lbu a6, 0(a4)
+; RV32-NEXT:    lbu a4, 1(a4)
 ; RV32-NEXT:    slli a3, a3, 8
-; RV32-NEXT:    or a3, a3, a6
+; RV32-NEXT:    or a3, a3, t1
+; RV32-NEXT:    lbu t1, 0(a5)
+; RV32-NEXT:    lbu a5, 1(a5)
+; RV32-NEXT:    slli a4, a4, 8
+; RV32-NEXT:    or a4, a4, a6
 ; RV32-NEXT:    lbu a6, 0(a7)
 ; RV32-NEXT:    lbu a7, 1(a7)
-; RV32-NEXT:    slli a4, a4, 8
-; RV32-NEXT:    or a4, a4, t1
+; RV32-NEXT:    slli a5, a5, 8
+; RV32-NEXT:    or a5, a5, t1
 ; RV32-NEXT:    lbu t1, 0(t0)
 ; RV32-NEXT:    lbu t0, 1(t0)
-; RV32-NEXT:    slli a5, a5, 8
-; RV32-NEXT:    or a5, a5, t2
 ; RV32-NEXT:    slli a7, a7, 8
 ; RV32-NEXT:    or a6, a7, a6
+; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; RV32-NEXT:    vmv.v.i v0, 15
 ; RV32-NEXT:    slli t0, t0, 8
 ; RV32-NEXT:    or a7, t0, t1
-; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32-NEXT:    vmv.v.x v9, a4
-; RV32-NEXT:    vslide1down.vx v8, v8, a2
+; RV32-NEXT:    vslide1down.vx v8, v8, a1
 ; RV32-NEXT:    vslide1down.vx v9, v9, a5
-; RV32-NEXT:    vslide1down.vx v10, v8, a3
+; RV32-NEXT:    vslide1down.vx v10, v8, a2
 ; RV32-NEXT:    vslide1down.vx v8, v9, a6
-; RV32-NEXT:    vmv.v.i v0, 15
 ; RV32-NEXT:    vslide1down.vx v8, v8, a7
-; RV32-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV32-NEXT:    vslide1down.vx v9, v10, a3
+; RV32-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV32-NEXT:    ret
 ;
 ; RV64V-LABEL: mgather_strided_unaligned:
@@ -14215,15 +14215,15 @@ define <8 x i16> @mgather_strided_unaligned(ptr %base) {
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
 ; RV64ZVE32F-NEXT:    slli a0, a0, 8
 ; RV64ZVE32F-NEXT:    or a0, a0, t1
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a2
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 1, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14258,15 +14258,15 @@ define <8 x i16> @mgather_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 26(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14303,15 +14303,15 @@ define <8 x i16> @mgather_strided_2xSEW_with_offset(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 30(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 2, i64 3, i64 6, i64 7, i64 10, i64 11, i64 14, i64 15>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14342,21 +14342,21 @@ define <8 x i16> @mgather_reverse_unit_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a2, 26(a0)
 ; RV64ZVE32F-NEXT:    lh a3, 28(a0)
 ; RV64ZVE32F-NEXT:    lh a4, 30(a0)
-; RV64ZVE32F-NEXT:    lh a5, 16(a0)
-; RV64ZVE32F-NEXT:    lh a6, 18(a0)
-; RV64ZVE32F-NEXT:    lh a7, 20(a0)
+; RV64ZVE32F-NEXT:    lh a5, 20(a0)
+; RV64ZVE32F-NEXT:    lh a6, 16(a0)
+; RV64ZVE32F-NEXT:    lh a7, 18(a0)
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a3
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 14, i64 15, i64 12, i64 13, i64 10, i64 11, i64 8, i64 9>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14387,21 +14387,21 @@ define <8 x i16> @mgather_reverse_strided_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a2, 22(a0)
 ; RV64ZVE32F-NEXT:    lh a3, 28(a0)
 ; RV64ZVE32F-NEXT:    lh a4, 30(a0)
-; RV64ZVE32F-NEXT:    lh a5, 4(a0)
-; RV64ZVE32F-NEXT:    lh a6, 6(a0)
-; RV64ZVE32F-NEXT:    lh a7, 12(a0)
+; RV64ZVE32F-NEXT:    lh a5, 12(a0)
+; RV64ZVE32F-NEXT:    lh a6, 4(a0)
+; RV64ZVE32F-NEXT:    lh a7, 6(a0)
 ; RV64ZVE32F-NEXT:    lh a0, 14(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a3
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a6
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 14, i64 15, i64 10, i64 11, i64 6, i64 7, i64 2, i64 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14437,15 +14437,15 @@ define <8 x i16> @mgather_gather_2xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14484,15 +14484,15 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 9, i32 10, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 2, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14531,15 +14531,15 @@ define <8 x i16> @mgather_gather_2xSEW_unaligned2(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 20(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a4
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a5
 ; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a0
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a3
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a0
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 1, i32 2, i32 9, i32 10, i32 4, i32 5, i32 2, i32 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14582,15 +14582,15 @@ define <8 x i16> @mgather_gather_4xSEW(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 8, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14630,15 +14630,15 @@ define <8 x i16> @mgather_gather_4xSEW_partial_align(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 22(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i32>  <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14687,15 +14687,15 @@ define <8 x i16> @mgather_shuffle_rotate(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 6(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 4, i64 5, i64 6, i64 7, i64 0, i64 1, i64 2, i64 3>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
@@ -14735,15 +14735,15 @@ define <8 x i16> @mgather_shuffle_vrgather(ptr %base) {
 ; RV64ZVE32F-NEXT:    lh a0, 14(a0)
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
 ; RV64ZVE32F-NEXT:    vmv.v.i v0, 15
-; RV64ZVE32F-NEXT:    vmv.v.x v8, a1
-; RV64ZVE32F-NEXT:    vmv.v.x v9, a5
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
-; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a7
-; RV64ZVE32F-NEXT:    vslide1down.vx v10, v8, a2
-; RV64ZVE32F-NEXT:    vslide1down.vx v8, v9, a0
-; RV64ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
+; RV64ZVE32F-NEXT:    vmv.v.x v8, a5
+; RV64ZVE32F-NEXT:    vmv.v.x v9, a1
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a6
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a3
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a4
+; RV64ZVE32F-NEXT:    vslide1down.vx v8, v8, a0
+; RV64ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; RV64ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
 ; RV64ZVE32F-NEXT:    ret
   %ptrs = getelementptr inbounds i16, ptr %base, <8 x i64>  <i64 0, i64 2, i64 3, i64 1, i64 4, i64 5, i64 6, i64 7>
   %v = call <8 x i16> @llvm.masked.gather.v8i16.v8p0(<8 x ptr> %ptrs, i32 4, <8 x i1> splat (i1 true), <8 x i16> poison)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
index b6267bf481c8..0909c11078ff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave2.ll
@@ -1340,26 +1340,26 @@ define <4 x i64> @unzip2a_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64:
@@ -1378,9 +1378,9 @@ entry:
 define <16 x i64> @unzip2a_dual_v16i64(<16 x i64> %a, <16 x i64> %b) {
 ; V-LABEL: unzip2a_dual_v16i64:
 ; V:       # %bb.0: # %entry
-; V-NEXT:    lui a0, 5
 ; V-NEXT:    vsetivli zero, 16, e16, m1, ta, ma
 ; V-NEXT:    vid.v v16
+; V-NEXT:    lui a0, 5
 ; V-NEXT:    addi a0, a0, 1365
 ; V-NEXT:    vmv.s.x v20, a0
 ; V-NEXT:    li a0, -256
@@ -1526,26 +1526,26 @@ define <4 x i64> @unzip2a_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_ra
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64_exact:
@@ -1574,26 +1574,26 @@ define <4 x i64> @unzip2a_dual_v4i64_exact_nf2(<4 x i64> %a, <4 x i64> %b) vscal
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v4i64_exact_nf2:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 0(a2)
-; ZVE32F-NEXT:    ld a2, 16(a2)
-; ZVE32F-NEXT:    ld a4, 0(a1)
+; ZVE32F-NEXT:    ld a3, 0(a1)
 ; ZVE32F-NEXT:    ld a1, 16(a1)
+; ZVE32F-NEXT:    ld a4, 0(a2)
+; ZVE32F-NEXT:    ld a2, 16(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2a_dual_v4i64_exact_nf2:
@@ -1651,76 +1651,76 @@ define <16 x i64> @unzip2a_dual_v16i64_exact(<16 x i64> %a, <16 x i64> %b) vscal
 ;
 ; ZVE32F-LABEL: unzip2a_dual_v16i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a6, 0(a1)
+; ZVE32F-NEXT:    ld a5, 96(a2)
+; ZVE32F-NEXT:    ld a7, 0(a1)
 ; ZVE32F-NEXT:    ld a4, 16(a1)
-; ZVE32F-NEXT:    ld a7, 32(a1)
+; ZVE32F-NEXT:    ld t0, 32(a1)
 ; ZVE32F-NEXT:    ld a3, 48(a1)
-; ZVE32F-NEXT:    ld a5, 80(a1)
-; ZVE32F-NEXT:    ld t0, 96(a1)
 ; ZVE32F-NEXT:    ld t1, 64(a1)
+; ZVE32F-NEXT:    ld a6, 80(a1)
+; ZVE32F-NEXT:    ld t2, 96(a1)
 ; ZVE32F-NEXT:    ld a1, 112(a1)
-; ZVE32F-NEXT:    srli t2, a6, 32
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
-; ZVE32F-NEXT:    vmv.v.x v8, t0
+; ZVE32F-NEXT:    vmv.v.x v8, a7
+; ZVE32F-NEXT:    srli a7, a7, 32
+; ZVE32F-NEXT:    vmv.v.x v9, t0
 ; ZVE32F-NEXT:    srli t0, t0, 32
-; ZVE32F-NEXT:    vmv.v.x v9, t1
-; ZVE32F-NEXT:    srli t1, t1, 32
-; ZVE32F-NEXT:    vmv.v.x v10, a6
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, t1
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
-; ZVE32F-NEXT:    vslide1down.vx v10, v10, t2
-; ZVE32F-NEXT:    ld t1, 32(a2)
-; ZVE32F-NEXT:    ld t0, 16(a2)
-; ZVE32F-NEXT:    ld t2, 0(a2)
-; ZVE32F-NEXT:    ld a6, 48(a2)
-; ZVE32F-NEXT:    vmv.v.x v11, t1
+; ZVE32F-NEXT:    vmv.v.x v10, t1
 ; ZVE32F-NEXT:    srli t1, t1, 32
-; ZVE32F-NEXT:    vmv.v.x v12, t2
+; ZVE32F-NEXT:    vmv.v.x v11, t2
 ; ZVE32F-NEXT:    srli t2, t2, 32
-; ZVE32F-NEXT:    vmv.v.x v13, a7
-; ZVE32F-NEXT:    srli a7, a7, 32
-; ZVE32F-NEXT:    vslide1down.vx v13, v13, a7
-; ZVE32F-NEXT:    vslide1down.vx v12, v12, t2
-; ZVE32F-NEXT:    vslide1down.vx v11, v11, t1
-; ZVE32F-NEXT:    ld a7, 64(a2)
-; ZVE32F-NEXT:    ld t1, 80(a2)
-; ZVE32F-NEXT:    ld t2, 96(a2)
-; ZVE32F-NEXT:    ld a2, 112(a2)
-; ZVE32F-NEXT:    vmv.v.x v14, a7
-; ZVE32F-NEXT:    srli a7, a7, 32
-; ZVE32F-NEXT:    vslide1down.vx v14, v14, a7
-; ZVE32F-NEXT:    vmv.v.x v15, t2
-; ZVE32F-NEXT:    srli a7, t2, 32
-; ZVE32F-NEXT:    vslide1down.vx v15, v15, a7
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, t2
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, t1
+; ZVE32F-NEXT:    vslide1down.vx v12, v9, t0
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
+; ZVE32F-NEXT:    ld t0, 0(a2)
+; ZVE32F-NEXT:    ld t1, 16(a2)
+; ZVE32F-NEXT:    ld t2, 32(a2)
+; ZVE32F-NEXT:    ld a7, 48(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, t0
+; ZVE32F-NEXT:    srli t0, t0, 32
+; ZVE32F-NEXT:    vmv.v.x v13, t2
+; ZVE32F-NEXT:    srli t2, t2, 32
+; ZVE32F-NEXT:    vslide1down.vx v13, v13, t2
+; ZVE32F-NEXT:    vslide1down.vx v14, v9, t0
+; ZVE32F-NEXT:    ld t0, 64(a2)
+; ZVE32F-NEXT:    ld t2, 112(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, a5
 ; ZVE32F-NEXT:    srli a5, a5, 32
+; ZVE32F-NEXT:    vslide1down.vx v15, v9, a5
+; ZVE32F-NEXT:    ld a2, 80(a2)
+; ZVE32F-NEXT:    vmv.v.x v9, t0
+; ZVE32F-NEXT:    srli a5, t0, 32
 ; ZVE32F-NEXT:    vslide1down.vx v16, v9, a5
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v9, v11, a1
 ; ZVE32F-NEXT:    srli a1, a1, 32
-; ZVE32F-NEXT:    vslide1down.vx v9, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v8, v10, a4
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, a6
+; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    vslide1down.vx v10, v10, a1
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a4
 ; ZVE32F-NEXT:    srli a4, a4, 32
-; ZVE32F-NEXT:    vslide1down.vx v10, v8, a4
-; ZVE32F-NEXT:    vslide1down.vx v8, v12, t0
-; ZVE32F-NEXT:    srli a1, t0, 32
-; ZVE32F-NEXT:    vslide1down.vx v12, v8, a1
+; ZVE32F-NEXT:    vslide1down.vx v11, v8, a4
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
 ; ZVE32F-NEXT:    vslide1down.vx v8, v14, t1
 ; ZVE32F-NEXT:    srli a1, t1, 32
 ; ZVE32F-NEXT:    vslide1down.vx v14, v8, a1
-; ZVE32F-NEXT:    vslidedown.vi v9, v16, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v8, v13, a3
+; ZVE32F-NEXT:    vslidedown.vi v9, v10, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v8, v12, a3
 ; ZVE32F-NEXT:    srli a3, a3, 32
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a3
-; ZVE32F-NEXT:    vslidedown.vi v8, v10, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v10, v11, a6
-; ZVE32F-NEXT:    srli a1, a6, 32
+; ZVE32F-NEXT:    vslidedown.vi v8, v11, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v10, v13, a7
+; ZVE32F-NEXT:    srli a1, a7, 32
 ; ZVE32F-NEXT:    vslide1down.vx v10, v10, a1
-; ZVE32F-NEXT:    vslidedown.vi v10, v12, 4, v0.t
-; ZVE32F-NEXT:    vslide1down.vx v11, v15, a2
+; ZVE32F-NEXT:    vslidedown.vi v10, v14, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v11, v15, t2
+; ZVE32F-NEXT:    srli a1, t2, 32
+; ZVE32F-NEXT:    vslide1down.vx v11, v11, a1
+; ZVE32F-NEXT:    vslide1down.vx v12, v16, a2
 ; ZVE32F-NEXT:    srli a2, a2, 32
-; ZVE32F-NEXT:    vslide1down.vx v11, v11, a2
-; ZVE32F-NEXT:    vslidedown.vi v11, v14, 4, v0.t
+; ZVE32F-NEXT:    vslide1down.vx v12, v12, a2
+; ZVE32F-NEXT:    vslidedown.vi v11, v12, 4, v0.t
 ; ZVE32F-NEXT:    vs4r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
@@ -1751,26 +1751,26 @@ define <4 x i64> @unzip2b_dual_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ;
 ; ZVE32F-LABEL: unzip2b_dual_v4i64:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 8(a2)
-; ZVE32F-NEXT:    ld a2, 24(a2)
-; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a3, 8(a1)
 ; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    ld a4, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vse32.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vse32.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2b_dual_v4i64:
@@ -1802,26 +1802,26 @@ define <4 x i64> @unzip2b_dual_v4i64_exact(<4 x i64> %a, <4 x i64> %b) vscale_ra
 ;
 ; ZVE32F-LABEL: unzip2b_dual_v4i64_exact:
 ; ZVE32F:       # %bb.0: # %entry
-; ZVE32F-NEXT:    ld a3, 8(a2)
-; ZVE32F-NEXT:    ld a2, 24(a2)
-; ZVE32F-NEXT:    ld a4, 8(a1)
+; ZVE32F-NEXT:    ld a3, 8(a1)
 ; ZVE32F-NEXT:    ld a1, 24(a1)
+; ZVE32F-NEXT:    ld a4, 8(a2)
+; ZVE32F-NEXT:    ld a2, 24(a2)
 ; ZVE32F-NEXT:    vsetivli zero, 8, e32, m1, ta, mu
 ; ZVE32F-NEXT:    vmv.v.i v0, 15
-; ZVE32F-NEXT:    srli a5, a2, 32
+; ZVE32F-NEXT:    srli a5, a1, 32
 ; ZVE32F-NEXT:    srli a6, a3, 32
-; ZVE32F-NEXT:    srli a7, a1, 32
+; ZVE32F-NEXT:    srli a7, a2, 32
 ; ZVE32F-NEXT:    srli t0, a4, 32
 ; ZVE32F-NEXT:    vmv.v.x v8, a4
 ; ZVE32F-NEXT:    vmv.v.x v9, a3
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, t0
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a6
-; ZVE32F-NEXT:    vslide1down.vx v8, v8, a1
-; ZVE32F-NEXT:    vslide1down.vx v9, v9, a2
+; ZVE32F-NEXT:    vslide1down.vx v8, v8, a2
+; ZVE32F-NEXT:    vslide1down.vx v9, v9, a1
 ; ZVE32F-NEXT:    vslide1down.vx v8, v8, a7
 ; ZVE32F-NEXT:    vslide1down.vx v9, v9, a5
-; ZVE32F-NEXT:    vslidedown.vi v9, v8, 4, v0.t
-; ZVE32F-NEXT:    vs1r.v v9, (a0)
+; ZVE32F-NEXT:    vslidedown.vi v8, v9, 4, v0.t
+; ZVE32F-NEXT:    vs1r.v v8, (a0)
 ; ZVE32F-NEXT:    ret
 ;
 ; ZIP-LABEL: unzip2b_dual_v4i64_exact:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
index 917613d5c786..371815697191 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int-interleave.ll
@@ -598,43 +598,54 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; ZIP-NEXT:    addi sp, sp, -16
 ; ZIP-NEXT:    .cfi_def_cfa_offset 16
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    slli a0, a0, 5
+; ZIP-NEXT:    li a1, 40
+; ZIP-NEXT:    mul a0, a0, a1
 ; ZIP-NEXT:    sub sp, sp, a0
-; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 32 * vlenb
+; ZIP-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x28, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 40 * vlenb
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    li a1, 24
-; ZIP-NEXT:    mul a0, a0, a1
+; ZIP-NEXT:    slli a0, a0, 5
 ; ZIP-NEXT:    add a0, sp, a0
 ; ZIP-NEXT:    addi a0, a0, 16
 ; ZIP-NEXT:    vs8r.v v16, (a0) # vscale x 64-byte Folded Spill
 ; ZIP-NEXT:    addi a0, sp, 16
 ; ZIP-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
-; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; ZIP-NEXT:    vslidedown.vi v24, v8, 16
 ; ZIP-NEXT:    li a0, 32
+; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
+; ZIP-NEXT:    vslidedown.vi v16, v8, 16
 ; ZIP-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; ZIP-NEXT:    ri.vzip2a.vv v16, v24, v0
+; ZIP-NEXT:    ri.vzip2a.vv v8, v16, v0
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    li a2, 24
-; ZIP-NEXT:    mul a1, a1, a2
+; ZIP-NEXT:    slli a1, a1, 3
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vl8r.v v24, (a1) # vscale x 64-byte Folded Reload
+; ZIP-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    slli a1, a1, 5
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 16
+; ZIP-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    vsetivli zero, 16, e32, m8, ta, ma
-; ZIP-NEXT:    vslidedown.vi v24, v24, 16
+; ZIP-NEXT:    vslidedown.vi v16, v16, 16
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    slli a1, a1, 4
+; ZIP-NEXT:    li a2, 24
+; ZIP-NEXT:    mul a1, a1, a2
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vs8r.v v24, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
 ; ZIP-NEXT:    lui a1, 699051
 ; ZIP-NEXT:    addi a1, a1, -1366
 ; ZIP-NEXT:    vmv.s.x v0, a1
 ; ZIP-NEXT:    csrr a1, vlenb
-; ZIP-NEXT:    slli a1, a1, 3
+; ZIP-NEXT:    slli a1, a1, 4
 ; ZIP-NEXT:    add a1, sp, a1
 ; ZIP-NEXT:    addi a1, a1, 16
-; ZIP-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    vs8r.v v16, (a1) # vscale x 64-byte Folded Spill
+; ZIP-NEXT:    csrr a1, vlenb
+; ZIP-NEXT:    li a2, 24
+; ZIP-NEXT:    mul a1, a1, a2
+; ZIP-NEXT:    add a1, sp, a1
+; ZIP-NEXT:    addi a1, a1, 16
+; ZIP-NEXT:    vl8r.v v16, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    csrr a1, vlenb
 ; ZIP-NEXT:    slli a1, a1, 4
 ; ZIP-NEXT:    add a1, sp, a1
@@ -646,19 +657,21 @@ define <64 x i32> @interleave_v32i32(<32 x i32> %x, <32 x i32> %y) {
 ; ZIP-NEXT:    addi a1, a1, 16
 ; ZIP-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    vsetvli zero, a0, e32, m8, ta, mu
-; ZIP-NEXT:    ri.vzip2a.vv v16, v8, v24, v0.t
+; ZIP-NEXT:    ri.vzip2a.vv v8, v24, v16, v0.t
+; ZIP-NEXT:    vmv.v.v v24, v8
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    li a1, 24
-; ZIP-NEXT:    mul a0, a0, a1
+; ZIP-NEXT:    slli a0, a0, 5
 ; ZIP-NEXT:    add a0, sp, a0
 ; ZIP-NEXT:    addi a0, a0, 16
-; ZIP-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; ZIP-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
 ; ZIP-NEXT:    addi a0, sp, 16
 ; ZIP-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
-; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v24
+; ZIP-NEXT:    ri.vzip2a.vv v0, v8, v16
 ; ZIP-NEXT:    vmv.v.v v8, v0
+; ZIP-NEXT:    vmv.v.v v16, v24
 ; ZIP-NEXT:    csrr a0, vlenb
-; ZIP-NEXT:    slli a0, a0, 5
+; ZIP-NEXT:    li a1, 40
+; ZIP-NEXT:    mul a0, a0, a1
 ; ZIP-NEXT:    add sp, sp, a0
 ; ZIP-NEXT:    .cfi_def_cfa sp, 16
 ; ZIP-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
index 8676803e20e3..f5216d82c81b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-int.ll
@@ -195,15 +195,15 @@ define <8 x i64> @vrgather_shuffle_xv_v8i64(<8 x i64> %x) {
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_0)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI12_0)
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV32-NEXT:    vmv.v.i v16, -1
 ; RV32-NEXT:    vle16.v v20, (a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI12_1)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI12_1)
 ; RV32-NEXT:    vle16.v v21, (a0)
+; RV32-NEXT:    vmv.v.i v16, -1
 ; RV32-NEXT:    li a0, 113
 ; RV32-NEXT:    vmv.s.x v0, a0
-; RV32-NEXT:    vrgatherei16.vv v12, v16, v20
-; RV32-NEXT:    vrgatherei16.vv v12, v8, v21, v0.t
+; RV32-NEXT:    vrgatherei16.vv v12, v16, v21
+; RV32-NEXT:    vrgatherei16.vv v12, v8, v20, v0.t
 ; RV32-NEXT:    vmv.v.v v8, v12
 ; RV32-NEXT:    ret
 ;
@@ -227,12 +227,12 @@ define <8 x i64> @vrgather_shuffle_xv_v8i64(<8 x i64> %x) {
 define <8 x i64> @vrgather_shuffle_vx_v8i64(<8 x i64> %x) {
 ; RV32-LABEL: vrgather_shuffle_vx_v8i64:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lui a0, %hi(.LCPI13_0)
-; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_0)
-; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
-; RV32-NEXT:    vle16.v v16, (a0)
 ; RV32-NEXT:    lui a0, %hi(.LCPI13_1)
 ; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_1)
+; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
+; RV32-NEXT:    vle16.v v16, (a0)
+; RV32-NEXT:    lui a0, %hi(.LCPI13_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI13_0)
 ; RV32-NEXT:    vle16.v v17, (a0)
 ; RV32-NEXT:    li a0, 140
 ; RV32-NEXT:    vmv.s.x v0, a0
@@ -388,14 +388,13 @@ define <8 x i8> @splat_ve2_we0_ins_i0ve4(<8 x i8> %v, <8 x i8> %w) {
 define <8 x i8> @splat_ve2_we0_ins_i0we4(<8 x i8> %v, <8 x i8> %w) {
 ; CHECK-LABEL: splat_ve2_we0_ins_i0we4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vrgather.vi v10, v8, 2
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vmv.v.i v11, 4
 ; CHECK-NEXT:    li a0, 67
 ; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 4
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vrgather.vi v10, v8, 2
+; CHECK-NEXT:    vrgather.vv v10, v9, v11, v0.t
 ; CHECK-NEXT:    vmv1r.v v8, v10
 ; CHECK-NEXT:    ret
   %shuff = shufflevector <8 x i8> %v, <8 x i8> %w, <8 x i32> <i32 12, i32 8, i32 2, i32 2, i32 2, i32 2, i32 8, i32 2>
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverser-float.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverse-float.ll
similarity index 71%
rename from llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverser-float.ll
rename to llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverse-float.ll
index 1d21cb558698..ad84aaccc217 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverser-float.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverse-float.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfh -verify-machineinstrs -riscv-v-vector-bits-min=128 \
+; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfh,+zvfbfmin -verify-machineinstrs -riscv-v-vector-bits-min=128 \
+; RUN:   < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfhmin,+zvfbfmin -verify-machineinstrs -riscv-v-vector-bits-min=128 \
 ; RUN:   < %s | FileCheck %s
 
 define <2 x double> @test_vp_reverse_v2f64_masked(<2 x double> %src, <2 x i1> %mask, i32 zeroext %evl) {
@@ -88,3 +90,32 @@ define <4 x half> @test_vp_reverse_v4f16(<4 x half> %src, i32 zeroext %evl) {
   %dst = call <4 x half> @llvm.experimental.vp.reverse.v4f16(<4 x half> %src, <4 x i1> splat (i1 1), i32 %evl)
   ret <4 x half> %dst
 }
+
+define <4 x bfloat> @test_vp_reverse_v4bf16_masked(<4 x bfloat> %src, <4 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_v4bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v9, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v10, v9, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %dst = call <4 x bfloat> @llvm.experimental.vp.reverse.v4bf16(<4 x bfloat> %src, <4 x i1> %mask, i32 %evl)
+  ret <4 x bfloat> %dst
+}
+
+define <4 x bfloat> @test_vp_reverse_v4bf16(<4 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_v4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a1
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+
+  %dst = call <4 x bfloat> @llvm.experimental.vp.reverse.v4bf16(<4 x bfloat> %src, <4 x i1> splat (i1 1), i32 %evl)
+  ret <4 x bfloat> %dst
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverser-int.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverse-int.ll
similarity index 100%
rename from llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverser-int.ll
rename to llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-reverse-int.ll
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
index 7245d8a11f56..fb22e5eaca2c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vp-splice.ll
@@ -1,86 +1,99 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zvfh -verify-machineinstrs -riscv-v-vector-bits-min=128 \
-; RUN:   < %s | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zvfh,+zvfbfmin -verify-machineinstrs -riscv-v-vector-bits-min=128 \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zvfhmin,+zvfbfmin -verify-machineinstrs -riscv-v-vector-bits-min=128 \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <2 x i64> @test_vp_splice_v2i64(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
 define <2 x i64> @test_vp_splice_v2i64_negative_offset(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <2 x i64> %v
+}
+
+define <2 x i64> @test_vp_splice_v2i64_zero_offset(<2 x i64> %va, <2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_v2i64_zero_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 0, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
 define <2 x i64> @test_vp_splice_v2i64_masked(<2 x i64> %va, <2 x i64> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2i64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x i64> @llvm.experimental.vp.splice.v2i64(<2 x i64> %va, <2 x i64> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x i64> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32_negative_offset(<4 x i32> %va, <4 x i32> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
 define <4 x i32> @test_vp_splice_v4i32_masked(<4 x i32> %va, <4 x i32> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4i32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x i32> %v
 }
 
@@ -169,82 +182,82 @@ define <16 x i8> @test_vp_splice_v16i8_masked(<16 x i8> %va, <16 x i8> %vb, <16
 define <2 x double> @test_vp_splice_v2f64(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <2 x double> @test_vp_splice_v2f64_negative_offset(<2 x double> %va, <2 x double> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e64, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <2 x double> @test_vp_splice_v2f64_masked(<2 x double> %va, <2 x double> %vb, <2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v2f64_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e64, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x double> @llvm.experimental.vp.splice.v2f64(<2 x double> %va, <2 x double> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x double> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32_negative_offset(<4 x float> %va, <4 x float> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32_negative_offset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e32, m1, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v8, v8, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    vslideup.vi v8, v9, 3
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
 define <4 x float> @test_vp_splice_v4f32_masked(<4 x float> %va, <4 x float> %vb, <4 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_v4f32_masked:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v8, v8, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, mu
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x float> @llvm.experimental.vp.splice.v4f32(<4 x float> %va, <4 x float> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x float> %v
 }
 
@@ -288,3 +301,103 @@ define <8 x half> @test_vp_splice_v8f16_masked(<8 x half> %va, <8 x half> %vb, <
   %v = call <8 x half> @llvm.experimental.vp.splice.v8f16(<8 x half> %va, <8 x half> %vb, i32 5, <8 x i1> %mask, i32 %evla, i32 %evlb)
   ret <8 x half> %v
 }
+
+define <4 x i32> @test_vp_splice_v4i32_with_firstelt(i32 %first, <4 x i32> %vb, <4 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_v4i32_with_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslide1up.vx v9, v8, a0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %va = insertelement <4 x i32> poison, i32 %first, i32 0
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %va, <4 x i32> %vb, i32 0, <4 x i1> %mask, i32 1, i32 %evl)
+  ret <4 x i32> %v
+}
+
+define <4 x i32> @test_vp_splice_v4i32_with_splat_firstelt(i32 %first, <4 x i32> %vb, <4 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_v4i32_with_splat_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslide1up.vx v9, v8, a0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %ins = insertelement <4 x i32> poison, i32 %first, i32 0
+  %splat = shufflevector <4 x i32> %ins, <4 x i32> poison, <4 x i32> zeroinitializer
+  %v = call <4 x i32> @llvm.experimental.vp.splice.v4i32(<4 x i32> %splat, <4 x i32> %vb, i32 0, <4 x i1> %mask, i32 1, i32 %evl)
+  ret <4 x i32> %v
+}
+
+define <4 x float> @test_vp_splice_nxv2f32_with_firstelt(float %first, <4 x float> %vb, <4 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_nxv2f32_with_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfslide1up.vf v9, v8, fa0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %va = insertelement <4 x float> poison, float %first, i32 0
+  %v = call <4 x float> @llvm.experimental.vp.splice.nxv2f32(<4 x float> %va, <4 x float> %vb, i32 0, <4 x i1> %mask, i32 1, i32 %evl)
+  ret <4 x float> %v
+}
+
+define <4 x half> @test_vp_splice_nxv2f16_with_firstelt(half %first, <4 x half> %vb, <4 x i1> %mask, i32 zeroext %evl) {
+; ZVFH-LABEL: test_vp_splice_nxv2f16_with_firstelt:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vfslide1up.vf v9, v8, fa0, v0.t
+; ZVFH-NEXT:    vmv1r.v v8, v9
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: test_vp_splice_nxv2f16_with_firstelt:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.w a1, fa0
+; ZVFHMIN-NEXT:    vsetivli zero, 4, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.s.x v9, a1
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
+; ZVFHMIN-NEXT:    vslideup.vi v9, v8, 1, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v8, v9
+; ZVFHMIN-NEXT:    ret
+  %va = insertelement <4 x half> poison, half %first, i32 0
+  %v = call <4 x half> @llvm.experimental.vp.splice.nxv2f16(<4 x half> %va, <4 x half> %vb, i32 0, <4 x i1> %mask, i32 1, i32 %evl)
+  ret <4 x half> %v
+}
+
+define <8 x bfloat> @test_vp_splice_v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_v8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+
+  %v = call <8 x bfloat> @llvm.experimental.vp.splice.v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, i32 5, <8 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <8 x bfloat> %v
+}
+
+define <8 x bfloat> @test_vp_splice_v8bf16_negative_offset(<8 x bfloat> %va, <8 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_v8bf16_negative_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetivli zero, 5, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    ret
+
+  %v = call <8 x bfloat> @llvm.experimental.vp.splice.v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, i32 -5, <8 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <8 x bfloat> %v
+}
+
+define <8 x bfloat> @test_vp_splice_v8bf16_masked(<8 x bfloat> %va, <8 x bfloat> %vb, <8 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_v8bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+  %v = call <8 x bfloat> @llvm.experimental.vp.splice.v8bf16(<8 x bfloat> %va, <8 x bfloat> %vb, i32 5, <8 x i1> %mask, i32 %evla, i32 %evlb)
+  ret <8 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
index 984bc5b2c735..b97fa1d3a51e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vselect.ll
@@ -25,8 +25,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -56,8 +56,8 @@ define void @vselect_vv_v6i32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -95,8 +95,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -126,8 +126,8 @@ define void @vselect_vx_v6i32(i32 %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -166,8 +166,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -197,8 +197,8 @@ define void @vselect_vi_v6i32(ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -236,8 +236,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
 ; RV32-NEXT:    vslide1down.vx v10, v10, a4
 ; RV32-NEXT:    vslide1down.vx v10, v10, a2
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -267,8 +267,8 @@ define void @vselect_vv_v6f32(ptr %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
 ; RV64-NEXT:    vslide1down.vx v10, v10, a4
 ; RV64-NEXT:    vslide1down.vx v10, v10, a2
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, tu, mu
@@ -306,8 +306,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -337,8 +337,8 @@ define void @vselect_vx_v6f32(float %a, ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -377,8 +377,8 @@ define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ; RV32-NEXT:    vslide1down.vx v10, v10, a0
 ; RV32-NEXT:    vslide1down.vx v10, v10, a3
 ; RV32-NEXT:    vslide1down.vx v10, v10, a1
-; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vi v10, v10, 2
 ; RV32-NEXT:    vand.vi v10, v10, 1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
@@ -408,8 +408,8 @@ define void @vselect_vfpzero_v6f32(ptr %b, ptr %cc, ptr %z) {
 ; RV64-NEXT:    vslide1down.vx v10, v10, a0
 ; RV64-NEXT:    vslide1down.vx v10, v10, a3
 ; RV64-NEXT:    vslide1down.vx v10, v10, a1
-; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vi v10, v10, 2
 ; RV64-NEXT:    vand.vi v10, v10, 1
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
index 2757e140ecde..3633885bfa7d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mgather-sdnode.ll
@@ -2373,3 +2373,219 @@ define <vscale x 1 x i8> @mgather_baseidx_zext_nxv1i1_nxv1i8(ptr %base, <vscale
   %v = call <vscale x 1 x i8> @llvm.masked.gather.nxv1i8.nxv1p0(<vscale x 1 x ptr> %ptrs, i32 1, <vscale x 1 x i1> %m, <vscale x 1 x i8> %passthru)
   ret <vscale x 1 x i8> %v
 }
+
+define <4 x i32> @scalar_prefix(ptr %base, i32 signext %index, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a1, a1, 10
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a2, 4
+; RV64-NEXT:    slli a1, a1, 10
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a2
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, i32 %index, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix_with_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vsll.vi v9, v9, 10
+; RV32-NEXT:    vadd.vx v9, v9, a0
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vadd.vv v8, v9, v8
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix_with_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a2, 1024
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vmv.v.x v9, a2
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v9
+; RV64-NEXT:    li a0, 4
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %broadcast.splat, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_constant_splat(ptr %base, <4 x i32> %vecidx) {
+; RV32-LABEL: scalar_prefix_with_constant_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a1, 5
+; RV32-NEXT:    add a0, a0, a1
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: scalar_prefix_with_constant_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 4
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a1
+; RV64-NEXT:    lui a1, 5
+; RV64-NEXT:    add a0, a0, a1
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> splat (i32 20), <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v9, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    li a0, 1024
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vmv.v.i v8, 4
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, i32 %index
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate_with_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate_with_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmv.v.x v9, a1
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v9, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate_with_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vmv.v.x v10, a0
+; RV64-NEXT:    li a0, 1024
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vwmaccus.vx v10, a0, v8
+; RV64-NEXT:    vmv.v.i v8, 4
+; RV64-NEXT:    vwmaccsu.vx v10, a1, v8
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %broadcast.splatinsert = insertelement <4 x i32> poison, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, <4 x i32> %broadcast.splat
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate_with_constant_splat(ptr %base, i32 %index, <4 x i32> %vecidx) {
+; RV32-LABEL: reassociate_with_constant_splat:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v8, v8, 10
+; RV32-NEXT:    addi a0, a0, 80
+; RV32-NEXT:    vluxei32.v v8, (a0), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reassociate_with_constant_splat:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 1024
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vwmulsu.vx v10, v8, a1
+; RV64-NEXT:    addi a0, a0, 80
+; RV64-NEXT:    vluxei64.v v8, (a0), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i32> %vecidx, <4 x i32> splat (i32 20)
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @diagonal(ptr %base, <4 x i64> %vecidx) {
+; RV32-LABEL: diagonal:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vnsrl.wi v10, v8, 0
+; RV32-NEXT:    vsll.vi v8, v10, 10
+; RV32-NEXT:    vadd.vx v8, v8, a0
+; RV32-NEXT:    vsll.vi v9, v10, 2
+; RV32-NEXT:    vadd.vv v8, v8, v9
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: diagonal:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vsll.vi v10, v8, 10
+; RV64-NEXT:    vadd.vx v10, v10, a0
+; RV64-NEXT:    vsll.vi v8, v8, 2
+; RV64-NEXT:    vadd.vv v10, v10, v8
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr inbounds nuw [256 x [256 x float]], ptr %base, i64 0, <4 x i64> %vecidx, <4 x i64> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @diagonal_i32(ptr %base, <4 x i32> %vecidx) {
+; RV32-LABEL: diagonal_i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vsll.vi v9, v8, 10
+; RV32-NEXT:    vadd.vx v9, v9, a0
+; RV32-NEXT:    vsll.vi v8, v8, 2
+; RV32-NEXT:    vadd.vv v8, v9, v8
+; RV32-NEXT:    vluxei32.v v8, (zero), v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: diagonal_i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; RV64-NEXT:    vsext.vf2 v10, v8
+; RV64-NEXT:    vsll.vi v8, v10, 10
+; RV64-NEXT:    vadd.vx v8, v8, a0
+; RV64-NEXT:    vsll.vi v10, v10, 2
+; RV64-NEXT:    vadd.vv v10, v8, v10
+; RV64-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; RV64-NEXT:    vluxei64.v v8, (zero), v10
+; RV64-NEXT:    ret
+  %gep = getelementptr inbounds nuw [256 x [256 x float]], ptr %base, i64 0, <4 x i32> %vecidx, <4 x i32> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
deleted file mode 100644
index 0c058b562f53..000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v -stop-after=finalize-isel | FileCheck %s
-
-declare <vscale x 2 x i32> @llvm.vp.merge.nxv2i32(<vscale x 2 x i1>, <vscale x 2 x i32>, <vscale x 2 x i32>, i32)
-declare <vscale x 2 x i32> @llvm.vp.select.nxv2i32(<vscale x 2 x i1>, <vscale x 2 x i32>, <vscale x 2 x i32>, i32)
-declare <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr, <vscale x 2 x i1>, i32)
-
-; Test result has chain output of true operand of merge.vvm.
-define void @vpmerge_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale x 2 x i1> %m, i32 zeroext %vl) {
-  ; CHECK-LABEL: name: vpmerge_vpload_store
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $v8, $x10, $v0, $x11
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v0
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   PseudoVSE32_V_M1 killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]], -1, 5 /* e32 */ :: (store (<vscale x 1 x s64>) into %ir.p)
-  ; CHECK-NEXT:   PseudoRET
-  %a = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr %p, <vscale x 2 x i1> splat (i1 -1), i32 %vl)
-  %b = call <vscale x 2 x i32> @llvm.vp.merge.nxv2i32(<vscale x 2 x i1> %m, <vscale x 2 x i32> %a, <vscale x 2 x i32> %passthru, i32 %vl)
-  store <vscale x 2 x i32> %b, ptr %p
-  ret void
-}
-
-define void @vpselect_vpload_store(<vscale x 2 x i32> %passthru, ptr %p, <vscale x 2 x i1> %m, i32 zeroext %vl) {
-  ; CHECK-LABEL: name: vpselect_vpload_store
-  ; CHECK: bb.0 (%ir-block.0):
-  ; CHECK-NEXT:   liveins: $v8, $x10, $v0, $x11
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v0
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr = COPY $x10
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vrnov0 = COPY $v8
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVLE32_V_M1_MASK:%[0-9]+]]:vrnov0 = PseudoVLE32_V_M1_MASK [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   PseudoVSE32_V_M1 killed [[PseudoVLE32_V_M1_MASK]], [[COPY2]], -1, 5 /* e32 */ :: (store (<vscale x 1 x s64>) into %ir.p)
-  ; CHECK-NEXT:   PseudoRET
-  %a = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr %p, <vscale x 2 x i1> splat (i1 -1), i32 %vl)
-  %b = call <vscale x 2 x i32> @llvm.vp.select.nxv2i32(<vscale x 2 x i1> %m, <vscale x 2 x i32> %a, <vscale x 2 x i32> %passthru, i32 %vl)
-  store <vscale x 2 x i32> %b, ptr %p
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
index f8581d8e21b3..eaa59eee7dc0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/select-fp.ll
@@ -3,9 +3,9 @@
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
+; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=ilp32d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfhmin,+v -target-abi=lp64d \
 ; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @select_nxv1f16(i1 zeroext %c, <vscale x 1 x half> %a, <vscale x 1 x half> %b) {
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 9972df97ad9f..8495dfe35072 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -299,9 +299,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
@@ -318,9 +317,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
@@ -628,9 +626,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1bf16(<vscale x 1 x bfloat> %va, bfloat
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
@@ -647,9 +644,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1bf16(<vscale x 1 x bfloat> %va, b
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v9, v8
-; CHECK-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vfmv.v.f v8, fa5
-; CHECK-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; CHECK-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; CHECK-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v8, v9
@@ -978,9 +974,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 ; CHECK-LABEL: fcmp_ord_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -998,9 +993,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, b
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1314,9 +1308,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8bf16(<vscale x 8 x bfloat> %va, bfloat
 ; CHECK-LABEL: fcmp_uno_vf_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1334,9 +1327,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8bf16(<vscale x 8 x bfloat> %va, b
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv8bf16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    fcvt.s.bf16 fa5, fa0
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfmv.v.f v12, fa5
 ; CHECK-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v12, v8
@@ -1968,9 +1960,8 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v8, v9
@@ -1981,9 +1972,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v9, v8
@@ -1997,9 +1987,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_swap_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v9, v8
@@ -2010,9 +1999,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfeq.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmand.mm v0, v8, v9
@@ -2451,9 +2439,8 @@ define <vscale x 1 x i1> @fcmp_uno_vv_nxv1f16(<vscale x 1 x half> %va, <vscale x
 define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v8, v9
@@ -2464,9 +2451,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v9, v8
@@ -2480,9 +2466,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f16(<vscale x 1 x half> %va, half %b,
 define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f16(<vscale x 1 x half> %va, half %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_swap_nxv1f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v9, fa0
 ; ZVFH-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v9, v8
@@ -2493,9 +2478,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f16(<vscale x 1 x half> %va, half
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v9, v8
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vfmv.v.f v8, fa5
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vmfne.vf v8, v8, fa5, v0.t
 ; ZVFHMIN-NEXT:    vmfne.vv v9, v9, v9, v0.t
 ; ZVFHMIN-NEXT:    vmor.mm v0, v8, v9
@@ -2966,9 +2950,8 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfeq.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v10, v12
@@ -2977,9 +2960,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ; ZVFHMIN-LABEL: fcmp_ord_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -2996,9 +2978,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_ord_vf_swap_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfeq.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfeq.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmand.mm v0, v12, v10
@@ -3007,9 +2988,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ; ZVFHMIN-LABEL: fcmp_ord_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfeq.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -3459,9 +3439,8 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8f16(<vscale x 8 x half> %va, <vscale x
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfne.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v10, v12
@@ -3470,9 +3449,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 ; ZVFHMIN-LABEL: fcmp_uno_vf_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -3489,9 +3467,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f16(<vscale x 8 x half> %va, half %b,
 define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f16(<vscale x 8 x half> %va, half %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; ZVFH-LABEL: fcmp_uno_vf_swap_nxv8f16:
 ; ZVFH:       # %bb.0:
-; ZVFH-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; ZVFH-NEXT:    vfmv.v.f v10, fa0
 ; ZVFH-NEXT:    vmfne.vf v12, v10, fa0, v0.t
 ; ZVFH-NEXT:    vmfne.vv v10, v8, v8, v0.t
 ; ZVFH-NEXT:    vmor.mm v0, v12, v10
@@ -3500,9 +3477,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f16(<vscale x 8 x half> %va, half
 ; ZVFHMIN-LABEL: fcmp_uno_vf_swap_nxv8f16:
 ; ZVFHMIN:       # %bb.0:
 ; ZVFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; ZVFHMIN-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; ZVFHMIN-NEXT:    vfmv.v.f v12, fa5
 ; ZVFHMIN-NEXT:    vmfne.vf v10, v12, fa5, v0.t
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v12, v8
@@ -4001,9 +3977,8 @@ define <vscale x 1 x i1> @fcmp_ord_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
 define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v8, v9
@@ -4017,9 +3992,8 @@ define <vscale x 1 x i1> @fcmp_ord_vf_nxv1f64(<vscale x 1 x double> %va, double
 define <vscale x 1 x i1> @fcmp_ord_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfeq.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v9, v8
@@ -4267,9 +4241,8 @@ define <vscale x 1 x i1> @fcmp_uno_vv_nxv1f64(<vscale x 1 x double> %va, <vscale
 define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v8, v9
@@ -4283,9 +4256,8 @@ define <vscale x 1 x i1> @fcmp_uno_vf_nxv1f64(<vscale x 1 x double> %va, double
 define <vscale x 1 x i1> @fcmp_uno_vf_swap_nxv1f64(<vscale x 1 x double> %va, double %b, <vscale x 1 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vmfne.vf v9, v9, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v8, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v9, v8
@@ -4551,9 +4523,8 @@ define <vscale x 8 x i1> @fcmp_ord_vv_nxv8f64(<vscale x 8 x double> %va, <vscale
 define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfeq.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v16, v24
@@ -4567,9 +4538,8 @@ define <vscale x 8 x i1> @fcmp_ord_vf_nxv8f64(<vscale x 8 x double> %va, double
 define <vscale x 8 x i1> @fcmp_ord_vf_swap_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_ord_vf_swap_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfeq.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfeq.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmand.mm v0, v24, v16
@@ -4820,9 +4790,8 @@ define <vscale x 8 x i1> @fcmp_uno_vv_nxv8f64(<vscale x 8 x double> %va, <vscale
 define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfne.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v16, v24
@@ -4836,9 +4805,8 @@ define <vscale x 8 x i1> @fcmp_uno_vf_nxv8f64(<vscale x 8 x double> %va, double
 define <vscale x 8 x i1> @fcmp_uno_vf_swap_nxv8f64(<vscale x 8 x double> %va, double %b, <vscale x 8 x i1> %m, i32 zeroext %evl) {
 ; CHECK-LABEL: fcmp_uno_vf_swap_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
+; CHECK-NEXT:    vfmv.v.f v16, fa0
 ; CHECK-NEXT:    vmfne.vf v24, v16, fa0, v0.t
 ; CHECK-NEXT:    vmfne.vv v16, v8, v8, v0.t
 ; CHECK-NEXT:    vmor.mm v0, v24, v16
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index 4753ab915bdf..5c04a09c9953 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -141,28 +141,26 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) {
 define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) {
 ; V-LABEL: vector_deinterleave_v8i64_v16i64:
 ; V:       # %bb.0:
-; V-NEXT:    li a0, 85
 ; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; V-NEXT:    vmv.v.i v0, -16
 ; V-NEXT:    vid.v v16
 ; V-NEXT:    vsetivli zero, 8, e64, m8, ta, ma
 ; V-NEXT:    vslidedown.vi v24, v8, 8
+; V-NEXT:    li a0, 85
+; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; V-NEXT:    vmv.v.i v0, -16
 ; V-NEXT:    vmv.s.x v12, a0
 ; V-NEXT:    li a0, 170
-; V-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; V-NEXT:    vadd.vv v20, v16, v16
-; V-NEXT:    vmv.s.x v21, a0
+; V-NEXT:    vadd.vv v13, v16, v16
+; V-NEXT:    vmv.s.x v20, a0
+; V-NEXT:    vadd.vi v21, v13, -8
 ; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
 ; V-NEXT:    vcompress.vm v16, v8, v12
 ; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; V-NEXT:    vadd.vi v22, v20, -8
-; V-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; V-NEXT:    vcompress.vm v12, v8, v21
-; V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; V-NEXT:    vadd.vi v8, v20, -7
+; V-NEXT:    vadd.vi v22, v13, -7
 ; V-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; V-NEXT:    vrgatherei16.vv v16, v24, v22, v0.t
-; V-NEXT:    vrgatherei16.vv v12, v24, v8, v0.t
+; V-NEXT:    vcompress.vm v12, v8, v20
+; V-NEXT:    vrgatherei16.vv v16, v24, v21, v0.t
+; V-NEXT:    vrgatherei16.vv v12, v24, v22, v0.t
 ; V-NEXT:    vmv.v.v v8, v16
 ; V-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index 9344c5209868..b11db3d61f69 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -380,31 +380,6 @@ define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
   ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res3
 }
 
-; TODO: Remove once recursive deinterleaving support is removed
-define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deinterleave_load_factor4_recursive(ptr %p) {
-; CHECK-LABEL: vector_deinterleave_load_factor4_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vlseg4e8.v v8, (a0)
-; CHECK-NEXT:    ret
-  %vec = load <vscale x 32 x i8>, ptr %p
-  %d0 = call {<vscale x 16 x i8>, <vscale x 16 x i8>} @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> %vec)
-  %d0.0 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } %d0, 1
-  %d1 = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %d0.0)
-  %t0 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d1, 0
-  %t2 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d1, 1
-  %d2 = call {<vscale x 8 x i8>, <vscale x 8 x i8>} @llvm.vector.deinterleave2.nxv16i8(<vscale x 16 x i8> %d0.1)
-  %t1 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d2, 0
-  %t3 = extractvalue { <vscale x 8 x i8>, <vscale x 8 x i8> } %d2, 1
-
-  %res0 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } undef, <vscale x 8 x i8> %t0, 0
-  %res1 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res0, <vscale x 8 x i8> %t1, 1
-  %res2 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res1, <vscale x 8 x i8> %t2, 2
-  %res3 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res2, <vscale x 8 x i8> %t3, 3
-  ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res3
-}
-
 define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @vector_deinterleave_load_factor5(ptr %p) {
 ; CHECK-LABEL: vector_deinterleave_load_factor5:
 ; CHECK:       # %bb.0:
@@ -500,45 +475,3 @@ define { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
   %res7 = insertvalue { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res6, <vscale x 8 x i8> %t7, 7
   ret { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } %res7
 }
-
-; TODO: Remove once recursive deinterleaving support is removed
-define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @vector_deinterleave_load_factor8_recursive(ptr %ptr) {
-; CHECK-LABEL: vector_deinterleave_load_factor8_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vlseg8e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %vec = load <vscale x 16 x i32>, ptr %ptr
-  %d0 = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %vec)
-  %d0.0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 1
-  %d1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.0)
-  %d1.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 0
-  %d1.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 1
-  %d2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.1)
-  %d2.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 0
-  %d2.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 1
-
-  %d3 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 0
-  %t4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 1
-  %d4 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.1)
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 0
-  %t6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 1
-  %d5 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.0)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 0
-  %t5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 1
-  %d6 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.1)
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 0
-  %t7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 1
-
-  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } undef, <vscale x 2 x i32> %t0, 0
-  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
-  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
-  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
-  %res4 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3, <vscale x 2 x i32> %t4, 4
-  %res5 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res4, <vscale x 2 x i32> %t5, 5
-  %res6 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res5, <vscale x 2 x i32> %t6, 6
-  %res7 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res6, <vscale x 2 x i32> %t7, 7
-  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res7
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index a5811e697634..af55aaa8fce8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -260,20 +260,6 @@ define void @vector_interleave_store_factor4(<vscale x 2 x i32> %a, <vscale x 2
   ret void
 }
 
-; TODO: Remove once recursive interleaving support is removed
-define void @vector_interleave_store_factor4_recursive(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, ptr %p) {
-; CHECK-LABEL: vector_interleave_store_factor4_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vsseg4e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %v0 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
-  %v1 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %d)
-  %v2 = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %v0, <vscale x 8 x i32> %v1)
-  store <vscale x 16 x i32> %v2, ptr %p
-  ret void
-}
-
 define void @vector_interleave_store_factor5(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, <vscale x 2 x i32> %e, ptr %p) {
 ; CHECK-LABEL: vector_interleave_store_factor5:
 ; CHECK:       # %bb.0:
@@ -317,23 +303,3 @@ define void @vector_interleave_store_factor8(<vscale x 2 x i32> %a, <vscale x 2
   store <vscale x 16 x i32> %v, ptr %p
   ret void
 }
-
-; TODO: Remove once recursive interleaving support is removed
-define void @vector_interleave_store_factor8_recursive(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscale x 2 x i32> %c, <vscale x 2 x i32> %d, <vscale x 2 x i32> %e, <vscale x 2 x i32> %f, <vscale x 2 x i32> %g, <vscale x 2 x i32> %h, ptr %p) {
-; CHECK-LABEL: vector_interleave_store_factor8_recursive:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vsseg8e32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %v0 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %a, <vscale x 2 x i32> %e)
-  %v1 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %c, <vscale x 2 x i32> %g)
-  %v2 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %v0, <vscale x 4 x i32> %v1)
-
-  %v3 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %b, <vscale x 2 x i32> %f)
-  %v4 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %d, <vscale x 2 x i32> %h)
-  %v5 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %v3, <vscale x 4 x i32> %v4)
-
-  %v6 = call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %v2, <vscale x 8 x i32> %v5)
-  store <vscale x 16 x i32> %v6, ptr %p
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll b/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
index 4a24e2342fab..3a461c872144 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-tuple-align.ll
@@ -11,7 +11,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv8i8(pt
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E8_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E8_V_M1 $noreg, [[COPY1]], [[COPY]], 3 /* e8 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 1)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E8_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E8_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -27,7 +27,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv4i16(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E16_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E16_V_M1 $noreg, [[COPY1]], [[COPY]], 4 /* e16 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 2)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E16_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E16_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -43,7 +43,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv2i32(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E32_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E32_V_M1 $noreg, [[COPY1]], [[COPY]], 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 4)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E32_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E32_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
@@ -59,7 +59,7 @@ define target("riscv.vector.tuple", <vscale x 8 x i8>, 2)  @test_vlseg_nxv1i64(p
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY $x11
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr = COPY $x10
   ; CHECK-NEXT:   [[PseudoVLSEG2E64_V_M1_:%[0-9]+]]:vrn2m1 = PseudoVLSEG2E64_V_M1 $noreg, [[COPY1]], [[COPY]], 6 /* e64 */, 2 /* tu, ma */ :: (load unknown-size from %ir.p, align 8)
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY [[PseudoVLSEG2E64_V_M1_]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vrn2m1 = COPY killed [[PseudoVLSEG2E64_V_M1_]]
   ; CHECK-NEXT:   $v8_v9 = COPY [[COPY2]]
   ; CHECK-NEXT:   PseudoRET implicit $v8_v9
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
index 35bed86d6117..03e6e6b7a624 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfma-vp-combine.ll
@@ -30,9 +30,8 @@ define <vscale x 1 x double> @test2(<vscale x 1 x double> %a, <vscale x 1 x i1>
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI1_0)(a1)
 ; CHECK-NEXT:    lui a1, %hi(.LCPI1_1)
 ; CHECK-NEXT:    fld fa4, %lo(.LCPI1_1)(a1)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa5
 ; CHECK-NEXT:    vfadd.vf v9, v9, fa4, v0.t
 ; CHECK-NEXT:    vfmul.vv v8, v8, v9, v0.t
 ; CHECK-NEXT:    ret
@@ -49,9 +48,8 @@ define <vscale x 1 x double> @test3(<vscale x 1 x double> %a, <vscale x 1 x doub
 ; CHECK-NEXT:    fld fa5, %lo(.LCPI2_0)(a1)
 ; CHECK-NEXT:    lui a1, %hi(.LCPI2_1)
 ; CHECK-NEXT:    fld fa4, %lo(.LCPI2_1)(a1)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v10, fa5
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v10, fa5
 ; CHECK-NEXT:    vfmul.vf v10, v10, fa4, v0.t
 ; CHECK-NEXT:    vfmadd.vv v10, v8, v9, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index 1cbb980aebff..0aa0cbceefc7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -3081,24 +3081,66 @@ define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <v
   ret <vscale x 1 x i8> %3
 }
 
-define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+define <vscale x 4 x i32> @vwsll_vi(<vscale x 4 x i16> %a, iXLen %vl) {
 ; NOVLOPT-LABEL: vwsll_vi:
 ; NOVLOPT:       # %bb.0:
 ; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; NOVLOPT-NEXT:    vwsll.vi v12, v8, 1
+; NOVLOPT-NEXT:    vwsll.vi v10, v8, 1
 ; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
-; NOVLOPT-NEXT:    vadd.vv v8, v12, v10
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
 ; NOVLOPT-NEXT:    ret
 ;
 ; VLOPT-LABEL: vwsll_vi:
 ; VLOPT:       # %bb.0:
 ; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; VLOPT-NEXT:    vwsll.vi v12, v8, 1
+; VLOPT-NEXT:    vwsll.vi v10, v8, 1
 ; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; VLOPT-NEXT:    vadd.vv v8, v12, v10
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
 ; VLOPT-NEXT:    ret
-  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a,iXLen 1, iXLen -1)
-  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %b, iXLen %vl)
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen 1, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwsll_vx(<vscale x 4 x i16> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vx v10, v8, a0
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vwsll_vv(<vscale x 4 x i16> %a, <vscale x 4 x i16> %b, iXLen %vl) {
+; NOVLOPT-LABEL: vwsll_vv:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; NOVLOPT-NEXT:    vwsll.vv v10, v8, v9
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vwsll_vv:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; VLOPT-NEXT:    vwsll.vv v10, v8, v9
+; VLOPT-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vwsll.nxv4i32.nxv4i16(<vscale x 4 x i32> poison, <vscale x 4 x i16> %a, <vscale x 4 x i16> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
   ret <vscale x 4 x i32> %2
 }
 
@@ -3434,6 +3476,172 @@ define <vscale x 4 x i32> @vid.v(<vscale x 4 x i32> %c, iXLen %vl) {
   ret <vscale x 4 x i32> %2
 }
 
+define <vscale x 4 x i32> @vslideup_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslideup_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslideup.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslideup_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslideup.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslideup(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslideup_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vslideup_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslideup.vi v10, v8, 2
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslideup_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vslideup.vi v10, v8, 2
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslideup(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 2, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslidedown_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslidedown_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslidedown.vx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslidedown_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslidedown.vx v8, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslidedown(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslidedown_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; NOVLOPT-LABEL: vslidedown_vi:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslidedown.vi v8, v8, 2
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslidedown_vi:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vslidedown.vi v8, v8, 2
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslidedown(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 2, iXLen -1, iXLen 3)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vslide1up_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslide1up_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslide1up.vx v10, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslide1up_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vslide1up.vx v10, v8, a0
+; VLOPT-NEXT:    vadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslide1up(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x float> @vfslide1up_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfslide1up_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfslide1up.vf v10, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v10
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfslide1up_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfslide1up.vf v10, v8, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v10
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfslide1up(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+; Negative test – not safe to reduce vl
+
+define <vscale x 4 x i32> @vslide1down_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; NOVLOPT-LABEL: vslide1down_vx:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vslide1down.vx v8, v8, a0
+; NOVLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vslide1down_vx:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli a2, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vslide1down.vx v8, v8, a0
+; VLOPT-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; VLOPT-NEXT:    vadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vslide1down(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %1, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+; Negative test – not safe to reduce vl
+
+define <vscale x 4 x float> @vfslide1down_vf(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfslide1down_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfslide1down.vf v8, v8, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v8, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfslide1down_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; VLOPT-NEXT:    vfslide1down.vf v8, v8, fa0
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfadd.vv v8, v8, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfslide1down(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %1, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
 define <vscale x 4 x float> @vfadd_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, iXLen %vl) {
 ; NOVLOPT-LABEL: vfadd_vv:
 ; NOVLOPT:       # %bb.0:
@@ -4540,6 +4748,46 @@ define <vscale x 4 x float> @vfsgnjx_vf(<vscale x 4 x float> %a, float %b, iXLen
   ret <vscale x 4 x float> %2
 }
 
+define <vscale x 4 x float> @vfmerge_vf(<vscale x 4 x float> %a, float %b, <vscale x 4 x i1> %c, iXLen %vl) {
+; NOVLOPT-LABEL: vfmerge_vf:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmerge.vfm v10, v8, fa0, v0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmerge_vf:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmerge.vfm v10, v8, fa0, v0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmerge(<vscale x 4 x float> poison, <vscale x 4 x float> %a, float %b, <vscale x 4 x i1> %c, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
+define <vscale x 4 x float> @vfmv_v_f(<vscale x 4 x float> %a, float %b, iXLen %vl) {
+; NOVLOPT-LABEL: vfmv_v_f:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfmv.v.f v10, fa0
+; NOVLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfadd.vv v8, v10, v8
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfmv_v_f:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; VLOPT-NEXT:    vfmv.v.f v10, fa0
+; VLOPT-NEXT:    vfadd.vv v8, v10, v8
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfmv.v.f(<vscale x 4 x float> poison, float %b, iXLen -1)
+  %2 = call <vscale x 4 x float> @llvm.riscv.vfadd.nxv4f32.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %1, <vscale x 4 x float> %a, iXLen 7, iXLen %vl)
+  ret <vscale x 4 x float> %2
+}
+
 define <vscale x 4 x float> @vfmacc_vv(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, iXLen %vl) {
 ; NOVLOPT-LABEL: vfmacc_vv:
 ; NOVLOPT:       # %bb.0:
@@ -5117,3 +5365,29 @@ define <vscale x 4 x double> @vfrsqrt7(<vscale x 4 x float> %a) {
   %2 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %1, iXLen 7, iXLen 6, iXLen 0)
   ret <vscale x 4 x double> %2
 }
+
+define <vscale x 4 x double> @vfrec7(<vscale x 4 x float> %a) {
+; NOVLOPT-LABEL: vfrec7:
+; NOVLOPT:       # %bb.0:
+; NOVLOPT-NEXT:    vsetivli zero, 7, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vmv2r.v v12, v8
+; NOVLOPT-NEXT:    fsrmi a0, 0
+; NOVLOPT-NEXT:    vfrec7.v v14, v8
+; NOVLOPT-NEXT:    fsrm a0
+; NOVLOPT-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
+; NOVLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; NOVLOPT-NEXT:    ret
+;
+; VLOPT-LABEL: vfrec7:
+; VLOPT:       # %bb.0:
+; VLOPT-NEXT:    vsetivli zero, 6, e32, m2, ta, ma
+; VLOPT-NEXT:    vmv2r.v v12, v8
+; VLOPT-NEXT:    fsrmi a0, 0
+; VLOPT-NEXT:    vfrec7.v v14, v8
+; VLOPT-NEXT:    fsrm a0
+; VLOPT-NEXT:    vfwmacc.vv v8, v12, v14
+; VLOPT-NEXT:    ret
+  %1 = call <vscale x 4 x float> @llvm.riscv.vfrec7.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> %a, iXLen 0, iXLen 7)
+  %2 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %1, iXLen 7, iXLen 6, iXLen 0)
+  ret <vscale x 4 x double> %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
new file mode 100644
index 000000000000..03204468dc14
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/vmerge-peephole.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc %s -o - -mtriple=riscv64 -mattr=+v -run-pass=riscv-vector-peephole -verify-machineinstrs | FileCheck %s
+
+---
+name: vle32
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %passthru:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %passthru, $noreg, %mask, %avl, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %passthru:vrnov0 = COPY $v8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %x, %mask, %avl, 5 /* e32 */
+...
+---
+name: vle32_no_passthru
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32_no_passthru
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %false:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %false, $noreg, %mask, %avl, 5 /* e32 */, 1 /* ta, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %false:vr = COPY $v8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %x, %mask, %avl, 5 /* e32 */
+...
+---
+name: vle32_move_past_passthru
+body: |
+  bb.0:
+    liveins: $x8, $v0, $v8
+    ; CHECK-LABEL: name: vle32_move_past_passthru
+    ; CHECK: liveins: $x8, $v0, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %avl:gprnox0 = COPY $x8
+    ; CHECK-NEXT: %passthru:vrnov0 = COPY $v8
+    ; CHECK-NEXT: %mask:vmv0 = COPY $v0
+    ; CHECK-NEXT: %y:vrnov0 = PseudoVLE32_V_M1_MASK %passthru, $noreg, %mask, %avl, 5 /* e32 */, 0 /* tu, mu */ :: (load unknown-size, align 1)
+    %avl:gprnox0 = COPY $x8
+    %x:vr = PseudoVLE32_V_M1 $noreg, $noreg, %avl, 5 /* e32 */, 2 /* tu, ma */ :: (load unknown-size)
+    %passthru:vrnov0 = COPY $v8
+    %mask:vmv0 = COPY $v0
+    %y:vrnov0 = PseudoVMERGE_VVM_M1 %passthru, %passthru, %x, %mask, %avl, 5 /* e32 */
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir b/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
index 2786f58826b9..ccabd5099071 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vmv-copy.mir
@@ -290,10 +290,10 @@ body:             |
     ; CHECK: liveins: $x2, $x10, $v8, $v13, $v4m4, $v16m4
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x0 = PseudoVSETVLI $x10, 66 /* e8, m4, ta, mu */, implicit-def $vl, implicit-def $vtype
-    ; CHECK-NEXT: early-clobber $v4m4 = PseudoVQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
+    ; CHECK-NEXT: early-clobber $v4m4 = PseudoSF_VQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $v16m4 = PseudoVMV_V_V_M4 undef $v16m4, $v4m4, $noreg, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     $x0 = PseudoVSETVLI  $x10, 66, implicit-def $vl, implicit-def $vtype
-    early-clobber $v4m4 = PseudoVQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3, 1, implicit $vl, implicit $vtype
+    early-clobber $v4m4 = PseudoSF_VQMACCUS_2x8x2_M4 renamable $v4m4, killed renamable $v13, killed renamable $v16m4, $noreg, 3, 1, implicit $vl, implicit $vtype
     $v16m4 = COPY renamable $v4m4
 ...
 ---
@@ -306,10 +306,10 @@ body:             |
     ; CHECK: liveins: $x2, $x10, $v8, $v13, $v4m4, $v16m2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x0 = PseudoVSETVLI $x10, 65 /* e8, m2, ta, mu */, implicit-def $vl, implicit-def $vtype
-    ; CHECK-NEXT: early-clobber $v4m4 = PseudoVQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
+    ; CHECK-NEXT: early-clobber $v4m4 = PseudoSF_VQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3 /* e8 */, 1 /* ta, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $v16m4 = VMV4R_V $v4m4, implicit $vtype
     $x0 = PseudoVSETVLI  $x10, 65, implicit-def $vl, implicit-def $vtype
-    early-clobber $v4m4 = PseudoVQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3, 1, implicit $vl, implicit $vtype
+    early-clobber $v4m4 = PseudoSF_VQMACCUS_4x8x4_M2 renamable $v4m4, killed renamable $v13, killed renamable $v16m2, $noreg, 3, 1, implicit $vl, implicit $vtype
     $v16m4 = COPY renamable $v4m4
 ...
 ---
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-reverse-float.ll b/llvm/test/CodeGen/RISCV/rvv/vp-reverse-float.ll
index 4bbd10df5254..9fafc4ac0667 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-reverse-float.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-reverse-float.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfh -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+v,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s
 
 define <vscale x 1 x half> @test_vp_reverse_nxv1f16_masked(<vscale x 1 x half> %src, <vscale x 1 x i1> %mask, i32 zeroext %evl) {
 ; CHECK-LABEL: test_vp_reverse_nxv1f16_masked:
@@ -435,3 +436,177 @@ define <vscale x 32 x half> @test_vp_reverse_nxv32f16(<vscale x 32 x half> %src,
   %dst = call <vscale x 32 x half> @llvm.experimental.vp.reverse.nxv32f16(<vscale x 32 x half> %src, <vscale x 32 x i1> splat (i1 1), i32 %evl)
   ret <vscale x 32 x half> %dst
 }
+
+define <vscale x 1 x bfloat> @test_vp_reverse_nxv1bf16_masked(<vscale x 1 x bfloat> %src, <vscale x 1 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv1bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vid.v v9, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v10, v9, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 1 x bfloat> @llvm.experimental.vp.reverse.nxv1bf16(<vscale x 1 x bfloat> %src, <vscale x 1 x i1> %mask, i32 %evl)
+  ret <vscale x 1 x bfloat> %dst
+}
+
+define <vscale x 1 x bfloat> @test_vp_reverse_nxv1bf16(<vscale x 1 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv1bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a1
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 1 x bfloat> @llvm.experimental.vp.reverse.nxv1bf16(<vscale x 1 x bfloat> %src, <vscale x 1 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 1 x bfloat> %dst
+}
+
+define <vscale x 2 x bfloat> @test_vp_reverse_nxv2bf16_masked(<vscale x 2 x bfloat> %src, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv2bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v9, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v10, v9, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 2 x bfloat> @llvm.experimental.vp.reverse.nxv2bf16(<vscale x 2 x bfloat> %src, <vscale x 2 x i1> %mask, i32 %evl)
+  ret <vscale x 2 x bfloat> %dst
+}
+
+define <vscale x 2 x bfloat> @test_vp_reverse_nxv2bf16(<vscale x 2 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a1
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 2 x bfloat> @llvm.experimental.vp.reverse.nxv2bf16(<vscale x 2 x bfloat> %src, <vscale x 2 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 2 x bfloat> %dst
+}
+
+define <vscale x 4 x bfloat> @test_vp_reverse_nxv4bf16_masked(<vscale x 4 x bfloat> %src, <vscale x 4 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv4bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v10, v9, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v9, v8, v10, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 4 x bfloat> @llvm.experimental.vp.reverse.nxv4bf16(<vscale x 4 x bfloat> %src, <vscale x 4 x i1> %mask, i32 %evl)
+  ret <vscale x 4 x bfloat> %dst
+}
+
+define <vscale x 4 x bfloat> @test_vp_reverse_nxv4bf16(<vscale x 4 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv4bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vx v10, v9, a1
+; CHECK-NEXT:    vrgather.vv v9, v8, v10
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 4 x bfloat> @llvm.experimental.vp.reverse.nxv4bf16(<vscale x 4 x bfloat> %src, <vscale x 4 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 4 x bfloat> %dst
+}
+
+define <vscale x 8 x bfloat> @test_vp_reverse_nxv8bf16_masked(<vscale x 8 x bfloat> %src, <vscale x 8 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv8bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v12, v10, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v10, v8, v12, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 8 x bfloat> @llvm.experimental.vp.reverse.nxv8bf16(<vscale x 8 x bfloat> %src, <vscale x 8 x i1> %mask, i32 %evl)
+  ret <vscale x 8 x bfloat> %dst
+}
+
+define <vscale x 8 x bfloat> @test_vp_reverse_nxv8bf16(<vscale x 8 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv8bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vx v12, v10, a1
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 8 x bfloat> @llvm.experimental.vp.reverse.nxv8bf16(<vscale x 8 x bfloat> %src, <vscale x 8 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 8 x bfloat> %dst
+}
+
+define <vscale x 16 x bfloat> @test_vp_reverse_nxv16bf16_masked(<vscale x 16 x bfloat> %src, <vscale x 16 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv16bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v12, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v16, v12, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v12, v8, v16, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 16 x bfloat> @llvm.experimental.vp.reverse.nxv16bf16(<vscale x 16 x bfloat> %src, <vscale x 16 x i1> %mask, i32 %evl)
+  ret <vscale x 16 x bfloat> %dst
+}
+
+define <vscale x 16 x bfloat> @test_vp_reverse_nxv16bf16(<vscale x 16 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv16bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    vrsub.vx v16, v12, a1
+; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 16 x bfloat> @llvm.experimental.vp.reverse.nxv16bf16(<vscale x 16 x bfloat> %src, <vscale x 16 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 16 x bfloat> %dst
+}
+
+define <vscale x 32 x bfloat> @test_vp_reverse_nxv32bf16_masked(<vscale x 32 x bfloat> %src, <vscale x 32 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv32bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vid.v v16, v0.t
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vrsub.vx v24, v16, a0, v0.t
+; CHECK-NEXT:    vrgather.vv v16, v8, v24, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+  %dst = call <vscale x 32 x bfloat> @llvm.experimental.vp.reverse.nxv32bf16(<vscale x 32 x bfloat> %src, <vscale x 32 x i1> %mask, i32 %evl)
+  ret <vscale x 32 x bfloat> %dst
+}
+
+define <vscale x 32 x bfloat> @test_vp_reverse_nxv32bf16(<vscale x 32 x bfloat> %src, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_reverse_nxv32bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a1, a0, -1
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vid.v v16
+; CHECK-NEXT:    vrsub.vx v24, v16, a1
+; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+
+  %dst = call <vscale x 32 x bfloat> @llvm.experimental.vp.reverse.nxv32bf16(<vscale x 32 x bfloat> %src, <vscale x 32 x i1> splat (i1 1), i32 %evl)
+  ret <vscale x 32 x bfloat> %dst
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
index 745cec4e7c4f..477f1daf10c2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice-mask-fixed-vectors.ll
@@ -19,15 +19,15 @@ define <2 x i1> @test_vp_splice_v2i1(<2 x i1> %va, <2 x i1> %vb, i32 zeroext %ev
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 1
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -43,15 +43,15 @@ define <2 x i1> @test_vp_splice_v2i1_negative_offset(<2 x i1> %va, <2 x i1> %vb,
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf8, ta, ma
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 -5, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 -1, <2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -67,16 +67,16 @@ define <2 x i1> @test_vp_splice_v2i1_masked(<2 x i1> %va, <2 x i1> %vb, <2 x i1>
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 1, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf8, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 5, <2 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <2 x i1> @llvm.experimental.vp.splice.v2i1(<2 x i1> %va, <2 x i1> %vb, i32 1, <2 x i1> %mask, i32 %evla, i32 %evlb)
   ret <2 x i1> %v
 }
 
@@ -92,15 +92,15 @@ define <4 x i1> @test_vp_splice_v4i1(<4 x i1> %va, <4 x i1> %vb, i32 zeroext %ev
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v9, 5
+; CHECK-NEXT:    vslidedown.vi v9, v9, 3
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a0
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
@@ -116,15 +116,15 @@ define <4 x i1> @test_vp_splice_v4i1_negative_offset(<4 x i1> %va, <4 x i1> %vb,
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v9, v10, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
-; CHECK-NEXT:    vsetivli zero, 5, e8, mf4, ta, ma
+; CHECK-NEXT:    addi a0, a0, -3
+; CHECK-NEXT:    vsetivli zero, 3, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v9, v9, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v9, v8, 5
+; CHECK-NEXT:    vslideup.vi v9, v8, 3
 ; CHECK-NEXT:    vmsne.vi v0, v9, 0
 ; CHECK-NEXT:    ret
 
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 -5, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 -3, <4 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
@@ -140,16 +140,16 @@ define <4 x i1> @test_vp_splice_v4i1_masked(<4 x i1> %va, <4 x i1> %vb, <4 x i1>
 ; CHECK-NEXT:    vmv.v.i v11, 0
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v10, v11, 1, v0
-; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    addi a0, a0, -3
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vsetvli zero, a0, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v10, 5, v0.t
+; CHECK-NEXT:    vslidedown.vi v10, v10, 3, v0.t
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
 ; CHECK-NEXT:    vslideup.vx v10, v8, a0, v0.t
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0, v0.t
 ; CHECK-NEXT:    ret
-  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 5, <4 x i1> %mask, i32 %evla, i32 %evlb)
+  %v = call <4 x i1> @llvm.experimental.vp.splice.v4i1(<4 x i1> %va, <4 x i1> %vb, i32 3, <4 x i1> %mask, i32 %evla, i32 %evlb)
   ret <4 x i1> %v
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll b/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
index ffeb49398910..9c8c5da75ff7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-splice.ll
@@ -1,18 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v -verify-machineinstrs \
-; RUN:   < %s | FileCheck %s
-
-declare <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i64>, i32, <vscale x 2 x i1>, i32, i32)
-
-declare <vscale x 1 x i64> @llvm.experimental.vp.splice.nxv1i64(<vscale x 1 x i64>, <vscale x 1 x i64>, i32, <vscale x 1 x i1>, i32, i32)
-declare <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32>, <vscale x 2 x i32>, i32, <vscale x 2 x i1>, i32, i32)
-declare <vscale x 4 x i16> @llvm.experimental.vp.splice.nxv4i16(<vscale x 4 x i16>, <vscale x 4 x i16>, i32, <vscale x 4 x i1>, i32, i32)
-declare <vscale x 8 x i8> @llvm.experimental.vp.splice.nxv8i8(<vscale x 8 x i8>, <vscale x 8 x i8>, i32, <vscale x 8 x i1>, i32, i32)
-
-declare <vscale x 1 x double> @llvm.experimental.vp.splice.nxv1f64(<vscale x 1 x double>, <vscale x 1 x double>, i32, <vscale x 1 x i1>, i32, i32)
-declare <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>, i32, <vscale x 2 x i1>, i32, i32)
-
-declare <vscale x 16 x i64> @llvm.experimental.vp.splice.nxv16i64(<vscale x 16 x i64>, <vscale x 16 x i64>, i32, <vscale x 16 x i1>, i32, i32)
+; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zfh,+zfbfmin,+zvfh,+zvfbfmin -verify-machineinstrs \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK,ZVFH
+; RUN: llc -mtriple riscv64 -mattr=+f,+d,+v,+zfh,+zfbfmin,+zvfhmin,+zvfbfmin -verify-machineinstrs \
+; RUN:   < %s | FileCheck %s --check-prefixes=CHECK,ZVFHMIN
 
 define <vscale x 2 x i64> @test_vp_splice_nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv2i64:
@@ -40,6 +30,16 @@ define <vscale x 2 x i64> @test_vp_splice_nxv2i64_negative_offset(<vscale x 2 x
   ret <vscale x 2 x i64> %v
 }
 
+define <vscale x 2 x i64> @test_vp_splice_nxv2i64_zero_offset(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2i64_zero_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, i32 0, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <vscale x 2 x i64> %v
+}
+
 define <vscale x 2 x i64> @test_vp_splice_nxv2i64_masked(<vscale x 2 x i64> %va, <vscale x 2 x i64> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
 ; CHECK-LABEL: test_vp_splice_nxv2i64_masked:
 ; CHECK:       # %bb.0:
@@ -295,10 +295,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vs
 ; CHECK-NEXT:    addi a5, a5, -1
 ; CHECK-NEXT:    slli a1, a4, 3
 ; CHECK-NEXT:    mv a7, a2
-; CHECK-NEXT:    bltu a2, a5, .LBB21_2
+; CHECK-NEXT:    bltu a2, a5, .LBB22_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a7, a5
-; CHECK-NEXT:  .LBB21_2:
+; CHECK-NEXT:  .LBB22_2:
 ; CHECK-NEXT:    addi sp, sp, -80
 ; CHECK-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
@@ -311,10 +311,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vs
 ; CHECK-NEXT:    slli a7, a7, 3
 ; CHECK-NEXT:    addi a6, sp, 64
 ; CHECK-NEXT:    mv t0, a2
-; CHECK-NEXT:    bltu a2, a4, .LBB21_4
+; CHECK-NEXT:    bltu a2, a4, .LBB22_4
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    mv t0, a4
-; CHECK-NEXT:  .LBB21_4:
+; CHECK-NEXT:  .LBB22_4:
 ; CHECK-NEXT:    vl8re64.v v24, (a5)
 ; CHECK-NEXT:    add a5, a6, a7
 ; CHECK-NEXT:    vl8re64.v v0, (a0)
@@ -328,10 +328,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64(<vscale x 16 x i64> %va, <vs
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vse64.v v16, (a6)
 ; CHECK-NEXT:    mv a0, a3
-; CHECK-NEXT:    bltu a3, a4, .LBB21_6
+; CHECK-NEXT:    bltu a3, a4, .LBB22_6
 ; CHECK-NEXT:  # %bb.5:
 ; CHECK-NEXT:    mv a0, a4
-; CHECK-NEXT:  .LBB21_6:
+; CHECK-NEXT:  .LBB22_6:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vse64.v v0, (a5)
 ; CHECK-NEXT:    sub a2, a3, a4
@@ -363,10 +363,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
 ; CHECK-NEXT:    addi a6, a6, -1
 ; CHECK-NEXT:    slli a1, a5, 3
 ; CHECK-NEXT:    mv a4, a2
-; CHECK-NEXT:    bltu a2, a6, .LBB22_2
+; CHECK-NEXT:    bltu a2, a6, .LBB23_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a4, a6
-; CHECK-NEXT:  .LBB22_2:
+; CHECK-NEXT:  .LBB23_2:
 ; CHECK-NEXT:    addi sp, sp, -80
 ; CHECK-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
@@ -379,10 +379,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
 ; CHECK-NEXT:    slli a4, a4, 3
 ; CHECK-NEXT:    addi a7, sp, 64
 ; CHECK-NEXT:    mv t0, a2
-; CHECK-NEXT:    bltu a2, a5, .LBB22_4
+; CHECK-NEXT:    bltu a2, a5, .LBB23_4
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    mv t0, a5
-; CHECK-NEXT:  .LBB22_4:
+; CHECK-NEXT:  .LBB23_4:
 ; CHECK-NEXT:    vl8re64.v v24, (a6)
 ; CHECK-NEXT:    add a6, a7, a4
 ; CHECK-NEXT:    vl8re64.v v0, (a0)
@@ -396,10 +396,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vse64.v v16, (a7)
 ; CHECK-NEXT:    mv a0, a3
-; CHECK-NEXT:    bltu a3, a5, .LBB22_6
+; CHECK-NEXT:    bltu a3, a5, .LBB23_6
 ; CHECK-NEXT:  # %bb.5:
 ; CHECK-NEXT:    mv a0, a5
-; CHECK-NEXT:  .LBB22_6:
+; CHECK-NEXT:  .LBB23_6:
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m8, ta, ma
 ; CHECK-NEXT:    vse64.v v0, (a6)
 ; CHECK-NEXT:    sub a2, a3, a5
@@ -410,10 +410,10 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
 ; CHECK-NEXT:    li a3, 8
 ; CHECK-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; CHECK-NEXT:    vse64.v v24, (a5)
-; CHECK-NEXT:    bltu a4, a3, .LBB22_8
+; CHECK-NEXT:    bltu a4, a3, .LBB23_8
 ; CHECK-NEXT:  # %bb.7:
 ; CHECK-NEXT:    li a4, 8
-; CHECK-NEXT:  .LBB22_8:
+; CHECK-NEXT:  .LBB23_8:
 ; CHECK-NEXT:    sub a2, a6, a4
 ; CHECK-NEXT:    add a1, a2, a1
 ; CHECK-NEXT:    vle64.v v16, (a1)
@@ -427,3 +427,151 @@ define <vscale x 16 x i64> @test_vp_splice_nxv16i64_negative_offset(<vscale x 16
   %v = call <vscale x 16 x i64> @llvm.experimental.vp.splice.nxv16i64(<vscale x 16 x i64> %va, <vscale x 16 x i64> %vb, i32 -1, <vscale x 16 x i1> splat (i1 1), i32 %evla, i32 %evlb)
   ret <vscale x 16 x i64> %v
 }
+
+define <vscale x 2 x half> @test_vp_splice_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2f16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <vscale x 2 x half> %v
+}
+
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_negative_offset(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2f16_negative_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetivli zero, 5, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <vscale x 2 x half> %v
+}
+
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_masked(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2f16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  ret <vscale x 2 x half> %v
+}
+
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2bf16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_negative_offset(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2bf16_negative_offset:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetivli zero, 5, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 5
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 -5, <vscale x 2 x i1> splat (i1 1), i32 %evla, i32 %evlb)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_masked(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evla, i32 zeroext %evlb) {
+; CHECK-LABEL: test_vp_splice_nxv2bf16_masked:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi a0, a0, -5
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 5, v0.t
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 5, <vscale x 2 x i1> %mask, i32 %evla, i32 %evlb)
+  ret <vscale x 2 x bfloat> %v
+}
+
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_nxv2i32_with_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslide1up.vx v9, v8, a0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %va = insertelement <vscale x 2 x i32> poison, i32 %first, i32 0
+  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %va, <vscale x 2 x i32> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
+  ret <vscale x 2 x i32> %v
+}
+
+define <vscale x 2 x i32> @test_vp_splice_nxv2i32_with_splat_firstelt(i32 %first, <vscale x 2 x i32> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_nxv2i32_with_splat_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vslide1up.vx v9, v8, a0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %ins = insertelement <vscale x 2 x i32> poison, i32 %first, i32 0
+  %splat = shufflevector <vscale x 2 x i32> %ins, <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+  %v = call <vscale x 2 x i32> @llvm.experimental.vp.splice.nxv2i32(<vscale x 2 x i32> %splat, <vscale x 2 x i32> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
+  ret <vscale x 2 x i32> %v
+}
+
+define <vscale x 2 x float> @test_vp_splice_nxv2f32_with_firstelt(float %first, <vscale x 2 x float> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_nxv2f32_with_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vfslide1up.vf v9, v8, fa0, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v9
+; CHECK-NEXT:    ret
+  %va = insertelement <vscale x 2 x float> poison, float %first, i32 0
+  %v = call <vscale x 2 x float> @llvm.experimental.vp.splice.nxv2f32(<vscale x 2 x float> %va, <vscale x 2 x float> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
+  ret <vscale x 2 x float> %v
+}
+
+define <vscale x 2 x half> @test_vp_splice_nxv2f16_with_firstelt(half %first, <vscale x 2 x half> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; ZVFH-LABEL: test_vp_splice_nxv2f16_with_firstelt:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFH-NEXT:    vfslide1up.vf v9, v8, fa0, v0.t
+; ZVFH-NEXT:    vmv1r.v v8, v9
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: test_vp_splice_nxv2f16_with_firstelt:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    fmv.x.h a1, fa0
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslide1up.vx v9, v8, a1, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v8, v9
+; ZVFHMIN-NEXT:    ret
+  %va = insertelement <vscale x 2 x half> poison, half %first, i32 0
+  %v = call <vscale x 2 x half> @llvm.experimental.vp.splice.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
+  ret <vscale x 2 x half> %v
+}
+
+define <vscale x 2 x bfloat> @test_vp_splice_nxv2bf16_with_firstelt(bfloat %first, <vscale x 2 x bfloat> %vb, <vscale x 2 x i1> %mask, i32 zeroext %evl) {
+; CHECK-LABEL: test_vp_splice_nxv2bf16_with_firstelt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    fmv.x.h a1, fa0
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslide1up.vx v9, v8, a1, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v9
+; CHECK-NEXT:    ret
+  %va = insertelement <vscale x 2 x bfloat> poison, bfloat %first, i32 0
+  %v = call <vscale x 2 x bfloat> @llvm.experimental.vp.splice.nxv2bf16(<vscale x 2 x bfloat> %va, <vscale x 2 x bfloat> %vb, i32 0, <vscale x 2 x i1> %mask, i32 1, i32 %evl)
+  ret <vscale x 2 x bfloat> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
index 186815405227..7fb822d20f89 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
@@ -84,15 +84,11 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 4
   %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
-  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
-  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
-  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+  %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
 
   %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
   %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
@@ -214,28 +210,15 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 8
   %wide.masked.load = call <vscale x 16 x i32> @llvm.vp.load.nxv16i32.p0(ptr %ptr, <vscale x 16 x i1> splat (i1 true), i32 %rvl)
-  %d0 = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 1
-  %d1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.0)
-  %d1.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 0
-  %d1.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 1
-  %d2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.1)
-  %d2.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 0
-  %d2.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 1
-
-  %d3 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 0
-  %t4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 1
-  %d4 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.1)
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 0
-  %t6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 1
-  %d5 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.0)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 0
-  %t5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 1
-  %d6 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.1)
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 0
-  %t7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 1
+  %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave8.nxv16i32(<vscale x 16 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
+  %t4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 4
+  %t5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 5
+  %t6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 6
+  %t7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 7
 
   %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
   %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
@@ -323,10 +306,8 @@ define void @store_factor4_v2(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, pt
 ; RV64-NEXT:    vsseg4e32.v v8, (a0)
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 8
-  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
-  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
-  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
-  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec2, ptr %ptr, <vscale x 4 x i1> splat (i1 true), i32 %rvl)
+  %interleaved.vec = call <vscale x 4 x i32> @llvm.vector.interleave4.nxv4i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1)
+  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec, ptr %ptr, <vscale x 4 x i1> splat (i1 true), i32 %rvl)
   ret void
 }
 
@@ -430,14 +411,8 @@ define void @store_factor8_v2(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, pt
 ; RV64-NEXT:    vsseg8e32.v v8, (a0)
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 8
-  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
-  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
-  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
-  %interleaved.vec3 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
-  %interleaved.vec4 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
-  %interleaved.vec5 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec3, <vscale x 2 x i32> %interleaved.vec4)
-  %interleaved.vec6 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %interleaved.vec2, <vscale x 4 x i32> %interleaved.vec5)
-  call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %interleaved.vec6, ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
+  %interleaved.vec = call <vscale x 8 x i32> @llvm.vector.interleave8.nxv8i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1)
+  call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %interleaved.vec, ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
   ret void
 }
 
@@ -485,19 +460,13 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2
 ; RV64-NEXT:    vlseg4e32.v v8, (a0), v0.t
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 4
-  %interleaved.mask0 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
-  %interleaved.mask1 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
-  %interleaved.mask2 = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %interleaved.mask0, <vscale x 4 x i1> %interleaved.mask1)
-  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %interleaved.mask2, i32 %rvl)
-  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
-  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
-  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+  %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> %interleaved.mask, i32 %rvl)
+  %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %t0 = extractvalue  { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+  %t1 = extractvalue  { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+  %t2 = extractvalue  { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+  %t3 = extractvalue  { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
 
   %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
   %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
@@ -567,37 +536,37 @@ define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v0
+; RV32-NEXT:    slli a2, a1, 1
 ; RV32-NEXT:    vmv.v.i v9, 0
-; RV32-NEXT:    li a2, -1
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    li a1, -1
+; RV32-NEXT:    vmerge.vim v10, v9, 1, v0
+; RV32-NEXT:    vwaddu.vv v11, v10, v10
+; RV32-NEXT:    vwmaccu.vx v11, a1, v10
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
 ; RV32-NEXT:    vmv.v.i v10, 0
+; RV32-NEXT:    srli a1, a1, 2
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmerge.vim v11, v9, 1, v0
-; RV32-NEXT:    vwaddu.vv v12, v11, v11
-; RV32-NEXT:    vwmaccu.vx v12, a2, v11
-; RV32-NEXT:    csrr a2, vlenb
-; RV32-NEXT:    srli a2, a2, 2
-; RV32-NEXT:    vmsne.vi v0, v12, 0
+; RV32-NEXT:    vmsne.vi v0, v11, 0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v11, v12, a2
+; RV32-NEXT:    vslidedown.vx v11, v11, a1
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
 ; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v11, 0
-; RV32-NEXT:    slli a3, a1, 1
 ; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
-; RV32-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v9, a2
-; RV32-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v9, a1
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
 ; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vnsrl.wx v13, v10, a1
 ; RV32-NEXT:    vmv.x.s a1, v10
 ; RV32-NEXT:    vnsrl.wi v12, v10, 0
-; RV32-NEXT:    srli a3, a3, 1
+; RV32-NEXT:    srli a2, a2, 1
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    vsetvli zero, a3, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
 ; RV32-NEXT:    vsseg2e32.v v12, (a0), v0.t
 ; RV32-NEXT:    mv a0, a1
 ; RV32-NEXT:    ret
@@ -674,205 +643,44 @@ define void @masked_store_factor4_v2(<vscale x 1 x i1> %mask, <vscale x 1 x i32>
 ; RV64-NEXT:    vsseg4e32.v v8, (a0), v0.t
 ; RV64-NEXT:    ret
   %rvl = mul i32 %evl, 4
-  %interleaved.mask0 = call <vscale x 2 x i1> @llvm.vector.interleave2.nxv2i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
-  %interleaved.mask1 = call <vscale x 2 x i1> @llvm.vector.interleave2.nxv2i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
-  %interleaved.mask2 = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %interleaved.mask0, <vscale x 2 x i1> %interleaved.mask1)
-  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
-  %interleaved.vec1 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v1, <vscale x 1 x i32> %v1)
-  %interleaved.vec2 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %interleaved.vec1)
-  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec2, ptr %ptr, <vscale x 4 x i1> %interleaved.mask2, i32 %rvl)
+  %interleaved.mask = call <vscale x 4 x i1> @llvm.vector.interleave4.nxv4i1(<vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask, <vscale x 1 x i1> %mask)
+  %interleaved.vec = call <vscale x 4 x i32> @llvm.vector.interleave4.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1, <vscale x 1 x i32> %v0, <vscale x 1 x i32> %v1)
+  call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> %interleaved.vec, ptr %ptr, <vscale x 4 x i1> %interleaved.mask, i32 %rvl)
   ret void
 }
 
 ; Negative tests
 
-; We should not transform this function because the deinterleave tree is not in a desired form.
-define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @incorrect_extract_value_index(ptr %ptr, i32 %evl) {
-; RV32-LABEL: incorrect_extract_value_index:
-; RV32:       # %bb.0:
-; RV32-NEXT:    slli a1, a1, 2
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV32-NEXT:    vnsrl.wi v12, v8, 0
-; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV32-NEXT:    vnsrl.wx v9, v12, a0
-; RV32-NEXT:    vnsrl.wi v8, v12, 0
-; RV32-NEXT:    vmv.v.v v10, v9
-; RV32-NEXT:    vmv.v.v v11, v9
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: incorrect_extract_value_index:
-; RV64:       # %bb.0:
-; RV64-NEXT:    slli a1, a1, 34
-; RV64-NEXT:    srli a1, a1, 32
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vle32.v v8, (a0)
-; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV64-NEXT:    vnsrl.wi v12, v8, 0
-; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV64-NEXT:    vnsrl.wx v9, v12, a0
-; RV64-NEXT:    vnsrl.wi v8, v12, 0
-; RV64-NEXT:    vmv.v.v v10, v9
-; RV64-NEXT:    vmv.v.v v11, v9
-; RV64-NEXT:    ret
-  %rvl = mul i32 %evl, 4
-  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
-  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
-  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
-
-  %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
-  %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
-  %res2 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res1, <vscale x 2 x i32> %t2, 2
-  %res3 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res2, <vscale x 2 x i32> %t3, 3
-  ret { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res3
-}
-
-; We should not transform this function because the expression is not a balanced tree.
-define {<vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32>} @not_balanced_load_tree(ptr %ptr, i32 %evl) {
-; RV32-LABEL: not_balanced_load_tree:
-; RV32:       # %bb.0:
-; RV32-NEXT:    slli a1, a1, 2
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vle32.v v12, (a0)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV32-NEXT:    vnsrl.wx v8, v12, a0
-; RV32-NEXT:    vnsrl.wi v16, v12, 0
-; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV32-NEXT:    vnsrl.wi v10, v16, 0
-; RV32-NEXT:    vnsrl.wx v11, v16, a0
-; RV32-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
-; RV32-NEXT:    vnsrl.wx v12, v11, a0
-; RV32-NEXT:    vnsrl.wi v11, v11, 0
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: not_balanced_load_tree:
-; RV64:       # %bb.0:
-; RV64-NEXT:    slli a1, a1, 34
-; RV64-NEXT:    srli a1, a1, 32
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vle32.v v12, (a0)
-; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV64-NEXT:    vnsrl.wx v8, v12, a0
-; RV64-NEXT:    vnsrl.wi v16, v12, 0
-; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV64-NEXT:    vnsrl.wi v10, v16, 0
-; RV64-NEXT:    vnsrl.wx v11, v16, a0
-; RV64-NEXT:    vsetvli a1, zero, e32, mf2, ta, ma
-; RV64-NEXT:    vnsrl.wx v12, v11, a0
-; RV64-NEXT:    vnsrl.wi v11, v11, 0
-; RV64-NEXT:    ret
-  %rvl = mul i32 %evl, 4
-  %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
-  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %t0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
-  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
-  %d1.1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
-  %d2 = call { <vscale x 1 x i32>, <vscale x 1 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 2 x i32> %d1.1)
-  %t2 = extractvalue { <vscale x 1 x i32>, <vscale x 1 x i32> } %d2, 0
-  %t3 = extractvalue { <vscale x 1 x i32>, <vscale x 1 x i32> } %d2, 1
-
-  %res0 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } poison, <vscale x 4 x i32> %t0, 0
-  %res1 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res0, <vscale x 2 x i32> %t1, 1
-  %res2 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res1, <vscale x 1 x i32> %t2, 2
-  %res3 = insertvalue { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res2, <vscale x 1 x i32> %t3, 3
-  ret { <vscale x 4 x i32>, <vscale x 2 x i32>, <vscale x 1 x i32>, <vscale x 1 x i32> } %res3
-}
-
-define void @not_balanced_store_tree(<vscale x 1 x i32> %v0, <vscale x 2 x i32> %v1, <vscale x 4 x i32> %v2, ptr %ptr, i32 %evl) {
-; RV32-LABEL: not_balanced_store_tree:
-; RV32:       # %bb.0:
-; RV32-NEXT:    slli a1, a1, 2
-; RV32-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
-; RV32-NEXT:    vwaddu.vv v12, v8, v8
-; RV32-NEXT:    li a2, -1
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    vwmaccu.vx v12, a2, v8
-; RV32-NEXT:    srli a3, a3, 3
-; RV32-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
-; RV32-NEXT:    vslidedown.vx v8, v12, a3
-; RV32-NEXT:    vslideup.vx v12, v8, a3
-; RV32-NEXT:    vwaddu.vv v16, v12, v9
-; RV32-NEXT:    vwmaccu.vx v16, a2, v9
-; RV32-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
-; RV32-NEXT:    vwaddu.vv v12, v16, v10
-; RV32-NEXT:    vwmaccu.vx v12, a2, v10
-; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV32-NEXT:    vse32.v v12, (a0)
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: not_balanced_store_tree:
-; RV64:       # %bb.0:
-; RV64-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
-; RV64-NEXT:    vwaddu.vv v12, v8, v8
-; RV64-NEXT:    li a2, -1
-; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a1, a1, 34
-; RV64-NEXT:    vwmaccu.vx v12, a2, v8
-; RV64-NEXT:    srli a3, a3, 3
-; RV64-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
-; RV64-NEXT:    vslidedown.vx v8, v12, a3
-; RV64-NEXT:    vslideup.vx v12, v8, a3
-; RV64-NEXT:    vwaddu.vv v16, v12, v9
-; RV64-NEXT:    vwmaccu.vx v16, a2, v9
-; RV64-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
-; RV64-NEXT:    vwaddu.vv v12, v16, v10
-; RV64-NEXT:    vwmaccu.vx v12, a2, v10
-; RV64-NEXT:    srli a1, a1, 32
-; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
-; RV64-NEXT:    vse32.v v12, (a0)
-; RV64-NEXT:    ret
-  %rvl = mul i32 %evl, 4
-  %interleaved.vec0 = call <vscale x 2 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 1 x i32> %v0, <vscale x 1 x i32> %v0)
-  %interleaved.vec1 = call <vscale x 4 x i32> @llvm.vector.interleave2.nxv2i32(<vscale x 2 x i32> %interleaved.vec0, <vscale x 2 x i32> %v1)
-  %interleaved.vec2 = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv4i32(<vscale x 4 x i32> %interleaved.vec1, <vscale x 4 x i32> %v2)
-  call void @llvm.vp.store.nxv8i32.p0(<vscale x 8 x i32> %interleaved.vec2, ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
-  ret void
-}
-
 define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1> %mask0, <vscale x 2 x i1> %mask1, ptr %ptr, i32 %evl) {
 ; RV32-LABEL: not_same_mask:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v9, v0
 ; RV32-NEXT:    vmv1r.v v0, v8
+; RV32-NEXT:    slli a1, a1, 1
 ; RV32-NEXT:    vmv.v.i v8, 0
 ; RV32-NEXT:    li a2, -1
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vmv.v.i v10, 0
-; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmerge.vim v11, v8, 1, v0
+; RV32-NEXT:    vmerge.vim v10, v8, 1, v0
 ; RV32-NEXT:    vmv1r.v v0, v9
 ; RV32-NEXT:    vmerge.vim v9, v8, 1, v0
-; RV32-NEXT:    vwaddu.vv v12, v9, v11
-; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    vwaddu.vv v11, v9, v10
+; RV32-NEXT:    vwmaccu.vx v11, a2, v10
 ; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vmv.v.i v9, 0
 ; RV32-NEXT:    srli a2, a2, 2
-; RV32-NEXT:    vmsne.vi v0, v12, 0
-; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v9, v12, a2
-; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; RV32-NEXT:    vmsne.vi v0, v9, 0
-; RV32-NEXT:    slli a1, a1, 1
-; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vmsne.vi v0, v11, 0
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v8, a2
+; RV32-NEXT:    vslidedown.vx v10, v11, a2
 ; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
+; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
+; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v9, v8, a2
+; RV32-NEXT:    vmsne.vi v0, v9, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
 ; RV32-NEXT:    li a0, 32
 ; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
@@ -931,48 +739,58 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1>
 define {<vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>} @invalid_evl(ptr %ptr, i32 %evl) {
 ; RV32-LABEL: invalid_evl:
 ; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    slli a2, a2, 2
+; RV32-NEXT:    sub sp, sp, a2
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb
 ; RV32-NEXT:    ori a1, a1, 1
 ; RV32-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; RV32-NEXT:    vle32.v v8, (a0)
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV32-NEXT:    vnsrl.wx v12, v8, a0
-; RV32-NEXT:    vnsrl.wi v14, v8, 0
+; RV32-NEXT:    addi a0, sp, 16
+; RV32-NEXT:    vs4r.v v8, (a0)
 ; RV32-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV32-NEXT:    vnsrl.wx v10, v14, a0
-; RV32-NEXT:    vnsrl.wi v8, v14, 0
-; RV32-NEXT:    vnsrl.wx v11, v12, a0
-; RV32-NEXT:    vnsrl.wi v9, v12, 0
+; RV32-NEXT:    vlseg4e32.v v8, (a0)
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 2
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    .cfi_def_cfa sp, 16
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    .cfi_def_cfa_offset 0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: invalid_evl:
 ; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    slli a2, a2, 2
+; RV64-NEXT:    sub sp, sp, a2
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x04, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 4 * vlenb
 ; RV64-NEXT:    ori a1, a1, 1
 ; RV64-NEXT:    slli a1, a1, 32
 ; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; RV64-NEXT:    vle32.v v8, (a0)
-; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; RV64-NEXT:    vnsrl.wx v12, v8, a0
-; RV64-NEXT:    vnsrl.wi v14, v8, 0
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vs4r.v v8, (a0)
 ; RV64-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; RV64-NEXT:    vnsrl.wx v10, v14, a0
-; RV64-NEXT:    vnsrl.wi v8, v14, 0
-; RV64-NEXT:    vnsrl.wx v11, v12, a0
-; RV64-NEXT:    vnsrl.wi v9, v12, 0
+; RV64-NEXT:    vlseg4e32.v v8, (a0)
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 2
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    .cfi_def_cfa sp, 16
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
 ; RV64-NEXT:    ret
   %rvl = or i32 %evl, 1
   %wide.masked.load = call <vscale x 8 x i32> @llvm.vp.load.nxv8i32.p0(ptr %ptr, <vscale x 8 x i1> splat (i1 true), i32 %rvl)
-  %d0 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
-  %d0.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d0, 1
-  %d1 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 0
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d1, 1
-  %d2 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d0.1)
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 0
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d2, 1
+  %d = call { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave4.nxv8i32(<vscale x 8 x i32> %wide.masked.load)
+  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 0
+  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 1
+  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 2
+  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %d, 3
 
   %res0 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> %t0, 0
   %res1 = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } %res0, <vscale x 2 x i32> %t1, 1
diff --git a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
index 75c00e406b4f..809276c751af 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vrgatherei16.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vrgatherei16.vv.nxv1i8(
@@ -1449,3 +1449,208 @@ entry:
 
   ret <vscale x 8 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vrgatherei16_vv_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv1bf16(
+    <vscale x 1 x bfloat> undef,
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i16> %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    <vscale x 1 x i16> %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vrgatherei16_vv_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv2bf16(
+    <vscale x 2 x bfloat> undef,
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i16> %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    <vscale x 2 x i16> %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vrgatherei16_vv_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v9
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv4bf16(
+    <vscale x 4 x bfloat> undef,
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i16> %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v9, v10, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    <vscale x 4 x i16> %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vrgatherei16_vv_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v10
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv8bf16(
+    <vscale x 8 x bfloat> undef,
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i16> %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v10, v12, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    <vscale x 8 x i16> %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vrgatherei16_vv_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v12
+; CHECK-NEXT:    vmv.v.v v8, v16
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv16bf16(
+    <vscale x 16 x bfloat> undef,
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i16> %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v12, v16, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    <vscale x 16 x i16> %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 32 x bfloat> @intrinsic_vrgatherei16_vv_nxv32bf16_nxv32bf16(<vscale x 32 x bfloat> %0, <vscale x 32 x i16> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_vv_nxv32bf16_nxv32bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v24, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v24
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.nxv32bf16(
+    <vscale x 32 x bfloat> undef,
+    <vscale x 32 x bfloat> %0,
+    <vscale x 32 x i16> %1,
+    iXLen %2)
+
+  ret <vscale x 32 x bfloat> %a
+}
+
+define <vscale x 32 x bfloat> @intrinsic_vrgatherei16_mask_vv_nxv32bf16_nxv32bf16(<vscale x 32 x bfloat> %0, <vscale x 32 x bfloat> %1, <vscale x 32 x i16> %2, <vscale x 32 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vrgatherei16_mask_vv_nxv32bf16_nxv32bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vl8re16.v v24, (a0)
+; CHECK-NEXT:    vsetvli zero, a1, e16, m8, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v24, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 32 x bfloat> @llvm.riscv.vrgatherei16.vv.mask.nxv32bf16(
+    <vscale x 32 x bfloat> %0,
+    <vscale x 32 x bfloat> %1,
+    <vscale x 32 x i16> %2,
+    <vscale x 32 x i1> %3,
+    iXLen %4, iXLen 1)
+
+  ret <vscale x 32 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 8b48dc43eca2..439974d56e38 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -307,9 +307,8 @@ define <vscale x 1 x double> @test16(i64 %avl, double %a, <vscale x 1 x double>
 ; CHECK-LABEL: test16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vsetvli a0, a0, e64, mf2, ta, ma
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vsetvli zero, a0, e64, m1, ta, ma
+; CHECK-NEXT:    vfmv.v.f v9, fa0
 ; CHECK-NEXT:    vfadd.vv v8, v9, v8
 ; CHECK-NEXT:    ret
 entry:
@@ -722,3 +721,35 @@ define i64 @avl_undef2() {
   %1 = tail call i64 @llvm.riscv.vsetvli(i64 poison, i64 2, i64 7)
   ret i64 %1
 }
+
+define i64 @vsetvli_vleff(ptr %s, i64 %evl) {
+; CHECK-LABEL: vsetvli_vleff:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:  .LBB37_1: # %while.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, ma
+; CHECK-NEXT:    vmv1r.v v9, v8
+; CHECK-NEXT:    vle16ff.v v9, (a0)
+; CHECK-NEXT:    csrr a2, vl
+; CHECK-NEXT:    beqz a2, .LBB37_1
+; CHECK-NEXT:  # %bb.2: # %while.end
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+entry:
+  br label %while.cond
+
+while.cond:
+  %new_vl.0 = phi i64 [ 0, %entry ], [ %1, %while.body ]
+  %cmp = icmp eq i64 %new_vl.0, 0
+  br i1 %cmp, label %while.body, label %while.end
+
+while.body:
+  %0 = tail call { <vscale x 4 x i16>, i64 } @llvm.riscv.vleff.nxv4i16.i64(<vscale x 4 x i16> zeroinitializer, ptr %s, i64 %evl)
+  %1 = extractvalue { <vscale x 4 x i16>, i64 } %0, 1
+  br label %while.cond
+
+while.end:
+  ret i64 0
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
index 6bd03eb0c222..fdd30c9a2c77 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.mir
@@ -100,6 +100,10 @@
     ret void
   }
 
+  define void @vsetvli_vleff() {
+    ret void
+  }
+
   declare <vscale x 1 x i64> @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, i64) #1
 
   declare <vscale x 1 x i64> @llvm.riscv.vle.nxv1i64.i64(<vscale x 1 x i64>, ptr nocapture, i64) #4
@@ -622,3 +626,41 @@ body: |
     dead $x0 = PseudoVSETIVLI 1, 208, implicit-def $vl, implicit-def $vtype
     %v:vr = COPY $v8, implicit $vtype
     %x = PseudoVSETVLI %x, 208, implicit-def $vl, implicit-def $vtype
+...
+---
+name:            vsetvli_vleff
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: vsetvli_vleff
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %vl:gpr = COPY $x0
+  ; CHECK-NEXT:   BNE $x0, $x0, %bb.2
+  ; CHECK-NEXT:   PseudoBR %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 0, 200 /* e16, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+  ; CHECK-NEXT:   $noreg, $x0 = PseudoVLE16FF_V_M1 $noreg, $noreg, 0, 4 /* e16 */, 2 /* tu, ma */, implicit $vl, implicit $vtype, implicit-def $vl
+  ; CHECK-NEXT:   %vl:gpr = PseudoReadVL implicit $vl
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   $x10 = COPY %vl
+  ; CHECK-NEXT:   PseudoRET implicit killed $x10
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+
+    %vl:gpr = COPY $x0
+    BNE $x0, $x0, %bb.2
+    PseudoBR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    $noreg, %vl:gpr = PseudoVLE16FF_V_M1 $noreg, $noreg, 0, 4 /* e16 */, 2 /* tu, ma */
+
+  bb.2:
+    $x10 = COPY %vl
+    PseudoRET implicit killed $x10
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
index f0f78c211c4a..22a90cc2c94a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslidedown.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslidedown.nxv1i8(
@@ -2463,3 +2463,343 @@ entry:
 
   ret <vscale x 4 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_mask_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslidedown_mask_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslidedown.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    <vscale x 1 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_mask_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslidedown_mask_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslidedown.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    <vscale x 2 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_mask_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslidedown_mask_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslidedown.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    <vscale x 4 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v10, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_mask_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v10, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v10, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslidedown_mask_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v10, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslidedown.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    <vscale x 8 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_mask_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, mu
+; CHECK-NEXT:    vslidedown.vx v8, v12, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v12, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslidedown_mask_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; CHECK-NEXT:    vslidedown.vi v8, v12, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslidedown.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    <vscale x 16 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
index 8e3c05611bc7..0291207ccdb4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vslideup.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin \
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
 declare <vscale x 1 x i8> @llvm.riscv.vslideup.nxv1i8(
@@ -2463,3 +2463,343 @@ entry:
 
   ret <vscale x 4 x double> %a
 }
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_mask_vx_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2, <vscale x 1 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen %2,
+    <vscale x 1 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 1 x bfloat> @intrinsic_vslideup_mask_vi_nxv1bf16_nxv1bf16(<vscale x 1 x bfloat> %0, <vscale x 1 x bfloat> %1, <vscale x 1 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv1bf16_nxv1bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf4, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 1 x bfloat> @llvm.riscv.vslideup.mask.nxv1bf16(
+    <vscale x 1 x bfloat> %0,
+    <vscale x 1 x bfloat> %1,
+    iXLen 9,
+    <vscale x 1 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 1 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_mask_vx_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2, <vscale x 2 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, mf2, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen %2,
+    <vscale x 2 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 2 x bfloat> @intrinsic_vslideup_mask_vi_nxv2bf16_nxv2bf16(<vscale x 2 x bfloat> %0, <vscale x 2 x bfloat> %1, <vscale x 2 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2bf16_nxv2bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, mf2, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 2 x bfloat> @llvm.riscv.vslideup.mask.nxv2bf16(
+    <vscale x 2 x bfloat> %0,
+    <vscale x 2 x bfloat> %1,
+    iXLen 9,
+    <vscale x 2 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 2 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_mask_vx_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2, <vscale x 4 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m1, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v9, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen %2,
+    <vscale x 4 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v9, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 4 x bfloat> @intrinsic_vslideup_mask_vi_nxv4bf16_nxv4bf16(<vscale x 4 x bfloat> %0, <vscale x 4 x bfloat> %1, <vscale x 4 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4bf16_nxv4bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m1, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v9, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 4 x bfloat> @llvm.riscv.vslideup.mask.nxv4bf16(
+    <vscale x 4 x bfloat> %0,
+    <vscale x 4 x bfloat> %1,
+    iXLen 9,
+    <vscale x 4 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 4 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_mask_vx_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2, <vscale x 8 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m2, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v10, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen %2,
+    <vscale x 8 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 8 x bfloat> @intrinsic_vslideup_mask_vi_nxv8bf16_nxv8bf16(<vscale x 8 x bfloat> %0, <vscale x 8 x bfloat> %1, <vscale x 8 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8bf16_nxv8bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m2, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v10, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 8 x bfloat> @llvm.riscv.vslideup.mask.nxv8bf16(
+    <vscale x 8 x bfloat> %0,
+    <vscale x 8 x bfloat> %1,
+    iXLen 9,
+    <vscale x 8 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 8 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v12, a0
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    iXLen %3,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_mask_vx_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2, <vscale x 16 x i1> %3, iXLen %4) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a1, e16, m4, tu, mu
+; CHECK-NEXT:    vslideup.vx v8, v12, a0, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen %2,
+    <vscale x 16 x i1> %3,
+    iXLen %4, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, iXLen %2) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v12, 9
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    iXLen %2,
+    iXLen 1)
+
+  ret <vscale x 16 x bfloat> %a
+}
+
+define <vscale x 16 x bfloat> @intrinsic_vslideup_mask_vi_nxv16bf16_nxv16bf16(<vscale x 16 x bfloat> %0, <vscale x 16 x bfloat> %1, <vscale x 16 x i1> %2, iXLen %3) nounwind {
+; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16bf16_nxv16bf16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, tu, mu
+; CHECK-NEXT:    vslideup.vi v8, v12, 9, v0.t
+; CHECK-NEXT:    ret
+entry:
+  %a = call <vscale x 16 x bfloat> @llvm.riscv.vslideup.mask.nxv16bf16(
+    <vscale x 16 x bfloat> %0,
+    <vscale x 16 x bfloat> %1,
+    iXLen 9,
+    <vscale x 16 x i1> %2,
+    iXLen %3, iXLen 0)
+
+  ret <vscale x 16 x bfloat> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
index f94e46771f49..c8b882b92b93 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll
@@ -1540,12 +1540,11 @@ define <vscale x 8 x i32> @vwadd_vx_splat_zext_i1(<vscale x 8 x i1> %va, i16 %b)
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slli a0, a0, 48
 ; RV64-NEXT:    srli a0, a0, 48
-; RV64-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a0
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; RV64-NEXT:    vmv.v.x v12, a0
+; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vwaddu.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
   %zb = zext i16 %b to i32
@@ -1615,12 +1614,11 @@ define <vscale x 8 x i32> @vwadd_vx_splat_sext_i1(<vscale x 8 x i1> %va, i16 %b)
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    slli a0, a0, 48
 ; RV64-NEXT:    srai a0, a0, 48
-; RV64-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; RV64-NEXT:    vmv.v.x v12, a0
-; RV64-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; RV64-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
 ; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, mu
+; RV64-NEXT:    vmv.v.x v12, a0
+; RV64-NEXT:    li a0, 1
 ; RV64-NEXT:    vwsub.vx v8, v12, a0, v0.t
 ; RV64-NEXT:    ret
   %sb = sext i16 %b to i32
diff --git a/llvm/test/CodeGen/RISCV/select-cond.ll b/llvm/test/CodeGen/RISCV/select-cond.ll
index a5f4677f73f1..d9f9ad379ee9 100644
--- a/llvm/test/CodeGen/RISCV/select-cond.ll
+++ b/llvm/test/CodeGen/RISCV/select-cond.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck %s --check-prefixes=RV32
 ; RUN: llc -mtriple=riscv32 -mattr=+xtheadcondmov -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefixes=RV32-THEAD
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcicm -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32-XQCICM
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s --check-prefixes=RV64
 ; RUN: llc -mtriple=riscv64 -mattr=+xmipscmov -verify-machineinstrs < %s \
@@ -26,6 +28,13 @@ define signext i32 @select_i32_trunc(i32 signext %cond, i32 signext %x, i32 sign
 ; RV32-THEAD-NEXT:    mv a0, a1
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_trunc:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    andi a0, a0, 1
+; RV32-XQCICM-NEXT:    qc.mveqi a1, a0, 0, a2
+; RV32-XQCICM-NEXT:    mv a0, a1
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_trunc:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    andi a3, a0, 1
@@ -64,6 +73,13 @@ define signext i32 @select_i32_param(i1 signext %cond, i32 signext %x, i32 signe
 ; RV32-THEAD-NEXT:    mv a0, a1
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_param:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    andi a0, a0, 1
+; RV32-XQCICM-NEXT:    qc.mveqi a1, a0, 0, a2
+; RV32-XQCICM-NEXT:    mv a0, a1
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_param:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    andi a3, a0, 1
@@ -100,6 +116,12 @@ define signext i32 @select_i32_eq(i32 signext %a, i32 signext %b, i32 signext %x
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_eq:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mveq a3, a0, a1, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_eq:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    beq a0, a1, .LBB2_2
@@ -136,6 +158,12 @@ define signext i32 @select_i32_ne(i32 signext %a, i32 signext %b, i32 signext %x
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_ne:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvne a3, a0, a1, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_ne:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bne a0, a1, .LBB3_2
@@ -172,6 +200,12 @@ define signext i32 @select_i32_ugt(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_ugt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvltu a3, a1, a0, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_ugt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bltu a1, a0, .LBB4_2
@@ -208,6 +242,12 @@ define signext i32 @select_i32_uge(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_uge:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvltu a2, a0, a1, a3
+; RV32-XQCICM-NEXT:    mv a0, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_uge:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bgeu a0, a1, .LBB5_2
@@ -244,6 +284,12 @@ define signext i32 @select_i32_ult(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_ult:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvltu a3, a0, a1, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_ult:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bltu a0, a1, .LBB6_2
@@ -280,6 +326,12 @@ define signext i32 @select_i32_ule(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_ule:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvltu a2, a1, a0, a3
+; RV32-XQCICM-NEXT:    mv a0, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_ule:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bgeu a1, a0, .LBB7_2
@@ -316,6 +368,12 @@ define signext i32 @select_i32_sgt(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_sgt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvlt a3, a1, a0, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_sgt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    blt a1, a0, .LBB8_2
@@ -352,6 +410,12 @@ define signext i32 @select_i32_sge(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_sge:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvlt a2, a0, a1, a3
+; RV32-XQCICM-NEXT:    mv a0, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_sge:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bge a0, a1, .LBB9_2
@@ -388,6 +452,12 @@ define signext i32 @select_i32_slt(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_slt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvlt a3, a0, a1, a2
+; RV32-XQCICM-NEXT:    mv a0, a3
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_slt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    blt a0, a1, .LBB10_2
@@ -424,6 +494,12 @@ define signext i32 @select_i32_sle(i32 signext %a, i32 signext %b, i32 signext %
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i32_sle:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    qc.mvlt a2, a1, a0, a3
+; RV32-XQCICM-NEXT:    mv a0, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i32_sle:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bge a1, a0, .LBB11_2
@@ -465,6 +541,15 @@ define i64 @select_i64_trunc(i64 %cond, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a0, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_trunc:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    mv a1, a3
+; RV32-XQCICM-NEXT:    andi a0, a0, 1
+; RV32-XQCICM-NEXT:    qc.mveqi a2, a0, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a1, a0, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_trunc:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    andi a3, a0, 1
@@ -507,6 +592,15 @@ define i64 @select_i64_param(i1 %cond, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a2
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_param:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    andi a0, a0, 1
+; RV32-XQCICM-NEXT:    qc.mveqi a1, a0, 0, a3
+; RV32-XQCICM-NEXT:    qc.mveqi a2, a0, 0, a4
+; RV32-XQCICM-NEXT:    mv a0, a1
+; RV32-XQCICM-NEXT:    mv a1, a2
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_param:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    andi a3, a0, 1
@@ -552,6 +646,17 @@ define i64 @select_i64_eq(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_eq:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    xor a1, a1, a3
+; RV32-XQCICM-NEXT:    xor a0, a0, a2
+; RV32-XQCICM-NEXT:    or a0, a0, a1
+; RV32-XQCICM-NEXT:    qc.mveqi a6, a0, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a7, a0, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_eq:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    beq a0, a1, .LBB14_2
@@ -597,6 +702,17 @@ define i64 @select_i64_ne(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_ne:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    xor a1, a1, a3
+; RV32-XQCICM-NEXT:    xor a0, a0, a2
+; RV32-XQCICM-NEXT:    or a0, a0, a1
+; RV32-XQCICM-NEXT:    qc.mvnei a6, a0, 0, a4
+; RV32-XQCICM-NEXT:    qc.mvnei a7, a0, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_ne:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bne a0, a1, .LBB15_2
@@ -647,6 +763,17 @@ define i64 @select_i64_ugt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_ugt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a2, a0
+; RV32-XQCICM-NEXT:    sltu a2, a3, a1
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a4, a2, 0, a6
+; RV32-XQCICM-NEXT:    qc.mveqi a5, a2, 0, a7
+; RV32-XQCICM-NEXT:    mv a0, a4
+; RV32-XQCICM-NEXT:    mv a1, a5
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_ugt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bltu a1, a0, .LBB16_2
@@ -697,6 +824,17 @@ define i64 @select_i64_uge(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_uge:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a0, a2
+; RV32-XQCICM-NEXT:    sltu a2, a1, a3
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a6, a2, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a7, a2, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_uge:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bgeu a0, a1, .LBB17_2
@@ -747,6 +885,17 @@ define i64 @select_i64_ult(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_ult:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a0, a2
+; RV32-XQCICM-NEXT:    sltu a2, a1, a3
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a4, a2, 0, a6
+; RV32-XQCICM-NEXT:    qc.mveqi a5, a2, 0, a7
+; RV32-XQCICM-NEXT:    mv a0, a4
+; RV32-XQCICM-NEXT:    mv a1, a5
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_ult:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bltu a0, a1, .LBB18_2
@@ -797,6 +946,17 @@ define i64 @select_i64_ule(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_ule:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a2, a0
+; RV32-XQCICM-NEXT:    sltu a2, a3, a1
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a6, a2, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a7, a2, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_ule:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bgeu a1, a0, .LBB19_2
@@ -847,6 +1007,17 @@ define i64 @select_i64_sgt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_sgt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a2, a0
+; RV32-XQCICM-NEXT:    slt a2, a3, a1
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a4, a2, 0, a6
+; RV32-XQCICM-NEXT:    qc.mveqi a5, a2, 0, a7
+; RV32-XQCICM-NEXT:    mv a0, a4
+; RV32-XQCICM-NEXT:    mv a1, a5
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_sgt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    blt a1, a0, .LBB20_2
@@ -897,6 +1068,17 @@ define i64 @select_i64_sge(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_sge:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a0, a2
+; RV32-XQCICM-NEXT:    slt a2, a1, a3
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a6, a2, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a7, a2, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_sge:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bge a0, a1, .LBB21_2
@@ -947,6 +1129,17 @@ define i64 @select_i64_slt(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_slt:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a0, a2
+; RV32-XQCICM-NEXT:    slt a2, a1, a3
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a4, a2, 0, a6
+; RV32-XQCICM-NEXT:    qc.mveqi a5, a2, 0, a7
+; RV32-XQCICM-NEXT:    mv a0, a4
+; RV32-XQCICM-NEXT:    mv a1, a5
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_slt:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    blt a0, a1, .LBB22_2
@@ -997,6 +1190,17 @@ define i64 @select_i64_sle(i64 %a, i64 %b, i64 %x, i64 %y) nounwind {
 ; RV32-THEAD-NEXT:    mv a1, a5
 ; RV32-THEAD-NEXT:    ret
 ;
+; RV32-XQCICM-LABEL: select_i64_sle:
+; RV32-XQCICM:       # %bb.0:
+; RV32-XQCICM-NEXT:    sltu a0, a2, a0
+; RV32-XQCICM-NEXT:    slt a2, a3, a1
+; RV32-XQCICM-NEXT:    qc.mveq a2, a1, a3, a0
+; RV32-XQCICM-NEXT:    qc.mveqi a6, a2, 0, a4
+; RV32-XQCICM-NEXT:    qc.mveqi a7, a2, 0, a5
+; RV32-XQCICM-NEXT:    mv a0, a6
+; RV32-XQCICM-NEXT:    mv a1, a7
+; RV32-XQCICM-NEXT:    ret
+;
 ; RV64-LABEL: select_i64_sle:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    bge a1, a0, .LBB23_2
diff --git a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
index 005a01bf1000..4066f62afad0 100644
--- a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
+++ b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
@@ -338,45 +338,3 @@ entry:
   %ret = add i32 %cond1, %cond2
   ret i32 %ret
 }
-
-define float @CascadedSelect(float noundef %a) {
-; RV32I-LABEL: CascadedSelect:
-; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    fmv.w.x fa5, a0
-; RV32I-NEXT:    lui a0, 260096
-; RV32I-NEXT:    fmv.w.x fa4, a0
-; RV32I-NEXT:    flt.s a0, fa4, fa5
-; RV32I-NEXT:    bnez a0, .LBB8_3
-; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    fmv.w.x fa4, zero
-; RV32I-NEXT:    flt.s a0, fa5, fa4
-; RV32I-NEXT:    bnez a0, .LBB8_3
-; RV32I-NEXT:  # %bb.2: # %entry
-; RV32I-NEXT:    fmv.s fa4, fa5
-; RV32I-NEXT:  .LBB8_3: # %entry
-; RV32I-NEXT:    fmv.x.w a0, fa4
-; RV32I-NEXT:    ret
-;
-; RV64I-LABEL: CascadedSelect:
-; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    fmv.w.x fa5, a0
-; RV64I-NEXT:    lui a0, 260096
-; RV64I-NEXT:    fmv.w.x fa4, a0
-; RV64I-NEXT:    flt.s a0, fa4, fa5
-; RV64I-NEXT:    bnez a0, .LBB8_3
-; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    fmv.w.x fa4, zero
-; RV64I-NEXT:    flt.s a0, fa5, fa4
-; RV64I-NEXT:    bnez a0, .LBB8_3
-; RV64I-NEXT:  # %bb.2: # %entry
-; RV64I-NEXT:    fmv.s fa4, fa5
-; RV64I-NEXT:  .LBB8_3: # %entry
-; RV64I-NEXT:    fmv.x.w a0, fa4
-; RV64I-NEXT:    ret
-entry:
-  %cmp = fcmp ogt float %a, 1.000000e+00
-  %cmp1 = fcmp olt float %a, 0.000000e+00
-  %.a = select i1 %cmp1, float 0.000000e+00, float %a
-  %retval.0 = select i1 %cmp, float 1.000000e+00, float %.a
-  ret float %retval.0
-}
diff --git a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
index 13c43a3875a0..061435c45ad0 100644
--- a/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
+++ b/llvm/test/CodeGen/RISCV/short-forward-branch-opt.ll
@@ -2,11 +2,13 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+c,+zbb -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=NOSFB %s
 ; RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -mattr=+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=SFB,NOZICOND,RV64SFB %s
+; RUN:   | FileCheck -check-prefixes=RV64SFB,RV64SFBSIFIVEU74 %s
+; RUN: llc -mtriple=riscv64 -mcpu=andes-ax45 -mattr=+zbb -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV64SFB,RV64SFBANDESAX45 %s
 ; RUN: llc -mtriple=riscv64 -mcpu=sifive-u74 -mattr=+zicond,+zbb \
-; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=SFB,ZICOND %s
+; RUN:   -verify-machineinstrs < %s | FileCheck -check-prefixes=ZICOND %s
 ; RUN: llc -mtriple=riscv32 -mcpu=sifive-e76 -mattr=+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=SFB,NOZICOND,RV32SFB %s
+; RUN:   | FileCheck -check-prefixes=RV32SFB %s
 
 ; The sifive-7-series can predicate a mv.
 
@@ -19,13 +21,37 @@ define signext i32 @test1(i32 signext %x, i32 signext %y, i32 signext %z) {
 ; NOSFB-NEXT:  .LBB0_2:
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test1:
-; SFB:       # %bb.0:
-; SFB-NEXT:    beqz a2, .LBB0_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a1
-; SFB-NEXT:  .LBB0_2:
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test1:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB0_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB0_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test1:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a2, 0, .LBB0_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB0_2:
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: test1:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    beqz a2, .LBB0_2
+; ZICOND-NEXT:  # %bb.1:
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:  .LBB0_2:
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test1:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    beqz a2, .LBB0_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a1
+; RV32SFB-NEXT:  .LBB0_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 %y
   ret i32 %b
@@ -42,13 +68,38 @@ define signext i32 @test2(i32 signext %x, i32 signext %y, i32 signext %z) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: test2:
-; SFB:       # %bb.0:
-; SFB-NEXT:    bnez a2, .LBB1_2
-; SFB-NEXT:  # %bb.1:
-; SFB-NEXT:    mv a0, a1
-; SFB-NEXT:  .LBB1_2:
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test2:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB1_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB1_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test2:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a2, 0, .LBB1_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    mv a1, a0
+; RV64SFBANDESAX45-NEXT:  .LBB1_2:
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: test2:
+; ZICOND:       # %bb.0:
+; ZICOND-NEXT:    bnez a2, .LBB1_2
+; ZICOND-NEXT:  # %bb.1:
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:  .LBB1_2:
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test2:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    bnez a2, .LBB1_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a1
+; RV32SFB-NEXT:  .LBB1_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %y, i32 %x
   ret i32 %b
@@ -67,18 +118,33 @@ define signext i32 @test3(i32 signext %v, i32 signext %w, i32 signext %x, i32 si
 ; NOSFB-NEXT:    addw a0, a1, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: test3:
-; RV64SFB:       # %bb.0:
-; RV64SFB-NEXT:    beqz a4, .LBB2_2
-; RV64SFB-NEXT:  # %bb.1:
-; RV64SFB-NEXT:    mv a2, a3
-; RV64SFB-NEXT:  .LBB2_2:
-; RV64SFB-NEXT:    bnez a4, .LBB2_4
-; RV64SFB-NEXT:  # %bb.3:
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:  .LBB2_4:
-; RV64SFB-NEXT:    addw a0, a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test3:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    beqz a4, .LBB2_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a2, a3
+; RV64SFBSIFIVEU74-NEXT:  .LBB2_2:
+; RV64SFBSIFIVEU74-NEXT:    bnez a4, .LBB2_4
+; RV64SFBSIFIVEU74-NEXT:  # %bb.3:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB2_4:
+; RV64SFBSIFIVEU74-NEXT:    addw a0, a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test3:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a4, 0, .LBB2_3
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a4, 0, .LBB2_4
+; RV64SFBANDESAX45-NEXT:  .LBB2_2:
+; RV64SFBANDESAX45-NEXT:    addw a0, a1, a2
+; RV64SFBANDESAX45-NEXT:    ret
+; RV64SFBANDESAX45-NEXT:  .LBB2_3:
+; RV64SFBANDESAX45-NEXT:    mv a1, a0
+; RV64SFBANDESAX45-NEXT:    nds.beqc a4, 0, .LBB2_2
+; RV64SFBANDESAX45-NEXT:  .LBB2_4:
+; RV64SFBANDESAX45-NEXT:    addw a0, a1, a3
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test3:
 ; ZICOND:       # %bb.0:
@@ -121,18 +187,34 @@ define signext i32 @test4(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    and a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test4:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    beqz a1, .LBB3_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    li a0, 0
-; NOZICOND-NEXT:  .LBB3_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test4:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB3_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    li a0, 0
+; RV64SFBSIFIVEU74-NEXT:  .LBB3_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test4:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a1, 0, .LBB3_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, 0
+; RV64SFBANDESAX45-NEXT:  .LBB3_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test4:
 ; ZICOND:       # %bb.0:
 ; ZICOND-NEXT:    czero.nez a0, a0, a1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test4:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    beqz a1, .LBB3_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    li a0, 0
+; RV32SFB-NEXT:  .LBB3_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 0
   ret i32 %b
@@ -147,18 +229,34 @@ define signext i32 @test5(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    and a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test5:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    bnez a1, .LBB4_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    li a0, 0
-; NOZICOND-NEXT:  .LBB4_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test5:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    bnez a1, .LBB4_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    li a0, 0
+; RV64SFBSIFIVEU74-NEXT:  .LBB4_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test5:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a1, 0, .LBB4_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, 0
+; RV64SFBANDESAX45-NEXT:  .LBB4_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test5:
 ; ZICOND:       # %bb.0:
 ; ZICOND-NEXT:    czero.eqz a0, a0, a1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test5:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    bnez a1, .LBB4_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    li a0, 0
+; RV32SFB-NEXT:  .LBB4_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 0, i32 %x
   ret i32 %b
@@ -173,14 +271,22 @@ define signext i32 @test6(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test6:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    li a2, -1
-; NOZICOND-NEXT:    beqz a1, .LBB5_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    mv a0, a2
-; NOZICOND-NEXT:  .LBB5_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test6:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    li a2, -1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB5_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:  .LBB5_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test6:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.beqc a1, 0, .LBB5_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, -1
+; RV64SFBANDESAX45-NEXT:  .LBB5_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test6:
 ; ZICOND:       # %bb.0:
@@ -188,6 +294,15 @@ define signext i32 @test6(i32 signext %x, i32 signext %z) {
 ; ZICOND-NEXT:    czero.nez a0, a0, a1
 ; ZICOND-NEXT:    addi a0, a0, -1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test6:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    li a2, -1
+; RV32SFB-NEXT:    beqz a1, .LBB5_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:  .LBB5_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 %x, i32 -1
   ret i32 %b
@@ -202,14 +317,22 @@ define signext i32 @test7(i32 signext %x, i32 signext %z) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; NOZICOND-LABEL: test7:
-; NOZICOND:       # %bb.0:
-; NOZICOND-NEXT:    li a2, -1
-; NOZICOND-NEXT:    bnez a1, .LBB6_2
-; NOZICOND-NEXT:  # %bb.1:
-; NOZICOND-NEXT:    mv a0, a2
-; NOZICOND-NEXT:  .LBB6_2:
-; NOZICOND-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: test7:
+; RV64SFBSIFIVEU74:       # %bb.0:
+; RV64SFBSIFIVEU74-NEXT:    li a2, -1
+; RV64SFBSIFIVEU74-NEXT:    bnez a1, .LBB6_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1:
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:  .LBB6_2:
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: test7:
+; RV64SFBANDESAX45:       # %bb.0:
+; RV64SFBANDESAX45-NEXT:    nds.bnec a1, 0, .LBB6_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    li a0, -1
+; RV64SFBANDESAX45-NEXT:  .LBB6_2:
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: test7:
 ; ZICOND:       # %bb.0:
@@ -217,6 +340,15 @@ define signext i32 @test7(i32 signext %x, i32 signext %z) {
 ; ZICOND-NEXT:    czero.eqz a0, a0, a1
 ; ZICOND-NEXT:    addi a0, a0, -1
 ; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: test7:
+; RV32SFB:       # %bb.0:
+; RV32SFB-NEXT:    li a2, -1
+; RV32SFB-NEXT:    bnez a1, .LBB6_2
+; RV32SFB-NEXT:  # %bb.1:
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:  .LBB6_2:
+; RV32SFB-NEXT:    ret
   %c = icmp eq i32 %z, 0
   %b = select i1 %c, i32 -1, i32 %x
   ret i32 %b
@@ -231,14 +363,40 @@ define i16 @select_xor_1(i16 %A, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_1:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a1, a1, 1
-; SFB-NEXT:    beqz a1, .LBB7_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xori a0, a0, 43
-; SFB-NEXT:  .LBB7_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_1:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a1, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB7_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a0, a0, 43
+; RV64SFBSIFIVEU74-NEXT:  .LBB7_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_1:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a1, 0, .LBB7_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xori a0, a0, 43
+; RV64SFBANDESAX45-NEXT:  .LBB7_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_1:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a1, a1, 1
+; ZICOND-NEXT:    beqz a1, .LBB7_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xori a0, a0, 43
+; ZICOND-NEXT:  .LBB7_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_1:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a1, a1, 1
+; RV32SFB-NEXT:    beqz a1, .LBB7_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xori a0, a0, 43
+; RV32SFB-NEXT:  .LBB7_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -258,14 +416,40 @@ define i16 @select_xor_1b(i16 %A, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_1b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a1, a1, 1
-; SFB-NEXT:    beqz a1, .LBB8_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xori a0, a0, 43
-; SFB-NEXT:  .LBB8_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_1b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a1, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a1, .LBB8_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a0, a0, 43
+; RV64SFBSIFIVEU74-NEXT:  .LBB8_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_1b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a1, 0, .LBB8_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    xori a0, a0, 43
+; RV64SFBANDESAX45-NEXT:  .LBB8_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_1b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a1, a1, 1
+; ZICOND-NEXT:    beqz a1, .LBB8_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xori a0, a0, 43
+; ZICOND-NEXT:  .LBB8_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_1b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a1, a1, 1
+; RV32SFB-NEXT:    beqz a1, .LBB8_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xori a0, a0, 43
+; RV32SFB-NEXT:  .LBB8_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -283,14 +467,40 @@ define i32 @select_xor_2(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_2:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB9_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xor a0, a0, a1
-; SFB-NEXT:  .LBB9_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_2:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB9_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xor a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB9_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_2:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB9_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xor a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB9_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_2:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB9_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xor a0, a0, a1
+; ZICOND-NEXT:  .LBB9_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_2:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB9_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xor a0, a0, a1
+; RV32SFB-NEXT:  .LBB9_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -310,14 +520,40 @@ define i32 @select_xor_2b(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    xor a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_xor_2b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB10_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    xor a0, a0, a1
-; SFB-NEXT:  .LBB10_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xor_2b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB10_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xor a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB10_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xor_2b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB10_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    xor a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB10_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_xor_2b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB10_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    xor a0, a0, a1
+; ZICOND-NEXT:  .LBB10_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_xor_2b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB10_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    xor a0, a0, a1
+; RV32SFB-NEXT:  .LBB10_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -335,14 +571,40 @@ define i32 @select_or(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB11_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB11_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB11_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB11_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB11_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB11_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB11_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB11_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB11_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB11_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp eq i8 %and, 0
@@ -362,14 +624,40 @@ define i32 @select_or_b(i32 %A, i32 %B, i8 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB12_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB12_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB12_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB12_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB12_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB12_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB12_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB12_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB12_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB12_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i8 %cond, 1
  %cmp10 = icmp ne i8 %and, 1
@@ -387,14 +675,40 @@ define i32 @select_or_1(i32 %A, i32 %B, i32 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_1:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB13_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB13_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_1:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB13_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB13_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_1:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB13_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB13_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_1:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB13_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB13_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_1:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB13_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB13_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i32 %cond, 1
  %cmp10 = icmp eq i32 %and, 0
@@ -414,14 +728,40 @@ define i32 @select_or_1b(i32 %A, i32 %B, i32 %cond) {
 ; NOSFB-NEXT:    or a0, a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; SFB-LABEL: select_or_1b:
-; SFB:       # %bb.0: # %entry
-; SFB-NEXT:    andi a2, a2, 1
-; SFB-NEXT:    beqz a2, .LBB14_2
-; SFB-NEXT:  # %bb.1: # %entry
-; SFB-NEXT:    or a0, a0, a1
-; SFB-NEXT:  .LBB14_2: # %entry
-; SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_or_1b:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a2, a2, 1
+; RV64SFBSIFIVEU74-NEXT:    beqz a2, .LBB14_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    or a0, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB14_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_or_1b:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bbc a2, 0, .LBB14_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or a0, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB14_2: # %entry
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_or_1b:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    andi a2, a2, 1
+; ZICOND-NEXT:    beqz a2, .LBB14_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    or a0, a0, a1
+; ZICOND-NEXT:  .LBB14_2: # %entry
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_or_1b:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    andi a2, a2, 1
+; RV32SFB-NEXT:    beqz a2, .LBB14_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a0, a0, a1
+; RV32SFB-NEXT:  .LBB14_2: # %entry
+; RV32SFB-NEXT:    ret
 entry:
  %and = and i32 %cond, 1
  %cmp10 = icmp ne i32 %and, 1
@@ -455,31 +795,55 @@ define void @sextw_removal_ccor(i1 %c, i32 signext %arg, i32 signext %arg1, i32
 ; NOSFB-NEXT:    addi sp, sp, 32
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: sextw_removal_ccor:
-; RV64SFB:       # %bb.0: # %bb
-; RV64SFB-NEXT:    addi sp, sp, -32
-; RV64SFB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    mv s0, a3
-; RV64SFB-NEXT:    andi a0, a0, 1
-; RV64SFB-NEXT:    mv s1, a2
-; RV64SFB-NEXT:    beqz a0, .LBB15_4
-; RV64SFB-NEXT:  # %bb.3: # %bb
-; RV64SFB-NEXT:    or s0, a3, a1
-; RV64SFB-NEXT:  .LBB15_4: # %bb
-; RV64SFB-NEXT:  .LBB15_1: # %bb2
-; RV64SFB-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64SFB-NEXT:    mv a0, s0
-; RV64SFB-NEXT:    call bar
-; RV64SFB-NEXT:    sllw s0, s0, s1
-; RV64SFB-NEXT:    bnez a0, .LBB15_1
-; RV64SFB-NEXT:  # %bb.2: # %bb7
-; RV64SFB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    addi sp, sp, 32
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccor:
+; RV64SFBSIFIVEU74:       # %bb.0: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    mv s0, a3
+; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    mv s1, a2
+; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB15_4
+; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
+; RV64SFBSIFIVEU74-NEXT:    or s0, a3, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB15_4: # %bb
+; RV64SFBSIFIVEU74-NEXT:  .LBB15_1: # %bb2
+; RV64SFBSIFIVEU74-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBSIFIVEU74-NEXT:    mv a0, s0
+; RV64SFBSIFIVEU74-NEXT:    call bar
+; RV64SFBSIFIVEU74-NEXT:    sllw s0, s0, s1
+; RV64SFBSIFIVEU74-NEXT:    bnez a0, .LBB15_1
+; RV64SFBSIFIVEU74-NEXT:  # %bb.2: # %bb7
+; RV64SFBSIFIVEU74-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, 32
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: sextw_removal_ccor:
+; RV64SFBANDESAX45:       # %bb.0: # %bb
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    mv s0, a3
+; RV64SFBANDESAX45-NEXT:    mv s1, a2
+; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB15_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    or s0, s0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB15_2: # %bb2
+; RV64SFBANDESAX45-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBANDESAX45-NEXT:    mv a0, s0
+; RV64SFBANDESAX45-NEXT:    call bar
+; RV64SFBANDESAX45-NEXT:    sllw s0, s0, s1
+; RV64SFBANDESAX45-NEXT:    bnez a0, .LBB15_2
+; RV64SFBANDESAX45-NEXT:  # %bb.3: # %bb7
+; RV64SFBANDESAX45-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, 32
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: sextw_removal_ccor:
 ; ZICOND:       # %bb.0: # %bb
@@ -574,31 +938,55 @@ define void @sextw_removal_ccaddw(i1 %c, i32 signext %arg, i32 signext %arg1, i3
 ; NOSFB-NEXT:    addi sp, sp, 32
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: sextw_removal_ccaddw:
-; RV64SFB:       # %bb.0: # %bb
-; RV64SFB-NEXT:    addi sp, sp, -32
-; RV64SFB-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64SFB-NEXT:    mv s1, a1
-; RV64SFB-NEXT:    andi a0, a0, 1
-; RV64SFB-NEXT:    mv s0, a2
-; RV64SFB-NEXT:    beqz a0, .LBB16_4
-; RV64SFB-NEXT:  # %bb.3: # %bb
-; RV64SFB-NEXT:    addw s1, a1, a3
-; RV64SFB-NEXT:  .LBB16_4: # %bb
-; RV64SFB-NEXT:  .LBB16_1: # %bb2
-; RV64SFB-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64SFB-NEXT:    mv a0, s1
-; RV64SFB-NEXT:    call bar
-; RV64SFB-NEXT:    sllw s1, s1, s0
-; RV64SFB-NEXT:    bnez a0, .LBB16_1
-; RV64SFB-NEXT:  # %bb.2: # %bb7
-; RV64SFB-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
-; RV64SFB-NEXT:    addi sp, sp, 32
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: sextw_removal_ccaddw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, -32
+; RV64SFBSIFIVEU74-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBSIFIVEU74-NEXT:    mv s1, a1
+; RV64SFBSIFIVEU74-NEXT:    andi a0, a0, 1
+; RV64SFBSIFIVEU74-NEXT:    mv s0, a2
+; RV64SFBSIFIVEU74-NEXT:    beqz a0, .LBB16_4
+; RV64SFBSIFIVEU74-NEXT:  # %bb.3: # %bb
+; RV64SFBSIFIVEU74-NEXT:    addw s1, a1, a3
+; RV64SFBSIFIVEU74-NEXT:  .LBB16_4: # %bb
+; RV64SFBSIFIVEU74-NEXT:  .LBB16_1: # %bb2
+; RV64SFBSIFIVEU74-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBSIFIVEU74-NEXT:    mv a0, s1
+; RV64SFBSIFIVEU74-NEXT:    call bar
+; RV64SFBSIFIVEU74-NEXT:    sllw s1, s1, s0
+; RV64SFBSIFIVEU74-NEXT:    bnez a0, .LBB16_1
+; RV64SFBSIFIVEU74-NEXT:  # %bb.2: # %bb7
+; RV64SFBSIFIVEU74-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBSIFIVEU74-NEXT:    addi sp, sp, 32
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: sextw_removal_ccaddw:
+; RV64SFBANDESAX45:       # %bb.0: # %bb
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, -32
+; RV64SFBANDESAX45-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64SFBANDESAX45-NEXT:    mv s0, a2
+; RV64SFBANDESAX45-NEXT:    mv s1, a1
+; RV64SFBANDESAX45-NEXT:    nds.bbc a0, 0, .LBB16_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1:
+; RV64SFBANDESAX45-NEXT:    addw s1, s1, a3
+; RV64SFBANDESAX45-NEXT:  .LBB16_2: # %bb2
+; RV64SFBANDESAX45-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64SFBANDESAX45-NEXT:    mv a0, s1
+; RV64SFBANDESAX45-NEXT:    call bar
+; RV64SFBANDESAX45-NEXT:    sllw s1, s1, s0
+; RV64SFBANDESAX45-NEXT:    bnez a0, .LBB16_2
+; RV64SFBANDESAX45-NEXT:  # %bb.3: # %bb7
+; RV64SFBANDESAX45-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64SFBANDESAX45-NEXT:    addi sp, sp, 32
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: sextw_removal_ccaddw:
 ; ZICOND:       # %bb.0: # %bb
@@ -677,14 +1065,23 @@ define i32 @select_sllw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sllw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB17_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sllw a2, a0, a1
-; RV64SFB-NEXT:  .LBB17_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sllw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB17_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sllw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB17_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sllw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB17_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sllw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB17_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sllw:
 ; ZICOND:       # %bb.0: # %entry
@@ -719,14 +1116,23 @@ define i32 @select_srlw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srlw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB18_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srlw a2, a0, a1
-; RV64SFB-NEXT:  .LBB18_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srlw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB18_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srlw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB18_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srlw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB18_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srlw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB18_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srlw:
 ; ZICOND:       # %bb.0: # %entry
@@ -761,14 +1167,23 @@ define i32 @select_sraw(i32 %A, i32 %B, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sraw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB19_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sraw a2, a0, a1
-; RV64SFB-NEXT:  .LBB19_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sraw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB19_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sraw a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB19_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sraw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB19_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sraw a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB19_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sraw:
 ; ZICOND:       # %bb.0: # %entry
@@ -803,14 +1218,23 @@ define i64 @select_sll(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sll:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB20_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sll a2, a0, a1
-; RV64SFB-NEXT:  .LBB20_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sll:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB20_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sll a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB20_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sll:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB20_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sll a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB20_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sll:
 ; ZICOND:       # %bb.0: # %entry
@@ -864,14 +1288,23 @@ define i64 @select_srl(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srl:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB21_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srl a2, a0, a1
-; RV64SFB-NEXT:  .LBB21_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srl:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB21_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srl a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB21_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srl:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB21_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srl a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB21_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srl:
 ; ZICOND:       # %bb.0: # %entry
@@ -925,14 +1358,23 @@ define i64 @select_sra(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sra:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB22_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sra a2, a0, a1
-; RV64SFB-NEXT:  .LBB22_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sra:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB22_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sra a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB22_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sra:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB22_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sra a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB22_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sra:
 ; ZICOND:       # %bb.0: # %entry
@@ -986,14 +1428,23 @@ define i32 @select_addiw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_addiw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB23_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    addiw a1, a0, 1234
-; RV64SFB-NEXT:  .LBB23_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_addiw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB23_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    addiw a1, a0, 1234
+; RV64SFBSIFIVEU74-NEXT:  .LBB23_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_addiw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB23_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    addiw a1, a0, 1234
+; RV64SFBANDESAX45-NEXT:  .LBB23_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_addiw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1028,14 +1479,23 @@ define i64 @select_addi(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_addi:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB24_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    addi a1, a0, 1234
-; RV64SFB-NEXT:  .LBB24_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_addi:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB24_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    addi a1, a0, 1234
+; RV64SFBSIFIVEU74-NEXT:  .LBB24_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_addi:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB24_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    addi a1, a0, 1234
+; RV64SFBANDESAX45-NEXT:  .LBB24_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_addi:
 ; ZICOND:       # %bb.0: # %entry
@@ -1077,14 +1537,23 @@ define i64 @select_andi(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_andi:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB25_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    andi a1, a0, 567
-; RV64SFB-NEXT:  .LBB25_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_andi:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB25_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andi a1, a0, 567
+; RV64SFBSIFIVEU74-NEXT:  .LBB25_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_andi:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB25_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    andi a1, a0, 567
+; RV64SFBANDESAX45-NEXT:  .LBB25_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_andi:
 ; ZICOND:       # %bb.0: # %entry
@@ -1124,14 +1593,23 @@ define i64 @select_ori(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_ori:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB26_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    ori a1, a0, 890
-; RV64SFB-NEXT:  .LBB26_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_ori:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB26_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    ori a1, a0, 890
+; RV64SFBSIFIVEU74-NEXT:  .LBB26_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_ori:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB26_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    ori a1, a0, 890
+; RV64SFBANDESAX45-NEXT:  .LBB26_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_ori:
 ; ZICOND:       # %bb.0: # %entry
@@ -1170,14 +1648,23 @@ define i64 @select_xori(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_xori:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB27_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    xori a1, a0, 321
-; RV64SFB-NEXT:  .LBB27_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xori:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB27_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xori a1, a0, 321
+; RV64SFBSIFIVEU74-NEXT:  .LBB27_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xori:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB27_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xori a1, a0, 321
+; RV64SFBANDESAX45-NEXT:  .LBB27_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_xori:
 ; ZICOND:       # %bb.0: # %entry
@@ -1216,14 +1703,23 @@ define i64 @select_slli(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_slli:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB28_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    slli a1, a0, 32
-; RV64SFB-NEXT:  .LBB28_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_slli:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB28_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slli a1, a0, 32
+; RV64SFBSIFIVEU74-NEXT:  .LBB28_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_slli:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB28_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    slli a1, a0, 32
+; RV64SFBANDESAX45-NEXT:  .LBB28_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_slli:
 ; ZICOND:       # %bb.0: # %entry
@@ -1263,14 +1759,23 @@ define i64 @select_srli(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srli:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB29_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srli a1, a0, 35
-; RV64SFB-NEXT:  .LBB29_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srli:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB29_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srli a1, a0, 35
+; RV64SFBSIFIVEU74-NEXT:  .LBB29_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srli:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB29_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srli a1, a0, 35
+; RV64SFBANDESAX45-NEXT:  .LBB29_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srli:
 ; ZICOND:       # %bb.0: # %entry
@@ -1310,14 +1815,23 @@ define i64 @select_srai(i64 %A, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srai:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB30_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srai a1, a0, 63
-; RV64SFB-NEXT:  .LBB30_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srai:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB30_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srai a1, a0, 63
+; RV64SFBSIFIVEU74-NEXT:  .LBB30_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srai:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB30_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srai a1, a0, 63
+; RV64SFBANDESAX45-NEXT:  .LBB30_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srai:
 ; ZICOND:       # %bb.0: # %entry
@@ -1357,14 +1871,23 @@ define i32 @select_slliw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_slliw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB31_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    slliw a1, a0, 3
-; RV64SFB-NEXT:  .LBB31_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_slliw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB31_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slliw a1, a0, 3
+; RV64SFBSIFIVEU74-NEXT:  .LBB31_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_slliw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB31_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    slliw a1, a0, 3
+; RV64SFBANDESAX45-NEXT:  .LBB31_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_slliw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1399,14 +1922,23 @@ define i32 @select_srliw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_srliw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB32_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    srliw a1, a0, 17
-; RV64SFB-NEXT:  .LBB32_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_srliw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB32_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srliw a1, a0, 17
+; RV64SFBSIFIVEU74-NEXT:  .LBB32_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_srliw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB32_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    srliw a1, a0, 17
+; RV64SFBANDESAX45-NEXT:  .LBB32_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_srliw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1441,14 +1973,23 @@ define i32 @select_sraiw(i32 %A, i32 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a1
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_sraiw:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a2, .LBB33_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    sraiw a1, a0, 31
-; RV64SFB-NEXT:  .LBB33_2: # %entry
-; RV64SFB-NEXT:    mv a0, a1
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_sraiw:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB33_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    sraiw a1, a0, 31
+; RV64SFBSIFIVEU74-NEXT:  .LBB33_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_sraiw:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB33_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    sraiw a1, a0, 31
+; RV64SFBANDESAX45-NEXT:  .LBB33_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_sraiw:
 ; ZICOND:       # %bb.0: # %entry
@@ -1559,14 +2100,23 @@ define i64 @select_andn(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_andn:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB36_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    andn a2, a0, a1
-; RV64SFB-NEXT:  .LBB36_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_andn:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB36_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    andn a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB36_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_andn:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB36_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    andn a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB36_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_andn:
 ; ZICOND:       # %bb.0: # %entry
@@ -1607,14 +2157,23 @@ define i64 @select_orn(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_orn:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB37_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    orn a2, a0, a1
-; RV64SFB-NEXT:  .LBB37_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_orn:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB37_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    orn a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB37_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_orn:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB37_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    orn a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB37_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_orn:
 ; ZICOND:       # %bb.0: # %entry
@@ -1655,14 +2214,23 @@ define i64 @select_xnor(i64 %A, i64 %B, i64 %C, i1 zeroext %cond) {
 ; NOSFB-NEXT:    mv a0, a2
 ; NOSFB-NEXT:    ret
 ;
-; RV64SFB-LABEL: select_xnor:
-; RV64SFB:       # %bb.0: # %entry
-; RV64SFB-NEXT:    bnez a3, .LBB38_2
-; RV64SFB-NEXT:  # %bb.1: # %entry
-; RV64SFB-NEXT:    xnor a2, a0, a1
-; RV64SFB-NEXT:  .LBB38_2: # %entry
-; RV64SFB-NEXT:    mv a0, a2
-; RV64SFB-NEXT:    ret
+; RV64SFBSIFIVEU74-LABEL: select_xnor:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    bnez a3, .LBB38_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    xnor a2, a0, a1
+; RV64SFBSIFIVEU74-NEXT:  .LBB38_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a2
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_xnor:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a3, 0, .LBB38_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    xnor a2, a0, a1
+; RV64SFBANDESAX45-NEXT:  .LBB38_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a2
+; RV64SFBANDESAX45-NEXT:    ret
 ;
 ; ZICOND-LABEL: select_xnor:
 ; ZICOND:       # %bb.0: # %entry
@@ -1692,3 +2260,130 @@ entry:
  %2 = select i1 %cond, i64 %C, i64 %1
  ret i64 %2
 }
+
+define i64 @select_bfoz(i64 %A, i64 %B, i1 zeroext %cond) {
+; NOSFB-LABEL: select_bfoz:
+; NOSFB:       # %bb.0: # %entry
+; NOSFB-NEXT:    bnez a2, .LBB39_2
+; NOSFB-NEXT:  # %bb.1: # %entry
+; NOSFB-NEXT:    slli a0, a0, 38
+; NOSFB-NEXT:    srli a1, a0, 61
+; NOSFB-NEXT:  .LBB39_2: # %entry
+; NOSFB-NEXT:    mv a0, a1
+; NOSFB-NEXT:    ret
+;
+; RV64SFBSIFIVEU74-LABEL: select_bfoz:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slli a0, a0, 38
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB39_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srli a1, a0, 61
+; RV64SFBSIFIVEU74-NEXT:  .LBB39_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_bfoz:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB39_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bfoz a1, a0, 25, 23
+; RV64SFBANDESAX45-NEXT:  .LBB39_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_bfoz:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    slli a0, a0, 38
+; ZICOND-NEXT:    bnez a2, .LBB39_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    srli a1, a0, 61
+; ZICOND-NEXT:  .LBB39_2: # %entry
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_bfoz:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    slli a0, a0, 6
+; RV32SFB-NEXT:    mv a1, a3
+; RV32SFB-NEXT:    bnez a4, .LBB39_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    srli a2, a0, 29
+; RV32SFB-NEXT:  .LBB39_2: # %entry
+; RV32SFB-NEXT:    bnez a4, .LBB39_4
+; RV32SFB-NEXT:  # %bb.3: # %entry
+; RV32SFB-NEXT:    li a1, 0
+; RV32SFB-NEXT:  .LBB39_4: # %entry
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:    ret
+entry:
+ %0 = lshr i64 %A, 23
+ %1 = and i64 %0, 7
+ %2 = select i1 %cond, i64 %B, i64 %1
+ ret i64 %2
+}
+
+define i64 @select_bfos(i64 %A, i64 %B, i1 zeroext %cond) {
+; NOSFB-LABEL: select_bfos:
+; NOSFB:       # %bb.0: # %entry
+; NOSFB-NEXT:    bnez a2, .LBB40_2
+; NOSFB-NEXT:  # %bb.1: # %entry
+; NOSFB-NEXT:    slli a0, a0, 31
+; NOSFB-NEXT:    srai a1, a0, 17
+; NOSFB-NEXT:  .LBB40_2: # %entry
+; NOSFB-NEXT:    mv a0, a1
+; NOSFB-NEXT:    ret
+;
+; RV64SFBSIFIVEU74-LABEL: select_bfos:
+; RV64SFBSIFIVEU74:       # %bb.0: # %entry
+; RV64SFBSIFIVEU74-NEXT:    slli a0, a0, 31
+; RV64SFBSIFIVEU74-NEXT:    bnez a2, .LBB40_2
+; RV64SFBSIFIVEU74-NEXT:  # %bb.1: # %entry
+; RV64SFBSIFIVEU74-NEXT:    srai a1, a0, 17
+; RV64SFBSIFIVEU74-NEXT:  .LBB40_2: # %entry
+; RV64SFBSIFIVEU74-NEXT:    mv a0, a1
+; RV64SFBSIFIVEU74-NEXT:    ret
+;
+; RV64SFBANDESAX45-LABEL: select_bfos:
+; RV64SFBANDESAX45:       # %bb.0: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bnec a2, 0, .LBB40_2
+; RV64SFBANDESAX45-NEXT:  # %bb.1: # %entry
+; RV64SFBANDESAX45-NEXT:    nds.bfos a1, a0, 14, 46
+; RV64SFBANDESAX45-NEXT:  .LBB40_2: # %entry
+; RV64SFBANDESAX45-NEXT:    mv a0, a1
+; RV64SFBANDESAX45-NEXT:    ret
+;
+; ZICOND-LABEL: select_bfos:
+; ZICOND:       # %bb.0: # %entry
+; ZICOND-NEXT:    slli a0, a0, 31
+; ZICOND-NEXT:    bnez a2, .LBB40_2
+; ZICOND-NEXT:  # %bb.1: # %entry
+; ZICOND-NEXT:    srai a1, a0, 17
+; ZICOND-NEXT:  .LBB40_2: # %entry
+; ZICOND-NEXT:    mv a0, a1
+; ZICOND-NEXT:    ret
+;
+; RV32SFB-LABEL: select_bfos:
+; RV32SFB:       # %bb.0: # %entry
+; RV32SFB-NEXT:    srli a5, a0, 1
+; RV32SFB-NEXT:    slli a6, a1, 31
+; RV32SFB-NEXT:    slli a0, a0, 31
+; RV32SFB-NEXT:    slli a1, a5, 15
+; RV32SFB-NEXT:    srli a0, a0, 17
+; RV32SFB-NEXT:    or a5, a6, a5
+; RV32SFB-NEXT:    bnez a4, .LBB40_2
+; RV32SFB-NEXT:  # %bb.1: # %entry
+; RV32SFB-NEXT:    or a2, a0, a1
+; RV32SFB-NEXT:  .LBB40_2: # %entry
+; RV32SFB-NEXT:    bnez a4, .LBB40_4
+; RV32SFB-NEXT:  # %bb.3: # %entry
+; RV32SFB-NEXT:    srai a3, a5, 17
+; RV32SFB-NEXT:  .LBB40_4: # %entry
+; RV32SFB-NEXT:    mv a0, a2
+; RV32SFB-NEXT:    mv a1, a3
+; RV32SFB-NEXT:    ret
+entry:
+ %0 = shl i64 %A, 31
+ %1 = ashr i64 %0, 17
+ %2 = select i1 %cond, i64 %B, i64 %1
+ ret i64 %2
+}
diff --git a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
index cd7f30d8f589..32753ca382fc 100644
--- a/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/wide-scalar-shift-legalization.ll
@@ -716,101 +716,92 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, a6, a5
-; RV32I-NEXT:    lbu a5, 8(a0)
-; RV32I-NEXT:    lbu a6, 9(a0)
-; RV32I-NEXT:    lbu t3, 10(a0)
-; RV32I-NEXT:    lbu t4, 11(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    or a5, a6, a5
+; RV32I-NEXT:    or a3, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    or t0, t2, t1
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 16
-; RV32I-NEXT:    slli t4, t4, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t2, 0(a1)
-; RV32I-NEXT:    lbu t4, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
+; RV32I-NEXT:    lbu t0, 12(a0)
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 16(sp)
 ; RV32I-NEXT:    sw zero, 20(sp)
 ; RV32I-NEXT:    sw zero, 24(sp)
 ; RV32I-NEXT:    sw zero, 28(sp)
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t4
-; RV32I-NEXT:    mv t2, sp
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, t0, a7
-; RV32I-NEXT:    or a5, t3, a5
-; RV32I-NEXT:    or a0, a0, a6
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a3, 0(sp)
-; RV32I-NEXT:    sw a4, 4(sp)
-; RV32I-NEXT:    sw a5, 8(sp)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    add a0, t2, a0
-; RV32I-NEXT:    lw a4, 4(a0)
-; RV32I-NEXT:    lw a5, 8(a0)
-; RV32I-NEXT:    lw a6, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a4, a1
-; RV32I-NEXT:    slli t0, a5, 1
-; RV32I-NEXT:    srl a6, a6, a1
-; RV32I-NEXT:    slli a4, a4, 1
-; RV32I-NEXT:    srl a5, a5, a1
-; RV32I-NEXT:    slli t1, a0, 1
-; RV32I-NEXT:    srl a0, a0, a1
-; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a4, a4, a3
-; RV32I-NEXT:    sll a3, t1, a3
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    mv t0, sp
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srli t3, a0, 3
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    andi a5, a0, 31
+; RV32I-NEXT:    andi t3, t3, 12
+; RV32I-NEXT:    xori a5, a5, 31
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    add t0, t0, t3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a3, 4(sp)
+; RV32I-NEXT:    sw a6, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a1, 4(t0)
+; RV32I-NEXT:    lw a3, 8(t0)
+; RV32I-NEXT:    lw a4, 0(t0)
+; RV32I-NEXT:    lw a6, 12(t0)
+; RV32I-NEXT:    srl a7, a1, a0
+; RV32I-NEXT:    slli t0, a3, 1
+; RV32I-NEXT:    srl a4, a4, a0
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srl a3, a3, a0
+; RV32I-NEXT:    slli t1, a6, 1
+; RV32I-NEXT:    srl a0, a6, a0
+; RV32I-NEXT:    sll a6, t0, a5
+; RV32I-NEXT:    sll a1, a1, a5
+; RV32I-NEXT:    sll a5, t1, a5
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
-; RV32I-NEXT:    or a1, a7, a1
-; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, a4, a1
+; RV32I-NEXT:    or a3, a3, a5
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
 ; RV32I-NEXT:    sb t1, 15(a2)
 ; RV32I-NEXT:    srli a0, a3, 16
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a4, 16
-; RV32I-NEXT:    srli t0, a4, 24
-; RV32I-NEXT:    srli t1, a4, 8
-; RV32I-NEXT:    srli t2, a1, 16
-; RV32I-NEXT:    srli t3, a1, 24
+; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 8
+; RV32I-NEXT:    srli a7, a1, 16
+; RV32I-NEXT:    srli t0, a1, 24
+; RV32I-NEXT:    srli t1, a1, 8
+; RV32I-NEXT:    srli t2, a6, 16
+; RV32I-NEXT:    srli t3, a6, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a6, 9(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a0, a1, 8
-; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a0, a6, 8
+; RV32I-NEXT:    sb a1, 0(a2)
 ; RV32I-NEXT:    sb t1, 1(a2)
 ; RV32I-NEXT:    sb a7, 2(a2)
 ; RV32I-NEXT:    sb t0, 3(a2)
-; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a6, 4(a2)
 ; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
@@ -952,102 +943,93 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, a6, a5
-; RV32I-NEXT:    lbu a5, 8(a0)
-; RV32I-NEXT:    lbu a6, 9(a0)
-; RV32I-NEXT:    lbu t3, 10(a0)
-; RV32I-NEXT:    lbu t4, 11(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a4, a4, a3
+; RV32I-NEXT:    or a5, a6, a5
+; RV32I-NEXT:    or a3, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli a6, a6, 8
-; RV32I-NEXT:    or a7, t0, a7
-; RV32I-NEXT:    or t0, t2, t1
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    lbu a6, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 16
-; RV32I-NEXT:    slli t4, t4, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t3, t4, t3
-; RV32I-NEXT:    or a6, t1, a6
-; RV32I-NEXT:    or a0, a0, t2
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t2, 0(a1)
-; RV32I-NEXT:    lbu t4, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t2
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
+; RV32I-NEXT:    lbu t0, 12(a0)
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 0(sp)
 ; RV32I-NEXT:    sw zero, 4(sp)
 ; RV32I-NEXT:    sw zero, 8(sp)
 ; RV32I-NEXT:    sw zero, 12(sp)
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t4
-; RV32I-NEXT:    addi t2, sp, 16
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    addi t0, sp, 16
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srli t3, a0, 3
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    andi a5, a0, 31
+; RV32I-NEXT:    andi t3, t3, 12
+; RV32I-NEXT:    or a3, t1, a3
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    sub a7, t0, t3
+; RV32I-NEXT:    sw a4, 16(sp)
+; RV32I-NEXT:    sw a3, 20(sp)
+; RV32I-NEXT:    sw a6, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    lw a1, 0(a7)
+; RV32I-NEXT:    lw a3, 4(a7)
+; RV32I-NEXT:    lw a4, 8(a7)
+; RV32I-NEXT:    lw a6, 12(a7)
+; RV32I-NEXT:    xori a5, a5, 31
+; RV32I-NEXT:    sll a7, a3, a0
+; RV32I-NEXT:    srli t0, a1, 1
+; RV32I-NEXT:    sll a6, a6, a0
+; RV32I-NEXT:    srli t1, a4, 1
+; RV32I-NEXT:    sll a4, a4, a0
+; RV32I-NEXT:    srli a3, a3, 1
+; RV32I-NEXT:    sll a0, a1, a0
+; RV32I-NEXT:    srl a1, t0, a5
+; RV32I-NEXT:    srl t0, t1, a5
+; RV32I-NEXT:    srl a3, a3, a5
+; RV32I-NEXT:    srli a5, a0, 16
+; RV32I-NEXT:    srli t1, a0, 24
+; RV32I-NEXT:    srli t2, a0, 8
+; RV32I-NEXT:    or a1, a7, a1
+; RV32I-NEXT:    or a6, a6, t0
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    or a4, t0, a7
-; RV32I-NEXT:    or a5, t3, a5
-; RV32I-NEXT:    or a0, a0, a6
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a3, 16(sp)
-; RV32I-NEXT:    sw a4, 20(sp)
-; RV32I-NEXT:    sw a5, 24(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    sub a0, t2, a0
-; RV32I-NEXT:    lw a4, 0(a0)
-; RV32I-NEXT:    lw a5, 4(a0)
-; RV32I-NEXT:    lw a6, 8(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    sll a7, a5, a1
-; RV32I-NEXT:    srli t0, a4, 1
-; RV32I-NEXT:    sll a0, a0, a1
-; RV32I-NEXT:    srli t1, a6, 1
-; RV32I-NEXT:    sll a6, a6, a1
-; RV32I-NEXT:    srli a5, a5, 1
-; RV32I-NEXT:    sll a1, a4, a1
-; RV32I-NEXT:    srl a4, t0, a3
-; RV32I-NEXT:    srl t0, t1, a3
-; RV32I-NEXT:    srl a3, a5, a3
-; RV32I-NEXT:    srli a5, a1, 16
-; RV32I-NEXT:    srli t1, a1, 24
-; RV32I-NEXT:    srli t2, a1, 8
-; RV32I-NEXT:    or a4, a7, a4
-; RV32I-NEXT:    or a0, a0, t0
-; RV32I-NEXT:    or a3, a6, a3
-; RV32I-NEXT:    sb a1, 0(a2)
+; RV32I-NEXT:    sb a0, 0(a2)
 ; RV32I-NEXT:    sb t2, 1(a2)
 ; RV32I-NEXT:    sb a5, 2(a2)
 ; RV32I-NEXT:    sb t1, 3(a2)
-; RV32I-NEXT:    srli a1, a3, 16
-; RV32I-NEXT:    srli a5, a3, 24
-; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a0, 16
-; RV32I-NEXT:    srli t0, a0, 24
-; RV32I-NEXT:    srli t1, a0, 8
-; RV32I-NEXT:    srli t2, a4, 16
-; RV32I-NEXT:    srli t3, a4, 24
+; RV32I-NEXT:    srli a0, a3, 16
+; RV32I-NEXT:    srli a4, a3, 24
+; RV32I-NEXT:    srli a5, a3, 8
+; RV32I-NEXT:    srli a7, a6, 16
+; RV32I-NEXT:    srli t0, a6, 24
+; RV32I-NEXT:    srli t1, a6, 8
+; RV32I-NEXT:    srli t2, a1, 16
+; RV32I-NEXT:    srli t3, a1, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
-; RV32I-NEXT:    sb a6, 9(a2)
-; RV32I-NEXT:    sb a1, 10(a2)
-; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a1, a4, 8
-; RV32I-NEXT:    sb a0, 12(a2)
+; RV32I-NEXT:    sb a5, 9(a2)
+; RV32I-NEXT:    sb a0, 10(a2)
+; RV32I-NEXT:    sb a4, 11(a2)
+; RV32I-NEXT:    srli a0, a1, 8
+; RV32I-NEXT:    sb a6, 12(a2)
 ; RV32I-NEXT:    sb t1, 13(a2)
 ; RV32I-NEXT:    sb a7, 14(a2)
 ; RV32I-NEXT:    sb t0, 15(a2)
-; RV32I-NEXT:    sb a4, 4(a2)
-; RV32I-NEXT:    sb a1, 5(a2)
+; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
 ; RV32I-NEXT:    addi sp, sp, 32
@@ -1186,82 +1168,73 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu t1, 6(a0)
 ; RV32I-NEXT:    lbu t2, 7(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
-; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    lbu a4, 8(a0)
-; RV32I-NEXT:    lbu t3, 9(a0)
-; RV32I-NEXT:    lbu t4, 10(a0)
-; RV32I-NEXT:    lbu t5, 11(a0)
 ; RV32I-NEXT:    slli a5, a5, 16
 ; RV32I-NEXT:    slli a6, a6, 24
 ; RV32I-NEXT:    slli t0, t0, 8
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    lbu a6, 8(a0)
+; RV32I-NEXT:    lbu a7, 9(a0)
+; RV32I-NEXT:    lbu t0, 10(a0)
+; RV32I-NEXT:    lbu t3, 11(a0)
 ; RV32I-NEXT:    slli t1, t1, 16
 ; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    slli a7, a7, 8
+; RV32I-NEXT:    slli t0, t0, 16
+; RV32I-NEXT:    slli t3, t3, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    or a6, a7, a6
+; RV32I-NEXT:    or a7, t3, t0
 ; RV32I-NEXT:    lbu t0, 12(a0)
-; RV32I-NEXT:    lbu t1, 13(a0)
-; RV32I-NEXT:    lbu t2, 14(a0)
-; RV32I-NEXT:    lbu a0, 15(a0)
-; RV32I-NEXT:    slli t3, t3, 8
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or a4, t3, a4
-; RV32I-NEXT:    or t3, t5, t4
-; RV32I-NEXT:    or t0, t1, t0
-; RV32I-NEXT:    lbu t1, 1(a1)
-; RV32I-NEXT:    lbu t4, 0(a1)
-; RV32I-NEXT:    lbu t5, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli t1, t1, 8
-; RV32I-NEXT:    or t1, t1, t4
-; RV32I-NEXT:    slli t5, t5, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or a1, a1, t5
-; RV32I-NEXT:    or a3, a5, a3
-; RV32I-NEXT:    mv a5, sp
-; RV32I-NEXT:    slli t2, t2, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    or t2, a0, t2
-; RV32I-NEXT:    srai a0, a0, 31
+; RV32I-NEXT:    lbu t2, 13(a0)
+; RV32I-NEXT:    lbu t3, 14(a0)
+; RV32I-NEXT:    lbu t4, 15(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    or a1, t2, t0
+; RV32I-NEXT:    mv t0, sp
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    srli a4, a0, 3
+; RV32I-NEXT:    or a5, t1, a5
+; RV32I-NEXT:    andi t1, a0, 31
+; RV32I-NEXT:    or t2, t4, t3
+; RV32I-NEXT:    srai t3, t4, 31
+; RV32I-NEXT:    andi a4, a4, 12
+; RV32I-NEXT:    xori t1, t1, 31
 ; RV32I-NEXT:    or a6, a7, a6
-; RV32I-NEXT:    or a4, t3, a4
-; RV32I-NEXT:    or a7, t2, t0
-; RV32I-NEXT:    or a1, a1, t1
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    sw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    or a1, t2, a1
+; RV32I-NEXT:    sw t3, 16(sp)
+; RV32I-NEXT:    sw t3, 20(sp)
+; RV32I-NEXT:    sw t3, 24(sp)
+; RV32I-NEXT:    sw t3, 28(sp)
+; RV32I-NEXT:    add a4, t0, a4
 ; RV32I-NEXT:    sw a3, 0(sp)
-; RV32I-NEXT:    sw a6, 4(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a7, 12(sp)
-; RV32I-NEXT:    srli a0, a1, 3
-; RV32I-NEXT:    andi a3, a1, 31
-; RV32I-NEXT:    andi a0, a0, 12
-; RV32I-NEXT:    xori a3, a3, 31
-; RV32I-NEXT:    add a0, a5, a0
-; RV32I-NEXT:    lw a4, 4(a0)
-; RV32I-NEXT:    lw a5, 8(a0)
-; RV32I-NEXT:    lw a6, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
-; RV32I-NEXT:    srl a7, a4, a1
-; RV32I-NEXT:    slli t0, a5, 1
-; RV32I-NEXT:    srl a6, a6, a1
-; RV32I-NEXT:    slli a4, a4, 1
-; RV32I-NEXT:    srl a5, a5, a1
-; RV32I-NEXT:    slli t1, a0, 1
-; RV32I-NEXT:    sra a0, a0, a1
-; RV32I-NEXT:    sll a1, t0, a3
-; RV32I-NEXT:    sll a4, a4, a3
-; RV32I-NEXT:    sll a3, t1, a3
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    sw a6, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a1, 4(a4)
+; RV32I-NEXT:    lw a3, 8(a4)
+; RV32I-NEXT:    lw a5, 0(a4)
+; RV32I-NEXT:    lw a4, 12(a4)
+; RV32I-NEXT:    srl a6, a1, a0
+; RV32I-NEXT:    slli a7, a3, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    srl a3, a3, a0
+; RV32I-NEXT:    slli t0, a4, 1
+; RV32I-NEXT:    sra a0, a4, a0
+; RV32I-NEXT:    sll a4, a7, t1
+; RV32I-NEXT:    sll a1, a1, t1
+; RV32I-NEXT:    sll a7, t0, t1
 ; RV32I-NEXT:    srli t0, a0, 16
 ; RV32I-NEXT:    srli t1, a0, 24
 ; RV32I-NEXT:    srli t2, a0, 8
-; RV32I-NEXT:    or a1, a7, a1
 ; RV32I-NEXT:    or a4, a6, a4
-; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a3, a3, a7
 ; RV32I-NEXT:    sb a0, 12(a2)
 ; RV32I-NEXT:    sb t2, 13(a2)
 ; RV32I-NEXT:    sb t0, 14(a2)
@@ -1269,21 +1242,21 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    srli a0, a3, 16
 ; RV32I-NEXT:    srli a5, a3, 24
 ; RV32I-NEXT:    srli a6, a3, 8
-; RV32I-NEXT:    srli a7, a4, 16
-; RV32I-NEXT:    srli t0, a4, 24
-; RV32I-NEXT:    srli t1, a4, 8
-; RV32I-NEXT:    srli t2, a1, 16
-; RV32I-NEXT:    srli t3, a1, 24
+; RV32I-NEXT:    srli a7, a1, 16
+; RV32I-NEXT:    srli t0, a1, 24
+; RV32I-NEXT:    srli t1, a1, 8
+; RV32I-NEXT:    srli t2, a4, 16
+; RV32I-NEXT:    srli t3, a4, 24
 ; RV32I-NEXT:    sb a3, 8(a2)
 ; RV32I-NEXT:    sb a6, 9(a2)
 ; RV32I-NEXT:    sb a0, 10(a2)
 ; RV32I-NEXT:    sb a5, 11(a2)
-; RV32I-NEXT:    srli a0, a1, 8
-; RV32I-NEXT:    sb a4, 0(a2)
+; RV32I-NEXT:    srli a0, a4, 8
+; RV32I-NEXT:    sb a1, 0(a2)
 ; RV32I-NEXT:    sb t1, 1(a2)
 ; RV32I-NEXT:    sb a7, 2(a2)
 ; RV32I-NEXT:    sb t0, 3(a2)
-; RV32I-NEXT:    sb a1, 4(a2)
+; RV32I-NEXT:    sb a4, 4(a2)
 ; RV32I-NEXT:    sb a0, 5(a2)
 ; RV32I-NEXT:    sb t2, 6(a2)
 ; RV32I-NEXT:    sb t3, 7(a2)
@@ -1299,19 +1272,17 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: lshr_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -1328,143 +1299,122 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli s8, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a5, a4, a3
-; RV64I-NEXT:    or a6, a6, s8
-; RV64I-NEXT:    or a3, t0, a7
-; RV64I-NEXT:    or a4, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
-; RV64I-NEXT:    slli s5, s5, 16
-; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
+; RV64I-NEXT:    lbu a0, 0(a1)
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    or a5, a6, a5
-; RV64I-NEXT:    mv a6, sp
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    mv s2, sp
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    slli s5, s5, 16
+; RV64I-NEXT:    slli s6, s6, 24
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
+; RV64I-NEXT:    xori s5, s5, 63
 ; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, t0, a7
-; RV64I-NEXT:    or a7, t2, t1
-; RV64I-NEXT:    or t0, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t1, s0, t6
-; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
-; RV64I-NEXT:    slli a3, a3, 32
-; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t2, t2, 32
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a3, a3, a5
-; RV64I-NEXT:    or a4, a7, a4
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, t2, t1
-; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    add s2, s2, s3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a6, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t1, a1
 ; RV64I-NEXT:    sd a3, 0(sp)
 ; RV64I-NEXT:    sd a4, 8(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    sd a5, 24(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    add a0, a6, a0
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 0(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    ld a1, 8(s2)
+; RV64I-NEXT:    ld a3, 16(s2)
+; RV64I-NEXT:    ld a4, 0(s2)
+; RV64I-NEXT:    ld a5, 24(s2)
+; RV64I-NEXT:    srl a6, a1, a0
+; RV64I-NEXT:    slli a7, a3, 1
+; RV64I-NEXT:    srl a4, a4, a0
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    srl a3, a3, a0
 ; RV64I-NEXT:    slli t0, a5, 1
-; RV64I-NEXT:    srl a6, a6, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a0, 1
-; RV64I-NEXT:    srl t2, a0, a1
-; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a4, a3
-; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a4, t2, 56
-; RV64I-NEXT:    srli t0, t2, 48
-; RV64I-NEXT:    srli t1, t2, 40
-; RV64I-NEXT:    srli t3, t2, 32
-; RV64I-NEXT:    srli t4, t2, 24
-; RV64I-NEXT:    srli t5, t2, 16
-; RV64I-NEXT:    srli t6, t2, 8
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a6, a1
-; RV64I-NEXT:    or a3, a5, a3
+; RV64I-NEXT:    srl a5, a5, a0
+; RV64I-NEXT:    sll a0, a7, s5
+; RV64I-NEXT:    sll a1, a1, s5
+; RV64I-NEXT:    sll a7, t0, s5
+; RV64I-NEXT:    srli t0, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    srli t4, a5, 24
+; RV64I-NEXT:    srli t5, a5, 16
+; RV64I-NEXT:    srli t6, a5, 8
+; RV64I-NEXT:    or a0, a6, a0
+; RV64I-NEXT:    or a1, a4, a1
+; RV64I-NEXT:    or a3, a3, a7
 ; RV64I-NEXT:    sb t3, 28(a2)
-; RV64I-NEXT:    sb t1, 29(a2)
-; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a4, 31(a2)
-; RV64I-NEXT:    sb t2, 24(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb t0, 31(a2)
+; RV64I-NEXT:    sb a5, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
@@ -1513,19 +1463,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb a1, 9(a2)
 ; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: lshr_32bytes:
@@ -1550,67 +1498,55 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s2, 13(a0)
-; RV32I-NEXT:    lbu s4, 14(a0)
-; RV32I-NEXT:    lbu s5, 15(a0)
-; RV32I-NEXT:    lbu s6, 16(a0)
-; RV32I-NEXT:    lbu s7, 17(a0)
-; RV32I-NEXT:    lbu s8, 18(a0)
-; RV32I-NEXT:    lbu s9, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t6, 7(a0)
+; RV32I-NEXT:    lbu s2, 8(a0)
+; RV32I-NEXT:    lbu s3, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s7, 12(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu s9, 14(a0)
+; RV32I-NEXT:    lbu s10, 15(a0)
+; RV32I-NEXT:    lbu s11, 16(a0)
+; RV32I-NEXT:    lbu ra, 17(a0)
+; RV32I-NEXT:    lbu t4, 18(a0)
+; RV32I-NEXT:    lbu s0, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s10, 20(a0)
-; RV32I-NEXT:    lbu s11, 21(a0)
-; RV32I-NEXT:    lbu ra, 22(a0)
-; RV32I-NEXT:    lbu a3, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t6, t6, 24
-; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu s1, 24(a0)
-; RV32I-NEXT:    lbu s3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s3, s3, 8
 ; RV32I-NEXT:    slli s4, s4, 16
 ; RV32I-NEXT:    slli s5, s5, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, s5, s4
-; RV32I-NEXT:    or t3, s7, s6
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s4, 29(a0)
-; RV32I-NEXT:    lbu s5, 30(a0)
-; RV32I-NEXT:    lbu s6, 31(a0)
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli s9, s9, 24
-; RV32I-NEXT:    slli s11, s11, 8
-; RV32I-NEXT:    slli ra, ra, 16
-; RV32I-NEXT:    slli a3, a3, 24
-; RV32I-NEXT:    or a0, s9, s8
-; RV32I-NEXT:    or s0, s11, s10
-; RV32I-NEXT:    or s2, a3, ra
-; RV32I-NEXT:    lbu a3, 0(a1)
-; RV32I-NEXT:    lbu s7, 1(a1)
-; RV32I-NEXT:    lbu s8, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    or a5, t0, a5
+; RV32I-NEXT:    or a6, t6, t3
+; RV32I-NEXT:    or a7, s3, s2
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s5, 25(a0)
+; RV32I-NEXT:    lbu s6, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    slli ra, ra, 8
+; RV32I-NEXT:    or s7, s8, s7
+; RV32I-NEXT:    or s2, s10, s9
+; RV32I-NEXT:    or s3, ra, s11
+; RV32I-NEXT:    lbu s4, 28(a0)
+; RV32I-NEXT:    lbu s8, 29(a0)
+; RV32I-NEXT:    lbu s9, 30(a0)
+; RV32I-NEXT:    lbu s10, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 56(sp)
 ; RV32I-NEXT:    sw zero, 60(sp)
 ; RV32I-NEXT:    sw zero, 64(sp)
@@ -1619,89 +1555,90 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw zero, 44(sp)
 ; RV32I-NEXT:    sw zero, 48(sp)
 ; RV32I-NEXT:    sw zero, 52(sp)
-; RV32I-NEXT:    slli s3, s3, 8
-; RV32I-NEXT:    or s1, s3, s1
-; RV32I-NEXT:    addi s3, sp, 8
 ; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s4, s4, 8
-; RV32I-NEXT:    slli s5, s5, 16
-; RV32I-NEXT:    slli s6, s6, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s4, t6
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a3, s7, a3
-; RV32I-NEXT:    or a1, a1, s8
-; RV32I-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, s4
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, a0, t3
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, t4, s1
-; RV32I-NEXT:    or t3, t6, t5
-; RV32I-NEXT:    or a0, a1, a3
-; RV32I-NEXT:    sw t0, 24(sp)
-; RV32I-NEXT:    sw t1, 28(sp)
-; RV32I-NEXT:    sw t2, 32(sp)
-; RV32I-NEXT:    sw t3, 36(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a5, 12(sp)
-; RV32I-NEXT:    sw a6, 16(sp)
-; RV32I-NEXT:    sw a7, 20(sp)
+; RV32I-NEXT:    slli s0, s0, 24
+; RV32I-NEXT:    or t4, s0, t4
+; RV32I-NEXT:    addi s0, sp, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    slli s6, s6, 16
+; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    or t1, t2, t1
 ; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    add a4, s3, a4
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    lw a5, 4(a4)
-; RV32I-NEXT:    lw a6, 8(a4)
-; RV32I-NEXT:    lw a7, 12(a4)
-; RV32I-NEXT:    lw t0, 16(a4)
-; RV32I-NEXT:    lw t1, 20(a4)
-; RV32I-NEXT:    lw t2, 24(a4)
-; RV32I-NEXT:    lw a4, 28(a4)
-; RV32I-NEXT:    srl t3, a5, a0
-; RV32I-NEXT:    slli t4, a6, 1
+; RV32I-NEXT:    or t2, s1, t5
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s5, t3
+; RV32I-NEXT:    or t6, t6, s6
+; RV32I-NEXT:    or s1, s8, s4
+; RV32I-NEXT:    or s4, s10, s9
+; RV32I-NEXT:    andi s5, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, s2, s7
+; RV32I-NEXT:    or a7, t4, s3
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, t6, t3
+; RV32I-NEXT:    or t2, s4, s1
+; RV32I-NEXT:    add s0, s0, s5
+; RV32I-NEXT:    sw a7, 24(sp)
+; RV32I-NEXT:    sw t0, 28(sp)
+; RV32I-NEXT:    sw t1, 32(sp)
+; RV32I-NEXT:    sw t2, 36(sp)
+; RV32I-NEXT:    sw a3, 8(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a6, 20(sp)
+; RV32I-NEXT:    lw a3, 0(s0)
+; RV32I-NEXT:    lw a4, 4(s0)
+; RV32I-NEXT:    lw a5, 8(s0)
+; RV32I-NEXT:    lw a6, 12(s0)
+; RV32I-NEXT:    lw a7, 16(s0)
+; RV32I-NEXT:    lw t0, 20(s0)
+; RV32I-NEXT:    lw t1, 24(s0)
+; RV32I-NEXT:    lw t2, 28(s0)
+; RV32I-NEXT:    srl t3, a4, a0
+; RV32I-NEXT:    slli t4, a5, 1
 ; RV32I-NEXT:    srl a3, a3, a0
-; RV32I-NEXT:    slli a5, a5, 1
-; RV32I-NEXT:    srl t5, a7, a0
-; RV32I-NEXT:    slli t6, t0, 1
-; RV32I-NEXT:    srl a6, a6, a0
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    srl s0, t1, a0
-; RV32I-NEXT:    slli s1, t2, 1
-; RV32I-NEXT:    srl t0, t0, a0
-; RV32I-NEXT:    slli t1, t1, 1
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl t5, a6, a0
+; RV32I-NEXT:    slli t6, a7, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a6, a6, 1
+; RV32I-NEXT:    srl s0, t0, a0
+; RV32I-NEXT:    slli s1, t1, 1
+; RV32I-NEXT:    srl a7, a7, a0
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    srl t1, t1, a0
+; RV32I-NEXT:    slli s2, t2, 1
 ; RV32I-NEXT:    srl t2, t2, a0
-; RV32I-NEXT:    slli s2, a4, 1
-; RV32I-NEXT:    srl s3, a4, a0
 ; RV32I-NEXT:    sll a0, t4, a1
-; RV32I-NEXT:    sll a4, a5, a1
-; RV32I-NEXT:    sll a5, t6, a1
-; RV32I-NEXT:    sll a7, a7, a1
-; RV32I-NEXT:    sll t4, s1, a1
-; RV32I-NEXT:    sll t1, t1, a1
-; RV32I-NEXT:    sll t6, s2, a1
-; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s2, s3, 16
-; RV32I-NEXT:    srli s4, s3, 8
+; RV32I-NEXT:    sll a4, a4, a1
+; RV32I-NEXT:    sll t4, t6, a1
+; RV32I-NEXT:    sll a6, a6, a1
+; RV32I-NEXT:    sll t6, s1, a1
+; RV32I-NEXT:    sll t0, t0, a1
+; RV32I-NEXT:    sll s1, s2, a1
+; RV32I-NEXT:    srli s2, t2, 24
+; RV32I-NEXT:    srli s3, t2, 16
+; RV32I-NEXT:    srli s4, t2, 8
 ; RV32I-NEXT:    or a0, t3, a0
 ; RV32I-NEXT:    or a1, a3, a4
-; RV32I-NEXT:    or a3, t5, a5
-; RV32I-NEXT:    or a4, a6, a7
-; RV32I-NEXT:    or a5, s0, t4
-; RV32I-NEXT:    or a6, t0, t1
-; RV32I-NEXT:    or a7, t2, t6
-; RV32I-NEXT:    sb s3, 28(a2)
+; RV32I-NEXT:    or a3, t5, t4
+; RV32I-NEXT:    or a4, a5, a6
+; RV32I-NEXT:    or a5, s0, t6
+; RV32I-NEXT:    or a6, a7, t0
+; RV32I-NEXT:    or a7, t1, s1
+; RV32I-NEXT:    sb t2, 28(a2)
 ; RV32I-NEXT:    sb s4, 29(a2)
-; RV32I-NEXT:    sb s2, 30(a2)
-; RV32I-NEXT:    sb s1, 31(a2)
+; RV32I-NEXT:    sb s3, 30(a2)
+; RV32I-NEXT:    sb s2, 31(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
 ; RV32I-NEXT:    srli t2, a7, 8
@@ -1775,19 +1712,17 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: shl_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -1804,146 +1739,125 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli s8, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a5, a4, a3
-; RV64I-NEXT:    or a6, a6, s8
-; RV64I-NEXT:    or a3, t0, a7
-; RV64I-NEXT:    or a4, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
-; RV64I-NEXT:    slli s5, s5, 16
-; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
+; RV64I-NEXT:    lbu a0, 0(a1)
 ; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
-; RV64I-NEXT:    or a5, a6, a5
-; RV64I-NEXT:    addi a6, sp, 32
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    addi s2, sp, 32
+; RV64I-NEXT:    slli s4, s4, 8
+; RV64I-NEXT:    slli s5, s5, 16
+; RV64I-NEXT:    slli s6, s6, 24
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
 ; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, t0, a7
-; RV64I-NEXT:    or a7, t2, t1
-; RV64I-NEXT:    or t0, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t1, s0, t6
-; RV64I-NEXT:    or t2, s5, s1
-; RV64I-NEXT:    or t3, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
-; RV64I-NEXT:    slli a3, a3, 32
-; RV64I-NEXT:    slli a7, a7, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t2, t2, 32
-; RV64I-NEXT:    slli a1, a1, 32
-; RV64I-NEXT:    or a3, a3, a5
-; RV64I-NEXT:    or a4, a7, a4
-; RV64I-NEXT:    or a0, a0, t0
-; RV64I-NEXT:    or a5, t2, t1
-; RV64I-NEXT:    or a1, a1, t3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    sub t2, s2, s3
+; RV64I-NEXT:    slli a4, a4, 32
+; RV64I-NEXT:    slli a6, a6, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t1, t1, 32
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t1, a1
 ; RV64I-NEXT:    sd a3, 32(sp)
 ; RV64I-NEXT:    sd a4, 40(sp)
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a5, 56(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    sub a0, a6, a0
-; RV64I-NEXT:    ld a4, 0(a0)
-; RV64I-NEXT:    ld a5, 8(a0)
-; RV64I-NEXT:    ld a6, 16(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    sll a7, a5, a1
-; RV64I-NEXT:    srli t0, a4, 1
-; RV64I-NEXT:    sll t1, a0, a1
-; RV64I-NEXT:    srli a0, a6, 1
-; RV64I-NEXT:    sll a6, a6, a1
-; RV64I-NEXT:    srli a5, a5, 1
-; RV64I-NEXT:    sll a4, a4, a1
-; RV64I-NEXT:    srl a1, t0, a3
-; RV64I-NEXT:    srl t0, a0, a3
-; RV64I-NEXT:    srl a3, a5, a3
-; RV64I-NEXT:    srli a5, a4, 56
-; RV64I-NEXT:    srli t2, a4, 48
-; RV64I-NEXT:    srli t3, a4, 40
-; RV64I-NEXT:    srli t4, a4, 32
-; RV64I-NEXT:    srli t5, a4, 24
-; RV64I-NEXT:    srli t6, a4, 16
-; RV64I-NEXT:    srli s0, a4, 8
-; RV64I-NEXT:    or a0, a7, a1
-; RV64I-NEXT:    or a1, t1, t0
-; RV64I-NEXT:    or a3, a6, a3
-; RV64I-NEXT:    sb t4, 4(a2)
-; RV64I-NEXT:    sb t3, 5(a2)
-; RV64I-NEXT:    sb t2, 6(a2)
-; RV64I-NEXT:    sb a5, 7(a2)
-; RV64I-NEXT:    sb a4, 0(a2)
-; RV64I-NEXT:    sb s0, 1(a2)
-; RV64I-NEXT:    sb t6, 2(a2)
-; RV64I-NEXT:    sb t5, 3(a2)
+; RV64I-NEXT:    sd a5, 48(sp)
+; RV64I-NEXT:    sd a1, 56(sp)
+; RV64I-NEXT:    ld a1, 0(t2)
+; RV64I-NEXT:    ld a3, 8(t2)
+; RV64I-NEXT:    ld a4, 16(t2)
+; RV64I-NEXT:    ld a5, 24(t2)
+; RV64I-NEXT:    xori a6, s5, 63
+; RV64I-NEXT:    sll a7, a3, a0
+; RV64I-NEXT:    srli t0, a1, 1
+; RV64I-NEXT:    sll a5, a5, a0
+; RV64I-NEXT:    srli t1, a4, 1
+; RV64I-NEXT:    sll a4, a4, a0
+; RV64I-NEXT:    srli a3, a3, 1
+; RV64I-NEXT:    sll t2, a1, a0
+; RV64I-NEXT:    srl a0, t0, a6
+; RV64I-NEXT:    srl a1, t1, a6
+; RV64I-NEXT:    srl a3, a3, a6
+; RV64I-NEXT:    srli a6, t2, 56
+; RV64I-NEXT:    srli t0, t2, 48
+; RV64I-NEXT:    srli t1, t2, 40
+; RV64I-NEXT:    srli t3, t2, 32
+; RV64I-NEXT:    srli t4, t2, 24
+; RV64I-NEXT:    srli t5, t2, 16
+; RV64I-NEXT:    srli t6, t2, 8
+; RV64I-NEXT:    or a0, a7, a0
+; RV64I-NEXT:    or a1, a5, a1
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    sb t3, 4(a2)
+; RV64I-NEXT:    sb t1, 5(a2)
+; RV64I-NEXT:    sb t0, 6(a2)
+; RV64I-NEXT:    sb a6, 7(a2)
+; RV64I-NEXT:    sb t2, 0(a2)
+; RV64I-NEXT:    sb t6, 1(a2)
+; RV64I-NEXT:    sb t5, 2(a2)
+; RV64I-NEXT:    sb t4, 3(a2)
 ; RV64I-NEXT:    srli a4, a3, 56
 ; RV64I-NEXT:    srli a5, a3, 48
 ; RV64I-NEXT:    srli a6, a3, 40
@@ -1989,19 +1903,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    sb a1, 9(a2)
 ; RV64I-NEXT:    sb a5, 10(a2)
 ; RV64I-NEXT:    sb a3, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: shl_32bytes:
@@ -2026,67 +1938,55 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s2, 13(a0)
-; RV32I-NEXT:    lbu s4, 14(a0)
-; RV32I-NEXT:    lbu s5, 15(a0)
-; RV32I-NEXT:    lbu s6, 16(a0)
-; RV32I-NEXT:    lbu s7, 17(a0)
-; RV32I-NEXT:    lbu s8, 18(a0)
-; RV32I-NEXT:    lbu s9, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t6, 7(a0)
+; RV32I-NEXT:    lbu s2, 8(a0)
+; RV32I-NEXT:    lbu s3, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s7, 12(a0)
+; RV32I-NEXT:    lbu s8, 13(a0)
+; RV32I-NEXT:    lbu s9, 14(a0)
+; RV32I-NEXT:    lbu s10, 15(a0)
+; RV32I-NEXT:    lbu s11, 16(a0)
+; RV32I-NEXT:    lbu ra, 17(a0)
+; RV32I-NEXT:    lbu t4, 18(a0)
+; RV32I-NEXT:    lbu s0, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s10, 20(a0)
-; RV32I-NEXT:    lbu s11, 21(a0)
-; RV32I-NEXT:    lbu ra, 22(a0)
-; RV32I-NEXT:    lbu a3, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli t3, t3, 16
 ; RV32I-NEXT:    slli t6, t6, 24
-; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu s1, 24(a0)
-; RV32I-NEXT:    lbu s3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s2, s2, 8
+; RV32I-NEXT:    slli s3, s3, 8
 ; RV32I-NEXT:    slli s4, s4, 16
 ; RV32I-NEXT:    slli s5, s5, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, s5, s4
-; RV32I-NEXT:    or t3, s7, s6
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s4, 29(a0)
-; RV32I-NEXT:    lbu s5, 30(a0)
-; RV32I-NEXT:    lbu s6, 31(a0)
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli s9, s9, 24
-; RV32I-NEXT:    slli s11, s11, 8
-; RV32I-NEXT:    slli ra, ra, 16
-; RV32I-NEXT:    slli a3, a3, 24
-; RV32I-NEXT:    or a0, s9, s8
-; RV32I-NEXT:    or s0, s11, s10
-; RV32I-NEXT:    or s2, a3, ra
-; RV32I-NEXT:    lbu a3, 0(a1)
-; RV32I-NEXT:    lbu s7, 1(a1)
-; RV32I-NEXT:    lbu s8, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
+; RV32I-NEXT:    or a5, t0, a5
+; RV32I-NEXT:    or a6, t6, t3
+; RV32I-NEXT:    or a7, s3, s2
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s5, 25(a0)
+; RV32I-NEXT:    lbu s6, 26(a0)
+; RV32I-NEXT:    lbu t6, 27(a0)
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    slli ra, ra, 8
+; RV32I-NEXT:    or s7, s8, s7
+; RV32I-NEXT:    or s2, s10, s9
+; RV32I-NEXT:    or s3, ra, s11
+; RV32I-NEXT:    lbu s4, 28(a0)
+; RV32I-NEXT:    lbu s8, 29(a0)
+; RV32I-NEXT:    lbu s9, 30(a0)
+; RV32I-NEXT:    lbu s10, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    sw zero, 24(sp)
 ; RV32I-NEXT:    sw zero, 28(sp)
 ; RV32I-NEXT:    sw zero, 32(sp)
@@ -2095,88 +1995,89 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    sw zero, 12(sp)
 ; RV32I-NEXT:    sw zero, 16(sp)
 ; RV32I-NEXT:    sw zero, 20(sp)
-; RV32I-NEXT:    slli s3, s3, 8
-; RV32I-NEXT:    or s1, s3, s1
-; RV32I-NEXT:    addi s3, sp, 40
 ; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s4, s4, 8
-; RV32I-NEXT:    slli s5, s5, 16
-; RV32I-NEXT:    slli s6, s6, 24
-; RV32I-NEXT:    slli s7, s7, 8
-; RV32I-NEXT:    slli s8, s8, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s4, t6
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a3, s7, a3
-; RV32I-NEXT:    or a1, a1, s8
-; RV32I-NEXT:    lw s4, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, s4
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, a0, t3
-; RV32I-NEXT:    or t1, s2, s0
-; RV32I-NEXT:    or t2, t4, s1
-; RV32I-NEXT:    or t3, t6, t5
-; RV32I-NEXT:    or a0, a1, a3
-; RV32I-NEXT:    sw t0, 56(sp)
-; RV32I-NEXT:    sw t1, 60(sp)
-; RV32I-NEXT:    sw t2, 64(sp)
-; RV32I-NEXT:    sw t3, 68(sp)
-; RV32I-NEXT:    sw a4, 40(sp)
-; RV32I-NEXT:    sw a5, 44(sp)
-; RV32I-NEXT:    sw a6, 48(sp)
-; RV32I-NEXT:    sw a7, 52(sp)
+; RV32I-NEXT:    slli s0, s0, 24
+; RV32I-NEXT:    or t4, s0, t4
+; RV32I-NEXT:    addi s0, sp, 40
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s5, s5, 8
+; RV32I-NEXT:    slli s6, s6, 16
+; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli s8, s8, 8
+; RV32I-NEXT:    slli s9, s9, 16
+; RV32I-NEXT:    slli s10, s10, 24
+; RV32I-NEXT:    or t1, t2, t1
 ; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    sub a3, s3, a4
-; RV32I-NEXT:    lw a4, 0(a3)
-; RV32I-NEXT:    lw a5, 4(a3)
-; RV32I-NEXT:    lw a6, 8(a3)
-; RV32I-NEXT:    lw a7, 12(a3)
-; RV32I-NEXT:    lw t0, 16(a3)
-; RV32I-NEXT:    lw t1, 20(a3)
-; RV32I-NEXT:    lw t2, 24(a3)
-; RV32I-NEXT:    lw a3, 28(a3)
-; RV32I-NEXT:    sll t3, a5, a0
-; RV32I-NEXT:    srli t4, a4, 1
-; RV32I-NEXT:    sll t5, a7, a0
-; RV32I-NEXT:    srli t6, a6, 1
-; RV32I-NEXT:    sll a6, a6, a0
-; RV32I-NEXT:    srli a5, a5, 1
-; RV32I-NEXT:    sll s0, t1, a0
-; RV32I-NEXT:    srli s1, t0, 1
-; RV32I-NEXT:    sll t0, t0, a0
-; RV32I-NEXT:    srli a7, a7, 1
-; RV32I-NEXT:    sll s2, a3, a0
-; RV32I-NEXT:    srli a3, t2, 1
+; RV32I-NEXT:    or t2, s1, t5
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s5, t3
+; RV32I-NEXT:    or t6, t6, s6
+; RV32I-NEXT:    or s1, s8, s4
+; RV32I-NEXT:    or s4, s10, s9
+; RV32I-NEXT:    andi s5, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, s2, s7
+; RV32I-NEXT:    or a7, t4, s3
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, t6, t3
+; RV32I-NEXT:    or t2, s4, s1
+; RV32I-NEXT:    sub t3, s0, s5
+; RV32I-NEXT:    sw a7, 56(sp)
+; RV32I-NEXT:    sw t0, 60(sp)
+; RV32I-NEXT:    sw t1, 64(sp)
+; RV32I-NEXT:    sw t2, 68(sp)
+; RV32I-NEXT:    sw a3, 40(sp)
+; RV32I-NEXT:    sw a4, 44(sp)
+; RV32I-NEXT:    sw a5, 48(sp)
+; RV32I-NEXT:    sw a6, 52(sp)
+; RV32I-NEXT:    lw a3, 0(t3)
+; RV32I-NEXT:    lw a4, 4(t3)
+; RV32I-NEXT:    lw a5, 8(t3)
+; RV32I-NEXT:    lw a6, 12(t3)
+; RV32I-NEXT:    lw a7, 16(t3)
+; RV32I-NEXT:    lw t0, 20(t3)
+; RV32I-NEXT:    lw t1, 24(t3)
+; RV32I-NEXT:    lw t2, 28(t3)
+; RV32I-NEXT:    sll t3, a4, a0
+; RV32I-NEXT:    srli t4, a3, 1
+; RV32I-NEXT:    sll t5, a6, a0
+; RV32I-NEXT:    srli t6, a5, 1
+; RV32I-NEXT:    sll a5, a5, a0
+; RV32I-NEXT:    srli a4, a4, 1
+; RV32I-NEXT:    sll s0, t0, a0
+; RV32I-NEXT:    srli s1, a7, 1
+; RV32I-NEXT:    sll a7, a7, a0
+; RV32I-NEXT:    srli a6, a6, 1
 ; RV32I-NEXT:    sll t2, t2, a0
-; RV32I-NEXT:    srli t1, t1, 1
-; RV32I-NEXT:    sll s3, a4, a0
+; RV32I-NEXT:    srli s2, t1, 1
+; RV32I-NEXT:    sll t1, t1, a0
+; RV32I-NEXT:    srli t0, t0, 1
+; RV32I-NEXT:    sll s3, a3, a0
 ; RV32I-NEXT:    srl a0, t4, a1
-; RV32I-NEXT:    srl a4, t6, a1
-; RV32I-NEXT:    srl a5, a5, a1
+; RV32I-NEXT:    srl a3, t6, a1
+; RV32I-NEXT:    srl a4, a4, a1
 ; RV32I-NEXT:    srl t4, s1, a1
-; RV32I-NEXT:    srl a7, a7, a1
-; RV32I-NEXT:    srl t6, a3, a1
-; RV32I-NEXT:    srl t1, t1, a1
+; RV32I-NEXT:    srl a6, a6, a1
+; RV32I-NEXT:    srl t6, s2, a1
+; RV32I-NEXT:    srl t0, t0, a1
 ; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s4, s3, 16
-; RV32I-NEXT:    srli s5, s3, 8
+; RV32I-NEXT:    srli s2, s3, 16
+; RV32I-NEXT:    srli s4, s3, 8
 ; RV32I-NEXT:    or a0, t3, a0
-; RV32I-NEXT:    or a1, t5, a4
-; RV32I-NEXT:    or a3, a6, a5
+; RV32I-NEXT:    or a1, t5, a3
+; RV32I-NEXT:    or a3, a5, a4
 ; RV32I-NEXT:    or a4, s0, t4
-; RV32I-NEXT:    or a5, t0, a7
-; RV32I-NEXT:    or a6, s2, t6
-; RV32I-NEXT:    or a7, t2, t1
+; RV32I-NEXT:    or a5, a7, a6
+; RV32I-NEXT:    or a6, t2, t6
+; RV32I-NEXT:    or a7, t1, t0
 ; RV32I-NEXT:    sb s3, 0(a2)
-; RV32I-NEXT:    sb s5, 1(a2)
-; RV32I-NEXT:    sb s4, 2(a2)
+; RV32I-NEXT:    sb s4, 1(a2)
+; RV32I-NEXT:    sb s2, 2(a2)
 ; RV32I-NEXT:    sb s1, 3(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
@@ -2251,19 +2152,17 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-LABEL: ashr_32bytes:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd s0, 152(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s1, 144(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s2, 136(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s3, 128(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s4, 120(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s5, 112(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s6, 104(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s7, 96(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s8, 88(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s9, 80(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s10, 72(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    sd s11, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    addi sp, sp, -144
+; RV64I-NEXT:    sd s0, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    lbu a4, 1(a0)
 ; RV64I-NEXT:    lbu a5, 2(a0)
@@ -2280,144 +2179,123 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    lbu s1, 13(a0)
 ; RV64I-NEXT:    lbu s2, 14(a0)
 ; RV64I-NEXT:    lbu s3, 15(a0)
+; RV64I-NEXT:    slli a4, a4, 8
+; RV64I-NEXT:    slli a5, a5, 16
+; RV64I-NEXT:    slli a6, a6, 24
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    lbu s4, 16(a0)
 ; RV64I-NEXT:    lbu s5, 17(a0)
 ; RV64I-NEXT:    lbu s6, 18(a0)
 ; RV64I-NEXT:    lbu s7, 19(a0)
-; RV64I-NEXT:    slli a4, a4, 8
-; RV64I-NEXT:    slli a5, a5, 16
-; RV64I-NEXT:    slli a6, a6, 24
 ; RV64I-NEXT:    slli t0, t0, 8
 ; RV64I-NEXT:    slli t1, t1, 16
 ; RV64I-NEXT:    slli t2, t2, 24
-; RV64I-NEXT:    or a3, a4, a3
-; RV64I-NEXT:    or a4, a6, a5
-; RV64I-NEXT:    or a5, t0, a7
-; RV64I-NEXT:    or a6, t2, t1
-; RV64I-NEXT:    lbu s8, 20(a0)
-; RV64I-NEXT:    lbu s9, 21(a0)
-; RV64I-NEXT:    lbu s10, 22(a0)
-; RV64I-NEXT:    lbu s11, 23(a0)
 ; RV64I-NEXT:    slli t4, t4, 8
 ; RV64I-NEXT:    slli t5, t5, 16
 ; RV64I-NEXT:    slli t6, t6, 24
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a6, t2, t1
+; RV64I-NEXT:    or a7, t4, t3
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    lbu t5, 20(a0)
+; RV64I-NEXT:    lbu t6, 21(a0)
+; RV64I-NEXT:    lbu s8, 22(a0)
+; RV64I-NEXT:    lbu s9, 23(a0)
 ; RV64I-NEXT:    slli s1, s1, 8
 ; RV64I-NEXT:    slli s2, s2, 16
 ; RV64I-NEXT:    slli s3, s3, 24
-; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or t0, t6, t5
-; RV64I-NEXT:    or t1, s1, s0
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    lbu t6, 24(a0)
-; RV64I-NEXT:    lbu s0, 25(a0)
-; RV64I-NEXT:    lbu s1, 26(a0)
-; RV64I-NEXT:    lbu s2, 27(a0)
 ; RV64I-NEXT:    slli s5, s5, 8
 ; RV64I-NEXT:    slli s6, s6, 16
 ; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    slli s9, s9, 8
+; RV64I-NEXT:    or t1, s1, s0
+; RV64I-NEXT:    or t2, s3, s2
 ; RV64I-NEXT:    or t3, s5, s4
 ; RV64I-NEXT:    or t4, s7, s6
-; RV64I-NEXT:    or t5, s9, s8
-; RV64I-NEXT:    lbu s3, 28(a0)
+; RV64I-NEXT:    lbu s0, 24(a0)
+; RV64I-NEXT:    lbu s1, 25(a0)
+; RV64I-NEXT:    lbu s2, 26(a0)
+; RV64I-NEXT:    lbu s3, 27(a0)
+; RV64I-NEXT:    slli t6, t6, 8
+; RV64I-NEXT:    slli s8, s8, 16
+; RV64I-NEXT:    slli s9, s9, 24
+; RV64I-NEXT:    slli s1, s1, 8
+; RV64I-NEXT:    or t5, t6, t5
+; RV64I-NEXT:    or t6, s9, s8
+; RV64I-NEXT:    or s0, s1, s0
+; RV64I-NEXT:    lbu s1, 28(a0)
 ; RV64I-NEXT:    lbu s4, 29(a0)
 ; RV64I-NEXT:    lbu s5, 30(a0)
 ; RV64I-NEXT:    lbu s6, 31(a0)
-; RV64I-NEXT:    slli s10, s10, 16
-; RV64I-NEXT:    slli s11, s11, 24
-; RV64I-NEXT:    slli s0, s0, 8
-; RV64I-NEXT:    slli s1, s1, 16
-; RV64I-NEXT:    slli s2, s2, 24
+; RV64I-NEXT:    lbu a0, 0(a1)
+; RV64I-NEXT:    slli s2, s2, 16
+; RV64I-NEXT:    slli s3, s3, 24
+; RV64I-NEXT:    or a1, s3, s2
+; RV64I-NEXT:    mv s2, sp
 ; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or a0, s11, s10
-; RV64I-NEXT:    or t6, s0, t6
-; RV64I-NEXT:    or s0, s2, s1
-; RV64I-NEXT:    or s1, s4, s3
-; RV64I-NEXT:    lbu s2, 0(a1)
-; RV64I-NEXT:    lbu s3, 1(a1)
-; RV64I-NEXT:    lbu s4, 2(a1)
-; RV64I-NEXT:    lbu s7, 3(a1)
 ; RV64I-NEXT:    slli s5, s5, 16
 ; RV64I-NEXT:    slli s6, s6, 24
-; RV64I-NEXT:    slli s3, s3, 8
-; RV64I-NEXT:    slli s4, s4, 16
-; RV64I-NEXT:    slli s7, s7, 24
-; RV64I-NEXT:    or s5, s6, s5
-; RV64I-NEXT:    or s2, s3, s2
-; RV64I-NEXT:    or s3, s7, s4
-; RV64I-NEXT:    lbu s4, 5(a1)
-; RV64I-NEXT:    lbu s6, 4(a1)
-; RV64I-NEXT:    lbu s7, 6(a1)
-; RV64I-NEXT:    lbu a1, 7(a1)
-; RV64I-NEXT:    slli s4, s4, 8
-; RV64I-NEXT:    or s4, s4, s6
-; RV64I-NEXT:    slli s7, s7, 16
-; RV64I-NEXT:    slli a1, a1, 24
-; RV64I-NEXT:    or a1, a1, s7
-; RV64I-NEXT:    mv s6, sp
+; RV64I-NEXT:    or s1, s4, s1
+; RV64I-NEXT:    srli s3, a0, 3
+; RV64I-NEXT:    or s4, s6, s5
+; RV64I-NEXT:    andi s5, a0, 63
+; RV64I-NEXT:    andi s3, s3, 24
+; RV64I-NEXT:    xori s5, s5, 63
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, a6, a5
 ; RV64I-NEXT:    or a5, t0, a7
 ; RV64I-NEXT:    or a6, t2, t1
 ; RV64I-NEXT:    or a7, t4, t3
-; RV64I-NEXT:    or a0, a0, t5
-; RV64I-NEXT:    or t0, s0, t6
-; RV64I-NEXT:    or t1, s5, s1
-; RV64I-NEXT:    or t2, s3, s2
-; RV64I-NEXT:    or a1, a1, s4
+; RV64I-NEXT:    or t0, t6, t5
+; RV64I-NEXT:    or a1, a1, s0
+; RV64I-NEXT:    or t1, s4, s1
+; RV64I-NEXT:    add s2, s2, s3
 ; RV64I-NEXT:    slli a4, a4, 32
 ; RV64I-NEXT:    slli a6, a6, 32
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    slli t3, t1, 32
-; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    slli t0, t0, 32
+; RV64I-NEXT:    slli t2, t1, 32
 ; RV64I-NEXT:    sraiw t1, t1, 31
 ; RV64I-NEXT:    or a3, a4, a3
 ; RV64I-NEXT:    or a4, a6, a5
-; RV64I-NEXT:    or a0, a0, a7
-; RV64I-NEXT:    or a5, t3, t0
-; RV64I-NEXT:    or a1, a1, t2
+; RV64I-NEXT:    or a5, t0, a7
+; RV64I-NEXT:    or a1, t2, a1
 ; RV64I-NEXT:    sd t1, 32(sp)
 ; RV64I-NEXT:    sd t1, 40(sp)
 ; RV64I-NEXT:    sd t1, 48(sp)
 ; RV64I-NEXT:    sd t1, 56(sp)
 ; RV64I-NEXT:    sd a3, 0(sp)
 ; RV64I-NEXT:    sd a4, 8(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    sd a5, 24(sp)
-; RV64I-NEXT:    srli a0, a1, 3
-; RV64I-NEXT:    andi a3, a1, 63
-; RV64I-NEXT:    andi a0, a0, 24
-; RV64I-NEXT:    xori a3, a3, 63
-; RV64I-NEXT:    add a0, s6, a0
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    ld a5, 16(a0)
-; RV64I-NEXT:    ld a6, 0(a0)
-; RV64I-NEXT:    ld a0, 24(a0)
-; RV64I-NEXT:    srl a7, a4, a1
+; RV64I-NEXT:    sd a5, 16(sp)
+; RV64I-NEXT:    sd a1, 24(sp)
+; RV64I-NEXT:    ld a1, 8(s2)
+; RV64I-NEXT:    ld a3, 16(s2)
+; RV64I-NEXT:    ld a4, 0(s2)
+; RV64I-NEXT:    ld a5, 24(s2)
+; RV64I-NEXT:    srl a6, a1, a0
+; RV64I-NEXT:    slli a7, a3, 1
+; RV64I-NEXT:    srl a4, a4, a0
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    srl a3, a3, a0
 ; RV64I-NEXT:    slli t0, a5, 1
-; RV64I-NEXT:    srl a6, a6, a1
-; RV64I-NEXT:    slli a4, a4, 1
-; RV64I-NEXT:    srl a5, a5, a1
-; RV64I-NEXT:    slli t1, a0, 1
-; RV64I-NEXT:    sra t2, a0, a1
-; RV64I-NEXT:    sll a0, t0, a3
-; RV64I-NEXT:    sll a1, a4, a3
-; RV64I-NEXT:    sll a3, t1, a3
-; RV64I-NEXT:    srli a4, t2, 56
-; RV64I-NEXT:    srli t0, t2, 48
-; RV64I-NEXT:    srli t1, t2, 40
-; RV64I-NEXT:    srli t3, t2, 32
-; RV64I-NEXT:    srli t4, t2, 24
-; RV64I-NEXT:    srli t5, t2, 16
-; RV64I-NEXT:    srli t6, t2, 8
-; RV64I-NEXT:    or a0, a7, a0
-; RV64I-NEXT:    or a1, a6, a1
-; RV64I-NEXT:    or a3, a5, a3
+; RV64I-NEXT:    sra a5, a5, a0
+; RV64I-NEXT:    sll a0, a7, s5
+; RV64I-NEXT:    sll a1, a1, s5
+; RV64I-NEXT:    sll a7, t0, s5
+; RV64I-NEXT:    srli t0, a5, 56
+; RV64I-NEXT:    srli t1, a5, 48
+; RV64I-NEXT:    srli t2, a5, 40
+; RV64I-NEXT:    srli t3, a5, 32
+; RV64I-NEXT:    srli t4, a5, 24
+; RV64I-NEXT:    srli t5, a5, 16
+; RV64I-NEXT:    srli t6, a5, 8
+; RV64I-NEXT:    or a0, a6, a0
+; RV64I-NEXT:    or a1, a4, a1
+; RV64I-NEXT:    or a3, a3, a7
 ; RV64I-NEXT:    sb t3, 28(a2)
-; RV64I-NEXT:    sb t1, 29(a2)
-; RV64I-NEXT:    sb t0, 30(a2)
-; RV64I-NEXT:    sb a4, 31(a2)
-; RV64I-NEXT:    sb t2, 24(a2)
+; RV64I-NEXT:    sb t2, 29(a2)
+; RV64I-NEXT:    sb t1, 30(a2)
+; RV64I-NEXT:    sb t0, 31(a2)
+; RV64I-NEXT:    sb a5, 24(a2)
 ; RV64I-NEXT:    sb t6, 25(a2)
 ; RV64I-NEXT:    sb t5, 26(a2)
 ; RV64I-NEXT:    sb t4, 27(a2)
@@ -2438,47 +2316,45 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV64I-NEXT:    srli s3, a0, 56
 ; RV64I-NEXT:    srli s4, a0, 48
 ; RV64I-NEXT:    srli s5, a0, 40
-; RV64I-NEXT:    srli s6, a0, 32
 ; RV64I-NEXT:    sb a7, 20(a2)
 ; RV64I-NEXT:    sb a6, 21(a2)
 ; RV64I-NEXT:    sb a5, 22(a2)
 ; RV64I-NEXT:    sb a4, 23(a2)
-; RV64I-NEXT:    srli a4, a0, 24
+; RV64I-NEXT:    srli a4, a0, 32
 ; RV64I-NEXT:    sb a3, 16(a2)
 ; RV64I-NEXT:    sb t2, 17(a2)
 ; RV64I-NEXT:    sb t1, 18(a2)
 ; RV64I-NEXT:    sb t0, 19(a2)
-; RV64I-NEXT:    srli a3, a0, 16
+; RV64I-NEXT:    srli a3, a0, 24
 ; RV64I-NEXT:    sb t6, 4(a2)
 ; RV64I-NEXT:    sb t5, 5(a2)
 ; RV64I-NEXT:    sb t4, 6(a2)
 ; RV64I-NEXT:    sb t3, 7(a2)
-; RV64I-NEXT:    srli a5, a0, 8
+; RV64I-NEXT:    srli a5, a0, 16
 ; RV64I-NEXT:    sb a1, 0(a2)
 ; RV64I-NEXT:    sb s2, 1(a2)
 ; RV64I-NEXT:    sb s1, 2(a2)
 ; RV64I-NEXT:    sb s0, 3(a2)
-; RV64I-NEXT:    sb s6, 12(a2)
+; RV64I-NEXT:    srli a1, a0, 8
+; RV64I-NEXT:    sb a4, 12(a2)
 ; RV64I-NEXT:    sb s5, 13(a2)
 ; RV64I-NEXT:    sb s4, 14(a2)
 ; RV64I-NEXT:    sb s3, 15(a2)
 ; RV64I-NEXT:    sb a0, 8(a2)
-; RV64I-NEXT:    sb a5, 9(a2)
-; RV64I-NEXT:    sb a3, 10(a2)
-; RV64I-NEXT:    sb a4, 11(a2)
-; RV64I-NEXT:    ld s0, 152(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s1, 144(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s2, 136(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s3, 128(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s4, 120(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s5, 112(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s6, 104(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s7, 96(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s8, 88(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s9, 80(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s10, 72(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    ld s11, 64(sp) # 8-byte Folded Reload
-; RV64I-NEXT:    addi sp, sp, 160
+; RV64I-NEXT:    sb a1, 9(a2)
+; RV64I-NEXT:    sb a5, 10(a2)
+; RV64I-NEXT:    sb a3, 11(a2)
+; RV64I-NEXT:    ld s0, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 144
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: ashr_32bytes:
@@ -2503,159 +2379,148 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
 ; RV32I-NEXT:    lbu a7, 3(a0)
 ; RV32I-NEXT:    lbu a5, 4(a0)
 ; RV32I-NEXT:    lbu t0, 5(a0)
-; RV32I-NEXT:    lbu t1, 6(a0)
-; RV32I-NEXT:    lbu t2, 7(a0)
-; RV32I-NEXT:    lbu t3, 8(a0)
-; RV32I-NEXT:    lbu t4, 9(a0)
-; RV32I-NEXT:    lbu t5, 10(a0)
-; RV32I-NEXT:    lbu t6, 11(a0)
-; RV32I-NEXT:    lbu s0, 12(a0)
-; RV32I-NEXT:    lbu s1, 13(a0)
-; RV32I-NEXT:    lbu s2, 14(a0)
-; RV32I-NEXT:    lbu s3, 15(a0)
-; RV32I-NEXT:    lbu s4, 16(a0)
-; RV32I-NEXT:    lbu s5, 17(a0)
-; RV32I-NEXT:    lbu s6, 18(a0)
-; RV32I-NEXT:    lbu s7, 19(a0)
+; RV32I-NEXT:    lbu t3, 6(a0)
+; RV32I-NEXT:    lbu t4, 7(a0)
+; RV32I-NEXT:    lbu t6, 8(a0)
+; RV32I-NEXT:    lbu s0, 9(a0)
+; RV32I-NEXT:    lbu s4, 10(a0)
+; RV32I-NEXT:    lbu s5, 11(a0)
+; RV32I-NEXT:    lbu s6, 12(a0)
+; RV32I-NEXT:    lbu s7, 13(a0)
+; RV32I-NEXT:    lbu s8, 14(a0)
+; RV32I-NEXT:    lbu s9, 15(a0)
+; RV32I-NEXT:    lbu s10, 16(a0)
+; RV32I-NEXT:    lbu s11, 17(a0)
+; RV32I-NEXT:    lbu s2, 18(a0)
+; RV32I-NEXT:    lbu s3, 19(a0)
 ; RV32I-NEXT:    slli a4, a4, 8
 ; RV32I-NEXT:    slli a6, a6, 16
 ; RV32I-NEXT:    slli a7, a7, 24
 ; RV32I-NEXT:    or a3, a4, a3
-; RV32I-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a4, a7, a6
-; RV32I-NEXT:    lbu s8, 20(a0)
-; RV32I-NEXT:    lbu s9, 21(a0)
-; RV32I-NEXT:    lbu s10, 22(a0)
-; RV32I-NEXT:    lbu s11, 23(a0)
+; RV32I-NEXT:    lbu t1, 20(a0)
+; RV32I-NEXT:    lbu t2, 21(a0)
+; RV32I-NEXT:    lbu t5, 22(a0)
+; RV32I-NEXT:    lbu s1, 23(a0)
 ; RV32I-NEXT:    slli t0, t0, 8
-; RV32I-NEXT:    slli t1, t1, 16
-; RV32I-NEXT:    slli t2, t2, 24
-; RV32I-NEXT:    slli t4, t4, 8
-; RV32I-NEXT:    slli t5, t5, 16
-; RV32I-NEXT:    slli t6, t6, 24
+; RV32I-NEXT:    slli t3, t3, 16
+; RV32I-NEXT:    slli t4, t4, 24
+; RV32I-NEXT:    slli s0, s0, 8
+; RV32I-NEXT:    slli s4, s4, 16
+; RV32I-NEXT:    slli s5, s5, 24
 ; RV32I-NEXT:    or a5, t0, a5
-; RV32I-NEXT:    or a6, t2, t1
-; RV32I-NEXT:    or a7, t4, t3
-; RV32I-NEXT:    or t0, t6, t5
-; RV32I-NEXT:    lbu ra, 24(a0)
-; RV32I-NEXT:    lbu a3, 25(a0)
-; RV32I-NEXT:    lbu t4, 26(a0)
-; RV32I-NEXT:    lbu t5, 27(a0)
-; RV32I-NEXT:    slli s1, s1, 8
+; RV32I-NEXT:    or a6, t4, t3
+; RV32I-NEXT:    or a7, s0, t6
+; RV32I-NEXT:    or t0, s5, s4
+; RV32I-NEXT:    lbu t3, 24(a0)
+; RV32I-NEXT:    lbu s4, 25(a0)
+; RV32I-NEXT:    lbu s5, 26(a0)
+; RV32I-NEXT:    lbu ra, 27(a0)
+; RV32I-NEXT:    slli s7, s7, 8
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    slli s9, s9, 24
+; RV32I-NEXT:    slli s11, s11, 8
+; RV32I-NEXT:    or t4, s7, s6
+; RV32I-NEXT:    or t6, s9, s8
+; RV32I-NEXT:    or s0, s11, s10
+; RV32I-NEXT:    lbu s6, 28(a0)
+; RV32I-NEXT:    lbu s7, 29(a0)
+; RV32I-NEXT:    lbu s8, 30(a0)
+; RV32I-NEXT:    lbu s9, 31(a0)
+; RV32I-NEXT:    lbu a0, 0(a1)
 ; RV32I-NEXT:    slli s2, s2, 16
 ; RV32I-NEXT:    slli s3, s3, 24
-; RV32I-NEXT:    slli s5, s5, 8
-; RV32I-NEXT:    or t1, s1, s0
-; RV32I-NEXT:    or t2, s3, s2
-; RV32I-NEXT:    or t3, s5, s4
-; RV32I-NEXT:    lbu t6, 28(a0)
-; RV32I-NEXT:    lbu s0, 29(a0)
-; RV32I-NEXT:    lbu s1, 30(a0)
-; RV32I-NEXT:    lbu a0, 31(a0)
-; RV32I-NEXT:    slli s6, s6, 16
-; RV32I-NEXT:    slli s7, s7, 24
-; RV32I-NEXT:    slli s9, s9, 8
-; RV32I-NEXT:    slli s10, s10, 16
-; RV32I-NEXT:    slli s11, s11, 24
-; RV32I-NEXT:    or s2, s7, s6
-; RV32I-NEXT:    or s3, s9, s8
-; RV32I-NEXT:    or s4, s11, s10
-; RV32I-NEXT:    lbu s5, 0(a1)
-; RV32I-NEXT:    lbu s6, 1(a1)
-; RV32I-NEXT:    lbu s7, 2(a1)
-; RV32I-NEXT:    lbu a1, 3(a1)
-; RV32I-NEXT:    slli a3, a3, 8
-; RV32I-NEXT:    or a3, a3, ra
-; RV32I-NEXT:    addi s8, sp, 8
-; RV32I-NEXT:    slli t4, t4, 16
-; RV32I-NEXT:    slli t5, t5, 24
-; RV32I-NEXT:    slli s0, s0, 8
-; RV32I-NEXT:    slli s1, s1, 16
-; RV32I-NEXT:    slli a0, a0, 24
-; RV32I-NEXT:    slli s6, s6, 8
-; RV32I-NEXT:    slli s7, s7, 16
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    or t4, t5, t4
-; RV32I-NEXT:    or t5, s0, t6
-; RV32I-NEXT:    or s1, a0, s1
-; RV32I-NEXT:    or t6, s6, s5
-; RV32I-NEXT:    or a1, a1, s7
-; RV32I-NEXT:    srai s0, a0, 31
-; RV32I-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
-; RV32I-NEXT:    or a4, a4, a0
-; RV32I-NEXT:    or a5, a6, a5
-; RV32I-NEXT:    or a6, t0, a7
-; RV32I-NEXT:    or a7, t2, t1
-; RV32I-NEXT:    or t0, s2, t3
-; RV32I-NEXT:    or t1, s4, s3
-; RV32I-NEXT:    or a3, t4, a3
+; RV32I-NEXT:    or s2, s3, s2
+; RV32I-NEXT:    addi s3, sp, 8
+; RV32I-NEXT:    slli t2, t2, 8
+; RV32I-NEXT:    slli t5, t5, 16
+; RV32I-NEXT:    slli s1, s1, 24
+; RV32I-NEXT:    slli s4, s4, 8
+; RV32I-NEXT:    slli s5, s5, 16
+; RV32I-NEXT:    slli ra, ra, 24
+; RV32I-NEXT:    slli s7, s7, 8
+; RV32I-NEXT:    slli s8, s8, 16
+; RV32I-NEXT:    slli s9, s9, 24
+; RV32I-NEXT:    or t1, t2, t1
+; RV32I-NEXT:    srli a1, a0, 3
 ; RV32I-NEXT:    or t2, s1, t5
-; RV32I-NEXT:    or a0, a1, t6
-; RV32I-NEXT:    sw s0, 56(sp)
-; RV32I-NEXT:    sw s0, 60(sp)
-; RV32I-NEXT:    sw s0, 64(sp)
-; RV32I-NEXT:    sw s0, 68(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s0, 44(sp)
-; RV32I-NEXT:    sw s0, 48(sp)
-; RV32I-NEXT:    sw s0, 52(sp)
-; RV32I-NEXT:    sw t0, 24(sp)
-; RV32I-NEXT:    sw t1, 28(sp)
-; RV32I-NEXT:    sw a3, 32(sp)
+; RV32I-NEXT:    andi t5, a0, 31
+; RV32I-NEXT:    or t3, s4, t3
+; RV32I-NEXT:    or s1, ra, s5
+; RV32I-NEXT:    or s4, s7, s6
+; RV32I-NEXT:    or s5, s9, s8
+; RV32I-NEXT:    srai s6, s9, 31
+; RV32I-NEXT:    andi s7, a1, 28
+; RV32I-NEXT:    xori a1, t5, 31
+; RV32I-NEXT:    or a3, a4, a3
+; RV32I-NEXT:    or a4, a6, a5
+; RV32I-NEXT:    or a5, t0, a7
+; RV32I-NEXT:    or a6, t6, t4
+; RV32I-NEXT:    or a7, s2, s0
+; RV32I-NEXT:    or t0, t2, t1
+; RV32I-NEXT:    or t1, s1, t3
+; RV32I-NEXT:    or t2, s5, s4
+; RV32I-NEXT:    sw s6, 56(sp)
+; RV32I-NEXT:    sw s6, 60(sp)
+; RV32I-NEXT:    sw s6, 64(sp)
+; RV32I-NEXT:    sw s6, 68(sp)
+; RV32I-NEXT:    sw s6, 40(sp)
+; RV32I-NEXT:    sw s6, 44(sp)
+; RV32I-NEXT:    sw s6, 48(sp)
+; RV32I-NEXT:    sw s6, 52(sp)
+; RV32I-NEXT:    add s3, s3, s7
+; RV32I-NEXT:    sw a7, 24(sp)
+; RV32I-NEXT:    sw t0, 28(sp)
+; RV32I-NEXT:    sw t1, 32(sp)
 ; RV32I-NEXT:    sw t2, 36(sp)
-; RV32I-NEXT:    sw a4, 8(sp)
-; RV32I-NEXT:    sw a5, 12(sp)
-; RV32I-NEXT:    sw a6, 16(sp)
-; RV32I-NEXT:    sw a7, 20(sp)
-; RV32I-NEXT:    srli a1, a0, 3
-; RV32I-NEXT:    andi a3, a0, 31
-; RV32I-NEXT:    andi a4, a1, 28
-; RV32I-NEXT:    xori a1, a3, 31
-; RV32I-NEXT:    add a4, s8, a4
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    lw a5, 4(a4)
-; RV32I-NEXT:    lw a6, 8(a4)
-; RV32I-NEXT:    lw a7, 12(a4)
-; RV32I-NEXT:    lw t0, 16(a4)
-; RV32I-NEXT:    lw t1, 20(a4)
-; RV32I-NEXT:    lw t2, 24(a4)
-; RV32I-NEXT:    lw a4, 28(a4)
-; RV32I-NEXT:    srl t3, a5, a0
-; RV32I-NEXT:    slli t4, a6, 1
+; RV32I-NEXT:    sw a3, 8(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a6, 20(sp)
+; RV32I-NEXT:    lw a3, 0(s3)
+; RV32I-NEXT:    lw a4, 4(s3)
+; RV32I-NEXT:    lw a5, 8(s3)
+; RV32I-NEXT:    lw a6, 12(s3)
+; RV32I-NEXT:    lw a7, 16(s3)
+; RV32I-NEXT:    lw t0, 20(s3)
+; RV32I-NEXT:    lw t1, 24(s3)
+; RV32I-NEXT:    lw t2, 28(s3)
+; RV32I-NEXT:    srl t3, a4, a0
+; RV32I-NEXT:    slli t4, a5, 1
 ; RV32I-NEXT:    srl a3, a3, a0
-; RV32I-NEXT:    slli a5, a5, 1
-; RV32I-NEXT:    srl t5, a7, a0
-; RV32I-NEXT:    slli t6, t0, 1
-; RV32I-NEXT:    srl a6, a6, a0
-; RV32I-NEXT:    slli a7, a7, 1
-; RV32I-NEXT:    srl s0, t1, a0
-; RV32I-NEXT:    slli s1, t2, 1
-; RV32I-NEXT:    srl t0, t0, a0
-; RV32I-NEXT:    slli t1, t1, 1
-; RV32I-NEXT:    srl t2, t2, a0
-; RV32I-NEXT:    slli s2, a4, 1
-; RV32I-NEXT:    sra s3, a4, a0
+; RV32I-NEXT:    slli a4, a4, 1
+; RV32I-NEXT:    srl t5, a6, a0
+; RV32I-NEXT:    slli t6, a7, 1
+; RV32I-NEXT:    srl a5, a5, a0
+; RV32I-NEXT:    slli a6, a6, 1
+; RV32I-NEXT:    srl s0, t0, a0
+; RV32I-NEXT:    slli s1, t1, 1
+; RV32I-NEXT:    srl a7, a7, a0
+; RV32I-NEXT:    slli t0, t0, 1
+; RV32I-NEXT:    srl t1, t1, a0
+; RV32I-NEXT:    slli s2, t2, 1
+; RV32I-NEXT:    sra t2, t2, a0
 ; RV32I-NEXT:    sll a0, t4, a1
-; RV32I-NEXT:    sll a4, a5, a1
-; RV32I-NEXT:    sll a5, t6, a1
-; RV32I-NEXT:    sll a7, a7, a1
-; RV32I-NEXT:    sll t4, s1, a1
-; RV32I-NEXT:    sll t1, t1, a1
-; RV32I-NEXT:    sll t6, s2, a1
-; RV32I-NEXT:    srli s1, s3, 24
-; RV32I-NEXT:    srli s2, s3, 16
-; RV32I-NEXT:    srli s4, s3, 8
+; RV32I-NEXT:    sll a4, a4, a1
+; RV32I-NEXT:    sll t4, t6, a1
+; RV32I-NEXT:    sll a6, a6, a1
+; RV32I-NEXT:    sll t6, s1, a1
+; RV32I-NEXT:    sll t0, t0, a1
+; RV32I-NEXT:    sll s1, s2, a1
+; RV32I-NEXT:    srli s2, t2, 24
+; RV32I-NEXT:    srli s3, t2, 16
+; RV32I-NEXT:    srli s4, t2, 8
 ; RV32I-NEXT:    or a0, t3, a0
 ; RV32I-NEXT:    or a1, a3, a4
-; RV32I-NEXT:    or a3, t5, a5
-; RV32I-NEXT:    or a4, a6, a7
-; RV32I-NEXT:    or a5, s0, t4
-; RV32I-NEXT:    or a6, t0, t1
-; RV32I-NEXT:    or a7, t2, t6
-; RV32I-NEXT:    sb s3, 28(a2)
+; RV32I-NEXT:    or a3, t5, t4
+; RV32I-NEXT:    or a4, a5, a6
+; RV32I-NEXT:    or a5, s0, t6
+; RV32I-NEXT:    or a6, a7, t0
+; RV32I-NEXT:    or a7, t1, s1
+; RV32I-NEXT:    sb t2, 28(a2)
 ; RV32I-NEXT:    sb s4, 29(a2)
-; RV32I-NEXT:    sb s2, 30(a2)
-; RV32I-NEXT:    sb s1, 31(a2)
+; RV32I-NEXT:    sb s3, 30(a2)
+; RV32I-NEXT:    sb s2, 31(a2)
 ; RV32I-NEXT:    srli t0, a7, 24
 ; RV32I-NEXT:    srli t1, a7, 16
 ; RV32I-NEXT:    srli t2, a7, 8
diff --git a/llvm/test/CodeGen/RISCV/xmips-cbop.ll b/llvm/test/CodeGen/RISCV/xmips-cbop.ll
new file mode 100644
index 000000000000..cbbd1de13192
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xmips-cbop.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+xmipscbop -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32XMIPSPREFETCH
+; RUN: llc -mtriple=riscv64 -mattr=+xmipscbop -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64XMIPSPREFETCH
+
+define void @prefetch_data_read(ptr noundef %ptr) nounwind {
+; RV32XMIPSPREFETCH-LABEL: prefetch_data_read:
+; RV32XMIPSPREFETCH:       # %bb.0: # %entry
+; RV32XMIPSPREFETCH-NEXT:    mips.pref 8, 1(a0)
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_data_read:
+; RV64XMIPSPREFETCH:       # %bb.0: # %entry
+; RV64XMIPSPREFETCH-NEXT:    mips.pref 8, 1(a0)
+; RV64XMIPSPREFETCH-NEXT:    ret
+entry:
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 1
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 0, i32 0, i32 1)
+  ret void
+}
+
+define void @prefetch_data_write(ptr noundef %ptr) nounwind  {
+; RV32XMIPSPREFETCH-LABEL: prefetch_data_write:
+; RV32XMIPSPREFETCH:       # %bb.0:
+; RV32XMIPSPREFETCH-NEXT:    addi a0, a0, 512
+; RV32XMIPSPREFETCH-NEXT:    mips.pref 9, 0(a0)
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_data_write:
+; RV64XMIPSPREFETCH:       # %bb.0:
+; RV64XMIPSPREFETCH-NEXT:    addi a0, a0, 512
+; RV64XMIPSPREFETCH-NEXT:    mips.pref 9, 0(a0)
+; RV64XMIPSPREFETCH-NEXT:    ret
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 512
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 1, i32 0, i32 1)
+  ret void
+}
+
+define void @prefetch_inst_read(ptr noundef %ptr) nounwind  {
+; RV32XMIPSPREFETCH-LABEL: prefetch_inst_read:
+; RV32XMIPSPREFETCH:       # %bb.0:
+; RV32XMIPSPREFETCH-NEXT:    ret
+;
+; RV64XMIPSPREFETCH-LABEL: prefetch_inst_read:
+; RV64XMIPSPREFETCH:       # %bb.0:
+; RV64XMIPSPREFETCH-NEXT:    ret
+  %arrayidx = getelementptr inbounds nuw i8, ptr %ptr, i64 512
+  tail call void @llvm.prefetch.p0(ptr nonnull %arrayidx, i32 0, i32 0, i32 0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/xqcicm.ll b/llvm/test/CodeGen/RISCV/xqcicm.ll
new file mode 100644
index 000000000000..4adb3dce265d
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcicm.ll
@@ -0,0 +1,679 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; Test that we are able to generate the Xqcicm instructions
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcicm -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s --check-prefixes=RV32IXQCICM
+
+define i32 @select_example(i32 %cond, i32 %x, i32 %y) {
+; RV32I-LABEL: select_example:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    andi a3, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a3, .LBB0_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:  .LBB0_2: # %entry
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_example:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    andi a0, a0, 1
+; RV32IXQCICM-NEXT:    qc.mveqi a1, a0, 0, a2
+; RV32IXQCICM-NEXT:    mv a0, a1
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cond_trunc = trunc i32 %cond to i1
+  %sel = select i1 %cond_trunc, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_eq(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_eq:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    beq a0, a1, .LBB1_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB1_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_eq:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mveqi a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp eq i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_eq1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_eq1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    beq a0, a1, .LBB2_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB2_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_eq1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mveqi a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp eq i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ne(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ne:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bne a0, a1, .LBB3_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB3_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ne:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvnei a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ne i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ne1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ne1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bne a0, a1, .LBB4_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB4_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ne1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvnei a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ne i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_slt(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_slt:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    blt a0, a1, .LBB5_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB5_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_slt:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp slt i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_slt1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_slt1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    blt a1, a0, .LBB6_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB6_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_slt1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a2, a0, 12, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp slt i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sle(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sle:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 12
+; RV32I-NEXT:    blt a0, a1, .LBB7_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB7_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sle:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a3, a0, 12, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sle i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sle1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sle1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 10
+; RV32I-NEXT:    blt a1, a0, .LBB8_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB8_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sle1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a2, a0, 11, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sle i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sgt(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sgt:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    blt a1, a0, .LBB9_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB9_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sgt:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a2, a0, 12, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sgt1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sgt1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    blt a0, a1, .LBB10_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB10_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sgt1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sge(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sge:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 10
+; RV32I-NEXT:    blt a1, a0, .LBB11_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB11_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sge:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a2, a0, 11, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sge i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sge1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sge1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 12
+; RV32I-NEXT:    blt a0, a1, .LBB12_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB12_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sge1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlti a3, a0, 12, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sge i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ule(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ule:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 12
+; RV32I-NEXT:    bltu a0, a1, .LBB13_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB13_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ule:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a3, a0, 12, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ule i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ule1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ule1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 10
+; RV32I-NEXT:    bltu a1, a0, .LBB14_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB14_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ule1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a2, a0, 11, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ule i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ugt(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ugt:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bltu a1, a0, .LBB15_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB15_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ugt:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a2, a0, 12, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ugt i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ugt1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ugt1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bltu a0, a1, .LBB16_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB16_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ugt1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ugt i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ult(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ult:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bltu a0, a1, .LBB17_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB17_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ult:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a3, a0, 11, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ult i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ult1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ult1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    bltu a1, a0, .LBB18_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB18_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ult1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a2, a0, 12, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ult i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_uge(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_uge:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 10
+; RV32I-NEXT:    bltu a1, a0, .LBB19_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB19_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_uge:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a2, a0, 11, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp uge i32 %a, 11
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_uge1(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_uge1:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    li a1, 12
+; RV32I-NEXT:    bltu a0, a1, .LBB20_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB20_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_uge1:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltui a3, a0, 12, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp uge i32 11, %a
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_eq_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_eq_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    beq a0, a1, .LBB21_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB21_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_eq_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mveq a3, a0, a1, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp eq i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ne_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ne_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bne a0, a1, .LBB22_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB22_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ne_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvne a3, a0, a1, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ne i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_slt_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_slt_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    blt a0, a1, .LBB23_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB23_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_slt_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlt a3, a0, a1, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp slt i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sge_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sge_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bge a0, a1, .LBB24_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB24_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sge_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlt a2, a0, a1, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sge i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sgt_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sgt_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    blt a1, a0, .LBB25_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB25_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sgt_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlt a3, a1, a0, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_sle_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_sle_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bge a1, a0, .LBB26_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB26_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_sle_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvlt a2, a1, a0, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp sle i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ugt_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ugt_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bltu a1, a0, .LBB27_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB27_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ugt_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltu a3, a1, a0, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ugt i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ult_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ult_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bltu a0, a1, .LBB28_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB28_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ult_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltu a3, a0, a1, a2
+; RV32IXQCICM-NEXT:    mv a0, a3
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ult i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_uge_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_uge_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bgeu a0, a1, .LBB29_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB29_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_uge_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltu a2, a0, a1, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp uge i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
+
+define i32 @select_cc_example_ule_reg(i32 %a, i32 %b, i32 %x, i32 %y) {
+; RV32I-LABEL: select_cc_example_ule_reg:
+; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    bgeu a1, a0, .LBB30_2
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:  .LBB30_2: # %entry
+; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCICM-LABEL: select_cc_example_ule_reg:
+; RV32IXQCICM:       # %bb.0: # %entry
+; RV32IXQCICM-NEXT:    qc.mvltu a2, a1, a0, a3
+; RV32IXQCICM-NEXT:    mv a0, a2
+; RV32IXQCICM-NEXT:    ret
+entry:
+  %cmp = icmp ule i32 %a, %b
+  %sel = select i1 %cmp, i32 %x, i32 %y
+  ret i32 %sel
+}
diff --git a/llvm/test/CodeGen/RISCV/xqcisls.ll b/llvm/test/CodeGen/RISCV/xqcisls.ll
index b9263d487b60..2bc4834ad355 100644
--- a/llvm/test/CodeGen/RISCV/xqcisls.ll
+++ b/llvm/test/CodeGen/RISCV/xqcisls.ll
@@ -205,3 +205,314 @@ define void @sw_ri(i32* %a, i32 %b, i32 %c) {
   store i32 %b, i32* %1
   ret void
 }
+
+define i8 @lrb_anyext(ptr %a, i64 %b) {
+; RV32I-LABEL: lrb_anyext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lrb_anyext:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lbu a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lrb_anyext:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrbu a0, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = getelementptr i8, ptr %a, i64 %b
+  %2 = load i8, ptr %1, align 1
+  ret i8 %2
+}
+
+define i64 @lrb(ptr %a, i64 %b) {
+; RV32I-LABEL: lrb:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lb a1, 0(a0)
+; RV32I-NEXT:    srai a2, a1, 31
+; RV32I-NEXT:    add a0, a1, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    add a2, a2, a2
+; RV32I-NEXT:    add a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lrb:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lb a1, 0(a0)
+; RV32IZBA-NEXT:    srai a2, a1, 31
+; RV32IZBA-NEXT:    add a0, a1, a1
+; RV32IZBA-NEXT:    sltu a1, a0, a1
+; RV32IZBA-NEXT:    add a2, a2, a2
+; RV32IZBA-NEXT:    add a1, a2, a1
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lrb:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrb a1, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    srai a2, a1, 31
+; RV32IZBAXQCISLS-NEXT:    add a0, a1, a1
+; RV32IZBAXQCISLS-NEXT:    sltu a1, a0, a1
+; RV32IZBAXQCISLS-NEXT:    add a2, a2, a2
+; RV32IZBAXQCISLS-NEXT:    add a1, a2, a1
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = getelementptr i8, ptr %a, i64 %b
+  %2 = load i8, ptr %1, align 1
+  %3 = sext i8 %2 to i64
+  %4 = add i64 %3, %3
+  ret i64 %4
+}
+
+define i8 @lurb_anyext(ptr %a, i32 %b) {
+; RV32I-LABEL: lurb_anyext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a0, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lurb_anyext:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lbu a0, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lurb_anyext:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrbu a0, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = zext i32 %b to i64
+  %2 = getelementptr i8, ptr %a, i64 %1
+  %3 = load i8, ptr %2, align 1
+  ret i8 %3
+}
+
+define i64 @lurb(ptr %a, i32 %b) {
+; RV32I-LABEL: lurb:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lb a1, 0(a0)
+; RV32I-NEXT:    srai a2, a1, 31
+; RV32I-NEXT:    add a0, a1, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    add a2, a2, a2
+; RV32I-NEXT:    add a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lurb:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lb a1, 0(a0)
+; RV32IZBA-NEXT:    srai a2, a1, 31
+; RV32IZBA-NEXT:    add a0, a1, a1
+; RV32IZBA-NEXT:    sltu a1, a0, a1
+; RV32IZBA-NEXT:    add a2, a2, a2
+; RV32IZBA-NEXT:    add a1, a2, a1
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lurb:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrb a1, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    srai a2, a1, 31
+; RV32IZBAXQCISLS-NEXT:    add a0, a1, a1
+; RV32IZBAXQCISLS-NEXT:    sltu a1, a0, a1
+; RV32IZBAXQCISLS-NEXT:    add a2, a2, a2
+; RV32IZBAXQCISLS-NEXT:    add a1, a2, a1
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = zext i32 %b to i64
+  %2 = getelementptr i8, ptr %a, i64 %1
+  %3 = load i8, ptr %2, align 1
+  %4 = sext i8 %3 to i64
+  %5 = add i64 %4, %4
+  ret i64 %5
+}
+
+define i64 @lrbu(ptr %a, i64 %b) {
+; RV32I-LABEL: lrbu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    add a0, a1, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lrbu:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lbu a1, 0(a0)
+; RV32IZBA-NEXT:    add a0, a1, a1
+; RV32IZBA-NEXT:    sltu a1, a0, a1
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lrbu:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrbu a1, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    add a0, a1, a1
+; RV32IZBAXQCISLS-NEXT:    sltu a1, a0, a1
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = getelementptr i8, ptr %a, i64 %b
+  %2 = load i8, ptr %1, align 1
+  %3 = zext i8 %2 to i64
+  %4 = add i64 %3, %3
+  ret i64 %4
+}
+
+define i64 @lurbu(ptr %a, i32 %b) {
+; RV32I-LABEL: lurbu:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    add a0, a1, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lurbu:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    lbu a1, 0(a0)
+; RV32IZBA-NEXT:    add a0, a1, a1
+; RV32IZBA-NEXT:    sltu a1, a0, a1
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lurbu:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    qc.lrbu a1, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    add a0, a1, a1
+; RV32IZBAXQCISLS-NEXT:    sltu a1, a0, a1
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = zext i32 %b to i64
+  %2 = getelementptr i8, ptr %a, i64 %1
+  %3 = load i8, ptr %2, align 1
+  %4 = zext i8 %3 to i64
+  %5 = add i64 %4, %4
+  ret i64 %5
+}
+
+define i64 @lrd_2(ptr %a, i64 %b) {
+; RV32I-LABEL: lrd_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 3
+; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    lw a1, 96(a0)
+; RV32I-NEXT:    lw a2, 100(a0)
+; RV32I-NEXT:    add a0, a1, a1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    add a2, a2, a2
+; RV32I-NEXT:    add a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lrd_2:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    sh3add a0, a1, a0
+; RV32IZBA-NEXT:    lw a1, 96(a0)
+; RV32IZBA-NEXT:    lw a2, 100(a0)
+; RV32IZBA-NEXT:    add a0, a1, a1
+; RV32IZBA-NEXT:    sltu a1, a0, a1
+; RV32IZBA-NEXT:    add a2, a2, a2
+; RV32IZBA-NEXT:    add a1, a2, a1
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lrd_2:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    addi a2, a0, 96
+; RV32IZBAXQCISLS-NEXT:    qc.lrw a2, a2, a1, 3
+; RV32IZBAXQCISLS-NEXT:    addi a0, a0, 100
+; RV32IZBAXQCISLS-NEXT:    qc.lrw a1, a0, a1, 3
+; RV32IZBAXQCISLS-NEXT:    add a0, a2, a2
+; RV32IZBAXQCISLS-NEXT:    sltu a2, a0, a2
+; RV32IZBAXQCISLS-NEXT:    add a1, a1, a1
+; RV32IZBAXQCISLS-NEXT:    add a1, a1, a2
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = add i64 %b, 12
+  %2 = getelementptr i64, ptr %a, i64 %1
+  %3 = load i64, ptr %2, align 8
+  %4 = add i64 %3, %3
+  ret i64 %4
+}
+
+define void @srb(ptr %a, i64 %b, i8 %c) {
+; RV32I-LABEL: srb:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a3, a3, a3
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    sb a3, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: srb:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a3, a3, a3
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    sb a3, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: srb:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    add a3, a3, a3
+; RV32IZBAXQCISLS-NEXT:    qc.srb a3, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = add i8 %c, %c
+  %2 = getelementptr i8, ptr %a, i64 %b
+  store i8 %1, ptr %2, align 1
+  ret void
+}
+
+define void @surb(ptr %a, i32 %b, i8 %c) {
+; RV32I-LABEL: surb:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    add a2, a2, a2
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    sb a2, 0(a0)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: surb:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    add a2, a2, a2
+; RV32IZBA-NEXT:    add a0, a0, a1
+; RV32IZBA-NEXT:    sb a2, 0(a0)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: surb:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    add a2, a2, a2
+; RV32IZBAXQCISLS-NEXT:    qc.srb a2, a0, a1, 0
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = zext i32 %b to i64
+  %2 = add i8 %c, %c
+  %3 = getelementptr i8, ptr %a, i64 %1
+  store i8 %2, ptr %3, align 1
+  ret void
+}
+
+define i64 @lrd_large_shift(ptr %a, i64 %b) {
+; RV32I-LABEL: lrd_large_shift:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a1, 5
+; RV32I-NEXT:    add a1, a1, a0
+; RV32I-NEXT:    lw a0, 384(a1)
+; RV32I-NEXT:    lw a1, 388(a1)
+; RV32I-NEXT:    ret
+;
+; RV32IZBA-LABEL: lrd_large_shift:
+; RV32IZBA:       # %bb.0:
+; RV32IZBA-NEXT:    slli a1, a1, 5
+; RV32IZBA-NEXT:    add a1, a1, a0
+; RV32IZBA-NEXT:    lw a0, 384(a1)
+; RV32IZBA-NEXT:    lw a1, 388(a1)
+; RV32IZBA-NEXT:    ret
+;
+; RV32IZBAXQCISLS-LABEL: lrd_large_shift:
+; RV32IZBAXQCISLS:       # %bb.0:
+; RV32IZBAXQCISLS-NEXT:    addi a2, a0, 384
+; RV32IZBAXQCISLS-NEXT:    addi a3, a0, 388
+; RV32IZBAXQCISLS-NEXT:    qc.lrw a0, a2, a1, 5
+; RV32IZBAXQCISLS-NEXT:    qc.lrw a1, a3, a1, 5
+; RV32IZBAXQCISLS-NEXT:    ret
+  %1 = add i64 %b, 12
+  %2 = shl i64 %1, 2
+  %3 = getelementptr i64, ptr %a, i64 %2
+  %4 = load i64, ptr %3, align 8
+  ret i64 %4
+}
diff --git a/llvm/test/CodeGen/RISCV/zicond-opts.ll b/llvm/test/CodeGen/RISCV/zicond-opts.ll
index 2512ba803cf4..35b06c4f4fb4 100644
--- a/llvm/test/CodeGen/RISCV/zicond-opts.ll
+++ b/llvm/test/CodeGen/RISCV/zicond-opts.ll
@@ -62,37 +62,30 @@ define i64 @rotate_l_nez(i64 %x, i64 %rot.amt, i1 %cond) {
 ; RV32ZICOND-LABEL: rotate_l_nez:
 ; RV32ZICOND:       # %bb.0:
 ; RV32ZICOND-NEXT:    andi a4, a4, 1
-; RV32ZICOND-NEXT:    bexti a3, a2, 5
-; RV32ZICOND-NEXT:    not a5, a2
-; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
-; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
-; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a0, a0, a4
-; RV32ZICOND-NEXT:    czero.eqz a1, a1, a4
-; RV32ZICOND-NEXT:    or a6, a7, a6
-; RV32ZICOND-NEXT:    or a3, a3, t0
-; RV32ZICOND-NEXT:    sll a7, a6, a2
-; RV32ZICOND-NEXT:    srli t0, a3, 1
-; RV32ZICOND-NEXT:    sll a2, a3, a2
-; RV32ZICOND-NEXT:    srli a3, a6, 1
-; RV32ZICOND-NEXT:    srl a6, t0, a5
-; RV32ZICOND-NEXT:    srl a3, a3, a5
-; RV32ZICOND-NEXT:    or a5, a7, a6
-; RV32ZICOND-NEXT:    or a2, a2, a3
 ; RV32ZICOND-NEXT:    czero.nez a2, a2, a4
-; RV32ZICOND-NEXT:    czero.nez a3, a5, a4
-; RV32ZICOND-NEXT:    or a0, a0, a2
-; RV32ZICOND-NEXT:    or a1, a1, a3
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    czero.nez a4, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a5, a1, a3
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a3
+; RV32ZICOND-NEXT:    not a3, a2
+; RV32ZICOND-NEXT:    or a4, a5, a4
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    sll a1, a4, a2
+; RV32ZICOND-NEXT:    srli a5, a0, 1
+; RV32ZICOND-NEXT:    sll a2, a0, a2
+; RV32ZICOND-NEXT:    srli a4, a4, 1
+; RV32ZICOND-NEXT:    srl a0, a5, a3
+; RV32ZICOND-NEXT:    srl a3, a4, a3
+; RV32ZICOND-NEXT:    or a0, a1, a0
+; RV32ZICOND-NEXT:    or a1, a2, a3
 ; RV32ZICOND-NEXT:    ret
 ;
 ; RV64ZICOND-LABEL: rotate_l_nez:
 ; RV64ZICOND:       # %bb.0:
 ; RV64ZICOND-NEXT:    andi a2, a2, 1
-; RV64ZICOND-NEXT:    rol a1, a0, a1
 ; RV64ZICOND-NEXT:    czero.nez a1, a1, a2
-; RV64ZICOND-NEXT:    czero.eqz a0, a0, a2
-; RV64ZICOND-NEXT:    or a0, a0, a1
+; RV64ZICOND-NEXT:    rol a0, a0, a1
 ; RV64ZICOND-NEXT:    ret
   %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
   %7 = select i1 %cond, i64 %x, i64 %6
@@ -104,37 +97,30 @@ define i64 @rotate_l_eqz(i64 %x, i64 %rot.amt, i1 %cond) {
 ; RV32ZICOND-LABEL: rotate_l_eqz:
 ; RV32ZICOND:       # %bb.0:
 ; RV32ZICOND-NEXT:    andi a4, a4, 1
-; RV32ZICOND-NEXT:    bexti a3, a2, 5
-; RV32ZICOND-NEXT:    not a5, a2
-; RV32ZICOND-NEXT:    czero.nez a6, a1, a3
-; RV32ZICOND-NEXT:    czero.eqz a7, a0, a3
-; RV32ZICOND-NEXT:    czero.nez t0, a0, a3
-; RV32ZICOND-NEXT:    czero.eqz a3, a1, a3
-; RV32ZICOND-NEXT:    czero.nez a0, a0, a4
-; RV32ZICOND-NEXT:    czero.nez a1, a1, a4
-; RV32ZICOND-NEXT:    or a6, a7, a6
-; RV32ZICOND-NEXT:    or a3, a3, t0
-; RV32ZICOND-NEXT:    sll a7, a6, a2
-; RV32ZICOND-NEXT:    srli t0, a3, 1
-; RV32ZICOND-NEXT:    sll a2, a3, a2
-; RV32ZICOND-NEXT:    srli a3, a6, 1
-; RV32ZICOND-NEXT:    srl a6, t0, a5
-; RV32ZICOND-NEXT:    srl a3, a3, a5
-; RV32ZICOND-NEXT:    or a5, a7, a6
-; RV32ZICOND-NEXT:    or a2, a2, a3
 ; RV32ZICOND-NEXT:    czero.eqz a2, a2, a4
-; RV32ZICOND-NEXT:    czero.eqz a3, a5, a4
-; RV32ZICOND-NEXT:    or a0, a2, a0
-; RV32ZICOND-NEXT:    or a1, a3, a1
+; RV32ZICOND-NEXT:    bexti a3, a2, 5
+; RV32ZICOND-NEXT:    czero.nez a4, a0, a3
+; RV32ZICOND-NEXT:    czero.eqz a5, a1, a3
+; RV32ZICOND-NEXT:    czero.nez a1, a1, a3
+; RV32ZICOND-NEXT:    czero.eqz a0, a0, a3
+; RV32ZICOND-NEXT:    not a3, a2
+; RV32ZICOND-NEXT:    or a4, a5, a4
+; RV32ZICOND-NEXT:    or a0, a0, a1
+; RV32ZICOND-NEXT:    sll a1, a4, a2
+; RV32ZICOND-NEXT:    srli a5, a0, 1
+; RV32ZICOND-NEXT:    sll a2, a0, a2
+; RV32ZICOND-NEXT:    srli a4, a4, 1
+; RV32ZICOND-NEXT:    srl a0, a5, a3
+; RV32ZICOND-NEXT:    srl a3, a4, a3
+; RV32ZICOND-NEXT:    or a0, a1, a0
+; RV32ZICOND-NEXT:    or a1, a2, a3
 ; RV32ZICOND-NEXT:    ret
 ;
 ; RV64ZICOND-LABEL: rotate_l_eqz:
 ; RV64ZICOND:       # %bb.0:
 ; RV64ZICOND-NEXT:    andi a2, a2, 1
-; RV64ZICOND-NEXT:    rol a1, a0, a1
-; RV64ZICOND-NEXT:    czero.nez a0, a0, a2
 ; RV64ZICOND-NEXT:    czero.eqz a1, a1, a2
-; RV64ZICOND-NEXT:    or a0, a1, a0
+; RV64ZICOND-NEXT:    rol a0, a0, a1
 ; RV64ZICOND-NEXT:    ret
   %6 = call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %rot.amt)
   %7 = select i1 %cond, i64 %6, i64 %x
diff --git a/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll b/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll
new file mode 100644
index 000000000000..aa60e13232b4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/capability-FloatControl2.ll
@@ -0,0 +1,22 @@
+; RUN: llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - | FileCheck %s --check-prefix=CHECK-NOEXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; RUN: llc -O0 -mtriple=spirv1.6-vulkan1.3-compute -spirv-ext=+SPV_KHR_float_controls2 %s -o - | FileCheck %s --check-prefix=CHECK-EXT
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute -spirv-ext=+SPV_KHR_float_controls2 %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-NOEXT-NOT: OpDecorate FPFastMathMode
+
+; CHECK-EXT: OpCapability FloatControls2
+; CHECK-EXT: OpExtension "SPV_KHR_float_controls2"
+; CHECK-EXT: OpDecorate {{%[0-9]+}} FPFastMathMode NotNaN|NotInf|NSZ|AllowRecip|Fast
+
+define hidden spir_func float @foo(float  %0) local_unnamed_addr {
+  %2 = fmul reassoc nnan ninf nsz arcp afn float %0, 2.000000e+00
+  ret float %2
+}
+
+define void @main() local_unnamed_addr #1 {
+  ret void
+}
+
+attributes #1 = { "hlsl.numthreads"="8,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
index 02d21a1abafa..4db0ba33d52c 100644
--- a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
+++ b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-but-one.ll
@@ -1,4 +1,6 @@
 ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=all,-SPV_INTEL_arbitrary_precision_integers %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=KHR %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown --spirv-ext=khr %s -o - | FileCheck %s
 
 define i6 @foo() {
   %call = tail call i32 @llvm.bitreverse.i32(i32 42)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll
new file mode 100644
index 000000000000..66fda60b5ac4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/MixedBufferLoadStore.ll
@@ -0,0 +1,36 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s --match-full-lines
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.signed = private unnamed_addr constant [7 x i8] c"signed\00", align 1
+@.str.unsigned = private unnamed_addr constant [9 x i8] c"unsigned\00", align 1
+
+; CHECK: {{.*}} OpFunction {{.*}}
+define void @main_unsigned() local_unnamed_addr #0 {
+entry:
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.unsigned)
+
+; CHECK: {{%[0-9]+}} = OpImageRead {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}}
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+; CHECK: OpImageWrite {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}}
+  store i32 %1, ptr %2, align 4
+  ret void
+}
+
+; CHECK: {{.*}} OpFunction {{.*}}
+define void @main_signed() local_unnamed_addr #0 {
+entry:
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.signed)
+
+; CHECK: {{%[0-9]+}} = OpImageRead {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}} SignExtend
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+; CHECK: OpImageWrite {{%[0-9]+}} {{%[0-9]+}} {{%[0-9]+}} SignExtend
+  store i32 %1, ptr %2, align 4
+  ret void
+}
+
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll
new file mode 100644
index 000000000000..354796ce9c1e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/SignedBufferLoadStore.ll
@@ -0,0 +1,137 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
+; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[v2int:%[0-9]+]] = OpTypeVector [[uint]] 2
+; CHECK-DAG: [[v4int:%[0-9]+]] = OpTypeVector [[uint]] 4
+; CHECK-DAG: [[zero:%[0-9]+]] = OpConstant [[uint]] 0
+; CHECK-DAG: [[one:%[0-9]+]] = OpConstant [[uint]] 1
+; CHECK-DAG: [[twenty:%[0-9]+]] = OpConstant [[uint]] 20
+; CHECK-DAG: [[twenty_three:%[0-9]+]] = OpConstant [[uint]] 23
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[uint]] Buffer 2 0 0 2 Unknown 
+; CHECK-DAG: [[ImagePtr:%[0-9]+]] = OpTypePointer UniformConstant [[ImageType]]
+; CHECK-DAG: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_scalar() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store i32 %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load i32, ptr %3, align 4
+  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store i32 %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector2() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <2 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <2 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <2 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <2 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector4() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]] SignExtend
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <4 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[R]] SignExtend
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <4 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]] SignExtend
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <4 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[R]] SignExtend
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <4 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none) "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll
new file mode 100644
index 000000000000..e22724f02fa7
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/UnsignedBufferLoadStore.ll
@@ -0,0 +1,137 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str.b0 = private unnamed_addr constant [3 x i8] c"B0\00", align 1
+
+; CHECK-DAG: [[uint:%[0-9]+]] = OpTypeInt 32 0
+; CHECK-DAG: [[v2int:%[0-9]+]] = OpTypeVector [[uint]] 2
+; CHECK-DAG: [[v4int:%[0-9]+]] = OpTypeVector [[uint]] 4
+; CHECK-DAG: [[zero:%[0-9]+]] = OpConstant [[uint]] 0
+; CHECK-DAG: [[one:%[0-9]+]] = OpConstant [[uint]] 1
+; CHECK-DAG: [[twenty:%[0-9]+]] = OpConstant [[uint]] 20
+; CHECK-DAG: [[twenty_three:%[0-9]+]] = OpConstant [[uint]] 23
+; CHECK-DAG: [[ImageType:%[0-9]+]] = OpTypeImage [[uint]] Buffer 2 0 0 2 Unknown 
+; CHECK-DAG: [[ImagePtr:%[0-9]+]] = OpTypePointer UniformConstant [[ImageType]]
+; CHECK-DAG: [[Var:%[0-9]+]] = OpVariable [[ImagePtr]] UniformConstant
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_scalar() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load i32, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store i32 %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+; CHECK: [[V:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load i32, ptr %3, align 4
+  
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store i32 %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector2() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <2 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[V]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <2 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+; CHECK: [[E0:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 0
+; CHECK: [[E1:%[0-9]+]] = OpCompositeExtract [[uint]] [[R]] 1
+; CHECK: [[V:%[0-9]+]] = OpCompositeConstruct [[v2int]] [[E0]] [[E1]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <2 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[V]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <2 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+; Function Attrs: mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none)
+; CHECK: OpFunction
+define void @main_vector4() local_unnamed_addr #0 {
+entry:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+  %s_h.i = tail call target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.Image_i32_5_2_0_0_2_0t(i32 3, i32 5, i32 1, i32 0, i1 false, ptr nonnull @.str.b0)
+
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[one]]
+  %0 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 1)
+  %1 = load <4 x i32>, ptr %0, align 4
+; CHECK: OpBranch [[bb_store:%[0-9]+]]
+  br label %bb_store
+
+; CHECK: [[bb_store]] = OpLabel
+bb_store:
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[zero]] [[R]]
+  %2 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 0)
+  store <4 x i32> %1, ptr %2, align 4
+; CHECK: OpBranch [[bb_both:%[0-9]+]]
+  br label %bb_both
+
+; CHECK: [[bb_both]] = OpLabel
+bb_both:
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: [[R:%[0-9]+]] = OpImageRead [[v4int]] [[H]] [[twenty_three]]
+  %3 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 23)
+  %4 = load <4 x i32>, ptr %3, align 4
+
+; CHECK: [[H:%[0-9]+]] = OpLoad [[ImageType]] [[Var]]
+; CHECK: OpImageWrite [[H]] [[twenty]] [[R]]
+  %5 = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.spv.resource.getpointer.p0.tspirv.Image_i32_5_2_0_0_2_0t(target("spirv.Image", i32, 5, 2, 0, 0, 2, 0) %s_h.i, i32 20)
+  store <4 x i32> %4, ptr %5, align 4
+  ret void
+}
+
+attributes #0 = { mustprogress nofree noinline norecurse nosync nounwind willreturn memory(readwrite, inaccessiblemem: none) "frame-pointer"="all" "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(none) }
diff --git a/llvm/test/CodeGen/SystemZ/pr60413.ll b/llvm/test/CodeGen/SystemZ/pr60413.ll
index 6dee3d303a6f..bbf4d50bd716 100644
--- a/llvm/test/CodeGen/SystemZ/pr60413.ll
+++ b/llvm/test/CodeGen/SystemZ/pr60413.ll
@@ -13,73 +13,67 @@ declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
 define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-LABEL: m:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stmg %r13, %r15, 104(%r15)
 ; CHECK-NEXT:    aghi %r15, -168
 ; CHECK-NEXT:    lhrl %r1, f+4
-; CHECK-NEXT:    sll %r1, 8
 ; CHECK-NEXT:    larl %r2, f
-; CHECK-NEXT:    ic %r1, 6(%r2)
-; CHECK-NEXT:    larl %r2, e
-; CHECK-NEXT:    lb %r0, 3(%r2)
-; CHECK-NEXT:    vlvgf %v1, %r1, 0
-; CHECK-NEXT:    vlvgf %v1, %r1, 1
-; CHECK-NEXT:    larl %r2, .LCPI0_0
-; CHECK-NEXT:    vl %v2, 0(%r2), 3
-; CHECK-NEXT:    vlvgf %v1, %r1, 3
-; CHECK-NEXT:    vlvgf %v3, %r1, 3
-; CHECK-NEXT:    vlvgf %v0, %r1, 1
-; CHECK-NEXT:    vperm %v4, %v1, %v0, %v2
-; CHECK-NEXT:    vlvgf %v0, %r1, 3
+; CHECK-NEXT:    llc %r2, 6(%r2)
+; CHECK-NEXT:    larl %r3, e
+; CHECK-NEXT:    lb %r0, 3(%r3)
+; CHECK-NEXT:    rosbg %r2, %r1, 32, 55, 8
+; CHECK-NEXT:    vlvgp %v0, %r2, %r0
+; CHECK-NEXT:    vlvgf %v0, %r2, 0
+; CHECK-NEXT:    vlvgf %v0, %r2, 2
+; CHECK-NEXT:    vlvgp %v1, %r0, %r2
+; CHECK-NEXT:    vlvgp %v2, %r2, %r2
+; CHECK-NEXT:    lr %r1, %r2
 ; CHECK-NEXT:    nilh %r1, 255
 ; CHECK-NEXT:    chi %r1, 128
 ; CHECK-NEXT:    ipm %r1
 ; CHECK-NEXT:    risbg %r1, %r1, 63, 191, 36
-; CHECK-NEXT:    vperm %v0, %v3, %v0, %v2
-; CHECK-NEXT:    larl %r2, .LCPI0_1
-; CHECK-NEXT:    vl %v5, 0(%r2), 3
-; CHECK-NEXT:    vgbm %v6, 30583
-; CHECK-NEXT:    vn %v0, %v0, %v6
-; CHECK-NEXT:    vn %v4, %v4, %v6
-; CHECK-NEXT:    vperm %v1, %v1, %v1, %v5
-; CHECK-NEXT:    vn %v1, %v1, %v6
-; CHECK-NEXT:    vperm %v2, %v0, %v3, %v2
-; CHECK-NEXT:    vn %v2, %v2, %v6
+; CHECK-NEXT:    vgbm %v3, 30583
+; CHECK-NEXT:    vn %v0, %v0, %v3
+; CHECK-NEXT:    vlvgf %v1, %r0, 0
+; CHECK-NEXT:    vlvgf %v1, %r0, 2
+; CHECK-NEXT:    vn %v1, %v1, %v3
+; CHECK-NEXT:    vrepf %v2, %v2, 1
+; CHECK-NEXT:    vn %v2, %v2, %v3
 ; CHECK-NEXT:    vrepif %v3, 127
-; CHECK-NEXT:    vchlf %v1, %v1, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    vlgvf %r2, %v1, 0
-; CHECK-NEXT:    risbg %r2, %r2, 48, 176, 15
-; CHECK-NEXT:    rosbg %r2, %r3, 49, 49, 14
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 50, 50, 13
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 51, 51, 12
-; CHECK-NEXT:    vchlf %v1, %v4, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 52, 52, 11
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 53, 53, 10
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 54, 54, 9
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 55, 55, 8
-; CHECK-NEXT:    vchlf %v1, %v2, %v3
-; CHECK-NEXT:    vlgvf %r3, %v1, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 56, 56, 7
-; CHECK-NEXT:    vlgvf %r3, %v1, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 57, 57, 6
-; CHECK-NEXT:    vlgvf %r3, %v1, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 58, 58, 5
-; CHECK-NEXT:    vlgvf %r3, %v1, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 59, 59, 4
 ; CHECK-NEXT:    vchlf %v0, %v0, %v3
-; CHECK-NEXT:    vlgvf %r3, %v0, 0
-; CHECK-NEXT:    rosbg %r2, %r3, 60, 60, 3
-; CHECK-NEXT:    vlgvf %r3, %v0, 1
-; CHECK-NEXT:    rosbg %r2, %r3, 61, 61, 2
-; CHECK-NEXT:    vlgvf %r3, %v0, 2
-; CHECK-NEXT:    rosbg %r2, %r3, 62, 62, 1
-; CHECK-NEXT:    vlgvf %r3, %v0, 3
-; CHECK-NEXT:    rosbg %r2, %r3, 63, 63, 0
+; CHECK-NEXT:    vlgvf %r13, %v0, 0
+; CHECK-NEXT:    vchlf %v2, %v2, %v3
+; CHECK-NEXT:    vlgvf %r3, %v2, 1
+; CHECK-NEXT:    nilf %r3, 1
+; CHECK-NEXT:    vlgvf %r4, %v2, 0
+; CHECK-NEXT:    risbg %r2, %r4, 48, 176, 15
+; CHECK-NEXT:    rosbg %r2, %r3, 32, 49, 14
+; CHECK-NEXT:    vlgvf %r5, %v2, 2
+; CHECK-NEXT:    nilf %r5, 1
+; CHECK-NEXT:    rosbg %r2, %r5, 32, 50, 13
+; CHECK-NEXT:    vlgvf %r14, %v2, 3
+; CHECK-NEXT:    nilf %r14, 1
+; CHECK-NEXT:    rosbg %r2, %r14, 32, 51, 12
+; CHECK-NEXT:    rosbg %r2, %r13, 52, 52, 11
+; CHECK-NEXT:    vlgvf %r13, %v0, 1
+; CHECK-NEXT:    rosbg %r2, %r13, 53, 53, 10
+; CHECK-NEXT:    vlgvf %r13, %v0, 2
+; CHECK-NEXT:    rosbg %r2, %r13, 54, 54, 9
+; CHECK-NEXT:    vlgvf %r13, %v0, 3
+; CHECK-NEXT:    rosbg %r2, %r13, 55, 55, 8
+; CHECK-NEXT:    vchlf %v0, %v1, %v3
+; CHECK-NEXT:    vlgvf %r13, %v0, 0
+; CHECK-NEXT:    rosbg %r2, %r13, 56, 56, 7
+; CHECK-NEXT:    vlgvf %r13, %v0, 1
+; CHECK-NEXT:    rosbg %r2, %r13, 57, 57, 6
+; CHECK-NEXT:    vlgvf %r13, %v0, 2
+; CHECK-NEXT:    rosbg %r2, %r13, 58, 58, 5
+; CHECK-NEXT:    vlgvf %r13, %v0, 3
+; CHECK-NEXT:    rosbg %r2, %r13, 59, 59, 4
+; CHECK-NEXT:    nilf %r4, 1
+; CHECK-NEXT:    rosbg %r2, %r4, 32, 60, 3
+; CHECK-NEXT:    rosbg %r2, %r3, 32, 61, 2
+; CHECK-NEXT:    rosbg %r2, %r5, 32, 62, 1
+; CHECK-NEXT:    or %r2, %r14
 ; CHECK-NEXT:    vlgvb %r4, %v0, 1
 ; CHECK-NEXT:    vlgvb %r3, %v0, 0
 ; CHECK-NEXT:    risbg %r3, %r3, 48, 176, 15
@@ -122,7 +116,7 @@ define dso_local void @m() local_unnamed_addr #1 {
 ; CHECK-NEXT:    nr %r2, %r0
 ; CHECK-NEXT:    larl %r1, g
 ; CHECK-NEXT:    stc %r2, 0(%r1)
-; CHECK-NEXT:    aghi %r15, 168
+; CHECK-NEXT:    lmg %r13, %r15, 272(%r15)
 ; CHECK-NEXT:    br %r14
 entry:
   %n = alloca i32, align 4
diff --git a/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll b/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
index e8d4b2828c84..10f9f28e599f 100644
--- a/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-max-min-zerosplat.ll
@@ -1,12 +1,15 @@
-; Test vector maximum/minimum with a zero splat on z14.
-;
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
 
+; Test vector maximum/minimum with a zero splat on z14.
+
 define <2 x double> @f1(<2 x double> %val) {
 ; CHECK-LABEL: f1:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfmaxdb %v24, %v24, %v0, 4
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfmaxdb %v24, %v24, %v0, 4
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ogt <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
   ret <2 x double> %ret
@@ -14,9 +17,10 @@ define <2 x double> @f1(<2 x double> %val) {
 
 define <2 x double> @f2(<2 x double> %val) {
 ; CHECK-LABEL: f2:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfmindb %v24, %v24, %v0, 4
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfmindb %v24, %v24, %v0, 4
+; CHECK-NEXT:    br %r14
   %cmp = fcmp olt <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
   ret <2 x double> %ret
@@ -24,9 +28,10 @@ define <2 x double> @f2(<2 x double> %val) {
 
 define <4 x float> @f3(<4 x float> %val) {
 ; CHECK-LABEL: f3:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfmaxsb %v24, %v24, %v0, 4
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfmaxsb %v24, %v24, %v0, 4
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ogt <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
   ret <4 x float> %ret
@@ -34,9 +39,10 @@ define <4 x float> @f3(<4 x float> %val) {
 
 define <4 x float> @f4(<4 x float> %val) {
 ; CHECK-LABEL: f4:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfminsb %v24, %v24, %v0, 4
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfminsb %v24, %v24, %v0, 4
+; CHECK-NEXT:    br %r14
   %cmp = fcmp olt <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
   ret <4 x float> %ret
@@ -44,10 +50,11 @@ define <4 x float> @f4(<4 x float> %val) {
 
 define <2 x double> @f5(<2 x double> %val) {
 ; CHECK-LABEL: f5:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfchedb	%v1, %v0, %v24
-; CHECK-NEXT: vsel	%v24, %v0, %v24, %v1
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfchedb %v0, %v0, %v24
+; CHECK-NEXT:    vnc %v24, %v24, %v0
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ugt <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
   ret <2 x double> %ret
@@ -55,10 +62,11 @@ define <2 x double> @f5(<2 x double> %val) {
 
 define <2 x double> @f6(<2 x double> %val) {
 ; CHECK-LABEL: f6:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfchedb	%v1, %v24, %v0
-; CHECK-NEXT: vsel	%v24, %v0, %v24, %v1
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfchedb %v0, %v24, %v0
+; CHECK-NEXT:    vnc %v24, %v24, %v0
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ult <2 x double> %val,  zeroinitializer
   %ret = select <2 x i1> %cmp, <2 x double> %val, <2 x double> zeroinitializer
   ret <2 x double> %ret
@@ -66,10 +74,11 @@ define <2 x double> @f6(<2 x double> %val) {
 
 define <4 x float> @f7(<4 x float> %val) {
 ; CHECK-LABEL: f7:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfchesb	%v1, %v0, %v24
-; CHECK-NEXT: vsel	%v24, %v0, %v24, %v1
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfchesb %v0, %v0, %v24
+; CHECK-NEXT:    vnc %v24, %v24, %v0
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ugt <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
   ret <4 x float> %ret
@@ -77,10 +86,11 @@ define <4 x float> @f7(<4 x float> %val) {
 
 define <4 x float> @f8(<4 x float> %val) {
 ; CHECK-LABEL: f8:
-; CHECK: vgbm %v0, 0
-; CHECK-NEXT: vfchesb	%v1, %v24, %v0
-; CHECK-NEXT: vsel	%v24, %v0, %v24, %v1
-; CHECK-NEXT: br %r14
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vfchesb %v0, %v24, %v0
+; CHECK-NEXT:    vnc %v24, %v24, %v0
+; CHECK-NEXT:    br %r14
   %cmp = fcmp ult <4 x float> %val,  zeroinitializer
   %ret = select <4 x i1> %cmp, <4 x float> %val, <4 x float> zeroinitializer
   ret <4 x float> %ret
diff --git a/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll b/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll
index 068b56d8cef8..d6aa99b5634d 100644
--- a/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-ada-relocations.ll
@@ -55,7 +55,9 @@ entry:
 
 declare signext i32 @callout(i32 signext)
 
-; CHECK:     .section    ".ada"
+; CHECK: stdin#C CSECT
+; CHECK: C_WSA64 CATTR ALIGN(4),FILL(0),DEFLOAD,NOTEXECUTABLE,RMODE(64),PART(stdin#S)
+; CHECK: stdin#S XATTR LINKAGE(XPLINK),REFERENCE(DATA),SCOPE(SECTION)
 ; CHECK:  .set L#DoFunc@indirect0, DoFunc
 ; CHECK:      .indirect_symbol   L#DoFunc@indirect0
 ; CHECK:  .quad V(L#DoFunc@indirect0)          * Offset 0 pointer to function descriptor DoFunc
diff --git a/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll b/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
index fc52de6c387d..a29646b8bcc6 100644
--- a/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-hlasm-out.ll
@@ -42,6 +42,7 @@ define void @foo() {
 ; CHECK: DS 0B
 ; CHECK-LABEL: L#.str.1 DS 0H
 ; CHECK: DC XL6'576F726C6400'
+; CHECK: END
 entry:
   %0 = load ptr, ptr @Greeting, align 8
   call void (ptr, ...) @outs(ptr noundef %0, ptr noundef @.str.1)
diff --git a/llvm/test/CodeGen/SystemZ/zos-intrinsics.ll b/llvm/test/CodeGen/SystemZ/zos-intrinsics.ll
index b1f0c9e5b8e5..c963d85d60f1 100644
--- a/llvm/test/CodeGen/SystemZ/zos-intrinsics.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-intrinsics.ll
@@ -30,7 +30,9 @@ declare double @llvm.sin.f64(double)
 declare fp128 @llvm.exp2.f128(fp128)
 
 ; Check the calls in the ADA.
-; CHECK: .section ".ada"
+; CHECK: stdin#C CSECT
+; CHECK: C_WSA64 CATTR ALIGN(4),FILL(0),DEFLOAD,NOTEXECUTABLE,RMODE(64),PART(stdin#S)
+; CHECK: stdin#S XATTR LINKAGE(XPLINK),REFERENCE(DATA),SCOPE(SECTION)
 
 ; Check that there is no call to sqrt.
 ; CHECK-NOT:  .quad   R(@@WSQT@B)
diff --git a/llvm/test/CodeGen/SystemZ/zos-landingpad.ll b/llvm/test/CodeGen/SystemZ/zos-landingpad.ll
index 63c332eb423e..307f28061761 100644
--- a/llvm/test/CodeGen/SystemZ/zos-landingpad.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-landingpad.ll
@@ -37,5 +37,7 @@ lpad:
 ; CHECK: Personality routine
 ; CHECK: LSDA location
 ; Check that the exception table is emitted into .lsda section.
-; CHECK: .section ".gcc_exception_table.test1"
+; CHECK:  stdin#C CSECT
+; CHECK:  C_WSA64 CATTR ALIGN(2),FILL(0),NOTEXECUTABLE,RMODE(64),PART(.gcc_exception_table.test1)
+; CHECK:  .gcc_exception_table.test1 XATTR LINKAGE(XPLINK),REFERENCE(DATA),SCOPE(SECTION)
 ; CHECK: GCC_except_table0:
diff --git a/llvm/test/MC/GOFF/ppa1.ll b/llvm/test/CodeGen/SystemZ/zos-ppa1.ll
similarity index 97%
rename from llvm/test/MC/GOFF/ppa1.ll
rename to llvm/test/CodeGen/SystemZ/zos-ppa1.ll
index fe2dc77bba2f..fc51e9c5eccc 100644
--- a/llvm/test/MC/GOFF/ppa1.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-ppa1.ll
@@ -12,7 +12,8 @@
 ; CHECK: *   Bit 1: 1 = Leaf function
 ; CHECK: *   Bit 2: 0 = Does not use alloca
 ; CHECK: L#func_end0:
-; CHECK: .section        ".ppa1"
+; CHECK: stdin#C CSECT
+; CHECK: C_CODE64 CATTR
 ; CHECK: L#PPA1_void_test_0:                     * PPA1
 ; CHECK:        .byte   2                               * Version
 ; CHECK:        .byte   206                             * LE Signature X'CE'
diff --git a/llvm/test/CodeGen/SystemZ/zos-ppa2.ll b/llvm/test/CodeGen/SystemZ/zos-ppa2.ll
index 07025091fb24..8f3847da32e3 100644
--- a/llvm/test/CodeGen/SystemZ/zos-ppa2.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-ppa2.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple s390x-ibm-zos -mcpu=z15 -asm-verbose=true < %s | FileCheck %s
 
-; CHECK:    .section    ".ppa2"
+; CHECK: C_CODE64 CATTR
 ; CHECK: L#PPA2:
 ; CHECK:    .byte   3
 ; CHECK:    .byte   231
@@ -19,7 +19,7 @@
 ; CHECK:    .quad   L#PPA2-CELQSTRT                 * A(PPA2-CELQSTRT)
 ; CHECK: L#PPA1_void_test_0:
 ; CHECK:    .long   L#PPA2-L#PPA1_void_test_0       * Offset to PPA2
-; CHECK:    .section    "B_IDRL"
+; CHECK: B_IDRL CATTR
 ; CHECK:    .byte   0
 ; CHECK:    .byte   3
 ; CHECK:    .short  30
diff --git a/llvm/test/CodeGen/SystemZ/zos-section-1.ll b/llvm/test/CodeGen/SystemZ/zos-section-1.ll
new file mode 100644
index 000000000000..b98584df54d5
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/zos-section-1.ll
@@ -0,0 +1,129 @@
+; RUN: llc <%s --mtriple s390x-ibm-zos --filetype=obj | \
+; RUN:   od -Ax -tx1 -v | FileCheck --ignore-case %s
+
+source_filename = "test.ll"
+
+declare void @other(...)
+
+define void @me() {
+entry:
+  tail call void @other()
+  ret void
+}
+
+; Header record:
+;  03 is prefix byte
+;  f. is header type
+;  .0 is flag
+;  00 is version
+; The 1 at offset 0x33 is the architecture level.
+; CHECK:      000000 03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000030 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+; ESD record, type SD.
+;  03 is prefix byte
+;  0. is header type
+;  .0 is flag
+;  00 is version
+;  00 is type = SD
+; The 01 at offset 0x57 is the id of the symbol.
+; The 60 at offset 0x89 is the tasking behavior.
+; The 01 at offset 0x91 is the binding scope.
+; The name begins at offset 0x97, and is test#C.
+; CHECK-NEXT: 000050 03 00 00 00 [[ROOTSD:00 00 00 01]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000060 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000080 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 60
+; CHECK-NEXT: 000090 00 01 00 00 00 00 00 06 a3 85 a2 a3 7b c3 00 00
+
+; ESD record, type ED.
+; The name is C_CODE64.
+; The regular expression matches the low byte of the length.
+; CHECK-NEXT: 0000a0 03 00 00 01 [[C_CODE64:00 00 00 02]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 0000b0 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 0000c0 00 00 00 00 00 00 00 00 01 80 00 00 00 00 00 00
+; CHECK-NEXT: 0000d0 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 08
+; CHECK-NEXT: 0000e0 00 00 03 00 00 00 00 08 c3 6d c3 d6 c4 c5 f6 f4
+
+; ESD record, type ED.
+; The name is C_@@QPPA2.
+; This record has a continuation, therefore the falg at offset 0xf1 is set.
+; CHECK-NEXT: 0000f0 03 01 00 01 [[C_QPPA2:00 00 00 03]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 000100 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000110 00 00 00 00 00 00 00 00 03 80 00 00 00 00 00 00
+; CHECK-NEXT: 000120 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 08
+; CHECK-NEXT: 000130 00 00 03 00 00 00 00 09 c3 6d 7c 7c d8 d7 d7 c1
+;
+; Continuation:
+; CHECK-NEXT: 000140 03 02 00 f2 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000150 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000160 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000170 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000180 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+; ESD record, type PR.
+; The name is .&ppa2.
+; CHECK-NEXT: 000190 03 00 00 03 [[PPA2:00 00 00 04]] [[C_QPPA2]] 00 00 00 00
+; CHECK-NEXT: 0001a0 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00
+; CHECK-NEXT: 0001b0 00 00 00 00 00 00 00 00 03 20 00 00 00 00 00 00
+; CHECK-NEXT: 0001c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 0001d0 00 01 03 00 00 00 00 06 4b 50 97 97 81 f2 00 00
+
+; ESD record, type ED.
+; The name is C_WSA64.
+; CHECK-NEXT: 0001e0 03 00 00 01 [[C_WSA64:00 00 00 05]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 0001f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000200 00 00 00 00 00 00 00 00 03 81 00 00 00 00 00 00
+; CHECK-NEXT: 000210 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 00
+; CHECK-NEXT: 000220 00 40 04 00 00 00 00 07 c3 6d e6 e2 c1 f6 f4 00
+
+; ESD record, type PR.
+; The name is test#S.
+; CHECK-NEXT: 000230 03 00 00 03 [[TESTS:00 00 00 06]] [[C_WSA64]] 00 00 00 00
+; CHECK-NEXT: 000240 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 000250 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00
+; CHECK-NEXT: 000260 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 000270 00 01 24 00 00 00 00 06 a3 85 a2 a3 7b e2 00 00
+
+; ESD record, type ED.
+; The name is B_IDRL.
+; CHECK-NEXT: 000280 03 00 00 01 [[BIDRL:00 00 00 07]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 000290 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 0002a0 00 00 00 00 00 00 00 00 01 80 00 00 00 00 00 00
+; CHECK-NEXT: 0002b0 00 00 00 00 00 00 00 00 00 00 00 00 00 04 10 08
+; CHECK-NEXT: 0002c0 00 80 03 00 00 00 00 06 c2 6d c9 c4 d9 d3 00 00
+
+; ESD record, type LD.
+; The name is test#C.
+; CHECK-NEXT: 0002d0 03 00 00 02 [[TESTC:00 00 00 08]] [[C_CODE64]] 00 00 00 00
+; CHECK-NEXT: 0002e0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0002f0 00 00 00 00 00 00 00 00 01 00 00 00 [[TESTS]]
+; CHECK-NEXT: 000300 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 02
+; CHECK-NEXT: 000310 00 01 20 00 00 00 00 06 a3 85 a2 a3 7b c3 00 00
+
+; Text record for the code section C_CODE64.
+; The regular expression matches the lower byte of the length.
+; CHECK-NEXT: 000320 03 11 00 00 [[C_CODE64]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000330 00 00 00 00 00 00 00 {{..}} 00 c3 00 c5 00 c5 00 f1
+
+; Text record for the section .&ppa2.
+; CHECK:      0003c0 03 10 00 00 [[PPA2]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0003d0 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; Text record for the ADA section test#S.
+; CHECK:      000410 03 10 00 00 [[TESTS]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000420 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; Text record for the section B_IDRL.
+; CHECK:      000460 03 10 00 01 [[BIDRL]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000470 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; End record.
+; CHECK:      0004b0 03 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0004c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0004d0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0004e0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0004f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
diff --git a/llvm/test/CodeGen/SystemZ/zos-section-2.ll b/llvm/test/CodeGen/SystemZ/zos-section-2.ll
new file mode 100644
index 000000000000..0f608c1206b9
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/zos-section-2.ll
@@ -0,0 +1,175 @@
+; RUN: llc <%s --mtriple s390x-ibm-zos --filetype=obj | \
+; RUN:   od -Ax -tx1 -v | FileCheck --ignore-case %s
+
+source_filename = "test.ll"
+
+@data = hidden global i32 42, align 4
+@bss = hidden global i64 0, align 8
+
+; Header record:
+;  03 is prefix byte
+;  f. is header type
+;  .0 is flag
+;  00 is version
+; The 1 at offset 0x33 is the architecture level.
+; CHECK:      000000 03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000030 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+; ESD record, type SD.
+;  03 is prefix byte
+;  0. is header type
+;  .0 is flag
+;  00 is version
+;  00 is type = SD
+; The 01 at offset 0x57 is the id of the symbol.
+; The 60 at offset 0x89 is the tasking behavior.
+; The 01 at offset 0x91 is the binding scope.
+; The name begins at offset 0x97, and is test#C.
+; CHECK-NEXT: 000050 03 00 00 00 [[ROOTSD:00 00 00 01]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000060 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000080 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 60
+; CHECK-NEXT: 000090 00 01 00 00 00 00 00 06 a3 85 a2 a3 7b c3 00 00
+
+; ESD record, type ED.
+; The name is C_CODE64.
+; The regular expression matches the low byte of the length.
+; CHECK-NEXT: 0000a0 03 00 00 01 [[C_CODE64:00 00 00 02]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 0000b0 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 0000c0 00 00 00 00 00 00 00 00 01 80 00 00 00 00 00 00
+; CHECK-NEXT: 0000d0 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 08
+; CHECK-NEXT: 0000e0 00 00 03 00 00 00 00 08 c3 6d c3 d6 c4 c5 f6 f4
+
+; ESD record, type ED.
+; The name is C_@@QPPA2.
+; This record has a continuation, therefore the falg at offset 0xf1 is set.
+; CHECK-NEXT: 0000f0 03 01 00 01 [[C_QPPA2:00 00 00 03]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 000100 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000110 00 00 00 00 00 00 00 00 03 80 00 00 00 00 00 00
+; CHECK-NEXT: 000120 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 08
+; CHECK-NEXT: 000130 00 00 03 00 00 00 00 09 c3 6d 7c 7c d8 d7 d7 c1
+;
+; Continuation:
+; CHECK-NEXT: 000140 03 02 00 f2 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000150 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000160 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000170 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000180 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+; ESD record, type PR.
+; The name is .&ppa2.
+; CHECK-NEXT: 000190 03 00 00 03 [[PPA2:00 00 00 04]] [[C_QPPA2]] 00 00 00 00
+; CHECK-NEXT: 0001a0 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00
+; CHECK-NEXT: 0001b0 00 00 00 00 00 00 00 00 03 20 00 00 00 00 00 00
+; CHECK-NEXT: 0001c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 0001d0 00 01 03 00 00 00 00 06 4b 50 97 97 81 f2 00 00
+
+; ESD record, type SD.
+; The name is data.
+; CHECK-NEXT: 0001e0 03 00 00 00 [[DATA:00 00 00 05]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0001f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000200 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000210 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000220 00 00 00 00 00 00 00 04 84 81 a3 81 00 00 00 00
+
+; ESD record, type ED.
+; The name is C_WSA64.
+; CHECK-NEXT: 000230 03 00 00 01 [[DATA_WSA:00 00 00 06]] [[DATA]] 00 00 00 00
+; CHECK-NEXT: 000240 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000250 00 00 00 00 00 00 00 00 03 80 00 00 00 00 00 00
+; CHECK-NEXT: 000260 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 00
+; CHECK-NEXT: 000270 00 40 02 00 00 00 00 07 c3 6d e6 e2 c1 f6 f4 00
+
+; ESD record, type PR.
+; The name is data.
+; CHECK-NEXT: 000280 03 00 00 03 [[DATA_PR:00 00 00 07]] [[DATA_WSA]] 00 00 00 00
+; CHECK-NEXT: 000290 00 00 00 00 00 00 00 00 [[DATA_LEN:00 00 00 04]] 00 00 00 00
+; CHECK-NEXT: 0002a0 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00
+; CHECK-NEXT: 0002b0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 0002c0 00 03 22 00 00 00 00 04 84 81 a3 81 00 00 00 00
+
+; ESD record, type SD.
+; The name is bss.
+; CHECK-NEXT: 0002d0 03 00 00 00 [[BSS:00 00 00 08]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0002e0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0002f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000300 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000310 00 00 00 00 00 00 00 03 82 a2 a2 00 00 00 00 00
+
+; ESD record, type ED.
+; The name is C_WSA64.
+; CHECK-NEXT: 000320 03 00 00 01 [[BSS_WSA:00 00 00 09]] [[BSS]] 00 00 00 00
+; CHECK-NEXT: 000330 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000340 00 00 00 00 00 00 00 00 03 80 00 00 00 00 00 00
+; CHECK-NEXT: 000350 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 00
+; CHECK-NEXT: 000360 00 40 03 00 00 00 00 07 c3 6d e6 e2 c1 f6 f4 00
+
+; ESD record, type PR.
+; The name is bss.
+; CHECK-NEXT: 000370 03 00 00 03 [[BSS_PR:00 00 00 0a]] [[BSS_WSA]] 00 00 00 00
+; CHECK-NEXT: 000380 00 00 00 00 00 00 00 00 [[BSS_LEN:00 00 00 08]] 00 00 00 00
+; CHECK-NEXT: 000390 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00
+; CHECK-NEXT: 0003a0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 0003b0 00 03 23 00 00 00 00 03 82 a2 a2 00 00 00 00 00
+
+; ESD record, type ED.
+; The name is C_WSA64.
+; CHECK-NEXT: 0003c0 03 00 00 01 [[C_WSA64:00 00 00 0b]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 0003d0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0003e0 00 00 00 00 00 00 00 00 03 81 00 00 00 00 00 00
+; CHECK-NEXT: 0003f0 00 00 00 00 00 00 00 00 00 00 00 00 00 04 01 00
+; CHECK-NEXT: 000400 00 40 04 00 00 00 00 07 c3 6d e6 e2 c1 f6 f4 00
+
+; ESD record, type PR.
+; The name is test#S.
+; CHECK-NEXT: 000410 03 00 00 03 [[TESTS:00 00 00 0c]] [[C_WSA64]] 00 00 00 00
+; CHECK-NEXT: 000420 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 000430 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00
+; CHECK-NEXT: 000440 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01
+; CHECK-NEXT: 000450 00 01 24 00 00 00 00 06 a3 85 a2 a3 7b e2 00 00
+
+; ESD record, type ED.
+; The name is B_IDRL.
+; CHECK-NEXT: 000460 03 00 00 01 [[BIDRL:00 00 00 0d]] [[ROOTSD]] 00 00 00 00
+; CHECK-NEXT: 000470 00 00 00 00 00 00 00 00 00 00 00 {{..}} 00 00 00 00
+; CHECK-NEXT: 000480 00 00 00 00 00 00 00 00 01 80 00 00 00 00 00 00
+; CHECK-NEXT: 000490 00 00 00 00 00 00 00 00 00 00 00 00 00 04 10 08
+; CHECK-NEXT: 0004a0 00 80 03 00 00 00 00 06 c2 6d c9 c4 d9 d3 00 00
+
+; ESD record, type LD.
+; The name is test#C.
+; CHECK-NEXT: 0004b0 03 00 00 02 [[TESTC:00 00 00 0e]] [[C_CODE64]] 00 00 00 00
+; CHECK-NEXT: 0004c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0004d0 00 00 00 00 00 00 00 00 01 00 00 00 [[TESTS]]
+; CHECK-NEXT: 0004e0 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 02
+; CHECK-NEXT: 0004f0 00 01 20 00 00 00 00 06 a3 85 a2 a3 7b c3 00 00
+
+; Text record for the code section C_CODE64.
+; The regular expression matches the lower byte of the length.
+; CHECK-NEXT: 000500 03 10 00 00 [[C_CODE64]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000510 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; Text record for the section .&ppa2.
+; CHECK:      000550 03 10 00 00 [[PPA2]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000560 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; Text record for the section data.
+; Length is 4, and the content is 0x2a = 42.
+; CHECK:      0005a0 03 10 00 00 [[DATA_PR]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 0005b0 00 00 00 00 00 00 00 04 00 00 00 2a 00 00 00 00
+
+; There is no text record for section bss!
+
+; Text record for the section B_IDRL.
+; CHECK:      0005f0 03 10 00 01 [[BIDRL]] 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000600 00 00 00 00 00 00 00 {{..}} {{.*}}
+
+; End record.
+; CHECK:      000640 03 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000650 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000660 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000670 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+; CHECK-NEXT: 000680 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
diff --git a/llvm/test/CodeGen/SystemZ/zos-simple-test.ll b/llvm/test/CodeGen/SystemZ/zos-simple-test.ll
index a46079818b13..f1f7cee8268f 100644
--- a/llvm/test/CodeGen/SystemZ/zos-simple-test.ll
+++ b/llvm/test/CodeGen/SystemZ/zos-simple-test.ll
@@ -5,9 +5,13 @@
 @a = global i32 0, align 4
 
 define signext i32 @main() {
-; CHECK: .section ".text"
+; CHECK: stdin#C CSECT
+; CHECK: C_CODE64 CATTR ALIGN(3),FILL(0),READONLY,RMODE(64)
 ; CHECK: main:
-; CHECK: .section "a"
+; CHECK: stdin#C CSECT
+; CHECK: C_WSA64 CATTR ALIGN(2),FILL(0),DEFLOAD,NOTEXECUTABLE,RMODE(64),PART(a)
+; CHECK: a XATTR LINKAGE(XPLINK),REFERENCE(DATA),SCOPE(EXPORT)
+; CHECK: a:
 entry:
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll b/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
index 9ee6ec345d96..1136246f6b14 100644
--- a/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-doublereduct.ll
@@ -244,15 +244,12 @@ define i32 @smax_i32(<8 x i32> %a, <4 x i32> %b) {
 define float @nested_add_f32(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-LABEL: nested_add_f32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vadd.f32 s6, s6, s7
-; CHECK-NEXT:    vadd.f32 s4, s4, s5
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vadd.f32 s4, s8, s9
 ; CHECK-NEXT:    vadd.f32 s2, s2, s3
 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
-; CHECK-NEXT:    vadd.f32 s4, s4, s6
-; CHECK-NEXT:    vadd.f32 s0, s0, s2
-; CHECK-NEXT:    vadd.f32 s2, s4, s9
-; CHECK-NEXT:    vadd.f32 s0, s0, s8
 ; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
 ; CHECK-NEXT:    bx lr
   %r1 = call fast float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %a)
   %a1 = fadd fast float %r1, %c
@@ -265,15 +262,12 @@ define float @nested_add_f32(<4 x float> %a, <4 x float> %b, float %c, float %d)
 define float @nested_mul_f32(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-LABEL: nested_mul_f32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmul.f32 s6, s6, s7
-; CHECK-NEXT:    vmul.f32 s4, s4, s5
+; CHECK-NEXT:    vmul.f32 q0, q0, q1
+; CHECK-NEXT:    vmul.f32 s4, s8, s9
 ; CHECK-NEXT:    vmul.f32 s2, s2, s3
 ; CHECK-NEXT:    vmul.f32 s0, s0, s1
-; CHECK-NEXT:    vmul.f32 s4, s4, s6
-; CHECK-NEXT:    vmul.f32 s0, s0, s2
-; CHECK-NEXT:    vmul.f32 s2, s4, s9
-; CHECK-NEXT:    vmul.f32 s0, s0, s8
 ; CHECK-NEXT:    vmul.f32 s0, s0, s2
+; CHECK-NEXT:    vmul.f32 s0, s0, s4
 ; CHECK-NEXT:    bx lr
   %r1 = call fast float @llvm.vector.reduce.fmul.f32.v4f32(float 1.0, <4 x float> %a)
   %a1 = fmul fast float %r1, %c
@@ -301,22 +295,17 @@ define i32 @nested_add_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_mul_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_mul_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vmov r8, r3, d2
-; CHECK-NEXT:    vmov r4, r5, d1
-; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    vmov r12, lr, d3
-; CHECK-NEXT:    mul r3, r8, r3
-; CHECK-NEXT:    muls r5, r4, r5
-; CHECK-NEXT:    mul r2, r12, lr
-; CHECK-NEXT:    muls r7, r6, r7
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vmul.i32 q0, q0, q1
+; CHECK-NEXT:    muls r0, r1, r0
+; CHECK-NEXT:    vmov r12, lr, d1
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    mul r12, r12, lr
 ; CHECK-NEXT:    muls r2, r3, r2
-; CHECK-NEXT:    mul r3, r7, r5
-; CHECK-NEXT:    muls r1, r2, r1
-; CHECK-NEXT:    muls r0, r3, r0
+; CHECK-NEXT:    mul r1, r2, r12
 ; CHECK-NEXT:    muls r0, r1, r0
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r7, pc}
   %r1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a)
   %a1 = mul i32 %r1, %c
   %r2 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %b)
@@ -328,22 +317,17 @@ define i32 @nested_mul_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_and_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_and_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vmov r2, r3, d2
-; CHECK-NEXT:    vmov r12, lr, d3
-; CHECK-NEXT:    vmov r8, r5, d1
-; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    ands r2, r3
-; CHECK-NEXT:    and.w r4, r12, lr
-; CHECK-NEXT:    ands r2, r4
-; CHECK-NEXT:    ands r1, r2
-; CHECK-NEXT:    and.w r2, r8, r5
-; CHECK-NEXT:    and.w r3, r6, r7
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vand q0, q0, q1
+; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    vmov r12, lr, d1
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    and.w r12, r12, lr
 ; CHECK-NEXT:    ands r2, r3
+; CHECK-NEXT:    and.w r2, r2, r12
 ; CHECK-NEXT:    ands r0, r2
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r7, pc}
   %r1 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %a)
   %a1 = and i32 %r1, %c
   %r2 = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %b)
@@ -355,22 +339,17 @@ define i32 @nested_and_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_or_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_or_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vmov r2, r3, d2
-; CHECK-NEXT:    vmov r12, lr, d3
-; CHECK-NEXT:    vmov r8, r5, d1
-; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    orrs r2, r3
-; CHECK-NEXT:    orr.w r4, r12, lr
-; CHECK-NEXT:    orrs r2, r4
-; CHECK-NEXT:    orrs r1, r2
-; CHECK-NEXT:    orr.w r2, r8, r5
-; CHECK-NEXT:    orr.w r3, r6, r7
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    vorr q0, q0, q1
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    vmov r12, lr, d1
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    orr.w r12, r12, lr
 ; CHECK-NEXT:    orrs r2, r3
+; CHECK-NEXT:    orr.w r2, r2, r12
 ; CHECK-NEXT:    orrs r0, r2
-; CHECK-NEXT:    orrs r0, r1
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r7, pc}
   %r1 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %a)
   %a1 = or i32 %r1, %c
   %r2 = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %b)
@@ -382,22 +361,17 @@ define i32 @nested_or_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_xor_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_xor_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    vmov r2, r3, d2
-; CHECK-NEXT:    vmov r12, lr, d3
-; CHECK-NEXT:    vmov r8, r5, d1
-; CHECK-NEXT:    vmov r6, r7, d0
-; CHECK-NEXT:    eors r2, r3
-; CHECK-NEXT:    eor.w r4, r12, lr
-; CHECK-NEXT:    eors r2, r4
-; CHECK-NEXT:    eors r1, r2
-; CHECK-NEXT:    eor.w r2, r8, r5
-; CHECK-NEXT:    eor.w r3, r6, r7
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    veor q0, q0, q1
+; CHECK-NEXT:    eors r0, r1
+; CHECK-NEXT:    vmov r12, lr, d1
+; CHECK-NEXT:    vmov r2, r3, d0
+; CHECK-NEXT:    eor.w r12, r12, lr
 ; CHECK-NEXT:    eors r2, r3
+; CHECK-NEXT:    eor.w r2, r2, r12
 ; CHECK-NEXT:    eors r0, r2
-; CHECK-NEXT:    eors r0, r1
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop {r7, pc}
   %r1 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %a)
   %a1 = xor i32 %r1, %c
   %r2 = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %b)
@@ -409,16 +383,13 @@ define i32 @nested_xor_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_smin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_smin_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mvn r3, #-2147483648
+; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    mvn r2, #-2147483648
-; CHECK-NEXT:    vminv.s32 r3, q1
+; CHECK-NEXT:    vmin.s32 q0, q0, q1
+; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    vminv.s32 r2, q0
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    csel r1, r3, r1, lt
 ; CHECK-NEXT:    cmp r2, r0
 ; CHECK-NEXT:    csel r0, r2, r0, lt
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    csel r0, r0, r1, lt
 ; CHECK-NEXT:    bx lr
   %r1 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %a)
   %a1 = call i32 @llvm.smin.i32(i32 %r1, i32 %c)
@@ -431,16 +402,13 @@ define i32 @nested_smin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_smax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_smax_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov.w r3, #-2147483648
+; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    mov.w r2, #-2147483648
-; CHECK-NEXT:    vmaxv.s32 r3, q1
+; CHECK-NEXT:    vmax.s32 q0, q0, q1
+; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    vmaxv.s32 r2, q0
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    csel r1, r3, r1, gt
 ; CHECK-NEXT:    cmp r2, r0
 ; CHECK-NEXT:    csel r0, r2, r0, gt
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    csel r0, r0, r1, gt
 ; CHECK-NEXT:    bx lr
   %r1 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
   %a1 = call i32 @llvm.smax.i32(i32 %r1, i32 %c)
@@ -453,16 +421,13 @@ define i32 @nested_smax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_umin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_umin_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    mov.w r3, #-1
+; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    mov.w r2, #-1
-; CHECK-NEXT:    vminv.u32 r3, q1
+; CHECK-NEXT:    vmin.u32 q0, q0, q1
+; CHECK-NEXT:    csel r0, r0, r1, lo
 ; CHECK-NEXT:    vminv.u32 r2, q0
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    csel r1, r3, r1, lo
 ; CHECK-NEXT:    cmp r2, r0
 ; CHECK-NEXT:    csel r0, r2, r0, lo
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    csel r0, r0, r1, lo
 ; CHECK-NEXT:    bx lr
   %r1 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %a)
   %a1 = call i32 @llvm.umin.i32(i32 %r1, i32 %c)
@@ -475,16 +440,13 @@ define i32 @nested_umin_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define i32 @nested_umax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: nested_umax_i32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmaxv.u32 r3, q1
+; CHECK-NEXT:    cmp r0, r1
+; CHECK-NEXT:    mov.w r2, #0
+; CHECK-NEXT:    vmax.u32 q0, q0, q1
+; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    vmaxv.u32 r2, q0
-; CHECK-NEXT:    cmp r3, r1
-; CHECK-NEXT:    csel r1, r3, r1, hi
 ; CHECK-NEXT:    cmp r2, r0
 ; CHECK-NEXT:    csel r0, r2, r0, hi
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    csel r0, r0, r1, hi
 ; CHECK-NEXT:    bx lr
   %r1 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %a)
   %a1 = call i32 @llvm.umax.i32(i32 %r1, i32 %c)
@@ -497,14 +459,11 @@ define i32 @nested_umax_i32(<4 x i32> %a, <4 x i32> %b, i32 %c, i32 %d) {
 define float @nested_fmin_float(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-LABEL: nested_fmin_float:
 ; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vminnm.f32 q0, q0, q1
 ; CHECK-NEXT:    vminnm.f32 s2, s2, s3
 ; CHECK-NEXT:    vminnm.f32 s0, s0, s1
 ; CHECK-NEXT:    vminnm.f32 s0, s0, s2
-; CHECK-NEXT:    vminnm.f32 s2, s6, s7
-; CHECK-NEXT:    vminnm.f32 s4, s4, s5
-; CHECK-NEXT:    vminnm.f32 s0, s0, s8
-; CHECK-NEXT:    vminnm.f32 s2, s4, s2
-; CHECK-NEXT:    vminnm.f32 s2, s2, s9
+; CHECK-NEXT:    vminnm.f32 s2, s8, s9
 ; CHECK-NEXT:    vminnm.f32 s0, s0, s2
 ; CHECK-NEXT:    bx lr
   %r1 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> %a)
@@ -518,14 +477,11 @@ define float @nested_fmin_float(<4 x float> %a, <4 x float> %b, float %c, float
 define float @nested_fmax_float(<4 x float> %a, <4 x float> %b, float %c, float %d) {
 ; CHECK-LABEL: nested_fmax_float:
 ; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmaxnm.f32 q0, q0, q1
 ; CHECK-NEXT:    vmaxnm.f32 s2, s2, s3
 ; CHECK-NEXT:    vmaxnm.f32 s0, s0, s1
 ; CHECK-NEXT:    vmaxnm.f32 s0, s0, s2
-; CHECK-NEXT:    vmaxnm.f32 s2, s6, s7
-; CHECK-NEXT:    vmaxnm.f32 s4, s4, s5
-; CHECK-NEXT:    vmaxnm.f32 s0, s0, s8
-; CHECK-NEXT:    vmaxnm.f32 s2, s4, s2
-; CHECK-NEXT:    vmaxnm.f32 s2, s2, s9
+; CHECK-NEXT:    vmaxnm.f32 s2, s8, s9
 ; CHECK-NEXT:    vmaxnm.f32 s0, s0, s2
 ; CHECK-NEXT:    bx lr
   %r1 = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %a)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll b/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
index 726237eb27f2..024de2b36667 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vselect-constants.ll
@@ -282,12 +282,12 @@ define arm_aapcs_vfpcc <2 x i64> @not_signbit_mask_v2i64(<2 x i64> %a, <2 x i64>
 ; CHECK-NEXT:    vmov r1, s1
 ; CHECK-NEXT:    movs r0, #0
 ; CHECK-NEXT:    vmov.i32 q2, #0x0
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    csetm r1, gt
+; CHECK-NEXT:    mvns r1, r1
+; CHECK-NEXT:    asrs r1, r1, #31
 ; CHECK-NEXT:    bfi r0, r1, #0, #8
 ; CHECK-NEXT:    vmov r1, s3
-; CHECK-NEXT:    cmp.w r1, #-1
-; CHECK-NEXT:    csetm r1, gt
+; CHECK-NEXT:    mvns r1, r1
+; CHECK-NEXT:    asrs r1, r1, #31
 ; CHECK-NEXT:    bfi r0, r1, #8, #8
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q1, q2
diff --git a/llvm/test/CodeGen/WebAssembly/PR40172.ll b/llvm/test/CodeGen/WebAssembly/PR40172.ll
index ed1630c4926d..e70b115e6747 100644
--- a/llvm/test/CodeGen/WebAssembly/PR40172.ll
+++ b/llvm/test/CodeGen/WebAssembly/PR40172.ll
@@ -10,9 +10,9 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK:  i32.sub $[[BASE:[0-9]+]]=,
 ; CHECK:  local.copy $[[ARG:[0-9]+]]=, $0{{$}}
-; CHECK:  i32.const $[[A0:[0-9]+]]=, 1{{$}}
-; CHECK:  i32.and $[[A1:[0-9]+]]=, $[[ARG]], $[[A0]]{{$}}
-; CHECK:  i32.store8 8($[[BASE]]), $[[A1]]{{$}}
+; CHECK:  i32.const $push[[A0:[0-9]+]]=, 1{{$}}
+; CHECK:  i32.and $push[[A1:[0-9]+]]=, $[[ARG]], $pop[[A0]]{{$}}
+; CHECK:  i32.store8 8($[[BASE]]), $pop[[A1]]{{$}}
 
 define void @test(i8 %byte) {
   %t = alloca { i8, i8 }, align 8
diff --git a/llvm/test/CodeGen/WebAssembly/PR41841.ll b/llvm/test/CodeGen/WebAssembly/PR41841.ll
index 4e3166adf7bb..76ec83ecf816 100644
--- a/llvm/test/CodeGen/WebAssembly/PR41841.ll
+++ b/llvm/test/CodeGen/WebAssembly/PR41841.ll
@@ -6,9 +6,9 @@ declare void @foo(i128)
 
 ; CHECK-LABEL: test_zext:
 ; CHECK-NEXT: .functype test_zext (i32) -> (){{$}}
-; CHECK-NEXT: i64.extend_i32_u $[[TMP3:[0-9]+]]=, $0{{$}}
-; CHECK-NEXT: i64.const $[[TMP4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: i64.and $[[TMP1:[0-9]+]]=, $[[TMP3]], $[[TMP4]]{{$}}
+; CHECK-NEXT: i64.extend_i32_u $push[[TMP3:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.const $push[[TMP4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: i64.and $[[TMP1:[0-9]+]]=, $pop[[TMP3]], $pop[[TMP4]]{{$}}
 ; CHECK-NEXT: i64.const $[[TMP2:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: call foo, $[[TMP1]], $[[TMP2]]{{$}}
 ; CHECK-NEXT: return{{$}}
@@ -23,11 +23,11 @@ next:                                             ; preds = %start
 
 ; CHECK-LABEL: test_sext:
 ; CHECK-NEXT:.functype test_sext (i32) -> (){{$}}
-; CHECK-NEXT: i64.extend_i32_u $[[TMP3:[0-9]+]]=, $0{{$}}
-; CHECK-NEXT: i64.const $[[TMP4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: i64.and $[[TMP5:[0-9]+]]=, $[[TMP3]], $[[TMP4]]{{$}}
-; CHECK-NEXT: i64.const $[[TMP6:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: i64.sub $[[TMP1:[0-9]+]]=, $[[TMP6]], $[[TMP5]]{{$}}
+; CHECK-NEXT: i64.extend_i32_u $push[[TMP3:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.const $push[[TMP4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: i64.and $[[TMP5:[0-9]+]]=, $pop[[TMP3]], $pop[[TMP4]]{{$}}
+; CHECK-NEXT: i64.const $push[[TMP6:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: i64.sub $[[TMP1:[0-9]+]]=, $pop[[TMP6]], $[[TMP5]]{{$}}
 ; CHECK-NEXT: local.copy $[[TMP2:[0-9]+]]=, $[[TMP1]]{{$}}
 ; CHECK-NEXT: call foo, $[[TMP1]], $[[TMP2]]{{$}}
 ; CHECK-NEXT: return{{$}}
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
index 5a4dc11cd328..d5fbf41d3903 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh-legacy.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/debug-code-stackification.ll b/llvm/test/CodeGen/WebAssembly/debug-code-stackification.ll
new file mode 100644
index 000000000000..98d0f029c169
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/debug-code-stackification.ll
@@ -0,0 +1,89 @@
+; RUN: llc < %s -O0 --filetype=obj -o - | llvm-dwarfdump - | FileCheck %s --check-prefixes DBG_USE
+
+target triple = "wasm32-unknown-unknown"
+
+declare i32 @extern_func(i32, i32)
+
+; We want to produce WASM local "DW_OP_WASM_location 0x00 <local number>" locations
+; in debug code instead of operand stack ("DW_OP_WASM_location 0x02 <depth>") locations
+; since local locations are more widely supported and can cover the entirety of the method.
+; DBG_USE: DW_TAG_subprogram
+; DBG_USE:   DW_AT_name ("single_non_dbg_use")
+; DBG_USE:   DW_TAG_variable
+; DBG_USE:     DW_AT_location
+; DBG_USE:       DW_OP_WASM_location 0x0
+; DBG_USE:     DW_AT_name    ("call_value")
+; DBG_USE:   DW_TAG_variable
+; DBG_USE:     DW_AT_location
+; DBG_USE:       DW_OP_WASM_location 0x0
+; DBG_USE:     DW_AT_name    ("sub_value")
+define i32 @single_non_dbg_use(i32 %0, i32 %1) !dbg !6 {
+  %call_value = call i32 @extern_func(i32 1, i32 2), !dbg !20
+  call void @llvm.dbg.value(metadata i32 %call_value, metadata !11, metadata !DIExpression()), !dbg !20
+  %div = udiv i32 %0, %1, !dbg !21
+  %sub = sub i32 %call_value, %div, !dbg !22
+  call void @llvm.dbg.value(metadata i32 %sub, metadata !12, metadata !DIExpression()), !dbg !22
+  ret i32 %sub, !dbg !23
+}
+
+!6 = distinct !DISubprogram(name: "single_non_dbg_use", scope: !1, file: !1, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9, !9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "call_value", scope: !6, type: !9)
+!12 = !DILocalVariable(name: "sub_value", scope: !6, type: !9)
+!20 = !DILocation(line: 20, scope: !6)
+!21 = !DILocation(line: 21, scope: !6)
+!22 = !DILocation(line: 22, scope: !6)
+!23 = !DILocation(line: 23, scope: !6)
+
+; Similarly for a singly-used frame base.
+; DBG_USE: DW_TAG_subprogram
+; DBG_USE:   DW_AT_frame_base (DW_OP_WASM_location 0x0
+; DBG_USE:   DW_AT_name ("single_use_frame_base")
+; DBG_USE:   DW_TAG_variable
+; DBG_USE:     DW_AT_location (DW_OP_fbreg +12)
+; DBG_USE:     DW_AT_name ("arg_value")
+define i32 @single_use_frame_base(i32 %0, i32 %1) !dbg !13 {
+  %arg_loc = alloca i32, !dbg !24
+  store i32 %1, ptr %arg_loc, !dbg !25
+  call void @llvm.dbg.declare(metadata ptr %arg_loc, metadata !14, metadata !DIExpression()), !dbg !25
+  ret i32 %0, !dbg !26
+}
+
+!13 = distinct !DISubprogram(name: "single_use_frame_base", scope: !1, file: !1, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!14 = !DILocalVariable(name: "arg_value", scope: !13, type: !9)
+!24 = !DILocation(line: 24, scope: !13)
+!25 = !DILocation(line: 25, scope: !13)
+!26 = !DILocation(line: 26, scope: !13)
+
+; Similarly for multivalue defs... But we can't really test
+; it due to https://github.com/llvm/llvm-project/issues/136506.
+; declare {i32, i32} @extern_func_multivalue(i32, i32)
+; 
+; define i32 @single_non_dbg_use_multivalue(i32 %0, i32 %1) !dbg !15 {
+;   %full_value = call {i32, i32} @extern_func_multivalue(i32 1, i32 2), !dbg !27
+;   %full_value_one = extractvalue {i32, i32} %full_value, 0, !dbg !27
+;   %full_value_two = extractvalue {i32, i32} %full_value, 1, !dbg !27
+;   %partial_value = call {i32, i32} @extern_func_multivalue(i32 %full_value_one, i32 %full_value_two), !dbg !28
+;   call void @llvm.dbg.value(metadata i32 %full_value_two, metadata !16, metadata !DIExpression()), !dbg !28
+;   %partial_value_one = extractvalue {i32, i32} %partial_value, 0, !dbg !28
+;   %partial_value_two = extractvalue {i32, i32} %partial_value, 1, !dbg !28
+;   call void @llvm.dbg.value(metadata i32 %partial_value_two, metadata !17, metadata !DIExpression()), !dbg !28
+;   ret i32 %partial_value_one, !dbg !29
+; }
+; 
+; !15 = distinct !DISubprogram(name: "single_non_dbg_use_multivalue", scope: !1, file: !1, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+; !16 = !DILocalVariable(name: "value_used", scope: !15, type: !9)
+; !17 = !DILocalVariable(name: "value_unused", scope: !15, type: !9)
+; !27 = !DILocation(line: 27, scope: !15)
+; !28 = !DILocation(line: 28, scope: !15)
+; !29 = !DILocation(line: 29, scope: !15)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "LLC", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.ll", directory: "")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/CodeGen/WebAssembly/eh-lsda.ll b/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
index ce55f7620e14..d5b28b279d41 100644
--- a/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
+++ b/llvm/test/CodeGen/WebAssembly/eh-lsda.ll
@@ -66,9 +66,9 @@ try.cont:                                         ; preds = %entry, %catch.start
 
 ; CHECK-LABEL: test1:
 ; In static linking, we load GCC_except_table as a constant directly.
-; NOPIC:      i[[PTR]].const  $push[[CONTEXT:.*]]=, __wasm_lpad_context
+; NOPIC:      i[[PTR]].const  $push[[CONTEXT:.*]]=, {{[48]}}
 ; NOPIC-NEXT: i[[PTR]].const  $push[[EXCEPT_TABLE:.*]]=, GCC_except_table1
-; NOPIC-NEXT: i[[PTR]].store  {{[48]}}($pop[[CONTEXT]]), $pop[[EXCEPT_TABLE]]
+; NOPIC-NEXT: i[[PTR]].store  __wasm_lpad_context($pop[[CONTEXT]]), $pop[[EXCEPT_TABLE]]
 
 ; In case of PIC, we make GCC_except_table symbols a relative on based on
 ; __memory_base.
diff --git a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
index b4ffd185e3ca..149e443903a8 100644
--- a/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception-legacy.ll
@@ -34,7 +34,7 @@ define void @throw(ptr %p) {
 ; CHECK:       call      foo
 ; CHECK:     catch     $[[EXN:[0-9]+]]=, __cpp_exception
 ; CHECK:       global.set  __stack_pointer
-; CHECK:       i32.{{store|const}} {{.*}} __wasm_lpad_context
+; CHECK:       i32.store __wasm_lpad_context
 ; CHECK:       call       $drop=, _Unwind_CallPersonality, $[[EXN]]
 ; CHECK:       block
 ; CHECK:         br_if     0
diff --git a/llvm/test/CodeGen/WebAssembly/exception-legacy.mir b/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
index 9273ceeadd0e..95929af4f4ce 100644
--- a/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
+++ b/llvm/test/CodeGen/WebAssembly/exception-legacy.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-late-eh-prepare -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -wasm-use-legacy-eh -wasm-enable-eh -exception-model=wasm -mattr=+exception-handling -run-pass wasm-late-eh-prepare -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/function-info.mir b/llvm/test/CodeGen/WebAssembly/function-info.mir
index d241d59b67a3..ae7c15efe872 100644
--- a/llvm/test/CodeGen/WebAssembly/function-info.mir
+++ b/llvm/test/CodeGen/WebAssembly/function-info.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -exception-model=wasm -mattr=+exception-handling,+multivalue,+simd128 -run-pass wasm-cfg-sort -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
+# RUN: llc -mtriple=wasm32-unknown-unknown -exception-model=wasm -wasm-enable-sjlj -mattr=+exception-handling,+multivalue,+simd128 -run-pass wasm-cfg-sort -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
 
 --- |
   target triple = "wasm32-unknown-unknown"
diff --git a/llvm/test/CodeGen/WebAssembly/inline-asm.ll b/llvm/test/CodeGen/WebAssembly/inline-asm.ll
index 4462cfb7aa0c..c378fd953a55 100644
--- a/llvm/test/CodeGen/WebAssembly/inline-asm.ll
+++ b/llvm/test/CodeGen/WebAssembly/inline-asm.ll
@@ -129,7 +129,18 @@ entry:
   ret i32 %ret
 }
 
+; CHECK-LABEL: v128_load
+; CHECK: local.get 0
+; CHECK-NEXT: v128.load 0
+; CHECK-NEXT: local.set 1
+define <4 x i32> @v128_load(ptr %v) #1 {
+entry:
+  %0 = tail call <4 x i32> asm "local.get $1\0Av128.load 0\0Alocal.set $0", "=r,r"(ptr %v)
+  ret <4 x i32> %0
+}
+
 attributes #0 = { nounwind }
+attributes #1 = { "target-features"="+simd128" }
 
 !0 = !{i32 47}
 !1 = !{i32 145}
diff --git a/llvm/test/CodeGen/WebAssembly/offset.ll b/llvm/test/CodeGen/WebAssembly/offset.ll
index 130508424f63..109607c91427 100644
--- a/llvm/test/CodeGen/WebAssembly/offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset.ll
@@ -118,6 +118,16 @@ define i32 @load_i32_from_global_address() {
   ret i32 %t
 }
 
+define i32 @load_i32_global_with_folded_gep_offset_nonconst_nuw(i32 %idx) {
+; CHECK-LABEL: load_i32_global_with_folded_gep_offset_nonconst_nuw:
+; CHECK: i32.const $push0=, 2
+; CHECK: i32.shl $push1=, $0, $pop0
+; CHECK: i32.load $push2=, gv($pop1)
+  %s = getelementptr nuw i32, ptr @gv, i32 %idx
+  %t = load i32, ptr %s
+  ret i32 %t
+}
+
 ;===----------------------------------------------------------------------------
 ; Loads: 64-bit
 ;===----------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/WebAssembly/pr51651.ll b/llvm/test/CodeGen/WebAssembly/pr51651.ll
index 56aa99b97f3f..73285746ef7c 100644
--- a/llvm/test/CodeGen/WebAssembly/pr51651.ll
+++ b/llvm/test/CodeGen/WebAssembly/pr51651.ll
@@ -9,17 +9,17 @@ define i32 @test(ptr %p, ptr %p2) {
 ; CHECK-NEXT:    i32.eqz $2=, $3
 ; CHECK-NEXT:    i32.store8 0($1), $3
 ; CHECK-NEXT:  # %bb.1: # %bb2
-; CHECK-NEXT:    i32.const $4=, 1
-; CHECK-NEXT:    i32.and $5=, $2, $4
 ; CHECK-NEXT:    block
-; CHECK-NEXT:    br_if 0, $5 # 0: down to label0
+; CHECK-NEXT:    i32.const $push0=, 1
+; CHECK-NEXT:    i32.and $push1=, $2, $pop0
+; CHECK-NEXT:    br_if 0, $pop1 # 0: down to label0
 ; CHECK-NEXT:  # %bb.2: # %bb4
-; CHECK-NEXT:    i32.const $6=, 0
-; CHECK-NEXT:    return $6
+; CHECK-NEXT:    i32.const $push2=, 0
+; CHECK-NEXT:    return $pop2
 ; CHECK-NEXT:  .LBB0_3: # %bb3
 ; CHECK-NEXT:    end_block # label0:
-; CHECK-NEXT:    i32.const $7=, 1
-; CHECK-NEXT:    return $7
+; CHECK-NEXT:    i32.const $push3=, 1
+; CHECK-NEXT:    return $pop3
   %v = load i8, ptr %p
   %v.ext = zext i8 %v to i32
   %cond = icmp eq i32 %v.ext, 0
diff --git a/llvm/test/CodeGen/WebAssembly/signext-zeroext-callsite.ll b/llvm/test/CodeGen/WebAssembly/signext-zeroext-callsite.ll
index e33337f27806..f7f0b68c1b66 100644
--- a/llvm/test/CodeGen/WebAssembly/signext-zeroext-callsite.ll
+++ b/llvm/test/CodeGen/WebAssembly/signext-zeroext-callsite.ll
@@ -12,7 +12,7 @@ declare i32 @foo(i1 signext noundef, i32 noundef)
 define i32 @callsite_nosignext() {
 ; CHECK-LABEL: callsite_nosignext:
 ; CHECK:         .functype callsite_nosignext () -> (i32)
-; CHECK-NEXT:    .local i32, i32, i32, i32, i32, i32
+; CHECK-NEXT:    .local i32, i32, i32
 ; CHECK-NEXT:  # %bb.0: # %start
 ; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    local.set 0
@@ -23,31 +23,21 @@ define i32 @callsite_nosignext() {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.set 3
-; CHECK-NEXT:    local.get 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i32.shr_s
-; CHECK-NEXT:    local.set 4
-; CHECK-NEXT:    local.get 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    call foo
-; CHECK-NEXT:    local.set 5
-; CHECK-NEXT:    local.get 5
 ; CHECK-NEXT:    return
 ;
 ; NO-FAST-ISEL-LABEL: callsite_nosignext:
 ; NO-FAST-ISEL:         .functype callsite_nosignext () -> (i32)
-; NO-FAST-ISEL-NEXT:    .local i32, i32, i32
+; NO-FAST-ISEL-NEXT:    .local i32
 ; NO-FAST-ISEL-NEXT:  # %bb.0: # %start
 ; NO-FAST-ISEL-NEXT:    i32.const 0
 ; NO-FAST-ISEL-NEXT:    local.set 0
 ; NO-FAST-ISEL-NEXT:    i32.const -1
-; NO-FAST-ISEL-NEXT:    local.set 1
-; NO-FAST-ISEL-NEXT:    local.get 1
 ; NO-FAST-ISEL-NEXT:    local.get 0
 ; NO-FAST-ISEL-NEXT:    call foo
-; NO-FAST-ISEL-NEXT:    local.set 2
-; NO-FAST-ISEL-NEXT:    local.get 2
 ; NO-FAST-ISEL-NEXT:    return
 start:
   %0 = call i32 @foo(i1 1, i32 0)
@@ -57,7 +47,7 @@ start:
 define i32 @callsite_signext() {
 ; CHECK-LABEL: callsite_signext:
 ; CHECK:         .functype callsite_signext () -> (i32)
-; CHECK-NEXT:    .local i32, i32, i32, i32, i32, i32
+; CHECK-NEXT:    .local i32, i32, i32
 ; CHECK-NEXT:  # %bb.0: # %start
 ; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    local.set 0
@@ -68,31 +58,21 @@ define i32 @callsite_signext() {
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.set 3
-; CHECK-NEXT:    local.get 3
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i32.shr_s
-; CHECK-NEXT:    local.set 4
-; CHECK-NEXT:    local.get 4
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    call foo
-; CHECK-NEXT:    local.set 5
-; CHECK-NEXT:    local.get 5
 ; CHECK-NEXT:    return
 ;
 ; NO-FAST-ISEL-LABEL: callsite_signext:
 ; NO-FAST-ISEL:         .functype callsite_signext () -> (i32)
-; NO-FAST-ISEL-NEXT:    .local i32, i32, i32
+; NO-FAST-ISEL-NEXT:    .local i32
 ; NO-FAST-ISEL-NEXT:  # %bb.0: # %start
 ; NO-FAST-ISEL-NEXT:    i32.const 0
 ; NO-FAST-ISEL-NEXT:    local.set 0
 ; NO-FAST-ISEL-NEXT:    i32.const -1
-; NO-FAST-ISEL-NEXT:    local.set 1
-; NO-FAST-ISEL-NEXT:    local.get 1
 ; NO-FAST-ISEL-NEXT:    local.get 0
 ; NO-FAST-ISEL-NEXT:    call foo
-; NO-FAST-ISEL-NEXT:    local.set 2
-; NO-FAST-ISEL-NEXT:    local.get 2
 ; NO-FAST-ISEL-NEXT:    return
 start:
   %0 = call i32 @foo(i1 signext 1, i32 0)
@@ -106,38 +86,28 @@ declare i32 @foo2(i1 zeroext noundef, i32 noundef)
 define i32 @callsite_nozeroext() {
 ; CHECK-LABEL: callsite_nozeroext:
 ; CHECK:         .functype callsite_nozeroext () -> (i32)
-; CHECK-NEXT:    .local i32, i32, i32, i32, i32
+; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0: # %start
 ; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    local.set 0
 ; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.set 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    local.set 3
-; CHECK-NEXT:    local.get 3
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    call foo2
-; CHECK-NEXT:    local.set 4
-; CHECK-NEXT:    local.get 4
 ; CHECK-NEXT:    return
 ;
 ; NO-FAST-ISEL-LABEL: callsite_nozeroext:
 ; NO-FAST-ISEL:         .functype callsite_nozeroext () -> (i32)
-; NO-FAST-ISEL-NEXT:    .local i32, i32, i32
+; NO-FAST-ISEL-NEXT:    .local i32
 ; NO-FAST-ISEL-NEXT:  # %bb.0: # %start
 ; NO-FAST-ISEL-NEXT:    i32.const 0
 ; NO-FAST-ISEL-NEXT:    local.set 0
 ; NO-FAST-ISEL-NEXT:    i32.const 1
-; NO-FAST-ISEL-NEXT:    local.set 1
-; NO-FAST-ISEL-NEXT:    local.get 1
 ; NO-FAST-ISEL-NEXT:    local.get 0
 ; NO-FAST-ISEL-NEXT:    call foo2
-; NO-FAST-ISEL-NEXT:    local.set 2
-; NO-FAST-ISEL-NEXT:    local.get 2
 ; NO-FAST-ISEL-NEXT:    return
 start:
   %0 = call i32 @foo2(i1 1, i32 0)
@@ -147,38 +117,28 @@ start:
 define i32 @callsite_zeroext() {
 ; CHECK-LABEL: callsite_zeroext:
 ; CHECK:         .functype callsite_zeroext () -> (i32)
-; CHECK-NEXT:    .local i32, i32, i32, i32, i32
+; CHECK-NEXT:    .local i32, i32
 ; CHECK-NEXT:  # %bb.0: # %start
 ; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    local.set 0
 ; CHECK-NEXT:    i32.const 0
 ; CHECK-NEXT:    local.set 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    local.set 2
 ; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    local.get 2
+; CHECK-NEXT:    i32.const 1
 ; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    local.set 3
-; CHECK-NEXT:    local.get 3
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    call foo2
-; CHECK-NEXT:    local.set 4
-; CHECK-NEXT:    local.get 4
 ; CHECK-NEXT:    return
 ;
 ; NO-FAST-ISEL-LABEL: callsite_zeroext:
 ; NO-FAST-ISEL:         .functype callsite_zeroext () -> (i32)
-; NO-FAST-ISEL-NEXT:    .local i32, i32, i32
+; NO-FAST-ISEL-NEXT:    .local i32
 ; NO-FAST-ISEL-NEXT:  # %bb.0: # %start
 ; NO-FAST-ISEL-NEXT:    i32.const 0
 ; NO-FAST-ISEL-NEXT:    local.set 0
 ; NO-FAST-ISEL-NEXT:    i32.const 1
-; NO-FAST-ISEL-NEXT:    local.set 1
-; NO-FAST-ISEL-NEXT:    local.get 1
 ; NO-FAST-ISEL-NEXT:    local.get 0
 ; NO-FAST-ISEL-NEXT:    call foo2
-; NO-FAST-ISEL-NEXT:    local.set 2
-; NO-FAST-ISEL-NEXT:    local.get 2
 ; NO-FAST-ISEL-NEXT:    return
 start:
   %0 = call i32 @foo2(i1 zeroext 1, i32 0)
diff --git a/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll b/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
index ca160c091b22..c2a43a839b1e 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-bitmask.ll
@@ -177,236 +177,20 @@ define i32 @bitmask_v32i8(<32 x i8> %v) {
 ; CHECK:         .functype bitmask_v32i8 (v128, v128) -> (i32)
 ; CHECK-NEXT:    .local v128
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    global.get __stack_pointer
 ; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.sub
-; CHECK-NEXT:    drop
 ; CHECK-NEXT:    local.get 0
 ; CHECK-NEXT:    v128.const 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
 ; CHECK-NEXT:    local.tee 2
 ; CHECK-NEXT:    i8x16.eq
-; CHECK-NEXT:    local.tee 0
-; CHECK-NEXT:    i8x16.extract_lane_u 0
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 2
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 2
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 3
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 3
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 4
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 4
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 5
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 5
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 6
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 6
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 7
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 7
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 8
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 8
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 9
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 9
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 10
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 10
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 11
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 11
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 12
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 12
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 13
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 13
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 14
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 14
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 15
-; CHECK-NEXT:    i32.const 15
+; CHECK-NEXT:    i8x16.bitmask
+; CHECK-NEXT:    i32.const 16
+; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.const 65535
-; CHECK-NEXT:    i32.and
 ; CHECK-NEXT:    local.get 1
 ; CHECK-NEXT:    local.get 2
 ; CHECK-NEXT:    i8x16.eq
-; CHECK-NEXT:    local.tee 0
-; CHECK-NEXT:    i8x16.extract_lane_u 15
-; CHECK-NEXT:    i32.const 31
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 14
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 30
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 13
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 29
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 12
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 28
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 11
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 27
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 10
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 26
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 9
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 25
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 8
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 24
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 7
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 23
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 6
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 22
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 5
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 21
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 4
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 20
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 3
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 19
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 2
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 18
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 1
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 17
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    local.get 0
-; CHECK-NEXT:    i8x16.extract_lane_u 0
-; CHECK-NEXT:    i32.const 1
-; CHECK-NEXT:    i32.and
-; CHECK-NEXT:    i32.const 16
-; CHECK-NEXT:    i32.shl
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
-; CHECK-NEXT:    i32.or
+; CHECK-NEXT:    i8x16.bitmask
+; CHECK-NEXT:    i32.add
 ; CHECK-NEXT:    # fallthrough-return
   %cmp = icmp eq <32 x i8> %v, zeroinitializer
   %bitmask = bitcast <32 x i1> %cmp to i32
diff --git a/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll b/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll
new file mode 100644
index 000000000000..e497edc08c4e
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-illegal-bitmask.ll
@@ -0,0 +1,596 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -O3 -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+
+target triple = "wasm64"
+
+
+define i16 @legal_bitcast_v16i8(<16 x i8> %x) {
+; CHECK-LABEL: legal_bitcast_v16i8:
+; CHECK:         .functype legal_bitcast_v16i8 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.const $push0=, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16
+; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
+; CHECK-NEXT:    i8x16.bitmask $push2=, $pop1
+; CHECK-NEXT:    return $pop2
+    %z = icmp eq <16 x i8> %x, splat (i8 16)
+    %res = bitcast <16 x i1> %z to i16
+    ret i16 %res
+}
+
+define i32 @optimize_illegal_bitcast_v32i8(<32 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8 (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push2=, 16
+; CHECK-NEXT:    v128.const $push10=, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32
+; CHECK-NEXT:    local.tee $push9=, $2=, $pop10
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $pop9
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i32.const $push8=, 16
+; CHECK-NEXT:    i32.add $push3=, $pop1, $pop8
+; CHECK-NEXT:    i32.shl $push4=, $pop2, $pop3
+; CHECK-NEXT:    i8x16.eq $push5=, $1, $2
+; CHECK-NEXT:    i8x16.bitmask $push6=, $pop5
+; CHECK-NEXT:    i32.add $push7=, $pop4, $pop6
+; CHECK-NEXT:    return $pop7
+    %z = icmp eq <32 x i8> %x, splat (i8 32)
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+define i32 @optimize_illegal_bitcast_v32i8_const_step_vec(<32 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8_const_step_vec:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8_const_step_vec (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push3=, 16
+; CHECK-NEXT:    v128.const $push0=, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
+; CHECK-NEXT:    i8x16.bitmask $push2=, $pop1
+; CHECK-NEXT:    i32.const $push10=, 16
+; CHECK-NEXT:    i32.add $push4=, $pop2, $pop10
+; CHECK-NEXT:    i32.shl $push5=, $pop3, $pop4
+; CHECK-NEXT:    v128.const $push6=, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32
+; CHECK-NEXT:    i8x16.eq $push7=, $1, $pop6
+; CHECK-NEXT:    i8x16.bitmask $push8=, $pop7
+; CHECK-NEXT:    i32.add $push9=, $pop5, $pop8
+; CHECK-NEXT:    return $pop9
+    %const_step_vec =  add <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8,
+                              i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16,
+                              i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24,
+                              i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32>, zeroinitializer
+    %z = icmp eq <32 x i8> %x, %const_step_vec
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+define i32 @optimize_illegal_bitcast_v32i8_non_const_vec(<32 x i8> %x, <32 x i8> %y) {
+; CHECK-LABEL: optimize_illegal_bitcast_v32i8_non_const_vec:
+; CHECK:         .functype optimize_illegal_bitcast_v32i8_non_const_vec (v128, v128, v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32.const $push2=, 16
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $2
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i32.const $push8=, 16
+; CHECK-NEXT:    i32.add $push3=, $pop1, $pop8
+; CHECK-NEXT:    i32.shl $push4=, $pop2, $pop3
+; CHECK-NEXT:    i8x16.eq $push5=, $1, $3
+; CHECK-NEXT:    i8x16.bitmask $push6=, $pop5
+; CHECK-NEXT:    i32.add $push7=, $pop4, $pop6
+; CHECK-NEXT:    return $pop7
+    %z = icmp eq <32 x i8> %x, %y
+    %res = bitcast <32 x i1> %z to i32
+    ret i32 %res
+}
+
+
+
+
+define i64 @optimize_illegal_bitcast_v64i8(<64 x i8> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v64i8:
+; CHECK:         .functype optimize_illegal_bitcast_v64i8 (v128, v128, v128, v128) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64.const $push3=, 16
+; CHECK-NEXT:    i64.const $push24=, 16
+; CHECK-NEXT:    i64.const $push23=, 16
+; CHECK-NEXT:    v128.const $push22=, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64
+; CHECK-NEXT:    local.tee $push21=, $4=, $pop22
+; CHECK-NEXT:    i8x16.eq $push0=, $0, $pop21
+; CHECK-NEXT:    i8x16.bitmask $push1=, $pop0
+; CHECK-NEXT:    i64.extend_i32_u $push2=, $pop1
+; CHECK-NEXT:    i64.const $push20=, 16
+; CHECK-NEXT:    i64.add $push4=, $pop2, $pop20
+; CHECK-NEXT:    i64.shl $push5=, $pop23, $pop4
+; CHECK-NEXT:    i8x16.eq $push6=, $1, $4
+; CHECK-NEXT:    i8x16.bitmask $push7=, $pop6
+; CHECK-NEXT:    i64.extend_i32_u $push8=, $pop7
+; CHECK-NEXT:    i64.add $push9=, $pop5, $pop8
+; CHECK-NEXT:    i64.shl $push10=, $pop24, $pop9
+; CHECK-NEXT:    i8x16.eq $push11=, $2, $4
+; CHECK-NEXT:    i8x16.bitmask $push12=, $pop11
+; CHECK-NEXT:    i64.extend_i32_u $push13=, $pop12
+; CHECK-NEXT:    i64.add $push14=, $pop10, $pop13
+; CHECK-NEXT:    i64.shl $push15=, $pop3, $pop14
+; CHECK-NEXT:    i8x16.eq $push16=, $3, $4
+; CHECK-NEXT:    i8x16.bitmask $push17=, $pop16
+; CHECK-NEXT:    i64.extend_i32_u $push18=, $pop17
+; CHECK-NEXT:    i64.add $push19=, $pop15, $pop18
+; CHECK-NEXT:    return $pop19
+    %z = icmp eq <64 x i8> %x, splat (i8 64)
+    %res = bitcast <64 x i1> %z to i64
+    ret i64 %res
+}
+
+define i64 @optimize_illegal_bitcast_v64i4(<64 x i4> %x) {
+; CHECK-LABEL: optimize_illegal_bitcast_v64i4:
+; CHECK:         .functype optimize_illegal_bitcast_v64i4 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> (i64)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    global.get $push355=, __stack_pointer
+; CHECK-NEXT:    i64.const $push356=, 16
+; CHECK-NEXT:    i64.sub $drop=, $pop355, $pop356
+; CHECK-NEXT:    i8x16.splat $push273=, $0
+; CHECK-NEXT:    i8x16.replace_lane $push274=, $pop273, 1, $1
+; CHECK-NEXT:    i8x16.replace_lane $push275=, $pop274, 2, $2
+; CHECK-NEXT:    i8x16.replace_lane $push276=, $pop275, 3, $3
+; CHECK-NEXT:    i8x16.replace_lane $push277=, $pop276, 4, $4
+; CHECK-NEXT:    i8x16.replace_lane $push278=, $pop277, 5, $5
+; CHECK-NEXT:    i8x16.replace_lane $push279=, $pop278, 6, $6
+; CHECK-NEXT:    i8x16.replace_lane $push280=, $pop279, 7, $7
+; CHECK-NEXT:    i8x16.replace_lane $push281=, $pop280, 8, $8
+; CHECK-NEXT:    i8x16.replace_lane $push282=, $pop281, 9, $9
+; CHECK-NEXT:    i8x16.replace_lane $push283=, $pop282, 10, $10
+; CHECK-NEXT:    i8x16.replace_lane $push284=, $pop283, 11, $11
+; CHECK-NEXT:    i8x16.replace_lane $push285=, $pop284, 12, $12
+; CHECK-NEXT:    i8x16.replace_lane $push286=, $pop285, 13, $13
+; CHECK-NEXT:    i8x16.replace_lane $push287=, $pop286, 14, $14
+; CHECK-NEXT:    i8x16.replace_lane $push288=, $pop287, 15, $15
+; CHECK-NEXT:    v128.const $push460=, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15
+; CHECK-NEXT:    local.tee $push459=, $64=, $pop460
+; CHECK-NEXT:    v128.and $push289=, $pop288, $pop459
+; CHECK-NEXT:    v128.const $push458=, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+; CHECK-NEXT:    local.tee $push457=, $65=, $pop458
+; CHECK-NEXT:    i8x16.eq $push456=, $pop289, $pop457
+; CHECK-NEXT:    local.tee $push455=, $66=, $pop456
+; CHECK-NEXT:    i8x16.extract_lane_u $push290=, $pop455, 0
+; CHECK-NEXT:    i32.const $push18=, 1
+; CHECK-NEXT:    i32.and $push291=, $pop290, $pop18
+; CHECK-NEXT:    i8x16.extract_lane_u $push292=, $66, 1
+; CHECK-NEXT:    i32.const $push454=, 1
+; CHECK-NEXT:    i32.and $push293=, $pop292, $pop454
+; CHECK-NEXT:    i32.const $push453=, 1
+; CHECK-NEXT:    i32.shl $push294=, $pop293, $pop453
+; CHECK-NEXT:    i32.or $push295=, $pop291, $pop294
+; CHECK-NEXT:    i8x16.extract_lane_u $push296=, $66, 2
+; CHECK-NEXT:    i32.const $push452=, 1
+; CHECK-NEXT:    i32.and $push297=, $pop296, $pop452
+; CHECK-NEXT:    i32.const $push121=, 2
+; CHECK-NEXT:    i32.shl $push298=, $pop297, $pop121
+; CHECK-NEXT:    i32.or $push299=, $pop295, $pop298
+; CHECK-NEXT:    i8x16.extract_lane_u $push300=, $66, 3
+; CHECK-NEXT:    i32.const $push451=, 1
+; CHECK-NEXT:    i32.and $push301=, $pop300, $pop451
+; CHECK-NEXT:    i32.const $push126=, 3
+; CHECK-NEXT:    i32.shl $push302=, $pop301, $pop126
+; CHECK-NEXT:    i32.or $push303=, $pop299, $pop302
+; CHECK-NEXT:    i8x16.extract_lane_u $push304=, $66, 4
+; CHECK-NEXT:    i32.const $push450=, 1
+; CHECK-NEXT:    i32.and $push305=, $pop304, $pop450
+; CHECK-NEXT:    i32.const $push131=, 4
+; CHECK-NEXT:    i32.shl $push306=, $pop305, $pop131
+; CHECK-NEXT:    i32.or $push307=, $pop303, $pop306
+; CHECK-NEXT:    i8x16.extract_lane_u $push308=, $66, 5
+; CHECK-NEXT:    i32.const $push449=, 1
+; CHECK-NEXT:    i32.and $push309=, $pop308, $pop449
+; CHECK-NEXT:    i32.const $push136=, 5
+; CHECK-NEXT:    i32.shl $push310=, $pop309, $pop136
+; CHECK-NEXT:    i32.or $push311=, $pop307, $pop310
+; CHECK-NEXT:    i8x16.extract_lane_u $push312=, $66, 6
+; CHECK-NEXT:    i32.const $push448=, 1
+; CHECK-NEXT:    i32.and $push313=, $pop312, $pop448
+; CHECK-NEXT:    i32.const $push141=, 6
+; CHECK-NEXT:    i32.shl $push314=, $pop313, $pop141
+; CHECK-NEXT:    i32.or $push315=, $pop311, $pop314
+; CHECK-NEXT:    i8x16.extract_lane_u $push316=, $66, 7
+; CHECK-NEXT:    i32.const $push447=, 1
+; CHECK-NEXT:    i32.and $push317=, $pop316, $pop447
+; CHECK-NEXT:    i32.const $push146=, 7
+; CHECK-NEXT:    i32.shl $push318=, $pop317, $pop146
+; CHECK-NEXT:    i32.or $push319=, $pop315, $pop318
+; CHECK-NEXT:    i8x16.extract_lane_u $push320=, $66, 8
+; CHECK-NEXT:    i32.const $push446=, 1
+; CHECK-NEXT:    i32.and $push321=, $pop320, $pop446
+; CHECK-NEXT:    i32.const $push151=, 8
+; CHECK-NEXT:    i32.shl $push322=, $pop321, $pop151
+; CHECK-NEXT:    i32.or $push323=, $pop319, $pop322
+; CHECK-NEXT:    i8x16.extract_lane_u $push324=, $66, 9
+; CHECK-NEXT:    i32.const $push445=, 1
+; CHECK-NEXT:    i32.and $push325=, $pop324, $pop445
+; CHECK-NEXT:    i32.const $push156=, 9
+; CHECK-NEXT:    i32.shl $push326=, $pop325, $pop156
+; CHECK-NEXT:    i32.or $push327=, $pop323, $pop326
+; CHECK-NEXT:    i8x16.extract_lane_u $push328=, $66, 10
+; CHECK-NEXT:    i32.const $push444=, 1
+; CHECK-NEXT:    i32.and $push329=, $pop328, $pop444
+; CHECK-NEXT:    i32.const $push161=, 10
+; CHECK-NEXT:    i32.shl $push330=, $pop329, $pop161
+; CHECK-NEXT:    i32.or $push331=, $pop327, $pop330
+; CHECK-NEXT:    i8x16.extract_lane_u $push332=, $66, 11
+; CHECK-NEXT:    i32.const $push443=, 1
+; CHECK-NEXT:    i32.and $push333=, $pop332, $pop443
+; CHECK-NEXT:    i32.const $push166=, 11
+; CHECK-NEXT:    i32.shl $push334=, $pop333, $pop166
+; CHECK-NEXT:    i32.or $push335=, $pop331, $pop334
+; CHECK-NEXT:    i8x16.extract_lane_u $push336=, $66, 12
+; CHECK-NEXT:    i32.const $push442=, 1
+; CHECK-NEXT:    i32.and $push337=, $pop336, $pop442
+; CHECK-NEXT:    i32.const $push171=, 12
+; CHECK-NEXT:    i32.shl $push338=, $pop337, $pop171
+; CHECK-NEXT:    i32.or $push339=, $pop335, $pop338
+; CHECK-NEXT:    i8x16.extract_lane_u $push340=, $66, 13
+; CHECK-NEXT:    i32.const $push441=, 1
+; CHECK-NEXT:    i32.and $push341=, $pop340, $pop441
+; CHECK-NEXT:    i32.const $push176=, 13
+; CHECK-NEXT:    i32.shl $push342=, $pop341, $pop176
+; CHECK-NEXT:    i32.or $push343=, $pop339, $pop342
+; CHECK-NEXT:    i8x16.extract_lane_u $push344=, $66, 14
+; CHECK-NEXT:    i32.const $push440=, 1
+; CHECK-NEXT:    i32.and $push345=, $pop344, $pop440
+; CHECK-NEXT:    i32.const $push181=, 14
+; CHECK-NEXT:    i32.shl $push346=, $pop345, $pop181
+; CHECK-NEXT:    i32.or $push347=, $pop343, $pop346
+; CHECK-NEXT:    i8x16.extract_lane_u $push348=, $66, 15
+; CHECK-NEXT:    i32.const $push185=, 15
+; CHECK-NEXT:    i32.shl $push349=, $pop348, $pop185
+; CHECK-NEXT:    i32.or $push350=, $pop347, $pop349
+; CHECK-NEXT:    i32.const $push188=, 65535
+; CHECK-NEXT:    i32.and $push351=, $pop350, $pop188
+; CHECK-NEXT:    i8x16.splat $push194=, $16
+; CHECK-NEXT:    i8x16.replace_lane $push195=, $pop194, 1, $17
+; CHECK-NEXT:    i8x16.replace_lane $push196=, $pop195, 2, $18
+; CHECK-NEXT:    i8x16.replace_lane $push197=, $pop196, 3, $19
+; CHECK-NEXT:    i8x16.replace_lane $push198=, $pop197, 4, $20
+; CHECK-NEXT:    i8x16.replace_lane $push199=, $pop198, 5, $21
+; CHECK-NEXT:    i8x16.replace_lane $push200=, $pop199, 6, $22
+; CHECK-NEXT:    i8x16.replace_lane $push201=, $pop200, 7, $23
+; CHECK-NEXT:    i8x16.replace_lane $push202=, $pop201, 8, $24
+; CHECK-NEXT:    i8x16.replace_lane $push203=, $pop202, 9, $25
+; CHECK-NEXT:    i8x16.replace_lane $push204=, $pop203, 10, $26
+; CHECK-NEXT:    i8x16.replace_lane $push205=, $pop204, 11, $27
+; CHECK-NEXT:    i8x16.replace_lane $push206=, $pop205, 12, $28
+; CHECK-NEXT:    i8x16.replace_lane $push207=, $pop206, 13, $29
+; CHECK-NEXT:    i8x16.replace_lane $push208=, $pop207, 14, $30
+; CHECK-NEXT:    i8x16.replace_lane $push209=, $pop208, 15, $31
+; CHECK-NEXT:    v128.and $push210=, $pop209, $64
+; CHECK-NEXT:    i8x16.eq $push439=, $pop210, $65
+; CHECK-NEXT:    local.tee $push438=, $66=, $pop439
+; CHECK-NEXT:    i8x16.extract_lane_u $push270=, $pop438, 15
+; CHECK-NEXT:    i32.const $push93=, 31
+; CHECK-NEXT:    i32.shl $push271=, $pop270, $pop93
+; CHECK-NEXT:    i8x16.extract_lane_u $push266=, $66, 14
+; CHECK-NEXT:    i32.const $push437=, 1
+; CHECK-NEXT:    i32.and $push267=, $pop266, $pop437
+; CHECK-NEXT:    i32.const $push89=, 30
+; CHECK-NEXT:    i32.shl $push268=, $pop267, $pop89
+; CHECK-NEXT:    i8x16.extract_lane_u $push262=, $66, 13
+; CHECK-NEXT:    i32.const $push436=, 1
+; CHECK-NEXT:    i32.and $push263=, $pop262, $pop436
+; CHECK-NEXT:    i32.const $push84=, 29
+; CHECK-NEXT:    i32.shl $push264=, $pop263, $pop84
+; CHECK-NEXT:    i8x16.extract_lane_u $push258=, $66, 12
+; CHECK-NEXT:    i32.const $push435=, 1
+; CHECK-NEXT:    i32.and $push259=, $pop258, $pop435
+; CHECK-NEXT:    i32.const $push79=, 28
+; CHECK-NEXT:    i32.shl $push260=, $pop259, $pop79
+; CHECK-NEXT:    i8x16.extract_lane_u $push254=, $66, 11
+; CHECK-NEXT:    i32.const $push434=, 1
+; CHECK-NEXT:    i32.and $push255=, $pop254, $pop434
+; CHECK-NEXT:    i32.const $push74=, 27
+; CHECK-NEXT:    i32.shl $push256=, $pop255, $pop74
+; CHECK-NEXT:    i8x16.extract_lane_u $push250=, $66, 10
+; CHECK-NEXT:    i32.const $push433=, 1
+; CHECK-NEXT:    i32.and $push251=, $pop250, $pop433
+; CHECK-NEXT:    i32.const $push69=, 26
+; CHECK-NEXT:    i32.shl $push252=, $pop251, $pop69
+; CHECK-NEXT:    i8x16.extract_lane_u $push246=, $66, 9
+; CHECK-NEXT:    i32.const $push432=, 1
+; CHECK-NEXT:    i32.and $push247=, $pop246, $pop432
+; CHECK-NEXT:    i32.const $push64=, 25
+; CHECK-NEXT:    i32.shl $push248=, $pop247, $pop64
+; CHECK-NEXT:    i8x16.extract_lane_u $push242=, $66, 8
+; CHECK-NEXT:    i32.const $push431=, 1
+; CHECK-NEXT:    i32.and $push243=, $pop242, $pop431
+; CHECK-NEXT:    i32.const $push59=, 24
+; CHECK-NEXT:    i32.shl $push244=, $pop243, $pop59
+; CHECK-NEXT:    i8x16.extract_lane_u $push238=, $66, 7
+; CHECK-NEXT:    i32.const $push430=, 1
+; CHECK-NEXT:    i32.and $push239=, $pop238, $pop430
+; CHECK-NEXT:    i32.const $push54=, 23
+; CHECK-NEXT:    i32.shl $push240=, $pop239, $pop54
+; CHECK-NEXT:    i8x16.extract_lane_u $push234=, $66, 6
+; CHECK-NEXT:    i32.const $push429=, 1
+; CHECK-NEXT:    i32.and $push235=, $pop234, $pop429
+; CHECK-NEXT:    i32.const $push49=, 22
+; CHECK-NEXT:    i32.shl $push236=, $pop235, $pop49
+; CHECK-NEXT:    i8x16.extract_lane_u $push230=, $66, 5
+; CHECK-NEXT:    i32.const $push428=, 1
+; CHECK-NEXT:    i32.and $push231=, $pop230, $pop428
+; CHECK-NEXT:    i32.const $push44=, 21
+; CHECK-NEXT:    i32.shl $push232=, $pop231, $pop44
+; CHECK-NEXT:    i8x16.extract_lane_u $push226=, $66, 4
+; CHECK-NEXT:    i32.const $push427=, 1
+; CHECK-NEXT:    i32.and $push227=, $pop226, $pop427
+; CHECK-NEXT:    i32.const $push39=, 20
+; CHECK-NEXT:    i32.shl $push228=, $pop227, $pop39
+; CHECK-NEXT:    i8x16.extract_lane_u $push222=, $66, 3
+; CHECK-NEXT:    i32.const $push426=, 1
+; CHECK-NEXT:    i32.and $push223=, $pop222, $pop426
+; CHECK-NEXT:    i32.const $push34=, 19
+; CHECK-NEXT:    i32.shl $push224=, $pop223, $pop34
+; CHECK-NEXT:    i8x16.extract_lane_u $push218=, $66, 2
+; CHECK-NEXT:    i32.const $push425=, 1
+; CHECK-NEXT:    i32.and $push219=, $pop218, $pop425
+; CHECK-NEXT:    i32.const $push29=, 18
+; CHECK-NEXT:    i32.shl $push220=, $pop219, $pop29
+; CHECK-NEXT:    i8x16.extract_lane_u $push214=, $66, 1
+; CHECK-NEXT:    i32.const $push424=, 1
+; CHECK-NEXT:    i32.and $push215=, $pop214, $pop424
+; CHECK-NEXT:    i32.const $push24=, 17
+; CHECK-NEXT:    i32.shl $push216=, $pop215, $pop24
+; CHECK-NEXT:    i8x16.extract_lane_u $push211=, $66, 0
+; CHECK-NEXT:    i32.const $push423=, 1
+; CHECK-NEXT:    i32.and $push212=, $pop211, $pop423
+; CHECK-NEXT:    i32.const $push20=, 16
+; CHECK-NEXT:    i32.shl $push213=, $pop212, $pop20
+; CHECK-NEXT:    i32.or $push217=, $pop216, $pop213
+; CHECK-NEXT:    i32.or $push221=, $pop220, $pop217
+; CHECK-NEXT:    i32.or $push225=, $pop224, $pop221
+; CHECK-NEXT:    i32.or $push229=, $pop228, $pop225
+; CHECK-NEXT:    i32.or $push233=, $pop232, $pop229
+; CHECK-NEXT:    i32.or $push237=, $pop236, $pop233
+; CHECK-NEXT:    i32.or $push241=, $pop240, $pop237
+; CHECK-NEXT:    i32.or $push245=, $pop244, $pop241
+; CHECK-NEXT:    i32.or $push249=, $pop248, $pop245
+; CHECK-NEXT:    i32.or $push253=, $pop252, $pop249
+; CHECK-NEXT:    i32.or $push257=, $pop256, $pop253
+; CHECK-NEXT:    i32.or $push261=, $pop260, $pop257
+; CHECK-NEXT:    i32.or $push265=, $pop264, $pop261
+; CHECK-NEXT:    i32.or $push269=, $pop268, $pop265
+; CHECK-NEXT:    i32.or $push272=, $pop271, $pop269
+; CHECK-NEXT:    i32.or $push352=, $pop351, $pop272
+; CHECK-NEXT:    i64.extend_i32_u $push353=, $pop352
+; CHECK-NEXT:    i8x16.splat $push96=, $32
+; CHECK-NEXT:    i8x16.replace_lane $push97=, $pop96, 1, $33
+; CHECK-NEXT:    i8x16.replace_lane $push98=, $pop97, 2, $34
+; CHECK-NEXT:    i8x16.replace_lane $push99=, $pop98, 3, $35
+; CHECK-NEXT:    i8x16.replace_lane $push100=, $pop99, 4, $36
+; CHECK-NEXT:    i8x16.replace_lane $push101=, $pop100, 5, $37
+; CHECK-NEXT:    i8x16.replace_lane $push102=, $pop101, 6, $38
+; CHECK-NEXT:    i8x16.replace_lane $push103=, $pop102, 7, $39
+; CHECK-NEXT:    i8x16.replace_lane $push104=, $pop103, 8, $40
+; CHECK-NEXT:    i8x16.replace_lane $push105=, $pop104, 9, $41
+; CHECK-NEXT:    i8x16.replace_lane $push106=, $pop105, 10, $42
+; CHECK-NEXT:    i8x16.replace_lane $push107=, $pop106, 11, $43
+; CHECK-NEXT:    i8x16.replace_lane $push108=, $pop107, 12, $44
+; CHECK-NEXT:    i8x16.replace_lane $push109=, $pop108, 13, $45
+; CHECK-NEXT:    i8x16.replace_lane $push110=, $pop109, 14, $46
+; CHECK-NEXT:    i8x16.replace_lane $push111=, $pop110, 15, $47
+; CHECK-NEXT:    v128.and $push112=, $pop111, $64
+; CHECK-NEXT:    i8x16.eq $push422=, $pop112, $65
+; CHECK-NEXT:    local.tee $push421=, $66=, $pop422
+; CHECK-NEXT:    i8x16.extract_lane_u $push113=, $pop421, 0
+; CHECK-NEXT:    i32.const $push420=, 1
+; CHECK-NEXT:    i32.and $push114=, $pop113, $pop420
+; CHECK-NEXT:    i8x16.extract_lane_u $push115=, $66, 1
+; CHECK-NEXT:    i32.const $push419=, 1
+; CHECK-NEXT:    i32.and $push116=, $pop115, $pop419
+; CHECK-NEXT:    i32.const $push418=, 1
+; CHECK-NEXT:    i32.shl $push117=, $pop116, $pop418
+; CHECK-NEXT:    i32.or $push118=, $pop114, $pop117
+; CHECK-NEXT:    i8x16.extract_lane_u $push119=, $66, 2
+; CHECK-NEXT:    i32.const $push417=, 1
+; CHECK-NEXT:    i32.and $push120=, $pop119, $pop417
+; CHECK-NEXT:    i32.const $push416=, 2
+; CHECK-NEXT:    i32.shl $push122=, $pop120, $pop416
+; CHECK-NEXT:    i32.or $push123=, $pop118, $pop122
+; CHECK-NEXT:    i8x16.extract_lane_u $push124=, $66, 3
+; CHECK-NEXT:    i32.const $push415=, 1
+; CHECK-NEXT:    i32.and $push125=, $pop124, $pop415
+; CHECK-NEXT:    i32.const $push414=, 3
+; CHECK-NEXT:    i32.shl $push127=, $pop125, $pop414
+; CHECK-NEXT:    i32.or $push128=, $pop123, $pop127
+; CHECK-NEXT:    i8x16.extract_lane_u $push129=, $66, 4
+; CHECK-NEXT:    i32.const $push413=, 1
+; CHECK-NEXT:    i32.and $push130=, $pop129, $pop413
+; CHECK-NEXT:    i32.const $push412=, 4
+; CHECK-NEXT:    i32.shl $push132=, $pop130, $pop412
+; CHECK-NEXT:    i32.or $push133=, $pop128, $pop132
+; CHECK-NEXT:    i8x16.extract_lane_u $push134=, $66, 5
+; CHECK-NEXT:    i32.const $push411=, 1
+; CHECK-NEXT:    i32.and $push135=, $pop134, $pop411
+; CHECK-NEXT:    i32.const $push410=, 5
+; CHECK-NEXT:    i32.shl $push137=, $pop135, $pop410
+; CHECK-NEXT:    i32.or $push138=, $pop133, $pop137
+; CHECK-NEXT:    i8x16.extract_lane_u $push139=, $66, 6
+; CHECK-NEXT:    i32.const $push409=, 1
+; CHECK-NEXT:    i32.and $push140=, $pop139, $pop409
+; CHECK-NEXT:    i32.const $push408=, 6
+; CHECK-NEXT:    i32.shl $push142=, $pop140, $pop408
+; CHECK-NEXT:    i32.or $push143=, $pop138, $pop142
+; CHECK-NEXT:    i8x16.extract_lane_u $push144=, $66, 7
+; CHECK-NEXT:    i32.const $push407=, 1
+; CHECK-NEXT:    i32.and $push145=, $pop144, $pop407
+; CHECK-NEXT:    i32.const $push406=, 7
+; CHECK-NEXT:    i32.shl $push147=, $pop145, $pop406
+; CHECK-NEXT:    i32.or $push148=, $pop143, $pop147
+; CHECK-NEXT:    i8x16.extract_lane_u $push149=, $66, 8
+; CHECK-NEXT:    i32.const $push405=, 1
+; CHECK-NEXT:    i32.and $push150=, $pop149, $pop405
+; CHECK-NEXT:    i32.const $push404=, 8
+; CHECK-NEXT:    i32.shl $push152=, $pop150, $pop404
+; CHECK-NEXT:    i32.or $push153=, $pop148, $pop152
+; CHECK-NEXT:    i8x16.extract_lane_u $push154=, $66, 9
+; CHECK-NEXT:    i32.const $push403=, 1
+; CHECK-NEXT:    i32.and $push155=, $pop154, $pop403
+; CHECK-NEXT:    i32.const $push402=, 9
+; CHECK-NEXT:    i32.shl $push157=, $pop155, $pop402
+; CHECK-NEXT:    i32.or $push158=, $pop153, $pop157
+; CHECK-NEXT:    i8x16.extract_lane_u $push159=, $66, 10
+; CHECK-NEXT:    i32.const $push401=, 1
+; CHECK-NEXT:    i32.and $push160=, $pop159, $pop401
+; CHECK-NEXT:    i32.const $push400=, 10
+; CHECK-NEXT:    i32.shl $push162=, $pop160, $pop400
+; CHECK-NEXT:    i32.or $push163=, $pop158, $pop162
+; CHECK-NEXT:    i8x16.extract_lane_u $push164=, $66, 11
+; CHECK-NEXT:    i32.const $push399=, 1
+; CHECK-NEXT:    i32.and $push165=, $pop164, $pop399
+; CHECK-NEXT:    i32.const $push398=, 11
+; CHECK-NEXT:    i32.shl $push167=, $pop165, $pop398
+; CHECK-NEXT:    i32.or $push168=, $pop163, $pop167
+; CHECK-NEXT:    i8x16.extract_lane_u $push169=, $66, 12
+; CHECK-NEXT:    i32.const $push397=, 1
+; CHECK-NEXT:    i32.and $push170=, $pop169, $pop397
+; CHECK-NEXT:    i32.const $push396=, 12
+; CHECK-NEXT:    i32.shl $push172=, $pop170, $pop396
+; CHECK-NEXT:    i32.or $push173=, $pop168, $pop172
+; CHECK-NEXT:    i8x16.extract_lane_u $push174=, $66, 13
+; CHECK-NEXT:    i32.const $push395=, 1
+; CHECK-NEXT:    i32.and $push175=, $pop174, $pop395
+; CHECK-NEXT:    i32.const $push394=, 13
+; CHECK-NEXT:    i32.shl $push177=, $pop175, $pop394
+; CHECK-NEXT:    i32.or $push178=, $pop173, $pop177
+; CHECK-NEXT:    i8x16.extract_lane_u $push179=, $66, 14
+; CHECK-NEXT:    i32.const $push393=, 1
+; CHECK-NEXT:    i32.and $push180=, $pop179, $pop393
+; CHECK-NEXT:    i32.const $push392=, 14
+; CHECK-NEXT:    i32.shl $push182=, $pop180, $pop392
+; CHECK-NEXT:    i32.or $push183=, $pop178, $pop182
+; CHECK-NEXT:    i8x16.extract_lane_u $push184=, $66, 15
+; CHECK-NEXT:    i32.const $push391=, 15
+; CHECK-NEXT:    i32.shl $push186=, $pop184, $pop391
+; CHECK-NEXT:    i32.or $push187=, $pop183, $pop186
+; CHECK-NEXT:    i32.const $push390=, 65535
+; CHECK-NEXT:    i32.and $push189=, $pop187, $pop390
+; CHECK-NEXT:    i8x16.splat $push0=, $48
+; CHECK-NEXT:    i8x16.replace_lane $push1=, $pop0, 1, $49
+; CHECK-NEXT:    i8x16.replace_lane $push2=, $pop1, 2, $50
+; CHECK-NEXT:    i8x16.replace_lane $push3=, $pop2, 3, $51
+; CHECK-NEXT:    i8x16.replace_lane $push4=, $pop3, 4, $52
+; CHECK-NEXT:    i8x16.replace_lane $push5=, $pop4, 5, $53
+; CHECK-NEXT:    i8x16.replace_lane $push6=, $pop5, 6, $54
+; CHECK-NEXT:    i8x16.replace_lane $push7=, $pop6, 7, $55
+; CHECK-NEXT:    i8x16.replace_lane $push8=, $pop7, 8, $56
+; CHECK-NEXT:    i8x16.replace_lane $push9=, $pop8, 9, $57
+; CHECK-NEXT:    i8x16.replace_lane $push10=, $pop9, 10, $58
+; CHECK-NEXT:    i8x16.replace_lane $push11=, $pop10, 11, $59
+; CHECK-NEXT:    i8x16.replace_lane $push12=, $pop11, 12, $60
+; CHECK-NEXT:    i8x16.replace_lane $push13=, $pop12, 13, $61
+; CHECK-NEXT:    i8x16.replace_lane $push14=, $pop13, 14, $62
+; CHECK-NEXT:    i8x16.replace_lane $push15=, $pop14, 15, $63
+; CHECK-NEXT:    v128.and $push16=, $pop15, $64
+; CHECK-NEXT:    i8x16.eq $push389=, $pop16, $65
+; CHECK-NEXT:    local.tee $push388=, $66=, $pop389
+; CHECK-NEXT:    i8x16.extract_lane_u $push92=, $pop388, 15
+; CHECK-NEXT:    i32.const $push387=, 31
+; CHECK-NEXT:    i32.shl $push94=, $pop92, $pop387
+; CHECK-NEXT:    i8x16.extract_lane_u $push87=, $66, 14
+; CHECK-NEXT:    i32.const $push386=, 1
+; CHECK-NEXT:    i32.and $push88=, $pop87, $pop386
+; CHECK-NEXT:    i32.const $push385=, 30
+; CHECK-NEXT:    i32.shl $push90=, $pop88, $pop385
+; CHECK-NEXT:    i8x16.extract_lane_u $push82=, $66, 13
+; CHECK-NEXT:    i32.const $push384=, 1
+; CHECK-NEXT:    i32.and $push83=, $pop82, $pop384
+; CHECK-NEXT:    i32.const $push383=, 29
+; CHECK-NEXT:    i32.shl $push85=, $pop83, $pop383
+; CHECK-NEXT:    i8x16.extract_lane_u $push77=, $66, 12
+; CHECK-NEXT:    i32.const $push382=, 1
+; CHECK-NEXT:    i32.and $push78=, $pop77, $pop382
+; CHECK-NEXT:    i32.const $push381=, 28
+; CHECK-NEXT:    i32.shl $push80=, $pop78, $pop381
+; CHECK-NEXT:    i8x16.extract_lane_u $push72=, $66, 11
+; CHECK-NEXT:    i32.const $push380=, 1
+; CHECK-NEXT:    i32.and $push73=, $pop72, $pop380
+; CHECK-NEXT:    i32.const $push379=, 27
+; CHECK-NEXT:    i32.shl $push75=, $pop73, $pop379
+; CHECK-NEXT:    i8x16.extract_lane_u $push67=, $66, 10
+; CHECK-NEXT:    i32.const $push378=, 1
+; CHECK-NEXT:    i32.and $push68=, $pop67, $pop378
+; CHECK-NEXT:    i32.const $push377=, 26
+; CHECK-NEXT:    i32.shl $push70=, $pop68, $pop377
+; CHECK-NEXT:    i8x16.extract_lane_u $push62=, $66, 9
+; CHECK-NEXT:    i32.const $push376=, 1
+; CHECK-NEXT:    i32.and $push63=, $pop62, $pop376
+; CHECK-NEXT:    i32.const $push375=, 25
+; CHECK-NEXT:    i32.shl $push65=, $pop63, $pop375
+; CHECK-NEXT:    i8x16.extract_lane_u $push57=, $66, 8
+; CHECK-NEXT:    i32.const $push374=, 1
+; CHECK-NEXT:    i32.and $push58=, $pop57, $pop374
+; CHECK-NEXT:    i32.const $push373=, 24
+; CHECK-NEXT:    i32.shl $push60=, $pop58, $pop373
+; CHECK-NEXT:    i8x16.extract_lane_u $push52=, $66, 7
+; CHECK-NEXT:    i32.const $push372=, 1
+; CHECK-NEXT:    i32.and $push53=, $pop52, $pop372
+; CHECK-NEXT:    i32.const $push371=, 23
+; CHECK-NEXT:    i32.shl $push55=, $pop53, $pop371
+; CHECK-NEXT:    i8x16.extract_lane_u $push47=, $66, 6
+; CHECK-NEXT:    i32.const $push370=, 1
+; CHECK-NEXT:    i32.and $push48=, $pop47, $pop370
+; CHECK-NEXT:    i32.const $push369=, 22
+; CHECK-NEXT:    i32.shl $push50=, $pop48, $pop369
+; CHECK-NEXT:    i8x16.extract_lane_u $push42=, $66, 5
+; CHECK-NEXT:    i32.const $push368=, 1
+; CHECK-NEXT:    i32.and $push43=, $pop42, $pop368
+; CHECK-NEXT:    i32.const $push367=, 21
+; CHECK-NEXT:    i32.shl $push45=, $pop43, $pop367
+; CHECK-NEXT:    i8x16.extract_lane_u $push37=, $66, 4
+; CHECK-NEXT:    i32.const $push366=, 1
+; CHECK-NEXT:    i32.and $push38=, $pop37, $pop366
+; CHECK-NEXT:    i32.const $push365=, 20
+; CHECK-NEXT:    i32.shl $push40=, $pop38, $pop365
+; CHECK-NEXT:    i8x16.extract_lane_u $push32=, $66, 3
+; CHECK-NEXT:    i32.const $push364=, 1
+; CHECK-NEXT:    i32.and $push33=, $pop32, $pop364
+; CHECK-NEXT:    i32.const $push363=, 19
+; CHECK-NEXT:    i32.shl $push35=, $pop33, $pop363
+; CHECK-NEXT:    i8x16.extract_lane_u $push27=, $66, 2
+; CHECK-NEXT:    i32.const $push362=, 1
+; CHECK-NEXT:    i32.and $push28=, $pop27, $pop362
+; CHECK-NEXT:    i32.const $push361=, 18
+; CHECK-NEXT:    i32.shl $push30=, $pop28, $pop361
+; CHECK-NEXT:    i8x16.extract_lane_u $push22=, $66, 1
+; CHECK-NEXT:    i32.const $push360=, 1
+; CHECK-NEXT:    i32.and $push23=, $pop22, $pop360
+; CHECK-NEXT:    i32.const $push359=, 17
+; CHECK-NEXT:    i32.shl $push25=, $pop23, $pop359
+; CHECK-NEXT:    i8x16.extract_lane_u $push17=, $66, 0
+; CHECK-NEXT:    i32.const $push358=, 1
+; CHECK-NEXT:    i32.and $push19=, $pop17, $pop358
+; CHECK-NEXT:    i32.const $push357=, 16
+; CHECK-NEXT:    i32.shl $push21=, $pop19, $pop357
+; CHECK-NEXT:    i32.or $push26=, $pop25, $pop21
+; CHECK-NEXT:    i32.or $push31=, $pop30, $pop26
+; CHECK-NEXT:    i32.or $push36=, $pop35, $pop31
+; CHECK-NEXT:    i32.or $push41=, $pop40, $pop36
+; CHECK-NEXT:    i32.or $push46=, $pop45, $pop41
+; CHECK-NEXT:    i32.or $push51=, $pop50, $pop46
+; CHECK-NEXT:    i32.or $push56=, $pop55, $pop51
+; CHECK-NEXT:    i32.or $push61=, $pop60, $pop56
+; CHECK-NEXT:    i32.or $push66=, $pop65, $pop61
+; CHECK-NEXT:    i32.or $push71=, $pop70, $pop66
+; CHECK-NEXT:    i32.or $push76=, $pop75, $pop71
+; CHECK-NEXT:    i32.or $push81=, $pop80, $pop76
+; CHECK-NEXT:    i32.or $push86=, $pop85, $pop81
+; CHECK-NEXT:    i32.or $push91=, $pop90, $pop86
+; CHECK-NEXT:    i32.or $push95=, $pop94, $pop91
+; CHECK-NEXT:    i32.or $push190=, $pop189, $pop95
+; CHECK-NEXT:    i64.extend_i32_u $push191=, $pop190
+; CHECK-NEXT:    i64.const $push192=, 32
+; CHECK-NEXT:    i64.shl $push193=, $pop191, $pop192
+; CHECK-NEXT:    i64.or $push354=, $pop353, $pop193
+; CHECK-NEXT:    return $pop354
+    %z = icmp eq <64 x i4> %x, splat (i4 64)
+    %res = bitcast <64 x i1> %z to i64
+    ret i64 %res
+}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll b/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll
new file mode 100644
index 000000000000..172ff53bfb45
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-setcc-reductions.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+target triple = "wasm64"
+
+define i32 @all_true_16_i8(<16 x i8> %v) {
+; CHECK-LABEL: all_true_16_i8:
+; CHECK:         .functype all_true_16_i8 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i8x16.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <16 x i8> %v, zeroinitializer
+  %2 = bitcast <16 x i1> %1 to i16
+  %3 = icmp eq i16 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: all_true_4_i32:
+; CHECK:         .functype all_true_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32x4.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_8_i16(<8 x i16> %v) {
+; CHECK-LABEL: all_true_8_i16:
+; CHECK:         .functype all_true_8_i16 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i16x8.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <8 x i16> %v, zeroinitializer
+  %2 = bitcast <8 x i1> %1 to i8
+  %3 = icmp eq i8 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_2_i64(<2 x i64> %v) {
+; CHECK-LABEL: all_true_2_i64:
+; CHECK:         .functype all_true_2_i64 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64x2.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <2 x i64> %v, zeroinitializer
+  %2 = bitcast <2 x i1> %1 to i2
+  %3 = icmp eq i2 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+define i32 @all_true_4_i64(<4 x i64> %v) {
+; CHECK-LABEL: all_true_4_i64:
+; CHECK:         .functype all_true_4_i64 (v128, v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.const $push9=, 0, 0
+; CHECK-NEXT:    local.tee $push8=, $2=, $pop9
+; CHECK-NEXT:    i64x2.eq $push1=, $0, $pop8
+; CHECK-NEXT:    i64x2.eq $push0=, $1, $2
+; CHECK-NEXT:    i8x16.shuffle $push2=, $pop1, $pop0, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27
+; CHECK-NEXT:    v128.any_true $push3=, $pop2
+; CHECK-NEXT:    i32.const $push4=, -1
+; CHECK-NEXT:    i32.xor $push5=, $pop3, $pop4
+; CHECK-NEXT:    i32.const $push6=, 1
+; CHECK-NEXT:    i32.and $push7=, $pop5, $pop6
+; CHECK-NEXT:    return $pop7
+  %1 = icmp eq <4 x i64> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, ne)), 0, ne
+;   => any_true (set_cc (X), 0, ne)
+;   => any_true (X)
+define i32 @any_true_1_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: any_true_1_4_i32:
+; CHECK:         .functype any_true_1_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.any_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp ne <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp ne i4 %2, 0
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, eq)), -1, ne
+;   => not all_true (set_cc (X), 0, eq)
+;   => not not any_true (X)
+;   => any_true (X)
+define i32 @any_true_2_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: any_true_2_4_i32:
+; CHECK:         .functype any_true_2_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.any_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp eq <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp ne i4 %2, -1
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
+; setcc (iN (bitcast (set_cc (vNi1 X), 0, ne)), -1, eq
+;   => all_true (set_cc (X), 0, ne)
+;   => all_true (X)
+define i32 @all_true_2_4_i32(<4 x i32> %v) {
+; CHECK-LABEL: all_true_2_4_i32:
+; CHECK:         .functype all_true_2_4_i32 (v128) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i32x4.all_true $push0=, $0
+; CHECK-NEXT:    return $pop0
+  %1 = icmp ne <4 x i32> %v, zeroinitializer
+  %2 = bitcast <4 x i1> %1 to i4
+  %3 = icmp eq i4 %2, -1
+  %conv3 = zext i1 %3 to i32
+  ret i32 %conv3
+}
+
+
diff --git a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
index e6497bca98dc..f7143711394f 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-vecreduce-bool.ll
@@ -1086,9 +1086,9 @@ define i1 @test_cmp_v16i8(<16 x i8> %x) {
 ; CHECK-LABEL: test_cmp_v16i8:
 ; CHECK:         .functype test_cmp_v16i8 (v128) -> (i32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    v128.const $push0=, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-; CHECK-NEXT:    i8x16.eq $push1=, $0, $pop0
-; CHECK-NEXT:    v128.any_true $push2=, $pop1
+; CHECK-NEXT:    i8x16.all_true $push0=, $0
+; CHECK-NEXT:    i32.const $push1=, 1
+; CHECK-NEXT:    i32.xor $push2=, $pop0, $pop1
 ; CHECK-NEXT:    return $pop2
   %zero = icmp eq <16 x i8> %x, zeroinitializer
   %ret = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %zero)
diff --git a/llvm/test/CodeGen/WebAssembly/suboptimal-compare.ll b/llvm/test/CodeGen/WebAssembly/suboptimal-compare.ll
index fadeb397a884..9a2716b5f246 100644
--- a/llvm/test/CodeGen/WebAssembly/suboptimal-compare.ll
+++ b/llvm/test/CodeGen/WebAssembly/suboptimal-compare.ll
@@ -4,32 +4,19 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: gh_80053:                               # @gh_80053
 ; CHECK-NEXT: .functype	gh_80053 (i32) -> (i32)
-; CHECK-NEXT: .local  	i32, i32, i32, i32, i32, i32
-; CHECK:      i32.const	0
-; CHECK-NEXT: local.set	1
+; CHECK:      block
 ; CHECK-NEXT: local.get	0
-; CHECK-NEXT: local.get	1
+; CHECK-NEXT: i32.const	0
 ; CHECK-NEXT: i32.eq  
-; CHECK-NEXT: local.set	2
 ; CHECK-NEXT: i32.const	1
-; CHECK-NEXT: local.set	3
-; CHECK-NEXT: local.get	2
-; CHECK-NEXT: local.get	3
 ; CHECK-NEXT: i32.and 
-; CHECK-NEXT: local.set	4
-; CHECK-NEXT: block   	
-; CHECK-NEXT:   local.get	4
-; CHECK-NEXT:   i32.eqz
-; CHECK-NEXT:   br_if   	0                               # 0: down to label0
-; CHECK:        i32.const	0
-; CHECK-NEXT:   local.set	5
-; CHECK-NEXT:   local.get	5
-; CHECK-NEXT:   return
+; CHECK-NEXT: i32.eqz
+; CHECK-NEXT: br_if   	0                               # 0: down to label0
+; CHECK:      i32.const	0
+; CHECK-NEXT: return
 ; CHECK-NEXT: .LBB0_2:                                # %BB03
 ; CHECK-NEXT: end_block                               # label0:
 ; CHECK-NEXT: i32.const	1
-; CHECK-NEXT: local.set	6
-; CHECK-NEXT: local.get	6
 ; CHECK-NEXT: return
 ; CHECK-NEXT: end_function
 define i1 @gh_80053(ptr) {
diff --git a/llvm/test/CodeGen/X86/GlobalISel/avoid-matchtable-crash.mir b/llvm/test/CodeGen/X86/GlobalISel/avoid-matchtable-crash.mir
index 2c70ed3b0451..31f1da5c674f 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/avoid-matchtable-crash.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/avoid-matchtable-crash.mir
@@ -1,4 +1,4 @@
-# RUN: not --crash llc -o - -run-pass=instruction-select -pass-remarks-missed=gisel %s 2>&1 | FileCheck %s
+# RUN: not llc -o - -run-pass=instruction-select -pass-remarks-missed=gisel %s 2>&1 | FileCheck %s
 --- |
   target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
   target triple = "x86_64--linux-gnu"
diff --git a/llvm/test/CodeGen/X86/GlobalISel/llvm.sincos.mir b/llvm/test/CodeGen/X86/GlobalISel/llvm.sincos.mir
new file mode 100644
index 000000000000..f8ce9ea8be65
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/llvm.sincos.mir
@@ -0,0 +1,189 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=i686-linux-gnu -run-pass=regbankselect,instruction-select %s -o - | FileCheck %s --check-prefixes GISEL-I686
+
+---
+name:            test_sincos_f32
+alignment:       16
+legalized:       true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 4, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1:
+    ; GISEL-I686-LABEL: name: test_sincos_f32
+    ; GISEL-I686: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.0, align 16)
+    ; GISEL-I686-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r %stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: [[LEA32r1:%[0-9]+]]:gr32 = LEA32r %stack.1, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: ADJCALLSTACKDOWN32 12, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY]], 1, $noreg, 0, $noreg, [[MOV32rm]] :: (store (s32) into stack, align 1)
+    ; GISEL-I686-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY1]], 1, $noreg, 4, $noreg, [[LEA32r]] :: (store (s32) into stack + 4, align 1)
+    ; GISEL-I686-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY2]], 1, $noreg, 8, $noreg, [[LEA32r1]] :: (store (s32) into stack + 8, align 1)
+    ; GISEL-I686-NEXT: CALLpcrel32 &sincosf, csr_32, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: ADJCALLSTACKUP32 12, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: [[LD_Fp32m:%[0-9]+]]:rfp32 = nofpexcept LD_Fp32m %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s32) from %stack.0)
+    ; GISEL-I686-NEXT: [[LD_Fp32m1:%[0-9]+]]:rfp32 = nofpexcept LD_Fp32m %stack.1, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s32) from %stack.1)
+    ; GISEL-I686-NEXT: $fp0 = COPY [[LD_Fp32m]]
+    ; GISEL-I686-NEXT: $fp1 = COPY [[LD_Fp32m1]]
+    ; GISEL-I686-NEXT: RET 0, implicit $fp0, implicit $fp1
+    %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %0:_(s32) = G_LOAD %1(p0) :: (invariant load (s32) from %fixed-stack.0, align 16)
+    %4:_(p0) = G_FRAME_INDEX %stack.0
+    %5:_(p0) = G_FRAME_INDEX %stack.1
+    ADJCALLSTACKDOWN32 12, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    %6:_(p0) = COPY $esp
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(p0) = G_PTR_ADD %6, %7(s32)
+    G_STORE %0(s32), %8(p0) :: (store (s32) into stack, align 1)
+    %9:_(p0) = COPY $esp
+    %10:_(s32) = G_CONSTANT i32 4
+    %11:_(p0) = G_PTR_ADD %9, %10(s32)
+    G_STORE %4(p0), %11(p0) :: (store (s32) into stack + 4, align 1)
+    %12:_(p0) = COPY $esp
+    %13:_(s32) = G_CONSTANT i32 8
+    %14:_(p0) = G_PTR_ADD %12, %13(s32)
+    G_STORE %5(p0), %14(p0) :: (store (s32) into stack + 8, align 1)
+    CALLpcrel32 &sincosf, csr_32, implicit $esp, implicit $ssp
+    ADJCALLSTACKUP32 12, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    %2:_(s32) = G_LOAD %4(p0) :: (load (s32) from %stack.0)
+    %3:_(s32) = G_LOAD %5(p0) :: (load (s32) from %stack.1)
+    $fp0 = COPY %2(s32)
+    $fp1 = COPY %3(s32)
+    RET 0, implicit $fp0, implicit $fp1
+...
+---
+name:            test_sincos_f64
+alignment:       16
+legalized:       true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 8, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1:
+    ; GISEL-I686-LABEL: name: test_sincos_f64
+    ; GISEL-I686: [[LEA32r:%[0-9]+]]:gr32 = LEA32r %fixed-stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load (s32) from %fixed-stack.0, align 16)
+    ; GISEL-I686-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[LEA32r]], 1, $noreg, 4, $noreg :: (invariant load (s32) from %fixed-stack.0 + 4, basealign 16)
+    ; GISEL-I686-NEXT: [[LEA32r1:%[0-9]+]]:gr32 = LEA32r %stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: [[LEA32r2:%[0-9]+]]:gr32 = LEA32r %stack.1, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: [[MOV32r0_:%[0-9]+]]:gr32_nosp = MOV32r0 implicit-def dead $eflags
+    ; GISEL-I686-NEXT: [[LEA32r3:%[0-9]+]]:gr32 = LEA32r [[COPY]], 1, [[MOV32r0_]], 0, $noreg
+    ; GISEL-I686-NEXT: MOV32mr [[COPY]], 1, $noreg, 0, $noreg, [[MOV32rm]] :: (store (s32) into stack, align 1)
+    ; GISEL-I686-NEXT: MOV32mr [[LEA32r3]], 1, $noreg, 4, $noreg, [[MOV32rm1]] :: (store (s32) into stack + 4, align 1)
+    ; GISEL-I686-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY1]], 1, $noreg, 8, $noreg, [[LEA32r1]] :: (store (s32) into stack + 8, align 1)
+    ; GISEL-I686-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY2]], 1, $noreg, 12, $noreg, [[LEA32r2]] :: (store (s32) into stack + 12, align 1)
+    ; GISEL-I686-NEXT: CALLpcrel32 &sincos, csr_32, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: $fp0 = IMPLICIT_DEF
+    ; GISEL-I686-NEXT: $fp1 = IMPLICIT_DEF
+    ; GISEL-I686-NEXT: RET 0, implicit $fp0, implicit $fp1
+    %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %25:_(s32) = G_LOAD %1(p0) :: (invariant load (s32) from %fixed-stack.0, align 16)
+    %17:_(s32) = G_CONSTANT i32 4
+    %26:_(p0) = G_PTR_ADD %1, %17(s32)
+    %27:_(s32) = G_LOAD %26(p0) :: (invariant load (s32) from %fixed-stack.0 + 4, basealign 16)
+    %4:_(p0) = G_FRAME_INDEX %stack.0
+    %5:_(p0) = G_FRAME_INDEX %stack.1
+    ADJCALLSTACKDOWN32 16, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    %6:_(p0) = COPY $esp
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(p0) = G_PTR_ADD %6, %7(s32)
+    G_STORE %25(s32), %8(p0) :: (store (s32) into stack, align 1)
+    %24:_(p0) = G_PTR_ADD %8, %17(s32)
+    G_STORE %27(s32), %24(p0) :: (store (s32) into stack + 4, align 1)
+    %9:_(p0) = COPY $esp
+    %10:_(s32) = G_CONSTANT i32 8
+    %11:_(p0) = G_PTR_ADD %9, %10(s32)
+    G_STORE %4(p0), %11(p0) :: (store (s32) into stack + 8, align 1)
+    %12:_(p0) = COPY $esp
+    %13:_(s32) = G_CONSTANT i32 12
+    %14:_(p0) = G_PTR_ADD %12, %13(s32)
+    G_STORE %5(p0), %14(p0) :: (store (s32) into stack + 12, align 1)
+    CALLpcrel32 &sincos, csr_32, implicit $esp, implicit $ssp
+    ADJCALLSTACKUP32 16, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    $fp0 = IMPLICIT_DEF
+    $fp1 = IMPLICIT_DEF
+    RET 0, implicit $fp0, implicit $fp1
+...
+---
+name:            test_sincos_f80
+alignment:       16
+legalized:       true
+fixedStack:
+  - { id: 0, type: default, offset: 0, size: 10, alignment: 16, stack-id: default,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 10, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 10, alignment: 16,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1:
+    ; GISEL-I686-LABEL: name: test_sincos_f80
+    ; GISEL-I686: [[LD_Fp80m:%[0-9]+]]:rfp80 = nofpexcept LD_Fp80m %fixed-stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (invariant load (s80) from %fixed-stack.0, align 16)
+    ; GISEL-I686-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r %stack.0, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: [[LEA32r1:%[0-9]+]]:gr32 = LEA32r %stack.1, 1, $noreg, 0, $noreg
+    ; GISEL-I686-NEXT: ADJCALLSTACKDOWN32 20, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: nofpexcept ST_FpP80m [[COPY]], 1, $noreg, 0, $noreg, [[LD_Fp80m]], implicit-def dead $fpsw, implicit $fpcw :: (store (s80) into stack, align 1)
+    ; GISEL-I686-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY1]], 1, $noreg, 12, $noreg, [[LEA32r]] :: (store (s32) into stack + 12, align 1)
+    ; GISEL-I686-NEXT: [[COPY2:%[0-9]+]]:gr32 = COPY $esp
+    ; GISEL-I686-NEXT: MOV32mr [[COPY2]], 1, $noreg, 16, $noreg, [[LEA32r1]] :: (store (s32) into stack + 16, align 1)
+    ; GISEL-I686-NEXT: CALLpcrel32 &sincosl, csr_32, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: ADJCALLSTACKUP32 20, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    ; GISEL-I686-NEXT: [[LD_Fp80m1:%[0-9]+]]:rfp80 = nofpexcept LD_Fp80m %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s80) from %stack.0, align 16)
+    ; GISEL-I686-NEXT: [[LD_Fp80m2:%[0-9]+]]:rfp80 = nofpexcept LD_Fp80m %stack.1, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (load (s80) from %stack.1, align 16)
+    ; GISEL-I686-NEXT: $fp0 = COPY [[LD_Fp80m1]]
+    ; GISEL-I686-NEXT: $fp1 = COPY [[LD_Fp80m2]]
+    ; GISEL-I686-NEXT: RET 0, implicit $fp0, implicit $fp1
+    %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %0:_(s80) = G_LOAD %1(p0) :: (invariant load (s80) from %fixed-stack.0, align 16)
+    %4:_(p0) = G_FRAME_INDEX %stack.0
+    %5:_(p0) = G_FRAME_INDEX %stack.1
+    ADJCALLSTACKDOWN32 20, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    %6:_(p0) = COPY $esp
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(p0) = G_PTR_ADD %6, %7(s32)
+    G_STORE %0(s80), %8(p0) :: (store (s80) into stack, align 1)
+    %9:_(p0) = COPY $esp
+    %10:_(s32) = G_CONSTANT i32 12
+    %11:_(p0) = G_PTR_ADD %9, %10(s32)
+    G_STORE %4(p0), %11(p0) :: (store (s32) into stack + 12, align 1)
+    %12:_(p0) = COPY $esp
+    %13:_(s32) = G_CONSTANT i32 16
+    %14:_(p0) = G_PTR_ADD %12, %13(s32)
+    G_STORE %5(p0), %14(p0) :: (store (s32) into stack + 16, align 1)
+    CALLpcrel32 &sincosl, csr_32, implicit $esp, implicit $ssp
+    ADJCALLSTACKUP32 20, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
+    %2:_(s80) = G_LOAD %4(p0) :: (load (s80) from %stack.0, align 16)
+    %3:_(s80) = G_LOAD %5(p0) :: (load (s80) from %stack.1, align 16)
+    $fp0 = COPY %2(s80)
+    $fp1 = COPY %3(s80)
+    RET 0, implicit $fp0, implicit $fp1
+...
diff --git a/llvm/test/CodeGen/X86/abds-neg.ll b/llvm/test/CodeGen/X86/abds-neg.ll
index 6e22d855dc83..f6d66ab47ce0 100644
--- a/llvm/test/CodeGen/X86/abds-neg.ll
+++ b/llvm/test/CodeGen/X86/abds-neg.ll
@@ -1058,15 +1058,15 @@ define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
-; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    sbbl %esi, %edx
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
@@ -1089,15 +1089,15 @@ define i64 @abd_subnsw_i64_undef(i64 %a, i64 %b) nounwind {
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
-; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    xorl %edx, %ecx
+; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl %edx, %eax
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    sbbl %esi, %edx
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %ecx, %edx
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
@@ -1121,11 +1121,11 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
@@ -1178,11 +1178,11 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %esi
 ; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
diff --git a/llvm/test/CodeGen/X86/abds-vector-128.ll b/llvm/test/CodeGen/X86/abds-vector-128.ll
index bcb42002fb08..148be83892b7 100644
--- a/llvm/test/CodeGen/X86/abds-vector-128.ll
+++ b/llvm/test/CodeGen/X86/abds-vector-128.ll
@@ -806,7 +806,7 @@ define <2 x i64> @abd_cmp_v2i64_multiuse_cmp(<2 x i64> %a, <2 x i64> %b) nounwin
 ; AVX512-NEXT:    vpmaxsq %xmm1, %xmm0, %xmm3
 ; AVX512-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
 ; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm0
 ; AVX512-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %cmp = icmp sge <2 x i64> %a, %b
diff --git a/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll b/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
index e70e5ff80d95..a7fe8983f2f2 100644
--- a/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
+++ b/llvm/test/CodeGen/X86/apx/kmov-domain-assignment.ll
@@ -22,8 +22,8 @@ define void @test_fcmp_storei1(i1 %cond, ptr %fptr, ptr %iptr, float %f1, float
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[VCMPSSZrri:%[0-9]+]]:vk1 = nofpexcept VCMPSSZrri [[COPY3]], [[COPY2]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vk16 = COPY [[VCMPSSZrri]]
-  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vk32 = COPY [[COPY7]]
+  ; CHECK-NEXT:   [[COPY7:%[0-9]+]]:vk16 = COPY killed [[VCMPSSZrri]]
+  ; CHECK-NEXT:   [[COPY8:%[0-9]+]]:vk32 = COPY killed [[COPY7]]
   ; CHECK-NEXT:   [[COPY9:%[0-9]+]]:vk8 = COPY [[COPY8]]
   ; CHECK-NEXT:   JMP_1 %bb.3
   ; CHECK-NEXT: {{  $}}
@@ -31,8 +31,8 @@ define void @test_fcmp_storei1(i1 %cond, ptr %fptr, ptr %iptr, float %f1, float
   ; CHECK-NEXT:   successors: %bb.3(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[VCMPSSZrri1:%[0-9]+]]:vk1 = nofpexcept VCMPSSZrri [[COPY1]], [[COPY]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vk16 = COPY [[VCMPSSZrri1]]
-  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vk32 = COPY [[COPY10]]
+  ; CHECK-NEXT:   [[COPY10:%[0-9]+]]:vk16 = COPY killed [[VCMPSSZrri1]]
+  ; CHECK-NEXT:   [[COPY11:%[0-9]+]]:vk32 = COPY killed [[COPY10]]
   ; CHECK-NEXT:   [[COPY12:%[0-9]+]]:vk8 = COPY [[COPY11]]
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.3.exit:
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index 0de308a9e073..217ccebdfb77 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -1734,20 +1734,20 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-LABEL: not_avg_v16i8_wide_constants:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps (%rdi), %xmm1
-; SSE2-NEXT:    movdqa (%rsi), %xmm0
+; SSE2-NEXT:    movdqa (%rsi), %xmm2
 ; SSE2-NEXT:    movaps %xmm1, -{{[0-9]+}}(%rsp)
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm1
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm3
+; SSE2-NEXT:    movd %eax, %xmm4
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm4
+; SSE2-NEXT:    movd %eax, %xmm3
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm5
@@ -1762,9 +1762,6 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm8
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    movd %eax, %xmm10
-; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
-; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm9
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
@@ -1774,6 +1771,9 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm12
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
+; SSE2-NEXT:    movd %eax, %xmm10
+; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT:    decl %eax
 ; SSE2-NEXT:    movd %eax, %xmm13
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
@@ -1783,45 +1783,43 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; SSE2-NEXT:    movd %eax, %xmm15
 ; SSE2-NEXT:    movzbl -{{[0-9]+}}(%rsp), %eax
 ; SSE2-NEXT:    decl %eax
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,0,0]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[0,0,0,0]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[0,0,0,0]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[0,0,0,0]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm1[0],xmm4[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT:    movsd {{.*#+}} xmm4 = xmm3[0],xmm4[1]
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
 ; SSE2-NEXT:    movapd %xmm4, %xmm5
 ; SSE2-NEXT:    andpd %xmm1, %xmm5
 ; SSE2-NEXT:    xorpd %xmm4, %xmm1
 ; SSE2-NEXT:    psrlw $1, %xmm1
 ; SSE2-NEXT:    paddw %xmm5, %xmm1
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm12[0,0,0,0]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[0,0,0,0]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT:    movsd {{.*#+}} xmm2 = xmm9[0],xmm2[1]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15]
-; SSE2-NEXT:    movapd %xmm2, %xmm3
-; SSE2-NEXT:    andpd %xmm0, %xmm3
-; SSE2-NEXT:    xorpd %xmm2, %xmm0
-; SSE2-NEXT:    psrlw $1, %xmm0
-; SSE2-NEXT:    paddw %xmm3, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm10[0],xmm0[1]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; SSE2-NEXT:    movapd %xmm0, %xmm3
+; SSE2-NEXT:    andpd %xmm2, %xmm3
+; SSE2-NEXT:    xorpd %xmm0, %xmm2
+; SSE2-NEXT:    psrlw $1, %xmm2
+; SSE2-NEXT:    paddw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
 ; SSE2-NEXT:    movdqu %xmm1, (%rax)
 ; SSE2-NEXT:    retq
 ;
@@ -1831,75 +1829,74 @@ define void @not_avg_v16i8_wide_constants(ptr %a, ptr %b) nounwind {
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX1-NEXT:    vpextrw $7, %xmm3, %edx
-; AVX1-NEXT:    vpextrw $6, %xmm3, %ecx
-; AVX1-NEXT:    vpextrw $5, %xmm3, %eax
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX1-NEXT:    vpextrd $2, %xmm5, %ecx
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; AVX1-NEXT:    vpextrd $2, %xmm4, %eax
+; AVX1-NEXT:    vpextrw $3, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm4
-; AVX1-NEXT:    vpextrw $4, %xmm3, %edx
-; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm5
-; AVX1-NEXT:    vpextrw $1, %xmm3, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm6
-; AVX1-NEXT:    vpextrw $0, %xmm3, %eax
+; AVX1-NEXT:    vpextrw $2, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm5
+; AVX1-NEXT:    vpextrw $1, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm6
+; AVX1-NEXT:    vpextrw $0, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm7
-; AVX1-NEXT:    vpextrw $3, %xmm3, %edx
-; AVX1-NEXT:    decq %rcx
-; AVX1-NEXT:    vmovq %rcx, %xmm8
-; AVX1-NEXT:    vpextrw $2, %xmm3, %ecx
-; AVX1-NEXT:    decq %rax
-; AVX1-NEXT:    vmovq %rax, %xmm3
-; AVX1-NEXT:    vpextrw $7, %xmm2, %eax
+; AVX1-NEXT:    vpextrw $3, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm8
+; AVX1-NEXT:    vpextrw $2, %xmm2, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm9
-; AVX1-NEXT:    vpextrw $6, %xmm2, %edx
-; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm10
-; AVX1-NEXT:    vpextrw $5, %xmm2, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm11
-; AVX1-NEXT:    vpextrw $4, %xmm2, %eax
+; AVX1-NEXT:    vpextrw $1, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm10
+; AVX1-NEXT:    vpextrw $0, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm11
+; AVX1-NEXT:    vpextrw $5, %xmm3, %edx
 ; AVX1-NEXT:    decl %edx
 ; AVX1-NEXT:    vmovd %edx, %xmm12
-; AVX1-NEXT:    vpextrw $1, %xmm2, %edx
+; AVX1-NEXT:    vpextrw $4, %xmm3, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm13
+; AVX1-NEXT:    vpextrw $5, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm14
+; AVX1-NEXT:    vpextrw $4, %xmm2, %edx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm15
+; AVX1-NEXT:    vpextrw $7, %xmm3, %edx
 ; AVX1-NEXT:    decl %ecx
-; AVX1-NEXT:    vmovd %ecx, %xmm13
-; AVX1-NEXT:    vpextrw $0, %xmm2, %ecx
-; AVX1-NEXT:    decl %eax
-; AVX1-NEXT:    vmovd %eax, %xmm14
-; AVX1-NEXT:    vpextrw $3, %xmm2, %eax
-; AVX1-NEXT:    decq %rdx
-; AVX1-NEXT:    vmovq %rdx, %xmm15
-; AVX1-NEXT:    vpextrw $2, %xmm2, %edx
-; AVX1-NEXT:    decq %rcx
-; AVX1-NEXT:    vmovq %rcx, %xmm2
+; AVX1-NEXT:    vmovd %ecx, %xmm3
+; AVX1-NEXT:    vpextrw $7, %xmm2, %ecx
+; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    vmovd %edx, %xmm2
 ; AVX1-NEXT:    decl %eax
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX1-NEXT:    vmovd %eax, %xmm5
-; AVX1-NEXT:    decl %edx
+; AVX1-NEXT:    decl %ecx
 ; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX1-NEXT:    vmovd %edx, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3,4,5],xmm4[6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,0,1,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[0,1,0,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm6[0,1,2,3,4,5],xmm4[6,7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm15[0],xmm2[1],xmm15[1],xmm2[2],xmm15[2],xmm2[3],xmm15[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,0,1,1]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vmovd %ecx, %xmm7
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
+; AVX1-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm8, %ymm6
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX1-NEXT:    vmovddup {{.*#+}} ymm3 = ymm6[0,0,2,2]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
+; AVX1-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,0,0,0,4,4,4,4]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1],ymm2[2,3],ymm4[4,5],ymm2[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vandps %ymm0, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
diff --git a/llvm/test/CodeGen/X86/avx512-vbroadcast.ll b/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
index 275df8c3675c..c50418feff6f 100644
--- a/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
+++ b/llvm/test/CodeGen/X86/avx512-vbroadcast.ll
@@ -210,7 +210,7 @@ define <16 x i32> @test_vbroadcast(<16 x float> %a0) {
 ; ALL:       # %bb.0: # %entry
 ; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; ALL-NEXT:    vcmpunordps %zmm1, %zmm0, %k1
-; ALL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; ALL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; ALL-NEXT:    knotw %k1, %k1
 ; ALL-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z}
 ; ALL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
index 3f6ddc6ecfd7..f66f0c0ceabc 100644
--- a/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
+++ b/llvm/test/CodeGen/X86/avx512fp16-cvt.ll
@@ -1054,4 +1054,22 @@ define <8 x half> @u64tof16(<8 x i64> %a) #0 {
   ret <8 x half> %1
 }
 
+define void @f16tou32(ptr %ptr) {
+; X64-LABEL: f16tou32:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvttph2udq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0
+; X64-NEXT:    vmovlps %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: f16tou32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvttph2udq {{\.?LCPI[0-9]+_[0-9]+}}{1to4}, %xmm0
+; X86-NEXT:    vmovlps %xmm0, (%eax)
+; X86-NEXT:    retl
+  %1 = fptoui <2 x half> splat (half 0xH7E00) to <2 x i32>
+  store <2 x i32> %1, ptr %ptr, align 8
+  ret void
+}
+
 attributes #0 = { "min-legal-vector-width"="256" "prefer-vector-width"="256" }
diff --git a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
index ba09ba8b6402..2aa79fafe59a 100644
--- a/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
+++ b/llvm/test/CodeGen/X86/change-unsafe-fp-math.ll
@@ -4,7 +4,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown \
 ; RUN:   | FileCheck %s --check-prefix=CHECK --check-prefix=SAFE
 
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math \
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown \
 ; RUN:   | FileCheck %s --check-prefix=CHECK --check-prefix=UNSAFE
 
 ; The div in these functions should be converted to a mul when unsafe-fp-math
@@ -12,14 +12,19 @@
 
 ; CHECK-LABEL: unsafe_fp_math_default0:
 define double @unsafe_fp_math_default0(double %x) {
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default0:
+define double @safe_fp_math_default0(double %x) {
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_off:
-define double @unsafe_fp_math_off(double %x) #0 {
+define double @unsafe_fp_math_off(double %x) {
 ; SAFE:      divsd
 ; UNSAFE:    divsd
   %div = fdiv double %x, 3.0
@@ -29,28 +34,37 @@ define double @unsafe_fp_math_off(double %x) #0 {
 ; CHECK-LABEL: unsafe_fp_math_default1:
 define double @unsafe_fp_math_default1(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default1:
+define double @safe_fp_math_default1(double %x) {
+; With unsafe math enabled, can change this div to a mul.
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_on:
-define double @unsafe_fp_math_on(double %x) #1 {
+define double @unsafe_fp_math_on(double %x) {
 ; SAFE:      mulsd
 ; UNSAFE:    mulsd
-  %div = fdiv double %x, 3.0
+  %div = fdiv arcp double %x, 3.0
   ret double %div
 }
 
 ; CHECK-LABEL: unsafe_fp_math_default2:
 define double @unsafe_fp_math_default2(double %x) {
 ; With unsafe math enabled, can change this div to a mul.
-; SAFE:      divsd
 ; UNSAFE:    mulsd
+  %div = fdiv arcp double %x, 3.0
+  ret double %div
+}
+; CHECK-LABEL: safe_fp_math_default2:
+define double @safe_fp_math_default2(double %x) {
+; With unsafe math enabled, can change this div to a mul.
+; SAFE:      divsd
   %div = fdiv double %x, 3.0
   ret double %div
 }
-
-attributes #0 = { "unsafe-fp-math"="false" }
-attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll b/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
new file mode 100644
index 000000000000..6291100f42c3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/dag-combiner-fma-folding.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=x86_64-- --start-before=x86-isel -mattr=+avx,+fma %s -o - | FileCheck %s
+; RUN: llc -mtriple=x86_64-- --start-before=x86-isel -mattr=+avx,+fma %s -o - -fp-contract=fast | FileCheck %s
+
+define double @fma_folding(double %x) {
+; CHECK-LABEL: fma_folding:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = [1.0E+0,0.0E+0]
+; CHECK-NEXT:    retq
+ %prod = fmul contract ninf nnan double %x, 0.0
+ %fused = fadd contract ninf nnan double %prod, 1.0
+ ret double %fused
+}
+
+define double @fma_no_folding(double %x) {
+; CHECK-LABEL: fma_no_folding:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; CHECK-NEXT:    retq
+  %fused = call contract nnan ninf double @llvm.fma.f64(double %x, double 0.0, double -0.0)
+  ret double %fused
+}
+
+define double @fma_no_fold_potential_nan(double %x) {
+; CHECK-LABEL: fma_no_fold_potential_nan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem
+; CHECK-NEXT:    retq
+ %fused = call contract double @llvm.fma.f64(double %x, double 0.0, double 1.0)
+ ret double %fused
+}
+
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 8f82a5bc6554..d869f8ec01a5 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -172,9 +172,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    sbbl %eax, %edi
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl 40(%ebp), %esi
-; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    movl 40(%ebp), %ecx
+; X86-NEXT:    movl %ecx, %edx
 ; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:    xorl %edx, %esi
 ; X86-NEXT:    movl 36(%ebp), %ecx
 ; X86-NEXT:    xorl %edx, %ecx
@@ -203,45 +204,45 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    orb %cl, %al
 ; X86-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT:    bsrl %esi, %edx
-; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    bsrl %eax, %ecx
+; X86-NEXT:    bsrl %eax, %edx
+; X86-NEXT:    xorl $31, %edx
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:    bsrl %esi, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
 ; X86-NEXT:    testl %esi, %esi
-; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    cmovel %edx, %ecx
 ; X86-NEXT:    bsrl %ebx, %edx
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    bsrl %edi, %edi
 ; X86-NEXT:    xorl $31, %edi
-; X86-NEXT:    orl $32, %edi
+; X86-NEXT:    addl $32, %edi
 ; X86-NEXT:    movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %edi
-; X86-NEXT:    orl $64, %edi
+; X86-NEXT:    addl $64, %edi
 ; X86-NEXT:    movl %eax, %edx
 ; X86-NEXT:    movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    orl %esi, %edx
 ; X86-NEXT:    cmovnel %ecx, %edi
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT:    bsrl %eax, %edx
+; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT:    addl $32, %edx
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    bsrl %eax, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    cmovnel %edx, %ecx
+; X86-NEXT:    cmovel %edx, %ecx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
 ; X86-NEXT:    bsrl %ebx, %esi
 ; X86-NEXT:    xorl $31, %esi
 ; X86-NEXT:    bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    orl $32, %edx
+; X86-NEXT:    addl $32, %edx
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    orl $64, %edx
+; X86-NEXT:    addl $64, %edx
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
 ; X86-NEXT:    orl %eax, %esi
 ; X86-NEXT:    cmovnel %ecx, %edx
@@ -379,9 +380,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %eax
 ; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT:    adcl $-1, %ecx
-; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    adcl $-1, %eax
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
 ; X86-NEXT:    adcl $-1, %ecx
 ; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 0bef9ee50bd5..7bbddefd8272 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -173,17 +173,17 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    bsrl 36(%ebp), %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
+; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %esi, %esi
 ; X86-NEXT:    cmovnel %edx, %ecx
 ; X86-NEXT:    bsrl %edi, %edx
 ; X86-NEXT:    xorl $31, %edx
 ; X86-NEXT:    bsrl %ebx, %eax
 ; X86-NEXT:    xorl $31, %eax
-; X86-NEXT:    orl $32, %eax
+; X86-NEXT:    addl $32, %eax
 ; X86-NEXT:    testl %edi, %edi
 ; X86-NEXT:    cmovnel %edx, %eax
-; X86-NEXT:    orl $64, %eax
+; X86-NEXT:    addl $64, %eax
 ; X86-NEXT:    movl 36(%ebp), %edx
 ; X86-NEXT:    orl %esi, %edx
 ; X86-NEXT:    cmovnel %ecx, %eax
@@ -193,7 +193,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    movl 20(%ebp), %ecx
 ; X86-NEXT:    bsrl %ecx, %ecx
 ; X86-NEXT:    xorl $31, %ecx
-; X86-NEXT:    orl $32, %ecx
+; X86-NEXT:    addl $32, %ecx
 ; X86-NEXT:    testl %ebx, %ebx
 ; X86-NEXT:    cmovnel %edx, %ecx
 ; X86-NEXT:    movl 16(%ebp), %edi
@@ -201,10 +201,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
 ; X86-NEXT:    xorl $31, %esi
 ; X86-NEXT:    bsrl 12(%ebp), %edx
 ; X86-NEXT:    xorl $31, %edx
-; X86-NEXT:    orl $32, %edx
+; X86-NEXT:    addl $32, %edx
 ; X86-NEXT:    testl %edi, %edi
 ; X86-NEXT:    cmovnel %esi, %edx
-; X86-NEXT:    orl $64, %edx
+; X86-NEXT:    addl $64, %edx
 ; X86-NEXT:    movl 20(%ebp), %edi
 ; X86-NEXT:    movl %edi, %esi
 ; X86-NEXT:    orl %ebx, %esi
diff --git a/llvm/test/CodeGen/X86/fdiv.ll b/llvm/test/CodeGen/X86/fdiv.ll
index 6d2db80a87bd..67bad09d5dd7 100644
--- a/llvm/test/CodeGen/X86/fdiv.ll
+++ b/llvm/test/CodeGen/X86/fdiv.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -enable-unsafe-fp-math | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 define double @exact(double %x) {
 ; Exact division by a constant converted to multiplication.
@@ -17,7 +17,7 @@ define double @inexact(double %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mulsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
-  %div = fdiv double %x, 0x41DFFFFFFFC00000
+  %div = fdiv arcp double %x, 0x41DFFFFFFFC00000
   ret double %div
 }
 
diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
index 95d350d45d90..eef87b5a9f85 100644
--- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
+++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll
@@ -272,30 +272,15 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fmaximumnum_zero0:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movapd %xmm1, %xmm0
-; SSE2-NEXT:    cmpordsd %xmm1, %xmm0
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm1, %xmm2
-; SSE2-NEXT:    xorpd %xmm3, %xmm3
-; SSE2-NEXT:    maxsd %xmm3, %xmm1
-; SSE2-NEXT:    andnpd %xmm1, %xmm0
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    maxsd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fmaximumnum_zero0:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vcmpordsd %xmm1, %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fmaximumnum_zero0:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    vcmpordsd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fmaximumnum_zero0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmaxsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_zero0:
 ; AVX10_2:       # %bb.0:
@@ -311,10 +296,8 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -326,33 +309,15 @@ define double @test_fmaximumnum_zero0(double %x, double %y) nounwind {
 define double @test_fmaximumnum_zero1(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fmaximumnum_zero1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    cmpordsd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm2
-; SSE2-NEXT:    andpd %xmm0, %xmm2
-; SSE2-NEXT:    xorpd %xmm3, %xmm3
-; SSE2-NEXT:    maxsd %xmm3, %xmm0
-; SSE2-NEXT:    andnpd %xmm0, %xmm1
-; SSE2-NEXT:    orpd %xmm2, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    xorpd %xmm1, %xmm1
+; SSE2-NEXT:    maxsd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fmaximumnum_zero1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fmaximumnum_zero1:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vcmpordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT:    vmovapd %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fmaximumnum_zero1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_zero1:
 ; AVX10_2:       # %bb.0:
@@ -368,10 +333,8 @@ define double @test_fmaximumnum_zero1(double %x, double %y) nounwind {
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -802,27 +765,13 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fminimumnum_zero0:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movapd %xmm1, %xmm0
-; SSE2-NEXT:    cmpordsd %xmm1, %xmm0
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm1, %xmm2
-; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    andnpd %xmm1, %xmm0
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fminimumnum_zero0:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vcmpordsd %xmm1, %xmm1, %xmm0
-; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fminimumnum_zero0:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcmpordsd %xmm1, %xmm1, %k1
-; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fminimumnum_zero0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_zero0:
 ; AVX10_2:       # %bb.0:
@@ -836,10 +785,8 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -851,30 +798,13 @@ define double @test_fminimumnum_zero0(double %x, double %y) nounwind {
 define double @test_fminimumnum_zero1(double %x, double %y) nounwind {
 ; SSE2-LABEL: test_fminimumnum_zero1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    cmpordsd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm2
-; SSE2-NEXT:    andpd %xmm0, %xmm2
 ; SSE2-NEXT:    minsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE2-NEXT:    andnpd %xmm0, %xmm1
-; SSE2-NEXT:    orpd %xmm2, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; AVX1-LABEL: test_fminimumnum_zero1:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; AVX1-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX512-LABEL: test_fminimumnum_zero1:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcmpordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vmovsd %xmm0, %xmm1, %xmm1 {%k1}
-; AVX512-NEXT:    vmovapd %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: test_fminimumnum_zero1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_zero1:
 ; AVX10_2:       # %bb.0:
@@ -888,10 +818,8 @@ define double @test_fminimumnum_zero1(double %x, double %y) nounwind {
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86-NEXT:    vcmpordsd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
-; X86-NEXT:    vmovlpd %xmm0, (%esp)
+; X86-NEXT:    vminsd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-NEXT:    vmovsd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
@@ -1156,13 +1084,19 @@ define <2 x double> @test_fminimumnum_vector_zero(<2 x double> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorpd %xmm1, %xmm1
 ; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_zero:
@@ -1174,7 +1108,9 @@ define <2 x double> @test_fminimumnum_vector_zero(<2 x double> %x) {
 ; X86-LABEL: test_fminimumnum_vector_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double 0., double 0.>)
   ret <2 x double> %r
@@ -1185,13 +1121,19 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpordps %xmm0, %xmm2
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_signed_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_signed_zero:
@@ -1202,7 +1144,9 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_signed_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float -0., float -0., float -0., float -0.>)
   ret <4 x float> %r
@@ -1211,17 +1155,23 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero(<4 x float> %x) {
 define <2 x double> @test_fminimumnum_vector_partially_zero(<2 x double> %x) {
 ; SSE2-LABEL: test_fminimumnum_vector_partially_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    xorpd %xmm1, %xmm1
-; SSE2-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm1
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm1
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    movhpd {{.*#+}} xmm2 = xmm2[0],mem[0]
+; SSE2-NEXT:    minpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm1, %xmm0
+; SSE2-NEXT:    andnpd %xmm2, %xmm1
+; SSE2-NEXT:    orpd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_partially_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_partially_zero:
@@ -1235,7 +1185,9 @@ define <2 x double> @test_fminimumnum_vector_partially_zero(<2 x double> %x) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
 ; X86-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0]
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double 0., double 5.>)
   ret <2 x double> %r
@@ -1302,15 +1254,12 @@ define <2 x double> @test_fminimumnum_vector_different_zeros(<2 x double> %x) {
 define <4 x float> @test_fmaximumnum_vector_non_zero(<4 x float> %x) {
 ; SSE2-LABEL: test_fmaximumnum_vector_non_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    maxps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_non_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_non_zero:
@@ -1320,8 +1269,7 @@ define <4 x float> @test_fmaximumnum_vector_non_zero(<4 x float> %x) {
 ;
 ; X86-LABEL: test_fmaximumnum_vector_non_zero:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovaps {{.*#+}} xmm1 = [5.0E+0,4.0E+0,3.0E+0,2.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float 5., float 4., float 3., float 2.>)
   ret <4 x float> %r
@@ -1370,13 +1318,19 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorpd %xmm1, %xmm1
 ; SSE2-NEXT:    minpd %xmm0, %xmm1
-; SSE2-NEXT:    movapd %xmm1, %xmm0
+; SSE2-NEXT:    movapd %xmm0, %xmm2
+; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
+; SSE2-NEXT:    andpd %xmm2, %xmm0
+; SSE2-NEXT:    andnpd %xmm1, %xmm2
+; SSE2-NEXT:    orpd %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_zero_first:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_zero_first:
@@ -1388,7 +1342,9 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 ; X86-LABEL: test_fminimumnum_vector_zero_first:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vminpd %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> <double 0., double 0.>, <2 x double> %x)
   ret <2 x double> %r
@@ -1397,20 +1353,12 @@ define <2 x double> @test_fminimumnum_vector_zero_first(<2 x double> %x) {
 define <2 x double> @test_fminimumnum_vector_signed_zero(<2 x double> %x) {
 ; SSE2-LABEL: test_fminimumnum_vector_signed_zero:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movapd %xmm0, %xmm1
-; SSE2-NEXT:    minpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT:    movapd %xmm0, %xmm2
-; SSE2-NEXT:    cmpordpd %xmm0, %xmm2
-; SSE2-NEXT:    andpd %xmm2, %xmm0
-; SSE2-NEXT:    andnpd %xmm1, %xmm2
-; SSE2-NEXT:    orpd %xmm2, %xmm0
+; SSE2-NEXT:    minpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fminimumnum_vector_signed_zero:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm1
-; AVX-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm2
-; AVX-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; AVX-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fminimumnum_vector_signed_zero:
@@ -1420,9 +1368,7 @@ define <2 x double> @test_fminimumnum_vector_signed_zero(<2 x double> %x) {
 ;
 ; X86-LABEL: test_fminimumnum_vector_signed_zero:
 ; X86:       # %bb.0:
-; X86-NEXT:    vcmpordpd %xmm0, %xmm0, %xmm1
-; X86-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm1, %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vminpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double -0., double -0.>)
   ret <2 x double> %r
@@ -1433,13 +1379,19 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero_first(<4 x float> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movaps {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; SSE2-NEXT:    maxps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm2
+; SSE2-NEXT:    cmpordps %xmm0, %xmm2
+; SSE2-NEXT:    andps %xmm2, %xmm0
+; SSE2-NEXT:    andnps %xmm1, %xmm2
+; SSE2-NEXT:    orps %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_signed_zero_first:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_signed_zero_first:
@@ -1450,7 +1402,9 @@ define <4 x float> @test_fmaximumnum_vector_signed_zero_first(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_signed_zero_first:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm0, %xmm1, %xmm1
+; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> <float -0., float -0., float -0., float -0.>, <4 x float> %x)
   ret <4 x float> %r
@@ -1460,21 +1414,13 @@ define <4 x float> @test_fmaximumnum_vector_zero(<4 x float> %x) {
 ; SSE2-LABEL: test_fmaximumnum_vector_zero:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    xorps %xmm1, %xmm1
-; SSE2-NEXT:    movaps %xmm0, %xmm2
-; SSE2-NEXT:    maxps %xmm1, %xmm2
-; SSE2-NEXT:    movaps %xmm0, %xmm1
-; SSE2-NEXT:    cmpordps %xmm0, %xmm1
-; SSE2-NEXT:    andps %xmm1, %xmm0
-; SSE2-NEXT:    andnps %xmm2, %xmm1
-; SSE2-NEXT:    orps %xmm1, %xmm0
+; SSE2-NEXT:    maxps %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: test_fmaximumnum_vector_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vmaxps %xmm1, %xmm0, %xmm1
-; AVX-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX10_2-LABEL: test_fmaximumnum_vector_zero:
@@ -1486,9 +1432,7 @@ define <4 x float> @test_fmaximumnum_vector_zero(<4 x float> %x) {
 ; X86-LABEL: test_fmaximumnum_vector_zero:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm1
-; X86-NEXT:    vcmpordps %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxps %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    retl
   %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> <float 0., float 0., float 0., float 0.>)
   ret <4 x float> %r
diff --git a/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
index 4e36d5a31c3d..bd32430fcedb 100644
--- a/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
+++ b/llvm/test/CodeGen/X86/fp-intrinsics-flags.ll
@@ -60,7 +60,7 @@ define i8 @f20s8(double %x) #0 {
 entry:
 ; CHECK-LABEL: name: f20s8
 ; CHECK: [[CVTTSD2SIrm:%[0-9]+]]:gr32 = CVTTSD2SIrm %fixed-stack.0, 1, $noreg, 0, $noreg, implicit $mxcsr :: (load (s64) from %fixed-stack.0, align 16)
-; CHECK: [[COPY:%[0-9]+]]:gr32_abcd = COPY [[CVTTSD2SIrm]]
+; CHECK: [[COPY:%[0-9]+]]:gr32_abcd = COPY killed [[CVTTSD2SIrm]]
 ; CHECK: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
 ; CHECK: $al = COPY [[COPY1]]
 ; CHECK: RET 0, $al
diff --git a/llvm/test/CodeGen/X86/fp128-select.ll b/llvm/test/CodeGen/X86/fp128-select.ll
index 0486c1c4d28e..659e4ddedc64 100644
--- a/llvm/test/CodeGen/X86/fp128-select.ll
+++ b/llvm/test/CodeGen/X86/fp128-select.ll
@@ -8,7 +8,7 @@
 ; RUN: llc < %s -O2 -mtriple=x86_64-linux-gnu -mattr=-sse \
 ; RUN:     -enable-legalize-types-checking | FileCheck %s --check-prefix=NOSSE
 
-define void @test_select(ptr %p, ptr %q, i1 zeroext %c) {
+define void @test_select(ptr %p, ptr %q, i1 zeroext %c) nounwind {
 ; SSE-LABEL: test_select:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    testl %edx, %edx
@@ -40,14 +40,11 @@ define void @test_select(ptr %p, ptr %q, i1 zeroext %c) {
 
 ; The uitofp will become a select_cc. This used to crash during type
 ; legalization because we didn't expect the operands to need to be softened.
-define fp128 @test_select_cc(fp128, fp128) {
+define fp128 @test_select_cc(fp128, fp128) nounwind {
 ; SSE-LABEL: test_select_cc:
 ; SSE:       # %bb.0: # %BB0
 ; SSE-NEXT:    pushq %rbx
-; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    subq $32, %rsp
-; SSE-NEXT:    .cfi_def_cfa_offset 48
-; SSE-NEXT:    .cfi_offset %rbx, -16
 ; SSE-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
 ; SSE-NEXT:    callq __netf2@PLT
@@ -70,28 +67,16 @@ define fp128 @test_select_cc(fp128, fp128) {
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:  .LBB1_5: # %BB2
 ; SSE-NEXT:    addq $32, %rsp
-; SSE-NEXT:    .cfi_def_cfa_offset 16
 ; SSE-NEXT:    popq %rbx
-; SSE-NEXT:    .cfi_def_cfa_offset 8
 ; SSE-NEXT:    retq
 ;
 ; NOSSE-LABEL: test_select_cc:
 ; NOSSE:       # %bb.0: # %BB0
 ; NOSSE-NEXT:    pushq %rbp
-; NOSSE-NEXT:    .cfi_def_cfa_offset 16
 ; NOSSE-NEXT:    pushq %r15
-; NOSSE-NEXT:    .cfi_def_cfa_offset 24
 ; NOSSE-NEXT:    pushq %r14
-; NOSSE-NEXT:    .cfi_def_cfa_offset 32
 ; NOSSE-NEXT:    pushq %r12
-; NOSSE-NEXT:    .cfi_def_cfa_offset 40
 ; NOSSE-NEXT:    pushq %rbx
-; NOSSE-NEXT:    .cfi_def_cfa_offset 48
-; NOSSE-NEXT:    .cfi_offset %rbx, -48
-; NOSSE-NEXT:    .cfi_offset %r12, -40
-; NOSSE-NEXT:    .cfi_offset %r14, -32
-; NOSSE-NEXT:    .cfi_offset %r15, -24
-; NOSSE-NEXT:    .cfi_offset %rbp, -16
 ; NOSSE-NEXT:    movq %rcx, %r15
 ; NOSSE-NEXT:    movq %rdx, %r12
 ; NOSSE-NEXT:    movq %rsi, %rbx
@@ -115,15 +100,10 @@ define fp128 @test_select_cc(fp128, fp128) {
 ; NOSSE-NEXT:    movq %rbx, %rdx
 ; NOSSE-NEXT:  .LBB1_2: # %BB2
 ; NOSSE-NEXT:    popq %rbx
-; NOSSE-NEXT:    .cfi_def_cfa_offset 40
 ; NOSSE-NEXT:    popq %r12
-; NOSSE-NEXT:    .cfi_def_cfa_offset 32
 ; NOSSE-NEXT:    popq %r14
-; NOSSE-NEXT:    .cfi_def_cfa_offset 24
 ; NOSSE-NEXT:    popq %r15
-; NOSSE-NEXT:    .cfi_def_cfa_offset 16
 ; NOSSE-NEXT:    popq %rbp
-; NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; NOSSE-NEXT:    retq
 BB0:
   %a = fcmp oeq fp128 %0, %1
diff --git a/llvm/test/CodeGen/X86/frame-pointer-reserved.ll b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
new file mode 100644
index 000000000000..bf5887fe5050
--- /dev/null
+++ b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @bar(i32, i32)
+
+define void @foo(i32 %0, i32 %1) nounwind "frame-pointer"="reserved" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    callq bar@PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+  call void @bar(i32 %0, i32 %1)
+  call void @bar(i32 %0, i32 %1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index 506a08808ff9..189de051011d 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -802,6 +802,126 @@ define i32 @freeze_fshr(i32 %a0, i32 %a1, i32 %a2) nounwind {
   ret i32 %z
 }
 
+define i32 @freeze_saddo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_saddo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    adcl $0, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_saddo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    adcl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = add i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_uaddo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_uaddo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    adcl $0, %eax
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_uaddo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    adcl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = add i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_ssubo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_ssubo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    setb %cl
+; X86-NEXT:    andl $1, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ssubo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = sub i32 %v, %a1
+  ret i32 %r
+}
+
+define i32 @freeze_usubo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
+; X86-LABEL: freeze_usubo:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    addb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    setb %cl
+; X86-NEXT:    andl $1, %ecx
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_usubo:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    addb %cl, %dl
+; X64-NEXT:    sbbl $0, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    retq
+  %b = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %a2, i8 %a3)
+  %b.o = extractvalue {i8, i1} %b, 1
+  %x = zext i1 %b.o to i32
+
+  %f0 = freeze i32 %a0
+  %o = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %f0, i32 %x)
+  %f = freeze {i32, i1} %o
+  %v = extractvalue {i32, i1} %f, 0
+  %r = sub i32 %v, %a1
+  ret i32 %r
+}
+
 define void @pr59676_frozen(ptr %dst, i32 %x.orig) {
 ; X86-LABEL: pr59676_frozen:
 ; X86:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/freeze-unary.ll b/llvm/test/CodeGen/X86/freeze-unary.ll
index f2c02de74c07..8602c385af83 100644
--- a/llvm/test/CodeGen/X86/freeze-unary.ll
+++ b/llvm/test/CodeGen/X86/freeze-unary.ll
@@ -126,6 +126,140 @@ define <4 x i32> @freeze_bitreverse_vec(<4 x i32> %a0) nounwind {
 }
 declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
 
+define i32 @freeze_ctlz(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz:
+; X86:       # %bb.0:
+; X86-NEXT:    bsrl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $63, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $63, %eax
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    xorl $31, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.ctlz.i32(i32 %a0, i1 0)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_ctlz_undef(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bsrl %eax, %ecx
+; X86-NEXT:    xorl $31, %ecx
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsrl %edi, %ecx
+; X64-NEXT:    xorl $31, %ecx
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.ctlz.i32(i32 %a0, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_ctlz_undef_nonzero(i32 %a0) nounwind {
+; X86-LABEL: freeze_ctlz_undef_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    bsrl %eax, %eax
+; X86-NEXT:    xorl $31, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ctlz_undef_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    bsrl %edi, %eax
+; X64-NEXT:    xorl $31, %eax
+; X64-NEXT:    retq
+  %y = or i32 %a0, 1
+  %x = call i32 @llvm.ctlz.i32(i32 %y, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %y, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz:
+; X86:       # %bb.0:
+; X86-NEXT:    bsfl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.cttz.i32(i32 %a0, i1 0)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz_undef(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bsfl %eax, %ecx
+; X86-NEXT:    movl $32, %eax
+; X86-NEXT:    cmovnel %ecx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    bsfl %edi, %ecx
+; X64-NEXT:    movl $32, %eax
+; X64-NEXT:    cmovnel %ecx, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.cttz.i32(i32 %a0, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %a0, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
+define i32 @freeze_cttz_undef_nonzero(i32 %a0) nounwind {
+; X86-LABEL: freeze_cttz_undef_nonzero:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    orl $1, %eax
+; X86-NEXT:    rep bsfl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_cttz_undef_nonzero:
+; X64:       # %bb.0:
+; X64-NEXT:    orl $1, %edi
+; X64-NEXT:    rep bsfl %edi, %eax
+; X64-NEXT:    retq
+  %y = or i32 %a0, 1
+  %x = call i32 @llvm.cttz.i32(i32 %y, i1 -1)
+  %f = freeze i32 %x
+  %c = icmp eq i32 %y, 0
+  %r = select i1 %c, i32 32, i32 %f
+  ret i32 %r
+}
+
 ; split parity pattern
 define i8 @freeze_ctpop(i8 %a0) nounwind {
 ; X86-LABEL: freeze_ctpop:
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 362b3b945f96..0f66d42697d9 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -171,15 +171,15 @@ define void @freeze_extractelement(ptr %origin0, ptr %origin1, ptr %dst) nounwin
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    vmovdqa (%edx), %xmm0
-; X86-NEXT:    vpand (%ecx), %xmm0, %xmm0
+; X86-NEXT:    vmovdqa (%ecx), %xmm0
+; X86-NEXT:    vpand (%edx), %xmm0, %xmm0
 ; X86-NEXT:    vpextrb $6, %xmm0, (%eax)
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_extractelement:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-NEXT:    vpand (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovdqa (%rsi), %xmm0
+; X64-NEXT:    vpand (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vpextrb $6, %xmm0, (%rdx)
 ; X64-NEXT:    retq
   %i0 = load <16 x i8>, ptr %origin0
@@ -198,8 +198,8 @@ define void @freeze_extractelement_escape(ptr %origin0, ptr %origin1, ptr %dst,
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    vmovdqa (%esi), %xmm0
-; X86-NEXT:    vpand (%edx), %xmm0, %xmm0
+; X86-NEXT:    vmovdqa (%edx), %xmm0
+; X86-NEXT:    vpand (%esi), %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
 ; X86-NEXT:    vpextrb $6, %xmm0, (%eax)
 ; X86-NEXT:    popl %esi
@@ -207,8 +207,8 @@ define void @freeze_extractelement_escape(ptr %origin0, ptr %origin1, ptr %dst,
 ;
 ; X64-LABEL: freeze_extractelement_escape:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-NEXT:    vpand (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovdqa (%rsi), %xmm0
+; X64-NEXT:    vpand (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    vpextrb $6, %xmm0, (%rdx)
 ; X64-NEXT:    retq
@@ -239,8 +239,8 @@ define void @freeze_extractelement_extra_use(ptr %origin0, ptr %origin1, i64 %id
 ; X86-NEXT:    movl 32(%ebp), %edx
 ; X86-NEXT:    movl 12(%ebp), %esi
 ; X86-NEXT:    movl 8(%ebp), %edi
-; X86-NEXT:    vmovaps (%edi), %xmm0
-; X86-NEXT:    vandps (%esi), %xmm0, %xmm0
+; X86-NEXT:    vmovaps (%esi), %xmm0
+; X86-NEXT:    vandps (%edi), %xmm0, %xmm0
 ; X86-NEXT:    vmovaps %xmm0, (%esp)
 ; X86-NEXT:    movzbl (%esp,%ecx), %ecx
 ; X86-NEXT:    cmpb (%esp,%eax), %cl
@@ -255,8 +255,8 @@ define void @freeze_extractelement_extra_use(ptr %origin0, ptr %origin1, i64 %id
 ; X64:       # %bb.0:
 ; X64-NEXT:    andl $15, %ecx
 ; X64-NEXT:    andl $15, %edx
-; X64-NEXT:    vmovaps (%rdi), %xmm0
-; X64-NEXT:    vandps (%rsi), %xmm0, %xmm0
+; X64-NEXT:    vmovaps (%rsi), %xmm0
+; X64-NEXT:    vandps (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    movzbl -24(%rsp,%rdx), %eax
 ; X64-NEXT:    cmpb -24(%rsp,%rcx), %al
@@ -342,42 +342,50 @@ define void @freeze_buildvector_single_repeated_maybe_poison_operand(ptr %origin
 define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_frozen_buildvectors:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
 ; X86-NEXT:    movl (%edx), %edx
 ; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vpinsrd $1, %edx, %xmm0, %xmm0
-; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
-; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; X86-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
+; X86-NEXT:    vbroadcastss {{.*#+}} xmm2 = [7,7,7,7]
+; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
 ; X86-NEXT:    vmovd %edx, %xmm0
 ; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X86-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7]
-; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5],xmm1[6,7]
+; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_frozen_buildvectors:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    movl (%rsi), %eax
+; X64-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
+; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; X64-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
+; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rdx)
 ; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3]
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
+; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i32, ptr %origin0
   %i0 = and i32 %i0.src, 15
   %i1.src = load i32, ptr %origin1
-  %i1 = and i32 %i0.src, 15
+  %i1 = and i32 %i1.src, 15
   %i2 = insertelement <4 x i32> poison, i32 %i0, i64 1
   %i3 = and <4 x i32> %i2, <i32 7, i32 7, i32 7, i32 7>
   %i4 = freeze <4 x i32> %i3
@@ -392,41 +400,43 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
 define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_buildvectors_only_one_frozen:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl (%edx), %edx
-; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X86-NEXT:    vmovd %edx, %xmm1
-; X86-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]
-; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5,6,7]
-; X86-NEXT:    vbroadcastss {{.*#+}} xmm2 = [7,7,7,7]
-; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; X86-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5,6,7]
+; X86-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vbroadcastss (%edx), %xmm2
 ; X86-NEXT:    vmovdqa %xmm0, (%ecx)
-; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,1,0,1]
-; X86-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_buildvectors_only_one_frozen:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    andl $15, %eax
-; X64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X64-NEXT:    vmovd %eax, %xmm1
-; X64-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X64-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; X64-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
-; X64-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X64-NEXT:    vmovdqa %xmm0, (%rdx)
-; X64-NEXT:    vpand %xmm2, %xmm1, %xmm0
-; X64-NEXT:    vmovdqa %xmm0, (%rcx)
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vbroadcastss %xmm0, %xmm0
+; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; X64-NEXT:    vbroadcastss {{.*#+}} xmm1 = [7,7,7,7]
+; X64-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vbroadcastss (%rsi), %xmm2
+; X64-NEXT:    vmovaps %xmm0, (%rdx)
+; X64-NEXT:    vandps %xmm1, %xmm2, %xmm0
+; X64-NEXT:    vmovaps %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i32, ptr %origin0
   %i0 = and i32 %i0.src, 15
   %i1.src = load i32, ptr %origin1
-  %i1 = and i32 %i0.src, 15
+  %i1 = and i32 %i1.src, 15
   %i2 = insertelement <4 x i32> poison, i32 %i0, i64 1
   %i3 = and <4 x i32> %i2, <i32 7, i32 7, i32 7, i32 7>
   %i4 = freeze <4 x i32> %i3
@@ -440,34 +450,40 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
 define void @freeze_two_buildvectors_one_undef_elt(ptr %origin0, ptr %origin1, ptr %dst0, ptr %dst1) nounwind {
 ; X86-LABEL: freeze_two_buildvectors_one_undef_elt:
 ; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl (%edx), %edx
-; X86-NEXT:    andl $15, %edx
-; X86-NEXT:    vmovddup {{.*#+}} xmm0 = [7,0,7,0]
-; X86-NEXT:    # xmm0 = mem[0,0]
-; X86-NEXT:    vmovd %edx, %xmm1
-; X86-NEXT:    vpand %xmm0, %xmm1, %xmm2
-; X86-NEXT:    vmovdqa %xmm2, (%ecx)
-; X86-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X86-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl (%esi), %esi
+; X86-NEXT:    andl $15, %esi
+; X86-NEXT:    vmovd %esi, %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm1 = [7,0,7,0]
+; X86-NEXT:    # xmm1 = mem[0,0]
+; X86-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm2 = mem[0,0]
+; X86-NEXT:    vmovdqa %xmm0, (%ecx)
+; X86-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X86-NEXT:    vmovdqa %xmm0, (%eax)
+; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: freeze_two_buildvectors_one_undef_elt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    andl $15, %eax
 ; X64-NEXT:    vmovd %eax, %xmm0
-; X64-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpmovsxbq {{.*#+}} xmm1 = [7,7]
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vpbroadcastq (%rsi), %xmm2
 ; X64-NEXT:    vmovdqa %xmm0, (%rdx)
+; X64-NEXT:    vpand %xmm1, %xmm2, %xmm0
 ; X64-NEXT:    vmovdqa %xmm0, (%rcx)
 ; X64-NEXT:    retq
   %i0.src = load i64, ptr %origin0
   %i0 = and i64 %i0.src, 15
   %i1.src = load i64, ptr %origin1
-  %i1 = and i64 %i0.src, 15
+  %i1 = and i64 %i1.src, 15
   %i2 = insertelement <2 x i64> poison, i64 %i0, i64 0
   %i3 = and <2 x i64> %i2, <i64 7, i64 7>
   %i4 = freeze <2 x i64> %i3
diff --git a/llvm/test/CodeGen/X86/isel-fabs-x87.ll b/llvm/test/CodeGen/X86/isel-fabs-x87.ll
index 8b846499946c..5579754a54b4 100644
--- a/llvm/test/CodeGen/X86/isel-fabs-x87.ll
+++ b/llvm/test/CodeGen/X86/isel-fabs-x87.ll
@@ -1,8 +1,79 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X86
-; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X64,SDAG-X64-ISEL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X64,FAST-X64-ISEL
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=X64,GISEL-X64-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse | FileCheck %s --check-prefixes=X86,SDAG-X86-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -fast-isel | FileCheck %s --check-prefixes=X86,FAST-X86-ISEL
+; RUN: llc < %s -mtriple=i686-- -mattr=+x87,-sse2,-sse -global-isel -global-isel-abort=0 | FileCheck %s --check-prefixes=X86,GISEL-X86-ISEL
+
+define void @test_float_abs(ptr %argptr)   {
+; SDAG-X64-ISEL-LABEL: test_float_abs:
+; SDAG-X64-ISEL:       # %bb.0:
+; SDAG-X64-ISEL-NEXT:    andb $127, 3(%rdi)
+; SDAG-X64-ISEL-NEXT:    retq
+;
+; FAST-X64-ISEL-LABEL: test_float_abs:
+; FAST-X64-ISEL:       # %bb.0:
+; FAST-X64-ISEL-NEXT:    andb $127, 3(%rdi)
+; FAST-X64-ISEL-NEXT:    retq
+;
+; GISEL-X64-ISEL-LABEL: test_float_abs:
+; GISEL-X64-ISEL:       # %bb.0:
+; GISEL-X64-ISEL-NEXT:    andl $2147483647, (%rdi) # imm = 0x7FFFFFFF
+; GISEL-X64-ISEL-NEXT:    retq
+;
+; SDAG-X86-ISEL-LABEL: test_float_abs:
+; SDAG-X86-ISEL:       # %bb.0:
+; SDAG-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SDAG-X86-ISEL-NEXT:    andb $127, 3(%eax)
+; SDAG-X86-ISEL-NEXT:    retl
+;
+; FAST-X86-ISEL-LABEL: test_float_abs:
+; FAST-X86-ISEL:       # %bb.0:
+; FAST-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; FAST-X86-ISEL-NEXT:    andb $127, 3(%eax)
+; FAST-X86-ISEL-NEXT:    retl
+;
+; GISEL-X86-ISEL-LABEL: test_float_abs:
+; GISEL-X86-ISEL:       # %bb.0:
+; GISEL-X86-ISEL-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-ISEL-NEXT:    andl $2147483647, (%eax) # imm = 0x7FFFFFFF
+; GISEL-X86-ISEL-NEXT:    retl
+     %arg = load  float, float* %argptr
+     %abs = tail call float @llvm.fabs.f32(float %arg)
+     store float %abs, ptr %argptr
+     ret void
+ }
+
+define void @test_double_abs(ptr %argptr)  {
+; SDAG-X64-ISEL-LABEL: test_double_abs:
+; SDAG-X64-ISEL:       # %bb.0:
+; SDAG-X64-ISEL-NEXT:    andb $127, 7(%rdi)
+; SDAG-X64-ISEL-NEXT:    retq
+;
+; FAST-X64-ISEL-LABEL: test_double_abs:
+; FAST-X64-ISEL:       # %bb.0:
+; FAST-X64-ISEL-NEXT:    andb $127, 7(%rdi)
+; FAST-X64-ISEL-NEXT:    retq
+;
+; GISEL-X64-ISEL-LABEL: test_double_abs:
+; GISEL-X64-ISEL:       # %bb.0:
+; GISEL-X64-ISEL-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; GISEL-X64-ISEL-NEXT:    andq %rax, (%rdi)
+; GISEL-X64-ISEL-NEXT:    retq
+;
+; X86-LABEL: test_double_abs:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    fldl (%eax)
+; X86-NEXT:    fabs
+; X86-NEXT:    fstpl (%eax)
+; X86-NEXT:    retl
+    %arg = load double, double* %argptr
+    %abs = tail call double @llvm.fabs.f64(double %arg)
+    store double %abs, double* %argptr
+    ret void
+}
 
 define x86_fp80 @test_x86_fp80_abs(x86_fp80 %arg) {
 ; X64-LABEL: test_x86_fp80_abs:
diff --git a/llvm/test/CodeGen/X86/isel-fabs.ll b/llvm/test/CodeGen/X86/isel-fabs.ll
index 10bd5799280a..c2d29248e49b 100644
--- a/llvm/test/CodeGen/X86/isel-fabs.ll
+++ b/llvm/test/CodeGen/X86/isel-fabs.ll
@@ -1,37 +1,61 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87 | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X64
-; RUN: llc < %s -mtriple=i686-- -mattr=-x87 | FileCheck %s --check-prefixes=X86
-; RUN: llc < %s -mtriple=i686-- -mattr=-x87 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=FASTISEL-X86
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87,+sse,+sse2 | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87,+sse,+sse2 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=X64
+; RUN: llc < %s -mtriple=x86_64-- -mattr=-x87,+sse,+sse2 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
+; RUN: llc < %s -mtriple=i686-- -mattr=-x87,+sse,+sse2 | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=i686-- -mattr=-x87,+sse,+sse2 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes=FASTISEL-X86
+; RUN: llc < %s -mtriple=i686-- -mattr=-x87,+sse,+sse2 -global-isel -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X86
 
-
-define float @test_float_abs(float %arg) {
+define float @test_float_abs(float %arg) nounwind {
 ; X64-LABEL: test_float_abs:
 ; X64:       # %bb.0:
 ; X64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; X64-NEXT:    retq
 ;
+; GISEL-X64-LABEL: test_float_abs:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    movd %xmm0, %eax
+; GISEL-X64-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
+; GISEL-X64-NEXT:    movd %eax, %xmm0
+; GISEL-X64-NEXT:    retq
+;
 ; X86-LABEL: test_float_abs:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
-; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-NEXT:    movd %xmm0, %eax
 ; X86-NEXT:    retl
 ;
 ; FASTISEL-X86-LABEL: test_float_abs:
 ; FASTISEL-X86:       # %bb.0:
-; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT:    andl $2147483647, %eax # imm = 0x7FFFFFFF
+; FASTISEL-X86-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; FASTISEL-X86-NEXT:    pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; FASTISEL-X86-NEXT:    movd %xmm0, %eax
 ; FASTISEL-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_float_abs:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    retl
     %abs = tail call float @llvm.fabs.f32(float %arg)
     ret float %abs
 }
 
-define double @test_double_abs(double %arg) {
+define double @test_double_abs(double %arg) nounwind {
 ; X64-LABEL: test_double_abs:
 ; X64:       # %bb.0:
 ; X64-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; X64-NEXT:    retq
 ;
+; GISEL-X64-LABEL: test_double_abs:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; GISEL-X64-NEXT:    movq %xmm0, %rcx
+; GISEL-X64-NEXT:    andq %rax, %rcx
+; GISEL-X64-NEXT:    movq %rcx, %xmm0
+; GISEL-X64-NEXT:    retq
+;
 ; X86-LABEL: test_double_abs:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -41,10 +65,26 @@ define double @test_double_abs(double %arg) {
 ;
 ; FASTISEL-X86-LABEL: test_double_abs:
 ; FASTISEL-X86:       # %bb.0:
+; FASTISEL-X86-NEXT:    pushl %ebp
+; FASTISEL-X86-NEXT:    movl %esp, %ebp
+; FASTISEL-X86-NEXT:    andl $-8, %esp
+; FASTISEL-X86-NEXT:    subl $8, %esp
+; FASTISEL-X86-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; FASTISEL-X86-NEXT:    andps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; FASTISEL-X86-NEXT:    movlps %xmm0, (%esp)
+; FASTISEL-X86-NEXT:    movl (%esp), %eax
 ; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; FASTISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT:    andl $2147483647, %edx # imm = 0x7FFFFFFF
+; FASTISEL-X86-NEXT:    movl %ebp, %esp
+; FASTISEL-X86-NEXT:    popl %ebp
 ; FASTISEL-X86-NEXT:    retl
+;
+; GISEL-X86-LABEL: test_double_abs:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    movl $-1, %eax
+; GISEL-X86-NEXT:    movl $2147483647, %edx # imm = 0x7FFFFFFF
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; GISEL-X86-NEXT:    retl
     %abs = tail call double @llvm.fabs.f64(double %arg)
     ret double %abs
 }
diff --git a/llvm/test/CodeGen/X86/legalize-sub-zero.ll b/llvm/test/CodeGen/X86/legalize-sub-zero.ll
index 51f17512cae6..296ae3f9e882 100644
--- a/llvm/test/CodeGen/X86/legalize-sub-zero.ll
+++ b/llvm/test/CodeGen/X86/legalize-sub-zero.ll
@@ -1,13 +1,17 @@
-; RUN: llc < %s -mtriple=i686-pc-win32
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=i686-pc-win32 | FileCheck %s
 
 ;target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 ;target triple = "i686-pc-win32"
 
 define void @test() nounwind {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $0, (%eax)
+; CHECK-NEXT:    retl
   %1 = fdiv <3 x double> zeroinitializer, undef
   %2 = fdiv <2 x double> zeroinitializer, undef
-  %3 = shufflevector <2 x double> %2, <2 x double> undef, <3 x i32> <i32 0, i32
-1, i32 undef>
+  %3 = shufflevector <2 x double> %2, <2 x double> undef, <3 x i32> <i32 0, i32 1, i32 undef>
   %4 = insertelement <3 x double> %3, double undef, i32 2
   %5 = bitcast <3 x double> %1 to <3 x i64>
   %6 = bitcast <3 x double> %4 to <3 x i64>
@@ -15,20 +19,16 @@ define void @test() nounwind {
   %8 = shufflevector <3 x i64> %7, <3 x i64> undef, <2 x i32> <i32 0, i32 1>
   %9 = xor <2 x i64> %8, zeroinitializer
   %10 = add nsw <2 x i64> %9, zeroinitializer
-  %11 = shufflevector <2 x i64> %10, <2 x i64> undef, <3 x i32> <i32 0, i32 1,
-i32 undef>
+  %11 = shufflevector <2 x i64> %10, <2 x i64> undef, <3 x i32> <i32 0, i32 1, i32 undef>
   %12 = insertelement <3 x i64> %11, i64 0, i32 2
-  %13 = shufflevector <3 x i64> %12, <3 x i64> undef, <4 x i32> <i32 0, i32 1,
-i32 2, i32 3>
+  %13 = shufflevector <3 x i64> %12, <3 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %14 = shufflevector <4 x i64> %13, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
   %15 = bitcast <2 x i64> %14 to <4 x i32>
-  %16 = shufflevector <4 x i32> %15, <4 x i32> undef, <4 x i32> <i32 0, i32 2,
-i32 0, i32 2>
+  %16 = shufflevector <4 x i32> %15, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 0, i32 2>
   %17 = bitcast <4 x i32> %16 to <2 x i64>
   %18 = shufflevector <2 x i64> %17, <2 x i64> undef, <2 x i32> <i32 0, i32 2>
   %19 = bitcast <2 x i64> %18 to <4 x i32>
-  %20 = shufflevector <4 x i32> %19, <4 x i32> undef, <3 x i32> <i32 0, i32 1,
-i32 2>
+  %20 = shufflevector <4 x i32> %19, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
   %21 = or <3 x i32> %20, zeroinitializer
   store <3 x i32> %21, ptr addrspace(1) undef, align 16
   ret void
diff --git a/llvm/test/CodeGen/X86/llvm.sincos.ll b/llvm/test/CodeGen/X86/llvm.sincos.ll
index 5734729a2c50..065710f91457 100644
--- a/llvm/test/CodeGen/X86/llvm.sincos.ll
+++ b/llvm/test/CodeGen/X86/llvm.sincos.ll
@@ -3,8 +3,9 @@
 ; RUN: llc < %s -mtriple=x86_64-linux-gnu -fast-isel  | FileCheck %s --check-prefixes=X64,FASTISEL-X64
 ; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel=0 -fast-isel=0  | FileCheck %s --check-prefixes=X86,SDAG-X86
 ; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel=0 -fast-isel=0  | FileCheck %s --check-prefixes=X64,SDAG-X64
-; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel=1 -global-isel-abort=2 | FileCheck %s --check-prefixes=X86,GISEL-X86
-; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel=1 -global-isel-abort=2 | FileCheck %s --check-prefixes=X64,GISEL-X64
+; TODO: The below RUN line will fails GISEL selection and will fallback to DAG selection due to lack of support for loads/stores in i686 mode, support is expected soon enough, for this reason the llvm/test/CodeGen/X86/GlobalISel/llvm.sincos.mir test is added for now because of the lack of support for i686 in GlobalISel.
+; RUN: llc < %s -mtriple=i686-linux-gnu -global-isel=1 -global-isel-abort=2 | FileCheck %s --check-prefixes=GISEL-X86
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -global-isel=1 -global-isel-abort=1 | FileCheck %s --check-prefixes=GISEL-X64
 
 define { float, float } @test_sincos_f32(float %Val) nounwind {
 ; X86-LABEL: test_sincos_f32:
@@ -32,6 +33,35 @@ define { float, float } @test_sincos_f32(float %Val) nounwind {
 ; X64-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64-NEXT:    popq %rax
 ; X64-NEXT:    retq
+;
+; GISEL-X86-LABEL: test_sincos_f32:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $28, %esp
+; GISEL-X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %edx
+; GISEL-X86-NEXT:    movl %eax, (%esp)
+; GISEL-X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movl %edx, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    calll sincosf
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    flds {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    addl $28, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: test_sincos_f32:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    pushq %rax
+; GISEL-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; GISEL-X64-NEXT:    movq %rsp, %rsi
+; GISEL-X64-NEXT:    callq sincosf
+; GISEL-X64-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT:    movl (%rsp), %ecx
+; GISEL-X64-NEXT:    movd %eax, %xmm0
+; GISEL-X64-NEXT:    movd %ecx, %xmm1
+; GISEL-X64-NEXT:    popq %rax
+; GISEL-X64-NEXT:    retq
   %res = call { float, float } @llvm.sincos.f32(float %Val)
   ret { float, float } %res
 }
@@ -62,6 +92,34 @@ define { double, double } @test_sincos_f64(double %Val) nounwind  {
 ; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; X64-NEXT:    addq $24, %rsp
 ; X64-NEXT:    retq
+;
+; GISEL-X86-LABEL: test_sincos_f64:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $44, %esp
+; GISEL-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fstpl (%esp)
+; GISEL-X86-NEXT:    calll sincos
+; GISEL-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldl {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    addl $44, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: test_sincos_f64:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $24, %rsp
+; GISEL-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; GISEL-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; GISEL-X64-NEXT:    callq sincos
+; GISEL-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; GISEL-X64-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; GISEL-X64-NEXT:    movq %rax, %xmm0
+; GISEL-X64-NEXT:    movq %rcx, %xmm1
+; GISEL-X64-NEXT:    addq $24, %rsp
+; GISEL-X64-NEXT:    retq
   %res = call { double, double } @llvm.sincos.f64(double %Val)
   ret { double, double } %res
 }
@@ -94,6 +152,36 @@ define { x86_fp80, x86_fp80 } @test_sincos_f80(x86_fp80 %Val) nounwind {
 ; X64-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-NEXT:    addq $56, %rsp
 ; X64-NEXT:    retq
+;
+; GISEL-X86-LABEL: test_sincos_f80:
+; GISEL-X86:       # %bb.0:
+; GISEL-X86-NEXT:    subl $60, %esp
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; GISEL-X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; GISEL-X86-NEXT:    fstpt (%esp)
+; GISEL-X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    calll sincosl
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fldt {{[0-9]+}}(%esp)
+; GISEL-X86-NEXT:    fxch %st(1)
+; GISEL-X86-NEXT:    addl $60, %esp
+; GISEL-X86-NEXT:    retl
+;
+; GISEL-X64-LABEL: test_sincos_f80:
+; GISEL-X64:       # %bb.0:
+; GISEL-X64-NEXT:    subq $56, %rsp
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
+; GISEL-X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rsi
+; GISEL-X64-NEXT:    fstpt (%rsp)
+; GISEL-X64-NEXT:    callq sincosl
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fldt {{[0-9]+}}(%rsp)
+; GISEL-X64-NEXT:    fxch %st(1)
+; GISEL-X64-NEXT:    addq $56, %rsp
+; GISEL-X64-NEXT:    retq
   %res = call { x86_fp80, x86_fp80 } @llvm.sincos.f80(x86_fp80 %Val)
   ret { x86_fp80, x86_fp80 } %res
 }
diff --git a/llvm/test/CodeGen/X86/ms-secure-hotpatch-attr.ll b/llvm/test/CodeGen/X86/ms-secure-hotpatch-attr.ll
new file mode 100644
index 000000000000..7e60e6a04750
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-secure-hotpatch-attr.ll
@@ -0,0 +1,38 @@
+; This tests directly annotating a function with marked_for_windows_hot_patching.
+;
+; RUN: llc -mtriple=x86_64-windows < %s | FileCheck %s
+
+source_filename = ".\\ms-secure-hotpatch-attr.ll"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.36.32537"
+
+@some_global_var = external global i32
+
+define noundef i32 @this_gets_hotpatched() #0 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #0 = { "marked_for_windows_hot_patching" mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_gets_hotpatched: # @this_gets_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movq __ref_some_global_var(%rip), %rax
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
+
+define noundef i32 @this_does_not_get_hotpatched() #1 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #1 = { mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_does_not_get_hotpatched: # @this_does_not_get_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movl some_global_var(%rip), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/ms-secure-hotpatch-bad-file.ll b/llvm/test/CodeGen/X86/ms-secure-hotpatch-bad-file.ll
new file mode 100644
index 000000000000..5fec1c48495a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-secure-hotpatch-bad-file.ll
@@ -0,0 +1,16 @@
+; RUN: not llc -mtriple=x86_64-windows --ms-secure-hotpatch-functions-file=%S/this-file-is-intentionally-missing-do-not-create-it.txt < %s 2>&1 | FileCheck %s
+; CHECK: failed to open hotpatch functions file
+
+source_filename = ".\\ms-secure-hotpatch.ll"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.36.32537"
+
+@some_global_var = external global i32
+
+define noundef i32 @this_gets_hotpatched() #0 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #0 = { "marked_for_windows_hot_patching" mustprogress noinline nounwind optnone uwtable }
diff --git a/llvm/test/CodeGen/X86/ms-secure-hotpatch-direct-global-access.ll b/llvm/test/CodeGen/X86/ms-secure-hotpatch-direct-global-access.ll
new file mode 100644
index 000000000000..5606b03760be
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-secure-hotpatch-direct-global-access.ll
@@ -0,0 +1,39 @@
+; This tests hotpatching functions that bypass double-indirection for global variables.
+;
+; RUN: llc -mtriple=x86_64-windows < %s | FileCheck %s
+
+source_filename = ".\\ms-secure-hotpatch-direct-global-access.ll"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.36.32537"
+
+@some_global_var = external global i32 #2
+
+define noundef i32 @this_gets_hotpatched() #0 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #0 = { "marked_for_windows_hot_patching" mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_gets_hotpatched: # @this_gets_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movl some_global_var(%rip), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
+
+define noundef i32 @this_does_not_get_hotpatched() #1 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #1 = { mustprogress noinline nounwind optnone uwtable }
+
+attributes #2 = { "allow_direct_access_in_hot_patch_function" }
+
+; CHECK: this_does_not_get_hotpatched: # @this_does_not_get_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movl some_global_var(%rip), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-file.ll b/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-file.ll
new file mode 100644
index 000000000000..bb79e08683aa
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-file.ll
@@ -0,0 +1,39 @@
+; This tests annotating a function with marked_for_windows_hot_patching by using --ms-hotpatch-functions-file.
+;
+; RUN: echo this_gets_hotpatched > %t.patch-functions.txt
+; RUN: llc -mtriple=x86_64-windows --ms-secure-hotpatch-functions-file=%t.patch-functions.txt < %s | FileCheck %s
+
+source_filename = ".\\ms-secure-hotpatch-functions-file.ll"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.36.32537"
+
+@some_global_var = external global i32
+
+define noundef i32 @this_gets_hotpatched() #0 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #0 = { mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_gets_hotpatched: # @this_gets_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movq __ref_some_global_var(%rip), %rax
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
+
+define noundef i32 @this_does_not_get_hotpatched() #1 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #1 = { mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_does_not_get_hotpatched: # @this_does_not_get_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movl some_global_var(%rip), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-list.ll b/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-list.ll
new file mode 100644
index 000000000000..b1da1a2db66d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ms-secure-hotpatch-functions-list.ll
@@ -0,0 +1,38 @@
+; This tests annotating a function with marked_for_windows_hot_patching by using --ms-hotpatch-functions-list.
+;
+; RUN: llc -mtriple=x86_64-windows --ms-secure-hotpatch-functions-list=this_gets_hotpatched < %s | FileCheck %s
+
+source_filename = ".\\ms-secure-hotpatch-functions-list.ll"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.36.32537"
+
+@some_global_var = external global i32
+
+define noundef i32 @this_gets_hotpatched() #0 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #0 = { mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_gets_hotpatched: # @this_gets_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movq __ref_some_global_var(%rip), %rax
+; CHECK-NEXT: movl (%rax), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
+
+define noundef i32 @this_does_not_get_hotpatched() #1 {
+    %1 = load i32, ptr @some_global_var
+    %2 = add i32 %1, 1
+    ret i32 %2
+}
+
+attributes #1 = { mustprogress noinline nounwind optnone uwtable }
+
+; CHECK: this_does_not_get_hotpatched: # @this_does_not_get_hotpatched
+; CHECK-NEXT: bb.0:
+; CHECK-NEXT: movl some_global_var(%rip), %eax
+; CHECK-NEXT: addl $1, %eax
+; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/pr134607.ll b/llvm/test/CodeGen/X86/pr134607.ll
index 5e824c22e5a2..2b383fc96429 100644
--- a/llvm/test/CodeGen/X86/pr134607.ll
+++ b/llvm/test/CodeGen/X86/pr134607.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -mtriple=i386-unknown-unknown -mattr=+sse -O3 | FileCheck %s --check-prefixes=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-sse2,+sse -O3 | FileCheck %s --check-prefixes=X64-SSE1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2,+sse -O3 | FileCheck %s --check-prefixes=X64-SSE2
@@ -5,16 +6,22 @@
 define void @store_v2f32_constant(ptr %v) {
 ; X86-LABEL: store_v2f32_constant:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl 4(%esp), %eax
-; X86-NEXT:    movaps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movaps {{.*#+}} xmm0 = [2.56E+2,5.12E+2,u,u]
+; X86-NEXT:    movlps %xmm0, (%eax)
+; X86-NEXT:    retl
+;
 ; X64-SSE1-LABEL: store_v2f32_constant:
 ; X64-SSE1:       # %bb.0:
-; X64-SSE1-NEXT:    movaps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-
+; X64-SSE1-NEXT:    movaps {{.*#+}} xmm0 = [2.56E+2,5.12E+2,u,u]
+; X64-SSE1-NEXT:    movlps %xmm0, (%rdi)
+; X64-SSE1-NEXT:    retq
+;
 ; X64-SSE2-LABEL: store_v2f32_constant:
 ; X64-SSE2:       # %bb.0:
-; X64-SSE2-NEXT:    movsd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT:    movsd {{.*#+}} xmm0 = [2.56E+2,5.12E+2,0.0E+0,0.0E+0]
+; X64-SSE2-NEXT:    movsd %xmm0, (%rdi)
+; X64-SSE2-NEXT:    retq
   store <2 x float> <float 2.560000e+02, float 5.120000e+02>, ptr %v, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/pr43157.ll b/llvm/test/CodeGen/X86/pr43157.ll
index 2b333782f43a..308a8b91d859 100644
--- a/llvm/test/CodeGen/X86/pr43157.ll
+++ b/llvm/test/CodeGen/X86/pr43157.ll
@@ -1,6 +1,7 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: not llc < %s -mtriple=i686-pc-linux -o - -mattr=+sse2 2>&1 | FileCheck %s --check-prefix=ERR
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -o - -mattr=+mmx | FileCheck %s
 
+; ERR: error: couldn't allocate input reg for constraint 'x'
 define void @foo(fp128 %x) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
index 2ac2be5545df..d2b292f1a799 100644
--- a/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
+++ b/llvm/test/CodeGen/X86/setcc-non-simple-type.ll
@@ -119,8 +119,8 @@ define void @failing(ptr %0, ptr %1) nounwind {
 ; CHECK-AVX2-NEXT:  .LBB0_2: # %vector.body
 ; CHECK-AVX2-NEXT:    # Parent Loop BB0_1 Depth=1
 ; CHECK-AVX2-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-AVX2-NEXT:    vmovdqu 1024(%rdx,%rsi), %xmm5
-; CHECK-AVX2-NEXT:    vmovdqu 1040(%rdx,%rsi), %xmm6
+; CHECK-AVX2-NEXT:    vmovdqu 1024(%rdx,%rsi), %ymm5
+; CHECK-AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; CHECK-AVX2-NEXT:    vpextrq $1, %xmm5, %rdi
 ; CHECK-AVX2-NEXT:    vpextrq $1, %xmm6, %r8
 ; CHECK-AVX2-NEXT:    vmovq %xmm5, %r9
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index c7da04171e6a..756019d0e98a 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -48,46 +48,20 @@ define <8 x i64> @shl_i512_1(<8 x i64> %a)  {
 ;
 ; ZNVER4-LABEL: shl_i512_1:
 ; ZNVER4:       # %bb.0:
-; ZNVER4-NEXT:    vextracti32x4 $3, %zmm0, %xmm1
-; ZNVER4-NEXT:    vmovq %xmm0, %rdx
-; ZNVER4-NEXT:    vpextrq $1, %xmm0, %r9
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %rax
-; ZNVER4-NEXT:    vmovq %xmm1, %rcx
 ; ZNVER4-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
-; ZNVER4-NEXT:    shrq $63, %rdx
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %rsi
-; ZNVER4-NEXT:    vmovq %xmm1, %rdi
-; ZNVER4-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; ZNVER4-NEXT:    leaq (%rdx,%r9,2), %rdx
-; ZNVER4-NEXT:    shrq $63, %r9
-; ZNVER4-NEXT:    vpsllq $1, %xmm0, %xmm0
-; ZNVER4-NEXT:    vmovq %xmm1, %r10
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %r8
-; ZNVER4-NEXT:    leaq (%r9,%r10,2), %r9
-; ZNVER4-NEXT:    shrq $63, %r10
-; ZNVER4-NEXT:    vmovq %rdx, %xmm4
-; ZNVER4-NEXT:    leaq (%r10,%r8,2), %r10
-; ZNVER4-NEXT:    shrq $63, %r8
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
-; ZNVER4-NEXT:    leaq (%r8,%rdi,2), %r8
-; ZNVER4-NEXT:    shrq $63, %rdi
-; ZNVER4-NEXT:    leaq (%rdi,%rsi,2), %rdi
-; ZNVER4-NEXT:    shrq $63, %rsi
-; ZNVER4-NEXT:    leaq (%rsi,%rcx,2), %rsi
-; ZNVER4-NEXT:    shrq $63, %rcx
-; ZNVER4-NEXT:    vmovq %r8, %xmm3
-; ZNVER4-NEXT:    leaq (%rcx,%rax,2), %rax
-; ZNVER4-NEXT:    vmovq %rsi, %xmm2
-; ZNVER4-NEXT:    vmovq %rax, %xmm1
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; ZNVER4-NEXT:    vmovq %rdi, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; ZNVER4-NEXT:    vmovq %r10, %xmm3
+; ZNVER4-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; ZNVER4-NEXT:    vpsllq $1, %xmm0, %xmm4
 ; ZNVER4-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
-; ZNVER4-NEXT:    vmovq %r9, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; ZNVER4-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; ZNVER4-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; ZNVER4-NEXT:    vpshldq $1, %xmm3, %xmm2, %xmm3
+; ZNVER4-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; ZNVER4-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[2,3,2,3,6,7,6,7]
+; ZNVER4-NEXT:    vpshldq $1, %ymm1, %ymm2, %ymm1
+; ZNVER4-NEXT:    vinserti128 $1, %xmm3, %ymm4, %ymm3
+; ZNVER4-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
+; ZNVER4-NEXT:    vpshufd {{.*#+}} zmm3 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
+; ZNVER4-NEXT:    vpshldq $1, %zmm0, %zmm3, %zmm0
+; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
 ; ZNVER4-NEXT:    retq
   %d = bitcast <8 x i64> %a to i512
   %s = shl i512 %d, 1
@@ -142,65 +116,21 @@ define <8 x i64> @lshr_i512_1(<8 x i64> %a)  {
 ;
 ; ZNVER4-LABEL: lshr_i512_1:
 ; ZNVER4:       # %bb.0:
-; ZNVER4-NEXT:    pushq %rbx
-; ZNVER4-NEXT:    .cfi_def_cfa_offset 16
-; ZNVER4-NEXT:    .cfi_offset %rbx, -16
+; ZNVER4-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; ZNVER4-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; ZNVER4-NEXT:    vmovq %xmm0, %r10
-; ZNVER4-NEXT:    vpextrq $1, %xmm0, %rsi
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %rcx
-; ZNVER4-NEXT:    vmovq %xmm1, %r9
-; ZNVER4-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
-; ZNVER4-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; ZNVER4-NEXT:    shrq %r10
-; ZNVER4-NEXT:    vpextrq $1, %xmm0, %rax
-; ZNVER4-NEXT:    vmovq %xmm0, %rdx
-; ZNVER4-NEXT:    vmovq %xmm1, %rdi
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %r11
-; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; ZNVER4-NEXT:    movq %rdx, %r8
-; ZNVER4-NEXT:    shrq %r8
-; ZNVER4-NEXT:    shlq $63, %rax
-; ZNVER4-NEXT:    movq %rdi, %rbx
-; ZNVER4-NEXT:    shrq %rbx
-; ZNVER4-NEXT:    shlq $63, %rdx
-; ZNVER4-NEXT:    shlq $63, %rdi
-; ZNVER4-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; ZNVER4-NEXT:    orq %r8, %rax
-; ZNVER4-NEXT:    movq %r11, %r8
-; ZNVER4-NEXT:    shlq $63, %r8
-; ZNVER4-NEXT:    shrq %r11
-; ZNVER4-NEXT:    orq %rbx, %r8
-; ZNVER4-NEXT:    movq %r9, %rbx
-; ZNVER4-NEXT:    orq %r11, %rdx
-; ZNVER4-NEXT:    movq %rsi, %r11
-; ZNVER4-NEXT:    shrq %r11
-; ZNVER4-NEXT:    shlq $63, %rbx
-; ZNVER4-NEXT:    shrq %r9
-; ZNVER4-NEXT:    shlq $63, %rsi
-; ZNVER4-NEXT:    vmovq %rax, %xmm4
-; ZNVER4-NEXT:    orq %r11, %rbx
-; ZNVER4-NEXT:    movq %rcx, %r11
-; ZNVER4-NEXT:    shlq $63, %r11
-; ZNVER4-NEXT:    shrq %rcx
-; ZNVER4-NEXT:    orq %r10, %rsi
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm0[0]
-; ZNVER4-NEXT:    orq %r9, %r11
-; ZNVER4-NEXT:    orq %rdi, %rcx
-; ZNVER4-NEXT:    vmovq %rbx, %xmm3
-; ZNVER4-NEXT:    vmovq %rcx, %xmm1
-; ZNVER4-NEXT:    vmovq %r11, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; ZNVER4-NEXT:    vmovq %rsi, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; ZNVER4-NEXT:    vmovq %r8, %xmm3
-; ZNVER4-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
-; ZNVER4-NEXT:    vmovq %rdx, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; ZNVER4-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
-; ZNVER4-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; ZNVER4-NEXT:    popq %rbx
-; ZNVER4-NEXT:    .cfi_def_cfa_offset 8
+; ZNVER4-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; ZNVER4-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; ZNVER4-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[2,3,2,3,6,7,6,7]
+; ZNVER4-NEXT:    vpshldq $63, %xmm4, %xmm2, %xmm4
+; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; ZNVER4-NEXT:    vpshldq $63, %ymm3, %ymm1, %ymm1
+; ZNVER4-NEXT:    vpshufd {{.*#+}} zmm3 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
+; ZNVER4-NEXT:    vpsrlq $1, %xmm2, %xmm2
+; ZNVER4-NEXT:    vpshldq $63, %zmm0, %zmm3, %zmm0
+; ZNVER4-NEXT:    vinserti128 $1, %xmm2, %ymm4, %ymm2
+; ZNVER4-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
 ; ZNVER4-NEXT:    retq
   %d = bitcast <8 x i64> %a to i512
   %s = lshr i512 %d, 1
@@ -255,65 +185,21 @@ define <8 x i64> @ashr_i512_1(<8 x i64> %a)  {
 ;
 ; ZNVER4-LABEL: ashr_i512_1:
 ; ZNVER4:       # %bb.0:
-; ZNVER4-NEXT:    pushq %rbx
-; ZNVER4-NEXT:    .cfi_def_cfa_offset 16
-; ZNVER4-NEXT:    .cfi_offset %rbx, -16
+; ZNVER4-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; ZNVER4-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; ZNVER4-NEXT:    vmovq %xmm0, %r10
-; ZNVER4-NEXT:    vpextrq $1, %xmm0, %rsi
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %rcx
-; ZNVER4-NEXT:    vmovq %xmm1, %r9
-; ZNVER4-NEXT:    vextracti32x4 $2, %zmm0, %xmm1
-; ZNVER4-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; ZNVER4-NEXT:    shrq %r10
-; ZNVER4-NEXT:    vpextrq $1, %xmm0, %rax
-; ZNVER4-NEXT:    vmovq %xmm0, %rdx
-; ZNVER4-NEXT:    vmovq %xmm1, %rdi
-; ZNVER4-NEXT:    vpextrq $1, %xmm1, %r11
-; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; ZNVER4-NEXT:    movq %rdx, %r8
-; ZNVER4-NEXT:    shrq %r8
-; ZNVER4-NEXT:    shlq $63, %rax
-; ZNVER4-NEXT:    movq %rdi, %rbx
-; ZNVER4-NEXT:    shrq %rbx
-; ZNVER4-NEXT:    shlq $63, %rdx
-; ZNVER4-NEXT:    shlq $63, %rdi
-; ZNVER4-NEXT:    vpsraq $1, %xmm0, %xmm0
-; ZNVER4-NEXT:    orq %r8, %rax
-; ZNVER4-NEXT:    movq %r11, %r8
-; ZNVER4-NEXT:    shlq $63, %r8
-; ZNVER4-NEXT:    shrq %r11
-; ZNVER4-NEXT:    orq %rbx, %r8
-; ZNVER4-NEXT:    movq %r9, %rbx
-; ZNVER4-NEXT:    orq %r11, %rdx
-; ZNVER4-NEXT:    movq %rsi, %r11
-; ZNVER4-NEXT:    shrq %r11
-; ZNVER4-NEXT:    shlq $63, %rbx
-; ZNVER4-NEXT:    shrq %r9
-; ZNVER4-NEXT:    shlq $63, %rsi
-; ZNVER4-NEXT:    vmovq %rax, %xmm4
-; ZNVER4-NEXT:    orq %r11, %rbx
-; ZNVER4-NEXT:    movq %rcx, %r11
-; ZNVER4-NEXT:    shlq $63, %r11
-; ZNVER4-NEXT:    shrq %rcx
-; ZNVER4-NEXT:    orq %r10, %rsi
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm4[0],xmm0[0]
-; ZNVER4-NEXT:    orq %r9, %r11
-; ZNVER4-NEXT:    orq %rdi, %rcx
-; ZNVER4-NEXT:    vmovq %rbx, %xmm3
-; ZNVER4-NEXT:    vmovq %rcx, %xmm1
-; ZNVER4-NEXT:    vmovq %r11, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; ZNVER4-NEXT:    vmovq %rsi, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; ZNVER4-NEXT:    vmovq %r8, %xmm3
-; ZNVER4-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
-; ZNVER4-NEXT:    vmovq %rdx, %xmm2
-; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; ZNVER4-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
-; ZNVER4-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; ZNVER4-NEXT:    popq %rbx
-; ZNVER4-NEXT:    .cfi_def_cfa_offset 8
+; ZNVER4-NEXT:    vextracti32x4 $3, %zmm0, %xmm2
+; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; ZNVER4-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
+; ZNVER4-NEXT:    vpshufd {{.*#+}} ymm3 = ymm0[2,3,2,3,6,7,6,7]
+; ZNVER4-NEXT:    vpshldq $63, %xmm4, %xmm2, %xmm4
+; ZNVER4-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; ZNVER4-NEXT:    vpshldq $63, %ymm3, %ymm1, %ymm1
+; ZNVER4-NEXT:    vpshufd {{.*#+}} zmm3 = zmm0[2,3,2,3,6,7,6,7,10,11,10,11,14,15,14,15]
+; ZNVER4-NEXT:    vpsraq $1, %xmm2, %xmm2
+; ZNVER4-NEXT:    vpshldq $63, %zmm0, %zmm3, %zmm0
+; ZNVER4-NEXT:    vinserti128 $1, %xmm2, %ymm4, %ymm2
+; ZNVER4-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; ZNVER4-NEXT:    vpunpcklqdq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
 ; ZNVER4-NEXT:    retq
   %d = bitcast <8 x i64> %a to i512
   %s = ashr i512 %d, 1
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
index 2c7da100344b..42617c1573be 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath-mir.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2,fma -stop-after=finalize-isel 2>&1 | FileCheck %s
+; RUN: llc -fp-contract=fast < %s -mtriple=x86_64-unknown-unknown -mattr=avx2,fma -stop-after=finalize-isel 2>&1 | FileCheck %s
 
 declare float @llvm.sqrt.f32(float) #2
 
@@ -35,15 +35,15 @@ define float @sqrt_ieee_ninf(float %f) #0 {
   ; CHECK-NEXT:   [[VFMADD213SSr1:%[0-9]+]]:fr32 = ninf afn nofpexcept VFMADD213SSr [[VMULSSrr2]], [[VMULSSrr3]], [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr4:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr [[VMULSSrr3]], [[VMOVSSrm_alt1]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr5:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr killed [[VMULSSrr4]], killed [[VFMADD213SSr1]], implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[VMULSSrr5]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY killed [[VMULSSrr5]]
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY [[COPY]]
   ; CHECK-NEXT:   [[VPBROADCASTDrm:%[0-9]+]]:vr128 = VPBROADCASTDrm $rip, 1, $noreg, %const.2, $noreg :: (load (s32) from constant-pool)
   ; CHECK-NEXT:   [[VPANDrr:%[0-9]+]]:vr128 = VPANDrr killed [[COPY2]], killed [[VPBROADCASTDrm]]
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY [[VPANDrr]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY killed [[VPANDrr]]
   ; CHECK-NEXT:   [[VCMPSSrmi:%[0-9]+]]:fr32 = nofpexcept VCMPSSrmi killed [[COPY3]], $rip, 1, $noreg, %const.3, $noreg, 1, implicit $mxcsr :: (load (s32) from constant-pool)
-  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vr128 = COPY [[VCMPSSrmi]]
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vr128 = COPY killed [[VCMPSSrmi]]
   ; CHECK-NEXT:   [[VPANDNrr:%[0-9]+]]:vr128 = VPANDNrr killed [[COPY4]], killed [[COPY1]]
-  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:fr32 = COPY killed [[VPANDNrr]]
   ; CHECK-NEXT:   $xmm0 = COPY [[COPY5]]
   ; CHECK-NEXT:   RET 0, $xmm0
   %call = tail call ninf afn float @llvm.sqrt.f32(float %f)
@@ -82,12 +82,12 @@ define float @sqrt_daz_ninf(float %f) #1 {
   ; CHECK-NEXT:   [[VFMADD213SSr1:%[0-9]+]]:fr32 = ninf afn nofpexcept VFMADD213SSr [[VMULSSrr2]], [[VMULSSrr3]], [[VMOVSSrm_alt]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr4:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr [[VMULSSrr3]], [[VMOVSSrm_alt1]], implicit $mxcsr
   ; CHECK-NEXT:   [[VMULSSrr5:%[0-9]+]]:fr32 = ninf afn nofpexcept VMULSSrr killed [[VMULSSrr4]], killed [[VFMADD213SSr1]], implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[VMULSSrr5]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY killed [[VMULSSrr5]]
   ; CHECK-NEXT:   [[FsFLD0SS:%[0-9]+]]:fr32 = FsFLD0SS
   ; CHECK-NEXT:   [[VCMPSSrri:%[0-9]+]]:fr32 = nofpexcept VCMPSSrri [[COPY]], killed [[FsFLD0SS]], 0, implicit $mxcsr
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY [[VCMPSSrri]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vr128 = COPY killed [[VCMPSSrri]]
   ; CHECK-NEXT:   [[VPANDNrr:%[0-9]+]]:vr128 = VPANDNrr killed [[COPY2]], killed [[COPY1]]
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY [[VPANDNrr]]
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:fr32 = COPY killed [[VPANDNrr]]
   ; CHECK-NEXT:   $xmm0 = COPY [[COPY3]]
   ; CHECK-NEXT:   RET 0, $xmm0
   %call = tail call ninf afn float @llvm.sqrt.f32(float %f)
@@ -144,6 +144,6 @@ define float @rsqrt_daz(float %f) #1 {
   ret float %div
 }
 
-attributes #0 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt:2" "denormal-fp-math"="ieee,ieee" }
-attributes #1 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt:2" "denormal-fp-math"="ieee,preserve-sign" }
+attributes #0 = { "reciprocal-estimates"="sqrt:2" "denormal-fp-math"="ieee,ieee" }
+attributes #1 = { "reciprocal-estimates"="sqrt:2" "denormal-fp-math"="ieee,preserve-sign" }
 attributes #2 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index 9f420bcede11..5cd604c62a16 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -183,7 +183,7 @@ define <4 x float> @sqrt_v4f32_check_denorms(<4 x float> %x) #3 {
   ret <4 x float> %call
 }
 
-define <4 x float> @sqrt_v4f32_check_denorms_ieee_ninf(<4 x float> %x) #3 {
+define <4 x float> @sqrt_v4f32_check_denorms_ieee_ninf(<4 x float> %x) #7 {
 ; SSE-LABEL: sqrt_v4f32_check_denorms_ieee_ninf:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtps %xmm0, %xmm1
@@ -230,11 +230,11 @@ define <4 x float> @sqrt_v4f32_check_denorms_ieee_ninf(<4 x float> %x) #3 {
 ; AVX512-NEXT:    vcmpleps %xmm0, %xmm2, %xmm0
 ; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
+  %call = tail call fast ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
   ret <4 x float> %call
 }
 
-define <4 x float> @sqrt_v4f32_check_denorms_dynamic_ninf(<4 x float> %x) #6 {
+define <4 x float> @sqrt_v4f32_check_denorms_dynamic_ninf(<4 x float> %x) #8 {
 ; SSE-LABEL: sqrt_v4f32_check_denorms_dynamic_ninf:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    rsqrtps %xmm0, %xmm1
@@ -281,7 +281,7 @@ define <4 x float> @sqrt_v4f32_check_denorms_dynamic_ninf(<4 x float> %x) #6 {
 ; AVX512-NEXT:    vcmpleps %xmm0, %xmm2, %xmm0
 ; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
-  %call = tail call ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
+  %call = tail call fast ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %x) #2
   ret <4 x float> %call
 }
 
@@ -1019,3 +1019,8 @@ attributes #3 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt"
 attributes #4 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="ieee,preserve-sign" }
 attributes #5 = { "unsafe-fp-math"="true" "reciprocal-estimates"="all:0" }
 attributes #6 = { "unsafe-fp-math"="true" "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,dynamic" }
+
+; Attributes without "unsafe-fp-math"="true"
+; TODO: Merge with previous attributes when this attribute can be deleted.
+attributes #7 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,ieee" } ; #3
+attributes #8 = { "reciprocal-estimates"="sqrt,vec-sqrt" "denormal-fp-math"="preserve-sign,dynamic" } ; #6
diff --git a/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll b/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
index 391963de7970..8d589c519eff 100644
--- a/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
+++ b/llvm/test/CodeGen/X86/statepoint-cmp-sunk-past-statepoint.ll
@@ -1,5 +1,6 @@
-; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction < %s | FileCheck --check-prefixes CHECK,CHECK-LV %s
-; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction -early-live-intervals < %s | FileCheck --check-prefixes CHECK,CHECK-LIS %s
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction < %s | FileCheck --check-prefix=CHECK-LV %s
+; RUN: llc -max-registers-for-gc-values=256 -verify-machineinstrs -stop-after twoaddressinstruction -early-live-intervals < %s | FileCheck --check-prefix=CHECK-LIS %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2"
 target triple = "x86_64-unknown-linux-gnu"
@@ -10,26 +11,54 @@ declare void @bar(ptr addrspace(1)) gc "statepoint-example"
 declare ptr @fake_personality_function()
 
 ; Simplest possible test demonstrating the problem
-
-; CHECK-LABEL: name: test
-; CHECK:  bb.0
-; CHECK-LV:     %0:gr64 = COPY killed $rdi
-; CHECK-LIS:    %0:gr64 = COPY $rdi
-; CHECK:        %1:gr64 = COPY %0
-; CHECK:        %1:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %1(tied-def 0), 2, 0, 2, 1, 0, 0
-; CHECK-LV:     TEST64rr killed %0, %0, implicit-def $eflags
-; CHECK-LIS:    TEST64rr %0, %0, implicit-def $eflags
-; CHECK:        JCC_1 %bb.2, 4, implicit killed $eflags
-; CHECK:        JMP_1 %bb.1
-; CHECK:      bb.1
-; CHECK-LV:     $rdi = COPY killed %1
-; CHECK-LV:     STATEPOINT 2, 5, 1, undef %3:gr64, killed $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0
-; CHECK-LIS:    $rdi = COPY %1
-; CHECK-LIS:    STATEPOINT 2, 5, 1, undef %3:gr64, $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0
-; CHECK:        RET 0
-; CHECK:      bb.2
-; CHECK:        RET 0
 define void @test(ptr addrspace(1) %a)  gc "statepoint-example" {
+  ; CHECK-LV-LABEL: name: test
+  ; CHECK-LV: bb.0.entry:
+  ; CHECK-LV-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-LV-NEXT:   liveins: $rdi
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY killed $rdi
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY1]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY]], [[COPY]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.2, 4, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.1
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.1.not_zero:
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   $rdi = COPY killed [[COPY1]]
+  ; CHECK-LV-NEXT:   STATEPOINT 2, 5, 1, undef %3:gr64, killed $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   RET 0
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.2.zero:
+  ; CHECK-LV-NEXT:   RET 0
+  ;
+  ; CHECK-LIS-LABEL: name: test
+  ; CHECK-LIS: bb.0.entry:
+  ; CHECK-LIS-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-LIS-NEXT:   liveins: $rdi
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = STATEPOINT 2, 5, 0, undef %2:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY1]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   TEST64rr [[COPY]], [[COPY]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.2, 4, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.1
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.1.not_zero:
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   $rdi = COPY [[COPY1]]
+  ; CHECK-LIS-NEXT:   STATEPOINT 2, 5, 1, undef %3:gr64, $rdi, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, 2, 0, csr_64, implicit-def $rsp, implicit-def $ssp
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   RET 0
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.2.zero:
+  ; CHECK-LIS-NEXT:   RET 0
 entry:
   %not7 = icmp eq ptr addrspace(1) %a, null
   %statepoint_token1745 = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 2, i32 5, ptr nonnull elementtype(void ()) @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(), "gc-live"(ptr addrspace(1) %a) ]
@@ -45,28 +74,138 @@ zero:
 }
 
 ; A bit more complex test, where both registers are used in same successor BB
-
-; CHECK-LABEL: name: test2
-; CHECK:  bb.2
-; CHECK:        %1:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %1(tied-def 0), 2, 0, 2, 1, 0, 0, csr_64
-; CHECK:        %10:gr64 = COPY %1
-; CHECK:        %10:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, %10(tied-def 0), 2, 0, 2, 1, 0, 0, csr_64
-; CHECK:        JMP_1 %bb.3
-; CHECK:      bb.3
-; CHECK:        %18:gr8 = COPY %17.sub_8bit
-; CHECK-LV:     TEST8rr killed %18, %18, implicit-def $eflags
-; CHECK-LIS:    TEST8rr %18, %18, implicit-def $eflags
-; CHECK:        JCC_1 %bb.5, 5, implicit killed $eflags
-; CHECK:        JMP_1 %bb.4
-; CHECK:      bb.4
-; CHECK:      bb.5
-; CHECK:        %3:gr64 = COPY %10
-; CHECK:        %4:gr64 = COPY killed %10
-; CHECK:        %4:gr64 = nuw ADD64ri32 %4, 8, implicit-def dead $eflags
-; CHECK:        TEST64rr killed %1, %1, implicit-def $eflags
-; CHECK:        JCC_1 %bb.1, 5, implicit killed $eflags
-; CHECK:        JMP_1 %bb.6
 define void @test2(ptr addrspace(1) %this, i32 %0, ptr addrspace(1) %p0, ptr addrspace(1) %p1) gc "statepoint-example" personality ptr @fake_personality_function {
+  ; CHECK-LV-LABEL: name: test2
+  ; CHECK-LV: bb.0.preheader:
+  ; CHECK-LV-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-LV-NEXT:   liveins: $rdx, $rcx
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY killed $rcx
+  ; CHECK-LV-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY killed $rdx
+  ; CHECK-LV-NEXT:   [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
+  ; CHECK-LV-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY killed [[COPY1]]
+  ; CHECK-LV-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY killed [[COPY]]
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.1.loop.head:
+  ; CHECK-LV-NEXT:   successors: %bb.6(0x04000000), %bb.2(0x7c000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY killed [[COPY3]]
+  ; CHECK-LV-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY killed [[COPY2]]
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY5]], [[COPY5]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.6, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.2
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.2.BB3:
+  ; CHECK-LV-NEXT:   successors: %bb.3(0x7ffff800), %bb.7(0x00000800)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY4:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY4]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-LV-NEXT:   [[COPY6:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY6]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LV-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   JMP_1 %bb.3
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.3.BB2:
+  ; CHECK-LV-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY7:%[0-9]+]]:gr8 = COPY [[MOV32r0_]].sub_8bit
+  ; CHECK-LV-NEXT:   TEST8rr killed [[COPY7]], [[COPY7]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   JCC_1 %bb.5, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.4
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.4.BB4:
+  ; CHECK-LV-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.5.tail:
+  ; CHECK-LV-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   [[COPY8:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-LV-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY killed [[COPY6]]
+  ; CHECK-LV-NEXT:   [[COPY9:%[0-9]+]]:gr64 = nuw ADD64ri32 [[COPY9]], 8, implicit-def dead $eflags
+  ; CHECK-LV-NEXT:   TEST64rr killed [[COPY4]], [[COPY4]], implicit-def $eflags
+  ; CHECK-LV-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY killed [[COPY9]]
+  ; CHECK-LV-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY killed [[COPY8]]
+  ; CHECK-LV-NEXT:   JCC_1 %bb.1, 5, implicit killed $eflags
+  ; CHECK-LV-NEXT:   JMP_1 %bb.6
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.6.BB1:
+  ; CHECK-LV-NEXT:   RET 0
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT: bb.7.BB6 (landing-pad):
+  ; CHECK-LV-NEXT:   liveins: $rax, $rdx
+  ; CHECK-LV-NEXT: {{  $}}
+  ; CHECK-LV-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LV-NEXT:   RET 0
+  ;
+  ; CHECK-LIS-LABEL: name: test2
+  ; CHECK-LIS: bb.0.preheader:
+  ; CHECK-LIS-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-LIS-NEXT:   liveins: $rdx, $rcx
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY:%[0-9]+]]:gr64 = COPY $rcx
+  ; CHECK-LIS-NEXT:   [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+  ; CHECK-LIS-NEXT:   [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags
+  ; CHECK-LIS-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY1]]
+  ; CHECK-LIS-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY]]
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.1.loop.head:
+  ; CHECK-LIS-NEXT:   successors: %bb.6(0x04000000), %bb.2(0x7c000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY4:%[0-9]+]]:gr64 = COPY [[COPY3]]
+  ; CHECK-LIS-NEXT:   [[COPY5:%[0-9]+]]:gr64 = COPY [[COPY2]]
+  ; CHECK-LIS-NEXT:   TEST64rr [[COPY5]], [[COPY5]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.6, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.2
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.2.BB3:
+  ; CHECK-LIS-NEXT:   successors: %bb.3(0x7ffff800), %bb.7(0x00000800)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY4:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %11:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY4]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   [[COPY6:%[0-9]+]]:gr64 = COPY [[COPY4]]
+  ; CHECK-LIS-NEXT:   [[COPY6:%[0-9]+]]:gr64 = STATEPOINT 2882400000, 0, 0, undef %13:gr64, 2, 0, 2, 0, 2, 0, 2, 1, [[COPY6]](tied-def 0), 2, 0, 2, 1, 0, 0, csr_64, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax
+  ; CHECK-LIS-NEXT:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.3
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.3.BB2:
+  ; CHECK-LIS-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY7:%[0-9]+]]:gr8 = COPY [[MOV32r0_]].sub_8bit
+  ; CHECK-LIS-NEXT:   TEST8rr [[COPY7]], [[COPY7]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.5, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.4
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.4.BB4:
+  ; CHECK-LIS-NEXT:   successors: %bb.5(0x80000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.5.tail:
+  ; CHECK-LIS-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   [[COPY8:%[0-9]+]]:gr64 = COPY [[COPY6]]
+  ; CHECK-LIS-NEXT:   [[COPY9:%[0-9]+]]:gr64 = COPY killed [[COPY6]]
+  ; CHECK-LIS-NEXT:   [[COPY9:%[0-9]+]]:gr64 = nuw ADD64ri32 [[COPY9]], 8, implicit-def dead $eflags
+  ; CHECK-LIS-NEXT:   TEST64rr killed [[COPY4]], [[COPY4]], implicit-def $eflags
+  ; CHECK-LIS-NEXT:   [[COPY2:%[0-9]+]]:gr64 = COPY [[COPY9]]
+  ; CHECK-LIS-NEXT:   [[COPY3:%[0-9]+]]:gr64 = COPY [[COPY8]]
+  ; CHECK-LIS-NEXT:   JCC_1 %bb.1, 5, implicit killed $eflags
+  ; CHECK-LIS-NEXT:   JMP_1 %bb.6
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.6.BB1:
+  ; CHECK-LIS-NEXT:   RET 0
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT: bb.7.BB6 (landing-pad):
+  ; CHECK-LIS-NEXT:   liveins: $rax, $rdx
+  ; CHECK-LIS-NEXT: {{  $}}
+  ; CHECK-LIS-NEXT:   EH_LABEL <mcsymbol >
+  ; CHECK-LIS-NEXT:   RET 0
 preheader:
   br label %loop.head
 
diff --git a/llvm/test/CodeGen/X86/unpredictable-brcond.ll b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
index 12411f1c49f2..bb44171d20d6 100644
--- a/llvm/test/CodeGen/X86/unpredictable-brcond.ll
+++ b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
@@ -87,7 +87,7 @@ define void @isint_branch(double %d) nounwind {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr128 = COPY [[COPY]]
   ; CHECK-NEXT:   [[CVTTPD2DQrr:%[0-9]+]]:vr128 = nofpexcept CVTTPD2DQrr killed [[COPY1]], implicit $mxcsr
   ; CHECK-NEXT:   [[CVTDQ2PDrr:%[0-9]+]]:vr128 = CVTDQ2PDrr killed [[CVTTPD2DQrr]]
-  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY [[CVTDQ2PDrr]]
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY killed [[CVTDQ2PDrr]]
   ; CHECK-NEXT:   nofpexcept UCOMISDrr [[COPY]], killed [[COPY2]], implicit-def $eflags, implicit $mxcsr
   ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
   ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 10, implicit $eflags
diff --git a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
index 36094fe56d57..84856aab8507 100644
--- a/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
+++ b/llvm/test/CodeGen/X86/urem-seteq-vec-tautological.ll
@@ -77,11 +77,9 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE2-LABEL: t1_all_odd_ne:
 ; CHECK-SSE2:       # %bb.0:
 ; CHECK-SSE2-NEXT:    pmuludq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; CHECK-SSE2-NEXT:    pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; CHECK-SSE2-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE2-NEXT:    retq
 ;
 ; CHECK-SSE41-LABEL: t1_all_odd_ne:
@@ -92,7 +90,7 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
 ; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
 ; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-SSE41-NEXT:    por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
 ; CHECK-SSE41-NEXT:    retq
 ;
 ; CHECK-AVX1-LABEL: t1_all_odd_ne:
@@ -102,7 +100,7 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; CHECK-AVX1-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-AVX1-NEXT:    retq
 ;
 ; CHECK-AVX2-LABEL: t1_all_odd_ne:
@@ -113,17 +111,16 @@ define <4 x i1> @t1_all_odd_ne(<4 x i32> %X) nounwind {
 ; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX2-NEXT:    vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
 ; CHECK-AVX2-NEXT:    retq
 ;
 ; CHECK-AVX512VL-LABEL: t1_all_odd_ne:
 ; CHECK-AVX512VL:       # %bb.0:
 ; CHECK-AVX512VL-NEXT:    vpmulld {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
 ; CHECK-AVX512VL-NEXT:    vpminud {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = ~xmm0
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1
+; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; CHECK-AVX512VL-NEXT:    vpternlogq {{.*#+}} xmm0 = m64bcst | (xmm0 ^ xmm1)
 ; CHECK-AVX512VL-NEXT:    retq
   %urem = urem <4 x i32> %X, <i32 3, i32 1, i32 1, i32 9>
   %cmp = icmp ne <4 x i32> %urem, <i32 0, i32 42, i32 0, i32 42>
diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll
index 7296cc27894c..283c6a303a58 100644
--- a/llvm/test/CodeGen/X86/var-permute-256.ll
+++ b/llvm/test/CodeGen/X86/var-permute-256.ll
@@ -2,12 +2,12 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop | FileCheck %s --check-prefix=XOP
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=INT256,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=INT256,AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq | FileCheck %s --check-prefixes=INT256,AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=INT256,AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi | FileCheck %s --check-prefixes=INT256,AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=INT256,AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=INT256,AVX512VL,AVX512VLDQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=INT256,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq | FileCheck %s --check-prefixes=INT256,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=INT256,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi | FileCheck %s --check-prefixes=INT256,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=INT256,AVX512VL,AVX512VLF
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=INT256,AVX512VL,AVX512VLF
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=INT256,AVX512VL,AVX512VLBW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+avx512vbmi | FileCheck %s --check-prefixes=INT256,AVX512VL,VLVBMI
 
@@ -393,17 +393,17 @@ define <16 x i16> @var_shuffle_v16i16(<16 x i16> %v, <16 x i16> %indices) nounwi
 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_v16i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
-; AVX512VLDQ-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
-; AVX512VLDQ-NEXT:    retq
+; AVX512VLF-LABEL: var_shuffle_v16i16:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512VLF-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
+; AVX512VLF-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v16i16:
 ; AVX512VLBW:       # %bb.0:
@@ -533,21 +533,57 @@ define <16 x i16> @var_shuffle_zero_v16i16(<16 x i16> %v, <16 x i16> %indices) n
 ; AVX2-NEXT:    vpandn %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_zero_v16i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX512VLDQ-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm2
-; AVX512VLDQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512VLDQ-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
-; AVX512VLDQ-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
-; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
-; AVX512VLDQ-NEXT:    vpandn %ymm1, %ymm2, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; AVX512F-LABEL: var_shuffle_zero_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512F-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512F-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpandn %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shuffle_zero_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512BW-NEXT:    vpcmpnleuw %zmm2, %zmm1, %k1
+; AVX512BW-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX512BW-NEXT:    vmovdqu16 %zmm2, %zmm1 {%k1}
+; AVX512BW-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512BW-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
+; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLF-LABEL: var_shuffle_zero_v16i16:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    vpmaxuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512VLF-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm2
+; AVX512VLF-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VLF-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512VLF-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512VLF-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
+; AVX512VLF-NEXT:    vpandn %ymm1, %ymm2, %ymm0
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_zero_v16i16:
 ; AVX512VLBW:       # %bb.0:
@@ -668,15 +704,15 @@ define <32 x i8> @var_shuffle_v32i8(<32 x i8> %v, <32 x i8> %indices) nounwind {
 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_v32i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm2 ^ ymm3))
-; AVX512VLDQ-NEXT:    retq
+; AVX512VLF-LABEL: var_shuffle_v32i8:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm2 ^ ymm3))
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v32i8:
 ; AVX512VLBW:       # %bb.0:
@@ -847,19 +883,51 @@ define <32 x i8> @var_shuffle_zero_v32i8(<32 x i8> %v, <32 x i8> %indices) nounw
 ; AVX2-NEXT:    vpandn %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_zero_v32i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX512VLDQ-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm2
-; AVX512VLDQ-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
-; AVX512VLDQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
-; AVX512VLDQ-NEXT:    vpandn %ymm1, %ymm2, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; AVX512F-LABEL: var_shuffle_zero_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512F-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm2
+; AVX512F-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpandn %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: var_shuffle_zero_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vpbroadcastb {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512BW-NEXT:    vpcmpnleub %zmm2, %zmm1, %k1
+; AVX512BW-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX512BW-NEXT:    vmovdqu8 %zmm2, %zmm1 {%k1}
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,2,3]
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm2, %ymm2
+; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLF-LABEL: var_shuffle_zero_v32i8:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    vpmaxub {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512VLF-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm2
+; AVX512VLF-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
+; AVX512VLF-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3]
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm1 = ymm3 ^ (ymm1 & (ymm0 ^ ymm3))
+; AVX512VLF-NEXT:    vpandn %ymm1, %ymm2, %ymm0
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_zero_v32i8:
 ; AVX512VLBW:       # %bb.0:
@@ -1493,17 +1561,17 @@ define <16 x i16> @var_shuffle_v16i16_from_v8i16(<8 x i16> %v, <16 x i16> %indic
 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_v16i16_from_v8i16:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512VLDQ-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
-; AVX512VLDQ-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
-; AVX512VLDQ-NEXT:    retq
+; AVX512VLF-LABEL: var_shuffle_v16i16_from_v8i16:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VLF-NEXT:    vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [514,514,514,514,514,514,514,514,514,514,514,514,514,514,514,514]
+; AVX512VLF-NEXT:    vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm2 ^ (ymm0 & (ymm3 ^ ymm2))
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v16i16_from_v8i16:
 ; AVX512VLBW:       # %bb.0:
@@ -1611,15 +1679,15 @@ define <32 x i8> @var_shuffle_v32i8_from_v16i8(<16 x i8> %v, <32 x i8> %indices)
 ; AVX512-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
 ;
-; AVX512VLDQ-LABEL: var_shuffle_v32i8_from_v16i8:
-; AVX512VLDQ:       # %bb.0:
-; AVX512VLDQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
-; AVX512VLDQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
-; AVX512VLDQ-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
-; AVX512VLDQ-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm2 ^ ymm3))
-; AVX512VLDQ-NEXT:    retq
+; AVX512VLF-LABEL: var_shuffle_v32i8_from_v16i8:
+; AVX512VLF:       # %bb.0:
+; AVX512VLF-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm2
+; AVX512VLF-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512VLF-NEXT:    vpshufb %ymm1, %ymm0, %ymm3
+; AVX512VLF-NEXT:    vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
+; AVX512VLF-NEXT:    vpternlogq {{.*#+}} ymm0 = ymm3 ^ (ymm0 & (ymm2 ^ ymm3))
+; AVX512VLF-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_shuffle_v32i8_from_v16i8:
 ; AVX512VLBW:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
index bde14e75dfc0..6aad9a6d82c7 100644
--- a/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
+++ b/llvm/test/CodeGen/X86/vec-strict-fptoint-128-fp16.ll
@@ -82,13 +82,11 @@ define <2 x i32> @strict_vector_fptosi_v2f16_to_v2i32(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i32:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; NOVL-NEXT:    vcvttph2dq %ymm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptosi.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -105,13 +103,11 @@ define <2 x i32> @strict_vector_fptoui_v2f16_to_v2i32(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i32:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; NOVL-NEXT:    vcvttph2udq %ymm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2usi %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2usi %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i32> @llvm.experimental.constrained.fptoui.v2i32.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -128,13 +124,12 @@ define <2 x i16> @strict_vector_fptosi_v2f16_to_v2i16(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptosi.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -151,13 +146,12 @@ define <2 x i16> @strict_vector_fptoui_v2f16_to_v2i16(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i16> @llvm.experimental.constrained.fptoui.v2i16.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -175,13 +169,13 @@ define <2 x i8> @strict_vector_fptosi_v2f16_to_v2i8(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v2f16_to_v2i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptosi.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -199,13 +193,13 @@ define <2 x i8> @strict_vector_fptoui_v2f16_to_v2i8(<2 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v2f16_to_v2i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vmovss {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm0
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <2 x i8> @llvm.experimental.constrained.fptoui.v2i8.v2f16(<2 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -335,12 +329,18 @@ define <4 x i16> @strict_vector_fptosi_v4f16_to_v4i16(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptosi.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -356,12 +356,18 @@ define <4 x i16> @strict_vector_fptoui_v4f16_to_v4i16(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i16:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i16> @llvm.experimental.constrained.fptoui.v4i16.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -378,12 +384,19 @@ define <4 x i8> @strict_vector_fptosi_v4f16_to_v4i8(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptosi_v4f16_to_v4i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptosi.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -400,12 +413,19 @@ define <4 x i8> @strict_vector_fptoui_v4f16_to_v4i8(<4 x half> %a) #0 {
 ;
 ; NOVL-LABEL: strict_vector_fptoui_v4f16_to_v4i8:
 ; NOVL:       # %bb.0:
-; NOVL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero
-; NOVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; NOVL-NEXT:    vinserti32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
-; NOVL-NEXT:    vzeroupper
+; NOVL-NEXT:    vpsrld $16, %xmm0, %xmm1
+; NOVL-NEXT:    vcvttsh2si %xmm1, %eax
+; NOVL-NEXT:    vcvttsh2si %xmm0, %ecx
+; NOVL-NEXT:    vmovd %ecx, %xmm1
+; NOVL-NEXT:    vpinsrd $1, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; NOVL-NEXT:    vcvttsh2si %xmm2, %eax
+; NOVL-NEXT:    vpinsrd $2, %eax, %xmm1, %xmm1
+; NOVL-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; NOVL-NEXT:    vcvttsh2si %xmm0, %eax
+; NOVL-NEXT:    vpinsrd $3, %eax, %xmm1, %xmm0
+; NOVL-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    retq
   %ret = call <4 x i8> @llvm.experimental.constrained.fptoui.v4i8.v4f16(<4 x half> %a,
                                               metadata !"fpexcept.strict") #0
@@ -554,7 +574,7 @@ define <8 x i8> @strict_vector_fptosi_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
 ; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptosi.v8i8.v8f16(<8 x half> %a,
@@ -573,8 +593,8 @@ define <8 x i8> @strict_vector_fptoui_v8f16_to_v8i8(<8 x half> %a) #0 {
 ; NOVL:       # %bb.0:
 ; NOVL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; NOVL-NEXT:    vinsertf32x4 $0, %xmm0, %zmm1, %zmm0
-; NOVL-NEXT:    vcvttph2uw %zmm0, %zmm0
-; NOVL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; NOVL-NEXT:    vcvttph2w %zmm0, %zmm0
+; NOVL-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
 ; NOVL-NEXT:    vzeroupper
 ; NOVL-NEXT:    retq
   %ret = call <8 x i8> @llvm.experimental.constrained.fptoui.v8i8.v8f16(<8 x half> %a,
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index 62ab5d82bfbb..910dd1ee6c41 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -2099,21 +2099,19 @@ define <4 x float> @uitofp_4i64_to_4f32_undef(<2 x i64> %a) {
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psrlq $1, %xmm2
 ; SSE41-NEXT:    por %xmm1, %xmm2
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm0
+; SSE41-NEXT:    pextrq $1, %xmm0, %rax
+; SSE41-NEXT:    cvtsi2ss %rax, %xmm3
+; SSE41-NEXT:    movq %xmm0, %rax
 ; SSE41-NEXT:    xorps %xmm2, %xmm2
 ; SSE41-NEXT:    cvtsi2ss %rax, %xmm2
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    xorps %xmm1, %xmm1
-; SSE41-NEXT:    cvtsi2ss %rax, %xmm1
-; SSE41-NEXT:    insertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],zero,zero
-; SSE41-NEXT:    movaps %xmm1, %xmm2
-; SSE41-NEXT:    addps %xmm1, %xmm2
-; SSE41-NEXT:    xorps %xmm3, %xmm3
-; SSE41-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,3],xmm3[2,3]
-; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE41-NEXT:    movaps %xmm1, %xmm0
+; SSE41-NEXT:    insertps {{.*#+}} xmm2 = xmm2[0],xmm3[0],zero,zero
+; SSE41-NEXT:    movaps %xmm2, %xmm3
+; SSE41-NEXT:    addps %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: uitofp_4i64_to_4f32_undef:
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index e88387a8b7c6..894186f9b343 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -138,22 +138,21 @@ define <2 x i64> @test_compress_v2i64(<2 x i64> %vec, <2 x i1> %mask, <2 x i64>
 ; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
 ; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
 ; AVX2-NEXT:    vmovq %xmm1, %rcx
+; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    movl %ecx, %edx
-; AVX2-NEXT:    subl %eax, %edx
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    subq %rax, %rcx
+; AVX2-NEXT:    movl %ecx, %eax
 ; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    cmpq $2, %rax
-; AVX2-NEXT:    cmovbq -24(%rsp,%rdx,8), %rsi
+; AVX2-NEXT:    cmpq $2, %rcx
+; AVX2-NEXT:    cmovbq -24(%rsp,%rax,8), %rsi
 ; AVX2-NEXT:    vmovq %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    movl %ecx, %ecx
-; AVX2-NEXT:    vpextrq $1, %xmm0, -24(%rsp,%rcx,8)
-; AVX2-NEXT:    cmpq $1, %rax
-; AVX2-NEXT:    movl $1, %ecx
-; AVX2-NEXT:    cmovbq %rax, %rcx
-; AVX2-NEXT:    movq %rsi, -24(%rsp,%rcx,8)
+; AVX2-NEXT:    vpextrq $1, %xmm0, -24(%rsp,%rdx,8)
+; AVX2-NEXT:    cmpq $1, %rcx
+; AVX2-NEXT:    movl $1, %eax
+; AVX2-NEXT:    cmovbq %rcx, %rax
+; AVX2-NEXT:    movl %eax, %eax
+; AVX2-NEXT:    movq %rsi, -24(%rsp,%rax,8)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    retq
 ;
@@ -188,18 +187,16 @@ define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x
 ; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vmovq %xmm1, %rcx
-; AVX2-NEXT:    movl %ecx, %edx
-; AVX2-NEXT:    subl %eax, %edx
-; AVX2-NEXT:    andl $1, %edx
+; AVX2-NEXT:    vpextrq $1, %xmm1, %rcx
+; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    andl $1, %eax
+; AVX2-NEXT:    movl %eax, %edx
+; AVX2-NEXT:    subq %rcx, %rax
+; AVX2-NEXT:    movl %eax, %ecx
+; AVX2-NEXT:    andl $1, %ecx
 ; AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX2-NEXT:    vmovlpd %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT:    andl $1, %ecx
-; AVX2-NEXT:    movl %ecx, %edx
 ; AVX2-NEXT:    vmovhpd %xmm0, -24(%rsp,%rdx,8)
-; AVX2-NEXT:    andl $1, %eax
-; AVX2-NEXT:    addq %rcx, %rax
 ; AVX2-NEXT:    cmpq $2, %rax
 ; AVX2-NEXT:    jb .LBB3_2
 ; AVX2-NEXT:  # %bb.1:
@@ -208,7 +205,8 @@ define <2 x double> @test_compress_v2f64(<2 x double> %vec, <2 x i1> %mask, <2 x
 ; AVX2-NEXT:    cmpq $1, %rax
 ; AVX2-NEXT:    movl $1, %ecx
 ; AVX2-NEXT:    cmovbq %rax, %rcx
-; AVX2-NEXT:    vmovsd %xmm1, -24(%rsp,%rcx,8)
+; AVX2-NEXT:    movl %ecx, %eax
+; AVX2-NEXT:    vmovsd %xmm1, -24(%rsp,%rax,8)
 ; AVX2-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
 ; AVX2-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
index 9c80720ae921..d77934adf4cd 100644
--- a/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
+++ b/llvm/test/CodeGen/X86/vector-constrained-fp-intrinsics-flags.ll
@@ -18,10 +18,10 @@ define <3 x float> @constrained_vector_fadd_v3f32() #0 {
 ; CHECK: [[ADDSSrr:%[0-9]+]]:fr32 = ADDSSrr [[MOVSSrm_alt]], killed [[FsFLD0SS]], implicit $mxcsr
 ; CHECK: [[ADDSSrm:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.1, $noreg, implicit $mxcsr :: (load (s32) from constant-pool)
 ; CHECK: [[ADDSSrm1:%[0-9]+]]:fr32 = ADDSSrm [[MOVSSrm_alt]], $rip, 1, $noreg, %const.2, $noreg, implicit $mxcsr :: (load (s32) from constant-pool)
-; CHECK: [[COPY:%[0-9]+]]:vr128 = COPY [[ADDSSrm1]]
-; CHECK: [[COPY1:%[0-9]+]]:vr128 = COPY [[ADDSSrm]]
+; CHECK: [[COPY:%[0-9]+]]:vr128 = COPY killed [[ADDSSrm1]]
+; CHECK: [[COPY1:%[0-9]+]]:vr128 = COPY killed [[ADDSSrm]]
 ; CHECK: [[UNPCKLPSrr:%[0-9]+]]:vr128 = UNPCKLPSrr [[COPY1]], killed [[COPY]]
-; CHECK: [[COPY2:%[0-9]+]]:vr128 = COPY [[ADDSSrr]]
+; CHECK: [[COPY2:%[0-9]+]]:vr128 = COPY killed [[ADDSSrr]]
 ; CHECK: [[UNPCKLPDrr:%[0-9]+]]:vr128 = UNPCKLPDrr [[UNPCKLPSrr]], killed [[COPY2]]
 ; CHECK: $xmm0 = COPY [[UNPCKLPDrr]]
 ; CHECK: RET 0, $xmm0
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
index ae4f85ce42a1..01aacc1e0625 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
@@ -1297,68 +1297,59 @@ define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-LABEL: load_i16_stride4_vf16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm0
-; AVX512-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX512-NEXT:    vpmovqw %ymm1, %xmm0
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm1
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm5[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $16, %ymm1, %ymm7
+; AVX512-NEXT:    vpmovqw %ymm7, %xmm7
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512-NEXT:    vpmovqw %zmm3, %xmm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm6
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm7
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm4, %zmm6
+; AVX512-NEXT:    vpmovqw %zmm6, %xmm6
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $32, %ymm1, %ymm6
+; AVX512-NEXT:    vpmovqw %ymm6, %xmm6
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm4, %zmm7
+; AVX512-NEXT:    vpmovqw %zmm7, %xmm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $48, %ymm1, %ymm1
+; AVX512-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm3, %zmm5
-; AVX512-NEXT:    vpmovqw %zmm5, %xmm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm3, %zmm8
-; AVX512-NEXT:    vpmovqw %zmm8, %xmm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpsrlq $48, %zmm3, %zmm3
-; AVX512-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT:    vpsrlq $48, %zmm4, %zmm2
+; AVX512-NEXT:    vpmovqw %zmm2, %xmm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa %ymm0, (%rsi)
-; AVX512-NEXT:    vmovdqa %ymm2, (%rdx)
-; AVX512-NEXT:    vmovdqa %ymm5, (%rcx)
+; AVX512-NEXT:    vmovdqa %ymm3, (%rdx)
+; AVX512-NEXT:    vmovdqa %ymm6, (%rcx)
 ; AVX512-NEXT:    vmovdqa %ymm1, (%r8)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1411,68 +1402,59 @@ define void @load_i16_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-LABEL: load_i16_stride4_vf16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX512DQ-NEXT:    vpmovqw %ymm1, %xmm0
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm5[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm1, %ymm7
+; AVX512DQ-NEXT:    vpmovqw %ymm7, %xmm7
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm6
-; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm4, %zmm6
+; AVX512DQ-NEXT:    vpmovqw %zmm6, %xmm6
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm1, %ymm6
+; AVX512DQ-NEXT:    vpmovqw %ymm6, %xmm6
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm2[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm5[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm4, %zmm7
+; AVX512DQ-NEXT:    vpmovqw %zmm7, %xmm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm6[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm3, %zmm5
-; AVX512DQ-NEXT:    vpmovqw %zmm5, %xmm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm1[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm3, %zmm8
-; AVX512DQ-NEXT:    vpmovqw %zmm8, %xmm8
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm3, %zmm3
-; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm4, %zmm2
+; AVX512DQ-NEXT:    vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rsi)
-; AVX512DQ-NEXT:    vmovdqa %ymm2, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa %ymm5, (%rcx)
+; AVX512DQ-NEXT:    vmovdqa %ymm3, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa %ymm6, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa %ymm1, (%r8)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -2713,128 +2695,110 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512-NEXT:    vmovdqa 192(%rdi), %ymm2
-; AVX512-NEXT:    vpmovqw %ymm2, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm6
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm7
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm7[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vpmovqw %ymm2, %xmm3
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vpmovqw %zmm1, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm5
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm7
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpmovqw %zmm1, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm3
-; AVX512-NEXT:    vpmovqw %ymm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm10
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm3
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm3[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX512-NEXT:    vpmovqw %zmm0, %xmm11
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm11
-; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm13
-; AVX512-NEXT:    vmovdqa 208(%rdi), %xmm14
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm13[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
+; AVX512-NEXT:    vpmovqw %ymm3, %xmm9
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm1, %zmm9
-; AVX512-NEXT:    vpmovqw %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm10
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm13
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]
+; AVX512-NEXT:    vpmovqw %zmm0, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $16, %ymm2, %ymm9
+; AVX512-NEXT:    vpmovqw %ymm9, %xmm9
+; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm12[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm1, %zmm8
+; AVX512-NEXT:    vpmovqw %zmm8, %xmm8
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $16, %ymm3, %ymm8
+; AVX512-NEXT:    vpmovqw %ymm8, %xmm8
 ; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm10[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
 ; AVX512-NEXT:    vpsrlq $16, %zmm0, %zmm9
 ; AVX512-NEXT:    vpmovqw %zmm9, %xmm9
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[0,1,2,3]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm6[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $32, %ymm2, %ymm8
+; AVX512-NEXT:    vpmovqw %ymm8, %xmm8
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm7[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm12
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm14[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm1, %zmm13
-; AVX512-NEXT:    vpmovqw %zmm13, %xmm13
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm11[2,0,2,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm1, %zmm9
+; AVX512-NEXT:    vpmovqw %zmm9, %xmm9
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $32, %ymm3, %ymm9
+; AVX512-NEXT:    vpmovqw %ymm9, %xmm9
+; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm10[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm0, %zmm14
-; AVX512-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm13[0,1,2,3],zmm12[0,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm0, %zmm11
+; AVX512-NEXT:    vpmovqw %zmm11, %xmm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $48, %ymm2, %ymm2
+; AVX512-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
 ; AVX512-NEXT:    vpsrlq $48, %zmm1, %zmm1
 ; AVX512-NEXT:    vpmovqw %zmm1, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $48, %ymm3, %ymm2
+; AVX512-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm11[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
 ; AVX512-NEXT:    vpsrlq $48, %zmm0, %zmm0
 ; AVX512-NEXT:    vpmovqw %zmm0, %xmm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rsi)
-; AVX512-NEXT:    vmovdqa64 %zmm5, (%rdx)
-; AVX512-NEXT:    vmovdqa64 %zmm12, (%rcx)
+; AVX512-NEXT:    vmovdqa64 %zmm4, (%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm6, (%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm8, (%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -2924,128 +2888,110 @@ define void @load_i16_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %ymm2
-; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm6
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm7[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm3
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm5[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm7
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm8[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm3
-; AVX512DQ-NEXT:    vpmovqw %ymm3, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm10
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm3[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm4[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm0, %xmm11
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm10[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm11
-; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm13
-; AVX512DQ-NEXT:    vmovdqa 208(%rdi), %xmm14
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm13[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1]
+; AVX512DQ-NEXT:    vpmovqw %ymm3, %xmm9
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm1, %zmm9
-; AVX512DQ-NEXT:    vpmovqw %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm10
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm11 = xmm10[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm13
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm14 = xmm13[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm14[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm12[6,7]
+; AVX512DQ-NEXT:    vpmovqw %zmm0, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm9[0,1,2,3],zmm4[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm2, %ymm9
+; AVX512DQ-NEXT:    vpmovqw %ymm9, %xmm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm12[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3,4,5],ymm6[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm1, %zmm8
+; AVX512DQ-NEXT:    vpmovqw %zmm8, %xmm8
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm3, %ymm8
+; AVX512DQ-NEXT:    vpmovqw %ymm8, %xmm8
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm10[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
 ; AVX512DQ-NEXT:    vpsrlq $16, %zmm0, %zmm9
 ; AVX512DQ-NEXT:    vpmovqw %zmm9, %xmm9
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[0,1,2,3]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm6[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm8[0,1,2,3],zmm6[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm2, %ymm8
+; AVX512DQ-NEXT:    vpmovqw %ymm8, %xmm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm5[0,1,2,0,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm7[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm12
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm14[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm13[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm1, %zmm13
-; AVX512DQ-NEXT:    vpmovqw %zmm13, %xmm13
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm11[2,0,2,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm7[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm1, %zmm9
+; AVX512DQ-NEXT:    vpmovqw %zmm9, %xmm9
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm3, %ymm9
+; AVX512DQ-NEXT:    vpmovqw %ymm9, %xmm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm10[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm0, %zmm14
-; AVX512DQ-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm13[0,1,2,3],zmm12[0,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm13[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm0, %zmm11
+; AVX512DQ-NEXT:    vpmovqw %zmm11, %xmm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4,5],ymm6[6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
 ; AVX512DQ-NEXT:    vpsrlq $48, %zmm1, %zmm1
 ; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm11[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
 ; AVX512DQ-NEXT:    vpsrlq $48, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpmovqw %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rsi)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm5, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm12, (%rcx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, (%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm6, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm8, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
@@ -5673,276 +5619,222 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-LABEL: load_i16_stride4_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $200, %rsp
-; AVX512-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512-NEXT:    vmovdqa64 384(%rdi), %zmm27
-; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm28
-; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm29
-; AVX512-NEXT:    vmovdqa 192(%rdi), %ymm0
-; AVX512-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 256(%rdi), %zmm22
+; AVX512-NEXT:    vmovdqa64 384(%rdi), %zmm23
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm29
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm30
+; AVX512-NEXT:    vmovdqa 192(%rdi), %ymm5
+; AVX512-NEXT:    vpmovqw %ymm5, %xmm0
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm14
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm13
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm13[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 240(%rdi), %xmm24
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 224(%rdi), %xmm26
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpmovqw %zmm29, %xmm1
+; AVX512-NEXT:    vpmovqw %zmm30, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512-NEXT:    vmovdqa 64(%rdi), %ymm6
+; AVX512-NEXT:    vpmovqw %ymm6, %xmm1
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm12
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm11
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT:    vmovdqa64 112(%rdi), %xmm27
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 96(%rdi), %xmm28
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
+; AVX512-NEXT:    vpmovqw %zmm29, %xmm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512-NEXT:    vmovdqa 448(%rdi), %ymm7
+; AVX512-NEXT:    vpmovqw %ymm7, %xmm0
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm8
+; AVX512-NEXT:    vmovdqa64 496(%rdi), %xmm16
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 480(%rdi), %xmm17
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT:    vpmovqw %zmm23, %xmm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT:    vmovdqa 320(%rdi), %ymm8
+; AVX512-NEXT:    vpmovqw %ymm8, %xmm11
+; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm15
+; AVX512-NEXT:    vmovdqa64 368(%rdi), %xmm18
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 352(%rdi), %xmm19
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT:    vpmovqw %zmm22, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $16, %ymm5, %ymm2
+; AVX512-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpmovqw %zmm28, %xmm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa 448(%rdi), %ymm0
-; AVX512-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm30, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $16, %ymm6, %ymm4
+; AVX512-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm29, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $16, %ymm7, %ymm2
+; AVX512-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa64 496(%rdi), %xmm24
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm24[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm4[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 480(%rdi), %xmm23
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm23[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm15[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpmovqw %zmm27, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa 320(%rdi), %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm23, %zmm1
+; AVX512-NEXT:    vpmovqw %zmm1, %xmm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $16, %ymm8, %ymm1
 ; AVX512-NEXT:    vpmovqw %ymm1, %xmm1
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vmovdqa64 368(%rdi), %xmm31
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm31[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 352(%rdi), %xmm25
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm25[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpmovqw %zmm26, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vmovdqa64 320(%rdi), %xmm30
-; AVX512-NEXT:    vmovdqa64 336(%rdi), %xmm17
-; AVX512-NEXT:    vmovdqa64 448(%rdi), %xmm18
-; AVX512-NEXT:    vmovdqa64 464(%rdi), %xmm19
-; AVX512-NEXT:    vmovdqa64 64(%rdi), %xmm20
-; AVX512-NEXT:    vmovdqa64 80(%rdi), %xmm21
-; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa 208(%rdi), %xmm1
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm29, %zmm6
-; AVX512-NEXT:    vpmovqw %zmm6, %xmm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm7[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm21[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm20[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm28, %zmm6
-; AVX512-NEXT:    vpmovqw %zmm6, %xmm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm4[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm15[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm19[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm18[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm27, %zmm3
-; AVX512-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm17[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm30[0,2,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT:    vpsrlq $16, %zmm22, %zmm2
+; AVX512-NEXT:    vpmovqw %zmm2, %xmm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $32, %ymm5, %ymm0
+; AVX512-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm30, %zmm1
+; AVX512-NEXT:    vpmovqw %zmm1, %xmm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $32, %ymm6, %ymm0
+; AVX512-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vpsrlq $16, %zmm26, %zmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm29, %zmm4
 ; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm22
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $32, %ymm7, %ymm1
+; AVX512-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm23, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $32, %ymm8, %ymm4
+; AVX512-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]
+; AVX512-NEXT:    vpsrlq $32, %zmm22, %zmm11
+; AVX512-NEXT:    vpmovqw %zmm11, %xmm11
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $48, %ymm5, %ymm4
+; AVX512-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm29, %zmm1
-; AVX512-NEXT:    vpmovqw %zmm1, %xmm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm11[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpsrlq $48, %zmm30, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $48, %ymm6, %ymm4
+; AVX512-NEXT:    vpmovqw %ymm4, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm21[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm21
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm20[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm16
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm28, %zmm8
-; AVX512-NEXT:    vpmovqw %zmm8, %xmm8
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm10 = xmm24[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm11 = xmm23[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm11[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm20 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm12 = xmm19[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm12[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm18[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm18 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm4 = xmm31[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm15 = xmm25[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm15[0,1,2,0,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm19 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm8 = xmm17[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm30[3,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
-; AVX512-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm14
-; AVX512-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm14[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm27, %zmm14
-; AVX512-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm14
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm14[6,7]
-; AVX512-NEXT:    vpsrlq $32, %zmm26, %zmm14
-; AVX512-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm0[0,1,2,3],zmm5[0,1,2,3]
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm0
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]
 ; AVX512-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm13[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vpsrlq $48, %zmm29, %zmm3
-; AVX512-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm5
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT:    vpsrlq $48, %zmm29, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512-NEXT:    vpsrlq $48, %ymm7, %ymm3
+; AVX512-NEXT:    vpmovqw %ymm3, %xmm3
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpsrlq $48, %zmm28, %zmm3
-; AVX512-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm0[0,1,2,3]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm11[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpsrlq $48, %zmm27, %zmm2
-; AVX512-NEXT:    vpmovqw %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vpsrlq $48, %zmm26, %zmm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT:    vpsrlq $48, %zmm23, %zmm3
 ; AVX512-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vmovaps %zmm2, 64(%rsi)
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vmovaps %zmm2, 64(%rdx)
-; AVX512-NEXT:    vmovups (%rsp), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512-NEXT:    vmovdqa64 %zmm17, 64(%rcx)
-; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512-NEXT:    vmovaps %zmm2, (%rcx)
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $48, %ymm8, %ymm3
+; AVX512-NEXT:    vpmovqw %ymm3, %xmm3
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpsrlq $48, %zmm22, %zmm4
+; AVX512-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
+; AVX512-NEXT:    vmovdqa64 %zmm21, 64(%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm20, (%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm31, 64(%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm25, (%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm2, 64(%rcx)
+; AVX512-NEXT:    vmovdqa64 %zmm24, (%rcx)
 ; AVX512-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%r8)
-; AVX512-NEXT:    addq $200, %rsp
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -6096,276 +5988,222 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-LABEL: load_i16_stride4_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $200, %rsp
-; AVX512DQ-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512DQ-NEXT:    vmovdqa64 384(%rdi), %zmm27
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm28
-; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm29
-; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 256(%rdi), %zmm22
+; AVX512DQ-NEXT:    vmovdqa64 384(%rdi), %zmm23
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm29
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm30
+; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %ymm5
+; AVX512DQ-NEXT:    vpmovqw %ymm5, %xmm0
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm14
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm14[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm9[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm13
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm13[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm6[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 240(%rdi), %xmm24
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm24[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm10[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %xmm26
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm14 = xmm26[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm14[0,1,0,2,4,5,6,7]
 ; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm29, %xmm1
+; AVX512DQ-NEXT:    vpmovqw %zmm30, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %ymm6
+; AVX512DQ-NEXT:    vpmovqw %ymm6, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm12
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm12[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm3[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm11
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm11[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm7[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT:    vmovdqa64 112(%rdi), %xmm27
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm27[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm3[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 96(%rdi), %xmm28
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm28[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm9[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
+; AVX512DQ-NEXT:    vpmovqw %zmm29, %xmm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-NEXT:    vmovdqa 448(%rdi), %ymm7
+; AVX512DQ-NEXT:    vpmovqw %ymm7, %xmm0
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vmovdqa64 496(%rdi), %xmm16
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm16[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm0[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 480(%rdi), %xmm17
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm17[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm1[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm12[0],xmm11[0],xmm12[1],xmm11[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT:    vpmovqw %zmm23, %xmm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa 320(%rdi), %ymm8
+; AVX512DQ-NEXT:    vpmovqw %ymm8, %xmm11
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vmovdqa64 368(%rdi), %xmm18
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm11 = xmm18[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm11[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %xmm19
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm19[0,2,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm12[0,1,0,2,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT:    vpmovqw %zmm22, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm2[0,1,2,3],zmm13[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm5, %ymm2
+; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm10[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm14[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm28, %xmm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 448(%rdi), %ymm0
-; AVX512DQ-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm30, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm29, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm25 = zmm3[0,1,2,3],zmm2[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm7, %ymm2
+; AVX512DQ-NEXT:    vpmovqw %ymm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vmovdqa64 496(%rdi), %xmm24
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm24[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm4[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 480(%rdi), %xmm23
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm23[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm15[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm27, %xmm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa 320(%rdi), %ymm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm23, %zmm1
+; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm8, %ymm1
 ; AVX512DQ-NEXT:    vpmovqw %ymm1, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 368(%rdi), %xmm31
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm31[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm5[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %xmm25
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm25[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm8[0,1,0,2,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpmovqw %zmm26, %xmm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %xmm30
-; AVX512DQ-NEXT:    vmovdqa64 336(%rdi), %xmm17
-; AVX512DQ-NEXT:    vmovdqa64 448(%rdi), %xmm18
-; AVX512DQ-NEXT:    vmovdqa64 464(%rdi), %xmm19
-; AVX512DQ-NEXT:    vmovdqa64 64(%rdi), %xmm20
-; AVX512DQ-NEXT:    vmovdqa64 80(%rdi), %xmm21
-; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa 208(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm9[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm0[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm29, %zmm6
-; AVX512DQ-NEXT:    vpmovqw %zmm6, %xmm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm7[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm21[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm20[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm28, %zmm6
-; AVX512DQ-NEXT:    vpmovqw %zmm6, %xmm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm4[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm15[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm11[0,1,1,3,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm12[0,1,1,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm19[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm18[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm27, %zmm3
-; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm5[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm8[0,1,1,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm17[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm5 = xmm30[0,2,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm22, %zmm2
+; AVX512DQ-NEXT:    vpmovqw %zmm2, %xmm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm31 = zmm1[0,1,2,3],zmm0[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm5, %ymm0
+; AVX512DQ-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm24[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm26[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm12[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm30, %zmm1
+; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm6, %ymm0
+; AVX512DQ-NEXT:    vpmovqw %ymm0, %xmm0
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm27[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm28[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm0[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm26, %zmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm29, %zmm4
 ; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm3[0,1,2,3],zmm2[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm14[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm22
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[3,1,2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm2[0,1,2,3],zmm1[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm7, %ymm1
+; AVX512DQ-NEXT:    vpmovqw %ymm1, %xmm1
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm16[3,1,2,3]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm13[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm17[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm1[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm23, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm8, %ymm4
+; AVX512DQ-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm18[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm15[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm19[3,1,2,3]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm10[0,1,2,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm14[0],xmm11[1],xmm14[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm22, %zmm11
+; AVX512DQ-NEXT:    vpmovqw %zmm11, %xmm11
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm4[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm2[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm5, %ymm4
+; AVX512DQ-NEXT:    vpmovqw %ymm4, %xmm4
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm12[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm1[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm6[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm7[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm29, %zmm1
-; AVX512DQ-NEXT:    vpmovqw %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm2[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm11[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm3[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm30, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpmovqw %ymm4, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm21[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm21
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm20[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm0[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm16
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm28, %zmm8
-; AVX512DQ-NEXT:    vpmovqw %zmm8, %xmm8
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm10 = xmm24[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm10[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm11 = xmm23[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm11[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm20 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm12 = xmm19[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm12[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm18[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm1[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm18 = xmm8[0],xmm4[0],xmm8[1],xmm4[1]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm4 = xmm31[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm4[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm15 = xmm25[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm15[0,1,2,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm19 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm8 = xmm17[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm8[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm30[3,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm9[2,0,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
-; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm18, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm14[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm27, %zmm14
-; AVX512DQ-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm14[6,7]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm26, %zmm14
-; AVX512DQ-NEXT:    vpmovqw %zmm14, %xmm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm0[0,1,2,3],zmm5[0,1,2,3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm0
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm9[0,1,3,1,4,5,6,7]
 ; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm13[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm6[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm7[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm29, %zmm3
-; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm3
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm5
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm29, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm3[0,1,2,3]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm7, %ymm3
+; AVX512DQ-NEXT:    vpmovqw %ymm3, %xmm3
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm28, %zmm3
-; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm0[0,1,2,3]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm10[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm11[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm12[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm4[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm13[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm27, %zmm2
-; AVX512DQ-NEXT:    vpmovqw %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7]
-; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm26, %zmm3
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm23, %zmm3
 ; AVX512DQ-NEXT:    vpmovqw %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm2[0,1,2,3],zmm1[0,1,2,3]
-; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovaps %zmm2, 64(%rsi)
-; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovaps %zmm2, 64(%rdx)
-; AVX512DQ-NEXT:    vmovups (%rsp), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, 64(%rcx)
-; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-NEXT:    vmovaps %zmm2, (%rcx)
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm8, %ymm3
+; AVX512DQ-NEXT:    vpmovqw %ymm3, %xmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm15[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm10[0,1,3,1,4,5,6,7]
+; AVX512DQ-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm22, %zmm4
+; AVX512DQ-NEXT:    vpmovqw %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm3[0,1,2,3],zmm1[0,1,2,3]
+; AVX512DQ-NEXT:    vmovdqa64 %zmm21, 64(%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm20, (%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm31, 64(%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm25, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 64(%rcx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm24, (%rcx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%r8)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%r8)
-; AVX512DQ-NEXT:    addq $200, %rsp
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
index 99932c0026b2..6770fb666060 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-8.ll
@@ -5975,343 +5975,275 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-LABEL: load_i8_stride8_vf32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm12
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm1
-; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm7
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vmovdqa 208(%rdi), %xmm9
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm9, %xmm2
-; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm8
-; AVX512-NEXT:    vpshufb %xmm4, %xmm8, %xmm6
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512-NEXT:    vpmovqb %zmm1, %xmm2
+; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm1
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm7 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
+; AVX512-NEXT:    vpshufb %xmm7, %xmm1, %xmm3
+; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm5
+; AVX512-NEXT:    vpshufb %xmm7, %xmm5, %xmm4
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vmovdqa 208(%rdi), %xmm6
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm6, %xmm4
+; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm10
+; AVX512-NEXT:    vpshufb %xmm8, %xmm10, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
 ; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm2
-; AVX512-NEXT:    vpmovqb %ymm2, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm6
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm1
-; AVX512-NEXT:    vpshufb %xmm3, %xmm1, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm18
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm1
-; AVX512-NEXT:    vpshufb %xmm4, %xmm1, %xmm11
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm21
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm3
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
-; AVX512-NEXT:    vpmovqb %zmm16, %xmm10
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpmovqb %ymm2, %xmm9
+; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm3
+; AVX512-NEXT:    vpshufb %xmm7, %xmm3, %xmm9
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm4
+; AVX512-NEXT:    vpshufb %xmm7, %xmm4, %xmm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm7
+; AVX512-NEXT:    vpshufb %xmm8, %xmm7, %xmm13
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm9
+; AVX512-NEXT:    vpshufb %xmm8, %xmm9, %xmm8
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512-NEXT:    vpmovqb %zmm16, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm17
-; AVX512-NEXT:    vmovdqa 160(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 176(%rdi), %xmm6
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm19
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm12
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vmovdqa 160(%rdi), %xmm11
+; AVX512-NEXT:    vmovdqa 176(%rdi), %xmm12
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm1, %xmm13
+; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm9, %xmm12
-; AVX512-NEXT:    vmovdqa64 %xmm9, %xmm22
-; AVX512-NEXT:    vpshufb %xmm14, %xmm8, %xmm13
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm20
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm12, %xmm6, %xmm13
-; AVX512-NEXT:    vpshufb %xmm12, %xmm10, %xmm12
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVX512-NEXT:    vmovdqa 128(%rdi), %xmm8
-; AVX512-NEXT:    vmovdqa 144(%rdi), %xmm13
-; AVX512-NEXT:    vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm0
-; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm26
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm1
+; AVX512-NEXT:    vpshufb %xmm14, %xmm6, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm10, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm24
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm5
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm4
-; AVX512-NEXT:    vpshufb %xmm14, %xmm4, %xmm2
-; AVX512-NEXT:    vpshufb %xmm14, %xmm3, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm3, %xmm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $8, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm13[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm13 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm13, %xmm11, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT:    vpsrlq $8, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm3, %xmm13
+; AVX512-NEXT:    vpshufb %xmm8, %xmm4, %xmm8
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512-NEXT:    vpshufb %xmm14, %xmm9, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512-NEXT:    vpsrlq $8, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm18
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm7, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm8
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm7
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm6, %xmm14
-; AVX512-NEXT:    vmovdqa %xmm6, %xmm12
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm6
-; AVX512-NEXT:    vpshufb %xmm14, %xmm6, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm20
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm6
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $16, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $16, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $16, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm19
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm25
-; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm14
-; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm22
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm12, %xmm14
-; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm27
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm12
-; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm5
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm15
-; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm11, %xmm6
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $24, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vpsrlq $24, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $24, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm1
-; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm9, %xmm7
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm9
-; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm0
-; AVX512-NEXT:    vpshufb %xmm4, %xmm0, %xmm14
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm22
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm13, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm25
-; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm1
-; AVX512-NEXT:    vpshufb %xmm3, %xmm15, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm10
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm23
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $32, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpsrlq $32, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $32, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm21
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm11, %xmm1
-; AVX512-NEXT:    vpshufb %xmm5, %xmm7, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm7, %xmm13
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm26
-; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm9, %xmm14
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm6
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm7
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm9
-; AVX512-NEXT:    vpshufb %xmm4, %xmm9, %xmm0
-; AVX512-NEXT:    vpshufb %xmm4, %xmm12, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm8
-; AVX512-NEXT:    vpshufb %xmm5, %xmm8, %xmm1
-; AVX512-NEXT:    vpshufb %xmm5, %xmm15, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm24
-; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm4
-; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $40, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm15
+; AVX512-NEXT:    vpshufb %xmm13, %xmm10, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpsrlq $40, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $40, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm22
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512-NEXT:    vpshufb %xmm10, %xmm11, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm23
-; AVX512-NEXT:    vpshufb %xmm10, %xmm13, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm11
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm25
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm6, %xmm15
+; AVX512-NEXT:    vpshufb %xmm14, %xmm10, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5,6],ymm8[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm13 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm12, %xmm15
+; AVX512-NEXT:    vpshufb %xmm13, %xmm11, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512-NEXT:    vpsrlq $48, %ymm2, %ymm15
+; AVX512-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm13
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512-NEXT:    vpshufb %xmm14, %xmm9, %xmm14
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $48, %zmm16, %zmm13
+; AVX512-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
+; AVX512-NEXT:    vpshufb %xmm5, %xmm10, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm9, %xmm0
-; AVX512-NEXT:    vmovdqa %xmm9, %xmm14
-; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vpshufb %xmm10, %xmm8, %xmm1
-; AVX512-NEXT:    vmovdqa %xmm8, %xmm9
-; AVX512-NEXT:    vpshufb %xmm10, %xmm15, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm10
-; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm4, %xmm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $48, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm2
-; AVX512-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
-; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5,6],ymm1[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm6 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm12, %xmm10
+; AVX512-NEXT:    vpshufb %xmm6, %xmm11, %xmm6
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpsrlq $56, %ymm2, %ymm2
+; AVX512-NEXT:    vpmovqb %ymm2, %xmm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm6, %xmm5
-; AVX512-NEXT:    vpshufb %xmm4, %xmm7, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm14, %xmm7
-; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm9, %xmm4
-; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $56, %zmm16, %zmm3
-; AVX512-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5],ymm2[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm2
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm7, %xmm2
+; AVX512-NEXT:    vpshufb %xmm5, %xmm9, %xmm3
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $56, %zmm16, %zmm2
+; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa64 %ymm17, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %ymm18, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %ymm19, (%rcx)
@@ -6320,9 +6252,9 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %ymm22, (%rax)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa %ymm0, (%rax)
+; AVX512-NEXT:    vmovdqa %ymm8, (%rax)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -6514,343 +6446,275 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-LABEL: load_i8_stride8_vf32:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm12
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm12, %xmm1
-; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm7, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vmovdqa 208(%rdi), %xmm9
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm9, %xmm2
-; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm8, %xmm6
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm2
+; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm7 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
+; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm1, %xmm3
+; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm5, %xmm4
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa 208(%rdi), %xmm6
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm6, %xmm4
+; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm10, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm2
-; AVX512DQ-NEXT:    vpmovqb %ymm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm6
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm1, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm18
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm1, %xmm11
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm21
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3]
-; AVX512DQ-NEXT:    vpmovqb %zmm16, %xmm10
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpmovqb %ymm2, %xmm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm3, %xmm9
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm7, %xmm4, %xmm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm7, %xmm13
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm9, %xmm8
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512DQ-NEXT:    vpmovqb %zmm16, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm17
-; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 176(%rdi), %xmm6
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm19
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm12
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm4[0],xmm12[1],xmm4[1],xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %xmm11
+; AVX512DQ-NEXT:    vmovdqa 176(%rdi), %xmm12
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm1, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm14 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm9, %xmm12
-; AVX512DQ-NEXT:    vmovdqa64 %xmm9, %xmm22
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm8, %xmm13
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm20
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1],xmm13[2],xmm12[2],xmm13[3],xmm12[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm12 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm6, %xmm13
-; AVX512DQ-NEXT:    vpshufb %xmm12, %xmm10, %xmm12
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %xmm8
-; AVX512DQ-NEXT:    vmovdqa 144(%rdi), %xmm13
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm1 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm26
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm6, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm10, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm24
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm3, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm3, %xmm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $8, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm13[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm13 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm11, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT:    vpsrlq $8, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm3, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm4, %xmm8
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm9, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512DQ-NEXT:    vpsrlq $8, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm18
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm7, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm6, %xmm14
-; AVX512DQ-NEXT:    vmovdqa %xmm6, %xmm12
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm6, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm20
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm19
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm25
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm14
-; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm22
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm12, %xmm14
-; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm27
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm12
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3,4],ymm4[5],ymm14[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm15
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm11, %xmm6
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $24, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $24, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $24, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm9, %xmm7
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm14[0],xmm4[0],xmm14[1],xmm4[1],xmm14[2],xmm4[2],xmm14[3],xmm4[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm0, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm22
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm14[0],xmm4[1],xmm14[1],xmm4[2],xmm14[2],xmm4[3],xmm14[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm13, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm25
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1],xmm14[2],xmm0[2],xmm14[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm23
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm10, %xmm15
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm21
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm11, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm7, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm7, %xmm13
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm26
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm9, %xmm14
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm4 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm9, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm12, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm8, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm24
-; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $40, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm13 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm10, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm14 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpsrlq $40, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm9, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $40, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm0, %ymm22
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm10 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm11, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm23
-; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm13, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm25
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm14 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm6, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm10, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm13 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm12, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm11, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm2, %ymm15
+; AVX512DQ-NEXT:    vpmovqb %ymm15, %xmm15
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4],ymm13[5],ymm15[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm9, %xmm14
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm16, %zmm13
+; AVX512DQ-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm5 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm9, %xmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm9, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm12, %xmm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm8, %xmm1
-; AVX512DQ-NEXT:    vmovdqa %xmm8, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm10, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm4, %xmm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm6 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm12, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm11, %xmm6
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpsrlq $56, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpmovqb %ymm2, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm4 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm6, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm7, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm5 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm14, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm12, %xmm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm9, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $56, %zmm16, %zmm3
-; AVX512DQ-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5],ymm2[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm7, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm9, %xmm3
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $56, %zmm16, %zmm2
+; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm17, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm18, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm19, (%rcx)
@@ -6859,9 +6723,9 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %ymm22, (%rax)
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rax)
+; AVX512DQ-NEXT:    vmovdqa %ymm8, (%rax)
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -7055,300 +6919,284 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa 240(%rdi), %xmm2
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm1
-; AVX512BW-NEXT:    vmovdqa 224(%rdi), %xmm6
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm6, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vmovdqa 208(%rdi), %xmm7
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm7, %xmm3
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm9, %xmm8
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512BW-NEXT:    vpmovqb %ymm3, %xmm3
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vpmovqb %zmm1, %xmm2
+; AVX512BW-NEXT:    vmovdqa 240(%rdi), %xmm1
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm1, %xmm3
+; AVX512BW-NEXT:    vmovdqa 224(%rdi), %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm4, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa 112(%rdi), %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm1, %xmm8
-; AVX512BW-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
-; AVX512BW-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm4, %xmm11
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm8
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm8, %xmm5
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
-; AVX512BW-NEXT:    vpmovqb %zmm16, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %xmm19
-; AVX512BW-NEXT:    vmovdqa 144(%rdi), %xmm11
-; AVX512BW-NEXT:    vmovdqa 160(%rdi), %xmm12
-; AVX512BW-NEXT:    vmovdqa 176(%rdi), %xmm13
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm14
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vmovdqa 208(%rdi), %xmm5
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm5, %xmm6
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm9
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm9, %xmm10
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5,6],ymm3[7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm6
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT:    vpmovqb %ymm2, %xmm10
+; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3,4],ymm6[5],ymm10[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5],ymm3[6,7]
+; AVX512BW-NEXT:    vmovdqa 112(%rdi), %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm3, %xmm10
+; AVX512BW-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512BW-NEXT:    vpshufb %xmm7, %xmm6, %xmm7
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]
+; AVX512BW-NEXT:    vmovdqa 80(%rdi), %xmm7
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm7, %xmm13
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm10
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm10, %xmm8
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512BW-NEXT:    vpmovqb %zmm17, %xmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512BW-NEXT:    vmovdqa 160(%rdi), %xmm11
+; AVX512BW-NEXT:    vmovdqa 176(%rdi), %xmm12
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm1, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm4, %xmm14
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm5, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm16
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm8, %xmm14
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $8, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512BW-NEXT:    vpsrlq $8, %ymm2, %ymm16
+; AVX512BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm3, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm8
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512BW-NEXT:    vpsrlq $8, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm19
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm19, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT:    vpsrlq $16, %ymm2, %ymm15
+; AVX512BW-NEXT:    vpmovqb %ymm15, %xmm15
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $16, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $16, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm20
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm19, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT:    vpsrlq $24, %ymm2, %ymm15
+; AVX512BW-NEXT:    vpmovqb %ymm15, %xmm15
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $24, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm23
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $24, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm21
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm16
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm16
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm15
+; AVX512BW-NEXT:    vpsrlq $32, %ymm2, %ymm16
+; AVX512BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm15
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $32, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm24
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $32, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm22
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm16
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm14
+; AVX512BW-NEXT:    vpsrlq $40, %ymm2, %ymm16
+; AVX512BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm14
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $40, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm25
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $40, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm0, %ymm23
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm14
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm14
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm15
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm5, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm16
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm14, %xmm8, %xmm14
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $48, %zmm16, %zmm10
-; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vpsrlq $48, %ymm2, %ymm16
+; AVX512BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $48, %zmm17, %zmm13
+; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm9
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm12, %xmm9
+; AVX512BW-NEXT:    vpshufb %xmm5, %xmm11, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpsrlq $56, %ymm2, %ymm2
+; AVX512BW-NEXT:    vpmovqb %ymm2, %xmm2
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm9, %xmm9
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm7, %xmm13, %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm7, %xmm12, %xmm7
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm9, %xmm11, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm9, %xmm19, %xmm9
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm4, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm6, %xmm8, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT:    vpsrlq $56, %zmm16, %zmm3
-; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm20, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %ymm21, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %ymm22, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %ymm23, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %ymm24, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %ymm25, (%r11)
-; AVX512BW-NEXT:    vmovdqa %ymm0, (%r10)
-; AVX512BW-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm10, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $56, %zmm17, %zmm2
+; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %ymm18, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %ymm19, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %ymm20, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %ymm21, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %ymm22, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %ymm23, (%r11)
+; AVX512BW-NEXT:    vmovdqa %ymm8, (%r10)
+; AVX512BW-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -7525,300 +7373,284 @@ define void @load_i8_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa 240(%rdi), %xmm2
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm2, %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa 224(%rdi), %xmm6
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm6, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vmovdqa 208(%rdi), %xmm7
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm5 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm7, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa 192(%rdi), %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm9, %xmm8
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512DQ-BW-NEXT:    vpmovqb %ymm3, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm17
+; AVX512DQ-BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm1, %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa 240(%rdi), %xmm1
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm7 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm1, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa 224(%rdi), %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm4, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa 112(%rdi), %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm1, %xmm8
-; AVX512DQ-BW-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm3, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3]
-; AVX512DQ-BW-NEXT:    vmovdqa 80(%rdi), %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm4, %xmm11
-; AVX512DQ-BW-NEXT:    vmovdqa 64(%rdi), %xmm8
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm8, %xmm5
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm11[0],xmm5[1],xmm11[1],xmm5[2],xmm11[2],xmm5[3],xmm11[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1,2],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm16, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512DQ-BW-NEXT:    vmovdqa64 128(%rdi), %xmm19
-; AVX512DQ-BW-NEXT:    vmovdqa 144(%rdi), %xmm11
-; AVX512DQ-BW-NEXT:    vmovdqa 160(%rdi), %xmm12
-; AVX512DQ-BW-NEXT:    vmovdqa 176(%rdi), %xmm13
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa 208(%rdi), %xmm5
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm5, %xmm6
+; AVX512DQ-BW-NEXT:    vmovdqa 192(%rdi), %xmm9
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm9, %xmm10
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm6
+; AVX512DQ-BW-NEXT:    vmovdqa 128(%rdi), %ymm2
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm2, %xmm10
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3,4],ymm6[5],ymm10[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa 112(%rdi), %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm3, %xmm10
+; AVX512DQ-BW-NEXT:    vmovdqa 96(%rdi), %xmm6
+; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm6, %xmm7
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm7[0],xmm10[0],xmm7[1],xmm10[1],xmm7[2],xmm10[2],xmm7[3],xmm10[3]
+; AVX512DQ-BW-NEXT:    vmovdqa 80(%rdi), %xmm7
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT:    vmovdqa 64(%rdi), %xmm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm10, %xmm8
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm8[0,1,2],xmm12[3]
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm17, %xmm12
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm12[0,1],xmm8[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm11[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm18
+; AVX512DQ-BW-NEXT:    vmovdqa 160(%rdi), %xmm11
+; AVX512DQ-BW-NEXT:    vmovdqa 176(%rdi), %xmm12
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm1, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm4, %xmm14
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm15[5],ymm10[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm8, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm21
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm3, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm8
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1],xmm8[2],xmm13[2],xmm8[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1,2],xmm8[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm13[0,1],xmm8[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm19
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm19, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %ymm2, %ymm15
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm15, %xmm15
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm22
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm20
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm15
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm19, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %ymm2, %ymm15
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm15, %xmm15
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm23
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm21
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm17[0],xmm14[0],xmm17[1],xmm14[1],xmm17[2],xmm14[2],xmm17[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm16[0],xmm14[0],xmm16[1],xmm14[1],xmm16[2],xmm14[2],xmm16[3],xmm14[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm17[0],xmm14[1],xmm17[1],xmm14[2],xmm17[2],xmm14[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm11, %xmm14
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm15
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5],ymm15[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm24
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm22
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm10 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm13[0],xmm8[0],xmm13[1],xmm8[1],xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm14
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm14
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5],ymm14[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm4, %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm25
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm7, %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm10, %xmm13
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm0, %ymm23
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm2, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm14
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm14 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm15
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm17[0],xmm15[0],xmm17[1],xmm15[1],xmm17[2],xmm15[2],xmm17[3],xmm15[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm5, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm9, %xmm16
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm16[0],xmm15[0],xmm16[1],xmm15[1],xmm16[2],xmm15[2],xmm16[3],xmm15[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3,4,5,6],ymm10[7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3,4,5,6],ymm8[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm15 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm13, %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm17[0],xmm15[1],xmm17[1],xmm15[2],xmm17[2],xmm15[3],xmm17[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm12, %xmm16
+; AVX512DQ-BW-NEXT:    vpshufb %xmm15, %xmm11, %xmm15
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm15[0],xmm16[0],xmm15[1],xmm16[1],xmm15[2],xmm16[2],xmm15[3],xmm16[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm11, %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm19, %xmm17
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm17 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm17, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm15[5],ymm5[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm10[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm0
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm4, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm8, %xmm14
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1],xmm14[2],xmm10[2],xmm14[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm16, %zmm10
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %ymm2, %ymm16
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm16, %xmm16
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm7, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm14, %xmm10, %xmm14
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm17, %zmm13
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm0[0,1,2,3],ymm8[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm9
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm5 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm6, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm12, %xmm9
+; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %ymm2, %ymm2
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm2, %xmm2
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm6 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm9, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm7 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm13, %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm7, %xmm12, %xmm7
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm9 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm11, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm9, %xmm19, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3,4],ymm7[5],ymm9[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm4, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm6, %xmm8, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm16, %zmm3
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm20, (%rsi)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm21, (%rdx)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm22, (%rcx)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm23, (%r8)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm24, (%r9)
-; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm25, (%r11)
-; AVX512DQ-BW-NEXT:    vmovdqa %ymm0, (%r10)
-; AVX512DQ-BW-NEXT:    vmovdqa %ymm1, (%rax)
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm3, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm17, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm18, (%rsi)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm19, (%rdx)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm20, (%rcx)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm21, (%r8)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm22, (%r9)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm23, (%r11)
+; AVX512DQ-BW-NEXT:    vmovdqa %ymm8, (%r10)
+; AVX512DQ-BW-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
@@ -13041,46 +12873,45 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512-LABEL: load_i8_stride8_vf64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    subq $520, %rsp # imm = 0x208
-; AVX512-NEXT:    vmovdqa64 256(%rdi), %zmm17
+; AVX512-NEXT:    subq $440, %rsp # imm = 0x1B8
+; AVX512-NEXT:    vmovdqa64 256(%rdi), %zmm30
 ; AVX512-NEXT:    vmovdqa64 384(%rdi), %zmm0
 ; AVX512-NEXT:    vpmovqb %zmm0, %xmm2
-; AVX512-NEXT:    vmovdqa 496(%rdi), %xmm7
+; AVX512-NEXT:    vmovdqa 496(%rdi), %xmm6
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm1
-; AVX512-NEXT:    vmovdqa 480(%rdi), %xmm8
-; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm1
+; AVX512-NEXT:    vmovdqa 480(%rdi), %xmm9
+; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm3
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm3
-; AVX512-NEXT:    vmovdqa 464(%rdi), %xmm11
+; AVX512-NEXT:    vmovdqa 464(%rdi), %xmm13
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
-; AVX512-NEXT:    vmovdqa 448(%rdi), %xmm15
-; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm4
+; AVX512-NEXT:    vmovdqa 448(%rdi), %xmm8
+; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm5
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa 384(%rdi), %ymm4
-; AVX512-NEXT:    vpmovqb %ymm4, %xmm4
+; AVX512-NEXT:    vmovdqa64 384(%rdi), %ymm18
+; AVX512-NEXT:    vpmovqb %ymm18, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512-NEXT:    vmovdqa 368(%rdi), %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm18
-; AVX512-NEXT:    vmovdqa 352(%rdi), %xmm14
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm4
+; AVX512-NEXT:    vmovdqa 368(%rdi), %xmm10
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512-NEXT:    vmovdqa 352(%rdi), %xmm12
+; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm4
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa 336(%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT:    vmovdqa 320(%rdi), %xmm6
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm19
+; AVX512-NEXT:    vmovdqa 336(%rdi), %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm21
+; AVX512-NEXT:    vmovdqa 320(%rdi), %xmm7
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm16
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
-; AVX512-NEXT:    vpmovqb %zmm17, %xmm4
+; AVX512-NEXT:    vpmovqb %zmm30, %xmm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
 ; AVX512-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
 ; AVX512-NEXT:    movb $-64, %al
@@ -13088,18 +12919,15 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
 ; AVX512-NEXT:    vmovdqa 240(%rdi), %xmm4
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm2
-; AVX512-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm25
-; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
+; AVX512-NEXT:    vmovdqa 224(%rdi), %xmm11
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm4
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
 ; AVX512-NEXT:    vmovdqa 208(%rdi), %xmm5
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm29
-; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm10
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm5
-; AVX512-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm22
+; AVX512-NEXT:    vmovdqa 192(%rdi), %xmm14
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm5
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512-NEXT:    vmovdqa64 128(%rdi), %zmm5
 ; AVX512-NEXT:    vpmovqb %zmm5, %xmm5
@@ -13107,8 +12935,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm4
-; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm5
-; AVX512-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm19
+; AVX512-NEXT:    vpmovqb %ymm19, %xmm5
 ; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
@@ -13119,655 +12947,597 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm30
+; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm4
+; AVX512-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm5
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm28
+; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm27
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512-NEXT:    vpmovqb %zmm16, %xmm1
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm20
+; AVX512-NEXT:    vpmovqb %zmm20, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm7, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm24
-; AVX512-NEXT:    vpshufb %xmm9, %xmm8, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm21
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm6, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm23
+; AVX512-NEXT:    vpshufb %xmm5, %xmm9, %xmm2
+; AVX512-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovdqa64 %xmm9, %xmm31
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm11, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm27
-; AVX512-NEXT:    vpshufb %xmm8, %xmm15, %xmm4
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm26
+; AVX512-NEXT:    vpshufb %xmm1, %xmm8, %xmm4
+; AVX512-NEXT:    vmovdqa %xmm8, %xmm6
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovdqa 416(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa 432(%rdi), %xmm11
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm5
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm22
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512-NEXT:    vmovdqa 384(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa 400(%rdi), %xmm12
-; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm6
-; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm7
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm31
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm0
-; AVX512-NEXT:    vpshufb %xmm9, %xmm0, %xmm5
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm23
-; AVX512-NEXT:    vpshufb %xmm9, %xmm14, %xmm6
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm26
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
-; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm14
-; AVX512-NEXT:    vpshufb %xmm8, %xmm14, %xmm7
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512-NEXT:    vpsrlq $8, %zmm17, %zmm6
-; AVX512-NEXT:    vpmovqb %zmm6, %xmm6
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm19
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm0
-; AVX512-NEXT:    vpshufb %xmm9, %xmm0, %xmm4
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm9, %xmm0, %xmm5
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm0
-; AVX512-NEXT:    vpshufb %xmm8, %xmm0, %xmm5
-; AVX512-NEXT:    vpshufb %xmm8, %xmm10, %xmm6
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vmovdqa 160(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa 176(%rdi), %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm25
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm20
 ; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa 128(%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vmovdqa 144(%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-NEXT:    vmovdqa 432(%rdi), %xmm7
+; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm29
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpsrlq $8, %ymm18, %ymm9
+; AVX512-NEXT:    vpmovqb %ymm9, %xmm9
+; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm10, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm24
+; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm9
+; AVX512-NEXT:    vmovdqa %xmm12, %xmm13
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm12
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm9
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm0
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm28
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm4 = xmm9[0,1,2],xmm4[3]
+; AVX512-NEXT:    vpsrlq $8, %zmm30, %zmm9
+; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512-NEXT:    vpshufb %xmm5, %xmm7, %xmm3
+; AVX512-NEXT:    vpshufb %xmm5, %xmm11, %xmm4
+; AVX512-NEXT:    vmovdqa %xmm11, %xmm15
+; AVX512-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm11
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm22
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512-NEXT:    vmovdqa 160(%rdi), %xmm14
+; AVX512-NEXT:    vmovdqa 176(%rdi), %xmm8
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm0
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm2
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
+; AVX512-NEXT:    vpsrlq $8, %ymm19, %ymm2
+; AVX512-NEXT:    vpmovqb %ymm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm9, %xmm2, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm7
-; AVX512-NEXT:    vpshufb %xmm8, %xmm7, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm3
-; AVX512-NEXT:    vpshufb %xmm8, %xmm3, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512-NEXT:    vpsrlq $8, %zmm16, %zmm2
+; AVX512-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm5, %xmm3, %xmm5
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
+; AVX512-NEXT:    vpsrlq $8, %zmm20, %zmm2
 ; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm13
-; AVX512-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm4, %xmm13, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm10
-; AVX512-NEXT:    vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
+; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm2
+; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm0
-; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm15, %xmm18
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm16
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm24
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm11
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm30
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm12
-; AVX512-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm4, %xmm12, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT:    vpshufb %xmm4, %xmm5, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm15
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm21
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $16, %zmm17, %zmm9
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm4, %xmm14, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm6
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm9
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm9
-; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm15
-; AVX512-NEXT:    vpshufb %xmm3, %xmm15, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm2, %xmm15, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpsrlq $16, %ymm18, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm6
+; AVX512-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
+; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm31
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm23
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm12
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT:    vmovdqa64 %zmm30, %zmm21
+; AVX512-NEXT:    vpsrlq $16, %zmm30, %zmm10
+; AVX512-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm17
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm24
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm13
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm22
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm25
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm29
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm7
-; AVX512-NEXT:    vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vpsrlq $16, %ymm19, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa (%rsp), %xmm11 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm7
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512-NEXT:    vpsrlq $16, %zmm16, %zmm2
-; AVX512-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX512-NEXT:    vpsrlq $16, %zmm20, %zmm1
+; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm2
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm10
-; AVX512-NEXT:    vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm4
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm28
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm9
-; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm11
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm8
+; AVX512-NEXT:    vmovdqa64 %xmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm9
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm13
-; AVX512-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $24, %zmm17, %zmm9
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
+; AVX512-NEXT:    vpsrlq $24, %ymm18, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT:    vpsrlq $24, %zmm30, %zmm10
+; AVX512-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm4
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm22
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm26
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm14
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm10
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm12
+; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm26
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm4
+; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT:    vpsrlq $24, %ymm19, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm28
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm15, %xmm29
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm30
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $24, %zmm16, %zmm1
+; AVX512-NEXT:    vpsrlq $24, %zmm20, %zmm1
 ; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm10
-; AVX512-NEXT:    vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm15
+; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm11
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm7
-; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm7
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm12
-; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm9
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
+; AVX512-NEXT:    vpsrlq $32, %ymm18, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm8
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm9
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm8
+; AVX512-NEXT:    vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vmovdqa64 %xmm23, %xmm13
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm10
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm31
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT:    vmovdqa64 %zmm21, %zmm23
+; AVX512-NEXT:    vpsrlq $32, %zmm21, %zmm10
+; AVX512-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm5
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm9
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $32, %zmm17, %zmm9
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm5
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm20
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm14
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm11
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm25
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm4 # 16-byte Reload
+; AVX512-NEXT:    vpsrlq $32, %ymm19, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm5
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm10
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm5
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $32, %zmm16, %zmm1
-; AVX512-NEXT:    vmovdqa64 %zmm16, %zmm18
+; AVX512-NEXT:    vpsrlq $32, %zmm20, %zmm1
 ; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
+; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm21
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm9
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm22
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm27
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm5
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm10
-; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm9
-; AVX512-NEXT:    vpshufb %xmm3, %xmm12, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm8
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm11
-; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $40, %zmm17, %zmm9
-; AVX512-NEXT:    vmovdqa64 %zmm17, %zmm23
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm30
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm6
-; AVX512-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm13
-; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm9
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm9
-; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; AVX512-NEXT:    vpsrlq $40, %ymm18, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm9
+; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm24
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT:    vpsrlq $40, %zmm23, %zmm10
+; AVX512-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vpshufb %xmm0, %xmm3, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm25
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm3, %xmm15, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm15, %xmm28
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm16
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm14, %xmm17
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm16
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm26
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm24
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm7
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT:    vpsrlq $40, %ymm19, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm4
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm8
+; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm6
+; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm7
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $40, %zmm18, %zmm1
-; AVX512-NEXT:    vmovdqa64 %zmm18, %zmm26
+; AVX512-NEXT:    vpsrlq $40, %zmm20, %zmm1
 ; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm1
-; AVX512-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm2
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm9
+; AVX512-NEXT:    vpshufb %xmm1, %xmm9, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm8
-; AVX512-NEXT:    vmovdqa64 %xmm5, %xmm21
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm10, %xmm18
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm5, %xmm15
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm8
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
 ; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm9
-; AVX512-NEXT:    vpshufb %xmm1, %xmm11, %xmm15
-; AVX512-NEXT:    vmovdqa64 %xmm11, %xmm22
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $48, %zmm17, %zmm9
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm13, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm13, %xmm19
-; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm17
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm6
-; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm9
+; AVX512-NEXT:    vpsrlq $48, %ymm18, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm10, %xmm10
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm10
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm14
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm12
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512-NEXT:    vpsrlq $48, %zmm23, %zmm10
+; AVX512-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm15, %xmm28
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm15
+; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm10
+; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm17
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm10
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm12
 ; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
-; AVX512-NEXT:    vmovdqa64 %xmm12, %xmm20
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm14
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm13
-; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm15
-; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm12
-; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm7, %xmm16
-; AVX512-NEXT:    vmovdqa (%rsp), %xmm7 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-NEXT:    vpsrlq $48, %ymm19, %ymm10
+; AVX512-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
+; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm10
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $48, %zmm26, %zmm1
+; AVX512-NEXT:    vpsrlq $48, %zmm20, %zmm1
 ; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm24
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm26
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512-NEXT:    vmovdqa64 %xmm29, %xmm1
-; AVX512-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
+; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm2
 ; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm6
+; AVX512-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm3
-; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm21, %xmm8
-; AVX512-NEXT:    vpshufb %xmm2, %xmm8, %xmm8
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512-NEXT:    vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512-NEXT:    vmovdqa64 %xmm18, %xmm9
-; AVX512-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm31, %xmm11
-; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm8
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm22, %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512-NEXT:    vpsrlq $56, %zmm23, %zmm9
-; AVX512-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm5
-; AVX512-NEXT:    vpshufb %xmm1, %xmm5, %xmm11
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm6
-; AVX512-NEXT:    vmovdqa64 %xmm20, %xmm5
-; AVX512-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
-; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512-NEXT:    vpsrlq $56, %ymm18, %ymm7
+; AVX512-NEXT:    vpmovqb %ymm7, %xmm7
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm13, %xmm6
+; AVX512-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512-NEXT:    vmovdqa64 %xmm24, %xmm7
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm7
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
+; AVX512-NEXT:    vpsrlq $56, %zmm23, %zmm7
+; AVX512-NEXT:    vpmovqb %zmm7, %xmm7
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
+; AVX512-NEXT:    vinserti32x4 $2, %xmm6, %zmm0, %zmm6
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm6 {%k1}
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512-NEXT:    vmovdqa64 %xmm28, %xmm7
+; AVX512-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm15, %xmm7
+; AVX512-NEXT:    vmovdqa64 %xmm17, %xmm9
+; AVX512-NEXT:    vpshufb %xmm1, %xmm9, %xmm9
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5,6],ymm5[7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
+; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpsrlq $56, %ymm19, %ymm3
+; AVX512-NEXT:    vpmovqb %ymm3, %xmm3
 ; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm0, %xmm15, %xmm3
-; AVX512-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm0, %xmm4, %xmm3
+; AVX512-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm30, %xmm3
 ; AVX512-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
 ; AVX512-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512-NEXT:    vpsrlq $56, %zmm26, %zmm1
+; AVX512-NEXT:    vpsrlq $56, %zmm20, %zmm1
 ; AVX512-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512-NEXT:    vinserti64x4 $0, %ymm0, %zmm6, %zmm0
 ; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -13782,10 +13552,10 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512-NEXT:    vmovaps %zmm1, (%rax)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512-NEXT:    vmovdqa64 %zmm24, (%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm26, (%rax)
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512-NEXT:    addq $520, %rsp # imm = 0x208
+; AVX512-NEXT:    addq $440, %rsp # imm = 0x1B8
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -14218,46 +13988,45 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512DQ-LABEL: load_i8_stride8_vf64:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    subq $520, %rsp # imm = 0x208
-; AVX512DQ-NEXT:    vmovdqa64 256(%rdi), %zmm17
+; AVX512DQ-NEXT:    subq $440, %rsp # imm = 0x1B8
+; AVX512DQ-NEXT:    vmovdqa64 256(%rdi), %zmm30
 ; AVX512DQ-NEXT:    vmovdqa64 384(%rdi), %zmm0
 ; AVX512DQ-NEXT:    vpmovqb %zmm0, %xmm2
-; AVX512DQ-NEXT:    vmovdqa 496(%rdi), %xmm7
+; AVX512DQ-NEXT:    vmovdqa 496(%rdi), %xmm6
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,0,8,0,0,0,8,0,0,0,8,0,0,0,8]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm1
-; AVX512DQ-NEXT:    vmovdqa 480(%rdi), %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm1
+; AVX512DQ-NEXT:    vmovdqa 480(%rdi), %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vmovdqa 464(%rdi), %xmm11
+; AVX512DQ-NEXT:    vmovdqa 464(%rdi), %xmm13
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [0,8,0,0,0,8,0,0,0,8,0,0,0,8,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
-; AVX512DQ-NEXT:    vmovdqa 448(%rdi), %xmm15
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm4
+; AVX512DQ-NEXT:    vmovdqa 448(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm5
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vmovdqa 384(%rdi), %ymm4
-; AVX512DQ-NEXT:    vpmovqb %ymm4, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 384(%rdi), %ymm18
+; AVX512DQ-NEXT:    vpmovqb %ymm18, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-NEXT:    vmovdqa 368(%rdi), %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm18
-; AVX512DQ-NEXT:    vmovdqa 352(%rdi), %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm4
+; AVX512DQ-NEXT:    vmovdqa 368(%rdi), %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-NEXT:    vmovdqa 352(%rdi), %xmm12
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm12, %xmm4
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa 336(%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vmovdqa 320(%rdi), %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm19
+; AVX512DQ-NEXT:    vmovdqa 336(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm21
+; AVX512DQ-NEXT:    vmovdqa 320(%rdi), %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm16
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
-; AVX512DQ-NEXT:    vpmovqb %zmm17, %xmm4
+; AVX512DQ-NEXT:    vpmovqb %zmm30, %xmm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
 ; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
 ; AVX512DQ-NEXT:    movb $-64, %al
@@ -14265,18 +14034,15 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm3 {%k1}
 ; AVX512DQ-NEXT:    vmovdqa 240(%rdi), %xmm4
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm25
-; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vmovdqa 224(%rdi), %xmm11
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm11, %xmm4
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
 ; AVX512DQ-NEXT:    vmovdqa 208(%rdi), %xmm5
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm29
-; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm5
-; AVX512DQ-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm22
+; AVX512DQ-NEXT:    vmovdqa 192(%rdi), %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm5
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %zmm5
 ; AVX512DQ-NEXT:    vpmovqb %zmm5, %xmm5
@@ -14284,8 +14050,8 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm5
-; AVX512DQ-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm19
+; AVX512DQ-NEXT:    vpmovqb %ymm19, %xmm5
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
@@ -14296,655 +14062,597 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm30
+; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm4
+; AVX512DQ-NEXT:    vmovdqa %xmm4, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm5
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm28
+; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm27
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm16
-; AVX512DQ-NEXT:    vpmovqb %zmm16, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %zmm20
+; AVX512DQ-NEXT:    vpmovqb %zmm20, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm9 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm7, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm24
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm8, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm21
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [0,0,1,9,0,0,1,9,0,0,1,9,0,0,1,9]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm6, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm23
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm9, %xmm2
+; AVX512DQ-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovdqa64 %xmm9, %xmm31
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm11, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm27
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm15, %xmm4
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [1,9,0,0,1,9,0,0,1,9,0,0,1,9,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm26
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm8, %xmm4
+; AVX512DQ-NEXT:    vmovdqa %xmm8, %xmm6
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovdqa 416(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa 432(%rdi), %xmm11
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm0, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm22
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
-; AVX512DQ-NEXT:    vmovdqa 384(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa 400(%rdi), %xmm12
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [1,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm0, %xmm7
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm31
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5],ymm6[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm0, %xmm5
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm23
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm14, %xmm6
-; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm26
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm0, %xmm6
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm14, %xmm7
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512DQ-NEXT:    vpsrlq $8, %zmm17, %zmm6
-; AVX512DQ-NEXT:    vpmovqb %zmm6, %xmm6
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm19
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm19 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm0, %xmm4
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm0, %xmm5
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm0, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm10, %xmm6
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa 176(%rdi), %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm25
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm0, %xmm20
-; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vmovdqa 144(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-NEXT:    vmovdqa 432(%rdi), %xmm7
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,1,9,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm29
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm0, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpsrlq $8, %ymm18, %ymm9
+; AVX512DQ-NEXT:    vpmovqb %ymm9, %xmm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4],ymm4[5],ymm9[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm10, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm24
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm12, %xmm9
+; AVX512DQ-NEXT:    vmovdqa %xmm12, %xmm13
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm12
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm12, %xmm9
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm28
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm4 = xmm9[0,1,2],xmm4[3]
+; AVX512DQ-NEXT:    vpsrlq $8, %zmm30, %zmm9
+; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm7, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm11, %xmm4
+; AVX512DQ-NEXT:    vmovdqa %xmm11, %xmm15
+; AVX512DQ-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm11
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm22
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %xmm14
+; AVX512DQ-NEXT:    vmovdqa 176(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm0
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpsrlq $8, %ymm19, %ymm2
+; AVX512DQ-NEXT:    vpmovqb %ymm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5],ymm2[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm10[6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm7, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3]
-; AVX512DQ-NEXT:    vpsrlq $8, %zmm16, %zmm2
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm3, %xmm5
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-NEXT:    vmovdqa (%rsp), %xmm3 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
+; AVX512DQ-NEXT:    vpsrlq $8, %zmm20, %zmm2
 ; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm13
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm13, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm10
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm10, %xmm2
+; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,2,10,0,0,2,10,0,0,2,10,0,0,2,10]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2,10,0,0,2,10,0,0,2,10,0,0,2,10,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm0
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm15, %xmm18
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm16
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm24
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [2,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm30
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5],ymm0[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm12
-; AVX512DQ-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm12, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm15
-; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm21
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm17, %zmm9
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm14, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,2,10,0,0,0,0,0,0,0,0,0,0,0,0]
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm15
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm15, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm18, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm6
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm31
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm12, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm23
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm12
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa64 %zmm30, %zmm21
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm30, %zmm10
+; AVX512DQ-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm17
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm24
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm22
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm25
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm29
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm7
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vpsrlq $16, %ymm19, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa (%rsp), %xmm11 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm7
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512DQ-NEXT:    vpsrlq $16, %zmm16, %zmm2
-; AVX512DQ-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
+; AVX512DQ-NEXT:    vpsrlq $16, %zmm20, %zmm1
+; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,3,11,0,0,3,11,0,0,3,11,0,0,3,11]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm1
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [3,11,0,0,3,11,0,0,3,11,0,0,3,11,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm10
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,3,11,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm28
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [3,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm9
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm12, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm13
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $24, %zmm17, %zmm9
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
+; AVX512DQ-NEXT:    vpsrlq $24, %ymm18, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT:    vpsrlq $24, %zmm30, %zmm10
+; AVX512DQ-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm22
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm26
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm12
+; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm26
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT:    vpsrlq $24, %ymm19, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm28
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-NEXT:    vmovdqa64 %xmm15, %xmm29
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm30
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $24, %zmm16, %zmm1
+; AVX512DQ-NEXT:    vpsrlq $24, %zmm20, %zmm1
 ; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,4,12,0,0,4,12,0,0,4,12,0,0,4,12]
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4,12,0,0,4,12,0,0,4,12,0,0,4,12,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm10
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,4,12,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [4,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm7, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm12
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm12, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm18, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm9
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm8
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm23, %xmm13
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm31
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa64 %zmm21, %zmm23
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm21, %zmm10
+; AVX512DQ-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm5
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm17, %zmm9
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm20
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm11
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm0
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa (%rsp), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT:    vpsrlq $32, %ymm19, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm5
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $32, %zmm16, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 %zmm16, %zmm18
+; AVX512DQ-NEXT:    vpsrlq $32, %zmm20, %zmm1
 ; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,5,13,0,0,5,13,0,0,5,13,0,0,5,13]
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm21
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [5,13,0,0,5,13,0,0,5,13,0,0,5,13,0,0]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm9
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm22
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,5,13,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm27
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [5,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm10
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm10, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm12, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm8
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $40, %zmm17, %zmm9
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, %zmm23
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm30
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm6, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm13
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm9
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
+; AVX512DQ-NEXT:    vpsrlq $40, %ymm18, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm24
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT:    vpsrlq $40, %zmm23, %zmm10
+; AVX512DQ-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm3, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm25
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm15, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm15, %xmm28
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm16
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm14, %xmm17
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm11, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm16
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm26
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm24
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm7
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vmovdqa (%rsp), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT:    vpsrlq $40, %ymm19, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm4
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm8
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm6
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $40, %zmm18, %zmm1
-; AVX512DQ-NEXT:    vmovdqa64 %zmm18, %zmm26
+; AVX512DQ-NEXT:    vpsrlq $40, %zmm20, %zmm1
 ; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
 ; AVX512DQ-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,6,14,0,0,6,14,0,0,6,14,0,0,6,14]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm1
-; AVX512DQ-NEXT:    vmovdqa64 %xmm2, %xmm29
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm2
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [6,14,0,0,6,14,0,0,6,14,0,0,6,14,0,0]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm9, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,6,14,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm8
-; AVX512DQ-NEXT:    vmovdqa64 %xmm5, %xmm21
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [6,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm10, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm10, %xmm18
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm5, %xmm15
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm8
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm9, %xmm9
 ; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm11, %xmm15
-; AVX512DQ-NEXT:    vmovdqa64 %xmm11, %xmm22
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm17, %zmm9
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm13, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm13, %xmm19
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm15
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm17
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm6
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm18, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4],ymm9[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm9
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm14
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm12
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1,2],xmm9[3]
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm23, %zmm10
+; AVX512DQ-NEXT:    vpmovqb %zmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm9 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm15, %xmm28
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm15
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm12
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm17
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm10[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm10
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm12
 ; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm12, %xmm20
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm14
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm13
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm15
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm12
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm7, %xmm16
-; AVX512DQ-NEXT:    vmovdqa (%rsp), %xmm7 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-NEXT:    vpsrlq $48, %ymm19, %ymm10
+; AVX512DQ-NEXT:    vpmovqb %ymm10, %xmm10
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3,4],ymm2[5],ymm10[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm10
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $48, %zmm26, %zmm1
+; AVX512DQ-NEXT:    vpsrlq $48, %zmm20, %zmm1
 ; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm24
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm26
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [0,0,7,15,0,0,7,15,0,0,7,15,0,0,7,15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm29, %xmm1
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
+; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm2
 ; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [7,15,0,0,7,15,0,0,7,15,0,0,7,15,0,0]
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm6
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm6, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm2 = [0,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm21, %xmm8
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm8, %xmm8
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1],xmm8[2],xmm3[2],xmm8[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovd {{.*#+}} xmm3 = [7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm18, %xmm9
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm9, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm31, %xmm11
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm11, %xmm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3,4],ymm8[5],ymm9[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm8
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm22, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1,2],xmm8[3]
-; AVX512DQ-NEXT:    vpsrlq $56, %zmm23, %zmm9
-; AVX512DQ-NEXT:    vpmovqb %zmm9, %xmm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm8, %zmm0, %zmm8
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm8 {%k1}
-; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm4
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm9
-; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm5, %xmm11
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3]
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm4 = ymm9[0,1,2,3,4,5,6],ymm4[7]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm6, %xmm6
-; AVX512DQ-NEXT:    vmovdqa64 %xmm20, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm5, %xmm2
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm5
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
-; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpsrlq $56, %ymm18, %ymm7
+; AVX512DQ-NEXT:    vpmovqb %ymm7, %xmm7
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5],ymm7[6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm13, %xmm6
+; AVX512DQ-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm24, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm14, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3]
+; AVX512DQ-NEXT:    vpsrlq $56, %zmm23, %zmm7
+; AVX512DQ-NEXT:    vpmovqb %zmm7, %xmm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm6 = xmm7[0,1],xmm6[2,3]
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm6, %zmm0, %zmm6
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm6 {%k1}
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vmovdqa64 %xmm28, %xmm7
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm15, %xmm7
+; AVX512DQ-NEXT:    vmovdqa64 %xmm17, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm9, %xmm9
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm12, %xmm2
+; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpsrlq $56, %ymm19, %ymm3
+; AVX512DQ-NEXT:    vpmovqb %ymm3, %xmm3
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm15, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm4, %xmm3
+; AVX512DQ-NEXT:    vpshufb %xmm0, %xmm8, %xmm0
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm16, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm30, %xmm3
 ; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512DQ-NEXT:    vpshufb %xmm1, %xmm10, %xmm1
 ; AVX512DQ-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3]
-; AVX512DQ-NEXT:    vpsrlq $56, %zmm26, %zmm1
+; AVX512DQ-NEXT:    vpsrlq $56, %zmm20, %zmm1
 ; AVX512DQ-NEXT:    vpmovqb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512DQ-NEXT:    vinserti64x4 $0, %ymm0, %zmm6, %zmm0
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovaps %zmm1, (%rsi)
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
@@ -14959,10 +14667,10 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-NEXT:    vmovaps %zmm1, (%rax)
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-NEXT:    vmovdqa64 %zmm24, (%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm26, (%rax)
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-NEXT:    addq $520, %rsp # imm = 0x208
+; AVX512DQ-NEXT:    addq $440, %rsp # imm = 0x1B8
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -15395,678 +15103,621 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512BW-LABEL: load_i8_stride8_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $744, %rsp # imm = 0x2E8
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm1
+; AVX512BW-NEXT:    subq $552, %rsp # imm = 0x228
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vpmovqb %zmm1, %xmm12
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512BW-NEXT:    vmovdqa64 496(%rdi), %xmm24
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm24, %xmm3
+; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm3
+; AVX512BW-NEXT:    vmovdqa64 496(%rdi), %xmm27
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm27, %xmm4
 ; AVX512BW-NEXT:    vmovdqa64 480(%rdi), %xmm25
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm25, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT:    vmovdqa64 464(%rdi), %xmm26
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm26, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %xmm30
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm30, %xmm6
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa 384(%rdi), %ymm4
-; AVX512BW-NEXT:    vpmovqb %ymm4, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm25, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512BW-NEXT:    vmovdqa64 368(%rdi), %xmm31
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm31, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 352(%rdi), %xmm27
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm27, %xmm6
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512BW-NEXT:    vmovdqa64 336(%rdi), %xmm22
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm22, %xmm11
-; AVX512BW-NEXT:    vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa 320(%rdi), %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm9, %xmm15
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
-; AVX512BW-NEXT:    vpmovqb %zmm1, %xmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 464(%rdi), %xmm31
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm18 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm31, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %xmm28
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm28, %xmm8
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm5
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %ymm16
+; AVX512BW-NEXT:    vpmovqb %ymm16, %xmm8
+; AVX512BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4],ymm5[5],ymm8[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512BW-NEXT:    vmovdqa64 368(%rdi), %xmm23
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm23, %xmm8
+; AVX512BW-NEXT:    vmovdqa64 352(%rdi), %xmm24
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm24, %xmm9
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512BW-NEXT:    vmovdqa64 336(%rdi), %xmm26
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm26, %xmm14
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %xmm19
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm19, %xmm15
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
+; AVX512BW-NEXT:    vpmovqb %zmm0, %xmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm13, %zmm0, %zmm21
 ; AVX512BW-NEXT:    movb $-64, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 240(%rdi), %xmm28
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm28, %xmm7
-; AVX512BW-NEXT:    vmovdqa64 224(%rdi), %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm18, %xmm10
-; AVX512BW-NEXT:    vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512BW-NEXT:    vmovdqa64 208(%rdi), %xmm17
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm17, %xmm10
-; AVX512BW-NEXT:    vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm8
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm8, %xmm16
-; AVX512BW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm10
-; AVX512BW-NEXT:    vpmovqb %ymm10, %xmm10
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
-; AVX512BW-NEXT:    vmovdqa 112(%rdi), %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm10, %xmm16
-; AVX512BW-NEXT:    vmovdqa 96(%rdi), %xmm14
-; AVX512BW-NEXT:    vpshufb %xmm12, %xmm14, %xmm12
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
-; AVX512BW-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm2, %xmm21
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512BW-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm2, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm21 {%k1}
+; AVX512BW-NEXT:    vmovdqa 240(%rdi), %xmm0
+; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm0, %xmm13
+; AVX512BW-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm0, %xmm15
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512BW-NEXT:    vmovdqa 208(%rdi), %xmm6
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm6, %xmm15
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm14
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm14, %xmm20
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm15
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm0
+; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpmovqb %ymm0, %xmm20
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512BW-NEXT:    vmovdqa 112(%rdi), %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm13, %xmm20
+; AVX512BW-NEXT:    vmovdqa 96(%rdi), %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm17, %xmm15, %xmm17
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm17[0],xmm20[0],xmm17[1],xmm20[1],xmm17[2],xmm20[2],xmm17[3],xmm20[3]
+; AVX512BW-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm12, %xmm22
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm7
+; AVX512BW-NEXT:    vpshufb %xmm18, %xmm7, %xmm18
+; AVX512BW-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm18[0],xmm22[0],xmm18[1],xmm22[1],xmm18[2],xmm22[2],xmm18[3],xmm22[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpmovqb %zmm10, %xmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm21, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 160(%rdi), %xmm6
-; AVX512BW-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa 384(%rdi), %xmm7
-; AVX512BW-NEXT:    vmovdqa64 400(%rdi), %xmm21
-; AVX512BW-NEXT:    vmovdqa64 416(%rdi), %xmm23
-; AVX512BW-NEXT:    vmovdqa64 432(%rdi), %xmm29
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vmovdqa64 %xmm24, %xmm19
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm1
-; AVX512BW-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vmovdqa64 %xmm26, %xmm12
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm26, %xmm24
-; AVX512BW-NEXT:    vmovdqa64 %xmm30, %xmm16
-; AVX512BW-NEXT:    vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm30, %xmm25
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm26, %xmm29, %xmm24
-; AVX512BW-NEXT:    vpshufb %xmm26, %xmm23, %xmm25
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm21, %xmm24
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm7, %xmm25
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm31, %xmm2
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm27, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm22, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm24
-; AVX512BW-NEXT:    vmovdqa64 %xmm9, %xmm31
-; AVX512BW-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
-; AVX512BW-NEXT:    vpsrlq $8, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm22
-; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %xmm28, %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm28, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm17, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm8, %xmm24
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa 176(%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 160(%rdi), %xmm20
+; AVX512BW-NEXT:    vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 176(%rdi), %xmm22
+; AVX512BW-NEXT:    vmovdqa64 416(%rdi), %xmm29
+; AVX512BW-NEXT:    vmovdqa 432(%rdi), %xmm8
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm1
+; AVX512BW-NEXT:    vmovdqa64 %xmm27, %xmm10
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm31, %xmm27
+; AVX512BW-NEXT:    vmovdqa64 %xmm28, %xmm11
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm28, %xmm28
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm3
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpshufb %xmm26, %xmm0, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm26, %xmm6, %xmm25
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %xmm6
-; AVX512BW-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa 144(%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm6, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm10, %xmm1
-; AVX512BW-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm14, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm15, %xmm3
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm28, %xmm4
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    vpshufb %xmm30, %xmm8, %xmm27
+; AVX512BW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm30, %xmm29, %xmm28
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpsrlq $8, %ymm16, %ymm27
+; AVX512BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm24, %xmm4
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT:    vpsrlq $8, %zmm5, %zmm3
-; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm19, %xmm18
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm19, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
+; AVX512BW-NEXT:    vpsrlq $8, %zmm9, %zmm4
+; AVX512BW-NEXT:    vpmovqb %zmm4, %xmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm4
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm14, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT:    vpshufb %xmm30, %xmm22, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm22, %xmm19
+; AVX512BW-NEXT:    vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm30, %xmm20, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm22 # 32-byte Reload
+; AVX512BW-NEXT:    vpsrlq $8, %ymm22, %ymm27
+; AVX512BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm12, %xmm20
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpsrlq $8, %zmm12, %zmm2
+; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm19, %xmm2
-; AVX512BW-NEXT:    vmovdqa64 %xmm19, %xmm20
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm11, %xmm3
+; AVX512BW-NEXT:    vmovdqa %xmm10, %xmm7
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
+; AVX512BW-NEXT:    vmovdqa64 %xmm25, %xmm28
+; AVX512BW-NEXT:    vmovdqa64 %xmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm25, %xmm3
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm12, %xmm25
-; AVX512BW-NEXT:    vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    vmovdqa64 %xmm31, %xmm30
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm11, %xmm31
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vmovdqa64 %xmm29, %xmm24
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm23, %xmm30
-; AVX512BW-NEXT:    vmovdqa64 %xmm23, %xmm16
-; AVX512BW-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    vmovdqa64 %xmm21, %xmm18
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm21, %xmm0
-; AVX512BW-NEXT:    vmovdqa64 %xmm7, %xmm17
-; AVX512BW-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm7, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm8, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vmovdqa64 %ymm16, %ymm10
+; AVX512BW-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpsrlq $16, %ymm16, %ymm27
+; AVX512BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm12, %xmm2
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm27, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpsrlq $16, %zmm22, %zmm13
-; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
+; AVX512BW-NEXT:    vmovdqa64 %xmm23, %xmm25
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm23, %xmm2
+; AVX512BW-NEXT:    vmovdqa64 %xmm24, %xmm16
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm24, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT:    vmovdqa64 %xmm26, %xmm8
+; AVX512BW-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT:    vpsrlq $16, %zmm9, %zmm5
+; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm9, %xmm0
-; AVX512BW-NEXT:    vmovdqa64 %xmm9, %xmm26
-; AVX512BW-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm21, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm21, %xmm0
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm17, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
+; AVX512BW-NEXT:    vmovdqa %xmm6, %xmm11
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm14, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm13
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm31, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm30, %xmm23, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm19, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm24 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm4
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512BW-NEXT:    vpsrlq $16, %ymm22, %ymm5
+; AVX512BW-NEXT:    vmovdqa64 %ymm22, %ymm23
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm14, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm13, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm13, %xmm19
+; AVX512BW-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
+; AVX512BW-NEXT:    vmovdqa64 %xmm15, %xmm22
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm3
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT:    vpsrlq $16, %zmm5, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm28
+; AVX512BW-NEXT:    vpsrlq $16, %zmm12, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6
 ; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm20, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
+; AVX512BW-NEXT:    vmovdqa %xmm7, %xmm15
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm30, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %xmm31, %xmm28
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm13, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpsrlq $24, %ymm10, %ymm27
+; AVX512BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5],ymm1[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm18, (%rsp) # 16-byte Spill
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm27
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT:    vpsrlq $24, %zmm9, %zmm5
+; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm14, %xmm27
+; AVX512BW-NEXT:    vmovdqa64 %xmm14, %xmm17
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm24, %xmm31
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm4
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT:    vmovdqa64 %ymm23, %ymm12
+; AVX512BW-NEXT:    vpsrlq $24, %ymm23, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vmovdqa64 %xmm19, %xmm10
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm19, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm26, %xmm24
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $24, %zmm6, %zmm3
+; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm25, %xmm3
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm4, %xmm4
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm30, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm28, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm28, %xmm23
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm24, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm30
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm17, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm13, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 %xmm13, %xmm19
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Reload
+; AVX512BW-NEXT:    vpsrlq $32, %ymm30, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa %xmm12, %xmm10
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm12, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm13
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
-; AVX512BW-NEXT:    vpsrlq $24, %zmm6, %zmm13
-; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm18, %xmm28
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512BW-NEXT:    vpsrlq $32, %zmm9, %zmm5
+; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm26, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 %xmm22, %xmm21
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm30
-; AVX512BW-NEXT:    vmovdqa64 %xmm29, %xmm22
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm11, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm11, %xmm18
+; AVX512BW-NEXT:    vmovdqa64 %xmm17, %xmm26
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm28
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm9, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 %xmm9, %xmm25
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512BW-NEXT:    vmovdqa %xmm7, %xmm9
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm31, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm23, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm7, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm7, %xmm28
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm3
+; AVX512BW-NEXT:    vmovdqa64 %xmm31, %xmm11
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
+; AVX512BW-NEXT:    vpsrlq $32, %ymm12, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
-; AVX512BW-NEXT:    vmovdqa64 %xmm14, %xmm23
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm15, %xmm3
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm17
-; AVX512BW-NEXT:    vpsrlq $24, %zmm28, %zmm2
+; AVX512BW-NEXT:    vpsrlq $32, %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm7
 ; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm20, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm3
-; AVX512BW-NEXT:    vmovdqa64 %xmm11, %xmm28
-; AVX512BW-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm26, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm30
-; AVX512BW-NEXT:    vmovdqa64 %xmm18, %xmm24
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa %xmm10, %xmm6
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm2
-; AVX512BW-NEXT:    vmovdqa64 %xmm27, %xmm7
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm10, %xmm13
-; AVX512BW-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpsrlq $32, %zmm10, %zmm13
-; AVX512BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm21, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm13
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm31, %xmm13
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm19, %xmm8, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm20
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpsrlq $32, %zmm17, %zmm3
-; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT:    vmovdqa %xmm8, %xmm13
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm16, %xmm4
+; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm3
+; AVX512BW-NEXT:    vmovdqa64 %xmm23, %xmm6
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm23, %xmm4
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vmovdqa (%rsp), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm24, %xmm19
-; AVX512BW-NEXT:    vmovdqa64 %xmm24, %xmm26
-; AVX512BW-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512BW-NEXT:    vmovdqa64 %xmm19, %xmm31
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm19, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpsrlq $40, %ymm30, %ymm5
+; AVX512BW-NEXT:    vmovdqa64 %ymm30, %ymm19
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm6, %xmm24
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm5, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT:    vmovdqa (%rsp), %xmm8 # 16-byte Reload
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm30
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
-; AVX512BW-NEXT:    vpsrlq $40, %zmm10, %zmm5
+; AVX512BW-NEXT:    vpsrlq $40, %zmm9, %zmm5
 ; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
-; AVX512BW-NEXT:    vmovdqa64 %xmm11, %xmm17
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm21, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT:    vmovdqa64 %xmm18, %xmm23
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm18, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm30
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm25, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm5
+; AVX512BW-NEXT:    vmovdqa64 %xmm11, %xmm18
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm8, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpsrlq $40, %ymm12, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vpsrlq $40, %zmm20, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512BW-NEXT:    vpsrlq $40, %zmm7, %zmm2
 ; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm1
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm17, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm5
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm26, %xmm19
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm30
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm31, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm30
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm24, %xmm3
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm30, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
-; AVX512BW-NEXT:    vpsrlq $48, %zmm10, %zmm5
+; AVX512BW-NEXT:    vpsrlq $48, %ymm19, %ymm30
+; AVX512BW-NEXT:    vpmovqb %ymm30, %xmm30
+; AVX512BW-NEXT:    vinserti32x4 $1, %xmm30, %ymm0, %ymm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm27, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm30
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512BW-NEXT:    vpsrlq $48, %zmm9, %zmm5
 ; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm21, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm19
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm23, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm30
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm9, %xmm18
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm28, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm4
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm5
-; AVX512BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm13, %xmm20, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-NEXT:    vpsrlq $48, %ymm12, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512BW-NEXT:    vmovdqa64 %xmm6, %xmm26
-; AVX512BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpsrlq $48, %zmm19, %zmm2
-; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm24, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $48, %zmm11, %zmm3
+; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm28, %xmm2
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm8, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm30
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm2
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
 ; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vmovdqa (%rsp), %xmm5 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm8
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
-; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm9
-; AVX512BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm13
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
+; AVX512BW-NEXT:    vpsrlq $56, %ymm19, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512BW-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm24, %xmm5
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm7, %xmm9
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm27, %xmm9
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm30, %xmm6
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512BW-NEXT:    vpsrlq $56, %zmm10, %zmm6
-; AVX512BW-NEXT:    vpmovqb %zmm6, %xmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm5
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm14, %xmm2
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm17, %xmm6
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm21, %xmm6
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm9
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm6
-; AVX512BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm4
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm31, %xmm6
-; AVX512BW-NEXT:    vpshufb %xmm8, %xmm20, %xmm8
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm1, %xmm23, %xmm1
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm4
-; AVX512BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm3
-; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512BW-NEXT:    vpsrlq $56, %zmm19, %zmm3
-; AVX512BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm1
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%r9)
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm6
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512BW-NEXT:    vpsrlq $56, %zmm9, %zmm5
+; AVX512BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm23, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm6
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm5
+; AVX512BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm3
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-NEXT:    vpsrlq $56, %ymm12, %ymm5
+; AVX512BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
+; AVX512BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512BW-NEXT:    vpsrlq $56, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%r9)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, (%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512BW-NEXT:    addq $744, %rsp # imm = 0x2E8
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT:    addq $552, %rsp # imm = 0x228
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
@@ -16428,678 +16079,621 @@ define void @load_i8_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512DQ-BW-LABEL: load_i8_stride8_vf64:
 ; AVX512DQ-BW:       # %bb.0:
-; AVX512DQ-BW-NEXT:    subq $744, %rsp # imm = 0x2E8
-; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm0, %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 256(%rdi), %zmm1
+; AVX512DQ-BW-NEXT:    subq $552, %rsp # imm = 0x228
+; AVX512DQ-BW-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm1, %xmm12
+; AVX512DQ-BW-NEXT:    vmovdqa64 256(%rdi), %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 496(%rdi), %xmm24
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm12 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm24, %xmm3
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 496(%rdi), %xmm27
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm17 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm27, %xmm4
 ; AVX512DQ-BW-NEXT:    vmovdqa64 480(%rdi), %xmm25
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm25, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 464(%rdi), %xmm26
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm26, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 448(%rdi), %xmm30
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm30, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5,6],ymm3[7]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vmovdqa 384(%rdi), %ymm4
-; AVX512DQ-BW-NEXT:    vpmovqb %ymm4, %xmm4
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm2[5],ymm4[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm2[0,1,2,3,4,5],ymm3[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa64 368(%rdi), %xmm31
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm31, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa64 352(%rdi), %xmm27
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm27, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 336(%rdi), %xmm22
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm22, %xmm11
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 320(%rdi), %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm9, %xmm15
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm15[0],xmm11[0],xmm15[1],xmm11[1],xmm15[2],xmm11[2],xmm15[3],xmm11[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3]
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm1, %xmm11
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, %zmm15
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm20
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm25, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 464(%rdi), %xmm31
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm18 = [0,8,0,8,0,8,0,8,0,8,0,8,0,8,0,8]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 448(%rdi), %xmm28
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm28, %xmm8
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5,6],ymm4[7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 384(%rdi), %ymm16
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm16, %xmm8
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3,4],ymm5[5],ymm8[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 368(%rdi), %xmm23
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm23, %xmm8
+; AVX512DQ-BW-NEXT:    vmovdqa64 352(%rdi), %xmm24
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm24, %xmm9
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 336(%rdi), %xmm26
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm26, %xmm14
+; AVX512DQ-BW-NEXT:    vmovdqa64 320(%rdi), %xmm19
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm19, %xmm15
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm14 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm14[0,1,2],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm0, %xmm14
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm13, %zmm0, %zmm21
 ; AVX512DQ-BW-NEXT:    movb $-64, %al
 ; AVX512DQ-BW-NEXT:    kmovd %eax, %k1
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm20 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 240(%rdi), %xmm28
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm28, %xmm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 224(%rdi), %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm18, %xmm10
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1],xmm10[2],xmm7[2],xmm10[3],xmm7[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 208(%rdi), %xmm17
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm17, %xmm10
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 192(%rdi), %xmm8
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm8, %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm16[0],xmm10[0],xmm16[1],xmm10[1],xmm16[2],xmm10[2],xmm16[3],xmm10[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm10[0,1,2,3,4,5,6],ymm7[7]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vmovdqa 128(%rdi), %ymm10
-; AVX512DQ-BW-NEXT:    vpmovqb %ymm10, %xmm10
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5],ymm10[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm7[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa 112(%rdi), %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm10, %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa 96(%rdi), %xmm14
-; AVX512DQ-BW-NEXT:    vpshufb %xmm12, %xmm14, %xmm12
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm12[0],xmm16[0],xmm12[1],xmm16[1],xmm12[2],xmm16[2],xmm12[3],xmm16[3]
-; AVX512DQ-BW-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm2, %xmm21
-; AVX512DQ-BW-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm2, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm21[0],xmm19[1],xmm21[1],xmm19[2],xmm21[2],xmm19[3],xmm21[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm21 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa 240(%rdi), %xmm0
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm0, %xmm13
+; AVX512DQ-BW-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm0, %xmm15
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512DQ-BW-NEXT:    vmovdqa 208(%rdi), %xmm6
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm6, %xmm15
+; AVX512DQ-BW-NEXT:    vmovdqa 192(%rdi), %xmm14
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm14, %xmm20
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm20[0],xmm15[0],xmm20[1],xmm15[1],xmm20[2],xmm15[2],xmm20[3],xmm15[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5,6],ymm13[7]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm15
+; AVX512DQ-BW-NEXT:    vmovdqa 128(%rdi), %ymm0
+; AVX512DQ-BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm0, %xmm20
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm15[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa 112(%rdi), %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm13, %xmm20
+; AVX512DQ-BW-NEXT:    vmovdqa 96(%rdi), %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm17, %xmm15, %xmm17
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm17[0],xmm20[0],xmm17[1],xmm20[1],xmm17[2],xmm20[2],xmm17[3],xmm20[3]
+; AVX512DQ-BW-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm12, %xmm22
+; AVX512DQ-BW-NEXT:    vmovdqa 64(%rdi), %xmm7
+; AVX512DQ-BW-NEXT:    vpshufb %xmm18, %xmm7, %xmm18
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm18[0],xmm22[0],xmm18[1],xmm22[1],xmm18[2],xmm22[2],xmm18[3],xmm22[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm10, %xmm2
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm21, %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 160(%rdi), %xmm6
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 384(%rdi), %xmm7
-; AVX512DQ-BW-NEXT:    vmovdqa64 400(%rdi), %xmm21
-; AVX512DQ-BW-NEXT:    vmovdqa64 416(%rdi), %xmm23
-; AVX512DQ-BW-NEXT:    vmovdqa64 432(%rdi), %xmm29
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, %xmm19
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm25, %xmm11
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm26, %xmm12
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm26, %xmm24
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm30, %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm30, %xmm25
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm26 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm29, %xmm24
-; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm23, %xmm25
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm21, %xmm24
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm7, %xmm25
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm24 = xmm25[0],xmm24[0],xmm25[1],xmm24[1],xmm25[2],xmm24[2],xmm25[3],xmm24[3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm24, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm31, %xmm2
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm27, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm22, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm9, %xmm24
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm9, %xmm31
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm15, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm15, %zmm22
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm28, %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm28, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm17, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm8, %xmm24
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm24[0],xmm3[0],xmm24[1],xmm3[1],xmm24[2],xmm3[2],xmm24[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vmovdqa 176(%rdi), %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vmovdqa64 160(%rdi), %xmm20
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vmovdqa64 176(%rdi), %xmm22
+; AVX512DQ-BW-NEXT:    vmovdqa64 416(%rdi), %xmm29
+; AVX512DQ-BW-NEXT:    vmovdqa 432(%rdi), %xmm8
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm27, %xmm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm31, %xmm27
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm28, %xmm11
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm28, %xmm28
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm3
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm0, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm26, %xmm6, %xmm25
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm25[0],xmm3[0],xmm25[1],xmm3[1],xmm25[2],xmm3[2],xmm25[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vmovdqa 128(%rdi), %xmm6
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa 144(%rdi), %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm0, %xmm0
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm6, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm30[0],xmm0[0],xmm30[1],xmm0[1],xmm30[2],xmm0[2],xmm30[3],xmm0[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5],ymm0[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm10, %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm14, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm15, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm28, %xmm4
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [1,9,1,9,1,9,1,9,1,9,1,9,1,9,1,9]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm8, %xmm27
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm29, %xmm28
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm27 = xmm28[0],xmm27[0],xmm28[1],xmm27[1],xmm28[2],xmm27[2],xmm28[3],xmm27[3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %ymm16, %ymm27
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm3[5],ymm4[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm24, %xmm4
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm5, %zmm3
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm19, %xmm18
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm19, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1,2],xmm3[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm9, %zmm4
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm4, %xmm4
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm4
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm22, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm22, %xmm19
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm20, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm27[0],xmm4[0],xmm27[1],xmm4[1],xmm27[2],xmm4[2],xmm27[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm22 # 32-byte Reload
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %ymm22, %ymm27
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm12, %xmm20
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vpsrlq $8, %zmm12, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm19, %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm19, %xmm20
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm11, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm10, %xmm7
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm10, %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm25, %xmm28
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm25, %xmm3
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm12, %xmm25
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm31, %xmm30
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm11, %xmm31
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm29, %xmm24
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm23, %xmm30
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm23, %xmm16
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm30 = [2,10,2,10,2,10,2,10,2,10,2,10,2,10,2,10]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm21, %xmm18
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm21, %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm7, %xmm17
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm7, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm19[0],xmm0[0],xmm19[1],xmm0[1],xmm19[2],xmm0[2],xmm19[3],xmm0[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm13[5],ymm0[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm8, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm16, %ymm10
+; AVX512DQ-BW-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %ymm16, %ymm27
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm0
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5],ymm0[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm12, %xmm2
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm22, %zmm6
-; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm22, %zmm13
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm23, %xmm25
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm23, %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, %xmm16
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm24, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm26, %xmm8
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
 ; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm9, %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm9, %xmm26
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm21 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1],xmm13[2],xmm0[2],xmm13[3],xmm0[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm22 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm21, %xmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm6, %xmm11
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5,6],ymm0[7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm23 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm30, %xmm23, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5,6],ymm0[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm19, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm24 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm4
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %ymm22, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm22, %ymm23
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm10, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm14, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm13, %xmm19
+; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm15, %xmm22
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm3
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm5, %zmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm5, %zmm28
+; AVX512DQ-BW-NEXT:    vpsrlq $16, %zmm12, %zmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm12, %zmm6
 ; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm20, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm2
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm7, %xmm15
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm30, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm31, %xmm28
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
+; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm13, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %ymm10, %ymm27
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm27, %xmm27
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm18, (%rsp) # 16-byte Spill
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm27
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm14, %xmm27
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm14, %xmm17
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm27[0],xmm5[0],xmm27[1],xmm5[1],xmm27[2],xmm5[2],xmm27[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, %xmm31
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm4
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm23, %ymm12
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %ymm23, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm19, %xmm10
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm19, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm26, %xmm24
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm6, %zmm3
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm25, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm4, %xmm4
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm30, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm28, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm28, %xmm23
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm24, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, (%rsp) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm19[0],xmm4[0],xmm19[1],xmm4[1],xmm19[2],xmm4[2],xmm19[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [3,11,3,11,3,11,3,11,3,11,3,11,3,11,3,11]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm17, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm13[0],xmm30[0],xmm13[1],xmm30[1],xmm13[2],xmm30[2],xmm13[3],xmm30[3]
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm13, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm13, %xmm19
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm30 # 32-byte Reload
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %ymm30, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm12, %xmm10
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm13[0,1,2],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm6, %zmm13
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm13[0,1],xmm4[2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm18, %xmm28
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
 ; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm26, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm22, %xmm21
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm30
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm29, %xmm22
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm11, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm11, %xmm18
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm17, %xmm26
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm28
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm28[0],xmm5[0],xmm28[1],xmm5[1],xmm28[2],xmm5[2],xmm28[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm9, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm9, %xmm25
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm7, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm13[0],xmm3[1],xmm13[1],xmm3[2],xmm13[2],xmm3[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm23, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm7, %xmm28
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm31, %xmm11
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3,4],ymm3[5],ymm13[6,7]
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm5, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm0
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm14, %xmm23
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm15, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm29 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm28, %zmm17
-; AVX512DQ-BW-NEXT:    vpsrlq $24, %zmm28, %zmm2
+; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm6, %zmm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm6, %zmm7
 ; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm20, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm3
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm11, %xmm28
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm16 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm24, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm26 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm26, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm19 = [4,12,4,12,4,12,4,12,4,12,4,12,4,12,4,12]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm18, %xmm24
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm18 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm18, %xmm2
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm30[0],xmm2[1],xmm30[1],xmm2[2],xmm30[2],xmm2[3],xmm30[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm13[5],ymm2[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa %xmm10, %xmm6
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm2
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm27, %xmm7
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm27, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm2[0],xmm13[1],xmm2[1],xmm13[2],xmm2[2],xmm13[3],xmm2[3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm10, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1,2],xmm2[3]
-; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm10, %zmm13
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm13, %xmm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm13[0,1],xmm2[2,3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1],xmm13[2],xmm1[2],xmm13[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm21, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm30[0],xmm13[0],xmm30[1],xmm13[1],xmm30[2],xmm13[2],xmm30[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm13[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm13
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm13[0],xmm4[1],xmm13[1],xmm4[2],xmm13[2],xmm4[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm31, %xmm13
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm19, %xmm8, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm13 = xmm19[0],xmm13[0],xmm19[1],xmm13[1],xmm19[2],xmm13[2],xmm19[3],xmm13[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3,4],ymm4[5],ymm13[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm17, %zmm20
-; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-BW-NEXT:    vpsrlq $32, %zmm17, %zmm3
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm1, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT:    vmovdqa %xmm8, %xmm13
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm16, %xmm4
+; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm17 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm3
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm23, %xmm6
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm23, %xmm4
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsp), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm15, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm13[0],xmm4[0],xmm13[1],xmm4[1],xmm13[2],xmm4[2],xmm13[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [5,13,5,13,5,13,5,13,5,13,5,13,5,13,5,13]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm24, %xmm19
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, %xmm26
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm19, %xmm31
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm19, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %ymm30, %ymm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %ymm30, %ymm19
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm6, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm6, %xmm24
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm5, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa (%rsp), %xmm8 # 16-byte Reload
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm10, %zmm5
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm9, %zmm5
 ; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
 ; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm11, %xmm17
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm21, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm18, %xmm23
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm18, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm25, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm9, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm11, %xmm18
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm31 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm8, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm20, %zmm2
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512DQ-BW-NEXT:    vpsrlq $40, %zmm7, %zmm2
 ; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512DQ-BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm28 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm28, %xmm1
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm17, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm6, %xmm5
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm13 = [6,14,6,14,6,14,6,14,6,14,6,14,6,14,6,14]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm26, %xmm19
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm18, %xmm30
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm19 = xmm30[0],xmm19[0],xmm30[1],xmm19[1],xmm30[2],xmm19[2],xmm30[3],xmm19[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm31, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm29, %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm19, %ymm0, %ymm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5],ymm3[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm24, %xmm3
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm30 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm30, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm5[0,1,2],xmm3[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm10, %zmm5
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %ymm19, %ymm30
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm30, %xmm30
+; AVX512DQ-BW-NEXT:    vinserti32x4 $1, %xmm30, %ymm0, %ymm2
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5],ymm2[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm2
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm8, %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1,2],xmm2[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm9, %zmm5
 ; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm5[0,1],xmm3[2,3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm3
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm17, %xmm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm5[0,1],xmm2[2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm2
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm21, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm22, %xmm19
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm19[0],xmm5[0],xmm19[1],xmm5[1],xmm19[2],xmm5[2],xmm19[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm23, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm30
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm30[0],xmm5[0],xmm30[1],xmm5[1],xmm30[2],xmm5[2],xmm30[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm9, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm9, %xmm18
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm4
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm31, %xmm5
-; AVX512DQ-BW-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm20 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm13, %xmm20, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3]
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm11, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm23, %xmm0
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
-; AVX512DQ-BW-NEXT:    vmovdqa64 %xmm6, %xmm26
-; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm29, %xmm2
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
-; AVX512DQ-BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm19, %zmm2
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm20, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm24, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $48, %zmm11, %zmm3
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
 ; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm1 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm28, %xmm2
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm8, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm30
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm0 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm15, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm13, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm2 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm17, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm6, %xmm4
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm3 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm12, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm31, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm5
 ; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vmovdqa (%rsp), %xmm5 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm15, %xmm8
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpbroadcastw {{.*#+}} xmm8 = [7,15,7,15,7,15,7,15,7,15,7,15,7,15,7,15]
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm9
-; AVX512DQ-BW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm6, %xmm13
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %ymm19, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
 ; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3,4],ymm5[5],ymm9[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm24, %xmm5
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm7, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm27, %xmm9
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm30, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm10, %zmm6
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm6, %xmm6
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
-; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm5
-; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm5 {%k1}
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm14, %xmm2
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm17, %xmm6
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm21, %xmm6
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm22, %xmm9
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm6[0],xmm9[1],xmm6[1],xmm9[2],xmm6[2],xmm9[3],xmm6[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm25, %xmm6
-; AVX512DQ-BW-NEXT:    vpshufb %xmm4, %xmm18, %xmm4
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm31, %xmm6
-; AVX512DQ-BW-NEXT:    vpshufb %xmm8, %xmm20, %xmm8
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5],ymm6[6,7]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm11, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm1, %xmm23, %xmm1
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm26, %xmm4
-; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm29, %xmm3
-; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1,2],xmm1[3]
-; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm19, %zmm3
-; AVX512DQ-BW-NEXT:    vpmovqb %zmm3, %xmm3
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm1
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%rcx)
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%r8)
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%r9)
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4],ymm4[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm25, %xmm4
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm16, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm27, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm8, %xmm6
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm9, %zmm5
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm5, %xmm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX512DQ-BW-NEXT:    vinserti32x4 $2, %xmm4, %zmm0, %zmm4
+; AVX512DQ-BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm4 {%k1}
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm21, %xmm1
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm14, %xmm5
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1],xmm5[2],xmm1[2],xmm5[3],xmm1[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm23, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm26, %xmm6
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5,6],ymm1[7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm28, %xmm5
+; AVX512DQ-BW-NEXT:    vpshufb %xmm3, %xmm18, %xmm3
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %ymm12, %ymm5
+; AVX512DQ-BW-NEXT:    vpmovqb %ymm5, %xmm5
+; AVX512DQ-BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm3[5],ymm5[6,7]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm10, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm0, %xmm22, %xmm0
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm20, %xmm3
+; AVX512DQ-BW-NEXT:    vpshufb %xmm2, %xmm24, %xmm2
+; AVX512DQ-BW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3]
+; AVX512DQ-BW-NEXT:    vpsrlq $56, %zmm11, %zmm2
+; AVX512DQ-BW-NEXT:    vpmovqb %zmm2, %xmm2
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512DQ-BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%rsi)
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%rdx)
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%rcx)
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%r8)
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%r9)
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-BW-NEXT:    vmovaps %zmm2, (%rax)
+; AVX512DQ-BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-BW-NEXT:    vmovaps %zmm1, (%rax)
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm30, (%rax)
 ; AVX512DQ-BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512DQ-BW-NEXT:    addq $744, %rsp # imm = 0x2E8
+; AVX512DQ-BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-BW-NEXT:    addq $552, %rsp # imm = 0x228
 ; AVX512DQ-BW-NEXT:    vzeroupper
 ; AVX512DQ-BW-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
index 58f6a66aeff7..c5e879c0135f 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
@@ -12,12 +12,12 @@
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver1 | FileCheck %s --check-prefixes=BMI
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver2 | FileCheck %s --check-prefixes=BMI
 ; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver3 | FileCheck %s --check-prefixes=BMI
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=BMI2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=BMI2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=BMI2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=BMI2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=BMI2
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=BMI2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=bdver4 | FileCheck %s --check-prefixes=BMI2-SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver1 | FileCheck %s --check-prefixes=BMI2-SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver2 | FileCheck %s --check-prefixes=BMI2-SLOW
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver3 | FileCheck %s --check-prefixes=BMI2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver4 | FileCheck %s --check-prefixes=BMI2-FAST
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=znver5 | FileCheck %s --check-prefixes=BMI2-FAST
 
 ; Verify that for the X86_64 processors that are known to have poor latency
 ; double precision shift instructions we do not generate 'shld' or 'shrd'
@@ -53,15 +53,23 @@ define i64 @lshift(i64 %a, i64 %b, i32 %c) nounwind readnone {
 ; BMI-NEXT:    orq %rdi, %rax
 ; BMI-NEXT:    retq
 ;
-; BMI2-LABEL: lshift:
-; BMI2:       # %bb.0: # %entry
-; BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
-; BMI2-NEXT:    shlxq %rdx, %rdi, %rcx
-; BMI2-NEXT:    notb %dl
-; BMI2-NEXT:    shrq %rsi
-; BMI2-NEXT:    shrxq %rdx, %rsi, %rax
-; BMI2-NEXT:    orq %rcx, %rax
-; BMI2-NEXT:    retq
+; BMI2-SLOW-LABEL: lshift:
+; BMI2-SLOW:       # %bb.0: # %entry
+; BMI2-SLOW-NEXT:    # kill: def $edx killed $edx def $rdx
+; BMI2-SLOW-NEXT:    shlxq %rdx, %rdi, %rcx
+; BMI2-SLOW-NEXT:    notb %dl
+; BMI2-SLOW-NEXT:    shrq %rsi
+; BMI2-SLOW-NEXT:    shrxq %rdx, %rsi, %rax
+; BMI2-SLOW-NEXT:    orq %rcx, %rax
+; BMI2-SLOW-NEXT:    retq
+;
+; BMI2-FAST-LABEL: lshift:
+; BMI2-FAST:       # %bb.0: # %entry
+; BMI2-FAST-NEXT:    movl %edx, %ecx
+; BMI2-FAST-NEXT:    movq %rdi, %rax
+; BMI2-FAST-NEXT:    # kill: def $cl killed $cl killed $ecx
+; BMI2-FAST-NEXT:    shldq %cl, %rsi, %rax
+; BMI2-FAST-NEXT:    retq
 entry:
   %sh_prom = zext i32 %c to i64
   %shl = shl i64 %a, %sh_prom
@@ -100,15 +108,23 @@ define i64 @rshift(i64 %a, i64 %b, i32 %c) nounwind readnone {
 ; BMI-NEXT:    orq %rdi, %rax
 ; BMI-NEXT:    retq
 ;
-; BMI2-LABEL: rshift:
-; BMI2:       # %bb.0: # %entry
-; BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
-; BMI2-NEXT:    shrxq %rdx, %rdi, %rcx
-; BMI2-NEXT:    notb %dl
-; BMI2-NEXT:    addq %rsi, %rsi
-; BMI2-NEXT:    shlxq %rdx, %rsi, %rax
-; BMI2-NEXT:    orq %rcx, %rax
-; BMI2-NEXT:    retq
+; BMI2-SLOW-LABEL: rshift:
+; BMI2-SLOW:       # %bb.0: # %entry
+; BMI2-SLOW-NEXT:    # kill: def $edx killed $edx def $rdx
+; BMI2-SLOW-NEXT:    shrxq %rdx, %rdi, %rcx
+; BMI2-SLOW-NEXT:    notb %dl
+; BMI2-SLOW-NEXT:    addq %rsi, %rsi
+; BMI2-SLOW-NEXT:    shlxq %rdx, %rsi, %rax
+; BMI2-SLOW-NEXT:    orq %rcx, %rax
+; BMI2-SLOW-NEXT:    retq
+;
+; BMI2-FAST-LABEL: rshift:
+; BMI2-FAST:       # %bb.0: # %entry
+; BMI2-FAST-NEXT:    movl %edx, %ecx
+; BMI2-FAST-NEXT:    movq %rdi, %rax
+; BMI2-FAST-NEXT:    # kill: def $cl killed $cl killed $ecx
+; BMI2-FAST-NEXT:    shrdq %cl, %rsi, %rax
+; BMI2-FAST-NEXT:    retq
 entry:
   %sh_prom = zext i32 %c to i64
   %shr = lshr i64 %a, %sh_prom
diff --git a/llvm/test/CodeGen/X86/x86-access-to-global.ll b/llvm/test/CodeGen/X86/x86-access-to-global.ll
new file mode 100644
index 000000000000..9e09a035ac51
--- /dev/null
+++ b/llvm/test/CodeGen/X86/x86-access-to-global.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i386-unknown-linux-gnu"
+
+@.str = external dso_local global i32
+
+define i1 @test() {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    calll .L0$pb
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 4
+; CHECK-NEXT:  .L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -4
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
+; CHECK-NEXT:    movl $.str@GOTOFF, %ecx
+; CHECK-NEXT:    addb %al, %cl
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retl
+  %i = ptrtoint ptr @.str to i8
+  %p = zext i8 %i to i32
+  %c = icmp eq i32 %p, 0
+  ret i1 %c
+}
diff --git a/llvm/test/CodeGen/Xtensa/float-arith.ll b/llvm/test/CodeGen/Xtensa/float-arith.ll
index 2b7186c44aeb..d9369a50c856 100644
--- a/llvm/test/CodeGen/Xtensa/float-arith.ll
+++ b/llvm/test/CodeGen/Xtensa/float-arith.ll
@@ -568,7 +568,6 @@ define float @round_f32(float %a) nounwind {
   ret float %res
 }
 
-
 define float @fneg_s(float %a) nounwind {
 ; XTENSA-LABEL: fneg_s:
 ; XTENSA:       # %bb.0:
@@ -601,3 +600,23 @@ define i32 @fptoui(float %f) {
   ret i32 %conv
 }
 
+define float @copysign_f32(float %a, float %b) {
+; XTENSA-LABEL: copysign_f32:
+; XTENSA:         .cfi_startproc
+; XTENSA-NEXT:  # %bb.0: # %entry
+; XTENSA-NEXT:    l32r a8, .LCPI35_0
+; XTENSA-NEXT:    and a8, a3, a8
+; XTENSA-NEXT:    l32r a9, .LCPI35_1
+; XTENSA-NEXT:    and a9, a2, a9
+; XTENSA-NEXT:    wfr f8, a9
+; XTENSA-NEXT:    movi a9, 0
+; XTENSA-NEXT:    beq a8, a9, .LBB35_2
+; XTENSA-NEXT:  # %bb.1:
+; XTENSA-NEXT:    neg.s f8, f8
+; XTENSA-NEXT:  .LBB35_2: # %entry
+; XTENSA-NEXT:    rfr a2, f8
+; XTENSA-NEXT:    ret
+entry:
+  %c = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/Xtensa/invalid-tls.ll b/llvm/test/CodeGen/Xtensa/invalid-tls.ll
new file mode 100644
index 000000000000..6aebca30b83d
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/invalid-tls.ll
@@ -0,0 +1,17 @@
+; RUN: not llc -mtriple=xtensa -mattr=+threadptr -relocation-model=pic -filetype=null < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=XTENSA-PIC %s
+; RUN: not llc -mtriple=xtensa -filetype=null < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=XTENSA-NO-THREADPTR %s
+
+; XTENSA-PIC: error: <unknown>:0:0: in function f i32 (): only local-exec and initial-exec TLS mode supported
+; XTENSA-PIC: error: <unknown>:0:0: in function f i32 (): PIC relocations are not supported
+
+; XTENSA-NO-THREADPTR: error: <unknown>:0:0: in function f i32 (): only emulated TLS supported
+
+@i = external thread_local global i32
+
+define i32 @f() {
+entry:
+  %tmp1 = load i32, ptr @i
+  ret i32 %tmp1
+}
diff --git a/llvm/test/CodeGen/Xtensa/select-cc-fp.ll b/llvm/test/CodeGen/Xtensa/select-cc-fp.ll
new file mode 100644
index 000000000000..ee45ef006123
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/select-cc-fp.ll
@@ -0,0 +1,457 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -mattr=+fp -disable-block-placement -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s
+
+define float @brcc_oeq(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_oeq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    oeq.s b0, f9, f8
+; CHECK-NEXT:    bf b0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI0_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI0_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp oeq float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ogt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ogt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ule.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI1_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI1_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ogt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_oge(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_oge:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ult.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI2_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI2_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp oge float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_olt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_olt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ule.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI3_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB3_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI3_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp olt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ole(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ole:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ult.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB4_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI4_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB4_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI4_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ole float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_one(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_one:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ueq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI5_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB5_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI5_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp one float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ord(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ord:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    un.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB6_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI6_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB6_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI6_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ord float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ueq(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ueq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ueq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB7_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI7_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB7_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI7_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ueq float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ugt(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ugt:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ole.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI8_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB8_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI8_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ugt float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_uge(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_uge:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    olt.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB9_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI9_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB9_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI9_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp uge float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ult(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ult:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    ole.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB10_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI10_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB10_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI10_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ult float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_ule(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_ule:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    olt.s b0, f8, f9
+; CHECK-NEXT:    bt b0, .LBB11_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI11_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB11_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI11_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp ule float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_une(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_une:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    oeq.s b0, f9, f8
+; CHECK-NEXT:    bt b0, .LBB12_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI12_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB12_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI12_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp une float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @brcc_uno(float %a, float %b) nounwind {
+; CHECK-LABEL: brcc_uno:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    wfr f8, a3
+; CHECK-NEXT:    wfr f9, a2
+; CHECK-NEXT:    un.s b0, f9, f8
+; CHECK-NEXT:    bf b0, .LBB13_2
+; CHECK-NEXT:  # %bb.1: # %t1
+; CHECK-NEXT:    l32r a8, .LCPI13_1
+; CHECK-NEXT:    wfr f8, a8
+; CHECK-NEXT:    add.s f8, f9, f8
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB13_2: # %t2
+; CHECK-NEXT:    l32r a8, .LCPI13_0
+; CHECK-NEXT:    wfr f9, a8
+; CHECK-NEXT:    add.s f8, f8, f9
+; CHECK-NEXT:    rfr a2, f8
+; CHECK-NEXT:    ret
+  %wb = fcmp uno float %a, %b
+  br i1 %wb, label %t1, label %t2
+t1:
+  %t1v = fadd float %a, 4.0
+  br label %exit
+t2:
+  %t2v = fadd float %b, 8.0
+  br label %exit
+exit:
+  %v = phi float [ %t1v, %t1 ], [ %t2v, %t2 ]
+  ret float %v
+}
+
+define float @copysign_f32(float %a, float %b) {
+entry:
+  %c = call float @llvm.copysign.f32(float %a, float %b)
+  ret float %c
+}
diff --git a/llvm/test/CodeGen/Xtensa/threadptr.ll b/llvm/test/CodeGen/Xtensa/threadptr.ll
new file mode 100644
index 000000000000..5e08addd840d
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/threadptr.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=xtensa -mattr=+threadptr < %s | FileCheck %s
+
+@i = external thread_local global i32
+
+define i32 @f() {
+; CHECK-LABEL: f:
+; CHECK:         .cfi_startproc
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    l32r a8, .LCPI0_0
+; CHECK-NEXT:    rur a9, threadptr
+; CHECK-NEXT:    add a8, a9, a8
+; CHECK-NEXT:    l32i a2, a8, 0
+; CHECK-NEXT:    ret
+entry:
+  %tmp1 = load i32, ptr @i
+  ret i32 %tmp1
+}
diff --git a/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll b/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll
index a8e900a7e91c..60a055ad66b6 100644
--- a/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll
+++ b/llvm/test/DebugInfo/AMDGPU/heterogeneous-dwarf-diop-diexpression-address-spaces.ll
@@ -102,6 +102,14 @@ define void @test_noop_convert(ptr addrspace(1) %p1) #0 !dbg !34 {
   ret void, !dbg !37
 }
 
+define void @test_noassert(ptr addrspace(1) %p1) #0 !dbg !38 {
+  ; Verify that this doesn't assert.
+    #dbg_value(ptr addrspace(1) %p1, !40, !DIExpression(DIOpArg(0, ptr addrspace(1)), DIOpConvert(ptr), DIOpReinterpret(i64), DIOpConstant(i64 1), DIOpAdd(), DIOpFragment(0, 32)), !41)
+    #dbg_value(i32 0, !40, !DIExpression(DIOpArg(0, i32), DIOpFragment(32, 16)), !41)
+    #dbg_value(i32 0, !40, !DIExpression(DW_OP_LLVM_poisoned, DW_OP_LLVM_fragment, 48, 16), !41)
+  ret void, !dbg !41
+}
+
 attributes #0 = { "frame-pointer"="all" }
 
 ; CHECK: [[PTR_AS_3]]: DW_TAG_pointer_type
@@ -160,3 +168,7 @@ attributes #0 = { "frame-pointer"="all" }
 !35 = !{!36}
 !36 = !DILocalVariable(name: "not_divergent", scope: !34, file: !1, line: 1, type: !14)
 !37 = !DILocation(line: 1, column: 1, scope: !34)
+!38 = distinct !DISubprogram(name: "test_noassert", linkageName: "test_noassert", scope: !1, file: !1, line: 1, type: !10, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !39)
+!39 = !{!40}
+!40 = !DILocalVariable(name: "frags", scope: !38, file: !1, line: 1, type: !14)
+!41 = !DILocation(line: 1, column: 1, scope: !38)
diff --git a/llvm/test/DebugInfo/AMDGPU/reg-sequence-salvage.ll b/llvm/test/DebugInfo/AMDGPU/reg-sequence-salvage.ll
index 0a4bd334a1a2..f284a49dd924 100644
--- a/llvm/test/DebugInfo/AMDGPU/reg-sequence-salvage.ll
+++ b/llvm/test/DebugInfo/AMDGPU/reg-sequence-salvage.ll
@@ -10,11 +10,11 @@ define i64 @test(ptr addrspace(1) %p) !dbg !11 {
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
-  ; CHECK-NEXT:   [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD killed [[COPY2]], 0, 0, implicit $exec, debug-instr-number 1 :: (load (s32) from %ir.p, addrspace 1)
-  ; CHECK-NEXT:   [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[GLOBAL_LOAD_DWORD]], implicit $exec, debug-instr-number 2
-  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[V_ASHRREV_I32_e64_]]
+  ; CHECK-NEXT:   [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD killed [[COPY2]], 0, 0, implicit $exec,  debug-instr-number 1 :: (load (s32) from %ir.p, addrspace 1)
+  ; CHECK-NEXT:   [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 31, [[GLOBAL_LOAD_DWORD]], implicit $exec,  debug-instr-number 2
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[V_ASHRREV_I32_e64_]]
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[GLOBAL_LOAD_DWORD]], %subreg.sub0, killed [[COPY3]], %subreg.sub1
-  ; CHECK-NEXT:   DBG_INSTR_REF !17, !DIExpression(DIOpArg(0, i32), DIOpArg(1, i32), DIOpComposite(2, i64)), dbg-instr-ref(1, 0), dbg-instr-ref(2, 0), debug-location !18
+  ; CHECK-NEXT:   DBG_INSTR_REF !17, !DIExpression(DIOpArg(0, i32), DIOpArg(1, i32), DIOpComposite(2, i64)), dbg-instr-ref(1, 0), dbg-instr-ref(2, 0),  debug-location !18
   ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[REG_SEQUENCE1]].sub1
   ; CHECK-NEXT:   $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
   ; CHECK-NEXT:   $vgpr1 = COPY [[COPY4]]
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/codegenprepare/sunk-addr.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/codegenprepare/sunk-addr.ll
index dc47f3f1920e..953548cc807a 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/codegenprepare/sunk-addr.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/codegenprepare/sunk-addr.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: x86-registered-target
 ; RUN: llc -start-before=codegenprepare -stop-after=codegenprepare \
 ; RUN:   -mtriple=x86_64-unknown-unknown %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg."
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that when CodeGenPrepare moves an address computation to a block it's
 ;; used in its dbg.assign uses are updated.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/dse/dse-after-memcpyopt-merge.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/dse/dse-after-memcpyopt-merge.ll
index 80212392ef85..ed8594be0d1d 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/dse/dse-after-memcpyopt-merge.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/dse/dse-after-memcpyopt-merge.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -S -passes=dse -o - | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: opt %s -S -passes=dse -o - | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Observed in the wild, but test is created by running memcpyopt on
 ;; assignment-tracking/memcpyopt/merge-stores.ll then manually inserting
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/dse/shorten-offset.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/dse/shorten-offset.ll
index 20019be7476d..0f6be3916a2d 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/dse/shorten-offset.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/dse/shorten-offset.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -S -passes=dse -o - | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: opt %s -S -passes=dse -o - | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Based on the test shorten.ll with some adjustments.
 ;;
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/inline/shared-alloca.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/inline/shared-alloca.ll
index 58b1cb71d9f0..6ea990090f4e 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/inline/shared-alloca.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/inline/shared-alloca.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S %s -passes=inline -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; The dbg.assign linked to the large alloca describes a variable sitting at
 ;; offset 0, size 64. Check:
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/do-not-remove-redundant-dbg.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/do-not-remove-redundant-dbg.ll
index e3b01e5354aa..26537e75d7a3 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/do-not-remove-redundant-dbg.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/do-not-remove-redundant-dbg.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=instcombine -S %s -o -  \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Based on the test remove-redundant-dbg.ll.
 ;;
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/remove-redundant-dbg.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/remove-redundant-dbg.ll
index 1e1ba56590d8..670f18b32834 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/remove-redundant-dbg.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/instcombine/remove-redundant-dbg.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that sroa removes redundant debug intrinsics after it makes a
 ;; change. This has a significant positive impact on peak memory and compiler
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/loop-vectorize/remove-redundant-dbg.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/loop-vectorize/remove-redundant-dbg.ll
index 130aff6501ad..a7a6c6be6ce1 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/loop-vectorize/remove-redundant-dbg.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/loop-vectorize/remove-redundant-dbg.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %s -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -S -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that loop-vectorize removes redundant debug intrinsics after it makes
 ;; a change. This has a significant positive impact on peak memory and compiler
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/phi.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/phi.ll
index 843f1bb75542..0397cf754415 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/phi.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/phi.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=mem2reg -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Test assignment tracking debug info when mem2reg promotes an alloca with
 ;; stores requiring insertion of a phi. Check the output when the stores are
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-block-alloca.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-block-alloca.ll
index 03337c27721d..6cef1d121c9a 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-block-alloca.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-block-alloca.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=mem2reg -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Test assignment tracking debug info when mem2reg promotes a single-block
 ;; alloca. Check the output when the stores are tagged and also untagged (test
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-store-alloca.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-store-alloca.ll
index ffe48412fb44..cc8bbb7bc2d9 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-store-alloca.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/single-store-alloca.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=mem2reg -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Test assignment tracking debug info when mem2reg promotes a single-store
 ;; alloca. Additionally, check that all the dbg.assigns linked to the alloca
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/store-to-part-of-alloca.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/store-to-part-of-alloca.ll
index eea921421686..ed6e1c0da375 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/store-to-part-of-alloca.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/mem2reg/store-to-part-of-alloca.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=mem2reg -S %s -o - | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: opt -passes=mem2reg -S %s -o - | FileCheck %s --implicit-check-not="#dbg_"
 
 ; CHECK: #dbg_value(i64 0, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 0, 32),
 
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/optnone.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/optnone.ll
index d1b395293ead..fbd34e63ae7d 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/optnone.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/optnone.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S %s -o - --passes=declare-to-assign \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Assignment tracking doesn't add any value when optimisations are disabled.
 ;; Check it doesn't get applied to functions marked optnone.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant-fwd-scan-linked.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant-fwd-scan-linked.ll
index 0a35bf71e1fa..fd52a866fe78 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant-fwd-scan-linked.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant-fwd-scan-linked.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=redundant-dbg-inst-elim -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; $ cat -n reduce.c
 ;;  1	void ext();
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant.ll
index 413bc3e3b4f6..693fb646e27c 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/remove-redundant.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=redundant-dbg-inst-elim -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Hand-written. Test how RemoveRedundantDbgInstrs interacts with dbg.assign
 ;; intrinsics. FileCehck directives are inline.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/salvage-value.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/salvage-value.ll
index 18009237022d..3f1a7269a948 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/salvage-value.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/salvage-value.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %s -S -o - -passes=instcombine \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Hand-written (the debug info doesn't necessarily make sense and isn't fully
 ;; formed). Test salvaging a dbg.assign value and address. Checks and comments
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/alloca-single-slice.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/alloca-single-slice.ll
index 8f8ac5d56b26..b12e6b6122a9 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/alloca-single-slice.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/alloca-single-slice.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa,verify -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ; Check that single sliced allocas retain their assignment tracking debug info.
 
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/complex.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/complex.ll
index 10c81cbe2598..0e8879e52ab9 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/complex.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/complex.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa -S -o - %s \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 ;
 ;; Based on llvm/test/DebugInfo/ARM/sroa-complex.ll
 ;; generated from:
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/fail-fragment.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/fail-fragment.ll
index 8898a814e22a..c10dba6d96e6 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/fail-fragment.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/fail-fragment.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that a dbg.assign for a promoted variable becomes a kill location if
 ;; it used a fragment that can't be split (the first check directive below).
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
index 6c3574de8ca4..ee7e9341ca59 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa,verify -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that the new slices of an alloca and memcpy intructions get dbg.assign
 ;; intrinsics with the correct fragment info.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/remove-redundant-dbg.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/remove-redundant-dbg.ll
index 1e1ba56590d8..670f18b32834 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/remove-redundant-dbg.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/remove-redundant-dbg.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that sroa removes redundant debug intrinsics after it makes a
 ;; change. This has a significant positive impact on peak memory and compiler
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
index 452ceaf1390f..73e954a27375 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa,verify -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ; Check that the new slices of an alloca and memset intructions get dbg.assign
 ; intrinsics with the correct fragment info. Ensure that only the
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store-2.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store-2.ll
index 49343e7576f3..c5a47f7b69d3 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store-2.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store-2.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -passes=sroa -sroa-skip-mem2reg %s \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; NOTE: This is the same as split-pre-fragmented-store.ll except the base
 ;; alloca's dbg.assign has been altered to contain a fragment of the full
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store.ll
index 5d1b74bd4304..b807f7e44989 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/split-pre-fragmented-store.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -S -passes=sroa -sroa-skip-mem2reg %s \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; IR hand-modified, originally generated from:
 ;; struct Pair { int a; int b; };
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/store.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/store.ll
index 53ea779de12d..5ea3ae474246 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/store.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/store.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa,verify -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ; Check that the new slices of an alloca and memset intructions get dbg.assign
 ; intrinsics with the correct fragment info. Ensure that only the
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
index af3070511e34..ded78f4ff83f 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/user-memcpy.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=sroa -S %s -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; Check that the fragments generated in SROA for a split alloca that has a
 ;; dbg.assign with non-zero-offset fragment are correct.
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/track-assignments.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/track-assignments.ll
index 709c6ca811ad..853ca65d3894 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/track-assignments.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/track-assignments.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes='declare-to-assign,verify' %s -S -o - \
-; RUN: | FileCheck %s --implicit-check-not="call void @llvm.dbg"
+; RUN: | FileCheck %s --implicit-check-not="#dbg_"
 
 ;; This test checks that `trackAssignments` is working correctly by using the
 ;; pass-wrapper `declare-to-assign`. Each function checks some specific
diff --git a/llvm/test/DebugInfo/Generic/debug-label.ll b/llvm/test/DebugInfo/Generic/debug-label.ll
index eff482a25ee0..0adb0608fafb 100644
--- a/llvm/test/DebugInfo/Generic/debug-label.ll
+++ b/llvm/test/DebugInfo/Generic/debug-label.ll
@@ -5,6 +5,7 @@
 ; CHECK-NEXT: DW_AT_name {{.*}}"top"
 ; CHECK-NEXT: DW_AT_decl_file [DW_FORM_data1] {{.*}}debug-label.c
 ; CHECK-NEXT: DW_AT_decl_line [DW_FORM_data1] {{.*}}4
+; CHECK-NEXT: DW_AT_decl_column [DW_FORM_data1] {{.*}}9
 ; CHECK-NEXT: DW_AT_low_pc [DW_FORM_addr] {{.*}}{{0x[0-9a-f]+}}
 ; CHECK: DW_TAG_label
 ; CHECK-NEXT: DW_AT_name {{.*}}"done"
@@ -22,6 +23,7 @@
 ; ASM-NEXT: DW_AT_name
 ; ASM: 1 {{.*}} DW_AT_decl_file
 ; ASM-NEXT: 4 {{.*}} DW_AT_decl_line
+; ASM-NEXT: 9 {{.*}} DW_AT_decl_column
 ; ASM-NEXT: [[TOP_LOW_PC]]{{.*}} DW_AT_low_pc
 ; ASM: DW_TAG_label
 ; ASM-NEXT: DW_AT_name
@@ -68,7 +70,7 @@ declare void @llvm.dbg.label(metadata)
 !7 = !DISubroutineType(types: !8)
 !8 = !{!9, !9, !9}
 !9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!10 = !DILabel(scope: !6, name: "top", file: !1, line: 4)
+!10 = !DILabel(scope: !6, name: "top", file: !1, line: 4, column: 9)
 !11 = !DILocation(line: 4, column: 1, scope: !6)
 !12 = !DILabel(scope: !15, name: "done", file: !1, line: 7)
 !13 = !DILocation(line: 7, column: 1, scope: !6)
diff --git a/llvm/test/DebugInfo/Generic/inline-alloca-ordering.ll b/llvm/test/DebugInfo/Generic/inline-alloca-ordering.ll
index f696c9397ba8..9585b171c083 100644
--- a/llvm/test/DebugInfo/Generic/inline-alloca-ordering.ll
+++ b/llvm/test/DebugInfo/Generic/inline-alloca-ordering.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s --passes=inline -o - -S | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt %s --passes=inline -o - -S | FileCheck %s --implicit-check-not=dbg_value
 
 ;; The inliner, specially, hoists all alloca instructions into the entry block
 ;; of the calling function. Ensure that it doesn't accidentally transfer the
diff --git a/llvm/test/DebugInfo/Generic/inline-dbg-values.ll b/llvm/test/DebugInfo/Generic/inline-dbg-values.ll
index 4710a10c9093..304e5810b88c 100644
--- a/llvm/test/DebugInfo/Generic/inline-dbg-values.ll
+++ b/llvm/test/DebugInfo/Generic/inline-dbg-values.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes='cgscc(inline)' -S %s -o - -S | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt -passes='cgscc(inline)' -S %s -o - -S | FileCheck %s --implicit-check-not=dbg_value
 
 ;; Test that dbg.value intrinsics are inlined, remapped, and have their
 ;; dilocation updated just like normal instructions. This becomes
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
index 33f7f673d91c..125ce1692758 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/inline-nodbg.ll
@@ -11,6 +11,8 @@
 
 ; CHECK: _Z1bv()
 ; CHECK: store i32 1, ptr @g, align 4, !dbg [[DBG:!.*]]
+
+; CHECK: distinct !DISubprogram(name: "b", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG]] = !DILocation(line: 3, scope: ![[#]])
 
 @g = hidden global i32 0, align 4
@@ -36,7 +38,7 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!15 = distinct !DISubprogram(name: "b", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!15 = distinct !DISubprogram(name: "b", scope: !1, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !16 = !DISubroutineType(types: !17)
 !17 = !{}
 !18 = !DILocation(line: 3, scope: !15, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
index b9b6abdd1e9c..0d5cc6d4a44d 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/inline.ll
@@ -8,6 +8,7 @@
 ; CHECK: %add.i = add nsw i32 %mul.i, 1, !dbg [[G1R2:!.*]]
 ; CHECK-NEXT: store i32 %add.i, ptr %x.i, align 4, !dbg [[G1R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "g", {{.*}}keyInstructions: true)
 ; CHECK: [[G1R2]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 2)
 ; CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
 
@@ -41,11 +42,10 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
 !19 = !DILocation(line: 2, scope: !11, atomGroup: 1, atomRank: 2)
 !20 = !DILocation(line: 2, scope: !11, atomGroup: 1, atomRank: 1)
-!23 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!23 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 4, type: !12, scopeLine: 4, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !24 = !DILocation(line: 5, scope: !23)
-
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
index 612e74792fac..175272e4e8bb 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-2-bbs.ll
@@ -27,6 +27,7 @@
 ; CHECK: bb.f2:
 ; CHECK-NEXT: store ptr @a, ptr %p, align 4, !dbg [[G4R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "foo", {{.*}}keyInstructions: true)
 ; CHECK: [[G1R2]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 2)
 ; CHECK: [[G1R1]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 1)
 ; CHECK: [[G3R2]] = !DILocation(line: 1, column: 1, scope: ![[#]], atomGroup: 3, atomRank: 2)
@@ -76,7 +77,7 @@ declare void @f2()
 !2 = !{i32 16}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !9 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
index 1541df346908..f285ebbdfda3 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-basic.ll
@@ -26,6 +26,7 @@
 ; CHECK-NEXT: %C = add i32 %v2, 1, !dbg [[G1R2:!.*]]
 ; CHECK-NEXT: store i32 %C, ptr %p, align 4, !dbg [[G1R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "test1", {{.*}}keyInstructions: true)
 ; CHECK: [[G2R2]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 2, atomRank: 2)
 ; CHECK: [[G2R1]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 2, atomRank: 1)
 ; CHECK: [[G1R2]] = !DILocation(line: 8, column: 1, scope: ![[#]], atomGroup: 1, atomRank: 2)
@@ -70,7 +71,7 @@ declare void @f3()
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 8, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
index 1ab1c1ac2e9f..a8a34d069894 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/jump-threading-dup-cond-br-on-phi-into-pred.ll
@@ -30,6 +30,7 @@
 ; CHECK-NEXT: %C = add i32 %v2, 1, !dbg [[G2R2:!.*]]
 ; CHECK-NEXT: store i32 %C, ptr %p, align 4, !dbg [[G2R1:!.*]]
 
+; CHECK: distinct !DISubprogram(name: "test5", {{.*}}keyInstructions: true)
 ; CHECK: [[G3R2]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 2)
 ; CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G1R1]] = !DILocation({{.*}}, atomGroup: 1, atomRank: 1)
@@ -79,7 +80,7 @@ declare void @f3()
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test5", linkageName: "test5", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test5", linkageName: "test5", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll
new file mode 100644
index 000000000000..104d75d8648b
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/link-two-modes.ll
@@ -0,0 +1,67 @@
+; RUN: split-file %s %t
+; RUN: llvm-as %t/key-instr-enabled.ll -o %t/key-instr-enabled.bc
+; RUN: llvm-as %t/key-instr-disabled.ll -o %t/key-instr-disabled.bc
+; RUN: llvm-link %t/key-instr-enabled.bc %t/key-instr-disabled.bc -o - | llvm-dis | FileCheck %s
+
+;; Check the Key Instructions metadata is preserved correctly when linking a
+;; modules with Key Instructions enabled/disabled.
+
+;; Key Instructions enabled.
+; CHECK: void @f() !dbg [[f:!.*]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret void, !dbg [[enabled:!.*]]
+; CHECK-NEXT: }
+
+;; Key Instructions disabled.
+; CHECK: void @g() !dbg [[g:!.*]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret void, !dbg [[disabled:!.*]]
+; CHECK-NEXT: }
+
+; CHECK: !llvm.dbg.cu = !{!0, !2}
+; CHECK-NEXT: !llvm.module.flags = !{!4}
+
+; CHECK: [[file1:!.*]] = !DIFile(filename: "key-instr-enabled.cpp", directory: "/")
+; CHECK: [[file2:!.*]] = !DIFile(filename: "key-instr-disabled.cpp", directory: "/")
+; CHECK: [[f]] = distinct !DISubprogram(name: "f", scope: [[file1]]{{.*}},  keyInstructions: true)
+; CHECK: [[enabled]] = !DILocation(line: 1, column: 11, scope: [[f]], atomGroup: 1, atomRank: 1)
+; CHECK: [[g]] = distinct !DISubprogram(name: "g", scope: [[file2]]
+; CHECK-NOT:                            keyInstructions
+; CHECK-SAME:                           )
+; CHECK: [[disabled]] = !DILocation(line: 1, column: 11, scope: [[g]])
+
+;--- key-instr-enabled.ll
+define dso_local void @f() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr-enabled.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
+
+;--- key-instr-disabled.ll
+define dso_local void @g() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr-disabled.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: false)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
index 6da7146b5b07..0ea0453c4a6c 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-rotate.ll
@@ -25,6 +25,7 @@
 ; CHECK:    [[CMP:%.*]] = icmp slt i32 {{.*}}, 100, !dbg [[G2R2:![0-9]+]]
 ; CHECK:    br i1 [[CMP]], label %for.body, label %for.end, !dbg [[G2R1:![0-9]+]]
 ;
+; CHECK: distinct !DISubprogram(name: "test1", {{.*}}keyInstructions: true)
 ; CHECK: [[G3R1]] = !DILocation(line: 4{{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G4R1]] = !DILocation(line: 6{{.*}}, atomGroup: 4, atomRank: 1)
 ; CHECK: [[G1R1]] = !DILocation(line: 4{{.*}}, atomGroup: 1, atomRank: 1)
@@ -69,6 +70,6 @@ attributes #1 = { noduplicate }
 !2 = !{i32 12}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test1", linkageName: "test1", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
index 6deb04cc00f8..d23afaed9a04 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll-runtime.ll
@@ -25,6 +25,7 @@
 ; CHECK-NEXT: store i64 %indvars.iv.next.2, ptr %p, align 4, !dbg [[G7R1:!.*]]
 ; CHECK-NEXT: %indvars.iv.next.3 = add nuw nsw i64 %indvars.iv, 4
 
+; CHECK: distinct !DISubprogram(name: "unroll", {{.*}}keyInstructions: true)
 ; CHECK: [[G2R1]] = !DILocation({{.*}}, atomGroup: 2, atomRank: 1)
 ; CHECK: [[G3R1]] = !DILocation({{.*}}, atomGroup: 3, atomRank: 1)
 ; CHECK: [[G4R1]] = !DILocation({{.*}}, atomGroup: 4, atomRank: 1)
@@ -63,7 +64,7 @@ for.body:                                         ; preds = %for.body, %for.body
 !2 = !{i32 17}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "unroll", linkageName: "unroll", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "unroll", linkageName: "unroll", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
index 557e8387e5bc..3b13bde8bfb3 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unroll.ll
@@ -41,7 +41,7 @@ attributes #0 = { nounwind ssp uwtable }
 !2 = !{i32 8}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "f", linkageName: "f", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !11 = !DILocation(line: 4, column: 1, scope: !5, atomGroup: 1, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
index 557950439ef3..fee7e845d4df 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
@@ -125,7 +125,7 @@ declare void @clobber()
 !2 = !{i32 13}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", linkageName: "partial_unswitch_true_successor_hoist_invariant", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", linkageName: "partial_unswitch_true_successor_hoist_invariant", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, scope: !5, atomGroup: 1, atomRank: 1)
@@ -142,6 +142,7 @@ declare void @clobber()
 !19 = !DILocation(line: 12, scope: !5, atomGroup: 12, atomRank: 1)
 !20 = !DILocation(line: 13, scope: !5, atomGroup: 13, atomRank: 1)
 ;.
+; CHECK: distinct !DISubprogram(name: "partial_unswitch_true_successor_hoist_invariant", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG8]] = !DILocation(line: 3{{.*}}, atomGroup: 24, atomRank: 1)
 ; CHECK: [[DBG9]] = !DILocation(line: 4{{.*}}, atomGroup: 25, atomRank: 1)
 ; CHECK: [[DBG10]] = !DILocation(line: 5{{.*}}, atomGroup: 26, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
index 33219a582fee..79f7a7d79610 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/parse.ll
@@ -1,5 +1,6 @@
 ; RUN: opt %s -o - -S| FileCheck %s
 
+; CHECK: distinct !DISubprogram(name: "f", {{.*}}keyInstructions: true)
 ; CHECK: !DILocation(line: 1, column: 11, scope: ![[#]], atomGroup: 1, atomRank: 1)
 
 define dso_local void @f() !dbg !10 {
@@ -13,7 +14,7 @@ entry:
 !1 = !DIFile(filename: "test.cpp", directory: "/")
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{!"clang version 21.0.0git"}
-!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
 !11 = !DISubroutineType(types: !12)
 !12 = !{null}
 !13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll
new file mode 100644
index 000000000000..79a06e51b264
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/roundtrip.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -o - -S  | llvm-as - | llvm-dis - | FileCheck %s
+
+; Key Instructions enabled.
+define dso_local void @f() !dbg !10 {
+; CHECK-LABEL: define dso_local void @f(
+; CHECK-SAME: ) !dbg [[DBG3:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret void, !dbg [[DBG6:![0-9]+]]
+;
+entry:
+  ret void, !dbg !13
+}
+
+; Key Instructions disabled.
+define dso_local void @g() !dbg !14 {
+; CHECK-LABEL: define dso_local void @g(
+; CHECK-SAME: ) !dbg [[DBG7:![0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    ret void, !dbg [[DBG8:![0-9]+]]
+;
+entry:
+  ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "key-instr.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: true)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, scope: !10, atomGroup: 1, atomRank: 1)
+!14 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 2, type: !11, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, keyInstructions: false)
+!15 = !DILocation(line: 2, scope: !14)
+;.
+; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}key-instr.cpp", directory: {{.*}})
+; CHECK: [[DBG3]] = distinct !DISubprogram(name: "f", scope: [[META1]], file: [[META1]], line: 1, type: [[META4:![0-9]+]], scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META0]], keyInstructions: true)
+; CHECK: [[META4]] = !DISubroutineType(types: [[META5:![0-9]+]])
+; CHECK: [[META5]] = !{null}
+; CHECK: [[DBG6]] = !DILocation(line: 1, scope: [[DBG3]], atomGroup: 1, atomRank: 1)
+; CHECK: [[DBG7]] = distinct !DISubprogram(name: "g", scope: [[META1]], file: [[META1]], line: 2, type: [[META4]], scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META0]])
+; CHECK: [[DBG8]] = !DILocation(line: 2, scope: [[DBG7]])
+;.
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
index 75a05ad93170..3ad956cabdb2 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
@@ -70,7 +70,7 @@ f:
 !4 = !{i32 0}
 !5 = !{i32 2, !"Debug Info Version", i32 3}
 !7 = !DISubroutineType(types: !2)
-!17 = distinct !DISubprogram(name: "f", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!17 = distinct !DISubprogram(name: "f", scope: null, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2, keyInstructions: true)
 !18 = !DILocation(line: 10, column: 10, scope: !17)
 !19 = !DILocation(line: 10, column: 10, scope: !17, atomGroup: 1, atomRank: 2)
 !20 = !DILocation(line: 10, column: 10, scope: !17, atomGroup: 2, atomRank: 2)
@@ -79,7 +79,7 @@ f:
 ; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: [[META2:![0-9]+]])
 ; CHECK: [[META1]] = !DIFile(filename: "a.ll", directory: {{.*}})
 ; CHECK: [[META2]] = !{}
-; CHECK: [[DBG6]] = distinct !DISubprogram(name: "f", scope: null, file: [[META1]], line: 1, type: [[META7:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]])
+; CHECK: [[DBG6]] = distinct !DISubprogram(name: "f", scope: null, file: [[META1]], line: 1, type: [[META7:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META2]], keyInstructions: true)
 ; CHECK: [[META7]] = !DISubroutineType(types: [[META2]])
 ; CHECK: [[DBG8]] = !DILocation(line: 10, column: 10, scope: [[DBG6]], atomGroup: 5, atomRank: 2)
 ; CHECK: [[DBG9]] = !DILocation(line: 10, column: 10, scope: [[DBG6]], atomGroup: 6, atomRank: 2)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
index f8477600c641..774c637e3b83 100644
--- a/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/simplifycfg-thread-phi.ll
@@ -13,6 +13,8 @@
 ; CHECK:   store i32 1{{.*}}, !dbg [[DBG1:!.*]]
 ; CHECK: if.end.1.critedge:
 ; CHECK:   store i32 1{{.*}}, !dbg [[DBG2:!.*]]
+;
+; CHECK: distinct !DISubprogram(name: "bar", {{.*}}keyInstructions: true)
 ; CHECK: [[DBG1]] = !DILocation(line: 1{{.*}}, atomGroup: 1
 ; CHECK: [[DBG2]] = !DILocation(line: 1{{.*}}, atomGroup: 2
 
@@ -56,7 +58,7 @@ declare void @foo()
 !2 = !{i32 15}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "bar", linkageName: "bar", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !8 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll b/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll
new file mode 100644
index 000000000000..0f8f505c51a5
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/Generic/verify.ll
@@ -0,0 +1,22 @@
+; RUN: opt %s -o - -S --passes=verify 2>&1 | FileCheck %s
+
+; CHECK: DbgLoc uses atomGroup but DISubprogram doesn't have Key Instructions enabled
+; CHECK-NEXT: ![[#]] = !DILocation(line: 1, column: 11, scope: ![[f:.*]], atomGroup: 1, atomRank: 1)
+; CHECK-NEXT: ![[f]] = distinct !DISubprogram(name: "f"
+; CHECK-NEXT: warning: ignoring invalid debug info
+
+define dso_local void @f() !dbg !10 {
+entry:
+  ret void, !dbg !13
+}
+
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 21.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 21.0.0git"}
+!10 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 1, column: 11, scope: !10, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll b/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
index a21b1da15a45..dcacd96e69a1 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/cgp-break-critical-edge.ll
@@ -46,7 +46,7 @@ if.else1:                                         ; preds = %if.else1, %preheade
 !2 = !{i32 11}
 !3 = !{i32 0}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!5 = distinct !DISubprogram(name: "test", linkageName: "test", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !6 = !DISubroutineType(types: !7)
 !7 = !{}
 !12 = !DILocation(line: 1, column: 1, scope: !5, atomGroup: 1, atomRank: 1)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
index 6988a6fb2ec3..154bb872684f 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic-ranks.ll
@@ -64,6 +64,6 @@ declare void @_Z12prologue_endv() local_unnamed_addr #1
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
index ba38bb2ffe90..b97b436ffc57 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-basic.ll
@@ -55,8 +55,8 @@ entry:
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
-!16 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!16 = distinct !DISubprogram(name: "g", scope: !1, file: !1, line: 5, type: !12, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !18 = distinct !DILocation(line: 6, scope: !16)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
index 3b7d5850e7e8..abe27e9870cd 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy-multi-key.mir
@@ -48,7 +48,7 @@
   !2 = !{i32 7, !"Dwarf Version", i32 5}
   !3 = !{i32 2, !"Debug Info Version", i32 3}
   !4 = !{!"clang version 19.0.0"}
-  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
   !6 = !DISubroutineType(types: !7)
   !7 = !{}
   !8 = !DILocalVariable(name: "x", scope: !5, file: !1, line: 1, type: !7)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
index 3aff5e22326e..222604493cc8 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-buoy.mir
@@ -44,7 +44,7 @@
   !2 = !{i32 7, !"Dwarf Version", i32 5}
   !3 = !{i32 2, !"Debug Info Version", i32 3}
   !4 = !{!"clang version 19.0.0"}
-  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+  !5 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
   !6 = !DISubroutineType(types: !7)
   !7 = !{}
   !8 = !DILocalVariable(name: "x", scope: !5, file: !1, line: 1, type: !7)
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
index 14dfa8a6a9cc..72fc4db6402a 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-calls.ll
@@ -109,6 +109,6 @@ declare i32  @g() local_unnamed_addr
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "fun", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir
new file mode 100644
index 000000000000..273d5cbdfda2
--- /dev/null
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-inline-modes.mir
@@ -0,0 +1,93 @@
+# RUN: llc %s --start-after=livedebugvalues --dwarf-use-key-instructions --filetype=obj -o - \
+# RUN: | llvm-objdump -d - --no-show-raw-insn \
+# RUN: | FileCheck %s --check-prefix=OBJ
+
+# RUN: llc %s --start-after=livedebugvalues --dwarf-use-key-instructions --filetype=obj -o - \
+# RUN: | llvm-dwarfdump - --debug-line \
+# RUN: | FileCheck %s --check-prefix=DBG
+
+# Test is_stmt emission for a key-instructions function inlined into a
+# not-key-instructions function, and vice versa.
+
+--- |
+  target triple = "x86_64-unknown-linux-gnu"
+
+  define hidden noundef i32 @key() local_unnamed_addr !dbg !5 {
+  entry:
+    ret i32 0
+  }
+
+  define hidden noundef i32 @not_key() local_unnamed_addr !dbg !9 {
+  entry:
+    ret i32 0
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2, !3}
+  !llvm.ident = !{!4}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_17, file: !1, producer: "clang version 21.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false, nameTableKind: None)
+  !1 = !DIFile(filename: "test.cpp", directory: "/")
+  !2 = !{i32 7, !"Dwarf Version", i32 5}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !4 = !{!"clang version 21.0.0"}
+  !5 = distinct !DISubprogram(name: "key", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{}
+  !9 = distinct !DISubprogram(name: "not_key", scope: !1, file: !1, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: false)
+  !10 = distinct !DILocation(line: 5, scope: !5)
+  !11 = distinct !DILocation(line: 9, scope: !9)
+...
+---
+name:            key
+alignment:       16
+body:             |
+  bb.0.entry:
+
+    ; OBJ: 0000000000000000 <key>:
+    ; OBJ-NEXT:  0: movl $0x1, %eax
+    ; OBJ-NEXT:  5: movl $0x2, %eax
+    ; OBJ-NEXT:  a: movl $0x3, %eax
+    ; OBJ-NEXT:  f: movl $0x4, %eax
+    ; OBJ-NEXT: 14: movl $0x5, %eax
+    ; OBJ-NEXT: 19: retq
+    ;
+    ; DBG:      Address            Line   Column File   ISA Discriminator OpIndex Flags
+    ; DBG-NEXT: ------------------ ------ ------ ------ --- ------------- ------- -------------
+    ; DBG-NEXT: 0x0000000000000000      1      0      0   0             0       0  is_stmt prologue_end
+    ; DBG-NEXT: 0x0000000000000005      2      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x000000000000000a      2      0      0   0             0       0
+    ; DBG-NEXT: 0x000000000000000f      3      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x0000000000000014      3      0      0   0             0       0
+    ;
+    $eax = MOV32ri 1,  debug-location !DILocation(line: 1, scope: !5)                            ; is_stmt (prologue_end)
+    $eax = MOV32ri 2,  debug-location !DILocation(line: 2, scope: !5, atomGroup: 1, atomRank: 1) ; is_stmt (key)
+    $eax = MOV32ri 3,  debug-location !DILocation(line: 2, scope: !9, inlinedAt: !10)
+    $eax = MOV32ri 4,  debug-location !DILocation(line: 3, scope: !9, inlinedAt: !10)            ; is_stmt (not_key)
+    $eax = MOV32ri 5,  debug-location !DILocation(line: 3, scope: !5, atomGroup: 1, atomRank: 2) ; is_stmt (key)
+    RET64 $eax,        debug-location !DILocation(line: 3, scope: !5, atomGroup: 1, atomRank: 1)
+...
+---
+name:            not_key
+alignment:       16
+body:             |
+  bb.0.entry:
+
+    ; OBJ: 0000000000000020 <not_key>:
+    ; OBJ-NEXT: 20: movl $0x1, %eax
+    ; OBJ-NEXT: 25: movl $0x2, %eax
+    ; OBJ-NEXT: 2a: movl $0x3, %eax
+    ; OBJ-NEXT: 2f: retq
+    ;
+    ;           Address            Line   Column File   ISA Discriminator OpIndex Flags
+    ;           ------------------ ------ ------ ------ --- ------------- ------- -------------
+    ; DBG-NEXT: 0x0000000000000020      1      0      0   0             0       0  is_stmt prologue_end
+    ; DBG-NEXT: 0x0000000000000025      2      0      0   0             0       0
+    ; DBG-NEXT: 0x000000000000002a      3      0      0   0             0       0  is_stmt
+    ; DBG-NEXT: 0x000000000000002f      3      0      0   0             0       0
+    ;
+    $eax = MOV32ri 1,  debug-location !DILocation(line: 1, scope: !9)                                            ; is_stmt (prologue_end)
+    $eax = MOV32ri 2,  debug-location !DILocation(line: 2, scope: !5, inlinedAt: !11, atomGroup: 1, atomRank: 2)
+    $eax = MOV32ri 3,  debug-location !DILocation(line: 3, scope: !5, inlinedAt: !11, atomGroup: 1, atomRank: 1) ; is_stmt (key)
+    RET64 $eax,        debug-location !DILocation(line: 3, scope: !9)
+...
diff --git a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
index cd840ecce07f..e9067e448942 100644
--- a/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/X86/dwarf-ranks-blocks.ll
@@ -60,6 +60,6 @@ declare void @_Z12prologue_endv() local_unnamed_addr #1
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{!"clang version 19.0.0"}
-!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!11 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !12, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, keyInstructions: true)
 !12 = !DISubroutineType(types: !13)
 !13 = !{}
diff --git a/llvm/test/DebugInfo/KeyInstructions/debugify.ll b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
index 881375873f32..551ae2794c2f 100644
--- a/llvm/test/DebugInfo/KeyInstructions/debugify.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
@@ -38,9 +38,12 @@ define i32 @boom() {
   ret i32 %retval
 }
 
+; CHECK: distinct !DISubprogram(name: "foo", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[RET1]] = !DILocation(line: 1, {{.*}}, atomGroup: 1, atomRank: 1
+; CHECK: distinct !DISubprogram(name: "bar", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[CALL1]] = !DILocation(line: 2, {{.*}}, atomGroup: 2, atomRank: 1
 ; CHECK-DAG: ![[ADD1]] = !DILocation(line: 3, {{.*}}, atomGroup: 3, atomRank: 1
 ; CHECK-DAG: ![[RET2]] = !DILocation(line: 4, {{.*}}, atomGroup: 4, atomRank: 1
+; CHECK: distinct !DISubprogram(name: "boom", {{.*}}keyInstructions: true)
 ; CHECK-DAG: ![[musttail]] = !DILocation(line: 5, {{.*}}, atomGroup: 5, atomRank: 1
 ; CHECK-DAG: ![[musttailRes]] = !DILocation(line: 6, {{.*}}, atomGroup: 6, atomRank: 1
diff --git a/llvm/test/DebugInfo/MIR/AArch64/entry_value_gets_propagated_aarch64.mir b/llvm/test/DebugInfo/MIR/AArch64/entry_value_gets_propagated_aarch64.mir
new file mode 100644
index 000000000000..7366ba4de64f
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/AArch64/entry_value_gets_propagated_aarch64.mir
@@ -0,0 +1,84 @@
+# RUN: llc --run-pass=livedebugvalues -o - %s | FileCheck %s --implicit-check-not=DBG_VALUE 
+# REQUIRES: aarch64-registered-target
+
+# This test covers the scenario where a DBG_VALUE created prior to LiveDebugValues has an entry-value expression.
+# It ensures that a clobbered stack copy doesn't crash if used as an entry-value because entry-values can't be clobbered.
+
+--- |
+  target triple = "aarch64-"
+  define i32 @baz(i32 swiftasync %arg1, i32 noundef %arg2, i1 %cond) !dbg !4 {
+    br i1 %cond, label %if.then, label %if.else, !dbg !14
+  if.then:                                          ; preds = %0
+    %call = call i32 @foo(i32 noundef %arg1), !dbg !15
+    br label %if.end, !dbg !18
+  if.else:                                          ; preds = %0
+    %call1 = call i32 @foo(i32 noundef %arg2), !dbg !19
+    br label %if.end
+  if.end:                                           ; preds = %if.else, %if.then
+    %temp.0 = phi i32 [ %call, %if.then ], [ %call1, %if.else ], !dbg !21
+    ret i32 %temp.0, !dbg !22
+  }
+  declare i32 @foo(i32)
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2, !3}
+  !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "ha", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+  !1 = !DIFile(filename: "test.c", directory: "hah")
+  !2 = !{i32 7, !"Dwarf Version", i32 4}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !4 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 3, type: !5, scopeLine: 3, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !8)
+  !5 = !DISubroutineType(types: !6)
+  !6 = !{!7, !7, !7, !7}
+  !7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !8 = !{!9, !10, !11, !12}
+  !9 = !DILocalVariable(name: "arg1", arg: 1, scope: !4, file: !1, line: 3, type: !7)
+  !10 = !DILocalVariable(name: "arg2", arg: 2, scope: !4, file: !1, line: 3, type: !7)
+  !11 = !DILocalVariable(name: "cond", arg: 3, scope: !4, file: !1, line: 3, type: !7)
+  !12 = !DILocalVariable(name: "local", scope: !4, file: !1, line: 4, type: !7)
+  !13 = !DILocation(line: 0, scope: !4)
+  !14 = !DILocation(line: 7, column: 7, scope: !4)
+  !15 = !DILocation(line: 8, column: 12, scope: !16)
+  !16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 7, column: 13)
+  !17 = distinct !DILexicalBlock(scope: !4, file: !1, line: 7, column: 7)
+  !18 = !DILocation(line: 9, column: 3, scope: !16)
+  !19 = !DILocation(line: 10, column: 12, scope: !20)
+  !20 = distinct !DILexicalBlock(scope: !17, file: !1, line: 9, column: 10)
+  !21 = !DILocation(line: 0, scope: !17)
+  !22 = !DILocation(line: 13, column: 3, scope: !4)
+name:            baz
+debugInstrRef:   true
+stack:
+  - { id: 1, name: '', type: spill-slot, offset: -24, size: 4, alignment: 4, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0 (%ir-block.0):
+    DBG_VALUE $w1, $noreg, !12, !DIExpression(DW_OP_LLVM_entry_value, 1),  debug-location !13
+
+  bb.1.if.then:
+    $w0 = LDRWui $sp, 2
+    BL @foo, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit-def $w0,  debug-location !15
+    $w1 = MOVi32imm 0
+
+  bb.2.if.else:
+    $w0 = LDRWui $sp, 3
+    BL @foo, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit-def $w0,  debug-location !19
+    STRWui killed $w0, $sp, 1
+    B %bb.3
+  
+  bb.3.if.end:
+    $w0 = LDRWui $sp, 1
+    $fp, $lr = frame-destroy LDPXi $sp, 2,  debug-location !22
+    $sp = frame-destroy ADDXri $sp, 32, 0,  debug-location !22
+    RET undef $lr, implicit killed $w0,  debug-location !22
+
+# CHECK-LABEL: bb.0
+# CHECK: DBG_VALUE $w1, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.1.if.then:
+# CHECK: DBG_VALUE $w1, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-NEXT: $w0 = LDRWui $sp, 2
+# CHECK-NEXT: BL @foo
+# CHECK-NEXT: $w1 = MOVi32imm 0
+# CHECK-NOT: DBG_VALUE $w1, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.2.if.else:
+# CHECK: DBG_VALUE $w1, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.3.if.end:
+# CHECK: DBG_VALUE $w1, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
\ No newline at end of file
diff --git a/llvm/test/DebugInfo/MIR/X86/entry_value_clobbered_stack_copy.mir b/llvm/test/DebugInfo/MIR/X86/entry_value_clobbered_stack_copy.mir
new file mode 100644
index 000000000000..8fe3f7b75a76
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/X86/entry_value_clobbered_stack_copy.mir
@@ -0,0 +1,52 @@
+# RUN: llc --run-pass=livedebugvalues -o - %s | FileCheck %s
+# REQUIRES: x86-registered-target
+
+# This test covers the scenario that saves a register on the stack, uses this register as an entry value DBG_VALUE location, and then clobbers it.
+
+--- |
+  target triple = "x86_64-"
+  define void @foo(ptr swiftasync %0) !dbg !4 {
+    call void @llvm.dbg.value(metadata ptr %0, metadata !9, metadata !DIExpression(DW_OP_LLVM_entry_value, 1)), !dbg !17
+    ret void
+  }
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+  !llvm.module.flags = !{!0}
+  !llvm.dbg.cu = !{!1}
+
+  !0 = !{i32 2, !"Debug Info Version", i32 3}
+  !1 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !2, producer: "blah", isOptimized: true, flags: "blah", runtimeVersion: 5, emissionKind: FullDebug)
+  !2 = !DIFile(filename: "blah", directory: "blah")
+  !3 = !{}
+  !4 = distinct !DISubprogram(name: "blah", linkageName: "blah", scope: !2, file: !2, line: 284, type: !7, unit: !1)
+  !7 = !DISubroutineType(types: !3)
+  !9 = !DILocalVariable(name: "self", arg: 3, scope: !4, file: !2, line: 328, type: !12, flags: DIFlagArtificial)
+  !12 = !DICompositeType(tag: DW_TAG_structure_type, name: "blah", scope: !2, file: !2, size: 64, elements: !3)
+  !17 = !DILocation(line: 328, column: 17, scope: !4)
+
+...
+---
+name:            foo
+alignment:       16
+debugInstrRef:   true
+tracksDebugUserValues: true
+liveins:
+  - { reg: '$r14', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -64, size: 8, alignment: 8,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0:
+    liveins: $r14
+    ; Put a copy of r14 on the stack.
+    MOV64mr $rbp, 1, $noreg, -48, $noreg, $r14 :: (store (s64) into %stack.0)
+    DBG_VALUE $r14, $noreg, !9, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !17
+    MOV64mi32 $noreg, 1, $noreg, 0, $noreg, 0, debug-location !17 :: (store (s64) into `ptr null`)
+    $r14 = MOV64rr killed $r13
+    ; Clobber $r14
+    RETI64 24
+# CHECK: bb.0:
+# CHECK:      MOV64mr $rbp, 1, $noreg, -48, $noreg, $r14
+# CHECK-NEXT: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-NOT:  DBG_VALUE
diff --git a/llvm/test/DebugInfo/MIR/X86/entry_value_gets_propagated_X86.mir b/llvm/test/DebugInfo/MIR/X86/entry_value_gets_propagated_X86.mir
new file mode 100644
index 000000000000..4e8863064c4f
--- /dev/null
+++ b/llvm/test/DebugInfo/MIR/X86/entry_value_gets_propagated_X86.mir
@@ -0,0 +1,98 @@
+# RUN: llc --run-pass=livedebugvalues -o - %s | FileCheck %s --implicit-check-not=DBG_VALUE 
+# REQUIRES: x86-registered-target
+
+# This test covers the scenario where a DBG_VALUE created prior to LiveDebugValues has an entry-value expression.
+# It ensures that a clobbered stack copy doesn't crash if used as an entry-value because entry-values can't be clobbered.
+
+--- |
+  target triple = "x86_64-"
+
+  define i32 @baz(i32 swiftasync %arg1, i32 noundef %arg2, i1 %cond) !dbg !9 {
+    tail call void @llvm.dbg.value(metadata i32 %arg1, metadata !17, metadata !DIExpression(DW_OP_LLVM_entry_value, 1)), !dbg !19
+    br i1 %cond, label %if.then, label %if.else, !dbg !22
+  if.then:
+    %call = call i32 @foo(i32 noundef %arg1), !dbg !23
+    br label %if.end, !dbg !25
+  if.else:
+    %call1 = call i32 @foo(i32 noundef %arg2), !dbg !26
+    br label %if.end
+  if.end:
+    %temp.0 = phi i32 [ %call, %if.then ], [ %call1, %if.else ], !dbg !28
+    ret i32 %temp.0, !dbg !29
+  }
+
+  declare i32 @foo(i32)
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2, !3}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "ha", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+  !1 = !DIFile(filename: "test.c", directory: "hah")
+  !2 = !{i32 7, !"Dwarf Version", i32 4}
+  !3 = !{i32 2, !"Debug Info Version", i32 3}
+  !9 = distinct !DISubprogram(name: "baz", scope: !1, file: !1, line: 3, type: !10, scopeLine: 3, unit: !0, retainedNodes: !13)
+  !10 = !DISubroutineType(types: !11)
+  !11 = !{!12, !12, !12, !12}
+  !12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !13 = !{!14, !15, !16, !17}
+  !14 = !DILocalVariable(name: "arg1", arg: 1, scope: !9, file: !1, line: 3, type: !12)
+  !15 = !DILocalVariable(name: "arg2", arg: 2, scope: !9, file: !1, line: 3, type: !12)
+  !16 = !DILocalVariable(name: "cond", arg: 3, scope: !9, file: !1, line: 3, type: !12)
+  !17 = !DILocalVariable(name: "local", scope: !9, file: !1, line: 4, type: !12)
+  !19 = !DILocation(line: 0, scope: !9)
+  !20 = !DILocation(line: 7, column: 7, scope: !21)
+  !21 = distinct !DILexicalBlock(scope: !9, file: !1, line: 7, column: 7)
+  !22 = !DILocation(line: 7, column: 7, scope: !9)
+  !23 = !DILocation(line: 8, column: 12, scope: !24)
+  !24 = distinct !DILexicalBlock(scope: !21, file: !1, line: 7, column: 13)
+  !25 = !DILocation(line: 9, column: 3, scope: !24)
+  !26 = !DILocation(line: 10, column: 12, scope: !27)
+  !27 = distinct !DILexicalBlock(scope: !21, file: !1, line: 9, column: 10)
+  !28 = !DILocation(line: 0, scope: !21)
+  !29 = !DILocation(line: 13, column: 3, scope: !9)
+
+...
+---
+name:            baz
+alignment:       16
+debugInstrRef: true
+tracksDebugUserValues: true
+liveins:
+  - { reg: '$r14', virtual-reg: '' }
+  - { reg: '$edi', virtual-reg: '' }
+  - { reg: '$esi', virtual-reg: '' }
+  - { reg: '$edx', virtual-reg: '' }
+body:             |
+  bb.0:
+    successors: %bb.2(0x40000000), %bb.1(0x40000000)
+    liveins: $r14, $edi, $edx, $esi
+    DBG_VALUE $r14, $noreg, !14, !DIExpression(DW_OP_LLVM_entry_value, 1), debug-location !19
+    CMP32ri killed renamable $edx, 0, implicit-def $eflags, debug-location !20
+    JCC_1 %bb.2, 4, implicit killed $eflags, debug-location !22
+  bb.1.if.then:
+    successors: %bb.3(0x80000000)
+    liveins: $edi, $r13
+    CALL64pcrel32 @foo, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit-def $eax, debug-location !23
+    $r14 = MOV64ri 0, debug-location !20
+    JMP_1 %bb.3, debug-location !25
+  bb.2.if.else:
+    successors: %bb.3(0x80000000)
+    liveins: $esi, $r13
+    $edi = MOV32rr killed $esi, debug-location !26
+    CALL64pcrel32 @foo, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit-def $eax, debug-location !26
+  bb.3.if.end:
+    liveins: $eax
+    $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp, debug-location !29
+    RET64 implicit $eax, debug-location !29
+# CHECK-LABEL: bb.0:
+# CHECK: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.1.if.then:
+# CHECK: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-NEXT: CALL64pcrel32 @foo
+# CHECK-NEXT: $r14 = MOV64ri 0
+# CHECK-NOT: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.2.if.else:
+# CHECK: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
+# CHECK-LABEL: bb.3.if.end:
+# CHECK: DBG_VALUE $r14, {{.*}}, !DIExpression(DW_OP_LLVM_entry_value, 1)
diff --git a/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll b/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
index 16c9e12accbc..776674d53533 100644
--- a/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
+++ b/llvm/test/DebugInfo/X86/debug-line-in-one-fragment.ll
@@ -43,15 +43,13 @@
 ;; single data fragment. (Check that we see one data fragment, then no more
 ;; fragments until the next section).
 ;
-; FRAGMENTS:       <MCSection Name:.debug_line Fragments:[
-; FRAGMENTS-NEXT:    <MCDataFragment<MCFragment
-; FRAGMENTS-NEXT:      Contents:[
-; FRAGMENTS-NEXT:      Fixups:[
+; FRAGMENTS:       Section Name:.debug_line
+; FRAGMENTS-NEXT:    Data
+; FRAGMENTS-NEXT:      Fixup
 ;
-; FRAGMENTS-NOT: MCDataFragment
-; FRAGMENTS-NOT: MCFragment
+; FRAGMENTS-NOT:     Data
 ;
-; FRAGMENTS:     <MCSection Name:.debug_line_str Fragments:[
+; FRAGMENTS:       Section Name:.debug_line_str
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll
index cebbc162fcb3..b178392eb950 100644
--- a/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll
+++ b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll
@@ -1,6 +1,8 @@
 ; RUN: llc %s -stop-after=finalize-isel -o - \
 ; RUN: | FileCheck %s --implicit-check-not=DBG_
 
+target triple = "x86_64-unknown-linux-gnu"
+
 ;; Based on optimized IR from C source:
 ;; int main () {
 ;;   char a1[__INT_MAX__];
diff --git a/llvm/test/DebugInfo/debug_frame_symbol.ll b/llvm/test/DebugInfo/debug_frame_symbol.ll
index 56ac55e46a82..0789ee371fda 100644
--- a/llvm/test/DebugInfo/debug_frame_symbol.ll
+++ b/llvm/test/DebugInfo/debug_frame_symbol.ll
@@ -22,7 +22,7 @@ entry:
   ret void, !dbg !12
 }
 
-attributes #0 = { noinline nounwind optnone ssp "frame-pointer"="non-leaf" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone ssp "frame-pointer"="non-leaf" "no-builtins" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3, !4, !5, !6}
diff --git a/llvm/test/Feature/intrinsics.ll b/llvm/test/Feature/intrinsics.ll
index 49fed0981bd7..b6abc0fff6db 100644
--- a/llvm/test/Feature/intrinsics.ll
+++ b/llvm/test/Feature/intrinsics.ll
@@ -3,8 +3,12 @@
 ; RUN: diff %t1.ll %t2.ll
 ; RUN: FileCheck %s < %t1.ll
 
+; CHECK: Unknown intrinsic
+; CHECK-NEXT: declare i1 @llvm.isunordered.f32(float, float)
 declare i1 @llvm.isunordered.f32(float, float)
 
+; CHECK: Unknown intrinsic
+; CHECK-NEXT: declare i1 @llvm.isunordered.f64(double, double)
 declare i1 @llvm.isunordered.f64(double, double)
 
 
@@ -60,7 +64,6 @@ define void @libm() {
 
 ; FIXME: test ALL the intrinsics in this file.
 
-; rdar://11542750
 ; CHECK: declare void @llvm.trap() #1
 declare void @llvm.trap()
 
diff --git a/llvm/test/FileCheck/var-escape.txt b/llvm/test/FileCheck/var-escape.txt
new file mode 100644
index 000000000000..b290b6671be3
--- /dev/null
+++ b/llvm/test/FileCheck/var-escape.txt
@@ -0,0 +1,29 @@
+; RUN: echo -e "WINPATH=A:\windows\style\path"              >  %t.1
+; RUN: echo -e "NOT_ESCAPED=shouldn't be escaped [a-Z]\+$"  >> %t.1
+; RUN: echo -e 'ESCAPED=\\ \f\v needs\to "be" escaped\000'  >> %t.1
+; RUN: echo -e '$NUMERIC=DEADBEEF'                          >> %t.1
+
+VARS:      WINPATH=[[WINPATH:.*]]
+VARS-NEXT: NOT_ESCAPED=[[NOT_ESCAPED:.*]]
+VARS-NEXT: ESCAPED=[[ESCAPED:.*]]
+VARS-NEXT: $NUMERIC=[[#%X,$NUMERIC:]]
+; Trigger a failed match, to show variables values
+VARS-NEXT: [[WINPATH]] [[NOT_ESCAPED]] [[ESCAPED]] [[#$NUMERIC + 0]]
+
+; RUN: %ProtectFileCheckOutput not FileCheck \
+; RUN:   -dump-input=never --strict-whitespace --check-prefix=VARS --input-file=%t.1 %s 2>&1 \
+; RUN: | FileCheck %s
+
+CHECK: with "WINPATH" equal to "A:\windows\style\path"
+CHECK: with "NOT_ESCAPED" equal to "shouldn't be escaped [a-Z]\+$"
+CHECK: with "ESCAPED" equal to "\\ \014\013 needs\to \"be\" escaped\000" (escaped value)
+CHECK: with "$NUMERIC + 0" equal to "DEADBEEF"
+
+; Test escaping of the name of a numeric substitution, which might contain
+; characters that need escaping. 
+; RUN: echo -e "VARNAME: [[#%,10\t]]" > %t.2
+; RUN: echo 42 \
+; RUN: | %ProtectFileCheckOutput not FileCheck \
+; RUN:   -dump-input=never -vv --strict-whitespace --check-prefix=VARNAME %t.2 2>&1 \
+; RUN: | FileCheck %s --check-prefix=CHECK-VARNAME
+CHECK-VARNAME: with "%,10\t" equal to "10"
diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
index 64fcfdcc6127..0859a7ebb4b0 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only-smallfn.ll
@@ -24,5 +24,5 @@ entry:
   store i8 2, ptr %arrayidx1, align 1
   ret void
 }
-attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
index 90e1ab35d9c3..5f122ad3bdb8 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/calls-only.ll
@@ -51,5 +51,5 @@ entry:
 ; CHECK-NOT:  store i64 -723401728380766731, ptr %126, align 1
   ret void
 }
-attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { noinline nounwind optnone sanitize_address ssp uwtable(sync) "frame-pointer"="non-leaf" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+crc,+crypto,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+lse,+neon,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
diff --git a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
index 6c1acf6d1377..2006a6db2ef4 100644
--- a/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
+++ b/llvm/test/Instrumentation/BoundsChecking/runtimes.ll
@@ -205,7 +205,7 @@ define void @f1(i64 %x) nounwind {
 ; TR-GUARD: attributes #[[ATTR3]] = { nomerge noreturn nounwind }
 ;.
 ; RT-GUARD: attributes #[[ATTR0]] = { nounwind }
-; RT-GUARD: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; RT-GUARD: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
 ; RT-GUARD: attributes #[[ATTR2]] = { nomerge nounwind }
 ;.
 ; TR: [[META0]] = !{}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll b/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
index 73e047e68ddc..c51dc1a37362 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/count-zeroes.ll
@@ -9,10 +9,14 @@ define i64 @test_ctlz_i64_zeropoison(i64 %v) #0 {
 ; CHECK-LABEL: @test_ctlz_i64_zeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V:%.*]], 0
-; CHECK-NEXT:    [[_MSCZ_BS1:%.*]] = or i1 [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS1]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = or i1 [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
 ; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V]], i1 true)
 ; CHECK-NEXT:    store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[RES]]
@@ -24,9 +28,13 @@ define i64 @test_ctlz_i64_nozeropoison(i64 %v) #0 {
 ; CHECK-LABEL: @test_ctlz_i64_nozeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
-; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.ctlz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_MAIN]] to i64
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.ctlz.i64(i64 [[V]], i1 false)
 ; CHECK-NEXT:    store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
@@ -39,10 +47,14 @@ define <2 x i64> @test_ctlz_v2i64_zeropoison(<2 x i64> %v) #0 {
 ; CHECK-LABEL: @test_ctlz_v2i64_zeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V:%.*]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_BS1:%.*]] = or <2 x i1> [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = or <2 x i1> [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 true)
 ; CHECK-NEXT:    store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
@@ -54,9 +66,13 @@ define <2 x i64> @test_ctlz_v2i64_nozeropoison(<2 x i64> %v) #0 {
 ; CHECK-LABEL: @test_ctlz_v2i64_nozeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_MAIN]] to <2 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[V]], i1 false)
 ; CHECK-NEXT:    store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
@@ -69,10 +85,14 @@ define i64 @test_cttz_i64_zeropoison(i64 %v) #0 {
 ; CHECK-LABEL: @test_cttz_i64_zeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V:%.*]], 0
-; CHECK-NEXT:    [[_MSCZ_BS1:%.*]] = or i1 [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS1]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = or i1 [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
 ; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V]], i1 true)
 ; CHECK-NEXT:    store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[RES]]
@@ -84,9 +104,13 @@ define i64 @test_cttz_i64_nozeropoison(i64 %v) #0 {
 ; CHECK-LABEL: @test_cttz_i64_nozeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_BS]] to i64
-; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.cttz.i64(i64 [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.cttz.i64(i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge i64 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and i1 [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext i1 [[_MSCZ_MAIN]] to i64
+; CHECK-NEXT:    [[RES:%.*]] = call i64 @llvm.cttz.i64(i64 [[V]], i1 false)
 ; CHECK-NEXT:    store i64 [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
@@ -99,10 +123,14 @@ define <2 x i64> @test_cttz_v2i64_zeropoison(<2 x i64> %v) #0 {
 ; CHECK-LABEL: @test_cttz_v2i64_zeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V:%.*]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_BS1:%.*]] = or <2 x i1> [[_MSCZ_BS]], [[_MSCZ_BZP]]
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS1]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_BZP:%.*]] = icmp eq <2 x i64> [[V]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = or <2 x i1> [[_MSCZ_MAIN]], [[_MSCZ_BZP]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
 ; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V]], i1 true)
 ; CHECK-NEXT:    store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
@@ -114,9 +142,13 @@ define <2 x i64> @test_cttz_v2i64_nozeropoison(<2 x i64> %v) #0 {
 ; CHECK-LABEL: @test_cttz_v2i64_nozeropoison(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSCZ_BS:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_BS]] to <2 x i64>
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V:%.*]], i1 false)
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[TMP1]], i1 false)
+; CHECK-NEXT:    [[_MSCZ_CMP_ZEROS:%.*]] = icmp uge <2 x i64> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSCZ_SHADOW_NOT_NULL:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[_MSCZ_MAIN:%.*]] = and <2 x i1> [[_MSCZ_CMP_ZEROS]], [[_MSCZ_SHADOW_NOT_NULL]]
+; CHECK-NEXT:    [[_MSCZ_OS:%.*]] = sext <2 x i1> [[_MSCZ_MAIN]] to <2 x i64>
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> [[V]], i1 false)
 ; CHECK-NEXT:    store <2 x i64> [[_MSCZ_OS]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> [[RES]]
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/or.ll b/llvm/test/Instrumentation/MemorySanitizer/or.ll
new file mode 100644
index 000000000000..27a1800aa495
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/or.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes=msan -msan-precise-disjoint-or=false 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-IMPRECISE
+; RUN: opt < %s -S -passes=msan -msan-precise-disjoint-or=true  2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PRECISE
+;
+; Test bitwise OR instructions, including "disjoint OR".
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i8 @test_or(i8 %a, i8 %b) sanitize_memory {
+; CHECK-LABEL: define i8 @test_or(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i8 [[A]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i8 [[B]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i8 [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i8 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i8 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[C:%.*]] = or i8 [[A]], [[B]]
+; CHECK-NEXT:    store i8 [[TMP9]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %c = or i8 %a, %b
+  ret i8 %c
+}
+
+define i8 @test_disjoint_or(i8 %a, i8 %b) sanitize_memory {
+; CHECK-LABEL: define i8 @test_disjoint_or(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i8 [[A]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i8 [[B]], -1
+; CHECK-NEXT:    [[TMP5:%.*]] = and i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i8 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i8 [[TMP1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i8 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or i8 [[TMP8]], [[TMP7]]
+;
+; CHECK-IMPRECISE:      [[C:%.*]] = or disjoint i8 [[A]], [[B]]
+; CHECK-IMPRECISE-NEXT: store i8 [[TMP11]], ptr @__msan_retval_tls, align 8
+;
+; CHECK-PRECISE:         [[TMP10:%.*]] = and i8 [[A]], [[B]]
+; CHECK-PRECISE-NEXT:    [[TMP12:%.*]] = or i8 [[TMP11]], [[TMP10]]
+; CHECK-PRECISE-NEXT:    [[C:%.*]] = or disjoint i8 [[A]], [[B]]
+; CHECK-PRECISE-NEXT:    store i8 [[TMP12]], ptr @__msan_retval_tls, align 8
+;
+; CHECK-NEXT:    ret i8 [[C]]
+;
+  %c = or disjoint i8 %a, %b
+  ret i8 %c
+}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/with-call-type-size.ll b/llvm/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
index 0b81e5682062..3b1ab3364b31 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
@@ -73,13 +73,30 @@ define <4 x i32> @test64(<4 x i32> %vec, i64 %idx, i32 %x) sanitize_memory {
 ; CHECK:         call void @__msan_maybe_warning_8(i64 zeroext %{{.*}}, i32 zeroext 0)
 ; CHECK:         ret <4 x i32>
 
-; Type size too large => inline check.
+; Type size too large => use variable-size handler.
 define <4 x i32> @test65(<4 x i32> %vec, i65 %idx, i32 %x) sanitize_memory {
   %vec1 = insertelement <4 x i32> %vec, i32 %x, i65 %idx
   ret <4 x i32> %vec1
 }
 ; CHECK-LABEL: @test65(
-; CHECK:         call void @__msan_warning_noreturn
+; CHECK:         %[[A:.*]] = zext i65 %1 to i128
+; CHECK:         call void @__msan_maybe_warning_N(ptr %{{.*}}, i64 zeroext 16, i32 zeroext 0)
+; CHECK:         ret <4 x i32>
+
+define <4 x i32> @test128(<4 x i32> %vec, i128 %idx, i32 %x) sanitize_memory {
+  %vec1 = insertelement <4 x i32> %vec, i32 %x, i128 %idx
+  ret <4 x i32> %vec1
+}
+; CHECK-LABEL: @test128(
+; CHECK:         call void @__msan_maybe_warning_N(ptr %{{.*}}, i64 zeroext 16, i32 zeroext 0)
+; CHECK:         ret <4 x i32>
+
+define <4 x i32> @test256(<4 x i32> %vec, i256 %idx, i32 %x) sanitize_memory {
+  %vec1 = insertelement <4 x i32> %vec, i32 %x, i256 %idx
+  ret <4 x i32> %vec1
+}
+; CHECK-LABEL: @test256(
+; CHECK:         call void @__msan_maybe_warning_N(ptr %{{.*}}, i64 zeroext 32, i32 zeroext 0)
 ; CHECK:         ret <4 x i32>
 
 define <4 x i32> @testUndef(<4 x i32> %vec, i32 %x) sanitize_memory {
diff --git a/llvm/test/LTO/AArch64/module-asm.ll b/llvm/test/LTO/AArch64/module-asm.ll
new file mode 100644
index 000000000000..321c7890e9df
--- /dev/null
+++ b/llvm/test/LTO/AArch64/module-asm.ll
@@ -0,0 +1,21 @@
+; RUN: llvm-as %s -o %t.o
+; RUN: llvm-lto %t.o --list-symbols-only | FileCheck %s
+
+; CHECK: ___foo    { function defined hidden }
+; CHECK: ___bar    { function defined default }
+; CHECK: _foo    { data defined default }
+; CHECK: ___foo    { asm extern }
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx12.0.0"
+
+module asm ".globl _foo"
+module asm "_foo = ___foo"
+
+define hidden i32 @__foo() {
+  ret i32 0
+}
+
+define i32 @__bar() {
+  ret i32 0
+}
diff --git a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
index 659c6b09a58c..dc1ed19c658e 100644
--- a/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SME2p1/directive-arch_extension-negative.s
@@ -23,5 +23,5 @@ bfadd za.h[w8, 3], {z20.h-z21.h}
 .arch_extension ssve-aes
 .arch_extension nossve-aes
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
-// CHECK: error: instruction requires: sve2 or ssve-aes
+// CHECK: error: instruction requires: sve or ssve-aes
 // CHECK: aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]
diff --git a/llvm/test/MC/AArch64/SVE2/aesd.s b/llvm/test/MC/AArch64/SVE2/aesd.s
index 44cc99c84291..0456e2849fc7 100644
--- a/llvm/test/MC/AArch64/SVE2/aesd.s
+++ b/llvm/test/MC/AArch64/SVE2/aesd.s
@@ -17,5 +17,5 @@
 aesd z0.b, z0.b, z31.b
 // CHECK-INST: aesd z0.b, z0.b, z31.b
 // CHECK-ENCODING: [0xe0,0xe7,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4522e7e0 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/aese.s b/llvm/test/MC/AArch64/SVE2/aese.s
index 12da19e06db1..28cf39084640 100644
--- a/llvm/test/MC/AArch64/SVE2/aese.s
+++ b/llvm/test/MC/AArch64/SVE2/aese.s
@@ -17,5 +17,5 @@
 aese z0.b, z0.b, z31.b
 // CHECK-INST: aese z0.b, z0.b, z31.b
 // CHECK-ENCODING: [0xe0,0xe3,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4522e3e0 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/aesimc.s b/llvm/test/MC/AArch64/SVE2/aesimc.s
index 7027f6a834e6..0fbcb15b12d6 100644
--- a/llvm/test/MC/AArch64/SVE2/aesimc.s
+++ b/llvm/test/MC/AArch64/SVE2/aesimc.s
@@ -17,11 +17,11 @@
 aesimc z0.b, z0.b
 // CHECK-INST: aesimc z0.b, z0.b
 // CHECK-ENCODING: [0x00,0xe4,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4520e400 <unknown>
 
 aesimc z31.b, z31.b
 // CHECK-INST: aesimc z31.b, z31.b
 // CHECK-ENCODING: [0x1f,0xe4,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4520e41f <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/aesmc.s b/llvm/test/MC/AArch64/SVE2/aesmc.s
index 1ad56b4762b2..4b0ebcca1de6 100644
--- a/llvm/test/MC/AArch64/SVE2/aesmc.s
+++ b/llvm/test/MC/AArch64/SVE2/aesmc.s
@@ -17,11 +17,11 @@
 aesmc z0.b, z0.b
 // CHECK-INST: aesmc z0.b, z0.b
 // CHECK-ENCODING: [0x00,0xe0,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4520e000 <unknown>
 
 aesmc z31.b, z31.b
 // CHECK-INST: aesmc z31.b, z31.b
 // CHECK-ENCODING: [0x1f,0xe0,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 4520e01f <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/bdep.s b/llvm/test/MC/AArch64/SVE2/bdep.s
index 44c848d0b3b5..30927d6956d1 100644
--- a/llvm/test/MC/AArch64/SVE2/bdep.s
+++ b/llvm/test/MC/AArch64/SVE2/bdep.s
@@ -14,23 +14,23 @@
 bdep z0.b, z1.b, z31.b
 // CHECK-INST: bdep z0.b, z1.b, z31.b
 // CHECK-ENCODING: [0x20,0xb4,0x1f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 451fb420 <unknown>
 
 bdep z0.h, z1.h, z31.h
 // CHECK-INST: bdep z0.h, z1.h, z31.h
 // CHECK-ENCODING: [0x20,0xb4,0x5f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 455fb420 <unknown>
 
 bdep z0.s, z1.s, z31.s
 // CHECK-INST: bdep z0.s, z1.s, z31.s
 // CHECK-ENCODING: [0x20,0xb4,0x9f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 459fb420 <unknown>
 
 bdep z0.d, z1.d, z31.d
 // CHECK-INST: bdep z0.d, z1.d, z31.d
 // CHECK-ENCODING: [0x20,0xb4,0xdf,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 45dfb420 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/bext.s b/llvm/test/MC/AArch64/SVE2/bext.s
index ea519c22cceb..8d5fc5788a50 100644
--- a/llvm/test/MC/AArch64/SVE2/bext.s
+++ b/llvm/test/MC/AArch64/SVE2/bext.s
@@ -14,23 +14,23 @@
 bext z0.b, z1.b, z31.b
 // CHECK-INST: bext z0.b, z1.b, z31.b
 // CHECK-ENCODING: [0x20,0xb0,0x1f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 451fb020 <unknown>
 
 bext z0.h, z1.h, z31.h
 // CHECK-INST: bext z0.h, z1.h, z31.h
 // CHECK-ENCODING: [0x20,0xb0,0x5f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 455fb020 <unknown>
 
 bext z0.s, z1.s, z31.s
 // CHECK-INST: bext z0.s, z1.s, z31.s
 // CHECK-ENCODING: [0x20,0xb0,0x9f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 459fb020 <unknown>
 
 bext z0.d, z1.d, z31.d
 // CHECK-INST: bext z0.d, z1.d, z31.d
 // CHECK-ENCODING: [0x20,0xb0,0xdf,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 45dfb020 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/bgrp.s b/llvm/test/MC/AArch64/SVE2/bgrp.s
index eb58d1351158..196eeb304b90 100644
--- a/llvm/test/MC/AArch64/SVE2/bgrp.s
+++ b/llvm/test/MC/AArch64/SVE2/bgrp.s
@@ -14,23 +14,23 @@
 bgrp z0.b, z1.b, z31.b
 // CHECK-INST: bgrp z0.b, z1.b, z31.b
 // CHECK-ENCODING: [0x20,0xb8,0x1f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 451fb820 <unknown>
 
 bgrp z0.h, z1.h, z31.h
 // CHECK-INST: bgrp z0.h, z1.h, z31.h
 // CHECK-ENCODING: [0x20,0xb8,0x5f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 455fb820 <unknown>
 
 bgrp z0.s, z1.s, z31.s
 // CHECK-INST: bgrp z0.s, z1.s, z31.s
 // CHECK-ENCODING: [0x20,0xb8,0x9f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 459fb820 <unknown>
 
 bgrp z0.d, z1.d, z31.d
 // CHECK-INST: bgrp z0.d, z1.d, z31.d
 // CHECK-ENCODING: [0x20,0xb8,0xdf,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK-ERROR: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-UNKNOWN: 45dfb820 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
index 2cfce3b232ff..c3e19a7ad4d0 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch-negative.s
@@ -26,7 +26,7 @@ sm4e z0.s, z0.s, z0.s
 .arch armv9-a+sve2-sha3
 .arch armv9-a+nosve2-sha3
 rax1 z0.d, z0.d, z0.d
-// CHECK: error: instruction requires: sve2-sha3
+// CHECK: error: instruction requires: sve-sha3
 // CHECK-NEXT: rax1 z0.d, z0.d, z0.d
 
 .arch armv9-a+ssve-bitperm
diff --git a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
index 2fab61597576..c2ec716e45d5 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-arch_extension-negative.s
@@ -9,7 +9,7 @@ tbx z0.b, z1.b, z2.b
 .arch_extension sve-aes
 .arch_extension nosve-aes
 aesd z23.b, z23.b, z13.b
-// CHECK: error: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK: error: instruction requires: sve-aes
 // CHECK-NEXT: aesd z23.b, z23.b, z13.b
 
 // nosve2-aes should disable sve-aes but not sve2.
@@ -28,19 +28,13 @@ sm4e z0.s, z0.s, z0.s
 .arch_extension sve2-sha3
 .arch_extension nosve2-sha3
 rax1 z0.d, z0.d, z0.d
-// CHECK: error: instruction requires: sve2-sha3
+// CHECK: error: instruction requires: sve-sha3
 // CHECK-NEXT: rax1 z0.d, z0.d, z0.d
 
 .arch_extension sve2-bitperm
 .arch_extension nosve2-bitperm
 bgrp z21.s, z10.s, z21.s
-// CHECK: error: instruction requires: sve2 or ssve-bitperm sve-bitperm
-// CHECK-NEXT: bgrp z21.s, z10.s, z21.s
-
-.arch_extension sve2-bitperm
-.arch_extension nosve2
-bgrp z21.s, z10.s, z21.s
-// CHECK: error: instruction requires: sve2 or ssve-bitperm
+// CHECK: error: instruction requires: sve-bitperm
 // CHECK-NEXT: bgrp z21.s, z10.s, z21.s
 
 .arch_extension f8f16mm
diff --git a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
index a50b99094942..956dcb1a82a6 100644
--- a/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
+++ b/llvm/test/MC/AArch64/SVE2/directive-cpu-negative.s
@@ -14,7 +14,7 @@ aesd z23.b, z23.b, z13.b
 
 .cpu generic+sve-aes+nosve-aes
 aesd z23.b, z23.b, z13.b
-// CHECK: error: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK: error: instruction requires: sve or ssve-aes sve-aes
 // CHECK-NEXT: aesd z23.b, z23.b, z13.b
 
 .cpu generic+sve2-sm4
@@ -26,7 +26,7 @@ sm4e z0.s, z0.s, z0.s
 .cpu generic+sve2-sha3
 .cpu generic+nosve2-sha3
 rax1 z0.d, z0.d, z0.d
-// CHECK: error: instruction requires: sve2-sha3
+// CHECK: error: instruction requires: sve or sme2p1 sve-sha3
 // CHECK-NEXT: rax1 z0.d, z0.d, z0.d
 
 .cpu generic+sve2+sve-bitperm
@@ -38,7 +38,7 @@ bgrp z21.s, z10.s, z21.s
 .cpu generic+ssve-bitperm
 .cpu generic+nossve-bitperm
 bgrp z21.s, z10.s, z21.s
-// CHECK: error: instruction requires: sve2 or ssve-bitperm sve-bitperm
+// CHECK: error: instruction requires: sve or ssve-bitperm sve-bitperm
 // CHECK-NEXT: bgrp z21.s, z10.s, z21.s
 
 .cpu generic+sve2+f8f16mm
diff --git a/llvm/test/MC/AArch64/SVE2/pmullb-128.s b/llvm/test/MC/AArch64/SVE2/pmullb-128.s
index dd9ae3c46f22..45ed1d3734f1 100644
--- a/llvm/test/MC/AArch64/SVE2/pmullb-128.s
+++ b/llvm/test/MC/AArch64/SVE2/pmullb-128.s
@@ -17,5 +17,5 @@
 pmullb z29.q, z30.d, z31.d
 // CHECK-INST: pmullb z29.q, z30.d, z31.d
 // CHECK-ENCODING: [0xdd,0x6b,0x1f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 451f6bdd <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/pmullt-128.s b/llvm/test/MC/AArch64/SVE2/pmullt-128.s
index b26679d9cb5f..9173e92f5a9a 100644
--- a/llvm/test/MC/AArch64/SVE2/pmullt-128.s
+++ b/llvm/test/MC/AArch64/SVE2/pmullt-128.s
@@ -17,5 +17,5 @@
 pmullt z29.q, z30.d, z31.d
 // CHECK-INST: pmullt z29.q, z30.d, z31.d
 // CHECK-ENCODING: [0xdd,0x6f,0x1f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes
 // CHECK-UNKNOWN: 451f6fdd <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2/rax1.s b/llvm/test/MC/AArch64/SVE2/rax1.s
index c110d0a83b1c..8bf70576e724 100644
--- a/llvm/test/MC/AArch64/SVE2/rax1.s
+++ b/llvm/test/MC/AArch64/SVE2/rax1.s
@@ -1,5 +1,9 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2-sha3 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve,+sve-sha3 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sme2p1,+sve-sha3 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 // RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sme < %s 2>&1 \
@@ -13,5 +17,5 @@
 rax1 z0.d, z1.d, z31.d
 // CHECK-INST: rax1 z0.d, z1.d, z31.d
 // CHECK-ENCODING: [0x20,0xf4,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2-sha3
+// CHECK-ERROR: instruction requires: sve or sme2p1 sve-sha3
 // CHECK-UNKNOWN: 453ff420 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesd.s b/llvm/test/MC/AArch64/SVE2p1/aesd.s
index 479335dc3616..fbf23215fdd4 100644
--- a/llvm/test/MC/AArch64/SVE2p1/aesd.s
+++ b/llvm/test/MC/AArch64/SVE2p1/aesd.s
@@ -18,36 +18,36 @@
 aesd    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101100-00000000
 // CHECK-INST: aesd    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4522ec00 <unknown>
 
 aesd    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101101-01010100
 // CHECK-INST: aesd    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xed,0x32,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4532ed54 <unknown>
 
 aesd    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101111-11111110
 // CHECK-INST: aesd    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xef,0x3a,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453aeffe <unknown>
 
 // x4
 aesd    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101100-00000000
 // CHECK-INST: aesd    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x26,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4526ec00 <unknown>
 
 aesd    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101101-10110100
 // CHECK-INST: aesd    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xed,0x2e,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452eedb4 <unknown>
 
 aesd    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101111-11111100
 // CHECK-INST: aesd    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xef,0x3e,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453eeffc <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesdimc.s b/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
index d1b0b130f9ec..9e7f38ee8fdb 100644
--- a/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
+++ b/llvm/test/MC/AArch64/SVE2p1/aesdimc.s
@@ -19,36 +19,36 @@
 aesdimc {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101100-00000000
 // CHECK-INST: aesdimc { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x23,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4523ec00 <unknown>
 
 aesdimc {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110011-11101101-01010100
 // CHECK-INST: aesdimc { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xed,0x33,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4533ed54 <unknown>
 
 aesdimc {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101111-11111110
 // CHECK-INST: aesdimc { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xef,0x3b,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453beffe <unknown>
 
 // x4
 aesdimc {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101100-00000000
 // CHECK-INST: aesdimc { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xec,0x27,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4527ec00 <unknown>
 
 aesdimc {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101111-11101101-10110100
 // CHECK-INST: aesdimc { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xed,0x2f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452fedb4 <unknown>
 
 aesdimc {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101111-11111100
 // CHECK-INST: aesdimc { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xef,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453feffc <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/aese.s b/llvm/test/MC/AArch64/SVE2p1/aese.s
index 1f8ef5e85a15..17f7d19b62ff 100644
--- a/llvm/test/MC/AArch64/SVE2p1/aese.s
+++ b/llvm/test/MC/AArch64/SVE2p1/aese.s
@@ -18,36 +18,36 @@
 aese    {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100010-11101000-00000000
 // CHECK-INST: aese    { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x22,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4522e800 <unknown>
 
 aese    {z20.b-z21.b}, {z20.b-z21.b}, z10.q[2]  // 01000101-00110010-11101001-01010100
 // CHECK-INST: aese    { z20.b, z21.b }, { z20.b, z21.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xe9,0x32,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4532e954 <unknown>
 
 aese    {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111010-11101011-11111110
 // CHECK-INST: aese    { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xeb,0x3a,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453aebfe <unknown>
 
 // x4
 aese    {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100110-11101000-00000000
 // CHECK-INST: aese    { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x26,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4526e800 <unknown>
 
 aese    {z20.b-z23.b}, {z20.b-z23.b}, z13.q[1]  // 01000101-00101110-11101001-10110100
 // CHECK-INST: aese    { z20.b - z23.b }, { z20.b - z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb4,0xe9,0x2e,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452ee9b4 <unknown>
 
 aese    {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111110-11101011-11111100
 // CHECK-INST: aese    { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xeb,0x3e,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453eebfc <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/aesemc.s b/llvm/test/MC/AArch64/SVE2p1/aesemc.s
index 5682ce87737a..2d58732d5046 100644
--- a/llvm/test/MC/AArch64/SVE2p1/aesemc.s
+++ b/llvm/test/MC/AArch64/SVE2p1/aesemc.s
@@ -18,36 +18,36 @@
 aesemc  {z0.b-z1.b}, {z0.b-z1.b}, z0.q[0]  // 01000101-00100011-11101000-00000000
 // CHECK-INST: aesemc  { z0.b, z1.b }, { z0.b, z1.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x23,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4523e800 <unknown>
 
 aesemc  {z22.b-z23.b}, {z22.b-z23.b}, z13.q[1]  // 01000101-00101011-11101001-10110110
 // CHECK-INST: aesemc  { z22.b, z23.b }, { z22.b, z23.b }, z13.q[1]
 // CHECK-ENCODING: [0xb6,0xe9,0x2b,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 452be9b6 <unknown>
 
 aesemc  {z30.b-z31.b}, {z30.b-z31.b}, z31.q[3]  // 01000101-00111011-11101011-11111110
 // CHECK-INST: aesemc  { z30.b, z31.b }, { z30.b, z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfe,0xeb,0x3b,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453bebfe <unknown>
 
 // x4
 aesemc  {z0.b-z3.b}, {z0.b-z3.b}, z0.q[0]  // 01000101-00100111-11101000-00000000
 // CHECK-INST: aesemc  { z0.b - z3.b }, { z0.b - z3.b }, z0.q[0]
 // CHECK-ENCODING: [0x00,0xe8,0x27,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4527e800 <unknown>
 
 aesemc  {z20.b-z23.b}, {z20.b-z23.b}, z10.q[2]  // 01000101-00110111-11101001-01010100
 // CHECK-INST: aesemc  { z20.b - z23.b }, { z20.b - z23.b }, z10.q[2]
 // CHECK-ENCODING: [0x54,0xe9,0x37,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4537e954 <unknown>
 
 aesemc  {z28.b-z31.b}, {z28.b-z31.b}, z31.q[3]  // 01000101-00111111-11101011-11111100
 // CHECK-INST: aesemc  { z28.b - z31.b }, { z28.b - z31.b }, z31.q[3]
 // CHECK-ENCODING: [0xfc,0xeb,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453febfc <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmlal.s b/llvm/test/MC/AArch64/SVE2p1/pmlal.s
index 6a3751e9ac25..596def7a52c2 100644
--- a/llvm/test/MC/AArch64/SVE2p1/pmlal.s
+++ b/llvm/test/MC/AArch64/SVE2p1/pmlal.s
@@ -17,17 +17,17 @@
 pmlal   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111100-00000000
 // CHECK-INST: pmlal   { z0.q, z1.q }, z0.d, z0.d
 // CHECK-ENCODING: [0x00,0xfc,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4520fc00 <unknown>
 
 pmlal   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111101-10110110
 // CHECK-INST: pmlal   { z22.q, z23.q }, z13.d, z8.d
 // CHECK-ENCODING: [0xb6,0xfd,0x28,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4528fdb6 <unknown>
 
 pmlal   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111111-11111110
 // CHECK-INST: pmlal   { z30.q, z31.q }, z31.d, z31.d
 // CHECK-ENCODING: [0xfe,0xff,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453ffffe <unknown>
diff --git a/llvm/test/MC/AArch64/SVE2p1/pmull.s b/llvm/test/MC/AArch64/SVE2p1/pmull.s
index 3686181c35b7..4b6b20931abf 100644
--- a/llvm/test/MC/AArch64/SVE2p1/pmull.s
+++ b/llvm/test/MC/AArch64/SVE2p1/pmull.s
@@ -17,17 +17,17 @@
 pmull   {z0.q-z1.q}, z0.d, z0.d  // 01000101-00100000-11111000-00000000
 // CHECK-INST: pmull   { z0.q, z1.q }, z0.d, z0.d
 // CHECK-ENCODING: [0x00,0xf8,0x20,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4520f800 <unknown>
 
 pmull   {z22.q-z23.q}, z13.d, z8.d  // 01000101-00101000-11111001-10110110
 // CHECK-INST: pmull   { z22.q, z23.q }, z13.d, z8.d
 // CHECK-ENCODING: [0xb6,0xf9,0x28,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 4528f9b6 <unknown>
 
 pmull   {z30.q-z31.q}, z31.d, z31.d  // 01000101-00111111-11111011-11111110
 // CHECK-INST: pmull   { z30.q, z31.q }, z31.d, z31.d
 // CHECK-ENCODING: [0xfe,0xfb,0x3f,0x45]
-// CHECK-ERROR: instruction requires: sve2 or ssve-aes sve-aes2
+// CHECK-ERROR: instruction requires: sve or ssve-aes sve-aes2
 // CHECK-UNKNOWN: 453ffbfe <unknown>
diff --git a/llvm/test/MC/AArch64/coff-relocations-branch26.s b/llvm/test/MC/AArch64/coff-relocations-branch26.s
index 4cd47032309c..51bbb4932f23 100644
--- a/llvm/test/MC/AArch64/coff-relocations-branch26.s
+++ b/llvm/test/MC/AArch64/coff-relocations-branch26.s
@@ -61,7 +61,7 @@ main:
 err:
     nop
     b .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:19: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
     .def .Lerr_target
     .scl 3
diff --git a/llvm/test/MC/AArch64/coff-relocations-diags.s b/llvm/test/MC/AArch64/coff-relocations-diags.s
index e03ba377dc18..41fdd03df8f3 100644
--- a/llvm/test/MC/AArch64/coff-relocations-diags.s
+++ b/llvm/test/MC/AArch64/coff-relocations-diags.s
@@ -1,17 +1,17 @@
 // RUN: not llvm-mc -triple aarch64-win32 -filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
 
   adrp x0, :got:symbol
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :got: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:12: error: relocation specifier :got: unsupported on COFF targets
   // CHECK-NEXT: adrp x0, :got:symbol
   // CHECK-NEXT: ^
 
   ldr x0, [x0, :got_lo12:symbol]
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :got_lo12: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:16: error: relocation specifier :got_lo12: unsupported on COFF targets
   // CHECK-NEXT: ldr x0, [x0, :got_lo12:symbol]
   // CHECK-NEXT: ^
 
   adrp x0, :tlsdesc:symbol
-  // CHECK: [[#@LINE-1]]:3: error: relocation specifier :tlsdesc: unsupported on COFF targets
+  // CHECK: [[#@LINE-1]]:12: error: relocation specifier :tlsdesc: unsupported on COFF targets
   // CHECK-NEXT: adrp x0, :tlsdesc:symbol
   // CHECK-NEXT: ^
   add x0, x0, :tlsdesc_lo12:symbol
diff --git a/llvm/test/MC/AArch64/ilp32-diagnostics.s b/llvm/test/MC/AArch64/ilp32-diagnostics.s
index 152903590335..aff01ed4c76a 100644
--- a/llvm/test/MC/AArch64/ilp32-diagnostics.s
+++ b/llvm/test/MC/AArch64/ilp32-diagnostics.s
@@ -3,102 +3,102 @@
 // RUN: FileCheck --check-prefix=ERROR %s --implicit-check-not=error: < %t2
 
 .xword sym-.
-// ERROR: [[#@LINE-1]]:8: error: 8 byte PC relative data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:11: error: 8 byte PC relative data relocation is not supported in ILP32
 
 .xword sym+16
-// ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 8 byte absolute data relocation is not supported in ILP32
 
 .xword sym@AUTH(da,42)
 // ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
 
 .xword sym@AUTH(da,42,addr)
-// ERROR: [[#@LINE-1]]:8: error: 8 byte absolute data relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 8 byte absolute data relocation is not supported in ILP32
 
 movz x7, #:abs_g3:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:11: error: absolute MOV relocation is not supported in ILP32
 // ERROR:        movz x7, #:abs_g3:some_label
 
 movz x3, #:abs_g2:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:abs_g2:some_label
 
 movz x19, #:abs_g2_s:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x19, #:abs_g2_s:some_label
 
 movk x5, #:abs_g2_nc:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x5, #:abs_g2_nc:some_label
 
 movz x19, #:abs_g1_s:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x19, #:abs_g1_s:some_label
 
 movk x5, #:abs_g1_nc:some_label
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x5, #:abs_g1_nc:some_label
 
 movz x3, #:dtprel_g2:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:dtprel_g2:var
 
 movk x9, #:dtprel_g1_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x9, #:dtprel_g1_nc:var
 
 movz x3, #:tprel_g2:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x3, #:tprel_g2:var
 
 movk x9, #:tprel_g1_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x9, #:tprel_g1_nc:var
 
 movz x15, #:gottprel_g1:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movz x15, #:gottprel_g1:var
 
 movk x13, #:gottprel_g0_nc:var
-// ERROR: [[#@LINE-1]]:1: error: absolute MOV relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: absolute MOV relocation is not supported in ILP32
 // ERROR: movk x13, #:gottprel_g0_nc:var
 
 ldr x10, [x0, #:gottprel_lo12:var]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 // ERROR: ldr x10, [x0, #:gottprel_lo12:var]
 
 ldr x24, [x23, #:got_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, [x23, #:got_auth_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 add x24, x23, #:got_auth_lo12:sym
-// ERROR: [[#@LINE-1]]:1: error: ADD AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADD AUTH relocation is not supported in ILP32
 
 ldr x24, [x23, :gottprel_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x10, [x0, #:gottprel_lo12:var]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 // ERROR: ldr x10, [x0, #:gottprel_lo12:var]
 
 ldr x24, [x23, #:got_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, [x23, :gottprel_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store relocation is not supported in ILP32
 
 ldr x24, :got_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: LDR AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:10: error: LDR AUTH relocation is not supported in ILP32
 
 adr x24, :got_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: ADR AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADR AUTH relocation is not supported in ILP32
 
 adrp x24, :tlsdesc_auth:sym
-// ERROR: [[#@LINE-1]]:1: error: ADRP AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADRP AUTH relocation is not supported in ILP32
 
 ldr x24, [x23, :tlsdesc_auth_lo12:sym]
-// ERROR: [[#@LINE-1]]:1: error: 64-bit load/store AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: 64-bit load/store AUTH relocation is not supported in ILP32
 
 add x24, x23, :tlsdesc_auth_lo12:sym
-// ERROR: [[#@LINE-1]]:1: error: ADD AUTH relocation is not supported in ILP32
+// ERROR: [[#@LINE-1]]:[[#]]: error: ADD AUTH relocation is not supported in ILP32
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
index 3dcf288bbbaa..bbd36a9140b9 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
@@ -13185,3 +13185,15 @@ v_pk_fmac_f16 v5, -4.0, v2
 
 v_pk_fmac_f16 v5, v1, v255
 // GFX10: encoding: [0x01,0xff,0x0b,0x78]
+
+v_pk_fmac_f16 v5, v1, v2
+// GFX10: encoding: [0x01,0x05,0x0a,0x78]
+
+v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX10: encoding: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0xff]
+
+v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX10: encoding: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0x00]
+
+v_pk_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX10: encoding: [0xe9,0x04,0x0a,0x78,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
index c151bf99b76c..6bb0f4b1dff2 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop3.s
@@ -8974,6 +8974,9 @@ v_mul_lo_u16 v5, v1, 0.5
 v_mul_lo_u16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x05,0xd7,0x01,0xef,0x01,0x00]
 
+v_mul_lo_u16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x05,0xd7,0x01,0x05,0x02,0x00]
+
 v_lshrrev_b16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9052,6 +9055,9 @@ v_lshrrev_b16 v5, v1, 0.5
 v_lshrrev_b16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x07,0xd7,0x01,0xef,0x01,0x00]
 
+v_lshrrev_b16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x07,0xd7,0x01,0x05,0x02,0x00]
+
 v_ashrrev_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9130,6 +9136,9 @@ v_ashrrev_i16 v5, v1, 0.5
 v_ashrrev_i16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x08,0xd7,0x01,0xef,0x01,0x00]
 
+v_ashrrev_i16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x08,0xd7,0x01,0x05,0x02,0x00]
+
 v_max_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9208,6 +9217,9 @@ v_max_u16 v5, v1, 0.5
 v_max_u16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x09,0xd7,0x01,0xef,0x01,0x00]
 
+v_max_u16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x09,0xd7,0x01,0x05,0x02,0x00]
+
 v_max_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9286,6 +9298,9 @@ v_max_i16 v5, v1, 0.5
 v_max_i16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x0a,0xd7,0x01,0xef,0x01,0x00]
 
+v_max_i16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x0a,0xd7,0x01,0x05,0x02,0x00]
+
 v_min_u16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9364,6 +9379,9 @@ v_min_u16 v5, v1, 0.5
 v_min_u16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x0b,0xd7,0x01,0xef,0x01,0x00]
 
+v_min_u16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x0b,0xd7,0x01,0x05,0x02,0x00]
+
 v_min_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0x05,0x02,0x00]
 
@@ -9442,6 +9460,9 @@ v_min_i16 v5, v1, 0.5
 v_min_i16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x0c,0xd7,0x01,0xef,0x01,0x00]
 
+v_min_i16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x0c,0xd7,0x01,0x05,0x02,0x00]
+
 v_add_nc_i16 v5, v1, v2
 // GFX10: encoding: [0x05,0x00,0x0d,0xd7,0x01,0x05,0x02,0x00]
 
@@ -10009,6 +10030,9 @@ v_lshlrev_b16 v5, v1, 0.5
 v_lshlrev_b16 v5, v1, -4.0
 // GFX10: encoding: [0x05,0x00,0x14,0xd7,0x01,0xef,0x01,0x00]
 
+v_lshlrev_b16 v5, v1, v2 op_sel:[1,1,1]
+// GFX10: encoding: [0x05,0x58,0x14,0xd7,0x01,0x05,0x02,0x00]
+
 v_mad_u16 v5, 0, v2, v3
 // GFX10: encoding: [0x05,0x00,0x40,0xd7,0x80,0x04,0x0e,0x04]
 
diff --git a/llvm/test/MC/AMDGPU/gfx10_unsupported_sdwa.s b/llvm/test/MC/AMDGPU/gfx10_unsupported_sdwa.s
index 88db110ad9c2..681b34e4c1c5 100644
--- a/llvm/test/MC/AMDGPU/gfx10_unsupported_sdwa.s
+++ b/llvm/test/MC/AMDGPU/gfx10_unsupported_sdwa.s
@@ -32,6 +32,9 @@ v_min_u16_sdwa v255, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD s
 v_mul_lo_u16_sdwa v255, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: sdwa variant of this instruction is not supported
 
+v_pk_fmac_f16_sdwa v255, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: sdwa variant of this instruction is not supported
+
 v_sub_co_u32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: sdwa variant of this instruction is not supported
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
new file mode 100644
index 000000000000..98436c9d6aa9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR %s
+
+ds_atomic_async_barrier_arrive_b64 v1 offset:65407
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_async_barrier_arrive_b64 v255 offset:1040
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v255 offset:1040 ; encoding: [0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_async_barrier_arrive_b64 v5
+// GFX1250: ds_atomic_async_barrier_arrive_b64 v5   ; encoding: [0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5]
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] ; encoding: [0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513 ; encoding: [0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535
+// GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535 ; encoding: [0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_load_tr.s b/llvm/test/MC/AMDGPU/gfx1250_asm_load_tr.s
new file mode 100644
index 000000000000..d475e3f3b7c0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_load_tr.s
@@ -0,0 +1,219 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-wavefrontsize32,+wavefrontsize64 %s 2>&1 | FileCheck --check-prefix=WAVESIZE-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: %s
+
+global_load_tr8_b64 v[2:3], v0, s[0:1]
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], v0, s[0:1] offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], v[4:5], off
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], v[4:5], off offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], v[4:5], off offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v1, v0, s[0:1]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr8_b64 v[2:3], s[3:4], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid register alignment
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v0, s[0:1]
+// GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v0, s[0:1] offset:64
+// GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v[4:5], off
+// GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v[4:5], off offset:64
+// GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], v[4:5], off offset:-64
+// GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v1, v0, s[0:1]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr4_b64 v[2:3], s[3:4], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid register alignment
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v0, s[0:1]
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v0, s[0:1] offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v[6:7], off
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v[6:7], off offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], v[6:7], off offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:-64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:3], v[6:7], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr16_b128 v[2:5], s[5:6], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid register alignment
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v0, s[0:1]
+// GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1]  ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[3:5], v0, s[0:1]
+// GFX1250: global_load_tr6_b96 v[3:5], v0, s[0:1]  ; encoding: [0x00,0x00,0x1d,0xee,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v0, s[0:1] offset:64
+// GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v[6:7], off
+// GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v[6:7], off offset:64
+// GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off offset:64 ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:4], v[6:7], off offset:-64
+// GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off offset:-64 ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:3], v[6:7], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+global_load_tr6_b96 v[2:5], s[5:6], off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid register alignment
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr8_b64 v[2:3], v0
+// GFX1250: ds_load_tr8_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr8_b64 v[2:3], v0 offset:64
+// GFX1250: ds_load_tr8_b64 v[2:3], v0 offset:64    ; encoding: [0x40,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr8_b64 v[2:3], v0 offset:-64
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr8_b64 v1, v0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr8_b64 v[2:3], s0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr4_b64 v[2:3], v0
+// GFX1250: ds_load_tr4_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr4_b64 v[2:3], v0 offset:64
+// GFX1250: ds_load_tr4_b64 v[2:3], v0 offset:64    ; encoding: [0x40,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr4_b64 v[2:3], v0 offset:-64
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr4_b64 v1, v0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr4_b64 v[2:3], s0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr16_b128 v[2:5], v0
+// GFX1250: ds_load_tr16_b128 v[2:5], v0            ; encoding: [0x00,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr16_b128 v[2:5], v0 offset:64
+// GFX1250: ds_load_tr16_b128 v[2:5], v0 offset:64  ; encoding: [0x40,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr16_b128 v[2:5], v0 offset:-64
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr16_b128 v[2:3], v0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr16_b128 v[2:5], s0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[2:4], v0
+// GFX1250: ds_load_tr6_b96 v[2:4], v0              ; encoding: [0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[3:5], v0
+// GFX1250: ds_load_tr6_b96 v[3:5], v0              ; encoding: [0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x03]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[2:4], v0 offset:64
+// GFX1250: ds_load_tr6_b96 v[2:4], v0 offset:64    ; encoding: [0x40,0x00,0xec,0xdb,0x00,0x00,0x00,0x02]
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[2:4], v0 offset:-64
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[2:3], v0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
+
+ds_load_tr6_b96 v[2:4], s0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// WAVESIZE-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction requires wavesize=32
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
index 48ec44b410c2..6ebc17468eed 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
@@ -16,3 +16,15 @@ s_wait_xcnt 0xf
 s_setprio_inc_wg 100
 // GFX1250: [0x64,0x00,0xbe,0xbf]
 // GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 1
+// GFX1250: s_monitor_sleep 1                       ; encoding: [0x01,0x00,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 32768
+// GFX1250: s_monitor_sleep 0x8000                  ; encoding: [0x00,0x80,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_monitor_sleep 0
+// GFX1250: s_monitor_sleep 0                       ; encoding: [0x00,0x00,0x84,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s b/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
new file mode 100644
index 000000000000..89bd507942a2
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
@@ -0,0 +1,569 @@
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+global_atomic_ordered_add_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; DOT4_F32_*, DOT2_F32_*, DOT2_F16 and DOT2_BF16
+
+v_dot4_f32_fp8_fp8 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_fp8_fp8 v0, v1, v2, v3 row_mirror
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_fp8_fp8 v0, v1, v2, v3 dpp8:[0,1,2,3,4,5,6,7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_fp8_bf8 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_fp8_bf8 v0, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_fp8_bf8 v0, v1, v2, v3 dpp8:[0,1,2,3,4,5,6,7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_fp8 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_fp8 v0, v1, v2, v3 row_shl:15
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_fp8 v0, v1, v2, v3 dpp8:[0,1,2,3,4,5,6,7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_bf8 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_bf8 v0, v1, v2, v3 row_share:15
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot4_f32_bf8_bf8 v0, v1, v2, v3 dpp8:[0,1,2,3,4,5,6,7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_f16_f16 v5, v1, v2, s3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_bf16_bf16 v5, v1, v2, s3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_f32_bf16 v5, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_dot2_f32_f16 v5, v1, v2, s3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; LDS-direct and parameter-load, VINTERP
+
+ds_direct_load v1 wait_va_vdst:15
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_param_load v1, attr0.x wait_va_vdst:15
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_direct_load v1 wait_va_vdst:15 wait_vm_vsrc:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_param_load v1, attr0.x wait_va_vdst:15 wait_vm_vsrc:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p10_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p2_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p10_f16_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p2_f16_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p10_rtz_f16_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_interp_p2_rtz_f16_f32 v0, v1, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; Export, S_WAIT_EXPCNT and S_WAIT_EVENT
+
+export mrt0 off, off, off, off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+export mrt7 v1, v1, v1, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+export mrtz v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+export pos0 v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+export pos3 v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+exp mrt0 off, off, off, off
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+exp mrt7 v1, v1, v1, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+exp mrtz v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+exp pos0 v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+exp pos3 v4, v3, v2, v1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_wait_event 0x3141
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_wait_expcnt 0x1234
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; Ray Tracing: DS_BVH_STACK ops
+
+ds_bvh_stack_rtn_b32 v255, v254, v253, v[249:252]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_bvh_stack_push4_pop1_rtn_b32 v1, v0, v1, v[2:5]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_bvh_stack_push8_pop1_rtn_b32 v1, v0, v1, v[2:9]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_bvh_stack_push8_pop2_rtn_b64 v[254:255], v253, v252, v[244:251]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; S_WAIT_*CNT instructions.
+
+s_wait_samplecnt 0x1234
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_wait_bvhcnt 0x1234
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; S_WAITCNT instruction.
+
+s_waitcnt 0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; All "TBUFFER" ops, and BUFFER_LOAD/STORE_FORMAT ops.
+
+tbuffer_load_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_8_UNORM] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_SINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_SINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_SSCALED] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_load_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_32_32_32_UINT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UNORM] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_32_FLOAT] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tbuffer_store_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_d16_x v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_d16_xy v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s3 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_d16_x v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_d16_xy v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s4 offset:8388607
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+;; Image load/store/atomic, Sample & Gather ops, image_bvh.
+
+image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_load_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_load_pck v5, v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_load_pck_sgn v5, v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_load_mip_pck v5, [v0, v1], s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_store v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_store_mip v[252:255], [v0, v1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_store_pck v5, v1, s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_store_mip_pck v5, [v0, v1], s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_swap v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_cmpswap v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_add_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_sub_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_min_int v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_min_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_max_int v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_max_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_and v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_or v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_xor v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_inc_uint v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_dec_uint v1, [v2, v3], s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_pk_add_f16 v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_pk_add_bf16 v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_add_flt v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_min_flt v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_atomic_max_flt v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_bvh_intersect_ray v[4:7], [v9, v10, v[11:13], v[14:16], v[17:19]], s[4:7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_bvh64_intersect_ray v[4:7], [v[9:10], v11, v[12:14], v[15:17], v[18:20]], s[4:7]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_bvh_dual_intersect_ray v[0:9], [v[0:1], v[11:12], v[3:5], v[6:8], v[9:10]], s[0:3]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_bvh8_intersect_ray v[0:9], [v[0:1], v[11:12], v[3:5], v[6:8], v9], s[0:3]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_get_resinfo v4, v32, s[96:103] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_l v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_lz v64, v32, s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_l v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_lz v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_o v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_l_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_lz_o v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_l_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_lz_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4 v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_l v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_b v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_lz v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_lz v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_o v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_lz_o v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_lz_o v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_get_lod v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_g16 v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_o_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_o_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_cl v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_cl v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_cl v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_cl v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_cl_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_cl_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_cl_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_o v64, [v32, v33, v34, v[35:37]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_cl_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_o_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_o_g16 v64, [v32, v33, v34, v[35:37]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_cl v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_b_cl v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_cl v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_l v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_b v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_b_cl v[64:67], [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4h v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_msaa_load v[1:4], [v5, v6, v7], s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s
new file mode 100644
index 000000000000..5b6bb477bdc9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vds_alias.s
@@ -0,0 +1,7 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+
+ds_load_tr_b64 v[2:3], v0
+// GFX1250: ds_load_tr8_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+
+ds_load_tr_b128 v[2:5], v0
+// GFX1250: ds_load_tr16_b128 v[2:5], v0            ; encoding: [0x00,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s
new file mode 100644
index 000000000000..07b4055f0ab9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat.s
@@ -0,0 +1,22 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_save s[0:1]
+// GFX1250: tensor_save s[0:1] ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_save s[0:1] offset:32
+// GFX1250: tensor_save s[0:1] offset:32 ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_stop
+// GFX1250: tensor_stop ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s
new file mode 100644
index 000000000000..6b2dd67b073e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vflat_alias.s
@@ -0,0 +1,37 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+global_load_tr_b64 v[2:3], v0, s[0:1]
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v0, s[0:1] offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_tr_b64 v[2:3], v[4:5], off
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v[4:5], off offset:64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+
+global_load_tr_b64 v[2:3], v[4:5], off offset:-64
+// GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+
+global_load_tr_b128 v[2:5], v0, s[0:1]
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v0, s[0:1] offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v0, s[0:1] offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_tr_b128 v[2:5], v[6:7], off
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v[6:7], off offset:64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+
+global_load_tr_b128 v[2:5], v[6:7], off offset:-64
+// GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:-64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s
new file mode 100644
index 000000000000..fec8ba19f93f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage.s
@@ -0,0 +1,34 @@
+; RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_load_to_lds s[0:3], s[4:11]
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19]
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11]
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19]
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
new file mode 100644
index 000000000000..3b2e03f1d4b9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
@@ -0,0 +1,25 @@
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+tensor_load_to_lds s[0:3], s[4:11] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_load_to_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for load instructions
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for load instructions
+
+tensor_store_from_lds s[0:3], s[4:11] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_store_from_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for store instructions
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] r128
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must set modifier r128=0
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for store instructions
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
new file mode 100644
index 000000000000..7b07c84d5668
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1-fake16.s
@@ -0,0 +1,89 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+v_cvt_f32_bf16 v5, v1
+// GFX1250: v_cvt_f32_bf16_e32 v5, v1               ; encoding: [0x01,0xe5,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, v127
+// GFX1250: v_cvt_f32_bf16_e32 v5, v127             ; encoding: [0x7f,0xe5,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, s1
+// GFX1250: v_cvt_f32_bf16_e32 v5, s1               ; encoding: [0x01,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, s105
+// GFX1250: v_cvt_f32_bf16_e32 v5, s105             ; encoding: [0x69,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, vcc_lo
+// GFX1250: v_cvt_f32_bf16_e32 v5, vcc_lo           ; encoding: [0x6a,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, vcc_hi
+// GFX1250: v_cvt_f32_bf16_e32 v5, vcc_hi           ; encoding: [0x6b,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, ttmp15
+// GFX1250: v_cvt_f32_bf16_e32 v5, ttmp15           ; encoding: [0x7b,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, m0
+// GFX1250: v_cvt_f32_bf16_e32 v5, m0               ; encoding: [0x7d,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, exec_lo
+// GFX1250: v_cvt_f32_bf16_e32 v5, exec_lo          ; encoding: [0x7e,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, exec_hi
+// GFX1250: v_cvt_f32_bf16_e32 v5, exec_hi          ; encoding: [0x7f,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, null
+// GFX1250: v_cvt_f32_bf16_e32 v5, null             ; encoding: [0x7c,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, -1
+// GFX1250: v_cvt_f32_bf16_e32 v5, -1               ; encoding: [0xc1,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, 0.5
+// GFX1250: v_cvt_f32_bf16_e32 v5, 0.5              ; encoding: [0xf0,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, src_scc
+// GFX1250: v_cvt_f32_bf16_e32 v5, src_scc          ; encoding: [0xfd,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v127, 0x8000
+// GFX1250: v_cvt_f32_bf16_e32 v127, 0x8000         ; encoding: [0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1, v2                ; encoding: [0x02,0xf1,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, s2
+// GFX1250: v_cvt_f16_bf8_e32 v1, s2                ; encoding: [0x02,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, 2
+// GFX1250: v_cvt_f16_bf8_e32 v1, 2                 ; encoding: [0x82,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1, 0x1234
+// GFX1250: v_cvt_f16_bf8_e32 v1, 0x1234            ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1, v2                ; encoding: [0x02,0xef,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, s2
+// GFX1250: v_cvt_f16_fp8_e32 v1, s2                ; encoding: [0x02,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, 2
+// GFX1250: v_cvt_f16_fp8_e32 v1, 2                 ; encoding: [0x82,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1, 0x1234
+// GFX1250: v_cvt_f16_fp8_e32 v1, 0x1234            ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v2
+// GFX1250: v_cvt_pk_f16_bf8 v1, v2                 ; encoding: [0x02,0xed,0x02,0x7e]
+
+v_cvt_pk_f16_bf8 v1, s2
+// GFX1250: v_cvt_pk_f16_bf8 v1, s2                 ; encoding: [0x02,0xec,0x02,0x7e]
+
+v_cvt_pk_f16_bf8 v1, 100
+// GFX1250: v_cvt_pk_f16_bf8 v1, 0x64               ; encoding: [0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v2
+// GFX1250: v_cvt_pk_f16_fp8 v1, v2                 ; encoding: [0x02,0xeb,0x02,0x7e]
+
+v_cvt_pk_f16_fp8 v1, s2
+// GFX1250: v_cvt_pk_f16_fp8 v1, s2                 ; encoding: [0x02,0xea,0x02,0x7e]
+
+v_cvt_pk_f16_fp8 v1, 100
+// GFX1250: v_cvt_pk_f16_fp8 v1, 0x64               ; encoding: [0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
new file mode 100644
index 000000000000..30c62c957874
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1.s
@@ -0,0 +1,98 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+v_cvt_f32_bf16 v5, v1
+// GFX1250: v_cvt_f32_bf16_e32 v5, v1               ; encoding: [0x01,0xe5,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, v127
+// GFX1250: v_cvt_f32_bf16_e32 v5, v127             ; encoding: [0x7f,0xe5,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, s1
+// GFX1250: v_cvt_f32_bf16_e32 v5, s1               ; encoding: [0x01,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, s105
+// GFX1250: v_cvt_f32_bf16_e32 v5, s105             ; encoding: [0x69,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, vcc_lo
+// GFX1250: v_cvt_f32_bf16_e32 v5, vcc_lo           ; encoding: [0x6a,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, vcc_hi
+// GFX1250: v_cvt_f32_bf16_e32 v5, vcc_hi           ; encoding: [0x6b,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, ttmp15
+// GFX1250: v_cvt_f32_bf16_e32 v5, ttmp15           ; encoding: [0x7b,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, m0
+// GFX1250: v_cvt_f32_bf16_e32 v5, m0               ; encoding: [0x7d,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, exec_lo
+// GFX1250: v_cvt_f32_bf16_e32 v5, exec_lo          ; encoding: [0x7e,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, exec_hi
+// GFX1250: v_cvt_f32_bf16_e32 v5, exec_hi          ; encoding: [0x7f,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, null
+// GFX1250: v_cvt_f32_bf16_e32 v5, null             ; encoding: [0x7c,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, -1
+// GFX1250: v_cvt_f32_bf16_e32 v5, -1               ; encoding: [0xc1,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, 0.5
+// GFX1250: v_cvt_f32_bf16_e32 v5, 0.5              ; encoding: [0xf0,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v5, src_scc
+// GFX1250: v_cvt_f32_bf16_e32 v5, src_scc          ; encoding: [0xfd,0xe4,0x0a,0x7e]
+
+v_cvt_f32_bf16 v127, 0x8000
+// GFX1250: v_cvt_f32_bf16_e32 v127, 0x8000         ; encoding: [0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00]
+
+v_cvt_f32_bf16 v5, v1.h
+// GFX1250: v_cvt_f32_bf16_e32 v5, v1.h             ; encoding: [0x81,0xe5,0x0a,0x7e]
+
+v_cvt_f16_bf8 v1.l, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, v2              ; encoding: [0x02,0xf1,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, s2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, s2              ; encoding: [0x02,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, 2
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, 2               ; encoding: [0x82,0xf0,0x02,0x7e]
+
+v_cvt_f16_bf8 v1.l, 0x1234
+// GFX1250: v_cvt_f16_bf8_e32 v1.l, 0x1234          ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1.h, v2
+// GFX1250: v_cvt_f16_bf8_e32 v1.h, v2              ; encoding: [0x02,0xf1,0x02,0x7f]
+
+v_cvt_f16_fp8 v1.l, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, v2              ; encoding: [0x02,0xef,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, s2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, s2              ; encoding: [0x02,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, 2
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, 2               ; encoding: [0x82,0xee,0x02,0x7e]
+
+v_cvt_f16_fp8 v1.l, 0x1234
+// GFX1250: v_cvt_f16_fp8_e32 v1.l, 0x1234          ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1.h, v2
+// GFX1250: v_cvt_f16_fp8_e32 v1.h, v2              ; encoding: [0x02,0xef,0x02,0x7f]
+
+v_cvt_pk_f16_bf8 v1, v2
+// GFX1250: v_cvt_pk_f16_bf8 v1, v2                 ; encoding: [0x02,0xed,0x02,0x7e]
+
+v_cvt_pk_f16_bf8 v1, s2
+// GFX1250: v_cvt_pk_f16_bf8 v1, s2                 ; encoding: [0x02,0xec,0x02,0x7e]
+
+v_cvt_pk_f16_bf8 v1, 100
+// GFX1250: v_cvt_pk_f16_bf8 v1, 0x64               ; encoding: [0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v2
+// GFX1250: v_cvt_pk_f16_fp8 v1, v2                 ; encoding: [0x02,0xeb,0x02,0x7e]
+
+v_cvt_pk_f16_fp8 v1, s2
+// GFX1250: v_cvt_pk_f16_fp8 v1, s2                 ; encoding: [0x02,0xea,0x02,0x7e]
+
+v_cvt_pk_f16_fp8 v1, 100
+// GFX1250: v_cvt_pk_f16_fp8 v1, 0x64               ; encoding: [0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
new file mode 100644
index 000000000000..e53812bb3fd0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16-fake16.s
@@ -0,0 +1,83 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_mirror
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_half_mirror
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shl:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shl:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shr:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shr:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_ror:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_ror:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX1250: v_cvt_f32_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
new file mode 100644
index 000000000000..bd767d14fab5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp16.s
@@ -0,0 +1,103 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16 v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_mirror
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_half_mirror
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shl:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shl:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shr:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_shr:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_ror:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_ror:15
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x60,0x09,0x13]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX1250: v_cvt_f32_bf16_dpp v127, -|v127| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1.h quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2.h quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.h quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xec,0x02,0x7e,0x82,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.h quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.h quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xea,0x02,0x7e,0x82,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
new file mode 100644
index 000000000000..cbc0ebd3edda
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8-fake16.s
@@ -0,0 +1,39 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_cvt_f32_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
new file mode 100644
index 000000000000..8b9980a31daf
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_dpp8.s
@@ -0,0 +1,59 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v127, v127 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX1250: v_cvt_f32_bf16_dpp v127, v127 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16 v5, v1.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_dpp v5, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_bf8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xec,0x02,0x7e,0x82,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_fp8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_err.s
new file mode 100644
index 000000000000..c393d3e81988
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop1_err.s
@@ -0,0 +1,36 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_pk_f16_bf8 v1, v2 clamp
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR-NEXT:{{^}}v_cvt_pk_f16_bf8 v1, v2 clamp
+// GFX1250-ERR-NEXT:{{^}}                        ^
+
+v_cvt_pk_f16_bf8 v1, v2 mul:2
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR-NEXT:{{^}}v_cvt_pk_f16_bf8 v1, v2 mul:2
+// GFX1250-ERR-NEXT:{{^}}                        ^
+
+v_cvt_pk_f16_fp8 v1, v2 clamp
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR-NEXT:{{^}}v_cvt_pk_f16_fp8 v1, v2 clamp
+// GFX1250-ERR-NEXT:{{^}}                        ^
+
+v_cvt_pk_f16_fp8 v1, v2 mul:2
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR-NEXT:{{^}}v_cvt_pk_f16_fp8 v1, v2 mul:2
+// GFX1250-ERR-NEXT:{{^}}                        ^
+
+v_cvt_f32_bf16 v5, v1 clamp
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR-NEXT:{{^}}v_cvt_f32_bf16 v5, v1 clamp
+// GFX1250-ERR-NEXT:{{^}}                      ^
+
+v_cvt_f32_bf16 v5, v1 mul:2
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR-NEXT:{{^}}v_cvt_f32_bf16 v5, v1 mul:2
+// GFX1250-ERR-NEXT:{{^}}                      ^
+
+v_cvt_f32_bf16 v5, v1 div:2
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR-NEXT:{{^}}v_cvt_f32_bf16 v5, v1 div:2
+// GFX1250-ERR-NEXT:{{^}}                      ^
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3-fake16.s
new file mode 100644
index 000000000000..0070c8ab9ee7
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3-fake16.s
@@ -0,0 +1,17 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+
+v_lshl_add_u64 v[2:3], s[4:5], v7, v[8:9]
+// GFX1250: v_lshl_add_u64 v[2:3], s[4:5], v7, v[8:9] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0e,0x22,0x04]
+
+v_lshl_add_u64 v[2:3], v[4:5], 0, 1
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 0, 1     ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x01,0x05,0x02]
+
+v_lshl_add_u64 v[2:3], v[4:5], 3, s[2:3]
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 3, s[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x07,0x09,0x00]
+
+v_lshl_add_u64 v[2:3], s[4:5], 4, v[2:3]
+// GFX1250: v_lshl_add_u64 v[2:3], s[4:5], 4, v[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x08,0x09,0x04]
+
+v_lshl_add_u64 v[2:3], v[4:5], v7, 12345
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], v7, 0x3039 ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0f,0xfe,0x03,0x39,0x30,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3.s
new file mode 100644
index 000000000000..553eacc8e7b6
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3.s
@@ -0,0 +1,17 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+
+v_lshl_add_u64 v[2:3], s[4:5], v7, v[8:9]
+// GFX1250: v_lshl_add_u64 v[2:3], s[4:5], v7, v[8:9] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0e,0x22,0x04]
+
+v_lshl_add_u64 v[2:3], v[4:5], 0, 1
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 0, 1     ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x01,0x05,0x02]
+
+v_lshl_add_u64 v[2:3], v[4:5], 3, s[2:3]
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 3, s[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x07,0x09,0x00]
+
+v_lshl_add_u64 v[2:3], s[4:5], 4, v[2:3]
+// GFX1250: v_lshl_add_u64 v[2:3], s[4:5], 4, v[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x08,0x09,0x04]
+
+v_lshl_add_u64 v[2:3], v[4:5], v7, 12345
+// GFX1250: v_lshl_add_u64 v[2:3], v[4:5], v7, 0x3039 ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0f,0xfe,0x03,0x39,0x30,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
new file mode 100644
index 000000000000..e2fafe415ff7
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
@@ -0,0 +1,11 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX125X-ERR,GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                          ^
+
+v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                          ^
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
new file mode 100644
index 000000000000..b333541a0f57
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1-fake16.s
@@ -0,0 +1,155 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+v_cvt_f32_bf16_e64 v5, v1
+// GFX1250: v_cvt_f32_bf16_e64 v5, v1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v255
+// GFX1250: v_cvt_f32_bf16_e64 v5, v255             ; encoding: [0x05,0x00,0xf2,0xd5,0xff,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s1
+// GFX1250: v_cvt_f32_bf16_e64 v5, s1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s105
+// GFX1250: v_cvt_f32_bf16_e64 v5, s105             ; encoding: [0x05,0x00,0xf2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_lo
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_lo           ; encoding: [0x05,0x00,0xf2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_hi
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_hi           ; encoding: [0x05,0x00,0xf2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, ttmp15
+// GFX1250: v_cvt_f32_bf16_e64 v5, ttmp15           ; encoding: [0x05,0x00,0xf2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, m0
+// GFX1250: v_cvt_f32_bf16_e64 v5, m0               ; encoding: [0x05,0x00,0xf2,0xd5,0x7d,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_lo
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_lo          ; encoding: [0x05,0x00,0xf2,0xd5,0x7e,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_hi
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_hi          ; encoding: [0x05,0x00,0xf2,0xd5,0x7f,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, null
+// GFX1250: v_cvt_f32_bf16_e64 v5, null             ; encoding: [0x05,0x00,0xf2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, -1
+// GFX1250: v_cvt_f32_bf16_e64 v5, -1               ; encoding: [0x05,0x00,0xf2,0xd5,0xc1,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, v1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v255 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, v255 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xff,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, s1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s105 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, s105 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_lo op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_hi op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, ttmp15 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, ttmp15 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, m0 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, m0 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x7d,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_lo op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7e,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_hi op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7f,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, null op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, null op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, -1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, -1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0xc1,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xfd,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150, v2
+// GFX1250: v_cvt_f16_bf8_e64 v150, v2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v150, s2
+// GFX1250: v_cvt_f16_bf8_e64 v150, s2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150, 2
+// GFX1250: v_cvt_f16_bf8_e64 v150, 2               ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150, 0x1234
+// GFX1250: v_cvt_f16_bf8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v128, v2 op_sel:[0,1]
+// GFX1250: v_cvt_f16_bf8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150, v2
+// GFX1250: v_cvt_f16_fp8_e64 v150, v2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150, s2
+// GFX1250: v_cvt_f16_fp8_e64 v150, s2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150, 2
+// GFX1250: v_cvt_f16_fp8_e64 v150, 2               ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150, 0x1234
+// GFX1250: v_cvt_f16_fp8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v128, v2 op_sel:[0,1]
+// GFX1250: v_cvt_f16_fp8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v150
+// GFX1250: v_cvt_pk_f16_bf8 v1, v150               ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v150 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, s2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v150
+// GFX1250: v_cvt_pk_f16_fp8 v1, v150               ; encoding: [0x01,0x00,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v150 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, s2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
new file mode 100644
index 000000000000..df595fe562e0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1.s
@@ -0,0 +1,158 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+v_cvt_f32_bf16_e64 v5, v1
+// GFX1250: v_cvt_f32_bf16_e64 v5, v1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v255
+// GFX1250: v_cvt_f32_bf16_e64 v5, v255             ; encoding: [0x05,0x00,0xf2,0xd5,0xff,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s1
+// GFX1250: v_cvt_f32_bf16_e64 v5, s1               ; encoding: [0x05,0x00,0xf2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s105
+// GFX1250: v_cvt_f32_bf16_e64 v5, s105             ; encoding: [0x05,0x00,0xf2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_lo
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_lo           ; encoding: [0x05,0x00,0xf2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_hi
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_hi           ; encoding: [0x05,0x00,0xf2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, ttmp15
+// GFX1250: v_cvt_f32_bf16_e64 v5, ttmp15           ; encoding: [0x05,0x00,0xf2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, m0
+// GFX1250: v_cvt_f32_bf16_e64 v5, m0               ; encoding: [0x05,0x00,0xf2,0xd5,0x7d,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_lo
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_lo          ; encoding: [0x05,0x00,0xf2,0xd5,0x7e,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_hi
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_hi          ; encoding: [0x05,0x00,0xf2,0xd5,0x7f,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, null
+// GFX1250: v_cvt_f32_bf16_e64 v5, null             ; encoding: [0x05,0x00,0xf2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, -1
+// GFX1250: v_cvt_f32_bf16_e64 v5, -1               ; encoding: [0x05,0x00,0xf2,0xd5,0xc1,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, v1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x01,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v255 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, v255 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xff,0x01,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, s1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x01,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, s105 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, s105 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x69,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_lo op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6a,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, vcc_hi op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, vcc_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x6b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, ttmp15 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, ttmp15 op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7b,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, m0 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, m0 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0x7d,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_lo op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_lo op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7e,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, exec_hi op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, exec_hi op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7f,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, null op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, null op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x7c,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, -1 op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, -1 op_sel:[1,0]  ; encoding: [0x05,0x08,0xf2,0xd5,0xc1,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1]
+// GFX1250: v_cvt_f32_bf16_e64 v5, src_scc op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xfd,0x00,0x00,0x00]
+
+v_cvt_f32_bf16_e64 v5, v128.h
+// GFX1250: v_cvt_f32_bf16_e64 v5, v128.h op_sel:[1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0x80,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, v2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, s2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, 2
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_bf8 v150.l, 0x1234
+// GFX1250: v_cvt_f16_bf8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3
+// GFX1250: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v128.h, v2
+// GFX1250: v_cvt_f16_bf8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:2
+// GFX1250: v_cvt_f16_bf8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, v2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, s2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, 2
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+v_cvt_f16_fp8 v150.l, 0x1234
+// GFX1250: v_cvt_f16_fp8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3
+// GFX1250: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v128.h, v2
+// GFX1250: v_cvt_f16_fp8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:2
+// GFX1250: v_cvt_f16_fp8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v150
+// GFX1250: v_cvt_pk_f16_bf8 v1, v150               ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, v150 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_bf8 v1, s2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_bf8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v150
+// GFX1250: v_cvt_pk_f16_fp8 v1, v150               ; encoding: [0x01,0x00,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, v150 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+v_cvt_pk_f16_fp8 v1, s2 op_sel:[1]
+// GFX1250: v_cvt_pk_f16_fp8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
new file mode 100644
index 000000000000..b4000ce9425f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16-fake16.s
@@ -0,0 +1,103 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_mirror
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128 op_sel:[1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128 op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x08,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 op_sel:[1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v2 op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x08,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
new file mode 100644
index 000000000000..82d8245c8624
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp16.s
@@ -0,0 +1,115 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_mirror
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_half_mirror
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:1
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:15
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f32_bf16_e64_dpp v5, v128.h quad_perm:[3,2,1,0]
+// GFX1250: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:3 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v128.l, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:3 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v128.l, v2 quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128.h quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128.h quad_perm:[0,1,2,3]
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128.h op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
new file mode 100644
index 000000000000..8e0bdfeb7885
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8-fake16.s
@@ -0,0 +1,75 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f16_bf8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v2 op_sel:[1] dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v2 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v2 op_sel:[1] dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v2 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf5,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
new file mode 100644
index 000000000000..eaf63eabea42
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop1_dpp8.s
@@ -0,0 +1,83 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_cvt_f16_bf8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_bf8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v1.h, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_bf8 v128.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_bf8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v1.h, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_f16_fp8 v128.l, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_f16_fp8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_bf8 v1, v128.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
+
+v_cvt_pk_f16_fp8 v1, v128.h dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250: v_cvt_pk_f16_fp8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf5,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+// GFX12-ERR: :[[@LINE-2]]:1: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx950_asm_features.s b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
index 7bc47914f40b..57fc573b37ba 100644
--- a/llvm/test/MC/AMDGPU/gfx950_asm_features.s
+++ b/llvm/test/MC/AMDGPU/gfx950_asm_features.s
@@ -40,14 +40,26 @@ global_load_lds_dwordx4 v2, s[4:5] offset:4
 // GFX950: v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 v_permlane16_swap_b32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+v_permlane16_swap_b32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 v_permlane16_swap_b32_e32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+v_permlane16_swap_b32_e32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
 v_permlane16_swap_b32_e64 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane16_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+v_permlane16_swap_b32_e64 v218, v219
+
 // FIXME: Parsed as bound_ctrl:1?
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00]
@@ -81,14 +93,26 @@ v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 fi:1
 // GFX950: v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 v_permlane32_swap_b32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+v_permlane32_swap_b32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 v_permlane32_swap_b32_e32 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+v_permlane32_swap_b32_e32 v218, v219
+
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
 v_permlane32_swap_b32_e64 v1, v2
 
+// NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
+// GFX950: v_permlane32_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+v_permlane32_swap_b32_e64 v218, v219
+
 // FIXME: Parsed as bound_ctrl:1?
 // NOT-GFX950: :[[@LINE+2]]:{{[0-9]+}}: error:
 // GFX950: v_permlane32_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop2_features.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop2_features.s
new file mode 100644
index 000000000000..4b5efd00a7ad
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop2_features.s
@@ -0,0 +1,91 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx908 -show-encoding %s | FileCheck --check-prefix=CHECK-MI %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx90a -show-encoding %s | FileCheck --check-prefix=CHECK-MI %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx942 -show-encoding %s | FileCheck --check-prefix=CHECK-MI %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx950 -show-encoding %s | FileCheck --check-prefix=CHECK-MI %s
+
+v_pk_fmac_f16 v5, v1, v2
+// CHECK-MI: [0x01,0x05,0x0a,0x78]
+
+v_pk_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// CHECK-MI: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0xff]
+
+v_pk_fmac_f16 v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// CHECK-MI: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0x00]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_0 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x00,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x01,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_2 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x02,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x03,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:WORD_0 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x04,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x05,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_SEXT src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x0e,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x16,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x16,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x00,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x01,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x02,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x03,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x04,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x05,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x00]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x01]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x02]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x03]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x04]
+
+v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x05]
+
+v_pk_fmac_f16_sdwa v5, v1, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+// CHECK-MI: [0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x16]
diff --git a/llvm/test/MC/AMDGPU/max-branch-distance.s b/llvm/test/MC/AMDGPU/max-branch-distance.s
index 0e3337a7e04b..46aa524f7ab4 100644
--- a/llvm/test/MC/AMDGPU/max-branch-distance.s
+++ b/llvm/test/MC/AMDGPU/max-branch-distance.s
@@ -1,5 +1,5 @@
 //  RUN: not llvm-mc -triple=amdgcn -filetype=obj -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERROR %s
-//  ERROR: max-branch-distance.s:7:3: error: branch size exceeds simm16
+//  ERROR: max-branch-distance.s:7:12: error: branch size exceeds simm16
 
 // fill v_nop
 LBB0_0:
diff --git a/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s b/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s
new file mode 100644
index 000000000000..cef1e8372900
--- /dev/null
+++ b/llvm/test/MC/ARM/Relocations/thumb-branch-out-of-range.s
@@ -0,0 +1,71 @@
+@ RUN: not llvm-mc %s -triple thumbv7-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv8-m.baseline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv8-m.mainline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv6m-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+@ RUN: not llvm-mc %s -triple thumbv5-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck -check-prefix=CHECKSHORT %s
+
+// Thumb BL has range +- 4 Megabytes if CPU does not support Thumb2 or does not
+// have v8-M baseline ops, it is +- 16 Megabytes otherwise.
+
+.code 16
+bl      shortend
+.space 0x3fffff
+shortend:
+// CHECKSHORT-NOT: error:
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      shortend2
+.space 0x400000
+shortend2:
+
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      end
+.space 0xffffff
+end:
+bl      end2
+.space 0xffffff
+.global end2
+end2:
+
+bl      end3
+.space 0x1000000
+.global end3
+end3:
+
+// CHECK-NOT: error:
+// CHECKSHORT-NOT: error:
+// CHECKSHORT: [[@LINE+2]]:[[#]]: error: Relocation out of range
+// CHECK: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl      end4
+.space 0x1000000
+end4:
+
+shortstart1:
+.space 0x3ffffc
+bl shortstart1
+
+shortstart2:
+.space 0x400000
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl shortstart2
+
+start1:
+.space 0xfffffc
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl start1
+
+.global start2
+start2:
+.space 0xfffffc
+bl start2
+
+.global start3
+start3:
+.space 0xfffffd
+bl start3
+
+// CHECK-NOT: error:
+start4:
+.space 0xfffffd
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
+// CHECKSHORT: [[@LINE+1]]:[[#]]: error: Relocation out of range
+bl start4
diff --git a/llvm/test/MC/ARM/Relocations/thumb-branch.s b/llvm/test/MC/ARM/Relocations/thumb-branch.s
new file mode 100644
index 000000000000..e0a487b719d8
--- /dev/null
+++ b/llvm/test/MC/ARM/Relocations/thumb-branch.s
@@ -0,0 +1,21 @@
+// RUN: llvm-mc -filetype=obj -triple=thumbv7a-linux-gnueabi %s -o %t
+// RUN: llvm-objdump -dr %t --triple=thumbv7a | FileCheck %s
+
+// CHECK:      f7ff fffe     bl      {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               00000000:  R_ARM_THM_CALL       foo
+// CHECK-NEXT: f7ff effe     blx     {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               00000004:  R_ARM_THM_CALL       callee_thumb_low
+// CHECK-NEXT: bf00          nop
+// CHECK-NEXT: f7ff effe     blx     {{.*}}                  @ imm = #-0x4
+// CHECK-NEXT:               0000000a:  R_ARM_THM_CALL       callee_thumb_high
+// CHECK-NEXT: 4770          bx      lr
+
+ .syntax unified
+ .section .text, "ax",%progbits
+ .thumb
+ bl foo
+
+ blx   callee_thumb_low
+ nop
+ blx   callee_thumb_high
+ bx lr
diff --git a/llvm/test/MC/ARM/Windows/branch-reloc-offset.s b/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
index e7d59cdcf6d4..f5b2ea704033 100644
--- a/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
+++ b/llvm/test/MC/ARM/Windows/branch-reloc-offset.s
@@ -65,11 +65,11 @@ err:
 // Test errors, if referencing a symbol with an offset
 
     b .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:19: error: cannot perform a PC-relative fixup with a non-zero symbol offset
     bl .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:20: error: cannot perform a PC-relative fixup with a non-zero symbol offset
     blx .Lerr_target+4
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:21: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
 // Test errors, if referencing a private label which lacks .def/.scl/.type/.endef, in another
 // section, without an offset. Such symbols are omitted from the output symbol table, so the
@@ -77,7 +77,7 @@ err:
 // section plus an offset, but such an offset is not supported with this relocation.
 
     b .Lerr_target2
-// ERR: [[#@LINE-1]]:5: error: cannot perform a PC-relative fixup with a non-zero symbol offset
+// ERR: [[#@LINE-1]]:7: error: cannot perform a PC-relative fixup with a non-zero symbol offset
 
     .def .Lerr_target
     .scl 3
diff --git a/llvm/test/MC/ARM/Windows/invalid-relocation.s b/llvm/test/MC/ARM/Windows/invalid-relocation.s
index 821de8dd5560..01528614c11d 100644
--- a/llvm/test/MC/ARM/Windows/invalid-relocation.s
+++ b/llvm/test/MC/ARM/Windows/invalid-relocation.s
@@ -9,4 +9,4 @@
 	.thumb_func
 	adr r0, invalid_relocation+1
 
-# CHECK: 10:2: error: unsupported relocation type
+# CHECK: 10:28: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/arm-memory-instructions-immediate.s b/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
index d72028edc9e2..23cf547226cc 100644
--- a/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
+++ b/llvm/test/MC/ARM/arm-memory-instructions-immediate.s
@@ -21,5 +21,5 @@ foo:
 // CHECK-NEXT: strb r0, [r1, #1024]
 .ifdef ERR
     str r0, [r1, 1b]
-// ERR:[[#@LINE-1]]:5: error: unsupported relocation type
+// ERR:[[#@LINE-1]]:18: error: unsupported relocation type
 .endif
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
index fdc49bcde4ed..d52ced6e20a1 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
@@ -12,7 +12,6 @@
 f:
         push    {r7, lr}
         mov     r7, sp
-        bl      g
         pop     {r7, pc}
 
 	.section	.data.rel.local,"aw",%progbits
@@ -20,16 +19,10 @@ ptr:
 	.long	f
 
 
-@@ make sure an R_ARM_THM_CALL relocation is generated for the call to g
-@CHECK:      Relocations [
-@CHECK-NEXT:   Section {{.*}} .rel.text {
-@CHECK-NEXT:     0x4 R_ARM_THM_CALL g
-@CHECK-NEXT:   }
-
 
 @@ make sure the relocation is with f. That is one way to make sure it includes
 @@ the thumb bit.
-@CHECK-NEXT:   Section ({{.*}}) .rel.data.rel.local {
+@CHECK:        Section ({{.*}}) .rel.data.rel.local {
 @CHECK-NEXT:     0x0 R_ARM_ABS32 f
 @CHECK-NEXT:   }
 @CHECK-NEXT: ]
diff --git a/llvm/test/MC/ARM/lower-upper-errors-2.s b/llvm/test/MC/ARM/lower-upper-errors-2.s
index 58c1f1179230..7b21b5a266cb 100644
--- a/llvm/test/MC/ARM/lower-upper-errors-2.s
+++ b/llvm/test/MC/ARM/lower-upper-errors-2.s
@@ -8,11 +8,11 @@
 // For errors that are reported earlier, when initially reading the
 // instructions, see lower-upper-errors.s.
 
-// CHECK: [[@LINE+1]]:1: error: unsupported relocation
+// CHECK: [[@LINE+1]]:15: error: unsupported relocation
 adds r0, r0, #foo
 
-// CHECK: [[@LINE+1]]:1: error: unsupported relocation
+// CHECK: [[@LINE+1]]:[[#]]: error: unsupported relocation
 add r9, r0, #foo
 
-// CHECK: [[@LINE+1]]:1: error: expected relocatable expression
+// CHECK: [[@LINE+1]]:11: error: expected relocatable expression
 movs r11, :upper8_15:#foo
diff --git a/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s b/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
index 0b745a97687d..b235bde45e99 100644
--- a/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
+++ b/llvm/test/MC/ARM/pcrel-ldrd-diff-section.s
@@ -16,8 +16,8 @@ bar:
     ldrd r0, r1, foo1    @ arm_pcrel_10_unscaled
     ldrd r0, r1, foo2-8  @ arm_pcrel_10_unscaled
 .ifdef ERR
-  @ ERR:[[#@LINE-3]]:5: error: unsupported relocation type
-  @ ERR:[[#@LINE-3]]:5: error: unsupported relocation type
+  @ ERR:[[#@LINE-3]]:18: error: unsupported relocation type
+  @ ERR:[[#@LINE-3]]:22: error: unsupported relocation type
 .endif
     bx lr
 
diff --git a/llvm/test/MC/ARM/pcrel-vldr-diff-section.s b/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
index 44b1032a8f63..ce058fccdcba 100644
--- a/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
+++ b/llvm/test/MC/ARM/pcrel-vldr-diff-section.s
@@ -9,6 +9,6 @@ vldr s0, foo     @ arm_pcrel_10 / t2_pcrel_10
 vldr d0, foo     @ arm_pcrel_10 / t2_pcrel_10
 vldr.16 s0,foo   @ arm_pcrel_9 / t2_pcrel_9
 
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
-@ CHECK: :[[#@LINE-4]]:1: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:10: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:10: error: unsupported relocation type
+@ CHECK: :[[#@LINE-4]]:12: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/quad-relocation.s b/llvm/test/MC/ARM/quad-relocation.s
index 50d8d82be584..89c1653b957a 100644
--- a/llvm/test/MC/ARM/quad-relocation.s
+++ b/llvm/test/MC/ARM/quad-relocation.s
@@ -3,7 +3,7 @@
 
 	.align 3
 symbol:
-@ CHECK:      :[[#@LINE+1]]:6: error: unsupported relocation type
+@ CHECK:      :[[#@LINE+1]]:7: error: unsupported relocation type
 .quad(symbol)
 @ CHECK:      :[[#@LINE+1]]:8: error: unsupported relocation type
 .8byte symbol
diff --git a/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s b/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
index 642484080b1c..12b5eae77f75 100644
--- a/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
+++ b/llvm/test/MC/ARM/t2-modified-immediate-fixup-error1.s
@@ -10,4 +10,4 @@
     .equ sym0, 0x01abcdef
 .L2:
     mov r0, .L2
-@ CHECK: :[[#@LINE-1]]:5: error: unsupported relocation type
+@ CHECK: :[[#@LINE-1]]:13: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/thumb-branches.s b/llvm/test/MC/ARM/thumb-branches.s
deleted file mode 100644
index c9e1660e1cdb..000000000000
--- a/llvm/test/MC/ARM/thumb-branches.s
+++ /dev/null
@@ -1,71 +0,0 @@
-@ RUN: not llvm-mc %s -triple thumbv7-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv8-m.baseline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv8-m.mainline-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv6m-none-eabi -filetype=obj -o /dev/null 2>&1 | FileCheck %s
-@ RUN: not llvm-mc %s -triple thumbv5-linux-gnueabi -filetype=obj -o /dev/null 2>&1 | FileCheck -check-prefix=CHECKSHORT %s
-
-// Thumb BL has range +- 4 Megabytes if CPU does not support Thumb2 or does not
-// have v8-M baseline ops, it is +- 16 Megabytes otherwise.
-
-        .code 16
-        bl      shortend
-        .space 0x3fffff
-shortend:
-// CHECKSHORT-NOT: error:
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      shortend2
-        .space 0x400000
-shortend2:
-
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      end
-        .space 0xffffff
-end:
-        bl      end2
-        .space 0xffffff
-        .global end2
-end2:
-
-        bl      end3
-        .space 0x1000000
-        .global end3
-end3:
-
-// CHECK-NOT: error:
-// CHECKSHORT-NOT: error:
-// CHECKSHORT: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
-// CHECK: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl      end4
-        .space 0x1000000
-end4:
-
-shortstart1:
-        .space 0x3ffffc
-        bl shortstart1
-
-shortstart2:
-        .space 0x400000
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl shortstart2
-
-start1:
-        .space 0xfffffc
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl start1
-
-        .global start2
-start2:
-        .space 0xfffffc
-        bl start2
-
-        .global start3
-start3:
-        .space 0xfffffd
-        bl start3
-
-// CHECK-NOT: error:
-start4:
-        .space 0xfffffd
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
-// CHECKSHORT: [[@LINE+1]]:{{[0-9]}}: error: Relocation out of range
-        bl start4
diff --git a/llvm/test/MC/ARM/thumb1-relax-adr.s b/llvm/test/MC/ARM/thumb1-relax-adr.s
index 97b566f4833e..f5c78612395f 100644
--- a/llvm/test/MC/ARM/thumb1-relax-adr.s
+++ b/llvm/test/MC/ARM/thumb1-relax-adr.s
@@ -4,4 +4,4 @@
         .global func1
 _func1:
         adr r0, _func2
-@ CHECK-ERROR: :[[#@LINE-1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE-1]]:17: error: unsupported relocation type
diff --git a/llvm/test/MC/ARM/thumb1-relax-bcc.s b/llvm/test/MC/ARM/thumb1-relax-bcc.s
index 78f3477c1971..7839334bc25d 100644
--- a/llvm/test/MC/ARM/thumb1-relax-bcc.s
+++ b/llvm/test/MC/ARM/thumb1-relax-bcc.s
@@ -5,7 +5,7 @@
 
         .global func1
 _func1:
-@ CHECK-ERROR: :[[#@LINE+1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE+1]]:13: error: unsupported relocation type
         bne _func2
 
 @ CHECK-ELF: f47f affe          bne.w {{.+}} @ imm = #-4
diff --git a/llvm/test/MC/ARM/thumb1-relax-br.s b/llvm/test/MC/ARM/thumb1-relax-br.s
index b58dd6c0b413..618c4b631ff3 100644
--- a/llvm/test/MC/ARM/thumb1-relax-br.s
+++ b/llvm/test/MC/ARM/thumb1-relax-br.s
@@ -9,7 +9,7 @@ _func1:
         @ There is no MachO relocation for Thumb1's unconditional branch, so
         @ this is unrepresentable. FIXME: I think ELF could represent this.
         b _func2
-@ CHECK-ERROR: :[[#@LINE-1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE-1]]:11: error: unsupported relocation type
 
 @ CHECK-MACHO: f7ff bffe          b.w {{.+}} @ imm = #-4
 @ CHECK-MACHO-NEXT: ARM_THUMB_RELOC_BR22
diff --git a/llvm/test/MC/ARM/thumb1-relax-ldrlit.s b/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
index 8f455c89d41e..261527ef00ba 100644
--- a/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
+++ b/llvm/test/MC/ARM/thumb1-relax-ldrlit.s
@@ -3,5 +3,5 @@
 
         .global func1
 _func1:
-@ CHECK-ERROR: :[[#@LINE+1]]:9: error: unsupported relocation type
+@ CHECK-ERROR: :[[#@LINE+1]]:17: error: unsupported relocation type
         ldr r0, _func2
diff --git a/llvm/test/MC/ARM/thumb2-branch-ranges.s b/llvm/test/MC/ARM/thumb2-branch-ranges.s
index fc8044acde8a..495ceb87f6c1 100644
--- a/llvm/test/MC/ARM/thumb2-branch-ranges.s
+++ b/llvm/test/MC/ARM/thumb2-branch-ranges.s
@@ -26,7 +26,7 @@ end3:   bx lr
 
 // branch to thumb function is resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: b.w end4
         b.w end4
         .space 0x1000000
@@ -47,7 +47,7 @@ end6:   bx lr
 
 // conditional branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: beq.w end7
         beq.w end7
         .space 0x100000
@@ -70,7 +70,7 @@ start3:
         .space 0x1000000
 // branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: b.w start3
         b.w start3
 
@@ -91,12 +91,12 @@ start6:
         .space 0x100000
 // branch to thumb function resolved at assembly time
 // CHECK-NOT: error:
-// CHECK: [[@LINE+2]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[@LINE+2]]:[[#]]: error: Relocation out of range
 // CHECK-LABEL: beq.w start6
         beq.w start6
 
 start7:
 // branch to thumb function resolved at assembly time
-// CHECK: [[#@LINE+1]]:{{[0-9]}}: error: Relocation out of range
+// CHECK: [[#@LINE+1]]:[[#]]: error: Relocation out of range
         b.w start8 - start7 + 0x1000000
 start8:
diff --git a/llvm/test/MC/BPF/bad-tied.s b/llvm/test/MC/BPF/bad-tied.s
new file mode 100644
index 000000000000..9cd47896c3dd
--- /dev/null
+++ b/llvm/test/MC/BPF/bad-tied.s
@@ -0,0 +1,12 @@
+# RUN: not llvm-mc -mcpu=v4 -triple bpfel < %s 2>&1 \
+# RUN:   | grep 'error: operand is not the same as the dst register' \
+# RUN:   | count 9
+        r0 = bswap16 r1
+        r0 = bswap32 r1
+        r0 = bswap64 r1
+        r0 = atomic_fetch_add((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_and((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_or((u64*)(r2 + 0), r1)
+        r0 = atomic_fetch_xor((u64*)(r2 + 0), r1)
+        w0 = xchg32_32(r2 + 0, w1)
+        r0 = xchg_64(r2 + 0, r1)
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp16.txt
index 1774efa4a65c..4a7471e6c6f9 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp16.txt
@@ -2476,3 +2476,10 @@
 # W32: v_cndmask_b32_dpp v5, -|v1|, -|v2|, vcc_lo quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff]
 # W64: v_cndmask_b32_dpp v5, -|v1|, -|v2|, vcc quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff]
 0xfa,0x04,0x0a,0x02,0x01,0xe4,0xf0,0xff
+
+# GFX10: v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0xff]
+0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0xff
+
+# GFX10: v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 ; encoding: [0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0x00]
+0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0x00
+
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp8.txt
index 40b8f24e4d72..233f93a5b8e7 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop2_dpp8.txt
@@ -222,3 +222,6 @@
 # W32: v_cndmask_b32_dpp v0, v1, v2, vcc_lo  dpp8:[0,1,2,3,4,5,6,7] fi:1 ; encoding: [0xea,0x04,0x00,0x02,0x01,0x88,0xc6,0xfa]
 # W64: v_cndmask_b32_dpp v0, v1, v2, vcc  dpp8:[0,1,2,3,4,5,6,7] fi:1 ; encoding: [0xea,0x04,0x00,0x02,0x01,0x88,0xc6,0xfa]
 0xea,0x04,0x00,0x02,0x01,0x88,0xc6,0xfa
+
+# GFX10: v_pk_fmac_f16_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x78,0x01,0x77,0x39,0x05]
+0xe9,0x04,0x0a,0x78,0x01,0x77,0x39,0x05
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
index 6da1423fe827..721babdd6424 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
@@ -1503,6 +1503,9 @@
 # GFX10: v_ashrrev_i16 v5, vcc_lo, v2            ; encoding: [0x05,0x00,0x08,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x08,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_ashrrev_i16 v5, v1, v2 op_sel:[1,1,1] ; encoding: [0x05,0x58,0x08,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x08,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_ashrrev_i32_e64 v255, v1, v2          ; encoding: [0xff,0x00,0x18,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x18,0xd5,0x01,0x05,0x02,0x00
 
@@ -8309,6 +8312,9 @@
 # GFX10: v_lshlrev_b16 v5, vcc_lo, v2            ; encoding: [0x05,0x00,0x14,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x14,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_lshlrev_b16 v5, v1, v2 op_sel:[1,1,1] ; encoding: [0x05,0x58,0x14,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x14,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_lshlrev_b32_e64 v255, v1, v2          ; encoding: [0xff,0x00,0x1a,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x1a,0xd5,0x01,0x05,0x02,0x00
 
@@ -8537,6 +8543,9 @@
 # GFX10: v_lshrrev_b16 v5, vcc_lo, v2            ; encoding: [0x05,0x00,0x07,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x07,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_lshrrev_b16 v5, v1, v2 op_sel:[1,1,1] ; encoding: [0x05,0x58,0x07,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x07,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_lshrrev_b32_e64 v255, v1, v2          ; encoding: [0xff,0x00,0x16,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x16,0xd5,0x01,0x05,0x02,0x00
 
@@ -11292,6 +11301,9 @@
 # GFX10: v_max_i16 v5, vcc_lo, v2                ; encoding: [0x05,0x00,0x0a,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x0a,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_max_i16 v5, v1, v2 op_sel:[1,1,1]     ; encoding: [0x05,0x58,0x0a,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x0a,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_max_i32_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x12,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x12,0xd5,0x01,0x05,0x02,0x00
 
@@ -11448,6 +11460,9 @@
 # GFX10: v_max_u16 v5, vcc_lo, v2                ; encoding: [0x05,0x00,0x09,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x09,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_max_u16 v5, v1, v2 op_sel:[1,1,1]     ; encoding: [0x05,0x58,0x09,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x09,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_max_u32_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x14,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x14,0xd5,0x01,0x05,0x02,0x00
 
@@ -13728,6 +13743,9 @@
 # GFX10: v_min_i16 v5, vcc_lo, v2                ; encoding: [0x05,0x00,0x0c,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x0c,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_min_i16 v5, v1, v2 op_sel:[1,1,1]     ; encoding: [0x05,0x58,0x0c,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x0c,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_min_i32_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x11,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x11,0xd5,0x01,0x05,0x02,0x00
 
@@ -13884,6 +13902,9 @@
 # GFX10: v_min_u16 v5, vcc_lo, v2                ; encoding: [0x05,0x00,0x0b,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x0b,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_min_u16 v5, v1, v2 op_sel:[1,1,1]     ; encoding: [0x05,0x58,0x0b,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x0b,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_min_u32_e64 v255, v1, v2              ; encoding: [0xff,0x00,0x13,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x13,0xd5,0x01,0x05,0x02,0x00
 
@@ -15228,6 +15249,9 @@
 # GFX10: v_mul_lo_u16 v5, vcc_lo, v2             ; encoding: [0x05,0x00,0x05,0xd7,0x6a,0x04,0x02,0x00]
 0x05,0x00,0x05,0xd7,0x6a,0x04,0x02,0x00
 
+# GFX10: v_mul_lo_u16 v5, v1, v2 op_sel:[1,1,1]  ; encoding: [0x05,0x58,0x05,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x58,0x05,0xd7,0x01,0x05,0x02,0x00
+
 # GFX10: v_mul_lo_u32 v255, v1, v2               ; encoding: [0xff,0x00,0x69,0xd5,0x01,0x05,0x02,0x00]
 0xff,0x00,0x69,0xd5,0x01,0x05,0x02,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
new file mode 100644
index 000000000000..e03c4327d981
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
@@ -0,0 +1,19 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX1250 %s
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
+0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v255 offset:1040 ; encoding: [0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00]
+0x10,0x04,0x58,0xd9,0xff,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_async_barrier_arrive_b64 v5   ; encoding: [0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00]
+0x00,0x00,0x58,0xd9,0x05,0x00,0x00,0x00
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[254:255], v2, v[4:5] offset:65535 ; encoding: [0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe]
+0xff,0xff,0xd4,0xd9,0x02,0x04,0x00,0xfe
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] ; encoding: [0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02]
+0x00,0x00,0xd4,0xd9,0x02,0x04,0x00,0x02
+
+# GFX1250: ds_atomic_barrier_arrive_rtn_b64 v[2:3], v2, v[4:5] offset:513 ; encoding: [0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02]
+0x01,0x02,0xd4,0xd9,0x02,0x04,0x00,0x02
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_load_tr.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_load_tr.txt
new file mode 100644
index 000000000000..b339986e1516
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_load_tr.txt
@@ -0,0 +1,103 @@
+# RUN: llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+# GFX1250: ds_load_tr16_b128 v[2:5], v0            ; encoding: [0x00,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
+0x00,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr16_b128 v[2:5], v0 offset:64  ; encoding: [0x40,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02]
+0x40,0x00,0xf0,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr4_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02]
+0x00,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr4_b64 v[2:3], v0 offset:64    ; encoding: [0x40,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02]
+0x40,0x00,0xe8,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr6_b96 v[2:4], v0              ; encoding: [0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x02]
+0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr6_b96 v[3:5], v0              ; encoding: [0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x03]
+0x00,0x00,0xec,0xdb,0x00,0x00,0x00,0x03
+
+# GFX1250: ds_load_tr6_b96 v[2:4], v0 offset:64    ; encoding: [0x40,0x00,0xec,0xdb,0x00,0x00,0x00,0x02]
+0x40,0x00,0xec,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr8_b64 v[2:3], v0              ; encoding: [0x00,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+0x00,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: ds_load_tr8_b64 v[2:3], v0 offset:64    ; encoding: [0x40,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02]
+0x40,0x00,0xf4,0xdb,0x00,0x00,0x00,0x02
+
+# GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr16_b128 v[2:5], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00
+
+# GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:-64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
+0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr16_b128 v[2:5], v[6:7], off offset:64 ; encoding: [0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+0x7c,0xc0,0x15,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00
+
+# GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX1250: global_load_tr4_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+0x00,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00
+
+# GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00
+
+# GFX1250: global_load_tr4_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+0x7c,0xc0,0x1c,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1]  ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: global_load_tr6_b96 v[3:5], v0, s[0:1]  ; encoding: [0x00,0x00,0x1d,0xee,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0x00,0x1d,0xee,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX1250: global_load_tr6_b96 v[2:4], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+0x00,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00]
+0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00
+
+# GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off offset:64 ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00]
+0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0x40,0x00,0x00
+
+# GFX1250: global_load_tr6_b96 v[2:4], v[6:7], off offset:-64 ; encoding: [0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff]
+0x7c,0x00,0x1d,0xee,0x02,0x00,0x00,0x00,0x06,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1]  ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:-64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr8_b64 v[2:3], v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00]
+0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00
+
+# GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:-64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff]
+0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0xc0,0xff,0xff
+
+# GFX1250: global_load_tr8_b64 v[2:3], v[4:5], off offset:64 ; encoding: [0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00]
+0x7c,0x00,0x16,0xee,0x02,0x00,0x00,0x00,0x04,0x40,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
index 55f74d3a31bf..220f9e5084f0 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
@@ -11,3 +11,12 @@
 
 # GFX1250: s_setprio_inc_wg 0x64 ; encoding: [0x64,0x00,0xbe,0xbf]
 0x64,0x00,0xbe,0xbf
+
+# GFX1250: s_monitor_sleep 0                       ; encoding: [0x00,0x00,0x84,0xbf]
+0x00,0x00,0x84,0xbf
+
+# GFX1250: s_monitor_sleep 0x8000                  ; encoding: [0x00,0x80,0x84,0xbf]
+0x00,0x80,0x84,0xbf
+
+# GFX1250: s_monitor_sleep 1                       ; encoding: [0x01,0x00,0x84,0xbf]
+0x01,0x00,0x84,0xbf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt
new file mode 100644
index 000000000000..6421c6f30e17
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vflat.txt
@@ -0,0 +1,16 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+# GFX1250: tensor_save s[0:1] ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_save s[0:1] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_save s[0:1] offset:32 ; encoding: [0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00]
+0x00,0x80,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00
+
+# GFX1250: tensor_stop ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x7c,0xc0,0x1b,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: tensor_stop th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00]
+0x7c,0xc0,0x1b,0xee,0x00,0x00,0x3c,0x00,0x00,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt
new file mode 100644
index 000000000000..9afaa075ea83
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vimage.txt
@@ -0,0 +1,25 @@
+# RUN: llvm-mc -disassemble -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11] th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_load_to_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x00,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11] th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x3c,0x00,0x00,0x04,0x7c,0x7c
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x00,0x00,0x00,0x04,0x0c,0x10
+
+# GFX1250: tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10]
+0x01,0x40,0x71,0xd0,0x00,0x00,0x68,0x00,0x00,0x04,0x0c,0x10
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
new file mode 100644
index 000000000000..622c57a20f86
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1.txt
@@ -0,0 +1,109 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_e32 v127, 0x8000         ; encoding: [0xff,0xe4,0xfe,0x7e,0x00,0x80,0x00,0x00]
+
+0xc1,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, -1               ; encoding: [0xc1,0xe4,0x0a,0x7e]
+
+0xf0,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, 0.5              ; encoding: [0xf0,0xe4,0x0a,0x7e]
+
+0x7f,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, exec_hi          ; encoding: [0x7f,0xe4,0x0a,0x7e]
+
+0x7e,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, exec_lo          ; encoding: [0x7e,0xe4,0x0a,0x7e]
+
+0x7d,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, m0               ; encoding: [0x7d,0xe4,0x0a,0x7e]
+
+0x7c,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, null             ; encoding: [0x7c,0xe4,0x0a,0x7e]
+
+0x01,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, s1               ; encoding: [0x01,0xe4,0x0a,0x7e]
+
+0x69,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, s105             ; encoding: [0x69,0xe4,0x0a,0x7e]
+
+0xfd,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, src_scc          ; encoding: [0xfd,0xe4,0x0a,0x7e]
+
+0x7b,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, ttmp15           ; encoding: [0x7b,0xe4,0x0a,0x7e]
+
+0x01,0xe5,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, v1.l             ; encoding: [0x01,0xe5,0x0a,0x7e]
+
+0x7f,0xe5,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, v127.l           ; encoding: [0x7f,0xe5,0x0a,0x7e]
+
+0x6b,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, vcc_hi           ; encoding: [0x6b,0xe4,0x0a,0x7e]
+
+0x6a,0xe4,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, vcc_lo           ; encoding: [0x6a,0xe4,0x0a,0x7e]
+
+0x81,0xe5,0x0a,0x7e
+# GFX1250: v_cvt_f32_bf16_e32 v5, v1.h             ; encoding: [0x81,0xe5,0x0a,0x7e]
+
+0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, 0x1234          ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, 0x1234            ; encoding: [0xff,0xf0,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+0x82,0xf0,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, 2               ; encoding: [0x82,0xf0,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, 2                 ; encoding: [0x82,0xf0,0x02,0x7e]
+
+0x02,0xf0,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, s2              ; encoding: [0x02,0xf0,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, s2                ; encoding: [0x02,0xf0,0x02,0x7e]
+
+0x02,0xf1,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.l, v2              ; encoding: [0x02,0xf1,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e32 v1, v2                ; encoding: [0x02,0xf1,0x02,0x7e]
+
+0x02,0xf1,0x02,0x7f
+# GFX1250-REAL16: v_cvt_f16_bf8_e32 v1.h, v2              ; encoding: [0x02,0xf1,0x02,0x7f]
+
+0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, 0x1234          ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, 0x1234            ; encoding: [0xff,0xee,0x02,0x7e,0x34,0x12,0x00,0x00]
+
+0x82,0xee,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, 2               ; encoding: [0x82,0xee,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, 2                 ; encoding: [0x82,0xee,0x02,0x7e]
+
+0x02,0xee,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, s2              ; encoding: [0x02,0xee,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, s2                ; encoding: [0x02,0xee,0x02,0x7e]
+
+0x02,0xef,0x02,0x7e
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.l, v2              ; encoding: [0x02,0xef,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e32 v1, v2                ; encoding: [0x02,0xef,0x02,0x7e]
+
+0x02,0xef,0x02,0x7f
+# GFX1250-REAL16: v_cvt_f16_fp8_e32 v1.h, v2              ; encoding: [0x02,0xef,0x02,0x7f]
+
+0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00
+# GFX1250: v_cvt_pk_f16_bf8 v1, 0x64               ; encoding: [0xff,0xec,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+0x02,0xec,0x02,0x7e
+# GFX1250: v_cvt_pk_f16_bf8 v1, s2                 ; encoding: [0x02,0xec,0x02,0x7e]
+
+0x02,0xed,0x02,0x7e
+# GFX1250-REAL16: v_cvt_pk_f16_bf8 v1, v2.l               ; encoding: [0x02,0xed,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8 v1, v2                 ; encoding: [0x02,0xed,0x02,0x7e]
+
+0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00
+# GFX1250: v_cvt_pk_f16_fp8 v1, 0x64               ; encoding: [0xff,0xea,0x02,0x7e,0x64,0x00,0x00,0x00]
+
+0x02,0xea,0x02,0x7e
+# GFX1250: v_cvt_pk_f16_fp8 v1, s2                 ; encoding: [0x02,0xea,0x02,0x7e]
+
+0x02,0xeb,0x02,0x7e
+# GFX1250-REAL16: v_cvt_pk_f16_fp8 v1, v2.l               ; encoding: [0x02,0xeb,0x02,0x7e]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8 v1, v2                 ; encoding: [0x02,0xeb,0x02,0x7e]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
new file mode 100644
index 000000000000..149a054742de
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp16.txt
@@ -0,0 +1,86 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30
+# GFX1250: v_cvt_f32_bf16_dpp v127, -|v127.l| row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xfa,0xe4,0xfe,0x7e,0x7f,0x6f,0x35,0x30]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0xe4,0x00,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0xe4,0x00,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x1b,0x00,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1b,0x00,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x41,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x41,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x40,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x40,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x21,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x21,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x2f,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x2f,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x50,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x50,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x5f,0x01,0x01
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x5f,0x01,0x01]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x01,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x01,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x0f,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x0f,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x11,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x11,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x1f,0x01,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x1f,0x01,0xff]
+
+0xfa,0xe4,0x0a,0x7e,0x01,0x60,0x09,0x13
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0xfa,0xe4,0x0a,0x7e,0x01,0x60,0x09,0x13]
+
+0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.h quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xe4,0x0a,0x7e,0x81,0x1b,0x00,0xff]
+
+0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x00,0xff]
+
+0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xf0,0x02,0x7e,0x02,0x39,0x04,0xff]
+
+0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xf0,0x02,0x7f,0x02,0xe4,0x00,0xff]
+
+0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x00,0xff]
+
+0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xee,0x02,0x7e,0x02,0x39,0x04,0xff]
+
+0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.h, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xee,0x02,0x7f,0x02,0xe4,0x00,0xff]
+
+0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xec,0x02,0x7e,0x02,0xe4,0x04,0xff]
+
+0xfa,0xec,0x02,0x7e,0x82,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.h quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xec,0x02,0x7e,0x82,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xec,0x02,0x7e,0x82,0xe4,0x00,0xff]
+
+0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_dpp v1, v2.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_dpp v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0xfa,0xea,0x02,0x7e,0x02,0xe4,0x04,0xff]
+
+0xfa,0xea,0x02,0x7e,0x82,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_dpp v1, v2.h quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xea,0x02,0x7e,0x82,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_dpp v1, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0xea,0x02,0x7e,0x82,0xe4,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
new file mode 100644
index 000000000000..274b58769911
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop1_dpp8.txt
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00
+# GFX1250: v_cvt_f32_bf16_dpp v127, v127.l dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xe9,0xe4,0xfe,0x7e,0x7f,0x00,0x00,0x00]
+
+0xe9,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xea,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xe4,0x0a,0x7e,0x01,0x77,0x39,0x05]
+
+0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05
+# GFX1250: v_cvt_f32_bf16_dpp v5, v1.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xe4,0x0a,0x7e,0x81,0x77,0x39,0x05]
+
+0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xf0,0x02,0x7f,0x02,0x77,0x39,0x05]
+
+0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xf0,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xee,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_dpp v1.h, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xee,0x02,0x7f,0x02,0x77,0x39,0x05]
+
+0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xec,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xec,0x02,0x7e,0x82,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xec,0x02,0x7e,0x82,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_dpp v1, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xec,0x02,0x7e,0x82,0x77,0x39,0x05]
+
+0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_dpp v1, v2.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_dpp v1, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0xea,0xea,0x02,0x7e,0x02,0x77,0x39,0x05]
+
+0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_dpp v1, v2.h dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_dpp v1, v130/*Invalid register, operand has 'VGPR_32_Lo128' register class*/ dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0xea,0x02,0x7e,0x82,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3.txt
new file mode 100644
index 000000000000..d9d8f60fe3d1
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3.txt
@@ -0,0 +1,21 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0x02,0x00,0x52,0xd6,0x04,0x0e,0x22,0x04
+# GFX1250: v_lshl_add_u64 v[2:3], s[4:5], v7, v[8:9] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0e,0x22,0x04]
+
+0x02,0x00,0x52,0xd6,0x04,0x01,0x05,0x02
+# GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 0, 1     ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x01,0x05,0x02]
+
+0x02,0x00,0x52,0xd6,0x04,0x07,0x09,0x00
+# GFX1250: v_lshl_add_u64 v[2:3], v[4:5], 3, s[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x07,0x09,0x00]
+
+0x02,0x00,0x52,0xd6,0x04,0x08,0x09,0x04
+# GFX1250: v_lshl_add_u64 v[2:3], s[4:5], 4, v[2:3] ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x08,0x09,0x04]
+
+0x02,0x00,0x52,0xd6,0x04,0x0f,0xfe,0x03,0x39,0x30,0x00,0x00
+# GFX1250: v_lshl_add_u64 v[2:3], v[4:5], v7, 0x3039 ; encoding: [0x02,0x00,0x52,0xd6,0x04,0x0f,0xfe,0x03,0x39,0x30,0x00,0x00]
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# GFX1250-FAKE16: {{.*}}
+# GFX1250-REAL16: {{.*}}
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
new file mode 100644
index 000000000000..b09ddc20b803
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1.txt
@@ -0,0 +1,105 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf8,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, 2               ; encoding: [0x96,0x00,0xf8,0xd5,0x82,0x00,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, s2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x00,0x00,0x00]
+
+0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v150, v2              ; encoding: [0x96,0x00,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_bf8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf8,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:1   ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:1     ; encoding: [0x01,0x10,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:2   ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:2     ; encoding: [0x01,0x08,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.l, v2 byte_sel:3   ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 byte_sel:3     ; encoding: [0x01,0x18,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, 0x1234        ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, 0x1234          ; encoding: [0x96,0x00,0xf7,0xd5,0xff,0x00,0x00,0x00,0x34,0x12,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, 2             ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, 2               ; encoding: [0x96,0x00,0xf7,0xd5,0x82,0x00,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, s2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, s2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x00,0x00,0x00]
+
+0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v150.l, v2            ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v150, v2              ; encoding: [0x96,0x00,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v128.h, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v128, v2 op_sel:[0,1] ; encoding: [0x80,0x40,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_f16_fp8_e64 v1.h, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64 v1, v2 op_sel:[0,1] byte_sel:2 ; encoding: [0x01,0x48,0xf7,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00
+# GFX1250: v_cvt_pk_f16_bf8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x00,0x00,0x00]
+
+0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_bf8 v1, v150.l             ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8 v1, v150               ; encoding: [0x01,0x00,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+0x01,0x08,0xf6,0xd5,0x96,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_bf8 v1, v150.h op_sel:[1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0x96,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf6,0xd5,0x96,0x01,0x00,0x00]
+
+0x01,0x08,0xf6,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_bf8 v1, v2.h op_sel:[1,0]  ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf6,0xd5,0x02,0x01,0x00,0x00]
+
+0x01,0x08,0xf5,0xd5,0x02,0x00,0x00,0x00
+# GFX1250: v_cvt_pk_f16_fp8 v1, s2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x00,0x00,0x00]
+
+0x01,0x00,0xf5,0xd5,0x96,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_fp8 v1, v150.l             ; encoding: [0x01,0x00,0xf5,0xd5,0x96,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8 v1, v150               ; encoding: [0x01,0x00,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+0x01,0x08,0xf5,0xd5,0x96,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_fp8 v1, v150.h op_sel:[1,0] ; encoding: [0x01,0x08,0xf5,0xd5,0x96,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8 v1, v150 op_sel:[1,0]  ; encoding: [0x01,0x08,0xf5,0xd5,0x96,0x01,0x00,0x00]
+
+0x01,0x08,0xf5,0xd5,0x02,0x01,0x00,0x00
+# GFX1250-REAL16: v_cvt_pk_f16_fp8 v1, v2.h op_sel:[1,0]  ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x01,0x00,0x00]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8 v1, v2 op_sel:[1,0]    ; encoding: [0x01,0x08,0xf5,0xd5,0x02,0x01,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
new file mode 100644
index 000000000000..dedb25599eea
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp16.txt
@@ -0,0 +1,115 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x41,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x40,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x01,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x11,0x01,0xff]
+
+0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1f,0x01,0xff]
+
+0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v128 op_sel:[1,0] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x08,0xf2,0xd5,0xfa,0x00,0x00,0x00,0x80,0x1b,0x00,0xff]
+
+0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v128, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf8,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x10,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x18,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x96,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0x39,0x00,0xff]
+
+0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x58,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v128.l, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v128, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x80,0x00,0xf7,0xd5,0xfa,0x00,0x00,0x00,0x02,0xe4,0x00,0xff]
+
+0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+
+0x01,0x08,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf6,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
+
+0x01,0x00,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_e64_dpp v1, v128.l quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_e64_dpp v1, v128 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x04,0xff]
+
+0x01,0x08,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_e64_dpp v1, v128.h op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_e64_dpp v1, v128 op_sel:[1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x08,0xf5,0xd5,0xfa,0x00,0x00,0x00,0x80,0xe4,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
new file mode 100644
index 000000000000..3868abe246e5
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_from_vop1_dpp8.txt
@@ -0,0 +1,87 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
+
+0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v1.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f32_bf16_e64_dpp v5, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f32_bf16_e64_dpp v5, v128 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x08,0xf2,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+
+0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x18,0xf8,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_bf8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_bf8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf8,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x10,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x18,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x96,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v1.h, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v1, v2 op_sel:[0,1] byte_sel:3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x58,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v128.l, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v128, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x80,0x00,0xf7,0xd5,0xe9,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_f16_fp8_e64_dpp v150.l, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_f16_fp8_e64_dpp v150, v2 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x96,0x00,0xf7,0xd5,0xea,0x00,0x00,0x00,0x02,0x77,0x39,0x05]
+
+0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf6,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+
+0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_bf8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_bf8_e64_dpp v1, v128 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf6,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+
+0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_e64_dpp v1, v128.l dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_e64_dpp v1, v128 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x01,0x00,0xf5,0xd5,0xea,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+
+0x01,0x08,0xf5,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05
+# GFX1250-REAL16: v_cvt_pk_f16_fp8_e64_dpp v1, v128.h op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf5,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_cvt_pk_f16_fp8_e64_dpp v1, v128 op_sel:[1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x01,0x08,0xf5,0xd5,0xe9,0x00,0x00,0x00,0x80,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index 8688410a8c9b..ad20788081c9 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -1,8 +1,10 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16,GFX1200,GFX1200-W32,GFX1200-W32-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16,GFX1200,GFX1200-W32,GFX1200-W32-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16,GFX1200,GFX1200-W64,GFX1200-W64-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16,GFX1200,GFX1200-W64,GFX1200-W64-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16,GFX1250,GFX1250-W32,GFX1250-W32-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16,GFX1250,GFX1250-W32,GFX1250-W32-FAKE16 %s
 
 0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -681,32 +683,32 @@
 # GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x87,0x0e,0xd6,0xea,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
 0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 
 0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v255, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
@@ -3754,7 +3756,6 @@
 # W64-REAL16: v_max3_num_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x13,0x2c,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x13,0x2c,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
 
-
 0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_max3_i16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_max3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -4565,7 +4566,6 @@
 # W64-REAL16: v_min3_num_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x13,0x2b,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x13,0x2b,0xd6,0xe9,0x04,0xc2,0x63,0x01,0x77,0x39,0x05]
 
-
 0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
 # W32-REAL16: v_min3_i16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 # W32-FAKE16: v_min3_i16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x4a,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -5047,100 +5047,112 @@
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xea,0xfe,0x03,0x00,0xff,0x00,0x00,0x00]
 
 0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 # op_sel[1:0] are ignored
 0x00,0x78,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
 0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1250: v_add_f32_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1250: v_add_f32_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 # op_sel[1:0] are ignored
 0x00,0x78,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# GFX1250: s_or_not1_b32 s70, s1, 8                ; encoding: [0x01,0x88,0x46,0x92]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
 0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1250: v_add_f32_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1250: v_add_f32_dpp v255, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xea,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
@@ -5719,3 +5731,13 @@
 # W32-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 # W64-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 # W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0xc7,0x6e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# GFX1200: {{.*}}
+# GFX1200-W32: {{.*}}
+# GFX1200-W64: {{.*}}
+# GFX1200-W64-FAKE16: {{.*}}
+# GFX1200-W64-REAL16: {{.*}}
+# GFX1250-W32: {{.*}}
+# GFX1250-W32-FAKE16: {{.*}}
+# GFX1250-W32-REAL16: {{.*}}
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
index 9fc9c58387b9..01821593b070 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx950.txt
@@ -47,9 +47,27 @@
 # GFX950:   v_permlane16_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb3,0x02,0x7e]
 0x02,0xb3,0x02,0x7e
 
+# GFX950: v_permlane16_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb3,0xb4,0x7f]
+0xdb,0xb3,0xb4,0x7f
+
+# GFX950: v_permlane16_swap_b32_e32 v218, v2      ; encoding: [0x02,0xb3,0xb4,0x7f]
+0x02,0xb3,0xb4,0x7f
+
+# GFX950: v_permlane16_swap_b32_e32 v2, v219      ; encoding: [0xdb,0xb3,0x04,0x7e]
+0xdb,0xb3,0x04,0x7e
+
 # GFX950:   v_permlane16_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x00,0x99,0xd1,0x02,0x01,0x00,0x00
 
+# GFX950: v_permlane16_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+0xda,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00
+
+# GFX950: v_permlane16_swap_b32_e64 v218, v2      ; encoding: [0xda,0x00,0x99,0xd1,0x02,0x01,0x00,0x00]
+0xda,0x00,0x99,0xd1,0x02,0x01,0x00,0x00
+
+# GFX950: v_permlane16_swap_b32_e64 v2, v219      ; encoding: [0x02,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00]
+0x02,0x00,0x99,0xd1,0xdb,0x01,0x00,0x00
+
 # GFX950:   v_permlane16_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x10,0x99,0xd1,0x02,0x01,0x00,0x00
 
@@ -63,9 +81,27 @@
 # GFX950:   v_permlane32_swap_b32_e32 v1, v2        ; encoding: [0x02,0xb5,0x02,0x7e]
 0x02,0xb5,0x02,0x7e
 
+# GFX950: v_permlane32_swap_b32_e32 v218, v219    ; encoding: [0xdb,0xb5,0xb4,0x7f]
+0xdb,0xb5,0xb4,0x7f
+
+# GFX950: v_permlane32_swap_b32_e32 v218, v2      ; encoding: [0x02,0xb5,0xb4,0x7f]
+0x02,0xb5,0xb4,0x7f
+
+# GFX950: v_permlane32_swap_b32_e32 v2, v219      ; encoding: [0xdb,0xb5,0x04,0x7e]
+0xdb,0xb5,0x04,0x7e
+
 # GFX950:   v_permlane32_swap_b32_e64 v1, v2        ; encoding: [0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00
 
+# GFX950: v_permlane32_swap_b32_e64 v218, v219    ; encoding: [0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+0xda,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00
+
+# GFX950: v_permlane32_swap_b32_e64 v218, v2      ; encoding: [0xda,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00]
+0xda,0x00,0x9a,0xd1,0x02,0x01,0x00,0x00
+
+# GFX950: v_permlane32_swap_b32_e64 v2, v219      ; encoding: [0x02,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00]
+0x02,0x00,0x9a,0xd1,0xdb,0x01,0x00,0x00
+
 # GFX950:   v_permlane32_swap_b32_e64 v1, v2 bound_ctrl:1 ; encoding: [0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00]
 0x01,0x10,0x9a,0xd1,0x02,0x01,0x00,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop2_features.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop2_features.txt
new file mode 100644
index 000000000000..2b8d58853847
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop2_features.txt
@@ -0,0 +1,92 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx908 -disassemble -show-encoding < %s | FileCheck -check-prefix=CHECK-MI %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx90a -disassemble -show-encoding < %s | FileCheck -check-prefix=CHECK-MI %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx942 -disassemble -show-encoding < %s | FileCheck -check-prefix=CHECK-MI %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx950 -disassemble -show-encoding < %s | FileCheck -check-prefix=CHECK-MI %s
+
+# CHECK-MI: v_pk_fmac_f16_e32 v5, v1, v2
+0x01,0x05,0x0a,0x78
+
+# CHECK-MI: v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf
+0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0xff
+
+# CHECK-MI: v_pk_fmac_f16_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+0xfa,0x04,0x0a,0x78,0x01,0xe4,0x00,0x00
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_0 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x00,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x01,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_2 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x02,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x03,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:WORD_0 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x04,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x05,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_SEXT src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x0e,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x16,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x16,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x00,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x01,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x02,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x03,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x04,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x05,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x06
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x00
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x01
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x02
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x03
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x04
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x05
+
+# CHECK-MI: v_pk_fmac_f16_sdwa v5, v1, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+0xf9,0x04,0x0a,0x78,0x01,0x06,0x06,0x16
+
diff --git a/llvm/test/MC/Disassembler/RISCV/c_lui_disasm.txt b/llvm/test/MC/Disassembler/RISCV/c_lui_disasm.txt
index 17889c15cbf9..0a83a200c931 100644
--- a/llvm/test/MC/Disassembler/RISCV/c_lui_disasm.txt
+++ b/llvm/test/MC/Disassembler/RISCV/c_lui_disasm.txt
@@ -10,265 +10,197 @@
 # RUN: llvm-mc --disassemble -triple=riscv32  -mattr=+c -mattr=+Zcmop \
 # RUN:     -M no-aliases --show-encoding < %s | \
 # RUN:   FileCheck --check-prefixes=GOOD,MOP %s
-#
-# RUN: llvm-mc --disassemble -triple=riscv32  -mattr=+c -mattr=+no-rvc-hints \
-# RUN:     -M no-aliases --show-encoding < %s 2>&1 | \
-# RUN:   FileCheck --check-prefix=NOHINTS %s
 
 # BAD: invalid instruction encoding
-# NOHINTS: invalid instruction encoding
 0x01 0x60
 
 # GOOD: c.lui zero, 1
-# NOHINTS: invalid instruction encoding
 0x05 0x60
 
 # GOOD: c.lui zero, 2
-# NOHINTS: invalid instruction encoding
 0x09 0x60
 
 # GOOD: c.lui zero, 3
-# NOHINTS: invalid instruction encoding
 0x0D 0x60
 
 # GOOD: c.lui zero, 4
-# NOHINTS: invalid instruction encoding
 0x11 0x060
 
 # GOOD: c.lui zero, 5
-# NOHINTS: invalid instruction encoding
 0x15 0x60
 
 # GOOD: c.lui zero, 6
-# NOHINTS: invalid instruction encoding
 0x19 0x60
 
 # GOOD: c.lui zero, 7
-# NOHINTS: invalid instruction encoding
 0x1D 0x60
 
 # GOOD: c.lui zero, 8
-# NOHINTS: invalid instruction encoding
 0x21 0x60
 
 # GOOD: c.lui zero, 9
-# NOHINTS: invalid instruction encoding
 0x25 0x60
 
 # GOOD: c.lui zero, 10
-# NOHINTS: invalid instruction encoding
 0x29 0x60
 
 # GOOD: c.lui zero, 11
-# NOHINTS: invalid instruction encoding
 0x2D 0x60
 
 # GOOD: c.lui zero, 12
-# NOHINTS: invalid instruction encoding
 0x31 0x60
 
 # GOOD: c.lui zero, 13
-# NOHINTS: invalid instruction encoding
 0x35 0x60
 
 # GOOD: c.lui zero, 14
-# NOHINTS: invalid instruction encoding
 0x39 0x60
 
 # GOOD: c.lui zero, 15
-# NOHINTS: invalid instruction encoding
 0x3D 0x60
 
 # GOOD: c.lui zero, 16
-# NOHINTS: invalid instruction encoding
 0x41 0x60
 
 # GOOD: c.lui zero, 17
-# NOHINTS: invalid instruction encoding
 0x45 0x60
 
 # GOOD: c.lui zero, 18
-# NOHINTS: invalid instruction encoding
 0x49 0x60
 
 # GOOD: c.lui zero, 19
-# NOHINTS: invalid instruction encoding
 0x4D 0x60
 
 # GOOD: c.lui zero, 20
-# NOHINTS: invalid instruction encoding
 0x51 0x60
 
 # GOOD: c.lui zero, 21
-# NOHINTS: invalid instruction encoding
 0x55 0x60
 
 # GOOD: c.lui zero, 22
-# NOHINTS: invalid instruction encoding
 0x59 0x60
 
 # GOOD: c.lui zero, 23
-# NOHINTS: invalid instruction encoding
 0x5D 0x60
 
 # GOOD: c.lui zero, 24
-# NOHINTS: invalid instruction encoding
 0x61 0x60
 
 # GOOD: c.lui zero, 25
-# NOHINTS: invalid instruction encoding
 0x65 0x60
 
 # GOOD: c.lui zero, 26
-# NOHINTS: invalid instruction encoding
 0x69 0x60
 
 # GOOD: c.lui zero, 27
-# NOHINTS: invalid instruction encoding
 0x6D 0x60
 
 # GOOD: c.lui zero, 28
-# NOHINTS: invalid instruction encoding
 0x71 0x60
 
 # GOOD: c.lui zero, 29
-# NOHINTS: invalid instruction encoding
 0x75 0x60
 
 # GOOD: c.lui zero, 30
-# NOHINTS: invalid instruction encoding
 0x79 0x60
 
 # GOOD: c.lui zero, 31
-# NOHINTS: invalid instruction encoding
 0x7D 0x60
 
 # GOOD: c.lui zero, 1048544
-# NOHINTS: invalid instruction encoding
 0x01 0x70
 
 # GOOD: c.lui zero, 1048545
-# NOHINTS: invalid instruction encoding
 0x05 0x70
 
 # GOOD: c.lui zero, 1048546
-# NOHINTS: invalid instruction encoding
 0x09 0x70
 
 # GOOD: c.lui zero, 1048547
-# NOHINTS: invalid instruction encoding
 0x0D 0x70
 
 # GOOD: c.lui zero, 1048548
-# NOHINTS: invalid instruction encoding
 0x11 0x70
 
 # GOOD: c.lui zero, 1048549
-# NOHINTS: invalid instruction encoding
 0x15 0x70
 
 # GOOD: c.lui zero, 1048550
-# NOHINTS: invalid instruction encoding
 0x19 0x70
 
 # GOOD: c.lui zero, 1048551
-# NOHINTS: invalid instruction encoding
 0x1D 0x70
 
 # GOOD: c.lui zero, 1048552
-# NOHINTS: invalid instruction encoding
 0x21 0x70
 
 # GOOD: c.lui zero, 1048553
-# NOHINTS: invalid instruction encoding
 0x25 0x70
 
 # GOOD: c.lui zero, 1048554
-# NOHINTS: invalid instruction encoding
 0x29 0x70
 
 # GOOD: c.lui zero, 1048555
-# NOHINTS: invalid instruction encoding
 0x2D 0x70
 
 # GOOD: c.lui zero, 1048556
-# NOHINTS: invalid instruction encoding
 0x31 0x70
 
 # GOOD: c.lui zero, 1048557
-# NOHINTS: invalid instruction encoding
 0x35 0x70
 
 # GOOD: c.lui zero, 1048558
-# NOHINTS: invalid instruction encoding
 0x39 0x70
 
 # GOOD: c.lui zero, 1048559
-# NOHINTS: invalid instruction encoding
 0x3D 0x70
 
 # GOOD: c.lui zero, 1048560
-# NOHINTS: invalid instruction encoding
 0x41 0x70
 
 # GOOD: c.lui zero, 1048561
-# NOHINTS: invalid instruction encoding
 0x45 0x70
 
 # GOOD: c.lui zero, 1048562
-# NOHINTS: invalid instruction encoding
 0x49 0x70
 
 # GOOD: c.lui zero, 1048563
-# NOHINTS: invalid instruction encoding
 0x4D 0x70
 
 # GOOD: c.lui zero, 1048564
-# NOHINTS: invalid instruction encoding
 0x51 0x70
 
 # GOOD: c.lui zero, 1048565
-# NOHINTS: invalid instruction encoding
 0x55 0x70
 
 # GOOD: c.lui zero, 1048566
-# NOHINTS: invalid instruction encoding
 0x59 0x70
 
 # GOOD: c.lui zero, 1048567
-# NOHINTS: invalid instruction encoding
 0x5D 0x70
 
 # GOOD: c.lui zero, 1048568
-# NOHINTS: invalid instruction encoding
 0x61 0x70
 
 # GOOD: c.lui zero, 1048569
-# NOHINTS: invalid instruction encoding
 0x65 0x70
 
 # GOOD: c.lui zero, 1048570
-# NOHINTS: invalid instruction encoding
 0x69 0x70
 
 # GOOD: c.lui zero, 1048571
-# NOHINTS: invalid instruction encoding
 0x6D 0x70
 
 # GOOD: c.lui zero, 1048572
-# NOHINTS: invalid instruction encoding
 0x71 0x70
 
 # GOOD: c.lui zero, 1048573
-# NOHINTS: invalid instruction encoding
 0x75 0x70
 
 # GOOD: c.lui zero, 1048574
-# NOHINTS: invalid instruction encoding
 0x79 0x70
 
 # GOOD: c.lui zero, 1048575
-# NOHINTS: invalid instruction encoding
 0x7D 0x70
 
 # BAD: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt b/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt
new file mode 100644
index 000000000000..e2f06c601fd5
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/dfpaccel.txt
@@ -0,0 +1,19 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+dfpaccel -disassemble %s | FileCheck -check-prefixes=CHECK-DFPACCEL %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with
+## DFPACCEL option enabled. Also verify that dissasembling without
+## DFPACCEL option generates warnings.
+
+[0xa0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64r_lo
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xb0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64r_hi
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
+
+[0xc0,0x3e,0xe3]
+# CHECK-DFPACCEL: rur	a3, f64s
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/fp.txt b/llvm/test/MC/Disassembler/Xtensa/fp.txt
index 993b9987c1bc..77ced9971fc1 100644
--- a/llvm/test/MC/Disassembler/Xtensa/fp.txt
+++ b/llvm/test/MC/Disassembler/Xtensa/fp.txt
@@ -7,209 +7,209 @@
 ## fp option generates warnings.
 
 [0x10,0x23,0xfa]
-# CHECK-FLOAT: abs.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: abs.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x0a]
-# CHECK-FLOAT: add.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: add.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xe0,0x23,0xfa]
-# CHECK-FLOAT: addexp.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: addexp.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xf0,0x23,0xfa]
-# CHECK-FLOAT: addexpm.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: addexpm.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xba]
-# CHECK-FLOAT: ceil.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ceil.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x35,0xfa]
-# CHECK-FLOAT: const.s f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: const.s	f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x70,0x23,0xfa]
-# CHECK-FLOAT: div0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: div0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x7a]
-# CHECK-FLOAT: divn.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: divn.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xca]
-# CHECK-FLOAT: float.s f2, a3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: float.s	f2, a3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xaa]
-# CHECK-FLOAT: floor.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: floor.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x03,0x02]
-# CHECK-FLOAT: lsi f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsi	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x83,0x02]
-# CHECK-FLOAT: lsip f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsip	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x08]
-# CHECK-FLOAT: lsx f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsx	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x18]
-# CHECK-FLOAT: lsxp f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: lsxp	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x4a]
-# CHECK-FLOAT: madd.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: madd.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x6a]
-# CHECK-FLOAT: maddn.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: maddn.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xd0,0x23,0xfa]
-# CHECK-FLOAT: mkdadj.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mkdadj.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xc0,0x23,0xfa]
-# CHECK-FLOAT: mksadj.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mksadj.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xfa]
-# CHECK-FLOAT: mov.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mov.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x8b]
-# CHECK-FLOAT: moveqz.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: moveqz.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xcb]
-# CHECK-FLOAT: movf.s f2, f3, b0
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movf.s	f2, f3, b0
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xbb]
-# CHECK-FLOAT: movgez.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movgez.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xab]
-# CHECK-FLOAT: movltz.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movltz.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x9b]
-# CHECK-FLOAT: movnez.s f2, f3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movnez.s	f2, f3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x00,0x23,0xdb]
-# CHECK-FLOAT: movt.s f2, f3, b0
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: movt.s	f2, f3, b0
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x5a]
-# CHECK-FLOAT: msub.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: msub.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x2a]
-# CHECK-FLOAT: mul.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: mul.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x60,0x23,0xfa]
-# CHECK-FLOAT: neg.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: neg.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xb0,0x23,0xfa]
-# CHECK-FLOAT: nexp01.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: nexp01.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x2b]
-# CHECK-FLOAT: oeq.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: oeq.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x6b]
-# CHECK-FLOAT: ole.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ole.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x4b]
-# CHECK-FLOAT: olt.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: olt.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x80,0x23,0xfa]
-# CHECK-FLOAT: recip0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: recip0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0xfa]
-# CHECK-FLOAT: rfr a2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rfr	a2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0x8a]
-# CHECK-FLOAT: round.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: round.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0xa0,0x23,0xfa]
-# CHECK-FLOAT: rsqrt0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rsqrt0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x90,0x23,0xfa]
-# CHECK-FLOAT: sqrt0.s f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: sqrt0.s	f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0x43,0x02]
-# CHECK-FLOAT: ssi f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssi	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x23,0xc3,0x02]
-# CHECK-FLOAT: ssip f2, a3, 8
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssip	f2, a3, 8
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x48]
-# CHECK-FLOAT: ssx f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssx	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x58]
-# CHECK-FLOAT: ssxp f2, a3, a4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ssxp	f2, a3, a4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x40,0x23,0x1a]
-# CHECK-FLOAT: sub.s f2, f3, f4
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: sub.s	f2, f3, f4
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0x9a]
-# CHECK-FLOAT: trunc.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: trunc.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x3b]
-# CHECK-FLOAT: ueq.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ueq.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xda]
-# CHECK-FLOAT: ufloat.s f2, a3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ufloat.s	f2, a3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x7b]
-# CHECK-FLOAT: ule.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ule.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x5b]
-# CHECK-FLOAT: ult.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: ult.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x30,0x02,0x1b]
-# CHECK-FLOAT: un.s b0, f2, f3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: un.s	b0, f2, f3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xea]
-# CHECK-FLOAT: utrunc.s a2, f3, 5
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: utrunc.s	a2, f3, 5
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x50,0x23,0xfa]
-# CHECK-FLOAT: wfr f2, a3
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: wfr	f2, a3
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x80,0x3e,0xe3]
-# CHECK-FLOAT: rur a3, fcr
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rur	a3, fcr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
 
 [0x90,0x3e,0xe3]
-# CHECK-FLOAT: rur a3, fsr
-# CHECK-CORE: [[#@LINE-2]]:2: warning: invalid instruction encoding
+# CHECK-FLOAT: rur	a3, fsr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/Xtensa/threadptr.txt b/llvm/test/MC/Disassembler/Xtensa/threadptr.txt
new file mode 100644
index 000000000000..1e8d75fa4003
--- /dev/null
+++ b/llvm/test/MC/Disassembler/Xtensa/threadptr.txt
@@ -0,0 +1,11 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=xtensa -mattr=+threadptr -disassemble %s | FileCheck -check-prefixes=CHECK-THREADPTR %s
+# RUN: not llvm-mc -triple=xtensa -disassemble %s 2>&1 | FileCheck --implicit-check-not=warning: -check-prefixes=CHECK-CORE %s
+
+## Verify that binary code is correctly disassembled with
+## THREADPTR option enabled. Also verify that dissasembling without
+## THREADPTR option generates warnings.
+
+[0x70,0x3e,0xe3]
+# CHECK-THREADPTR: rur	a3, threadptr
+# CHECK-CORE: :[[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/ELF/bad-expr.s b/llvm/test/MC/ELF/bad-expr.s
index 2346f5493b98..e7b5f1952101 100644
--- a/llvm/test/MC/ELF/bad-expr.s
+++ b/llvm/test/MC/ELF/bad-expr.s
@@ -7,6 +7,6 @@ x:
 // CHECK: [[@LINE+1]]:{{[0-9]+}}: error: symbol '__executable_start' can not be undefined in a subtraction expression
         .quad   x-__executable_start
 
-// CHECK: [[@LINE+1]]:7: error: expected relocatable expression
+// CHECK: [[@LINE+1]]:11: error: expected relocatable expression
 .long bar - foo@got
 foo:
diff --git a/llvm/test/MC/ELF/comdat.s b/llvm/test/MC/ELF/comdat.s
deleted file mode 100644
index e5862fab9521..000000000000
--- a/llvm/test/MC/ELF/comdat.s
+++ /dev/null
@@ -1,100 +0,0 @@
-// RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --symbols --sd - | FileCheck %s
-
-// Test that we produce the group sections and that they are before the members
-
-// CHECK:        Section {
-// CHECK:          Index: 3
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 1
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 04000000 05000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-// CHECK:        Section {
-// CHECK:          Index: 6
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 8
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 2
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 07000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-// CHECK:        Section {
-// CHECK:          Index: 8
-// CHECK-NEXT:     Name: .group
-// CHECK-NEXT:     Type: SHT_GROUP
-// CHECK-NEXT:     Flags [
-// CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x0
-// CHECK-NEXT:     Offset:
-// CHECK-NEXT:     Size: 12
-// CHECK-NEXT:     Link:
-// CHECK-NEXT:     Info: 3
-// CHECK-NEXT:     AddressAlignment: 4
-// CHECK-NEXT:     EntrySize: 4
-// CHECK-NEXT:     SectionData (
-// CHECK-NEXT:       0000:    01000000 09000000 0A000000
-// CHECK-NEXT:     )
-// CHECK-NEXT:   }
-
-// Test that g1 and g2 are local, but g3 is an undefined global.
-
-// CHECK:        Symbol {
-// CHECK:          Name: g1
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .foo
-// CHECK-NEXT:   }
-// CHECK-NEXT:   Symbol {
-// CHECK-NEXT:     Name: g2
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Local
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: .group
-// CHECK-NEXT:   }
-
-// CHECK:        Symbol {
-// CHECK:          Name: g3
-// CHECK-NEXT:     Value: 0x0
-// CHECK-NEXT:     Size: 0
-// CHECK-NEXT:     Binding: Global
-// CHECK-NEXT:     Type: None
-// CHECK-NEXT:     Other: 0
-// CHECK-NEXT:     Section: Undefined (0x0)
-// CHECK-NEXT:   }
-
-
-	.section	.foo,"axG",@progbits,g1,comdat
-g1:
-        nop
-
-        .section	.bar,"ax?",@progbits
-        nop
-
-        .section	.zed,"axG",@progbits,g2,comdat
-        nop
-
-        .section	.baz,"axG",@progbits,g3,comdat
-        .long g3
diff --git a/llvm/test/MC/ELF/group.s b/llvm/test/MC/ELF/group.s
new file mode 100644
index 000000000000..8f6097e8a66f
--- /dev/null
+++ b/llvm/test/MC/ELF/group.s
@@ -0,0 +1,59 @@
+# RUN: llvm-mc -filetype=obj -triple x86_64 %s -o %t
+# RUN: llvm-readelf -SsX %t | FileCheck %s
+# RUN: llvm-readelf -g %t | FileCheck %s --check-prefix=GROUP
+
+## Test that we produce the group sections and that they are before the members
+
+# CHECK:       [ 3] .group            GROUP           0000000000000000 {{.*}} 00000c 04     14   1  4
+# CHECK-NEXT:  [ 4] .foo              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 5] .bar              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 6] .group            GROUP           0000000000000000 {{.*}} 000008 04     14   2  4
+# CHECK-NEXT:  [ 7] .zed              PROGBITS        0000000000000000 {{.*}} 000001 00 AXG  0   0  1
+# CHECK-NEXT:  [ 8] .group            GROUP           0000000000000000 {{.*}} 00000c 04     14   4  4
+# CHECK-NEXT:  [ 9] .baz              PROGBITS        0000000000000000 {{.*}} 000004 00 AXG  0   0  1
+# CHECK-NEXT:  [10] .rela.baz         RELA            0000000000000000 {{.*}} 000018 18   G 14   9  8
+# CHECK-NEXT:  [11] sec               PROGBITS        0000000000000000 {{.*}} 000000 00      0   0  1
+# CHECK-NEXT:  [12] .group            GROUP           0000000000000000 {{.*}} 000008 04     14   3  4
+# CHECK-NEXT:  [13] .qux              PROGBITS        0000000000000000 {{.*}} 000000 00 AXG  0   0  1
+# CHECK-NEXT:  [14] .symtab           SYMTAB
+
+## Test that g1 and g2 are local, but g3 is an undefined global.
+
+# CHECK:      0000000000000000     0 NOTYPE  LOCAL  DEFAULT    4 (.foo)    g1
+# CHECK-NEXT: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT    6 (.group)  g2
+# CHECK-NEXT: 0000000000000000     0 SECTION LOCAL  DEFAULT   11 (sec)     sec
+# CHECK-NEXT: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT  UND           g3
+
+# GROUP:       COMDAT group section [    3] `.group' [g1] contains 2 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    4]   .foo
+# GROUP-NEXT:     [    5]   .bar
+# GROUP-EMPTY:
+# GROUP-NEXT:  COMDAT group section [    6] `.group' [g2] contains 1 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    7]   .zed
+# GROUP-EMPTY:
+# GROUP-NEXT:  COMDAT group section [    8] `.group' [g3] contains 2 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [    9]   .baz
+# GROUP-NEXT:     [   10]   .rela.baz
+# GROUP-EMPTY:
+# GROUP-NEXT:  group section [   12] `.group' [] contains 1 sections:
+# GROUP-NEXT:     [Index]    Name
+# GROUP-NEXT:     [   13]   .qux
+
+	.section	.foo,"axG",@progbits,g1,comdat
+g1:
+        nop
+
+        .section	.bar,"ax?",@progbits
+        nop
+
+        .section	.zed,"axG",@progbits,g2,comdat
+        nop
+
+        .section	.baz,"axG",@progbits,g3,comdat
+        .long g3
+
+.section sec
+.section .qux,"axG",@progbits,sec
diff --git a/llvm/test/MC/ELF/mc-dump.s b/llvm/test/MC/ELF/mc-dump.s
new file mode 100644
index 000000000000..9c31daed3450
--- /dev/null
+++ b/llvm/test/MC/ELF/mc-dump.s
@@ -0,0 +1,53 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t -debug-only=mc-dump-pre,mc-dump 2>&1 | FileCheck %s --match-full-lines --strict-whitespace
+
+#CHECK-LABEL:assembler backend - pre-layout
+#      CHECK:MCSection Name:.text
+#CHECK-LABEL:assembler backend - final-layout
+#      CHECK:Sections:[
+# CHECK-NEXT:MCSection Name:.text
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .text
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4 Nops
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 _start
+# CHECK-NEXT:0 Org Offset:3 Value:0
+# CHECK-NEXT:3 Relaxable Size:2 <MCInst #1996 <MCOperand Expr:.Ltmp0>>
+# CHECK-NEXT:  Fixup @1 Value:.Ltmp0-1 Kind:4001
+# CHECK-NEXT:5 Data Size:16 [48,8b,04,25,00,00,00,00,48,8b,04,25,00,00,00,00]
+# CHECK-NEXT:  Fixup @4 Value:f0@<variant 11> Kind:4017
+# CHECK-NEXT:  Fixup @12 Value:_start@<variant 11> Kind:4017
+# CHECK-NEXT:  Symbol @16 .Ltmp0 Temporary
+# CHECK-NEXT:MCSection Name:.data
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .data
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4
+# CHECK-NEXT:0 Data Size:4 [01,00,00,00]
+# CHECK-NEXT:4 Fill Value:0 ValueSize:1 NumValues:1
+# CHECK-NEXT:5 LEB Value:.Ltmp0-_start Signed:0
+# CHECK-NEXT:]
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t -debug-only=mc-dump -save-temp-labels -g 2>&1 | FileCheck %s --check-prefix=CHECK2
+
+#      CHECK2:5 Data Size:16 [48,8b,04,25,00,00,00,00,48,8b,04,25,00,00,00,00]
+# CHECK2-NEXT:  Fixup @4 Value:f0@<variant 11> Kind:4017
+# CHECK2-NEXT:  Fixup @12 Value:_start@<variant 11> Kind:4017
+# CHECK2-NEXT:  Symbol @16 .Ltmp1
+# CHECK2-NEXT:  Symbol @0 .Ltmp3 Temporary
+# CHECK2-NEXT:  Symbol @8 .Ltmp4 Temporary
+# CHECK2-NEXT:  Symbol @16 .Ltmp5 Temporary
+# CHECK2-NEXT:  Symbol @16 .Lsec_end0 Temporary
+
+_start:
+var = _start
+.org 3
+jmp 1f
+movq f0@GOTPCREL, %rax
+movq _start@GOTPCREL, %rax
+1:
+
+.data
+.p2align 2
+.long 1
+.space 1
+.uleb128 1b-_start
diff --git a/llvm/test/MC/ELF/relocation-alias.s b/llvm/test/MC/ELF/relocation-alias.s
index 9720a55556ed..154ba460ba68 100644
--- a/llvm/test/MC/ELF/relocation-alias.s
+++ b/llvm/test/MC/ELF/relocation-alias.s
@@ -2,8 +2,8 @@
 # RUN: llvm-objdump --no-print-imm-hex -dr %t | FileCheck %s
 # RUN: llvm-readelf -s %t | FileCheck %s --check-prefix=SYM
 
-# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR0=1 %s 2>&1 | FileCheck %s --check-prefix=ERR0 --implicit-check-not=error:
-# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR=1 %s 2>&1 | FileCheck %s --check-prefix=ERR
+# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR0=1 %s -o %t 2>&1 | FileCheck %s --check-prefix=ERR0 --implicit-check-not=error:
+# RUN: not llvm-mc -filetype=obj -triple x86_64 --defsym ERR=1 %s -o %t 2>&1 | FileCheck %s --check-prefix=ERR
 
 ## If a fixup symbol is equated to an undefined symbol, convert the fixup
 ## to be against the target symbol, even if there is a variant (@PLT).
@@ -51,7 +51,7 @@ memcpy_spec1 = __GI_memcpy@PLT+1
 .ifdef ERR
 .text
 ## Note, GNU as emits a relocation for this erroneous fixup.
-# ERR: {{.*}}.s:[[#@LINE+2]]:1: error: expected relocatable expression
+# ERR: {{.*}}.s:[[#@LINE+2]]:6: error: expected relocatable expression
 memcpy_plus_1 = __GI_memcpy+1
 call memcpy_plus_1@PLT
 .endif
diff --git a/llvm/test/MC/ELF/weakref.s b/llvm/test/MC/ELF/weakref.s
index a86162526953..49508463f8c5 100644
--- a/llvm/test/MC/ELF/weakref.s
+++ b/llvm/test/MC/ELF/weakref.s
@@ -113,6 +113,6 @@ alias:
 .weakref cycle1, cycle0
 call cycle0
 # ERR2: <unknown>:0: error: cyclic dependency detected for symbol 'cycle0'
-# ERR2: [[#@LINE-2]]:1: error: expected relocatable expression
+# ERR2: [[#@LINE-2]]:6: error: expected relocatable expression
 
 .endif
diff --git a/llvm/test/MC/GOFF/empty-goff.s b/llvm/test/MC/GOFF/empty-goff.s
index f6d402863d71..ba1b0e0a4907 100644
--- a/llvm/test/MC/GOFF/empty-goff.s
+++ b/llvm/test/MC/GOFF/empty-goff.s
@@ -17,8 +17,8 @@
 *  03 is prefix byte
 *  4. is header type
 *  .0 is version
-* CHECK: 000050 03 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00
-* CHECK: 000060 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
-* CHECK: 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
-* CHECK: 000080 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
-* CHECK: 000090 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+* CHECK: {{([[:xdigit:]]{6})}} 03 40 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+* CHECK: {{([[:xdigit:]]{6})}} 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+* CHECK: {{([[:xdigit:]]{6})}} 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+* CHECK: {{([[:xdigit:]]{6})}} 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+* CHECK: {{([[:xdigit:]]{6})}} 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
diff --git a/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s b/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
index c72eef7cd991..c668729e44c9 100644
--- a/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
+++ b/llvm/test/MC/LoongArch/Relocations/fixups-diagnostics.s
@@ -1,20 +1,20 @@
 # RUN: not llvm-mc --triple=loongarch64 --filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
 
-  beq $a0, $a1, unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
-  beqz $a0, unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
-  b unaligned # CHECK: :[[#@LINE]]:3: error: fixup value must be 4-byte aligned
+  beq $a0, $a1, unaligned # CHECK: :[[#@LINE]]:17: error: fixup value must be 4-byte aligned
+  beqz $a0, unaligned # CHECK: :[[#@LINE]]:13: error: fixup value must be 4-byte aligned
+  b unaligned # CHECK: :[[#@LINE]]:5: error: fixup value must be 4-byte aligned
   .byte 0
 unaligned:
   .byte 0
   .byte 0
   .byte 0
 
-  beq $a0, $a1, out_of_range_b18 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-131072, 131071]
+  beq $a0, $a1, out_of_range_b18 # CHECK: :[[#@LINE]]:17: error: fixup value out of range [-131072, 131071]
   .space 1<<18
 out_of_range_b18:
-  beqz $a0, out_of_range_b23 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-4194304, 4194303]
+  beqz $a0, out_of_range_b23 # CHECK: :[[#@LINE]]:13: error: fixup value out of range [-4194304, 4194303]
   .space 1<<23
 out_of_range_b23:
-  b out_of_range_b28 # CHECK: :[[#@LINE]]:3: error: fixup value out of range [-134217728, 134217727]
+  b out_of_range_b28 # CHECK: :[[#@LINE]]:5: error: fixup value out of range [-134217728, 134217727]
   .space 1<<28
 out_of_range_b28:
diff --git a/llvm/test/MC/M68k/Control/branch-pc-rel.s b/llvm/test/MC/M68k/Control/branch-pc-rel.s
index 912959ba94fa..bfeed3addbf2 100644
--- a/llvm/test/MC/M68k/Control/branch-pc-rel.s
+++ b/llvm/test/MC/M68k/Control/branch-pc-rel.s
@@ -2,11 +2,9 @@
 
 	; CHECK:      beq  .LBB0_1
 	; CHECK-SAME: encoding: [0x67,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	beq	.LBB0_1
 	; CHECK:      bra  .LBB0_2
 	; CHECK-SAME: encoding: [0x60,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/bsr.s b/llvm/test/MC/M68k/Control/bsr.s
index 6b7a7d5c6ecd..71ceb56ea66a 100644
--- a/llvm/test/MC/M68k/Control/bsr.s
+++ b/llvm/test/MC/M68k/Control/bsr.s
@@ -2,11 +2,9 @@
 
 	; CHECK:      bsr.b   .LBB0_1
 	; CHECK-SAME: encoding: [0x61,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	bsr.b .LBB0_1
 	; CHECK:      bsr.w   .LBB0_2
 	; CHECK-SAME: encoding: [0x61,0x00,A,A]
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_2, kind: FK_PCRel_2
 	bsr.w	.LBB0_2
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/bsr32.s b/llvm/test/MC/M68k/Control/bsr32.s
index 58ed4a29a353..5151d6281ef3 100644
--- a/llvm/test/MC/M68k/Control/bsr32.s
+++ b/llvm/test/MC/M68k/Control/bsr32.s
@@ -2,15 +2,12 @@
 
 	; CHECK:      bsr.b   .LBB0_1
 	; CHECK-SAME: encoding: [0x61,A]
-        ; CHECK:      fixup A - offset: 1, value: .LBB0_1-1, kind: FK_PCRel_1
 	bsr.b .LBB0_1
 	; CHECK:      bsr.w   .LBB0_2
 	; CHECK-SAME: encoding: [0x61,0x00,A,A]
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_2, kind: FK_PCRel_2
 	bsr.w	.LBB0_2
   ; CHECK:     bsr.l   .LBB0_3
   ; CHECK-SAME: encoding: [0x61,0xff,A,A,A,A] 
-        ; CHECK:      fixup A - offset: 2, value: .LBB0_3, kind: FK_PCRel_4
   bsr.l .LBB0_3  
 .LBB0_1:
 	; CHECK:      add.l  #0, %d0
diff --git a/llvm/test/MC/M68k/Control/call-pc-rel.s b/llvm/test/MC/M68k/Control/call-pc-rel.s
index ee0147ef52c9..26d30530ca91 100644
--- a/llvm/test/MC/M68k/Control/call-pc-rel.s
+++ b/llvm/test/MC/M68k/Control/call-pc-rel.s
@@ -16,12 +16,10 @@ PCI:
 	; CHECK:     jsr  (BACKWARD,%pc,%d0)
         ; CHECK-OBJ: jsr  (250,%pc,%d0)
 	; CHECK-SAME: encoding: [0x4e,0xbb,0x08,A]
-        ; CHECK:      fixup A - offset: 3, value: BACKWARD+1, kind: FK_PCRel_1
 	jsr	(BACKWARD,%pc,%d0)
 	; CHECK:     jsr  (FORWARD,%pc,%d0)
         ; CHECK-OBJ: jsr  (10,%pc,%d0)
 	; CHECK-SAME: encoding: [0x4e,0xbb,0x08,A]
-        ; CHECK:      fixup A - offset: 3, value: FORWARD+1, kind: FK_PCRel_1
 	jsr	(FORWARD,%pc,%d0)
 
 ; CHECK-LABEL: PCD:
@@ -29,12 +27,10 @@ PCD:
 	; CHECK:     jsr  (BACKWARD,%pc)
 	; CHECK-OBJ: jsr  (65522,%pc)
 	; CHECK-SAME: encoding: [0x4e,0xba,A,A]
-        ; CHECK:      fixup A - offset: 2, value: BACKWARD, kind: FK_PCRel_2
 	jsr	(BACKWARD,%pc)
 	; CHECK:     jsr  (FORWARD,%pc)
 	; CHECK-OBJ: jsr  (2,%pc)
 	; CHECK-SAME: encoding: [0x4e,0xba,A,A]
-        ; CHECK:      fixup A - offset: 2, value: FORWARD, kind: FK_PCRel_2
 	jsr	(FORWARD,%pc)
 
 ; CHECK-LABEL: FORWARD:
diff --git a/llvm/test/MC/M68k/Relaxations/PIC/branch.s b/llvm/test/MC/M68k/Relaxations/PIC/branch.s
index 5035ca3ce50a..e3384e57eae0 100644
--- a/llvm/test/MC/M68k/Relaxations/PIC/branch.s
+++ b/llvm/test/MC/M68k/Relaxations/PIC/branch.s
@@ -1,13 +1,12 @@
 ; RUN: llvm-mc -triple=m68k -motorola-integers -filetype=obj --position-independent < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; CHECK-LABEL: <TIGHT>:
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -18,7 +17,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -29,7 +27,6 @@ RELAXED:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relaxations/PIC/branch32.s b/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
index 7e40f055aff1..1d5d4f09267c 100644
--- a/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
+++ b/llvm/test/MC/M68k/Relaxations/PIC/branch32.s
@@ -1,7 +1,7 @@
 ; RUN: llvm-mc -triple=m68k --mcpu=M68020 -motorola-integers -filetype=obj --position-independent < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; Branch relocations are relaxed as part of object layout, so -show-encoding still 
 ; shows them as 1-byte relocactions 
@@ -10,7 +10,6 @@
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -21,7 +20,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -34,7 +32,6 @@ RELAXED_32:
 	; CHECK-NEXT: 00 00
 	; CHECK-NEXT: 80 04
 	; INSTR: bra .LBB2_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
 	bra	.LBB2_1
 	.space 0x8000  ; Greater than i16::MAX.
 .LBB2_1:
@@ -45,7 +42,6 @@ RELAXED_32:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relaxations/branch32.s b/llvm/test/MC/M68k/Relaxations/branch32.s
index ba5c371dada8..766de3c1da19 100644
--- a/llvm/test/MC/M68k/Relaxations/branch32.s
+++ b/llvm/test/MC/M68k/Relaxations/branch32.s
@@ -1,13 +1,12 @@
 ; RUN: llvm-mc -triple=m68k --mcpu=M68020 -motorola-integers -filetype=obj < %s \
 ; RUN:     | llvm-objdump -d - | FileCheck %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; CHECK-LABEL: <TIGHT>:
 TIGHT:
 	; CHECK: bra  $7f
 	; INSTR: bra .LBB0_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB0_2-1, kind: FK_PCRel_1
 	bra	.LBB0_2
 	.space 0x7F  ; i8::MAX
 .LBB0_2:
@@ -18,7 +17,6 @@ TIGHT:
 RELAXED:
 	; CHECK: bra  $82
 	; INSTR: bra .LBB1_2 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB1_2-1, kind: FK_PCRel_1
 	bra	.LBB1_2
 	.space 0x80  ; Greater than i8::MAX
 .LBB1_2:
@@ -31,7 +29,6 @@ RELAXED_32:
 	; CHECK-NEXT: 00 00
 	; CHECK-NEXT: 80 04
 	; INSTR: bra .LBB2_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB2_1-1, kind: FK_PCRel_1
 	bra	.LBB2_1
 	.space 0x8000  ; Greater than i16::MAX.
 .LBB2_1:
@@ -42,7 +39,6 @@ RELAXED_32:
 ZERO:
 	; CHECK: bra  $2
 	; INSTR: bra .LBB3_1 ; encoding: [0x60,A]
-	; FIXUP: fixup A - offset: 1, value: .LBB3_1-1, kind: FK_PCRel_1
 	bra	.LBB3_1
 .LBB3_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-abs.s b/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
index 0df09c8cb5cc..8a77c40117a9 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-abs.s
@@ -1,26 +1,22 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_32 dst 0x0
 ; INSTR: move.l dst, %d0
-; FIXUP: fixup A - offset: 2, value: dst, kind: FK_Data_4
 move.l	dst, %d0
 
 ; Relocating immediate values
 
 ; RELOC: R_68K_8 str8 0x0
 ; INSTR: move.b  #str8,  (4,%sp)
-; FIXUP: fixup A - offset: 3, value: str8, kind: FK_Data_1
 move.b  #str8,  (4,%sp)
 
 ; RELOC: R_68K_16 str16 0x0
 ; INSTR: move.w  #str16,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str16, kind: FK_Data_2
 move.w  #str16, (4,%sp)
 
 ; RELOC: R_68K_32 str32 0x0
 ; INSTR: move.l  #str32,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str32, kind: FK_Data_4
 move.l  #str32, (4,%sp)
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s b/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
index ca1ce52de1ca..ef6c25926f02 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-gotoff.s
@@ -1,19 +1,16 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTOFF8 dst1 0x0
 ; INSTR: move.l  (dst1@GOTOFF,%a5,%d0), %d0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTOFF, kind: FK_Data_1
 move.l	(dst1@GOTOFF,%a5,%d0), %d0
 
 ; RELOC: R_68K_GOTOFF16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTOFF,%a5), %d0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTOFF, kind: FK_Data_2
 move.l	(dst2@GOTOFF,%a5), %d0
 
 ; RELOC: R_68K_GOTPCREL16 dst3 0x0
 ; INSTR: lea     (dst3@GOTPCREL,%pc), %a5
-; FIXUP: fixup A - offset: 2, value: dst3@GOTPCREL, kind: FK_PCRel_2
 lea	(dst3@GOTPCREL,%pc), %a5
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s b/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
index 9f79dd94f819..be84c090bc7d 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-gotpcrel.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTPCREL8 dst1 0x1
 ; INSTR: move.l  (dst1@GOTPCREL,%pc,%d0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTPCREL+1, kind: FK_PCRel_1
 move.l	(dst1@GOTPCREL,%pc,%d0), %a0
 
 ; RELOC: R_68K_GOTPCREL16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTPCREL,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTPCREL, kind: FK_PCRel_2
 move.l	(dst2@GOTPCREL,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s b/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
index ef57ca589e64..8eb186121649 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/data-pc-rel.s
@@ -1,20 +1,17 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj --position-independent %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding --position-independent %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PC8 dst1 0x1
 ; INSTR: move.l  (dst1,%pc,%a0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1+1, kind: FK_PCRel_1
 move.l	(dst1,%pc,%a0), %a0
 
 ; RELOC: R_68K_PC16 dst2 0x0
 ; INSTR: move.l  (dst2,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2, kind: FK_PCRel_2
 move.l	(dst2,%pc), %a0
 
 ; Shouldn't have any relocation
 ; RELOC-NOT: R_68K_PC
 ; INSTR: move.l  (0,%pc), %a0
-; FIXUP-NOT: fixup
 move.l	(0,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/PIC/text-plt.s b/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
index 0ccd1758004a..d5a103079c61 100644
--- a/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
+++ b/llvm/test/MC/M68k/Relocations/PIC/text-plt.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 --position-independent -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 --position-independent -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PLT16 target 0x0
 ; INSTR: jsr     (target@PLT,%pc)
-; FIXUP: fixup A - offset: 2, value: target@PLT, kind: FK_PCRel_2
 jsr	(target@PLT,%pc)
 
 ; RELOC: R_68K_PLT32  __tls_get_addr 0x0
 ; INSTR: bsr.l   __tls_get_addr@PLT
-; FIXUP: fixup A - offset: 2, value: __tls_get_addr@PLT, kind: FK_PCRel_4
 bsr.l __tls_get_addr@PLT
diff --git a/llvm/test/MC/M68k/Relocations/data-abs.s b/llvm/test/MC/M68k/Relocations/data-abs.s
index b15007d8da36..4ddbd326d171 100644
--- a/llvm/test/MC/M68k/Relocations/data-abs.s
+++ b/llvm/test/MC/M68k/Relocations/data-abs.s
@@ -1,26 +1,22 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_32 dst 0x0
 ; INSTR: move.l dst, %d0
-; FIXUP: fixup A - offset: 2, value: dst, kind: FK_Data_4
 move.l	dst, %d0
 
 ; Relocating immediate values
 
 ; RELOC: R_68K_8 str8 0x0
 ; INSTR: move.b  #str8,  (4,%sp)
-; FIXUP: fixup A - offset: 3, value: str8, kind: FK_Data_1
 move.b  #str8,  (4,%sp)
 
 ; RELOC: R_68K_16 str16 0x0
 ; INSTR: move.w  #str16,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str16, kind: FK_Data_2
 move.w  #str16, (4,%sp)
 
 ; RELOC: R_68K_32 str32 0x0
 ; INSTR: move.l  #str32,  (4,%sp)
-; FIXUP: fixup A - offset: 2, value: str32, kind: FK_Data_4
 move.l  #str32, (4,%sp)
diff --git a/llvm/test/MC/M68k/Relocations/data-gotoff.s b/llvm/test/MC/M68k/Relocations/data-gotoff.s
index e0c3affc6d1d..dc56e691937f 100644
--- a/llvm/test/MC/M68k/Relocations/data-gotoff.s
+++ b/llvm/test/MC/M68k/Relocations/data-gotoff.s
@@ -1,19 +1,16 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTOFF8 dst1 0x0
 ; INSTR: move.l  (dst1@GOTOFF,%a5,%d0), %d0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTOFF, kind: FK_Data_1
 move.l	(dst1@GOTOFF,%a5,%d0), %d0
 
 ; RELOC: R_68K_GOTOFF16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTOFF,%a5), %d0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTOFF, kind: FK_Data_2
 move.l	(dst2@GOTOFF,%a5), %d0
 
 ; RELOC: R_68K_GOTPCREL16 dst3 0x0
 ; INSTR: lea     (dst3@GOTPCREL,%pc), %a5
-; FIXUP: fixup A - offset: 2, value: dst3@GOTPCREL, kind: FK_PCRel_2
 lea	(dst3@GOTPCREL,%pc), %a5
diff --git a/llvm/test/MC/M68k/Relocations/data-gotpcrel.s b/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
index ac3ba96d1f09..ceffc77dd6e3 100644
--- a/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
+++ b/llvm/test/MC/M68k/Relocations/data-gotpcrel.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_GOTPCREL8 dst1 0x1
 ; INSTR: move.l  (dst1@GOTPCREL,%pc,%d0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1@GOTPCREL+1, kind: FK_PCRel_1
 move.l	(dst1@GOTPCREL,%pc,%d0), %a0
 
 ; RELOC: R_68K_GOTPCREL16 dst2 0x0
 ; INSTR: move.l  (dst2@GOTPCREL,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2@GOTPCREL, kind: FK_PCRel_2
 move.l	(dst2@GOTPCREL,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/data-pc-rel.s b/llvm/test/MC/M68k/Relocations/data-pc-rel.s
index 909681da5a95..23e0bcc73269 100644
--- a/llvm/test/MC/M68k/Relocations/data-pc-rel.s
+++ b/llvm/test/MC/M68k/Relocations/data-pc-rel.s
@@ -1,20 +1,17 @@
 ; RUN: llvm-mc -triple m68k -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PC8 dst1 0x1
 ; INSTR: move.l  (dst1,%pc,%a0), %a0
-; FIXUP: fixup A - offset: 3, value: dst1+1, kind: FK_PCRel_1
 move.l	(dst1,%pc,%a0), %a0
 
 ; RELOC: R_68K_PC16 dst2 0x0
 ; INSTR: move.l  (dst2,%pc), %a0
-; FIXUP: fixup A - offset: 2, value: dst2, kind: FK_PCRel_2
 move.l	(dst2,%pc), %a0
 
 ; Shouldn't have any relocation
 ; RELOC-NOT: R_68K_PC
 ; INSTR: move.l  (0,%pc), %a0
-; FIXUP-NOT: fixup
 move.l	(0,%pc), %a0
diff --git a/llvm/test/MC/M68k/Relocations/text-plt.s b/llvm/test/MC/M68k/Relocations/text-plt.s
index 7de04b8b2182..131e66947444 100644
--- a/llvm/test/MC/M68k/Relocations/text-plt.s
+++ b/llvm/test/MC/M68k/Relocations/text-plt.s
@@ -1,14 +1,12 @@
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -filetype=obj %s -o - \
 ; RUN:   | llvm-readobj -r - | FileCheck -check-prefix=RELOC %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -show-encoding %s -o - \
-; RUN:   | FileCheck -check-prefix=INSTR -check-prefix=FIXUP %s
+; RUN:   | FileCheck -check-prefix=INSTR %s
 
 ; RELOC: R_68K_PLT16 target 0x0
 ; INSTR: jsr     (target@PLT,%pc)
-; FIXUP: fixup A - offset: 2, value: target@PLT, kind: FK_PCRel_2
 jsr	(target@PLT,%pc)
 
 ; RELOC: R_68K_PLT32  __tls_get_addr 0x0
 ; INSTR: bsr.l   __tls_get_addr@PLT
-; FIXUP: fixup A - offset: 2, value: __tls_get_addr@PLT, kind: FK_PCRel_4
 bsr.l __tls_get_addr@PLT
diff --git a/llvm/test/MC/M68k/Relocations/tls.s b/llvm/test/MC/M68k/Relocations/tls.s
index ec7e11f17fe7..50f67eb5f158 100644
--- a/llvm/test/MC/M68k/Relocations/tls.s
+++ b/llvm/test/MC/M68k/Relocations/tls.s
@@ -1,17 +1,12 @@
-; RUN: llvm-mc -triple m68k -show-encoding %s | FileCheck -check-prefix=INSTR %s
+; RUN: llvm-mc -triple m68k %s | FileCheck -check-prefix=INSTR %s
 ; RUN: llvm-mc -triple m68k --mcpu=M68020 -filetype=obj %s -o %t
 ; RUN: llvm-readelf -rs %t | FileCheck --check-prefix=READELF %s --implicit-check-not=TLS
 
 ; INSTR:      move.l  (le@TPOFF,%a0), %d0
-; INSTR-NEXT:                fixup A - offset: 2, value: le@TPOFF, kind: FK_Data_2
 ; INSTR-NEXT: move.l  (ie@GOTTPOFF,%a0), %a2
-; INSTR-NEXT:                fixup A - offset: 2, value: ie@GOTTPOFF, kind: FK_Data_2
 ; INSTR-NEXT: lea     (gd@TLSGD,%a0), %a0
-; INSTR-NEXT:                fixup A - offset: 2, value: gd@TLSGD, kind: FK_Data_2
 ; INSTR-NEXT: move.l  (ld@TLSLD,%a0), %d0
-; INSTR-NEXT:                fixup A - offset: 2, value: ld@TLSLD, kind: FK_Data_2
 ; INSTR-NEXT: lea     (ld@TLSLDM,%a2), %a1
-; INSTR-NEXT:                fixup A - offset: 2, value: ld@TLSLDM, kind: FK_Data_2
 
 ; READELF:      R_68K_TLS_LE16         00000000   le + 0
 ; READELF-NEXT: R_68K_TLS_IE16         00000000   ie + 0
diff --git a/llvm/test/MC/M68k/pc-rel.s b/llvm/test/MC/M68k/pc-rel.s
index a4d76d341bb0..f5d98ec855e6 100644
--- a/llvm/test/MC/M68k/pc-rel.s
+++ b/llvm/test/MC/M68k/pc-rel.s
@@ -3,7 +3,6 @@
         ; A fixup whose size is multiple of a word.
 	; CHECK:      cmpi.l  #87, (.LBB0_1,%pc)
 	; CHECK-SAME: encoding: [0x0c,0xba,0x00,0x00,0x00,0x57,A,A]
-        ; CHECK:      fixup A - offset: 6, value: .LBB0_1, kind: FK_PCRel_2
 	cmpi.l  #87, (.LBB0_1,%pc)
 
         ; A fixup that is smaller than a word.
@@ -11,7 +10,6 @@
         ; tested by `Control/branch-pc-rel.s`.
 	; CHECK:      cmpi.l  #94, (.LBB0_2,%pc,%a0)
 	; CHECK-SAME: encoding: [0x0c,0xbb,0x00,0x00,0x00,0x5e,0x88,A]
-        ; CHECK:      fixup A - offset: 7, value: .LBB0_2+1, kind: FK_PCRel_1
 	cmpi.l  #94, (.LBB0_2,%pc,%a0)
 .LBB0_1:
 	add.l	#0, %d0
diff --git a/llvm/test/MC/Mips/reloc-directive-bad-obj.s b/llvm/test/MC/Mips/reloc-directive-bad-obj.s
index 94317d0830df..86d6d0cc66c5 100644
--- a/llvm/test/MC/Mips/reloc-directive-bad-obj.s
+++ b/llvm/test/MC/Mips/reloc-directive-bad-obj.s
@@ -1,9 +1,9 @@
-# RUN: not llvm-mc -triple mips-unknown-linux %s -show-encoding \
-# RUN:     -target-abi=o32 -filetype=obj 2>&1 | FileCheck %s
-	.text
-	nop
-	.reloc foo, R_MIPS_32, .text  # CHECK: :[[@LINE]]:2: error: unresolved relocation offset
-	nop
-	nop
-	.reloc bar, R_MIPS_32, .text  # CHECK: :[[@LINE]]:2: error: unresolved relocation offset
-	nop
+# RUN: not llvm-mc -triple mips-unknown-linux %s \
+# RUN:     -target-abi=o32 -filetype=obj -o /dev/null 2>&1 | FileCheck %s
+.text
+nop
+.reloc foo, R_MIPS_32, .text  # CHECK: :[[@LINE]]:24: error: unresolved relocation offset
+nop
+nop
+.reloc bar, R_MIPS_32, .text  # CHECK: :[[@LINE]]:24: error: unresolved relocation offset
+nop
diff --git a/llvm/test/MC/Mips/unsupported-relocation.s b/llvm/test/MC/Mips/unsupported-relocation.s
index 311c0c7d60f2..4cbcff2f168b 100644
--- a/llvm/test/MC/Mips/unsupported-relocation.s
+++ b/llvm/test/MC/Mips/unsupported-relocation.s
@@ -10,4 +10,4 @@ foo:
         .byte   x
 # CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
         .byte   x+1
-# CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
+# CHECK: :[[@LINE-1]]:18: error: MIPS does not support one byte relocations
diff --git a/llvm/test/MC/PowerPC/ppc64-abs-reloc.s b/llvm/test/MC/PowerPC/ppc64-abs-reloc.s
index d38e1f4b8dc1..4fa449d9a01a 100644
--- a/llvm/test/MC/PowerPC/ppc64-abs-reloc.s
+++ b/llvm/test/MC/PowerPC/ppc64-abs-reloc.s
@@ -1,9 +1,6 @@
 # RUN: llvm-mc -triple powerpc64le-unknown-linux-gnu %s -filetype=obj -o - | \
 # RUN:    llvm-objdump -D -r - | FileCheck %s
 
-# RUN: llvm-mc -triple powerpc64-ibm-aix-xcoff %s -filetype=obj -o - | \
-# RUN:    llvm-objdump -D -r - | FileCheck %s
-
 test:                                   # @test
         add 5, 3, 4
         extsw 3, 5
diff --git a/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s b/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
index 24fbccdb8abd..2b97dca147bf 100644
--- a/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
+++ b/llvm/test/MC/RISCV/Relocations/data-directive-specifier.s
@@ -28,21 +28,21 @@ data1:
 .word %gotpcrel(extern+4), %gotpcrel(extern-5)
 
 .ifdef ERR
-# ERR: [[#@LINE+1]]:7: error: %pltpcrel can only be used in a .word directive
+# ERR: [[#@LINE+1]]:8: error: %pltpcrel can only be used in a .word directive
 .quad %pltpcrel(g)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %pltpcrel(g-.)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %pltpcrel(extern - und)
 
-# ERR: [[#@LINE+1]]:7: error: %gotpcrel can only be used in a .word directive
+# ERR: [[#@LINE+1]]:8: error: %gotpcrel can only be used in a .word directive
 .quad %gotpcrel(g)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %gotpcrel(extern - .)
 
-# ERR: [[#@LINE+1]]:7: error: expected relocatable expression
+# ERR: [[#@LINE+1]]:8: error: expected relocatable expression
 .word %gotpcrel(extern - und)
 .endif
diff --git a/llvm/test/MC/RISCV/Relocations/expr-err.s b/llvm/test/MC/RISCV/Relocations/expr-err.s
index 897004b64636..481a379321a2 100644
--- a/llvm/test/MC/RISCV/Relocations/expr-err.s
+++ b/llvm/test/MC/RISCV/Relocations/expr-err.s
@@ -3,26 +3,26 @@
 .Ltmp1:
 	.quad	tls
 
-	lui a0, %hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	lw a0, %lo(tls+0-.Ltmp1)(t0)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	lui a0, %tprel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	add a0, a0, tp, %tprel_add(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	addi a0, a0, %tprel_lo(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %tls_ie_pcrel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %tls_gd_pcrel_hi(tls+0-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %pcrel_hi(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	auipc a0, %got_pcrel_hi(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
-	addi a0, a0, %pcrel_lo(tls-.Ltmp1)
-# CHECK: :[[#@LINE-1]]:2: error: expected relocatable expression
+lui a0, %hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:10: error: expected relocatable expression
+lw a0, %lo(tls+0-.Ltmp1)(t0)
+# CHECK: :[[#@LINE-1]]:9: error: expected relocatable expression
+lui a0, %tprel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:10: error: expected relocatable expression
+add a0, a0, tp, %tprel_add(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:18: error: expected relocatable expression
+addi a0, a0, %tprel_lo(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:15: error: expected relocatable expression
+auipc a0, %tls_ie_pcrel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %tls_gd_pcrel_hi(tls+0-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %pcrel_hi(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+auipc a0, %got_pcrel_hi(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:12: error: expected relocatable expression
+addi a0, a0, %pcrel_lo(tls-.Ltmp1)
+# CHECK: :[[#@LINE-1]]:15: error: expected relocatable expression
 
 #	tail tls+32
 #	tail tls-tls
diff --git a/llvm/test/MC/RISCV/Relocations/mc-dump.s b/llvm/test/MC/RISCV/Relocations/mc-dump.s
new file mode 100644
index 000000000000..98205925da1c
--- /dev/null
+++ b/llvm/test/MC/RISCV/Relocations/mc-dump.s
@@ -0,0 +1,20 @@
+# REQUIRES: asserts
+# RUN: llvm-mc -filetype=obj --triple=riscv64 --mattr=+relax %s -debug-only=mc-dump -o /dev/null 2>&1 | FileCheck %s
+
+#      CHECK:Sections:[
+# CHECK-NEXT:MCSection Name:.text
+# CHECK-NEXT:0 Data Size:0 []
+# CHECK-NEXT:  Symbol @0 .text
+# CHECK-NEXT:0 Align Align:4 Value:0 ValueSize:1 MaxBytesToEmit:4 Nops
+# CHECK-NEXT:0 Data LinkerRelaxable Size:8 [97,00,00,00,e7,80,00,00]
+# CHECK-NEXT:  Fixup @0 Value:specifier(19,ext) Kind:4022
+# CHECK-NEXT:  Symbol @0 $x
+# CHECK-NEXT:8 Align Align:8 Value:0 ValueSize:1 MaxBytesToEmit:8 Nops
+# CHECK-NEXT:12 Data Size:4 [13,05,30,00]
+# CHECK-NEXT:16 Align Align:8 Value:0 ValueSize:1 MaxBytesToEmit:8 Nops
+# CHECK-NEXT:]
+
+call ext
+.p2align 3
+li x10, 3
+.p2align 3
diff --git a/llvm/test/MC/RISCV/fixups-diagnostics.s b/llvm/test/MC/RISCV/fixups-diagnostics.s
index 3c99673b81fc..d19f4050d97e 100644
--- a/llvm/test/MC/RISCV/fixups-diagnostics.s
+++ b/llvm/test/MC/RISCV/fixups-diagnostics.s
@@ -1,9 +1,9 @@
 # RUN: not llvm-mc -triple riscv32 -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
 
-  jal a0, far_distant # CHECK: :[[@LINE]]:3: error: fixup value out of range
-  jal a0, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  jal a0, far_distant # CHECK: :[[@LINE]]:11: error: fixup value out of range
+  jal a0, unaligned # CHECK: :[[@LINE]]:11: error: fixup value must be 2-byte aligned
 
-  blt t0, t1, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  blt t0, t1, unaligned # CHECK: :[[@LINE]]:15: error: fixup value must be 2-byte aligned
 
   .byte 0
 unaligned:
diff --git a/llvm/test/MC/RISCV/option-exact-long-jump-disable.s b/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
index 6e4b5fdd31e1..79ca919f8b9d 100644
--- a/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
+++ b/llvm/test/MC/RISCV/option-exact-long-jump-disable.s
@@ -40,12 +40,12 @@ bar:
   .text
 ## Branches to defined out-of-range symbols should report an error
 test_defined_out_of_range:
-  bne a0, a1, 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  bne a0, a1, 1f # CHECK: :[[#@LINE]]:15: error: fixup value out of range
   .skip (1 << 12)
 1:
-  c.beqz a0, 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  c.beqz a0, 1f # CHECK: :[[#@LINE]]:14: error: fixup value out of range
   .skip (1 << 8)
 1:
-  c.j 1f # CHECK: :[[#@LINE]]:3: error: fixup value out of range
+  c.j 1f # CHECK: :[[#@LINE]]:7: error: fixup value out of range
   .skip (1 << 11)
 1:
diff --git a/llvm/test/MC/RISCV/pcrel-lo12-invalid.s b/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
index 74a1f2fac923..e6dce0d6c698 100644
--- a/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
+++ b/llvm/test/MC/RISCV/pcrel-lo12-invalid.s
@@ -2,6 +2,6 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=+relax -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
 
 1:
-  addi a0, a0, %pcrel_lo(1b) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
-  addi a0, a0, %pcrel_lo(0x123456) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
-  addi a0, a0, %pcrel_lo(foo) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(1b) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(0x123456) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
+  addi a0, a0, %pcrel_lo(foo) # CHECK: :[[@LINE]]:17: error: could not find corresponding %pcrel_hi
diff --git a/llvm/test/MC/RISCV/rv32-relaxation-xqci.s b/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
index 03c9914bb37d..b38aa373c90f 100644
--- a/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
+++ b/llvm/test/MC/RISCV/rv32-relaxation-xqci.s
@@ -160,9 +160,9 @@ EXT_JUMP_NEGATIVE:
 .space 0x1000
 
   jal t1, EXT_JUMP
-# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+# ERROR: [[@LINE-1]]:11: error: fixup value out of range
   jal t1, EXT_JUMP_NEGATIVE
-# ERROR: [[@LINE-1]]:3: error: fixup value out of range
+# ERROR: [[@LINE-1]]:11: error: fixup value out of range
 
 .space 0x1000
 .space 0x100000
diff --git a/llvm/test/MC/RISCV/rv32c-invalid.s b/llvm/test/MC/RISCV/rv32c-invalid.s
index 8dddbf887c87..413573af1c5e 100644
--- a/llvm/test/MC/RISCV/rv32c-invalid.s
+++ b/llvm/test/MC/RISCV/rv32c-invalid.s
@@ -1,6 +1,6 @@
-# RUN: not llvm-mc -triple=riscv32 -mattr=+c -mattr=+no-rvc-hints < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+c < %s 2>&1 \
 # RUN:     | FileCheck %s
-# RUN: not llvm-mc -triple=riscv32 -mattr=+zca -mattr=+no-rvc-hints < %s 2>&1 \
+# RUN: not llvm-mc -triple=riscv32 -mattr=+zca < %s 2>&1 \
 # RUN:     | FileCheck %s
 
 ## GPRC
@@ -23,16 +23,10 @@ c.lwsp  x0, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding
 c.lwsp  zero, 4(sp) # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
 c.jr  x0 # CHECK: :[[@LINE]]:7: error: register must be a GPR excluding zero (x0)
 c.jalr  zero # CHECK: :[[@LINE]]:9: error: register must be a GPR excluding zero (x0)
-c.addi  x0, x0, 1 # CHECK: :[[@LINE]]:13: error: immediate must be zero
-c.li  zero, 2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
-c.slli  zero, zero, 4 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
-c.mv  zero, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
 c.mv  ra, x0 # CHECK: :[[@LINE]]:11: error: register must be a GPR excluding zero (x0)
 c.add  ra, ra, x0 # CHECK: :[[@LINE]]:16: error: invalid operand for instruction
-c.add  zero, zero, sp # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
 
 ## GPRNoX0X2
-c.lui x0, 4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
 c.lui x2, 4 # CHECK: :[[@LINE]]:7: error: register must be a GPR excluding zero (x0) and sp (x2){{$}}
 
 ## SP
@@ -57,7 +51,6 @@ c.andi a0, %lo(foo) # CHECK: :[[@LINE]]:12: error: immediate must be an integer
 c.andi a0, %hi(foo) # CHECK: :[[@LINE]]:12: error: immediate must be an integer in the range [-32, 31]
 
 ## simm6nonzero
-c.addi t0, 0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RVC Hint Instructions{{$}}
 c.addi t0, -33 # CHECK: :[[@LINE]]:12: error: immediate must be non-zero in the range [-32, 31]
 c.addi t0, 32 # CHECK: :[[@LINE]]:12: error: immediate must be non-zero in the range [-32, 31]
 c.addi t0, foo # CHECK: :[[@LINE]]:12: error: immediate must be non-zero in the range [-32, 31]
diff --git a/llvm/test/MC/RISCV/rv32p-invalid.s b/llvm/test/MC/RISCV/rv32p-invalid.s
new file mode 100644
index 000000000000..da3c67b73d0a
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32p-invalid.s
@@ -0,0 +1,19 @@
+# RUN: not llvm-mc -triple=riscv32 --mattr=+experimental-p %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+# Imm overflow
+pli.h a0, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
+pli.b a0, 0x200
+# CHECK-ERROR: immediate must be an integer in the range [0, 255]
+
+pslli.b a6, a7, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+psslai.h t0, t1, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+sslai a4, a5, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/rv32p-valid.s b/llvm/test/MC/RISCV/rv32p-valid.s
index e2e366593f9b..c755acddc712 100644
--- a/llvm/test/MC/RISCV/rv32p-valid.s
+++ b/llvm/test/MC/RISCV/rv32p-valid.s
@@ -1,36 +1,79 @@
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-p -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-p < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r --no-print-imm-hex - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ,CHECK-OBJ %s
 
-# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
-# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
-sh1add a0, a1, a2
 # CHECK-ASM-AND-OBJ: clz a0, a1
 # CHECK-ASM: encoding: [0x13,0x95,0x05,0x60]
 clz a0, a1
+# CHECK-ASM-AND-OBJ: cls a1, a2
+# CHECK-ASM: encoding: [0x93,0x15,0x36,0x60]
+cls a1, a2
 # CHECK-ASM-AND-OBJ: sext.b a2, a3
 # CHECK-ASM: encoding: [0x13,0x96,0x46,0x60]
 sext.b a2, a3
 # CHECK-ASM-AND-OBJ: sext.h t0, t1
 # CHECK-ASM: encoding: [0x93,0x12,0x53,0x60]
 sext.h t0, t1
+# CHECK-ASM-AND-OBJ: abs a4, a5
+# CHECK-ASM: encoding: [0x13,0x97,0x77,0x60]
+abs a4, a5
+# CHECK-ASM-AND-OBJ: rev8 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x69]
+rev8 s0, s1
+# CHECK-ASM-AND-OBJ: rev s2, s3
+# CHECK-ASM: encoding: [0x13,0xd9,0xf9,0x69]
+rev s2, s3
+# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
+# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
+sh1add a0, a1, a2
+# CHECK-ASM-AND-OBJ: pack s0, s1, s2
+# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
+pack s0, s1, s2
 # CHECK-ASM-AND-OBJ: min t0, t1, t2
 # CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
 min t0, t1, t2
-# CHECK-ASM-AND-OBJ: minu t0, t1, t2
-# CHECK-ASM: encoding: [0xb3,0x52,0x73,0x0a]
-minu t0, t1, t2
+# CHECK-ASM-AND-OBJ: minu ra, sp, gp
+# CHECK-ASM: encoding: [0xb3,0x50,0x31,0x0a]
+minu ra, sp, gp
 # CHECK-ASM-AND-OBJ: max t3, t4, t5
 # CHECK-ASM: encoding: [0x33,0xee,0xee,0x0b]
 max t3, t4, t5
 # CHECK-ASM-AND-OBJ: maxu a4, a5, a6
 # CHECK-ASM: encoding: [0x33,0xf7,0x07,0x0b]
 maxu a4, a5, a6
-# CHECK-ASM-AND-OBJ: pack s0, s1, s2
-# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
-pack s0, s1, s2
-# CHECK-ASM-AND-OBJ: rev8 s0, s1
-# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x69]
-rev8 s0, s1
+# CHECK-ASM-AND-OBJ: pslli.b a6, a7, 0
+# CHECK-ASM: encoding: [0x1b,0xa8,0x88,0x80]
+pslli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: pslli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x20,0x11,0x81]
+pslli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: psslai.h t0, t1, 2
+# CHECK-ASM: encoding: [0x9b,0x22,0x23,0xd1]
+psslai.h t0, t1, 2
+# CHECK-ASM-AND-OBJ: sslai a4, a5, 3
+# CHECK-ASM: encoding: [0x1b,0xa7,0x37,0xd2]
+sslai a4, a5, 3
+# CHECK-ASM-AND-OBJ: pli.h a5, 16
+# CHECK-ASM: encoding: [0x9b,0x27,0x10,0xb0]
+pli.h a5, 16
+# CHECK-ASM-AND-OBJ: pli.b a6, 16
+# CHECK-ASM: encoding: [0x1b,0x28,0x10,0xb4]
+pli.b a6, 16
+# CHECK-ASM-AND-OBJ: psext.h.b a7, a0
+# CHECK-ASM: encoding: [0x9b,0x28,0x45,0xe0]
+psext.h.b a7, a0
+# CHECK-ASM-AND-OBJ: psabs.h a1, a2
+# CHECK-ASM: encoding: [0x9b,0x25,0x76,0xe0]
+psabs.h a1, a2
+# CHECK-ASM-AND-OBJ: psabs.b t0, t1
+# CHECK-ASM: encoding: [0x9b,0x22,0x73,0xe4]
+psabs.b t0, t1
+# CHECK-ASM-AND-OBJ: plui.h gp, 32
+# CHECK-ASM: encoding: [0x9b,0x21,0x20,0xf0]
+plui.h gp, 32
+# CHECK-OBJ: plui.h gp, -412
+# CHECK-ASM: plui.h gp, 612
+# CHECK-ASM: encoding: [0x9b,0xa1,0x64,0xf0]
+plui.h gp, 612
diff --git a/llvm/test/MC/RISCV/rv64p-invalid.s b/llvm/test/MC/RISCV/rv64p-invalid.s
new file mode 100644
index 000000000000..572a099fba4d
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv64p-invalid.s
@@ -0,0 +1,21 @@
+# RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-p %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+# Imm overflow
+pli.h a0, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400
+# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
+pli.w a1, -0x201
+# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
+
+pslli.b a6, a7, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+pslli.w ra, sp, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+psslai.h t0, t1, 100
+# CHECK-ERROR: immediate must be an integer in the range [0, 15]
+psslai.w a4, a5, -1
+# CHECK-ERROR: error: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/rv64p-valid.s b/llvm/test/MC/RISCV/rv64p-valid.s
index 1ec5ee312e10..6c48ad3469b4 100644
--- a/llvm/test/MC/RISCV/rv64p-valid.s
+++ b/llvm/test/MC/RISCV/rv64p-valid.s
@@ -1,39 +1,110 @@
 # RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-p -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-p < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-p -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj --triple=riscv64 -mattr=+experimental-p < %s \
+# RUN:     | llvm-objdump --triple=riscv64 --mattr=+experimental-p -M no-aliases --no-print-imm-hex -d -r - \
+# RUN:     | FileCheck --check-prefixes=CHECK-ASM-AND-OBJ,CHECK-OBJ %s
 
-# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
-# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
-sh1add a0, a1, a2
 # CHECK-ASM-AND-OBJ: clz a0, a1
 # CHECK-ASM: encoding: [0x13,0x95,0x05,0x60]
 clz a0, a1
-# CHECK-ASM-AND-OBJ: clzw s0, s1
-# CHECK-ASM: encoding: [0x1b,0x94,0x04,0x60]
-clzw s0, s1
+# CHECK-ASM-AND-OBJ: cls a1, a2
+# CHECK-ASM: encoding: [0x93,0x15,0x36,0x60]
+cls a1, a2
 # CHECK-ASM-AND-OBJ: sext.b a2, a3
 # CHECK-ASM: encoding: [0x13,0x96,0x46,0x60]
 sext.b a2, a3
 # CHECK-ASM-AND-OBJ: sext.h t0, t1
 # CHECK-ASM: encoding: [0x93,0x12,0x53,0x60]
 sext.h t0, t1
+# CHECK-ASM-AND-OBJ: abs a4, a5
+# CHECK-ASM: encoding: [0x13,0x97,0x77,0x60]
+abs a4, a5
+# CHECK-ASM-AND-OBJ: rev16 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x04,0x6b]
+rev16 s0, s1
+# CHECK-ASM-AND-OBJ: rev8 s0, s1
+# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x6b]
+rev8 s0, s1
+# CHECK-ASM-AND-OBJ: rev s2, s3
+# CHECK-ASM: encoding: [0x13,0xd9,0xf9,0x6b]
+rev s2, s3
+# CHECK-ASM-AND-OBJ: clzw s0, s1
+# CHECK-ASM: encoding: [0x1b,0x94,0x04,0x60]
+clzw s0, s1
+# CHECK-ASM-AND-OBJ: clsw s2, s3
+# CHECK-ASM: encoding: [0x1b,0x99,0x39,0x60]
+clsw s2, s3
+# CHECK-ASM-AND-OBJ: absw s2, s3
+# CHECK-ASM: encoding: [0x1b,0x99,0x79,0x60]
+absw s2, s3
+# CHECK-ASM-AND-OBJ: sh1add a0, a1, a2
+# CHECK-ASM: encoding: [0x33,0xa5,0xc5,0x20]
+sh1add a0, a1, a2
+# CHECK-ASM-AND-OBJ: pack s0, s1, s2
+# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
+pack s0, s1, s2
 # CHECK-ASM-AND-OBJ: min t0, t1, t2
 # CHECK-ASM: encoding: [0xb3,0x42,0x73,0x0a]
 min t0, t1, t2
-# CHECK-ASM-AND-OBJ: minu t0, t1, t2
-# CHECK-ASM: encoding: [0xb3,0x52,0x73,0x0a]
-minu t0, t1, t2
+# CHECK-ASM-AND-OBJ: minu ra, sp, gp
+# CHECK-ASM: encoding: [0xb3,0x50,0x31,0x0a]
+minu ra, sp, gp
 # CHECK-ASM-AND-OBJ: max t3, t4, t5
 # CHECK-ASM: encoding: [0x33,0xee,0xee,0x0b]
 max t3, t4, t5
 # CHECK-ASM-AND-OBJ: maxu a4, a5, a6
 # CHECK-ASM: encoding: [0x33,0xf7,0x07,0x0b]
 maxu a4, a5, a6
-# CHECK-ASM-AND-OBJ: pack s0, s1, s2
-# CHECK-ASM: encoding: [0x33,0xc4,0x24,0x09]
-pack s0, s1, s2
-# CHECK-ASM-AND-OBJ: rev8 s0, s1
-# CHECK-ASM: encoding: [0x13,0xd4,0x84,0x6b]
-rev8 s0, s1
+# CHECK-ASM-AND-OBJ: pslli.b a6, a7
+# CHECK-ASM: encoding: [0x1b,0xa8,0x88,0x80]
+pslli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: pslli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x20,0x11,0x81]
+pslli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: pslli.w ra, sp, 2
+# CHECK-ASM: encoding: [0x9b,0x20,0x21,0x82]
+pslli.w ra, sp, 2
+# CHECK-ASM-AND-OBJ: psslai.h t0, t1, 3
+# CHECK-ASM: encoding: [0x9b,0x22,0x33,0xd1]
+psslai.h t0, t1, 3
+# CHECK-ASM-AND-OBJ: psslai.w a4, a5, 4
+# CHECK-ASM: encoding: [0x1b,0xa7,0x47,0xd2]
+psslai.w a4, a5, 4
+# CHECK-ASM-AND-OBJ: pli.h a5, 5
+# CHECK-ASM: encoding: [0x9b,0x27,0x05,0xb0]
+pli.h a5, 5
+# CHECK-ASM-AND-OBJ: pli.w a5, 5
+# CHECK-ASM: encoding: [0x9b,0x27,0x05,0xb2]
+pli.w a5, 5
+# CHECK-ASM-AND-OBJ: pli.b a6, 6
+# CHECK-ASM: encoding: [0x1b,0x28,0x06,0xb4]
+pli.b a6, 6
+# CHECK-ASM-AND-OBJ: psext.h.b t3, a2
+# CHECK-ASM: encoding: [0x1b,0x2e,0x46,0xe0]
+psext.h.b t3, a2
+# CHECK-ASM-AND-OBJ: psext.w.b a2, s0
+# CHECK-ASM: encoding: [0x1b,0x26,0x44,0xe2]
+psext.w.b a2, s0
+# CHECK-ASM-AND-OBJ: psext.w.h t1, t3
+# CHECK-ASM: encoding: [0x1b,0x23,0x5e,0xe2]
+psext.w.h t1, t3
+# CHECK-ASM-AND-OBJ: psabs.h t1, t5
+# CHECK-ASM: encoding: [0x1b,0x23,0x7f,0xe0]
+psabs.h t1, t5
+# CHECK-ASM-AND-OBJ: psabs.b a0, s2
+# CHECK-ASM: encoding: [0x1b,0x25,0x79,0xe4]
+psabs.b a0, s2
+# CHECK-ASM-AND-OBJ: plui.h s2, 4
+# CHECK-ASM: encoding: [0x1b,0x29,0x04,0xf0]
+plui.h s2, 4
+# CHECK-OBJ: plui.h gp, -412
+# CHECK-ASM: plui.h gp, 612
+# CHECK-ASM: encoding: [0x9b,0xa1,0x64,0xf0]
+plui.h gp, 612
+# CHECK-ASM-AND-OBJ: plui.w a2, 1
+# CHECK-ASM: encoding: [0x1b,0x26,0x01,0xf2]
+plui.w a2, 1
+# CHECK-OBJ: plui.w a2, -1
+# CHECK-ASM: plui.w a2, 1023
+# CHECK-ASM: encoding: [0x1b,0xa6,0xff,0xf3]
+plui.w a2, 1023
diff --git a/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s b/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
index 5168c42da70d..faa0e29f13f8 100644
--- a/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
+++ b/llvm/test/MC/RISCV/rvv/xsfvfwmacc.s
@@ -11,11 +11,11 @@
 sf.vfwmacc.4x4x4 v8, v4, v20
 # CHECK-INST: sf.vfwmacc.4x4x4 v8, v4, v20
 # CHECK-ENCODING: [0x5b,0x14,0x42,0xf3]
-# CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))
+# CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4))
 # CHECK-UNKNOWN: f342145b <unknown>
 
 sf.vfwmacc.4x4x4 v0, v4, v20
 # CHECK-INST: sf.vfwmacc.4x4x4 v0, v4, v20
 # CHECK-ENCODING: [0x5b,0x10,0x42,0xf3]
-# CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction and 4-by-4))
+# CHECK-ERROR: instruction requires the following: 'XSfvfwmaccqqq' (SiFive Matrix Multiply Accumulate Instruction (4-by-4))
 # CHECK-UNKNOWN: f342105b <unknown>
diff --git a/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
index e52f8129129d..2f799977f764 100644
--- a/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
+++ b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
@@ -1,7 +1,7 @@
 # RUN: not llvm-mc -triple riscv32 -filetype obj -mattr=+xandesperf < %s -o /dev/null 2>&1 | FileCheck %s
 
-  nds.bbc t0, 7, far_distant # CHECK: :[[@LINE]]:3: error: fixup value out of range
-  nds.bbc t0, 7, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+  nds.bbc t0, 7, far_distant # CHECK: :[[@LINE]]:18: error: fixup value out of range
+  nds.bbc t0, 7, unaligned # CHECK: :[[@LINE]]:18: error: fixup value must be 2-byte aligned
 
   .byte 0
 unaligned:
diff --git a/llvm/test/MC/RISCV/xandesvsintload-valid.s b/llvm/test/MC/RISCV/xandesvsintload-valid.s
new file mode 100644
index 000000000000..19efba86a610
--- /dev/null
+++ b/llvm/test/MC/RISCV/xandesvsintload-valid.s
@@ -0,0 +1,39 @@
+# XAndesVSIntLoad - Andes Vector INT4 Load Extension
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+xandesvsintload -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+xandesvsintload < %s \
+# RUN:     | llvm-objdump --mattr=+xandesvsintload -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ %s
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xandesvsintload -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+xandesvsintload < %s \
+# RUN:     | llvm-objdump --mattr=+xandesvsintload -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ %s
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+
+# CHECK-OBJ: nds.vln8.v v8, (a0)
+# CHECK-ASM: nds.vln8.v v8, (a0)
+# CHECK-ASM: encoding: [0x5b,0x44,0x25,0x06]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vln8.v v8, (a0)
+
+# CHECK-OBJ: nds.vln8.v v8, (a0), v0.t
+# CHECK-ASM: nds.vln8.v v8, (a0), v0.t
+# CHECK-ASM: encoding: [0x5b,0x44,0x25,0x04]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vln8.v v8, (a0), v0.t
+
+# CHECK-OBJ: nds.vlnu8.v v8, (a0)
+# CHECK-ASM: nds.vlnu8.v v8, (a0)
+# CHECK-ASM: encoding: [0x5b,0x44,0x35,0x06]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vlnu8.v v8, (a0)
+
+# CHECK-OBJ: nds.vlnu8.v v8, (a0), v0.t
+# CHECK-ASM: nds.vlnu8.v v8, (a0), v0.t
+# CHECK-ASM: encoding: [0x5b,0x44,0x35,0x04]
+# CHECK-ERROR: instruction requires the following: 'XAndesVSIntLoad' (Andes Vector INT4 Load Extension){{$}}
+nds.vlnu8.v v8, (a0), v0.t
diff --git a/llvm/test/MC/RISCV/xmips-invalid.s b/llvm/test/MC/RISCV/xmips-invalid.s
index b3834e7b3407..a17d2e347f8d 100644
--- a/llvm/test/MC/RISCV/xmips-invalid.s
+++ b/llvm/test/MC/RISCV/xmips-invalid.s
@@ -1,5 +1,14 @@
 # RUN: not llvm-mc -triple=riscv64 < %s 2>&1 | FileCheck %s -check-prefixes=CHECK-FEATURE
-# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+Xmipscbop < %s 2>&1 | FileCheck %s
+
+mips.pref   8, 512(a0)
+# CHECK: error: immediate offset must be in the range [0, 511]
+
+mips.pref	8
+# CHECK: error: too few operands for instruction
+
+mips.pref	8, 511(a0)
+# CHECK-FEATURE: error: instruction requires the following: 'Xmipscbop' (MIPS hardware prefetch)
 
 mips.ccmov x0, x1, 0x10
 # CHECK: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/xmips-valid.s b/llvm/test/MC/RISCV/xmips-valid.s
index 9f31e4fa2038..c5755ee81b49 100644
--- a/llvm/test/MC/RISCV/xmips-valid.s
+++ b/llvm/test/MC/RISCV/xmips-valid.s
@@ -1,9 +1,21 @@
-# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmov -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+xmipscbop -M no-aliases -show-encoding \
 # RUN:   | FileCheck -check-prefixes=CHECK-INST,CHECK-ENC %s
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmov < %s \
-# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmov -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+xmipslsp,+xmipscmov,+xmipscbop < %s \
+# RUN:   | llvm-objdump --mattr=+xmipslsp,+xmipscmov,+xmipscbop -M no-aliases -d - \
 # RUN:   | FileCheck -check-prefix=CHECK-DIS %s
 
+# CHECK-INST: mips.pref 8, 511(a0)
+# CHECK-ENC:  encoding: [0x0b,0x04,0xf5,0x1f]
+mips.pref	8, 511(a0)
+
+# CHECK-DIS: mips.pref  0x8, 0x1ff(a0)
+
+# CHECK-INST: mips.pref 9, 0(a0)
+# CHECK-ENC:  encoding: [0x8b,0x04,0x05,0x00]
+mips.pref	9, 0(a0)
+
+# CHECK-DIS: mips.pref  0x9, 0x0(a0)
+
 # CHECK-INST: mips.ccmov	s0, s1, s2, s3
 # CHECK-ENC:  encoding: [0x0b,0x34,0x99,0x9e]
 mips.ccmov s0, s1, s2, s3
diff --git a/llvm/test/MC/RISCV/xqcicm-valid.s b/llvm/test/MC/RISCV/xqcicm-valid.s
index 0eb253ff5e43..a9ce30e15048 100644
--- a/llvm/test/MC/RISCV/xqcicm-valid.s
+++ b/llvm/test/MC/RISCV/xqcicm-valid.s
@@ -135,3 +135,9 @@ qc.mveqi x9, x9, 0, x12
 # CHECK-ENC: encoding: [0x06,0xae]
 qc.mvltui x9, x9, 1, x12
 
+# Following instruction should not be compressed
+
+# CHECK-INST: qc.mveqi a0, s1, 0, a2
+# CHECK-ENC: encoding: [0x5b,0x85,0x04,0x64]
+qc.mveqi x10, x9, 0, x12
+
diff --git a/llvm/test/MC/VE/data-reloc-error.s b/llvm/test/MC/VE/data-reloc-error.s
index e312ac4e02d1..8c03a796e312 100644
--- a/llvm/test/MC/VE/data-reloc-error.s
+++ b/llvm/test/MC/VE/data-reloc-error.s
@@ -15,12 +15,12 @@ a:
 
 # CHECK:      data-reloc-error.s:10:7: error: 1-byte data relocation is not supported
 # CHECK-NEXT: .byte _GLOBAL_OFFSET_TABLE_
-# CHECK:      data-reloc-error.s:11:7: error: 1-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:11:29: error: 1-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .byte _GLOBAL_OFFSET_TABLE_ - .
 # CHECK:      data-reloc-error.s:12:8: error: 2-byte data relocation is not supported
 # CHECK-NEXT: .2byte _GLOBAL_OFFSET_TABLE_
-# CHECK:      data-reloc-error.s:13:8: error: 2-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:13:30: error: 2-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .2byte _GLOBAL_OFFSET_TABLE_ - .
-# CHECK:      data-reloc-error.s:14:8: error: 8-byte pc-relative data relocation is not supported
+# CHECK:      data-reloc-error.s:14:30: error: 8-byte pc-relative data relocation is not supported
 # CHECK-NEXT: .8byte _GLOBAL_OFFSET_TABLE_ - .
 
diff --git a/llvm/test/MC/X86/Relocations/x86-16.s b/llvm/test/MC/X86/Relocations/x86-16.s
new file mode 100644
index 000000000000..f8aaa6a9bf62
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-16.s
@@ -0,0 +1,13 @@
+# RUN: llvm-mc -filetype=obj -triple=i386-unknown-code16 %s -o %t
+# RUN: llvm-objdump -dr --triple=i386-unknown-code16 %t | FileCheck %s
+
+# CHECK:       e9 03 00                      jmp     {{.*}} <foo>
+# CHECK-NEXT:  e9 00 00                      jmp     {{.*}} <foo>
+# CHECK-LABEL: <foo>:
+# CHECK-NEXT:  0f 84 fc ff                   je      {{.*}} <foo>
+# CHECK-NEXT:  0f 84 f8 ff                   je      {{.*}} <foo>
+{disp32} jmp foo
+jmp.d32 foo
+foo:
+{disp32} je foo
+je.d32 foo
diff --git a/llvm/test/MC/X86/Relocations/x86-32.s b/llvm/test/MC/X86/Relocations/x86-32.s
new file mode 100644
index 000000000000..f2b256517cca
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-32.s
@@ -0,0 +1,17 @@
+# RUN: llvm-mc -filetype=obj -triple=i686 %s -o %t
+# RUN: llvm-objdump -dr --no-addresses %t | sed 's/#.*//;/^ *$/d' | FileCheck %s
+
+# CHECK:       e9 05 00 00 00                jmp     {{.*}} <foo>
+# CHECK-NEXT:  e9 00 00 00 00                jmp     {{.*}} <foo>
+# CHECK-LABEL: <foo>:
+# CHECK-NEXT:  0f 84 fa ff ff ff             je      {{.*}} <foo>
+# CHECK-NEXT:  0f 84 f4 ff ff ff             je      {{.*}} <foo>
+{disp32} jmp foo
+jmp.d32 foo
+foo:
+{disp32} je foo
+je.d32 foo
+
+# CHECK-NEXT: c1 0b 0a                               rorl    $0xa, (%ebx)
+# CHECK-NEXT:                R_386_8 .text
+                rorl    $foo, (%ebx)
diff --git a/llvm/test/MC/X86/Relocations/x86-64.s b/llvm/test/MC/X86/Relocations/x86-64.s
new file mode 100644
index 000000000000..402e6ef9356a
--- /dev/null
+++ b/llvm/test/MC/X86/Relocations/x86-64.s
@@ -0,0 +1,102 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -dr --no-addresses %t | sed 's/#.*//;/^ *$/d' | FileCheck %s
+
+# CHECK:      8b 05 00 00 00 00                      movl    (%rip), %eax
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+# CHECK-NEXT: c6 05 00 00 00 00 0c                   movb    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: 66 c7 05 00 00 00 00 0c 00             movw    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x6
+# CHECK-NEXT: c7 05 00 00 00 00 0c 00 00 00          movl    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 48 c7 05 00 00 00 00 0c 00 00 00       movq    $0xc, (%rip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 67 8b 05 00 00 00 00                   movl    (%eip), %eax
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+# CHECK-NEXT: 67 c6 05 00 00 00 00 0c                movb    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: 67 66 c7 05 00 00 00 00 0c 00          movw    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x6
+# CHECK-NEXT: 67 c7 05 00 00 00 00 0c 00 00 00       movl    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+# CHECK-NEXT: 67 48 c7 05 00 00 00 00 0c 00 00 00    movq    $0xc, (%eip)
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x8
+movl	foo(%rip), %eax
+movb	$12, foo(%rip)
+movw	$12, foo(%rip)
+movl	$12, foo(%rip)
+movq	$12, foo(%rip)
+
+movl	foo(%eip), %eax
+movb	$12, foo(%eip)
+movw	$12, foo(%eip)
+movl	$12, foo(%eip)
+movq	$12, foo(%eip)
+
+# CHECK-NEXT: 48 8b 05 00 00 00 00                   movq    (%rip), %rax
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 4c 8b 35 00 00 00 00                   movq    (%rip), %r14
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 67 48 8b 05 00 00 00 00                movq    (%eip), %rax
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+# CHECK-NEXT: 67 4c 8b 35 00 00 00 00                movq    (%eip), %r14
+# CHECK-NEXT:                R_X86_64_REX_GOTPCRELX  foo-0x4
+movq foo@GOTPCREL(%rip), %rax
+movq foo@GOTPCREL(%rip), %r14
+movq foo@GOTPCREL(%eip), %rax
+movq foo@GOTPCREL(%eip), %r14
+
+# CHECK-NEXT: 66 0f 38 00 0d 00 00 00 00             pshufb  (%rip), %xmm1
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x4
+pshufb foo(%rip), %xmm1
+
+## PR15040
+# CHECK-NEXT: c4 e3 f9 6a 05 00 00 00 00 10          vfmaddss        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 6a 05 00 00 00 00 10          vfmaddss        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 6b 05 00 00 00 00 10          vfmaddsd        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 6b 05 00 00 00 00 10          vfmaddsd        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 68 05 00 00 00 00 10          vfmaddps        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 68 05 00 00 00 00 10          vfmaddps        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 f9 69 05 00 00 00 00 10          vfmaddpd        (%rip), %xmm1, %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 79 69 05 00 00 00 00 10          vfmaddpd        %xmm1, (%rip), %xmm0, %xmm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 fd 68 05 00 00 00 00 10          vfmaddps        (%rip), %ymm1, %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 7d 68 05 00 00 00 00 10          vfmaddps        %ymm1, (%rip), %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 fd 69 05 00 00 00 00 10          vfmaddpd        (%rip), %ymm1, %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+# CHECK-NEXT: c4 e3 7d 69 05 00 00 00 00 10          vfmaddpd        %ymm1, (%rip), %ymm0, %ymm0
+# CHECK-NEXT:                R_X86_64_PC32   foo-0x5
+vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddss   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddsd   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddps   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
+vfmaddpd   %xmm1, foo(%rip),%xmm0, %xmm0
+vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
+vfmaddps   %ymm1, foo(%rip),%ymm0, %ymm0
+vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
+vfmaddpd   %ymm1, foo(%rip),%ymm0, %ymm0
+
+# CHECK-NEXT:<l1>:
+# CHECK-NEXT: 90                                     nop
+# CHECK-NEXT: c4 e2 79 00 05 f6 ff ff ff             vpshufb -0xa(%rip), %xmm0, %xmm0
+# CHECK-NEXT:<l2>:
+# CHECK-NEXT: 90                                     nop
+# CHECK-NEXT: c4 e3 7d 4a 05 f5 ff ff ff 10          vblendvps       %ymm1, -0xb(%rip), %ymm0, %ymm0
+l1:
+  nop
+  vpshufb l1(%rip), %xmm0, %xmm0
+l2:
+  nop
+  vblendvps %ymm1, l2(%rip), %ymm0, %ymm0
diff --git a/llvm/test/MC/X86/avx-64-att.s b/llvm/test/MC/X86/avx-64-att.s
index 39ee048c3736..f29142549bf5 100644
--- a/llvm/test/MC/X86/avx-64-att.s
+++ b/llvm/test/MC/X86/avx-64-att.s
@@ -4116,19 +4116,6 @@ vdivpd  -4(%rcx,%rbx,8), %xmm10, %xmm11
 // CHECK: encoding: [0xc4,0xa1,0x78,0x29,0x1c,0x18]
           vmovaps %xmm3, (%rax,%r11)
 
-// CHECK: vpshufb _foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe2,0x79,0x00,0x05,A,A,A,A]
-// CHECK: kind: reloc_riprel_4byte
-_foo:
-  nop
-  vpshufb _foo(%rip), %xmm0, %xmm0
-
-// CHECK: vblendvps %ymm1, _foo2(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x4a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: _foo2-5
-_foo2:
-  nop
-  vblendvps %ymm1, _foo2(%rip), %ymm0, %ymm0
 
 // CHECK: vgatherdpd %xmm0, (%rdi,%xmm1,2), %xmm2
 // CHECK: encoding: [0xc4,0xe2,0xf9,0x92,0x14,0x4f]
diff --git a/llvm/test/MC/X86/fma4-att.s b/llvm/test/MC/X86/fma4-att.s
index c9bd954e9049..bdd45ad1717d 100644
--- a/llvm/test/MC/X86/fma4-att.s
+++ b/llvm/test/MC/X86/fma4-att.s
@@ -77,66 +77,6 @@
 // CHECK: encoding: [0xc4,0xe3,0xfd,0x69,0xc2,0x10]
           vfmaddpd   %ymm2, %ymm1, %ymm0, %ymm0
 
-// PR15040
-// CHECK: vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x6a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddss  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddss   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x6a,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddss   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x6b,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddsd  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddsd   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x6b,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddsd   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddps   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0xf9,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd  foo(%rip), %xmm1, %xmm0, %xmm0
-
-// CHECK: vfmaddpd   %xmm1, foo(%rip), %xmm0, %xmm0
-// CHECK: encoding: [0xc4,0xe3,0x79,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd   %xmm1, foo(%rip),%xmm0, %xmm0
-
-// CHECK: vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0xfd,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps  foo(%rip), %ymm1, %ymm0, %ymm0
-
-// CHECK: vfmaddps   %ymm1, foo(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x68,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddps   %ymm1, foo(%rip),%ymm0, %ymm0
-
-// CHECK: vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0xfd,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd  foo(%rip), %ymm1, %ymm0, %ymm0
-
-// CHECK: vfmaddpd   %ymm1, foo(%rip), %ymm0, %ymm0
-// CHECK: encoding: [0xc4,0xe3,0x7d,0x69,0x05,A,A,A,A,0x10]
-// CHECK: fixup A - offset: 5, value: foo-5, kind: reloc_riprel_4byte
-          vfmaddpd   %ymm1, foo(%rip),%ymm0, %ymm0
 
 // vfmsub
 // CHECK: vfmsubss  (%rcx), %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/MC/X86/gotpcrel-non-globals.ll b/llvm/test/MC/X86/gotpcrel-non-globals.ll
new file mode 100644
index 000000000000..222d2d73ff72
--- /dev/null
+++ b/llvm/test/MC/X86/gotpcrel-non-globals.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; Check that we emit the `@bar_*` symbols, and that we don't emit multiple symbols.
+
+; CHECK-LABEL: .Lrel_0:
+; CHECK: .long   foo_0@GOTPCREL+0
+; CHECK-LABEL: .Lrel_1_failed:
+; CHECK: .long   bar_1-foo_0
+; CHECK-LABEL: .Lrel_2:
+; CHECK: .long   foo_2@GOTPCREL+0
+
+; CHECK: bar_0:
+; CHECK: bar_1:
+; CHECK: bar_2_indirect:
+
+@rel_0 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @bar_0 to i64), i64 ptrtoint (ptr @rel_0 to i64)) to i32)]
+@rel_1_failed = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @bar_1 to i64), i64 ptrtoint (ptr @foo_0 to i64)) to i32)]
+@rel_2 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @bar_2_indirect to i64), i64 ptrtoint (ptr @rel_2 to i64)) to i32)]
+@bar_0 = internal unnamed_addr constant ptr @foo_0, align 8
+@bar_1 = internal unnamed_addr constant ptr @foo_1, align 8
+@bar_2_indirect = internal unnamed_addr constant ptr @foo_2, align 8
+@foo_0 = external global ptr, align 8
+@foo_1 = external global ptr, align 8
+@foo_2 = external global ptr, align 8
+
+define void @foo(ptr %arg0, ptr %arg1) {
+  store ptr @bar_0, ptr %arg0, align 8
+  store ptr @bar_1, ptr %arg1, align 8
+  store ptr getelementptr (i8, ptr @bar_2_indirect, i32 1), ptr %arg1, align 8
+  ret void
+}
diff --git a/llvm/test/MC/X86/macho-reloc-errors-x86.s b/llvm/test/MC/X86/macho-reloc-errors-x86.s
index 4af202220073..16c656dc472d 100644
--- a/llvm/test/MC/X86/macho-reloc-errors-x86.s
+++ b/llvm/test/MC/X86/macho-reloc-errors-x86.s
@@ -10,6 +10,6 @@ defined:
         .section __DATA,__tim2
 later:
 
-// CHECK-ERROR: 3:9: error: symbol 'thing' can not be undefined in a subtraction expression
-// CHECK-ERROR: 4:9: error: symbol 'thing2' can not be undefined in a subtraction expression
-// CHECK-ERROR: 5:9: error: Section too large, can't encode r_address (0x100000b) into 24 bits of scattered relocation entry.
+// CHECK-ERROR: 3:24: error: symbol 'thing' can not be undefined in a subtraction expression
+// CHECK-ERROR: 4:26: error: symbol 'thing2' can not be undefined in a subtraction expression
+// CHECK-ERROR: 5:24: error: Section too large, can't encode r_address (0x100000b) into 24 bits of scattered relocation entry.
diff --git a/llvm/test/MC/X86/macho-reloc-errors-x86_64.s b/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
index daf84a7dbdc4..040b66656dd2 100644
--- a/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
+++ b/llvm/test/MC/X86/macho-reloc-errors-x86_64.s
@@ -9,11 +9,11 @@
         jmp thing@PLT
         mov %rax, thing@TLVP
 
-// CHECK-ERROR: 3:9: error: 32-bit absolute addressing is not supported in 64-bit mode
-// CHECK-ERROR: 4:9: error: expected relocatable expression
-// CHECK-ERROR: 5:9: error: unsupported pc-relative relocation of difference
-// CHECK-ERROR: 6:9: error: unsupported relocation with identical base
-// CHECK-ERROR: 7:9: error: unsupported relocation with subtraction expression, symbol 'thing' can not be undefined in a subtraction expression
-// CHECK-ERROR: 8:9: error: unsupported symbol modifier in relocation
-// CHECK-ERROR: 9:9: error: unsupported symbol modifier in branch relocation
-// CHECK-ERROR: 10:9: error: TLVP symbol modifier should have been rip-rel
+// CHECK-ERROR: 3:19: error: 32-bit absolute addressing is not supported in 64-bit mode
+// CHECK-ERROR: 4:28: error: expected relocatable expression
+// CHECK-ERROR: 5:25: error: unsupported pc-relative relocation of difference
+// CHECK-ERROR: 6:24: error: unsupported relocation with identical base
+// CHECK-ERROR: 7:24: error: unsupported relocation with subtraction expression, symbol 'thing' can not be undefined in a subtraction expression
+// CHECK-ERROR: 8:19: error: unsupported symbol modifier in relocation
+// CHECK-ERROR: 9:13: error: unsupported symbol modifier in branch relocation
+// CHECK-ERROR: 10:19: error: TLVP symbol modifier should have been rip-rel
diff --git a/llvm/test/MC/X86/pltoff.s b/llvm/test/MC/X86/pltoff.s
index a30ee6ccb098..2f3551b6a837 100644
--- a/llvm/test/MC/X86/pltoff.s
+++ b/llvm/test/MC/X86/pltoff.s
@@ -14,6 +14,6 @@ movabsq $puts@PLTOFF, %rax
 movabsq $.text@PLTOFF, %rax
 
 .ifdef ERR
-# ERR: {{.*}}.s:[[#@LINE+1]]:1: error: 64 bit reloc applied to a field with a different size
+# ERR: {{.*}}.s:[[#@LINE+1]]:7: error: 64 bit reloc applied to a field with a different size
 movl $puts@PLTOFF, %eax
 .endif
diff --git a/llvm/test/MC/X86/register-assignment.s b/llvm/test/MC/X86/register-assignment.s
index 4c7b07df050b..043f010efda2 100644
--- a/llvm/test/MC/X86/register-assignment.s
+++ b/llvm/test/MC/X86/register-assignment.s
@@ -1,12 +1,14 @@
 // RUN: llvm-mc -triple x86_64-unknown-unknown %s -o -      | FileCheck %s
 // RUN: not llvm-mc -triple x86_64 --defsym ERR=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
 	
-// CHECK-NOT: .set var_xdata
+// CHECK: var_xdata = %rcx
 var_xdata = %rcx
 
 // CHECK: xorq %rcx, %rcx
 xorq var_xdata, var_xdata
 
+unused_reg = %xmm0
+
 // CHECK: .data
 // CHECK-NEXT: .byte 1	
 .data 
diff --git a/llvm/test/MC/X86/unused_reg_var_assign.s b/llvm/test/MC/X86/unused_reg_var_assign.s
deleted file mode 100644
index 53e5cd6a2747..000000000000
--- a/llvm/test/MC/X86/unused_reg_var_assign.s
+++ /dev/null
@@ -1,3 +0,0 @@
-# RUN: llvm-mc -triple x86_64-unknown-linux-gnu -filetype obj %s -o %t
-	
-reg_i = %xmm0
diff --git a/llvm/test/MC/X86/x86-16.s b/llvm/test/MC/X86/x86-16.s
index be0563e5b1d6..b0a4bda56fcb 100644
--- a/llvm/test/MC/X86/x86-16.s
+++ b/llvm/test/MC/X86/x86-16.s
@@ -1056,17 +1056,6 @@ xsusldtrk
 // CHECK: encoding: [0xf2,0x0f,0x01,0xe9]
 xresldtrk
 
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-2, kind: FK_PCRel_2
-{disp32} jmp foo
-foo:
-
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-2, kind: FK_PCRel_2
-{disp32} je foo
-
 // CHECK: movl nearer, %ebx
 // CHECK:  encoding: [0x66,0x8b,0x1e,A,A]
 // CHECK:  fixup A - offset: 3, value: nearer, kind: FK_Data_2
diff --git a/llvm/test/MC/X86/x86-32-coverage.s b/llvm/test/MC/X86/x86-32-coverage.s
index 5475946a9d21..d585b4bc0e95 100644
--- a/llvm/test/MC/X86/x86-32-coverage.s
+++ b/llvm/test/MC/X86/x86-32-coverage.s
@@ -1421,12 +1421,6 @@
 // CHECK:  encoding: [0xd1,0x0d,0x78,0x56,0x34,0x12]
         	rorl	0x12345678
 
-// CHECK: rorl  $foo, (%ebx)
-// INTEL: ror dword ptr [ebx], foo
-// CHECK:  encoding: [0xc1,0x0b,A]
-// CHECK:    fixup A - offset: 2, value: foo, kind: FK_Data_1
-                rorl    $foo, (%ebx)
-
 // CHECK: shll	$0, 3735928559(%ebx,%ecx,8)
 // CHECK:  encoding: [0xc1,0xa4,0xcb,0xef,0xbe,0xad,0xde,0x00]
         	sall	$0,0xdeadbeef(%ebx,%ecx,8)
diff --git a/llvm/test/MC/X86/x86-32.s b/llvm/test/MC/X86/x86-32.s
index e3d473a05e59..8bfcf82c0cf7 100644
--- a/llvm/test/MC/X86/x86-32.s
+++ b/llvm/test/MC/X86/x86-32.s
@@ -1,3 +1,4 @@
+/// See Relocations/x86-32.s for fixup/relocation tests.
 // RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
 
 	pause
@@ -1118,25 +1119,6 @@ ptwritel 0xdeadbeef(%ebx,%ecx,8)
 // CHECK:  encoding: [0xf3,0x0f,0xae,0xe0]
 ptwritel %eax
 
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-4, kind: FK_PCRel_4
-// CHECK: jmp foo
-// CHECK:  encoding: [0xe9,A,A,A,A]
-// CHECK:  fixup A - offset: 1, value: foo-4, kind: FK_PCRel_4
-{disp32} jmp foo
-jmp.d32 foo
-foo:
-
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
-// CHECK: je foo
-// CHECK:  encoding: [0x0f,0x84,A,A,A,A]
-// CHECK:  fixup A - offset: 2, value: foo-4, kind: FK_PCRel_4
-{disp32} je foo
-je.d32 foo
-
 // CHECK: ljmpl *%cs:305419896
 // CHECK:  encoding: [0x2e,0xff,0x2d,0x78,0x56,0x34,0x12]
 ljmp %cs:*0x12345678
diff --git a/llvm/test/MC/X86/x86-64.s b/llvm/test/MC/X86/x86-64.s
index 108d1220107e..911f4674bd2c 100644
--- a/llvm/test/MC/X86/x86-64.s
+++ b/llvm/test/MC/X86/x86-64.s
@@ -1,3 +1,4 @@
+/// See Relocations/x86-64.s for fixup/relocation tests.
 // RUN: llvm-mc -triple x86_64-unknown-unknown -show-encoding %s > %t 2> %t.err
 // RUN: FileCheck < %t %s
 // RUN: FileCheck --check-prefix=CHECK-STDERR < %t.err %s
@@ -597,86 +598,15 @@ fdivp %st, %st(1) // CHECK: encoding: [0xde,0xf1]
 fdivp %st(1), %st // CHECK: encoding: [0xde,0xf1]
 
 
-movl	foo(%rip), %eax
-// CHECK: movl	foo(%rip), %eax
-// CHECK: encoding: [0x8b,0x05,A,A,A,A]
-// CHECK: fixup A - offset: 2, value: foo-4, kind: reloc_riprel_4byte
-
-movb	$12, foo(%rip)
-// CHECK: movb	$12, foo(%rip)
-// CHECK: encoding: [0xc6,0x05,A,A,A,A,0x0c]
-// CHECK:    fixup A - offset: 2, value: foo-5, kind: reloc_riprel_4byte
-
-movw	$12, foo(%rip)
-// CHECK: movw	$12, foo(%rip)
-// CHECK: encoding: [0x66,0xc7,0x05,A,A,A,A,0x0c,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-6, kind: reloc_riprel_4byte
-
-movl	$12, foo(%rip)
-// CHECK: movl	$12, foo(%rip)
-// CHECK: encoding: [0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 2, value: foo-8, kind: reloc_riprel_4byte
-
 // rdar://37247000
 movl	$12, 1024(%rip)
 // CHECK: movl	$12, 1024(%rip)
 // CHECK: encoding: [0xc7,0x05,0x00,0x04,0x00,0x00,0x0c,0x00,0x00,0x00]
 
-movq	$12, foo(%rip)
-// CHECK:  movq	$12, foo(%rip)
-// CHECK: encoding: [0x48,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-8, kind: reloc_riprel_4byte
-
-movl	foo(%eip), %eax
-// CHECK: movl	foo(%eip), %eax
-// CHECK: encoding: [0x67,0x8b,0x05,A,A,A,A]
-// CHECK: fixup A - offset: 3, value: foo-4, kind: reloc_riprel_4byte
-
-movb	$12, foo(%eip)
-// CHECK: movb	$12, foo(%eip)
-// CHECK: encoding: [0x67,0xc6,0x05,A,A,A,A,0x0c]
-// CHECK:    fixup A - offset: 3, value: foo-5, kind: reloc_riprel_4byte
-
-movw	$12, foo(%eip)
-// CHECK: movw	$12, foo(%eip)
-// CHECK: encoding: [0x67,0x66,0xc7,0x05,A,A,A,A,0x0c,0x00]
-// CHECK:    fixup A - offset: 4, value: foo-6, kind: reloc_riprel_4byte
-
-movl	$12, foo(%eip)
-// CHECK: movl	$12, foo(%eip)
-// CHECK: encoding: [0x67,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 3, value: foo-8, kind: reloc_riprel_4byte
-
-movq	$12, foo(%eip)
-// CHECK:  movq	$12, foo(%eip)
-// CHECK: encoding: [0x67,0x48,0xc7,0x05,A,A,A,A,0x0c,0x00,0x00,0x00]
-// CHECK:    fixup A - offset: 4, value: foo-8, kind: reloc_riprel_4byte
-
 // CHECK: addq	$-424, %rax
 // CHECK: encoding: [0x48,0x05,0x58,0xfe,0xff,0xff]
 addq $-424, %rax
 
-
-// CHECK: movq	_foo@GOTPCREL(%rip), %rax
-// CHECK:  encoding: [0x48,0x8b,0x05,A,A,A,A]
-// CHECK:  fixup A - offset: 3, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%rip), %rax
-
-// CHECK: movq	_foo@GOTPCREL(%rip), %r14
-// CHECK:  encoding: [0x4c,0x8b,0x35,A,A,A,A]
-// CHECK:  fixup A - offset: 3, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%rip), %r14
-
-// CHECK: movq	_foo@GOTPCREL(%eip), %rax
-// CHECK:  encoding: [0x67,0x48,0x8b,0x05,A,A,A,A]
-// CHECK:  fixup A - offset: 4, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%eip), %rax
-
-// CHECK: movq	_foo@GOTPCREL(%eip), %r14
-// CHECK:  encoding: [0x67,0x4c,0x8b,0x35,A,A,A,A]
-// CHECK:  fixup A - offset: 4, value: _foo@GOTPCREL-4, kind: reloc_riprel_4byte_movq_load
-movq _foo@GOTPCREL(%eip), %r14
-
 // CHECK: movq	(%r13,%rax,8), %r13
 // CHECK:  encoding: [0x4d,0x8b,0x6c,0xc5,0x00]
 movq 0x00(%r13,%rax,8),%r13
diff --git a/llvm/test/MC/X86/x86_64-encoding.s b/llvm/test/MC/X86/x86_64-encoding.s
index 247b1dc656bc..557687e9f6f8 100644
--- a/llvm/test/MC/X86/x86_64-encoding.s
+++ b/llvm/test/MC/X86/x86_64-encoding.s
@@ -133,12 +133,6 @@ movq %mm1, %r8
 // CHECK:  encoding: [0x48,0x0f,0x7e,0xca]
 movq %mm1, %rdx
 
-// rdar://7840289
-// CHECK: pshufb	CPI1_0(%rip), %xmm1
-// CHECK:  encoding: [0x66,0x0f,0x38,0x00,0x0d,A,A,A,A]
-// CHECK:  fixup A - offset: 5, value: CPI1_0-4
-pshufb	CPI1_0(%rip), %xmm1
-
 // CHECK: sha1rnds4 $1, %xmm1, %xmm2
 // CHECK:   encoding: [0x0f,0x3a,0xcc,0xd1,0x01]
 sha1rnds4 $1, %xmm1, %xmm2
diff --git a/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s b/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
index 9a81ade9fe28..0904e992a8a3 100644
--- a/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
+++ b/llvm/test/MC/Xtensa/Relocations/fixups-diagnostics.s
@@ -2,13 +2,13 @@
 
   .align 4
 
-  beq a0, a1, LBL1 # CHECK: :[[@LINE]]:3: error: fixup value out of range
+  beq a0, a1, LBL1 # CHECK: :[[@LINE]]:15: error: fixup value out of range
 LBL0:
-  beqz a0, LBL2 # CHECK: :[[@LINE]]:3: error: fixup value out of range
+  beqz a0, LBL2 # CHECK: :[[@LINE]]:12: error: fixup value out of range
 
-  call0 LBL0 # CHECK: :[[@LINE]]:3: error: fixup value must be 4-byte aligned
+  call0 LBL0 # CHECK: :[[@LINE]]:9: error: fixup value must be 4-byte aligned
 
-  loop a3, LBL0 # CHECK: :[[@LINE]]:3: error: loop fixup value out of range
+  loop a3, LBL0 # CHECK: :[[@LINE]]:12: error: loop fixup value out of range
 
   .space 1<<8
 LBL1:
diff --git a/llvm/test/MC/Xtensa/dfpaccel.s b/llvm/test/MC/Xtensa/dfpaccel.s
new file mode 100644
index 000000000000..13e8c2865f40
--- /dev/null
+++ b/llvm/test/MC/Xtensa/dfpaccel.s
@@ -0,0 +1,53 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+dfpaccel \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur a3, f64r_lo
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur a3, 234
+
+# CHECK-INST: rur a3, f64r_lo
+# CHECK: encoding: [0xa0,0x3e,0xe3]
+rur.f64r_lo a3
+
+# CHECK-INST: wur a3, f64r_lo
+# CHECK: encoding: [0x30,0xea,0xf3]
+wur a3, f64r_lo
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur a3, f64r_hi
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur a3, 235
+
+# CHECK-INST: rur a3, f64r_hi
+# CHECK: encoding: [0xb0,0x3e,0xe3]
+rur.f64r_hi a3
+
+# CHECK-INST: wur a3, f64r_hi
+# CHECK: encoding: [0x30,0xeb,0xf3]
+wur a3, f64r_hi
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur a3, f64s
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur a3, 236
+
+# CHECK-INST: rur a3, f64s
+# CHECK: encoding: [0xc0,0x3e,0xe3]
+rur.f64s a3
+
+# CHECK-INST: wur a3, f64s
+# CHECK: encoding: [0x30,0xec,0xf3]
+wur a3, f64s
diff --git a/llvm/test/MC/Xtensa/threadptr.s b/llvm/test/MC/Xtensa/threadptr.s
new file mode 100644
index 000000000000..c29d1ea91ed8
--- /dev/null
+++ b/llvm/test/MC/Xtensa/threadptr.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc %s -triple=xtensa -show-encoding --mattr=+threadptr \
+# RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+
+.align	4
+LBL0:
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur a3, threadptr
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur a3, 231
+
+# CHECK-INST: rur a3, threadptr
+# CHECK: encoding: [0x70,0x3e,0xe3]
+rur.threadptr a3
+
+# CHECK-INST: wur a3, threadptr
+# CHECK: encoding: [0x30,0xe7,0xf3]
+wur a3, threadptr
diff --git a/llvm/test/Other/fatlto.ll b/llvm/test/Other/fatlto.ll
new file mode 100644
index 000000000000..2f7ce4bf817a
--- /dev/null
+++ b/llvm/test/Other/fatlto.ll
@@ -0,0 +1,7 @@
+; RUN: opt -debug-pass-manager -passes='fatlto-pre-link<O2>' -verify-analysis-invalidation=0 -disable-output %s 2>&1 | FileCheck %s
+; RUN: opt -debug-pass-manager -passes='fatlto-pre-link<O2;thinlto>' -verify-analysis-invalidation=0 -disable-output %s 2>&1 | FileCheck %s --check-prefixes=CHECK,THINLTO
+
+; CHECK: Running pass: EmbedBitcodePass on [module]
+; THINLTO: Running analysis: ModuleSummaryIndexAnalysis on [module]
+; CHECK: Running pass: FatLtoCleanup on [module]
+; CHECK-NEXT: Running pass: LowerTypeTestsPass on [module]
diff --git a/llvm/test/Other/pipeline-alias-errors.ll b/llvm/test/Other/pipeline-alias-errors.ll
new file mode 100644
index 000000000000..de7ef8da7e2c
--- /dev/null
+++ b/llvm/test/Other/pipeline-alias-errors.ll
@@ -0,0 +1,20 @@
+; RUN: not opt -passes="default" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="default<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="default-post-link" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="default-post-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="thinlto-pre-link" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="thinlto-pre-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="thinlto" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="thinlto<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="lto-pre-link" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="lto-pre-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="lto" < %s 2>&1 | FileCheck %s --check-prefix=MISSING-OPT-LEVEL
+; RUN: not opt -passes="lto<foo>" < %s 2>&1 | FileCheck %s --check-prefix=INVALID-OPT-LEVEL
+; RUN: not opt -passes="fatlto-pre-link" < %s 2>&1 | FileCheck %s --check-prefix=FATLTO-MISSING-OPT-LEVEL
+; RUN: not opt -passes="fatlto-pre-link<foo>" < %s 2>&1 | FileCheck %s --check-prefix=FATLTO-INVALID-PARAMS
+
+; MISSING-OPT-LEVEL: invalid optimization level ''
+; INVALID-OPT-LEVEL: invalid optimization level 'foo'
+
+; FATLTO-MISSING-OPT-LEVEL: missing optimization level for fatlto-pre-link pipeline
+; FATLTO-INVALID-PARAMS: invalid fatlto-pre-link pass parameter 'foo'
diff --git a/llvm/test/TableGen/CompressWriteLatencyEntry.td b/llvm/test/TableGen/CompressWriteLatencyEntry.td
index 88273e885844..d6a9f0ac0dd7 100644
--- a/llvm/test/TableGen/CompressWriteLatencyEntry.td
+++ b/llvm/test/TableGen/CompressWriteLatencyEntry.td
@@ -33,10 +33,10 @@ def Read_D : SchedRead;
 // CHECK-NEXT: }; // MyTargetReadAdvanceTable
 
 // CHECK:  static const llvm::MCSchedClassDesc SchedModel_ASchedClasses[] = {
-// CHECK-NEXT:  {DBGFIELD("InvalidSchedClass")  8191, false, false, false, 0, 0,  0, 0,  0, 0},
-// CHECK-NEXT:  {DBGFIELD("Inst_A")             1, false, false, false,  0, 0,  1, 1,  0, 0}, // #1
-// CHECK-NEXT:  {DBGFIELD("Inst_B")             1, false, false, false,  0, 0,  2, 1,  0, 0}, // #2
-// CHECK-NEXT:  {DBGFIELD("Inst_C")             1, false, false, false,  0, 0,  1, 1,  1, 1}, // #3
+// CHECK-NEXT:  {DBGFIELD(1)  8191, false, false, false, 0, 0,  0, 0,  0, 0},
+// CHECK-NEXT:  {DBGFIELD(/*Inst_A*/ {{[0-9]+}})             1, false, false, false,  0, 0,  1, 1,  0, 0}, // #1
+// CHECK-NEXT:  {DBGFIELD(/*Inst_B*/ {{[0-9]+}})             1, false, false, false,  0, 0,  2, 1,  0, 0}, // #2
+// CHECK-NEXT:  {DBGFIELD(/*Inst_C*/ {{[0-9]+}})             1, false, false, false,  0, 0,  1, 1,  1, 1}, // #3
 // CHECK-NEXT: }; // SchedModel_ASchedClasses
 
 let SchedModel = SchedModel_A in {
diff --git a/llvm/test/TableGen/DecoderEmitterFnTable.td b/llvm/test/TableGen/DecoderEmitterFnTable.td
new file mode 100644
index 000000000000..7bed18c19a51
--- /dev/null
+++ b/llvm/test/TableGen/DecoderEmitterFnTable.td
@@ -0,0 +1,84 @@
+// RUN: llvm-tblgen -gen-disassembler -use-fn-table-in-decode-to-mcinst -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+  let InstructionSet = archInstrInfo;
+}
+
+let Namespace = "arch" in {
+  def R0 : Register<"r0">;
+  def R1 : Register<"r1">;
+  def R2 : Register<"r2">;
+  def R3 : Register<"r3">;
+}
+def Regs : RegisterClass<"Regs", [i32], 32, (add R0, R1, R2, R3)>;
+
+class TestInstruction : Instruction {
+  let Size = 1;
+  let OutOperandList = (outs);
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+}
+
+// Define instructions to generate 4 cases in decodeToMCInst.
+// Lower 2 bits define the number of operands. Each register operand
+// needs 2 bits to encode.
+
+// An instruction with no inputs. Encoded with lower 2 bits = 0 and upper
+// 6 bits = 0 as well.
+def Inst0 : TestInstruction {
+  let Inst = 0x0;
+  let InOperandList = (ins);
+  let AsmString = "Inst0";
+}
+
+// An instruction with a single input. Encoded with lower 2 bits = 1 and the
+// single input in bits 2-3.
+def Inst1 : TestInstruction {
+  bits<2> r0;
+  let Inst{1-0} = 1;
+  let Inst{3-2} = r0;
+  let InOperandList = (ins Regs:$r0);
+  let AsmString = "Inst1";
+}
+
+// An instruction with two inputs. Encoded with lower 2 bits = 2 and the
+// inputs in bits 2-3 and 4-5.
+def Inst2 : TestInstruction {
+  bits<2> r0;
+  bits<2> r1;
+  let Inst{1-0} = 2;
+  let Inst{3-2} = r0;
+  let Inst{5-4} = r1;
+  let InOperandList = (ins Regs:$r0, Regs:$r1);
+  let AsmString = "Inst2";
+}
+
+// An instruction with three inputs. Encoded with lower 2 bits = 3 and the
+// inputs in bits 2-3 and 4-5 and 6-7.
+def Inst3 : TestInstruction {
+  bits<2> r0;
+  bits<2> r1;
+  bits<2> r2;
+  let Inst{1-0} = 3;
+  let Inst{3-2} = r0;
+  let Inst{5-4} = r1;
+  let Inst{7-6} = r2;
+  let InOperandList = (ins Regs:$r0, Regs:$r1, Regs:$r2);
+  let AsmString = "Inst3";
+}
+
+// CHECK-LABEL: DecodeStatus decodeFn0(DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const MCDisassembler *Decoder, bool &DecodeComplete)
+// CHECK-LABEL: DecodeStatus decodeFn1(DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const MCDisassembler *Decoder, bool &DecodeComplete)
+// CHECK-LABEL: DecodeStatus decodeFn2(DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const MCDisassembler *Decoder, bool &DecodeComplete)
+// CHECK-LABEL: DecodeStatus decodeFn3(DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const MCDisassembler *Decoder, bool &DecodeComplete)
+// CHECK-LABEL: decodeToMCInst(unsigned Idx, DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const MCDisassembler *Decoder, bool &DecodeComplete)
+// CHECK: static constexpr DecodeFnTy decodeFnTable[]
+// CHECK-NEXT: decodeFn0,
+// CHECK-NEXT: decodeFn1,
+// CHECK-NEXT: decodeFn2,
+// CHECK-NEXT: decodeFn3,
+// CHECK: return decodeFnTable[Idx](S, insn, MI, Address, Decoder, DecodeComplete)
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
index a8290a57e321..7399ef726d0e 100644
--- a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s -o - 2>%t
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -o - 2>%t
 // RUN: FileCheck %s < %t
 
 include "llvm/Target/Target.td"
diff --git a/llvm/test/TableGen/InvalidMCSchedClassDesc.td b/llvm/test/TableGen/InvalidMCSchedClassDesc.td
index de5392237a84..e43edd417458 100644
--- a/llvm/test/TableGen/InvalidMCSchedClassDesc.td
+++ b/llvm/test/TableGen/InvalidMCSchedClassDesc.td
@@ -1,13 +1,13 @@
 // RUN: llvm-tblgen -gen-subtarget -I %p/../../include %s 2>&1 | FileCheck %s
-// Check if it is valid MCSchedClassDesc if didn't have the resources. 
+// Check if it is valid MCSchedClassDesc if didn't have the resources.
 
 include "llvm/Target/Target.td"
 
 def MyTarget : Target;
 
 let OutOperandList = (outs), InOperandList = (ins) in {
-  def Inst_A : Instruction; 
-  def Inst_B : Instruction; 
+  def Inst_A : Instruction;
+  def Inst_B : Instruction;
 }
 
 let CompleteModel = 0 in {
@@ -18,8 +18,8 @@ let CompleteModel = 0 in {
 
 // Inst_B didn't have the resoures, and it is invalid.
 // CHECK: SchedModel_ASchedClasses[] = {
-// CHECK: {DBGFIELD("Inst_A")             1
-// CHECK-NEXT: {DBGFIELD("Inst_B")             8191
+// CHECK: {DBGFIELD(/*Inst_A*/ 19)             1
+// CHECK-NEXT: {DBGFIELD(/*Inst_B*/ 26)             8191
 let SchedModel = SchedModel_A in {
   def Write_A : SchedWriteRes<[]>;
   def : InstRW<[Write_A], (instrs Inst_A)>;
@@ -27,18 +27,18 @@ let SchedModel = SchedModel_A in {
 
 // Inst_A didn't have the resoures, and it is invalid.
 // CHECK: SchedModel_BSchedClasses[] = {
-// CHECK: {DBGFIELD("Inst_A")             8191
-// CHECK-NEXT: {DBGFIELD("Inst_B")             1 
+// CHECK: {DBGFIELD(/*Inst_A*/ 19)             8191
+// CHECK-NEXT: {DBGFIELD(/*Inst_B*/ 26)             1
 let SchedModel = SchedModel_B in {
-  def Write_B: SchedWriteRes<[]>; 
+  def Write_B: SchedWriteRes<[]>;
   def : InstRW<[Write_B], (instrs Inst_B)>;
 }
 
 // CHECK: SchedModel_CSchedClasses[] = {
-// CHECK: {DBGFIELD("Inst_A")             1
-// CHECK-NEXT: {DBGFIELD("Inst_B")             1
+// CHECK: {DBGFIELD(/*Inst_A*/ 19)             1
+// CHECK-NEXT: {DBGFIELD(/*Inst_B*/ 26)             1
 let SchedModel = SchedModel_C in {
-  def Write_C: SchedWriteRes<[]>; 
+  def Write_C: SchedWriteRes<[]>;
   def : InstRW<[Write_C], (instrs Inst_A, Inst_B)>;
 }
 
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter.td b/llvm/test/TableGen/RuntimeLibcallEmitter.td
new file mode 100644
index 000000000000..d4030dd0a7e9
--- /dev/null
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter.td
@@ -0,0 +1,77 @@
+// RUN: llvm-tblgen -gen-runtime-libcalls -I %p/../../include %s | FileCheck %s
+
+include "llvm/IR/RuntimeLibcallsImpl.td"
+
+
+def SHL_I32 : RuntimeLibcall;
+def SRL_I64 : RuntimeLibcall;
+
+def SQRT_F128 : RuntimeLibcall;
+def SQRT_F80 : RuntimeLibcall;
+def BZERO : RuntimeLibcall;
+
+// Test default names.
+let IsDefault = true in {
+  def __ashlsi3 : RuntimeLibcallImpl<SHL_I32>;
+  def __lshrdi3 : RuntimeLibcallImpl<SRL_I64>;
+
+  def sqrtl_f128 : RuntimeLibcallImpl<SQRT_F128, "sqrtl">;
+  def sqrtl_f80 : RuntimeLibcallImpl<SQRT_F80, "sqrtl">;
+}
+
+// Ignore non-default in initDefaultLibCallNames.
+def bzero : RuntimeLibcallImpl<BZERO>;
+
+// All entries should be emitted in Libcall enum.
+// CHECK: #ifdef GET_RUNTIME_LIBCALL_ENUM
+// CHECK-NEXT: namespace llvm {
+// CHECK-NEXT: namespace RTLIB {
+// CHECK-NEXT: enum Libcall : unsigned short {
+// CHECK-NEXT: BZERO = 0,
+// CHECK-NEXT: SHL_I32 = 1,
+// CHECK-NEXT: SQRT_F80 = 2,
+// CHECK-NEXT: SQRT_F128 = 3,
+// CHECK-NEXT: SRL_I64 = 4,
+// CHECK-NEXT: UNKNOWN_LIBCALL = 5
+// CHECK-NEXT: };
+// CHECK-EMPTY:
+// CHECK-NEXT:enum LibcallImpl : unsigned short {
+// CHECK-NEXT:  Unsupported = 0,
+// CHECK-NEXT:  __ashlsi3 = 1, // __ashlsi3
+// CHECK-NEXT:  __lshrdi3 = 2, // __lshrdi3
+// CHECK-NEXT:  bzero = 3, // bzero
+// CHECK-NEXT:  sqrtl_f80 = 4, // sqrtl
+// CHECK-NEXT:  sqrtl_f128 = 5, // sqrtl
+// CHECK-NEXT:  NumLibcallImpls = 6
+// CHECK-NEXT: };
+// CHECK-NEXT: } // End namespace RTLIB
+// CHECK-NEXT: } // End namespace llvm
+// CHECK-NEXT: #endif
+
+// CHECK: #ifdef GET_INIT_RUNTIME_LIBCALL_NAMES
+// CHECK-NEXT: const RTLIB::LibcallImpl llvm::RTLIB::RuntimeLibcallsInfo::DefaultLibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {
+// CHECK-NEXT:   RTLIB::Unsupported, // RTLIB::BZERO
+// CHECK-NEXT:   RTLIB::__ashlsi3, // RTLIB::SHL_I32
+// CHECK-NEXT:   RTLIB::sqrtl_f80, // RTLIB::SQRT_F80
+// CHECK-NEXT:   RTLIB::sqrtl_f128, // RTLIB::SQRT_F128
+// CHECK-NEXT:   RTLIB::__lshrdi3, // RTLIB::SRL_I64
+// CHECK-NEXT:   RTLIB::Unsupported
+// CHECK-NEXT: };
+// CHECK-EMPTY:
+// CHECK-NEXT: const char *const llvm::RTLIB::RuntimeLibcallsInfo::LibCallImplNames[RTLIB::NumLibcallImpls] = {
+// CHECK-NEXT: nullptr, // RTLIB::Unsupported
+// CHECK-NEXT: "__ashlsi3", // RTLIB::__ashlsi3
+// CHECK-NEXT: "__lshrdi3", // RTLIB::__lshrdi3
+// CHECK-NEXT: "bzero", // RTLIB::bzero
+// CHECK-NEXT: "sqrtl", // RTLIB::sqrtl_f80
+// CHECK-NEXT: "sqrtl", // RTLIB::sqrtl_f128
+// CHECK-NEXT: };
+
+// CHECK: const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::ImplToLibcall[RTLIB::NumLibcallImpls] = {
+// CHECK-NEXT: RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported
+// CHECK-NEXT: RTLIB::SHL_I32, // RTLIB::__ashlsi3
+// CHECK-NEXT: RTLIB::SRL_I64, // RTLIB::__lshrdi3
+// CHECK-NEXT: RTLIB::BZERO, // RTLIB::bzero
+// CHECK-NEXT: RTLIB::SQRT_F80, // RTLIB::sqrtl_f80
+// CHECK-NEXT: RTLIB::SQRT_F128, // RTLIB::sqrtl_f128
+// CHECK-NEXT: };
diff --git a/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll b/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
new file mode 100644
index 000000000000..d5aa6b10b5ad
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
@@ -0,0 +1,431 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=attributor %s -S | FileCheck %s --check-prefix=AMDGCN
+
+@G = addrspace(1) global i32 zeroinitializer, align 4
+declare void @clobber(i32) #0
+declare void @clobber.p5(ptr addrspace(5)) #0
+declare ptr addrspace(1) @get_ptr() #0
+declare noalias ptr addrspace(1) @get_noalias_ptr() #0
+declare noalias ptr addrspace(1) @get_untouched_ptr() #1
+
+define void @test_nonkernel(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define void @test_nonkernel(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7:[0-9]+]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be !invariant.load, as the caller may modify %ptr
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_plain(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_plain(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be !invariant.load, as %ptr may alias a pointer in @clobber
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_noalias_ptr(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_ptr(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0:![0-9]+]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_gep(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_gep(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
+  %val = load i32, ptr addrspace(1) %gep, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_noalias_gep(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_gep(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
+  %val = load i32, ptr addrspace(1) %gep, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_swap(ptr addrspace(1) noalias %ptr, i32 inreg %swap) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_swap(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]], i32 inreg [[SWAP:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    store i32 [[SWAP]], ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; cannot be !invariant.load due to the write to %ptr
+  store i32 %swap, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_volatile(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_volatile(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load volatile i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load volatile i32, ptr addrspace(1) %ptr, align 4
+  ;; volatiles loads cannot be !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_unordered(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_unordered(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load atomic i32, ptr addrspace(1) %ptr unordered, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_monotonic(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_monotonic(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] monotonic, align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load atomic i32, ptr addrspace(1) %ptr monotonic, align 4
+  ;; atomic loads with ordering guarantees may have side effects
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_global() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_global(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) @G, align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) @G, align 4
+  ;; is not an !invariant.load as global variables may change
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_alloca(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_alloca(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; AMDGCN-NEXT:    store i32 [[VAL]], ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    call void @clobber.p5(ptr addrspace(5) noundef align 4 [[ALLOCA]]) #[[ATTR7]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[LOAD]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  %alloca = alloca i32, addrspace(5)
+  store i32 %val, ptr addrspace(5) %alloca
+  %load = load i32, ptr addrspace(5) %alloca
+  call void @clobber.p5(ptr addrspace(5) %alloca)
+  call void @clobber(i32 %load)
+  ret void
+}
+
+define internal void @copy.i32(ptr addrspace(5) %alloca, i32 %qty) {
+; AMDGCN-LABEL: define internal void @copy.i32(
+; AMDGCN-SAME: ptr addrspace(5) noalias nofree noundef writeonly align 4 captures(none) dereferenceable_or_null(4) [[ALLOCA:%.*]], i32 [[QTY:%.*]]) #[[ATTR4:[0-9]+]] {
+; AMDGCN-NEXT:    store i32 [[QTY]], ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    ret void
+;
+  store i32 %qty, ptr addrspace(5) %alloca
+  ret void
+}
+
+define amdgpu_kernel void @test_internal_alloca(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_internal_alloca(
+; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
+; AMDGCN-NEXT:    call void @copy.i32(ptr addrspace(5) noalias nofree noundef writeonly align 4 captures(none) dereferenceable_or_null(4) [[ALLOCA]], i32 [[VAL]]) #[[ATTR8:[0-9]+]]
+; AMDGCN-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(5) [[ALLOCA]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[LOAD]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  %alloca = alloca i32, addrspace(5)
+  call void @copy.i32(ptr addrspace(5) %alloca, i32 %val)
+  %load = load i32, ptr addrspace(5) %alloca
+  call void @clobber(i32 %load)
+  ret void
+}
+
+define internal i32 @test_internal_noalias_load(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_noalias_load(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5:[0-9]+]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; is an !invariant.load due to its only caller @test_call_internal_noalias
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal_noalias(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_noalias(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_noalias_load(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9:[0-9]+]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_noalias_load(ptr addrspace(1) %ptr)
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define internal i32 @test_internal_load(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_load(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since the pointer in @test_call_internal may alias
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_load(ptr addrspace(1) nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_load(ptr addrspace(1) %ptr)
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define internal i32 @test_internal_written(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define internal i32 @test_internal_written(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR5]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    ret i32 [[VAL]]
+;
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; cannot be an !invariant.load because of the write in caller @test_call_internal_written
+  ret i32 %val
+}
+
+define amdgpu_kernel void @test_call_internal_written(ptr addrspace(1) noalias %ptr, i32 inreg %x) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_written(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree captures(none) [[PTR:%.*]], i32 inreg [[X:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_written(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR9]]
+; AMDGCN-NEXT:    store i32 [[X]], ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %val = call i32 @test_internal_written(ptr addrspace(1) %ptr)
+  store i32 %x, ptr addrspace(1) %ptr
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_ptr() #[[ATTR7]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since %ptr may alias
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_noalias_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_noalias_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_noalias_ptr() #[[ATTR7]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_noalias_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; may not be an !invariant.load since %ptr may have been written to before returning
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_call_untouched_ptr() {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_call_untouched_ptr(
+; AMDGCN-SAME: ) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = call noalias align 4 ptr addrspace(1) @get_untouched_ptr() #[[ATTR10:[0-9]+]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = call ptr addrspace(1) @get_untouched_ptr()
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_make_buffer(ptr addrspace(1) %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer(
+; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR11:[0-9]+]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
+  %val = load i32, ptr addrspace(7) %rsrc, align 4
+  ;; original %ptr may alias
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_make_buffer_noalias(ptr addrspace(1) noalias %ptr) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer_noalias(
+; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR11]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
+  %val = load i32, ptr addrspace(7) %rsrc, align 4
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_selected_load(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load(
+; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; either pointer yields an !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_selected_load_partial_noalias(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load_partial_noalias(
+; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; %ptr.false may alias, so no !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_branch_load(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load(
+; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:  [[ENTRY:.*:]]
+; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
+; AMDGCN:       [[TRUE]]:
+; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR7]]
+; AMDGCN-NEXT:    br label %[[FINISH:.*]]
+; AMDGCN:       [[FALSE]]:
+; AMDGCN-NEXT:    br label %[[FINISH]]
+; AMDGCN:       [[FINISH]]:
+; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %true, label %false
+true:
+  call void @clobber(i32 1)
+  br label %finish
+false:
+  br label %finish
+finish:
+  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; either pointer yields an !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+define amdgpu_kernel void @test_branch_load_partial_noalias(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
+; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load_partial_noalias(
+; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
+; AMDGCN-NEXT:  [[ENTRY:.*:]]
+; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
+; AMDGCN:       [[TRUE]]:
+; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR7]]
+; AMDGCN-NEXT:    br label %[[FINISH:.*]]
+; AMDGCN:       [[FALSE]]:
+; AMDGCN-NEXT:    br label %[[FINISH]]
+; AMDGCN:       [[FINISH]]:
+; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
+; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR7]]
+; AMDGCN-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %true, label %false
+true:
+  call void @clobber(i32 1)
+  br label %finish
+false:
+  br label %finish
+finish:
+  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
+  %val = load i32, ptr addrspace(1) %ptr, align 4
+  ;; ptr.false may alias, so no !invariant.load
+  call void @clobber(i32 %val)
+  ret void
+}
+
+attributes #0 = { nofree norecurse nosync nounwind willreturn }
+attributes #1 = { nofree norecurse nosync nounwind willreturn readonly }
+;.
+; AMDGCN: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
index b0446479dac4..6641fdb9b4ff 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/PR16052.ll
@@ -68,7 +68,7 @@ define i64 @fn2c() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[CONV:%.*]] = sext i32 undef to i64
 ; CGSCC-NEXT:    [[ADD:%.*]] = add i64 42, [[CONV]]
-; CGSCC-NEXT:    [[CALL2:%.*]] = call i64 @fn1(i64 [[ADD]]) #[[ATTR2]], !range [[RNG0:![0-9]+]]
+; CGSCC-NEXT:    [[CALL2:%.*]] = call range(i64 -2147483606, 2147483690) i64 @fn1(i64 [[ADD]]) #[[ATTR2]]
 ; CGSCC-NEXT:    ret i64 [[CALL2]]
 ;
 entry:
@@ -91,13 +91,11 @@ entry:
   ret i64 %cond
 }
 ;.
+; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+;.
 ; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR2]] = { nofree nosync willreturn }
 ;.
-; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
-;.
-; CGSCC: [[RNG0]] = !{i64 -2147483606, i64 2147483690}
-;.
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 07e2d5ea1575..5bff2a2e6b20 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -207,7 +207,6 @@ define void @f7_1(ptr %ptr, i1 %cnd) {
 ; CHECK-LABEL: define {{[^@]+}}@f7_1
 ; CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], i1 noundef [[CND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[A:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
-; CHECK-NEXT:    [[PTR_0:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[CND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
diff --git a/llvm/test/Transforms/Attributor/range.ll b/llvm/test/Transforms/Attributor/range.ll
index 1b27fc5f16f5..38f8a829cf41 100644
--- a/llvm/test/Transforms/Attributor/range.ll
+++ b/llvm/test/Transforms/Attributor/range.ll
@@ -19,7 +19,7 @@ define i32 @test0-range-check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-range-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[A:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3:[0-9]+]], !range [[RNG0]]
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    ret i32 [[A]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
@@ -32,6 +32,40 @@ define i32 @test0-range-check(ptr %p) {
   ret i32 %a
 }
 
+define i32 @test0-range-check-smaller-current-range-attr(ptr %p) {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
+; TUNIT-LABEL: define {{[^@]+}}@test0-range-check-smaller-current-range-attr
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 2, 5) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
+; TUNIT-NEXT:    ret i32 [[A]]
+;
+; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
+; CGSCC-LABEL: define {{[^@]+}}@test0-range-check-smaller-current-range-attr
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[A:%.*]] = tail call range(i32 2, 5) i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    ret i32 [[A]]
+;
+  %a = tail call range(i32 2, 5)  i32 @test0(ptr %p)
+  ret i32 %a
+}
+
+define i32 @test0-range-check-larger-current-range-attr(ptr %p) {
+; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
+; TUNIT-LABEL: define {{[^@]+}}@test0-range-check-larger-current-range-attr
+; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
+; TUNIT-NEXT:    [[A:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
+; TUNIT-NEXT:    ret i32 [[A]]
+;
+; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: read)
+; CGSCC-LABEL: define {{[^@]+}}@test0-range-check-larger-current-range-attr
+; CGSCC-SAME: (ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P:%.*]]) #[[ATTR1]] {
+; CGSCC-NEXT:    [[A:%.*]] = tail call range(i32 0, 100) i32 @test0(ptr nofree noundef nonnull readonly align 4 captures(none) dereferenceable(4) [[P]]) #[[ATTR5]]
+; CGSCC-NEXT:    ret i32 [[A]]
+;
+  %a = tail call range(i32 0, 100) i32 @test0(ptr %p)
+  ret i32 %a
+}
+
 declare void @use3-dummy(i1, i1, i1)
 define void @use3(i1, i1, i1) {
 ; CHECK-LABEL: define {{[^@]+}}@use3
@@ -48,7 +82,7 @@ define void @test0-icmp-check(ptr %p){
   ; ret = [0, 10)
 ; TUNIT-LABEL: define {{[^@]+}}@test0-icmp-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) {
-; TUNIT-NEXT:    [[RET:%.*]] = tail call i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG0]]
+; TUNIT-NEXT:    [[RET:%.*]] = tail call range(i32 0, 10) i32 @test0(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CMP_EQ_1:%.*]] = icmp eq i32 [[RET]], 10
 ; TUNIT-NEXT:    [[CMP_EQ_2:%.*]] = icmp eq i32 [[RET]], 9
 ; TUNIT-NEXT:    [[CMP_EQ_3:%.*]] = icmp eq i32 [[RET]], 8
@@ -284,7 +318,7 @@ define i1 @test1-check(ptr %p) {
 ; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
 ; TUNIT-LABEL: define {{[^@]+}}@test1-check
 ; TUNIT-SAME: (ptr nofree readonly align 4 captures(none) [[P:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT:    [[RES:%.*]] = tail call i32 @test1(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]], !range [[RNG2:![0-9]+]]
+; TUNIT-NEXT:    [[RES:%.*]] = tail call range(i32 200, 1091) i32 @test1(ptr nofree noundef readonly align 4 captures(none) [[P]]) #[[ATTR3]]
 ; TUNIT-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RES]], 500
 ; TUNIT-NEXT:    ret i1 [[CMP]]
 ;
@@ -624,7 +658,7 @@ define dso_local i32 @test4-g2(i32 %u) {
 ; TUNIT-LABEL: define {{[^@]+}}@test4-g2
 ; TUNIT-SAME: (i32 [[U:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
-; TUNIT-NEXT:    [[CALL:%.*]] = tail call i32 @test4-f2(i32 [[U]]) #[[ATTR4]], !range [[RNG3:![0-9]+]]
+; TUNIT-NEXT:    [[CALL:%.*]] = tail call range(i32 1, -2147483648) i32 @test4-f2(i32 [[U]]) #[[ATTR4]]
 ; TUNIT-NEXT:    ret i32 [[CALL]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -1760,6 +1794,7 @@ declare void @barney(i32 signext, i32 signext)
 
 !0 = !{i32 0, i32 10}
 !1 = !{i32 10, i32 100}
+!2 = !{i32 2, i32 5}
 ;.
 ; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
 ; TUNIT: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
@@ -1778,8 +1813,6 @@ declare void @barney(i32 signext, i32 signext)
 ;.
 ; TUNIT: [[RNG0]] = !{i32 0, i32 10}
 ; TUNIT: [[RNG1]] = !{i32 10, i32 100}
-; TUNIT: [[RNG2]] = !{i32 200, i32 1091}
-; TUNIT: [[RNG3]] = !{i32 1, i32 -2147483648}
 ;.
 ; CGSCC: [[RNG0]] = !{i32 0, i32 10}
 ; CGSCC: [[RNG1]] = !{i32 10, i32 100}
diff --git a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
index 374d5ba7ff52..75ab54059f30 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
@@ -342,14 +342,6 @@ define %S.2 @t3.helper() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[S_2:%.*]], align 8
 ; CHECK-NEXT:    call void @ext1(ptr noundef nonnull align 8 dereferenceable(24) [[RETVAL]])
-; CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load ptr, ptr [[RETVAL]], align 8
-; CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [[S_2]] poison, ptr [[DOTFCA_0_LOAD]], 0
-; CHECK-NEXT:    [[DOTFCA_1_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 1
-; CHECK-NEXT:    [[DOTFCA_1_LOAD:%.*]] = load i64, ptr [[DOTFCA_1_GEP]], align 8
-; CHECK-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_0_INSERT]], i64 [[DOTFCA_1_LOAD]], 1
-; CHECK-NEXT:    [[DOTFCA_2_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 2
-; CHECK-NEXT:    [[DOTFCA_2_LOAD:%.*]] = load i64, ptr [[DOTFCA_2_GEP]], align 8
-; CHECK-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_1_INSERT]], i64 [[DOTFCA_2_LOAD]], 2
 ; CHECK-NEXT:    ret [[S_2]] zeroinitializer
 ;
 entry:
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index 7c31951d900b..463ecb3003a8 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -1114,7 +1114,7 @@ define i32 @test(i1 %c) {
 ; TUNIT-LABEL: define {{[^@]+}}@test
 ; TUNIT-SAME: (i1 [[C:%.*]]) {
 ; TUNIT-NEXT:    [[R1:%.*]] = call i32 @ctx_test1(i1 noundef [[C]])
-; TUNIT-NEXT:    [[R2:%.*]] = call i32 @ctx_test2(i1 noundef [[C]]), !range [[RNG0:![0-9]+]]
+; TUNIT-NEXT:    [[R2:%.*]] = call range(i32 0, -2147483648) i32 @ctx_test2(i1 noundef [[C]])
 ; TUNIT-NEXT:    [[ADD:%.*]] = add i32 [[R1]], [[R2]]
 ; TUNIT-NEXT:    ret i32 [[ADD]]
 ;
@@ -1689,8 +1689,6 @@ define i32 @readExtInitZeroInit() {
 ; TUNIT: attributes #[[ATTR15]] = { nosync nounwind memory(read) }
 ; TUNIT: attributes #[[ATTR16]] = { nounwind memory(write) }
 ;.
-; TUNIT: [[RNG0]] = !{i32 0, i32 -2147483648}
-;.
 ; CGSCC: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn }
 ; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
 ; CGSCC: attributes #[[ATTR2]] = { memory(readwrite, argmem: none) }
diff --git a/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll b/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
index fcdc2d7c1c71..c1c4c12909a7 100644
--- a/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
+++ b/llvm/test/Transforms/CallSiteSplitting/callsite-split-preserve-debug.ll
@@ -1,4 +1,4 @@
-; RUN: opt -passes=callsite-splitting -S < %s | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt -passes=callsite-splitting -S < %s | FileCheck %s --implicit-check-not=dbg_value
 
 ;; Test that DebugLocs are preserved, and that dbg.values are duplicated.
 
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
index 6ef3400812fc..e27d5d772a7a 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
@@ -110,5 +110,71 @@ define void @splat_ptr_scatter(ptr %ptr, <4 x i1> %mask, <4 x i32> %val) {
   ret void
 }
 
+define <4 x i32> @scalar_prefix(ptr %base, i64 %index, <4 x i64> %vecidx) {
+; CHECK-LABEL: @scalar_prefix(
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [256 x i32], ptr [[BASE:%.*]], i64 [[INDEX:%.*]], <4 x i64> [[VECIDX:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP2]], i32 4, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %gep = getelementptr [256 x i32], ptr %base, i64 %index, <4 x i64> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_splat(ptr %base, i64 %index, <4 x i64> %vecidx) {
+; CHECK-LABEL: @scalar_prefix_with_splat(
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [256 x i32], ptr [[BASE:%.*]], <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[VECIDX:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP2]], i32 4, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %broadcast.splatinsert = insertelement <4 x i64> poison, i64 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i64> %broadcast.splat, <4 x i64> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_prefix_with_constant_splat(ptr %base, <4 x i64> %vecidx) {
+; CHECK-LABEL: @scalar_prefix_with_constant_splat(
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [256 x i32], ptr [[BASE:%.*]], <4 x i64> splat (i64 20), <4 x i64> [[VECIDX:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP2]], i32 4, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i64> splat (i64 20), <4 x i64> %vecidx
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate(ptr %base, i64 %index, <4 x i64> %vecidx) {
+; CHECK-LABEL: @reassociate(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [256 x i32], ptr [[BASE:%.*]], <4 x i64> [[VECIDX:%.*]], i64 [[INDEX:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[GEP]], i32 4, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i64> %vecidx, i64 %index
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @reassociate_with_splat(ptr %base, i64 %index, <4 x i64> %vecidx) {
+; CHECK-LABEL: @reassociate_with_splat(
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX:%.*]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [256 x i32], ptr [[BASE:%.*]], <4 x i64> [[VECIDX:%.*]], <4 x i64> [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[GEP]], i32 4, <4 x i1> splat (i1 true), <4 x i32> undef)
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %broadcast.splatinsert = insertelement <4 x i64> poison, i64 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> poison, <4 x i32> zeroinitializer
+
+  %gep = getelementptr [256 x i32], ptr %base, <4 x i64> %vecidx, <4 x i64> %broadcast.splat
+  %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+  ret <4 x i32> %res
+}
+
+
 declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)
 declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32, <4 x i1>)
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll
new file mode 100644
index 000000000000..d0d87b38e058
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addr-recreate.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -p 'require<profile-summary>,codegenprepare' -cgpp-huge-func=0 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+@globalptr = external global ptr
+declare ptr @get_ptr(i64)
+
+; Can't recreate invoke instruction
+
+define void @addr_from_invoke() personality ptr null {
+; CHECK-LABEL: define void @addr_from_invoke() personality ptr null {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[PTR:%.*]] = invoke ptr @get_ptr(i64 0)
+; CHECK-NEXT:            to label %[[BODY_1:.*]] unwind label %[[EHCLEANUP:.*]]
+; CHECK:       [[EHCLEANUP]]:
+; CHECK-NEXT:    [[PAD:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    cleanupret from [[PAD]] unwind to caller
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[GEP1:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP0]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr = invoke ptr @get_ptr(i64 0) to label %body.1 unwind label %ehcleanup
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+ehcleanup:
+  %pad = cleanuppad within none []
+  cleanupret from %pad unwind to caller
+
+body.1:
+  %gep = getelementptr { i32 }, ptr %ptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
+
+define void @addr_from_arg(ptr %ptr, i1 %p) {
+; CHECK-LABEL: define void @addr_from_arg(
+; CHECK-SAME: ptr [[PTR:%.*]], i1 [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[P]], label %[[BODY_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast ptr [[PTR]] to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr [[PTR]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %p, label %body.1, label %exit
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+exit:
+  ret void
+
+body.1:
+  %gep = getelementptr { i32 }, ptr %ptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
+
+define void @addr_from_global(i1 %p) {
+; CHECK-LABEL: define void @addr_from_global(
+; CHECK-SAME: i1 [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[P]], label %[[BODY_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[BODY_1]]:
+; CHECK-NEXT:    [[GEP1:%.*]] = bitcast ptr @globalptr to ptr
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[UNUSED:%.*]] = load <4 x i32>, ptr @globalptr, align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @globalptr, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %p, label %body.1, label %exit
+
+body.2:
+  %unused = load <4 x i32>, ptr %gep, align 4
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  ret void
+
+exit:
+  ret void
+
+body.1:
+  %gep = getelementptr { i32 }, ptr @globalptr, i64 0, i32 0
+  store <4 x i32> zeroinitializer, ptr %gep, align 4
+  br label %body.2
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
index 61107a9b7016..63fd1845594c 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-base.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=true -addr-sink-new-select=true -disable-cgp-delete-phis  %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-YES
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -addr-sink-new-phis=false -addr-sink-new-select=true -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NO
 target datalayout =
@@ -6,7 +7,31 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Can we sink for different base if there is no phi for base?
 define i32 @test1(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test1
+; CHECK-YES-LABEL: define i32 @test1(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test1(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -16,10 +41,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
   ret i32 %v
@@ -27,7 +48,18 @@ fallthrough:
 
 ; Can we sink for different base if there is phi for base?
 define i32 @test2(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test2
+; CHECK-LABEL: define i32 @test2(
+; CHECK-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -37,7 +69,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -46,7 +77,33 @@ fallthrough:
 
 ; Can we sink for different base if there is phi for base but not valid one?
 define i32 @test3(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test3
+; CHECK-YES-LABEL: define i32 @test3(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test3(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -56,11 +113,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %b = phi ptr [%b2, %entry], [%b1, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -69,7 +121,31 @@ fallthrough:
 
 ; Can we sink for different base if both addresses are in the same block?
 define i32 @test4(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test4
+; CHECK-YES-LABEL: define i32 @test4(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test4(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -79,10 +155,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
   ret i32 %v
@@ -91,7 +163,18 @@ fallthrough:
 ; Can we sink for different base if there is phi for base?
 ; Both addresses are in the same block.
 define i32 @test5(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test5
+; CHECK-LABEL: define i32 @test5(
+; CHECK-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -101,7 +184,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -111,7 +193,33 @@ fallthrough:
 ; Can we sink for different base if there is phi for base but not valid one?
 ; Both addresses are in the same block.
 define i32 @test6(i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test6
+; CHECK-YES-LABEL: define i32 @test6(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test6(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B2]], %[[ENTRY]] ], [ [[B1]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   %a2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -121,11 +229,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %b = phi ptr [%b2, %entry], [%b1, %if.then]
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %c, align 4
@@ -134,14 +237,56 @@ fallthrough:
 
 ; case with a loop. No phi node.
 define i32 @test7(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test7
+; CHECK-YES-LABEL: define i32 @test7(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[FALLTHROUGH]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test7(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
   %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
   %c3 = phi ptr [%a1, %entry], [%c, %fallthrough]
   br i1 %cond, label %if.then, label %fallthrough
@@ -151,10 +296,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %v = load volatile i32, ptr %c, align 4
   %iv.inc = add i32 %iv, 1
@@ -167,7 +308,30 @@ exit:
 
 ; case with a loop. There is phi node.
 define i32 @test8(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test8
+; CHECK-LABEL: define i32 @test8(
+; CHECK-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[B]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
@@ -183,7 +347,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: getelementptr inbounds i8, {{.+}} 40
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b = phi ptr [%b3, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -197,14 +360,60 @@ exit:
 
 ; case with a loop. There is phi node but it does not fit.
 define i32 @test9(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test9
+; CHECK-YES-LABEL: define i32 @test9(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[FALLTHROUGH]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[FALLTHROUGH]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[B:%.*]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test9(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[FALLTHROUGH:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    [[B3:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[FALLTHROUGH]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[B:%.*]] = phi ptr [ [[B3]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
   %iv = phi i32 [0, %entry], [%iv.inc, %fallthrough]
   %c3 = phi ptr [%a1, %entry], [%c, %fallthrough]
   %b3 = phi ptr [%b1, %entry], [%b2, %fallthrough]
@@ -215,11 +424,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b = phi ptr [%b3, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -233,7 +437,45 @@ exit:
 
 ; Case through a loop. No phi node.
 define i32 @test10(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test10
+; CHECK-YES-LABEL: define i32 @test10(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test10(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -243,10 +485,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: br
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   br label %loop
 
@@ -257,14 +495,31 @@ loop:
   br i1 %cmp, label %loop, label %exit
 
 exit:
-; CHECK-YES: sunkaddr
   %v = load volatile i32, ptr %c, align 4
   ret i32 %v
 }
 
 ; Case through a loop. There is a phi.
 define i32 @test11(i32 %N, i1 %cond, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test11
+; CHECK-LABEL: define i32 @test11(
+; CHECK-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[B:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[FALLTHROUGH]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br i1 %cond, label %if.then, label %fallthrough
@@ -274,9 +529,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK: phi
-; CHECK: phi
-; CHECK: br
   %c = phi ptr [%a1, %entry], [%a2, %if.then]
   %b = phi ptr [%b1, %entry], [%b2, %if.then]
   br label %loop
@@ -288,25 +540,80 @@ loop:
   br i1 %cmp, label %loop, label %exit
 
 exit:
-; CHECK: sunkaddr
   %v = load volatile i32, ptr %c, align 4
   ret i32 %v
 }
 
 ; Complex case with address value from previous iteration.
 define i32 @test12(i32 %N, i1 %cond, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test12
+; CHECK-YES-LABEL: define i32 @test12(
+; CHECK-YES-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-YES-NEXT:    br label %[[LOOP:.*]]
+; CHECK-YES:       [[LOOP]]:
+; CHECK-YES-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-YES-NEXT:    [[SUNK_PHI1:%.*]] = phi ptr [ [[SUNK_PHI:%.*]], %[[BACKEDGE]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[BACKEDGE]] ]
+; CHECK-YES-NEXT:    [[B4:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B5:%.*]], %[[BACKEDGE]] ]
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[SUNK_PHI1]], %[[LOOP]] ]
+; CHECK-YES-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[B6:%.*]] = phi ptr [ [[B4]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load volatile i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[A4:%.*]] = getelementptr inbounds i64, ptr [[B4]], i64 5
+; CHECK-YES-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 20
+; CHECK-YES-NEXT:    br i1 [[CMP]], label %[[BACKEDGE]], label %[[IF_THEN_2:.*]]
+; CHECK-YES:       [[IF_THEN_2]]:
+; CHECK-YES-NEXT:    br label %[[BACKEDGE]]
+; CHECK-YES:       [[BACKEDGE]]:
+; CHECK-YES-NEXT:    [[B5]] = phi ptr [ [[B4]], %[[FALLTHROUGH]] ], [ [[B6]], %[[IF_THEN_2]] ]
+; CHECK-YES-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-YES-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-YES-NEXT:    br i1 [[CMP2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-YES:       [[EXIT]]:
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test12(
+; CHECK-NO-SAME: i32 [[N:%.*]], i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[LOOP:.*]]
+; CHECK-NO:       [[LOOP]]:
+; CHECK-NO-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NO-NEXT:    [[C3:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[C:%.*]], %[[BACKEDGE]] ]
+; CHECK-NO-NEXT:    [[B4:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B5:%.*]], %[[BACKEDGE]] ]
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C]] = phi ptr [ [[C3]], %[[LOOP]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[B6:%.*]] = phi ptr [ [[B4]], %[[LOOP]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load volatile i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[A4:%.*]] = getelementptr inbounds i64, ptr [[B4]], i64 5
+; CHECK-NO-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 20
+; CHECK-NO-NEXT:    br i1 [[CMP]], label %[[BACKEDGE]], label %[[IF_THEN_2:.*]]
+; CHECK-NO:       [[IF_THEN_2]]:
+; CHECK-NO-NEXT:    br label %[[BACKEDGE]]
+; CHECK-NO:       [[BACKEDGE]]:
+; CHECK-NO-NEXT:    [[B5]] = phi ptr [ [[B4]], %[[FALLTHROUGH]] ], [ [[B6]], %[[IF_THEN_2]] ]
+; CHECK-NO-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NO-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[IV_INC]], [[N]]
+; CHECK-NO-NEXT:    br i1 [[CMP2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NO:       [[EXIT]]:
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds i64, ptr %b1, i64 5
   br label %loop
 
 loop:
-; CHECK-LABEL: loop:
-; CHECK-YES: sunk_phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: br
   %iv = phi i32 [0, %entry], [%iv.inc, %backedge]
   %c3 = phi ptr [%a1, %entry], [%c, %backedge]
   %b4 = phi ptr [%b1, %entry], [%b5, %backedge]
@@ -317,11 +624,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO: phi
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%c3, %loop], [%a2, %if.then]
   %b6 = phi ptr [%b4, %loop], [%b2, %if.then]
   %v = load volatile i32, ptr %c, align 4
@@ -345,7 +647,35 @@ exit:
 %struct.S = type {i32, i32}
 ; Case with index
 define i32 @test13(i1 %cond, ptr %b1, ptr %b2, i64 %Index) {
-; CHECK-LABEL: @test13
+; CHECK-YES-LABEL: define i32 @test13(
+; CHECK-YES-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], i64 [[INDEX:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    [[I2:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-YES-NEXT:    [[SUNKADDR1:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 [[SUNKADDR]]
+; CHECK-YES-NEXT:    [[SUNKADDR2:%.*]] = getelementptr inbounds i8, ptr [[SUNKADDR1]], i64 4
+; CHECK-YES-NEXT:    [[V:%.*]] = load i32, ptr [[SUNKADDR2]], align 4
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test13(
+; CHECK-NO-SAME: i1 [[COND:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], i64 [[INDEX:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[A1:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], ptr [[B1]], i64 [[INDEX]], i32 1
+; CHECK-NO-NEXT:    br i1 [[COND]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    [[I2:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NO-NEXT:    [[A2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[B2]], i64 [[INDEX]], i32 1
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[A:%.*]] = phi ptr [ [[A1]], %[[ENTRY]] ], [ [[A2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %a1 = getelementptr inbounds %struct.S, ptr %b1, i64 %Index, i32 1
   br i1 %cond, label %if.then, label %fallthrough
@@ -356,10 +686,6 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO-NEXT: phi
-; CHECK-NO-NEXT: load
   %a = phi ptr [%a1, %entry], [%a2, %if.then]
   %v = load i32, ptr %a, align 4
   ret i32 %v
@@ -367,22 +693,55 @@ fallthrough:
 
 ; Select of Select case.
 define i64 @test14(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test14
+; CHECK-LABEL: define i64 @test14(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[S12:%.*]] = select i1 [[C1]], ptr [[B1]], ptr [[B2]]
+; CHECK-NEXT:    [[S21:%.*]] = select i1 [[C2]], ptr [[S12]], ptr [[B3]]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S21]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-NEXT:    ret i64 [[V]]
+;
 entry:
-; CHECK-LABEL: entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
   %g3 = getelementptr inbounds i64, ptr %b3, i64 5
   %s1 = select i1 %c1, ptr %g1, ptr %g2
   %s2 = select i1 %c2, ptr %s1, ptr %g3
-; CHECK: sunkaddr
   %v = load i64 , ptr %s2, align 8
   ret i64 %v
 }
 
 ; Select of Phi case.
 define i64 @test15(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test15
+; CHECK-YES-LABEL: define i64 @test15(
+; CHECK-YES-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN]] ], [ [[B1]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[SUNK_PHI]], ptr [[B3]]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S11]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-YES-NEXT:    ret i64 [[V]]
+;
+; CHECK-NO-LABEL: define i64 @test15(
+; CHECK-NO-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr [[B3]], i64 5
+; CHECK-NO-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[P1:%.*]] = phi ptr [ [[G1]], %[[ENTRY]] ], [ [[G2]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[S1:%.*]] = select i1 [[C2]], ptr [[P1]], ptr [[G3]]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i64, ptr [[S1]], align 8
+; CHECK-NO-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -393,20 +752,27 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p1 = phi ptr [%g1, %entry], [%g2, %if.then]
   %s1 = select i1 %c2, ptr %p1, ptr %g3
-; CHECK-YES: sunkaddr
-; CHECK-NO: phi
-; CHECK-NO-NEXT: select
-; CHECK-NO-NEXT: load
   %v = load i64 , ptr %s1, align 8
   ret i64 %v
 }
 
 ; Select of Phi case. Phi exists
 define i64 @test16(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test16
+; CHECK-LABEL: define i64 @test16(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi ptr [ [[B1]], %[[ENTRY]] ], [ [[B2]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[P]], ptr [[B3]]
+; CHECK-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[S11]], i64 40
+; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -417,18 +783,43 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p = phi ptr [%b1, %entry], [%b2, %if.then]
   %p1 = phi ptr [%g1, %entry], [%g2, %if.then]
   %s1 = select i1 %c2, ptr %p1, ptr %g3
-; CHECK: sunkaddr
   %v = load i64 , ptr %s1, align 8
   ret i64 %v
 }
 
 ; Phi of Select case.
 define i64 @test17(i1 %c1, i1 %c2, ptr %b1, ptr %b2, ptr %b3) {
-; CHECK-LABEL: @test17
+; CHECK-YES-LABEL: define i64 @test17(
+; CHECK-YES-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*]]:
+; CHECK-YES-NEXT:    [[S11:%.*]] = select i1 [[C2]], ptr [[B1]], ptr [[B2]]
+; CHECK-YES-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B3]], %[[IF_THEN]] ], [ [[S11]], %[[ENTRY]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V:%.*]] = load i64, ptr [[SUNKADDR]], align 8
+; CHECK-YES-NEXT:    ret i64 [[V]]
+;
+; CHECK-NO-LABEL: define i64 @test17(
+; CHECK-NO-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]], ptr [[B3:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*]]:
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr [[B3]], i64 5
+; CHECK-NO-NEXT:    [[S1:%.*]] = select i1 [[C2]], ptr [[G1]], ptr [[G2]]
+; CHECK-NO-NEXT:    br i1 [[C1]], label %[[IF_THEN:.*]], label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[P1:%.*]] = phi ptr [ [[S1]], %[[ENTRY]] ], [ [[G3]], %[[IF_THEN]] ]
+; CHECK-NO-NEXT:    [[V:%.*]] = load i64, ptr [[P1]], align 8
+; CHECK-NO-NEXT:    ret i64 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b1, i64 5
   %g2 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -440,18 +831,48 @@ if.then:
   br label %fallthrough
 
 fallthrough:
-; CHECK-LABEL: fallthrough:
   %p1 = phi ptr [%s1, %entry], [%g3, %if.then]
-; CHECK-YES: sunkaddr
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %v = load i64 , ptr %p1, align 8
   ret i64 %v
 }
 
 ; The same two addr modes by different paths
 define i32 @test18(i1 %cond1, i1 %cond2, ptr %b1, ptr %b2) {
-; CHECK-LABEL: @test18
+; CHECK-YES-LABEL: define i32 @test18(
+; CHECK-YES-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-YES-NEXT:  [[ENTRY:.*:]]
+; CHECK-YES-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK-YES:       [[IF_THEN1]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK-YES:       [[IF_THEN2]]:
+; CHECK-YES-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-YES:       [[FALLTHROUGH]]:
+; CHECK-YES-NEXT:    [[SUNK_PHI:%.*]] = phi ptr [ [[B2]], %[[IF_THEN2]] ], [ [[B1]], %[[IF_THEN1]] ]
+; CHECK-YES-NEXT:    [[SUNKADDR:%.*]] = getelementptr inbounds i8, ptr [[SUNK_PHI]], i64 40
+; CHECK-YES-NEXT:    [[V1:%.*]] = load i32, ptr [[SUNKADDR]], align 4
+; CHECK-YES-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-YES-NEXT:    [[V2:%.*]] = load i32, ptr [[G1_1]], align 4
+; CHECK-YES-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-YES-NEXT:    ret i32 [[V]]
+;
+; CHECK-NO-LABEL: define i32 @test18(
+; CHECK-NO-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B1:%.*]], ptr [[B2:%.*]]) {
+; CHECK-NO-NEXT:  [[ENTRY:.*:]]
+; CHECK-NO-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK-NO:       [[IF_THEN1]]:
+; CHECK-NO-NEXT:    [[G2:%.*]] = getelementptr inbounds i64, ptr [[B1]], i64 5
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK-NO:       [[IF_THEN2]]:
+; CHECK-NO-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK-NO:       [[FALLTHROUGH]]:
+; CHECK-NO-NEXT:    [[C:%.*]] = phi ptr [ [[G2]], %[[IF_THEN1]] ], [ [[G1]], %[[IF_THEN2]] ]
+; CHECK-NO-NEXT:    [[V1:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NO-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NO-NEXT:    [[V2:%.*]] = load i32, ptr [[G1_1]], align 4
+; CHECK-NO-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NO-NEXT:    ret i32 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b2, i64 5
   br i1 %cond1, label %if.then1, label %if.then2
@@ -467,10 +888,6 @@ if.then3:
   br label %fallthrough
 
 fallthrough:
-; CHECK-YES: sunk_phi
-; CHECK-NO-LABEL: fallthrough:
-; CHECK-NO: phi
-; CHECK-NO-NEXT: load
   %c = phi ptr [%g2, %if.then1], [%g1, %if.then2], [%g1, %if.then3]
   %v1 = load i32, ptr %c, align 4
   %g1_1 = getelementptr inbounds i64, ptr %b2, i64 5
@@ -481,7 +898,31 @@ fallthrough:
 
 ; Different types but null is the first?
 define i32 @test19(i1 %cond1, i1 %cond2, ptr %b2, ptr %b1) {
-; CHECK-LABEL: @test19
+; CHECK-LABEL: define i32 @test19(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]], ptr [[B2:%.*]], ptr [[B1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    br i1 [[COND1]], label %[[IF_THEN1:.*]], label %[[IF_THEN2:.*]]
+; CHECK:       [[IF_THEN1]]:
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, ptr [[B1]], i64 40
+; CHECK-NEXT:    [[BC2:%.*]] = bitcast ptr [[G2]] to ptr
+; CHECK-NEXT:    br label %[[FALLTHROUGH:.*]]
+; CHECK:       [[IF_THEN2]]:
+; CHECK-NEXT:    [[BC1_1:%.*]] = bitcast ptr [[G1]] to ptr
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FALLTHROUGH]], label %[[IF_THEN3:.*]]
+; CHECK:       [[IF_THEN3]]:
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i64, ptr null, i64 5
+; CHECK-NEXT:    [[BC1_2:%.*]] = bitcast ptr [[G3]] to ptr
+; CHECK-NEXT:    br label %[[FALLTHROUGH]]
+; CHECK:       [[FALLTHROUGH]]:
+; CHECK-NEXT:    [[C:%.*]] = phi ptr [ [[BC2]], %[[IF_THEN1]] ], [ [[BC1_1]], %[[IF_THEN2]] ], [ [[BC1_2]], %[[IF_THEN3]] ]
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[C]], align 4
+; CHECK-NEXT:    [[G1_1:%.*]] = getelementptr inbounds i64, ptr [[B2]], i64 5
+; CHECK-NEXT:    [[BC1_1_1:%.*]] = bitcast ptr [[G1_1]] to ptr
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[BC1_1_1]], align 4
+; CHECK-NEXT:    [[V:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
 entry:
   %g1 = getelementptr inbounds i64, ptr %b2, i64 5
   %bc1 = bitcast ptr %g1 to ptr
@@ -502,7 +943,6 @@ if.then3:
   br label %fallthrough
 
 fallthrough:
-; CHECK-NOT: sunk_phi
   %c = phi ptr [%bc2, %if.then1], [%bc1_1, %if.then2], [%bc1_2, %if.then3]
   %v1 = load i32, ptr %c, align 4
   %g1_1 = getelementptr inbounds i64, ptr %b2, i64 5
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
index 6a6b029b67b5..2945a007bf57 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-two-phi.ll
@@ -1,9 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes='require<profile-summary>,function(codegenprepare)' -disable-complex-addr-modes=false -disable-cgp-delete-phis %s | FileCheck %s --check-prefix=CHECK
 target datalayout =
 "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr null, i64 undef
+; CHECK-NEXT:    br label %[[START:.*]]
+; CHECK:       [[START]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = phi ptr [ [[TMP0]], %[[ENTRY]] ], [ [[VAL1]], %[[START]] ]
+; CHECK-NEXT:    [[VAL2:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[VAL2]], %[[START]] ]
+; CHECK-NEXT:    [[SUNKADDR2:%.*]] = bitcast ptr [[VAL2]] to ptr
+; CHECK-NEXT:    [[LOADX:%.*]] = load i64, ptr [[SUNKADDR2]], align 8
+; CHECK-NEXT:    br label %[[START]]
+;
 entry:
   %0 = getelementptr inbounds i64, ptr null, i64 undef
   br label %start
@@ -18,7 +30,6 @@ slowpath:
   br label %exit
 
 exit:
-; CHECK: sunkaddr
   %val3 = phi ptr [ undef, %slowpath ], [ %val2, %start ]
   %val4 = phi ptr [ %elem1, %slowpath ], [ %val1, %start ]
   %val5 = phi ptr [ undef, %slowpath ], [ %val2, %start ]
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
index 5f7701c357ec..0934393a667e 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
@@ -1,4 +1,4 @@
-; Tests whether resume function would remain dbg.value infomation.
+; Tests whether resume function retains dbg.value information.
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
index a220073248ba..5f8e9c9c1d16 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -19,7 +19,6 @@ entry:
   %3 = call i8 @llvm.coro.suspend(token none, i1 false), !dbg !17
   %conv = sext i8 %3 to i32, !dbg !17
   %late_local = alloca i32, align 4
-  call void @coro.devirt.trigger(ptr null)
   switch i32 %conv, label %sw.default [
     i32 0, label %sw.bb
     i32 1, label %sw.bb1
@@ -113,12 +112,6 @@ declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 ; Function Attrs: nounwind
 declare i1 @llvm.coro.end(ptr, i1, token) #5
 
-; Function Attrs: alwaysinline
-define private void @coro.devirt.trigger(ptr) #6 {
-entry:
-  ret void
-}
-
 ; Function Attrs: argmemonly nounwind readonly
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #2
 
@@ -170,15 +163,20 @@ attributes #7 = { noduplicate }
 !31 = !DILocalVariable(name: "allocated", scope: !6, file: !7, line: 55, type: !11)
 !32 = !DILocalVariable(name: "inline_asm", scope: !6, file: !7, line: 55, type: !11)
 
+; Check that the original function is visible and capture its debug info id.
 ; CHECK: define ptr @flink(i32 %x) #0 personality i32 0 !dbg ![[ORIG:[0-9]+]]
+
+; Check that the resume function is present and capture its debug info id.
+; Also check that it contains `#dbg_declare` and `#dbg_value` debug instructions
+; making the debug variables available to the debugger.
+;
 ; CHECK: define internal fastcc void @flink.resume(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[RESUME:[0-9]+]]
 ; CHECK: entry.resume:
 ; CHECK: %[[DBG_PTR:.*]] = alloca ptr
-; CHECK: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_COROHDL:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst,
-; CHECK: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_X:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, [[EXPR_TAIL:.*]])
-; CHECK: store ptr {{.*}}, ptr %[[DBG_PTR]]
+; CHECK-NEXT: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_COROHDL:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst,
+; CHECK-NEXT: #dbg_declare(ptr %[[DBG_PTR]], ![[RESUME_X:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, [[EXPR_TAIL:.*]])
+; CHECK-NEXT: store ptr {{.*}}, ptr %[[DBG_PTR]]
 ; CHECK-NOT: alloca ptr
-; CHECK: call void @coro.devirt.trigger(ptr null)
 ; CHECK: #dbg_value(i8 0, ![[RESUME_CONST:[0-9]+]], !DIExpression(DW_OP_LLVM_convert, 8, DW_ATE_signed, DW_OP_LLVM_convert, 32, DW_ATE_signed),
 ; CHECK: #dbg_value(ptr %[[DBG_PTR]], ![[RESUME_DIRECT:[0-9]+]], !DIExpression(DW_OP_deref, DW_OP_plus_uconst, {{[0-9]+}}, DW_OP_deref),
 ; Note that keeping the undef value here could be acceptable, too.
@@ -194,18 +192,26 @@ attributes #7 = { noduplicate }
 ; CHECK: [[DEFAULT_DEST]]:
 ; CHECK-NOT: {{.*}}:
 ; CHECK: #dbg_value(i32 %[[CALLBR_RES]]
+
+; Check that the destroy and cleanup functions are present and capture their debug info id.
+;
 ; CHECK: define internal fastcc void @flink.destroy(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[DESTROY:[0-9]+]]
 ; CHECK: define internal fastcc void @flink.cleanup(ptr noundef nonnull align 8 dereferenceable(40) %0) #0 personality i32 0 !dbg ![[CLEANUP:[0-9]+]]
 
+; Check that the linkage name of the original function is set correctly.
+;
 ; CHECK: ![[ORIG]] = distinct !DISubprogram(name: "f", linkageName: "flink"
-
 ; CHECK: ![[RESUME]] = distinct !DISubprogram(name: "f", linkageName: "flink.resume"
+
+; Check that metadata for local variables in the resume function is set correctly.
+;
 ; CHECK: ![[RESUME_COROHDL]] = !DILocalVariable(name: "coro_hdl", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_X]] = !DILocalVariable(name: "x", arg: 1, scope: ![[RESUME]]
 ; CHECK: ![[RESUME_CONST]] = !DILocalVariable(name: "direct_const", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT]] = !DILocalVariable(name: "direct_mem", scope: ![[RESUME]]
 ; CHECK: ![[RESUME_DIRECT_VALUE]] = !DILocalVariable(name: "direct_value", scope: ![[RESUME]]
 
+; Check that the linkage names are set correctly for the destroy and cleanup functions.
+;
 ; CHECK: ![[DESTROY]] = distinct !DISubprogram(name: "f", linkageName: "flink.destroy"
-
 ; CHECK: ![[CLEANUP]] = distinct !DISubprogram(name: "f", linkageName: "flink.cleanup"
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll
new file mode 100644
index 000000000000..c8cea268d677
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg-labels.ll
@@ -0,0 +1,147 @@
+; Tests that we add DILabels for the suspend points.
+;
+; We check both the generated LLVM:
+; RUN: opt < %s -passes='cgscc(coro-split)' -S | FileCheck %s
+;
+; And the debug info:
+; REQUIRES: object-emission
+; RUN: opt < %s -passes='cgscc(coro-split),coro-cleanup' \
+; RUN:   | %llc_dwarf -O0 -filetype=obj -o - \
+; RUN:   | llvm-dwarfdump - \
+; RUN:   | FileCheck %s -check-prefix=DWARF
+
+source_filename = "coro.c"
+
+declare void @bar(...) local_unnamed_addr #2
+declare void @baz(...) local_unnamed_addr #2
+
+; Roughly equivalent to:
+;
+; task f() {
+;   while (true) {
+;     bar();
+;     co_await std::suspend_always{};
+;     baz();
+;     co_await std::suspend_always{};
+;   }
+; }
+
+; Function Attrs: nounwind uwtable
+define ptr @f() #3 !dbg !16 {
+entry:
+  %0 = tail call token @llvm.coro.id(i32 0, ptr null, ptr @f, ptr null), !dbg !26
+  %1 = tail call i64 @llvm.coro.size.i64(), !dbg !26
+  %frame = tail call ptr @malloc(i64 %1), !dbg !26
+  %2 = tail call ptr @llvm.coro.begin(token %0, ptr %frame) #4, !dbg !26
+  br label %loop1, !dbg !27
+
+loop1:                                         ; preds = %for.cond, %entry
+  tail call void (...) @bar() #7, !dbg !33
+  %3 = tail call token @llvm.coro.save(ptr null), !dbg !34
+  %4 = tail call i8 @llvm.coro.suspend(token %3, i1 false), !dbg !34
+  switch i8 %4, label %coro_Suspend [
+    i8 0, label %loop2
+    i8 1, label %coro_Cleanup
+  ], !dbg !34
+
+loop2:                                         ; preds = %for.cond, %entry
+  tail call void (...) @baz() #7, !dbg !35
+  %5 = tail call token @llvm.coro.save(ptr null), !dbg !36
+  %6 = tail call i8 @llvm.coro.suspend(token %5, i1 false), !dbg !36
+  switch i8 %6, label %coro_Suspend [
+    i8 0, label %loop1
+    i8 1, label %coro_Cleanup
+  ], !dbg !36
+
+coro_Cleanup:                                     ; preds = %for.cond
+  %7 = tail call ptr @llvm.coro.free(token %0, ptr %2), !dbg !37
+  tail call void @free(ptr nonnull %7), !dbg !37
+  br label %coro_Suspend, !dbg !37
+
+coro_Suspend:                                     ; preds = %for.cond, %if.then, %coro_Cleanup
+  tail call i1 @llvm.coro.end(ptr null, i1 false, token none) #4, !dbg !40
+  ret ptr %2, !dbg !41
+}
+
+; Check that the resume function contains the `#dbg_label` instructions.
+; CHECK-LABEL:   define ptr @f() #1 !dbg !6 {
+; CHECK:         resume.0:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[RESUME_0:[0-9]+]], !{{[0-9]+}})
+; CHECK:         resume.1:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[RESUME_1:[0-9]+]], !{{[0-9]+}})
+
+; Check that the destroy function contains the `#dbg_label` instructions.
+; CHECK-LABEL:   define internal fastcc void @f.destroy({{.*}}) #1 !dbg !38 {
+; CHECK:         resume.0:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[DESTROY_0:[0-9]+]], !{{[0-9]+}})
+; CHECK:         resume.1:        ; preds = %resume.entry
+; CHECK-NEXT:        #dbg_label(![[DESTROY_1:[0-9]+]], !{{[0-9]+}})
+
+; Check that the DILabels are correct.
+; CHECK: ![[RESUME_0]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_0", file: !{{[0-9]*}}, line: 12, column: 6, isArtificial: true, coroSuspendIdx: 0)
+; CHECK: ![[RESUME_1]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_1", file: !{{[0-9]*}}, line: 14, column: 6, isArtificial: true, coroSuspendIdx: 1)
+; CHECK: ![[DESTROY_0]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_0", file: !{{[0-9]*}}, line: 12, column: 6, isArtificial: true, coroSuspendIdx: 0)
+; CHECK: ![[DESTROY_1]] = !DILabel(scope: !{{[0-9]+}}, name: "__coro_resume_1", file: !{{[0-9]*}}, line: 14, column: 6, isArtificial: true, coroSuspendIdx: 1)
+
+; DWARF:        {{.*}}DW_TAG_label
+; DWARF-NEXT:    DW_AT_name ("__coro_resume_0")
+; DWARF-NEXT:    DW_AT_decl_file
+; DWARF-NEXT:    DW_AT_decl_line    (12)
+; DWARF-NEXT:    DW_AT_decl_column  (6)
+; DWARF-NEXT:    DW_AT_artificial   (true)
+; DWARF-NEXT:    DW_AT_LLVM_coro_suspend_idx (0x00)
+; DWARF-NEXT:    DW_AT_low_pc
+
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #5
+
+; Function Attrs: nounwind
+declare noalias ptr @malloc(i64) local_unnamed_addr #6
+declare i64 @llvm.coro.size.i64() #1
+declare ptr @llvm.coro.begin(token, ptr writeonly) #7
+declare token @llvm.coro.save(ptr) #7
+declare i8 @llvm.coro.suspend(token, i1) #7
+declare ptr @llvm.coro.free(token, ptr nocapture readonly) #5
+declare void @free(ptr nocapture) local_unnamed_addr #6
+declare i1 @llvm.coro.end(ptr, i1, token) #7
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind uwtable presplitcoroutine "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { noduplicate }
+attributes #5 = { argmemonly nounwind readonly }
+attributes #6 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #7 = { nounwind }
+attributes #8 = { alwaysinline nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 - manually edited", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "coro.c", directory: "/home/gor/build/bin")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 4.0.0 - manually edited"}
+!9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!16 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 8, type: !17, isLocal: false, isDefinition: true, scopeLine: 8, isOptimized: true, unit: !0, retainedNodes: !20)
+!17 = !DISubroutineType(types: !18)
+!18 = !{!19}
+!19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64, align: 64)
+!20 = !{!21, !22, !24}
+!21 = !DILocalVariable(name: "coro_hdl", scope: !16, file: !1, line: 9, type: !19)
+!22 = !DILocalVariable(name: "i", scope: !23, file: !1, line: 11, type: !9)
+!23 = distinct !DILexicalBlock(scope: !16, file: !1, line: 11, column: 3)
+!24 = !DILocalVariable(name: "coro_mem", scope: !16, file: !1, line: 16, type: !19)
+!26 = !DILocation(line: 9, column: 3, scope: !16)
+!27 = !DILocation(line: 10, column: 8, scope: !23)
+!33 = !DILocation(line: 11, column: 6, scope: !23)
+!34 = !DILocation(line: 12, column: 6, scope: !23)
+!35 = !DILocation(line: 13, column: 6, scope: !23)
+!36 = !DILocation(line: 14, column: 6, scope: !23)
+!37 = !DILocation(line: 16, column: 3, scope: !16)
+!40 = !DILocation(line: 16, column: 3, scope: !16)
+!41 = !DILocation(line: 17, column: 1, scope: !16)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-no-lieftime.ll b/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
similarity index 100%
rename from llvm/test/Transforms/Coroutines/coro-split-no-lieftime.ll
rename to llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
diff --git a/llvm/test/Transforms/DCE/dbg-value-removal.ll b/llvm/test/Transforms/DCE/dbg-value-removal.ll
index 724ad7ba3a09..ec850ac77bed 100644
--- a/llvm/test/Transforms/DCE/dbg-value-removal.ll
+++ b/llvm/test/Transforms/DCE/dbg-value-removal.ll
@@ -4,7 +4,7 @@
 ; All dbg.value with location "!dbg !19" are redundant in the input.
 ; FIXME: We do not handle non-overlapping/overlapping fragments perfectly yet.
 
-define dso_local i16 @main(i16 %a1, i16 %a2) local_unnamed_addr #0 !dbg !7 {
+define dso_local i16 @main(i16 %a1, i16 %a2) local_unnamed_addr !dbg !7 {
 ; CHECK-LABEL: @main(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[BB0:%.*]]
@@ -23,6 +23,8 @@ define dso_local i16 @main(i16 %a1, i16 %a2) local_unnamed_addr #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_value(i16 [[A2]], [[META12]], !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value), [[META18]])
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
+; CHECK-NEXT:      #dbg_declare(i16 %a1, !19, !DIExpression(), [[META18]])
+; CHECK-NEXT:      #dbg_label([[META21:![0-9]+]], [[META18]])
 ; CHECK-NEXT:      #dbg_value(i16 [[A1]], [[META13]], !DIExpression(DW_OP_LLVM_fragment, 0, 8), [[META18]])
 ; CHECK-NEXT:      #dbg_value(i16 [[A1]], [[META13]], !DIExpression(DW_OP_LLVM_fragment, 8, 8), [[META18]])
 ; CHECK-NEXT:    [[T2:%.*]] = call i16 @bar(i16 [[T1]])
@@ -40,51 +42,49 @@ entry:
   br label %bb0
 
 bb0:
-  call void @llvm.dbg.value(metadata i16 999, metadata !12, metadata !DIExpression()), !dbg !19
-  call void @llvm.dbg.value(metadata i16 996, metadata !13, metadata !DIExpression()), !dbg !19
-  call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !17
-  call void @llvm.dbg.value(metadata i16 998, metadata !12, metadata !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value)), !dbg !19
-  call void @llvm.dbg.value(metadata i16 14, metadata !14, metadata !DIExpression()), !dbg !16
-  call void @llvm.dbg.value(metadata i16 997, metadata !12, metadata !DIExpression()), !dbg !19
-  call void @llvm.dbg.value(metadata i16 13, metadata !13, metadata !DIExpression()), !dbg !16
-  call void @llvm.dbg.value(metadata i16 12, metadata !12, metadata !DIExpression()), !dbg !16
+    #dbg_value(i16 999, !12, !DIExpression(), !19)
+    #dbg_value(i16 996, !13, !DIExpression(), !19)
+    #dbg_value(i16 13, !13, !DIExpression(), !17)
+    #dbg_value(i16 998, !12, !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value), !19)
+    #dbg_value(i16 14, !14, !DIExpression(), !16)
+    #dbg_value(i16 997, !12, !DIExpression(), !19)
+    #dbg_value(i16 13, !13, !DIExpression(), !16)
+    #dbg_value(i16 12, !12, !DIExpression(), !16)
   br label %bb1
 
 bb1:
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !14, metadata !DIExpression()), !dbg !16
-  call void @llvm.dbg.value(metadata i16 888, metadata !13, metadata !DIExpression()), !dbg !16
-  call void @llvm.dbg.value(metadata i16 %a2, metadata !12, metadata !DIExpression()), !dbg !16
+    #dbg_value(i16 %a1, !14, !DIExpression(), !16)
+    #dbg_value(i16 888, !13, !DIExpression(), !16)
+    #dbg_value(i16 %a2, !12, !DIExpression(), !16)
   %t1 = call i16 @bar(i16 0)
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !14, metadata !DIExpression()), !dbg !19
-  call void @llvm.dbg.value(metadata i16 %t1, metadata !13, metadata !DIExpression()), !dbg !16
-  call void @llvm.dbg.value(metadata i16 %a2, metadata !12, metadata !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value)), !dbg !16
+    #dbg_value(i16 %a1, !14, !DIExpression(), !19)
+    #dbg_value(i16 %t1, !13, !DIExpression(), !16)
+    #dbg_value(i16 %a2, !12, !DIExpression(DW_OP_constu, 2, DW_OP_shr, DW_OP_stack_value), !16)
   br label %bb2
 
 bb2:
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !19
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !19
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !16
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !19)
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 8, 8), !19)
+    ; Check dbg_declare and dbg_label don't interfere with the backward scan.
+    #dbg_declare(i16 %a1, !21, !DIExpression(), !16)
+    #dbg_label(!20, !16)
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !16)
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 8, 8), !16)
   %t2 = call i16 @bar(i16 %t1)
-  call void @llvm.dbg.value(metadata i16 %t2, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !16
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 8, 8)), !dbg !19
+    #dbg_value(i16 %t2, !13, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !16)
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 8, 8), !19)
   br label %bb3
 
 bb3:
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !19
-  call void @llvm.dbg.value(metadata i16 %a1, metadata !13, metadata !DIExpression()), !dbg !16
+    #dbg_value(i16 %a1, !13, !DIExpression(DW_OP_LLVM_fragment, 0, 8), !19)
+    #dbg_value(i16 %a1, !13, !DIExpression(), !16)
   br label %exit
 
 exit:
   ret i16 %t2
 }
 
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
-declare i16 @bar(i16) #2
-
-attributes #0 = { noinline nounwind }
-attributes #1 = { nounwind readnone speculatable willreturn }
-attributes #2 = { noinline nounwind readnone }
+declare i16 @bar(i16)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
@@ -110,3 +110,5 @@ attributes #2 = { noinline nounwind readnone }
 !17 = !DILocation(line: 0, scope: !7, inlinedAt: !18)
 !18 = !DILocation(line: 1, scope: !7)
 !19 = !DILocation(line: 77, scope: !7)
+!20 = !DILabel(scope: !7, name: "label", file: !1, line: 3)
+!21 = !DILocalVariable(name: "z", scope: !7, file: !1, line: 10, type: !10)
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index f8e594b5626a..6c04e15edc37 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -425,7 +425,7 @@ define ptr @test25(ptr %p) nounwind {
 ; CHECK-NEXT:    [[P_4:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 4
 ; CHECK-NEXT:    [[TMP:%.*]] = load i8, ptr [[P_4]], align 1
 ; CHECK-NEXT:    store i8 0, ptr [[P_4]], align 1
-; CHECK-NEXT:    [[Q:%.*]] = call ptr @strdup(ptr [[P]]) #[[ATTR13:[0-9]+]]
+; CHECK-NEXT:    [[Q:%.*]] = call ptr @strdup(ptr [[P]]) #[[ATTR14:[0-9]+]]
 ; CHECK-NEXT:    store i8 [[TMP]], ptr [[P_4]], align 1
 ; CHECK-NEXT:    ret ptr [[Q]]
 ;
@@ -855,3 +855,36 @@ bb:
   store ptr null, ptr null, align 8
   ret void
 }
+
+define void @test_dead_on_return(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return(
+; CHECK-NEXT:    ret void
+;
+  store i8 0, ptr %p
+  ret void
+}
+
+define void @test_dead_on_return_maythrow(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return_maythrow(
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    ret void
+;
+  store i8 0, ptr %p
+  call void @maythrow()
+  ret void
+}
+
+define ptr @test_dead_on_return_ptr_returned(ptr dead_on_return %p) {
+; CHECK-LABEL: @test_dead_on_return_ptr_returned(
+; CHECK-NEXT:    [[LOCAL_VAR:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    call void @opaque(ptr [[LOCAL_VAR]])
+; CHECK-NEXT:    ret ptr [[P:%.*]]
+;
+  %local.var = alloca ptr
+  call void @opaque(ptr %local.var)
+  store ptr %local.var, ptr %p
+  ret ptr %p
+}
+
+declare void @opaque(ptr)
+declare void @maythrow() memory(none)
diff --git a/llvm/test/Transforms/EarlyCSE/writeonly.ll b/llvm/test/Transforms/EarlyCSE/writeonly.ll
index 0bfffa3c825a..c09b913f9ff2 100644
--- a/llvm/test/Transforms/EarlyCSE/writeonly.ll
+++ b/llvm/test/Transforms/EarlyCSE/writeonly.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=early-cse -earlycse-debug-hash < %s | FileCheck %s
+; RUN: opt -S -passes=early-cse -earlycse-debug-hash < %s | FileCheck %s --check-prefixes=CHECK,NO-MSSA
+; RUN: opt -S -passes='early-cse<memssa>' < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 @var = global i32 undef
 declare void @foo() nounwind
@@ -15,3 +16,148 @@ define void @test() {
   store i32 2, ptr @var
   ret void
 }
+
+declare void @writeonly_void() memory(write)
+
+; Can CSE writeonly calls, including non-nounwind/willreturn.
+define void @writeonly_cse() {
+; CHECK-LABEL: @writeonly_cse(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    ret void
+;
+  call void @writeonly_void()
+  call void @writeonly_void()
+  ret void
+}
+
+; Can CSE, loads do not matter.
+define i32 @writeonly_cse_intervening_load(ptr %p) {
+; CHECK-LABEL: @writeonly_cse_intervening_load(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  call void @writeonly_void()
+  %v = load i32, ptr %p
+  call void @writeonly_void()
+  ret i32 %v
+}
+
+; Cannot CSE, the store may be to the same memory.
+define void @writeonly_cse_intervening_store(ptr %p) {
+; CHECK-LABEL: @writeonly_cse_intervening_store(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    ret void
+;
+  call void @writeonly_void()
+  store i32 0, ptr %p
+  call void @writeonly_void()
+  ret void
+}
+
+; Can CSE, the store does not alias the writeonly call.
+define void @writeonly_cse_intervening_noalias_store(ptr noalias %p) {
+; NO-MSSA-LABEL: @writeonly_cse_intervening_noalias_store(
+; NO-MSSA-NEXT:    call void @writeonly_void()
+; NO-MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; NO-MSSA-NEXT:    call void @writeonly_void()
+; NO-MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @writeonly_cse_intervening_noalias_store(
+; MSSA-NEXT:    call void @writeonly_void()
+; MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    ret void
+;
+  call void @writeonly_void()
+  store i32 0, ptr %p
+  call void @writeonly_void()
+  ret void
+}
+
+; Cannot CSE loads across writeonly call.
+define i32 @load_cse_across_writeonly(ptr %p) {
+; CHECK-LABEL: @load_cse_across_writeonly(
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %v1 = load i32, ptr %p
+  call void @writeonly_void()
+  %v2 = load i32, ptr %p
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; Can CSE loads across eliminated writeonly call.
+define i32 @load_cse_across_csed_writeonly(ptr %p) {
+; CHECK-LABEL: @load_cse_across_csed_writeonly(
+; CHECK-NEXT:    call void @writeonly_void()
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  call void @writeonly_void()
+  %v1 = load i32, ptr %p
+  call void @writeonly_void()
+  %v2 = load i32, ptr %p
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+declare i32 @writeonly(ptr %p) memory(write)
+
+; Can CSE writeonly calls with arg and return.
+define i32 @writeonly_ret_cse(ptr %p) {
+; CHECK-LABEL: @writeonly_ret_cse(
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @writeonly(ptr [[P:%.*]])
+; CHECK-NEXT:    ret i32 0
+;
+  %v1 = call i32 @writeonly(ptr %p)
+  %v2 = call i32 @writeonly(ptr %p)
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+; Cannot CSE writeonly calls with different arguments.
+define i32 @writeonly_different_args(ptr %p1, ptr %p2) {
+; CHECK-LABEL: @writeonly_different_args(
+; CHECK-NEXT:    [[V1:%.*]] = call i32 @writeonly(ptr [[P1:%.*]])
+; CHECK-NEXT:    [[V2:%.*]] = call i32 @writeonly(ptr [[P2:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = sub i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %v1 = call i32 @writeonly(ptr %p1)
+  %v2 = call i32 @writeonly(ptr %p2)
+  %res = sub i32 %v1, %v2
+  ret i32 %res
+}
+
+declare void @callee()
+
+; These are weird cases where the same call is both readonly and writeonly
+; based on call-site attributes. I believe this implies that both calls are
+; actually readnone and safe to CSE, but leave them alone to be conservative.
+define void @readonly_and_writeonly() {
+; CHECK-LABEL: @readonly_and_writeonly(
+; CHECK-NEXT:    call void @callee() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    call void @callee() #[[ATTR1]]
+; CHECK-NEXT:    ret void
+;
+  call void @callee() memory(read)
+  call void @callee() memory(write)
+  ret void
+}
+
+define void @writeonly_and_readonly() {
+; CHECK-LABEL: @writeonly_and_readonly(
+; CHECK-NEXT:    call void @callee() #[[ATTR1]]
+; CHECK-NEXT:    call void @callee() #[[ATTR2]]
+; CHECK-NEXT:    ret void
+;
+  call void @callee() memory(write)
+  call void @callee() memory(read)
+  ret void
+}
diff --git a/llvm/test/Transforms/EmbedBitcode/embed-wpd.ll b/llvm/test/Transforms/EmbedBitcode/embed-wpd.ll
index 54931be42b4e..f1f7712f5403 100644
--- a/llvm/test/Transforms/EmbedBitcode/embed-wpd.ll
+++ b/llvm/test/Transforms/EmbedBitcode/embed-wpd.ll
@@ -1,13 +1,12 @@
 ; RUN: opt --mtriple x86_64-unknown-linux-gnu < %s -passes="embed-bitcode<thinlto>" -S | FileCheck %s
 
-; CHECK: $_ZTV3Foo = comdat any
+; CHECK-NOT: $_ZTV3Foo = comdat any
 $_ZTV3Foo = comdat any
 
 $_ZTI3Foo = comdat any
 
-;; ThinLTOBitcodeWriter will remove the vtable for Foo, and make it an external symbol
-; CHECK: @_ZTV3Foo = linkonce_odr hidden unnamed_addr constant { [5 x ptr] } { [5 x ptr] [ptr null, ptr @_ZTI3Foo, ptr @_ZN3FooD2Ev, ptr @_ZN3FooD0Ev, ptr @_ZNKSt13runtime_error4whatEv] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5
-; CHECK-NOT: @foo = external unnamed_addr constant { [5 x ptr] }, align 8
+; CHECK: @_ZTV3Foo = external hidden unnamed_addr constant { [5 x ptr] }, align 8
+; CHECK: @_ZTI3Foo = linkonce_odr hidden constant { ptr, ptr, ptr } { ptr getelementptr inbounds (ptr, ptr @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2), ptr @_ZTS3Foo, ptr @_ZTISt13runtime_error }, comdat, align 8
 ; CHECK: @llvm.embedded.object = private constant {{.*}}, section ".llvm.lto", align 1
 ; CHECK: @llvm.compiler.used = appending global [1 x ptr] [ptr @llvm.embedded.object], section "llvm.metadata"
 @_ZTV3Foo = linkonce_odr hidden unnamed_addr constant { [5 x ptr] } { [5 x ptr] [ptr null, ptr @_ZTI3Foo, ptr @_ZN3FooD2Ev, ptr @_ZN3FooD0Ev, ptr @_ZNKSt13runtime_error4whatEv] }, comdat, align 8, !type !0, !type !1, !type !2, !type !3, !type !4, !type !5
diff --git a/llvm/test/Transforms/FunctionAttrs/initializes.ll b/llvm/test/Transforms/FunctionAttrs/initializes.ll
index 937595b5e9b7..193d0fd58988 100644
--- a/llvm/test/Transforms/FunctionAttrs/initializes.ll
+++ b/llvm/test/Transforms/FunctionAttrs/initializes.ll
@@ -443,7 +443,7 @@ define void @memset_neg(ptr %p) {
 }
 
 define void @memset_volatile(ptr %p) {
-; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: write)
+; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: write, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define void @memset_volatile(
 ; CHECK-SAME: ptr writeonly [[P:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[P]], i8 2, i64 9, i1 true)
@@ -478,7 +478,7 @@ define void @memcpy(ptr %p, ptr %p2) {
 }
 
 define void @memcpy_volatile(ptr %p, ptr %p2) {
-; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
+; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define void @memcpy_volatile(
 ; CHECK-SAME: ptr writeonly [[P:%.*]], ptr readonly [[P2:%.*]]) #[[ATTR6:[0-9]+]] {
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 9, i1 true)
@@ -541,7 +541,7 @@ define void @memmove(ptr %p, ptr %p2) {
 }
 
 define void @memmove_volatile(ptr %p, ptr %p2) {
-; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
+; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: define void @memmove_volatile(
 ; CHECK-SAME: ptr writeonly [[P:%.*]], ptr readonly [[P2:%.*]]) #[[ATTR6]] {
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[P]], ptr [[P2]], i64 9, i1 true)
@@ -649,3 +649,17 @@ define void @range_overflows_signed_64_bit_int(ptr %arg) {
   store i32 0, ptr %getelementptr
   ret void
 }
+
+; We should bail if the memset range overflows a signed 64-bit int.
+define void @memset_large_offset_nonzero_size(ptr %dst) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
+; CHECK-LABEL: define void @memset_large_offset_nonzero_size(
+; CHECK-SAME: ptr writeonly captures(none) [[DST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 9223372036854775805
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[OFFSET]], i8 0, i64 3, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %offset = getelementptr inbounds i8, ptr %dst, i64 9223372036854775805
+  call void @llvm.memset.p0.i64(ptr %offset, i8 0, i64 3, i1 false)
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/nosync.ll b/llvm/test/Transforms/FunctionAttrs/nosync.ll
index de5398f17ce5..9abfbb21a71a 100644
--- a/llvm/test/Transforms/FunctionAttrs/nosync.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nosync.ll
@@ -236,7 +236,7 @@ declare void @llvm.memset(ptr %dest, i8 %val, i32 %len, i1 %isvolatile)
 
 ; negative, checking volatile intrinsics.
 define i32 @memcpy_volatile(ptr %ptr1, ptr %ptr2) {
-; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite)
+; CHECK: Function Attrs: mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite)
 ; CHECK-LABEL: @memcpy_volatile(
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[PTR1:%.*]], ptr [[PTR2:%.*]], i32 8, i1 true)
 ; CHECK-NEXT:    ret i32 4
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
deleted file mode 100644
index ab04e0a5bc69..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-dest-non-array.ll
+++ /dev/null
@@ -1,39 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_struct()  {
-; CHECK-LABEL: define void @memcpy_struct() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca { i8, i8, i8 }, align 1
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca {i8, i8, i8}, align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  ret void
-}
-
-
-@.i8_multi = private unnamed_addr constant [2 x [3 x i8]] [[3 x i8] [i8 1, i8 2, i8 3], [3 x i8] [i8 4, i8 5, i8 6]] , align 1
-
-define  void @memcpy_array_multidimensional()  {
-; CHECK-LABEL: define void @memcpy_array_multidimensional() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [2 x [3 x i8]], align 1
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [2 x [3 x i8]], align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8_multi, i32 3, i1 false)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
deleted file mode 100644
index f435ffdeed2c..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-global-dest.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK: [3 x i8]
-@other = private unnamed_addr global [3 x i8] [i8 1, i8 2, i8 3] , align 1
-; CHECK: [4 x i8]
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_multiple()  {
-; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull @other)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  %call1 = call i32 @bar(ptr nonnull %something)
-  %call2 = call i32 @bar(ptr nonnull @other)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) @other, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
deleted file mode 100644
index c7ca7271fd3d..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-byte-array.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i16 = private unnamed_addr constant [5 x i16] [i16 1, i16 2, i16 3, i16 4, i16 5] , align 1
-
-define  void @memcpy_i16_array()  {
-; CHECK-LABEL: define void @memcpy_i16_array() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [6 x i16], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 12, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [5 x i16], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.i16, i32 10, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
deleted file mode 100644
index 3d9c42fe1f3d..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-non-const-global.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = unnamed_addr global [3 x i8] c"12\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [3 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
-; CHECK-NEXT:    [[CALL1:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.str, i32 3, i1 false)
-  %call1 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
deleted file mode 100644
index e37925a78d2c..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-string-multi-use.ll
+++ /dev/null
@@ -1,33 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.i8 = private unnamed_addr constant [3 x i8] [i8 1, i8 2, i8 3] , align 1
-
-define  void @memcpy_multiple()  {
-; CHECK-LABEL: define void @memcpy_multiple() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING2:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[SOMETHING1:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    [[SOMETHING3:%.*]] = alloca [4 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING2]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING1]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) [[SOMETHING3]], ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 4, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING2]])
-; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING1]])
-; CHECK-NEXT:    [[CALL4:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING3]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [3 x i8], align 1
-  %something1 = alloca [3 x i8], align 1
-  %something2 = alloca [3 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something1, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(3) %something2, ptr noundef nonnull align 1 dereferenceable(3) @.i8, i32 3, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  %call3 = call i32 @bar(ptr nonnull %something1)
-  %call4 = call i32 @bar(ptr nonnull %something2)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
deleted file mode 100644
index 8ea9e2804370..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-1.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = private unnamed_addr constant [10 x i8] c"123456789\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [12 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 12, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [10 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(10) %something, ptr noundef nonnull align 1 dereferenceable(10) @.str, i32 10, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
deleted file mode 100644
index ad3620b14ea2..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-2.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
-
-define  void @foo()  {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [64 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) [[SOMETHING]], ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 64, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [62 x i8], align 1
-  call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(62) %something, ptr noundef nonnull align 1 dereferenceable(62) @.str, i32 62, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something)
-  ret void
-}
-
-declare i32 @bar(...)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
deleted file mode 100644
index b8e02c3f996d..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-lengths-dont-match.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-; CHECK: [17 x i8]
-@.str = private unnamed_addr constant [17 x i8] c"aaaaaaaaaaaaaaaa\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [20 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 17, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [20 x i8], align 1
-  call void @llvm.lifetime.start(i64 20, ptr nonnull %something) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 17, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something) #3
-  call void @llvm.lifetime.end(i64 20, ptr nonnull %something) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
deleted file mode 100644
index 4ac31aa2f976..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-more-than-64-bytes.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK: [65 x i8]
-; CHECK-NOT: [68 x i8]
-@.str = private unnamed_addr constant [65 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaazzz\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [65 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 65, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[SOMETHING]], ptr align 1 @.str, i32 65, i1 false)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 65, ptr nonnull [[SOMETHING]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [65 x i8], align 1
-  call void @llvm.lifetime.start(i64 65, ptr nonnull %something) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %something, ptr align 1 @.str, i32 65, i1 false)
-  %call2 = call i32 @bar(ptr nonnull %something) #3
-  call void @llvm.lifetime.end(i64 65, ptr nonnull %something) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
deleted file mode 100644
index 64f57884cd39..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-ptrtoint.ll
+++ /dev/null
@@ -1,54 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-@f.string1 = private unnamed_addr constant [45 x i8] c"The quick brown dog jumps over the lazy fox.\00", align 1
-
-; Function Attrs: nounwind
-define  i32 @f() {
-; CHECK-LABEL: define i32 @f() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[STRING1:%.*]] = alloca [48 x i8], align 1
-; CHECK-NEXT:    [[POS:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[TOKEN:%.*]] = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 45, ptr [[STRING1]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[STRING1]], ptr align 1 @f.string1, i32 48, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[POS]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TOKEN]])
-; CHECK-NEXT:    [[CALL:%.*]] = call ptr @strchr(ptr [[STRING1]], i32 101)
-; CHECK-NEXT:    store ptr [[CALL]], ptr [[TOKEN]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[TOKEN]], align 4
-; CHECK-NEXT:    [[SUB_PTR_LHS_CAST:%.*]] = ptrtoint ptr [[TMP1]] to i32
-; CHECK-NEXT:    [[SUB_PTR_RHS_CAST:%.*]] = ptrtoint ptr [[STRING1]] to i32
-; CHECK-NEXT:    [[SUB_PTR_SUB:%.*]] = sub i32 [[SUB_PTR_LHS_CAST]], [[SUB_PTR_RHS_CAST]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUB_PTR_SUB]], 1
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[POS]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[POS]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TOKEN]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[POS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 45, ptr [[STRING1]])
-; CHECK-NEXT:    ret i32 [[TMP2]]
-;
-entry:
-  %string1 = alloca [45 x i8], align 1
-  %pos = alloca i32, align 4
-  %token = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0i8(i64 45, ptr %string1)
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %string1, ptr align 1 @f.string1, i32 45, i1 false)
-  call void @llvm.lifetime.start.p0i8(i64 4, ptr %pos)
-  call void @llvm.lifetime.start.p0i8(i64 4, ptr %token)
-  %call = call ptr @strchr(ptr %string1, i32 101)
-  store ptr %call, ptr %token, align 4
-  %0 = load ptr, ptr %token, align 4
-  %sub.ptr.lhs.cast = ptrtoint ptr %0 to i32
-  %sub.ptr.rhs.cast = ptrtoint ptr %string1 to i32
-  %sub.ptr.sub = sub i32 %sub.ptr.lhs.cast, %sub.ptr.rhs.cast
-  %add = add nsw i32 %sub.ptr.sub, 1
-  store i32 %add, ptr %pos, align 4
-  %1 = load i32, ptr %pos, align 4
-  call void @llvm.lifetime.end.p0i8(i64 4, ptr %token)
-  call void @llvm.lifetime.end.p0i8(i64 4, ptr %pos)
-  call void @llvm.lifetime.end.p0i8(i64 45, ptr %string1)
-  ret i32 %1
-}
-
-declare ptr @strchr(ptr, i32)
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
deleted file mode 100644
index 5367572704b1..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-struct-test.ll
+++ /dev/null
@@ -1,45 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-%struct.P = type { i32, [13 x i8] }
-
-; CHECK-NOT: [16 x i8]
-@.str = private unnamed_addr constant [13 x i8] c"hello world\0A\00", align 1
-
-; Function Attrs: nounwind
-define  i32 @main()   {
-; CHECK-LABEL: define i32 @main() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[P:%.*]] = alloca [[STRUCT_P:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[P]])
-; CHECK-NEXT:    store i32 10, ptr [[P]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [[STRUCT_P]], ptr [[P]], i32 0, i32 1, i32 0
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 1 [[ARRAYDECAY]], ptr align 1 @.str, i32 13, i1 false)
-; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(ptr [[ARRAYDECAY]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[P]])
-; CHECK-NEXT:    ret i32 0
-;
-entry:
-  %p = alloca %struct.P, align 4
-  call void @llvm.lifetime.start(i64 20, ptr nonnull %p) #2
-  store i32 10, ptr %p, align 4, !tbaa !1
-  %arraydecay = getelementptr inbounds %struct.P, ptr %p, i32 0, i32 1, i32 0
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 %arraydecay, ptr align 1 @.str, i32 13, i1 false)
-  %puts = call i32 @puts(ptr %arraydecay)
-  call void @llvm.lifetime.end(i64 20, ptr nonnull %p) #2
-  ret i32 0
-}
-
-declare i32 @puts(ptr nocapture readonly) #2
-
-!1 = !{!2, !3, i64 0}
-!2 = !{!"P", !3, i64 0, !4, i64 4}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
-;.
-; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]], i64 0}
-; CHECK: [[META1]] = !{!"P", [[META2]], i64 0, [[META3:![0-9]+]], i64 4}
-; CHECK: [[META2]] = !{!"int", [[META3]], i64 0}
-; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
-; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
-;.
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll b/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
deleted file mode 100644
index b735a7788742..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/arm-widen-strings-volatile.ll
+++ /dev/null
@@ -1,29 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -mtriple=arm-none-eabi -passes=globalopt -S | FileCheck %s
-
-; CHECK-NOT: [64 x i8]
-@.str = private unnamed_addr constant [62 x i8] c"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
-
-; Function Attrs: nounwind
-define  void @foo()   {
-; CHECK-LABEL: define void @foo() local_unnamed_addr {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[SOMETHING:%.*]] = alloca [62 x i8], align 1
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [62 x i8], ptr [[SOMETHING]], i32 0, i32 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 62, ptr nonnull [[TMP0]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 1 [[TMP0]], ptr align 1 @.str, i32 62, i1 true)
-; CHECK-NEXT:    [[CALL2:%.*]] = call i32 @bar(ptr nonnull [[TMP0]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 62, ptr nonnull [[TMP0]])
-; CHECK-NEXT:    ret void
-;
-entry:
-  %something = alloca [62 x i8], align 1
-  %0 = getelementptr inbounds [62 x i8], ptr %something, i32 0, i32 0
-  call void @llvm.lifetime.start(i64 62, ptr nonnull %0) #3
-  call void @llvm.memcpy.p0i8.p0i8.i32(ptr align 1 nonnull %0, ptr align 1 @.str, i32 62, i1 true)
-  %call2 = call i32 @bar(ptr nonnull %0) #3
-  call void @llvm.lifetime.end(i64 62, ptr nonnull %0) #3
-  ret void
-}
-
-declare i32 @bar(...)  #2
diff --git a/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg b/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
deleted file mode 100644
index 7c23e4f98013..000000000000
--- a/llvm/test/Transforms/GlobalOpt/ARM/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not "ARM" in config.root.targets:
-    config.unsupported = True
diff --git a/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
index 8bc71148352d..5bc733f5622c 100644
--- a/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
+++ b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -S -passes='function(instsimplify),hotcoldsplit' -hotcoldsplit-threshold=-1 -debug < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='function(instsimplify),hotcoldsplit' -hotcoldsplit-threshold=-1 < %s 2>&1 | FileCheck %s
 ; RUN: opt -passes='function(instcombine),hotcoldsplit,function(instsimplify)' %s -o /dev/null
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/Transforms/InferAlignment/gep-recurrence.ll b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll
new file mode 100644
index 000000000000..50b968d206b8
--- /dev/null
+++ b/llvm/test/Transforms/InferAlignment/gep-recurrence.ll
@@ -0,0 +1,574 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=infer-alignment -S %s | FileCheck %s
+
+target datalayout = "p1:64:64:64:32"
+
+declare i1 @cond()
+
+define void @recur_i8_128(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_128(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 128
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 128
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_128_no_nusw(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_128_no_nusw(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 128
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr i8, ptr %iv, i64 128
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_64(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_64(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 64
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 64
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_63(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_63(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 63
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 63
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_32(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_32(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 32
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 32
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_16(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_16(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 16
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 16
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_8(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_8(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 8
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 8
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_4(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_4(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 4
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 4
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_2(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_2(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 2
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 2
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_1(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_1(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 1
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 1
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_unknown_step(ptr align 128 %dst, i64 %off) {
+; CHECK-LABEL: define void @recur_i8_unknown_step(
+; CHECK-SAME: ptr align 128 [[DST:%.*]], i64 [[OFF:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 [[OFF]]
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 %off
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_step_known_multiple(ptr align 128 %dst, i64 %off) {
+; CHECK-LABEL: define void @recur_i8_step_known_multiple(
+; CHECK-SAME: ptr align 128 [[DST:%.*]], i64 [[OFF:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[UREM:%.*]] = urem i64 [[OFF]], 128
+; CHECK-NEXT:    [[C_UREM:%.*]] = icmp eq i64 [[UREM]], 0
+; CHECK-NEXT:    [[C_POS:%.*]] = icmp sge i64 [[OFF]], 0
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[C_UREM]], [[C_POS]]
+; CHECK-NEXT:    br i1 [[AND]], label %[[LOOP:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 [[OFF]]
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %urem = urem i64 %off, 128
+  %c.urem = icmp eq i64 %urem, 0
+  %c.pos = icmp sge i64 %off, 0
+  %and = and i1 %c.urem, %c.pos
+  br i1 %and, label %loop, label %exit
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 %off
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_i16_128(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_i16_128(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i16 128
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i16 128
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_i8_132(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_i8_132(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i8 -124
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i8 132
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i32_4(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i32_4(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i32, ptr [[IV]], i64 4
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i32, ptr %iv, i64 4
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i32_3(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i32_3(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i32, ptr [[IV]], i64 4
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i32, ptr %iv, i64 4
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_neg_128(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_neg_128(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -128
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 -128
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_neg64(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_neg64(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -64
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 -64
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_neg_63(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_neg_63(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -63
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 -63
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @recur_i8_neg_32(ptr align 128 %dst) {
+; CHECK-LABEL: define void @recur_i8_neg_32(
+; CHECK-SAME: ptr align 128 [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    store i64 0, ptr [[IV]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr nusw i8, ptr [[IV]], i64 -32
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %dst, %entry ], [ %iv.next, %loop ]
+  store i64 0, ptr %iv, align 1
+  %iv.next = getelementptr nusw i8, ptr %iv, i64 -32
+  %c = call i1 @cond()
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/simplify-crosscallsite.ll b/llvm/test/Transforms/Inline/simplify-crosscallsite.ll
new file mode 100644
index 000000000000..112f858aac0e
--- /dev/null
+++ b/llvm/test/Transforms/Inline/simplify-crosscallsite.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes=inline | FileCheck %s
+
+define i32 @callee([2 x i32] %agg) {
+; CHECK-LABEL: define i32 @callee(
+; CHECK-SAME: [2 x i32] [[AGG:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = extractvalue [2 x i32] [[AGG]], 0
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[V]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[IS_NULL:.*]], label %[[NON_NULL:.*]]
+; CHECK:       [[IS_NULL]]:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       [[NON_NULL]]:
+; CHECK-NEXT:    [[R:%.*]] = call i32 @callee([2 x i32] [[AGG]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %v = extractvalue [2 x i32] %agg, 0
+  %c = icmp eq i32 %v, 0
+  br i1 %c, label %is_null, label %non_null
+
+is_null:
+  ret i32 0
+
+non_null:
+  %r = call i32 @callee([2 x i32] %agg)
+  ret i32 %r
+}
+
+define i32 @caller_simplified(i32 %arg) {
+; CHECK-LABEL: define i32 @caller_simplified(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[AGG0:%.*]] = insertvalue [2 x i32] poison, i32 0, 0
+; CHECK-NEXT:    [[AGG1:%.*]] = insertvalue [2 x i32] [[AGG0]], i32 [[ARG]], 1
+; CHECK-NEXT:    ret i32 0
+;
+  %agg0 = insertvalue [2 x i32] poison, i32 0, 0
+  %agg1 = insertvalue [2 x i32] %agg0, i32 %arg, 1
+  %v = call i32 @callee([2 x i32] %agg1)
+  ret i32 %v
+}
+
+define i32 @caller_not_simplified(i32 %arg) {
+; CHECK-LABEL: define i32 @caller_not_simplified(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[AGG0:%.*]] = insertvalue [2 x i32] poison, i32 1, 0
+; CHECK-NEXT:    [[AGG1:%.*]] = insertvalue [2 x i32] [[AGG0]], i32 [[ARG]], 1
+; CHECK-NEXT:    [[V:%.*]] = call i32 @callee([2 x i32] [[AGG1]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %agg0 = insertvalue [2 x i32] poison, i32 1, 0
+  %agg1 = insertvalue [2 x i32] %agg0, i32 %arg, 1
+  %v = call i32 @callee([2 x i32] %agg1)
+  ret i32 %v
+}
+
+define i32 @caller_not_simplified2(i32 %arg) {
+; CHECK-LABEL: define i32 @caller_not_simplified2(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[AGG0:%.*]] = insertvalue [2 x i32] poison, i32 0, 1
+; CHECK-NEXT:    [[AGG1:%.*]] = insertvalue [2 x i32] [[AGG0]], i32 [[ARG]], 0
+; CHECK-NEXT:    [[V:%.*]] = call i32 @callee([2 x i32] [[AGG1]])
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %agg0 = insertvalue [2 x i32] poison, i32 0, 1
+  %agg1 = insertvalue [2 x i32] %agg0, i32 %arg, 0
+  %v = call i32 @callee([2 x i32] %agg1)
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll b/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll
index d9311008bd68..361a2b828091 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/fmed3.ll
@@ -521,6 +521,96 @@ define float @fmed3_neg2_3_snan1_f32(float %x, float %y) #1 {
   ret float %med3
 }
 
+define float @fmed3_inf_x_y_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_inf_x_y_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_inf_x_y_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0x7FF0000000000000, float %x, float %y)
+  ret float %med3
+}
+
+define float @fmed3_x_inf_y_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_x_inf_y_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_x_inf_y_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 0x7FF0000000000000, float %y)
+  ret float %med3
+}
+
+define float @fmed3_x_y_inf_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_x_y_inf_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_x_y_inf_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.maximumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0x7FF0000000000000)
+  ret float %med3
+}
+
+define float @fmed3_ninf_x_y_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_ninf_x_y_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_ninf_x_y_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float 0xFFF0000000000000, float %x, float %y)
+  ret float %med3
+}
+
+define float @fmed3_x_ninf_y_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_x_ninf_y_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_x_ninf_y_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float 0xFFF0000000000000, float %y)
+  ret float %med3
+}
+
+define float @fmed3_x_y_ninf_f32(float %x, float %y) #1 {
+; IEEE1-LABEL: define float @fmed3_x_y_ninf_f32(
+; IEEE1-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE1-NEXT:    [[MED3:%.*]] = call float @llvm.minnum.f32(float [[X]], float [[Y]])
+; IEEE1-NEXT:    ret float [[MED3]]
+;
+; IEEE0-LABEL: define float @fmed3_x_y_ninf_f32(
+; IEEE0-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR1]] {
+; IEEE0-NEXT:    [[MED3:%.*]] = call float @llvm.minimumnum.f32(float [[X]], float [[Y]])
+; IEEE0-NEXT:    ret float [[MED3]]
+;
+  %med3 = call float @llvm.amdgcn.fmed3.f32(float %x, float %y, float 0xFFF0000000000000)
+  ret float %med3
+}
+
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.fmed3 with default mode implied by shader CC
 ; --------------------------------------------------------------------
diff --git a/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg b/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg
new file mode 100644
index 000000000000..f9dd98a21cc3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/SystemZ/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "SystemZ" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll b/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll
new file mode 100644
index 000000000000..9148f9c6a6c4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/SystemZ/printf-opt-alignment.ll
@@ -0,0 +1,17 @@
+; RUN: opt < %s --passes=instcombine -S | FileCheck %s
+;
+; Check that string replacements inserted by the instcombiner are properly aligned.
+; The specific case checked replaces `printf("foo\n")` with `puts("foo")`
+
+target datalayout = "i8:8:16"
+
+@msg1 = constant [17 x i8] c"Alignment Check\0A\00", align 2
+; CHECK: c"Alignment Check\00", align 2
+
+; Function Attrs: noinline nounwind
+define dso_local void @foo() #0 {
+  %call = call signext i32 (ptr, ...) @printf(ptr noundef @msg1)
+  ret void
+}
+
+declare signext i32 @printf(ptr noundef, ...) #1
diff --git a/llvm/test/Transforms/InstCombine/binop-recurrence.ll b/llvm/test/Transforms/InstCombine/binop-recurrence.ll
new file mode 100644
index 000000000000..b5eef4e3f516
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/binop-recurrence.ll
@@ -0,0 +1,1408 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+; Reassociate add.
+define i8 @add_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @add_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, and maintain nuw if all ops have it.
+define i8 @add_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add nuw i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if op1 doesn't have it.
+define i8 @add_op1_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_op1_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if op2 doesn't have it.
+define i8 @add_op2_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_op2_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nuw if rdx doesn't have it.
+define i8 @add_rdx_no_nuw(i32 %n) {
+; CHECK-LABEL: define i8 @add_rdx_no_nuw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw i8 %pn, 2
+  %op2 = add nuw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, drop nsw even if all ops have it.
+define i8 @add_no_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @add_no_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nsw i8 %pn, 2
+  %op2 = add nsw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate add, keep nuw/nsw if all ops have them.
+define i8 @add_nuw_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @add_nuw_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add nuw nsw i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add nuw nsw i8 %pn, 2
+  %op2 = add nuw nsw i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add nuw nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate fixed-length vector operands.
+define <16 x i8> @add_v16i8(i32 %n) {
+; CHECK-LABEL: define <16 x i8> @add_v16i8(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi <16 x i8> [ splat (i8 1), %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add <16 x i8> [[REDUCED_PHI]], splat (i8 5)
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret <16 x i8> [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi <16 x i8> [ splat (i8 0), %entry ], [ %op1, %body ]
+  %pn2 = phi <16 x i8> [ splat (i8 1), %entry ], [ %op2, %body ]
+  %op1 = add <16 x i8> %pn, splat (i8 2)
+  %op2 = add <16 x i8> %pn2, splat (i8 3)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add <16 x i8> %op2, %op1
+  ret <16 x i8> %rdx
+}
+
+; Reassociate scalable vector operands.
+define <vscale x 16 x i8> @add_nxv16i8(i32 %n) {
+; CHECK-LABEL: define <vscale x 16 x i8> @add_nxv16i8(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi <vscale x 16 x i8> [ splat (i8 1), %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add <vscale x 16 x i8> [[REDUCED_PHI]], splat (i8 5)
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret <vscale x 16 x i8> [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi <vscale x 16 x i8> [ splat (i8 0), %entry ], [ %op1, %body ]
+  %pn2 = phi <vscale x 16 x i8> [ splat (i8 1), %entry ], [ %op2, %body ]
+  %op1 = add <vscale x 16 x i8> %pn, splat (i8 2)
+  %op2 = add <vscale x 16 x i8> %pn2, splat (i8 3)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add <vscale x 16 x i8> %op2, %op1
+  ret <vscale x 16 x i8> %rdx
+}
+
+; Check other opcodes.
+
+; Reassociate mul.
+define i8 @mul_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @mul_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 2, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = mul i8 [[REDUCED_PHI]], 15
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 2, %entry ], [ %op2, %body ]
+  %op1 = mul i8 %pn, 3
+  %op2 = mul i8 %pn2, 5
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate mul, don't expect any flags to be propagated.
+define i8 @mul_reassoc_no_nuw_no_nsw(i32 %n) {
+; CHECK-LABEL: define i8 @mul_reassoc_no_nuw_no_nsw(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 2, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = mul i8 [[REDUCED_PHI]], 15
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 2, %entry ], [ %op2, %body ]
+  %op1 = mul nuw nsw i8 %pn, 3
+  %op2 = mul nuw nsw i8 %pn2, 5
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul nuw nsw i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate and, although it should already be optimized separately.
+define i8 @and_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @and_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 3
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 31, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 63, %entry ], [ %op2, %body ]
+  %op1 = and i8 %pn, 3
+  %op2 = and i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = and i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate or, although it should already be optimized separately.
+define i8 @or_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @or_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 7
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = or i8 %pn, 3
+  %op2 = or i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = or i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate or and propagate disjoint if all ops have it.
+; Note: This can't currently be seen as the results get optimized away.
+define i8 @or_disjoint(i32 %n) {
+; CHECK-LABEL: define i8 @or_disjoint(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 7
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = or disjoint i8 %pn, 3
+  %op2 = or disjoint i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = or disjoint i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate xor.
+define i8 @xor_reassoc(i32 %n) {
+; CHECK-LABEL: define i8 @xor_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = xor i8 [[REDUCED_PHI]], 4
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = xor i8 %pn, 3
+  %op2 = xor i8 %pn2, 7
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = xor i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate fadd if reassoc and nsz are present on all instructions.
+define float @fadd_reassoc_nsz(i32 %n) {
+; CHECK-LABEL: define float @fadd_reassoc_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = fadd reassoc nsz float [[REDUCED_PHI]], 5.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd reassoc nsz float %pn, 2.0
+  %op2 = fadd reassoc nsz float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd reassoc nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Reassociate fmul if reassoc and nsz are present on all instructions.
+define float @fmul_reassoc_nsz(i32 %n) {
+; CHECK-LABEL: define float @fmul_reassoc_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi float [ 2.000000e+00, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = fmul reassoc nsz float [[REDUCED_PHI]], 1.200000e+01
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 1.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 2.0, %entry ], [ %op2, %body ]
+  %op1 = fmul reassoc nsz float %pn, 3.0
+  %op2 = fmul reassoc nsz float %pn2, 4.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fmul reassoc nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Don't reassociate without `reassoc'.
+define float @fadd_no_reassoc(i32 %n) {
+; CHECK-LABEL: define float @fadd_no_reassoc(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = fadd nsz float [[PN]], 2.000000e+00
+; CHECK-NEXT:    [[OP2]] = fadd nsz float [[PN2]], 3.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = fadd nsz float [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd nsz float %pn, 2.0
+  %op2 = fadd nsz float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd nsz float %op2, %op1
+  ret float %rdx
+}
+
+; Don't reassociate without `nsz'.
+define float @fadd_no_nsz(i32 %n) {
+; CHECK-LABEL: define float @fadd_no_nsz(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi float [ 1.000000e+00, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = fadd reassoc float [[PN]], 2.000000e+00
+; CHECK-NEXT:    [[OP2]] = fadd reassoc float [[PN2]], 3.000000e+00
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = fadd reassoc float [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret float [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi float [ 0.0, %entry ], [ %op1, %body ]
+  %pn2 = phi float [ 1.0, %entry ], [ %op2, %body ]
+  %op1 = fadd reassoc float %pn, 2.0
+  %op2 = fadd reassoc float %pn2, 3.0
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = fadd reassoc float %op2, %op1
+  ret float %rdx
+}
+
+; Check commuted operands.
+
+; Reassociate, even if op1 has commuted operands.
+define i8 @add_op1_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_op1_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 2, %pn
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if op2 has commuted operands.
+define i8 @add_op2_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_op2_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 3, %pn2
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if rdx has commuted operands.
+define i8 @add_rdx_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_rdx_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op1, %op2
+  ret i8 %rdx
+}
+
+; Reassociate, even if pn has commuted incoming values.
+define i8 @add_pn_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_pn_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ %op1, %body ], [ 0, %entry ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Reassociate, even if pn2 has commuted incoming values.
+define i8 @add_pn2_commuted(i32 %n) {
+; CHECK-LABEL: define i8 @add_pn2_commuted(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[REDUCED_PHI:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[RDX:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[RDX]] = add i8 [[REDUCED_PHI]], 5
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ %op2, %body ], [ 1, %entry ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op1, %op2
+  ret i8 %rdx
+}
+
+; Check the instructions have the same opcodes.
+
+; Don't reassociate if the first op doesn't match the rest.
+define i8 @no_mul_add_add(i32 %n) {
+; CHECK-LABEL: define i8 @no_mul_add_add(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = shl i8 [[PN]], 1
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = mul i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the second op doesn't match the rest.
+define i8 @no_add_mul_add(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_mul_add(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = mul i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = mul i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the third op doesn't match the rest.
+define i8 @no_add_add_mul(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_add_mul(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = mul i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = mul i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check the number of uses of pn, pn2, op1 and op2.
+
+; Don't reassociate if pn has more uses.
+define i8 @no_add_pn_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_pn_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[PN]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %pn)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if pn2 has more uses.
+define i8 @no_add_pn2_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_pn2_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[PN2]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %pn2)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op1 has more uses.
+define i8 @no_add_op1_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op1_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[OP1]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %op1)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op2 has more uses.
+define i8 @no_add_op2_use(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op2_use(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    tail call void @use(i8 [[OP2]])
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  tail call void @use(i8 %op2)
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check that init1, init2, c1 and c2 are constants.
+; Note: It should be possible to support non-constant operands, we just don't
+; do so yet.
+
+; Don't reassociate if the initial value of pn isn't constant.
+define i8 @no_add_init1(i8 %init1, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_init1(
+; CHECK-SAME: i8 [[INIT1:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ [[INIT1]], %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ %init1, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the initial value of pn2 isn't constant.
+define i8 @no_add_init2(i8 %init2, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_init2(
+; CHECK-SAME: i8 [[INIT2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ [[INIT2]], %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ %init2, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the recurrence operand of op1 isn't constant.
+define i8 @no_add_c1(i8 %c1, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_c1(
+; CHECK-SAME: i8 [[C1:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], [[C1]]
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, %c1
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the recurrence operand of op2 isn't constant.
+define i8 @no_add_c2(i8 %c2, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_c2(
+; CHECK-SAME: i8 [[C2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], [[C2]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, %c2
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Check the incoming values of pn and pn2.
+
+; Don't reassociate if op1 doesn't recurse to pn.
+define i8 @no_add_op1_to_pn(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op1_to_pn(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[OP1:%.*]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op2, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if op2 doesn't recurse to pn2.
+define i8 @no_add_op2_to_pn2(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op2_to_pn2(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP1]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[OP2:%.*]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op1, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the phis have more than two incoming values.
+define i8 @no_add_phis_more_incoming_values(i1 %c, i32 %n) {
+; CHECK-LABEL: define i8 @no_add_phis_more_incoming_values(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[BODY:.*]], label %[[OTHER:.*]]
+; CHECK:       [[OTHER]]:
+; CHECK-NEXT:    br label %[[BODY]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 1, %[[OTHER]] ], [ [[I_NEXT:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ 0, %[[OTHER]] ], [ [[OP1:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ 1, %[[OTHER]] ], [ [[OP2:%.*]], %[[BODY]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br i1 %c, label %body, label %other
+
+other:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ 1, %other ], [ %i.next, %body ]
+  %pn = phi i8 [ 0, %entry ], [ 0, %other ], [ %op1, %body ]
+  %pn2 = phi i8 [ 1, %entry ], [ 1, %other ], [ %op2, %body ]
+  %op1 = add i8 %pn, 2
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+; Don't reassociate if the ops span different blocks.
+define i8 @no_add_op_multi_block(i32 %n) {
+; CHECK-LABEL: define i8 @no_add_op_multi_block(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[BODY:.*]]
+; CHECK:       [[BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OTHER:.*]] ]
+; CHECK-NEXT:    [[PN:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[OP1:%.*]], %[[OTHER]] ]
+; CHECK-NEXT:    [[PN2:%.*]] = phi i8 [ 1, %[[ENTRY]] ], [ [[OP2:%.*]], %[[OTHER]] ]
+; CHECK-NEXT:    [[OP1]] = add i8 [[PN]], 2
+; CHECK-NEXT:    br label %[[OTHER]]
+; CHECK:       [[OTHER]]:
+; CHECK-NEXT:    [[OP2]] = add i8 [[PN2]], 3
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[BODY]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RDX:%.*]] = add i8 [[OP2]], [[OP1]]
+; CHECK-NEXT:    ret i8 [[RDX]]
+;
+entry:
+  br label %body
+
+body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %other ]
+  %pn = phi i8 [ 0, %entry ], [ %op1, %other ]
+  %pn2 = phi i8 [ 1, %entry ], [ %op2, %other ]
+  %op1 = add i8 %pn, 2
+  br label %other
+
+other:
+  %op2 = add i8 %pn2, 3
+  %i.next = add nuw nsw i32 %i, 1
+  %cmp = icmp eq i32 %i.next, %n
+  br i1 %cmp, label %exit, label %body
+
+exit:
+  %rdx = add i8 %op2, %op1
+  ret i8 %rdx
+}
+
+declare void @use(i8)
diff --git a/llvm/test/Transforms/InstCombine/compare-3way.ll b/llvm/test/Transforms/InstCombine/compare-3way.ll
index 5d443cd45238..f5bbe0c9efd2 100644
--- a/llvm/test/Transforms/InstCombine/compare-3way.ll
+++ b/llvm/test/Transforms/InstCombine/compare-3way.ll
@@ -3,7 +3,7 @@
 
 declare void @use(i32)
 
-; These 18 exercise all combinations of signed comparison
+; These exercise all combinations of signed comparison
 ; for each of the three values produced by your typical
 ; 3way compare function (-1, 0, 1)
 
@@ -81,8 +81,8 @@ unreached:
 define void @test_low_sle(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_sle
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -105,8 +105,8 @@ unreached:
 define void @test_low_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -130,8 +130,8 @@ unreached:
 define void @test_low_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_low_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -154,8 +154,8 @@ unreached:
 define void @test_mid_sgt(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_sgt
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -178,8 +178,8 @@ unreached:
 define void @test_mid_slt(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_slt
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -252,8 +252,8 @@ unreached:
 define void @test_mid_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[EQ:%.*]] = icmp eq i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[EQ]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -277,8 +277,8 @@ unreached:
 define void @test_mid_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_mid_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[EQ:%.*]] = icmp eq i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[EQ]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -348,8 +348,8 @@ unreached:
 define void @test_high_sge(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_sge
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -396,8 +396,8 @@ unreached:
 define void @test_high_ne(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_ne
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[NORMAL:%.*]], label [[UNREACHED:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -421,8 +421,8 @@ unreached:
 define void @test_high_eq(i64 %a, i64 %b) {
 ; CHECK-LABEL: define void @test_high_eq
 ; CHECK-SAME: (i64 [[A:%.*]], i64 [[B:%.*]]) {
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[A]], [[B]]
-; CHECK-NEXT:    br i1 [[TMP1]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[UNREACHED:%.*]], label [[NORMAL:%.*]]
 ; CHECK:       normal:
 ; CHECK-NEXT:    ret void
 ; CHECK:       unreached:
@@ -560,3 +560,75 @@ unreached:
   call void @use(i32 %result)
   ret void
 }
+
+define i32 @smax_smin_to_scmp(i32 %x) {
+; CHECK-LABEL: define i32 @smax_smin_to_scmp
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  ret i32 %cond5
+}
+
+define i16 @smax_smin_to_scmp_i16(i16 %x) {
+; CHECK-LABEL: define i16 @smax_smin_to_scmp_i16
+; CHECK-SAME: (i16 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i16 @llvm.scmp.i16.i16(i16 [[X]], i16 0)
+; CHECK-NEXT:    ret i16 [[COND5]]
+;
+  %cond = call i16 @llvm.smax.i16(i16 %x, i16 -1)
+  %cond5 = call i16 @llvm.smin.i16(i16 %cond, i16 1)
+  ret i16 %cond5
+}
+
+; Test the reversed pattern: smax(smin(X, 1), -1) -> scmp(X, 0)
+define i32 @smin_smax_to_scmp(i32 %x) {
+; CHECK-LABEL: define i32 @smin_smax_to_scmp
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.scmp.i32.i32(i32 [[X]], i32 0)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smin.i32(i32 %x, i32 1)
+  %cond5 = call i32 @llvm.smax.i32(i32 %cond, i32 -1)
+  ret i32 %cond5
+}
+
+define i32 @test_max_min_neg(i32 %x) {
+; CHECK-LABEL: define i32 @test_max_min_neg
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -2)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 1)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -2)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  ret i32 %cond5
+}
+
+define i32 @test_max_min_neg_2(i32 %x) {
+; CHECK-LABEL: define i32 @test_max_min_neg_2
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -1)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 2)
+; CHECK-NEXT:    ret i32 [[COND5]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 2)
+  ret i32 %cond5
+}
+
+define i32 @test_multiple_uses(i32 %x) {
+; CHECK-LABEL: define i32 @test_multiple_uses
+; CHECK-SAME: (i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 -1)
+; CHECK-NEXT:    [[COND5:%.*]] = call i32 @llvm.smin.i32(i32 [[COND]], i32 1)
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[COND]], [[COND5]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %cond = call i32 @llvm.smax.i32(i32 %x, i32 -1)
+  %cond5 = call i32 @llvm.smin.i32(i32 %cond, i32 1)
+  %sum = add i32 %cond, %cond5
+  ret i32 %sum
+}
diff --git a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
index 4c5d7a7dc011..a1f0f01e2c47 100644
--- a/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
+++ b/llvm/test/Transforms/InstCombine/ctpop-cttz.ll
@@ -127,3 +127,24 @@ define <2 x i32> @ctpop3v_poison(<2 x i32> %0) {
   %5 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %4)
   ret <2 x i32> %5
 }
+
+define i32 @ctpop_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @ctpop_xor(
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[FALSE:%.*]], label [[TRUE:%.*]]
+; CHECK:       true:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[RET:%.*]] = call range(i32 1, 33) i32 @llvm.ctpop.i32(i32 [[XOR]])
+; CHECK-NEXT:    ret i32 [[RET]]
+; CHECK:       false:
+; CHECK-NEXT:    ret i32 0
+;
+  %cmp = icmp ne i32 %x, %y
+  br i1 %cmp, label %true, label %false
+true:
+  %xor = xor i32 %x, %y
+  %ret = call i32 @llvm.ctpop.i32(i32 %xor)
+  ret i32 %ret
+false:
+  ret i32 0
+}
diff --git a/llvm/test/Transforms/InstCombine/debuginfo-variables.ll b/llvm/test/Transforms/InstCombine/debuginfo-variables.ll
index 1c9c2a5d5020..bcfd6d2eb1e1 100644
--- a/llvm/test/Transforms/InstCombine/debuginfo-variables.ll
+++ b/llvm/test/Transforms/InstCombine/debuginfo-variables.ll
@@ -1,5 +1,4 @@
 ; RUN: opt < %s -passes=debugify,instcombine -S | FileCheck %s
-
 ; RUN: opt < %s -passes=debugify,instcombine --debugify-diop-diexprs -S | FileCheck %s --check-prefix DIOP-DBGINFO
 
 declare void @escape32(i32)
@@ -191,14 +190,14 @@ define void @test_gep(ptr %A) {
   ret void
 }
 
-define void @test_gep_var_offset(ptr %A, i64 %B, i64 %C) {
+define void @test_gep_var_offset(ptr %A, i64 %B, i8 %C) {
 ; CHECK-LABEL: @test_gep_var_offset(
-; CHECK-NEXT:  #dbg_value(!DIArgList(ptr %A, i64 %B, i64 %C), {{.*}}, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_constu, 8, DW_OP_mul, DW_OP_plus, DW_OP_LLVM_arg, 2, DW_OP_constu, 2, DW_OP_mul, DW_OP_plus, DW_OP_plus_uconst, 88, DW_OP_stack_value),
+; CHECK-NEXT:  #dbg_value(!DIArgList(ptr %A, i64 %B, i8 %C), {{.*}}, !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_constu, 8, DW_OP_mul, DW_OP_plus, DW_OP_LLVM_arg, 2, DW_OP_constu, 2, DW_OP_mul, DW_OP_plus, DW_OP_plus_uconst, 88, DW_OP_stack_value),
 
 ; DIOP-DBGINFO-LABEL: @test_gep_var_offset(
-; DIOP-DBGINFO-NEXT:  #dbg_value(!DIArgList(ptr %A, i64 %B, i64 %C), {{.*}}, !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64), DIOpArg(1, i64), DIOpConstant(i64 8), DIOpMul(), DIOpAdd(), DIOpArg(2, i64), DIOpConstant(i64 2), DIOpMul(), DIOpAdd(), DIOpConstant(i64 88), DIOpAdd(), DIOpReinterpret(ptr)),
+; DIOP-DBGINFO-NEXT:  #dbg_value(!DIArgList(ptr %A, i64 %B, i8 %C), {{.*}}, !DIExpression(DIOpArg(0, ptr), DIOpReinterpret(i64), DIOpArg(1, i64), DIOpConstant(i64 8), DIOpMul(), DIOpAdd(), DIOpArg(2, i8), DIOpSExt(i64), DIOpConstant(i64 2), DIOpMul(), DIOpAdd(), DIOpConstant(i64 88), DIOpAdd(), DIOpReinterpret(ptr)),
 
   ; This is the following expression in infix: i64(A) + B*8 + C*2 + 88
-  %1 = getelementptr %struct.S, ptr %A, i32 1, i32 1, i64 %B, i32 0, i64 %C
+  %1 = getelementptr %struct.S, ptr %A, i32 1, i32 1, i64 %B, i32 0, i8 %C
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/fold-phi-arg-gep-to-phi-negative.ll b/llvm/test/Transforms/InstCombine/fold-phi-arg-gep-to-phi-negative.ll
index 0bbb1035b109..c5f9dc186170 100644
--- a/llvm/test/Transforms/InstCombine/fold-phi-arg-gep-to-phi-negative.ll
+++ b/llvm/test/Transforms/InstCombine/fold-phi-arg-gep-to-phi-negative.ll
@@ -11,13 +11,13 @@ define <16 x ptr> @test(i1 %tobool) {
 ; CHECK-NEXT:    [[LANE_15:%.*]] = insertelement <16 x ptr> poison, ptr [[LANE_0]], i64 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label %[[F1:.*]], label %[[F0:.*]]
 ; CHECK:       [[F0]]:
-; CHECK-NEXT:    [[MM_VECTORGEP:%.*]] = getelementptr inbounds [[FOO]], <16 x ptr> [[LANE_15]], <16 x i64> zeroinitializer, <16 x i32> splat (i32 1)
+; CHECK-NEXT:    [[MM_VECTORGEP1:%.*]] = getelementptr inbounds i8, <16 x ptr> [[LANE_15]], i64 2
 ; CHECK-NEXT:    br label %[[MERGE:.*]]
 ; CHECK:       [[F1]]:
-; CHECK-NEXT:    [[MM_VECTORGEP2:%.*]] = getelementptr inbounds [[FOO]], <16 x ptr> [[LANE_15]], <16 x i64> zeroinitializer, <16 x i32> splat (i32 2)
+; CHECK-NEXT:    [[MM_VECTORGEP22:%.*]] = getelementptr inbounds i8, <16 x ptr> [[LANE_15]], i64 4
 ; CHECK-NEXT:    br label %[[MERGE]]
 ; CHECK:       [[MERGE]]:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x ptr> [ [[MM_VECTORGEP]], %[[F0]] ], [ [[MM_VECTORGEP2]], %[[F1]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x ptr> [ [[MM_VECTORGEP1]], %[[F0]] ], [ [[MM_VECTORGEP22]], %[[F1]] ]
 ; CHECK-NEXT:    ret <16 x ptr> [[VEC_PHI]]
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll b/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll
new file mode 100644
index 000000000000..dddd63d101e6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-shuffle-ext.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p instcombine -S %s | FileCheck %s
+
+define <4 x i16> @ext_identity_mask_first_vector_first_half_4xi16(<8 x i8> %x) {
+; CHECK-LABEL: define <4 x i16> @ext_identity_mask_first_vector_first_half_4xi16(
+; CHECK-SAME: <8 x i8> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i16> %shuffle
+}
+
+define <3 x i32> @ext_identity_mask_first_vector_first_half_3xi32(<4 x i16> %x) {
+; CHECK-LABEL: define <3 x i32> @ext_identity_mask_first_vector_first_half_3xi32(
+; CHECK-SAME: <4 x i16> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[E_1]], <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i32> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <4 x i16> %x to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %e.1, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x i32> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask1(<8 x i8> %in) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask1(
+; CHECK-SAME: <8 x i8> [[IN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[IN]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %in to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask2(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask2(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  ret <4 x i16> %shuffle
+}
+
+define <5 x i32> @ext_identity_mask_first_vector_first_half_5xi32(<4 x i16> %x) {
+; CHECK-LABEL: define <5 x i32> @ext_identity_mask_first_vector_first_half_5xi32(
+; CHECK-SAME: <4 x i16> [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <4 x i16> [[X]] to <4 x i32>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[E_1]], <4 x i32> poison, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT:    ret <5 x i32> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <4 x i16> %x to <4 x i32>
+  %shuffle = shufflevector <4 x i32> %e.1, <4 x i32> %e.1, <5 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4>
+  ret <5 x i32> %shuffle
+}
+
+define <4 x i16> @ext_no_identity_mask_first_vector_second_half(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @ext_no_identity_mask_first_vector_second_half(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[X]] to <8 x i16>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> [[Y]], <4 x i32> <i32 0, i32 9, i32 1, i32 10>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 0, i32 9, i32 1, i32 10>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i16> @select_second_op(<8 x i8> %x, <8 x i16> %y) {
+; CHECK-LABEL: define <4 x i16> @select_second_op(
+; CHECK-SAME: <8 x i8> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[Y]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i16> [[SHUFFLE]]
+;
+entry:
+  %e.1 = zext <8 x i8> %x to <8 x i16>
+  %shuffle = shufflevector <8 x i16> %e.1, <8 x i16> %y, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+  ret <4 x i16> %shuffle
+}
+
+define <4 x i32> @load_i32_zext_to_v4i32(ptr %di) {
+; CHECK-LABEL: define <4 x i32> @load_i32_zext_to_v4i32(
+; CHECK-SAME: ptr [[DI:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[DI]], align 4
+; CHECK-NEXT:    [[VEC_INS:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[L]], i64 0
+; CHECK-NEXT:    [[VEC_BC:%.*]] = bitcast <2 x i32> [[VEC_INS]] to <8 x i8>
+; CHECK-NEXT:    [[E_1:%.*]] = zext <8 x i8> [[VEC_BC]] to <8 x i16>
+; CHECK-NEXT:    [[VEC_SHUFFLE:%.*]] = shufflevector <8 x i16> [[E_1]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXT_2:%.*]] = zext nneg <4 x i16> [[VEC_SHUFFLE]] to <4 x i32>
+; CHECK-NEXT:    ret <4 x i32> [[EXT_2]]
+;
+entry:
+  %l = load i32, ptr %di
+  %vec.ins = insertelement <2 x i32> <i32 poison, i32 0>, i32 %l, i64 0
+  %vec.bc = bitcast <2 x i32> %vec.ins to <8 x i8>
+  %e.1 = zext <8 x i8> %vec.bc to <8 x i16>
+  %vec.shuffle = shufflevector <8 x i16> %e.1, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %ext.2 = zext nneg <4 x i16> %vec.shuffle to <4 x i32>
+  ret <4 x i32> %ext.2
+}
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
index 8875ce1c566f..9733f1b732c3 100644
--- a/llvm/test/Transforms/InstCombine/freeze.ll
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -877,6 +877,104 @@ exit:                                             ; preds = %loop
   ret void
 }
 
+; The recurrence for the GEP offset can't produce poison so the freeze should
+; be pushed through to the ptr, but this is not currently supported.
+define void @fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
+; CHECK-LABEL: @fold_phi_gep_phi_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
+; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi ptr [ %init, %entry ], [ %i.next.fr, %loop ]
+  %off = phi i64 [ %n, %entry ], [ %off.next, %loop ]
+  %off.next = shl i64 %off, 3
+  %i.next = getelementptr i8, ptr %i, i64 %off.next
+  %i.next.fr = freeze ptr %i.next
+  %cond = icmp eq ptr %i.next.fr, %end
+  br i1 %cond, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; Offset is still guaranteed not to be poison, so the freeze could be moved
+; here if we strip inbounds from the GEP, but this is not currently supported.
+define void @fold_phi_gep_inbounds_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
+; CHECK-LABEL: @fold_phi_gep_inbounds_phi_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
+; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi ptr [ %init, %entry ], [ %i.next.fr, %loop ]
+  %off = phi i64 [ %n, %entry ], [ %off.next, %loop ]
+  %off.next = shl i64 %off, 3
+  %i.next = getelementptr inbounds i8, ptr %i, i64 %off.next
+  %i.next.fr = freeze ptr %i.next
+  %cond = icmp eq ptr %i.next.fr, %end
+  br i1 %cond, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
+; GEP can produce poison, check freeze isn't moved.
+define void @cant_fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 %n) {
+; CHECK-LABEL: @cant_fold_phi_gep_phi_offset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
+; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %loop, %entry
+  %i = phi ptr [ %init, %entry ], [ %i.next.fr, %loop ]
+  %off = phi i64 [ %n, %entry ], [ %off.next, %loop ]
+  %off.next = shl i64 %off, 3
+  %i.next = getelementptr inbounds i8, ptr %i, i64 %off.next
+  %i.next.fr = freeze ptr %i.next
+  %cond = icmp eq ptr %i.next.fr, %end
+  br i1 %cond, label %loop, label %exit
+
+exit:                                             ; preds = %loop
+  ret void
+}
+
 define void @fold_phi_multiple_insts(i32 %init, i32 %n) {
 ; CHECK-LABEL: @fold_phi_multiple_insts(
 ; CHECK-NEXT:  entry:
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index 7568c6edc429..752ff0cae5b7 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -247,11 +247,11 @@ define <2 x i1> @test13_vector2(i64 %X, <2 x ptr> %P) nounwind {
 
 define <2 x i1> @test13_fixed_fixed(i64 %X, ptr %P, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: @test13_fixed_fixed(
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[DOTSPLATINSERT]], <i64 3, i64 0>
-; CHECK-NEXT:    [[A_IDX:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[B_IDX:%.*]] = shl nsw <2 x i64> [[Y:%.*]], splat (i64 4)
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i64> [[A_IDX]], [[B_IDX]]
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds <2 x i64>, ptr [[P:%.*]], i64 0, i64 [[X:%.*]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[A1]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x ptr> [[DOTSPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds <2 x i64>, ptr [[P]], <2 x i64> [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x ptr> [[DOTSPLAT]], [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %A = getelementptr inbounds <2 x i64>, ptr %P, <2 x i64> zeroinitializer, i64 %X
@@ -688,9 +688,9 @@ define i32 @test28() nounwind  {
 ; CHECK-NEXT:    [[INDVAR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDVAR_NEXT:%.*]], [[BB10]] ]
 ; CHECK-NEXT:    [[T12_REC:%.*]] = xor i32 [[INDVAR]], -1
 ; CHECK-NEXT:    [[TMP0:%.*]] = sext i32 [[T12_REC]] to i64
-; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds [[STRUCT_X:%.*]], ptr [[T45]], i64 [[TMP0]]
+; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds i8, ptr [[T45]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[T16:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str1, ptr nonnull [[T12]]) #[[ATTR0]]
-; CHECK-NEXT:    [[T84:%.*]] = icmp eq ptr [[T12]], [[ORIENTATIONS]]
+; CHECK-NEXT:    [[T84:%.*]] = icmp eq i32 [[INDVAR]], 0
 ; CHECK-NEXT:    [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1
 ; CHECK-NEXT:    br i1 [[T84]], label [[BB17:%.*]], label [[BB10]]
 ; CHECK:       bb17:
diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll
index 260462896c39..3f104056fb1f 100644
--- a/llvm/test/Transforms/InstCombine/icmp-gep.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll
@@ -221,6 +221,34 @@ define i1 @eq_base_inbounds_commute_use(i64 %y) {
   ret i1 %r
 }
 
+define i1 @ne_base_inbounds_use_scaled(ptr %x, i64 %y) {
+; CHECK-LABEL: @ne_base_inbounds_use_scaled(
+; CHECK-NEXT:    [[G_IDX:%.*]] = shl nsw i64 [[Y:%.*]], 3
+; CHECK-NEXT:    [[G:%.*]] = getelementptr inbounds i8, ptr [[X:%.*]], i64 [[G_IDX]]
+; CHECK-NEXT:    call void @use(ptr [[G]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[Y]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr inbounds i64, ptr %x, i64 %y
+  call void @use(ptr %g)
+  %r = icmp ne ptr %g, %x
+  ret i1 %r
+}
+
+define i1 @ne_base_use_scaled(ptr %x, i64 %y) {
+; CHECK-LABEL: @ne_base_use_scaled(
+; CHECK-NEXT:    [[G_IDX_MASK:%.*]] = shl i64 [[Y:%.*]], 3
+; CHECK-NEXT:    [[G:%.*]] = getelementptr i8, ptr [[X:%.*]], i64 [[G_IDX_MASK]]
+; CHECK-NEXT:    call void @use(ptr [[G]])
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[G_IDX_MASK]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %g = getelementptr i64, ptr %x, i64 %y
+  call void @use(ptr %g)
+  %r = icmp ne ptr %g, %x
+  ret i1 %r
+}
+
 define i1 @eq_bitcast_base(ptr %p, i64 %x) {
 ; CHECK-LABEL: @eq_bitcast_base(
 ; CHECK-NEXT:    [[GEP_IDX_MASK:%.*]] = and i64 [[X:%.*]], 9223372036854775807
@@ -757,3 +785,67 @@ define i1 @gep_diff_base_same_indices_nuw_nusw(ptr %x, ptr %y, i64 %z) {
   %cmp = icmp ult ptr %gep1, %gep2
   ret i1 %cmp
 }
+
+define i1 @gep_multiple_eq(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_multiple_eq(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[DOTMASK:%.*]] = and i64 [[GEP1_IDX1]], 4611686018427387903
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[DOTMASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp eq ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_multiple_eq_commuted(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_multiple_eq_commuted(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[DOTMASK:%.*]] = and i64 [[GEP1_IDX1]], 4611686018427387903
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[DOTMASK]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp eq ptr %base, %gep2
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_nuw(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = sub i64 0, [[IDX2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IDX:%.*]], [[GEP1_IDX1]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %base, i64 %idx
+  %gep2 = getelementptr nuw i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_not_all_nuw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_not_all_nuw(
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr nuw i32, ptr [[BASE:%.*]], i64 [[IDX:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i32, ptr [[GEP1]], i64 [[IDX2:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt ptr [[GEP2]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr nuw i32, ptr %base, i64 %idx
+  %gep2 = getelementptr i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
+
+define i1 @gep_mugtiple_ugt_inbounds_nusw(ptr %base, i64 %idx, i64 %idx2) {
+; CHECK-LABEL: @gep_mugtiple_ugt_inbounds_nusw(
+; CHECK-NEXT:    [[GEP1_IDX1:%.*]] = add i64 [[IDX:%.*]], [[IDX2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[GEP1_IDX1]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %gep1 = getelementptr inbounds i32, ptr %base, i64 %idx
+  %gep2 = getelementptr nusw i32, ptr %gep1, i64 %idx2
+  %cmp = icmp ugt ptr %gep2, %base
+  ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 365c17b35a46..a090f9c4d261 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -1900,6 +1900,108 @@ define i1 @icmp_add1_sle(i32 %x, i32 %y) {
   ret i1 %cmp
 }
 
+define i1 @icmp_slt_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_slt_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -16
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_slt_offset_with_smaller_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_slt_offset_with_smaller_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -8
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_sle_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_sle_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, 16
+  %cmp = icmp sle i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_offset_with_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_ule_offset_with_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHLX]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nuw i64 %shlx, 16
+  %cmp = icmp ule i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+; TODO: Handle non-power-of-2 divisors
+define i1 @icmp_ule_offset_with_common_non_pow2_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @icmp_ule_offset_with_common_non_pow2_divisor(
+; CHECK-NEXT:    [[MULX:%.*]] = mul nuw i64 [[X:%.*]], 7
+; CHECK-NEXT:    [[MULY:%.*]] = mul nuw i64 [[Y:%.*]], 7
+; CHECK-NEXT:    [[MULX_OFFSET:%.*]] = add nuw i64 [[MULX]], 7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MULX_OFFSET]], [[MULY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %mulx = mul nuw i64 %x, 7
+  %muly = mul nuw i64 %y, 7
+  %mulx_offset = add nuw i64 %mulx, 7
+  %cmp = icmp ule i64 %mulx_offset, %muly
+  ret i1 %cmp
+}
+
+define i1 @neg_icmp_slt_offset_without_common_divisor(i64 %x, i64 %y) {
+; CHECK-LABEL: @neg_icmp_slt_offset_without_common_divisor(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], -32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, -32
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
+define i1 @neg_icmp_slt_offset_with_wrong_sign(i64 %x, i64 %y) {
+; CHECK-LABEL: @neg_icmp_slt_offset_with_wrong_sign(
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %shlx = shl i64 %x, 4
+  %shly = shl i64 %y, 4
+  %shlx_offset = add nsw i64 %shlx, 16
+  %cmp = icmp slt i64 %shlx_offset, %shly
+  ret i1 %cmp
+}
+
 define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add20_sge_add57(
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y:%.*]], 37
diff --git a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
index 331de12ddd33..c2663d863893 100644
--- a/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
+++ b/llvm/test/Transforms/InstCombine/matrix-multiplication-negation.ll
@@ -205,9 +205,10 @@ define <4 x double> @matrix_multiply_two_operands_negated_with_same_size(<2 x do
 
 define <2 x double> @matrix_multiply_two_operands_with_multiple_uses(<6 x double> %a, <3 x double> %b) {
 ; CHECK-LABEL: @matrix_multiply_two_operands_with_multiple_uses(
-; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x double> @llvm.matrix.multiply.v2f64.v6f64.v3f64(<6 x double> [[A:%.*]], <3 x double> [[B:%.*]], i32 2, i32 3, i32 1)
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x double> [[A]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[RES_3:%.*]] = fsub <2 x double> [[RES]], [[TMP1]]
+; CHECK-NEXT:    [[A_NEG:%.*]] = fneg <6 x double> [[A:%.*]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <2 x double> @llvm.matrix.multiply.v2f64.v6f64.v3f64(<6 x double> [[A]], <3 x double> [[B:%.*]], i32 2, i32 3, i32 1)
+; CHECK-NEXT:    [[RES_2:%.*]] = shufflevector <6 x double> [[A_NEG]], <6 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[RES_3:%.*]] = fadd <2 x double> [[RES_2]], [[RES]]
 ; CHECK-NEXT:    ret <2 x double> [[RES_3]]
 ;
   %a.neg = fneg <6 x double> %a
diff --git a/llvm/test/Transforms/InstCombine/pr39908.ll b/llvm/test/Transforms/InstCombine/pr39908.ll
index ca143f417fb2..5d13a331c6d2 100644
--- a/llvm/test/Transforms/InstCombine/pr39908.ll
+++ b/llvm/test/Transforms/InstCombine/pr39908.ll
@@ -7,9 +7,7 @@ target datalayout = "p:32:32"
 
 define i1 @test(ptr %p, i32 %n) {
 ; CHECK-LABEL: @test(
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[N:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[N:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i32 0, i32 %n, i32 0, i32 0
@@ -22,9 +20,7 @@ define i1 @test(ptr %p, i32 %n) {
 define i1 @test64(ptr %p, i64 %n) {
 ; CHECK-LABEL: @test64(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[TMP1]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i64 0, i64 %n, i32 0, i64 0
@@ -37,9 +33,7 @@ define i1 @test64(ptr %p, i64 %n) {
 define i1 @test64_overflow(ptr %p, i64 %n) {
 ; CHECK-LABEL: @test64_overflow(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[N:%.*]] to i32
-; CHECK-NEXT:    [[END:%.*]] = getelementptr inbounds [0 x %S], ptr [[P:%.*]], i32 0, i32 [[TMP1]], i32 0, i32 0
-; CHECK-NEXT:    [[LAST:%.*]] = getelementptr inbounds i8, ptr [[END]], i32 -8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[LAST]], [[P]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %end = getelementptr inbounds [0 x %S], ptr %p, i64 0, i64 %n, i32 0, i64 8589934592
diff --git a/llvm/test/Transforms/InstCombine/sub-after-sle-is-non-negative.ll b/llvm/test/Transforms/InstCombine/sub-after-sle-is-non-negative.ll
new file mode 100644
index 000000000000..8395895b1af6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-after-sle-is-non-negative.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+declare void @subroutine(i16)
+
+define void @test_as_arg(i8 %a, i8 %b) {
+; CHECK-LABEL: define void @test_as_arg(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_END:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext nneg i8 [[SUB]] to i16
+; CHECK-NEXT:    call void @subroutine(i16 [[CONV]])
+; CHECK-NEXT:    br label %[[COND_END]]
+; CHECK:       [[COND_END]]:
+; CHECK-NEXT:    ret void
+;
+  %cmp = icmp sgt i8 %a, %b
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %sub = sub nsw i8 %b, %a
+  %conv = sext i8 %sub to i16
+  call void @subroutine(i16 %conv)
+  br label %cond.end
+
+cond.end:
+  ret void
+}
+
+define i16 @test_as_retval(i8 %a, i8 %b) {
+; CHECK-LABEL: define i16 @test_as_retval(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_TRUE]]:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = zext nneg i8 [[SUB]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %cmp = icmp sgt i8 %a, %b
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:
+  ret i16 0
+
+cond.false:
+  %sub = sub nsw i8 %b, %a
+  %conv = sext i8 %sub to i16
+  ret i16 %conv
+}
+
+define void @test_as_arg_wrong_icmp(i8 %a, i8 %b) {
+; CHECK-LABEL: define void @test_as_arg_wrong_icmp(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_END:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[SUB]] to i16
+; CHECK-NEXT:    call void @subroutine(i16 [[CONV]])
+; CHECK-NEXT:    br label %[[COND_END]]
+; CHECK:       [[COND_END]]:
+; CHECK-NEXT:    ret void
+;
+  %cmp = icmp slt i8 %a, %b
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %sub = sub nsw i8 %b, %a
+  %conv = sext i8 %sub to i16
+  call void @subroutine(i16 %conv)
+  br label %cond.end
+
+cond.end:
+  ret void
+}
+
+define void @test_as_arg_missing_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: define void @test_as_arg_missing_nsw(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_END:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[SUB]] to i16
+; CHECK-NEXT:    call void @subroutine(i16 [[CONV]])
+; CHECK-NEXT:    br label %[[COND_END]]
+; CHECK:       [[COND_END]]:
+; CHECK-NEXT:    ret void
+;
+  %cmp = icmp sgt i8 %a, %b
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %sub = sub i8 %b, %a
+  %conv = sext i8 %sub to i16
+  call void @subroutine(i16 %conv)
+  br label %cond.end
+
+cond.end:
+  ret void
+}
+
+define i16 @test_as_retval_wrong_icmp(i8 %a, i8 %b) {
+; CHECK-LABEL: define i16 @test_as_retval_wrong_icmp(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_TRUE]]:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[SUB]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %cmp = icmp slt i8 %a, %b
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:
+  ret i16 0
+
+cond.false:
+  %sub = sub nsw i8 %b, %a
+  %conv = sext i8 %sub to i16
+  ret i16 %conv
+}
+
+define i16 @test_as_retval_missing_nsw(i8 %a, i8 %b) {
+; CHECK-LABEL: define i16 @test_as_retval_missing_nsw(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], [[B]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+; CHECK:       [[COND_TRUE]]:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[B]], [[A]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[SUB]] to i16
+; CHECK-NEXT:    ret i16 [[CONV]]
+;
+  %cmp = icmp sgt i8 %a, %b
+  br i1 %cmp, label %cond.true, label %cond.false
+
+cond.true:
+  ret i16 0
+
+cond.false:
+  %sub = sub i8 %b, %a
+  %conv = sext i8 %sub to i16
+  ret i16 %conv
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
index 45a30350aafe..11af6b4a0197 100644
--- a/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -943,6 +943,70 @@ define i64 @multiple_geps_two_chains_gep_base(ptr %base, i64 %base.idx, i64 %idx
   ret i64 %d
 }
 
+define i64 @multiple_geps_two_chains_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4) {
+; CHECK-LABEL: @multiple_geps_two_chains_multi_use(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
+; CHECK-NEXT:    [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
+; CHECK-NEXT:    [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P3_IDX]]
+; CHECK-NEXT:    [[P4_IDX1:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[P4_IDX1]]
+; CHECK-NEXT:    call void @use(ptr [[P5]])
+; CHECK-NEXT:    call void @use(ptr [[P4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[P3_IDX]], [[P4_IDX1]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
+  %p2 = getelementptr inbounds i32, ptr %p1, i64 %idx2
+  %p3 = getelementptr inbounds i32, ptr %base, i64 %idx3
+  %p4 = getelementptr inbounds i32, ptr %p3, i64 %idx4
+  call void @use(ptr %p2)
+  call void @use(ptr %p4)
+  %i1 = ptrtoint ptr %p4 to i64
+  %i2 = ptrtoint ptr %p2 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
+define i64 @multiple_geps_two_chains_partial_multi_use(ptr %base, i64 %idx1, i64 %idx2, i64 %idx3, i64 %idx4, i64 %idx5, i64 %idx6) {
+; CHECK-LABEL: @multiple_geps_two_chains_partial_multi_use(
+; CHECK-NEXT:    [[P2_IDX:%.*]] = shl nsw i64 [[IDX2:%.*]], 2
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 [[P2_IDX]]
+; CHECK-NEXT:    [[P4_IDX:%.*]] = shl nsw i64 [[IDX4:%.*]], 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[P4_IDX]]
+; CHECK-NEXT:    [[P3_IDX:%.*]] = shl nsw i64 [[IDX3:%.*]], 2
+; CHECK-NEXT:    [[P4_IDX1:%.*]] = shl nsw i64 [[IDX7:%.*]], 2
+; CHECK-NEXT:    [[P5:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[P4_IDX1]]
+; CHECK-NEXT:    [[P5_IDX:%.*]] = shl nsw i64 [[IDX5:%.*]], 2
+; CHECK-NEXT:    [[P4:%.*]] = getelementptr inbounds i8, ptr [[P5]], i64 [[P5_IDX]]
+; CHECK-NEXT:    [[P6_IDX:%.*]] = shl nsw i64 [[IDX6:%.*]], 2
+; CHECK-NEXT:    call void @use(ptr [[P3]])
+; CHECK-NEXT:    call void @use(ptr [[P4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[P2_IDX]], [[P4_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[TMP1]], [[P3_IDX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[P4_IDX1]], [[P5_IDX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[P6_IDX]]
+; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub nsw i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i64 [[GEPDIFF]]
+;
+  %p1 = getelementptr inbounds i32, ptr %base, i64 %idx1
+  %p2 = getelementptr inbounds i32, ptr %p1, i64 %idx2
+  %p3 = getelementptr inbounds i32, ptr %p2, i64 %idx3
+  %p4 = getelementptr inbounds i32, ptr %base, i64 %idx4
+  %p5 = getelementptr inbounds i32, ptr %p4, i64 %idx5
+  %p6 = getelementptr inbounds i32, ptr %p5, i64 %idx6
+  call void @use(ptr %p2)
+  call void @use(ptr %p5)
+  %i1 = ptrtoint ptr %p6 to i64
+  %i2 = ptrtoint ptr %p3 to i64
+  %d = sub i64 %i2, %i1
+  ret i64 %d
+}
+
 define i64 @multiple_geps_inbounds(ptr %base, i64 %idx, i64 %idx2) {
 ; CHECK-LABEL: @multiple_geps_inbounds(
 ; CHECK-NEXT:    [[D:%.*]] = add nsw i64 [[IDX:%.*]], [[IDX2:%.*]]
@@ -1025,3 +1089,72 @@ define <2 x i64> @splat_geps_multiple(ptr %base, i64 %idx0, <2 x i64> %idx1, <2
   %d = sub <2 x i64> %gep2.int, %gep1.int
   ret <2 x i64> %d
 }
+
+define i64 @nuw_ptrdiff_shl_nsw(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_shl_nsw(
+; CHECK-NEXT:    [[OFFSET:%.*]] = shl nuw nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = shl nsw i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_shl_nonsw(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_shl_nonsw(
+; CHECK-NEXT:    [[OFFSET:%.*]] = shl i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = shl i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_mul_nsw_nneg_scale(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_nneg_scale(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, 3
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+define i64 @nuw_ptrdiff_mul_nsw_unknown_scale(ptr %base, i64 %idx, i64 %scale) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_unknown_scale(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nsw i64 [[IDX:%.*]], [[SCALE:%.*]]
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, %scale
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
+
+declare void @usei64(i64)
+
+define i64 @nuw_ptrdiff_mul_nsw_nneg_scale_multiuse(ptr %base, i64 %idx) {
+; CHECK-LABEL: @nuw_ptrdiff_mul_nsw_nneg_scale_multiuse(
+; CHECK-NEXT:    [[OFFSET:%.*]] = mul nsw i64 [[IDX:%.*]], 3
+; CHECK-NEXT:    call void @usei64(i64 [[OFFSET]])
+; CHECK-NEXT:    ret i64 [[OFFSET]]
+;
+  %offset = mul nsw i64 %idx, 3
+  call void @usei64(i64 %offset)
+  %gep = getelementptr inbounds i8, ptr %base, i64 %offset
+  %lhs = ptrtoint ptr %gep to i64
+  %rhs = ptrtoint ptr %base to i64
+  %diff = sub nuw i64 %lhs, %rhs
+  ret i64 %diff
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
index a240dfe7d271..cbfdebc6cb41 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
@@ -525,9 +525,7 @@ define ptr @gep_splat_base_w_s_idx(ptr %base) {
 
 define ptr @gep_splat_base_w_cv_idx(ptr %base) {
 ; CHECK-LABEL: @gep_splat_base_w_cv_idx(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> <i64 poison, i64 1>
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 4
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> poison, ptr %base, i32 0
@@ -539,9 +537,8 @@ define ptr @gep_splat_base_w_cv_idx(ptr %base) {
 
 define ptr @gep_splat_base_w_vidx(ptr %base, <2 x i64> %idxvec) {
 ; CHECK-LABEL: @gep_splat_base_w_vidx(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]]
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[IDXVEC:%.*]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i32, ptr [[BASE:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> poison, ptr %base, i32 0
@@ -597,10 +594,7 @@ define ptr @gep_sbase_w_splat_idx(ptr %base, i64 %idx) {
 }
 define ptr @gep_splat_both(ptr %base, i64 %idx) {
 ; CHECK-LABEL: @gep_splat_both(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> poison, i64 [[IDX:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC2]]
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i32, ptr [[BASE:%.*]], i64 [[IDX:%.*]]
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> poison, ptr %base, i32 0
@@ -641,9 +635,9 @@ define ptr @gep_demanded_lane_undef(ptr %base, i64 %idx) {
 ;; indices.
 define ptr @PR41624(<2 x ptr> %a) {
 ; CHECK-LABEL: @PR41624(
-; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> splat (i64 5), <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x ptr> [[W]], i64 0
-; CHECK-NEXT:    ret ptr [[R]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x ptr> [[W:%.*]], i64 0
+; CHECK-NEXT:    [[R1:%.*]] = getelementptr i8, ptr [[R]], i64 40
+; CHECK-NEXT:    ret ptr [[R1]]
 ;
   %w = getelementptr { i32, i32 }, <2 x ptr> %a, <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
   %r = extractelement <2 x ptr> %w, i32 0
@@ -657,8 +651,8 @@ define ptr @PR41624(<2 x ptr> %a) {
 define ptr @zero_sized_type_extract(<4 x i64> %arg, i64 %arg1) {
 ; CHECK-LABEL: @zero_sized_type_extract(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds [0 x i32], <4 x ptr> <ptr @global, ptr poison, ptr poison, ptr poison>, <4 x i64> <i64 0, i64 poison, i64 poison, i64 poison>, <4 x i64> [[ARG:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = extractelement <4 x ptr> [[T]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i64> [[ARG:%.*]], i64 0
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds [0 x i32], ptr @global, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    ret ptr [[T2]]
 ;
 bb:
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
index ee7ef9955e64..f0ef33dcc6b4 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -528,9 +528,7 @@ define ptr @gep_splat_base_w_s_idx(ptr %base) {
 
 define ptr @gep_splat_base_w_cv_idx(ptr %base) {
 ; CHECK-LABEL: @gep_splat_base_w_cv_idx(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> <i64 poison, i64 1>
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 4
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> undef, ptr %base, i32 0
@@ -542,9 +540,8 @@ define ptr @gep_splat_base_w_cv_idx(ptr %base) {
 
 define ptr @gep_splat_base_w_vidx(ptr %base, <2 x i64> %idxvec) {
 ; CHECK-LABEL: @gep_splat_base_w_vidx(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]]
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[IDXVEC:%.*]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i32, ptr [[BASE:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> undef, ptr %base, i32 0
@@ -600,10 +597,7 @@ define ptr @gep_sbase_w_splat_idx(ptr %base, i64 %idx) {
 }
 define ptr @gep_splat_both(ptr %base, i64 %idx) {
 ; CHECK-LABEL: @gep_splat_both(
-; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x ptr> poison, ptr [[BASE:%.*]], i64 1
-; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> poison, i64 [[IDX:%.*]], i64 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x ptr> [[BASEVEC2]], <2 x i64> [[IDXVEC2]]
-; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x ptr> [[GEP]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = getelementptr i32, ptr [[BASE:%.*]], i64 [[IDX:%.*]]
 ; CHECK-NEXT:    ret ptr [[EE]]
 ;
   %basevec1 = insertelement <2 x ptr> undef, ptr %base, i32 0
@@ -644,9 +638,9 @@ define ptr @gep_demanded_lane_undef(ptr %base, i64 %idx) {
 ;; indices.
 define ptr @PR41624(<2 x ptr> %a) {
 ; CHECK-LABEL: @PR41624(
-; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> splat (i64 5), <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x ptr> [[W]], i64 0
-; CHECK-NEXT:    ret ptr [[R]]
+; CHECK-NEXT:    [[R:%.*]] = extractelement <2 x ptr> [[W:%.*]], i64 0
+; CHECK-NEXT:    [[R1:%.*]] = getelementptr i8, ptr [[R]], i64 40
+; CHECK-NEXT:    ret ptr [[R1]]
 ;
   %w = getelementptr { i32, i32 }, <2 x ptr> %a, <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
   %r = extractelement <2 x ptr> %w, i32 0
@@ -660,8 +654,8 @@ define ptr @PR41624(<2 x ptr> %a) {
 define ptr @zero_sized_type_extract(<4 x i64> %arg, i64 %arg1) {
 ; CHECK-LABEL: @zero_sized_type_extract(
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[T:%.*]] = getelementptr inbounds [0 x i32], <4 x ptr> <ptr @global, ptr poison, ptr poison, ptr poison>, <4 x i64> <i64 0, i64 poison, i64 poison, i64 poison>, <4 x i64> [[ARG:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = extractelement <4 x ptr> [[T]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i64> [[ARG:%.*]], i64 0
+; CHECK-NEXT:    [[T2:%.*]] = getelementptr inbounds [0 x i32], ptr @global, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    ret ptr [[T2]]
 ;
 bb:
diff --git a/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll b/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
new file mode 100644
index 000000000000..1ec3dc3e4b40
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
@@ -0,0 +1,52 @@
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+define float @extract_from_zero_init_shuffle(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_zero_init_shuffle(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 0
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> zeroinitializer
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+
+define float @extract_from_general_splat(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_general_splat(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_general_scalable_splat(<vscale x 2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_general_scalable_splat(
+; CHECK-NEXT:    %extract = extractelement <vscale x 2 x float> %vec, i64 0
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <vscale x 2 x float> %vec, <vscale x 2 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %extract = extractelement <vscale x 4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_splat_with_poison_0(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_splat_with_poison_0(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
+
+define float @extract_from_splat_with_poison_1(<2 x float> %vec, i64 %idx) {
+; CHECK-LABEL: @extract_from_splat_with_poison_1(
+; CHECK-NEXT:    %extract = extractelement <2 x float> %vec, i64 1
+; CHECK-NEXT:    ret float %extract
+;
+  %shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 poison, i32 1, i32 1>
+  %extract = extractelement <4 x float> %shuffle, i64 %idx
+  ret float %extract
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
index 9aa050e8cd50..86fc5bbf72e7 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
@@ -61,8 +61,7 @@ define float @test6(<4 x float> %X) {
 
 define float @testvscale6(<vscale x 4 x float> %X) {
 ; CHECK-LABEL: @testvscale6(
-; CHECK-NEXT:    [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
@@ -1278,9 +1277,9 @@ define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
 
 define <2 x float> @fneg(<2 x float> %x) {
 ; CHECK-LABEL: @fneg(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %splat = shufflevector <2 x float> %x, <2 x float> poison, <2 x i32> zeroinitializer
   %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle.ll b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
index 83919e743d38..39f76f18b13c 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle.ll
@@ -67,8 +67,7 @@ define float @test6(<4 x float> %X) {
 
 define float @testvscale6(<vscale x 4 x float> %X) {
 ; CHECK-LABEL: @testvscale6(
-; CHECK-NEXT:    [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
@@ -1382,9 +1381,9 @@ define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
 
 define <2 x float> @fneg(<2 x float> %x) {
 ; CHECK-LABEL: @fneg(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
   %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
@@ -1906,9 +1905,9 @@ define <4 x i32> @PR46872(<4 x i32> %x) {
 
 define <2 x float> @fabs_unary_shuf(<2 x float> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = call nnan nsz <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP1]])
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = call nnan nsz <2 x float> @llvm.fabs.v2f32(<2 x float> [[TMP1:%.*]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %nx = call nsz nnan <2 x float> @llvm.fabs.v2f32(<2 x float> %x)
   %r = shufflevector <2 x float> %nx, <2 x float> poison, <2 x i32> <i32 1, i32 0>
@@ -1917,9 +1916,9 @@ define <2 x float> @fabs_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fabs_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = call ninf <4 x half> @llvm.fabs.v4f16(<4 x half> [[TMP1]])
-; CHECK-NEXT:    ret <4 x half> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = call ninf <2 x half> @llvm.fabs.v2f16(<2 x half> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    ret <4 x half> [[TMP1]]
 ;
   %nx = call ninf <2 x half> @llvm.fabs.v2f16(<2 x half> %x)
   %r = shufflevector <2 x half> %nx, <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
@@ -1928,9 +1927,9 @@ define <4 x half> @fabs_unary_shuf_widen(<2 x half> %x) {
 
 define <2 x double> @fabs_unary_shuf_narrow(<4 x double> %x) {
 ; CHECK-LABEL: @fabs_unary_shuf_narrow(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = call nsz <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP1]])
-; CHECK-NEXT:    ret <2 x double> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = call nsz <4 x double> @llvm.fabs.v4f64(<4 x double> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %nx = call nsz <4 x double> @llvm.fabs.v4f64(<4 x double> %x)
   %r = shufflevector <4 x double> %nx, <4 x double> poison, <2 x i32> <i32 3, i32 0>
@@ -2021,9 +2020,9 @@ define <2 x float> @fabs_shuf_use3(<2 x float> %x, <2 x float> %y) {
 
 define <2 x float> @fneg_unary_shuf(<2 x float> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = fneg nnan nsz <2 x float> [[TMP1]]
-; CHECK-NEXT:    ret <2 x float> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = fneg nnan nsz <2 x float> [[TMP1:%.*]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <2 x float> [[R]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    ret <2 x float> [[R1]]
 ;
   %nx = fneg nsz nnan <2 x float> %x
   %r = shufflevector <2 x float> %nx, <2 x float> poison, <2 x i32> <i32 1, i32 0>
@@ -2032,9 +2031,9 @@ define <2 x float> @fneg_unary_shuf(<2 x float> %x) {
 
 define <4 x half> @fneg_unary_shuf_widen(<2 x half> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_widen(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X:%.*]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = fneg ninf <4 x half> [[TMP1]]
-; CHECK-NEXT:    ret <4 x half> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = fneg ninf <2 x half> [[X1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[X]], <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
+; CHECK-NEXT:    ret <4 x half> [[TMP1]]
 ;
   %nx = fneg ninf <2 x half> %x
   %r = shufflevector <2 x half> %nx, <2 x half> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 poison>
@@ -2043,9 +2042,9 @@ define <4 x half> @fneg_unary_shuf_widen(<2 x half> %x) {
 
 define <2 x double> @fneg_unary_shuf_narrow(<4 x double> %x) {
 ; CHECK-LABEL: @fneg_unary_shuf_narrow(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X:%.*]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = fneg nsz <2 x double> [[TMP1]]
-; CHECK-NEXT:    ret <2 x double> [[R]]
+; CHECK-NEXT:    [[X:%.*]] = fneg nsz <4 x double> [[X1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[X]], <4 x double> poison, <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
   %nx = fneg nsz <4 x double> %x
   %r = shufflevector <4 x double> %nx, <4 x double> poison, <2 x i32> <i32 3, i32 0>
diff --git a/llvm/test/Transforms/InstCombine/vector-reverse.ll b/llvm/test/Transforms/InstCombine/vector-reverse.ll
index c9c68d2241b3..ee60d31d1ddb 100644
--- a/llvm/test/Transforms/InstCombine/vector-reverse.ll
+++ b/llvm/test/Transforms/InstCombine/vector-reverse.ll
@@ -17,6 +17,18 @@ define <vscale x 4 x i32> @binop_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i3
   ret <vscale x 4 x i32> %add
 }
 
+define <vscale x 4 x i32> @binop_intrinsic_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse(
+; CHECK-NEXT:    [[ADD:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[A_REV:%.*]], <vscale x 4 x i32> [[B_REV:%.*]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %a.rev has multiple uses
 define <vscale x 4 x i32> @binop_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @binop_reverse_1(
@@ -33,6 +45,22 @@ define <vscale x 4 x i32> @binop_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %add
 }
 
+; %a.rev has multiple uses
+define <vscale x 4 x i32> @binop_intrinsic_reverse_1(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse_1(
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[B1:%.*]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP1]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  call void @use_nxv4i32(<vscale x 4 x i32>  %a.rev)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %b.rev has multiple uses
 define <vscale x 4 x i32> @binop_reverse_2(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @binop_reverse_2(
@@ -67,6 +95,24 @@ define <vscale x 4 x i32> @binop_reverse_3(<vscale x 4 x i32> %a, <vscale x 4 x
   ret <vscale x 4 x i32> %add
 }
 
+; %a.rev and %b.rev have multiple uses
+define <vscale x 4 x i32> @binop_intrinsic_reverse_3(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @binop_intrinsic_reverse_3(
+; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]])
+; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[A_REV]])
+; CHECK-NEXT:    call void @use_nxv4i32(<vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    [[SMAX:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[A_REV]], <vscale x 4 x i32> [[B_REV]])
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[SMAX]]
+;
+  %a.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
+  %b.rev = tail call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %b)
+  call void @use_nxv4i32(<vscale x 4 x i32> %a.rev)
+  call void @use_nxv4i32(<vscale x 4 x i32> %b.rev)
+  %smax = call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a.rev, <vscale x 4 x i32> %b.rev)
+  ret <vscale x 4 x i32> %smax
+}
+
 ; %a.rev used as both operands
 define <vscale x 4 x i32> @binop_reverse_4(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: @binop_reverse_4(
@@ -161,9 +207,9 @@ define <vscale x 4 x i32> @binop_reverse_splat_LHS_1(<vscale x 4 x i32> %a, i32
 
 define <vscale x 4 x float> @unop_reverse(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: @unop_reverse(
-; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[A:%.*]])
-; CHECK-NEXT:    [[NEG:%.*]] = fneg fast <vscale x 4 x float> [[A_REV]]
-; CHECK-NEXT:    ret <vscale x 4 x float> [[NEG]]
+; CHECK-NEXT:    [[NEG:%.*]] = fneg fast <vscale x 4 x float> [[A_REV:%.*]]
+; CHECK-NEXT:    [[NEG1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[NEG]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[NEG1]]
 ;
   %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
   %neg = fneg fast <vscale x 4 x float> %a.rev
@@ -184,6 +230,17 @@ define <vscale x 4 x float> @unop_reverse_1(<vscale x 4 x float> %a) {
   ret <vscale x 4 x float> %neg
 }
 
+define <vscale x 4 x float> @unop_intrinsic_reverse(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @unop_intrinsic_reverse(
+; CHECK-NEXT:    [[NEG:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    [[ABS:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[NEG]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ABS]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %abs = call <vscale x 4 x float> @llvm.fabs(<vscale x 4 x float> %a.rev)
+  ret <vscale x 4 x float> %abs
+}
+
 define <vscale x 4 x i1> @icmp_reverse(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
 ; CHECK-LABEL: @icmp_reverse(
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <vscale x 4 x i32> [[A:%.*]], [[B:%.*]]
@@ -629,6 +686,18 @@ define <vscale x 4 x float> @reverse_binop_reverse(<vscale x 4 x float> %a, <vsc
   ret <vscale x 4 x float> %add.rev
 }
 
+define <vscale x 4 x float> @reverse_binop_intrinsic_reverse(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: @reverse_binop_intrinsic_reverse(
+; CHECK-NEXT:    [[ADD:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]], <vscale x 4 x float> [[B_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ADD]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %b)
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a.rev, <vscale x 4 x float> %b.rev)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
 define <vscale x 4 x float> @reverse_binop_reverse_splat_RHS(<vscale x 4 x float> %a, float %b) {
 ; CHECK-LABEL: @reverse_binop_reverse_splat_RHS(
 ; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
@@ -659,6 +728,63 @@ define <vscale x 4 x float> @reverse_binop_reverse_splat_LHS(<vscale x 4 x float
   ret <vscale x 4 x float> %div.rev
 }
 
+define <vscale x 4 x float> @reverse_binop_reverse_intrinsic_splat_RHS(<vscale x 4 x float> %a, float %b) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_RHS(
+; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[B_INSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]], <vscale x 4 x float> [[B_SPLAT]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[MAXNUM]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.insert = insertelement <vscale x 4 x float> poison, float %b, i32 0
+  %b.splat = shufflevector <vscale x 4 x float> %b.insert, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a.rev, <vscale x 4 x float> %b.splat)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
+define <vscale x 4 x float> @reverse_binop_reverse_intrinsic_splat_LHS(<vscale x 4 x float> %a, float %b) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_LHS(
+; CHECK-NEXT:    [[B_INSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[B_INSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[MAXNUM:%.*]] = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> [[B_SPLAT]], <vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[MAXNUM]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %b.insert = insertelement <vscale x 4 x float> poison, float %b, i32 0
+  %b.splat = shufflevector <vscale x 4 x float> %b.insert, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
+  %maxnum = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %b.splat, <vscale x 4 x float> %a.rev)
+  %maxnum.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %maxnum)
+  ret <vscale x 4 x float> %maxnum.rev
+}
+
+; Negative test: Make sure that splats with poison aren't considered splats
+define <4 x float> @reverse_binop_reverse_intrinsic_splat_with_poison(<4 x float> %a) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_splat_with_poison(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> <float 1.000000e+00, float poison, float 1.000000e+00, float 1.000000e+00>)
+; CHECK-NEXT:    [[MAXNUM:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[MAXNUM_REV:%.*]] = tail call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> [[MAXNUM]])
+; CHECK-NEXT:    ret <4 x float> [[MAXNUM_REV]]
+;
+  %a.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %a)
+  %maxnum = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 1.0, float 1.0, float poison, float 1.0>, <4 x float> %a.rev)
+  %maxnum.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %maxnum)
+  ret <4 x float> %maxnum.rev
+}
+
+define <4 x float> @reverse_binop_reverse_intrinsic_constant_RHS(<4 x float> %a) {
+; CHECK-LABEL: @reverse_binop_reverse_intrinsic_constant_RHS(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[A:%.*]], <4 x float> <float 3.000000e+00, float 2.000000e+00, float 1.000000e+00, float 0.000000e+00>)
+; CHECK-NEXT:    [[MAXNUM:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[MAXNUM_REV:%.*]] = tail call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> [[MAXNUM]])
+; CHECK-NEXT:    ret <4 x float> [[MAXNUM_REV]]
+;
+  %a.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %a)
+  %maxnum = call <4 x float> @llvm.maxnum.v4f32(<4 x float> <float 0.0, float 1.0, float 2.0, float 3.0>, <4 x float> %a.rev)
+  %maxnum.rev = tail call <4 x float> @llvm.vector.reverse(<4 x float> %maxnum)
+  ret <4 x float> %maxnum.rev
+}
+
 define <vscale x 4 x i1> @reverse_fcmp_reverse(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
 ; CHECK-LABEL: @reverse_fcmp_reverse(
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp fast olt <vscale x 4 x float> [[A:%.*]], [[B:%.*]]
@@ -695,6 +821,27 @@ define <vscale x 4 x float> @reverse_unop_reverse(<vscale x 4 x float> %a) {
   ret <vscale x 4 x float> %neg.rev
 }
 
+define <vscale x 4 x float> @reverse_unop_intrinsic_reverse(<vscale x 4 x float> %a) {
+; CHECK-LABEL: @reverse_unop_intrinsic_reverse(
+; CHECK-NEXT:    [[ABS:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> [[A_REV:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[ABS]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %abs = call <vscale x 4 x float> @llvm.fabs(<vscale x 4 x float> %a.rev)
+  %abs.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %abs)
+  ret <vscale x 4 x float> %abs.rev
+}
+
+define <vscale x 4 x float> @reverse_unop_intrinsic_reverse_scalar_arg(<vscale x 4 x float> %a, i32 %power) {
+; CHECK-LABEL: @reverse_unop_intrinsic_reverse_scalar_arg(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.powi.nxv4f32.i32(<vscale x 4 x float> [[A:%.*]], i32 [[POWER:%.*]])
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP1]]
+;
+  %a.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
+  %powi = call <vscale x 4 x float> @llvm.powi.nxv4f32(<vscale x 4 x float> %a.rev, i32 %power)
+  %powi.rev = tail call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %powi)
+  ret <vscale x 4 x float> %powi.rev
+}
 
 declare void @use_nxv4i1(<vscale x 4 x i1>)
 declare void @use_nxv4i32(<vscale x 4 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
index 994ecc2246bb..b882d7dfff91 100644
--- a/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
@@ -55,7 +55,7 @@ define <2 x i1> @test5(<2 x ptr> %a) {
 
 define <2 x ptr> @test7(<2 x ptr> %a) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> <i64 5, i64 9>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> <i64 5, i64 9>, i32 0
 ; CHECK-NEXT:    ret <2 x ptr> [[W]]
 ;
   %w = getelementptr {i32, i32}, <2 x ptr> %a, <2 x i32> <i32 5, i32 9>, <2 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/vector_gep1.ll b/llvm/test/Transforms/InstCombine/vector_gep1.ll
index ab99ee71fc8d..92de12213bee 100644
--- a/llvm/test/Transforms/InstCombine/vector_gep1.ll
+++ b/llvm/test/Transforms/InstCombine/vector_gep1.ll
@@ -55,7 +55,7 @@ define <2 x i1> @test5(<2 x ptr> %a) {
 
 define <2 x ptr> @test7(<2 x ptr> %a) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> <i64 5, i64 9>, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[W:%.*]] = getelementptr { i32, i32 }, <2 x ptr> [[A:%.*]], <2 x i64> <i64 5, i64 9>, i32 0
 ; CHECK-NEXT:    ret <2 x ptr> [[W]]
 ;
   %w = getelementptr {i32, i32}, <2 x ptr> %a, <2 x i32> <i32 5, i32 9>, <2 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
index 2655c2035460..36ed39a3d324 100644
--- a/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
@@ -89,12 +89,12 @@ define i8 @extractelement_bitcast_insert_extra_use_bitcast(<vscale x 2 x i32> %a
   ret i8 %r
 }
 
+; while it may be that the extract is out-of-bounds, any valid index
+; is going to yield %v (because the mask is all-zeros).
+
 define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -104,10 +104,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 
 define i32 @extractelement_shuffle_invalid_index(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_invalid_index(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
index 07090e9099ae..9ac8a92abb68 100644
--- a/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_extractelement.ll
@@ -53,12 +53,12 @@ define i8 @extractelement_bitcast_useless_insert(<vscale x 2 x i32> %a, i32 %x)
   ret i8 %r
 }
 
+; while in these tests it may be that the extract is out-of-bounds,
+; any valid index is going to yield %v (because the mask is all-zeros).
+
 define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
@@ -68,10 +68,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
 
 define i32 @extractelement_shuffle_invalid_index(i32 %v) {
 ; CHECK-LABEL: @extractelement_shuffle_invalid_index(
-; CHECK-NEXT:    [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
-; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
-; CHECK-NEXT:    ret i32 [[R]]
+; CHECK-NEXT:    ret i32 [[V:%.*]]
 ;
   %in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
   %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll b/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll
index c5c17d65524c..d824d6d35643 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/atan-intrinsic.ll
@@ -1,6 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
-; XFAIL: target={{.*}}-aix{{.*}}
 
 define double @test_atan_0() {
 ; CHECK-LABEL: define double @test_atan_0() {
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/calls.ll b/llvm/test/Transforms/InstSimplify/ConstProp/calls.ll
index 61a30c781c0f..26fb8c0d7a1c 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/calls.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/calls.ll
@@ -202,5 +202,17 @@ entry:
   ret float %0
 }
 
+define float @test_atan_negzero() nounwind uwtable ssp {
+entry:
+; CHECK-LABEL: @test_atan_negzero(
+; CHECK:    ret float -0.000000e+00
+;
+; FNOBUILTIN-LABEL: @test_atan_negzero(
+; FNOBUILTIN:  ret float -0.000000e+00
+;
+  %1 = call float @atanf(float -0.0)
+  ret float %1
+}
+
 declare double @llvm.pow.f64(double, double) nounwind readonly
 declare float @llvm.pow.f32(float, float) nounwind readonly
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 910750adf9d4..de71af414c25 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -36,7 +36,7 @@ define {i8, i1} @test_uadd3(i8 %v) {
 
 define {i8, i1} @test_uadd3_poison(i8 %v) {
 ; CHECK-LABEL: @test_uadd3_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %v, i8 poison)
   ret {i8, i1} %result
@@ -52,7 +52,7 @@ define {i8, i1} @test_uadd4(i8 %v) {
 
 define {i8, i1} @test_uadd4_poison(i8 %v) {
 ; CHECK-LABEL: @test_uadd4_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 poison, i8 %v)
   ret {i8, i1} %result
@@ -86,7 +86,7 @@ define {i8, i1} @test_sadd3(i8 %v) {
 
 define {i8, i1} @test_sadd3_poison(i8 %v) {
 ; CHECK-LABEL: @test_sadd3_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v, i8 poison)
   ret {i8, i1} %result
@@ -102,7 +102,7 @@ define {i8, i1} @test_sadd4(i8 %v) {
 
 define {i8, i1} @test_sadd4_poison(i8 %v) {
 ; CHECK-LABEL: @test_sadd4_poison(
-; CHECK-NEXT:    ret { i8, i1 } { i8 -1, i1 false }
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 poison, i8 %v)
   ret {i8, i1} %result
@@ -126,7 +126,7 @@ define {i8, i1} @test_usub2(i8 %V) {
 
 define {i8, i1} @test_usub2_poison(i8 %V) {
 ; CHECK-LABEL: @test_usub2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -142,7 +142,7 @@ define {i8, i1} @test_usub3(i8 %V) {
 
 define {i8, i1} @test_usub3_poison(i8 %V) {
 ; CHECK-LABEL: @test_usub3_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -166,7 +166,7 @@ define {i8, i1} @test_ssub2(i8 %V) {
 
 define {i8, i1} @test_ssub2_poison(i8 %V) {
 ; CHECK-LABEL: @test_ssub2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -182,7 +182,7 @@ define {i8, i1} @test_ssub3(i8 %V) {
 
 define {i8, i1} @test_ssub3_poison(i8 %V) {
 ; CHECK-LABEL: @test_ssub3_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -206,7 +206,7 @@ define {i8, i1} @test_umul2(i8 %V) {
 
 define {i8, i1} @test_umul2_poison(i8 %V) {
 ; CHECK-LABEL: @test_umul2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -230,7 +230,7 @@ define {i8, i1} @test_umul4(i8 %V) {
 
 define {i8, i1} @test_umul4_poison(i8 %V) {
 ; CHECK-LABEL: @test_umul4_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
@@ -254,7 +254,7 @@ define {i8, i1} @test_smul2(i8 %V) {
 
 define {i8, i1} @test_smul2_poison(i8 %V) {
 ; CHECK-LABEL: @test_smul2_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %V, i8 poison)
   ret {i8, i1} %x
@@ -278,7 +278,7 @@ define {i8, i1} @test_smul4(i8 %V) {
 
 define {i8, i1} @test_smul4_poison(i8 %V) {
 ; CHECK-LABEL: @test_smul4_poison(
-; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+; CHECK-NEXT:    ret { i8, i1 } poison
 ;
   %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 poison, i8 %V)
   ret {i8, i1} %x
diff --git a/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
index e45aa3fd09ce..8578aa9fa84b 100644
--- a/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/fold-intrinsics.ll
@@ -33,7 +33,7 @@ define void @powi(double %V, ptr%P) {
 define void @powi_i16(float %V, ptr%P) {
 ; CHECK-LABEL: @powi_i16(
 ; CHECK-NEXT:    store volatile float 1.000000e+00, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    store volatile float [[V:%.*]], ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float [[D:%.*]], ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %B = tail call float @llvm.powi.f32.i16(float %V, i16 0) nounwind
@@ -44,3 +44,46 @@ define void @powi_i16(float %V, ptr%P) {
 
   ret void
 }
+
+define i32 @test_ctpop_poison(i32 %a) {
+; CHECK-LABEL: @test_ctpop_poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %res = tail call i32 @llvm.ctpop.i32(i32 poison)
+  ret i32 %res
+}
+
+define void @pow_poison(i16 %arg_int,float %arg_flt, ptr %P) {
+; CHECK-LABEL: @pow_poison(
+; CHECK-NEXT:    store volatile float poison, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile float poison, ptr [[P]], align 4
+; CHECK-NEXT:    store volatile <2 x float> poison, ptr [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %2 = tail call float @llvm.powi(float poison, i16 %arg_int) nounwind
+  store volatile float %2, ptr %P
+
+  %3 = tail call float @llvm.pow(float poison, float %arg_flt) nounwind
+  store volatile float %3, ptr %P
+
+  %4 = tail call float @llvm.powi(float %arg_flt, i16 poison) nounwind
+  store volatile float %4, ptr %P
+
+  %5 = tail call float @llvm.pow(float %arg_flt, float poison) nounwind
+  store volatile float %5, ptr %P
+
+  %6 = tail call float @llvm.powi(float poison, i16 poison) nounwind
+  store volatile float %6, ptr %P
+
+  %7 = tail call float @llvm.pow(float poison, float poison) nounwind
+  store volatile float %7, ptr %P
+
+  %8 = tail call <2 x float> @llvm.pow(<2 x float> poison, <2 x float> poison) nounwind
+  store volatile <2 x float> %8, ptr %P
+
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index 25a35062e60b..ead4c0153516 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -90,7 +90,7 @@ define i8 @uadd_scalar_undef(i8 %a) {
 
 define i8 @uadd_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @uadd_scalar_poison(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %x5 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 poison)
   ret i8 %x5
@@ -106,7 +106,7 @@ define <2 x i8> @uadd_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @uadd_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
   ret <2 x i8> %x5v
@@ -122,7 +122,7 @@ define i8 @uadd_scalar_undef_commute(i8 %a) {
 
 define i8 @uadd_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @uadd_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %x6 = call i8 @llvm.uadd.sat.i8(i8 poison, i8 %a)
   ret i8 %x6
@@ -138,7 +138,7 @@ define <2 x i8> @uadd_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @uadd_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
   ret <2 x i8> %x5v
@@ -222,7 +222,7 @@ define i8 @sadd_scalar_undef(i8 %a) {
 
 define i8 @sadd_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_poison(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %y5 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 poison)
   ret i8 %y5
@@ -238,7 +238,7 @@ define <2 x i8> @sadd_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @sadd_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y5v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
   ret <2 x i8> %y5v
@@ -254,7 +254,7 @@ define i8 @sadd_scalar_undef_commute(i8 %a) {
 
 define i8 @sadd_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 -1
+; CHECK-NEXT:    ret i8 poison
 ;
   %y6 = call i8 @llvm.sadd.sat.i8(i8 poison, i8 %a)
   ret i8 %y6
@@ -270,7 +270,7 @@ define <2 x i8> @sadd_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @sadd_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> splat (i8 -1)
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y6v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
   ret <2 x i8> %y6v
@@ -334,7 +334,7 @@ define i8 @usub_scalar_undef(i8 %a) {
 
 define i8 @usub_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_poison(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %x4 = call i8 @llvm.usub.sat.i8(i8 %a, i8 poison)
   ret i8 %x4
@@ -350,7 +350,7 @@ define <2 x i8> @usub_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @usub_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x4v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
   ret <2 x i8> %x4v
@@ -366,7 +366,7 @@ define i8 @usub_scalar_undef_commute(i8 %a) {
 
 define i8 @usub_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %x5 = call i8 @llvm.usub.sat.i8(i8 poison, i8 %a)
   ret i8 %x5
@@ -382,7 +382,7 @@ define <2 x i8> @usub_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @usub_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %x5v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
   ret <2 x i8> %x5v
@@ -466,7 +466,7 @@ define i8 @ssub_scalar_undef(i8 %a) {
 
 define i8 @ssub_scalar_poison(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_poison(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %y4 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 poison)
   ret i8 %y4
@@ -482,7 +482,7 @@ define <2 x i8> @ssub_vector_undef(<2 x i8> %a) {
 
 define <2 x i8> @ssub_vector_poison(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_poison(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y4v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
   ret <2 x i8> %y4v
@@ -498,7 +498,7 @@ define i8 @ssub_scalar_undef_commute(i8 %a) {
 
 define i8 @ssub_scalar_poison_commute(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_poison_commute(
-; CHECK-NEXT:    ret i8 0
+; CHECK-NEXT:    ret i8 poison
 ;
   %y5 = call i8 @llvm.ssub.sat.i8(i8 poison, i8 %a)
   ret i8 %y5
@@ -514,7 +514,7 @@ define <2 x i8> @ssub_vector_undef_commute(<2 x i8> %a) {
 
 define <2 x i8> @ssub_vector_poison_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_poison_commute(
-; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> poison
 ;
   %y5v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
   ret <2 x i8> %y5v
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll
index 61a68692ff5b..c565066541d1 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-deinterleave4.ll
@@ -16,17 +16,13 @@ define void @deinterleave4(ptr %src) {
 ;
 
   %load = load <vscale x 16 x i32>, ptr %src, align 4
-  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
-  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
-  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
-  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
-  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
-  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 1
-  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
-  %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 0
-  %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
-  %sum = add <vscale x 4 x i32> %5, %7
-  %sub = sub <vscale x 4 x i32> %6, %8
+  %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %load)
+  %1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 0
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 1
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 2
+  %4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 3
+  %sum = add <vscale x 4 x i32> %1, %2
+  %sub = sub <vscale x 4 x i32> %3, %4
   ret void
 }
 
@@ -58,17 +54,13 @@ define void @wide_deinterleave4(ptr %src) {
 ; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 32 x i32>, ptr %src, align 4
-  %deinterleave_src = tail call { <vscale x 16 x i32>, <vscale x 16 x i32> } @llvm.vector.deinterleave2.nxv32i32(<vscale x 32 x i32> %load)
-  %3 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave_src, 0
-  %4 = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i32> } %deinterleave_src, 1
-  %deinterleave_half1 = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %3)
-  %5 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half1, 0
-  %6 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half1, 1
-  %deinterleave_half2 = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %4)
-  %7 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half2, 0
-  %8 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_half2, 1
-  %sum = add <vscale x 8 x i32> %5, %7
-  %sub = sub <vscale x 8 x i32> %6, %8
+  %deinterleave = tail call { <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave4.nxv32i32(<vscale x 32 x i32> %load)
+  %1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave, 0
+  %2 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave, 1
+  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave, 2
+  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave, 3
+  %sum = add <vscale x 8 x i32> %1, %2
+  %sub = sub <vscale x 8 x i32> %3, %4
   ret void
 }
 
@@ -87,52 +79,36 @@ define void @mix_deinterleave4_deinterleave2(ptr %src) {
 ;
 
   %load = load <vscale x 16 x i32>, ptr %src, align 4
-  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
-  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
-  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
-  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
-  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
-  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 1
-  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
-  %7 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 0
-  %8 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
+  %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %load)
+  %1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 0
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 1
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 2
+  %4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 3
 
   %load2 = load <vscale x 8 x i32>, ptr %src, align 4
-  %deinterleave_src2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 8 x i32> %load2)
-  %ld2_1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_src2, 0
-  %ld2_2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_src2, 1
+  %deinterleave2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 8 x i32> %load2)
+  %ld2_1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave2, 0
+  %ld2_2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave2, 1
   ret void
 }
 
 define void @negative_deinterleave4_test(ptr %src) {
 ; CHECK-LABEL: define void @negative_deinterleave4_test
 ; CHECK-SAME: (ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 0
-; CHECK-NEXT:    [[LDN:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> splat (i1 true), ptr [[TMP1]])
-; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 0
-; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP2]], i64 0)
-; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN]], 1
-; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP4]], i64 0)
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <vscale x 4 x i32>, ptr [[SRC]], i64 2
-; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> splat (i1 true), ptr [[TMP6]])
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP7]], i64 4)
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[LDN1]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP9]], i64 4)
-; CHECK-NEXT:    [[DEINTERLEAVE_HALF1:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP8]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE_HALF1]], 0
-; CHECK-NEXT:    [[DEINTERLEAVE_HALF2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[TMP10]])
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE_HALF2]], 1
+; CHECK-NEXT:    [[LOAD:%.*]] = load <vscale x 16 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[DEINTERLEAVE:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[LOAD]])
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[DEINTERLEAVE]], 2
 ; CHECK-NEXT:    ret void
 ;
   %load = load <vscale x 16 x i32>, ptr %src, align 4
-  %deinterleave_src = tail call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %load)
-  %3 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 0
-  %4 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %deinterleave_src, 1
-  %deinterleave_half1 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %3)
-  %5 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half1, 0
-  %deinterleave_half2 = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %4)
-  %6 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave_half2, 1
+  %deinterleave = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> %load)
+  %1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 0
+  %2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 1
+  %3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 2
+  %4 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleave, 2
 
   ret void
 }
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll
index 085089978d8f..a61db6577d56 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleave4.ll
@@ -8,9 +8,7 @@ define void @interleave4(ptr %dst, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b,
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[C]], <vscale x 4 x i32> [[D]], <vscale x 4 x i1> splat (i1 true), ptr [[DST]])
 ; CHECK-NEXT:    ret void
 ;
-  %interleaved.half1 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
-  %interleaved.half2 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %d)
-  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %interleaved.half1, <vscale x 8 x i32> %interleaved.half2)
+  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
   store <vscale x 16 x i32> %interleaved.vec, ptr %dst, align 4
   ret void
 }
@@ -32,9 +30,7 @@ define void @wide_interleave4(ptr %dst, <vscale x 8 x i32> %a, <vscale x 8 x i32
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[TMP10]], <vscale x 4 x i1> splat (i1 true), ptr [[TMP6]])
 ; CHECK-NEXT:    ret void
 ;
-  %interleaved.half1 = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %c)
-  %interleaved.half2 = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %b, <vscale x 8 x i32> %d)
-  %interleaved.vec = tail call <vscale x 32 x i32> @llvm.vector.interleave2.nxv32i32(<vscale x 16 x i32> %interleaved.half1, <vscale x 16 x i32> %interleaved.half2)
+  %interleaved.vec = tail call <vscale x 32 x i32> @llvm.vector.interleave4.nxv32i32(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i32> %c, <vscale x 8 x i32> %d)
   store <vscale x 32 x i32> %interleaved.vec, ptr %dst, align 4
   ret void
 }
@@ -46,9 +42,7 @@ define void @mix_interleave4_interleave2(ptr %dst1, ptr %dst2, <vscale x 4 x i32
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st2.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[C]], <vscale x 4 x i1> splat (i1 true), ptr [[DST2]])
 ; CHECK-NEXT:    ret void
 ;
-  %interleaved.half1 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
-  %interleaved.half2 = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %d)
-  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %interleaved.half1, <vscale x 8 x i32> %interleaved.half2)
+  %interleaved.vec = tail call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d)
   store <vscale x 16 x i32> %interleaved.vec, ptr %dst1, align 4
 
   %interleaved = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %c)
@@ -64,8 +58,7 @@ define void @duplicate_by_interleave(<vscale x 4 x i32> %A, <vscale x 4 x i32> %
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[A]], <vscale x 4 x i32> [[B]], <vscale x 4 x i32> [[B]], <vscale x 4 x i1> splat (i1 true), ptr [[AB_DUPLICATE]])
 ; CHECK-NEXT:    ret void
 ;
-  %interleave = tail call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> %A, <vscale x 4 x i32> %B)
-  %duplicate_by_interleave = tail call <vscale x 16 x i32> @llvm.vector.interleave2.nxv16i32(<vscale x 8 x i32> %interleave, <vscale x 8 x i32> %interleave)
-  store <vscale x 16 x i32> %duplicate_by_interleave, ptr %AB_duplicate, align 4
+  %interleave = tail call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> %A, <vscale x 4 x i32> %A, <vscale x 4 x i32> %B, <vscale x 4 x i32> %B)
+  store <vscale x 16 x i32> %interleave, ptr %AB_duplicate, align 4
   ret void
 }
diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
index 87b16d17aa5f..72c1f22032bb 100644
--- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll
@@ -175,53 +175,6 @@ define void @load_factor4_vscale(ptr %ptr) {
   ret void
 }
 
-; TODO: Remove once recursive deinterleaving support is removed
-define void @load_factor4_vscale_recursive(ptr %ptr) {
-; RV32-LABEL: @load_factor4_vscale_recursive(
-; RV32-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 16 x i8>, 4) @llvm.riscv.vlseg4.triscv.vector.tuple_nxv16i8_4t.p0.i32(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) poison, ptr [[PTR:%.*]], i32 -1, i32 5)
-; RV32-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 0)
-; RV32-NEXT:    [[TMP3:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } poison, <vscale x 4 x i32> [[TMP2]], 0
-; RV32-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 1)
-; RV32-NEXT:    [[TMP5:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP3]], <vscale x 4 x i32> [[TMP4]], 1
-; RV32-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 2)
-; RV32-NEXT:    [[TMP7:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP5]], <vscale x 4 x i32> [[TMP6]], 2
-; RV32-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 3)
-; RV32-NEXT:    [[TMP9:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP7]], <vscale x 4 x i32> [[TMP8]], 3
-; RV32-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 0
-; RV32-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 1
-; RV32-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 2
-; RV32-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 3
-; RV32-NEXT:    ret void
-;
-; RV64-LABEL: @load_factor4_vscale_recursive(
-; RV64-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 16 x i8>, 4) @llvm.riscv.vlseg4.triscv.vector.tuple_nxv16i8_4t.p0.i64(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) poison, ptr [[PTR:%.*]], i64 -1, i64 5)
-; RV64-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 0)
-; RV64-NEXT:    [[TMP3:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } poison, <vscale x 4 x i32> [[TMP2]], 0
-; RV64-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 1)
-; RV64-NEXT:    [[TMP5:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP3]], <vscale x 4 x i32> [[TMP4]], 1
-; RV64-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 2)
-; RV64-NEXT:    [[TMP7:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP5]], <vscale x 4 x i32> [[TMP6]], 2
-; RV64-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.riscv.tuple.extract.nxv4i32.triscv.vector.tuple_nxv16i8_4t(target("riscv.vector.tuple", <vscale x 16 x i8>, 4) [[TMP1]], i32 3)
-; RV64-NEXT:    [[TMP9:%.*]] = insertvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP7]], <vscale x 4 x i32> [[TMP8]], 3
-; RV64-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 0
-; RV64-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 1
-; RV64-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 2
-; RV64-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP9]], 3
-; RV64-NEXT:    ret void
-;
-  %interleaved.vec = load <vscale x 16 x i32>, ptr %ptr
-  %d0 = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %interleaved.vec)
-  %d0.0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 1
-  %d1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.0)
-  %t0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 0
-  %t1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 1
-  %d2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.1)
-  %t2 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 0
-  %t3 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 1
-  ret void
-}
-
 define void @load_factor5(ptr %ptr) {
 ; RV32-LABEL: @load_factor5(
 ; RV32-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>, <4 x i32> } @llvm.riscv.seg5.load.mask.v4i32.p0.i32(ptr [[PTR:%.*]], <4 x i1> splat (i1 true), i32 4)
@@ -590,91 +543,6 @@ define void @load_factor8_vscale(ptr %ptr) {
   ret void
 }
 
-; TODO: Remove once recursive deinterleaving support is removed
-define void @load_factor8_vscale_recursive(ptr %ptr) {
-; RV32-LABEL: @load_factor8_vscale_recursive(
-; RV32-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.vlseg8.triscv.vector.tuple_nxv8i8_8t.p0.i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) poison, ptr [[PTR:%.*]], i32 -1, i32 5)
-; RV32-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 0)
-; RV32-NEXT:    [[TMP3:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> [[TMP2]], 0
-; RV32-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 1)
-; RV32-NEXT:    [[TMP5:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP3]], <vscale x 2 x i32> [[TMP4]], 1
-; RV32-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 2)
-; RV32-NEXT:    [[TMP7:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP5]], <vscale x 2 x i32> [[TMP6]], 2
-; RV32-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 3)
-; RV32-NEXT:    [[TMP9:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP7]], <vscale x 2 x i32> [[TMP8]], 3
-; RV32-NEXT:    [[TMP10:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 4)
-; RV32-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP9]], <vscale x 2 x i32> [[TMP10]], 4
-; RV32-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 5)
-; RV32-NEXT:    [[TMP13:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP11]], <vscale x 2 x i32> [[TMP12]], 5
-; RV32-NEXT:    [[TMP14:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 6)
-; RV32-NEXT:    [[TMP15:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP13]], <vscale x 2 x i32> [[TMP14]], 6
-; RV32-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 7)
-; RV32-NEXT:    [[TMP17:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP15]], <vscale x 2 x i32> [[TMP16]], 7
-; RV32-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 0
-; RV32-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 1
-; RV32-NEXT:    [[TMP20:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 2
-; RV32-NEXT:    [[TMP21:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 3
-; RV32-NEXT:    [[TMP22:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 4
-; RV32-NEXT:    [[TMP23:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 5
-; RV32-NEXT:    [[TMP24:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 6
-; RV32-NEXT:    [[TMP25:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 7
-; RV32-NEXT:    ret void
-;
-; RV64-LABEL: @load_factor8_vscale_recursive(
-; RV64-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.vlseg8.triscv.vector.tuple_nxv8i8_8t.p0.i64(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) poison, ptr [[PTR:%.*]], i64 -1, i64 5)
-; RV64-NEXT:    [[TMP2:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 0)
-; RV64-NEXT:    [[TMP3:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } poison, <vscale x 2 x i32> [[TMP2]], 0
-; RV64-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 1)
-; RV64-NEXT:    [[TMP5:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP3]], <vscale x 2 x i32> [[TMP4]], 1
-; RV64-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 2)
-; RV64-NEXT:    [[TMP7:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP5]], <vscale x 2 x i32> [[TMP6]], 2
-; RV64-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 3)
-; RV64-NEXT:    [[TMP9:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP7]], <vscale x 2 x i32> [[TMP8]], 3
-; RV64-NEXT:    [[TMP10:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 4)
-; RV64-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP9]], <vscale x 2 x i32> [[TMP10]], 4
-; RV64-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 5)
-; RV64-NEXT:    [[TMP13:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP11]], <vscale x 2 x i32> [[TMP12]], 5
-; RV64-NEXT:    [[TMP14:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 6)
-; RV64-NEXT:    [[TMP15:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP13]], <vscale x 2 x i32> [[TMP14]], 6
-; RV64-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_8t(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], i32 7)
-; RV64-NEXT:    [[TMP17:%.*]] = insertvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP15]], <vscale x 2 x i32> [[TMP16]], 7
-; RV64-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 0
-; RV64-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 1
-; RV64-NEXT:    [[TMP20:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 2
-; RV64-NEXT:    [[TMP21:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 3
-; RV64-NEXT:    [[TMP22:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 4
-; RV64-NEXT:    [[TMP23:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 5
-; RV64-NEXT:    [[TMP24:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 6
-; RV64-NEXT:    [[TMP25:%.*]] = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } [[TMP17]], 7
-; RV64-NEXT:    ret void
-;
-  %interleaved.vec = load <vscale x 16 x i32>, ptr %ptr
-  %d0 = call { <vscale x 8 x i32>, <vscale x 8 x i32> } @llvm.vector.deinterleave2.nxv16i32(<vscale x 16 x i32> %interleaved.vec)
-  %d0.0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 0
-  %d0.1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d0, 1
-
-  %d1 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.0)
-  %d1.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 0
-  %d1.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d1, 1
-  %d2 = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> %d0.1)
-  %d2.0 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 0
-  %d2.1 = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } %d2, 1
-
-  %d3 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.0)
-  %t0 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 0
-  %t1 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d3, 1
-  %d4 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d1.1)
-  %t2 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 0
-  %t3 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d4, 1
-  %d5 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.0)
-  %t4 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 0
-  %t5 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d5, 1
-  %d6 = call { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.vector.deinterleave2.nxv4i32(<vscale x 4 x i32> %d2.1)
-  %t6 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 0
-  %t7 = extractvalue { <vscale x 2 x i32>, <vscale x 2 x i32> } %d6, 1
-  ret void
-}
-
 
 define void @store_factor2(ptr %ptr, <8 x i8> %v0, <8 x i8> %v1) {
 ; RV32-LABEL: @store_factor2(
@@ -808,31 +676,6 @@ define void @store_factor4_vscale(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x
   ret void
 }
 
-; TODO: Remove once recursive interleaving support is removed
-define void @store_factor4_vscale_recursive(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1) {
-; RV32-LABEL: @store_factor4_vscale_recursive(
-; RV32-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) poison, <vscale x 8 x i8> [[V0:%.*]], i32 0)
-; RV32-NEXT:    [[TMP2:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP1]], <vscale x 8 x i8> [[V0]], i32 1)
-; RV32-NEXT:    [[TMP3:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP2]], <vscale x 8 x i8> [[V1:%.*]], i32 2)
-; RV32-NEXT:    [[TMP4:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP3]], <vscale x 8 x i8> [[V1]], i32 3)
-; RV32-NEXT:    call void @llvm.riscv.vsseg4.triscv.vector.tuple_nxv8i8_4t.p0.i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP4]], ptr [[PTR:%.*]], i32 -1, i32 3)
-; RV32-NEXT:    ret void
-;
-; RV64-LABEL: @store_factor4_vscale_recursive(
-; RV64-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) poison, <vscale x 8 x i8> [[V0:%.*]], i32 0)
-; RV64-NEXT:    [[TMP2:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP1]], <vscale x 8 x i8> [[V0]], i32 1)
-; RV64-NEXT:    [[TMP3:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP2]], <vscale x 8 x i8> [[V1:%.*]], i32 2)
-; RV64-NEXT:    [[TMP4:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP3]], <vscale x 8 x i8> [[V1]], i32 3)
-; RV64-NEXT:    call void @llvm.riscv.vsseg4.triscv.vector.tuple_nxv8i8_4t.p0.i64(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) [[TMP4]], ptr [[PTR:%.*]], i64 -1, i64 3)
-; RV64-NEXT:    ret void
-;
-  %i0 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
-  %i1 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
-  %i2 = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 16 x i8> %i0, <vscale x 16 x i8> %i1)
-  store <vscale x 32 x i8> %i2, ptr %ptr, align 4
-  ret void
-}
-
 define void @store_factor5_vscale(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscale x 8 x i8> %v2, <vscale x 8 x i8> %v3, <vscale x 8 x i8> %v4) {
 ; RV32-LABEL: @store_factor5_vscale(
 ; RV32-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 5) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_5t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 5) poison, <vscale x 8 x i8> [[V0:%.*]], i32 0)
@@ -1013,45 +856,6 @@ define void @store_factor8_vscale(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x
   ret void
 }
 
-; TODO: Remove once recursive interleaving support is removed
-define void @store_factor8_vscale_recursive(ptr %ptr, <vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscale x 8 x i8> %v2, <vscale x 8 x i8> %v3) {
-; RV32-LABEL: @store_factor8_vscale_recursive(
-; RV32-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) poison, <vscale x 8 x i8> [[V0:%.*]], i32 0)
-; RV32-NEXT:    [[TMP2:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], <vscale x 8 x i8> [[V2:%.*]], i32 1)
-; RV32-NEXT:    [[TMP3:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP2]], <vscale x 8 x i8> [[V0]], i32 2)
-; RV32-NEXT:    [[TMP4:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP3]], <vscale x 8 x i8> [[V2]], i32 3)
-; RV32-NEXT:    [[TMP5:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP4]], <vscale x 8 x i8> [[V1:%.*]], i32 4)
-; RV32-NEXT:    [[TMP6:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP5]], <vscale x 8 x i8> [[V3:%.*]], i32 5)
-; RV32-NEXT:    [[TMP7:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP6]], <vscale x 8 x i8> [[V1]], i32 6)
-; RV32-NEXT:    [[TMP8:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP7]], <vscale x 8 x i8> [[V3]], i32 7)
-; RV32-NEXT:    call void @llvm.riscv.vsseg8.triscv.vector.tuple_nxv8i8_8t.p0.i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP8]], ptr [[PTR:%.*]], i32 -1, i32 3)
-; RV32-NEXT:    ret void
-;
-; RV64-LABEL: @store_factor8_vscale_recursive(
-; RV64-NEXT:    [[TMP1:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) poison, <vscale x 8 x i8> [[V0:%.*]], i32 0)
-; RV64-NEXT:    [[TMP2:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP1]], <vscale x 8 x i8> [[V2:%.*]], i32 1)
-; RV64-NEXT:    [[TMP3:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP2]], <vscale x 8 x i8> [[V0]], i32 2)
-; RV64-NEXT:    [[TMP4:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP3]], <vscale x 8 x i8> [[V2]], i32 3)
-; RV64-NEXT:    [[TMP5:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP4]], <vscale x 8 x i8> [[V1:%.*]], i32 4)
-; RV64-NEXT:    [[TMP6:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP5]], <vscale x 8 x i8> [[V3:%.*]], i32 5)
-; RV64-NEXT:    [[TMP7:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP6]], <vscale x 8 x i8> [[V1]], i32 6)
-; RV64-NEXT:    [[TMP8:%.*]] = call target("riscv.vector.tuple", <vscale x 8 x i8>, 8) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_8t.nxv8i8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP7]], <vscale x 8 x i8> [[V3]], i32 7)
-; RV64-NEXT:    call void @llvm.riscv.vsseg8.triscv.vector.tuple_nxv8i8_8t.p0.i64(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) [[TMP8]], ptr [[PTR:%.*]], i64 -1, i64 3)
-; RV64-NEXT:    ret void
-;
-  %i0 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
-  %i1 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1)
-  %i2 = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 16 x i8> %i0, <vscale x 16 x i8> %i1)
-
-  %i3 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v2, <vscale x 8 x i8> %v3)
-  %i4 = call <vscale x 16 x i8> @llvm.vector.interleave2.nxv8i8(<vscale x 8 x i8> %v2, <vscale x 8 x i8> %v3)
-  %i5 = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv16i8(<vscale x 16 x i8> %i3, <vscale x 16 x i8> %i4)
-
-  %i6 = call <vscale x 64 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 32 x i8> %i2, <vscale x 32 x i8> %i5)
-  store <vscale x 64 x i8> %i6, ptr %ptr, align 4
-  ret void
-}
-
 define void @load_factor2_fp128(ptr %ptr) {
 ; RV32-LABEL: @load_factor2_fp128(
 ; RV32-NEXT:    [[INTERLEAVED_VEC:%.*]] = load <4 x fp128>, ptr [[PTR:%.*]], align 16
diff --git a/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll b/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll
new file mode 100644
index 000000000000..806bd820009b
--- /dev/null
+++ b/llvm/test/Transforms/LICM/salvage-hoisted-minmax.ll
@@ -0,0 +1,59 @@
+; RUN: opt -S -passes=licm %s | FileCheck %s
+
+; Check that hoistMinMax() in LICM salvages the debug values for the hoisted
+; cmp instructions.
+
+define i32 @test_ult(i32 %start, i32 %inv_1, i32 %inv_2) !dbg !5 {
+; CHECK-LABEL: define i32 @test_ult(
+; CHECK-SAME: i32 [[START:%.*]], i32 [[INV_1:%.*]], i32 [[INV_2:%.*]])
+; CHECK:       [[LOOP:.*]]:
+; CHECK:           #dbg_value(!DIArgList(i32 [[IV:%.*]], i32 [[INV_1]]), [[META9:![0-9]+]], !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_lt, DW_OP_stack_value), [[META14:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(!DIArgList(i32 [[IV]], i32 [[INV_2]]), [[META11:![0-9]+]], !DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_lt, DW_OP_stack_value), [[META15:![0-9]+]])
+; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV]], [[INVARIANT_UMIN:%.*]], !dbg [[DBG16:![0-9]+]]
+entry:
+  br label %loop, !dbg !16
+
+loop:                                             ; preds = %loop, %entry
+  %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop ], !dbg !17
+  %cmp_1 = icmp ult i32 %iv, %inv_1, !dbg !18
+    #dbg_value(i1 %cmp_1, !11, !DIExpression(), !18)
+  %cmp_2 = icmp ult i32 %iv, %inv_2, !dbg !19
+    #dbg_value(i1 %cmp_2, !13, !DIExpression(), !19)
+  %loop_cond = and i1 %cmp_1, %cmp_2, !dbg !20
+  %iv.next = add i32 %iv, 1, !dbg !21
+  br i1 %loop_cond, label %loop, label %exit, !dbg !22
+
+exit:                                             ; preds = %loop
+  ret i32 %iv, !dbg !23
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.debugify = !{!2, !3}
+!llvm.module.flags = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "temp.ll", directory: "/")
+!2 = !{i32 8}
+!3 = !{i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test_ult", linkageName: "test_ult", scope: null, file: !1, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!6 = !DISubroutineType(types: !7)
+!7 = !{}
+!8 = !{!11, !13}
+!11 = !DILocalVariable(name: "2", scope: !5, file: !1, line: 3, type: !12)
+!12 = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+!13 = !DILocalVariable(name: "3", scope: !5, file: !1, line: 4, type: !12)
+!16 = !DILocation(line: 1, column: 1, scope: !5)
+!17 = !DILocation(line: 2, column: 1, scope: !5)
+!18 = !DILocation(line: 3, column: 1, scope: !5)
+!19 = !DILocation(line: 4, column: 1, scope: !5)
+!20 = !DILocation(line: 5, column: 1, scope: !5)
+!21 = !DILocation(line: 6, column: 1, scope: !5)
+!22 = !DILocation(line: 7, column: 1, scope: !5)
+!23 = !DILocation(line: 8, column: 1, scope: !5)
+;.
+; CHECK: [[META9]] = !DILocalVariable(name: "2",
+; CHECK: [[META11]] = !DILocalVariable(name: "3",
+; CHECK: [[META14]] = !DILocation(line: 3,
+; CHECK: [[META15]] = !DILocation(line: 4,
+;.
diff --git a/llvm/test/Transforms/LoopDeletion/diundef.ll b/llvm/test/Transforms/LoopDeletion/diundef.ll
index d4fb3275007f..0a52845d272b 100644
--- a/llvm/test/Transforms/LoopDeletion/diundef.ll
+++ b/llvm/test/Transforms/LoopDeletion/diundef.ll
@@ -1,4 +1,4 @@
-; RUN: opt %s -passes=loop-deletion -S | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt %s -passes=loop-deletion -S | FileCheck %s --implicit-check-not=dbg_value
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
diff --git a/llvm/test/Transforms/LoopFusion/four_loops.ll b/llvm/test/Transforms/LoopFusion/four_loops.ll
index 7f7f0f19b59b..1728686ebc15 100644
--- a/llvm/test/Transforms/LoopFusion/four_loops.ll
+++ b/llvm/test/Transforms/LoopFusion/four_loops.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 @A = common global [1024 x i32] zeroinitializer, align 16
@@ -5,21 +6,77 @@
 @C = common global [1024 x i32] zeroinitializer, align 16
 @D = common global [1024 x i32] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br label %[[LOOP2BODY:bb[0-9]+]]
-; CHECK: [[LOOP2BODY]]
-; CHECK: br label %[[LOOP3BODY:bb[0-9]+]]
-; CHECK: [[LOOP3BODY]]
-; CHECK: br label %[[LOOP4BODY:bb[0-9]+]]
-; CHECK: [[LOOP4BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]+]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOPEXIT:bb[0-9]+]]
-; CHECK: ret void
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB15:.*]]
+; CHECK:       [[BB15]]:
+; CHECK-NEXT:    [[DOT08:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP23:%.*]], %[[BB61:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV107:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT11:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT016:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP36:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV75:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT8:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT024:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP49:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV43:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT5:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[DOT032:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP62:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[BB61]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT08]], -3
+; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw i64 [[INDVARS_IV107]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP16]] to i32
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nsw i32 [[TMP]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[INDVARS_IV107]] to i32
+; CHECK-NEXT:    [[TMP20:%.*]] = srem i32 [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 [[INDVARS_IV107]]
+; CHECK-NEXT:    store i32 [[TMP20]], ptr [[TMP21]], align 4
+; CHECK-NEXT:    br label %[[BB22:.*]]
+; CHECK:       [[BB22]]:
+; CHECK-NEXT:    [[TMP28:%.*]] = add nsw i32 [[DOT016]], -3
+; CHECK-NEXT:    [[TMP29:%.*]] = add nuw nsw i64 [[INDVARS_IV75]], 3
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[TMP29]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = mul nsw i32 [[TMP28]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = trunc i64 [[INDVARS_IV75]] to i32
+; CHECK-NEXT:    [[TMP33:%.*]] = srem i32 [[TMP31]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV75]]
+; CHECK-NEXT:    store i32 [[TMP33]], ptr [[TMP34]], align 4
+; CHECK-NEXT:    br label %[[BB35:.*]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    [[TMP41:%.*]] = add nsw i32 [[DOT024]], -3
+; CHECK-NEXT:    [[TMP42:%.*]] = add nuw nsw i64 [[INDVARS_IV43]], 3
+; CHECK-NEXT:    [[TMP43:%.*]] = trunc i64 [[TMP42]] to i32
+; CHECK-NEXT:    [[TMP44:%.*]] = mul nsw i32 [[TMP41]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = trunc i64 [[INDVARS_IV43]] to i32
+; CHECK-NEXT:    [[TMP46:%.*]] = srem i32 [[TMP44]], [[TMP45]]
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds [1024 x i32], ptr @C, i64 0, i64 [[INDVARS_IV43]]
+; CHECK-NEXT:    store i32 [[TMP46]], ptr [[TMP47]], align 4
+; CHECK-NEXT:    br label %[[BB48:.*]]
+; CHECK:       [[BB48]]:
+; CHECK-NEXT:    [[TMP54:%.*]] = add nsw i32 [[DOT032]], -3
+; CHECK-NEXT:    [[TMP55:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP56:%.*]] = trunc i64 [[TMP55]] to i32
+; CHECK-NEXT:    [[TMP57:%.*]] = mul nsw i32 [[TMP54]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP59:%.*]] = srem i32 [[TMP57]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [1024 x i32], ptr @D, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP59]], ptr [[TMP60]], align 4
+; CHECK-NEXT:    br label %[[BB61]]
+; CHECK:       [[BB52:.*]]:
+; CHECK-NEXT:    br label %[[BB63:.*]]
+; CHECK:       [[BB61]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT11]] = add nuw nsw i64 [[INDVARS_IV107]], 1
+; CHECK-NEXT:    [[TMP23]] = add nuw nsw i32 [[DOT08]], 1
+; CHECK-NEXT:    [[EXITCOND12:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT11]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT8]] = add nuw nsw i64 [[INDVARS_IV75]], 1
+; CHECK-NEXT:    [[TMP36]] = add nuw nsw i32 [[DOT016]], 1
+; CHECK-NEXT:    [[EXITCOND9:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT8]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT5]] = add nuw nsw i64 [[INDVARS_IV43]], 1
+; CHECK-NEXT:    [[TMP49]] = add nuw nsw i32 [[DOT024]], 1
+; CHECK-NEXT:    [[EXITCOND6:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT5]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP62]] = add nuw nsw i32 [[DOT032]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB15]], label %[[BB52]]
+; CHECK:       [[BB63]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb15
 
diff --git a/llvm/test/Transforms/LoopFusion/guarded_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
index c45772b0b029..c26935908d4d 100644
--- a/llvm/test/Transforms/LoopFusion/guarded_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/guarded_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests if we are able to fuse two guarded loops which have constant but
@@ -6,33 +7,75 @@
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
-; CHECK-LABEL: void @main(ptr noalias %A)
-; CHECK-NEXT:  entry:
-; CHECK:         br i1 %cmp4, label %for.first.entry, label %for.end
-; CHECK:       for.first.entry
-; CHECK-NEXT:    br label %for.first.peel.begin
-; CHECK:       for.first.peel.begin:
-; CHECK-NEXT:    br label %for.first.peel
-; CHECK:       for.first.peel:
-; CHECK:         br label %for.first.peel.next
-; CHECK:       for.first.peel.next:
-; CHECK-NEXT:    br label %for.first.peel2
-; CHECK:       for.first.peel2:
-; CHECK:         br label %for.first.peel.next1
-; CHECK:       for.first.peel.next1:
-; CHECK-NEXT:    br label %for.first.peel.next11
-; CHECK:       for.first.peel.next11:
-; CHECK-NEXT:    br label %for.first.entry.peel.newph
-; CHECK:       for.first.entry.peel.newph:
-; CHECK:         br label %for.first
-; CHECK:       for.first:
-; CHECK:         br i1 %cmp3, label %for.first, label %for.second.exit
-; CHECK:       for.second.exit:
-; CHECK:         br label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 define void @main(ptr noalias %A) {
+; CHECK-LABEL: define void @main(
+; CHECK-SAME: ptr noalias [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i64 0, 45
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp slt i64 2, 45
+; CHECK-NEXT:    br i1 [[CMP4]], label %[[FOR_FIRST_ENTRY:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_FIRST_ENTRY]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_BEGIN:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_BEGIN]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_PEEL]]:
+; CHECK-NEXT:    [[SUB_PEEL:%.*]] = sub nsw i64 0, 3
+; CHECK-NEXT:    [[ADD_PEEL:%.*]] = add nsw i64 0, 3
+; CHECK-NEXT:    [[MUL_PEEL:%.*]] = mul nsw i64 [[SUB_PEEL]], [[ADD_PEEL]]
+; CHECK-NEXT:    [[REM_PEEL:%.*]] = srem i64 [[MUL_PEEL]], 0
+; CHECK-NEXT:    [[CONV_PEEL:%.*]] = trunc i64 [[REM_PEEL]] to i32
+; CHECK-NEXT:    [[ARRAYIDX_PEEL:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 0
+; CHECK-NEXT:    store i32 [[CONV_PEEL]], ptr [[ARRAYIDX_PEEL]], align 4
+; CHECK-NEXT:    [[INC_PEEL:%.*]] = add nsw i64 0, 1
+; CHECK-NEXT:    [[CMP_PEEL:%.*]] = icmp slt i64 [[INC_PEEL]], 45
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL2:.*]]
+; CHECK:       [[FOR_FIRST_PEEL2]]:
+; CHECK-NEXT:    [[SUB_PEEL3:%.*]] = sub nsw i64 [[INC_PEEL]], 3
+; CHECK-NEXT:    [[ADD_PEEL4:%.*]] = add nsw i64 [[INC_PEEL]], 3
+; CHECK-NEXT:    [[MUL_PEEL5:%.*]] = mul nsw i64 [[SUB_PEEL3]], [[ADD_PEEL4]]
+; CHECK-NEXT:    [[REM_PEEL6:%.*]] = srem i64 [[MUL_PEEL5]], [[INC_PEEL]]
+; CHECK-NEXT:    [[CONV_PEEL7:%.*]] = trunc i64 [[REM_PEEL6]] to i32
+; CHECK-NEXT:    [[ARRAYIDX_PEEL8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INC_PEEL]]
+; CHECK-NEXT:    store i32 [[CONV_PEEL7]], ptr [[ARRAYIDX_PEEL8]], align 4
+; CHECK-NEXT:    [[INC_PEEL9:%.*]] = add nsw i64 [[INC_PEEL]], 1
+; CHECK-NEXT:    [[CMP_PEEL10:%.*]] = icmp slt i64 [[INC_PEEL9]], 45
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT1:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT1]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT11:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT11]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_ENTRY_PEEL_NEWPH:.*]]
+; CHECK:       [[FOR_FIRST_ENTRY_PEEL_NEWPH]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_FIRST]] ], [ [[INC_PEEL9]], %[[FOR_FIRST_ENTRY_PEEL_NEWPH]] ]
+; CHECK-NEXT:    [[I1_02:%.*]] = phi i64 [ [[INC14:%.*]], %[[FOR_FIRST]] ], [ 2, %[[FOR_FIRST_ENTRY_PEEL_NEWPH]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[I_05]], 3
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[I_05]], 3
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[SUB]], [[ADD]]
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 [[MUL]], [[I_05]]
+; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 [[REM]] to i32
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_05]]
+; CHECK-NEXT:    store i32 [[CONV]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_05]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INC]], 45
+; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i64 [[I1_02]], 3
+; CHECK-NEXT:    [[ADD8:%.*]] = add nsw i64 [[I1_02]], 3
+; CHECK-NEXT:    [[MUL9:%.*]] = mul nsw i64 [[SUB7]], [[ADD8]]
+; CHECK-NEXT:    [[REM10:%.*]] = srem i64 [[MUL9]], [[I1_02]]
+; CHECK-NEXT:    [[CONV11:%.*]] = trunc i64 [[REM10]] to i32
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[I1_02]]
+; CHECK-NEXT:    store i32 [[CONV11]], ptr [[ARRAYIDX12]], align 4
+; CHECK-NEXT:    [[INC14]] = add nsw i64 [[I1_02]], 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i64 [[INC14]], 45
+; CHECK-NEXT:    br i1 [[CMP3]], label %[[FOR_FIRST]], label %[[FOR_SECOND_EXIT:.*]]
+; CHECK:       [[FOR_SECOND_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_END]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %cmp4 = icmp slt i64 0, 45
   br i1 %cmp4, label %for.first.entry, label %for.second.guard
diff --git a/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
index 1ab9e538a3dc..add32738bc5a 100644
--- a/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/guarded_unsafeblock_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests that we do not fuse two guarded loops together.
@@ -6,28 +7,42 @@
 ; loops unsafe to fuse together.
 ; The expected output of this test is the function as below.
 
-; CHECK-LABEL: void @unsafe_exitblock(ptr noalias %A, ptr noalias %B)
-; CHECK:       for.first.guard
-; CHECK:         br i1 %cmp3, label %for.first.preheader, label %for.second.guard
-; CHECK:       for.first.preheader:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br i1 %cmp, label %for.first, label %for.first.exit
-; CHECK:       for.first.exit:
-; CHECK-NEXT:    call void @bar()
-; CHECK-NEXT:    br label %for.second.guard
-; CHECK:       for.second.guard:
-; CHECK:         br i1 %cmp21, label %for.second.preheader, label %for.end
-; CHECK:       for.second.preheader:
-; CHECK-NEXT:    br label %for.second
-; CHECK:       for.second:
-; CHECK:         br i1 %cmp2, label %for.second, label %for.second.exit
-; CHECK:       for.second.exit:
-; CHECK-NEXT:    br label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 define void @unsafe_exitblock(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @unsafe_exitblock(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_GUARD:.*:]]
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i64 0, 45
+; CHECK-NEXT:    br i1 [[CMP3]], label %[[FOR_FIRST_PREHEADER:.*]], label %[[FOR_SECOND_GUARD:.*]]
+; CHECK:       [[FOR_FIRST_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_FIRST]] ], [ 0, %[[FOR_FIRST_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_04]]
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_04]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INC]], 45
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_FIRST]], label %[[FOR_FIRST_EXIT:.*]]
+; CHECK:       [[FOR_FIRST_EXIT]]:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label %[[FOR_SECOND_GUARD]]
+; CHECK:       [[FOR_SECOND_GUARD]]:
+; CHECK-NEXT:    [[CMP21:%.*]] = icmp slt i64 2, 45
+; CHECK-NEXT:    br i1 [[CMP21]], label %[[FOR_SECOND_PREHEADER:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_SECOND_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND:.*]]
+; CHECK:       [[FOR_SECOND]]:
+; CHECK-NEXT:    [[J_02:%.*]] = phi i64 [ [[INC6:%.*]], %[[FOR_SECOND]] ], [ 2, %[[FOR_SECOND_PREHEADER]] ]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[J_02]]
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[INC6]] = add nsw i64 [[J_02]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i64 [[INC6]], 45
+; CHECK-NEXT:    br i1 [[CMP2]], label %[[FOR_SECOND]], label %[[FOR_SECOND_EXIT:.*]]
+; CHECK:       [[FOR_SECOND_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_END]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.guard:
   %cmp3 = icmp slt i64 0, 45
   br i1 %cmp3, label %for.first.preheader, label %for.second.guard
diff --git a/llvm/test/Transforms/LoopFusion/hoist_preheader.ll b/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
index 433090fbd994..89698f5cc806 100644
--- a/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
+++ b/llvm/test/Transforms/LoopFusion/hoist_preheader.ll
@@ -1,16 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 define void @hoist_preheader(i32 %N) {
-
-; CHECK:pre1:
-; CHECK-NEXT:  %hoistme = add i32 1, %N
-; CHECK-NEXT:  %hoistme2 = add i32 1, %hoistme
-; CHECK-NEXT:  br label %body1
+; CHECK-LABEL: define void @hoist_preheader(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[PRE1:.*]]:
+; CHECK-NEXT:    [[HOISTME:%.*]] = add i32 1, [[N]]
+; CHECK-NEXT:    [[HOISTME2:%.*]] = add i32 1, [[HOISTME]]
+; CHECK-NEXT:    br label %[[BODY1:.*]]
+; CHECK:       [[BODY1]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 1, [[I]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N]]
+; CHECK-NEXT:    [[I_NEXT2]] = add i32 1, [[I2]]
+; CHECK-NEXT:    [[COND2:%.*]] = icmp ne i32 [[I2]], [[N]]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BODY1]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 pre1:
   br label %body1
 
-; CHECK: body1:
-; CHECK-NOT:  %hoistme
 body1:  ; preds = %pre1, %body1
   %i = phi i32 [%i_next, %body1], [0, %pre1]
   %i_next = add i32 1, %i
diff --git a/llvm/test/Transforms/LoopFusion/inner_loops.ll b/llvm/test/Transforms/LoopFusion/inner_loops.ll
index 3c90905339fa..88ac6621baff 100644
--- a/llvm/test/Transforms/LoopFusion/inner_loops.ll
+++ b/llvm/test/Transforms/LoopFusion/inner_loops.ll
@@ -1,26 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s 2>&1 | FileCheck %s
 
 @A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 @B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]*]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY:bb[0-9]*]], label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP1BODY]]
-; CHECK: br label %[[LOOP1LATCH:bb[0-9]*]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK: br label %[[LOOP2PREHEADER:bb[0-9]+]]
-; CHECK: [[LOOP2PREHEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2EXIT:bb[0-9]*]]
-; CHECK: [[LOOP2BODY]]
-; CHECK: br label %[[LOOP2LATCH:bb[0-9]+]]
-; CHECK: [[LOOP2LATCH]]
-; CHECK: br label %[[LOOP1HEADER]]
-; CHECK: ret void
 
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB9:.*]]
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[INDVARS_IV6:%.*]] = phi i64 [ [[INDVARS_IV_NEXT7:%.*]], %[[BB35:.*]] ], [ 0, %[[BB]] ]
+; CHECK-NEXT:    [[DOT0:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP36:%.*]], %[[BB35]] ]
+; CHECK-NEXT:    [[EXITCOND8:%.*]] = icmp ne i64 [[INDVARS_IV6]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND8]], label %[[BB11:.*]], label %[[BB10:.*]]
+; CHECK:       [[BB10]]:
+; CHECK-NEXT:    br label %[[BB37:.*]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    br label %[[BB12:.*]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[BB21:.*]] ], [ 0, %[[BB11]] ]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB14:.*]], label %[[BB23_PREHEADER:.*]]
+; CHECK:       [[BB23_PREHEADER]]:
+; CHECK-NEXT:    br label %[[BB23:.*]]
+; CHECK:       [[BB14]]:
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT0]], -3
+; CHECK-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[INDVARS_IV6]], 3
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = mul nsw i32 [[TMP]], [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i64 [[INDVARS_IV6]] to i32
+; CHECK-NEXT:    [[TMP19:%.*]] = srem i32 [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @A, i64 0, i64 [[INDVARS_IV6]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[TMP19]], ptr [[TMP20]], align 4
+; CHECK-NEXT:    br label %[[BB21]]
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br label %[[BB12]]
+; CHECK:       [[BB23]]:
+; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ [[INDVARS_IV_NEXT4:%.*]], %[[BB33:.*]] ], [ 0, %[[BB23_PREHEADER]] ]
+; CHECK-NEXT:    [[EXITCOND5:%.*]] = icmp ne i64 [[INDVARS_IV3]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND5]], label %[[BB25:.*]], label %[[BB35]]
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = add nsw i32 [[DOT0]], -3
+; CHECK-NEXT:    [[TMP27:%.*]] = add nuw nsw i64 [[INDVARS_IV6]], 3
+; CHECK-NEXT:    [[TMP28:%.*]] = trunc i64 [[TMP27]] to i32
+; CHECK-NEXT:    [[TMP29:%.*]] = mul nsw i32 [[TMP26]], [[TMP28]]
+; CHECK-NEXT:    [[TMP30:%.*]] = trunc i64 [[INDVARS_IV6]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = srem i32 [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @B, i64 0, i64 [[INDVARS_IV6]], i64 [[INDVARS_IV3]]
+; CHECK-NEXT:    store i32 [[TMP31]], ptr [[TMP32]], align 4
+; CHECK-NEXT:    br label %[[BB33]]
+; CHECK:       [[BB33]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nuw nsw i64 [[INDVARS_IV3]], 1
+; CHECK-NEXT:    br label %[[BB23]]
+; CHECK:       [[BB35]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT7]] = add nuw nsw i64 [[INDVARS_IV6]], 1
+; CHECK-NEXT:    [[TMP36]] = add nuw nsw i32 [[DOT0]], 1
+; CHECK-NEXT:    br label %[[BB9]]
+; CHECK:       [[BB37]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb9
 
diff --git a/llvm/test/Transforms/LoopFusion/loop_nest.ll b/llvm/test/Transforms/LoopFusion/loop_nest.ll
index 260e7c86eeed..39dcd19fb05a 100644
--- a/llvm/test/Transforms/LoopFusion/loop_nest.ll
+++ b/llvm/test/Transforms/LoopFusion/loop_nest.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 ;
 ;    int A[1024][1024];
@@ -19,28 +20,61 @@
 @A = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 @B = common global [1024 x [1024 x i32]] zeroinitializer, align 16
 
-; CHECK: void @dep_free
-; CHECK-NEXT: bb:
-; CHECK-NEXT: br label %[[LOOP1HEADER:bb[0-9]+]]
-; CHECK: [[LOOP1HEADER]]
-; CHECK: br label %[[LOOP3HEADER:bb[0-9]+]]
-; CHECK: [[LOOP3HEADER]]
-; CHECK: br label %[[LOOP2HEADER:bb[0-9]+]]
-; CHECK: [[LOOP2HEADER]]
-; CHECK: br label %[[LOOP4HEADER:bb[0-9]+]]
-; CHECK: [[LOOP4HEADER]]
-; CHECK: br i1 %{{.*}}, label %[[LOOP3HEADER]], label %[[LOOP1LATCH:bb[0-9]+]]
-; CHECK: [[LOOP1LATCH]]
-; CHECK-NEXT: %inc.outer.fc0 = add nuw nsw i64 %indvars.iv105, 1
-; CHECK-NEXT: %add.outer.fc0 = add nuw nsw i32 %.06, 1
-; CHECK-NEXT: %cmp.outer.fc0 = icmp ne i64 %inc.outer.fc0, 100
-; CHECK: br i1 %{{.*}}, label %[[LOOP1HEADER]], label %[[LOOP1EXIT:bb[0-9]*]]
-; CHECK: ret void
 
 ; TODO: The current version of loop fusion does not allow the inner loops to be
 ; fused because they are not control flow equivalent and adjacent. These are
 ; limitations that can be addressed in future improvements to fusion.
 define void @dep_free() {
+; CHECK-LABEL: define void @dep_free() {
+; CHECK-NEXT:  [[BB:.*]]:
+; CHECK-NEXT:    br label %[[BB16:.*]]
+; CHECK:       [[BB16]]:
+; CHECK-NEXT:    [[DOT06:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[ADD_OUTER_FC0:%.*]], %[[BB45:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV105:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INC_OUTER_FC0:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    [[DOT023:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP46:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    [[INDVARS_IV42:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[INDVARS_IV_NEXT5:%.*]], %[[BB45]] ]
+; CHECK-NEXT:    br label %[[BB18:.*]]
+; CHECK:       [[BB18]]:
+; CHECK-NEXT:    [[INDVARS_IV74:%.*]] = phi i64 [ 0, %[[BB16]] ], [ [[INDVARS_IV_NEXT8:%.*]], %[[BB43:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, %[[BB16]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[BB43]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT06]], -3
+; CHECK-NEXT:    [[TMP19:%.*]] = add nuw nsw i64 [[INDVARS_IV105]], 3
+; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nsw i32 [[TMP]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[INDVARS_IV105]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = srem i32 [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @A, i64 0, i64 [[INDVARS_IV105]], i64 [[INDVARS_IV74]]
+; CHECK-NEXT:    store i32 [[TMP23]], ptr [[TMP24]], align 4
+; CHECK-NEXT:    br label %[[BB25:.*]]
+; CHECK:       [[BB25]]:
+; CHECK-NEXT:    [[TMP36:%.*]] = add nsw i32 [[DOT023]], -3
+; CHECK-NEXT:    [[TMP37:%.*]] = add nuw nsw i64 [[INDVARS_IV42]], 3
+; CHECK-NEXT:    [[TMP38:%.*]] = trunc i64 [[TMP37]] to i32
+; CHECK-NEXT:    [[TMP39:%.*]] = mul nsw i32 [[TMP36]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = trunc i64 [[INDVARS_IV42]] to i32
+; CHECK-NEXT:    [[TMP41:%.*]] = srem i32 [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [1024 x [1024 x i32]], ptr @B, i64 0, i64 [[INDVARS_IV42]], i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP41]], ptr [[TMP42]], align 4
+; CHECK-NEXT:    br label %[[BB43]]
+; CHECK:       [[BB31:.*]]:
+; CHECK-NEXT:    br label %[[BB47:.*]]
+; CHECK:       [[BB43]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT8]] = add nuw nsw i64 [[INDVARS_IV74]], 1
+; CHECK-NEXT:    [[EXITCOND9:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT8]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[BB18]], label %[[BB45]]
+; CHECK:       [[BB45]]:
+; CHECK-NEXT:    [[INC_OUTER_FC0]] = add nuw nsw i64 [[INDVARS_IV105]], 1
+; CHECK-NEXT:    [[ADD_OUTER_FC0]] = add nuw nsw i32 [[DOT06]], 1
+; CHECK-NEXT:    [[CMP_OUTER_FC0:%.*]] = icmp ne i64 [[INC_OUTER_FC0]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT5]] = add nuw nsw i64 [[INDVARS_IV42]], 1
+; CHECK-NEXT:    [[TMP46]] = add nuw nsw i32 [[DOT023]], 1
+; CHECK-NEXT:    [[EXITCOND6:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT5]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND6]], label %[[BB16]], label %[[BB31]]
+; CHECK:       [[BB47]]:
+; CHECK-NEXT:    ret void
+;
 bb:
   br label %bb16
 
diff --git a/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
index 1bc70ef4f3aa..c9e57b671a5a 100644
--- a/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
+++ b/llvm/test/Transforms/LoopFusion/nonadjacent_peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests that we do not fuse these two loops together. These loops do not have
@@ -6,29 +7,57 @@
 ; peeling).
 ; The expected output of this test is the function below.
 
-; CHECK-LABEL: void @function(ptr noalias %arg)
-; CHECK-NEXT:  for.first.preheader:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br label %for.first.latch
-; CHECK:       for.first.latch:
-; CHECK:         br i1 %exitcond4, label %for.first, label %for.first.exit
-; CHECK:       for.first.exit:
-; CHECK-NEXT:    br label %for.next
-; CHECK:       for.next:
-; CHECK-NEXT:    br label %for.second.preheader
-; CHECK:       for.second.preheader:
-; CHECK:         br label %for.second
-; CHECK:       for.second:
-; CHECK:         br label %for.second.latch
-; CHECK:       for.second.latch:
-; CHECK:         br i1 %exitcond, label %for.second, label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
 define void @function(ptr noalias %arg) {
+; CHECK-LABEL: define void @function(
+; CHECK-SAME: ptr noalias [[ARG:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_PREHEADER:.*]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[DOT014:%.*]] = phi i32 [ 0, %[[FOR_FIRST_PREHEADER]] ], [ [[TMP15:%.*]], %[[FOR_FIRST_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ 0, %[[FOR_FIRST_PREHEADER]] ], [ [[INDVARS_IV_NEXT3:%.*]], %[[FOR_FIRST_LATCH]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT014]], -3
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[INDVARS_IV23]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i32 [[TMP]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[INDVARS_IV23]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = srem i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH]]
+; CHECK:       [[FOR_FIRST_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[TMP15]] = add nuw nsw i32 [[DOT014]], 1
+; CHECK-NEXT:    [[EXITCOND4:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND4]], label %[[FOR_FIRST]], label %[[FOR_FIRST_EXIT:.*]]
+; CHECK:       [[FOR_FIRST_EXIT]]:
+; CHECK-NEXT:    br label %[[FOR_NEXT:.*]]
+; CHECK:       [[FOR_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND_PREHEADER:.*]]
+; CHECK:       [[FOR_SECOND_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_SECOND:.*]]
+; CHECK:       [[FOR_SECOND]]:
+; CHECK-NEXT:    [[DOT02:%.*]] = phi i32 [ 0, %[[FOR_SECOND_PREHEADER]] ], [ [[TMP28:%.*]], %[[FOR_SECOND_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 3, %[[FOR_SECOND_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw i32 [[DOT02]], -3
+; CHECK-NEXT:    [[TMP21:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nsw i32 [[TMP20]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = srem i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    br label %[[FOR_SECOND_LATCH]]
+; CHECK:       [[FOR_SECOND_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP28]] = add nuw nsw i32 [[DOT02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_SECOND]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.preheader:
   br label %for.first
 
diff --git a/llvm/test/Transforms/LoopFusion/peel.ll b/llvm/test/Transforms/LoopFusion/peel.ll
index 0ee3a3b06803..e8a4817f4c06 100644
--- a/llvm/test/Transforms/LoopFusion/peel.ll
+++ b/llvm/test/Transforms/LoopFusion/peel.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion -loop-fusion-peel-max-count=3 < %s | FileCheck %s
 
 ; Tests whether we can fuse two loops together if they have constant but a
@@ -17,45 +18,106 @@
 ;     B[i] = ((i-6)*(i+3)) % i;
 ; }
 
-; CHECK-LABEL: void @function(ptr noalias %arg)
-; CHECK-NEXT:  for.first.preheader:
-; CHECK-NEXT:    br label %for.first.peel.begin
-; CHECK:       for.first.peel.begin:
-; CHECK-NEXT:    br label %for.first.peel
-; CHECK:       for.first.peel:
-; CHECK:         br label %for.first.latch.peel
-; CHECK:       for.first.latch.peel:
-; CHECK:         br label %for.first.peel.next
-; CHECK:       for.first.peel.next:
-; CHECK-NEXT:    br label %for.first.peel2
-; CHECK:       for.first.peel2:
-; CHECK:         br label %for.first.latch.peel10
-; CHECK:       for.first.latch.peel10:
-; CHECK:         br label %for.first.peel.next1
-; CHECK:       for.first.peel.next1:
-; CHECK-NEXT:    br label %for.first.peel15
-; CHECK:       for.first.peel15:
-; CHECK:         br label %for.first.latch.peel23
-; CHECK:       for.first.latch.peel23:
-; CHECK:         br label %for.first.peel.next14
-; CHECK:       for.first.peel.next14:
-; CHECK-NEXT:    br label %for.first.peel.next27
-; CHECK:       for.first.peel.next27:
-; CHECK-NEXT:    br label %for.first.preheader.peel.newph
-; CHECK:       for.first.preheader.peel.newph:
-; CHECK-NEXT:    br label %for.first
-; CHECK:       for.first:
-; CHECK:         br label %for.first.latch
-; CHECK:       for.first.latch:
-; CHECK:         br label %for.second.latch
-; CHECK:       for.second.latch:
-; CHECK:         br i1 %exitcond, label %for.first, label %for.end
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
 define void @function(ptr noalias %arg) {
+; CHECK-LABEL: define void @function(
+; CHECK-SAME: ptr noalias [[ARG:%.*]]) {
+; CHECK-NEXT:  [[FOR_FIRST_PREHEADER:.*:]]
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_BEGIN:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_BEGIN]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_PEEL]]:
+; CHECK-NEXT:    [[TMP_PEEL:%.*]] = add nsw i32 0, -3
+; CHECK-NEXT:    [[TMP8_PEEL:%.*]] = add nuw nsw i64 0, 3
+; CHECK-NEXT:    [[TMP9_PEEL:%.*]] = trunc i64 [[TMP8_PEEL]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL:%.*]] = mul nsw i32 [[TMP_PEEL]], [[TMP9_PEEL]]
+; CHECK-NEXT:    [[TMP11_PEEL:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[TMP12_PEEL:%.*]] = srem i32 [[TMP10_PEEL]], [[TMP11_PEEL]]
+; CHECK-NEXT:    [[TMP13_PEEL:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 0
+; CHECK-NEXT:    store i32 [[TMP12_PEEL]], ptr [[TMP13_PEEL]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL:%.*]] = add nuw nsw i64 0, 1
+; CHECK-NEXT:    [[TMP15_PEEL:%.*]] = add nuw nsw i32 0, 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL2:.*]]
+; CHECK:       [[FOR_FIRST_PEEL2]]:
+; CHECK-NEXT:    [[TMP_PEEL3:%.*]] = add nsw i32 [[TMP15_PEEL]], -3
+; CHECK-NEXT:    [[TMP8_PEEL4:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL]], 3
+; CHECK-NEXT:    [[TMP9_PEEL5:%.*]] = trunc i64 [[TMP8_PEEL4]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL6:%.*]] = mul nsw i32 [[TMP_PEEL3]], [[TMP9_PEEL5]]
+; CHECK-NEXT:    [[TMP11_PEEL7:%.*]] = trunc i64 [[INDVARS_IV_NEXT3_PEEL]] to i32
+; CHECK-NEXT:    [[TMP12_PEEL8:%.*]] = srem i32 [[TMP10_PEEL6]], [[TMP11_PEEL7]]
+; CHECK-NEXT:    [[TMP13_PEEL9:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV_NEXT3_PEEL]]
+; CHECK-NEXT:    store i32 [[TMP12_PEEL8]], ptr [[TMP13_PEEL9]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL10:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL10]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL11:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL]], 1
+; CHECK-NEXT:    [[TMP15_PEEL12:%.*]] = add nuw nsw i32 [[TMP15_PEEL]], 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL13:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL11]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT1:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT1]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL15:.*]]
+; CHECK:       [[FOR_FIRST_PEEL15]]:
+; CHECK-NEXT:    [[TMP_PEEL16:%.*]] = add nsw i32 [[TMP15_PEEL12]], -3
+; CHECK-NEXT:    [[TMP8_PEEL17:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL11]], 3
+; CHECK-NEXT:    [[TMP9_PEEL18:%.*]] = trunc i64 [[TMP8_PEEL17]] to i32
+; CHECK-NEXT:    [[TMP10_PEEL19:%.*]] = mul nsw i32 [[TMP_PEEL16]], [[TMP9_PEEL18]]
+; CHECK-NEXT:    [[TMP11_PEEL20:%.*]] = trunc i64 [[INDVARS_IV_NEXT3_PEEL11]] to i32
+; CHECK-NEXT:    [[TMP12_PEEL21:%.*]] = srem i32 [[TMP10_PEEL19]], [[TMP11_PEEL20]]
+; CHECK-NEXT:    [[TMP13_PEEL22:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV_NEXT3_PEEL11]]
+; CHECK-NEXT:    store i32 [[TMP12_PEEL21]], ptr [[TMP13_PEEL22]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH_PEEL23:.*]]
+; CHECK:       [[FOR_FIRST_LATCH_PEEL23]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3_PEEL24:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT3_PEEL11]], 1
+; CHECK-NEXT:    [[TMP15_PEEL25:%.*]] = add nuw nsw i32 [[TMP15_PEEL12]], 1
+; CHECK-NEXT:    [[EXITCOND4_PEEL26:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3_PEEL24]], 100
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT14:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT14]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PEEL_NEXT27:.*]]
+; CHECK:       [[FOR_FIRST_PEEL_NEXT27]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST_PREHEADER_PEEL_NEWPH:.*]]
+; CHECK:       [[FOR_FIRST_PREHEADER_PEEL_NEWPH]]:
+; CHECK-NEXT:    br label %[[FOR_FIRST:.*]]
+; CHECK:       [[FOR_FIRST]]:
+; CHECK-NEXT:    [[DOT014:%.*]] = phi i32 [ [[TMP15_PEEL25]], %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[TMP15:%.*]], %[[FOR_SECOND_LATCH:.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV23:%.*]] = phi i64 [ [[INDVARS_IV_NEXT3_PEEL24]], %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[INDVARS_IV_NEXT3:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[DOT02:%.*]] = phi i32 [ 0, %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[TMP28:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 3, %[[FOR_FIRST_PREHEADER_PEEL_NEWPH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_SECOND_LATCH]] ]
+; CHECK-NEXT:    [[TMP:%.*]] = add nsw i32 [[DOT014]], -3
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[INDVARS_IV23]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw i32 [[TMP]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[INDVARS_IV23]] to i32
+; CHECK-NEXT:    [[TMP12:%.*]] = srem i32 [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARG]], i64 [[INDVARS_IV23]]
+; CHECK-NEXT:    store i32 [[TMP12]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    br label %[[FOR_FIRST_LATCH:.*]]
+; CHECK:       [[FOR_FIRST_LATCH]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = add nsw i32 [[DOT02]], -3
+; CHECK-NEXT:    [[TMP21:%.*]] = add nuw nsw i64 [[INDVARS_IV1]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP21]] to i32
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nsw i32 [[TMP20]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[INDVARS_IV1]] to i32
+; CHECK-NEXT:    [[TMP25:%.*]] = srem i32 [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 [[INDVARS_IV1]]
+; CHECK-NEXT:    store i32 [[TMP25]], ptr [[TMP26]], align 4
+; CHECK-NEXT:    br label %[[FOR_SECOND_LATCH]]
+; CHECK:       [[FOR_SECOND_LATCH]]:
+; CHECK-NEXT:    [[INDVARS_IV_NEXT3]] = add nuw nsw i64 [[INDVARS_IV23]], 1
+; CHECK-NEXT:    [[TMP15]] = add nuw nsw i32 [[DOT014]], 1
+; CHECK-NEXT:    [[EXITCOND4:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT3]], 100
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; CHECK-NEXT:    [[TMP28]] = add nuw nsw i32 [[DOT02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_FIRST]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 for.first.preheader:
   br label %for.first
 
diff --git a/llvm/test/Transforms/LoopFusion/sink_preheader.ll b/llvm/test/Transforms/LoopFusion/sink_preheader.ll
index 7e4559cfc85e..be416db30da6 100644
--- a/llvm/test/Transforms/LoopFusion/sink_preheader.ll
+++ b/llvm/test/Transforms/LoopFusion/sink_preheader.ll
@@ -1,19 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
 
 define void @sink_preheader(i32 %N) {
-; CHECK-LABEL: @sink_preheader(
-; CHECK-NEXT:  pre1:
-; CHECK-NEXT:    br label [[BODY1:%.*]]
-; CHECK:       body1:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[BODY1]] ], [ 0, [[PRE1:%.*]] ]
-; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], [[BODY1]] ], [ 0, [[PRE1]] ]
+; CHECK-LABEL: define void @sink_preheader(
+; CHECK-SAME: i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[PRE1:.*]]:
+; CHECK-NEXT:    br label %[[BODY1:.*]]
+; CHECK:       [[BODY1]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
+; CHECK-NEXT:    [[I2:%.*]] = phi i32 [ [[I_NEXT2:%.*]], %[[BODY1]] ], [ 0, %[[PRE1]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 1, [[I]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N:%.*]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ne i32 [[I]], [[N]]
 ; CHECK-NEXT:    [[BARRIER:%.*]] = add i32 1, [[N]]
 ; CHECK-NEXT:    [[I_NEXT2]] = add i32 1, [[I2]]
 ; CHECK-NEXT:    [[COND2:%.*]] = icmp ne i32 [[I2]], [[N]]
-; CHECK-NEXT:    br i1 [[COND2]], label [[BODY1]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[COND2]], label %[[BODY1]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[SINKME:%.*]] = add i32 1, [[BARRIER]]
 ; CHECK-NEXT:    [[SINKME2:%.*]] = add i32 1, [[BARRIER]]
 ; CHECK-NEXT:    [[SINKME3:%.*]] = add i32 1, [[SINKME2]]
diff --git a/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll b/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll
new file mode 100644
index 000000000000..d0b6447d92fe
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/position-in-pipeline.ll
@@ -0,0 +1,48 @@
+; RUN: opt -passes='default<O3>' -enable-loopinterchange -disable-output \
+; RUN:     -disable-verify -verify-analysis-invalidation=0 \
+; RUN:     -debug-pass-manager=quiet %s 2>&1 | FileCheck %s
+
+; Test the position of LoopInterchange in the pass pipeline.
+
+; CHECK-NOT:  Running pass: LoopInterchangePass
+; CHECK:      Running pass: ControlHeightReductionPass
+; CHECK-NEXT: Running pass: LoopSimplifyPass
+; CHECK-NEXT: Running pass: LCSSAPass
+; CHECK-NEXT: Running pass: LoopRotatePass
+; CHECK-NEXT: Running pass: LoopDeletionPass
+; CHECK-NEXT: Running pass: LoopRotatePass
+; CHECK-NEXT: Running pass: LoopDeletionPass
+; CHECK-NEXT: Running pass: LoopInterchangePass
+; CHECK-NEXT: Running pass: LoopDistributePass
+; CHECK-NEXT: Running pass: InjectTLIMappings
+; CHECK-NEXT: Running pass: LoopVectorizePass
+
+
+define void @foo(ptr %a, i32 %n) {
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.i.latch ]
+  br label %for.j
+
+for.j:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.next, %for.j ]
+  %tmp = mul i32 %i, %n
+  %offset = add i32 %tmp, %j
+  %idx = getelementptr inbounds i32, ptr %a, i32 %offset
+  %load = load i32, ptr %idx, align 4
+  %inc = add i32 %load, 1
+  store i32 %inc, ptr %idx, align 4
+  %j.next = add i32 %j, 1
+  %j.exit = icmp eq i32 %j.next, %n
+  br i1 %j.exit, label %for.i.latch, label %for.j
+
+for.i.latch:
+  %i.next = add i32 %i, 1
+  %i.exit = icmp eq i32 %i.next, %n
+  br i1 %i.exit, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopRotate/dbgvalue.ll b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
index 9651aeb43057..939554f741b0 100644
--- a/llvm/test/Transforms/LoopRotate/dbgvalue.ll
+++ b/llvm/test/Transforms/LoopRotate/dbgvalue.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -passes=loop-rotate -verify-memoryssa < %s | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt -S -passes=loop-rotate -verify-memoryssa < %s | FileCheck %s --implicit-check-not=dbg_value
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata) nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) nounwind readnone
diff --git a/llvm/test/Transforms/LoopRotate/delete-dbg-values.ll b/llvm/test/Transforms/LoopRotate/delete-dbg-values.ll
index ed6012b8a079..be2c8cb96f57 100644
--- a/llvm/test/Transforms/LoopRotate/delete-dbg-values.ll
+++ b/llvm/test/Transforms/LoopRotate/delete-dbg-values.ll
@@ -1,4 +1,4 @@
-; RUN: opt --passes=loop-rotate -o - -S %s | FileCheck %s --implicit-check-not=dbg.value
+; RUN: opt --passes=loop-rotate -o - -S %s | FileCheck %s --implicit-check-not=dbg_value
 ;
 ;; Test some fine-grained behaviour of loop-rotate's de-duplication of
 ;; dbg.values. The intrinsic on the first branch should be seen and
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
index 9435c544fc81..1f619898ea78 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
@@ -92,7 +92,7 @@ for.cond.cleanup:                                 ; preds = %for.body
 ; CHECK-V1-IC1: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK-V1-IC1: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK-V1-IC1: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 7}
 ; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
 ; CHECK-V1-IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META2]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
index e017b689067f..19ee763e6ffa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -disable-output \
+; RUN: opt -S < %s -p loop-vectorize -disable-output \
 ; RUN:   -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefixes=CHECK
 
 target triple = "aarch64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll
new file mode 100644
index 000000000000..e6d5c541560c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/licm-calls.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @licm_replicate_call(double %x, ptr %dst) {
+; CHECK-LABEL: define void @licm_replicate_call(
+; CHECK-SAME: double [[X:%.*]], ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[TMP1]], i32 1
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT:    [[TMP4:%.*]] = uitofp <2 x i32> [[VEC_IND]] to <2 x double>
+; CHECK-NEXT:    [[TMP5:%.*]] = uitofp <2 x i32> [[STEP_ADD]] to <2 x double>
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds double, ptr [[TMP8]], i32 2
+; CHECK-NEXT:    store <2 x double> [[TMP6]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    store <2 x double> [[TMP7]], ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[STEP_ADD]], splat (i32 2)
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[IV_AS_FP:%.*]] = uitofp i32 [[IV_TRUNC]] to double
+; CHECK-NEXT:    [[P:%.*]] = tail call double @llvm.pow.f64(double [[X]], double 3.000000e+00)
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[P]], [[IV_AS_FP]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[MUL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 128
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.trunc = trunc i64 %iv to i32
+  %iv.as.fp = uitofp i32 %iv.trunc to double
+  %p = tail call double @llvm.pow.f64(double %x, double 3.000000e+00)
+  %mul = fmul double %p, %iv.as.fp
+  %gep.dst = getelementptr inbounds double, ptr %dst, i64 %iv
+  store double %mul, ptr %gep.dst, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 128
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare double @llvm.pow.f64(double, double)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index 295fe1b891e0..b02b314ecbd6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -1030,6 +1030,472 @@ for.body:                                         ; preds = %for.body.preheader,
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
 }
 
+
+define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP9]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
+; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP17]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP18]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_madd_extadd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP15]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add = add nsw i32 %res, %mul.ab
+  %add2 = add i32 %add, %c.ext
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
+define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP5]])
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE2]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP6]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP9]], align 1
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP11]]
+; CHECK-SVE-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[TMP12]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_extadd_extadd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP11]])
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE2]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP12]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %add = add nsw i32 %res, %a.ext
+  %add2 = add i32 %add, %b.ext
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
+
+define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
+; CHECK-NEON-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-NEON-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEON-NEXT:  entry:
+; CHECK-NEON-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEON-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-NEON-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-NEON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEON:       vector.ph:
+; CHECK-NEON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 16
+; CHECK-NEON-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-NEON:       vector.body:
+; CHECK-NEON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEON-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEON-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEON-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEON-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i32 0
+; CHECK-NEON-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEON-NEXT:    [[TMP7:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP8:%.*]] = sext <16 x i8> [[WIDE_LOAD1]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[TMP9:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP9]])
+; CHECK-NEON-NEXT:    [[TMP10:%.*]] = mul nsw <16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-NEON-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[PARTIAL_REDUCE]], <16 x i32> [[TMP10]])
+; CHECK-NEON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEON-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEON:       middle.block:
+; CHECK-NEON-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-NEON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEON-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-NEON:       scalar.ph:
+;
+; CHECK-SVE-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-SVE-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-NEXT:  entry:
+; CHECK-SVE-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE:       vector.ph:
+; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE:       vector.body:
+; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-NEXT:    [[TMP13:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
+; CHECK-SVE-NEXT:    [[TMP16:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
+; CHECK-SVE-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP16]], [[TMP17]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-SVE:       middle.block:
+; CHECK-SVE-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP18]])
+; CHECK-SVE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE:       scalar.ph:
+;
+; CHECK-SVE-MAXBW-LABEL: define i32 @chained_partial_reduce_extadd_madd(
+; CHECK-SVE-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-SVE-MAXBW-NEXT:  entry:
+; CHECK-SVE-MAXBW-NEXT:    [[CMP28_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
+; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-SVE-MAXBW:       vector.ph:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
+; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-SVE-MAXBW:       vector.body:
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[C]], i64 [[INDEX]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP9]], i32 0
+; CHECK-SVE-MAXBW-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 1
+; CHECK-SVE-MAXBW-NEXT:    [[TMP13:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP14:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP15]])
+; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
+; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP16]])
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-SVE-MAXBW:       middle.block:
+; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-SVE-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-SVE-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK-SVE-MAXBW:       scalar.ph:
+;
+entry:
+  %cmp28.not = icmp ult i32 %N, 2
+  %div27 = lshr i32 %N, 1
+  %wide.trip.count = zext nneg i32 %div27 to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  %res.0.lcssa = phi i32 [ %add2, %for.body ]
+  ret i32 %res.0.lcssa
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %res = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %a.ptr = getelementptr inbounds nuw i8, ptr %a, i64 %indvars.iv
+  %b.ptr = getelementptr inbounds nuw i8, ptr %b, i64 %indvars.iv
+  %c.ptr = getelementptr inbounds nuw i8, ptr %c, i64 %indvars.iv
+  %a.val = load i8, ptr %a.ptr, align 1
+  %b.val = load i8, ptr %b.ptr, align 1
+  %c.val = load i8, ptr %c.ptr, align 1
+  %a.ext = sext i8 %a.val to i32
+  %b.ext = sext i8 %b.val to i32
+  %c.ext = sext i8 %c.val to i32
+  %add = add nsw i32 %res, %c.ext
+  %mul.ab = mul nsw i32 %a.ext, %b.ext
+  %add2 = add i32 %add, %mul.ab
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !loop !1
+}
+
+
 attributes #0 = { vscale_range(1,16) }
 
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
index d2a82269e1b8..1d3741482212 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
@@ -72,8 +72,8 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
@@ -94,14 +94,14 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP19:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP18]], [[TMP11]]
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
-; CHECK-NOI8MM-NEXT:    [[TMP22]] = add <vscale x 8 x i32> [[TMP20]], [[VEC_PHI]]
-; CHECK-NOI8MM-NEXT:    [[TMP23]] = add <vscale x 8 x i32> [[TMP21]], [[VEC_PHI1]]
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
 ; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
-; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i32> [[TMP23]], [[TMP22]]
-; CHECK-NOI8MM-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[BIN_RDX]])
+; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-NOI8MM-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[BIN_RDX]])
 ; CHECK-NOI8MM-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-NOI8MM:       scalar.ph:
@@ -195,8 +195,8 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP22:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
@@ -217,14 +217,14 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP19:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP18]], [[TMP11]]
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
-; CHECK-NOI8MM-NEXT:    [[TMP22]] = add <vscale x 8 x i32> [[TMP20]], [[VEC_PHI]]
-; CHECK-NOI8MM-NEXT:    [[TMP23]] = add <vscale x 8 x i32> [[TMP21]], [[VEC_PHI1]]
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
+; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
 ; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
-; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i32> [[TMP23]], [[TMP22]]
-; CHECK-NOI8MM-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[BIN_RDX]])
+; CHECK-NOI8MM-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 2 x i32> [[PARTIAL_REDUCE5]], [[PARTIAL_REDUCE]]
+; CHECK-NOI8MM-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[BIN_RDX]])
 ; CHECK-NOI8MM-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-NOI8MM:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index a44b9bfb3b44..887f95ce31f4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -2590,514 +2590,8 @@ exit:                                 ; preds = %for.cond.cleanup.loopexit, %ent
 }
 
 
-; == Partial reductions with add of an extend
-
-define i32 @zext_add_reduc_i8_i32(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i32
-  %add = add i32 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-
-define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <8 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr [[TMP1]], i32 8
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[WIDE_LOAD2]] to <8 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <8 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i16_i64(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 4 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv4i64(<vscale x 4 x i64> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i16, ptr %a, i64 %iv
-  %load.a = load i16, ptr %gep.a, align 2
-  %ext.a = zext i16 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(ptr %a) #1 {
-; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP10]] = add <vscale x 8 x i64> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP10]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = zext i8 %load.a to i64
-  %add = add i64 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i64 %add
-}
-
-
-define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT:  entry:
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVE1:       vector.body:
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-INTERLEAVE1:       middle.block:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVE1:       scalar.ph:
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT:  entry:
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-INTERLEAVED:       vector.body:
-; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
-; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
-; CHECK-INTERLEAVED:       middle.block:
-; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-INTERLEAVED:       scalar.ph:
-;
-; CHECK-MAXBW-LABEL: define i32 @sext_add_reduc_i8_i32(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT:  entry:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK-MAXBW:       vector.body:
-; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
-; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP12]] = add <vscale x 8 x i32> [[TMP9]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP12]])
-; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK-MAXBW:       scalar.ph:
-;
-entry:
-  br label %for.body
-
-for.body:                                         ; preds = %for.body, %entry
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
-  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
-  %gep.a = getelementptr i8, ptr %a, i64 %iv
-  %load.a = load i8, ptr %gep.a, align 1
-  %ext.a = sext i8 %load.a to i32
-  %add = add i32 %ext.a, %accum
-  %iv.next = add i64 %iv, 1
-  %exitcond.not = icmp eq i64 %iv.next, 1025
-  br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit:                        ; preds = %for.body
-  ret i32 %add
-}
-
-define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #1 {
-; CHECK-INTERLEAVE1-LABEL: define dso_local void @dotp_high_register_pressure(
+define dso_local void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32 %n) #1 {
+; CHECK-INTERLEAVE1-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3184,7 +2678,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP36]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP36]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
@@ -3198,7 +2692,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
 ;
-; CHECK-INTERLEAVED-LABEL: define dso_local void @dotp_high_register_pressure(
+; CHECK-INTERLEAVED-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3285,7 +2779,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE21]])
 ; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE20]])
@@ -3299,7 +2793,7 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-INTERLEAVED:       scalar.ph:
 ;
-; CHECK-MAXBW-LABEL: define dso_local void @dotp_high_register_pressure(
+; CHECK-MAXBW-LABEL: define dso_local void @not_dotp_high_register_pressure(
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[SUM:%.*]], i32 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[CMP100:%.*]] = icmp sgt i32 [[N]], 0
@@ -3337,14 +2831,14 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[TMP36:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[TMP21:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE21:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP1]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI4:%.*]] = phi <4 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI6:%.*]] = phi <4 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI7:%.*]] = phi <4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i32 0
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP9]], align 1
@@ -3362,40 +2856,40 @@ define dso_local void @dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum, i32
 ; CHECK-MAXBW-NEXT:    [[STRIDED_VEC14:%.*]] = shufflevector <128 x i8> [[WIDE_VEC]], <128 x i8> poison, <16 x i32> <i32 7, i32 15, i32 23, i32 31, i32 39, i32 47, i32 55, i32 63, i32 71, i32 79, i32 87, i32 95, i32 103, i32 111, i32 119, i32 127>
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = sext <16 x i8> [[STRIDED_VEC]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = mul nsw <16 x i32> [[TMP13]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP15]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP14]])
-; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = sext <16 x i8> [[STRIDED_VEC8]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP29]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP18]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP16]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI7]], <16 x i32> [[TMP14]])
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = sext <16 x i8> [[STRIDED_VEC8]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <16 x i32> [[TMP15]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE15]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI6]], <16 x i32> [[TMP16]])
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = sext <16 x i8> [[STRIDED_VEC9]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP31]])
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <16 x i32> [[TMP17]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE16]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI5]], <16 x i32> [[TMP18]])
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = sext <16 x i8> [[STRIDED_VEC10]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = mul nsw <16 x i32> [[TMP19]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP24]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP20]])
-; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = sext <16 x i8> [[STRIDED_VEC11]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP32]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP27]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP22]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE17]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI4]], <16 x i32> [[TMP20]])
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = sext <16 x i8> [[STRIDED_VEC11]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul nsw <16 x i32> [[TMP21]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE18]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP22]])
 ; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = sext <16 x i8> [[STRIDED_VEC12]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP30]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP34]])
+; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = mul nsw <16 x i32> [[TMP23]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE19]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP24]])
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = sext <16 x i8> [[STRIDED_VEC13]] to <16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = mul nsw <16 x i32> [[TMP25]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP33]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP26]])
-; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = sext <16 x i8> [[STRIDED_VEC14]] to <16 x i32>
-; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP35]], [[TMP10]]
-; CHECK-MAXBW-NEXT:    [[TMP36]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE20]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP26]])
+; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = sext <16 x i8> [[STRIDED_VEC14]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = mul nsw <16 x i32> [[TMP27]], [[TMP10]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE21]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP28]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP38:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP36]])
-; CHECK-MAXBW-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP33]])
-; CHECK-MAXBW-NEXT:    [[TMP40:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP30]])
-; CHECK-MAXBW-NEXT:    [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]])
-; CHECK-MAXBW-NEXT:    [[TMP42:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP24]])
-; CHECK-MAXBW-NEXT:    [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP21]])
-; CHECK-MAXBW-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
-; CHECK-MAXBW-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE21]])
+; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE20]])
+; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE19]])
+; CHECK-MAXBW-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE18]])
+; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE17]])
+; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-MAXBW-NEXT:    [[TMP36:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE15]])
+; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
new file mode 100644
index 000000000000..5d5ee570da0f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
@@ -0,0 +1,918 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+; == Partial reductions with add of an extend
+
+define i32 @zext_add_reduc_i8_i32_sve(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3]] = add <16 x i32> [[TMP2]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP3]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP6]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_sve(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+define i32 @zext_add_reduc_i8_i32_neon(ptr %a) #2 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP2]])
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP0]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP3]])
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE3]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_neon(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP2]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-MAXBW-NEXT:    br i1 [[TMP3]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i8_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i64>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 16 x i64> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <8 x i64> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <8 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i16, ptr [[TMP1]], i32 8
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP2]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <8 x i16>, ptr [[TMP3]], align 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <8 x i16> [[WIDE_LOAD2]] to <8 x i64>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <8 x i64> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <8 x i64> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <8 x i64> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i64 @zext_add_reduc_i16_i64(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i16, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 2
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 8 x i64> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i16, ptr %a, i64 %iv
+  %load.a = load i16, ptr %gep.a, align 2
+  %ext.a = zext i16 %load.a to i64
+  %add = add i64 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i64 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP3]])
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE3]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP7]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i8> @llvm.masked.load.nxv4i8.p0(ptr [[TMP7]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i8> poison)
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_predicated(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP7]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 1025)
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[TMP10]], i32 0
+; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_add_reduc_i8_i32_mul_use(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       for.body:
+; CHECK-INTERLEAVE1-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVE1-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-INTERLEAVE1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVE1:       for.exit:
+; CHECK-INTERLEAVE1-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-INTERLEAVED-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVED:       for.exit:
+; CHECK-INTERLEAVED-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_add_reduc_i8_i32_mul_use(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-MAXBW:       for.body:
+; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[ACCUM2:%.*]] = phi i32 [ 13, [[ENTRY]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT:    [[LOAD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-MAXBW-NEXT:    [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT:    [[ADD]] = add i32 [[EXT_A]], [[ACCUM]]
+; CHECK-MAXBW-NEXT:    [[ADD2]] = add i32 [[EXT_A]], [[ACCUM2]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-MAXBW-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
+; CHECK-MAXBW-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-MAXBW:       for.exit:
+; CHECK-MAXBW-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    ret i32 [[ADD_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %accum2 = phi i32 [ 13, %entry ], [ %add2, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %add2 = add i32 %ext.a, %accum2
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = sub <16 x i32> [[VEC_PHI]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = sub <16 x i32> [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = sub <16 x i32> [[VEC_PHI1]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP10]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP10]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %add = sub i32 %accum, %ext.a
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+
+define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4]] = add <16 x i32> [[TMP3]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i32 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add <16 x i32> [[TMP4]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add <16 x i32> [[TMP5]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define i32 @sext_add_reduc_i8_i32(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = sext i8 %load.a to i32
+  %add = add i32 %ext.a, %accum
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1025
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
+attributes #1 = { vscale_range(1,16) "target-features"="+neon,+dotprod,+sve" "cpu"="neoverse-v2" }
+attributes #2 = { "target-features"="+neon,+dotprod" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index 5925b7014b12..3bb20e2a81d8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization | FileCheck %s --check-prefixes=CHECK
+; RUN: opt -S < %s -p loop-vectorize | FileCheck %s --check-prefixes=CHECK
 
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
index 9dfe70ddf1b0..9fee8a390504 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4
-; RUN: opt -p loop-vectorize -enable-early-exit-vectorization -force-vector-interleave=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -force-vector-interleave=4 -S %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
@@ -31,13 +31,43 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 16
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP19]]
+; CHECK-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP37:%.*]] = mul nuw i64 [[TMP36]], 32
+; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP39]], 48
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP40]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP29]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 16 x i8>, ptr [[TMP38]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP41]], align 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 32
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP24]]
+; CHECK-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 48
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[TMP27]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 16 x i8>, ptr [[TMP22]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 16 x i8>, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 16 x i8>, ptr [[TMP28]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD5]], [[WIDE_LOAD6]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
 ; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP11]])
+; CHECK-NEXT:    [[TMP33:%.*]] = or <vscale x 16 x i1> [[TMP11]], [[TMP30]]
+; CHECK-NEXT:    [[TMP34:%.*]] = or <vscale x 16 x i1> [[TMP33]], [[TMP31]]
+; CHECK-NEXT:    [[TMP35:%.*]] = or <vscale x 16 x i1> [[TMP34]], [[TMP32]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP35]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -47,8 +77,27 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 510, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       vector.early.exit:
+; CHECK-NEXT:    [[TMP63:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP63]], 16
+; CHECK-NEXT:    [[TMP44:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP32]], i1 true)
+; CHECK-NEXT:    [[TMP62:%.*]] = mul i64 [[TMP42]], 3
+; CHECK-NEXT:    [[TMP45:%.*]] = add i64 [[TMP62]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP31]], i1 true)
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP42]], 2
+; CHECK-NEXT:    [[TMP50:%.*]] = add i64 [[TMP58]], [[TMP46]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp ne i64 [[TMP46]], [[TMP42]]
+; CHECK-NEXT:    [[TMP51:%.*]] = select i1 [[TMP47]], i64 [[TMP50]], i64 [[TMP45]]
+; CHECK-NEXT:    [[TMP52:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP30]], i1 true)
+; CHECK-NEXT:    [[TMP64:%.*]] = mul i64 [[TMP42]], 1
+; CHECK-NEXT:    [[TMP56:%.*]] = add i64 [[TMP64]], [[TMP52]]
+; CHECK-NEXT:    [[TMP53:%.*]] = icmp ne i64 [[TMP52]], [[TMP42]]
+; CHECK-NEXT:    [[TMP57:%.*]] = select i1 [[TMP53]], i64 [[TMP56]], i64 [[TMP51]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP11]], i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX1]], [[TMP15]]
+; CHECK-NEXT:    [[TMP65:%.*]] = mul i64 [[TMP42]], 0
+; CHECK-NEXT:    [[TMP60:%.*]] = add i64 [[TMP65]], [[TMP15]]
+; CHECK-NEXT:    [[TMP59:%.*]] = icmp ne i64 [[TMP15]], [[TMP42]]
+; CHECK-NEXT:    [[TMP61:%.*]] = select i1 [[TMP59]], i64 [[TMP60]], i64 [[TMP57]]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX1]], [[TMP61]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 3, [[TMP16]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 8c2958769a61..b349c55d3e09 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -367,10 +367,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_ST2:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i32 [[TMP5]], 3
-; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw i32 2, [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i64 2, [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
@@ -381,10 +379,8 @@ define void @test_reversed_load2_store2(ptr noalias nocapture readonly %A, ptr n
 ; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <vscale x 4 x i32> [[REVERSE]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = sub nsw <vscale x 4 x i32> [[REVERSE1]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_ST2]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw nsw i32 [[TMP15]], 3
-; CHECK-NEXT:    [[TMP17:%.*]] = sub nsw i32 2, [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = sub nsw i64 2, [[TMP15]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    [[REVERSE2:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP12]])
 ; CHECK-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP13]])
@@ -1577,10 +1573,8 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_XYZT:%.*]], ptr [[A:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i32 [[TMP6]], 4
-; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw i32 4, [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = sub nsw i64 4, [[TMP6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 16 x i32>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave4.nxv16i32(<vscale x 16 x i32> [[WIDE_VEC]])
@@ -1597,10 +1591,8 @@ define void @interleave_deinterleave_reverse(ptr noalias nocapture readonly %A,
 ; CHECK-NEXT:    [[TMP19:%.*]] = mul nsw <vscale x 4 x i32> [[REVERSE4]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw nsw <vscale x 4 x i32> [[REVERSE5]], [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_XYZT]], ptr [[B:%.*]], i64 [[OFFSET_IDX]], i32 0
-; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 4
-; CHECK-NEXT:    [[TMP24:%.*]] = sub nsw i32 4, [[TMP23]]
-; CHECK-NEXT:    [[TMP25:%.*]] = sext i32 [[TMP24]] to i64
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP25:%.*]] = sub nsw i64 4, [[TMP22]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP25]]
 ; CHECK-NEXT:    [[REVERSE6:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP17]])
 ; CHECK-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP18]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
index 217127765054..173766cc0a65 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
@@ -141,3 +141,310 @@ loop:
 exit:
   ret void
 }
+
+define void @test_interleave_store_one_constant(ptr noalias %src, ptr noalias %dst, i64 %n) #0 {
+; CHECK-LABEL: define void @test_interleave_store_one_constant(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 0, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[DST]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
+; CHECK-NEXT:    [[MUL2:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT3:%.*]] = extractvalue { i64, i1 } [[MUL2]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW4:%.*]] = extractvalue { i64, i1 } [[MUL2]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 0, [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[SCEVGEP]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK5]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr double, ptr [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr double, ptr [[TMP13]], i32 2
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr double, ptr [[TMP13]], i32 4
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr double, ptr [[TMP13]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <2 x double>, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <2 x double>, ptr [[TMP17]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP19:%.*]] = fmul <2 x double> [[WIDE_LOAD6]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul <2 x double> [[WIDE_LOAD7]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul <2 x double> [[WIDE_LOAD8]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <2 x double> [[TMP18]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP22]], align 8
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <2 x double> [[TMP19]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC9:%.*]] = shufflevector <4 x double> [[TMP27]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC9]], ptr [[TMP23]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC10:%.*]] = shufflevector <4 x double> [[TMP28]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC10]], ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <2 x double> [[TMP21]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC11:%.*]] = shufflevector <4 x double> [[TMP29]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC11]], ptr [[TMP25]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF12:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NEXT:    [[N_VEC13:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF12]]
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX14:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr double, ptr [[SRC]], i64 [[INDEX14]]
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr double, ptr [[TMP31]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <2 x double>, ptr [[TMP32]], align 8
+; CHECK-NEXT:    [[TMP33:%.*]] = fmul <2 x double> [[WIDE_LOAD15]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[INDEX14]]
+; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <2 x double> [[TMP33]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP35]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT17]] = add nuw i64 [[INDEX14]], 2
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC13]]
+; CHECK-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC13]]
+; CHECK-NEXT:    br i1 [[CMP_N18]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[GEP_SRC]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[L]], 5.000000e+00
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store double [[MUL]], ptr [[GEP_DST]], align 8
+; CHECK-NEXT:    [[GEP_DST_1:%.*]] = getelementptr [2 x double], ptr [[DST]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_DST_1]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr double, ptr %src, i64 %iv
+  %l = load double, ptr %gep.src, align 8
+  %mul = fmul double %l, 5.000000e+00
+  %gep.dst = getelementptr [2 x double], ptr %dst, i64 %iv
+  store double %mul, ptr %gep.dst, align 8
+  %gep.dst.1 = getelementptr [2 x double], ptr %dst, i64 %iv, i32 1
+  store double 0.000000e+00, ptr %gep.dst.1, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @single_fmul_used_by_each_member(ptr noalias %A, ptr noalias %B, ptr noalias %C, i64 %n) #0 {
+; CHECK-LABEL: define void @single_fmul_used_by_each_member(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 8
+; CHECK-NEXT:    [[MUL:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i64, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i64, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 0, [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult ptr [[TMP2]], [[SCEVGEP]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP3]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 0, [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[MUL_RESULT2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult ptr [[TMP6]], [[B]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW3]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[C]], i64 8
+; CHECK-NEXT:    [[MUL5:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT6:%.*]] = extractvalue { i64, i1 } [[MUL5]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW7:%.*]] = extractvalue { i64, i1 } [[MUL5]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 0, [[MUL_RESULT6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[SCEVGEP4]], i64 [[MUL_RESULT6]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult ptr [[TMP10]], [[SCEVGEP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = or i1 [[TMP11]], [[MUL_OVERFLOW7]]
+; CHECK-NEXT:    [[MUL8:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 16, i64 [[N]])
+; CHECK-NEXT:    [[MUL_RESULT9:%.*]] = extractvalue { i64, i1 } [[MUL8]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW10:%.*]] = extractvalue { i64, i1 } [[MUL8]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i64 0, [[MUL_RESULT9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[C]], i64 [[MUL_RESULT9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[C]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW10]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or i1 [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or i1 [[TMP17]], [[TMP12]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or i1 [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK11:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK11]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr double, ptr [[TMP23]], i32 0
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr double, ptr [[TMP23]], i32 2
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr double, ptr [[TMP23]], i32 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr double, ptr [[TMP23]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, ptr [[TMP24]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <2 x double>, ptr [[TMP25]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <2 x double>, ptr [[TMP26]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <2 x double>, ptr [[TMP27]], align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = fmul <2 x double> [[WIDE_LOAD]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP29:%.*]] = fmul <2 x double> [[WIDE_LOAD12]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP30:%.*]] = fmul <2 x double> [[WIDE_LOAD13]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP31:%.*]] = fmul <2 x double> [[WIDE_LOAD14]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[TMP22]]
+; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> [[TMP28]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP36]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP32]], align 8
+; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> [[TMP29]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC15:%.*]] = shufflevector <4 x double> [[TMP37]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP33]], align 8
+; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> [[TMP30]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC16:%.*]] = shufflevector <4 x double> [[TMP38]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP34]], align 8
+; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> [[TMP31]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC17:%.*]] = shufflevector <4 x double> [[TMP39]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP35]], align 8
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP42:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP43:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[TMP22]]
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP40]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC15]], ptr [[TMP41]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC16]], ptr [[TMP42]], align 8
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC17]], ptr [[TMP43]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF22:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NEXT:    [[N_VEC23:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF22]]
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr double, ptr [[A]], i64 [[INDEX24]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr double, ptr [[TMP45]], i32 0
+; CHECK-NEXT:    [[TMP47:%.*]] = load double, ptr [[TMP46]], align 8
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP47]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP48:%.*]] = fmul <2 x double> [[BROADCAST_SPLAT]], splat (double 5.000000e+00)
+; CHECK-NEXT:    [[TMP49:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[INDEX24]]
+; CHECK-NEXT:    store <2 x double> [[TMP48]], ptr [[TMP49]], align 8
+; CHECK-NEXT:    [[TMP50:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[INDEX24]]
+; CHECK-NEXT:    store <2 x double> [[TMP48]], ptr [[TMP50]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT25]] = add nuw i64 [[INDEX24]], 1
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[TMP51]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N26:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[CMP_N26]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr double, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[GEP_A]], align 8
+; CHECK-NEXT:    [[DIV:%.*]] = fmul double [[L]], 5.000000e+00
+; CHECK-NEXT:    [[GEP_B_1:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_B_1]], align 8
+; CHECK-NEXT:    [[GEP_C_1:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]], i32 1
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_C_1]], align 8
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr { double, double }, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_B]], align 8
+; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr { double, double }, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT:    store double [[DIV]], ptr [[GEP_C]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr double, ptr %A, i64 %iv
+  %l = load double, ptr %gep.A, align 8
+  %div = fmul double %l, 5.000000e+00
+  %gep.B.1 = getelementptr {double, double}, ptr %B, i64 %iv, i32 1
+  store double %div, ptr %gep.B.1, align 8
+  %gep.C.1 = getelementptr {double, double}, ptr %C, i64 %iv, i32 1
+  store double %div, ptr %gep.C.1, align 8
+  %gep.B = getelementptr {double, double}, ptr %B, i64 %iv
+  store double %div, ptr %gep.B, align 8
+  %gep.C = getelementptr {double, double}, ptr %C, i64 %iv
+  store double %div, ptr %gep.C, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-cpu"="neoverse-v2" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
new file mode 100644
index 000000000000..5efd821ba990
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-derived-ivs.ll
@@ -0,0 +1,303 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF4 %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64-apple-macosx15.0.0"
+
+define void @derived_int_ivs(ptr noalias %a, ptr noalias %b, i64 %end) {
+; VF2-LABEL: define void @derived_int_ivs(
+; VF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[END:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[END]], -32
+; VF2-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; VF2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 2
+; VF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 2
+; VF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VF2-NEXT:    [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP4:%.*]] = add i64 16, [[TMP3]]
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP5:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 16, [[TMP5]]
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x double> [[STRIDED_VEC]], <2 x double> [[STRIDED_VEC1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @derived_int_ivs(
+; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[END:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[END]], -32
+; VF4-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; VF4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4
+; VF4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4
+; VF4-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; VF4-NEXT:    [[TMP3:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP4:%.*]] = add i64 16, [[TMP3]]
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP5:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 16, [[TMP5]]
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP8:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x double> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 16, %entry ], [ %iv.next, %loop ]
+  %gep.a.0 = getelementptr inbounds i8, ptr %a, i64 %iv
+  %l.0 = load double, ptr %gep.a.0, align 8
+  %gep.b.0 = getelementptr inbounds i8, ptr %b, i64 %iv
+  store double %l.0, ptr %gep.b.0, align 8
+  %gep.a.1 = getelementptr inbounds i8, ptr %gep.a.0, i64 8
+  %l.1 = load double, ptr %gep.a.1, align 8
+  %gep.b.1 = getelementptr inbounds i8, ptr %gep.b.0, i64 8
+  store double %l.1, ptr %gep.b.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 16
+  %ec = icmp eq i64 %iv.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @derived_pointer_ivs(ptr noalias %a, ptr noalias %b, ptr %end) {
+; VF2-LABEL: define void @derived_pointer_ivs(
+; VF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr [[END:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    [[A5:%.*]] = ptrtoint ptr [[A]] to i64
+; VF2-NEXT:    [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; VF2-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; VF2-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[END4]], -16
+; VF2-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[A5]]
+; VF2-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 4
+; VF2-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; VF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; VF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; VF2:       [[VECTOR_MEMCHECK]]:
+; VF2-NEXT:    [[TMP4:%.*]] = add i64 [[END1]], -16
+; VF2-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], [[A2]]
+; VF2-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 4
+; VF2-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 4
+; VF2-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 16
+; VF2-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; VF2-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; VF2-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP3]]
+; VF2-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; VF2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; VF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; VF2-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
+; VF2-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
+; VF2-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX6]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x double>, ptr [[NEXT_GEP]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <4 x double> [[WIDE_VEC]], <4 x double> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP13:%.*]] = shufflevector <2 x double> [[STRIDED_VEC]], <2 x double> [[STRIDED_VEC8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP7]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; VF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @derived_pointer_ivs(
+; VF4-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr [[END:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    [[A5:%.*]] = ptrtoint ptr [[A]] to i64
+; VF4-NEXT:    [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; VF4-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; VF4-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[END4]], -16
+; VF4-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[A5]]
+; VF4-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; VF4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; VF4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; VF4:       [[VECTOR_MEMCHECK]]:
+; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[END1]], -16
+; VF4-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], [[A2]]
+; VF4-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP5]], 4
+; VF4-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 4
+; VF4-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], 16
+; VF4-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP8]]
+; VF4-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP8]]
+; VF4-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[B]], [[SCEVGEP3]]
+; VF4-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP]]
+; VF4-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF4-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; VF4-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; VF4-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], 16
+; VF4-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP11]]
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[OFFSET_IDX6:%.*]] = mul i64 [[INDEX]], 16
+; VF4-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX6]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x double>, ptr [[NEXT_GEP]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <8 x double> [[WIDE_VEC]], <8 x double> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP13:%.*]] = shufflevector <4 x double> [[STRIDED_VEC]], <4 x double> [[STRIDED_VEC8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x double> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP7]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; VF4-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %ptr.iv.1 = phi ptr [ %a, %entry ], [ %ptr.iv.1.next, %loop ]
+  %ptr.iv.2 = phi ptr [ %b, %entry ], [ %ptr.iv.2.next, %loop ]
+  %l.0 = load double, ptr %ptr.iv.1, align 8
+  store double %l.0, ptr %ptr.iv.2, align 8
+  %gep.a.1 = getelementptr inbounds i8, ptr %ptr.iv.1, i64 8
+  %l.1 = load double, ptr %gep.a.1, align 8
+  %gep.b.1 = getelementptr inbounds i8, ptr %ptr.iv.2, i64 8
+  store double %l.1, ptr %gep.b.1, align 8
+  %ptr.iv.1.next = getelementptr inbounds i8, ptr %ptr.iv.1, i64 16
+  %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 16
+  %ec = icmp eq ptr %ptr.iv.1.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @narrow_with_uniform_add_and_gep(ptr noalias %p) {
+; VF2-LABEL: define void @narrow_with_uniform_add_and_gep(
+; VF2-SAME: ptr noalias [[P:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 0
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP0]]
+; VF2-NEXT:    [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[STRIDED_VEC]], splat (i64 1)
+; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[STRIDED_VEC1]], splat (i64 1)
+; VF2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
+; VF2-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @narrow_with_uniform_add_and_gep(
+; VF4-SAME: ptr noalias [[P:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; VF4-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[OFFSET_IDX]], 0
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP0]]
+; VF4-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[STRIDED_VEC]], splat (i64 1)
+; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[STRIDED_VEC1]], splat (i64 1)
+; VF4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512
+; VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [0, %entry], [%iv.next, %loop]
+
+  %iv0 = add nuw nsw i64 %iv, 0
+  %gep0 = getelementptr i64, ptr %p, i64 %iv0
+  %load0 = load i64, ptr %gep0
+  %add0 = add i64 %load0, 1
+  store i64 %add0, ptr %gep0
+
+  %iv1 = add nuw nsw i64 %iv, 1
+  %gep1 = getelementptr i64, ptr %p, i64 %iv1
+  %load1 = load i64, ptr %gep1
+  %add1 = add i64 %load1, 1
+  store i64 %add1, ptr %gep1
+
+  %iv.next = add nuw nsw i64 %iv, 2
+  %done = icmp eq i64 %iv.next, 1024
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
index 00054a8ccdea..8cd850b3ea98 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory.ll
@@ -204,3 +204,388 @@ loop:
 exit:
   ret void
 }
+
+define void @same_constant_store_interleave_group(i64 %x, ptr noalias %dst) {
+; VF2-LABEL: define void @same_constant_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    store <2 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @same_constant_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    store <8 x i64> zeroinitializer, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 0, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @different_constants_store_interleave_group(i64 %x, i64 %y, ptr noalias %dst) {
+; VF2-LABEL: define void @different_constants_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    store <4 x i64> <i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @different_constants_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    store <8 x i64> <i64 0, i64 1, i64 0, i64 1, i64 0, i64 1, i64 0, i64 1>, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 1, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @same_live_in_store_interleave_group(i64 %x, ptr noalias %dst) {
+; VF2-LABEL: define void @same_live_in_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @same_live_in_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %x, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %x, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @different_live_ins_store_interleave_group(i64 %x, i64 %y, ptr noalias %dst) {
+; VF2-LABEL: define void @different_live_ins_store_interleave_group(
+; VF2-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[X]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[Y]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @different_live_ins_store_interleave_group(
+; VF4-SAME: i64 [[X:%.*]], i64 [[Y:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[Y]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %x, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %y, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @single_uniform_load_store_interleave_group(ptr noalias %src, ptr noalias %dst) {
+; VF2-LABEL: define void @single_uniform_load_store_interleave_group(
+; VF2-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP3]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @single_uniform_load_store_interleave_group(
+; VF4-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP3]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %l.0 = load i64, ptr %src, align 8
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %l.0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  store i64 %l.0, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @multiple_uniform_load_store_interleave_group(ptr noalias %src.0, ptr noalias %src.1, ptr noalias %dst) {
+; VF2-LABEL: define void @multiple_uniform_load_store_interleave_group(
+; VF2-SAME: ptr noalias [[SRC_0:%.*]], ptr noalias [[SRC_1:%.*]], ptr noalias [[DST:%.*]]) {
+; VF2-NEXT:  [[ENTRY:.*:]]
+; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF2:       [[VECTOR_PH]]:
+; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF2:       [[VECTOR_BODY]]:
+; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF2-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC_0]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP1]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
+; VF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; VF2-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> [[BROADCAST_SPLAT2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; VF2-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; VF2-NEXT:    store <4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF2-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF2:       [[MIDDLE_BLOCK]]:
+; VF2-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF2:       [[SCALAR_PH]]:
+;
+; VF4-LABEL: define void @multiple_uniform_load_store_interleave_group(
+; VF4-SAME: ptr noalias [[SRC_0:%.*]], ptr noalias [[SRC_1:%.*]], ptr noalias [[DST:%.*]]) {
+; VF4-NEXT:  [[ENTRY:.*:]]
+; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4:       [[VECTOR_PH]]:
+; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4:       [[VECTOR_BODY]]:
+; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; VF4-NEXT:    [[TMP1:%.*]] = load i64, ptr [[SRC_0]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP3:%.*]] = load i64, ptr [[SRC_1]], align 8
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
+; VF4-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; VF4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLAT]], <4 x i64> [[BROADCAST_SPLAT2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VF4-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; VF4-NEXT:    store <8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP2]], align 8
+; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF4:       [[MIDDLE_BLOCK]]:
+; VF4-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; VF4:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %l.0 = load i64, ptr %src.0, align 8
+  %dst.0 = getelementptr inbounds i64, ptr %dst, i64 %mul.2
+  store i64 %l.0, ptr %dst.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %dst.1 = getelementptr inbounds i64, ptr %dst, i64 %add.1
+  %l.1 = load i64, ptr %src.1, align 8
+  store i64 %l.1, ptr %dst.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
index 307ab3a87e9d..399d676c4fee 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
@@ -43,13 +43,12 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<[[ACC]]>, ir<[[REDUCE]]>
-; CHECK-NEXT:   EMIT vp<[[EXTRACT:%.+]]> = extract-last-element vp<[[RED_RESULT]]>
 ; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, vp<[[VEC_TC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[EXTRACT]]> from middle.block)
+; CHECK-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll b/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
index 3d925b57b408..dbc6121c8b76 100644
--- a/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/AMDGPU/assert-vplan-cost-model.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=loop-vectorize -amdgpu-coerce-illegal-types=1 < %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -passes=loop-vectorize < %s -S -o - | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
index f179a3ae04d2..ddd334d2982f 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions.ll
@@ -1525,8 +1525,8 @@ define i64 @mla_and_add_together_16_64(ptr nocapture noundef readonly %x, i32 no
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[TMP0]], align 2
-; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP1]], [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <8 x i64> [[TMP4]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])
 ; CHECK-NEXT:    [[TMP5]] = add i64 [[TMP3]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
index e40f51fd7bd7..25f52b2a99dd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
@@ -8,14 +8,51 @@ define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-LABEL: define void @test_wide_integer_induction(
 ; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VEC_IND]], ptr align 8 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[TMP8]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[IV]], ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT:    store i64 [[IV1]], ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
@@ -68,3 +105,10 @@ for.body:
 for.cond.cleanup:
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index c063e6ff3f48..d485a7432423 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -367,7 +367,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index c0548dd8ca54..45357dd6bf0d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -14,42 +14,42 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
-; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP9]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP15]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP14]], <vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP18]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP8]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP9]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP17]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP16]], <vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -107,36 +107,53 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_EVL-LABEL: define void @masked_strided_factor2
 ; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] {
 ; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_EVL:       vector.ph:
 ; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
-; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; PREDICATED_EVL:       for.body:
-; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
-; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; PREDICATED_EVL:       if.then:
-; PREDICATED_EVL-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[MUL]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
-; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; PREDICATED_EVL-NEXT:    [[ADD:%.*]] = or disjoint i32 [[MUL]], 1
-; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[ADD]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
-; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
-; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[MUL]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP4]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I]], ptr [[ARRAYIDX6]], align 1
-; PREDICATED_EVL-NEXT:    [[SUB:%.*]] = sub i8 0, [[SPEC_SELECT_I]]
-; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[ADD]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP5]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB]], ptr [[ARRAYIDX11]], align 1
-; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
-; PREDICATED_EVL:       for.inc:
-; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
-; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; PREDICATED_EVL:       for.end:
-; PREDICATED_EVL-NEXT:    ret void
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; PREDICATED_EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_EVL:       vector.body:
+; PREDICATED_EVL-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[AVL:%.*]] = sub i32 1024, [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP8]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP9]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP12]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; PREDICATED_EVL-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x ptr> align 1 [[TMP15]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
+; PREDICATED_EVL-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP16]], <vscale x 16 x ptr> align 1 [[TMP18]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_EVL-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PREDICATED_EVL:       middle.block:
+; PREDICATED_EVL-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_EVL:       scalar.ph:
 ;
 entry:
   %conv = zext i8 %guard to i32
@@ -182,58 +199,58 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
-; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 2)
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 3)
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP12]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP14]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP16]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP17]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP18]]
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP19]], i32 1, <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP21:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP20]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP22:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP23:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP22]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP24:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP24]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP25]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP26:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP26]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP27]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP28:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP28]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP29]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP30:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_EPILOGUE-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP30]]
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP23]], <vscale x 16 x ptr> [[TMP31]], i32 1, <vscale x 16 x i1> [[TMP7]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 2)
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 3)
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP13]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP14]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP15]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP16]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP17]]
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> [[TMP18]], i32 1, <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP22:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP21]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP23:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP23]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP19]], <vscale x 16 x ptr> [[TMP24]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP25:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP25]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> [[TMP26]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP27:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP27]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> [[TMP28]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP29:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; SCALAR_EPILOGUE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP29]]
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> [[TMP30]], i32 1, <vscale x 16 x i1> [[TMP6]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -307,52 +324,69 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_EVL-LABEL: define void @masked_strided_factor4
 ; PREDICATED_EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] {
 ; PREDICATED_EVL-NEXT:  entry:
+; PREDICATED_EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; PREDICATED_EVL:       vector.ph:
 ; PREDICATED_EVL-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
-; PREDICATED_EVL-NEXT:    br label [[FOR_BODY:%.*]]
-; PREDICATED_EVL:       for.body:
-; PREDICATED_EVL-NEXT:    [[IX_024:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ]
-; PREDICATED_EVL-NEXT:    [[CMP1:%.*]] = icmp samesign ugt i32 [[IX_024]], [[CONV]]
-; PREDICATED_EVL-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
-; PREDICATED_EVL:       if.then:
-; PREDICATED_EVL-NEXT:    [[IDX0:%.*]] = shl nuw nsw i32 [[IX_024]], 2
-; PREDICATED_EVL-NEXT:    [[IDX1:%.*]] = or disjoint i32 [[IDX0]], 1
-; PREDICATED_EVL-NEXT:    [[IDX2:%.*]] = or disjoint i32 [[IDX0]], 2
-; PREDICATED_EVL-NEXT:    [[IDX3:%.*]] = or disjoint i32 [[IDX0]], 3
-; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = zext nneg i32 [[IDX0]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
-; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAY1IDX0]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = zext nneg i32 [[IDX1]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
-; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = load i8, ptr [[ARRAY1IDX1]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = zext nneg i32 [[IDX2]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP4]]
-; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = load i8, ptr [[ARRAY1IDX2]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = zext nneg i32 [[IDX3]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY1IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP6]]
-; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAY1IDX3]], align 1
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I1:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP1]], i8 [[TMP3]])
-; PREDICATED_EVL-NEXT:    [[SUB1:%.*]] = sub i8 0, [[SPEC_SELECT_I1]]
-; PREDICATED_EVL-NEXT:    [[SPEC_SELECT_I2:%.*]] = call i8 @llvm.smax.i8(i8 [[TMP5]], i8 [[TMP7]])
-; PREDICATED_EVL-NEXT:    [[SUB2:%.*]] = sub i8 0, [[SPEC_SELECT_I2]]
-; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = zext nneg i32 [[IDX0]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX0:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP8]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I1]], ptr [[ARRAY3IDX0]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = zext nneg i32 [[IDX1]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP9]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB1]], ptr [[ARRAY3IDX1]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[IDX2]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP10]]
-; PREDICATED_EVL-NEXT:    store i8 [[SPEC_SELECT_I2]], ptr [[ARRAY3IDX2]], align 1
-; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg i32 [[IDX3]] to i64
-; PREDICATED_EVL-NEXT:    [[ARRAY3IDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[Q]], i64 [[TMP11]]
-; PREDICATED_EVL-NEXT:    store i8 [[SUB2]], ptr [[ARRAY3IDX3]], align 1
-; PREDICATED_EVL-NEXT:    br label [[FOR_INC]]
-; PREDICATED_EVL:       for.inc:
-; PREDICATED_EVL-NEXT:    [[INC]] = add nuw nsw i32 [[IX_024]], 1
-; PREDICATED_EVL-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1024
-; PREDICATED_EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; PREDICATED_EVL:       for.end:
-; PREDICATED_EVL-NEXT:    ret void
+; PREDICATED_EVL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
+; PREDICATED_EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
+; PREDICATED_EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
+; PREDICATED_EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
+; PREDICATED_EVL-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; PREDICATED_EVL-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; PREDICATED_EVL:       vector.body:
+; PREDICATED_EVL-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_EVL-NEXT:    [[AVL:%.*]] = sub i32 1024, [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; PREDICATED_EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; PREDICATED_EVL-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_EVL-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 2)
+; PREDICATED_EVL-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
+; PREDICATED_EVL-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 2)
+; PREDICATED_EVL-NEXT:    [[TMP10:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 3)
+; PREDICATED_EVL-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP12]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP13]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP14]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP15:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP15]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP16]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP17:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], <vscale x 16 x i64> [[TMP17]]
+; PREDICATED_EVL-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 16 x i8> @llvm.vp.gather.nxv16i8.nxv16p0(<vscale x 16 x ptr> align 1 [[TMP18]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER3]])
+; PREDICATED_EVL-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; PREDICATED_EVL-NEXT:    [[TMP21:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER4]], <vscale x 16 x i8> [[WIDE_MASKED_GATHER5]])
+; PREDICATED_EVL-NEXT:    [[TMP22:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP21]]
+; PREDICATED_EVL-NEXT:    [[TMP23:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP23]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP19]], <vscale x 16 x ptr> align 1 [[TMP24]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP25:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP25]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP20]], <vscale x 16 x ptr> align 1 [[TMP26]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP27:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP27]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP21]], <vscale x 16 x ptr> align 1 [[TMP28]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[TMP29:%.*]] = zext nneg <vscale x 16 x i32> [[TMP10]] to <vscale x 16 x i64>
+; PREDICATED_EVL-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP29]]
+; PREDICATED_EVL-NEXT:    call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> [[TMP22]], <vscale x 16 x ptr> align 1 [[TMP30]], <vscale x 16 x i1> [[TMP6]], i32 [[TMP5]])
+; PREDICATED_EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]]
+; PREDICATED_EVL-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_EVL-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_EVL-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; PREDICATED_EVL:       middle.block:
+; PREDICATED_EVL-NEXT:    br label [[FOR_END:%.*]]
+; PREDICATED_EVL:       scalar.ph:
 ;
 entry:
   %conv = zext i8 %guard to i32
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
index f404b8949288..4f3b8d84310b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
@@ -4,11 +4,11 @@
 
 ; REQUIRES: asserts
 
-; Make sure we do not vectorize a loop with a widened int induction.
-define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
+; Make sure we do not vectorize a loop with a widened pointer induction.
+define void @test_wide_pointer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-NOT: LV: Vector loop of width {{.+}} costs:
 ;
-; CHECK: define void @test_wide_integer_induction(
+; CHECK: define void @test_wide_pointer_induction(
 ; CHECK-NOT: vector.body
 ;
 entry:
@@ -16,9 +16,11 @@ entry:
 
 loop:
   %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.ptr = phi ptr [ %a, %entry ], [ %iv.ptr.next, %loop ]
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  store i64 %iv, ptr %arrayidx, align 8
+  store ptr %iv.ptr, ptr %arrayidx, align 8
   %iv.next = add nuw nsw i64 %iv, 1
+  %iv.ptr.next = getelementptr i64, ptr %iv.ptr, i32 1
   %exitcond.not = icmp eq i64 %iv.next, %N
   br i1 %exitcond.not, label %exit, label %loop
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
index 93bd44f5c622..e4892db592d6 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
@@ -9,7 +9,7 @@
 define void @vp_select(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; DEBUGLOC-LABEL: define void @vp_select(
 ; DEBUGLOC: vector.body:
-; DEBUGLOC:   = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> %{{.+}}, <vscale x 4 x i32> %{{.+}}, <vscale x 4 x i32> %{{.+}}, i32 %{{.+}}), !dbg ![[SELLOC:[0-9]+]]
+; DEBUGLOC:   = select <vscale x 4 x i1> %{{.+}}, <vscale x 4 x i32> %{{.+}}, <vscale x 4 x i32> %{{.+}}, !dbg ![[SELLOC:[0-9]+]]
 ; DEBUGLOC: loop:
 ; DEBUGLOC:   = select i1 %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, !dbg ![[SELLOC]]
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
index 09b274de3021..29b27cdb7556 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
@@ -40,7 +40,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP8:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP22:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP22]]
 ; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP10]]
 ; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 [[TMP11]]
 ; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP13]], align 4
@@ -48,7 +49,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; RV64-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP16]]
 ; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 [[TMP17]]
 ; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])
@@ -98,7 +100,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP11:%.*]] = mul i32 0, [[TMP10]]
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 1, [[TMP10]]
+; RV32-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP10]], 1
+; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP24]]
 ; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 [[TMP11]]
 ; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 [[TMP12]]
 ; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -107,7 +110,8 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 1, [[TMP17]]
+; RV32-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP25]]
 ; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 [[TMP18]]
 ; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP20]], i32 [[TMP19]]
 ; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])
@@ -157,11 +161,13 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP9:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP11:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP12:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP12:%.*]] = mul i64 -1, [[TMP32]]
 ; RV64-UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP11]]
 ; RV64-UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP12]]
 ; RV64-UF2-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul i64 -1, [[TMP33]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP15]]
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP16]]
 ; RV64-UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -172,11 +178,13 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = add <vscale x 4 x i32> [[REVERSE2]], splat (i32 1)
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP34]]
 ; RV64-UF2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP22]]
 ; RV64-UF2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP24]], i64 [[TMP23]]
 ; RV64-UF2-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP27:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP35:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP27:%.*]] = mul i64 -1, [[TMP35]]
 ; RV64-UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP26]]
 ; RV64-UF2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP28]], i64 [[TMP27]]
 ; RV64-UF2-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP19]])
@@ -246,7 +254,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP8:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP22:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP22]]
 ; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[TMP10]]
 ; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP11]]
 ; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
@@ -254,7 +263,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    [[TMP14:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
 ; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP5]]
+; RV64-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; RV64-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[TMP16]]
 ; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[TMP17]]
 ; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP14]])
@@ -304,7 +314,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP11:%.*]] = mul i32 0, [[TMP10]]
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 1, [[TMP10]]
+; RV32-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP10]], 1
+; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP24]]
 ; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i32 [[TMP11]]
 ; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i32 [[TMP12]]
 ; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
@@ -313,7 +324,8 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]]
 ; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 1, [[TMP17]]
+; RV32-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP25]]
 ; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 [[TMP18]]
 ; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i32 [[TMP19]]
 ; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP15]])
@@ -363,11 +375,13 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP9:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP11:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP12:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP12:%.*]] = mul i64 -1, [[TMP32]]
 ; RV64-UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP11]]
 ; RV64-UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP12]]
 ; RV64-UF2-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul i64 -1, [[TMP33]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP15]]
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP16]]
 ; RV64-UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
@@ -378,11 +392,13 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = fadd <vscale x 4 x float> [[REVERSE2]], splat (float 1.000000e+00)
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]]
 ; RV64-UF2-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP34]]
 ; RV64-UF2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[TMP22]]
 ; RV64-UF2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[TMP23]]
 ; RV64-UF2-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP5]]
-; RV64-UF2-NEXT:    [[TMP27:%.*]] = sub i64 1, [[TMP5]]
+; RV64-UF2-NEXT:    [[TMP35:%.*]] = sub i64 [[TMP5]], 1
+; RV64-UF2-NEXT:    [[TMP27:%.*]] = mul i64 -1, [[TMP35]]
 ; RV64-UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[TMP26]]
 ; RV64-UF2-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP27]]
 ; RV64-UF2-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP19]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index b23b0ce759d4..b4e49a60e088 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -274,7 +274,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body.preheader> in BB:for.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.body.preheader> in BB: for.body.preheader
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.body.preheader: ; preds = %entry
 ; CHECK-NEXT:    %0 = zext i32 %n to i64
@@ -282,7 +282,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %2 = mul nuw i64 %1, 4
 ; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
 ; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.scevcheck> in BB: vector.scevcheck
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.scevcheck: ; preds = %for.body.preheader
 ; CHECK-NEXT:    %3 = add nsw i64 %0, -1
@@ -297,8 +297,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %9 = icmp ugt i64 %3, 4294967295
 ; CHECK-NEXT:    %10 = or i1 %8, %9
 ; CHECK-NEXT:    br i1 %10, label %scalar.ph, label %vector.memcheck
-; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.memcheck> in BB:vector.memcheck
+; CHECK-NEXT:  LV: draw edge from for.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.memcheck> in BB: vector.memcheck
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
 ; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
@@ -307,8 +307,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
 ; CHECK-NEXT:    br i1 %diff.check, label %scalar.ph, label %vector.ph
-; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.ph> in BB:vector.ph
+; CHECK-NEXT:  LV: draw edge from vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.ph> in BB: vector.ph
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
 ; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
@@ -321,10 +321,10 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
 ; CHECK-NEXT:    %20 = sub i32 %n, %.cast
 ; CHECK-NEXT:    br
-; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: draw edge from vector.memcheck
 ; CHECK-NEXT:  LV: created vector.body
-; CHECK-NEXT:  LV: draw edge fromvector.ph
-; CHECK-NEXT:  LV: vectorizing VPBB:vector.body in BB:vector.body
+; CHECK-NEXT:  LV: draw edge from vector.ph
+; CHECK-NEXT:  LV: vectorizing VPBB: vector.body in BB: vector.body
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.body: ; preds = %vector.body, %vector.ph
 ; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ]
@@ -334,45 +334,47 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %22 = zext i32 %21 to i64
 ; CHECK-NEXT:    %23 = getelementptr inbounds i32, ptr %B, i64 %22
 ; CHECK-NEXT:    %24 = mul i64 0, %18
-; CHECK-NEXT:    %25 = sub i64 1, %18
-; CHECK-NEXT:    %26 = getelementptr inbounds i32, ptr %23, i64 %24
-; CHECK-NEXT:    %27 = getelementptr inbounds i32, ptr %26, i64 %25
-; CHECK-NEXT:    %wide.load = load <vscale x 4 x i32>, ptr %27, align 4
+; CHECK-NEXT:    %25 = sub i64 %18, 1
+; CHECK-NEXT:    %26 = mul i64 -1, %25
+; CHECK-NEXT:    %27 = getelementptr inbounds i32, ptr %23, i64 %24
+; CHECK-NEXT:    %28 = getelementptr inbounds i32, ptr %27, i64 %26
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x i32>, ptr %28, align 4
 ; CHECK-NEXT:    %reverse = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %wide.load)
-; CHECK-NEXT:    %28 = add <vscale x 4 x i32> %reverse, splat (i32 1)
-; CHECK-NEXT:    %29 = getelementptr inbounds i32, ptr %A, i64 %22
-; CHECK-NEXT:    %30 = mul i64 0, %18
-; CHECK-NEXT:    %31 = sub i64 1, %18
-; CHECK-NEXT:    %32 = getelementptr inbounds i32, ptr %29, i64 %30
-; CHECK-NEXT:    %33 = getelementptr inbounds i32, ptr %32, i64 %31
-; CHECK-NEXT:    %reverse4 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %28)
-; CHECK-NEXT:    store <vscale x 4 x i32> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %29 = add <vscale x 4 x i32> %reverse, splat (i32 1)
+; CHECK-NEXT:    %30 = getelementptr inbounds i32, ptr %A, i64 %22
+; CHECK-NEXT:    %31 = mul i64 0, %18
+; CHECK-NEXT:    %32 = sub i64 %18, 1
+; CHECK-NEXT:    %33 = mul i64 -1, %32
+; CHECK-NEXT:    %34 = getelementptr inbounds i32, ptr %30, i64 %31
+; CHECK-NEXT:    %35 = getelementptr inbounds i32, ptr %34, i64 %33
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %29)
+; CHECK-NEXT:    store <vscale x 4 x i32> %reverse4, ptr %35, align 4
 ; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
-; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
-; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:    %36 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %36, <null operand!>, label %vector.body
 ; CHECK-NEXT:  LV: created middle.block
-; CHECK-NEXT:  LV: draw edge fromvector.body
-; CHECK-NEXT:  LV: vectorizing VPBB:middle.block in BB:middle.block
+; CHECK-NEXT:  LV: draw edge from vector.body
+; CHECK-NEXT:  LV: vectorizing VPBB: middle.block in BB: middle.block
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  middle.block: ; preds = %vector.body
 ; CHECK-NEXT:    %cmp.n = icmp eq i64 %0, %n.vec
 ; CHECK-NEXT:    br i1 %cmp.n, <null operand!>, <null operand!>
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.cond.cleanup.loopexit> in BB:for.cond.cleanup.loopexit
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.cond.cleanup.loopexit> in BB: for.cond.cleanup.loopexit
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.cond.cleanup.loopexit: ; preds = %for.body
 ; CHECK-NEXT:    br label %for.cond.cleanup
-; CHECK-NEXT:  LV: draw edge frommiddle.block
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<scalar.ph> in BB:scalar.ph
+; CHECK-NEXT:  LV: draw edge from middle.block
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<scalar.ph> in BB: scalar.ph
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
 ; CHECK-NEXT:    %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
 ; CHECK-NEXT:    %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
 ; CHECK-NEXT:    br label %for.body
-; CHECK-NEXT:  LV: draw edge frommiddle.block
-; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
-; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
-; CHECK-NEXT:  LV: draw edge fromvector.memcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body> in BB:for.body
+; CHECK-NEXT:  LV: draw edge from middle.block
+; CHECK-NEXT:  LV: draw edge from for.body.preheader
+; CHECK-NEXT:  LV: draw edge from vector.scevcheck
+; CHECK-NEXT:  LV: draw edge from vector.memcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.body> in BB: for.body
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.body: ; preds = %for.body, %scalar.ph
 ; CHECK-NEXT:    %indvars.iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %indvars.iv.next, %for.body ]
@@ -380,14 +382,14 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
-; CHECK-NEXT:    %35 = load i32, ptr %arrayidx, align 4
-; CHECK-NEXT:    %add9 = add i32 %35, 1
+; CHECK-NEXT:    %37 = load i32, ptr %arrayidx, align 4
+; CHECK-NEXT:    %add9 = add i32 %37, 1
 ; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds i32, ptr %A, i64 %idxprom
 ; CHECK-NEXT:    store i32 %add9, ptr %arrayidx3, align 4
 ; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:    %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:    br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
-; CHECK-NEXT:  LV: draw edge fromscalar.ph
+; CHECK-NEXT:  LV: draw edge from scalar.ph
 ; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
 ; CHECK-NEXT:  LV: Vectorizing: innermost loop.
 ; CHECK-EMPTY:
@@ -683,7 +685,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:  No successors
 ; CHECK-NEXT:  }
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body.preheader> in BB:for.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.body.preheader> in BB: for.body.preheader
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.body.preheader: ; preds = %entry
 ; CHECK-NEXT:    %0 = zext i32 %n to i64
@@ -691,7 +693,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %2 = mul nuw i64 %1, 4
 ; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
 ; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.scevcheck> in BB: vector.scevcheck
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.scevcheck: ; preds = %for.body.preheader
 ; CHECK-NEXT:    %3 = add nsw i64 %0, -1
@@ -706,8 +708,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %9 = icmp ugt i64 %3, 4294967295
 ; CHECK-NEXT:    %10 = or i1 %8, %9
 ; CHECK-NEXT:    br i1 %10, label %scalar.ph, label %vector.memcheck
-; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.memcheck> in BB:vector.memcheck
+; CHECK-NEXT:  LV: draw edge from for.body.preheader
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.memcheck> in BB: vector.memcheck
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
 ; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
@@ -716,8 +718,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
 ; CHECK-NEXT:    br i1 %diff.check, label %scalar.ph, label %vector.ph
-; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.ph> in BB:vector.ph
+; CHECK-NEXT:  LV: draw edge from vector.scevcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<vector.ph> in BB: vector.ph
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
 ; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
@@ -730,10 +732,10 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
 ; CHECK-NEXT:    %20 = sub i32 %n, %.cast
 ; CHECK-NEXT:    br
-; CHECK-NEXT:  LV: draw edge fromvector.memcheck
+; CHECK-NEXT:  LV: draw edge from vector.memcheck
 ; CHECK-NEXT:  LV: created vector.body
-; CHECK-NEXT:  LV: draw edge fromvector.ph
-; CHECK-NEXT:  LV: vectorizing VPBB:vector.body in BB:vector.body
+; CHECK-NEXT:  LV: draw edge from vector.ph
+; CHECK-NEXT:  LV: vectorizing VPBB: vector.body in BB: vector.body
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.body: ; preds = %vector.body, %vector.ph
 ; CHECK-NEXT:    %index = phi i64 [ 0, %vector.ph ]
@@ -743,45 +745,47 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %22 = zext i32 %21 to i64
 ; CHECK-NEXT:    %23 = getelementptr inbounds float, ptr %B, i64 %22
 ; CHECK-NEXT:    %24 = mul i64 0, %18
-; CHECK-NEXT:    %25 = sub i64 1, %18
-; CHECK-NEXT:    %26 = getelementptr inbounds float, ptr %23, i64 %24
-; CHECK-NEXT:    %27 = getelementptr inbounds float, ptr %26, i64 %25
-; CHECK-NEXT:    %wide.load = load <vscale x 4 x float>, ptr %27, align 4
+; CHECK-NEXT:    %25 = sub i64 %18, 1
+; CHECK-NEXT:    %26 = mul i64 -1, %25
+; CHECK-NEXT:    %27 = getelementptr inbounds float, ptr %23, i64 %24
+; CHECK-NEXT:    %28 = getelementptr inbounds float, ptr %27, i64 %26
+; CHECK-NEXT:    %wide.load = load <vscale x 4 x float>, ptr %28, align 4
 ; CHECK-NEXT:    %reverse = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %wide.load)
-; CHECK-NEXT:    %28 = fadd <vscale x 4 x float> %reverse, splat (float 1.000000e+00)
-; CHECK-NEXT:    %29 = getelementptr inbounds float, ptr %A, i64 %22
-; CHECK-NEXT:    %30 = mul i64 0, %18
-; CHECK-NEXT:    %31 = sub i64 1, %18
-; CHECK-NEXT:    %32 = getelementptr inbounds float, ptr %29, i64 %30
-; CHECK-NEXT:    %33 = getelementptr inbounds float, ptr %32, i64 %31
-; CHECK-NEXT:    %reverse4 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %28)
-; CHECK-NEXT:    store <vscale x 4 x float> %reverse4, ptr %33, align 4
+; CHECK-NEXT:    %29 = fadd <vscale x 4 x float> %reverse, splat (float 1.000000e+00)
+; CHECK-NEXT:    %30 = getelementptr inbounds float, ptr %A, i64 %22
+; CHECK-NEXT:    %31 = mul i64 0, %18
+; CHECK-NEXT:    %32 = sub i64 %18, 1
+; CHECK-NEXT:    %33 = mul i64 -1, %32
+; CHECK-NEXT:    %34 = getelementptr inbounds float, ptr %30, i64 %31
+; CHECK-NEXT:    %35 = getelementptr inbounds float, ptr %34, i64 %33
+; CHECK-NEXT:    %reverse4 = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %29)
+; CHECK-NEXT:    store <vscale x 4 x float> %reverse4, ptr %35, align 4
 ; CHECK-NEXT:    %index.next = add nuw i64 %index, %18
-; CHECK-NEXT:    %34 = icmp eq i64 %index.next, %n.vec
-; CHECK-NEXT:    br i1 %34, <null operand!>, label %vector.body
+; CHECK-NEXT:    %36 = icmp eq i64 %index.next, %n.vec
+; CHECK-NEXT:    br i1 %36, <null operand!>, label %vector.body
 ; CHECK-NEXT:  LV: created middle.block
-; CHECK-NEXT:  LV: draw edge fromvector.body
-; CHECK-NEXT:  LV: vectorizing VPBB:middle.block in BB:middle.block
+; CHECK-NEXT:  LV: draw edge from vector.body
+; CHECK-NEXT:  LV: vectorizing VPBB: middle.block in BB: middle.block
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  middle.block: ; preds = %vector.body
 ; CHECK-NEXT:    %cmp.n = icmp eq i64 %0, %n.vec
 ; CHECK-NEXT:    br i1 %cmp.n, <null operand!>, <null operand!>
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.cond.cleanup.loopexit> in BB:for.cond.cleanup.loopexit
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.cond.cleanup.loopexit> in BB: for.cond.cleanup.loopexit
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.cond.cleanup.loopexit: ; preds = %for.body
 ; CHECK-NEXT:    br label %for.cond.cleanup
-; CHECK-NEXT:  LV: draw edge frommiddle.block
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<scalar.ph> in BB:scalar.ph
+; CHECK-NEXT:  LV: draw edge from middle.block
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<scalar.ph> in BB: scalar.ph
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  scalar.ph: ; preds = %vector.memcheck, %vector.scevcheck, %for.body.preheader
 ; CHECK-NEXT:    %bc.resume.val = phi i64 [ %19, %middle.block ], [ %0, %for.body.preheader ], [ %0, %vector.scevcheck ], [ %0, %vector.memcheck ]
 ; CHECK-NEXT:    %bc.resume.val5 = phi i32 [ %20, %middle.block ], [ %n, %for.body.preheader ], [ %n, %vector.scevcheck ], [ %n, %vector.memcheck ]
 ; CHECK-NEXT:    br label %for.body
-; CHECK-NEXT:  LV: draw edge frommiddle.block
-; CHECK-NEXT:  LV: draw edge fromfor.body.preheader
-; CHECK-NEXT:  LV: draw edge fromvector.scevcheck
-; CHECK-NEXT:  LV: draw edge fromvector.memcheck
-; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<for.body> in BB:for.body
+; CHECK-NEXT:  LV: draw edge from middle.block
+; CHECK-NEXT:  LV: draw edge from for.body.preheader
+; CHECK-NEXT:  LV: draw edge from vector.scevcheck
+; CHECK-NEXT:  LV: draw edge from vector.memcheck
+; CHECK-NEXT:  LV: vectorizing VPBB: ir-bb<for.body> in BB: for.body
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  for.body: ; preds = %for.body, %scalar.ph
 ; CHECK-NEXT:    %indvars.iv = phi i64 [ %bc.resume.val, %scalar.ph ], [ %indvars.iv.next, %for.body ]
@@ -789,14 +793,14 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:    %i.0 = add nsw i32 %i.0.in8, -1
 ; CHECK-NEXT:    %idxprom = zext i32 %i.0 to i64
 ; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
-; CHECK-NEXT:    %35 = load float, ptr %arrayidx, align 4
-; CHECK-NEXT:    %conv1 = fadd float %35, 1.000000e+00
+; CHECK-NEXT:    %37 = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:    %conv1 = fadd float %37, 1.000000e+00
 ; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, ptr %A, i64 %idxprom
 ; CHECK-NEXT:    store float %conv1, ptr %arrayidx3, align 4
 ; CHECK-NEXT:    %cmp = icmp ugt i64 %indvars.iv, 1
 ; CHECK-NEXT:    %indvars.iv.next = add nsw i64 %indvars.iv, -1
 ; CHECK-NEXT:    br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
-; CHECK-NEXT:  LV: draw edge fromscalar.ph
+; CHECK-NEXT:  LV: draw edge from scalar.ph
 ; CHECK-NEXT:  LV: Interleaving disabled by the pass manager
 ; CHECK-NEXT:  LV: Vectorizing: innermost loop.
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index f059730a245b..1e1ed49f9f2e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -44,7 +44,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-NEXT:    [[TMP23:%.*]] = ashr <vscale x 8 x i32> [[TMP15]], zeroinitializer
 ; CHECK-NEXT:    [[VP_OP3:%.*]] = or <vscale x 8 x i32> [[TMP23]], zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ult <vscale x 8 x i32> [[TMP15]], zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x i32> @llvm.vp.select.nxv8i32(<vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[VP_OP3]], <vscale x 8 x i32> zeroinitializer, i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP17:%.*]] = select <vscale x 8 x i1> [[TMP16]], <vscale x 8 x i32> [[VP_OP3]], <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP24:%.*]] = trunc <vscale x 8 x i32> [[TMP17]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i8.nxv8p0(<vscale x 8 x i8> [[TMP24]], <vscale x 8 x ptr> align 1 [[BROADCAST_SPLAT]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = trunc <vscale x 8 x i32> [[VP_OP]] to <vscale x 8 x i16>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
index 38ed3f24ab23..091eb8720260 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -953,16 +953,55 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_ptrtoint(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], <vscale x 2 x i64> [[VEC_IND]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = ptrtoint <vscale x 2 x ptr> [[TMP14]] to <vscale x 2 x i64>
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br label %[[EXIT:.*]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64
 ; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    store i64 [[TMP0]], ptr [[GEP2]], align 8
 ; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP49:![0-9]+]]
 ; IF-EVL:       [[EXIT]]:
 ; IF-EVL-NEXT:    ret void
 ;
@@ -1047,4 +1086,6 @@ exit:
 ; IF-EVL: [[LOOP45]] = distinct !{[[LOOP45]], [[META6]]}
 ; IF-EVL: [[LOOP46]] = distinct !{[[LOOP46]], [[META6]], [[META7]], [[META8]]}
 ; IF-EVL: [[LOOP47]] = distinct !{[[LOOP47]], [[META6]]}
+; IF-EVL: [[LOOP48]] = distinct !{[[LOOP48]], [[META6]], [[META7]], [[META8]]}
+; IF-EVL: [[LOOP49]] = distinct !{[[LOOP49]], [[META8]], [[META6]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
index e5364136135d..2926011857ae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
@@ -47,7 +47,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-OUTLOOP-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP18]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP11]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP19:%.*]] = select <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP:%.*]] = add <vscale x 4 x i32> [[TMP19]], [[VEC_PHI]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP20]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[VP_OP]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP11]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP11]] to i64
@@ -101,7 +101,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], splat (i32 3)
-; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer, i32 [[TMP12]])
+; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
 ; IF-EVL-INLOOP-NEXT:    [[TMP22]] = add i32 [[TMP21]], [[VEC_PHI]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP12]] to i64
@@ -514,41 +514,127 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @step_cond_add(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP:       vector.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP10]], splat (i32 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP11]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL-OUTLOOP:       vector.body:
+; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT1:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul i32 1, [[TMP12]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP19]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP12]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP12]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT1]] = add i64 [[TMP20]], [[EVL_BASED_IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[EVL_BASED_IV]], [[TMP8]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N_VEC]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-OUTLOOP:       middle.block:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-OUTLOOP:       scalar.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP37:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[EVL_BASED_IV]] to i32
+; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP37]], [[IV_TRUNC]]
 ; IF-EVL-OUTLOOP-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[TMP37]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[ADD]] = add nsw i32 [[SELECT]], [[RDX]]
-; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw nsw i64 [[EVL_BASED_IV]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL-OUTLOOP:       for.end:
-; IF-EVL-OUTLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP22]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-OUTLOOP-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; IF-EVL-INLOOP-LABEL: define i32 @step_cond_add(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP:       vector.ph:
+; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP:       vector.body:
+; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT1:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[ADD:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-INLOOP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0
+; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP15:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-INLOOP-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
+; IF-EVL-INLOOP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[ADD]] = add i32 [[TMP17]], [[RDX]]
+; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT1]] = add i64 [[TMP19]], [[EVL_BASED_IV1]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[EVL_BASED_IV]], [[TMP8]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-INLOOP-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N_VEC]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-INLOOP:       middle.block:
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-INLOOP:       scalar.ph:
+; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-INLOOP:       for.body:
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX1:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[EVL_BASED_IV]] to i32
+; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP28]], [[IV_TRUNC]]
 ; IF-EVL-INLOOP-NEXT:    [[SELECT:%.*]] = select i1 [[CMP]], i32 [[TMP28]], i32 0
-; IF-EVL-INLOOP-NEXT:    [[ADD]] = add nsw i32 [[SELECT]], [[RDX]]
-; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add nuw nsw i64 [[EVL_BASED_IV]], 1
-; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL-INLOOP-NEXT:    [[ADD1]] = add nsw i32 [[SELECT]], [[RDX1]]
+; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; IF-EVL-INLOOP:       for.end:
-; IF-EVL-INLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD1]], [[FOR_BODY]] ], [ [[ADD]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-INLOOP-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 ; NO-VP-OUTLOOP-LABEL: define i32 @step_cond_add(
@@ -696,49 +782,148 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @step_cond_add_pred(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP:       vector.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-OUTLOOP-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i64> [[TMP10]], splat (i64 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP11]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
+; IF-EVL-OUTLOOP-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP13]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ], [ [[TMP37:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP:       vector.body:
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT7:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-OUTLOOP-NEXT:    [[TMP15:%.*]] = mul i32 1, [[TMP14]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP15]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[TMP17:%.*]] = mul i64 1, [[TMP16]]
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP38:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; IF-EVL-OUTLOOP-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i32 0
+; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp sle <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND2]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP23:%.*]] = select <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i1> [[TMP21]], <vscale x 4 x i1> zeroinitializer
+; IF-EVL-OUTLOOP-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP22]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP24]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
+; IF-EVL-OUTLOOP-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP14]] to i64
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP25]], [[IV]]
+; IF-EVL-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
+; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND_NEXT7]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT4]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK1:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-OUTLOOP:       middle.block:
+; IF-EVL-OUTLOOP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP24]])
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-OUTLOOP:       scalar.ph:
+; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-OUTLOOP:       for.body:
+; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP38:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP38]], [[IV_TRUNC]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[MIDDLE_BLOCK]]
 ; IF-EVL-OUTLOOP:       if.then:
 ; IF-EVL-OUTLOOP-NEXT:    [[ADD_PRED:%.*]] = add nsw i32 [[BC_MERGE_RDX]], [[TMP38]]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[MIDDLE_BLOCK]]
 ; IF-EVL-OUTLOOP:       for.inc:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP37]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL-OUTLOOP:       for.end:
-; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[TMP37]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[RDX_ADD]], [[MIDDLE_BLOCK]] ], [ [[TMP27]], [[MIDDLE_BLOCK1]] ]
 ; IF-EVL-OUTLOOP-NEXT:    ret i32 [[RDX_ADD_LCSSA]]
 ;
 ; IF-EVL-INLOOP-LABEL: define i32 @step_cond_add_pred(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP:       vector.ph:
+; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
+; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
-; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ], [ [[TMP32:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP:       vector.body:
+; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[IV]]
+; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; IF-EVL-INLOOP-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; IF-EVL-INLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[ARRAYIDX]], i32 0
+; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP15:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND]]
+; IF-EVL-INLOOP-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> [[TMP15]], i32 [[TMP11]])
+; IF-EVL-INLOOP-NEXT:    [[TMP17]] = add i32 [[TMP16]], [[VEC_PHI]]
+; IF-EVL-INLOOP-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-INLOOP-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP18]], [[IV]]
+; IF-EVL-INLOOP-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; IF-EVL-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-INLOOP-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK1:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL-INLOOP:       middle.block:
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL-INLOOP:       scalar.ph:
+; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
+; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL-INLOOP:       for.body:
+; IF-EVL-INLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; IF-EVL-INLOOP-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP35]], [[IV_TRUNC]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[MIDDLE_BLOCK]]
 ; IF-EVL-INLOOP:       if.then:
 ; IF-EVL-INLOOP-NEXT:    [[ADD_PRED:%.*]] = add nsw i32 [[BC_MERGE_RDX]], [[TMP35]]
 ; IF-EVL-INLOOP-NEXT:    br label [[MIDDLE_BLOCK]]
 ; IF-EVL-INLOOP:       for.inc:
-; IF-EVL-INLOOP-NEXT:    [[TMP32]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-INLOOP-NEXT:    [[RDX_ADD]] = phi i32 [ [[ADD_PRED]], [[IF_THEN]] ], [ [[BC_MERGE_RDX]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
 ; IF-EVL-INLOOP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7]]
+; IF-EVL-INLOOP-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; IF-EVL-INLOOP:       for.end:
-; IF-EVL-INLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[TMP32]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX_ADD_LCSSA:%.*]] = phi i32 [ [[RDX_ADD]], [[MIDDLE_BLOCK]] ], [ [[TMP17]], [[MIDDLE_BLOCK1]] ]
 ; IF-EVL-INLOOP-NEXT:    ret i32 [[RDX_ADD_LCSSA]]
 ;
 ; NO-VP-OUTLOOP-LABEL: define i32 @step_cond_add_pred(
@@ -906,8 +1091,10 @@ for.end:
 ; IF-EVL-OUTLOOP: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ; IF-EVL-OUTLOOP: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
 ; IF-EVL-OUTLOOP: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
-; IF-EVL-OUTLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; IF-EVL-OUTLOOP: [[META8]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL-OUTLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-OUTLOOP: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; IF-EVL-OUTLOOP: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-OUTLOOP: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
 ;.
 ; IF-EVL-INLOOP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; IF-EVL-INLOOP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -916,8 +1103,10 @@ for.end:
 ; IF-EVL-INLOOP: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ; IF-EVL-INLOOP: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
 ; IF-EVL-INLOOP: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
-; IF-EVL-INLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]]}
-; IF-EVL-INLOOP: [[META8]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL-INLOOP: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-INLOOP: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; IF-EVL-INLOOP: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL-INLOOP: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
 ;.
 ; NO-VP-OUTLOOP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; NO-VP-OUTLOOP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
index 172335d0e447..0490d63f67d4 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
@@ -592,6 +592,151 @@ for.end:
   ret i32 %for1
 }
 
+define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
+; IF-EVL-LABEL: define void @first_order_recurrence_indvar(
+; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL:       [[VECTOR_PH]]:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP2]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP18]], 2
+; IF-EVL-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
+; IF-EVL-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP12:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP12]]
+; IF-EVL-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; IF-EVL-NEXT:    [[TMP19:%.*]] = mul nuw i32 [[TMP13]], 2
+; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP19]], 1
+; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 33, i32 [[TMP10]]
+; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IF-EVL:       [[VECTOR_BODY]]:
+; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[PREV_EVL:%.*]] = phi i32 [ [[TMP5]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[TC]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP7]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP20]] = add <vscale x 2 x i64> [[VEC_IND]], splat (i64 42)
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP20]], i32 -1, <vscale x 2 x i1> splat (i1 true), i32 [[PREV_EVL]], i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP9]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr align 8 [[TMP17]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL:       [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
+; IF-EVL:       [[SCALAR_PH]]:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
+; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 33, %[[ENTRY]] ]
+; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
+; IF-EVL:       [[FOR_BODY]]:
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP14]] = add i64 [[IV1]], 42
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[IV1]]
+; IF-EVL-NEXT:    store i64 [[FOR1]], ptr [[ARRAYIDX]], align 8
+; IF-EVL-NEXT:    [[IV1_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV1_NEXT]], [[TC]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; IF-EVL:       [[FOR_END]]:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: define void @first_order_recurrence_indvar(
+; NO-VP-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT:  [[ENTRY:.*]]:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NO-VP:       [[VECTOR_PH]]:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP4]], 2
+; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
+; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP9]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP16:%.*]] = mul nuw i32 [[TMP14]], 2
+; NO-VP-NEXT:    [[TMP20:%.*]] = sub i32 [[TMP16]], 1
+; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 33, i32 [[TMP20]]
+; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
+; NO-VP:       [[VECTOR_BODY]]:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 2 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP12]] = add <vscale x 2 x i64> [[VEC_IND]], splat (i64 42)
+; NO-VP-NEXT:    [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP12]], i32 -1)
+; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[INDEX]]
+; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i64, ptr [[TMP11]], i32 0
+; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP13]], ptr [[TMP15]], align 8
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-VP-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; NO-VP:       [[MIDDLE_BLOCK]]:
+; NO-VP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP17]], 2
+; NO-VP-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP21]], 1
+; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 2 x i64> [[TMP12]], i32 [[TMP19]]
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; NO-VP:       [[SCALAR_PH]]:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 33, %[[ENTRY]] ]
+; NO-VP-NEXT:    br label %[[FOR_BODY:.*]]
+; NO-VP:       [[FOR_BODY]]:
+; NO-VP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
+; NO-VP-NEXT:    [[FOR1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP18]] = add i64 [[IV1]], 42
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[IV1]]
+; NO-VP-NEXT:    store i64 [[FOR1]], ptr [[ARRAYIDX]], align 8
+; NO-VP-NEXT:    [[IV1_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV1_NEXT]], [[TC]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; NO-VP:       [[FOR_END]]:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %indvars = phi i64 [ 0, %entry ], [ %indvars.next, %for.body ]
+  %for1 = phi i64 [ 33, %entry ], [ %x, %for.body ]
+
+  %x = add i64 %indvars, 42
+
+  %arrayidx = getelementptr inbounds nuw i64, ptr %A, i64 %indvars
+  store i64 %for1, ptr %arrayidx
+
+  %indvars.next = add nuw nsw i64 %indvars, 1
+  %exitcond.not = icmp eq i64 %indvars.next, %TC
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
 ;.
@@ -606,6 +751,8 @@ for.end:
 ; IF-EVL: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
 ; IF-EVL: [[LOOP9]] = distinct !{[[LOOP9]], [[META10:![0-9]+]]}
 ; IF-EVL: [[META10]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; IF-EVL: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]], [[META3]]}
+; IF-EVL: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META1]]}
 ;.
 ; NO-VP: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; NO-VP: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -617,4 +764,6 @@ for.end:
 ; NO-VP: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
 ; NO-VP: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
 ; NO-VP: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; NO-VP: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; NO-VP: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
index a52da79ee396..2b7a9fbdfd80 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-gather-scatter.ll
@@ -12,18 +12,58 @@
 define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %index, i64 %n) {
 ; IF-EVL-LABEL: @gather_scatter(
 ; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
+; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = sub i64 [[N]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], <vscale x 2 x i64> [[VEC_IND]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x float> @llvm.vp.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP15]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> [[WIDE_MASKED_GATHER2]], <vscale x 2 x ptr> align 4 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP11]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], [[TMP8]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N_VEC]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
+; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], i64 [[INDVARS_IV]]
+; IF-EVL-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX]], i64 [[INDVARS_IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
-; IF-EVL-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN:%.*]], i64 [[TMP0]]
+; IF-EVL-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN]], i64 [[TMP0]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-; IF-EVL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], i64 [[TMP0]]
+; IF-EVL-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[OUT]], i64 [[TMP0]]
 ; IF-EVL-NEXT:    store float [[TMP1]], ptr [[ARRAYIDX7]], align 4
-; IF-EVL-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]]
-; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; IF-EVL-NEXT:    [[INDVARS_IV_NEXT1]] = add nuw nsw i64 [[INDVARS_IV1]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT1]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
 ; IF-EVL:       for.end:
 ; IF-EVL-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index 83deebe42d38..8b1441450dd9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -602,7 +602,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp slt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -723,7 +723,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp sgt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -844,7 +844,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -965,7 +965,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp ugt <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i32> @llvm.vp.select.nxv4i32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -1311,7 +1311,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = fcmp fast olt <vscale x 4 x float> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x float> @llvm.vp.select.nxv4f32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
@@ -1432,7 +1432,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = fcmp fast ogt <vscale x 4 x float> [[VP_OP_LOAD]], [[VEC_PHI]]
-; IF-EVL-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x float> @llvm.vp.select.nxv4f32(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
 ; IF-EVL-NEXT:    [[TMP15]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP9]])
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP9]] to i64
 ; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index 96db5bf4e9ac..91d94e52d099 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -33,7 +33,8 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP7]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP18]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 1, [[TMP18]]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP18]], 1
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 -1, [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP9]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP10]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -41,7 +42,8 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP7]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = mul i64 0, [[TMP19]]
-; IF-EVL-NEXT:    [[TMP15:%.*]] = sub i64 1, [[TMP19]]
+; IF-EVL-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP19]], 1
+; IF-EVL-NEXT:    [[TMP15:%.*]] = mul i64 -1, [[TMP23]]
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP13]], i64 [[TMP14]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[TMP22]], i64 [[TMP15]]
 ; IF-EVL-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -136,7 +138,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[PTR1:%.*]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP26]]
-; IF-EVL-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP26]]
+; IF-EVL-NEXT:    [[TMP15:%.*]] = sub i64 [[TMP26]], 1
+; IF-EVL-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP15]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[TMP16]], i64 [[TMP17]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP19]], i64 [[TMP18]]
 ; IF-EVL-NEXT:    [[VP_REVERSE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.experimental.vp.reverse.nxv4i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -145,7 +148,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = zext i32 [[TMP5]] to i64
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = mul i64 0, [[TMP27]]
-; IF-EVL-NEXT:    [[TMP23:%.*]] = sub i64 1, [[TMP27]]
+; IF-EVL-NEXT:    [[TMP30:%.*]] = sub i64 [[TMP27]], 1
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 -1, [[TMP30]]
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[TMP21]], i64 [[TMP22]]
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 [[TMP23]]
 ; IF-EVL-NEXT:    [[VP_REVERSE5:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_REVERSE]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]])
@@ -261,7 +265,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP9]]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 1, [[TMP9]]
+; IF-EVL-NEXT:    [[TMP29:%.*]] = sub i64 [[TMP9]], 1
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP29]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP10]]
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP11]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
@@ -271,7 +276,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP16]]
-; IF-EVL-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP16]]
+; IF-EVL-NEXT:    [[TMP30:%.*]] = sub i64 [[TMP16]], 1
+; IF-EVL-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP30]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP15]], i64 [[TMP17]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP19]], i64 [[TMP18]]
 ; IF-EVL-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
@@ -279,7 +285,8 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[D:%.*]], i64 [[OFFSET_IDX]]
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP6]] to i64
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 0, [[TMP22]]
-; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 1, [[TMP22]]
+; IF-EVL-NEXT:    [[TMP31:%.*]] = sub i64 [[TMP22]], 1
+; IF-EVL-NEXT:    [[TMP24:%.*]] = mul i64 -1, [[TMP31]]
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP21]], i64 [[TMP23]]
 ; IF-EVL-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[TMP24]]
 ; IF-EVL-NEXT:    [[VP_REVERSE2:%.*]] = call <vscale x 16 x i8> @llvm.experimental.vp.reverse.nxv16i8(<vscale x 16 x i8> [[WIDE_MASKED_GATHER]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP6]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index 5c94ce180578..984b64c55ce1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -34,7 +34,8 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP11]] to i64
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = sub i64 1, [[TMP15]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i64 [[TMP15]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 -1, [[TMP23]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[ARRAYIDX13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[TMP18]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
index 83ae1f2d704e..bb61f431d064 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll
@@ -10,7 +10,6 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -24,7 +23,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -67,7 +66,6 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -81,7 +79,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -124,7 +122,6 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -138,7 +135,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -181,7 +178,6 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -195,7 +191,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -238,7 +234,6 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -252,7 +247,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -290,7 +285,6 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -304,7 +298,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -342,7 +336,6 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -356,7 +349,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -398,7 +391,6 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -412,7 +404,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -454,7 +446,6 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -468,7 +459,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
index 5c5492eec743..2e1bcaaf3fc2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-cast-intrinsics.ll
@@ -9,7 +9,6 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -23,7 +22,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -63,7 +62,6 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -77,7 +75,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -115,7 +113,6 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -129,7 +126,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -167,7 +164,6 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -181,7 +177,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -219,7 +215,6 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -233,7 +228,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -271,7 +266,6 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -285,7 +279,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -323,7 +317,6 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -337,7 +330,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -375,7 +368,6 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -389,7 +381,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -427,7 +419,6 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -441,7 +432,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -479,7 +470,6 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -493,7 +483,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]>  = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:     vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:     CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:     vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:     WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -527,8 +517,37 @@ exit:
 }
 
 define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
-; IF-EVL: LV: Vectorization is possible but not beneficial.
+; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
+; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF={1}' {
+; IF-EVL-NEXT: Live-in vp<[[VFUF:%.+]]> = VF * UF
+; IF-EVL-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; IF-EVL-NEXT: Live-in ir<[[N:%.+]]> = original trip-count
+
+; IF-EVL: vector.ph:
+; IF-EVL-NEXT: Successor(s): vector loop
 
+; IF-EVL: <x1> vector loop: {
+; IF-EVL-NEXT:   vector.body:
+; IF-EVL-NEXT:     EMIT vp<[[INDEX:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[INDEX_NEXT:%.+]]>
+; IF-EVL-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[INDEX_EVL:%.+]]> = phi ir<0>, vp<[[INDEX_EVL_NEXT:%.+]]>
+; IF-EVL-NEXT:     ir<[[IV:%.+]]> = WIDEN-INDUCTION  ir<0>, ir<1>, vp<[[EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[AVL:%.+]]> = sub ir<[[N]]>, vp<[[INDEX_EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; IF-EVL-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[INDEX_EVL]]>, ir<1>, vp<[[EVL]]>
+; IF-EVL-NEXT:     WIDEN-GEP Inv[Var] ir<[[GEP:%.+]]> = getelementptr inbounds ir<%b>, ir<[[IV]]>
+; IF-EVL-NEXT:     WIDEN-CAST ir<[[PTRTOINT:%.+]]> = ptrtoint ir<[[GEP]]> to i64
+; IF-EVL-NEXT:     CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[SCALAR_STEPS]]>
+; IF-EVL-NEXT:     vp<[[VECTOR_PTR:%.+]]> = vector-pointer ir<[[GEP2]]>
+; IF-EVL-NEXT:     WIDEN vp.store vp<[[VECTOR_PTR]]>, ir<[[PTRTOINT]]>, vp<[[EVL]]>
+; IF-EVL-NEXT:     EMIT-SCALAR vp<[[ZEXT:%.+]]> = zext vp<[[EVL]]> to i64
+; IF-EVL-NEXT:     EMIT vp<[[INDEX_EVL_NEXT]]> = add vp<[[ZEXT]]>, vp<[[INDEX_EVL]]>
+; IF-EVL-NEXT:     EMIT vp<[[INDEX_NEXT]]> = add vp<[[INDEX]]>, vp<[[VFUF]]>
+; IF-EVL-NEXT:     EMIT branch-on-count vp<[[INDEX_NEXT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT:   No successors
+; IF-EVL-NEXT: }
+; IF-EVL-NEXT: Successor(s): middle.block
 entry:
   br label %loop
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index 0fd27f335b64..aa15a20a21d7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -28,7 +28,6 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 
 ; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-OUTLOOP-NEXT: Live-in ir<%n> = original trip-count
@@ -47,7 +46,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-OUTLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-OUTLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-OUTLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; IF-EVL-OUTLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-OUTLOOP-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -63,16 +62,14 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-EMPTY:
 ; IF-EVL-OUTLOOP-NEXT: middle.block:
 ; IF-EVL-OUTLOOP-NEXT:   EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, vp<[[RDX_SELECT]]>
-; IF-EVL-OUTLOOP-NEXT:   EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
 ; IF-EVL-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>
 ; IF-EVL-OUTLOOP-EMPTY:
 ; IF-EVL-OUTLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-OUTLOOP-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; IF-EVL-OUTLOOP-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
 ; IF-EVL-OUTLOOP-NEXT: No successors
 ;
 
 ; IF-EVL-INLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-INLOOP-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-INLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-INLOOP-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-INLOOP-NEXT: Live-in ir<%n> = original trip-count
@@ -88,7 +85,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%n>, vp<[[EVL_PHI]]>
 ; IF-EVL-INLOOP-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-INLOOP-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-INLOOP-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; IF-EVL-INLOOP-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-INLOOP-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -103,11 +100,10 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP-NEXT: middle.block:
 ; IF-EVL-INLOOP-NEXT:   EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
-; IF-EVL-INLOOP-NEXT:   EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
 ; IF-EVL-INLOOP-NEXT: Successor(s): ir-bb<for.end>
 ; IF-EVL-INLOOP-EMPTY:
 ; IF-EVL-INLOOP-NEXT: ir-bb<for.end>:
-; IF-EVL-INLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; IF-EVL-INLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
 ; IF-EVL-INLOOP-NEXT: No successors
 ;
 
@@ -138,13 +134,12 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: middle.block:
 ; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
-; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
 ; NO-VP-OUTLOOP-NEXT:   EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
 ; NO-VP-OUTLOOP-NEXT:   EMIT branch-on-cond vp<[[BOC]]>
 ; NO-VP-OUTLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-OUTLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; NO-VP-OUTLOOP-NEXT:  IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
 ; NO-VP-OUTLOOP-NEXT: No successors
 ; NO-VP-OUTLOOP-EMPTY:
 ; NO-VP-OUTLOOP-NEXT: scalar.ph:
@@ -187,13 +182,12 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: middle.block:
 ; NO-VP-INLOOP-NEXT:   EMIT vp<[[RDX:%.+]]> = compute-reduction-result ir<[[RDX_PHI]]>, ir<[[ADD]]>
-; NO-VP-INLOOP-NEXT:   EMIT vp<[[RDX_EX:%.+]]> = extract-last-element vp<[[RDX]]>
 ; NO-VP-INLOOP-NEXT:   EMIT vp<[[BOC:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
 ; NO-VP-INLOOP-NEXT:   EMIT branch-on-cond vp<[[BOC]]>
 ; NO-VP-INLOOP-NEXT: Successor(s): ir-bb<for.end>, scalar.ph
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: ir-bb<for.end>:
-; NO-VP-INLOOP-NEXT:   IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX_EX]]> from middle.block)
+; NO-VP-INLOOP-NEXT:   IR %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RDX]]> from middle.block)
 ; NO-VP-INLOOP-NEXT: No successors
 ; NO-VP-INLOOP-EMPTY:
 ; NO-VP-INLOOP-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index fc81bba4e9c2..563e515726c5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -15,7 +15,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
 ;
 ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
-; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
@@ -29,7 +28,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:    EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
 ; IF-EVL-NEXT:    EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
 ; IF-EVL-NEXT:    EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
-; IF-EVL-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[VF]]>
+; IF-EVL-NEXT:    vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
 ; IF-EVL-NEXT:    CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
 ; IF-EVL-NEXT:    vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
 ; IF-EVL-NEXT:    WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
@@ -97,7 +96,7 @@ for.cond.cleanup:
 
 define void @safe_dep(ptr %p) {
 ; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2},UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
 ; CHECK-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
 ; CHECK-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in ir<512> = original trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
deleted file mode 100644
index 8c0e4da51f72..000000000000
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ /dev/null
@@ -1,77 +0,0 @@
-; REQUIRES: asserts
-
- ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \
- ; RUN: -force-tail-folding-style=data-with-evl \
- ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \
- ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s
-
- define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
- ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
- ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
- ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}'
- ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
- ;
- ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
- ; IF-EVL-NEXT: Live-in ir<[[VFUF:%.+]]> = VF * UF
- ; IF-EVL-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
- ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count
-
- ; IF-EVL:      ir-bb<entry>:
- ; IF-EVL-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
-
- ; IF-EVL:      ir-bb<vector.ph>:
- ; IF-EVL-NEXT:   IR   %0 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %1 = mul nuw i64 %0, 4
- ; IF-EVL-NEXT:   IR   %2 = sub i64 %1, 1
- ; IF-EVL-NEXT:   IR   %n.rnd.up = add i64 %N, %2
- ; IF-EVL-NEXT:   IR   %n.mod.vf = urem i64 %n.rnd.up, %1
- ; IF-EVL-NEXT:   IR   %n.vec = sub i64 %n.rnd.up, %n.mod.vf
- ; IF-EVL-NEXT:   IR   %3 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %4 = mul nuw i64 %3, 4
- ; IF-EVL-NEXT: Successor(s): vector.body
- ; IF-EVL-EMPTY:
- ; IF-EVL-NEXT: vector.body:
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[IV_NEXT_EXIT:%.+]]>, vector.body ]
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[EVL_PHI:%.+]]>  = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[IV_NEX:%.+]]>, vector.body ]
- ; IF-EVL-NEXT:   EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
- ; IF-EVL-NEXT:   WIDEN ir<[[SUB:%.+]]> = sub ir<0>, ir<[[LD2]]>
- ; IF-EVL-NEXT:   WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<[[LD2]]>, ir<[[SUB]]>, vp<[[EVL]]>)
- ; IF-EVL-NEXT:   WIDEN ir<[[ADD:%.+]]> = add vp<[[SELECT]]>, ir<[[LD1]]>
- ; IF-EVL-NEXT:   CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
- ; IF-EVL-NEXT:   WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
- ; IF-EVL-NEXT:   EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
- ; IF-EVL-NEXT:   EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
- ; IF-EVL-NEXT:   EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<[[VFUF]]>
- ; IF-EVL-NEXT:   EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>,  ir<[[VTC]]>
-
- entry:
-   br label %for.body
-
- for.body:
-   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
-   %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
-   %0 = load i32, ptr %arrayidx, align 4
-   %arrayidx3 = getelementptr inbounds i32, ptr %c, i64 %indvars.iv
-   %1 = load i32, ptr %arrayidx3, align 4
-   %cmp4 = icmp sgt i32 %0, %1
-   %2 = sub i32 0, %1
-   %cond.p = select i1 %cmp4, i32 %1, i32 %2
-   %cond = add i32 %cond.p, %0
-   %arrayidx15 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
-   store i32 %cond, ptr %arrayidx15, align 4
-   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-   %exitcond.not = icmp eq i64 %indvars.iv.next, %N
-   br i1 %exitcond.not, label %exit, label %for.body
-
- exit:
-   ret void
- }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 2c6fe4f5c808..147e949808b5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
-; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
+; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index 3361068c9922..86fa5d682041 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -393,12 +393,6 @@ define void @test_for_tried_to_force_scalar(ptr noalias %A, ptr noalias %B, ptr
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x float> [[STRIDED_VEC]], i32 3
 ; CHECK-NEXT:    store float [[TMP30]], ptr [[C:%.*]], align 4
-; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 0
-; CHECK-NEXT:    [[TMP38:%.*]] = load float, ptr [[TMP31]], align 4
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 1
-; CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[TMP33]], align 4
-; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 2
-; CHECK-NEXT:    [[TMP34:%.*]] = load float, ptr [[TMP35]], align 4
 ; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x ptr> [[TMP29]], i32 3
 ; CHECK-NEXT:    [[TMP36:%.*]] = load float, ptr [[TMP37]], align 4
 ; CHECK-NEXT:    store float [[TMP36]], ptr [[B:%.*]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
index cdc7839bfc0f..95258e65bbe3 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-ptradd-with-replicated-operand.ll
@@ -32,42 +32,31 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 104
 ; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[OFFSET_IDX]], 112
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 120
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP0]]
+; CHECK-NEXT:    [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP3]]
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP4]]
+; CHECK-NEXT:    [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP7]]
-; CHECK-NEXT:    [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP8]]
+; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP10]]
-; CHECK-NEXT:    [[NEXT_GEP12:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP11]]
-; CHECK-NEXT:    [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP12]]
-; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP13]]
-; CHECK-NEXT:    [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP14]]
+; CHECK-NEXT:    [[NEXT_GEP17:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP11]]
+; CHECK-NEXT:    [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP12]]
+; CHECK-NEXT:    [[NEXT_GEP18:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP13]]
+; CHECK-NEXT:    [[NEXT_GEP19:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[M]], i64 [[TMP15]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 4
-; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP3]], i64 4
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i64 4
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[NEXT_GEP5]], i64 4
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[NEXT_GEP6]], i64 4
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[NEXT_GEP7]], i64 4
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[NEXT_GEP8]], i64 4
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[NEXT_GEP9]], i64 4
-; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP10]], i64 4
-; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i64 4
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[NEXT_GEP12]], i64 4
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[NEXT_GEP13]], i64 4
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[NEXT_GEP14]], i64 4
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[NEXT_GEP15]], i64 4
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[NEXT_GEP16]], i64 4
-; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP16]], i32 -4
-; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP20]], i32 -4
-; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP24]], i32 -4
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[TMP28]], i32 -4
+; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[TMP27]], i32 -4
+; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i8, ptr [[TMP28]], i32 -4
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[TMP29]], i32 -4
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[TMP30]], i32 -4
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP32]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    [[STRIDED_VEC17:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
@@ -85,7 +74,7 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
 ; CHECK-NEXT:    [[TMP38:%.*]] = add <4 x i32> [[STRIDED_VEC23]], [[STRIDED_VEC22]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = add <4 x i32> [[STRIDED_VEC26]], [[STRIDED_VEC25]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <4 x i32> [[TMP36]], i32 0
-; CHECK-NEXT:    store i32 [[TMP40]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT:    store i32 [[TMP40]], ptr [[NEXT_GEP12]], align 4
 ; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[TMP36]], i32 1
 ; CHECK-NEXT:    store i32 [[TMP41]], ptr [[NEXT_GEP2]], align 4
 ; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <4 x i32> [[TMP36]], i32 2
@@ -93,7 +82,7 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
 ; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <4 x i32> [[TMP36]], i32 3
 ; CHECK-NEXT:    store i32 [[TMP43]], ptr [[NEXT_GEP4]], align 4
 ; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <4 x i32> [[TMP37]], i32 0
-; CHECK-NEXT:    store i32 [[TMP44]], ptr [[NEXT_GEP5]], align 4
+; CHECK-NEXT:    store i32 [[TMP44]], ptr [[NEXT_GEP13]], align 4
 ; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <4 x i32> [[TMP37]], i32 1
 ; CHECK-NEXT:    store i32 [[TMP45]], ptr [[NEXT_GEP6]], align 4
 ; CHECK-NEXT:    [[TMP46:%.*]] = extractelement <4 x i32> [[TMP37]], i32 2
@@ -101,19 +90,19 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
 ; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <4 x i32> [[TMP37]], i32 3
 ; CHECK-NEXT:    store i32 [[TMP47]], ptr [[NEXT_GEP8]], align 4
 ; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <4 x i32> [[TMP38]], i32 0
-; CHECK-NEXT:    store i32 [[TMP48]], ptr [[NEXT_GEP9]], align 4
+; CHECK-NEXT:    store i32 [[TMP48]], ptr [[NEXT_GEP14]], align 4
 ; CHECK-NEXT:    [[TMP49:%.*]] = extractelement <4 x i32> [[TMP38]], i32 1
 ; CHECK-NEXT:    store i32 [[TMP49]], ptr [[NEXT_GEP10]], align 4
 ; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <4 x i32> [[TMP38]], i32 2
 ; CHECK-NEXT:    store i32 [[TMP50]], ptr [[NEXT_GEP11]], align 4
 ; CHECK-NEXT:    [[TMP51:%.*]] = extractelement <4 x i32> [[TMP38]], i32 3
-; CHECK-NEXT:    store i32 [[TMP51]], ptr [[NEXT_GEP12]], align 4
+; CHECK-NEXT:    store i32 [[TMP51]], ptr [[NEXT_GEP17]], align 4
 ; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <4 x i32> [[TMP39]], i32 0
-; CHECK-NEXT:    store i32 [[TMP52]], ptr [[NEXT_GEP13]], align 4
+; CHECK-NEXT:    store i32 [[TMP52]], ptr [[NEXT_GEP15]], align 4
 ; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <4 x i32> [[TMP39]], i32 1
-; CHECK-NEXT:    store i32 [[TMP53]], ptr [[NEXT_GEP14]], align 4
+; CHECK-NEXT:    store i32 [[TMP53]], ptr [[NEXT_GEP18]], align 4
 ; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <4 x i32> [[TMP39]], i32 2
-; CHECK-NEXT:    store i32 [[TMP54]], ptr [[NEXT_GEP15]], align 4
+; CHECK-NEXT:    store i32 [[TMP54]], ptr [[NEXT_GEP19]], align 4
 ; CHECK-NEXT:    [[TMP55:%.*]] = extractelement <4 x i32> [[TMP39]], i32 3
 ; CHECK-NEXT:    store i32 [[TMP55]], ptr [[NEXT_GEP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
@@ -123,11 +112,11 @@ define ptr @test_interleave_ptradd_with_replicated_op(ptr %m) #0 {
 ; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[M]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 97, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i32 [ 97, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL26]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
 ; CHECK-NEXT:    [[P_4:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4
 ; CHECK-NEXT:    [[L_1:%.*]] = load i32, ptr [[P_4]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
index 644f10b617eb..db9be20f7d19 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
@@ -10,6 +10,8 @@ define ptr @test(ptr noalias %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x ptr> [[TMP16]], ptr [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i64> [[VEC_IND]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
deleted file mode 100644
index 3e88672f2924..000000000000
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
+++ /dev/null
@@ -1,407 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -p loop-vectorize -force-vector-width=2 -S %s | FileCheck %s
-
-declare void @llvm.assume(i1)
-
-; %a is known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_non_constant_trip_count_access_i8(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
-; CHECK-LABEL: define void @deref_assumption_in_preheader_non_constant_trip_count_access_i8(
-; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[N]]) ]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP17:%.*]] = load i8, ptr [[TMP16]], align 1
-; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i8> poison, i8 [[TMP17]], i32 0
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
-; CHECK:       [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i8> [[TMP9]], i8 [[TMP13]], i32 1
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i8> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i8> [[WIDE_LOAD]], <2 x i8> [[WIDE_LOAD1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    store <2 x i8> [[PREDPHI]], ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    [[L_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i8 [[L_B]], 0
-; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
-; CHECK:       [[LOOP_THEN]]:
-; CHECK-NEXT:    [[L_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[MERGE:%.*]] = phi i8 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP_HEADER]] ]
-; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[IV]]
-; CHECK-NEXT:    store i8 [[MERGE]], ptr [[GEP_C]], align 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 %n) ]
-  br label %loop.header
-
-loop.header:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
-  %gep.a = getelementptr inbounds i8, ptr %a, i64 %iv
-  %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv
-  %l.b = load i8, ptr %gep.b, align 1
-  %c.1 = icmp sge i8 %l.b, 0
-  br i1 %c.1, label %loop.latch, label %loop.then
-
-loop.then:
-  %l.a = load i8, ptr %gep.a, align 1
-  br label %loop.latch
-
-loop.latch:
-  %merge = phi i8 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
-  %gep.c = getelementptr inbounds i8, ptr %c, i64 %iv
-  store i8 %merge, ptr %gep.c, align 1
-  %iv.next = add nuw nsw i64 %iv, 1
-  %ec = icmp eq i64 %iv.next, %n
-  br i1 %ec, label %exit, label %loop.header
-
-exit:
-  ret void
-}
-
-; %a is known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
-; CHECK-LABEL: define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[N]], 4
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[MUL]]) ]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 1
-; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
-; CHECK:       [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
-; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
-; CHECK:       [[LOOP_THEN]]:
-; CHECK-NEXT:    [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 1
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP_HEADER]] ]
-; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
-; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_C]], align 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  %mul = mul nsw nuw i64 %n, 4
-  call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 %mul) ]
-  br label %loop.header
-
-loop.header:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
-  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
-  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
-  %l.b = load i32, ptr %gep.b, align 1
-  %c.1 = icmp sge i32 %l.b, 0
-  br i1 %c.1, label %loop.latch, label %loop.then
-
-loop.then:
-  %l.a = load i32, ptr %gep.a, align 1
-  br label %loop.latch
-
-loop.latch:
-  %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
-  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
-  store i32 %merge, ptr %gep.c, align 1
-  %iv.next = add nuw nsw i64 %iv, 1
-  %ec = icmp eq i64 %iv.next, %n
-  br i1 %ec, label %exit, label %loop.header
-
-exit:
-  ret void
-}
-
-
-; %a is NOT known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
-; CHECK-LABEL: define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[N]]) ]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 1
-; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
-; CHECK:       [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
-; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
-; CHECK:       [[LOOP_THEN]]:
-; CHECK-NEXT:    [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 1
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP_HEADER]] ]
-; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
-; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_C]], align 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 %n) ]
-  br label %loop.header
-
-loop.header:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
-  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
-  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
-  %l.b = load i32, ptr %gep.b, align 1
-  %c.1 = icmp sge i32 %l.b, 0
-  br i1 %c.1, label %loop.latch, label %loop.then
-
-loop.then:
-  %l.a = load i32, ptr %gep.a, align 1
-  br label %loop.latch
-
-loop.latch:
-  %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
-  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
-  store i32 %merge, ptr %gep.c, align 1
-  %iv.next = add nuw nsw i64 %iv, 1
-  %ec = icmp eq i64 %iv.next, %n
-  br i1 %ec, label %exit, label %loop.header
-
-exit:
-  ret void
-}
-
-; %a is NOT known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
-; CHECK-LABEL: define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 100) ]
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
-; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP0]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
-; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 1
-; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
-; CHECK:       [[PRED_LOAD_CONTINUE]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
-; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
-; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
-; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 1
-; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
-; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
-; CHECK:       [[LOOP_THEN]]:
-; CHECK-NEXT:    [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 1
-; CHECK-NEXT:    br label %[[LOOP_LATCH]]
-; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP_HEADER]] ]
-; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
-; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_C]], align 1
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    ret void
-;
-entry:
-  call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 100) ]
-  br label %loop.header
-
-loop.header:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
-  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
-  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
-  %l.b = load i32, ptr %gep.b, align 1
-  %c.1 = icmp sge i32 %l.b, 0
-  br i1 %c.1, label %loop.latch, label %loop.then
-
-loop.then:
-  %l.a = load i32, ptr %gep.a, align 1
-  br label %loop.latch
-
-loop.latch:
-  %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ]
-  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
-  store i32 %merge, ptr %gep.c, align 1
-  %iv.next = add nuw nsw i64 %iv, 1
-  %ec = icmp eq i64 %iv.next, %n
-  br i1 %ec, label %exit, label %loop.header
-
-exit:
-  ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index 22b34079755c..e9b8e8cdda52 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 3a08681a2fb9..13fd2eff8729 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
 
 define i64 @loop_contains_store(ptr %dest) {
 ; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
@@ -165,35 +165,6 @@ exit:
   ret void
 }
 
-define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
-; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
-entry:
-  %n_bytes = mul nuw nsw i32 %n, 2
-  call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
-  %tc = sext i32 %n to i64
-  br label %for.body
-
-for.body:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
-  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
-  %data = load i16, ptr %st.addr, align 2
-  %inc = add nsw i16 %data, 1
-  store i16 %inc, ptr %st.addr, align 2
-  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
-  %ee.val = load i16, ptr %ee.addr, align 2
-  %ee.cond = icmp sgt i16 %ee.val, 500
-  br i1 %ee.cond, label %exit, label %for.inc
-
-for.inc:
-  %iv.next = add nuw nsw i64 %iv, 1
-  %counted.cond = icmp eq i64 %iv.next, %tc
-  br i1 %counted.cond, label %exit, label %for.body
-
-exit:
-  ret void
-}
-
 define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
 ; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
 ; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index 0a2bb8d5682f..c101d6a19aa2 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
-; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
-; CHECK-NEXT:    [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
-; CHECK-NEXT:    [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
+; CHECK-NEXT:    [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
 ; CHECK-NEXT:    br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       vec.epilog.scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 9d2719a6153c..9deab9063d71 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -213,11 +213,10 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:  EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]>
-; CHECK-NEXT:  EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:   IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT:   IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
 ; CHECK-NEXT: No successors
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 7684d274a75c..17c2be64f1a3 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -1055,16 +1055,10 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 10
 ; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 12
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 14
-; UNROLL-NO-IC-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2
 ; UNROLL-NO-IC-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2
-; UNROLL-NO-IC-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP7]]
-; UNROLL-NO-IC-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
-; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
+; UNROLL-NO-IC-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP9]]
 ; UNROLL-NO-IC-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; UNROLL-NO-IC-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP11]], align 4
-; UNROLL-NO-IC-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 4
 ; UNROLL-NO-IC-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
 ; UNROLL-NO-IC-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
 ; UNROLL-NO-IC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
@@ -1152,16 +1146,10 @@ define i32 @PR30183(i32 %pre_load, ptr %a, ptr %b, i64 %n) {
 ; SINK-AFTER-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; SINK-AFTER-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; SINK-AFTER-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 6
-; SINK-AFTER-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 2
-; SINK-AFTER-NEXT:    [[TMP8:%.*]] = add nuw nsw i64 [[TMP4]], 2
 ; SINK-AFTER-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP5]], 2
 ; SINK-AFTER-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 2
-; SINK-AFTER-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP7]]
-; SINK-AFTER-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP8]]
-; SINK-AFTER-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP9]]
+; SINK-AFTER-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP9]]
 ; SINK-AFTER-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP10]]
-; SINK-AFTER-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP11]], align 4
-; SINK-AFTER-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP12]], align 4
 ; SINK-AFTER-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP13]], align 4
 ; SINK-AFTER-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP14]], align 4
 ; SINK-AFTER-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
index 0306567c5ce9..a02fddc4cf72 100644
--- a/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+++ b/llvm/test/Transforms/LoopVectorize/invariant-store-vectorization-2.ll
@@ -134,18 +134,14 @@ define void @inv_val_store_to_inv_address_conditional_inv(ptr %a, i64 %n, ptr %b
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX2]], 9223372036854775804
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = insertelement <4 x i32> poison, i32 [[K]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i1> poison, i1 [[CMP]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[NTRUNC]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT6]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[PREDPHI]], i64 0
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP3]], align 4, !alias.scope [[META9:![0-9]+]], !noalias [[META12:![0-9]+]]
-; CHECK-NEXT:    store i32 [[TMP2]], ptr [[A]], align 4, !alias.scope [[META12]]
+; CHECK-NEXT:    store i32 [[K]], ptr [[A]], align 4, !alias.scope [[META12]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index cf10d32e063a..d224da795997 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -1,26 +1,185 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck %s
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S < %s | FileCheck --check-prefixes=CHECK,IC1VF4 %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -S < %s | FileCheck --check-prefixes=CHECK,IC4VF4 %s
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=1 -S < %s | FileCheck --check-prefixes=CHECK,IC4VF1 %s
 
 define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
-; CHECK-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
-; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
-; CHECK-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC1VF4-SAME: ptr [[A:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 19999, i64 19998, i64 19997, i64 19996>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = icmp sgt <4 x i64> [[REVERSE]], splat (i64 3)
+; IC1VF4-NEXT:    [[TMP4]] = select <4 x i1> [[TMP3]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC1VF4-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC1VF4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[TMP4]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP6]], 9223372036854775807
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP6]], i64 331
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC1VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC1VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC4VF4-SAME: ptr [[A:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 19999, i64 19998, i64 19997, i64 19996>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i64> [[STEP_ADD]], splat (i64 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i64> [[STEP_ADD_2]], splat (i64 -4)
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC4VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC4VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 0
+; IC4VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 -3
+; IC4VF4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -4
+; IC4VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP3]], i32 -3
+; IC4VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -8
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 -3
+; IC4VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i32 -12
+; IC4VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 -3
+; IC4VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; IC4VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
+; IC4VF4-NEXT:    [[REVERSE5:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD4]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i64>, ptr [[TMP6]], align 8
+; IC4VF4-NEXT:    [[REVERSE7:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD6]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; IC4VF4-NEXT:    [[REVERSE9:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD8]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC4VF4-NEXT:    [[TMP9:%.*]] = icmp sgt <4 x i64> [[REVERSE]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP10:%.*]] = icmp sgt <4 x i64> [[REVERSE5]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP11:%.*]] = icmp sgt <4 x i64> [[REVERSE7]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP12:%.*]] = icmp sgt <4 x i64> [[REVERSE9]], splat (i64 3)
+; IC4VF4-NEXT:    [[TMP13]] = select <4 x i1> [[TMP9]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP14]] = select <4 x i1> [[TMP10]], <4 x i64> [[STEP_ADD]], <4 x i64> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP15]] = select <4 x i1> [[TMP11]], <4 x i64> [[STEP_ADD_2]], <4 x i64> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP16]] = select <4 x i1> [[TMP12]], <4 x i64> [[STEP_ADD_3]], <4 x i64> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 -4)
+; IC4VF4-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC4VF4-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[TMP13]], <4 x i64> [[TMP14]])
+; IC4VF4-NEXT:    [[RDX_MINMAX10:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[RDX_MINMAX]], <4 x i64> [[TMP15]])
+; IC4VF4-NEXT:    [[RDX_MINMAX11:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> [[RDX_MINMAX10]], <4 x i64> [[TMP16]])
+; IC4VF4-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> [[RDX_MINMAX11]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[TMP18]], 9223372036854775807
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP18]], i64 331
+; IC4VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC4VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC4VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
+; IC4VF1-SAME: ptr [[A:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i64 [ 9223372036854775807, %[[VECTOR_PH]] ], [ [[TMP18:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 19999, [[INDEX]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; IC4VF1-NEXT:    [[TMP11:%.*]] = icmp sgt i64 [[TMP7]], 3
+; IC4VF1-NEXT:    [[TMP12:%.*]] = icmp sgt i64 [[TMP8]], 3
+; IC4VF1-NEXT:    [[TMP13:%.*]] = icmp sgt i64 [[TMP9]], 3
+; IC4VF1-NEXT:    [[TMP14:%.*]] = icmp sgt i64 [[TMP10]], 3
+; IC4VF1-NEXT:    [[TMP15]] = select i1 [[TMP11]], i64 [[OFFSET_IDX]], i64 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP16]] = select i1 [[TMP12]], i64 [[TMP0]], i64 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP17]] = select i1 [[TMP13]], i64 [[TMP1]], i64 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP18]] = select i1 [[TMP14]], i64 [[TMP2]], i64 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20000
+; IC4VF1-NEXT:    br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP15]], i64 [[TMP16]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i64 @llvm.smin.i64(i64 [[RDX_MINMAX]], i64 [[TMP17]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i64 @llvm.smin.i64(i64 [[RDX_MINMAX4]], i64 [[TMP18]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i64 [[RDX_MINMAX5]], 9223372036854775807
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[RDX_MINMAX5]], i64 331
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 19999, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 331, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
+; IC4VF1-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
+; IC4VF1-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_A_3]], i64 [[IV]], i64 [[RDX]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i64 [[IV]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i64 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -43,23 +202,354 @@ exit:                                             ; preds = %loop
 @table = constant [13 x i16] [i16 10, i16 35, i16 69, i16 147, i16 280, i16 472, i16 682, i16 1013, i16 1559, i16 2544, i16 4553, i16 6494, i16 10000], align 1
 
 define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
-; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
-; CHECK-SAME: i16 noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
-; CHECK-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
-; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC1VF4-SAME: i16 noundef [[VAL:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
+; IC1VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP2]], align 1
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i16> [[WIDE_LOAD]], <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = icmp ugt <4 x i16> [[REVERSE]], [[BROADCAST_SPLAT]]
+; IC1VF4-NEXT:    [[TMP4:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC1VF4-NEXT:    [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP4]], <4 x i16> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC1VF4-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC1VF4-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[TMP5]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP7]], 32767
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC4VF4-SAME: i16 noundef [[VAL:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
+; IC4VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
+; IC4VF4-NEXT:    [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
+; IC4VF4-NEXT:    [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
+; IC4VF4-NEXT:    [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
+; IC4VF4-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4:       [[PRED_LOAD_IF]]:
+; IC4VF4-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 1
+; IC4VF4-NEXT:    [[TMP8:%.*]] = insertelement <4 x i16> poison, i16 [[TMP7]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE]]:
+; IC4VF4-NEXT:    [[TMP9:%.*]] = phi <4 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; IC4VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4:       [[PRED_LOAD_IF15]]:
+; IC4VF4-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT:    [[TMP13:%.*]] = load i16, ptr [[TMP12]], align 1
+; IC4VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i16> [[TMP9]], i16 [[TMP13]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE16]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE16]]:
+; IC4VF4-NEXT:    [[TMP15:%.*]] = phi <4 x i16> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
+; IC4VF4-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4:       [[PRED_LOAD_IF17]]:
+; IC4VF4-NEXT:    [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT:    [[TMP19:%.*]] = load i16, ptr [[TMP18]], align 1
+; IC4VF4-NEXT:    [[TMP20:%.*]] = insertelement <4 x i16> [[TMP15]], i16 [[TMP19]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE18]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE18]]:
+; IC4VF4-NEXT:    [[TMP21:%.*]] = phi <4 x i16> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
+; IC4VF4-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4:       [[PRED_LOAD_IF19]]:
+; IC4VF4-NEXT:    [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF4-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT:    [[TMP25:%.*]] = load i16, ptr [[TMP24]], align 1
+; IC4VF4-NEXT:    [[TMP26:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[TMP25]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE20]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE20]]:
+; IC4VF4-NEXT:    [[TMP27:%.*]] = phi <4 x i16> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
+; IC4VF4-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4:       [[PRED_LOAD_IF21]]:
+; IC4VF4-NEXT:    [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
+; IC4VF4-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT:    [[TMP31:%.*]] = load i16, ptr [[TMP30]], align 1
+; IC4VF4-NEXT:    [[TMP32:%.*]] = insertelement <4 x i16> poison, i16 [[TMP31]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE22]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE22]]:
+; IC4VF4-NEXT:    [[TMP33:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
+; IC4VF4-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4:       [[PRED_LOAD_IF23]]:
+; IC4VF4-NEXT:    [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
+; IC4VF4-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT:    [[TMP37:%.*]] = load i16, ptr [[TMP36]], align 1
+; IC4VF4-NEXT:    [[TMP38:%.*]] = insertelement <4 x i16> [[TMP33]], i16 [[TMP37]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE24]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE24]]:
+; IC4VF4-NEXT:    [[TMP39:%.*]] = phi <4 x i16> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
+; IC4VF4-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4:       [[PRED_LOAD_IF25]]:
+; IC4VF4-NEXT:    [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
+; IC4VF4-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT:    [[TMP43:%.*]] = load i16, ptr [[TMP42]], align 1
+; IC4VF4-NEXT:    [[TMP44:%.*]] = insertelement <4 x i16> [[TMP39]], i16 [[TMP43]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE26]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE26]]:
+; IC4VF4-NEXT:    [[TMP45:%.*]] = phi <4 x i16> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
+; IC4VF4-NEXT:    [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4:       [[PRED_LOAD_IF27]]:
+; IC4VF4-NEXT:    [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
+; IC4VF4-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT:    [[TMP49:%.*]] = load i16, ptr [[TMP48]], align 1
+; IC4VF4-NEXT:    [[TMP50:%.*]] = insertelement <4 x i16> [[TMP45]], i16 [[TMP49]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE28]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE28]]:
+; IC4VF4-NEXT:    [[TMP51:%.*]] = phi <4 x i16> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
+; IC4VF4-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4:       [[PRED_LOAD_IF29]]:
+; IC4VF4-NEXT:    [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
+; IC4VF4-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT:    [[TMP55:%.*]] = load i16, ptr [[TMP54]], align 1
+; IC4VF4-NEXT:    [[TMP56:%.*]] = insertelement <4 x i16> poison, i16 [[TMP55]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE30]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE30]]:
+; IC4VF4-NEXT:    [[TMP57:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
+; IC4VF4-NEXT:    [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4:       [[PRED_LOAD_IF31]]:
+; IC4VF4-NEXT:    [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
+; IC4VF4-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT:    [[TMP61:%.*]] = load i16, ptr [[TMP60]], align 1
+; IC4VF4-NEXT:    [[TMP62:%.*]] = insertelement <4 x i16> [[TMP57]], i16 [[TMP61]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE32]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE32]]:
+; IC4VF4-NEXT:    [[TMP63:%.*]] = phi <4 x i16> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
+; IC4VF4-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4:       [[PRED_LOAD_IF33]]:
+; IC4VF4-NEXT:    [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
+; IC4VF4-NEXT:    [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT:    [[TMP67:%.*]] = load i16, ptr [[TMP66]], align 1
+; IC4VF4-NEXT:    [[TMP68:%.*]] = insertelement <4 x i16> [[TMP63]], i16 [[TMP67]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE34]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE34]]:
+; IC4VF4-NEXT:    [[TMP69:%.*]] = phi <4 x i16> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
+; IC4VF4-NEXT:    [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4:       [[PRED_LOAD_IF35]]:
+; IC4VF4-NEXT:    [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
+; IC4VF4-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT:    [[TMP73:%.*]] = load i16, ptr [[TMP72]], align 1
+; IC4VF4-NEXT:    [[TMP74:%.*]] = insertelement <4 x i16> [[TMP69]], i16 [[TMP73]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE36]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE36]]:
+; IC4VF4-NEXT:    [[TMP75:%.*]] = phi <4 x i16> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
+; IC4VF4-NEXT:    [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4:       [[PRED_LOAD_IF37]]:
+; IC4VF4-NEXT:    [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
+; IC4VF4-NEXT:    [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT:    [[TMP79:%.*]] = load i16, ptr [[TMP78]], align 1
+; IC4VF4-NEXT:    [[TMP80:%.*]] = insertelement <4 x i16> poison, i16 [[TMP79]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE38]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE38]]:
+; IC4VF4-NEXT:    [[TMP81:%.*]] = phi <4 x i16> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
+; IC4VF4-NEXT:    [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4:       [[PRED_LOAD_IF39]]:
+; IC4VF4-NEXT:    [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
+; IC4VF4-NEXT:    [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT:    [[TMP85:%.*]] = load i16, ptr [[TMP84]], align 1
+; IC4VF4-NEXT:    [[TMP86:%.*]] = insertelement <4 x i16> [[TMP81]], i16 [[TMP85]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE40]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE40]]:
+; IC4VF4-NEXT:    [[TMP87:%.*]] = phi <4 x i16> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
+; IC4VF4-NEXT:    [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4:       [[PRED_LOAD_IF41]]:
+; IC4VF4-NEXT:    [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
+; IC4VF4-NEXT:    [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT:    [[TMP91:%.*]] = load i16, ptr [[TMP90]], align 1
+; IC4VF4-NEXT:    [[TMP92:%.*]] = insertelement <4 x i16> [[TMP87]], i16 [[TMP91]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE42]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE42]]:
+; IC4VF4-NEXT:    [[TMP93:%.*]] = phi <4 x i16> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
+; IC4VF4-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_IF43]]:
+; IC4VF4-NEXT:    [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
+; IC4VF4-NEXT:    [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT:    [[TMP97:%.*]] = load i16, ptr [[TMP96]], align 1
+; IC4VF4-NEXT:    [[TMP98:%.*]] = insertelement <4 x i16> [[TMP93]], i16 [[TMP97]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE44]]:
+; IC4VF4-NEXT:    [[TMP99:%.*]] = phi <4 x i16> [ [[TMP93]], %[[PRED_LOAD_CONTINUE42]] ], [ [[TMP98]], %[[PRED_LOAD_IF43]] ]
+; IC4VF4-NEXT:    [[TMP100:%.*]] = icmp ugt <4 x i16> [[TMP27]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP101:%.*]] = icmp ugt <4 x i16> [[TMP51]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP102:%.*]] = icmp ugt <4 x i16> [[TMP75]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP103:%.*]] = icmp ugt <4 x i16> [[TMP99]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
+; IC4VF4-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
+; IC4VF4-NEXT:    [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
+; IC4VF4-NEXT:    [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
+; IC4VF4-NEXT:    [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
+; IC4VF4-NEXT:    br label %[[EXIT:.*]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
+; IC4VF1-SAME: i16 noundef [[VAL:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP3]], align 1
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load i16, ptr [[TMP4]], align 1
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load i16, ptr [[TMP5]], align 1
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load i16, ptr [[TMP6]], align 1
+; IC4VF1-NEXT:    [[TMP11:%.*]] = icmp ugt i16 [[TMP7]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP12:%.*]] = icmp ugt i16 [[TMP8]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP13:%.*]] = icmp ugt i16 [[TMP9]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP14:%.*]] = icmp ugt i16 [[TMP10]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP15:%.*]] = add nsw i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP16:%.*]] = add nsw i16 [[TMP0]], -1
+; IC4VF1-NEXT:    [[TMP17:%.*]] = add nsw i16 [[TMP1]], -1
+; IC4VF1-NEXT:    [[TMP18:%.*]] = add nsw i16 [[TMP2]], -1
+; IC4VF1-NEXT:    [[TMP19]] = select i1 [[TMP11]], i16 [[TMP15]], i16 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP20]] = select i1 [[TMP12]], i16 [[TMP16]], i16 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP21]] = select i1 [[TMP13]], i16 [[TMP17]], i16 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP22]] = select i1 [[TMP14]], i16 [[TMP18]], i16 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC4VF1-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP19]], i16 [[TMP20]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX]], i16 [[TMP21]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX4]], i16 [[TMP22]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[RDX_MINMAX5]], 32767
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -83,23 +573,354 @@ exit:                                             ; preds = %loop
 @tablef = constant [13 x half] [half 10.0, half 35.0, half 69.0, half 147.0, half 280.0, half 472.0, half 682.0, half 1013.0, half 1559.0, half 2544.0, half 4556.0, half 6496.0, half 10000.0], align 1
 
 define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
-; CHECK-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
-; CHECK-SAME: half noundef [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
-; CHECK-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
-; CHECK-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
-; CHECK-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
-; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT:.*]], label %[[LOOP]]
-; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ]
-; CHECK-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC1VF4-SAME: half noundef [[VAL:%.*]]) {
+; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC1VF4:       [[VECTOR_PH]]:
+; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
+; IC1VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x half> [[BROADCAST_SPLATINSERT]], <4 x half> poison, <4 x i32> zeroinitializer
+; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC1VF4:       [[VECTOR_BODY]]:
+; IC1VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; IC1VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC1VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC1VF4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC1VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds half, ptr [[TMP0]], i32 0
+; IC1VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 -3
+; IC1VF4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x half>, ptr [[TMP2]], align 1
+; IC1VF4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x half> [[WIDE_LOAD]], <4 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; IC1VF4-NEXT:    [[TMP3:%.*]] = fcmp ugt <4 x half> [[REVERSE]], [[BROADCAST_SPLAT]]
+; IC1VF4-NEXT:    [[TMP4:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC1VF4-NEXT:    [[TMP5]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP4]], <4 x i16> [[VEC_PHI]]
+; IC1VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC1VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC1VF4-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC1VF4-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC1VF4:       [[MIDDLE_BLOCK]]:
+; IC1VF4-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[TMP5]])
+; IC1VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP7]], 32767
+; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
+; IC1VF4-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC1VF4:       [[SCALAR_PH]]:
+; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC1VF4-NEXT:    br label %[[LOOP:.*]]
+; IC1VF4:       [[LOOP]]:
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC1VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC1VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC1VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC1VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC1VF4:       [[EXIT]]:
+; IC1VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC1VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC4VF4-SAME: half noundef [[VAL:%.*]]) {
+; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF4:       [[VECTOR_PH]]:
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x half> [[BROADCAST_SPLATINSERT]], <4 x half> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF4:       [[VECTOR_BODY]]:
+; IC4VF4-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44:.*]] ]
+; IC4VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 12, i16 11, i16 10, i16 9>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP108:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP109:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP110:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i16> [ splat (i16 32767), %[[VECTOR_PH]] ], [ [[TMP111:%.*]], %[[PRED_LOAD_CONTINUE44]] ]
+; IC4VF4-NEXT:    [[STEP_ADD:%.*]] = add <4 x i16> [[VEC_IND]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_2:%.*]] = add <4 x i16> [[STEP_ADD]], splat (i16 -4)
+; IC4VF4-NEXT:    [[STEP_ADD_3:%.*]] = add <4 x i16> [[STEP_ADD_2]], splat (i16 -4)
+; IC4VF4-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF4-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; IC4VF4-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> poison, <4 x i32> zeroinitializer
+; IC4VF4-NEXT:    [[VEC_IV:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 0, i32 1, i32 2, i32 3>
+; IC4VF4-NEXT:    [[VEC_IV8:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 4, i32 5, i32 6, i32 7>
+; IC4VF4-NEXT:    [[VEC_IV11:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 8, i32 9, i32 10, i32 11>
+; IC4VF4-NEXT:    [[VEC_IV14:%.*]] = add <4 x i32> [[BROADCAST_SPLAT5]], <i32 12, i32 13, i32 14, i32 15>
+; IC4VF4-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IV]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i32> [[VEC_IV8]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV11]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV14]], splat (i32 11)
+; IC4VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; IC4VF4:       [[PRED_LOAD_IF]]:
+; IC4VF4-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 0
+; IC4VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP5]]
+; IC4VF4-NEXT:    [[TMP7:%.*]] = load half, ptr [[TMP6]], align 1
+; IC4VF4-NEXT:    [[TMP8:%.*]] = insertelement <4 x half> poison, half [[TMP7]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE]]:
+; IC4VF4-NEXT:    [[TMP9:%.*]] = phi <4 x half> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
+; IC4VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF15:.*]], label %[[PRED_LOAD_CONTINUE16:.*]]
+; IC4VF4:       [[PRED_LOAD_IF15]]:
+; IC4VF4-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP11]]
+; IC4VF4-NEXT:    [[TMP13:%.*]] = load half, ptr [[TMP12]], align 1
+; IC4VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x half> [[TMP9]], half [[TMP13]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE16]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE16]]:
+; IC4VF4-NEXT:    [[TMP15:%.*]] = phi <4 x half> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF15]] ]
+; IC4VF4-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP16]], label %[[PRED_LOAD_IF17:.*]], label %[[PRED_LOAD_CONTINUE18:.*]]
+; IC4VF4:       [[PRED_LOAD_IF17]]:
+; IC4VF4-NEXT:    [[TMP17:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP17]]
+; IC4VF4-NEXT:    [[TMP19:%.*]] = load half, ptr [[TMP18]], align 1
+; IC4VF4-NEXT:    [[TMP20:%.*]] = insertelement <4 x half> [[TMP15]], half [[TMP19]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE18]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE18]]:
+; IC4VF4-NEXT:    [[TMP21:%.*]] = phi <4 x half> [ [[TMP15]], %[[PRED_LOAD_CONTINUE16]] ], [ [[TMP20]], %[[PRED_LOAD_IF17]] ]
+; IC4VF4-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP22]], label %[[PRED_LOAD_IF19:.*]], label %[[PRED_LOAD_CONTINUE20:.*]]
+; IC4VF4:       [[PRED_LOAD_IF19]]:
+; IC4VF4-NEXT:    [[TMP23:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF4-NEXT:    [[TMP24:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP23]]
+; IC4VF4-NEXT:    [[TMP25:%.*]] = load half, ptr [[TMP24]], align 1
+; IC4VF4-NEXT:    [[TMP26:%.*]] = insertelement <4 x half> [[TMP21]], half [[TMP25]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE20]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE20]]:
+; IC4VF4-NEXT:    [[TMP27:%.*]] = phi <4 x half> [ [[TMP21]], %[[PRED_LOAD_CONTINUE18]] ], [ [[TMP26]], %[[PRED_LOAD_IF19]] ]
+; IC4VF4-NEXT:    [[TMP28:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP28]], label %[[PRED_LOAD_IF21:.*]], label %[[PRED_LOAD_CONTINUE22:.*]]
+; IC4VF4:       [[PRED_LOAD_IF21]]:
+; IC4VF4-NEXT:    [[TMP29:%.*]] = add i16 [[OFFSET_IDX]], -4
+; IC4VF4-NEXT:    [[TMP30:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP29]]
+; IC4VF4-NEXT:    [[TMP31:%.*]] = load half, ptr [[TMP30]], align 1
+; IC4VF4-NEXT:    [[TMP32:%.*]] = insertelement <4 x half> poison, half [[TMP31]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE22]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE22]]:
+; IC4VF4-NEXT:    [[TMP33:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE20]] ], [ [[TMP32]], %[[PRED_LOAD_IF21]] ]
+; IC4VF4-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP34]], label %[[PRED_LOAD_IF23:.*]], label %[[PRED_LOAD_CONTINUE24:.*]]
+; IC4VF4:       [[PRED_LOAD_IF23]]:
+; IC4VF4-NEXT:    [[TMP35:%.*]] = add i16 [[OFFSET_IDX]], -5
+; IC4VF4-NEXT:    [[TMP36:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP35]]
+; IC4VF4-NEXT:    [[TMP37:%.*]] = load half, ptr [[TMP36]], align 1
+; IC4VF4-NEXT:    [[TMP38:%.*]] = insertelement <4 x half> [[TMP33]], half [[TMP37]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE24]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE24]]:
+; IC4VF4-NEXT:    [[TMP39:%.*]] = phi <4 x half> [ [[TMP33]], %[[PRED_LOAD_CONTINUE22]] ], [ [[TMP38]], %[[PRED_LOAD_IF23]] ]
+; IC4VF4-NEXT:    [[TMP40:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP40]], label %[[PRED_LOAD_IF25:.*]], label %[[PRED_LOAD_CONTINUE26:.*]]
+; IC4VF4:       [[PRED_LOAD_IF25]]:
+; IC4VF4-NEXT:    [[TMP41:%.*]] = add i16 [[OFFSET_IDX]], -6
+; IC4VF4-NEXT:    [[TMP42:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP41]]
+; IC4VF4-NEXT:    [[TMP43:%.*]] = load half, ptr [[TMP42]], align 1
+; IC4VF4-NEXT:    [[TMP44:%.*]] = insertelement <4 x half> [[TMP39]], half [[TMP43]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE26]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE26]]:
+; IC4VF4-NEXT:    [[TMP45:%.*]] = phi <4 x half> [ [[TMP39]], %[[PRED_LOAD_CONTINUE24]] ], [ [[TMP44]], %[[PRED_LOAD_IF25]] ]
+; IC4VF4-NEXT:    [[TMP46:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP46]], label %[[PRED_LOAD_IF27:.*]], label %[[PRED_LOAD_CONTINUE28:.*]]
+; IC4VF4:       [[PRED_LOAD_IF27]]:
+; IC4VF4-NEXT:    [[TMP47:%.*]] = add i16 [[OFFSET_IDX]], -7
+; IC4VF4-NEXT:    [[TMP48:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP47]]
+; IC4VF4-NEXT:    [[TMP49:%.*]] = load half, ptr [[TMP48]], align 1
+; IC4VF4-NEXT:    [[TMP50:%.*]] = insertelement <4 x half> [[TMP45]], half [[TMP49]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE28]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE28]]:
+; IC4VF4-NEXT:    [[TMP51:%.*]] = phi <4 x half> [ [[TMP45]], %[[PRED_LOAD_CONTINUE26]] ], [ [[TMP50]], %[[PRED_LOAD_IF27]] ]
+; IC4VF4-NEXT:    [[TMP52:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP52]], label %[[PRED_LOAD_IF29:.*]], label %[[PRED_LOAD_CONTINUE30:.*]]
+; IC4VF4:       [[PRED_LOAD_IF29]]:
+; IC4VF4-NEXT:    [[TMP53:%.*]] = add i16 [[OFFSET_IDX]], -8
+; IC4VF4-NEXT:    [[TMP54:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP53]]
+; IC4VF4-NEXT:    [[TMP55:%.*]] = load half, ptr [[TMP54]], align 1
+; IC4VF4-NEXT:    [[TMP56:%.*]] = insertelement <4 x half> poison, half [[TMP55]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE30]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE30]]:
+; IC4VF4-NEXT:    [[TMP57:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE28]] ], [ [[TMP56]], %[[PRED_LOAD_IF29]] ]
+; IC4VF4-NEXT:    [[TMP58:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP58]], label %[[PRED_LOAD_IF31:.*]], label %[[PRED_LOAD_CONTINUE32:.*]]
+; IC4VF4:       [[PRED_LOAD_IF31]]:
+; IC4VF4-NEXT:    [[TMP59:%.*]] = add i16 [[OFFSET_IDX]], -9
+; IC4VF4-NEXT:    [[TMP60:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP59]]
+; IC4VF4-NEXT:    [[TMP61:%.*]] = load half, ptr [[TMP60]], align 1
+; IC4VF4-NEXT:    [[TMP62:%.*]] = insertelement <4 x half> [[TMP57]], half [[TMP61]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE32]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE32]]:
+; IC4VF4-NEXT:    [[TMP63:%.*]] = phi <4 x half> [ [[TMP57]], %[[PRED_LOAD_CONTINUE30]] ], [ [[TMP62]], %[[PRED_LOAD_IF31]] ]
+; IC4VF4-NEXT:    [[TMP64:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP64]], label %[[PRED_LOAD_IF33:.*]], label %[[PRED_LOAD_CONTINUE34:.*]]
+; IC4VF4:       [[PRED_LOAD_IF33]]:
+; IC4VF4-NEXT:    [[TMP65:%.*]] = add i16 [[OFFSET_IDX]], -10
+; IC4VF4-NEXT:    [[TMP66:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP65]]
+; IC4VF4-NEXT:    [[TMP67:%.*]] = load half, ptr [[TMP66]], align 1
+; IC4VF4-NEXT:    [[TMP68:%.*]] = insertelement <4 x half> [[TMP63]], half [[TMP67]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE34]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE34]]:
+; IC4VF4-NEXT:    [[TMP69:%.*]] = phi <4 x half> [ [[TMP63]], %[[PRED_LOAD_CONTINUE32]] ], [ [[TMP68]], %[[PRED_LOAD_IF33]] ]
+; IC4VF4-NEXT:    [[TMP70:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP70]], label %[[PRED_LOAD_IF35:.*]], label %[[PRED_LOAD_CONTINUE36:.*]]
+; IC4VF4:       [[PRED_LOAD_IF35]]:
+; IC4VF4-NEXT:    [[TMP71:%.*]] = add i16 [[OFFSET_IDX]], -11
+; IC4VF4-NEXT:    [[TMP72:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP71]]
+; IC4VF4-NEXT:    [[TMP73:%.*]] = load half, ptr [[TMP72]], align 1
+; IC4VF4-NEXT:    [[TMP74:%.*]] = insertelement <4 x half> [[TMP69]], half [[TMP73]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE36]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE36]]:
+; IC4VF4-NEXT:    [[TMP75:%.*]] = phi <4 x half> [ [[TMP69]], %[[PRED_LOAD_CONTINUE34]] ], [ [[TMP74]], %[[PRED_LOAD_IF35]] ]
+; IC4VF4-NEXT:    [[TMP76:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
+; IC4VF4-NEXT:    br i1 [[TMP76]], label %[[PRED_LOAD_IF37:.*]], label %[[PRED_LOAD_CONTINUE38:.*]]
+; IC4VF4:       [[PRED_LOAD_IF37]]:
+; IC4VF4-NEXT:    [[TMP77:%.*]] = add i16 [[OFFSET_IDX]], -12
+; IC4VF4-NEXT:    [[TMP78:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP77]]
+; IC4VF4-NEXT:    [[TMP79:%.*]] = load half, ptr [[TMP78]], align 1
+; IC4VF4-NEXT:    [[TMP80:%.*]] = insertelement <4 x half> poison, half [[TMP79]], i32 0
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE38]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE38]]:
+; IC4VF4-NEXT:    [[TMP81:%.*]] = phi <4 x half> [ poison, %[[PRED_LOAD_CONTINUE36]] ], [ [[TMP80]], %[[PRED_LOAD_IF37]] ]
+; IC4VF4-NEXT:    [[TMP82:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1
+; IC4VF4-NEXT:    br i1 [[TMP82]], label %[[PRED_LOAD_IF39:.*]], label %[[PRED_LOAD_CONTINUE40:.*]]
+; IC4VF4:       [[PRED_LOAD_IF39]]:
+; IC4VF4-NEXT:    [[TMP83:%.*]] = add i16 [[OFFSET_IDX]], -13
+; IC4VF4-NEXT:    [[TMP84:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP83]]
+; IC4VF4-NEXT:    [[TMP85:%.*]] = load half, ptr [[TMP84]], align 1
+; IC4VF4-NEXT:    [[TMP86:%.*]] = insertelement <4 x half> [[TMP81]], half [[TMP85]], i32 1
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE40]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE40]]:
+; IC4VF4-NEXT:    [[TMP87:%.*]] = phi <4 x half> [ [[TMP81]], %[[PRED_LOAD_CONTINUE38]] ], [ [[TMP86]], %[[PRED_LOAD_IF39]] ]
+; IC4VF4-NEXT:    [[TMP88:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2
+; IC4VF4-NEXT:    br i1 [[TMP88]], label %[[PRED_LOAD_IF41:.*]], label %[[PRED_LOAD_CONTINUE42:.*]]
+; IC4VF4:       [[PRED_LOAD_IF41]]:
+; IC4VF4-NEXT:    [[TMP89:%.*]] = add i16 [[OFFSET_IDX]], -14
+; IC4VF4-NEXT:    [[TMP90:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP89]]
+; IC4VF4-NEXT:    [[TMP91:%.*]] = load half, ptr [[TMP90]], align 1
+; IC4VF4-NEXT:    [[TMP92:%.*]] = insertelement <4 x half> [[TMP87]], half [[TMP91]], i32 2
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE42]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE42]]:
+; IC4VF4-NEXT:    [[TMP93:%.*]] = phi <4 x half> [ [[TMP87]], %[[PRED_LOAD_CONTINUE40]] ], [ [[TMP92]], %[[PRED_LOAD_IF41]] ]
+; IC4VF4-NEXT:    [[TMP94:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
+; IC4VF4-NEXT:    br i1 [[TMP94]], label %[[PRED_LOAD_IF43:.*]], label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_IF43]]:
+; IC4VF4-NEXT:    [[TMP95:%.*]] = add i16 [[OFFSET_IDX]], -15
+; IC4VF4-NEXT:    [[TMP96:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP95]]
+; IC4VF4-NEXT:    [[TMP97:%.*]] = load half, ptr [[TMP96]], align 1
+; IC4VF4-NEXT:    [[TMP98:%.*]] = insertelement <4 x half> [[TMP93]], half [[TMP97]], i32 3
+; IC4VF4-NEXT:    br label %[[PRED_LOAD_CONTINUE44]]
+; IC4VF4:       [[PRED_LOAD_CONTINUE44]]:
+; IC4VF4-NEXT:    [[TMP99:%.*]] = phi <4 x half> [ [[TMP93]], %[[PRED_LOAD_CONTINUE42]] ], [ [[TMP98]], %[[PRED_LOAD_IF43]] ]
+; IC4VF4-NEXT:    [[TMP100:%.*]] = fcmp ugt <4 x half> [[TMP27]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP101:%.*]] = fcmp ugt <4 x half> [[TMP51]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP102:%.*]] = fcmp ugt <4 x half> [[TMP75]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP103:%.*]] = fcmp ugt <4 x half> [[TMP99]], [[BROADCAST_SPLAT]]
+; IC4VF4-NEXT:    [[TMP104:%.*]] = add nsw <4 x i16> [[VEC_IND]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP105:%.*]] = add nsw <4 x i16> [[STEP_ADD]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP106:%.*]] = add nsw <4 x i16> [[STEP_ADD_2]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP107:%.*]] = add nsw <4 x i16> [[STEP_ADD_3]], splat (i16 -1)
+; IC4VF4-NEXT:    [[TMP108]] = select <4 x i1> [[TMP100]], <4 x i16> [[TMP104]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP109]] = select <4 x i1> [[TMP101]], <4 x i16> [[TMP105]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP110]] = select <4 x i1> [[TMP102]], <4 x i16> [[TMP106]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP111]] = select <4 x i1> [[TMP103]], <4 x i16> [[TMP107]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[TMP112:%.*]] = select <4 x i1> [[TMP0]], <4 x i16> [[TMP108]], <4 x i16> [[VEC_PHI]]
+; IC4VF4-NEXT:    [[TMP113:%.*]] = select <4 x i1> [[TMP1]], <4 x i16> [[TMP109]], <4 x i16> [[VEC_PHI1]]
+; IC4VF4-NEXT:    [[TMP114:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> [[TMP110]], <4 x i16> [[VEC_PHI2]]
+; IC4VF4-NEXT:    [[TMP115:%.*]] = select <4 x i1> [[TMP3]], <4 x i16> [[TMP111]], <4 x i16> [[VEC_PHI3]]
+; IC4VF4-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; IC4VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD_3]], splat (i16 -4)
+; IC4VF4-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC4VF4:       [[MIDDLE_BLOCK]]:
+; IC4VF4-NEXT:    [[RDX_MINMAX:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[TMP112]], <4 x i16> [[TMP113]])
+; IC4VF4-NEXT:    [[RDX_MINMAX45:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX]], <4 x i16> [[TMP114]])
+; IC4VF4-NEXT:    [[RDX_MINMAX46:%.*]] = call <4 x i16> @llvm.smin.v4i16(<4 x i16> [[RDX_MINMAX45]], <4 x i16> [[TMP115]])
+; IC4VF4-NEXT:    [[TMP116:%.*]] = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> [[RDX_MINMAX46]])
+; IC4VF4-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[TMP116]], 32767
+; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
+; IC4VF4-NEXT:    br label %[[EXIT:.*]]
+; IC4VF4:       [[SCALAR_PH]]:
+; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
+; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
+; IC4VF4-NEXT:    br label %[[LOOP:.*]]
+; IC4VF4:       [[LOOP]]:
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC4VF4-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF4-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF4-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF4-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC4VF4:       [[EXIT]]:
+; IC4VF4-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF4-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
+;
+; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
+; IC4VF1-SAME: half noundef [[VAL:%.*]]) {
+; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4VF1:       [[VECTOR_PH]]:
+; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4VF1:       [[VECTOR_BODY]]:
+; IC4VF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP19:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI1:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI2:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[VEC_PHI3:%.*]] = phi i16 [ 32767, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
+; IC4VF1-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
+; IC4VF1-NEXT:    [[OFFSET_IDX:%.*]] = sub i16 12, [[DOTCAST]]
+; IC4VF1-NEXT:    [[TMP0:%.*]] = add i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], -2
+; IC4VF1-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], -3
+; IC4VF1-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[OFFSET_IDX]]
+; IC4VF1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP0]]
+; IC4VF1-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP1]]
+; IC4VF1-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[TMP2]]
+; IC4VF1-NEXT:    [[TMP7:%.*]] = load half, ptr [[TMP3]], align 1
+; IC4VF1-NEXT:    [[TMP8:%.*]] = load half, ptr [[TMP4]], align 1
+; IC4VF1-NEXT:    [[TMP9:%.*]] = load half, ptr [[TMP5]], align 1
+; IC4VF1-NEXT:    [[TMP10:%.*]] = load half, ptr [[TMP6]], align 1
+; IC4VF1-NEXT:    [[TMP11:%.*]] = fcmp ugt half [[TMP7]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP12:%.*]] = fcmp ugt half [[TMP8]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP13:%.*]] = fcmp ugt half [[TMP9]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP14:%.*]] = fcmp ugt half [[TMP10]], [[VAL]]
+; IC4VF1-NEXT:    [[TMP15:%.*]] = add nsw i16 [[OFFSET_IDX]], -1
+; IC4VF1-NEXT:    [[TMP16:%.*]] = add nsw i16 [[TMP0]], -1
+; IC4VF1-NEXT:    [[TMP17:%.*]] = add nsw i16 [[TMP1]], -1
+; IC4VF1-NEXT:    [[TMP18:%.*]] = add nsw i16 [[TMP2]], -1
+; IC4VF1-NEXT:    [[TMP19]] = select i1 [[TMP11]], i16 [[TMP15]], i16 [[VEC_PHI]]
+; IC4VF1-NEXT:    [[TMP20]] = select i1 [[TMP12]], i16 [[TMP16]], i16 [[VEC_PHI1]]
+; IC4VF1-NEXT:    [[TMP21]] = select i1 [[TMP13]], i16 [[TMP17]], i16 [[VEC_PHI2]]
+; IC4VF1-NEXT:    [[TMP22]] = select i1 [[TMP14]], i16 [[TMP18]], i16 [[VEC_PHI3]]
+; IC4VF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; IC4VF1-NEXT:    [[TMP23:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; IC4VF1-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IC4VF1:       [[MIDDLE_BLOCK]]:
+; IC4VF1-NEXT:    [[RDX_MINMAX:%.*]] = call i16 @llvm.smin.i16(i16 [[TMP19]], i16 [[TMP20]])
+; IC4VF1-NEXT:    [[RDX_MINMAX4:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX]], i16 [[TMP21]])
+; IC4VF1-NEXT:    [[RDX_MINMAX5:%.*]] = call i16 @llvm.smin.i16(i16 [[RDX_MINMAX4]], i16 [[TMP22]])
+; IC4VF1-NEXT:    [[RDX_SELECT_CMP:%.*]] = icmp ne i16 [[RDX_MINMAX5]], 32767
+; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
+; IC4VF1-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4VF1:       [[SCALAR_PH]]:
+; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 12, %[[ENTRY]] ]
+; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4VF1-NEXT:    br label %[[LOOP:.*]]
+; IC4VF1:       [[LOOP]]:
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
+; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
+; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
+; IC4VF1-NEXT:    [[IV_NEXT]] = add nsw i16 [[IV]], -1
+; IC4VF1-NEXT:    [[SPEC_SELECT]] = select i1 [[CMP_TABLE_VAL]], i16 [[IV_NEXT]], i16 [[RDX]]
+; IC4VF1-NEXT:    [[EXIT_COND:%.*]] = icmp eq i16 [[IV_NEXT]], 0
+; IC4VF1-NEXT:    br i1 [[EXIT_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; IC4VF1:       [[EXIT]]:
+; IC4VF1-NEXT:    [[SPEC_SELECT_LCSSA:%.*]] = phi i16 [ [[SPEC_SELECT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; IC4VF1-NEXT:    ret i16 [[SPEC_SELECT_LCSSA]]
 ;
 entry:
   br label %loop
@@ -199,4 +1020,3 @@ loop:                                             ; preds = %entry, %loop
 exit:                                             ; preds = %loop
   ret i64 %spec.select
 }
-
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index f801443b85d3..7b7735434e67 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -1101,7 +1101,6 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
 ; VEC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[TMP0]]
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0
 ; VEC-NEXT:    store <2 x i16> zeroinitializer, ptr [[TMP2]], align 2
-; VEC-NEXT:    [[TMP4:%.*]] = add i32 [[STEP_2]], [[TMP0]]
 ; VEC-NEXT:    [[TMP5:%.*]] = add i32 [[STEP_2]], [[TMP6]]
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; VEC-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
@@ -1293,8 +1292,6 @@ define i32 @iv_ext_used_outside( ptr %dst) {
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP1]], i32 0
 ; VEC-NEXT:    store <2 x i32> zeroinitializer, ptr [[TMP2]], align 4
 ; VEC-NEXT:    [[TMP5:%.*]] = add nuw nsw <2 x i16> [[VEC_IND]], splat (i16 1)
-; VEC-NEXT:    [[TMP3:%.*]] = extractelement <2 x i16> [[TMP5]], i32 0
-; VEC-NEXT:    [[TMP4:%.*]] = zext nneg i16 [[TMP3]] to i32
 ; VEC-NEXT:    [[TMP8:%.*]] = extractelement <2 x i16> [[TMP5]], i32 1
 ; VEC-NEXT:    [[TMP7:%.*]] = zext nneg i16 [[TMP8]] to i32
 ; VEC-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -1389,9 +1386,7 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
 ; VEC-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[TMP0]], i32 0
 ; VEC-NEXT:    [[TMP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 -1
 ; VEC-NEXT:    store <2 x i16> splat (i16 1), ptr [[TMP2]], align 2
-; VEC-NEXT:    [[TMP3:%.*]] = add i64 2, -1
 ; VEC-NEXT:    [[TMP5:%.*]] = add i64 1, -1
-; VEC-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], 1
 ; VEC-NEXT:    [[TMP6:%.*]] = add i64 [[TMP5]], 1
 ; VEC-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; VEC:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index 7a33fd092e29..8a326c9d0c08 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -669,6 +669,8 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_SRC]], i64 0, i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <2 x i32> poison, i32 [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP12]], i32 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll b/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
index 0e753a535cd2..94af5b7c7607 100644
--- a/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/multi_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/multi_early_exit_live_outs.ll
index 4027f6a0f5df..7759c10032e9 100644
--- a/llvm/test/Transforms/LoopVectorize/multi_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/multi_early_exit_live_outs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
new file mode 100644
index 000000000000..6a17b5f61795
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/narrow-to-single-scalar.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+define void @narrow_select_to_single_scalar(i1 %invar.cond, ptr noalias %A, ptr noalias %B, ptr noalias %C) {
+; CHECK-LABEL: define void @narrow_select_to_single_scalar(
+; CHECK-SAME: i1 [[INVAR_COND:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[INVAR_COND]], i16 0, i16 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i16, ptr [[C]], i16 [[TMP0]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = add i16 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = add i16 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr [[TMP6]], align 1
+; CHECK-NEXT:    store i16 [[TMP7]], ptr [[B]], align 1
+; CHECK-NEXT:    store i16 0, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 1024, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i16 [[IV]]
+; CHECK-NEXT:    [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 1
+; CHECK-NEXT:    store i16 [[L_0]], ptr [[B]], align 1
+; CHECK-NEXT:    [[INVAR_SEL:%.*]] = select i1 [[INVAR_COND]], i16 0, i16 1
+; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr i16, ptr [[C]], i16 [[INVAR_SEL]]
+; CHECK-NEXT:    store i16 0, ptr [[GEP_C]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i16 [[IV]], 1024
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i16 [ 0, %entry ], [ %iv.next, %loop.header ]
+  %gep.A = getelementptr i8, ptr %A, i16 %iv
+  %l.0 = load i16, ptr %gep.A, align 1
+  store i16 %l.0, ptr %B, align 1
+  %invar.sel = select i1 %invar.cond, i16 0, i16 1
+  %gep.C = getelementptr i16, ptr %C, i16 %invar.sel
+  store i16 0, ptr %gep.C, align 1
+  %iv.next = add i16 %iv, 1
+  %ec = icmp ne i16 %iv, 1024
+  br i1 %ec, label %loop.header, label %exit
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 796c1d116aa1..13cc1b657d23 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
-; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
-; CHECK-NEXT:    [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
-; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
new file mode 100644
index 000000000000..2747895f06a7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
@@ -0,0 +1,198 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -S %s | FileCheck %s
+
+declare noalias noundef ptr @calloc(i64 noundef, i64 noundef)
+
+define void @reuse_lcssa_phi_for_add_rec1(ptr %head) {
+; CHECK-LABEL: define void @reuse_lcssa_phi_for_add_rec1(
+; CHECK-SAME: ptr [[HEAD:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi ptr [ [[HEAD]], %[[ENTRY]] ], [ [[L_1:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[L_1]] = load ptr, ptr [[FOR]], align 8
+; CHECK-NEXT:    [[IV_2_NEXT]] = add nuw nsw i32 [[IV_2]], 1
+; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq ptr [[L_1]], null
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
+; CHECK-NEXT:    br i1 [[EC_1]], label %[[PH:.*]], label %[[LOOP_1]]
+; CHECK:       [[PH]]:
+; CHECK-NEXT:    [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_2_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SRC_2:%.*]] = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8)
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[IV_2_LCSSA]], 1
+; CHECK-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 1)
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[IV_LCSSA]], [[N_VEC]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[IV_LCSSA]], [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[SRC_2]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr ptr, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr ptr, ptr [[TMP6]], i32 -1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x ptr>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <2 x ptr> [[WIDE_LOAD]], <2 x ptr> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x ptr> [[REVERSE]], i32 0
+; CHECK-NEXT:    store ptr null, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x ptr> [[REVERSE]], i32 1
+; CHECK-NEXT:    store ptr null, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[IV_LCSSA]], %[[PH]] ]
+; CHECK-NEXT:    br label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[IV_3:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_3:%.*]], %[[LOOP_2]] ]
+; CHECK-NEXT:    [[IV_NEXT_3]] = add nsw i64 [[IV_3]], -1
+; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr ptr, ptr [[SRC_2]], i64 [[IV_3]]
+; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[GEP_SRC_2]], align 8
+; CHECK-NEXT:    store ptr null, ptr [[L]], align 8
+; CHECK-NEXT:    [[T:%.*]] = trunc nuw i64 [[IV_3]] to i32
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp sgt i32 [[T]], 1
+; CHECK-NEXT:    br i1 [[EC_2]], label %[[LOOP_2]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.1 ]
+  %iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop.1 ]
+  %for = phi ptr [ %head, %entry ], [ %l.1, %loop.1 ]
+  %l.1 = load ptr, ptr %for, align 8
+  %iv.2.next = add nuw nsw i32 %iv.2, 1
+  %ec.1 = icmp eq ptr %l.1, null
+  %iv.next = add nuw i64 %iv, 1
+  br i1 %ec.1, label %ph, label %loop.1
+
+ph:
+  %iv.lcssa = phi i64 [ %iv, %loop.1 ]
+  %iv.2.next.lcssa = phi i32 [ %iv.2.next, %loop.1 ]
+  %src.2 = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8)
+  br label %loop.2
+
+loop.2:
+  %iv.3 = phi i64 [ %iv.lcssa, %ph ], [ %iv.next.3, %loop.2 ]
+  %iv.next.3 = add nsw i64 %iv.3, -1
+  %gep.src.2 = getelementptr ptr, ptr %src.2, i64 %iv.3
+  %l = load ptr, ptr %gep.src.2, align 8
+  store ptr null, ptr %l, align 8
+  %t = trunc nuw i64 %iv.3 to i32
+  %ec.2 = icmp sgt i32 %t, 1
+  br i1 %ec.2, label %loop.2, label %for.end
+
+for.end:
+  ret void
+}
+
+declare i32 @val()
+
+define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) {
+; CHECK-LABEL: define void @runtime_checks_ptr_inductions(
+; CHECK-SAME: ptr [[DST_1:%.*]], ptr [[DST_2:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[DST_11:%.*]] = ptrtoint ptr [[DST_1]] to i64
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[PTR_IV_1:%.*]] = phi ptr [ [[DST_1]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @val()
+; CHECK-NEXT:    [[SEL_DST:%.*]] = select i1 [[C]], ptr [[DST_1]], ptr [[DST_2]]
+; CHECK-NEXT:    [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1
+; CHECK-NEXT:    [[EC_1:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
+; CHECK-NEXT:    br i1 [[EC_1]], label %[[LOOP_2_HEADER_PREHEADER:.*]], label %[[LOOP_1]]
+; CHECK:       [[LOOP_2_HEADER_PREHEADER]]:
+; CHECK-NEXT:    [[SEL_DST_LCSSA2:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SEL_DST_LCSSA:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[SEL_DST_LCSSA23:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA2]] to i64
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDVAR_LCSSA]], [[DST_11]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA23]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR_IV_1_LCSSA]], i64 1022
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[SEL_DST_LCSSA]], i64 1022
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_1_LCSSA]], i64 [[INDEX]]
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[SEL_DST_LCSSA]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP4]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; CHECK-NEXT:    store <2 x i8> [[WIDE_LOAD]], ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1022
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1023, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_2_HEADER_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP_2_HEADER:.*]]
+; CHECK:       [[LOOP_2_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[DEC7:%.*]], %[[LOOP_2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[EC_2:%.*]] = icmp eq i32 [[IV]], 1024
+; CHECK-NEXT:    br i1 [[EC_2]], label %[[EXIT:.*]], label %[[LOOP_2_LATCH]]
+; CHECK:       [[LOOP_2_LATCH]]:
+; CHECK-NEXT:    [[DEC7]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[PTR_IV_3_NEXT]] = getelementptr i8, ptr [[PTR_IV_3]], i64 1
+; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[PTR_IV_3]], align 1
+; CHECK-NEXT:    [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 1
+; CHECK-NEXT:    store i8 [[L]], ptr [[PTR_IV_2]], align 1
+; CHECK-NEXT:    br label %[[LOOP_2_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %ptr.iv.1 = phi ptr [ %dst.1, %entry ], [ %ptr.iv.1.next, %loop.1 ]
+  %call = call i32 @val()
+  %sel.dst = select i1 %c, ptr %dst.1, ptr %dst.2
+  %ptr.iv.1.next = getelementptr i8, ptr %ptr.iv.1, i64 1
+  %ec.1 = icmp eq i32 %call, 0
+  br i1 %ec.1, label %loop.2.header, label %loop.1
+
+loop.2.header:
+  %iv = phi i32 [ 1, %loop.1 ], [ %dec7, %loop.2.latch ]
+  %ptr.iv.2 = phi ptr [ %ptr.iv.1, %loop.1 ], [ %ptr.iv.2.next, %loop.2.latch ]
+  %ptr.iv.3 = phi ptr [ %sel.dst, %loop.1 ], [ %ptr.iv.3.next, %loop.2.latch ]
+  %ec.2 = icmp eq i32 %iv, 1024
+  br i1 %ec.2, label %exit, label %loop.2.latch
+
+loop.2.latch:
+  %dec7 = add i32 %iv, 1
+  %ptr.iv.3.next = getelementptr i8, ptr %ptr.iv.3, i64 1
+  %l = load i8, ptr %ptr.iv.3, align 1
+  %ptr.iv.2.next = getelementptr i8, ptr %ptr.iv.2, i64 1
+  store i8 %l, ptr %ptr.iv.2, align 1
+  br label %loop.2.header
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index 079f6b73e886..e901d9801a14 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
 ; CHECK-NEXT:    [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
 ; CHECK-NEXT:    [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
new file mode 100644
index 000000000000..0fe893abec86
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 | FileCheck %s
+
+define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size(ptr noalias %p1, ptr noalias %p2) nofree nosync {
+; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK:       loop.inc:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK:       loop.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
+;
+entry:
+  call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 1024) ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 1024) ]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 1024
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ]
+  ret i64 %retval
+}
+
+define i64 @early_exit_alignment_and_deref_via_assumption_with_constant_size_too_small(ptr noalias %p1, ptr noalias %p2) nofree nosync {
+; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_via_assumption_with_constant_size_too_small(
+; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]]
+; CHECK:       loop.inc:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1025
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]
+; CHECK:       loop.end:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ]
+; CHECK-NEXT:    ret i64 [[RETVAL]]
+;
+entry:
+  call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 1024) ]
+  call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 1024) ]
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+  %ld1 = load i8, ptr %arrayidx, align 1
+  %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+  %ld2 = load i8, ptr %arrayidx1, align 1
+  %cmp3 = icmp eq i8 %ld1, %ld2
+  br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 1025
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ]
+  ret i64 %retval
+}
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
new file mode 100644
index 000000000000..de8a3c5a8eaf
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; REQUIRES: asserts
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 \
+; RUN:   -debug-only=loop-vectorize -S %s 2>%t | FileCheck --check-prefix=VF4IC4 %s
+; RUN: cat %t | FileCheck --check-prefix=DEBUG %s
+
+declare void @init_mem(ptr, i64);
+
+; DEBUG: Interleaving not supported for loops with uncountable early exits
+
+define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
+; VF4IC4-LABEL: define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
+; VF4IC4-NEXT:  [[ENTRY:.*]]:
+; VF4IC4-NEXT:    [[SRC:%.*]] = alloca [128 x i32], align 4
+; VF4IC4-NEXT:    call void @init_mem(ptr [[SRC]])
+; VF4IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4IC4:       [[VECTOR_PH]]:
+; VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4IC4:       [[VECTOR_BODY]]:
+; VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
+; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; VF4IC4-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10)
+; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; VF4IC4-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; VF4IC4-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4IC4:       [[MIDDLE_SPLIT]]:
+; VF4IC4-NEXT:    br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; VF4IC4:       [[MIDDLE_BLOCK]]:
+; VF4IC4-NEXT:    br i1 true, label %[[E2:.*]], label %[[SCALAR_PH]]
+; VF4IC4:       [[VECTOR_EARLY_EXIT]]:
+; VF4IC4-NEXT:    br label %[[E1:.*]]
+; VF4IC4:       [[SCALAR_PH]]:
+; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 128, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; VF4IC4-NEXT:    br label %[[LOOP_HEADER:.*]]
+; VF4IC4:       [[LOOP_HEADER]]:
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; VF4IC4-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; VF4IC4-NEXT:    [[C_1:%.*]] = icmp eq i32 [[L]], 10
+; VF4IC4-NEXT:    br i1 [[C_1]], label %[[E1]], label %[[LOOP_LATCH]]
+; VF4IC4:       [[LOOP_LATCH]]:
+; VF4IC4-NEXT:    [[INC]] = add nuw i64 [[IV]], 1
+; VF4IC4-NEXT:    [[C_2:%.*]] = icmp eq i64 [[INC]], 128
+; VF4IC4-NEXT:    br i1 [[C_2]], label %[[E2]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF4IC4:       [[E1]]:
+; VF4IC4-NEXT:    [[P1:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ]
+; VF4IC4-NEXT:    ret i64 [[P1]]
+; VF4IC4:       [[E2]]:
+; VF4IC4-NEXT:    [[P2:%.*]] = phi i64 [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; VF4IC4-NEXT:    ret i64 [[P2]]
+;
+entry:
+  %src = alloca [128 x i32]
+  call void @init_mem(ptr %src)
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ %inc, %loop.latch ], [ 0, %entry ]
+  %gep.src = getelementptr inbounds i32, ptr %src, i64 %iv
+  %l = load i32, ptr %gep.src
+  %c.1 = icmp eq i32 %l, 10
+  br i1 %c.1, label %e1, label %loop.latch
+
+loop.latch:
+  %inc = add nuw i64 %iv, 1
+  %c.2 = icmp eq i64 %inc, 128
+  br i1 %c.2, label %e2, label %loop.header, !llvm.loop !0
+
+e1:
+  %p1 = phi i64 [ 0, %loop.header ]
+  ret i64 %p1
+
+e2:
+  %p2 = phi i64 [ 1, %loop.latch ]
+  ret i64 %p2
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 4}
+;.
+; VF4IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; VF4IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; VF4IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; VF4IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
index 1f8cfa1bfd11..6dc17e0993a2 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
@@ -1,9 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4
-; RUN: opt -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -force-vector-interleave=4 -S %s | FileCheck --check-prefix=VF4IC4 %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=4 -S %s | FileCheck --check-prefix=VF4IC4 %s
 
 declare void @init_mem(ptr, i64);
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4-LABEL: define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4-NEXT:  entry:
@@ -16,10 +15,22 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP14]], align 4
 ; VF4IC4-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10)
+; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD1]], splat (i32 10)
+; VF4IC4-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD2]], splat (i32 10)
+; VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10)
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; VF4IC4-NEXT:    [[TMP9:%.*]] = or <4 x i1> [[TMP2]], [[TMP6]]
+; VF4IC4-NEXT:    [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP7]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = or <4 x i1> [[TMP10]], [[TMP8]]
+; VF4IC4-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
 ; VF4IC4-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
 ; VF4IC4-NEXT:    br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -75,7 +86,6 @@ e2:
   ret i64 %p2
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @same_exit_block_pre_inc_use1() {
 ; VF4IC4-LABEL: define i64 @same_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:  entry:
@@ -91,13 +101,31 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 0
+; VF4IC4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 4
+; VF4IC4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 8
+; VF4IC4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 12
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP20]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP22]], align 1
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD5]]
+; VF4IC4-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; VF4IC4-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; VF4IC4-NEXT:    [[TMP14:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; VF4IC4-NEXT:    [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP12]]
+; VF4IC4-NEXT:    [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP13]]
+; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
 ; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
 ; VF4IC4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
@@ -106,7 +134,20 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP33:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; VF4IC4-NEXT:    [[TMP34:%.*]] = add i64 12, [[TMP33]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 true)
+; VF4IC4-NEXT:    [[TMP24:%.*]] = add i64 8, [[TMP35]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = icmp ne i64 [[TMP35]], 4
+; VF4IC4-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i64 [[TMP24]], i64 [[TMP34]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP11]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 4, [[TMP26]]
+; VF4IC4-NEXT:    [[TMP27:%.*]] = icmp ne i64 [[TMP26]], 4
+; VF4IC4-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i64 [[TMP28]], i64 [[TMP25]]
+; VF4IC4-NEXT:    [[TMP30:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP32:%.*]] = add i64 0, [[TMP30]]
+; VF4IC4-NEXT:    [[TMP31:%.*]] = icmp ne i64 [[TMP30]], 4
+; VF4IC4-NEXT:    [[TMP8:%.*]] = select i1 [[TMP31]], i64 [[TMP32]], i64 [[TMP29]]
 ; VF4IC4-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
@@ -155,7 +196,6 @@ loop.end:
   ret i64 %retval
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; VF4IC4-LABEL: define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; VF4IC4-NEXT:  entry:
@@ -170,10 +210,22 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4IC4-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
+; VF4IC4-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 4
+; VF4IC4-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 8
+; VF4IC4-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP12]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP14]], align 1
 ; VF4IC4-NEXT:    [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], splat (i8 72)
+; VF4IC4-NEXT:    [[TMP15:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD1]], splat (i8 72)
+; VF4IC4-NEXT:    [[TMP16:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], splat (i8 72)
+; VF4IC4-NEXT:    [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], splat (i8 72)
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; VF4IC4-NEXT:    [[TMP9:%.*]] = or <4 x i1> [[TMP2]], [[TMP15]]
+; VF4IC4-NEXT:    [[TMP10:%.*]] = or <4 x i1> [[TMP9]], [[TMP16]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = or <4 x i1> [[TMP10]], [[TMP17]]
+; VF4IC4-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; VF4IC4-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
 ; VF4IC4-NEXT:    br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -182,7 +234,20 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 true)
+; VF4IC4-NEXT:    [[TMP29:%.*]] = add i64 12, [[TMP28]]
+; VF4IC4-NEXT:    [[TMP30:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 true)
+; VF4IC4-NEXT:    [[TMP19:%.*]] = add i64 8, [[TMP30]]
+; VF4IC4-NEXT:    [[TMP18:%.*]] = icmp ne i64 [[TMP30]], 4
+; VF4IC4-NEXT:    [[TMP20:%.*]] = select i1 [[TMP18]], i64 [[TMP19]], i64 [[TMP29]]
+; VF4IC4-NEXT:    [[TMP21:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP15]], i1 true)
+; VF4IC4-NEXT:    [[TMP23:%.*]] = add i64 4, [[TMP21]]
+; VF4IC4-NEXT:    [[TMP22:%.*]] = icmp ne i64 [[TMP21]], 4
+; VF4IC4-NEXT:    [[TMP24:%.*]] = select i1 [[TMP22]], i64 [[TMP23]], i64 [[TMP20]]
+; VF4IC4-NEXT:    [[TMP25:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true)
+; VF4IC4-NEXT:    [[TMP27:%.*]] = add i64 0, [[TMP25]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = icmp ne i64 [[TMP25]], 4
+; VF4IC4-NEXT:    [[TMP6:%.*]] = select i1 [[TMP26]], i64 [[TMP27]], i64 [[TMP24]]
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], [[TMP6]]
 ; VF4IC4-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP7]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
@@ -224,7 +289,6 @@ loop.end:
   ret ptr %retval
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @same_exit_block_post_inc_use() {
 ; VF4IC4-LABEL: define i64 @same_exit_block_post_inc_use() {
 ; VF4IC4-NEXT:  entry:
@@ -240,13 +304,31 @@ define i64 @same_exit_block_post_inc_use() {
 ; VF4IC4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 0
+; VF4IC4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 4
+; VF4IC4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 8
+; VF4IC4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 12
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP20]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP22]], align 1
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD5]]
+; VF4IC4-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; VF4IC4-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; VF4IC4-NEXT:    [[TMP14:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; VF4IC4-NEXT:    [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP12]]
+; VF4IC4-NEXT:    [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP13]]
+; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
 ; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
 ; VF4IC4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -255,7 +337,20 @@ define i64 @same_exit_block_post_inc_use() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP33:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; VF4IC4-NEXT:    [[TMP34:%.*]] = add i64 12, [[TMP33]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 true)
+; VF4IC4-NEXT:    [[TMP24:%.*]] = add i64 8, [[TMP35]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = icmp ne i64 [[TMP35]], 4
+; VF4IC4-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i64 [[TMP24]], i64 [[TMP34]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP11]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 4, [[TMP26]]
+; VF4IC4-NEXT:    [[TMP27:%.*]] = icmp ne i64 [[TMP26]], 4
+; VF4IC4-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i64 [[TMP28]], i64 [[TMP25]]
+; VF4IC4-NEXT:    [[TMP30:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP32:%.*]] = add i64 0, [[TMP30]]
+; VF4IC4-NEXT:    [[TMP31:%.*]] = icmp ne i64 [[TMP30]], 4
+; VF4IC4-NEXT:    [[TMP8:%.*]] = select i1 [[TMP31]], i64 [[TMP32]], i64 [[TMP29]]
 ; VF4IC4-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
@@ -304,7 +399,6 @@ loop.end:
   ret i64 %retval
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @diff_exit_block_pre_inc_use1() {
 ; VF4IC4-LABEL: define i64 @diff_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:  entry:
@@ -320,13 +414,31 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 0
+; VF4IC4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 4
+; VF4IC4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 8
+; VF4IC4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 12
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP20]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP22]], align 1
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD5]]
+; VF4IC4-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; VF4IC4-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; VF4IC4-NEXT:    [[TMP14:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; VF4IC4-NEXT:    [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP12]]
+; VF4IC4-NEXT:    [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP13]]
+; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
 ; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
 ; VF4IC4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -335,7 +447,20 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP33:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; VF4IC4-NEXT:    [[TMP34:%.*]] = add i64 12, [[TMP33]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 true)
+; VF4IC4-NEXT:    [[TMP24:%.*]] = add i64 8, [[TMP35]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = icmp ne i64 [[TMP35]], 4
+; VF4IC4-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i64 [[TMP24]], i64 [[TMP34]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP11]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 4, [[TMP26]]
+; VF4IC4-NEXT:    [[TMP27:%.*]] = icmp ne i64 [[TMP26]], 4
+; VF4IC4-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i64 [[TMP28]], i64 [[TMP25]]
+; VF4IC4-NEXT:    [[TMP30:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP32:%.*]] = add i64 0, [[TMP30]]
+; VF4IC4-NEXT:    [[TMP31:%.*]] = icmp ne i64 [[TMP30]], 4
+; VF4IC4-NEXT:    [[TMP8:%.*]] = select i1 [[TMP31]], i64 [[TMP32]], i64 [[TMP29]]
 ; VF4IC4-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
@@ -391,7 +516,6 @@ loop.end:
   ret i64 %retval2
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @diff_exit_block_post_inc_use1() {
 ; VF4IC4-LABEL: define i64 @diff_exit_block_post_inc_use1() {
 ; VF4IC4-NEXT:  entry:
@@ -407,13 +531,31 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; VF4IC4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 0
+; VF4IC4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 4
+; VF4IC4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 8
+; VF4IC4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 12
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP20]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP21]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP22]], align 1
 ; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD5]]
+; VF4IC4-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; VF4IC4-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; VF4IC4-NEXT:    [[TMP14:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; VF4IC4-NEXT:    [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP12]]
+; VF4IC4-NEXT:    [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP13]]
+; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
 ; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
 ; VF4IC4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
@@ -422,7 +564,20 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP33:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; VF4IC4-NEXT:    [[TMP34:%.*]] = add i64 12, [[TMP33]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 true)
+; VF4IC4-NEXT:    [[TMP24:%.*]] = add i64 8, [[TMP35]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = icmp ne i64 [[TMP35]], 4
+; VF4IC4-NEXT:    [[TMP25:%.*]] = select i1 [[TMP23]], i64 [[TMP24]], i64 [[TMP34]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP11]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 4, [[TMP26]]
+; VF4IC4-NEXT:    [[TMP27:%.*]] = icmp ne i64 [[TMP26]], 4
+; VF4IC4-NEXT:    [[TMP29:%.*]] = select i1 [[TMP27]], i64 [[TMP28]], i64 [[TMP25]]
+; VF4IC4-NEXT:    [[TMP30:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP32:%.*]] = add i64 0, [[TMP30]]
+; VF4IC4-NEXT:    [[TMP31:%.*]] = icmp ne i64 [[TMP30]], 4
+; VF4IC4-NEXT:    [[TMP8:%.*]] = select i1 [[TMP31]], i64 [[TMP32]], i64 [[TMP29]]
 ; VF4IC4-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
@@ -478,7 +633,6 @@ loop.end:
   ret i64 %retval2
 }
 
-; FIXME: Computes incorrect final IV when early exit is taken.
 define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; VF4IC4-LABEL: define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; VF4IC4-NEXT:  entry:
@@ -495,16 +649,46 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
 ; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 -3
-; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
-; VF4IC4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
-; VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -3
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -4
+; VF4IC4-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 -3
+; VF4IC4-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -8
+; VF4IC4-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 -3
+; VF4IC4-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -12
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 -3
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
 ; VF4IC4-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD1]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp ne <4 x i8> [[REVERSE]], [[REVERSE2]]
+; VF4IC4-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
+; VF4IC4-NEXT:    [[REVERSE10:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD9]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD11:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
+; VF4IC4-NEXT:    [[REVERSE12:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD11]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[REVERSE14:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD13]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP25]], i32 0
+; VF4IC4-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP26]], i32 -3
+; VF4IC4-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP25]], i32 -4
+; VF4IC4-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP28]], i32 -3
+; VF4IC4-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[TMP25]], i32 -8
+; VF4IC4-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i8, ptr [[TMP30]], i32 -3
+; VF4IC4-NEXT:    [[TMP45:%.*]] = getelementptr inbounds i8, ptr [[TMP25]], i32 -12
+; VF4IC4-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i8, ptr [[TMP45]], i32 -3
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP27]], align 1
+; VF4IC4-NEXT:    [[REVERSE8:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD7]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1
+; VF4IC4-NEXT:    [[REVERSE11:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD10]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i8>, ptr [[TMP44]], align 1
+; VF4IC4-NEXT:    [[REVERSE13:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD12]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x i8>, ptr [[TMP46]], align 1
+; VF4IC4-NEXT:    [[REVERSE15:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD14]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp ne <4 x i8> [[REVERSE2]], [[REVERSE8]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i8> [[REVERSE10]], [[REVERSE11]]
+; VF4IC4-NEXT:    [[TMP20:%.*]] = icmp ne <4 x i8> [[REVERSE12]], [[REVERSE13]]
+; VF4IC4-NEXT:    [[TMP21:%.*]] = icmp ne <4 x i8> [[REVERSE14]], [[REVERSE15]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; VF4IC4-NEXT:    [[TMP22:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = or <4 x i1> [[TMP22]], [[TMP20]]
+; VF4IC4-NEXT:    [[TMP24:%.*]] = or <4 x i1> [[TMP23]], [[TMP21]]
+; VF4IC4-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
 ; VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1008
 ; VF4IC4-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
 ; VF4IC4-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
@@ -513,7 +697,20 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 false, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; VF4IC4-NEXT:    [[TMP41:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP21]], i1 true)
+; VF4IC4-NEXT:    [[TMP42:%.*]] = add i64 12, [[TMP41]]
+; VF4IC4-NEXT:    [[TMP43:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP20]], i1 true)
+; VF4IC4-NEXT:    [[TMP32:%.*]] = add i64 8, [[TMP43]]
+; VF4IC4-NEXT:    [[TMP31:%.*]] = icmp ne i64 [[TMP43]], 4
+; VF4IC4-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i64 [[TMP32]], i64 [[TMP42]]
+; VF4IC4-NEXT:    [[TMP34:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP19]], i1 true)
+; VF4IC4-NEXT:    [[TMP36:%.*]] = add i64 4, [[TMP34]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = icmp ne i64 [[TMP34]], 4
+; VF4IC4-NEXT:    [[TMP37:%.*]] = select i1 [[TMP35]], i64 [[TMP36]], i64 [[TMP33]]
+; VF4IC4-NEXT:    [[TMP38:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; VF4IC4-NEXT:    [[TMP40:%.*]] = add i64 0, [[TMP38]]
+; VF4IC4-NEXT:    [[TMP39:%.*]] = icmp ne i64 [[TMP38]], 4
+; VF4IC4-NEXT:    [[TMP10:%.*]] = select i1 [[TMP39]], i64 [[TMP40]], i64 [[TMP37]]
 ; VF4IC4-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
 ; VF4IC4-NEXT:    [[TMP12:%.*]] = sub i64 1023, [[TMP11]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
@@ -577,13 +774,31 @@ define i8 @same_exit_block_use_loaded_value() {
 ; VF4IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
+; VF4IC4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 4
+; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 8
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 12
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
-; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
-; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 0
+; VF4IC4-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 4
+; VF4IC4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 8
+; VF4IC4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 12
+; VF4IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP29]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP8]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
+; VF4IC4-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD4]]
+; VF4IC4-NEXT:    [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
+; VF4IC4-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; VF4IC4-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; VF4IC4-NEXT:    [[TMP14:%.*]] = or <4 x i1> [[TMP4]], [[TMP11]]
+; VF4IC4-NEXT:    [[TMP15:%.*]] = or <4 x i1> [[TMP14]], [[TMP12]]
+; VF4IC4-NEXT:    [[TMP16:%.*]] = or <4 x i1> [[TMP15]], [[TMP13]]
+; VF4IC4-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
 ; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; VF4IC4-NEXT:    [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]
 ; VF4IC4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
@@ -592,7 +807,20 @@ define i8 @same_exit_block_use_loaded_value() {
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE1:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 true)
+; VF4IC4-NEXT:    [[TMP20:%.*]] = add i64 12, [[FIRST_ACTIVE_LANE1]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 true)
+; VF4IC4-NEXT:    [[TMP22:%.*]] = add i64 8, [[FIRST_ACTIVE_LANE8]]
+; VF4IC4-NEXT:    [[TMP21:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE8]], 4
+; VF4IC4-NEXT:    [[TMP23:%.*]] = select i1 [[TMP21]], i64 [[TMP22]], i64 [[TMP20]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP11]], i1 true)
+; VF4IC4-NEXT:    [[TMP25:%.*]] = add i64 4, [[FIRST_ACTIVE_LANE9]]
+; VF4IC4-NEXT:    [[TMP24:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE9]], 4
+; VF4IC4-NEXT:    [[TMP26:%.*]] = select i1 [[TMP24]], i64 [[TMP25]], i64 [[TMP23]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE10]]
+; VF4IC4-NEXT:    [[TMP27:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE10]], 4
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = select i1 [[TMP27]], i64 [[TMP28]], i64 [[TMP26]]
 ; VF4IC4-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
@@ -657,25 +885,68 @@ define i8 @same_exit_block_reverse_use_loaded_value() {
 ; VF4IC4-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; VF4IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0
 ; VF4IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 -3
+; VF4IC4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -4
+; VF4IC4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 -3
+; VF4IC4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -8
+; VF4IC4-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[TMP18]], i32 -3
+; VF4IC4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 -12
+; VF4IC4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 -3
 ; VF4IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
 ; VF4IC4-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; VF4IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; VF4IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP3]], i32 0
-; VF4IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 -3
-; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1
+; VF4IC4-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1
 ; VF4IC4-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD1]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp ne <4 x i8> [[REVERSE]], [[REVERSE2]]
+; VF4IC4-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP37]], align 1
+; VF4IC4-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD3]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i8>, ptr [[TMP8]], align 1
+; VF4IC4-NEXT:    [[REVERSE6:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD5]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
+; VF4IC4-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0
+; VF4IC4-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i8, ptr [[TMP38]], i32 -3
+; VF4IC4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 -4
+; VF4IC4-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 -3
+; VF4IC4-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 -8
+; VF4IC4-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 -3
+; VF4IC4-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 -12
+; VF4IC4-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP16]], i32 -3
+; VF4IC4-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP39]], align 1
+; VF4IC4-NEXT:    [[REVERSE8:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD7]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1
+; VF4IC4-NEXT:    [[REVERSE10:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD9]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD11:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1
+; VF4IC4-NEXT:    [[REVERSE12:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD11]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1
+; VF4IC4-NEXT:    [[REVERSE14:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD13]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF4IC4-NEXT:    [[TMP6:%.*]] = icmp ne <4 x i8> [[REVERSE]], [[REVERSE8]]
+; VF4IC4-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i8> [[REVERSE2]], [[REVERSE10]]
+; VF4IC4-NEXT:    [[TMP20:%.*]] = icmp ne <4 x i8> [[REVERSE4]], [[REVERSE12]]
+; VF4IC4-NEXT:    [[TMP21:%.*]] = icmp ne <4 x i8> [[REVERSE6]], [[REVERSE14]]
 ; VF4IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; VF4IC4-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
-; VF4IC4-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1008
-; VF4IC4-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
-; VF4IC4-NEXT:    br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; VF4IC4-NEXT:    [[TMP22:%.*]] = or <4 x i1> [[TMP6]], [[TMP19]]
+; VF4IC4-NEXT:    [[TMP23:%.*]] = or <4 x i1> [[TMP22]], [[TMP20]]
+; VF4IC4-NEXT:    [[TMP24:%.*]] = or <4 x i1> [[TMP23]], [[TMP21]]
+; VF4IC4-NEXT:    [[TMP25:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP24]])
+; VF4IC4-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1008
+; VF4IC4-NEXT:    [[TMP27:%.*]] = or i1 [[TMP25]], [[TMP26]]
+; VF4IC4-NEXT:    br i1 [[TMP27]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; VF4IC4:       middle.split:
-; VF4IC4-NEXT:    br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; VF4IC4-NEXT:    br i1 [[TMP25]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
 ; VF4IC4:       middle.block:
 ; VF4IC4-NEXT:    br i1 false, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
 ; VF4IC4:       vector.early.exit:
-; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE1:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP21]], i1 true)
+; VF4IC4-NEXT:    [[TMP28:%.*]] = add i64 12, [[FIRST_ACTIVE_LANE1]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP20]], i1 true)
+; VF4IC4-NEXT:    [[TMP30:%.*]] = add i64 8, [[FIRST_ACTIVE_LANE15]]
+; VF4IC4-NEXT:    [[TMP29:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE15]], 4
+; VF4IC4-NEXT:    [[TMP31:%.*]] = select i1 [[TMP29]], i64 [[TMP30]], i64 [[TMP28]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE16:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP19]], i1 true)
+; VF4IC4-NEXT:    [[TMP33:%.*]] = add i64 4, [[FIRST_ACTIVE_LANE16]]
+; VF4IC4-NEXT:    [[TMP32:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE16]], 4
+; VF4IC4-NEXT:    [[TMP34:%.*]] = select i1 [[TMP32]], i64 [[TMP33]], i64 [[TMP31]]
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE17:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
+; VF4IC4-NEXT:    [[TMP36:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE17]]
+; VF4IC4-NEXT:    [[TMP35:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE17]], 4
+; VF4IC4-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = select i1 [[TMP35]], i64 [[TMP36]], i64 [[TMP34]]
 ; VF4IC4-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = extractelement <4 x i8> [[REVERSE]], i64 [[FIRST_ACTIVE_LANE]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
index 00e21f102038..2069570d4a10 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
index ec1e8fa1e1b3..a21666a31b6a 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_with_outer_loop.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_with_outer_loop.ll
index 1aa63550e031..5c1bde0a156c 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_with_outer_loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_with_outer_loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S < %s -passes='loop-vectorize,verify<loops>,print<loops>' -disable-output -enable-early-exit-vectorization 2>&1 | FileCheck %s
+; RUN: opt -S < %s -passes='loop-vectorize,verify<loops>,print<loops>' -disable-output 2>&1 | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
index fe53334cb25a..717d1f9ae6fd 100644
--- a/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll
@@ -23,19 +23,19 @@ define void @struct_return_1xi64_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 3
 ; VF4-NEXT:    [[TMP9:%.*]] = tail call { i64 } @fn1(float [[TMP8]]) #[[ATTR0]]
 ; VF4-NEXT:    [[TMP10:%.*]] = extractvalue { i64 } [[TMP3]], 0
-; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
+; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
 ; VF4-NEXT:    [[TMP12:%.*]] = insertvalue { <4 x i64> } poison, <4 x i64> [[TMP11]], 0
 ; VF4-NEXT:    [[TMP13:%.*]] = extractvalue { i64 } [[TMP5]], 0
 ; VF4-NEXT:    [[TMP14:%.*]] = extractvalue { <4 x i64> } [[TMP12]], 0
-; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP13]], i32 1
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP13]], i64 1
 ; VF4-NEXT:    [[TMP16:%.*]] = insertvalue { <4 x i64> } [[TMP12]], <4 x i64> [[TMP15]], 0
 ; VF4-NEXT:    [[TMP17:%.*]] = extractvalue { i64 } [[TMP7]], 0
 ; VF4-NEXT:    [[TMP18:%.*]] = extractvalue { <4 x i64> } [[TMP16]], 0
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP17]], i32 2
+; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP17]], i64 2
 ; VF4-NEXT:    [[TMP20:%.*]] = insertvalue { <4 x i64> } [[TMP16]], <4 x i64> [[TMP19]], 0
 ; VF4-NEXT:    [[TMP21:%.*]] = extractvalue { i64 } [[TMP9]], 0
 ; VF4-NEXT:    [[TMP22:%.*]] = extractvalue { <4 x i64> } [[TMP20]], 0
-; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP21]], i32 3
+; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP21]], i64 3
 ; VF4-NEXT:    [[TMP24:%.*]] = insertvalue { <4 x i64> } [[TMP20]], <4 x i64> [[TMP23]], 0
 ; VF4-NEXT:    [[TMP25:%.*]] = extractvalue { <4 x i64> } [[TMP24]], 0
 ; VF4-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[OUT_A]], i64 [[INDEX]]
@@ -64,22 +64,22 @@ define void @struct_return_1xi64_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF2IC2-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1
 ; VF2IC2-NEXT:    [[TMP6:%.*]] = tail call { i64 } @fn1(float [[TMP5]]) #[[ATTR0]]
 ; VF2IC2-NEXT:    [[TMP7:%.*]] = extractvalue { i64 } [[TMP4]], 0
-; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i64 0
 ; VF2IC2-NEXT:    [[TMP9:%.*]] = insertvalue { <2 x i64> } poison, <2 x i64> [[TMP8]], 0
 ; VF2IC2-NEXT:    [[TMP10:%.*]] = extractvalue { i64 } [[TMP6]], 0
 ; VF2IC2-NEXT:    [[TMP11:%.*]] = extractvalue { <2 x i64> } [[TMP9]], 0
-; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i32 1
+; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x i64> [[TMP11]], i64 [[TMP10]], i64 1
 ; VF2IC2-NEXT:    [[TMP13:%.*]] = insertvalue { <2 x i64> } [[TMP9]], <2 x i64> [[TMP12]], 0
 ; VF2IC2-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 0
 ; VF2IC2-NEXT:    [[TMP15:%.*]] = tail call { i64 } @fn1(float [[TMP14]]) #[[ATTR0]]
 ; VF2IC2-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 1
 ; VF2IC2-NEXT:    [[TMP17:%.*]] = tail call { i64 } @fn1(float [[TMP16]]) #[[ATTR0]]
 ; VF2IC2-NEXT:    [[TMP18:%.*]] = extractvalue { i64 } [[TMP15]], 0
-; VF2IC2-NEXT:    [[TMP19:%.*]] = insertelement <2 x i64> poison, i64 [[TMP18]], i32 0
+; VF2IC2-NEXT:    [[TMP19:%.*]] = insertelement <2 x i64> poison, i64 [[TMP18]], i64 0
 ; VF2IC2-NEXT:    [[TMP20:%.*]] = insertvalue { <2 x i64> } poison, <2 x i64> [[TMP19]], 0
 ; VF2IC2-NEXT:    [[TMP21:%.*]] = extractvalue { i64 } [[TMP17]], 0
 ; VF2IC2-NEXT:    [[TMP22:%.*]] = extractvalue { <2 x i64> } [[TMP20]], 0
-; VF2IC2-NEXT:    [[TMP23:%.*]] = insertelement <2 x i64> [[TMP22]], i64 [[TMP21]], i32 1
+; VF2IC2-NEXT:    [[TMP23:%.*]] = insertelement <2 x i64> [[TMP22]], i64 [[TMP21]], i64 1
 ; VF2IC2-NEXT:    [[TMP24:%.*]] = insertvalue { <2 x i64> } [[TMP20]], <2 x i64> [[TMP23]], 0
 ; VF2IC2-NEXT:    [[TMP25:%.*]] = extractvalue { <2 x i64> } [[TMP13]], 0
 ; VF2IC2-NEXT:    [[TMP26:%.*]] = extractvalue { <2 x i64> } [[TMP24]], 0
@@ -133,35 +133,35 @@ define void @struct_return_2xf32_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 3
 ; VF4-NEXT:    [[TMP9:%.*]] = tail call { float, float } @fn2(float [[TMP8]]) #[[ATTR1]]
 ; VF4-NEXT:    [[TMP10:%.*]] = extractvalue { float, float } [[TMP3]], 0
-; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> poison, float [[TMP10]], i32 0
+; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> poison, float [[TMP10]], i64 0
 ; VF4-NEXT:    [[TMP12:%.*]] = insertvalue { <4 x float>, <4 x float> } poison, <4 x float> [[TMP11]], 0
 ; VF4-NEXT:    [[TMP13:%.*]] = extractvalue { float, float } [[TMP3]], 1
 ; VF4-NEXT:    [[TMP14:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP12]], 1
-; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP13]], i32 0
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP13]], i64 0
 ; VF4-NEXT:    [[TMP16:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP12]], <4 x float> [[TMP15]], 1
 ; VF4-NEXT:    [[TMP17:%.*]] = extractvalue { float, float } [[TMP5]], 0
 ; VF4-NEXT:    [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP16]], 0
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP17]], i32 1
+; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP17]], i64 1
 ; VF4-NEXT:    [[TMP20:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP16]], <4 x float> [[TMP19]], 0
 ; VF4-NEXT:    [[TMP21:%.*]] = extractvalue { float, float } [[TMP5]], 1
 ; VF4-NEXT:    [[TMP22:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP20]], 1
-; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x float> [[TMP22]], float [[TMP21]], i32 1
+; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x float> [[TMP22]], float [[TMP21]], i64 1
 ; VF4-NEXT:    [[TMP24:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP20]], <4 x float> [[TMP23]], 1
 ; VF4-NEXT:    [[TMP25:%.*]] = extractvalue { float, float } [[TMP7]], 0
 ; VF4-NEXT:    [[TMP26:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP24]], 0
-; VF4-NEXT:    [[TMP27:%.*]] = insertelement <4 x float> [[TMP26]], float [[TMP25]], i32 2
+; VF4-NEXT:    [[TMP27:%.*]] = insertelement <4 x float> [[TMP26]], float [[TMP25]], i64 2
 ; VF4-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP24]], <4 x float> [[TMP27]], 0
 ; VF4-NEXT:    [[TMP29:%.*]] = extractvalue { float, float } [[TMP7]], 1
 ; VF4-NEXT:    [[TMP30:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP28]], 1
-; VF4-NEXT:    [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP29]], i32 2
+; VF4-NEXT:    [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP29]], i64 2
 ; VF4-NEXT:    [[TMP32:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP28]], <4 x float> [[TMP31]], 1
 ; VF4-NEXT:    [[TMP33:%.*]] = extractvalue { float, float } [[TMP9]], 0
 ; VF4-NEXT:    [[TMP34:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP32]], 0
-; VF4-NEXT:    [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP33]], i32 3
+; VF4-NEXT:    [[TMP35:%.*]] = insertelement <4 x float> [[TMP34]], float [[TMP33]], i64 3
 ; VF4-NEXT:    [[TMP36:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP32]], <4 x float> [[TMP35]], 0
 ; VF4-NEXT:    [[TMP37:%.*]] = extractvalue { float, float } [[TMP9]], 1
 ; VF4-NEXT:    [[TMP38:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP36]], 1
-; VF4-NEXT:    [[TMP39:%.*]] = insertelement <4 x float> [[TMP38]], float [[TMP37]], i32 3
+; VF4-NEXT:    [[TMP39:%.*]] = insertelement <4 x float> [[TMP38]], float [[TMP37]], i64 3
 ; VF4-NEXT:    [[TMP40:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP36]], <4 x float> [[TMP39]], 1
 ; VF4-NEXT:    [[TMP41:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP40]], 0
 ; VF4-NEXT:    [[TMP42:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP40]], 1
@@ -194,38 +194,38 @@ define void @struct_return_2xf32_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF2IC2-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1
 ; VF2IC2-NEXT:    [[TMP6:%.*]] = tail call { float, float } @fn2(float [[TMP5]]) #[[ATTR1]]
 ; VF2IC2-NEXT:    [[TMP7:%.*]] = extractvalue { float, float } [[TMP4]], 0
-; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i32 0
+; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> poison, float [[TMP7]], i64 0
 ; VF2IC2-NEXT:    [[TMP9:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP8]], 0
 ; VF2IC2-NEXT:    [[TMP10:%.*]] = extractvalue { float, float } [[TMP4]], 1
 ; VF2IC2-NEXT:    [[TMP11:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP9]], 1
-; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP10]], i32 0
+; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP10]], i64 0
 ; VF2IC2-NEXT:    [[TMP13:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP9]], <2 x float> [[TMP12]], 1
 ; VF2IC2-NEXT:    [[TMP14:%.*]] = extractvalue { float, float } [[TMP6]], 0
 ; VF2IC2-NEXT:    [[TMP15:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP13]], 0
-; VF2IC2-NEXT:    [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i32 1
+; VF2IC2-NEXT:    [[TMP16:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP14]], i64 1
 ; VF2IC2-NEXT:    [[TMP17:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP13]], <2 x float> [[TMP16]], 0
 ; VF2IC2-NEXT:    [[TMP18:%.*]] = extractvalue { float, float } [[TMP6]], 1
 ; VF2IC2-NEXT:    [[TMP19:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP17]], 1
-; VF2IC2-NEXT:    [[TMP20:%.*]] = insertelement <2 x float> [[TMP19]], float [[TMP18]], i32 1
+; VF2IC2-NEXT:    [[TMP20:%.*]] = insertelement <2 x float> [[TMP19]], float [[TMP18]], i64 1
 ; VF2IC2-NEXT:    [[TMP21:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP17]], <2 x float> [[TMP20]], 1
 ; VF2IC2-NEXT:    [[TMP22:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 0
 ; VF2IC2-NEXT:    [[TMP23:%.*]] = tail call { float, float } @fn2(float [[TMP22]]) #[[ATTR1]]
 ; VF2IC2-NEXT:    [[TMP24:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 1
 ; VF2IC2-NEXT:    [[TMP25:%.*]] = tail call { float, float } @fn2(float [[TMP24]]) #[[ATTR1]]
 ; VF2IC2-NEXT:    [[TMP26:%.*]] = extractvalue { float, float } [[TMP23]], 0
-; VF2IC2-NEXT:    [[TMP27:%.*]] = insertelement <2 x float> poison, float [[TMP26]], i32 0
+; VF2IC2-NEXT:    [[TMP27:%.*]] = insertelement <2 x float> poison, float [[TMP26]], i64 0
 ; VF2IC2-NEXT:    [[TMP28:%.*]] = insertvalue { <2 x float>, <2 x float> } poison, <2 x float> [[TMP27]], 0
 ; VF2IC2-NEXT:    [[TMP29:%.*]] = extractvalue { float, float } [[TMP23]], 1
 ; VF2IC2-NEXT:    [[TMP30:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP28]], 1
-; VF2IC2-NEXT:    [[TMP31:%.*]] = insertelement <2 x float> [[TMP30]], float [[TMP29]], i32 0
+; VF2IC2-NEXT:    [[TMP31:%.*]] = insertelement <2 x float> [[TMP30]], float [[TMP29]], i64 0
 ; VF2IC2-NEXT:    [[TMP32:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP28]], <2 x float> [[TMP31]], 1
 ; VF2IC2-NEXT:    [[TMP33:%.*]] = extractvalue { float, float } [[TMP25]], 0
 ; VF2IC2-NEXT:    [[TMP34:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP32]], 0
-; VF2IC2-NEXT:    [[TMP35:%.*]] = insertelement <2 x float> [[TMP34]], float [[TMP33]], i32 1
+; VF2IC2-NEXT:    [[TMP35:%.*]] = insertelement <2 x float> [[TMP34]], float [[TMP33]], i64 1
 ; VF2IC2-NEXT:    [[TMP36:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP32]], <2 x float> [[TMP35]], 0
 ; VF2IC2-NEXT:    [[TMP37:%.*]] = extractvalue { float, float } [[TMP25]], 1
 ; VF2IC2-NEXT:    [[TMP38:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP36]], 1
-; VF2IC2-NEXT:    [[TMP39:%.*]] = insertelement <2 x float> [[TMP38]], float [[TMP37]], i32 1
+; VF2IC2-NEXT:    [[TMP39:%.*]] = insertelement <2 x float> [[TMP38]], float [[TMP37]], i64 1
 ; VF2IC2-NEXT:    [[TMP40:%.*]] = insertvalue { <2 x float>, <2 x float> } [[TMP36]], <2 x float> [[TMP39]], 1
 ; VF2IC2-NEXT:    [[TMP41:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP21]], 0
 ; VF2IC2-NEXT:    [[TMP42:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP40]], 0
@@ -290,51 +290,51 @@ define void @struct_return_3xi32_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[WIDE_LOAD]], i32 3
 ; VF4-NEXT:    [[TMP9:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP8]]) #[[ATTR2]]
 ; VF4-NEXT:    [[TMP10:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 0
-; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
+; VF4-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i64 0
 ; VF4-NEXT:    [[TMP12:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP11]], 0
 ; VF4-NEXT:    [[TMP13:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 1
 ; VF4-NEXT:    [[TMP14:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP12]], 1
-; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP13]], i32 0
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP13]], i64 0
 ; VF4-NEXT:    [[TMP16:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP12]], <4 x i32> [[TMP15]], 1
 ; VF4-NEXT:    [[TMP17:%.*]] = extractvalue { i32, i32, i32 } [[TMP3]], 2
 ; VF4-NEXT:    [[TMP18:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP16]], 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP17]], i32 0
+; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP17]], i64 0
 ; VF4-NEXT:    [[TMP20:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP16]], <4 x i32> [[TMP19]], 2
 ; VF4-NEXT:    [[TMP21:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 0
 ; VF4-NEXT:    [[TMP22:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP20]], 0
-; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP21]], i32 1
+; VF4-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP22]], i32 [[TMP21]], i64 1
 ; VF4-NEXT:    [[TMP24:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP20]], <4 x i32> [[TMP23]], 0
 ; VF4-NEXT:    [[TMP25:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 1
 ; VF4-NEXT:    [[TMP26:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP24]], 1
-; VF4-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP25]], i32 1
+; VF4-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP25]], i64 1
 ; VF4-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP24]], <4 x i32> [[TMP27]], 1
 ; VF4-NEXT:    [[TMP29:%.*]] = extractvalue { i32, i32, i32 } [[TMP5]], 2
 ; VF4-NEXT:    [[TMP30:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP28]], 2
-; VF4-NEXT:    [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP29]], i32 1
+; VF4-NEXT:    [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP29]], i64 1
 ; VF4-NEXT:    [[TMP32:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP28]], <4 x i32> [[TMP31]], 2
 ; VF4-NEXT:    [[TMP33:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 0
 ; VF4-NEXT:    [[TMP34:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP32]], 0
-; VF4-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP33]], i32 2
+; VF4-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP33]], i64 2
 ; VF4-NEXT:    [[TMP36:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP32]], <4 x i32> [[TMP35]], 0
 ; VF4-NEXT:    [[TMP37:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 1
 ; VF4-NEXT:    [[TMP38:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP36]], 1
-; VF4-NEXT:    [[TMP39:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP37]], i32 2
+; VF4-NEXT:    [[TMP39:%.*]] = insertelement <4 x i32> [[TMP38]], i32 [[TMP37]], i64 2
 ; VF4-NEXT:    [[TMP40:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP36]], <4 x i32> [[TMP39]], 1
 ; VF4-NEXT:    [[TMP41:%.*]] = extractvalue { i32, i32, i32 } [[TMP7]], 2
 ; VF4-NEXT:    [[TMP42:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP40]], 2
-; VF4-NEXT:    [[TMP43:%.*]] = insertelement <4 x i32> [[TMP42]], i32 [[TMP41]], i32 2
+; VF4-NEXT:    [[TMP43:%.*]] = insertelement <4 x i32> [[TMP42]], i32 [[TMP41]], i64 2
 ; VF4-NEXT:    [[TMP44:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP40]], <4 x i32> [[TMP43]], 2
 ; VF4-NEXT:    [[TMP45:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 0
 ; VF4-NEXT:    [[TMP46:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP44]], 0
-; VF4-NEXT:    [[TMP47:%.*]] = insertelement <4 x i32> [[TMP46]], i32 [[TMP45]], i32 3
+; VF4-NEXT:    [[TMP47:%.*]] = insertelement <4 x i32> [[TMP46]], i32 [[TMP45]], i64 3
 ; VF4-NEXT:    [[TMP48:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP44]], <4 x i32> [[TMP47]], 0
 ; VF4-NEXT:    [[TMP49:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 1
 ; VF4-NEXT:    [[TMP50:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP48]], 1
-; VF4-NEXT:    [[TMP51:%.*]] = insertelement <4 x i32> [[TMP50]], i32 [[TMP49]], i32 3
+; VF4-NEXT:    [[TMP51:%.*]] = insertelement <4 x i32> [[TMP50]], i32 [[TMP49]], i64 3
 ; VF4-NEXT:    [[TMP52:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP48]], <4 x i32> [[TMP51]], 1
 ; VF4-NEXT:    [[TMP53:%.*]] = extractvalue { i32, i32, i32 } [[TMP9]], 2
 ; VF4-NEXT:    [[TMP54:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP52]], 2
-; VF4-NEXT:    [[TMP55:%.*]] = insertelement <4 x i32> [[TMP54]], i32 [[TMP53]], i32 3
+; VF4-NEXT:    [[TMP55:%.*]] = insertelement <4 x i32> [[TMP54]], i32 [[TMP53]], i64 3
 ; VF4-NEXT:    [[TMP56:%.*]] = insertvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP52]], <4 x i32> [[TMP55]], 2
 ; VF4-NEXT:    [[TMP57:%.*]] = extractvalue { <4 x i32>, <4 x i32>, <4 x i32> } [[TMP56]], 0
 ; VF4-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, ptr [[DST_A]], i64 [[INDEX]]
@@ -371,54 +371,54 @@ define void @struct_return_3xi32_replicate(ptr noalias %in, ptr noalias writeonl
 ; VF2IC2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[WIDE_LOAD]], i32 1
 ; VF2IC2-NEXT:    [[TMP6:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP5]]) #[[ATTR2]]
 ; VF2IC2-NEXT:    [[TMP7:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 0
-; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
+; VF2IC2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i64 0
 ; VF2IC2-NEXT:    [[TMP9:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP8]], 0
 ; VF2IC2-NEXT:    [[TMP10:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 1
 ; VF2IC2-NEXT:    [[TMP11:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP9]], 1
-; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP10]], i32 0
+; VF2IC2-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP10]], i64 0
 ; VF2IC2-NEXT:    [[TMP13:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP9]], <2 x i32> [[TMP12]], 1
 ; VF2IC2-NEXT:    [[TMP14:%.*]] = extractvalue { i32, i32, i32 } [[TMP4]], 2
 ; VF2IC2-NEXT:    [[TMP15:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP13]], 2
-; VF2IC2-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP14]], i32 0
+; VF2IC2-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[TMP14]], i64 0
 ; VF2IC2-NEXT:    [[TMP17:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP13]], <2 x i32> [[TMP16]], 2
 ; VF2IC2-NEXT:    [[TMP18:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 0
 ; VF2IC2-NEXT:    [[TMP19:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP17]], 0
-; VF2IC2-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP18]], i32 1
+; VF2IC2-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP18]], i64 1
 ; VF2IC2-NEXT:    [[TMP21:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP17]], <2 x i32> [[TMP20]], 0
 ; VF2IC2-NEXT:    [[TMP22:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 1
 ; VF2IC2-NEXT:    [[TMP23:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP21]], 1
-; VF2IC2-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP22]], i32 1
+; VF2IC2-NEXT:    [[TMP24:%.*]] = insertelement <2 x i32> [[TMP23]], i32 [[TMP22]], i64 1
 ; VF2IC2-NEXT:    [[TMP25:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP21]], <2 x i32> [[TMP24]], 1
 ; VF2IC2-NEXT:    [[TMP26:%.*]] = extractvalue { i32, i32, i32 } [[TMP6]], 2
 ; VF2IC2-NEXT:    [[TMP27:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP25]], 2
-; VF2IC2-NEXT:    [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TMP26]], i32 1
+; VF2IC2-NEXT:    [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TMP26]], i64 1
 ; VF2IC2-NEXT:    [[TMP29:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP25]], <2 x i32> [[TMP28]], 2
 ; VF2IC2-NEXT:    [[TMP30:%.*]] = extractelement <2 x i32> [[WIDE_LOAD1]], i32 0
 ; VF2IC2-NEXT:    [[TMP31:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP30]]) #[[ATTR2]]
 ; VF2IC2-NEXT:    [[TMP32:%.*]] = extractelement <2 x i32> [[WIDE_LOAD1]], i32 1
 ; VF2IC2-NEXT:    [[TMP33:%.*]] = tail call { i32, i32, i32 } @fn3(i32 [[TMP32]]) #[[ATTR2]]
 ; VF2IC2-NEXT:    [[TMP34:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 0
-; VF2IC2-NEXT:    [[TMP35:%.*]] = insertelement <2 x i32> poison, i32 [[TMP34]], i32 0
+; VF2IC2-NEXT:    [[TMP35:%.*]] = insertelement <2 x i32> poison, i32 [[TMP34]], i64 0
 ; VF2IC2-NEXT:    [[TMP36:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } poison, <2 x i32> [[TMP35]], 0
 ; VF2IC2-NEXT:    [[TMP37:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 1
 ; VF2IC2-NEXT:    [[TMP38:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP36]], 1
-; VF2IC2-NEXT:    [[TMP39:%.*]] = insertelement <2 x i32> [[TMP38]], i32 [[TMP37]], i32 0
+; VF2IC2-NEXT:    [[TMP39:%.*]] = insertelement <2 x i32> [[TMP38]], i32 [[TMP37]], i64 0
 ; VF2IC2-NEXT:    [[TMP40:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP36]], <2 x i32> [[TMP39]], 1
 ; VF2IC2-NEXT:    [[TMP41:%.*]] = extractvalue { i32, i32, i32 } [[TMP31]], 2
 ; VF2IC2-NEXT:    [[TMP42:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP40]], 2
-; VF2IC2-NEXT:    [[TMP43:%.*]] = insertelement <2 x i32> [[TMP42]], i32 [[TMP41]], i32 0
+; VF2IC2-NEXT:    [[TMP43:%.*]] = insertelement <2 x i32> [[TMP42]], i32 [[TMP41]], i64 0
 ; VF2IC2-NEXT:    [[TMP44:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP40]], <2 x i32> [[TMP43]], 2
 ; VF2IC2-NEXT:    [[TMP45:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 0
 ; VF2IC2-NEXT:    [[TMP46:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP44]], 0
-; VF2IC2-NEXT:    [[TMP47:%.*]] = insertelement <2 x i32> [[TMP46]], i32 [[TMP45]], i32 1
+; VF2IC2-NEXT:    [[TMP47:%.*]] = insertelement <2 x i32> [[TMP46]], i32 [[TMP45]], i64 1
 ; VF2IC2-NEXT:    [[TMP48:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP44]], <2 x i32> [[TMP47]], 0
 ; VF2IC2-NEXT:    [[TMP49:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 1
 ; VF2IC2-NEXT:    [[TMP50:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP48]], 1
-; VF2IC2-NEXT:    [[TMP51:%.*]] = insertelement <2 x i32> [[TMP50]], i32 [[TMP49]], i32 1
+; VF2IC2-NEXT:    [[TMP51:%.*]] = insertelement <2 x i32> [[TMP50]], i32 [[TMP49]], i64 1
 ; VF2IC2-NEXT:    [[TMP52:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP48]], <2 x i32> [[TMP51]], 1
 ; VF2IC2-NEXT:    [[TMP53:%.*]] = extractvalue { i32, i32, i32 } [[TMP33]], 2
 ; VF2IC2-NEXT:    [[TMP54:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP52]], 2
-; VF2IC2-NEXT:    [[TMP55:%.*]] = insertelement <2 x i32> [[TMP54]], i32 [[TMP53]], i32 1
+; VF2IC2-NEXT:    [[TMP55:%.*]] = insertelement <2 x i32> [[TMP54]], i32 [[TMP53]], i64 1
 ; VF2IC2-NEXT:    [[TMP56:%.*]] = insertvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP52]], <2 x i32> [[TMP55]], 2
 ; VF2IC2-NEXT:    [[TMP57:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP29]], 0
 ; VF2IC2-NEXT:    [[TMP58:%.*]] = extractvalue { <2 x i32>, <2 x i32>, <2 x i32> } [[TMP56]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
index 3bbcb2b62c38..61ed49296070 100644
--- a/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S -enable-early-exit-vectorization -debug %s 2>&1 | FileCheck %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S -debug %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index fe533672f2ca..67d99dc73362 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -229,6 +229,10 @@ define void @redundant_branch_and_blends_without_mask(ptr %A) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x ptr> [[TMP35]], ptr [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x ptr> [[TMP36]], ptr [[TMP7]], i32 2
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x ptr> [[TMP37]], ptr [[TMP8]], i32 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
index 5b2a95f1b368..3cd60e8d7f59 100644
--- a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization  -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize  -force-vector-width=4 | FileCheck %s
 
 declare void @init_mem(ptr, i64);
 
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
index 1a8361ce2bac..2f192ad7a9ea 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=1 -enable-early-exit-vectorization -S %s | FileCheck --check-prefixes=VF8UF1 %s
-; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=2 -enable-early-exit-vectorization -S %s | FileCheck --check-prefixes=VF8UF2 %s
-; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=16 -force-vector-interleave=1 -enable-early-exit-vectorization -S %s | FileCheck --check-prefixes=VF16UF1 %s
+; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF8UF1 %s
+; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=8 -force-vector-interleave=2 -S %s | FileCheck --check-prefixes=VF8UF2 %s
+; RUN: opt -passes='loop-vectorize,verify<loops>' -force-vector-width=16 -force-vector-interleave=1 -S %s | FileCheck --check-prefixes=VF16UF1 %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -56,12 +56,16 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ; VF8UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; VF8UF2:       [[VECTOR_BODY]]:
 ; VF8UF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 0
+; VF8UF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 8
 ; VF8UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; VF8UF2-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
 ; VF8UF2-NEXT:    [[TMP3:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; VF8UF2-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
+; VF8UF2-NEXT:    [[TMP6:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer
+; VF8UF2-NEXT:    [[TMP4:%.*]] = or <8 x i1> [[TMP3]], [[TMP6]]
+; VF8UF2-NEXT:    [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP4]])
 ; VF8UF2-NEXT:    br label %[[MIDDLE_SPLIT:.*]]
 ; VF8UF2:       [[MIDDLE_SPLIT]]:
-; VF8UF2-NEXT:    br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; VF8UF2-NEXT:    br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
 ; VF8UF2:       [[MIDDLE_BLOCK]]:
 ; VF8UF2-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; VF8UF2:       [[VECTOR_EARLY_EXIT]]:
@@ -189,17 +193,26 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ; VF8UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; VF8UF2:       [[VECTOR_BODY]]:
 ; VF8UF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 0
+; VF8UF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 8
 ; VF8UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; VF8UF2-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
 ; VF8UF2-NEXT:    [[TMP3:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
-; VF8UF2-NEXT:    [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]])
+; VF8UF2-NEXT:    [[TMP6:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer
+; VF8UF2-NEXT:    [[TMP4:%.*]] = or <8 x i1> [[TMP3]], [[TMP6]]
+; VF8UF2-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP4]])
 ; VF8UF2-NEXT:    br label %[[MIDDLE_SPLIT:.*]]
 ; VF8UF2:       [[MIDDLE_SPLIT]]:
-; VF8UF2-NEXT:    br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; VF8UF2-NEXT:    br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
 ; VF8UF2:       [[MIDDLE_BLOCK]]:
 ; VF8UF2-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; VF8UF2:       [[VECTOR_EARLY_EXIT]]:
+; VF8UF2-NEXT:    [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP6]], i1 true)
+; VF8UF2-NEXT:    [[TMP10:%.*]] = add i64 8, [[TMP8]]
 ; VF8UF2-NEXT:    [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP3]], i1 true)
 ; VF8UF2-NEXT:    [[TMP5:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE]]
+; VF8UF2-NEXT:    [[TMP9:%.*]] = icmp ne i64 [[FIRST_ACTIVE_LANE]], 8
+; VF8UF2-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP5]], i64 [[TMP10]]
+; VF8UF2-NEXT:    [[TMP12:%.*]] = add i64 0, [[TMP11]]
 ; VF8UF2-NEXT:    br label %[[EXIT]]
 ; VF8UF2:       [[SCALAR_PH]]:
 ; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
@@ -215,7 +228,7 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ; VF8UF2-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 16
 ; VF8UF2-NEXT:    br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 ; VF8UF2:       [[EXIT]]:
-; VF8UF2-NEXT:    [[RES:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ], [ [[TMP5]], %[[VECTOR_EARLY_EXIT]] ]
+; VF8UF2-NEXT:    [[RES:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT]] ]
 ; VF8UF2-NEXT:    ret i64 [[RES]]
 ;
 ; VF16UF1-LABEL: define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
index 978f1b80d26d..4af3fa9202c7 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
@@ -36,13 +36,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result fast ir<%red>, ir<%red.next>
-; CHECK-NEXT:   EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
 ; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:  IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT:  IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
@@ -179,13 +178,12 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result nnan ninf nsz ir<%sum.07>, ir<[[MULADD]]>
-; CHECK-NEXT:   EMIT vp<[[RED_EX:%.+]]> = extract-last-element vp<[[RED_RES]]>
 ; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<%n>, vp<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>
-; CHECK-NEXT:   IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[RED_EX]]> from middle.block)
+; CHECK-NEXT:   IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[RED_RES]]> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph
@@ -240,14 +238,13 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
-; CHECK-NEXT:   EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
+; CHECK-NEXT:   EMIT vp<[[RDX_RES:%.+]]> = compute-find-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
 ; CHECK-NEXT:   EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<%cmp.n>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
 ; CHECK-EMPTY:
 ; CHECK:      ir-bb<exit>:
-; CHECK-NEXT:   IR   %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[EXT]]> from middle.block)
+; CHECK-NEXT:   IR   %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[RDX_RES]]> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
@@ -287,12 +284,12 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR:%.+]]> = vector-pointer ir<%arrayidx>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD:%.+]]> = load vp<[[ADDR]]>
-; CHECK-NEXT:     EXTENDED-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> zext to i64)
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> zext to i64)
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
@@ -332,7 +329,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
@@ -340,7 +337,7 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX1:%.+]]> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR1:%.+]]> = vector-pointer ir<[[ARRAYIDX1]]>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD1:%.+]]> = load vp<[[ADDR1]]>
-; CHECK-NEXT:     MULACC-REDUCE ir<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (mul nsw ir<[[LOAD0]]>, ir<[[LOAD1]]>)
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT]]> = ir<[[RDX]]> + reduce.add (mul nsw ir<[[LOAD0]]>, ir<[[LOAD1]]>)
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
@@ -382,7 +379,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[IV_NEXT:%.+]]>
-; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
+; CHECK-NEXT:     WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_NEXT:%.+]]>
 ; CHECK-NEXT:     vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX0:%.+]]> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR0:%.+]]> = vector-pointer ir<[[ARRAYIDX0]]>
@@ -390,7 +387,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
 ; CHECK-NEXT:     CLONE ir<[[ARRAYIDX1:%.+]]> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
 ; CHECK-NEXT:     vp<[[ADDR1:%.+]]> = vector-pointer ir<[[ARRAYIDX1]]>
 ; CHECK-NEXT:     WIDEN ir<[[LOAD1:%.+]]> = load vp<[[ADDR1]]>
-; CHECK-NEXT:     MULACC-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> sext to i64), (ir<[[LOAD1]]> sext to i64))
+; CHECK-NEXT:     EXPRESSION vp<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> sext to i64), (ir<[[LOAD1]]> sext to i64))
 ; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
diff --git a/llvm/test/Transforms/MergeICmps/X86/blockaddresses.ll b/llvm/test/Transforms/MergeICmps/X86/blockaddresses.ll
new file mode 100644
index 000000000000..d8f47a1d080f
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/blockaddresses.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=mergeicmps -S | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%type = type { i64, i64, i64, i64 }
+
+define fastcc i1 @test(ptr byval(%type) %a, ptr byval(%type) %b, i1 %cond) {
+; CHECK-LABEL: define fastcc i1 @test(
+; CHECK-SAME: ptr byval([[TYPE:%.*]]) [[A:%.*]], ptr byval([[TYPE]]) [[B:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[VALUE401:%.*]] = select i1 [[COND]], ptr blockaddress(@test, %[[BLOCK307:.*]]), ptr blockaddress(@test, %[[EXIT:.*]])
+; CHECK-NEXT:    indirectbr ptr [[VALUE401]], [label %[[BLOCK307]], label %exit]
+; CHECK:       [[BLOCK307]]:
+; CHECK-NEXT:    [[VALUE403:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    [[VALUE404:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    [[CMP_I:%.*]] = icmp eq i64 [[VALUE403]], [[VALUE404]]
+; CHECK-NEXT:    br i1 [[CMP_I]], label %[[LAND_LHS_TRUE_I:.*]], label %[[TEST_EXIT:.*]]
+; CHECK:       [[LAND_LHS_TRUE_I]]:
+; CHECK-NEXT:    [[B_I:%.*]] = getelementptr [[TYPE]], ptr [[A]], i64 0, i32 0
+; CHECK-NEXT:    [[VALUE405:%.*]] = load i64, ptr [[B_I]], align 8
+; CHECK-NEXT:    [[B3_I:%.*]] = getelementptr [[TYPE]], ptr [[B]], i64 0, i32 0
+; CHECK-NEXT:    [[VALUE406:%.*]] = load i64, ptr [[B3_I]], align 8
+; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp eq i64 [[VALUE405]], [[VALUE406]]
+; CHECK-NEXT:    br i1 [[CMP4_I]], label %[[LAND_LHS_TRUE5_I:.*]], label %[[TEST_EXIT]]
+; CHECK:       [[LAND_LHS_TRUE5_I]]:
+; CHECK-NEXT:    [[C_I:%.*]] = getelementptr [[TYPE]], ptr [[A]], i64 0, i32 2
+; CHECK-NEXT:    [[VALUE407:%.*]] = load i64, ptr [[C_I]], align 8
+; CHECK-NEXT:    [[C6_I:%.*]] = getelementptr [[TYPE]], ptr [[B]], i64 0, i32 2
+; CHECK-NEXT:    [[VALUE408:%.*]] = load i64, ptr [[C6_I]], align 8
+; CHECK-NEXT:    [[CMP7_I:%.*]] = icmp eq i64 [[VALUE407]], [[VALUE408]]
+; CHECK-NEXT:    br i1 [[CMP7_I]], label %[[LAND_RHS_I:.*]], label %[[TEST_EXIT]]
+; CHECK:       [[LAND_RHS_I]]:
+; CHECK-NEXT:    [[D_I:%.*]] = getelementptr [[TYPE]], ptr [[A]], i64 0, i32 3
+; CHECK-NEXT:    [[VALUE409:%.*]] = load i64, ptr [[D_I]], align 8
+; CHECK-NEXT:    [[D8_I:%.*]] = getelementptr [[TYPE]], ptr [[B]], i64 0, i32 3
+; CHECK-NEXT:    [[VALUE410:%.*]] = load i64, ptr [[D8_I]], align 8
+; CHECK-NEXT:    [[CMP9_I:%.*]] = icmp eq i64 [[VALUE409]], [[VALUE410]]
+; CHECK-NEXT:    br label %[[TEST_EXIT]]
+; CHECK:       [[TEST_EXIT]]:
+; CHECK-NEXT:    [[VALUE411:%.*]] = phi i1 [ false, %[[LAND_LHS_TRUE5_I]] ], [ false, %[[LAND_LHS_TRUE_I]] ], [ false, %[[BLOCK307]] ], [ [[CMP9_I]], %[[LAND_RHS_I]] ]
+; CHECK-NEXT:    ret i1 false
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %Value401 = select i1 %cond, ptr blockaddress(@test, %Block307), ptr blockaddress(@test, %exit)
+  indirectbr ptr %Value401, [label %Block307, label %exit]
+
+Block307:                                         ; preds = %entry
+  %Value403 = load i64, ptr %a, align 8
+  %Value404 = load i64, ptr %a, align 8
+  %cmp.i = icmp eq i64 %Value403, %Value404
+  br i1 %cmp.i, label %land.lhs.true.i, label %test.exit
+
+land.lhs.true.i:                                  ; preds = %Block307
+  %b.i = getelementptr %type, ptr %a, i64 0, i32 0
+  %Value405 = load i64, ptr %b.i, align 8
+  %b3.i = getelementptr %type, ptr %b, i64 0, i32 0
+  %Value406 = load i64, ptr %b3.i, align 8
+  %cmp4.i = icmp eq i64 %Value405, %Value406
+  br i1 %cmp4.i, label %land.lhs.true5.i, label %test.exit
+
+land.lhs.true5.i:                                 ; preds = %land.lhs.true.i
+  %c.i = getelementptr %type, ptr %a, i64 0, i32 2
+  %Value407 = load i64, ptr %c.i, align 8
+  %c6.i = getelementptr %type, ptr %b, i64 0, i32 2
+  %Value408 = load i64, ptr %c6.i, align 8
+  %cmp7.i = icmp eq i64 %Value407, %Value408
+  br i1 %cmp7.i, label %land.rhs.i, label %test.exit
+
+land.rhs.i:                                       ; preds = %land.lhs.true5.i
+  %d.i = getelementptr %type, ptr %a, i64 0, i32 3
+  %Value409 = load i64, ptr %d.i, align 8
+  %d8.i = getelementptr %type, ptr %b, i64 0, i32 3
+  %Value410 = load i64, ptr %d8.i, align 8
+  %cmp9.i = icmp eq i64 %Value409, %Value410
+  br label %test.exit
+
+test.exit:                    ; preds = %land.rhs.i, %land.lhs.true5.i, %land.lhs.true.i, %Block307
+  %Value411 = phi i1 [ false, %land.lhs.true5.i ], [ false, %land.lhs.true.i ], [ false, %Block307 ], [ %cmp9.i, %land.rhs.i ]
+  ret i1 false
+
+exit:                                            ; preds = %entry
+  ret i1 false
+}
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
index 3b0e3250cd6d..ab8ebefd204d 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion.ll
@@ -1,6 +1,10 @@
 ; RUN: opt < %s -passes=pgo-icall-prom -S -icp-total-percent-threshold=50 | FileCheck %s --check-prefix=ICALL-PROM
 ; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS-REMARK
 ; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=20 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=PASS2-REMARK
+; Test minimum count threshold - should prevent func1 promotion (count 10 < threshold 15)
+; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-minimum-count-threshold=15 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=MIN-COUNT-15
+; Test edge case - threshold exactly at count value
+; RUN: opt < %s -passes=pgo-icall-prom -S -pass-remarks=pgo-icall-prom -icp-minimum-count-threshold=10 -icp-remaining-percent-threshold=0 -icp-total-percent-threshold=0 -icp-max-prom=4 2>&1 | FileCheck %s --check-prefix=MIN-COUNT-10
 
 ; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func4 with count 1030 out of 1600
 ; PASS-REMARK: remark: <unknown>:0:0: Promote indirect call to func2 with count 410 out of 570
@@ -12,6 +16,16 @@
 ; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func3
 ; PASS2-REMARK-NOT: remark: <unknown>:0:0: Promote indirect call to func1
 
+; MIN-COUNT-15: remark: <unknown>:0:0: Promote indirect call to func4 with count 1030 out of 1600
+; MIN-COUNT-15: remark: <unknown>:0:0: Promote indirect call to func2 with count 410 out of 570
+; MIN-COUNT-15: remark: <unknown>:0:0: Promote indirect call to func3 with count 150 out of 160
+; MIN-COUNT-15-NOT: remark: <unknown>:0:0: Promote indirect call to func1
+
+; MIN-COUNT-10: remark: <unknown>:0:0: Promote indirect call to func4 with count 1030 out of 1600
+; MIN-COUNT-10: remark: <unknown>:0:0: Promote indirect call to func2 with count 410 out of 570
+; MIN-COUNT-10: remark: <unknown>:0:0: Promote indirect call to func3 with count 150 out of 160
+; MIN-COUNT-10: remark: <unknown>:0:0: Promote indirect call to func1 with count 10 out of 10
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
index 22e4239009dd..bbf893f6127b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/blendv-select.ll
@@ -12,20 +12,10 @@
 ;
 
 define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d) {
-; SSE-LABEL: @x86_pblendvb_v4f64_v2f64(
-; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; SSE-NEXT:    ret <4 x double> [[DOTV]]
-;
-; AVX2-LABEL: @x86_pblendvb_v4f64_v2f64(
-; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; AVX2-NEXT:    ret <4 x double> [[DOTV]]
-;
-; AVX512-LABEL: @x86_pblendvb_v4f64_v2f64(
-; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
-; AVX512-NEXT:    ret <4 x double> [[DOTV]]
+; CHECK-LABEL: @x86_pblendvb_v4f64_v2f64(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select <4 x i1> [[CMP]], <4 x double> [[B:%.*]], <4 x double> [[A:%.*]]
+; CHECK-NEXT:    ret <4 x double> [[DOTV]]
 ;
   %a.bc = bitcast <4 x double> %a to <32 x i8>
   %b.bc = bitcast <4 x double> %b to <32 x i8>
@@ -46,20 +36,10 @@ define <4 x double> @x86_pblendvb_v4f64_v2f64(<4 x double> %a, <4 x double> %b,
 }
 
 define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
-; SSE-LABEL: @x86_pblendvb_v8f32_v4f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; SSE-NEXT:    ret <8 x float> [[DOTV]]
-;
-; AVX2-LABEL: @x86_pblendvb_v8f32_v4f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; AVX2-NEXT:    ret <8 x float> [[DOTV]]
-;
-; AVX512-LABEL: @x86_pblendvb_v8f32_v4f32(
-; AVX512-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
-; AVX512-NEXT:    ret <8 x float> [[DOTV]]
+; CHECK-LABEL: @x86_pblendvb_v8f32_v4f32(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <8 x float> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[DOTV:%.*]] = select <8 x i1> [[CMP]], <8 x float> [[B:%.*]], <8 x float> [[A:%.*]]
+; CHECK-NEXT:    ret <8 x float> [[DOTV]]
 ;
   %a.bc = bitcast <8 x float> %a to <32 x i8>
   %b.bc = bitcast <8 x float> %b to <32 x i8>
@@ -80,20 +60,10 @@ define <8 x float> @x86_pblendvb_v8f32_v4f32(<8 x float> %a, <8 x float> %b, <8
 }
 
 define <4 x i64> @x86_pblendvb_v4i64_v2i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v4i64_v2i64(
-; SSE-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; SSE-NEXT:    ret <4 x i64> [[TMP2]]
-;
-; AVX2-LABEL: @x86_pblendvb_v4i64_v2i64(
-; AVX2-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; AVX2-NEXT:    ret <4 x i64> [[TMP2]]
-;
-; AVX512-LABEL: @x86_pblendvb_v4i64_v2i64(
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
-; AVX512-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
-; AVX512-NEXT:    ret <4 x i64> [[TMP1]]
+; CHECK-LABEL: @x86_pblendvb_v4i64_v2i64(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <4 x i64> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[CMP]], <4 x i64> [[B:%.*]], <4 x i64> [[A:%.*]]
+; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -216,35 +186,15 @@ define <4 x i64> @x86_pblendvb_v16i16_v8i16(<4 x i64> %a, <4 x i64> %b, <4 x i64
 }
 
 define <4 x i64> @x86_pblendvb_v32i8_v16i8(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c, <4 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v32i8_v16i8(
-; SSE-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; SSE-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; SSE-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; SSE-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; SSE-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
-; SSE-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; SSE-NEXT:    ret <4 x i64> [[RES]]
-;
-; AVX2-LABEL: @x86_pblendvb_v32i8_v16i8(
-; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; AVX2-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[TMP3]], <32 x i8> [[TMP2]], <32 x i8> [[TMP1]]
-; AVX2-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; AVX2-NEXT:    ret <4 x i64> [[RES]]
-;
-; AVX512-LABEL: @x86_pblendvb_v32i8_v16i8(
-; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
-; AVX512-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
-; AVX512-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
-; AVX512-NEXT:    ret <4 x i64> [[RES]]
+; CHECK-LABEL: @x86_pblendvb_v32i8_v16i8(
+; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <4 x i64> [[A:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <4 x i64> [[B:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <4 x i64> [[C:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <4 x i64> [[D:%.*]] to <32 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <32 x i8> [[C_BC]], [[D_BC]]
+; CHECK-NEXT:    [[CONCAT:%.*]] = select <32 x i1> [[CMP]], <32 x i8> [[B_BC]], <32 x i8> [[A_BC]]
+; CHECK-NEXT:    [[RES:%.*]] = bitcast <32 x i8> [[CONCAT]] to <4 x i64>
+; CHECK-NEXT:    ret <4 x i64> [[RES]]
 ;
   %a.bc = bitcast <4 x i64> %a to <32 x i8>
   %b.bc = bitcast <4 x i64> %b to <32 x i8>
@@ -424,35 +374,15 @@ define <8 x i64> @x86_pblendvb_v32i16_v16i16(<8 x i64> %a, <8 x i64> %b, <8 x i6
 }
 
 define <8 x i64> @x86_pblendvb_v64i8_v32i8(<8 x i64> %a, <8 x i64> %b, <8 x i64> %c, <8 x i64> %d) {
-; SSE-LABEL: @x86_pblendvb_v64i8_v32i8(
-; SSE-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; SSE-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; SSE-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; SSE-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; SSE-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; SSE-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
-; SSE-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; SSE-NEXT:    ret <8 x i64> [[RES]]
-;
-; AVX2-LABEL: @x86_pblendvb_v64i8_v32i8(
-; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[TMP2:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; AVX2-NEXT:    [[TMP3:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; AVX2-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[TMP3]], <64 x i8> [[TMP2]], <64 x i8> [[TMP1]]
-; AVX2-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; AVX2-NEXT:    ret <8 x i64> [[RES]]
-;
-; AVX512-LABEL: @x86_pblendvb_v64i8_v32i8(
-; AVX512-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
-; AVX512-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
-; AVX512-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
-; AVX512-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
-; AVX512-NEXT:    ret <8 x i64> [[RES]]
+; CHECK-LABEL: @x86_pblendvb_v64i8_v32i8(
+; CHECK-NEXT:    [[A_BC:%.*]] = bitcast <8 x i64> [[A:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[B_BC:%.*]] = bitcast <8 x i64> [[B:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[C_BC:%.*]] = bitcast <8 x i64> [[C:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[D_BC:%.*]] = bitcast <8 x i64> [[D:%.*]] to <64 x i8>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <64 x i8> [[C_BC]], [[D_BC]]
+; CHECK-NEXT:    [[CONCAT:%.*]] = select <64 x i1> [[CMP]], <64 x i8> [[B_BC]], <64 x i8> [[A_BC]]
+; CHECK-NEXT:    [[RES:%.*]] = bitcast <64 x i8> [[CONCAT]] to <8 x i64>
+; CHECK-NEXT:    ret <8 x i64> [[RES]]
 ;
   %a.bc = bitcast <8 x i64> %a to <64 x i8>
   %b.bc = bitcast <8 x i64> %b to <64 x i8>
@@ -477,30 +407,22 @@ define <2 x i64> @PR66513(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c, <2 x i64> %s
 ; CHECK-LABEL: @PR66513(
 ; CHECK-NEXT:    [[I:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[CMP_I23:%.*]] = icmp sgt <4 x i32> [[I]], zeroinitializer
-; CHECK-NEXT:    [[SEXT_I24:%.*]] = sext <4 x i1> [[CMP_I23]] to <4 x i32>
-; CHECK-NEXT:    [[I1:%.*]] = bitcast <4 x i32> [[SEXT_I24]] to <2 x i64>
 ; CHECK-NEXT:    [[I2:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[CMP_I21:%.*]] = icmp sgt <4 x i32> [[I2]], zeroinitializer
-; CHECK-NEXT:    [[SEXT_I22:%.*]] = sext <4 x i1> [[CMP_I21]] to <4 x i32>
-; CHECK-NEXT:    [[I3:%.*]] = bitcast <4 x i32> [[SEXT_I22]] to <2 x i64>
 ; CHECK-NEXT:    [[I4:%.*]] = bitcast <2 x i64> [[C:%.*]] to <4 x i32>
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp sgt <4 x i32> [[I4]], zeroinitializer
-; CHECK-NEXT:    [[SEXT_I:%.*]] = sext <4 x i1> [[CMP_I]] to <4 x i32>
+; CHECK-NEXT:    [[NARROW:%.*]] = select <4 x i1> [[CMP_I21]], <4 x i1> [[CMP_I23]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[XOR_I_INNER1:%.*]] = xor <4 x i1> [[NARROW]], [[CMP_I]]
+; CHECK-NEXT:    [[NARROW3:%.*]] = select <4 x i1> [[CMP_I23]], <4 x i1> [[XOR_I_INNER1]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[AND_I25_INNER2:%.*]] = and <4 x i1> [[XOR_I_INNER1]], [[CMP_I21]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[SRC:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[NARROW]], <4 x i32> [[TMP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <2 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[NARROW3]], <4 x i32> [[TMP3]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <2 x i64> [[B]] to <4 x i32>
+; CHECK-NEXT:    [[SEXT_I:%.*]] = select <4 x i1> [[AND_I25_INNER2]], <4 x i32> [[TMP5]], <4 x i32> [[TMP4]]
 ; CHECK-NEXT:    [[I5:%.*]] = bitcast <4 x i32> [[SEXT_I]] to <2 x i64>
-; CHECK-NEXT:    [[AND_I27:%.*]] = and <2 x i64> [[I3]], [[I1]]
-; CHECK-NEXT:    [[XOR_I:%.*]] = xor <2 x i64> [[AND_I27]], [[I5]]
-; CHECK-NEXT:    [[AND_I26:%.*]] = and <2 x i64> [[XOR_I]], [[I1]]
-; CHECK-NEXT:    [[AND_I25:%.*]] = and <2 x i64> [[XOR_I]], [[I3]]
-; CHECK-NEXT:    [[AND_I:%.*]] = and <2 x i64> [[AND_I27]], [[SRC:%.*]]
-; CHECK-NEXT:    [[I6:%.*]] = bitcast <2 x i64> [[AND_I]] to <16 x i8>
-; CHECK-NEXT:    [[I7:%.*]] = bitcast <2 x i64> [[A]] to <16 x i8>
-; CHECK-NEXT:    [[I8:%.*]] = bitcast <2 x i64> [[AND_I26]] to <16 x i8>
-; CHECK-NEXT:    [[I9:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[I6]], <16 x i8> [[I7]], <16 x i8> [[I8]])
-; CHECK-NEXT:    [[I12:%.*]] = bitcast <2 x i64> [[B]] to <16 x i8>
-; CHECK-NEXT:    [[I13:%.*]] = bitcast <2 x i64> [[AND_I25]] to <16 x i8>
-; CHECK-NEXT:    [[I14:%.*]] = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> [[I9]], <16 x i8> [[I12]], <16 x i8> [[I13]])
-; CHECK-NEXT:    [[I15:%.*]] = bitcast <16 x i8> [[I14]] to <2 x i64>
-; CHECK-NEXT:    ret <2 x i64> [[I15]]
+; CHECK-NEXT:    ret <2 x i64> [[I5]]
 ;
   %i = bitcast <2 x i64> %a to <4 x i32>
   %cmp.i23 = icmp sgt <4 x i32> %i, zeroinitializer
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
index 9bfd92ef35a4..798df4cd4ff5 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
@@ -132,7 +132,8 @@ define <8 x i16> @add_v8i16_76u43210(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP4]], [[TMP5]]
 ; SSE2-NEXT:    [[HADD41:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP6]], <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[HADD6:%.*]] = shufflevector <8 x i16> [[HADD41]], <8 x i16> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 12, i32 poison>
-; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[HADD6]], <8 x i32> <i32 7, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    [[HADD7:%.*]] = shufflevector <8 x i16> [[HADD6]], <8 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 6, i32 15>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[HADD7]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:    ret <8 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v8i16_76u43210(
@@ -290,13 +291,10 @@ define <16 x i16> @add_v16i16_0123u56789uBCDEF(<16 x i16> %a, <16 x i16> %b) {
 ; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v16i16_0123u56789uBCDEF(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 26, i32 poison, i32 30, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP4]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 27, i32 poison, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 24, i32 26, i32 28, i32 30>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 8, i32 10, i32 poison, i32 14, i32 24, i32 26, i32 28, i32 30>
 ; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 25, i32 27, i32 29, i32 31>
 ; SSE4-NEXT:    [[TMP9:%.*]] = add <16 x i16> [[TMP7]], [[TMP8]]
 ; SSE4-NEXT:    ret <16 x i16> [[TMP9]]
@@ -404,11 +402,9 @@ define <16 x i16> @add_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
 ; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @add_v16i16_FEuCBA98765432u0(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 27, i32 28, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 poison, i32 11, i32 12, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 26, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP4]], [[TMP5]]
 ; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
index 13b4d7da97c9..fd160b7c5702 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
@@ -132,7 +132,8 @@ define <8 x i16> @sub_v8i16_76u43210(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP4]], [[TMP5]]
 ; SSE2-NEXT:    [[HSUB41:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP6]], <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison, i32 poison>
 ; SSE2-NEXT:    [[HSUB6:%.*]] = shufflevector <8 x i16> [[HSUB41]], <8 x i16> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 12, i32 poison>
-; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[HSUB6]], <8 x i32> <i32 7, i32 14, i32 poison, i32 12, i32 11, i32 10, i32 9, i32 8>
+; SSE2-NEXT:    [[HSUB7:%.*]] = shufflevector <8 x i16> [[HSUB6]], <8 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 poison, i32 6, i32 15>
+; SSE2-NEXT:    [[RESULT:%.*]] = shufflevector <8 x i16> [[HSUB7]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 poison, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; SSE2-NEXT:    ret <8 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v8i16_76u43210(
@@ -290,13 +291,10 @@ define <16 x i16> @sub_v16i16_0123u56789uBCDEF(<16 x i16> %a, <16 x i16> %b) {
 ; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v16i16_0123u56789uBCDEF(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 poison, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 26, i32 poison, i32 30, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i16> [[TMP4]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 27, i32 poison, i32 31, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[TMP5]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 24, i32 26, i32 28, i32 30>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 18, i32 20, i32 22, i32 8, i32 10, i32 poison, i32 14, i32 24, i32 26, i32 28, i32 30>
 ; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 6, i32 7, i32 8, i32 9, i32 poison, i32 11, i32 25, i32 27, i32 29, i32 31>
 ; SSE4-NEXT:    [[TMP9:%.*]] = sub <16 x i16> [[TMP7]], [[TMP8]]
 ; SSE4-NEXT:    ret <16 x i16> [[TMP9]]
@@ -398,11 +396,9 @@ define <16 x i16> @sub_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
 ; SSE2-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; SSE4-LABEL: @sub_v16i16_FEuCBA98765432u0(
-; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 26, i32 28, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 poison, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 27, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP4]], [[TMP5]]
 ; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
new file mode 100644
index 000000000000..125675fe5046
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt --Os -S %s -o - | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "arm64e-apple-ios19.0.0"
+
+; This test checks if we generate a phi for the <vscale 16 x float> value or we manage
+; to promote it during inlining.
+
+; Function Attrs: mustprogress optsize ssp uwtable(sync)
+define void @pluto() #0 {
+; CHECK-LABEL: define void @pluto(
+; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP2]], <vscale x 16 x float> [[TMP3]], <vscale x 16 x float> undef
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[SPEC_SELECT]], i64 0)
+; CHECK-NEXT:    br label %[[BB5:.*]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP4]])
+; CHECK-NEXT:    br label %[[BB5]]
+;
+  br label %1
+
+1:                                                ; preds = %1, %0
+  call void @ham() #5
+  br label %1
+}
+
+; Function Attrs: alwaysinline mustprogress optsize ssp uwtable(sync)
+define void @ham() #1 {
+; CHECK-LABEL: define void @ham(
+; CHECK-SAME: ) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[SNORK_EXIT:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr inttoptr (i64 48 to ptr), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[TMP0]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP1]], <vscale x 16 x float> [[TMP2]], <vscale x 16 x float> undef
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[SPEC_SELECT]], i64 0)
+; CHECK-NEXT:    tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP3]])
+; CHECK-NEXT:    ret void
+;
+  %1 = alloca <vscale x 16 x float>, align 16
+  %2 = load i64, ptr inttoptr (i64 48 to ptr), align 8
+  %3 = call i64 @snork(i64 noundef %2, ptr noundef nonnull align 16 %1) #5
+  %4 = load <vscale x 16 x float>, ptr %1, align 16
+  %5 = call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> %4, i64 0)
+  call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> %5)
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float>, <vscale x 4 x float>, i64 immarg) #2
+
+; Function Attrs: alwaysinline mustprogress nounwind optsize ssp uwtable(sync)
+define i64 @snork(i64 noundef %0, ptr noundef nonnull align 16 %1) #3 {
+; CHECK-LABEL: define noundef i64 @snork(
+; CHECK-SAME: i64 noundef [[TMP0:%.*]], ptr noundef nonnull writeonly align 16 captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i64 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[BB4:.*]], label %[[BB6:.*]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT:    store <vscale x 16 x float> [[TMP5]], ptr [[TMP1]], align 16
+; CHECK-NEXT:    br label %[[BB6]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    ret i64 0
+;
+  %3 = icmp sgt i64 %0, 0
+  br i1 %3, label %4, label %6
+
+4:                                                ; preds = %2
+  %5 = call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+  store <vscale x 16 x float> %5, ptr %1, align 16
+  br label %6
+
+6:                                                ; preds = %4, %2
+  ret i64 0
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float>, i64 immarg) #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
+declare void @llvm.aarch64.sme.mopa.nxv4f32(i32 immarg, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>) #4
+
+attributes #0 = { mustprogress optsize ssp uwtable(sync) "aarch64_new_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #1 = { alwaysinline mustprogress optsize ssp uwtable(sync) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #3 = { alwaysinline mustprogress nounwind optsize ssp uwtable(sync) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "frame-pointer"="non-leaf" "no-builtin-calloc" "no-builtin-stpcpy" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m4" }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
+attributes #5 = { optsize "aarch64_inout_za" "aarch64_pstate_sm_enabled" "no-builtin-calloc" "no-builtin-stpcpy" }
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
index e53b8687af91..45f18dd56739 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
@@ -24,7 +24,8 @@ define void @test_fill_with_foreach([2 x i64] %elems.coerce) {
 ; CHECK-NEXT:    [[ELEMS_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x i64] [[ELEMS_COERCE]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = inttoptr i64 [[ELEMS_COERCE_FCA_0_EXTRACT]] to ptr
 ; CHECK-NEXT:    [[ELEMS_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x i64] [[ELEMS_COERCE]], 1
-; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[ELEMS_COERCE_FCA_1_EXTRACT]]
+; CHECK-NEXT:    [[ADD_PTR_I_IDX:%.*]] = shl nsw i64 [[ELEMS_COERCE_FCA_1_EXTRACT]], 2
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 [[ADD_PTR_I_IDX]]
 ; CHECK-NEXT:    [[CMP_NOT_I_I_I_I:%.*]] = icmp slt i64 [[ELEMS_COERCE_FCA_1_EXTRACT]], 0
 ; CHECK-NEXT:    br i1 [[CMP_NOT_I_I_I_I]], label [[ERROR:%.*]], label [[FOR_COND_PREHEADER_SPLIT:%.*]]
 ; CHECK:       for.cond.preheader.split:
diff --git a/llvm/test/Transforms/PhaseOrdering/memset-dse.ll b/llvm/test/Transforms/PhaseOrdering/memset-dse.ll
new file mode 100644
index 000000000000..ce398ddc4194
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/memset-dse.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -O2 < %s | FileCheck %s
+
+declare void @do_something(ptr %a)
+
+; We want only one memset in the if block (rather than only one memset in
+; the entry block).
+define void @test(i1 %c) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i1 [[C:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:    [[A:%.*]] = alloca [32 x i8], align 1
+; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[COMMON_RET:.*]]
+; CHECK:       [[COMMON_RET]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(32) [[A]], i8 0, i64 32, i1 false)
+; CHECK-NEXT:    call void @do_something(ptr nonnull [[A]])
+; CHECK-NEXT:    br label %[[COMMON_RET]]
+;
+  %a = alloca [32 x i8]
+  call void @llvm.memset(ptr %a, i8 0, i64 32, i1 false)
+  br i1 %c, label %if, label %else
+if:
+  call void @llvm.memset(ptr %a, i8 0, i64 32, i1 false)
+  call void @do_something(ptr %a)
+  ret void
+
+else:
+  ret void
+}
diff --git a/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll b/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
index 78b8a4aa126c..322c38d090fe 100644
--- a/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
+++ b/llvm/test/Transforms/RelLookupTableConverter/unnamed_addr.ll
@@ -20,6 +20,14 @@
 @y3 = internal unnamed_addr constant ptr @x0
 @load_relative_2.table = private unnamed_addr constant [4 x ptr] [ptr @y3, ptr @y2, ptr @y1, ptr @y0]
 
+@b0 = private unnamed_addr constant [8 x i8] c"00000000"
+@b1 = private unnamed_addr constant [8 x i8] c"11111111"
+@b2 = private unnamed_addr constant [8 x i8] c"22222222"
+@load_relative_3.table = private unnamed_addr constant [3 x ptr] [
+  ptr getelementptr inbounds (i8, ptr @b0, i64 8),
+  ptr getelementptr inbounds (i8, ptr @b1, i64 8),
+  ptr getelementptr inbounds (i8, ptr @b2, i64 8)]
+
 ;.
 ; x86_64-apple-darwin: @a0 = private constant i32 0
 ; x86_64-apple-darwin: @a1 = private constant i32 1
@@ -34,6 +42,10 @@
 ; x86_64-apple-darwin: @y2 = internal constant ptr @x1
 ; x86_64-apple-darwin: @y3 = internal constant ptr @x0
 ; x86_64-apple-darwin: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; x86_64-apple-darwin: @b0 = private constant [8 x i8] c"00000000"
+; x86_64-apple-darwin: @b1 = private constant [8 x i8] c"11111111"
+; x86_64-apple-darwin: @b2 = private constant [8 x i8] c"22222222"
+; x86_64-apple-darwin: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 ; aarch64: @a0 = private constant i32 0
 ; aarch64: @a1 = private constant i32 1
@@ -48,6 +60,10 @@
 ; aarch64: @y2 = internal constant ptr @x1
 ; aarch64: @y3 = internal constant ptr @x0
 ; aarch64: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; aarch64: @b0 = private constant [8 x i8] c"00000000"
+; aarch64: @b1 = private constant [8 x i8] c"11111111"
+; aarch64: @b2 = private constant [8 x i8] c"22222222"
+; aarch64: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 ; x86_64: @a0 = private unnamed_addr constant i32 0
 ; x86_64: @a1 = private unnamed_addr constant i32 1
@@ -62,6 +78,10 @@
 ; x86_64: @y2 = internal unnamed_addr constant ptr @x1
 ; x86_64: @y3 = internal unnamed_addr constant ptr @x0
 ; x86_64: @load_relative_2.table.rel = private unnamed_addr constant [4 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr @y3 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y2 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y1 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr @y0 to i64), i64 ptrtoint (ptr @load_relative_2.table.rel to i64)) to i32)], align 4
+; x86_64: @b0 = private unnamed_addr constant [8 x i8] c"00000000"
+; x86_64: @b1 = private unnamed_addr constant [8 x i8] c"11111111"
+; x86_64: @b2 = private unnamed_addr constant [8 x i8] c"22222222"
+; x86_64: @load_relative_3.table.rel = private unnamed_addr constant [3 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b0, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b1, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (ptr getelementptr inbounds (i8, ptr @b2, i64 8) to i64), i64 ptrtoint (ptr @load_relative_3.table.rel to i64)) to i32)], align 4
 ;.
 define ptr @load_relative_1(i64 %offset) {
 ; x86_64-apple-darwin-LABEL: define ptr @load_relative_1(
@@ -110,6 +130,31 @@ define ptr @load_relative_2(i64 %offset) {
   %load = load ptr, ptr %gep
   ret ptr %load
 }
+
+define ptr @load_relative_3(i64 %offset) {
+; x86_64-apple-darwin-LABEL: define ptr @load_relative_3(
+; x86_64-apple-darwin-SAME: i64 [[OFFSET:%.*]]) {
+; x86_64-apple-darwin-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; x86_64-apple-darwin-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; x86_64-apple-darwin-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+; aarch64-LABEL: define ptr @load_relative_3(
+; aarch64-SAME: i64 [[OFFSET:%.*]]) {
+; aarch64-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; aarch64-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; aarch64-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+; x86_64-LABEL: define ptr @load_relative_3(
+; x86_64-SAME: i64 [[OFFSET:%.*]]) {
+; x86_64-NEXT:    [[RELTABLE_SHIFT:%.*]] = shl i64 [[OFFSET]], 2
+; x86_64-NEXT:    [[RELTABLE_INTRINSIC:%.*]] = call ptr @llvm.load.relative.i64(ptr @load_relative_3.table.rel, i64 [[RELTABLE_SHIFT]])
+; x86_64-NEXT:    ret ptr [[RELTABLE_INTRINSIC]]
+;
+  %gep = getelementptr inbounds [3 x ptr], ptr @load_relative_3.table, i64 0, i64 %offset
+  %load = load ptr, ptr %gep
+  ret ptr %load
+}
+
 ;.
 ; x86_64-apple-darwin: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
 ;.
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
index e460f558f472..8dc88f7b9671 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -passes=slp-vectorizer,instcombine -pass-remarks-output=%t | FileCheck %s
+; RUN: opt < %s -S -passes=slp-vectorizer -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
-; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='slp-vectorizer,instcombine' -pass-remarks-output=%t | FileCheck %s
+; RUN: opt < %s -S -aa-pipeline=basic-aa -passes='slp-vectorizer' -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
@@ -16,10 +16,10 @@ target triple = "aarch64--linux-gnu"
 
 define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-LABEL: @gather_multiple_use(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i64 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i64 2
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], splat (i32 15)
 ; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], splat (i32 65537)
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <4 x i32> [[TMP6]], splat (i32 65535)
@@ -57,22 +57,26 @@ define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
 @data = global [6 x [258 x i8]] zeroinitializer, align 1
 define void @gather_load(ptr noalias %ptr) {
 ; CHECK-LABEL: @gather_load(
-; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 8
-; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 258), align 1
+; CHECK-NEXT:    [[ARRAYIDX182:%.*]] = getelementptr inbounds i16, ptr [[PTR:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX183:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX185:%.*]] = getelementptr inbounds i16, ptr [[PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX149:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 1, i64 0
+; CHECK-NEXT:    [[L0:%.*]] = load i8, ptr [[ARRAYIDX149]], align 1
 ; CHECK-NEXT:    [[CONV150:%.*]] = zext i8 [[L0]] to i16
-; CHECK-NEXT:    [[ADD152:%.*]] = add nuw nsw i16 [[CONV150]], 10
-; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 517), align 1
+; CHECK-NEXT:    [[ADD152:%.*]] = add i16 10, [[CONV150]]
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 2, i64 1
+; CHECK-NEXT:    [[L1:%.*]] = load i8, ptr [[ARRAYIDX155]], align 1
 ; CHECK-NEXT:    [[CONV156:%.*]] = zext i8 [[L1]] to i16
-; CHECK-NEXT:    [[ADD158:%.*]] = add nuw nsw i16 [[CONV156]], 20
-; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 776), align 1
+; CHECK-NEXT:    [[ADD158:%.*]] = add i16 20, [[CONV156]]
+; CHECK-NEXT:    [[ARRAYIDX161:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 3, i64 2
+; CHECK-NEXT:    [[L2:%.*]] = load i8, ptr [[ARRAYIDX161]], align 1
 ; CHECK-NEXT:    [[CONV162:%.*]] = zext i8 [[L2]] to i16
-; CHECK-NEXT:    [[ADD164:%.*]] = add nuw nsw i16 [[CONV162]], 30
-; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr getelementptr inbounds nuw (i8, ptr @data, i64 1035), align 1
+; CHECK-NEXT:    [[ADD164:%.*]] = add i16 30, [[CONV162]]
+; CHECK-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds [6 x [258 x i8]], ptr @data, i64 0, i64 4, i64 3
+; CHECK-NEXT:    [[L3:%.*]] = load i8, ptr [[ARRAYIDX167]], align 1
 ; CHECK-NEXT:    [[CONV168:%.*]] = zext i8 [[L3]] to i16
-; CHECK-NEXT:    [[ADD170:%.*]] = add nuw nsw i16 [[CONV168]], 40
+; CHECK-NEXT:    [[ADD170:%.*]] = add i16 40, [[CONV168]]
 ; CHECK-NEXT:    store i16 [[ADD152]], ptr [[ARRAYIDX182]], align 2
 ; CHECK-NEXT:    store i16 [[ADD158]], ptr [[ARRAYIDX183]], align 2
 ; CHECK-NEXT:    store i16 [[ADD164]], ptr [[ARRAYIDX184]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
index 900d5f293b5b..f83628ed17b6 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-reduce.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer,dce,instcombine < %s | FileCheck %s --check-prefix=GENERIC
-; RUN: opt -S -mcpu=kryo -passes=slp-vectorizer,dce,instcombine < %s | FileCheck %s --check-prefix=KRYO
+; RUN: opt -S -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=GENERIC
+; RUN: opt -S -mcpu=kryo -passes=slp-vectorizer < %s | FileCheck %s --check-prefix=KRYO
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -36,57 +36,49 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; GENERIC-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
-; GENERIC-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
+; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
+; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]
 ; GENERIC-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
-; GENERIC-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
+; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]
 ; GENERIC-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
-; GENERIC-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
+; GENERIC-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
+; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]
 ; GENERIC-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
-; GENERIC-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
+; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
+; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]
 ; GENERIC-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
-; GENERIC-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
+; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
+; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]
 ; GENERIC-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
-; GENERIC-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
+; GENERIC-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
+; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]
 ; GENERIC-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
-; GENERIC-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
+; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
+; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]
 ; GENERIC-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
-; GENERIC-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
-; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
+; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
+; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]
 ; GENERIC-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
 ; GENERIC-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP28]] to i32
 ; GENERIC-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
@@ -109,57 +101,49 @@ define i32 @gather_reduce_8x16_i32(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; KRYO-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
-; KRYO-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
-; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
+; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
+; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i32 [[TMP5]]
 ; KRYO-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
-; KRYO-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
+; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP8]]
 ; KRYO-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
-; KRYO-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
+; KRYO-NEXT:    [[TMP9:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
+; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP9]]
 ; KRYO-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
-; KRYO-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
-; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
+; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
+; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP11]]
 ; KRYO-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
-; KRYO-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
-; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
+; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
+; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP14]]
 ; KRYO-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
-; KRYO-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
-; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
+; KRYO-NEXT:    [[TMP15:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
+; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP15]]
 ; KRYO-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
-; KRYO-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
-; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
+; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
+; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP17]]
 ; KRYO-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
-; KRYO-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
-; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
+; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
+; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i32 [[TMP20]]
 ; KRYO-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
 ; KRYO-NEXT:    [[CONV65:%.*]] = zext i16 [[TMP28]] to i32
 ; KRYO-NEXT:    [[ADD66]] = add nsw i32 [[ADD57]], [[CONV65]]
@@ -293,55 +277,55 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; GENERIC-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; GENERIC-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; GENERIC-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; GENERIC-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; GENERIC-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; GENERIC-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; GENERIC-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
-; GENERIC-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
+; GENERIC-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]
+; GENERIC-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
 ; GENERIC-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
 ; GENERIC-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; GENERIC-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
+; GENERIC-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; GENERIC-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
 ; GENERIC-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
 ; GENERIC-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; GENERIC-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; GENERIC-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
+; GENERIC-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
 ; GENERIC-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
 ; GENERIC-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; GENERIC-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; GENERIC-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
+; GENERIC-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
 ; GENERIC-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
 ; GENERIC-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; GENERIC-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; GENERIC-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
+; GENERIC-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
 ; GENERIC-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
 ; GENERIC-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; GENERIC-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; GENERIC-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
+; GENERIC-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
 ; GENERIC-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
 ; GENERIC-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; GENERIC-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; GENERIC-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
+; GENERIC-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
 ; GENERIC-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
 ; GENERIC-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; GENERIC-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; GENERIC-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
+; GENERIC-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
 ; GENERIC-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
 ; GENERIC-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
 ; GENERIC-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
@@ -366,55 +350,55 @@ define i32 @gather_reduce_8x16_i64(ptr nocapture readonly %a, ptr nocapture read
 ; KRYO-NEXT:    [[I_0103:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[SUM_0102:%.*]] = phi i32 [ [[ADD66]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
 ; KRYO-NEXT:    [[A_ADDR_0101:%.*]] = phi ptr [ [[INCDEC_PTR58:%.*]], [[FOR_BODY]] ], [ [[A:%.*]], [[FOR_BODY_PREHEADER]] ]
-; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds nuw i8, ptr [[A_ADDR_0101]], i64 16
+; KRYO-NEXT:    [[INCDEC_PTR58]] = getelementptr inbounds i16, ptr [[A_ADDR_0101]], i64 8
 ; KRYO-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr [[A_ADDR_0101]], align 2
 ; KRYO-NEXT:    [[TMP1:%.*]] = zext <8 x i16> [[TMP0]] to <8 x i32>
 ; KRYO-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr [[B:%.*]], align 2
 ; KRYO-NEXT:    [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
-; KRYO-NEXT:    [[TMP4:%.*]] = sub nsw <8 x i32> [[TMP1]], [[TMP3]]
-; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i64 0
+; KRYO-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP1]], [[TMP3]]
+; KRYO-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP4]], i32 0
 ; KRYO-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[G:%.*]], i64 [[TMP6]]
 ; KRYO-NEXT:    [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; KRYO-NEXT:    [[CONV3:%.*]] = zext i16 [[TMP7]] to i32
-; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[SUM_0102]], [[CONV3]]
-; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i64 1
+; KRYO-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV3]], [[SUM_0102]]
+; KRYO-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP4]], i32 1
 ; KRYO-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP9]]
 ; KRYO-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
 ; KRYO-NEXT:    [[CONV11:%.*]] = zext i16 [[TMP10]] to i32
 ; KRYO-NEXT:    [[ADD12:%.*]] = add nsw i32 [[ADD]], [[CONV11]]
-; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i64 2
+; KRYO-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[TMP4]], i32 2
 ; KRYO-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP12]]
 ; KRYO-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX19]], align 2
 ; KRYO-NEXT:    [[CONV20:%.*]] = zext i16 [[TMP13]] to i32
 ; KRYO-NEXT:    [[ADD21:%.*]] = add nsw i32 [[ADD12]], [[CONV20]]
-; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i64 3
+; KRYO-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP4]], i32 3
 ; KRYO-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP15]]
 ; KRYO-NEXT:    [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; KRYO-NEXT:    [[CONV29:%.*]] = zext i16 [[TMP16]] to i32
 ; KRYO-NEXT:    [[ADD30:%.*]] = add nsw i32 [[ADD21]], [[CONV29]]
-; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i64 4
+; KRYO-NEXT:    [[TMP17:%.*]] = extractelement <8 x i32> [[TMP4]], i32 4
 ; KRYO-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP17]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP18]]
 ; KRYO-NEXT:    [[TMP19:%.*]] = load i16, ptr [[ARRAYIDX37]], align 2
 ; KRYO-NEXT:    [[CONV38:%.*]] = zext i16 [[TMP19]] to i32
 ; KRYO-NEXT:    [[ADD39:%.*]] = add nsw i32 [[ADD30]], [[CONV38]]
-; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i64 5
+; KRYO-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP4]], i32 5
 ; KRYO-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX46:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP21]]
 ; KRYO-NEXT:    [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX46]], align 2
 ; KRYO-NEXT:    [[CONV47:%.*]] = zext i16 [[TMP22]] to i32
 ; KRYO-NEXT:    [[ADD48:%.*]] = add nsw i32 [[ADD39]], [[CONV47]]
-; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i64 6
+; KRYO-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP4]], i32 6
 ; KRYO-NEXT:    [[TMP24:%.*]] = sext i32 [[TMP23]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX55:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP24]]
 ; KRYO-NEXT:    [[TMP25:%.*]] = load i16, ptr [[ARRAYIDX55]], align 2
 ; KRYO-NEXT:    [[CONV56:%.*]] = zext i16 [[TMP25]] to i32
 ; KRYO-NEXT:    [[ADD57:%.*]] = add nsw i32 [[ADD48]], [[CONV56]]
-; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i64 7
+; KRYO-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP4]], i32 7
 ; KRYO-NEXT:    [[TMP27:%.*]] = sext i32 [[TMP26]] to i64
 ; KRYO-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i16, ptr [[G]], i64 [[TMP27]]
 ; KRYO-NEXT:    [[TMP28:%.*]] = load i16, ptr [[ARRAYIDX64]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index c43b79e138a3..89e133bb1c6a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer' -slp-threshold=-7 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 ; These tests check that we remove from consideration pairs of seed
@@ -46,9 +46,9 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[Z:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -58,29 +58,25 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD16]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i64 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = zext nneg i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP6]]
 ; CHECK-NEXT:    [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[SUM_032]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP5]], i64 1
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP7]]
 ; CHECK-NEXT:    [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i32> [[TMP4]], [[TMP2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
-; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP10]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP9]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
-; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
 ; CHECK-NEXT:    [[ADD16]] = add nsw i32 [[ADD11]], [[T12]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1
@@ -143,7 +139,7 @@ define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
 ; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[Y:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[Y:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup.loopexit:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
@@ -153,26 +149,22 @@ define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[OP_RDX]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[T4:%.*]] = shl nuw nsw i32 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i64 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = zext nneg i32 [[TMP4]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[G:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
-; CHECK-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP9]]
 ; CHECK-NEXT:    [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
 ; CHECK-NEXT:    [[T11:%.*]] = add nsw i32 [[T4]], [[Z:%.*]]
-; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[T11]] to i64
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i64 [[TMP9]]
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[T11]]
 ; CHECK-NEXT:    [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[T10]], i64 2
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[T12]], i64 3
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP12]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[T10]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[T12]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP8]], <2 x i32> [[TMP6]], i64 0)
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP13]])
 ; CHECK-NEXT:    [[OP_RDX]] = add i32 [[TMP14]], [[SUM_032]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[INDVARS_IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
index 5808d64d925c..ac476c521a59 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr2.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;test_i16_extend NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer,dce,instcombine -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
+; RUN: opt -S -mtriple=aarch64--linux-gnu -passes='slp-vectorizer' -slp-threshold=-5 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 
@@ -25,42 +25,42 @@ define void @test_i16_extend(ptr %p.1, ptr %p.2, i32 %idx.i32) {
 ; CHECK-LABEL: @test_i16_extend(
 ; CHECK-NEXT:    [[P_0:%.*]] = load ptr, ptr @global, align 8
 ; CHECK-NEXT:    [[IDX_0:%.*]] = zext i32 [[IDX_I32:%.*]] to i64
-; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds nuw i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
-; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds nuw i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T53:%.*]] = getelementptr inbounds i16, ptr [[P_1:%.*]], i64 [[IDX_0]]
+; CHECK-NEXT:    [[T56:%.*]] = getelementptr inbounds i16, ptr [[P_2:%.*]], i64 [[IDX_0]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[T53]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i16> [[TMP1]] to <8 x i32>
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr [[T56]], align 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i16> [[TMP3]] to <8 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = sub <8 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[T60:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[L_1:%.*]] = load i32, ptr [[T60]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i32> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
 ; CHECK-NEXT:    [[T71:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[L_2:%.*]] = load i32, ptr [[T71]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i64 2
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
 ; CHECK-NEXT:    [[T82:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[L_3:%.*]] = load i32, ptr [[T82]], align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i64 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[TMP5]], i32 3
 ; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
 ; CHECK-NEXT:    [[T93:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[L_4:%.*]] = load i32, ptr [[T93]], align 4
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i64 4
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
 ; CHECK-NEXT:    [[T104:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[L_5:%.*]] = load i32, ptr [[T104]], align 4
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i64 5
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
 ; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
 ; CHECK-NEXT:    [[T115:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP17]]
 ; CHECK-NEXT:    [[L_6:%.*]] = load i32, ptr [[T115]], align 4
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i64 6
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP5]], i32 6
 ; CHECK-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP18]] to i64
 ; CHECK-NEXT:    [[T126:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[L_7:%.*]] = load i32, ptr [[T126]], align 4
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i64 7
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP5]], i32 7
 ; CHECK-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP20]] to i64
 ; CHECK-NEXT:    [[T137:%.*]] = getelementptr inbounds i32, ptr [[P_0]], i64 [[TMP21]]
 ; CHECK-NEXT:    [[L_8:%.*]] = load i32, ptr [[T137]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
index f138065101c4..6c5220d13b7a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loadorder.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=slp-vectorizer,instcombine -mtriple=aarch64--linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64--linux-gnu < %s | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64"
@@ -133,7 +133,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -165,7 +165,7 @@ define i16 @reduce_blockstrided2(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[ARRAYIDX28]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM15]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[ARRAYIDX32]], align 2
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i16, ptr [[ARRAYIDX33]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX36]], align 2
@@ -254,9 +254,9 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-LABEL: @reduce_blockstrided3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[L0:%.*]] = load i16, ptr [[X:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 1
 ; CHECK-NEXT:    [[L1:%.*]] = load i16, ptr [[ARRAYIDX1]], align 2
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 2
 ; CHECK-NEXT:    [[L2:%.*]] = load i16, ptr [[ARRAYIDX2]], align 2
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM]]
@@ -269,10 +269,12 @@ define i16 @reduce_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[ADD8]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i16, ptr [[X]], i64 [[IDXPROM9]]
 ; CHECK-NEXT:    [[L6:%.*]] = load i16, ptr [[ARRAYIDX10]], align 2
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[STRIDE]], 3
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
 ; CHECK-NEXT:    [[L8:%.*]] = load i16, ptr [[Y:%.*]], align 2
-; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 1
 ; CHECK-NEXT:    [[L9:%.*]] = load i16, ptr [[ARRAYIDX15]], align 2
-; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds nuw i8, ptr [[Y]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 2
 ; CHECK-NEXT:    [[L10:%.*]] = load i16, ptr [[ARRAYIDX16]], align 2
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i16, ptr [[Y]], i64 [[IDXPROM]]
 ; CHECK-NEXT:    [[L12:%.*]] = load i16, ptr [[ARRAYIDX20]], align 2
@@ -347,9 +349,9 @@ define i16 @reduce_blockstrided4(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[Y]], align 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr [[ARRAYIDX20]], align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i16> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i16> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <8 x i16> [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP6]])
 ; CHECK-NEXT:    ret i16 [[TMP7]]
 ;
@@ -414,33 +416,128 @@ define i32 @reduce_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
+; TODO: Dead code must be removed below.
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV_2:%.*]] = zext i8 [[TMP32]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; CHECK-NEXT:    [[CONV2_2:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; CHECK-NEXT:    [[CONV4_2:%.*]] = zext i8 [[TMP34]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; CHECK-NEXT:    [[CONV6_2:%.*]] = zext i8 [[TMP35]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 1
+; CHECK-NEXT:    [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; CHECK-NEXT:    [[CONV9_2:%.*]] = zext i8 [[TMP36]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; CHECK-NEXT:    [[CONV11_2:%.*]] = zext i8 [[TMP37]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 5
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; CHECK-NEXT:    [[CONV14_2:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 5
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; CHECK-NEXT:    [[CONV16_2:%.*]] = zext i8 [[TMP7]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; CHECK-NEXT:    [[CONV21_2:%.*]] = zext i8 [[TMP8]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 2
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; CHECK-NEXT:    [[CONV23_2:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 6
+; CHECK-NEXT:    [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; CHECK-NEXT:    [[CONV26_2:%.*]] = zext i8 [[TMP41]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 6
+; CHECK-NEXT:    [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; CHECK-NEXT:    [[CONV28_2:%.*]] = zext i8 [[TMP42]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 3
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; CHECK-NEXT:    [[CONV33_2:%.*]] = zext i8 [[TMP43]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 3
+; CHECK-NEXT:    [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; CHECK-NEXT:    [[CONV35_2:%.*]] = zext i8 [[TMP44]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 7
+; CHECK-NEXT:    [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; CHECK-NEXT:    [[CONV38_2:%.*]] = zext i8 [[TMP14]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 7
+; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; CHECK-NEXT:    [[CONV40_2:%.*]] = zext i8 [[TMP15]] to i32
+; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[ADD_PTR_2]], align 1
+; CHECK-NEXT:    [[CONV_3:%.*]] = zext i8 [[TMP16]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i8, ptr [[ADD_PTR64_2]], align 1
+; CHECK-NEXT:    [[CONV2_3:%.*]] = zext i8 [[TMP17]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[TMP48:%.*]] = load i8, ptr [[ARRAYIDX3_3]], align 1
+; CHECK-NEXT:    [[CONV4_3:%.*]] = zext i8 [[TMP48]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[TMP49:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
+; CHECK-NEXT:    [[CONV6_3:%.*]] = zext i8 [[TMP49]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 1
+; CHECK-NEXT:    [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; CHECK-NEXT:    [[CONV9_3:%.*]] = zext i8 [[TMP50]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 1
+; CHECK-NEXT:    [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; CHECK-NEXT:    [[CONV11_3:%.*]] = zext i8 [[TMP51]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 5
+; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX13_3]], align 1
+; CHECK-NEXT:    [[CONV14_3:%.*]] = zext i8 [[TMP22]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 5
+; CHECK-NEXT:    [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; CHECK-NEXT:    [[CONV16_3:%.*]] = zext i8 [[TMP23]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 2
+; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; CHECK-NEXT:    [[CONV21_3:%.*]] = zext i8 [[TMP24]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; CHECK-NEXT:    [[CONV23_3:%.*]] = zext i8 [[TMP25]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 6
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX25_3]], align 1
+; CHECK-NEXT:    [[CONV26_3:%.*]] = zext i8 [[TMP26]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 6
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; CHECK-NEXT:    [[CONV28_3:%.*]] = zext i8 [[TMP27]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; CHECK-NEXT:    [[CONV33_3:%.*]] = zext i8 [[TMP28]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; CHECK-NEXT:    [[CONV35_3:%.*]] = zext i8 [[TMP29]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 7
+; CHECK-NEXT:    [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX37_3]], align 1
+; CHECK-NEXT:    [[CONV38_3:%.*]] = zext i8 [[TMP30]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 7
+; CHECK-NEXT:    [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; CHECK-NEXT:    [[CONV40_3:%.*]] = zext i8 [[TMP31]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP6]], <16 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP8]], <16 x i8> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP38:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP39:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP38]], <4 x i8> [[TMP4]], i64 4)
+; CHECK-NEXT:    [[TMP40:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP39]], <4 x i8> [[TMP1]], i64 8)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP40]], <4 x i8> [[TMP5]], i64 12)
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext <16 x i8> [[TMP10]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP12]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP14]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP16]], <16 x i8> [[TMP17]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP45:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP46:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP45]], <4 x i8> [[TMP12]], i64 4)
+; CHECK-NEXT:    [[TMP47:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP46]], <4 x i8> [[TMP3]], i64 8)
+; CHECK-NEXT:    [[TMP18:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP47]], <4 x i8> [[TMP13]], i64 12)
 ; CHECK-NEXT:    [[TMP19:%.*]] = zext <16 x i8> [[TMP18]] to <16 x i32>
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw nsw <16 x i32> [[TMP11]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = mul <16 x i32> [[TMP11]], [[TMP19]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP20]])
 ; CHECK-NEXT:    ret i32 [[TMP21]]
 ;
@@ -677,7 +774,7 @@ entry:
 define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocapture noundef readonly %y, ptr nocapture noundef writeonly %z, i32 noundef %stride) {
 ; CHECK-LABEL: @store_blockstrided3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[X:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD4:%.*]] = add nsw i32 [[STRIDE:%.*]], 1
 ; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[ADD4]] to i64
@@ -696,7 +793,7 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ADD26:%.*]] = add nsw i32 [[MUL21]], 1
 ; CHECK-NEXT:    [[IDXPROM27:%.*]] = sext i32 [[ADD26]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds nuw i8, ptr [[Y:%.*]], i64 8
+; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i32, ptr [[Y:%.*]], i64 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX35]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM5]]
 ; CHECK-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM11]]
@@ -705,20 +802,21 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[ARRAYIDX60:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM23]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX60]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX65:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 [[IDXPROM27]]
-; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds nuw i8, ptr [[Z:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds i32, ptr [[Z:%.*]], i64 1
 ; CHECK-NEXT:    [[MUL73:%.*]] = mul nsw i32 [[TMP3]], [[TMP0]]
-; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 24
+; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 6
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i32>, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x i32>, ptr [[ARRAYIDX6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <2 x i32>, ptr [[Y]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = load <2 x i32>, ptr [[ARRAYIDX41]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nsw <2 x i32> [[TMP8]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nsw <2 x i32> [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> [[TMP11]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[ARRAYIDX84:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 28
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[ARRAYIDX84:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 7
 ; CHECK-NEXT:    [[MUL81:%.*]] = mul nsw i32 [[TMP4]], [[TMP1]]
 ; CHECK-NEXT:    [[MUL87:%.*]] = mul nsw i32 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds nuw i8, ptr [[Z]], i64 44
+; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i32, ptr [[Z]], i64 11
 ; CHECK-NEXT:    [[TMP17:%.*]] = load <2 x i32>, ptr [[ARRAYIDX28]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = load <2 x i32>, ptr [[ARRAYIDX64]], align 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = load <2 x i32>, ptr [[ARRAYIDX49]], align 4
@@ -727,9 +825,10 @@ define void @store_blockstrided3(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    store <4 x i32> [[TMP12]], ptr [[ARRAYIDX72]], align 4
 ; CHECK-NEXT:    store i32 [[MUL81]], ptr [[ARRAYIDX76]], align 4
 ; CHECK-NEXT:    store i32 [[MUL87]], ptr [[ARRAYIDX88]], align 4
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nsw <2 x i32> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nsw <2 x i32> [[TMP16]], [[TMP18]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> [[TMP21]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x i32> [[TMP17]], <2 x i32> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nsw <4 x i32> [[TMP21]], [[TMP22]]
+; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 ; CHECK-NEXT:    store <4 x i32> [[TMP19]], ptr [[ARRAYIDX84]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -837,9 +936,10 @@ define void @store_blockstrided4(ptr nocapture noundef readonly %x, ptr nocaptur
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i16>, ptr [[Y]], align 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr [[ARRAYIDX20]], align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i16> [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i16> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[TMP2]], <4 x i16> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <8 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP7]], <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 ; CHECK-NEXT:    store <8 x i16> [[TMP6]], ptr [[DST0:%.*]], align 2
 ; CHECK-NEXT:    ret void
 ;
@@ -912,35 +1012,132 @@ define void @store_blockstrided4x4(ptr nocapture noundef readonly %p1, i32 nound
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[OFF1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[OFF2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
-; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0:%.*]], i64 16
-; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 32
-; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds nuw i8, ptr [[DST0]], i64 48
+; TODO: Dead code must be removed below.
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i8, ptr [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV_2:%.*]] = zext i8 [[TMP32]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP33:%.*]] = load i8, ptr [[ADD_PTR64_1]], align 1
+; CHECK-NEXT:    [[CONV2_2:%.*]] = zext i8 [[TMP33]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[TMP34:%.*]] = load i8, ptr [[ARRAYIDX3_2]], align 1
+; CHECK-NEXT:    [[CONV4_2:%.*]] = zext i8 [[TMP34]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[TMP35:%.*]] = load i8, ptr [[ARRAYIDX5_2]], align 1
+; CHECK-NEXT:    [[CONV6_2:%.*]] = zext i8 [[TMP35]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 1
+; CHECK-NEXT:    [[TMP36:%.*]] = load i8, ptr [[ARRAYIDX8_2]], align 1
+; CHECK-NEXT:    [[CONV9_2:%.*]] = zext i8 [[TMP36]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = load i8, ptr [[ARRAYIDX10_2]], align 1
+; CHECK-NEXT:    [[CONV11_2:%.*]] = zext i8 [[TMP37]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 5
+; CHECK-NEXT:    [[TMP38:%.*]] = load i8, ptr [[ARRAYIDX13_2]], align 1
+; CHECK-NEXT:    [[CONV14_2:%.*]] = zext i8 [[TMP38]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 5
+; CHECK-NEXT:    [[TMP39:%.*]] = load i8, ptr [[ARRAYIDX15_2]], align 1
+; CHECK-NEXT:    [[CONV16_2:%.*]] = zext i8 [[TMP39]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 2
+; CHECK-NEXT:    [[TMP40:%.*]] = load i8, ptr [[ARRAYIDX20_2]], align 1
+; CHECK-NEXT:    [[CONV21_2:%.*]] = zext i8 [[TMP40]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 2
+; CHECK-NEXT:    [[TMP41:%.*]] = load i8, ptr [[ARRAYIDX22_2]], align 1
+; CHECK-NEXT:    [[CONV23_2:%.*]] = zext i8 [[TMP41]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 6
+; CHECK-NEXT:    [[TMP42:%.*]] = load i8, ptr [[ARRAYIDX25_2]], align 1
+; CHECK-NEXT:    [[CONV26_2:%.*]] = zext i8 [[TMP42]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 6
+; CHECK-NEXT:    [[TMP43:%.*]] = load i8, ptr [[ARRAYIDX27_2]], align 1
+; CHECK-NEXT:    [[CONV28_2:%.*]] = zext i8 [[TMP43]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 3
+; CHECK-NEXT:    [[TMP44:%.*]] = load i8, ptr [[ARRAYIDX32_2]], align 1
+; CHECK-NEXT:    [[CONV33_2:%.*]] = zext i8 [[TMP44]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 3
+; CHECK-NEXT:    [[TMP45:%.*]] = load i8, ptr [[ARRAYIDX34_2]], align 1
+; CHECK-NEXT:    [[CONV35_2:%.*]] = zext i8 [[TMP45]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 7
+; CHECK-NEXT:    [[TMP46:%.*]] = load i8, ptr [[ARRAYIDX37_2]], align 1
+; CHECK-NEXT:    [[CONV38_2:%.*]] = zext i8 [[TMP46]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 7
+; CHECK-NEXT:    [[TMP47:%.*]] = load i8, ptr [[ARRAYIDX39_2]], align 1
+; CHECK-NEXT:    [[CONV40_2:%.*]] = zext i8 [[TMP47]] to i32
+; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[TMP48:%.*]] = load i8, ptr [[ADD_PTR_2]], align 1
+; CHECK-NEXT:    [[CONV_3:%.*]] = zext i8 [[TMP48]] to i32
+; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
+; CHECK-NEXT:    [[TMP49:%.*]] = load i8, ptr [[ADD_PTR64_2]], align 1
+; CHECK-NEXT:    [[CONV2_3:%.*]] = zext i8 [[TMP49]] to i32
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[TMP50:%.*]] = load i8, ptr [[ARRAYIDX3_3]], align 1
+; CHECK-NEXT:    [[CONV4_3:%.*]] = zext i8 [[TMP50]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[TMP51:%.*]] = load i8, ptr [[ARRAYIDX5_3]], align 1
+; CHECK-NEXT:    [[CONV6_3:%.*]] = zext i8 [[TMP51]] to i32
+; CHECK-NEXT:    [[ARRAYIDX8_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 1
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[ARRAYIDX8_3]], align 1
+; CHECK-NEXT:    [[CONV9_3:%.*]] = zext i8 [[TMP20]] to i32
+; CHECK-NEXT:    [[ARRAYIDX10_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 1
+; CHECK-NEXT:    [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX10_3]], align 1
+; CHECK-NEXT:    [[CONV11_3:%.*]] = zext i8 [[TMP21]] to i32
+; CHECK-NEXT:    [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 5
+; CHECK-NEXT:    [[TMP22:%.*]] = load i8, ptr [[ARRAYIDX13_3]], align 1
+; CHECK-NEXT:    [[CONV14_3:%.*]] = zext i8 [[TMP22]] to i32
+; CHECK-NEXT:    [[ARRAYIDX15_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 5
+; CHECK-NEXT:    [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX15_3]], align 1
+; CHECK-NEXT:    [[CONV16_3:%.*]] = zext i8 [[TMP23]] to i32
+; CHECK-NEXT:    [[ARRAYIDX20_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 2
+; CHECK-NEXT:    [[TMP24:%.*]] = load i8, ptr [[ARRAYIDX20_3]], align 1
+; CHECK-NEXT:    [[CONV21_3:%.*]] = zext i8 [[TMP24]] to i32
+; CHECK-NEXT:    [[ARRAYIDX22_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 2
+; CHECK-NEXT:    [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX22_3]], align 1
+; CHECK-NEXT:    [[CONV23_3:%.*]] = zext i8 [[TMP25]] to i32
+; CHECK-NEXT:    [[ARRAYIDX25_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 6
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[ARRAYIDX25_3]], align 1
+; CHECK-NEXT:    [[CONV26_3:%.*]] = zext i8 [[TMP26]] to i32
+; CHECK-NEXT:    [[ARRAYIDX27_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 6
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX27_3]], align 1
+; CHECK-NEXT:    [[CONV28_3:%.*]] = zext i8 [[TMP27]] to i32
+; CHECK-NEXT:    [[ARRAYIDX32_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 3
+; CHECK-NEXT:    [[TMP28:%.*]] = load i8, ptr [[ARRAYIDX32_3]], align 1
+; CHECK-NEXT:    [[CONV33_3:%.*]] = zext i8 [[TMP28]] to i32
+; CHECK-NEXT:    [[ARRAYIDX34_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 3
+; CHECK-NEXT:    [[TMP29:%.*]] = load i8, ptr [[ARRAYIDX34_3]], align 1
+; CHECK-NEXT:    [[CONV35_3:%.*]] = zext i8 [[TMP29]] to i32
+; CHECK-NEXT:    [[ARRAYIDX37_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 7
+; CHECK-NEXT:    [[TMP30:%.*]] = load i8, ptr [[ARRAYIDX37_3]], align 1
+; CHECK-NEXT:    [[CONV38_3:%.*]] = zext i8 [[TMP30]] to i32
+; CHECK-NEXT:    [[ARRAYIDX39_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 7
+; CHECK-NEXT:    [[TMP31:%.*]] = load i8, ptr [[ARRAYIDX39_3]], align 1
+; CHECK-NEXT:    [[CONV40_3:%.*]] = zext i8 [[TMP31]] to i32
+; CHECK-NEXT:    [[DST4:%.*]] = getelementptr inbounds i32, ptr [[DST0:%.*]], i64 4
+; CHECK-NEXT:    [[DST8:%.*]] = getelementptr inbounds i32, ptr [[DST0]], i64 8
+; CHECK-NEXT:    [[DST12:%.*]] = getelementptr inbounds i32, ptr [[DST0]], i64 12
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext <4 x i8> [[TMP0]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i32> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = zext <4 x i8> [[TMP7]] to <4 x i32>
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP6]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i8>, ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = zext <4 x i8> [[TMP10]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_1]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = zext <4 x i8> [[TMP12]] to <4 x i32>
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw nsw <4 x i32> [[TMP11]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul <4 x i32> [[TMP11]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load <4 x i8>, ptr [[ADD_PTR64]], align 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = zext <4 x i8> [[TMP15]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP17:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_1]], align 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = zext <4 x i8> [[TMP17]] to <4 x i32>
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw nsw <4 x i32> [[TMP16]], [[TMP18]]
+; CHECK-NEXT:    [[TMP19:%.*]] = mul <4 x i32> [[TMP16]], [[TMP18]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[DST0]], align 4
 ; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[DST4]], align 4
 ; CHECK-NEXT:    store <4 x i32> [[TMP14]], ptr [[DST8]], align 4
@@ -1198,20 +1395,20 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[ST1:%.*]] to i64
 ; CHECK-NEXT:    [[IDX_EXT63:%.*]] = sext i32 [[ST2:%.*]] to i64
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds nuw i8, ptr [[P1:%.*]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds nuw i8, ptr [[P2:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i8, ptr [[P1:%.*]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i8, ptr [[P2:%.*]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_1:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_1]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 4
 ; CHECK-NEXT:    [[ADD_PTR_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_1]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR64_2:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_1]], i64 [[IDX_EXT63]]
-; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR_2]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds nuw i8, ptr [[ADD_PTR64_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX3_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR_2]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX5_3:%.*]] = getelementptr inbounds i8, ptr [[ADD_PTR64_2]], i64 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr [[P1]], align 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[P2]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3]], align 1
@@ -1225,33 +1422,29 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_2]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_2]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ADD_PTR_2]], align 1
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP0]], <4 x i8> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <16 x i8> [[TMP15]], <16 x i8> [[TMP16]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP13:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP0]], i64 0)
+; CHECK-NEXT:    [[TMP14:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP13]], <4 x i8> [[TMP4]], i64 4)
+; CHECK-NEXT:    [[TMP15:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP14]], <4 x i8> [[TMP8]], i64 8)
+; CHECK-NEXT:    [[TMP17:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP15]], <4 x i8> [[TMP12]], i64 12)
 ; CHECK-NEXT:    [[TMP18:%.*]] = zext <16 x i8> [[TMP17]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = load <4 x i8>, ptr [[ADD_PTR64_2]], align 1
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP5]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <4 x i8> [[TMP9]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <16 x i8> [[TMP20]], <16 x i8> [[TMP21]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x i8> [[TMP19]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <16 x i8> [[TMP22]], <16 x i8> [[TMP23]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP22:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP1]], i64 0)
+; CHECK-NEXT:    [[TMP20:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP22]], <4 x i8> [[TMP5]], i64 4)
+; CHECK-NEXT:    [[TMP21:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP20]], <4 x i8> [[TMP9]], i64 8)
+; CHECK-NEXT:    [[TMP24:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP21]], <4 x i8> [[TMP19]], i64 12)
 ; CHECK-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP26:%.*]] = sub nsw <16 x i32> [[TMP18]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load <4 x i8>, ptr [[ARRAYIDX3_3]], align 1
-; CHECK-NEXT:    [[TMP28:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP29:%.*]] = shufflevector <4 x i8> [[TMP10]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP30:%.*]] = shufflevector <16 x i8> [[TMP28]], <16 x i8> [[TMP29]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP31:%.*]] = shufflevector <4 x i8> [[TMP27]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <16 x i8> [[TMP30]], <16 x i8> [[TMP31]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP29:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP30:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP29]], <4 x i8> [[TMP6]], i64 4)
+; CHECK-NEXT:    [[TMP28:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP30]], <4 x i8> [[TMP10]], i64 8)
+; CHECK-NEXT:    [[TMP32:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP28]], <4 x i8> [[TMP27]], i64 12)
 ; CHECK-NEXT:    [[TMP33:%.*]] = zext <16 x i8> [[TMP32]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP34:%.*]] = load <4 x i8>, ptr [[ARRAYIDX5_3]], align 1
-; CHECK-NEXT:    [[TMP35:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <16 x i8> [[TMP35]], <16 x i8> [[TMP36]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <4 x i8> [[TMP34]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <16 x i8> [[TMP37]], <16 x i8> [[TMP38]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    [[TMP35:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> poison, <4 x i8> [[TMP3]], i64 0)
+; CHECK-NEXT:    [[TMP36:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP35]], <4 x i8> [[TMP7]], i64 4)
+; CHECK-NEXT:    [[TMP37:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP36]], <4 x i8> [[TMP11]], i64 8)
+; CHECK-NEXT:    [[TMP39:%.*]] = call <16 x i8> @llvm.vector.insert.v16i8.v4i8(<16 x i8> [[TMP37]], <4 x i8> [[TMP34]], i64 12)
 ; CHECK-NEXT:    [[TMP40:%.*]] = zext <16 x i8> [[TMP39]] to <16 x i32>
 ; CHECK-NEXT:    [[TMP41:%.*]] = sub nsw <16 x i32> [[TMP33]], [[TMP40]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = shl nsw <16 x i32> [[TMP41]], splat (i32 16)
@@ -1259,11 +1452,11 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP44:%.*]] = shufflevector <16 x i32> [[TMP43]], <16 x i32> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 ; CHECK-NEXT:    [[TMP45:%.*]] = add nsw <16 x i32> [[TMP43]], [[TMP44]]
 ; CHECK-NEXT:    [[TMP46:%.*]] = sub nsw <16 x i32> [[TMP43]], [[TMP44]]
-; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i32> [[TMP46]], <16 x i32> [[TMP45]], <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 4, i32 21, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 12, i32 29, i32 14, i32 31>
+; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <16 x i32> [[TMP45]], <16 x i32> [[TMP46]], <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
 ; CHECK-NEXT:    [[TMP48:%.*]] = shufflevector <16 x i32> [[TMP47]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9, i32 14, i32 15, i32 12, i32 13>
 ; CHECK-NEXT:    [[TMP49:%.*]] = add nsw <16 x i32> [[TMP47]], [[TMP48]]
 ; CHECK-NEXT:    [[TMP50:%.*]] = sub nsw <16 x i32> [[TMP47]], [[TMP48]]
-; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <16 x i32> [[TMP50]], <16 x i32> [[TMP49]], <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 4, i32 5, i32 22, i32 23, i32 8, i32 9, i32 26, i32 27, i32 12, i32 13, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <16 x i32> [[TMP49]], <16 x i32> [[TMP50]], <16 x i32> <i32 16, i32 17, i32 2, i32 3, i32 20, i32 21, i32 6, i32 7, i32 24, i32 25, i32 10, i32 11, i32 28, i32 29, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP52:%.*]] = shufflevector <16 x i32> [[TMP51]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    [[TMP53:%.*]] = sub nsw <16 x i32> [[TMP51]], [[TMP52]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = add nsw <16 x i32> [[TMP51]], [[TMP52]]
@@ -1271,7 +1464,7 @@ define dso_local i32 @full(ptr nocapture noundef readonly %p1, i32 noundef %st1,
 ; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <16 x i32> [[TMP55]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP57:%.*]] = add nsw <16 x i32> [[TMP55]], [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = sub nsw <16 x i32> [[TMP55]], [[TMP56]]
-; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP58]], <16 x i32> [[TMP57]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <16 x i32> [[TMP57]], <16 x i32> [[TMP58]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP60:%.*]] = lshr <16 x i32> [[TMP59]], splat (i32 15)
 ; CHECK-NEXT:    [[TMP61:%.*]] = and <16 x i32> [[TMP60]], splat (i32 65537)
 ; CHECK-NEXT:    [[TMP62:%.*]] = mul nuw <16 x i32> [[TMP61]], splat (i32 65535)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
index c84c33339135..0f47c6b3ac90 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -167,7 +167,9 @@ define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP3_31]]
 ;
   %v0.0 = extractelement <2 x i32> %v0, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
index e4fcb1ed08be..370ed1f258ac 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -167,7 +167,9 @@ define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i32> [[V0]], [[V1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <2 x i32> [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3_31:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; CHECK-NEXT:    ret <4 x i32> [[TMP3_31]]
 ;
   %v0.0 = extractelement <2 x i32> %v0, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
index c48dac19368f..c36e156473f9 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -264,8 +264,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -273,8 +276,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -317,19 +323,27 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX8-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
@@ -354,25 +368,35 @@ bb:
 }
 
 define <4 x i8> @uadd_sat_v4i8(<4 x i8> %arg0, <4 x i8> %arg1) {
-; GCN-LABEL: @uadd_sat_v4i8(
-; GCN-NEXT:  bb:
-; GCN-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
-; GCN-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
-; GCN-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
-; GCN-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
-; GCN-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
-; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
-; GCN-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
-; GCN-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
-; GCN-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
-; GCN-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
-; GCN-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
-; GCN-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
-; GCN-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
-; GCN-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
-; GCN-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
-; GCN-NEXT:    ret <4 x i8> [[INS_3]]
+; GFX7-LABEL: @uadd_sat_v4i8(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
+; GFX7-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
+; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
+; GFX7-NEXT:    ret <4 x i8> [[INS_3]]
+;
+; GFX8-LABEL: @uadd_sat_v4i8(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <4 x i8> [[TMP0]]
+;
+; GFX9-LABEL: @uadd_sat_v4i8(
+; GFX9-NEXT:  bb:
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX9-NEXT:    ret <4 x i8> [[TMP0]]
 ;
 bb:
   %arg0.0 = extractelement <4 x i8> %arg0, i64 0
@@ -395,25 +419,35 @@ bb:
 }
 
 define <4 x i8> @usub_sat_v4i8(<4 x i8> %arg0, <4 x i8> %arg1) {
-; GCN-LABEL: @usub_sat_v4i8(
-; GCN-NEXT:  bb:
-; GCN-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
-; GCN-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
-; GCN-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
-; GCN-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
-; GCN-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
-; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
-; GCN-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
-; GCN-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
-; GCN-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
-; GCN-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
-; GCN-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
-; GCN-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
-; GCN-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
-; GCN-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
-; GCN-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
-; GCN-NEXT:    ret <4 x i8> [[INS_3]]
+; GFX7-LABEL: @usub_sat_v4i8(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
+; GFX7-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
+; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
+; GFX7-NEXT:    ret <4 x i8> [[INS_3]]
+;
+; GFX8-LABEL: @usub_sat_v4i8(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <4 x i8> [[TMP0]]
+;
+; GFX9-LABEL: @usub_sat_v4i8(
+; GFX9-NEXT:  bb:
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX9-NEXT:    ret <4 x i8> [[TMP0]]
 ;
 bb:
   %arg0.0 = extractelement <4 x i8> %arg0, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
index 9da610b48f74..798cc1543c02 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -12,7 +12,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -47,7 +47,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -82,7 +82,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -117,7 +117,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -152,7 +152,7 @@ define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -177,7 +177,7 @@ define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -202,7 +202,7 @@ define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -227,7 +227,7 @@ define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -255,7 +255,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    ret <3 x i16> [[INS_2]]
@@ -264,8 +264,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:  bb:
 ; GFX8-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX8-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX8-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -273,8 +276,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP3:%.*]] = call <3 x i16> @llvm.uadd.sat.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP3]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -309,7 +315,7 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
 ; GFX7-NEXT:    [[ADD_3:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
@@ -317,19 +323,27 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX8-LABEL: @uadd_sat_v4i16(
 ; GFX8-NEXT:  bb:
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX8-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX8-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX8-NEXT:    ret <4 x i16> [[INS_31]]
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG2:%.*]])
-; GFX9-NEXT:    [[ARG1:%.*]] = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG2]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG2:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP4]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG2]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP4]], <2 x i16> [[TMP7]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
@@ -354,25 +368,35 @@ bb:
 }
 
 define <4 x i8> @uadd_sat_v4i8(<4 x i8> %arg0, <4 x i8> %arg1, ptr addrspace(1) %dst) {
-; GCN-LABEL: @uadd_sat_v4i8(
-; GCN-NEXT:  bb:
-; GCN-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
-; GCN-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
-; GCN-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
-; GCN-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
-; GCN-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
-; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
-; GCN-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
-; GCN-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
-; GCN-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
-; GCN-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
-; GCN-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
-; GCN-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
-; GCN-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
-; GCN-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
-; GCN-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
-; GCN-NEXT:    ret <4 x i8> [[INS_3]]
+; GFX7-LABEL: @uadd_sat_v4i8(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
+; GFX7-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> undef, i8 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
+; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
+; GFX7-NEXT:    ret <4 x i8> [[INS_3]]
+;
+; GFX8-LABEL: @uadd_sat_v4i8(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <4 x i8> [[TMP0]]
+;
+; GFX9-LABEL: @uadd_sat_v4i8(
+; GFX9-NEXT:  bb:
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX9-NEXT:    ret <4 x i8> [[TMP0]]
 ;
 bb:
   %arg0.0 = extractelement <4 x i8> %arg0, i64 0
@@ -394,25 +418,35 @@ bb:
   ret <4 x i8> %ins.3
 }
 define <4 x i8> @usub_sat_v4i8(<4 x i8> %arg0, <4 x i8> %arg1) {
-; GCN-LABEL: @usub_sat_v4i8(
-; GCN-NEXT:  bb:
-; GCN-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
-; GCN-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
-; GCN-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
-; GCN-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
-; GCN-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
-; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
-; GCN-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
-; GCN-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
-; GCN-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
-; GCN-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
-; GCN-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
-; GCN-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> poison, i8 [[ADD_0]], i64 0
-; GCN-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
-; GCN-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
-; GCN-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
-; GCN-NEXT:    ret <4 x i8> [[INS_3]]
+; GFX7-LABEL: @usub_sat_v4i8(
+; GFX7-NEXT:  bb:
+; GFX7-NEXT:    [[ARG0_0:%.*]] = extractelement <4 x i8> [[ARG0:%.*]], i64 0
+; GFX7-NEXT:    [[ARG0_1:%.*]] = extractelement <4 x i8> [[ARG0]], i64 1
+; GFX7-NEXT:    [[ARG0_2:%.*]] = extractelement <4 x i8> [[ARG0]], i64 2
+; GFX7-NEXT:    [[ARG0_3:%.*]] = extractelement <4 x i8> [[ARG0]], i64 3
+; GFX7-NEXT:    [[ARG1_0:%.*]] = extractelement <4 x i8> [[ARG1:%.*]], i64 0
+; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i8> [[ARG1]], i64 1
+; GFX7-NEXT:    [[ARG1_2:%.*]] = extractelement <4 x i8> [[ARG1]], i64 2
+; GFX7-NEXT:    [[ARG1_3:%.*]] = extractelement <4 x i8> [[ARG1]], i64 3
+; GFX7-NEXT:    [[ADD_0:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_0]], i8 [[ARG1_0]])
+; GFX7-NEXT:    [[ADD_1:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_1]], i8 [[ARG1_1]])
+; GFX7-NEXT:    [[ADD_2:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_2]], i8 [[ARG1_2]])
+; GFX7-NEXT:    [[ADD_3:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[ARG0_3]], i8 [[ARG1_3]])
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i8> undef, i8 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i8> [[INS_0]], i8 [[ADD_1]], i64 1
+; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i8> [[INS_1]], i8 [[ADD_2]], i64 2
+; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i8> [[INS_2]], i8 [[ADD_3]], i64 3
+; GFX7-NEXT:    ret <4 x i8> [[INS_3]]
+;
+; GFX8-LABEL: @usub_sat_v4i8(
+; GFX8-NEXT:  bb:
+; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX8-NEXT:    ret <4 x i8> [[TMP0]]
+;
+; GFX9-LABEL: @usub_sat_v4i8(
+; GFX9-NEXT:  bb:
+; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ARG0:%.*]], <4 x i8> [[ARG1:%.*]])
+; GFX9-NEXT:    ret <4 x i8> [[TMP0]]
 ;
 bb:
   %arg0.0 = extractelement <4 x i8> %arg0, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/i8.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/i8.ll
index d68e273b56ca..8d518c538a2a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/i8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/i8.ll
@@ -2,7 +2,6 @@
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GFX7 %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GFX8PLUS,GFX8 %s
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GFX8PLUS,GFX9 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer -amdgpu-coerce-illegal-types=1 %s | FileCheck -check-prefixes=VECI8 %s
 
 define protected amdgpu_kernel void @phi(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
 ; GCN-LABEL: @vectorizePHI(
@@ -80,31 +79,13 @@ define protected amdgpu_kernel void @phi(ptr addrspace(3) %inptr0, ptr addrspace
 ; GFX8PLUS-LABEL: @phi(
 ; GFX8PLUS-NEXT:  entry:
 ; GFX8PLUS-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0:%.*]], i32 0
-; GFX8PLUS-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8PLUS-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8PLUS-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8PLUS-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX8PLUS-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8PLUS-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX8PLUS-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX8PLUS-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8PLUS-NEXT:    br label [[DO_BODY:%.*]]
 ; GFX8PLUS:       do.body:
-; GFX8PLUS-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], [[ENTRY:%.*]] ], [ [[OTHERELE3:%.*]], [[DO_BODY]] ]
-; GFX8PLUS-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], [[ENTRY]] ], [ [[OTHERELE2:%.*]], [[DO_BODY]] ]
-; GFX8PLUS-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], [[ENTRY]] ], [ [[OTHERELE1:%.*]], [[DO_BODY]] ]
-; GFX8PLUS-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], [[ENTRY]] ], [ [[OTHERELE0:%.*]], [[DO_BODY]] ]
-; GFX8PLUS-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8PLUS-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8PLUS-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8PLUS-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX8PLUS-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX8PLUS-NEXT:    [[VEC01:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
-; GFX8PLUS-NEXT:    [[VEC02:%.*]] = insertelement <16 x i8> [[VEC01]], i8 [[OTHERELE2]], i64 10
-; GFX8PLUS-NEXT:    [[VEC03:%.*]] = insertelement <16 x i8> [[VEC02]], i8 [[OTHERELE3]], i64 11
-; GFX8PLUS-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8PLUS-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX8PLUS-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX8PLUS-NEXT:    [[VEC13:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX8PLUS-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[DO_BODY]] ]
+; GFX8PLUS-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8PLUS-NEXT:    [[VEC03:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8PLUS-NEXT:    [[VEC13:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8PLUS-NEXT:    store <16 x i8> [[VEC13]], ptr addrspace(3) [[INPTR1:%.*]], align 2
 ; GFX8PLUS-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG:%.*]], 0
 ; GFX8PLUS-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[DO_BODY]]
@@ -113,24 +94,6 @@ define protected amdgpu_kernel void @phi(ptr addrspace(3) %inptr0, ptr addrspace
 ; GFX8PLUS-NEXT:    store <16 x i8> [[VEC03]], ptr [[OUT1:%.*]], align 16
 ; GFX8PLUS-NEXT:    ret void
 ;
-; VECI8-LABEL: @phi(
-; VECI8-NEXT:  entry:
-; VECI8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0:%.*]], i32 0
-; VECI8-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
-; VECI8-NEXT:    br label [[DO_BODY:%.*]]
-; VECI8:       do.body:
-; VECI8-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[DO_BODY]] ]
-; VECI8-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
-; VECI8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    store <16 x i8> [[TMP4]], ptr addrspace(3) [[INPTR1:%.*]], align 2
-; VECI8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG:%.*]], 0
-; VECI8-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[DO_BODY]]
-; VECI8:       exit:
-; VECI8-NEXT:    store <16 x i8> [[TMP4]], ptr [[OUT:%.*]], align 16
-; VECI8-NEXT:    store <16 x i8> [[TMP3]], ptr [[OUT1:%.*]], align 16
-; VECI8-NEXT:    ret void
-;
 entry:
   %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
   %ele0 = load i8, ptr addrspace(3) %gep0, align 8
@@ -242,61 +205,26 @@ define protected amdgpu_kernel void @arith_phi(ptr addrspace(3) %inptr0, ptr %ou
 ; GFX8PLUS-LABEL: @arith_phi(
 ; GFX8PLUS-NEXT:  entry:
 ; GFX8PLUS-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0:%.*]], i32 0
-; GFX8PLUS-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8PLUS-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8PLUS-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
 ; GFX8PLUS-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX8PLUS-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
 ; GFX8PLUS-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX8PLUS-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX8PLUS-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8PLUS-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG:%.*]], 0
 ; GFX8PLUS-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[BB_1:%.*]]
 ; GFX8PLUS:       bb.1:
-; GFX8PLUS-NEXT:    [[ADD0:%.*]] = add i8 [[ELE0]], 1
-; GFX8PLUS-NEXT:    [[ADD1:%.*]] = add i8 [[ELE1]], 1
-; GFX8PLUS-NEXT:    [[ADD2:%.*]] = add i8 [[ELE2]], 1
-; GFX8PLUS-NEXT:    [[ADD3:%.*]] = add i8 [[ELE3]], 1
+; GFX8PLUS-NEXT:    [[TMP1:%.*]] = add <4 x i8> [[TMP0]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; GFX8PLUS-NEXT:    br label [[EXIT]]
 ; GFX8PLUS:       exit:
-; GFX8PLUS-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], [[ENTRY:%.*]] ], [ [[ADD0]], [[BB_1]] ]
-; GFX8PLUS-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], [[ENTRY]] ], [ [[ADD1]], [[BB_1]] ]
-; GFX8PLUS-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], [[ENTRY]] ], [ [[ADD2]], [[BB_1]] ]
-; GFX8PLUS-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], [[ENTRY]] ], [ [[ADD3]], [[BB_1]] ]
+; GFX8PLUS-NEXT:    [[TMP3:%.*]] = phi <4 x i8> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2]], [[BB_1]] ]
 ; GFX8PLUS-NEXT:    [[OTHERELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8PLUS-NEXT:    [[OTHERELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
 ; GFX8PLUS-NEXT:    [[OTHERELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
 ; GFX8PLUS-NEXT:    [[OTHERELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX8PLUS-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8PLUS-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX8PLUS-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX8PLUS-NEXT:    [[VEC13:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX8PLUS-NEXT:    [[VEC13:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8PLUS-NEXT:    store <16 x i8> [[VEC13]], ptr [[OUT:%.*]], align 2
 ; GFX8PLUS-NEXT:    ret void
 ;
-; VECI8-LABEL: @arith_phi(
-; VECI8-NEXT:  entry:
-; VECI8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0:%.*]], i32 0
-; VECI8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; VECI8-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; VECI8-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; VECI8-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
-; VECI8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG:%.*]], 0
-; VECI8-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[BB_1:%.*]]
-; VECI8:       bb.1:
-; VECI8-NEXT:    [[TMP1:%.*]] = add <4 x i8> [[TMP0]], splat (i8 1)
-; VECI8-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; VECI8-NEXT:    br label [[EXIT]]
-; VECI8:       exit:
-; VECI8-NEXT:    [[TMP3:%.*]] = phi <4 x i8> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP2]], [[BB_1]] ]
-; VECI8-NEXT:    [[OTHERELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; VECI8-NEXT:    [[OTHERELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; VECI8-NEXT:    [[OTHERELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; VECI8-NEXT:    [[OTHERELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; VECI8-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP5]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    store <16 x i8> [[TMP4]], ptr [[OUT:%.*]], align 2
-; VECI8-NEXT:    ret void
-;
 entry:
   %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
   %ele0 = load i8, ptr addrspace(3) %gep0, align 8
@@ -405,97 +333,28 @@ define protected amdgpu_kernel void @arith(<16 x i8> %invec, ptr %out, i32 %flag
 ;
 ; GFX8PLUS-LABEL: @arith(
 ; GFX8PLUS-NEXT:  entry:
-; GFX8PLUS-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC:%.*]], i64 0
-; GFX8PLUS-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX8PLUS-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX8PLUS-NEXT:    [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
-; GFX8PLUS-NEXT:    [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
-; GFX8PLUS-NEXT:    [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
-; GFX8PLUS-NEXT:    [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
-; GFX8PLUS-NEXT:    [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
-; GFX8PLUS-NEXT:    [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
-; GFX8PLUS-NEXT:    [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
-; GFX8PLUS-NEXT:    [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
-; GFX8PLUS-NEXT:    [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
-; GFX8PLUS-NEXT:    [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
-; GFX8PLUS-NEXT:    [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
-; GFX8PLUS-NEXT:    [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
-; GFX8PLUS-NEXT:    [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
-; GFX8PLUS-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX8PLUS-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX8PLUS-NEXT:    [[MUL2:%.*]] = mul i8 [[EL2]], 1
-; GFX8PLUS-NEXT:    [[MUL3:%.*]] = mul i8 [[EL3]], 1
-; GFX8PLUS-NEXT:    [[MUL4:%.*]] = mul i8 [[EL4]], 1
-; GFX8PLUS-NEXT:    [[MUL5:%.*]] = mul i8 [[EL5]], 1
-; GFX8PLUS-NEXT:    [[MUL6:%.*]] = mul i8 [[EL6]], 1
-; GFX8PLUS-NEXT:    [[MUL7:%.*]] = mul i8 [[EL7]], 1
-; GFX8PLUS-NEXT:    [[MUL8:%.*]] = mul i8 [[EL8]], 1
-; GFX8PLUS-NEXT:    [[MUL9:%.*]] = mul i8 [[EL9]], 1
-; GFX8PLUS-NEXT:    [[MUL10:%.*]] = mul i8 [[EL10]], 1
-; GFX8PLUS-NEXT:    [[MUL11:%.*]] = mul i8 [[EL11]], 1
-; GFX8PLUS-NEXT:    [[MUL12:%.*]] = mul i8 [[EL12]], 1
-; GFX8PLUS-NEXT:    [[MUL13:%.*]] = mul i8 [[EL13]], 1
-; GFX8PLUS-NEXT:    [[MUL14:%.*]] = mul i8 [[EL14]], 1
-; GFX8PLUS-NEXT:    [[MUL15:%.*]] = mul i8 [[EL15]], 1
-; GFX8PLUS-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX8PLUS-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX8PLUS-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX8PLUS-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX8PLUS-NEXT:    [[ADD4:%.*]] = add i8 [[MUL4]], 1
-; GFX8PLUS-NEXT:    [[ADD5:%.*]] = add i8 [[MUL5]], 1
-; GFX8PLUS-NEXT:    [[ADD6:%.*]] = add i8 [[MUL6]], 1
-; GFX8PLUS-NEXT:    [[ADD7:%.*]] = add i8 [[MUL7]], 1
-; GFX8PLUS-NEXT:    [[ADD8:%.*]] = add i8 [[MUL8]], 1
-; GFX8PLUS-NEXT:    [[ADD9:%.*]] = add i8 [[MUL9]], 1
-; GFX8PLUS-NEXT:    [[ADD10:%.*]] = add i8 [[MUL10]], 1
-; GFX8PLUS-NEXT:    [[ADD11:%.*]] = add i8 [[MUL11]], 1
-; GFX8PLUS-NEXT:    [[ADD12:%.*]] = add i8 [[MUL12]], 1
-; GFX8PLUS-NEXT:    [[ADD13:%.*]] = add i8 [[MUL13]], 1
-; GFX8PLUS-NEXT:    [[ADD14:%.*]] = add i8 [[MUL14]], 1
-; GFX8PLUS-NEXT:    [[ADD15:%.*]] = add i8 [[MUL15]], 1
-; GFX8PLUS-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX8PLUS-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX8PLUS-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
-; GFX8PLUS-NEXT:    [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
-; GFX8PLUS-NEXT:    [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
-; GFX8PLUS-NEXT:    [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
-; GFX8PLUS-NEXT:    [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
-; GFX8PLUS-NEXT:    [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
-; GFX8PLUS-NEXT:    [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
-; GFX8PLUS-NEXT:    [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
-; GFX8PLUS-NEXT:    [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
-; GFX8PLUS-NEXT:    [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
-; GFX8PLUS-NEXT:    [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
-; GFX8PLUS-NEXT:    [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
-; GFX8PLUS-NEXT:    [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
-; GFX8PLUS-NEXT:    [[VECINS15:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX8PLUS-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8PLUS-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GFX8PLUS-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[TMP3]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP5:%.*]] = add <4 x i8> [[TMP4]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GFX8PLUS-NEXT:    [[TMP7:%.*]] = mul <4 x i8> [[TMP6]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP8:%.*]] = add <4 x i8> [[TMP7]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GFX8PLUS-NEXT:    [[TMP10:%.*]] = mul <4 x i8> [[TMP9]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP11:%.*]] = add <4 x i8> [[TMP10]], splat (i8 1)
+; GFX8PLUS-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8PLUS-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8PLUS-NEXT:    [[VECINS71:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GFX8PLUS-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8PLUS-NEXT:    [[VECINS112:%.*]] = shufflevector <16 x i8> [[VECINS71]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; GFX8PLUS-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8PLUS-NEXT:    [[VECINS15:%.*]] = shufflevector <16 x i8> [[VECINS112]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; GFX8PLUS-NEXT:    store <16 x i8> [[VECINS15]], ptr [[OUT:%.*]], align 16
 ; GFX8PLUS-NEXT:    ret void
 ;
-; VECI8-LABEL: @arith(
-; VECI8-NEXT:  entry:
-; VECI8-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC:%.*]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; VECI8-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
-; VECI8-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
-; VECI8-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; VECI8-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[TMP3]], splat (i8 1)
-; VECI8-NEXT:    [[TMP5:%.*]] = add <4 x i8> [[TMP4]], splat (i8 1)
-; VECI8-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; VECI8-NEXT:    [[TMP7:%.*]] = mul <4 x i8> [[TMP6]], splat (i8 1)
-; VECI8-NEXT:    [[TMP8:%.*]] = add <4 x i8> [[TMP7]], splat (i8 1)
-; VECI8-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
-; VECI8-NEXT:    [[TMP10:%.*]] = mul <4 x i8> [[TMP9]], splat (i8 1)
-; VECI8-NEXT:    [[TMP11:%.*]] = add <4 x i8> [[TMP10]], splat (i8 1)
-; VECI8-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[VECINS71:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; VECI8-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[VECINS112:%.*]] = shufflevector <16 x i8> [[VECINS71]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; VECI8-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; VECI8-NEXT:    [[VECINS153:%.*]] = shufflevector <16 x i8> [[VECINS112]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; VECI8-NEXT:    store <16 x i8> [[VECINS153]], ptr [[OUT:%.*]], align 16
-; VECI8-NEXT:    ret void
-;
 entry:
   %el0 = extractelement <16 x i8> %invec, i64  0
   %el1 = extractelement <16 x i8> %invec, i64  1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
index 46c6c10125b9..57ca3db07568 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/min_max.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer,instcombine %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-LABEL: @uadd_sat_v2i16(
@@ -12,7 +12,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -24,7 +24,7 @@ define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -54,7 +54,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -66,7 +66,7 @@ define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -96,7 +96,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -108,7 +108,7 @@ define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -138,7 +138,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX7-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -150,7 +150,7 @@ define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.smax.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <2 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    ret <2 x i16> [[INS_1]]
 ;
@@ -180,7 +180,7 @@ define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.umin.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.umin.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -205,7 +205,7 @@ define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.umax.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.umax.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -230,7 +230,7 @@ define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.smin.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -255,7 +255,7 @@ define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
 ; GCN-NEXT:    [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
 ; GCN-NEXT:    [[ADD_0:%.*]] = call i32 @llvm.smax.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
 ; GCN-NEXT:    [[ADD_1:%.*]] = call i32 @llvm.smax.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
-; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT:    [[INS_0:%.*]] = insertelement <2 x i32> undef, i32 [[ADD_0]], i64 0
 ; GCN-NEXT:    [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
 ; GCN-NEXT:    ret <2 x i32> [[INS_1]]
 ;
@@ -283,7 +283,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    ret <3 x i16> [[INS_2]]
@@ -299,7 +299,7 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX8-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <3 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX8-NEXT:    ret <3 x i16> [[INS_2]]
@@ -308,8 +308,11 @@ define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
 ; GFX9-NEXT:  bb:
 ; GFX9-NEXT:    [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
 ; GFX9-NEXT:    [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
-; GFX9-NEXT:    [[TMP0:%.*]] = call <3 x i16> @llvm.umin.v3i16(<3 x i16> [[ARG0]], <3 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP2:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP1]])
 ; GFX9-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; GFX9-NEXT:    [[INS_2:%.*]] = insertelement <3 x i16> [[TMP0]], i16 [[ADD_2]], i64 2
 ; GFX9-NEXT:    ret <3 x i16> [[INS_2]]
 ;
@@ -344,7 +347,7 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX7-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
 ; GFX7-NEXT:    [[ADD_2:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
 ; GFX7-NEXT:    [[ADD_3:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
-; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX7-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX7-NEXT:    [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
 ; GFX7-NEXT:    [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
@@ -358,9 +361,10 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ; GFX8-NEXT:    [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
 ; GFX8-NEXT:    [[ADD_0:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
 ; GFX8-NEXT:    [[ADD_1:%.*]] = call i16 @llvm.umin.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
-; GFX8-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
-; GFX8-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT:    [[TMP1:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP3]])
+; GFX8-NEXT:    [[INS_0:%.*]] = insertelement <4 x i16> undef, i16 [[ADD_0]], i64 0
 ; GFX8-NEXT:    [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
 ; GFX8-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[INS_1]], <4 x i16> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
@@ -368,10 +372,14 @@ define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
 ;
 ; GFX9-LABEL: @uadd_sat_v4i16(
 ; GFX9-NEXT:  bb:
-; GFX9-NEXT:    [[TMP0:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0:%.*]], <4 x i16> [[ARG1:%.*]])
-; GFX9-NEXT:    [[TMP1:%.*]] = call <4 x i16> @llvm.umin.v4i16(<4 x i16> [[ARG0]], <4 x i16> [[ARG1]])
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i16> [[TMP1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP8:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP6]], <2 x i16> [[TMP7]])
 ; GFX9-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
+; GFX9-NEXT:    [[TMP5:%.*]] = call <2 x i16> @llvm.umin.v2i16(<2 x i16> [[TMP2]], <2 x i16> [[TMP4]])
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <2 x i16> [[TMP8]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[INS_31:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; GFX9-NEXT:    ret <4 x i16> [[INS_31]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
index 03dd882059ee..5593fe9bab6e 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/phi-result-use-order.ll
@@ -95,26 +95,11 @@ bb1:
 define <4 x i8> @phisi8(i1 %cmp1, <4 x i8> %in1, <4 x i8> %in2)  {
 ; CHECK-LABEL: @phisi8(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x i8> [[IN1:%.*]], i64 0
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x i8> [[IN1]], i64 1
-; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x i8> [[IN1]], i64 2
-; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x i8> [[IN1]], i64 3
 ; CHECK-NEXT:    br i1 [[CMP1:%.*]], label [[BB1:%.*]], label [[BB0:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x i8> [[IN2:%.*]], i64 0
-; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x i8> [[IN2]], i64 1
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x i8> [[IN2]], i64 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x i8> [[IN2]], i64 3
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[C0:%.*]] = phi i8 [ [[A0]], [[ENTRY:%.*]] ], [ [[B0]], [[BB0]] ]
-; CHECK-NEXT:    [[C1:%.*]] = phi i8 [ [[A1]], [[ENTRY]] ], [ [[B1]], [[BB0]] ]
-; CHECK-NEXT:    [[C2:%.*]] = phi i8 [ [[A2]], [[ENTRY]] ], [ [[B2]], [[BB0]] ]
-; CHECK-NEXT:    [[C3:%.*]] = phi i8 [ [[A3]], [[ENTRY]] ], [ [[B3]], [[BB0]] ]
-; CHECK-NEXT:    [[O0:%.*]] = insertelement <4 x i8> undef, i8 [[C0]], i64 0
-; CHECK-NEXT:    [[O1:%.*]] = insertelement <4 x i8> [[O0]], i8 [[C1]], i64 1
-; CHECK-NEXT:    [[O2:%.*]] = insertelement <4 x i8> [[O1]], i8 [[C2]], i64 2
-; CHECK-NEXT:    [[O3:%.*]] = insertelement <4 x i8> [[O2]], i8 [[C3]], i64 3
+; CHECK-NEXT:    [[O3:%.*]] = phi <4 x i8> [ [[IN1:%.*]], [[ENTRY:%.*]] ], [ [[IN2:%.*]], [[BB0]] ]
 ; CHECK-NEXT:    ret <4 x i8> [[O3]]
 ;
 entry:
@@ -147,26 +132,11 @@ bb1:
 define <4 x i8> @phisi8_reverse(i1 %cmp1, <4 x i8> %in1, <4 x i8> %in2)  {
 ; CHECK-LABEL: @phisi8_reverse(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A0:%.*]] = extractelement <4 x i8> [[IN1:%.*]], i64 0
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <4 x i8> [[IN1]], i64 1
-; CHECK-NEXT:    [[A2:%.*]] = extractelement <4 x i8> [[IN1]], i64 2
-; CHECK-NEXT:    [[A3:%.*]] = extractelement <4 x i8> [[IN1]], i64 3
 ; CHECK-NEXT:    br i1 [[CMP1:%.*]], label [[BB1:%.*]], label [[BB0:%.*]]
 ; CHECK:       bb0:
-; CHECK-NEXT:    [[B0:%.*]] = extractelement <4 x i8> [[IN2:%.*]], i64 0
-; CHECK-NEXT:    [[B1:%.*]] = extractelement <4 x i8> [[IN2]], i64 1
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x i8> [[IN2]], i64 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x i8> [[IN2]], i64 3
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[C3:%.*]] = phi i8 [ [[A3]], [[ENTRY:%.*]] ], [ [[B3]], [[BB0]] ]
-; CHECK-NEXT:    [[C2:%.*]] = phi i8 [ [[A2]], [[ENTRY]] ], [ [[B2]], [[BB0]] ]
-; CHECK-NEXT:    [[C1:%.*]] = phi i8 [ [[A1]], [[ENTRY]] ], [ [[B1]], [[BB0]] ]
-; CHECK-NEXT:    [[C0:%.*]] = phi i8 [ [[A0]], [[ENTRY]] ], [ [[B0]], [[BB0]] ]
-; CHECK-NEXT:    [[O0:%.*]] = insertelement <4 x i8> undef, i8 [[C0]], i64 0
-; CHECK-NEXT:    [[O1:%.*]] = insertelement <4 x i8> [[O0]], i8 [[C1]], i64 1
-; CHECK-NEXT:    [[O2:%.*]] = insertelement <4 x i8> [[O1]], i8 [[C2]], i64 2
-; CHECK-NEXT:    [[O3:%.*]] = insertelement <4 x i8> [[O2]], i8 [[C3]], i64 3
+; CHECK-NEXT:    [[O3:%.*]] = phi <4 x i8> [ [[IN1:%.*]], [[ENTRY:%.*]] ], [ [[IN2:%.*]], [[BB0]] ]
 ; CHECK-NEXT:    ret <4 x i8> [[O3]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
index 5f8419251e86..e899ad749bb0 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/reduction.ll
@@ -552,13 +552,7 @@ entry:
 define i8 @reduction_v4i8(<4 x i8> %a) {
 ; GCN-LABEL: @reduction_v4i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x i8> [[A:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x i8> [[A]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x i8> [[A]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x i8> [[A]], i64 3
-; GCN-NEXT:    [[ADD1:%.*]] = add i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[ADD2:%.*]] = add i8 [[ELT2]], [[ADD1]]
-; GCN-NEXT:    [[ADD3:%.*]] = add i8 [[ELT3]], [[ADD2]]
+; GCN-NEXT:    [[ADD3:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[A:%.*]])
 ; GCN-NEXT:    ret i8 [[ADD3]]
 ;
 entry:
@@ -577,21 +571,7 @@ entry:
 define i8 @reduction_v8i8(<8 x i8> %vec8) {
 ; GCN-LABEL: @reduction_v8i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <8 x i8> [[VEC8:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <8 x i8> [[VEC8]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <8 x i8> [[VEC8]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <8 x i8> [[VEC8]], i64 3
-; GCN-NEXT:    [[ELT4:%.*]] = extractelement <8 x i8> [[VEC8]], i64 4
-; GCN-NEXT:    [[ELT5:%.*]] = extractelement <8 x i8> [[VEC8]], i64 5
-; GCN-NEXT:    [[ELT6:%.*]] = extractelement <8 x i8> [[VEC8]], i64 6
-; GCN-NEXT:    [[ELT7:%.*]] = extractelement <8 x i8> [[VEC8]], i64 7
-; GCN-NEXT:    [[ADD1:%.*]] = add i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[ADD2:%.*]] = add i8 [[ELT2]], [[ADD1]]
-; GCN-NEXT:    [[ADD3:%.*]] = add i8 [[ELT3]], [[ADD2]]
-; GCN-NEXT:    [[ADD4:%.*]] = add i8 [[ELT4]], [[ADD3]]
-; GCN-NEXT:    [[ADD5:%.*]] = add i8 [[ELT5]], [[ADD4]]
-; GCN-NEXT:    [[ADD6:%.*]] = add i8 [[ELT6]], [[ADD5]]
-; GCN-NEXT:    [[ADD7:%.*]] = add i8 [[ELT7]], [[ADD6]]
+; GCN-NEXT:    [[ADD7:%.*]] = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> [[VEC8:%.*]])
 ; GCN-NEXT:    ret i8 [[ADD7]]
 ;
 entry:
@@ -618,16 +598,7 @@ entry:
 define i8 @reduction_umin_v4i8(<4 x i8> %vec4) {
 ; GCN-LABEL: @reduction_umin_v4i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x i8> [[VEC4:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x i8> [[VEC4]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x i8> [[VEC4]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x i8> [[VEC4]], i64 3
-; GCN-NEXT:    [[CMP1:%.*]] = icmp ult i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[MIN1:%.*]] = select i1 [[CMP1]], i8 [[ELT1]], i8 [[ELT0]]
-; GCN-NEXT:    [[CMP2:%.*]] = icmp ult i8 [[ELT2]], [[MIN1]]
-; GCN-NEXT:    [[MIN2:%.*]] = select i1 [[CMP2]], i8 [[ELT2]], i8 [[MIN1]]
-; GCN-NEXT:    [[CMP3:%.*]] = icmp ult i8 [[ELT3]], [[MIN2]]
-; GCN-NEXT:    [[MIN3:%.*]] = select i1 [[CMP3]], i8 [[ELT3]], i8 [[MIN2]]
+; GCN-NEXT:    [[MIN3:%.*]] = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> [[VEC4:%.*]])
 ; GCN-NEXT:    ret i8 [[MIN3]]
 ;
 entry:
@@ -649,28 +620,7 @@ entry:
 define i8 @reduction_icmp_v8i8(<8 x i8> %vec8) {
 ; GCN-LABEL: @reduction_icmp_v8i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <8 x i8> [[VEC8:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <8 x i8> [[VEC8]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <8 x i8> [[VEC8]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <8 x i8> [[VEC8]], i64 3
-; GCN-NEXT:    [[ELT4:%.*]] = extractelement <8 x i8> [[VEC8]], i64 4
-; GCN-NEXT:    [[ELT5:%.*]] = extractelement <8 x i8> [[VEC8]], i64 5
-; GCN-NEXT:    [[ELT6:%.*]] = extractelement <8 x i8> [[VEC8]], i64 6
-; GCN-NEXT:    [[ELT7:%.*]] = extractelement <8 x i8> [[VEC8]], i64 7
-; GCN-NEXT:    [[CMP0:%.*]] = icmp ult i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i8 [[ELT1]], i8 [[ELT0]]
-; GCN-NEXT:    [[CMP1:%.*]] = icmp ult i8 [[ELT2]], [[MIN1]]
-; GCN-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i8 [[ELT2]], i8 [[MIN1]]
-; GCN-NEXT:    [[CMP2:%.*]] = icmp ult i8 [[ELT3]], [[MIN2]]
-; GCN-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i8 [[ELT3]], i8 [[MIN2]]
-; GCN-NEXT:    [[CMP3:%.*]] = icmp ult i8 [[ELT4]], [[MIN3]]
-; GCN-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i8 [[ELT4]], i8 [[MIN3]]
-; GCN-NEXT:    [[CMP4:%.*]] = icmp ult i8 [[ELT5]], [[MIN4]]
-; GCN-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i8 [[ELT5]], i8 [[MIN4]]
-; GCN-NEXT:    [[CMP5:%.*]] = icmp ult i8 [[ELT6]], [[MIN5]]
-; GCN-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i8 [[ELT6]], i8 [[MIN5]]
-; GCN-NEXT:    [[CMP6:%.*]] = icmp ult i8 [[ELT7]], [[MIN6]]
-; GCN-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i8 [[ELT7]], i8 [[MIN6]]
+; GCN-NEXT:    [[MIN7:%.*]] = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> [[VEC8:%.*]])
 ; GCN-NEXT:    ret i8 [[MIN7]]
 ;
 entry:
@@ -706,52 +656,7 @@ entry:
 define i8 @reduction_smin_v16i8(<16 x i8> %vec16) {
 ; GCN-LABEL: @reduction_smin_v16i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <16 x i8> [[VEC16:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <16 x i8> [[VEC16]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <16 x i8> [[VEC16]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <16 x i8> [[VEC16]], i64 3
-; GCN-NEXT:    [[ELT4:%.*]] = extractelement <16 x i8> [[VEC16]], i64 4
-; GCN-NEXT:    [[ELT5:%.*]] = extractelement <16 x i8> [[VEC16]], i64 5
-; GCN-NEXT:    [[ELT6:%.*]] = extractelement <16 x i8> [[VEC16]], i64 6
-; GCN-NEXT:    [[ELT7:%.*]] = extractelement <16 x i8> [[VEC16]], i64 7
-; GCN-NEXT:    [[ELT8:%.*]] = extractelement <16 x i8> [[VEC16]], i64 8
-; GCN-NEXT:    [[ELT9:%.*]] = extractelement <16 x i8> [[VEC16]], i64 9
-; GCN-NEXT:    [[ELT10:%.*]] = extractelement <16 x i8> [[VEC16]], i64 10
-; GCN-NEXT:    [[ELT11:%.*]] = extractelement <16 x i8> [[VEC16]], i64 11
-; GCN-NEXT:    [[ELT12:%.*]] = extractelement <16 x i8> [[VEC16]], i64 12
-; GCN-NEXT:    [[ELT13:%.*]] = extractelement <16 x i8> [[VEC16]], i64 13
-; GCN-NEXT:    [[ELT14:%.*]] = extractelement <16 x i8> [[VEC16]], i64 14
-; GCN-NEXT:    [[ELT15:%.*]] = extractelement <16 x i8> [[VEC16]], i64 15
-; GCN-NEXT:    [[CMP0:%.*]] = icmp slt i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[MIN1:%.*]] = select i1 [[CMP0]], i8 [[ELT1]], i8 [[ELT0]]
-; GCN-NEXT:    [[CMP1:%.*]] = icmp slt i8 [[ELT2]], [[MIN1]]
-; GCN-NEXT:    [[MIN2:%.*]] = select i1 [[CMP1]], i8 [[ELT2]], i8 [[MIN1]]
-; GCN-NEXT:    [[CMP2:%.*]] = icmp slt i8 [[ELT3]], [[MIN2]]
-; GCN-NEXT:    [[MIN3:%.*]] = select i1 [[CMP2]], i8 [[ELT3]], i8 [[MIN2]]
-; GCN-NEXT:    [[CMP3:%.*]] = icmp slt i8 [[ELT4]], [[MIN3]]
-; GCN-NEXT:    [[MIN4:%.*]] = select i1 [[CMP3]], i8 [[ELT4]], i8 [[MIN3]]
-; GCN-NEXT:    [[CMP4:%.*]] = icmp slt i8 [[ELT5]], [[MIN4]]
-; GCN-NEXT:    [[MIN5:%.*]] = select i1 [[CMP4]], i8 [[ELT5]], i8 [[MIN4]]
-; GCN-NEXT:    [[CMP5:%.*]] = icmp slt i8 [[ELT6]], [[MIN5]]
-; GCN-NEXT:    [[MIN6:%.*]] = select i1 [[CMP5]], i8 [[ELT6]], i8 [[MIN5]]
-; GCN-NEXT:    [[CMP6:%.*]] = icmp slt i8 [[ELT7]], [[MIN6]]
-; GCN-NEXT:    [[MIN7:%.*]] = select i1 [[CMP6]], i8 [[ELT7]], i8 [[MIN6]]
-; GCN-NEXT:    [[CMP7:%.*]] = icmp slt i8 [[ELT8]], [[MIN7]]
-; GCN-NEXT:    [[MIN8:%.*]] = select i1 [[CMP7]], i8 [[ELT8]], i8 [[MIN7]]
-; GCN-NEXT:    [[CMP8:%.*]] = icmp slt i8 [[ELT9]], [[MIN8]]
-; GCN-NEXT:    [[MIN9:%.*]] = select i1 [[CMP8]], i8 [[ELT9]], i8 [[MIN8]]
-; GCN-NEXT:    [[CMP9:%.*]] = icmp slt i8 [[ELT10]], [[MIN9]]
-; GCN-NEXT:    [[MIN10:%.*]] = select i1 [[CMP9]], i8 [[ELT10]], i8 [[MIN9]]
-; GCN-NEXT:    [[CMP10:%.*]] = icmp slt i8 [[ELT11]], [[MIN10]]
-; GCN-NEXT:    [[MIN11:%.*]] = select i1 [[CMP10]], i8 [[ELT11]], i8 [[MIN10]]
-; GCN-NEXT:    [[CMP11:%.*]] = icmp slt i8 [[ELT12]], [[MIN11]]
-; GCN-NEXT:    [[MIN12:%.*]] = select i1 [[CMP11]], i8 [[ELT12]], i8 [[MIN11]]
-; GCN-NEXT:    [[CMP12:%.*]] = icmp slt i8 [[ELT13]], [[MIN12]]
-; GCN-NEXT:    [[MIN13:%.*]] = select i1 [[CMP12]], i8 [[ELT13]], i8 [[MIN12]]
-; GCN-NEXT:    [[CMP13:%.*]] = icmp slt i8 [[ELT14]], [[MIN13]]
-; GCN-NEXT:    [[MIN14:%.*]] = select i1 [[CMP13]], i8 [[ELT14]], i8 [[MIN13]]
-; GCN-NEXT:    [[CMP14:%.*]] = icmp slt i8 [[ELT15]], [[MIN14]]
-; GCN-NEXT:    [[MIN15:%.*]] = select i1 [[CMP14]], i8 [[ELT15]], i8 [[MIN14]]
+; GCN-NEXT:    [[MIN15:%.*]] = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> [[VEC16:%.*]])
 ; GCN-NEXT:    ret i8 [[MIN15]]
 ;
 entry:
@@ -817,16 +722,7 @@ entry:
 define i8 @reduction_umax_v4i8(<4 x i8> %vec4) {
 ; GCN-LABEL: @reduction_umax_v4i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x i8> [[VEC4:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x i8> [[VEC4]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x i8> [[VEC4]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x i8> [[VEC4]], i64 3
-; GCN-NEXT:    [[CMP1:%.*]] = icmp ugt i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i8 [[ELT1]], i8 [[ELT0]]
-; GCN-NEXT:    [[CMP2:%.*]] = icmp ugt i8 [[ELT2]], [[MAX1]]
-; GCN-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i8 [[ELT2]], i8 [[MAX1]]
-; GCN-NEXT:    [[CMP3:%.*]] = icmp ugt i8 [[ELT3]], [[MAX2]]
-; GCN-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i8 [[ELT3]], i8 [[MAX2]]
+; GCN-NEXT:    [[MAX3:%.*]] = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> [[VEC4:%.*]])
 ; GCN-NEXT:    ret i8 [[MAX3]]
 ;
 entry:
@@ -848,16 +744,7 @@ entry:
 define i8 @reduction_smax_v4i8(<4 x i8> %vec4) {
 ; GCN-LABEL: @reduction_smax_v4i8(
 ; GCN-NEXT:  entry:
-; GCN-NEXT:    [[ELT0:%.*]] = extractelement <4 x i8> [[VEC4:%.*]], i64 0
-; GCN-NEXT:    [[ELT1:%.*]] = extractelement <4 x i8> [[VEC4]], i64 1
-; GCN-NEXT:    [[ELT2:%.*]] = extractelement <4 x i8> [[VEC4]], i64 2
-; GCN-NEXT:    [[ELT3:%.*]] = extractelement <4 x i8> [[VEC4]], i64 3
-; GCN-NEXT:    [[CMP1:%.*]] = icmp sgt i8 [[ELT1]], [[ELT0]]
-; GCN-NEXT:    [[MAX1:%.*]] = select i1 [[CMP1]], i8 [[ELT1]], i8 [[ELT0]]
-; GCN-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[ELT2]], [[MAX1]]
-; GCN-NEXT:    [[MAX2:%.*]] = select i1 [[CMP2]], i8 [[ELT2]], i8 [[MAX1]]
-; GCN-NEXT:    [[CMP3:%.*]] = icmp sgt i8 [[ELT3]], [[MAX2]]
-; GCN-NEXT:    [[MAX3:%.*]] = select i1 [[CMP3]], i8 [[ELT3]], i8 [[MAX2]]
+; GCN-NEXT:    [[MAX3:%.*]] = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> [[VEC4:%.*]])
 ; GCN-NEXT:    ret i8 [[MAX3]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
index b9b1bc1be681..605eccf26cd3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
@@ -21,28 +21,20 @@ define protected amdgpu_kernel void @arith_2(<16 x i8> %invec, ptr %out, i32 %fl
 ; GFX8-LABEL: define protected amdgpu_kernel void @arith_2(
 ; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; GFX8-NEXT:  [[ENTRY:.*:]]
-; GFX8-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX8-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX8-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX8-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX8-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX8-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX8-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT:    [[TMP1:%.*]] = mul <2 x i8> [[TMP0]], splat (i8 1)
+; GFX8-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], splat (i8 1)
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
 ; GFX8-NEXT:    ret void
 ;
 ; GFX9-LABEL: define protected amdgpu_kernel void @arith_2(
 ; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; GFX9-NEXT:  [[ENTRY:.*:]]
-; GFX9-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX9-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX9-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX9-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX9-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX9-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX9-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <2 x i32> <i32 0, i32 1>
+; GFX9-NEXT:    [[TMP1:%.*]] = mul <2 x i8> [[TMP0]], splat (i8 1)
+; GFX9-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], splat (i8 1)
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
 ; GFX9-NEXT:    ret void
 ;
@@ -82,17 +74,14 @@ define protected amdgpu_kernel void @arith_3(<16 x i8> %invec, ptr %out, i32 %fl
 ; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*:]]
 ; GFX8-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX8-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX8-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX8-NEXT:    [[MUL2:%.*]] = mul i8 [[EL0]], 1
-; GFX8-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX8-NEXT:    [[MUL3:%.*]] = mul i8 [[EL2]], 1
-; GFX8-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX8-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX8-NEXT:    [[MUL3:%.*]] = mul i8 [[EL0]], 1
 ; GFX8-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX8-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD2]], i64 0
-; GFX8-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX8-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD3]], i64 2
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <2 x i32> <i32 1, i32 2>
+; GFX8-NEXT:    [[TMP1:%.*]] = mul <2 x i8> [[TMP0]], splat (i8 1)
+; GFX8-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], splat (i8 1)
+; GFX8-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD3]], i64 0
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VECINS2:%.*]] = shufflevector <16 x i8> [[VECINS0]], <16 x i8> [[TMP3]], <16 x i32> <i32 0, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[VECINS2]], ptr [[OUT]], align 16
 ; GFX8-NEXT:    ret void
 ;
@@ -100,17 +89,14 @@ define protected amdgpu_kernel void @arith_3(<16 x i8> %invec, ptr %out, i32 %fl
 ; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*:]]
 ; GFX9-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX9-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX9-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX9-NEXT:    [[MUL2:%.*]] = mul i8 [[EL0]], 1
-; GFX9-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX9-NEXT:    [[MUL3:%.*]] = mul i8 [[EL2]], 1
-; GFX9-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX9-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX9-NEXT:    [[MUL3:%.*]] = mul i8 [[EL0]], 1
 ; GFX9-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX9-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD2]], i64 0
-; GFX9-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX9-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD3]], i64 2
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <2 x i32> <i32 1, i32 2>
+; GFX9-NEXT:    [[TMP1:%.*]] = mul <2 x i8> [[TMP0]], splat (i8 1)
+; GFX9-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], splat (i8 1)
+; GFX9-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD3]], i64 0
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VECINS2:%.*]] = shufflevector <16 x i8> [[VECINS0]], <16 x i8> [[TMP3]], <16 x i32> <i32 0, i32 16, i32 17, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[VECINS2]], ptr [[OUT]], align 16
 ; GFX9-NEXT:    ret void
 ;
@@ -157,44 +143,20 @@ define protected amdgpu_kernel void @arith_4(<16 x i8> %invec, ptr %out, i32 %fl
 ; GFX8-LABEL: define protected amdgpu_kernel void @arith_4(
 ; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*:]]
-; GFX8-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX8-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX8-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX8-NEXT:    [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
-; GFX8-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX8-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX8-NEXT:    [[MUL2:%.*]] = mul i8 [[EL2]], 1
-; GFX8-NEXT:    [[MUL3:%.*]] = mul i8 [[EL3]], 1
-; GFX8-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX8-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX8-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX8-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX8-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX8-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX8-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
+; GFX8-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
 ; GFX8-NEXT:    ret void
 ;
 ; GFX9-LABEL: define protected amdgpu_kernel void @arith_4(
 ; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*:]]
-; GFX9-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX9-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX9-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX9-NEXT:    [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
-; GFX9-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX9-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX9-NEXT:    [[MUL2:%.*]] = mul i8 [[EL2]], 1
-; GFX9-NEXT:    [[MUL3:%.*]] = mul i8 [[EL3]], 1
-; GFX9-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX9-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX9-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX9-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX9-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX9-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX9-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX9-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
+; GFX9-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
 ; GFX9-NEXT:    ret void
 ;
@@ -293,140 +255,50 @@ define protected amdgpu_kernel void @arith_16(<16 x i8> %invec, ptr %out, i32 %f
 ; GFX8-LABEL: define protected amdgpu_kernel void @arith_16(
 ; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*:]]
-; GFX8-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX8-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX8-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX8-NEXT:    [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
-; GFX8-NEXT:    [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
-; GFX8-NEXT:    [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
-; GFX8-NEXT:    [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
-; GFX8-NEXT:    [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
-; GFX8-NEXT:    [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
-; GFX8-NEXT:    [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
-; GFX8-NEXT:    [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
-; GFX8-NEXT:    [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
-; GFX8-NEXT:    [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
-; GFX8-NEXT:    [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
-; GFX8-NEXT:    [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
-; GFX8-NEXT:    [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
-; GFX8-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX8-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX8-NEXT:    [[MUL2:%.*]] = mul i8 [[EL2]], 1
-; GFX8-NEXT:    [[MUL3:%.*]] = mul i8 [[EL3]], 1
-; GFX8-NEXT:    [[MUL4:%.*]] = mul i8 [[EL4]], 1
-; GFX8-NEXT:    [[MUL5:%.*]] = mul i8 [[EL5]], 1
-; GFX8-NEXT:    [[MUL6:%.*]] = mul i8 [[EL6]], 1
-; GFX8-NEXT:    [[MUL7:%.*]] = mul i8 [[EL7]], 1
-; GFX8-NEXT:    [[MUL8:%.*]] = mul i8 [[EL8]], 1
-; GFX8-NEXT:    [[MUL9:%.*]] = mul i8 [[EL9]], 1
-; GFX8-NEXT:    [[MUL10:%.*]] = mul i8 [[EL10]], 1
-; GFX8-NEXT:    [[MUL11:%.*]] = mul i8 [[EL11]], 1
-; GFX8-NEXT:    [[MUL12:%.*]] = mul i8 [[EL12]], 1
-; GFX8-NEXT:    [[MUL13:%.*]] = mul i8 [[EL13]], 1
-; GFX8-NEXT:    [[MUL14:%.*]] = mul i8 [[EL14]], 1
-; GFX8-NEXT:    [[MUL15:%.*]] = mul i8 [[EL15]], 1
-; GFX8-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX8-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX8-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX8-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX8-NEXT:    [[ADD4:%.*]] = add i8 [[MUL4]], 1
-; GFX8-NEXT:    [[ADD5:%.*]] = add i8 [[MUL5]], 1
-; GFX8-NEXT:    [[ADD6:%.*]] = add i8 [[MUL6]], 1
-; GFX8-NEXT:    [[ADD7:%.*]] = add i8 [[MUL7]], 1
-; GFX8-NEXT:    [[ADD8:%.*]] = add i8 [[MUL8]], 1
-; GFX8-NEXT:    [[ADD9:%.*]] = add i8 [[MUL9]], 1
-; GFX8-NEXT:    [[ADD10:%.*]] = add i8 [[MUL10]], 1
-; GFX8-NEXT:    [[ADD11:%.*]] = add i8 [[MUL11]], 1
-; GFX8-NEXT:    [[ADD12:%.*]] = add i8 [[MUL12]], 1
-; GFX8-NEXT:    [[ADD13:%.*]] = add i8 [[MUL13]], 1
-; GFX8-NEXT:    [[ADD14:%.*]] = add i8 [[MUL14]], 1
-; GFX8-NEXT:    [[ADD15:%.*]] = add i8 [[MUL15]], 1
-; GFX8-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX8-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX8-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
-; GFX8-NEXT:    [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
-; GFX8-NEXT:    [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
-; GFX8-NEXT:    [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
-; GFX8-NEXT:    [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
-; GFX8-NEXT:    [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
-; GFX8-NEXT:    [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
-; GFX8-NEXT:    [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
-; GFX8-NEXT:    [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
-; GFX8-NEXT:    [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
-; GFX8-NEXT:    [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
-; GFX8-NEXT:    [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
-; GFX8-NEXT:    [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
-; GFX8-NEXT:    [[VECINS153:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX8-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
+; GFX8-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GFX8-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[TMP3]], splat (i8 1)
+; GFX8-NEXT:    [[TMP5:%.*]] = add <4 x i8> [[TMP4]], splat (i8 1)
+; GFX8-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GFX8-NEXT:    [[TMP7:%.*]] = mul <4 x i8> [[TMP6]], splat (i8 1)
+; GFX8-NEXT:    [[TMP8:%.*]] = add <4 x i8> [[TMP7]], splat (i8 1)
+; GFX8-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GFX8-NEXT:    [[TMP10:%.*]] = mul <4 x i8> [[TMP9]], splat (i8 1)
+; GFX8-NEXT:    [[TMP11:%.*]] = add <4 x i8> [[TMP10]], splat (i8 1)
+; GFX8-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VECINS71:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GFX8-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VECINS112:%.*]] = shufflevector <16 x i8> [[VECINS71]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; GFX8-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VECINS153:%.*]] = shufflevector <16 x i8> [[VECINS112]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; GFX8-NEXT:    store <16 x i8> [[VECINS153]], ptr [[OUT]], align 16
 ; GFX8-NEXT:    ret void
 ;
 ; GFX9-LABEL: define protected amdgpu_kernel void @arith_16(
 ; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*:]]
-; GFX9-NEXT:    [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
-; GFX9-NEXT:    [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
-; GFX9-NEXT:    [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
-; GFX9-NEXT:    [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
-; GFX9-NEXT:    [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
-; GFX9-NEXT:    [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
-; GFX9-NEXT:    [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
-; GFX9-NEXT:    [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
-; GFX9-NEXT:    [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
-; GFX9-NEXT:    [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
-; GFX9-NEXT:    [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
-; GFX9-NEXT:    [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
-; GFX9-NEXT:    [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
-; GFX9-NEXT:    [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
-; GFX9-NEXT:    [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
-; GFX9-NEXT:    [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
-; GFX9-NEXT:    [[MUL0:%.*]] = mul i8 [[EL0]], 1
-; GFX9-NEXT:    [[MUL1:%.*]] = mul i8 [[EL1]], 1
-; GFX9-NEXT:    [[MUL2:%.*]] = mul i8 [[EL2]], 1
-; GFX9-NEXT:    [[MUL3:%.*]] = mul i8 [[EL3]], 1
-; GFX9-NEXT:    [[MUL4:%.*]] = mul i8 [[EL4]], 1
-; GFX9-NEXT:    [[MUL5:%.*]] = mul i8 [[EL5]], 1
-; GFX9-NEXT:    [[MUL6:%.*]] = mul i8 [[EL6]], 1
-; GFX9-NEXT:    [[MUL7:%.*]] = mul i8 [[EL7]], 1
-; GFX9-NEXT:    [[MUL8:%.*]] = mul i8 [[EL8]], 1
-; GFX9-NEXT:    [[MUL9:%.*]] = mul i8 [[EL9]], 1
-; GFX9-NEXT:    [[MUL10:%.*]] = mul i8 [[EL10]], 1
-; GFX9-NEXT:    [[MUL11:%.*]] = mul i8 [[EL11]], 1
-; GFX9-NEXT:    [[MUL12:%.*]] = mul i8 [[EL12]], 1
-; GFX9-NEXT:    [[MUL13:%.*]] = mul i8 [[EL13]], 1
-; GFX9-NEXT:    [[MUL14:%.*]] = mul i8 [[EL14]], 1
-; GFX9-NEXT:    [[MUL15:%.*]] = mul i8 [[EL15]], 1
-; GFX9-NEXT:    [[ADD0:%.*]] = add i8 [[MUL0]], 1
-; GFX9-NEXT:    [[ADD1:%.*]] = add i8 [[MUL1]], 1
-; GFX9-NEXT:    [[ADD2:%.*]] = add i8 [[MUL2]], 1
-; GFX9-NEXT:    [[ADD3:%.*]] = add i8 [[MUL3]], 1
-; GFX9-NEXT:    [[ADD4:%.*]] = add i8 [[MUL4]], 1
-; GFX9-NEXT:    [[ADD5:%.*]] = add i8 [[MUL5]], 1
-; GFX9-NEXT:    [[ADD6:%.*]] = add i8 [[MUL6]], 1
-; GFX9-NEXT:    [[ADD7:%.*]] = add i8 [[MUL7]], 1
-; GFX9-NEXT:    [[ADD8:%.*]] = add i8 [[MUL8]], 1
-; GFX9-NEXT:    [[ADD9:%.*]] = add i8 [[MUL9]], 1
-; GFX9-NEXT:    [[ADD10:%.*]] = add i8 [[MUL10]], 1
-; GFX9-NEXT:    [[ADD11:%.*]] = add i8 [[MUL11]], 1
-; GFX9-NEXT:    [[ADD12:%.*]] = add i8 [[MUL12]], 1
-; GFX9-NEXT:    [[ADD13:%.*]] = add i8 [[MUL13]], 1
-; GFX9-NEXT:    [[ADD14:%.*]] = add i8 [[MUL14]], 1
-; GFX9-NEXT:    [[ADD15:%.*]] = add i8 [[MUL15]], 1
-; GFX9-NEXT:    [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
-; GFX9-NEXT:    [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
-; GFX9-NEXT:    [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
-; GFX9-NEXT:    [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
-; GFX9-NEXT:    [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
-; GFX9-NEXT:    [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
-; GFX9-NEXT:    [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
-; GFX9-NEXT:    [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
-; GFX9-NEXT:    [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
-; GFX9-NEXT:    [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
-; GFX9-NEXT:    [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
-; GFX9-NEXT:    [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
-; GFX9-NEXT:    [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
-; GFX9-NEXT:    [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
-; GFX9-NEXT:    [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
-; GFX9-NEXT:    [[VECINS153:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX9-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX9-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[TMP0]], splat (i8 1)
+; GFX9-NEXT:    [[TMP2:%.*]] = add <4 x i8> [[TMP1]], splat (i8 1)
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; GFX9-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[TMP3]], splat (i8 1)
+; GFX9-NEXT:    [[TMP5:%.*]] = add <4 x i8> [[TMP4]], splat (i8 1)
+; GFX9-NEXT:    [[TMP6:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; GFX9-NEXT:    [[TMP7:%.*]] = mul <4 x i8> [[TMP6]], splat (i8 1)
+; GFX9-NEXT:    [[TMP8:%.*]] = add <4 x i8> [[TMP7]], splat (i8 1)
+; GFX9-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[INVEC]], <16 x i8> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; GFX9-NEXT:    [[TMP10:%.*]] = mul <4 x i8> [[TMP9]], splat (i8 1)
+; GFX9-NEXT:    [[TMP11:%.*]] = add <4 x i8> [[TMP10]], splat (i8 1)
+; GFX9-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP5]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VECINS71:%.*]] = shufflevector <16 x i8> [[TMP12]], <16 x i8> [[TMP13]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; GFX9-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP8]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VECINS112:%.*]] = shufflevector <16 x i8> [[VECINS71]], <16 x i8> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
+; GFX9-NEXT:    [[TMP15:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VECINS153:%.*]] = shufflevector <16 x i8> [[VECINS112]], <16 x i8> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
 ; GFX9-NEXT:    store <16 x i8> [[VECINS153]], ptr [[OUT]], align 16
 ; GFX9-NEXT:    ret void
 ;
@@ -529,19 +401,13 @@ define protected amdgpu_kernel void @phi_2(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*]]:
 ; GFX8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX8-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX8-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX8:       [[DO_BODY]]:
-; GFX8-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
-; GFX8-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8-NEXT:    [[VEC111:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
+; GFX8-NEXT:    [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT:    [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VEC111:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[VEC111]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX8-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -554,19 +420,13 @@ define protected amdgpu_kernel void @phi_2(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*]]:
 ; GFX9-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX9-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX9-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX9-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX9-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX9:       [[DO_BODY]]:
-; GFX9-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
-; GFX9-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX9-NEXT:    [[VEC111:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
+; GFX9-NEXT:    [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT:    [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VEC111:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[VEC111]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX9-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX9-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -637,24 +497,18 @@ define protected amdgpu_kernel void @phi_3(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*]]:
 ; GFX8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX8-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX8-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
 ; GFX8-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
 ; GFX8-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX8:       [[DO_BODY]]:
 ; GFX8-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX8-NEXT:    [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT:    [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[VEC02:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[OTHERELE2]], i64 10
-; GFX8-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8-NEXT:    [[VEC111:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
+; GFX8-NEXT:    [[VEC111:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC111]], i8 [[PHI1]], i64 10
 ; GFX8-NEXT:    store <16 x i8> [[VEC12]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
@@ -668,24 +522,18 @@ define protected amdgpu_kernel void @phi_3(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*]]:
 ; GFX9-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX9-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX9-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX9-NEXT:    [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX9-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
 ; GFX9-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
 ; GFX9-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX9:       [[DO_BODY]]:
 ; GFX9-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
+; GFX9-NEXT:    [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT:    [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX9-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[VEC02:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[OTHERELE2]], i64 10
-; GFX9-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX9-NEXT:    [[VEC111:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
+; GFX9-NEXT:    [[VEC111:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC111]], i8 [[PHI1]], i64 10
 ; GFX9-NEXT:    store <16 x i8> [[VEC12]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX9-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
@@ -769,31 +617,13 @@ define protected amdgpu_kernel void @phi_4(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*]]:
 ; GFX8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX8-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX8-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX8-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX8-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX8:       [[DO_BODY]]:
-; GFX8-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX8-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX8-NEXT:    [[VEC01:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
-; GFX8-NEXT:    [[VEC02:%.*]] = insertelement <16 x i8> [[VEC01]], i8 [[OTHERELE2]], i64 10
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC02]], i8 [[OTHERELE3]], i64 11
-; GFX8-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX8-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX8-NEXT:    [[VEC131:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX8-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT:    [[VEC131:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[VEC131]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX8-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -806,31 +636,13 @@ define protected amdgpu_kernel void @phi_4(ptr addrspace(3) %inptr0, ptr addrspa
 ; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*]]:
 ; GFX9-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX9-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX9-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX9-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX9-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX9-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX9-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX9:       [[DO_BODY]]:
-; GFX9-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX9-NEXT:    [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
-; GFX9-NEXT:    [[VEC01:%.*]] = insertelement <16 x i8> [[VEC00]], i8 [[OTHERELE1]], i64 9
-; GFX9-NEXT:    [[VEC02:%.*]] = insertelement <16 x i8> [[VEC01]], i8 [[OTHERELE2]], i64 10
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC02]], i8 [[OTHERELE3]], i64 11
-; GFX9-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX9-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX9-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX9-NEXT:    [[VEC131:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX9-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT:    [[VEC131:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[VEC131]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX9-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX9-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -882,31 +694,13 @@ define protected amdgpu_kernel void @phi_4_with_stores(ptr addrspace(3) %inptr0,
 ; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX7-NEXT:  [[ENTRY:.*]]:
 ; GFX7-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX7-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX7-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX7-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX7-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX7-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX7-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX7-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX7-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX7-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX7:       [[DO_BODY]]:
-; GFX7-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
-; GFX7-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX7-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX7-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX7-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX7-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX7-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX7-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX7-NEXT:    store i8 [[PHI3]], ptr addrspace(3) [[GEP0]], align 2
-; GFX7-NEXT:    store i8 [[PHI2]], ptr addrspace(3) [[GEP1]], align 2
-; GFX7-NEXT:    store i8 [[PHI1]], ptr addrspace(3) [[GEP2]], align 2
-; GFX7-NEXT:    store i8 [[PHI0]], ptr addrspace(3) [[GEP3]], align 2
-; GFX7-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX7-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX7-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX7-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX7-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT:    store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX7-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX7-NEXT:    store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX7-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX7-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -919,31 +713,13 @@ define protected amdgpu_kernel void @phi_4_with_stores(ptr addrspace(3) %inptr0,
 ; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*]]:
 ; GFX8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX8-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX8-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX8-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX8-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX8-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX8-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX8:       [[DO_BODY]]:
-; GFX8-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX8-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX8-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX8-NEXT:    store i8 [[PHI3]], ptr addrspace(3) [[GEP0]], align 2
-; GFX8-NEXT:    store i8 [[PHI2]], ptr addrspace(3) [[GEP1]], align 2
-; GFX8-NEXT:    store i8 [[PHI1]], ptr addrspace(3) [[GEP2]], align 2
-; GFX8-NEXT:    store i8 [[PHI0]], ptr addrspace(3) [[GEP3]], align 2
-; GFX8-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX8-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX8-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX8-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX8-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT:    store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX8-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX8-NEXT:    store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX8-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX8-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -956,31 +732,13 @@ define protected amdgpu_kernel void @phi_4_with_stores(ptr addrspace(3) %inptr0,
 ; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*]]:
 ; GFX9-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
-; GFX9-NEXT:    [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 1
-; GFX9-NEXT:    [[ELE1:%.*]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
-; GFX9-NEXT:    [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 3
-; GFX9-NEXT:    [[ELE3:%.*]] = load i8, ptr addrspace(3) [[GEP3]], align 1
+; GFX9-NEXT:    [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
 ; GFX9-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX9:       [[DO_BODY]]:
-; GFX9-NEXT:    [[PHI0:%.*]] = phi i8 [ [[ELE3]], %[[ENTRY]] ], [ [[OTHERELE3:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI2:%.*]] = phi i8 [ [[ELE1]], %[[ENTRY]] ], [ [[OTHERELE1:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
-; GFX9-NEXT:    [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
-; GFX9-NEXT:    [[OTHERELE1]] = load i8, ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    [[OTHERELE3]] = load i8, ptr addrspace(3) [[GEP3]], align 1
-; GFX9-NEXT:    store i8 [[PHI3]], ptr addrspace(3) [[GEP0]], align 2
-; GFX9-NEXT:    store i8 [[PHI2]], ptr addrspace(3) [[GEP1]], align 2
-; GFX9-NEXT:    store i8 [[PHI1]], ptr addrspace(3) [[GEP2]], align 2
-; GFX9-NEXT:    store i8 [[PHI0]], ptr addrspace(3) [[GEP3]], align 2
-; GFX9-NEXT:    [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
-; GFX9-NEXT:    [[VEC11:%.*]] = insertelement <16 x i8> [[VEC10]], i8 [[PHI2]], i64 9
-; GFX9-NEXT:    [[VEC12:%.*]] = insertelement <16 x i8> [[VEC11]], i8 [[PHI1]], i64 10
-; GFX9-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[VEC12]], i8 [[PHI0]], i64 11
+; GFX9-NEXT:    [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT:    [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT:    store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX9-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; GFX9-NEXT:    store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
 ; GFX9-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
 ; GFX9-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
@@ -1031,10 +789,6 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
 ; GFX7-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
 ; GFX7-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX7-NEXT:  [[ENTRY:.*]]:
-; GFX7-NEXT:    [[TMP4:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
-; GFX7-NEXT:    [[TMP1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
-; GFX7-NEXT:    [[TMP2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
-; GFX7-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
 ; GFX7-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX7:       [[DO_BODY]]:
 ; GFX7-NEXT:    [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -1044,22 +798,12 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
 ; GFX7-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
 ; GFX7:       [[EXIT]]:
 ; GFX7-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX7-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
-; GFX7-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
-; GFX7-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
-; GFX7-NEXT:    store i8 [[TMP3]], ptr addrspace(3) [[GEP0]], align 1
-; GFX7-NEXT:    store i8 [[TMP2]], ptr addrspace(3) [[GEP1]], align 1
-; GFX7-NEXT:    store i8 [[TMP1]], ptr addrspace(3) [[GEP2]], align 1
-; GFX7-NEXT:    store i8 [[TMP4]], ptr addrspace(3) [[GEP3]], align 1
+; GFX7-NEXT:    store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
 ; GFX7-NEXT:    ret void
 ;
 ; GFX8-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
 ; GFX8-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX8-NEXT:  [[ENTRY:.*]]:
-; GFX8-NEXT:    [[TMP4:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
-; GFX8-NEXT:    [[TMP1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
-; GFX8-NEXT:    [[TMP2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
-; GFX8-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
 ; GFX8-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX8:       [[DO_BODY]]:
 ; GFX8-NEXT:    [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -1069,22 +813,12 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
 ; GFX8-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
 ; GFX8:       [[EXIT]]:
 ; GFX8-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX8-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
-; GFX8-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
-; GFX8-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
-; GFX8-NEXT:    store i8 [[TMP3]], ptr addrspace(3) [[GEP0]], align 1
-; GFX8-NEXT:    store i8 [[TMP2]], ptr addrspace(3) [[GEP1]], align 1
-; GFX8-NEXT:    store i8 [[TMP1]], ptr addrspace(3) [[GEP2]], align 1
-; GFX8-NEXT:    store i8 [[TMP4]], ptr addrspace(3) [[GEP3]], align 1
+; GFX8-NEXT:    store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
 ; GFX8-NEXT:    ret void
 ;
 ; GFX9-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
 ; GFX9-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
 ; GFX9-NEXT:  [[ENTRY:.*]]:
-; GFX9-NEXT:    [[TMP4:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
-; GFX9-NEXT:    [[TMP1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
-; GFX9-NEXT:    [[TMP2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
-; GFX9-NEXT:    [[TMP3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
 ; GFX9-NEXT:    br label %[[DO_BODY:.*]]
 ; GFX9:       [[DO_BODY]]:
 ; GFX9-NEXT:    [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -1094,13 +828,7 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
 ; GFX9-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
 ; GFX9:       [[EXIT]]:
 ; GFX9-NEXT:    [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX9-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
-; GFX9-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
-; GFX9-NEXT:    [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
-; GFX9-NEXT:    store i8 [[TMP3]], ptr addrspace(3) [[GEP0]], align 1
-; GFX9-NEXT:    store i8 [[TMP2]], ptr addrspace(3) [[GEP1]], align 1
-; GFX9-NEXT:    store i8 [[TMP1]], ptr addrspace(3) [[GEP2]], align 1
-; GFX9-NEXT:    store i8 [[TMP4]], ptr addrspace(3) [[GEP3]], align 1
+; GFX9-NEXT:    store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
 ; GFX9-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
index 7faf11c34d76..88a03bb72c27 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/no-vectorize-rotate.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
 
 ; Regression test for a bug in the SLP vectorizer that was causing
 ; these rotates to be incorrectly combined into a vector rotate.
@@ -9,16 +9,16 @@ target triple = "wasm32-unknown-unknown"
 
 define void @foo(<2 x i64> %x, <4 x i32> %y, ptr %out) #0 {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[A:%.*]] = extractelement <2 x i64> [[X:%.*]], i64 0
-; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x i32> [[Y:%.*]], i64 2
+; CHECK-NEXT:    [[A:%.*]] = extractelement <2 x i64> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[B:%.*]] = extractelement <4 x i32> [[Y:%.*]], i32 2
 ; CHECK-NEXT:    [[CONV6:%.*]] = zext i32 [[B]] to i64
 ; CHECK-NEXT:    [[C:%.*]] = tail call i64 @llvm.fshl.i64(i64 [[A]], i64 [[A]], i64 [[CONV6]])
 ; CHECK-NEXT:    store i64 [[C]], ptr [[OUT:%.*]], align 8
-; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i64> [[X]], i64 1
-; CHECK-NEXT:    [[E:%.*]] = extractelement <4 x i32> [[Y]], i64 3
+; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i64> [[X]], i32 1
+; CHECK-NEXT:    [[E:%.*]] = extractelement <4 x i32> [[Y]], i32 3
 ; CHECK-NEXT:    [[CONV17:%.*]] = zext i32 [[E]] to i64
 ; CHECK-NEXT:    [[F:%.*]] = tail call i64 @llvm.fshl.i64(i64 [[D]], i64 [[D]], i64 [[CONV17]])
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[OUT]], i32 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[OUT]], i32 1
 ; CHECK-NEXT:    store i64 [[F]], ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index 77d36f010766..92ffbad73d5f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -1,21 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
 
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
-; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
-; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
 ; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
 ; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
@@ -24,31 +24,31 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
 ; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
 ; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
-; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
-; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
-; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -56,20 +56,20 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX-LABEL: @ceil_floor(
-; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX-NEXT:    [[TMP8:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP9]])
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP10]])
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i64 3
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -77,20 +77,20 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index 18d79752b0b4..f504821b5fa6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -1,21 +1,21 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=AVX2
 
 define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-LABEL: @ceil_floor(
-; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
-; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i64 4
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
 ; SSE-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
 ; SSE-NEXT:    [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
 ; SSE-NEXT:    [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
@@ -24,75 +24,75 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
 ; SSE-NEXT:    [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
 ; SSE-NEXT:    [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
 ; SSE-NEXT:    [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
-; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i64 1
-; SSE-NEXT:    [[R23:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
-; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i64 4
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i64 6
-; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT:    [[R71:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x float> [[R71]]
 ;
 ; SLM-LABEL: @ceil_floor(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; SLM-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SLM-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; SLM-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; SLM-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; SLM-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX-LABEL: @ceil_floor(
-; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX-NEXT:    [[TMP8:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP9]])
 ; AVX-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i64 3
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP10]])
+; AVX-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX-NEXT:    ret <8 x float> [[R71]]
 ;
 ; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i64 0
-; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[A]], i64 3
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
+; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
 ; AVX2-NEXT:    [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT:    [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
 ; AVX2-NEXT:    [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
-; AVX2-NEXT:    [[TMP3:%.*]] = call <8 x float> @llvm.ceil.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
-; AVX2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.floor.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i64 0
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i64 3
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
+; AVX2-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
+; AVX2-NEXT:    [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
index 3e2c305dbed6..65e5458b25d2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-LABEL: @sitofp_uitofp(
@@ -12,7 +12,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @sitofp_uitofp(
@@ -20,7 +21,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @sitofp_uitofp(
@@ -74,7 +76,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @fptosi_fptoui(
@@ -82,7 +85,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @fptosi_fptoui(
@@ -133,36 +137,50 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 define <8 x float> @fneg_fabs(<8 x float> %a) {
 ; SSE2-LABEL: @fneg_fabs(
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SSE2-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SSE2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; SLM-LABEL: @fneg_fabs(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SLM-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SLM-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SLM-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SLM-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SLM-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SLM-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX-LABEL: @fneg_fabs(
-; AVX-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX2-LABEL: @fneg_fabs(
-; AVX2-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX512-LABEL: @fneg_fabs(
-; AVX512-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX512-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
   %a0 = extractelement <8 x float> %a, i32 0
@@ -214,7 +232,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @sext_zext(
@@ -222,7 +241,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @sext_zext(
@@ -275,7 +295,9 @@ define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; CHECK-NEXT:    [[R71:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
   %a0 = extractelement <4 x i32> %a, i32 0
@@ -321,7 +343,7 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
index 880523d6474a..fad46870ec47 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-LABEL: @sitofp_uitofp(
@@ -12,7 +12,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @sitofp_uitofp(
@@ -20,7 +21,8 @@ define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = uitofp <4 x i32> [[TMP3]] to <4 x float>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP6]], <4 x float> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @sitofp_uitofp(
@@ -74,7 +76,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @fptosi_fptoui(
@@ -82,7 +85,8 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = fptoui <4 x float> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @fptosi_fptoui(
@@ -133,36 +137,50 @@ define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
 define <8 x float> @fneg_fabs(<8 x float> %a) {
 ; SSE2-LABEL: @fneg_fabs(
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE2-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SSE2-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SSE2-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SSE2-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SSE2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SSE2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SSE2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; SLM-LABEL: @fneg_fabs(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP3:%.*]] = bitcast <4 x float> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP3:%.*]] = fneg <4 x float> [[TMP1]]
-; SLM-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP2]])
-; SLM-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP4:%.*]] = bitcast <4 x float> [[TMP2]] to <4 x i32>
+; SLM-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP3]], splat (i32 -2147483648)
+; SLM-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP4]], splat (i32 2147483647)
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
+; SLM-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP8]] to <8 x float>
 ; SLM-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX-LABEL: @fneg_fabs(
-; AVX-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX2-LABEL: @fneg_fabs(
-; AVX2-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX2-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX2-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
 ; AVX512-LABEL: @fneg_fabs(
-; AVX512-NEXT:    [[TMP1:%.*]] = fneg <8 x float> [[A:%.*]]
-; AVX512-NEXT:    [[TMP2:%.*]] = call <8 x float> @llvm.fabs.v8f32(<8 x float> [[A]])
-; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT:    [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT:    [[DOTUNCASTED:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
 ; AVX512-NEXT:    ret <8 x float> [[DOTUNCASTED]]
 ;
   %a0 = extractelement <8 x float> %a, i32 0
@@ -214,7 +232,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SSE2-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE2-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SSE2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @sext_zext(
@@ -222,7 +241,8 @@ define <8 x i32> @sext_zext(<8 x i16> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX-LABEL: @sext_zext(
@@ -275,7 +295,9 @@ define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[B:%.*]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP3:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x float>
-; CHECK-NEXT:    [[R71:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
 ;
   %a0 = extractelement <4 x i32> %a, i32 0
@@ -321,7 +343,7 @@ define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16
 ; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> [[TMP10]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R52:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[R71:%.*]] = shufflevector <8 x float> [[R52]], <8 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; CHECK-NEXT:    ret <8 x float> [[R71]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
index 5cee6984df04..99b13bdc0508 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
@@ -1,34 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
 
 define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fadd_fsub_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fadd_fsub_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fadd_fsub_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fadd_fsub_v8f32(
@@ -80,35 +92,51 @@ define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fmul_fdiv_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v8f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX2-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX2-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v8f32(
@@ -154,30 +182,39 @@ define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SSE-LABEL: @fmul_fdiv_v4f32_const(
-; SSE-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; SSE-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v4f32_const(
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
-; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT:    [[AB2:%.*]] = fdiv float [[A2]], 1.000000e+00
+; SLM-NEXT:    [[AB3:%.*]] = fdiv float [[A3]], 5.000000e-01
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v4f32_const(
-; AVX-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v4f32_const(
-; AVX2-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX2-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX2-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v4f32_const(
-; AVX512-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX512-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
index 9a2f959ac63b..7f9475917b56 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -1,34 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefix=AVX512
 
 define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fadd_fsub_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fadd_fsub_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fadd_fsub_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fsub <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fadd_fsub_v8f32(
@@ -80,35 +92,51 @@ define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @fmul_fdiv_v8f32(
-; SSE-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SSE-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SSE-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SSE-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v8f32(
-; SLM-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; SLM-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; SLM-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; SLM-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v8f32(
-; AVX-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v8f32(
-; AVX2-NEXT:    [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
-; AVX2-NEXT:    [[TMP3:%.*]] = fmul <8 x float> [[A]], [[B]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP9:%.*]] = shufflevector <8 x float> [[TMP3:%.*]], <8 x float> poison, <4 x i32> <i32 1, i32 2, i32 5, i32 6>
+; AVX2-NEXT:    [[TMP10:%.*]] = fdiv <4 x float> [[TMP2]], [[TMP9]]
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> poison, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
+; AVX2-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[TMP11]], [[TMP4]]
+; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP8:%.*]] = call <8 x float> @llvm.vector.insert.v8f32.v4f32(<8 x float> [[TMP7]], <4 x float> [[TMP10]], i64 4)
+; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 5, i32 1, i32 2, i32 6, i32 7, i32 3>
 ; AVX2-NEXT:    ret <8 x float> [[TMP5]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v8f32(
@@ -154,30 +182,39 @@ define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
 
 define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SSE-LABEL: @fmul_fdiv_v4f32_const(
-; SSE-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; SSE-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v4f32_const(
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i64 3
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
 ; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
-; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT:    [[AB2:%.*]] = fdiv float [[A2]], 1.000000e+00
+; SLM-NEXT:    [[AB3:%.*]] = fdiv float [[A3]], 5.000000e-01
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[A2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i64 3
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[TMP3]], float [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
 ;
 ; AVX-LABEL: @fmul_fdiv_v4f32_const(
-; AVX-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX2-LABEL: @fmul_fdiv_v4f32_const(
-; AVX2-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX2-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX2-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; AVX512-LABEL: @fmul_fdiv_v4f32_const(
-; AVX512-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP2:%.*]] = fdiv <4 x float> [[A]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 5.000000e-01>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
 ; AVX512-NEXT:    ret <4 x float> [[TMP1]]
 ;
   %a0 = extractelement <4 x float> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
index f8c5df994453..11ab7770a538 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -1,26 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @add_sub_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @add_sub_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32(
@@ -130,19 +136,25 @@ define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_shl_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32(
@@ -204,7 +216,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SSE-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32_const(
@@ -212,7 +225,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32_const(
@@ -262,24 +276,25 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 
 define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_lshr_shl_v8i32(
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
-; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i64 6
-; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i64 7
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[A]], [[B]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP7:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP10]]
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
 ; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @ashr_lshr_shl_v8i32(
@@ -293,7 +308,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SLM-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SLM-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SLM-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX1-LABEL: @ashr_lshr_shl_v8i32(
@@ -307,7 +324,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX1-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX2-LABEL: @ashr_lshr_shl_v8i32(
@@ -321,7 +340,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @ashr_lshr_shl_v8i32(
@@ -335,7 +356,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX512-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -407,100 +430,130 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
 
 define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SSE-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SSE-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SSE-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SSE-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SSE-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SSE-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SSE-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SLM-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SLM-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SLM-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SLM-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SLM-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SLM-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SLM-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SLM-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; AVX1-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX1-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX1-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; AVX1-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX1-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX1-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX1-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; AVX1-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
-; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; AVX1-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; AVX1-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX2-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX2-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX2-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX2-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX2-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX2-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX512-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX512-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX512-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX512-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX512-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX512-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX512-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> poison, i32 [[AB1]], i64 1
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 poison, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -533,26 +586,28 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; SSE-LABEL: @add_sub_v8i32_splat(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SSE-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @add_sub_v8i32_splat(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SLM-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32_splat(
-; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX1-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX1-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX1-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -560,7 +615,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX1-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX2-LABEL: @add_sub_v8i32_splat(
-; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX2-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -568,7 +623,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX512-LABEL: @add_sub_v8i32_splat(
-; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX512-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index b84ef027f67c..9589ec24d49d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -1,26 +1,32 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
 
 define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @add_sub_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @add_sub_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32(
@@ -130,19 +136,25 @@ define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_shl_v8i32(
-; SSE-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SSE-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32(
-; SLM-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
-; SLM-NEXT:    [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP2:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP8:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP4]]
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SLM-NEXT:    [[TMP9:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shl <4 x i32> [[TMP3]], [[TMP9]]
+; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP7]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32(
@@ -204,7 +216,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SSE-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; SLM-LABEL: @ashr_shl_v8i32_const(
@@ -212,7 +225,8 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 ; SLM-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], splat (i32 2)
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], splat (i32 3)
-; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP6]], <4 x i32> [[TMP4]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX1-LABEL: @ashr_shl_v8i32_const(
@@ -262,24 +276,25 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
 
 define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_lshr_shl_v8i32(
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
-; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i64 6
-; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i64 7
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SSE-NEXT:    [[TMP6:%.*]] = lshr <8 x i32> [[A]], [[B]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
+; SSE-NEXT:    [[TMP7:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP10]]
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
 ; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @ashr_lshr_shl_v8i32(
@@ -293,7 +308,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SLM-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; SLM-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; SLM-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; SLM-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX1-LABEL: @ashr_lshr_shl_v8i32(
@@ -307,7 +324,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX1-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX1-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX1-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX2-LABEL: @ashr_lshr_shl_v8i32(
@@ -321,7 +340,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @ashr_lshr_shl_v8i32(
@@ -335,7 +356,9 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; AVX512-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP9:%.*]] = shl <4 x i32> [[TMP6]], [[TMP7]]
 ; AVX512-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[TMP11]], <8 x i32> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -407,100 +430,130 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
 
 define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 ; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SSE-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SSE-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SSE-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SSE-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SSE-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SSE-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SSE-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; SLM-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; SLM-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; SLM-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; SLM-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SLM-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; SLM-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; SLM-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; SLM-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; SLM-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SLM-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i64 2
-; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i64 3
-; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
-; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i64 6
-; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i64 7
+; AVX1-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX1-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX1-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
 ; AVX1-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX1-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX1-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX1-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
 ; AVX1-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
-; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i64 2
-; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i64 3
-; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i64 5
-; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i64 6
-; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i64 7
+; AVX1-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; AVX1-NEXT:    ret <8 x i32> [[R7]]
 ;
 ; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX2-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX2-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX2-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX2-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX2-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX2-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX2-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX2-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX2-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
+; AVX2-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX2-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; AVX2-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX2-NEXT:    ret <8 x i32> [[R71]]
 ;
 ; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i64 1
-; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i64 5
+; AVX512-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX512-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX512-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; AVX512-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX512-NEXT:    [[AB0:%.*]] = sdiv i32 [[A0]], undef
 ; AVX512-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
 ; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
 ; AVX512-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
+; AVX512-NEXT:    [[AB4:%.*]] = sdiv i32 [[A4]], undef
 ; AVX512-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
 ; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
 ; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 undef, i32 poison, i32 poison, i32 poison>, i32 [[AB1]], i64 1
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
 ; AVX512-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 poison, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB5]], i64 5
+; AVX512-NEXT:    [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
 ; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 poison, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; AVX512-NEXT:    [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
 ; AVX512-NEXT:    ret <8 x i32> [[R71]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
@@ -533,26 +586,28 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
 define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; SSE-LABEL: @add_sub_v8i32_splat(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SSE-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @add_sub_v8i32_splat(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i64 0
+; SLM-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i32 0
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; SLM-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
 ; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP3]], [[TMP5]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT:    [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> [[TMP8]], <4 x i32> [[TMP6]], i64 4)
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX1-LABEL: @add_sub_v8i32_splat(
-; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX1-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX1-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX1-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX1-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -560,7 +615,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX1-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX2-LABEL: @add_sub_v8i32_splat(
-; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX2-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX2-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
@@ -568,7 +623,7 @@ define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
 ; AVX2-NEXT:    ret <8 x i32> [[TMP5]]
 ;
 ; AVX512-LABEL: @add_sub_v8i32_splat(
-; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i64 0
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[B:%.*]], i32 0
 ; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX512-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
 ; AVX512-NEXT:    [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
index 3ee6c55e429b..c2bd9e648e4f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
 
 define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @g(
@@ -86,8 +86,8 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[Y]], <2 x i32> <i32 3, i32 6>
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i8> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sdiv i8 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    ret i8 [[TMP8]]
 ;
@@ -107,14 +107,14 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 
 define i8 @k(<4 x i8> %x) {
 ; CHECK-LABEL: @k(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
@@ -136,14 +136,14 @@ define i8 @k_bb(<4 x i8> %x) {
 ; CHECK-LABEL: @k_bb(
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
index 60e008d12e6e..e4e45eb99fbd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 | FileCheck %s
 
 define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @g(
@@ -39,8 +39,14 @@ define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
 
 define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-LABEL: @h_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> <i8 undef, i8 poison, i8 poison, i8 poison>, <4 x i32> <i32 4, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[Y:%.*]], <4 x i32> <i32 0, i32 1, i32 5, i32 6>
+; CHECK-NEXT:    [[X0:%.*]] = extractelement <4 x i8> undef, i32 0
+; CHECK-NEXT:    [[X3:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 3
+; CHECK-NEXT:    [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT:    [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i8> poison, i8 [[X0]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[X3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[Y1]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i8> [[TMP5]], i8 [[Y2]], i32 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[TMP2]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i8> [[TMP3]]
 ;
@@ -87,8 +93,8 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[Y]], <2 x i32> <i32 3, i32 6>
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <2 x i8> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i8> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP5]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = sdiv i8 [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    ret i8 [[TMP8]]
 ;
@@ -108,14 +114,14 @@ define i8 @j(<4 x i8> %x, <4 x i8> %y) {
 
 define i8 @k(<4 x i8> %x) {
 ; CHECK-LABEL: @k(
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
@@ -137,14 +143,14 @@ define i8 @k_bb(<4 x i8> %x) {
 ; CHECK-LABEL: @k_bb(
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    [[TMP1:%.*]] = mul <4 x i8> [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i8> [[X]], [[X]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i8> [[TMP1:%.*]], <4 x i8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <2 x i8> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = mul <2 x i8> [[TMP3]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i8> [[TMP4]], <2 x i8> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i8> [[TMP10]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i8> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i8> [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv i8 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    ret i8 [[TMP9]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
index fd5f09bf2adc..7d5ea22436c7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
 
 ;
 ; Check that we can commute operands based on the predicate.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
index 35619d6d3ad1..25aa24b4cc81 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64--- -mattr=+avx  | FileCheck %s
 
 ;
 ; Check that we can commute operands based on the predicate.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll b/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
index d87c40511fcf..3a7184f8e051 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
@@ -7,26 +7,16 @@ define void @test(i64 %v) {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> <i64 0, i64 poison>, i64 [[V]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 0, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP9]], false
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 0, [[TMP2]]
-; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP18:%.*]] = and i1 [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i64> [[TMP2]], <8 x i64> <i64 poison, i64 0, i64 0, i64 0, i64 poison, i64 0, i64 0, i64 0>, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 4, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <8 x i64> zeroinitializer, [[TMP3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult i64 0, 0
+; CHECK-NEXT:    [[TMP6:%.*]] = freeze <8 x i1> [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i1 [[TMP18]], i1 false
-; CHECK-NEXT:    br i1 [[TMP20]], label %[[BB_I107_PREHEADER:.*]], label %[[BB_I27_I_PREHEADER:.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = freeze i1 [[TMP20]]
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = select i1 [[TMP8]], i1 false, i1 false
+; CHECK-NEXT:    br i1 [[OP_RDX1]], label %[[BB_I107_PREHEADER:.*]], label %[[BB_I27_I_PREHEADER:.*]]
 ; CHECK:       [[BB_I107_PREHEADER]]:
 ; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
 ; CHECK-NEXT:    [[DOTSROA_1278_10_EXTRACT_SHIFT83_I1622_1:%.*]] = xor i64 0, [[TMP21]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
index b85ec5bce819..815ad044ca07 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 ;
 ; 128-bit vectors
@@ -169,21 +169,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -214,14 +214,14 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; PR50392
 define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b) {
 ; CHECK-LABEL: @test_v4f64_partial_swizzle(
-; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; CHECK-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; CHECK-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i64 3
+; CHECK-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[TMP4]], double [[R3]], i32 3
 ; CHECK-NEXT:    ret <4 x double> [[R03]]
 ;
   %a0 = extractelement <4 x double> %a, i64 0
@@ -243,21 +243,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -305,21 +305,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -351,21 +351,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -413,21 +413,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
index b83d35541bba..fb037b87da54 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
 
 ;
 ; 128-bit vectors
@@ -169,21 +169,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -214,15 +214,15 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; PR50392
 define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64_partial_swizzle(
-; SSE-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; SSE-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; SSE-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; SSE-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; SSE-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; SSE-NEXT:    [[R0212:%.*]] = insertelement <4 x double> [[TMP4]], double 0.000000e+00, i64 1
-; SSE-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R0212]], double [[R3]], i64 3
+; SSE-NEXT:    [[R021:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+; SSE-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R021]], double [[R3]], i32 3
 ; SSE-NEXT:    ret <4 x double> [[R03]]
 ;
 ; SLM-LABEL: @test_v4f64_partial_swizzle(
@@ -232,21 +232,21 @@ define <4 x double> @test_v4f64_partial_swizzle(<4 x double> %a, <4 x double> %b
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> poison, <2 x i32> <i32 1, i32 2>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[B]], <4 x double> poison, <2 x i32> <i32 0, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
-; SLM-NEXT:    [[R00:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, double [[R0]], i64 0
+; SLM-NEXT:    [[R00:%.*]] = insertelement <4 x double> zeroinitializer, double [[R0]], i32 0
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SLM-NEXT:    [[R031:%.*]] = shufflevector <4 x double> [[R00]], <4 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
 ; SLM-NEXT:    ret <4 x double> [[R031]]
 ;
 ; AVX-LABEL: @test_v4f64_partial_swizzle(
-; AVX-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i64 2
-; AVX-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i64 3
+; AVX-NEXT:    [[B2:%.*]] = extractelement <4 x double> [[B:%.*]], i32 2
+; AVX-NEXT:    [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
 ; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B]], <2 x i32> <i32 0, i32 4>
 ; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; AVX-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; AVX-NEXT:    [[R3:%.*]] = fadd double [[B2]], [[B3]]
 ; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
-; AVX-NEXT:    [[R0212:%.*]] = insertelement <4 x double> [[TMP4]], double 0.000000e+00, i64 1
-; AVX-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R0212]], double [[R3]], i64 3
+; AVX-NEXT:    [[R021:%.*]] = shufflevector <4 x double> zeroinitializer, <4 x double> [[TMP4]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+; AVX-NEXT:    [[R03:%.*]] = insertelement <4 x double> [[R021]], double [[R3]], i32 3
 ; AVX-NEXT:    ret <4 x double> [[R03]]
 ;
   %a0 = extractelement <4 x double> %a, i64 0
@@ -268,21 +268,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fadd <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -330,21 +330,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -376,21 +376,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = add <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -438,21 +438,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = add <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = add <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = add <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
index 40b6a8c32f5d..6f655b3e9ae0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
 
 ;
 ; 128-bit vectors
@@ -148,21 +148,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -194,21 +194,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -256,21 +256,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -302,21 +302,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -364,21 +364,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
index 09113323d3ab..2ab81d9b8d30 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer,instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
 
 ;
 ; 128-bit vectors
@@ -148,21 +148,21 @@ define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
 ; SSE-LABEL: @test_v4f64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4f64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <2 x double> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <4 x double> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x double> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4f64(
@@ -194,21 +194,21 @@ define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
 ; SSE-LABEL: @test_v8f32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8f32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = fsub <4 x float> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = fsub <4 x float> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = fsub <8 x float> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x float> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8f32(
@@ -256,21 +256,21 @@ define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; SSE-LABEL: @test_v4i64(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v4i64(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 4>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 2, i32 6>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 1, i32 5>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <2 x i32> <i32 3, i32 7>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <2 x i64> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <2 x i64> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <4 x i64> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <4 x i64> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v4i64(
@@ -302,21 +302,21 @@ define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @test_v8i32(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v8i32(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <4 x i32> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <4 x i32> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <8 x i32> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <8 x i32> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v8i32(
@@ -364,21 +364,21 @@ define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; SSE-LABEL: @test_v16i16(
 ; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SSE-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SSE-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; SLM-LABEL: @test_v16i16(
 ; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
 ; SLM-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SLM-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
 ; SLM-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
-; SLM-NEXT:    [[TMP5:%.*]] = sub <8 x i16> [[TMP1]], [[TMP3]]
-; SLM-NEXT:    [[TMP6:%.*]] = sub <8 x i16> [[TMP2]], [[TMP4]]
-; SLM-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP5]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT:    [[TMP7:%.*]] = sub <16 x i16> [[TMP5]], [[TMP6]]
 ; SLM-NEXT:    ret <16 x i16> [[TMP7]]
 ;
 ; AVX-LABEL: @test_v16i16(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 9c9a89ac76d8..58fb5f772207 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+sse2     -S | FileCheck %s --check-prefix=SSE
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx      -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx2     -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx512f  -S | FileCheck %s --check-prefix=AVX
-; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer,instcombine -mattr=+avx512vl -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+sse2     -S | FileCheck %s --check-prefix=SSE
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx      -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx2     -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx512f  -S | FileCheck %s --check-prefix=AVX
+; RUN: opt < %s -slp-threshold=-6 -passes=slp-vectorizer -mattr=+avx512vl -S | FileCheck %s --check-prefix=AVX
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -17,15 +17,15 @@ target triple = "x86_64-unknown-linux-gnu"
 define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_zext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -33,15 +33,15 @@ define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ;
 ; AVX-LABEL: @PR31243_zext(
 ; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; AVX-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; AVX-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i64 [[TMP6]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -76,15 +76,15 @@ entry:
 define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ; SSE-LABEL: @PR31243_sext(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; SSE-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
-; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; SSE-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
+; SSE-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; SSE-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; SSE-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; SSE-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -92,15 +92,15 @@ define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
 ;
 ; AVX-LABEL: @PR31243_sext(
 ; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i64 0
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i64 1
+; AVX-NEXT:    [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
 ; AVX-NEXT:    [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i64 0
-; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i64
-; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[TMP4]]
-; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i64 1
-; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i64
-; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[TMP6]]
+; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; AVX-NEXT:    [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
+; AVX-NEXT:    [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
+; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; AVX-NEXT:    [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
+; AVX-NEXT:    [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
 ; AVX-NEXT:    [[T6:%.*]] = load i8, ptr [[T4]], align 1
 ; AVX-NEXT:    [[T7:%.*]] = load i8, ptr [[T5]], align 1
 ; AVX-NEXT:    [[T8:%.*]] = add i8 [[T6]], [[T7]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
index 61938d01e57a..de7252134543 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-threshold=-100 -S -mtriple=i386-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 
@@ -10,16 +10,16 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 define void @shuffle_operands1(ptr noalias %from, ptr noalias %to, double %v1, double %v2) {
 ; CHECK-LABEL: @shuffle_operands1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 ; SSE2-LABEL: @shuffle_operands1(
 ; SSE2-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i64 0
-; SSE2-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V1:%.*]], i32 0
+; SSE2-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V2:%.*]], i32 1
 ; SSE2-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; SSE2-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    ret void
@@ -43,7 +43,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -57,7 +57,7 @@ define void @vecload_vs_broadcast(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP0]], [[TMP2]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -91,7 +91,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -105,7 +105,7 @@ define void @vecload_vs_broadcast2(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -139,7 +139,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -153,7 +153,7 @@ define void @vecload_vs_broadcast3(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i64 0
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[P]], i32 0
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP0]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -185,12 +185,12 @@ define void @shuffle_nodes_match1(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    br label [[LP:%.*]]
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr i8, ptr [[FROM:%.*]], i32 8
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, ptr [[FROM:%.*]], i64 1
 ; CHECK-NEXT:    [[V0_1:%.*]] = load double, ptr [[FROM]], align 4
 ; CHECK-NEXT:    [[V0_2:%.*]] = load double, ptr [[FROM_1]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i32 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
@@ -205,7 +205,7 @@ define void @shuffle_nodes_match1(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -239,7 +239,7 @@ define void @vecload_vs_broadcast4(ptr noalias %from, ptr noalias %to, double %v
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -253,7 +253,7 @@ define void @vecload_vs_broadcast4(ptr noalias %from, ptr noalias %to, double %v
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], [[TMP1]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -286,13 +286,13 @@ define void @shuffle_nodes_match2(ptr noalias %from, ptr noalias %to, double %v1
 ; CHECK-NEXT:    br label [[LP:%.*]]
 ; CHECK:       lp:
 ; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr i8, ptr [[FROM:%.*]], i32 8
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, ptr [[FROM:%.*]], i64 1
 ; CHECK-NEXT:    [[V0_1:%.*]] = load double, ptr [[FROM]], align 4
 ; CHECK-NEXT:    [[V0_2:%.*]] = load double, ptr [[FROM_1]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> poison, double [[V0_1]], i32 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> poison, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i32 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    store <2 x double> [[TMP4]], ptr [[TO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -306,7 +306,7 @@ define void @shuffle_nodes_match2(ptr noalias %from, ptr noalias %to, double %v1
 ; SSE2-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
 ; SSE2-NEXT:    [[TMP0:%.*]] = load <2 x double>, ptr [[FROM:%.*]], align 4
 ; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
-; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i64 1
+; SSE2-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
 ; SSE2-NEXT:    [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
 ; SSE2-NEXT:    store <2 x double> [[TMP3]], ptr [[TO:%.*]], align 4
 ; SSE2-NEXT:    br i1 [[C:%.*]], label [[LP]], label [[EXT:%.*]]
@@ -347,23 +347,20 @@ define void @good_load_order() {
 ; CHECK:       for.body3:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP13:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
 ; CHECK-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; CHECK-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; CHECK-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
@@ -382,23 +379,20 @@ define void @good_load_order() {
 ; SSE2:       for.body3:
 ; SSE2-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP13:%.*]], [[FOR_BODY3]] ]
 ; SSE2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
-; SSE2-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
-; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP3]]
-; SSE2-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP4]]
-; SSE2-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; SSE2-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
-; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP6]]
+; SSE2-NEXT:    [[TMP2:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; SSE2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP2]]
+; SSE2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV]]
+; SSE2-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], 4
+; SSE2-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[TMP3]]
 ; SSE2-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX31]], align 4
 ; SSE2-NEXT:    [[TMP8:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
 ; SSE2-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 2>
-; SSE2-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP1]], i64 0
+; SSE2-NEXT:    [[TMP12:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; SSE2-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP12]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
 ; SSE2-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
 ; SSE2-NEXT:    store <4 x float> [[TMP11]], ptr [[ARRAYIDX5]], align 4
 ; SSE2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
-; SSE2-NEXT:    [[TMP12:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
-; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i32 0, i32 [[TMP12]]
+; SSE2-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], ptr @a, i64 0, i64 [[INDVARS_IV_NEXT]]
 ; SSE2-NEXT:    [[TMP13]] = load float, ptr [[ARRAYIDX41]], align 4
 ; SSE2-NEXT:    [[MUL45:%.*]] = fmul float [[TMP13]], [[TMP7]]
 ; SSE2-NEXT:    store float [[MUL45]], ptr [[ARRAYIDX31]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
index 2588012847d0..6fd2de8ad8ab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr40522.ll
@@ -1,13 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-- -mcpu=corei7 < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-- -mcpu=corei7 < %s | FileCheck %s
 
 define void @test1(float %a, float %b, float %c, float %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
 ; CHECK-NEXT:    ret void
@@ -30,10 +30,10 @@ entry:
 define void @test1_vec(float %a, float %b, float %c, float %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test1_vec(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> [[TMP1]], float [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = fptosi <4 x float> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 16, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
@@ -54,10 +54,10 @@ entry:
 define void @test2(i32 %a, i32 %b, i32 %c, i32 %d, ptr nocapture %p) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C:%.*]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[D:%.*]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C:%.*]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[D:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], splat (i32 1)
 ; CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[P:%.*]], align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
@@ -79,10 +79,10 @@ entry:
 
 define void @test2_vec(i32 %0, i32 %1, i32 %2, i32 %3, ptr nocapture %4) {
 ; CHECK-LABEL: @test2_vec(
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0:%.*]], i64 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP1:%.*]], i64 1
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP2:%.*]], i64 2
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP3:%.*]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP2:%.*]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP3:%.*]], i32 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <4 x i32> [[TMP9]], splat (i32 1)
 ; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr [[TMP4:%.*]], align 16, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
index bae127527bbb..9e4f10ec7b34 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
@@ -1,19 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2   -slp-threshold=-1 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx    | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2   | FileCheck %s
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2   -slp-threshold=-1 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx    | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2   | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s
 
 define void @store_i32(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i32(
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <4 x i32> [[TMP4]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = lshr <4 x i32> [[TMP7]], splat (i32 15)
-; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP8]], <4 x i32> splat (i32 255))
-; CHECK-NEXT:    store <4 x i32> [[TMP9]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ult <4 x i32> [[TMP8]], splat (i32 255)
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP8]], <4 x i32> splat (i32 255)
+; CHECK-NEXT:    store <4 x i32> [[TMP10]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i32, ptr %0, align 4, !tbaa !2
@@ -50,13 +51,14 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i8(
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[TMP0:%.*]], align 1, !tbaa [[TBAA4:![0-9]+]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP7]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = lshr <4 x i32> [[TMP8]], splat (i32 15)
-; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> [[TMP9]], <4 x i32> splat (i32 255))
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc nuw <4 x i32> [[TMP10]] to <4 x i8>
-; CHECK-NEXT:    store <4 x i8> [[TMP11]], ptr [[TMP0]], align 1, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <4 x i32> [[TMP9]], splat (i32 255)
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP9]], <4 x i32> splat (i32 255)
+; CHECK-NEXT:    [[TMP12:%.*]] = trunc <4 x i32> [[TMP11]] to <4 x i8>
+; CHECK-NEXT:    store <4 x i8> [[TMP12]], ptr [[TMP0]], align 1, !tbaa [[TBAA4]]
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i8, ptr %0, align 1, !tbaa !6
@@ -101,7 +103,7 @@ define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) {
 ; CHECK-LABEL: @store_i64(
 ; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
index a9c0eb3f9f2b..1b11c3dcc081 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47623.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512
 
 
 @b = global [8 x i32] zeroinitializer, align 16
@@ -13,14 +13,14 @@ define void @foo() {
 ; SSE-LABEL: @foo(
 ; SSE-NEXT:    [[TMP1:%.*]] = load i32, ptr @b, align 16
 ; SSE-NEXT:    store i32 [[TMP1]], ptr @a, align 16
-; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds nuw (i8, ptr @b, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 4), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 8), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 12), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 16), align 16
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 20), align 4
-; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
-; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds nuw (i8, ptr @a, i64 28), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load i32, ptr getelementptr inbounds ([8 x i32], ptr @b, i64 0, i64 2), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 2), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 4), align 16
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[TMP1]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 6), align 8
+; SSE-NEXT:    store i32 [[TMP2]], ptr getelementptr inbounds ([8 x i32], ptr @a, i64 0, i64 7), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @foo(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
index a0e52c13ec62..db38a6201739 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
@@ -1,89 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; SSE-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; SSE-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; SSE-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; SSE-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; SSE-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX2-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX2-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512F-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512F-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX512F-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512F-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512F-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512VL-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512VL-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX512VL-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512VL-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512VL-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    ret void
 ;
   %3 = getelementptr inbounds i32, ptr %1, i64 1
@@ -107,64 +122,64 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX2-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512F-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -172,8 +187,8 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -207,39 +222,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 5
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 6
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 7
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -247,41 +262,38 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
-; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i64 2
-; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i64 3
-; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i64 4
-; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i64 5
-; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i64 6
-; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i64 7
+; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i32 0
+; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i32 2
+; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i32 3
+; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i32 4
+; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i32 5
+; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i32 6
+; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i32 7
 ; AVX-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_3(
-; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 24
-; AVX2-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP1]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <4 x i32> <i32 0, i32 11, i32 4, i32 15>
-; AVX2-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr nonnull [[TMP14]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> poison, <4 x i32> <i32 12, i32 3, i32 0, i32 15>
-; AVX2-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP3:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr [[TMP1:%.*]], i32 4, <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <22 x i32> [[TMP3]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
+; AVX2-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[TMP4]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
+; AVX2-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_3(
@@ -343,20 +355,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, ptr [[T0:%.*]], i64 1
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 2
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 3
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 4
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 5
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 6
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 7
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -384,13 +396,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -399,27 +411,24 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    [[T23:%.*]] = load i32, ptr [[T22]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T27:%.*]] = load i32, ptr [[T26]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
-; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i64 1
-; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i64 2
-; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i64 3
-; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i64 4
-; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i64 5
-; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i64 6
-; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i32 1
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i32 2
+; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i32 3
+; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i32 4
+; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i32 5
+; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i32 6
+; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i32 7
 ; AVX-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_4(
-; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 24
-; AVX2-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[T1]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <4 x i32> <i32 0, i32 11, i32 4, i32 15>
-; AVX2-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr nonnull [[T26]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP3]], <16 x i32> poison, <4 x i32> <i32 12, i32 3, i32 0, i32 15>
-; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP1:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr [[T1:%.*]], i32 4, <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <22 x i32> [[TMP1]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
+; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
+; AVX2-NEXT:    store <8 x i32> [[TMP4]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_4(
@@ -487,153 +496,154 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
 ; SSE-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP17:%.*]] = shufflevector <4 x float> [[TMP15]], <4 x float> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
-; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; SSE-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP12]], <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP21:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
+; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i32 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
-; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
-; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i64 2
-; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i64 3
-; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i64 0
-; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i64 1
-; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i64 2
-; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i64 3
+; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i32 0
+; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i32 1
+; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i32 2
+; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i32 3
+; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i32 0
+; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i32 1
+; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i32 2
+; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i32 3
 ; SSE-NEXT:    [[TMP48:%.*]] = fdiv <4 x float> [[TMP43]], [[TMP47]]
 ; SSE-NEXT:    store <4 x float> [[TMP48]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX2-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX2-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX2-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX2-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_div(
 ; AVX512F-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512F-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512F-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
@@ -643,6 +653,7 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX512VL-LABEL: @gather_load_div(
 ; AVX512VL-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
index 6c5638819dce..bfa361080496 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
@@ -1,89 +1,104 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
-; RUN:  opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+sse2     | FileCheck %s --check-prefixes=SSE
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx      | FileCheck %s --check-prefixes=AVX
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx2     | FileCheck %s --check-prefixes=AVX2
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512f  | FileCheck %s --check-prefixes=AVX512F
+; RUN:  opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 
 define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; SSE-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; SSE-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; SSE-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; SSE-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; SSE-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; SSE-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; SSE-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; SSE-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX2-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX2-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512F-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512F-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512F-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512F-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512F-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512F-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512F-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512F-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512F-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512F-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512F-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX512F-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512F-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512F-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512F-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512F-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX512VL-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
-; AVX512VL-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
-; AVX512VL-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX512VL-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX512VL-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i64 1
-; AVX512VL-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i64 2
-; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i64 3
+; AVX512VL-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX512VL-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
+; AVX512VL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX512VL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
+; AVX512VL-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; AVX512VL-NEXT:    [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP7]], i32 1
+; AVX512VL-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP10]], i32 2
+; AVX512VL-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 3
 ; AVX512VL-NEXT:    [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
-; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    store <4 x i32> [[TMP14]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    ret void
 ;
   %3 = getelementptr inbounds i32, ptr %1, i64 1
@@ -107,64 +122,64 @@ define void @gather_load(ptr noalias nocapture %0, ptr noalias nocapture readonl
 
 define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_2(
-; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; SSE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; SSE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP5]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; SSE-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP9]], ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; SSE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; SSE-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP13]], ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
 ; SSE-NEXT:    store i32 [[TMP17]], ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_2(
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_2(
-; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
+; AVX2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
 ; AVX2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 10
 ; AVX2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 3
 ; AVX2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0
-; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i64 1
-; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i64 2
-; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i64 3
+; AVX2-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP6]], i32 1
+; AVX2-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP8]], i32 2
+; AVX2-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP10]], i32 3
 ; AVX2-NEXT:    [[TMP15:%.*]] = add nsw <4 x i32> [[TMP14]], <i32 1, i32 2, i32 3, i32 4>
 ; AVX2-NEXT:    store <4 x i32> [[TMP15]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_2(
-; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512F-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512F-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512F-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -172,8 +187,8 @@ define void @gather_load_2(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; AVX512F-NEXT:    ret void
 ;
 ; AVX512VL-LABEL: @gather_load_2(
-; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 4
-; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr nonnull [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i64 1
+; AVX512VL-NEXT:    [[TMP4:%.*]] = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr [[TMP3]], i32 4, <10 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true>, <10 x i32> poison), !tbaa [[TBAA0]]
 ; AVX512VL-NEXT:    [[TMP5:%.*]] = shufflevector <10 x i32> [[TMP4]], <10 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 9>
 ; AVX512VL-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[TMP5]], <i32 1, i32 3, i32 4, i32 2>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 1, i32 2>
@@ -207,39 +222,39 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ; SSE-LABEL: @gather_load_3(
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 1
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 4
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP0:%.*]], i64 1
 ; SSE-NEXT:    store i32 [[TMP4]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; SSE-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP8:%.*]] = add i32 [[TMP7]], 2
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2
 ; SSE-NEXT:    store i32 [[TMP8]], ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; SSE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; SSE-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], 3
-; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 12
+; SSE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3
 ; SSE-NEXT:    store i32 [[TMP12]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; SSE-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; SSE-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP16:%.*]] = add i32 [[TMP15]], 4
-; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; SSE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
 ; SSE-NEXT:    store i32 [[TMP16]], ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; SSE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; SSE-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP20:%.*]] = add i32 [[TMP19]], 1
-; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 20
+; SSE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 5
 ; SSE-NEXT:    store i32 [[TMP20]], ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; SSE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; SSE-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP24:%.*]] = add i32 [[TMP23]], 2
-; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 24
+; SSE-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 6
 ; SSE-NEXT:    store i32 [[TMP24]], ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; SSE-NEXT:    [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP28:%.*]] = add i32 [[TMP27]], 3
-; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 28
+; SSE-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 7
 ; SSE-NEXT:    store i32 [[TMP28]], ptr [[TMP25]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; SSE-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP32:%.*]] = add i32 [[TMP31]], 4
 ; SSE-NEXT:    store i32 [[TMP32]], ptr [[TMP29]], align 4, !tbaa [[TBAA0]]
@@ -247,41 +262,38 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 ;
 ; AVX-LABEL: @gather_load_3(
 ; AVX-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 44
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
 ; AVX-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
 ; AVX-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 60
+; AVX-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
 ; AVX-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 72
+; AVX-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
 ; AVX-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 36
+; AVX-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 9
 ; AVX-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 24
+; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
 ; AVX-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 84
+; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
 ; AVX-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i64 1
-; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i64 2
-; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i64 3
-; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i64 4
-; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i64 5
-; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i64 6
-; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i64 7
+; AVX-NEXT:    [[TMP18:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i32 0
+; AVX-NEXT:    [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP5]], i32 1
+; AVX-NEXT:    [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP7]], i32 2
+; AVX-NEXT:    [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP9]], i32 3
+; AVX-NEXT:    [[TMP22:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP11]], i32 4
+; AVX-NEXT:    [[TMP23:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP13]], i32 5
+; AVX-NEXT:    [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP15]], i32 6
+; AVX-NEXT:    [[TMP25:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP17]], i32 7
 ; AVX-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_3(
-; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1:%.*]], i64 24
-; AVX2-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[TMP1]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> poison, <4 x i32> <i32 0, i32 11, i32 4, i32 15>
-; AVX2-NEXT:    [[TMP6:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr nonnull [[TMP14]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> poison, <4 x i32> <i32 12, i32 3, i32 0, i32 15>
-; AVX2-NEXT:    [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP26:%.*]] = add <8 x i32> [[TMP25]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <8 x i32> [[TMP26]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP3:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr [[TMP1:%.*]], i32 4, <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <22 x i32> [[TMP3]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
+; AVX2-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[TMP4]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
+; AVX2-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
+; AVX2-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_3(
@@ -343,20 +355,20 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
 
 define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture readonly %t1) {
 ; SSE-LABEL: @gather_load_4(
-; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds nuw i8, ptr [[T0:%.*]], i64 4
-; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 8
-; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 12
-; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 16
-; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 20
-; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 24
-; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds nuw i8, ptr [[T0]], i64 28
-; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; SSE-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, ptr [[T0:%.*]], i64 1
+; SSE-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; SSE-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 2
+; SSE-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; SSE-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 3
+; SSE-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; SSE-NEXT:    [[T17:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 4
+; SSE-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; SSE-NEXT:    [[T21:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 5
+; SSE-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; SSE-NEXT:    [[T25:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 6
+; SSE-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; SSE-NEXT:    [[T29:%.*]] = getelementptr inbounds i32, ptr [[T0]], i64 7
+; SSE-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; SSE-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -384,13 +396,13 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_4(
-; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 44
-; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 16
-; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 60
-; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 72
-; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 36
-; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 24
-; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds nuw i8, ptr [[T1]], i64 84
+; AVX-NEXT:    [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1:%.*]], i64 11
+; AVX-NEXT:    [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 9
+; AVX-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX-NEXT:    [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
 ; AVX-NEXT:    [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[TBAA0]]
@@ -399,27 +411,24 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 ; AVX-NEXT:    [[T23:%.*]] = load i32, ptr [[T22]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T27:%.*]] = load i32, ptr [[T26]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
-; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i64 1
-; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i64 2
-; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i64 3
-; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i64 4
-; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i64 5
-; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i64 6
-; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i64 7
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[T7]], i32 1
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[T11]], i32 2
+; AVX-NEXT:    [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T15]], i32 3
+; AVX-NEXT:    [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T19]], i32 4
+; AVX-NEXT:    [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T23]], i32 5
+; AVX-NEXT:    [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T27]], i32 6
+; AVX-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[T31]], i32 7
 ; AVX-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
 ; AVX-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_4(
-; AVX2-NEXT:    [[T26:%.*]] = getelementptr inbounds nuw i8, ptr [[T1:%.*]], i64 24
-; AVX2-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr [[T1]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <4 x i32> <i32 0, i32 11, i32 4, i32 15>
-; AVX2-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr nonnull [[T26]], i32 4, <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <16 x i32> poison), !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP3]], <16 x i32> poison, <4 x i32> <i32 12, i32 3, i32 0, i32 15>
-; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP9:%.*]] = add <8 x i32> [[TMP8]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
-; AVX2-NEXT:    store <8 x i32> [[TMP9]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP1:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr [[T1:%.*]], i32 4, <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[TBAA0]]
+; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <22 x i32> [[TMP1]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
+; AVX2-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[TMP2]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
+; AVX2-NEXT:    [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
+; AVX2-NEXT:    store <8 x i32> [[TMP4]], ptr [[T0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_4(
@@ -487,153 +496,154 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
 define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture readonly %1) {
 ; SSE-LABEL: @gather_load_div(
 ; SSE-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; SSE-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; SSE-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; SSE-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; SSE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; SSE-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0:%.*]], i64 16
+; SSE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP0:%.*]], i64 4
 ; SSE-NEXT:    [[TMP10:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
 ; SSE-NEXT:    [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP15:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 4, i32 poison, i32 poison>
-; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP17:%.*]] = shufflevector <4 x float> [[TMP15]], <4 x float> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 4, i32 poison>
-; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP18:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP19:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
 ; SSE-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> [[TMP12]], <4 x i32> <i32 1, i32 2, i32 poison, i32 poison>
 ; SSE-NEXT:    [[TMP21:%.*]] = shufflevector <4 x float> [[TMP20]], <4 x float> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
-; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i64 3
+; SSE-NEXT:    [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP8]], i32 3
 ; SSE-NEXT:    [[TMP23:%.*]] = fdiv <4 x float> [[TMP19]], [[TMP22]]
 ; SSE-NEXT:    store <4 x float> [[TMP23]], ptr [[TMP0]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; SSE-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; SSE-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; SSE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; SSE-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP26]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; SSE-NEXT:    [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; SSE-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; SSE-NEXT:    [[TMP31:%.*]] = load float, ptr [[TMP30]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; SSE-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; SSE-NEXT:    [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; SSE-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; SSE-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; SSE-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; SSE-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; SSE-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; SSE-NEXT:    [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4, !tbaa [[TBAA0]]
-; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i64 0
-; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i64 1
-; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i64 2
-; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i64 3
-; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i64 0
-; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i64 1
-; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i64 2
-; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i64 3
+; SSE-NEXT:    [[TMP40:%.*]] = insertelement <4 x float> poison, float [[TMP25]], i32 0
+; SSE-NEXT:    [[TMP41:%.*]] = insertelement <4 x float> [[TMP40]], float [[TMP29]], i32 1
+; SSE-NEXT:    [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP33]], i32 2
+; SSE-NEXT:    [[TMP43:%.*]] = insertelement <4 x float> [[TMP42]], float [[TMP37]], i32 3
+; SSE-NEXT:    [[TMP44:%.*]] = insertelement <4 x float> poison, float [[TMP27]], i32 0
+; SSE-NEXT:    [[TMP45:%.*]] = insertelement <4 x float> [[TMP44]], float [[TMP31]], i32 1
+; SSE-NEXT:    [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP35]], i32 2
+; SSE-NEXT:    [[TMP47:%.*]] = insertelement <4 x float> [[TMP46]], float [[TMP39]], i32 3
 ; SSE-NEXT:    [[TMP48:%.*]] = fdiv <4 x float> [[TMP43]], [[TMP47]]
 ; SSE-NEXT:    store <4 x float> [[TMP48]], ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @gather_load_div(
 ; AVX-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX-NEXT:    ret void
 ;
 ; AVX2-LABEL: @gather_load_div(
 ; AVX2-NEXT:    [[TMP3:%.*]] = load float, ptr [[TMP1:%.*]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 40
-; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
-; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 12
-; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 176
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 10
+; AVX2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 13
+; AVX2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 3
+; AVX2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 44
 ; AVX2-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 68
+; AVX2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 17
 ; AVX2-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 132
+; AVX2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 33
 ; AVX2-NEXT:    [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 32
+; AVX2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 8
 ; AVX2-NEXT:    [[TMP14:%.*]] = load float, ptr [[TMP13]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 120
+; AVX2-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 30
 ; AVX2-NEXT:    [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 20
+; AVX2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 5
 ; AVX2-NEXT:    [[TMP18:%.*]] = load float, ptr [[TMP17]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 108
+; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 27
 ; AVX2-NEXT:    [[TMP20:%.*]] = load float, ptr [[TMP19]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 20
 ; AVX2-NEXT:    [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 92
+; AVX2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 23
 ; AVX2-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP23]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP25:%.*]] = load <2 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP26:%.*]] = load <2 x float>, ptr [[TMP4]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    [[TMP27:%.*]] = load <2 x float>, ptr [[TMP5]], align 4, !tbaa [[TBAA0]]
-; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i64 0
+; AVX2-NEXT:    [[TMP28:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
 ; AVX2-NEXT:    [[TMP29:%.*]] = shufflevector <2 x float> [[TMP26]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP30:%.*]] = shufflevector <8 x float> [[TMP28]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP31:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP32:%.*]] = shufflevector <8 x float> [[TMP30]], <8 x float> [[TMP31]], <8 x i32> <i32 0, i32 1, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT:    [[TMP33:%.*]] = shufflevector <2 x float> [[TMP27]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP34:%.*]] = shufflevector <8 x float> [[TMP32]], <8 x float> [[TMP33]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i64 4
-; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i64 5
-; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i64 6
-; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i64 7
+; AVX2-NEXT:    [[TMP35:%.*]] = insertelement <8 x float> [[TMP34]], float [[TMP10]], i32 4
+; AVX2-NEXT:    [[TMP36:%.*]] = insertelement <8 x float> [[TMP35]], float [[TMP14]], i32 5
+; AVX2-NEXT:    [[TMP37:%.*]] = insertelement <8 x float> [[TMP36]], float [[TMP18]], i32 6
+; AVX2-NEXT:    [[TMP38:%.*]] = insertelement <8 x float> [[TMP37]], float [[TMP22]], i32 7
 ; AVX2-NEXT:    [[TMP39:%.*]] = shufflevector <2 x float> [[TMP25]], <2 x float> [[TMP27]], <8 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; AVX2-NEXT:    [[TMP40:%.*]] = shufflevector <8 x float> [[TMP39]], <8 x float> [[TMP29]], <8 x i32> <i32 0, i32 1, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i64 3
-; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i64 4
-; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i64 5
-; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i64 6
-; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i64 7
+; AVX2-NEXT:    [[TMP41:%.*]] = insertelement <8 x float> [[TMP40]], float [[TMP8]], i32 3
+; AVX2-NEXT:    [[TMP42:%.*]] = insertelement <8 x float> [[TMP41]], float [[TMP12]], i32 4
+; AVX2-NEXT:    [[TMP43:%.*]] = insertelement <8 x float> [[TMP42]], float [[TMP16]], i32 5
+; AVX2-NEXT:    [[TMP44:%.*]] = insertelement <8 x float> [[TMP43]], float [[TMP20]], i32 6
+; AVX2-NEXT:    [[TMP45:%.*]] = insertelement <8 x float> [[TMP44]], float [[TMP24]], i32 7
 ; AVX2-NEXT:    [[TMP46:%.*]] = fdiv <8 x float> [[TMP38]], [[TMP45]]
 ; AVX2-NEXT:    store <8 x float> [[TMP46]], ptr [[TMP0:%.*]], align 4, !tbaa [[TBAA0]]
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512F-LABEL: @gather_load_div(
 ; AVX512F-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512F-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512F-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512F-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512F-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
@@ -643,6 +653,7 @@ define void @gather_load_div(ptr noalias nocapture %0, ptr noalias nocapture rea
 ;
 ; AVX512VL-LABEL: @gather_load_div(
 ; AVX512VL-NEXT:    [[TMP3:%.*]] = call <45 x float> @llvm.masked.load.v45f32.p0(ptr [[TMP1:%.*]], i32 4, <45 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <45 x float> poison), !tbaa [[TBAA0]]
+; AVX512VL-NEXT:    [[TMP4:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <16 x i32> <i32 0, i32 3, i32 4, i32 5, i32 8, i32 10, i32 11, i32 13, i32 14, i32 17, i32 20, i32 23, i32 27, i32 30, i32 33, i32 44>
 ; AVX512VL-NEXT:    [[TMP7:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 0, i32 3, i32 5, i32 8, i32 10, i32 14, i32 17, i32 20>
 ; AVX512VL-NEXT:    [[TMP8:%.*]] = shufflevector <45 x float> [[TMP3]], <45 x float> poison, <8 x i32> <i32 4, i32 11, i32 27, i32 30, i32 13, i32 44, i32 33, i32 23>
 ; AVX512VL-NEXT:    [[TMP9:%.*]] = fdiv <8 x float> [[TMP7]], [[TMP8]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
index 42a50384787c..a4949bc67b0f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47642.ll
@@ -1,15 +1,15 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer,instcombine -S < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S < %s | FileCheck %s
 ; These code should be fully vectorized by D57059 patch
 
 target triple = "x86_64-unknown-linux-gnu"
 
 define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> poison, i32 [[F:%.*]], i64 0
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <4 x i32> undef, i32 [[F:%.*]], i32 0
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[F]], 1
-; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i64 0
+; CHECK-NEXT:    [[VECINIT1:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[ADD]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
@@ -28,7 +28,7 @@ define <4 x i32> @foo(<4 x i32> %x, i32 %f) {
 
 define <4 x i32> @bar(<4 x i32> %x, i32 %f) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F:%.*]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[F:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
index e7239f906c59..bada001ebbc6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr49081.ll
@@ -1,12 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple=x86_64-- -passes=slp-vectorizer,instcombine -S < %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -passes=slp-vectorizer -S < %s | FileCheck %s
 
 ; These conversions should be vectorized by reviews.llvm.org/D57059
 
 define dso_local <4 x float> @foo(<4 x i32> %0) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP0:%.*]] to <4 x float>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i32> [[TMP1:%.*]], <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <4 x i32> [[TMP0]] to <4 x float>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 1, i32 0, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[TMP3]]
 ;
   %2 = extractelement <4 x i32> %0, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
index 09a5ace101e6..3fd9e126f468 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
@@ -15,15 +15,9 @@
 define void @test(ptr noalias %p, ptr noalias %p1) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[I:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr i32, ptr [[P]], i64 32
-; CHECK-NEXT:    [[I2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr i32, ptr [[P]], i64 33
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[ARRAYIDX11]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = call <35 x i32> @llvm.masked.load.v35i32.p0(ptr [[P:%.*]], i32 4, <35 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, <35 x i32> poison)
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <35 x i32> [[TMP0]], <35 x i32> poison, <4 x i32> <i32 0, i32 32, i32 33, i32 34>
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[P]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[I]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[I2]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.vector.insert.v4i32.v2i32(<4 x i32> [[TMP3]], <2 x i32> [[TMP0]], i64 2)
 ; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[TMP4]], [[TMP1]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr [[P1:%.*]], align 4
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
index 54120f5402f5..70ab6d09a923 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reverse_extract_elements.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,instcombine -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 | FileCheck %s
 
 define float @dotf(<4 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @dotf(
diff --git a/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll b/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
index 94c361caf90f..0688a235f80e 100644
--- a/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extractelement-insertpoint.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86_64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu -o - < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu -o - < %s | FileCheck %s %}
 ; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -o - < %s | FileCheck %s %}
 
 define i32 @crash() {
diff --git a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
index b85c78ec8d2d..cec99c694391 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec-shufflevector.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s
-; RUN: opt -passes=slp-vectorizer,instcombine -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s --check-prefix COMBINE
+; RUN: opt -passes=slp-vectorizer -S -slp-revec -slp-max-reg-size=1024 -slp-threshold=-100 %s | FileCheck %s --check-prefix COMBINE
 
 define void @test1(ptr %in, ptr %out) {
 ; CHECK-LABEL: @test1(
@@ -19,8 +19,14 @@ define void @test1(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test1(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 1
-; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP0]] to <8 x i64>
-; COMBINE-NEXT:    store <8 x i64> [[TMP1]], ptr [[OUT:%.*]], align 8
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i64, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP5:%.*]] = zext <8 x i32> [[TMP4]] to <8 x i64>
+; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i64> [[TMP5]], <8 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    store <8 x i64> [[TMP1]], ptr [[OUT]], align 8
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -61,9 +67,14 @@ define void @test2(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test2(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 1
-; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP0]] to <8 x i64>
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i64, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP5:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = zext <8 x i32> [[TMP4]] to <8 x i64>
+; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
 ; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    store <8 x i64> [[TMP2]], ptr [[OUT:%.*]], align 8
+; COMBINE-NEXT:    store <8 x i64> [[TMP2]], ptr [[OUT]], align 8
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -99,8 +110,11 @@ define void @test3(<16 x i32> %0, ptr %out) {
 ;
 ; COMBINE-LABEL: @test3(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0:%.*]], <16 x i32> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
-; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[TMP3:%.*]] = call <64 x i32> @llvm.vector.insert.v64i32.v16i32(<64 x i32> poison, <16 x i32> [[TMP0:%.*]], i64 0)
+; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <64 x i32> [[TMP3]], <64 x i32> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <64 x i32> [[TMP3]], <64 x i32> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -134,8 +148,12 @@ define void @test4(ptr %in, ptr %out) {
 ; COMBINE-LABEL: @test4(
 ; COMBINE-NEXT:  entry:
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, ptr [[IN:%.*]], align 4
-; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[OUT:%.*]] = getelementptr inbounds i32, ptr [[OUT1:%.*]], i64 0
+; COMBINE-NEXT:    [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> [[TMP0]], i64 0)
+; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; COMBINE-NEXT:    store <16 x i32> [[TMP1]], ptr [[OUT]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -165,7 +183,11 @@ define void @test5(ptr %out) {
 ;
 ; COMBINE-LABEL: @test5(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    store <8 x i32> zeroinitializer, ptr [[OUT:%.*]], align 4
+; COMBINE-NEXT:    [[TMP0:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> poison, <8 x i32> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP1:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v8i32(<16 x i32> [[TMP0]], <8 x i32> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[OUT:%.*]], i64 0
+; COMBINE-NEXT:    store <8 x i32> [[TMP2]], ptr [[TMP3]], align 4
 ; COMBINE-NEXT:    ret void
 ;
 entry:
@@ -219,28 +241,37 @@ define void @test6(ptr %in0, ptr %in1, ptr %in2) {
 ;
 ; COMBINE-LABEL: @test6(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    [[GEP1:%.*]] = getelementptr inbounds nuw i8, ptr [[IN0:%.*]], i64 32
+; COMBINE-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[IN0:%.*]], i64 32
 ; COMBINE-NEXT:    [[LOAD2:%.*]] = load <4 x float>, ptr [[GEP1]], align 16
 ; COMBINE-NEXT:    [[TMP0:%.*]] = load <8 x float>, ptr [[IN0]], align 16
 ; COMBINE-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr [[IN1:%.*]], align 1
-; COMBINE-NEXT:    [[TMP2:%.*]] = uitofp <32 x i8> [[TMP1]] to <32 x float>
+; COMBINE-NEXT:    [[TMP10:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP11:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP4:%.*]] = zext <32 x i8> [[TMP11]] to <32 x i16>
+; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <32 x i16> [[TMP4]], <32 x i16> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP19:%.*]] = shufflevector <32 x i16> [[TMP4]], <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP2:%.*]] = uitofp <32 x i16> [[TMP19]] to <32 x float>
 ; COMBINE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[TMP3]], <16 x float> [[TMP4]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; COMBINE-NEXT:    [[TMP5:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP3]], <4 x float> [[LOAD2]], i64 8)
 ; COMBINE-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[TMP5]], <16 x float> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; COMBINE-NEXT:    [[TMP7:%.*]] = fmul <32 x float> [[TMP6]], [[TMP2]]
-; COMBINE-NEXT:    [[GEP10:%.*]] = getelementptr inbounds nuw i8, ptr [[IN1]], i64 32
-; COMBINE-NEXT:    [[GEP11:%.*]] = getelementptr inbounds nuw i8, ptr [[IN2:%.*]], i64 128
+; COMBINE-NEXT:    [[GEP10:%.*]] = getelementptr inbounds i8, ptr [[IN1]], i64 32
+; COMBINE-NEXT:    [[GEP11:%.*]] = getelementptr inbounds i8, ptr [[IN2:%.*]], i64 128
 ; COMBINE-NEXT:    [[TMP8:%.*]] = load <8 x float>, ptr [[IN0]], align 16
 ; COMBINE-NEXT:    store <32 x float> [[TMP7]], ptr [[IN2]], align 16
 ; COMBINE-NEXT:    [[LOAD5:%.*]] = load <16 x i8>, ptr [[GEP10]], align 1
-; COMBINE-NEXT:    [[TMP9:%.*]] = uitofp <16 x i8> [[LOAD5]] to <16 x float>
-; COMBINE-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[LOAD2]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP11:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP10]], <16 x float> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP13:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-; COMBINE-NEXT:    [[TMP14:%.*]] = shufflevector <4 x float> [[TMP13]], <4 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; COMBINE-NEXT:    [[TMP15:%.*]] = shufflevector <16 x float> [[TMP12]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; COMBINE-NEXT:    [[TMP13:%.*]] = call <32 x i8> @llvm.vector.insert.v32i8.v16i8(<32 x i8> poison, <16 x i8> [[LOAD5]], i64 0)
+; COMBINE-NEXT:    [[TMP14:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP24:%.*]] = shufflevector <32 x i8> [[TMP13]], <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP25:%.*]] = zext <16 x i8> [[TMP24]] to <16 x i16>
+; COMBINE-NEXT:    [[TMP26:%.*]] = shufflevector <16 x i16> [[TMP25]], <16 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP18:%.*]] = shufflevector <16 x i16> [[TMP25]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; COMBINE-NEXT:    [[TMP9:%.*]] = uitofp <16 x i16> [[TMP18]] to <16 x float>
+; COMBINE-NEXT:    [[TMP20:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> poison, <4 x float> [[LOAD2]], i64 0)
+; COMBINE-NEXT:    [[TMP21:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> [[TMP8]], i64 0)
+; COMBINE-NEXT:    [[TMP22:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP20]], <4 x float> [[TMP21]], i64 4)
+; COMBINE-NEXT:    [[TMP23:%.*]] = call <4 x float> @llvm.vector.extract.v4f32.v8f32(<8 x float> [[TMP8]], i64 4)
+; COMBINE-NEXT:    [[TMP15:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP22]], <4 x float> [[TMP23]], i64 8)
 ; COMBINE-NEXT:    [[TMP16:%.*]] = shufflevector <16 x float> [[TMP15]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3>
 ; COMBINE-NEXT:    [[TMP17:%.*]] = fmul <16 x float> [[TMP16]], [[TMP9]]
 ; COMBINE-NEXT:    store <16 x float> [[TMP17]], ptr [[GEP11]], align 16
@@ -353,7 +384,21 @@ define i32 @test7() {
 ;
 ; COMBINE-LABEL: @test7(
 ; COMBINE-NEXT:  entry:
-; COMBINE-NEXT:    store <16 x float> poison, ptr null, align 16
+; COMBINE-NEXT:    [[TMP0:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP1:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP0]], <8 x float> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP2:%.*]] = fsub <16 x float> [[TMP1]], [[TMP1]]
+; COMBINE-NEXT:    [[TMP3:%.*]] = fadd <16 x float> [[TMP1]], [[TMP1]]
+; COMBINE-NEXT:    [[TMP4:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP3]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP6:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> poison, <4 x float> zeroinitializer, i64 0)
+; COMBINE-NEXT:    [[TMP7:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP6]], <4 x float> zeroinitializer, i64 4)
+; COMBINE-NEXT:    [[TMP8:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP7]], <4 x float> zeroinitializer, i64 8)
+; COMBINE-NEXT:    [[TMP9:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v4f32(<16 x float> [[TMP8]], <4 x float> zeroinitializer, i64 12)
+; COMBINE-NEXT:    [[TMP10:%.*]] = fadd <16 x float> [[TMP9]], [[TMP5]]
+; COMBINE-NEXT:    [[TMP11:%.*]] = fsub <16 x float> [[TMP9]], [[TMP5]]
+; COMBINE-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP10]], <16 x float> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; COMBINE-NEXT:    [[TMP13:%.*]] = fadd <16 x float> [[TMP9]], [[TMP12]]
+; COMBINE-NEXT:    store <16 x float> [[TMP13]], ptr null, align 16
 ; COMBINE-NEXT:    ret i32 0
 ;
 entry:
diff --git a/llvm/test/Transforms/SROA/select-gep.ll b/llvm/test/Transforms/SROA/select-gep.ll
index 1342a2ca4ea2..b48b0f77aa99 100644
--- a/llvm/test/Transforms/SROA/select-gep.ll
+++ b/llvm/test/Transforms/SROA/select-gep.ll
@@ -201,6 +201,47 @@ define i32 @test_select_idx_mem2reg(i1 %c) {
   ret i32 %res
 }
 
+; Test gep with a select-like zext index unfolding on an alloca that is
+; splittable and promotable.
+define i64 @test_select_like_zext_idx_mem2reg(i1 %c) {
+; CHECK-LABEL: @test_select_like_zext_idx_mem2reg(
+; CHECK-NEXT:    [[IDX:%.*]] = zext i1 [[C:%.*]] to i64
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i64 2, i64 1
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %alloca = alloca [2 x i64], align 8
+  store i64 1, ptr %alloca
+  %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
+  store i64 2, ptr %gep1
+  %idx = zext i1 %c to i64
+  %gep2 = getelementptr inbounds i64, ptr %alloca, i64 %idx
+  %res = load i64, ptr %gep2
+  ret i64 %res
+}
+
+; Test gep with a zext index that is not equivalent to a select. No unfolding
+; or promotion should take place.
+define i64 @test_zext_unlike_select_idx_mem2reg(i8 %c) {
+; CHECK-LABEL: @test_zext_unlike_select_idx_mem2reg(
+; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [2 x i64], align 8
+; CHECK-NEXT:    store i64 1, ptr [[ALLOCA]], align 4
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 1
+; CHECK-NEXT:    store i64 2, ptr [[GEP1]], align 4
+; CHECK-NEXT:    [[IDX:%.*]] = zext i8 [[C:%.*]] to i64
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[ALLOCA]], i64 [[IDX]]
+; CHECK-NEXT:    [[RES:%.*]] = load i64, ptr [[GEP2]], align 4
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %alloca = alloca [2 x i64], align 8
+  store i64 1, ptr %alloca
+  %gep1 = getelementptr inbounds i64, ptr %alloca, i64 1
+  store i64 2, ptr %gep1
+  %idx = zext i8 %c to i64
+  %gep2 = getelementptr inbounds i64, ptr %alloca, i64 %idx
+  %res = load i64, ptr %gep2
+  ret i64 %res
+}
+
 ; Test gep of index select unfolding on an alloca that escaped, and as such
 ; is not splittable or promotable.
 ; FIXME: Ideally, no transform would take place in this case.
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
index 2ac94afd9591..4c2aa1821168 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch-of-powers-of-two.ll
@@ -34,7 +34,8 @@ define i32 @switch_of_powers(i32 %x) {
 ; RV64ZBB-LABEL: @switch_of_powers(
 ; RV64ZBB-NEXT:  entry:
 ; RV64ZBB-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true)
-; RV64ZBB-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers, i32 0, i32 [[TMP0]]
+; RV64ZBB-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; RV64ZBB-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers, i64 0, i64 [[TMP1]]
 ; RV64ZBB-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; RV64ZBB-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
index 6c9450bb8998..11fca66cbf71 100644
--- a/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
+++ b/llvm/test/Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
@@ -27,7 +27,8 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -67,7 +68,8 @@ define i8 @char(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -114,7 +116,8 @@ define void @h(i32 %x) {
 ; CHECK-NEXT:    [[SWITCH_SHIFTAMT:%.*]] = mul nuw nsw i32 [[X]], 8
 ; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 89655594, [[SWITCH_SHIFTAMT]]
 ; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -159,7 +162,8 @@ define ptr @foostring(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
index 98a2b33d87e3..5bc4b38cf4c1 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/disable-lookup-table.ll
@@ -50,7 +50,8 @@ define i32 @bar(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.bar, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.bar, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
index cf244e6b6345..3abde486c4e3 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-covered-bug.ll
@@ -10,8 +10,8 @@ define i64 @test(i3 %arg) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i3 [[ARG:%.*]], -4
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i64], ptr @switch.table.test, i32 0, i4 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i64], ptr @switch.table.test, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[V3:%.*]] = add i64 [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    ret i64 [[V3]]
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
index 529826758c13..d307ca463291 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-of-powers-of-two.ll
@@ -8,7 +8,8 @@ define i32 @switch_of_powers_two(i32 %arg) {
 ; CHECK-SAME: i32 [[ARG:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[ARG]], i1 true)
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers_two, i32 0, i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.switch_of_powers_two, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
index 37001f4fba2a..63c16a2f7b84 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-table-bug.ll
@@ -10,8 +10,8 @@ define i64 @_TFO6reduce1E5toRawfS0_FT_Si(i2) {
 ; CHECK-LABEL: @_TFO6reduce1E5toRawfS0_FT_Si(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TMP0:%.*]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table._TFO6reduce1E5toRawfS0_FT_Si, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table._TFO6reduce1E5toRawfS0_FT_Si, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    ret i64 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
index ec42efb12aad..b5548de8195c 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-bitcast.ll
@@ -9,7 +9,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define { ptr, i64 } @switch_to_lookup_bitcast(i8 %0) unnamed_addr {
 ; CHECK-LABEL: @switch_to_lookup_bitcast(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_bitcast, i32 0, i8 [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_bitcast, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { ptr, i64 } undef, ptr [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { ptr, i64 } [[TMP1]], i64 1, 1
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
index c7d1c2a237ef..4463e8931700 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-gep.ll
@@ -9,7 +9,8 @@ target triple = "x86_64-unknown-linux-gnu"
 define { ptr, i64 } @switch_to_lookup_gep(i8 %0) unnamed_addr {
 ; CHECK-LABEL: @switch_to_lookup_gep(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_gep, i32 0, i8 [[TMP0:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[TMP0:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x ptr], ptr @switch.table.switch_to_lookup_gep, i64 0, i64 [[TMP3]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { ptr, i64 } undef, ptr [[SWITCH_LOAD]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { ptr, i64 } [[TMP1]], i64 1, 1
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
index 7c97827d108d..4ee7a2710afb 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
@@ -13,7 +13,8 @@ define i1 @zot(i32 %arg) {
 ; CHECK-NEXT:    %0 = icmp ult i32 %arg, 3
 ; CHECK-NEXT:    br i1 %0, label %switch.lookup, label %bb6
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    %switch.gep = getelementptr inbounds [3 x ptr], ptr @switch.table.zot, i32 0, i32 %arg
+; CHECK-NEXT:    %1 = zext i32 %arg to i64
+; CHECK-NEXT:    %switch.gep = getelementptr inbounds [3 x ptr], ptr @switch.table.zot, i64 0, i64 %1
 ; CHECK-NEXT:    %switch.load = load ptr, ptr %switch.gep, align 8
 ; CHECK-NEXT:    br label %bb6
 ; CHECK:       bb6:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
index e188cd8057df..169faff7e228 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
@@ -17,7 +17,7 @@ define i8 @switch_to_lookup_i64(i64 %x){
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i8 [ [[SWITCH_LOAD:%.*]], [[SWITCH_LOOKUP]] ], [ 10, [[START:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[COMMON_RET_OP]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i64, i32 0, i64 [[X]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i64, i64 0, i64 [[X]]
 ; CHECK-NEXT:    [[SWITCH_LOAD]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
@@ -61,7 +61,8 @@ define i8 @switch_to_lookup_i128(i128 %x){
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i8 [ [[SWITCH_LOAD:%.*]], [[SWITCH_LOOKUP]] ], [ 10, [[START:%.*]] ]
 ; CHECK-NEXT:    ret i8 [[COMMON_RET_OP]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i128, i32 0, i128 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i8], ptr @switch.table.switch_to_lookup_i128, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index 95bf296af9b0..9124b356a46b 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -50,7 +50,8 @@ define i32 @f(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.f, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -90,7 +91,8 @@ define i8 @char(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i8], ptr @switch.table.char, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i8, ptr [[SWITCH_GEP]], align 1
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -137,7 +139,8 @@ define void @h(i32 %x) {
 ; CHECK-NEXT:    [[SWITCH_SHIFTAMT:%.*]] = mul nuw nsw i32 [[X]], 8
 ; CHECK-NEXT:    [[SWITCH_DOWNSHIFT:%.*]] = lshr i32 89655594, [[SWITCH_SHIFTAMT]]
 ; CHECK-NEXT:    [[SWITCH_MASKED:%.*]] = trunc i32 [[SWITCH_DOWNSHIFT]] to i8
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x float], ptr @switch.table.h, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load float, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -182,7 +185,8 @@ define ptr @foostring(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x ptr], ptr @switch.table.foostring, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load ptr, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -221,9 +225,11 @@ define i32 @earlyreturncrash(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[SW_EPILOG:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
-; CHECK-NEXT:    [[SWITCH_GEP1:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash.1, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP1:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.earlyreturncrash.1, i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[SWITCH_LOAD2:%.*]] = load i32, ptr [[SWITCH_GEP1]], align 4
 ; CHECK-NEXT:    br label [[SW_EPILOG]]
 ; CHECK:       sw.epilog:
@@ -404,7 +410,8 @@ define i32 @large(i32 %x) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 199
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [199 x i32], ptr @switch.table.large, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [199 x i32], ptr @switch.table.large, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -835,7 +842,8 @@ define i32 @cprop(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 7
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.cprop, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [7 x i32], ptr @switch.table.cprop, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -885,7 +893,8 @@ define i32 @unreachable_case(i32 %x)  {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 9
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i32], ptr @switch.table.unreachable_case, i32 0, i32 [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [9 x i32], ptr @switch.table.unreachable_case, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -920,7 +929,8 @@ return:
 define i32 @unreachable_default(i32 %x)  {
 ; CHECK-LABEL: @unreachable_default(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.unreachable_default, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.unreachable_default, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1000,7 +1010,8 @@ define i32 @nodefaultnoholes(i32 %c) {
 ; CHECK-NEXT:    call void @exit(i32 1)
 ; CHECK-NEXT:    unreachable
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.nodefaultnoholes, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.nodefaultnoholes, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1037,7 +1048,8 @@ define i32 @nodefaultwithholes(i32 %c) {
 ; CHECK-NEXT:    call void @exit(i32 1)
 ; CHECK-NEXT:    unreachable
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.nodefaultwithholes, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.nodefaultwithholes, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1102,7 +1114,8 @@ define i32 @threecases(i32 %c) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[C:%.*]], 3
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[RETURN:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.threecases, i32 0, i32 [[C]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[C]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.threecases, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return:
@@ -1561,8 +1574,8 @@ define i32 @covered_switch_with_bit_tests(i3) {
 ; CHECK-NEXT:    [[SWITCH_LOBIT:%.*]] = trunc i8 [[SWITCH_SHIFTED]] to i1
 ; CHECK-NEXT:    br i1 [[SWITCH_LOBIT]], label [[SWITCH_LOOKUP:%.*]], label [[L6:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.covered_switch_with_bit_tests, i32 0, i4 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i3 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.covered_switch_with_bit_tests, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[L6]]
 ; CHECK:       l6:
@@ -1714,8 +1727,8 @@ define i32 @signed_overflow1(i8 %n) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TRUNC]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow1, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow1, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -1750,8 +1763,8 @@ define i32 @signed_overflow2(i8 %n) {
 ; CHECK-NEXT:  start:
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[N:%.*]] to i2
 ; CHECK-NEXT:    [[SWITCH_TABLEIDX:%.*]] = sub i2 [[TRUNC]], -2
-; CHECK-NEXT:    [[SWITCH_TABLEIDX_ZEXT:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i3
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow2, i32 0, i3 [[SWITCH_TABLEIDX_ZEXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i2 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.signed_overflow2, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2215,7 +2228,8 @@ return:
 define i32 @constant_hole_unreachable_default_firstundef(i32 %x) {
 ; CHECK-LABEL: @constant_hole_unreachable_default_firstundef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_firstundef, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_firstundef, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2241,7 +2255,8 @@ return:
 define i32 @constant_hole_unreachable_default_lastundef(i32 %x) {
 ; CHECK-LABEL: @constant_hole_unreachable_default_lastundef(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_lastundef, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.constant_hole_unreachable_default_lastundef, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
@@ -2359,7 +2374,8 @@ return:
 define i32 @linearmap_hole_unreachable_default(i32 %x) {
 ; CHECK-LABEL: @linearmap_hole_unreachable_default(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.linearmap_hole_unreachable_default, i32 0, i32 [[X:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.linearmap_hole_unreachable_default, i64 0, i64 [[TMP0]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    ret i32 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
index fb607c72a0e3..0f78e236b424 100644
--- a/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
+++ b/llvm/test/Transforms/SimplifyCFG/preserve-branchweights.ll
@@ -38,45 +38,12 @@ Z:
   ret void
 }
 
-; Make sure the metadata name string is "branch_weights" before propagating it.
-
-define void @fake_weights(i1 %a, i1 %b) {
-; CHECK-LABEL: @fake_weights(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A_NOT:%.*]] = xor i1 [[A:%.*]], true
-; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
-; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A_NOT]], i1 [[C]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       common.ret:
-; CHECK-NEXT:    ret void
-; CHECK:       Y:
-; CHECK-NEXT:    call void @helper(i32 0)
-; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
-; CHECK:       Z:
-; CHECK-NEXT:    call void @helper(i32 1)
-; CHECK-NEXT:    br label [[COMMON_RET]]
-;
-entry:
-  br i1 %a, label %Y, label %X, !prof !12
-X:
-  %c = or i1 %b, false
-  br i1 %c, label %Z, label %Y, !prof !1
-
-Y:
-  call void @helper(i32 0)
-  ret void
-
-Z:
-  call void @helper(i32 1)
-  ret void
-}
-
 define void @test2(i1 %a, i1 %b) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       Y:
@@ -107,7 +74,7 @@ define void @test3(i1 %a, i1 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF2:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       Y:
@@ -138,7 +105,7 @@ define void @test4(i1 %a, i1 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = or i1 [[B:%.*]], false
 ; CHECK-NEXT:    [[OR_COND:%.*]] = select i1 [[A:%.*]], i1 [[C]], i1 false
-; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[Z:%.*]], label [[Y:%.*]], !prof [[PROF2]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       Y:
@@ -1120,7 +1087,6 @@ exit:
 !9 = !{!"branch_weights", i32 7, i32 6}
 !10 = !{!"branch_weights", i32 672646, i32 21604207}
 !11 = !{!"branch_weights", i32 6960, i32 21597248}
-!12 = !{!"these_are_not_the_branch_weights_you_are_looking_for", i32 3, i32 5}
 !13 = !{!"branch_weights", i32 2, i32 3}
 !14 = !{!"branch_weights", i32 4, i32 7}
 !15 = !{!"branch_weights", i32 99, i32 1}
@@ -1136,8 +1102,8 @@ exit:
 ; CHECK: attributes #[[ATTR2:[0-9]+]] = { noredzone nounwind ssp memory(none) }
 ;.
 ; CHECK: [[PROF0]] = !{!"branch_weights", i32 5, i32 11}
-; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 3}
-; CHECK: [[PROF2]] = !{!"branch_weights", i32 1, i32 5}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 5}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 1, i32 3}
 ; CHECK: [[PROF3]] = !{!"branch_weights", i32 7, i32 1, i32 2}
 ; CHECK: [[PROF4]] = !{!"branch_weights", i32 49, i32 12, i32 24, i32 35}
 ; CHECK: [[PROF5]] = !{!"branch_weights", i32 11, i32 5}
diff --git a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
index 467ede9b75c3..c2afba965702 100644
--- a/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
+++ b/llvm/test/Transforms/SimplifyCFG/rangereduce.ll
@@ -11,7 +11,8 @@ define i32 @test1(i32 %a) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test1, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test1, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -80,7 +81,8 @@ define i32 @test3(i32 %a) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[SWITCH_TABLEIDX]], 3
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.test3, i32 0, i32 [[SWITCH_TABLEIDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[SWITCH_TABLEIDX]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [3 x i32], ptr @switch.table.test3, i64 0, i64 [[TMP2]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -185,7 +187,8 @@ define i32 @test6(i32 %a) optsize {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 4
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test6, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i32], ptr @switch.table.test6, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -248,7 +251,8 @@ define i32 @test8(i32 %a) optsize {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 5
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.test8, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [5 x i32], ptr @switch.table.test8, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
@@ -280,7 +284,8 @@ define i32 @test9(i32 %a) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 8
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[SWITCH_LOOKUP:%.*]], label [[COMMON_RET:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.test9, i32 0, i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [8 x i32], ptr @switch.table.test9, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ; CHECK:       common.ret:
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll b/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
index bead0dc4c567..72eea9a064eb 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-dead-default-lookup-table.ll
@@ -8,7 +8,7 @@ define i64 @test_1(i64 %0) {
 ; CHECK-SAME: i64 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  switch.lookup:
 ; CHECK-NEXT:    [[TMP1:%.*]] = urem i64 [[TMP0]], 4
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table.test_1, i32 0, i64 [[TMP1]]
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [4 x i64], ptr @switch.table.test_1, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]], align 8
 ; CHECK-NEXT:    ret i64 [[SWITCH_LOAD]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
index 50998e447b71..fe2e897125eb 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll
@@ -309,3 +309,444 @@ end:
   %t0 = phi i8 [ 42, %case1 ], [ 42, %case2 ], [ 44, %case3 ], [ 44, %case4 ]
   ret i8 %t0
 }
+
+define i1 @no_range(i8 %f) {
+; CHECK-LABEL: @no_range(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 60
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 60
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 60, label %bb2
+  i8 61, label %bb2
+  i8 62, label %bb2
+  i8 63, label %bb2
+  i8 124, label %bb2
+  i8 188, label %bb2
+  i8 252, label %bb2
+  i8 189, label %bb2
+  i8 190, label %bb2
+  i8 191, label %bb2
+  i8 125, label %bb2
+  i8 126, label %bb2
+  i8 127, label %bb2
+  i8 253, label %bb2
+  i8 254, label %bb2
+  i8 255, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_no_range(i8 %f) {
+; CHECK-LABEL: @negative_no_range(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 52, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 61, label [[BB2]]
+; CHECK-NEXT:      i8 62, label [[BB2]]
+; CHECK-NEXT:      i8 63, label [[BB2]]
+; CHECK-NEXT:      i8 124, label [[BB2]]
+; CHECK-NEXT:      i8 -68, label [[BB2]]
+; CHECK-NEXT:      i8 -4, label [[BB2]]
+; CHECK-NEXT:      i8 -67, label [[BB2]]
+; CHECK-NEXT:      i8 -66, label [[BB2]]
+; CHECK-NEXT:      i8 -65, label [[BB2]]
+; CHECK-NEXT:      i8 125, label [[BB2]]
+; CHECK-NEXT:      i8 126, label [[BB2]]
+; CHECK-NEXT:      i8 127, label [[BB2]]
+; CHECK-NEXT:      i8 -3, label [[BB2]]
+; CHECK-NEXT:      i8 -2, label [[BB2]]
+; CHECK-NEXT:      i8 -1, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 52, label %bb2
+  i8 61, label %bb2
+  i8 62, label %bb2
+  i8 63, label %bb2
+  i8 124, label %bb2
+  i8 188, label %bb2
+  i8 252, label %bb2
+  i8 189, label %bb2
+  i8 190, label %bb2
+  i8 191, label %bb2
+  i8 125, label %bb2
+  i8 126, label %bb2
+  i8 127, label %bb2
+  i8 253, label %bb2
+  i8 254, label %bb2
+  i8 255, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+; Using ranges.
+
+define i1 @range0to4odd(i8 range(i8 0, 4) %f) {
+; CHECK-LABEL: @range0to4odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range1to4odd(i8 range(i8 1, 4) %f) {
+; CHECK-LABEL: @range1to4odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to8odd(i8 range(i8 0, 8) %f) {
+; CHECK-LABEL: @range0to8odd(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 1, label %bb2
+  i8 3, label %bb2
+  i8 5, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to8most_significant_bit(i8 range(i8 0, 8) %f) {
+; CHECK-LABEL: @range0to8most_significant_bit(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 4, label %bb2
+  i8 5, label %bb2
+  i8 6, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_middle_two_bits(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_middle_two_bits(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 7, label [[BB2]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_pow_2(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_pow_2(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -2
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to5even(i8 range(i8 0, 5) %f) {
+; CHECK-LABEL: @negative_range0to5even(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i8 [[F:%.*]], 2
+; CHECK-NEXT:    [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -3
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  switch i8 %f, label %bb1 [
+  i8 2, label %bb2
+  i8 4, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_corner_case(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_corner_case(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[F:%.*]], 15
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[COND]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[DOT]]
+;
+  switch i8 %f, label %bb1 [
+  i8 15, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_corner_case(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_corner_case(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[SWITCH_SELECTCMP_CASE1:%.*]] = icmp eq i8 [[F:%.*]], 15
+; CHECK-NEXT:    [[SWITCH_SELECTCMP_CASE2:%.*]] = icmp eq i8 [[F]], 8
+; CHECK-NEXT:    [[SWITCH_SELECTCMP:%.*]] = or i1 [[SWITCH_SELECTCMP_CASE1]], [[SWITCH_SELECTCMP_CASE2]]
+; CHECK-NEXT:    [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 15, label %bb2
+  i8 8,  label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+; Out of range scenarios. Check if the cases, that have a value out of range
+; are eliminated and the optimization is performed.
+
+define i1 @range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_out_of_range_non_prime(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 22, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @range0to15_out_of_range_non_prime_more(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @range0to15_out_of_range_non_prime_more(
+; CHECK-NEXT:  bb3:
+; CHECK-NEXT:    [[TMP0:%.*]] = and i8 [[F:%.*]], 6
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 22, label %bb2
+  i8 23, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_out_of_range_non_prime(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:      i8 15, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 14, label %bb2
+  i8 15, label %bb2
+  i8 23, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_out_of_range(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_out_of_range(
+; CHECK-NEXT:    switch i8 [[F:%.*]], label [[BB3:%.*]] [
+; CHECK-NEXT:      i8 6, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 7, label [[BB2]]
+; CHECK-NEXT:      i8 14, label [[BB2]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret i1 [[_0_SROA_0_0]]
+;
+  switch i8 %f, label %bb1 [
+  i8 6, label %bb2
+  i8 7, label %bb2
+  i8 14, label %bb2
+  i8 150, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
+
+define i1 @negative_range0to15_all_out_of_range(i8 range(i8 0, 16) %f) {
+; CHECK-LABEL: @negative_range0to15_all_out_of_range(
+; CHECK-NEXT:  bb1:
+; CHECK-NEXT:    ret i1 false
+;
+  switch i8 %f, label %bb1 [
+  i8 22, label %bb2
+  i8 23, label %bb2
+  i8 30, label %bb2
+  i8 31, label %bb2
+  ]
+bb1:
+  br label %bb3
+bb2:
+  br label %bb3
+bb3:
+  %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ]
+  ret i1 %phi
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
index 51c15774d476..bbd1505b2597 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
@@ -186,7 +186,8 @@ define void @switch_lookup_with_nonconst_range(i32 %x, i1 %cond) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[ADD]], 6
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[SWITCH_LOOKUP:%.*]], label [[LOR_END:%.*]]
 ; CHECK:       switch.lookup:
-; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.switch_lookup_with_nonconst_range, i32 0, i32 [[ADD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[ADD]] to i64
+; CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds [6 x i32], ptr @switch.table.switch_lookup_with_nonconst_range, i64 0, i64 [[TMP1]]
 ; CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i32, ptr [[SWITCH_GEP]], align 4
 ; CHECK-NEXT:    br label [[LOR_END]]
 ; CHECK:       lor.end:
diff --git a/llvm/test/Transforms/Util/Debugify/loc-only.ll b/llvm/test/Transforms/Util/Debugify/loc-only.ll
index 1a1c531622f1..d5331be3457f 100644
--- a/llvm/test/Transforms/Util/Debugify/loc-only.ll
+++ b/llvm/test/Transforms/Util/Debugify/loc-only.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -passes=debugify -S < %s | FileCheck --check-prefixes=ALL,VALUE %s
-; RUN: opt -passes=debugify -debugify-level=locations -S < %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg.value %s
+; RUN: opt -passes=debugify -debugify-level=locations -S < %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg_value %s
 
 
 ; ALL-LABEL: @test
diff --git a/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll b/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
index b438b5e24b3a..38752ac4ff57 100644
--- a/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
+++ b/llvm/test/Transforms/Util/trivial-auto-var-init-crash-20210521.ll
@@ -23,7 +23,7 @@ bb:
 declare void @llvm.dbg.value(metadata, metadata, metadata) #2
 
 attributes #0 = { argmemonly nofree nosync nounwind willreturn }
-attributes #1 = { "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" }
+attributes #1 = { "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2" }
 attributes #2 = { nofree nosync nounwind readnone speculatable willreturn }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll b/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
index 09c03991ad7c..60700412686e 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/ext-extract.ll
@@ -9,15 +9,23 @@ define void @zext_v4i8_all_lanes_used(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_all_lanes_used(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 255
+; CHECK-NEXT:    [[TMP9:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP9]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -68,13 +76,20 @@ define void @zext_v4i8_3_lanes_used_1(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_1(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP6]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -93,13 +108,19 @@ define void @zext_v4i8_3_lanes_used_2(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_3_lanes_used_2(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 24
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP7]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -118,11 +139,17 @@ define void @zext_v4i8_2_lanes_used_1(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_1(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP3]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -139,11 +166,16 @@ define void @zext_v4i8_2_lanes_used_2(<4 x i8> %src) {
 ; CHECK-LABEL: define void @zext_v4i8_2_lanes_used_2(
 ; CHECK-SAME: <4 x i8> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i8> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i8> [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 255
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP1]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP5]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP3]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -160,15 +192,22 @@ define void @zext_v4i8_all_lanes_used_noundef(<4 x i8> noundef %src) {
 ; CHECK-LABEL: define void @zext_v4i8_all_lanes_used_noundef(
 ; CHECK-SAME: <4 x i8> noundef [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <4 x i8> [[SRC]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 24
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr i32 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP4:%.*]] = and i32 [[TMP3]], 255
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr i32 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], 255
+; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP0]], 255
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i8> [[SRC]] to <4 x i32>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i32> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i32> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i32> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i32> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP8]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP6]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP4]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP1]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -221,15 +260,23 @@ define void @zext_v4i16_all_lanes_used(<4 x i16> %src) {
 ; CHECK-LABEL: define void @zext_v4i16_all_lanes_used(
 ; CHECK-SAME: <4 x i16> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <4 x i16> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i16> [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 48
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], 65535
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i64 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[TMP6]], 65535
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP1]], 65535
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <4 x i16> [[SRC]] to <4 x i64>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <4 x i64> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <4 x i64> [[EXT9]], i64 1
 ; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <4 x i64> [[EXT9]], i64 2
 ; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <4 x i64> [[EXT9]], i64 3
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_2]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_3]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP9]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP7]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP5]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -250,11 +297,15 @@ define void @zext_v2i32_all_lanes_used(<2 x i32> %src) {
 ; CHECK-LABEL: define void @zext_v2i32_all_lanes_used(
 ; CHECK-SAME: <2 x i32> [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze <2 x i32> [[SRC]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP1]], 4294967295
 ; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <2 x i32> [[SRC]] to <2 x i64>
 ; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <2 x i64> [[EXT9]], i64 0
 ; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <2 x i64> [[EXT9]], i64 1
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_0]])
-; CHECK-NEXT:    call void @use.i64(i64 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP5]])
+; CHECK-NEXT:    call void @use.i64(i64 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -266,3 +317,32 @@ entry:
   call void @use.i64(i64 %ext.1)
   ret void
 }
+
+define void @zext_nxv4i8_all_lanes_used(<vscale x 4 x i8> %src) {
+; CHECK-LABEL: define void @zext_nxv4i8_all_lanes_used(
+; CHECK-SAME: <vscale x 4 x i8> [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[EXT9:%.*]] = zext nneg <vscale x 4 x i8> [[SRC]] to <vscale x 4 x i32>
+; CHECK-NEXT:    [[EXT_0:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 0
+; CHECK-NEXT:    [[EXT_1:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = extractelement <vscale x 4 x i32> [[EXT9]], i64 3
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_0]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_2]])
+; CHECK-NEXT:    call void @use.i32(i32 [[EXT_3]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ext9 = zext nneg <vscale x 4 x i8> %src to <vscale x 4 x i32>
+  %ext.0 = extractelement <vscale x 4 x i32> %ext9, i64 0
+  %ext.1 = extractelement <vscale x 4 x i32> %ext9, i64 1
+  %ext.2 = extractelement <vscale x 4 x i32> %ext9, i64 2
+  %ext.3 = extractelement <vscale x 4 x i32> %ext9, i64 3
+
+  call void @use.i32(i32 %ext.0)
+  call void @use.i32(i32 %ext.1)
+  call void @use.i32(i32 %ext.2)
+  call void @use.i32(i32 %ext.3)
+  ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
index 605c0772f6e3..391215b3e415 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extract-insert-store-scalarization.ll
@@ -6,13 +6,12 @@ target triple = "arm64-apple-darwin"
 define void @load_extract_insert_store_const_idx(ptr %A) {
 ; CHECK-LABEL: @load_extract_insert_store_const_idx(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <225 x double>, ptr [[A:%.*]], i32 0, i64 0
-; CHECK-NEXT:    [[EXT_0:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT:    [[EXT_0:%.*]] = load double, ptr [[TMP0:%.*]], align 8
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul double 2.000000e+01, [[EXT_0]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i32 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <225 x double>, ptr [[TMP0]], i32 0, i64 1
 ; CHECK-NEXT:    [[EXT_1:%.*]] = load double, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[EXT_1]], [[MUL]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <225 x double>, ptr [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <225 x double>, ptr [[TMP0]], i64 0, i64 1
 ; CHECK-NEXT:    store double [[SUB]], ptr [[TMP2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
index 551d6d1cabd4..e6e5f5196d3d 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll
@@ -15,8 +15,7 @@ define i32 @load_extract_idx_0(ptr %x) {
 
 define i32 @vscale_load_extract_idx_0(ptr %x) {
 ; CHECK-LABEL: @vscale_load_extract_idx_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1]], align 16
+; CHECK-NEXT:    [[R:%.*]] = load i32, ptr [[TMP1:%.*]], align 16
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %lv = load <vscale x 4 x i32>, ptr %x
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll b/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
index 3c672efbb5a0..761ad80d560e 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shrink-types.ll
@@ -7,9 +7,8 @@ define i32 @test_and(<16 x i32> %a, ptr %b) {
 ; CHECK-LABEL: @test_and(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[B:%.*]], align 1
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc <16 x i32> [[A:%.*]] to <16 x i8>
-; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[WIDE_LOAD]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[TMP1]] to <16 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <16 x i32> [[TMP0]], [[A:%.*]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -26,9 +25,8 @@ define i32 @test_mask_or(<16 x i32> %a, ptr %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[B:%.*]], align 1
 ; CHECK-NEXT:    [[A_MASKED:%.*]] = and <16 x i32> [[A:%.*]], splat (i32 16)
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc <16 x i32> [[A_MASKED]] to <16 x i8>
-; CHECK-NEXT:    [[TMP1:%.*]] = or <16 x i8> [[WIDE_LOAD]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[TMP1]] to <16 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i32> [[TMP0]], [[A_MASKED]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
@@ -47,15 +45,13 @@ define i32 @multiuse(<16 x i32> %u, <16 x i32> %v, ptr %b) {
 ; CHECK-NEXT:    [[U_MASKED:%.*]] = and <16 x i32> [[U:%.*]], splat (i32 255)
 ; CHECK-NEXT:    [[V_MASKED:%.*]] = and <16 x i32> [[V:%.*]], splat (i32 255)
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[B:%.*]], align 1
-; CHECK-NEXT:    [[TMP0:%.*]] = lshr <16 x i8> [[WIDE_LOAD]], splat (i8 4)
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <16 x i32> [[V_MASKED]] to <16 x i8>
-; CHECK-NEXT:    [[TMP2:%.*]] = or <16 x i8> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = and <16 x i8> [[WIDE_LOAD]], splat (i8 15)
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc <16 x i32> [[U_MASKED]] to <16 x i8>
-; CHECK-NEXT:    [[TMP6:%.*]] = or <16 x i8> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr <16 x i8> [[WIDE_LOAD]], splat (i8 4)
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext <16 x i8> [[TMP6]] to <16 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <16 x i32> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP7]], [[V_MASKED]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and <16 x i32> [[TMP0]], splat (i32 15)
+; CHECK-NEXT:    [[TMP5:%.*]] = or <16 x i32> [[TMP4]], [[U_MASKED]]
+; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <16 x i32> [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP8]])
 ; CHECK-NEXT:    ret i32 [[TMP9]]
 ;
@@ -81,9 +77,8 @@ define i32 @phi_bug(<16 x i32> %a, ptr %b) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[A_PHI:%.*]] = phi <16 x i32> [ [[A:%.*]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[WIDE_LOAD_PHI:%.*]] = phi <16 x i8> [ [[WIDE_LOAD]], [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc <16 x i32> [[A_PHI]] to <16 x i8>
-; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i8> [[WIDE_LOAD_PHI]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <16 x i8> [[TMP1]] to <16 x i32>
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <16 x i8> [[WIDE_LOAD_PHI]] to <16 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <16 x i32> [[TMP0]], [[A_PHI]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[TMP2]])
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index 1c128c8f56a0..a40d514a520c 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -997,10 +997,8 @@ define <4 x i64> @bitcast_smax_v8i32_v4i32(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-NEXT:    [[A_BC0:%.*]] = bitcast <4 x i64> [[A:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[B_BC0:%.*]] = bitcast <4 x i64> [[B:%.*]] to <8 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <8 x i32> [[A_BC0]], [[B_BC0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[B]], <4 x i64> [[B]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[TMP2]] to <8 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[A]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[B_BC1:%.*]] = bitcast <4 x i64> [[B]] to <8 x i32>
+; CHECK-NEXT:    [[A_BC1:%.*]] = bitcast <4 x i64> [[A]] to <8 x i32>
 ; CHECK-NEXT:    [[CONCAT:%.*]] = select <8 x i1> [[CMP]], <8 x i32> [[B_BC1]], <8 x i32> [[A_BC1]]
 ; CHECK-NEXT:    [[RES:%.*]] = bitcast <8 x i32> [[CONCAT]] to <4 x i64>
 ; CHECK-NEXT:    ret <4 x i64> [[RES]]
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
index 46a622148c87..59ece31631d8 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/vecreduce-shuffle.ll
@@ -16,8 +16,7 @@ define i32 @reducebase_v4i32(<4 x i32> %a, <4 x i32> %b) {
 
 define i32 @reduceshuffle_onein_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v4i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %x = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -27,8 +26,7 @@ define i32 @reduceshuffle_onein_v4i32(<4 x i32> %a) {
 
 define i32 @reduceshuffle_onein_const_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_const_v4i32(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[X:%.*]] = xor <4 x i32> [[S]], splat (i32 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <4 x i32> [[S:%.*]], splat (i32 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -222,8 +220,7 @@ define i32 @reducebase_v16i32(<16 x i32> %a, <16 x i32> %b) {
 
 define i32 @reduceshuffle_onein_v16i32(<16 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v16i32(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <16 x i32> [[A:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %x = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -233,8 +230,7 @@ define i32 @reduceshuffle_onein_v16i32(<16 x i32> %a) {
 
 define i32 @reduceshuffle_onein_ext_v16i32(<16 x i32> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_ext_v16i32(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <16 x i32> [[A:%.*]], <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[X:%.*]] = xor <16 x i32> [[S]], splat (i32 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <16 x i32> [[S:%.*]], splat (i32 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[X]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -353,8 +349,7 @@ define i16 @reducebase_v16i16(<16 x i16> %a, <16 x i16> %b) {
 
 define i16 @reduceshuffle_onein_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_v16i16(
-; CHECK-NEXT:    [[X:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X:%.*]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %x = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
@@ -364,8 +359,7 @@ define i16 @reduceshuffle_onein_v16i16(<16 x i16> %a) {
 
 define i16 @reduceshuffle_onein_ext_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: @reduceshuffle_onein_ext_v16i16(
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT:    [[X:%.*]] = xor <16 x i16> [[S]], splat (i16 -1)
+; CHECK-NEXT:    [[X:%.*]] = xor <16 x i16> [[S:%.*]], splat (i16 -1)
 ; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> [[X]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
index 3ce2c8ecebaf..7f941a364929 100644
--- a/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/RISCV/vpintrin-scalarization.ll
@@ -486,10 +486,7 @@ define <vscale x 1 x i64> @urem_nxv1i64_unspeculatable(i64 %x, i64 %y, i32 zeroe
 
 define <vscale x 1 x i64> @sdiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = sdiv i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> splat (i64 1)
 ;
 ; NO-VEC-COMBINE-LABEL: @sdiv_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -530,10 +527,7 @@ define <vscale x 1 x i64> @sdiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @udiv_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = udiv i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> splat (i64 1)
 ;
 ; NO-VEC-COMBINE-LABEL: @udiv_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -574,10 +568,7 @@ define <vscale x 1 x i64> @udiv_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @srem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = srem i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> zeroinitializer
 ;
 ; NO-VEC-COMBINE-LABEL: @srem_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -618,10 +609,7 @@ define <vscale x 1 x i64> @srem_nxv1i64_anymask_knownvl(i64 %x, i64 %y, <vscale
 
 define <vscale x 1 x i64> @urem_nxv1i64_allonesmask_knownvl(i64 %x, i64 %y) {
 ; VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
-; VEC-COMBINE-NEXT:    [[TMP1:%.*]] = urem i64 [[X:%.*]], [[X]]
-; VEC-COMBINE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-NEXT:    [[RES:%.*]] = shufflevector <vscale x 1 x i64> [[DOTSPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
-; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> [[RES]]
+; VEC-COMBINE-NEXT:    ret <vscale x 1 x i64> zeroinitializer
 ;
 ; NO-VEC-COMBINE-LABEL: @urem_nxv1i64_allonesmask_knownvl(
 ; NO-VEC-COMBINE-NEXT:    [[MASK_HEAD:%.*]] = insertelement <vscale x 1 x i1> poison, i1 true, i32 0
@@ -1581,8 +1569,7 @@ define <1 x i64> @add_v1i64_allonesmask(<1 x i64> %x, i64 %y, i32 zeroext %evl)
 ; VEC-COMBINE-64-NEXT:    [[MASK:%.*]] = shufflevector <1 x i1> [[SPLAT]], <1 x i1> poison, <1 x i32> zeroinitializer
 ; VEC-COMBINE-64-NEXT:    [[TMP1:%.*]] = add i64 [[Y:%.*]], 42
 ; VEC-COMBINE-64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <1 x i64> poison, i64 [[TMP1]], i64 0
-; VEC-COMBINE-64-NEXT:    [[TMP2:%.*]] = shufflevector <1 x i64> [[DOTSPLATINSERT]], <1 x i64> poison, <1 x i32> zeroinitializer
-; VEC-COMBINE-64-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.vp.mul.v1i64(<1 x i64> [[X:%.*]], <1 x i64> [[TMP2]], <1 x i1> [[MASK]], i32 [[EVL:%.*]])
+; VEC-COMBINE-64-NEXT:    [[TMP3:%.*]] = call <1 x i64> @llvm.vp.mul.v1i64(<1 x i64> [[X:%.*]], <1 x i64> [[DOTSPLATINSERT]], <1 x i1> [[MASK]], i32 [[EVL:%.*]])
 ; VEC-COMBINE-64-NEXT:    ret <1 x i64> [[TMP3]]
 ;
 ; NO-VEC-COMBINE-LABEL: @add_v1i64_allonesmask(
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index 7cbe1c6cec90..d11fb1426c94 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -567,3 +567,22 @@ define float @constant_fold_crash_commute(<4 x float> %x) {
   %c = fadd float %b, %a
   ret float %c
 }
+
+; Test case for https://github.com/llvm/llvm-project/issues/147218.
+define i64 @instsimplify_folder_crash(<4 x i64> %in) {
+; CHECK-LABEL: @instsimplify_folder_crash(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[SHUFFLE_1:%.*]] = shufflevector <4 x i64> [[IN:%.*]], <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    [[E_0:%.*]] = extractelement <4 x i64> zeroinitializer, i64 0
+; CHECK-NEXT:    [[E_1:%.*]] = extractelement <4 x i64> [[SHUFFLE_1]], i64 1
+; CHECK-NEXT:    [[OR:%.*]] = or i64 [[E_1]], [[E_0]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+entry:
+  %shuffle.1 = shufflevector <4 x i64> %in, <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+  %e.0 = extractelement <4 x i64> zeroinitializer, i64 0
+  %e.1 = extractelement <4 x i64> %shuffle.1, i64 1
+  %shift = shufflevector <4 x i64> %shuffle.1, <4 x i64> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+  %or = or i64 %e.1, %e.0
+  ret i64 %or
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll b/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
index cd2bc757eb9d..5358e0419e7a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll
@@ -48,13 +48,13 @@ define <4 x float> @ext2_v4f32(<4 x float> %x, <4 x float> %y) {
 define <4 x float> @ext2_v2f32v4f32(<2 x float> %x, <4 x float> %y) {
 ; CHECK-LABEL: @ext2_v2f32v4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 poison>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
-  %e = extractelement <2 x float> %x, i32 2
+  %e = extractelement <2 x float> %x, i32 1
   %n = fneg float %e
-  %r = insertelement <4 x float> %y, float %n, i32 2
+  %r = insertelement <4 x float> %y, float %n, i32 1
   ret <4 x float> %r
 }
 
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
index d46c8c0de403..b26e5ec2698a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
@@ -6,8 +6,7 @@
 define void @multiple_extract(ptr %p) {
 ; CHECK-LABEL: @multiple_extract(
 ; CHECK-NEXT:    [[VP:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i32>, ptr [[VP]], i32 0, i64 0
-; CHECK-NEXT:    [[E0:%.*]] = load i32, ptr [[TMP1]], align 16
+; CHECK-NEXT:    [[E0:%.*]] = load i32, ptr [[VP]], align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i32>, ptr [[VP]], i32 0, i64 1
 ; CHECK-NEXT:    [[E1:%.*]] = load i32, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    store i32 [[E0]], ptr [[P]], align 4
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
index 40437ca34522..977da754ec5a 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -657,11 +657,10 @@ define <2 x float> @load_f32_insert_v2f32_msan(ptr align 16 dereferenceable(16)
 ; PR30986 - split vector loads for scalarized operations
 define <2 x i64> @PR30986(ptr %0) {
 ; CHECK-LABEL: @PR30986(
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2:%.*]], align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP2]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[TMP8]], i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index 84c223be8862..388b655641b7 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -553,8 +553,7 @@ define <4 x float> @load_v2f32_extract_insert_v4f32(ptr align 16 dereferenceable
 
 define <4 x float> @load_v8f32_extract_insert_v4f32(ptr align 16 dereferenceable(16) %p) nofree nosync {
 ; SSE2-LABEL: @load_v8f32_extract_insert_v4f32(
-; SSE2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <8 x float>, ptr [[P:%.*]], i32 0, i32 0
-; SSE2-NEXT:    [[S:%.*]] = load float, ptr [[TMP1]], align 4
+; SSE2-NEXT:    [[S:%.*]] = load float, ptr [[TMP1:%.*]], align 4
 ; SSE2-NEXT:    [[R:%.*]] = insertelement <4 x float> undef, float [[S]], i32 0
 ; SSE2-NEXT:    ret <4 x float> [[R]]
 ;
@@ -590,8 +589,7 @@ define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(ptr align 16 derefer
 define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceable(16) %p) nofree nosync {
 ; CHECK-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <2 x i16>, ptr [[P:%.*]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <2 x i16>, ptr [[GEP]], i32 0, i32 0
-; CHECK-NEXT:    [[S:%.*]] = load i16, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[S:%.*]] = load i16, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = insertelement <8 x i16> undef, i16 [[S]], i64 0
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
@@ -605,11 +603,10 @@ define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(ptr align 1 dereferenceab
 ; PR30986 - split vector loads for scalarized operations
 define <2 x i64> @PR30986(ptr %0) {
 ; CHECK-LABEL: @PR30986(
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0:%.*]], i32 0, i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2:%.*]], align 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP3]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> undef, i64 [[TMP4]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP0]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds <2 x i64>, ptr [[TMP2]], i32 0, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = tail call i64 @llvm.ctpop.i64(i64 [[TMP7]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[TMP8]], i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/pr114901.ll b/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
index d6917e1007cf..982944b8b7bd 100644
--- a/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/pr114901.ll
@@ -17,8 +17,7 @@ define i1 @PR114901(<4 x i32> %a) {
 ; AVX-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; AVX-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A]], <i32 poison, i32 -8, i32 poison, i32 42>
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP2:%.*]] = ashr <4 x i1> [[SHIFT]], [[TMP1]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[SHIFT]], i64 1
 ; AVX-NEXT:    ret i1 [[R]]
 ;
   %e1 = extractelement <4 x i32> %a, i32 1
@@ -43,8 +42,7 @@ define i1 @PR114901_flip(<4 x i32> %a) {
 ; AVX-SAME: <4 x i32> [[A:%.*]]) #[[ATTR0]] {
 ; AVX-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[A]], <i32 poison, i32 -8, i32 poison, i32 42>
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> poison, <4 x i32> <i32 poison, i32 3, i32 poison, i32 poison>
-; AVX-NEXT:    [[TMP2:%.*]] = ashr <4 x i1> [[TMP1]], [[SHIFT]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1
+; AVX-NEXT:    [[R:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
 ; AVX-NEXT:    ret i1 [[R]]
 ;
   %e1 = extractelement <4 x i32> %a, i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
index 2588f9116f32..0c995bde9f25 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-selects.ll
@@ -34,18 +34,12 @@ define <8 x i16> @src_v4tov8_i16(<4 x i1> %a, <4 x i1> %b, <4 x i16> %x, <4 x i1
 define <8 x i16> @src_v8tov8_i16(<8 x i1> %a, <8 x i1> %b, <8 x i16> %x, <8 x i16> %y, <8 x i16> %z) {
 ; SSE-LABEL: define <8 x i16> @src_v8tov8_i16(
 ; SSE-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i1> [[A]], <8 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X]], <8 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[Z]], <8 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
+; SSE-NEXT:    [[RES:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
 ; SSE-NEXT:    ret <8 x i16> [[RES]]
 ;
 ; AVX2-LABEL: define <8 x i16> @src_v8tov8_i16(
 ; AVX2-SAME: <8 x i1> [[A:%.*]], <8 x i1> [[B:%.*]], <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]], <8 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i1> [[A]], <8 x i1> [[B]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i16> [[X]], <8 x i16> [[Y]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[Z]], <8 x i16> [[X]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[TMP2]], <8 x i16> [[TMP3]]
+; AVX2-NEXT:    [[RES:%.*]] = select <8 x i1> [[A]], <8 x i16> [[X]], <8 x i16> [[Z]]
 ; AVX2-NEXT:    ret <8 x i16> [[RES]]
 ;
 ; AVX512-LABEL: define <8 x i16> @src_v8tov8_i16(
diff --git a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
index 58b7f8de004d..9e43a28bf1e5 100644
--- a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
+++ b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
@@ -152,12 +152,12 @@ define <vscale x 4 x float> @fma_scalable(float %x, float %y, float %z) {
   ret <vscale x 4 x float> %v
 }
 
-; TODO: We should be able to scalarize this if we preserve the scalar argument.
 define <4 x float> @scalar_argument(float %x) {
 ; CHECK-LABEL: define <4 x float> @scalar_argument(
 ; CHECK-SAME: float [[X:%.*]]) {
-; CHECK-NEXT:    [[X_INSERT:%.*]] = insertelement <4 x float> poison, float [[X]], i32 0
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> [[X_INSERT]], i32 42)
+; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.powi.f32.i32(float [[X]], i32 42)
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> poison, i32 42)
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[V]]
 ;
   %x.insert = insertelement <4 x float> poison, float %x, i32 0
diff --git a/llvm/test/Transforms/lower-builtin-allow-check-pipeline.ll b/llvm/test/Transforms/lower-builtin-allow-check-pipeline.ll
new file mode 100644
index 000000000000..7b00be97e050
--- /dev/null
+++ b/llvm/test/Transforms/lower-builtin-allow-check-pipeline.ll
@@ -0,0 +1,3 @@
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[7]=0;cutoffs[8]=1>)' -S -o - -print-pipeline-passes | FileCheck %s
+
+; CHECK: lower-allow-check<cutoffs[8]=1>
diff --git a/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll b/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
index 3422ab1b56d3..3030cc02f868 100644
--- a/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
+++ b/llvm/test/Transforms/lower-builtin-allow-check-remarks.ll
@@ -1,12 +1,24 @@
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-random-rate=1 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-random-rate=0 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=REMOVE
 
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=0 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check)' -lower-allow-check-percentile-cutoff-hot=1000000 -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=REMOVE
+
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[7]=0;runtime_check=1000000>)' -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=MIXED
+; RUN: opt < %s -passes='require<profile-summary>,function(lower-allow-check<cutoffs[7]=1000000;runtime_check=0>)' -pass-remarks=lower-allow-check -pass-remarks-missed=lower-allow-check -S 2>&1 | FileCheck %s --check-prefixes=MIXED2
+
 ; CHECK: remark: <unknown>:0:0: Allowed check: Kind=test_check F=test_runtime BB=entry1
 ; CHECK: remark: <unknown>:0:0: Allowed check: Kind=7 F=test_ubsan BB=entry2
 
 ; REMOVE: remark: <unknown>:0:0: Removed check: Kind=test_check F=test_runtime BB=entry1
 ; REMOVE: remark: <unknown>:0:0: Removed check: Kind=7 F=test_ubsan BB=entry2
 
+; MIXED: remark: <unknown>:0:0: Removed check: Kind=test_check F=test_runtime BB=entry1
+; MIXED: remark: <unknown>:0:0: Allowed check: Kind=7 F=test_ubsan BB=entry2
+
+; MIXED2: remark: <unknown>:0:0: Allowed check: Kind=test_check F=test_runtime BB=entry1
+; MIXED2: remark: <unknown>:0:0: Removed check: Kind=7 F=test_ubsan BB=entry2
+
 target triple = "x86_64-pc-linux-gnu"
 
 define i1 @test_runtime() local_unnamed_addr {
diff --git a/llvm/test/Verifier/assume-bundles.ll b/llvm/test/Verifier/assume-bundles.ll
index afe6cc0ab4c6..047e8d9bba3e 100644
--- a/llvm/test/Verifier/assume-bundles.ll
+++ b/llvm/test/Verifier/assume-bundles.ll
@@ -9,6 +9,8 @@ define void @func(ptr %P, i32 %P1, ptr %P2, ptr %P3) {
   call void @llvm.assume(i1 true) ["adazdazd"()]
 ; CHECK: the second argument should be a constant integral value
   call void @llvm.assume(i1 true) ["dereferenceable"(ptr %P, i32 %P1)]
+; CHECK: the second argument should be a constant integral value
+  call void @llvm.assume(i1 true) ["dereferenceable"(ptr %P, float 1.5)]
 ; CHECK: too many arguments
   call void @llvm.assume(i1 true) ["dereferenceable"(ptr %P, i32 8, i32 8)]
 ; CHECK: this attribute should have 2 arguments
@@ -28,5 +30,7 @@ define void @func(ptr %P, i32 %P1, ptr %P2, ptr %P3) {
   call void @llvm.assume(i1 true) ["separate_storage"(ptr %P)]
 ; CHECK: arguments to separate_storage assumptions should be pointers
   call void @llvm.assume(i1 true) ["separate_storage"(ptr %P, i32 123)]
+; CHECK: this attribute should have 2 arguments
+  call void @llvm.assume(i1 true) ["align"(ptr %P, i32 4), "dereferenceable"(ptr %P)]
   ret void
 }
diff --git a/llvm/test/Verifier/branch-weight.ll b/llvm/test/Verifier/branch-weight.ll
new file mode 100644
index 000000000000..4c87a9835901
--- /dev/null
+++ b/llvm/test/Verifier/branch-weight.ll
@@ -0,0 +1,151 @@
+; Test MD_prof validation
+
+; RUN: split-file %s %t
+
+; RUN: opt -passes=verify %t/valid.ll --disable-output
+
+; RUN: not opt -passes=verify %t/wrong-count.ll --disable-output 2>&1 | FileCheck %s --check-prefix=WRONG-COUNT
+; RUN: not opt -passes=verify %t/invalid-name1.ll --disable-output 2>&1 | FileCheck %s
+; RUN: not opt -passes=verify %t/invalid-name2.ll --disable-output 2>&1 | FileCheck %s
+
+; RUN: opt -passes=verify %t/unknown-correct.ll --disable-output
+
+; RUN: not opt -passes=verify %t/unknown-invalid.ll --disable-output 2>&1 | FileCheck %s --check-prefix=EXTRA-ARGS
+; RUN: not opt -passes=verify %t/unknown-on-function1.ll --disable-output 2>&1 | FileCheck %s --check-prefix=ON-FUNCTION1
+; RUN: not opt -passes=verify %t/unknown-on-function2.ll --disable-output 2>&1 | FileCheck %s --check-prefix=ON-FUNCTION2
+; RUN: not opt -passes=verify %t/invalid-unknown-placement.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-UNKNOWN-PLACEMENT
+
+;--- valid.ll
+declare void @to_invoke()
+declare i32 @__gxx_personality_v0(...)
+
+define void @invoker() personality ptr @__gxx_personality_v0 {
+  invoke void @to_invoke() to label %exit unwind label %lpad, !prof !0
+lpad:
+  %ll = landingpad {ptr, i32}
+  cleanup
+  ret void
+exit:
+  ret void
+}
+
+define i32 @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %exit, !prof !0
+yes:
+  switch i32 %a, label %exit [ i32 1, label %case_b
+                               i32 2, label %case_c], !prof !1
+case_b:
+  br label %exit
+case_c:
+  br label %exit
+exit:
+  %r = select i1 %c, i32 1, i32 2, !prof !0
+  ret i32 %r
+}
+!0 = !{!"branch_weights", i32 1, i32 2}
+!1 = !{!"branch_weights", i32 1, i32 2, i32 3}
+
+;--- wrong-count.ll
+define void @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %no, !prof !0
+yes:
+  ret void
+no:
+  ret void
+}
+!0 = !{!"branch_weights", i32 1, i32 2, i32 3}
+
+; WRONG-COUNT: Wrong number of operands
+
+;--- invalid-name1.ll
+define void @test(i1 %0) {
+  br i1 %0, label %2, label %3, !prof !0
+2:
+  ret void
+3:
+  ret void
+}
+!0 = !{!"invalid", i32 1, i32 2}
+
+;--- invalid-name2.ll
+define void @test(i1 %0) {
+  br i1 %0, label %2, label %3, !prof !0
+2:
+  ret void
+3:
+  ret void
+}
+
+!0 = !{!"function_entry_count", i32 1}
+
+; CHECK: expected either branch_weights or VP profile name
+
+;--- unknown-correct.ll
+declare void @to_invoke()
+declare i32 @__gxx_personality_v0(...)
+
+define void @invoker() personality ptr @__gxx_personality_v0 {
+  invoke void @to_invoke() to label %exit unwind label %lpad, !prof !0
+lpad:
+  %ll = landingpad {ptr, i32}
+  cleanup
+  ret void
+exit:
+  ret void
+}
+
+define i32 @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %exit, !prof !0
+yes:
+  switch i32 %a, label %exit [ i32 1, label %case_b
+                               i32 2, label %case_c], !prof !0
+case_b:
+  br label %exit
+case_c:
+  br label %exit
+exit:
+  %r = select i1 %c, i32 1, i32 2, !prof !0
+  ret i32 %r
+}
+
+!0 = !{!"unknown"}
+
+;--- unknown-invalid.ll
+define void @test(i32 %a) {
+  %c = icmp eq i32 %a, 0
+  br i1 %c, label %yes, label %no, !prof !0
+yes:
+  ret void
+no:
+  ret void
+}
+
+!0 = !{!"unknown", i32 12, i32 67}
+; EXTRA-ARGS: 'unknown' !prof should have no additional operands
+
+;--- unknown-on-function1.ll
+define void @test() !prof !0 {
+  ret void
+}
+
+!0 = !{!"unknown"}
+; ON-FUNCTION1: 'unknown' !prof metadata should appear only on instructions supporting the 'branch_weights' metadata
+
+;--- unknown-on-function2.ll
+define void @test() !prof !0 {
+  ret void
+}
+
+!0 = !{!"unknown", i64 123}
+; ON-FUNCTION2: first operand should be 'function_entry_count' or 'synthetic_function_entry_count'
+
+;--- invalid-unknown-placement.ll
+define i32 @test() {
+  %r = add i32 1, 2, !prof !0
+  ret i32 %r
+}
+!0 = !{!"unknown"}
+; INVALID-UNKNOWN-PLACEMENT: 'unknown' !prof should only appear on instructions on which 'branch_weights' would
diff --git a/llvm/test/Verifier/dead-on-return.ll b/llvm/test/Verifier/dead-on-return.ll
new file mode 100644
index 000000000000..eb1d67fa319c
--- /dev/null
+++ b/llvm/test/Verifier/dead-on-return.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: Attribute 'dead_on_return' applied to incompatible type!
+; CHECK-NEXT: ptr @arg_not_pointer
+define void @arg_not_pointer(i32 dead_on_return %arg) {
+  ret void
+}
diff --git a/llvm/test/Verifier/value-profile.ll b/llvm/test/Verifier/value-profile.ll
new file mode 100644
index 000000000000..f38bc7728a70
--- /dev/null
+++ b/llvm/test/Verifier/value-profile.ll
@@ -0,0 +1,38 @@
+; Test MD_prof "VP" validation
+
+; RUN: split-file %s %t
+; RUN: opt -passes=verify %t/valid.ll --disable-output
+; RUN: not opt -passes=verify %t/invalid-kind.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-KIND
+; RUN: not opt -passes=verify %t/invalid-count.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-COUNT
+; RUN: not opt -passes=verify %t/invalid-place.ll --disable-output 2>&1 | FileCheck %s --check-prefix=INVALID-PLACE
+
+;--- valid.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 0, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+
+;--- invalid-kind.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 3, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-KIND: Invalid VP !prof kind
+
+;--- invalid-count.ll
+define void @test(ptr %0) {
+  call void %0(), !prof !0
+  ret void
+}
+!0 = !{!"VP", i32 1, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-COUNT: VP !prof should have an even number of arguments after 'VP'
+
+;--- invalid-place.ll
+define i32 @test(i32 %0) {
+  %r = add i32 %0, 1, !prof !0
+  ret i32 %r
+}
+!0 = !{!"VP", i32 1, i32 20, i64 1234, i64 10, i64 5678, i64 5}
+; INVALID-PLACE: VP !prof indirect call or memop size expected to be applied to CallBase instructions only
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
index b8779b9d54ea..51cafacd57ad 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
@@ -19,11 +19,7 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
 ; CHECK-NEXT:    st.param.v2.b64 [param0], {%rd2, %rd1};
 ; CHECK-NEXT:    st.param.v2.b64 [param0+16], {%rd4, %rd3};
 ; CHECK-NEXT:    .param .align 16 .b8 retval0[32];
-; CHECK-NEXT:    call.uni (retval0),
-; CHECK-NEXT:    callee_St8x4,
-; CHECK-NEXT:    (
-; CHECK-NEXT:    param0
-; CHECK-NEXT:    );
+; CHECK-NEXT:    call.uni (retval0), callee_St8x4, (param0);
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [retval0];
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [retval0+16];
 ; CHECK-NEXT:    } // callseq 0
@@ -58,8 +54,8 @@ define internal fastcc [4 x i64] @callee_St8x4(ptr nocapture noundef readonly by
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [callee_St8x4_param_0];
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [callee_St8x4_param_0+16];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
 ; CHECK-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd3, %rd4};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
 ; CHECK-NEXT:    ret;
   %1 = load i64, ptr %in, align 8
   %arrayidx.1 = getelementptr inbounds [4 x i64], ptr %in, i64 0, i64 1
diff --git a/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c b/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
index d1ec7e70d443..c66f760bf4ca 100644
--- a/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
+++ b/llvm/test/tools/dsymutil/X86/global_downgraded_to_static.c
@@ -1,4 +1,4 @@
-// REQUIRES : system-darwin
+// REQUIRES: system-darwin
 // RUN: dsymutil -oso-prepend-path %p/.. -dump-debug-map \
 // RUN: %p/../Inputs/global_downgraded_to_static.x86_64 2>&1 | FileCheck %s
 //
diff --git a/llvm/test/tools/dxil-dis/shuffle.ll b/llvm/test/tools/dxil-dis/shuffle.ll
index 0739afc87ff2..5f7ecba45edd 100644
--- a/llvm/test/tools/dxil-dis/shuffle.ll
+++ b/llvm/test/tools/dxil-dis/shuffle.ll
@@ -1,4 +1,4 @@
-; RUN: llc --filetype=obj %s -o - 2>&1 | dxil-dis -o - | FileCheck %s
+; RUN: llc -start-after=dxil-flatten-arrays --filetype=obj %s -o - 2>&1 | dxil-dis -o - | FileCheck %s
 target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
 target triple = "dxil-unknown-shadermodel6.7-library"
 
diff --git a/llvm/test/tools/llc/new-pm/pipeline.mir b/llvm/test/tools/llc/new-pm/pipeline.mir
index 761a3a424ee6..1113f2a26c44 100644
--- a/llvm/test/tools/llc/new-pm/pipeline.mir
+++ b/llvm/test/tools/llc/new-pm/pipeline.mir
@@ -1,10 +1,18 @@
 # RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir -passes=no-op-machine-function --print-pipeline-passes -filetype=null < %s | FileCheck %s --match-full-lines
 # RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir -passes='require<machine-dom-tree>,print<machine-dom-tree>' -print-pipeline-passes < %s | FileCheck --check-prefix=ANALYSIS %s
 
+# Check same nested pass managers
+# RUN: llc -mtriple=x86_64-pc-linux-gnu -x mir \
+# RUN:     -passes='function(machine-function(machine-function(machine-cp)))' \
+# RUN:     -print-pipeline-passes < %s | FileCheck %s --check-prefix=NESTED
+
+# RUN: not llc -mtriple=x86_64-pc-linux-gnu -x mir -passes='machine-function(whoops(verify))' -print-pipeline-passes < %s 2>&1
+
 # CHECK: function(machine-function(no-op-machine-function)),PrintMIRPreparePass,function(machine-function(verify,print))
 
 # ANALYSIS: require<machine-dom-tree>,print<machine-dom-tree>
 
+# NESTED: function(machine-function(machine-cp))
 ---
 name: f
 body: |
diff --git a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
index f1a30bc7964c..7a50bcaf212c 100644
--- a/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
+++ b/llvm/test/tools/llc/new-pm/x86_64-regalloc-pipeline.mir
@@ -1,4 +1,4 @@
-# REQUIRES x86_64-registered-target
+# REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=fast -print-pipeline-passes %s -o - 2>&1 | FileCheck %s
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -enable-new-pm -O3 -regalloc-npm=greedy -print-pipeline-passes %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-GREEDY
 
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
index 33c517da91b5..3887b43c6eda 100644
--- a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
@@ -33,4 +33,4 @@ CHECK-SAME:     "mcdc":{"count":0,"covered":0,"notcovered":0,"percent":0},
 CHECK-SAME:     "regions":{"count":1,"covered":1,"notcovered":0,"percent":100}}}
 CHECK-SAME: ],
 CHECK-SAME: "type":"llvm.coverage.json.export"
-CHECK-SAME: "version":"2.0.1"
+CHECK-SAME: "version":"3.0.0"
diff --git a/llvm/test/tools/llvm-cov/mcdc-export-json.test b/llvm/test/tools/llvm-cov/mcdc-export-json.test
index c2cebf52db8c..3f9201d338cf 100644
--- a/llvm/test/tools/llvm-cov/mcdc-export-json.test
+++ b/llvm/test/tools/llvm-cov/mcdc-export-json.test
@@ -1,10 +1,10 @@
 // RUN: llvm-profdata merge %S/Inputs/mcdc-general.proftext -o %t.profdata
 // RUN: llvm-cov export --format=text %S/Inputs/mcdc-general.o -instr-profile %t.profdata | FileCheck %s
 
-// CHECK: 12,7,12,27,0,5,[true,true,true,true]
-// CHECK: 15,7,15,13,0,5,[true,true]
-// CHECK: 15,19,15,25,0,5,[true,false]
-// CHECK: 18,7,19,15,0,5,[true,true,false,true]
+// CHECK: 12,7,12,27,2,4,0,5,[true,true,true,true]
+// CHECK: 15,7,15,13,1,2,0,5,[true,true]
+// CHECK: 15,19,15,25,1,1,0,5,[true,false]
+// CHECK: 18,7,19,15,1,3,0,5,[true,true,false,true]
 // CHECK: "mcdc":{"count":12,"covered":10,"notcovered":2,"percent":83.333333333333343}
 
 Instructions for regenerating the test:
diff --git a/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/wasm-32bit-tombstone.s b/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/wasm-32bit-tombstone.s
new file mode 100644
index 000000000000..3609b3c8ecc3
--- /dev/null
+++ b/llvm/test/tools/llvm-debuginfo-analyzer/WebAssembly/wasm-32bit-tombstone.s
@@ -0,0 +1,107 @@
+# REQUIRES: webassembly-registered-target
+
+# Test that DWARF tombstones are correctly detected/respected in wasm
+# 32 bit object files.
+
+# The test case was produced by the following steps:
+#
+# // test-clang.cpp
+# void foo() {
+# }
+#
+# 1) clang --target=wasm32 -S -g test-clang.cpp
+#                             -o Inputs/wasm-32bit-tombstone.s
+#
+# 2) Creating a single function, tombstoning it in the assembly, by
+#    manually changing the DW_AT_low_pc for the DW_TAG_subprogram:
+#    .Lfunc_begin0 to 0xffffffff to mark the function as dead code:
+#
+#	   .int8	2                          # Abbrev [2] 0x26:0x1b DW_TAG_subprogram
+#	   .int32	.Lfunc_begin0              # DW_AT_low_pc  <---------
+#	   .int32	.Lfunc_end0-.Lfunc_begin0  # DW_AT_high_pc
+
+#	   .int8	2                          # Abbrev [2] 0x26:0x1b DW_TAG_subprogram
+#	   .int32	0xffffffff                 # DW_AT_low_pc  <---------
+#	   .int32	.Lfunc_end0-.Lfunc_begin0  # DW_AT_high_pc
+
+# RUN: llvm-mc -arch=wasm32 -filetype=obj       \
+# RUN:         %p/wasm-32bit-tombstone.s        \
+# RUN:         -o %t.wasm-32bit-tombstone.wasm
+
+# RUN: llvm-debuginfo-analyzer --select-elements=Discarded         \
+# RUN:                         --print=elements                    \
+# RUN:                         %t.wasm-32bit-tombstone.wasm 2>&1 | \
+# RUN: FileCheck --strict-whitespace -check-prefix=ONE %s
+
+# ONE: Logical View:
+# ONE-NEXT:           {File} '{{.*}}wasm-32bit-tombstone.wasm'
+# ONE-EMPTY:
+# ONE-NEXT:           {CompileUnit} 'test-clang.cpp'
+# ONE-NEXT:           {Function} not_inlined 'foo' -> 'void'
+
+# RUN: llvm-dwarfdump --debug-info %t.wasm-32bit-tombstone.wasm | \
+# RUN: FileCheck %s --check-prefix=TWO
+
+# TWO:      DW_TAG_subprogram
+# TWO-NEXT:   DW_AT_low_pc	(dead code)
+# TWO-NEXT:   DW_AT_high_pc
+# TWO-NEXT:   DW_AT_name	("foo")
+
+	.text
+	.file	"test-clang.cpp"
+	.functype	_Z3foov () -> ()
+	.section	.text._Z3foov,"",@
+_Z3foov:                                # @_Z3foov
+.Lfunc_begin0:
+	.functype	_Z3foov () -> ()
+	return
+	end_function
+.Lfunc_end0:
+                                        # -- End function
+	.section	.debug_abbrev,"",@
+	.int8	1                               # Abbreviation Code
+	.int8	17                              # DW_TAG_compile_unit
+	.int8	1                               # DW_CHILDREN_yes
+	.int8	3                               # DW_AT_name
+	.int8	14                              # DW_FORM_strp
+	.int8	17                              # DW_AT_low_pc
+	.int8	1                               # DW_FORM_addr
+	.int8	18                              # DW_AT_high_pc
+	.int8	6                               # DW_FORM_data4
+	.int8	0                               # EOM(1)
+	.int8	0                               # EOM(2)
+	.int8	2                               # Abbreviation Code
+	.int8	46                              # DW_TAG_subprogram
+	.int8	0                               # DW_CHILDREN_no
+	.int8	17                              # DW_AT_low_pc
+	.int8	1                               # DW_FORM_addr
+	.int8	18                              # DW_AT_high_pc
+	.int8	6                               # DW_FORM_data4
+	.int8	3                               # DW_AT_name
+	.int8	14                              # DW_FORM_strp
+	.int8	0                               # EOM(1)
+	.int8	0                               # EOM(2)
+	.int8	0                               # EOM(3)
+	.section	.debug_info,"",@
+.Lcu_begin0:
+	.int32	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.int16	4                               # DWARF version number
+	.int32	.debug_abbrev0                  # Offset Into Abbrev. Section
+	.int8	4                               # Address Size (in bytes)
+	.int8	1                               # Abbrev [1] 0xb:0x37 DW_TAG_compile_unit
+	.int32	.Linfo_string1                  # DW_AT_name
+	.int32	.Lfunc_begin0                   # DW_AT_low_pc
+	.int32	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.int8	2                               # Abbrev [2] 0x26:0x1b DW_TAG_subprogram
+	.int32	0xffffffff                   # DW_AT_low_pc
+	.int32	.Lfunc_end0-.Lfunc_begin0       # DW_AT_high_pc
+	.int32	.Linfo_string4                  # DW_AT_name
+	.int8	0                               # End Of Children Mark
+.Ldebug_info_end0:
+	.section	.debug_str,"S",@
+.Linfo_string1:
+	.asciz	"test-clang.cpp"                # string offset=176
+.Linfo_string4:
+	.asciz	"foo"                           # string offset=241
+	.ident	"clang version 19.0.0"
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll b/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
index ec16137cee1d..9a2d0a52d975 100644
--- a/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/template_ref_ptr.ll
@@ -31,7 +31,7 @@ entry:
   ret i32 0, !dbg !31
 }
 
-attributes #0 = { mustprogress noinline norecurse nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { mustprogress noinline norecurse nounwind optnone ssp uwtable(sync) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 
 !llvm.module.flags = !{!18, !19, !20, !21, !22, !23, !24}
 !llvm.dbg.cu = !{!2}
diff --git a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
index 64c18669264e..6023b99702e2 100644
--- a/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
+++ b/llvm/test/tools/llvm-gsymutil/ARM_AArch64/macho-gsym-callsite-info-obj.test
@@ -167,9 +167,9 @@ entry:
   ret i32 0, !dbg !60
 }
 
-attributes #0 = { minsize mustprogress nofree noinline norecurse nounwind optsize ssp memory(readwrite, argmem: none) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #1 = { minsize mustprogress noinline nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
-attributes #2 = { minsize mustprogress norecurse nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,+zcm,+zcz" }
+attributes #0 = { minsize mustprogress nofree noinline norecurse nounwind optsize ssp memory(readwrite, argmem: none) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #1 = { minsize mustprogress noinline nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+attributes #2 = { minsize mustprogress norecurse nounwind optsize ssp "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="apple-m1" "target-features"="+aes,+altnzcv,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fptoint,+fullfp16,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+sha3,+specrestrict,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
 attributes #3 = { minsize optsize }
 attributes #4 = { minsize nounwind optsize }
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
index 99e39567b1ad..ef9d4463ebe5 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-sve-instructions.s
@@ -5066,19 +5066,19 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  2      2     1.00                        movs	p0.b, p0/z, p0.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15.b
 # CHECK-NEXT:  2      2     1.00                        movs	p15.b, p15/z, p15.b
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ID_AA64ZFR0_EL1
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL1
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL12
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL2
-# CHECK-NEXT:  1      1     0.10                  U     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  1      1     0.20                  U     mrs	x3, ZCR_EL3
 # CHECK-NEXT:  1      4     1.00                        msb	z0.b, p7/m, z1.b, z31.b
 # CHECK-NEXT:  2      5     2.00                        msb	z0.d, p7/m, z1.d, z31.d
 # CHECK-NEXT:  1      4     1.00                        msb	z0.h, p7/m, z1.h, z31.h
 # CHECK-NEXT:  1      4     1.00                        msb	z0.s, p7/m, z1.s, z31.s
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL1, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL12, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL2, x3
-# CHECK-NEXT:  1      1     0.10                  U     msr	ZCR_EL3, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL1, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL12, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL2, x3
+# CHECK-NEXT:  1      1     0.20                  U     msr	ZCR_EL3, x3
 # CHECK-NEXT:  1      4     1.00                        mul	z0.b, p7/m, z0.b, z31.b
 # CHECK-NEXT:  1      4     1.00                        mul	z0.b, z1.b, z2.b
 # CHECK-NEXT:  2      5     2.00                        mul	z0.d, p7/m, z0.d, z31.d
diff --git a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
index 5ffaf9138d48..dee46a304582 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Neoverse/N2-writeback.s
@@ -1185,10 +1185,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1197,13 +1197,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1215,14 +1215,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s }, [x27], #8
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [1] Code Region - G02
 
@@ -1231,10 +1231,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1243,13 +1243,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1261,14 +1261,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.16b }, [x27], #16
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [2] Code Region - G03
 
@@ -1277,10 +1277,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.95
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1289,13 +1289,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     D===eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeeeER.   ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     . D==eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .  D==eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1307,14 +1307,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    2.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    2.0       <total>
 
 # CHECK:      [3] Code Region - G04
 
@@ -1323,10 +1323,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        1900
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.74
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 3.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1334,14 +1334,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1352,15 +1352,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.0    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [4] Code Region - G05
 
@@ -1369,10 +1369,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1380,14 +1380,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1398,15 +1398,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], #32
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [5] Code Region - G06
 
@@ -1415,10 +1415,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1426,14 +1426,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1444,15 +1444,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [6] Code Region - G07
 
@@ -1461,10 +1461,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.53
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 4.3
+# CHECK-NEXT: Block RThroughput: 4.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
@@ -1472,14 +1472,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1490,15 +1490,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [7] Code Region - G08
 
@@ -1507,7 +1507,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.92
 # CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -1518,14 +1518,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1536,15 +1536,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [8] Code Region - G09
 
@@ -1553,7 +1553,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.92
 # CHECK-NEXT: IPC:               1.97
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -1564,14 +1564,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER . .   ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1582,26 +1582,26 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [9] Code Region - G10
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      608
+# CHECK-NEXT: Total Cycles:      708
 # CHECK-NEXT: Total uOps:        2700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.64
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.81
+# CHECK-NEXT: IPC:               1.41
 # CHECK-NEXT: Block RThroughput: 5.7
 
 # CHECK:      Timeline view:
@@ -1610,14 +1610,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeER .  .   ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE----R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE----R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER  .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeER.   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,7]     . D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,9]     . D===eE-----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER   .   ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE----R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeER .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,7]     .   DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeeeeER   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1628,42 +1628,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 9.     1     4.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    2.2       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.2    2.2       <total>
 
 # CHECK:      [10] Code Region - G11
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      675
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.48
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    . .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER . .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE-----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeER   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1673,43 +1673,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.5       <total>
 
 # CHECK:      [11] Code Region - G12
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      675
+# CHECK-NEXT: Total Cycles:      1008
 # CHECK-NEXT: Total uOps:        3000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.44
-# CHECK-NEXT: IPC:               1.48
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    2.98
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 6.7
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeER.   .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eE-----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER   .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-----R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER  .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE-----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER.   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eE-----R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    . .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER . .   ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeER   ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1719,25 +1719,25 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    2.5       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.5       <total>
 
 # CHECK:      [12] Code Region - G13
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1210
+# CHECK-NEXT: Total Cycles:      1212
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.31
 # CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 5.7
@@ -1746,16 +1746,16 @@ add x0, x27, 1
 # CHECK-NEXT:                     0123456789
 # CHECK-NEXT: Index     0123456789          01
 
-# CHECK:      [0,0]     DeeeeeeeER.    .    ..   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE-----R.    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER    .    ..   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-----R    .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeeER   .    ..   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE-----R   .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeER .    ..   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE-----R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D========eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,9]     .  D=========eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeER.    .    .  .   ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE-----R.    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .    .  .   ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-----R   .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeER .    .  .   ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE-----R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER    .  .   ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-----R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . D======eeeeeeeeER   ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,9]     .    . D=======eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1765,16 +1765,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 9.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.3    0.2    2.6       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     7.0    0.0    0.0       ld1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 9.     1     8.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.4    2.6       <total>
 
 # CHECK:      [13] Code Region - G14
 
@@ -1783,10 +1783,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.50
 # CHECK-NEXT: IPC:               0.25
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1794,14 +1794,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1812,15 +1812,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.b }[8], [x27], #1
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld1	{ v1.b }[8], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [14] Code Region - G15
 
@@ -1829,10 +1829,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.50
 # CHECK-NEXT: IPC:               0.25
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
@@ -1840,14 +1840,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1858,15 +1858,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.h }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld1	{ v1.s }[0], [x27], #4
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [15] Code Region - G16
 
@@ -1875,10 +1875,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      1203
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.66
 # CHECK-NEXT: IPC:               0.83
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1886,14 +1886,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1904,15 +1904,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1	{ v1.d }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.2d }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], #2
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [16] Code Region - G17
 
@@ -1921,10 +1921,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1932,14 +1932,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1950,15 +1950,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.4s }, [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], #1
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.8h }, [x27], #2
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], #1
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [17] Code Region - G18
 
@@ -1967,10 +1967,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.92
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -1978,14 +1978,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -1996,15 +1996,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.2d }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld1r	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld1r	{ v1.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld1r	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [18] Code Region - G19
 
@@ -2013,10 +2013,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.71
 # CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -2024,14 +2024,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,5]     .  DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,7]     .   DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,9]     .    DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2042,42 +2042,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld1r	{ v1.8h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld1r	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.1    3.0       <total>
 
 # CHECK:      [19] Code Region - G20
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      909
 # CHECK-NEXT: Total uOps:        2900
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.69
-# CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.19
+# CHECK-NEXT: IPC:               1.10
+# CHECK-NEXT: Block RThroughput: 5.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     01234567
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeeeeER.   ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D=eeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .  D==eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    . .   ld2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE------R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   . .   ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,3]     . DeE------R   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeeeER . .   ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,5]     .   DeE------R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeeeeeER .   ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,7]     .    .DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeeeER   ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .  DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2087,43 +2087,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.2    0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.4    3.0       <total>
 
 # CHECK:      [20] Code Region - G21
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total Cycles:      709
 # CHECK-NEXT: Total uOps:        2700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.29
-# CHECK-NEXT: IPC:               1.96
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.81
+# CHECK-NEXT: IPC:               1.41
+# CHECK-NEXT: Block RThroughput: 5.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
+# CHECK-NEXT:                     012345
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2	{ v1.2s, v2.2s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .   ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER  .   ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER .   ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .   DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeeeeeER   ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2134,15 +2134,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.2s, v2.2s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    3.0       <total>
 
 # CHECK:      [21] Code Region - G22
 
@@ -2151,25 +2151,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      3310
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.79
 # CHECK-NEXT: IPC:               0.30
-# CHECK-NEXT: Block RThroughput: 5.0
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789          0123456789
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     . D=======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,3]     . D=======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,5]     .  D==============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .   D=====================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    D============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2179,16 +2179,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     24.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 3.     1     8.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 5.     1     15.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld2	{ v1.b, v2.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     22.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     29.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.0   0.1    3.0       <total>
 
 # CHECK:      [22] Code Region - G23
 
@@ -2197,7 +2197,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.62
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2208,14 +2208,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], #4
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,3]     D=========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D=======================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .D========================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,9]     . D===============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2226,15 +2226,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     24.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     25.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     31.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 9.     1     32.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     16.7   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld2	{ v1.h, v2.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
 
 # CHECK:      [23] Code Region - G24
 
@@ -2243,7 +2243,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      2603
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.96
 # CHECK-NEXT: IPC:               0.38
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2254,14 +2254,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .  .   ld2	{ v1.s, v2.s }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R    .    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D========eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,3]     D=========eE------R .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D===============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D================eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D================eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,7]     .D=================eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D================eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D=================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .  .   ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,3]     .D========eE------R .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER .   ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D===============eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D==============eeeeeeeeER.   ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D===============eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==============eeeeeeeeER   ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D===============eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2272,15 +2272,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2	{ v1.s, v2.s }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     9.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 3.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     16.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     17.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 7.     1     18.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     17.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     18.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     12.5   0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     15.0   0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 7.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     15.0   0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     16.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     11.3   0.1    3.0       <total>
 
 # CHECK:      [24] Code Region - G25
 
@@ -2289,7 +2289,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.90
 # CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2300,14 +2300,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.2s, v2.2s }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2318,15 +2318,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], #4
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], #4
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [25] Code Region - G26
 
@@ -2335,7 +2335,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      510
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.90
 # CHECK-NEXT: IPC:               1.96
 # CHECK-NEXT: Block RThroughput: 5.0
@@ -2346,14 +2346,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.16b, v2.16b }, [x27], #2
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2364,15 +2364,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], #2
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld2r	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    3.0       <total>
 
 # CHECK:      [26] Code Region - G27
 
@@ -2381,10 +2381,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      609
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.60
 # CHECK-NEXT: IPC:               1.64
-# CHECK-NEXT: Block RThroughput: 5.5
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     01234
@@ -2392,14 +2392,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeeeeeER   .   ld2r	{ v1.4s, v2.4s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,9]     . D===eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeeeER .   ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeeeER.   ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeeeER   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,9]     .    DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2410,42 +2410,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld2r	{ v1.4s, v2.4s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    3.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld2r	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ld2r	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ld2r	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.1    3.0       <total>
 
 # CHECK:      [27] Code Region - G28
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2455,43 +2455,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [28] Code Region - G29
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.01
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2501,43 +2501,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [29] Code Region - G30
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      2010
+# CHECK-NEXT: Total Cycles:      2011
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.84
 # CHECK-NEXT: IPC:               0.50
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .    .    .   .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .    .    .   .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .    .    .   .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R .    .    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D========eeeeeeeeER   .   .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,7]     .  D=========eE------R   .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,9]     .   D================eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789 
+
+# CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .   ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .    .    .    .   ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.    .    .    .   ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.    .    .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D======eeeeeeeeER  .    .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,7]     .    . D======eE------R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D============eeeeeeeeER   ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,9]     .    .   D============eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2547,16 +2547,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     9.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 7.     1     10.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     16.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 9.     1     17.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     6.3    0.2    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     13.0   0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 9.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     4.6    0.3    3.0       <total>
 
 # CHECK:      [30] Code Region - G31
 
@@ -2565,7 +2565,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.87
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 7.5
@@ -2575,15 +2575,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2593,16 +2593,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [31] Code Region - G32
 
@@ -2611,7 +2611,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        3500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    0.87
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 7.5
@@ -2621,15 +2621,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2639,43 +2639,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [32] Code Region - G33
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2685,43 +2685,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], #12
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], #6
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [33] Code Region - G34
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.01
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2731,43 +2731,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], #3
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], #3
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [34] Code Region - G35
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      759
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        3700
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.87
-# CHECK-NEXT: IPC:               1.32
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.67
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 7.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    ..   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   ..   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER ..   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE------R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeeeER..   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE------R..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE------R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2777,43 +2777,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld3r	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld3r	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld3r	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld3r	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [35] Code Region - G36
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      959
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.80
-# CHECK-NEXT: IPC:               1.04
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.55
+# CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 9.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER    .  .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeeeER .  .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,5]     . D==eE------R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===eeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,9]     .   D====eE-------R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   .   ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .   .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE-------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   .   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER  .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2823,43 +2823,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld3r	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 9.     1     5.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.4    3.2       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.2       <total>
 
 # CHECK:      [36] Code Region - G37
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,3]     .D=eE-------R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,5]     . D===eE-------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER .   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE-------R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,3]     .  DeE-------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeeER  .   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeE-------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeER.   ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2869,43 +2869,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 5.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.3       <total>
 
 # CHECK:      [37] Code Region - G38
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1009
+# CHECK-NEXT: Total Cycles:      1010
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.76
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.75
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789          
 
 # CHECK:      [0,0]     DeeeeeeeeER    .   .   ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeeER  .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE-------R  .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeeER  .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE-------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE-------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeeER .   .   ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE-------R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   .   ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeeER .   ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE-------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeeER   ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE-------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2915,16 +2915,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    7.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    7.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.3       <total>
 
 # CHECK:      [38] Code Region - G39
 
@@ -2933,7 +2933,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.12
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -2943,15 +2943,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -2961,16 +2961,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [39] Code Region - G40
 
@@ -2979,7 +2979,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      4003
 # CHECK-NEXT: Total uOps:        4500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.12
 # CHECK-NEXT: IPC:               0.25
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -2989,15 +2989,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123456789          012
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,1]     D=eE------R    .    .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .    .    .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,5]     . D===============eE------R   .    .    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=====================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .  D======================eE------R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D============================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .   D=============================eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .    .    .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .    .    .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D============eeeeeeeeER   .    .    . .   ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,5]     .    D============eE------R   .    .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D==================eeeeeeeeER.    . .   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .    . D==================eE------R.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D========================eeeeeeeeER   ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   D========================eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3007,16 +3007,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     15.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 5.     1     16.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     22.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 7.     1     23.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     29.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 9.     1     30.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     15.5   0.1    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     13.0   0.0    0.0       ld4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 5.     1     13.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     19.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: 7.     1     19.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     25.0   0.0    0.0       ld4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 9.     1     25.0   0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     13.0   0.1    3.0       <total>
 
 # CHECK:      [40] Code Region - G41
 
@@ -3025,7 +3025,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      2103
 # CHECK-NEXT: Total uOps:        4600
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.19
 # CHECK-NEXT: IPC:               0.48
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -3035,15 +3035,15 @@ add x0, x27, 1
 # CHECK-NEXT: Index     0123456789          0123
 
 # CHECK:      [0,0]     DeeeeeeeeER    .    .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,1]     D=eE------R    .    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .D=======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D========eE------R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=======eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,5]     . D========eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=========eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==========eE------R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=========eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: [0,9]     .   D==========eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . D======eeeeeeeeER .  .   ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  D======eE------R .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   D=====eeeeeeeeER.  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,5]     .    D=====eE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .D======eeeeeeeeER.   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . D======eE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  D=====eeeeeeeeER   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: [0,9]     .    .   D=====eE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3053,43 +3053,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     8.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 3.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     8.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 5.     1     9.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     10.0   2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
-# CHECK-NEXT: 7.     1     11.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     10.0   0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
-# CHECK-NEXT: 9.     1     11.0   0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     7.9    0.3    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     7.0    0.0    0.0       ld4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 3.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     6.0    0.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 5.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     7.0    2.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #32
+# CHECK-NEXT: 7.     1     7.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     6.0    0.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #16
+# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     5.4    0.3    3.0       <total>
 
 # CHECK:      [41] Code Region - G42
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3099,43 +3099,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #8
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #16
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #8
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #4
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [42] Code Region - G43
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1008
+# CHECK-NEXT: Total Cycles:      1009
 # CHECK-NEXT: Total uOps:        4700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.66
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012345678
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789         
 
 # CHECK:      [0,0]     DeeeeeeeeER    .  .   ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R    .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER   .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R   .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeeER  .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D===eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D====eeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D=====eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeE------R    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .  .   ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.  .   ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeeER .   ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE------R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeeeER   ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3145,43 +3145,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    2.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     6.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.5    3.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ld4r	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       ld4r	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    3.0       <total>
 
 # CHECK:      [43] Code Region - G44
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      708
+# CHECK-NEXT: Total Cycles:      808
 # CHECK-NEXT: Total uOps:        3900
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.51
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.83
+# CHECK-NEXT: IPC:               1.24
+# CHECK-NEXT: Block RThroughput: 7.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeeER   .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE------R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE------R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeeeER   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE------R   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeE-R   ldp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .  D===eE-----R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===eeeeeeER   ldp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     .  D====eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     012345
+# CHECK-NEXT: Index     0123456789      
+
+# CHECK:      [0,0]     DeeeeeeeeER    .   ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE------R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeeER  .   ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE------R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeeeER.   ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE------R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER.   ldp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .    .D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    . DeeeeeeER   ldp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .    . D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3191,43 +3191,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ld4r	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    1.0       ldp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.9    0.3    2.8       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ld4r	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       ld4r	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       ldp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.4    2.6       <total>
 
 # CHECK:      [44] Code Region - G45
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      507
+# CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.52
-# CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER ..   ldp	q1, q2, [x27], #992
-# CHECK-NEXT: [0,1]     D=eE----R ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER..   ldp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     .D=eE----R..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER.   ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeeER   ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     . D==eE----R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeE-R   ldp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     .  D==eE---R   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeeeeeER . .   ldp	q1, q2, [x27], #992
+# CHECK-NEXT: [0,1]     .DeE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeeeER.   ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeeER   ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .    DeE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeE-R   ldp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .    .DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3237,16 +3237,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	q1, q2, [x27], #992
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       ldp	s1, s2, [x27, #248]!
 # CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    1.0       ldp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.2    0.1    2.0       <total>
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    1.0       ldp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    2.0       <total>
 
 # CHECK:      [45] Code Region - G46
 
@@ -3255,7 +3255,7 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      1006
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    1.99
 # CHECK-NEXT: IPC:               0.99
 # CHECK-NEXT: Block RThroughput: 10.0
@@ -3266,14 +3266,14 @@ add x0, x27, 1
 
 # CHECK:      [0,0]     DeeeeER   .    .   ldp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eE--R   .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .    .   ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eE--R  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .    .   ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eE--R .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeER    .   ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: [0,7]     .D===eE---R    .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D======eeeeeER   ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: [0,9]     . D=======eE---R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .    .   ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eE--R  .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .    .   ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eE--R .    .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeER    .   ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: [0,7]     .  D=eE---R    .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D====eeeeeER   ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: [0,9]     .   D=====eE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3284,15 +3284,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     7.0    4.0    0.0       ldpsw	x1, x2, [x27, #248]!
-# CHECK-NEXT: 9.     1     8.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.5    0.5    1.2       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldpsw	x1, x2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     5.0    4.0    0.0       ldpsw	x1, x2, [x27, #248]!
+# CHECK-NEXT: 9.     1     6.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.5    1.2       <total>
 
 # CHECK:      [46] Code Region - G47
 
@@ -3301,25 +3301,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ldr	s1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ldr	d1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldr	q1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ldr	h1, [x27], #254
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ldr	s1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ldr	d1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ldr	q1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3330,15 +3330,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	s1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldr	q1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldr	h1, [x27], #254
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldr	s1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldr	d1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldr	q1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [47] Code Region - G48
 
@@ -3347,25 +3347,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      508
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.94
 # CHECK-NEXT: IPC:               1.97
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeeeeeER . .   ldr	b1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE----R . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeeeER. .   ldr	h1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   ldr	s1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeeeER.   ldr	d1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeeeER   ldr	q1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D===eE----R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeeeER. .   ldr	h1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eE----R. .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER .   ldr	s1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeeeER.   ldr	d1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeeeER   ldr	q1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .   D=eE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3376,15 +3376,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	b1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	h1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	s1, [x27, #254]!
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	d1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldr	q1, [x27, #254]!
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    2.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       ldr	h1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       ldr	s1, [x27, #254]!
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       ldr	d1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       ldr	q1, [x27, #254]!
+# CHECK-NEXT: 9.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    2.0       <total>
 
 # CHECK:      [48] Code Region - G49
 
@@ -3393,25 +3393,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldr	w1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldr	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldr	w1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldr	x1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrb	w1, [x27], #254
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldr	w1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldr	x1, [x27, #254]!
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrb	w1, [x27], #254
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3423,14 +3423,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	w1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldr	x1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldr	w1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldr	x1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrb	w1, [x27], #254
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldr	w1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldr	x1, [x27, #254]!
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrb	w1, [x27], #254
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [49] Code Region - G50
 
@@ -3439,25 +3439,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldrb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldrh	w1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrh	w1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldrsb	w1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrsb	x1, [x27], #254
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldrh	w1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldrsb	w1, [x27], #254
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrsb	x1, [x27], #254
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3469,14 +3469,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrh	w1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrh	w1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldrsb	w1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrsb	x1, [x27], #254
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrh	w1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldrsb	w1, [x27], #254
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrsb	x1, [x27], #254
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [50] Code Region - G51
 
@@ -3485,25 +3485,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        1500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    2.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER  .   ldrsb	x1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER .   ldrsh	w1, [x27], #254
-# CHECK-NEXT: [0,5]     D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   ldrsh	x1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeeeER   ldrsh	w1, [x27, #254]!
-# CHECK-NEXT: [0,9]     .D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER .   ldrsh	w1, [x27], #254
+# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeeeER.   ldrsh	x1, [x27], #254
+# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     . D==eeeeER   ldrsh	w1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .  D==eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3515,14 +3515,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsb	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsb	x1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsh	w1, [x27], #254
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       ldrsh	x1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       ldrsh	w1, [x27, #254]!
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    1.0       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrsh	w1, [x27], #254
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       ldrsh	x1, [x27], #254
+# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       ldrsh	w1, [x27, #254]!
+# CHECK-NEXT: 9.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
 
 # CHECK:      [51] Code Region - G52
 
@@ -3531,10 +3531,10 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        1700
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 3.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
@@ -3542,13 +3542,13 @@ add x0, x27, 1
 # CHECK:      [0,0]     DeeeeER .   ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eE--R .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D=eeeeER.   ldrsw	x1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     D==eeeeER   ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: [0,5]     D===eE--R   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeE-R   st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eE-R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .D===eeER   st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: [0,9]     .D====eER   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .D=eeeeER   ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: [0,5]     .D==eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     . D=eeE-R   st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: [0,7]     . D==eE-R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .  D=eeER   st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: [0,9]     .  D==eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3560,14 +3560,14 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldrsh	x1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     2.0    0.0    0.0       ldrsw	x1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       ldrsw	x1, [x27, #254]!
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    1.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.1    0.8       <total>
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       ldrsw	x1, [x27, #254]!
+# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    1.0       st1	{ v1.1d }, [x27], #8
+# CHECK-NEXT: 7.     1     3.0    0.0    1.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], #16
+# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.2    0.1    0.8       <total>
 
 # CHECK:      [52] Code Region - G53
 
@@ -3576,24 +3576,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3604,15 +3604,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s }, [x27], #8
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h }, [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8b }, [x27], #8
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [53] Code Region - G54
 
@@ -3621,24 +3621,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3649,15 +3649,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b }, [x27], #16
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.2s }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.4h }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [54] Code Region - G55
 
@@ -3666,24 +3666,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3694,15 +3694,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b }, [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8h }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.16b }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [55] Code Region - G56
 
@@ -3711,24 +3711,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.76
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeER.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,7]     .   DeER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3738,42 +3738,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.1    0.0       <total>
 
 # CHECK:      [56] Code Region - G57
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      504
+# CHECK-NEXT: Total Cycles:      604
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.16
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.30
+# CHECK-NEXT: IPC:               1.66
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.  .   st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeER.   st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER  .   st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeER .   st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeER.   st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    DeER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeER   st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3783,42 +3783,42 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.1    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    0.0       <total>
 
 # CHECK:      [57] Code Region - G58
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      504
+# CHECK-NEXT: Total Cycles:      604
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.16
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.30
+# CHECK-NEXT: IPC:               1.66
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     012345678
+# CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeeER.  .   st1	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK:      [0,0]     DeeER.   .   st1	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,3]     . DeER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER .   st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,7]     .   DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeER   st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3829,41 +3829,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    0.0       <total>
 
 # CHECK:      [58] Code Region - G59
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.84
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.39
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3873,42 +3874,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [59] Code Region - G60
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.12
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.59
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3918,42 +3920,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.9    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [60] Code Region - G61
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.84
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.39
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeER  .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -3963,43 +3966,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.7    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [61] Code Region - G62
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.11
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.59
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.    .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER  .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,5]     . D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,7]     .  D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4009,43 +4012,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [62] Code Region - G63
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      804
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        4200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.22
-# CHECK-NEXT: IPC:               1.24
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.19
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 8.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.    ..   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,1]     D=eER.    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    ..   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,3]     .D=eER    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER  ..   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,5]     . D==eER  ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER ..   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D==eER ..   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D===eeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D====eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4055,42 +4058,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    2.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.4    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.1d, v2.1d, v3.1d, v4.1d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [63] Code Region - G64
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      703
+# CHECK-NEXT: Total Cycles:      1003
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.41
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 7.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.79
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeER.   .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER   .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeER .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeER.   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeER.    . .   st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeER . .   st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeER . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeER .   st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeER .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeER   st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4100,16 +4104,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.1    0.2    0.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st1	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st1	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    0.0       <total>
 
 # CHECK:      [64] Code Region - G65
 
@@ -4118,25 +4122,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      706
 # CHECK-NEXT: Total uOps:        3200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.53
 # CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 5.5
+# CHECK-NEXT: Block RThroughput: 6.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789   
 
 # CHECK:      [0,0]     DeeER.    . .   st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeER    . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeER .   st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: [0,5]     . D===eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D===eeeeER.   st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: [0,7]     . D====eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D===eeeeER   st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .  D====eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,1]     .DeER.    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeER   . .   st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeER   . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER .   st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: [0,5]     .   D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    DeeeeER.   st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: [0,7]     .    D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeER   st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4146,16 +4150,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st1	{ v1.b }[0], [x27], #1
-# CHECK-NEXT: 5.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
-# CHECK-NEXT: 7.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.1    0.3    0.6       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st1	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st1	{ v1.b }[0], [x27], #1
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.b }[8], [x27], #1
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.3    0.6       <total>
 
 # CHECK:      [65] Code Region - G66
 
@@ -4164,25 +4168,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.b }[8], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4193,15 +4197,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.b }[8], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], #2
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], #2
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st1	{ v1.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [66] Code Region - G67
 
@@ -4210,25 +4214,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.35
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st1	{ v1.s }[0], [x27], #4
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: [0,9]     .    DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4239,15 +4243,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st1	{ v1.s }[0], [x27], #4
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st1	{ v1.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], #8
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st1	{ v1.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.4    0.1    1.0       <total>
 
 # CHECK:      [67] Code Region - G68
 
@@ -4256,25 +4260,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.74
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.2s, v2.2s }, [x27], #16
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: [0,5]     .  DeE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: [0,9]     .    DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4285,42 +4289,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.2s, v2.2s }, [x27], #16
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], #16
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.8b, v2.8b }, [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.1    1.0       <total>
 
 # CHECK:      [68] Code Region - G69
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total Cycles:      705
 # CHECK-NEXT: Total uOps:        2600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.14
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 4.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.69
+# CHECK-NEXT: IPC:               1.42
+# CHECK-NEXT: Block RThroughput: 5.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: [0,7]     . D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789  
+
+# CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.16b, v2.16b }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeER ..   st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE--R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeER..   st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: [0,5]     .  D=eE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeER.   st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: [0,7]     .   D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeER   st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .DeE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4330,43 +4334,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.1    1.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st2	{ v1.2d, v2.2d }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.2s, v2.2s }, [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.4h, v2.4h }, [x27], x28
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.4s, v2.4s }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.2    1.0       <total>
 
 # CHECK:      [69] Code Region - G70
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total Cycles:      606
 # CHECK-NEXT: Total uOps:        2400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.74
-# CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.96
+# CHECK-NEXT: IPC:               1.65
+# CHECK-NEXT: Block RThroughput: 4.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeER   .   st2	{ v1.8b, v2.8b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT:                     01
+# CHECK-NEXT: Index     0123456789  
+
+# CHECK:      [0,0]     DeeeeER   ..   st2	{ v1.8b, v2.8b }, [x27], x28
+# CHECK-NEXT: [0,1]     D=eE--R   ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  ..   st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: [0,3]     . DeE--R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .  DeeeeER..   st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: [0,5]     .   DeE--R..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeER.   st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: [0,7]     .   D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    DeeeeER   st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: [0,9]     .    D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4377,15 +4381,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.8b, v2.8b }, [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.8h, v2.8h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st2	{ v1.16b, v2.16b }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[0], [x27], #2
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], #2
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.3    0.2    1.0       <total>
 
 # CHECK:      [70] Code Region - G71
 
@@ -4394,25 +4398,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.b, v2.b }[0], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4423,15 +4427,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.b, v2.b }[0], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.b, v2.b }[8], [x27], x28
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], #4
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[4], [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.h, v2.h }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [71] Code Region - G72
 
@@ -4440,25 +4444,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      506
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.95
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   st2	{ v1.h, v2.h }[4], [x27], x28
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: [0,3]     D==eE--R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: [0,5]     .D==eE--R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: [0,7]     .D===eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: [0,9]     . D===eE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeeeER  .   st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .D=eE--R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeER .   st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: [0,5]     . D=eE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeeeER.   st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: [0,7]     .  D=eE--R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeeeER   st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .   D=eE--R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4469,42 +4473,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2	{ v1.h, v2.h }[4], [x27], x28
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
-# CHECK-NEXT: 3.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
-# CHECK-NEXT: 7.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], #8
+# CHECK-NEXT: 3.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st2	{ v1.s, v2.s }[0], [x27], x28
+# CHECK-NEXT: 5.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], #16
+# CHECK-NEXT: 7.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st2	{ v1.d, v2.d }[0], [x27], x28
+# CHECK-NEXT: 9.     1     2.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    1.0       <total>
 
 # CHECK:      [72] Code Region - G73
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      607
+# CHECK-NEXT: Total Cycles:      707
 # CHECK-NEXT: Total uOps:        2800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.61
-# CHECK-NEXT: IPC:               1.65
-# CHECK-NEXT: Block RThroughput: 4.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.96
+# CHECK-NEXT: IPC:               1.41
+# CHECK-NEXT: Block RThroughput: 5.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeER .    . .   st2g	x26, [x27], #4064
-# CHECK-NEXT: [0,1]     D=eER.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.    . .   st2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,3]     D==eER    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeeeeeER .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: [0,5]     .D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     . D=eeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: [0,7]     . D==eE---R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .  D==eeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: [0,9]     .  D===eE---R   add	x0, x27, #1
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789    
+
+# CHECK:      [0,0]     DeER .    .  .   st2g	x26, [x27], #4064
+# CHECK-NEXT: [0,1]     D=eER.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.    .  .   st2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,3]     .D=eER    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: [0,5]     .  DeE----R  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .   DeeeeeER .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: [0,7]     .    DeE---R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .DeeeeeER   st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: [0,9]     .    . DeE---R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4515,42 +4519,42 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st2g	x26, [x27], #4064
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       st2g	x26, [x27, #4064]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
-# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
-# CHECK-NEXT: 9.     1     4.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.2    1.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st2g	x26, [x27, #4064]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], #24
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], #24
+# CHECK-NEXT: 9.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.3    1.0       <total>
 
 # CHECK:      [73] Code Region - G74
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      708
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        3800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.37
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 7.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.77
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 7.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeER .   .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE---R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER.    ..   st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE---R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4560,43 +4564,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], #48
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], #48
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], #48
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.2d, v2.2d, v3.2d }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.9       <total>
 
 # CHECK:      [74] Code Region - G75
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      707
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        3400
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.81
-# CHECK-NEXT: IPC:               1.41
-# CHECK-NEXT: Block RThroughput: 6.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.38
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 6.8
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeER  .  .   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE---R  .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeER .  .   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE---R .  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . DeeeeeeER  .   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: [0,5]     . D=eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  DeeeeeER  .   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: [0,7]     .  D=eE---R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D=eeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: [0,9]     .   D==eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeER  .    ..   st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeER.    ..   st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE---R.    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeER ..   st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4606,43 +4610,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.2s, v2.2s, v3.2s }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     1.0    0.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
-# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     1.0    0.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
-# CHECK-NEXT: 7.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
-# CHECK-NEXT: 9.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.7    0.2    1.7       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.4h, v2.4h, v3.4h }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.4s, v2.4s, v3.4s }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.8b, v2.8b, v3.8b }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.8h, v2.8h, v3.8h }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.7       <total>
 
 # CHECK:      [75] Code Region - G76
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4652,43 +4656,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.16b, v2.16b, v3.16b }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], #3
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], #3
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.b, v2.b, v3.b }[8], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [76] Code Region - G77
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4698,43 +4702,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], #6
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], #6
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st3	{ v1.h, v2.h, v3.h }[4], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], #12
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [77] Code Region - G78
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      807
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.20
-# CHECK-NEXT: IPC:               1.24
-# CHECK-NEXT: Block RThroughput: 8.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.17
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.4
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeER  .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: [0,7]     .  D==eE---R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeER ..   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: [0,7]     .    . DeE---R ..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4744,43 +4748,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st3	{ v1.s, v2.s, v3.s }[0], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.9       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], #24
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st3	{ v1.d, v2.d, v3.d }[0], [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], #32
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    1.9       <total>
 
 # CHECK:      [78] Code Region - G79
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1205
+# CHECK-NEXT: Total Cycles:      1307
 # CHECK-NEXT: Total uOps:        5800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.81
-# CHECK-NEXT: IPC:               0.83
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.44
+# CHECK-NEXT: IPC:               0.77
 # CHECK-NEXT: Block RThroughput: 12.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: [0,1]     D=eE----R .    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeeER    ..   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: [0,3]     . DeE-----R    ..   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .  DeeeeeeER   ..   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: [0,5]     .  D=eE----R   ..   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .   D==eeeeeeeER.   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: [0,7]     .    D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D=eeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: [0,9]     .    . D=eE-----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          
+
+# CHECK:      [0,0]     DeeeeeeER .    .   .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
+# CHECK-NEXT: [0,1]     .DeE----R .    .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeeER   .   .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: [0,3]     .   DeE----R   .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    DeeeeeeER .   .   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
+# CHECK-NEXT: [0,5]     .    .DeE----R .   .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    . D=eeeeeeeER .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: [0,7]     .    .   DeE-----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeeeER   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: [0,9]     .    .    . DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4790,43 +4794,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], #32
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
-# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], #64
+# CHECK-NEXT: 3.     1     1.0    1.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], #32
-# CHECK-NEXT: 5.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     2.0    0.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
-# CHECK-NEXT: 9.     1     2.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT:        1     1.8    0.4    2.3       <total>
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    2.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], #64
+# CHECK-NEXT: 7.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], #64
+# CHECK-NEXT: 9.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.8    2.1       <total>
 
 # CHECK:      [79] Code Region - G80
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1006
+# CHECK-NEXT: Total Cycles:      1107
 # CHECK-NEXT: Total uOps:        4800
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.77
-# CHECK-NEXT: IPC:               0.99
-# CHECK-NEXT: Block RThroughput: 9.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.34
+# CHECK-NEXT: IPC:               0.90
+# CHECK-NEXT: Block RThroughput: 9.6
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     012345
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeER  .    .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: [0,1]     D=eE---R  .    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.    .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE----R.    .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D==eeeeeeER  .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: [0,5]     . D===eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D==eeeeeeeER.   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: [0,7]     .   D==eE-----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    D==eeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: [0,9]     .    D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789        
+
+# CHECK:      [0,0]     DeeeeeER  .    . .   st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
+# CHECK-NEXT: [0,1]     .DeE---R  .    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    . .   st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE----R    . .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  . .   st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: [0,5]     .    DeE----R  . .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeeER .   st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: [0,7]     .    .  DeE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .   DeeeeeeER   st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: [0,9]     .    .    DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4836,43 +4840,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.2d, v2.2d, v3.2d, v4.2d }, [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    3.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    2.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.6    0.4    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    3.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.2s, v2.2s, v3.2s, v4.2s }, [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.4h, v2.4h, v3.4h, v4.4h }, [x27], x28
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.4s, v2.4s, v3.4s, v4.4s }, [x27], x28
+# CHECK-NEXT: 7.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.8b, v2.8b, v3.8b, v4.8b }, [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.6    1.9       <total>
 
 # CHECK:      [80] Code Region - G81
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      1058
+# CHECK-NEXT: Total Cycles:      1207
 # CHECK-NEXT: Total uOps:        5200
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    4.91
-# CHECK-NEXT: IPC:               0.95
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.31
+# CHECK-NEXT: IPC:               0.83
 # CHECK-NEXT: Block RThroughput: 10.5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234567
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeeER.    . .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: [0,1]     .DeE-----R.    . .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     . DeeeeeeeER   . .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: [0,3]     .  DeE-----R   . .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .   D==eeeeeeER. .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: [0,5]     .   D===eE----R. .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .    D===eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: [0,7]     .    D====eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .    .D===eeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: [0,9]     .    .D====eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     012345678
+# CHECK-NEXT: Index     0123456789         
+
+# CHECK:      [0,0]     DeeeeeeeER.    .  .   st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
+# CHECK-NEXT: [0,1]     . DeE----R.    .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .  DeeeeeeeER  .  .   st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
+# CHECK-NEXT: [0,3]     .    DeE----R  .  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .    .DeeeeeeER.  .   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: [0,5]     .    . DeE----R.  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .  D=eeeeeeER.   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: [0,7]     .    .   D=eE----R.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .    DeeeeeeER   st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: [0,9]     .    .    .DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4882,43 +4886,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.8h, v2.8h, v3.8h, v4.8h }, [x27], x28
-# CHECK-NEXT: 1.     1     1.0    0.0    5.0       add	x0, x27, #1
+# CHECK-NEXT: 1.     1     1.0    1.0    4.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.16b, v2.16b, v3.16b, v4.16b }, [x27], x28
-# CHECK-NEXT: 3.     1     1.0    0.0    5.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    3.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
-# CHECK-NEXT: 5.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
-# CHECK-NEXT: 7.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     4.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
-# CHECK-NEXT: 9.     1     5.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.9    0.6    2.2       <total>
+# CHECK-NEXT: 3.     1     1.0    1.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], #4
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    2.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], #4
+# CHECK-NEXT: 7.     1     2.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[0], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.2    0.7    2.0       <total>
 
 # CHECK:      [81] Code Region - G82
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      757
+# CHECK-NEXT: Total Cycles:      1007
 # CHECK-NEXT: Total uOps:        4000
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.28
-# CHECK-NEXT: IPC:               1.32
-# CHECK-NEXT: Block RThroughput: 7.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.97
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 8.0
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01234
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   .   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: [0,1]     D=eE----R .   .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: [0,3]     .D=eE----R.   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeeeER  .   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: [0,5]     . D==eE----R  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeeeER .   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: [0,9]     .   D===eE----R   add	x0, x27, #1
+# CHECK-NEXT:                     0123456
+# CHECK-NEXT: Index     0123456789       
+
+# CHECK:      [0,0]     DeeeeeeER .    ..   st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
+# CHECK-NEXT: [0,1]     .DeE----R .    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER    ..   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: [0,3]     .  DeE----R    ..   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeeeER  ..   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: [0,5]     .    DeE----R  ..   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeeeER..   st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE----R..   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeeeeeeER   st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: [0,9]     .    .   DeE----R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4928,43 +4932,43 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.b, v2.b, v3.b, v4.b }[8], [x27], x28
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
-# CHECK-NEXT: 5.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
-# CHECK-NEXT: 9.     1     4.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], #8
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], #8
+# CHECK-NEXT: 5.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    0.0       st4	{ v1.h, v2.h, v3.h, v4.h }[4], [x27], x28
+# CHECK-NEXT: 9.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.0    0.5    2.0       <total>
 
 # CHECK:      [82] Code Region - G83
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      1000
-# CHECK-NEXT: Total Cycles:      704
+# CHECK-NEXT: Total Cycles:      904
 # CHECK-NEXT: Total uOps:        3600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.11
-# CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 6.5
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    3.98
+# CHECK-NEXT: IPC:               1.11
+# CHECK-NEXT: Block RThroughput: 7.2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
-
-# CHECK:      [0,0]     DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: [0,1]     D=eE----R .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     .DeeeeeeER.   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: [0,3]     .D=eE----R.   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     . D=eeeeER.   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: [0,5]     . D==eE--R.   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .  D=eeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: [0,7]     .  D==eE--R   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     .   D==eE-R   stg	x26, [x27], #4064
-# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
+# CHECK-NEXT:                     012
+# CHECK-NEXT: Index     0123456789   
+
+# CHECK:      [0,0]     DeeeeeeER . .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
+# CHECK-NEXT: [0,1]     .DeE----R . .   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     . DeeeeeeER .   st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: [0,3]     .  DeE----R .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     .   DeeeeER .   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: [0,5]     .    DeE--R .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .    .DeeeeER   st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: [0,7]     .    . DeE--R   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .    .  DeE-R   stg	x26, [x27], #4064
+# CHECK-NEXT: [0,9]     .    .  D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -4974,16 +4978,16 @@ add x0, x27, 1
 
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], #16
-# CHECK-NEXT: 1.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     1.0    0.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
-# CHECK-NEXT: 3.     1     2.0    0.0    4.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
-# CHECK-NEXT: 5.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     2.0    0.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
-# CHECK-NEXT: 7.     1     3.0    0.0    2.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    1.0    1.0       stg	x26, [x27], #4064
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.3    0.3    1.3       <total>
+# CHECK-NEXT: 1.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 2.     1     1.0    1.0    0.0       st4	{ v1.s, v2.s, v3.s, v4.s }[0], [x27], x28
+# CHECK-NEXT: 3.     1     1.0    0.0    4.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], #32
+# CHECK-NEXT: 5.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    1.0    0.0       st4	{ v1.d, v2.d, v3.d, v4.d }[0], [x27], x28
+# CHECK-NEXT: 7.     1     1.0    0.0    2.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    1.0    1.0       stg	x26, [x27], #4064
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.1    0.5    1.3       <total>
 
 # CHECK:      [83] Code Region - G84
 
@@ -4992,24 +4996,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stg	x26, [x27, #4064]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stgp	x1, x2, [x27], #992
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   stp	s1, s2, [x27], #248
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   stp	d1, d2, [x27], #496
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stgp	x1, x2, [x27], #992
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   stp	s1, s2, [x27], #248
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   stp	d1, d2, [x27], #496
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5020,15 +5024,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stg	x26, [x27, #4064]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stgp	x1, x2, [x27], #992
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       stp	s1, s2, [x27], #248
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stp	d1, d2, [x27], #496
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stgp	x1, x2, [x27], #992
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stgp	x1, x2, [x27, #992]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       stp	s1, s2, [x27], #248
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       stp	d1, d2, [x27], #496
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [84] Code Region - G85
 
@@ -5037,25 +5041,25 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      704
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.55
 # CHECK-NEXT: IPC:               1.42
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeER.    .   stp	q1, q2, [x27], #992
 # CHECK-NEXT: [0,1]     D==eER    .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D==eeER   .   stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D===eER   .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D==eeER  .   stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D===eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D===eeER .   stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: [0,7]     . D====eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D====eER.   stp	w1, w2, [x27], #248
-# CHECK-NEXT: [0,9]     . D=====eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .D=eeER   .   stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D==eER   .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . D=eeER  .   stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D==eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  D=eeER .   stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: [0,7]     .   D==eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   D==eER.   stp	w1, w2, [x27], #248
+# CHECK-NEXT: [0,9]     .   D===eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5066,15 +5070,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	q1, q2, [x27], #992
 # CHECK-NEXT: 1.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     3.0    0.0    0.0       stp	s1, s2, [x27, #248]!
-# CHECK-NEXT: 3.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     3.0    0.0    0.0       stp	d1, d2, [x27, #496]!
-# CHECK-NEXT: 5.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     4.0    0.0    0.0       stp	q1, q2, [x27, #992]!
-# CHECK-NEXT: 7.     1     5.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     5.0    0.0    0.0       stp	w1, w2, [x27], #248
-# CHECK-NEXT: 9.     1     6.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     3.8    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	s1, s2, [x27, #248]!
+# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stp	d1, d2, [x27, #496]!
+# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     2.0    0.0    0.0       stp	q1, q2, [x27, #992]!
+# CHECK-NEXT: 7.     1     3.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stp	w1, w2, [x27], #248
+# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.6    0.1    0.0       <total>
 
 # CHECK:      [85] Code Region - G86
 
@@ -5083,24 +5087,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2200
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.37
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.0
+# CHECK-NEXT: Block RThroughput: 4.4
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   stp	x1, x2, [x27], #496
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   str	b1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   str	h1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   str	b1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   str	h1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5111,15 +5115,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       stp	x1, x2, [x27], #496
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stp	w1, w2, [x27, #248]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stp	x1, x2, [x27, #496]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	b1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stp	w1, w2, [x27, #248]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stp	x1, x2, [x27, #496]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	b1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	h1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [86] Code Region - G87
 
@@ -5128,24 +5132,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2500
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.96
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.8
+# CHECK-NEXT: Block RThroughput: 5.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   str	s1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27], #254
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eeER.   str	b1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eeER   str	h1, [x27, #254]!
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   str	d1, [x27], #254
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   str	q1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeeER.   str	b1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeeER   str	h1, [x27, #254]!
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5156,15 +5160,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27], #254
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	b1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	h1, [x27, #254]!
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	d1, [x27], #254
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       str	q1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	b1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	h1, [x27, #254]!
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [87] Code Region - G88
 
@@ -5173,24 +5177,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2300
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    4.56
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 3.3
+# CHECK-NEXT: Block RThroughput: 4.6
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeeER.  .   str	s1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eeER  .   str	d1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eeER .   str	q1, [x27, #254]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   str	w1, [x27], #254
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   str	x1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeeER  .   str	d1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeeER .   str	q1, [x27, #254]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   str	w1, [x27], #254
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   str	x1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5201,15 +5205,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	s1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	d1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       str	q1, [x27, #254]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       str	w1, [x27], #254
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       str	x1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	d1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       str	q1, [x27, #254]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       str	w1, [x27], #254
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       str	x1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [88] Code Region - G89
 
@@ -5218,24 +5222,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   str	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   str	x1, [x27, #254]!
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   strb	w1, [x27], #254
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   strb	w1, [x27, #254]!
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   strh	w1, [x27], #254
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   str	x1, [x27, #254]!
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   strb	w1, [x27], #254
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   strb	w1, [x27, #254]!
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   strh	w1, [x27], #254
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5246,15 +5250,15 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       str	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       str	x1, [x27, #254]!
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       strb	w1, [x27], #254
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       strb	w1, [x27, #254]!
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       strh	w1, [x27], #254
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       str	x1, [x27, #254]!
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       strb	w1, [x27], #254
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       strb	w1, [x27, #254]!
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       strh	w1, [x27], #254
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [89] Code Region - G90
 
@@ -5263,24 +5267,24 @@ add x0, x27, 1
 # CHECK-NEXT: Total Cycles:      504
 # CHECK-NEXT: Total uOps:        2000
 
-# CHECK:      Dispatch Width:    10
+# CHECK:      Dispatch Width:    5
 # CHECK-NEXT: uOps Per Cycle:    3.97
 # CHECK-NEXT: IPC:               1.98
-# CHECK-NEXT: Block RThroughput: 2.5
+# CHECK-NEXT: Block RThroughput: 4.0
 
 # CHECK:      Timeline view:
 # CHECK-NEXT: Index     012345678
 
 # CHECK:      [0,0]     DeER .  .   strh	w1, [x27, #254]!
 # CHECK-NEXT: [0,1]     D=eER.  .   add	x0, x27, #1
-# CHECK-NEXT: [0,2]     D=eER.  .   stz2g	x26, [x27], #4064
-# CHECK-NEXT: [0,3]     D==eER  .   add	x0, x27, #1
-# CHECK-NEXT: [0,4]     .D=eER  .   stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: [0,5]     .D==eER .   add	x0, x27, #1
-# CHECK-NEXT: [0,6]     .D==eER .   stzg	x26, [x27], #4064
-# CHECK-NEXT: [0,7]     .D===eER.   add	x0, x27, #1
-# CHECK-NEXT: [0,8]     . D==eER.   stzg	x26, [x27, #4064]!
-# CHECK-NEXT: [0,9]     . D===eER   add	x0, x27, #1
+# CHECK-NEXT: [0,2]     .DeER.  .   stz2g	x26, [x27], #4064
+# CHECK-NEXT: [0,3]     .D=eER  .   add	x0, x27, #1
+# CHECK-NEXT: [0,4]     . DeER  .   stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: [0,5]     . D=eER .   add	x0, x27, #1
+# CHECK-NEXT: [0,6]     .  DeER .   stzg	x26, [x27], #4064
+# CHECK-NEXT: [0,7]     .  D=eER.   add	x0, x27, #1
+# CHECK-NEXT: [0,8]     .   DeER.   stzg	x26, [x27, #4064]!
+# CHECK-NEXT: [0,9]     .   D=eER   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5291,36 +5295,36 @@ add x0, x27, 1
 # CHECK:            [0]    [1]    [2]    [3]
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       strh	w1, [x27, #254]!
 # CHECK-NEXT: 1.     1     2.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 2.     1     2.0    0.0    0.0       stz2g	x26, [x27], #4064
-# CHECK-NEXT: 3.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 4.     1     2.0    0.0    0.0       stz2g	x26, [x27, #4064]!
-# CHECK-NEXT: 5.     1     3.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 6.     1     3.0    0.0    0.0       stzg	x26, [x27], #4064
-# CHECK-NEXT: 7.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT: 8.     1     3.0    0.0    0.0       stzg	x26, [x27, #4064]!
-# CHECK-NEXT: 9.     1     4.0    0.0    0.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.7    0.1    0.0       <total>
+# CHECK-NEXT: 2.     1     1.0    0.0    0.0       stz2g	x26, [x27], #4064
+# CHECK-NEXT: 3.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 4.     1     1.0    0.0    0.0       stz2g	x26, [x27, #4064]!
+# CHECK-NEXT: 5.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 6.     1     1.0    0.0    0.0       stzg	x26, [x27], #4064
+# CHECK-NEXT: 7.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT: 8.     1     1.0    0.0    0.0       stzg	x26, [x27, #4064]!
+# CHECK-NEXT: 9.     1     2.0    0.0    0.0       add	x0, x27, #1
+# CHECK-NEXT:        1     1.5    0.1    0.0       <total>
 
 # CHECK:      [90] Code Region - G91
 
 # CHECK:      Iterations:        100
 # CHECK-NEXT: Instructions:      400
-# CHECK-NEXT: Total Cycles:      110
+# CHECK-NEXT: Total Cycles:      143
 # CHECK-NEXT: Total uOps:        600
 
-# CHECK:      Dispatch Width:    10
-# CHECK-NEXT: uOps Per Cycle:    5.45
-# CHECK-NEXT: IPC:               3.64
-# CHECK-NEXT: Block RThroughput: 1.0
+# CHECK:      Dispatch Width:    5
+# CHECK-NEXT: uOps Per Cycle:    4.20
+# CHECK-NEXT: IPC:               2.80
+# CHECK-NEXT: Block RThroughput: 1.2
 
 # CHECK:      Timeline view:
 # CHECK-NEXT:                     0
-# CHECK-NEXT: Index     0123456789
+# CHECK-NEXT: Index     0123456789 
 
 # CHECK:      [0,0]     DeeeeER   .   ldr	x1, [x27], #254
 # CHECK-NEXT: [0,1]     D=eE--R   .   add	x0, x27, #1
 # CHECK-NEXT: [0,2]     D====eeeeER   ldr	x2, [x1], #254
-# CHECK-NEXT: [0,3]     D=eE------R   add	x0, x27, #1
+# CHECK-NEXT: [0,3]     .DeE------R   add	x0, x27, #1
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -5332,5 +5336,5 @@ add x0, x27, 1
 # CHECK-NEXT: 0.     1     1.0    1.0    0.0       ldr	x1, [x27], #254
 # CHECK-NEXT: 1.     1     2.0    0.0    2.0       add	x0, x27, #1
 # CHECK-NEXT: 2.     1     5.0    0.0    0.0       ldr	x2, [x1], #254
-# CHECK-NEXT: 3.     1     2.0    0.0    6.0       add	x0, x27, #1
-# CHECK-NEXT:        1     2.5    0.3    2.0       <total>
+# CHECK-NEXT: 3.     1     1.0    0.0    6.0       add	x0, x27, #1
+# CHECK-NEXT:        1     2.3    0.3    2.0       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/independent-load-stores.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/independent-load-stores.s
new file mode 100644
index 000000000000..7a615dbacc22
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/independent-load-stores.s
@@ -0,0 +1,148 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -timeline -timeline-max-iterations=1 < %s | FileCheck %s -check-prefixes=ALL,NOALIAS
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -timeline -timeline-max-iterations=1 -noalias=false < %s | FileCheck %s -check-prefixes=ALL,YESALIAS
+
+  addq	$44, 64(%r14)
+  addq	$44, 128(%r14)
+  addq	$44, 192(%r14)
+  addq	$44, 256(%r14)
+  addq	$44, 320(%r14)
+  addq	$44, 384(%r14)
+  addq	$44, 448(%r14)
+  addq	$44, 512(%r14)
+  addq	$44, 576(%r14)
+  addq	$44, 640(%r14)
+
+# ALL:           Iterations:        100
+# ALL-NEXT:      Instructions:      1000
+
+# NOALIAS-NEXT:  Total Cycles:      681
+# YESALIAS-NEXT: Total Cycles:      12003
+
+# ALL-NEXT:      Total uOps:        4000
+
+# ALL:           Dispatch Width:    8
+
+# NOALIAS-NEXT:  uOps Per Cycle:    5.87
+# NOALIAS-NEXT:  IPC:               1.47
+
+# YESALIAS-NEXT: uOps Per Cycle:    0.33
+# YESALIAS-NEXT: IPC:               0.08
+
+# ALL-NEXT:      Block RThroughput: 6.7
+
+# ALL:           Instruction Info:
+# ALL-NEXT:      [1]: #uOps
+# ALL-NEXT:      [2]: Latency
+# ALL-NEXT:      [3]: RThroughput
+# ALL-NEXT:      [4]: MayLoad
+# ALL-NEXT:      [5]: MayStore
+# ALL-NEXT:      [6]: HasSideEffects (U)
+
+# ALL:           [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 64(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 128(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 192(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 256(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 320(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 384(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 448(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 512(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 576(%r14)
+# ALL-NEXT:       4      12    0.67    *      *            addq	$44, 640(%r14)
+
+# ALL:           Resources:
+# ALL-NEXT:      [0]   - LNLPPort00
+# ALL-NEXT:      [1]   - LNLPPort01
+# ALL-NEXT:      [2]   - LNLPPort02
+# ALL-NEXT:      [3]   - LNLPPort03
+# ALL-NEXT:      [4]   - LNLPPort04
+# ALL-NEXT:      [5]   - LNLPPort05
+# ALL-NEXT:      [6]   - LNLPPort10
+# ALL-NEXT:      [7]   - LNLPPort11
+# ALL-NEXT:      [8]   - LNLPPort20
+# ALL-NEXT:      [9]   - LNLPPort21
+# ALL-NEXT:      [10]  - LNLPPort22
+# ALL-NEXT:      [11]  - LNLPPort25
+# ALL-NEXT:      [12]  - LNLPPort26
+# ALL-NEXT:      [13]  - LNLPPort27
+# ALL-NEXT:      [14]  - LNLPPortInvalid
+# ALL-NEXT:      [15]  - LNLPVPort00
+# ALL-NEXT:      [16]  - LNLPVPort01
+# ALL-NEXT:      [17]  - LNLPVPort02
+# ALL-NEXT:      [18]  - LNLPVPort03
+
+# ALL:           Resource pressure per iteration:
+# ALL-NEXT:      [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# ALL-NEXT:       -     3.33    -     3.33    -     3.34   5.00   5.00   6.66   6.66   6.68   3.33   3.33   3.34    -      -      -      -      -
+
+# ALL:           Resource pressure by instruction:
+# ALL-NEXT:      [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# ALL-NEXT:       -     0.33    -     0.33    -     0.34    -     1.00   0.66   0.66   0.68   0.33   0.33   0.34    -      -      -      -      -     addq	$44, 64(%r14)
+# ALL-NEXT:       -     0.33    -     0.34    -     0.33   1.00    -     0.66   0.68   0.66   0.33   0.34   0.33    -      -      -      -      -     addq	$44, 128(%r14)
+# ALL-NEXT:       -     0.34    -     0.33    -     0.33    -     1.00   0.68   0.66   0.66   0.34   0.33   0.33    -      -      -      -      -     addq	$44, 192(%r14)
+# ALL-NEXT:       -     0.33    -     0.33    -     0.34   1.00    -     0.66   0.66   0.68   0.33   0.33   0.34    -      -      -      -      -     addq	$44, 256(%r14)
+# ALL-NEXT:       -     0.33    -     0.34    -     0.33    -     1.00   0.66   0.68   0.66   0.33   0.34   0.33    -      -      -      -      -     addq	$44, 320(%r14)
+# ALL-NEXT:       -     0.34    -     0.33    -     0.33   1.00    -     0.68   0.66   0.66   0.34   0.33   0.33    -      -      -      -      -     addq	$44, 384(%r14)
+# ALL-NEXT:       -     0.33    -     0.33    -     0.34    -     1.00   0.66   0.66   0.68   0.33   0.33   0.34    -      -      -      -      -     addq	$44, 448(%r14)
+# ALL-NEXT:       -     0.33    -     0.34    -     0.33   1.00    -     0.66   0.68   0.66   0.33   0.34   0.33    -      -      -      -      -     addq	$44, 512(%r14)
+# ALL-NEXT:       -     0.34    -     0.33    -     0.33    -     1.00   0.68   0.66   0.66   0.34   0.33   0.33    -      -      -      -      -     addq	$44, 576(%r14)
+# ALL-NEXT:       -     0.33    -     0.33    -     0.34   1.00    -     0.66   0.66   0.68   0.33   0.33   0.34    -      -      -      -      -     addq	$44, 640(%r14)
+
+# ALL:           Timeline view:
+
+# NOALIAS-NEXT:                      0123456789
+# NOALIAS-NEXT:  Index     0123456789          0
+
+# YESALIAS-NEXT:                     0123456789          0123456789          0123456789          01234
+# YESALIAS-NEXT: Index     0123456789          0123456789          0123456789          0123456789
+
+# NOALIAS:       [0,0]     DeeeeeeeeeeeeER.    .   addq	$44, 64(%r14)
+# NOALIAS-NEXT:  [0,1]     DeeeeeeeeeeeeER.    .   addq	$44, 128(%r14)
+# NOALIAS-NEXT:  [0,2]     .DeeeeeeeeeeeeER    .   addq	$44, 192(%r14)
+# NOALIAS-NEXT:  [0,3]     .D=eeeeeeeeeeeeER   .   addq	$44, 256(%r14)
+# NOALIAS-NEXT:  [0,4]     . DeeeeeeeeeeeeER   .   addq	$44, 320(%r14)
+# NOALIAS-NEXT:  [0,5]     . D=eeeeeeeeeeeeER  .   addq	$44, 384(%r14)
+# NOALIAS-NEXT:  [0,6]     .  D=eeeeeeeeeeeeER .   addq	$44, 448(%r14)
+# NOALIAS-NEXT:  [0,7]     .  D=eeeeeeeeeeeeER .   addq	$44, 512(%r14)
+# NOALIAS-NEXT:  [0,8]     .   D=eeeeeeeeeeeeER.   addq	$44, 576(%r14)
+# NOALIAS-NEXT:  [0,9]     .   D==eeeeeeeeeeeeER   addq	$44, 640(%r14)
+
+# YESALIAS:      [0,0]     DeeeeeeeeeeeeER.    .    .    .    .    .    .    .    .    .    .    .   .   addq	$44, 64(%r14)
+# YESALIAS-NEXT: [0,1]     D============eeeeeeeeeeeeER   .    .    .    .    .    .    .    .    .   .   addq	$44, 128(%r14)
+# YESALIAS-NEXT: [0,2]     .D=======================eeeeeeeeeeeeER .    .    .    .    .    .    .   .   addq	$44, 192(%r14)
+# YESALIAS-NEXT: [0,3]     .D===================================eeeeeeeeeeeeER    .    .    .    .   .   addq	$44, 256(%r14)
+# YESALIAS-NEXT: [0,4]     . D==============================================eeeeeeeeeeeeER  .    .   .   addq	$44, 320(%r14)
+# YESALIAS-NEXT: [0,5]     . D==========================================================eeeeeeeeeeeeER   addq	$44, 384(%r14)
+# YESALIAS-NEXT: Truncated display due to cycle limit
+
+# ALL:           Average Wait times (based on the timeline view):
+# ALL-NEXT:      [0]: Executions
+# ALL-NEXT:      [1]: Average time spent waiting in a scheduler's queue
+# ALL-NEXT:      [2]: Average time spent waiting in a scheduler's queue while ready
+# ALL-NEXT:      [3]: Average time elapsed from WB until retire stage
+
+# ALL:                 [0]    [1]    [2]    [3]
+# ALL-NEXT:      0.     1     1.0    1.0    0.0       addq	$44, 64(%r14)
+
+# NOALIAS-NEXT:  1.     1     1.0    0.0    0.0       addq	$44, 128(%r14)
+# NOALIAS-NEXT:  2.     1     1.0    1.0    0.0       addq	$44, 192(%r14)
+# NOALIAS-NEXT:  3.     1     2.0    1.0    0.0       addq	$44, 256(%r14)
+# NOALIAS-NEXT:  4.     1     1.0    0.0    0.0       addq	$44, 320(%r14)
+# NOALIAS-NEXT:  5.     1     2.0    1.0    0.0       addq	$44, 384(%r14)
+# NOALIAS-NEXT:  6.     1     2.0    1.0    0.0       addq	$44, 448(%r14)
+# NOALIAS-NEXT:  7.     1     2.0    0.0    0.0       addq	$44, 512(%r14)
+# NOALIAS-NEXT:  8.     1     2.0    1.0    0.0       addq	$44, 576(%r14)
+# NOALIAS-NEXT:  9.     1     3.0    1.0    0.0       addq	$44, 640(%r14)
+# NOALIAS-NEXT:         1     1.7    0.7    0.0       <total>
+
+# YESALIAS-NEXT: 1.     1     13.0   0.0    0.0       addq	$44, 128(%r14)
+# YESALIAS-NEXT: 2.     1     24.0   0.0    0.0       addq	$44, 192(%r14)
+# YESALIAS-NEXT: 3.     1     36.0   0.0    0.0       addq	$44, 256(%r14)
+# YESALIAS-NEXT: 4.     1     47.0   0.0    0.0       addq	$44, 320(%r14)
+# YESALIAS-NEXT: 5.     1     59.0   0.0    0.0       addq	$44, 384(%r14)
+# YESALIAS-NEXT: 6.     1     70.0   0.0    0.0       addq	$44, 448(%r14)
+# YESALIAS-NEXT: 7.     1     82.0   0.0    0.0       addq	$44, 512(%r14)
+# YESALIAS-NEXT: 8.     1     93.0   0.0    0.0       addq	$44, 576(%r14)
+# YESALIAS-NEXT: 9.     1     105.0  0.0    0.0       addq	$44, 640(%r14)
+# YESALIAS-NEXT:        1     53.0   0.1    0.0       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/partially-overlapping-groups.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/partially-overlapping-groups.s
new file mode 100644
index 000000000000..c118824ba79c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/partially-overlapping-groups.s
@@ -0,0 +1,21 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -all-views=false -summary-view < %s | FileCheck %s
+
+# Issue #57548
+
+# Do not crash when simulating instructions that consume partially overlapping
+# resource groups.
+
+vpsllw %xmm1, %ymm0, %ymm0
+vpsllw %xmm1, %xmm2, %xmm1
+vpand %ymm1, %ymm0, %ymm0
+
+# CHECK:      Iterations:        100
+# CHECK-NEXT: Instructions:      300
+# CHECK-NEXT: Total Cycles:      503
+# CHECK-NEXT: Total uOps:        500
+
+# CHECK:      Dispatch Width:    8
+# CHECK-NEXT: uOps Per Cycle:    0.99
+# CHECK-NEXT: IPC:               0.60
+# CHECK-NEXT: Block RThroughput: 1.0
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-aes.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-aes.s
new file mode 100644
index 000000000000..d94cad161dea
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-aes.s
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+aesdec          %xmm0, %xmm2
+aesdec          (%rax), %xmm2
+
+aesdeclast      %xmm0, %xmm2
+aesdeclast      (%rax), %xmm2
+
+aesenc          %xmm0, %xmm2
+aesenc          (%rax), %xmm2
+
+aesenclast      %xmm0, %xmm2
+aesenclast      (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     2.00                        aesdec	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   aesdec	(%rax), %xmm2
+# CHECK-NEXT:  1      3     2.00                        aesdeclast	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   aesdeclast	(%rax), %xmm2
+# CHECK-NEXT:  1      3     2.00                        aesenc	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   aesenc	(%rax), %xmm2
+# CHECK-NEXT:  1      3     2.00                        aesenclast	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   aesenclast	(%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.33   9.33   9.33    -      -      -      -     16.00  16.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     aesdec	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     aesdec	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     aesdeclast	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     aesdeclast	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     aesenc	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     aesenc	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     aesenclast	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     aesenclast	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx1.s
new file mode 100644
index 000000000000..cbf995140edf
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx1.s
@@ -0,0 +1,2428 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vaddpd            %xmm0, %xmm1, %xmm2
+vaddpd            (%rax), %xmm1, %xmm2
+
+vaddpd            %ymm0, %ymm1, %ymm2
+vaddpd            (%rax), %ymm1, %ymm2
+
+vaddps            %xmm0, %xmm1, %xmm2
+vaddps            (%rax), %xmm1, %xmm2
+
+vaddps            %ymm0, %ymm1, %ymm2
+vaddps            (%rax), %ymm1, %ymm2
+
+vaddsd            %xmm0, %xmm1, %xmm2
+vaddsd            (%rax), %xmm1, %xmm2
+
+vaddss            %xmm0, %xmm1, %xmm2
+vaddss            (%rax), %xmm1, %xmm2
+
+vaddsubpd         %xmm0, %xmm1, %xmm2
+vaddsubpd         (%rax), %xmm1, %xmm2
+
+vaddsubpd         %ymm0, %ymm1, %ymm2
+vaddsubpd         (%rax), %ymm1, %ymm2
+
+vaddsubps         %xmm0, %xmm1, %xmm2
+vaddsubps         (%rax), %xmm1, %xmm2
+
+vaddsubps         %ymm0, %ymm1, %ymm2
+vaddsubps         (%rax), %ymm1, %ymm2
+
+vaesdec           %xmm0, %xmm1, %xmm2
+vaesdec           (%rax), %xmm1, %xmm2
+
+vaesdeclast       %xmm0, %xmm1, %xmm2
+vaesdeclast       (%rax), %xmm1, %xmm2
+
+vaesenc           %xmm0, %xmm1, %xmm2
+vaesenc           (%rax), %xmm1, %xmm2
+
+vaesenclast       %xmm0, %xmm1, %xmm2
+vaesenclast       (%rax), %xmm1, %xmm2
+
+vandnpd           %xmm0, %xmm1, %xmm2
+vandnpd           (%rax), %xmm1, %xmm2
+
+vandnpd           %ymm0, %ymm1, %ymm2
+vandnpd           (%rax), %ymm1, %ymm2
+
+vandnps           %xmm0, %xmm1, %xmm2
+vandnps           (%rax), %xmm1, %xmm2
+
+vandnps           %ymm0, %ymm1, %ymm2
+vandnps           (%rax), %ymm1, %ymm2
+
+vandpd            %xmm0, %xmm1, %xmm2
+vandpd            (%rax), %xmm1, %xmm2
+
+vandpd            %ymm0, %ymm1, %ymm2
+vandpd            (%rax), %ymm1, %ymm2
+
+vandps            %xmm0, %xmm1, %xmm2
+vandps            (%rax), %xmm1, %xmm2
+
+vandps            %ymm0, %ymm1, %ymm2
+vandps            (%rax), %ymm1, %ymm2
+
+vblendpd          $11, %xmm0, %xmm1, %xmm2
+vblendpd          $11, (%rax), %xmm1, %xmm2
+
+vblendpd          $11, %ymm0, %ymm1, %ymm2
+vblendpd          $11, (%rax), %ymm1, %ymm2
+
+vblendps          $11, %xmm0, %xmm1, %xmm2
+vblendps          $11, (%rax), %xmm1, %xmm2
+
+vblendps          $11, %ymm0, %ymm1, %ymm2
+vblendps          $11, (%rax), %ymm1, %ymm2
+
+vblendvpd         %xmm3, %xmm0, %xmm1, %xmm2
+vblendvpd         %xmm3, (%rax), %xmm1, %xmm2
+
+vblendvpd         %ymm3, %ymm0, %ymm1, %ymm2
+vblendvpd         %ymm3, (%rax), %ymm1, %ymm2
+
+vblendvps         %xmm3, %xmm0, %xmm1, %xmm2
+vblendvps         %xmm3, (%rax), %xmm1, %xmm2
+
+vblendvps         %ymm3, %ymm0, %ymm1, %ymm2
+vblendvps         %ymm3, (%rax), %ymm1, %ymm2
+
+vbroadcastf128    (%rax), %ymm2
+
+vbroadcastsd      (%rax), %ymm2
+
+vbroadcastss      (%rax), %xmm2
+vbroadcastss      (%rax), %ymm2
+
+vcmppd            $0, %xmm0, %xmm1, %xmm2
+vcmppd            $0, (%rax), %xmm1, %xmm2
+
+vcmppd            $0, %ymm0, %ymm1, %ymm2
+vcmppd            $0, (%rax), %ymm1, %ymm2
+
+vcmpps            $0, %xmm0, %xmm1, %xmm2
+vcmpps            $0, (%rax), %xmm1, %xmm2
+
+vcmpps            $0, %ymm0, %ymm1, %ymm2
+vcmpps            $0, (%rax), %ymm1, %ymm2
+
+vcmpsd            $0, %xmm0, %xmm1, %xmm2
+vcmpsd            $0, (%rax), %xmm1, %xmm2
+
+vcmpss            $0, %xmm0, %xmm1, %xmm2
+vcmpss            $0, (%rax), %xmm1, %xmm2
+
+vcomisd           %xmm0, %xmm1
+vcomisd           (%rax), %xmm1
+
+vcomiss           %xmm0, %xmm1
+vcomiss           (%rax), %xmm1
+
+vcvtdq2pd         %xmm0, %xmm2
+vcvtdq2pd         (%rax), %xmm2
+
+vcvtdq2pd         %xmm0, %ymm2
+vcvtdq2pd         (%rax), %ymm2
+
+vcvtdq2ps         %xmm0, %xmm2
+vcvtdq2ps         (%rax), %xmm2
+
+vcvtdq2ps         %ymm0, %ymm2
+vcvtdq2ps         (%rax), %ymm2
+
+vcvtpd2dqx        %xmm0, %xmm2
+vcvtpd2dqx        (%rax), %xmm2
+
+vcvtpd2dqy        %ymm0, %xmm2
+vcvtpd2dqy        (%rax), %xmm2
+
+vcvtpd2psx        %xmm0, %xmm2
+vcvtpd2psx        (%rax), %xmm2
+
+vcvtpd2psy        %ymm0, %xmm2
+vcvtpd2psy        (%rax), %xmm2
+
+vcvtps2dq         %xmm0, %xmm2
+vcvtps2dq         (%rax), %xmm2
+
+vcvtps2dq         %ymm0, %ymm2
+vcvtps2dq         (%rax), %ymm2
+
+vcvtps2pd         %xmm0, %xmm2
+vcvtps2pd         (%rax), %xmm2
+
+vcvtps2pd         %xmm0, %ymm2
+vcvtps2pd         (%rax), %ymm2
+
+vcvtsd2si         %xmm0, %ecx
+vcvtsd2si         %xmm0, %rcx
+vcvtsd2si         (%rax), %ecx
+vcvtsd2si         (%rax), %rcx
+
+vcvtsd2ss         %xmm0, %xmm1, %xmm2
+vcvtsd2ss         (%rax), %xmm1, %xmm2
+
+vcvtsi2sdl        %ecx, %xmm0, %xmm2
+vcvtsi2sdq        %rcx, %xmm0, %xmm2
+vcvtsi2sdl        (%rax), %xmm0, %xmm2
+vcvtsi2sdq        (%rax), %xmm0, %xmm2
+
+vcvtsi2ssl        %ecx, %xmm0, %xmm2
+vcvtsi2ssq        %rcx, %xmm0, %xmm2
+vcvtsi2ssl        (%rax), %xmm0, %xmm2
+vcvtsi2ssq        (%rax), %xmm0, %xmm2
+
+vcvtss2sd         %xmm0, %xmm1, %xmm2
+vcvtss2sd         (%rax), %xmm1, %xmm2
+
+vcvtss2si         %xmm0, %ecx
+vcvtss2si         %xmm0, %rcx
+vcvtss2si         (%rax), %ecx
+vcvtss2si         (%rax), %rcx
+
+vcvttpd2dqx       %xmm0, %xmm2
+vcvttpd2dqx       (%rax), %xmm2
+
+vcvttpd2dqy       %ymm0, %xmm2
+vcvttpd2dqy       (%rax), %xmm2
+
+vcvttps2dq        %xmm0, %xmm2
+vcvttps2dq        (%rax), %xmm2
+
+vcvttps2dq        %ymm0, %ymm2
+vcvttps2dq        (%rax), %ymm2
+
+vcvttsd2si        %xmm0, %ecx
+vcvttsd2si        %xmm0, %rcx
+vcvttsd2si        (%rax), %ecx
+vcvttsd2si        (%rax), %rcx
+
+vcvttss2si        %xmm0, %ecx
+vcvttss2si        %xmm0, %rcx
+vcvttss2si        (%rax), %ecx
+vcvttss2si        (%rax), %rcx
+
+vdivpd            %xmm0, %xmm1, %xmm2
+vdivpd            (%rax), %xmm1, %xmm2
+
+vdivpd            %ymm0, %ymm1, %ymm2
+vdivpd            (%rax), %ymm1, %ymm2
+
+vdivps            %xmm0, %xmm1, %xmm2
+vdivps            (%rax), %xmm1, %xmm2
+
+vdivps            %ymm0, %ymm1, %ymm2
+vdivps            (%rax), %ymm1, %ymm2
+
+vdivsd            %xmm0, %xmm1, %xmm2
+vdivsd            (%rax), %xmm1, %xmm2
+
+vdivss            %xmm0, %xmm1, %xmm2
+vdivss            (%rax), %xmm1, %xmm2
+
+vdppd             $22, %xmm0, %xmm1, %xmm2
+vdppd             $22, (%rax), %xmm1, %xmm2
+
+vdpps             $22, %xmm0, %xmm1, %xmm2
+vdpps             $22, (%rax), %xmm1, %xmm2
+
+vdpps             $22, %ymm0, %ymm1, %ymm2
+vdpps             $22, (%rax), %ymm1, %ymm2
+
+vextractf128      $1, %ymm0, %xmm2
+vextractf128      $1, %ymm0, (%rax)
+
+vextractps        $1, %xmm0, %rcx
+vextractps        $1, %xmm0, (%rax)
+
+vhaddpd           %xmm0, %xmm1, %xmm2
+vhaddpd           (%rax), %xmm1, %xmm2
+
+vhaddpd           %ymm0, %ymm1, %ymm2
+vhaddpd           (%rax), %ymm1, %ymm2
+
+vhaddps           %xmm0, %xmm1, %xmm2
+vhaddps           (%rax), %xmm1, %xmm2
+
+vhaddps           %ymm0, %ymm1, %ymm2
+vhaddps           (%rax), %ymm1, %ymm2
+
+vhsubpd           %xmm0, %xmm1, %xmm2
+vhsubpd           (%rax), %xmm1, %xmm2
+
+vhsubpd           %ymm0, %ymm1, %ymm2
+vhsubpd           (%rax), %ymm1, %ymm2
+
+vhsubps           %xmm0, %xmm1, %xmm2
+vhsubps           (%rax), %xmm1, %xmm2
+
+vhsubps           %ymm0, %ymm1, %ymm2
+vhsubps           (%rax), %ymm1, %ymm2
+
+vinsertf128       $1, %xmm0, %ymm1, %ymm2
+vinsertf128       $1, (%rax), %ymm1, %ymm2
+
+vinsertps         $1, %xmm0, %xmm1, %xmm2
+vinsertps         $1, (%rax), %xmm1, %xmm2
+
+vlddqu            (%rax), %xmm2
+vlddqu            (%rax), %ymm2
+
+vldmxcsr          (%rax)
+
+vmaskmovdqu       %xmm0, %xmm1
+
+vmaskmovpd        (%rax), %xmm0, %xmm2
+vmaskmovpd        (%rax), %ymm0, %ymm2
+
+vmaskmovpd        %xmm0, %xmm1, (%rax)
+vmaskmovpd        %ymm0, %ymm1, (%rax)
+
+vmaskmovps        (%rax), %xmm0, %xmm2
+vmaskmovps        (%rax), %ymm0, %ymm2
+
+vmaskmovps        %xmm0, %xmm1, (%rax)
+vmaskmovps        %ymm0, %ymm1, (%rax)
+
+vmaxpd            %xmm0, %xmm1, %xmm2
+vmaxpd            (%rax), %xmm1, %xmm2
+
+vmaxpd            %ymm0, %ymm1, %ymm2
+vmaxpd            (%rax), %ymm1, %ymm2
+
+vmaxps            %xmm0, %xmm1, %xmm2
+vmaxps            (%rax), %xmm1, %xmm2
+
+vmaxps            %ymm0, %ymm1, %ymm2
+vmaxps            (%rax), %ymm1, %ymm2
+
+vmaxsd            %xmm0, %xmm1, %xmm2
+vmaxsd            (%rax), %xmm1, %xmm2
+
+vmaxss            %xmm0, %xmm1, %xmm2
+vmaxss            (%rax), %xmm1, %xmm2
+
+vminpd            %xmm0, %xmm1, %xmm2
+vminpd            (%rax), %xmm1, %xmm2
+
+vminpd            %ymm0, %ymm1, %ymm2
+vminpd            (%rax), %ymm1, %ymm2
+
+vminps            %xmm0, %xmm1, %xmm2
+vminps            (%rax), %xmm1, %xmm2
+
+vminps            %ymm0, %ymm1, %ymm2
+vminps            (%rax), %ymm1, %ymm2
+
+vminsd            %xmm0, %xmm1, %xmm2
+vminsd            (%rax), %xmm1, %xmm2
+
+vminss            %xmm0, %xmm1, %xmm2
+vminss            (%rax), %xmm1, %xmm2
+
+vmovapd           %xmm0, %xmm2
+vmovapd           %xmm0, (%rax)
+vmovapd           (%rax), %xmm2
+
+vmovapd           %ymm0, %ymm2
+vmovapd           %ymm0, (%rax)
+vmovapd           (%rax), %ymm2
+
+vmovaps           %xmm0, %xmm2
+vmovaps           %xmm0, (%rax)
+vmovaps           (%rax), %xmm2
+
+vmovaps           %ymm0, %ymm2
+vmovaps           %ymm0, (%rax)
+vmovaps           (%rax), %ymm2
+
+vmovd             %eax, %xmm2
+vmovd             (%rax), %xmm2
+
+vmovd             %xmm0, %ecx
+vmovd             %xmm0, (%rax)
+
+vmovddup          %xmm0, %xmm2
+vmovddup          (%rax), %xmm2
+
+vmovddup          %ymm0, %ymm2
+vmovddup          (%rax), %ymm2
+
+vmovdqa           %xmm0, %xmm2
+vmovdqa           %xmm0, (%rax)
+vmovdqa           (%rax), %xmm2
+
+vmovdqa           %ymm0, %ymm2
+vmovdqa           %ymm0, (%rax)
+vmovdqa           (%rax), %ymm2
+
+vmovdqu           %xmm0, %xmm2
+vmovdqu           %xmm0, (%rax)
+vmovdqu           (%rax), %xmm2
+
+vmovdqu           %ymm0, %ymm2
+vmovdqu           %ymm0, (%rax)
+vmovdqu           (%rax), %ymm2
+
+vmovhlps          %xmm0, %xmm1, %xmm2
+vmovlhps          %xmm0, %xmm1, %xmm2
+
+vmovhpd           %xmm0, (%rax)
+vmovhpd           (%rax), %xmm1, %xmm2
+
+vmovhps           %xmm0, (%rax)
+vmovhps           (%rax), %xmm1, %xmm2
+
+vmovlpd           %xmm0, (%rax)
+vmovlpd           (%rax), %xmm1, %xmm2
+
+vmovlps           %xmm0, (%rax)
+vmovlps           (%rax), %xmm1, %xmm2
+
+vmovmskpd         %xmm0, %rcx
+vmovmskpd         %ymm0, %rcx
+
+vmovmskps         %xmm0, %rcx
+vmovmskps         %ymm0, %rcx
+
+vmovntdq          %xmm0, (%rax)
+vmovntdq          %ymm0, (%rax)
+
+vmovntdqa         (%rax), %xmm2
+vmovntdqa         (%rax), %ymm2
+
+vmovntpd          %xmm0, (%rax)
+vmovntpd          %ymm0, (%rax)
+
+vmovntps          %xmm0, (%rax)
+vmovntps          %ymm0, (%rax)
+
+vmovq             %xmm0, %xmm2
+
+vmovq             %rax, %xmm2
+vmovq             (%rax), %xmm2
+
+vmovq             %xmm0, %rcx
+vmovq             %xmm0, (%rax)
+
+vmovsd            %xmm0, %xmm1, %xmm2
+vmovsd            %xmm0, (%rax)
+vmovsd            (%rax), %xmm2
+
+vmovshdup         %xmm0, %xmm2
+vmovshdup         (%rax), %xmm2
+
+vmovshdup         %ymm0, %ymm2
+vmovshdup         (%rax), %ymm2
+
+vmovsldup         %xmm0, %xmm2
+vmovsldup         (%rax), %xmm2
+
+vmovsldup         %ymm0, %ymm2
+vmovsldup         (%rax), %ymm2
+
+vmovss            %xmm0, %xmm1, %xmm2
+vmovss            %xmm0, (%rax)
+vmovss            (%rax), %xmm2
+
+vmovupd           %xmm0, %xmm2
+vmovupd           %xmm0, (%rax)
+vmovupd           (%rax), %xmm2
+
+vmovupd           %ymm0, %ymm2
+vmovupd           %ymm0, (%rax)
+vmovupd           (%rax), %ymm2
+
+vmovups           %xmm0, %xmm2
+vmovups           %xmm0, (%rax)
+vmovups           (%rax), %xmm2
+
+vmovups           %ymm0, %ymm2
+vmovups           %ymm0, (%rax)
+vmovups           (%rax), %ymm2
+
+vmpsadbw          $1, %xmm0, %xmm1, %xmm2
+vmpsadbw          $1, (%rax), %xmm1, %xmm2
+
+vmulpd            %xmm0, %xmm1, %xmm2
+vmulpd            (%rax), %xmm1, %xmm2
+
+vmulpd            %ymm0, %ymm1, %ymm2
+vmulpd            (%rax), %ymm1, %ymm2
+
+vmulps            %xmm0, %xmm1, %xmm2
+vmulps            (%rax), %xmm1, %xmm2
+
+vmulps            %ymm0, %ymm1, %ymm2
+vmulps            (%rax), %ymm1, %ymm2
+
+vmulsd            %xmm0, %xmm1, %xmm2
+vmulsd            (%rax), %xmm1, %xmm2
+
+vmulss            %xmm0, %xmm1, %xmm2
+vmulss            (%rax), %xmm1, %xmm2
+
+vorpd             %xmm0, %xmm1, %xmm2
+vorpd             (%rax), %xmm1, %xmm2
+
+vorpd             %ymm0, %ymm1, %ymm2
+vorpd             (%rax), %ymm1, %ymm2
+
+vorps             %xmm0, %xmm1, %xmm2
+vorps             (%rax), %xmm1, %xmm2
+
+vorps             %ymm0, %ymm1, %ymm2
+vorps             (%rax), %ymm1, %ymm2
+
+vpabsb            %xmm0, %xmm2
+vpabsb            (%rax), %xmm2
+
+vpabsd            %xmm0, %xmm2
+vpabsd            (%rax), %xmm2
+
+vpabsw            %xmm0, %xmm2
+vpabsw            (%rax), %xmm2
+
+vpackssdw         %xmm0, %xmm1, %xmm2
+vpackssdw         (%rax), %xmm1, %xmm2
+
+vpacksswb         %xmm0, %xmm1, %xmm2
+vpacksswb         (%rax), %xmm1, %xmm2
+
+vpackusdw         %xmm0, %xmm1, %xmm2
+vpackusdw         (%rax), %xmm1, %xmm2
+
+vpackuswb         %xmm0, %xmm1, %xmm2
+vpackuswb         (%rax), %xmm1, %xmm2
+
+vpaddb            %xmm0, %xmm1, %xmm2
+vpaddb            (%rax), %xmm1, %xmm2
+
+vpaddd            %xmm0, %xmm1, %xmm2
+vpaddd            (%rax), %xmm1, %xmm2
+
+vpaddq            %xmm0, %xmm1, %xmm2
+vpaddq            (%rax), %xmm1, %xmm2
+
+vpaddsb           %xmm0, %xmm1, %xmm2
+vpaddsb           (%rax), %xmm1, %xmm2
+
+vpaddsw           %xmm0, %xmm1, %xmm2
+vpaddsw           (%rax), %xmm1, %xmm2
+
+vpaddusb          %xmm0, %xmm1, %xmm2
+vpaddusb          (%rax), %xmm1, %xmm2
+
+vpaddusw          %xmm0, %xmm1, %xmm2
+vpaddusw          (%rax), %xmm1, %xmm2
+
+vpaddw            %xmm0, %xmm1, %xmm2
+vpaddw            (%rax), %xmm1, %xmm2
+
+vpalignr          $1, %xmm0, %xmm1, %xmm2
+vpalignr          $1, (%rax), %xmm1, %xmm2
+
+vpand             %xmm0, %xmm1, %xmm2
+vpand             (%rax), %xmm1, %xmm2
+
+vpandn            %xmm0, %xmm1, %xmm2
+vpandn            (%rax), %xmm1, %xmm2
+
+vpavgb            %xmm0, %xmm1, %xmm2
+vpavgb            (%rax), %xmm1, %xmm2
+
+vpavgw            %xmm0, %xmm1, %xmm2
+vpavgw            (%rax), %xmm1, %xmm2
+
+vpblendvb         %xmm3, %xmm0, %xmm1, %xmm2
+vpblendvb         %xmm3, (%rax), %xmm1, %xmm2
+
+vpblendw          $11, %xmm0, %xmm1, %xmm2
+vpblendw          $11, (%rax), %xmm1, %xmm2
+
+vpclmulqdq        $11, %xmm0, %xmm1, %xmm2
+vpclmulqdq        $11, (%rax), %xmm1, %xmm2
+
+vpcmpeqb          %xmm0, %xmm1, %xmm2
+vpcmpeqb          (%rax), %xmm1, %xmm2
+
+vpcmpeqd          %xmm0, %xmm1, %xmm2
+vpcmpeqd          (%rax), %xmm1, %xmm2
+
+vpcmpeqq          %xmm0, %xmm1, %xmm2
+vpcmpeqq          (%rax), %xmm1, %xmm2
+
+vpcmpeqw          %xmm0, %xmm1, %xmm2
+vpcmpeqw          (%rax), %xmm1, %xmm2
+
+vpcmpestri        $1, %xmm0, %xmm2
+vpcmpestri        $1, (%rax), %xmm2
+
+vpcmpestrm        $1, %xmm0, %xmm2
+vpcmpestrm        $1, (%rax), %xmm2
+
+vpcmpgtb          %xmm0, %xmm1, %xmm2
+vpcmpgtb          (%rax), %xmm1, %xmm2
+
+vpcmpgtd          %xmm0, %xmm1, %xmm2
+vpcmpgtd          (%rax), %xmm1, %xmm2
+
+vpcmpgtq          %xmm0, %xmm1, %xmm2
+vpcmpgtq          (%rax), %xmm1, %xmm2
+
+vpcmpgtw          %xmm0, %xmm1, %xmm2
+vpcmpgtw          (%rax), %xmm1, %xmm2
+
+vpcmpistri        $1, %xmm0, %xmm2
+vpcmpistri        $1, (%rax), %xmm2
+
+vpcmpistrm        $1, %xmm0, %xmm2
+vpcmpistrm        $1, (%rax), %xmm2
+
+vperm2f128        $1, %ymm0, %ymm1, %ymm2
+vperm2f128        $1, (%rax), %ymm1, %ymm2
+
+vpermilpd         $1, %xmm0, %xmm2
+vpermilpd         $1, (%rax), %xmm2
+vpermilpd         %xmm0, %xmm1, %xmm2
+vpermilpd         (%rax), %xmm1, %xmm2
+
+vpermilpd         $1, %ymm0, %ymm2
+vpermilpd         $1, (%rax), %ymm2
+vpermilpd         %ymm0, %ymm1, %ymm2
+vpermilpd         (%rax), %ymm1, %ymm2
+
+vpermilps         $1, %xmm0, %xmm2
+vpermilps         $1, (%rax), %xmm2
+vpermilps         %xmm0, %xmm1, %xmm2
+vpermilps         (%rax), %xmm1, %xmm2
+
+vpermilps         $1, %ymm0, %ymm2
+vpermilps         $1, (%rax), %ymm2
+vpermilps         %ymm0, %ymm1, %ymm2
+vpermilps         (%rax), %ymm1, %ymm2
+
+vpextrb           $1, %xmm0, %ecx
+vpextrb           $1, %xmm0, (%rax)
+
+vpextrd           $1, %xmm0, %ecx
+vpextrd           $1, %xmm0, (%rax)
+
+vpextrq           $1, %xmm0, %rcx
+vpextrq           $1, %xmm0, (%rax)
+
+vpextrw           $1, %xmm0, %ecx
+vpextrw           $1, %xmm0, (%rax)
+
+vphaddd           %xmm0, %xmm1, %xmm2
+vphaddd           (%rax), %xmm1, %xmm2
+
+vphaddsw          %xmm0, %xmm1, %xmm2
+vphaddsw          (%rax), %xmm1, %xmm2
+
+vphaddw           %xmm0, %xmm1, %xmm2
+vphaddw           (%rax), %xmm1, %xmm2
+
+vphminposuw       %xmm0, %xmm2
+vphminposuw       (%rax), %xmm2
+
+vphsubd           %xmm0, %xmm1, %xmm2
+vphsubd           (%rax), %xmm1, %xmm2
+
+vphsubsw          %xmm0, %xmm1, %xmm2
+vphsubsw          (%rax), %xmm1, %xmm2
+
+vphsubw           %xmm0, %xmm1, %xmm2
+vphsubw           (%rax), %xmm1, %xmm2
+
+vpinsrb           $1, %eax, %xmm1, %xmm2
+vpinsrb           $1, (%rax), %xmm1, %xmm2
+
+vpinsrd           $1, %eax, %xmm1, %xmm2
+vpinsrd           $1, (%rax), %xmm1, %xmm2
+
+vpinsrq           $1, %rax, %xmm1, %xmm2
+vpinsrq           $1, (%rax), %xmm1, %xmm2
+
+vpinsrw           $1, %eax, %xmm1, %xmm2
+vpinsrw           $1, (%rax), %xmm1, %xmm2
+
+vpmaddubsw        %xmm0, %xmm1, %xmm2
+vpmaddubsw        (%rax), %xmm1, %xmm2
+
+vpmaddwd          %xmm0, %xmm1, %xmm2
+vpmaddwd          (%rax), %xmm1, %xmm2
+
+vpmaxsb           %xmm0, %xmm1, %xmm2
+vpmaxsb           (%rax), %xmm1, %xmm2
+
+vpmaxsd           %xmm0, %xmm1, %xmm2
+vpmaxsd           (%rax), %xmm1, %xmm2
+
+vpmaxsw           %xmm0, %xmm1, %xmm2
+vpmaxsw           (%rax), %xmm1, %xmm2
+
+vpmaxub           %xmm0, %xmm1, %xmm2
+vpmaxub           (%rax), %xmm1, %xmm2
+
+vpmaxud           %xmm0, %xmm1, %xmm2
+vpmaxud           (%rax), %xmm1, %xmm2
+
+vpmaxuw           %xmm0, %xmm1, %xmm2
+vpmaxuw           (%rax), %xmm1, %xmm2
+
+vpminsb           %xmm0, %xmm1, %xmm2
+vpminsb           (%rax), %xmm1, %xmm2
+
+vpminsd           %xmm0, %xmm1, %xmm2
+vpminsd           (%rax), %xmm1, %xmm2
+
+vpminsw           %xmm0, %xmm1, %xmm2
+vpminsw           (%rax), %xmm1, %xmm2
+
+vpminub           %xmm0, %xmm1, %xmm2
+vpminub           (%rax), %xmm1, %xmm2
+
+vpminud           %xmm0, %xmm1, %xmm2
+vpminud           (%rax), %xmm1, %xmm2
+
+vpminuw           %xmm0, %xmm1, %xmm2
+vpminuw           (%rax), %xmm1, %xmm2
+
+vpmovmskb         %xmm0, %rcx
+
+vpmovsxbd         %xmm0, %xmm2
+vpmovsxbd         (%rax), %xmm2
+
+vpmovsxbq         %xmm0, %xmm2
+vpmovsxbq         (%rax), %xmm2
+
+vpmovsxbw         %xmm0, %xmm2
+vpmovsxbw         (%rax), %xmm2
+
+vpmovsxdq         %xmm0, %xmm2
+vpmovsxdq         (%rax), %xmm2
+
+vpmovsxwd         %xmm0, %xmm2
+vpmovsxwd         (%rax), %xmm2
+
+vpmovsxwq         %xmm0, %xmm2
+vpmovsxwq         (%rax), %xmm2
+
+vpmovzxbd         %xmm0, %xmm2
+vpmovzxbd         (%rax), %xmm2
+
+vpmovzxbq         %xmm0, %xmm2
+vpmovzxbq         (%rax), %xmm2
+
+vpmovzxbw         %xmm0, %xmm2
+vpmovzxbw         (%rax), %xmm2
+
+vpmovzxdq         %xmm0, %xmm2
+vpmovzxdq         (%rax), %xmm2
+
+vpmovzxwd         %xmm0, %xmm2
+vpmovzxwd         (%rax), %xmm2
+
+vpmovzxwq         %xmm0, %xmm2
+vpmovzxwq         (%rax), %xmm2
+
+vpmuldq           %xmm0, %xmm1, %xmm2
+vpmuldq           (%rax), %xmm1, %xmm2
+
+vpmulhrsw         %xmm0, %xmm1, %xmm2
+vpmulhrsw         (%rax), %xmm1, %xmm2
+
+vpmulhuw          %xmm0, %xmm1, %xmm2
+vpmulhuw          (%rax), %xmm1, %xmm2
+
+vpmulhw           %xmm0, %xmm1, %xmm2
+vpmulhw           (%rax), %xmm1, %xmm2
+
+vpmulld           %xmm0, %xmm1, %xmm2
+vpmulld           (%rax), %xmm1, %xmm2
+
+vpmullw           %xmm0, %xmm1, %xmm2
+vpmullw           (%rax), %xmm1, %xmm2
+
+vpmuludq          %xmm0, %xmm1, %xmm2
+vpmuludq          (%rax), %xmm1, %xmm2
+
+vpor              %xmm0, %xmm1, %xmm2
+vpor              (%rax), %xmm1, %xmm2
+
+vpsadbw           %xmm0, %xmm1, %xmm2
+vpsadbw           (%rax), %xmm1, %xmm2
+
+vpshufb           %xmm0, %xmm1, %xmm2
+vpshufb           (%rax), %xmm1, %xmm2
+
+vpshufd           $1, %xmm0, %xmm2
+vpshufd           $1, (%rax), %xmm2
+
+vpshufhw          $1, %xmm0, %xmm2
+vpshufhw          $1, (%rax), %xmm2
+
+vpshuflw          $1, %xmm0, %xmm2
+vpshuflw          $1, (%rax), %xmm2
+
+vpsignb           %xmm0, %xmm1, %xmm2
+vpsignb           (%rax), %xmm1, %xmm2
+
+vpsignd           %xmm0, %xmm1, %xmm2
+vpsignd           (%rax), %xmm1, %xmm2
+
+vpsignw           %xmm0, %xmm1, %xmm2
+vpsignw           (%rax), %xmm1, %xmm2
+
+vpslld            $1, %xmm0, %xmm2
+vpslld            %xmm0, %xmm1, %xmm2
+vpslld            (%rax), %xmm1, %xmm2
+
+vpslldq           $1, %xmm1, %xmm2
+
+vpsllq            $1, %xmm0, %xmm2
+vpsllq            %xmm0, %xmm1, %xmm2
+vpsllq            (%rax), %xmm1, %xmm2
+
+vpsllw            $1, %xmm0, %xmm2
+vpsllw            %xmm0, %xmm1, %xmm2
+vpsllw            (%rax), %xmm1, %xmm2
+
+vpsrad            $1, %xmm0, %xmm2
+vpsrad            %xmm0, %xmm1, %xmm2
+vpsrad            (%rax), %xmm1, %xmm2
+
+vpsraw            $1, %xmm0, %xmm2
+vpsraw            %xmm0, %xmm1, %xmm2
+vpsraw            (%rax), %xmm1, %xmm2
+
+vpsrld            $1, %xmm0, %xmm2
+vpsrld            %xmm0, %xmm1, %xmm2
+vpsrld            (%rax), %xmm1, %xmm2
+
+vpsrldq           $1, %xmm1, %xmm2
+
+vpsrlq            $1, %xmm0, %xmm2
+vpsrlq            %xmm0, %xmm1, %xmm2
+vpsrlq            (%rax), %xmm1, %xmm2
+
+vpsrlw            $1, %xmm0, %xmm2
+vpsrlw            %xmm0, %xmm1, %xmm2
+vpsrlw            (%rax), %xmm1, %xmm2
+
+vpsubb            %xmm0, %xmm1, %xmm2
+vpsubb            (%rax), %xmm1, %xmm2
+
+vpsubd            %xmm0, %xmm1, %xmm2
+vpsubd            (%rax), %xmm1, %xmm2
+
+vpsubq            %xmm0, %xmm1, %xmm2
+vpsubq            (%rax), %xmm1, %xmm2
+
+vpsubsb           %xmm0, %xmm1, %xmm2
+vpsubsb           (%rax), %xmm1, %xmm2
+
+vpsubsw           %xmm0, %xmm1, %xmm2
+vpsubsw           (%rax), %xmm1, %xmm2
+
+vpsubusb          %xmm0, %xmm1, %xmm2
+vpsubusb          (%rax), %xmm1, %xmm2
+
+vpsubusw          %xmm0, %xmm1, %xmm2
+vpsubusw          (%rax), %xmm1, %xmm2
+
+vpsubw            %xmm0, %xmm1, %xmm2
+vpsubw            (%rax), %xmm1, %xmm2
+
+vptest            %xmm0, %xmm1
+vptest            (%rax), %xmm1
+
+vptest            %ymm0, %ymm1
+vptest            (%rax), %ymm1
+
+vpunpckhbw        %xmm0, %xmm1, %xmm2
+vpunpckhbw        (%rax), %xmm1, %xmm2
+
+vpunpckhdq        %xmm0, %xmm1, %xmm2
+vpunpckhdq        (%rax), %xmm1, %xmm2
+
+vpunpckhqdq       %xmm0, %xmm1, %xmm2
+vpunpckhqdq       (%rax), %xmm1, %xmm2
+
+vpunpckhwd        %xmm0, %xmm1, %xmm2
+vpunpckhwd        (%rax), %xmm1, %xmm2
+
+vpunpcklbw        %xmm0, %xmm1, %xmm2
+vpunpcklbw        (%rax), %xmm1, %xmm2
+
+vpunpckldq        %xmm0, %xmm1, %xmm2
+vpunpckldq        (%rax), %xmm1, %xmm2
+
+vpunpcklqdq       %xmm0, %xmm1, %xmm2
+vpunpcklqdq       (%rax), %xmm1, %xmm2
+
+vpunpcklwd        %xmm0, %xmm1, %xmm2
+vpunpcklwd        (%rax), %xmm1, %xmm2
+
+vpxor             %xmm0, %xmm1, %xmm2
+vpxor             (%rax), %xmm1, %xmm2
+
+vrcpps            %xmm0, %xmm2
+vrcpps            (%rax), %xmm2
+
+vrcpps            %ymm0, %ymm2
+vrcpps            (%rax), %ymm2
+
+vrcpss            %xmm0, %xmm1, %xmm2
+vrcpss            (%rax), %xmm1, %xmm2
+
+vroundpd          $1, %xmm0, %xmm2
+vroundpd          $1, (%rax), %xmm2
+
+vroundpd          $1, %ymm0, %ymm2
+vroundpd          $1, (%rax), %ymm2
+
+vroundps          $1, %xmm0, %xmm2
+vroundps          $1, (%rax), %xmm2
+
+vroundps          $1, %ymm0, %ymm2
+vroundps          $1, (%rax), %ymm2
+
+vroundsd          $1, %xmm0, %xmm1, %xmm2
+vroundsd          $1, (%rax), %xmm1, %xmm2
+
+vroundss          $1, %xmm0, %xmm1, %xmm2
+vroundss          $1, (%rax), %xmm1, %xmm2
+
+vrsqrtps          %xmm0, %xmm2
+vrsqrtps          (%rax), %xmm2
+
+vrsqrtps          %ymm0, %ymm2
+vrsqrtps          (%rax), %ymm2
+
+vrsqrtss          %xmm0, %xmm1, %xmm2
+vrsqrtss          (%rax), %xmm1, %xmm2
+
+vshufpd           $1, %xmm0, %xmm1, %xmm2
+vshufpd           $1, (%rax), %xmm1, %xmm2
+
+vshufpd           $1, %ymm0, %ymm1, %ymm2
+vshufpd           $1, (%rax), %ymm1, %ymm2
+
+vshufps           $1, %xmm0, %xmm1, %xmm2
+vshufps           $1, (%rax), %xmm1, %xmm2
+
+vshufps           $1, %ymm0, %ymm1, %ymm2
+vshufps           $1, (%rax), %ymm1, %ymm2
+
+vsqrtpd           %xmm0, %xmm2
+vsqrtpd           (%rax), %xmm2
+
+vsqrtpd           %ymm0, %ymm2
+vsqrtpd           (%rax), %ymm2
+
+vsqrtps           %xmm0, %xmm2
+vsqrtps           (%rax), %xmm2
+
+vsqrtps           %ymm0, %ymm2
+vsqrtps           (%rax), %ymm2
+
+vsqrtsd           %xmm0, %xmm1, %xmm2
+vsqrtsd           (%rax), %xmm1, %xmm2
+
+vsqrtss           %xmm0, %xmm1, %xmm2
+vsqrtss           (%rax), %xmm1, %xmm2
+
+vstmxcsr          (%rax)
+
+vsubpd            %xmm0, %xmm1, %xmm2
+vsubpd            (%rax), %xmm1, %xmm2
+
+vsubpd            %ymm0, %ymm1, %ymm2
+vsubpd            (%rax), %ymm1, %ymm2
+
+vsubps            %xmm0, %xmm1, %xmm2
+vsubps            (%rax), %xmm1, %xmm2
+
+vsubps            %ymm0, %ymm1, %ymm2
+vsubps            (%rax), %ymm1, %ymm2
+
+vsubsd            %xmm0, %xmm1, %xmm2
+vsubsd            (%rax), %xmm1, %xmm2
+
+vsubss            %xmm0, %xmm1, %xmm2
+vsubss            (%rax), %xmm1, %xmm2
+
+vtestpd          %xmm0, %xmm1
+vtestpd          (%rax), %xmm1
+
+vtestpd          %ymm0, %ymm1
+vtestpd          (%rax), %ymm1
+
+vtestps          %xmm0, %xmm1
+vtestps          (%rax), %xmm1
+
+vtestps          %ymm0, %ymm1
+vtestps          (%rax), %ymm1
+
+vucomisd          %xmm0, %xmm1
+vucomisd          (%rax), %xmm1
+
+vucomiss          %xmm0, %xmm1
+vucomiss          (%rax), %xmm1
+
+vunpckhpd         %xmm0, %xmm1, %xmm2
+vunpckhpd         (%rax), %xmm1, %xmm2
+
+vunpckhpd         %ymm0, %ymm1, %ymm2
+vunpckhpd         (%rax), %ymm1, %ymm2
+
+vunpckhps         %xmm0, %xmm1, %xmm2
+vunpckhps         (%rax), %xmm1, %xmm2
+
+vunpckhps         %ymm0, %ymm1, %ymm2
+vunpckhps         (%rax), %ymm1, %ymm2
+
+vunpcklpd         %xmm0, %xmm1, %xmm2
+vunpcklpd         (%rax), %xmm1, %xmm2
+
+vunpcklpd         %ymm0, %ymm1, %ymm2
+vunpcklpd         (%rax), %ymm1, %ymm2
+
+vunpcklps         %xmm0, %xmm1, %xmm2
+vunpcklps         (%rax), %xmm1, %xmm2
+
+vunpcklps         %ymm0, %ymm1, %ymm2
+vunpcklps         (%rax), %ymm1, %ymm2
+
+vxorpd            %xmm0, %xmm1, %xmm2
+vxorpd            (%rax), %xmm1, %xmm2
+
+vxorpd            %ymm0, %ymm1, %ymm2
+vxorpd            (%rax), %ymm1, %ymm2
+
+vxorps            %xmm0, %xmm1, %xmm2
+vxorps            (%rax), %xmm1, %xmm2
+
+vxorps            %ymm0, %ymm1, %ymm2
+vxorps            (%rax), %ymm1, %ymm2
+
+vzeroall
+vzeroupper
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        vaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vaddsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.33                        vaddss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vaddsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vaddsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vaddsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vaddsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vaddsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vaddsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vaddsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     2.00                        vaesdec	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   vaesdec	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     2.00                        vaesdeclast	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   vaesdeclast	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     2.00                        vaesenc	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   vaesenc	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     2.00                        vaesenclast	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      11    2.33    *                   vaesenclast	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vandnpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vandnpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vandnps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vandnps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandnps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vandnps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vandpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vandpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vandpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vandps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vandps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vandps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vblendpd	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vblendpd	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vblendpd	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vblendpd	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vblendps	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vblendps	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vblendps	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vblendps	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vblendvpd	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vblendvpd	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     0.75                        vblendvpd	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    0.75    *                   vblendvpd	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     1.00                        vblendvps	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vblendvps	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     0.75                        vblendvps	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    0.75    *                   vblendvps	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      7     2.33    *                   vbroadcastf128	(%rax), %ymm2
+# CHECK-NEXT:  1      7     2.33    *                   vbroadcastsd	(%rax), %ymm2
+# CHECK-NEXT:  1      6     2.00    *                   vbroadcastss	(%rax), %xmm2
+# CHECK-NEXT:  1      7     2.33    *                   vbroadcastss	(%rax), %ymm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   vcmpeqpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    2.33    *                   vcmpeqpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   vcmpeqps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    2.33    *                   vcmpeqps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   vcmpeqsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcmpeqss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   vcmpeqss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vcomisd	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   vcomisd	(%rax), %xmm1
+# CHECK-NEXT:  1      3     1.50                        vcomiss	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   vcomiss	(%rax), %xmm1
+# CHECK-NEXT:  2      5     0.50                        vcvtdq2pd	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %xmm2
+# CHECK-NEXT:  2      7     2.00                        vcvtdq2pd	%xmm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vcvtdq2pd	(%rax), %ymm2
+# CHECK-NEXT:  1      4     2.00                        vcvtdq2ps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtdq2ps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50    *                   vcvtdq2ps	(%rax), %ymm2
+# CHECK-NEXT:  2      5     0.50                        vcvtpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   vcvtpd2dqx	(%rax), %xmm2
+# CHECK-NEXT:  2      7     0.50                        vcvtpd2dq	%ymm0, %xmm2
+# CHECK-NEXT:  3      15    0.50    *                   vcvtpd2dqy	(%rax), %xmm2
+# CHECK-NEXT:  2      5     0.50                        vcvtpd2ps	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   vcvtpd2psx	(%rax), %xmm2
+# CHECK-NEXT:  2      7     0.50                        vcvtpd2ps	%ymm0, %xmm2
+# CHECK-NEXT:  3      15    0.50    *                   vcvtpd2psy	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcvtps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtps2dq	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcvtps2dq	%ymm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vcvtps2dq	(%rax), %ymm2
+# CHECK-NEXT:  2      5     2.00                        vcvtps2pd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtps2pd	(%rax), %xmm2
+# CHECK-NEXT:  2      7     2.00                        vcvtps2pd	%xmm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vcvtps2pd	(%rax), %ymm2
+# CHECK-NEXT:  2      7     3.50                        vcvtsd2si	%xmm0, %ecx
+# CHECK-NEXT:  2      7     3.50                        vcvtsd2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   vcvtsd2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   vcvtsd2si	(%rax), %rcx
+# CHECK-NEXT:  2      5     0.50                        vcvtsd2ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      11    2.00    *                   vcvtsd2ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     2.00                        vcvtsi2sd	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  2      8     2.00                        vcvtsi2sd	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtsi2sdl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtsi2sdq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      8     2.00                        vcvtsi2ss	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  3      9     2.00                        vcvtsi2ss	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtsi2ssl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  3      11    2.00    *                   vcvtsi2ssq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      5     2.00                        vcvtss2sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvtss2sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      7     3.50                        vcvtss2si	%xmm0, %ecx
+# CHECK-NEXT:  3      8     3.50                        vcvtss2si	%xmm0, %rcx
+# CHECK-NEXT:  2      11    3.50    *                   vcvtss2si	(%rax), %ecx
+# CHECK-NEXT:  2      11    3.50    *                   vcvtss2si	(%rax), %rcx
+# CHECK-NEXT:  2      5     0.50                        vcvttpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   vcvttpd2dqx	(%rax), %xmm2
+# CHECK-NEXT:  2      7     0.50                        vcvttpd2dq	%ymm0, %xmm2
+# CHECK-NEXT:  3      15    0.50    *                   vcvttpd2dqy	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcvttps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vcvttps2dq	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vcvttps2dq	%ymm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vcvttps2dq	(%rax), %ymm2
+# CHECK-NEXT:  2      7     3.50                        vcvttsd2si	%xmm0, %ecx
+# CHECK-NEXT:  2      7     3.50                        vcvttsd2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   vcvttsd2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   vcvttsd2si	(%rax), %rcx
+# CHECK-NEXT:  2      7     3.50                        vcvttss2si	%xmm0, %ecx
+# CHECK-NEXT:  3      8     3.50                        vcvttss2si	%xmm0, %rcx
+# CHECK-NEXT:  2      11    3.50    *                   vcvttss2si	(%rax), %ecx
+# CHECK-NEXT:  2      11    3.50    *                   vcvttss2si	(%rax), %rcx
+# CHECK-NEXT:  1      10    5.00                        vdivpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   vdivpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      10    5.00                        vdivpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      17    5.00    *                   vdivpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      7     3.50                        vdivps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      13    3.50    *                   vdivps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      7     3.50                        vdivps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      14    3.50    *                   vdivps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      10    5.00                        vdivsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      20    0.50    *                   vdivsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      7     3.50                        vdivss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      13    3.50    *                   vdivss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      8     1.00                        vdppd	$22, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      13    1.00    *                   vdppd	$22, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  5      11    1.50                        vdpps	$22, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  6      18    1.50    *                   vdpps	$22, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  5      12    1.50                        vdpps	$22, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  6      19    1.50    *                   vdpps	$22, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vextractf128	$1, %ymm0, %xmm2
+# CHECK-NEXT:  2      1     0.50           *            vextractf128	$1, %ymm0, (%rax)
+# CHECK-NEXT:  2      4     0.50                        vextractps	$1, %xmm0, %ecx
+# CHECK-NEXT:  3      12    0.50           *            vextractps	$1, %xmm0, (%rax)
+# CHECK-NEXT:  3      5     0.67                        vhaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   vhaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      4     2.50                        vhaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      11    2.50    *                   vhaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      5     0.67                        vhaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   vhaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      4     2.50                        vhaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      11    2.50    *                   vhaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      5     0.67                        vhsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   vhsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      4     2.50                        vhsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      11    2.50    *                   vhsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      5     0.67                        vhsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   vhsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      4     2.50                        vhsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      11    2.50    *                   vhsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vinsertf128	$1, %xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vinsertf128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vinsertps	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vinsertps	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   vlddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      7     0.33    *                   vlddqu	(%rax), %ymm2
+# CHECK-NEXT:  3      7     0.50    *      *      U     vldmxcsr	(%rax)
+# CHECK-NEXT:  2      1     0.50    *      *      U     vmaskmovdqu	%xmm0, %xmm1
+# CHECK-NEXT:  2      8     0.33    *                   vmaskmovpd	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      9     0.33    *                   vmaskmovpd	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  3      14    0.50    *      *            vmaskmovpd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  3      14    0.50    *      *            vmaskmovpd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  2      8     0.33    *                   vmaskmovps	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      9     0.33    *                   vmaskmovps	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  3      14    0.50    *      *            vmaskmovps	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  3      14    0.50    *      *            vmaskmovps	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  1      4     2.00                        vmaxpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vmaxpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vmaxpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vmaxpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vmaxps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vmaxps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vmaxps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vmaxps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vmaxsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vmaxsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vmaxss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vmaxss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vminpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vminpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vminpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vminpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vminps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vminps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vminps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vminps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vminsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vminsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vminss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vminss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovapd	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovapd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   vmovapd	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovapd	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovapd	%ymm0, (%rax)
+# CHECK-NEXT:  1      8     2.33    *                   vmovapd	(%rax), %ymm2
+# CHECK-NEXT:  0      1     0.00                        vmovaps	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovaps	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   vmovaps	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovaps	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovaps	%ymm0, (%rax)
+# CHECK-NEXT:  1      8     2.33    *                   vmovaps	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.33                        vmovd	%eax, %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   vmovd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        vmovd	%xmm0, %ecx
+# CHECK-NEXT:  2      12    0.50           *            vmovd	%xmm0, (%rax)
+# CHECK-NEXT:  1      1     0.50                        vmovddup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   vmovddup	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vmovddup	%ymm0, %ymm2
+# CHECK-NEXT:  1      7     2.33    *                   vmovddup	(%rax), %ymm2
+# CHECK-NEXT:  2      1     0.50                        vmovdqa	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovdqa	%xmm0, (%rax)
+# CHECK-NEXT:  1      6     0.33    *                   vmovdqa	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovdqa	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovdqa	%ymm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   vmovdqa	(%rax), %ymm2
+# CHECK-NEXT:  2      1     0.50                        vmovdqu	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovdqu	%xmm0, (%rax)
+# CHECK-NEXT:  1      6     0.33    *                   vmovdqu	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovdqu	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovdqu	%ymm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   vmovdqu	(%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vmovhlps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vmovlhps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovhpd	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.50    *                   vmovhpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovhps	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.50    *                   vmovhps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovlpd	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.33    *                   vmovlpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovlps	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.33    *                   vmovlps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vmovmskpd	%xmm0, %ecx
+# CHECK-NEXT:  1      4     1.50                        vmovmskpd	%ymm0, %ecx
+# CHECK-NEXT:  1      3     1.50                        vmovmskps	%xmm0, %ecx
+# CHECK-NEXT:  1      4     1.50                        vmovmskps	%ymm0, %ecx
+# CHECK-NEXT:  2      521   0.50           *            vmovntdq	%xmm0, (%rax)
+# CHECK-NEXT:  2      507   0.50           *            vmovntdq	%ymm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   vmovntdqa	(%rax), %xmm2
+# CHECK-NEXT:  1      8     0.33    *                   vmovntdqa	(%rax), %ymm2
+# CHECK-NEXT:  2      473   0.50           *            vmovntpd	%xmm0, (%rax)
+# CHECK-NEXT:  2      542   0.50           *            vmovntpd	%ymm0, (%rax)
+# CHECK-NEXT:  2      470   0.50           *            vmovntps	%xmm0, (%rax)
+# CHECK-NEXT:  2      494   0.50           *            vmovntps	%ymm0, (%rax)
+# CHECK-NEXT:  1      1     0.25                        vmovq	%xmm0, %xmm2
+# CHECK-NEXT:  1      3     0.33                        vmovq	%rax, %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   vmovq	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        vmovq	%xmm0, %rcx
+# CHECK-NEXT:  2      12    0.50           *            vmovq	%xmm0, (%rax)
+# CHECK-NEXT:  1      1     0.25                        vmovsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovsd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   vmovsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vmovshdup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   vmovshdup	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vmovshdup	%ymm0, %ymm2
+# CHECK-NEXT:  1      7     2.33    *                   vmovshdup	(%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vmovsldup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   vmovsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vmovsldup	%ymm0, %ymm2
+# CHECK-NEXT:  1      7     2.33    *                   vmovsldup	(%rax), %ymm2
+# CHECK-NEXT:  1      1     0.25                        vmovss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovss	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   vmovss	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovupd	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovupd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   vmovupd	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovupd	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovupd	%ymm0, (%rax)
+# CHECK-NEXT:  1      8     2.33    *                   vmovupd	(%rax), %ymm2
+# CHECK-NEXT:  0      1     0.00                        vmovups	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            vmovups	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   vmovups	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        vmovups	%ymm0, %ymm2
+# CHECK-NEXT:  2      12    0.50           *            vmovups	%ymm0, (%rax)
+# CHECK-NEXT:  1      8     2.33    *                   vmovups	(%rax), %ymm2
+# CHECK-NEXT:  2      4     1.00                        vmpsadbw	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      11    1.00    *                   vmpsadbw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vmulpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vmulpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vmulpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vmulpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vmulps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vmulps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vmulps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vmulps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vmulsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vmulsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vmulss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vmulss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vorpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vorpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vorpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vorpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vorps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vorps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vorps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vorps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpabsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpabsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpabsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpabsw	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpackssdw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpackssdw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpacksswb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpacksswb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpackusdw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpackusdw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpackuswb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpackuswb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpaddb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpaddq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpaddsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpaddusb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddusb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpaddusw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddusw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.33                        vpalignr	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpalignr	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpand	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpand	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpandn	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpandn	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpavgb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpavgb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpavgw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpavgw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     1.00                        vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vpclmulqdq	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpeqb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpeqd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpeqq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpeqw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  8      16    3.00                        vpcmpestri	$1, %xmm0, %xmm2
+# CHECK-NEXT:  8      31    3.00    *                   vpcmpestri	$1, (%rax), %xmm2
+# CHECK-NEXT:  9      16    3.00                        vpcmpestrm	$1, %xmm0, %xmm2
+# CHECK-NEXT:  9      17    3.00    *                   vpcmpestrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpgtb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpgtd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpcmpgtq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpcmpgtw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      11    3.00                        vpcmpistri	$1, %xmm0, %xmm2
+# CHECK-NEXT:  4      31    3.00    *                   vpcmpistri	$1, (%rax), %xmm2
+# CHECK-NEXT:  3      11    3.00                        vpcmpistrm	$1, %xmm0, %xmm2
+# CHECK-NEXT:  4      15    3.00    *                   vpcmpistrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpermilpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpermilpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpermilpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpermilpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpermilpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpermilpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpermilps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpermilps	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpermilps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpermilps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpermilps	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpermilps	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpermilps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpermilps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  2      4     1.50                        vpextrb	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     0.50           *            vpextrb	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      4     1.50                        vpextrd	$1, %xmm0, %ecx
+# CHECK-NEXT:  3      12    0.50           *            vpextrd	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      4     1.50                        vpextrq	$1, %xmm0, %rcx
+# CHECK-NEXT:  3      12    0.50           *            vpextrq	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      4     1.50                        vpextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     0.50           *            vpextrw	$1, %xmm0, (%rax)
+# CHECK-NEXT:  3      2     1.00                        vphaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      2     1.00                        vphaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      2     1.00                        vphaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      5     0.50                        vphminposuw	%xmm0, %xmm2
+# CHECK-NEXT:  1      11    0.50    *                   vphminposuw	(%rax), %xmm2
+# CHECK-NEXT:  3      2     1.00                        vphsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      2     1.00                        vphsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      2     1.00                        vphsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   vphsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      5     1.33                        vpinsrb	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      5     1.33                        vpinsrd	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpinsrd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      5     1.33                        vpinsrq	$1, %rax, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      5     1.33                        vpinsrw	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmaddubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmaddwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmaddwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxub	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxub	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxud	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxud	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpmaxuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminub	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminub	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminud	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminud	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpminuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpminuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpmovmskb	%xmm0, %ecx
+# CHECK-NEXT:  1      1     0.50                        vpmovsxbd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxbd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovsxbq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxbq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovsxbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovsxdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovsxwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovsxwq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovsxwq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxbd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxbd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxbq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxbq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpmovzxwq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpmovzxwq	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpmuldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpmuldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhrsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmulhrsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmulhuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmulhw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50                        vpmulld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      18    0.50    *                   vpmulld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpmullw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpmullw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     1.50                        vpmuludq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   vpmuludq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpor	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpor	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vpsadbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vpsadbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpshufb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpshufd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpshufd	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpshufhw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpshufhw	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpshuflw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vpshuflw	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsignb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsignb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsignd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsignd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsignw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsignw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpslld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpslld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpslldq	$1, %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsllq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsllq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsllw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsllw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrad	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsrad	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsraw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsraw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsrld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrldq	$1, %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsrlq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      2     0.50                        vpsrlw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsubsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsubusb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubusb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsubusw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubusw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      4     0.50                        vptest	%xmm0, %xmm1
+# CHECK-NEXT:  3      10    2.00    *                   vptest	(%rax), %xmm1
+# CHECK-NEXT:  2      100   1.50                        vptest	%ymm0, %ymm1
+# CHECK-NEXT:  3      11    2.00    *                   vptest	(%rax), %ymm1
+# CHECK-NEXT:  1      1     0.50                        vpunpckhbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpckhbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpckhdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhqdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpckhqdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpckhwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpcklbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpckldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklqdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpcklqdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vpunpcklwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpxor	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpxor	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vrcpps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vrcpps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vrcpps	%ymm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vrcpps	(%rax), %ymm2
+# CHECK-NEXT:  1      4     2.00                        vrcpss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vrcpss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     4.00                        vroundpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   vroundpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  2      8     4.00                        vroundpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  3      15    4.00    *                   vroundpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  2      8     4.00                        vroundps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   vroundps	$1, (%rax), %xmm2
+# CHECK-NEXT:  2      8     4.00                        vroundps	$1, %ymm0, %ymm2
+# CHECK-NEXT:  3      15    4.00    *                   vroundps	$1, (%rax), %ymm2
+# CHECK-NEXT:  2      8     4.00                        vroundsd	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   vroundsd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      8     4.00                        vroundss	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   vroundss	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vrsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vrsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        vrsqrtps	%ymm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vrsqrtps	(%rax), %ymm2
+# CHECK-NEXT:  1      4     2.00                        vrsqrtss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vrsqrtss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vshufpd	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vshufpd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vshufpd	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vshufpd	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vshufps	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   vshufps	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vshufps	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vshufps	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      15    7.50                        vsqrtpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      21    7.50    *                   vsqrtpd	(%rax), %xmm2
+# CHECK-NEXT:  1      15    7.50                        vsqrtpd	%ymm0, %ymm2
+# CHECK-NEXT:  1      22    7.50    *                   vsqrtpd	(%rax), %ymm2
+# CHECK-NEXT:  1      10    5.00                        vsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   vsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  1      10    5.00                        vsqrtps	%ymm0, %ymm2
+# CHECK-NEXT:  1      17    5.00    *                   vsqrtps	(%rax), %ymm2
+# CHECK-NEXT:  1      15    7.50                        vsqrtsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      24    1.00    *                   vsqrtsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      10    5.00                        vsqrtss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   vsqrtss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  4      100   3.00           *      U     vstmxcsr	(%rax)
+# CHECK-NEXT:  1      3     0.50                        vsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vsubsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vsubsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.33                        vsubss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   vsubss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      3     0.50                        vtestpd	%xmm0, %xmm1
+# CHECK-NEXT:  2      7     0.50    *                   vtestpd	(%rax), %xmm1
+# CHECK-NEXT:  1      5     0.50                        vtestpd	%ymm0, %ymm1
+# CHECK-NEXT:  2      11    0.50    *                   vtestpd	(%rax), %ymm1
+# CHECK-NEXT:  1      3     0.50                        vtestps	%xmm0, %xmm1
+# CHECK-NEXT:  2      7     0.50    *                   vtestps	(%rax), %xmm1
+# CHECK-NEXT:  1      100   1.50                        vtestps	%ymm0, %ymm1
+# CHECK-NEXT:  2      11    0.50    *                   vtestps	(%rax), %ymm1
+# CHECK-NEXT:  1      3     1.50                        vucomisd	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   vucomisd	(%rax), %xmm1
+# CHECK-NEXT:  1      3     1.50                        vucomiss	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   vucomiss	(%rax), %xmm1
+# CHECK-NEXT:  1      1     0.50                        vunpckhpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vunpckhpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vunpckhpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vunpckhpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vunpckhps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vunpckhps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vunpckhps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vunpckhps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vunpcklpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vunpcklpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vunpcklpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vunpcklpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vunpcklps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   vunpcklps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vunpcklps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vunpcklps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vxorpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vxorpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vxorps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vxorps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vxorps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vxorps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  10     16    1.67                  U     vzeroall
+# CHECK-NEXT:  0      0     0.00                  U     vzeroupper
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 28.67  28.67  3.67   28.67  3.67   28.67  19.00  19.00  459.00 459.00 459.00 12.33  12.33  12.33   -     565.50 565.50 193.50 193.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vaddss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vaddsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vaddsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesdec	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesdec	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesdeclast	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesdeclast	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesenc	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesenc	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesenclast	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesenclast	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandnpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vandnpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandnpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vandnpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandnps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vandnps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandnps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vandnps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vandpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vandpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vandps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vandps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vandps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vblendpd	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vblendpd	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vblendpd	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vblendpd	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vblendps	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vblendps	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vblendps	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vblendps	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vblendvpd	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vblendvpd	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   vblendvpd	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.75   0.75   0.75   0.75   vblendvpd	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vblendvps	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vblendvps	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   vblendvps	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.75   0.75   0.75   0.75   vblendvps	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vbroadcastf128	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vbroadcastsd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vbroadcastss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vbroadcastss	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcmpeqpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcmpeqpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcmpeqps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcmpeqps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcmpeqsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcmpeqss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcmpeqss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vcomisd	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vcomisd	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vcomiss	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vcomiss	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtdq2pd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vcvtdq2pd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   1.50   1.50   vcvtdq2pd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcvtdq2pd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtdq2ps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtdq2ps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vcvtdq2ps	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vcvtdq2ps	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2dqx	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2dq	%ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2dqy	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2ps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2psx	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2ps	%ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvtpd2psy	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtps2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtps2dq	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcvtps2dq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   vcvtps2pd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtps2pd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   1.50   1.50   vcvtps2pd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcvtps2pd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvtsd2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvtsd2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvtsd2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvtsd2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtsd2ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00   0.50   0.50   vcvtsd2ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtsi2sd	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtsi2sd	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtsi2sdl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtsi2sdq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvtsi2ss	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   vcvtsi2ss	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtsi2ssl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00   0.50   0.50   vcvtsi2ssq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   vcvtss2sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvtss2sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvtss2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50   0.50   0.50   vcvtss2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvtss2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvtss2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvttpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvttpd2dqx	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvttpd2dq	%ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vcvttpd2dqy	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvttps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vcvttps2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vcvttps2dq	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcvttps2dq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvttsd2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvttsd2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvttsd2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvttsd2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vcvttss2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50   0.50   0.50   vcvttss2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvttss2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     vcvttss2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vdivpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     vdivpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vdivpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     5.00   5.00    -      -     vdivpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vdivps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     3.50   3.50    -      -     vdivps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vdivps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     3.50   3.50    -      -     vdivps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vdivsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vdivsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     vdivss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     3.50   3.50    -      -     vdivss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   0.50   0.50   vdppd	$22, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00   1.00   0.50   0.50   vdppd	$22, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   1.50   1.50   vdpps	$22, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.50   1.50   1.50   1.50   vdpps	$22, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   1.50   1.50   vdpps	$22, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.50   1.50   1.50   1.50   vdpps	$22, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vextractf128	$1, %ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vextractf128	$1, %ymm0, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vextractps	$1, %xmm0, %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vextractps	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vhaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   vhaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   vhaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     2.50   2.50   vhaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vhaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   vhaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   vhaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     2.50   2.50   vhaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vhsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   vhsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   vhsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     2.50   2.50   vhsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vhsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   vhsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   vhsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     2.50   2.50   vhsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vinsertf128	$1, %xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vinsertf128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vinsertps	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vinsertps	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vlddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vlddqu	(%rax), %ymm2
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vldmxcsr	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmaskmovdqu	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmaskmovpd	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmaskmovpd	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vmaskmovpd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vmaskmovpd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmaskmovps	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmaskmovps	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vmaskmovps	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vmaskmovps	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vmaxpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vmaxpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vmaxps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vmaxps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vmaxsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vmaxss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vmaxss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vminpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vminpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vminps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vminps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vminsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vminss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vminss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovapd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovapd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovapd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovapd	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovapd	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovapd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovaps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovaps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovaps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovaps	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovaps	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovaps	(%rax), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vmovd	%eax, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vmovd	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovddup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovddup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovddup	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovddup	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vmovdqa	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovdqa	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovdqa	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovdqa	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovdqa	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovdqa	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vmovdqu	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovdqu	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovdqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovdqu	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovdqu	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovdqu	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovhlps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovlhps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovhpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vmovhpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovhps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vmovhps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovlpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmovlpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovlps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   vmovlps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmovmskpd	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmovmskpd	%ymm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmovmskps	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmovmskps	%ymm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntdq	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntdq	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovntdqa	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovntdqa	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntpd	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovntps	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vmovq	%xmm0, %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vmovq	%rax, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vmovq	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovq	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vmovsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovsd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovshdup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovshdup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovshdup	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovshdup	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovsldup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vmovsldup	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovsldup	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vmovss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovss	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovupd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovupd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovupd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovupd	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovupd	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovupd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovups	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovups	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vmovups	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vmovups	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vmovups	%ymm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vmovups	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   vmpsadbw	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   vmpsadbw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vmulpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     1.50   1.50    -      -     vmulpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vmulps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     1.50   1.50    -      -     vmulps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vmulsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vmulss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vmulss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vorpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vorpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vorpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vorpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vorps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vorps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vorps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vorps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpabsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpabsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpabsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackssdw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   vpackssdw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpacksswb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   vpacksswb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackusdw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   vpackusdw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackuswb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   vpackuswb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpaddb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpaddq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpaddsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddusb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpaddusb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddusw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpaddusw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpalignr	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpalignr	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpand	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpand	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpandn	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpandn	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpavgb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpavgb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpavgw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpavgw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpblendvb	%xmm3, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vpblendvb	%xmm3, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpblendw	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpblendw	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpclmulqdq	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpclmulqdq	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpeqb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpeqd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpeqq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpeqw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT: 3.67   1.00   0.67   1.00   0.67   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpestri	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.17   1.17   0.17   1.17   0.17   1.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vpcmpestri	$1, (%rax), %xmm2
+# CHECK-NEXT: 3.50   1.50   0.50   1.50   0.50   1.50    -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpestrm	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.33   1.33   0.33   1.33   0.33   1.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vpcmpestrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpgtb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpgtd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpcmpgtq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   vpcmpgtq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpcmpgtw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistri	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vpcmpistri	$1, (%rax), %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vpcmpistrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpermilpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpermilps	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilps	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilps	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermilps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermilps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   vpextrb	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   vpextrb	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   vpextrd	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   vpextrd	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   vpextrq	$1, %xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   vpextrq	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   vpextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   vpextrw	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   vphaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   1.00   1.00   vphaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vphminposuw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vphminposuw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   vphsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   1.00   1.00   vphsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpinsrb	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpinsrb	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpinsrd	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpinsrd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpinsrq	$1, %rax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpinsrq	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpinsrw	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpinsrw	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmaddubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmaddubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmaddwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmaddwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxub	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxub	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxud	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxud	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpmaxuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminub	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminub	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminud	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminud	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpminuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmovmskb	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxbd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxbq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxwq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovsxwq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxbd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxbq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxwq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpmovzxwq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmuldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vpmuldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhrsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmulhrsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhuw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmulhuw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmulhw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpmulld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vpmulld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmullw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpmullw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmuludq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vpmuludq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpor	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpor	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpsadbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpsadbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpshufb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpshufd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufhw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpshufhw	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshuflw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vpshuflw	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsignb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsignd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsignw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpslld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpslld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpslldq	$1, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsllq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsllw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrad	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrad	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsraw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsraw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpsrldq	$1, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrlq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrlw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpsubb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpsubq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubsb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsubsb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubusb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsubusb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubusw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsubusw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vptest	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50   0.50   0.50   vptest	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   vptest	%ymm0, %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50   0.50   0.50   vptest	(%rax), %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhqdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhqdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklbw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklbw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckldq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckldq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklqdq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklqdq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklwd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklwd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpxor	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpxor	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrcpps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vrcpps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrcpps	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vrcpps	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrcpss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vrcpss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     vroundpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     4.00   4.00    -      -     vroundpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     vroundps	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundps	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     4.00   4.00    -      -     vroundps	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundsd	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     vroundsd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     vroundss	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     vroundss	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vrsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrsqrtps	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vrsqrtps	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vrsqrtss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vrsqrtss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vshufpd	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vshufpd	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vshufpd	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vshufpd	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vshufps	$1, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   vshufps	$1, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vshufps	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vshufps	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50   7.50    -      -     vsqrtpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     7.50   7.50    -      -     vsqrtpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50   7.50    -      -     vsqrtpd	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     7.50   7.50    -      -     vsqrtpd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     vsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vsqrtps	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     5.00   5.00    -      -     vsqrtps	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50   7.50    -      -     vsqrtsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vsqrtsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     vsqrtss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     vsqrtss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     3.00   3.00    -      -     vstmxcsr	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vsubsd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubsd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vsubss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vsubss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vtestpd	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vtestpd	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vtestpd	%ymm0, %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vtestpd	(%rax), %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vtestps	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vtestps	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vtestps	%ymm0, %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vtestps	(%rax), %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vucomisd	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vucomisd	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vucomiss	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     vucomiss	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpckhpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vunpckhpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpckhpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vunpckhpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpckhps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vunpckhps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpckhps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vunpckhps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpcklpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vunpcklpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpcklpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vunpcklpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpcklps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vunpcklps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vunpcklps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vunpcklps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vxorpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vxorpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vxorpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vxorpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vxorps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vxorps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vxorps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vxorps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT: 1.33   2.00   1.33   2.00   1.33   2.00    -      -      -      -      -      -      -      -      -      -      -      -      -     vzeroall
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     vzeroupper
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx2.s
new file mode 100644
index 000000000000..8109dc36ca70
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avx2.s
@@ -0,0 +1,1092 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vbroadcasti128  (%rax), %ymm0
+
+vbroadcastsd    %xmm0, %ymm0
+vbroadcastss    %xmm0, %ymm0
+
+vextracti128    $1, %ymm0, %xmm2
+vextracti128    $1, %ymm0, (%rax)
+
+vgatherdpd      %xmm0, (%rax,%xmm1,2), %xmm2
+vgatherdpd      %ymm0, (%rax,%xmm1,2), %ymm2
+
+vgatherdps      %xmm0, (%rax,%xmm1,2), %xmm2
+vgatherdps      %ymm0, (%rax,%ymm1,2), %ymm2
+
+vgatherqpd      %xmm0, (%rax,%xmm1,2), %xmm2
+vgatherqpd      %ymm0, (%rax,%ymm1,2), %ymm2
+
+vgatherqps      %xmm0, (%rax,%xmm1,2), %xmm2
+vgatherqps      %xmm0, (%rax,%ymm1,2), %xmm2
+
+vinserti128     $1, %xmm0, %ymm1, %ymm2
+vinserti128     $1, (%rax), %ymm1, %ymm2
+
+vmovntdqa       (%rax), %ymm0
+
+vmpsadbw        $1, %ymm0, %ymm1, %ymm2
+vmpsadbw        $1, (%rax), %ymm1, %ymm2
+
+vpabsb          %ymm0, %ymm2
+vpabsb          (%rax), %ymm2
+
+vpabsd          %ymm0, %ymm2
+vpabsd          (%rax), %ymm2
+
+vpabsw          %ymm0, %ymm2
+vpabsw          (%rax), %ymm2
+
+vpackssdw       %ymm0, %ymm1, %ymm2
+vpackssdw       (%rax), %ymm1, %ymm2
+
+vpacksswb       %ymm0, %ymm1, %ymm2
+vpacksswb       (%rax), %ymm1, %ymm2
+
+vpackusdw       %ymm0, %ymm1, %ymm2
+vpackusdw       (%rax), %ymm1, %ymm2
+
+vpackuswb       %ymm0, %ymm1, %ymm2
+vpackuswb       (%rax), %ymm1, %ymm2
+
+vpaddb          %ymm0, %ymm1, %ymm2
+vpaddb          (%rax), %ymm1, %ymm2
+
+vpaddd          %ymm0, %ymm1, %ymm2
+vpaddd          (%rax), %ymm1, %ymm2
+
+vpaddq          %ymm0, %ymm1, %ymm2
+vpaddq          (%rax), %ymm1, %ymm2
+
+vpaddsb         %ymm0, %ymm1, %ymm2
+vpaddsb         (%rax), %ymm1, %ymm2
+
+vpaddsw         %ymm0, %ymm1, %ymm2
+vpaddsw         (%rax), %ymm1, %ymm2
+
+vpaddusb        %ymm0, %ymm1, %ymm2
+vpaddusb        (%rax), %ymm1, %ymm2
+
+vpaddusw        %ymm0, %ymm1, %ymm2
+vpaddusw        (%rax), %ymm1, %ymm2
+
+vpaddw          %ymm0, %ymm1, %ymm2
+vpaddw          (%rax), %ymm1, %ymm2
+
+vpalignr        $1, %ymm0, %ymm1, %ymm2
+vpalignr        $1, (%rax), %ymm1, %ymm2
+
+vpand           %ymm0, %ymm1, %ymm2
+vpand           (%rax), %ymm1, %ymm2
+
+vpandn          %ymm0, %ymm1, %ymm2
+vpandn          (%rax), %ymm1, %ymm2
+
+vpavgb          %ymm0, %ymm1, %ymm2
+vpavgb          (%rax), %ymm1, %ymm2
+
+vpavgw          %ymm0, %ymm1, %ymm2
+vpavgw          (%rax), %ymm1, %ymm2
+
+vpblendd        $11, %xmm0, %xmm1, %xmm2
+vpblendd        $11, (%rax), %xmm1, %xmm2
+
+vpblendd        $11, %ymm0, %ymm1, %ymm2
+vpblendd        $11, (%rax), %ymm1, %ymm2
+
+vpblendvb       %ymm3, %ymm0, %ymm1, %ymm2
+vpblendvb       %ymm3, (%rax), %ymm1, %ymm2
+
+vpblendw        $11, %ymm0, %ymm1, %ymm2
+vpblendw        $11, (%rax), %ymm1, %ymm2
+
+vpbroadcastb    %xmm0, %xmm0
+vpbroadcastb    (%rax), %xmm0
+
+vpbroadcastb    %xmm0, %ymm0
+vpbroadcastb    (%rax), %ymm0
+
+vpbroadcastd    %xmm0, %xmm0
+vpbroadcastd    (%rax), %xmm0
+
+vpbroadcastd    %xmm0, %ymm0
+vpbroadcastd    (%rax), %ymm0
+
+vpbroadcastq    %xmm0, %xmm0
+vpbroadcastq    (%rax), %xmm0
+
+vpbroadcastq    %xmm0, %ymm0
+vpbroadcastq    (%rax), %ymm0
+
+vpbroadcastw    %xmm0, %xmm0
+vpbroadcastw    (%rax), %xmm0
+
+vpbroadcastw    %xmm0, %ymm0
+vpbroadcastw    (%rax), %ymm0
+
+vpcmpeqb        %ymm0, %ymm1, %ymm2
+vpcmpeqb        (%rax), %ymm1, %ymm2
+
+vpcmpeqd        %ymm0, %ymm1, %ymm2
+vpcmpeqd        (%rax), %ymm1, %ymm2
+
+vpcmpeqq        %ymm0, %ymm1, %ymm2
+vpcmpeqq        (%rax), %ymm1, %ymm2
+
+vpcmpeqw        %ymm0, %ymm1, %ymm2
+vpcmpeqw        (%rax), %ymm1, %ymm2
+
+vpcmpgtb        %ymm0, %ymm1, %ymm2
+vpcmpgtb        (%rax), %ymm1, %ymm2
+
+vpcmpgtd        %ymm0, %ymm1, %ymm2
+vpcmpgtd        (%rax), %ymm1, %ymm2
+
+vpcmpgtq        %ymm0, %ymm1, %ymm2
+vpcmpgtq        (%rax), %ymm1, %ymm2
+
+vpcmpgtw        %ymm0, %ymm1, %ymm2
+vpcmpgtw        (%rax), %ymm1, %ymm2
+
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
+vpermd          %ymm0, %ymm1, %ymm2
+vpermd          (%rax), %ymm1, %ymm2
+
+vpermpd         $1, %ymm0, %ymm2
+vpermpd         $1, (%rax), %ymm2
+
+vpermps         %ymm0, %ymm1, %ymm2
+vpermps         (%rax), %ymm1, %ymm2
+
+vpermq          $1, %ymm0, %ymm2
+vpermq          $1, (%rax), %ymm2
+
+vpgatherdd      %xmm0, (%rax,%xmm1,2), %xmm2
+vpgatherdd      %ymm0, (%rax,%ymm1,2), %ymm2
+
+vpgatherdq      %xmm0, (%rax,%xmm1,2), %xmm2
+vpgatherdq      %ymm0, (%rax,%xmm1,2), %ymm2
+
+vpgatherqd      %xmm0, (%rax,%xmm1,2), %xmm2
+vpgatherqd      %xmm0, (%rax,%ymm1,2), %xmm2
+
+vpgatherqq      %xmm0, (%rax,%xmm1,2), %xmm2
+vpgatherqq      %ymm0, (%rax,%ymm1,2), %ymm2
+
+vphaddd         %ymm0, %ymm1, %ymm2
+vphaddd         (%rax), %ymm1, %ymm2
+
+vphaddsw        %ymm0, %ymm1, %ymm2
+vphaddsw        (%rax), %ymm1, %ymm2
+
+vphaddw         %ymm0, %ymm1, %ymm2
+vphaddw         (%rax), %ymm1, %ymm2
+
+vphsubd         %ymm0, %ymm1, %ymm2
+vphsubd         (%rax), %ymm1, %ymm2
+
+vphsubsw        %ymm0, %ymm1, %ymm2
+vphsubsw        (%rax), %ymm1, %ymm2
+
+vphsubw         %ymm0, %ymm1, %ymm2
+vphsubw         (%rax), %ymm1, %ymm2
+
+vpmaddubsw      %ymm0, %ymm1, %ymm2
+vpmaddubsw      (%rax), %ymm1, %ymm2
+
+vpmaddwd        %ymm0, %ymm1, %ymm2
+vpmaddwd        (%rax), %ymm1, %ymm2
+
+vpmaskmovd      (%rax), %xmm0, %xmm2
+vpmaskmovd      (%rax), %ymm0, %ymm2
+
+vpmaskmovd      %xmm0, %xmm1, (%rax)
+vpmaskmovd      %ymm0, %ymm1, (%rax)
+
+vpmaskmovq      (%rax), %xmm0, %xmm2
+vpmaskmovq      (%rax), %ymm0, %ymm2
+
+vpmaskmovq      %xmm0, %xmm1, (%rax)
+vpmaskmovq      %ymm0, %ymm1, (%rax)
+
+vpmaxsb         %ymm0, %ymm1, %ymm2
+vpmaxsb         (%rax), %ymm1, %ymm2
+
+vpmaxsd         %ymm0, %ymm1, %ymm2
+vpmaxsd         (%rax), %ymm1, %ymm2
+
+vpmaxsw         %ymm0, %ymm1, %ymm2
+vpmaxsw         (%rax), %ymm1, %ymm2
+
+vpmaxub         %ymm0, %ymm1, %ymm2
+vpmaxub         (%rax), %ymm1, %ymm2
+
+vpmaxud         %ymm0, %ymm1, %ymm2
+vpmaxud         (%rax), %ymm1, %ymm2
+
+vpmaxuw         %ymm0, %ymm1, %ymm2
+vpmaxuw         (%rax), %ymm1, %ymm2
+
+vpminsb         %ymm0, %ymm1, %ymm2
+vpminsb         (%rax), %ymm1, %ymm2
+
+vpminsd         %ymm0, %ymm1, %ymm2
+vpminsd         (%rax), %ymm1, %ymm2
+
+vpminsw         %ymm0, %ymm1, %ymm2
+vpminsw         (%rax), %ymm1, %ymm2
+
+vpminub         %ymm0, %ymm1, %ymm2
+vpminub         (%rax), %ymm1, %ymm2
+
+vpminud         %ymm0, %ymm1, %ymm2
+vpminud         (%rax), %ymm1, %ymm2
+
+vpminuw         %ymm0, %ymm1, %ymm2
+vpminuw         (%rax), %ymm1, %ymm2
+
+vpmovmskb       %ymm0, %rcx
+
+vpmovsxbd       %xmm0, %ymm2
+vpmovsxbd       (%rax), %ymm2
+
+vpmovsxbq       %xmm0, %ymm2
+vpmovsxbq       (%rax), %ymm2
+
+vpmovsxbw       %xmm0, %ymm2
+vpmovsxbw       (%rax), %ymm2
+
+vpmovsxdq       %xmm0, %ymm2
+vpmovsxdq       (%rax), %ymm2
+
+vpmovsxwd       %xmm0, %ymm2
+vpmovsxwd       (%rax), %ymm2
+
+vpmovsxwq       %xmm0, %ymm2
+vpmovsxwq       (%rax), %ymm2
+
+vpmovzxbd       %xmm0, %ymm2
+vpmovzxbd       (%rax), %ymm2
+
+vpmovzxbq       %xmm0, %ymm2
+vpmovzxbq       (%rax), %ymm2
+
+vpmovzxbw       %xmm0, %ymm2
+vpmovzxbw       (%rax), %ymm2
+
+vpmovzxdq       %xmm0, %ymm2
+vpmovzxdq       (%rax), %ymm2
+
+vpmovzxwd       %xmm0, %ymm2
+vpmovzxwd       (%rax), %ymm2
+
+vpmovzxwq       %xmm0, %ymm2
+vpmovzxwq       (%rax), %ymm2
+
+vpmuldq         %ymm0, %ymm1, %ymm2
+vpmuldq         (%rax), %ymm1, %ymm2
+
+vpmulhrsw       %ymm0, %ymm1, %ymm2
+vpmulhrsw       (%rax), %ymm1, %ymm2
+
+vpmulhuw        %ymm0, %ymm1, %ymm2
+vpmulhuw        (%rax), %ymm1, %ymm2
+
+vpmulhw         %ymm0, %ymm1, %ymm2
+vpmulhw         (%rax), %ymm1, %ymm2
+
+vpmulld         %ymm0, %ymm1, %ymm2
+vpmulld         (%rax), %ymm1, %ymm2
+
+vpmullw         %ymm0, %ymm1, %ymm2
+vpmullw         (%rax), %ymm1, %ymm2
+
+vpmuludq        %ymm0, %ymm1, %ymm2
+vpmuludq        (%rax), %ymm1, %ymm2
+
+vpor            %ymm0, %ymm1, %ymm2
+vpor            (%rax), %ymm1, %ymm2
+
+vpsadbw         %ymm0, %ymm1, %ymm2
+vpsadbw         (%rax), %ymm1, %ymm2
+
+vpshufb         %ymm0, %ymm1, %ymm2
+vpshufb         (%rax), %ymm1, %ymm2
+
+vpshufd         $1, %ymm0, %ymm2
+vpshufd         $1, (%rax), %ymm2
+
+vpshufhw        $1, %ymm0, %ymm2
+vpshufhw        $1, (%rax), %ymm2
+
+vpshuflw        $1, %ymm0, %ymm2
+vpshuflw        $1, (%rax), %ymm2
+
+vpsignb         %ymm0, %ymm1, %ymm2
+vpsignb         (%rax), %ymm1, %ymm2
+
+vpsignd         %ymm0, %ymm1, %ymm2
+vpsignd         (%rax), %ymm1, %ymm2
+
+vpsignw         %ymm0, %ymm1, %ymm2
+vpsignw         (%rax), %ymm1, %ymm2
+
+vpslld          $1, %ymm0, %ymm2
+vpslld          %xmm0, %ymm1, %ymm2
+vpslld          (%rax), %ymm1, %ymm2
+
+vpslldq         $1, %ymm1, %ymm2
+
+vpsllq          $1, %ymm0, %ymm2
+vpsllq          %xmm0, %ymm1, %ymm2
+vpsllq          (%rax), %ymm1, %ymm2
+
+vpsllvd         %xmm0, %xmm1, %xmm2
+vpsllvd         (%rax), %xmm1, %xmm2
+
+vpsllvd         %ymm0, %ymm1, %ymm2
+vpsllvd         (%rax), %ymm1, %ymm2
+
+vpsllvq         %xmm0, %xmm1, %xmm2
+vpsllvq         (%rax), %xmm1, %xmm2
+
+vpsllvq         %ymm0, %ymm1, %ymm2
+vpsllvq         (%rax), %ymm1, %ymm2
+
+vpsllw          $1, %ymm0, %ymm2
+vpsllw          %xmm0, %ymm1, %ymm2
+vpsllw          (%rax), %ymm1, %ymm2
+
+vpsrad          $1, %ymm0, %ymm2
+vpsrad          %xmm0, %ymm1, %ymm2
+vpsrad          (%rax), %ymm1, %ymm2
+
+vpsravd         %xmm0, %xmm1, %xmm2
+vpsravd         (%rax), %xmm1, %xmm2
+
+vpsravd         %ymm0, %ymm1, %ymm2
+vpsravd         (%rax), %ymm1, %ymm2
+
+vpsraw          $1, %ymm0, %ymm2
+vpsraw          %xmm0, %ymm1, %ymm2
+vpsraw          (%rax), %ymm1, %ymm2
+
+vpsrld          $1, %ymm0, %ymm2
+vpsrld          %xmm0, %ymm1, %ymm2
+vpsrld          (%rax), %ymm1, %ymm2
+
+vpsrldq         $1, %ymm1, %ymm2
+
+vpsrlq          $1, %ymm0, %ymm2
+vpsrlq          %xmm0, %ymm1, %ymm2
+vpsrlq          (%rax), %ymm1, %ymm2
+
+vpsrlvd         %xmm0, %xmm1, %xmm2
+vpsrlvd         (%rax), %xmm1, %xmm2
+
+vpsrlvd         %ymm0, %ymm1, %ymm2
+vpsrlvd         (%rax), %ymm1, %ymm2
+
+vpsrlvq         %xmm0, %xmm1, %xmm2
+vpsrlvq         (%rax), %xmm1, %xmm2
+
+vpsrlvq         %ymm0, %ymm1, %ymm2
+vpsrlvq         (%rax), %ymm1, %ymm2
+
+vpsrlw          $1, %ymm0, %ymm2
+vpsrlw          %xmm0, %ymm1, %ymm2
+vpsrlw          (%rax), %ymm1, %ymm2
+
+vpsubb          %ymm0, %ymm1, %ymm2
+vpsubb          (%rax), %ymm1, %ymm2
+
+vpsubd          %ymm0, %ymm1, %ymm2
+vpsubd          (%rax), %ymm1, %ymm2
+
+vpsubq          %ymm0, %ymm1, %ymm2
+vpsubq          (%rax), %ymm1, %ymm2
+
+vpsubsb         %ymm0, %ymm1, %ymm2
+vpsubsb         (%rax), %ymm1, %ymm2
+
+vpsubsw         %ymm0, %ymm1, %ymm2
+vpsubsw         (%rax), %ymm1, %ymm2
+
+vpsubusb        %ymm0, %ymm1, %ymm2
+vpsubusb        (%rax), %ymm1, %ymm2
+
+vpsubusw        %ymm0, %ymm1, %ymm2
+vpsubusw        (%rax), %ymm1, %ymm2
+
+vpsubw          %ymm0, %ymm1, %ymm2
+vpsubw          (%rax), %ymm1, %ymm2
+
+vpunpckhbw      %ymm0, %ymm1, %ymm2
+vpunpckhbw      (%rax), %ymm1, %ymm2
+
+vpunpckhdq      %ymm0, %ymm1, %ymm2
+vpunpckhdq      (%rax), %ymm1, %ymm2
+
+vpunpckhqdq     %ymm0, %ymm1, %ymm2
+vpunpckhqdq     (%rax), %ymm1, %ymm2
+
+vpunpckhwd      %ymm0, %ymm1, %ymm2
+vpunpckhwd      (%rax), %ymm1, %ymm2
+
+vpunpcklbw      %ymm0, %ymm1, %ymm2
+vpunpcklbw      (%rax), %ymm1, %ymm2
+
+vpunpckldq      %ymm0, %ymm1, %ymm2
+vpunpckldq      (%rax), %ymm1, %ymm2
+
+vpunpcklqdq     %ymm0, %ymm1, %ymm2
+vpunpcklqdq     (%rax), %ymm1, %ymm2
+
+vpunpcklwd      %ymm0, %ymm1, %ymm2
+vpunpcklwd      (%rax), %ymm1, %ymm2
+
+vpxor           %ymm0, %ymm1, %ymm2
+vpxor           (%rax), %ymm1, %ymm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      8     0.33    *                   vbroadcasti128	(%rax), %ymm0
+# CHECK-NEXT:  1      3     0.50                        vbroadcastsd	%xmm0, %ymm0
+# CHECK-NEXT:  1      3     0.50                        vbroadcastss	%xmm0, %ymm0
+# CHECK-NEXT:  1      3     0.50                        vextracti128	$1, %ymm0, %xmm2
+# CHECK-NEXT:  2      1     0.50           *            vextracti128	$1, %ymm0, (%rax)
+# CHECK-NEXT:  5      20    0.67    *                   vgatherdpd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vgatherdpd	%ymm0, (%rax,%xmm1,2), %ymm2
+# CHECK-NEXT:  8      28    1.33    *                   vgatherdps	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  12     30    2.67    *                   vgatherdps	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  5      20    0.67    *                   vgatherqpd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vgatherqpd	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  5      20    0.67    *                   vgatherqps	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vgatherqps	%xmm0, (%rax,%ymm1,2), %xmm2
+# CHECK-NEXT:  1      3     0.50                        vinserti128	$1, %xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vinserti128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      8     0.33    *                   vmovntdqa	(%rax), %ymm0
+# CHECK-NEXT:  2      4     1.00                        vmpsadbw	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  3      12    1.00    *                   vmpsadbw	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm0, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpabsb	(%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpabsd	%ymm0, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpabsd	(%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm0, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpabsw	(%rax), %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpackssdw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpackssdw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpacksswb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpacksswb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpackusdw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpackusdw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpackuswb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpackuswb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpaddb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpaddq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpaddsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpaddusb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddusb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpaddusw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddusw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.33                        vpalignr	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpalignr	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpand	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpand	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpandn	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpandn	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpavgb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpavgb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpavgw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpavgw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpblendd	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpblendd	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpblendd	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpblendd	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      3     0.75                        vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      9     2.33    *                   vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.33                        vpbroadcastb	%xmm0, %xmm0
+# CHECK-NEXT:  2      8     0.50    *                   vpbroadcastb	(%rax), %xmm0
+# CHECK-NEXT:  1      3     0.50                        vpbroadcastb	%xmm0, %ymm0
+# CHECK-NEXT:  1      8     2.33    *                   vpbroadcastb	(%rax), %ymm0
+# CHECK-NEXT:  1      1     0.33                        vpbroadcastd	%xmm0, %xmm0
+# CHECK-NEXT:  1      6     2.00    *                   vpbroadcastd	(%rax), %xmm0
+# CHECK-NEXT:  1      3     0.50                        vpbroadcastd	%xmm0, %ymm0
+# CHECK-NEXT:  1      7     2.33    *                   vpbroadcastd	(%rax), %ymm0
+# CHECK-NEXT:  1      1     0.33                        vpbroadcastq	%xmm0, %xmm0
+# CHECK-NEXT:  1      6     2.00    *                   vpbroadcastq	(%rax), %xmm0
+# CHECK-NEXT:  1      3     0.50                        vpbroadcastq	%xmm0, %ymm0
+# CHECK-NEXT:  1      7     2.33    *                   vpbroadcastq	(%rax), %ymm0
+# CHECK-NEXT:  1      1     0.33                        vpbroadcastw	%xmm0, %xmm0
+# CHECK-NEXT:  1      6     2.00    *                   vpbroadcastw	(%rax), %xmm0
+# CHECK-NEXT:  1      3     0.50                        vpbroadcastw	%xmm0, %ymm0
+# CHECK-NEXT:  1      7     2.33    *                   vpbroadcastw	(%rax), %ymm0
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpeqb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpeqd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpeqq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpeqw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpeqw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpgtb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpgtd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpermd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpermd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpermpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vpermpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpermps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vpermps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpermq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vpermq	$1, (%rax), %ymm2
+# CHECK-NEXT:  8      28    1.33    *                   vpgatherdd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  12     30    2.67    *                   vpgatherdd	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  5      20    0.67    *                   vpgatherdq	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vpgatherdq	%ymm0, (%rax,%xmm1,2), %ymm2
+# CHECK-NEXT:  5      20    0.67    *                   vpgatherqd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vpgatherqd	%xmm0, (%rax,%ymm1,2), %xmm2
+# CHECK-NEXT:  5      20    0.67    *                   vpgatherqq	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  8      29    1.33    *                   vpgatherqq	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      9     2.33    *                   vphaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      9     2.33    *                   vphsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  3      2     1.00                        vphsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  4      10    1.00    *                   vphsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmaddubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmaddubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmaddwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmaddwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  2      7     2.00    *                   vpmaskmovd	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      8     2.33    *                   vpmaskmovd	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  3      14    0.50    *      *            vpmaskmovd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  3      14    0.50    *      *            vpmaskmovd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  2      7     2.00    *                   vpmaskmovq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      8     2.33    *                   vpmaskmovq	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  3      14    0.50    *      *            vpmaskmovq	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  3      14    0.50    *      *            vpmaskmovq	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  1      1     0.50                        vpmaxsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxsd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxsd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxub	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxub	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxud	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxud	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpmaxuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpmaxuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminsd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminsd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminub	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminub	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminud	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminud	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpminuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpminuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpmovmskb	%ymm0, %ecx
+# CHECK-NEXT:  1      3     0.50                        vpmovsxbd	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxbd	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovsxbq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxbq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovsxbw	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxbw	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovsxdq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxdq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovsxwd	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxwd	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovsxwq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovsxwq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxbd	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxbd	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxbq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxbq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxbw	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxbw	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxdq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxdq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxwd	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxwd	(%rax), %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpmovzxwq	%xmm0, %ymm2
+# CHECK-NEXT:  2      10    2.33    *                   vpmovzxwq	(%rax), %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpmuldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpmuldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhrsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmulhrsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmulhuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmulhw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmulhw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  2      10    0.50                        vpmulld	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  3      18    0.50    *                   vpmulld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpmullw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpmullw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.50                        vpmuludq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      10    2.33    *                   vpmuludq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpor	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpor	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     0.50                        vpsadbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      11    0.50    *                   vpsadbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpshufb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpshufd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpshufd	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpshufhw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpshufhw	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpshuflw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      8     2.33    *                   vpshuflw	$1, (%rax), %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsignb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsignb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsignd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsignd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsignw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsignw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpslld	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpslld	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpslld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpslldq	$1, %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsllq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsllq	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsllq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsllvd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsllvd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsllvd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsllvd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsllvq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsllvq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsllvq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsllvq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsllw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsllw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsllw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrad	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsrad	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsrad	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsravd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsravd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsravd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsravd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsraw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsraw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsraw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrld	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsrld	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsrld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrldq	$1, %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsrlq	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsrlq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlvd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsrlvd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlvd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsrlvd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlvq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vpsrlvq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlvq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsrlvq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsrlw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  2      4     0.50                        vpsrlw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.33    *                   vpsrlw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsubsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsubusb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubusb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpsubusw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubusw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpckhbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpckhdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhqdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpckhqdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckhwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpckhwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpcklbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpckldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpckldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklqdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpcklqdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vpunpcklwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     0.50    *                   vpunpcklwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vpxor	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vpxor	(%rax), %ymm1, %ymm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     7.00    -     7.00    -     7.00   2.50   2.50   289.67 289.67 289.67 1.67   1.67   1.67    -     113.25 113.25 109.25 109.25
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vbroadcasti128	(%rax), %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vbroadcastsd	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vbroadcastss	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vextracti128	$1, %ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     vextracti128	$1, %ymm0, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vgatherdpd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vgatherdpd	%ymm0, (%rax,%xmm1,2), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vgatherdps	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     2.67   2.67   2.67    -      -      -      -     0.50   0.50   1.00   1.00   vgatherdps	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vgatherqpd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vgatherqpd	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vgatherqps	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vgatherqps	%xmm0, (%rax,%ymm1,2), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vinserti128	$1, %xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vinserti128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     vmovntdqa	(%rax), %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   vmpsadbw	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   vmpsadbw	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsb	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpabsb	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsd	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpabsd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpabsw	%ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpabsw	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackssdw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpackssdw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpacksswb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpacksswb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackusdw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpackusdw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpackuswb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpackuswb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpaddb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpaddq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpaddsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddusb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpaddusb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpaddusw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpaddusw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpalignr	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpalignr	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpand	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpand	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpandn	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpandn	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpavgb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpavgb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpavgw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpavgw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpblendd	$11, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpblendd	$11, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpblendd	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpblendd	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   vpblendvb	%ymm3, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.75   0.75   0.75   0.75   vpblendvb	%ymm3, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpblendw	$11, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpblendw	$11, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpbroadcastb	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpbroadcastb	(%rax), %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpbroadcastb	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpbroadcastb	(%rax), %ymm0
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpbroadcastd	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vpbroadcastd	(%rax), %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpbroadcastd	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vpbroadcastd	(%rax), %ymm0
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpbroadcastq	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vpbroadcastq	(%rax), %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpbroadcastq	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vpbroadcastq	(%rax), %ymm0
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     vpbroadcastw	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     vpbroadcastw	(%rax), %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpbroadcastw	%xmm0, %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -      -      -     vpbroadcastw	(%rax), %ymm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpeqb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpeqd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpeqq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpeqw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpeqw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpgtb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpgtd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   vpcmpgtq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpcmpgtq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpermd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermpd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermpd	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpermq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpermq	$1, (%rax), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vpgatherdd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     2.67   2.67   2.67    -      -      -      -     0.50   0.50   1.00   1.00   vpgatherdd	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vpgatherdq	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vpgatherdq	%ymm0, (%rax,%xmm1,2), %ymm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vpgatherqd	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vpgatherqd	%xmm0, (%rax,%ymm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.67   0.67   0.67    -      -      -      -     0.50   0.50   0.50   0.50   vpgatherqq	%xmm0, (%rax,%xmm1,2), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     1.33   1.33   1.33    -      -      -      -     0.50   0.50   1.00   1.00   vpgatherqq	%ymm0, (%rax,%ymm1,2), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphaddd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphaddd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   vphaddsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50   1.00   1.00   vphaddsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphaddw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphaddw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   vphsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50   1.00   1.00   vphsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   vphsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   vphsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmaddubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmaddubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmaddwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmaddwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpmaskmovd	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpmaskmovd	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vpmaskmovd	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vpmaskmovd	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   vpmaskmovq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpmaskmovq	(%rax), %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vpmaskmovq	%xmm0, %xmm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     vpmaskmovq	%ymm0, %ymm1, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxsd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxub	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxub	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxud	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxud	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpmaxuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpmaxuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminsd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminub	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminub	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminud	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminud	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpminuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpminuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmovmskb	%ymm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxbd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxbq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxbw	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxbw	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxdq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxdq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxwd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxwd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovsxwq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovsxwq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxbd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxbq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxbw	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxbw	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxdq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxdq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxwd	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxwd	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpmovzxwq	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpmovzxwq	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmuldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     1.50   1.50    -      -     vpmuldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhrsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmulhrsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhuw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmulhuw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmulhw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmulhw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpmulld	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   vpmulld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpmullw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpmullw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     vpmuludq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     1.50   1.50    -      -     vpmuludq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpor	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpor	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpsadbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpsadbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpshufb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufd	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpshufd	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshufhw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpshufhw	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpshuflw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     0.50   0.50   vpshuflw	$1, (%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsignb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsignd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsignw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsignw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpslld	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpslld	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpslld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpslldq	$1, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsllq	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsllq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllvd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsllvd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllvd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsllvd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllvq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsllvq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllvq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsllvq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsllw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsllw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsllw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrad	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrad	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrad	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsravd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsravd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsravd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsravd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsraw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsraw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsraw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrld	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrld	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrld	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpsrldq	$1, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlq	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrlq	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrlq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlvd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsrlvd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlvd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrlvd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlvq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vpsrlvq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlvq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrlvq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsrlw	$1, %ymm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vpsrlw	%xmm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsrlw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpsubb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpsubd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpsubq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubsb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsubsb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubsw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsubsw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubusb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsubusb	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpsubusw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vpsubusw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpsubw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpsubw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhqdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhqdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckhwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckhwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklbw	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklbw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpckldq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpckldq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklqdq	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklqdq	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpunpcklwd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   vpunpcklwd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   vpxor	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.25   0.25   0.25   0.25   vpxor	(%rax), %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxgfni.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxgfni.s
new file mode 100644
index 000000000000..1cc5f46769e8
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxgfni.s
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vgf2p8affineinvqb   $0, %xmm0, %xmm1, %xmm2
+vgf2p8affineinvqb   $0, (%rax), %xmm1, %xmm2
+
+vgf2p8affineinvqb   $0, %ymm0, %ymm1, %ymm2
+vgf2p8affineinvqb   $0, (%rax), %ymm1, %ymm2
+
+vgf2p8affineqb      $0, %xmm0, %xmm1, %xmm2
+vgf2p8affineqb      $0, (%rax), %xmm1, %xmm2
+
+vgf2p8affineqb      $0, %ymm0, %ymm1, %ymm2
+vgf2p8affineqb      $0, (%rax), %ymm1, %ymm2
+
+vgf2p8mulb          %xmm0, %xmm1, %xmm2
+vgf2p8mulb          (%rax), %xmm1, %xmm2
+
+vgf2p8mulb          %ymm0, %ymm1, %ymm2
+vgf2p8mulb          (%rax), %ymm1, %ymm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     2.00                        vgf2p8affineinvqb	$0, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vgf2p8affineinvqb	$0, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vgf2p8affineinvqb	$0, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vgf2p8affineinvqb	$0, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vgf2p8affineqb	$0, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vgf2p8affineqb	$0, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vgf2p8affineqb	$0, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vgf2p8affineqb	$0, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     0.50                        vgf2p8mulb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   vgf2p8mulb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vgf2p8mulb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     2.33    *                   vgf2p8mulb	(%rax), %ymm1, %ymm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     13.00  13.00  13.00   -      -      -      -     18.00  18.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vgf2p8affineinvqb	$0, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vgf2p8affineinvqb	$0, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vgf2p8affineinvqb	$0, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vgf2p8affineinvqb	$0, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vgf2p8affineqb	$0, %xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vgf2p8affineqb	$0, (%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vgf2p8affineqb	$0, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vgf2p8affineqb	$0, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vgf2p8mulb	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     vgf2p8mulb	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vgf2p8mulb	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     0.50   0.50    -      -     vgf2p8mulb	(%rax), %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxvnni.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxvnni.s
new file mode 100644
index 000000000000..38d9959675f1
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-avxvnni.s
@@ -0,0 +1,96 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vpdpbusd    %xmm0, %xmm1, %xmm2
+vpdpbusd    (%rax), %xmm1, %xmm2
+
+vpdpbusd    %ymm0, %ymm1, %ymm2
+vpdpbusd    (%rax), %ymm1, %ymm2
+
+vpdpbusds   %xmm0, %xmm1, %xmm2
+vpdpbusds   (%rax), %xmm1, %xmm2
+
+vpdpbusds   %ymm0, %ymm1, %ymm2
+vpdpbusds   (%rax), %ymm1, %ymm2
+
+vpdpwssd    %xmm0, %xmm1, %xmm2
+vpdpwssd    (%rax), %xmm1, %xmm2
+
+vpdpwssd    %ymm0, %ymm1, %ymm2
+vpdpwssd    (%rax), %ymm1, %ymm2
+
+vpdpwssds   %xmm0, %xmm1, %xmm2
+vpdpwssds   (%rax), %xmm1, %xmm2
+
+vpdpwssds   %ymm0, %ymm1, %ymm2
+vpdpwssds   (%rax), %ymm1, %ymm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     2.00                        vpdpbusd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpdpbusd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpdpbusd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpdpbusd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpdpbusds	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpdpbusds	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpdpbusds	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpdpbusds	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpdpwssd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpdpwssd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpdpwssd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpdpwssd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vpdpwssds	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vpdpwssds	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vpdpwssds	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vpdpwssds	(%rax), %ymm1, %ymm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     17.33  17.33  17.33   -      -      -      -     32.00  32.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpbusd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpdpbusd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpbusd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpdpbusd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpbusds	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpdpbusds	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpbusds	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpdpbusds	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpwssd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpdpwssd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpwssd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpdpwssd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpwssds	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vpdpwssds	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vpdpwssds	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vpdpwssds	(%rax), %ymm1, %ymm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-bmi2.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-bmi2.s
new file mode 100644
index 000000000000..a92f6372f3b9
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-bmi2.s
@@ -0,0 +1,152 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+bzhi        %eax, %ebx, %ecx
+bzhi        %eax, (%rbx), %ecx
+
+bzhi        %rax, %rbx, %rcx
+bzhi        %rax, (%rbx), %rcx
+
+mulx        %eax, %ebx, %ecx
+mulx        (%rax), %ebx, %ecx
+
+mulx        %rax, %rbx, %rcx
+mulx        (%rax), %rbx, %rcx
+
+pdep        %eax, %ebx, %ecx
+pdep        (%rax), %ebx, %ecx
+
+pdep        %rax, %rbx, %rcx
+pdep        (%rax), %rbx, %rcx
+
+pext        %eax, %ebx, %ecx
+pext        (%rax), %ebx, %ecx
+
+pext        %rax, %rbx, %rcx
+pext        (%rax), %rbx, %rcx
+
+rorx        $1, %eax, %ecx
+rorx        $1, (%rax), %ecx
+
+rorx        $1, %rax, %rcx
+rorx        $1, (%rax), %rcx
+
+sarx        %eax, %ebx, %ecx
+sarx        %eax, (%rbx), %ecx
+
+sarx        %rax, %rbx, %rcx
+sarx        %rax, (%rbx), %rcx
+
+shlx        %eax, %ebx, %ecx
+shlx        %eax, (%rbx), %ecx
+
+shlx        %rax, %rbx, %rcx
+shlx        %rax, (%rbx), %rcx
+
+shrx        %eax, %ebx, %ecx
+shrx        %eax, (%rbx), %ecx
+
+shrx        %rax, %rbx, %rcx
+shrx        %rax, (%rbx), %rcx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.33                        bzhil	%eax, %ebx, %ecx
+# CHECK-NEXT:  2      7     0.33    *                   bzhil	%eax, (%rbx), %ecx
+# CHECK-NEXT:  1      3     0.33                        bzhiq	%rax, %rbx, %rcx
+# CHECK-NEXT:  2      7     0.33    *                   bzhiq	%rax, (%rbx), %rcx
+# CHECK-NEXT:  3      4     0.67                        mulxl	%eax, %ebx, %ecx
+# CHECK-NEXT:  4      8     0.67    *                   mulxl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  2      4     0.67                        mulxq	%rax, %rbx, %rcx
+# CHECK-NEXT:  3      8     0.67    *                   mulxq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  1      3     1.00                        pdepl	%eax, %ebx, %ecx
+# CHECK-NEXT:  1      7     1.33    *                   pdepl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  1      3     1.00                        pdepq	%rax, %rbx, %rcx
+# CHECK-NEXT:  1      7     1.33    *                   pdepq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  1      3     1.00                        pextl	%eax, %ebx, %ecx
+# CHECK-NEXT:  1      7     1.33    *                   pextl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  1      3     1.00                        pextq	%rax, %rbx, %rcx
+# CHECK-NEXT:  1      7     1.33    *                   pextq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  1      1     0.33                        rorxl	$1, %eax, %ecx
+# CHECK-NEXT:  2      6     0.33    *                   rorxl	$1, (%rax), %ecx
+# CHECK-NEXT:  1      1     0.33                        rorxq	$1, %rax, %rcx
+# CHECK-NEXT:  2      6     0.33    *                   rorxq	$1, (%rax), %rcx
+# CHECK-NEXT:  1      3     0.33                        sarxl	%eax, %ebx, %ecx
+# CHECK-NEXT:  2      8     0.33    *                   sarxl	%eax, (%rbx), %ecx
+# CHECK-NEXT:  1      3     0.33                        sarxq	%rax, %rbx, %rcx
+# CHECK-NEXT:  2      8     0.33    *                   sarxq	%rax, (%rbx), %rcx
+# CHECK-NEXT:  1      3     0.33                        shlxl	%eax, %ebx, %ecx
+# CHECK-NEXT:  2      8     0.33    *                   shlxl	%eax, (%rbx), %ecx
+# CHECK-NEXT:  1      3     0.33                        shlxq	%rax, %rbx, %rcx
+# CHECK-NEXT:  2      8     0.33    *                   shlxq	%rax, (%rbx), %rcx
+# CHECK-NEXT:  1      3     0.33                        shrxl	%eax, %ebx, %ecx
+# CHECK-NEXT:  2      8     0.33    *                   shrxl	%eax, (%rbx), %ecx
+# CHECK-NEXT:  1      3     0.33                        shrxq	%rax, %rbx, %rcx
+# CHECK-NEXT:  2      8     0.33    *                   shrxq	%rax, (%rbx), %rcx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 5.00   13.00  5.00   13.00  5.00   13.00   -      -     9.33   9.33   9.33    -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bzhil	%eax, %ebx, %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bzhil	%eax, (%rbx), %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bzhiq	%rax, %rbx, %rcx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bzhiq	%rax, (%rbx), %rcx
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     mulxl	%eax, %ebx, %ecx
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mulxl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     mulxq	%rax, %rbx, %rcx
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mulxq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pdepl	%eax, %ebx, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     pdepl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pdepq	%rax, %rbx, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     pdepq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pextl	%eax, %ebx, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     pextl	(%rax), %ebx, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pextq	%rax, %rbx, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     pextq	(%rax), %rbx, %rcx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     rorxl	$1, %eax, %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     rorxl	$1, (%rax), %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     rorxq	$1, %rax, %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     rorxq	$1, (%rax), %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarxl	%eax, %ebx, %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sarxl	%eax, (%rbx), %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarxq	%rax, %rbx, %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sarxq	%rax, (%rbx), %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shlxl	%eax, %ebx, %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     shlxl	%eax, (%rbx), %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shlxq	%rax, %rbx, %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     shlxq	%rax, (%rbx), %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrxl	%eax, %ebx, %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     shrxl	%eax, (%rbx), %ecx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrxq	%rax, %rbx, %rcx
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     shrxq	%rax, (%rbx), %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clflushopt.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clflushopt.s
new file mode 100644
index 000000000000..0632f801d177
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clflushopt.s
@@ -0,0 +1,44 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+clflushopt (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  3      2     0.50    *      *      U     clflushopt	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     clflushopt	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clwb.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clwb.s
new file mode 100644
index 000000000000..28fc2e7446a9
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-clwb.s
@@ -0,0 +1,44 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+clwb (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  3      5     0.50    *      *      U     clwb	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     clwb	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmov.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmov.s
new file mode 100644
index 000000000000..8185a70fcd47
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmov.s
@@ -0,0 +1,334 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+cmovow    %si, %di
+cmovnow   %si, %di
+cmovbw    %si, %di
+cmovaew   %si, %di
+cmovew    %si, %di
+cmovnew   %si, %di
+cmovbew   %si, %di
+cmovaw    %si, %di
+cmovsw    %si, %di
+cmovnsw   %si, %di
+cmovpw    %si, %di
+cmovnpw   %si, %di
+cmovlw    %si, %di
+cmovgew   %si, %di
+cmovlew   %si, %di
+cmovgw    %si, %di
+
+cmovow    (%rax), %di
+cmovnow   (%rax), %di
+cmovbw    (%rax), %di
+cmovaew   (%rax), %di
+cmovew    (%rax), %di
+cmovnew   (%rax), %di
+cmovbew   (%rax), %di
+cmovaw    (%rax), %di
+cmovsw    (%rax), %di
+cmovnsw   (%rax), %di
+cmovpw    (%rax), %di
+cmovnpw   (%rax), %di
+cmovlw    (%rax), %di
+cmovgew   (%rax), %di
+cmovlew   (%rax), %di
+cmovgw    (%rax), %di
+
+cmovol    %esi, %edi
+cmovnol   %esi, %edi
+cmovbl    %esi, %edi
+cmovael   %esi, %edi
+cmovel    %esi, %edi
+cmovnel   %esi, %edi
+cmovbel   %esi, %edi
+cmoval    %esi, %edi
+cmovsl    %esi, %edi
+cmovnsl   %esi, %edi
+cmovpl    %esi, %edi
+cmovnpl   %esi, %edi
+cmovll    %esi, %edi
+cmovgel   %esi, %edi
+cmovlel   %esi, %edi
+cmovgl    %esi, %edi
+
+cmovol    (%rax), %edi
+cmovnol   (%rax), %edi
+cmovbl    (%rax), %edi
+cmovael   (%rax), %edi
+cmovel    (%rax), %edi
+cmovnel   (%rax), %edi
+cmovbel   (%rax), %edi
+cmoval    (%rax), %edi
+cmovsl    (%rax), %edi
+cmovnsl   (%rax), %edi
+cmovpl    (%rax), %edi
+cmovnpl   (%rax), %edi
+cmovll    (%rax), %edi
+cmovgel   (%rax), %edi
+cmovlel   (%rax), %edi
+cmovgl    (%rax), %edi
+
+cmovoq    %rsi, %rdi
+cmovnoq   %rsi, %rdi
+cmovbq    %rsi, %rdi
+cmovaeq   %rsi, %rdi
+cmoveq    %rsi, %rdi
+cmovneq   %rsi, %rdi
+cmovbeq   %rsi, %rdi
+cmovaq    %rsi, %rdi
+cmovsq    %rsi, %rdi
+cmovnsq   %rsi, %rdi
+cmovpq    %rsi, %rdi
+cmovnpq   %rsi, %rdi
+cmovlq    %rsi, %rdi
+cmovgeq   %rsi, %rdi
+cmovleq   %rsi, %rdi
+cmovgq    %rsi, %rdi
+
+cmovoq    (%rax), %rdi
+cmovnoq   (%rax), %rdi
+cmovbq    (%rax), %rdi
+cmovaeq   (%rax), %rdi
+cmoveq    (%rax), %rdi
+cmovneq   (%rax), %rdi
+cmovbeq   (%rax), %rdi
+cmovaq    (%rax), %rdi
+cmovsq    (%rax), %rdi
+cmovnsq   (%rax), %rdi
+cmovpq    (%rax), %rdi
+cmovnpq   (%rax), %rdi
+cmovlq    (%rax), %rdi
+cmovgeq   (%rax), %rdi
+cmovleq   (%rax), %rdi
+cmovgq    (%rax), %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.17                        cmovow	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovnow	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovbw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovaew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovnew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovbew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovaw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovsw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovnsw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovpw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovnpw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovlw	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovgew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovlew	%si, %di
+# CHECK-NEXT:  1      1     0.17                        cmovgw	%si, %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovow	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovnow	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovbw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovaew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovnew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovbew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovaw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovsw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovnsw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovpw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovnpw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovlw	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovgew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovlew	(%rax), %di
+# CHECK-NEXT:  1      5     1.33    *                   cmovgw	(%rax), %di
+# CHECK-NEXT:  1      1     0.17                        cmovol	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovnol	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovbl	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovael	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovel	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovnel	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovbel	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmoval	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovsl	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovnsl	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovpl	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovnpl	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovll	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovgel	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovlel	%esi, %edi
+# CHECK-NEXT:  1      1     0.17                        cmovgl	%esi, %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovol	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnol	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovbl	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovael	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovel	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnel	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovbel	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmoval	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovsl	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnsl	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovpl	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnpl	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovll	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovgel	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovlel	(%rax), %edi
+# CHECK-NEXT:  1      5     1.33    *                   cmovgl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.17                        cmovoq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovnoq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovbq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovaeq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmoveq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovneq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovbeq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovaq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovsq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovnsq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovpq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovnpq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovlq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovgeq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovleq	%rsi, %rdi
+# CHECK-NEXT:  1      1     0.17                        cmovgq	%rsi, %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovoq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnoq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovbq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovaeq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmoveq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovneq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovbeq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovaq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovsq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnsq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovpq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovnpq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovlq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovgeq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovleq	(%rax), %rdi
+# CHECK-NEXT:  1      5     1.33    *                   cmovgq	(%rax), %rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 16.00  16.00  16.00  16.00  16.00  16.00   -      -     64.00  64.00  64.00   -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovow	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnow	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovaew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovaw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovsw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnsw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovpw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnpw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovlw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovlew	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovow	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnow	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovaew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovaw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovsw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnsw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovpw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnpw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovlw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovlew	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovol	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnol	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovael	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovel	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnel	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbel	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmoval	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovsl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnsl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovpl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnpl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovll	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgel	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovlel	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovol	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnol	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovael	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovel	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnel	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbel	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmoval	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovsl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnsl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovpl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnpl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovll	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgel	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovlel	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovoq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnoq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovaeq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmoveq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovneq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovbeq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovaq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovsq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnsq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovpq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovnpq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovlq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgeq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovleq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cmovgq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovoq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnoq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovaeq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmoveq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovneq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovbeq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovaq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovsq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnsq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovpq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovnpq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovlq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgeq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovleq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     cmovgq	(%rax), %rdi
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmpxchg.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmpxchg.s
new file mode 100644
index 000000000000..9297bbc8b54e
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-cmpxchg.s
@@ -0,0 +1,53 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+cmpxchg8b  (%rax)
+cmpxchg16b (%rax)
+lock cmpxchg8b  (%rax)
+lock cmpxchg16b (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  16     25    2.33    *      *            cmpxchg8b	(%rax)
+# CHECK-NEXT:  22     32    2.83    *      *            cmpxchg16b	(%rax)
+# CHECK-NEXT:  16     25    2.33    *      *            lock		cmpxchg8b	(%rax)
+# CHECK-NEXT:  22     32    2.83    *      *            lock		cmpxchg16b	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 16.33  9.00   12.33  5.00   12.33  5.00   2.00   2.00   1.33   1.33   1.33   1.33   1.33   1.33    -      -      -     2.00   2.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 3.00   2.67   3.00   0.67   3.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     cmpxchg8b	(%rax)
+# CHECK-NEXT: 5.17   1.83   3.17   1.83   3.17   1.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -     1.00   1.00   cmpxchg16b	(%rax)
+# CHECK-NEXT: 3.00   2.67   3.00   0.67   3.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		cmpxchg8b	(%rax)
+# CHECK-NEXT: 5.17   1.83   3.17   1.83   3.17   1.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -     1.00   1.00   lock		cmpxchg16b	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-f16c.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-f16c.s
new file mode 100644
index 000000000000..b6fa7a599cf4
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-f16c.s
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vcvtph2ps   %xmm0, %xmm2
+vcvtph2ps   (%rax), %xmm2
+
+vcvtph2ps   %xmm0, %ymm2
+vcvtph2ps   (%rax), %ymm2
+
+vcvtps2ph   $0, %xmm0, %xmm2
+vcvtps2ph   $0, %xmm0, (%rax)
+
+vcvtps2ph   $0, %ymm0, %xmm2
+vcvtps2ph   $0, %ymm0, (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      5     0.50                        vcvtph2ps	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50    *                   vcvtph2ps	(%rax), %xmm2
+# CHECK-NEXT:  2      7     2.00                        vcvtph2ps	%xmm0, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vcvtph2ps	(%rax), %ymm2
+# CHECK-NEXT:  2      5     2.00                        vcvtps2ph	$0, %xmm0, %xmm2
+# CHECK-NEXT:  2      5     2.00           *            vcvtps2ph	$0, %xmm0, (%rax)
+# CHECK-NEXT:  2      7     2.00                        vcvtps2ph	$0, %ymm0, %xmm2
+# CHECK-NEXT:  3      5     2.00           *            vcvtps2ph	$0, %ymm0, (%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00   1.00   2.67   2.67   2.67   0.67   0.67   0.67    -     13.00  13.00  4.00   4.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   vcvtph2ps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     vcvtph2ps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   1.50   1.50   vcvtph2ps	%xmm0, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vcvtph2ps	(%rax), %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   vcvtps2ph	$0, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     2.00   2.00    -      -     vcvtps2ph	$0, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   1.50   1.50   vcvtps2ph	$0, %ymm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     2.00   2.00    -      -     vcvtps2ph	$0, %ymm0, (%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fma.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fma.s
new file mode 100644
index 000000000000..626f9306cbf6
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fma.s
@@ -0,0 +1,712 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vfmadd132pd %xmm0, %xmm1, %xmm2
+vfmadd132pd (%rax), %xmm1, %xmm2
+
+vfmadd132pd %ymm0, %ymm1, %ymm2
+vfmadd132pd (%rax), %ymm1, %ymm2
+
+vfmadd213pd %xmm0, %xmm1, %xmm2
+vfmadd213pd (%rax), %xmm1, %xmm2
+
+vfmadd213pd %ymm0, %ymm1, %ymm2
+vfmadd213pd (%rax), %ymm1, %ymm2
+
+vfmadd231pd %xmm0, %xmm1, %xmm2
+vfmadd231pd (%rax), %xmm1, %xmm2
+
+vfmadd231pd %ymm0, %ymm1, %ymm2
+vfmadd231pd (%rax), %ymm1, %ymm2
+
+vfmadd132ps %xmm0, %xmm1, %xmm2
+vfmadd132ps (%rax), %xmm1, %xmm2
+
+vfmadd132ps %ymm0, %ymm1, %ymm2
+vfmadd132ps (%rax), %ymm1, %ymm2
+
+vfmadd213ps %xmm0, %xmm1, %xmm2
+vfmadd213ps (%rax), %xmm1, %xmm2
+
+vfmadd213ps %ymm0, %ymm1, %ymm2
+vfmadd213ps (%rax), %ymm1, %ymm2
+
+vfmadd231ps %xmm0, %xmm1, %xmm2
+vfmadd231ps (%rax), %xmm1, %xmm2
+
+vfmadd231ps %ymm0, %ymm1, %ymm2
+vfmadd231ps (%rax), %ymm1, %ymm2
+
+vfmadd132sd %xmm0, %xmm1, %xmm2
+vfmadd132sd (%rax), %xmm1, %xmm2
+
+vfmadd213sd %xmm0, %xmm1, %xmm2
+vfmadd213sd (%rax), %xmm1, %xmm2
+
+vfmadd231sd %xmm0, %xmm1, %xmm2
+vfmadd231sd (%rax), %xmm1, %xmm2
+
+vfmadd132ss %xmm0, %xmm1, %xmm2
+vfmadd132ss (%rax), %xmm1, %xmm2
+
+vfmadd213ss %xmm0, %xmm1, %xmm2
+vfmadd213ss (%rax), %xmm1, %xmm2
+
+vfmadd231ss %xmm0, %xmm1, %xmm2
+vfmadd231ss (%rax), %xmm1, %xmm2
+
+vfmaddsub132pd %xmm0, %xmm1, %xmm2
+vfmaddsub132pd (%rax), %xmm1, %xmm2
+
+vfmaddsub132pd %ymm0, %ymm1, %ymm2
+vfmaddsub132pd (%rax), %ymm1, %ymm2
+
+vfmaddsub213pd %xmm0, %xmm1, %xmm2
+vfmaddsub213pd (%rax), %xmm1, %xmm2
+
+vfmaddsub213pd %ymm0, %ymm1, %ymm2
+vfmaddsub213pd (%rax), %ymm1, %ymm2
+
+vfmaddsub231pd %xmm0, %xmm1, %xmm2
+vfmaddsub231pd (%rax), %xmm1, %xmm2
+
+vfmaddsub231pd %ymm0, %ymm1, %ymm2
+vfmaddsub231pd (%rax), %ymm1, %ymm2
+
+vfmaddsub132ps %xmm0, %xmm1, %xmm2
+vfmaddsub132ps (%rax), %xmm1, %xmm2
+
+vfmaddsub132ps %ymm0, %ymm1, %ymm2
+vfmaddsub132ps (%rax), %ymm1, %ymm2
+
+vfmaddsub213ps %xmm0, %xmm1, %xmm2
+vfmaddsub213ps (%rax), %xmm1, %xmm2
+
+vfmaddsub213ps %ymm0, %ymm1, %ymm2
+vfmaddsub213ps (%rax), %ymm1, %ymm2
+
+vfmaddsub231ps %xmm0, %xmm1, %xmm2
+vfmaddsub231ps (%rax), %xmm1, %xmm2
+
+vfmaddsub231ps %ymm0, %ymm1, %ymm2
+vfmaddsub231ps (%rax), %ymm1, %ymm2
+
+vfmsub132pd %xmm0, %xmm1, %xmm2
+vfmsub132pd (%rax), %xmm1, %xmm2
+
+vfmsub132pd %ymm0, %ymm1, %ymm2
+vfmsub132pd (%rax), %ymm1, %ymm2
+
+vfmsub213pd %xmm0, %xmm1, %xmm2
+vfmsub213pd (%rax), %xmm1, %xmm2
+
+vfmsub213pd %ymm0, %ymm1, %ymm2
+vfmsub213pd (%rax), %ymm1, %ymm2
+
+vfmsub231pd %xmm0, %xmm1, %xmm2
+vfmsub231pd (%rax), %xmm1, %xmm2
+
+vfmsub231pd %ymm0, %ymm1, %ymm2
+vfmsub231pd (%rax), %ymm1, %ymm2
+
+vfmsub132ps %xmm0, %xmm1, %xmm2
+vfmsub132ps (%rax), %xmm1, %xmm2
+
+vfmsub132ps %ymm0, %ymm1, %ymm2
+vfmsub132ps (%rax), %ymm1, %ymm2
+
+vfmsub213ps %xmm0, %xmm1, %xmm2
+vfmsub213ps (%rax), %xmm1, %xmm2
+
+vfmsub213ps %ymm0, %ymm1, %ymm2
+vfmsub213ps (%rax), %ymm1, %ymm2
+
+vfmsub231ps %xmm0, %xmm1, %xmm2
+vfmsub231ps (%rax), %xmm1, %xmm2
+
+vfmsub231ps %ymm0, %ymm1, %ymm2
+vfmsub231ps (%rax), %ymm1, %ymm2
+
+vfmsub132sd %xmm0, %xmm1, %xmm2
+vfmsub132sd (%rax), %xmm1, %xmm2
+
+vfmsub213sd %xmm0, %xmm1, %xmm2
+vfmsub213sd (%rax), %xmm1, %xmm2
+
+vfmsub231sd %xmm0, %xmm1, %xmm2
+vfmsub231sd (%rax), %xmm1, %xmm2
+
+vfmsub132ss %xmm0, %xmm1, %xmm2
+vfmsub132ss (%rax), %xmm1, %xmm2
+
+vfmsub213ss %xmm0, %xmm1, %xmm2
+vfmsub213ss (%rax), %xmm1, %xmm2
+
+vfmsub231ss %xmm0, %xmm1, %xmm2
+vfmsub231ss (%rax), %xmm1, %xmm2
+
+vfmsubadd132pd %xmm0, %xmm1, %xmm2
+vfmsubadd132pd (%rax), %xmm1, %xmm2
+
+vfmsubadd132pd %ymm0, %ymm1, %ymm2
+vfmsubadd132pd (%rax), %ymm1, %ymm2
+
+vfmsubadd213pd %xmm0, %xmm1, %xmm2
+vfmsubadd213pd (%rax), %xmm1, %xmm2
+
+vfmsubadd213pd %ymm0, %ymm1, %ymm2
+vfmsubadd213pd (%rax), %ymm1, %ymm2
+
+vfmsubadd231pd %xmm0, %xmm1, %xmm2
+vfmsubadd231pd (%rax), %xmm1, %xmm2
+
+vfmsubadd231pd %ymm0, %ymm1, %ymm2
+vfmsubadd231pd (%rax), %ymm1, %ymm2
+
+vfmsubadd132ps %xmm0, %xmm1, %xmm2
+vfmsubadd132ps (%rax), %xmm1, %xmm2
+
+vfmsubadd132ps %ymm0, %ymm1, %ymm2
+vfmsubadd132ps (%rax), %ymm1, %ymm2
+
+vfmsubadd213ps %xmm0, %xmm1, %xmm2
+vfmsubadd213ps (%rax), %xmm1, %xmm2
+
+vfmsubadd213ps %ymm0, %ymm1, %ymm2
+vfmsubadd213ps (%rax), %ymm1, %ymm2
+
+vfmsubadd231ps %xmm0, %xmm1, %xmm2
+vfmsubadd231ps (%rax), %xmm1, %xmm2
+
+vfmsubadd231ps %ymm0, %ymm1, %ymm2
+vfmsubadd231ps (%rax), %ymm1, %ymm2
+
+vfnmadd132pd %xmm0, %xmm1, %xmm2
+vfnmadd132pd (%rax), %xmm1, %xmm2
+
+vfnmadd132pd %ymm0, %ymm1, %ymm2
+vfnmadd132pd (%rax), %ymm1, %ymm2
+
+vfnmadd213pd %xmm0, %xmm1, %xmm2
+vfnmadd213pd (%rax), %xmm1, %xmm2
+
+vfnmadd213pd %ymm0, %ymm1, %ymm2
+vfnmadd213pd (%rax), %ymm1, %ymm2
+
+vfnmadd231pd %xmm0, %xmm1, %xmm2
+vfnmadd231pd (%rax), %xmm1, %xmm2
+
+vfnmadd231pd %ymm0, %ymm1, %ymm2
+vfnmadd231pd (%rax), %ymm1, %ymm2
+
+vfnmadd132ps %xmm0, %xmm1, %xmm2
+vfnmadd132ps (%rax), %xmm1, %xmm2
+
+vfnmadd132ps %ymm0, %ymm1, %ymm2
+vfnmadd132ps (%rax), %ymm1, %ymm2
+
+vfnmadd213ps %xmm0, %xmm1, %xmm2
+vfnmadd213ps (%rax), %xmm1, %xmm2
+
+vfnmadd213ps %ymm0, %ymm1, %ymm2
+vfnmadd213ps (%rax), %ymm1, %ymm2
+
+vfnmadd231ps %xmm0, %xmm1, %xmm2
+vfnmadd231ps (%rax), %xmm1, %xmm2
+
+vfnmadd231ps %ymm0, %ymm1, %ymm2
+vfnmadd231ps (%rax), %ymm1, %ymm2
+
+vfnmadd132sd %xmm0, %xmm1, %xmm2
+vfnmadd132sd (%rax), %xmm1, %xmm2
+
+vfnmadd213sd %xmm0, %xmm1, %xmm2
+vfnmadd213sd (%rax), %xmm1, %xmm2
+
+vfnmadd231sd %xmm0, %xmm1, %xmm2
+vfnmadd231sd (%rax), %xmm1, %xmm2
+
+vfnmadd132ss %xmm0, %xmm1, %xmm2
+vfnmadd132ss (%rax), %xmm1, %xmm2
+
+vfnmadd213ss %xmm0, %xmm1, %xmm2
+vfnmadd213ss (%rax), %xmm1, %xmm2
+
+vfnmadd231ss %xmm0, %xmm1, %xmm2
+vfnmadd231ss (%rax), %xmm1, %xmm2
+
+vfnmsub132pd %xmm0, %xmm1, %xmm2
+vfnmsub132pd (%rax), %xmm1, %xmm2
+
+vfnmsub132pd %ymm0, %ymm1, %ymm2
+vfnmsub132pd (%rax), %ymm1, %ymm2
+
+vfnmsub213pd %xmm0, %xmm1, %xmm2
+vfnmsub213pd (%rax), %xmm1, %xmm2
+
+vfnmsub213pd %ymm0, %ymm1, %ymm2
+vfnmsub213pd (%rax), %ymm1, %ymm2
+
+vfnmsub231pd %xmm0, %xmm1, %xmm2
+vfnmsub231pd (%rax), %xmm1, %xmm2
+
+vfnmsub231pd %ymm0, %ymm1, %ymm2
+vfnmsub231pd (%rax), %ymm1, %ymm2
+
+vfnmsub132ps %xmm0, %xmm1, %xmm2
+vfnmsub132ps (%rax), %xmm1, %xmm2
+
+vfnmsub132ps %ymm0, %ymm1, %ymm2
+vfnmsub132ps (%rax), %ymm1, %ymm2
+
+vfnmsub213ps %xmm0, %xmm1, %xmm2
+vfnmsub213ps (%rax), %xmm1, %xmm2
+
+vfnmsub213ps %ymm0, %ymm1, %ymm2
+vfnmsub213ps (%rax), %ymm1, %ymm2
+
+vfnmsub231ps %xmm0, %xmm1, %xmm2
+vfnmsub231ps (%rax), %xmm1, %xmm2
+
+vfnmsub231ps %ymm0, %ymm1, %ymm2
+vfnmsub231ps (%rax), %ymm1, %ymm2
+
+vfnmsub132sd %xmm0, %xmm1, %xmm2
+vfnmsub132sd (%rax), %xmm1, %xmm2
+
+vfnmsub213sd %xmm0, %xmm1, %xmm2
+vfnmsub213sd (%rax), %xmm1, %xmm2
+
+vfnmsub231sd %xmm0, %xmm1, %xmm2
+vfnmsub231sd (%rax), %xmm1, %xmm2
+
+vfnmsub132ss %xmm0, %xmm1, %xmm2
+vfnmsub132ss (%rax), %xmm1, %xmm2
+
+vfnmsub213ss %xmm0, %xmm1, %xmm2
+vfnmsub213ss (%rax), %xmm1, %xmm2
+
+vfnmsub231ss %xmm0, %xmm1, %xmm2
+vfnmsub231ss (%rax), %xmm1, %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     2.00                        vfmadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmadd231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmadd231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmaddsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmaddsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmaddsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsub231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsub231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfmsubadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfmsubadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfmsubadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmadd231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmadd231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      11    2.33    *                   vfnmsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      4     2.00                        vfnmsub231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   vfnmsub231ss	(%rax), %xmm1, %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     204.00 204.00 204.00  -      -      -      -     384.00 384.00  -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmadd231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmadd231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmaddsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmaddsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmaddsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsub231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsub231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfmsubadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfmsubadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfmsubadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmadd231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmadd231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmadd231ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub132pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub132pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub213pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub213pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231pd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub231pd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231pd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub231pd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub132ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub132ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub213ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub213ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231ps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub231ps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231ps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vfnmsub231ps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub132sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub213sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231sd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub231sd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub132ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub132ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub213ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub213ss	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vfnmsub231ss	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     vfnmsub231ss	(%rax), %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fsgsbase.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fsgsbase.s
new file mode 100644
index 000000000000..eadfbe64bac0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-fsgsbase.s
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.17    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.17    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.17    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.17    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.17    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.17    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.17    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.17    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 1.33   1.33   1.33   1.33   1.33   1.33    -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdfsbasel	%eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdfsbaseq	%rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdgsbasel	%eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdgsbaseq	%rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     wrfsbasel	%edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     wrfsbaseq	%rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     wrgsbasel	%edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     wrgsbaseq	%rdi
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-gfni.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-gfni.s
new file mode 100644
index 000000000000..cc82c5b358e5
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-gfni.s
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+gf2p8affineinvqb    $0, %xmm0, %xmm1
+gf2p8affineinvqb    $0, (%rax), %xmm1
+
+gf2p8affineqb       $0, %xmm0, %xmm1
+gf2p8affineqb       $0, (%rax), %xmm1
+
+gf2p8mulb           %xmm0, %xmm1
+gf2p8mulb           (%rax), %xmm1
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     2.00                        gf2p8affineinvqb	$0, %xmm0, %xmm1
+# CHECK-NEXT:  2      10    2.00    *                   gf2p8affineinvqb	$0, (%rax), %xmm1
+# CHECK-NEXT:  1      3     2.00                        gf2p8affineqb	$0, %xmm0, %xmm1
+# CHECK-NEXT:  2      10    2.00    *                   gf2p8affineqb	$0, (%rax), %xmm1
+# CHECK-NEXT:  1      3     2.00                        gf2p8mulb	%xmm0, %xmm1
+# CHECK-NEXT:  2      10    2.00    *                   gf2p8mulb	(%rax), %xmm1
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     6.00   6.00   6.00    -      -      -      -     12.00  12.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     gf2p8affineinvqb	$0, %xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     gf2p8affineinvqb	$0, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     gf2p8affineqb	$0, %xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     gf2p8affineqb	$0, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     gf2p8mulb	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     gf2p8mulb	(%rax), %xmm1
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lea.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lea.s
new file mode 100644
index 000000000000..bb917d048254
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lea.s
@@ -0,0 +1,448 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+lea 0(), %cx
+lea 0(), %ecx
+lea 0(), %rcx
+lea (%eax), %cx
+lea (%eax), %ecx
+lea (%eax), %rcx
+lea (%rax), %cx
+lea (%rax), %ecx
+lea (%rax), %rcx
+lea (, %ebx), %cx
+lea (, %ebx), %ecx
+lea (, %ebx), %rcx
+lea (, %rbx), %cx
+lea (, %rbx), %ecx
+lea (, %rbx), %rcx
+lea (, %ebx, 1), %cx
+lea (, %ebx, 1), %ecx
+lea (, %ebx, 1), %rcx
+lea (, %rbx, 1), %cx
+lea (, %rbx, 1), %ecx
+lea (, %rbx, 1), %rcx
+lea (, %ebx, 2), %cx
+lea (, %ebx, 2), %ecx
+lea (, %ebx, 2), %rcx
+lea (, %rbx, 2), %cx
+lea (, %rbx, 2), %ecx
+lea (, %rbx, 2), %rcx
+lea (%eax, %ebx), %cx
+lea (%eax, %ebx), %ecx
+lea (%eax, %ebx), %rcx
+lea (%rax, %rbx), %cx
+lea (%rax, %rbx), %ecx
+lea (%rax, %rbx), %rcx
+lea (%eax, %ebx, 1), %cx
+lea (%eax, %ebx, 1), %ecx
+lea (%eax, %ebx, 1), %rcx
+lea (%rax, %rbx, 1), %cx
+lea (%rax, %rbx, 1), %ecx
+lea (%rax, %rbx, 1), %rcx
+lea (%eax, %ebx, 2), %cx
+lea (%eax, %ebx, 2), %ecx
+lea (%eax, %ebx, 2), %rcx
+lea (%rax, %rbx, 2), %cx
+lea (%rax, %rbx, 2), %ecx
+lea (%rax, %rbx, 2), %rcx
+
+lea -16(), %cx
+lea -16(), %ecx
+lea -16(), %rcx
+lea -16(%eax), %cx
+lea -16(%eax), %ecx
+lea -16(%eax), %rcx
+lea -16(%rax), %cx
+lea -16(%rax), %ecx
+lea -16(%rax), %rcx
+lea -16(, %ebx), %cx
+lea -16(, %ebx), %ecx
+lea -16(, %ebx), %rcx
+lea -16(, %rbx), %cx
+lea -16(, %rbx), %ecx
+lea -16(, %rbx), %rcx
+lea -16(, %ebx, 1), %cx
+lea -16(, %ebx, 1), %ecx
+lea -16(, %ebx, 1), %rcx
+lea -16(, %rbx, 1), %cx
+lea -16(, %rbx, 1), %ecx
+lea -16(, %rbx, 1), %rcx
+lea -16(, %ebx, 2), %cx
+lea -16(, %ebx, 2), %ecx
+lea -16(, %ebx, 2), %rcx
+lea -16(, %rbx, 2), %cx
+lea -16(, %rbx, 2), %ecx
+lea -16(, %rbx, 2), %rcx
+lea -16(%eax, %ebx), %cx
+lea -16(%eax, %ebx), %ecx
+lea -16(%eax, %ebx), %rcx
+lea -16(%rax, %rbx), %cx
+lea -16(%rax, %rbx), %ecx
+lea -16(%rax, %rbx), %rcx
+lea -16(%eax, %ebx, 1), %cx
+lea -16(%eax, %ebx, 1), %ecx
+lea -16(%eax, %ebx, 1), %rcx
+lea -16(%rax, %rbx, 1), %cx
+lea -16(%rax, %rbx, 1), %ecx
+lea -16(%rax, %rbx, 1), %rcx
+lea -16(%eax, %ebx, 2), %cx
+lea -16(%eax, %ebx, 2), %ecx
+lea -16(%eax, %ebx, 2), %rcx
+lea -16(%rax, %rbx, 2), %cx
+lea -16(%rax, %rbx, 2), %ecx
+lea -16(%rax, %rbx, 2), %rcx
+
+lea 1024(), %cx
+lea 1024(), %ecx
+lea 1024(), %rcx
+lea 1024(%eax), %cx
+lea 1024(%eax), %ecx
+lea 1024(%eax), %rcx
+lea 1024(%rax), %cx
+lea 1024(%rax), %ecx
+lea 1024(%rax), %rcx
+lea 1024(, %ebx), %cx
+lea 1024(, %ebx), %ecx
+lea 1024(, %ebx), %rcx
+lea 1024(, %rbx), %cx
+lea 1024(, %rbx), %ecx
+lea 1024(, %rbx), %rcx
+lea 1024(, %ebx, 1), %cx
+lea 1024(, %ebx, 1), %ecx
+lea 1024(, %ebx, 1), %rcx
+lea 1024(, %rbx, 1), %cx
+lea 1024(, %rbx, 1), %ecx
+lea 1024(, %rbx, 1), %rcx
+lea 1024(, %ebx, 2), %cx
+lea 1024(, %ebx, 2), %ecx
+lea 1024(, %ebx, 2), %rcx
+lea 1024(, %rbx, 2), %cx
+lea 1024(, %rbx, 2), %ecx
+lea 1024(, %rbx, 2), %rcx
+lea 1024(%eax, %ebx), %cx
+lea 1024(%eax, %ebx), %ecx
+lea 1024(%eax, %ebx), %rcx
+lea 1024(%rax, %rbx), %cx
+lea 1024(%rax, %rbx), %ecx
+lea 1024(%rax, %rbx), %rcx
+lea 1024(%eax, %ebx, 1), %cx
+lea 1024(%eax, %ebx, 1), %ecx
+lea 1024(%eax, %ebx, 1), %rcx
+lea 1024(%rax, %rbx, 1), %cx
+lea 1024(%rax, %rbx, 1), %ecx
+lea 1024(%rax, %rbx, 1), %rcx
+lea 1024(%eax, %ebx, 2), %cx
+lea 1024(%eax, %ebx, 2), %ecx
+lea 1024(%eax, %ebx, 2), %rcx
+lea 1024(%rax, %rbx, 2), %cx
+lea 1024(%rax, %rbx, 2), %ecx
+lea 1024(%rax, %rbx, 2), %rcx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      2     0.33                        leaw	0, %cx
+# CHECK-NEXT:  1      1     1.00                        leal	0, %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	0, %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%eax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%eax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%eax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%rax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%rax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%rax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(,%rbx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%eax,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%eax,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	(%rax,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	(%rax,%rbx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16, %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16, %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16, %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%eax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%eax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%eax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%rax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%rax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%rax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(,%rbx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%eax,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%eax,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	-16(%rax,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	-16(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	-16(%rax,%rbx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024, %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024, %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024, %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%eax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%eax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%eax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%rax), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%rax), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%rax), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(,%rbx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%eax,%ebx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%eax,%ebx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%eax,%ebx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%rax,%rbx), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%rax,%rbx), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%rax,%rbx), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%eax,%ebx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%eax,%ebx,2), %rcx
+# CHECK-NEXT:  2      2     0.33                        leaw	1024(%rax,%rbx,2), %cx
+# CHECK-NEXT:  1      1     1.00                        leal	1024(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  1      1     1.00                        leaq	1024(%rax,%rbx,2), %rcx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 7.50   112.50 7.50   22.50  7.50   22.50   -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	0, %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	0, %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	0, %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%eax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%eax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%eax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%rax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%rax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(,%rbx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%eax,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%eax,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	(%rax,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	(%rax,%rbx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16, %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16, %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16, %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%eax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%eax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%eax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%rax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%rax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(,%rbx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%eax,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%eax,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	-16(%rax,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	-16(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	-16(%rax,%rbx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024, %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024, %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024, %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%eax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%eax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%eax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%rax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%rax), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(,%rbx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%eax,%ebx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%eax,%ebx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%eax,%ebx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%rax,%rbx), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%rax,%rbx), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%rax,%rbx), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%eax,%ebx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%eax,%ebx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%eax,%ebx,2), %rcx
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     leaw	1024(%rax,%rbx,2), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leal	1024(%rax,%rbx,2), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leaq	1024(%rax,%rbx,2), %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lzcnt.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lzcnt.s
new file mode 100644
index 000000000000..51dfe77e0741
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-lzcnt.s
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+lzcntw      %cx, %cx
+lzcntw      (%rax), %cx
+
+lzcntl      %eax, %ecx
+lzcntl      (%rax), %ecx
+
+lzcntq      %rax, %rcx
+lzcntq      (%rax), %rcx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00                        lzcntw	%cx, %cx
+# CHECK-NEXT:  2      7     1.00    *                   lzcntw	(%rax), %cx
+# CHECK-NEXT:  1      3     1.00                        lzcntl	%eax, %ecx
+# CHECK-NEXT:  2      7     1.00    *                   lzcntl	(%rax), %ecx
+# CHECK-NEXT:  1      3     1.00                        lzcntq	%rax, %rcx
+# CHECK-NEXT:  2      7     1.00    *                   lzcntq	(%rax), %rcx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     6.00    -      -      -      -      -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lzcntw	%cx, %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     lzcntw	(%rax), %cx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lzcntl	%eax, %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     lzcntl	(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lzcntq	%rax, %rcx
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     lzcntq	(%rax), %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-mmx.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-mmx.s
new file mode 100644
index 000000000000..10da6cf733bc
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-mmx.s
@@ -0,0 +1,404 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+emms
+
+movd        %eax, %mm2
+movd        (%rax), %mm2
+
+movd        %mm0, %ecx
+movd        %mm0, (%rax)
+
+movq        %rax, %mm2
+movq        (%rax), %mm2
+
+movq        %mm0, %rcx
+movq        %mm0, (%rax)
+
+packsswb    %mm0, %mm2
+packsswb    (%rax), %mm2
+
+packssdw    %mm0, %mm2
+packssdw    (%rax), %mm2
+
+packuswb    %mm0, %mm2
+packuswb    (%rax), %mm2
+
+paddb       %mm0, %mm2
+paddb       (%rax), %mm2
+
+paddd       %mm0, %mm2
+paddd       (%rax), %mm2
+
+paddsb      %mm0, %mm2
+paddsb      (%rax), %mm2
+
+paddsw      %mm0, %mm2
+paddsw      (%rax), %mm2
+
+paddusb     %mm0, %mm2
+paddusb     (%rax), %mm2
+
+paddusw     %mm0, %mm2
+paddusw     (%rax), %mm2
+
+paddw       %mm0, %mm2
+paddw       (%rax), %mm2
+
+pand        %mm0, %mm2
+pand        (%rax), %mm2
+
+pandn       %mm0, %mm2
+pandn       (%rax), %mm2
+
+pcmpeqb     %mm0, %mm2
+pcmpeqb     (%rax), %mm2
+
+pcmpeqd     %mm0, %mm2
+pcmpeqd     (%rax), %mm2
+
+pcmpeqw     %mm0, %mm2
+pcmpeqw     (%rax), %mm2
+
+pcmpgtb     %mm0, %mm2
+pcmpgtb     (%rax), %mm2
+
+pcmpgtd     %mm0, %mm2
+pcmpgtd     (%rax), %mm2
+
+pcmpgtw     %mm0, %mm2
+pcmpgtw     (%rax), %mm2
+
+pmaddwd     %mm0, %mm2
+pmaddwd     (%rax), %mm2
+
+pmulhw      %mm0, %mm2
+pmulhw      (%rax), %mm2
+
+pmullw      %mm0, %mm2
+pmullw      (%rax), %mm2
+
+por         %mm0, %mm2
+por         (%rax), %mm2
+
+pslld       $1, %mm2
+pslld       %mm0, %mm2
+pslld       (%rax), %mm2
+
+psllq       $1, %mm2
+psllq       %mm0, %mm2
+psllq       (%rax), %mm2
+
+psllw       $1, %mm2
+psllw       %mm0, %mm2
+psllw       (%rax), %mm2
+
+psrad       $1, %mm2
+psrad       %mm0, %mm2
+psrad       (%rax), %mm2
+
+psraw       $1, %mm2
+psraw       %mm0, %mm2
+psraw       (%rax), %mm2
+
+psrld       $1, %mm2
+psrld       %mm0, %mm2
+psrld       (%rax), %mm2
+
+psrlq       $1, %mm2
+psrlq       %mm0, %mm2
+psrlq       (%rax), %mm2
+
+psrlw       $1, %mm2
+psrlw       %mm0, %mm2
+psrlw       (%rax), %mm2
+
+psubb       %mm0, %mm2
+psubb       (%rax), %mm2
+
+psubd       %mm0, %mm2
+psubd       (%rax), %mm2
+
+psubsb      %mm0, %mm2
+psubsb      (%rax), %mm2
+
+psubsw      %mm0, %mm2
+psubsw      (%rax), %mm2
+
+psubusb     %mm0, %mm2
+psubusb     (%rax), %mm2
+
+psubusw     %mm0, %mm2
+psubusw     (%rax), %mm2
+
+psubw       %mm0, %mm2
+psubw       (%rax), %mm2
+
+punpckhbw   %mm0, %mm2
+punpckhbw   (%rax), %mm2
+
+punpckhdq   %mm0, %mm2
+punpckhdq   (%rax), %mm2
+
+punpckhwd   %mm0, %mm2
+punpckhwd   (%rax), %mm2
+
+punpcklbw   %mm0, %mm2
+punpcklbw   (%rax), %mm2
+
+punpckldq   %mm0, %mm2
+punpckldq   (%rax), %mm2
+
+punpcklwd   %mm0, %mm2
+punpcklwd   (%rax), %mm2
+
+pxor        %mm0, %mm2
+pxor        (%rax), %mm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  10     10    4.00    *      *      U     emms
+# CHECK-NEXT:  1      3     0.33                        movd	%eax, %mm2
+# CHECK-NEXT:  1      8     0.33    *                   movd	(%rax), %mm2
+# CHECK-NEXT:  1      3     0.50                        movd	%mm0, %ecx
+# CHECK-NEXT:  2      18    0.50           *      U     movd	%mm0, (%rax)
+# CHECK-NEXT:  1      3     0.33                        movq	%rax, %mm2
+# CHECK-NEXT:  1      8     0.33    *                   movq	(%rax), %mm2
+# CHECK-NEXT:  1      3     0.50                        movq	%mm0, %rcx
+# CHECK-NEXT:  2      12    0.50           *            movq	%mm0, (%rax)
+# CHECK-NEXT:  2      4     0.67                        packsswb	%mm0, %mm2
+# CHECK-NEXT:  3      12    0.67    *                   packsswb	(%rax), %mm2
+# CHECK-NEXT:  2      4     0.67                        packssdw	%mm0, %mm2
+# CHECK-NEXT:  3      12    0.67    *                   packssdw	(%rax), %mm2
+# CHECK-NEXT:  2      4     0.67                        packuswb	%mm0, %mm2
+# CHECK-NEXT:  3      12    0.67    *                   packuswb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        paddb	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   paddb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        paddd	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   paddd	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        paddsb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   paddsb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        paddsw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   paddsw	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        paddusb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   paddusb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        paddusw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   paddusw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        paddw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   paddw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        pand	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.33    *                   pand	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        pandn	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.33    *                   pandn	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpeqb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpeqb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpeqd	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpeqd	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpeqw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpeqw	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpgtb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpgtb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpgtd	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpgtd	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpgtw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pcmpgtw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmaddwd	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmaddwd	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmulhw	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmulhw	(%rax), %mm2
+# CHECK-NEXT:  1      5     0.50                        pmullw	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmullw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        por	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.33    *                   por	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pslld	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        pslld	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   pslld	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psllq	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psllq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psllq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psllw	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psllw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psllw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psrad	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psrad	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psrad	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psraw	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psraw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psraw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psrld	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psrld	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psrld	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psrlq	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psrlq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psrlq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psrlw	$1, %mm2
+# CHECK-NEXT:  1      1     0.50                        psrlw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psrlw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        psubb	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psubb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        psubd	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psubd	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        psubsb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psubsb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        psubsw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psubsw	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        psubusb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psubusb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        psubusw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psubusw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        psubw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psubw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpckhbw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpckhbw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpckhdq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpckhdq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpckhwd	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpckhwd	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpcklbw	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpcklbw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpckldq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpckldq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        punpcklwd	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   punpcklwd	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        pxor	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.33    *                   pxor	(%rax), %mm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     7.67    -     6.67    -     6.67   1.00   1.00   15.33  15.33  15.33  0.67   0.67   0.67    -     53.00  25.00  8.50   8.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00   0.50   0.50   emms
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movd	%eax, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movd	%mm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movd	%mm0, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movq	%rax, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movq	%mm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movq	%mm0, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     packsswb	%mm0, %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     packsswb	(%rax), %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     packssdw	%mm0, %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     packssdw	(%rax), %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     packuswb	%mm0, %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     packuswb	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     paddb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     paddb	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     paddd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     paddd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     paddsb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     paddsb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     paddsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     paddsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     paddusb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     paddusb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     paddusw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     paddusw	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     paddw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     paddw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pand	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   pand	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pandn	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   pandn	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpeqb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpeqb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpeqd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpeqd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpeqw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpeqw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpgtb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpgtd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pcmpgtw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaddwd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmaddwd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmulhw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmulhw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmullw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmullw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   por	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   por	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pslld	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pslld	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pslld	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllq	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psllq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllw	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psllw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrad	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrad	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrad	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psraw	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psraw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psraw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrld	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrld	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrld	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlq	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrlq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlw	$1, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrlw	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     psubb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psubb	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     psubd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psubd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psubsb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psubsb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psubsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psubsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psubusb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psubusb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psubusw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psubusw	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     psubw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psubw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhbw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhbw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhdq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhdq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhwd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhwd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpcklbw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpcklbw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckldq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckldq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpcklwd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpcklwd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pxor	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   pxor	(%rax), %mm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-movbe.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-movbe.s
new file mode 100644
index 000000000000..72fd96dffc7b
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-movbe.s
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+movbe  %cx, (%rax)
+movbe  (%rax), %cx
+
+movbe  %ecx, (%rax)
+movbe  (%rax), %ecx
+
+movbe  %rcx, (%rax)
+movbe  (%rax), %rcx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  3      12    0.50           *            movbew	%cx, (%rax)
+# CHECK-NEXT:  3      100   1.33    *                   movbew	(%rax), %cx
+# CHECK-NEXT:  3      12    0.50           *            movbel	%ecx, (%rax)
+# CHECK-NEXT:  3      100   1.33    *                   movbel	(%rax), %ecx
+# CHECK-NEXT:  4      12    0.50           *            movbeq	%rcx, (%rax)
+# CHECK-NEXT:  3      100   1.33    *                   movbeq	(%rax), %rcx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.67   2.67   0.67   2.67   0.67   2.67   1.50   1.50   4.00   4.00   4.00   1.00   1.00   1.00    -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movbew	%cx, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     movbew	(%rax), %cx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movbel	%ecx, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     movbel	(%rax), %ecx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movbeq	%rcx, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     movbeq	(%rax), %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-pclmul.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-pclmul.s
new file mode 100644
index 000000000000..078f7a82c5d0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-pclmul.s
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+pclmulqdq     $11, %xmm0, %xmm2
+pclmulqdq     $11, (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        pclmulqdq	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   pclmulqdq	$11, (%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pclmulqdq	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pclmulqdq	$11, (%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-popcnt.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-popcnt.s
new file mode 100644
index 000000000000..11cfa559ba50
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-popcnt.s
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+popcntw     %cx, %cx
+popcntw     (%rax), %cx
+
+popcntl     %eax, %ecx
+popcntl     (%rax), %ecx
+
+popcntq     %rax, %rcx
+popcntq     (%rax), %rcx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.33                        popcntw	%cx, %cx
+# CHECK-NEXT:  2      7     0.33    *                   popcntw	(%rax), %cx
+# CHECK-NEXT:  1      3     0.33                        popcntl	%eax, %ecx
+# CHECK-NEXT:  2      7     0.33    *                   popcntl	(%rax), %ecx
+# CHECK-NEXT:  1      3     0.33                        popcntq	%rax, %rcx
+# CHECK-NEXT:  2      7     0.33    *                   popcntq	(%rax), %rcx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     2.00    -     2.00    -     2.00    -      -     1.00   1.00   1.00    -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     popcntw	%cx, %cx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     popcntw	(%rax), %cx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     popcntl	%eax, %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     popcntl	(%rax), %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     popcntq	%rax, %rcx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     popcntq	(%rax), %rcx
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-prefetchw.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-prefetchw.s
new file mode 100644
index 000000000000..2f2b85e627be
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-prefetchw.s
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+prefetch    (%rax)
+prefetchw   (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     0.33    *      *            prefetch	(%rax)
+# CHECK-NEXT:  1      4     0.33    *      *            prefetchw	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetch	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetchw	(%rax)
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdrand.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdrand.s
new file mode 100644
index 000000000000..2a41c00c4aef
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdrand.s
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+rdrand   %ax
+rdrand   %eax
+rdrand   %rax
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.17                  U     rdrandw	%ax
+# CHECK-NEXT:  1      100   0.17                  U     rdrandl	%eax
+# CHECK-NEXT:  1      100   0.17                  U     rdrandq	%rax
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdrandw	%ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdrandl	%eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdrandq	%rax
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdseed.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdseed.s
new file mode 100644
index 000000000000..7b42d8b402ec
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-rdseed.s
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+rdseed   %ax
+rdseed   %eax
+rdseed   %rax
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.17                  U     rdseedw	%ax
+# CHECK-NEXT:  1      100   0.17                  U     rdseedl	%eax
+# CHECK-NEXT:  1      100   0.17                  U     rdseedq	%rax
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdseedw	%ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdseedl	%eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdseedq	%rax
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse1.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse1.s
new file mode 100644
index 000000000000..e0b4b45c331d
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse1.s
@@ -0,0 +1,472 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+addps       %xmm0, %xmm2
+addps       (%rax), %xmm2
+
+addss       %xmm0, %xmm2
+addss       (%rax), %xmm2
+
+andnps      %xmm0, %xmm2
+andnps      (%rax), %xmm2
+
+andps       %xmm0, %xmm2
+andps       (%rax), %xmm2
+
+cmpps       $0, %xmm0, %xmm2
+cmpps       $0, (%rax), %xmm2
+
+cmpss       $0, %xmm0, %xmm2
+cmpss       $0, (%rax), %xmm2
+
+comiss      %xmm0, %xmm1
+comiss      (%rax), %xmm1
+
+cvtpi2ps    %mm0, %xmm2
+cvtpi2ps    (%rax), %xmm2
+
+cvtps2pi    %xmm0, %mm2
+cvtps2pi    (%rax), %mm2
+
+cvtsi2ss    %ecx, %xmm2
+cvtsi2ss    %rcx, %xmm2
+cvtsi2ss    (%rax), %xmm2
+cvtsi2ss    (%rax), %xmm2
+
+cvtss2si    %xmm0, %ecx
+cvtss2si    %xmm0, %rcx
+cvtss2si    (%rax), %ecx
+cvtss2si    (%rax), %rcx
+
+cvttps2pi   %xmm0, %mm2
+cvttps2pi   (%rax), %mm2
+
+cvttss2si   %xmm0, %ecx
+cvttss2si   %xmm0, %rcx
+cvttss2si   (%rax), %ecx
+cvttss2si   (%rax), %rcx
+
+divps       %xmm0, %xmm2
+divps       (%rax), %xmm2
+
+divss       %xmm0, %xmm2
+divss       (%rax), %xmm2
+
+ldmxcsr     (%rax)
+
+maskmovq    %mm0, %mm1
+
+maxps       %xmm0, %xmm2
+maxps       (%rax), %xmm2
+
+maxss       %xmm0, %xmm2
+maxss       (%rax), %xmm2
+
+minps       %xmm0, %xmm2
+minps       (%rax), %xmm2
+
+minss       %xmm0, %xmm2
+minss       (%rax), %xmm2
+
+movaps      %xmm0, %xmm2
+movaps      %xmm0, (%rax)
+movaps      (%rax), %xmm2
+
+movhlps     %xmm0, %xmm2
+movlhps     %xmm0, %xmm2
+
+movhps      %xmm0, (%rax)
+movhps      (%rax), %xmm2
+
+movlps      %xmm0, (%rax)
+movlps      (%rax), %xmm2
+
+movmskps    %xmm0, %rcx
+
+movntps     %xmm0, (%rax)
+movntq      %mm0, (%rax)
+
+movss       %xmm0, %xmm2
+movss       %xmm0, (%rax)
+movss       (%rax), %xmm2
+
+movups      %xmm0, %xmm2
+movups      %xmm0, (%rax)
+movups      (%rax), %xmm2
+
+mulps       %xmm0, %xmm2
+mulps       (%rax), %xmm2
+
+mulss       %xmm0, %xmm2
+mulss       (%rax), %xmm2
+
+orps        %xmm0, %xmm2
+orps        (%rax), %xmm2
+
+pavgb       %mm0, %mm2
+pavgb       (%rax), %mm2
+
+pavgw       %mm0, %mm2
+pavgw       (%rax), %mm2
+
+pextrw      $1, %mm0, %rcx
+
+pinsrw      $1, %rax, %mm2
+pinsrw      $1, (%rax), %mm2
+
+pmaxsw      %mm0, %mm2
+pmaxsw      (%rax), %mm2
+
+pmaxub      %mm0, %mm2
+pmaxub      (%rax), %mm2
+
+pminsw      %mm0, %mm2
+pminsw      (%rax), %mm2
+
+pminub      %mm0, %mm2
+pminub      (%rax), %mm2
+
+pmovmskb    %mm0, %rcx
+
+pmulhuw     %mm0, %mm2
+pmulhuw     (%rax), %mm2
+
+prefetcht0  (%rax)
+prefetcht1  (%rax)
+prefetcht2  (%rax)
+prefetchnta (%rax)
+
+psadbw      %mm0, %mm2
+psadbw      (%rax), %mm2
+
+pshufw      $1, %mm0, %mm2
+pshufw      $1, (%rax), %mm2
+
+rcpps       %xmm0, %xmm2
+rcpps       (%rax), %xmm2
+
+rcpss       %xmm0, %xmm2
+rcpss       (%rax), %xmm2
+
+rsqrtps     %xmm0, %xmm2
+rsqrtps     (%rax), %xmm2
+
+rsqrtss     %xmm0, %xmm2
+rsqrtss     (%rax), %xmm2
+
+sfence
+
+shufps      $1, %xmm0, %xmm2
+shufps      $1, (%rax), %xmm2
+
+sqrtps      %xmm0, %xmm2
+sqrtps      (%rax), %xmm2
+
+sqrtss      %xmm0, %xmm2
+sqrtss      (%rax), %xmm2
+
+stmxcsr     (%rax)
+
+subps       %xmm0, %xmm2
+subps       (%rax), %xmm2
+
+subss       %xmm0, %xmm2
+subss       (%rax), %xmm2
+
+ucomiss     %xmm0, %xmm1
+ucomiss     (%rax), %xmm1
+
+unpckhps    %xmm0, %xmm2
+unpckhps    (%rax), %xmm2
+
+unpcklps    %xmm0, %xmm2
+unpcklps    (%rax), %xmm2
+
+xorps       %xmm0, %xmm2
+xorps       (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        addps	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addps	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.33                        addss	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addss	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        andnps	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   andnps	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        andps	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   andps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        cmpeqps	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cmpeqps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        cmpeqss	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cmpeqss	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        comiss	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   comiss	(%rax), %xmm1
+# CHECK-NEXT:  2      100   2.50                        cvtpi2ps	%mm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtpi2ps	(%rax), %xmm2
+# CHECK-NEXT:  2      100   2.00                        cvtps2pi	%xmm0, %mm2
+# CHECK-NEXT:  2      13    2.67    *                   cvtps2pi	(%rax), %mm2
+# CHECK-NEXT:  2      8     2.00                        cvtsi2ss	%ecx, %xmm2
+# CHECK-NEXT:  3      9     2.00                        cvtsi2ss	%rcx, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  2      7     3.50                        cvtss2si	%xmm0, %ecx
+# CHECK-NEXT:  3      8     3.50                        cvtss2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   cvtss2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   cvtss2si	(%rax), %rcx
+# CHECK-NEXT:  2      5     2.00                        cvttps2pi	%xmm0, %mm2
+# CHECK-NEXT:  2      13    2.67    *                   cvttps2pi	(%rax), %mm2
+# CHECK-NEXT:  2      7     3.50                        cvttss2si	%xmm0, %ecx
+# CHECK-NEXT:  3      8     3.50                        cvttss2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   cvttss2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   cvttss2si	(%rax), %rcx
+# CHECK-NEXT:  1      7     3.50                        divps	%xmm0, %xmm2
+# CHECK-NEXT:  1      13    3.50    *                   divps	(%rax), %xmm2
+# CHECK-NEXT:  1      7     3.50                        divss	%xmm0, %xmm2
+# CHECK-NEXT:  1      13    3.50    *                   divss	(%rax), %xmm2
+# CHECK-NEXT:  4      7     0.75    *      *      U     ldmxcsr	(%rax)
+# CHECK-NEXT:  4      12    1.00    *      *      U     maskmovq	%mm0, %mm1
+# CHECK-NEXT:  1      4     2.00                        maxps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   maxps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        maxss	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   maxss	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        minps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   minps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        minss	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   minss	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        movaps	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movaps	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   movaps	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        movhlps	%xmm0, %xmm2
+# CHECK-NEXT:  1      1     0.50                        movlhps	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movhps	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.50    *                   movhps	(%rax), %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movlps	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.33    *                   movlps	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        movmskps	%xmm0, %ecx
+# CHECK-NEXT:  2      518   0.50           *            movntps	%xmm0, (%rax)
+# CHECK-NEXT:  2      511   0.50    *      *      U     movntq	%mm0, (%rax)
+# CHECK-NEXT:  1      1     0.25                        movss	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movss	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   movss	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        movups	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movups	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   movups	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        mulps	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   mulps	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        mulss	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   mulss	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        orps	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   orps	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        pavgb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pavgb	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pavgw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pavgw	(%rax), %mm2
+# CHECK-NEXT:  2      4     0.50                        pextrw	$1, %mm0, %ecx
+# CHECK-NEXT:  2      4     0.67                        pinsrw	$1, %eax, %mm2
+# CHECK-NEXT:  2      9     0.33    *                   pinsrw	$1, (%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pmaxsw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pmaxsw	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pmaxub	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pmaxub	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pminsw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pminsw	(%rax), %mm2
+# CHECK-NEXT:  1      1     1.00                        pminub	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pminub	(%rax), %mm2
+# CHECK-NEXT:  1      3     0.50                        pmovmskb	%mm0, %ecx
+# CHECK-NEXT:  1      5     0.50                        pmulhuw	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmulhuw	(%rax), %mm2
+# CHECK-NEXT:  1      0     0.33    *      *            prefetcht0	(%rax)
+# CHECK-NEXT:  1      0     0.33    *      *            prefetcht1	(%rax)
+# CHECK-NEXT:  1      0     0.33    *      *            prefetcht2	(%rax)
+# CHECK-NEXT:  1      0     0.33    *      *            prefetchnta	(%rax)
+# CHECK-NEXT:  1      3     0.50                        psadbw	%mm0, %mm2
+# CHECK-NEXT:  2      11    0.50    *                   psadbw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pshufw	$1, %mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   pshufw	$1, (%rax), %mm2
+# CHECK-NEXT:  1      4     2.00                        rcpps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   rcpps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        rcpss	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   rcpss	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        rsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   rsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        rsqrtss	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   rsqrtss	(%rax), %xmm2
+# CHECK-NEXT:  2      2     0.50    *      *      U     sfence
+# CHECK-NEXT:  1      1     0.50                        shufps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   shufps	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      10    5.00                        sqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   sqrtps	(%rax), %xmm2
+# CHECK-NEXT:  1      10    5.00                        sqrtss	%xmm0, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   sqrtss	(%rax), %xmm2
+# CHECK-NEXT:  4      100   3.00           *      U     stmxcsr	(%rax)
+# CHECK-NEXT:  1      3     0.50                        subps	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   subps	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.33                        subss	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   subss	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        ucomiss	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   ucomiss	(%rax), %xmm1
+# CHECK-NEXT:  1      1     0.50                        unpckhps	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   unpckhps	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        unpcklps	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   unpcklps	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        xorps	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   xorps	(%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.33   4.67   0.33   4.67   0.33   4.67   5.00   5.00   74.33  74.33  74.33  3.67   3.67   3.67    -     157.25 145.25 15.25  15.25
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addps	(%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   andnps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   andnps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   andps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   andps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cmpeqps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cmpeqps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cmpeqss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cmpeqss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     comiss	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     comiss	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50    -      -     cvtpi2ps	%mm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtpi2ps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   cvtps2pi	%xmm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.67   2.67   2.67    -      -      -      -     2.00   2.00    -      -     cvtps2pi	(%rax), %mm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvtsi2ss	%ecx, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvtsi2ss	%rcx, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvtss2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50   0.50   0.50   cvtss2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvtss2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvtss2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.25   2.25   0.25   0.25   cvttps2pi	%xmm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.67   2.67   2.67    -      -      -      -     2.00   2.00    -      -     cvttps2pi	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvttss2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50   0.50   0.50   cvttss2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvttss2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvttss2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     divps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     3.50   3.50    -      -     divps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     divss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     3.50   3.50    -      -     divss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.33   0.33   0.33    -     0.75   0.75   0.75   0.75   ldmxcsr	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     1.00   1.00    -      -     maskmovq	%mm0, %mm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     maxps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     maxps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     maxss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     maxss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     minps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     minps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     minss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     minss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movaps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movaps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movaps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   movhlps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   movlhps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movhps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   movhps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movlps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   movlps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     movmskps	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntps	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntq	%mm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movss	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movups	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movups	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movups	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     mulps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     mulps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     mulss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     mulss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   orps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pavgb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pavgb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pavgw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pavgw	(%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pextrw	$1, %mm0, %ecx
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     pinsrw	$1, %eax, %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     pinsrw	$1, (%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pmaxsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pmaxsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pmaxub	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pmaxub	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pminsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pminsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pminub	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pminub	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmovmskb	%mm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmulhuw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmulhuw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetcht0	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetcht1	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetcht2	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     prefetchnta	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   psadbw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   psadbw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pshufw	$1, %mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pshufw	$1, (%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     rcpps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     rcpps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     rcpss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     rcpss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     rsqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     rsqrtps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     rsqrtss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     rsqrtss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     sfence
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   shufps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   shufps	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     sqrtps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     sqrtps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     sqrtss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     sqrtss	(%rax), %xmm2
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50    -      -      -     0.33   0.33   0.33    -     3.00   3.00    -      -     stmxcsr	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   subps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   subps	(%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   subss	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     ucomiss	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     ucomiss	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   unpckhps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   unpckhps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   unpcklps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   unpcklps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xorps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   xorps	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse2.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse2.s
new file mode 100644
index 000000000000..db62ae4162f8
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse2.s
@@ -0,0 +1,971 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+addpd       %xmm0, %xmm2
+addpd       (%rax), %xmm2
+
+addsd       %xmm0, %xmm2
+addsd       (%rax), %xmm2
+
+andnpd      %xmm0, %xmm2
+andnpd      (%rax), %xmm2
+
+andpd       %xmm0, %xmm2
+andpd       (%rax), %xmm2
+
+clflush     (%rax)
+
+cmppd       $0, %xmm0, %xmm2
+cmppd       $0, (%rax), %xmm2
+
+cmpsd       $0, %xmm0, %xmm2
+cmpsd       $0, (%rax), %xmm2
+
+comisd      %xmm0, %xmm1
+comisd      (%rax), %xmm1
+
+cvtdq2pd    %xmm0, %xmm2
+cvtdq2pd    (%rax), %xmm2
+
+cvtdq2ps    %xmm0, %xmm2
+cvtdq2ps    (%rax), %xmm2
+
+cvtpd2dq    %xmm0, %xmm2
+cvtpd2dq    (%rax), %xmm2
+
+cvtpd2pi    %xmm0, %mm2
+cvtpd2pi    (%rax), %mm2
+
+cvtpd2ps    %xmm0, %xmm2
+cvtpd2ps    (%rax), %xmm2
+
+cvtpi2pd    %mm0, %xmm2
+cvtpi2pd    (%rax), %xmm2
+
+cvtps2dq    %xmm0, %xmm2
+cvtps2dq    (%rax), %xmm2
+
+cvtps2pd    %xmm0, %xmm2
+cvtps2pd    (%rax), %xmm2
+
+cvtsd2si    %xmm0, %ecx
+cvtsd2si    %xmm0, %rcx
+cvtsd2si    (%rax), %ecx
+cvtsd2si    (%rax), %rcx
+
+cvtsd2ss    %xmm0, %xmm2
+cvtsd2ss    (%rax), %xmm2
+
+cvtsi2sd    %ecx, %xmm2
+cvtsi2sd    %rcx, %xmm2
+cvtsi2sd    (%rax), %xmm2
+cvtsi2sd    (%rax), %xmm2
+
+cvtss2sd    %xmm0, %xmm2
+cvtss2sd    (%rax), %xmm2
+
+cvttpd2dq   %xmm0, %xmm2
+cvttpd2dq   (%rax), %xmm2
+
+cvttpd2pi   %xmm0, %mm2
+cvttpd2pi   (%rax), %mm2
+
+cvttps2dq   %xmm0, %xmm2
+cvttps2dq   (%rax), %xmm2
+
+cvttsd2si   %xmm0, %ecx
+cvttsd2si   %xmm0, %rcx
+cvttsd2si   (%rax), %ecx
+cvttsd2si   (%rax), %rcx
+
+divpd       %xmm0, %xmm2
+divpd       (%rax), %xmm2
+
+divsd       %xmm0, %xmm2
+divsd       (%rax), %xmm2
+
+lfence
+
+maskmovdqu  %xmm0, %xmm1
+
+maxpd       %xmm0, %xmm2
+maxpd       (%rax), %xmm2
+
+maxsd       %xmm0, %xmm2
+maxsd       (%rax), %xmm2
+
+mfence
+
+minpd       %xmm0, %xmm2
+minpd       (%rax), %xmm2
+
+minsd       %xmm0, %xmm2
+minsd       (%rax), %xmm2
+
+movapd      %xmm0, %xmm2
+movapd      %xmm0, (%rax)
+movapd      (%rax), %xmm2
+
+movd        %eax, %xmm2
+movd        (%rax), %xmm2
+
+movd        %xmm0, %ecx
+movd        %xmm0, (%rax)
+
+movdqa      %xmm0, %xmm2
+movdqa      %xmm0, (%rax)
+movdqa      (%rax), %xmm2
+
+movdqu      %xmm0, %xmm2
+movdqu      %xmm0, (%rax)
+movdqu      (%rax), %xmm2
+
+movdq2q     %xmm0, %mm2
+
+movhpd      %xmm0, (%rax)
+movhpd      (%rax), %xmm2
+
+movlpd      %xmm0, (%rax)
+movlpd      (%rax), %xmm2
+
+movmskpd    %xmm0, %rcx
+
+movntil     %eax, (%rax)
+movntiq     %rax, (%rax)
+
+movntdq     %xmm0, (%rax)
+movntpd     %xmm0, (%rax)
+
+movq        %xmm0, %xmm2
+
+movq        %rax, %xmm2
+movq        (%rax), %xmm2
+
+movq        %xmm0, %rcx
+movq        %xmm0, (%rax)
+
+movq2dq     %mm0, %xmm2
+
+movsd       %xmm0, %xmm2
+movsd       %xmm0, (%rax)
+movsd       (%rax), %xmm2
+
+movupd      %xmm0, %xmm2
+movupd      %xmm0, (%rax)
+movupd      (%rax), %xmm2
+
+mulpd       %xmm0, %xmm2
+mulpd       (%rax), %xmm2
+
+mulsd       %xmm0, %xmm2
+mulsd       (%rax), %xmm2
+
+orpd        %xmm0, %xmm2
+orpd        (%rax), %xmm2
+
+packssdw    %xmm0, %xmm2
+packssdw    (%rax), %xmm2
+
+packsswb    %xmm0, %xmm2
+packsswb    (%rax), %xmm2
+
+packuswb    %xmm0, %xmm2
+packuswb    (%rax), %xmm2
+
+paddb       %xmm0, %xmm2
+paddb       (%rax), %xmm2
+
+paddd       %xmm0, %xmm2
+paddd       (%rax), %xmm2
+
+paddq       %mm0, %mm2
+paddq       (%rax), %mm2
+
+paddq       %xmm0, %xmm2
+paddq       (%rax), %xmm2
+
+paddsb      %xmm0, %xmm2
+paddsb      (%rax), %xmm2
+
+paddsw      %xmm0, %xmm2
+paddsw      (%rax), %xmm2
+
+paddusb     %xmm0, %xmm2
+paddusb     (%rax), %xmm2
+
+paddusw     %xmm0, %xmm2
+paddusw     (%rax), %xmm2
+
+paddw       %xmm0, %xmm2
+paddw       (%rax), %xmm2
+
+pand        %xmm0, %xmm2
+pand        (%rax), %xmm2
+
+pandn       %xmm0, %xmm2
+pandn       (%rax), %xmm2
+
+pavgb       %xmm0, %xmm2
+pavgb       (%rax), %xmm2
+
+pavgw       %xmm0, %xmm2
+pavgw       (%rax), %xmm2
+
+pcmpeqb     %xmm0, %xmm2
+pcmpeqb     (%rax), %xmm2
+
+pcmpeqd     %xmm0, %xmm2
+pcmpeqd     (%rax), %xmm2
+
+pcmpeqw     %xmm0, %xmm2
+pcmpeqw     (%rax), %xmm2
+
+pcmpgtb     %xmm0, %xmm2
+pcmpgtb     (%rax), %xmm2
+
+pcmpgtd     %xmm0, %xmm2
+pcmpgtd     (%rax), %xmm2
+
+pcmpgtw     %xmm0, %xmm2
+pcmpgtw     (%rax), %xmm2
+
+pextrw      $1, %xmm0, %rcx
+
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
+pmaddwd     %xmm0, %xmm2
+pmaddwd     (%rax), %xmm2
+
+pmaxsw      %xmm0, %xmm2
+pmaxsw      (%rax), %xmm2
+
+pmaxub      %xmm0, %xmm2
+pmaxub      (%rax), %xmm2
+
+pminsw      %xmm0, %xmm2
+pminsw      (%rax), %xmm2
+
+pminub      %xmm0, %xmm2
+pminub      (%rax), %xmm2
+
+pmovmskb    %xmm0, %rcx
+
+pmulhuw     %xmm0, %xmm2
+pmulhuw     (%rax), %xmm2
+
+pmulhw      %xmm0, %xmm2
+pmulhw      (%rax), %xmm2
+
+pmullw      %xmm0, %xmm2
+pmullw      (%rax), %xmm2
+
+pmuludq     %mm0, %mm2
+pmuludq     (%rax), %mm2
+
+pmuludq     %xmm0, %xmm2
+pmuludq     (%rax), %xmm2
+
+por         %xmm0, %xmm2
+por         (%rax), %xmm2
+
+psadbw      %xmm0, %xmm2
+psadbw      (%rax), %xmm2
+
+pshufd      $1, %xmm0, %xmm2
+pshufd      $1, (%rax), %xmm2
+
+pshufhw     $1, %xmm0, %xmm2
+pshufhw     $1, (%rax), %xmm2
+
+pshuflw     $1, %xmm0, %xmm2
+pshuflw     $1, (%rax), %xmm2
+
+pslld       $1, %xmm2
+pslld       %xmm0, %xmm2
+pslld       (%rax), %xmm2
+
+pslldq      $1, %xmm2
+
+psllq       $1, %xmm2
+psllq       %xmm0, %xmm2
+psllq       (%rax), %xmm2
+
+psllw       $1, %xmm2
+psllw       %xmm0, %xmm2
+psllw       (%rax), %xmm2
+
+psrad       $1, %xmm2
+psrad       %xmm0, %xmm2
+psrad       (%rax), %xmm2
+
+psraw       $1, %xmm2
+psraw       %xmm0, %xmm2
+psraw       (%rax), %xmm2
+
+psrld       $1, %xmm2
+psrld       %xmm0, %xmm2
+psrld       (%rax), %xmm2
+
+psrldq      $1, %xmm2
+
+psrlq       $1, %xmm2
+psrlq       %xmm0, %xmm2
+psrlq       (%rax), %xmm2
+
+psrlw       $1, %xmm2
+psrlw       %xmm0, %xmm2
+psrlw       (%rax), %xmm2
+
+psubb       %xmm0, %xmm2
+psubb       (%rax), %xmm2
+
+psubd       %xmm0, %xmm2
+psubd       (%rax), %xmm2
+
+psubq       %mm0, %mm2
+psubq       (%rax), %mm2
+
+psubq       %xmm0, %xmm2
+psubq       (%rax), %xmm2
+
+psubsb      %xmm0, %xmm2
+psubsb      (%rax), %xmm2
+
+psubsw      %xmm0, %xmm2
+psubsw      (%rax), %xmm2
+
+psubusb     %xmm0, %xmm2
+psubusb     (%rax), %xmm2
+
+psubusw     %xmm0, %xmm2
+psubusw     (%rax), %xmm2
+
+psubw       %xmm0, %xmm2
+psubw       (%rax), %xmm2
+
+punpckhbw   %xmm0, %xmm2
+punpckhbw   (%rax), %xmm2
+
+punpckhdq   %xmm0, %xmm2
+punpckhdq   (%rax), %xmm2
+
+punpckhqdq  %xmm0, %xmm2
+punpckhqdq  (%rax), %xmm2
+
+punpckhwd   %xmm0, %xmm2
+punpckhwd   (%rax), %xmm2
+
+punpcklbw   %xmm0, %xmm2
+punpcklbw   (%rax), %xmm2
+
+punpckldq   %xmm0, %xmm2
+punpckldq   (%rax), %xmm2
+
+punpcklqdq  %xmm0, %xmm2
+punpcklqdq  (%rax), %xmm2
+
+punpcklwd   %xmm0, %xmm2
+punpcklwd   (%rax), %xmm2
+
+pxor        %xmm0, %xmm2
+pxor        (%rax), %xmm2
+
+shufpd      $1, %xmm0, %xmm2
+shufpd      $1, (%rax), %xmm2
+
+sqrtpd      %xmm0, %xmm2
+sqrtpd      (%rax), %xmm2
+
+sqrtsd      %xmm0, %xmm2
+sqrtsd      (%rax), %xmm2
+
+subpd       %xmm0, %xmm2
+subpd       (%rax), %xmm2
+
+subsd       %xmm0, %xmm2
+subsd       (%rax), %xmm2
+
+ucomisd     %xmm0, %xmm1
+ucomisd     (%rax), %xmm1
+
+unpckhpd    %xmm0, %xmm2
+unpckhpd    (%rax), %xmm2
+
+unpcklpd    %xmm0, %xmm2
+unpcklpd    (%rax), %xmm2
+
+xorpd       %xmm0, %xmm2
+xorpd       (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        addpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        addsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        andnpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   andnpd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        andpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   andpd	(%rax), %xmm2
+# CHECK-NEXT:  4      2     0.50    *      *      U     clflush	(%rax)
+# CHECK-NEXT:  1      4     2.00                        cmpeqpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cmpeqpd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        cmpeqsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cmpeqsd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        comisd	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   comisd	(%rax), %xmm1
+# CHECK-NEXT:  2      5     0.50                        cvtdq2pd	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   cvtdq2pd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        cvtdq2ps	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtdq2ps	(%rax), %xmm2
+# CHECK-NEXT:  2      5     0.50                        cvtpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   cvtpd2dq	(%rax), %xmm2
+# CHECK-NEXT:  2      100   2.00                        cvtpd2pi	%xmm0, %mm2
+# CHECK-NEXT:  3      13    2.67    *                   cvtpd2pi	(%rax), %mm2
+# CHECK-NEXT:  2      5     0.50                        cvtpd2ps	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   cvtpd2ps	(%rax), %xmm2
+# CHECK-NEXT:  2      100   2.00                        cvtpi2pd	%mm0, %xmm2
+# CHECK-NEXT:  2      11    0.50    *                   cvtpi2pd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        cvtps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cvtps2dq	(%rax), %xmm2
+# CHECK-NEXT:  2      5     2.00                        cvtps2pd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtps2pd	(%rax), %xmm2
+# CHECK-NEXT:  2      7     3.50                        cvtsd2si	%xmm0, %ecx
+# CHECK-NEXT:  2      7     3.50                        cvtsd2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   cvtsd2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   cvtsd2si	(%rax), %rcx
+# CHECK-NEXT:  2      5     0.50                        cvtsd2ss	%xmm0, %xmm2
+# CHECK-NEXT:  3      11    2.00    *                   cvtsd2ss	(%rax), %xmm2
+# CHECK-NEXT:  2      8     2.00                        cvtsi2sd	%ecx, %xmm2
+# CHECK-NEXT:  2      8     2.00                        cvtsi2sd	%rcx, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  2      5     2.00                        cvtss2sd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   cvtss2sd	(%rax), %xmm2
+# CHECK-NEXT:  2      5     0.50                        cvttpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  3      12    0.50    *                   cvttpd2dq	(%rax), %xmm2
+# CHECK-NEXT:  2      100   2.00                        cvttpd2pi	%xmm0, %mm2
+# CHECK-NEXT:  3      13    2.67    *                   cvttpd2pi	(%rax), %mm2
+# CHECK-NEXT:  1      4     2.00                        cvttps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    2.00    *                   cvttps2dq	(%rax), %xmm2
+# CHECK-NEXT:  2      7     3.50                        cvttsd2si	%xmm0, %ecx
+# CHECK-NEXT:  2      7     3.50                        cvttsd2si	%xmm0, %rcx
+# CHECK-NEXT:  3      11    3.50    *                   cvttsd2si	(%rax), %ecx
+# CHECK-NEXT:  3      11    3.50    *                   cvttsd2si	(%rax), %rcx
+# CHECK-NEXT:  1      10    5.00                        divpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      16    5.00    *                   divpd	(%rax), %xmm2
+# CHECK-NEXT:  1      10    5.00                        divsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      20    0.50    *                   divsd	(%rax), %xmm2
+# CHECK-NEXT:  2      2     0.50    *      *      U     lfence
+# CHECK-NEXT:  2      1     0.50    *      *      U     maskmovdqu	%xmm0, %xmm1
+# CHECK-NEXT:  1      4     2.00                        maxpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   maxpd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        maxsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  2      3     0.50    *      *      U     mfence
+# CHECK-NEXT:  1      4     2.00                        minpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   minpd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        minsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   minsd	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        movapd	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movapd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   movapd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.33                        movd	%eax, %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   movd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        movd	%xmm0, %ecx
+# CHECK-NEXT:  2      12    0.50           *            movd	%xmm0, (%rax)
+# CHECK-NEXT:  2      1     0.50                        movdqa	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movdqa	%xmm0, (%rax)
+# CHECK-NEXT:  1      6     0.33    *                   movdqa	(%rax), %xmm2
+# CHECK-NEXT:  2      1     0.50                        movdqu	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movdqu	%xmm0, (%rax)
+# CHECK-NEXT:  1      6     0.33    *                   movdqu	(%rax), %xmm2
+# CHECK-NEXT:  2      3     0.50                        movdq2q	%xmm0, %mm2
+# CHECK-NEXT:  2      12    0.50           *            movhpd	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.50    *                   movhpd	(%rax), %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movlpd	%xmm0, (%rax)
+# CHECK-NEXT:  2      8     0.33    *                   movlpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        movmskpd	%xmm0, %ecx
+# CHECK-NEXT:  2      518   0.50           *            movntil	%eax, (%rax)
+# CHECK-NEXT:  2      512   0.50           *            movntiq	%rax, (%rax)
+# CHECK-NEXT:  2      512   0.50           *            movntdq	%xmm0, (%rax)
+# CHECK-NEXT:  2      518   0.50           *            movntpd	%xmm0, (%rax)
+# CHECK-NEXT:  1      1     0.25                        movq	%xmm0, %xmm2
+# CHECK-NEXT:  1      3     0.33                        movq	%rax, %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   movq	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        movq	%xmm0, %rcx
+# CHECK-NEXT:  2      12    0.50           *            movq	%xmm0, (%rax)
+# CHECK-NEXT:  2      3     0.50                        movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  1      1     0.25                        movsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movsd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     0.33    *                   movsd	(%rax), %xmm2
+# CHECK-NEXT:  0      1     0.00                        movupd	%xmm0, %xmm2
+# CHECK-NEXT:  2      12    0.50           *            movupd	%xmm0, (%rax)
+# CHECK-NEXT:  1      7     2.00    *                   movupd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        mulpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   mulpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        mulsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   mulsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        orpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   orpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        packssdw	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   packssdw	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        packsswb	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   packsswb	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        packuswb	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   packuswb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        paddb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        paddd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.33                        paddq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   paddq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        paddq	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        paddsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        paddsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddsw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        paddusb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddusb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        paddusw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddusw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        paddw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   paddw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        pand	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pand	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        pandn	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pandn	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pavgb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pavgb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pavgw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pavgw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpeqb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpeqb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpeqd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpeqd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpeqw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpeqw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpgtb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpgtd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpgtw	(%rax), %xmm2
+# CHECK-NEXT:  2      4     1.50                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      5     1.33                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      7     2.00    *                   pinsrw	$1, (%rax), %xmm0
+# CHECK-NEXT:  1      4     2.00                        pmaddwd	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmaddwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxsw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmaxub	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxub	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminsw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminub	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminub	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        pmovmskb	%xmm0, %ecx
+# CHECK-NEXT:  1      4     2.00                        pmulhuw	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmulhuw	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        pmulhw	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmulhw	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        pmullw	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmullw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmuludq	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmuludq	(%rax), %mm2
+# CHECK-NEXT:  1      3     1.50                        pmuludq	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   pmuludq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        por	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   por	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        psadbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   psadbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pshufd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   pshufd	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pshufhw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   pshufhw	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pshuflw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   pshuflw	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pslld	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        pslld	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pslld	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pslldq	$1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        psllq	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psllq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psllq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psllw	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psllw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psllw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psrad	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psrad	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psrad	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psraw	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psraw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psraw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psrld	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psrld	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psrld	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psrldq	$1, %xmm2
+# CHECK-NEXT:  1      1     0.50                        psrlq	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psrlq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psrlq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psrlw	$1, %xmm2
+# CHECK-NEXT:  2      2     0.50                        psrlw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   psrlw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.33                        psubq	%mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   psubq	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.25                        psubq	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psubsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psubsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubsw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psubusb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubusb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        psubusw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubusw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psubw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpckhbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpckhbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpckhdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpckhdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpckhqdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpckhqdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpckhwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpckhwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpcklbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpcklbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpckldq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpckldq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpcklqdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpcklqdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        punpcklwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   punpcklwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        pxor	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pxor	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        shufpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   shufpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      15    7.50                        sqrtpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      21    7.50    *                   sqrtpd	(%rax), %xmm2
+# CHECK-NEXT:  1      15    7.50                        sqrtsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      24    1.00    *                   sqrtsd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        subpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   subpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        subsd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   subsd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        ucomisd	%xmm0, %xmm1
+# CHECK-NEXT:  2      9     2.00    *                   ucomisd	(%rax), %xmm1
+# CHECK-NEXT:  1      1     0.50                        unpckhpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   unpckhpd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        unpcklpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   unpcklpd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        xorpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   xorpd	(%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 1.50   5.50   0.50   5.50   0.50   5.50   8.50   8.50   166.33 166.33 166.33 5.33   5.33   5.33    -     216.25 216.25 53.75  53.75
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   andnpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   andnpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   andpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   andpd	(%rax), %xmm2
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     clflush	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cmpeqpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cmpeqpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cmpeqsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cmpeqsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     comisd	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     comisd	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   cvtdq2pd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     cvtdq2pd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvtdq2ps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtdq2ps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   cvtpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   cvtpd2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvtpd2pi	%xmm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.67   2.67   2.67    -      -      -      -     2.00   2.00   0.50   0.50   cvtpd2pi	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   cvtpd2ps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   cvtpd2ps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvtpi2pd	%mm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     cvtpi2pd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvtps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtps2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvtps2pd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtps2pd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvtsd2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvtsd2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvtsd2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvtsd2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   cvtsd2ss	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00   0.50   0.50   cvtsd2ss	(%rax), %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvtsi2sd	%ecx, %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvtsi2sd	%rcx, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvtss2sd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvtss2sd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   cvttpd2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   cvttpd2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00   0.50   0.50   cvttpd2pi	%xmm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.67   2.67   2.67    -      -      -      -     2.00   2.00   0.50   0.50   cvttpd2pi	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     cvttps2dq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     cvttps2dq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvttsd2si	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.50   3.50    -      -     cvttsd2si	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvttsd2si	(%rax), %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.33   1.33   1.33    -      -      -      -     3.50   3.50    -      -     cvttsd2si	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     divpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     5.00   5.00    -      -     divpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   5.00    -      -     divsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   divsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     lfence
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     maskmovdqu	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     maxpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     maxpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     maxsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     mfence
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     minpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     minpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     minsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     minsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movapd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movapd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movapd	(%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movd	%eax, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movd	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   movdqa	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movdqa	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movdqa	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   movdqu	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movdqu	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movdqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   movdq2q	%xmm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movhpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   movhpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movlpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   movlpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     movmskpd	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntil	%eax, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntiq	%rax, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntdq	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movntpd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movq	%xmm0, %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movq	%rax, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     movq	%xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movq	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.25   0.25   movq2dq	%mm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   movsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movsd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     movupd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     movupd	%xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movupd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     mulpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     mulpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     mulsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     mulsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   orpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   orpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   packssdw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   packssdw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   packsswb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   packsswb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   packuswb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   packuswb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   paddb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   paddb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   paddd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   paddd	(%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     paddq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     paddq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   paddq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   paddq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     paddsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     paddsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     paddsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     paddsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     paddusb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     paddusb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     paddusw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     paddusw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   paddw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   paddw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pand	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   pand	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pandn	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   pandn	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pavgb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pavgb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pavgw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pavgw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpeqb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpeqb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpeqd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpeqd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpeqw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpeqw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpgtb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpgtb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpgtd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpgtd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpgtw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpgtw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pinsrw	$1, (%rax), %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmaddwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmaddwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxub	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxub	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminub	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminub	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     pmovmskb	%xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmulhuw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmulhuw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmulhw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmulhw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmullw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmullw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmuludq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmuludq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     pmuludq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     pmuludq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   por	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   por	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   psadbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   psadbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pshufd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pshufd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pshufhw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pshufhw	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pshuflw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pshuflw	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pslld	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   pslld	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pslld	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pslldq	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllq	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psllq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psllq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psllw	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psllw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psllw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrad	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psrad	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrad	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psraw	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psraw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psraw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrld	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psrld	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrld	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   psrldq	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlq	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psrlq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrlq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psrlw	$1, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   psrlw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psrlw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   psubb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   psubb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   psubd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   psubd	(%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     psubq	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     psubq	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   psubq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   psubq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psubsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psubsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psubsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psubsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psubusb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psubusb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psubusw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psubusw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   psubw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   psubw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhqdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhqdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckhwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckhwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpcklbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpcklbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpckldq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpckldq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpcklqdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpcklqdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   punpcklwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   punpcklwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pxor	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   pxor	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   shufpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   shufpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50   7.50    -      -     sqrtpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     7.50   7.50    -      -     sqrtpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.50   7.50    -      -     sqrtsd	%xmm0, %xmm2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sqrtsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   subpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   subpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   subsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   subsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50    -      -     ucomisd	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50    -      -     ucomisd	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   unpckhpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   unpckhpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   unpcklpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   unpcklpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   xorpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   xorpd	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse3.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse3.s
new file mode 100644
index 000000000000..bb6f33396f8e
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse3.s
@@ -0,0 +1,115 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+addsubpd  %xmm0, %xmm2
+addsubpd  (%rax),  %xmm2
+
+addsubps  %xmm0, %xmm2
+addsubps  (%rax), %xmm2
+
+haddpd    %xmm0, %xmm2
+haddpd    (%rax), %xmm2
+
+haddps    %xmm0, %xmm2
+haddps    (%rax), %xmm2
+
+hsubpd    %xmm0, %xmm2
+hsubpd    (%rax), %xmm2
+
+hsubps    %xmm0, %xmm2
+hsubps    (%rax), %xmm2
+
+lddqu     (%rax), %xmm2
+
+monitor
+
+movddup   %xmm0, %xmm2
+movddup   (%rax), %xmm2
+
+movshdup  %xmm0, %xmm2
+movshdup  (%rax), %xmm2
+
+movsldup  %xmm0, %xmm2
+movsldup  (%rax), %xmm2
+
+mwait
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        addsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addsubpd	(%rax), %xmm2
+# CHECK-NEXT:  1      3     0.50                        addsubps	%xmm0, %xmm2
+# CHECK-NEXT:  2      10    0.50    *                   addsubps	(%rax), %xmm2
+# CHECK-NEXT:  3      4     2.50                        haddpd	%xmm0, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   haddpd	(%rax), %xmm2
+# CHECK-NEXT:  3      4     2.50                        haddps	%xmm0, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   haddps	(%rax), %xmm2
+# CHECK-NEXT:  3      4     2.50                        hsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   hsubpd	(%rax), %xmm2
+# CHECK-NEXT:  3      4     2.50                        hsubps	%xmm0, %xmm2
+# CHECK-NEXT:  4      10    2.50    *                   hsubps	(%rax), %xmm2
+# CHECK-NEXT:  1      6     0.33    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.17                  U     monitor
+# CHECK-NEXT:  1      1     0.50                        movddup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   movddup	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        movshdup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   movshdup	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        movsldup	%xmm0, %xmm2
+# CHECK-NEXT:  1      6     2.00    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.17    *      *      U     mwait
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     15.00  15.00  15.00   -      -      -      -      -      -     23.50  23.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addsubpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   addsubps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   addsubps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   haddpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   haddpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   haddps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   haddps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   hsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   hsubpd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.50   2.50   hsubps	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     2.50   2.50   hsubps	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     monitor
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   movddup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movddup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   movshdup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movshdup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   movsldup	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     mwait
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse41.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse41.s
new file mode 100644
index 000000000000..96e55fe4e969
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse41.s
@@ -0,0 +1,377 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+blendpd     $11, %xmm0, %xmm2
+blendpd     $11, (%rax), %xmm2
+
+blendps     $11, %xmm0, %xmm2
+blendps     $11, (%rax), %xmm2
+
+blendvpd    %xmm0, %xmm2
+blendvpd    (%rax), %xmm2
+
+blendvps    %xmm0, %xmm2
+blendvps    (%rax), %xmm2
+
+dppd        $22, %xmm0, %xmm2
+dppd        $22, (%rax), %xmm2
+
+dpps        $22, %xmm0, %xmm2
+dpps        $22, (%rax), %xmm2
+
+extractps   $1, %xmm0, %rcx
+extractps   $1, %xmm0, (%rax)
+
+insertps    $1, %xmm0, %xmm2
+insertps    $1, (%rax), %xmm2
+
+movntdqa    (%rax), %xmm2
+
+mpsadbw     $1, %xmm0, %xmm2
+mpsadbw     $1, (%rax), %xmm2
+
+packusdw    %xmm0, %xmm2
+packusdw    (%rax), %xmm2
+
+pblendvb    %xmm0, %xmm2
+pblendvb    (%rax), %xmm2
+
+pblendw     $11, %xmm0, %xmm2
+pblendw     $11, (%rax), %xmm2
+
+pcmpeqq     %xmm0, %xmm2
+pcmpeqq     (%rax), %xmm2
+
+pextrb      $1, %xmm0, %ecx
+pextrb      $1, %xmm0, (%rax)
+
+pextrd      $1, %xmm0, %ecx
+pextrd      $1, %xmm0, (%rax)
+
+pextrq      $1, %xmm0, %rcx
+pextrq      $1, %xmm0, (%rax)
+
+pextrw      $1, %xmm0, (%rax)
+
+phminposuw  %xmm0, %xmm2
+phminposuw  (%rax), %xmm2
+
+pinsrb      $1, %eax, %xmm1
+pinsrb      $1, (%rax), %xmm1
+
+pinsrd      $1, %eax, %xmm1
+pinsrd      $1, (%rax), %xmm1
+
+pinsrq      $1, %rax, %xmm1
+pinsrq      $1, (%rax), %xmm1
+
+pmaxsb      %xmm0, %xmm2
+pmaxsb      (%rax), %xmm2
+
+pmaxsd      %xmm0, %xmm2
+pmaxsd      (%rax), %xmm2
+
+pmaxud      %xmm0, %xmm2
+pmaxud      (%rax), %xmm2
+
+pmaxuw      %xmm0, %xmm2
+pmaxuw      (%rax), %xmm2
+
+pminsb      %xmm0, %xmm2
+pminsb      (%rax), %xmm2
+
+pminsd      %xmm0, %xmm2
+pminsd      (%rax), %xmm2
+
+pminud      %xmm0, %xmm2
+pminud      (%rax), %xmm2
+
+pminuw      %xmm0, %xmm2
+pminuw      (%rax), %xmm2
+
+pmovsxbd    %xmm0, %xmm2
+pmovsxbd    (%rax), %xmm2
+
+pmovsxbq    %xmm0, %xmm2
+pmovsxbq    (%rax), %xmm2
+
+pmovsxbw    %xmm0, %xmm2
+pmovsxbw    (%rax), %xmm2
+
+pmovsxdq    %xmm0, %xmm2
+pmovsxdq    (%rax), %xmm2
+
+pmovsxwd    %xmm0, %xmm2
+pmovsxwd    (%rax), %xmm2
+
+pmovsxwq    %xmm0, %xmm2
+pmovsxwq    (%rax), %xmm2
+
+pmovzxbd    %xmm0, %xmm2
+pmovzxbd    (%rax), %xmm2
+
+pmovzxbq    %xmm0, %xmm2
+pmovzxbq    (%rax), %xmm2
+
+pmovzxbw    %xmm0, %xmm2
+pmovzxbw    (%rax), %xmm2
+
+pmovzxdq    %xmm0, %xmm2
+pmovzxdq    (%rax), %xmm2
+
+pmovzxwd    %xmm0, %xmm2
+pmovzxwd    (%rax), %xmm2
+
+pmovzxwq    %xmm0, %xmm2
+pmovzxwq    (%rax), %xmm2
+
+pmuldq      %xmm0, %xmm2
+pmuldq      (%rax), %xmm2
+
+pmulld      %xmm0, %xmm2
+pmulld      (%rax), %xmm2
+
+ptest       %xmm0, %xmm1
+ptest       (%rax), %xmm1
+
+roundpd     $1, %xmm0, %xmm2
+roundpd     $1, (%rax), %xmm2
+
+roundps     $1, %xmm0, %xmm2
+roundps     $1, (%rax), %xmm2
+
+roundsd     $1, %xmm0, %xmm2
+roundsd     $1, (%rax), %xmm2
+
+roundss     $1, %xmm0, %xmm2
+roundss     $1, (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.25                        blendpd	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   blendpd	$11, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        blendps	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   blendps	$11, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        blendvpd	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.33    *                   blendvpd	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        blendvps	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.33    *                   blendvps	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  3      8     1.00                        dppd	$22, %xmm0, %xmm2
+# CHECK-NEXT:  4      13    1.00    *                   dppd	$22, (%rax), %xmm2
+# CHECK-NEXT:  5      11    1.50                        dpps	$22, %xmm0, %xmm2
+# CHECK-NEXT:  6      18    1.50    *                   dpps	$22, (%rax), %xmm2
+# CHECK-NEXT:  2      4     0.50                        extractps	$1, %xmm0, %ecx
+# CHECK-NEXT:  3      12    0.50           *            extractps	$1, %xmm0, (%rax)
+# CHECK-NEXT:  1      1     0.50                        insertps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   insertps	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      7     0.33    *                   movntdqa	(%rax), %xmm2
+# CHECK-NEXT:  2      4     1.00                        mpsadbw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      11    1.00    *                   mpsadbw	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        packusdw	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   packusdw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     0.75                        pblendvb	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  4      8     2.00    *                   pblendvb	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.25                        pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpeqq	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pcmpeqq	(%rax), %xmm2
+# CHECK-NEXT:  2      4     1.50                        pextrb	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     0.50           *            pextrb	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      4     1.50                        pextrd	$1, %xmm0, %ecx
+# CHECK-NEXT:  3      12    0.50           *            pextrd	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      4     1.50                        pextrq	$1, %xmm0, %rcx
+# CHECK-NEXT:  3      12    0.50           *            pextrq	$1, %xmm0, (%rax)
+# CHECK-NEXT:  2      2     0.50           *            pextrw	$1, %xmm0, (%rax)
+# CHECK-NEXT:  1      5     0.50                        phminposuw	%xmm0, %xmm2
+# CHECK-NEXT:  1      11    0.50    *                   phminposuw	(%rax), %xmm2
+# CHECK-NEXT:  2      5     1.33                        pinsrb	$1, %eax, %xmm1
+# CHECK-NEXT:  2      7     2.00    *                   pinsrb	$1, (%rax), %xmm1
+# CHECK-NEXT:  2      5     1.33                        pinsrd	$1, %eax, %xmm1
+# CHECK-NEXT:  2      7     2.00    *                   pinsrd	$1, (%rax), %xmm1
+# CHECK-NEXT:  2      5     1.33                        pinsrq	$1, %rax, %xmm1
+# CHECK-NEXT:  2      7     2.00    *                   pinsrq	$1, (%rax), %xmm1
+# CHECK-NEXT:  1      1     0.50                        pmaxsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmaxsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmaxud	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxud	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmaxuw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pmaxuw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminud	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminud	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pminuw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pminuw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxbd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxbd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxbq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxbq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovsxwq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovsxwq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxbd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxbd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxbq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxbq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxbw	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxbw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxdq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxdq	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxwd	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxwd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        pmovzxwq	%xmm0, %xmm2
+# CHECK-NEXT:  2      8     0.50    *                   pmovzxwq	(%rax), %xmm2
+# CHECK-NEXT:  1      4     2.00                        pmuldq	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmuldq	(%rax), %xmm2
+# CHECK-NEXT:  2      10    0.50                        pmulld	%xmm0, %xmm2
+# CHECK-NEXT:  3      18    0.50    *                   pmulld	(%rax), %xmm2
+# CHECK-NEXT:  2      4     0.50                        ptest	%xmm0, %xmm1
+# CHECK-NEXT:  3      10    2.00    *                   ptest	(%rax), %xmm1
+# CHECK-NEXT:  2      8     4.00                        roundpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   roundpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  2      8     4.00                        roundps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   roundps	$1, (%rax), %xmm2
+# CHECK-NEXT:  2      8     4.00                        roundsd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   roundsd	$1, (%rax), %xmm2
+# CHECK-NEXT:  2      8     4.00                        roundss	$1, %xmm0, %xmm2
+# CHECK-NEXT:  3      14    4.00    *                   roundss	$1, (%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     4.67    -     4.67    -     4.67   2.50   2.50   53.00  53.00  53.00  1.67   1.67   1.67    -     62.50  62.50  35.00  35.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   blendpd	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   blendpd	$11, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   blendps	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   blendps	$11, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   blendvpd	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   blendvpd	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   blendvps	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   0.25   0.25   blendvps	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   0.50   0.50   dppd	$22, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00   1.00   0.50   0.50   dppd	$22, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   1.50   1.50   dpps	$22, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.50   1.50   1.50   1.50   dpps	$22, (%rax), %xmm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   extractps	$1, %xmm0, %ecx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     extractps	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   insertps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   insertps	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movntdqa	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   mpsadbw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     1.00   1.00   mpsadbw	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   packusdw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   packusdw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   pblendvb	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.75   0.75   0.75   0.75   pblendvb	%xmm0, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   pblendw	$11, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.25   0.25   0.25   0.25   pblendw	$11, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pcmpeqq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pcmpeqq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   pextrb	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   pextrb	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   pextrd	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   pextrd	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   0.50   0.50   pextrq	$1, %xmm0, %rcx
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   pextrq	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -     0.50   0.50   pextrw	$1, %xmm0, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     phminposuw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     phminposuw	(%rax), %xmm2
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pinsrb	$1, %eax, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pinsrb	$1, (%rax), %xmm1
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pinsrd	$1, %eax, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pinsrd	$1, (%rax), %xmm1
+# CHECK-NEXT:  -     1.33    -     1.33    -     1.33    -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pinsrq	$1, %rax, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pinsrq	$1, (%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxud	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxud	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaxuw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pmaxuw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminud	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminud	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pminuw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pminuw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxbd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxbd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxbq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxbq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovsxwq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovsxwq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxbd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxbd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxbq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxbq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxbw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxbw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxdq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxdq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxwd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxwd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pmovzxwq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   pmovzxwq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmuldq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmuldq	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   pmulld	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   pmulld	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   ptest	%xmm0, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     1.50   1.50   0.50   0.50   ptest	(%rax), %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     roundpd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     roundpd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     roundps	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     roundps	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     roundsd	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     roundsd	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     4.00   4.00    -      -     roundss	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     4.00   4.00    -      -     roundss	$1, (%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse42.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse42.s
new file mode 100644
index 000000000000..5bb1989d92d0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-sse42.s
@@ -0,0 +1,110 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+crc32b      %al, %ecx
+crc32b      (%rax), %ecx
+
+crc32l      %eax, %ecx
+crc32l      (%rax), %ecx
+
+crc32w      %ax, %ecx
+crc32w      (%rax), %ecx
+
+crc32b      %al, %rcx
+crc32b      (%rax), %rcx
+
+crc32q      %rax, %rcx
+crc32q      (%rax), %rcx
+
+pcmpestri   $1, %xmm0, %xmm2
+pcmpestri   $1, (%rax), %xmm2
+
+pcmpestrm   $1, %xmm0, %xmm2
+pcmpestrm   $1, (%rax), %xmm2
+
+pcmpistri   $1, %xmm0, %xmm2
+pcmpistri   $1, (%rax), %xmm2
+
+pcmpistrm   $1, %xmm0, %xmm2
+pcmpistrm   $1, (%rax), %xmm2
+
+pcmpgtq     %xmm0, %xmm2
+pcmpgtq     (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00                        crc32b	%al, %ecx
+# CHECK-NEXT:  1      7     1.33    *                   crc32b	(%rax), %ecx
+# CHECK-NEXT:  1      3     1.00                        crc32l	%eax, %ecx
+# CHECK-NEXT:  1      7     1.33    *                   crc32l	(%rax), %ecx
+# CHECK-NEXT:  1      3     1.00                        crc32w	%ax, %ecx
+# CHECK-NEXT:  1      7     1.33    *                   crc32w	(%rax), %ecx
+# CHECK-NEXT:  1      3     1.00                        crc32b	%al, %rcx
+# CHECK-NEXT:  1      7     1.33    *                   crc32b	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        crc32q	%rax, %rcx
+# CHECK-NEXT:  1      7     1.33    *                   crc32q	(%rax), %rcx
+# CHECK-NEXT:  8      16    3.00                        pcmpestri	$1, %xmm0, %xmm2
+# CHECK-NEXT:  8      31    3.00    *                   pcmpestri	$1, (%rax), %xmm2
+# CHECK-NEXT:  9      16    3.00                        pcmpestrm	$1, %xmm0, %xmm2
+# CHECK-NEXT:  9      17    3.00    *                   pcmpestrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  3      11    3.00                        pcmpistri	$1, %xmm0, %xmm2
+# CHECK-NEXT:  4      31    3.00    *                   pcmpistri	$1, (%rax), %xmm2
+# CHECK-NEXT:  3      11    3.00                        pcmpistrm	$1, %xmm0, %xmm2
+# CHECK-NEXT:  4      15    3.00    *                   pcmpistrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  1      3     1.50                        pcmpgtq	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     2.00    *                   pcmpgtq	(%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 25.67  15.00  1.67   15.00  1.67   15.00   -      -     10.00  10.00  10.00   -      -      -      -      -      -     3.00   3.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     crc32b	%al, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     crc32b	(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     crc32l	%eax, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     crc32l	(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     crc32w	%ax, %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     crc32w	(%rax), %ecx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     crc32b	%al, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     crc32b	(%rax), %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     crc32q	%rax, %rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     1.33   1.33   1.33    -      -      -      -      -      -      -      -     crc32q	(%rax), %rcx
+# CHECK-NEXT: 3.67   1.00   0.67   1.00   0.67   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     pcmpestri	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.17   1.17   0.17   1.17   0.17   1.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     pcmpestri	$1, (%rax), %xmm2
+# CHECK-NEXT: 3.50   1.50   0.50   1.50   0.50   1.50    -      -      -      -      -      -      -      -      -      -      -      -      -     pcmpestrm	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.33   1.33   0.33   1.33   0.33   1.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     pcmpestrm	$1, (%rax), %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     pcmpistri	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     pcmpistri	$1, (%rax), %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     pcmpistrm	$1, %xmm0, %xmm2
+# CHECK-NEXT: 3.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     pcmpistrm	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.50   1.50   pcmpgtq	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     1.50   1.50   pcmpgtq	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-ssse3.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-ssse3.s
new file mode 100644
index 000000000000..ae3568cef5b2
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-ssse3.s
@@ -0,0 +1,264 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+pabsb       %mm0, %mm2
+pabsb       (%rax), %mm2
+
+pabsb       %xmm0, %xmm2
+pabsb       (%rax), %xmm2
+
+pabsd       %mm0, %mm2
+pabsd       (%rax), %mm2
+
+pabsd       %xmm0, %xmm2
+pabsd       (%rax), %xmm2
+
+pabsw       %mm0, %mm2
+pabsw       (%rax), %mm2
+
+pabsw       %xmm0, %xmm2
+pabsw       (%rax), %xmm2
+
+palignr     $1, %mm0, %mm2
+palignr     $1, (%rax), %mm2
+
+palignr     $1, %xmm0, %xmm2
+palignr     $1, (%rax), %xmm2
+
+phaddd      %mm0, %mm2
+phaddd      (%rax), %mm2
+
+phaddd      %xmm0, %xmm2
+phaddd      (%rax), %xmm2
+
+phaddsw     %mm0, %mm2
+phaddsw     (%rax), %mm2
+
+phaddsw     %xmm0, %xmm2
+phaddsw     (%rax), %xmm2
+
+phaddw      %mm0, %mm2
+phaddw      (%rax), %mm2
+
+phaddw      %xmm0, %xmm2
+phaddw      (%rax), %xmm2
+
+phsubd      %mm0, %mm2
+phsubd      (%rax), %mm2
+
+phsubd      %xmm0, %xmm2
+phsubd      (%rax), %xmm2
+
+phsubsw     %mm0, %mm2
+phsubsw     (%rax), %mm2
+
+phsubsw     %xmm0, %xmm2
+phsubsw     (%rax), %xmm2
+
+phsubw      %mm0, %mm2
+phsubw      (%rax), %mm2
+
+phsubw      %xmm0, %xmm2
+phsubw      (%rax), %xmm2
+
+pmaddubsw   %mm0, %mm2
+pmaddubsw   (%rax), %mm2
+
+pmaddubsw   %xmm0, %xmm2
+pmaddubsw   (%rax), %xmm2
+
+pmulhrsw    %mm0, %mm2
+pmulhrsw    (%rax), %mm2
+
+pmulhrsw    %xmm0, %xmm2
+pmulhrsw    (%rax), %xmm2
+
+pshufb      %mm0, %mm2
+pshufb      (%rax), %mm2
+
+pshufb      %xmm0, %xmm2
+pshufb      (%rax), %xmm2
+
+psignb      %mm0, %mm2
+psignb      (%rax), %mm2
+
+psignb      %xmm0, %xmm2
+psignb      (%rax), %xmm2
+
+psignd      %mm0, %mm2
+psignd      (%rax), %mm2
+
+psignd      %xmm0, %xmm2
+psignd      (%rax), %xmm2
+
+psignw      %mm0, %mm2
+psignw      (%rax), %mm2
+
+psignw      %xmm0, %xmm2
+psignw      (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        pabsb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pabsb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pabsb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pabsb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        pabsd	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pabsd	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pabsd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pabsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        pabsw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   pabsw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pabsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   pabsw	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        palignr	$1, %mm0, %mm2
+# CHECK-NEXT:  2      9     0.50    *                   palignr	$1, (%rax), %mm2
+# CHECK-NEXT:  1      1     0.33                        palignr	$1, %xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   palignr	$1, (%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phaddd	%mm0, %mm2
+# CHECK-NEXT:  4      11    1.00    *                   phaddd	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phaddd	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     0.67                        phaddsw	%mm0, %mm2
+# CHECK-NEXT:  4      11    0.67    *                   phaddsw	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phaddsw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddsw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phaddw	%mm0, %mm2
+# CHECK-NEXT:  4      11    1.00    *                   phaddw	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phaddw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phaddw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phsubd	%mm0, %mm2
+# CHECK-NEXT:  4      11    1.00    *                   phsubd	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phsubd	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     0.67                        phsubsw	%mm0, %mm2
+# CHECK-NEXT:  4      11    0.67    *                   phsubsw	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phsubsw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubsw	(%rax), %xmm2
+# CHECK-NEXT:  3      3     1.00                        phsubw	%mm0, %mm2
+# CHECK-NEXT:  4      11    1.00    *                   phsubw	(%rax), %mm2
+# CHECK-NEXT:  3      2     1.00                        phsubw	%xmm0, %xmm2
+# CHECK-NEXT:  4      9     1.00    *                   phsubw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmaddubsw	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmaddubsw	(%rax), %mm2
+# CHECK-NEXT:  1      4     2.00                        pmaddubsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmaddubsw	(%rax), %xmm2
+# CHECK-NEXT:  1      5     0.50                        pmulhrsw	%mm0, %mm2
+# CHECK-NEXT:  2      13    0.50    *                   pmulhrsw	(%rax), %mm2
+# CHECK-NEXT:  1      4     2.00                        pmulhrsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      10    2.00    *                   pmulhrsw	(%rax), %xmm2
+# CHECK-NEXT:  2      3     0.50                        pshufb	%mm0, %mm2
+# CHECK-NEXT:  3      11    0.50    *                   pshufb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        pshufb	%xmm0, %xmm2
+# CHECK-NEXT:  2      7     2.00    *                   pshufb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        psignb	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psignb	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psignb	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psignb	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        psignd	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psignd	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psignd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psignd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        psignw	%mm0, %mm2
+# CHECK-NEXT:  2      9     1.00    *                   psignw	(%rax), %mm2
+# CHECK-NEXT:  1      1     0.50                        psignw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     2.00    *                   psignw	(%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     3.00    -     3.00    -     3.00    -      -     27.33  27.33  27.33   -      -      -      -     36.50  24.50  28.00  28.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pabsb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pabsb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pabsb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pabsb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pabsd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pabsd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pabsd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pabsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pabsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     pabsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pabsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     pabsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   palignr	$1, %mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   palignr	$1, (%rax), %mm2
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     palignr	$1, %xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   palignr	$1, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phaddd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phaddd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phaddd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phaddd	(%rax), %xmm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   phaddsw	%mm0, %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     phaddsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   phaddsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   1.00   1.00   phaddsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phaddw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phaddw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phaddw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phaddw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phsubd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phsubd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phsubd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phsubd	(%rax), %xmm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   phsubsw	%mm0, %mm2
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     phsubsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   1.00   1.00   phsubsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   1.00   1.00   phsubsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phsubw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phsubw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   1.25   1.25   phsubw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.25   0.25   1.25   1.25   phsubw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmaddubsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmaddubsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmaddubsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmaddubsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     pmulhrsw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     pmulhrsw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     pmulhrsw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     2.00   2.00    -      -     pmulhrsw	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   pshufb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50   0.50   0.50   pshufb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   pshufb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -      -      -     0.50   0.50   pshufb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psignb	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psignb	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psignb	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psignb	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psignd	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psignd	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psignd	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psignd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     psignw	%mm0, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     1.00    -      -      -     psignw	(%rax), %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     psignw	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.00   2.00   2.00    -      -      -      -     0.50   0.50    -      -     psignw	(%rax), %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vaes.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vaes.s
new file mode 100644
index 000000000000..65a8e7893714
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vaes.s
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vaesdec          %ymm0, %ymm1, %ymm3
+vaesdec          (%rax), %ymm1, %ymm3
+
+vaesdeclast      %ymm0, %ymm1, %ymm3
+vaesdeclast      (%rax), %ymm1, %ymm3
+
+vaesenc          %ymm0, %ymm1, %ymm3
+vaesenc          (%rax), %ymm1, %ymm3
+
+vaesenclast      %ymm0, %ymm1, %ymm3
+vaesenclast      (%rax), %ymm1, %ymm3
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     2.00                        vaesdec	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  2      11    2.33    *                   vaesdec	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  1      3     2.00                        vaesdeclast	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  2      11    2.33    *                   vaesdeclast	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  1      3     2.00                        vaesenc	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  2      11    2.33    *                   vaesenc	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  1      3     2.00                        vaesenclast	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  2      11    2.33    *                   vaesenclast	(%rax), %ymm1, %ymm3
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.33   9.33   9.33    -      -      -      -     16.00  16.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesdec	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesdec	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesdeclast	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesdeclast	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesenc	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesenc	(%rax), %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     2.00   2.00    -      -     vaesenclast	%ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -     2.00   2.00    -      -     vaesenclast	(%rax), %ymm1, %ymm3
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vpclmulqdq.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vpclmulqdq.s
new file mode 100644
index 000000000000..dd0e1ef249c4
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-vpclmulqdq.s
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+vpclmulqdq    $11,  %ymm0, %ymm1, %ymm3
+vpclmulqdq    $11, (%rax), %ymm1, %ymm3
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        vpclmulqdq	$11, %ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  2      10    2.33    *                   vpclmulqdq	$11, (%rax), %ymm1, %ymm3
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     2.00   2.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   vpclmulqdq	$11, %ymm0, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     2.33   2.33   2.33    -      -      -      -      -      -     1.50   1.50   vpclmulqdq	$11, (%rax), %ymm1, %ymm3
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_32.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_32.s
new file mode 100644
index 000000000000..f88c52e0cc33
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_32.s
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=i686-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+aaa
+
+aad
+aad $7
+
+aam
+aam $7
+
+aas
+
+bound %bx, (%eax)
+bound %ebx, (%eax)
+
+daa
+
+das
+
+into
+
+leave
+
+salc
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.17                        aaa
+# CHECK-NEXT:  1      100   0.17                        aad
+# CHECK-NEXT:  1      100   0.17                        aad	$7
+# CHECK-NEXT:  1      100   0.17                        aam
+# CHECK-NEXT:  1      100   0.17                        aam	$7
+# CHECK-NEXT:  1      100   0.17                        aas
+# CHECK-NEXT:  1      100   0.17                  U     bound	%bx, (%eax)
+# CHECK-NEXT:  1      100   0.17                  U     bound	%ebx, (%eax)
+# CHECK-NEXT:  1      100   0.17                        daa
+# CHECK-NEXT:  1      100   0.17                        das
+# CHECK-NEXT:  1      100   0.17                  U     into
+# CHECK-NEXT:  1      1     0.33    *                   leave
+# CHECK-NEXT:  1      1     0.17                  U     salc
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 2.00   2.33   2.00   2.33   2.00   2.33    -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aaa
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aad
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aad	$7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aam
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aam	$7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     aas
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     bound	%bx, (%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     bound	%ebx, (%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     daa
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     das
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     into
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     leave
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     salc
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_64.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_64.s
new file mode 100644
index 000000000000..c0fb7f6e6923
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x86_64.s
@@ -0,0 +1,2886 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+adcb $0, %al
+adcb $0, %dil
+adcb $0, (%rax)
+lock adcb $0, (%rax)
+adcb $7, %al
+adcb $7, %dil
+adcb $7, (%rax)
+lock adcb $7, (%rax)
+adcb %sil, %dil
+adcb %sil, (%rax)
+lock adcb %sil, (%rax)
+adcb (%rax), %dil
+
+adcw $0, %ax
+adcw $0, %di
+adcw $0, (%rax)
+lock adcw $0, (%rax)
+adcw $511, %ax
+adcw $511, %di
+adcw $511, (%rax)
+lock adcw $511, (%rax)
+adcw $7, %di
+adcw $7, (%rax)
+lock adcw $7, (%rax)
+adcw %si, %di
+adcw %si, (%rax)
+lock adcw %si, (%rax)
+adcw (%rax), %di
+
+adcl $0, %eax
+adcl $0, %edi
+adcl $0, (%rax)
+lock adcl $0, (%rax)
+adcl $665536, %eax
+adcl $665536, %edi
+adcl $665536, (%rax)
+lock adcl $665536, (%rax)
+adcl $7, %edi
+adcl $7, (%rax)
+lock adcl $7, (%rax)
+adcl %esi, %edi
+adcl %esi, (%rax)
+lock adcl %esi, (%rax)
+adcl (%rax), %edi
+
+adcq $0, %rax
+adcq $0, %rdi
+adcq $0, (%rax)
+lock adcq $0, (%rax)
+adcq $665536, %rax
+adcq $665536, %rdi
+adcq $665536, (%rax)
+lock adcq $665536, (%rax)
+adcq $7, %rdi
+adcq $7, (%rax)
+lock adcq $7, (%rax)
+adcq %rsi, %rdi
+adcq %rsi, (%rax)
+lock adcq %rsi, (%rax)
+adcq (%rax), %rdi
+
+addb $7, %al
+addb $7, %dil
+addb $7, (%rax)
+lock addb $7, (%rax)
+addb %sil, %dil
+addb %sil, (%rax)
+lock addb %sil, (%rax)
+addb (%rax), %dil
+
+addw $511, %ax
+addw $511, %di
+addw $511, (%rax)
+lock addw $511, (%rax)
+addw $7, %di
+addw $7, (%rax)
+lock addw $7, (%rax)
+addw %si, %di
+addw %si, (%rax)
+lock addw %si, (%rax)
+addw (%rax), %di
+
+addl $665536, %eax
+addl $665536, %edi
+addl $665536, (%rax)
+lock addl $665536, (%rax)
+addl $7, %edi
+addl $7, (%rax)
+lock addl $7, (%rax)
+addl %esi, %edi
+addl %esi, (%rax)
+lock addl %esi, (%rax)
+addl (%rax), %edi
+
+addq $665536, %rax
+addq $665536, %rdi
+addq $665536, (%rax)
+lock addq $665536, (%rax)
+addq $7, %rdi
+addq $7, (%rax)
+lock addq $7, (%rax)
+addq %rsi, %rdi
+addq %rsi, (%rax)
+lock addq %rsi, (%rax)
+addq (%rax), %rdi
+
+andb $7, %al
+andb $7, %dil
+andb $7, (%rax)
+lock andb $7, (%rax)
+andb %sil, %dil
+andb %sil, (%rax)
+lock andb %sil, (%rax)
+andb (%rax), %dil
+
+andw $511, %ax
+andw $511, %di
+andw $511, (%rax)
+lock andw $511, (%rax)
+andw $7, %di
+andw $7, (%rax)
+lock andw $7, (%rax)
+andw %si, %di
+andw %si, (%rax)
+lock andw %si, (%rax)
+andw (%rax), %di
+
+andl $665536, %eax
+andl $665536, %edi
+andl $665536, (%rax)
+lock andl $665536, (%rax)
+andl $7, %edi
+andl $7, (%rax)
+lock andl $7, (%rax)
+andl %esi, %edi
+andl %esi, (%rax)
+lock andl %esi, (%rax)
+andl (%rax), %edi
+
+andq $665536, %rax
+andq $665536, %rdi
+andq $665536, (%rax)
+lock andq $665536, (%rax)
+andq $7, %rdi
+andq $7, (%rax)
+lock andq $7, (%rax)
+andq %rsi, %rdi
+andq %rsi, (%rax)
+lock andq %rsi, (%rax)
+andq (%rax), %rdi
+
+bsfw %si, %di
+bsrw %si, %di
+bsfw (%rax), %di
+bsrw (%rax), %di
+
+bsfl %esi, %edi
+bsrl %esi, %edi
+bsfl (%rax), %edi
+bsrl (%rax), %edi
+
+bsfq %rsi, %rdi
+bsrq %rsi, %rdi
+bsfq (%rax), %rdi
+bsrq (%rax), %rdi
+
+bswap %eax
+bswap %rax
+
+btw  %si, %di
+btcw %si, %di
+btrw %si, %di
+btsw %si, %di
+btw  %si, (%rax)
+btcw %si, (%rax)
+btrw %si, (%rax)
+btsw %si, (%rax)
+lock btcw %si, (%rax)
+lock btrw %si, (%rax)
+lock btsw %si, (%rax)
+btw  $7, %di
+btcw $7, %di
+btrw $7, %di
+btsw $7, %di
+btw  $7, (%rax)
+btcw $7, (%rax)
+btrw $7, (%rax)
+btsw $7, (%rax)
+lock btcw $7, (%rax)
+lock btrw $7, (%rax)
+lock btsw $7, (%rax)
+
+btl  %esi, %edi
+btcl %esi, %edi
+btrl %esi, %edi
+btsl %esi, %edi
+btl  %esi, (%rax)
+btcl %esi, (%rax)
+btrl %esi, (%rax)
+btsl %esi, (%rax)
+lock btcl %esi, (%rax)
+lock btrl %esi, (%rax)
+lock btsl %esi, (%rax)
+btl  $7, %edi
+btcl $7, %edi
+btrl $7, %edi
+btsl $7, %edi
+btl  $7, (%rax)
+btcl $7, (%rax)
+btrl $7, (%rax)
+btsl $7, (%rax)
+lock btcl $7, (%rax)
+lock btrl $7, (%rax)
+lock btsl $7, (%rax)
+
+btq  %rsi, %rdi
+btcq %rsi, %rdi
+btrq %rsi, %rdi
+btsq %rsi, %rdi
+btq  %rsi, (%rax)
+btcq %rsi, (%rax)
+btrq %rsi, (%rax)
+btsq %rsi, (%rax)
+lock btcq %rsi, (%rax)
+lock btrq %rsi, (%rax)
+lock btsq %rsi, (%rax)
+btq  $7, %rdi
+btcq $7, %rdi
+btrq $7, %rdi
+btsq $7, %rdi
+btq  $7, (%rax)
+btcq $7, (%rax)
+btrq $7, (%rax)
+btsq $7, (%rax)
+lock btcq $7, (%rax)
+lock btrq $7, (%rax)
+lock btsq $7, (%rax)
+
+cbw
+cwde
+cdqe
+cwd
+cdq
+cqo
+
+clc
+cld
+cmc
+
+cmpb $7, %al
+cmpb $7, %dil
+cmpb $7, (%rax)
+cmpb %sil, %dil
+cmpb %sil, (%rax)
+cmpb (%rax), %dil
+
+cmpw $511, %ax
+cmpw $511, %di
+cmpw $511, (%rax)
+cmpw $7, %di
+cmpw $7, (%rax)
+cmpw %si, %di
+cmpw %si, (%rax)
+cmpw (%rax), %di
+
+cmpl $665536, %eax
+cmpl $665536, %edi
+cmpl $665536, (%rax)
+cmpl $7, %edi
+cmpl $7, (%rax)
+cmpl %esi, %edi
+cmpl %esi, (%rax)
+cmpl (%rax), %edi
+
+cmpq $665536, %rax
+cmpq $665536, %rdi
+cmpq $665536, (%rax)
+cmpq $7, %rdi
+cmpq $7, (%rax)
+cmpq %rsi, %rdi
+cmpq %rsi, (%rax)
+cmpq (%rax), %rdi
+
+cmpsb
+cmpsw
+cmpsl
+cmpsq
+
+cmpxchgb %cl, %bl
+cmpxchgb %cl, (%rbx)
+lock cmpxchgb %cl, (%rbx)
+
+cmpxchgw %cx, %bx
+cmpxchgw %cx, (%rbx)
+lock cmpxchgw %cx, (%rbx)
+
+cmpxchgl %ecx, %ebx
+cmpxchgl %ecx, (%rbx)
+lock cmpxchgl %ecx, (%rbx)
+
+cmpxchgq %rcx, %rbx
+cmpxchgq %rcx, (%rbx)
+lock cmpxchgq %rcx, (%rbx)
+
+cpuid
+
+decb %dil
+decb (%rax)
+lock decb (%rax)
+decw %di
+decw (%rax)
+lock decw (%rax)
+decl %edi
+decl (%rax)
+lock decl (%rax)
+decq %rdi
+decq (%rax)
+lock decq (%rax)
+
+divb %dil
+divb (%rax)
+divw %si
+divw (%rax)
+divl %edx
+divl (%rax)
+divq %rcx
+divq (%rax)
+
+enter $7, $4095
+
+idivb %dil
+idivb (%rax)
+idivw %si
+idivw (%rax)
+idivl %edx
+idivl (%rax)
+idivq %rcx
+idivq (%rax)
+
+imulb %dil
+imulb (%rax)
+
+imulw %di
+imulw (%rax)
+imulw %si, %di
+imulw (%rax), %di
+imulw $511, %si, %di
+imulw $511, (%rax), %di
+imulw $7, %si, %di
+imulw $7, (%rax), %di
+
+imull %edi
+imull (%rax)
+imull %esi, %edi
+imull (%rax), %edi
+imull $665536, %esi, %edi
+imull $665536, (%rax), %edi
+imull $7, %esi, %edi
+imull $7, (%rax), %edi
+
+imulq %rdi
+imulq (%rax)
+imulq %rsi, %rdi
+imulq (%rax), %rdi
+imulq $665536, %rsi, %rdi
+imulq $665536, (%rax), %rdi
+imulq $7, %rsi, %rdi
+imulq $7, (%rax), %rdi
+
+inb $7,  %al
+inb %dx, %al
+inw $7,  %ax
+inw %dx, %ax
+inl $7,  %eax
+inl %dx, %eax
+
+incb %dil
+incb (%rax)
+lock incb (%rax)
+incw %di
+incw (%rax)
+lock incw (%rax)
+incl %edi
+incl (%rax)
+lock incl (%rax)
+incq %rdi
+incq (%rax)
+lock incq (%rax)
+
+insb
+insw
+insl
+
+int $7
+
+invlpg  (%rax)
+invlpga %rax, %ecx
+
+leave
+
+lodsb
+lodsw
+lodsl
+lodsq
+
+loop 0
+loope 0
+loopne 0
+
+movsb
+movsw
+movsl
+movsq
+
+movsbw %al, %di
+movzbw %al, %di
+movsbw (%rax), %di
+movzbw (%rax), %di
+movsbl %al, %edi
+movzbl %al, %edi
+movsbl (%rax), %edi
+movzbl (%rax), %edi
+movsbq %al, %rdi
+movzbq %al, %rdi
+movsbq (%rax), %rdi
+movzbq (%rax), %rdi
+
+movswl %ax, %edi
+movzwl %ax, %edi
+movswl (%rax), %edi
+movzwl (%rax), %edi
+movswq %ax, %rdi
+movzwq %ax, %rdi
+movswq (%rax), %rdi
+movzwq (%rax), %rdi
+
+movslq %eax, %rdi
+movslq (%rax), %rdi
+
+mulb %dil
+mulb (%rax)
+mulw %si
+mulw (%rax)
+mull %edx
+mull (%rax)
+mulq %rcx
+mulq (%rax)
+
+negb %dil
+negb (%r8)
+lock negb (%r8)
+negw %si
+negw (%r9)
+lock negw (%r9)
+negl %edx
+negl (%rax)
+lock negl (%rax)
+negq %rcx
+negq (%r10)
+lock negq (%r10)
+
+nop
+nopw %di
+nopw (%rcx)
+nopl %esi
+nopl (%r8)
+nopq %rdx
+nopq (%r9)
+
+notb %dil
+notb (%r8)
+lock notb (%r8)
+notw %si
+notw (%r9)
+lock notw (%r9)
+notl %edx
+notl (%rax)
+lock notl (%rax)
+notq %rcx
+notq (%r10)
+lock notq (%r10)
+
+orb $7, %al
+orb $7, %dil
+orb $7, (%rax)
+lock orb $7, (%rax)
+orb %sil, %dil
+orb %sil, (%rax)
+lock orb %sil, (%rax)
+orb (%rax), %dil
+
+orw $511, %ax
+orw $511, %di
+orw $511, (%rax)
+lock orw $511, (%rax)
+orw $7, %di
+orw $7, (%rax)
+lock orw $7, (%rax)
+orw %si, %di
+orw %si, (%rax)
+lock orw %si, (%rax)
+orw (%rax), %di
+
+orl $665536, %eax
+orl $665536, %edi
+orl $665536, (%rax)
+lock orl $665536, (%rax)
+orl $7, %edi
+orl $7, (%rax)
+lock orl $7, (%rax)
+orl %esi, %edi
+orl %esi, (%rax)
+lock orl %esi, (%rax)
+orl (%rax), %edi
+
+orq $665536, %rax
+orq $665536, %rdi
+orq $665536, (%rax)
+lock orq $665536, (%rax)
+orq $7, %rdi
+orq $7, (%rax)
+lock orq $7, (%rax)
+orq %rsi, %rdi
+orq %rsi, (%rax)
+lock orq %rsi, (%rax)
+orq (%rax), %rdi
+
+outb %al,  $7
+outb %al,  %dx
+outw %ax,  $7
+outw %ax,  %dx
+outl %eax, $7
+outl %eax, %dx
+
+outsb
+outsw
+outsl
+
+pause
+
+rclb %dil
+rcrb %dil
+rclb (%rax)
+rcrb (%rax)
+rclb $7, %dil
+rcrb $7, %dil
+rclb $7, (%rax)
+rcrb $7, (%rax)
+rclb %cl, %dil
+rcrb %cl, %dil
+rclb %cl, (%rax)
+rcrb %cl, (%rax)
+
+rclw %di
+rcrw %di
+rclw (%rax)
+rcrw (%rax)
+rclw $7, %di
+rcrw $7, %di
+rclw $7, (%rax)
+rcrw $7, (%rax)
+rclw %cl, %di
+rcrw %cl, %di
+rclw %cl, (%rax)
+rcrw %cl, (%rax)
+
+rcll %edi
+rcrl %edi
+rcll (%rax)
+rcrl (%rax)
+rcll $7, %edi
+rcrl $7, %edi
+rcll $7, (%rax)
+rcrl $7, (%rax)
+rcll %cl, %edi
+rcrl %cl, %edi
+rcll %cl, (%rax)
+rcrl %cl, (%rax)
+
+rclq %rdi
+rcrq %rdi
+rclq (%rax)
+rcrq (%rax)
+rclq $7, %rdi
+rcrq $7, %rdi
+rclq $7, (%rax)
+rcrq $7, (%rax)
+rclq %cl, %rdi
+rcrq %cl, %rdi
+rclq %cl, (%rax)
+rcrq %cl, (%rax)
+
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
+rolb %dil
+rorb %dil
+rolb (%rax)
+rorb (%rax)
+rolb $7, %dil
+rorb $7, %dil
+rolb $7, (%rax)
+rorb $7, (%rax)
+rolb %cl, %dil
+rorb %cl, %dil
+rolb %cl, (%rax)
+rorb %cl, (%rax)
+
+rolw %di
+rorw %di
+rolw (%rax)
+rorw (%rax)
+rolw $7, %di
+rorw $7, %di
+rolw $7, (%rax)
+rorw $7, (%rax)
+rolw %cl, %di
+rorw %cl, %di
+rolw %cl, (%rax)
+rorw %cl, (%rax)
+
+roll %edi
+rorl %edi
+roll (%rax)
+rorl (%rax)
+roll $7, %edi
+rorl $7, %edi
+roll $7, (%rax)
+rorl $7, (%rax)
+roll %cl, %edi
+rorl %cl, %edi
+roll %cl, (%rax)
+rorl %cl, (%rax)
+
+rolq %rdi
+rorq %rdi
+rolq (%rax)
+rorq (%rax)
+rolq $7, %rdi
+rorq $7, %rdi
+rolq $7, (%rax)
+rorq $7, (%rax)
+rolq %cl, %rdi
+rorq %cl, %rdi
+rolq %cl, (%rax)
+rorq %cl, (%rax)
+
+sahf
+
+sarb %dil
+shlb %dil
+shrb %dil
+sarb (%rax)
+shlb (%rax)
+shrb (%rax)
+sarb $7, %dil
+shlb $7, %dil
+shrb $7, %dil
+sarb $7, (%rax)
+shlb $7, (%rax)
+shrb $7, (%rax)
+sarb %cl, %dil
+shlb %cl, %dil
+shrb %cl, %dil
+sarb %cl, (%rax)
+shlb %cl, (%rax)
+shrb %cl, (%rax)
+
+sarw %di
+shlw %di
+shrw %di
+sarw (%rax)
+shlw (%rax)
+shrw (%rax)
+sarw $7, %di
+shlw $7, %di
+shrw $7, %di
+sarw $7, (%rax)
+shlw $7, (%rax)
+shrw $7, (%rax)
+sarw %cl, %di
+shlw %cl, %di
+shrw %cl, %di
+sarw %cl, (%rax)
+shlw %cl, (%rax)
+shrw %cl, (%rax)
+
+sarl %edi
+shll %edi
+shrl %edi
+sarl (%rax)
+shll (%rax)
+shrl (%rax)
+sarl $7, %edi
+shll $7, %edi
+shrl $7, %edi
+sarl $7, (%rax)
+shll $7, (%rax)
+shrl $7, (%rax)
+sarl %cl, %edi
+shll %cl, %edi
+shrl %cl, %edi
+sarl %cl, (%rax)
+shll %cl, (%rax)
+shrl %cl, (%rax)
+
+sarq %rdi
+shlq %rdi
+shrq %rdi
+sarq (%rax)
+shlq (%rax)
+shrq (%rax)
+sarq $7, %rdi
+shlq $7, %rdi
+shrq $7, %rdi
+sarq $7, (%rax)
+shlq $7, (%rax)
+shrq $7, (%rax)
+sarq %cl, %rdi
+shlq %cl, %rdi
+shrq %cl, %rdi
+sarq %cl, (%rax)
+shlq %cl, (%rax)
+shrq %cl, (%rax)
+
+sbbb $0, %al
+sbbb $0, %dil
+sbbb $0, (%rax)
+lock sbbb $0, (%rax)
+sbbb $7, %al
+sbbb $7, %dil
+sbbb $7, (%rax)
+lock sbbb $7, (%rax)
+sbbb %sil, %dil
+sbbb %sil, (%rax)
+lock sbbb %sil, (%rax)
+sbbb (%rax), %dil
+
+sbbw $0, %ax
+sbbw $0, %di
+sbbw $0, (%rax)
+lock sbbw $0, (%rax)
+sbbw $511, %ax
+sbbw $511, %di
+sbbw $511, (%rax)
+lock sbbw $511, (%rax)
+sbbw $7, %di
+sbbw $7, (%rax)
+lock sbbw $7, (%rax)
+sbbw %si, %di
+sbbw %si, (%rax)
+lock sbbw %si, (%rax)
+sbbw (%rax), %di
+
+sbbl $0, %eax
+sbbl $0, %edi
+sbbl $0, (%rax)
+lock sbbl $0, (%rax)
+sbbl $665536, %eax
+sbbl $665536, %edi
+sbbl $665536, (%rax)
+lock sbbl $665536, (%rax)
+sbbl $7, %edi
+sbbl $7, (%rax)
+lock sbbl $7, (%rax)
+sbbl %esi, %edi
+sbbl %esi, (%rax)
+lock sbbl %esi, (%rax)
+sbbl (%rax), %edi
+
+sbbq $0, %rax
+sbbq $0, %rdi
+sbbq $0, (%rax)
+lock sbbq $0, (%rax)
+sbbq $665536, %rax
+sbbq $665536, %rdi
+sbbq $665536, (%rax)
+lock sbbq $665536, (%rax)
+sbbq $7, %rdi
+sbbq $7, (%rax)
+lock sbbq $7, (%rax)
+sbbq %rsi, %rdi
+sbbq %rsi, (%rax)
+lock sbbq %rsi, (%rax)
+sbbq (%rax), %rdi
+
+scasb
+scasw
+scasl
+scasq
+
+seto  %al
+seto  (%rax)
+setno %al
+setno (%rax)
+setb  %al
+setb  (%rax)
+setnb %al
+setnb (%rax)
+setz  %al
+setz  (%rax)
+setnz %al
+setnz (%rax)
+seta  %al
+seta  (%rax)
+setna %al
+setna (%rax)
+sets  %al
+sets  (%rax)
+setns %al
+setns (%rax)
+setp  %al
+setp  (%rax)
+setnp %al
+setnp (%rax)
+setl  %al
+setl  (%rax)
+setnl %al
+setnl (%rax)
+setg  %al
+setg  (%rax)
+setng %al
+setng (%rax)
+
+shldw %cl, %si, %di
+shrdw %cl, %si, %di
+shldw %cl, %si, (%rax)
+shrdw %cl, %si, (%rax)
+shldw $7, %si, %di
+shrdw $7, %si, %di
+shldw $7, %si, (%rax)
+shrdw $7, %si, (%rax)
+
+shldl %cl, %esi, %edi
+shrdl %cl, %esi, %edi
+shldl %cl, %esi, (%rax)
+shrdl %cl, %esi, (%rax)
+shldl $7, %esi, %edi
+shrdl $7, %esi, %edi
+shldl $7, %esi, (%rax)
+shrdl $7, %esi, (%rax)
+
+shldq %cl, %rsi, %rdi
+shrdq %cl, %rsi, %rdi
+shldq %cl, %rsi, (%rax)
+shrdq %cl, %rsi, (%rax)
+shldq $7, %rsi, %rdi
+shrdq $7, %rsi, %rdi
+shldq $7, %rsi, (%rax)
+shrdq $7, %rsi, (%rax)
+
+stc
+std
+
+stosb
+stosw
+stosl
+stosq
+
+subb $7, %al
+subb $7, %dil
+subb $7, (%rax)
+lock subb $7, (%rax)
+subb %sil, %dil
+subb %sil, (%rax)
+lock subb %sil, (%rax)
+subb (%rax), %dil
+
+subw $511, %ax
+subw $511, %di
+subw $511, (%rax)
+lock subw $511, (%rax)
+subw $7, %di
+subw $7, (%rax)
+lock subw $7, (%rax)
+subw %si, %di
+subw %si, (%rax)
+lock subw %si, (%rax)
+subw (%rax), %di
+
+subl $665536, %eax
+subl $665536, %edi
+subl $665536, (%rax)
+lock subl $665536, (%rax)
+subl $7, %edi
+subl $7, (%rax)
+lock subl $7, (%rax)
+subl %esi, %edi
+subl %esi, (%rax)
+lock subl %esi, (%rax)
+subl (%rax), %edi
+
+subq $665536, %rax
+subq $665536, %rdi
+subq $665536, (%rax)
+lock subq $665536, (%rax)
+subq $7, %rdi
+subq $7, (%rax)
+lock subq $7, (%rax)
+subq %rsi, %rdi
+subq %rsi, (%rax)
+lock subq %rsi, (%rax)
+subq (%rax), %rdi
+
+testb $7, %al
+testb $7, %dil
+testb $7, (%rax)
+testb %sil, %dil
+testb %sil, (%rax)
+
+testw $511, %ax
+testw $511, %di
+testw $511, (%rax)
+testw $7, %di
+testw $7, (%rax)
+testw %si, %di
+testw %si, (%rax)
+
+testl $665536, %eax
+testl $665536, %edi
+testl $665536, (%rax)
+testl $7, %edi
+testl $7, (%rax)
+testl %esi, %edi
+testl %esi, (%rax)
+
+testq $665536, %rax
+testq $665536, %rdi
+testq $665536, (%rax)
+testq $7, %rdi
+testq $7, (%rax)
+testq %rsi, %rdi
+testq %rsi, (%rax)
+
+ud2
+
+wrmsr
+
+xaddb %bl, %cl
+xaddb %bl, (%rcx)
+lock xaddb %bl, (%rcx)
+
+xaddw %bx, %cx
+xaddw %ax, (%rbx)
+lock xaddw %ax, (%rbx)
+
+xaddl %ebx, %ecx
+xaddl %eax, (%rbx)
+lock xaddl %eax, (%rbx)
+
+xaddq %rbx, %rcx
+xaddq %rax, (%rbx)
+lock xaddq %rax, (%rbx)
+
+xchgb %bl, %cl
+xchgb %bl, (%rbx)
+lock xchgb %bl, (%rbx)
+
+xchgw %ax, %bx
+xchgw %bx, %cx
+xchgw %ax, (%rbx)
+lock xchgw %ax, (%rbx)
+
+xchgl %eax, %ebx
+xchgl %ebx, %ecx
+xchgl %eax, (%rbx)
+lock xchgl %eax, (%rbx)
+
+xchgq %rax, %rbx
+xchgq %rbx, %rcx
+xchgq %rax, (%rbx)
+lock xchgq %rax, (%rbx)
+
+xlatb
+
+xorb $7, %al
+xorb $7, %dil
+xorb $7, (%rax)
+lock xorb $7, (%rax)
+xorb %sil, %dil
+xorb %sil, (%rax)
+lock xorb %sil, (%rax)
+xorb (%rax), %dil
+
+xorw $511, %ax
+xorw $511, %di
+xorw $511, (%rax)
+lock xorw $511, (%rax)
+xorw $7, %di
+xorw $7, (%rax)
+lock xorw $7, (%rax)
+xorw %si, %di
+xorw %si, (%rax)
+lock xorw %si, (%rax)
+xorw (%rax), %di
+
+xorl $665536, %eax
+xorl $665536, %edi
+xorl $665536, (%rax)
+lock xorl $665536, (%rax)
+xorl $7, %edi
+xorl $7, (%rax)
+lock xorl $7, (%rax)
+xorl %esi, %edi
+xorl %esi, (%rax)
+lock xorl %esi, (%rax)
+xorl (%rax), %edi
+
+xorq $665536, %rax
+xorq $665536, %rdi
+xorq $665536, (%rax)
+lock xorq $665536, (%rax)
+xorq $7, %rdi
+xorq $7, (%rax)
+lock xorq $7, (%rax)
+xorq %rsi, %rdi
+xorq %rsi, (%rax)
+lock xorq %rsi, (%rax)
+xorq (%rax), %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.33                        adcb	$0, %al
+# CHECK-NEXT:  1      1     0.33                        adcb	$0, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            adcb	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcb	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcb	$7, %al
+# CHECK-NEXT:  1      1     0.33                        adcb	$7, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            adcb	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcb	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcb	%sil, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            adcb	%sil, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   adcb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        adcw	$0, %ax
+# CHECK-NEXT:  1      1     0.33                        adcw	$0, %di
+# CHECK-NEXT:  5      12    0.50    *      *            adcw	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcw	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        adcw	$511, %di
+# CHECK-NEXT:  5      12    0.50    *      *            adcw	$511, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcw	$7, %di
+# CHECK-NEXT:  5      12    0.50    *      *            adcw	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcw	%si, %di
+# CHECK-NEXT:  6      12    0.50    *      *            adcw	%si, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		adcw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   adcw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        adcl	$0, %eax
+# CHECK-NEXT:  1      1     0.33                        adcl	$0, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            adcl	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcl	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcl	$665536, %eax
+# CHECK-NEXT:  1      1     0.33                        adcl	$665536, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            adcl	$665536, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcl	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcl	$7, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            adcl	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcl	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcl	%esi, %edi
+# CHECK-NEXT:  6      12    0.50    *      *            adcl	%esi, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		adcl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   adcl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        adcq	$0, %rax
+# CHECK-NEXT:  1      1     0.33                        adcq	$0, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            adcq	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcq	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcq	$665536, %rax
+# CHECK-NEXT:  1      1     0.33                        adcq	$665536, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            adcq	$665536, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcq	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcq	$7, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            adcq	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		adcq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        adcq	%rsi, %rdi
+# CHECK-NEXT:  6      12    0.50    *      *            adcq	%rsi, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		adcq	%rsi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   adcq	(%rax), %rdi
+# CHECK-NEXT:  1      1     0.33                        addb	$7, %al
+# CHECK-NEXT:  1      1     0.33                        addb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            addb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		addb	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addb	%sil, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            addb	%sil, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		addb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   addb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        addw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        addw	$511, %di
+# CHECK-NEXT:  4      12    0.67    *      *            addw	$511, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addw	$7, %di
+# CHECK-NEXT:  4      12    0.67    *      *            addw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addw	%si, %di
+# CHECK-NEXT:  4      12    0.67    *      *            addw	%si, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   addw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        addl	$665536, %eax
+# CHECK-NEXT:  1      1     0.33                        addl	$665536, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            addl	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addl	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addl	$7, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            addl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addl	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addl	%esi, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            addl	%esi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   addl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        addq	$665536, %rax
+# CHECK-NEXT:  1      1     0.33                        addq	$665536, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            addq	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addq	$665536, (%rax)
+# CHECK-NEXT:  0      1     0.00                        addq	$7, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            addq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        addq	%rsi, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            addq	%rsi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		addq	%rsi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   addq	(%rax), %rdi
+# CHECK-NEXT:  1      2     0.17                        andb	$7, %al
+# CHECK-NEXT:  1      2     0.17                        andb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            andb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		andb	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andb	%sil, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            andb	%sil, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		andb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   andb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        andw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        andw	$511, %di
+# CHECK-NEXT:  4      12    0.67    *      *            andw	$511, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andw	$511, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andw	$7, %di
+# CHECK-NEXT:  4      12    0.67    *      *            andw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andw	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andw	%si, %di
+# CHECK-NEXT:  4      12    0.67    *      *            andw	%si, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   andw	(%rax), %di
+# CHECK-NEXT:  1      2     0.17                        andl	$665536, %eax
+# CHECK-NEXT:  1      2     0.17                        andl	$665536, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            andl	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andl	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andl	$7, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            andl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andl	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andl	%esi, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            andl	%esi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   andl	(%rax), %edi
+# CHECK-NEXT:  1      2     0.17                        andq	$665536, %rax
+# CHECK-NEXT:  1      2     0.17                        andq	$665536, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            andq	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andq	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andq	$7, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            andq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andq	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        andq	%rsi, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            andq	%rsi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		andq	%rsi, (%rax)
+# CHECK-NEXT:  2      7     0.33    *                   andq	(%rax), %rdi
+# CHECK-NEXT:  1      3     0.33                        bsfw	%si, %di
+# CHECK-NEXT:  1      3     0.33                        bsrw	%si, %di
+# CHECK-NEXT:  2      7     0.33    *                   bsfw	(%rax), %di
+# CHECK-NEXT:  2      7     0.33    *                   bsrw	(%rax), %di
+# CHECK-NEXT:  1      3     0.33                        bsfl	%esi, %edi
+# CHECK-NEXT:  1      3     0.33                        bsrl	%esi, %edi
+# CHECK-NEXT:  2      7     0.33    *                   bsfl	(%rax), %edi
+# CHECK-NEXT:  2      7     0.33    *                   bsrl	(%rax), %edi
+# CHECK-NEXT:  1      3     0.33                        bsfq	%rsi, %rdi
+# CHECK-NEXT:  1      3     0.33                        bsrq	%rsi, %rdi
+# CHECK-NEXT:  2      7     0.33    *                   bsfq	(%rax), %rdi
+# CHECK-NEXT:  2      7     0.33    *                   bsrq	(%rax), %rdi
+# CHECK-NEXT:  1      1     0.33                        bswapl	%eax
+# CHECK-NEXT:  2      2     0.67                        bswapq	%rax
+# CHECK-NEXT:  1      1     0.33                        btw	%si, %di
+# CHECK-NEXT:  1      1     0.33                        btcw	%si, %di
+# CHECK-NEXT:  1      1     0.33                        btrw	%si, %di
+# CHECK-NEXT:  1      1     0.33                        btsw	%si, %di
+# CHECK-NEXT:  10     11    1.50    *                   btw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btcw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btrw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btsw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btcw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btrw	%si, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btsw	%si, (%rax)
+# CHECK-NEXT:  1      1     0.33                        btw	$7, %di
+# CHECK-NEXT:  1      1     0.33                        btcw	$7, %di
+# CHECK-NEXT:  1      1     0.33                        btrw	$7, %di
+# CHECK-NEXT:  1      1     0.33                        btsw	$7, %di
+# CHECK-NEXT:  2      6     0.33    *                   btw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btcw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btrw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btsw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btcw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btrw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btsw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        btl	%esi, %edi
+# CHECK-NEXT:  1      1     0.33                        btcl	%esi, %edi
+# CHECK-NEXT:  1      1     0.33                        btrl	%esi, %edi
+# CHECK-NEXT:  1      1     0.33                        btsl	%esi, %edi
+# CHECK-NEXT:  10     11    1.50    *                   btl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btcl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btrl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            btsl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btcl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btrl	%esi, (%rax)
+# CHECK-NEXT:  11     18    1.33    *      *            lock		btsl	%esi, (%rax)
+# CHECK-NEXT:  1      1     0.33                        btl	$7, %edi
+# CHECK-NEXT:  1      1     0.33                        btcl	$7, %edi
+# CHECK-NEXT:  1      1     0.33                        btrl	$7, %edi
+# CHECK-NEXT:  1      1     0.33                        btsl	$7, %edi
+# CHECK-NEXT:  2      6     0.33    *                   btl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btcl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btrl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btsl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btcl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btrl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btsl	$7, (%rax)
+# CHECK-NEXT:  1      3     1.00                        btq	%rsi, %rdi
+# CHECK-NEXT:  1      3     1.00                        btcq	%rsi, %rdi
+# CHECK-NEXT:  1      3     1.00                        btrq	%rsi, %rdi
+# CHECK-NEXT:  1      3     1.00                        btsq	%rsi, %rdi
+# CHECK-NEXT:  9      10    1.33    *                   btq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            btcq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            btrq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            btsq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            lock		btcq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            lock		btrq	%rsi, (%rax)
+# CHECK-NEXT:  10     17    1.17    *      *            lock		btsq	%rsi, (%rax)
+# CHECK-NEXT:  1      1     0.33                        btq	$7, %rdi
+# CHECK-NEXT:  1      1     0.33                        btcq	$7, %rdi
+# CHECK-NEXT:  1      1     0.33                        btrq	$7, %rdi
+# CHECK-NEXT:  1      1     0.33                        btsq	$7, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   btq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btcq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btrq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            btsq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btcq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btrq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.50    *      *            lock		btsq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.50                        cbtw
+# CHECK-NEXT:  1      1     0.50                        cwtl
+# CHECK-NEXT:  1      1     0.50                        cltq
+# CHECK-NEXT:  2      2     0.33                        cwtd
+# CHECK-NEXT:  1      1     0.33                        cltd
+# CHECK-NEXT:  1      1     0.33                        cqto
+# CHECK-NEXT:  0      1     0.00                  U     clc
+# CHECK-NEXT:  2      3     0.33                  U     cld
+# CHECK-NEXT:  1      1     0.33                  U     cmc
+# CHECK-NEXT:  1      1     0.33                        cmpb	$7, %al
+# CHECK-NEXT:  1      1     0.33                        cmpb	$7, %dil
+# CHECK-NEXT:  2      6     0.33    *                   cmpb	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpb	%sil, %dil
+# CHECK-NEXT:  2      6     0.33    *                   cmpb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   cmpb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        cmpw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        cmpw	$511, %di
+# CHECK-NEXT:  2      6     0.33    *                   cmpw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpw	$7, %di
+# CHECK-NEXT:  2      6     0.33    *                   cmpw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpw	%si, %di
+# CHECK-NEXT:  2      6     0.33    *                   cmpw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   cmpw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        cmpl	$665536, %eax
+# CHECK-NEXT:  1      1     0.33                        cmpl	$665536, %edi
+# CHECK-NEXT:  2      6     0.33    *                   cmpl	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpl	$7, %edi
+# CHECK-NEXT:  2      6     0.33    *                   cmpl	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpl	%esi, %edi
+# CHECK-NEXT:  2      6     0.33    *                   cmpl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   cmpl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        cmpq	$665536, %rax
+# CHECK-NEXT:  1      1     0.33                        cmpq	$665536, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   cmpq	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpq	$7, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   cmpq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        cmpq	%rsi, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   cmpq	%rsi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   cmpq	(%rax), %rdi
+# CHECK-NEXT:  7      6     0.83                  U     cmpsb	%es:(%rdi), (%rsi)
+# CHECK-NEXT:  7      6     0.83                  U     cmpsw	%es:(%rdi), (%rsi)
+# CHECK-NEXT:  7      6     0.83                  U     cmpsl	%es:(%rdi), (%rsi)
+# CHECK-NEXT:  7      6     0.83                  U     cmpsq	%es:(%rdi), (%rsi)
+# CHECK-NEXT:  5      3     0.83                        cmpxchgb	%cl, %bl
+# CHECK-NEXT:  6      13    0.67    *      *            cmpxchgb	%cl, (%rbx)
+# CHECK-NEXT:  6      13    0.67    *      *            lock		cmpxchgb	%cl, (%rbx)
+# CHECK-NEXT:  5      3     0.83                        cmpxchgw	%cx, %bx
+# CHECK-NEXT:  6      12    0.67    *      *            cmpxchgw	%cx, (%rbx)
+# CHECK-NEXT:  6      12    0.67    *      *            lock		cmpxchgw	%cx, (%rbx)
+# CHECK-NEXT:  5      3     0.83                        cmpxchgl	%ecx, %ebx
+# CHECK-NEXT:  6      12    0.67    *      *            cmpxchgl	%ecx, (%rbx)
+# CHECK-NEXT:  6      12    0.67    *      *            lock		cmpxchgl	%ecx, (%rbx)
+# CHECK-NEXT:  5      3     0.83                        cmpxchgq	%rcx, %rbx
+# CHECK-NEXT:  6      12    0.67    *      *            cmpxchgq	%rcx, (%rbx)
+# CHECK-NEXT:  6      12    0.67    *      *            lock		cmpxchgq	%rcx, (%rbx)
+# CHECK-NEXT:  26     18    6.00                  U     cpuid
+# CHECK-NEXT:  1      1     0.33                        decb	%dil
+# CHECK-NEXT:  4      13    0.50    *      *            decb	(%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		decb	(%rax)
+# CHECK-NEXT:  1      1     0.33                        decw	%di
+# CHECK-NEXT:  4      12    0.67    *      *            decw	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		decw	(%rax)
+# CHECK-NEXT:  1      1     0.33                        decl	%edi
+# CHECK-NEXT:  4      12    0.67    *      *            decl	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		decl	(%rax)
+# CHECK-NEXT:  0      1     0.00                        decq	%rdi
+# CHECK-NEXT:  4      12    0.67    *      *            decq	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		decq	(%rax)
+# CHECK-NEXT:  3      17    3.00                  U     divb	%dil
+# CHECK-NEXT:  3      22    3.00    *             U     divb	(%rax)
+# CHECK-NEXT:  4      16    3.00                  U     divw	%si
+# CHECK-NEXT:  5      20    3.00    *             U     divw	(%rax)
+# CHECK-NEXT:  4      15    3.00                  U     divl	%edx
+# CHECK-NEXT:  5      19    3.00    *             U     divl	(%rax)
+# CHECK-NEXT:  3      18    3.00                  U     divq	%rcx
+# CHECK-NEXT:  4      22    3.00    *             U     divq	(%rax)
+# CHECK-NEXT:  57     126   7.67                  U     enter	$7, $4095
+# CHECK-NEXT:  3      17    1.00                  U     idivb	%dil
+# CHECK-NEXT:  3      22    1.00    *             U     idivb	(%rax)
+# CHECK-NEXT:  4      16    1.00                  U     idivw	%si
+# CHECK-NEXT:  5      20    1.00    *             U     idivw	(%rax)
+# CHECK-NEXT:  4      15    1.00                  U     idivl	%edx
+# CHECK-NEXT:  5      19    1.00    *             U     idivl	(%rax)
+# CHECK-NEXT:  3      18    1.00                  U     idivq	%rcx
+# CHECK-NEXT:  4      22    1.00    *             U     idivq	(%rax)
+# CHECK-NEXT:  1      3     0.33                        imulb	%dil
+# CHECK-NEXT:  2      7     0.33    *                   imulb	(%rax)
+# CHECK-NEXT:  4      5     0.67                        imulw	%di
+# CHECK-NEXT:  5      9     0.67    *                   imulw	(%rax)
+# CHECK-NEXT:  1      3     0.33                        imulw	%si, %di
+# CHECK-NEXT:  2      7     0.33    *                   imulw	(%rax), %di
+# CHECK-NEXT:  2      4     0.33                        imulw	$511, %si, %di
+# CHECK-NEXT:  3      8     0.33    *                   imulw	$511, (%rax), %di
+# CHECK-NEXT:  2      4     0.33                        imulw	$7, %si, %di
+# CHECK-NEXT:  3      8     0.33    *                   imulw	$7, (%rax), %di
+# CHECK-NEXT:  3      4     0.33                        imull	%edi
+# CHECK-NEXT:  4      8     0.33    *                   imull	(%rax)
+# CHECK-NEXT:  1      3     0.33                        imull	%esi, %edi
+# CHECK-NEXT:  2      7     0.33    *                   imull	(%rax), %edi
+# CHECK-NEXT:  1      2     0.33                        imull	$665536, %esi, %edi
+# CHECK-NEXT:  2      6     0.33    *                   imull	$665536, (%rax), %edi
+# CHECK-NEXT:  1      2     0.33                        imull	$7, %esi, %edi
+# CHECK-NEXT:  2      6     0.33    *                   imull	$7, (%rax), %edi
+# CHECK-NEXT:  2      4     0.33                        imulq	%rdi
+# CHECK-NEXT:  3      8     0.33    *                   imulq	(%rax)
+# CHECK-NEXT:  1      3     0.33                        imulq	%rsi, %rdi
+# CHECK-NEXT:  2      7     0.33    *                   imulq	(%rax), %rdi
+# CHECK-NEXT:  1      2     0.33                        imulq	$665536, %rsi, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   imulq	$665536, (%rax), %rdi
+# CHECK-NEXT:  1      2     0.33                        imulq	$7, %rsi, %rdi
+# CHECK-NEXT:  2      6     0.33    *                   imulq	$7, (%rax), %rdi
+# CHECK-NEXT:  1      100   0.17                  U     inb	$7, %al
+# CHECK-NEXT:  1      100   0.17                  U     inb	%dx, %al
+# CHECK-NEXT:  1      100   0.17                  U     inw	$7, %ax
+# CHECK-NEXT:  1      100   0.17                  U     inw	%dx, %ax
+# CHECK-NEXT:  1      100   0.17                  U     inl	$7, %eax
+# CHECK-NEXT:  1      100   0.17                  U     inl	%dx, %eax
+# CHECK-NEXT:  1      1     0.33                        incb	%dil
+# CHECK-NEXT:  4      13    0.50    *      *            incb	(%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		incb	(%rax)
+# CHECK-NEXT:  1      1     0.33                        incw	%di
+# CHECK-NEXT:  4      12    0.67    *      *            incw	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		incw	(%rax)
+# CHECK-NEXT:  1      1     0.33                        incl	%edi
+# CHECK-NEXT:  4      12    0.67    *      *            incl	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		incl	(%rax)
+# CHECK-NEXT:  0      1     0.00                        incq	%rdi
+# CHECK-NEXT:  4      12    0.67    *      *            incq	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		incq	(%rax)
+# CHECK-NEXT:  1      100   0.17                  U     insb	%dx, %es:(%rdi)
+# CHECK-NEXT:  1      100   0.17                  U     insw	%dx, %es:(%rdi)
+# CHECK-NEXT:  1      100   0.17                  U     insl	%dx, %es:(%rdi)
+# CHECK-NEXT:  1      100   0.17    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.17                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.17                  U     invlpga
+# CHECK-NEXT:  1      1     0.33    *                   leave
+# CHECK-NEXT:  1      100   0.17                  U     lodsb	(%rsi), %al
+# CHECK-NEXT:  1      100   0.17                  U     lodsw	(%rsi), %ax
+# CHECK-NEXT:  1      100   0.17                  U     lodsl	(%rsi), %eax
+# CHECK-NEXT:  1      100   0.17                  U     lodsq	(%rsi), %rax
+# CHECK-NEXT:  1      1     0.33                  U     loop	0
+# CHECK-NEXT:  11     3     2.00                  U     loope	0
+# CHECK-NEXT:  11     2     2.00                  U     loopne	0
+# CHECK-NEXT:  7      8     0.67                  U     movsb	(%rsi), %es:(%rdi)
+# CHECK-NEXT:  7      7     0.67                  U     movsw	(%rsi), %es:(%rdi)
+# CHECK-NEXT:  7      7     0.67                  U     movsl	(%rsi), %es:(%rdi)
+# CHECK-NEXT:  7      7     0.67                  U     movsq	(%rsi), %es:(%rdi)
+# CHECK-NEXT:  1      1     0.33                        movsbw	%al, %di
+# CHECK-NEXT:  1      1     0.33                        movzbw	%al, %di
+# CHECK-NEXT:  2      6     0.33    *                   movsbw	(%rax), %di
+# CHECK-NEXT:  2      6     0.33    *                   movzbw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        movsbl	%al, %edi
+# CHECK-NEXT:  1      0     0.17                        movzbl	%al, %edi
+# CHECK-NEXT:  1      6     0.33    *                   movsbl	(%rax), %edi
+# CHECK-NEXT:  1      4     0.33    *                   movzbl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        movsbq	%al, %rdi
+# CHECK-NEXT:  1      0     0.17                        movzbq	%al, %rdi
+# CHECK-NEXT:  1      6     0.33    *                   movsbq	(%rax), %rdi
+# CHECK-NEXT:  1      4     0.33    *                   movzbq	(%rax), %rdi
+# CHECK-NEXT:  1      1     0.33                        movswl	%ax, %edi
+# CHECK-NEXT:  1      1     0.33                        movzwl	%ax, %edi
+# CHECK-NEXT:  1      6     0.33    *                   movswl	(%rax), %edi
+# CHECK-NEXT:  1      4     0.33    *                   movzwl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        movswq	%ax, %rdi
+# CHECK-NEXT:  1      1     0.33                        movzwq	%ax, %rdi
+# CHECK-NEXT:  1      6     0.33    *                   movswq	(%rax), %rdi
+# CHECK-NEXT:  1      4     0.33    *                   movzwq	(%rax), %rdi
+# CHECK-NEXT:  1      1     0.33                        movslq	%eax, %rdi
+# CHECK-NEXT:  1      6     0.33    *                   movslq	(%rax), %rdi
+# CHECK-NEXT:  1      3     0.33                        mulb	%dil
+# CHECK-NEXT:  2      7     0.33    *                   mulb	(%rax)
+# CHECK-NEXT:  4      5     0.67                        mulw	%si
+# CHECK-NEXT:  5      9     0.67    *                   mulw	(%rax)
+# CHECK-NEXT:  3      4     0.33                        mull	%edx
+# CHECK-NEXT:  4      8     0.33    *                   mull	(%rax)
+# CHECK-NEXT:  2      4     0.33                        mulq	%rcx
+# CHECK-NEXT:  3      8     0.33    *                   mulq	(%rax)
+# CHECK-NEXT:  1      1     0.33                        negb	%dil
+# CHECK-NEXT:  4      13    0.50    *      *            negb	(%r8)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		negb	(%r8)
+# CHECK-NEXT:  1      1     0.33                        negw	%si
+# CHECK-NEXT:  4      12    0.67    *      *            negw	(%r9)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		negw	(%r9)
+# CHECK-NEXT:  1      1     0.33                        negl	%edx
+# CHECK-NEXT:  4      12    0.67    *      *            negl	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		negl	(%rax)
+# CHECK-NEXT:  1      1     0.33                        negq	%rcx
+# CHECK-NEXT:  4      12    0.67    *      *            negq	(%r10)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		negq	(%r10)
+# CHECK-NEXT:  0      1     0.00                        nop
+# CHECK-NEXT:  0      1     0.00                        nopw	%di
+# CHECK-NEXT:  0      1     0.00                        nopw	(%rcx)
+# CHECK-NEXT:  0      1     0.00                        nopl	%esi
+# CHECK-NEXT:  0      1     0.00                        nopl	(%r8)
+# CHECK-NEXT:  0      1     0.00                        nopq	%rdx
+# CHECK-NEXT:  0      1     0.00                        nopq	(%r9)
+# CHECK-NEXT:  1      1     0.33                        notb	%dil
+# CHECK-NEXT:  4      13    0.50    *      *            notb	(%r8)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		notb	(%r8)
+# CHECK-NEXT:  1      1     0.33                        notw	%si
+# CHECK-NEXT:  4      12    0.67    *      *            notw	(%r9)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		notw	(%r9)
+# CHECK-NEXT:  1      1     0.33                        notl	%edx
+# CHECK-NEXT:  4      12    0.67    *      *            notl	(%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		notl	(%rax)
+# CHECK-NEXT:  1      1     0.33                        notq	%rcx
+# CHECK-NEXT:  4      12    0.67    *      *            notq	(%r10)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		notq	(%r10)
+# CHECK-NEXT:  1      2     0.17                        orb	$7, %al
+# CHECK-NEXT:  1      2     0.17                        orb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            orb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		orb	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orb	%sil, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            orb	%sil, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		orb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   orb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        orw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        orw	$511, %di
+# CHECK-NEXT:  4      12    0.67    *      *            orw	$511, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orw	$511, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orw	$7, %di
+# CHECK-NEXT:  4      12    0.67    *      *            orw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orw	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orw	%si, %di
+# CHECK-NEXT:  4      12    0.67    *      *            orw	%si, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   orw	(%rax), %di
+# CHECK-NEXT:  1      2     0.17                        orl	$665536, %eax
+# CHECK-NEXT:  1      2     0.17                        orl	$665536, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            orl	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orl	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orl	$7, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            orl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orl	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orl	%esi, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            orl	%esi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   orl	(%rax), %edi
+# CHECK-NEXT:  1      2     0.17                        orq	$665536, %rax
+# CHECK-NEXT:  1      2     0.17                        orq	$665536, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            orq	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orq	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orq	$7, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            orq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orq	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        orq	%rsi, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            orq	%rsi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		orq	%rsi, (%rax)
+# CHECK-NEXT:  2      7     0.33    *                   orq	(%rax), %rdi
+# CHECK-NEXT:  1      100   0.17                  U     outb	%al, $7
+# CHECK-NEXT:  1      100   0.17                  U     outb	%al, %dx
+# CHECK-NEXT:  1      100   0.17                  U     outw	%ax, $7
+# CHECK-NEXT:  1      100   0.17                  U     outw	%ax, %dx
+# CHECK-NEXT:  1      100   0.17                  U     outl	%eax, $7
+# CHECK-NEXT:  1      100   0.17                  U     outl	%eax, %dx
+# CHECK-NEXT:  1      100   0.17                  U     outsb	(%rsi), %dx
+# CHECK-NEXT:  1      100   0.17                  U     outsw	(%rsi), %dx
+# CHECK-NEXT:  1      100   0.17                  U     outsl	(%rsi), %dx
+# CHECK-NEXT:  0      1     0.00    *      *      U     pause
+# CHECK-NEXT:  3      2     0.67                        rclb	%dil
+# CHECK-NEXT:  3      2     0.67                        rcrb	%dil
+# CHECK-NEXT:  6      13    0.67    *      *            rclb	(%rax)
+# CHECK-NEXT:  6      13    0.67    *      *            rcrb	(%rax)
+# CHECK-NEXT:  3      2     0.67                        rclb	$7, %dil
+# CHECK-NEXT:  3      2     0.67                        rcrb	$7, %dil
+# CHECK-NEXT:  6      13    0.67    *      *            rclb	$7, (%rax)
+# CHECK-NEXT:  6      13    0.67    *      *            rcrb	$7, (%rax)
+# CHECK-NEXT:  9      7     1.67                        rclb	%cl, %dil
+# CHECK-NEXT:  10     9     1.67                        rcrb	%cl, %dil
+# CHECK-NEXT:  11     20    1.67    *      *            rclb	%cl, (%rax)
+# CHECK-NEXT:  12     20    1.50    *      *            rcrb	%cl, (%rax)
+# CHECK-NEXT:  3      2     0.67                        rclw	%di
+# CHECK-NEXT:  3      2     0.67                        rcrw	%di
+# CHECK-NEXT:  6      12    0.67    *      *            rclw	(%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrw	(%rax)
+# CHECK-NEXT:  3      2     0.67                        rclw	$7, %di
+# CHECK-NEXT:  3      2     0.67                        rcrw	$7, %di
+# CHECK-NEXT:  6      12    0.67    *      *            rclw	$7, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrw	$7, (%rax)
+# CHECK-NEXT:  7      8     1.17                        rclw	%cl, %di
+# CHECK-NEXT:  7      8     1.17                        rcrw	%cl, %di
+# CHECK-NEXT:  10     19    1.67    *      *            rclw	%cl, (%rax)
+# CHECK-NEXT:  10     19    1.67    *      *            rcrw	%cl, (%rax)
+# CHECK-NEXT:  3      2     0.67                        rcll	%edi
+# CHECK-NEXT:  3      2     0.67                        rcrl	%edi
+# CHECK-NEXT:  6      12    0.67    *      *            rcll	(%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrl	(%rax)
+# CHECK-NEXT:  3      2     0.67                        rcll	$7, %edi
+# CHECK-NEXT:  3      2     0.67                        rcrl	$7, %edi
+# CHECK-NEXT:  6      12    0.67    *      *            rcll	$7, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrl	$7, (%rax)
+# CHECK-NEXT:  7      8     1.17                        rcll	%cl, %edi
+# CHECK-NEXT:  7      8     1.17                        rcrl	%cl, %edi
+# CHECK-NEXT:  10     19    1.67    *      *            rcll	%cl, (%rax)
+# CHECK-NEXT:  10     19    1.67    *      *            rcrl	%cl, (%rax)
+# CHECK-NEXT:  3      2     0.67                        rclq	%rdi
+# CHECK-NEXT:  3      2     0.67                        rcrq	%rdi
+# CHECK-NEXT:  6      12    0.67    *      *            rclq	(%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrq	(%rax)
+# CHECK-NEXT:  3      2     0.67                        rclq	$7, %rdi
+# CHECK-NEXT:  3      2     0.67                        rcrq	$7, %rdi
+# CHECK-NEXT:  6      12    0.67    *      *            rclq	$7, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            rcrq	$7, (%rax)
+# CHECK-NEXT:  7      8     1.17                        rclq	%cl, %rdi
+# CHECK-NEXT:  7      8     1.17                        rcrq	%cl, %rdi
+# CHECK-NEXT:  10     19    1.67    *      *            rclq	%cl, (%rax)
+# CHECK-NEXT:  10     19    1.67    *      *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.17                  U     rdmsr
+# CHECK-NEXT:  1      100   0.17                  U     rdpmc
+# CHECK-NEXT:  15     18    2.50                  U     rdtsc
+# CHECK-NEXT:  1      100   0.17                  U     rdtscp
+# CHECK-NEXT:  2      1     0.67                        rolb	%dil
+# CHECK-NEXT:  2      1     0.67                        rorb	%dil
+# CHECK-NEXT:  5      13    0.67    *      *            rolb	(%rax)
+# CHECK-NEXT:  5      13    0.67    *      *            rorb	(%rax)
+# CHECK-NEXT:  2      1     0.67                        rolb	$7, %dil
+# CHECK-NEXT:  2      1     0.67                        rorb	$7, %dil
+# CHECK-NEXT:  5      13    0.67    *      *            rolb	$7, (%rax)
+# CHECK-NEXT:  5      13    0.67    *      *            rorb	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        rolb	%cl, %dil
+# CHECK-NEXT:  2      2     0.67                        rorb	%cl, %dil
+# CHECK-NEXT:  5      13    0.67    *      *            rolb	%cl, (%rax)
+# CHECK-NEXT:  5      13    0.67    *      *            rorb	%cl, (%rax)
+# CHECK-NEXT:  2      1     0.67                        rolw	%di
+# CHECK-NEXT:  2      1     0.67                        rorw	%di
+# CHECK-NEXT:  5      12    0.67    *      *            rolw	(%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorw	(%rax)
+# CHECK-NEXT:  2      1     0.67                        rolw	$7, %di
+# CHECK-NEXT:  2      1     0.67                        rorw	$7, %di
+# CHECK-NEXT:  5      12    0.67    *      *            rolw	$7, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorw	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        rolw	%cl, %di
+# CHECK-NEXT:  2      2     0.67                        rorw	%cl, %di
+# CHECK-NEXT:  5      12    0.67    *      *            rolw	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorw	%cl, (%rax)
+# CHECK-NEXT:  2      1     0.67                        roll	%edi
+# CHECK-NEXT:  2      1     0.67                        rorl	%edi
+# CHECK-NEXT:  5      12    0.67    *      *            roll	(%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorl	(%rax)
+# CHECK-NEXT:  2      1     0.67                        roll	$7, %edi
+# CHECK-NEXT:  2      1     0.67                        rorl	$7, %edi
+# CHECK-NEXT:  5      12    0.67    *      *            roll	$7, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorl	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        roll	%cl, %edi
+# CHECK-NEXT:  2      2     0.67                        rorl	%cl, %edi
+# CHECK-NEXT:  5      12    0.67    *      *            roll	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorl	%cl, (%rax)
+# CHECK-NEXT:  2      1     0.67                        rolq	%rdi
+# CHECK-NEXT:  2      1     0.67                        rorq	%rdi
+# CHECK-NEXT:  5      12    0.67    *      *            rolq	(%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorq	(%rax)
+# CHECK-NEXT:  2      1     0.67                        rolq	$7, %rdi
+# CHECK-NEXT:  2      1     0.67                        rorq	$7, %rdi
+# CHECK-NEXT:  5      12    0.67    *      *            rolq	$7, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorq	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        rolq	%cl, %rdi
+# CHECK-NEXT:  2      2     0.67                        rorq	%cl, %rdi
+# CHECK-NEXT:  5      12    0.67    *      *            rolq	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            rorq	%cl, (%rax)
+# CHECK-NEXT:  2      4     0.67                        sahf
+# CHECK-NEXT:  1      1     0.33                        sarb	%dil
+# CHECK-NEXT:  1      1     0.33                        shlb	%dil
+# CHECK-NEXT:  1      1     0.33                        shrb	%dil
+# CHECK-NEXT:  4      13    0.50    *      *            sarb	(%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            shlb	(%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            shrb	(%rax)
+# CHECK-NEXT:  1      1     0.33                        sarb	$7, %dil
+# CHECK-NEXT:  1      1     0.33                        shlb	$7, %dil
+# CHECK-NEXT:  1      1     0.33                        shrb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            sarb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            shlb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            shrb	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        sarb	%cl, %dil
+# CHECK-NEXT:  2      2     0.67                        shlb	%cl, %dil
+# CHECK-NEXT:  2      2     0.67                        shrb	%cl, %dil
+# CHECK-NEXT:  5      13    0.67    *      *            sarb	%cl, (%rax)
+# CHECK-NEXT:  5      13    0.67    *      *            shlb	%cl, (%rax)
+# CHECK-NEXT:  5      13    0.67    *      *            shrb	%cl, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sarw	%di
+# CHECK-NEXT:  1      1     0.33                        shlw	%di
+# CHECK-NEXT:  1      1     0.33                        shrw	%di
+# CHECK-NEXT:  5      5     1.67    *      *            sarw	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shlw	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrw	(%rax)
+# CHECK-NEXT:  1      1     0.33                        sarw	$7, %di
+# CHECK-NEXT:  1      1     0.33                        shlw	$7, %di
+# CHECK-NEXT:  1      1     0.33                        shrw	$7, %di
+# CHECK-NEXT:  5      5     1.67    *      *            sarw	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shlw	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrw	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        sarw	%cl, %di
+# CHECK-NEXT:  2      2     0.67                        shlw	%cl, %di
+# CHECK-NEXT:  2      2     0.67                        shrw	%cl, %di
+# CHECK-NEXT:  5      12    0.67    *      *            sarw	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shlw	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shrw	%cl, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sarl	%edi
+# CHECK-NEXT:  1      1     0.33                        shll	%edi
+# CHECK-NEXT:  1      1     0.33                        shrl	%edi
+# CHECK-NEXT:  5      5     1.67    *      *            sarl	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shll	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrl	(%rax)
+# CHECK-NEXT:  1      1     0.33                        sarl	$7, %edi
+# CHECK-NEXT:  1      1     0.33                        shll	$7, %edi
+# CHECK-NEXT:  1      1     0.33                        shrl	$7, %edi
+# CHECK-NEXT:  5      5     1.67    *      *            sarl	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shll	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrl	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        sarl	%cl, %edi
+# CHECK-NEXT:  2      2     0.67                        shll	%cl, %edi
+# CHECK-NEXT:  2      2     0.67                        shrl	%cl, %edi
+# CHECK-NEXT:  5      12    0.67    *      *            sarl	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shll	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shrl	%cl, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sarq	%rdi
+# CHECK-NEXT:  1      1     0.33                        shlq	%rdi
+# CHECK-NEXT:  1      1     0.33                        shrq	%rdi
+# CHECK-NEXT:  5      5     1.67    *      *            sarq	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shlq	(%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrq	(%rax)
+# CHECK-NEXT:  1      1     0.33                        sarq	$7, %rdi
+# CHECK-NEXT:  1      1     0.33                        shlq	$7, %rdi
+# CHECK-NEXT:  1      1     0.33                        shrq	$7, %rdi
+# CHECK-NEXT:  5      5     1.67    *      *            sarq	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shlq	$7, (%rax)
+# CHECK-NEXT:  5      5     1.67    *      *            shrq	$7, (%rax)
+# CHECK-NEXT:  2      2     0.67                        sarq	%cl, %rdi
+# CHECK-NEXT:  2      2     0.67                        shlq	%cl, %rdi
+# CHECK-NEXT:  2      2     0.67                        shrq	%cl, %rdi
+# CHECK-NEXT:  5      12    0.67    *      *            sarq	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shlq	%cl, (%rax)
+# CHECK-NEXT:  5      12    0.67    *      *            shrq	%cl, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbb	$0, %al
+# CHECK-NEXT:  1      1     0.33                        sbbb	$0, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            sbbb	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbb	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbb	$7, %al
+# CHECK-NEXT:  1      1     0.33                        sbbb	$7, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            sbbb	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbb	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbb	%sil, %dil
+# CHECK-NEXT:  5      12    0.50    *      *            sbbb	%sil, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   sbbb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        sbbw	$0, %ax
+# CHECK-NEXT:  1      1     0.33                        sbbw	$0, %di
+# CHECK-NEXT:  5      12    0.50    *      *            sbbw	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbw	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        sbbw	$511, %di
+# CHECK-NEXT:  5      12    0.50    *      *            sbbw	$511, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbw	$7, %di
+# CHECK-NEXT:  5      12    0.50    *      *            sbbw	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbw	%si, %di
+# CHECK-NEXT:  6      12    0.50    *      *            sbbw	%si, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		sbbw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   sbbw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        sbbl	$0, %eax
+# CHECK-NEXT:  1      1     0.33                        sbbl	$0, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbl	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbl	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbl	$665536, %eax
+# CHECK-NEXT:  1      1     0.33                        sbbl	$665536, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbl	$665536, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbl	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbl	$7, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbl	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbl	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbl	%esi, %edi
+# CHECK-NEXT:  6      12    0.50    *      *            sbbl	%esi, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		sbbl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   sbbl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        sbbq	$0, %rax
+# CHECK-NEXT:  1      1     0.33                        sbbq	$0, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbq	$0, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbq	$0, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbq	$665536, %rax
+# CHECK-NEXT:  1      1     0.33                        sbbq	$665536, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbq	$665536, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbq	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbq	$7, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            sbbq	$7, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		sbbq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        sbbq	%rsi, %rdi
+# CHECK-NEXT:  6      12    0.50    *      *            sbbq	%rsi, (%rax)
+# CHECK-NEXT:  6      12    0.50    *      *            lock		sbbq	%rsi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   sbbq	(%rax), %rdi
+# CHECK-NEXT:  1      100   0.17                  U     scasb	%es:(%rdi), %al
+# CHECK-NEXT:  1      100   0.17                  U     scasw	%es:(%rdi), %ax
+# CHECK-NEXT:  1      100   0.17                  U     scasl	%es:(%rdi), %eax
+# CHECK-NEXT:  1      100   0.17                  U     scasq	%es:(%rdi), %rax
+# CHECK-NEXT:  2      2     0.67                        seto	%al
+# CHECK-NEXT:  4      13    0.67           *            seto	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setno	%al
+# CHECK-NEXT:  4      13    0.67           *            setno	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setb	%al
+# CHECK-NEXT:  4      13    0.67           *            setb	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setae	%al
+# CHECK-NEXT:  4      13    0.67           *            setae	(%rax)
+# CHECK-NEXT:  2      2     0.67                        sete	%al
+# CHECK-NEXT:  4      13    0.67           *            sete	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setne	%al
+# CHECK-NEXT:  4      13    0.67           *            setne	(%rax)
+# CHECK-NEXT:  2      2     0.67                        seta	%al
+# CHECK-NEXT:  4      13    0.67           *            seta	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setbe	%al
+# CHECK-NEXT:  4      13    0.67           *            setbe	(%rax)
+# CHECK-NEXT:  2      2     0.67                        sets	%al
+# CHECK-NEXT:  4      13    0.67           *            sets	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setns	%al
+# CHECK-NEXT:  4      13    0.67           *            setns	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setp	%al
+# CHECK-NEXT:  4      13    0.67           *            setp	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setnp	%al
+# CHECK-NEXT:  4      13    0.67           *            setnp	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setl	%al
+# CHECK-NEXT:  4      13    0.67           *            setl	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setge	%al
+# CHECK-NEXT:  4      13    0.67           *            setge	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setg	%al
+# CHECK-NEXT:  4      13    0.67           *            setg	(%rax)
+# CHECK-NEXT:  2      2     0.67                        setle	%al
+# CHECK-NEXT:  4      13    0.67           *            setle	(%rax)
+# CHECK-NEXT:  3      5     0.67                        shldw	%cl, %si, %di
+# CHECK-NEXT:  3      5     0.67                        shrdw	%cl, %si, %di
+# CHECK-NEXT:  6      12    0.67    *      *            shldw	%cl, %si, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            shrdw	%cl, %si, (%rax)
+# CHECK-NEXT:  1      3     0.33                        shldw	$7, %si, %di
+# CHECK-NEXT:  1      3     0.33                        shrdw	$7, %si, %di
+# CHECK-NEXT:  5      12    0.50    *      *            shldw	$7, %si, (%rax)
+# CHECK-NEXT:  5      13    0.50    *      *            shrdw	$7, %si, (%rax)
+# CHECK-NEXT:  3      5     0.67                        shldl	%cl, %esi, %edi
+# CHECK-NEXT:  3      5     0.67                        shrdl	%cl, %esi, %edi
+# CHECK-NEXT:  6      12    0.67    *      *            shldl	%cl, %esi, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            shrdl	%cl, %esi, (%rax)
+# CHECK-NEXT:  1      3     0.33                        shldl	$7, %esi, %edi
+# CHECK-NEXT:  1      3     0.33                        shrdl	$7, %esi, %edi
+# CHECK-NEXT:  5      12    0.50    *      *            shldl	$7, %esi, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            shrdl	$7, %esi, (%rax)
+# CHECK-NEXT:  3      5     0.67                        shldq	%cl, %rsi, %rdi
+# CHECK-NEXT:  3      5     0.67                        shrdq	%cl, %rsi, %rdi
+# CHECK-NEXT:  6      12    0.67    *      *            shldq	%cl, %rsi, (%rax)
+# CHECK-NEXT:  6      12    0.67    *      *            shrdq	%cl, %rsi, (%rax)
+# CHECK-NEXT:  1      3     0.33                        shldq	$7, %rsi, %rdi
+# CHECK-NEXT:  1      3     0.33                        shrdq	$7, %rsi, %rdi
+# CHECK-NEXT:  5      12    0.50    *      *            shldq	$7, %rsi, (%rax)
+# CHECK-NEXT:  5      12    0.50    *      *            shrdq	$7, %rsi, (%rax)
+# CHECK-NEXT:  1      1     0.33                  U     stc
+# CHECK-NEXT:  2      6     0.33                  U     std
+# CHECK-NEXT:  4      8     0.50                  U     stosb	%al, %es:(%rdi)
+# CHECK-NEXT:  4      7     0.50                  U     stosw	%ax, %es:(%rdi)
+# CHECK-NEXT:  4      7     0.50                  U     stosl	%eax, %es:(%rdi)
+# CHECK-NEXT:  4      7     0.50                  U     stosq	%rax, %es:(%rdi)
+# CHECK-NEXT:  1      1     0.33                        subb	$7, %al
+# CHECK-NEXT:  1      1     0.33                        subb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            subb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		subb	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subb	%sil, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            subb	%sil, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		subb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   subb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        subw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        subw	$511, %di
+# CHECK-NEXT:  4      12    0.67    *      *            subw	$511, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subw	$7, %di
+# CHECK-NEXT:  4      12    0.67    *      *            subw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subw	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subw	%si, %di
+# CHECK-NEXT:  4      12    0.67    *      *            subw	%si, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   subw	(%rax), %di
+# CHECK-NEXT:  1      1     0.33                        subl	$665536, %eax
+# CHECK-NEXT:  1      1     0.33                        subl	$665536, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            subl	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subl	$665536, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subl	$7, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            subl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subl	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subl	%esi, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            subl	%esi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   subl	(%rax), %edi
+# CHECK-NEXT:  1      1     0.33                        subq	$665536, %rax
+# CHECK-NEXT:  1      1     0.33                        subq	$665536, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            subq	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subq	$665536, (%rax)
+# CHECK-NEXT:  0      1     0.00                        subq	$7, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            subq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subq	$7, (%rax)
+# CHECK-NEXT:  1      1     0.33                        subq	%rsi, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            subq	%rsi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		subq	%rsi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   subq	(%rax), %rdi
+# CHECK-NEXT:  1      2     0.17                        testb	$7, %al
+# CHECK-NEXT:  1      2     0.17                        testb	$7, %dil
+# CHECK-NEXT:  2      7     0.33    *                   testb	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testb	%sil, %dil
+# CHECK-NEXT:  2      7     0.33    *                   testb	%sil, (%rax)
+# CHECK-NEXT:  1      1     0.33                        testw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        testw	$511, %di
+# CHECK-NEXT:  2      7     0.33    *                   testw	$511, (%rax)
+# CHECK-NEXT:  1      1     0.33                        testw	$7, %di
+# CHECK-NEXT:  2      7     0.33    *                   testw	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testw	%si, %di
+# CHECK-NEXT:  2      7     0.33    *                   testw	%si, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testl	$665536, %eax
+# CHECK-NEXT:  1      2     0.17                        testl	$665536, %edi
+# CHECK-NEXT:  2      7     0.33    *                   testl	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testl	$7, %edi
+# CHECK-NEXT:  2      7     0.33    *                   testl	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testl	%esi, %edi
+# CHECK-NEXT:  2      7     0.33    *                   testl	%esi, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testq	$665536, %rax
+# CHECK-NEXT:  1      2     0.17                        testq	$665536, %rdi
+# CHECK-NEXT:  2      7     0.33    *                   testq	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testq	$7, %rdi
+# CHECK-NEXT:  2      7     0.33    *                   testq	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        testq	%rsi, %rdi
+# CHECK-NEXT:  2      7     0.33    *                   testq	%rsi, (%rax)
+# CHECK-NEXT:  1      100   0.17    *             U     ud2
+# CHECK-NEXT:  144    100   21.00                 U     wrmsr
+# CHECK-NEXT:  3      2     0.50                        xaddb	%bl, %cl
+# CHECK-NEXT:  5      13    0.50    *      *            xaddb	%bl, (%rcx)
+# CHECK-NEXT:  5      13    0.50    *      *            lock		xaddb	%bl, (%rcx)
+# CHECK-NEXT:  3      2     0.50                        xaddw	%bx, %cx
+# CHECK-NEXT:  5      12    0.50    *      *            xaddw	%ax, (%rbx)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		xaddw	%ax, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xaddl	%ebx, %ecx
+# CHECK-NEXT:  5      12    0.50    *      *            xaddl	%eax, (%rbx)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		xaddl	%eax, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xaddq	%rbx, %rcx
+# CHECK-NEXT:  5      12    0.50    *      *            xaddq	%rax, (%rbx)
+# CHECK-NEXT:  5      12    0.50    *      *            lock		xaddq	%rax, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xchgb	%bl, %cl
+# CHECK-NEXT:  8      40    0.83    *      *            xchgb	%bl, (%rbx)
+# CHECK-NEXT:  8      40    0.83    *      *            lock		xchgb	%bl, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xchgw	%bx, %ax
+# CHECK-NEXT:  3      2     0.50                        xchgw	%bx, %cx
+# CHECK-NEXT:  8      39    0.83    *      *            xchgw	%ax, (%rbx)
+# CHECK-NEXT:  8      39    0.83    *      *            lock		xchgw	%ax, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xchgl	%ebx, %eax
+# CHECK-NEXT:  3      2     0.50                        xchgl	%ebx, %ecx
+# CHECK-NEXT:  8      39    0.83    *      *            xchgl	%eax, (%rbx)
+# CHECK-NEXT:  8      39    0.83    *      *            lock		xchgl	%eax, (%rbx)
+# CHECK-NEXT:  3      2     0.50                        xchgq	%rbx, %rax
+# CHECK-NEXT:  3      2     0.50                        xchgq	%rbx, %rcx
+# CHECK-NEXT:  9      39    1.00    *      *            xchgq	%rax, (%rbx)
+# CHECK-NEXT:  9      39    1.00    *      *            lock		xchgq	%rax, (%rbx)
+# CHECK-NEXT:  3      7     0.33    *                   xlatb
+# CHECK-NEXT:  1      2     0.17                        xorb	$7, %al
+# CHECK-NEXT:  1      2     0.17                        xorb	$7, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            xorb	$7, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		xorb	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorb	%sil, %dil
+# CHECK-NEXT:  4      13    0.50    *      *            xorb	%sil, (%rax)
+# CHECK-NEXT:  4      13    0.50    *      *            lock		xorb	%sil, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   xorb	(%rax), %dil
+# CHECK-NEXT:  1      1     0.33                        xorw	$511, %ax
+# CHECK-NEXT:  1      1     0.33                        xorw	$511, %di
+# CHECK-NEXT:  4      12    0.67    *      *            xorw	$511, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorw	$511, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorw	$7, %di
+# CHECK-NEXT:  4      12    0.67    *      *            xorw	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorw	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorw	%si, %di
+# CHECK-NEXT:  4      12    0.67    *      *            xorw	%si, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorw	%si, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   xorw	(%rax), %di
+# CHECK-NEXT:  1      2     0.17                        xorl	$665536, %eax
+# CHECK-NEXT:  1      2     0.17                        xorl	$665536, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            xorl	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorl	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorl	$7, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            xorl	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorl	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorl	%esi, %edi
+# CHECK-NEXT:  4      12    0.67    *      *            xorl	%esi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorl	%esi, (%rax)
+# CHECK-NEXT:  2      6     0.33    *                   xorl	(%rax), %edi
+# CHECK-NEXT:  1      2     0.17                        xorq	$665536, %rax
+# CHECK-NEXT:  1      2     0.17                        xorq	$665536, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            xorq	$665536, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorq	$665536, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorq	$7, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            xorq	$7, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorq	$7, (%rax)
+# CHECK-NEXT:  1      2     0.17                        xorq	%rsi, %rdi
+# CHECK-NEXT:  4      12    0.67    *      *            xorq	%rsi, (%rax)
+# CHECK-NEXT:  4      12    0.67    *      *            lock		xorq	%rsi, (%rax)
+# CHECK-NEXT:  2      7     0.33    *                   xorq	(%rax), %rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 270.17 357.17 266.17 314.17 266.17 314.17 194.00 194.00 225.67 225.67 225.67 127.67 127.67 127.67  -     12.25  12.25  13.75  13.75
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcb	$0, %al
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcb	$0, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcb	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcb	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcb	$7, %al
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcb	$7, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcb	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcb	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcb	%sil, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcb	%sil, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcb	%sil, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     adcb	(%rax), %dil
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	$0, %ax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	$0, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcw	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcw	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	$511, %ax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	$511, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcw	$511, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcw	$511, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	$7, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcw	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcw	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcw	%si, %di
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcw	%si, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcw	%si, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     adcw	(%rax), %di
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	$0, %eax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	$0, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcl	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcl	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	$665536, %eax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	$665536, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcl	$665536, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcl	$665536, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	$7, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcl	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcl	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcl	%esi, %edi
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcl	%esi, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcl	%esi, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     adcl	(%rax), %edi
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	$0, %rax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	$0, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcq	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcq	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	$665536, %rax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	$665536, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcq	$665536, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcq	$665536, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	$7, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcq	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcq	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     adcq	%rsi, %rdi
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     adcq	%rsi, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		adcq	%rsi, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     adcq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addb	$7, %al
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     addb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		addb	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     addb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		addb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     addb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addw	$511, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     addw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addl	$665536, %eax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addl	$665536, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     addl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addq	$665536, %rax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addq	$665536, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addq	$665536, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     addq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     addq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     addq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		addq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     addq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andb	$7, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     andb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		andb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     andb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		andb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     andb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     andw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     andw	$511, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andw	$511, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andw	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     andw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andl	$665536, %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andl	$665536, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andl	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andl	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     andl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andq	$665536, %rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andq	$665536, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andq	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andq	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     andq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     andq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		andq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     andq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsfw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsrw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsfw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsrw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsfl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsrl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsfl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsrl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsfq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bsrq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsfq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     bsrq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     bswapl	%eax
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     bswapq	%rax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btcw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btrw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btsw	%si, %di
+# CHECK-NEXT: 1.67   1.33   1.67   1.33   1.67   1.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrw	%si, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btcw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btrw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btsw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btcl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btrl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btsl	%esi, %edi
+# CHECK-NEXT: 1.67   1.33   1.67   1.33   1.67   1.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrl	%esi, (%rax)
+# CHECK-NEXT: 1.50   1.17   1.50   1.17   1.50   1.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btcl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btrl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btsl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsl	$7, (%rax)
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     btq	%rsi, %rdi
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     btcq	%rsi, %rdi
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     btrq	%rsi, %rdi
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     btsq	%rsi, %rdi
+# CHECK-NEXT: 1.50   1.83   1.50   0.83   1.50   0.83    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrq	%rsi, (%rax)
+# CHECK-NEXT: 1.33   1.67   1.33   0.67   1.33   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btcq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btrq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     btsq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     btq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btcq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btrq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     btsq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btcq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btrq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		btsq	$7, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   cbtw
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   cwtl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   cltq
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cwtd
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     cltd
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     cqto
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     clc
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     cld
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmc
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpb	$7, %al
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpb	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpw	$511, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpw	$7, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpl	$665536, %eax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpl	$665536, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpl	$7, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpq	$665536, %rax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpq	$665536, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpq	$7, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     cmpq	(%rax), %rdi
+# CHECK-NEXT: 0.83   0.83   0.83   0.83   0.83   0.83    -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpsb	%es:(%rdi), (%rsi)
+# CHECK-NEXT: 0.83   0.83   0.83   0.83   0.83   0.83    -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpsw	%es:(%rdi), (%rsi)
+# CHECK-NEXT: 0.83   0.83   0.83   0.83   0.83   0.83    -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpsl	%es:(%rdi), (%rsi)
+# CHECK-NEXT: 0.83   0.83   0.83   0.83   0.83   0.83    -      -     0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpsq	%es:(%rdi), (%rsi)
+# CHECK-NEXT: 1.17   0.50   1.17   0.50   1.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpxchgb	%cl, %bl
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     cmpxchgb	%cl, (%rbx)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		cmpxchgb	%cl, (%rbx)
+# CHECK-NEXT: 1.17   0.50   1.17   0.50   1.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpxchgw	%cx, %bx
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpxchgw	%cx, (%rbx)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     lock		cmpxchgw	%cx, (%rbx)
+# CHECK-NEXT: 1.17   0.50   1.17   0.50   1.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpxchgl	%ecx, %ebx
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpxchgl	%ecx, (%rbx)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     lock		cmpxchgl	%ecx, (%rbx)
+# CHECK-NEXT: 1.17   0.50   1.17   0.50   1.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     cmpxchgq	%rcx, %rbx
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     cmpxchgq	%rcx, (%rbx)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.67   0.67   0.67    -      -      -      -      -      -      -      -     lock		cmpxchgq	%rcx, (%rbx)
+# CHECK-NEXT: 5.33   7.67   3.33   1.67   3.33   1.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -     0.50   0.50    -      -     cpuid
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     decb	%dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     decb	(%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		decb	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     decw	%di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     decw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		decw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     decl	%edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     decl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		decl	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     decq	%rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     decq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		decq	(%rax)
+# CHECK-NEXT:  -     3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     divb	%dil
+# CHECK-NEXT:  -     3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     divb	(%rax)
+# CHECK-NEXT: 0.17   3.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     divw	%si
+# CHECK-NEXT: 0.17   3.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     divw	(%rax)
+# CHECK-NEXT: 0.17   3.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     divl	%edx
+# CHECK-NEXT: 0.17   3.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     divl	(%rax)
+# CHECK-NEXT:  -     3.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     divq	%rcx
+# CHECK-NEXT:  -     3.00    -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     divq	(%rax)
+# CHECK-NEXT: 9.00   5.00   7.00   3.00   7.00   3.00   2.00   2.00   4.67   4.67   4.67   1.67   1.67   1.67    -      -      -      -      -     enter	$7, $4095
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     idivb	%dil
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     idivb	(%rax)
+# CHECK-NEXT: 0.17   1.17   0.17   1.17   0.17   1.17    -      -      -      -      -      -      -      -      -      -      -      -      -     idivw	%si
+# CHECK-NEXT: 0.17   1.17   0.17   1.17   0.17   1.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     idivw	(%rax)
+# CHECK-NEXT: 0.17   1.17   0.17   1.17   0.17   1.17    -      -      -      -      -      -      -      -      -      -      -      -      -     idivl	%edx
+# CHECK-NEXT: 0.17   1.17   0.17   1.17   0.17   1.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     idivl	(%rax)
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     idivq	%rcx
+# CHECK-NEXT:  -     1.00    -     1.00    -     1.00    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     idivq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imulb	%dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulb	(%rax)
+# CHECK-NEXT: 0.33   0.67   0.33   0.67   0.33   0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     imulw	%di
+# CHECK-NEXT: 0.33   0.67   0.33   0.67   0.33   0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imulw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulw	(%rax), %di
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imulw	$511, %si, %di
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulw	$511, (%rax), %di
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imulw	$7, %si, %di
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulw	$7, (%rax), %di
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imull	%edi
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imull	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imull	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imull	(%rax), %edi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imull	$665536, %esi, %edi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imull	$665536, (%rax), %edi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imull	$7, %esi, %edi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imull	$7, (%rax), %edi
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imulq	%rdi
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     imulq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imulq	$665536, %rsi, %rdi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulq	$665536, (%rax), %rdi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     imulq	$7, %rsi, %rdi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     imulq	$7, (%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inb	$7, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inb	%dx, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inw	$7, %ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inw	%dx, %ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inl	$7, %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     inl	%dx, %eax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     incb	%dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     incb	(%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		incb	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     incw	%di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     incw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		incw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     incl	%edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     incl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		incl	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     incq	%rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     incq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		incq	(%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     insb	%dx, %es:(%rdi)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     insw	%dx, %es:(%rdi)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     insl	%dx, %es:(%rdi)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     int	$7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     invlpg	(%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     invlpga
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     leave
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     lodsb	(%rsi), %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     lodsw	(%rsi), %ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     lodsl	(%rsi), %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     lodsq	(%rsi), %rax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     loop	0
+# CHECK-NEXT: 2.67   1.00   2.67   1.00   2.67   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     loope	0
+# CHECK-NEXT: 2.67   1.00   2.67   1.00   2.67   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     loopne	0
+# CHECK-NEXT: 0.67   0.67   0.67   0.67   0.67   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     movsb	(%rsi), %es:(%rdi)
+# CHECK-NEXT: 0.67   0.67   0.67   0.67   0.67   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     movsw	(%rsi), %es:(%rdi)
+# CHECK-NEXT: 0.67   0.67   0.67   0.67   0.67   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     movsl	(%rsi), %es:(%rdi)
+# CHECK-NEXT: 0.67   0.67   0.67   0.67   0.67   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     movsq	(%rsi), %es:(%rdi)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movsbw	%al, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movzbw	%al, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movsbw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movzbw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movsbl	%al, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     movzbl	%al, %edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movsbl	(%rax), %edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movzbl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movsbq	%al, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     movzbq	%al, %rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movsbq	(%rax), %rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movzbq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movswl	%ax, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movzwl	%ax, %edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movswl	(%rax), %edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movzwl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movswq	%ax, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movzwq	%ax, %rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movswq	(%rax), %rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movzwq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     movslq	%eax, %rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     movslq	(%rax), %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     mulb	%dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mulb	(%rax)
+# CHECK-NEXT: 0.33   0.67   0.33   0.67   0.33   0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     mulw	%si
+# CHECK-NEXT: 0.33   0.67   0.33   0.67   0.33   0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mulw	(%rax)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     mull	%edx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mull	(%rax)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     mulq	%rcx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     mulq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     negb	%dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     negb	(%r8)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		negb	(%r8)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     negw	%si
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     negw	(%r9)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		negw	(%r9)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     negl	%edx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     negl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		negl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     negq	%rcx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     negq	(%r10)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		negq	(%r10)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nop
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopw	%di
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopw	(%rcx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopl	%esi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopl	(%r8)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopq	%rdx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     nopq	(%r9)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     notb	%dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     notb	(%r8)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		notb	(%r8)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     notw	%si
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     notw	(%r9)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		notw	(%r9)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     notl	%edx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     notl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		notl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     notq	%rcx
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     notq	(%r10)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		notq	(%r10)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orb	$7, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     orb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		orb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     orb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		orb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     orb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     orw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     orw	$511, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orw	$511, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orw	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     orw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orl	$665536, %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orl	$665536, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orl	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orl	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     orl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orq	$665536, %rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orq	$665536, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orq	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orq	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     orq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     orq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		orq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     orq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outb	%al, $7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outb	%al, %dx
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outw	%ax, $7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outw	%ax, %dx
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outl	%eax, $7
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outl	%eax, %dx
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outsb	(%rsi), %dx
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outsw	(%rsi), %dx
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     outsl	(%rsi), %dx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     pause
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclb	%dil
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrb	%dil
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclb	(%rax)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrb	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclb	$7, %dil
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrb	$7, %dil
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclb	$7, (%rax)
+# CHECK-NEXT: 0.83   0.17   0.83   0.17   0.83   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrb	$7, (%rax)
+# CHECK-NEXT: 2.00   1.00   2.00   1.00   2.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rclb	%cl, %dil
+# CHECK-NEXT: 1.83   1.50   1.83   1.50   1.83   1.50    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrb	%cl, %dil
+# CHECK-NEXT: 1.83   0.83   1.83   0.83   1.83   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclb	%cl, (%rax)
+# CHECK-NEXT: 1.67   1.33   1.67   1.33   1.67   1.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrb	%cl, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclw	%di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrw	%di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclw	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrw	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclw	$7, %di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrw	$7, %di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclw	$7, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrw	$7, (%rax)
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rclw	%cl, %di
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrw	%cl, %di
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclw	%cl, (%rax)
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrw	%cl, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcll	%edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrl	%edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcll	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrl	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcll	$7, %edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrl	$7, %edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcll	$7, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrl	$7, (%rax)
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rcll	%cl, %edi
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrl	%cl, %edi
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcll	%cl, (%rax)
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrl	%cl, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclq	%rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrq	%rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclq	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrq	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rclq	$7, %rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrq	$7, %rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclq	$7, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrq	$7, (%rax)
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rclq	%cl, %rdi
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rcrq	%cl, %rdi
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rclq	%cl, (%rax)
+# CHECK-NEXT: 0.33   2.00   0.33   2.00   0.33   2.00   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rcrq	%cl, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdmsr
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdpmc
+# CHECK-NEXT: 2.83   2.17   2.83   2.17   2.83   2.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdtsc
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     rdtscp
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolb	%dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorb	%dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolb	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorb	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolb	$7, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorb	$7, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolb	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorb	$7, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rolb	%cl, %dil
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rorb	%cl, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolb	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorb	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolw	%di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorw	%di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolw	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorw	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolw	$7, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorw	$7, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolw	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorw	$7, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rolw	%cl, %di
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rorw	%cl, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolw	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorw	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     roll	%edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorl	%edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     roll	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorl	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     roll	$7, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorl	$7, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     roll	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorl	$7, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     roll	%cl, %edi
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rorl	%cl, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     roll	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorl	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolq	%rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorq	%rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolq	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorq	(%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolq	$7, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorq	$7, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolq	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorq	$7, (%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rolq	%cl, %rdi
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     rorq	%cl, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rolq	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     rorq	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sahf
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarb	%dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlb	%dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrb	%dil
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarb	(%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shlb	(%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrb	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarb	$7, %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlb	$7, %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrb	$7, %dil
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarb	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shlb	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrb	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarb	%cl, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shlb	%cl, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrb	%cl, %dil
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarb	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shlb	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrb	%cl, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarw	%di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlw	%di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrw	%di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shlw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrw	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shlw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrw	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarw	%cl, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shlw	%cl, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrw	%cl, %di
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarw	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shlw	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrw	%cl, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarl	%edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shll	%edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrl	%edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shll	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrl	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shll	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shll	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrl	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarl	%cl, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shll	%cl, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrl	%cl, %edi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarl	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shll	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrl	%cl, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarq	%rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlq	%rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrq	%rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shlq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrq	(%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     sarq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shlq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     sarq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shlq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   1.67   1.67   1.67   0.33   0.33   0.33    -      -      -      -      -     shrq	$7, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sarq	%cl, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shlq	%cl, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -      -     shrq	%cl, %rdi
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sarq	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shlq	%cl, (%rax)
+# CHECK-NEXT: 0.67    -     0.67    -     0.67    -     0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrq	%cl, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbb	$0, %al
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbb	$0, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbb	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbb	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbb	$7, %al
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbb	$7, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbb	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbb	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbb	%sil, %dil
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbb	%sil, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbb	%sil, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sbbb	(%rax), %dil
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	$0, %ax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	$0, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbw	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbw	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	$511, %ax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	$511, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbw	$511, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbw	$511, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	$7, %di
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbw	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbw	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbw	%si, %di
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbw	%si, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbw	%si, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sbbw	(%rax), %di
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	$0, %eax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	$0, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbl	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbl	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	$665536, %eax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	$665536, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbl	$665536, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbl	$665536, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	$7, %edi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbl	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbl	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbl	%esi, %edi
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbl	%esi, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbl	%esi, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sbbl	(%rax), %edi
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	$0, %rax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	$0, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbq	$0, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbq	$0, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	$665536, %rax
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	$665536, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbq	$665536, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbq	$665536, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	$7, %rdi
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbq	$7, (%rax)
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbq	$7, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -      -     sbbq	%rsi, %rdi
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     sbbq	%rsi, (%rax)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		sbbq	%rsi, (%rax)
+# CHECK-NEXT: 0.33    -     0.33    -     0.33    -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     sbbq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     scasb	%es:(%rdi), %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     scasw	%es:(%rdi), %ax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     scasl	%es:(%rdi), %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     scasq	%es:(%rdi), %rax
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     seto	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     seto	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setno	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setno	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setb	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setb	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setae	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setae	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     sete	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     sete	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setne	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setne	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     seta	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     seta	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setbe	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setbe	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     sets	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     sets	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setns	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setns	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setp	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setp	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setnp	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setnp	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setl	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setl	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setge	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setge	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setg	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setg	(%rax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     setle	%al
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     setle	(%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shldw	%cl, %si, %di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdw	%cl, %si, %di
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldw	%cl, %si, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdw	%cl, %si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shldw	$7, %si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdw	$7, %si, %di
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldw	$7, %si, (%rax)
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdw	$7, %si, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shldl	%cl, %esi, %edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdl	%cl, %esi, %edi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldl	%cl, %esi, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdl	%cl, %esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shldl	$7, %esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdl	$7, %esi, %edi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldl	$7, %esi, (%rax)
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdl	$7, %esi, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shldq	%cl, %rsi, %rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdq	%cl, %rsi, %rdi
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldq	%cl, %rsi, (%rax)
+# CHECK-NEXT: 0.17   0.83   0.17   0.83   0.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdq	%cl, %rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shldq	$7, %rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     shrdq	$7, %rsi, %rdi
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shldq	$7, %rsi, (%rax)
+# CHECK-NEXT: 0.17   0.50   0.17   0.50   0.17   0.50   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     shrdq	$7, %rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     stc
+# CHECK-NEXT: 0.50   0.17   0.50   0.17   0.50   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     std
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     stosb	%al, %es:(%rdi)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     stosw	%ax, %es:(%rdi)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     stosl	%eax, %es:(%rdi)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     stosq	%rax, %es:(%rdi)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subb	$7, %al
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     subb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		subb	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     subb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		subb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     subb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subw	$511, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     subw	(%rax), %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subl	$665536, %eax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subl	$665536, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     subl	(%rax), %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subq	$665536, %rax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subq	$665536, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subq	$665536, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     subq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     subq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     subq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		subq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     subq	(%rax), %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testb	$7, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testb	%sil, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     testw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     testw	$511, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     testw	$7, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testw	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testw	%si, %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testl	$665536, %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testl	$665536, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testl	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testl	$7, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testl	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testl	%esi, %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testq	$665536, %rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testq	$665536, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testq	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testq	$7, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testq	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     testq	%rsi, %rdi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     testq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     ud2
+# CHECK-NEXT: 21.00  10.33  21.00  10.33  21.00  10.33  0.50   0.50    -      -      -     0.33   0.33   0.33    -     11.75  11.75  12.25  12.25  wrmsr
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xaddb	%bl, %cl
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xaddb	%bl, (%rcx)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xaddb	%bl, (%rcx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xaddw	%bx, %cx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xaddw	%ax, (%rbx)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xaddw	%ax, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xaddl	%ebx, %ecx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xaddl	%eax, (%rbx)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xaddl	%eax, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xaddq	%rbx, %rcx
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xaddq	%rax, (%rbx)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xaddq	%rax, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgb	%bl, %cl
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xchgb	%bl, (%rbx)
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xchgb	%bl, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgw	%bx, %ax
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgw	%bx, %cx
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xchgw	%ax, (%rbx)
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xchgw	%ax, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgl	%ebx, %eax
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgl	%ebx, %ecx
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xchgl	%eax, (%rbx)
+# CHECK-NEXT: 1.00   0.67   1.00   0.67   1.00   0.67   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xchgl	%eax, (%rbx)
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgq	%rbx, %rax
+# CHECK-NEXT: 0.50   0.50   0.50   0.50   0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -     xchgq	%rbx, %rcx
+# CHECK-NEXT: 1.17   0.83   1.17   0.83   1.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xchgq	%rax, (%rbx)
+# CHECK-NEXT: 1.17   0.83   1.17   0.83   1.17   0.83   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xchgq	%rax, (%rbx)
+# CHECK-NEXT: 0.33   0.33   0.33   0.33   0.33   0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     xlatb
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorb	$7, %al
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorb	$7, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xorb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xorb	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorb	%sil, %dil
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     xorb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17   0.50   0.50   0.33   0.33   0.33   0.33   0.33   0.33    -      -      -      -      -     lock		xorb	%sil, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     xorb	(%rax), %dil
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     xorw	$511, %ax
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     xorw	$511, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorw	$511, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorw	$511, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorw	$7, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorw	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorw	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorw	%si, %di
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorw	%si, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorw	%si, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     xorw	(%rax), %di
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorl	$665536, %eax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorl	$665536, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorl	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorl	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorl	$7, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorl	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorl	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorl	%esi, %edi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorl	%esi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorl	%esi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     xorl	(%rax), %edi
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorq	$665536, %rax
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorq	$665536, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorq	$665536, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorq	$665536, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorq	$7, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorq	$7, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorq	$7, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     xorq	%rsi, %rdi
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     xorq	%rsi, (%rax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33   0.50   0.50   0.67   0.67   0.67   0.33   0.33   0.33    -      -      -      -      -     lock		xorq	%rsi, (%rax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     xorq	(%rax), %rdi
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x87.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x87.s
new file mode 100644
index 000000000000..eab70ac4bcaa
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-x87.s
@@ -0,0 +1,532 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+f2xm1
+
+fabs
+
+fadd %st, %st(1)
+fadd %st(2)
+fadds (%ecx)
+faddl (%ecx)
+faddp %st(1)
+faddp %st(2)
+fiadds (%ecx)
+fiaddl (%ecx)
+
+fbld (%ecx)
+fbstp (%eax)
+
+fchs
+
+fnclex
+
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
+
+fcom %st(1)
+fcom %st(3)
+fcoms (%ecx)
+fcoml (%eax)
+fcomp %st(1)
+fcomp %st(3)
+fcomps (%ecx)
+fcompl (%eax)
+fcompp
+
+fcomi %st(3)
+fcompi %st(3)
+
+fcos
+
+fdecstp
+
+fdiv %st, %st(1)
+fdiv %st(2)
+fdivs (%ecx)
+fdivl (%eax)
+fdivp %st(1)
+fdivp %st(2)
+fidivs (%ecx)
+fidivl (%eax)
+
+fdivr %st, %st(1)
+fdivr %st(2)
+fdivrs (%ecx)
+fdivrl (%eax)
+fdivrp %st(1)
+fdivrp %st(2)
+fidivrs (%ecx)
+fidivrl (%eax)
+
+ffree %st(0)
+
+ficoms (%ecx)
+ficoml (%eax)
+ficomps (%ecx)
+ficompl (%eax)
+
+filds (%edx)
+fildl (%ecx)
+fildll (%eax)
+
+fincstp
+
+fninit
+
+fists (%edx)
+fistl (%ecx)
+fistps (%edx)
+fistpl (%ecx)
+fistpll (%eax)
+
+fisttps (%edx)
+fisttpl (%ecx)
+fisttpll (%eax)
+
+fld %st(0)
+flds (%edx)
+fldl (%ecx)
+fldt (%eax)
+
+fldcw (%eax)
+fldenv (%eax)
+
+fld1
+fldl2e
+fldl2t
+fldlg2
+fldln2
+fldpi
+fldz
+
+fmul %st, %st(1)
+fmul %st(2)
+fmuls (%ecx)
+fmull (%eax)
+fmulp %st(1)
+fmulp %st(2)
+fimuls (%ecx)
+fimull (%eax)
+
+fnop
+
+fpatan
+
+fprem
+fprem1
+
+fptan
+
+frndint
+
+frstor (%eax)
+
+fnsave (%eax)
+
+fscale
+
+fsin
+
+fsincos
+
+fsqrt
+
+fst %st(0)
+fsts (%edx)
+fstl (%ecx)
+fstp %st(0)
+fstpl (%edx)
+fstpl (%ecx)
+fstpt (%eax)
+
+fnstcw (%eax)
+fnstenv (%eax)
+fnstsw (%eax)
+
+frstor (%eax)
+fsave (%eax)
+
+fsub %st, %st(1)
+fsub %st(2)
+fsubs (%ecx)
+fsubl (%eax)
+fsubp %st(1)
+fsubp %st(2)
+fisubs (%ecx)
+fisubl (%eax)
+
+fsubr %st, %st(1)
+fsubr %st(2)
+fsubrs (%ecx)
+fsubrl (%eax)
+fsubrp %st(1)
+fsubrp %st(2)
+fisubrs (%ecx)
+fisubrl (%eax)
+
+ftst
+
+fucom %st(1)
+fucom %st(3)
+fucomp %st(1)
+fucomp %st(3)
+fucompp
+
+fucomi %st(3)
+fucompi %st(3)
+
+fwait
+
+fxam
+
+fxch %st(1)
+fxch %st(3)
+
+fxrstor (%eax)
+fxsave (%eax)
+
+fxtract
+
+fyl2x
+fyl2xp1
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.17                  U     f2xm1
+# CHECK-NEXT:  1      1     0.50                  U     fabs
+# CHECK-NEXT:  1      2     0.50                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     fadd	%st(2), %st
+# CHECK-NEXT:  2      10    0.33    *             U     fadds	(%ecx)
+# CHECK-NEXT:  2      10    0.33    *             U     faddl	(%ecx)
+# CHECK-NEXT:  1      2     0.50                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     faddp	%st, %st(2)
+# CHECK-NEXT:  3      13    0.67    *             U     fiadds	(%ecx)
+# CHECK-NEXT:  3      13    0.67    *             U     fiaddl	(%ecx)
+# CHECK-NEXT:  1      100   0.17    *             U     fbld	(%ecx)
+# CHECK-NEXT:  2      1     0.50           *      U     fbstp	(%eax)
+# CHECK-NEXT:  1      1     0.50                  U     fchs
+# CHECK-NEXT:  4      4     1.00                  U     fnclex
+# CHECK-NEXT:  1      3     0.50                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     0.50                  U     fcmovu	%st(1), %st
+# CHECK-NEXT:  1      1     0.50                  U     fcom	%st(1)
+# CHECK-NEXT:  1      1     0.50                  U     fcom	%st(3)
+# CHECK-NEXT:  2      8     0.50    *             U     fcoms	(%ecx)
+# CHECK-NEXT:  2      8     0.50    *             U     fcoml	(%eax)
+# CHECK-NEXT:  1      1     0.50                  U     fcomp	%st(1)
+# CHECK-NEXT:  1      1     0.50                  U     fcomp	%st(3)
+# CHECK-NEXT:  2      8     0.50    *             U     fcomps	(%ecx)
+# CHECK-NEXT:  2      8     0.50    *             U     fcompl	(%eax)
+# CHECK-NEXT:  1      100   0.17                  U     fcompp
+# CHECK-NEXT:  1      1     0.50                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      1     0.50                  U     fcompi	%st(3), %st
+# CHECK-NEXT:  1      100   0.17                  U     fcos
+# CHECK-NEXT:  2      2     0.67                  U     fdecstp
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2), %st
+# CHECK-NEXT:  2      22    1.00    *             U     fdivs	(%ecx)
+# CHECK-NEXT:  2      22    1.00    *             U     fdivl	(%eax)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(2)
+# CHECK-NEXT:  3      25    1.00    *             U     fidivs	(%ecx)
+# CHECK-NEXT:  3      25    1.00    *             U     fidivl	(%eax)
+# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2), %st
+# CHECK-NEXT:  2      27    1.00    *             U     fdivrs	(%ecx)
+# CHECK-NEXT:  2      27    1.00    *             U     fdivrl	(%eax)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(2)
+# CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
+# CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
+# CHECK-NEXT:  1      100   0.17                  U     ffree	%st(0)
+# CHECK-NEXT:  3      11    0.67    *             U     ficoms	(%ecx)
+# CHECK-NEXT:  3      11    0.67    *             U     ficoml	(%eax)
+# CHECK-NEXT:  3      11    0.67    *             U     ficomps	(%ecx)
+# CHECK-NEXT:  3      11    0.67    *             U     ficompl	(%eax)
+# CHECK-NEXT:  2      10    0.33    *             U     filds	(%edx)
+# CHECK-NEXT:  2      10    0.33    *             U     fildl	(%ecx)
+# CHECK-NEXT:  2      10    0.33    *             U     fildll	(%eax)
+# CHECK-NEXT:  1      1     0.33                  U     fincstp
+# CHECK-NEXT:  15     75    3.00                  U     fninit
+# CHECK-NEXT:  2      12    0.50           *      U     fists	(%edx)
+# CHECK-NEXT:  2      12    0.50           *      U     fistl	(%ecx)
+# CHECK-NEXT:  2      12    0.50           *      U     fistps	(%edx)
+# CHECK-NEXT:  2      12    0.50           *      U     fistpl	(%ecx)
+# CHECK-NEXT:  2      12    0.50           *      U     fistpll	(%eax)
+# CHECK-NEXT:  2      12    0.50           *      U     fisttps	(%edx)
+# CHECK-NEXT:  2      12    0.50           *      U     fisttpl	(%ecx)
+# CHECK-NEXT:  2      12    0.50           *      U     fisttpll	(%eax)
+# CHECK-NEXT:  1      1     0.17                  U     fld	%st(0)
+# CHECK-NEXT:  1      4     0.33    *             U     flds	(%edx)
+# CHECK-NEXT:  1      4     0.33    *             U     fldl	(%ecx)
+# CHECK-NEXT:  1      4     0.33    *             U     fldt	(%eax)
+# CHECK-NEXT:  3      7     0.50    *             U     fldcw	(%eax)
+# CHECK-NEXT:  64     62    10.25   *             U     fldenv	(%eax)
+# CHECK-NEXT:  2      1     1.00                  U     fld1
+# CHECK-NEXT:  2      1     1.00                  U     fldl2e
+# CHECK-NEXT:  2      1     1.00                  U     fldl2t
+# CHECK-NEXT:  2      1     1.00                  U     fldlg2
+# CHECK-NEXT:  2      1     1.00                  U     fldln2
+# CHECK-NEXT:  2      1     1.00                  U     fldpi
+# CHECK-NEXT:  1      1     0.50                  U     fldz
+# CHECK-NEXT:  1      4     0.25                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      4     0.25                  U     fmul	%st(2), %st
+# CHECK-NEXT:  2      11    0.50    *             U     fmuls	(%ecx)
+# CHECK-NEXT:  2      11    0.50    *             U     fmull	(%eax)
+# CHECK-NEXT:  1      4     0.25                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      4     0.25                  U     fmulp	%st, %st(2)
+# CHECK-NEXT:  3      14    0.50    *             U     fimuls	(%ecx)
+# CHECK-NEXT:  3      14    0.50    *             U     fimull	(%eax)
+# CHECK-NEXT:  1      1     0.33                  U     fnop
+# CHECK-NEXT:  1      100   0.17                  U     fpatan
+# CHECK-NEXT:  1      100   0.17                  U     fprem
+# CHECK-NEXT:  1      100   0.17                  U     fprem1
+# CHECK-NEXT:  1      100   0.17                  U     fptan
+# CHECK-NEXT:  1      100   0.17                  U     frndint
+# CHECK-NEXT:  1      100   0.17    *             U     frstor	(%eax)
+# CHECK-NEXT:  1      100   0.17           *      U     fnsave	(%eax)
+# CHECK-NEXT:  1      100   0.17                  U     fscale
+# CHECK-NEXT:  1      100   0.17                  U     fsin
+# CHECK-NEXT:  1      100   0.17                  U     fsincos
+# CHECK-NEXT:  1      21    7.00                  U     fsqrt
+# CHECK-NEXT:  1      1     0.17                  U     fst	%st(0)
+# CHECK-NEXT:  1      1     0.50           *      U     fsts	(%edx)
+# CHECK-NEXT:  1      1     0.50           *      U     fstl	(%ecx)
+# CHECK-NEXT:  1      1     0.17                  U     fstp	%st(0)
+# CHECK-NEXT:  2      1     0.50           *      U     fstpl	(%edx)
+# CHECK-NEXT:  2      1     0.50           *      U     fstpl	(%ecx)
+# CHECK-NEXT:  2      1     0.50           *      U     fstpt	(%eax)
+# CHECK-NEXT:  3      2     0.50           *      U     fnstcw	(%eax)
+# CHECK-NEXT:  100    106   10.00          *      U     fnstenv	(%eax)
+# CHECK-NEXT:  3      3     0.50           *      U     fnstsw	(%eax)
+# CHECK-NEXT:  1      100   0.17    *             U     frstor	(%eax)
+# CHECK-NEXT:  2      2     0.50                  U     wait
+# CHECK-NEXT:  1      100   0.17           *      U     fnsave	(%eax)
+# CHECK-NEXT:  1      2     0.50                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     fsub	%st(2), %st
+# CHECK-NEXT:  2      10    0.33    *             U     fsubs	(%ecx)
+# CHECK-NEXT:  2      10    0.33    *             U     fsubl	(%eax)
+# CHECK-NEXT:  1      2     0.50                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     fsubp	%st, %st(2)
+# CHECK-NEXT:  3      13    0.67    *             U     fisubs	(%ecx)
+# CHECK-NEXT:  3      13    0.67    *             U     fisubl	(%eax)
+# CHECK-NEXT:  1      2     0.50                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     fsubr	%st(2), %st
+# CHECK-NEXT:  2      10    0.33    *             U     fsubrs	(%ecx)
+# CHECK-NEXT:  2      10    0.33    *             U     fsubrl	(%eax)
+# CHECK-NEXT:  1      2     0.50                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      2     0.50                  U     fsubrp	%st, %st(2)
+# CHECK-NEXT:  3      13    0.67    *             U     fisubrs	(%ecx)
+# CHECK-NEXT:  3      13    0.67    *             U     fisubrl	(%eax)
+# CHECK-NEXT:  1      2     0.33                  U     ftst
+# CHECK-NEXT:  1      1     0.50                  U     fucom	%st(1)
+# CHECK-NEXT:  1      1     0.50                  U     fucom	%st(3)
+# CHECK-NEXT:  1      1     0.50                  U     fucomp	%st(1)
+# CHECK-NEXT:  1      1     0.50                  U     fucomp	%st(3)
+# CHECK-NEXT:  1      2     0.33                  U     fucompp
+# CHECK-NEXT:  1      1     0.50                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      1     0.50                  U     fucompi	%st(3), %st
+# CHECK-NEXT:  2      2     0.50                  U     wait
+# CHECK-NEXT:  1      2     0.33                  U     fxam
+# CHECK-NEXT:  15     17    2.00                  U     fxch	%st(1)
+# CHECK-NEXT:  15     17    2.00                  U     fxch	%st(3)
+# CHECK-NEXT:  90     63    13.50   *      *      U     fxrstor	(%eax)
+# CHECK-NEXT:  110    100   19.00   *      *      U     fxsave	(%eax)
+# CHECK-NEXT:  1      100   0.17                  U     fxtract
+# CHECK-NEXT:  1      100   0.17                  U     fyl2x
+# CHECK-NEXT:  1      100   0.17                  U     fyl2xp1
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 37.67  44.00  21.67  33.00  21.67  33.00  32.50  32.50  32.33  32.33  32.33  15.33  15.33  15.33  7.00   53.00  53.00  64.00  64.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     f2xm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fabs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fadd	%st(2), %st
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fadds	(%ecx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     faddl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   faddp	%st, %st(2)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fiadds	(%ecx)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fiaddl	(%ecx)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fbld	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fbstp	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fchs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   1.00   1.00   fnclex
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcmovu	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcom	%st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcom	%st(3)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     fcoms	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     fcoml	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcomp	%st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcomp	%st(3)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     fcomps	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -     0.50   0.50    -      -     fcompl	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fcompp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fcompi	%st(3), %st
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fcos
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -      -      -      -      -      -      -      -      -      -      -      -     fdecstp
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdiv	%st(2), %st
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fdivs	(%ecx)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fdivl	(%eax)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivp	%st, %st(2)
+# CHECK-NEXT: 1.00   0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fidivs	(%ecx)
+# CHECK-NEXT: 1.00   0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fidivl	(%eax)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivr	%st(2), %st
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fdivrs	(%ecx)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fdivrl	(%eax)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdivrp	%st, %st(2)
+# CHECK-NEXT: 1.00   0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fidivrs	(%ecx)
+# CHECK-NEXT: 1.00   0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fidivrl	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     ffree	%st(0)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ficoms	(%ecx)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ficoml	(%eax)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ficomps	(%ecx)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     ficompl	(%eax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     filds	(%edx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fildl	(%ecx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fildll	(%eax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     fincstp
+# CHECK-NEXT:  -     3.00    -     3.00    -     3.00    -      -      -      -      -      -      -      -      -     1.50   1.50   1.50   1.50   fninit
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fists	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fistl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fistps	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fistpl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fistpll	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fisttps	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fisttpl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50    -      -      -     0.33   0.33   0.33    -      -      -      -      -     fisttpll	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fld	%st(0)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     flds	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fldl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fldt	(%eax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   fldcw	(%eax)
+# CHECK-NEXT: 3.33   1.67   3.33   1.67   3.33   1.67    -      -     2.67   2.67   2.67    -      -      -      -     9.75   9.75   10.75  10.75  fldenv	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fld1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fldl2e
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fldl2t
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fldlg2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fldln2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   1.00   fldpi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fldz
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmul	%st(2), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.75   0.75   fmuls	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.75   0.75   fmull	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.25   0.25   0.25   0.25   fmulp	%st, %st(2)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -     0.25   0.25   0.75   0.75   fimuls	(%ecx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -     0.25   0.25   0.75   0.75   fimull	(%eax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     fnop
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fpatan
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fprem
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fprem1
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fptan
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     frndint
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fscale
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fsin
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fsincos
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -     7.00   0.50   0.50    -      -     fsqrt
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fst	%st(0)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fsts	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fstl	(%ecx)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fstp	%st(0)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fstpl	(%edx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fstpl	(%ecx)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -      -      -     fstpt	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   fnstcw	(%eax)
+# CHECK-NEXT: 7.00   6.33   7.00   5.33   7.00   5.33   5.50   5.50   3.67   3.67   3.67    -      -      -      -     7.50   7.50   12.50  12.50  fnstenv	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50   fnstsw	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   wait
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsub	%st(2), %st
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fsubs	(%ecx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fsubl	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubp	%st, %st(2)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fisubs	(%ecx)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fisubl	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubr	%st(2), %st
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fsubrs	(%ecx)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fsubrl	(%eax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   fsubrp	%st, %st(2)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fisubrs	(%ecx)
+# CHECK-NEXT:  -     0.67    -     0.67    -     0.67    -      -     0.33   0.33   0.33    -      -      -      -      -      -      -      -     fisubrl	(%eax)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     ftst
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucom	%st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucom	%st(3)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucomp	%st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucomp	%st(3)
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     fucompp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     fucompi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   wait
+# CHECK-NEXT:  -     0.33    -     0.33    -     0.33    -      -      -      -      -      -      -      -      -      -      -      -      -     fxam
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -     2.50   2.50   1.50   1.50   fxch	%st(1)
+# CHECK-NEXT: 1.33   1.00   1.33   1.00   1.33   1.00    -      -      -      -      -      -      -      -      -     2.50   2.50   1.50   1.50   fxch	%st(3)
+# CHECK-NEXT: 0.67   0.33   0.67   0.33   0.67   0.33    -      -     11.00  11.00  11.00   -      -      -      -     11.75  11.75  15.25  15.25  fxrstor	(%eax)
+# CHECK-NEXT: 4.17   12.50  4.17   2.50   4.17   2.50   19.00  19.00  0.67   0.67   0.67   12.67  12.67  12.67   -      -      -     1.00   1.00   fxsave	(%eax)
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fxtract
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fyl2x
+# CHECK-NEXT: 0.17   0.17   0.17   0.17   0.17   0.17    -      -      -      -      -      -      -      -      -      -      -      -      -     fyl2xp1
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-xsave.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-xsave.s
new file mode 100644
index 000000000000..2d9748021d62
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/resources-xsave.s
@@ -0,0 +1,60 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -instruction-tables < %s | FileCheck %s
+
+xgetbv
+
+xrstor  (%rax)
+
+xrstors (%rax)
+
+xsave   (%rax)
+
+xsetbv
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  23     4     3.33                  U     xgetbv
+# CHECK-NEXT:  31     37    5.25    *      *      U     xrstor	(%rax)
+# CHECK-NEXT:  31     37    5.25    *      *      U     xrstors	(%rax)
+# CHECK-NEXT:  140    42    15.00   *      *      U     xsave	(%rax)
+# CHECK-NEXT:  54     5     8.67    *      *      U     xsetbv
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT: 32.33  22.33  32.33  22.33  32.33  22.33  0.50   0.50   2.00   2.00   2.00   0.33   0.33   0.33    -     25.00  25.00  28.50  28.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT: 3.83   2.83   3.83   2.83   3.83   2.83    -      -      -      -      -      -      -      -      -     0.75   0.75   0.75   0.75   xgetbv
+# CHECK-NEXT: 2.67   0.33   2.67   0.33   2.67   0.33    -      -     0.33   0.33   0.33    -      -      -      -     5.25   5.25   5.25   5.25   xrstor	(%rax)
+# CHECK-NEXT: 2.67   0.33   2.67   0.33   2.67   0.33    -      -     0.33   0.33   0.33    -      -      -      -     5.25   5.25   5.25   5.25   xrstors	(%rax)
+# CHECK-NEXT: 14.67  10.00  14.67  10.00  14.67  10.00  0.50   0.50   1.33   1.33   1.33   0.33   0.33   0.33    -     13.25  13.25  16.75  16.75  xsave	(%rax)
+# CHECK-NEXT: 8.50   8.83   8.50   8.83   8.50   8.83    -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50   xsetbv
diff --git a/llvm/test/tools/llvm-mca/X86/LunarlakeP/zero-idioms.s b/llvm/test/tools/llvm-mca/X86/LunarlakeP/zero-idioms.s
new file mode 100644
index 000000000000..a98fe5071952
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/LunarlakeP/zero-idioms.s
@@ -0,0 +1,502 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=lunarlake -timeline -register-file-stats -iterations=1 < %s | FileCheck %s
+
+# On SKL, renamer-based zeroing does not work for:
+#  - 16 and 8-bit GPRs
+#  - MMX
+#  - ANDN variants
+
+subl  %eax, %eax
+subq  %rax, %rax
+xorl  %eax, %eax
+xorq  %rax, %rax
+
+pcmpgtb   %mm2, %mm2
+pcmpgtd   %mm2, %mm2
+# pcmpgtq   %mm2, %mm2 # invalid operand for instruction
+pcmpgtw   %mm2, %mm2
+
+pcmpgtb   %xmm2, %xmm2
+pcmpgtd   %xmm2, %xmm2
+pcmpgtq   %xmm2, %xmm2
+pcmpgtw   %xmm2, %xmm2
+
+vpcmpgtb  %xmm3, %xmm3, %xmm3
+vpcmpgtd  %xmm3, %xmm3, %xmm3
+vpcmpgtq  %xmm3, %xmm3, %xmm3
+vpcmpgtw  %xmm3, %xmm3, %xmm3
+
+vpcmpgtb  %xmm3, %xmm3, %xmm5
+vpcmpgtd  %xmm3, %xmm3, %xmm5
+vpcmpgtq  %xmm3, %xmm3, %xmm5
+vpcmpgtw  %xmm3, %xmm3, %xmm5
+
+vpcmpgtb  %ymm3, %ymm3, %ymm3
+vpcmpgtd  %ymm3, %ymm3, %ymm3
+vpcmpgtq  %ymm3, %ymm3, %ymm3
+vpcmpgtw  %ymm3, %ymm3, %ymm3
+
+vpcmpgtb  %ymm3, %ymm3, %ymm5
+vpcmpgtd  %ymm3, %ymm3, %ymm5
+vpcmpgtq  %ymm3, %ymm3, %ymm5
+vpcmpgtw  %ymm3, %ymm3, %ymm5
+
+psubb   %mm2, %mm2
+psubd   %mm2, %mm2
+psubq   %mm2, %mm2
+psubw   %mm2, %mm2
+psubb   %xmm2, %xmm2
+psubd   %xmm2, %xmm2
+psubq   %xmm2, %xmm2
+psubw   %xmm2, %xmm2
+vpsubb  %xmm3, %xmm3, %xmm3
+vpsubd  %xmm3, %xmm3, %xmm3
+vpsubq  %xmm3, %xmm3, %xmm3
+vpsubw  %xmm3, %xmm3, %xmm3
+vpsubb  %ymm3, %ymm3, %ymm3
+vpsubd  %ymm3, %ymm3, %ymm3
+vpsubq  %ymm3, %ymm3, %ymm3
+vpsubw  %ymm3, %ymm3, %ymm3
+
+vpsubb  %xmm3, %xmm3, %xmm5
+vpsubd  %xmm3, %xmm3, %xmm5
+vpsubq  %xmm3, %xmm3, %xmm5
+vpsubw  %xmm3, %xmm3, %xmm5
+vpsubb  %ymm3, %ymm3, %ymm5
+vpsubd  %ymm3, %ymm3, %ymm5
+vpsubq  %ymm3, %ymm3, %ymm5
+vpsubw  %ymm3, %ymm3, %ymm5
+
+andnps  %xmm0, %xmm0
+andnpd  %xmm1, %xmm1
+vandnps %xmm2, %xmm2, %xmm2
+vandnpd %xmm1, %xmm1, %xmm1
+vandnps %ymm2, %ymm2, %ymm2
+vandnpd %ymm1, %ymm1, %ymm1
+pandn   %mm2, %mm2
+pandn   %xmm2, %xmm2
+vpandn  %xmm3, %xmm3, %xmm3
+vpandn  %ymm3, %ymm3, %ymm3
+
+vandnps %xmm2, %xmm2, %xmm5
+vandnpd %xmm1, %xmm1, %xmm5
+vpandn  %xmm3, %xmm3, %xmm5
+vandnps %ymm2, %ymm2, %ymm5
+vandnpd %ymm1, %ymm1, %ymm5
+vpandn  %ymm3, %ymm3, %ymm5
+
+xorps  %xmm0, %xmm0
+xorpd  %xmm1, %xmm1
+vxorps %xmm2, %xmm2, %xmm2
+vxorpd %xmm1, %xmm1, %xmm1
+vxorps %ymm2, %ymm2, %ymm2
+vxorpd %ymm1, %ymm1, %ymm1
+pxor   %mm2, %mm2
+pxor   %xmm2, %xmm2
+vpxor  %xmm3, %xmm3, %xmm3
+vpxor  %ymm3, %ymm3, %ymm3
+
+vxorps %xmm4, %xmm4, %xmm5
+vxorpd %xmm1, %xmm1, %xmm3
+vxorps %ymm4, %ymm4, %ymm5
+vxorpd %ymm1, %ymm1, %ymm3
+vpxor  %xmm3, %xmm3, %xmm5
+vpxor  %ymm3, %ymm3, %ymm5
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      83
+# CHECK-NEXT: Total Cycles:      32
+# CHECK-NEXT: Total uOps:        83
+
+# CHECK:      Dispatch Width:    8
+# CHECK-NEXT: uOps Per Cycle:    2.59
+# CHECK-NEXT: IPC:               2.59
+# CHECK-NEXT: Block RThroughput: 13.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.33                        subl	%eax, %eax
+# CHECK-NEXT:  1      1     0.33                        subq	%rax, %rax
+# CHECK-NEXT:  1      2     0.17                        xorl	%eax, %eax
+# CHECK-NEXT:  1      2     0.17                        xorq	%rax, %rax
+# CHECK-NEXT:  1      1     1.00                        pcmpgtb	%mm2, %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpgtd	%mm2, %mm2
+# CHECK-NEXT:  1      1     1.00                        pcmpgtw	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT:  1      3     1.50                        pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      3     1.50                        vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.33                        psubb	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.33                        psubd	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.33                        psubq	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.33                        psubw	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.25                        psubb	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubd	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubq	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        psubw	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.25                        andnps	%xmm0, %xmm0
+# CHECK-NEXT:  1      1     0.25                        andnpd	%xmm1, %xmm1
+# CHECK-NEXT:  1      1     0.25                        vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT:  1      1     0.25                        vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT:  1      1     0.25                        pandn	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.25                        pandn	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  1      1     0.25                        xorps	%xmm0, %xmm0
+# CHECK-NEXT:  1      1     0.25                        xorpd	%xmm1, %xmm1
+# CHECK-NEXT:  1      1     0.25                        vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT:  1      1     0.25                        vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT:  1      1     0.25                        pxor	%mm2, %mm2
+# CHECK-NEXT:  1      1     0.25                        pxor	%xmm2, %xmm2
+# CHECK-NEXT:  1      1     0.25                        vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT:  1      1     0.25                        vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT:  1      1     0.25                        vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT:  1      1     0.25                        vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  1      1     0.25                        vpxor	%ymm3, %ymm3, %ymm5
+
+# CHECK:      Register File statistics:
+# CHECK-NEXT: Total number of mappings created:    87
+# CHECK-NEXT: Max number of mappings used:         68
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - LNLPPort00
+# CHECK-NEXT: [1]   - LNLPPort01
+# CHECK-NEXT: [2]   - LNLPPort02
+# CHECK-NEXT: [3]   - LNLPPort03
+# CHECK-NEXT: [4]   - LNLPPort04
+# CHECK-NEXT: [5]   - LNLPPort05
+# CHECK-NEXT: [6]   - LNLPPort10
+# CHECK-NEXT: [7]   - LNLPPort11
+# CHECK-NEXT: [8]   - LNLPPort20
+# CHECK-NEXT: [9]   - LNLPPort21
+# CHECK-NEXT: [10]  - LNLPPort22
+# CHECK-NEXT: [11]  - LNLPPort25
+# CHECK-NEXT: [12]  - LNLPPort26
+# CHECK-NEXT: [13]  - LNLPPort27
+# CHECK-NEXT: [14]  - LNLPPortInvalid
+# CHECK-NEXT: [15]  - LNLPVPort00
+# CHECK-NEXT: [16]  - LNLPVPort01
+# CHECK-NEXT: [17]  - LNLPVPort02
+# CHECK-NEXT: [18]  - LNLPVPort03
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]
+# CHECK-NEXT:  -     2.00   1.00   2.00   1.00   2.00    -      -      -      -      -      -      -      -      -     22.00  22.00  20.00  21.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16]   [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     subl	%eax, %eax
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     subq	%rax, %rax
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -     xorl	%eax, %eax
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     xorq	%rax, %rax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtb	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtd	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtw	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00   pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00    -     vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00    -     vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00    -     vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     3.00    -     vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     psubb	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     psubd	%mm2, %mm2
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     psubq	%mm2, %mm2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     psubw	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   psubb	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   psubd	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   psubq	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   psubw	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     andnps	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     andnpd	%xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pandn	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pandn	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     xorps	%xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     xorpd	%xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     pxor	%mm2, %mm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     pxor	%xmm2, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -      -     vpxor	%ymm3, %ymm3, %ymm5
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789          01
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DeER .    .    .    .    .    ..   subl	%eax, %eax
+# CHECK-NEXT: [0,1]     D=eER.    .    .    .    .    ..   subq	%rax, %rax
+# CHECK-NEXT: [0,2]     D==eeER   .    .    .    .    ..   xorl	%eax, %eax
+# CHECK-NEXT: [0,3]     D====eeER .    .    .    .    ..   xorq	%rax, %rax
+# CHECK-NEXT: [0,4]     DeE-----R .    .    .    .    ..   pcmpgtb	%mm2, %mm2
+# CHECK-NEXT: [0,5]     D=eE----R .    .    .    .    ..   pcmpgtd	%mm2, %mm2
+# CHECK-NEXT: [0,6]     D==eE---R .    .    .    .    ..   pcmpgtw	%mm2, %mm2
+# CHECK-NEXT: [0,7]     DeE-----R .    .    .    .    ..   pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT: [0,8]     .DeE----R .    .    .    .    ..   pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT: [0,9]     .D=eeeE-R .    .    .    .    ..   pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT: [0,10]    .D====eER .    .    .    .    ..   pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT: [0,11]    .D=eE---R .    .    .    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,12]    .D==eE--R .    .    .    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,13]    .D===eeeER.    .    .    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,14]    .D======eER    .    .    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,15]    .D=======eER   .    .    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,16]    . D======eER   .    .    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,17]    . D======eeeER .    .    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,18]    . D=======eE-R .    .    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,19]    . D=======eE-R .    .    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,20]    . D========eER .    .    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,21]    . D=========eeeER   .    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,22]    . D============eER  .    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,23]    . D=============eER .    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,24]    .  D============eER .    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,25]    .  D============eeeER    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,26]    .  D=============eE-R    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,27]    .  DeE--------------R    .    ..   psubb	%mm2, %mm2
+# CHECK-NEXT: [0,28]    .  D=eE-------------R    .    ..   psubd	%mm2, %mm2
+# CHECK-NEXT: [0,29]    .  D==eE------------R    .    ..   psubq	%mm2, %mm2
+# CHECK-NEXT: [0,30]    .  D===eE-----------R    .    ..   psubw	%mm2, %mm2
+# CHECK-NEXT: [0,31]    .  D===eE-----------R    .    ..   psubb	%xmm2, %xmm2
+# CHECK-NEXT: [0,32]    .   D===eE----------R    .    ..   psubd	%xmm2, %xmm2
+# CHECK-NEXT: [0,33]    .   D====eE---------R    .    ..   psubq	%xmm2, %xmm2
+# CHECK-NEXT: [0,34]    .   D=====eE--------R    .    ..   psubw	%xmm2, %xmm2
+# CHECK-NEXT: [0,35]    .   D===========eE--R    .    ..   vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,36]    .   D============eE-R    .    ..   vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,37]    .   D=============eER    .    ..   vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,38]    .   D==============eER   .    ..   vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,39]    .   D===============eER  .    ..   vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,40]    .    D===============eER .    ..   vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,41]    .    D================eER.    ..   vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,42]    .    D=================eER    ..   vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,43]    .    D==================eER   ..   vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,44]    .    D==================eER   ..   vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,45]    .    D==================eER   ..   vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,46]    .    D==================eER   ..   vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,47]    .    D===================eER  ..   vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,48]    .    .D==================eER  ..   vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,49]    .    .D==================eER  ..   vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,50]    .    .D==================eER  ..   vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,51]    .    .DeE------------------R  ..   andnps	%xmm0, %xmm0
+# CHECK-NEXT: [0,52]    .    .DeE------------------R  ..   andnpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,53]    .    .D====eE--------------R  ..   vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,54]    .    .D=eE-----------------R  ..   vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,55]    .    .D=====eE-------------R  ..   vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,56]    .    . D===eE--------------R  ..   vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,57]    .    . DeE-----------------R  ..   pandn	%mm2, %mm2
+# CHECK-NEXT: [0,58]    .    . D=====eE------------R  ..   pandn	%xmm2, %xmm2
+# CHECK-NEXT: [0,59]    .    . D==================eER ..   vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,60]    .    . D===================eER..   vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,61]    .    . D======eE-------------R..   vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: [0,62]    .    . D====eE---------------R..   vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: [0,63]    .    . D====================eER.   vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,64]    .    .  D=====eE--------------R.   vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: [0,65]    .    .  D===eE----------------R.   vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: [0,66]    .    .  D===================eER.   vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,67]    .    .  D====eE---------------R.   xorps	%xmm0, %xmm0
+# CHECK-NEXT: [0,68]    .    .  D====eE---------------R.   xorpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,69]    .    .  D=====eE--------------R.   vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,70]    .    .  D======eE-------------R.   vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,71]    .    .  D======eE-------------R.   vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,72]    .    .   D=======eE-----------R.   vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,73]    .    .   D=====eE-------------R.   pxor	%mm2, %mm2
+# CHECK-NEXT: [0,74]    .    .   D========eE----------R.   pxor	%xmm2, %xmm2
+# CHECK-NEXT: [0,75]    .    .   D==================eER.   vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,76]    .    .   D===================eER   vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,77]    .    .   D=========eE----------R   vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: [0,78]    .    .   D=========eE----------R   vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: [0,79]    .    .   D=========eE----------R   vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: [0,80]    .    .    D=======eE-----------R   vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: [0,81]    .    .    D=========eE---------R   vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,82]    .    .    D=========eE---------R   vpxor	%ymm3, %ymm3, %ymm5
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     1     1.0    1.0    0.0       subl	%eax, %eax
+# CHECK-NEXT: 1.     1     2.0    0.0    0.0       subq	%rax, %rax
+# CHECK-NEXT: 2.     1     3.0    0.0    0.0       xorl	%eax, %eax
+# CHECK-NEXT: 3.     1     5.0    0.0    0.0       xorq	%rax, %rax
+# CHECK-NEXT: 4.     1     1.0    1.0    5.0       pcmpgtb	%mm2, %mm2
+# CHECK-NEXT: 5.     1     2.0    0.0    4.0       pcmpgtd	%mm2, %mm2
+# CHECK-NEXT: 6.     1     3.0    0.0    3.0       pcmpgtw	%mm2, %mm2
+# CHECK-NEXT: 7.     1     1.0    1.0    5.0       pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT: 8.     1     1.0    0.0    4.0       pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT: 9.     1     2.0    0.0    1.0       pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT: 10.    1     5.0    0.0    0.0       pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT: 11.    1     2.0    2.0    3.0       vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 12.    1     3.0    0.0    2.0       vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 13.    1     4.0    0.0    0.0       vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 14.    1     7.0    0.0    0.0       vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 15.    1     8.0    0.0    0.0       vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 16.    1     7.0    0.0    0.0       vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 17.    1     7.0    0.0    0.0       vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 18.    1     8.0    1.0    1.0       vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 19.    1     8.0    1.0    1.0       vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 20.    1     9.0    0.0    0.0       vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 21.    1     10.0   0.0    0.0       vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 22.    1     13.0   0.0    0.0       vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 23.    1     14.0   0.0    0.0       vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 24.    1     13.0   0.0    0.0       vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 25.    1     13.0   0.0    0.0       vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 26.    1     14.0   1.0    1.0       vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 27.    1     1.0    0.0    14.0      psubb	%mm2, %mm2
+# CHECK-NEXT: 28.    1     2.0    0.0    13.0      psubd	%mm2, %mm2
+# CHECK-NEXT: 29.    1     3.0    0.0    12.0      psubq	%mm2, %mm2
+# CHECK-NEXT: 30.    1     4.0    0.0    11.0      psubw	%mm2, %mm2
+# CHECK-NEXT: 31.    1     4.0    0.0    11.0      psubb	%xmm2, %xmm2
+# CHECK-NEXT: 32.    1     4.0    0.0    10.0      psubd	%xmm2, %xmm2
+# CHECK-NEXT: 33.    1     5.0    0.0    9.0       psubq	%xmm2, %xmm2
+# CHECK-NEXT: 34.    1     6.0    0.0    8.0       psubw	%xmm2, %xmm2
+# CHECK-NEXT: 35.    1     12.0   0.0    2.0       vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 36.    1     13.0   0.0    1.0       vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 37.    1     14.0   0.0    0.0       vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 38.    1     15.0   0.0    0.0       vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 39.    1     16.0   0.0    0.0       vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 40.    1     16.0   0.0    0.0       vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 41.    1     17.0   0.0    0.0       vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 42.    1     18.0   0.0    0.0       vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 43.    1     19.0   0.0    0.0       vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 44.    1     19.0   0.0    0.0       vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 45.    1     19.0   0.0    0.0       vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 46.    1     19.0   0.0    0.0       vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 47.    1     20.0   1.0    0.0       vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 48.    1     19.0   1.0    0.0       vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 49.    1     19.0   1.0    0.0       vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 50.    1     19.0   1.0    0.0       vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 51.    1     1.0    1.0    18.0      andnps	%xmm0, %xmm0
+# CHECK-NEXT: 52.    1     1.0    1.0    18.0      andnpd	%xmm1, %xmm1
+# CHECK-NEXT: 53.    1     5.0    0.0    14.0      vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: 54.    1     2.0    0.0    17.0      vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: 55.    1     6.0    0.0    13.0      vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: 56.    1     4.0    2.0    14.0      vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: 57.    1     1.0    0.0    17.0      pandn	%mm2, %mm2
+# CHECK-NEXT: 58.    1     6.0    0.0    12.0      pandn	%xmm2, %xmm2
+# CHECK-NEXT: 59.    1     19.0   2.0    0.0       vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 60.    1     20.0   0.0    0.0       vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 61.    1     7.0    0.0    13.0      vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: 62.    1     5.0    0.0    15.0      vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: 63.    1     21.0   0.0    0.0       vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 64.    1     6.0    0.0    14.0      vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: 65.    1     4.0    0.0    16.0      vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: 66.    1     20.0   0.0    0.0       vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 67.    1     5.0    5.0    15.0      xorps	%xmm0, %xmm0
+# CHECK-NEXT: 68.    1     5.0    1.0    15.0      xorpd	%xmm1, %xmm1
+# CHECK-NEXT: 69.    1     6.0    0.0    14.0      vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: 70.    1     7.0    1.0    13.0      vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: 71.    1     7.0    0.0    13.0      vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: 72.    1     8.0    1.0    11.0      vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: 73.    1     6.0    6.0    13.0      pxor	%mm2, %mm2
+# CHECK-NEXT: 74.    1     9.0    2.0    10.0      pxor	%xmm2, %xmm2
+# CHECK-NEXT: 75.    1     19.0   0.0    0.0       vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 76.    1     20.0   0.0    0.0       vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 77.    1     10.0   10.0   10.0      vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: 78.    1     10.0   1.0    10.0      vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: 79.    1     10.0   10.0   10.0      vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: 80.    1     8.0    0.0    11.0      vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: 81.    1     10.0   1.0    9.0       vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 82.    1     10.0   1.0    9.0       vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:        1     9.0    0.7    5.6       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/show-encoding.s b/llvm/test/tools/llvm-mca/X86/show-encoding.s
index 9d2015664271..b20b7fc31cde 100644
--- a/llvm/test/tools/llvm-mca/X86/show-encoding.s
+++ b/llvm/test/tools/llvm-mca/X86/show-encoding.s
@@ -74,4 +74,4 @@
 # WITHENCODINGS-NEXT:  1      1     0.50                         6     c4 c1 30 c6 c1 ff             vshufps	$255, %xmm9, %xmm9, %xmm0
 # WITHENCODINGS-NEXT:  1      1     1.00           *             6     c4 01 78 11 0c 8b             vmovups	%xmm9, (%r11,%r9,4)
 # WITHENCODINGS-NEXT:  1      1     0.50                         3     44 39 c6                      cmpl	%r8d, %esi
-# WITHENCODINGS-NEXT:  1      1     0.50                         6     0f 8c 00 00 00 00             jl	-90
+# WITHENCODINGS-NEXT:  1      1     0.50                         2     7c 00                         jl	-90
diff --git a/llvm/test/tools/llvm-objcopy/COFF/string-table.test b/llvm/test/tools/llvm-objcopy/COFF/string-table.test
new file mode 100644
index 000000000000..aef710b69412
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/COFF/string-table.test
@@ -0,0 +1,34 @@
+## Show that section names appear before symbol names in the COFF strtab.
+## These names are only added to the strtab if they exceed 8 characters.
+
+# RUN: yaml2obj %s -o %t
+# RUN: touch %t.sec
+# RUN: llvm-objcopy --add-section=.debug_str=%t.sec %t %t1
+# RUN: llvm-readobj --string-table %t1 | FileCheck %s
+
+# CHECK:      StringTable {
+# CHECK-NEXT:   Length: 37
+# CHECK-NEXT:   [     4] .debug_str
+# CHECK-NEXT:   [     f] symbol_zzz
+# CHECK-NEXT:   [    1a] symbol_aaa
+# CHECK-NEXT: }
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+symbols:
+  - Name:            symbol_aaa
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            symbol_zzz
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
diff --git a/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-executable.yaml b/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-executable.yaml
new file mode 100644
index 000000000000..d318ea01b4c3
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-executable.yaml
@@ -0,0 +1,67 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump %t -d --symbolize-operands --no-show-raw-insn --no-leading-addr | \
+# RUN:   FileCheck %s --match-full-lines -DABS_ADRP_VAL=0x6000
+# RUN: llvm-objdump %t -d --symbolize-operands --no-show-raw-insn --no-leading-addr --adjust-vma=0x2000 | \
+# RUN:   FileCheck %s --match-full-lines -DABS_ADRP_VAL=0x8000
+
+## Expect to find the branch labels and global variable name.
+# CHECK:      <_start>:
+# CHECK-NEXT:   ldr x0, <symbol>
+# CHECK-NEXT: <L0>:
+# CHECK-NEXT:   adrp x1, [[ABS_ADRP_VAL]] <symbol+0xff4>
+# CHECK-NEXT:   adr x2, <symbol>
+# CHECK-NEXT:   cmp x1, x2
+# CHECK-NEXT:   b.eq <L1>
+# CHECK-NEXT:   b <L0>
+# CHECK-NEXT: <L1>:
+# CHECK-NEXT:   cbz x2, <L0>
+# CHECK-NEXT:   ret
+
+## Machine code generated with:
+# llvm-mc --arch=aarch64 --filetype=obj -o tmp.o <<EOF
+#   .text
+#   .p2align 14
+#   .globl .start
+# _start:
+#   ldr x0, symbol
+# 1:
+#   adrp x1, symbol + 0x1000
+#   adr x2, symbol
+#   cmp x1, x2
+#   b.eq 2f
+#   b 1b
+# 2:
+#   cbz x2, 1b
+#   ret
+#
+#   .data
+#   .p2align 12
+#   .skip 12
+# symbol:
+# EOF
+# ld.lld -shared --nmagic -o tmp.so tmp.o
+# llvm-objdump -s tmp.so --section=.text
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_AARCH64
+Sections:
+  - Name:    .text
+    Type:    SHT_PROGBITS
+    Address: 0x4000
+    Flags:   [SHF_ALLOC, SHF_EXECINSTR]
+    Content: '60800058010000d0228000103f0002eb40000054fcffff1762ffffb4c0035fd6'
+  - Name:    .data
+    Type:    SHT_PROGBITS
+    Flags:   [SHF_ALLOC, SHF_WRITE]
+    Address: 0x5000
+Symbols:
+  - Name:    _start
+    Section: .text
+    Value:   0x4000
+  - Name:    symbol
+    Section: .data
+    Value:   0x500c
diff --git a/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-relocatable.s b/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-relocatable.s
new file mode 100644
index 000000000000..17af1bfcde33
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/AArch64/symbolize-operands-relocatable.s
@@ -0,0 +1,79 @@
+# RUN: llvm-mc --triple=aarch64-elf --filetype=obj < %s | \
+# RUN:   llvm-objdump -d -r --symbolize-operands --no-show-raw-insn --no-leading-addr - | \
+# RUN:   FileCheck %s --match-full-lines
+
+# CHECK:      <fn1>:
+# CHECK-NEXT:   b <L0>
+# CHECK-NEXT:   tbz x0, #0x2c, <L2>
+# CHECK-NEXT: <L0>:
+# CHECK-NEXT:   b.eq <L1>
+# CHECK-NEXT: <L1>:
+# CHECK-NEXT:   cbz x1, <L0>
+# CHECK-NEXT: <L2>:
+# CHECK-NEXT:   nop
+# CHECK-NEXT: <L3>:
+# CHECK-NEXT:   bl <L3>
+# CHECK-NEXT:     R_AARCH64_CALL26 fn2
+# CHECK-NEXT:   bl <fn2>
+# CHECK-NEXT:   adr x0, <L2>
+# CHECK-NEXT: <L4>:
+# CHECK-NEXT:   adr x1, <L4>
+# CHECK-NEXT:     R_AARCH64_ADR_PREL_LO21 fn2
+# CHECK-NEXT:   adr x2, <fn2>
+# CHECK-NEXT:   ldr w0, <L2>
+# CHECK-NEXT: <L5>:
+# CHECK-NEXT:   ldr w0, <L5>
+# CHECK-NEXT:     R_AARCH64_LD_PREL_LO19 fn2
+# CHECK-NEXT:   ret
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   nop
+# CHECK-EMPTY:
+# CHECK-NEXT: <fn2>:
+# CHECK-NEXT:   bl <L0>
+# CHECK-NEXT:   adrp x3, 0x0 <fn1>
+# CHECK-NEXT:     R_AARCH64_ADR_PREL_PG_HI21 fn2
+# CHECK-NEXT:   add x3, x3, #0x0
+# CHECK-NEXT:     R_AARCH64_ADD_ABS_LO12_NC fn2
+# CHECK-NEXT:   adrp x3, 0x0 <fn1>
+# CHECK-NEXT:     R_AARCH64_ADR_PREL_PG_HI21 fn2
+# CHECK-NEXT:   ldr x0, [x3]
+# CHECK-NEXT:     R_AARCH64_LDST64_ABS_LO12_NC fn2
+# CHECK-NEXT:   ret
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   nop
+# CHECK-NEXT: <L0>:
+# CHECK-NEXT:   ret
+
+    .p2align 4
+    .global fn1
+fn1:
+    b 0f
+    tbz x0, 44, 2f
+0:  b.eq 1f
+1:  cbz x1, 0b
+2:  nop
+    bl fn2
+    bl .Lfn2
+    adr x0, 2b
+    adr x1, fn2
+    adr x2, .Lfn2
+    ldr w0, 2b
+    ldr w0, fn2
+    ret
+
+    .p2align 4
+    .global fn2
+fn2:
+.Lfn2: ## Local label for non-interposable call.
+    bl .Lfn3
+    ## In future, we might identify the pairs and symbolize the operands properly.
+    adrp x3, fn2
+    add x3, x3, :lo12:fn2
+    adrp x3, fn2
+    ldr x0, [x3, :lo12:fn2]
+    ret
+
+    .p2align 4
+.Lfn3: ## Private function
+    ret
diff --git a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
index cc7faea67bed..737e902f4023 100644
--- a/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
+++ b/llvm/test/tools/llvm-objdump/X86/elf-bbaddrmap-disassemble-symbolize-operands.yaml
@@ -33,10 +33,10 @@
 # ATT-NEXT: <BB0>:
 # ATT-NEXT:   pushq  %rax
 # ATT-NEXT:   movl   %edx, %eax
-# ATT-NEXT:   je     <BB2>
-# ATT-NEXT: <BB1>:
-# ATT-NEXT:   xorl   %esi, %esi
+# ATT-NEXT:   je     <BB4>
 # ATT-NEXT: <BB2>:
+# ATT-NEXT:   xorl   %esi, %esi
+# ATT-NEXT: <BB4>:
 # ATT-NEXT:   callq  <bar>
 # ATT-NEXT:   retq
 # ATT:      <foo.cold>:
@@ -60,10 +60,10 @@
 # INTEL-NEXT: <BB0>:
 # INTEL-NEXT:   push  rax
 # INTEL-NEXT:   mov   eax, edx
-# INTEL-NEXT:   je    <BB2>
-# INTEL-NEXT: <BB1>:
-# INTEL-NEXT:   xor   esi, esi
+# INTEL-NEXT:   je    <BB4>
 # INTEL-NEXT: <BB2>:
+# INTEL-NEXT:   xor   esi, esi
+# INTEL-NEXT: <BB4>:
 # INTEL-NEXT:   call  <bar>
 # INTEL-NEXT:   ret
 # INTEL:      <foo.cold>:
@@ -139,17 +139,20 @@ Sections:
     Type:   SHT_LLVM_BB_ADDR_MAP
     Link:   .text.bar
     Entries:
-      - Version: 1
+      - Version: 2
         BBRanges:
          - BaseAddress: 0x5000
            BBEntries:
-            - AddressOffset: 0x0
+            - ID:            0
+              AddressOffset: 0x0
               Size:          0x1
               Metadata:      0x1
-            - AddressOffset: 0x4
+            - ID:            2
+              AddressOffset: 0x4
               Size:          0x2
               Metadata:      0x0
-            - AddressOffset: 0x0
+            - ID:            4
+              AddressOffset: 0x0
               Size:          0x6
               Metadata:      0x0
 
@@ -230,17 +233,20 @@ Sections:
                AddressOffset: 0x0
                Size:          0x1
                Metadata:      0x2
-      - Version: 1
+      - Version: 2
         BBRanges:
          - BaseAddress: 0x5000
            BBEntries:
-            - AddressOffset: 0x0
+            - ID:            0
+              AddressOffset: 0x0
               Size:          0x1
               Metadata:      0x1
-            - AddressOffset: 0x4
+            - ID:            2
+              AddressOffset: 0x4
               Size:          0x2
               Metadata:      0x0
-            - AddressOffset: 0x0
+            - ID:            4
+              AddressOffset: 0x0
               Size:          0x6
               Metadata:      0x0
 
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
new file mode 100644
index 000000000000..ed08c850cb4a
--- /dev/null
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
@@ -0,0 +1,124 @@
+ <html>
+  <head>
+  <style>
+  table, th, td {
+    border: 1px solid black;
+  }
+  table.center {
+    margin-left: auto;
+    margin-right: auto;
+  }
+  </style>
+  </head>
+  <body>
+  <table>
+  <caption><b>Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>LLVM IR Instruction</th>
+    <th>Function Name</th>
+    <th>Basic Block Name</th>
+    <th>Action</th>
+    <th>Origin</th>
+  </tr>
+  </tr>
+    <tr>
+    <td>test.ll</td>
+    <td>LoopVectorizePass</td>
+    <td>add</td>
+    <td>fn</td>
+    <td>no-name</td>
+    <td>not-generate</td>
+    <td><details><summary>View Origin StackTrace</summary><pre>Stack Trace 0:
+ #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9
+ #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11
+ #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5
+ #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14
+ #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9
+ #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9
+ #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9
+ #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22
+ #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22
+ #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31
+#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8
+#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8
+#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12
+#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5</pre></details></td>
+    </tr>
+  <tr>
+</table>
+<br>
+<table>
+  <caption><b>Summary of Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+    <tr>
+    <td>LoopVectorizePass</td>
+    <td>1</td>
+    </tr>
+  <tr>
+</table>
+<br>
+<br>
+<table>
+  <caption><b>SP Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Function Name</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of SP Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<br>
+<table>
+  <caption><b>Variable Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Variable</th>
+    <th>Function</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of Variable Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+</body>
+  </html>
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
index f70ebbf86b0a..d8941cbdedbe 100644
--- a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-skipped.html
@@ -128,6 +128,15 @@
     <td>drop</td>
     </tr>
   <tr>
+  </tr>
+    <tr>
+    <td>wrstabs.c</td>
+    <td>Simplify the CFG</td>
+    <td>tindex</td>
+    <td>stab_bool_type</td>
+    <td>drop</td>
+    </tr>
+  <tr>
   </tr>
     <tr>
     <td>prdbg.c</td>
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
new file mode 100644
index 000000000000..ab59bc295add
--- /dev/null
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
@@ -0,0 +1 @@
+{"file":"test.ll", "pass":"LoopVectorizePass", "bugs": [[{"action":"not-generate","bb-name":"no-name","fn-name":"fn","instr":"add","metadata":"DILocation", "origin": "Stack Trace 0:\n #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9\n #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11\n #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5\n #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14\n #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9\n #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9\n #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9\n #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22\n #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22\n #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31\n#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8\n#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8\n#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12\n#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5"}]]}
diff --git a/llvm/test/tools/llvm-original-di-preservation/basic.test b/llvm/test/tools/llvm-original-di-preservation/basic.test
index 81f987aa221b..5ef670b42c66 100644
--- a/llvm/test/tools/llvm-original-di-preservation/basic.test
+++ b/llvm/test/tools/llvm-original-di-preservation/basic.test
@@ -1,5 +1,6 @@
 RUN: %llvm-original-di-preservation %p/Inputs/sample.json %t.html | FileCheck %s
 RUN: diff -w %p/Inputs/expected-sample.html %t.html
+CHECK: The {{.+}}.html generated.
 CHECK-NOT: Skipped lines:
 
 RUN: %llvm-original-di-preservation %p/Inputs/corrupted.json %t2.html | FileCheck %s -check-prefix=CORRUPTED
@@ -9,5 +10,8 @@ CORRUPTED: Skipped bugs: 1
 
 RUN: %llvm-original-di-preservation -compress %p/Inputs/sample.json %t3.html | FileCheck %s -check-prefix=COMPRESSED
 RUN: diff -w %p/Inputs/expected-compressed.html %t3.html
+COMPRESSED: The {{.+}}.html generated.
 COMPRESSED-NOT: Skipped lines:
 
+RUN: %llvm-original-di-preservation %p/Inputs/origin.json %t4.html | FileCheck %s
+RUN: diff -w %p/Inputs/expected-origin.html %t4.html
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index ed1a80d3c1bc..c322b4f6c982 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -35,6 +35,7 @@
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
@@ -877,7 +878,8 @@ static void exitOnLazyCallThroughFailure() { exit(1); }
 Expected<orc::ThreadSafeModule>
 loadModule(StringRef Path, orc::ThreadSafeContext TSCtx) {
   SMDiagnostic Err;
-  auto M = parseIRFile(Path, Err, *TSCtx.getContext());
+  auto M = TSCtx.withContextDo(
+      [&](LLVMContext *Ctx) { return parseIRFile(Path, Err, *Ctx); });
   if (!M) {
     std::string ErrMsg;
     {
diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c
index 95b99bb17cd6..545cb79e1dd5 100644
--- a/llvm/tools/llvm-c-test/debuginfo.c
+++ b/llvm/tools/llvm-c-test/debuginfo.c
@@ -130,12 +130,12 @@ int llvm_test_dibuilder(void) {
   LLVMMetadataRef FooParamLocation =
     LLVMDIBuilderCreateDebugLocation(LLVMGetGlobalContext(), 42, 0,
                                      ReplaceableFunctionMetadata, NULL);
-  LLVMMetadataRef FunctionMetadata =
-    LLVMDIBuilderCreateFunction(DIB, File, "foo", 3, "foo", 3,
-                                File, 42, FunctionTy, true, true,
-                                42, 0, false);
+  LLVMMetadataRef FunctionMetadata = LLVMDIBuilderCreateFunction(
+      DIB, File, "foo", 3, "foo", 3, File, 42, NULL, true, true, 42, 0, false);
   LLVMMetadataReplaceAllUsesWith(ReplaceableFunctionMetadata, FunctionMetadata);
 
+  LLVMDISubprogramReplaceType(FunctionMetadata, FunctionTy);
+
   LLVMMetadataRef FooParamExpression =
     LLVMDIBuilderCreateExpression(DIB, NULL, 0);
   LLVMMetadataRef FooParamVar1 =
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index 4088c1b053aa..024693a24cc2 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -62,7 +62,7 @@
 #include <utility>
 
 /// The semantic version combined as a string.
-#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.1"
+#define LLVM_COVERAGE_EXPORT_JSON_STR "3.0.0"
 
 /// Unique type identifier for JSON coverage export.
 #define LLVM_COVERAGE_EXPORT_JSON_TYPE_STR "llvm.coverage.json.export"
@@ -110,8 +110,10 @@ json::Array gatherConditions(const coverage::MCDCRecord &Record) {
 
 json::Array renderMCDCRecord(const coverage::MCDCRecord &Record) {
   const llvm::coverage::CounterMappingRegion &CMR = Record.getDecisionRegion();
+  const auto [TrueDecisions, FalseDecisions] = Record.getDecisions();
   return json::Array({CMR.LineStart, CMR.ColumnStart, CMR.LineEnd,
-                      CMR.ColumnEnd, CMR.ExpandedFileID, int64_t(CMR.Kind),
+                      CMR.ColumnEnd, TrueDecisions, FalseDecisions,
+                      CMR.ExpandedFileID, int64_t(CMR.Kind),
                       gatherConditions(Record)});
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/Analysis.cpp b/llvm/tools/llvm-exegesis/lib/Analysis.cpp
index be10c32cf08d..fb843285ada2 100644
--- a/llvm/tools/llvm-exegesis/lib/Analysis.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Analysis.cpp
@@ -137,9 +137,9 @@ void Analysis::printInstructionRowCsv(const size_t PointId,
   std::tie(SchedClassId, std::ignore) = ResolvedSchedClass::resolveSchedClassId(
       State_.getSubtargetInfo(), State_.getInstrInfo(), MCI);
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  const MCSchedClassDesc *const SCDesc =
-      State_.getSubtargetInfo().getSchedModel().getSchedClassDesc(SchedClassId);
-  writeEscaped<kEscapeCsv>(OS, SCDesc->Name);
+  StringRef SCDescName =
+      State_.getSubtargetInfo().getSchedModel().getSchedClassName(SchedClassId);
+  writeEscaped<kEscapeCsv>(OS, SCDescName);
 #else
   OS << SchedClassId;
 #endif
@@ -563,7 +563,8 @@ Error Analysis::run<Analysis::PrintSchedClassInconsistencies>(
     OS << "<div class=\"inconsistency\"><p>Sched Class <span "
           "class=\"sched-class-name\">";
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-    writeEscaped<kEscapeHtml>(OS, RSCAndPoints.RSC.SCDesc->Name);
+    writeEscaped<kEscapeHtml>(OS, SI.getSchedModel().getSchedClassName(
+                                      RSCAndPoints.RSC.SchedClassId));
 #else
     OS << RSCAndPoints.RSC.SchedClassId;
 #endif
diff --git a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
index e0e796cee804..66c770d9ca86 100644
--- a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
+++ b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
@@ -12,6 +12,7 @@
 #include <tuple>
 
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/InterleavedRange.h"
 
 namespace llvm {
 namespace exegesis {
@@ -293,15 +294,8 @@ void Instruction::dump(const MCRegisterInfo &RegInfo,
   }
   for (const auto &Var : Variables) {
     Stream << "- Var" << Var.getIndex();
-    Stream << " [";
-    bool IsFirst = true;
-    for (auto OperandIndex : Var.TiedOperands) {
-      if (!IsFirst)
-        Stream << ",";
-      Stream << "Op" << OperandIndex;
-      IsFirst = false;
-    }
-    Stream << "]";
+    Stream << " ";
+    Stream << llvm::interleaved_array(Var.TiedOperands, ",");
     Stream << "\n";
   }
   if (hasMemoryOperands())
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index df356178b116..de8dc16f2eba 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -38,6 +38,7 @@
 #include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/ExecutionEngine/Orc/SectCreate.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h"
diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp
index 21953ee98d6a..05e9502e3abb 100644
--- a/llvm/tools/llvm-lto/llvm-lto.cpp
+++ b/llvm/tools/llvm-lto/llvm-lto.cpp
@@ -475,6 +475,8 @@ static void testLTOModule(const TargetOptions &Options) {
         printLTOSymbolAttributes(Module->getSymbolAttributes(I));
         outs() << "\n";
       }
+      for (int I = 0, E = Module->getAsmUndefSymbolCount(); I != E; ++I)
+        outs() << Module->getAsmUndefSymbolName(I) << "    { asm extern }\n";
     }
     if (QueryHasCtorDtor)
       outs() << Filename
diff --git a/llvm/tools/llvm-mc/Disassembler.cpp b/llvm/tools/llvm-mc/Disassembler.cpp
index 16897054fbea..607184e3b724 100644
--- a/llvm/tools/llvm-mc/Disassembler.cpp
+++ b/llvm/tools/llvm-mc/Disassembler.cpp
@@ -35,7 +35,7 @@ typedef std::pair<std::vector<unsigned char>, std::vector<const char *>>
 static bool PrintInsts(const MCDisassembler &DisAsm, const ByteArrayTy &Bytes,
                        SourceMgr &SM, MCStreamer &Streamer, bool InAtomicBlock,
                        const MCSubtargetInfo &STI) {
-  ArrayRef<uint8_t> Data(Bytes.first.data(), Bytes.first.size());
+  ArrayRef<uint8_t> Data(Bytes.first);
 
   // Disassemble it to strings.
   uint64_t Size;
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 7823b02298e2..e7d8083da19d 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -41,8 +41,7 @@ PressureTracker::PressureTracker(const MCSchedModel &Model)
   }
 
   ResourceUsers.resize(NextResourceUsersIdx);
-  std::fill(ResourceUsers.begin(), ResourceUsers.end(),
-            std::make_pair<unsigned, unsigned>(~0U, 0U));
+  llvm::fill(ResourceUsers, std::make_pair<unsigned, unsigned>(~0U, 0U));
 }
 
 void PressureTracker::getResourceUsers(uint64_t ResourceMask,
diff --git a/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp b/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
index 4ef8053bff41..54566dc860c5 100644
--- a/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
+++ b/llvm/tools/llvm-mca/Views/RegisterFileStatistics.cpp
@@ -38,10 +38,10 @@ RegisterFileStatistics::RegisterFileStatistics(const MCSubtargetInfo &sti)
   unsigned NumRegFiles = std::max(PI.NumRegisterFiles, 1U);
 
   PRFUsage.resize(NumRegFiles);
-  std::fill(PRFUsage.begin(), PRFUsage.end(), RFUEmpty);
+  llvm::fill(PRFUsage, RFUEmpty);
 
   MoveElimInfo.resize(NumRegFiles);
-  std::fill(MoveElimInfo.begin(), MoveElimInfo.end(), MEIEmpty);
+  llvm::fill(MoveElimInfo, MEIEmpty);
 }
 
 void RegisterFileStatistics::updateRegisterFileUsage(
diff --git a/llvm/tools/llvm-mca/Views/TimelineView.cpp b/llvm/tools/llvm-mca/Views/TimelineView.cpp
index 2eca48aadfd7..193bceca190d 100644
--- a/llvm/tools/llvm-mca/Views/TimelineView.cpp
+++ b/llvm/tools/llvm-mca/Views/TimelineView.cpp
@@ -28,14 +28,14 @@ TimelineView::TimelineView(const MCSubtargetInfo &sti, MCInstPrinter &Printer,
   NumInstructions *= Iterations;
   Timeline.resize(NumInstructions);
   TimelineViewEntry InvalidTVEntry = {-1, 0, 0, 0, 0};
-  std::fill(Timeline.begin(), Timeline.end(), InvalidTVEntry);
+  llvm::fill(Timeline, InvalidTVEntry);
 
   WaitTimeEntry NullWTEntry = {0, 0, 0};
-  std::fill(WaitTime.begin(), WaitTime.end(), NullWTEntry);
+  llvm::fill(WaitTime, NullWTEntry);
 
   std::pair<unsigned, int> NullUsedBufferEntry = {/* Invalid resource ID*/ 0,
                                                   /* unknown buffer size */ -1};
-  std::fill(UsedBuffer.begin(), UsedBuffer.end(), NullUsedBufferEntry);
+  llvm::fill(UsedBuffer, NullUsedBufferEntry);
 }
 
 void TimelineView::onReservedBuffers(const InstRef &IR,
diff --git a/llvm/tools/llvm-ml/Disassembler.cpp b/llvm/tools/llvm-ml/Disassembler.cpp
index 71063deeeec4..e55561f3b7f8 100644
--- a/llvm/tools/llvm-ml/Disassembler.cpp
+++ b/llvm/tools/llvm-ml/Disassembler.cpp
@@ -33,7 +33,7 @@ typedef std::pair<std::vector<unsigned char>, std::vector<const char *>>
 static bool PrintInsts(const MCDisassembler &DisAsm, const ByteArrayTy &Bytes,
                        SourceMgr &SM, raw_ostream &Out, MCStreamer &Streamer,
                        bool InAtomicBlock, const MCSubtargetInfo &STI) {
-  ArrayRef<uint8_t> Data(Bytes.first.data(), Bytes.first.size());
+  ArrayRef<uint8_t> Data(Bytes.first);
 
   // Disassemble it to strings.
   uint64_t Size;
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 13e38111edf0..f19b448333fa 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -1496,8 +1496,9 @@ collectLocalBranchTargets(ArrayRef<uint8_t> Bytes, MCInstrAnalysis *MIA,
   // Supported by certain targets.
   const bool isPPC = STI->getTargetTriple().isPPC();
   const bool isX86 = STI->getTargetTriple().isX86();
+  const bool isAArch64 = STI->getTargetTriple().isAArch64();
   const bool isBPF = STI->getTargetTriple().isBPF();
-  if (!isPPC && !isX86 && !isBPF)
+  if (!isPPC && !isX86 && !isAArch64 && !isBPF)
     return;
 
   if (MIA)
diff --git a/llvm/tools/llvm-pdbutil/MinimalSymbolDumper.cpp b/llvm/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
index 479d02583518..66a091f50d6b 100644
--- a/llvm/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
+++ b/llvm/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
@@ -955,3 +955,11 @@ Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR,
       JumpTable.EntriesCount);
   return Error::success();
 }
+
+Error MinimalSymbolDumper::visitKnownRecord(CVSymbol &CVR,
+                                            HotPatchFuncSym &JumpTable) {
+  AutoIndent Indent(P, 7);
+  P.formatLine("function = {0}, name = {1}", typeIndex(JumpTable.Function),
+               JumpTable.Name);
+  return Error::success();
+}
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index a7cbd4bfc838..45eac90aef93 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -1705,7 +1705,10 @@ static WeightedFile parseWeightedFile(const StringRef &WeightedFilename) {
   if (WeightStr.getAsInteger(10, Weight) || Weight < 1)
     exitWithError("input weight must be a positive integer");
 
-  return {std::string(FileName), Weight};
+  llvm::SmallString<128> ResolvedFileName;
+  llvm::sys::fs::expand_tilde(FileName, ResolvedFileName);
+
+  return {std::string(ResolvedFileName), Weight};
 }
 
 static void addWeightedInput(WeightedFileVector &WNI, const WeightedFile &WF) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index b47c77c5f2ff..db686c3b597e 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -108,8 +108,6 @@ using namespace llvm;
 using namespace sampleprof;
 
 namespace llvm {
-extern cl::opt<int> ProfileSummaryCutoffHot;
-extern cl::opt<bool> UseContextLessSummary;
 
 namespace sampleprof {
 
diff --git a/llvm/tools/llvm-remarkutil/RemarkCounter.cpp b/llvm/tools/llvm-remarkutil/RemarkCounter.cpp
index 0e6198cbb520..7d5c84815b3b 100644
--- a/llvm/tools/llvm-remarkutil/RemarkCounter.cpp
+++ b/llvm/tools/llvm-remarkutil/RemarkCounter.cpp
@@ -13,6 +13,7 @@
 #include "RemarkCounter.h"
 #include "RemarkUtilRegistry.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/Regex.h"
 
 using namespace llvm;
@@ -197,23 +198,11 @@ Error ArgumentCounter::print(StringRef OutputFileName) {
 
   auto OF = std::move(*MaybeOF);
   OF->os() << groupByToStr(Group) << ",";
-  unsigned Idx = 0;
-  for (auto [Key, _] : ArgumentSetIdxMap) {
-    OF->os() << Key;
-    if (Idx != ArgumentSetIdxMap.size() - 1)
-      OF->os() << ",";
-    Idx++;
-  }
+  OF->os() << llvm::interleaved(llvm::make_first_range(ArgumentSetIdxMap), ",");
   OF->os() << "\n";
   for (auto [Header, CountVector] : CountByKeysMap) {
     OF->os() << Header << ",";
-    unsigned Idx = 0;
-    for (auto Count : CountVector) {
-      OF->os() << Count;
-      if (Idx != ArgumentSetIdxMap.size() - 1)
-        OF->os() << ",";
-      Idx++;
-    }
+    OF->os() << llvm::interleaved(CountVector, ",");
     OF->os() << "\n";
   }
   return Error::success();
diff --git a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
index 40a7b86078d8..aa44f35a2d9c 100644
--- a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -921,7 +921,7 @@ static int linkAndVerify() {
     RuntimeDyldChecker::MemoryRegionInfo SecInfo;
     SecInfo.setTargetAddress(Dyld.getSectionLoadAddress(*SectionID));
     StringRef SecContent = Dyld.getSectionContent(*SectionID);
-    SecInfo.setContent(ArrayRef<char>(SecContent.data(), SecContent.size()));
+    SecInfo.setContent(ArrayRef<char>(SecContent));
     return SecInfo;
   };
 
@@ -945,8 +945,7 @@ static int linkAndVerify() {
                                  SI.Offset);
     StringRef SecContent =
         Dyld.getSectionContent(SI.SectionID).substr(SI.Offset);
-    StubMemInfo.setContent(
-        ArrayRef<char>(SecContent.data(), SecContent.size()));
+    StubMemInfo.setContent(ArrayRef<char>(SecContent));
     return StubMemInfo;
   };
 
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index bb64d42ccced..467a4da27dcd 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -322,6 +322,15 @@ lto_symbol_attributes lto_module_get_symbol_attribute(lto_module_t mod,
   return unwrap(mod)->getSymbolAttributes(index);
 }
 
+unsigned int lto_module_get_num_asm_undef_symbols(lto_module_t mod) {
+  return unwrap(mod)->getAsmUndefSymbolCount();
+}
+
+const char *lto_module_get_asm_undef_symbol_name(lto_module_t mod,
+                                                 unsigned int index) {
+  return unwrap(mod)->getAsmUndefSymbolName(index).data();
+}
+
 const char* lto_module_get_linkeropts(lto_module_t mod) {
   return unwrap(mod)->getLinkerOpts().data();
 }
diff --git a/llvm/tools/lto/lto.exports b/llvm/tools/lto/lto.exports
index 4164c3919a97..850e159b725a 100644
--- a/llvm/tools/lto/lto.exports
+++ b/llvm/tools/lto/lto.exports
@@ -14,6 +14,8 @@ lto_module_get_macho_cputype
 lto_module_get_num_symbols
 lto_module_get_symbol_attribute
 lto_module_get_symbol_name
+lto_module_get_num_asm_undef_symbols
+lto_module_get_asm_undef_symbol_name
 lto_module_get_target_triple
 lto_module_set_target_triple
 lto_module_is_object_file
diff --git a/llvm/unittests/ADT/ArrayRefTest.cpp b/llvm/unittests/ADT/ArrayRefTest.cpp
index 39a4a9b6a178..b5fc0a100571 100644
--- a/llvm/unittests/ADT/ArrayRefTest.cpp
+++ b/llvm/unittests/ADT/ArrayRefTest.cpp
@@ -8,10 +8,16 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Allocator.h"
-#include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 #include <limits>
 #include <vector>
+#if __has_include(<version>)
+#include <version>
+#endif
+#ifdef __cpp_lib_span
+#include <span>
+#endif
+
 using namespace llvm;
 
 // Check that the ArrayRef-of-pointer converting constructor only allows adding
@@ -98,6 +104,48 @@ TEST(ArrayRefTest, DropFront) {
   EXPECT_TRUE(AR1.drop_front(2).equals(AR2));
 }
 
+TEST(ArrayRefTest, ConsumeFront) {
+  static const int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  ArrayRef<int> AR1(TheNumbers);
+  ArrayRef<int> AR2(&TheNumbers[2], AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[0]);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[1]);
+  EXPECT_TRUE(AR1.equals(AR2));
+}
+
+TEST(ArrayRefTest, ConsumeBack) {
+  static const int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  ArrayRef<int> AR1(TheNumbers);
+  ArrayRef<int> AR2(TheNumbers, AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[5]);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[4]);
+  EXPECT_TRUE(AR1.equals(AR2));
+}
+
+TEST(ArrayRefTest, MutableArryaRefConsumeFront) {
+  int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  MutableArrayRef<int> AR1(TheNumbers);
+  MutableArrayRef<int> AR2(&TheNumbers[2], AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[0]);
+  EXPECT_EQ(&AR1.consume_front(), &TheNumbers[1]);
+  EXPECT_TRUE(AR1.equals(AR2));
+
+  AR1.consume_front() = 33;
+  EXPECT_EQ(TheNumbers[2], 33);
+}
+
+TEST(ArrayRefTest, MutableArryaRefConsumeBack) {
+  int TheNumbers[] = {4, 8, 15, 16, 23, 42};
+  MutableArrayRef<int> AR1(TheNumbers);
+  MutableArrayRef<int> AR2(TheNumbers, AR1.size() - 2);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[5]);
+  EXPECT_EQ(&AR1.consume_back(), &TheNumbers[4]);
+  EXPECT_TRUE(AR1.equals(AR2));
+
+  AR1.consume_back() = 33;
+  EXPECT_EQ(TheNumbers[3], 33);
+}
+
 TEST(ArrayRefTest, DropWhile) {
   static const int TheNumbers[] = {1, 3, 5, 8, 10, 11};
   ArrayRef<int> AR1(TheNumbers);
@@ -406,4 +454,25 @@ TEST(ArrayRefTest, MutableArrayRefDeductionGuides) {
   }
 }
 
+#ifdef __cpp_lib_span
+static_assert(std::is_constructible_v<ArrayRef<int>, std::span<const int>>,
+              "should be able to construct ArrayRef from const std::span");
+static_assert(std::is_constructible_v<std::span<const int>, ArrayRef<int>>,
+              "should be able to construct const std::span from ArrayRef");
+static_assert(std::is_constructible_v<ArrayRef<int>, std::span<int>>,
+              "should be able to construct ArrayRef from mutable std::span");
+static_assert(!std::is_constructible_v<std::span<int>, ArrayRef<int>>,
+              "cannot construct mutable std::span from ArrayRef");
+
+static_assert(
+    !std::is_constructible_v<MutableArrayRef<int>, std::span<const int>>,
+    "cannot construct MutableArrayRef from const std::span");
+static_assert(
+    std::is_constructible_v<std::span<const int>, MutableArrayRef<int>>,
+    "should be able to construct const std::span from MutableArrayRef");
+static_assert(
+    std::is_constructible_v<MutableArrayRef<int>, std::span<int>>,
+    "should be able to construct MutableArrayRef from mutable std::span");
+#endif
+
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/DynamicAPIntTest.cpp b/llvm/unittests/ADT/DynamicAPIntTest.cpp
index 932b750608b3..fafd8fda0868 100644
--- a/llvm/unittests/ADT/DynamicAPIntTest.cpp
+++ b/llvm/unittests/ADT/DynamicAPIntTest.cpp
@@ -31,6 +31,16 @@ class TypeNames {
 };
 TYPED_TEST_SUITE(IntTest, TypeList, TypeNames);
 
+TYPED_TEST(IntTest, ValueInit) {
+  APInt Large(65, 0, true);
+  Large.setBit(64);
+  TypeParam DynLarge(1ll << 63);
+  EXPECT_EQ(TypeParam(Large), DynLarge + DynLarge);
+  APInt Small(64, -1, true);
+  TypeParam DynSmall(Small.getSExtValue());
+  EXPECT_EQ(TypeParam(Small), DynSmall);
+}
+
 TYPED_TEST(IntTest, ops) {
   TypeParam Two(2), Five(5), Seven(7), Ten(10);
   EXPECT_EQ(Five + Five, Ten);
diff --git a/llvm/unittests/ADT/STLExtrasTest.cpp b/llvm/unittests/ADT/STLExtrasTest.cpp
index 286cfa745fd1..9fda0d912a2f 100644
--- a/llvm/unittests/ADT/STLExtrasTest.cpp
+++ b/llvm/unittests/ADT/STLExtrasTest.cpp
@@ -1591,6 +1591,17 @@ TEST(STLExtrasTest, Includes) {
   }
 }
 
+TEST(STLExtrasTest, Fill) {
+  std::vector<int> V1 = {1, 2, 3};
+  std::vector<int> V2;
+  int Val = 4;
+  fill(V1, Val);
+  EXPECT_THAT(V1, ElementsAre(Val, Val, Val));
+  V2.resize(4);
+  fill(V2, Val);
+  EXPECT_THAT(V2, ElementsAre(Val, Val, Val, Val));
+}
+
 struct Foo;
 struct Bar {};
 
diff --git a/llvm/unittests/ADT/STLForwardCompatTest.cpp b/llvm/unittests/ADT/STLForwardCompatTest.cpp
index b856386aa3e4..e3d500aa7b55 100644
--- a/llvm/unittests/ADT/STLForwardCompatTest.cpp
+++ b/llvm/unittests/ADT/STLForwardCompatTest.cpp
@@ -45,6 +45,25 @@ TYPED_TEST(STLForwardCompatRemoveCVRefTest, RemoveCVRefT) {
                             llvm::remove_cvref_t<From>>::value));
 }
 
+template <typename T> class TypeIdentityTest : public ::testing::Test {
+public:
+  using TypeIdentity = llvm::type_identity<T>;
+};
+
+struct A {
+  struct B {};
+};
+using TypeIdentityTestTypes =
+    ::testing::Types<int, volatile int, A, const A::B>;
+
+TYPED_TEST_SUITE(TypeIdentityTest, TypeIdentityTestTypes, /*NameGenerator*/);
+
+TYPED_TEST(TypeIdentityTest, Identity) {
+  // TestFixture is the instantiated TypeIdentityTest.
+  EXPECT_TRUE(
+      (std::is_same_v<TypeParam, typename TestFixture::TypeIdentity::type>));
+}
+
 TEST(TransformTest, TransformStd) {
   std::optional<int> A;
 
diff --git a/llvm/unittests/Analysis/AliasAnalysisTest.cpp b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
index 5d990521d483..06066b1b92c5 100644
--- a/llvm/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/AliasAnalysisTest.cpp
@@ -147,7 +147,8 @@ class AliasAnalysisTest : public testing::Test {
   std::unique_ptr<BasicAAResult> BAR;
   std::unique_ptr<AAResults> AAR;
 
-  AliasAnalysisTest() : M("AliasAnalysisTest", C), TLI(TLII) {}
+  AliasAnalysisTest()
+      : M("AliasAnalysisTest", C), TLII(M.getTargetTriple()), TLI(TLII) {}
 
   AAResults &getAAResults(Function &F) {
     // Reset the Function AA results first to clear out any references.
diff --git a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
index f5c73ff63934..bae1f1c508af 100644
--- a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
@@ -66,7 +66,8 @@ class BasicAATest : public testing::Test {
 
 public:
   BasicAATest()
-      : M("BasicAATest", C), B(C), DL(DLString), TLI(TLII), F(nullptr) {}
+      : M("BasicAATest", C), B(C), DL(DLString), TLII(M.getTargetTriple()),
+        TLI(TLII), F(nullptr) {}
 };
 
 // Check that a function arg can't trivially alias a global when we're accessing
diff --git a/llvm/unittests/Analysis/DDGTest.cpp b/llvm/unittests/Analysis/DDGTest.cpp
index da6f53fb0141..7fcdfdb62da4 100644
--- a/llvm/unittests/Analysis/DDGTest.cpp
+++ b/llvm/unittests/Analysis/DDGTest.cpp
@@ -29,7 +29,7 @@ static void runTest(Module &M, StringRef FuncName,
   auto *F = M.getFunction(FuncName);
   ASSERT_NE(F, nullptr) << "Could not find " << FuncName;
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
diff --git a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
index abc16e1aff41..7b00bd03cace 100644
--- a/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/FunctionPropertiesAnalysisTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/FunctionPropertiesAnalysis.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/IR2Vec.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Dominators.h"
@@ -20,15 +21,20 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Transforms/Utils/Cloning.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <cstring>
 
 using namespace llvm;
+using namespace testing;
 
 namespace llvm {
 LLVM_ABI extern cl::opt<bool> EnableDetailedFunctionProperties;
 LLVM_ABI extern cl::opt<bool> BigBasicBlockInstructionThreshold;
 LLVM_ABI extern cl::opt<bool> MediumBasicBlockInstrutionThreshold;
+LLVM_ABI extern cl::opt<float> ir2vec::OpcWeight;
+LLVM_ABI extern cl::opt<float> ir2vec::TypeWeight;
+LLVM_ABI extern cl::opt<float> ir2vec::ArgWeight;
 } // namespace llvm
 
 namespace {
@@ -36,17 +42,64 @@ namespace {
 class FunctionPropertiesAnalysisTest : public testing::Test {
 public:
   FunctionPropertiesAnalysisTest() {
+    createTestVocabulary(1);
+    MAM.registerPass([&] { return IR2VecVocabAnalysis(Vocabulary); });
+    MAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+    FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
     FAM.registerPass([&] { return DominatorTreeAnalysis(); });
     FAM.registerPass([&] { return LoopAnalysis(); });
     FAM.registerPass([&] { return PassInstrumentationAnalysis(); });
+
+    ir2vec::OpcWeight = 1.0;
+    ir2vec::TypeWeight = 1.0;
+    ir2vec::ArgWeight = 1.0;
+  }
+
+private:
+  float OriginalOpcWeight = ir2vec::OpcWeight;
+  float OriginalTypeWeight = ir2vec::TypeWeight;
+  float OriginalArgWeight = ir2vec::ArgWeight;
+
+  void createTestVocabulary(unsigned Dim) {
+    llvm::SmallVector<std::string, 32> SampleEntities = {
+        "add",        "sub",      "mul",        "icmp",          "br",
+        "ret",        "store",    "load",       "alloca",        "phi",
+        "call",       "voidTy",   "floatTy",    "integerTy",     "functionTy",
+        "structTy",   "arrayTy",  "pointerTy",  "vectorTy",      "emptyTy",
+        "labelTy",    "tokenTy",  "metadataTy", "unknownTy",     "function",
+        "pointer",    "constant", "variable",   "getelementptr", "invoke",
+        "landingpad", "resume",   "catch",      "cleanup"};
+    float EmbVal = 0.1f;
+
+    // Helper lambda to add entries to the vocabulary
+    auto addEntry = [&](std::string key) {
+      Vocabulary[key] = ir2vec::Embedding(Dim, EmbVal);
+      EmbVal += 0.1f;
+    };
+
+    for (auto &Name : SampleEntities)
+      addEntry(Name);
   }
 
 protected:
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<LoopInfo> LI;
   FunctionAnalysisManager FAM;
+  ModuleAnalysisManager MAM;
+  ir2vec::Vocab Vocabulary;
+
+  void TearDown() override {
+    // Restore original IR2Vec weights
+    ir2vec::OpcWeight = OriginalOpcWeight;
+    ir2vec::TypeWeight = OriginalTypeWeight;
+    ir2vec::ArgWeight = OriginalArgWeight;
+  }
 
   FunctionPropertiesInfo buildFPI(Function &F) {
+    // FunctionPropertiesInfo assumes IR2VecVocabAnalysis has been run to
+    // use IR2Vec.
+    auto VocabResult = MAM.getResult<IR2VecVocabAnalysis>(*F.getParent());
+    (void)VocabResult;
     return FunctionPropertiesInfo::getFunctionPropertiesInfo(F, FAM);
   }
 
@@ -71,6 +124,12 @@ class FunctionPropertiesAnalysisTest : public testing::Test {
             return CB;
     return nullptr;
   }
+
+  std::unique_ptr<ir2vec::Embedder> createEmbedder(const Function &F) {
+    auto Emb = ir2vec::Embedder::create(IR2VecKind::Symbolic, F, Vocabulary);
+    EXPECT_TRUE(static_cast<bool>(Emb));
+    return Emb;
+  }
 };
 
 TEST_F(FunctionPropertiesAnalysisTest, BasicTest) {
@@ -113,6 +172,8 @@ define internal i32 @top() {
   EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
   EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
   EXPECT_EQ(BranchesFeatures.TopLevelLoopCount, 0);
+  EXPECT_TRUE(BranchesFeatures.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*BranchesFunction)->getFunctionVector()));
 
   Function *TopFunction = M->getFunction("top");
   FunctionPropertiesInfo TopFeatures = buildFPI(*TopFunction);
@@ -120,6 +181,8 @@ define internal i32 @top() {
   EXPECT_EQ(TopFeatures.BlocksReachedFromConditionalInstruction, 0);
   EXPECT_EQ(TopFeatures.Uses, 0);
   EXPECT_EQ(TopFeatures.DirectCallsToDefinedFunctions, 1);
+  EXPECT_TRUE(TopFeatures.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*TopFunction)->getFunctionVector()));
   EXPECT_EQ(BranchesFeatures.LoadInstCount, 0);
   EXPECT_EQ(BranchesFeatures.StoreInstCount, 0);
   EXPECT_EQ(BranchesFeatures.MaxLoopDepth, 0);
@@ -159,6 +222,9 @@ define internal i32 @top() {
   EXPECT_EQ(DetailedBranchesFeatures.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedBranchesFeatures.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedBranchesFeatures.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(
+      DetailedBranchesFeatures.getFunctionEmbedding().approximatelyEquals(
+          createEmbedder(*BranchesFunction)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -210,6 +276,8 @@ define i64 @f1() {
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -240,6 +308,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 3;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.DirectCallsToDefinedFunctions = 0;
@@ -252,6 +322,9 @@ define i32 @f2(i32 %a) {
   auto IR = llvm::InlineFunction(*CB, IFI);
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
+
   EXPECT_TRUE(FPU.finishAndTest(FAM));
   EXPECT_EQ(FPI, ExpectedFinal);
 }
@@ -294,6 +367,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 9;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.DirectCallsToDefinedFunctions = 0;
@@ -307,6 +382,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -356,6 +434,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.TotalInstructionCount = 9;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal;
   ExpectedFinal.BasicBlockCount = 6;
@@ -374,6 +454,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -422,6 +505,8 @@ declare i32 @__gxx_personality_v0(...)
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size());
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount());
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
 }
 
 TEST_F(FunctionPropertiesAnalysisTest, InvokeUnreachableHandler) {
@@ -475,6 +560,8 @@ declare i32 @__gxx_personality_v0(...)
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -581,6 +668,8 @@ define void @outer() personality i8* null {
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -637,6 +726,8 @@ if.then:
   EXPECT_EQ(static_cast<size_t>(FPI.BasicBlockCount), F1->size() - 1);
   EXPECT_EQ(static_cast<size_t>(FPI.TotalInstructionCount),
             F1->getInstructionCount() - 2);
+  EXPECT_TRUE(FPI.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EXPECT_EQ(FPI, FunctionPropertiesInfo::getFunctionPropertiesInfo(*F1, FAM));
 }
 
@@ -689,6 +780,8 @@ define i32 @f2(i32 %a) {
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
   ExpectedInitial.MaxLoopDepth = 1;
   ExpectedInitial.TopLevelLoopCount = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 6;
@@ -705,6 +798,9 @@ define i32 @f2(i32 %a) {
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -733,7 +829,7 @@ cond.false:                                       ; preds = %entry
 extra2:
   br label %cond.end
 
-cond.end:                                         ; preds = %cond.false, %cond.true
+cond.end:                                         ; preds = %extra2, %cond.true
   %cond = phi i64 [ %conv2, %cond.true ], [ %call3, %extra ]
   ret i64 %cond
 }
@@ -757,6 +853,8 @@ declare void @llvm.trap()
   ExpectedInitial.BlocksReachedFromConditionalInstruction = 2;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 4;
@@ -772,6 +870,9 @@ declare void @llvm.trap()
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -817,6 +918,8 @@ declare void @f3()
   ExpectedInitial.BlocksReachedFromConditionalInstruction = 0;
   ExpectedInitial.Uses = 1;
   ExpectedInitial.DirectCallsToDefinedFunctions = 1;
+  ExpectedInitial.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
 
   FunctionPropertiesInfo ExpectedFinal = ExpectedInitial;
   ExpectedFinal.BasicBlockCount = 6;
@@ -832,6 +935,9 @@ declare void @f3()
   EXPECT_TRUE(IR.isSuccess());
   invalidate(*F1);
   EXPECT_TRUE(FPU.finishAndTest(FAM));
+
+  ExpectedFinal.setFunctionEmbeddingForTest(
+      createEmbedder(*F1)->getFunctionVector());
   EXPECT_EQ(FPI, ExpectedFinal);
 }
 
@@ -885,6 +991,8 @@ define i64 @f1(i64 %e) {
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -910,6 +1018,8 @@ declare float @llvm.cos.f32(float)
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 0);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 0);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -943,6 +1053,8 @@ declare float @f5()
   EXPECT_EQ(DetailedF1Properties.CallReturnsPointerCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallWithManyArgumentsCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallWithPointerArgumentCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -972,6 +1084,8 @@ define i64 @f1(i64 %a) {
   EnableDetailedFunctionProperties.setValue(true);
   FunctionPropertiesInfo DetailedF1Properties = buildFPI(*F1);
   EXPECT_EQ(DetailedF1Properties.CriticalEdgeCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
@@ -997,6 +1111,8 @@ declare <4 x ptr> @f4()
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorIntCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorFloatCount, 1);
   EXPECT_EQ(DetailedF1Properties.CallReturnsVectorPointerCount, 1);
+  EXPECT_TRUE(DetailedF1Properties.getFunctionEmbedding().approximatelyEquals(
+      createEmbedder(*F1)->getFunctionVector()));
   EnableDetailedFunctionProperties.setValue(false);
 }
 
diff --git a/llvm/unittests/Analysis/IR2VecTest.cpp b/llvm/unittests/Analysis/IR2VecTest.cpp
index 90d07d080443..70d4808dc6d5 100644
--- a/llvm/unittests/Analysis/IR2VecTest.cpp
+++ b/llvm/unittests/Analysis/IR2VecTest.cpp
@@ -109,6 +109,18 @@ TEST(EmbeddingTest, ConstructorsAndAccessors) {
   }
 }
 
+TEST(EmbeddingTest, AddVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {0.5, 1.5, -1.0};
+
+  Embedding E3 = E1 + E2;
+  EXPECT_THAT(E3, ElementsAre(1.5, 3.5, 2.0));
+
+  // Check that E1 and E2 are unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
+}
+
 TEST(EmbeddingTest, AddVectors) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {0.5, 1.5, -1.0};
@@ -120,6 +132,18 @@ TEST(EmbeddingTest, AddVectors) {
   EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
+TEST(EmbeddingTest, SubtractVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = {0.5, 1.5, -1.0};
+
+  Embedding E3 = E1 - E2;
+  EXPECT_THAT(E3, ElementsAre(0.5, 0.5, 4.0));
+
+  // Check that E1 and E2 are unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
+}
+
 TEST(EmbeddingTest, SubtractVectors) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {0.5, 1.5, -1.0};
@@ -131,6 +155,21 @@ TEST(EmbeddingTest, SubtractVectors) {
   EXPECT_THAT(E2, ElementsAre(0.5, 1.5, -1.0));
 }
 
+TEST(EmbeddingTest, ScaleVector) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  E1 *= 0.5f;
+  EXPECT_THAT(E1, ElementsAre(0.5, 1.0, 1.5));
+}
+
+TEST(EmbeddingTest, ScaleVectorOutOfPlace) {
+  Embedding E1 = {1.0, 2.0, 3.0};
+  Embedding E2 = E1 * 0.5f;
+  EXPECT_THAT(E2, ElementsAre(0.5, 1.0, 1.5));
+
+  // Check that E1 is unchanged
+  EXPECT_THAT(E1, ElementsAre(1.0, 2.0, 3.0));
+}
+
 TEST(EmbeddingTest, AddScaledVector) {
   Embedding E1 = {1.0, 2.0, 3.0};
   Embedding E2 = {2.0, 0.5, -1.0};
@@ -148,14 +187,14 @@ TEST(EmbeddingTest, ApproximatelyEqual) {
   EXPECT_TRUE(E1.approximatelyEquals(E2)); // Diff = 1e-7
 
   Embedding E3 = {1.00002, 2.00002, 3.00002}; // Diff = 2e-5
-  EXPECT_FALSE(E1.approximatelyEquals(E3));
+  EXPECT_FALSE(E1.approximatelyEquals(E3, 1e-6));
   EXPECT_TRUE(E1.approximatelyEquals(E3, 3e-5));
 
   Embedding E_clearly_within = {1.0000005, 2.0000005, 3.0000005}; // Diff = 5e-7
   EXPECT_TRUE(E1.approximatelyEquals(E_clearly_within));
 
   Embedding E_clearly_outside = {1.00001, 2.00001, 3.00001}; // Diff = 1e-5
-  EXPECT_FALSE(E1.approximatelyEquals(E_clearly_outside));
+  EXPECT_FALSE(E1.approximatelyEquals(E_clearly_outside, 1e-6));
 
   Embedding E4 = {1.0, 2.0, 3.5}; // Large diff
   EXPECT_FALSE(E1.approximatelyEquals(E4, 0.01));
@@ -174,6 +213,12 @@ TEST(EmbeddingTest, AccessOutOfBounds) {
   EXPECT_DEATH(E[4] = 4.0, "Index out of bounds");
 }
 
+TEST(EmbeddingTest, MismatchedDimensionsAddVectorsOutOfPlace) {
+  Embedding E1 = {1.0, 2.0};
+  Embedding E2 = {1.0};
+  EXPECT_DEATH(E1 + E2, "Vectors must have the same dimension");
+}
+
 TEST(EmbeddingTest, MismatchedDimensionsAddVectors) {
   Embedding E1 = {1.0, 2.0};
   Embedding E2 = {1.0};
@@ -210,10 +255,7 @@ TEST(IR2VecTest, CreateSymbolicEmbedder) {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx), false);
   Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
 
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  EXPECT_TRUE(static_cast<bool>(Result));
-
-  auto *Emb = Result->get();
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
   EXPECT_NE(Emb, nullptr);
 }
 
@@ -228,12 +270,6 @@ TEST(IR2VecTest, CreateInvalidMode) {
   // static_cast an invalid int to IR2VecKind
   auto Result = Embedder::create(static_cast<IR2VecKind>(-1), *F, V);
   EXPECT_FALSE(static_cast<bool>(Result));
-
-  std::string ErrMsg;
-  llvm::handleAllErrors(
-      Result.takeError(),
-      [&](const llvm::ErrorInfoBase &EIB) { ErrMsg = EIB.message(); });
-  EXPECT_NE(ErrMsg.find("Unknown IR2VecKind"), std::string::npos);
 }
 
 TEST(IR2VecTest, LookupVocab) {
@@ -281,7 +317,7 @@ TEST(IR2VecTest, IR2VecVocabResultValidity) {
   EXPECT_EQ(validResult.getDimension(), 2u);
 }
 
-// Fixture for IR2Vec tests requiring IR setup and weight management.
+// Fixture for IR2Vec tests requiring IR setup.
 class IR2VecTestFixture : public ::testing::Test {
 protected:
   Vocab V;
@@ -292,14 +328,10 @@ class IR2VecTestFixture : public ::testing::Test {
   Instruction *AddInst = nullptr;
   Instruction *RetInst = nullptr;
 
-  float OriginalOpcWeight = ::OpcWeight;
-  float OriginalTypeWeight = ::TypeWeight;
-  float OriginalArgWeight = ::ArgWeight;
-
   void SetUp() override {
     V = {{"add", {1.0, 2.0}},
-         {"integerTy", {0.5, 0.5}},
-         {"constant", {0.2, 0.3}},
+         {"integerTy", {0.25, 0.25}},
+         {"constant", {0.04, 0.06}},
          {"variable", {0.0, 0.0}},
          {"unknownTy", {0.0, 0.0}}};
 
@@ -316,25 +348,11 @@ class IR2VecTestFixture : public ::testing::Test {
     AddInst = BinaryOperator::CreateAdd(Arg, Const, "add", BB);
     RetInst = ReturnInst::Create(Ctx, AddInst, BB);
   }
-
-  void setWeights(float OpcWeight, float TypeWeight, float ArgWeight) {
-    ::OpcWeight = OpcWeight;
-    ::TypeWeight = TypeWeight;
-    ::ArgWeight = ArgWeight;
-  }
-
-  void TearDown() override {
-    // Restore original global weights
-    ::OpcWeight = OriginalOpcWeight;
-    ::TypeWeight = OriginalTypeWeight;
-    ::ArgWeight = OriginalArgWeight;
-  }
 };
 
 TEST_F(IR2VecTestFixture, GetInstVecMap) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &InstMap = Emb->getInstVecMap();
 
@@ -355,9 +373,8 @@ TEST_F(IR2VecTestFixture, GetInstVecMap) {
 }
 
 TEST_F(IR2VecTestFixture, GetBBVecMap) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &BBMap = Emb->getBBVecMap();
 
@@ -372,9 +389,8 @@ TEST_F(IR2VecTestFixture, GetBBVecMap) {
 }
 
 TEST_F(IR2VecTestFixture, GetBBVector) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &BBVec = Emb->getBBVector(*BB);
 
@@ -384,9 +400,8 @@ TEST_F(IR2VecTestFixture, GetBBVector) {
 }
 
 TEST_F(IR2VecTestFixture, GetFunctionVector) {
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
+  auto Emb = Embedder::create(IR2VecKind::Symbolic, *F, V);
+  ASSERT_TRUE(static_cast<bool>(Emb));
 
   const auto &FuncVec = Emb->getFunctionVector();
 
@@ -397,23 +412,6 @@ TEST_F(IR2VecTestFixture, GetFunctionVector) {
               ElementsAre(DoubleNear(1.29, 1e-6), DoubleNear(2.31, 1e-6)));
 }
 
-TEST_F(IR2VecTestFixture, GetFunctionVectorWithCustomWeights) {
-  setWeights(1.0, 1.0, 1.0);
-
-  auto Result = Embedder::create(IR2VecKind::Symbolic, *F, V);
-  ASSERT_TRUE(static_cast<bool>(Result));
-  auto Emb = std::move(*Result);
-
-  const auto &FuncVec = Emb->getFunctionVector();
-
-  EXPECT_EQ(FuncVec.size(), 2u);
-
-  // Expected: 1*([1.0 2.0] + [0.0 0.0]) + 1*([0.5 0.5] + [0.0 0.0]) + 1*([0.2
-  // 0.3] + [0.0 0.0])
-  EXPECT_THAT(FuncVec,
-              ElementsAre(DoubleNear(1.7, 1e-6), DoubleNear(2.8, 1e-6)));
-}
-
 TEST(IR2VecTest, IR2VecVocabAnalysisWithPrepopulatedVocab) {
   Vocab InitialVocab = {{"key1", {1.1, 2.2}}, {"key2", {3.3, 4.4}}};
   Vocab ExpectedVocab = InitialVocab;
diff --git a/llvm/unittests/Analysis/IVDescriptorsTest.cpp b/llvm/unittests/Analysis/IVDescriptorsTest.cpp
index ce9383d15f46..453800abf9ca 100644
--- a/llvm/unittests/Analysis/IVDescriptorsTest.cpp
+++ b/llvm/unittests/Analysis/IVDescriptorsTest.cpp
@@ -26,7 +26,7 @@ static void runWithLoopInfoAndSE(
   auto *F = M.getFunction(FuncName);
   ASSERT_NE(F, nullptr) << "Could not find " << FuncName;
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index 13377a808209..c4f5b22318e3 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -180,7 +180,7 @@ loop.end:
   auto *F2 = dyn_cast<Function>(GV2);
   ASSERT_TRUE(F1 && F2);
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
 
   auto IsDerefReadOnlyLoop = [&TLI](Function *F) -> bool {
diff --git a/llvm/unittests/Analysis/LoopInfoTest.cpp b/llvm/unittests/Analysis/LoopInfoTest.cpp
index 5e9edce04971..627e59d121c5 100644
--- a/llvm/unittests/Analysis/LoopInfoTest.cpp
+++ b/llvm/unittests/Analysis/LoopInfoTest.cpp
@@ -38,7 +38,7 @@ static void runWithLoopInfoPlus(
   auto *F = M.getFunction(FuncName);
   ASSERT_NE(F, nullptr) << "Could not find " << FuncName;
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
diff --git a/llvm/unittests/Analysis/LoopNestTest.cpp b/llvm/unittests/Analysis/LoopNestTest.cpp
index 366be2e87211..107ce85299f8 100644
--- a/llvm/unittests/Analysis/LoopNestTest.cpp
+++ b/llvm/unittests/Analysis/LoopNestTest.cpp
@@ -25,7 +25,7 @@ static void runTest(
   auto *F = M.getFunction(FuncName);
   ASSERT_NE(F, nullptr) << "Could not find " << FuncName;
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
diff --git a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
index 6eef0b5f9171..aa4d712cde09 100644
--- a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
+++ b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
@@ -14,6 +14,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -23,7 +24,7 @@
 using namespace llvm;
 using namespace llvm::memprof;
 
-extern cl::opt<bool> MemProfKeepAllNotColdContexts;
+LLVM_ABI extern cl::opt<bool> MemProfKeepAllNotColdContexts;
 
 namespace {
 
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index fda3a713b576..a6f002b63dff 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -70,7 +70,8 @@ class MemorySSATest : public testing::Test {
 
 public:
   MemorySSATest()
-      : M("MemorySSATest", C), B(C), DL(DLString), TLI(TLII), F(nullptr) {}
+      : M("MemorySSATest", C), B(C), DL(DLString), TLII(M.getTargetTriple()),
+        TLI(TLII), F(nullptr) {}
 };
 
 TEST_F(MemorySSATest, CreateALoad) {
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index 95a4affdd778..9b88e423e802 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -39,7 +39,8 @@ class ScalarEvolutionsTest : public testing::Test {
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<LoopInfo> LI;
 
-  ScalarEvolutionsTest() : M("", Context), TLII(), TLI(TLII) {}
+  ScalarEvolutionsTest()
+      : M("", Context), TLII(M.getTargetTriple()), TLI(TLII) {}
 
   ScalarEvolution buildSE(Function &F) {
     AC.reset(new AssumptionCache(F));
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 2f1bcbae4fc5..b33419545efa 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -26,7 +26,7 @@ class TargetLibraryInfoTest : public testing::Test {
 
   std::unique_ptr<Module> M;
 
-  TargetLibraryInfoTest() : TLI(TLII) {}
+  TargetLibraryInfoTest() : TLII(Triple()), TLI(TLII) {}
 
   void parseAssembly(const char *Assembly) {
     SMDiagnostic Error;
diff --git a/llvm/unittests/Analysis/UnrollAnalyzerTest.cpp b/llvm/unittests/Analysis/UnrollAnalyzerTest.cpp
index 0ff08d19957e..d5ba1757ce35 100644
--- a/llvm/unittests/Analysis/UnrollAnalyzerTest.cpp
+++ b/llvm/unittests/Analysis/UnrollAnalyzerTest.cpp
@@ -27,7 +27,7 @@ runUnrollAnalyzer(Module &M, StringRef FuncName,
   auto *F = M.getFunction(FuncName);
   ASSERT_NE(F, nullptr) << "Could not find " << FuncName;
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 6031898f7f67..7a48105a1dc9 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -16,6 +16,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -909,11 +910,11 @@ TEST(ValueTracking, propagatesPoison) {
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 0},
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 1},
       {false, "call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt)", 2},
-      {false, "call float @llvm.sqrt.f32(float %fx)", 0},
-      {false, "call float @llvm.powi.f32.i32(float %fx, i32 %x)", 0},
+      {true, "call float @llvm.sqrt.f32(float %fx)", 0},
+      {true, "call float @llvm.powi.f32.i32(float %fx, i32 %x)", 0},
       {false, "call float @llvm.sin.f32(float %fx)", 0},
       {false, "call float @llvm.cos.f32(float %fx)", 0},
-      {false, "call float @llvm.pow.f32(float %fx, float %fy)", 0},
+      {true, "call float @llvm.pow.f32(float %fx, float %fy)", 0},
       {false, "call float @llvm.exp.f32(float %fx)", 0},
       {false, "call float @llvm.exp2.f32(float %fx)", 0},
       {false, "call float @llvm.log.f32(float %fx)", 0},
@@ -1257,6 +1258,58 @@ TEST_F(ValueTrackingTest, computePtrAlignment) {
   EXPECT_EQ(getKnownAlignment(A, DL, CxtI3, &AC, &DT), Align(16));
 }
 
+TEST_F(ValueTrackingTest, MatchBinaryIntrinsicRecurrenceUMax) {
+  auto M = parseModule(R"(
+    define i8 @test(i8 %a, i8 %b) {
+    entry:
+      br label %loop
+    loop:
+      %iv = phi i8 [ %iv.next, %loop ], [ 0, %entry ]
+      %umax.acc = phi i8 [ %umax, %loop ], [ %a, %entry ]
+      %umax = call i8 @llvm.umax.i8(i8 %umax.acc, i8 %b)
+      %iv.next = add nuw i8 %iv, 1
+      %cmp = icmp ult i8 %iv.next, 10
+      br i1 %cmp, label %loop, label %exit
+    exit:
+      ret i8 %umax
+    }
+  )");
+
+  auto *F = M->getFunction("test");
+  auto *II = &cast<IntrinsicInst>(findInstructionByName(F, "umax"));
+  auto *UMaxAcc = &cast<PHINode>(findInstructionByName(F, "umax.acc"));
+  PHINode *PN;
+  Value *Init, *OtherOp;
+  EXPECT_TRUE(matchSimpleBinaryIntrinsicRecurrence(II, PN, Init, OtherOp));
+  EXPECT_EQ(UMaxAcc, PN);
+  EXPECT_EQ(F->getArg(0), Init);
+  EXPECT_EQ(F->getArg(1), OtherOp);
+}
+
+TEST_F(ValueTrackingTest, MatchBinaryIntrinsicRecurrenceNegativeFSHR) {
+  auto M = parseModule(R"(
+    define i8 @test(i8 %a, i8 %b, i8 %c) {
+    entry:
+      br label %loop
+    loop:
+      %iv = phi i8 [ %iv.next, %loop ], [ 0, %entry ]
+      %fshr.acc = phi i8 [ %fshr, %loop ], [ %a, %entry ]
+      %fshr = call i8 @llvm.fshr.i8(i8 %fshr.acc, i8 %b, i8 %c)
+      %iv.next = add nuw i8 %iv, 1
+      %cmp = icmp ult i8 %iv.next, 10
+      br i1 %cmp, label %loop, label %exit
+    exit:
+      ret i8 %fshr
+    }
+  )");
+
+  auto *F = M->getFunction("test");
+  auto *II = &cast<IntrinsicInst>(findInstructionByName(F, "fshr"));
+  PHINode *PN;
+  Value *Init, *OtherOp;
+  EXPECT_FALSE(matchSimpleBinaryIntrinsicRecurrence(II, PN, Init, OtherOp));
+}
+
 TEST_F(ComputeKnownBitsTest, ComputeKnownBits) {
   parseAssembly(
       "define i32 @test(i32 %a, i32 %b) {\n"
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index 2b1fa75a1475..dc531e601374 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -110,6 +110,14 @@ TEST_F(SelectionDAGPatternMatchTest, matchValueType) {
 
   using namespace SDPatternMatch;
   EXPECT_TRUE(sd_match(Op0, m_SpecificVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op0, m_SpecificScalarVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op0, m_SpecificScalarVT(Int32VT, m_Value())));
+  EXPECT_FALSE(sd_match(Op0, m_SpecificScalarVT(Float32VT)));
+  EXPECT_FALSE(sd_match(Op0, m_SpecificScalarVT(Float32VT, m_Value())));
+  EXPECT_TRUE(sd_match(Op2, m_SpecificVectorElementVT(Int32VT)));
+  EXPECT_TRUE(sd_match(Op2, m_SpecificVectorElementVT(Int32VT, m_Value())));
+  EXPECT_FALSE(sd_match(Op2, m_SpecificVectorElementVT(Float32VT)));
+  EXPECT_FALSE(sd_match(Op2, m_SpecificVectorElementVT(Float32VT, m_Value())));
   EVT BindVT;
   EXPECT_TRUE(sd_match(Op1, m_VT(BindVT)));
   EXPECT_EQ(BindVT, Float32VT);
@@ -178,6 +186,12 @@ TEST_F(SelectionDAGPatternMatchTest, matchTernaryOp) {
   SDValue ExtractELT =
       DAG->getNode(ISD::EXTRACT_VECTOR_ELT, DL, Int32VT, V1, Op3);
 
+  SDValue Ch = DAG->getEntryNode();
+  SDValue BasePtr = DAG->getRegister(1, MVT::i64);
+  SDValue Offset = DAG->getUNDEF(MVT::i64);
+  MachinePointerInfo PtrInfo;
+  SDValue Load = DAG->getLoad(MVT::i32, DL, Ch, BasePtr, PtrInfo);
+
   using namespace SDPatternMatch;
   ISD::CondCode CC;
   EXPECT_TRUE(sd_match(ICMP_UGT, m_SetCC(m_Value(), m_Value(),
@@ -230,6 +244,13 @@ TEST_F(SelectionDAGPatternMatchTest, matchTernaryOp) {
   EXPECT_FALSE(sd_match(
       InsertSubvector,
       m_InsertSubvector(m_Specific(V2), m_Specific(V3), m_SpecificInt(3))));
+
+  EXPECT_TRUE(sd_match(
+      Load, m_Load(m_Specific(Ch), m_Specific(BasePtr), m_Specific(Offset))));
+
+  EXPECT_FALSE(
+      sd_match(Load.getValue(1), m_Load(m_Specific(Ch), m_Specific(BasePtr),
+                                        m_Specific(Offset))));
 }
 
 TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
@@ -264,6 +285,11 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue Rotl = DAG->getNode(ISD::ROTL, DL, Int32VT, Op0, Op1);
   SDValue Rotr = DAG->getNode(ISD::ROTR, DL, Int32VT, Op1, Op0);
 
+  SDValue SMulLoHi = DAG->getNode(ISD::SMUL_LOHI, DL,
+                                  DAG->getVTList(Int32VT, Int32VT), Op0, Op1);
+  SDValue PartsDiff =
+      DAG->getNode(ISD::SUB, DL, Int32VT, SMulLoHi, SMulLoHi.getValue(1));
+
   SDValue ICMP_GT = DAG->getSetCC(DL, MVT::i1, Op0, Op1, ISD::SETGT);
   SDValue ICMP_GE = DAG->getSetCC(DL, MVT::i1, Op0, Op1, ISD::SETGE);
   SDValue ICMP_UGT = DAG->getSetCC(DL, MVT::i1, Op0, Op1, ISD::SETUGT);
@@ -347,6 +373,15 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(UMinLikeULT, m_UMinLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(UMinLikeULE, m_UMinLike(m_Value(), m_Value())));
 
+  // By default, it matches any of the results.
+  EXPECT_TRUE(
+      sd_match(PartsDiff, m_Sub(m_Opc(ISD::SMUL_LOHI), m_Opc(ISD::SMUL_LOHI))));
+  // Matching a specific result.
+  EXPECT_TRUE(sd_match(PartsDiff, m_Sub(m_Opc(ISD::SMUL_LOHI),
+                                        m_Result<1>(m_Opc(ISD::SMUL_LOHI)))));
+  EXPECT_FALSE(sd_match(PartsDiff, m_Sub(m_Opc(ISD::SMUL_LOHI),
+                                         m_Result<0>(m_Opc(ISD::SMUL_LOHI)))));
+
   SDValue BindVal;
   EXPECT_TRUE(sd_match(SFAdd, m_ChainedBinOp(ISD::STRICT_FADD, m_Value(BindVal),
                                              m_Deferred(BindVal))));
@@ -790,3 +825,89 @@ TEST_F(SelectionDAGPatternMatchTest, matchReassociatableOp) {
   EXPECT_FALSE(sd_match(
       ORS0123, m_ReassociatableOr(m_Value(), m_Value(), m_Value(), m_Value())));
 }
+
+TEST_F(SelectionDAGPatternMatchTest, MatchZeroOneAllOnes) {
+  using namespace SDPatternMatch;
+
+  SDLoc DL;
+  EVT VT = EVT::getIntegerVT(Context, 32);
+
+  // Scalar constant 0
+  SDValue Zero = DAG->getConstant(0, DL, VT);
+  EXPECT_TRUE(sd_match(Zero, DAG.get(), llvm::SDPatternMatch::m_Zero()));
+  EXPECT_FALSE(sd_match(Zero, DAG.get(), m_One()));
+  EXPECT_FALSE(sd_match(Zero, DAG.get(), m_AllOnes()));
+
+  // Scalar constant 1
+  SDValue One = DAG->getConstant(1, DL, VT);
+  EXPECT_FALSE(sd_match(One, DAG.get(), m_Zero()));
+  EXPECT_TRUE(sd_match(One, DAG.get(), m_One()));
+  EXPECT_FALSE(sd_match(One, DAG.get(), m_AllOnes()));
+
+  // Scalar constant -1
+  SDValue AllOnes =
+      DAG->getConstant(APInt::getAllOnes(VT.getSizeInBits()), DL, VT);
+  EXPECT_FALSE(sd_match(AllOnes, DAG.get(), m_Zero()));
+  EXPECT_FALSE(sd_match(AllOnes, DAG.get(), m_One()));
+  EXPECT_TRUE(sd_match(AllOnes, DAG.get(), m_AllOnes()));
+
+  EVT VecF32 = EVT::getVectorVT(Context, MVT::f32, 4);
+  EVT VecVT = EVT::getVectorVT(Context, MVT::i32, 4);
+
+  // m_Zero: splat vector of 0 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(0, DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_TRUE(sd_match(Bitcasted, DAG.get(), m_Zero()));
+  }
+
+  // m_One: splat vector of 1 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(1, DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_FALSE(sd_match(Bitcasted, DAG.get(), m_One()));
+  }
+
+  // m_AllOnes: splat vector of -1 → bitcast
+  {
+    SDValue SplatVal = DAG->getConstant(APInt::getAllOnes(32), DL, MVT::i32);
+    SDValue VecSplat = DAG->getSplatBuildVector(VecVT, DL, SplatVal);
+    SDValue Bitcasted = DAG->getNode(ISD::BITCAST, DL, VecF32, VecSplat);
+    EXPECT_TRUE(sd_match(Bitcasted, DAG.get(), m_AllOnes()));
+  }
+
+  // splat vector with one undef → default should NOT match
+  SDValue Undef = DAG->getUNDEF(MVT::i32);
+
+  {
+    // m_Zero: Undef + constant 0
+    SDValue Zero = DAG->getConstant(0, DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, Zero);
+    Ops[2] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_Zero()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_Zero(true)));
+  }
+
+  {
+    // m_One: Undef + constant 1
+    SDValue One = DAG->getConstant(1, DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, One);
+    Ops[1] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_One()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_One(true)));
+  }
+
+  {
+    // m_AllOnes: Undef + constant -1
+    SDValue AllOnes = DAG->getConstant(APInt::getAllOnes(32), DL, MVT::i32);
+    SmallVector<SDValue, 4> Ops(4, AllOnes);
+    Ops[0] = Undef;
+    SDValue Vec = DAG->getBuildVector(VecVT, DL, Ops);
+    EXPECT_FALSE(sd_match(Vec, DAG.get(), m_AllOnes()));
+    EXPECT_TRUE(sd_match(Vec, DAG.get(), m_AllOnes(true)));
+  }
+}
diff --git a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
index 6c04ab774c2e..d4b45ea0dca0 100644
--- a/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManagerTest.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h"
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
 #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h"
 #include "llvm/Support/FormatVariadic.h"
diff --git a/llvm/unittests/ExecutionEngine/Orc/EPCGenericMemoryAccessTest.cpp b/llvm/unittests/ExecutionEngine/Orc/EPCGenericMemoryAccessTest.cpp
index 3eec564ee629..2bcff3356b86 100644
--- a/llvm/unittests/ExecutionEngine/Orc/EPCGenericMemoryAccessTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/EPCGenericMemoryAccessTest.cpp
@@ -9,6 +9,7 @@
 #include "OrcTestCommon.h"
 
 #include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/Testing/Support/Error.h"
 
 using namespace llvm;
@@ -28,6 +29,16 @@ CWrapperFunctionResult testWriteUInts(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
+CWrapperFunctionResult testWritePointers(const char *ArgData, size_t ArgSize) {
+  return WrapperFunction<void(SPSSequence<SPSMemoryAccessPointerWrite>)>::
+      handle(ArgData, ArgSize,
+             [](std::vector<tpctypes::PointerWrite> Ws) {
+               for (auto &W : Ws)
+                 *W.Addr.template toPtr<void **>() = W.Value.toPtr<void *>();
+             })
+          .release();
+}
+
 CWrapperFunctionResult testWriteBuffers(const char *ArgData, size_t ArgSize) {
   return WrapperFunction<void(SPSSequence<SPSMemoryAccessBufferWrite>)>::handle(
              ArgData, ArgSize,
@@ -39,76 +50,296 @@ CWrapperFunctionResult testWriteBuffers(const char *ArgData, size_t ArgSize) {
       .release();
 }
 
-CWrapperFunctionResult testWritePointers(const char *ArgData, size_t ArgSize) {
-  return WrapperFunction<void(SPSSequence<SPSMemoryAccessPointerWrite>)>::
-      handle(ArgData, ArgSize,
-             [](std::vector<tpctypes::PointerWrite> Ws) {
-               for (auto &W : Ws)
-                 *W.Addr.template toPtr<uint64_t *>() = W.Value.getValue();
+template <typename ReadT>
+CWrapperFunctionResult testReadUInts(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<ReadT>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(ArgData, ArgSize,
+                                         [](std::vector<ExecutorAddr> Rs) {
+                                           std::vector<ReadT> Result;
+                                           Result.reserve(Rs.size());
+                                           for (auto &R : Rs)
+                                             Result.push_back(
+                                                 *R.template toPtr<ReadT *>());
+                                           return Result;
+                                         })
+      .release();
+}
+
+CWrapperFunctionResult testReadPointers(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<SPSExecutorAddr>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(
+             ArgData, ArgSize,
+             [](std::vector<ExecutorAddr> Rs) {
+               std::vector<ExecutorAddr> Result;
+               Result.reserve(Rs.size());
+               for (auto &R : Rs)
+                 Result.push_back(
+                     ExecutorAddr::fromPtr(*R.template toPtr<void **>()));
+               return Result;
              })
-          .release();
+      .release();
+}
+
+CWrapperFunctionResult testReadBuffers(const char *ArgData, size_t ArgSize) {
+  using SPSSig =
+      SPSSequence<SPSSequence<uint8_t>>(SPSSequence<SPSExecutorAddrRange>);
+  return WrapperFunction<SPSSig>::handle(
+             ArgData, ArgSize,
+             [](std::vector<ExecutorAddrRange> Rs) {
+               std::vector<std::vector<uint8_t>> Result;
+               Result.reserve(Rs.size());
+               for (auto &R : Rs) {
+                 Result.push_back({});
+                 Result.back().resize(R.size());
+                 memcpy(reinterpret_cast<char *>(Result.back().data()),
+                        R.Start.toPtr<char *>(), R.size());
+               }
+               return Result;
+             })
+      .release();
+}
+
+CWrapperFunctionResult testReadStrings(const char *ArgData, size_t ArgSize) {
+  using SPSSig = SPSSequence<SPSString>(SPSSequence<SPSExecutorAddr>);
+  return WrapperFunction<SPSSig>::handle(
+             ArgData, ArgSize,
+             [](std::vector<ExecutorAddr> Rs) {
+               std::vector<std::string> Result;
+               Result.reserve(Rs.size());
+               for (auto &R : Rs)
+                 Result.push_back(std::string(R.toPtr<char *>()));
+               return Result;
+             })
+      .release();
+}
+
+class EPCGenericMemoryAccessTest : public testing::Test {
+public:
+  EPCGenericMemoryAccessTest() {
+    EPC = cantFail(SelfExecutorProcessControl::Create());
+
+    EPCGenericMemoryAccess::FuncAddrs FAs;
+    FAs.WriteUInt8s = ExecutorAddr::fromPtr(
+        &testWriteUInts<tpctypes::UInt8Write, SPSMemoryAccessUInt8Write>);
+    FAs.WriteUInt16s = ExecutorAddr::fromPtr(
+        &testWriteUInts<tpctypes::UInt16Write, SPSMemoryAccessUInt16Write>);
+    FAs.WriteUInt32s = ExecutorAddr::fromPtr(
+        &testWriteUInts<tpctypes::UInt32Write, SPSMemoryAccessUInt32Write>);
+    FAs.WriteUInt64s = ExecutorAddr::fromPtr(
+        &testWriteUInts<tpctypes::UInt64Write, SPSMemoryAccessUInt64Write>);
+    FAs.WritePointers = ExecutorAddr::fromPtr(&testWritePointers);
+    FAs.WriteBuffers = ExecutorAddr::fromPtr(&testWriteBuffers);
+    FAs.ReadUInt8s = ExecutorAddr::fromPtr(&testReadUInts<uint8_t>);
+    FAs.ReadUInt16s = ExecutorAddr::fromPtr(&testReadUInts<uint16_t>);
+    FAs.ReadUInt32s = ExecutorAddr::fromPtr(&testReadUInts<uint32_t>);
+    FAs.ReadUInt64s = ExecutorAddr::fromPtr(&testReadUInts<uint64_t>);
+    FAs.ReadPointers = ExecutorAddr::fromPtr(&testReadPointers);
+    FAs.ReadBuffers = ExecutorAddr::fromPtr(&testReadBuffers);
+    FAs.ReadStrings = ExecutorAddr::fromPtr(&testReadStrings);
+
+    MemAccess = std::make_unique<EPCGenericMemoryAccess>(*EPC, FAs);
+  }
+
+  ~EPCGenericMemoryAccessTest() { cantFail(EPC->disconnect()); }
+
+protected:
+  std::shared_ptr<SelfExecutorProcessControl> EPC;
+  std::unique_ptr<MemoryAccess> MemAccess;
+
+  static const uint8_t UInt8_1_TestValue;
+  static const uint8_t UInt8_2_TestValue;
+  static const uint16_t UInt16_TestValue;
+  static const uint32_t UInt32_TestValue;
+  static const uint64_t UInt64_TestValue;
+  static const void *Pointer_TestValue;
+
+  static const char Buffer_TestValue[21];
+  static const char *String_TestValue;
+};
+
+const uint8_t EPCGenericMemoryAccessTest::UInt8_1_TestValue = 1;
+const uint8_t EPCGenericMemoryAccessTest::UInt8_2_TestValue = 2;
+const uint16_t EPCGenericMemoryAccessTest::UInt16_TestValue = 3;
+const uint32_t EPCGenericMemoryAccessTest::UInt32_TestValue = 4;
+const uint64_t EPCGenericMemoryAccessTest::UInt64_TestValue = 5;
+
+const void *EPCGenericMemoryAccessTest::Pointer_TestValue =
+    reinterpret_cast<void *>(uintptr_t(0x6));
+
+const char EPCGenericMemoryAccessTest::Buffer_TestValue[21] = {
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A,
+    0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14};
+const char *EPCGenericMemoryAccessTest::String_TestValue = "hello, world!";
+
+TEST_F(EPCGenericMemoryAccessTest, WriteUInt8s) {
+  uint8_t UInt8_1_Storage = 0;
+  uint8_t UInt8_2_Storage = 0;
+
+  auto Err = MemAccess->writeUInt8s(
+      {{ExecutorAddr::fromPtr(&UInt8_1_Storage), UInt8_1_TestValue},
+       {ExecutorAddr::fromPtr(&UInt8_2_Storage), UInt8_2_TestValue}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(UInt8_1_Storage, UInt8_1_TestValue);
+  EXPECT_EQ(UInt8_2_Storage, UInt8_2_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadUInt8s) {
+  uint8_t UInt8_1_Storage = UInt8_1_TestValue;
+  uint8_t UInt8_2_Storage = UInt8_2_TestValue;
+
+  auto Vals =
+      MemAccess->readUInt8s({{ExecutorAddr::fromPtr(&UInt8_1_Storage),
+                              ExecutorAddr::fromPtr(&UInt8_2_Storage)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type::value_type, uint8_t>);
+  if (!Vals)
+    return ADD_FAILURE() << toString(Vals.takeError());
+
+  EXPECT_EQ(Vals->size(), 2U);
+  if (Vals->size() >= 1)
+    EXPECT_EQ((*Vals)[0], UInt8_1_TestValue);
+  if (Vals->size() >= 2)
+    EXPECT_EQ((*Vals)[1], UInt8_2_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, WriteUInt16s) {
+  uint16_t UInt16_Storage = 0;
+
+  auto Err = MemAccess->writeUInt16s(
+      {{ExecutorAddr::fromPtr(&UInt16_Storage), UInt16_TestValue}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(UInt16_Storage, UInt16_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadUInt16s) {
+  uint16_t UInt16_Storage = UInt16_TestValue;
+
+  auto Vals =
+      MemAccess->readUInt16s({{ExecutorAddr::fromPtr(&UInt16_Storage)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type::value_type, uint16_t>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1)
+      EXPECT_EQ((*Vals)[0], UInt16_TestValue);
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
+}
+
+TEST_F(EPCGenericMemoryAccessTest, WriteUInt32s) {
+  uint32_t UInt32_Storage = 0;
+
+  auto Err = MemAccess->writeUInt32s(
+      {{ExecutorAddr::fromPtr(&UInt32_Storage), UInt32_TestValue}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(UInt32_Storage, UInt32_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadUInt32s) {
+  uint32_t UInt32_Storage = UInt32_TestValue;
+
+  auto Vals =
+      MemAccess->readUInt32s({{ExecutorAddr::fromPtr(&UInt32_Storage)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type::value_type, uint32_t>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1)
+      EXPECT_EQ((*Vals)[0], UInt32_TestValue);
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
+}
+
+TEST_F(EPCGenericMemoryAccessTest, WriteUInt64s) {
+  uint64_t UInt64_Storage = 0;
+
+  auto Err = MemAccess->writeUInt64s(
+      {{ExecutorAddr::fromPtr(&UInt64_Storage), UInt64_TestValue}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(UInt64_Storage, UInt64_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadUInt64s) {
+  uint64_t UInt64_Storage = UInt64_TestValue;
+
+  auto Vals =
+      MemAccess->readUInt64s({{ExecutorAddr::fromPtr(&UInt64_Storage)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type::value_type, uint64_t>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1)
+      EXPECT_EQ((*Vals)[0], UInt64_TestValue);
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
+}
+
+TEST_F(EPCGenericMemoryAccessTest, WritePointers) {
+  void *Pointer_Storage = nullptr;
+
+  auto Err =
+      MemAccess->writePointers({{ExecutorAddr::fromPtr(&Pointer_Storage),
+                                 ExecutorAddr::fromPtr(Pointer_TestValue)}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(Pointer_Storage, Pointer_TestValue);
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadPointers) {
+  auto Vals =
+      MemAccess->readPointers({{ExecutorAddr::fromPtr(&Pointer_TestValue)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type::value_type, ExecutorAddr>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1)
+      EXPECT_EQ((*Vals)[0], ExecutorAddr::fromPtr(Pointer_TestValue));
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
+}
+
+TEST_F(EPCGenericMemoryAccessTest, WriteBuffers) {
+  char Buffer_Storage[sizeof(Buffer_TestValue)];
+  memset(Buffer_Storage, 0, sizeof(Buffer_TestValue));
+
+  auto Err = MemAccess->writeBuffers(
+      {{ExecutorAddr::fromPtr(&Buffer_Storage),
+        ArrayRef(Buffer_TestValue, sizeof(Buffer_TestValue))}});
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  EXPECT_EQ(ArrayRef(Buffer_Storage, sizeof(Buffer_TestValue)),
+            ArrayRef(Buffer_TestValue, sizeof(Buffer_TestValue)));
+}
+
+TEST_F(EPCGenericMemoryAccessTest, ReadBuffers) {
+  char Buffer_Storage[sizeof(Buffer_TestValue)];
+  memcpy(Buffer_Storage, Buffer_TestValue, sizeof(Buffer_TestValue));
+
+  auto Vals = MemAccess->readBuffers({{ExecutorAddrRange(
+      ExecutorAddr::fromPtr(&Buffer_Storage), sizeof(Buffer_Storage))}});
+  static_assert(std::is_same_v<decltype(Vals)::value_type::value_type,
+                               std::vector<uint8_t>>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1) {
+      EXPECT_EQ((*Vals)[0].size(), sizeof(Buffer_Storage));
+      EXPECT_EQ(
+          memcmp((*Vals)[0].data(), Buffer_TestValue, sizeof(Buffer_Storage)),
+          0);
+    }
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
 }
 
-TEST(EPCGenericMemoryAccessTest, MemWrites) {
-  auto SelfEPC = cantFail(SelfExecutorProcessControl::Create());
-
-  EPCGenericMemoryAccess::FuncAddrs FAs;
-  FAs.WriteUInt8s = ExecutorAddr::fromPtr(
-      &testWriteUInts<tpctypes::UInt8Write, SPSMemoryAccessUInt8Write>);
-  FAs.WriteUInt16s = ExecutorAddr::fromPtr(
-      &testWriteUInts<tpctypes::UInt16Write, SPSMemoryAccessUInt16Write>);
-  FAs.WriteUInt32s = ExecutorAddr::fromPtr(
-      &testWriteUInts<tpctypes::UInt32Write, SPSMemoryAccessUInt32Write>);
-  FAs.WriteUInt64s = ExecutorAddr::fromPtr(
-      &testWriteUInts<tpctypes::UInt64Write, SPSMemoryAccessUInt64Write>);
-  FAs.WriteBuffers = ExecutorAddr::fromPtr(&testWriteBuffers);
-  FAs.WritePointers = ExecutorAddr::fromPtr(&testWritePointers);
-
-  auto MemAccess = std::make_unique<EPCGenericMemoryAccess>(*SelfEPC, FAs);
-
-  uint8_t Test_UInt8_1 = 0;
-  uint8_t Test_UInt8_2 = 0;
-  uint16_t Test_UInt16 = 0;
-  uint32_t Test_UInt32 = 0;
-  uint64_t Test_UInt64 = 0;
-  uint64_t Test_Pointer = 0;
-  char Test_Buffer[21];
-
-  auto Err1 =
-      MemAccess->writeUInt8s({{ExecutorAddr::fromPtr(&Test_UInt8_1), 1},
-                              {ExecutorAddr::fromPtr(&Test_UInt8_2), 0xFE}});
-
-  EXPECT_THAT_ERROR(std::move(Err1), Succeeded());
-  EXPECT_EQ(Test_UInt8_1, 1U);
-  EXPECT_EQ(Test_UInt8_2, 0xFE);
-
-  auto Err2 =
-      MemAccess->writeUInt16s({{ExecutorAddr::fromPtr(&Test_UInt16), 1}});
-  EXPECT_THAT_ERROR(std::move(Err2), Succeeded());
-  EXPECT_EQ(Test_UInt16, 1U);
-
-  auto Err3 =
-      MemAccess->writeUInt32s({{ExecutorAddr::fromPtr(&Test_UInt32), 1}});
-  EXPECT_THAT_ERROR(std::move(Err3), Succeeded());
-  EXPECT_EQ(Test_UInt32, 1U);
-
-  auto Err4 =
-      MemAccess->writeUInt64s({{ExecutorAddr::fromPtr(&Test_UInt64), 1}});
-  EXPECT_THAT_ERROR(std::move(Err4), Succeeded());
-  EXPECT_EQ(Test_UInt64, 1U);
-
-  StringRef TestMsg("test-message");
-  auto Err5 =
-      MemAccess->writeBuffers({{ExecutorAddr::fromPtr(&Test_Buffer), TestMsg}});
-  EXPECT_THAT_ERROR(std::move(Err5), Succeeded());
-  EXPECT_EQ(StringRef(Test_Buffer, TestMsg.size()), TestMsg);
-
-  auto Err6 = MemAccess->writePointers(
-      {{ExecutorAddr::fromPtr(&Test_Pointer), ExecutorAddr(1U)}});
-  EXPECT_THAT_ERROR(std::move(Err6), Succeeded());
-  EXPECT_EQ(Test_Pointer, 1U);
-
-  cantFail(SelfEPC->disconnect());
+TEST_F(EPCGenericMemoryAccessTest, ReadStrings) {
+  auto Vals =
+      MemAccess->readStrings({{ExecutorAddr::fromPtr(String_TestValue)}});
+  static_assert(
+      std::is_same_v<decltype(Vals)::value_type, std::vector<std::string>>);
+  if (Vals) {
+    EXPECT_EQ(Vals->size(), 1U);
+    if (Vals->size() == 1)
+      EXPECT_EQ((*Vals)[0], std::string(String_TestValue));
+  } else
+    EXPECT_THAT_ERROR(Vals.takeError(), Succeeded());
 }
 
 } // namespace
diff --git a/llvm/unittests/ExecutionEngine/Orc/ExecutionSessionWrapperFunctionCallsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ExecutionSessionWrapperFunctionCallsTest.cpp
index 80b9c157f762..1f7067502b15 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ExecutionSessionWrapperFunctionCallsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ExecutionSessionWrapperFunctionCallsTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
@@ -109,7 +110,7 @@ TEST(ExecutionSessionWrapperFunctionCalls, RegisterAsyncHandlerAndRun) {
         EXPECT_TRUE(SPSArgList<int32_t>::deserialize(IB, Result));
         RP.set_value(Result);
       },
-      AddAsyncTagAddr, ArrayRef<char>(ArgBuffer.data(), ArgBuffer.size()));
+      AddAsyncTagAddr, ArrayRef<char>(ArgBuffer));
 
   EXPECT_EQ(RF.get(), (int32_t)3);
 
diff --git a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
index 0c2c06ec75d6..8a6549b1b594 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ObjectLinkingLayerTest.cpp
@@ -19,6 +19,8 @@
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 
+#include "OrcTestCommon.h"
+
 using namespace llvm;
 using namespace llvm::jitlink;
 using namespace llvm::orc;
diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h
index 0981f4b8132b..469de2a3665a 100644
--- a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h
+++ b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h
@@ -17,6 +17,7 @@
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/Core.h"
 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
+#include "llvm/ExecutionEngine/Orc/InProcessMemoryAccess.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
@@ -82,6 +83,47 @@ class CoreAPIsBasedStandardTest : public testing::Test {
   unique_function<void(std::unique_ptr<Task>)> DispatchOverride;
 };
 
+/// A ExecutorProcessControl instance that asserts if any of its methods are
+/// used. Suitable for use is unit tests, and by ORC clients who haven't moved
+/// to ExecutorProcessControl-based APIs yet.
+class UnsupportedExecutorProcessControl : public ExecutorProcessControl,
+                                          private InProcessMemoryAccess {
+public:
+  UnsupportedExecutorProcessControl(
+      std::shared_ptr<SymbolStringPool> SSP = nullptr,
+      std::unique_ptr<TaskDispatcher> D = nullptr, const std::string &TT = "",
+      unsigned PageSize = 0)
+      : ExecutorProcessControl(
+            SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>(),
+            D ? std::move(D) : std::make_unique<InPlaceTaskDispatcher>()),
+        InProcessMemoryAccess(Triple(TT).isArch64Bit()) {
+    this->TargetTriple = Triple(TT);
+    this->PageSize = PageSize;
+    this->MemAccess = this;
+  }
+
+  Expected<int32_t> runAsMain(ExecutorAddr MainFnAddr,
+                              ArrayRef<std::string> Args) override {
+    llvm_unreachable("Unsupported");
+  }
+
+  Expected<int32_t> runAsVoidFunction(ExecutorAddr VoidFnAddr) override {
+    llvm_unreachable("Unsupported");
+  }
+
+  Expected<int32_t> runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) override {
+    llvm_unreachable("Unsupported");
+  }
+
+  void callWrapperAsync(ExecutorAddr WrapperFnAddr,
+                        IncomingWFRHandler OnComplete,
+                        ArrayRef<char> ArgBuffer) override {
+    llvm_unreachable("Unsupported");
+  }
+
+  Error disconnect() override { return Error::success(); }
+};
+
 } // end namespace orc
 
 class OrcNativeTarget {
diff --git a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
index a0ae9308d33a..61b99ddf0995 100644
--- a/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
@@ -128,29 +128,25 @@ TEST(RTDyldObjectLinkingLayerTest, TestOverrideObjectFlags) {
   };
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B1(FooEntry);
     B1.CreateRetVoid();
 
     Function *BarImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "bar");
-    BasicBlock *BarEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", BarImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "bar");
+    BasicBlock *BarEntry = BasicBlock::Create(*Ctx, "entry", BarImpl);
     IRBuilder<> B2(BarEntry);
     B2.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   // Create a simple stack and set the override flags option.
@@ -207,21 +203,19 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
   };
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B(FooEntry);
     B.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   // Create a simple stack and set the override flags option.
@@ -258,21 +252,19 @@ TEST(RTDyldObjectLinkingLayerTest, TestMemoryBufferNamePropagation) {
     GTEST_SKIP();
 
   // Create a module with two void() functions: foo and bar.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
   ThreadSafeModule M;
   {
-    ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
+    auto Ctx = std::make_unique<LLVMContext>();
+    ModuleBuilder MB(*Ctx, TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
     Function *FooImpl = MB.createFunctionDecl(
-        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
-        "foo");
-    BasicBlock *FooEntry =
-        BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
+        FunctionType::get(Type::getVoidTy(*Ctx), {}, false), "foo");
+    BasicBlock *FooEntry = BasicBlock::Create(*Ctx, "entry", FooImpl);
     IRBuilder<> B1(FooEntry);
     B1.CreateRetVoid();
 
-    M = ThreadSafeModule(MB.takeModule(), std::move(TSCtx));
+    M = ThreadSafeModule(MB.takeModule(), std::move(Ctx));
   }
 
   ExecutionSession ES{std::make_unique<UnsupportedExecutorProcessControl>()};
diff --git a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
index 71fd255a68c8..cd10ffe53d0f 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ReOptimizeLayerTest.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/Support/CodeGen.h"
@@ -171,8 +172,8 @@ TEST_F(ReOptimizeLayerTest, BasicReOptimization) {
       });
   EXPECT_THAT_ERROR(ROLayer->reigsterRuntimeFunctions(*JD), Succeeded());
 
-  ThreadSafeContext Ctx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("<main>", *Ctx.getContext());
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("<main>", *Ctx);
   M->setTargetTriple(Triple(sys::getProcessTriple()));
 
   (void)createRetFunction(M.get(), "main", 42);
diff --git a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
index 0449ec6f7fbc..700500f41d88 100644
--- a/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/SharedMemoryMapperTest.cpp
@@ -9,6 +9,7 @@
 #include "OrcTestCommon.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 #include "llvm/ExecutionEngine/Orc/MemoryMapper.h"
+#include "llvm/ExecutionEngine/Orc/SelfExecutorProcessControl.h"
 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
 #include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"
 #include "llvm/Testing/Support/Error.h"
diff --git a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
index 5b094e2edd58..bbb9e8d3d6a7 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
@@ -7,6 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/IRReader/IRReader.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
 #include "gtest/gtest.h"
 
 #include <atomic>
@@ -18,23 +25,42 @@ using namespace llvm::orc;
 
 namespace {
 
+const llvm::StringRef FooSrc = R"(
+  define void @foo() {
+    ret void
+  }
+)";
+
+static ThreadSafeModule parseModule(llvm::StringRef Source,
+                                    llvm::StringRef Name) {
+  auto Ctx = std::make_unique<LLVMContext>();
+  SMDiagnostic Err;
+  auto M = parseIR(MemoryBufferRef(Source, Name), Err, *Ctx);
+  if (!M) {
+    Err.print("Testcase source failed to parse: ", errs());
+    exit(1);
+  }
+  return ThreadSafeModule(std::move(M), std::move(Ctx));
+}
+
 TEST(ThreadSafeModuleTest, ContextWhollyOwnedByOneModule) {
   // Test that ownership of a context can be transferred to a single
   // ThreadSafeModule.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
-  ThreadSafeModule TSM(std::move(M), std::move(TSCtx));
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
 }
 
 TEST(ThreadSafeModuleTest, ContextOwnershipSharedByTwoModules) {
   // Test that ownership of a context can be shared between more than one
   // ThreadSafeModule.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
+  auto Ctx = std::make_unique<LLVMContext>();
 
-  auto M1 = std::make_unique<Module>("M1", *TSCtx.getContext());
-  ThreadSafeModule TSM1(std::move(M1), TSCtx);
+  auto M1 = std::make_unique<Module>("M1", *Ctx);
+  auto M2 = std::make_unique<Module>("M2", *Ctx);
 
-  auto M2 = std::make_unique<Module>("M2", *TSCtx.getContext());
+  ThreadSafeContext TSCtx(std::move(Ctx));
+  ThreadSafeModule TSM1(std::move(M1), TSCtx);
   ThreadSafeModule TSM2(std::move(M2), std::move(TSCtx));
 }
 
@@ -45,12 +71,14 @@ TEST(ThreadSafeModuleTest, ContextOwnershipSharedWithClient) {
 
   {
     // Create and destroy a module.
-    auto M1 = std::make_unique<Module>("M1", *TSCtx.getContext());
+    auto M1 = TSCtx.withContextDo(
+        [](LLVMContext *Ctx) { return std::make_unique<Module>("M1", *Ctx); });
     ThreadSafeModule TSM1(std::move(M1), TSCtx);
   }
 
   // Verify that the context is still available for re-use.
-  auto M2 = std::make_unique<Module>("M2", *TSCtx.getContext());
+  auto M2 = TSCtx.withContextDo(
+      [](LLVMContext *Ctx) { return std::make_unique<Module>("M2", *Ctx); });
   ThreadSafeModule TSM2(std::move(M2), std::move(TSCtx));
 }
 
@@ -59,60 +87,69 @@ TEST(ThreadSafeModuleTest, ThreadSafeModuleMoveAssignment) {
   // to the field order) to ensure that overwriting with an empty
   // ThreadSafeModule does not destroy the context early.
   ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
+  auto M = TSCtx.withContextDo(
+      [](LLVMContext *Ctx) { return std::make_unique<Module>("M", *Ctx); });
   ThreadSafeModule TSM(std::move(M), std::move(TSCtx));
   TSM = ThreadSafeModule();
 }
 
-TEST(ThreadSafeModuleTest, BasicContextLockAPI) {
-  // Test that basic lock API calls work.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto M = std::make_unique<Module>("M", *TSCtx.getContext());
-  ThreadSafeModule TSM(std::move(M), TSCtx);
-
-  { auto L = TSCtx.getLock(); }
+TEST(ThreadSafeModuleTest, WithContextDoPreservesContext) {
+  // Test that withContextDo passes through the LLVMContext that was used
+  // to create the ThreadSafeContext.
 
-  { auto L = TSM.getContext().getLock(); }
-}
-
-TEST(ThreadSafeModuleTest, ContextLockPreservesContext) {
-  // Test that the existence of a context lock preserves the attached
-  // context.
-  // The trick to verify this is a bit of a hack: We attach a Module
-  // (without the ThreadSafeModule wrapper) to the context, then verify
-  // that this Module destructs safely (which it will not if its context
-  // has been destroyed) even though all references to the context have
-  // been thrown away (apart from the lock).
-
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  auto L = TSCtx.getLock();
-  auto &Ctx = *TSCtx.getContext();
-  auto M = std::make_unique<Module>("M", Ctx);
-  TSCtx = ThreadSafeContext();
+  auto Ctx = std::make_unique<LLVMContext>();
+  LLVMContext *OriginalCtx = Ctx.get();
+  ThreadSafeContext TSCtx(std::move(Ctx));
+  TSCtx.withContextDo(
+      [&](LLVMContext *ClosureCtx) { EXPECT_EQ(ClosureCtx, OriginalCtx); });
 }
 
 TEST(ThreadSafeModuleTest, WithModuleDo) {
   // Test non-const version of withModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                       TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.withModuleDo([](Module &M) {});
 }
 
 TEST(ThreadSafeModuleTest, WithModuleDoConst) {
   // Test const version of withModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  const ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                             TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  const ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.withModuleDo([](const Module &M) {});
 }
 
 TEST(ThreadSafeModuleTest, ConsumingModuleDo) {
   // Test consumingModuleDo.
-  ThreadSafeContext TSCtx(std::make_unique<LLVMContext>());
-  ThreadSafeModule TSM(std::make_unique<Module>("M", *TSCtx.getContext()),
-                       TSCtx);
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = std::make_unique<Module>("M", *Ctx);
+  ThreadSafeModule TSM(std::move(M), std::move(Ctx));
   TSM.consumingModuleDo([](std::unique_ptr<Module> M) {});
 }
 
+TEST(ThreadSafeModuleTest, CloneToNewContext) {
+  auto TSM1 = parseModule(FooSrc, "foo.ll");
+  auto TSM2 = cloneToNewContext(TSM1);
+  TSM2.withModuleDo([&](Module &NewM) {
+    EXPECT_FALSE(verifyModule(NewM, &errs()));
+    TSM1.withModuleDo([&](Module &OrigM) {
+      EXPECT_NE(&NewM.getContext(), &OrigM.getContext());
+    });
+  });
+}
+
+TEST(ObjectFormatsTest, CloneToContext) {
+  auto TSM1 = parseModule(FooSrc, "foo.ll");
+
+  auto TSCtx = ThreadSafeContext(std::make_unique<LLVMContext>());
+  auto TSM2 = cloneToContext(TSM1, TSCtx);
+
+  TSM2.withModuleDo([&](Module &M) {
+    EXPECT_FALSE(verifyModule(M, &errs()));
+    TSCtx.withContextDo(
+        [&](LLVMContext *Ctx) { EXPECT_EQ(&M.getContext(), Ctx); });
+  });
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/FileCheck/FileCheckTest.cpp b/llvm/unittests/FileCheck/FileCheckTest.cpp
index 91fce69b9a6c..2c4130330e97 100644
--- a/llvm/unittests/FileCheck/FileCheckTest.cpp
+++ b/llvm/unittests/FileCheck/FileCheckTest.cpp
@@ -1439,7 +1439,7 @@ TEST_F(FileCheckTest, Substitution) {
   // Substitution of an undefined string variable fails and error holds that
   // variable's name.
   StringSubstitution StringSubstitution(&Context, "VAR404", 42);
-  Expected<std::string> SubstValue = StringSubstitution.getResult();
+  Expected<std::string> SubstValue = StringSubstitution.getResultRegex();
   expectUndefErrors({"VAR404"}, SubstValue.takeError());
 
   // Numeric substitution blocks constituted of defined numeric variables are
@@ -1452,20 +1452,20 @@ TEST_F(FileCheckTest, Substitution) {
       std::move(NVarUse), ExpressionFormat(ExpressionFormat::Kind::HexUpper));
   NumericSubstitution SubstitutionN(&Context, "N", std::move(ExpressionN),
                                     /*InsertIdx=*/30);
-  SubstValue = SubstitutionN.getResult();
+  SubstValue = SubstitutionN.getResultRegex();
   ASSERT_THAT_EXPECTED(SubstValue, Succeeded());
   EXPECT_EQ("A", *SubstValue);
 
   // Substitution of an undefined numeric variable fails, error holds name of
   // undefined variable.
   NVar.clearValue();
-  SubstValue = SubstitutionN.getResult();
+  SubstValue = SubstitutionN.getResultRegex();
   expectUndefErrors({"N"}, SubstValue.takeError());
 
   // Substitution of a defined string variable returns the right value.
   Pattern P(Check::CheckPlain, &Context, 1);
   StringSubstitution = llvm::StringSubstitution(&Context, "FOO", 42);
-  SubstValue = StringSubstitution.getResult();
+  SubstValue = StringSubstitution.getResultRegex();
   ASSERT_THAT_EXPECTED(SubstValue, Succeeded());
   EXPECT_EQ("BAR", *SubstValue);
 }
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index 4048143b3681..281d509227a4 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -19,6 +19,8 @@ add_llvm_unittest(LLVMFrontendTests
   OpenMPParsingTest.cpp
   OpenMPCompositionTest.cpp
   OpenMPDecompositionTest.cpp
+  OpenMPDirectiveNameTest.cpp
+  OpenMPDirectiveNameParserTest.cpp
 
   DEPENDS
   acc_gen
diff --git a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
index dde82ac34ea2..98b33fdfb8c1 100644
--- a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
+++ b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignature.h"
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
@@ -17,7 +17,7 @@ TEST(HLSLRootSignatureTest, DescriptorCBVClauseDump) {
   DescriptorTableClause Clause;
   Clause.Type = ClauseType::CBuffer;
   Clause.Reg = {RegisterType::BReg, 0};
-  Clause.setDefaultFlags();
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -37,7 +37,7 @@ TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
   Clause.NumDescriptors = NumDescriptorsUnbounded;
   Clause.Space = 42;
   Clause.Offset = 3;
-  Clause.Flags = DescriptorRangeFlags::None;
+  Clause.Flags = llvm::dxbc::DescriptorRangeFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -50,13 +50,20 @@ TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
 }
 
 TEST(HLSLRootSignatureTest, DescriptorUAVClauseDump) {
+  using llvm::dxbc::DescriptorRangeFlags;
   DescriptorTableClause Clause;
   Clause.Type = ClauseType::UAV;
   Clause.Reg = {RegisterType::UReg, 92374};
   Clause.NumDescriptors = 3298;
   Clause.Space = 932847;
   Clause.Offset = 1;
-  Clause.Flags = DescriptorRangeFlags::ValidFlags;
+  auto ValidDescriptorRangeFlags =
+      DescriptorRangeFlags::DescriptorsVolatile |
+      DescriptorRangeFlags::DataVolatile |
+      DescriptorRangeFlags::DataStaticWhileSetAtExecute |
+      DescriptorRangeFlags::DataStatic |
+      DescriptorRangeFlags::DescriptorsStaticKeepingBufferBoundsChecks;
+  Clause.Flags = ValidDescriptorRangeFlags;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -80,7 +87,7 @@ TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
   Clause.NumDescriptors = 2;
   Clause.Space = 42;
   Clause.Offset = DescriptorTableOffsetAppend;
-  Clause.Flags = DescriptorRangeFlags::ValidSamplerFlags;
+  Clause.Flags = llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -93,10 +100,44 @@ TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
   EXPECT_EQ(Out, Expected);
 }
 
+TEST(HLSLRootSignatureTest, DescriptorCBVV10ClauseDump) {
+  DescriptorTableClause Clause;
+  Clause.Type = ClauseType::CBuffer;
+  Clause.Reg = {RegisterType::BReg, 0};
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Clause;
+  OS.flush();
+
+  std::string Expected = "CBV(b0, numDescriptors = 1, space = 0, "
+                         "offset = DescriptorTableOffsetAppend, "
+                         "flags = DescriptorsVolatile | DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
+TEST(HLSLRootSignatureTest, DescriptorSamplerV10ClauseDump) {
+  DescriptorTableClause Clause;
+  Clause.Type = ClauseType::Sampler;
+  Clause.Reg = {RegisterType::SReg, 0};
+  Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Clause;
+  OS.flush();
+
+  std::string Expected = "Sampler(s0, numDescriptors = 1, space = 0, offset = "
+                         "DescriptorTableOffsetAppend, "
+                         "flags = DescriptorsVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
 TEST(HLSLRootSignatureTest, DescriptorTableDump) {
   DescriptorTable Table;
   Table.NumClauses = 4;
-  Table.Visibility = ShaderVisibility::Geometry;
+  Table.Visibility = llvm::dxbc::ShaderVisibility::Geometry;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -112,7 +153,7 @@ TEST(HLSLRootSignatureTest, RootCBVDump) {
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::CBuffer;
   Descriptor.Reg = {RegisterType::BReg, 0};
-  Descriptor.setDefaultFlags();
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -125,13 +166,47 @@ TEST(HLSLRootSignatureTest, RootCBVDump) {
   EXPECT_EQ(Out, Expected);
 }
 
+TEST(HLSLRootSignatureTest, RootSRV10Dump) {
+  RootDescriptor Descriptor;
+  Descriptor.Type = DescriptorType::SRV;
+  Descriptor.Reg = {RegisterType::TReg, 0};
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Descriptor;
+  OS.flush();
+
+  std::string Expected = "RootSRV(t0, space = 0, "
+                         "visibility = All, "
+                         "flags = DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
+TEST(HLSLRootSignatureTest, RootUAVV10Dump) {
+  RootDescriptor Descriptor;
+  Descriptor.Type = DescriptorType::UAV;
+  Descriptor.Reg = {RegisterType::UReg, 0};
+  Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
+
+  std::string Out;
+  llvm::raw_string_ostream OS(Out);
+  OS << Descriptor;
+  OS.flush();
+
+  std::string Expected = "RootUAV(u0, space = 0, "
+                         "visibility = All, "
+                         "flags = DataVolatile)";
+  EXPECT_EQ(Out, Expected);
+}
+
 TEST(HLSLRootSignatureTest, RootSRVDump) {
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::SRV;
   Descriptor.Reg = {RegisterType::TReg, 0};
   Descriptor.Space = 42;
-  Descriptor.Visibility = ShaderVisibility::Geometry;
-  Descriptor.Flags = RootDescriptorFlags::None;
+  Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Geometry;
+  Descriptor.Flags = llvm::dxbc::RootDescriptorFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -144,12 +219,17 @@ TEST(HLSLRootSignatureTest, RootSRVDump) {
 }
 
 TEST(HLSLRootSignatureTest, RootUAVDump) {
+  using llvm::dxbc::RootDescriptorFlags;
   RootDescriptor Descriptor;
   Descriptor.Type = DescriptorType::UAV;
   Descriptor.Reg = {RegisterType::UReg, 92374};
   Descriptor.Space = 932847;
-  Descriptor.Visibility = ShaderVisibility::Hull;
-  Descriptor.Flags = RootDescriptorFlags::ValidFlags;
+  Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Hull;
+  auto ValidRootDescriptorFlags =
+      RootDescriptorFlags::DataVolatile |
+      RootDescriptorFlags::DataStaticWhileSetAtExecute |
+      RootDescriptorFlags::DataStatic;
+  Descriptor.Flags = ValidRootDescriptorFlags;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -194,18 +274,18 @@ TEST(HLSLRootSignatureTest, DefinedStaticSamplerDump) {
   StaticSampler Sampler;
   Sampler.Reg = {RegisterType::SReg, 0};
 
-  Sampler.Filter = SamplerFilter::ComparisonMinMagLinearMipPoint;
-  Sampler.AddressU = TextureAddressMode::Mirror;
-  Sampler.AddressV = TextureAddressMode::Border;
-  Sampler.AddressW = TextureAddressMode::Clamp;
+  Sampler.Filter = llvm::dxbc::SamplerFilter::ComparisonMinMagLinearMipPoint;
+  Sampler.AddressU = llvm::dxbc::TextureAddressMode::Mirror;
+  Sampler.AddressV = llvm::dxbc::TextureAddressMode::Border;
+  Sampler.AddressW = llvm::dxbc::TextureAddressMode::Clamp;
   Sampler.MipLODBias = 4.8f;
   Sampler.MaxAnisotropy = 32;
-  Sampler.CompFunc = ComparisonFunc::NotEqual;
-  Sampler.BorderColor = StaticBorderColor::OpaqueBlack;
+  Sampler.CompFunc = llvm::dxbc::ComparisonFunc::NotEqual;
+  Sampler.BorderColor = llvm::dxbc::StaticBorderColor::OpaqueBlack;
   Sampler.MinLOD = 1.0f;
   Sampler.MaxLOD = 32.0f;
   Sampler.Space = 7;
-  Sampler.Visibility = ShaderVisibility::Domain;
+  Sampler.Visibility = llvm::dxbc::ShaderVisibility::Domain;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -249,7 +329,7 @@ TEST(HLSLRootSignatureTest, SetRootConstantsDump) {
   Constants.Num32BitConstants = 983;
   Constants.Reg = {RegisterType::BReg, 34593};
   Constants.Space = 7;
-  Constants.Visibility = ShaderVisibility::Pixel;
+  Constants.Visibility = llvm::dxbc::ShaderVisibility::Pixel;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -262,7 +342,7 @@ TEST(HLSLRootSignatureTest, SetRootConstantsDump) {
 }
 
 TEST(HLSLRootSignatureTest, NoneRootFlagsDump) {
-  RootFlags Flags = RootFlags::None;
+  llvm::dxbc::RootFlags Flags = llvm::dxbc::RootFlags::None;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
@@ -274,11 +354,23 @@ TEST(HLSLRootSignatureTest, NoneRootFlagsDump) {
 }
 
 TEST(HLSLRootSignatureTest, AllRootFlagsDump) {
-  RootFlags Flags = RootFlags::ValidFlags;
+  using llvm::dxbc::RootFlags;
+  auto ValidRootFlags = RootFlags::AllowInputAssemblerInputLayout |
+                        RootFlags::DenyVertexShaderRootAccess |
+                        RootFlags::DenyHullShaderRootAccess |
+                        RootFlags::DenyDomainShaderRootAccess |
+                        RootFlags::DenyGeometryShaderRootAccess |
+                        RootFlags::DenyPixelShaderRootAccess |
+                        RootFlags::AllowStreamOutput |
+                        RootFlags::LocalRootSignature |
+                        RootFlags::DenyAmplificationShaderRootAccess |
+                        RootFlags::DenyMeshShaderRootAccess |
+                        RootFlags::CBVSRVUAVHeapDirectlyIndexed |
+                        RootFlags::SamplerHeapDirectlyIndexed;
 
   std::string Out;
   llvm::raw_string_ostream OS(Out);
-  OS << Flags;
+  OS << ValidRootFlags;
   OS.flush();
 
   std::string Expected = "RootFlags("
diff --git a/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp b/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
index 0ef6fe84f0ec..be3f51e0e83d 100644
--- a/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
+++ b/llvm/unittests/Frontend/HLSLRootSignatureRangesTest.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
+#include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
new file mode 100644
index 000000000000..0363a08cc0f0
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp
@@ -0,0 +1,171 @@
+//===- llvm/unittests/Frontend/OpenMPDirectiveNameParserTest.cpp ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/DirectiveNameParser.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "gtest/gtest.h"
+
+#include <cctype>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+using namespace llvm;
+
+static const omp::DirectiveNameParser &getParser() {
+  static omp::DirectiveNameParser Parser(omp::SourceLanguage::C |
+                                         omp::SourceLanguage::Fortran);
+  return Parser;
+}
+
+static std::vector<std::string> tokenize(StringRef S) {
+  std::vector<std::string> Tokens;
+
+  using TokenIterator = std::istream_iterator<std::string>;
+  std::string Copy = S.str();
+  std::istringstream Stream(Copy);
+
+  for (auto I = TokenIterator(Stream), E = TokenIterator(); I != E; ++I)
+    Tokens.push_back(*I);
+  return Tokens;
+}
+
+static std::string &prepareParamName(std::string &Name) {
+  for (size_t I = 0, E = Name.size(); I != E; ++I) {
+    // The parameter name must only have alphanumeric characters.
+    if (!isalnum(Name[I]))
+      Name[I] = 'X';
+  }
+  return Name;
+}
+
+namespace llvm {
+template <> struct enum_iteration_traits<omp::Directive> {
+  static constexpr bool is_iterable = true;
+};
+} // namespace llvm
+
+// Test tokenizing.
+
+class Tokenize : public testing::TestWithParam<omp::Directive> {};
+
+static bool isEqual(const SmallVector<StringRef> &A,
+                    const std::vector<std::string> &B) {
+  if (A.size() != B.size())
+    return false;
+
+  for (size_t I = 0, E = A.size(); I != E; ++I) {
+    if (A[I] != StringRef(B[I]))
+      return false;
+  }
+  return true;
+}
+
+TEST_P(Tokenize, T) {
+  omp::Directive DirId = GetParam();
+  StringRef Name = omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion);
+
+  SmallVector<StringRef> tokens1 = omp::DirectiveNameParser::tokenize(Name);
+  std::vector<std::string> tokens2 = tokenize(Name);
+  ASSERT_TRUE(isEqual(tokens1, tokens2));
+}
+
+static std::string
+getParamName1(const testing::TestParamInfo<Tokenize::ParamType> &Info) {
+  omp::Directive DirId = Info.param;
+  std::string Name =
+      omp::getOpenMPDirectiveName(DirId, omp::FallbackVersion).str();
+  return prepareParamName(Name);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    DirectiveNameParserTest, Tokenize,
+    testing::ValuesIn(
+        llvm::enum_seq(static_cast<omp::Directive>(0),
+                       static_cast<omp::Directive>(omp::Directive_enumSize))),
+    getParamName1);
+
+// Test parsing of valid names.
+
+using ValueType = std::tuple<omp::Directive, unsigned>;
+
+class ParseValid : public testing::TestWithParam<ValueType> {};
+
+TEST_P(ParseValid, T) {
+  auto [DirId, Version] = GetParam();
+  if (DirId == omp::Directive::OMPD_unknown)
+    return;
+
+  std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str();
+
+  // Tokenize and parse
+  auto &Parser = getParser();
+  auto *State = Parser.initial();
+  ASSERT_TRUE(State != nullptr);
+
+  std::vector<std::string> Tokens = tokenize(Name);
+  for (auto &Tok : Tokens) {
+    State = Parser.consume(State, Tok);
+    ASSERT_TRUE(State != nullptr);
+  }
+
+  ASSERT_EQ(State->Value, DirId);
+}
+
+static std::string
+getParamName2(const testing::TestParamInfo<ParseValid::ParamType> &Info) {
+  auto [DirId, Version] = Info.param;
+  std::string Name = omp::getOpenMPDirectiveName(DirId, Version).str() + "v" +
+                     std::to_string(Version);
+  return prepareParamName(Name);
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    DirectiveNameParserTest, ParseValid,
+    testing::Combine(testing::ValuesIn(llvm::enum_seq(
+                         static_cast<omp::Directive>(0),
+                         static_cast<omp::Directive>(omp::Directive_enumSize))),
+                     testing::ValuesIn(omp::getOpenMPVersions())),
+    getParamName2);
+
+// Test parsing of invalid names
+
+class ParseInvalid : public testing::TestWithParam<std::string> {};
+
+TEST_P(ParseInvalid, T) {
+  std::string Name = GetParam();
+
+  auto &Parser = getParser();
+  auto *State = Parser.initial();
+  ASSERT_TRUE(State != nullptr);
+
+  std::vector<std::string> Tokens = tokenize(Name);
+  for (auto &Tok : Tokens)
+    State = Parser.consume(State, Tok);
+
+  ASSERT_TRUE(State == nullptr || State->Value == omp::Directive::OMPD_unknown);
+}
+
+namespace {
+using namespace std;
+
+INSTANTIATE_TEST_SUITE_P(DirectiveNameParserTest, ParseInvalid,
+                         testing::Values(
+                             // Names that contain invalid tokens
+                             "bad"s, "target teams invalid"s,
+                             "target sections parallel"s,
+                             "target teams distribute parallel for wrong"s,
+                             // Valid beginning, but not a complete name
+                             "begin declare"s,
+                             // Complete name with extra tokens
+                             "distribute simd target"s));
+} // namespace
diff --git a/llvm/unittests/Frontend/OpenMPDirectiveNameTest.cpp b/llvm/unittests/Frontend/OpenMPDirectiveNameTest.cpp
new file mode 100644
index 000000000000..da648157ee9b
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenMPDirectiveNameTest.cpp
@@ -0,0 +1,96 @@
+//===- llvm/unittests/Frontend/OpenMPDirectiveNameTest.cpp ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Frontend/OpenMP/OMP.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::omp;
+
+const DenseMap<Directive, StringRef> &Expected52() {
+  static const DenseMap<Directive, StringRef> Names{
+      {OMPD_begin_declare_target, "begin declare target"},
+      {OMPD_begin_declare_variant, "begin declare variant"},
+      {OMPD_cancellation_point, "cancellation point"},
+      {OMPD_declare_mapper, "declare mapper"},
+      {OMPD_declare_reduction, "declare reduction"},
+      {OMPD_declare_simd, "declare simd"},
+      {OMPD_declare_target, "declare target"},
+      {OMPD_declare_variant, "declare variant"},
+      {OMPD_end_declare_target, "end declare target"},
+      {OMPD_end_declare_variant, "end declare variant"},
+      {OMPD_target_data, "target data"},
+      {OMPD_target_enter_data, "target enter data"},
+      {OMPD_target_exit_data, "target exit data"},
+      {OMPD_target_update, "target update"},
+  };
+  return Names;
+}
+
+const DenseMap<Directive, StringRef> &Expected60() {
+  static const DenseMap<Directive, StringRef> Names{
+      {OMPD_begin_declare_target, "begin declare_target"},
+      {OMPD_begin_declare_variant, "begin declare_variant"},
+      {OMPD_cancellation_point, "cancellation_point"},
+      {OMPD_declare_mapper, "declare_mapper"},
+      {OMPD_declare_reduction, "declare_reduction"},
+      {OMPD_declare_simd, "declare_simd"},
+      {OMPD_declare_target, "declare_target"},
+      {OMPD_declare_variant, "declare_variant"},
+      {OMPD_end_declare_target, "end declare_target"},
+      {OMPD_end_declare_variant, "end declare_variant"},
+      {OMPD_target_data, "target_data"},
+      {OMPD_target_enter_data, "target_enter_data"},
+      {OMPD_target_exit_data, "target_exit_data"},
+      {OMPD_target_update, "target_update"},
+  };
+  return Names;
+}
+
+class VersionTest : public testing::TestWithParam<unsigned> {
+public:
+  void SetUp() override {
+    Version = GetParam();
+
+    if (Version < 60)
+      KindToName = &Expected52();
+    else
+      KindToName = &Expected60();
+  }
+
+  const DenseMap<Directive, StringRef> *KindToName;
+  unsigned Version;
+};
+
+INSTANTIATE_TEST_SUITE_P(OpenMPDirectiveNames, VersionTest,
+                         testing::ValuesIn(getOpenMPVersions()));
+
+TEST_P(VersionTest, DirectiveName) {
+  for (auto [Kind, Name] : *KindToName)
+    ASSERT_EQ(Name, getOpenMPDirectiveName(Kind, Version));
+}
+
+TEST(OpenMPDirectiveNames, DirectiveKind52) {
+  for (auto [Kind, Name] : Expected52()) {
+    auto [K, R] = getOpenMPDirectiveKindAndVersions(Name);
+    ASSERT_EQ(K, Kind);
+    // Expect the name to be valid in 5.2, but not in 6.0.
+    EXPECT_TRUE(52 <= R.Max && R.Max < 60);
+  }
+}
+
+TEST(OpenMPDirectiveNames, DirectiveKind60) {
+  for (auto [Kind, Name] : Expected60()) {
+    auto [K, R] = getOpenMPDirectiveKindAndVersions(Name);
+    ASSERT_EQ(K, Kind);
+    // Expect the name to be valid in 6.0 and later.
+    EXPECT_TRUE(60 <= R.Min);
+  }
+}
diff --git a/llvm/unittests/FuzzMutate/StrategiesTest.cpp b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
index d140aa159a3e..a7d8b6c33608 100644
--- a/llvm/unittests/FuzzMutate/StrategiesTest.cpp
+++ b/llvm/unittests/FuzzMutate/StrategiesTest.cpp
@@ -89,9 +89,12 @@ void IterateOnSource(StringRef Source, IRMutator &Mutator) {
   }
 }
 
-static void mutateAndVerifyModule(StringRef Source,
-                                  std::unique_ptr<IRMutator> &Mutator,
-                                  int repeat = 100) {
+using ModuleVerifier = std::function<void(Module &)>;
+
+static void
+mutateAndVerifyModule(StringRef Source, std::unique_ptr<IRMutator> &Mutator,
+                      int repeat = 100,
+                      ArrayRef<ModuleVerifier> ExtraModuleVerifiers = {}) {
   LLVMContext Ctx;
   auto M = parseAssembly(Source.data(), Ctx);
   std::mt19937 mt(Seed);
@@ -99,13 +102,19 @@ static void mutateAndVerifyModule(StringRef Source,
   for (int i = 0; i < repeat; i++) {
     Mutator->mutateModule(*M, RandInt(mt), IRMutator::getModuleSize(*M) + 1024);
     ASSERT_FALSE(verifyModule(*M, &errs()));
+    for (auto &ModuleVerifier : ExtraModuleVerifiers) {
+      ModuleVerifier(*M);
+    }
   }
 }
+
 template <class Strategy>
-static void mutateAndVerifyModule(StringRef Source, int repeat = 100) {
+static void
+mutateAndVerifyModule(StringRef Source, int repeat = 100,
+                      ArrayRef<ModuleVerifier> ExtraModuleVerifiers = {}) {
   auto Mutator = createMutator<Strategy>();
   ASSERT_TRUE(Mutator);
-  mutateAndVerifyModule(Source, Mutator, repeat);
+  mutateAndVerifyModule(Source, Mutator, repeat, ExtraModuleVerifiers);
 }
 
 TEST(InjectorIRStrategyTest, EmptyModule) {
@@ -745,4 +754,53 @@ TEST(AllStrategies, SkipEHPad) {
   mutateAndVerifyModule<InjectorIRStrategy>(Source);
   mutateAndVerifyModule<InstModificationIRStrategy>(Source);
 }
+
+TEST(AllStrategies, SpecialTerminator) {
+  StringRef Source = "\n\
+    declare amdgpu_cs_chain void @callee(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })\n\
+    define amdgpu_cs_chain void @chain_to_chain(<3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr) {\n\
+      call void(ptr, i64, <3 x i32>, { i32, ptr addrspace(5), i32, i32 }, i32, ...) @llvm.amdgcn.cs.chain(ptr @callee, i64 -1, <3 x i32> inreg %sgpr, { i32, ptr addrspace(5), i32, i32 } %vgpr, i32 0) \n\
+      unreachable\n\
+    }\n\
+  ";
+  mutateAndVerifyModule<InjectorIRStrategy>(Source);
+  mutateAndVerifyModule<InsertCFGStrategy>(Source);
+  mutateAndVerifyModule<InsertFunctionStrategy>(Source);
+  mutateAndVerifyModule<InsertPHIStrategy>(Source);
+  mutateAndVerifyModule<InstModificationIRStrategy>(Source);
+  mutateAndVerifyModule<ShuffleBlockStrategy>(Source);
+  mutateAndVerifyModule<SinkInstructionStrategy>(Source);
+}
+
+TEST(AllStrategies, AMDGCNLegalAddrspace) {
+  StringRef Source = "\n\
+    target triple = \"amdgcn-amd-amdhsa\"\n\
+    ; minimum values required by the fuzzer (e.g., default addrspace for allocas and globals)\n\
+    target datalayout = \"A5-G1\"\n\
+    define amdgpu_gfx void @strict_wwm_amdgpu_cs_main(<4 x i32> inreg %desc, i32 %index) {\n\
+    %desc.int = bitcast <4 x i32> %desc to i128\n\
+    %desc.ptr = inttoptr i128 %desc.int to ptr addrspace(8)\n\
+    ret void\n\
+  }\n\
+  ";
+
+  ModuleVerifier AddrSpaceVerifier = [](Module &M) {
+    Function *F = M.getFunction("strict_wwm_amdgpu_cs_main");
+    EXPECT_TRUE(F != nullptr);
+    for (BasicBlock &BB : *F) {
+      for (Instruction &I : BB) {
+        if (StoreInst *S = dyn_cast<StoreInst>(&I)) {
+          EXPECT_TRUE(S->getPointerAddressSpace() != 8);
+        } else if (LoadInst *L = dyn_cast<LoadInst>(&I)) {
+          EXPECT_TRUE(L->getPointerAddressSpace() != 8);
+        }
+      }
+    }
+  };
+
+  int Repeat = 100;
+  mutateAndVerifyModule<SinkInstructionStrategy>(Source, Repeat,
+                                                 {AddrSpaceVerifier});
+}
+
 } // namespace
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index cb1a56a26f06..aac0f96845f2 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -164,7 +164,7 @@ TEST(BasicBlockDbgInfoTest, DropSourceAtomOnSplit) {
     !1 = !DIFile(filename: "dummy", directory: "dummy")
     !2 = !{i32 7, !"Dwarf Version", i32 5}
     !3 = !{i32 2, !"Debug Info Version", i32 3}
-    !10 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13)
+    !10 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 1, type: !11, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13, keyInstructions: true)
     !11 = !DISubroutineType(types: !12)
     !12 = !{null}
     !13 = !{}
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index ac7914fe6b9f..b2fdd8604ba0 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -1302,4 +1302,33 @@ TEST(DIBuilder, CompositeTypes) {
   EXPECT_EQ(Enum->getTag(), dwarf::DW_TAG_enumeration_type);
 }
 
+TEST(DIBuilder, DynamicOffsetAndSize) {
+  LLVMContext Ctx;
+  auto M = std::make_unique<Module>("MyModule", Ctx);
+  DIBuilder DIB(*M);
+  DIScope *Scope = DISubprogram::getDistinct(
+      Ctx, nullptr, "", "", nullptr, 0, nullptr, 0, nullptr, 0, 0,
+      DINode::FlagZero, DISubprogram::SPFlagZero, nullptr);
+  DIFile *F = DIB.createFile("main.adb", "/");
+
+  DIVariable *Len = DIB.createAutoVariable(Scope, "length", F, 0, nullptr,
+                                           false, DINode::FlagZero, dwarf::DW_MSPACE_LLVM_none, 0);
+
+  DICompositeType *Struct = DIB.createStructType(
+      Scope, "some_record", F, 18, Len, 8, DINode::FlagZero, nullptr, {});
+  EXPECT_EQ(Struct->getTag(), dwarf::DW_TAG_structure_type);
+
+  SmallVector<uint64_t, 4> ops;
+  ops.push_back(llvm::dwarf::DW_OP_push_object_address);
+  DIExpression::appendOffset(ops, 3);
+  ops.push_back(llvm::dwarf::DW_OP_deref);
+  DIExpression *Expr = DIB.createExpression(ops);
+
+  DIDerivedType *Field = DIB.createMemberType(Scope, "field", F, 23, Len, 0,
+                                              Expr, DINode::FlagZero, Struct);
+
+  EXPECT_EQ(Field->getRawOffsetInBits(), Expr);
+  EXPECT_EQ(Field->getRawSizeInBits(), Len);
+}
+
 } // end namespace
diff --git a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
index e1ce671852c8..6716796e71c5 100644
--- a/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
+++ b/llvm/unittests/IR/DebugTypeODRUniquingTest.cpp
@@ -141,12 +141,12 @@ TEST(DebugTypeODRUniquingTest, buildODRTypeFields) {
   DO_FOR_FIELD(BaseType)                                                       \
   DO_FOR_FIELD(Elements)                                                       \
   DO_FOR_FIELD(VTableHolder)                                                   \
-  DO_FOR_FIELD(TemplateParams)
+  DO_FOR_FIELD(TemplateParams)                                                 \
+  DO_FOR_FIELD(SizeInBits)                                                     \
+  DO_FOR_FIELD(OffsetInBits)
 #define FOR_EACH_INLINEFIELD()                                                 \
   DO_FOR_FIELD(Line)                                                           \
-  DO_FOR_FIELD(SizeInBits)                                                     \
   DO_FOR_FIELD(AlignInBits)                                                    \
-  DO_FOR_FIELD(OffsetInBits)                                                   \
   DO_FOR_FIELD(NumExtraInhabitants)                                            \
   DO_FOR_FIELD(RuntimeLang)                                                    \
   DO_FOR_FIELD(EnumKind)
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 520735dfc326..4f2ede332108 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -920,9 +920,11 @@ TEST_F(IRBuilderTest, DIBuilder) {
     // --------------------------
     DILocation *LabelLoc = DILocation::get(Ctx, 1, 0, BarScope);
     DILabel *AlwaysPreserveLabel = DIB.createLabel(
-        BarScope, "meles_meles", File, 1, /*AlwaysPreserve*/ true);
-    DILabel *Label =
-        DIB.createLabel(BarScope, "badger", File, 1, /*AlwaysPreserve*/ false);
+        BarScope, "meles_meles", File, 1, /*Column*/ 0, /*IsArtificial*/ false,
+        /*CoroSuspendIdx*/ std::nullopt, /*AlwaysPreserve*/ true);
+    DILabel *Label = DIB.createLabel(
+        BarScope, "badger", File, 1, /*Column*/ 0, /*IsArtificial*/ false,
+        /*CoroSuspendIdx*/ std::nullopt, /*AlwaysPreserve*/ false);
 
     { /* dbg.label | DbgLabelRecord */
       // Insert before I and check order.
diff --git a/llvm/unittests/IR/LegacyPassManagerTest.cpp b/llvm/unittests/IR/LegacyPassManagerTest.cpp
index 71241e44831e..00ceebaff416 100644
--- a/llvm/unittests/IR/LegacyPassManagerTest.cpp
+++ b/llvm/unittests/IR/LegacyPassManagerTest.cpp
@@ -359,7 +359,8 @@ namespace llvm {
     struct CustomOptPassGate : public OptPassGate {
       bool Skip;
       CustomOptPassGate(bool Skip) : Skip(Skip) { }
-      bool shouldRunPass(const StringRef PassName, StringRef IRDescription) override {
+      bool shouldRunPass(StringRef PassName,
+                         StringRef IRDescription) const override {
         return !Skip;
       }
       bool isEnabled() const override { return true; }
diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp
index c788a80a560a..9f08c8a5c460 100644
--- a/llvm/unittests/IR/MetadataTest.cpp
+++ b/llvm/unittests/IR/MetadataTest.cpp
@@ -3034,12 +3034,13 @@ TEST_F(DISubprogramTest, get) {
   assert(!IsLocalToUnit && IsDefinition && !IsOptimized &&
          "bools and SPFlags have to match");
   SPFlags |= DISubprogram::SPFlagDefinition;
+  bool KeyInstructions = false;
 
   auto *N = DISubprogram::get(
       Context, Scope, Name, LinkageName, File, Line, Type, ScopeLine,
       ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags, Unit,
       TemplateParams, Declaration, RetainedNodes, ThrownTypes, Annotations,
-      TargetFuncName);
+      TargetFuncName, KeyInstructions);
 
   EXPECT_EQ(dwarf::DW_TAG_subprogram, N->getTag());
   EXPECT_EQ(Scope, N->getScope());
@@ -3064,125 +3065,138 @@ TEST_F(DISubprogramTest, get) {
   EXPECT_EQ(ThrownTypes, N->getThrownTypes().get());
   EXPECT_EQ(Annotations, N->getAnnotations().get());
   EXPECT_EQ(TargetFuncName, N->getTargetFuncName());
+  EXPECT_EQ(KeyInstructions, N->getKeyInstructionsEnabled());
   EXPECT_EQ(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
 
   EXPECT_NE(N, DISubprogram::get(Context, getCompositeType(), Name, LinkageName,
                                  File, Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, "other", LinkageName, File,
                                  Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
-  EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, "other", File, Line,
-                                 Type, ScopeLine, ContainingType, VirtualIndex,
-                                 ThisAdjustment, Flags, SPFlags, Unit,
-                                 TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
+  EXPECT_NE(N, DISubprogram::get(
+                   Context, Scope, Name, "other", File, Line, Type, ScopeLine,
+                   ContainingType, VirtualIndex, ThisAdjustment, Flags, SPFlags,
+                   Unit, TemplateParams, Declaration, RetainedNodes,
+                   ThrownTypes, Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, getFile(),
                                  Line, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File,
                                  Line + 1, Type, ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  getSubroutineType(), ScopeLine, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagLocalToUnit, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagDefinition, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine + 1, ContainingType,
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, getCompositeType(),
                                  VirtualIndex, ThisAdjustment, Flags, SPFlags,
                                  Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagVirtual, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType,
                                  VirtualIndex + 1, ThisAdjustment, Flags,
                                  SPFlags, Unit, TemplateParams, Declaration,
                                  RetainedNodes, ThrownTypes, Annotations,
-                                 TargetFuncName));
+                                 TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(
                    Context, Scope, Name, LinkageName, File, Line, Type,
                    ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
                    Flags, SPFlags ^ DISubprogram::SPFlagOptimized, Unit,
                    TemplateParams, Declaration, RetainedNodes, ThrownTypes,
-                   Annotations, TargetFuncName));
+                   Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, nullptr,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
-  EXPECT_NE(N,
-            DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
-                              Type, ScopeLine, ContainingType, VirtualIndex,
-                              ThisAdjustment, Flags, SPFlags, Unit, getTuple(),
-                              Declaration, RetainedNodes, ThrownTypes,
-                              Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
+  EXPECT_NE(N, DISubprogram::get(
+                   Context, Scope, Name, LinkageName, File, Line, Type,
+                   ScopeLine, ContainingType, VirtualIndex, ThisAdjustment,
+                   Flags, SPFlags, Unit, getTuple(), Declaration, RetainedNodes,
+                   ThrownTypes, Annotations, TargetFuncName, KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, getSubprogram(), RetainedNodes,
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, getTuple(),
-                                 ThrownTypes, Annotations, TargetFuncName));
+                                 ThrownTypes, Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 getTuple(), Annotations, TargetFuncName));
+                                 getTuple(), Annotations, TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, getTuple(), TargetFuncName));
+                                 ThrownTypes, getTuple(), TargetFuncName,
+                                 KeyInstructions));
   EXPECT_NE(N, DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
                                  Type, ScopeLine, ContainingType, VirtualIndex,
                                  ThisAdjustment, Flags, SPFlags, Unit,
                                  TemplateParams, Declaration, RetainedNodes,
-                                 ThrownTypes, Annotations, "other"));
+                                 ThrownTypes, Annotations, "other",
+                                 KeyInstructions));
+  EXPECT_NE(N,
+            DISubprogram::get(Context, Scope, Name, LinkageName, File, Line,
+                              Type, ScopeLine, ContainingType, VirtualIndex,
+                              ThisAdjustment, Flags, SPFlags, Unit,
+                              TemplateParams, Declaration, RetainedNodes,
+                              ThrownTypes, Annotations, TargetFuncName, true));
 
   TempDISubprogram Temp = N->clone();
   EXPECT_EQ(N, MDNode::replaceWithUniqued(std::move(Temp)));
diff --git a/llvm/unittests/Object/DXContainerTest.cpp b/llvm/unittests/Object/DXContainerTest.cpp
index 28012073d3c7..396d060a75bf 100644
--- a/llvm/unittests/Object/DXContainerTest.cpp
+++ b/llvm/unittests/Object/DXContainerTest.cpp
@@ -1062,8 +1062,8 @@ TEST(RootSignature, ParseDescriptorTable) {
         0x3c, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x03, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
         0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
-        0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00,
-        0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+        0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
         0x00};
     DXContainer C =
@@ -1088,7 +1088,7 @@ TEST(RootSignature, ParseDescriptorTable) {
     auto *DescriptorTableView =
         dyn_cast<DirectX::DescriptorTableView>(&*ParamView);
     ASSERT_TRUE(DescriptorTableView != nullptr);
-    auto Table = DescriptorTableView->read(2);
+    auto Table = DescriptorTableView->read<dxbc::RTS0::v2::DescriptorRange>();
 
     ASSERT_THAT_ERROR(Table.takeError(), Succeeded());
 
@@ -1140,7 +1140,7 @@ TEST(RootSignature, ParseDescriptorTable) {
     auto *DescriptorTableView =
         dyn_cast<DirectX::DescriptorTableView>(&*ParamView);
     ASSERT_TRUE(DescriptorTableView != nullptr);
-    auto Table = DescriptorTableView->read(1);
+    auto Table = DescriptorTableView->read<dxbc::RTS0::v1::DescriptorRange>();
 
     ASSERT_THAT_ERROR(Table.takeError(), Succeeded());
 
diff --git a/llvm/unittests/Object/ELFObjectFileTest.cpp b/llvm/unittests/Object/ELFObjectFileTest.cpp
index 423f92ea07b3..21606fff1f32 100644
--- a/llvm/unittests/Object/ELFObjectFileTest.cpp
+++ b/llvm/unittests/Object/ELFObjectFileTest.cpp
@@ -737,7 +737,7 @@ TEST(ELFObjectFileTest, ReadBBAddrMap) {
     Type: SHT_LLVM_BB_ADDR_MAP
     Link: 2
     Entries:
-      - Version: 1
+      - Version: 2
         BBRanges:
           - BaseAddress: 0x33333
             BBEntries:
@@ -904,7 +904,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
   SmallString<128> UnsupportedLowVersionYamlString(CommonYamlString);
   UnsupportedLowVersionYamlString += R"(
       - Version: 1
-        Feature: 0x4
+        Feature: 0x0
         BBRanges:
           - BBEntries:
               - AddressOffset: 0x0
@@ -915,8 +915,7 @@ TEST(ELFObjectFileTest, InvalidDecodePGOAnalysisMap) {
   {
     SCOPED_TRACE("unsupported version");
     DoCheck(UnsupportedLowVersionYamlString,
-            "version should be >= 2 for SHT_LLVM_BB_ADDR_MAP when PGO features "
-            "are enabled: version = 1 feature = 4");
+            "unsupported SHT_LLVM_BB_ADDR_MAP version: 1");
   }
 
   // Check that we fail when function entry count is enabled but not provided.
diff --git a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
index 879712f833e1..9e6f7cc4f7db 100644
--- a/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
+++ b/llvm/unittests/ObjectYAML/DXContainerYAMLTest.cpp
@@ -459,8 +459,8 @@ TEST(RootSignature, ParseDescriptorTableV11) {
       0x3c, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x03, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
       0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
-      0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00,
-      0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x2a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+      0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
       0x00};
 
diff --git a/llvm/unittests/ProfileData/InstrProfTest.cpp b/llvm/unittests/ProfileData/InstrProfTest.cpp
index dcdacb903791..6467695355be 100644
--- a/llvm/unittests/ProfileData/InstrProfTest.cpp
+++ b/llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -135,7 +135,7 @@ TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("foo", 0x1234);
+  auto R = Reader->getInstrProfRecord("foo", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
   ASSERT_EQ(2U, R->Counts.size());
   ASSERT_EQ(1U, R->Counts[0]);
@@ -251,7 +251,7 @@ TEST_F(InstrProfTest, test_writer_merge) {
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
+  auto R = Reader->getInstrProfRecord("func1", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
   ASSERT_EQ(1U, R->Counts.size());
   ASSERT_EQ(42U, R->Counts[0]);
@@ -600,7 +600,7 @@ TEST_F(InstrProfTest, test_memprof_merge) {
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("func1", 0x1234);
+  auto R = Reader->getInstrProfRecord("func1", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
   ASSERT_EQ(1U, R->Counts.size());
   ASSERT_EQ(42U, R->Counts[0]);
@@ -800,7 +800,7 @@ TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) {
   // Set reader value prof data endianness.
   Reader->setValueProfDataEndianness(getEndianness());
 
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+  auto R = Reader->getInstrProfRecord("caller", 0x1234);
   ASSERT_THAT_ERROR(R.takeError(), Succeeded());
 
   // Test the number of instrumented indirect call sites and the number of
@@ -874,7 +874,7 @@ TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {
   Writer.addRecord(std::move(Record), Err);
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+  auto R = Reader->getInstrProfRecord("caller", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
 
   LLVMContext Ctx;
@@ -1051,7 +1051,7 @@ TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) {
 
   // Test the number of instrumented value sites and the number of profiled
   // values for each site.
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+  auto R = Reader->getInstrProfRecord("caller", 0x1234);
   EXPECT_THAT_ERROR(R.takeError(), Succeeded());
   // For indirect calls.
   ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));
@@ -1190,13 +1190,11 @@ TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_saturation) {
   readProfile(std::move(Profile));
 
   // Verify saturation of counts.
-  Expected<InstrProfRecord> ReadRecord1 =
-      Reader->getInstrProfRecord("foo", 0x1234);
+  auto ReadRecord1 = Reader->getInstrProfRecord("foo", 0x1234);
   ASSERT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());
   EXPECT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]);
 
-  Expected<InstrProfRecord> ReadRecord2 =
-      Reader->getInstrProfRecord("baz", 0x5678);
+  auto ReadRecord2 = Reader->getInstrProfRecord("baz", 0x5678);
   ASSERT_TRUE(bool(ReadRecord2));
   ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind));
   auto VD = ReadRecord2->getValueArrayForSite(ValueKind, 0);
@@ -1241,7 +1239,7 @@ TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_site_trunc) {
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
-  Expected<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+  auto R = Reader->getInstrProfRecord("caller", 0x1234);
   ASSERT_THAT_ERROR(R.takeError(), Succeeded());
   ASSERT_EQ(2U, R->getNumValueSites(ValueKind));
   auto VD = R->getValueArrayForSite(ValueKind, 0);
diff --git a/llvm/unittests/SandboxIR/UtilsTest.cpp b/llvm/unittests/SandboxIR/UtilsTest.cpp
index a30fc253a1a7..2bef48616beb 100644
--- a/llvm/unittests/SandboxIR/UtilsTest.cpp
+++ b/llvm/unittests/SandboxIR/UtilsTest.cpp
@@ -89,7 +89,7 @@ define void @foo(ptr %ptr) {
 )IR");
   llvm::Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   AssumptionCache AC(LLVMF);
diff --git a/llvm/unittests/Support/Base64Test.cpp b/llvm/unittests/Support/Base64Test.cpp
index e6e54bb53098..1c4766f2d393 100644
--- a/llvm/unittests/Support/Base64Test.cpp
+++ b/llvm/unittests/Support/Base64Test.cpp
@@ -31,7 +31,7 @@ void TestBase64Decode(StringRef Input, StringRef Expected,
   if (ExpectedErrorMessage.empty()) {
     ASSERT_THAT_ERROR(decodeBase64(Input, DecodedBytes), Succeeded());
     EXPECT_EQ(llvm::ArrayRef<char>(DecodedBytes),
-              llvm::ArrayRef<char>(Expected.data(), Expected.size()));
+              llvm::ArrayRef<char>(Expected));
   } else {
     ASSERT_THAT_ERROR(decodeBase64(Input, DecodedBytes),
                       FailedWithMessage(ExpectedErrorMessage));
diff --git a/llvm/unittests/Support/ParallelTest.cpp b/llvm/unittests/Support/ParallelTest.cpp
index ccf5f2bb9132..041067d06888 100644
--- a/llvm/unittests/Support/ParallelTest.cpp
+++ b/llvm/unittests/Support/ParallelTest.cpp
@@ -67,17 +67,17 @@ TEST(Parallel, TransformReduce) {
 
   // Check that we handle non-divisible task sizes as above.
   uint32_t range[2050];
-  std::fill(std::begin(range), std::end(range), 1);
+  llvm::fill(range, 1);
   sum = parallelTransformReduce(range, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 2050U);
 
-  std::fill(std::begin(range), std::end(range), 2);
+  llvm::fill(range, 2);
   sum = parallelTransformReduce(range, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 4100U);
 
   // Avoid one large task.
   uint32_t range2[3060];
-  std::fill(std::begin(range2), std::end(range2), 1);
+  llvm::fill(range2, 1);
   sum = parallelTransformReduce(range2, 0U, std::plus<uint32_t>(), identity);
   EXPECT_EQ(sum, 3060U);
 }
diff --git a/llvm/unittests/Support/RISCVAttributeParserTest.cpp b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
index 777dc4d0f4d4..7e862822fdd7 100644
--- a/llvm/unittests/Support/RISCVAttributeParserTest.cpp
+++ b/llvm/unittests/Support/RISCVAttributeParserTest.cpp
@@ -45,7 +45,7 @@ static bool testAttribute(unsigned Tag, unsigned Value, unsigned ExpectedTag,
   cantFail(Parser.parse(Bytes, llvm::endianness::little));
 
   std::optional<unsigned> Attr = Parser.getAttributeValue("", ExpectedTag);
-  return Attr && *Attr == ExpectedValue;
+  return Attr == ExpectedValue;
 }
 
 static bool testTagString(unsigned Tag, const char *name) {
diff --git a/llvm/unittests/Support/TrailingObjectsTest.cpp b/llvm/unittests/Support/TrailingObjectsTest.cpp
index 2590f375b659..9184a4dd0cc2 100644
--- a/llvm/unittests/Support/TrailingObjectsTest.cpp
+++ b/llvm/unittests/Support/TrailingObjectsTest.cpp
@@ -45,9 +45,10 @@ class Class1 final : private TrailingObjects<Class1, short> {
   template <typename... Ty>
   using FixedSizeStorage = TrailingObjects::FixedSizeStorage<Ty...>;
 
-  using TrailingObjects::totalSizeToAlloc;
   using TrailingObjects::additionalSizeToAlloc;
   using TrailingObjects::getTrailingObjects;
+  using TrailingObjects::getTrailingObjectsNonStrict;
+  using TrailingObjects::totalSizeToAlloc;
 };
 
 // Here, there are two singular optional object types appended. Note
@@ -123,11 +124,11 @@ TEST(TrailingObjects, OneArg) {
   EXPECT_EQ(Class1::totalSizeToAlloc<short>(3),
             sizeof(Class1) + sizeof(short) * 3);
 
-  EXPECT_EQ(C->getTrailingObjects<short>(), reinterpret_cast<short *>(C + 1));
+  EXPECT_EQ(C->getTrailingObjects(), reinterpret_cast<short *>(C + 1));
   EXPECT_EQ(C->get(0), 1);
   EXPECT_EQ(C->get(2), 3);
 
-  EXPECT_EQ(C->getTrailingObjects(), C->getTrailingObjects<short>());
+  EXPECT_EQ(C->getTrailingObjects(), C->getTrailingObjectsNonStrict<short>());
 
   delete C;
 }
diff --git a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
index 0e6c1172ee9b..2744dbc0907b 100644
--- a/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
+++ b/llvm/unittests/Target/RISCV/RISCVInstrInfoTest.cpp
@@ -374,9 +374,9 @@ TEST_P(RISCVInstrInfoTest, GetDestEEW) {
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VMSEQ_VV), 4), 0u);
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VMAND_MM), 0), 0u);
   EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VIOTA_M), 3), 3u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VQMACCU_2x8x2), 3), 5u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::VFWMACC_4x4x4), 4), 5u);
-  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::THVdotVMAQA_VV), 5), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::SF_VQMACCU_2x8x2), 3), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::SF_VFWMACC_4x4x4), 4), 5u);
+  EXPECT_EQ(RISCV::getDestLog2EEW(TII->get(RISCV::TH_VMAQA_VV), 5), 5u);
 }
 
 } // namespace
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index c13d33e32c6d..0a9ac9bb0596 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -122,6 +122,14 @@ TEST(getLinuxHostCPUName, AArch64) {
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
                                               "CPU part        : 0xd48"),
             "cortex-x2");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
+                                              "CPU part        : 0xd85\n"
+                                              "CPU part        : 0xd87"),
+            "cortex-x925");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
+                                              "CPU part        : 0xd87\n"
+                                              "CPU part        : 0xd85"),
+            "cortex-x925");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
                                               "CPU part        : 0xc00"),
             "falkor");
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 66e335a33a3f..9780eba5dc7a 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1133,6 +1133,7 @@ R"(All available -march extensions for RISC-V
     xandesvbfhcvt        5.0
     xandesvdot           5.0
     xandesvpackfph       5.0
+    xandesvsintload      5.0
     xcvalu               1.0
     xcvbi                1.0
     xcvbitmanip          1.0
@@ -1140,6 +1141,7 @@ R"(All available -march extensions for RISC-V
     xcvmac               1.0
     xcvmem               1.0
     xcvsimd              1.0
+    xmipscbop            1.0
     xmipscmov            1.0
     xmipslsp             1.0
     xsfcease             1.0
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index c4efb991ab6f..1d7ad8ac075f 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1164,6 +1164,7 @@ INSTANTIATE_TEST_SUITE_P(
                       AArch64CPUTestParams("a64fx", "armv8.2-a"),
                       AArch64CPUTestParams("fujitsu-monaka", "armv9.3-a"),
                       AArch64CPUTestParams("carmel", "armv8.2-a"),
+                      AArch64CPUTestParams("gb10", "armv9.2-a"),
                       AArch64CPUTestParams("grace", "armv9-a"),
                       AArch64CPUTestParams("olympus", "armv9.2-a"),
                       AArch64CPUTestParams("saphira", "armv8.4-a"),
@@ -1260,7 +1261,7 @@ INSTANTIATE_TEST_SUITE_P(
     AArch64CPUAliasTestParams::PrintToStringParamName);
 
 // Note: number of CPUs includes aliases.
-static constexpr unsigned NumAArch64CPUArchs = 90;
+static constexpr unsigned NumAArch64CPUArchs = 91;
 
 TEST(TargetParserTest, testAArch64CPUArchList) {
   SmallVector<StringRef, NumAArch64CPUArchs> List;
@@ -1440,7 +1441,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
       AArch64::AEK_PCDPHINT,     AArch64::AEK_POPS,
       AArch64::AEK_SVEAES,       AArch64::AEK_SME_MOP4,
       AArch64::AEK_SME_TMOP,     AArch64::AEK_SVEBITPERM,
-      AArch64::AEK_SSVE_BITPERM,
+      AArch64::AEK_SSVE_BITPERM, AArch64::AEK_SVESHA3,
   };
 
   std::vector<StringRef> Features;
@@ -1479,6 +1480,7 @@ TEST(TargetParserTest, AArch64ExtensionFeatures) {
   EXPECT_TRUE(llvm::is_contained(Features, "+sve-aes"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-aes"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sm4"));
+  EXPECT_TRUE(llvm::is_contained(Features, "+sve-sha3"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-sha3"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve-bitperm"));
   EXPECT_TRUE(llvm::is_contained(Features, "+sve2-bitperm"));
@@ -1651,6 +1653,7 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
       {"sve-f16f32mm", "nosve-f16f32mm", "+sve-f16f32mm", "-sve-f16f32mm"},
       {"sve2", "nosve2", "+sve2", "-sve2"},
       {"sve-aes", "nosve-aes", "+sve-aes", "-sve-aes"},
+      {"sve-sha3", "nosve-sha3", "+sve-sha3", "-sve-sha3"},
       {"sve2-aes", "nosve2-aes", "+sve2-aes", "-sve2-aes"},
       {"sve2-sm4", "nosve2-sm4", "+sve2-sm4", "-sve2-sm4"},
       {"sve2-sha3", "nosve2-sha3", "+sve2-sha3", "-sve2-sha3"},
@@ -2159,6 +2162,12 @@ AArch64ExtensionDependenciesBaseArchTestParams
          {"sve2"},
          {"sve2-aes", "sve-aes"}},
 
+        // -sve2-sha3 should disable sve-sha3 (only)
+        {AArch64::ARMV9_6A,
+         {"sve2", "sve-sha3", "nosve2-sha3"},
+         {"sve2"},
+         {"sve2-sha3", "sve-sha3"}},
+
         // sme-tmop -> sme
         {AArch64::ARMV8A, {"nosme2", "sme-tmop"}, {"sme2", "sme-tmop"}, {}},
         {AArch64::ARMV8A, {"sme-tmop", "nosme2"}, {}, {"sme2", "sme-tmop"}},
diff --git a/llvm/unittests/TargetParser/TripleTest.cpp b/llvm/unittests/TargetParser/TripleTest.cpp
index 0f6d07657c93..7817e8232296 100644
--- a/llvm/unittests/TargetParser/TripleTest.cpp
+++ b/llvm/unittests/TargetParser/TripleTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TargetParser/Triple.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/VersionTuple.h"
 #include "gtest/gtest.h"
@@ -2281,6 +2282,44 @@ TEST(TripleTest, XROS) {
   EXPECT_EQ(VersionTuple(17), Version);
 }
 
+TEST(TripleTest, BridgeOS) {
+  Triple T;
+  VersionTuple Version;
+
+  T = Triple("arm64-apple-bridgeos");
+  EXPECT_TRUE(T.isBridgeOS());
+  EXPECT_FALSE(T.isXROS());
+  EXPECT_TRUE(T.isOSDarwin());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_FALSE(T.isSimulatorEnvironment());
+  EXPECT_EQ(T.getOSName(), "bridgeos");
+  Version = T.getOSVersion();
+  EXPECT_EQ(VersionTuple(0), Version);
+
+  T = Triple("arm64-apple-bridgeos1.0");
+  EXPECT_TRUE(T.isBridgeOS());
+  EXPECT_FALSE(T.isXROS());
+  EXPECT_TRUE(T.isOSDarwin());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_FALSE(T.isSimulatorEnvironment());
+  EXPECT_EQ(T.getOSName(), "bridgeos1.0");
+  Version = T.getOSVersion();
+  EXPECT_EQ(VersionTuple(1), Version);
+
+  T = Triple("arm64-apple-bridgeos9.0");
+  EXPECT_TRUE(T.isBridgeOS());
+  EXPECT_FALSE(T.isXROS());
+  EXPECT_TRUE(T.isOSDarwin());
+  EXPECT_FALSE(T.isiOS());
+  EXPECT_FALSE(T.isMacOSX());
+  EXPECT_FALSE(T.isSimulatorEnvironment());
+  EXPECT_EQ(T.getOSName(), "bridgeos9.0");
+  Version = T.getOSVersion();
+  EXPECT_EQ(VersionTuple(9), Version);
+}
+
 TEST(TripleTest, getOSVersion) {
   Triple T;
   VersionTuple Version;
@@ -2719,6 +2758,157 @@ TEST(TripleTest, FileFormat) {
   EXPECT_EQ(Triple::DXContainer, Triple("dxil-apple-macosx").getObjectFormat());
 }
 
+TEST(TripleTest, DefaultExceptionHandling) {
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-unknown-linux-gnu").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-unknown-freebsd").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("wasm32-unknown-wasi-wasm").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("wasm64-unknown-wasi-wasm").getDefaultExceptionHandling());
+
+  EXPECT_EQ(
+      ExceptionHandling::ARM,
+      Triple("arm-unknown-linux-android16").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("armv6-unknown-netbsd-eabi").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::ARM,
+            Triple("armv7-suse-linux-gnueabihf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::ARM,
+            Triple("thumbv7-linux-gnueabihf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("thumbv7-none-netbsd").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::ZOS,
+            Triple("s390x-ibm-zos").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("systemz-ibm-linux").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("sparcel-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("amdgcn--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("nvptx64--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("spirv32--").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("spirv64--").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-windows-msvc-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i686-pc-windows-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-apple-darwin9").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-unknown-uefi").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI, // ???
+            Triple("x86_64-unknown-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("riscv32-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("riscv64-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xcore-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xtensa-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("lanai-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arc-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(
+      ExceptionHandling::DwarfCFI,
+      Triple("loongarch32-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(
+      ExceptionHandling::DwarfCFI,
+      Triple("loongarch64-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("msp430-unknown-unknown").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("csky-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::AIX,
+            Triple("powerpc-ibm-aix").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::AIX,
+            Triple("powerpc---xcoff").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("powerpc-apple-macosx").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-cygwin-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-pc-cygwin-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-pc-win32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("x86_64-pc-windows-coreclr").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("ve-unknown-linux").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::None,
+            Triple("ve-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("armv7k-apple-watchos").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::SjLj,
+            Triple("armv7-apple-ios").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips64-mti-linux-gnu").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mips-pc-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("mips-pc-windows-msvc-coff").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("mipsel-windows--coff").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("mipsel-windows-msvc").getDefaultExceptionHandling());
+
+  EXPECT_EQ(
+      ExceptionHandling::WinEH,
+      Triple("aarch64-unknown-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("aarch64-unknown-unknown").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arm64_32-apple-ios").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("arm64-apple-macosx").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("xcore-xmos-elf").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i386-pc-windows-msvc").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-mingw32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::WinEH,
+            Triple("i386-pc-win32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i386-pc-mingw32").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("i686-pc-cygwin").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-pc-mingw64").getDefaultExceptionHandling());
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-win32-gnu").getDefaultExceptionHandling());
+
+  EXPECT_EQ(ExceptionHandling::DwarfCFI,
+            Triple("x86_64-scei-ps4").getDefaultExceptionHandling());
+}
+
 TEST(TripleTest, NormalizeWindows) {
   EXPECT_EQ("i686-pc-windows-msvc", Triple::normalize("i686-pc-win32"));
   EXPECT_EQ("i686-unknown-windows-msvc", Triple::normalize("i686-win32"));
diff --git a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
index 0dd819a10e0b..220dfedbee14 100644
--- a/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
+++ b/llvm/unittests/Transforms/Instrumentation/MemProfUseTest.cpp
@@ -91,7 +91,7 @@ declare !dbg !19 void @_Z2f3v()
   auto *F = M->getFunction("_Z3foov");
   ASSERT_NE(F, nullptr);
 
-  TargetLibraryInfoWrapperPass WrapperPass;
+  TargetLibraryInfoWrapperPass WrapperPass(M->getTargetTriple());
   auto &TLI = WrapperPass.getTLI(*F);
   auto Calls = extractCallsFromIR(*M, TLI);
 
@@ -191,7 +191,7 @@ declare !dbg !25 void @_Z2g2v() local_unnamed_addr
   auto *F = M->getFunction("_Z3foov");
   ASSERT_NE(F, nullptr);
 
-  TargetLibraryInfoWrapperPass WrapperPass;
+  TargetLibraryInfoWrapperPass WrapperPass(M->getTargetTriple());
   auto &TLI = WrapperPass.getTLI(*F);
   auto Calls = extractCallsFromIR(*M, TLI);
 
@@ -282,7 +282,7 @@ attributes #2 = { builtin allocsize(0) }
   auto *F = M->getFunction("_Z3foov");
   ASSERT_NE(F, nullptr);
 
-  TargetLibraryInfoWrapperPass WrapperPass;
+  TargetLibraryInfoWrapperPass WrapperPass(M->getTargetTriple());
   auto &TLI = WrapperPass.getTLI(*F);
   auto Calls = extractCallsFromIR(*M, TLI);
 
@@ -398,7 +398,7 @@ attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "t
   auto *F = M->getFunction("_Z3foov");
   ASSERT_NE(F, nullptr);
 
-  TargetLibraryInfoWrapperPass WrapperPass;
+  TargetLibraryInfoWrapperPass WrapperPass(M->getTargetTriple());
   auto &TLI = WrapperPass.getTLI(*F);
   auto Calls = extractCallsFromIR(*M, TLI);
 
diff --git a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
index e58daed88785..40a8c1d8d3da 100644
--- a/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
@@ -180,7 +180,7 @@ define i32 @foo(i32 %n) {
   LoopInfo LI(DT);
 
   DataLayout DL("e-i64:64-f80:128-n8:16:32:64-S128");
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   AAResults AA(TLI);
@@ -255,7 +255,7 @@ declare void @sink_alt() cold
 
   LoopInfo LI(DT);
 
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
 
   AAResults AA(TLI);
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index 09b32bf09df0..b8b035751637 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -1192,8 +1192,8 @@ TEST_F(CloneInstruction, cloneKeyInstructions) {
     !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
     !1 = !DIFile(filename: "test.cpp",  directory: "")
     !2 = !{i32 1, !"Debug Info Version", i32 3}
-    !3 = distinct !DISubprogram(name: "test", scope: !0, unit: !0)
-    !4 = distinct !DISubprogram(name: "inlined", scope: !0, unit: !0, retainedNodes: !{!5})
+    !3 = distinct !DISubprogram(name: "test", scope: !0, unit: !0, keyInstructions: true)
+    !4 = distinct !DISubprogram(name: "inlined", scope: !0, unit: !0, retainedNodes: !{!5}, keyInstructions: true)
     !5 = !DILocalVariable(name: "awaitables", scope: !4)
     !6 = !DILocation(line: 1, scope: !4, inlinedAt: !8, atomGroup: 1, atomRank: 1)
     !7 = !DILocation(line: 2, scope: !3, atomGroup: 1, atomRank: 1)
diff --git a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
index 9200fab43686..9466977d0064 100644
--- a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
@@ -38,7 +38,7 @@ static void run(Module &M, StringRef FuncName,
   auto *F = M.getFunction(FuncName);
   DominatorTree DT(*F);
   PostDominatorTree PDT(*F);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   AliasAnalysis AA(TLI);
@@ -227,7 +227,7 @@ TEST(CodeMoverUtils, IsControlFlowEquivalentCondNestTest) {
   //       i = 2;
   // }
   std::unique_ptr<Module> M =
-      parseIR(C, R"(define void @foo(i32* %i, i1 %cond1, i1 %cond2) { 
+      parseIR(C, R"(define void @foo(i32* %i, i1 %cond1, i1 %cond2) {
          entry:
            br i1 %cond1, label %if.outer.first, label %if.first.end
          if.outer.first:
@@ -282,8 +282,8 @@ TEST(CodeMoverUtils, IsControlFlowEquivalentImbalanceTest) {
   //   if (cond1)
   //     i = 4;
   // }
-  std::unique_ptr<Module> M = parseIR(
-      C, R"(define void @foo(i32* %i, i1 %cond1, i1 %cond2, i1 %cond3) { 
+  std::unique_ptr<Module> M =
+      parseIR(C, R"(define void @foo(i32* %i, i1 %cond1, i1 %cond2, i1 %cond3) {
          entry:
            br i1 %cond1, label %if.outer.first, label %if.first.end
          if.outer.first:
diff --git a/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp b/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
index c276f2ea62fe..70cdf0aba356 100644
--- a/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LoopRotationUtilsTest.cpp
@@ -79,7 +79,7 @@ deopt.exit:
   LoopInfo LI(DT);
   AssumptionCache AC(*F);
   TargetTransformInfo TTI(M->getDataLayout());
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   ScalarEvolution SE(*F, TLI, AC, DT, LI);
   SimplifyQuery SQ(M->getDataLayout());
@@ -150,7 +150,7 @@ deopt.exit:
   LoopInfo LI(DT);
   AssumptionCache AC(*F);
   TargetTransformInfo TTI(M->getDataLayout());
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   ScalarEvolution SE(*F, TLI, AC, DT, LI);
   SimplifyQuery SQ(M->getDataLayout());
diff --git a/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp b/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
index cd4717181d2c..c22a3582bee8 100644
--- a/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp
@@ -33,7 +33,7 @@ static void run(Module &M, StringRef FuncName,
                     Test) {
   Function *F = M.getFunction(FuncName);
   DominatorTree DT(*F);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M.getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   AssumptionCache AC(*F);
   LoopInfo LI(DT);
diff --git a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
index 6aff8406790d..55eae64fe0f6 100644
--- a/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
+++ b/llvm/unittests/Transforms/Utils/ScalarEvolutionExpanderTest.cpp
@@ -42,7 +42,8 @@ class ScalarEvolutionExpanderTest : public testing::Test {
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<LoopInfo> LI;
 
-  ScalarEvolutionExpanderTest() : M("", Context), TLII(), TLI(TLII) {}
+  ScalarEvolutionExpanderTest()
+      : M("", Context), TLII(M.getTargetTriple()), TLI(TLII) {}
 
   ScalarEvolution buildSE(Function &F) {
     AC.reset(new AssumptionCache(F));
diff --git a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
index fb02c89c77a1..eec10110e6af 100644
--- a/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/llvm/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -63,7 +63,7 @@ while.end:                                        ; preds = %while.cond
   DominatorTree DT(*F);
   LoopInfo LI(DT);
   AssumptionCache AC(*F);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   ScalarEvolution SE(*F, TLI, AC, DT, LI);
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index 9a7ee8214d10..721eda16c7e1 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -30,21 +30,26 @@ struct DependencyGraphTest : public testing::Test {
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<BasicAAResult> BAA;
   std::unique_ptr<AAResults> AA;
+  std::unique_ptr<TargetLibraryInfoImpl> TLII;
+  std::unique_ptr<TargetLibraryInfo> TLI;
 
   void parseIR(LLVMContext &C, const char *IR) {
     SMDiagnostic Err;
     M = parseAssemblyString(IR, Err, C);
-    if (!M)
+    if (!M) {
       Err.print("DependencyGraphTest", errs());
+      return;
+    }
+
+    TLII = std::make_unique<TargetLibraryInfoImpl>(M->getTargetTriple());
+    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
   }
 
   AAResults &getAA(llvm::Function &LLVMF) {
-    TargetLibraryInfoImpl TLII;
-    TargetLibraryInfo TLI(TLII);
-    AA = std::make_unique<AAResults>(TLI);
+    AA = std::make_unique<AAResults>(*TLI);
     AC = std::make_unique<AssumptionCache>(LLVMF);
     DT = std::make_unique<DominatorTree>(LLVMF);
-    BAA = std::make_unique<BasicAAResult>(M->getDataLayout(), LLVMF, TLI, *AC,
+    BAA = std::make_unique<BasicAAResult>(M->getDataLayout(), LLVMF, *TLI, *AC,
                                           DT.get());
     AA->addAAResult(*BAA);
     return *AA;
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index 99519d17d0e8..586f846caf26 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -38,8 +38,6 @@ struct LegalityTest : public testing::Test {
 
   void getAnalyses(llvm::Function &LLVMF) {
     DT = std::make_unique<DominatorTree>(LLVMF);
-    TLII = std::make_unique<TargetLibraryInfoImpl>();
-    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
     AC = std::make_unique<AssumptionCache>(LLVMF);
     LI = std::make_unique<LoopInfo>(*DT);
     SE = std::make_unique<ScalarEvolution>(LLVMF, *TLI, *AC, *DT, *LI);
@@ -52,8 +50,13 @@ struct LegalityTest : public testing::Test {
   void parseIR(LLVMContext &C, const char *IR) {
     SMDiagnostic Err;
     M = parseAssemblyString(IR, Err, C);
-    if (!M)
+    if (!M) {
       Err.print("LegalityTest", errs());
+      return;
+    }
+
+    TLII = std::make_unique<TargetLibraryInfoImpl>(M->getTargetTriple());
+    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
   }
 };
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
index 6f6e8a089bac..4c38fa069ba6 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SchedulerTest.cpp
@@ -30,21 +30,26 @@ struct SchedulerTest : public testing::Test {
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<BasicAAResult> BAA;
   std::unique_ptr<AAResults> AA;
+  std::unique_ptr<TargetLibraryInfoImpl> TLII;
+  std::unique_ptr<TargetLibraryInfo> TLI;
 
   void parseIR(LLVMContext &C, const char *IR) {
     SMDiagnostic Err;
     M = parseAssemblyString(IR, Err, C);
-    if (!M)
+    if (!M) {
       Err.print("SchedulerTest", errs());
+      return;
+    }
+
+    TLII = std::make_unique<TargetLibraryInfoImpl>(M->getTargetTriple());
+    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
   }
 
   AAResults &getAA(llvm::Function &LLVMF) {
-    TargetLibraryInfoImpl TLII;
-    TargetLibraryInfo TLI(TLII);
-    AA = std::make_unique<AAResults>(TLI);
+    AA = std::make_unique<AAResults>(*TLI);
     AC = std::make_unique<AssumptionCache>(LLVMF);
     DT = std::make_unique<DominatorTree>(LLVMF);
-    BAA = std::make_unique<BasicAAResult>(M->getDataLayout(), LLVMF, TLI, *AC,
+    BAA = std::make_unique<BasicAAResult>(M->getDataLayout(), LLVMF, *TLI, *AC,
                                           DT.get());
     AA->addAAResult(*BAA);
     return *AA;
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
index 99a13801c7c3..7f9a59bd428a 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
@@ -185,7 +185,7 @@ define void @foo(ptr %ptrA, float %val, ptr %ptr) {
   Function &LLVMF = *M->getFunction("foo");
 
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -240,7 +240,7 @@ define void @foo(ptr %ptrA, float %val, ptr %ptrB) {
   Function &LLVMF = *M->getFunction("foo");
 
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -305,7 +305,7 @@ define void @foo(ptr noalias %ptr, float %val) {
 )IR");
   Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -315,7 +315,8 @@ define void @foo(ptr noalias %ptr, float %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 4);
@@ -349,7 +350,7 @@ define void @foo(ptr noalias %ptr, float %val) {
 )IR");
   Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -359,7 +360,8 @@ define void @foo(ptr noalias %ptr, float %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 4);
@@ -409,7 +411,7 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0, i64 %val1) {
 )IR");
   Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -419,7 +421,8 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0, i64 %val1) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 3);
@@ -450,7 +453,7 @@ define void @foo(ptr noalias %ptr, float %v, <2 x float> %val) {
 )IR");
   Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -460,7 +463,8 @@ define void @foo(ptr noalias %ptr, float %v, <2 x float> %val) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false);
 
   // Find the stores
   auto It = std::next(BB->begin(), 3);
@@ -493,7 +497,7 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0) {
 )IR");
   Function &LLVMF = *M->getFunction("foo");
   DominatorTree DT(LLVMF);
-  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
   TargetLibraryInfo TLI(TLII);
   DataLayout DL(M->getDataLayout());
   LoopInfo LI(DT);
@@ -503,7 +507,8 @@ define void @foo(ptr noalias %ptr, <2 x float> %val0) {
   sandboxir::Context Ctx(C);
   auto &F = *Ctx.createFunction(&LLVMF);
   auto BB = F.begin();
-  sandboxir::SeedCollector SC(&*BB, SE);
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/false,
+                              /*CollectLoads=*/true);
 
   // Find the loads
   auto It = std::next(BB->begin(), 2);
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
index 1751701967e6..2bfea6908305 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/VecUtilsTest.cpp
@@ -37,12 +37,16 @@ struct VecUtilsTest : public testing::Test {
   void parseIR(const char *IR) {
     SMDiagnostic Err;
     M = parseAssemblyString(IR, Err, C);
-    if (!M)
+    if (!M) {
       Err.print("VecUtilsTest", errs());
+      return;
+    }
+
+    TLII = std::make_unique<TargetLibraryInfoImpl>(M->getTargetTriple());
+    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
   }
+
   ScalarEvolution &getSE(llvm::Function &LLVMF) {
-    TLII = std::make_unique<TargetLibraryInfoImpl>();
-    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
     AC = std::make_unique<AssumptionCache>(LLVMF);
     DT = std::make_unique<DominatorTree>(LLVMF);
     LI = std::make_unique<LoopInfo>(*DT);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 3f43a2ef690c..118bf67320a3 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -18,8 +18,6 @@ namespace {
 
 class VPlanSlpTest : public VPlanTestIRBase {
 protected:
-  TargetLibraryInfoImpl TLII;
-  TargetLibraryInfo TLI;
   DataLayout DL;
 
   std::unique_ptr<AssumptionCache> AC;
@@ -31,20 +29,19 @@ class VPlanSlpTest : public VPlanTestIRBase {
   std::unique_ptr<InterleavedAccessInfo> IAI;
 
   VPlanSlpTest()
-      : TLII(), TLI(TLII),
-        DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
+      : DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
            "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
            "16:32:64-S128") {}
 
   VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L,
                                                    VPlan &Plan) {
     AC.reset(new AssumptionCache(F));
-    SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
-    BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT));
-    AARes.reset(new AAResults(TLI));
+    SE.reset(new ScalarEvolution(F, *TLI, *AC, *DT, *LI));
+    BasicAA.reset(new BasicAAResult(DL, F, *TLI, *AC, &*DT));
+    AARes.reset(new AAResults(*TLI));
     AARes->addAAResult(*BasicAA);
     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
-    LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &TLI, &*AARes, &*DT, &*LI));
+    LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &*TLI, &*AARes, &*DT, &*LI));
     IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
     IAI->analyzeInterleaving(false);
     return {Plan, *IAI};
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 8b4fd066aaa2..8012d9ed3870 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -706,13 +706,15 @@ TEST_F(VPBasicBlockTest, reassociateBlocks) {
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 TEST_F(VPBasicBlockTest, print) {
-  VPInstruction *TC = new VPInstruction(Instruction::Add, {});
+  VPInstruction *TC = new VPInstruction(Instruction::PHI, {});
   VPlan &Plan = getPlan(TC);
+  IntegerType *Int32 = IntegerType::get(C, 32);
+  VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
   VPBasicBlock *VPBB0 = Plan.getEntry();
   VPBB0->appendRecipe(TC);
 
-  VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
-  VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1});
+  VPInstruction *I1 = new VPInstruction(Instruction::Add, {Val, Val});
+  VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1, Val});
   VPInstruction *I3 = new VPInstruction(Instruction::Br, {I1, I2});
 
   VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
@@ -722,7 +724,7 @@ TEST_F(VPBasicBlockTest, print) {
   VPBB1->setName("bb1");
 
   VPInstruction *I4 = new VPInstruction(Instruction::Mul, {I2, I1});
-  VPInstruction *I5 = new VPInstruction(Instruction::Ret, {I4});
+  VPInstruction *I5 = new VPInstruction(Instruction::Br, {I4});
   VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
   VPBB2->appendRecipe(I4);
   VPBB2->appendRecipe(I5);
@@ -752,14 +754,14 @@ edge [fontname=Courier, fontsize=30]
 compound=true
   N0 [label =
     "preheader:\l" +
-    "  EMIT vp\<%1\> = add \l" +
+    "  EMIT-SCALAR vp\<%1\> = phi \l" +
     "Successor(s): bb1\l"
   ]
   N0 -> N1 [ label=""]
   N1 [label =
     "bb1:\l" +
-    "  EMIT vp\<%2\> = add \l" +
-    "  EMIT vp\<%3\> = sub vp\<%2\>\l" +
+    "  EMIT vp\<%2\> = add ir\<1\>, ir\<1\>\l" +
+    "  EMIT vp\<%3\> = sub vp\<%2\>, ir\<1\>\l" +
     "  EMIT br vp\<%2\>, vp\<%3\>\l" +
     "Successor(s): bb2\l"
   ]
@@ -767,7 +769,7 @@ compound=true
   N2 [label =
     "bb2:\l" +
     "  EMIT vp\<%5\> = mul vp\<%3\>, vp\<%2\>\l" +
-    "  EMIT ret vp\<%5\>\l" +
+    "  EMIT br vp\<%5\>\l" +
     "Successor(s): ir-bb\<scalar.header\>\l"
   ]
   N2 -> N3 [ label=""]
@@ -780,8 +782,8 @@ compound=true
   EXPECT_EQ(ExpectedStr, FullDump);
 
   const char *ExpectedBlock1Str = R"(bb1:
-  EMIT vp<%2> = add 
-  EMIT vp<%3> = sub vp<%2>
+  EMIT vp<%2> = add ir<1>, ir<1>
+  EMIT vp<%3> = sub vp<%2>, ir<1>
   EMIT br vp<%2>, vp<%3>
 Successor(s): bb2
 )";
@@ -793,7 +795,7 @@ Successor(s): bb2
   // Ensure that numbering is good when dumping the second block in isolation.
   const char *ExpectedBlock2Str = R"(bb2:
   EMIT vp<%5> = mul vp<%3>, vp<%2>
-  EMIT ret vp<%5>
+  EMIT br vp<%5>
 Successor(s): ir-bb<scalar.header>
 )";
   std::string Block2Dump;
@@ -909,9 +911,12 @@ TEST_F(VPBasicBlockTest, cloneAndPrint) {
   VPlan &Plan = getPlan(nullptr);
   VPBasicBlock *VPBB0 = Plan.getEntry();
 
-  VPInstruction *I1 = new VPInstruction(Instruction::Add, {});
-  VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1});
-  VPInstruction *I3 = new VPInstruction(Instruction::Br, {I1, I2});
+  IntegerType *Int32 = IntegerType::get(C, 32);
+  VPValue *Val = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+
+  VPInstruction *I1 = new VPInstruction(Instruction::Add, {Val, Val});
+  VPInstruction *I2 = new VPInstruction(Instruction::Sub, {I1, Val});
+  VPInstruction *I3 = new VPInstruction(Instruction::Store, {I1, I2});
 
   VPBasicBlock *VPBB1 = Plan.createVPBasicBlock("");
   VPBB1->appendRecipe(I1);
@@ -932,9 +937,9 @@ compound=true
   N0 -> N1 [ label=""]
   N1 [label =
     "bb1:\l" +
-    "  EMIT vp\<%1\> = add \l" +
-    "  EMIT vp\<%2\> = sub vp\<%1\>\l" +
-    "  EMIT br vp\<%1\>, vp\<%2\>\l" +
+    "  EMIT vp\<%1\> = add ir\<1\>, ir\<1\>\l" +
+    "  EMIT vp\<%2\> = sub vp\<%1\>, ir\<1\>\l" +
+    "  EMIT store vp\<%1\>, vp\<%2\>\l" +
     "No successors\l"
   ]
 }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index e2ad65b93e3d..7dfd11a48b59 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -31,8 +31,6 @@ namespace llvm {
 /// given loop entry block.
 class VPlanTestIRBase : public testing::Test {
 protected:
-  TargetLibraryInfoImpl TLII;
-  TargetLibraryInfo TLI;
   DataLayout DL;
 
   std::unique_ptr<LLVMContext> Ctx;
@@ -41,10 +39,11 @@ class VPlanTestIRBase : public testing::Test {
   std::unique_ptr<DominatorTree> DT;
   std::unique_ptr<AssumptionCache> AC;
   std::unique_ptr<ScalarEvolution> SE;
+  std::unique_ptr<TargetLibraryInfoImpl> TLII;
+  std::unique_ptr<TargetLibraryInfo> TLI;
 
   VPlanTestIRBase()
-      : TLII(), TLI(TLII),
-        DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
+      : DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
            "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
            "16:32:64-S128"),
         Ctx(new LLVMContext) {}
@@ -53,6 +52,8 @@ class VPlanTestIRBase : public testing::Test {
     SMDiagnostic Err;
     M = parseAssemblyString(ModuleString, Err, *Ctx);
     EXPECT_TRUE(M);
+    TLII = std::make_unique<TargetLibraryInfoImpl>(M->getTargetTriple());
+    TLI = std::make_unique<TargetLibraryInfo>(*TLII);
     return *M;
   }
 
@@ -60,7 +61,7 @@ class VPlanTestIRBase : public testing::Test {
     DT.reset(new DominatorTree(F));
     LI.reset(new LoopInfo(*DT));
     AC.reset(new AssumptionCache(F));
-    SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
+    SE.reset(new ScalarEvolution(F, *TLI, *AC, *DT, *LI));
   }
 
   /// Build the VPlan for the loop starting from \p LoopHeader.
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index b6d9c9f3a158..6dec4110decb 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -1541,7 +1541,7 @@ void AsmMatcherInfo::buildInfo() {
     Variant.Name = AsmVariant->getValueAsString("Name");
     Variant.AsmVariantNo = AsmVariant->getValueAsInt("Variant");
 
-    for (const CodeGenInstruction *CGI : Target.getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *CGI : Target.getInstructions()) {
 
       // If the tblgen -match-prefix option is specified (for tblgen hackers),
       // filter the set of instructions we consider.
@@ -1712,22 +1712,21 @@ void AsmMatcherInfo::buildInstructionOperandReference(MatchableInfo *II,
   MatchableInfo::AsmOperand *Op = &II->AsmOperands[AsmOpIdx];
 
   // Map this token to an operand.
-  unsigned Idx;
-  if (!Operands.hasOperandNamed(OperandName, Idx))
+  std::optional<unsigned> Idx = Operands.findOperandNamed(OperandName);
+  if (!Idx)
     PrintFatalError(II->TheDef->getLoc(),
                     "error: unable to find operand: '" + OperandName + "'");
-
   // If the instruction operand has multiple suboperands, but the parser
   // match class for the asm operand is still the default "ImmAsmOperand",
   // then handle each suboperand separately.
-  if (Op->SubOpIdx == -1 && Operands[Idx].MINumOperands > 1) {
-    const Record *Rec = Operands[Idx].Rec;
+  if (Op->SubOpIdx == -1 && Operands[*Idx].MINumOperands > 1) {
+    const Record *Rec = Operands[*Idx].Rec;
     assert(Rec->isSubClassOf("Operand") && "Unexpected operand!");
     const Record *MatchClass = Rec->getValueAsDef("ParserMatchClass");
     if (MatchClass && MatchClass->getValueAsString("Name") == "Imm") {
       // Insert remaining suboperands after AsmOpIdx in II->AsmOperands.
       StringRef Token = Op->Token; // save this in case Op gets moved
-      for (unsigned SI = 1, SE = Operands[Idx].MINumOperands; SI != SE; ++SI) {
+      for (unsigned SI = 1, SE = Operands[*Idx].MINumOperands; SI != SE; ++SI) {
         MatchableInfo::AsmOperand NewAsmOp(/*IsIsolatedToken=*/true, Token);
         NewAsmOp.SubOpIdx = SI;
         II->AsmOperands.insert(II->AsmOperands.begin() + AsmOpIdx + SI,
@@ -1740,7 +1739,7 @@ void AsmMatcherInfo::buildInstructionOperandReference(MatchableInfo *II,
   }
 
   // Set up the operand class.
-  Op->Class = getOperandClass(Operands[Idx], Op->SubOpIdx);
+  Op->Class = getOperandClass(Operands[*Idx], Op->SubOpIdx);
   Op->OrigSrcOpName = OperandName;
 
   // If the named operand is tied, canonicalize it to the untied operand.
@@ -1752,14 +1751,14 @@ void AsmMatcherInfo::buildInstructionOperandReference(MatchableInfo *II,
   //   "inc $dst"
   // so that we know how to provide the $dst operand when filling in the result.
   int OITied = -1;
-  if (Operands[Idx].MINumOperands == 1)
-    OITied = Operands[Idx].getTiedRegister();
+  if (Operands[*Idx].MINumOperands == 1)
+    OITied = Operands[*Idx].getTiedRegister();
   if (OITied != -1) {
     // The tied operand index is an MIOperand index, find the operand that
     // contains it.
-    std::pair<unsigned, unsigned> Idx = Operands.getSubOperandNumber(OITied);
-    OperandName = Operands[Idx.first].Name;
-    Op->SubOpIdx = Idx.second;
+    auto [OpIdx, SubopIdx] = Operands.getSubOperandNumber(OITied);
+    OperandName = Operands[OpIdx].Name;
+    Op->SubOpIdx = SubopIdx;
   }
 
   Op->SrcOpName = OperandName;
@@ -3725,7 +3724,8 @@ void AsmMatcherEmitter::run(raw_ostream &OS) {
         "<< ActualIdx);\n";
   OS << "      if (ActualIdx < Operands.size())\n";
   OS << "        DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \" (\";\n";
-  OS << "                        Operands[ActualIdx]->print(dbgs()); dbgs() << "
+  OS << "                        Operands[ActualIdx]->print(dbgs(), "
+        "*getContext().getAsmInfo()); dbgs() << "
         "\"): \");\n";
   OS << "      else\n";
   OS << "        DEBUG_WITH_TYPE(\"asm-matcher\", dbgs() << \": \");\n";
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index a3ea0f5e32e9..cbe645778fa3 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -1314,7 +1314,7 @@ AsmWriterEmitter::AsmWriterEmitter(const RecordKeeper &R)
   unsigned Variant = AsmWriter->getValueAsInt("Variant");
 
   // Get the instruction numbering.
-  NumberedInstructions = Target.getInstructionsByEnumValue();
+  NumberedInstructions = Target.getInstructions();
 
   for (const auto &[Idx, I] : enumerate(NumberedInstructions)) {
     if (!I->AsmString.empty() && I->TheDef->getName() != "PHI")
diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt
index 1f4d3a7bc123..b4a66ecce644 100644
--- a/llvm/utils/TableGen/Basic/CMakeLists.txt
+++ b/llvm/utils/TableGen/Basic/CMakeLists.txt
@@ -15,6 +15,7 @@ add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL DISABLE_LLVM_LINK_LLV
   DirectiveEmitter.cpp
   IntrinsicEmitter.cpp
   RISCVTargetDefEmitter.cpp
+  RuntimeLibcallsEmitter.cpp
   SDNodeProperties.cpp
   TableGen.cpp
   TargetFeaturesEmitter.cpp
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
new file mode 100644
index 000000000000..9d77631862ee
--- /dev/null
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -0,0 +1,345 @@
+//===- RuntimeLibcallEmitter.cpp - Properties from RuntimeLibcalls.td -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+using namespace llvm;
+
+namespace {
+
+class RuntimeLibcall {
+  const Record *TheDef = nullptr;
+
+public:
+  RuntimeLibcall() = delete;
+  RuntimeLibcall(const Record *Def) : TheDef(Def) { assert(Def); }
+
+  ~RuntimeLibcall() { assert(TheDef); }
+
+  const Record *getDef() const { return TheDef; }
+
+  StringRef getName() const { return TheDef->getName(); }
+
+  void emitEnumEntry(raw_ostream &OS) const {
+    OS << "RTLIB::" << TheDef->getValueAsString("Name");
+  }
+};
+
+class RuntimeLibcallImpl {
+  const Record *TheDef;
+  const RuntimeLibcall *Provides = nullptr;
+
+public:
+  RuntimeLibcallImpl(
+      const Record *Def,
+      const DenseMap<const Record *, const RuntimeLibcall *> &ProvideMap)
+      : TheDef(Def) {
+    if (const Record *ProvidesDef = Def->getValueAsDef("Provides"))
+      Provides = ProvideMap.lookup(ProvidesDef);
+  }
+
+  ~RuntimeLibcallImpl() {}
+
+  const Record *getDef() const { return TheDef; }
+
+  StringRef getName() const { return TheDef->getName(); }
+
+  const RuntimeLibcall *getProvides() const { return Provides; }
+
+  StringRef getLibcallFuncName() const {
+    return TheDef->getValueAsString("LibCallFuncName");
+  }
+
+  void emitQuotedLibcallFuncName(raw_ostream &OS) const {
+    OS << '\"' << getLibcallFuncName() << '\"';
+  }
+
+  bool isDefault() const { return TheDef->getValueAsBit("IsDefault"); }
+
+  void emitEnumEntry(raw_ostream &OS) const {
+    OS << "RTLIB::" << TheDef->getName();
+  }
+};
+
+class RuntimeLibcallEmitter {
+private:
+  const RecordKeeper &Records;
+
+  DenseMap<const Record *, const RuntimeLibcall *> Def2RuntimeLibcall;
+
+  const RuntimeLibcall *getRuntimeLibcall(const Record *Def) const {
+    return Def2RuntimeLibcall.lookup(Def);
+  }
+
+  std::vector<RuntimeLibcall> RuntimeLibcallDefList;
+  std::vector<RuntimeLibcallImpl> RuntimeLibcallImplDefList;
+
+  DenseMap<const RuntimeLibcall *, const RuntimeLibcallImpl *>
+      LibCallToDefaultImpl;
+
+  void
+  emitTargetOverrideFunc(raw_ostream &OS, StringRef FuncName,
+                         ArrayRef<RuntimeLibcallImpl> LibCallImplList) const;
+
+  void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
+
+  void emitWindowsArm64LibCallNameOverrides(raw_ostream &OS) const;
+
+  void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
+  void emitGetInitRuntimeLibcallUtils(raw_ostream &OS) const;
+
+public:
+  RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R) {
+
+    ArrayRef<const Record *> AllRuntimeLibcalls =
+        Records.getAllDerivedDefinitions("RuntimeLibcall");
+
+    RuntimeLibcallDefList.reserve(AllRuntimeLibcalls.size());
+
+    for (const Record *RuntimeLibcallDef : AllRuntimeLibcalls) {
+      RuntimeLibcallDefList.emplace_back(RuntimeLibcallDef);
+      Def2RuntimeLibcall[RuntimeLibcallDef] = &RuntimeLibcallDefList.back();
+    }
+
+    for (RuntimeLibcall &LibCall : RuntimeLibcallDefList)
+      Def2RuntimeLibcall[LibCall.getDef()] = &LibCall;
+
+    ArrayRef<const Record *> AllRuntimeLibcallImpls =
+        Records.getAllDerivedDefinitions("RuntimeLibcallImpl");
+    RuntimeLibcallImplDefList.reserve(AllRuntimeLibcallImpls.size());
+
+    for (const Record *LibCallImplDef : AllRuntimeLibcallImpls) {
+      RuntimeLibcallImplDefList.emplace_back(LibCallImplDef,
+                                             Def2RuntimeLibcall);
+
+      RuntimeLibcallImpl &LibCallImpl = RuntimeLibcallImplDefList.back();
+
+      // const RuntimeLibcallImpl &LibCallImpl =
+      // RuntimeLibcallImplDefList.back();
+      if (LibCallImpl.isDefault()) {
+        const RuntimeLibcall *Provides = LibCallImpl.getProvides();
+        if (!Provides)
+          PrintFatalError(LibCallImplDef->getLoc(),
+                          "default implementations must provide a libcall");
+        LibCallToDefaultImpl[Provides] = &LibCallImpl;
+      }
+    }
+  }
+
+  std::vector<RuntimeLibcallImpl>
+  getRuntimeLibcallImplSet(StringRef Name) const {
+    std::vector<RuntimeLibcallImpl> Result;
+    ArrayRef<const Record *> ImplSet =
+        Records.getAllDerivedDefinitionsIfDefined(Name);
+    Result.reserve(ImplSet.size());
+
+    for (const Record *LibCallImplDef : ImplSet)
+      Result.emplace_back(LibCallImplDef, Def2RuntimeLibcall);
+    return Result;
+  }
+
+  void run(raw_ostream &OS);
+};
+
+} // End anonymous namespace.
+
+/// Emit a method \p FuncName of RTLIB::RuntimeLibcallsInfo to override the
+/// libcall names in \p LibCallImplList.
+void RuntimeLibcallEmitter::emitTargetOverrideFunc(
+    raw_ostream &OS, StringRef FuncName,
+    ArrayRef<RuntimeLibcallImpl> LibCallImplList) const {
+  OS << "void llvm::RTLIB::RuntimeLibcallsInfo::" << FuncName << "() {\n";
+
+  if (LibCallImplList.empty()) {
+    OS << "  llvm_unreachable(\"override set not defined\");\n";
+  } else {
+    // for (const Record *LibCallImpl : LibCallImplList) {
+    for (const RuntimeLibcallImpl &LibCallImpl : LibCallImplList) {
+      const RuntimeLibcall *Provides = LibCallImpl.getProvides();
+      OS << "  LibcallImpls[";
+      Provides->emitEnumEntry(OS);
+      OS << "] = ";
+      LibCallImpl.emitEnumEntry(OS);
+      OS << ";\n";
+    }
+  }
+
+  OS << "}\n\n";
+}
+
+void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
+  OS << "#ifdef GET_RUNTIME_LIBCALL_ENUM\n"
+        "namespace llvm {\n"
+        "namespace RTLIB {\n"
+        "enum Libcall : unsigned short {\n";
+
+  size_t CallTypeEnumVal = 0;
+  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
+    StringRef Name = LibCall.getName();
+    OS << "  " << Name << " = " << CallTypeEnumVal++ << ",\n";
+  }
+
+  // TODO: Emit libcall names as string offset table.
+
+  OS << "  UNKNOWN_LIBCALL = " << CallTypeEnumVal
+     << "\n};\n\n"
+        "enum LibcallImpl : unsigned short {\n"
+        "  Unsupported = 0,\n";
+
+  // FIXME: Emit this in a different namespace. And maybe use enum class.
+  size_t LibCallImplEnumVal = 1;
+  for (const RuntimeLibcallImpl &LibCall : RuntimeLibcallImplDefList) {
+    OS << "  " << LibCall.getName() << " = " << LibCallImplEnumVal++ << ", // "
+       << LibCall.getLibcallFuncName() << '\n';
+  }
+
+  OS << "  NumLibcallImpls = " << LibCallImplEnumVal
+     << "\n};\n"
+        "} // End namespace RTLIB\n"
+        "} // End namespace llvm\n"
+        "#endif\n\n";
+}
+
+void RuntimeLibcallEmitter::emitWindowsArm64LibCallNameOverrides(
+    raw_ostream &OS) const {
+  // FIXME: Stop treating this as a special case
+  OS << "void "
+        "llvm::RTLIB::RuntimeLibcallsInfo::setWindowsArm64LibCallNameOverrides("
+        ") {\n"
+        "  static const RTLIB::LibcallImpl "
+        "WindowsArm64RoutineImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {\n";
+  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
+    auto I = LibCallToDefaultImpl.find(&LibCall);
+    if (I == LibCallToDefaultImpl.end())
+      OS << "    RTLIB::Unsupported,";
+    else {
+      const RuntimeLibcallImpl *LibCallImpl = I->second;
+      assert(LibCallImpl);
+      OS << "    RTLIB::arm64ec_" << LibCallImpl->getName() << ',';
+    }
+
+    OS << " // ";
+    LibCall.emitEnumEntry(OS);
+    OS << '\n';
+  }
+
+  OS << "    RTLIB::Unsupported // RTLIB::UNKNOWN_LIBCALL\n"
+        "  };\n\n"
+        "  std::memcpy(LibcallImpls, WindowsArm64RoutineImpls,\n"
+        "              sizeof(LibcallImpls));\n"
+        "  static_assert(sizeof(LibcallImpls) == "
+        "sizeof(WindowsArm64RoutineImpls),\n"
+        "                \"libcall array size should match\");\n"
+        "}\n#endif\n\n";
+}
+
+void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
+    raw_ostream &OS) const {
+  // TODO: Emit libcall names as string offset table.
+
+  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_NAMES\n"
+        "const RTLIB::LibcallImpl "
+        "llvm::RTLIB::RuntimeLibcallsInfo::"
+        "DefaultLibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {\n";
+
+  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
+    auto I = LibCallToDefaultImpl.find(&LibCall);
+    if (I == LibCallToDefaultImpl.end()) {
+      OS << "  RTLIB::Unsupported,";
+    } else {
+      const RuntimeLibcallImpl *LibCallImpl = I->second;
+      OS << "  ";
+      LibCallImpl->emitEnumEntry(OS);
+      OS << ",";
+    }
+
+    OS << " // ";
+    LibCall.emitEnumEntry(OS);
+    OS << '\n';
+  }
+
+  OS << "  RTLIB::Unsupported\n"
+        "};\n\n";
+
+  // Emit the implementation names
+  OS << "const char *const llvm::RTLIB::RuntimeLibcallsInfo::"
+        "LibCallImplNames[RTLIB::NumLibcallImpls] = {\n"
+        "  nullptr, // RTLIB::Unsupported\n";
+
+  for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
+    OS << "  \"" << LibCallImpl.getLibcallFuncName() << "\", // ";
+    LibCallImpl.emitEnumEntry(OS);
+    OS << '\n';
+  }
+
+  OS << "};\n\n";
+
+  // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
+  OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
+        "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
+        "  RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported\n";
+
+  for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
+    const RuntimeLibcall *Provides = LibCallImpl.getProvides();
+    OS << "  ";
+    Provides->emitEnumEntry(OS);
+    OS << ", // ";
+    LibCallImpl.emitEnumEntry(OS);
+    OS << '\n';
+  }
+  OS << "};\n\n";
+
+  std::vector<RuntimeLibcallImpl> ZOSRuntimeLibcallImplList =
+      getRuntimeLibcallImplSet("ZOSRuntimeLibcallImpl");
+  emitTargetOverrideFunc(OS, "setZOSLibCallNameOverrides",
+                         ZOSRuntimeLibcallImplList);
+
+  std::vector<RuntimeLibcallImpl> PPCRuntimeLibcallImplList =
+      getRuntimeLibcallImplSet("PPCRuntimeLibcallImpl");
+  emitTargetOverrideFunc(OS, "setPPCLibCallNameOverrides",
+                         PPCRuntimeLibcallImplList);
+
+  emitWindowsArm64LibCallNameOverrides(OS);
+}
+
+void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallUtils(
+    raw_ostream &OS) const {
+  // FIXME: Hack we shouldn't really need
+  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_UTILS\n"
+        "static inline bool isAtomicLibCall(llvm::RTLIB::Libcall LC) {\n"
+        "  switch (LC) {\n";
+  for (const RuntimeLibcall &LibCall : RuntimeLibcallDefList) {
+    StringRef Name = LibCall.getName();
+    if (Name.contains("ATOMIC")) {
+      OS << "  case ";
+      LibCall.emitEnumEntry(OS);
+      OS << ":\n";
+    }
+  }
+
+  OS << "    return true;\n"
+        "  default:\n"
+        "    return false;\n"
+        "  }\n\n"
+        "  llvm_unreachable(\"covered switch over libcalls\");\n"
+        "}\n#endif\n\n";
+}
+
+void RuntimeLibcallEmitter::run(raw_ostream &OS) {
+  emitSourceFileHeader("Runtime LibCalls Source Fragment", OS, Records);
+  emitGetRuntimeLibcallEnum(OS);
+  emitGetInitRuntimeLibcallNames(OS);
+  emitGetInitRuntimeLibcallUtils(OS);
+}
+
+static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
+    X("gen-runtime-libcalls", "Generate RuntimeLibcalls");
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index 2fe40450abfd..3ab6f40a6dee 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -123,12 +123,11 @@ bool CodeEmitterGen::addCodeToMergeInOperand(const Record *R,
   // operand number. Non-matching operands are assumed to be in
   // order.
   unsigned OpIdx;
-  std::pair<unsigned, unsigned> SubOp;
-  if (CGI.Operands.hasSubOperandAlias(VarName, SubOp)) {
-    OpIdx = CGI.Operands[SubOp.first].MIOperandNo + SubOp.second;
-  } else if (CGI.Operands.hasOperandNamed(VarName, OpIdx)) {
+  if (auto SubOp = CGI.Operands.findSubOperandAlias(VarName)) {
+    OpIdx = CGI.Operands[SubOp->first].MIOperandNo + SubOp->second;
+  } else if (auto MayBeOpIdx = CGI.Operands.findOperandNamed(VarName)) {
     // Get the machine operand number for the indicated operand.
-    OpIdx = CGI.Operands[OpIdx].MIOperandNo;
+    OpIdx = CGI.Operands[*MayBeOpIdx].MIOperandNo;
   } else {
     PrintError(R, Twine("No operand named ") + VarName + " in record " +
                       R->getName());
@@ -476,7 +475,7 @@ void CodeEmitterGen::run(raw_ostream &O) {
   Target.reverseBitsForLittleEndianEncoding();
 
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   if (Target.hasVariableLengthEncodings()) {
     emitVarLenCodeEmitter(Records, O);
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index eccd1aff422d..90f0a2ac8c26 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -358,7 +358,7 @@ const Record *MapTableEmitter::getInstrForColumn(const Record *KeyInstr,
 
 unsigned MapTableEmitter::emitBinSearchTable(raw_ostream &OS) {
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
   StringRef Namespace = Target.getInstNamespace();
   ArrayRef<const ListInit *> ValueCols = InstrMapDesc.getValueCols();
   unsigned NumCol = ValueCols.size();
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index 3a4ca1b45156..c3b406a54507 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -2103,7 +2103,7 @@ TreePatternNodePtr TreePatternNode::clone() const {
 /// RemoveAllTypes - Recursively strip all the types of this tree.
 void TreePatternNode::RemoveAllTypes() {
   // Reset to unknown type.
-  std::fill(Types.begin(), Types.end(), TypeSetByHwMode());
+  llvm::fill(Types, TypeSetByHwMode());
   if (isLeaf())
     return;
   for (TreePatternNode &Child : children())
@@ -3187,7 +3187,7 @@ bool TreePattern::InferAllTypes(
           return true;
         }
 
-        const SmallVectorImpl<TreePatternNode *> &InNodes = InIter->second;
+        ArrayRef<TreePatternNode *> InNodes = InIter->second;
 
         // The input types should be fully resolved by now.
         for (TreePatternNode *Node : Nodes) {
@@ -4122,8 +4122,7 @@ void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern,
 }
 
 void CodeGenDAGPatterns::InferInstructionFlags() {
-  ArrayRef<const CodeGenInstruction *> Instructions =
-      Target.getInstructionsByEnumValue();
+  ArrayRef<const CodeGenInstruction *> Instructions = Target.getInstructions();
 
   unsigned Errors = 0;
 
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
index 2ec3683e116e..3dbf2b9b446e 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
@@ -183,8 +183,8 @@ CGIOperandList::CGIOperandList(const Record *R) : TheDef(R) {
       // If we have no explicit sub-op dag, but have an top-level encoder
       // method, the single encoder will multiple sub-ops, itself.
       OpInfo.EncoderMethodNames[0] = EncoderMethod;
-      for (unsigned j = 1; j < NumOps; ++j)
-        OpInfo.DoNotEncode[j] = true;
+      OpInfo.DoNotEncode.set();
+      OpInfo.DoNotEncode[0] = false;
     }
 
     MIOperandNo += NumOps;
@@ -199,36 +199,31 @@ CGIOperandList::CGIOperandList(const Record *R) : TheDef(R) {
 /// specified name, abort.
 ///
 unsigned CGIOperandList::getOperandNamed(StringRef Name) const {
-  unsigned OpIdx;
-  if (hasOperandNamed(Name, OpIdx))
-    return OpIdx;
+  std::optional<unsigned> OpIdx = findOperandNamed(Name);
+  if (OpIdx)
+    return *OpIdx;
   PrintFatalError(TheDef->getLoc(), "'" + TheDef->getName() +
                                         "' does not have an operand named '$" +
                                         Name + "'!");
 }
 
-/// hasOperandNamed - Query whether the instruction has an operand of the
-/// given name. If so, return true and set OpIdx to the index of the
-/// operand. Otherwise, return false.
-bool CGIOperandList::hasOperandNamed(StringRef Name, unsigned &OpIdx) const {
+/// findOperandNamed - Query whether the instruction has an operand of the
+/// given name. If so, the index of the operand. Otherwise, return std::nullopt.
+std::optional<unsigned> CGIOperandList::findOperandNamed(StringRef Name) const {
   assert(!Name.empty() && "Cannot search for operand with no name!");
-  for (unsigned i = 0, e = OperandList.size(); i != e; ++i)
-    if (OperandList[i].Name == Name) {
-      OpIdx = i;
-      return true;
-    }
-  return false;
+  for (const auto &[Index, Opnd] : enumerate(OperandList))
+    if (Opnd.Name == Name)
+      return Index;
+  return std::nullopt;
 }
 
-bool CGIOperandList::hasSubOperandAlias(
-    StringRef Name, std::pair<unsigned, unsigned> &SubOp) const {
+std::optional<std::pair<unsigned, unsigned>>
+CGIOperandList::findSubOperandAlias(StringRef Name) const {
   assert(!Name.empty() && "Cannot search for operand with no name!");
   auto SubOpIter = SubOpAliases.find(Name);
-  if (SubOpIter != SubOpAliases.end()) {
-    SubOp = SubOpIter->second;
-    return true;
-  }
-  return false;
+  if (SubOpIter != SubOpAliases.end())
+    return SubOpIter->second;
+  return std::nullopt;
 }
 
 std::pair<unsigned, unsigned>
@@ -251,9 +246,7 @@ CGIOperandList::ParseOperandName(StringRef Op, bool AllowWholeOp) {
     OpName = OpName.substr(0, DotIdx);
   }
 
-  unsigned OpIdx;
-
-  if (std::pair<unsigned, unsigned> SubOp; hasSubOperandAlias(OpName, SubOp)) {
+  if (auto SubOp = findSubOperandAlias(OpName)) {
     // Found a name for a piece of an operand, just return it directly.
     if (!SubOpName.empty()) {
       PrintFatalError(
@@ -262,10 +255,10 @@ CGIOperandList::ParseOperandName(StringRef Op, bool AllowWholeOp) {
               ": Cannot use dotted suboperand name within suboperand '" +
               OpName + "'");
     }
-    return SubOp;
+    return *SubOp;
   }
 
-  OpIdx = getOperandNamed(OpName);
+  unsigned OpIdx = getOperandNamed(OpName);
 
   if (SubOpName.empty()) { // If no suboperand name was specified:
     // If one was needed, throw.
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h
index 0db12b551b43..1ae913c262b2 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.h
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.h
@@ -127,10 +127,9 @@ class CGIOperandList {
     /// getTiedOperand - If this operand is tied to another one, return the
     /// other operand number.  Otherwise, return -1.
     int getTiedRegister() const {
-      for (const CGIOperandList::ConstraintInfo &CI : Constraints) {
+      for (const CGIOperandList::ConstraintInfo &CI : Constraints)
         if (CI.isTied())
           return CI.getTiedOperand();
-      }
       return -1;
     }
   };
@@ -176,13 +175,13 @@ class CGIOperandList {
   /// specified name, abort.
   unsigned getOperandNamed(StringRef Name) const;
 
-  /// hasOperandNamed - Query whether the instruction has an operand of the
-  /// given name. If so, return true and set OpIdx to the index of the
-  /// operand. Otherwise, return false.
-  bool hasOperandNamed(StringRef Name, unsigned &OpIdx) const;
+  /// findOperandNamed - Query whether the instruction has an operand of the
+  /// given name. If so, the index of the operand. Otherwise, return
+  /// std::nullopt.
+  std::optional<unsigned> findOperandNamed(StringRef Name) const;
 
-  bool hasSubOperandAlias(StringRef Name,
-                          std::pair<unsigned, unsigned> &SubOp) const;
+  std::optional<std::pair<unsigned, unsigned>>
+  findSubOperandAlias(StringRef Name) const;
 
   /// ParseOperandName - Parse an operand name like "$foo" or "$foo.bar",
   /// where $foo is a whole operand and $foo.bar refers to a suboperand.
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index 57a243158692..6260eeee331e 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -881,7 +881,7 @@ bool CodeGenRegisterClass::hasType(const ValueTypeByHwMode &VT) const {
   if (VT.isSimple()) {
     MVT T = VT.getSimple();
     for (const ValueTypeByHwMode &OurVT : VTs) {
-      if (llvm::count_if(OurVT, [T](auto &&P) { return P.second == T; }))
+      if (llvm::is_contained(llvm::make_second_range(OurVT), T))
         return true;
     }
   }
diff --git a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
index af7e43929bcf..50346c29e886 100644
--- a/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenSchedule.cpp
@@ -77,14 +77,12 @@ struct InstRegexOp : public SetTheory::Operator {
 
   void apply(SetTheory &ST, const DagInit *Expr, SetTheory::RecSet &Elts,
              ArrayRef<SMLoc> Loc) override {
-    ArrayRef<const CodeGenInstruction *> Instructions =
-        Target.getInstructionsByEnumValue();
-
-    unsigned NumGeneric = Target.getNumFixedInstructions();
-    unsigned NumPseudos = Target.getNumPseudoInstructions();
-    auto Generics = Instructions.slice(0, NumGeneric);
-    auto Pseudos = Instructions.slice(NumGeneric, NumPseudos);
-    auto NonPseudos = Instructions.slice(NumGeneric + NumPseudos);
+    ArrayRef<const CodeGenInstruction *> Generics =
+        Target.getGenericInstructions();
+    ArrayRef<const CodeGenInstruction *> Pseudos =
+        Target.getTargetPseudoInstructions();
+    ArrayRef<const CodeGenInstruction *> NonPseudos =
+        Target.getTargetNonPseudoInstructions();
 
     for (const Init *Arg : Expr->getArgs()) {
       const StringInit *SI = dyn_cast<StringInit>(Arg);
@@ -145,7 +143,7 @@ struct InstRegexOp : public SetTheory::Operator {
       }
 
       // Target instructions are split into two ranges: pseudo instructions
-      // first, than non-pseudos. Each range is in lexicographical order
+      // first, then non-pseudos. Each range is in lexicographical order
       // sorted by name. Find the sub-ranges that start with our prefix.
       struct Comp {
         bool operator()(const CodeGenInstruction *LHS, StringRef RHS) {
@@ -589,7 +587,7 @@ void CodeGenSchedModels::collectSchedRW() {
 
   // Find all SchedReadWrites referenced by instruction defs.
   ConstRecVec SWDefs, SRDefs;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     const Record *SchedDef = Inst->TheDef;
     if (SchedDef->isValueUnset("SchedRW"))
       continue;
@@ -838,7 +836,7 @@ void CodeGenSchedModels::collectSchedClasses() {
 
   // Create a SchedClass for each unique combination of itinerary class and
   // SchedRW list.
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     const Record *ItinDef = Inst->TheDef->getValueAsDef("Itinerary");
     IdxVec Writes, Reads;
     if (!Inst->TheDef->isValueUnset("SchedRW"))
@@ -863,7 +861,7 @@ void CodeGenSchedModels::collectSchedClasses() {
   LLVM_DEBUG(
       dbgs()
       << "\n+++ ITINERARIES and/or MACHINE MODELS (collectSchedClasses) +++\n");
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     StringRef InstName = Inst->TheDef->getName();
     unsigned SCIdx = getSchedClassIdx(*Inst);
     if (!SCIdx) {
@@ -1292,8 +1290,8 @@ class PredTransitions {
 
   PredTransitions(CodeGenSchedModels &sm) : SchedModels(sm) {}
 
-  bool substituteVariantOperand(const SmallVectorImpl<unsigned> &RWSeq,
-                                bool IsRead, unsigned StartIdx);
+  bool substituteVariantOperand(ArrayRef<unsigned> RWSeq, bool IsRead,
+                                unsigned StartIdx);
 
   bool substituteVariants(const PredTransition &Trans);
 
@@ -1526,8 +1524,8 @@ void PredTransitions::pushVariant(const TransVariant &VInfo, bool IsRead) {
 // operand. StartIdx is an index into TransVec where partial results
 // starts. RWSeq must be applied to all transitions between StartIdx and the end
 // of TransVec.
-bool PredTransitions::substituteVariantOperand(
-    const SmallVectorImpl<unsigned> &RWSeq, bool IsRead, unsigned StartIdx) {
+bool PredTransitions::substituteVariantOperand(ArrayRef<unsigned> RWSeq,
+                                               bool IsRead, unsigned StartIdx) {
   bool Subst = false;
   // Visit each original RW within the current sequence.
   for (unsigned int RWI : RWSeq) {
@@ -1591,7 +1589,7 @@ bool PredTransitions::substituteVariants(const PredTransition &Trans) {
 }
 
 static void addSequences(CodeGenSchedModels &SchedModels,
-                         const SmallVectorImpl<SmallVector<unsigned, 4>> &Seqs,
+                         ArrayRef<SmallVector<unsigned, 4>> Seqs,
                          IdxVec &Result, bool IsRead) {
   for (const auto &S : Seqs)
     if (!S.empty())
@@ -1924,7 +1922,7 @@ void CodeGenSchedModels::checkCompleteness() {
     const bool HasItineraries = ProcModel.hasItineraries();
     if (!ProcModel.ModelDef->getValueAsBit("CompleteModel"))
       continue;
-    for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *Inst : Target.getInstructions()) {
       if (Inst->hasNoSchedulingInfo)
         continue;
       if (ProcModel.isUnsupported(*Inst))
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.cpp b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
index f519582387db..a9002c5860fa 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.cpp
@@ -88,7 +88,7 @@ StringRef CodeGenTarget::getName() const { return TargetRec->getName(); }
 /// namespace. The namespace is cached because it is requested multiple times.
 StringRef CodeGenTarget::getInstNamespace() const {
   if (InstNamespace.empty()) {
-    for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) {
+    for (const CodeGenInstruction *Inst : getInstructions()) {
       // We are not interested in the "TargetOpcode" namespace.
       if (Inst->Namespace != "TargetOpcode") {
         InstNamespace = Inst->Namespace;
@@ -211,10 +211,9 @@ void CodeGenTarget::ReadInstructions() const {
 
   // Parse the instructions defined in the .td file.
   for (const Record *R : Insts) {
-    auto &Inst = Instructions[R];
-    Inst = std::make_unique<CodeGenInstruction>(R);
-    if (Inst->isVariableLengthEncoding())
-      HasVariableLengthEncodings = true;
+    auto [II, _] =
+        InstructionMap.try_emplace(R, std::make_unique<CodeGenInstruction>(R));
+    HasVariableLengthEncodings |= II->second->isVariableLengthEncoding();
   }
 }
 
@@ -242,9 +241,9 @@ unsigned CodeGenTarget::getNumFixedInstructions() {
 /// Return all of the instructions defined by the target, ordered by
 /// their enum value.
 void CodeGenTarget::ComputeInstrsByEnum() const {
-  const auto &Insts = getInstructions();
+  const auto &InstMap = getInstructionMap();
   for (const char *Name : FixedInstrs) {
-    const CodeGenInstruction *Instr = GetInstByName(Name, Insts, Records);
+    const CodeGenInstruction *Instr = GetInstByName(Name, InstMap, Records);
     assert(Instr && "Missing target independent instruction");
     assert(Instr->Namespace == "TargetOpcode" && "Bad namespace");
     InstrsByEnum.push_back(Instr);
@@ -253,23 +252,24 @@ void CodeGenTarget::ComputeInstrsByEnum() const {
   assert(EndOfPredefines == getNumFixedInstructions() &&
          "Missing generic opcode");
 
-  for (const auto &I : Insts) {
-    const CodeGenInstruction *CGI = I.second.get();
+  for (const auto &[_, CGIUp] : InstMap) {
+    const CodeGenInstruction *CGI = CGIUp.get();
     if (CGI->Namespace != "TargetOpcode") {
       InstrsByEnum.push_back(CGI);
-      if (CGI->TheDef->getValueAsBit("isPseudo"))
-        ++NumPseudoInstructions;
+      NumPseudoInstructions += CGI->TheDef->getValueAsBit("isPseudo");
     }
   }
 
-  assert(InstrsByEnum.size() == Insts.size() && "Missing predefined instr");
+  assert(InstrsByEnum.size() == InstMap.size() && "Missing predefined instr");
 
   // All of the instructions are now in random order based on the map iteration.
   llvm::sort(
       InstrsByEnum.begin() + EndOfPredefines, InstrsByEnum.end(),
       [](const CodeGenInstruction *Rec1, const CodeGenInstruction *Rec2) {
-        const auto &D1 = *Rec1->TheDef;
-        const auto &D2 = *Rec2->TheDef;
+        const Record &D1 = *Rec1->TheDef;
+        const Record &D2 = *Rec2->TheDef;
+        // Sort all pseudo instructions before non-pseudo ones, and sort by name
+        // within.
         return std::tuple(!D1.getValueAsBit("isPseudo"), D1.getName()) <
                std::tuple(!D2.getValueAsBit("isPseudo"), D2.getName());
       });
diff --git a/llvm/utils/TableGen/Common/CodeGenTarget.h b/llvm/utils/TableGen/Common/CodeGenTarget.h
index 52871f33a301..ac138344c83c 100644
--- a/llvm/utils/TableGen/Common/CodeGenTarget.h
+++ b/llvm/utils/TableGen/Common/CodeGenTarget.h
@@ -59,7 +59,7 @@ class CodeGenTarget {
   const Record *TargetRec;
 
   mutable DenseMap<const Record *, std::unique_ptr<CodeGenInstruction>>
-      Instructions;
+      InstructionMap;
   mutable std::unique_ptr<CodeGenRegBank> RegBank;
   mutable ArrayRef<const Record *> RegAltNameIndices;
   mutable SmallVector<ValueTypeByHwMode, 8> LegalValueTypes;
@@ -154,43 +154,52 @@ class CodeGenTarget {
 
 private:
   DenseMap<const Record *, std::unique_ptr<CodeGenInstruction>> &
-  getInstructions() const {
-    if (Instructions.empty())
+  getInstructionMap() const {
+    if (InstructionMap.empty())
       ReadInstructions();
-    return Instructions;
+    return InstructionMap;
   }
 
 public:
   CodeGenInstruction &getInstruction(const Record *InstRec) const {
-    if (Instructions.empty())
-      ReadInstructions();
-    auto I = Instructions.find(InstRec);
-    assert(I != Instructions.end() && "Not an instruction");
+    auto I = getInstructionMap().find(InstRec);
+    assert(I != InstructionMap.end() && "Not an instruction");
     return *I->second;
   }
 
   /// Returns the number of predefined instructions.
   static unsigned getNumFixedInstructions();
 
-  /// Returns the number of pseudo instructions.
-  unsigned getNumPseudoInstructions() const {
-    if (InstrsByEnum.empty())
-      ComputeInstrsByEnum();
-    return NumPseudoInstructions;
-  }
-
   /// Return all of the instructions defined by the target, ordered by their
   /// enum value.
   /// The following order of instructions is also guaranteed:
   /// - fixed / generic instructions as declared in TargetOpcodes.def, in order;
   /// - pseudo instructions in lexicographical order sorted by name;
   /// - other instructions in lexicographical order sorted by name.
-  ArrayRef<const CodeGenInstruction *> getInstructionsByEnumValue() const {
+  ArrayRef<const CodeGenInstruction *> getInstructions() const {
     if (InstrsByEnum.empty())
       ComputeInstrsByEnum();
     return InstrsByEnum;
   }
 
+  // Functions that return various slices of `getInstructions`, ordered by
+  // their enum values.
+  ArrayRef<const CodeGenInstruction *> getGenericInstructions() const {
+    return getInstructions().take_front(getNumFixedInstructions());
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetInstructions() const {
+    return getInstructions().drop_front(getNumFixedInstructions());
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetPseudoInstructions() const {
+    return getTargetInstructions().take_front(NumPseudoInstructions);
+  }
+
+  ArrayRef<const CodeGenInstruction *> getTargetNonPseudoInstructions() const {
+    return getTargetInstructions().drop_front(NumPseudoInstructions);
+  }
+
   /// Return the integer enum value corresponding to this instruction record.
   unsigned getInstrIntValue(const Record *R) const {
     if (InstrsByEnum.empty())
@@ -198,12 +207,6 @@ class CodeGenTarget {
     return getInstruction(R).EnumVal;
   }
 
-  typedef ArrayRef<const CodeGenInstruction *>::const_iterator inst_iterator;
-  inst_iterator inst_begin() const {
-    return getInstructionsByEnumValue().begin();
-  }
-  inst_iterator inst_end() const { return getInstructionsByEnumValue().end(); }
-
   /// Return whether instructions have variable length encodings on this target.
   bool hasVariableLengthEncodings() const { return HasVariableLengthEncodings; }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 327ac5f87777..70141ba738bd 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1448,7 +1448,7 @@ void InstructionOpcodeMatcher::initOpcodeValuesMap(
     const CodeGenTarget &Target) {
   OpcodeValues.clear();
 
-  for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue())
+  for (const CodeGenInstruction *I : Target.getInstructions())
     OpcodeValues[I] = Target.getInstrIntValue(I->TheDef);
 }
 
diff --git a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index e5e03b8bd8bd..77a2ae60ad6f 100644
--- a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -226,7 +226,7 @@ void VarLenInst::buildRec(const DagInit *DI) {
 void VarLenCodeEmitterGen::run(raw_ostream &OS) {
   CodeGenTarget Target(Records);
 
-  auto NumberedInstructions = Target.getInstructionsByEnumValue();
+  auto NumberedInstructions = Target.getInstructions();
 
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
     const Record *R = CGI->TheDef;
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index 2dfeea36e213..9a84343b98d9 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -98,6 +98,10 @@ class CompressInstEmitter {
     // Tied operand index within the instruction.
     int TiedOpIdx = -1;
   };
+  struct ArgData {
+    unsigned DAGOpNo;
+    unsigned MIOpNo;
+  };
   struct CompressPat {
     // The source instruction definition.
     CodeGenInstruction Source;
@@ -114,8 +118,9 @@ class CompressInstEmitter {
 
     bool IsCompressOnly;
     CompressPat(const CodeGenInstruction &S, const CodeGenInstruction &D,
-                std::vector<const Record *> RF, IndexedMap<OpData> &SourceMap,
-                IndexedMap<OpData> &DestMap, bool IsCompressOnly)
+                std::vector<const Record *> RF,
+                const IndexedMap<OpData> &SourceMap,
+                const IndexedMap<OpData> &DestMap, bool IsCompressOnly)
         : Source(S), Dest(D), PatReqFeatures(std::move(RF)),
           SourceOperandMap(SourceMap), DestOperandMap(DestMap),
           IsCompressOnly(IsCompressOnly) {}
@@ -126,26 +131,23 @@ class CompressInstEmitter {
   std::vector<CompressPat> CompressPatterns;
   void addDagOperandMapping(const Record *Rec, const DagInit *Dag,
                             const CodeGenInstruction &Inst,
-                            IndexedMap<OpData> &OperandMap, bool IsSourceInst,
-                            unsigned *SourceLastTiedOpPtr);
+                            IndexedMap<OpData> &OperandMap,
+                            StringMap<ArgData> &Operands, bool IsSourceInst);
   void evaluateCompressPat(const Record *Compress);
   void emitCompressInstEmitter(raw_ostream &OS, EmitterType EType);
   bool validateTypes(const Record *DagOpType, const Record *InstOpType,
                      bool IsSourceInst);
   bool validateRegister(const Record *Reg, const Record *RegClass);
-  void createDagOperandMapping(const Record *Rec,
-                               StringMap<unsigned> &SourceOperands,
-                               StringMap<unsigned> &DestOperands,
-                               const DagInit *SourceDag, const DagInit *DestDag,
-                               IndexedMap<OpData> &SourceOperandMap);
+  void checkDagOperandMapping(const Record *Rec,
+                              const StringMap<ArgData> &DestOperands,
+                              const DagInit *SourceDag, const DagInit *DestDag);
 
   void createInstOperandMapping(const Record *Rec, const DagInit *SourceDag,
                                 const DagInit *DestDag,
                                 IndexedMap<OpData> &SourceOperandMap,
                                 IndexedMap<OpData> &DestOperandMap,
-                                StringMap<unsigned> &SourceOperands,
-                                const CodeGenInstruction &DestInst,
-                                unsigned SourceLastTiedOp);
+                                StringMap<ArgData> &SourceOperands,
+                                const CodeGenInstruction &DestInst);
 
 public:
   CompressInstEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
@@ -197,6 +199,10 @@ bool CompressInstEmitter::validateTypes(const Record *DagOpType,
   return true;
 }
 
+static bool validateArgsTypes(const Init *Arg1, const Init *Arg2) {
+  return cast<DefInit>(Arg1)->getDef() == cast<DefInit>(Arg2)->getDef();
+}
+
 /// The patterns in the Dag contain different types of operands:
 /// Register operands, e.g.: GPRC:$rs1; Fixed registers, e.g: X1; Immediate
 /// operands, e.g.: simm6:$imm; Fixed immediate operands, e.g.: 0. This function
@@ -208,11 +214,12 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
                                                const DagInit *Dag,
                                                const CodeGenInstruction &Inst,
                                                IndexedMap<OpData> &OperandMap,
-                                               bool IsSourceInst,
-                                               unsigned *SourceLastTiedOpPtr) {
+                                               StringMap<ArgData> &Operands,
+                                               bool IsSourceInst) {
   unsigned NumMIOperands = 0;
-  for (const auto &Op : Inst.Operands)
-    NumMIOperands += Op.MINumOperands;
+  if (!Inst.Operands.empty())
+    NumMIOperands =
+        Inst.Operands.back().MIOperandNo + Inst.Operands.back().MINumOperands;
   OperandMap.grow(NumMIOperands);
 
   // TiedCount keeps track of the number of operands skipped in Inst
@@ -222,16 +229,12 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
   // are represented.
   unsigned TiedCount = 0;
   unsigned OpNo = 0;
-  if (IsSourceInst)
-    *SourceLastTiedOpPtr = std::numeric_limits<unsigned int>::max();
   for (const auto &Opnd : Inst.Operands) {
     int TiedOpIdx = Opnd.getTiedRegister();
     if (-1 != TiedOpIdx) {
+      assert((unsigned)TiedOpIdx < OpNo);
       // Set the entry in OperandMap for the tied operand we're skipping.
-      OperandMap[OpNo].Kind = OperandMap[TiedOpIdx].Kind;
-      OperandMap[OpNo].Data = OperandMap[TiedOpIdx].Data;
-      if (IsSourceInst)
-        *SourceLastTiedOpPtr = OpNo;
+      OperandMap[OpNo] = OperandMap[TiedOpIdx];
       ++OpNo;
       ++TiedCount;
       continue;
@@ -288,6 +291,30 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
       } else {
         llvm_unreachable("Unhandled CompressPat argument type!");
       }
+
+      // Create a mapping between the operand name in the Dag (e.g. $rs1) and
+      // its index in the list of Dag operands and check that operands with the
+      // same name have the same type. For example in 'C_ADD $rs1, $rs2' we
+      // generate the mapping $rs1 --> 0, $rs2 ---> 1. If the operand appears
+      // twice in the same Dag (tied in the compressed instruction), we note
+      // the previous index in the TiedOpIdx field.
+      StringRef ArgName = Dag->getArgNameStr(DAGOpNo);
+      if (ArgName.empty())
+        continue;
+
+      if (IsSourceInst) {
+        auto It = Operands.find(ArgName);
+        if (It != Operands.end()) {
+          OperandMap[OpNo].TiedOpIdx = It->getValue().MIOpNo;
+          if (!validateArgsTypes(Dag->getArg(It->getValue().DAGOpNo),
+                                 Dag->getArg(DAGOpNo)))
+            PrintFatalError(Rec->getLoc(),
+                            "Input Operand '" + ArgName +
+                                "' has a mismatched tied operand!");
+        }
+      }
+
+      Operands[ArgName] = {DAGOpNo, OpNo};
     }
   }
 }
@@ -297,14 +324,11 @@ static bool verifyDagOpCount(const CodeGenInstruction &Inst, const DagInit *Dag,
                              bool IsSource) {
   unsigned NumMIOperands = 0;
 
-  // Use this to count number of tied Operands in Source Inst in this function.
-  // This counter is required here to error out when there is a Source
-  // Inst with two or more tied operands.
-  unsigned SourceInstTiedOpCount = 0;
+  unsigned TiedOpCount = 0;
   for (const auto &Op : Inst.Operands) {
     NumMIOperands += Op.MINumOperands;
     if (Op.getTiedRegister() != -1)
-      SourceInstTiedOpCount++;
+      TiedOpCount++;
   }
 
   if (Dag->getNumArgs() == NumMIOperands)
@@ -312,7 +336,7 @@ static bool verifyDagOpCount(const CodeGenInstruction &Inst, const DagInit *Dag,
 
   // Source instructions are non compressed instructions and have at most one
   // tied operand.
-  if (IsSource && (SourceInstTiedOpCount >= 2))
+  if (IsSource && (TiedOpCount > 1))
     PrintFatalError(Inst.TheDef->getLoc(),
                     "Input operands for Inst '" + Inst.TheDef->getName() +
                         "' and input Dag operand count mismatch");
@@ -325,68 +349,36 @@ static bool verifyDagOpCount(const CodeGenInstruction &Inst, const DagInit *Dag,
 
   // The Instruction might have tied operands so the Dag might have
   // a fewer operand count.
-  unsigned RealCount = NumMIOperands;
-  for (const auto &Operand : Inst.Operands)
-    if (Operand.getTiedRegister() != -1)
-      --RealCount;
-
-  if (Dag->getNumArgs() != RealCount)
+  if (Dag->getNumArgs() != (NumMIOperands - TiedOpCount))
     PrintFatalError(Inst.TheDef->getLoc(),
                     "Inst '" + Inst.TheDef->getName() +
                         "' and Dag operand count mismatch");
   return true;
 }
 
-static bool validateArgsTypes(const Init *Arg1, const Init *Arg2) {
-  return cast<DefInit>(Arg1)->getDef() == cast<DefInit>(Arg2)->getDef();
-}
-
-// Creates a mapping between the operand name in the Dag (e.g. $rs1) and
-// its index in the list of Dag operands and checks that operands with the same
-// name have the same types. For example in 'C_ADD $rs1, $rs2' we generate the
-// mapping $rs1 --> 0, $rs2 ---> 1. If the operand appears twice in the (tied)
-// same Dag we use the last occurrence for indexing.
-void CompressInstEmitter::createDagOperandMapping(
-    const Record *Rec, StringMap<unsigned> &SourceOperands,
-    StringMap<unsigned> &DestOperands, const DagInit *SourceDag,
-    const DagInit *DestDag, IndexedMap<OpData> &SourceOperandMap) {
-  for (unsigned I = 0; I < DestDag->getNumArgs(); ++I) {
-    // Skip fixed immediates and registers, they were handled in
-    // addDagOperandMapping.
-    if ("" == DestDag->getArgNameStr(I))
-      continue;
-    DestOperands[DestDag->getArgNameStr(I)] = I;
-  }
+// Check that all names in the source DAG appear in the destionation DAG.
+void CompressInstEmitter::checkDagOperandMapping(
+    const Record *Rec, const StringMap<ArgData> &DestOperands,
+    const DagInit *SourceDag, const DagInit *DestDag) {
 
   for (unsigned I = 0; I < SourceDag->getNumArgs(); ++I) {
     // Skip fixed immediates and registers, they were handled in
     // addDagOperandMapping.
-    if ("" == SourceDag->getArgNameStr(I))
+    StringRef ArgName = SourceDag->getArgNameStr(I);
+    if (ArgName.empty())
       continue;
 
-    StringMap<unsigned>::iterator It =
-        SourceOperands.find(SourceDag->getArgNameStr(I));
-    if (It != SourceOperands.end()) {
-      // Operand sharing the same name in the Dag should be mapped as tied.
-      SourceOperandMap[I].TiedOpIdx = It->getValue();
-      if (!validateArgsTypes(SourceDag->getArg(It->getValue()),
-                             SourceDag->getArg(I)))
-        PrintFatalError(Rec->getLoc(),
-                        "Input Operand '" + SourceDag->getArgNameStr(I) +
-                            "' has a mismatched tied operand!\n");
-    }
-    It = DestOperands.find(SourceDag->getArgNameStr(I));
+    auto It = DestOperands.find(ArgName);
     if (It == DestOperands.end())
-      PrintFatalError(Rec->getLoc(), "Operand " + SourceDag->getArgNameStr(I) +
+      PrintFatalError(Rec->getLoc(), "Operand " + ArgName +
                                          " defined in Input Dag but not used in"
-                                         " Output Dag!\n");
+                                         " Output Dag!");
     // Input Dag operand types must match output Dag operand type.
-    if (!validateArgsTypes(DestDag->getArg(It->getValue()),
+    if (!validateArgsTypes(DestDag->getArg(It->getValue().DAGOpNo),
                            SourceDag->getArg(I)))
       PrintFatalError(Rec->getLoc(), "Type mismatch between Input and "
                                      "Output Dag operand '" +
-                                         SourceDag->getArgNameStr(I) + "'!");
-    SourceOperands[SourceDag->getArgNameStr(I)] = I;
+                                         ArgName + "'!");
   }
 }
 
@@ -396,8 +388,7 @@ void CompressInstEmitter::createDagOperandMapping(
 void CompressInstEmitter::createInstOperandMapping(
     const Record *Rec, const DagInit *SourceDag, const DagInit *DestDag,
     IndexedMap<OpData> &SourceOperandMap, IndexedMap<OpData> &DestOperandMap,
-    StringMap<unsigned> &SourceOperands, const CodeGenInstruction &DestInst,
-    unsigned SourceLastTiedOp) {
+    StringMap<ArgData> &SourceOperands, const CodeGenInstruction &DestInst) {
   // TiedCount keeps track of the number of operands skipped in Inst
   // operands list to get to the corresponding Dag operand.
   unsigned TiedCount = 0;
@@ -407,8 +398,8 @@ void CompressInstEmitter::createInstOperandMapping(
     int TiedInstOpIdx = Operand.getTiedRegister();
     if (TiedInstOpIdx != -1) {
       ++TiedCount;
-      DestOperandMap[OpNo].Data = DestOperandMap[TiedInstOpIdx].Data;
-      DestOperandMap[OpNo].Kind = DestOperandMap[TiedInstOpIdx].Kind;
+      assert((unsigned)TiedInstOpIdx < OpNo);
+      DestOperandMap[OpNo] = DestOperandMap[TiedInstOpIdx];
       if (DestOperandMap[OpNo].Kind == OpData::Operand)
         // No need to fill the SourceOperandMap here since it was mapped to
         // destination operand 'TiedInstOpIdx' in a previous iteration.
@@ -427,29 +418,21 @@ void CompressInstEmitter::createInstOperandMapping(
         continue;
 
       unsigned DagArgIdx = OpNo - TiedCount;
-      StringMap<unsigned>::iterator SourceOp =
-          SourceOperands.find(DestDag->getArgNameStr(DagArgIdx));
+      StringRef ArgName = DestDag->getArgNameStr(DagArgIdx);
+      auto SourceOp = SourceOperands.find(ArgName);
       if (SourceOp == SourceOperands.end())
         PrintFatalError(Rec->getLoc(),
-                        "Output Dag operand '" +
-                            DestDag->getArgNameStr(DagArgIdx) +
+                        "Output Dag operand '" + ArgName +
                             "' has no matching input Dag operand.");
 
-      assert(DestDag->getArgNameStr(DagArgIdx) ==
-                 SourceDag->getArgNameStr(SourceOp->getValue()) &&
+      assert(ArgName ==
+                 SourceDag->getArgNameStr(SourceOp->getValue().DAGOpNo) &&
              "Incorrect operand mapping detected!\n");
 
-      // Following four lines ensure the correct handling of a single tied
-      // operand in the Source Inst. SourceDagOp points to the position of
-      // appropriate Dag argument which is not correct in presence of tied
-      // operand in the Source Inst and must be incremented by 1 to reflect
-      // correct position of the operand in Source Inst
-      unsigned SourceDagOp = SourceOp->getValue();
-      if (SourceDagOp >= SourceLastTiedOp)
-        SourceDagOp++;
-      DestOperandMap[OpNo].Data.Operand = SourceDagOp;
-      SourceOperandMap[SourceDagOp].Data.Operand = OpNo;
-      LLVM_DEBUG(dbgs() << "    " << SourceDagOp << " ====> " << OpNo << "\n");
+      unsigned SourceOpNo = SourceOp->getValue().MIOpNo;
+      DestOperandMap[OpNo].Data.Operand = SourceOpNo;
+      SourceOperandMap[SourceOpNo].Data.Operand = OpNo;
+      LLVM_DEBUG(dbgs() << "    " << SourceOpNo << " ====> " << OpNo << "\n");
     }
   }
 }
@@ -508,25 +491,24 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
   // Fill the mapping from the source to destination instructions.
 
   IndexedMap<OpData> SourceOperandMap;
-  unsigned SourceLastTiedOp; // postion of the last tied operand in Source Inst
+  // Map from arg name to DAG operand number and MI operand number.
+  StringMap<ArgData> SourceOperands;
   // Create a mapping between source Dag operands and source Inst operands.
   addDagOperandMapping(Rec, SourceDag, SourceInst, SourceOperandMap,
-                       /*IsSourceInst*/ true, &SourceLastTiedOp);
+                       SourceOperands, /*IsSourceInst*/ true);
 
   IndexedMap<OpData> DestOperandMap;
+  // Map from arg name to DAG operand number and MI operand number.
+  StringMap<ArgData> DestOperands;
   // Create a mapping between destination Dag operands and destination Inst
   // operands.
-  addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap,
-                       /*IsSourceInst*/ false, nullptr);
+  addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap, DestOperands,
+                       /*IsSourceInst*/ false);
 
-  StringMap<unsigned> SourceOperands;
-  StringMap<unsigned> DestOperands;
-  createDagOperandMapping(Rec, SourceOperands, DestOperands, SourceDag, DestDag,
-                          SourceOperandMap);
+  checkDagOperandMapping(Rec, DestOperands, SourceDag, DestDag);
   // Create operand mapping between the source and destination instructions.
   createInstOperandMapping(Rec, SourceDag, DestDag, SourceOperandMap,
-                           DestOperandMap, SourceOperands, DestInst,
-                           SourceLastTiedOp);
+                           DestOperandMap, SourceOperands, DestInst);
 
   // Get the target features for the CompressPat.
   std::vector<const Record *> PatReqFeatures;
@@ -535,9 +517,9 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
     return R->getValueAsBit("AssemblerMatcherPredicate");
   });
 
-  CompressPatterns.push_back(CompressPat(
-      SourceInst, DestInst, std::move(PatReqFeatures), SourceOperandMap,
-      DestOperandMap, Rec->getValueAsBit("isCompressOnly")));
+  CompressPatterns.emplace_back(SourceInst, DestInst, std::move(PatReqFeatures),
+                                SourceOperandMap, DestOperandMap,
+                                Rec->getValueAsBit("isCompressOnly"));
 }
 
 static void
@@ -624,7 +606,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS,
   if (!AsmWriter->getValueAsInt("PassSubtarget"))
     PrintFatalError(AsmWriter->getLoc(),
                     "'PassSubtarget' is false. SubTargetInfo object is needed "
-                    "for target features.\n");
+                    "for target features.");
 
   StringRef TargetName = Target.getName();
 
@@ -780,7 +762,7 @@ void CompressInstEmitter::emitCompressInstEmitter(raw_ostream &OS,
               << ").getReg() ==  MI.getOperand("
               << SourceOperandMap[OpNo].TiedOpIdx << ").getReg()) &&\n";
         else
-          PrintFatalError("Unexpected tied operand types!\n");
+          PrintFatalError("Unexpected tied operand types!");
       }
       for (unsigned SubOp = 0; SubOp != SourceOperand.MINumOperands; ++SubOp) {
         // Check for fixed immediates\registers in the source instruction.
diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
index 49c33dc5ca14..d3fba6a92357 100644
--- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -1158,7 +1158,7 @@ void MatcherTableEmitter::EmitPredicateFunctions(raw_ostream &OS) {
   EmitNodePredicatesFunction(
       NodePredicatesWithOperands,
       "CheckNodePredicateWithOperands(SDValue Op, unsigned PredNo, "
-      "const SmallVectorImpl<SDValue> &Operands) const",
+      "ArrayRef<SDValue> Operands) const",
       OS);
 
   // Emit CompletePattern matchers.
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index 05bca75bcd4e..8d8189983270 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -282,8 +282,8 @@ static void ContractNodes(std::unique_ptr<Matcher> &InputMatcherPtr,
 #endif
 
         if (ResultsMatch) {
-          const SmallVectorImpl<MVT::SimpleValueType> &VTs = EN->getVTList();
-          const SmallVectorImpl<unsigned> &Operands = EN->getOperandList();
+          ArrayRef<MVT::SimpleValueType> VTs = EN->getVTList();
+          ArrayRef<unsigned> Operands = EN->getOperandList();
           MatcherPtr->reset(new MorphNodeToMatcher(
               EN->getInstruction(), VTs, Operands, EN->hasChain(),
               EN->hasInGlue(), EN->hasOutGlue(), EN->hasMemRefs(),
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 2e8ff2aa47d9..a50fd06435a1 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
@@ -83,6 +84,14 @@ static cl::opt<bool> LargeTable(
              "in the table instead of the default 16 bits."),
     cl::init(false), cl::cat(DisassemblerEmitterCat));
 
+static cl::opt<bool> UseFnTableInDecodeToMCInst(
+    "use-fn-table-in-decode-to-mcinst",
+    cl::desc(
+        "Use a table of function pointers instead of a switch case in the\n"
+        "generated `decodeToMCInst` function. Helps improve compile time\n"
+        "of the generated code."),
+    cl::init(false), cl::cat(DisassemblerEmitterCat));
+
 STATISTIC(NumEncodings, "Number of encodings considered");
 STATISTIC(NumEncodingsLackingDisasm,
           "Number of encodings without disassembler info");
@@ -207,13 +216,6 @@ struct EncodingIDAndOpcode {
 using EncodingIDsVec = std::vector<EncodingIDAndOpcode>;
 using NamespacesHwModesMap = std::map<std::string, std::set<StringRef>>;
 
-raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) {
-  if (Value.EncodingDef != Value.Inst->TheDef)
-    OS << Value.EncodingDef->getName() << ":";
-  OS << Value.Inst->TheDef->getName();
-  return OS;
-}
-
 class DecoderEmitter {
   const RecordKeeper &RK;
   std::vector<EncodingAndInst> NumberedEncodings;
@@ -244,84 +246,96 @@ class DecoderEmitter {
   StringRef PredicateNamespace;
 };
 
-} // end anonymous namespace
-
 // The set (BIT_TRUE, BIT_FALSE, BIT_UNSET) represents a ternary logic system
 // for a bit value.
 //
 // BIT_UNFILTERED is used as the init value for a filter position.  It is used
 // only for filter processings.
-typedef enum : uint8_t {
-  BIT_FALSE,     // '0'
-  BIT_TRUE,      // '1'
-  BIT_UNSET,     // '?'
-  BIT_UNFILTERED // unfiltered
-} bit_value_t;
-
-static bool ValueSet(bit_value_t V) {
-  return (V == BIT_TRUE || V == BIT_FALSE);
-}
-
-static bool ValueNotSet(bit_value_t V) { return (V == BIT_UNSET); }
-
-static int Value(bit_value_t V) {
-  return ValueNotSet(V) ? -1 : (V == BIT_FALSE ? 0 : 1);
-}
+struct BitValue {
+  enum bit_value_t : uint8_t {
+    BIT_FALSE,     // '0'
+    BIT_TRUE,      // '1'
+    BIT_UNSET,     // '?', printed as '_'
+    BIT_UNFILTERED // unfiltered, printed as '.'
+  };
 
-static bit_value_t bitFromBits(const BitsInit &bits, unsigned index) {
-  if (const BitInit *bit = dyn_cast<BitInit>(bits.getBit(index)))
-    return bit->getValue() ? BIT_TRUE : BIT_FALSE;
+  BitValue(bit_value_t V) : V(V) {}
+  explicit BitValue(const Init *Init) {
+    if (const auto *Bit = dyn_cast<BitInit>(Init))
+      V = Bit->getValue() ? BIT_TRUE : BIT_FALSE;
+    else
+      V = BIT_UNSET;
+  }
+  BitValue(const BitsInit &Bits, unsigned Idx) : BitValue(Bits.getBit(Idx)) {}
 
-  // The bit is uninitialized.
-  return BIT_UNSET;
-}
+  bool isSet() const { return V == BIT_TRUE || V == BIT_FALSE; }
+  bool isUnset() const { return V == BIT_UNSET; }
+  std::optional<uint64_t> getValue() const {
+    if (isSet())
+      return static_cast<uint64_t>(V);
+    return std::nullopt;
+  }
 
-// Prints the bit value for each position.
-static void dumpBits(raw_ostream &OS, const BitsInit &bits) {
-  for (unsigned index = bits.getNumBits(); index > 0; --index) {
-    switch (bitFromBits(bits, index - 1)) {
-    case BIT_TRUE:
-      OS << "1";
-      break;
+  // For printing a bit value.
+  operator StringRef() const {
+    switch (V) {
     case BIT_FALSE:
-      OS << "0";
-      break;
+      return "0";
+    case BIT_TRUE:
+      return "1";
     case BIT_UNSET:
-      OS << "_";
-      break;
-    default:
-      llvm_unreachable("unexpected return value from bitFromBits");
+      return "_";
+    case BIT_UNFILTERED:
+      return ".";
     }
+    llvm_unreachable("Unknow bit value");
   }
+
+  bool operator==(bit_value_t Other) const { return Other == V; }
+  bool operator!=(bit_value_t Other) const { return Other != V; }
+
+private:
+  bit_value_t V;
+};
+
+} // end anonymous namespace
+
+static raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) {
+  if (Value.EncodingDef != Value.Inst->TheDef)
+    OS << Value.EncodingDef->getName() << ":";
+  OS << Value.Inst->TheDef->getName();
+  return OS;
 }
 
-static const BitsInit &getBitsField(const Record &def, StringRef str) {
-  const RecordVal *RV = def.getValue(str);
+// Prints the bit value for each position.
+static void dumpBits(raw_ostream &OS, const BitsInit &Bits) {
+  for (const Init *Bit : reverse(Bits.getBits()))
+    OS << BitValue(Bit);
+}
+
+static const BitsInit &getBitsField(const Record &Def, StringRef FieldName) {
+  const RecordVal *RV = Def.getValue(FieldName);
   if (const BitsInit *Bits = dyn_cast<BitsInit>(RV->getValue()))
     return *Bits;
 
-  // variable length instruction
-  VarLenInst VLI = VarLenInst(cast<DagInit>(RV->getValue()), RV);
+  // Handle variable length instructions.
+  VarLenInst VLI(cast<DagInit>(RV->getValue()), RV);
   SmallVector<const Init *, 16> Bits;
 
   for (const auto &SI : VLI) {
-    if (const BitsInit *BI = dyn_cast<BitsInit>(SI.Value)) {
-      for (unsigned Idx = 0U; Idx < BI->getNumBits(); ++Idx) {
-        Bits.push_back(BI->getBit(Idx));
-      }
-    } else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value)) {
+    if (const BitsInit *BI = dyn_cast<BitsInit>(SI.Value))
+      llvm::append_range(Bits, BI->getBits());
+    else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value))
       Bits.push_back(BI);
-    } else {
-      for (unsigned Idx = 0U; Idx < SI.BitWidth; ++Idx)
-        Bits.push_back(UnsetInit::get(def.getRecords()));
-    }
+    else
+      Bits.append(SI.BitWidth, UnsetInit::get(Def.getRecords()));
   }
 
-  return *BitsInit::get(def.getRecords(), Bits);
+  return *BitsInit::get(Def.getRecords(), Bits);
 }
 
 // Representation of the instruction to work on.
-typedef std::vector<bit_value_t> insn_t;
+typedef std::vector<BitValue> insn_t;
 
 namespace {
 
@@ -472,7 +486,7 @@ class FilterChooser {
 
   // Array of bit values passed down from our parent.
   // Set to all BIT_UNFILTERED's for Parent == NULL.
-  std::vector<bit_value_t> FilterBitValues;
+  std::vector<BitValue> FilterBitValues;
 
   // Links to the FilterChooser above us in the decoding tree.
   const FilterChooser *Parent;
@@ -498,15 +512,15 @@ class FilterChooser {
                 const std::map<unsigned, std::vector<OperandInfo>> &Ops,
                 unsigned BW, const DecoderEmitter *E)
       : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
-        FilterBitValues(BW, BIT_UNFILTERED), Parent(nullptr), BestIndex(-1),
-        BitWidth(BW), Emitter(E) {
+        FilterBitValues(BW, BitValue::BIT_UNFILTERED), Parent(nullptr),
+        BestIndex(-1), BitWidth(BW), Emitter(E) {
     doFilter();
   }
 
   FilterChooser(ArrayRef<EncodingAndInst> Insts,
                 ArrayRef<EncodingIDAndOpcode> IDs,
                 const std::map<unsigned, std::vector<OperandInfo>> &Ops,
-                const std::vector<bit_value_t> &ParentFilterBitValues,
+                const std::vector<BitValue> &ParentFilterBitValues,
                 const FilterChooser &parent)
       : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
         FilterBitValues(ParentFilterBitValues), Parent(&parent), BestIndex(-1),
@@ -524,7 +538,7 @@ class FilterChooser {
   void insnWithID(insn_t &Insn, unsigned Opcode) const {
     const Record *EncodingDef = AllInstructions[Opcode].EncodingDef;
     const BitsInit &Bits = getBitsField(*EncodingDef, "Inst");
-    Insn.resize(std::max(BitWidth, Bits.getNumBits()), BIT_UNSET);
+    Insn.resize(std::max(BitWidth, Bits.getNumBits()), BitValue::BIT_UNSET);
     // We may have a SoftFail bitmask, which specifies a mask where an encoding
     // may differ from the value in "Inst" and yet still be valid, but the
     // disassembler should return SoftFail instead of Success.
@@ -533,22 +547,13 @@ class FilterChooser {
     const RecordVal *RV = EncodingDef->getValue("SoftFail");
     const BitsInit *SFBits = RV ? dyn_cast<BitsInit>(RV->getValue()) : nullptr;
     for (unsigned i = 0; i < Bits.getNumBits(); ++i) {
-      if (SFBits && bitFromBits(*SFBits, i) == BIT_TRUE)
-        Insn[i] = BIT_UNSET;
+      if (SFBits && BitValue(*SFBits, i) == BitValue::BIT_TRUE)
+        Insn[i] = BitValue::BIT_UNSET;
       else
-        Insn[i] = bitFromBits(Bits, i);
+        Insn[i] = BitValue(Bits, i);
     }
   }
 
-  // Emit the name of the encoding/instruction pair.
-  void emitNameWithID(raw_ostream &OS, unsigned Opcode) const {
-    const Record *EncodingDef = AllInstructions[Opcode].EncodingDef;
-    const Record *InstDef = AllInstructions[Opcode].Inst->TheDef;
-    if (EncodingDef != InstDef)
-      OS << EncodingDef->getName() << ":";
-    OS << InstDef->getName();
-  }
-
   // Populates the field of the insn given the start position and the number of
   // consecutive bits to scan for.
   //
@@ -560,7 +565,7 @@ class FilterChooser {
 
   /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
   /// filter array as a series of chars.
-  void dumpFilterArray(raw_ostream &OS, ArrayRef<bit_value_t> Filter) const;
+  void dumpFilterArray(raw_ostream &OS, ArrayRef<BitValue> Filter) const;
 
   /// dumpStack - dumpStack traverses the filter chooser chain and calls
   /// dumpFilterArray on each filter chooser up to the top level one.
@@ -571,8 +576,8 @@ class FilterChooser {
     return Filters[BestIndex];
   }
 
-  bool PositionFiltered(unsigned i) const {
-    return ValueSet(FilterBitValues[i]);
+  bool PositionFiltered(unsigned Idx) const {
+    return FilterBitValues[Idx].isSet();
   }
 
   // Calculates the island(s) needed to decode the instruction.
@@ -689,12 +694,12 @@ Filter::Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
 // match the remaining undecoded encoding bits against the singleton.
 void Filter::recurse() {
   // Starts by inheriting our parent filter chooser's filter bit values.
-  std::vector<bit_value_t> BitValueArray(Owner.FilterBitValues);
+  std::vector<BitValue> BitValueArray(Owner.FilterBitValues);
 
   if (!VariableInstructions.empty()) {
     // Conservatively marks each segment position as BIT_UNSET.
     for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
-      BitValueArray[StartBit + bitIndex] = BIT_UNSET;
+      BitValueArray[StartBit + bitIndex] = BitValue::BIT_UNSET;
 
     // Delegates to an inferior filter chooser for further processing on this
     // group of instructions whose segment values are variable.
@@ -715,12 +720,10 @@ void Filter::recurse() {
   // Otherwise, create sub choosers.
   for (const auto &Inst : FilteredInstructions) {
     // Marks all the segment positions with either BIT_TRUE or BIT_FALSE.
-    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex) {
-      if (Inst.first & (1ULL << bitIndex))
-        BitValueArray[StartBit + bitIndex] = BIT_TRUE;
-      else
-        BitValueArray[StartBit + bitIndex] = BIT_FALSE;
-    }
+    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
+      BitValueArray[StartBit + bitIndex] = Inst.first & (1ULL << bitIndex)
+                                               ? BitValue::BIT_TRUE
+                                               : BitValue::BIT_FALSE;
 
     // Delegates to an inferior filter chooser for further processing on this
     // category of instructions.
@@ -1066,31 +1069,67 @@ void DecoderEmitter::emitPredicateFunction(formatted_raw_ostream &OS,
 void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
                                          DecoderSet &Decoders,
                                          indent Indent) const {
-  // The decoder function is just a big switch statement based on the
-  // input decoder index.
-  OS << Indent << "template <typename InsnType>\n";
-  OS << Indent << "static DecodeStatus decodeToMCInst(DecodeStatus S,"
-     << " unsigned Idx, InsnType insn, MCInst &MI,\n";
-  OS << Indent << "                                   uint64_t "
-     << "Address, const MCDisassembler *Decoder, bool &DecodeComplete) {\n";
-  Indent += 2;
-  OS << Indent << "DecodeComplete = true;\n";
+  // The decoder function is just a big switch statement or a table of function
+  // pointers based on the input decoder index.
+
   // TODO: When InsnType is large, using uint64_t limits all fields to 64 bits
   // It would be better for emitBinaryParser to use a 64-bit tmp whenever
   // possible but fall back to an InsnType-sized tmp for truly large fields.
-  OS << Indent
-     << "using TmpType = "
-        "std::conditional_t<std::is_integral<InsnType>::"
-        "value, InsnType, uint64_t>;\n";
-  OS << Indent << "TmpType tmp;\n";
-  OS << Indent << "switch (Idx) {\n";
-  OS << Indent << "default: llvm_unreachable(\"Invalid index!\");\n";
-  for (const auto &[Index, Decoder] : enumerate(Decoders)) {
-    OS << Indent << "case " << Index << ":\n";
-    OS << Decoder;
-    OS << Indent + 2 << "return S;\n";
+  StringRef TmpTypeDecl =
+      "using TmpType = std::conditional_t<std::is_integral<InsnType>::value, "
+      "InsnType, uint64_t>;\n";
+  StringRef DecodeParams =
+      "DecodeStatus S, InsnType insn, MCInst &MI, uint64_t Address, const "
+      "MCDisassembler *Decoder, bool &DecodeComplete";
+
+  if (UseFnTableInDecodeToMCInst) {
+    // Emit a function for each case first.
+    for (const auto &[Index, Decoder] : enumerate(Decoders)) {
+      OS << Indent << "template <typename InsnType>\n";
+      OS << Indent << "DecodeStatus decodeFn" << Index << "(" << DecodeParams
+         << ") {\n";
+      Indent += 2;
+      OS << Indent << TmpTypeDecl;
+      OS << Indent << "[[maybe_unused]] TmpType tmp;\n";
+      OS << Decoder;
+      OS << Indent << "return S;\n";
+      Indent -= 2;
+      OS << Indent << "}\n\n";
+    }
+  }
+
+  OS << Indent << "// Handling " << Decoders.size() << " cases.\n";
+  OS << Indent << "template <typename InsnType>\n";
+  OS << Indent << "static DecodeStatus decodeToMCInst(unsigned Idx, "
+     << DecodeParams << ") {\n";
+  Indent += 2;
+  OS << Indent << "DecodeComplete = true;\n";
+
+  if (UseFnTableInDecodeToMCInst) {
+    // Build a table of function pointers.
+    OS << Indent << "using DecodeFnTy = DecodeStatus (*)(" << DecodeParams
+       << ");\n";
+    OS << Indent << "static constexpr DecodeFnTy decodeFnTable[] = {\n";
+    for (size_t Index : llvm::seq(Decoders.size()))
+      OS << Indent + 2 << "decodeFn" << Index << ",\n";
+    OS << Indent << "};\n";
+    OS << Indent << "if (Idx >= " << Decoders.size() << ")\n";
+    OS << Indent + 2 << "llvm_unreachable(\"Invalid index!\");\n";
+    OS << Indent
+       << "return decodeFnTable[Idx](S, insn, MI, Address, Decoder, "
+          "DecodeComplete);\n";
+  } else {
+    OS << Indent << TmpTypeDecl;
+    OS << Indent << "TmpType tmp;\n";
+    OS << Indent << "switch (Idx) {\n";
+    OS << Indent << "default: llvm_unreachable(\"Invalid index!\");\n";
+    for (const auto &[Index, Decoder] : enumerate(Decoders)) {
+      OS << Indent << "case " << Index << ":\n";
+      OS << Decoder;
+      OS << Indent + 2 << "return S;\n";
+    }
+    OS << Indent << "}\n";
   }
-  OS << Indent << "}\n";
   Indent -= 2;
   OS << Indent << "}\n";
 }
@@ -1107,10 +1146,10 @@ std::pair<bool, uint64_t> FilterChooser::fieldFromInsn(const insn_t &Insn,
   uint64_t Field = 0;
 
   for (unsigned i = 0; i < NumBits; ++i) {
-    if (Insn[StartBit + i] == BIT_UNSET)
+    if (Insn[StartBit + i] == BitValue::BIT_UNSET)
       return {false, Field};
 
-    if (Insn[StartBit + i] == BIT_TRUE)
+    if (Insn[StartBit + i] == BitValue::BIT_TRUE)
       Field = Field | (1ULL << i);
   }
 
@@ -1120,23 +1159,9 @@ std::pair<bool, uint64_t> FilterChooser::fieldFromInsn(const insn_t &Insn,
 /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
 /// filter array as a series of chars.
 void FilterChooser::dumpFilterArray(raw_ostream &OS,
-                                    ArrayRef<bit_value_t> Filter) const {
-  for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--) {
-    switch (Filter[bitIndex - 1]) {
-    case BIT_UNFILTERED:
-      OS << ".";
-      break;
-    case BIT_UNSET:
-      OS << "_";
-      break;
-    case BIT_TRUE:
-      OS << "1";
-      break;
-    case BIT_FALSE:
-      OS << "0";
-      break;
-    }
-  }
+                                    ArrayRef<BitValue> Filter) const {
+  for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--)
+    OS << Filter[bitIndex - 1];
 }
 
 /// dumpStack - dumpStack traverses the filter chooser chain and calls
@@ -1167,28 +1192,28 @@ unsigned FilterChooser::getIslands(std::vector<Island> &Islands,
   unsigned State = 0;
 
   for (unsigned i = 0; i < BitWidth; ++i) {
-    int64_t Val = Value(Insn[i]);
+    std::optional<uint64_t> Val = Insn[i].getValue();
     bool Filtered = PositionFiltered(i);
     switch (State) {
     default:
       llvm_unreachable("Unreachable code!");
     case 0:
     case 1:
-      if (Filtered || Val == -1) {
+      if (Filtered || !Val) {
         State = 1; // Still in Water
       } else {
         State = 2; // Into the Island
         StartBit = i;
-        FieldVal = Val;
+        FieldVal = *Val;
       }
       break;
     case 2:
-      if (Filtered || Val == -1) {
+      if (Filtered || !Val) {
         State = 1; // Into the Water
         Islands.push_back({StartBit, i - StartBit, FieldVal});
       } else {
         State = 2; // Still in Island
-        FieldVal |= Val << (i - StartBit);
+        FieldVal |= *Val << (i - StartBit);
       }
       break;
     }
@@ -1267,7 +1292,8 @@ std::pair<unsigned, bool> FilterChooser::getDecoderIndex(DecoderSet &Decoders,
   // FIXME: emitDecoder() function can take a buffer directly rather than
   // a stream.
   raw_svector_ostream S(Decoder);
-  bool HasCompleteDecoder = emitDecoder(S, indent(4), Opc);
+  indent Indent(UseFnTableInDecodeToMCInst ? 2 : 4);
+  bool HasCompleteDecoder = emitDecoder(S, Indent, Opc);
 
   // Using the full decoder string as the key value here is a bit
   // heavyweight, but is effective. If the string comparisons become a
@@ -1408,22 +1434,19 @@ void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
   APInt PositiveMask(BitWidth, 0ULL);
   APInt NegativeMask(BitWidth, 0ULL);
   for (unsigned i = 0; i < BitWidth; ++i) {
-    bit_value_t B = bitFromBits(*SFBits, i);
-    bit_value_t IB = bitFromBits(*InstBits, i);
+    BitValue B(*SFBits, i);
+    BitValue IB(*InstBits, i);
 
-    if (B != BIT_TRUE)
+    if (B != BitValue::BIT_TRUE)
       continue;
 
-    switch (IB) {
-    case BIT_FALSE:
+    if (IB == BitValue::BIT_FALSE) {
       // The bit is meant to be false, so emit a check to see if it is true.
       PositiveMask.setBit(i);
-      break;
-    case BIT_TRUE:
+    } else if (IB == BitValue::BIT_TRUE) {
       // The bit is meant to be true, so emit a check to see if it is false.
       NegativeMask.setBit(i);
-      break;
-    default:
+    } else {
       // The bit is not set; this must be an error!
       errs() << "SoftFail Conflict: bit SoftFail{" << i << "} in "
              << AllInstructions[Opc] << " is set but Inst{" << i
@@ -1606,8 +1629,7 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   // FILTERED bit positions provide no entropy and are not worthy of pursuing.
   // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position.
   for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex)
-    if (FilterBitValues[BitIndex] == BIT_TRUE ||
-        FilterBitValues[BitIndex] == BIT_FALSE)
+    if (FilterBitValues[BitIndex].isSet())
       bitAttrs[BitIndex] = ATTR_FILTERED;
 
   for (const auto &OpcPair : Opcodes) {
@@ -1618,17 +1640,17 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) {
       switch (bitAttrs[BitIndex]) {
       case ATTR_NONE:
-        if (insn[BitIndex] == BIT_UNSET)
+        if (insn[BitIndex] == BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_ALL_UNSET;
         else
           bitAttrs[BitIndex] = ATTR_ALL_SET;
         break;
       case ATTR_ALL_SET:
-        if (insn[BitIndex] == BIT_UNSET)
+        if (insn[BitIndex] == BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_MIXED;
         break;
       case ATTR_ALL_UNSET:
-        if (insn[BitIndex] != BIT_UNSET)
+        if (insn[BitIndex] != BitValue::BIT_UNSET)
           bitAttrs[BitIndex] = ATTR_MIXED;
         break;
       case ATTR_MIXED:
@@ -1823,7 +1845,7 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
     return;
   }
 
-  // We don't know how to decode these instructions!  Dump the
+  // We don't know how to decode these instructions! Dump the
   // conflict set and bail.
 
   // Print out useful conflict information for postmortem analysis.
@@ -1832,14 +1854,12 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
   dumpStack(errs(), "\t\t");
 
   for (auto Opcode : Opcodes) {
-    errs() << '\t';
-    emitNameWithID(errs(), Opcode.EncodingID);
-    errs() << " ";
-    dumpBits(
-        errs(),
-        getBitsField(*AllInstructions[Opcode.EncodingID].EncodingDef, "Inst"));
+    const EncodingAndInst &Enc = AllInstructions[Opcode.EncodingID];
+    errs() << '\t' << Enc << ' ';
+    dumpBits(errs(), getBitsField(*Enc.EncodingDef, "Inst"));
     errs() << '\n';
   }
+  PrintFatalError("Decoding conflict encountered");
 }
 
 static std::string findOperandDecoderMethod(const Record *Record) {
@@ -2371,7 +2391,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
        << "      makeUp(insn, Len);";
   }
   OS << R"(
-      S = decodeToMCInst(S, DecodeIdx, insn, MI, Address, DisAsm, DecodeComplete);
+      S = decodeToMCInst(DecodeIdx, S, insn, MI, Address, DisAsm, DecodeComplete);
       assert(DecodeComplete);
 
       LLVM_DEBUG(dbgs() << Loc << ": OPC_Decode: opcode " << Opc
@@ -2393,7 +2413,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
       MCInst TmpMI;
       TmpMI.setOpcode(Opc);
       bool DecodeComplete;
-      S = decodeToMCInst(S, DecodeIdx, insn, TmpMI, Address, DisAsm, DecodeComplete);
+      S = decodeToMCInst(DecodeIdx, S, insn, TmpMI, Address, DisAsm, DecodeComplete);
       LLVM_DEBUG(dbgs() << Loc << ": OPC_TryDecode: opcode " << Opc
                    << ", using decoder " << DecodeIdx << ": ");
 
@@ -2552,7 +2572,7 @@ namespace {
   if (HwModeNames.empty())
     HwModeNames.push_back("");
 
-  const auto &NumberedInstructions = Target.getInstructionsByEnumValue();
+  const auto &NumberedInstructions = Target.getInstructions();
   NumberedEncodings.reserve(NumberedInstructions.size());
   for (const auto &NumberedInstruction : NumberedInstructions) {
     const Record *InstDef = NumberedInstruction->TheDef;
diff --git a/llvm/utils/TableGen/DisassemblerEmitter.cpp b/llvm/utils/TableGen/DisassemblerEmitter.cpp
index 70d835e699ff..15a11a3a9940 100644
--- a/llvm/utils/TableGen/DisassemblerEmitter.cpp
+++ b/llvm/utils/TableGen/DisassemblerEmitter.cpp
@@ -103,8 +103,7 @@ static void emitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
   if (Target.getName() == "X86") {
     DisassemblerTables Tables;
 
-    for (const auto &[Idx, NumberedInst] :
-         enumerate(Target.getInstructionsByEnumValue()))
+    for (const auto &[Idx, NumberedInst] : enumerate(Target.getInstructions()))
       RecognizableInstr::processInstr(Tables, *NumberedInst, Idx);
 
     if (Tables.hasConflicts()) {
@@ -120,7 +119,7 @@ static void emitDisassembler(const RecordKeeper &Records, raw_ostream &OS) {
   // below (which depends on a Size table-gen Record), and also uses a custom
   // disassembler.
   if (Target.getName() == "WebAssembly") {
-    emitWebAssemblyDisassemblerTables(OS, Target.getInstructionsByEnumValue());
+    emitWebAssemblyDisassemblerTables(OS, Target.getInstructions());
     return;
   }
 
diff --git a/llvm/utils/TableGen/InstrDocsEmitter.cpp b/llvm/utils/TableGen/InstrDocsEmitter.cpp
index a8a234675dec..7ca34ad05a72 100644
--- a/llvm/utils/TableGen/InstrDocsEmitter.cpp
+++ b/llvm/utils/TableGen/InstrDocsEmitter.cpp
@@ -72,7 +72,7 @@ static void EmitInstrDocs(const RecordKeeper &RK, raw_ostream &OS) {
   writeTitle(Title, OS);
   OS << "\n";
 
-  for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *II : Target.getInstructions()) {
     const Record *Inst = II->TheDef;
 
     // Don't print the target-independent instructions.
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index b6f16789bf95..8a220338bf27 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -91,10 +91,10 @@ class InstrInfoEmitter {
       ArrayRef<const CodeGenInstruction *> NumberedInstructions);
   void emitOperandNameMappings(
       raw_ostream &OS, const CodeGenTarget &Target,
-      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+      ArrayRef<const CodeGenInstruction *> TargetInstructions);
   void emitLogicalOperandSizeMappings(
       raw_ostream &OS, StringRef Namespace,
-      ArrayRef<const CodeGenInstruction *> NumberedInstructions);
+      ArrayRef<const CodeGenInstruction *> TargetInstructions);
 
   // Operand information.
   unsigned CollectOperandInfo(OperandInfoListTy &OperandInfoList,
@@ -201,7 +201,7 @@ InstrInfoEmitter::CollectOperandInfo(OperandInfoListTy &OperandInfoList,
                                      OperandInfoMapTy &OperandInfoMap) {
   const CodeGenTarget &Target = CDP.getTargetInfo();
   unsigned Offset = 0;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     OperandInfoTy OperandInfo = GetOperandInfo(*Inst);
     if (OperandInfoMap.try_emplace(OperandInfo, Offset).second) {
       OperandInfoList.push_back(OperandInfo);
@@ -234,9 +234,12 @@ void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
 /// - A function called getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
 ///   for looking up the operand index for an instruction, given a value from
 ///   OpName enum
+///
+/// Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
+/// we can just skip them. Hence accept just the TargetInstructions.
 void InstrInfoEmitter::emitOperandNameMappings(
     raw_ostream &OS, const CodeGenTarget &Target,
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+    ArrayRef<const CodeGenInstruction *> TargetInstructions) {
   StringRef Namespace = Target.getInstNamespace();
 
   /// To facilitate assigning OpName enum values in the sorted alphabetical
@@ -260,9 +263,7 @@ void InstrInfoEmitter::emitOperandNameMappings(
 
   // Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
   // we can just skip them.
-  const unsigned NumFixedInsts = Target.getNumFixedInstructions();
-  for (const CodeGenInstruction *Inst :
-       NumberedInstructions.drop_front(NumFixedInsts)) {
+  for (const CodeGenInstruction *Inst : TargetInstructions) {
     if (!Inst->TheDef->getValueAsBit("UseNamedOperandTable"))
       continue;
     std::map<unsigned, unsigned> OpList;
@@ -476,19 +477,18 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "#endif // GET_INSTRINFO_MEM_OPERAND_SIZE\n\n";
 }
 
+// Fixed/Predefined instructions do not have UseLogicalOperandMappings
+// enabled, so we can just skip them. Hence accept TargetInstructions.
 void InstrInfoEmitter::emitLogicalOperandSizeMappings(
     raw_ostream &OS, StringRef Namespace,
-    ArrayRef<const CodeGenInstruction *> NumberedInstructions) {
+    ArrayRef<const CodeGenInstruction *> TargetInstructions) {
   std::map<std::vector<unsigned>, unsigned> LogicalOpSizeMap;
   std::map<unsigned, std::vector<std::string>> InstMap;
 
   size_t LogicalOpListSize = 0U;
   std::vector<unsigned> LogicalOpList;
 
-  // Fixed/Predefined instructions do not have UseLogicalOperandMappings
-  // enabled, so we can just skip them.
-  const unsigned NumFixedInsts = CDP.getTargetInfo().getNumFixedInstructions();
-  for (const auto *Inst : NumberedInstructions.drop_front(NumFixedInsts)) {
+  for (const auto *Inst : TargetInstructions) {
     if (!Inst->TheDef->getValueAsBit("UseLogicalOperandMappings"))
       continue;
 
@@ -645,7 +645,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
       Target.getName(), "", "computeAvailableFeatures", SubtargetFeatures, OS);
 
   std::vector<std::vector<const Record *>> FeatureBitsets;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     FeatureBitsets.emplace_back();
     for (const Record *Predicate :
          Inst->TheDef->getValueAsListOfDefs("Predicates")) {
@@ -696,7 +696,7 @@ void InstrInfoEmitter::emitFeatureVerifier(raw_ostream &OS,
      << "  static constexpr " << getMinimalTypeForRange(FeatureBitsets.size())
      << " RequiredFeaturesRefs[] = {\n";
   unsigned InstIdx = 0;
-  for (const CodeGenInstruction *Inst : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *Inst : Target.getInstructions()) {
     OS << "    CEFBS";
     unsigned NumPredicates = 0;
     for (const Record *Predicate :
@@ -818,7 +818,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   const CodeGenTarget &Target = CDP.getTargetInfo();
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   emitEnums(OS, NumberedInstructions);
 
@@ -1057,9 +1057,12 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
 
+  ArrayRef<const CodeGenInstruction *> TargetInstructions =
+      Target.getTargetInstructions();
+
   if (HasUseNamedOperandTable) {
     Timer.startTimer("Emit operand name mappings");
-    emitOperandNameMappings(OS, Target, NumberedInstructions);
+    emitOperandNameMappings(OS, Target, TargetInstructions);
   }
 
   Timer.startTimer("Emit operand type mappings");
@@ -1067,7 +1070,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   if (HasUseLogicalOperandMappings) {
     Timer.startTimer("Emit logical operand size mappings");
-    emitLogicalOperandSizeMappings(OS, TargetName, NumberedInstructions);
+    emitLogicalOperandSizeMappings(OS, TargetName, TargetInstructions);
   }
 
   Timer.startTimer("Emit helper methods");
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 77618b24cf11..408fe544d260 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -27,6 +27,8 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/StringToOffsetTable.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
 #include <cassert>
@@ -1380,6 +1382,10 @@ void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
   }
   OS << "}; // " << Target << "ReadAdvanceTable\n";
 
+  // Pool all SchedClass names in a string table.
+  StringToOffsetTable StrTab;
+  unsigned InvalidNameOff = StrTab.GetOrAddStringOffset("InvalidSchedClass");
+
   // Emit a SchedClass table for each processor.
   for (const auto &[Idx, Proc] : enumerate(SchedModels.procModels())) {
     if (!Proc.hasInstrSchedModel())
@@ -1397,14 +1403,15 @@ void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
     // name and position.
     assert(SchedModels.getSchedClass(0).Name == "NoInstrModel" &&
            "invalid class not first");
-    OS << "  {DBGFIELD(\"InvalidSchedClass\")  "
+    OS << "  {DBGFIELD(" << InvalidNameOff << ")  "
        << MCSchedClassDesc::InvalidNumMicroOps
        << ", false, false, false, 0, 0,  0, 0,  0, 0},\n";
 
     for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) {
       MCSchedClassDesc &MCDesc = SCTab[SCIdx];
       const CodeGenSchedClass &SchedClass = SchedModels.getSchedClass(SCIdx);
-      OS << "  {DBGFIELD(\"" << SchedClass.Name << "\") ";
+      unsigned NameOff = StrTab.GetOrAddStringOffset(SchedClass.Name);
+      OS << "  {DBGFIELD(/*" << SchedClass.Name << "*/ " << NameOff << ") ";
       if (SchedClass.Name.size() < 18)
         OS.indent(18 - SchedClass.Name.size());
       OS << MCDesc.NumMicroOps << ", " << (MCDesc.BeginGroup ? "true" : "false")
@@ -1419,6 +1426,8 @@ void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
     }
     OS << "}; // " << Proc.ModelName << "SchedClasses\n";
   }
+
+  StrTab.EmitStringTableDef(OS, Target + "SchedClassNames");
 }
 
 void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
@@ -1472,6 +1481,8 @@ void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
     else
       OS << "  nullptr, nullptr, 0, 0,"
          << " // No instruction-level machine model.\n";
+    OS << "  DBGVAL_OR_NULLPTR(&" << Target
+       << "SchedClassNames), // SchedClassNames\n";
     if (PM.hasItineraries())
       OS << "  " << PM.ItinsDef->getName() << ",\n";
     else
@@ -1493,8 +1504,10 @@ void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
      << "#endif\n"
      << "#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)\n"
      << "#define DBGFIELD(x) x,\n"
+     << "#define DBGVAL_OR_NULLPTR(x) x\n"
      << "#else\n"
      << "#define DBGFIELD(x)\n"
+     << "#define DBGVAL_OR_NULLPTR(x) nullptr\n"
      << "#endif\n";
 
   if (SchedModels.hasItineraries()) {
@@ -1512,10 +1525,11 @@ void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
   }
   emitSchedClassTables(SchedTables, OS);
 
-  OS << "\n#undef DBGFIELD\n";
-
   // Emit the processor machine model
   emitProcessorModels(OS);
+
+  OS << "\n#undef DBGFIELD\n";
+  OS << "\n#undef DBGVAL_OR_NULLPTR\n";
 }
 
 static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index bcc5712b9154..d63570a88a4c 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -625,7 +625,7 @@ void X86FoldTablesEmitter::run(raw_ostream &OS) {
   std::map<uint8_t, std::vector<const CodeGenInstruction *>> RegInsts;
 
   ArrayRef<const CodeGenInstruction *> NumberedInstructions =
-      Target.getInstructionsByEnumValue();
+      Target.getInstructions();
 
   for (const CodeGenInstruction *Inst : NumberedInstructions) {
     const Record *Rec = Inst->TheDef;
diff --git a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
index 5a10aba8f357..2e8351c95198 100644
--- a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
+++ b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
@@ -373,8 +373,7 @@ void X86InstrMappingEmitter::emitSSE2AVXTable(
 void X86InstrMappingEmitter::run(raw_ostream &OS) {
   emitSourceFileHeader("X86 instruction mapping", OS);
 
-  ArrayRef<const CodeGenInstruction *> Insts =
-      Target.getInstructionsByEnumValue();
+  ArrayRef<const CodeGenInstruction *> Insts = Target.getInstructions();
   printClassDef(OS);
   emitCompressEVEXTable(Insts, OS);
   emitNFTransformTable(Insts, OS);
diff --git a/llvm/utils/TableGen/X86MnemonicTables.cpp b/llvm/utils/TableGen/X86MnemonicTables.cpp
index ddbfb2af9869..f469fcabf7c3 100644
--- a/llvm/utils/TableGen/X86MnemonicTables.cpp
+++ b/llvm/utils/TableGen/X86MnemonicTables.cpp
@@ -40,7 +40,7 @@ void X86MnemonicTablesEmitter::run(raw_ostream &OS) {
   // Hold all instructions grouped by mnemonic
   StringMap<SmallVector<const CodeGenInstruction *, 0>> MnemonicToCGInstrMap;
 
-  for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue()) {
+  for (const CodeGenInstruction *I : Target.getInstructions()) {
     const Record *Def = I->TheDef;
     // Filter non-X86 instructions.
     if (!Def->isSubClassOf("X86Inst"))
diff --git a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
index f4a679345d71..d4ec80b3d5be 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Core/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Core") {
     "//llvm/lib/BinaryFormat",
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/Demangle",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCDisassembler",
diff --git a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
index 5100478114d9..b856d1c2f7fb 100644
--- a/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/lib/Rewrite/BUILD.gn
@@ -14,6 +14,7 @@ static_library("Rewrite") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWP",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/ExecutionEngine/JITLink",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
index 21e348da90b9..18b13d5c382c 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
@@ -51,6 +51,7 @@ static_library("modernize") {
     "UseNullptrCheck.cpp",
     "UseOverrideCheck.cpp",
     "UseRangesCheck.cpp",
+    "UseScopedLockCheck.cpp",
     "UseStartsEndsWithCheck.cpp",
     "UseStdFormatCheck.cpp",
     "UseStdNumbersCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
index 98c78dc15b32..7c05c77d9753 100644
--- a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -94,7 +94,6 @@ static_library("Driver") {
     "ToolChains/PPCFreeBSD.cpp",
     "ToolChains/PPCLinux.cpp",
     "ToolChains/PS4CPU.cpp",
-    "ToolChains/RISCVToolchain.cpp",
     "ToolChains/SPIRV.cpp",
     "ToolChains/SPIRVOpenMP.cpp",
     "ToolChains/SYCL.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
index 4b79e989545b..21266ed7f905 100644
--- a/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Interpreter/BUILD.gn
@@ -31,6 +31,7 @@ static_library("Interpreter") {
     "Interpreter.cpp",
     "InterpreterUtils.cpp",
     "InterpreterValuePrinter.cpp",
+    "RemoteJITUtils.cpp",
     "Value.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 99f8e2fdc2e0..ab72bf3c9c00 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -48,6 +48,7 @@ write_lit_config("lit_site_cfg") {
     "LLVM_LINK_LLVM_DYLIB=0",
     "CLANG_LINK_CLANG_DYLIB=0",
     "LLVM_WITH_Z3=0",
+    "TEST_WITH_Z3_MOCK=0",
     "CLANG_BUILD_EXAMPLES=0",
     "CLANG_BUILT_STANDALONE=0",
     "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
@@ -57,15 +58,6 @@ write_lit_config("lit_site_cfg") {
     "CLANG_PLUGIN_SUPPORT=0",  # FIXME: Analysis/plugins need global -fPIC
     "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin", dir),
     "CLANG_VENDOR_UTI=org.llvm.clang",
-
-    # This is only used if LLVM_USE_SANITIZER includes lsan and the host
-    # OS is macOS. Since the GN build currently never uses LLVM_USE_SANITIZER,
-    # this is never read.  If it's ever needed,
-    # utils/gn/build/toolchain/BUILD.gn should get the compiler from a variable
-    # that's also read here -- but that should happen after multi-toolchain
-    # builds exist, to make sure it's a toolchain var.
-    "CMAKE_CXX_COMPILER=c++",
-    "CMAKE_C_COMPILER=cc",
     "ENABLE_BACKTRACES=1",
     "ENABLE_SHARED=0",
     "LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS=0",
@@ -77,7 +69,6 @@ write_lit_config("lit_site_cfg") {
     "LLVM_USE_SANITIZER=",
     "LLVM_VERSION_MAJOR=$llvm_version_major",
     "Python3_EXECUTABLE=$python_path",
-    "USE_Z3_SOLVER=",
     "PPC_LINUX_DEFAULT_IEEELONGDOUBLE=0",
   ]
 
diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
index 5ba8f2171eab..b1ebfcefbfcd 100644
--- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -46,3 +46,10 @@ group("unittests") {
   }
   testonly = true
 }
+
+# CMake puts most tests into an AllClangUnitTests target. The GN build
+# doesn't do this. This keeps the sync script happy.
+group("dummy_sources") {
+  sources = [ "AllClangUnitTests.cpp" ]
+  not_needed(sources)
+}
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 7b9966d9bfc5..6051674a790e 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -1155,6 +1155,7 @@ if (current_toolchain == default_toolchain) {
       "__iterator/ostreambuf_iterator.h",
       "__iterator/permutable.h",
       "__iterator/prev.h",
+      "__iterator/product_iterator.h",
       "__iterator/projected.h",
       "__iterator/ranges_iterator_traits.h",
       "__iterator/readable_traits.h",
@@ -1167,13 +1168,19 @@ if (current_toolchain == default_toolchain) {
       "__iterator/unreachable_sentinel.h",
       "__iterator/wrap_iter.h",
       "__locale",
+      "__locale_dir/check_grouping.h",
+      "__locale_dir/get_c_locale.h",
       "__locale_dir/locale_base_api.h",
       "__locale_dir/locale_base_api/android.h",
       "__locale_dir/locale_base_api/bsd_locale_fallbacks.h",
       "__locale_dir/locale_base_api/ibm.h",
       "__locale_dir/locale_base_api/musl.h",
       "__locale_dir/locale_base_api/openbsd.h",
+      "__locale_dir/messages.h",
+      "__locale_dir/money.h",
+      "__locale_dir/num.h",
       "__locale_dir/pad_and_output.h",
+      "__locale_dir/scan_keyword.h",
       "__locale_dir/support/apple.h",
       "__locale_dir/support/bsd_like.h",
       "__locale_dir/support/freebsd.h",
@@ -1182,6 +1189,9 @@ if (current_toolchain == default_toolchain) {
       "__locale_dir/support/no_locale/characters.h",
       "__locale_dir/support/no_locale/strtonum.h",
       "__locale_dir/support/windows.h",
+      "__locale_dir/time.h",
+      "__locale_dir/wbuffer_convert.h",
+      "__locale_dir/wstring_convert.h",
       "__math/abs.h",
       "__math/copysign.h",
       "__math/error_functions.h",
@@ -1434,9 +1444,8 @@ if (current_toolchain == default_toolchain) {
       "__tuple/tuple_size.h",
       "__tuple/tuple_types.h",
       "__type_traits/add_cv_quals.h",
-      "__type_traits/add_lvalue_reference.h",
       "__type_traits/add_pointer.h",
-      "__type_traits/add_rvalue_reference.h",
+      "__type_traits/add_reference.h",
       "__type_traits/aligned_storage.h",
       "__type_traits/aligned_union.h",
       "__type_traits/alignment_of.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
index 1b4ab1df0a35..672770ef8ed5 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/Utility/BUILD.gn
@@ -38,7 +38,6 @@ static_library("Utility") {
     "OpenBSDSignals.cpp",
     "RegisterContextDarwin_arm.cpp",
     "RegisterContextDarwin_arm64.cpp",
-    "RegisterContextDarwin_i386.cpp",
     "RegisterContextDarwin_riscv32.cpp",
     "RegisterContextDarwin_x86_64.cpp",
     "RegisterContextDummy.cpp",
@@ -51,7 +50,6 @@ static_library("Utility") {
     "RegisterContextLinux_s390x.cpp",
     "RegisterContextLinux_x86_64.cpp",
     "RegisterContextMach_arm.cpp",
-    "RegisterContextMach_i386.cpp",
     "RegisterContextMach_x86_64.cpp",
     "RegisterContextMemory.cpp",
     "RegisterContextNetBSD_i386.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
index fa8f8b49721c..8963e65f4a42 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Process/mach-core/BUILD.gn
@@ -18,6 +18,7 @@ static_library("mach-core") {
   include_dirs = [ "//lldb/source" ]
   sources = [
     "ProcessMachCore.cpp",
+    "RegisterContextUnifiedCore.cpp",
     "ThreadMachCore.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index f4ee2599c01c..e9f91057cd32 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -304,6 +304,7 @@ write_cmake_config("llvm-config") {
     "LLVM_ENABLE_DEBUGLOC_TRACKING_ORIGIN=",
     "LLVM_ENABLE_DUMP=",
     "LLVM_ENABLE_HTTPLIB=",
+    "LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS=",
     "LLVM_FORCE_USE_OLD_TOOLCHAIN=",
     "LLVM_HAS_ATOMICS=1",
     "LLVM_HAS_LOGF128=",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
index 87e58608617a..7cf5a8148f92 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
@@ -6,6 +6,12 @@ tablegen("Attributes") {
   tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
+tablegen("RuntimeLibcalls") {
+  visibility = [ ":public_tablegen" ]
+  args = [ "-gen-runtime-libcalls" ]
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
+}
+
 tablegen("IntrinsicImpl") {
   visibility = [ "//llvm/lib/IR" ]
   args = [ "-gen-intrinsic-impl" ]
@@ -117,6 +123,9 @@ group("public_tablegen") {
     # IR's public headers include IntrinsicEnums.inc.
     ":IntrinsicEnums",
 
+    # IR's public headers include RuntimeLibcalls.inc.
+    ":RuntimeLibcalls",
+
     # FIXME: These are only included from .cpp files at first glance.
     # Try removing them from the public_tablegen target.
     ":IntrinsicsAArch64",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
index 87368557c42c..8b38f1a9f562 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
@@ -6,6 +6,7 @@ static_library("AsmPrinter") {
     "//llvm/lib/CodeGen",
     "//llvm/lib/DebugInfo/CodeView",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/MSF",
     "//llvm/lib/IR",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
index 91b892dde8d8..a4160535c7cf 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
@@ -253,6 +253,7 @@ static_library("CodeGen") {
     "WasmEHPrepare.cpp",
     "WinEHPrepare.cpp",
     "WindowScheduler.cpp",
+    "WindowsSecureHotPatching.cpp",
     "XRayInstrumentation.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
index b3a8e0331e68..3258f519cf5a 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Classic/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Classic") {
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DWARFLinker",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
index 117508a5cc92..9844d3ea39b1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DWARFLinker/Parallel/BUILD.gn
@@ -6,6 +6,7 @@ static_library("Parallel") {
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DWARFLinker",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
index cb46f7cf55fe..f2e518de8186 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/BUILD.gn
@@ -2,6 +2,7 @@ static_library("DWARF") {
   output_name = "LLVMDebugInfoDWARF"
   deps = [
     "//llvm/lib/BinaryFormat",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
@@ -12,7 +13,6 @@ static_library("DWARF") {
     "DWARFAcceleratorTable.cpp",
     "DWARFAddressRange.cpp",
     "DWARFCFIPrinter.cpp",
-    "DWARFCFIProgram.cpp",
     "DWARFCompileUnit.cpp",
     "DWARFContext.cpp",
     "DWARFDebugAbbrev.cpp",
@@ -28,7 +28,7 @@ static_library("DWARF") {
     "DWARFDebugRangeList.cpp",
     "DWARFDebugRnglists.cpp",
     "DWARFDie.cpp",
-    "DWARFExpression.cpp",
+    "DWARFExpressionPrinter.cpp",
     "DWARFFormValue.cpp",
     "DWARFGdbIndex.cpp",
     "DWARFListTable.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn
new file mode 100644
index 000000000000..a8cf7845b023
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/DWARF/LowLevel/BUILD.gn
@@ -0,0 +1,12 @@
+static_library("LowLevel") {
+  output_name = "LLVMDebugInfoDWARFLowLevel"
+  deps = [
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Support",
+    "//llvm/lib/TargetParser",
+  ]
+  sources = [
+    "DWARFCFIProgram.cpp",
+    "DWARFExpression.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
index cca2cb7ee501..ecccdf5ea63e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/LogicalView/BUILD.gn
@@ -4,6 +4,7 @@ static_library("LogicalView") {
     "//llvm/lib/BinaryFormat",
     "//llvm/lib/DebugInfo/CodeView",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/PDB",
     "//llvm/lib/Demangle",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 7eb9311b292e..a68cee4286f8 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -38,6 +38,7 @@ static_library("Orc") {
     "IRCompileLayer.cpp",
     "IRPartitionLayer.cpp",
     "IRTransformLayer.cpp",
+    "InProcessMemoryAccess.cpp",
     "IndirectionUtils.cpp",
     "JITLinkRedirectableSymbolManager.cpp",
     "JITLinkReentryTrampolines.cpp",
@@ -64,6 +65,7 @@ static_library("Orc") {
     "ReOptimizeLayer.cpp",
     "RedirectionManager.cpp",
     "SectCreate.cpp",
+    "SelfExecutorProcessControl.cpp",
     "SimpleRemoteEPC.cpp",
     "SpeculateAnalyses.cpp",
     "Speculation.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
index 7346de64f6c9..4c1c613529f7 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/HLSL/BUILD.gn
@@ -7,6 +7,8 @@ static_library("HLSL") {
   sources = [
     "CBuffer.cpp",
     "HLSLResource.cpp",
-    "HLSLRootSignatureUtils.cpp",
+    "HLSLRootSignature.cpp",
+    "RootSignatureMetadata.cpp",
+    "RootSignatureValidations.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
index affa26d20bc9..227a67f4da15 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenMP/BUILD.gn
@@ -13,6 +13,7 @@ static_library("OpenMP") {
   ]
   public_deps = [ "//llvm/include/llvm/Frontend/OpenMP:public_tablegen" ]
   sources = [
+    "DirectiveNameParser.cpp",
     "OMP.cpp",
     "OMPContext.cpp",
     "OMPIRBuilder.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
index c76962ae92c1..5c96bd8c8c5d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
@@ -60,6 +60,7 @@ static_library("MC") {
     "MCSectionCOFF.cpp",
     "MCSectionDXContainer.cpp",
     "MCSectionELF.cpp",
+    "MCSectionGOFF.cpp",
     "MCSectionMachO.cpp",
     "MCSectionWasm.cpp",
     "MCSectionXCOFF.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
index 90f997c58fa9..28063dc794a9 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ProfileData/BUILD.gn
@@ -2,6 +2,7 @@ static_library("ProfileData") {
   output_name = "LLVMProfileData"
   deps = [
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/Symbolize",
     "//llvm/lib/IR",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 628e544a687e..d4adeddd9b4e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -188,6 +188,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPURemoveIncompatibleFunctions.cpp",
     "AMDGPUReserveWWMRegs.cpp",
     "AMDGPUResourceUsageAnalysis.cpp",
+    "AMDGPURewriteAGPRCopyMFMA.cpp",
     "AMDGPURewriteOutArguments.cpp",
     "AMDGPURewriteUndefForPHI.cpp",
     "AMDGPUSelectionDAGInfo.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/MCTargetDesc/BUILD.gn
index 29aeba47ee3e..1ec5591cd659 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/LoongArch/MCTargetDesc/BUILD.gn
@@ -58,7 +58,6 @@ static_library("MCTargetDesc") {
     "LoongArchInstPrinter.cpp",
     "LoongArchMCAsmInfo.cpp",
     "LoongArchMCCodeEmitter.cpp",
-    "LoongArchMCExpr.cpp",
     "LoongArchMCTargetDesc.cpp",
     "LoongArchMatInt.cpp",
     "LoongArchTargetStreamer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
index 97b21360b02b..00033a7b6572 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
@@ -58,7 +58,6 @@ static_library("MCTargetDesc") {
     "PPCInstPrinter.cpp",
     "PPCMCAsmInfo.cpp",
     "PPCMCCodeEmitter.cpp",
-    "PPCMCExpr.cpp",
     "PPCMCTargetDesc.cpp",
     "PPCPredicates.cpp",
     "PPCXCOFFObjectWriter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
index 8b5c7a667fa1..acf97582ac5d 100644
--- a/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
@@ -14,6 +14,7 @@ driver_executable("dsymutil") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWARFLinker/Parallel",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Option",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
index ab044d165a79..877f9fdce1f7 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfdump/BUILD.gn
@@ -1,6 +1,7 @@
 executable("llvm-dwarfdump") {
   deps = [
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
index 3654acef9fdf..8c8088656ee1 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-dwarfutil/BUILD.gn
@@ -13,6 +13,7 @@ executable("llvm-dwarfutil") {
     "//llvm/lib/DWARFLinker/Classic",
     "//llvm/lib/DWARFLinker/Parallel",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/ObjCopy",
     "//llvm/lib/Object",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
index e70c7603c4a4..4e9d162b28c0 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
@@ -45,6 +45,7 @@ driver_executable("llvm-objdump") {
     "//llvm/include/llvm/Config:config",
     "//llvm/lib/CodeGen",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/DebugInfo/PDB",
     "//llvm/lib/DebugInfo/Symbolize",
     "//llvm/lib/Debuginfod",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
index d8b6697ee073..d0f91a5cdeb0 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
@@ -5,6 +5,7 @@ unittest("DebugInfoDWARFTests") {
     "//llvm/lib/CodeGen",
     "//llvm/lib/CodeGen/AsmPrinter",
     "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/DWARF/LowLevel",
     "//llvm/lib/MC",
     "//llvm/lib/Object",
     "//llvm/lib/ObjectYAML",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
index 7338fb159419..c29277c808d2 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
@@ -19,6 +19,8 @@ unittest("LLVMFrontendTests") {
     "OpenMPCompositionTest.cpp",
     "OpenMPContextTest.cpp",
     "OpenMPDecompositionTest.cpp",
+    "OpenMPDirectiveNameParserTest.cpp",
+    "OpenMPDirectiveNameTest.cpp",
     "OpenMPIRBuilderTest.cpp",
     "OpenMPParsingTest.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
index 5838118f2711..43916cef756f 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
@@ -10,6 +10,7 @@ source_set("Basic") {
     "DirectiveEmitter.cpp",
     "IntrinsicEmitter.cpp",
     "RISCVTargetDefEmitter.cpp",
+    "RuntimeLibcallsEmitter.cpp",
     "SDNodeProperties.cpp",
     "TableGen.cpp",
     "TargetFeaturesEmitter.cpp",
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index c134cda90e2e..5459d431d696 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -629,7 +629,8 @@ def use_clang(
                 ToolSubst(
                     "%clang_analyze_cc1",
                     command="%clang_cc1",
-                    extra_args=["-analyze", "%analyze", "-setup-static-analyzer"]
+                    # -setup-static-analyzer ensures that __clang_analyzer__ is defined
+                    extra_args=["-analyze", "-setup-static-analyzer"]
                     + additional_flags,
                 ),
                 ToolSubst(
diff --git a/llvm/utils/llvm-original-di-preservation.py b/llvm/utils/llvm-original-di-preservation.py
index dc1fa518ca8e..03793b1136f8 100755
--- a/llvm/utils/llvm-original-di-preservation.py
+++ b/llvm/utils/llvm-original-di-preservation.py
@@ -13,7 +13,8 @@
 
 
 class DILocBug:
-    def __init__(self, action, bb_name, fn_name, instr):
+    def __init__(self, origin, action, bb_name, fn_name, instr):
+        self.origin = origin
         self.action = action
         self.bb_name = bb_name
         self.fn_name = fn_name
@@ -79,6 +80,15 @@ def generate_html_report(
         table_title_di_loc
     )
 
+    # If any DILocation bug has an origin stack trace, we emit an extra column in the table, which we must therefore
+    # determine up-front.
+    has_origin_col = any(
+        x.origin is not None
+        for per_file_bugs in di_location_bugs.values()
+        for per_pass_bugs in per_file_bugs.values()
+        for x in per_pass_bugs
+    )
+
     header_di_loc = [
         "File",
         "LLVM Pass Name",
@@ -87,6 +97,8 @@ def generate_html_report(
         "Basic Block Name",
         "Action",
     ]
+    if has_origin_col:
+        header_di_loc.append("Origin")
 
     for column in header_di_loc:
         table_di_loc += "    <th>{0}</th>\n".format(column.strip())
@@ -112,6 +124,13 @@ def generate_html_report(
                 row.append(x.fn_name)
                 row.append(x.bb_name)
                 row.append(x.action)
+                if has_origin_col:
+                    if x.origin is not None:
+                        row.append(
+                            f"<details><summary>View Origin StackTrace</summary><pre>{x.origin}</pre></details>"
+                        )
+                    else:
+                        row.append("")
                 row.append("    </tr>\n")
             # Dump the bugs info into the table.
             for column in row:
@@ -428,9 +447,9 @@ def Main():
         sys.exit(1)
 
     # Use the defaultdict in order to make multidim dicts.
-    di_location_bugs = defaultdict(lambda: defaultdict(dict))
-    di_subprogram_bugs = defaultdict(lambda: defaultdict(dict))
-    di_variable_bugs = defaultdict(lambda: defaultdict(dict))
+    di_location_bugs = defaultdict(lambda: defaultdict(list))
+    di_subprogram_bugs = defaultdict(lambda: defaultdict(list))
+    di_variable_bugs = defaultdict(lambda: defaultdict(list))
 
     # Use the ordered dict to make a summary.
     di_location_bugs_summary = OrderedDict()
@@ -470,9 +489,9 @@ def Main():
                 skipped_lines += 1
                 continue
 
-            di_loc_bugs = []
-            di_sp_bugs = []
-            di_var_bugs = []
+            di_loc_bugs = di_location_bugs[bugs_file][bugs_pass]
+            di_sp_bugs = di_subprogram_bugs[bugs_file][bugs_pass]
+            di_var_bugs = di_variable_bugs[bugs_file][bugs_pass]
 
             # Omit duplicated bugs.
             di_loc_set = set()
@@ -487,6 +506,7 @@ def Main():
 
                 if bugs_metadata == "DILocation":
                     try:
+                        origin = bug.get("origin")
                         action = bug["action"]
                         bb_name = bug["bb-name"]
                         fn_name = bug["fn-name"]
@@ -494,7 +514,7 @@ def Main():
                     except:
                         skipped_bugs += 1
                         continue
-                    di_loc_bug = DILocBug(action, bb_name, fn_name, instr)
+                    di_loc_bug = DILocBug(origin, action, bb_name, fn_name, instr)
                     if not str(di_loc_bug) in di_loc_set:
                         di_loc_set.add(str(di_loc_bug))
                         if opts.compress:
diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py
index 8343dee937f7..e9591b00e2b5 100755
--- a/llvm/utils/release/github-upload-release.py
+++ b/llvm/utils/release/github-upload-release.py
@@ -42,18 +42,45 @@ def create_release(repo, release, tag=None, name=None, message=None):
         name = "LLVM {}".format(release)
 
     if not message:
+        # Note that these lines are not length limited because if we do so, GitHub
+        # assumes that should be how it is laid out on the page. We want GitHub to
+        # do the reflowing for us instead.
         message = dedent(
             """\
-            LLVM {} Release
+LLVM {release} Release
 
-            # A note on binaries
+## Package Types
 
-            Volunteers make binaries for the LLVM project, which will be uploaded
-            when they have had time to test and build these binaries. They might
-            not be available directly or not at all for each release. We suggest
-            you use the binaries from your distribution or build your own if you
-            rely on a specific platform or configuration."""
-        ).format(release)
+Each platform has one binary release package. The file name starts with either `LLVM-` or `clang+llvm-` and ends with the platform's name. For example, `LLVM-{release}-Linux-ARM64.tar.xz` contains LLVM binaries for Arm64 Linux.
+
+Except for Windows. Where `LLVM-*.exe` is an installer intended for using LLVM as a toolchain and `clang+llvm-` contains the contents of the installer, plus libraries and tools not normally used in a toolchain. You most likely want the `LLVM-` installer, unless you are developing software which itself uses LLVM, in which case choose `clang+llvm-`.
+
+If you do not find a release package for your platform, you may be able to find a community built package on the LLVM Discourse forum thread for this release. Remember that these are built by volunteers and may not always be available.
+
+If you rely on a platform or configuration that is not one of the defaults, we suggest you use the binaries that your platform provides, or build your own release packages.
+
+In addition, source archives are available:
+* `<sub-project>-{release}.src.tar.xz` are archives of the sources of specific sub-projects of `llvm-project` (except for `test-suite` which is an archive of the [LLVM Test Suite](https://github.com/llvm/llvm-test-suite)).
+* To get all the `llvm-project` source code for this release, choose `llvm-project-{release}.src.tar.xz`.
+
+## Verifying Packages
+
+All packages come with a matching `.sig` or `.jsonl` file. You should use these to verify the integrity of the packages.
+
+If it has a `.sig` file, it should have been signed by the release managers using GPG. Download the keys from the [LLVM website](https://releases.llvm.org/release-keys.asc), import them into your keyring and use them to verify the file:
+```
+$ gpg --import release-keys.asc
+$ gpg --verify <package file name>.sig <package file name>
+```
+
+If it has a `.jsonl` file, use [gh](https://cli.github.com/manual/gh_attestation_verify) to verify the package:
+```
+gh attestation verify --repo llvm/llvm-project <package file name>
+(if you are able to connect to GitHub)
+gh attestation verify --repo llvm/llvm-project <package file name> --bundle <package file name>.jsonl
+(using attestation file on disk)
+```"""
+        ).format(release=release)
 
     prerelease = True if "rc" in release else False
 
diff --git a/mlir/cmake/modules/IRDLToCpp.cmake b/mlir/cmake/modules/IRDLToCpp.cmake
index 8470ccdf5516..3fa1601c694d 100644
--- a/mlir/cmake/modules/IRDLToCpp.cmake
+++ b/mlir/cmake/modules/IRDLToCpp.cmake
@@ -5,7 +5,7 @@ function(add_irdl_to_cpp_target target irdl_file)
 
     # The command output depends on the executable to ensure IRDL sources are properly rebuilt
     # if the tool changes.
-    DEPENDS ${MLIR_IRDL_TO_CPP_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/${irdl_file}
+    DEPENDS ${MLIR_IRDL_TO_CPP_TARGET} ${MLIR_IRDL_TO_CPP_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/${irdl_file}
     COMMENT "Building ${irdl_file}..."
   )
   add_custom_target(${target} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${irdl_file}.cpp.inc)
diff --git a/mlir/docs/Bindings/Python.md b/mlir/docs/Bindings/Python.md
index b8bd0f507a51..bef9e7f54948 100644
--- a/mlir/docs/Bindings/Python.md
+++ b/mlir/docs/Bindings/Python.md
@@ -51,7 +51,10 @@ python -m pip install --upgrade pip
 # packages will be installed there.
 python -m pip install -r mlir/python/requirements.txt
 
-# Now run `cmake`, `ninja`, et al.
+# Now run your build command with `cmake`, `ninja`, et al.
+
+# Run mlir tests. For example, to run python bindings tests only using ninja:
+ninja check-mlir-python
 ```
 
 For interactive use, it is sufficient to add the
@@ -859,7 +862,7 @@ mutually exclusive with a more complete mapping of the backing constructs.
 
 ## Testing
 
-Tests should be added in the `test/Bindings/Python` directory and should
+Tests should be added in the `mlir/test/python` directory and should
 typically be `.py` files that have a lit run line.
 
 We use `lit` and `FileCheck` based tests:
diff --git a/mlir/docs/Tutorials/transform/Ch1.md b/mlir/docs/Tutorials/transform/Ch1.md
index b0fdf085854c..7463a9afbe97 100644
--- a/mlir/docs/Tutorials/transform/Ch1.md
+++ b/mlir/docs/Tutorials/transform/Ch1.md
@@ -19,13 +19,13 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
   %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
     ins(%biased, %c0f : tensor<512x512xf32>, f32)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
@@ -41,7 +41,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">):
+      %arg2: !transform.op<"linalg.elementwise">):
     transform.yield
   }
 }
@@ -72,11 +72,11 @@ To check or debug a transform sequence, it is possible to print various entities
 transform.sequence failures(propagate) {
 ^bb0(%arg0: !transform.any_op,
      %arg1: !transform.op<"linalg.matmul">,
-     %arg2: !transform.op<"linalg.elemwise_binary">):
+     %arg2: !transform.op<"linalg.elementwise">):
   transform.debug.emit_remark_at %arg1, "matmul"
       : !transform.op<"linalg.matmul">
   transform.debug.emit_remark_at %arg2, "elemwise_binaries"
-      : !transform.op<"linalg.elemwise_binary">
+      : !transform.op<"linalg.elementwise">
   transform.yield
 }
 ```
@@ -89,10 +89,10 @@ Since we don’t want to recompile the compiler every time we change a transform
 ```sh
 $ mlir-opt sequence.mlir --pass-pipeline="
     builtin.module(transform-interpreter{
-        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary})"
+        debug-bind-trailing-args=linalg.matmul,linalg.elementwise})"
 ```
 
-The `sequence.mlir` file contains _both_ the payload IR function _and_ the transform IR sequence nested in the same module. The transform interpreter pass will apply the `@__transform_main` named sequence to the anchor operation of the pass. In our case, we also asked the interpreter pass to associate the two extra arguments of the top-level sequence with all `linalg.matmul` and `linalg.elemwise_binary` payload operations through the respective pass options. Running this pass results in the expected remarks:
+The `sequence.mlir` file contains _both_ the payload IR function _and_ the transform IR sequence nested in the same module. The transform interpreter pass will apply the `@__transform_main` named sequence to the anchor operation of the pass. In our case, we also asked the interpreter pass to associate the two extra arguments of the top-level sequence with all `linalg.matmul` and `linalg.elementwise` payload operations through the respective pass options. Running this pass results in the expected remarks:
 
 ```sh
 sequence.mlir:7:13: remark: matmul
@@ -100,13 +100,13 @@ sequence.mlir:7:13: remark: matmul
             ^
 sequence.mlir:7:13: note: see current operation: %0 = linalg.matmul ins(%arg0, %arg1 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 sequence.mlir:10:13: remark: elemwise_binaries
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
             ^
-sequence.mlir:10:13: note: see current operation: %1 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
+sequence.mlir:10:13: note: see current operation: %1 = linalg.elementwise kind=#linalg.elementwise_kind<add>> ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 sequence.mlir:14:13: remark: elemwise_binaries
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
             ^
-sequence.mlir:14:13: note: see current operation: %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>} ins(%1, %cst : tensor<512x512xf32>, f32) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
+sequence.mlir:14:13: note: see current operation: %2 = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>> ins(%1, %cst : tensor<512x512xf32>, f32) outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
 ```
 
 Note that `%arg2` is associated with both elementwise payload operations. Any handle is associated with a list of entities. Individual transformations may or may not care about the order of elements in that list.
@@ -121,7 +121,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     %loop, %tiled = transform.structured.tile_using_forall %arg1
                     tile_sizes [4, 32]
@@ -163,10 +163,10 @@ func.func @fc_relu(%arg0: tensor<512x512xf32>,
           : tensor<4x32xf32> into tensor<512x512xf32>
     }
   }
-  %1 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+  %1 = linalg.elementwise kind=#linalg.elementwise_kind<add>>
     ins(%0, %arg2 : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
-  %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+  %2 = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>>
     ins(%1, %cst : tensor<512x512xf32>, f32)
     outs(%arg3 : tensor<512x512xf32>) -> tensor<512x512xf32>
   return %2 : tensor<512x512xf32>
@@ -185,7 +185,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     %loop, %tiled = transform.structured.tile_using_forall %arg1 tile_sizes [4, 32]
         : (!transform.op<"linalg.matmul">) -> (!transform.any_op, !transform.any_op)
@@ -219,7 +219,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // We can cast one type to another as long as operations are compatible
     // with both types. This creates "aliasing" handles.
     %casted = transform.cast %arg1 : !transform.op<"linalg.matmul">
@@ -248,7 +248,7 @@ sequence.mlir:28:3: error: op uses a handle invalidated by a previously executed
   transform.debug.emit_remark_at %matmul, "elemwise_binaries" : !transform.op<"linalg.matmul">
   ^
 sequence.mlir:21:29: note: handle to invalidated ops
-^bb0(%root: !transform.any_op, %matmul: !transform.op<"linalg.matmul">, %elemwise: !transform.op<"linalg.elemwise_binary">):
+^bb0(%root: !transform.any_op, %matmul: !transform.op<"linalg.matmul">, %elemwise: !transform.op<"linalg.elementwise">):
                             ^
 sequence.mlir:27:19: note: invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them
   %loop, %tiled = transform.structured.tile_using_forall %mm tile_sizes [4, 32]
@@ -263,12 +263,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
@@ -308,12 +308,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
           -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
@@ -384,7 +384,7 @@ test/Examples/transform/Ch1/invalidation-2.mlir:106:18: note: invalidated by thi
   %func, %call = transform.loop.outline %outline_target {func_name = "outlined"}
                  ^
 test/Examples/transform/Ch1/invalidation-2.mlir:24:13: note: ancestor payload op
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
             ^
 test/Examples/transform/Ch1/invalidation-2.mlir:24:13: note: nested payload op
   %matmul = linalg.matmul ins(%lhs, %rhs: tensor<512x512xf32>, tensor<512x512xf32>)
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index 6a6cefd8785a..0f45f5607bab 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -290,12 +290,12 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
     %add, %max = transform.split_handle %arg2
-        : (!transform.op<"linalg.elemwise_binary">)
+        : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It
diff --git a/mlir/docs/Tutorials/transform/Ch4.md b/mlir/docs/Tutorials/transform/Ch4.md
index ac60fcb91468..f9a0dff5c90e 100644
--- a/mlir/docs/Tutorials/transform/Ch4.md
+++ b/mlir/docs/Tutorials/transform/Ch4.md
@@ -42,13 +42,13 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
             outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
   %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
     ins(%biased, %c0f : tensor<512x512xf32>, f32)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
@@ -59,7 +59,7 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 
 In Chapter 1, we were calling the test transform interpreter pass with
 additional arguments, `bind-first-extra-to-ops=linalg.matmul
-bind-second-extra-to-ops=linalg.elemwise_binary`, to provide initial
+bind-second-extra-to-ops=linalg.elementwise`, to provide initial
 associations for operation handles. Instead, we can use match operations to
 discover relevant operations in the payload IR. Match operations can be combined
 with “regular” transform operations using, e.g., the
@@ -97,7 +97,7 @@ module @transforms attributes { transform.with_named_sequence } {
   // rewriter sequence on success.
   transform.named_sequence @match_elemwise(
       %entry: !transform.any_op {transform.readonly}) -> !transform.any_op {
-    transform.match.operation_name %entry ["linalg.elemwise_binary"]
+    transform.match.operation_name %entry ["linalg.elementwise"]
       : !transform.any_op
     transform.yield %entry : !transform.any_op
   }
@@ -127,7 +127,7 @@ module @transforms attributes { transform.with_named_sequence } {
 This script can be executed using the non-test interpreter pass running on the
 root operation of the translation unit without additional flags: `mlir-opt
 --transform-interpreter`. It will emit corresponding remarks at
-`linalg.elemwise_binary` and `linalg.matmul` operations. In debug builds, the
+`linalg.elementwise` and `linalg.matmul` operations. In debug builds, the
 infrastructure provides a convenient method to understand the matching process
 by passing `-debug-only=transform-matcher` to `mlir-opt` or a derived tool. It
 will print the silenceable failure messages produced by the match operations
@@ -169,7 +169,7 @@ transform.named_sequence @match_matmul_elemwise(
     %last: !transform.any_op {transform.readonly})
     -> (!transform.any_op, !transform.any_op, !transform.any_op) {
   // The last operation must be an elementwise binary.
-  transform.match.operation_name %last ["linalg.elemwise_binary"]
+  transform.match.operation_name %last ["linalg.elementwise"]
     : !transform.any_op
   // Its first operand must be defined by another operation, to which we
   // will get a handle here. We are guaranteed that the first operand exists
@@ -179,7 +179,7 @@ transform.named_sequence @match_matmul_elemwise(
   %middle = transform.get_producer_of_operand %last[0]
     : (!transform.any_op) -> !transform.any_op
   // The defining operation must itself be an elementwise binary.
-  transform.match.operation_name %middle ["linalg.elemwise_binary"]
+  transform.match.operation_name %middle ["linalg.elementwise"]
     : !transform.any_op
   // And the first operand of that operation must be defined by yet another
   // operation.
@@ -399,7 +399,7 @@ transform.named_sequence @match_matmul_elemwise(
     -> (!transform.any_op, !transform.any_op, !transform.any_op,
         !transform.param<i32>) {
   // The last operation must be an elementwise binary.
-  transform.match.operation_name %last ["linalg.elemwise_binary"]
+  transform.match.operation_name %last ["linalg.elementwise"]
     : !transform.any_op
 
   // One of its operands must be defined by another operation, to which we
@@ -413,7 +413,7 @@ transform.named_sequence @match_matmul_elemwise(
     %def = transform.get_defining_op %operand
       : (!transform.any_value) -> !transform.any_op
     // The defining operation must itself be an elementwise binary.
-    transform.match.operation_name %def ["linalg.elemwise_binary"]
+    transform.match.operation_name %def ["linalg.elementwise"]
       : !transform.any_op
     transform.yield %def : !transform.any_op
   }
diff --git a/mlir/docs/Tutorials/transform/ChH.md b/mlir/docs/Tutorials/transform/ChH.md
index aadfddb89393..ce24c5add631 100644
--- a/mlir/docs/Tutorials/transform/ChH.md
+++ b/mlir/docs/Tutorials/transform/ChH.md
@@ -290,7 +290,7 @@ scf.forall (%co) in (2) {
   scf.forall (%n, %y, %xo) in (5, 80, 20) {
     tensor.extract_slice
     // Implicit dimensions [ni=0:1, y=0:1, xi=0:5, ci=0:64]
-    %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> } // ...
+    %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed> // ...
     scf.forall.in_parallel {
       tensor.parallel_insert_slice // ...
     }
diff --git a/mlir/examples/toy/Ch2/include/toy/Ops.td b/mlir/examples/toy/Ch2/include/toy/Ops.td
index 1a1b136af71f..ef65c9c8d682 100644
--- a/mlir/examples/toy/Ch2/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch2/include/toy/Ops.td
@@ -297,7 +297,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
index bf4c099fe05d..96925bebf07b 100644
--- a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, {});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch3/include/toy/Ops.td b/mlir/examples/toy/Ch3/include/toy/Ops.td
index 021802b5c4fc..485980420a20 100644
--- a/mlir/examples/toy/Ch3/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch3/include/toy/Ops.td
@@ -298,7 +298,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
index bf4c099fe05d..c8cba82cb63a 100644
--- a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch4/include/toy/Ops.td b/mlir/examples/toy/Ch4/include/toy/Ops.td
index 4441e48ca53c..0b32b1b0c772 100644
--- a/mlir/examples/toy/Ch4/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch4/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
index b56e2f7c9008..9371815577b1 100644
--- a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch5/include/toy/Ops.td b/mlir/examples/toy/Ch5/include/toy/Ops.td
index 5b7c966de6f0..d11d18dc93cf 100644
--- a/mlir/examples/toy/Ch5/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch5/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
index b56e2f7c9008..9371815577b1 100644
--- a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch6/include/toy/Ops.td b/mlir/examples/toy/Ch6/include/toy/Ops.td
index fdbc239a171d..63950f467455 100644
--- a/mlir/examples/toy/Ch6/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch6/include/toy/Ops.td
@@ -335,7 +335,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
index de1eafc4d867..22f75e042fe0 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToLLVM.cpp
@@ -92,7 +92,7 @@ class PrintOpLowering : public ConversionPattern {
       auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
       auto loop =
           rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      for (Operation &nested : *loop.getBody())
+      for (Operation &nested : make_early_inc_range(*loop.getBody()))
         rewriter.eraseOp(&nested);
       loopIvs.push_back(loop.getInductionVar());
 
diff --git a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
index b56e2f7c9008..9371815577b1 100644
--- a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
@@ -120,7 +120,7 @@ class MLIRGenImpl {
     // Arguments type are uniformly unranked tensors.
     llvm::SmallVector<mlir::Type, 4> argTypes(proto.getArgs().size(),
                                               getType(VarType{}));
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
diff --git a/mlir/examples/toy/Ch7/include/toy/Ops.td b/mlir/examples/toy/Ch7/include/toy/Ops.td
index 71ab7b0aeebb..bdf8ad0bc21a 100644
--- a/mlir/examples/toy/Ch7/include/toy/Ops.td
+++ b/mlir/examples/toy/Ch7/include/toy/Ops.td
@@ -360,7 +360,7 @@ def ReturnOp : Toy_Op<"return", [Pure, HasParent<"FuncOp">,
 
   // Allow building a ReturnOp with no return operand.
   let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
+    OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>
   ];
 
   // Provide extra utility definitions on the c++ operation class definition.
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
index a20a2c888a17..54eeb275ae85 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToLLVM.cpp
@@ -92,7 +92,7 @@ class PrintOpLowering : public ConversionPattern {
       auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
       auto loop =
           rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step);
-      for (Operation &nested : *loop.getBody())
+      for (Operation &nested : make_early_inc_range(*loop.getBody()))
         rewriter.eraseOp(&nested);
       loopIvs.push_back(loop.getInductionVar());
 
diff --git a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
index e554e375209f..2490f17163b8 100644
--- a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
@@ -182,7 +182,7 @@ class MLIRGenImpl {
         return nullptr;
       argTypes.push_back(type);
     }
-    auto funcType = builder.getFunctionType(argTypes, std::nullopt);
+    auto funcType = builder.getFunctionType(argTypes, /*results=*/{});
     return builder.create<mlir::toy::FuncOp>(location, proto.getName(),
                                              funcType);
   }
@@ -441,10 +441,10 @@ class MLIRGenImpl {
     for (auto &var : lit.getValues()) {
       if (auto *number = llvm::dyn_cast<NumberExprAST>(var.get())) {
         attrElements.push_back(getConstantAttr(*number));
-        typeElements.push_back(getType(std::nullopt));
+        typeElements.push_back(getType(/*shape=*/{}));
       } else if (auto *lit = llvm::dyn_cast<LiteralExprAST>(var.get())) {
         attrElements.push_back(getConstantAttr(*lit));
-        typeElements.push_back(getType(std::nullopt));
+        typeElements.push_back(getType(/*shape=*/{}));
       } else {
         auto *structLit = llvm::cast<StructLiteralExprAST>(var.get());
         auto attrTypePair = getConstantAttr(*structLit);
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 503a2a7e6f0c..969154abe883 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -113,8 +113,12 @@ class ConvertToLLVMPattern : public ConversionPattern {
   Type getVoidType() const;
 
   /// Get the MLIR type wrapping the LLVM i8* type.
+  [[deprecated("Use getPtrType() instead!")]]
   Type getVoidPtrType() const;
 
+  /// Get the MLIR type wrapping the LLVM ptr type.
+  Type getPtrType(unsigned addressSpace = 0) const;
+
   /// Create a constant Op producing a value of `resultType` from an index-typed
   /// integer attribute.
   static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index d58558ac3288..eadb5d932679 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -898,6 +898,40 @@ def AMDGPU_GatherToLDSOp :
   let hasVerifier = 1;
 }
 
+def AMDGPU_TransposeLoadOp :
+    AMDGPU_Op<"transpose_load", [SameVariadicOperandSize]>,
+    Arguments<(ins Arg<AnyMemRef, "buffer to transpose load from", [MemRead]>:$src, Variadic<Index>:$srcIndices)>,
+    Results<(outs AnyTypeOf<[AnyVectorOfNonZeroRank]>:$result)> {
+  let summary = "MLIR wrapper for CDNA Transpose Load instructions";
+  let description = [{
+    The `amdgpu.transpose_load` op is a wrapper around the `ds_read_tr` instructions.
+    The transpose load op represents a subgroup load from LDS memory,
+    where the subgroup of threads collectively reads a matrix from the source
+    memref, with each thread reading a vector of the matrix, and gets a transposed matrix
+    in as the result. That is, each thread reads a vector of the col-major matrix at different
+    indices, and the thread's read result is a vector of the corresponding row of the transposed
+    matrix.
+
+    This op is a direct wrapper around the ROCDL `ds_read_tr` family intrinsics. Please refer
+    to the CDNA4 ISA documentation for more details about its exact semantics.
+
+    Format example:
+    ```
+    %0 = amdgpu.transpose_load %src[%srcIndices] : memref<128x256xf16> -> vector<4xf16>
+    ```
+    Operands:
+    * `$src`: LDS memref to read from.
+    * `$srcIndices`: indices into `$src` to read from for this thread.
+    * `$result`: target register this transpose load instruction will write to.
+
+    Note: Lowering is only supported on gfx950 and up.
+  }];
+  let assemblyFormat = [{
+    $src `[` $srcIndices `]` attr-dict `:` type($src) `->` type($result)
+  }];
+  let hasVerifier = 1;
+}
+
 def AMDGPU_ScaledMFMAOp :
     AMDGPU_Op<"scaled_mfma", [AllTypesMatch<["destC", "destD"]>,
                         Pure]>,
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
index a52ee2ee89ca..cc2f543e79f6 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
@@ -23,7 +23,7 @@ namespace amdgpu {
 
 #define GEN_PASS_DECL_AMDGPUEMULATEATOMICSPASS
 #define GEN_PASS_DECL_AMDGPURESOLVESTRIDEDMETADATAPASS
-#define GEN_PASS_DECL_AMDGPUTRANSFERREADTOLOADPASS
+#define GEN_PASS_DECL_AMDGPUMASKEDLOADTOLOADPASS
 #define GEN_PASS_REGISTRATION
 #include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
 
@@ -35,8 +35,8 @@ void populateAmdgpuEmulateAtomicsPatterns(ConversionTarget &target,
 void populateAmdgpuResolveStridedMetadataPatterns(RewritePatternSet &patterns,
                                                   PatternBenefit benefit = 1);
 
-void populateAmdgpuTransferReadToLoadPatterns(RewritePatternSet &patterns,
-                                              PatternBenefit benefit = 1);
+void populateAmdgpuMaskedloadToLoadPatterns(RewritePatternSet &patterns,
+                                            PatternBenefit benefit = 1);
 
 } // namespace amdgpu
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
index 0e858108acf3..8d0e6829ab0c 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.td
@@ -51,8 +51,8 @@ def AmdgpuResolveStridedMetadataPass : Pass<"amdgpu-resolve-strided-metadata"> {
   ];
 }
 
-def AmdgpuTransferReadToLoadPass : Pass<"amdgpu-transfer-read-to-load"> {
-  let summary = "Lower the operations from the vector transfer_read to vector load";
+def AmdgpuMaskedloadToLoadPass : Pass<"amdgpu-maskedload-to-load"> {
+  let summary = "Lower the operations from the vector maskedload to vector load";
   let description = [{
     This pass creates a transfer read op lowering optimization. The lowering
     will produce a conditional check at runtime. If within bounds, a vector
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 19fbcf64b236..e52b7d2090d5 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -243,17 +243,16 @@ def AffineForOp : Affine_Op<"for",
   let regions = (region SizedRegion<1>:$region);
 
   let skipDefaultBuilders = 1;
-  let builders = [
-    OpBuilder<(ins "int64_t":$lowerBound, "int64_t":$upperBound,
-      CArg<"int64_t", "1">:$step, CArg<"ValueRange", "std::nullopt">:$iterArgs,
-      CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
-           "nullptr">:$bodyBuilder)>,
-    OpBuilder<(ins "ValueRange":$lbOperands, "AffineMap":$lbMap,
-      "ValueRange":$ubOperands, "AffineMap":$ubMap, CArg<"int64_t", "1">:$step,
-      CArg<"ValueRange", "std::nullopt">:$iterArgs,
-      CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
-           "nullptr">:$bodyBuilder)>
-  ];
+  let builders =
+      [OpBuilder<(ins "int64_t":$lowerBound, "int64_t":$upperBound,
+           CArg<"int64_t", "1">:$step, CArg<"ValueRange", "{}">:$iterArgs,
+           CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
+                "nullptr">:$bodyBuilder)>,
+       OpBuilder<(ins "ValueRange":$lbOperands, "AffineMap":$lbMap,
+           "ValueRange":$ubOperands, "AffineMap":$ubMap,
+           CArg<"int64_t", "1">:$step, CArg<"ValueRange", "{}">:$iterArgs,
+           CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
+                "nullptr">:$bodyBuilder)>];
 
   let extraClassDeclaration = [{
     /// Defining the function type we use for building the body of affine.for.
@@ -920,9 +919,7 @@ def AffineYieldOp : Affine_Op<"yield", [Pure, Terminator, ReturnLike,
 
   let arguments = (ins Variadic<AnyType>:$operands);
 
-  let builders = [
-    OpBuilder<(ins), [{ build($_builder, $_state, std::nullopt); }]>
-  ];
+  let builders = [OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
   let hasVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.td b/mlir/include/mlir/Dialect/Affine/Passes.td
index 67f9138589c4..6ad45b828f65 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.td
+++ b/mlir/include/mlir/Dialect/Affine/Passes.td
@@ -414,6 +414,23 @@ def SimplifyAffineStructures : Pass<"affine-simplify-structures", "func::FuncOp"
   let constructor = "mlir::affine::createSimplifyAffineStructuresPass()";
 }
 
+def SimplifyAffineMinMaxPass : InterfacePass<"affine-simplify-min-max", "FunctionOpInterface"> {
+  let summary = "Simplify affine min/max/apply";
+  let description = [{
+    Apply the SimplifyAffineMaxOp, SimplifyAffineMinOp and SimplifyAffineApplyOp
+    patterns in addition to AffineMin/Max canonicalization patterns until a
+    fixed point is reached.
+    These patterns apply ValueBoundsOp interface on AffineMin/Max ops and
+    additional simplifications such as:
+    ```
+       min(x, y, cst) / cst -> 1
+    ```
+    when x, y, cst are all >= 0.
+    This is typically useful to extract more static informationfrom IR after
+    tiling but can also come at a cost due to Presburger-style analysis.
+  }];
+}
+
 def AffineExpandIndexOps : Pass<"affine-expand-index-ops"> {
   let summary = "Lower affine operations operating on indices into more fundamental operations";
   let constructor = "mlir::affine::createAffineExpandIndexOpsPass()";
diff --git a/mlir/include/mlir/Dialect/Affine/Utils.h b/mlir/include/mlir/Dialect/Affine/Utils.h
index ae5a68a6be15..ac11f5a7c24c 100644
--- a/mlir/include/mlir/Dialect/Affine/Utils.h
+++ b/mlir/include/mlir/Dialect/Affine/Utils.h
@@ -198,10 +198,9 @@ AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
 /// of its input list. `indexRemap`'s dimensional inputs are expected to
 /// correspond to memref's indices, and its symbolic inputs if any should be
 /// provided in `symbolOperands`.
-///
-/// `domOpFilter`, if non-null, restricts the replacement to only those
-/// operations that are dominated by the former; similarly, `postDomOpFilter`
-/// restricts replacement to only those operations that are postdominated by it.
+//
+/// If `userFilterFn` is specified, restrict replacement to only those users
+/// that pass the specified filter (i.e., the filter returns true).
 ///
 /// 'allowNonDereferencingOps', if set, allows replacement of non-dereferencing
 /// uses of a memref without any requirement for access index rewrites as long
@@ -224,13 +223,14 @@ AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min,
 //  d1, d2) -> (d0 - d1, d2), and %ii will be the extra operand. Without any
 //  extra operands, note that 'indexRemap' would just be applied to existing
 //  indices (%i, %j).
+//
 //  TODO: allow extraIndices to be added at any position.
 LogicalResult replaceAllMemRefUsesWith(
     Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices = {},
     AffineMap indexRemap = AffineMap(), ArrayRef<Value> extraOperands = {},
-    ArrayRef<Value> symbolOperands = {}, Operation *domOpFilter = nullptr,
-    Operation *postDomOpFilter = nullptr, bool allowNonDereferencingOps = false,
-    bool replaceInDeallocOp = false);
+    ArrayRef<Value> symbolOperands = {},
+    llvm::function_ref<bool(Operation *)> userFilterFn = nullptr,
+    bool allowNonDereferencingOps = false, bool replaceInDeallocOp = false);
 
 /// Performs the same replacement as the other version above but only for the
 /// dereferencing uses of `oldMemRef` in `op`, except in cases where
diff --git a/mlir/include/mlir/Dialect/Arith/IR/Arith.h b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
index 0bee876ac9bf..7c50c2036ffd 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/Arith.h
+++ b/mlir/include/mlir/Dialect/Arith/IR/Arith.h
@@ -65,6 +65,10 @@ class ConstantIntOp : public arith::ConstantOp {
   static void build(OpBuilder &builder, OperationState &result, Type type,
                     int64_t value);
 
+  /// Build a constant int op that produces an integer from an APInt
+  static void build(OpBuilder &builder, OperationState &result, Type type,
+                    const APInt &value);
+
   inline int64_t value() {
     return cast<IntegerAttr>(arith::ConstantOp::getValue()).getInt();
   }
@@ -154,6 +158,11 @@ Value getReductionOp(AtomicRMWKind op, OpBuilder &builder, Location loc,
                      Value lhs, Value rhs);
 
 arith::CmpIPredicate invertPredicate(arith::CmpIPredicate pred);
+
+/// Creates an `arith.constant` operation with a zero value of type `type`. This
+/// method asserts if `type` is invalid for representing zero with
+/// `arith.constant`.
+Value getZeroConstant(OpBuilder &builder, Location loc, Type type);
 } // namespace arith
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
index 475b11f12c5f..ce86ff2cfd92 100644
--- a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
+++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
@@ -222,6 +222,34 @@ def UsmmlaOp : ArmNeon_OverloadedOperandsWithOneResultIntrOp<"usmmla",[1], [
     "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
 }
 
+def BfmmlaOp : ArmNeon_IntrOp<"bfmmla", [], [], 1, [
+                 Pure,
+                 AllTypesMatch<["src1", "src2"]>,
+                 AllTypesMatch<["acc", "res"]>,
+               ]> {
+  let summary = "BFloat16 matrix multiply-accumulate to single-precision";
+  let description = [{
+    BFMMLA: BFloat16 matrix multiply-accumulate to single-precision.
+
+    The operation multiplies the 2x4 BFloat16 matrix in the first source vector
+    with the 4x2 BFloat16 matrix in the second source vector, then accumulates
+    this intermediate result with the 2x2 Float32 matrix in the accumulator
+    vector, yielding the final 2x2 Float32 result.
+
+    Source:
+    https://developer.arm.com/architectures/instruction-sets/intrinsics/vbfmmlaq_f32
+  }];
+  // Supports (vector<8xbf16>, vector<8xbf16>) -> (vector<2xf32>)
+  let arguments = (ins
+    NeonVectorOfLength<4, F32>:$acc,
+    NeonVectorOfLength<8, BF16>:$src1,
+    NeonVectorOfLength<8, BF16>:$src2
+  );
+  let results = (outs NeonVectorOfLength<4, F32>:$res);
+  let assemblyFormat =
+    "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
+}
+
 class ArmNeon_2dOp<string mnemonic, list<Trait> traits = []>
     : Op</*dialect=*/ArmNeon_Dialect,
          /*opName=*/"2d." # mnemonic,
diff --git a/mlir/include/mlir/Dialect/ArmNeon/Transforms.h b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
index 52ebea2d0ffd..2f0f634a9677 100644
--- a/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
+++ b/mlir/include/mlir/Dialect/ArmNeon/Transforms.h
@@ -13,7 +13,7 @@ namespace mlir {
 class RewritePatternSet;
 
 namespace arm_neon {
-void populateLowerContractionToSMMLAPatternPatterns(
+void populateLowerContractionToNeonI8MMPatternPatterns(
     RewritePatternSet &patterns);
 } // namespace arm_neon
 
diff --git a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
index 7385bb73b449..8988df680b8f 100644
--- a/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
+++ b/mlir/include/mlir/Dialect/ArmSVE/IR/ArmSVE.td
@@ -293,6 +293,35 @@ def UsmmlaOp : ArmSVE_Op<"usmmla", [Pure,
     "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($dst)";
 }
 
+def BfmmlaOp : ArmSVE_IntrOp<"bfmmla", [Pure,
+                                        AllTypesMatch<["src1", "src2"]>,
+                                        AllTypesMatch<["acc", "res"]>,
+                                        ]> {
+  let summary = "BFloat16 matrix multiply-accumulate";
+  let description = [{
+    BFMMLA: BFloat16 matrix multiply-accumulate into 2×2 matrices";
+
+    This operation multiplies the 2x4 BFloat16 matrix held in each 128-bit
+    segment of the first source vector by the 4x2 BFloat16 matrix in the
+    corresponding segment of the second source vector, then accumulates
+    this intermediate result with the 2x2 Float32 matrix in the corresponding
+    segment of the accumulator vector, yielding the final 2x2 Float32
+    segment of the result.
+
+    Source:
+    https://developer.arm.com/documentation/100987/0000
+  }];
+  // Supports (vector<[8]xbf16>, vector<[8]xbf16>) -> (vector<[4]xf32>)
+  let arguments = (ins
+          ScalableVectorOfLengthAndType<[4], [F32]>:$acc,
+          ScalableVectorOfLengthAndType<[8], [BF16]>:$src1,
+          ScalableVectorOfLengthAndType<[8], [BF16]>:$src2
+  );
+  let results = (outs ScalableVectorOfLengthAndType<[4], [F32]>:$res);
+  let assemblyFormat =
+    "$acc `,` $src1 `,` $src2 attr-dict `:` type($src1) `to` type($res)";
+}
+
 class SvboolTypeConstraint<string lhsArg, string rhsArg> : TypesMatchWith<
       "expected corresponding svbool type widened to [16]xi1",
       lhsArg, rhsArg,
diff --git a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
index 6dbcdefbc933..a8455c237c86 100644
--- a/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
+++ b/mlir/include/mlir/Dialect/Async/IR/AsyncOps.td
@@ -288,7 +288,7 @@ def Async_ReturnOp : Async_Op<"return",
 
   let arguments = (ins Variadic<AnyType>:$operands);
 
-  let builders = [OpBuilder<(ins), [{build($_builder, $_state, std::nullopt);}]>];
+  let builders = [OpBuilder<(ins), [{build($_builder, $_state, {});}]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
   let hasVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
index c1529a36465a..6245f88db3d1 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
@@ -712,7 +712,7 @@ AliasingOpOperandList defaultGetAliasingOpOperands(Value value,
 /// This is the default implementation of
 /// BufferizableOpInterface::getBufferType. Should not be called from other
 /// places.
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 defaultGetBufferType(Value value, const BufferizationOptions &options,
                      const BufferizationState &state,
                      SmallVector<Value> &invocationStack);
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
index cafe05fe5f18..246ae77f327c 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td
@@ -525,7 +525,7 @@ def BufferizableOpInterface : OpInterface<"BufferizableOpInterface"> {
           Note: This interface method should never be called directly from user
           code. Always use `bufferization::getBufferType`.
         }],
-        /*retType=*/"::mlir::FailureOr<::mlir::BaseMemRefType>",
+        /*retType=*/"::mlir::FailureOr<::mlir::bufferization::BufferLikeType>",
         /*methodName=*/"getBufferType",
         /*args=*/(ins "::mlir::Value":$value,
                       "const ::mlir::bufferization::BufferizationOptions &":$options,
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 32c53ea9c494..f175b15c8770 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -111,7 +111,7 @@ def Bufferization_AllocTensorOp : Bufferization_Op<"alloc_tensor",
     AliasingValueList getAliasingValues(
         OpOperand &opOperand, const AnalysisState &state);
 
-    FailureOr<BaseMemRefType> getBufferType(
+    FailureOr<BufferLikeType> getBufferType(
         Value value, const BufferizationOptions &options,
         const BufferizationState &state,
         SmallVector<Value> &invocationStack);
@@ -478,10 +478,10 @@ def Bufferization_ToTensorOp : Bufferization_Op<"to_tensor", [
 
     bool isWritable(Value value, const AnalysisState &state);
 
-    FailureOr<BaseMemRefType> getBufferType(
+    FailureOr<BufferLikeType> getBufferType(
         Value value, const BufferizationOptions &options,
         const BufferizationState &state, SmallVector<Value> &invocationStack) {
-      return ::llvm::cast<BaseMemRefType>(getBuffer().getType());
+      return getBuffer().getType();
     }
   }];
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
index cbb6054fcf88..a2bfcb7ed2b7 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
@@ -18,7 +18,6 @@
 
 namespace mlir::bufferization {
 struct BufferizationOptions;
-class BufferizationState;
 class BufferLikeType;
 } // namespace mlir::bufferization
 
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
index f56c10555f02..e8a81c74bd77 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
@@ -32,7 +32,7 @@ template <typename ConcreteModel, typename ConcreteOp>
 struct OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
     : public BufferizableOpInterface::ExternalModel<ConcreteModel, ConcreteOp> {
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -110,7 +110,7 @@ struct OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
     if (!bufferType)
       return op->emitOpError("could not infer buffer type of block argument");
 
-    return bufferType;
+    return cast<BufferLikeType>(bufferType);
   }
 
 protected:
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index 9ecdb74f4d82..7fe2da8f7e04 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1593,4 +1593,101 @@ def EmitC_SwitchOp : EmitC_Op<"switch", [RecursiveMemoryEffects,
   let hasVerifier = 1;
 }
 
+def EmitC_ClassOp
+    : EmitC_Op<"class", [AutomaticAllocationScope, IsolatedFromAbove,
+                         OpAsmOpInterface, SymbolTable,
+                         Symbol]#GraphRegionNoTerminator.traits> {
+  let summary =
+      "Represents a C++ class definition, encapsulating fields and methods.";
+
+  let description = [{
+    The `emitc.class` operation defines a C++ class, acting as a container
+    for its data fields (`emitc.field`) and methods (`emitc.func`).
+    It creates a distinct scope, isolating its contents from the surrounding
+    MLIR region, similar to how C++ classes encapsulate their internals.
+
+    Example:
+
+    ```mlir
+    emitc.class @modelClass {
+      emitc.field @fieldName0 : !emitc.array<1xf32> = {emitc.opaque = "input_tensor"}
+      emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+      }
+    }
+    // Class with a final speciferAdd commentMore actions
+    emitc.class final @modelClass {
+      emitc.field @fieldName0 : !emitc.array<1xf32> = {emitc.opaque = "input_tensor"}
+      emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+      }
+    }
+    ```
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name, UnitAttr:$final_specifier);
+
+  let regions = (region AnyRegion:$body);
+
+  let extraClassDeclaration = [{
+    // Returns the body block containing class members and methods.
+    Block &getBlock();
+  }];
+
+  let hasCustomAssemblyFormat = 1;
+
+  let assemblyFormat =
+      [{ (`final` $final_specifier^)? $sym_name attr-dict-with-keyword $body }];
+}
+
+def EmitC_FieldOp : EmitC_Op<"field", [Symbol]> {
+  let summary = "A field within a class";
+  let description = [{
+    The `emitc.field` operation declares a named field within an `emitc.class`
+    operation. The field's type must be an EmitC type. 
+
+    Example:
+
+    ```mlir
+    // Example with an attribute:
+    emitc.field @fieldName0 : !emitc.array<1xf32>  {emitc.opaque = "another_feature"}
+    // Example with no attribute:
+    emitc.field @fieldName0 : !emitc.array<1xf32>
+    ```
+  }];
+
+  let arguments = (ins SymbolNameAttr:$sym_name, TypeAttr:$type,
+      OptionalAttr<AnyAttr>:$attrs);
+
+  let assemblyFormat = [{ $sym_name `:` $type ($attrs^)? attr-dict}];
+
+  let hasVerifier = 1;
+}
+
+def EmitC_GetFieldOp
+    : EmitC_Op<"get_field", [Pure, DeclareOpInterfaceMethods<
+                                       SymbolUserOpInterface>]> {
+  let summary = "Obtain access to a field within a class instance";
+  let description = [{
+     The `emitc.get_field` operation retrieves the lvalue of a
+     named field from a given class instance.
+
+     Example:
+
+     ```mlir
+     %0 = get_field @fieldName0 : !emitc.array<1xf32>
+     ```
+  }];
+
+  let arguments = (ins FlatSymbolRefAttr:$field_name);
+  let results = (outs AnyTypeOf<[EmitC_ArrayType, EmitC_LValueType]>:$result);
+  let assemblyFormat = "$field_name `:` type($result) attr-dict";
+}
+
 #endif // MLIR_DIALECT_EMITC_IR_EMITC
diff --git a/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.h b/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.h
index 5a103f181c76..1af4aa06fa81 100644
--- a/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.h
@@ -15,6 +15,7 @@ namespace mlir {
 namespace emitc {
 
 #define GEN_PASS_DECL_FORMEXPRESSIONSPASS
+#define GEN_PASS_DECL_WRAPFUNCINCLASSPASS
 #include "mlir/Dialect/EmitC/Transforms/Passes.h.inc"
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.td b/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.td
index f46b705ca2df..74c49132b61f 100644
--- a/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/EmitC/Transforms/Passes.td
@@ -20,4 +20,42 @@ def FormExpressionsPass : Pass<"form-expressions"> {
   let dependentDialects = ["emitc::EmitCDialect"];
 }
 
+def WrapFuncInClassPass : Pass<"wrap-emitc-func-in-class"> {
+  let summary = "Wrap functions in classes, using arguments as fields.";
+  let description = [{
+    This pass transforms `emitc.func` operations into `emitc.class` operations.
+    Function arguments become fields of the class, and the function body is moved
+    to a new `execute` method within the class.
+    If the corresponding function argument has attributes (accessed via `argAttrs`), 
+    these attributes are attached to the field operation. 
+    Otherwise, the field is created without additional attributes.
+
+    Example:
+    
+    ```mlir
+    emitc.func @model(%input_data : !emitc.array<1xf32> {emitc.opaque = "input_tensor"}) attributes { } {
+      %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+      %1 = subscript %input_data[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+      return
+    }
+    // becomes 
+    emitc.class @modelClass {
+      emitc.field @input_tensor : !emitc.array<1xf32> {emitc.opaque = "input_tensor"}
+      emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @input_tensor : !emitc.array<1xf32>
+        %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+      }
+    }
+    ```
+  }];
+  let dependentDialects = ["emitc::EmitCDialect"];
+  let options = [Option<
+      "namedAttribute", "named-attribute", "std::string",
+      /*default=*/"",
+      "Attribute key used to extract field names from function argument's "
+      "dictionary attributes">];
+}
+
 #endif // MLIR_DIALECT_EMITC_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/EmitC/Transforms/Transforms.h b/mlir/include/mlir/Dialect/EmitC/Transforms/Transforms.h
index 2574acd7d48e..a4e8fe10ff85 100644
--- a/mlir/include/mlir/Dialect/EmitC/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/EmitC/Transforms/Transforms.h
@@ -28,6 +28,10 @@ ExpressionOp createExpression(Operation *op, OpBuilder &builder);
 /// Populates `patterns` with expression-related patterns.
 void populateExpressionPatterns(RewritePatternSet &patterns);
 
+/// Populates 'patterns' with func-related patterns.
+void populateFuncPatterns(RewritePatternSet &patterns,
+                          StringRef namedAttribute);
+
 } // namespace emitc
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
index cdaeb6461afb..06ce4f16c867 100644
--- a/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
+++ b/mlir/include/mlir/Dialect/Func/IR/FuncOps.td
@@ -385,7 +385,7 @@ def ReturnOp : Func_Op<"return", [Pure, HasParent<"FuncOp">,
   let arguments = (ins Variadic<AnyType>:$operands);
 
   let builders = [OpBuilder<(ins), [{
-    build($_builder, $_state, std::nullopt);
+    build($_builder, $_state, {});
   }]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td b/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
index 6e0f6f1d78ed..63f228ca3157 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUDeviceMappingAttr.td
@@ -228,6 +228,30 @@ def GPUThreadMappingAttr
   }];
 }
 
+def GPULaneMappingAttr
+    : GPU_Attr<"GPULaneMapping", "lane", [
+      DeclareAttrInterfaceMethods<DeviceMappingAttrInterface> ]> {
+  let parameters = (ins
+    EnumParameter<MappingIdEnum>:$lane
+  );
+  let assemblyFormat = "`<` params `>`";
+  let description = [{
+    An attribute that allows defining lane parallelism for GPU devices.
+
+    It can be consumed by lowering to generate GPU.
+
+    #### 3D mapping mode
+
+    Unsupported
+
+    #### Linear mapping mode
+
+    The linear lane id is obtained by linearizing the index of the lane.
+    If required, predication occurs on the linear id. This allows specifying
+    predication on a 1D subset of the (linearized) lanes.
+  }];
+}
+
 def GPUMemorySpaceMappingAttr : GPU_Attr<"GPUMemorySpaceMapping", "memory_space", [
   DeclareAttrInterfaceMethods<DeviceMappingAttrInterface> ] >  {
   let parameters = (ins
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index a81b2e83ddef..1dbaf5db7b61 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1304,8 +1304,8 @@ def GPU_ShuffleOp : GPU_Op<
     Results<(outs AnyIntegerOrFloatOr1DVector:$shuffleResult, I1:$valid)> {
   let summary = "Shuffles values within a subgroup.";
   let description = [{
-    The "shuffle" op moves values to a across lanes (a.k.a., invocations,
-    work items) within the same subgroup. The `width` argument specifies the
+    The "shuffle" op moves values across lanes in a subgroup (a.k.a., local
+    invocation) within the same subgroup. The `width` argument specifies the
     number of lanes that participate in the shuffle, and must be uniform
     across all lanes. Further, the first `width` lanes of the subgroup must
     be active.
@@ -1366,6 +1366,54 @@ def GPU_ShuffleOp : GPU_Op<
   ];
 }
 
+def GPU_RotateOp : GPU_Op<
+    "rotate", [Pure, AllTypesMatch<["value", "rotateResult"]>]>,
+    Arguments<(ins AnyIntegerOrFloatOr1DVector:$value, I32:$offset, I32:$width)>,
+    Results<(outs AnyIntegerOrFloatOr1DVector:$rotateResult, I1:$valid)> {
+  let summary = "Rotate values within a subgroup.";
+  let description = [{
+    The "rotate" op moves values across lanes in a subgroup (a.k.a., local
+    invocations) within the same subgroup. The `width` argument specifies the
+    number of lanes that participate in the rotation, and must be uniform across
+    all participating lanes. Further, the first `width` lanes of the subgroup
+    must be active.
+
+    `width` must be a power of two, and `offset` must be in the range
+    `[0, width)`.
+
+    Return the `rotateResult` of the invocation whose id within the group is
+    calculated as follows:
+
+    ```mlir
+    Invocation ID = ((LaneId + offset) & (width - 1)) + (LaneId & ~(width - 1))
+    ```
+
+    Returns the `rotateResult` and `true` if the current lane id is smaller than
+    `width`, and poison value and `false` otherwise.
+
+    example:
+
+    ```mlir
+    %offset = arith.constant 1 : i32
+    %width = arith.constant 16 : i32
+    %1, %2 = gpu.rotate %0, %offset, %width : f32
+    ```
+
+    For lane `k`, returns the value from lane `(k + cst1) % width`.
+  }];
+
+  let assemblyFormat = [{
+    $value `,` $offset `,` $width attr-dict `:` type($value)
+  }];
+
+  let builders = [
+    // Helper function that creates a rotate with constant offset/width.
+    OpBuilder<(ins "Value":$value, "int32_t":$offset, "int32_t":$width)>
+  ];
+
+  let hasVerifier = 1;
+}
+
 def GPU_BarrierOp : GPU_Op<"barrier"> {
   let summary = "Synchronizes all work items of a workgroup.";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
index 52fc6f4d5c71..de512ded59fe 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/Utils.h
@@ -28,27 +28,24 @@ namespace transform {
 namespace gpu {
 
 /// Helper type for functions that generate ids for the mapping of a scf.forall.
-/// Operates on both 1) an "original" basis that represents the individual
-/// thread and block ids and 2) a "scaled" basis that represents grouped ids
-/// (e.g. block clusters, warpgroups and warps).
-/// The mapping of ids is done in the "scaled" basis (i.e. when mapping to warps
-/// a division by 32 occurs).
-/// The predication is in the "original" basis using the "active" quantities
-/// (`activeMappingSizes`, `availableMappingSizes` and `activeIdOps`).
 struct IdBuilderResult {
-  // Ops used to replace the forall induction variables.
+  /// Error message, if not empty then building the ids failed.
+  std::string errorMsg;
+  /// Values used to replace the forall induction variables.
   SmallVector<Value> mappingIdOps;
-  // Available mapping sizes used to predicate the forall body when they are
-  // larger than the predicate mapping sizes.
-  SmallVector<int64_t> availableMappingSizes;
-  // Actual mapping sizes used to predicate the forall body when they are
-  // smaller than the available mapping sizes.
-  SmallVector<int64_t> activeMappingSizes;
-  // Ops used to predicate the forall body when activeMappingSizes is smaller
-  // than the available mapping sizes.
-  SmallVector<Value> activeIdOps;
+  /// Values used to predicate the forall body when activeMappingSizes is
+  /// smaller than the available mapping sizes.
+  SmallVector<Value> predicateOps;
 };
 
+inline raw_ostream &operator<<(raw_ostream &os, const IdBuilderResult &res) {
+  llvm::interleaveComma(res.mappingIdOps, os << "----mappingIdOps: ");
+  os << "\n";
+  llvm::interleaveComma(res.predicateOps, os << "----predicateOps: ");
+  os << "\n";
+  return os;
+}
+
 /// Common gpu id builder type, allows the configuration of lowering for various
 /// mapping schemes. Takes:
 ///   - A rewriter with insertion point set before the forall op to rewrite.
@@ -117,6 +114,15 @@ struct GpuThreadIdBuilder : public GpuIdBuilder {
   GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping = false);
 };
 
+/// Builder for lane id.
+/// The `idBuilder` method returns nD values used for indexing rewrites as well
+/// as 1D sizes for predicate generation.
+/// This `useLinearMapping` case is the only supported case.
+struct GpuLaneIdBuilder : public GpuIdBuilder {
+  GpuLaneIdBuilder(MLIRContext *ctx, int64_t warpSize, bool unused);
+  int64_t warpSize = 32;
+};
+
 /// Determine if the size of the kernel configuration is supported by the
 /// GPU architecture being used.
 /// TODO this is currently hardwired to CUDA, parameterize and generalize.
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index 9c5bbae1022f..cfad07e57021 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -87,3 +87,13 @@ mlir_tablegen(VCIXConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(VCIXOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=vcix)
 mlir_tablegen(VCIXOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=vcix)
 add_public_tablegen_target(MLIRVCIXConversionsIncGen)
+
+add_mlir_dialect(XeVMOps xevm)
+add_mlir_doc(XeVMOps XeVMDialect Dialects/ -gen-dialect-doc -dialect=xevm)
+set(LLVM_TARGET_DEFINITIONS XeVMOps.td)
+mlir_tablegen(XeVMConversions.inc -gen-llvmir-conversions)
+mlir_tablegen(XeVMOpsEnums.h.inc -gen-enum-decls)
+mlir_tablegen(XeVMOpsEnums.cpp.inc -gen-enum-defs)
+mlir_tablegen(XeVMOpsAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=xevm)
+mlir_tablegen(XeVMOpsAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=xevm)
+add_public_tablegen_target(MLIRXeVMConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index b97b5ac932c9..e08c7b796933 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -235,8 +235,7 @@ class LLVM_MemOpPatterns {
   }];
   code setInvariantGroupCode = [{
     if ($invariantGroup) {
-      llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(),
-                                                 std::nullopt);
+      llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), {});
       inst->setMetadata(llvm::LLVMContext::MD_invariant_group, metadata);
     }
   }];
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 939e7a09a73a..f4c164009832 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -425,7 +425,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load",
     auto *inst = builder.CreateLoad($_resultType, $addr, $volatile_);
     $res = inst;
     if ($invariant) {
-      llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), std::nullopt);
+      llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), {});
       inst->setMetadata(llvm::LLVMContext::MD_invariant_load, metadata);
     }
     if ($dereferenceable)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 418931b93126..6895e946b8a4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -965,19 +965,21 @@ def NVVM_ElectSyncOp : NVVM_Op<"elect.sync">
   let summary = "Elect one leader thread";
   let description = [{
     The `elect.sync` instruction elects one predicated active leader
-    thread from among a set of threads specified in membermask.
-    The membermask is set to `0xFFFFFFFF` for the current version
-    of this Op. The predicate result is set to `True` for the
-    leader thread, and `False` for all other threads.
+    thread from among a set of threads specified in the `membermask`.
+    When the `membermask` is not provided explicitly, a default value
+    of `0xFFFFFFFF` is used. The predicate result is set to `True` for
+    the leader thread, and `False` for all other threads.
 
     [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync)
   }];
 
+  let arguments = (ins Optional<I32>:$membermask);
   let results = (outs I1:$pred);
-  let assemblyFormat = "attr-dict `->` type(results)";  
+  let assemblyFormat = "($membermask^)? attr-dict `->` type(results)";
   string llvmBuilder = [{
     auto *resultTuple = createIntrinsicCall(builder,
-        llvm::Intrinsic::nvvm_elect_sync, {builder.getInt32(0xFFFFFFFF)});
+        llvm::Intrinsic::nvvm_elect_sync,
+        {$membermask ? $membermask : builder.getInt32(0xFFFFFFFF)});
     // Extract the second value into $pred
     $pred = builder.CreateExtractValue(resultTuple, 1);
   }];
diff --git a/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h b/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h
new file mode 100644
index 000000000000..85bc19ad5979
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/XeVMDialect.h
@@ -0,0 +1,28 @@
+//===-- XeVMDialect.h - MLIR XeVM target definitions ------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_
+#define MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_
+
+#include "mlir/Bytecode/BytecodeOpInterface.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsEnums.h.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.h.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOps.h.inc"
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsDialect.h.inc"
+
+#endif /* MLIR_DIALECT_LLVMIR_XEVMDIALECT_H_ */
diff --git a/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td
new file mode 100644
index 000000000000..b5e81d595d74
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/XeVMOps.td
@@ -0,0 +1,561 @@
+//===-- XeVMOps.td - XeVM dialect definition ---------------*- tablegen -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef XEVMIR_OPS
+#define XEVMIR_OPS
+
+include "mlir/Dialect/GPU/IR/CompilationAttrInterfaces.td"
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+include "mlir/IR/OpBase.td"
+include "mlir/IR/EnumAttr.td"
+
+def XeVM_Dialect : Dialect {
+  let name = "xevm";
+  let cppNamespace = "::mlir::xevm";
+  let summary = "The XeVM dialect that extends LLVM dialect and models Intel "
+                "GPU's hardware features.";
+  let description = [{
+    The XeVM dialect is extension to the LLVM dialect that models hardware
+    features of Intel GPUs. The dialect is designed to work with the Xe
+    architecture for Intel GPUs, supporting advanced operations like 2D block
+    loads, stores, prefetch and matrix multiply-add (MMA) operations.
+  }];
+  let dependentDialects = ["LLVM::LLVMDialect"];
+
+  let extraClassDeclaration = [{
+    /// Get the name for the attribute used to specify cache control
+    /// decorations.
+    static constexpr ::llvm::StringRef getCacheControlsAttrName() {
+      return ::llvm::StringLiteral("xevm.DecorationCacheControl");
+    }
+  }];
+
+  let useDefaultAttributePrinterParser = 1;
+}
+
+class XeVM_Attr<string attrName, string attrMnemonic, list<Trait> traits = []>
+    : AttrDef<XeVM_Dialect, attrName, traits> {
+  let mnemonic = attrMnemonic;
+}
+
+class XeVM_Op<string mnemonic, list<Trait> traits = []>
+    : LLVM_OpBase<XeVM_Dialect, mnemonic, traits> {
+
+  code extraBaseClassDeclaration = [{
+    void printProperties(::mlir::MLIRContext *ctx,
+            ::mlir::OpAsmPrinter &p, const Properties &prop,
+            ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
+      Attribute propAttr = getPropertiesAsAttr(ctx, prop);
+      if (propAttr)
+        p << "<" << propAttr << ">";
+    }
+
+    static ::mlir::ParseResult parseProperties(::mlir::OpAsmParser &parser,
+                                     ::mlir::OperationState &result) {
+      if (mlir::succeeded(parser.parseOptionalLess())) {
+        if (parser.parseAttribute(result.propertiesAttr) || parser.parseGreater())
+          return failure();
+      }
+      return success();
+    }
+
+  }];
+}
+
+def XeVM_ElemType : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, TF32, F16, BF16]>;
+
+//===----------------------------------------------------------------------===//
+// XeVM Load Cache Control
+//   L1, L2, L3 - cache levels
+//   uc - uncached
+//   c - cached
+//   s - streaming
+//   ir - invalidated after read
+//===----------------------------------------------------------------------===//
+
+def LoadCacheControl_L1uc_L2uc_L3uc
+    : I32EnumAttrCase<"L1UC_L2UC_L3UC", 1, "L1uc_L2uc_L3uc">;
+def LoadCacheControl_L1uc_L2uc_L3c
+    : I32EnumAttrCase<"L1UC_L2UC_L3C", 2, "L1uc_L2uc_L3c">;
+def LoadCacheControl_L1uc_L2c_L3uc
+    : I32EnumAttrCase<"L1UC_L2C_L3UC", 3, "L1uc_L2c_L3uc">;
+def LoadCacheControl_L1uc_L2c_L3c
+    : I32EnumAttrCase<"L1UC_L2C_L3C", 4, "L1uc_L2c_L3c">;
+def LoadCacheControl_L1c_L2uc_L3uc
+    : I32EnumAttrCase<"L1C_L2UC_L3UC", 5, "L1c_L2uc_L3uc">;
+def LoadCacheControl_L1c_L2uc_L3c
+    : I32EnumAttrCase<"L1C_L2UC_L3C", 6, "L1c_L2uc_L3c">;
+def LoadCacheControl_L1c_L2c_L3uc
+    : I32EnumAttrCase<"L1C_L2C_L3UC", 7, "L1c_L2c_L3uc">;
+def LoadCacheControl_L1c_L2c_L3c
+    : I32EnumAttrCase<"L1C_L2C_L3C", 8, "L1c_L2c_L3c">;
+def LoadCacheControl_L1s_L2uc_L3uc
+    : I32EnumAttrCase<"L1S_L2UC_L3UC", 9, "L1s_L2uc_L3uc">;
+def LoadCacheControl_L1s_L2uc_L3c
+    : I32EnumAttrCase<"L1S_L2UC_L3C", 10, "L1s_L2uc_L3c">;
+def LoadCacheControl_L1s_L2c_L3uc
+    : I32EnumAttrCase<"L1S_L2C_L3UC", 11, "L1s_L2c_L3uc">;
+def LoadCacheControl_L1s_L2c_L3c
+    : I32EnumAttrCase<"L1S_L2C_L3C", 12, "L1s_L2c_L3c">;
+def LoadCacheControlInvalidateRead
+    : I32EnumAttrCase<"INVALIDATE_READ", 13, "ir">;
+
+def XeVM_LoadCacheControl
+    : I32EnumAttr<
+          "LoadCacheControl", "XeVM load ops cache control",
+          [LoadCacheControl_L1uc_L2uc_L3uc, LoadCacheControl_L1uc_L2uc_L3c,
+           LoadCacheControl_L1uc_L2c_L3uc, LoadCacheControl_L1uc_L2c_L3c,
+           LoadCacheControl_L1c_L2uc_L3uc, LoadCacheControl_L1c_L2uc_L3c,
+           LoadCacheControl_L1c_L2c_L3uc, LoadCacheControl_L1c_L2c_L3c,
+           LoadCacheControl_L1s_L2uc_L3uc, LoadCacheControl_L1s_L2uc_L3c,
+           LoadCacheControl_L1s_L2c_L3uc, LoadCacheControl_L1s_L2c_L3c,
+           LoadCacheControlInvalidateRead]> {
+  let cppNamespace = "::mlir::xevm";
+  let genSpecializedAttr = 0;
+}
+
+def XeVM_LoadCacheControlAttr
+    : EnumAttr<XeVM_Dialect, XeVM_LoadCacheControl, "load_cache_control"> {
+  let summary = [{Describe the cache settings for load operators}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+//===----------------------------------------------------------------------===//
+// XeVM Store Cache Control
+//   L1, L2, L3 - cache levels
+//   uc - uncached
+//   wb - write-back
+//   wt - write-through
+//   s - streaming
+//===----------------------------------------------------------------------===//
+
+def StoreCacheControl_L1uc_L2uc_L3uc
+    : I32EnumAttrCase<"L1UC_L2UC_L3UC", 1, "L1uc_L2uc_L3uc">;
+def StoreCacheControl_L1uc_L2uc_L3wb
+    : I32EnumAttrCase<"L1UC_L2UC_L3WB", 2, "L1uc_L2uc_L3wb">;
+def StoreCacheControl_L1uc_L2wb_L3uc
+    : I32EnumAttrCase<"L1UC_L2WB_L3UC", 3, "L1uc_L2wb_L3uc">;
+def StoreCacheControl_L1uc_L2wb_L3wb
+    : I32EnumAttrCase<"L1UC_L2WB_L3WB", 4, "L1uc_L2wb_L3wb">;
+def StoreCacheControl_L1wt_L2uc_L3uc
+    : I32EnumAttrCase<"L1WT_L2UC_L3UC", 5, "L1wt_L2uc_L3uc">;
+def StoreCacheControl_L1wt_L2uc_L3wb
+    : I32EnumAttrCase<"L1WT_L2UC_L3WB", 6, "L1wt_L2uc_L3wb">;
+def StoreCacheControl_L1wt_L2wb_L3uc
+    : I32EnumAttrCase<"L1WT_L2WB_L3UC", 7, "L1wt_L2wb_L3uc">;
+def StoreCacheControl_L1wt_L2wb_L3wb
+    : I32EnumAttrCase<"L1WT_L2WB_L3WB", 8, "L1wt_L2wb_L3wb">;
+def StoreCacheControl_L1s_L2uc_L3uc
+    : I32EnumAttrCase<"L1S_L2UC_L3UC", 9, "L1s_L2uc_L3uc">;
+def StoreCacheControl_L1s_L2uc_L3wb
+    : I32EnumAttrCase<"L1S_L2UC_L3WB", 10, "L1s_L2uc_L3wb">;
+def StoreCacheControl_L1s_L2wb_L3uc
+    : I32EnumAttrCase<"L1S_L2WB_L3UC", 11, "L1s_L2wb_L3uc">;
+def StoreCacheControl_L1s_L2wb_L3wb
+    : I32EnumAttrCase<"L1S_L2WB_L3WB", 12, "L1s_L2wb_L3wb">;
+def StoreCacheControl_L1wb_L2uc_L3uc
+    : I32EnumAttrCase<"L1WB_L2UC_L3UC", 13, "L1wb_L2uc_L3uc">;
+def StoreCacheControl_L1wb_L2wb_L3uc
+    : I32EnumAttrCase<"L1WB_L2WB_L3UC", 14, "L1wb_L2wb_L3uc">;
+def StoreCacheControl_L1wb_L2uc_L3wb
+    : I32EnumAttrCase<"L1WB_L2UC_L3WB", 15, "L1wb_L2uc_L3wb">;
+
+def XeVM_StoreCacheControl
+    : I32EnumAttr<
+          "StoreCacheControl", "XeVM store ops cache control",
+          [StoreCacheControl_L1uc_L2uc_L3uc, StoreCacheControl_L1uc_L2uc_L3wb,
+           StoreCacheControl_L1uc_L2wb_L3uc, StoreCacheControl_L1uc_L2wb_L3wb,
+           StoreCacheControl_L1wt_L2uc_L3uc, StoreCacheControl_L1wt_L2uc_L3wb,
+           StoreCacheControl_L1wt_L2wb_L3uc, StoreCacheControl_L1wt_L2wb_L3wb,
+           StoreCacheControl_L1s_L2uc_L3uc, StoreCacheControl_L1s_L2uc_L3wb,
+           StoreCacheControl_L1s_L2wb_L3uc, StoreCacheControl_L1s_L2wb_L3wb,
+           StoreCacheControl_L1wb_L2uc_L3uc, StoreCacheControl_L1wb_L2wb_L3uc,
+           StoreCacheControl_L1wb_L2uc_L3wb]> {
+  let cppNamespace = "::mlir::xevm";
+  let genSpecializedAttr = 0;
+}
+
+def XeVM_StoreCacheControlAttr
+    : EnumAttr<XeVM_Dialect, XeVM_StoreCacheControl, "store_cache_control"> {
+  let summary = [{Describe the cache settings for store operators}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def XeVM_BlockLoad2dOp
+    : XeVM_Op<"blockload2d">,
+      Results<(outs FixedVectorOfRankAndType<[1], [XeVM_ElemType]>:$res)>,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemRead]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          I32Attr:$v_blocks, I1Attr:$transpose, I1Attr:$pack_register,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block load";
+
+  let description = [{
+    The `xevm.blockload2d` operation loads a two dimensional matrix tile
+    from a base matrix residing in global memory. The parameters are:
+      $ptr - the base address of the base matrix containing the tile to load
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of
+      the tile to load in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element type
+        - 32 for f32, tf32
+        - 16 for f16, int16, bf16
+        - 8 for int8
+      $v_blocks - number of consecutive tiles in innermost dimension direction to load
+      $transpose - transpose the tile in registers (useful for 32 bit element type)
+      $pack_register - pack element types narrower than register bit width.
+        [M, N] => [M/factor, N, factor] where factor is register_size_in_bits / elem_size_in_bits
+      $cache_control - an enumerator that sets the cache behaviour
+
+    Notes:
+      - the $transpose and $pack_register parameters are mutual exclusive
+      - transposing the tile loaded is used for A matrix in backward path or used for the B matrix operand
+        (D = C + A * B), where A has row-major layout and B should have column-major layout in memory.
+      - if the tile loaded contains out of bound elements of the matrix, they are filled with 0.
+
+    Example:
+    ```mlir
+      %base_width_a = arith.constant 32 : i32
+      %base_height_a = arith.constant 8 : i32
+      %base_pitch_a = arith.constant 32 : i32
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      %loaded_a = xevm.blockload2d %src, %base_width_a, %base_height_a, %base_pitch_a, %x, %y
+                    <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32,
+                      v_blocks=1 : i32, transpose=false : i32, pack_register=false,
+                      cache_control=#xevm.load_cache_control<Default>}>
+                    : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeVM_BlockStore2dOp
+    : XeVM_Op<"blockstore2d">,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemWrite]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          FixedVectorOfRankAndType<[1], [XeVM_ElemType]>:$stored_val,
+          OptionalAttr<XeVM_StoreCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block store";
+
+  let description = [{
+    The `xevm.blockstore2d` operation stores a two dimensional tile into a
+    larger matrix residing in global memory. The parameters are:
+      $ptr - the base address of the target matrix where to store the tile
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of the tile to store
+      in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element
+        - 32 for f32, tf32
+        - 16 for f16, int16, bf16
+        - 8 for int8
+      $cache_control - an enumerator that sets the cache behaviour
+      $stored_val - the tile to store
+
+    Example:
+    ```mlir
+      %base_width_c = arith.constant 64 : i32
+      %base_height_c = arith.constant 8 : i32
+      %base_pitch_c = arith.constant 64 : i32
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      xevm.blockstore2d %dst, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %src
+        <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32,
+          cache_control=#xevm.load_cache_control<Default>}>
+        : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+    /// Default value for v_blocks is 1.
+    constexpr uint32_t getVBlocks() {
+      return 1;
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+def MemScopeLane : I32EnumAttrCase<"LANE", 0, "lane">;
+def MemScopeSubgroup : I32EnumAttrCase<"SUBGROUP", 1, "subgroup">;
+def MemScopeWorkgroup : I32EnumAttrCase<"WORKGROUP", 2, "workgroup">;
+def MemScopeCluster : I32EnumAttrCase<"CLUSTER", 3, "cluster">;
+def MemScopeDevice : I32EnumAttrCase<"DEVICE", 4, "device">;
+def MemScopeSystem : I32EnumAttrCase<"SYSTEM", 5, "system">;
+
+def XeVM_MemScope
+    : I32EnumAttr<"MemScope", "XeVM memory scope",
+                  [MemScopeLane, MemScopeSubgroup, MemScopeWorkgroup,
+                   MemScopeCluster, MemScopeDevice, MemScopeSystem]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::xevm";
+}
+def XeVM_MemScopeAttr : EnumAttr<XeVM_Dialect, XeVM_MemScope, "mem_scope"> {
+  let summary = [{Describe memory scopes}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def AddrSpacePrivate : I32EnumAttrCase<"PRIVATE", 0, "private">;
+def AddrSpaceGlobal : I32EnumAttrCase<"GLOBAL", 1, "global">;
+def AddrSpaceConstant : I32EnumAttrCase<"CONSTANT", 2, "constant">;
+def AddrSpaceShared : I32EnumAttrCase<"SHARED", 3, "shared">;
+def AddrSpaceGeneric : I32EnumAttrCase<"GENERIC", 4, "generic">;
+
+def XeVM_AddrSpace
+    : I32EnumAttr<"AddrSpace", "Address spaces",
+                  [AddrSpacePrivate, AddrSpaceGlobal, AddrSpaceConstant,
+                   AddrSpaceShared, AddrSpaceGeneric]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "mlir::xevm";
+}
+def XeVM_AddrSpaceAttr : EnumAttr<XeVM_Dialect, XeVM_AddrSpace, "addr_space"> {
+  let summary = [{Describe address spaces}];
+  let assemblyFormat = "`<` $value `>`";
+}
+
+def XeVM_MemfenceOp
+    : XeVM_Op<"memfence">,
+      Arguments<(ins XeVM_MemScopeAttr:$scope,
+          DefaultValuedAttr<XeVM_AddrSpaceAttr,
+                            "mlir::xevm::AddrSpace::GENERIC">:$addrspace)> {
+  let summary = "Work-item's memory fence.";
+  let description = [{
+    This operation ensures that all prior memory accesses of this
+    work-item to `addrspace` are visible to all other work-items in `scope`.
+    Parameters description:
+    $scope - specify the memory scope at which all other work-items should observe
+                memory operations prior to the fence.
+    $addrspace - specify the address space of work-item's memory accesses
+                to be affected by the fence.
+  }];
+  let assemblyFormat = [{prop-dict  attr-dict}];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+}
+
+def XeVM_PrefetchOp
+    : XeVM_Op<"prefetch">,
+      Arguments<(ins Arg<AnyTypeOf<[LLVM_PointerInAddressSpace<1>,
+                                    LLVM_PointerInAddressSpace<4>]>>:$ptr,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+  let summary = "Prefetch data into a cache subsystem.";
+  let description = [{
+    Work-item issues a prefetch from global memory to cache:
+    $ptr - LLVM pointer with address space. Address space must be 1 (global)
+      or 4 (generic)
+    $cache_control - specify caching options
+  }];
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+}
+
+def XeVM_BlockPrefetch2dOp
+    : XeVM_Op<"blockprefetch2d">,
+      Arguments<(ins Arg<LLVM_AnyPointer, "", [MemRead]>:$ptr, I32:$base_width,
+          I32:$base_height, I32:$base_pitch, I32:$x, I32:$y,
+          I32Attr:$elem_size_in_bits, I32Attr:$tile_width, I32Attr:$tile_height,
+          I32Attr:$v_blocks,
+          OptionalAttr<XeVM_LoadCacheControlAttr>:$cache_control)> {
+
+  let summary = "2D block prefetch";
+
+  let description = [{
+    The `xevm.blockprefetch2d` operation prefetches a two dimensional tile
+    from a larger base matrix residing in global memory. The parameters are:
+      $ptr - the base address of the base matrix containing the tile to prefetch
+      $base_width - the width of the base matrix in number of bytes.
+      $base_height - the number of rows in the base matrix
+      $base_pitch - the physical stride between the first columns of the current
+         row and the subsequent row in number of bytes.
+      $x, $y, $tile_width, $tile_height - the starting offsets and shape of tile
+      to prefetch in number of elements.
+      $elem_size_in_bits - the size in bits of the matrix element
+      - 32 for f32, bf32
+      - 16 for f16, int16, bf16
+      - 8 for int8, int4, int2
+      $v_blocks - number of tiles in innermost dimension direction to prefetch
+      $cache_control - an enumerator that sets the cache behaviour
+
+    Example:
+    ```mlir
+      xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y
+        <{elem_size_in_bits=8 : i32, tile_width=32 : i32, tile_height=8 : i32,
+          v_blocks=1 : i32, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}>
+        : (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+    ```
+  }];
+
+  let assemblyFormat = [{
+    operands prop-dict attr-dict `:` `(` type(operands) `)`
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeVM_MatrixElemType
+    : AnyTypeOf<[AnyI8, AnyI16, AnyI32, F32, TF32, F16, BF16]>;
+
+/// Enum attribute of the different element types.
+def XeVM_ET_BF16 : I32EnumAttrCase<"BF16", 8, "bf16">;
+def XeVM_ET_F16 : I32EnumAttrCase<"F16", 9, "f16">;
+def XeVM_ET_S8 : I32EnumAttrCase<"S8", 10, "s8">;
+def XeVM_ET_U8 : I32EnumAttrCase<"U8", 11, "u8">;
+def XeVM_ET_S4 : I32EnumAttrCase<"S4", 12, "s4">;
+def XeVM_ET_U4 : I32EnumAttrCase<"U4", 13, "u4">;
+def XeVM_ET_TF32 : I32EnumAttrCase<"TF32", 14, "tf32">;
+def XeVM_ET_F32 : I32EnumAttrCase<"F32", 15, "f32">;
+def XeVM_ET_S32 : I32EnumAttrCase<"S32", 16, "s32">;
+
+def XeVM_ElemTypeAttr : I32EnumAttr<"ElemType", "XeVM element type",
+                                    [XeVM_ET_BF16, XeVM_ET_F16, XeVM_ET_S8,
+                                     XeVM_ET_U8, XeVM_ET_S4, XeVM_ET_U4,
+                                     XeVM_ET_TF32, XeVM_ET_F32, XeVM_ET_S32]> {
+  let cppNamespace = "::mlir::xevm";
+}
+
+def XeVM_MMAShapeAttr : XeVM_Attr<"MMAShape", "mma_shape"> {
+  let description = [{
+    MMA operation is represented as D=AxB+C, where
+      A has the shape MxK.
+      B has the shape KxN.
+      D and C have the shape MxN.
+    This attribute encodes the shape of all matrices that participate in MMA.
+  }];
+  let parameters = (ins "int":$m, "int":$n, "int":$k);
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def XeVM_MMATypesAttr : XeVM_Attr<"MMATypes", "mma_types"> {
+  let parameters = (ins "xevm::ElemType":$d, "xevm::ElemType":$a,
+      "xevm::ElemType":$b, OptionalParameter<"xevm::ElemType">:$c);
+  let assemblyFormat = "`<` struct(params) `>`";
+}
+
+def XeVM_MMAOp
+    : XeVM_Op<"mma">,
+      Results<(outs FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$d)>,
+      Arguments<(ins FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$a,
+          FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>:$b,
+          Optional<FixedVectorOfRankAndType<[1], [XeVM_MatrixElemType]>>:$c,
+          XeVM_MMAShapeAttr:$shape, XeVM_MMATypesAttr:$types)> {
+
+  let summary = "Subgroup matrix multiply-add";
+
+  let description = [{
+    The `xevm.mma` is a cooperative operation where all threads/lanes in
+    a subgroup participates and carries out matrix multiplication plus accumulation:
+
+      D = C + A x B
+
+      where the A, B, C input matrices and the result D have shapes:
+        D : MxN
+        C : MxN
+        A : MxK
+        B : KxN
+
+    Parameters:
+      `a` - vector of matrix A elements.
+      `b` - vector of matrix B elements.
+      `c` - (optional) vector of matrix C elements.
+      `shape` - the shape of the matrices, specified as `M`, `N`, and `K` values.
+      `types` - the data types of the matrices, specified as `D`, `A`, `B`, and optionally `C`.
+
+    Example:
+    ```mlir
+      %d = xevm.mma %a, %b, %c { shape=<m=8, n=16, k=16>, types=<d=f32, a=f16, b=f16, c=f32> }
+             : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+    ```
+  }];
+
+  let assemblyFormat = [{
+    $a `,` $b (`,` $c^)? ` `
+    `{`
+      `shape` `=` $shape `,`
+      `types` `=` $types
+    `}` attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// XeVM target attribute.
+//===----------------------------------------------------------------------===//
+
+def XeVM_TargetAttr : XeVM_Attr<"XeVMTarget", "target"> {
+  let description = [{
+    GPU target attribute for controlling compilation of Intel GPU targets. All
+    parameters decay into default values if not present.
+
+    Examples:
+
+    1. Target with default values.
+    ```
+      gpu.module @mymodule [#xevm.target] attributes {...} {
+        ...
+      }
+    ```
+  }];
+  let parameters =
+      (ins DefaultValuedParameter<"int", "2",
+                                  "Optimization level to apply.">:$O,
+          StringRefParameter<"Target triple.",
+                             "\"spirv64-unknown-unknown\"">:$triple,
+          StringRefParameter<"Target chip.", "\"bmg\"">:$chip,
+          OptionalParameter<"::mlir::DictionaryAttr",
+                            "Target specific flags.">:$flags,
+          OptionalParameter<"::mlir::ArrayAttr",
+                            "Files to link to the LLVM module.">:$linkFiles);
+  let assemblyFormat = [{
+    (`<` struct($O, $triple, $chip, $flags, $linkFiles)^ `>`)?
+  }];
+  let builders = [AttrBuilder<
+      (ins CArg<"int", "2">:$optLevel,
+          CArg<"::llvm::StringRef", "\"spirv64-unknown-unknown\"">:$triple,
+          CArg<"::llvm::StringRef", "\"bmg\"">:$chip,
+          CArg<"::mlir::DictionaryAttr", "nullptr">:$targetFlags,
+          CArg<"::mlir::ArrayAttr", "nullptr">:$linkFiles),
+      [{
+      return Base::get($_ctxt, optLevel, triple, chip, targetFlags, linkFiles);
+    }]>];
+  let skipDefaultBuilders = 1;
+  let genVerifyDecl = 1;
+}
+
+#endif // XEVMIR_OPS
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h b/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
index 57bf6305a469..bb0ac414bcc2 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
@@ -16,6 +16,7 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -26,6 +27,9 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
+
+#include "llvm/ADT/STLFunctionalExtras.h"
+
 #include <optional>
 
 namespace mlir {
@@ -100,6 +104,20 @@ OpFoldResult createFoldedDimOp(OpBuilder &b, Location loc, Value val,
 
 #include "mlir/Dialect/Linalg/IR/LinalgOpsEnums.h.inc"
 
+namespace mlir {
+namespace linalg {
+
+/// Converts the given `m` and `r` parameters to a WinogradConv2DFmr enumeration
+/// value.
+std::optional<WinogradConv2DFmr> getWinogradConv2DFmr(int64_t m, int64_t r);
+
+/// Converts the given WinogradConv2DFmr enumeration value to a pair of
+/// m and r parameters.
+std::pair<int64_t, int64_t> getFmrFromWinogradConv2DFmr(WinogradConv2DFmr fmr);
+
+} // namespace linalg
+} // namespace mlir
+
 //===----------------------------------------------------------------------===//
 // Linalg Attributes
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
index 33601c5d6dad..a459656b982e 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgBase.td
@@ -52,7 +52,8 @@ def Linalg_Dialect : Dialect {
         kMemoizedIndexingMapsAttrName = "linalg.memoized_indexing_maps";
 
     using RegionBuilderFunType = llvm::function_ref<
-      void(ImplicitLocOpBuilder &b, Block &, ArrayRef<NamedAttribute>)>;
+      void(ImplicitLocOpBuilder &b, Block &, ArrayRef<NamedAttribute>,
+           function_ref<InFlightDiagnostic()>)>;
     RegionBuilderFunType getRegionBuilder(StringRef name) {
       return namedStructuredOpRegionBuilders.lookup(name);
     }
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
index ce68afe471fe..1109db973f52 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgEnums.td
@@ -122,4 +122,19 @@ def TypeFn : I32EnumAttr<"TypeFn", "", [
   let cppNamespace = "::mlir::linalg";
 }
 
+/// We use F(m, r) to define the size of minimal filtering algorithms.
+/// m is the output dimension and r is the filter dimension. We can get
+/// the input dimension, alpha, from the formula, alpha = m + r - 1.
+///
+/// For example, when m = 2 and r = 3, we know its input size is 4.
+/// The Conv2D will operate on 4x4 input data with 3x3 filter and get
+/// 2x2 output result.
+def WinogradConv2DFmr : I32EnumAttr<"WinogradConv2DFmr", "", [
+      I32EnumAttrCase<"F_2_3", 0>,
+      I32EnumAttrCase<"F_4_3", 1>,
+      I32EnumAttrCase<"F_2_5", 2>,
+]>{
+  let cppNamespace = "mlir::linalg";
+}
+
 #endif // LINALG_ENUMS
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index ca1cba8747bd..ba73cfbbed84 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -720,7 +720,7 @@ def LinalgStructuredInterface
         Returns a null function if this named op does not define a region
         builder.
       }],
-      /*retTy=*/"std::function<void(ImplicitLocOpBuilder &, Block &, ArrayRef<NamedAttribute>)>",
+      /*retTy=*/"std::function<void(ImplicitLocOpBuilder &, Block &, ArrayRef<NamedAttribute>, function_ref<InFlightDiagnostic()>)>",
       /*methodName=*/"getRegionBuilder",
       (ins),
       [{ return ConcreteOp::getRegionBuilder(); }]
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index 6344861c53ac..3637147c5a90 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -44,56 +44,6 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: I
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: elemwise_unary
-  cpp_class_name: ElemwiseUnaryOp
-  doc: |-
-    Applies the unary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: I
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: O
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: fun
-    kind: unary_fn_attr
-    default_fn: exp
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-  iterator_types: []
-  assignments:
-  - !ScalarAssign
-    arg: O
-    value: !ScalarExpression
-      scalar_fn:
-        kind: unary
-        attr_name: fun
-        operands:
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: I
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: exp
   cpp_class_name: ExpOp
@@ -549,70 +499,6 @@ structured_op: !LinalgStructuredOpConfig
         - !ScalarExpression
           scalar_arg: I
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: elemwise_binary
-  cpp_class_name: ElemwiseBinaryOp
-  doc: |-
-    Applies the binary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: lhs
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: rhs
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: O
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<() -> ()>
-  - !LinalgOperandDefConfig
-    name: fun
-    kind: binary_fn_attr
-    default_fn: add
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-    - affine_map<() -> ()>
-  iterator_types: []
-  assignments:
-  - !ScalarAssign
-    arg: O
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        attr_name: fun
-        operands:
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: lhs
-        - !ScalarExpression
-          scalar_fn:
-            kind: type
-            attr_name: cast
-            type_var: U
-            operands:
-            - !ScalarExpression
-              scalar_arg: rhs
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: add
   cpp_class_name: AddOp
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index 1b48bf5fcb23..7ff44c2e1d2e 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -183,15 +183,13 @@ def Linalg_WinogradFilterTransformOp : Linalg_Op<"winograd_filter_transform",
 
   let arguments = (ins TensorRankOf<[AnyType], [4]>:$filter,
                        TensorRankOf<[AnyType], [4]>:$output,
-                       I64Attr:$m,
-                       I64Attr:$r
+                       WinogradConv2DFmr:$fmr
   );
 
   let results = (outs TensorRankOf<[AnyType], [4]>:$result);
   let assemblyFormat = [{
     attr-dict
-    `m` `(` $m `)`
-    `r` `(` $r `)`
+    `fmr` `(` $fmr `)`
     `ins` `(` $filter `:` type($filter) `)`
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
@@ -254,15 +252,13 @@ def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
 
   let arguments = (ins TensorRankOf<[AnyType], [4]>:$input,
                        TensorRankOf<[AnyType], [6]>:$output,
-                       I64Attr:$m,
-                       I64Attr:$r
+                       WinogradConv2DFmr:$fmr
   );
 
   let results = (outs TensorRankOf<[AnyType], [6]>:$result);
   let assemblyFormat = [{
     attr-dict
-    `m` `(` $m `)`
-    `r` `(` $r `)`
+    `fmr` `(` $fmr `)`
     `ins` `(` $input `:` type($input) `)`
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
@@ -343,15 +339,13 @@ def Linalg_WinogradOutputTransformOp : Linalg_Op<"winograd_output_transform",
 
   let arguments = (ins TensorRankOf<[AnyType], [6]>:$value,
                        TensorRankOf<[AnyType], [4]>:$output,
-                       I64Attr:$m,
-                       I64Attr:$r
+                       WinogradConv2DFmr:$fmr
   );
 
   let results = (outs TensorRankOf<[AnyType], [4]>:$result);
   let assemblyFormat = [{
     attr-dict
-    `m` `(` $m `)`
-    `r` `(` $r `)`
+    `fmr` `(` $fmr `)`
     `ins` `(` $value `:` type($value) `)`
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
index 1e48a5e3a20e..c384e8b63838 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgRelayoutOps.td
@@ -93,17 +93,21 @@ def Linalg_PackOp : Linalg_RelayoutOp<"pack", [
     tensor of rank `n + k` with a tiled and packed layout (maybe with padding)
     and optionally transposes the tiled source tensor dimensions.
 
-    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
-    being tiled, where `0 < k <= n`. The order of the dimensions matters:
-     - The tiled dimensions (of size `inner_tiles`) are added to the end of the result
-    tensor in the order in which they appear in `inner_dims_pos`.
-     - `inner_dims_pos[i]` specifies the source tensor dimension tiled by
-    `inner_tiles[i]`.
-
     `inner_tiles` (mandatory) specifies `k` tile sizes. These tile sizes
     correspond to the least significant ("inner") result tensor dimension sizes,
     in the same order. Tile sizes can be static or dynamic.
 
+    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
+    being tiled, where `0 <= k <= n`.
+     - `inner_dims_pos[i]` specifies the source tensor dimension tiled by
+    `inner_tiles[i]` where `0 <= i < k`. All the values in `inner_dims_pos` are
+    within [0, n).
+     - The tiled dimensions (of size `inner_tiles`) are added to the end of the
+     result tensor in the order in which they appear, i.e.
+     `shape(result)[rank(result) + i] = inner_tiles[i]` for `0 <= i < k`.
+     - The following relationship for the tiled dimensions holds:
+     `shape(result)[inner_dims_pos[i]] = shape(source)[inner_dims_pos[i]] / inner_tiles[i]`.
+
     Example: If `inner_tiles = [16, 32]`, the result tensor has a shape of
     `...x16x32`. If `inner_dims_pos = [0, 1]`, the 0th source dimension is tiled
     by 16 and the 1st source dimension is tiled by 32. Other source dimensions
@@ -116,7 +120,19 @@ def Linalg_PackOp : Linalg_RelayoutOp<"pack", [
     %0 = linalg.pack %source inner_dims_pos = [0, 1] inner_tiles = [8, 32]
         into %dest : tensor<128x256xf32> -> tensor<16x8 x 8x32 xf32>
     //                                             \  /   \  /
-    //                                       outer dims  inner dims
+    //                                 Outer Dims: 16x8   Inner Dims: 8x32
+
+    // CHW to CHWhw
+    %0 = linalg.pack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2]
+        into %dest : tensor<3x20x24xf32> -> tensor<3x10x6 x 4x2 xf32>
+    //                                              \  /    \ /
+    //                                 Outer Dims: 3x10x6  Inner Dims: 4x2
+
+    // HCW to HCWhw
+    %0 = linalg.pack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2]
+        into %dest : tensor<18x3x32xf32> -> tensor<9x3x8 x 4x2 xf32>
+    //                                              \  /   \ /
+    //                                 Outer Dims: 9x3x8  Inner Dims: 4x2
     ```
 
     `outer_dims_perm` (optional) specifies a permutation for the outer
@@ -246,13 +262,6 @@ def Linalg_UnPackOp : Linalg_RelayoutOp<"unpack"> {
     The "unpack" operation converts a source tensor of rank `n` with a tiled and
     packed layout to a result tensor of rank `n - k`.
 
-    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions with
-    which the last `k` source tensor dimensions are combined, where
-    `0 < k <= n/2`. Each `inner_dims_pos` element must be `>= 0` and `< n - k`.
-    The order of the dimensions in `inner_dims_pos` matters: dimension
-    `inner_dims_pos[i]` is combined with dimension `n - k + i` (assuming that
-    `outer_dims_perm` is not specified).
-
     `inner_tiles` (mandatory) specifies `k` tile sizes. These tile sizes
     correspond to the least significant ("inner") source tensor dimension sizes.
     The behavior of this op is undefined if:
@@ -262,21 +271,50 @@ def Linalg_UnPackOp : Linalg_RelayoutOp<"unpack"> {
       `inner_dims_pos[i]` (assuming that `outer_dims_perm` is not specified)
       evenly.
 
+    `inner_dims_pos` (mandatory) specifies `k` result tensor (i.e. unpacked
+    tensor) dimensions that were tiled with the `inner_tiles` to create the
+    packed source tensor. The source tensor (i.e. packed tensor) dimensions can
+    be unpacked given `inner_dims_pos` as follows.
+    - For `0 <= i < k` the following relationship holds:
+    `shape(result)[inner_dims_pos[i]] <= shape(source)[n-k+i] * shape(source)[inner_dims_pos[i]]`.
+    - For `0 <= j < n-k` and `j` not in `inner_dims_pos` the following relationship holds:
+    `shape(result)[j] = shape(source)[j]`.
+
     `outer_dims_perm` (optional) specifies a permutation for the outer
     dimensions. If specified, it must have `n - k` elements. If specified, this
     permutation is applied before combining any dimensions.
 
-    Example:
+    Note, the unpack operation may drop any padding introduced by the pack
+    operation and hence the following holds
+    `NumElementsOf(source) >= NumElementsOf(result)`.
+
+    Examples:
 
     ```mlir
     // NCnc to NC:
     %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [8, 32]
-        into %dest : tensor<16x8x8x32xf32> -> tensor<128x256xf32>
+        into %dest : tensor<16x8 x 8x32 xf32> -> tensor<128x256xf32>
+    //                      \  /   \  /
+    //          Outer Dims: 16x8  Inner Dims: 8x32
 
     // CK to KCck:
     %0 = linalg.unpack %source outer_dims_perm = [1, 0] inner_dims_pos = [0, 1]
-        inner_tiles = [8, 32] into %dest
-        : tensor<8x16x8x32xf32> -> tensor<128x256xf32>
+        inner_tiles = [8, 32]
+        into %dest : tensor<8x16 x 8x32 xf32> -> tensor<128x256xf32>
+    //                      \  /   \  /
+    //          Outer Dims: 8x16  Inner Dims: 8x32
+
+    // CHW to CHWhw:
+    %0 = linalg.unpack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2]
+        into %dest : tensor<3x10x6 x 4x2 xf32> -> tensor<3x20x24xf32>
+    //                       \  /    \ /
+    //          Outer Dims: 3x10x6  Inner Dims: 4x2
+
+    // HCW to HCWhw
+    %0 = linalg.unpack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2]
+        into %dest : tensor<9x3x8 x 4x2 xf32> -> tensor<18x3x32xf32>
+    //                       \  /   \ /
+    //          Outer Dims: 9x3x8   Inner Dims: 4x2
     ```
   }];
   let arguments = (ins AnyRankedTensor:$source,
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index 61783812920b..7bbc56f549c0 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -192,7 +192,8 @@ def GenericOp : LinalgStructuredBase_Op<"generic", [
     }
 
     static std::function<void(ImplicitLocOpBuilder &,
-                              Block &, ArrayRef<NamedAttribute>)>
+                              Block &, ArrayRef<NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
     getRegionBuilder() {
       return nullptr;
     }
@@ -300,7 +301,8 @@ def MapOp : LinalgStructuredBase_Op<"map", [
     }
 
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-                              mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
     getRegionBuilder() {
       return nullptr;
     }
@@ -380,7 +382,8 @@ def ReduceOp : LinalgStructuredBase_Op<"reduce", [
 
     // Implement functions necessary for DestinationStyleOpInterface.
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-                              mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
     getRegionBuilder() {
       return nullptr;
     }
@@ -449,13 +452,14 @@ def TransposeOp : LinalgStructuredBase_Op<"transpose", [
     MutableOperandRange getDpsInitsMutable() { return getInitMutable(); }
 
     static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
-        mlir::ArrayRef<mlir::NamedAttribute>) {
+        mlir::ArrayRef<mlir::NamedAttribute>, function_ref<InFlightDiagnostic()> emitError) {
       OpBuilder::InsertionGuard guard(b);
       b.create<linalg::YieldOp>(b.getLoc(), block.getArgument(0));
     }
 
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-        mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
       return regionBuilder;
     }
@@ -521,13 +525,15 @@ def BroadcastOp : LinalgStructuredBase_Op<"broadcast", [
     MutableOperandRange getDpsInitsMutable() { return getInitMutable(); }
 
     static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
-        mlir::ArrayRef<mlir::NamedAttribute>) {
+                              mlir::ArrayRef<mlir::NamedAttribute>, 
+                              function_ref<InFlightDiagnostic()> emitError) {
       OpBuilder::InsertionGuard guard(b);
       b.create<linalg::YieldOp>(b.getLoc(), block.getArgument(0));
     }
 
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-        mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
       return regionBuilder;
     }
@@ -631,10 +637,12 @@ def ElementwiseOp : LinalgStructuredBase_Op<"elementwise", [
       /// Implements the block region builder for the elementwiseOp. This is
       /// called by the 'fillStructuredOpRegion'.
       static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
+                                Block &block, ArrayRef<NamedAttribute> attrs,
+                                function_ref<InFlightDiagnostic()> emitError);
 
       static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
+                                Block &, ArrayRef<NamedAttribute>,
+                                function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
         return regionBuilder;
       }
@@ -771,7 +779,8 @@ def MatmulOp : LinalgStructuredBase_Op<"matmul", [
 
       /// Implements the block region builder.
       static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
+                                Block &block, ArrayRef<NamedAttribute> attrs,
+                                function_ref<InFlightDiagnostic()> emitError);
 
       /// Returns a list of AffineMap with the default matmul indexing charactristic.
       static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
@@ -780,7 +789,8 @@ def MatmulOp : LinalgStructuredBase_Op<"matmul", [
       bool isValidLhsRhsBroadcastMap(AffineMap bcastMap);
 
       static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
+                                Block &, ArrayRef<NamedAttribute>,
+                                function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
         return regionBuilder;
       }
@@ -916,10 +926,12 @@ def ContractOp : LinalgStructuredBase_Op<"contract", [
     static unsigned getNumRegionArgs();
 
     static void regionBuilder(ImplicitLocOpBuilder &b,
-                              Block &block, ArrayRef<NamedAttribute> attrs);
+                              Block &block, ArrayRef<NamedAttribute> attrs,
+                              function_ref<InFlightDiagnostic()> emitError);
 
     static std::function<void(ImplicitLocOpBuilder &,
-                              Block &, ArrayRef<NamedAttribute>)>
+                              Block &, ArrayRef<NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
     getRegionBuilder() {
       return regionBuilder;
     }
@@ -1033,9 +1045,11 @@ def BatchMatmulOp : LinalgStructuredBase_Op<"batch_matmul", !listconcat([AttrSiz
 
       SmallVector<utils::IteratorType> getIteratorTypesArray();
       static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
+                                Block &block, ArrayRef<NamedAttribute> attrs,
+                                function_ref<InFlightDiagnostic()> emitError);
       static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
+                                Block &, ArrayRef<NamedAttribute>,
+                                function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
         return regionBuilder;
       }
@@ -1161,7 +1175,8 @@ def BatchReduceMatmulOp : LinalgStructuredBase_Op<"batch_reduce_matmul", [
 
       /// Implements the block region builder.
       static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
+                                Block &block, ArrayRef<NamedAttribute> attrs,
+                                function_ref<InFlightDiagnostic()> emitError);
 
       /// Returns a list of AffineMap with the default batch_reduce_matmul indexing charactristic.
       static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
@@ -1170,7 +1185,8 @@ def BatchReduceMatmulOp : LinalgStructuredBase_Op<"batch_reduce_matmul", [
       bool isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS = true);
 
       static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
+                                Block &, ArrayRef<NamedAttribute>,
+                              function_ref<InFlightDiagnostic()>)>
       getRegionBuilder() {
         return regionBuilder;
       }
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 9d6ce653e285..4360055e7869 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -9,6 +9,7 @@
 #ifndef LINALG_TRANSFORM_OPS
 #define LINALG_TRANSFORM_OPS
 
+include "mlir/Dialect/Linalg/IR/LinalgEnums.td"
 include "mlir/Dialect/Linalg/TransformOps/LinalgTransformEnums.td"
 include "mlir/Dialect/Transform/IR/TransformAttrs.td"
 include "mlir/Dialect/Transform/IR/TransformDialect.td"
@@ -1401,6 +1402,7 @@ def PromoteOp : Op<Transform_Dialect, "structured.promote",
                        DefaultValuedAttr<I64ArrayAttr, "{}">:$operands_to_promote,
                        DefaultValuedAttr<BoolArrayAttr, "{}">:$use_full_tile_buffers,
                        UnitAttr:$use_full_tiles_by_default,
+                       UnitAttr:$use_original_subview_size,
                        UnitAttr:$use_alloca,
                        OptionalAttr<AnyAttr>:$memory_space,
                        OptionalAttr<DeviceMappingArrayAttr>:$mapping,
@@ -2902,8 +2904,7 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   }];
 
   let arguments = (ins TransformHandleTypeInterface:$target,
-                       I64Attr:$m,
-                       I64Attr:$r);
+                       WinogradConv2DFmr:$fmr);
   let results = (outs TransformHandleTypeInterface:$transformed);
 
   let assemblyFormat =
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index a6dab03d6473..74280fdd82f4 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -37,6 +37,7 @@ class BufferizationState;
 namespace linalg {
 
 class LinalgOp;
+enum class WinogradConv2DFmr : uint32_t;
 
 //===----------------------------------------------------------------------===//
 // Utils.
@@ -355,12 +356,6 @@ struct PadTilingInterfaceOptions {
     paddingValues.assign(pv.begin(), pv.end());
     return *this;
   }
-  /// A list of iterator dimensions to pad.
-  SmallVector<int64_t> paddingDimensions;
-  PadTilingInterfaceOptions &setPaddingDimensions(ArrayRef<int64_t> pd) {
-    paddingDimensions.assign(pd.begin(), pd.end());
-    return *this;
-  }
   /// A list of iterator dimensions sizes to pad to.
   SmallVector<OpFoldResult> paddingSizes;
   PadTilingInterfaceOptions &setPaddingSizes(ArrayRef<OpFoldResult> m) {
@@ -427,6 +422,13 @@ struct LinalgPromotionOptions {
     useFullTileBuffersDefault = use;
     return *this;
   }
+  /// If true, buffers will be allocated with the original subview size. This
+  /// may result in more dynamic allocations, in case of dynamic sizes.
+  bool useOriginalSubviewSize = false;
+  LinalgPromotionOptions &setUseOriginalSubviewSize(bool originalSize) {
+    useOriginalSubviewSize = originalSize;
+    return *this;
+  }
   /// Alignment of promoted buffer. If `std::nullopt` do not specify alignment.
   std::optional<unsigned> alignment;
   LinalgPromotionOptions &setAlignment(unsigned align) {
@@ -801,7 +803,8 @@ FailureOr<LinalgOp> specializeGenericOp(RewriterBase &rewriter,
                                         GenericOp genericOp);
 
 /// Create a new buffer using the `allocationFn` provided. The size of this
-/// buffer is the smallest constant bounding size along each dimension that
+/// buffer is either the original subview size when 'useOriginalSubviewSize' is
+/// set to true or the smallest constant bounding size along each dimension that
 /// can be computed for the size of the result of `subView`. Returns the
 /// allocated buffer as `fullLocalView` and the view that matches the size of
 /// the result of subview operation as `partialLocalView`.
@@ -811,6 +814,7 @@ struct PromotionInfo {
 };
 FailureOr<PromotionInfo>
 promoteSubviewAsNewBuffer(OpBuilder &b, Location loc, memref::SubViewOp subView,
+                          bool useOriginalSubviewSize,
                           const AllocBufferCallbackFn &allocationFn,
                           DataLayout &layout);
 
@@ -860,17 +864,23 @@ LogicalResult deallocateGPUPrivateMemory(OpBuilder &, Value /*buffer*/);
 /// to work (these are checked by the vectorizer itself).
 bool hasVectorizationImpl(Operation *);
 
-/// Emit a suitable vector form for an operation. If provided,
-/// `inputVectorSizes` are used to vectorize this operation. `inputVectorSizes`
-/// must match the rank of the iteration space of the operation and the sizes
-/// must be smaller or equal than their counterpart interation space sizes, if
-/// static. `inputVectorShapes` also allows the vectorization of operations with
-/// dynamic shapes.
-LogicalResult vectorize(RewriterBase &rewriter, Operation *op,
-                        ArrayRef<int64_t> inputVectorSizes = {},
-                        ArrayRef<bool> inputScalableVecDims = {},
-                        bool vectorizeNDExtract = false,
-                        bool flatten1DDepthwiseConv = false);
+/// Transformation information returned after vectorizing.
+struct VectorizationResult {
+  /// Results of the vectorization transform to replace the original operation.
+  SmallVector<Value> replacements;
+};
+/// Returns a `VectorizationResult` containing the results of the vectorized op,
+/// or failure if the transformation fails. If provided, `inputVectorSizes` are
+/// used to vectorize this operation. `inputVectorSizes` must match the rank of
+/// the iteration space of the operation and the input vector sizes must be
+/// greater than or equal to their counterpart iteration space sizes, if static.
+/// `inputVectorShapes` also allows the vectorization of operations with dynamic
+/// shapes.
+FailureOr<VectorizationResult>
+vectorize(RewriterBase &rewriter, Operation *op,
+          ArrayRef<int64_t> inputVectorSizes = {},
+          ArrayRef<bool> inputScalableVecDims = {},
+          bool vectorizeNDExtract = false, bool flatten1DDepthwiseConv = false);
 
 /// Emit a suitable vector form for a Copy op with fully static shape.
 LogicalResult vectorizeCopy(RewriterBase &builder, memref::CopyOp copyOp);
@@ -1426,8 +1436,8 @@ FailureOr<Operation *> transposeBatchMatmul(RewriterBase &rewriter,
 /// F(m x m, r x r). m is the dimension size of output and r is the dimension
 /// size of filter.
 FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
-                                      linalg::Conv2DNhwcFhwcOp op, int64_t m,
-                                      int64_t r);
+                                      linalg::Conv2DNhwcFhwcOp op,
+                                      WinogradConv2DFmr fmr);
 
 /// Rewrite linalg.winograd_filter_transform. The data layout of the filter is
 /// FHWC. The transformation matrix is 2-dimension. We need to extract H x W
@@ -1968,8 +1978,8 @@ void populateBlockPackMatmulPatterns(RewritePatternSet &patterns,
                                      const ControlBlockPackMatmulFn &controlFn);
 
 /// Patterns to apply Winograd Conv2D algorithm F(m x m, r x r).
-void populateWinogradConv2DPatterns(RewritePatternSet &patterns, int64_t m,
-                                    int64_t r);
+void populateWinogradConv2DPatterns(RewritePatternSet &patterns,
+                                    WinogradConv2DFmr fmr);
 
 /// Patterns to decompose Winograd operators.
 void populateDecomposeWinogradOpsPatterns(RewritePatternSet &patterns);
@@ -1983,10 +1993,15 @@ void populateDecomposeWinogradOpsPatterns(RewritePatternSet &patterns);
 /// convert to a `linalg.dot`.
 void populateContractionOpRankReducingPatterns(RewritePatternSet &patterns);
 
+/// Function type which is used to control folding operations like `tensor.pad`
+/// and `tensor.extract_slice` into linalg.pack/unpack ops.
+using ControlFoldIntoPackUnpackFn = std::function<bool(OpOperand *opOperand)>;
 /// Populates `patterns` with patterns that fold operations like `tensor.pad`
 /// and `tensor.extract_slice` into `tensor.pack` and `tensor.unpack` operations
 /// respectively.
-void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns);
+void populateFoldIntoPackAndUnpackPatterns(
+    RewritePatternSet &patterns,
+    const ControlFoldIntoPackUnpackFn &controlFn = nullptr);
 
 /// Populates `patterns` with patterns that fold operations like `linalg.pack`
 /// and `linalg.unpack` into `tensor.empty`.
diff --git a/mlir/include/mlir/Dialect/MLProgram/IR/MLProgramOps.td b/mlir/include/mlir/Dialect/MLProgram/IR/MLProgramOps.td
index fd213824b30c..095f6bee2c34 100644
--- a/mlir/include/mlir/Dialect/MLProgram/IR/MLProgramOps.td
+++ b/mlir/include/mlir/Dialect/MLProgram/IR/MLProgramOps.td
@@ -462,7 +462,7 @@ def MLProgram_OutputOp : MLProgram_Op<"output", [
   let arguments = (ins Variadic<AnyType>:$operands);
 
   let builders = [OpBuilder<(ins), [{
-    build($_builder, $_state, std::nullopt);
+    build($_builder, $_state, {});
   }]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
@@ -488,7 +488,7 @@ def MLProgram_ReturnOp : MLProgram_Op<"return", [
   let arguments = (ins Variadic<AnyType>:$operands);
 
   let builders = [OpBuilder<(ins), [{
-    build($_builder, $_state, std::nullopt);
+    build($_builder, $_state, {});
   }]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index b0fb5b078514..09bb3932ef29 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -2333,6 +2333,7 @@ def MemRef_ViewOp : MemRef_Op<"view", [
 
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td b/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
index a8d135caa74f..f3e40aaa2907 100644
--- a/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/Passes.td
@@ -182,6 +182,68 @@ def ResolveShapedTypeResultDimsPass : Pass<"resolve-shaped-type-result-dims"> {
   ];
 }
 
+def ReifyResultShapesPass : Pass<"reify-result-shapes"> {
+  let summary ="Reifies the results of `tensor::PadOp` and `tensor::ConcatOp`.";
+  let description = [{
+    This pass reifies the shapes of a subset of `ReifyRankedShapedTypeOpInterface`
+    ops with `tensor` results.
+    
+    The pass currently only supports result shape type reification for:
+      - tensor::PadOp
+      - tensor::ConcatOp
+    It addresses a representation gap where implicit op semantics are needed to
+    infer static result types from dynamic operands.
+    But it does so by using `ReifyRankedShapedTypeOpInterface` as the source of
+    truth rather than the op itself. As a consequence, this cannot generalize
+    today.
+
+    TODO: in the future, we should consider coupling this information with op
+    "transfer functions" (e.g. `IndexingMapOpInterface`) to provide a source of
+    truth that can work across result shape inference, canonicalization and op
+    verifiers.
+
+    The pass replaces the operations with their reified versions, when more
+    static information can be derived, and inserts casts when results shapes
+    are updated.
+
+    Example:
+    ```mlir
+    #map = affine_map<(d0) -> (-d0 + 256)>
+    func.func @func(%arg0: f32, %arg1: index, %arg2: tensor<64x?x64xf32>)
+        -> tensor<1x?x64xf32>
+    {
+      %0 = affine.apply #map(%arg1)
+      %extracted_slice = tensor.extract_slice %arg2[0, 0, 0] [1, %arg1, 64] [1, 1, 1]
+        : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+      %padded = tensor.pad %extracted_slice low[0, 0, 0] high[0, %0, 0] {
+      ^bb0(%arg3: index, %arg4: index, %arg5: index):
+        tensor.yield %arg0 : f32
+      } : tensor<1x?x64xf32> to tensor<1x?x64xf32>
+      return %padded : tensor<1x?x64xf32>
+    }
+
+    // mlir-opt --reify-result-shapes
+    #map = affine_map<()[s0] -> (-s0 + 256)>
+    func.func @func(%arg0: f32, %arg1: index, %arg2: tensor<64x?x64xf32>)
+        -> tensor<1x?x64xf32>
+    {
+      %0 = affine.apply #map()[%arg1]
+      %extracted_slice = tensor.extract_slice %arg2[0, 0, 0] [1, %arg1, 64] [1, 1, 1]
+        : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+      %padded = tensor.pad %extracted_slice low[0, 0, 0] high[0, %0, 0] {
+      ^bb0(%arg3: index, %arg4: index, %arg5: index):
+        tensor.yield %arg0 : f32
+      } : tensor<1x?x64xf32> to tensor<1x256x64xf32>
+      %cast = tensor.cast %padded : tensor<1x256x64xf32> to tensor<1x?x64xf32>
+      return %cast : tensor<1x?x64xf32>
+    }
+    ```
+  }];
+  let dependentDialects = [
+    "affine::AffineDialect", "memref::MemRefDialect", "tensor::TensorDialect"
+  ];
+}
+
 def ExpandStridedMetadataPass : Pass<"expand-strided-metadata"> {
   let summary = "Expand memref operations into easier to analyze constructs";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
index c2b8cb05be92..33e3d94f02b1 100644
--- a/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/MemRef/Transforms/Transforms.h
@@ -23,6 +23,7 @@ class RewritePatternSet;
 class RewriterBase;
 class Value;
 class ValueRange;
+class ReifyRankedShapedTypeOpInterface;
 
 namespace arith {
 class WideIntEmulationConverter;
@@ -208,7 +209,6 @@ FailureOr<Value> replaceWithIndependentOp(RewriterBase &rewriter,
 memref::AllocaOp allocToAlloca(
     RewriterBase &rewriter, memref::AllocOp alloc,
     function_ref<bool(memref::AllocOp, memref::DeallocOp)> filter = nullptr);
-
 } // namespace memref
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
index c4d512b60bc5..7213fde45c69 100644
--- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
+++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
@@ -44,7 +44,7 @@ class MeshSharding {
   ::mlir::FlatSymbolRefAttr mesh;
   SmallVector<MeshAxesAttr> split_axes;
   SmallVector<MeshAxis> partial_axes;
-  ReductionKind partial_type;
+  ReductionKind partial_type = ReductionKind::Sum;
   SmallVector<int64_t> static_halo_sizes;
   SmallVector<int64_t> static_sharded_dims_offsets;
   SmallVector<Value> dynamic_halo_sizes;
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 34312655115a..66378f116784 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -199,6 +199,41 @@ def OpenACC_DataClauseEnum : I64EnumAttr<"DataClause",
 def OpenACC_DataClauseAttr : EnumAttr<OpenACC_Dialect, OpenACC_DataClauseEnum,
                                       "data_clause">;
 
+// Data clause modifiers:
+// * readonly: Added in OpenACC 2.7 to copyin and cache.
+// * zero: Added in OpenACC 3.0 for create and copyout.
+// * always, alwaysin, alwaysout: Added in OpenACC 3.4 for
+//       copy, copyin, and copyout clauses.
+// * capture: Added in OpenACC 3.4 for copy, copyin, copyout and create clauses.
+def OpenACC_DataClauseModifierNone : I32BitEnumAttrCaseNone<"none">;
+// All of the modifiers below are bit flags - so the value noted is `1 << bit`.
+// Thus the `zero` modifier is `1 << 0` = 1, `readonly` is `1 << 1` = 2, etc.
+def OpenACC_DataClauseModifierZero : I32BitEnumAttrCaseBit<"zero", 0>;
+def OpenACC_DataClauseModifierReadonly : I32BitEnumAttrCaseBit<"readonly", 1>;
+def OpenACC_DataClauseModifierAlwaysIn : I32BitEnumAttrCaseBit<"alwaysin", 2>;
+def OpenACC_DataClauseModifierAlwaysOut : I32BitEnumAttrCaseBit<"alwaysout", 3>;
+def OpenACC_DataClauseModifierAlways : I32BitEnumAttrCaseGroup<"always",
+  [OpenACC_DataClauseModifierAlwaysIn, OpenACC_DataClauseModifierAlwaysOut]>;
+def OpenACC_DataClauseModifierCapture : I32BitEnumAttrCaseBit<"capture", 4>;
+
+def OpenACC_DataClauseModifierEnum : I32BitEnumAttr<
+    "DataClauseModifier",
+    "Captures data clause modifiers",
+    [
+      OpenACC_DataClauseModifierNone, OpenACC_DataClauseModifierZero,
+      OpenACC_DataClauseModifierReadonly, OpenACC_DataClauseModifierAlwaysIn,
+      OpenACC_DataClauseModifierAlwaysOut, OpenACC_DataClauseModifierAlways,
+      OpenACC_DataClauseModifierCapture]> {
+  let separator = ",";
+  let cppNamespace = "::mlir::acc";
+  let genSpecializedAttr = 0;
+  let printBitEnumPrimaryGroups = 1;
+}
+
+def OpenACC_DataClauseModifierAttr : EnumAttr<OpenACC_Dialect,
+                                      OpenACC_DataClauseModifierEnum,
+                                      "data_clause_modifier">;
+
 class OpenACC_Attr<string name, string attrMnemonic,
                    list<Trait> traits = [],
                    string baseCppClass = "::mlir::Attribute">
@@ -477,6 +512,8 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           DefaultValuedAttr<OpenACC_DataClauseAttr, clause>:$dataClause,
           DefaultValuedAttr<BoolAttr, "true">:$structured,
           DefaultValuedAttr<BoolAttr, "false">:$implicit,
+          DefaultValuedAttr<OpenACC_DataClauseModifierAttr,
+            "mlir::acc::DataClauseModifier::none">:$modifiers,
           OptionalAttr<StrAttr>:$name));
 
   let description = !strconcat(extraDescription, [{
@@ -506,6 +543,7 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
     counters (2.6.7).
     - `implicit`: Whether this is an implicitly generated operation, such as copies
     done to satisfy "Variables with Implicitly Determined Data Attributes" in 2.6.2.
+    - `modifiers`: Keeps track of the data clause modifiers (eg zero, readonly, etc)
     - `name`: Holds the name of variable as specified in user clause (including bounds).
 
     The async values attached to the data entry operation imply that the data
@@ -584,7 +622,8 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit), /*name=*/nullptr);
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
+          /*name=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$var,
                    "bool":$structured, "bool":$implicit,
@@ -601,9 +640,23 @@ class OpenACC_DataEntryOp<string mnemonic, string clause, string extraDescriptio
           /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit),
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
           /*name=*/$_builder.getStringAttr(name));
-      }]>];
+      }]>,
+    OpBuilder<(ins "::mlir::Type":$accVarType, "::mlir::Value":$var,
+                   "::mlir::Type":$varType, "::mlir::Value":$varPtrPtr,
+                   "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit, "::mlir::StringAttr":$name),
+      [{
+        build($_builder, $_state, accVarType, var, varType, varPtrPtr, bounds,
+          asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
+          structured, implicit, ::mlir::acc::DataClauseModifier::none, name);
+      }]>,
+    ];
 }
 
 //===----------------------------------------------------------------------===//
@@ -817,9 +870,7 @@ def OpenACC_CacheOp : OpenACC_DataEntryOp<"cache",
 
   let extraClassDeclaration = extraClassDeclarationBase # [{
     /// Check if this is a cache with readonly modifier.
-    bool isCacheReadonly() {
-      return getDataClause() == acc::DataClause::acc_cache_readonly;
-    }
+    bool isCacheReadonly();
   }];
 }
 
@@ -840,6 +891,8 @@ class OpenACC_DataExitOp<string mnemonic, string clause, string extraDescription
                        DefaultValuedAttr<OpenACC_DataClauseAttr,clause>:$dataClause,
                        DefaultValuedAttr<BoolAttr, "true">:$structured,
                        DefaultValuedAttr<BoolAttr, "false">:$implicit,
+                       DefaultValuedAttr<OpenACC_DataClauseModifierAttr,
+                         "mlir::acc::DataClauseModifier::none">:$modifiers,
                        OptionalAttr<StrAttr>:$name));
 
   let description = !strconcat(extraDescription, [{
@@ -861,6 +914,7 @@ class OpenACC_DataExitOp<string mnemonic, string clause, string extraDescription
     counters (2.6.7).
     - `implicit`: Whether this is an implicitly generated operation, such as copies
     done to satisfy "Variables with Implicitly Determined Data Attributes" in 2.6.2.
+    - `modifiers`: Keeps track of the data clause modifiers (eg zero, always, etc)
     - `name`: Holds the name of variable as specified in user clause (including bounds).
 
     The async values attached to the data exit operation imply that the data
@@ -944,7 +998,8 @@ class OpenACC_DataExitOpWithVarPtr<string mnemonic, string clause>
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit), /*name=*/nullptr);
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
+          /*name=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$accVar,
                    "::mlir::Value":$var,
@@ -961,9 +1016,22 @@ class OpenACC_DataExitOpWithVarPtr<string mnemonic, string clause>
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit),
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
           /*name=*/$_builder.getStringAttr(name));
-      }]>];
+      }]>,
+    OpBuilder<(ins "::mlir::Value":$accVar, "::mlir::Value":$var,
+                   "::mlir::Type":$varType, "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit, "::mlir::StringAttr":$name),
+      [{
+        build($_builder, $_state, accVar, var, varType, bounds,
+          asyncOperands, asyncOperandsDeviceType, asyncOnly, dataClause,
+          structured, implicit, ::mlir::acc::DataClauseModifier::none, name);
+      }]>,
+    ];
 
   code extraClassDeclarationDataExit = [{
     mlir::TypedValue<mlir::acc::PointerLikeType> getVarPtr() {
@@ -998,7 +1066,8 @@ class OpenACC_DataExitOpNoVarPtr<string mnemonic, string clause> :
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit), /*name=*/nullptr);
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
+          /*name=*/nullptr);
       }]>,
     OpBuilder<(ins "::mlir::Value":$accVar,
                    "bool":$structured, "bool":$implicit,
@@ -1009,9 +1078,20 @@ class OpenACC_DataExitOpNoVarPtr<string mnemonic, string clause> :
           bounds, /*asyncOperands=*/{}, /*asyncOperandsDeviceType=*/nullptr,
           /*asyncOnly=*/nullptr, /*dataClause=*/nullptr,
           /*structured=*/$_builder.getBoolAttr(structured),
-          /*implicit=*/$_builder.getBoolAttr(implicit),
+          /*implicit=*/$_builder.getBoolAttr(implicit), /*modifiers=*/nullptr,
           /*name=*/$_builder.getStringAttr(name));
-      }]>
+      }]>,
+    OpBuilder<(ins "::mlir::Value":$accVar, "::mlir::ValueRange":$bounds,
+                   "::mlir::ValueRange":$asyncOperands,
+                   "::mlir::ArrayAttr":$asyncOperandsDeviceType,
+                   "::mlir::ArrayAttr":$asyncOnly,
+                   "::mlir::acc::DataClause":$dataClause, "bool":$structured,
+                   "bool":$implicit, "::mlir::StringAttr":$name),
+      [{
+        build($_builder, $_state, accVar, bounds, asyncOperands,
+              asyncOperandsDeviceType, asyncOnly, dataClause, structured,
+              implicit, ::mlir::acc::DataClauseModifier::none, name);
+      }]>,
   ];
 
   code extraClassDeclarationDataExit = [{
@@ -1930,6 +2010,25 @@ def OpenACC_EnterDataOp : OpenACC_Op<"enter_data",
 
     /// The i-th data operand passed.
     Value getDataOperand(unsigned i);
+
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async'. DeviceType array is supplied even though it
+    /// should always be empty, so this can mirror other versions of this
+    /// function.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait'. DeviceType array is supplied
+    /// even though it should always be empty, so this can mirror other versions
+    /// of this function.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
@@ -1984,6 +2083,26 @@ def OpenACC_ExitDataOp : OpenACC_Op<"exit_data",
 
     /// The i-th data operand passed.
     Value getDataOperand(unsigned i);
+
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async'. DeviceType array is supplied even though it
+    /// should always be empty, so this can mirror other versions of this
+    /// function.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments). DeviceType array is supplied even though it should always be
+    /// empty, so this can mirror other versions of this function.
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait'. DeviceType array is supplied
+    /// even though it should always be empty, so this can mirror other versions
+    /// of this function.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
@@ -2652,18 +2771,15 @@ def OpenACC_RoutineOp : OpenACC_Op<"routine", [IsolatedFromAbove]> {
     must be modified as well.
   }];
 
-  let arguments = (ins SymbolNameAttr:$sym_name,
-                       SymbolNameAttr:$func_name,
-                       OptionalAttr<StrArrayAttr>:$bindName,
-                       OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
-                       OptionalAttr<DeviceTypeArrayAttr>:$worker,
-                       OptionalAttr<DeviceTypeArrayAttr>:$vector,
-                       OptionalAttr<DeviceTypeArrayAttr>:$seq,
-                       UnitAttr:$nohost,
-                       UnitAttr:$implicit,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gang,
-                       OptionalAttr<I64ArrayAttr>:$gangDim,
-                       OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
+  let arguments = (ins SymbolNameAttr:$sym_name, SymbolRefAttr:$func_name,
+      OptionalAttr<StrArrayAttr>:$bindName,
+      OptionalAttr<DeviceTypeArrayAttr>:$bindNameDeviceType,
+      OptionalAttr<DeviceTypeArrayAttr>:$worker,
+      OptionalAttr<DeviceTypeArrayAttr>:$vector,
+      OptionalAttr<DeviceTypeArrayAttr>:$seq, UnitAttr:$nohost,
+      UnitAttr:$implicit, OptionalAttr<DeviceTypeArrayAttr>:$gang,
+      OptionalAttr<I64ArrayAttr>:$gangDim,
+      OptionalAttr<DeviceTypeArrayAttr>:$gangDimDeviceType);
 
   let extraClassDeclaration = [{
     static StringRef getGangDimKeyword() { return "dim"; }
@@ -2909,6 +3025,21 @@ def OpenACC_UpdateOp : OpenACC_Op<"update",
     /// Return the wait devnum value clause for the given device_type if
     /// present.
     mlir::Value getWaitDevnum(mlir::acc::DeviceType deviceType);
+    /// Add an entry to the 'async-only' attribute (clause spelled without
+    /// arguments)for each of the additional device types (or a none if it is
+    /// empty).
+    void addAsyncOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add a value to the 'async' with the current list of device types.
+    void addAsyncOperand(MLIRContext *, mlir::Value,
+                         llvm::ArrayRef<DeviceType>);
+    /// Add an entry to the 'wait-only' attribute (clause spelled without
+    /// arguments)for each of the additional device types (or a none if it is
+    /// empty).
+    void addWaitOnly(MLIRContext *, llvm::ArrayRef<DeviceType>);
+    /// Add an array-like entry  to the 'wait' with the current list of device
+    /// types.
+    void addWaitOperands(MLIRContext *, bool hasDevnum, mlir::ValueRange,
+                         llvm::ArrayRef<DeviceType>);
   }];
 
   let assemblyFormat = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index f9a85626a3f1..faf820dcfdb2 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -15,14 +15,10 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 #define MLIR_DIALECT_OPENMP_OPENMPCLAUSEOPERANDS_H_
 
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "llvm/ADT/SmallVector.h"
 
-#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
-
-#define GET_ATTRDEF_CLASSES
-#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
-
 #include "mlir/Dialect/OpenMP/OpenMPClauseOps.h.inc"
 
 namespace mlir {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
index 248ac2eb72c6..ab11a6094e3e 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPDialect.h
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/OpenACCMPOpsInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
@@ -33,8 +34,6 @@
 
 #include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.h.inc"
 
-#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
-
 #define GET_OP_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOps.h.inc"
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
index 989ab1710c21..5ebf7c8090c1 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
@@ -13,7 +13,8 @@
 #ifndef MLIR_DIALECT_OPENMP_OPENMPINTERFACES_H_
 #define MLIR_DIALECT_OPENMP_OPENMPINTERFACES_H_
 
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
new file mode 100644
index 000000000000..6d196decfad0
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsAttributes.h
@@ -0,0 +1,18 @@
+//===- OpenMPOpsAttributes.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h"
+#include "mlir/IR/BuiltinAttributes.h"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSATTRIBUTES_H_
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
new file mode 100644
index 000000000000..edf316ea9ca8
--- /dev/null
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsEnums.h
@@ -0,0 +1,19 @@
+//===- OpenMPOpsEnums.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+#define MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
+
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.h.inc"
+
+#endif // MLIR_DIALECT_OPENMP_OPENMPOPSENUMS_H_
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index b51b61b3d2cb..8b14cef7437d 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -248,12 +248,10 @@ def ForOp : SCF_Op<"for",
   let regions = (region SizedRegion<1>:$region);
 
   let skipDefaultBuilders = 1;
-  let builders = [
-    OpBuilder<(ins "Value":$lowerBound, "Value":$upperBound, "Value":$step,
-      CArg<"ValueRange", "std::nullopt">:$initArgs,
+  let builders = [OpBuilder<(ins "Value":$lowerBound, "Value":$upperBound,
+      "Value":$step, CArg<"ValueRange", "{}">:$initArgs,
       CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
-           "nullptr">)>
-  ];
+           "nullptr">)>];
 
   let extraClassDeclaration = [{
     using BodyBuilderFn =
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h b/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
index 7b43aa43c751..3205da6e448f 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/TileUsingInterface.h
@@ -313,19 +313,23 @@ tileConsumerAndFuseProducersUsingSCF(RewriterBase &rewriter,
                                      TilingInterface consumer,
                                      const SCFTileAndFuseOptions &options);
 
-/// Fuse the consumer of the source of `candidateSliceOp` by computing the
-/// required slice of the consumer in-place.  Note that the method
-/// replaces the uses of `candidateSliceOp` with the tiled and fused consumer
-/// value but does not delete the slice operation.
+/// Fuse the consumer `candidateSlices` by computing the required slice of the
+/// consumer in-place. All the entries of `candidateSlices` are expected to map
+/// to the same consumer. The method returns an error if the consumer cannot be
+/// tiled in a manner that is consistent for all the passed slices. Note that
+/// the method replaces the uses of `candidateSlices` with the tiled and fused
+/// consumer value but does not delete the slice operations.
 struct SCFFuseConsumerOfSliceResult {
-  OpOperand *origConsumerOperand; // Original untiled consumer's operand.
-  OpOperand
-      *tiledAndFusedConsumerOperand; // Tiled and fused consumer's operand.
+  // Original untiled consumer operands.
+  SmallVector<OpOperand *> origConsumerOperands;
+  // Tiled and fused consumer operands.
+  SmallVector<OpOperand *> tiledAndFusedConsumerOperands;
   SmallVector<Operation *> tiledOps;
 };
 FailureOr<scf::SCFFuseConsumerOfSliceResult>
-tileAndFuseConsumerOfSlice(RewriterBase &rewriter, Operation *candidateSliceOp,
-                           MutableArrayRef<LoopLikeOpInterface> loops);
+tileAndFuseConsumerOfSlices(RewriterBase &rewriter,
+                            ArrayRef<Operation *> candidateSlices,
+                            MutableArrayRef<LoopLikeOpInterface> loops);
 
 /// Method to lower an `op` that implements the `TilingInterface` to
 /// loops/scalars.
diff --git a/mlir/include/mlir/Dialect/SMT/IR/SMTOps.td b/mlir/include/mlir/Dialect/SMT/IR/SMTOps.td
index 1872c00b74f1..3143ab7de1b1 100644
--- a/mlir/include/mlir/Dialect/SMT/IR/SMTOps.td
+++ b/mlir/include/mlir/Dialect/SMT/IR/SMTOps.td
@@ -227,7 +227,7 @@ def YieldOp : SMTOp<"yield", [
   let arguments = (ins Variadic<AnyType>:$values);
   let assemblyFormat = "($values^ `:` qualified(type($values)))? attr-dict";
   let builders = [OpBuilder<(ins), [{
-    build($_builder, $_state, std::nullopt);
+    build($_builder, $_state, {});
   }]>];
 }
 
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index d2ba76cdad90..d874817e6888 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -422,6 +422,8 @@ def SPV_NV_ray_tracing_motion_blur       : I32EnumAttrCase<"SPV_NV_ray_tracing_m
 
 def SPV_NVX_multiview_per_view_attributes : I32EnumAttrCase<"SPV_NVX_multiview_per_view_attributes", 5015>;
 
+def SPV_ARM_tensors                      : I32EnumAttrCase<"SPV_ARM_tensors", 6000>;
+
 def SPIRV_ExtensionAttr :
     SPIRV_I32EnumAttr<"Extension", "supported SPIR-V extensions", "ext", [
       SPV_KHR_16bit_storage, SPV_KHR_8bit_storage, SPV_KHR_device_group,
@@ -445,6 +447,7 @@ def SPIRV_ExtensionAttr :
       SPV_EXT_shader_atomic_float_add, SPV_EXT_shader_atomic_float_min_max,
       SPV_EXT_shader_image_int64, SPV_EXT_shader_atomic_float16_add,
       SPV_EXT_mesh_shader,
+      SPV_ARM_tensors,
       SPV_AMD_gpu_shader_half_float_fetch, SPV_AMD_shader_ballot,
       SPV_AMD_shader_explicit_vertex_parameter, SPV_AMD_shader_fragment_mask,
       SPV_AMD_shader_image_load_store_lod, SPV_AMD_texture_gather_bias_lod,
@@ -1311,6 +1314,24 @@ def SPIRV_C_GeometryStreams                             : I32EnumAttrCase<"Geome
 def SPIRV_C_MultiViewport                               : I32EnumAttrCase<"MultiViewport", 57> {
   list<I32EnumAttrCase> implies = [SPIRV_C_Geometry];
 }
+def SPIRV_C_TensorsARM                                  : I32EnumAttrCase<"TensorsARM", 4174> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_Int8];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
+def SPIRV_C_StorageTensorArrayDynamicIndexingEXT        : I32EnumAttrCase<"StorageTensorArrayDynamicIndexingEXT", 4175> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_TensorsARM, SPIRV_C_Shader];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
+def SPIRV_C_StorageTensorArrayNonUniformIndexingEXT     : I32EnumAttrCase<"StorageTensorArrayNonUniformIndexingEXT", 4176> {
+  list<I32EnumAttrCase> implies = [SPIRV_C_TensorsARM, SPIRV_C_ShaderNonUniform];
+  list<Availability> availability = [
+    Extension<[SPV_ARM_tensors]>
+  ];
+}
 def SPIRV_C_WorkgroupMemoryExplicitLayout8BitAccessKHR  : I32EnumAttrCase<"WorkgroupMemoryExplicitLayout8BitAccessKHR", 4429> {
   list<I32EnumAttrCase> implies = [SPIRV_C_WorkgroupMemoryExplicitLayoutKHR];
   list<Availability> availability = [
@@ -1523,6 +1544,8 @@ def SPIRV_CapabilityAttr :
       SPIRV_C_IntegerFunctions2INTEL, SPIRV_C_TessellationPointSize,
       SPIRV_C_GeometryPointSize, SPIRV_C_ImageCubeArray, SPIRV_C_ImageRect,
       SPIRV_C_GeometryStreams, SPIRV_C_MultiViewport,
+      SPIRV_C_TensorsARM, SPIRV_C_StorageTensorArrayDynamicIndexingEXT,
+      SPIRV_C_StorageTensorArrayNonUniformIndexingEXT,
       SPIRV_C_WorkgroupMemoryExplicitLayout8BitAccessKHR, SPIRV_C_VariablePointers,
       SPIRV_C_RayTraversalPrimitiveCullingKHR, SPIRV_C_SampleMaskOverrideCoverageNV,
       SPIRV_C_GeometryShaderPassthroughNV, SPIRV_C_PerViewAttributesNV,
@@ -4179,7 +4202,7 @@ def SPIRV_IsPtrType : CPred<"::llvm::isa<::mlir::spirv::PointerType>($_self)">;
 def SPIRV_IsRTArrayType : CPred<"::llvm::isa<::mlir::spirv::RuntimeArrayType>($_self)">;
 def SPIRV_IsSampledImageType : CPred<"::llvm::isa<::mlir::spirv::SampledImageType>($_self)">;
 def SPIRV_IsStructType : CPred<"::llvm::isa<::mlir::spirv::StructType>($_self)">;
-
+def SPIRV_IsTensorArmType : CPred<"::llvm::isa<::mlir::spirv::TensorArmType>($_self)">;
 
 // See https://www.khronos.org/registry/spir-v/specs/unified1/SPIRV.html#_types
 // for the definition of the following types and type categories.
@@ -4217,6 +4240,8 @@ def SPIRV_AnyStruct : DialectType<SPIRV_Dialect, SPIRV_IsStructType,
                                 "any SPIR-V struct type">;
 def SPIRV_AnySampledImage : DialectType<SPIRV_Dialect, SPIRV_IsSampledImageType,
                                 "any SPIR-V sampled image type">;
+def SPIRV_AnyTensorArm : DialectType<SPIRV_Dialect, SPIRV_IsTensorArmType,
+                                 "any SPIR-V tensorArm type">;
 
 def SPIRV_Numerical : AnyTypeOf<[SPIRV_Integer, SPIRV_AnyFloat]>;
 def SPIRV_Scalar : AnyTypeOf<[SPIRV_Numerical, SPIRV_Bool]>;
@@ -4228,7 +4253,7 @@ def SPIRV_Type : AnyTypeOf<[
     SPIRV_Void, SPIRV_Bool, SPIRV_Integer, SPIRV_AnyFloat, SPIRV_Vector,
     SPIRV_AnyPtr, SPIRV_AnyArray, SPIRV_AnyRTArray, SPIRV_AnyStruct,
     SPIRV_AnyCooperativeMatrix, SPIRV_AnyMatrix, SPIRV_AnySampledImage,
-    SPIRV_AnyImage
+    SPIRV_AnyImage, SPIRV_AnyTensorArm
   ]>;
 
 def SPIRV_SignedInt : SignedIntOfWidths<[8, 16, 32, 64]>;
@@ -4525,6 +4550,7 @@ def SPIRV_OC_OpGroupNonUniformBitwiseXor      : I32EnumAttrCase<"OpGroupNonUnifo
 def SPIRV_OC_OpGroupNonUniformLogicalAnd      : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
 def SPIRV_OC_OpGroupNonUniformLogicalOr       : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
 def SPIRV_OC_OpGroupNonUniformLogicalXor      : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
+def SPIRV_OC_OpTypeTensorARM                  : I32EnumAttrCase<"OpTypeTensorARM", 4163>;
 def SPIRV_OC_OpSubgroupBallotKHR              : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
 def SPIRV_OC_OpGroupNonUniformRotateKHR       : I32EnumAttrCase<"OpGroupNonUniformRotateKHR", 4431>;
 def SPIRV_OC_OpSDot                           : I32EnumAttrCase<"OpSDot", 4450>;
@@ -4638,7 +4664,9 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpGroupNonUniformFMax, SPIRV_OC_OpGroupNonUniformBitwiseAnd,
       SPIRV_OC_OpGroupNonUniformBitwiseOr, SPIRV_OC_OpGroupNonUniformBitwiseXor,
       SPIRV_OC_OpGroupNonUniformLogicalAnd, SPIRV_OC_OpGroupNonUniformLogicalOr,
-      SPIRV_OC_OpGroupNonUniformLogicalXor, SPIRV_OC_OpSubgroupBallotKHR,
+      SPIRV_OC_OpGroupNonUniformLogicalXor,
+      SPIRV_OC_OpTypeTensorARM,
+      SPIRV_OC_OpSubgroupBallotKHR,
       SPIRV_OC_OpGroupNonUniformRotateKHR, SPIRV_OC_OpSDot, SPIRV_OC_OpUDot,
       SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat, SPIRV_OC_OpUDotAccSat,
       SPIRV_OC_OpSUDotAccSat, SPIRV_OC_OpTypeCooperativeMatrixKHR,
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
index 46732ba19afe..fd75532ae3d7 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVCooperativeMatrixOps.td
@@ -112,7 +112,7 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
   }];
 
   let assemblyFormat = [{
-    $pointer `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? attr-dict `:`
+    $pointer `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? ( `,` $alignment^ )? attr-dict `:`
       type(operands) `->` type($result)
   }];
 
@@ -123,11 +123,13 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
     Capability<[SPIRV_C_CooperativeMatrixKHR]>
   ];
 
+  // TODO: Add scope operand for MakePointer*. See #145485.
   let arguments = (ins
     SPIRV_AnyPtr:$pointer,
     SPIRV_KHR_CooperativeMatrixLayoutAttr:$matrix_layout,
     SPIRV_Integer:$stride,
-    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand
+    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand,
+    OptionalAttr<I32Attr>:$alignment
   );
 
   let results = (outs
@@ -139,7 +141,7 @@ def SPIRV_KHRCooperativeMatrixLoadOp : SPIRV_KhrVendorOp<"CooperativeMatrixLoad"
                    "spirv::ConstantOp":$stride,
                    "spirv::CooperativeMatrixLayoutKHR":$layout), [{
       build($_builder, $_state, result, pointer, layout, stride,
-            spirv::MemoryAccessAttr{});
+            spirv::MemoryAccessAttr{}, IntegerAttr{});
     }]>
   ];
 }
@@ -194,7 +196,7 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
   }];
 
   let assemblyFormat = [{
-    $pointer `,` $object `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? attr-dict `:`
+    $pointer `,` $object `,` $stride `,` $matrix_layout ( `,` $memory_operand^ )? ( `,` $alignment^ )? attr-dict `:`
       type(operands)
   }];
 
@@ -205,12 +207,14 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
     Capability<[SPIRV_C_CooperativeMatrixKHR]>
   ];
 
+  // TODO: Add scope operand for MakePointer*. See #145485.
   let arguments = (ins
     SPIRV_AnyPtr:$pointer,
     SPIRV_AnyCooperativeMatrix:$object,
     SPIRV_KHR_CooperativeMatrixLayoutAttr:$matrix_layout,
     SPIRV_Integer:$stride,
-    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand
+    OptionalAttr<SPIRV_MemoryAccessAttr>:$memory_operand,
+    OptionalAttr<I32Attr>:$alignment
   );
 
   let results = (outs);
@@ -220,7 +224,7 @@ def SPIRV_KHRCooperativeMatrixStoreOp : SPIRV_KhrVendorOp<"CooperativeMatrixStor
                    "spirv::ConstantOp":$stride,
                    "spirv::CooperativeMatrixLayoutKHR":$layout), [{
       build($_builder, $_state, pointer, object, layout, stride,
-            spirv::MemoryAccessAttr{});
+            spirv::MemoryAccessAttr{}, IntegerAttr{});
     }]>
   ];
 }
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
index 787535d0a6bd..212cba61d396 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVTypes.h
@@ -29,6 +29,7 @@ namespace spirv {
 namespace detail {
 struct ArrayTypeStorage;
 struct CooperativeMatrixTypeStorage;
+struct TensorArmTypeStorage;
 struct ImageTypeStorage;
 struct MatrixTypeStorage;
 struct PointerTypeStorage;
@@ -96,7 +97,8 @@ class ScalarType : public SPIRVType {
   std::optional<int64_t> getSizeInBytes();
 };
 
-// SPIR-V composite type: VectorType, SPIR-V ArrayType, or SPIR-V StructType.
+// SPIR-V composite type: VectorType, SPIR-V ArrayType, SPIR-V
+// StructType, or SPIR-V TensorArmType.
 class CompositeType : public SPIRVType {
 public:
   using SPIRVType::SPIRVType;
@@ -477,6 +479,46 @@ class MatrixType : public Type::TypeBase<MatrixType, CompositeType,
                        std::optional<StorageClass> storage = std::nullopt);
 };
 
+/// SPIR-V TensorARM Type
+class TensorArmType
+    : public Type::TypeBase<TensorArmType, CompositeType,
+                            detail::TensorArmTypeStorage, ShapedType::Trait> {
+public:
+  using Base::Base;
+
+  using ShapedTypeTraits = ShapedType::Trait<TensorArmType>;
+  using ShapedTypeTraits::getDimSize;
+  using ShapedTypeTraits::getDynamicDimIndex;
+  using ShapedTypeTraits::getElementTypeBitWidth;
+  using ShapedTypeTraits::getNumDynamicDims;
+  using ShapedTypeTraits::getNumElements;
+  using ShapedTypeTraits::getRank;
+  using ShapedTypeTraits::hasStaticShape;
+  using ShapedTypeTraits::isDynamicDim;
+
+  static constexpr StringLiteral name = "spirv.arm.tensor";
+
+  // TensorArm supports minimum rank of 1, hence an empty shape here means
+  // unranked.
+  static TensorArmType get(ArrayRef<int64_t> shape, Type elementType);
+  TensorArmType cloneWith(std::optional<ArrayRef<int64_t>> shape,
+                          Type elementType) const;
+
+  static LogicalResult
+  verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
+                   ArrayRef<int64_t> shape, Type elementType);
+
+  Type getElementType() const;
+  ArrayRef<int64_t> getShape() const;
+  bool hasRank() const { return !getShape().empty(); }
+  operator ShapedType() const { return llvm::cast<ShapedType>(*this); }
+
+  void getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
+                     std::optional<StorageClass> storage = std::nullopt);
+  void getCapabilities(SPIRVType::CapabilityArrayRefVector &capabilities,
+                       std::optional<StorageClass> storage = std::nullopt);
+};
+
 } // namespace spirv
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 8bccba426ab1..cbf1223298f9 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -678,9 +678,7 @@ def Shape_YieldOp : Shape_Op<"yield",
 
   let arguments = (ins Variadic<AnyType>:$operands);
 
-  let builders = [OpBuilder<(ins),
-    [{ build($_builder, $_state, std::nullopt); }]>
-  ];
+  let builders = [OpBuilder<(ins), [{ build($_builder, $_state, {}); }]>];
 
   let assemblyFormat = "attr-dict ($operands^ `:` type($operands))?";
   let hasVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index 35d0b1662841..7d396e5c64c2 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -1256,6 +1256,7 @@ def Tensor_CollapseShapeOp : Tensor_ReassociativeReshapeOp<"collapse_shape"> {
 
 def Tensor_PadOp : Tensor_Op<"pad", [
     DeclareOpInterfaceMethods<OpAsmOpInterface, ["getAsmResultNames"]>,
+    DeclareOpInterfaceMethods<ReifyRankedShapedTypeOpInterface>,
     AttrSizedOperandSegments,
     Pure,
     SingleBlockImplicitTerminator<"mlir::tensor::YieldOp">]> {
@@ -1770,8 +1771,7 @@ def Tensor_SplatOp : Tensor_Op<"splat", [
   ]> {
   let summary = "tensor splat or broadcast operation";
   let description = [{
-    Broadcast the operand to all elements of the result tensor. The operand is
-    required to be of integer/index/float type.
+    Broadcast the operand to all elements of the result tensor.
 
     An additional argument of type `index` must be provided for each dynamic
     dimension present in the result type.
@@ -1794,8 +1794,7 @@ def Tensor_SplatOp : Tensor_Op<"splat", [
     ```
   }];
 
-  let arguments = (ins AnyTypeOf<[AnySignlessInteger, Index, AnyFloat],
-                                 "integer/index/float type">:$input,
+  let arguments = (ins AnyType:$input,
                        Variadic<Index>:$dynamicSizes);
   let results = (outs AnyRankedTensor:$aggregate);
 
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
index 18981337742e..87deef9ca746 100644
--- a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
@@ -31,12 +31,16 @@ namespace tensor {
 FailureOr<TilingResult> replaceExtractSliceWithTiledProducer(
     OpBuilder &builder, tensor::ExtractSliceOp sliceOp, OpResult producerOp);
 
-/// Method to swap an `tensor.insert_slice` with its consumer when the
-/// consumer implements the `TilingInterface`.
+/// Method to swap `tensor.insert_slice`s with their consumers when the
+/// consumer implements the `TilingInterface`. The size of `sliceOps` and
+/// `consumerOperands` is expected to be the same. Every entry in
+/// `consumerOperands` represents a use of the the corresponding
+/// entry in `sliceOps` in the consumer. All entries of `consumerOperands` is
+/// expected to be uses in the same consumer.
 FailureOr<TilingResult>
-replaceInsertSliceWithTiledConsumer(OpBuilder &builder,
-                                    OffsetSizeAndStrideOpInterface sliceOp,
-                                    OpOperand &consumerOp);
+replaceInsertSlicesWithTiledConsumer(OpBuilder &builder,
+                                     ArrayRef<tensor::InsertSliceOp> sliceOps,
+                                     ArrayRef<OpOperand *> consumerOperands);
 
 //===----------------------------------------------------------------------===//
 // Populate functions.
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h b/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
index cdbebd581d9f..9ee5079559d2 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TargetEnv.h
@@ -43,32 +43,22 @@ class TargetEnv {
   bool allows(Profile prof) const { return enabledProfiles.count(prof) != 0; }
 
   bool allowsAnyOf(ArrayRef<Profile> profs) const {
-    const auto *chosen = llvm::find_if(
-        profs, [this](tosa::Profile prof) { return allows(prof); });
-    return chosen != profs.end() ? true : false;
+    return llvm::any_of(profs, [&](Profile prof) { return allows(prof); });
   }
 
   bool allowsAllOf(ArrayRef<Profile> profs) const {
-    bool is_allowed = true;
-    llvm::for_each(profs,
-                   [&](tosa::Profile prof) { is_allowed &= allows(prof); });
-    return is_allowed;
+    return llvm::all_of(profs, [&](Profile prof) { return allows(prof); });
   }
 
   // Returns true if the given extension is allowed.
   bool allows(Extension ext) const { return enabledExtensions.count(ext) != 0; }
 
   bool allowsAnyOf(ArrayRef<Extension> exts) const {
-    const auto *chosen = llvm::find_if(
-        exts, [this](tosa::Extension ext) { return allows(ext); });
-    return chosen != exts.end() ? true : false;
+    return llvm::any_of(exts, [&](Extension ext) { return allows(ext); });
   }
 
   bool allowsAllOf(ArrayRef<Extension> exts) const {
-    bool is_allowed = true;
-    llvm::for_each(exts,
-                   [&](tosa::Extension ext) { is_allowed &= allows(ext); });
-    return is_allowed;
+    return llvm::all_of(exts, [&](Extension ext) { return allows(ext); });
   }
 
 private:
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index f93dd901535c..20889558be31 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -1490,9 +1490,9 @@ def Tosa_SelectOp : Tosa_ElementwiseOp<"select"> {
   }];
 
   let arguments = (ins
-    Tosa_I1Tensor:$input1,
-    Tosa_Tensor:$input2,
-    Tosa_Tensor:$input3
+    Tosa_I1Tensor:$input1,  // pred
+    Tosa_Tensor:$input2,    // on true
+    Tosa_Tensor:$input3     // on false
   );
 
   let results = (outs
@@ -1512,6 +1512,13 @@ def Tosa_SelectOp : Tosa_ElementwiseOp<"select"> {
     operands attr-dict `:` `(` type($input1) `,` type($input2) `,` type($input3)
     `)` `->` type($output)
   }];
+
+  let extraClassDeclaration = [{
+    // Custom getters for readability
+    ::mlir::TypedValue<::mlir::TensorType> getPred() { return getInput1(); }
+    ::mlir::TypedValue<::mlir::TensorType> getOnTrue() { return getInput2(); }
+    ::mlir::TypedValue<::mlir::TensorType> getOnFalse() { return getInput3(); }
+  }];
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
index d005a4cc6859..b96682843538 100644
--- a/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Tosa/Transforms/Passes.td
@@ -127,4 +127,18 @@ def TosaReduceTransposes : Pass<"tosa-reduce-transposes", "func::FuncOp"> {
   }];
 }
 
+def TosaConvertIntegerTypeToSignless : Pass<"tosa-convert-integer-type-to-signless", "func::FuncOp"> {
+  let summary = "Convert integer types to signless";
+  let description = [{
+    This pass converts signed or unsigned integer types to signless. It
+    currently does this greedily for all operators and can also change the
+    signature of the function. Should the signature of the entrypoint
+    function change, it will be the responsibility of the user to carry
+    signedness information of the inputs and outputs independently.
+
+    This can be a useful transformation for conversion to other formats
+    that require strict adherence to the TOSA specification.
+  }];
+}
+
 #endif // MLIR_DIALECT_TOSA_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td
index 0275f241fda3..4a6898e36d34 100644
--- a/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td
+++ b/mlir/include/mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td
@@ -20,7 +20,7 @@ include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.td"
 include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.td"
 include "mlir/Dialect/Transform/IR/TransformDialect.td"
 
-def DebugEmitRemarkAtOp : TransformDialectOp<"debug.emit_remark_at",
+def EmitRemarkAtOp : TransformDialectOp<"debug.emit_remark_at",
   [MatchOpInterface,
    DeclareOpInterfaceMethods<TransformOpInterface>,
    MemoryEffectsOpInterface, NavigationTransformOpTrait]> {
@@ -39,7 +39,7 @@ def DebugEmitRemarkAtOp : TransformDialectOp<"debug.emit_remark_at",
   let assemblyFormat = "$at `,` $message attr-dict `:` type($at)";
 }
 
-def DebugEmitParamAsRemarkOp 
+def EmitParamAsRemarkOp
   : TransformDialectOp<"debug.emit_param_as_remark",
     [MatchOpInterface,
      DeclareOpInterfaceMethods<TransformOpInterface>,
diff --git a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
index 61c2a50e514c..704e39e90884 100644
--- a/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/ReshapeOpsUtils.h
@@ -14,6 +14,7 @@
 #ifndef MLIR_DIALECT_UTILS_RESHAPEOPSUTILS_H
 #define MLIR_DIALECT_UTILS_RESHAPEOPSUTILS_H
 
+#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
@@ -305,8 +306,43 @@ struct ComposeCollapseOfExpandOp : public OpRewritePattern<CollapseOpTy> {
       rewriter.replaceOpWithNewOp<CollapseOpTy>(
           collapseOp, resultType, expandOp.getSrc(), composedReassociation);
     } else if (srcRank < resultRank) {
+      // Compute the dynamic output shape for the new expand_shape op.
+      Location loc = collapseOp.getLoc();
+      SmallVector<OpFoldResult> origOutputShape =
+          expandOp.getMixedOutputShape();
+      SmallVector<OpFoldResult> newOutputShape;
+      for (const ReassociationIndices &indices :
+           collapseOp.getReassociationIndices()) {
+        int64_t numStaticElems = 1;
+        SmallVector<Value> dynamicSizes;
+        for (int64_t idx : indices) {
+          OpFoldResult size = origOutputShape[idx];
+          if (std::optional<int64_t> maybeCst = getConstantIntValue(size)) {
+            numStaticElems *= maybeCst.value();
+            continue;
+          }
+          dynamicSizes.push_back(cast<Value>(size));
+        }
+        if (dynamicSizes.empty()) {
+          newOutputShape.push_back(rewriter.getIndexAttr(numStaticElems));
+          continue;
+        }
+
+        // There is at least one dynamic size, so we can initialize `result` to
+        // the first dynamic size.
+        Value result = dynamicSizes[0];
+        for (Value v : llvm::drop_begin(dynamicSizes))
+          result = rewriter.create<arith::MulIOp>(loc, result, v);
+        if (numStaticElems != 1) {
+          result = rewriter.create<arith::MulIOp>(
+              loc, result,
+              rewriter.create<arith::ConstantIndexOp>(loc, numStaticElems));
+        }
+        newOutputShape.push_back(result);
+      }
       rewriter.replaceOpWithNewOp<ExpandOpTy>(
-          collapseOp, resultType, expandOp.getSrc(), composedReassociation);
+          collapseOp, resultType, expandOp.getSrc(), composedReassociation,
+          newOutputShape);
     } else {
       // Collapses/expansions that do not change the rank are not allowed. Use
       // a cast instead.
diff --git a/mlir/include/mlir/Dialect/Vector/IR/Vector.td b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
index 1922cc63ef35..5125ae7c1371 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/Vector.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/Vector.td
@@ -21,7 +21,10 @@ def Vector_Dialect : Dialect {
 
   let useDefaultAttributePrinterParser = 1;
   let hasConstantMaterializer = 1;
-  let dependentDialects = ["arith::ArithDialect"];
+  let dependentDialects = [
+    "arith::ArithDialect",
+    "ub::UBDialect"
+  ];
 }
 
 // Base class for Vector dialect ops.
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 02e62930a742..ec2c87ca1cf4 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -703,8 +703,9 @@ def Vector_ExtractOp :
      InferTypeOpAdaptorWithIsCompatible]> {
   let summary = "extract operation";
   let description = [{
-    Takes an n-D vector and a k-D position and extracts the (n-k)-D vector at
-    the proper position. Degenerates to an element type if n-k is zero.
+    Extracts an (n − k)-D result sub-vector from an n-D source vector at a
+    specified k-D position. When n = k, the result degenerates to a scalar
+    element.
 
     Static and dynamic indices must be greater or equal to zero and less than
     the size of the corresponding dimension. The result is undefined if any
@@ -716,7 +717,6 @@ def Vector_ExtractOp :
     ```mlir
     %1 = vector.extract %0[3]: vector<8x16xf32> from vector<4x8x16xf32>
     %2 = vector.extract %0[2, 1, 3]: f32 from vector<4x8x16xf32>
-    %3 = vector.extract %1[]: vector<f32> from vector<f32>
     %4 = vector.extract %0[%a, %b, %c]: f32 from vector<4x8x16xf32>
     %5 = vector.extract %0[2, %b]: vector<16xf32> from vector<4x8x16xf32>
     %6 = vector.extract %10[-1, %c]: f32 from vector<4x16xf32>
@@ -949,9 +949,9 @@ def Vector_InsertOp :
      AllTypesMatch<["dest", "result"]>]> {
   let summary = "insert operation";
   let description = [{
-    Takes an n-D source vector, an (n+k)-D destination vector and a k-D position
-    and inserts the n-D source into the (n+k)-D destination at the proper
-    position. Degenerates to a scalar or a 0-d vector source type when n = 0.
+    Inserts an (n - k)-D sub-vector (value-to-store) into an n-D destination
+    vector at a specified k-D position. When n = 0, value-to-store degenerates
+    to a scalar element inserted into the n-D destination vector.
 
     Static and dynamic indices must be greater or equal to zero and less than
     the size of the corresponding dimension. The result is undefined if any
@@ -963,8 +963,7 @@ def Vector_InsertOp :
     ```mlir
     %2 = vector.insert %0, %1[3] : vector<8x16xf32> into vector<4x8x16xf32>
     %5 = vector.insert %3, %4[2, 1, 3] : f32 into vector<4x8x16xf32>
-    %8 = vector.insert %6, %7[] : f32 into vector<f32>
-    %11 = vector.insert %9, %10[%a, %b, %c] : vector<f32> into vector<4x8x16xf32>
+    %11 = vector.insert %9, %10[%a, %b, %c] : f32 into vector<4x8x16xf32>
     %12 = vector.insert %4, %10[2, %b] : vector<16xf32> into vector<4x8x16xf32>
     %13 = vector.insert %20, %1[-1, %c] : f32 into vector<4x16xf32>
     ```
@@ -1544,30 +1543,29 @@ def Vector_TransferReadOp :
   }];
 
   let builders = [
-    /// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
+    /// 1. Builder that sets padding to `padding` or poison if not provided and
+    /// an empty mask (variant with attrs).
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
+                   "std::optional<Value>":$padding,
                    "AffineMapAttr":$permutationMapAttr,
                    "ArrayAttr":$inBoundsAttr)>,
-    /// 2. Builder that sets padding to zero and an empty mask (variant without attrs).
+    /// 2. Builder that sets padding to `padding` or poison if not provided and
+    /// an empty mask (variant without attrs).
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
+                   "std::optional<Value>":$padding,
                    "AffineMap":$permutationMap,
                    CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
-    /// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
+    /// 3. Builder that sets padding to `padding` or poison if not provided and
+    /// permutation map to 'getMinorIdentityMap'.
     OpBuilder<(ins "VectorType":$vectorType,
                    "Value":$source,
                    "ValueRange":$indices,
-                   "Value":$padding,
-                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
-    /// 4. Builder that sets padding to zero and permutation map to
-    /// 'getMinorIdentityMap'.
-    OpBuilder<(ins "VectorType":$vectorType,
-                   "Value":$source,
-                   "ValueRange":$indices,
-                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>,
+                   "std::optional<Value>":$padding,
+                   CArg<"std::optional<ArrayRef<bool>>", "::std::nullopt">:$inBounds)>
   ];
 
   let extraClassDeclaration = [{
@@ -2494,6 +2492,7 @@ def Vector_ConstantMaskOp :
 
   let assemblyFormat = "$mask_dim_sizes attr-dict `:` type(results)";
   let hasVerifier = 1;
+  let hasFolder = 1;
 }
 
 def Vector_CreateMaskOp :
@@ -2920,8 +2919,8 @@ def Vector_SplatOp : Vector_Op<"splat", [
   ]> {
   let summary = "vector splat or broadcast operation";
   let description = [{
-    Broadcast the operand to all elements of the result vector. The operand is
-    required to be of integer/index/float type.
+    Broadcast the operand to all elements of the result vector. The type of the
+    operand must match the element type of the vector type.
 
     Example:
 
@@ -2931,8 +2930,7 @@ def Vector_SplatOp : Vector_Op<"splat", [
     ```
   }];
 
-  let arguments = (ins AnyTypeOf<[AnySignlessInteger, Index, AnyFloat],
-                                 "integer/index/float type">:$input);
+  let arguments = (ins AnyType:$input);
   let results = (outs AnyVectorOfAnyRank:$aggregate);
 
   let builders = [
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index e6c7efc47593..daab65ec893b 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -80,9 +80,6 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
         information e.g., memref<?x?xf16>, the strides information has to be explicitly
         passed via the "strides" and "const_strides" argument.
 
-    In SIMT mode, tensor descriptor is augmented with `LayoutAttr` which describes the
-    mapping of the tensor descriptor to the work items.
-
     Example 1 (suppose the tensor shape inferred by the compiler is 8x16):
     ```mlir
     %0 = memref.alloc() : memref<1024x1024xf32>
@@ -106,15 +103,6 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
     %c1 = arith.constant 1 : index
     %1 = xegpu.create_nd_tdesc %0[%c0, %c0], [%h, %w], [%w, %c1]: ui64 -> TensorDesc<8x16xf32>
     ```
-
-    Example 4 (SIMT mode):
-    ```mlir
-    %0 = memref.alloc() : memref<1024x1024xf32>
-    %c0 = arith.constant 0 : index
-    %c1 = arith.constant 8 : index
-    %1 = xegpu.create_nd_tdesc %0[%c0, %c0] : memref<1024x1024xf32>
-          -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-    ```
   }];
 
   let arguments = (ins
@@ -301,9 +289,7 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [
     fp32 or fp64. It implies that vnni and transpose cannot exit at the
     same time.
 
-    In SIMT mode, LoadNdOp expects the tensor descriptor to be augmented with `LayoutAttr`
-    which describes the mapping of the tensor to the work items. In this case, result
-    vector represents the data to be loaded by each work-item.
+    In SIMT mode, result vector represents the data to be loaded by each work-item.
 
     Example 1:
     ```mlir
@@ -317,8 +303,7 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [
     ```mlir
       xegpu.load_nd %1 {l1_hint = #xegpu.cache_hint<cached>,
                         l2_hint = #xegpu.cache_hint<uncached>}>
-        : !xegpu.tensor_desc<8x16xf32,
-          #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x1xf32>
+        : !xegpu.tensor_desc<8x16xf32> -> vector<8xf32>
     ```
 
 
@@ -359,9 +344,7 @@ def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [
     of cache, L1, L2 and L3. If hardware does not have a correspoding cache,
     Corresponding cache hint attribute will be masked.
 
-    In SIMT mode, StoreNdOp expects the tensor descriptor to be augmented with `LayoutAttr`
-    which describes the mapping of the tensor to the work items. In this case, input
-    vector represents the data to be stored by each work-item.
+    In SIMT mode, the input vector represents the data to be stored by each work-item.
 
     Example 1:
     ```mlir
@@ -375,8 +358,7 @@ def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [
       xegpu.store_nd %3, %2 {l1_hint = #xegpu.cache_hint<uncached>,
                              l2_hint = #xegpu.cache_hint<write_back>,
                              l3_hint = #xegpu.cache_hint<write_through>}
-                             : vector<8x1xf16>, !xegpu.tensor_desc<8x16xf16,
-                               #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+                             : vector<8xf16>, !xegpu.tensor_desc<8x16xf16>
     ```
 
 
@@ -410,15 +392,10 @@ def XeGPU_UpdateNdOffsetOp : XeGPU_Op<"update_nd_offset",
     The offsets are relative offset to the current position in the number
     of elements. It will result in a same type TensorDesc as the input.
 
-  Example 1:
+  Example:
   ```
     %2 = xegpu.update_nd_offset %1, [0, 16]: !xegpu.tensor_desc<8x16xf32>
   ```
-  Example 2 (SIMT mode):
-  ```
-    %2 = xegpu.update_nd_offset %1, [0, 16]:
-      !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  ```
   }];
 
   let arguments = (ins
@@ -476,11 +453,6 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     match the dimension of offsets. It may also has a second dimension corresponding to
     the chunk_size if the chunk size is larger than 1.
 
-    In SIMT mode, similar to `create_nd_tdesc` the resulting tensor descriptor is augmented
-    with `LayoutAttr` which describes the mapping of the tensor descriptor to the work items.
-    In this case, the first dimension of the tensor descriptor represents the work-items, and
-    the second dimension represents the chunk size.
-
     Example 1: It assumes subgroup size is 4, and accesses a[0], a[16], a[32], a[64]
     ```mlir
     %a = memref.alloc() : memref<1024xf32>
@@ -505,15 +477,6 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     %1 = xegpu.create_tdesc %0, %off : memref<1024xf32>, vector<4xindex>
           -> TensorDesc<4x8xf32, #xegpu.scattered_tdesc_attr<chunk_size = 8>>
     ```
-
-    Example 4: SIMT mode
-    ```mlir
-    %0 = memref.alloc() : memref<1024xf32>
-    %off = arith.constant dense<[0, 16, 32, 64]> : vector<4xindex>
-    %1 = xegpu.create_tdesc %0, %off : memref<1024xf32>, vector<4xindex>
-          -> TensorDesc<4x8xf32, #xegpu.scattered_tdesc_attr<chunk_size = 8>,
-          #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>
-    ```
   }];
 
   let arguments = (ins XeGPU_BaseAddrType: $source,
@@ -609,54 +572,44 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [
   let description = [{ It (aka. load) load data per each work-item. The output
     describes the data being loaded at the subgroup level, so its size is
     consistent with the number of work-items in a subgroup. When the chunk size
-    is larger than 2, the output vector is a 2D vector, with dim-1 correspoding
-    to work-items, and dim-0 corresponding to the chunk size loaded by each work-item.
-    Specially, there is a transpose effect on the result (as compared to the TensorDesc)
-    due to the hardware implementation. Therefore, a transpose attribute is introduced
-    on purpose, making sure users are aware of this implicit transformation.
-
+    is larger than 2, the output vector is a 2D vector, with dim-0 correspoding
+    to work-items, and dim-1 corresponding to the chunk size loaded by each work-item.
     The mask operand masks out memory access so that it is safe to pass out-of-boundary
     addresses/offsets as long as they are masked. It applies to slots of SIMD lanes.
 
-    In SIMT mode, LoadGatherOp expects the tensor descriptor to be augmented with `LayoutAttr`
-    which describes the mapping of the tensor to the work items. In this case, result vector
-    represents the data to be loaded by each work-item. Each work-item recieves a `chunk_size`
-    number of elements.
+    In SIMT mode, the result vector represents the data to be loaded by each work-item.
+    Each work-item recieves a `chunk_size` number of elements.
 
   Example 1:
   ```mlir
-    %2 = xegpu.load %1, %0 {l1_hint = #xegpu.cache_hint<cached>,
-                            l2_hint = #xegpu.cache_hint<uncached>,
-                            l3_hint = #xegpu.cache_hint<uncached>}
+    %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<uncached>,
+                             l3_hint = #xegpu.cache_hint<uncached>}>
           : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<memory_space=global>>,
             vector<16xi1> -> vector<16xf32>
   ```
 
   Example 2:
   ```mlir
-    %2 = xegpu.load %1, %0 {transpose,
-                            l1_hint = #xegpu.cache_hint<cached>,
-                            l2_hint = #xegpu.cache_hint<uncached>,
-                            l3_hint = #xegpu.cache_hint<uncached>}
+    %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<uncached>,
+                             l3_hint = #xegpu.cache_hint<uncached>}>
           : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>>,
-            vector<16xi1> -> vector<8x16xf32>
+            vector<16xi1> -> vector<16x8xf32>
   ```
   Example 3 (SIMT mode):
   ```mlir
-    %2 = xegpu.load %1, %0 {transpose,
-                            l1_hint = #xegpu.cache_hint<cached>,
-                            l2_hint = #xegpu.cache_hint<uncached>,
-                            l3_hint = #xegpu.cache_hint<uncached>}
-          : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>,
-            !xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>
-            vector<16xi1> -> vector<8x1xf32>
+    %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<uncached>,
+                             l3_hint = #xegpu.cache_hint<uncached>}>
+          : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>>
+            vector<16xi1> -> vector<8xf32>
   ```
 
   }];
 
   let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
                        XeGPU_MaskType: $mask,
-                       OptionalAttr<UnitAttr>: $transpose,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
@@ -699,44 +652,38 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [
   has transpose effect, which is similar to `load_gather`. Therefore, a transpose attribute is
   introduced on purpose, making sure users are aware of this implicit transformation.
 
-  In SIMT mode, StoreScatterOp expects the tensor descriptor to be augmented with `LayoutAttr`
-  which describes the mapping of the tensor to the work items. In this case, input vector
-  represents the data to be stored by each work-item. Each work-item recieves a `chunk_size`
-  number of elements.
+  In SIMT mode, the input vector represents the data to be stored by each work-item.
+  Each work-item stores a `chunk_size` number of elements.
 
   Example 1:
   ```mlir
-    xegpu.store %0, %1, %2 {l1_hint = #xegpu.cache_hint<uncached>,
-                                 l2_hint = #xegpu.cache_hint<write_back>,
-                                 l3_hint = #xegpu.cache_hint<write_through>}
+    xegpu.store %0, %1, %2 <{l1_hint = #xegpu.cache_hint<uncached>,
+                             l2_hint = #xegpu.cache_hint<write_back>,
+                             l3_hint = #xegpu.cache_hint<write_through>}>
           : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scattered_tdesc_attr<>>, vector<16xi1>
   ```
 
   Example 2:
   ```mlir
-    xegpu.store %0, %1, %2 {transpose,
-                                 l1_hint = #xegpu.cache_hint<uncached>,
-                                 l2_hint = #xegpu.cache_hint<write_back>,
-                                 l3_hint = #xegpu.cache_hint<write_through>}
-          : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>>, vector<16xi1>
+    xegpu.store %0, %1, %2 <{l1_hint = #xegpu.cache_hint<uncached>,
+                             l2_hint = #xegpu.cache_hint<write_back>,
+                             l3_hint = #xegpu.cache_hint<write_through>}>
+          : vector<16x8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>>, vector<16xi1>
   ```
+
   Example 3 (SIMT mode):
   ```mlir
-    xegpu.store %0, %1, %2 {transpose,
-                                 l1_hint = #xegpu.cache_hint<uncached>,
-                                 l2_hint = #xegpu.cache_hint<write_back>,
-                                 l3_hint = #xegpu.cache_hint<write_through>}
-          : vector<8x1xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>,
-            !xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>> vector<16xi1>
+    xegpu.store %0, %1, %2 <{l1_hint = #xegpu.cache_hint<uncached>,
+                             l2_hint = #xegpu.cache_hint<write_back>,
+                             l3_hint = #xegpu.cache_hint<write_through>}>
+          : vector<8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>> vector<16xi1>
   ```
-
   }];
 
   let arguments = (ins
     XeGPU_ValueType: $value,
     XeGPU_TensorDesc: $TensorDesc,
     XeGPU_MaskType: $mask,
-    OptionalAttr<UnitAttr>: $transpose,
     OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
     OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
     OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
@@ -773,20 +720,13 @@ def XeGPU_UpdateOffsetOp: XeGPU_Op<"update_offset",
     update the offset per work-item, so its offsets contains values representing
     shifts for each work-item.
 
-    Example 1:
+    Example:
     ```mlir
       %off = arith.constant dense<[32, 32, 32, 32]> : vector<4xindex>
       %2 = xegpu.update_offset %1, %off :
               !xegpu.tensor_desc<4x2xf32, #xegpu.scattered_tdesc_attr<chunk_size=2>>, vector<4xindex>
     ```
 
-    Example 2 (SIMT mode):
-    ```mlir
-      %off = arith.constant dense<[32, 32, 32, 32]> : vector<4xindex>
-      %2 = xegpu.update_offset %1, %off :
-              !xegpu.tensor_desc<4x2xf32, #xegpu.scattered_tdesc_attr<chunk_size=2>,
-              #xegpu.layout<lane_layout = [4, 1], lane_data = [1, 1]>>, vector<4xindex>
-    ```
   }];
 
   let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h
new file mode 100644
index 000000000000..8aa9536cb67c
--- /dev/null
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h
@@ -0,0 +1,30 @@
+//===- XeGPUTargetInfo.h - Target constants ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_XEGPU_IR_XEGPUTARGETINFO_H_
+#define MLIR_DIALECT_XEGPU_IR_XEGPUTARGETINFO_H_
+
+namespace mlir {
+namespace xegpu {
+/// HW dependent constants.
+/// TODO: These constants should be queried from the target information.
+namespace targetinfo {
+constexpr unsigned subgroupSize = 16; // How many lanes in a subgroup.
+/// If DPAS A or B operands have low precision element types they must be packed
+/// according to the following sizes.
+constexpr unsigned packedSizeInBitsForDefault =
+    16; // Minimum packing size per register for DPAS A.
+constexpr unsigned packedSizeInBitsForDpasB =
+    32; // Minimum packing size per register for DPAS B.
+constexpr unsigned packedSizeInBitsForGatherScatter =
+    32; // Minimum packing size per register for Gather and Scatter ops.
+} // namespace targetinfo
+} // namespace xegpu
+} // namespace mlir
+
+#endif // MLIR_DIALECT_XEGPU_IR_XEGPUTARGETINFO_H_
diff --git a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
index 772cf7364964..6fea10185402 100644
--- a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
+++ b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
@@ -24,20 +24,6 @@ class LayoutAttr;
 class TensorDescType;
 } // namespace xegpu
 
-namespace xegpu {
-/// HW dependent constants.
-/// TODO: These constants should be queried from the target information.
-namespace targetinfo {
-constexpr unsigned subgroupSize = 16; // How many lanes in a subgroup.
-/// If DPAS A or B operands have low precision element types they must be packed
-/// according to the following sizes.
-constexpr unsigned packedSizeInBitsForDefault =
-    16; // Minimum packing size per register for DPAS A.
-constexpr unsigned packedSizeInBitsForDpasB =
-    32; // Minimum packing size per register for DPAS B.
-} // namespace targetinfo
-} // namespace xegpu
-
 namespace xegpu {
 
 /// Flatten a set of ValueRange into a single SmallVector<Value>
diff --git a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
index f355dfb8648e..6e72f7c23bdc 100644
--- a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
+++ b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
@@ -68,8 +68,8 @@ makeStridedMemRefDescriptor(T *ptr, T *alignedPtr, ArrayRef<int64_t> shape,
   assert(shape.size() == N);
   assert(shapeAlloc.size() == N);
   StridedMemRefType<T, N> descriptor;
-  descriptor.basePtr = static_cast<T *>(ptr);
-  descriptor.data = static_cast<T *>(alignedPtr);
+  descriptor.basePtr = ptr;
+  descriptor.data = alignedPtr;
   descriptor.offset = 0;
   std::copy(shape.begin(), shape.end(), descriptor.sizes);
   auto strides = makeStrides<N>(shapeAlloc);
@@ -89,8 +89,8 @@ makeStridedMemRefDescriptor(T *ptr, T *alignedPtr, ArrayRef<int64_t> shape = {},
   assert(shape.size() == N);
   assert(shapeAlloc.size() == N);
   StridedMemRefType<T, 0> descriptor;
-  descriptor.basePtr = static_cast<T *>(ptr);
-  descriptor.data = static_cast<T *>(alignedPtr);
+  descriptor.basePtr = ptr;
+  descriptor.data = alignedPtr;
   descriptor.offset = 0;
   return descriptor;
 }
diff --git a/mlir/include/mlir/IR/EnumAttr.td b/mlir/include/mlir/IR/EnumAttr.td
index 3f7f747ac20d..ff6cec6d4116 100644
--- a/mlir/include/mlir/IR/EnumAttr.td
+++ b/mlir/include/mlir/IR/EnumAttr.td
@@ -39,8 +39,11 @@ class EnumCase<string sym, int intVal, string strVal, int widthVal> {
 class IntEnumAttrCaseBase<I intType, string sym, string strVal, int intVal> :
     EnumCase<sym, intVal, strVal, intType.bitwidth>,
     SignlessIntegerAttrBase<intType, "case " # strVal> {
-  let predicate =
-    CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getInt() == " # intVal>;
+  let predicate = CPred<[{
+    ::llvm::cast<::mlir::IntegerAttr>($_self).getValue().eq(::llvm::APInt(}]
+      # intType.bitwidth # ", "
+      # intVal # 
+    "))">;
 }
 
 // Cases of integer enums with a specific type. By default, the string
diff --git a/mlir/include/mlir/IR/Matchers.h b/mlir/include/mlir/IR/Matchers.h
index 1dce055db1b4..e577909621cb 100644
--- a/mlir/include/mlir/IR/Matchers.h
+++ b/mlir/include/mlir/IR/Matchers.h
@@ -88,7 +88,7 @@ struct constant_op_binder {
 
     // Fold the constant to an attribute.
     SmallVector<OpFoldResult, 1> foldedOp;
-    LogicalResult result = op->fold(/*operands=*/std::nullopt, foldedOp);
+    LogicalResult result = op->fold(/*operands=*/{}, foldedOp);
     (void)result;
     assert(succeeded(result) && "expected ConstantLike op to be foldable");
 
diff --git a/mlir/include/mlir/IR/ODSSupport.h b/mlir/include/mlir/IR/ODSSupport.h
index b24a2470470f..036ce4e5b725 100644
--- a/mlir/include/mlir/IR/ODSSupport.h
+++ b/mlir/include/mlir/IR/ODSSupport.h
@@ -118,6 +118,9 @@ convertFromAttribute(SmallVectorImpl<int32_t> &storage, Attribute attr,
 /// Convert the provided ArrayRef<int64_t> to a DenseI64ArrayAttr attribute.
 Attribute convertToAttribute(MLIRContext *ctx, ArrayRef<int64_t> storage);
 
+/// Convert the provided ArrayRef<int32_t> to a DenseI32ArrayAttr attribute.
+Attribute convertToAttribute(MLIRContext *ctx, ArrayRef<int32_t> storage);
+
 } // namespace mlir
 
 #endif // MLIR_IR_ODSSUPPORT_H
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 31f54413a5ff..663c256c848d 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -272,7 +272,7 @@ class OpFoldResult : public PointerUnion<Attribute, Value> {
   using PointerUnion<Attribute, Value>::PointerUnion;
 
 public:
-  void dump() const { llvm::errs() << *this << "\n"; }
+  LLVM_DUMP_METHOD void dump() const { llvm::errs() << *this << "\n"; }
 
   MLIRContext *getContext() const {
     PointerUnion pu = *this;
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index 8710b970e8d7..d70aa346eaa1 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -207,6 +207,9 @@ class AsmPrinter {
   /// provide a valid type for the attribute.
   virtual void printAttributeWithoutType(Attribute attr);
 
+  /// Print the given named attribute.
+  virtual void printNamedAttribute(NamedAttribute attr);
+
   /// Print the alias for the given attribute, return failure if no alias could
   /// be printed.
   virtual LogicalResult printAlias(Attribute attr);
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index b6659a7f915c..afeb784b85a1 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -810,8 +810,7 @@ class RewritePatternSet {
   RewritePatternSet &add(ConstructorArg &&arg, ConstructorArgs &&...args) {
     // The following expands a call to emplace_back for each of the pattern
     // types 'Ts'.
-    (addImpl<Ts>(/*debugLabels=*/std::nullopt,
-                 std::forward<ConstructorArg>(arg),
+    (addImpl<Ts>(/*debugLabels=*/{}, std::forward<ConstructorArg>(arg),
                  std::forward<ConstructorArgs>(args)...),
      ...);
     return *this;
@@ -894,7 +893,7 @@ class RewritePatternSet {
   RewritePatternSet &insert(ConstructorArg &&arg, ConstructorArgs &&...args) {
     // The following expands a call to emplace_back for each of the pattern
     // types 'Ts'.
-    (addImpl<Ts>(/*debugLabels=*/std::nullopt, arg, args...), ...);
+    (addImpl<Ts>(/*debugLabels=*/{}, arg, args...), ...);
     return *this;
   }
 
diff --git a/mlir/include/mlir/IR/Unit.h b/mlir/include/mlir/IR/Unit.h
index 63117a7664a7..0e99f8e25f32 100644
--- a/mlir/include/mlir/IR/Unit.h
+++ b/mlir/include/mlir/IR/Unit.h
@@ -22,7 +22,7 @@ class Region;
 class Block;
 class Value;
 
-/// IRUnit is a union of the different types of IR objects that consistute the
+/// IRUnit is a union of the different types of IR objects that constitute the
 /// IR structure (other than Type and Attribute), that is Operation, Region, and
 /// Block.
 class IRUnit : public PointerUnion<Operation *, Region *, Block *, Value> {
diff --git a/mlir/include/mlir/IR/Visitors.h b/mlir/include/mlir/IR/Visitors.h
index 15abf2559e5c..893f66ae33de 100644
--- a/mlir/include/mlir/IR/Visitors.h
+++ b/mlir/include/mlir/IR/Visitors.h
@@ -14,6 +14,7 @@
 #define MLIR_IR_VISITORS_H
 
 #include "mlir/Support/LLVM.h"
+#include "mlir/Support/WalkResult.h"
 #include "llvm/ADT/STLExtras.h"
 
 namespace mlir {
@@ -23,41 +24,6 @@ class Operation;
 class Block;
 class Region;
 
-/// A utility result that is used to signal how to proceed with an ongoing walk:
-///   * Interrupt: the walk will be interrupted and no more operations, regions
-///   or blocks will be visited.
-///   * Advance: the walk will continue.
-///   * Skip: the walk of the current operation, region or block and their
-///   nested elements that haven't been visited already will be skipped and will
-///   continue with the next operation, region or block.
-class WalkResult {
-  enum ResultEnum { Interrupt, Advance, Skip } result;
-
-public:
-  WalkResult(ResultEnum result = Advance) : result(result) {}
-
-  /// Allow LogicalResult to interrupt the walk on failure.
-  WalkResult(LogicalResult result)
-      : result(failed(result) ? Interrupt : Advance) {}
-
-  /// Allow diagnostics to interrupt the walk.
-  WalkResult(Diagnostic &&) : result(Interrupt) {}
-  WalkResult(InFlightDiagnostic &&) : result(Interrupt) {}
-
-  bool operator==(const WalkResult &rhs) const { return result == rhs.result; }
-  bool operator!=(const WalkResult &rhs) const { return result != rhs.result; }
-
-  static WalkResult interrupt() { return {Interrupt}; }
-  static WalkResult advance() { return {Advance}; }
-  static WalkResult skip() { return {Skip}; }
-
-  /// Returns true if the walk was interrupted.
-  bool wasInterrupted() const { return result == Interrupt; }
-
-  /// Returns true if the walk was skipped.
-  bool wasSkipped() const { return result == Skip; }
-};
-
 /// Traversal order for region, block and operation walk utilities.
 enum class WalkOrder { PreOrder, PostOrder };
 
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 261b0e00bdf8..c6fcf1a0d510 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -46,6 +46,7 @@
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/AllInterfaces.h"
 #include "mlir/Dialect/Linalg/Transforms/RuntimeOpVerification.h"
@@ -152,7 +153,8 @@ inline void registerAllDialects(DialectRegistry &registry) {
                   ub::UBDialect,
                   vector::VectorDialect,
                   x86vector::X86VectorDialect,
-                  xegpu::XeGPUDialect>();
+                  xegpu::XeGPUDialect,
+                  xevm::XeVMDialect>();
   // clang-format on
 
   // Register all external models.
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
index 49925fe7701c..1a2c05fc16ed 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
@@ -366,9 +366,8 @@ def ReifyRankedShapedTypeOpInterface :
 
         `reifiedReturnShapes` is populated with one vector per op result. Each
         of those vectors contains an OpFoldResult for each dimension of the
-        shaped type. In case a dimension in the type is static, the
-        corresponding entry is an IntegerAttr. Otherwise, it is a Value. The
-        given builder may be used to insert ops that compute result shapes.
+        shaped type. The given builder may be used to insert ops that compute
+        result shapes.
 
         If the shape of a particular result cannot be computed it must be empty.
       }],
diff --git a/mlir/include/mlir/Interfaces/TilingInterface.td b/mlir/include/mlir/Interfaces/TilingInterface.td
index 0de37338c95e..0c0fc88aec95 100644
--- a/mlir/include/mlir/Interfaces/TilingInterface.td
+++ b/mlir/include/mlir/Interfaces/TilingInterface.td
@@ -202,28 +202,28 @@ def TilingInterface : OpInterface<"TilingInterface"> {
       InterfaceMethod<
         /*desc=*/[{
           Method to generate the tiled implementation of an operation that uses
-          exactly a tile of the given operand.
+          the exact tiles of the given operands.
 
           This method is required to allow operations to be "tiled and fused"
-          with an (already tiled) producer. Given a tile of the producer, this
-          method generates the tile of the consumer that uses exactly this
-          produced tile. In some sense it is the "reverse" of
+          with an (already tiled) producer. Given tiles of the producer, this
+          method generates the tile of the consumer that uses exactly these
+          produced tiles. In some sense it is the "reverse" of
           `generateResultTileValue`.
-          - `operandNumber` is the result of the producer used by the consumer.
-          - `offsets` is the offset of the slice of the producer result used by
-            the tiled implementation of the consumer.
-          - `sizes` is the size of the slice of the producer result used by the
+          - `operandNumbers` is the list of operands whose tiles are "producers".
+          - `allOffsets` is the offset of the slice of the producer used by the
+            tiled implementation of the consumer.
+          - `allSizes` is the size of the slice of the producer used by the
             consumer.
-          If it is illegal to fuse with a producer along the given operand for
+          If it is illegal to fuse with a producer along the given operand tiles for
           an operation, the implementation should return a failure.
         }],
         /*retType=*/"::mlir::FailureOr<::mlir::TilingResult>",
-        /*methodName=*/"getTiledImplementationFromOperandTile",
+        /*methodName=*/"getTiledImplementationFromOperandTiles",
         /*args=*/(ins
           "::mlir::OpBuilder &":$b,
-          "unsigned":$operandNumber,
-          "::mlir::ArrayRef<::mlir::OpFoldResult>":$offsets,
-          "::mlir::ArrayRef<::mlir::OpFoldResult>":$sizes),
+          "::mlir::ArrayRef<unsigned>":$operandNumbers,
+          "::mlir::ArrayRef<::mlir::SmallVector<::mlir::OpFoldResult>>":$allOffsets,
+          "::mlir::ArrayRef<::mlir::SmallVector<::mlir::OpFoldResult>>":$allSizes),
         /*methodBody=*/"",
         /*defaultImplementation=*/[{
           return failure();
@@ -235,16 +235,17 @@ def TilingInterface : OpInterface<"TilingInterface"> {
           tile of the operand.
 
           This method is required to allow operations to be "tiled and fused"
-          with an (already tiled) producer. Given a tile of an operand,
-          returns the tile of the iteration space that uses this tile.
-          - `operandNumber` is the result of the producer used by the consumer.
-          - `offsets` is the offset of the slice of the producer result used by
+          with an (already tiled) producer. Given tiles of operands,
+          returns the tile of the iteration space that uses these tiles.
+          - `operandNumbers` is the list of operands whose tiles are "produced"
+            by the producer(s).
+          - `allOffsets` is the offset of the slice of the producers used by
             the tiled implementation of the consumer.
-          - `sizes` is the size of the slice of the producer result used by the
+          - `allSizes` is the size of the slice of the producers used by the
             consumer.
-          If it is illegal to fuse with a producer along the given operand for
-          an operation, or if this mapping cannot be computed, the
-          implementation should return a failure.
+          If it is illegal to fuse with the producer slices for an operation,
+          or if this mapping cannot be computed, the implementation should
+          return a failure.
 
           Note that unlike the "tile consumer and fuse producer" case, the
           "tile producer and fuse consumer" requires an additional method to get
@@ -285,17 +286,17 @@ def TilingInterface : OpInterface<"TilingInterface"> {
           transformation. It does not provide guarantees on whether such a
           transformation is profitable.
 
-          For most cases `getTiledImplementationFromOperandTile` could be a
-          implemented using `getIterationDomainTileFromOperandTile` +
+          For most cases `getTiledImplementationFromOperandTiles` could be a
+          implemented using `getIterationDomainTileFromOperandTiles` +
           `getTiledImplementation` methods.
         }],
         /*retType=*/"::llvm::LogicalResult",
-        /*methodName=*/"getIterationDomainTileFromOperandTile",
+        /*methodName=*/"getIterationDomainTileFromOperandTiles",
         /*args=*/(ins
           "::mlir::OpBuilder &":$b,
-          "unsigned":$operandNumber,
-          "::mlir::ArrayRef<::mlir::OpFoldResult> ":$offsets,
-          "::mlir::ArrayRef<::mlir::OpFoldResult> ":$sizes,
+          "::mlir::ArrayRef<unsigned>":$operandNumbers,
+          "::mlir::ArrayRef<::mlir::SmallVector<::mlir::OpFoldResult>> ":$allOffsets,
+          "::mlir::ArrayRef<::mlir::SmallVector<::mlir::OpFoldResult>> ":$allSizes,
           "::mlir::SmallVectorImpl<::mlir::OpFoldResult> &":$iterDomainOffsets,
           "::mlir::SmallVectorImpl<::mlir::OpFoldResult> &":$iterDomainSizes),
         /*methodBody=*/"",
diff --git a/mlir/include/mlir/Support/StateStack.h b/mlir/include/mlir/Support/StateStack.h
new file mode 100644
index 000000000000..ef0f5d198b45
--- /dev/null
+++ b/mlir/include/mlir/Support/StateStack.h
@@ -0,0 +1,128 @@
+//===- StateStack.h - Utility for storing a stack of state ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines utilities for storing a stack of generic context.
+// The context can be arbitrary data, possibly including file-scoped types. Data
+// must be derived from StateStackFrameBase and implement MLIR TypeID.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SUPPORT_STACKFRAME_H
+#define MLIR_SUPPORT_STACKFRAME_H
+
+#include "mlir/Support/TypeID.h"
+#include "mlir/Support/WalkResult.h"
+#include <memory>
+
+namespace mlir {
+
+/// Common CRTP base class for StateStack frames.
+class StateStackFrame {
+public:
+  virtual ~StateStackFrame() = default;
+  TypeID getTypeID() const { return typeID; }
+
+protected:
+  explicit StateStackFrame(TypeID typeID) : typeID(typeID) {}
+
+private:
+  const TypeID typeID;
+  virtual void anchor();
+};
+
+/// Concrete CRTP base class for StateStack frames. This is used for keeping a
+/// stack of common state useful for recursive IR conversions. For example, when
+/// translating operations with regions, users of StateStack can store state on
+/// StateStack before entering the region and inspect it when converting
+/// operations nested within that region. Users are expected to derive this
+/// class and put any relevant information into fields of the derived class. The
+/// usual isa/dyn_cast functionality is available for instances of derived
+/// classes.
+template <typename Derived>
+class StateStackFrameBase : public StateStackFrame {
+public:
+  explicit StateStackFrameBase() : StateStackFrame(TypeID::get<Derived>()) {}
+};
+
+class StateStack {
+public:
+  /// Creates a stack frame of type `T` on StateStack. `T` must
+  /// be derived from `StackFrameBase<T>` and constructible from the provided
+  /// arguments. Doing this before entering the region of the op being
+  /// translated makes the frame available when translating ops within that
+  /// region.
+  template <typename T, typename... Args>
+  void stackPush(Args &&...args) {
+    static_assert(std::is_base_of<StateStackFrame, T>::value,
+                  "can only push instances of StackFrame on StateStack");
+    stack.push_back(std::make_unique<T>(std::forward<Args>(args)...));
+  }
+
+  /// Pops the last element from the StateStack.
+  void stackPop() { stack.pop_back(); }
+
+  /// Calls `callback` for every StateStack frame of type `T`
+  /// starting from the top of the stack.
+  template <typename T>
+  WalkResult stackWalk(llvm::function_ref<WalkResult(T &)> callback) {
+    static_assert(std::is_base_of<StateStackFrame, T>::value,
+                  "expected T derived from StackFrame");
+    if (!callback)
+      return WalkResult::skip();
+    for (std::unique_ptr<StateStackFrame> &frame : llvm::reverse(stack)) {
+      if (T *ptr = dyn_cast_or_null<T>(frame.get())) {
+        WalkResult result = callback(*ptr);
+        if (result.wasInterrupted())
+          return result;
+      }
+    }
+    return WalkResult::advance();
+  }
+
+  /// Get the top instance of frame type `T` or nullptr if none are found
+  template <typename T>
+  T *getStackTop() {
+    T *top = nullptr;
+    stackWalk<T>([&](T &frame) -> mlir::WalkResult {
+      top = &frame;
+      return mlir::WalkResult::interrupt();
+    });
+    return top;
+  }
+
+private:
+  SmallVector<std::unique_ptr<StateStackFrame>> stack;
+};
+
+/// RAII object calling stackPush/stackPop on construction/destruction.
+/// HostClass could be a StateStack or some other class which forwards calls to
+/// one.
+template <typename T, typename HostClass = StateStack>
+struct SaveStateStack {
+  template <typename... Args>
+  explicit SaveStateStack(HostClass &host, Args &&...args) : host(host) {
+    host.template stackPush<T>(std::forward<Args>(args)...);
+  }
+  ~SaveStateStack() { host.stackPop(); }
+
+private:
+  HostClass &host;
+};
+
+} // namespace mlir
+
+namespace llvm {
+template <typename T>
+struct isa_impl<T, ::mlir::StateStackFrame> {
+  static inline bool doit(const ::mlir::StateStackFrame &frame) {
+    return frame.getTypeID() == ::mlir::TypeID::get<T>();
+  }
+};
+} // namespace llvm
+
+#endif // MLIR_SUPPORT_STACKFRAME_H
diff --git a/mlir/include/mlir/Support/WalkResult.h b/mlir/include/mlir/Support/WalkResult.h
new file mode 100644
index 000000000000..cd3b1e156279
--- /dev/null
+++ b/mlir/include/mlir/Support/WalkResult.h
@@ -0,0 +1,59 @@
+//===- WalkResult.h - Status of completed walk ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Result kind for completed walk.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_SUPPORT_WALKRESULT_H
+#define MLIR_SUPPORT_WALKRESULT_H
+
+#include "mlir/Support/LLVM.h"
+
+namespace mlir {
+class Diagnostic;
+class InFlightDiagnostic;
+
+/// A utility result that is used to signal how to proceed with an ongoing walk:
+///   * Interrupt: the walk will be interrupted and no more operations, regions
+///   or blocks will be visited.
+///   * Advance: the walk will continue.
+///   * Skip: the walk of the current operation, region or block and their
+///   nested elements that haven't been visited already will be skipped and will
+///   continue with the next operation, region or block.
+class WalkResult {
+  enum ResultEnum { Interrupt, Advance, Skip } result;
+
+public:
+  WalkResult(ResultEnum result = Advance) : result(result) {}
+
+  /// Allow LogicalResult to interrupt the walk on failure.
+  WalkResult(LogicalResult result)
+      : result(failed(result) ? Interrupt : Advance) {}
+
+  /// Allow diagnostics to interrupt the walk.
+  WalkResult(Diagnostic &&) : result(Interrupt) {}
+  WalkResult(InFlightDiagnostic &&) : result(Interrupt) {}
+
+  bool operator==(const WalkResult &rhs) const { return result == rhs.result; }
+  bool operator!=(const WalkResult &rhs) const { return result != rhs.result; }
+
+  static WalkResult interrupt() { return {Interrupt}; }
+  static WalkResult advance() { return {Advance}; }
+  static WalkResult skip() { return {Skip}; }
+
+  /// Returns true if the walk was interrupted.
+  bool wasInterrupted() const { return result == Interrupt; }
+
+  /// Returns true if the walk was skipped.
+  bool wasSkipped() const { return result == Skip; }
+};
+
+} // namespace mlir
+
+#endif
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 0f136c5c46d7..79e8bb6add0d 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -18,6 +18,7 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
+#include "mlir/Support/StateStack.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
@@ -271,33 +272,6 @@ class ModuleTranslation {
   /// it if it does not exist.
   llvm::NamedMDNode *getOrInsertNamedModuleMetadata(StringRef name);
 
-  /// Common CRTP base class for ModuleTranslation stack frames.
-  class StackFrame {
-  public:
-    virtual ~StackFrame() = default;
-    TypeID getTypeID() const { return typeID; }
-
-  protected:
-    explicit StackFrame(TypeID typeID) : typeID(typeID) {}
-
-  private:
-    const TypeID typeID;
-    virtual void anchor();
-  };
-
-  /// Concrete CRTP base class for ModuleTranslation stack frames. When
-  /// translating operations with regions, users of ModuleTranslation can store
-  /// state on ModuleTranslation stack before entering the region and inspect
-  /// it when converting operations nested within that region. Users are
-  /// expected to derive this class and put any relevant information into fields
-  /// of the derived class. The usual isa/dyn_cast functionality is available
-  /// for instances of derived classes.
-  template <typename Derived>
-  class StackFrameBase : public StackFrame {
-  public:
-    explicit StackFrameBase() : StackFrame(TypeID::get<Derived>()) {}
-  };
-
   /// Creates a stack frame of type `T` on ModuleTranslation stack. `T` must
   /// be derived from `StackFrameBase<T>` and constructible from the provided
   /// arguments. Doing this before entering the region of the op being
@@ -305,46 +279,22 @@ class ModuleTranslation {
   /// region.
   template <typename T, typename... Args>
   void stackPush(Args &&...args) {
-    static_assert(
-        std::is_base_of<StackFrame, T>::value,
-        "can only push instances of StackFrame on ModuleTranslation stack");
-    stack.push_back(std::make_unique<T>(std::forward<Args>(args)...));
+    stack.stackPush<T>(std::forward<Args>(args)...);
   }
 
   /// Pops the last element from the ModuleTranslation stack.
-  void stackPop() { stack.pop_back(); }
+  void stackPop() { stack.stackPop(); }
 
   /// Calls `callback` for every ModuleTranslation stack frame of type `T`
   /// starting from the top of the stack.
   template <typename T>
   WalkResult stackWalk(llvm::function_ref<WalkResult(T &)> callback) {
-    static_assert(std::is_base_of<StackFrame, T>::value,
-                  "expected T derived from StackFrame");
-    if (!callback)
-      return WalkResult::skip();
-    for (std::unique_ptr<StackFrame> &frame : llvm::reverse(stack)) {
-      if (T *ptr = dyn_cast_or_null<T>(frame.get())) {
-        WalkResult result = callback(*ptr);
-        if (result.wasInterrupted())
-          return result;
-      }
-    }
-    return WalkResult::advance();
+    return stack.stackWalk(callback);
   }
 
   /// RAII object calling stackPush/stackPop on construction/destruction.
   template <typename T>
-  struct SaveStack {
-    template <typename... Args>
-    explicit SaveStack(ModuleTranslation &m, Args &&...args)
-        : moduleTranslation(m) {
-      moduleTranslation.stackPush<T>(std::forward<Args>(args)...);
-    }
-    ~SaveStack() { moduleTranslation.stackPop(); }
-
-  private:
-    ModuleTranslation &moduleTranslation;
-  };
+  using SaveStack = SaveStateStack<T, ModuleTranslation>;
 
   SymbolTableCollection &symbolTable() { return symbolTableCollection; }
 
@@ -468,7 +418,7 @@ class ModuleTranslation {
 
   /// Stack of user-specified state elements, useful when translating operations
   /// with regions.
-  SmallVector<std::unique_ptr<StackFrame>> stack;
+  StateStack stack;
 
   /// A cache for the symbol tables constructed during symbols lookup.
   SymbolTableCollection symbolTableCollection;
@@ -510,14 +460,4 @@ llvm::CallInst *createIntrinsicCall(
 } // namespace LLVM
 } // namespace mlir
 
-namespace llvm {
-template <typename T>
-struct isa_impl<T, ::mlir::LLVM::ModuleTranslation::StackFrame> {
-  static inline bool
-  doit(const ::mlir::LLVM::ModuleTranslation::StackFrame &frame) {
-    return frame.getTypeID() == ::mlir::TypeID::get<T>();
-  }
-};
-} // namespace llvm
-
 #endif // MLIR_TARGET_LLVMIR_MODULETRANSLATION_H
diff --git a/mlir/include/mlir/Tools/PDLL/AST/Nodes.h b/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
index 9ad94839890b..07e265582e96 100644
--- a/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
+++ b/mlir/include/mlir/Tools/PDLL/AST/Nodes.h
@@ -903,8 +903,8 @@ class UserConstraintDecl final
                                         ArrayRef<VariableDecl *> results,
                                         const CompoundStmt *body,
                                         Type resultType) {
-    return createImpl(ctx, name, inputs, /*nativeInputTypes=*/std::nullopt,
-                      results, /*codeBlock=*/std::nullopt, body, resultType);
+    return createImpl(ctx, name, inputs, /*nativeInputTypes=*/{}, results,
+                      /*codeBlock=*/std::nullopt, body, resultType);
   }
 
   /// Return the name of the constraint.
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index 81858812d262..f6437657c9a9 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -221,12 +221,13 @@ class TypeConverter {
   ///
   /// The conversion functions take a non-null Type or subclass of Type and a
   /// non-null Attribute (or subclass of Attribute), and returns a
-  /// `AttributeConversionResult`. This result can either contan an `Attribute`,
-  /// which may be `nullptr`, representing the conversion's success,
-  /// `AttributeConversionResult::na()` (the default empty value), indicating
-  /// that the conversion function did not apply and that further conversion
-  /// functions should be checked, or `AttributeConversionResult::abort()`
-  /// indicating that the conversion process should be aborted.
+  /// `AttributeConversionResult`. This result can either contain an
+  /// `Attribute`, which may be `nullptr`, representing the conversion's
+  /// success, `AttributeConversionResult::na()` (the default empty value),
+  /// indicating that the conversion function did not apply and that further
+  /// conversion functions should be checked, or
+  /// `AttributeConversionResult::abort()` indicating that the conversion
+  /// process should be aborted.
   ///
   /// Registered conversion functions are callled in the reverse of the order in
   /// which they were registered.
diff --git a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
index 953d65e467b7..d05374f667a5 100644
--- a/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DenseAnalysis.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Casting.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
index 016e59dcb744..e625f626d12f 100644
--- a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Casting.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
index 6cc5ec639f44..f4b02b496a5c 100644
--- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
+++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Analysis//FlatLinearValueConstraints.h"
 
-#include "mlir/Analysis/Presburger/LinearTransform.h"
 #include "mlir/Analysis/Presburger/PresburgerSpace.h"
 #include "mlir/Analysis/Presburger/Simplex.h"
 #include "mlir/Analysis/Presburger/Utils.h"
@@ -17,7 +16,6 @@
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InterleavedRange.h"
diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
index 2181d9484f22..17e48e0d069b 100644
--- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
@@ -25,16 +25,12 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
 #include <functional>
 #include <memory>
-#include <numeric>
 #include <optional>
-#include <sstream>
-#include <string>
 #include <utility>
 #include <vector>
 
@@ -2218,7 +2214,7 @@ IntegerRelation::unionBoundingBox(const IntegerRelation &otherCst) {
       auto constOtherLb = otherCst.getConstantBound(BoundType::LB, d);
       if (!constLb.has_value() || !constOtherLb.has_value())
         return failure();
-      std::fill(minLb.begin(), minLb.end(), 0);
+      llvm::fill(minLb, 0);
       minLb.back() = std::min(*constLb, *constOtherLb);
     }
 
@@ -2234,12 +2230,12 @@ IntegerRelation::unionBoundingBox(const IntegerRelation &otherCst) {
       auto constOtherUb = otherCst.getConstantBound(BoundType::UB, d);
       if (!constUb.has_value() || !constOtherUb.has_value())
         return failure();
-      std::fill(maxUb.begin(), maxUb.end(), 0);
+      llvm::fill(maxUb, 0);
       maxUb.back() = std::max(*constUb, *constOtherUb);
     }
 
-    std::fill(newLb.begin(), newLb.end(), 0);
-    std::fill(newUb.begin(), newUb.end(), 0);
+    llvm::fill(newLb, 0);
+    llvm::fill(newUb, 0);
 
     // The divisor for lb, ub, otherLb, otherUb at this point is lbDivisor,
     // and so it's the divisor for newLb and newUb as well.
diff --git a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
index 3ed77998c92a..55d17ad47e99 100644
--- a/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
+++ b/mlir/lib/Analysis/Presburger/PWMAFunction.cpp
@@ -143,7 +143,7 @@ void MultiAffineFunction::mergeDivs(MultiAffineFunction &other) {
   SmallVector<DynamicAPInt, 8> div(other.divs.getNumVars() + 1);
   for (unsigned i = 0; i < nDivs; ++i) {
     // Zero fill.
-    std::fill(div.begin(), div.end(), 0);
+    llvm::fill(div, 0);
     // Fill div with dividend from `divs`. Do not fill the constant.
     std::copy(divs.getDividend(i).begin(), divs.getDividend(i).end() - 1,
               div.begin());
diff --git a/mlir/lib/Analysis/Presburger/Simplex.cpp b/mlir/lib/Analysis/Presburger/Simplex.cpp
index 4eb6e6da2950..08290db55f2c 100644
--- a/mlir/lib/Analysis/Presburger/Simplex.cpp
+++ b/mlir/lib/Analysis/Presburger/Simplex.cpp
@@ -18,7 +18,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <functional>
diff --git a/mlir/lib/Analysis/Presburger/Utils.cpp b/mlir/lib/Analysis/Presburger/Utils.cpp
index f12bd5239245..2aaa4c033d0c 100644
--- a/mlir/lib/Analysis/Presburger/Utils.cpp
+++ b/mlir/lib/Analysis/Presburger/Utils.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/Presburger/PresburgerSpace.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/Support/LogicalResult.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstdint>
diff --git a/mlir/lib/Analysis/SliceAnalysis.cpp b/mlir/lib/Analysis/SliceAnalysis.cpp
index 4cdf47a405c0..36a9812bd797 100644
--- a/mlir/lib/Analysis/SliceAnalysis.cpp
+++ b/mlir/lib/Analysis/SliceAnalysis.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 ///
 /// Implements Analysis functions specific to slicing in Function.
diff --git a/mlir/lib/AsmParser/AttributeParser.cpp b/mlir/lib/AsmParser/AttributeParser.cpp
index 2474e88373e0..dcb21e7d0831 100644
--- a/mlir/lib/AsmParser/AttributeParser.cpp
+++ b/mlir/lib/AsmParser/AttributeParser.cpp
@@ -717,7 +717,7 @@ DenseElementsAttr TensorLiteralParser::getHexAttr(SMLoc loc, ShapedType type) {
   if (parseElementAttrHexValues(p, *hexStorage, data))
     return nullptr;
 
-  ArrayRef<char> rawData(data.data(), data.size());
+  ArrayRef<char> rawData(data);
   bool detectedSplat = false;
   if (!DenseElementsAttr::isValidRawBuffer(type, rawData, detectedSplat)) {
     p.emitError(loc) << "elements hex data size is invalid for provided type: "
diff --git a/mlir/lib/Bindings/Python/IRAttributes.cpp b/mlir/lib/Bindings/Python/IRAttributes.cpp
index 12725a0ed093..8f79caf08a6d 100644
--- a/mlir/lib/Bindings/Python/IRAttributes.cpp
+++ b/mlir/lib/Bindings/Python/IRAttributes.cpp
@@ -1670,7 +1670,7 @@ class PyStridedLayoutAttribute
         [](int64_t rank, DefaultingPyMlirContext ctx) {
           auto dynamic = mlirShapedTypeGetDynamicStrideOrOffset();
           std::vector<int64_t> strides(rank);
-          std::fill(strides.begin(), strides.end(), dynamic);
+          llvm::fill(strides, dynamic);
           MlirAttribute attr = mlirStridedLayoutAttrGet(
               ctx->get(), dynamic, strides.size(), strides.data());
           return PyStridedLayoutAttribute(ctx->getRef(), attr);
diff --git a/mlir/lib/CAPI/Dialect/Linalg.cpp b/mlir/lib/CAPI/Dialect/Linalg.cpp
index 0c4f6e88e707..21db18dfd47e 100644
--- a/mlir/lib/CAPI/Dialect/Linalg.cpp
+++ b/mlir/lib/CAPI/Dialect/Linalg.cpp
@@ -38,7 +38,7 @@ void mlirLinalgFillBuiltinNamedOpRegion(MlirOperation mlirOp) {
   Region &region = op->getRegion(0);
   Block *body = b.createBlock(&region, /*insertPt=*/{}, argTypes, argLocs);
   b.setInsertionPointToStart(body);
-  fun(b, *body, op->getAttrs());
+  fun(b, *body, op->getAttrs(), /*emitError=*/{});
 }
 
 MLIR_CAPI_EXPORTED bool mlirLinalgIsAContractionOp(MlirOperation op) {
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 700563460f52..910fe1b1d93c 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -1100,6 +1100,81 @@ struct WMMAOpLowering : public ConvertOpToLLVMPattern<WMMAOp> {
   }
 };
 
+struct TransposeLoadOpLowering
+    : public ConvertOpToLLVMPattern<TransposeLoadOp> {
+  TransposeLoadOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<TransposeLoadOp>(converter), chipset(chipset) {}
+
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(TransposeLoadOp op, TransposeLoadOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (chipset != kGfx950)
+      return op.emitOpError("Non-gfx950 chipset not supported");
+
+    Location loc = op.getLoc();
+    auto srcMemRefType = cast<MemRefType>(op.getSrc().getType());
+
+    // Elements in subbyte memrefs are stored non-contiguously,
+    // reject if source is sub-byte memref. Use emulated memrefs instead.
+    size_t srcElementSize =
+        srcMemRefType.getElementType().getIntOrFloatBitWidth();
+    if (srcElementSize < 8)
+      return op.emitOpError("Expect source memref to have at least 8 bits "
+                            "element size, got ")
+             << srcElementSize;
+
+    auto resultType = cast<VectorType>(op.getResult().getType());
+    Value srcPtr =
+        getStridedElementPtr(rewriter, loc, srcMemRefType, adaptor.getSrc(),
+                             (adaptor.getSrcIndices()));
+
+    size_t numElements = resultType.getNumElements();
+    size_t elementTypeSize =
+        resultType.getElementType().getIntOrFloatBitWidth();
+
+    // ROCDL transpose load intrinsics return vectors of 32-bit integers, if
+    // the element size is smaller than 16 bits.
+    Type rocdlResultType = VectorType::get((numElements * elementTypeSize) / 32,
+                                           rewriter.getIntegerType(32));
+    Type llvmResultType = typeConverter->convertType(resultType);
+
+    switch (elementTypeSize) {
+    case 4: {
+      assert(numElements == 16);
+      auto rocdlOp =
+          rewriter.create<ROCDL::ds_read_tr4_b64>(loc, rocdlResultType, srcPtr);
+      rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
+      break;
+    }
+    case 6: {
+      assert(numElements == 16);
+      auto rocdlOp =
+          rewriter.create<ROCDL::ds_read_tr6_b96>(loc, rocdlResultType, srcPtr);
+      rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
+      break;
+    }
+    case 8: {
+      assert(numElements == 8);
+      auto rocdlOp =
+          rewriter.create<ROCDL::ds_read_tr8_b64>(loc, rocdlResultType, srcPtr);
+      rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op, llvmResultType, rocdlOp);
+      break;
+    }
+    case 16: {
+      assert(numElements == 4);
+      rewriter.replaceOpWithNewOp<ROCDL::ds_read_tr16_b64>(op, llvmResultType,
+                                                           srcPtr);
+      break;
+    }
+    default:
+      return op.emitOpError("Unsupported element size for transpose load");
+    }
+    return success();
+  }
+};
+
 struct GatherToLDSOpLowering : public ConvertOpToLLVMPattern<GatherToLDSOp> {
   GatherToLDSOpLowering(const LLVMTypeConverter &converter, Chipset chipset)
       : ConvertOpToLLVMPattern<GatherToLDSOp>(converter), chipset(chipset) {}
@@ -1749,7 +1824,7 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
            MFMAOpLowering, ScaledMFMAOpLowering, WMMAOpLowering,
            ExtPackedFp8OpLowering, ScaledExtPackedOpLowering,
            PackedScaledTruncOpLowering, PackedTrunc2xFp8OpLowering,
-           PackedStochRoundFp8OpLowering, GatherToLDSOpLowering>(converter,
-                                                                 chipset);
+           PackedStochRoundFp8OpLowering, GatherToLDSOpLowering,
+           TransposeLoadOpLowering>(converter, chipset);
   patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
 }
diff --git a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
index 4fbe6a03f6ba..3b143ca1ef9c 100644
--- a/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
+++ b/mlir/lib/Conversion/AffineToStandard/AffineToStandard.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index b8e5aa87244f..f7bf581adc9e 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Pass/Pass.h"
 #include <type_traits>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
index 0c3f942b5cbd..21ea444e3182 100644
--- a/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
+++ b/mlir/lib/Conversion/ArmSMEToLLVM/ArmSMEToLLVM.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
 #include "mlir/Dialect/ArmSME/Utils/Utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index c95e375ce9af..94f7caa315cf 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -10,7 +10,6 @@
 
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
index 0b5a5d8e7106..b9991f36cdaa 100644
--- a/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
+++ b/mlir/lib/Conversion/BufferizationToMemRef/BufferizationToMemRef.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
@@ -111,8 +110,8 @@ struct CloneOpConversion : public OpConversionPattern<bufferization::CloneOp> {
             rewriter.create<memref::CastOp>(op->getLoc(), memrefType, alloc);
     }
 
-    rewriter.replaceOp(op, alloc);
     rewriter.create<memref::CopyOp>(loc, op.getInput(), alloc);
+    rewriter.replaceOp(op, alloc);
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
index 6956c81f2a2d..e5e862315941 100644
--- a/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
+++ b/mlir/lib/Conversion/ComplexToLLVM/ComplexToLLVM.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCOMPLEXTOLLVMPASS
diff --git a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
index 050bedf67b54..56269d189873 100644
--- a/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
+++ b/mlir/lib/Conversion/ComplexToLibm/ComplexToLibm.cpp
@@ -11,7 +11,6 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include <optional>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp b/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
index 5e7d2d849153..4225f815ab69 100644
--- a/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
+++ b/mlir/lib/Conversion/ComplexToSPIRV/ComplexToSPIRV.cpp
@@ -12,11 +12,9 @@
 
 #include "mlir/Conversion/ComplexToSPIRV/ComplexToSPIRV.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "complex-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 5089179435f1..0c832c452718 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -14,9 +14,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include <memory>
 #include <type_traits>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index cf416e140c85..88a8b7fb185c 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/PrintCallHelper.h"
-#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -25,8 +24,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/StringRef.h"
-#include <functional>
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTCONTROLFLOWTOLLVMPASS
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 1c592d665f3e..9831dcaaaccc 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/ControlFlowToSCF/ControlFlowToSCF.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
diff --git a/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp b/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
index d8f3e9951095..caaffc5a3315 100644
--- a/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.cpp
@@ -11,16 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/ControlFlowToSPIRV/ControlFlowToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "cf-to-spirv-pattern"
diff --git a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
index 2cf44051ce5f..ed5d6d4a7fe4 100644
--- a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
+++ b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
@@ -9,11 +9,9 @@
 #include "mlir/Analysis/DataLayoutAnalysis.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include <memory>
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 72977e6cc0e4..36235636d6ba 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -20,33 +20,22 @@
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
-#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
-#include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Type.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormatVariadic.h"
-#include <algorithm>
-#include <functional>
 #include <optional>
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp b/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
index a5c2b92ca191..7cc9680236b0 100644
--- a/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
+++ b/mlir/lib/Conversion/FuncToSPIRV/FuncToSPIRV.cpp
@@ -11,14 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "func-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 512820bab409..167cabbc57db 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -21,9 +21,7 @@
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/GPUCommon/GPUToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
@@ -40,8 +38,6 @@
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "gpu-to-llvm"
 
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index cef250232daf..1ef6edea93c5 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index e543ca98d8d4..456bfaba980c 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
-#include "mlir/Transforms/Passes.h"
 
 #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
@@ -34,14 +33,12 @@
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
-#include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/FormatVariadic.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
 #include "../GPUCommon/IndexIntrinsicsOpLowering.h"
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 93c76d267c51..b99ed261ecfa 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -13,13 +13,11 @@
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include <optional>
@@ -122,6 +120,16 @@ class GPUShuffleConversion final : public OpConversionPattern<gpu::ShuffleOp> {
                   ConversionPatternRewriter &rewriter) const override;
 };
 
+/// Pattern to convert a gpu.rotate op into a spirv.GroupNonUniformRotateKHROp.
+class GPURotateConversion final : public OpConversionPattern<gpu::RotateOp> {
+public:
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(gpu::RotateOp rotateOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
 class GPUPrintfConversion final : public OpConversionPattern<gpu::PrintfOp> {
 public:
   using OpConversionPattern::OpConversionPattern;
@@ -251,8 +259,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
   }
   auto newFuncOp = rewriter.create<spirv::FuncOp>(
       funcOp.getLoc(), funcOp.getName(),
-      rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
-                               std::nullopt));
+      rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   for (const auto &namedAttr : funcOp->getAttrs()) {
     if (namedAttr.getName() == funcOp.getFunctionTypeAttrName() ||
         namedAttr.getName() == SymbolTable::getSymbolAttrName())
@@ -489,6 +496,41 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Rotate
+//===----------------------------------------------------------------------===//
+
+LogicalResult GPURotateConversion::matchAndRewrite(
+    gpu::RotateOp rotateOp, OpAdaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  const spirv::TargetEnv &targetEnv =
+      getTypeConverter<SPIRVTypeConverter>()->getTargetEnv();
+  unsigned subgroupSize =
+      targetEnv.getAttr().getResourceLimits().getSubgroupSize();
+  IntegerAttr widthAttr;
+  if (!matchPattern(rotateOp.getWidth(), m_Constant(&widthAttr)) ||
+      widthAttr.getValue().getZExtValue() > subgroupSize)
+    return rewriter.notifyMatchFailure(
+        rotateOp,
+        "rotate width is not a constant or larger than target subgroup size");
+
+  Location loc = rotateOp.getLoc();
+  auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
+  Value rotateResult = rewriter.create<spirv::GroupNonUniformRotateKHROp>(
+      loc, scope, adaptor.getValue(), adaptor.getOffset(), adaptor.getWidth());
+  Value validVal;
+  if (widthAttr.getValue().getZExtValue() == subgroupSize) {
+    validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(), loc, rewriter);
+  } else {
+    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+                                              laneId, adaptor.getWidth());
+  }
+
+  rewriter.replaceOp(rotateOp, {rotateResult, validVal});
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Group ops
 //===----------------------------------------------------------------------===//
@@ -777,7 +819,7 @@ void mlir::populateGPUToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
                                       RewritePatternSet &patterns) {
   patterns.add<
       GPUBarrierConversion, GPUFuncOpConversion, GPUModuleConversion,
-      GPUReturnOpConversion, GPUShuffleConversion,
+      GPUReturnOpConversion, GPUShuffleConversion, GPURotateConversion,
       LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>,
       LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>,
       LaunchConfigConversion<gpu::BlockDimOp, spirv::BuiltIn::WorkgroupSize>,
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
index 509b6343057b..0b2c06a08db2 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRVPass.cpp
@@ -21,7 +21,6 @@
 #include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
index d2f5e3585355..7bb86b5ce1dd 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/WmmaOpsToSPIRV.cpp
@@ -12,10 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
-#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
diff --git a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
index 7c441830e1e3..4821962f989e 100644
--- a/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
+++ b/mlir/lib/Conversion/IndexToSPIRV/IndexToSPIRV.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 using namespace index;
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 86fb9166b722..c5f72f7e10b8 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -46,10 +46,13 @@ Type ConvertToLLVMPattern::getVoidType() const {
   return LLVM::LLVMVoidType::get(&getTypeConverter()->getContext());
 }
 
-Type ConvertToLLVMPattern::getVoidPtrType() const {
-  return LLVM::LLVMPointerType::get(&getTypeConverter()->getContext());
+Type ConvertToLLVMPattern::getPtrType(unsigned addressSpace) const {
+  return LLVM::LLVMPointerType::get(&getTypeConverter()->getContext(),
+                                    addressSpace);
 }
 
+Type ConvertToLLVMPattern::getVoidPtrType() const { return getPtrType(); }
+
 Value ConvertToLLVMPattern::createIndexAttrConstant(OpBuilder &builder,
                                                     Location loc,
                                                     Type resultType,
@@ -273,7 +276,7 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
             ? builder
                   .create<LLVM::CallOp>(loc, mallocFunc.value(), allocationSize)
                   .getResult()
-            : builder.create<LLVM::AllocaOp>(loc, getVoidPtrType(),
+            : builder.create<LLVM::AllocaOp>(loc, getPtrType(),
                                              IntegerType::get(getContext(), 8),
                                              allocationSize,
                                              /*alignment=*/0);
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 57c8f4402cf4..cf47da972469 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
-#include "MemRefDescriptor.h"
 #include "mlir/Conversion/LLVMCommon/MemRefBuilder.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 9504bcb0eff0..c3f213147b7a 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Pass/Pass.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTLINALGTOSTANDARDPASS
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index c4792884eb34..a0ce7d3b75fc 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
index e065c804507a..93d8b49b93bd 100644
--- a/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
+++ b/mlir/lib/Conversion/MathToROCDL/MathToROCDL.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/PatternMatch.h"
@@ -21,7 +20,6 @@
 #include "mlir/Transforms/DialectConversion.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
-#include "../GPUCommon/IndexIntrinsicsOpLowering.h"
 #include "../GPUCommon/OpToFuncCallLowering.h"
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 
diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 501bfa223fb1..59db14ed816b 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -12,14 +12,12 @@
 
 #include "../SPIRVCommon/Pattern.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "math-to-spirv-pattern"
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index 4b7b2cc224ce..0caf51606a50 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -24,7 +24,6 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/Pass/Pass.h"
-#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/Support/MathExtras.h"
 #include <optional>
 
@@ -408,8 +407,7 @@ struct AllocaScopeOpLowering
 
     // Save stack and then branch into the body of the region.
     rewriter.setInsertionPointToEnd(currentBlock);
-    auto stackSaveOp =
-        rewriter.create<LLVM::StackSaveOp>(loc, getVoidPtrType());
+    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
     rewriter.create<LLVM::BrOp>(loc, ValueRange(), beforeBody);
 
     // Replace the alloca_scope return with a branch that jumps out of the body.
@@ -1122,8 +1120,7 @@ class MemRefCopyOpLowering : public ConvertOpToLLVMPattern<memref::CopyOp> {
     };
 
     // Save stack position before promoting descriptors
-    auto stackSaveOp =
-        rewriter.create<LLVM::StackSaveOp>(loc, getVoidPtrType());
+    auto stackSaveOp = rewriter.create<LLVM::StackSaveOp>(loc, getPtrType());
 
     auto srcMemRefType = dyn_cast<MemRefType>(srcType);
     Value unrankedSource =
@@ -1249,7 +1246,7 @@ struct MemorySpaceCastOpLowering
                                              result, resultAddrSpace, sizes);
       Value resultUnderlyingSize = sizes.front();
       Value resultUnderlyingDesc = rewriter.create<LLVM::AllocaOp>(
-          loc, getVoidPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
+          loc, getPtrType(), rewriter.getI8Type(), resultUnderlyingSize);
       result.setMemRefDescPtr(rewriter, loc, resultUnderlyingDesc);
 
       // Copy pointers, performing address space casts.
@@ -1530,8 +1527,7 @@ struct MemRefReshapeOpLowering
     UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, addressSpace, sizes);
     Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
-        loc, getVoidPtrType(), IntegerType::get(getContext(), 8),
-        sizes.front());
+        loc, getPtrType(), IntegerType::get(getContext(), 8), sizes.front());
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
     // Extract pointers and offset from the source memref.
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index f8867c65fe7d..b866afbce98b 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -21,7 +21,6 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Visitors.h"
-#include "llvm/Support/Debug.h"
 #include <cassert>
 #include <optional>
 
diff --git a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
index 8950985d3b67..479725aae8af 100644
--- a/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
+++ b/mlir/lib/Conversion/OpenACCToSCF/OpenACCToSCF.cpp
@@ -8,11 +8,9 @@
 
 #include "mlir/Conversion/OpenACCToSCF/ConvertOpenACCToSCF.h"
 
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 7a0a7f86bc1e..77a270865357 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -12,7 +12,6 @@
 #include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
 #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
-#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 3b4209d47c5c..7d20109b3db5 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -9,14 +9,12 @@
 #include "mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h"
 
 #include "PredicateTree.h"
-#include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/Pass/Pass.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 
@@ -636,7 +634,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
   builder.setInsertionPointToEnd(rewriterModule.getBody());
   auto rewriterFunc = builder.create<pdl_interp::FuncOp>(
       pattern.getLoc(), "pdl_generated_rewriter",
-      builder.getFunctionType(std::nullopt, std::nullopt));
+      builder.getFunctionType({}, {}));
   rewriterSymbolTable.insert(rewriterFunc);
 
   // Generate the rewriter function body.
@@ -703,7 +701,7 @@ SymbolRefAttr PatternLowering::generateRewriter(
   // Update the signature of the rewrite function.
   rewriterFunc.setType(builder.getFunctionType(
       llvm::to_vector<8>(rewriterFunc.front().getArgumentTypes()),
-      /*results=*/std::nullopt));
+      /*results=*/{}));
 
   builder.create<pdl_interp::FinalizeOp>(rewriter.getLoc());
   return SymbolRefAttr::get(
@@ -990,8 +988,8 @@ void PDLToPDLInterpPass::runOnOperation() {
   auto matcherFunc = builder.create<pdl_interp::FuncOp>(
       module.getLoc(), pdl_interp::PDLInterpDialect::getMatcherFunctionName(),
       builder.getFunctionType(builder.getType<pdl::OperationType>(),
-                              /*results=*/std::nullopt),
-      /*attrs=*/std::nullopt);
+                              /*results=*/{}),
+      /*attrs=*/ArrayRef<NamedAttribute>());
 
   // Create a nested module to hold the functions invoked for rewriting the IR
   // after a successful match.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
index ac8ed4fdff7c..3e434ea7e14c 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
@@ -9,11 +9,8 @@
 #include "PredicateTree.h"
 #include "RootOrdering.h"
 
-#include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
-#include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/TypeSwitch.h"
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 114d634629d7..ac6dcf2513e9 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -19,8 +19,6 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Transforms/Transforms.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 345e8494194e..dcb48529a74e 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -13,13 +13,10 @@
 #include "mlir/Conversion/SCFToEmitC/SCFToEmitC.h"
 
 #include "mlir/Conversion/ConvertToEmitC/ToEmitCInterface.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/EmitC/IR/EmitC.h"
 #include "mlir/Dialect/EmitC/Transforms/TypeConversions.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index 84959d5ff8ab..844e66e927c4 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -29,7 +29,6 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
 #include "mlir/Transforms/RegionUtils.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
 
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
index 99631705851f..c816356cf3f9 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPUPass.cpp
@@ -10,14 +10,8 @@
 
 #include "mlir/Conversion/SCFToGPU/SCFToGPU.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/Support/CommandLine.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTAFFINEFORTOGPUPASS
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 71786e856c6d..584ac2f11b67 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index a59001de299f..78d13278fef5 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -12,10 +12,8 @@
 
 #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/Support/FormatVariadic.h"
 
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
index 3ef1d84ee264..d974f184c08f 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRVPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Conversion/IndexToSPIRV/IndexToSPIRV.h"
 #include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
 #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index 0e9eb9799c3e..d7ae9f0e94fe 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -11,8 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVMPass.h"
-
 #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
 #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
@@ -269,8 +267,9 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       copyInfo.push_back(info);
     }
     // Create a call to the kernel and copy the data back.
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, kernelFunc,
-                                              ArrayRef<Value>());
+    Operation *callOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+        op, kernelFunc, ArrayRef<Value>());
+    rewriter.setInsertionPointAfter(callOp);
     for (CopyInfo info : copyInfo)
       copy(loc, info.src, info.dst, info.size, rewriter);
     return success();
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index 2e4561e7cc3e..cea9d1fdec80 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -11,11 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h"
-#include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Conversion/SPIRVCommon/AttrToLLVMConverter.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
@@ -23,7 +21,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 
 #define DEBUG_TYPE "spirv-to-llvm-pattern"
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index 1f55fb7456f6..da9ad3dd6732 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -11,10 +11,8 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
-#include "mlir/Pass/PassRegistry.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp b/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
index b0ca0ca13d06..55eb11c87d23 100644
--- a/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
+++ b/mlir/lib/Conversion/TensorToLinalg/TensorToLinalg.cpp
@@ -12,9 +12,6 @@
 
 #include "mlir/Conversion/TensorToLinalg/TensorToLinalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "tensor-to-linalg-pattern"
 
diff --git a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
index ef3f4bc22fb8..2c4d27502a52 100644
--- a/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
+++ b/mlir/lib/Conversion/TensorToSPIRV/TensorToSPIRV.cpp
@@ -11,14 +11,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Conversion/TensorToSPIRV/TensorToSPIRV.h"
-#include "../SPIRVCommon/Pattern.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/AffineMap.h"
-#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "tensor-to-spirv-pattern"
 
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
index 9dea12355a51..40ad63610e23 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArith.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace tosa;
diff --git a/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp b/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
index ede3c9e0040f..46bdf2272ef0 100644
--- a/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
+++ b/mlir/lib/Conversion/TosaToArith/TosaToArithPass.cpp
@@ -14,11 +14,8 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOARITHPASS
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 923f5f67b865..1444b764e7c3 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -27,11 +27,9 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 
-#include <numeric>
 #include <type_traits>
 
 using namespace mlir;
@@ -310,8 +308,8 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     auto shifted =
         rewriter.create<arith::ShRSIOp>(loc, resultTypes, args[0], subtract)
             ->getResults();
-    auto truncated =
-        rewriter.create<arith::TruncIOp>(loc, i1Ty, shifted, std::nullopt);
+    auto truncated = rewriter.create<arith::TruncIOp>(
+        loc, i1Ty, shifted, ArrayRef<NamedAttribute>());
     auto isInputOdd =
         rewriter.create<arith::AndIOp>(loc, i1Ty, truncated, i1one);
 
@@ -552,20 +550,20 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && bitExtend)
       return rewriter.create<arith::ExtFOp>(loc, resultTypes, args,
-                                            std::nullopt);
+                                            ArrayRef<NamedAttribute>());
 
     if (isa<FloatType>(srcTy) && isa<FloatType>(dstTy) && !bitExtend)
       return rewriter.create<arith::TruncFOp>(loc, resultTypes, args,
-                                              std::nullopt);
+                                              ArrayRef<NamedAttribute>());
 
     // 1-bit integers need to be treated as signless.
     if (srcTy.isInteger(1) && arith::UIToFPOp::areCastCompatible(srcTy, dstTy))
       return rewriter.create<arith::UIToFPOp>(loc, resultTypes, args,
-                                              std::nullopt);
+                                              ArrayRef<NamedAttribute>());
 
     if (srcTy.isInteger(1) && isa<IntegerType>(dstTy) && bitExtend)
       return rewriter.create<arith::ExtUIOp>(loc, resultTypes, args,
-                                             std::nullopt);
+                                             ArrayRef<NamedAttribute>());
 
     // Unsigned integers need an unrealized cast so that they can be passed
     // to UIToFP.
@@ -583,7 +581,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
     // All other si-to-fp conversions should be handled by SIToFP.
     if (arith::SIToFPOp::areCastCompatible(srcTy, dstTy))
       return rewriter.create<arith::SIToFPOp>(loc, resultTypes, args,
-                                              std::nullopt);
+                                              ArrayRef<NamedAttribute>());
 
     // Casting to boolean, floats need to only be checked as not-equal to zero.
     if (isa<FloatType>(srcTy) && dstTy.isInteger(1)) {
@@ -690,7 +688,7 @@ static Value createLinalgBodyCalculationForElementwiseOp(
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && bitExtend)
       return rewriter.create<arith::ExtSIOp>(loc, resultTypes, args,
-                                             std::nullopt);
+                                             ArrayRef<NamedAttribute>());
 
     if (isa<IntegerType>(srcTy) && isa<IntegerType>(dstTy) && !bitExtend) {
       return rewriter.create<arith::TruncIOp>(loc, dstTy, args[0]);
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index c460a8bb2f4b..b89fde4fbc17 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -14,21 +14,13 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
-#include "mlir/Interfaces/InferTypeOpInterface.h"
-
-#include <numeric>
 #include <type_traits>
 
 using namespace mlir;
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
index 80df3908991d..3733d043f9eb 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamedPass.cpp
@@ -20,11 +20,9 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
-#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOLINALGNAMED
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
index 01a7cd7ac94d..c6a3ba9f1439 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgPass.cpp
@@ -21,11 +21,9 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
-#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/Passes.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
index 8c39f5e8a636..dba6f79d840a 100644
--- a/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
+++ b/mlir/lib/Conversion/TosaToMLProgram/TosaToMLProgramPass.cpp
@@ -13,7 +13,6 @@
 #include "mlir/Conversion/TosaToMLProgram/TosaToMLProgram.h"
 #include "mlir/Dialect/MLProgram/IR/MLProgram.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
index ef144fc7f0d5..03f9d20ad69d 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCF.cpp
@@ -14,9 +14,7 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace tosa;
diff --git a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
index 3fd72223d313..14dfd98f5d47 100644
--- a/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
+++ b/mlir/lib/Conversion/TosaToSCF/TosaToSCFPass.cpp
@@ -16,11 +16,9 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
-#include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOSCFPASS
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index 615c7ca1cfd1..3c7101202bb9 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -14,7 +14,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
 #include "mlir/IR/PatternMatch.h"
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
index e1dbe7db3048..a7f415c57484 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensorPass.cpp
@@ -17,9 +17,7 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_TOSATOTENSORPASS
diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
index 9017bcb5283c..18adaa793787 100644
--- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
+++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp
@@ -12,8 +12,6 @@
 
 #include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
 
-#include <type_traits>
-
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -28,11 +26,9 @@
 #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Region.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index d53d11f87efe..501d98862672 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -1985,6 +1985,38 @@ struct VectorFromElementsLowering
   }
 };
 
+/// Conversion pattern for a `vector.to_elements`.
+struct VectorToElementsLowering
+    : public ConvertOpToLLVMPattern<vector::ToElementsOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::ToElementsOp toElementsOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = toElementsOp.getLoc();
+    auto idxType = typeConverter->convertType(rewriter.getIndexType());
+    Value source = adaptor.getSource();
+
+    SmallVector<Value> results(toElementsOp->getNumResults());
+    for (auto [idx, element] : llvm::enumerate(toElementsOp.getElements())) {
+      // Create an extractelement operation only for results that are not dead.
+      if (element.use_empty())
+        continue;
+
+      auto constIdx = rewriter.create<LLVM::ConstantOp>(
+          loc, idxType, rewriter.getIntegerAttr(idxType, idx));
+      auto llvmType = typeConverter->convertType(element.getType());
+
+      Value result = rewriter.create<LLVM::ExtractElementOp>(loc, llvmType,
+                                                             source, constIdx);
+      results[idx] = result;
+    }
+
+    rewriter.replaceOp(toElementsOp, results);
+    return success();
+  }
+};
+
 /// Conversion pattern for vector.step.
 struct VectorScalableStepOpLowering
     : public ConvertOpToLLVMPattern<vector::StepOp> {
@@ -2035,7 +2067,8 @@ void mlir::populateVectorToLLVMConversionPatterns(
                VectorScalableInsertOpLowering, VectorScalableExtractOpLowering,
                MaskedReductionOpConversion, VectorInterleaveOpLowering,
                VectorDeinterleaveOpLowering, VectorFromElementsLowering,
-               VectorScalableStepOpLowering>(converter);
+               VectorToElementsLowering, VectorScalableStepOpLowering>(
+      converter);
 }
 
 void mlir::populateVectorToLLVMMatrixConversionPatterns(
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index 293e01a5bf4d..549d0210af7a 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/ArmNeon/Transforms.h"
 #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
 #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
@@ -85,7 +84,7 @@ void ConvertVectorToLLVMPass::runOnOperation() {
     populateVectorGatherLoweringPatterns(patterns);
     if (armI8MM) {
       if (armNeon)
-        arm_neon::populateLowerContractionToSMMLAPatternPatterns(patterns);
+        arm_neon::populateLowerContractionToNeonI8MMPatternPatterns(patterns);
       if (armSVE)
         populateLowerContractionToSVEI8MMPatternPatterns(patterns);
     }
diff --git a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
index cc5623068ab1..9f5b8fcca6c2 100644
--- a/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
+++ b/mlir/lib/Conversion/VectorToSCF/VectorToSCF.cpp
@@ -12,7 +12,6 @@
 
 #include <numeric>
 #include <optional>
-#include <type_traits>
 
 #include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
 
@@ -20,7 +19,6 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
@@ -29,7 +27,6 @@
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTVECTORTOSCF
@@ -1324,6 +1321,8 @@ struct UnrollTransferReadConversion
     for (int64_t i = 0; i < dimSize; ++i) {
       Value iv = rewriter.create<arith::ConstantIndexOp>(loc, i);
 
+      // FIXME: Rename this lambda - it does much more than just
+      // in-bounds-check generation.
       vec = generateInBoundsCheck(
           rewriter, xferOp, iv, unpackedDim(xferOp), TypeRange(vecType),
           /*inBoundsCase=*/
@@ -1338,12 +1337,21 @@ struct UnrollTransferReadConversion
             insertionIndices.push_back(rewriter.getIndexAttr(i));
 
             auto inBoundsAttr = dropFirstElem(b, xferOp.getInBoundsAttr());
+
             auto newXferOp = b.create<vector::TransferReadOp>(
                 loc, newXferVecType, xferOp.getBase(), xferIndices,
                 AffineMapAttr::get(unpackedPermutationMap(b, xferOp)),
                 xferOp.getPadding(), Value(), inBoundsAttr);
             maybeAssignMask(b, xferOp, newXferOp, i);
-            return b.create<vector::InsertOp>(loc, newXferOp, vec,
+
+            Value valToInser = newXferOp.getResult();
+            if (newXferVecType.getRank() == 0) {
+              // vector.insert does not accept rank-0 as the non-indexed
+              // argument. Extract the scalar before inserting.
+              valToInser = b.create<vector::ExtractOp>(loc, valToInser,
+                                                       SmallVector<int64_t>());
+            }
+            return b.create<vector::InsertOp>(loc, valToInser, vec,
                                               insertionIndices);
           },
           /*outOfBoundsCase=*/
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
index de2af69eba9e..21d8e1d9f115 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRV.cpp
@@ -1022,6 +1022,51 @@ struct VectorStepOpConvert final : OpConversionPattern<vector::StepOp> {
   }
 };
 
+struct VectorToElementOpConvert final
+    : OpConversionPattern<vector::ToElementsOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::ToElementsOp toElementsOp, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    SmallVector<Value> results(toElementsOp->getNumResults());
+    Location loc = toElementsOp.getLoc();
+
+    // Input vectors of size 1 are converted to scalars by the type converter.
+    // We cannot use `spirv::CompositeExtractOp` directly in this case.
+    // For a scalar source, the result is just the scalar itself.
+    if (isa<spirv::ScalarType>(adaptor.getSource().getType())) {
+      results[0] = adaptor.getSource();
+      rewriter.replaceOp(toElementsOp, results);
+      return success();
+    }
+
+    Type srcElementType = toElementsOp.getElements().getType().front();
+    Type elementType = getTypeConverter()->convertType(srcElementType);
+    if (!elementType)
+      return rewriter.notifyMatchFailure(
+          toElementsOp,
+          llvm::formatv("failed to convert element type '{0}' to SPIR-V",
+                        srcElementType));
+
+    for (auto [idx, element] : llvm::enumerate(toElementsOp.getElements())) {
+      // Create an CompositeExtract operation only for results that are not
+      // dead.
+      if (element.use_empty())
+        continue;
+
+      Value result = rewriter.create<spirv::CompositeExtractOp>(
+          loc, elementType, adaptor.getSource(),
+          rewriter.getI32ArrayAttr({static_cast<int32_t>(idx)}));
+      results[idx] = result;
+    }
+
+    rewriter.replaceOp(toElementsOp, results);
+    return success();
+  }
+};
+
 } // namespace
 #define CL_INT_MAX_MIN_OPS                                                     \
   spirv::CLUMaxOp, spirv::CLUMinOp, spirv::CLSMaxOp, spirv::CLSMinOp
@@ -1039,8 +1084,8 @@ void mlir::populateVectorToSPIRVPatterns(
       VectorExtractElementOpConvert, VectorExtractOpConvert,
       VectorExtractStridedSliceOpConvert, VectorFmaOpConvert<spirv::GLFmaOp>,
       VectorFmaOpConvert<spirv::CLFmaOp>, VectorFromElementsOpConvert,
-      VectorInsertElementOpConvert, VectorInsertOpConvert,
-      VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
+      VectorToElementOpConvert, VectorInsertElementOpConvert,
+      VectorInsertOpConvert, VectorReductionPattern<GL_INT_MAX_MIN_OPS>,
       VectorReductionPattern<CL_INT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<CL_FLOAT_MAX_MIN_OPS>,
       VectorReductionFloatMinMax<GL_FLOAT_MAX_MIN_OPS>, VectorShapeCast,
diff --git a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
index 710c74a70f9e..b3ef23085c18 100644
--- a/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
+++ b/mlir/lib/Conversion/VectorToSPIRV/VectorToSPIRVPass.cpp
@@ -17,7 +17,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 namespace mlir {
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index adcee1933e8a..0ec7129a40a6 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include <algorithm>
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 0d0add309466..4613d1446196 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -24,6 +24,7 @@
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include <limits>
@@ -524,6 +525,39 @@ LogicalResult GatherToLDSOp::verify() {
   return success();
 }
 
+LogicalResult TransposeLoadOp::verify() {
+  MemRefType srcType = cast<MemRefType>(getSrc().getType());
+
+  if (!hasWorkgroupMemorySpace(srcType.getMemorySpace()))
+    return emitOpError("source memory address space must be Workgroup");
+
+  auto transferType = cast<VectorType>(getType());
+  size_t numElements = transferType.getNumElements();
+  size_t elementTypeSize =
+      transferType.getElementType().getIntOrFloatBitWidth();
+
+  // ElementSize -> NumElements
+  const llvm::SmallDenseMap<size_t, size_t> KValidLoadSizeMap = {
+      {4, 16},
+      {6, 16},
+      {8, 8},
+      {16, 4},
+  };
+
+  auto validNumElems = KValidLoadSizeMap.find(elementTypeSize);
+  if (validNumElems == KValidLoadSizeMap.end()) {
+    return emitOpError("Unsupported element type size for transpose load: ")
+           << elementTypeSize << " bits";
+  }
+  if (numElements != validNumElems->second) {
+    return emitOpError(
+               "Transferring type size mismatch: expected num of elements: ")
+           << validNumElems->second;
+  }
+
+  return success();
+}
+
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc"
 
 #define GET_ATTRDEF_CLASSES
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
index 8709a27e0168..17bbe54ea6c0 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_mlir_dialect_library(MLIRAMDGPUTransforms
   EmulateAtomics.cpp
   ResolveStridedMetadata.cpp
-  TransferReadToLoad.cpp
+  MaskedloadToLoad.cpp
 
   ADDITIONAL_HEADER_DIRS
   {$MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/AMDGPU/Transforms
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
new file mode 100644
index 000000000000..9a368f372c29
--- /dev/null
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/MaskedloadToLoad.cpp
@@ -0,0 +1,167 @@
+//===- MaskedloadToLoad.cpp - Lowers maskedload to load -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
+
+#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/LogicalResult.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/Support/MathExtras.h"
+
+namespace mlir::amdgpu {
+#define GEN_PASS_DEF_AMDGPUMASKEDLOADTOLOADPASS
+#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
+} // namespace mlir::amdgpu
+
+using namespace mlir;
+using namespace mlir::amdgpu;
+
+/// This pattern supports lowering of: `vector.maskedload` to `vector.load`
+/// and `arith.select` if the memref is in buffer address space.
+static LogicalResult baseInBufferAddrSpace(PatternRewriter &rewriter,
+                                           vector::MaskedLoadOp maskedOp) {
+  auto memRefType = dyn_cast<MemRefType>(maskedOp.getBase().getType());
+  if (!memRefType)
+    return rewriter.notifyMatchFailure(maskedOp, "not a memref source");
+
+  Attribute addrSpace = memRefType.getMemorySpace();
+  if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(addrSpace))
+    return rewriter.notifyMatchFailure(maskedOp, "no address space");
+
+  if (dyn_cast<amdgpu::AddressSpaceAttr>(addrSpace).getValue() !=
+      amdgpu::AddressSpace::FatRawBuffer)
+    return rewriter.notifyMatchFailure(maskedOp, "not in buffer address space");
+
+  return success();
+}
+
+static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
+                                           vector::MaskedLoadOp maskedOp) {
+  VectorType vectorType = maskedOp.getVectorType();
+  Value load = builder.create<vector::LoadOp>(
+      loc, vectorType, maskedOp.getBase(), maskedOp.getIndices());
+  Value res = builder.create<arith::SelectOp>(
+      loc, vectorType, maskedOp.getMask(), load, maskedOp.getPassThru());
+  return res;
+}
+
+static constexpr char kMaskedloadNeedsMask[] =
+    "amdgpu.buffer_maskedload_needs_mask";
+
+namespace {
+
+struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedOp,
+                                PatternRewriter &rewriter) const override {
+    if (maskedOp->hasAttr(kMaskedloadNeedsMask))
+      return failure();
+
+    if (failed(baseInBufferAddrSpace(rewriter, maskedOp))) {
+      return failure();
+    }
+
+    Location loc = maskedOp.getLoc();
+    Value src = maskedOp.getBase();
+
+    VectorType vectorType = maskedOp.getVectorType();
+    int64_t vectorSize = vectorType.getNumElements();
+    int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
+    SmallVector<OpFoldResult> indices = maskedOp.getIndices();
+
+    auto stridedMetadata =
+        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
+    SmallVector<OpFoldResult> strides =
+        stridedMetadata.getConstifiedMixedStrides();
+    SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
+    OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
+    memref::LinearizedMemRefInfo linearizedInfo;
+    OpFoldResult linearizedIndices;
+    std::tie(linearizedInfo, linearizedIndices) =
+        memref::getLinearizedMemRefOffsetAndSize(rewriter, loc, elementBitWidth,
+                                                 elementBitWidth, offset, sizes,
+                                                 strides, indices);
+
+    // delta = bufferSize - linearizedOffset
+    Value vectorSizeOffset =
+        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
+    Value linearIndex =
+        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
+    Value totalSize = getValueOrCreateConstantIndexOp(
+        rewriter, loc, linearizedInfo.linearizedSize);
+    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
+
+    // 1) check if delta < vectorSize
+    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
+
+    // 2) check if (detla % elements_per_word != 0)
+    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
+        loc, llvm::divideCeil(32, elementBitWidth));
+    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::ne,
+        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
+        rewriter.create<arith::ConstantIndexOp>(loc, 0));
+
+    // We take the fallback of maskedload default lowering only it is both
+    // out-of-bounds and not word aligned. The fallback ensures correct results
+    // when loading at the boundary of the buffer since buffer load returns
+    // inconsistent zeros for the whole word when boundary is crossed.
+    Value ifCondition =
+        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
+
+    auto thenBuilder = [&](OpBuilder &builder, Location loc) {
+      Operation *read = builder.clone(*maskedOp.getOperation());
+      read->setAttr(kMaskedloadNeedsMask, builder.getUnitAttr());
+      Value readResult = read->getResult(0);
+      builder.create<scf::YieldOp>(loc, readResult);
+    };
+
+    auto elseBuilder = [&](OpBuilder &builder, Location loc) {
+      Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp);
+      rewriter.create<scf::YieldOp>(loc, res);
+    };
+
+    auto ifOp =
+        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
+
+    rewriter.replaceOp(maskedOp, ifOp);
+
+    return success();
+  }
+};
+
+} // namespace
+
+void mlir::amdgpu::populateAmdgpuMaskedloadToLoadPatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<MaskedLoadLowering>(patterns.getContext(), benefit);
+}
+
+struct AmdgpuMaskedloadToLoadPass final
+    : amdgpu::impl::AmdgpuMaskedloadToLoadPassBase<AmdgpuMaskedloadToLoadPass> {
+  void runOnOperation() override {
+    RewritePatternSet patterns(&getContext());
+    populateAmdgpuMaskedloadToLoadPatterns(patterns);
+    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) {
+      return signalPassFailure();
+    }
+  }
+};
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
deleted file mode 100644
index f5b12a9524cc..000000000000
--- a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
+++ /dev/null
@@ -1,239 +0,0 @@
-//===- TransferReadToLoad.cpp - Lowers masked transfer read to load -------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
-
-#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Dialect/Vector/IR/VectorOps.h"
-#include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/TypeUtilities.h"
-#include "mlir/Pass/Pass.h"
-#include "mlir/Support/LogicalResult.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/MathExtras.h"
-
-namespace mlir::amdgpu {
-#define GEN_PASS_DEF_AMDGPUTRANSFERREADTOLOADPASS
-#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
-} // namespace mlir::amdgpu
-
-using namespace mlir;
-using namespace mlir::amdgpu;
-
-/// This pattern supports lowering of:
-/// `vector.transfer_read` to a combination of `vector.load`, `arith.select` and
-/// `vector.broadcast` if all of the following hold:
-/// - The transfer op is masked.
-/// - The memref is in buffer address space.
-/// - Stride of most minor memref dimension must be 1.
-/// - Out-of-bounds masking is not required.
-/// - If the memref's element type is a vector type then it coincides with the
-///   result type.
-/// - The permutation map doesn't perform permutation (broadcasting is allowed).
-/// Note: those conditions mostly come from TransferReadToVectorLoadLowering
-/// pass.
-static LogicalResult transferPreconditions(
-    PatternRewriter &rewriter, VectorTransferOpInterface xferOp,
-    bool &requiresBroadcasting, VectorType &unbroadcastedVectorType) {
-  if (!xferOp.getMask())
-    return rewriter.notifyMatchFailure(xferOp, "Only support masked transfer");
-
-  // Permutations are handled by VectorToSCF or
-  // populateVectorTransferPermutationMapLoweringPatterns.
-  // We let the 0-d corner case pass-through as it is supported.
-  SmallVector<unsigned> broadcastedDims;
-  if (!xferOp.getPermutationMap().isMinorIdentityWithBroadcasting(
-          &broadcastedDims))
-    return rewriter.notifyMatchFailure(xferOp, "not minor identity + bcast");
-
-  auto memRefType = dyn_cast<MemRefType>(xferOp.getShapedType());
-  if (!memRefType)
-    return rewriter.notifyMatchFailure(xferOp, "not a memref source");
-
-  Attribute addrSpace = memRefType.getMemorySpace();
-  if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(addrSpace))
-    return rewriter.notifyMatchFailure(xferOp, "no address space");
-
-  if (dyn_cast<amdgpu::AddressSpaceAttr>(addrSpace).getValue() !=
-      amdgpu::AddressSpace::FatRawBuffer)
-    return rewriter.notifyMatchFailure(xferOp, "not in buffer address space");
-
-  // Non-unit strides are handled by VectorToSCF.
-  if (!memRefType.isLastDimUnitStride())
-    return rewriter.notifyMatchFailure(xferOp, "!= 1 stride needs VectorToSCF");
-
-  if (memRefType.getElementTypeBitWidth() < 8)
-    return rewriter.notifyMatchFailure(xferOp, "unsupported sub-byte type");
-
-  // If there is broadcasting involved then we first load the unbroadcasted
-  // vector, and then broadcast it with `vector.broadcast`.
-  ArrayRef<int64_t> vectorShape = xferOp.getVectorType().getShape();
-  SmallVector<int64_t> unbroadcastedVectorShape(vectorShape);
-  for (unsigned i : broadcastedDims)
-    unbroadcastedVectorShape[i] = 1;
-  unbroadcastedVectorType = xferOp.getVectorType().cloneWith(
-      unbroadcastedVectorShape, xferOp.getVectorType().getElementType());
-  requiresBroadcasting = !broadcastedDims.empty();
-
-  // `vector.load` supports vector types as memref's elements only when the
-  // resulting vector type is the same as the element type.
-  auto memrefElTy = memRefType.getElementType();
-  if (isa<VectorType>(memrefElTy) && memrefElTy != unbroadcastedVectorType)
-    return rewriter.notifyMatchFailure(xferOp, "incompatible element type");
-
-  // Otherwise, element types of the memref and the vector must match.
-  if (!isa<VectorType>(memrefElTy) &&
-      memrefElTy != xferOp.getVectorType().getElementType())
-    return rewriter.notifyMatchFailure(xferOp, "non-matching element type");
-
-  // Out-of-bounds dims are handled by MaterializeTransferMask.
-  if (xferOp.hasOutOfBoundsDim())
-    return rewriter.notifyMatchFailure(xferOp, "out-of-bounds needs mask");
-
-  if (xferOp.getVectorType().getRank() != 1)
-    // vector.maskedload operates on 1-D vectors.
-    return rewriter.notifyMatchFailure(
-        xferOp, "vector type is not rank 1, can't create masked load, needs "
-                "VectorToSCF");
-
-  return success();
-}
-
-static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
-                                           vector::TransferReadOp readOp,
-                                           bool requiresBroadcasting,
-                                           VectorType unbroadcastedVectorType) {
-  Value fill = builder.create<vector::SplatOp>(loc, unbroadcastedVectorType,
-                                               readOp.getPadding());
-  Value load = builder.create<vector::LoadOp>(
-      loc, unbroadcastedVectorType, readOp.getBase(), readOp.getIndices());
-  Value res = builder.create<arith::SelectOp>(loc, unbroadcastedVectorType,
-                                              readOp.getMask(), load, fill);
-  // Insert a broadcasting op if required.
-  if (requiresBroadcasting) {
-    res = builder.create<vector::BroadcastOp>(loc, readOp.getVectorType(), res);
-  }
-  return res;
-}
-
-static constexpr char kTransferReadNeedsMask[] =
-    "amdgpu.buffer_transfer_read_needs_mask";
-
-namespace {
-
-struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
-                                PatternRewriter &rewriter) const override {
-    if (readOp->hasAttr(kTransferReadNeedsMask))
-      return failure();
-
-    bool requiresBroadcasting = false;
-    VectorType unbroadcastedVectorType;
-    if (failed(transferPreconditions(rewriter, readOp, requiresBroadcasting,
-                                     unbroadcastedVectorType))) {
-      return failure();
-    }
-
-    Location loc = readOp.getLoc();
-    Value src = readOp.getBase();
-
-    VectorType vectorType = readOp.getVectorType();
-    int64_t vectorSize = vectorType.getNumElements();
-    int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
-    SmallVector<OpFoldResult> indices = readOp.getIndices();
-
-    auto stridedMetadata =
-        rewriter.create<memref::ExtractStridedMetadataOp>(loc, src);
-    SmallVector<OpFoldResult> strides =
-        stridedMetadata.getConstifiedMixedStrides();
-    SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
-    OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
-    memref::LinearizedMemRefInfo linearizedInfo;
-    OpFoldResult linearizedIndices;
-    std::tie(linearizedInfo, linearizedIndices) =
-        memref::getLinearizedMemRefOffsetAndSize(rewriter, loc, elementBitWidth,
-                                                 elementBitWidth, offset, sizes,
-                                                 strides, indices);
-
-    // delta = bufferSize - linearizedOffset
-    Value vectorSizeOffset =
-        rewriter.create<arith::ConstantIndexOp>(loc, vectorSize);
-    Value linearIndex =
-        getValueOrCreateConstantIndexOp(rewriter, loc, linearizedIndices);
-    Value totalSize = getValueOrCreateConstantIndexOp(
-        rewriter, loc, linearizedInfo.linearizedSize);
-    Value delta = rewriter.create<arith::SubIOp>(loc, totalSize, linearIndex);
-
-    // 1) check if delta < vectorSize
-    Value isOutofBounds = rewriter.create<arith::CmpIOp>(
-        loc, arith::CmpIPredicate::ult, delta, vectorSizeOffset);
-
-    // 2) check if (detla % elements_per_word != 0)
-    Value elementsPerWord = rewriter.create<arith::ConstantIndexOp>(
-        loc, llvm::divideCeil(32, elementBitWidth));
-    Value isNotWordAligned = rewriter.create<arith::CmpIOp>(
-        loc, arith::CmpIPredicate::ne,
-        rewriter.create<arith::RemUIOp>(loc, delta, elementsPerWord),
-        rewriter.create<arith::ConstantIndexOp>(loc, 0));
-
-    // We take the fallback of transfer_read default lowering only it is both
-    // out-of-bounds and not word aligned. The fallback ensures correct results
-    // when loading at the boundary of the buffer since buffer load returns
-    // inconsistent zeros for the whole word when boundary is crossed.
-    Value ifCondition =
-        rewriter.create<arith::AndIOp>(loc, isOutofBounds, isNotWordAligned);
-
-    auto thenBuilder = [&](OpBuilder &builder, Location loc) {
-      Operation *read = builder.clone(*readOp.getOperation());
-      read->setAttr(kTransferReadNeedsMask, builder.getUnitAttr());
-      Value readResult = read->getResult(0);
-      builder.create<scf::YieldOp>(loc, readResult);
-    };
-
-    auto elseBuilder = [&](OpBuilder &builder, Location loc) {
-      Value res = createVectorLoadForMaskedLoad(
-          builder, loc, readOp, requiresBroadcasting, unbroadcastedVectorType);
-      rewriter.create<scf::YieldOp>(loc, res);
-    };
-
-    auto ifOp =
-        rewriter.create<scf::IfOp>(loc, ifCondition, thenBuilder, elseBuilder);
-
-    rewriter.replaceOp(readOp, ifOp);
-
-    return success();
-  }
-};
-
-} // namespace
-
-void mlir::amdgpu::populateAmdgpuTransferReadToLoadPatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<TransferReadLowering>(patterns.getContext(), benefit);
-}
-
-struct AmdgpuTransferReadToLoadPass final
-    : amdgpu::impl::AmdgpuTransferReadToLoadPassBase<
-          AmdgpuTransferReadToLoadPass> {
-  void runOnOperation() override {
-    RewritePatternSet patterns(&getContext());
-    populateAmdgpuTransferReadToLoadPatterns(patterns);
-    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns)))) {
-      return signalPassFailure();
-    }
-  }
-};
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
index 6f79665c2bb6..a8067349344b 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineAnalysis.cpp
@@ -391,7 +391,7 @@ static void addOrderingConstraints(const FlatAffineValueConstraints &srcDomain,
   unsigned numCommonLoops = getNumCommonLoops(srcDomain, dstDomain);
   unsigned numCommonLoopConstraints = std::min(numCommonLoops, loopDepth);
   for (unsigned i = 0; i < numCommonLoopConstraints; ++i) {
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     eq[i] = -1;
     eq[i + numSrcDims] = 1;
     if (i == loopDepth - 1) {
@@ -433,7 +433,7 @@ static void computeDirectionVector(
   // Constraint variables format:
   // [num-common-loops][num-src-dim-ids][num-dst-dim-ids][num-symbols][constant]
   for (unsigned j = 0; j < numCommonLoops; ++j) {
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     eq[j] = 1;
     eq[j + numCommonLoops] = 1;
     eq[j + numCommonLoops + numSrcDims] = -1;
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
index f344d59234e6..150a08bf17e1 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
@@ -526,7 +526,7 @@ LogicalResult mlir::affine::getRelationFromMap(AffineMap &map,
   SmallVector<int64_t, 8> eq(localVarCst.getNumCols());
   for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
     // Zero fill.
-    std::fill(eq.begin(), eq.end(), 0);
+    llvm::fill(eq, 0);
     // Fill equality.
     for (unsigned j = 0, f = oldDimNum; j < f; ++j)
       eq[j] = flatExprs[i][j];
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index 3144ca723df9..87fc8173a964 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -1432,7 +1432,7 @@ LogicalResult mlir::affine::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
 
     // Check for a negative index.
     FlatAffineValueConstraints lcst(*region.getConstraints());
-    std::fill(ineq.begin(), ineq.end(), 0);
+    llvm::fill(ineq, 0);
     // d_i <= -1;
     lcst.addBound(BoundType::UB, r, -1);
     outOfBounds = !lcst.isEmpty();
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 3b4d51d914d8..bd4b2e56808b 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -42,6 +42,7 @@ using llvm::divideFloorSigned;
 using llvm::mod;
 
 #define DEBUG_TYPE "affine-ops"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
 
 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc"
 
@@ -1046,59 +1047,85 @@ simplifyMapWithOperands(AffineMap &map, ArrayRef<Value> operands) {
                        map.getContext());
 }
 
-/// Assuming `dimOrSym` is a quantity in `map` that is defined by `minOp`.
-/// Assuming that the quantity is of the form:
-///   `affine_min(f(x, y), symbolic_cst)`.
-/// This function checks that `0 < affine_min(f(x, y), symbolic_cst)` and
-/// proceeds with replacing the patterns:
+/// Assuming `dimOrSym` is a quantity in the apply op map `map` and defined by
+/// `minOp = affine_min(x_1, ..., x_n)`. This function checks that:
+/// `0 < affine_min(x_1, ..., x_n)` and proceeds with replacing the patterns:
 /// ```
-///   dimOrSym.ceildiv(symbolic_cst)
-///   (dimOrSym + symbolic_cst - 1).floordiv(symbolic_cst)
+///   dimOrSym.ceildiv(x_k)
+///   (dimOrSym + x_k - 1).floordiv(x_k)
 /// ```
-/// by `1`.
+/// by `1` for all `k` in `1, ..., n`. This is possible because `x / x_k <= 1`.
 ///
-/// Additionally, allows the caller to pass `affineMinKnownToBeNonNegative` to
-/// inject static information that may not be statically discoverable.
 ///
 /// Warning: ValueBoundsConstraintSet::computeConstantBound is needed to check
-/// for the nonnegative case, if `affineMinKnownToBeNonNegative` is false.
-static LogicalResult replaceAffineMinBoundingBoxExpression(
-    AffineMinOp minOp, AffineExpr dimOrSym, AffineMap *map,
-    bool affineMinKnownToBeNonNegative = false) {
-  auto affineMinMap = minOp.getAffineMap();
-  if (!affineMinKnownToBeNonNegative) {
-    ValueRange values = minOp->getOperands();
-    for (unsigned i = 0, e = affineMinMap.getNumResults(); i < e; ++i) {
-      AffineMap row = affineMinMap.getSubMap(ArrayRef<unsigned>{i});
-      FailureOr<int64_t> lowerBound =
-          ValueBoundsConstraintSet::computeConstantBound(
-              presburger::BoundType::LB, {row, values},
-              /*stopCondition=*/nullptr,
-              /*closedUB=*/true);
-      if (failed(lowerBound) || lowerBound.value() <= 0)
-        return failure();
+/// `minOp` is positive.
+static LogicalResult replaceAffineMinBoundingBoxExpression(AffineMinOp minOp,
+                                                           AffineExpr dimOrSym,
+                                                           AffineMap *map,
+                                                           ValueRange dims,
+                                                           ValueRange syms) {
+  AffineMap affineMinMap = minOp.getAffineMap();
+
+  LLVM_DEBUG({
+    DBGS() << "replaceAffineMinBoundingBoxExpression: `" << minOp << "`\n";
+  });
+
+  // Check the value is positive.
+  for (unsigned i = 0, e = affineMinMap.getNumResults(); i < e; ++i) {
+    // Compare each expression in the minimum against 0.
+    if (!ValueBoundsConstraintSet::compare(
+            getAsIndexOpFoldResult(minOp.getContext(), 0),
+            ValueBoundsConstraintSet::ComparisonOperator::LT,
+            ValueBoundsConstraintSet::Variable(affineMinMap.getSliceMap(i, 1),
+                                               minOp.getOperands())))
+      return failure();
+  }
+
+  /// Convert affine symbols and dimensions in minOp to symbols or dimensions in
+  /// the apply op affine map.
+  DenseMap<AffineExpr, AffineExpr> dimSymConversionTable;
+  SmallVector<unsigned> unmappedDims, unmappedSyms;
+  for (auto [i, dim] : llvm::enumerate(minOp.getDimOperands())) {
+    auto it = llvm::find(dims, dim);
+    if (it == dims.end()) {
+      unmappedDims.push_back(i);
+      continue;
+    }
+    dimSymConversionTable[getAffineDimExpr(i, minOp.getContext())] =
+        getAffineDimExpr(it.getIndex(), minOp.getContext());
+  }
+  for (auto [i, sym] : llvm::enumerate(minOp.getSymbolOperands())) {
+    auto it = llvm::find(syms, sym);
+    if (it == syms.end()) {
+      unmappedSyms.push_back(i);
+      continue;
     }
+    dimSymConversionTable[getAffineSymbolExpr(i, minOp.getContext())] =
+        getAffineSymbolExpr(it.getIndex(), minOp.getContext());
   }
 
-  AffineMap initialMap = *map;
-  for (unsigned i = 0, e = affineMinMap.getNumResults(); i != e; ++i) {
-    auto m = affineMinMap.getSubMap(ArrayRef<unsigned>{i});
-    AffineExpr expr = m.getResult(0);
-    if (!expr.isSymbolicOrConstant())
+  // Create the replacement map.
+  DenseMap<AffineExpr, AffineExpr> repl;
+  AffineExpr c1 = getAffineConstantExpr(1, minOp.getContext());
+  for (AffineExpr expr : affineMinMap.getResults()) {
+    // If we cannot express the result in terms of the apply map symbols and
+    // sims then continue.
+    if (llvm::any_of(unmappedDims,
+                     [&](unsigned i) { return expr.isFunctionOfDim(i); }) ||
+        llvm::any_of(unmappedSyms,
+                     [&](unsigned i) { return expr.isFunctionOfSymbol(i); }))
       continue;
 
-    DenseMap<AffineExpr, AffineExpr> repl;
+    AffineExpr convertedExpr = expr.replace(dimSymConversionTable);
+
     // dimOrSym.ceilDiv(expr) -> 1
-    repl[dimOrSym.ceilDiv(expr)] = getAffineConstantExpr(1, minOp.getContext());
+    repl[dimOrSym.ceilDiv(convertedExpr)] = c1;
     // (dimOrSym + expr - 1).floorDiv(expr) -> 1
-    repl[(dimOrSym + expr - 1).floorDiv(expr)] =
-        getAffineConstantExpr(1, minOp.getContext());
-    auto newMap = map->replace(repl);
-    if (newMap == *map)
-      continue;
-    *map = newMap;
+    repl[(dimOrSym + convertedExpr - 1).floorDiv(convertedExpr)] = c1;
   }
-
+  AffineMap initialMap = *map;
+  *map = initialMap.replace(repl, initialMap.getNumDims(),
+                            initialMap.getNumSymbols());
   return success(*map != initialMap);
 }
 
@@ -1127,11 +1154,11 @@ static LogicalResult replaceDimOrSym(AffineMap *map,
   if (!v)
     return failure();
 
-  auto minOp = v.getDefiningOp<AffineMinOp>();
-  if (minOp && replaceAffineMin) {
+  if (auto minOp = v.getDefiningOp<AffineMinOp>(); minOp && replaceAffineMin) {
     AffineExpr dimOrSym = isDimReplacement ? getAffineDimExpr(pos, ctx)
                                            : getAffineSymbolExpr(pos, ctx);
-    return replaceAffineMinBoundingBoxExpression(minOp, dimOrSym, map);
+    return replaceAffineMinBoundingBoxExpression(minOp, dimOrSym, map, dims,
+                                                 syms);
   }
 
   auto affineApply = v.getDefiningOp<AffineApplyOp>();
@@ -1241,6 +1268,12 @@ void mlir::affine::fullyComposeAffineMapAndOperands(
   })) {
     composeAffineMapAndOperands(map, operands, composeAffineMin);
   }
+  // Additional trailing step for AffineMinOps in case no chains of AffineApply.
+  if (composeAffineMin && llvm::any_of(*operands, [](Value v) {
+        return isa_and_nonnull<AffineMinOp>(v.getDefiningOp());
+      })) {
+    composeAffineMapAndOperands(map, operands, composeAffineMin);
+  }
 }
 
 AffineApplyOp
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 4b4eb9ce37b4..8f42586e5d18 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -445,10 +445,15 @@ static Value createPrivateMemRef(AffineForOp forOp,
   // Replace all users of 'oldMemRef' with 'newMemRef'.
   Operation *domFilter =
       getDominanceFilterForPrivateMemRefRepl(sliceInsertionBlock, storeOps);
+  auto userFilterFn = [&](Operation *user) {
+    auto domInfo = std::make_unique<DominanceInfo>(
+        domFilter->getParentOfType<FunctionOpInterface>());
+    return domInfo->dominates(domFilter, user);
+  };
   LogicalResult res = replaceAllMemRefUsesWith(
       oldMemRef, newMemRef, /*extraIndices=*/{}, indexRemap,
       /*extraOperands=*/outerIVs,
-      /*symbolOperands=*/{}, domFilter);
+      /*symbolOperands=*/{}, userFilterFn);
   assert(succeeded(res) &&
          "replaceAllMemrefUsesWith should always succeed here");
   (void)res;
@@ -997,6 +1002,8 @@ struct GreedyFusion {
           if (producerConsumerMemrefs.count(
                   cast<AffineWriteOpInterface>(op).getMemRef()))
             dstMemrefOps.push_back(op);
+        if (dstMemrefOps.empty())
+          continue;
         unsigned dstLoopDepthTest =
             getInnermostCommonLoopDepth(dstMemrefOps) - numSurroundingLoops;
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
index 45bda68bb863..b1b0143a08ee 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopTiling.cpp
@@ -121,7 +121,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   // If the cache size is zero, set the minimum valid tile size. No good reason
   // to pick another specific size over this.
   if (cacheSizeInKiB == 0) {
-    std::fill(tileSizes->begin(), tileSizes->end(), 1);
+    llvm::fill(*tileSizes, 1);
     return;
   }
 
@@ -136,8 +136,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   std::optional<int64_t> fp = getMemoryFootprintBytes(band[0], 0);
   if (!fp) {
     // Fill with default tile sizes if footprint is unknown.
-    std::fill(tileSizes->begin(), tileSizes->end(),
-              LoopTiling::kDefaultTileSize);
+    llvm::fill(*tileSizes, LoopTiling::kDefaultTileSize);
     if (avoidMaxMinBounds)
       adjustToDivisorsOfTripCounts(band, tileSizes);
     LLVM_DEBUG(
@@ -151,7 +150,7 @@ void LoopTiling::getTileSizes(ArrayRef<AffineForOp> band,
   uint64_t excessFactor = llvm::divideCeil(*fp, cacheSizeBytes);
   if (excessFactor <= 1) {
     // No need of any tiling - set tile size to 1.
-    std::fill(tileSizes->begin(), tileSizes->end(), 1);
+    llvm::fill(*tileSizes, 1);
     return;
   }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
index 4be99aa19738..92cb7075005a 100644
--- a/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/PipelineDataTransfer.cpp
@@ -115,13 +115,16 @@ static bool doubleBuffer(Value oldMemRef, AffineForOp forOp) {
 
   // replaceAllMemRefUsesWith will succeed unless the forOp body has
   // non-dereferencing uses of the memref (dealloc's are fine though).
-  if (failed(replaceAllMemRefUsesWith(
-          oldMemRef, newMemRef,
-          /*extraIndices=*/{ivModTwoOp},
-          /*indexRemap=*/AffineMap(),
-          /*extraOperands=*/{},
-          /*symbolOperands=*/{},
-          /*domOpFilter=*/&*forOp.getBody()->begin()))) {
+  auto userFilterFn = [&](Operation *user) {
+    auto domInfo = std::make_unique<DominanceInfo>(
+        forOp->getParentOfType<FunctionOpInterface>());
+    return domInfo->dominates(&*forOp.getBody()->begin(), user);
+  };
+  if (failed(replaceAllMemRefUsesWith(oldMemRef, newMemRef,
+                                      /*extraIndices=*/{ivModTwoOp},
+                                      /*indexRemap=*/AffineMap(),
+                                      /*extraOperands=*/{},
+                                      /*symbolOperands=*/{}, userFilterFn))) {
     LLVM_DEBUG(
         forOp.emitError("memref replacement for double buffering failed"));
     ivModTwoOp.erase();
diff --git a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
index c992badcfa49..2cb78d2b94c6 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SimplifyAffineMinMax.cpp
@@ -10,13 +10,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Affine/Passes.h"
+
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
 #include "mlir/IR/PatternMatch.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
+#include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/IntEqClasses.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
 
 #define DEBUG_TYPE "affine-min-max"
 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
@@ -44,6 +49,12 @@ static bool simplifyAffineMinMaxOp(RewriterBase &rewriter, AffineOp affineOp) {
       [&](unsigned i) {
         return Variable(affineMap.getSliceMap(i, 1), operands);
       });
+  LLVM_DEBUG({
+    DBGS() << "- constructed variables are: "
+           << llvm::interleaved_array(llvm::map_range(
+                  variables, [](const Variable &v) { return v.getMap(); }))
+           << "`\n";
+  });
 
   // Get the comparison operation.
   ComparisonOperator cmpOp =
@@ -125,8 +136,17 @@ static bool simplifyAffineMinMaxOp(RewriterBase &rewriter, AffineOp affineOp) {
   for (auto [k, bound] : bounds)
     results.push_back(bound->getMap().getResult(0));
 
-  affineMap = AffineMap::get(affineMap.getNumDims(), affineMap.getNumSymbols(),
-                             results, rewriter.getContext());
+  LLVM_DEBUG({
+    DBGS() << "- starting from map: " << affineMap << "\n";
+    DBGS() << "- creating new map with: \n";
+    DBGS() << "--- dims: " << affineMap.getNumDims() << "\n";
+    DBGS() << "--- syms: " << affineMap.getNumSymbols() << "\n";
+    DBGS() << "--- res: " << llvm::interleaved_array(results) << "\n";
+  });
+
+  affineMap =
+      AffineMap::get(0, affineMap.getNumSymbols() + affineMap.getNumDims(),
+                     results, rewriter.getContext());
 
   // Update the affine op.
   rewriter.modifyOpInPlace(affineOp, [&]() { affineOp.setMap(affineMap); });
@@ -172,3 +192,74 @@ LogicalResult mlir::affine::simplifyAffineMinMaxOps(RewriterBase &rewriter,
     *modified = changed;
   return success();
 }
+
+namespace {
+
+struct SimplifyAffineMaxOp : public OpRewritePattern<AffineMaxOp> {
+  using OpRewritePattern<AffineMaxOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(AffineMaxOp affineOp,
+                                PatternRewriter &rewriter) const override {
+    return success(simplifyAffineMaxOp(rewriter, affineOp));
+  }
+};
+
+struct SimplifyAffineMinOp : public OpRewritePattern<AffineMinOp> {
+  using OpRewritePattern<AffineMinOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(AffineMinOp affineOp,
+                                PatternRewriter &rewriter) const override {
+    return success(simplifyAffineMinOp(rewriter, affineOp));
+  }
+};
+
+struct SimplifyAffineApplyOp : public OpRewritePattern<AffineApplyOp> {
+  using OpRewritePattern<AffineApplyOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(AffineApplyOp affineOp,
+                                PatternRewriter &rewriter) const override {
+    AffineMap map = affineOp.getAffineMap();
+    SmallVector<Value> operands{affineOp->getOperands().begin(),
+                                affineOp->getOperands().end()};
+    fullyComposeAffineMapAndOperands(&map, &operands,
+                                     /*composeAffineMin=*/true);
+
+    // No change => failure to apply.
+    if (map == affineOp.getAffineMap())
+      return failure();
+
+    rewriter.modifyOpInPlace(affineOp, [&]() {
+      affineOp.setMap(map);
+      affineOp->setOperands(operands);
+    });
+    return success();
+  }
+};
+
+} // namespace
+
+namespace mlir {
+namespace affine {
+#define GEN_PASS_DEF_SIMPLIFYAFFINEMINMAXPASS
+#include "mlir/Dialect/Affine/Passes.h.inc"
+} // namespace affine
+} // namespace mlir
+
+/// Creates a simplification pass for affine min/max/apply.
+struct SimplifyAffineMinMaxPass
+    : public affine::impl::SimplifyAffineMinMaxPassBase<
+          SimplifyAffineMinMaxPass> {
+  void runOnOperation() override;
+};
+
+void SimplifyAffineMinMaxPass::runOnOperation() {
+  FunctionOpInterface func = getOperation();
+  RewritePatternSet patterns(func.getContext());
+  AffineMaxOp::getCanonicalizationPatterns(patterns, func.getContext());
+  AffineMinOp::getCanonicalizationPatterns(patterns, func.getContext());
+  patterns.add<SimplifyAffineMaxOp, SimplifyAffineMinOp, SimplifyAffineApplyOp>(
+      func.getContext());
+  FrozenRewritePatternSet frozenPatterns(std::move(patterns));
+  if (failed(applyPatternsGreedily(func, std::move(frozenPatterns))))
+    return signalPassFailure();
+}
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index f6f192a6d964..7fae260767e0 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -1257,7 +1257,8 @@ static Operation *vectorizeAffineLoad(AffineLoadOp loadOp,
   LLVM_DEBUG(permutationMap.print(dbgs()));
 
   auto transfer = state.builder.create<vector::TransferReadOp>(
-      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices, permutationMap);
+      loadOp.getLoc(), vectorType, loadOp.getMemRef(), indices,
+      /*padding=*/std::nullopt, permutationMap);
 
   // Register replacement for future uses in the scope.
   state.registerOpVectorReplacement(loadOp, transfer);
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 4aa1fe318efa..0501616ad912 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -1967,6 +1967,12 @@ static LogicalResult generateCopy(
   if (begin == end)
     return success();
 
+  // Record the last op in the block for which we are performing copy
+  // generation. We later do the memref replacement only in [begin, lastCopyOp]
+  // so that the original memref's used in the data movement code themselves
+  // don't get replaced.
+  Operation *lastCopyOp = end->getPrevNode();
+
   // Is the copy out point at the end of the block where we are doing
   // explicit copying.
   bool isCopyOutAtEndOfBlock = (end == copyOutPlacementStart);
@@ -2143,12 +2149,6 @@ static LogicalResult generateCopy(
     }
   }
 
-  // Record the last operation where we want the memref replacement to end. We
-  // later do the memref replacement only in [begin, postDomFilter] so
-  // that the original memref's used in the data movement code themselves don't
-  // get replaced.
-  auto postDomFilter = std::prev(end);
-
   // Create fully composed affine maps for each memref.
   auto memAffineMap = b.getMultiDimIdentityMap(memIndices.size());
   fullyComposeAffineMapAndOperands(&memAffineMap, &memIndices);
@@ -2244,13 +2244,17 @@ static LogicalResult generateCopy(
   if (!isBeginAtStartOfBlock)
     prevOfBegin = std::prev(begin);
 
+  auto userFilterFn = [&](Operation *user) {
+    auto *ancestorUser = block->findAncestorOpInBlock(*user);
+    return ancestorUser && !ancestorUser->isBeforeInBlock(&*begin) &&
+           !lastCopyOp->isBeforeInBlock(ancestorUser);
+  };
+
   // *Only* those uses within the range [begin, end) of 'block' are replaced.
   (void)replaceAllMemRefUsesWith(memref, fastMemRef,
                                  /*extraIndices=*/{}, indexRemap,
                                  /*extraOperands=*/regionSymbols,
-                                 /*symbolOperands=*/{},
-                                 /*domOpFilter=*/&*begin,
-                                 /*postDomOpFilter=*/&*postDomFilter);
+                                 /*symbolOperands=*/{}, userFilterFn);
 
   *nBegin = isBeginAtStartOfBlock ? block->begin() : std::next(prevOfBegin);
 
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index cde822310785..66b3f2a4f93a 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -1305,9 +1305,9 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
 LogicalResult mlir::affine::replaceAllMemRefUsesWith(
     Value oldMemRef, Value newMemRef, ArrayRef<Value> extraIndices,
     AffineMap indexRemap, ArrayRef<Value> extraOperands,
-    ArrayRef<Value> symbolOperands, Operation *domOpFilter,
-    Operation *postDomOpFilter, bool allowNonDereferencingOps,
-    bool replaceInDeallocOp) {
+    ArrayRef<Value> symbolOperands,
+    llvm::function_ref<bool(Operation *)> userFilterFn,
+    bool allowNonDereferencingOps, bool replaceInDeallocOp) {
   unsigned newMemRefRank = cast<MemRefType>(newMemRef.getType()).getRank();
   (void)newMemRefRank; // unused in opt mode
   unsigned oldMemRefRank = cast<MemRefType>(oldMemRef.getType()).getRank();
@@ -1328,61 +1328,52 @@ LogicalResult mlir::affine::replaceAllMemRefUsesWith(
 
   std::unique_ptr<DominanceInfo> domInfo;
   std::unique_ptr<PostDominanceInfo> postDomInfo;
-  if (domOpFilter)
-    domInfo = std::make_unique<DominanceInfo>(
-        domOpFilter->getParentOfType<FunctionOpInterface>());
-
-  if (postDomOpFilter)
-    postDomInfo = std::make_unique<PostDominanceInfo>(
-        postDomOpFilter->getParentOfType<FunctionOpInterface>());
 
   // Walk all uses of old memref; collect ops to perform replacement. We use a
   // DenseSet since an operation could potentially have multiple uses of a
   // memref (although rare), and the replacement later is going to erase ops.
   DenseSet<Operation *> opsToReplace;
-  for (auto *op : oldMemRef.getUsers()) {
-    // Skip this use if it's not dominated by domOpFilter.
-    if (domOpFilter && !domInfo->dominates(domOpFilter, op))
-      continue;
-
-    // Skip this use if it's not post-dominated by postDomOpFilter.
-    if (postDomOpFilter && !postDomInfo->postDominates(postDomOpFilter, op))
+  for (auto *user : oldMemRef.getUsers()) {
+    // Check if this user doesn't pass the filter.
+    if (userFilterFn && !userFilterFn(user))
       continue;
 
     // Skip dealloc's - no replacement is necessary, and a memref replacement
     // at other uses doesn't hurt these dealloc's.
-    if (hasSingleEffect<MemoryEffects::Free>(op, oldMemRef) &&
+    if (hasSingleEffect<MemoryEffects::Free>(user, oldMemRef) &&
         !replaceInDeallocOp)
       continue;
 
     // Check if the memref was used in a non-dereferencing context. It is fine
     // for the memref to be used in a non-dereferencing way outside of the
     // region where this replacement is happening.
-    if (!isa<AffineMapAccessInterface>(*op)) {
+    if (!isa<AffineMapAccessInterface>(*user)) {
       if (!allowNonDereferencingOps) {
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Memref replacement failed: non-deferencing memref op: \n"
-                   << *op << '\n');
+        LLVM_DEBUG(
+            llvm::dbgs()
+            << "Memref replacement failed: non-deferencing memref user: \n"
+            << *user << '\n');
         return failure();
       }
       // Non-dereferencing ops with the MemRefsNormalizable trait are
       // supported for replacement.
-      if (!op->hasTrait<OpTrait::MemRefsNormalizable>()) {
+      if (!user->hasTrait<OpTrait::MemRefsNormalizable>()) {
         LLVM_DEBUG(llvm::dbgs() << "Memref replacement failed: use without a "
                                    "memrefs normalizable trait: \n"
-                                << *op << '\n');
+                                << *user << '\n');
         return failure();
       }
     }
 
-    // We'll first collect and then replace --- since replacement erases the op
-    // that has the use, and that op could be postDomFilter or domFilter itself!
-    opsToReplace.insert(op);
+    // We'll first collect and then replace --- since replacement erases the
+    // user that has the use, and that user could be postDomFilter or domFilter
+    // itself!
+    opsToReplace.insert(user);
   }
 
-  for (auto *op : opsToReplace) {
+  for (auto *user : opsToReplace) {
     if (failed(replaceAllMemRefUsesWith(
-            oldMemRef, newMemRef, op, extraIndices, indexRemap, extraOperands,
+            oldMemRef, newMemRef, user, extraIndices, indexRemap, extraOperands,
             symbolOperands, allowNonDereferencingOps)))
       llvm_unreachable("memref replacement guaranteed to succeed here");
   }
@@ -1763,8 +1754,7 @@ LogicalResult mlir::affine::normalizeMemRef(AllocLikeOp allocOp) {
                                       /*indexRemap=*/layoutMap,
                                       /*extraOperands=*/{},
                                       /*symbolOperands=*/symbolOperands,
-                                      /*domOpFilter=*/nullptr,
-                                      /*postDomOpFilter=*/nullptr,
+                                      /*userFilterFn=*/nullptr,
                                       /*allowNonDereferencingOps=*/true))) {
     // If it failed (due to escapes for example), bail out.
     newAlloc.erase();
@@ -1854,8 +1844,7 @@ mlir::affine::normalizeMemRef(memref::ReinterpretCastOp reinterpretCastOp) {
                                       /*indexRemap=*/oldLayoutMap,
                                       /*extraOperands=*/{},
                                       /*symbolOperands=*/oldStrides,
-                                      /*domOpFilter=*/nullptr,
-                                      /*postDomOpFilter=*/nullptr,
+                                      /*userFilterFn=*/nullptr,
                                       /*allowNonDereferencingOps=*/true))) {
     // If it failed (due to escapes for example), bail out.
     newReinterpretCast.erase();
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 5194f2b58669..ae2a00cedf6f 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -262,6 +262,12 @@ void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
                            builder.getIntegerAttr(type, value));
 }
 
+void arith::ConstantIntOp::build(OpBuilder &builder, OperationState &result,
+                                 Type type, const APInt &value) {
+  arith::ConstantOp::build(builder, result, type,
+                           builder.getIntegerAttr(type, value));
+}
+
 bool arith::ConstantIntOp::classof(Operation *op) {
   if (auto constOp = dyn_cast_or_null<arith::ConstantOp>(op))
     return constOp.getType().isSignlessInteger();
@@ -292,6 +298,16 @@ bool arith::ConstantIndexOp::classof(Operation *op) {
   return false;
 }
 
+Value mlir::arith::getZeroConstant(OpBuilder &builder, Location loc,
+                                   Type type) {
+  // TODO: Incorporate this check to `FloatAttr::get*`.
+  assert(!isa<Float8E8M0FNUType>(getElementTypeOrSelf(type)) &&
+         "type doesn't have a zero representation");
+  TypedAttr zeroAttr = builder.getZeroAttr(type);
+  assert(zeroAttr && "unsupported type for zero attribute");
+  return builder.create<arith::ConstantOp>(loc, zeroAttr);
+}
+
 //===----------------------------------------------------------------------===//
 // AddIOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index 85d1b5ac73bf..afee162053be 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -181,7 +181,7 @@ struct SelectOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -196,17 +196,17 @@ struct SelectOpInterface
     if (failed(trueType) || failed(falseType))
       return failure();
     if (*trueType == *falseType)
-      return *trueType;
+      return cast<BufferLikeType>(*trueType);
     if (trueType->getMemorySpace() != falseType->getMemorySpace())
       return op->emitError("inconsistent memory space on true/false operands");
 
     // If the buffers have different types, they differ only in their layout
     // map.
     auto memrefType = llvm::cast<MemRefType>(*trueType);
-    return getMemRefTypeWithFullyDynamicLayout(
+    return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
         RankedTensorType::get(memrefType.getShape(),
                               memrefType.getElementType()),
-        memrefType.getMemorySpace());
+        memrefType.getMemorySpace()));
   }
 };
 
diff --git a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
index e81fc6a8b598..d07e6a52d8b5 100644
--- a/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
+++ b/mlir/lib/Dialect/ArmNeon/TransformOps/ArmNeonVectorTransformOps.cpp
@@ -20,7 +20,7 @@ using namespace mlir;
 
 void transform::ApplyArmNeonContractionToI8MMPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
-  arm_neon::populateLowerContractionToSMMLAPatternPatterns(patterns);
+  arm_neon::populateLowerContractionToNeonI8MMPatternPatterns(patterns);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
index 84fb1b0116d2..06bafde451cb 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/CMakeLists.txt
@@ -1,5 +1,5 @@
 add_mlir_dialect_library(MLIRArmNeonTransforms
-  LowerContractionToSMMLAPattern.cpp
+  LowerContractionToNeonI8MMPattern.cpp
 
   DEPENDS
   MLIRArmNeonIncGen
diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
similarity index 59%
rename from mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
rename to mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
index 5ce3d2b28aeb..7180884c77e9 100644
--- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp
+++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToNeonI8MMPattern.cpp
@@ -1,4 +1,4 @@
-//===- LowerContractionToSMMLAPattern.cpp - Contract to SMMLA ---*- C++ -*-===//
+//===- LowerContractionToNeonI8MMPattern.cpp - Contract to I8MM -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements lowering patterns from vector.contract to
-// arm_neon.intr.smmla
+// This file implements lowering patterns from vector.contract to operations
+// that map to instructions from the Neon FEAT_I8MM extension.
 //
-//===---
+// TODO: There may be opportunities to unify this with a similar pattern
+// for SVE. See:
+//   https://github.com/llvm/llvm-project/issues/145559
+//   LowerContractionToSVEI8MMPattern.cpp
+//
+//===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
@@ -37,12 +42,87 @@ static Type matchContainerType(Type element, Type container) {
   return element;
 }
 
+// Get the operand of a `vector.contract`. This function is intended to abstract
+// away from the particular way a value is extended before feeding it into the
+// `vector.contract` - via zero-extend or an explicit or implicit sign-extend
+// (for implicit sign-extension see `vector.contract` documentation).
+//
+// The template parameter `Op` indicates the extension operation (explicit or
+// implicit) for which we are checking.
+//
+// Return success only for extensions from `iN` (N <= 8) to `i32`.
+template <typename Op>
+std::optional<Value> getExtOperand(Value v) {
+
+  static_assert(llvm::is_one_of<Op, arith::ExtSIOp, arith::ExtUIOp>::value,
+                "Must be instantiated with either sign- or zero- extension op");
+
+  // If the operand is not defined by an explicit extend operation of the
+  // accepted operation type allow for an implicit sign-extension.
+  auto extOp = dyn_cast_or_null<Op>(v.getDefiningOp());
+  if (!extOp) {
+    if constexpr (std::is_same<Op, arith::ExtSIOp>::value) {
+      auto eltTy = cast<VectorType>(v.getType()).getElementType();
+      if (!eltTy.isSignlessInteger() || eltTy.getIntOrFloatBitWidth() > 8)
+        return {};
+      return v;
+    }
+    return {};
+  }
+
+  // If the operand is defined by an explicit extend operation of the accepted
+  // operation type, check it's extended from `iN` (N <= 8) to `i32`.
+  auto inOp = extOp.getIn();
+  auto inTy = dyn_cast<VectorType>(inOp.getType());
+  if (!inTy)
+    return {};
+  auto inEltTy = inTy.getElementType();
+  if (!inEltTy.isSignlessInteger() || inEltTy.getIntOrFloatBitWidth() > 8)
+    return {};
+
+  auto outTy = dyn_cast<VectorType>(extOp.getType());
+  if (!(outTy && outTy.getElementType().isSignlessInteger(32)))
+    return {};
+
+  return inOp;
+}
+
+// Designate the operation (resp. instruction) used to do sub-tile matrix
+// multiplications.
+enum class MMLA {
+  Signed,      // smmla
+  Unsigned,    // ummla
+  Mixed,       // usmmla
+  MixedSwapped // usmmla with LHS and RHS swapped
+};
+
+// Create the matrix mulitply and accumulate operation according to `op`.
+Value createMMLA(PatternRewriter &rewriter, MMLA op, Location loc,
+                 mlir::Type accType, Value acc, Value lhs, Value rhs) {
+  switch (op) {
+  case MMLA::Signed:
+    return rewriter.createOrFold<arm_neon::SmmlaOp>(loc, accType, acc, lhs,
+                                                    rhs);
+  case MMLA::Unsigned:
+    return rewriter.createOrFold<arm_neon::UmmlaOp>(loc, accType, acc, lhs,
+                                                    rhs);
+  case MMLA::Mixed:
+    return rewriter.createOrFold<arm_neon::UsmmlaOp>(loc, accType, acc, lhs,
+                                                     rhs);
+  case MMLA::MixedSwapped:
+    // The accumulator comes transposed and the result will be transposed
+    // later, so all we have to do here is swap the operands.
+    return rewriter.createOrFold<arm_neon::UsmmlaOp>(loc, accType, acc, rhs,
+                                                     lhs);
+  }
+}
+
 /// Lowering from a vector::contractOp arm neon smmla intrinsic. This will tile
 /// any vector.contract into multiple smmla instructions with unrolling so long
 /// as [2,2,8] is a divisor of its shape. It can also process vecmats with dimM
 /// = 1 (either explicitly or inferred if LHS has only dimK) If no unrolling is
 /// necessary, a single smmla instruction is emitted.
-class LowerContractionToSMMLAPattern
+class LowerContractionToNeonI8MMPattern
     : public OpRewritePattern<vector::ContractionOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
@@ -88,39 +168,64 @@ class LowerContractionToSMMLAPattern
       return failure();
     }
 
-    // Check two extsi inputs Rhs Lhs for contract.
-    arith::ExtSIOp origLhsExtOp =
-        dyn_cast_or_null<arith::ExtSIOp>(op.getLhs().getDefiningOp());
-    arith::ExtSIOp origRhsExtOp =
-        dyn_cast_or_null<arith::ExtSIOp>(op.getRhs().getDefiningOp());
-    if (!origLhsExtOp || !origRhsExtOp) {
+    // Check inputs are sign-/zero- extensions from iN (N <= 8) to i32. Get the
+    // values before the extension. All four signed/unsigned combinations for
+    // input operands are supported, but they are lowered to different
+    // operations. Determine which is the appropriate operation to lower to.
+    MMLA mmlaOp = MMLA::Signed;
+    auto maybeLhs = getExtOperand<arith::ExtSIOp>(op.getLhs());
+    if (!maybeLhs) {
+      mmlaOp = MMLA::Unsigned;
+      maybeLhs = getExtOperand<arith::ExtUIOp>(op.getLhs());
+    }
+    if (!maybeLhs)
       return failure();
+
+    auto maybeRhs = getExtOperand<arith::ExtSIOp>(op.getRhs());
+    if (maybeRhs) {
+      if (mmlaOp == MMLA::Unsigned)
+        mmlaOp = MMLA::Mixed;
+    } else {
+      if (mmlaOp == MMLA::Signed)
+        mmlaOp = MMLA::MixedSwapped;
+      maybeRhs = getExtOperand<arith::ExtUIOp>(op.getRhs());
     }
+    if (!maybeRhs)
+      return failure();
+
+    Value origLhs = *maybeLhs;
+    Value origRhs = *maybeRhs;
 
     // Match any iX to i32 for X<8 then turn into an i8 output. Feed into
     // following neon instruction. Check inputs for extsi are <=i8
-    Value extsiLhs;
-    Value extsiRhs;
-    if (auto lhsExtInType =
-            dyn_cast<mlir::VectorType>(origLhsExtOp.getIn().getType())) {
+    Value extLhs;
+    Value extRhs;
+    if (auto lhsExtInType = dyn_cast<mlir::VectorType>(origLhs.getType())) {
       if (lhsExtInType.getElementTypeBitWidth() <= 8) {
         Type targetLhsExtTy =
             matchContainerType(rewriter.getI8Type(), lhsExtInType);
-        extsiLhs = rewriter.createOrFold<arith::ExtSIOp>(loc, targetLhsExtTy,
-                                                         origLhsExtOp.getIn());
+        if (mmlaOp == MMLA::Signed || mmlaOp == MMLA::Mixed)
+          extLhs = rewriter.createOrFold<arith::ExtSIOp>(loc, targetLhsExtTy,
+                                                         origLhs);
+        else
+          extLhs = rewriter.createOrFold<arith::ExtUIOp>(loc, targetLhsExtTy,
+                                                         origLhs);
       }
     }
-    if (auto rhsExtInType =
-            dyn_cast<mlir::VectorType>(origRhsExtOp.getIn().getType())) {
+    if (auto rhsExtInType = dyn_cast<mlir::VectorType>(origRhs.getType())) {
       if (rhsExtInType.getElementTypeBitWidth() <= 8) {
         Type targetRhsExtTy =
             matchContainerType(rewriter.getI8Type(), rhsExtInType);
-        extsiRhs = rewriter.createOrFold<arith::ExtSIOp>(loc, targetRhsExtTy,
-                                                         origRhsExtOp.getIn());
+        if (mmlaOp == MMLA::Unsigned || mmlaOp == MMLA::Mixed)
+          extRhs = rewriter.createOrFold<arith::ExtUIOp>(loc, targetRhsExtTy,
+                                                         origRhs);
+        else
+          extRhs = rewriter.createOrFold<arith::ExtSIOp>(loc, targetRhsExtTy,
+                                                         origRhs);
       }
     }
 
-    if (!extsiLhs || !extsiRhs) {
+    if (!extLhs || !extRhs) {
       return failure();
     }
 
@@ -155,11 +260,11 @@ class LowerContractionToSMMLAPattern
       AffineMap lhsPermutationMap = op.getIndexingMapsArray()[0];
       SmallVector<int64_t> lhsOffsets =
           applyPermutationMap(lhsPermutationMap, ArrayRef<int64_t>(offsets));
-      Value tiledLhs = extractOperand(extsiLhs, lhsPermutationMap, lhsOffsets);
+      Value tiledLhs = extractOperand(extLhs, lhsPermutationMap, lhsOffsets);
       AffineMap rhsPermutationMap = op.getIndexingMapsArray()[1];
       SmallVector<int64_t> rhsOffsets =
           applyPermutationMap(rhsPermutationMap, ArrayRef<int64_t>(offsets));
-      Value tiledRhs = extractOperand(extsiRhs, rhsPermutationMap, rhsOffsets);
+      Value tiledRhs = extractOperand(extRhs, rhsPermutationMap, rhsOffsets);
       AffineMap accPermutationMap = op.getIndexingMapsArray()[2];
       SmallVector<int64_t> accOffsets =
           applyPermutationMap(accPermutationMap, ArrayRef<int64_t>(offsets));
@@ -191,6 +296,13 @@ class LowerContractionToSMMLAPattern
         tiledAcc = expandForSMMLA(tiledAcc, outputExpandedType);
       }
 
+      // Transpose ACC if doing signed by unsigned multiplication, because we're
+      // using the instruction for unsigned by signed multiplication with
+      // reversed operands.
+      if (mmlaOp == MMLA::MixedSwapped)
+        tiledAcc = rewriter.create<vector::TransposeOp>(
+            loc, tiledAcc, ArrayRef<int64_t>({1, 0}));
+
       // Collapse tiled operands to 1D vectors required by smmla intrinsic
       auto collapsedInputType =
           VectorType::get(inputExpandedType.getNumElements(), inputElementType);
@@ -211,15 +323,21 @@ class LowerContractionToSMMLAPattern
       }
 
       // Insert contract op
-      kAcc = rewriter.createOrFold<arm_neon::SmmlaOp>(
-          op.getLoc(), collapsedRes.getType(), collapsedRes, collapsedLhs,
-          collapsedRhs);
+      kAcc = createMMLA(rewriter, mmlaOp, op.getLoc(), collapsedRes.getType(),
+                        collapsedRes, collapsedLhs, collapsedRhs);
 
       // Reshape output back to 2D
       Value tiledRes = rewriter.createOrFold<vector::ShapeCastOp>(
           kAcc.getLoc(), tiledAcc.getType(), kAcc);
 
-      // With vecmat, only one row of tiled ACC can be inserted into file result
+      // Because of the reversed operands the result is obtained transposed.
+      // Transpose it back,
+      if (mmlaOp == MMLA::MixedSwapped)
+        tiledRes = rewriter.create<vector::TransposeOp>(
+            loc, tiledRes, ArrayRef<int64_t>({1, 0}));
+
+      // With vecmat, only one row of tiled ACC can be inserted into the final
+      // result
       if (isVecmat) {
         tiledRes = rewriter.createOrFold<vector::ExtractOp>(loc, tiledRes, 0);
       }
@@ -239,8 +357,8 @@ class LowerContractionToSMMLAPattern
 
 } // namespace
 
-void mlir::arm_neon::populateLowerContractionToSMMLAPatternPatterns(
+void mlir::arm_neon::populateLowerContractionToNeonI8MMPatternPatterns(
     RewritePatternSet &patterns) {
   MLIRContext *context = patterns.getContext();
-  patterns.add<LowerContractionToSMMLAPattern>(context, /*benefit=*/2);
+  patterns.add<LowerContractionToNeonI8MMPattern>(context, /*benefit=*/2);
 }
diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 95965872f409..1e8e1265affa 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -724,59 +724,6 @@ struct LiftIllegalVectorTransposeToMemory
   }
 };
 
-/// A rewrite to turn unit dim transpose-like vector.shape_casts into
-/// vector.transposes. The shape_cast has to be from an illegal vector type to a
-/// legal one (as defined by isLegalVectorType).
-///
-/// The reasoning for this is if we've got to this pass and we still have
-/// shape_casts of illegal types, then they likely will not cancel out. Turning
-/// them into transposes gives LiftIllegalVectorTransposeToMemory a chance to
-/// eliminate them.
-///
-/// Example:
-///
-///  BEFORE:
-///  ```mlir
-///  %0 = vector.shape_cast %a : vector<[4]x1xf32> to vector<1x[4]xf32>
-///  ```
-///
-///  AFTER:
-///  ```mlir
-///  %0 = vector.transpose %0, [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32>
-///  ```
-struct ConvertIllegalShapeCastOpsToTransposes
-    : public OpRewritePattern<vector::ShapeCastOp> {
-  using OpRewritePattern<vector::ShapeCastOp>::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(vector::ShapeCastOp shapeCastOp,
-                                PatternRewriter &rewriter) const override {
-    auto sourceType = shapeCastOp.getSourceVectorType();
-    auto resultType = shapeCastOp.getResultVectorType();
-    if (isLegalVectorType(sourceType) || !isLegalVectorType(resultType))
-      return rewriter.notifyMatchFailure(shapeCastOp,
-                                         kMatchFailureNotIllegalToLegal);
-
-    // Note: If we know that `sourceType` is an illegal vector type (and 2D)
-    // then dim 0 is scalable and dim 1 is fixed.
-    if (sourceType.getRank() != 2 || sourceType.getDimSize(1) != 1)
-      return rewriter.notifyMatchFailure(
-          shapeCastOp, "expected source to be a 2D scalable vector with a "
-                       "trailing unit dim");
-
-    auto loc = shapeCastOp.getLoc();
-    auto transpose = rewriter.create<vector::TransposeOp>(
-        loc, shapeCastOp.getSource(), ArrayRef<int64_t>{1, 0});
-
-    if (resultType.getRank() == 1)
-      rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(shapeCastOp, resultType,
-                                                       transpose);
-    else
-      rewriter.replaceOp(shapeCastOp, transpose);
-
-    return success();
-  }
-};
-
 /// Rewrites an illegal/unsupported SVE transfer_write(transpose) to instead use
 /// the ZA state. This workaround rewrite to support these transposes when ZA is
 /// available.
@@ -920,6 +867,116 @@ struct LowerIllegalTransposeStoreViaZA
   }
 };
 
+/// Lower `vector.transfer_read` of a scalable column to `scf::for`
+///
+/// Lowers a "read" of a scalable column from a MemRef for which there is no
+/// hardware pperation that we could use to a loop over the rows to read and
+/// loads one element at a time.
+///
+///  BEFORE:
+///  ```
+///  %res = vector.transfer_read %mem[%a, %b] (...)
+///    : memref<?x?xf32>, vector<[4]x1xf32>
+///  ```
+///
+///  AFTER:
+///  ```
+///    %cst = arith.constant (...) : vector<[4]xf32>
+///    %vscale = vector.vscale
+///    %c4_vscale = arith.muli %vscale, %c4 : index
+///    %scf = scf.for %lb = %c0 to %c4_vscale step %c1 iter_args(%arg4 = %cst)
+///      -> (vector<[4]xf32>) {
+///
+///        %load = memref.load %mem[%arg3 + %a, %b] : memref<?x?xf32>
+///        %vec = vector.insert %load, %cst [%arg3] : f32 into vector<[4]xf32>
+///        scf.yield %vec : vector<[4]xf32>
+///    }
+///    %res = vector.shape_cast %scf : vector<[4]xf32> to vector<[4]x1xf32>
+///  ```
+///
+///  TODO: This transformation isn't specific to SME - move it to the SVE
+///  dialect.
+///  TODO: Check the in_bounds attribute and generate vector.maskedload if
+///  required.
+struct LowerColumnTransferReadToLoops
+    : public OpRewritePattern<vector::TransferReadOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
+                                PatternRewriter &rewriter) const override {
+    // NOTE: This is a fairly low-level transformation, so we shouldn't be
+    // adding support for Tensors without good rationale.
+    if (readOp.hasPureTensorSemantics())
+      return rewriter.notifyMatchFailure(
+          readOp, "Tensor semantics are unsupported (either bufferize or "
+                  "extend this pattern)");
+
+    auto resType = readOp.getVectorType();
+
+    if (resType.getRank() != 2)
+      return rewriter.notifyMatchFailure(readOp,
+                                         "Only 2D vectors are supported!");
+
+    if (resType.getShape()[1] != 1)
+      return rewriter.notifyMatchFailure(
+          readOp, "The trailing output dim is != 1 (not supported ATM)");
+
+    if (!resType.getScalableDims()[0] || resType.getScalableDims()[1])
+      return rewriter.notifyMatchFailure(
+          readOp, "Expected the leading dim to be scalable and the trailing "
+                  "dim to be fixed.");
+
+    // Create new result type - similar to the original vector with the
+    // trailing unit dim collapsed.
+    int64_t numRows = resType.getShape()[0];
+    VectorType newResType = VectorType::get(numRows, resType.getElementType(),
+                                            /*scalableDims=*/{true});
+
+    // Create a loop over all rows and load one element at a time.
+    auto loc = readOp.getLoc();
+    auto lowerBound = rewriter.create<arith::ConstantIndexOp>(loc, 0);
+    auto createVscaleMultiple =
+        vector::makeVscaleConstantBuilder(rewriter, loc);
+    auto upperBound = createVscaleMultiple(numRows);
+    auto step = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+    Value init = rewriter.create<arith::ConstantOp>(
+        loc, newResType, DenseElementsAttr::get(newResType, 0.0f));
+
+    scf::ForOp loadLoop;
+    {
+      OpBuilder::InsertionGuard g(rewriter);
+      loadLoop = rewriter.create<scf::ForOp>(loc, lowerBound, upperBound, step,
+                                             ValueRange{init});
+      rewriter.setInsertionPointToStart(loadLoop.getBody());
+
+      auto tileSliceIndex = loadLoop.getInductionVar();
+
+      auto idx0 = rewriter.create<arith::AddIOp>(loc, tileSliceIndex,
+                                                 readOp.getIndices()[0]);
+      auto idx1 = readOp.getIndices()[1];
+
+      Value scalar = rewriter.create<memref::LoadOp>(
+          loc, readOp.getBase(), SmallVector<Value>({idx0, idx1}));
+
+      Operation *updateInit = rewriter.create<vector::InsertOp>(
+          loc, scalar, loadLoop.getRegionIterArg(0), tileSliceIndex);
+
+      rewriter.create<scf::YieldOp>(loc, updateInit->getResult(0));
+    }
+
+    // The read operation has been "legalized", but since the original result
+    // type was a 2D vector, we need to cast before returning the result. This
+    // ShapeCast should cancel-out with some other ShapeCast (i.e. it's a
+    // no-op).
+    auto sc = rewriter.create<vector::ShapeCastOp>(
+        loc, readOp.getResult().getType(), loadLoop.getResult(0));
+
+    rewriter.replaceOp(readOp, sc);
+
+    return success();
+  }
+};
+
 struct VectorLegalizationPass
     : public arm_sme::impl::VectorLegalizationBase<VectorLegalizationPass> {
   void runOnOperation() override {
@@ -941,10 +998,10 @@ struct VectorLegalizationPass
 
     // Apply preprocessing patterns.
     RewritePatternSet rewritePatterns(context);
-    rewritePatterns.add<FoldExtractFromVectorOfSMELikeCreateMasks,
-                        LiftIllegalVectorTransposeToMemory,
-                        ConvertIllegalShapeCastOpsToTransposes,
-                        LowerIllegalTransposeStoreViaZA>(context);
+    rewritePatterns
+        .add<FoldExtractFromVectorOfSMELikeCreateMasks,
+             LowerColumnTransferReadToLoops, LiftIllegalVectorTransposeToMemory,
+             LowerIllegalTransposeStoreViaZA>(context);
     if (failed(
             applyPatternsGreedily(getOperation(), std::move(rewritePatterns))))
       return signalPassFailure();
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
index 35f2a02cc4ec..006332b48325 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeForLLVMExport.cpp
@@ -220,7 +220,8 @@ void mlir::populateArmSVELegalizeForLLVMExportPatterns(
 void mlir::configureArmSVELegalizeForExportTarget(
     LLVMConversionTarget &target) {
   // clang-format off
-  target.addLegalOp<ConvertFromSvboolIntrOp,
+  target.addLegalOp<BfmmlaOp,
+                    ConvertFromSvboolIntrOp,
                     ConvertToSvboolIntrOp,
                     DupQLaneIntrOp,
                     PselIntrOp,
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
index d2ac850a5f70..3dbb93b8a066 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp
@@ -298,16 +298,157 @@ struct LegalizeSVEMaskLoadConversion : public OpRewritePattern<memref::LoadOp> {
   }
 };
 
+/// Transforms a `transfer_read` operation so it reads vector of a type that
+/// can be mapped to an LLVM type ("LLVM-legal" type). This is done by
+/// collapsing trailing dimensions so we obtain a vector type with a single
+/// scalable dimension in the rightmost position.
+///
+/// Example:
+/// ```
+/// %v = vector.transfer_read %M[%i, %j, %c0, %c0], %c0_i8
+///   {in_bounds = [false, true, true, true]}
+///   : memref<?x?x2x8xi8>, vector<2x[4]x2x8xi8>
+/// ```
+/// is rewritten to
+/// ```
+/// %collapse_shape = memref.collapse_shape %M [[0], [1, 2, 3]]
+///   : memref<?x?x2x8xi8> into memref<?x?xi8>
+/// %0 = vector.transfer_read  %collapse_shape[%i, %j], %c0_i8
+///   {in_bounds = [false, true]}
+///   : memref<?x?xi8>, vector<2x[64]xi8>
+/// %1 = vector.shape_cast %0 : vector<2x[64]xi8> to vector<2x[4]x2x8xi8>
+/// ```
+struct LegalizeTransferRead : public OpRewritePattern<vector::TransferReadOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
+                                PatternRewriter &rewriter) const override {
+
+    // Do not try to transform masked reads. For example, if we have a transfer
+    // to a `vector<[4]x4xi8>` we could have a mask like
+    //    1 1 1 0
+    //    1 1 1 0
+    //    1 1 1 0
+    //    0 0 0 0
+    // Flattening this mask would look like
+    //    1 1 1 0 1 1 1 0 1 1 1 0 0 0 0 0
+    // and we have not yet figured out an efficient way to build such a mask,
+    // neither from the mask operand, nor from the original `vector.create_mask`
+    // operation (if visible at all).
+    if (readOp.isMasked() || readOp.getMask())
+      return rewriter.notifyMatchFailure(readOp,
+                                         "masked transfers not-supported");
+
+    // General permutation maps are not supported. The issue is with transpose,
+    // broadcast, and other forms of non-identify mapping in the minor
+    // dimensions which is impossible to represent after collapsing (at least
+    // because the resulting "collapsed" maps would have smaller number of
+    // dimension indices).
+    // TODO: We have not had yet the need for it, but some forms of permutation
+    // maps with identity in the minor dimensions voukld be supported, for
+    // example `(i, j, k, p) -> (j, i, k, p)` where we need to collapse only `k`
+    // and `p`.
+    if (!readOp.getPermutationMap().isMinorIdentity())
+      return rewriter.notifyMatchFailure(readOp, "non-identity permutation");
+
+    // We handle transfers of vectors with rank >= 2 and a single scalable
+    // dimension. This transformation aims to transform an LLVM-illegal type
+    // into an LLVM-legal type and one dimensional vectors are already
+    // LLVM-legal, even if scalable. A value of a vector type with more than one
+    // scalable dimension is impossible to represent using a vector type with no
+    // scalable dimensions or a single one. For example a `vector<[4]x[4]xi8>`
+    // would have `4 * 4 * vscale * vscale` elements and this quantity is
+    // impossible to represent as `N` or `N * vscale` (where `N` is a constant).
+    VectorType origVT = readOp.getVectorType();
+    ArrayRef<bool> origScalableDims = origVT.getScalableDims();
+    const int64_t origVRank = origVT.getRank();
+    if (origVRank < 2 || origVT.getNumScalableDims() != 1)
+      return rewriter.notifyMatchFailure(readOp, "wrong dimensions");
+
+    // Number of trailing dimensions to collapse, including the scalable
+    // dimension.  Nothing to do if the single scalable dimension is already the
+    // last one.
+    const int64_t numCollapseDims = std::distance(
+        llvm::find(origScalableDims, true), origScalableDims.end());
+    if (numCollapseDims < 2)
+      return rewriter.notifyMatchFailure(readOp,
+                                         "scalable dimension is trailing");
+
+    // We want a simple memref (not a tensor) with contiguous elements for at
+    // least all the trailing dimensions up to and including the scalable one.
+    auto memTy = dyn_cast<MemRefType>(readOp.getBase().getType());
+    if (!(memTy && memTy.areTrailingDimsContiguous(numCollapseDims)))
+      return rewriter.notifyMatchFailure(
+          readOp, "non-contiguous memref dimensions to collapse");
+
+    // The dimensions to collapse (excluding the scalable one) of the vector and
+    // the memref must match. A dynamic memref dimension is considered
+    // non-matching. The transfers from the dimensions to collapse must be
+    // in-bounds (it follows the corresponding indices would be zero). This
+    // guarantees that the operation transfers a contiguous block
+    // and no padding is necessary.
+    if (!llvm::equal(memTy.getShape().take_back(numCollapseDims - 1),
+                     origVT.getShape().take_back(numCollapseDims - 1)))
+      return rewriter.notifyMatchFailure(
+          readOp, "memref and vector dimensions do not match");
+
+    SmallVector<bool> origInBounds = readOp.getInBoundsValues();
+    if (!llvm::all_of(
+            ArrayRef<bool>(origInBounds).take_back(numCollapseDims - 1),
+            [](bool v) { return v; }))
+      return rewriter.notifyMatchFailure(
+          readOp, "out-of-bounds transfer from a dimension to collapse");
+
+    // Collapse the trailing dimensions of the memref.
+    SmallVector<ReassociationIndices> reassoc;
+    for (int64_t i = 0; i < memTy.getRank() - numCollapseDims + 1; ++i)
+      reassoc.push_back({i});
+    for (int64_t i = memTy.getRank() - numCollapseDims + 1; i < memTy.getRank();
+         ++i)
+      reassoc.back().push_back(i);
+    if (!memref::CollapseShapeOp::isGuaranteedCollapsible(memTy, reassoc))
+      return failure();
+    Value collapsedMem = rewriter.create<memref::CollapseShapeOp>(
+        readOp.getLoc(), readOp.getBase(), reassoc);
+
+    // Get a vector type with collapsed trailing dimensions.
+    SmallVector<int64_t> shape(origVT.getShape());
+    for (int64_t i = origVRank - numCollapseDims + 1; i < origVRank; ++i)
+      shape[origVRank - numCollapseDims] *= shape[i];
+    shape.pop_back_n(numCollapseDims - 1);
+    auto collapsedVT =
+        VectorType::get(shape, origVT.getElementType(),
+                        origScalableDims.drop_back(numCollapseDims - 1));
+
+    // Drop the extra (zero) indices.
+    auto indices = readOp.getIndices().drop_back(numCollapseDims - 1);
+
+    // Create the new `transfer_read`.
+    auto newReadOp = rewriter.create<vector::TransferReadOp>(
+        readOp.getLoc(), collapsedVT, collapsedMem, indices,
+        readOp.getPadding(),
+        ArrayRef<bool>(origInBounds).drop_back(numCollapseDims - 1));
+
+    // Cast back to the original vector type.
+    auto toOrigShape = rewriter.create<vector::ShapeCastOp>(readOp.getLoc(),
+                                                            origVT, newReadOp);
+
+    rewriter.replaceOp(readOp, toOrigShape);
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::arm_sve::populateLegalizeVectorStoragePatterns(
     RewritePatternSet &patterns) {
-  patterns.add<RelaxScalableVectorAllocaAlignment,
-               LegalizeSVEMaskAllocation<memref::AllocaOp>,
-               LegalizeSVEMaskAllocation<memref::AllocOp>,
-               LegalizeSVEMaskTypeCastConversion,
-               LegalizeSVEMaskStoreConversion, LegalizeSVEMaskLoadConversion>(
-      patterns.getContext());
+  patterns
+      .add<RelaxScalableVectorAllocaAlignment,
+           LegalizeSVEMaskAllocation<memref::AllocaOp>,
+           LegalizeSVEMaskAllocation<memref::AllocOp>,
+           LegalizeSVEMaskTypeCastConversion, LegalizeSVEMaskStoreConversion,
+           LegalizeSVEMaskLoadConversion, LegalizeTransferRead>(
+          patterns.getContext());
 }
 
 namespace {
diff --git a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
index a1209fe8230e..b7703ff0393e 100644
--- a/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
+++ b/mlir/lib/Dialect/ArmSVE/Transforms/LowerContractionToSVEI8MMPattern.cpp
@@ -1,4 +1,4 @@
-//===- LowerContractionToSMMLAPattern.cpp - Contract to SMMLA ---*- C++ -*-===//
+//===- LowerContractionToSVEI8MMPattern.cpp - Contract to I8MM --*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -9,6 +9,11 @@
 // This file implements lowering patterns from vector.contract to operations
 // that map to instructions from the SVE FEAT_I8MM extension.
 //
+// TODO: There may be opportunities to unify this with a similar pattern
+// for Neon. See:
+//   https://github.com/llvm/llvm-project/issues/145559
+//   LowerContractionToNeonI8MMPattern.cpp
+//
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index d3bb250bb8ab..b834afef7da7 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -309,7 +309,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
     return;
   assert(type.getNumInputs() == argAttrs.size());
   call_interface_impl::addArgAndResultAttrs(
-      builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
+      builder, state, argAttrs, /*resultAttrs=*/{},
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
 
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
index c289faaf9983..8601bb5aaada 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncToAsyncRuntime.cpp
@@ -723,8 +723,8 @@ class YieldOpLowering : public OpConversionPattern<async::YieldOp> {
       // Switch the coroutine completion token to available state.
       rewriter.create<RuntimeSetAvailableOp>(loc, *coro.asyncToken);
 
-    rewriter.eraseOp(op);
     rewriter.create<cf::BranchOp>(loc, coro.cleanup);
+    rewriter.eraseOp(op);
 
     return success();
   }
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 2ab182c9b7b2..8f17a82fabe0 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -7,13 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/AsmState.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
@@ -195,9 +194,12 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
         reifiedShapes = true;
         auto &shape =
             resultDims[llvm::cast<OpResult>(shapedValue).getResultNumber()];
-        for (const auto &dim : enumerate(tensorType.getShape()))
-          if (ShapedType::isDynamic(dim.value()))
-            dynamicSizes.push_back(cast<Value>(shape[dim.index()]));
+        for (const auto &dim : enumerate(tensorType.getShape())) {
+          if (ShapedType::isDynamic(dim.value())) {
+            dynamicSizes.push_back(
+                getValueOrCreateConstantIndexOp(b, loc, shape[dim.index()]));
+          }
+        }
       }
     }
 
@@ -945,7 +947,7 @@ AliasingOpOperandList bufferization::detail::defaultGetAliasingOpOperands(
   return AliasingOpOperandList(std::move(result));
 }
 
-FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
+FailureOr<BufferLikeType> bufferization::detail::defaultGetBufferType(
     Value value, const BufferizationOptions &options,
     const BufferizationState &bufferizationState,
     SmallVector<Value> &invocationStack) {
@@ -953,8 +955,10 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
   auto tensorType = cast<TensorType>(value.getType());
 
   // No further analysis is possible for a block argument.
-  if (llvm::isa<BlockArgument>(value))
-    return bufferization::getMemRefType(tensorType, options);
+  if (llvm::isa<BlockArgument>(value)) {
+    return cast<BufferLikeType>(
+        bufferization::getMemRefType(tensorType, options));
+  }
 
   // Value is an OpResult.
   Operation *op = getOwnerOfValue(value);
@@ -966,8 +970,8 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
     // If the OpResult has an equivalent OpOperand, both OpResult and
     // OpOperand bufferize to the exact same buffer type.
     Value equivalentOperand = aliases.getAliases().front().opOperand->get();
-    return asMemRefType(getBufferType(equivalentOperand, options,
-                                      bufferizationState, invocationStack));
+    return getBufferType(equivalentOperand, options, bufferizationState,
+                         invocationStack);
   }
 
   // If we do not know the memory space and there is no default memory space,
@@ -977,7 +981,8 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
   if (!memSpace.has_value())
     return op->emitError("could not infer memory space");
 
-  return getMemRefType(tensorType, options, /*layout=*/{}, *memSpace);
+  return cast<BufferLikeType>(
+      getMemRefType(tensorType, options, /*layout=*/{}, *memSpace));
 }
 
 bool bufferization::detail::defaultIsRepetitiveRegion(
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index 9bd87d66c7d3..66949c96798d 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -222,7 +222,7 @@ AliasingValueList AllocTensorOp::getAliasingValues(OpOperand &opOperand,
   return {};
 }
 
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 AllocTensorOp::getBufferType(Value value, const BufferizationOptions &options,
                              const BufferizationState &state,
                              SmallVector<Value> &invocationStack) {
@@ -245,7 +245,8 @@ AllocTensorOp::getBufferType(Value value, const BufferizationOptions &options,
     return getOperation()->emitError("could not infer memory space");
   }
 
-  return getMemRefTypeWithStaticIdentityLayout(getType(), memorySpace);
+  return cast<BufferLikeType>(
+      getMemRefTypeWithStaticIdentityLayout(getType(), memorySpace));
 }
 
 LogicalResult AllocTensorOp::verify() {
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index 453ed43bcadd..bd2aebca6807 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -211,7 +211,7 @@ struct CallOpInterface
     return result;
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -229,12 +229,13 @@ struct CallOpInterface
     Type resultType =
         funcType.getResult(cast<OpResult>(value).getResultNumber());
     if (auto bufferizedType = dyn_cast<BaseMemRefType>(resultType))
-      return bufferizedType;
+      return cast<BufferLikeType>(bufferizedType);
 
     // Otherwise, call the type converter to compute the bufferized type.
     auto tensorType = cast<TensorType>(resultType);
-    return options.functionArgTypeConverterFn(
-        tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp, options);
+    return cast<BufferLikeType>(options.functionArgTypeConverterFn(
+        tensorType, *options.defaultMemorySpaceFn(tensorType), funcOp,
+        options));
   }
 
   /// All function arguments are writable. It is the responsibility of the
@@ -396,7 +397,7 @@ struct FuncOpInterface
     return getAliasingBranchOpOperands(op, cast<BlockArgument>(value), state);
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -405,8 +406,8 @@ struct FuncOpInterface
 
     // Function arguments are special.
     if (bbArg.getOwner() == &funcOp.getBody().front())
-      return getBufferizedFunctionArgType(funcOp, bbArg.getArgNumber(),
-                                          options);
+      return cast<BufferLikeType>(
+          getBufferizedFunctionArgType(funcOp, bbArg.getArgNumber(), options));
 
     return OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel::
         getBufferType(op, value, options, state, invocationStack);
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index e602210c2dc6..27298e892e59 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -601,7 +601,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
     return;
   assert(type.getNumInputs() == argAttrs.size());
   call_interface_impl::addArgAndResultAttrs(
-      builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
+      builder, state, argAttrs, /*resultAttrs=*/{},
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
 
@@ -1400,6 +1400,49 @@ void FileOp::build(OpBuilder &builder, OperationState &state, StringRef id) {
       builder.getNamedAttr("id", builder.getStringAttr(id)));
 }
 
+//===----------------------------------------------------------------------===//
+// FieldOp
+//===----------------------------------------------------------------------===//
+LogicalResult FieldOp::verify() {
+  if (!isSupportedEmitCType(getType()))
+    return emitOpError("expected valid emitc type");
+
+  Operation *parentOp = getOperation()->getParentOp();
+  if (!parentOp || !isa<emitc::ClassOp>(parentOp))
+    return emitOpError("field must be nested within an emitc.class operation");
+
+  StringAttr symName = getSymNameAttr();
+  if (!symName || symName.getValue().empty())
+    return emitOpError("field must have a non-empty symbol name");
+
+  if (!getAttrs())
+    return success();
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// GetFieldOp
+//===----------------------------------------------------------------------===//
+LogicalResult GetFieldOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
+  mlir::FlatSymbolRefAttr fieldNameAttr = getFieldNameAttr();
+  FieldOp fieldOp =
+      symbolTable.lookupNearestSymbolFrom<FieldOp>(*this, fieldNameAttr);
+  if (!fieldOp)
+    return emitOpError("field '")
+           << fieldNameAttr << "' not found in the class";
+
+  Type getFieldResultType = getResult().getType();
+  Type fieldType = fieldOp.getType();
+
+  if (fieldType != getFieldResultType)
+    return emitOpError("result type ")
+           << getFieldResultType << " does not match field '" << fieldNameAttr
+           << "' type " << fieldType;
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/EmitC/Transforms/CMakeLists.txt b/mlir/lib/Dialect/EmitC/Transforms/CMakeLists.txt
index 19b80b22bd84..baf67afc3007 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/EmitC/Transforms/CMakeLists.txt
@@ -2,6 +2,7 @@ add_mlir_dialect_library(MLIREmitCTransforms
   Transforms.cpp
   FormExpressions.cpp
   TypeConversions.cpp
+  WrapFuncInClass.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/EmitC/Transforms
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
new file mode 100644
index 000000000000..17d436f6df02
--- /dev/null
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -0,0 +1,112 @@
+//===- WrapFuncInClass.cpp - Wrap Emitc Funcs in classes -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/EmitC/IR/EmitC.h"
+#include "mlir/Dialect/EmitC/Transforms/Passes.h"
+#include "mlir/Dialect/EmitC/Transforms/Transforms.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/WalkPatternRewriteDriver.h"
+
+using namespace mlir;
+using namespace emitc;
+
+namespace mlir {
+namespace emitc {
+#define GEN_PASS_DEF_WRAPFUNCINCLASSPASS
+#include "mlir/Dialect/EmitC/Transforms/Passes.h.inc"
+
+namespace {
+struct WrapFuncInClassPass
+    : public impl::WrapFuncInClassPassBase<WrapFuncInClassPass> {
+  using WrapFuncInClassPassBase::WrapFuncInClassPassBase;
+  void runOnOperation() override {
+    Operation *rootOp = getOperation();
+
+    RewritePatternSet patterns(&getContext());
+    populateFuncPatterns(patterns, namedAttribute);
+
+    walkAndApplyPatterns(rootOp, std::move(patterns));
+  }
+};
+
+} // namespace
+} // namespace emitc
+} // namespace mlir
+
+class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
+public:
+  WrapFuncInClass(MLIRContext *context, StringRef attrName)
+      : OpRewritePattern<emitc::FuncOp>(context), attributeName(attrName) {}
+
+  LogicalResult matchAndRewrite(emitc::FuncOp funcOp,
+                                PatternRewriter &rewriter) const override {
+
+    auto className = funcOp.getSymNameAttr().str() + "Class";
+    ClassOp newClassOp = rewriter.create<ClassOp>(funcOp.getLoc(), className);
+
+    SmallVector<std::pair<StringAttr, TypeAttr>> fields;
+    rewriter.createBlock(&newClassOp.getBody());
+    rewriter.setInsertionPointToStart(&newClassOp.getBody().front());
+
+    auto argAttrs = funcOp.getArgAttrs();
+    for (auto [idx, val] : llvm::enumerate(funcOp.getArguments())) {
+      StringAttr fieldName;
+      Attribute argAttr = nullptr;
+
+      fieldName = rewriter.getStringAttr("fieldName" + std::to_string(idx));
+      if (argAttrs && idx < argAttrs->size())
+        argAttr = (*argAttrs)[idx];
+
+      TypeAttr typeAttr = TypeAttr::get(val.getType());
+      fields.push_back({fieldName, typeAttr});
+      rewriter.create<emitc::FieldOp>(funcOp.getLoc(), fieldName, typeAttr,
+                                      argAttr);
+    }
+
+    rewriter.setInsertionPointToEnd(&newClassOp.getBody().front());
+    FunctionType funcType = funcOp.getFunctionType();
+    Location loc = funcOp.getLoc();
+    FuncOp newFuncOp =
+        rewriter.create<emitc::FuncOp>(loc, ("execute"), funcType);
+
+    rewriter.createBlock(&newFuncOp.getBody());
+    newFuncOp.getBody().takeBody(funcOp.getBody());
+
+    rewriter.setInsertionPointToStart(&newFuncOp.getBody().front());
+    std::vector<Value> newArguments;
+    newArguments.reserve(fields.size());
+    for (auto &[fieldName, attr] : fields) {
+      GetFieldOp arg =
+          rewriter.create<emitc::GetFieldOp>(loc, attr.getValue(), fieldName);
+      newArguments.push_back(arg);
+    }
+
+    for (auto [oldArg, newArg] :
+         llvm::zip(newFuncOp.getArguments(), newArguments)) {
+      rewriter.replaceAllUsesWith(oldArg, newArg);
+    }
+
+    llvm::BitVector argsToErase(newFuncOp.getNumArguments(), true);
+    if (failed(newFuncOp.eraseArguments(argsToErase)))
+      newFuncOp->emitOpError("failed to erase all arguments using BitVector");
+
+    rewriter.replaceOp(funcOp, newClassOp);
+    return success();
+  }
+
+private:
+  StringRef attributeName;
+};
+
+void mlir::emitc::populateFuncPatterns(RewritePatternSet &patterns,
+                                       StringRef namedAttribute) {
+  patterns.add<WrapFuncInClass>(patterns.getContext(), namedAttribute);
+}
diff --git a/mlir/lib/Dialect/Func/IR/FuncOps.cpp b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
index aa068966ba60..9d317f20521f 100644
--- a/mlir/lib/Dialect/Func/IR/FuncOps.cpp
+++ b/mlir/lib/Dialect/Func/IR/FuncOps.cpp
@@ -193,7 +193,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
     return;
   assert(type.getNumInputs() == argAttrs.size());
   call_interface_impl::addArgAndResultAttrs(
-      builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
+      builder, state, argAttrs, /*resultAttrs=*/{},
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
 
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 39f626b55829..d009c3b40452 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -106,6 +106,20 @@ int64_t GPUThreadMappingAttr::getRelativeIndex() const {
              : getMappingId();
 }
 
+int64_t GPULaneMappingAttr::getMappingId() const {
+  return static_cast<int64_t>(getLane());
+}
+
+bool GPULaneMappingAttr::isLinearMapping() const {
+  return getMappingId() >= static_cast<int64_t>(MappingId::LinearDim0);
+}
+
+int64_t GPULaneMappingAttr::getRelativeIndex() const {
+  return isLinearMapping()
+             ? getMappingId() - static_cast<int64_t>(MappingId::LinearDim0)
+             : getMappingId();
+}
+
 int64_t GPUMemorySpaceMappingAttr::getMappingId() const {
   return static_cast<int64_t>(getAddressSpace());
 }
@@ -1147,8 +1161,7 @@ void LaunchFuncOp::build(OpBuilder &builder, OperationState &result,
   prop.kernel = kernelSymbol;
   size_t segmentSizesLen = std::size(prop.operandSegmentSizes);
   // Initialize the segment sizes to 1.
-  for (auto &sz : prop.operandSegmentSizes)
-    sz = 1;
+  llvm::fill(prop.operandSegmentSizes, 1);
   prop.operandSegmentSizes[0] = asyncDependencies.size();
   if (!clusterSize.has_value()) {
     prop.operandSegmentSizes[segmentSizesLen - 4] = 0;
@@ -1196,8 +1209,7 @@ void LaunchFuncOp::build(OpBuilder &builder, OperationState &result,
   prop.kernel = kernel;
   size_t segmentSizesLen = std::size(prop.operandSegmentSizes);
   // Initialize the segment sizes to 1.
-  for (auto &sz : prop.operandSegmentSizes)
-    sz = 1;
+  llvm::fill(prop.operandSegmentSizes, 1);
   prop.operandSegmentSizes[0] = 0;
   if (!clusterSize.has_value()) {
     prop.operandSegmentSizes[segmentSizesLen - 4] = 0;
@@ -1331,6 +1343,49 @@ void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value value,
         mode);
 }
 
+//===----------------------------------------------------------------------===//
+// RotateOp
+//===----------------------------------------------------------------------===//
+
+void RotateOp::build(OpBuilder &builder, OperationState &result, Value value,
+                     int32_t offset, int32_t width) {
+  build(builder, result, value,
+        builder.create<arith::ConstantOp>(result.location,
+                                          builder.getI32IntegerAttr(offset)),
+        builder.create<arith::ConstantOp>(result.location,
+                                          builder.getI32IntegerAttr(width)));
+}
+
+LogicalResult RotateOp::verify() {
+  auto offsetConstOp = getOffset().getDefiningOp<arith::ConstantOp>();
+  if (!offsetConstOp)
+    return emitOpError() << "offset is not a constant value";
+
+  auto offsetIntAttr =
+      llvm::dyn_cast<mlir::IntegerAttr>(offsetConstOp.getValue());
+
+  auto widthConstOp = getWidth().getDefiningOp<arith::ConstantOp>();
+  if (!widthConstOp)
+    return emitOpError() << "width is not a constant value";
+
+  auto widthIntAttr =
+      llvm::dyn_cast<mlir::IntegerAttr>(widthConstOp.getValue());
+
+  llvm::APInt offsetValue = offsetIntAttr.getValue();
+  llvm::APInt widthValue = widthIntAttr.getValue();
+
+  if (!widthValue.isPowerOf2())
+    return emitOpError() << "width must be a power of two";
+
+  if (offsetValue.sge(widthValue) || offsetValue.slt(0)) {
+    int64_t widthValueInt = widthValue.getSExtValue();
+    return emitOpError() << "offset must be in the range [0, " << widthValueInt
+                         << ")";
+  }
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // BarrierOp
 //===----------------------------------------------------------------------===//
@@ -2302,7 +2357,7 @@ void WarpExecuteOnLane0Op::build(OpBuilder &builder, OperationState &result,
                                  TypeRange resultTypes, Value laneId,
                                  int64_t warpSize) {
   build(builder, result, resultTypes, laneId, warpSize,
-        /*operands=*/std::nullopt, /*argTypes=*/std::nullopt);
+        /*operands=*/{}, /*argTypes=*/{});
 }
 
 void WarpExecuteOnLane0Op::build(OpBuilder &builder, OperationState &result,
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 6446235c06fb..63f87d9b5877 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -313,11 +313,14 @@ checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
                                      llvm::IsaPred<GPUWarpMappingAttr>);
   bool hasThreadMapping = llvm::any_of(forallOp.getMapping().value(),
                                        llvm::IsaPred<GPUThreadMappingAttr>);
+  bool hasLaneMapping = llvm::any_of(forallOp.getMapping().value(),
+                                     llvm::IsaPred<GPULaneMappingAttr>);
   int64_t countMappingTypes = 0;
   countMappingTypes += hasBlockMapping ? 1 : 0;
   countMappingTypes += hasWarpgroupMapping ? 1 : 0;
   countMappingTypes += hasWarpMapping ? 1 : 0;
   countMappingTypes += hasThreadMapping ? 1 : 0;
+  countMappingTypes += hasLaneMapping ? 1 : 0;
   if (countMappingTypes > 1) {
     return definiteFailureHelper(
         transformOp, forallOp,
@@ -330,7 +333,8 @@ checkMappingAttributeTypes(std::optional<TransformOpInterface> transformOp,
         "scf.forall op requires a mapping attribute of kind 'block'");
   }
   if (std::is_same<MappingKindType, ThreadMappingKind>::value &&
-      !hasThreadMapping && !hasWarpMapping && !hasWarpgroupMapping) {
+      !hasLaneMapping && !hasThreadMapping && !hasWarpMapping &&
+      !hasWarpgroupMapping) {
     return definiteFailureHelper(transformOp, forallOp,
                                  "scf.forall op requires a mapping attribute "
                                  "of kind 'thread' or 'warp'");
@@ -473,13 +477,24 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   SmallVector<int64_t> originalBasis(availableMappingSizes);
   bool originalBasisWasProvided = !originalBasis.empty();
   if (!originalBasisWasProvided) {
+    LDBG("----originalBasis was not provided, deriving it and there will be no "
+         "predication");
     originalBasis = forallMappingSizes;
     while (originalBasis.size() < 3)
       originalBasis.push_back(1);
+  } else {
+    LDBG("----originalBasis was provided, using it, there will be predication");
   }
+  LLVM_DEBUG(
+      llvm::interleaveComma(originalBasis, DBGS() << "------originalBasis: ");
+      llvm::dbgs() << "\n");
 
   IdBuilderResult builderResult =
       gpuIdBuilder.idBuilder(rewriter, loc, forallMappingSizes, originalBasis);
+  if (!builderResult.errorMsg.empty())
+    return definiteFailureHelper(transformOp, forallOp, builderResult.errorMsg);
+
+  LLVM_DEBUG(DBGS() << builderResult);
 
   // Step 4. Map the induction variables to the mappingIdOps, this may involve
   // a permutation.
@@ -490,6 +505,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
            forallMappingAttrs.getArrayRef().take_front(forallOp.getRank()))) {
     auto mappingAttr = cast<DeviceMappingAttrInterface>(dim);
     Value peIdOp = mappingIdOps[mappingAttr.getRelativeIndex()];
+    LDBG("----map: " << iv << " to " << peIdOp);
     bvm.map(iv, peIdOp);
   }
 
@@ -498,32 +514,7 @@ static DiagnosedSilenceableFailure rewriteOneForallCommonImpl(
   // originalBasis and no predication occurs.
   Value predicate;
   if (originalBasisWasProvided) {
-    SmallVector<int64_t> activeMappingSizes = builderResult.activeMappingSizes;
-    SmallVector<int64_t> availableMappingSizes =
-        builderResult.availableMappingSizes;
-    SmallVector<Value> activeIdOps = builderResult.activeIdOps;
-    LDBG("----activeMappingSizes: " << llvm::interleaved(activeMappingSizes));
-    LDBG("----availableMappingSizes: "
-         << llvm::interleaved(availableMappingSizes));
-    LDBG("----activeIdOps: " << llvm::interleaved(activeIdOps));
-    for (auto [activeId, activeMappingSize, availableMappingSize] :
-         llvm::zip_equal(activeIdOps, activeMappingSizes,
-                         availableMappingSizes)) {
-      if (activeMappingSize > availableMappingSize) {
-        return definiteFailureHelper(
-            transformOp, forallOp,
-            "Trying to map to fewer GPU threads than loop iterations but "
-            "overprovisioning is not yet supported. "
-            "Try additional tiling of the before mapping or map to more "
-            "threads.");
-      }
-      if (activeMappingSize == availableMappingSize)
-        continue;
-      Value idx =
-          rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
-      Value tmpPredicate = rewriter.create<arith::CmpIOp>(
-          loc, arith::CmpIPredicate::ult, activeId, idx);
-      LDBG("----predicate: " << tmpPredicate);
+    for (Value tmpPredicate : builderResult.predicateOps) {
       predicate = predicate ? rewriter.create<arith::AndIOp>(loc, predicate,
                                                              tmpPredicate)
                             : tmpPredicate;
@@ -790,6 +781,9 @@ getThreadIdBuilder(std::optional<TransformOpInterface> transformOp,
           .Case([&](GPUThreadMappingAttr) {
             return GpuThreadIdBuilder(ctx, useLinearMapping);
           })
+          .Case([&](GPULaneMappingAttr) {
+            return GpuLaneIdBuilder(ctx, warpSize, useLinearMapping);
+          })
           .Default([&](DeviceMappingAttrInterface) -> GpuIdBuilder {
             llvm_unreachable("unknown mapping attribute");
           });
diff --git a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
index 9853e8082839..795d643c0591 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/Utils.cpp
@@ -47,12 +47,57 @@ using namespace mlir::transform::gpu;
 #define LDBG(X) LLVM_DEBUG(DBGS() << (X) << "\n")
 #define DBGS_ALIAS() (llvm::dbgs() << '[' << DEBUG_TYPE_ALIAS << "] ")
 
+/// Build predicates to filter execution by only the activeIds. Along each
+/// dimension, 3 cases appear:
+///   1. activeMappingSize > availableMappingSize: this is an unsupported case
+///      as this requires additional looping. An error message is produced to
+///      advise the user to tile more or to use more threads.
+///   2. activeMappingSize == availableMappingSize: no predication is needed.
+///   3. activeMappingSize < availableMappingSize: only a subset of threads
+///      should be active and we produce the boolean `id < activeMappingSize`
+///      for further use in building predicated execution.
+static FailureOr<SmallVector<Value>>
+buildPredicates(RewriterBase &rewriter, Location loc, ArrayRef<Value> activeIds,
+                ArrayRef<int64_t> activeMappingSizes,
+                ArrayRef<int64_t> availableMappingSizes,
+                std::string &errorMsg) {
+  // clang-format off
+  LLVM_DEBUG(
+    llvm::interleaveComma(
+      activeMappingSizes, DBGS() << "----activeMappingSizes: ");
+    DBGS() << "\n";
+    llvm::interleaveComma(
+      availableMappingSizes, DBGS() << "----availableMappingSizes: ");
+    DBGS() << "\n";);
+  // clang-format on
+
+  SmallVector<Value> predicateOps;
+  for (auto [activeId, activeMappingSize, availableMappingSize] :
+       llvm::zip_equal(activeIds, activeMappingSizes, availableMappingSizes)) {
+    if (activeMappingSize > availableMappingSize) {
+      errorMsg = "Trying to map to fewer GPU threads than loop iterations but "
+                 "overprovisioning is not yet supported. Try additional tiling "
+                 "before mapping or map to more threads.";
+      return failure();
+    }
+    if (activeMappingSize == availableMappingSize)
+      continue;
+    Value idx = rewriter.create<arith::ConstantIndexOp>(loc, activeMappingSize);
+    Value pred = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+                                                activeId, idx);
+    predicateOps.push_back(pred);
+  }
+  return predicateOps;
+}
+
 /// Return a flattened thread id for the workgroup with given sizes.
 template <typename ThreadOrBlockIdOp>
 static Value buildLinearId(RewriterBase &rewriter, Location loc,
                            ArrayRef<OpFoldResult> originalBasisOfr) {
-  LLVM_DEBUG(DBGS() << "----buildLinearId with originalBasisOfr:  "
-                    << llvm::interleaved(originalBasisOfr) << "\n");
+  LLVM_DEBUG(llvm::interleaveComma(
+                 originalBasisOfr,
+                 DBGS() << "----buildLinearId with originalBasisOfr:  ");
+             llvm::dbgs() << "\n");
   assert(originalBasisOfr.size() == 3 && "expected 3 sizes");
   IndexType indexType = rewriter.getIndexType();
   AffineExpr tx, ty, tz, bdx, bdy;
@@ -79,44 +124,43 @@ static GpuIdBuilderFnType commonLinearIdBuilderFn(int64_t multiplicity = 1) {
   auto res = [multiplicity](RewriterBase &rewriter, Location loc,
                             ArrayRef<int64_t> forallMappingSizes,
                             ArrayRef<int64_t> originalBasis) {
+    // 1. Compute linearId.
     SmallVector<OpFoldResult> originalBasisOfr =
         getAsIndexOpFoldResult(rewriter.getContext(), originalBasis);
-    OpFoldResult linearId =
+    Value physicalLinearId =
         buildLinearId<ThreadOrBlockIdOp>(rewriter, loc, originalBasisOfr);
+
+    // 2. Compute scaledLinearId.
+    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
+    OpFoldResult scaledLinearId = affine::makeComposedFoldedAffineApply(
+        rewriter, loc, d0.floorDiv(multiplicity), {physicalLinearId});
+
+    // 3. Compute remapped indices.
+    SmallVector<Value> ids;
     // Sizes in [0 .. n] -> [n .. 0] order to properly compute strides in
     // "row-major" order.
     SmallVector<int64_t> reverseBasisSizes(llvm::reverse(forallMappingSizes));
     SmallVector<int64_t> strides = computeStrides(reverseBasisSizes);
-    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
-    OpFoldResult scaledLinearId = affine::makeComposedFoldedAffineApply(
-        rewriter, loc, d0.floorDiv(multiplicity), {linearId});
     SmallVector<AffineExpr> delinearizingExprs = delinearize(d0, strides);
-    SmallVector<Value> ids;
     // Reverse back to be in [0 .. n] order.
     for (AffineExpr e : llvm::reverse(delinearizingExprs)) {
       ids.push_back(
           affine::makeComposedAffineApply(rewriter, loc, e, {scaledLinearId}));
     }
 
-    LLVM_DEBUG(DBGS() << "--delinearization basis: "
-                      << llvm::interleaved(reverseBasisSizes) << "\n";
-               DBGS() << "--delinearization strides: "
-                      << llvm::interleaved(strides) << "\n";
-               DBGS() << "--delinearization exprs: "
-                      << llvm::interleaved(delinearizingExprs) << "\n";
-               DBGS() << "--ids: " << llvm::interleaved(ids) << "\n");
-
-    // Return n-D ids for indexing and 1-D size + id for predicate generation.
-    return IdBuilderResult{
-        /*mappingIdOps=*/ids,
-        /*availableMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(originalBasis)},
-        // `forallMappingSizes` iterate in the scaled basis, they need to be
-        // scaled back into the original basis to provide tight
-        // activeMappingSizes quantities for predication.
-        /*activeMappingSizes=*/
-        SmallVector<int64_t>{computeProduct(forallMappingSizes) * multiplicity},
-        /*activeIdOps=*/SmallVector<Value>{cast<Value>(linearId)}};
+    // 4. Handle predicates using physicalLinearId.
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    FailureOr<SmallVector<Value>> maybePredicateOps =
+        buildPredicates(rewriter, loc, physicalLinearId,
+                        computeProduct(forallMappingSizes) * multiplicity,
+                        computeProduct(originalBasis), errorMsg);
+    if (succeeded(maybePredicateOps))
+      predicateOps = *maybePredicateOps;
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/ids,
+                           /*predicateOps=*/predicateOps};
   };
 
   return res;
@@ -143,19 +187,70 @@ static GpuIdBuilderFnType common3DIdBuilderFn(int64_t multiplicity = 1) {
     // In the 3-D mapping case, unscale the first dimension by the multiplicity.
     SmallVector<int64_t> forallMappingSizeInOriginalBasis(forallMappingSizes);
     forallMappingSizeInOriginalBasis[0] *= multiplicity;
-    return IdBuilderResult{
-        /*mappingIdOps=*/scaledIds,
-        /*availableMappingSizes=*/SmallVector<int64_t>{originalBasis},
-        // `forallMappingSizes` iterate in the scaled basis, they need to be
-        // scaled back into the original basis to provide tight
-        // activeMappingSizes quantities for predication.
-        /*activeMappingSizes=*/
-        SmallVector<int64_t>{forallMappingSizeInOriginalBasis},
-        /*activeIdOps=*/ids};
+
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    FailureOr<SmallVector<Value>> maybePredicateOps =
+        buildPredicates(rewriter, loc, ids, forallMappingSizeInOriginalBasis,
+                        originalBasis, errorMsg);
+    if (succeeded(maybePredicateOps))
+      predicateOps = *maybePredicateOps;
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/scaledIds,
+                           /*predicateOps=*/predicateOps};
   };
   return res;
 }
 
+/// Create a lane id builder that takes the `originalBasis` and decompose
+/// it in the basis of `forallMappingSizes`. The linear id builder returns an
+/// n-D vector of ids for indexing and 1-D size + id for predicate generation.
+static GpuIdBuilderFnType laneIdBuilderFn(int64_t warpSize) {
+  auto res = [warpSize](RewriterBase &rewriter, Location loc,
+                        ArrayRef<int64_t> forallMappingSizes,
+                        ArrayRef<int64_t> originalBasis) {
+    // 1. Compute linearId.
+    SmallVector<OpFoldResult> originalBasisOfr =
+        getAsIndexOpFoldResult(rewriter.getContext(), originalBasis);
+    Value physicalLinearId =
+        buildLinearId<ThreadIdOp>(rewriter, loc, originalBasisOfr);
+
+    // 2. Compute laneId.
+    AffineExpr d0 = getAffineDimExpr(0, rewriter.getContext());
+    OpFoldResult laneId = affine::makeComposedFoldedAffineApply(
+        rewriter, loc, d0 % warpSize, {physicalLinearId});
+
+    // 3. Compute remapped indices.
+    SmallVector<Value> ids;
+    // Sizes in [0 .. n] -> [n .. 0] order to properly compute strides in
+    // "row-major" order.
+    SmallVector<int64_t> reverseBasisSizes(llvm::reverse(forallMappingSizes));
+    SmallVector<int64_t> strides = computeStrides(reverseBasisSizes);
+    SmallVector<AffineExpr> delinearizingExprs = delinearize(d0, strides);
+    // Reverse back to be in [0 .. n] order.
+    for (AffineExpr e : llvm::reverse(delinearizingExprs)) {
+      ids.push_back(
+          affine::makeComposedAffineApply(rewriter, loc, e, {laneId}));
+    }
+
+    // 4. Handle predicates using laneId.
+    std::string errorMsg;
+    SmallVector<Value> predicateOps;
+    FailureOr<SmallVector<Value>> maybePredicateOps = buildPredicates(
+        rewriter, loc, cast<Value>(laneId), computeProduct(forallMappingSizes),
+        computeProduct(originalBasis), errorMsg);
+    if (succeeded(maybePredicateOps))
+      predicateOps = *maybePredicateOps;
+
+    return IdBuilderResult{/*errorMsg=*/errorMsg,
+                           /*mappingIdOps=*/ids,
+                           /*predicateOps=*/predicateOps};
+  };
+
+  return res;
+}
+
 namespace mlir {
 namespace transform {
 namespace gpu {
@@ -221,6 +316,16 @@ GpuThreadIdBuilder::GpuThreadIdBuilder(MLIRContext *ctx, bool useLinearMapping)
                   : common3DIdBuilderFn<ThreadIdOp>(/*multiplicity=*/1);
 }
 
+GpuLaneIdBuilder::GpuLaneIdBuilder(MLIRContext *ctx, int64_t warpSize,
+                                   bool unused)
+    : GpuIdBuilder(ctx, /*useLinearMapping=*/true,
+                   [](MLIRContext *ctx, MappingId id) {
+                     return GPULaneMappingAttr::get(ctx, id);
+                   }),
+      warpSize(warpSize) {
+  idBuilder = laneIdBuilderFn(/*periodicity=*/warpSize);
+}
+
 DiagnosedSilenceableFailure checkGpuLimits(TransformOpInterface transformOp,
                                            std::optional<int64_t> gridDimX,
                                            std::optional<int64_t> gridDimY,
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index 695d43b04cff..f63af8da2808 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -111,7 +111,8 @@ static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
       getFlatOffsetAndStrides(rewriter, loc, source, offsetsTemp);
   MemRefType retType = inferCastResultType(base, offset);
   return rewriter.create<memref::ReinterpretCastOp>(loc, retType, base, offset,
-                                                    std::nullopt, std::nullopt);
+                                                    ArrayRef<OpFoldResult>(),
+                                                    ArrayRef<OpFoldResult>());
 }
 
 static bool needFlatten(Value val) {
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index d83fd3800eb9..d987b72e9835 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -110,3 +110,26 @@ add_mlir_dialect_library(MLIRVCIXDialect
   MLIRLLVMDialect
   MLIRSideEffectInterfaces
   )
+
+add_mlir_dialect_library(MLIRXeVMDialect
+  IR/XeVMDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
+
+  DEPENDS
+  MLIRGPUCompilationAttrInterfacesIncGen
+  MLIRXeVMOpsIncGen
+  MLIRXeVMConversionsIncGen
+  intrinsics_gen
+
+  LINK_COMPONENTS
+  AsmParser
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRDialectUtils
+  MLIRIR
+  MLIRLLVMDialect
+  MLIRSideEffectInterfaces
+)
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index a12aef0dfad3..6dcd94e6eea1 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -128,7 +128,6 @@ REGISTER_ENUM_TYPE(Visibility);
 /// second template argument.
 template <typename EnumTy, typename RetTy = EnumTy>
 static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
-                                      OperationState &result,
                                       EnumTy defaultValue) {
   SmallVector<StringRef, 10> names;
   for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
@@ -1396,14 +1395,14 @@ ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   result.addAttribute(
       getTailCallKindAttrName(result.name),
       TailCallKindAttr::get(parser.getContext(),
                             parseOptionalLLVMKeyword<TailCallKind>(
-                                parser, result, LLVM::TailCallKind::None)));
+                                parser, LLVM::TailCallKind::None)));
 
   // Parse a function pointer for indirect calls.
   if (parseOptionalCallFuncPtr(parser, operands))
@@ -1631,8 +1630,8 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   // Parse a function pointer for indirect calls.
   if (parseOptionalCallFuncPtr(parser, operands))
@@ -2245,8 +2244,7 @@ void GlobalOp::print(OpAsmPrinter &p) {
                            getGlobalTypeAttrName(), getConstantAttrName(),
                            getValueAttrName(), getLinkageAttrName(),
                            getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
-                           getVisibility_AttrName(), getComdatAttrName(),
-                           getUnnamedAddrAttrName()});
+                           getVisibility_AttrName(), getComdatAttrName()});
 
   // Print the trailing type unless it's a string global.
   if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
@@ -2298,16 +2296,16 @@ static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
                                              OperationState &result) {
   MLIRContext *ctx = parser.getContext();
   // Parse optional linkage, default to External.
-  result.addAttribute(OpType::getLinkageAttrName(result.name),
-                      LLVM::LinkageAttr::get(
-                          ctx, parseOptionalLLVMKeyword<Linkage>(
-                                   parser, result, LLVM::Linkage::External)));
+  result.addAttribute(
+      OpType::getLinkageAttrName(result.name),
+      LLVM::LinkageAttr::get(ctx, parseOptionalLLVMKeyword<Linkage>(
+                                      parser, LLVM::Linkage::External)));
 
   // Parse optional visibility, default to Default.
   result.addAttribute(OpType::getVisibility_AttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
-                              parser, result, LLVM::Visibility::Default)));
+                              parser, LLVM::Visibility::Default)));
 
   if (succeeded(parser.parseOptionalKeyword("thread_local")))
     result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
@@ -2317,7 +2315,7 @@ static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
   result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
-                              parser, result, LLVM::UnnamedAddr::None)));
+                              parser, LLVM::UnnamedAddr::None)));
 
   return success();
 }
@@ -2588,7 +2586,7 @@ void AliasOp::print(OpAsmPrinter &p) {
                           {SymbolTable::getSymbolAttrName(),
                            getAliasTypeAttrName(), getLinkageAttrName(),
                            getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
-                           getVisibility_AttrName(), getUnnamedAddrAttrName()});
+                           getVisibility_AttrName()});
 
   // Print the trailing type.
   p << " : " << getType() << ' ';
@@ -2774,7 +2772,7 @@ void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
   assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
          "expected as many argument attribute lists as arguments");
   call_interface_impl::addArgAndResultAttrs(
-      builder, result, argAttrs, /*resultAttrs=*/std::nullopt,
+      builder, result, argAttrs, /*resultAttrs=*/{},
       getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
 }
 
@@ -2825,29 +2823,28 @@ buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef<Type> inputs,
 //
 ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
   // Default to external linkage if no keyword is provided.
-  result.addAttribute(
-      getLinkageAttrName(result.name),
-      LinkageAttr::get(parser.getContext(),
-                       parseOptionalLLVMKeyword<Linkage>(
-                           parser, result, LLVM::Linkage::External)));
+  result.addAttribute(getLinkageAttrName(result.name),
+                      LinkageAttr::get(parser.getContext(),
+                                       parseOptionalLLVMKeyword<Linkage>(
+                                           parser, LLVM::Linkage::External)));
 
   // Parse optional visibility, default to Default.
   result.addAttribute(getVisibility_AttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
-                              parser, result, LLVM::Visibility::Default)));
+                              parser, LLVM::Visibility::Default)));
 
   // Parse optional UnnamedAddr, default to None.
   result.addAttribute(getUnnamedAddrAttrName(result.name),
                       parser.getBuilder().getI64IntegerAttr(
                           parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
-                              parser, result, LLVM::UnnamedAddr::None)));
+                              parser, LLVM::UnnamedAddr::None)));
 
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
       getCConvAttrName(result.name),
-      CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
-                                              parser, result, LLVM::CConv::C)));
+      CConvAttr::get(parser.getContext(),
+                     parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
 
   StringAttr nameAttr;
   SmallVector<OpAsmParser::Argument> entryArgs;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
new file mode 100644
index 000000000000..9e497829ba72
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/XeVMDialect.cpp
@@ -0,0 +1,366 @@
+//===-- XeVMDialect.cpp - XeVM dialect registration -------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/DialectImplementation.h"
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MathExtras.h"
+
+using namespace mlir;
+using namespace mlir::xevm;
+
+#include "mlir/Dialect/LLVMIR/XeVMOpsDialect.cpp.inc"
+#include "mlir/Dialect/LLVMIR/XeVMOpsEnums.cpp.inc"
+
+namespace {
+static constexpr uint32_t subgroupSize = 16;
+
+template <typename Op>
+LogicalResult verifyMatrixInput(Op op) {
+  static_assert(llvm::is_one_of<Op, BlockLoad2dOp, BlockStore2dOp,
+                                BlockPrefetch2dOp>::value,
+                "Unexpected template parameter");
+
+  std::optional<int64_t> width = getConstantIntValue(op.getBaseWidth());
+  std::optional<int64_t> pitch = getConstantIntValue(op.getBasePitch());
+  if (pitch && width && *pitch < *width)
+    return op->emitOpError(
+        "4th operand (base pitch) should be >= 2nd operand (base width)");
+
+  uint32_t elemSize = op.getElemSizeInBits();
+  if (elemSize < 8 || !llvm::isPowerOf2_32(elemSize) || elemSize > 32)
+    return op->emitOpError("expecting 'elem_size_in_bits' to be 8, 16, or 32");
+
+  uint32_t tileHeight = op.getTileHeight();
+  if (tileHeight > 32 || !llvm::isPowerOf2_32(tileHeight))
+    return op->emitOpError("expecting tile_height to be 1, 2, 4, 8, 16, or 32");
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks > 8 || !llvm::isPowerOf2_32(vBlocks))
+    return op->emitOpError("expecting v_blocks to be 1, 2, 4, or 8");
+
+  return success();
+}
+
+LogicalResult verify2DBlockLoadRestriction(BlockLoad2dOp op) {
+  VectorType resTy = op.getRes().getType();
+  if (!resTy.getElementType().isIntOrFloat())
+    return op.emitOpError()
+           << "expecting result element type to be int or float";
+  unsigned resElemTySize = resTy.getElementType().getIntOrFloatBitWidth();
+  unsigned resSize = resTy.getNumElements() * resElemTySize;
+  unsigned expectedSize = op.getElemSizeInBits() * op.getTileHeight() *
+                          op.getTileWidth() * op.getVBlocks() / subgroupSize;
+  if (resSize != expectedSize)
+    return op.emitOpError() << "result size of " << resSize
+                            << " bits does not match the expected size of "
+                            << expectedSize << " bits";
+
+  if (op.getTranspose() && op.getPackRegister())
+    return op.emitOpError("transpose and pack_register are mutually exclusive");
+
+  if (!op.getTranspose() && !op.getPackRegister()) {
+    uint32_t tileHeight = op.getTileHeight();
+    if (tileHeight < 1 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 1 and 32");
+
+    uint32_t tileWidth = op.getTileWidth();
+    uint32_t vBlocks = op.getVBlocks();
+    switch (op.getElemSizeInBits()) {
+    case 8:
+      if (tileWidth < 4 || tileWidth > 64)
+        return op.emitOpError("expecting tile_width to be between 4 and 64");
+      if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+        return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+      if (tileWidth * vBlocks > 64)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 64 for 8 bit elements");
+      break;
+    case 16:
+      if (tileWidth < 2 || tileWidth > 32)
+        return op.emitOpError("expecting tile_width to be between 2 and 32");
+      if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+        return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+      if (tileWidth * vBlocks > 32)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 32 for 16 bit elements");
+      break;
+    case 32:
+      if (tileWidth < 1 || tileWidth > 16)
+        return op.emitOpError("expecting tile_width to be between 1 and 16");
+      if (vBlocks != 1 && vBlocks != 2)
+        return op.emitOpError("expecting v_blocks to be 1 or 2");
+      if (tileWidth * vBlocks > 16)
+        return op.emitOpError(
+            "tile_width * v_blocks should be less than or equal "
+            "to 16 for 32 bit elements");
+      break;
+    case 64:
+      if (tileWidth < 1 || tileWidth > 8)
+        return op.emitOpError("expecting tile_width to be between 1 and 8");
+      if (vBlocks != 1)
+        return op.emitOpError("expecting v_blocks to be 1");
+      break;
+    default:
+      return op.emitOpError(
+          "expecting elem_size_in_bits to be 8, 16, 32, or 64");
+    }
+
+    return success();
+  }
+
+  if (op.getTranspose()) {
+    assert(!op.getPackRegister() && "Expecting pack_register should be false");
+
+    uint32_t vBlocks = op.getVBlocks();
+    if (vBlocks != 1)
+      return op.emitOpError("expecting v_blocks to be 1");
+
+    uint32_t tileHeight = op.getTileHeight();
+    uint32_t tileWidth = op.getTileWidth();
+    switch (op.getElemSizeInBits()) {
+    case 32:
+      if (tileHeight < 1 || tileHeight > 32)
+        return op.emitOpError("expecting tile_height to be between 1 and 32");
+      if (tileWidth < 1 || tileWidth > 8)
+        return op.emitOpError("expecting tile_width to be between 1 and 8");
+      break;
+    case 64:
+      if (tileHeight != 8)
+        return op.emitOpError(
+            "expecting tile_height to be 8 for 64 bit elements");
+      if (tileWidth != 1 && tileWidth != 2 && tileWidth != 4)
+        return op.emitOpError("expecting tile_width to be 1, 2, or 4");
+      break;
+    default:
+      return op.emitOpError("transpose is only supported for 32 and 64 bit "
+                            "elements");
+    }
+
+    return success();
+  }
+
+  assert(op.getPackRegister() && !op.getTranspose() &&
+         "Expecting pack_register should be true and transpose should be "
+         "false");
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks != 1 && vBlocks != 2 && vBlocks != 4)
+    return op.emitOpError("expecting v_blocks to be 1, 2, or 4");
+
+  uint32_t tileHeight = op.getTileHeight();
+  uint32_t tileWidth = op.getTileWidth();
+  switch (op.getElemSizeInBits()) {
+  case 8:
+    if (tileHeight < 4 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 4 and 32");
+    if (tileWidth < 4 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 4 and 16");
+    break;
+  case 16:
+    if (tileHeight < 2 || tileHeight > 32)
+      return op.emitOpError("expecting tile_height to be between 2 and 32");
+    if (tileWidth < 2 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 2 and 16");
+    if (tileWidth * vBlocks > 32)
+      return op.emitOpError(
+          "tile_width * v_blocks should be less than or equal "
+          "to 32 for 16 bit elements");
+    break;
+  default:
+    return op.emitOpError("pack_register is only supported for 8 and 16 bit "
+                          "elements");
+  }
+
+  return success();
+}
+
+static LogicalResult verify2DBlockStoreRestriction(BlockStore2dOp op) {
+  uint32_t tileHeight = op.getTileHeight();
+  if (tileHeight < 1 || tileHeight > 8)
+    return op.emitOpError("expecting tile_height to be between 1 and 8");
+
+  uint32_t tileWidth = op.getTileWidth();
+  switch (op.getElemSizeInBits()) {
+  case 8:
+    if (tileWidth < 4 || tileWidth > 64)
+      return op.emitOpError("expecting tile_width to be between 4 and 64");
+    break;
+  case 16:
+    if (tileWidth < 2 || tileWidth > 32)
+      return op.emitOpError("expecting tile_width to be between 2 and 32");
+    break;
+  case 32:
+    if (tileWidth < 1 || tileWidth > 16)
+      return op.emitOpError("expecting tile_width to be between 1 and 16");
+    break;
+  case 64:
+    if (tileWidth < 1 || tileWidth > 8)
+      return op.emitOpError("expecting tile_width to be between 1 and 8");
+    break;
+  default:
+    return op.emitOpError("expecting elem_size_in_bits to be 8, 16, 32, or 64");
+  }
+
+  uint32_t vBlocks = op.getVBlocks();
+  if (vBlocks != 1)
+    return op.emitOpError("expecting v_blocks to be 1");
+  return success();
+}
+
+} // namespace
+
+LogicalResult BlockLoad2dOp::verify() {
+  if (verify2DBlockLoadRestriction(*this).failed())
+    return failure();
+
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  VectorType resTy = getRes().getType();
+  if (!resTy.getElementType().isIntOrFloat())
+    return emitOpError() << "expecting result element type to be int of float";
+  unsigned resElemTySize = resTy.getElementType().getIntOrFloatBitWidth();
+  if (getElemSizeInBits() == 32 || getPackRegister()) {
+    if (resElemTySize != 32)
+      return emitOpError() << "expecting result element type to be 32 bits";
+  }
+
+  uint32_t tileWidth = getTileWidth();
+  if (getPackRegister()) {
+    if (tileWidth != 16)
+      return emitOpError(
+          "tile_width when pack_register is true should be equal "
+          "to subgroup size (16 elements)");
+    return success();
+  }
+
+  return success();
+}
+
+LogicalResult BlockStore2dOp::verify() {
+  if (verify2DBlockStoreRestriction(*this).failed())
+    return failure();
+
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  uint32_t tileWidth = getTileWidth();
+  switch (getElemSizeInBits()) {
+  case 8:
+    if (tileWidth != 16 && tileWidth != 32)
+      return emitOpError("tile_width for 8 bit elements should be equal to "
+                         "16 or 32");
+    break;
+  case 16:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 16 bit elements should be equal "
+                         "to 16");
+    break;
+  case 32:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 32 bit elements should be equal "
+                         "to 16");
+    break;
+  default:
+    llvm_unreachable("unexpected element size");
+  }
+
+  return success();
+}
+
+LogicalResult BlockPrefetch2dOp::verify() {
+  if (verifyMatrixInput(*this).failed())
+    return failure();
+
+  uint32_t tileWidth = getTileWidth();
+  switch (getElemSizeInBits()) {
+  case 8:
+    if (tileWidth != 16 && tileWidth != 32)
+      return emitOpError("tile_width for 8 bit elements should be equal to "
+                         "16 or 32");
+    break;
+  case 16:
+    if (tileWidth != 16)
+      return emitOpError("tile_width for 16 bit elements should be equal "
+                         "to 16");
+    break;
+  case 32:
+    if (tileWidth != 8 && tileWidth != 16)
+      return emitOpError(
+          "tile_width for 32 bit elements should be equal to 8 or 16");
+    break;
+  default:
+    llvm_unreachable("unexpected element size");
+  }
+
+  return success();
+}
+
+LogicalResult MMAOp::verify() {
+  if (getC()) {
+    if (getResult().getType() != getC().getType())
+      return emitOpError("type of C operand must match result type");
+  }
+  return success();
+}
+
+LogicalResult
+XeVMTargetAttr::verify(function_ref<InFlightDiagnostic()> emitError, int O,
+                       StringRef triple, StringRef chip, DictionaryAttr flags,
+                       ArrayAttr linkFiles) {
+  if (O < 0 || O > 3) {
+    return emitError()
+           << "The optimization level must be a number between 0 and 3.";
+  }
+  if (triple.empty()) {
+    return emitError() << "The target triple cannot be empty.";
+  }
+  if (chip.empty()) {
+    return emitError() << "The target chip cannot be empty.";
+  }
+  if (linkFiles) {
+    for (Attribute fileAttr : linkFiles) {
+      if (auto fileStrAttr = llvm::dyn_cast<StringAttr>(fileAttr)) {
+        StringRef filePath = fileStrAttr.getValue();
+        if (filePath.empty()) {
+          return emitError() << "File paths in linkFiles cannot be empty.";
+        }
+        if (!llvm::sys::fs::exists(filePath)) {
+          return emitError() << "File '" << filePath << "' does not exist.";
+        }
+      }
+    }
+  }
+  return success();
+}
+
+void XeVMDialect::initialize() {
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/LLVMIR/XeVMOps.cpp.inc"
+      >();
+
+  addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc"
+      >();
+  declarePromisedInterface<mlir::gpu::TargetAttrInterface,
+                           mlir::xevm::XeVMTargetAttr>();
+}
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOps.cpp.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc"
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
index c256b18dd2b1..b6e168e95ee8 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgDialect.cpp
@@ -11,27 +11,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/Interfaces/DestinationStyleOpInterface.h"
-#include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
-#include "mlir/Parser/Parser.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/InliningUtils.h"
 
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index ca7f31dd6b51..94f2002fc51f 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -25,8 +25,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
-#include <numeric>
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 0cd563b687ce..75ec1030069c 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -19,20 +19,16 @@
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypeInterfaces.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
@@ -42,7 +38,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -117,8 +112,9 @@ OpFoldResult linalg::createFoldedDimOp(OpBuilder &b, Location loc, Value source,
 // Support for named Linalg ops defined in ods-gen.
 //===----------------------------------------------------------------------===//
 
-using RegionBuilderFn = llvm::function_ref<void(ImplicitLocOpBuilder &, Block &,
-                                                ArrayRef<NamedAttribute>)>;
+using RegionBuilderFn = llvm::function_ref<void(
+    ImplicitLocOpBuilder &, Block &, ArrayRef<NamedAttribute>,
+    function_ref<InFlightDiagnostic()>)>;
 
 /// Fills the region of a structured operation using the provided
 /// `regionBuilder`. The method is used by both named structured ops created by
@@ -128,6 +124,7 @@ using RegionBuilderFn = llvm::function_ref<void(ImplicitLocOpBuilder &, Block &,
 static void fillStructuredOpRegion(OpBuilder &opBuilder, Region &region,
                                    TypeRange inputTypes, TypeRange outputTypes,
                                    ArrayRef<NamedAttribute> attrs,
+                                   function_ref<InFlightDiagnostic()> emitError,
                                    RegionBuilderFn regionBuilder) {
   SmallVector<Type, 8> argTypes;
   SmallVector<Location, 8> argLocs;
@@ -148,7 +145,7 @@ static void fillStructuredOpRegion(OpBuilder &opBuilder, Region &region,
 
   opBuilder.setInsertionPointToStart(body);
   ImplicitLocOpBuilder b(opBuilder.getUnknownLoc(), opBuilder);
-  regionBuilder(b, *body, attrs);
+  regionBuilder(b, *body, attrs, emitError);
 
   // indexing_maps is an auto-generated method.
 
@@ -184,7 +181,8 @@ static void buildStructuredOp(OpBuilder &b, OperationState &state,
   // Create and fill the region of the structured operation.
   Region &region = *state.addRegion();
   fillStructuredOpRegion(b, region, TypeRange(inputs), TypeRange(outputs),
-                         state.attributes.getAttrs(), regionBuilder);
+                         state.attributes.getAttrs(), /*emitError=*/{},
+                         regionBuilder);
 }
 
 static void buildMatmulOp(OpBuilder &b, OperationState &state,
@@ -329,7 +327,7 @@ static void printCommonStructuredOpParts(OpAsmPrinter &p, ValueRange inputs,
 static ParseResult parseNamedStructuredOpRegion(
     OpAsmParser &parser, Region &region, unsigned numRegionArgs,
     TypeRange inputTypes, TypeRange outputTypes, ArrayRef<NamedAttribute> attrs,
-    RegionBuilderFn regionBuilder) {
+    RegionBuilderFn regionBuilder, SMLoc loc) {
   if (numRegionArgs != inputTypes.size() + outputTypes.size()) {
     return parser.emitError(
         parser.getCurrentLocation(),
@@ -339,9 +337,15 @@ static ParseResult parseNamedStructuredOpRegion(
   }
 
   OpBuilder opBuilder(parser.getContext());
-  fillStructuredOpRegion(opBuilder, region, inputTypes, outputTypes, attrs,
-                         regionBuilder);
-  return success();
+  ParseResult result = success();
+  fillStructuredOpRegion(
+      opBuilder, region, inputTypes, outputTypes, attrs,
+      [&]() {
+        result = failure();
+        return parser.emitError(loc);
+      },
+      regionBuilder);
+  return result;
 }
 
 static ParseResult
@@ -358,6 +362,7 @@ static ParseResult parseNamedStructuredOp(OpAsmParser &parser,
                                           RegionBuilderFn regionBuilder) {
   // TODO: Enable when ods-gen supports captures.
   SmallVector<Type, 1> inputTypes, outputTypes;
+  SMLoc loc = parser.getCurrentLocation();
   if (parseCommonStructuredOpParts(parser, result, inputTypes, outputTypes))
     return failure();
 
@@ -375,7 +380,7 @@ static ParseResult parseNamedStructuredOp(OpAsmParser &parser,
   std::unique_ptr<Region> region = std::make_unique<Region>();
   if (parseNamedStructuredOpRegion(parser, *region, numRegionArgs, inputTypes,
                                    outputTypes, result.attributes.getAttrs(),
-                                   regionBuilder))
+                                   regionBuilder, loc))
     return failure();
   result.addRegion(std::move(region));
 
@@ -435,9 +440,15 @@ class RegionBuilderHelper {
       : builder(builder), block(block) {}
 
   // Build the unary functions defined by OpDSL.
-  Value buildUnaryFn(UnaryFn unaryFn, Value arg) {
-    if (!isFloatingPoint(arg))
+  Value buildUnaryFn(UnaryFn unaryFn, Value arg,
+                     function_ref<InFlightDiagnostic()> emitError = {}) {
+    if (!isFloatingPoint(arg)) {
+      if (emitError) {
+        emitError() << "unsupported non numeric type";
+        return nullptr;
+      }
       llvm_unreachable("unsupported non numeric type");
+    }
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     switch (unaryFn) {
@@ -472,18 +483,34 @@ class RegionBuilderHelper {
     case UnaryFn::erf:
       return builder.create<math::ErfOp>(arg.getLoc(), arg);
     }
+    if (emitError) {
+      emitError() << "unsupported unary function";
+      return nullptr;
+    }
     llvm_unreachable("unsupported unary function");
   }
 
   // Build the binary functions defined by OpDSL.
-  Value buildBinaryFn(BinaryFn binaryFn, Value arg0, Value arg1) {
+  // If emitError is provided, an error will be emitted if the operation is not
+  // supported and a nullptr will be returned, otherwise an assertion will be
+  // raised.
+  Value buildBinaryFn(BinaryFn binaryFn, Value arg0, Value arg1,
+                      function_ref<InFlightDiagnostic()> emitError = {}) {
     bool allComplex = isComplex(arg0) && isComplex(arg1);
     bool allFloatingPoint = isFloatingPoint(arg0) && isFloatingPoint(arg1);
     bool allInteger = isInteger(arg0) && isInteger(arg1);
     bool allBool = allInteger && arg0.getType().getIntOrFloatBitWidth() == 1 &&
                    arg1.getType().getIntOrFloatBitWidth() == 1;
-    if (!allComplex && !allFloatingPoint && !allInteger)
+    if (!allComplex && !allFloatingPoint && !allInteger) {
+      if (emitError) {
+        emitError()
+            << "Cannot build binary Linalg operation: expects allComplex, "
+               "allFloatingPoint, or allInteger, got "
+            << arg0.getType() << " and " << arg1.getType();
+        return nullptr;
+      }
       llvm_unreachable("unsupported non numeric type");
+    }
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     switch (binaryFn) {
@@ -500,8 +527,13 @@ class RegionBuilderHelper {
         return builder.create<complex::SubOp>(arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
         return builder.create<arith::SubFOp>(arg0.getLoc(), arg0, arg1);
-      if (allBool)
+      if (allBool) {
+        if (emitError) {
+          emitError() << "unsupported operation: sub with bools";
+          return nullptr;
+        }
         llvm_unreachable("unsupported operation: sub with bools");
+      }
       return builder.create<arith::SubIOp>(arg0.getLoc(), arg0, arg1);
     case BinaryFn::mul:
       if (allComplex)
@@ -516,12 +548,22 @@ class RegionBuilderHelper {
         return builder.create<complex::DivOp>(arg0.getLoc(), arg0, arg1);
       if (allFloatingPoint)
         return builder.create<arith::DivFOp>(arg0.getLoc(), arg0, arg1);
-      if (allBool)
+      if (allBool) {
+        if (emitError) {
+          emitError() << "unsupported operation: div with bools";
+          return nullptr;
+        }
         llvm_unreachable("unsupported operation: div with bools");
+      }
       return builder.create<arith::DivSIOp>(arg0.getLoc(), arg0, arg1);
     case BinaryFn::div_unsigned:
-      if (!allInteger || allBool)
+      if (!allInteger || allBool) {
+        if (emitError) {
+          emitError() << "unsupported operation: unsigned div not on uint";
+          return nullptr;
+        }
         llvm_unreachable("unsupported operation: unsigned div not on uint");
+      }
       return builder.create<arith::DivUIOp>(arg0.getLoc(), arg0, arg1);
     case BinaryFn::max_signed:
       assert(!allComplex);
@@ -547,12 +589,16 @@ class RegionBuilderHelper {
       assert(allFloatingPoint);
       return builder.create<math::PowFOp>(arg0.getLoc(), arg0, arg1);
     }
+    if (emitError) {
+      emitError() << "unsupported binary function";
+      return nullptr;
+    }
     llvm_unreachable("unsupported binary function");
   }
 
   // Build the ternary functions defined by OpDSL.
-  Value buildTernaryFn(TernaryFn ternaryFn, Value arg0, Value arg1,
-                       Value arg2) {
+  Value buildTernaryFn(TernaryFn ternaryFn, Value arg0, Value arg1, Value arg2,
+                       function_ref<InFlightDiagnostic()> emitError = {}) {
     bool headBool =
         isInteger(arg0) && arg0.getType().getIntOrFloatBitWidth() == 1;
     bool tailFloatingPoint =
@@ -566,17 +612,26 @@ class RegionBuilderHelper {
         llvm_unreachable("unsupported non numeric type");
       return builder.create<arith::SelectOp>(arg0.getLoc(), arg0, arg1, arg2);
     }
+    if (emitError) {
+      emitError() << "unsupported ternary function";
+      return nullptr;
+    }
     llvm_unreachable("unsupported ternary function");
   }
 
   // Build the type functions defined by OpDSL.
-  Value buildTypeFn(TypeFn typeFn, Type toType, Value operand) {
+  Value buildTypeFn(TypeFn typeFn, Type toType, Value operand,
+                    function_ref<InFlightDiagnostic()> emitError = {}) {
     switch (typeFn) {
     case TypeFn::cast_signed:
       return cast(toType, operand, false);
     case TypeFn::cast_unsigned:
       return cast(toType, operand, true);
     }
+    if (emitError) {
+      emitError() << "unsupported type conversion function";
+      return nullptr;
+    }
     llvm_unreachable("unsupported type conversion function");
   }
 
@@ -617,6 +672,13 @@ class RegionBuilderHelper {
     OpBuilder::InsertionGuard g(builder);
     builder.setInsertionPointToEnd(&block);
     auto loc = operand.getLoc();
+    if (isa<UnknownLoc>(loc)) {
+      if (operand.getDefiningOp())
+        loc = operand.getDefiningOp()->getLoc();
+      else if (operand.getParentBlock() &&
+               operand.getParentBlock()->getParentOp())
+        loc = operand.getParentBlock()->getParentOp()->getLoc();
+    }
     return convertScalarToDtype(builder, loc, operand, toType, isUnsignedCast);
   }
 
@@ -2989,8 +3051,9 @@ LogicalResult WinogradFilterTransformOp::verify() {
   ArrayRef<int64_t> filterShape = filterType.getShape();
   int64_t filterH = filterShape[getFilterHDim()];
   int64_t filterW = filterShape[getFilterWDim()];
-  int64_t r = getR();
-  int64_t m = getM();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
 
   if (filterH != r && filterH != 1)
     return emitOpError("expect filter height either equals to r or 1");
@@ -3046,8 +3109,9 @@ LogicalResult WinogradFilterTransformOp::getResultTilePosition(
   ArrayRef<int64_t> filterShape = filterType.getShape();
   int64_t filterH = filterShape[getFilterHDim()];
   int64_t filterW = filterShape[getFilterWDim()];
-  int64_t m = getM();
-  int64_t r = getR();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   int64_t alpha = m + r - 1;
   int64_t alphaH = filterH != 1 ? alpha : 1;
   int64_t alphaW = filterW != 1 ? alpha : 1;
@@ -3124,8 +3188,9 @@ LogicalResult WinogradInputTransformOp::verify() {
   ArrayRef<int64_t> inputShape = inputType.getShape();
   int64_t inputH = inputShape[getInputHDim()];
   int64_t inputW = inputShape[getInputWDim()];
-  int m = getM();
-  int r = getR();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   int64_t tileSize = m + r - 1;
 
   auto outputType = cast<ShapedType>(getOutput().getType());
@@ -3194,8 +3259,9 @@ LogicalResult WinogradInputTransformOp::getResultTilePosition(
   int64_t outputAlphaH = outputShape[getOutputAlphaHDim()];
   int64_t outputAlphaW = outputShape[getOutputAlphaWDim()];
 
-  int64_t m = getM();
-  int64_t r = getR();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   int64_t alpha = m + r - 1;
   int64_t alphaH = outputAlphaH != 1 ? alpha : 1;
   int64_t alphaW = outputAlphaW != 1 ? alpha : 1;
@@ -3224,8 +3290,9 @@ WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
                                                  ArrayRef<OpFoldResult> offsets,
                                                  ArrayRef<OpFoldResult> sizes) {
   IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
-  int64_t m = getM();
-  int64_t r = getR();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
 
   ShapedType outputType = getOutputOperandType();
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -3303,8 +3370,9 @@ LogicalResult WinogradOutputTransformOp::verify() {
   int64_t valueW = valueShape[getValueAlphaWDim()];
   int64_t valueTileH = valueShape[getValueTileHDim()];
   int64_t valueTileW = valueShape[getValueTileWDim()];
-  int m = getM();
-  int r = getR();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   bool leftTransform = valueH != 1;
   bool rightTransform = valueW != 1;
 
@@ -3365,7 +3433,9 @@ LogicalResult WinogradOutputTransformOp::getResultTilePosition(
     OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
     ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
     SmallVector<OpFoldResult> &resultSizes) {
-  int64_t m = getM();
+  WinogradConv2DFmr fmr = getFmr();
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
 
   Location loc = getLoc();
   MLIRContext *context = builder.getContext();
@@ -3623,6 +3693,27 @@ verifyExtendedBatchVariantMatmulSemantic(OpTy batchVariantMatmulOp,
 namespace mlir {
 namespace linalg {
 
+std::optional<WinogradConv2DFmr> getWinogradConv2DFmr(int64_t m, int64_t r) {
+  if (m == 2 && r == 3)
+    return WinogradConv2DFmr::F_2_3;
+  if (m == 4 && r == 3)
+    return WinogradConv2DFmr::F_4_3;
+  if (m == 2 && r == 5)
+    return WinogradConv2DFmr::F_2_5;
+  return std::nullopt;
+}
+
+std::pair<int64_t, int64_t> getFmrFromWinogradConv2DFmr(WinogradConv2DFmr fmr) {
+  switch (fmr) {
+  case WinogradConv2DFmr::F_2_3:
+    return {2, 3};
+  case WinogradConv2DFmr::F_4_3:
+    return {4, 3};
+  case WinogradConv2DFmr::F_2_5:
+    return {2, 5};
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // MatMulOp
 //===----------------------------------------------------------------------===//
@@ -3665,9 +3756,15 @@ bool MatmulOp::hasUserDefinedMaps() {
 /// Implements the block region builder for the MatmulOp. This is called by
 /// 'fillStructuredOpRegion'.
 void MatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                             ArrayRef<NamedAttribute> attrs) {
-  assert(3 > 0 && block.getNumArguments() == 3 &&
-         "MatmulOp regionBuilder expects 3 (>=0) args");
+                             ArrayRef<NamedAttribute> attrs,
+                             function_ref<InFlightDiagnostic()> emitError) {
+  if (emitError && block.getNumArguments() != 3) {
+    emitError() << "MatmulOp regionBuilder expects 3 args, got "
+                << block.getNumArguments();
+    return;
+  }
+  assert(block.getNumArguments() == 3 &&
+         "MatmulOp regionBuilder expects 3 args");
   RegionBuilderHelper helper(b, block);
   SmallVector<Value> yields;
 
@@ -3684,9 +3781,13 @@ void MatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
                                     block.getArgument(0));
   Value value2 = helper.buildTypeFn(castVal, block.getArgument(2).getType(),
                                     block.getArgument(1));
-  Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2);
-  Value value4 =
-      helper.buildBinaryFn(BinaryFn::add, block.getArgument(2), value3);
+  Value value3 = helper.buildBinaryFn(BinaryFn::mul, value1, value2, emitError);
+  if (!value3)
+    return;
+  Value value4 = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2),
+                                      value3, emitError);
+  if (!value4)
+    return;
   yields.push_back(value4);
   helper.yieldOutputs(yields);
 }
@@ -3814,7 +3915,13 @@ unsigned ContractOp::getNumRegionArgs() { return 3; }
 
 /// Implement block region builder, which is called by 'fillStructuredOpRegion'.
 void ContractOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                               ArrayRef<NamedAttribute> attrs) {
+                               ArrayRef<NamedAttribute> attrs,
+                               function_ref<InFlightDiagnostic()> emitError) {
+  if (emitError && block.getNumArguments() != 3) {
+    emitError() << "ContractOp regionBuilder expects 3 args, got "
+                << block.getNumArguments();
+    return;
+  }
   assert(block.getNumArguments() == 3 &&
          "ContractOp regionBuilder expects 3 args");
   RegionBuilderHelper helper(b, block);
@@ -3834,10 +3941,14 @@ void ContractOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
       helper.buildTypeFn(castSignedness, outType, block.getArgument(0));
   Value rhsAtOutType =
       helper.buildTypeFn(castSignedness, outType, block.getArgument(1));
-  Value productAtOutType =
-      helper.buildBinaryFn(BinaryFn::mul, lhsAtOutType, rhsAtOutType);
+  Value productAtOutType = helper.buildBinaryFn(BinaryFn::mul, lhsAtOutType,
+                                                rhsAtOutType, emitError);
+  if (!productAtOutType)
+    return;
   Value result = helper.buildBinaryFn(BinaryFn::add, block.getArgument(2),
-                                      productAtOutType);
+                                      productAtOutType, emitError);
+  if (!result)
+    return;
   helper.yieldOutputs({result});
 }
 
@@ -4029,10 +4140,16 @@ bool BatchMatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS) {
   return isValid;
 }
 
-void BatchMatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                                  ArrayRef<NamedAttribute> attrs) {
+void BatchMatmulOp::regionBuilder(
+    ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
+    function_ref<InFlightDiagnostic()> emitError) {
+  if (emitError && block.getNumArguments() != 3) {
+    emitError() << "BatchMatmulOp regionBuilder expects 3 args, got "
+                << block.getNumArguments();
+    return;
+  }
   assert(block.getNumArguments() == 3 &&
-         "BatchMatmulOp regionBuilder expects 3 (>=0) args");
+         "BatchMatmulOp regionBuilder expects 3 args");
   RegionBuilderHelper helper(b, block);
   SmallVector<Value> yields;
 
@@ -4304,8 +4421,9 @@ LogicalResult ElementwiseOp::verify() {
 
 /// Implements the block region builder for the ElementwiseOp. This is called by
 /// 'fillStructuredOpRegion'.
-void ElementwiseOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                                  ArrayRef<NamedAttribute> attrs) {
+void ElementwiseOp::regionBuilder(
+    ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
+    function_ref<InFlightDiagnostic()> emitError) {
   ElementwiseKind elemwiseKind;
   for (auto attr : attrs) {
     if (attr.getName() == b.getStringAttr("kind")) {
@@ -4319,6 +4437,13 @@ void ElementwiseOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
   ArityGroupAndKind groupAndKind = getArityGroupAndKind(elemwiseKind);
   auto arityGroup = groupAndKind.arityGroup;
   auto kind = groupAndKind.kind;
+  if (emitError && block.getNumArguments() !=
+                       getArityGroupAsUInt(arityGroup) + 1 /*output*/) {
+    emitError() << "Elementwise regionBuilder expects "
+                << (getArityGroupAsUInt(arityGroup) + 1) << " args, got "
+                << block.getNumArguments();
+    return;
+  }
   assert(block.getNumArguments() ==
              getArityGroupAsUInt(arityGroup) + 1 /*output*/
          && "Elementwise regionBuilder number of block args mismatch");
@@ -5502,10 +5627,16 @@ bool BatchReduceMatmulOp::isValidLhsRhsBroadcastMap(AffineMap bcastMap,
   return isValid;
 }
 
-void BatchReduceMatmulOp::regionBuilder(ImplicitLocOpBuilder &b, Block &block,
-                                        ArrayRef<NamedAttribute> attrs) {
+void BatchReduceMatmulOp::regionBuilder(
+    ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs,
+    function_ref<InFlightDiagnostic()> emitError) {
+  if (emitError && block.getNumArguments() != 3) {
+    emitError() << "BatchReduceMatmulOp regionBuilder expects 3 args, got "
+                << block.getNumArguments();
+    return;
+  }
   assert(block.getNumArguments() == 3 &&
-         "BatchReduceMatmulOp regionBuilder expects 3 (>=0) args");
+         "BatchReduceMatmulOp regionBuilder expects 3 args");
   RegionBuilderHelper helper(b, block);
   SmallVector<Value> yields;
 
diff --git a/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
index 8d6d9dc690b5..f56ef485069f 100644
--- a/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Dialect/Linalg/IR/ValueBoundsOpInterfaceImpl.h"
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Interfaces/IndexingMapOpInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
index 4591802ce74a..a1a91ca060b1 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/DialectExtension.cpp
@@ -14,11 +14,9 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/TransformOps/LinalgMatchOps.h"
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h"
-#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
-#include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
index 67bb1de59dd4..2fe72a326651 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgMatchOps.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/MatchInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/Interfaces/FunctionImplementation.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/InterleavedRange.h"
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 2355edea2df6..7ec3f3445281 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -24,9 +24,6 @@
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/SCF/Transforms/TileUsingInterface.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
-#include "mlir/Dialect/Transform/IR/TransformDialect.h"
-#include "mlir/Dialect/Transform/IR/TransformOps.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
@@ -39,7 +36,6 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Support/TypeID.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
@@ -2408,6 +2404,9 @@ transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter,
   if (getUseFullTilesByDefault())
     promotionOptions = promotionOptions.setUseFullTileBuffersByDefault(
         getUseFullTilesByDefault());
+  if (getUseOriginalSubviewSize())
+    promotionOptions =
+        promotionOptions.setUseOriginalSubviewSize(getUseOriginalSubviewSize());
   if (getUseAlloca())
     promotionOptions = promotionOptions.setUseAlloca(getUseAlloca());
   if (!getUseFullTileBuffers().empty())
@@ -3823,9 +3822,14 @@ struct VectorizationPattern : public RewritePattern {
     if (!linalg::hasVectorizationImpl(op))
       return rewriter.notifyMatchFailure(op,
                                          "Unsupported Op, cannot vectorize");
-    return vectorize(rewriter, op, /*inputVectorSizes=*/{},
-                     /*inputScalableVecDims=*/{}, vectorizeNDExtract,
-                     flatten1DDepthwiseConv);
+    FailureOr<VectorizationResult> vectorResults =
+        vectorize(rewriter, op, /*inputVectorSizes=*/{},
+                  /*inputScalableVecDims=*/{}, vectorizeNDExtract,
+                  flatten1DDepthwiseConv);
+    if (failed(vectorResults))
+      return failure();
+    rewriter.replaceOp(op, vectorResults->replacements);
+    return success();
   }
 
 private:
@@ -3914,13 +3918,14 @@ DiagnosedSilenceableFailure transform::VectorizeOp::apply(
       return mlir::emitSilenceableFailure(target->getLoc())
              << "Unsupported Op, cannot vectorize";
     }
-
-    if (failed(linalg::vectorize(rewriter, target, vectorSizes,
-                                 getScalableSizes(),
-                                 getVectorizeNdExtract().value_or(false)))) {
+    FailureOr<VectorizationResult> vectorResults =
+        linalg::vectorize(rewriter, target, vectorSizes, getScalableSizes(),
+                          getVectorizeNdExtract().value_or(false));
+    if (failed(vectorResults)) {
       return mlir::emitSilenceableFailure(target->getLoc())
              << "Attempted to vectorize, but failed";
     }
+    rewriter.replaceOp(target, vectorResults->replacements);
   }
 
   return DiagnosedSilenceableFailure::success();
@@ -4244,7 +4249,7 @@ DiagnosedSilenceableFailure transform::WinogradConv2DOp::applyToOne(
   bool supported = TypeSwitch<Operation *, bool>(target)
                        .Case([&](linalg::Conv2DNhwcFhwcOp op) {
                          maybeTransformed =
-                             winogradConv2D(rewriter, op, getM(), getR());
+                             winogradConv2D(rewriter, op, getFmr());
                          return true;
                        })
                        .Default([&](Operation *op) { return false; });
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
index 81842e4bea63..3908d73f5e0e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
@@ -14,7 +14,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
 
 #include <optional>
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp b/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
index 428422e6e875..9a16318342d1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BubbleUpExtractSlice.cpp
@@ -13,12 +13,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 using namespace mlir::linalg;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
index 9044d89c80bd..1f6d96ca0f81 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -8,11 +8,9 @@
 
 #include "mlir/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
index b53180b5cf7c..edcb39d6e9a9 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConstantFold.cpp
@@ -10,13 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <optional>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
index 81d44ba04fa1..a7732b939e70 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertConv2DToImg2Col.cpp
@@ -19,7 +19,6 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 573420f6a9aa..39e2aac27e21 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -14,7 +14,6 @@
 //===----------------------------------------------------------------------===//
 //
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
@@ -24,7 +23,6 @@
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/Debug.h"
 
 using namespace mlir;
 using namespace mlir::tensor;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
index 7188987e5e93..70574903f711 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp
@@ -6,17 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Linalg/Passes.h"
-
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -358,6 +353,12 @@ static GenericOp packGenericOp(RewriterBase &rewriter, GenericOp genericOp,
   return newGenericOp;
 }
 
+static bool isGenericOutsNotUsed(linalg::GenericOp genericOp) {
+  return llvm::all_of(genericOp.getDpsInitsMutable(), [&](OpOperand &operand) {
+    return genericOp.getMatchingBlockArgument(&operand).use_empty();
+  });
+}
+
 /// Bubbles up linalg.pack op through a producer generic op. This
 /// swap pack(generic) to generic(pack). The new generic op works on packed
 /// domain; pack ops are created for input and output operands. E.g.,
@@ -470,12 +471,15 @@ bubbleUpPackOpThroughGenericOp(RewriterBase &rewriter, linalg::PackOp packOp,
       getOrCreatePackedViewOfOperand(rewriter, genericOp.getLoc(), *packInfo,
                                      genericOp, opOperand);
 
-  // If the dps init operand of the generic is a tensor.empty forward the pack
-  // op destination.
+  // Forward the new tensor.empty as a destination if it is one of the following
+  // situations:
+  // 1) The dps init operand is a tensor.empty.
+  // 2) The dps init is a write-only operand, i.e., it is not used in the
+  // genericOp
   Value dest = packedOutOperand;
-  if (auto initTensor = genericOp.getDpsInitOperand(0)
-                            ->get()
-                            .getDefiningOp<tensor::EmptyOp>()) {
+  auto initTensor =
+      genericOp.getDpsInitOperand(0)->get().getDefiningOp<tensor::EmptyOp>();
+  if (initTensor || isGenericOutsNotUsed(genericOp)) {
     dest = packOpDest;
   }
   // pack(unpack) isn't naively foldable because the unpack op can be from
@@ -1101,12 +1105,15 @@ pushDownUnPackOpThroughGenericOp(RewriterBase &rewriter, GenericOp genericOp,
                                      genericOp, genericOp.getDpsInitOperand(0));
   auto destPack = packedOutOperand.getDefiningOp<linalg::PackOp>();
 
-  // If the dps init operand of the generic is a tensor.empty, do not pack it
-  // and forward the new tensor.empty as a destination.
+  // Forward the new tensor.empty as a destination if it is one of the following
+  // situations:
+  // 1) The dps init operand is a tensor.empty.
+  // 2) The dps init is a write-only operand, i.e., it is not used in the
+  // genericOp
   Value dest = packedOutOperand;
-  if (auto initTensor = genericOp.getDpsInitOperand(0)
-                            ->get()
-                            .getDefiningOp<tensor::EmptyOp>()) {
+  auto initTensor =
+      genericOp.getDpsInitOperand(0)->get().getDefiningOp<tensor::EmptyOp>();
+  if (initTensor || isGenericOutsNotUsed(genericOp)) {
     if (destPack)
       dest = destPack.getDest();
   }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
index daeae2c9d947..692bf595267d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DecomposeGenericByUnfoldingPermutation.cpp
@@ -6,11 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 //
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include <map>
-#include <optional>
 #include <utility>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
index 20ab5ecbe3c4..ef24eb881d68 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Detensorize.cpp
@@ -9,15 +9,12 @@
 #include "mlir/Dialect/Linalg/Passes.h"
 
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Func/Transforms/FuncConversions.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include <iterator>
-#include <memory>
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index c5d9a729a413..e0062d15e61c 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -22,15 +22,12 @@
 #include "mlir/Dialect/MemRef/Transforms/Transforms.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
index 28d6752fc2d3..c4af09ca0142 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseToLinalg.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Dialect/Linalg/Passes.h"
 
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp b/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
index b710bde87f9f..b6c4b98301e4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/EliminateEmptyTensors.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
-#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
 #include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
index fcf049e9ce72..b1c0c3b161b2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FoldIntoElementwise.cpp
@@ -17,7 +17,6 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGFOLDINTOELEMENTWISEPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
index 703db8373c31..44469bc404a7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusePadOpWithLinalgProducer.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 4fc8a1755443..9bc7be262384 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -10,10 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
-#include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -23,17 +20,9 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/RegionUtils.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
-#include <optional>
-#include <set>
-
 #define DEBUG_TYPE "linalg-fusion"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
index 78cee47c497e..05f2157b77ae 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Generalization.cpp
@@ -13,17 +13,12 @@
 
 #include "mlir/Dialect/Linalg/Passes.h"
 
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/Debug.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGGENERALIZENAMEDOPSPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index bfeeb7756860..94ed46442180 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -20,7 +20,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/Tensor/Utils/Utils.h"
-#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/Matchers.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 707b63ff9335..f2e51c29f324 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -15,23 +15,14 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/Utils.h"
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Utils/Utils.h"
-#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Utils/VectorUtils.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 
 using llvm::dbgs;
@@ -303,23 +294,51 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
       //   1. indices, vector type and permutation map are the same (i.e., the
       //      transfer_read/transfer_write ops are matching),
       //   2. source operands for transfer.{read|write} do not originate from
-      //      Ops implementing ViewLikeOpInterface.
+      //      nor have users that are Ops implementing ViewLikeOpInterface.
       //   3. no other operations in the loop access the same memref except
       //      for transfer_read/transfer_write accessing statically disjoint
       //      slices.
+
+      // Check 1.
       if (transferRead.getIndices() != transferWrite.getIndices() ||
           transferRead.getVectorType() != transferWrite.getVectorType() ||
           transferRead.getPermutationMap() != transferWrite.getPermutationMap())
         return WalkResult::advance();
 
-      auto *source = transferRead.getBase().getDefiningOp();
-      if (source && isa_and_nonnull<ViewLikeOpInterface>(source))
-        return WalkResult::advance();
+      // Check 2. Note, since both xfer Ops share the source, we only need to
+      // look at one of them.
+      auto base = transferRead.getBase();
+      auto *source = base.getDefiningOp();
+      if (source) {
+        // NOTE: We treat `memref.assume_alignment` as a special case.
+        //
+        // The idea is that it is safe to look past AssumeAlignmemtOp (i.e.
+        // MemRef _before_ alignment) iff:
+        //  1. It has exactly two uses (these have to be the xfer Ops
+        //     being looked at).
+        //  2. The original MemRef has only one use (i.e.
+        //     AssumeAlignmentOp).
+        //
+        // Relaxing these conditions will most likely require proper alias
+        // analysis.
+        if (auto assume = dyn_cast<memref::AssumeAlignmentOp>(source)) {
+          Value memPreAlignment = assume.getMemref();
+          auto numInLoopUses =
+              llvm::count_if(base.getUses(), [&loop](OpOperand &use) {
+                return loop->isAncestor(use.getOwner());
+              });
+
+          if (numInLoopUses && memPreAlignment.hasOneUse())
+            source = memPreAlignment.getDefiningOp();
+        }
+        if (isa_and_nonnull<ViewLikeOpInterface>(source))
+          return WalkResult::advance();
+      }
 
-      source = transferWrite.getBase().getDefiningOp();
-      if (source && isa_and_nonnull<ViewLikeOpInterface>(source))
+      if (llvm::any_of(base.getUsers(), llvm::IsaPred<ViewLikeOpInterface>))
         return WalkResult::advance();
 
+      // Check 3.
       // TODO: may want to memoize this information for performance but it
       // likely gets invalidated often.
       DominanceInfo dom(loop);
@@ -358,7 +377,8 @@ void mlir::linalg::hoistRedundantVectorTransfers(Operation *root,
       // Hoist write after.
       transferWrite->moveAfter(loop);
 
-      // Rewrite `loop` with new yields by cloning and erase the original loop.
+      // Rewrite `loop` with new yields by cloning and erase the original
+      // loop.
       IRRewriter rewriter(transferRead.getContext());
       NewYieldValuesFn yieldFn = [&](OpBuilder &b, Location loc,
                                      ArrayRef<BlockArgument> newBBArgs) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index 785330736b39..a92a0c83e031 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -18,14 +18,9 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/raw_ostream.h"
-#include <type_traits>
 
 #define DEBUG_TYPE "linalg-interchange"
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 5b2ec4a6ff52..488041a43a2e 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -10,7 +10,6 @@
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
index 92cfba2549a3..ee1957aaa6a5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/MeshShardingInterfaceImpl.cpp
@@ -29,15 +29,12 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
-#include "mlir/Interfaces/TilingInterface.h"
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include <iterator>
 #include <numeric>
 #include <optional>
-#include <utility>
 
 namespace mlir::linalg {
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
index 0984b6988b93..2afa2f9b71c2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PackAndUnpackPatterns.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/IR/PatternMatch.h"
 
@@ -197,7 +197,9 @@ struct SimplifyUnPackToCollapseShape : public OpRewritePattern<UnPackOp> {
 /// Fold a `pad` -> `pack` into `pack` if they have the same padding values and
 /// the pad op has zero low paddings, or if `pack` has no padding values.
 struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
-  using OpRewritePattern<PackOp>::OpRewritePattern;
+public:
+  FoldPadWithPackOp(MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<PackOp>(context), controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(PackOp packOp,
                                 PatternRewriter &rewriter) const override {
@@ -206,6 +208,10 @@ struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
     if (!padOp || padOp.getNofold() || !padOp.hasZeroLowPad())
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&packOp.getSourceMutable()))
+      return failure();
+
     Value constantPaddingValue = padOp.getConstantPaddingValue();
     if (!constantPaddingValue)
       return failure();
@@ -220,13 +226,20 @@ struct FoldPadWithPackOp : public OpRewritePattern<PackOp> {
         packOp.getOuterDimsPerm());
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold a `unpack` -> `extract_slice` into the `unpack` since it already
 /// has extract_slice semantics.
 struct FoldUnpackWithExtractSliceOp
     : public OpRewritePattern<tensor::ExtractSliceOp> {
-  using OpRewritePattern<tensor::ExtractSliceOp>::OpRewritePattern;
+public:
+  FoldUnpackWithExtractSliceOp(MLIRContext *context,
+                               ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<tensor::ExtractSliceOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(tensor::ExtractSliceOp sliceOp,
                                 PatternRewriter &rewriter) const override {
@@ -234,6 +247,10 @@ struct FoldUnpackWithExtractSliceOp
     if (!unpackOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&sliceOp.getSourceMutable()))
+      return failure();
+
     if (sliceOp.getResultType().getRank() != unpackOp.getDestType().getRank()) {
       return rewriter.notifyMatchFailure(
           sliceOp, "rank-reduced folding is not supported");
@@ -255,6 +272,9 @@ struct FoldUnpackWithExtractSliceOp
         unpackOp.getMixedTiles(), unpackOp.getOuterDimsPerm());
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 // Applies 'permutation' on 'inVec' and stores the result in resVec.
@@ -284,7 +304,12 @@ static bool checkAndPermute(ArrayRef<int64_t> permutation,
 /// semantics.
 struct FoldProducerPackWithConsumerLinalgTransposeOp
     : public OpInterfaceRewritePattern<linalg::LinalgOp> {
-  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
+
+public:
+  FoldProducerPackWithConsumerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpInterfaceRewritePattern<linalg::LinalgOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
@@ -293,6 +318,10 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
     if (!packOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&linalgOp->getOpOperand(0)))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -331,13 +360,20 @@ struct FoldProducerPackWithConsumerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'transpose' -> 'pack' into 'pack' since 'pack' already has transpose
 /// semantics.
 struct FoldConsumerPackWithProducerLinalgTransposeOp
     : public OpRewritePattern<PackOp> {
-  using OpRewritePattern<PackOp>::OpRewritePattern;
+
+public:
+  FoldConsumerPackWithProducerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<PackOp>(context), controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(PackOp packOp,
                                 PatternRewriter &rewriter) const override {
@@ -345,6 +381,10 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
     if (!linalgOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&packOp.getSourceMutable()))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -375,13 +415,21 @@ struct FoldConsumerPackWithProducerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'unpack' -> 'transpose' into 'unpack' since 'unpack' already has
 /// transpose semantics.
 struct FoldProducerUnPackWithConsumerLinalgTransposeOp
     : public OpInterfaceRewritePattern<linalg::LinalgOp> {
-  using OpInterfaceRewritePattern<linalg::LinalgOp>::OpInterfaceRewritePattern;
+
+public:
+  FoldProducerUnPackWithConsumerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpInterfaceRewritePattern<linalg::LinalgOp>(context),
+        controlFn(std::move(controlFn)) {}
 
   LogicalResult matchAndRewrite(linalg::LinalgOp linalgOp,
                                 PatternRewriter &rewriter) const override {
@@ -390,6 +438,10 @@ struct FoldProducerUnPackWithConsumerLinalgTransposeOp
     if (!unPackOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&linalgOp->getOpOperand(0)))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -416,6 +468,9 @@ struct FoldProducerUnPackWithConsumerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// Fold 'transpose' -> 'unpack' into 'unpack' since 'unpack' already has
@@ -424,12 +479,21 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
     : public OpRewritePattern<UnPackOp> {
   using OpRewritePattern<UnPackOp>::OpRewritePattern;
 
+public:
+  FoldConsumerUnPackWithProducerLinalgTransposeOp(
+      MLIRContext *context, ControlFoldIntoPackUnpackFn controlFn)
+      : OpRewritePattern<UnPackOp>(context), controlFn(std::move(controlFn)) {}
+
   LogicalResult matchAndRewrite(UnPackOp unPackOp,
                                 PatternRewriter &rewriter) const override {
     auto linalgOp = unPackOp.getSource().getDefiningOp<linalg::LinalgOp>();
     if (!linalgOp)
       return failure();
 
+    // User controlled folding function.
+    if (controlFn && !controlFn(&unPackOp.getSourceMutable()))
+      return failure();
+
     FailureOr<SmallVector<int64_t>> maybePerm =
         getTransposeOpPermutation(linalgOp);
     if (failed(maybePerm))
@@ -474,6 +538,9 @@ struct FoldConsumerUnPackWithProducerLinalgTransposeOp
 
     return success();
   }
+
+private:
+  ControlFoldIntoPackUnpackFn controlFn;
 };
 
 /// tensor.empty does not define any tensor contents, so an unpadded pack
@@ -521,13 +588,14 @@ struct FoldEmptyTensorWithUnPackOp : public OpRewritePattern<UnPackOp> {
 
 } // namespace
 
-void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns) {
+void populateFoldIntoPackAndUnpackPatterns(
+    RewritePatternSet &patterns, const ControlFoldIntoPackUnpackFn &controlFn) {
   patterns.insert<FoldUnpackWithExtractSliceOp, FoldPadWithPackOp,
                   FoldProducerPackWithConsumerLinalgTransposeOp,
                   FoldConsumerPackWithProducerLinalgTransposeOp,
                   FoldConsumerUnPackWithProducerLinalgTransposeOp,
                   FoldProducerUnPackWithConsumerLinalgTransposeOp>(
-      patterns.getContext());
+      patterns.getContext(), controlFn);
 }
 
 void populateSimplifyPackAndUnpackPatterns(RewritePatternSet &patterns) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index eda3373b4d63..5eb3761f7aca 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -261,9 +261,8 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     // 2.a. Skip scalar-like operands.
     Type operandType = operand.getType();
     if (!isa<RankedTensorType>(operandType)) {
-      assert(!isa<ShapedType>(operandType) ||
-             isa<VectorType>(operandType) &&
-                 "Unexpected non-vector ShapedType");
+      assert((!isa<ShapedType>(operandType) || isa<VectorType>(operandType)) &&
+             "Unexpected non-vector ShapedType");
       newOperands.push_back(operand);
       continue;
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index d599ddd220dd..b274502e1690 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -18,9 +18,6 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Passes.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
-#include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
@@ -30,7 +27,6 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
 using namespace mlir;
@@ -148,6 +144,9 @@ struct LinalgOpInstancePromotionOptions {
   llvm::SmallSet<int64_t, 4> operandsNumbersToCopyIn;
   /// True if the full view should be used for the promoted buffer.
   DenseMap<Value, bool> useFullTileBuffers;
+  /// True if the original subview size should be used. This means the full tile
+  /// buffer is the same size as the partial view.
+  bool useOriginalSubviewSize;
 
   /// Callback functions for allocation and deallocation of promoted buffers, as
   /// well as to copy the data into and out of these buffers.
@@ -170,6 +169,7 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
       options.useFullTileBuffers.value_or(llvm::SmallBitVector());
   vUseFullTileBuffers.resize(linalgOp->getNumOperands(),
                              options.useFullTileBuffersDefault);
+  useOriginalSubviewSize = options.useOriginalSubviewSize;
 
   for (OpOperand &opOperand : linalgOp->getOpOperands()) {
     int64_t operandNumber = opOperand.getOperandNumber();
@@ -237,7 +237,8 @@ LinalgOpInstancePromotionOptions::LinalgOpInstancePromotionOptions(
 // by a partial `copy` op.
 FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
     OpBuilder &b, Location loc, memref::SubViewOp subView,
-    const AllocBufferCallbackFn &allocationFn, DataLayout &layout) {
+    bool useOriginalSubviewSize, const AllocBufferCallbackFn &allocationFn,
+    DataLayout &layout) {
   auto viewType = subView.getType();
   auto rank = viewType.getRank();
   SmallVector<Value, 4> fullSizes;
@@ -254,7 +255,8 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
     // to look for the bound.
     LLVM_DEBUG(llvm::dbgs() << "Extract tightest: " << rangeValue.size << "\n");
     Value size;
-    if (auto attr = llvm::dyn_cast_if_present<Attribute>(rangeValue.size)) {
+    if (llvm::isa_and_present<Attribute>(rangeValue.size) ||
+        useOriginalSubviewSize) {
       size = getValueOrCreateConstantIndexOp(b, loc, rangeValue.size);
     } else {
       FailureOr<int64_t> upperBound =
@@ -295,7 +297,8 @@ promoteSubViews(ImplicitLocOpBuilder &b,
     memref::SubViewOp subView =
         cast<memref::SubViewOp>(v.second.getDefiningOp());
     auto promotionInfo = promoteSubviewAsNewBuffer(
-        b, b.getLoc(), subView, options.allocationFn, layout);
+        b, b.getLoc(), subView, options.useOriginalSubviewSize,
+        options.allocationFn, layout);
     if (failed(promotionInfo))
       return failure();
     promotionInfoMap[v.first] = *promotionInfo;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
index 933f5966cc6b..455e1a6d146d 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
@@ -18,9 +18,7 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Support/TypeID.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/Support/Debug.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_LINALGSPECIALIZEGENERICOPSPASS
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 8a5a2e54cdda..4741afe8a417 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -10,13 +10,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Linalg/Passes.h"
-
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -26,12 +23,9 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/ValueRange.h"
 #include "mlir/Transforms/FoldUtils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Support/CommandLine.h"
 #include <utility>
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
index 19d484a3bb70..513cecef29b6 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TilingInterfaceImpl.cpp
@@ -22,8 +22,11 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "mlir/Interfaces/ValueBoundsOpInterface.h"
+#include "llvm/Support/Debug.h"
 #include <optional>
 
+#define DEBUG_TYPE "linalg-tiling-interface-impl"
+
 using namespace mlir;
 using namespace mlir::linalg;
 
@@ -148,55 +151,82 @@ struct LinalgOpTilingInterface
   /// Utility to fetch the offsets and sizes when applied as per the indexing
   /// map of the linalg op. This helps in fusing the linalg op as a consumer of
   /// a given slice op.
-  void
-  getMappedOffsetAndSize(LinalgOp linalgOp, OpBuilder &b, AffineMap indexingMap,
-                         ArrayRef<OpFoldResult> offsets,
-                         ArrayRef<OpFoldResult> sizes,
-                         SmallVectorImpl<OpFoldResult> &mappedOffsets,
-                         SmallVectorImpl<OpFoldResult> &mappedSizes) const {
-    unsigned numLoops = linalgOp.getNumLoops();
-    auto tilingInterfaceOp = cast<TilingInterface>(linalgOp.getOperation());
-    mappedOffsets.resize(numLoops);
-    mappedSizes.resize(numLoops);
-    if (!indexingMap.isPermutation()) {
-      SmallVector<Range> iterationDomain =
-          tilingInterfaceOp.getIterationDomain(b);
-      for (const auto &&[index, value] : llvm::enumerate(iterationDomain)) {
-        mappedOffsets[index] = value.offset;
-        mappedSizes[index] = value.size;
+  static LogicalResult
+  getMappedOffsetAndSize(LinalgOp linalgOp, OpBuilder &b,
+                         ArrayRef<AffineMap> indexingMaps,
+                         ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+                         ArrayRef<SmallVector<OpFoldResult>> allSizes,
+                         SmallVectorImpl<OpFoldResult> &mappedOffsetsVec,
+                         SmallVectorImpl<OpFoldResult> &mappedSizesVec) {
+    DenseMap<unsigned, OpFoldResult> mappedOffsets, mappedSizes;
+
+    for (auto [indexingMap, offsets, sizes] :
+         llvm::zip_equal(indexingMaps, allOffsets, allSizes)) {
+      for (auto [resultExpr, offset, size] :
+           llvm::zip_equal(indexingMap.getResults(), offsets, sizes)) {
+        auto dimExpr = dyn_cast<AffineDimExpr>(resultExpr);
+        if (!dimExpr)
+          continue;
+        unsigned position = dimExpr.getPosition();
+        auto it = mappedOffsets.find(position);
+        if (it != mappedOffsets.end()) {
+          OpFoldResult seenOffset = it->second;
+          OpFoldResult seenSize = mappedSizes.lookup(position);
+          if (seenOffset != offset || seenSize != size) {
+            LLVM_DEBUG({
+              llvm::dbgs() << "inconsistent iteration space mapping from "
+                              "offsets/sizes of operands/results";
+            });
+            return failure();
+          }
+        } else {
+          mappedOffsets[position] = offset;
+          mappedSizes[position] = size;
+        }
       }
     }
-    for (const auto &&[index, value] :
-         llvm::enumerate(indexingMap.getResults())) {
-      unsigned dimPosition = cast<AffineDimExpr>(value).getPosition();
-      mappedOffsets[dimPosition] = offsets[index];
-      mappedSizes[dimPosition] = sizes[index];
+
+    // Aggregate from the given operand offsets and sizes, or default to
+    // iteration space values.
+    SmallVector<Range> iterationDomain =
+        cast<TilingInterface>(linalgOp.getOperation()).getIterationDomain(b);
+    mappedOffsetsVec.resize(iterationDomain.size());
+    mappedSizesVec.resize(iterationDomain.size());
+    for (auto [index, domain] : llvm::enumerate(iterationDomain)) {
+      auto it = mappedOffsets.find(index);
+      if (it != mappedOffsets.end()) {
+        mappedOffsetsVec[index] = it->second;
+        mappedSizesVec[index] = mappedSizes.lookup(index);
+        continue;
+      }
+      mappedOffsetsVec[index] = domain.offset;
+      mappedSizesVec[index] = domain.size;
     }
+    return success();
   }
 
   /// Method to return the position of the result tile computed by the tiled
   /// operation.
-  LogicalResult getIterationDomainTileFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes,
+  LogicalResult getIterationDomainTileFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes,
       SmallVectorImpl<OpFoldResult> &iterDomainOffsets,
       SmallVectorImpl<OpFoldResult> &iterDomainSizes) const {
     auto linalgOp = cast<LinalgOp>(op);
 
-    // Check that the indexing map used for the operand is a projected
-    // permutation. This could be relaxed with a more general approach that can
-    // map the offsets and sizes from the operand to iteration space tiles
-    // (filling in full extent for dimensions not used to access the result).
-    AffineMap indexingMap =
-        linalgOp.getMatchingIndexingMap(&op->getOpOperand(operandNumber));
-    if (!indexingMap.isProjectedPermutation()) {
-      return op->emitError()
-             << "unhandled get iter domain position when operand is not "
-                "accessed using a permuted projection";
+    std::optional<SmallVector<OpFoldResult>> iterationSpaceOffsets,
+        iterationSpaceSizes;
+    SmallVector<AffineMap> indexingMaps =
+        llvm::map_to_vector(operandNumbers, [&](unsigned operandNumber) {
+          OpOperand &opOperand = linalgOp->getOpOperand(operandNumber);
+          return linalgOp.getMatchingIndexingMap(&opOperand);
+        });
+    if (failed(getMappedOffsetAndSize(linalgOp, b, indexingMaps, allOffsets,
+                                      allSizes, iterDomainOffsets,
+                                      iterDomainSizes))) {
+      return failure();
     }
-
-    getMappedOffsetAndSize(linalgOp, b, indexingMap, offsets, sizes,
-                           iterDomainOffsets, iterDomainSizes);
     return success();
   }
 
@@ -247,8 +277,13 @@ struct LinalgOpTilingInterface
           "accessed using a permuted projection");
     }
 
-    getMappedOffsetAndSize(linalgOp, b, indexingMap, offsets, sizes,
-                           iterDomainOffsets, iterDomainSizes);
+    SmallVector<OpFoldResult> allOffsets = llvm::to_vector(offsets);
+    SmallVector<OpFoldResult> allSizes = llvm::to_vector(sizes);
+    auto status =
+        getMappedOffsetAndSize(linalgOp, b, indexingMap, {allOffsets},
+                               {allSizes}, iterDomainOffsets, iterDomainSizes);
+    (void)status;
+    assert(succeeded(status) && "unexpected error in offset calculation");
     return success();
   }
 
@@ -279,12 +314,13 @@ struct LinalgOpTilingInterface
 
   /// Method to generate the tiled implementation of an operation from the tile
   /// of the operand.
-  FailureOr<TilingResult> getTiledImplementationFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes) const {
+  FailureOr<TilingResult> getTiledImplementationFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes) const {
     SmallVector<OpFoldResult> mappedOffsets, mappedSizes;
-    if (failed(getIterationDomainTileFromOperandTile(
-            op, b, operandNumber, offsets, sizes, mappedOffsets,
+    if (failed(getIterationDomainTileFromOperandTiles(
+            op, b, operandNumbers, allOffsets, allSizes, mappedOffsets,
             mappedSizes))) {
       return failure();
     }
@@ -837,13 +873,20 @@ struct PackOpTiling
   /// Method to return the position of iteration domain tile computed by the
   /// tiled operation. In current `tensor.pack` context, the `resultOffsets` and
   /// `resultSizes` only cover outer dimensions.
-  LogicalResult getIterationDomainTileFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes,
+  LogicalResult getIterationDomainTileFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes,
       SmallVectorImpl<OpFoldResult> &resultOffsets,
       SmallVectorImpl<OpFoldResult> &resultSizes) const {
-    if (operandNumber != 0)
+    if (operandNumbers.size() != 1 || operandNumbers[0] != 0) {
+      LLVM_DEBUG(
+          { llvm::dbgs() << "unsupported operands for consumer fusion"; });
       return failure();
+    }
+
+    ArrayRef<OpFoldResult> offsets(allOffsets[0]);
+    ArrayRef<OpFoldResult> sizes(allSizes[0]);
 
     auto packOp = cast<PackOp>(op);
     // It is not trivial to infer dest tile from source tile if `packOp` has
@@ -904,11 +947,18 @@ struct PackOpTiling
   }
 
   /// Method to return the tiled implementation of tensor.pack as a consumer.
-  FailureOr<TilingResult> getTiledImplementationFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes) const {
-    if (operandNumber != 0)
+  FailureOr<TilingResult> getTiledImplementationFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes) const {
+    if (operandNumbers.size() != 1 || operandNumbers[0] != 0) {
+      LLVM_DEBUG(
+          { llvm ::dbgs() << "unhandled operands for consumer fusion"; });
       return failure();
+    }
+
+    ArrayRef<OpFoldResult> offsets(allOffsets[0]);
+    ArrayRef<OpFoldResult> sizes(allSizes[0]);
 
     auto packOp = cast<PackOp>(op);
     Location loc = packOp.getLoc();
@@ -923,8 +973,8 @@ struct PackOpTiling
     tiledOperands.push_back(sourceSlice);
 
     SmallVector<OpFoldResult> outerDimOffsets, outerDimSizes;
-    if (failed(getIterationDomainTileFromOperandTile(
-            op, b, /*operandNumber=*/0, offsets, sizes, outerDimOffsets,
+    if (failed(getIterationDomainTileFromOperandTiles(
+            op, b, operandNumbers, allOffsets, allSizes, outerDimOffsets,
             outerDimSizes)))
       return failure();
 
@@ -1182,12 +1232,21 @@ struct UnPackOpTiling
 
   /// Method to return the position of iteration domain tile computed by the
   /// tiled operation.
-  LogicalResult getIterationDomainTileFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes,
+  LogicalResult getIterationDomainTileFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes,
       SmallVectorImpl<OpFoldResult> &resultOffsets,
       SmallVectorImpl<OpFoldResult> &resultSizes) const {
+    if (operandNumbers.size() != 1) {
+      LLVM_DEBUG({ llvm::dbgs() << "unable to handle multiple operands"; });
+      return failure();
+    }
     auto unPackOp = cast<UnPackOp>(op);
+    unsigned operandNumber = operandNumbers[0];
+    ArrayRef<OpFoldResult> offsets(allOffsets[0]);
+    ArrayRef<OpFoldResult> sizes(allSizes[0]);
+
     // If the operand tile is the dest, then no adjustment is needed.
     if (operandNumber == unPackOp.getDestMutable().getOperandNumber()) {
       resultOffsets = llvm::to_vector(offsets);
@@ -1241,10 +1300,18 @@ struct UnPackOpTiling
   }
 
   /// Method to return the tiled implementation of tensor.unpack as a consumer.
-  FailureOr<TilingResult> getTiledImplementationFromOperandTile(
-      Operation *op, OpBuilder &b, unsigned operandNumber,
-      ArrayRef<OpFoldResult> offsets, ArrayRef<OpFoldResult> sizes) const {
+  FailureOr<TilingResult> getTiledImplementationFromOperandTiles(
+      Operation *op, OpBuilder &b, ArrayRef<unsigned> operandNumbers,
+      ArrayRef<SmallVector<OpFoldResult>> allOffsets,
+      ArrayRef<SmallVector<OpFoldResult>> allSizes) const {
+    if (operandNumbers.size() != 1 || operandNumbers[0] != 0) {
+      LLVM_DEBUG({ llvm::dbgs() << "unhandled operands for consumer fusion"; });
+      return failure();
+    }
     auto unPackOp = cast<UnPackOp>(op);
+    ArrayRef<OpFoldResult> offsets(allOffsets[0]);
+    ArrayRef<OpFoldResult> sizes(allSizes[0]);
+
     // tensor.unpack op is fusible (as a consumer) only if inner dims are not
     // tiled.
     int64_t numTiles = unPackOp.getInnerDimsPos().size();
@@ -1259,8 +1326,8 @@ struct UnPackOpTiling
     // Fetch offset/size for creating the slice of the dest operand of
     // unpack op.
     SmallVector<OpFoldResult> outputOffsets, outputSizes;
-    if (failed(getIterationDomainTileFromOperandTile(
-            op, b, /*operandNumber=*/0, offsets, sizes, outputOffsets,
+    if (failed(getIterationDomainTileFromOperandTiles(
+            op, b, operandNumbers, allOffsets, allSizes, outputOffsets,
             outputSizes)))
       return failure();
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index 615d1f66414b..eab74dab4eb7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -26,16 +26,11 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/AffineExpr.h"
-#include "mlir/IR/Matchers.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
-#include <type_traits>
 #include <utility>
 
 #define DEBUG_TYPE "linalg-transforms"
@@ -1178,13 +1173,6 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   int64_t destRank = packOp.getDestRank();
   int64_t numTiles = destRank - srcRank;
 
-  if (!llvm::all_of(packOp.getInnerDimsPos(),
-                    [&srcRank, &numTiles](int64_t dimPos) {
-                      return dimPos >= (srcRank - numTiles - 1);
-                    }))
-    return rewriter.notifyMatchFailure(
-        packOp, "Attempting to tile non-trailing source dims!");
-
   // 1. Extract the inner tile sizes.
   // Where possible, values are replaced with constant attributes (to match the
   // behaviour of `getPackOpSourceOrPaddedSource`).
@@ -1204,16 +1192,24 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   //    %init = tensor.empty()
   //    %transposed_tile = linalg.transpose ins(%source_or_padded_source),
   //                                        outs(%init)
-  // Two assumptions are made:
-  //  1. All outer dims are 1 - the corresponding transposition doesn't matter.
-  //  2. Inner dims position correspond to the trailing `numTiles` dims.
-  SmallVector<int64_t> tilesPermNormalized =
-      getPackUnpackNormalizedPerm(srcRank, packOp.getInnerDimsPos());
+  // Assumptions made:
+  //  1. All outer dims are 1 - the corresponding transposition order doesn't
+  //     matter, but requires all dim indices to be present.
   SmallVector<int64_t> srcPermForTranspose;
-  for (int64_t i = 0; i < (srcRank - numTiles); i++)
+  ArrayRef<int64_t> innerDimPos(packOp.getInnerDimsPos());
+  for (int64_t i = 0; i < srcRank; i++) {
+    // We assume the `k` dimensions of the inner dim position, where `k` is the
+    // rank of the inner tiling, correspond to the last `k` indices of the
+    // transpose permutation. This is done by adding the indices not contained
+    // in the inner dimension position in order from 0 to `n`. Where n is the
+    // rank of the source tensor. For example if we have a source tensor with
+    // indices [0, 1, 2, 3] and inner dim position of [3, 0], the remaining
+    // indices are [1, 2]. and the transpose will be [1, 2, 3, 0].
+    if (llvm::is_contained(innerDimPos, i))
+      continue;
     srcPermForTranspose.push_back(i);
-
-  srcPermForTranspose.append(SmallVector<int64_t>(packOp.getInnerDimsPos()));
+  }
+  srcPermForTranspose.append(innerDimPos.begin(), innerDimPos.end());
 
   LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n"
                     << "perm: " << llvm::interleaved(srcPermForTranspose)
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
index 436d485ab936..092aecceef6b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeConv2D.cpp
@@ -12,14 +12,8 @@
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/ValueRange.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/RWMutex.h"
-#include <memory>
-#include <numeric>
 
 namespace mlir {
 namespace linalg {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index e624f589917d..934781d1cab7 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -11,7 +11,6 @@
 
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 #define DEBUG_TYPE "linalg-transpose-matmul"
 
@@ -38,7 +37,7 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
         matmulOp, "only matmul ops with non-extended semantics are supported");
   }
 
-  if (!bufferization::hasTensorSemantics(matmulOp))
+  if (!matmulOp.hasPureTensorSemantics())
     return rewriter.notifyMatchFailure(
         matmulOp, "only matmul ops with tensors are supported");
 
@@ -93,7 +92,7 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
         batchMatmulOp, "ops with user-defined maps are not supported");
   }
 
-  if (!bufferization::hasTensorSemantics(batchMatmulOp))
+  if (!batchMatmulOp.hasPureTensorSemantics())
     return rewriter.notifyMatchFailure(
         batchMatmulOp, "only matmul ops with tensors are supported");
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index ff8e0b8977ae..2864c5b807d6 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -19,7 +19,6 @@
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-#include "mlir/Dialect/Tensor/Utils/Utils.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
@@ -38,12 +37,10 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TypeSwitch.h"
-#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
-#include <type_traits>
 
 using namespace mlir;
 using namespace mlir::linalg;
@@ -551,9 +548,10 @@ enum class Conv1DOpOrder {
   Nwc  // Corresponds to operation that traverses the input in (n, w, c) order.
 };
 
-/// Helper data structure to represent the result of vectorization.
-/// In certain specific cases, like terminators, we do not want to propagate/
-enum VectorizationStatus {
+/// Helper data structure to represent the result of vectorization for a single
+/// operation. In certain specific cases, like terminators, we do not want to
+/// propagate.
+enum VectorizationHookStatus {
   /// Op failed to vectorize.
   Failure = 0,
   /// Op vectorized and custom function took care of replacement logic
@@ -564,9 +562,12 @@ enum VectorizationStatus {
   // TODO: support values if Op vectorized to Many-Ops whose results we need to
   // aggregate for replacement.
 };
-struct VectorizationResult {
+/// VectorizationHookResult contains the vectorized op returned from a
+/// CustomVectorizationHook. This is an internal implementation detail of
+/// linalg vectorization, not to be confused with VectorizationResult.
+struct VectorizationHookResult {
   /// Return status from vectorizing the current op.
-  enum VectorizationStatus status = VectorizationStatus::Failure;
+  enum VectorizationHookStatus status = VectorizationHookStatus::Failure;
   /// New vectorized operation to replace the current op.
   /// Replacement behavior is specified by `status`.
   Operation *newOp;
@@ -728,22 +729,22 @@ using CustomVectorizationPrecondition =
 // assuming all its vectorized operands are already in the IRMapping.
 // Return nullptr if the Operation cannot be vectorized.
 using CustomVectorizationHook =
-    std::function<VectorizationResult(Operation *, const IRMapping &)>;
+    std::function<VectorizationHookResult(Operation *, const IRMapping &)>;
 
 /// Helper function to vectorize the terminator of a `linalgOp`. New result
 /// vector values are appended to `newResults`. Return
-/// VectorizationStatus::NoReplace to signal the vectorization algorithm that it
-/// should not try to map produced operations and instead return the results
-/// using the `newResults` vector making them available to the vectorization
-/// algorithm for RAUW. This function is meant to be used as a
+/// VectorizationHookStatus::NoReplace to signal the vectorization algorithm
+/// that it should not try to map produced operations and instead return the
+/// results using the `newResults` vector making them available to the
+/// vectorization algorithm for RAUW. This function is meant to be used as a
 /// CustomVectorizationHook.
-static VectorizationResult
+static VectorizationHookResult
 vectorizeLinalgYield(RewriterBase &rewriter, Operation *op,
                      const IRMapping &bvm, VectorizationState &state,
                      LinalgOp linalgOp, SmallVectorImpl<Value> &newResults) {
   auto yieldOp = dyn_cast<linalg::YieldOp>(op);
   if (!yieldOp)
-    return VectorizationResult{VectorizationStatus::Failure, nullptr};
+    return VectorizationHookResult{VectorizationHookStatus::Failure, nullptr};
   for (const auto &output : llvm::enumerate(yieldOp.getValues())) {
     // TODO: Scan for an opportunity for reuse.
     // TODO: use a map.
@@ -755,20 +756,20 @@ vectorizeLinalgYield(RewriterBase &rewriter, Operation *op,
       newResults.push_back(newResult);
   }
 
-  return VectorizationResult{VectorizationStatus::NoReplace, nullptr};
+  return VectorizationHookResult{VectorizationHookStatus::NoReplace, nullptr};
 }
 
 /// Helper function to vectorize the index operations of a `linalgOp`. Return
-/// VectorizationStatus::NewOp to signal the vectorization algorithm that it
+/// VectorizationHookStatus::NewOp to signal the vectorization algorithm that it
 /// should map the produced operations. This function is meant to be used as a
 /// CustomVectorizationHook.
-static VectorizationResult vectorizeLinalgIndex(RewriterBase &rewriter,
-                                                VectorizationState &state,
-                                                Operation *op,
-                                                LinalgOp linalgOp) {
+static VectorizationHookResult vectorizeLinalgIndex(RewriterBase &rewriter,
+                                                    VectorizationState &state,
+                                                    Operation *op,
+                                                    LinalgOp linalgOp) {
   IndexOp indexOp = dyn_cast<linalg::IndexOp>(op);
   if (!indexOp)
-    return VectorizationResult{VectorizationStatus::Failure, nullptr};
+    return VectorizationHookResult{VectorizationHookStatus::Failure, nullptr};
   auto loc = indexOp.getLoc();
   // Compute the static loop sizes of the index op.
   ArrayRef<int64_t> targetShape = state.getCanonicalVecShape();
@@ -782,7 +783,7 @@ static VectorizationResult vectorizeLinalgIndex(RewriterBase &rewriter,
   // dimension of the iteration space since the vectorization algorithm in this
   // case can handle the broadcast.
   if (dim == targetShape.size() - 1)
-    return VectorizationResult{VectorizationStatus::NewOp, indexSteps};
+    return VectorizationHookResult{VectorizationHookStatus::NewOp, indexSteps};
   // Otherwise permute the targetShape to move the index dimension last,
   // broadcast the one-dimensional index vector to the permuted shape, and
   // finally transpose the broadcasted index vector to undo the permutation.
@@ -800,7 +801,7 @@ static VectorizationResult vectorizeLinalgIndex(RewriterBase &rewriter,
   std::swap(transposition.back(), transposition[dim]);
   auto transposeOp =
       rewriter.create<vector::TransposeOp>(loc, broadCastOp, transposition);
-  return VectorizationResult{VectorizationStatus::NewOp, transposeOp};
+  return VectorizationHookResult{VectorizationHookStatus::NewOp, transposeOp};
 }
 
 /// Helper function to check if the tensor.extract can be vectorized by the
@@ -918,8 +919,7 @@ static bool isLoopInvariantIdx(LinalgOp &linalgOp, Value &val,
   // TODO: We could try analysing the corresponding affine map here.
   auto *block = linalgOp.getBlock();
   if (isa<BlockArgument>(val))
-    return llvm::all_of(block->getArguments(),
-                        [&val](Value v) { return (v != val); });
+    return !llvm::is_contained(block->getArguments(), val);
 
   Operation *defOp = val.getDefiningOp();
   assert(defOp && "This is neither a block argument nor an operation result");
@@ -978,8 +978,7 @@ static bool isContiguousLoadIdx(LinalgOp &linalgOp, Value &val,
   // TODO: We could try analysing the corresponding affine map here.
   auto *block = linalgOp.getBlock();
   if (isa<BlockArgument>(val))
-    return llvm::all_of(block->getArguments(),
-                        [&val](Value v) { return (v != val); });
+    return !llvm::is_contained(block->getArguments(), val);
 
   Operation *defOp = val.getDefiningOp();
   assert(defOp && "This is neither a block argument nor an operation result");
@@ -1098,15 +1097,15 @@ getTensorExtractMemoryAccessPattern(tensor::ExtractOp extractOp,
 }
 
 /// Helper function to vectorize the tensor.extract operations. Returns
-/// VectorizationStatus::NewOp to signal the vectorization algorithm that it
+/// VectorizationHookStatus::NewOp to signal the vectorization algorithm that it
 /// should map the produced operations. This function is meant to be used as a
 /// CustomVectorizationHook.
-static VectorizationResult
+static VectorizationHookResult
 vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
                        Operation *op, LinalgOp linalgOp, const IRMapping &bvm) {
   tensor::ExtractOp extractOp = dyn_cast<tensor::ExtractOp>(op);
   if (!extractOp)
-    return VectorizationResult{VectorizationStatus::Failure, nullptr};
+    return VectorizationHookResult{VectorizationHookStatus::Failure, nullptr};
   auto loc = extractOp.getLoc();
 
   // Compute the static loop sizes of the extract op.
@@ -1138,7 +1137,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
     gatherOp = state.maskOperation(rewriter, gatherOp, linalgOp);
 
     LDBG("Vectorised as gather load: " << extractOp << "\n");
-    return VectorizationResult{VectorizationStatus::NewOp, gatherOp};
+    return VectorizationHookResult{VectorizationHookStatus::NewOp, gatherOp};
   }
 
   // 2. Handle:
@@ -1189,7 +1188,7 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
 
     auto transferReadOp = rewriter.create<vector::TransferReadOp>(
         loc, resultType, extractOp.getTensor(), transferReadIdxs,
-        permutationMap, inBounds);
+        /*padding=*/std::nullopt, permutationMap, inBounds);
 
     // Mask this broadcasting xfer_read here rather than relying on the generic
     // path (the generic path assumes identity masking map, which wouldn't be
@@ -1202,7 +1201,8 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
         mlir::vector::maskOperation(rewriter, transferReadOp, allTrue);
 
     LDBG("Vectorised as scalar broadcast load: " << extractOp << "\n");
-    return VectorizationResult{VectorizationStatus::NewOp, maskedReadOp};
+    return VectorizationHookResult{VectorizationHookStatus::NewOp,
+                                   maskedReadOp};
   }
 
   // 2b. Handle contiguous access.
@@ -1224,11 +1224,12 @@ vectorizeTensorExtract(RewriterBase &rewriter, VectorizationState &state,
   }
 
   auto transferReadOp = rewriter.create<vector::TransferReadOp>(
-      loc, resultType, extractOp.getTensor(), transferReadIdxs, permutationMap,
-      inBounds);
+      loc, resultType, extractOp.getTensor(), transferReadIdxs,
+      /*padding=*/std::nullopt, permutationMap, inBounds);
 
   LDBG("Vectorised as contiguous load: " << extractOp);
-  return VectorizationResult{VectorizationStatus::NewOp, transferReadOp};
+  return VectorizationHookResult{VectorizationHookStatus::NewOp,
+                                 transferReadOp};
 }
 
 /// Emit reduction operations if the shapes of the value to reduce is different
@@ -1268,9 +1269,9 @@ static Operation *reduceIfNeeded(OpBuilder &b, LinalgOp linalgOp, Operation *op,
 /// This function assumes all operands of `op` have been vectorized and are in
 /// the `bvm` mapping. As a consequence, this function is meant to be called  on
 /// a topologically-sorted list of ops.
-/// This function does not update `bvm` but returns a VectorizationStatus that
-/// instructs the caller what `bvm` update needs to occur.
-static VectorizationResult
+/// This function does not update `bvm` but returns a VectorizationHookStatus
+/// that instructs the caller what `bvm` update needs to occur.
+static VectorizationHookResult
 vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
                LinalgOp linalgOp, Operation *op, const IRMapping &bvm,
                ArrayRef<CustomVectorizationHook> customVectorizationHooks) {
@@ -1279,8 +1280,8 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
   // 1. Try to apply any CustomVectorizationHook.
   if (!customVectorizationHooks.empty()) {
     for (auto &customFunc : customVectorizationHooks) {
-      VectorizationResult result = customFunc(op, bvm);
-      if (result.status == VectorizationStatus::Failure)
+      VectorizationHookResult result = customFunc(op, bvm);
+      if (result.status == VectorizationHookStatus::Failure)
         continue;
       return result;
     }
@@ -1289,11 +1290,12 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
   // 2. Constant ops don't get vectorized but rather broadcasted at their users.
   // Clone so that the constant is not confined to the linalgOp block .
   if (isa<arith::ConstantOp, func::ConstantOp>(op))
-    return VectorizationResult{VectorizationStatus::NewOp, rewriter.clone(*op)};
+    return VectorizationHookResult{VectorizationHookStatus::NewOp,
+                                   rewriter.clone(*op)};
 
   // 3. Only ElementwiseMappable are allowed in the generic vectorization.
   if (!OpTrait::hasElementwiseMappableTraits(op))
-    return VectorizationResult{VectorizationStatus::Failure, nullptr};
+    return VectorizationHookResult{VectorizationHookStatus::Failure, nullptr};
 
   // 4 . Check if the operation is a reduction.
   SmallVector<std::pair<Value, Value>> reductionOperands;
@@ -1316,7 +1318,7 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
         reduceIfNeeded(rewriter, linalgOp, op, reductionOperands[0].first,
                        reductionOperands[0].second, bvm);
     if (reduceOp)
-      return VectorizationResult{VectorizationStatus::NewOp, reduceOp};
+      return VectorizationHookResult{VectorizationHookStatus::NewOp, reduceOp};
   }
 
   // 5. Generic vectorization path for ElementwiseMappable ops.
@@ -1356,8 +1358,8 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
             : resultType);
   }
   //   d. Build and return the new op.
-  return VectorizationResult{
-      VectorizationStatus::NewOp,
+  return VectorizationHookResult{
+      VectorizationHookStatus::NewOp,
       rewriter.create(op->getLoc(), op->getName().getIdentifier(), vecOperands,
                       resultTypes, op->getAttrs())};
 }
@@ -1379,7 +1381,7 @@ vectorizeOneOp(RewriterBase &rewriter, VectorizationState &state,
 /// performed to the maximal common vector size implied by the `linalgOp`
 /// iteration space. This eager broadcasting is introduced in the
 /// permutation_map of the vector.transfer_read operations. The eager
-/// broadcasting makes it trivial to detrmine where broadcast, transposes and
+/// broadcasting makes it trivial to determine where broadcast, transposes and
 /// reductions should occur, without any bookkeeping. The tradeoff is that, in
 /// the absence of good canonicalizations, the amount of work increases.
 /// This is not deemed a problem as we expect canonicalizations and foldings to
@@ -1434,7 +1436,8 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     SmallVector<Value> indices(linalgOp.getShape(opOperand).size(), zero);
 
     Operation *read = rewriter.create<vector::TransferReadOp>(
-        loc, readType, opOperand->get(), indices, readMap);
+        loc, readType, opOperand->get(), indices,
+        /*padding=*/std::nullopt, readMap);
     read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
 
@@ -1461,34 +1464,34 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
   SmallVector<CustomVectorizationHook> hooks;
   // 4a. Register CustomVectorizationHook for yieldOp.
   CustomVectorizationHook vectorizeYield =
-      [&](Operation *op, const IRMapping &bvm) -> VectorizationResult {
+      [&](Operation *op, const IRMapping &bvm) -> VectorizationHookResult {
     return vectorizeLinalgYield(rewriter, op, bvm, state, linalgOp, newResults);
   };
   hooks.push_back(vectorizeYield);
 
   // 4b. Register CustomVectorizationHook for indexOp.
   CustomVectorizationHook vectorizeIndex =
-      [&](Operation *op, const IRMapping &bvm) -> VectorizationResult {
+      [&](Operation *op, const IRMapping &bvm) -> VectorizationHookResult {
     return vectorizeLinalgIndex(rewriter, state, op, linalgOp);
   };
   hooks.push_back(vectorizeIndex);
 
   // 4c. Register CustomVectorizationHook for extractOp.
   CustomVectorizationHook vectorizeExtract =
-      [&](Operation *op, const IRMapping &bvm) -> VectorizationResult {
+      [&](Operation *op, const IRMapping &bvm) -> VectorizationHookResult {
     return vectorizeTensorExtract(rewriter, state, op, linalgOp, bvm);
   };
   hooks.push_back(vectorizeExtract);
 
   // 5. Iteratively call `vectorizeOneOp` to each op in the slice.
   for (Operation &op : block->getOperations()) {
-    VectorizationResult result =
+    VectorizationHookResult result =
         vectorizeOneOp(rewriter, state, linalgOp, &op, bvm, hooks);
-    if (result.status == VectorizationStatus::Failure) {
+    if (result.status == VectorizationHookStatus::Failure) {
       LDBG("failed to vectorize: " << op << "\n");
       return failure();
     }
-    if (result.status == VectorizationStatus::NewOp) {
+    if (result.status == VectorizationHookStatus::NewOp) {
       Operation *maybeMaskedOp =
           state.maskOperation(rewriter, result.newOp, linalgOp);
       LDBG("New vector op: " << *maybeMaskedOp << "\n");
@@ -1653,9 +1656,9 @@ createWriteOrMaskedWrite(OpBuilder &builder, Location loc, Value vecToStore,
   }
 
   // If missing, initialize the write indices to 0.
-  assert(writeIndices.empty() ||
-         writeIndices.size() == static_cast<size_t>(destRank) &&
-             "Invalid number of write indices!");
+  assert((writeIndices.empty() ||
+          writeIndices.size() == static_cast<size_t>(destRank)) &&
+         "Invalid number of write indices!");
   if (writeIndices.empty()) {
     auto zero = builder.create<arith::ConstantIndexOp>(loc, 0);
     writeIndices.assign(destRank, zero);
@@ -2525,17 +2528,11 @@ bool mlir::linalg::hasVectorizationImpl(Operation *op) {
              tensor::InsertSliceOp>(op);
 }
 
-/// Emit a suitable vector form for an operation. If provided,
-/// `inputVectorSizes` are used to vectorize this operation.
-/// `inputVectorSizes` must match the rank of the iteration space of the
-/// operation and the input vector sizes must be greater than or equal to
-/// their counterpart iteration space sizes, if static. `inputVectorShapes`
-/// also allows the vectorization of operations with dynamic shapes.
-LogicalResult mlir::linalg::vectorize(RewriterBase &rewriter, Operation *op,
-                                      ArrayRef<int64_t> inputVectorSizes,
-                                      ArrayRef<bool> inputScalableVecDims,
-                                      bool vectorizeNDExtract,
-                                      bool flatten1DDepthwiseConv) {
+FailureOr<VectorizationResult>
+mlir::linalg::vectorize(RewriterBase &rewriter, Operation *op,
+                        ArrayRef<int64_t> inputVectorSizes,
+                        ArrayRef<bool> inputScalableVecDims,
+                        bool vectorizeNDExtract, bool flatten1DDepthwiseConv) {
   LDBG("Attempting to vectorize:\n" << *op << "\n");
   LDBG("Input vector sizes: ");
   LLVM_DEBUG(llvm::interleaveComma(inputVectorSizes, llvm::dbgs()));
@@ -2617,12 +2614,7 @@ LogicalResult mlir::linalg::vectorize(RewriterBase &rewriter, Operation *op,
     return failure();
   }
 
-  if (!results.empty())
-    rewriter.replaceOp(op, results);
-  else
-    rewriter.eraseOp(op);
-
-  return success();
+  return VectorizationResult{results};
 }
 
 LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
@@ -2647,6 +2639,7 @@ LogicalResult mlir::linalg::vectorizeCopy(RewriterBase &rewriter,
 
   Value readValue = rewriter.create<vector::TransferReadOp>(
       loc, readType, copyOp.getSource(), indices,
+      /*padding=*/std::nullopt,
       rewriter.getMultiDimIdentityMap(srcType.getRank()));
   if (cast<VectorType>(readValue.getType()).getRank() == 0) {
     readValue =
@@ -3493,15 +3486,18 @@ struct Conv1DGenerator
     SmallVector<Value> resPadding(resShape.size(), zero);
 
     // Read the whole lhs, rhs and res in one shot (with zero padding).
-    Value lhs = rewriter.create<vector::TransferReadOp>(loc, lhsType, lhsShaped,
-                                                        lhsPadding);
+    Value lhs = rewriter.create<vector::TransferReadOp>(
+        loc, lhsType, lhsShaped, lhsPadding,
+        /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     // This is needed only for Conv.
     Value rhs = nullptr;
     if (oper == ConvOperationKind::Conv)
-      rhs = rewriter.create<vector::TransferReadOp>(loc, rhsType, rhsShaped,
-                                                    rhsPadding);
-    Value res = rewriter.create<vector::TransferReadOp>(loc, resType, resShaped,
-                                                        resPadding);
+      rhs = rewriter.create<vector::TransferReadOp>(
+          loc, rhsType, rhsShaped, rhsPadding,
+          /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
+    Value res = rewriter.create<vector::TransferReadOp>(
+        loc, resType, resShaped, resPadding,
+        /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
 
     // The base vectorization case for channeled convolution is input:
     // {n,w,c}, weight: {kw,c,f}, output: {n,w,f}. To reuse the base pattern
@@ -3748,19 +3744,22 @@ struct Conv1DGenerator
     // Read lhs slice of size {n, w * strideW + kw * dilationW, c} @ [0, 0,
     // 0].
     Value lhs = rewriter.create<vector::TransferReadOp>(
-        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero});
+        loc, lhsType, lhsShaped, ValueRange{zero, zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, lhsEltType));
     auto maybeMaskedLhs = maybeMaskXferOp(
         lhsType.getShape(), lhsType.getScalableDims(), lhs.getDefiningOp());
 
     // Read rhs slice of size {kw, c} @ [0, 0].
-    Value rhs = rewriter.create<vector::TransferReadOp>(loc, rhsType, rhsShaped,
-                                                        ValueRange{zero, zero});
+    Value rhs = rewriter.create<vector::TransferReadOp>(
+        loc, rhsType, rhsShaped, ValueRange{zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, rhsEltType));
     auto maybeMaskedRhs = maybeMaskXferOp(
         rhsType.getShape(), rhsType.getScalableDims(), rhs.getDefiningOp());
 
     // Read res slice of size {n, w, c} @ [0, 0, 0].
     Value res = rewriter.create<vector::TransferReadOp>(
-        loc, resType, resShaped, ValueRange{zero, zero, zero});
+        loc, resType, resShaped, ValueRange{zero, zero, zero},
+        /*padding=*/arith::getZeroConstant(rewriter, loc, resEltType));
     auto maybeMaskedRes = maybeMaskXferOp(
         resType.getShape(), resType.getScalableDims(), res.getDefiningOp());
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index e4221d474841..9fd084487e3f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -18,7 +18,6 @@
 #include "mlir/Dialect/Linalg/Utils/Utils.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/Support/MathExtras.h"
 
 namespace mlir {
@@ -38,6 +37,15 @@ namespace {
 ///
 /// The following tables define these constant transformation matrices for
 /// F(2 x 2, 3 x 3), F(4 x 4, 3 x 3), and F(2 x 2, 5 x 5)
+///
+/// To add more transformation matrices, we need to add the following
+/// items:
+/// 1. Add the constant transformation matrix to the corresponding
+///   G, GT, BT, B, AT, or A array.
+/// 2. Add the corresponding TransformMatrix to the GMatrices, GTMatrices,
+///   BTMatrices, BMatrices, ATMatrices, or AMatrices map.
+/// 3. Add a enum value F_m_r to WinogradConv2DFmr enum.
+///
 constexpr float G_2x2_3x3[] = {
    -1,     0,   0,
  1./2, -1./2, 1./2,
@@ -176,19 +184,6 @@ constexpr float A_2x2_5x5[] = {
 };
 // clang-format on
 
-using TransformMapKeyTy = std::pair<int, int>;
-
-/// We use F(m, r) to define the size of minimal filtering algorithms.
-/// m is the output dimension and r is the filter dimension. We can get
-/// the input dimension, alpha, from the formula, alpha = m + r - 1.
-///
-/// For example, when m = 2 and r = 3, we know its input size is 4.
-/// The Conv2D will operate on 4x4 input data with 3x3 filter and get
-/// 2x2 output result.
-constexpr TransformMapKeyTy F_2_3{2, 3};
-constexpr TransformMapKeyTy F_4_3{4, 3};
-constexpr TransformMapKeyTy F_2_5{2, 5};
-
 /// Structure to keep information of constant transform matrices.
 struct TransformMatrix {
   TransformMatrix(const float *table, int64_t rows, int64_t cols,
@@ -344,22 +339,22 @@ Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
 ///     %ret = linalg.matmul %ret, GT
 ///     %inserted = insert %ret into filter<h x w x c x f>
 Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
-                      Value retValue, int64_t m, int64_t r,
+                      Value retValue, WinogradConv2DFmr fmr,
                       bool leftTransform = true, bool rightTransform = true) {
   // Map from (m, r) to G transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       GMatrices = {
-          {F_2_3, TransformMatrix(G_2x2_3x3, 4, 3)},
-          {F_4_3, TransformMatrix(G_4x4_3x3, 6, 3)},
-          {F_2_5, TransformMatrix(G_2x2_5x5, 6, 5)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(G_2x2_3x3, 4, 3)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(G_4x4_3x3, 6, 3)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(G_2x2_5x5, 6, 5)},
       };
 
   // Map from (m, r) to GT transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       GTMatrices = {
-          {F_2_3, TransformMatrix(GT_2x2_3x3, 3, 4)},
-          {F_4_3, TransformMatrix(GT_4x4_3x3, 3, 6)},
-          {F_2_5, TransformMatrix(GT_2x2_5x5, 5, 6)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(GT_2x2_3x3, 3, 4)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(GT_4x4_3x3, 3, 6)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(GT_2x2_5x5, 5, 6)},
       };
 
   auto filterType = cast<ShapedType>(filter.getType());
@@ -370,6 +365,8 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
   int64_t filterW = filterShape[2];
   int64_t filterC = filterShape[3];
 
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   if (filterH != r && filterH != 1)
     return Value();
   if (filterW != r && filterW != 1)
@@ -387,14 +384,13 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
                             zeroIdx, filterH, filterW, /*loopNorFIdx=*/0,
                             /*loopCorFIdx=*/3, /*heightIdx=*/1, /*widthIdx=*/2);
 
-    TransformMapKeyTy key = {m, r};
     int64_t retRows = 1;
     Value matmulRetValue = extractFilter;
     Value zero = builder.create<arith::ConstantOp>(
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix G.
-      auto it = GMatrices.find(key);
+      auto it = GMatrices.find(fmr);
       if (it == GMatrices.end())
         return {};
       const TransformMatrix &GMatrix = it->second;
@@ -416,7 +412,7 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 
     if (rightTransform) {
       // Get constant transform matrix GT.
-      auto it = GTMatrices.find(key);
+      auto it = GTMatrices.find(fmr);
       if (it == GTMatrices.end())
         return {};
       const TransformMatrix &GTMatrix = it->second;
@@ -476,24 +472,26 @@ Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
 ///                            %output<alphaH x alphaW x tileH x tileW x N x C>
 ///                            at [0, 0, %h, %w, %n, %c]
 Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
-                     Value retValue, int64_t m, int64_t r,
+                     Value retValue, WinogradConv2DFmr fmr,
                      bool leftTransform = true, bool rightTransform = true) {
   // Map from (m, r) to BT transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       BTMatrices = {
-          {F_2_3, TransformMatrix(BT_2x2_3x3, 4, 4)},
-          {F_4_3, TransformMatrix(BT_4x4_3x3, 6, 6)},
-          {F_2_5, TransformMatrix(BT_2x2_5x5, 6, 6)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(BT_2x2_3x3, 4, 4)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(BT_4x4_3x3, 6, 6)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(BT_2x2_5x5, 6, 6)},
       };
 
   // Map from (m, r) to B transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       BMatrices = {
-          {F_2_3, TransformMatrix(B_2x2_3x3, 4, 4)},
-          {F_4_3, TransformMatrix(B_4x4_3x3, 6, 6)},
-          {F_2_5, TransformMatrix(B_2x2_5x5, 6, 6)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(B_2x2_3x3, 4, 4)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(B_4x4_3x3, 6, 6)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(B_2x2_5x5, 6, 6)},
       };
 
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   auto inputType = cast<ShapedType>(input.getType());
   Type elementType = inputType.getElementType();
   auto inputShape = inputType.getShape(); // N, H, W, C
@@ -529,7 +527,6 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
                             widthOffset, alphaH, alphaW, /*loopNorFIdx=*/0,
                             /*loopCorFIdx=*/3, /*heightIdx=*/1, /*widthIdx=*/2);
 
-    TransformMapKeyTy key = {m, r};
     int64_t retRows = 1;
     int64_t retCols = 1;
     Value matmulRetValue = extractInput;
@@ -537,7 +534,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
         loc, rewriter.getZeroAttr(elementType));
     if (leftTransform) {
       // Get constant transform matrix BT.
-      auto it = BTMatrices.find(key);
+      auto it = BTMatrices.find(fmr);
       if (it == BTMatrices.end())
         return {};
       const TransformMatrix &BTMatrix = it->second;
@@ -560,7 +557,7 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
 
     if (rightTransform) {
       // Get constant transform matrix B.
-      auto it = BMatrices.find(key);
+      auto it = BMatrices.find(fmr);
       if (it == BMatrices.end())
         return {};
       const TransformMatrix &BMatrix = it->second;
@@ -696,24 +693,26 @@ static Value matrixMultiply(RewriterBase &rewriter, Location loc,
 ///                            output<N x H x W x F>
 ///                            at [%n, (%h x m), (%w x m), %f]
 Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
-                      Value output, int64_t m, int64_t r,
+                      Value output, WinogradConv2DFmr fmr,
                       bool leftTransform = true, bool rightTransform = true) {
   // Map from (m, r) to AT transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       ATMatrices = {
-          {F_2_3, TransformMatrix(AT_2x2_3x3, 2, 4)},
-          {F_4_3, TransformMatrix(AT_4x4_3x3, 4, 6, 32)},
-          {F_2_5, TransformMatrix(AT_2x2_5x5, 2, 6, 16)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(AT_2x2_3x3, 2, 4)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(AT_4x4_3x3, 4, 6, 32)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(AT_2x2_5x5, 2, 6, 16)},
       };
 
   // Map from (m, r) to A transform matrix.
-  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
+  static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
       AMatrices = {
-          {F_2_3, TransformMatrix(A_2x2_3x3, 4, 2)},
-          {F_4_3, TransformMatrix(A_4x4_3x3, 6, 4, 32)},
-          {F_2_5, TransformMatrix(A_2x2_5x5, 6, 2, 16)},
+          {WinogradConv2DFmr::F_2_3, TransformMatrix(A_2x2_3x3, 4, 2)},
+          {WinogradConv2DFmr::F_4_3, TransformMatrix(A_4x4_3x3, 6, 4, 32)},
+          {WinogradConv2DFmr::F_2_5, TransformMatrix(A_2x2_5x5, 6, 2, 16)},
       };
 
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   auto valueType = cast<ShapedType>(value.getType());
   Type elementType = valueType.getElementType();
   auto valueShape = valueType.getShape(); // H, W, TileH, TileW, N, F
@@ -743,9 +742,8 @@ Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
                             FIter, 2, 3, /*loopNorFIdx=*/4,
                             /*loopCorFIdx=*/5, /*heightIdx=*/0, /*widthIdx=*/1);
 
-    const TransformMapKeyTy key = {m, r};
-    const TransformMatrix &AMatrix = AMatrices.at(key);
-    const TransformMatrix &ATMatrix = ATMatrices.at(key);
+    const TransformMatrix &AMatrix = AMatrices.at(fmr);
+    const TransformMatrix &ATMatrix = ATMatrices.at(fmr);
     int64_t scalarFactor = (rightTransform ? AMatrix.scalarFactor : 1) *
                            (leftTransform ? ATMatrix.scalarFactor : 1);
     int64_t retCols = rightTransform ? AMatrix.cols : 1;
@@ -903,7 +901,7 @@ static bool hasAllOneValues(DenseIntElementsAttr attr) {
 /// linalg.winograd_*_transform ops.
 static FailureOr<Operation *>
 winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
-                     int64_t m, int64_t r) {
+                     WinogradConv2DFmr fmr) {
   if (!convOp.hasPureTensorSemantics())
     return rewriter.notifyMatchFailure(
         convOp, "expected pure tensor semantics for linalg.conv_2d_nhwc_fhwc");
@@ -946,6 +944,8 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   int64_t outputW = outputShape[2];
   int64_t outputF = outputShape[3];
 
+  int64_t m, r;
+  std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
   // Only support F(m x m, r x r), F(m x 1, r x 1) or F(1 x m, 1 x r).
   bool isSupportedFilter = false;
   if (filterH == filterW && filterH == r)
@@ -959,17 +959,6 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
     return rewriter.notifyMatchFailure(
         convOp, "only support filter (r x r), (r x 1) or (1 x r)");
 
-  // Currently, we support (m, r) = (2, 3) or (4, 3) or (2, 5).
-  static const llvm::SmallVector<TransformMapKeyTy, 3> validConfigs = {
-      F_2_3, F_4_3, F_2_5};
-
-  TransformMapKeyTy key = {m, r};
-  auto it = llvm::find(validConfigs, key);
-  // If we cannot find the constant transformation matrix, it means we do
-  // not support this configuration yet.
-  if (it == validConfigs.end())
-    return failure();
-
   // All the criterias are satisfied. We can do Winograd Conv2D.
   Location loc = convOp.getLoc();
 
@@ -993,7 +982,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
                                                     filterElementType);
   auto transformedFilter = rewriter.create<linalg::WinogradFilterTransformOp>(
-      loc, retType, filter, retValue, m, r);
+      loc, retType, filter, retValue, fmr);
 
   // --- Create operation for input transform ---
 
@@ -1012,7 +1001,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
                                               inputElementType);
   auto transformedInput = rewriter.create<linalg::WinogradInputTransformOp>(
-      loc, retType, input, retValue, m, r);
+      loc, retType, input, retValue, fmr);
 
   Type outputElementType = outputType.getElementType();
   Value matmulRet = matrixMultiply(rewriter, loc, transformedFilter,
@@ -1035,7 +1024,7 @@ winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
   }
 
   Value transformedOutput = rewriter.create<linalg::WinogradOutputTransformOp>(
-      loc, outputType, matmulRet, output, m, r);
+      loc, outputType, matmulRet, output, fmr);
 
   // When output size is not aligned with output tile size, extract the
   // value from the padded buffer.
@@ -1067,8 +1056,8 @@ decomposeWinogradFilterTransformHelper(RewriterBase &rewriter,
   // For F(1 x m, 1 x r), we only need to do right side transform.
   bool rightTransform = filterW != 1;
   Value transformedFilter =
-      filterTransform(rewriter, loc, filter, op.getOutput(), op.getM(),
-                      op.getR(), leftTransform, rightTransform);
+      filterTransform(rewriter, loc, filter, op.getOutput(), op.getFmr(),
+                      leftTransform, rightTransform);
   if (!transformedFilter)
     return failure();
 
@@ -1094,8 +1083,8 @@ decomposeWinogradInputTransformHelper(RewriterBase &rewriter,
   // For F(1 x m, 1 x r), we only need to do right side transform.
   bool rightTransform = outputW != 1;
   Value transformedInput =
-      inputTransform(rewriter, loc, op.getInput(), op.getOutput(), op.getM(),
-                     op.getR(), leftTransform, rightTransform);
+      inputTransform(rewriter, loc, op.getInput(), op.getOutput(), op.getFmr(),
+                     leftTransform, rightTransform);
   if (!transformedInput)
     return failure();
 
@@ -1120,8 +1109,8 @@ decomposeWinogradOutputTransformHelper(RewriterBase &rewriter,
   // For F(1 x m, 1 x r), we only need to do right side transform.
   bool rightTransform = valueW != 1;
   Value transformedOutput =
-      outputTransform(rewriter, loc, value, op.getOutput(), op.getM(),
-                      op.getR(), leftTransform, rightTransform);
+      outputTransform(rewriter, loc, value, op.getOutput(), op.getFmr(),
+                      leftTransform, rightTransform);
   if (!transformedOutput)
     return failure();
 
@@ -1171,28 +1160,28 @@ class WinogradConv2DNhwcFhwc final
     : public OpRewritePattern<linalg::Conv2DNhwcFhwcOp> {
 public:
   using OpRewritePattern::OpRewritePattern;
-  WinogradConv2DNhwcFhwc(mlir::MLIRContext *context, int64_t m, int64_t r)
-      : OpRewritePattern(context), m(m), r(r) {}
+  WinogradConv2DNhwcFhwc(mlir::MLIRContext *context, WinogradConv2DFmr fmr)
+      : OpRewritePattern(context), fmr(fmr) {}
 
   LogicalResult matchAndRewrite(linalg::Conv2DNhwcFhwcOp convOp,
                                 PatternRewriter &rewriter) const override {
-    if (failed(winogradConv2DHelper(rewriter, convOp, m, r)))
+    if (failed(winogradConv2DHelper(rewriter, convOp, fmr)))
       return failure();
 
     return success();
   }
 
 private:
-  int64_t m;
-  int64_t r;
+  WinogradConv2DFmr fmr;
 };
+
 } // end anonymous namespace
 
 //===----------------------------------------------------------------------===//
 FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
-                                      linalg::Conv2DNhwcFhwcOp op, int64_t m,
-                                      int64_t r) {
-  return winogradConv2DHelper(rewriter, op, m, r);
+                                      linalg::Conv2DNhwcFhwcOp op,
+                                      linalg::WinogradConv2DFmr fmr) {
+  return winogradConv2DHelper(rewriter, op, fmr);
 }
 
 FailureOr<Operation *>
@@ -1213,11 +1202,11 @@ decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
   return decomposeWinogradOutputTransformHelper(rewriter, op);
 }
 
-void populateWinogradConv2DPatterns(RewritePatternSet &patterns, int64_t m,
-                                    int64_t r) {
+void populateWinogradConv2DPatterns(RewritePatternSet &patterns,
+                                    WinogradConv2DFmr fmr) {
   MLIRContext *context = patterns.getContext();
   // TODO: Support more Conv2D data layout, e.g., conv_2d_nchw_fchw
-  patterns.insert<WinogradConv2DNhwcFhwc>(context, m, r);
+  patterns.insert<WinogradConv2DNhwcFhwc>(context, fmr);
 }
 
 void populateDecomposeWinogradOpsPatterns(RewritePatternSet &patterns) {
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 209309ddb413..37d29937282e 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Dialect/Affine/Analysis/AffineStructures.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
-#include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/Affine/LoopUtils.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Arith/Utils/Utils.h"
@@ -31,8 +30,6 @@
 #include "mlir/IR/AffineExprVisitor.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/OpImplementation.h"
-#include "mlir/Pass/Pass.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include <optional>
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index d56b32193765..4ce251d6dd22 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -1567,11 +1567,23 @@ LogicalResult GlobalOp::verify() {
     // Check that the type of the initial value is compatible with the type of
     // the global variable.
     if (auto elementsAttr = llvm::dyn_cast<ElementsAttr>(initValue)) {
-      Type initType = elementsAttr.getType();
-      Type tensorType = getTensorTypeFromMemRefType(memrefType);
-      if (initType != tensorType)
-        return emitOpError("initial value expected to be of type ")
-               << tensorType << ", but was of type " << initType;
+      // Check the element types match.
+      auto initElementType =
+          cast<TensorType>(elementsAttr.getType()).getElementType();
+      auto memrefElementType = memrefType.getElementType();
+
+      if (initElementType != memrefElementType)
+        return emitOpError("initial value element expected to be of type ")
+               << memrefElementType << ", but was of type " << initElementType;
+
+      // Check the shapes match, given that memref globals can only produce
+      // statically shaped memrefs and elements literal type must have a static
+      // shape we can assume both types are shaped.
+      auto initShape = elementsAttr.getShapedType().getShape();
+      auto memrefShape = memrefType.getShape();
+      if (initShape != memrefShape)
+        return emitOpError("initial value shape expected to be ")
+               << memrefShape << " but was " << initShape;
     }
   }
 
@@ -2917,27 +2929,32 @@ static bool haveCompatibleStrides(MemRefType t1, MemRefType t2,
 }
 
 static LogicalResult produceSubViewErrorMsg(SliceVerificationResult result,
-                                            Operation *op, Type expectedType) {
+                                            SubViewOp op, Type expectedType) {
   auto memrefType = llvm::cast<ShapedType>(expectedType);
   switch (result) {
   case SliceVerificationResult::Success:
     return success();
   case SliceVerificationResult::RankTooLarge:
     return op->emitError("expected result rank to be smaller or equal to ")
-           << "the source rank. ";
+           << "the source rank, but got " << op.getType();
   case SliceVerificationResult::SizeMismatch:
     return op->emitError("expected result type to be ")
            << expectedType
-           << " or a rank-reduced version. (mismatch of result sizes) ";
+           << " or a rank-reduced version. (mismatch of result sizes), but got "
+           << op.getType();
   case SliceVerificationResult::ElemTypeMismatch:
     return op->emitError("expected result element type to be ")
-           << memrefType.getElementType();
+           << memrefType.getElementType() << ", but got " << op.getType();
   case SliceVerificationResult::MemSpaceMismatch:
-    return op->emitError("expected result and source memory spaces to match.");
+    return op->emitError(
+               "expected result and source memory spaces to match, but got ")
+           << op.getType();
   case SliceVerificationResult::LayoutMismatch:
     return op->emitError("expected result type to be ")
            << expectedType
-           << " or a rank-reduced version. (mismatch of result layout) ";
+           << " or a rank-reduced version. (mismatch of result layout), but "
+              "got "
+           << op.getType();
   }
   llvm_unreachable("unexpected subview verification result");
 }
@@ -3446,6 +3463,16 @@ LogicalResult ViewOp::verify() {
 
 Value ViewOp::getViewSource() { return getSource(); }
 
+OpFoldResult ViewOp::fold(FoldAdaptor adaptor) {
+  MemRefType sourceMemrefType = getSource().getType();
+  MemRefType resultMemrefType = getResult().getType();
+
+  if (resultMemrefType == sourceMemrefType && resultMemrefType.hasStaticShape())
+    return getViewSource();
+
+  return {};
+}
+
 namespace {
 
 struct ViewOpShapeFolder : public OpRewritePattern<ViewOp> {
diff --git a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
index 637f5ec1c9f9..9049faccadef 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/Transforms/CMakeLists.txt
@@ -13,6 +13,7 @@ add_mlir_dialect_library(MLIRMemRefTransforms
   IndependenceTransforms.cpp
   MultiBuffer.cpp
   NormalizeMemRefs.cpp
+  ReifyResultShapes.cpp
   ResolveShapedTypeResultDims.cpp
   RuntimeOpVerification.cpp
 
diff --git a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
index b40896269081..d6fcb8d9f050 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/NormalizeMemRefs.cpp
@@ -297,8 +297,7 @@ void NormalizeMemRefs::updateFunctionSignature(func::FuncOp funcOp,
                                           /*indexRemap=*/layoutMap,
                                           /*extraOperands=*/{},
                                           /*symbolOperands=*/{},
-                                          /*domOpFilter=*/nullptr,
-                                          /*postDomOpFilter=*/nullptr,
+                                          /*userFilterFn=*/nullptr,
                                           /*allowNonDereferencingOps=*/true,
                                           /*replaceInDeallocOp=*/true))) {
         // If it failed (due to escapes for example), bail out.
@@ -407,8 +406,7 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(func::FuncOp funcOp,
                                         /*indexRemap=*/layoutMap,
                                         /*extraOperands=*/{},
                                         /*symbolOperands=*/{},
-                                        /*domOpFilter=*/nullptr,
-                                        /*postDomOpFilter=*/nullptr,
+                                        /*userFilterFn=*/nullptr,
                                         /*allowNonDereferencingOps=*/true,
                                         /*replaceInDeallocOp=*/true))) {
       // If it failed (due to escapes for example), bail out. Removing the
@@ -457,8 +455,7 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(func::FuncOp funcOp,
                                               /*indexRemap=*/layoutMap,
                                               /*extraOperands=*/{},
                                               /*symbolOperands=*/{},
-                                              /*domOpFilter=*/nullptr,
-                                              /*postDomOpFilter=*/nullptr,
+                                              /*userFilterFn=*/nullptr,
                                               /*allowNonDereferencingOps=*/true,
                                               /*replaceInDeallocOp=*/true))) {
             newOp->erase();
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
new file mode 100644
index 000000000000..e6b9e2f7e821
--- /dev/null
+++ b/mlir/lib/Dialect/MemRef/Transforms/ReifyResultShapes.cpp
@@ -0,0 +1,159 @@
+//===- ReifyResultShapes.cpp - Reify result shapes ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This transform reifies result shapes of `ReifyRankedShapedTypeOpInterface`
+// operations with ranked `memref` and `tensor` results.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/MemRef/Transforms/Passes.h"
+
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Interfaces/DestinationStyleOpInterface.h"
+#include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "llvm/Support/InterleavedRange.h"
+
+#define DEBUG_TYPE "reify-result-shapes"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
+
+namespace mlir {
+namespace memref {
+#define GEN_PASS_DEF_REIFYRESULTSHAPESPASS
+#include "mlir/Dialect/MemRef/Transforms/Passes.h.inc"
+} // namespace memref
+} // namespace mlir
+
+using namespace mlir;
+
+/// Reifies the results of `op`, potentially replacing `op` with a reified
+/// version. Returns `failure` if `mlir::reifyResultShapes` returned failure,
+/// otherwise it always succeeds. Users of this transform should always expect
+/// it to modify the IR, even when it fails. If any of the result types changes,
+/// the transform will insert cast operations to the old type to keep the IR
+/// consistent.
+static LogicalResult reifyOpResultShapes(RewriterBase &rewriter,
+                                         ReifyRankedShapedTypeOpInterface op) {
+  LLVM_DEBUG({ DBGS() << " reifying op: " << op << "\n"; });
+  // Get the reified out shapes.
+  ReifiedRankedShapedTypeDims reifiedResultShapes;
+  if (failed(mlir::reifyResultShapes(rewriter, op, reifiedResultShapes)) ||
+      reifiedResultShapes.empty()) {
+    return op->emitWarning() << "failed to get the reified shapes";
+  }
+
+  bool modified = false;
+  // Compute the new output types.
+  SmallVector<Type> outTypes;
+  for (const auto &[oldTy, reifiedShape] :
+       llvm::zip(op->getResultTypes(), reifiedResultShapes)) {
+    // Skip if it's not a memref or tensor type.
+    if (!isa<RankedTensorType, MemRefType>(oldTy)) {
+      outTypes.push_back(oldTy);
+      continue;
+    }
+
+    ShapedType shapedTy = dyn_cast<ShapedType>(oldTy);
+
+    SmallVector<int64_t> shape = llvm::to_vector(shapedTy.getShape());
+    for (auto &&[dim, ofr] : llvm::zip_equal(shape, reifiedShape)) {
+      std::optional<int64_t> maybeCst = getConstantIntValue(ofr);
+      // If the reified dim is dynamic set it appropriately.
+      if (!maybeCst.has_value()) {
+        dim = ShapedType::kDynamic;
+        continue;
+      }
+      // Set the static dim.
+      dim = *maybeCst;
+    }
+
+    // If the shape didn't change continue.
+    if (shape == shapedTy.getShape()) {
+      outTypes.push_back(oldTy);
+      continue;
+    }
+    modified = true;
+    outTypes.push_back(shapedTy.cloneWith(shape, shapedTy.getElementType()));
+  }
+
+  // Return if we don't need to update.
+  if (!modified) {
+    LLVM_DEBUG({ DBGS() << "- op doesn't require update\n"; });
+    return success();
+  }
+
+  LLVM_DEBUG({
+    DBGS() << "- oldTypes: " << llvm::interleaved_array(op->getResultTypes())
+           << " \n";
+    DBGS() << "- outTypes: " << llvm::interleaved_array(outTypes) << " \n";
+  });
+
+  // We now have outTypes that need to be turned to cast ops.
+  Location loc = op->getLoc();
+  SmallVector<Value> newResults;
+  // TODO: `mlir::reifyResultShapes` and op verifiers may not agree atm.
+  // This is a confluence problem that will need to be addressed.
+  // For now, we know PadOp and ConcatOp are fine.
+  assert((isa<tensor::PadOp, tensor::ConcatOp>(op.getOperation())) &&
+         "incorrect op");
+  Operation *newOp = rewriter.clone(*op);
+  for (auto [reifiedTy, oldRes] : llvm::zip(outTypes, op->getResults())) {
+    OpResult newRes = newOp->getResult(oldRes.getResultNumber());
+    Type oldTy = oldRes.getType();
+    // Continue if the type remained invariant or is not shaped.
+    if (oldTy == reifiedTy || !isa<MemRefType, RankedTensorType>(oldTy)) {
+      newResults.push_back(newRes);
+      continue;
+    }
+
+    // Update the type.
+    newRes.setType(reifiedTy);
+    if (isa<RankedTensorType>(reifiedTy)) {
+      newResults.push_back(rewriter.create<tensor::CastOp>(loc, oldTy, newRes));
+    } else {
+      assert(isa<MemRefType>(reifiedTy) && "expected a memref type");
+      newResults.push_back(rewriter.create<memref::CastOp>(loc, oldTy, newRes));
+    }
+  }
+
+  LLVM_DEBUG({
+    DBGS() << "- reified results " << llvm::interleaved_array(newResults)
+           << "\n";
+  });
+  rewriter.replaceOp(op, newResults);
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// Pass registration
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct ReifyResultShapesPass final
+    : public memref::impl::ReifyResultShapesPassBase<ReifyResultShapesPass> {
+  void runOnOperation() override;
+};
+} // namespace
+
+void ReifyResultShapesPass::runOnOperation() {
+  SmallVector<ReifyRankedShapedTypeOpInterface> ops;
+  getOperation()->walk([&](ReifyRankedShapedTypeOpInterface op) {
+    // Handle ops that are not DPS and that do not carry an tied operand shapes.
+    // For now, limit to tensor::PadOp and tensor::ConcatOp.
+    if (!isa<tensor::PadOp, tensor::ConcatOp>(op.getOperation()))
+      return;
+    ops.push_back(op);
+  });
+  IRRewriter rewriter(&getContext());
+  for (ReifyRankedShapedTypeOpInterface op : ops) {
+    rewriter.setInsertionPoint(op);
+    (void)reifyOpResultShapes(rewriter, op);
+  }
+}
diff --git a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
index 6751fafaf177..b823837bf826 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
@@ -56,10 +56,10 @@ static llvm::raw_ostream &operator<<(llvm::raw_ostream &stream,
 template <typename Stream, typename Range>
 static Stream &printRange(Stream &stream, Range &&range) {
   stream << "[";
-  llvm::for_each(range, [&stream](auto &v) {
+  for (auto &v : range) {
     stream << v;
     stream << ", ";
-  });
+  }
   return stream << "]";
 }
 
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 0dfead98b7e7..80c807e774a7 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -321,6 +321,24 @@ static LogicalResult checkVarAndAccVar(Op op) {
   return success();
 }
 
+template <typename Op>
+static LogicalResult checkNoModifier(Op op) {
+  if (op.getModifiers() != acc::DataClauseModifier::none)
+    return op.emitError("no data clause modifiers are allowed");
+  return success();
+}
+
+template <typename Op>
+static LogicalResult
+checkValidModifier(Op op, acc::DataClauseModifier validModifiers) {
+  if (acc::bitEnumContainsAny(op.getModifiers(), ~validModifiers))
+    return op.emitError(
+        "invalid data clause modifiers: " +
+        acc::stringifyDataClauseModifier(op.getModifiers() & ~validModifiers));
+
+  return success();
+}
+
 static ParseResult parseVar(mlir::OpAsmParser &parser,
                             OpAsmParser::UnresolvedOperand &var) {
   // Either `var` or `varPtr` keyword is required.
@@ -447,6 +465,8 @@ LogicalResult acc::PrivateOp::verify() {
         "data clause associated with private operation must match its intent");
   if (failed(checkVarAndVarType(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -459,6 +479,8 @@ LogicalResult acc::FirstprivateOp::verify() {
                      "match its intent");
   if (failed(checkVarAndVarType(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -471,6 +493,8 @@ LogicalResult acc::ReductionOp::verify() {
                      "match its intent");
   if (failed(checkVarAndVarType(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -485,6 +509,8 @@ LogicalResult acc::DevicePtrOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -499,6 +525,8 @@ LogicalResult acc::PresentOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -518,11 +546,17 @@ LogicalResult acc::CopyinOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkValidModifier(*this, acc::DataClauseModifier::readonly |
+                                           acc::DataClauseModifier::always |
+                                           acc::DataClauseModifier::capture)))
+    return failure();
   return success();
 }
 
 bool acc::CopyinOp::isCopyinReadonly() {
-  return getDataClause() == acc::DataClause::acc_copyin_readonly;
+  return getDataClause() == acc::DataClause::acc_copyin_readonly ||
+         acc::bitEnumContainsAny(getModifiers(),
+                                 acc::DataClauseModifier::readonly);
 }
 
 //===----------------------------------------------------------------------===//
@@ -541,13 +575,20 @@ LogicalResult acc::CreateOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  // this op is the entry part of copyout, so it also needs to allow all
+  // modifiers allowed on copyout.
+  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |
+                                           acc::DataClauseModifier::always |
+                                           acc::DataClauseModifier::capture)))
+    return failure();
   return success();
 }
 
 bool acc::CreateOp::isCreateZero() {
   // The zero modifier is encoded in the data clause.
   return getDataClause() == acc::DataClause::acc_create_zero ||
-         getDataClause() == acc::DataClause::acc_copyout_zero;
+         getDataClause() == acc::DataClause::acc_copyout_zero ||
+         acc::bitEnumContainsAny(getModifiers(), acc::DataClauseModifier::zero);
 }
 
 //===----------------------------------------------------------------------===//
@@ -561,6 +602,8 @@ LogicalResult acc::NoCreateOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -575,6 +618,8 @@ LogicalResult acc::AttachOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -590,6 +635,8 @@ LogicalResult acc::DeclareDeviceResidentOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -605,6 +652,8 @@ LogicalResult acc::DeclareLinkOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -626,11 +675,16 @@ LogicalResult acc::CopyoutOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |
+                                           acc::DataClauseModifier::always |
+                                           acc::DataClauseModifier::capture)))
+    return failure();
   return success();
 }
 
 bool acc::CopyoutOp::isCopyoutZero() {
-  return getDataClause() == acc::DataClause::acc_copyout_zero;
+  return getDataClause() == acc::DataClause::acc_copyout_zero ||
+         acc::bitEnumContainsAny(getModifiers(), acc::DataClauseModifier::zero);
 }
 
 //===----------------------------------------------------------------------===//
@@ -652,6 +706,13 @@ LogicalResult acc::DeleteOp::verify() {
         " or specify original clause this operation was decomposed from");
   if (!getAccVar())
     return emitError("must have device pointer");
+  // This op is the exit part of copyin and create - thus allow all modifiers
+  // allowed on either case.
+  if (failed(checkValidModifier(*this, acc::DataClauseModifier::zero |
+                                           acc::DataClauseModifier::readonly |
+                                           acc::DataClauseModifier::always |
+                                           acc::DataClauseModifier::capture)))
+    return failure();
   return success();
 }
 
@@ -667,6 +728,8 @@ LogicalResult acc::DetachOp::verify() {
         " or specify original clause this operation was decomposed from");
   if (!getAccVar())
     return emitError("must have device pointer");
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -686,6 +749,8 @@ LogicalResult acc::UpdateHostOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -702,6 +767,8 @@ LogicalResult acc::UpdateDeviceOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -718,6 +785,8 @@ LogicalResult acc::UseDeviceOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkNoModifier(*this)))
+    return failure();
   return success();
 }
 
@@ -735,9 +804,17 @@ LogicalResult acc::CacheOp::verify() {
     return failure();
   if (failed(checkVarAndAccVar(*this)))
     return failure();
+  if (failed(checkValidModifier(*this, acc::DataClauseModifier::readonly)))
+    return failure();
   return success();
 }
 
+bool acc::CacheOp::isCacheReadonly() {
+  return getDataClause() == acc::DataClause::acc_cache_readonly ||
+         acc::bitEnumContainsAny(getModifiers(),
+                                 acc::DataClauseModifier::readonly);
+}
+
 template <typename StructureOp>
 static ParseResult parseRegions(OpAsmParser &parser, OperationState &state,
                                 unsigned nRegions = 1) {
@@ -3092,6 +3169,53 @@ void ExitDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<RemoveConstantIfCondition<ExitDataOp>>(context);
 }
 
+void ExitDataOp::addAsyncOnly(MLIRContext *context,
+                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  setAsyncAttr(mlir::UnitAttr::get(context));
+}
+
+void ExitDataOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  getAsyncOperandMutable().append(newValue);
+}
+
+void ExitDataOp::addWaitOnly(MLIRContext *context,
+                             llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  setWaitAttr(mlir::UnitAttr::get(context));
+}
+
+void ExitDataOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  // if hasDevnum, the first value is the devnum. The 'rest' go into the
+  // operands list.
+  if (hasDevnum) {
+    getWaitDevnumMutable().append(newValues.front());
+    newValues = newValues.drop_front();
+  }
+
+  getWaitOperandsMutable().append(newValues);
+}
+
 //===----------------------------------------------------------------------===//
 // EnterDataOp
 //===----------------------------------------------------------------------===//
@@ -3141,6 +3265,53 @@ void EnterDataOp::getCanonicalizationPatterns(RewritePatternSet &results,
   results.add<RemoveConstantIfCondition<EnterDataOp>>(context);
 }
 
+void EnterDataOp::addAsyncOnly(
+    MLIRContext *context, llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  setAsyncAttr(mlir::UnitAttr::get(context));
+}
+
+void EnterDataOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getAsyncAttr());
+  assert(!getAsyncOperand());
+
+  getAsyncOperandMutable().append(newValue);
+}
+
+void EnterDataOp::addWaitOnly(MLIRContext *context,
+                              llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  setWaitAttr(mlir::UnitAttr::get(context));
+}
+
+void EnterDataOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  assert(effectiveDeviceTypes.empty());
+  assert(!getWaitAttr());
+  assert(getWaitOperands().empty());
+  assert(!getWaitDevnum());
+
+  // if hasDevnum, the first value is the devnum. The 'rest' go into the
+  // operands list.
+  if (hasDevnum) {
+    getWaitDevnumMutable().append(newValues.front());
+    newValues = newValues.drop_front();
+  }
+
+  getWaitOperandsMutable().append(newValues);
+}
+
 //===----------------------------------------------------------------------===//
 // AtomicReadOp
 //===----------------------------------------------------------------------===//
@@ -3683,6 +3854,49 @@ mlir::Value UpdateOp::getWaitDevnum(mlir::acc::DeviceType deviceType) {
                             deviceType);
 }
 
+void UpdateOp::addAsyncOnly(MLIRContext *context,
+                            llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setAsyncOnlyAttr(addDeviceTypeAffectedOperandHelper(
+      context, getAsyncOnlyAttr(), effectiveDeviceTypes));
+}
+
+void UpdateOp::addAsyncOperand(
+    MLIRContext *context, mlir::Value newValue,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setAsyncOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
+      context, getAsyncOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValue,
+      getAsyncOperandsMutable()));
+}
+
+void UpdateOp::addWaitOnly(MLIRContext *context,
+                           llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+  setWaitOnlyAttr(addDeviceTypeAffectedOperandHelper(context, getWaitOnlyAttr(),
+                                                     effectiveDeviceTypes));
+}
+
+void UpdateOp::addWaitOperands(
+    MLIRContext *context, bool hasDevnum, mlir::ValueRange newValues,
+    llvm::ArrayRef<DeviceType> effectiveDeviceTypes) {
+
+  llvm::SmallVector<int32_t> segments;
+  if (getWaitOperandsSegments())
+    llvm::copy(*getWaitOperandsSegments(), std::back_inserter(segments));
+
+  setWaitOperandsDeviceTypeAttr(addDeviceTypeAffectedOperandHelper(
+      context, getWaitOperandsDeviceTypeAttr(), effectiveDeviceTypes, newValues,
+      getWaitOperandsMutable(), segments));
+  setWaitOperandsSegments(segments);
+
+  llvm::SmallVector<mlir::Attribute> hasDevnums;
+  if (getHasWaitDevnumAttr())
+    llvm::copy(getHasWaitDevnumAttr(), std::back_inserter(hasDevnums));
+  hasDevnums.insert(
+      hasDevnums.end(),
+      std::max(effectiveDeviceTypes.size(), static_cast<size_t>(1)),
+      mlir::BoolAttr::get(context, hasDevnum));
+  setHasWaitDevnumAttr(mlir::ArrayAttr::get(context, hasDevnums));
+}
+
 //===----------------------------------------------------------------------===//
 // WaitOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 6f8ad5bf21c0..4d2188a69ed6 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -15,11 +15,14 @@
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/OpenACCMPCommon/Interfaces/AtomicInterfaces.h"
+#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
+#include "mlir/IR/SymbolTable.h"
+#include "mlir/Interfaces/FoldInterfaces.h"
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
@@ -2642,6 +2645,23 @@ LogicalResult SimdOp::verify() {
     return emitError()
            << "'omp.composite' attribute present in non-composite wrapper";
 
+  // Firstprivate is not allowed for SIMD in the standard. Check that none of
+  // the private decls are for firstprivate.
+  std::optional<ArrayAttr> privateSyms = getPrivateSyms();
+  if (privateSyms) {
+    for (const Attribute &sym : *privateSyms) {
+      auto symRef = cast<SymbolRefAttr>(sym);
+      omp::PrivateClauseOp privatizer =
+          SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
+              getOperation(), symRef);
+      if (!privatizer)
+        return emitError() << "Cannot find privatizer '" << symRef << "'";
+      if (privatizer.getDataSharingType() ==
+          DataSharingClauseType::FirstPrivate)
+        return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
+    }
+  }
+
   return success();
 }
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 0db752b6d67b..59a9202eb876 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -764,7 +764,7 @@ LoopNest mlir::scf::buildLoopNest(
     ValueRange steps,
     function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilder) {
   // Delegate to the main function by wrapping the body builder.
-  return buildLoopNest(builder, loc, lbs, ubs, steps, std::nullopt,
+  return buildLoopNest(builder, loc, lbs, ubs, steps, {},
                        [&bodyBuilder](OpBuilder &nestedBuilder,
                                       Location nestedLoc, ValueRange ivs,
                                       ValueRange) -> ValueVector {
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 58562536be61..d36d91249ed3 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -274,7 +274,7 @@ struct IfOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -313,15 +313,15 @@ struct IfOpInterface
 
     // Best case: Both branches have the exact same buffer type.
     if (thenBufferType == elseBufferType)
-      return thenBufferType;
+      return cast<BufferLikeType>(thenBufferType);
 
     // Memory space mismatch.
     if (thenBufferType.getMemorySpace() != elseBufferType.getMemorySpace())
       return op->emitError("inconsistent memory space on then/else branches");
 
     // Layout maps are different: Promote to fully dynamic layout map.
-    return getMemRefTypeWithFullyDynamicLayout(
-        cast<TensorType>(opResult.getType()), thenBufferType.getMemorySpace());
+    return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
+        cast<TensorType>(opResult.getType()), thenBufferType.getMemorySpace()));
   }
 };
 
@@ -392,7 +392,7 @@ struct IndexSwitchOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -436,7 +436,7 @@ struct IndexSwitchOpInterface
           cast<TensorType>(value.getType()), bufferType.getMemorySpace());
     }
 
-    return bufferType;
+    return cast<BufferLikeType>(bufferType);
   }
 };
 
@@ -522,13 +522,13 @@ getBbArgReplacements(RewriterBase &rewriter, Block::BlockArgListType bbArgs,
 /// If both buffer types are equal, no casts are needed the computed buffer type
 /// can be used directly. Otherwise, the buffer types can only differ in their
 /// layout map and a cast must be inserted.
-static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
+static FailureOr<BufferLikeType> computeLoopRegionIterArgBufferType(
     Operation *loopOp, BlockArgument iterArg, Value initArg, Value yieldedValue,
     const BufferizationOptions &options, const BufferizationState &state,
     SmallVector<Value> &invocationStack) {
   // Determine the buffer type of the init_arg.
-  auto initArgBufferType = bufferization::detail::asMemRefType(
-      bufferization::getBufferType(initArg, options, state, invocationStack));
+  auto initArgBufferType =
+      bufferization::getBufferType(initArg, options, state, invocationStack);
   if (failed(initArgBufferType))
     return failure();
 
@@ -547,16 +547,15 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
   }
 
   // Compute the buffer type of the yielded value.
-  BaseMemRefType yieldedValueBufferType;
+  BufferLikeType yieldedValueBufferType;
   if (isa<BaseMemRefType>(yieldedValue.getType())) {
     // scf.yield was already bufferized.
-    yieldedValueBufferType = cast<BaseMemRefType>(yieldedValue.getType());
+    yieldedValueBufferType = cast<BufferLikeType>(yieldedValue.getType());
   } else {
     // Note: This typically triggers a recursive call for the buffer type of
     // the iter_arg.
-    auto maybeBufferType =
-        bufferization::detail::asMemRefType(bufferization::getBufferType(
-            yieldedValue, options, state, invocationStack));
+    auto maybeBufferType = bufferization::getBufferType(yieldedValue, options,
+                                                        state, invocationStack);
     if (failed(maybeBufferType))
       return failure();
     yieldedValueBufferType = *maybeBufferType;
@@ -584,8 +583,8 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
         "expected same shape");
   }
 #endif // NDEBUG
-  return getMemRefTypeWithFullyDynamicLayout(
-      iterTensorType, yieldedBufferType.getMemorySpace());
+  return cast<BufferLikeType>(getMemRefTypeWithFullyDynamicLayout(
+      iterTensorType, yieldedBufferType.getMemorySpace()));
 }
 
 /// Return `true` if the given loop may have 0 iterations.
@@ -708,7 +707,7 @@ struct ForOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -719,12 +718,8 @@ struct ForOpInterface
     if (auto opResult = dyn_cast<OpResult>(value)) {
       // The type of an OpResult must match the corresponding iter_arg type.
       BlockArgument bbArg = forOp.getTiedLoopRegionIterArg(opResult);
-      auto bufferType =
-          bufferization::getBufferType(bbArg, options, state, invocationStack);
-      if (failed(bufferType))
-        return failure();
-      assert(isa<BaseMemRefType>(*bufferType) && "expected memref type");
-      return cast<BaseMemRefType>(*bufferType);
+      return bufferization::getBufferType(bbArg, options, state,
+                                          invocationStack);
     }
 
     // Compute result/argument number.
@@ -1047,7 +1042,7 @@ struct WhileOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -1081,10 +1076,10 @@ struct WhileOpInterface
     Value conditionYieldedVal = whileOp.getConditionOp().getArgs()[resultNum];
     if (!isa<TensorType>(conditionYieldedVal.getType())) {
       // scf.condition was already bufferized.
-      return cast<BaseMemRefType>(conditionYieldedVal.getType());
+      return cast<BufferLikeType>(conditionYieldedVal.getType());
     }
-    return bufferization::detail::asMemRefType(bufferization::getBufferType(
-        conditionYieldedVal, options, state, invocationStack));
+    return bufferization::getBufferType(conditionYieldedVal, options, state,
+                                        invocationStack);
   }
 
   /// Assert that yielded values of an scf.while op are equivalent to their
@@ -1303,7 +1298,7 @@ struct ForallOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -1312,15 +1307,15 @@ struct ForallOpInterface
     if (auto bbArg = dyn_cast<BlockArgument>(value))
       // A tensor block argument has the same bufferized type as the
       // corresponding output operand.
-      return bufferization::detail::asMemRefType(
-          bufferization::getBufferType(forallOp.getTiedOpOperand(bbArg)->get(),
-                                       options, state, invocationStack));
+      return bufferization::getBufferType(
+          forallOp.getTiedOpOperand(bbArg)->get(), options, state,
+          invocationStack);
 
     // The bufferized result type is the same as the bufferized type of the
     // corresponding output operand.
-    return bufferization::detail::asMemRefType(bufferization::getBufferType(
+    return bufferization::getBufferType(
         forallOp.getOutputs()[cast<OpResult>(value).getResultNumber()], options,
-        state, invocationStack));
+        state, invocationStack);
   }
 
   bool isRepetitiveRegion(Operation *op, unsigned index) const {
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 4aacbe739ca5..bcecef5e6e0a 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -106,6 +106,20 @@ bool LoopPipelinerInternal::initializeLoopInfo(
   lb = forOp.getLowerBound();
   step = forOp.getStep();
 
+  std::vector<std::pair<Operation *, unsigned>> schedule;
+  options.getScheduleFn(forOp, schedule);
+  if (schedule.empty()) {
+    LDBG("--empty schedule -> BAIL");
+    return false;
+  }
+
+  opOrder.reserve(schedule.size());
+  for (auto &opSchedule : schedule) {
+    maxStage = std::max(maxStage, opSchedule.second);
+    stages[opSchedule.first] = opSchedule.second;
+    opOrder.push_back(opSchedule.first);
+  }
+
   dynamicLoop = true;
   auto upperBoundCst = getConstantIntValue(ub);
   auto lowerBoundCst = getConstantIntValue(lb);
@@ -124,7 +138,7 @@ bool LoopPipelinerInternal::initializeLoopInfo(
       return false;
     }
     int64_t numIteration = llvm::divideCeilSigned(ubImm - lbImm, stepImm);
-    if (numIteration > maxStage) {
+    if (numIteration >= maxStage) {
       dynamicLoop = false;
     } else if (!options.supportDynamicLoops) {
       LDBG("--fewer loop iterations than pipeline stages -> BAIL");
@@ -137,19 +151,6 @@ bool LoopPipelinerInternal::initializeLoopInfo(
     LDBG("--no epilogue or predicate set -> BAIL");
     return false;
   }
-  std::vector<std::pair<Operation *, unsigned>> schedule;
-  options.getScheduleFn(forOp, schedule);
-  if (schedule.empty()) {
-    LDBG("--empty schedule -> BAIL");
-    return false;
-  }
-
-  opOrder.reserve(schedule.size());
-  for (auto &opSchedule : schedule) {
-    maxStage = std::max(maxStage, opSchedule.second);
-    stages[opSchedule.first] = opSchedule.second;
-    opOrder.push_back(opSchedule.first);
-  }
 
   // All operations need to have a stage.
   for (Operation &op : forOp.getBody()->without_terminator()) {
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index ddcae8481a5b..995120ad8680 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -2047,53 +2047,119 @@ getUntiledConsumerFromSlice(RewriterBase &rewriter,
 
 /// A utility to fetch an untiled consumer of
 /// tensor.insert_slice/tensor.parallel_insert_slice.
-static FailureOr<OpOperand *>
-getUntiledConsumerFromSlice(RewriterBase &rewriter, Operation *sliceOp,
-                            MutableArrayRef<LoopLikeOpInterface> loops) {
+static FailureOr<SmallVector<OpOperand *>> getUntiledConsumerOperandsFromSlices(
+    RewriterBase &rewriter, ArrayRef<Operation *> sliceOps,
+    MutableArrayRef<LoopLikeOpInterface> loops) {
   assert(!loops.empty() && "unexpected empty loops");
-  if (auto insertSlice = dyn_cast<tensor::InsertSliceOp>(sliceOp)) {
-    return getUntiledConsumerFromSlice(rewriter, insertSlice, loops);
-  } else if (auto parallelInsertSlice =
-                 dyn_cast<tensor::ParallelInsertSliceOp>(sliceOp)) {
-    return getUntiledConsumerFromSlice(rewriter, parallelInsertSlice, loops);
-  } else {
-    return failure();
+  assert(!sliceOps.empty() && "unexpected empty list of candidate slices");
+  SmallVector<OpOperand *> fusedOperands;
+  for (auto sliceOp : sliceOps) {
+    FailureOr<OpOperand *> fusedOperand =
+        TypeSwitch<Operation *, FailureOr<OpOperand *>>(sliceOp)
+            .Case<tensor::InsertSliceOp, tensor::ParallelInsertSliceOp>(
+                [&](auto op) {
+                  return getUntiledConsumerFromSlice(rewriter, op, loops);
+                })
+            .Default([&](Operation *op) {
+              return rewriter.notifyMatchFailure(op, "unhandled slice type");
+            });
+    if (failed(fusedOperand)) {
+      return failure();
+    }
+    if (!fusedOperands.empty() &&
+        fusedOperand.value()->getOwner() != fusedOperands.front()->getOwner()) {
+      return rewriter.notifyMatchFailure(
+          fusedOperand.value()->getOwner(),
+          "all candidate slices must be to the same consumer");
+    }
+    fusedOperands.push_back(fusedOperand.value());
   }
+  return fusedOperands;
+}
+
+template <typename InsertSliceOpTy>
+static tensor::InsertSliceOp cloneAsInsertSlice(RewriterBase &rewriter,
+                                                InsertSliceOpTy sliceOp);
+
+template <>
+tensor::InsertSliceOp
+cloneAsInsertSlice<tensor::InsertSliceOp>(RewriterBase &rewriter,
+                                          tensor::InsertSliceOp insertSliceOp) {
+  return cast<tensor::InsertSliceOp>(
+      rewriter.clone(*insertSliceOp.getOperation()));
+}
+
+template <>
+tensor::InsertSliceOp cloneAsInsertSlice<tensor::ParallelInsertSliceOp>(
+    RewriterBase &rewriter, tensor::ParallelInsertSliceOp insertSliceOp) {
+  return rewriter.create<tensor::InsertSliceOp>(
+      insertSliceOp->getLoc(), insertSliceOp.getSource(),
+      insertSliceOp.getDest(), insertSliceOp.getMixedOffsets(),
+      insertSliceOp.getMixedSizes(), insertSliceOp.getMixedStrides());
+}
+
+static SmallVector<tensor::InsertSliceOp>
+cloneAsInsertSlices(RewriterBase &rewriter,
+                    ArrayRef<Operation *> candidateSlices) {
+  assert(!candidateSlices.empty() &&
+         "unexpected empty list of slices to clone");
+  SmallVector<tensor::InsertSliceOp> clonedSlices;
+  for (auto sliceOp : candidateSlices) {
+    TypeSwitch<Operation *>(sliceOp)
+        .Case<tensor::InsertSliceOp, tensor::ParallelInsertSliceOp>(
+            [&](auto op) {
+              auto clonedOp = cloneAsInsertSlice(rewriter, op);
+              clonedSlices.push_back(clonedOp);
+            })
+        .Default([&](Operation *op) {
+          // Assert here assuming this has already been checked.
+          assert(0 && "unexpected slice type while cloning as insert slice");
+        });
+  }
+  return clonedSlices;
 }
 
 /// Implementation of fusing consumer of a single slice by computing the
 /// slice of the consumer in-place for scf loop.
 FailureOr<scf::SCFFuseConsumerOfSliceResult>
-mlir::scf::tileAndFuseConsumerOfSlice(
-    RewriterBase &rewriter, Operation *candidateSliceOp,
+mlir::scf::tileAndFuseConsumerOfSlices(
+    RewriterBase &rewriter, ArrayRef<Operation *> candidateSlices,
     MutableArrayRef<LoopLikeOpInterface> loops) {
+  if (candidateSlices.empty()) {
+    return rewriter.notifyMatchFailure(
+        rewriter.getUnknownLoc(),
+        "no candidate slices provided for consumer fusion");
+  }
   // Return if `loops` is empty, return an error for now. Caller is expected
   // to handle this case.
   if (loops.empty()) {
-    return candidateSliceOp->emitOpError(
+    return rewriter.notifyMatchFailure(
+        candidateSlices.front(),
         "cannot call tile and fuse consumer with an empty loop nest");
   }
-  if (!isa<tensor::InsertSliceOp, tensor::ParallelInsertSliceOp>(
-          candidateSliceOp))
-    return failure();
+
+  if (!(llvm::all_of(candidateSlices, llvm::IsaPred<tensor::InsertSliceOp>) ||
+        llvm::all_of(candidateSlices,
+                     llvm::IsaPred<tensor::ParallelInsertSliceOp>))) {
+    return rewriter.notifyMatchFailure(
+        candidateSlices.front(),
+        "candidates slices need to be all `tensor.extract_slice`s or "
+        "`tensor.parallel_insert_slice`s");
+  }
 
   // 1. Get the consumer of scf.for for the result yielded by
   // tensor.insert_slice/parallel_insert_slice.
-  FailureOr<OpOperand *> maybeConsumerOpOperand =
-      getUntiledConsumerFromSlice(rewriter, candidateSliceOp, loops);
-  if (failed(maybeConsumerOpOperand)) {
-    return rewriter.notifyMatchFailure(candidateSliceOp,
-                                       "could not fetch consumer to fuse");
-  }
-  OpOperand *consumerOpOperand = *maybeConsumerOpOperand;
-  Operation *consumerOp = consumerOpOperand->getOwner();
-  unsigned operandNumber = consumerOpOperand->getOperandNumber();
-  unsigned resultNumber = 0;
-  if (auto producerResult = dyn_cast<OpResult>(consumerOpOperand->get())) {
-    resultNumber = producerResult.getResultNumber();
-  } else {
-    return rewriter.notifyMatchFailure(
-        consumerOp, "consumer op's operand doesn't seem to be an OpResult");
+  SmallVector<OpOperand *> consumerOpOperands;
+  Operation *consumerOp;
+  {
+    FailureOr<SmallVector<OpOperand *>> maybeConsumerOpOperand =
+        getUntiledConsumerOperandsFromSlices(rewriter, candidateSlices, loops);
+    if (failed(maybeConsumerOpOperand)) {
+      return rewriter.notifyMatchFailure(candidateSlices.front(),
+                                         "could not fetch consumer to fuse");
+    }
+    std::swap(consumerOpOperands, maybeConsumerOpOperand.value());
+    consumerOp = consumerOpOperands.front()->getOwner();
   }
 
   LoopLikeOpInterface outerMostLoop = loops.front();
@@ -2113,16 +2179,14 @@ mlir::scf::tileAndFuseConsumerOfSlice(
   if (!dstOp)
     return rewriter.notifyMatchFailure(consumerOp,
                                        "consumer op is not DPS operation");
-  SmallVector<Value> dpsInits =
-      llvm::map_to_vector(dstOp.getDpsInits(), [](Value v) { return v; });
-  if (llvm::is_contained(dpsInits, outerMostLoop->getResult(resultNumber))) {
+  if (llvm::any_of(consumerOpOperands, [&](OpOperand *opOperand) {
+        return dstOp.isDpsInit(opOperand);
+      })) {
     return rewriter.notifyMatchFailure(
         consumerOp,
         "consumer op taking the result of scf.for as init is not supported");
   }
-  SmallVector<Value> newInits = dpsInits;
-
-  Location loc = outerMostLoop->getLoc();
+  SmallVector<Value> newInits = llvm::to_vector(dstOp.getDpsInits());
 
   // 3. Move the whole loop structure right before firstUserOfLoop, the
   // dominance should be already ensured by `checkAssumptionForLoop`.
@@ -2137,43 +2201,52 @@ mlir::scf::tileAndFuseConsumerOfSlice(
   // tensor.insert_slice. In the scf.for case this is a clone of the
   // candidateSliceOp whereas in the scf.forall case this is created from the
   // operands of tensor.parallel_insert_slice.
-  tensor::InsertSliceOp clonedInsertSliceOp;
   if (auto sliceOp =
-          dyn_cast<tensor::ParallelInsertSliceOp>(candidateSliceOp)) {
+          dyn_cast<tensor::ParallelInsertSliceOp>(candidateSlices.front())) {
     auto newForallOp = cast<scf::ForallOp>(innerMostLoop.getOperation());
     rewriter.setInsertionPoint(newForallOp.getTerminator());
-    clonedInsertSliceOp = rewriter.create<tensor::InsertSliceOp>(
-        loc, sliceOp.getSource(), sliceOp.getDest(), sliceOp.getMixedOffsets(),
-        sliceOp.getMixedSizes(), sliceOp.getMixedStrides());
   } else {
-    rewriter.setInsertionPoint(candidateSliceOp);
-    clonedInsertSliceOp =
-        cast<tensor::InsertSliceOp>(rewriter.clone(*candidateSliceOp));
+    rewriter.setInsertionPoint(candidateSlices.front());
   }
+  // 5.a. Clone all the candidate slices as equivalent insert slice ops.
+  SmallVector<tensor::InsertSliceOp> clonedInsertSlices =
+      cloneAsInsertSlices(rewriter, candidateSlices);
 
-  // 5.a. Clone consumer op.
+  // 5.b. Clone consumer op.
   auto clonedConsumerOp = cast<TilingInterface>(rewriter.clone(*consumerOp));
+  SmallVector<unsigned> operandNumbers =
+      llvm::map_to_vector(consumerOpOperands, [](OpOperand *opOperand) {
+        return opOperand->getOperandNumber();
+      });
+  SmallVector<OpOperand *> clonedOpFusedOperandsList =
+      llvm::map_to_vector(operandNumbers, [&](unsigned operandNum) {
+        return &clonedConsumerOp->getOpOperand(operandNum);
+      });
 
-  // 5.b. Replace all uses of the loop result with the result of the cloned
+  // 5.c. Replace all uses of the loop result with the result of the cloned
   // tensor.insert_slice.
-  OpOperand &operandToReplace = clonedConsumerOp->getOpOperand(operandNumber);
   rewriter.modifyOpInPlace(clonedConsumerOp, [&]() {
-    operandToReplace.set(clonedInsertSliceOp.getResult());
+    for (auto [operandToReplace, clonedSliceOp] :
+         llvm::zip_equal(clonedOpFusedOperandsList, clonedInsertSlices)) {
+      operandToReplace->set(clonedSliceOp.getResult());
+    }
   });
 
   // 6. Perform tiling of the cloned consumer and replace the operand at
   // `operandNumber` with the source of the cloned tensor.insert_slice op.
-  auto ossSliceOp =
-      cast<OffsetSizeAndStrideOpInterface>(clonedInsertSliceOp.getOperation());
   FailureOr<TilingResult> tileAndFuseResult =
-      tensor::replaceInsertSliceWithTiledConsumer(
-          rewriter, ossSliceOp, clonedConsumerOp->getOpOperand(operandNumber));
+      tensor::replaceInsertSlicesWithTiledConsumer(rewriter, clonedInsertSlices,
+                                                   clonedOpFusedOperandsList);
   if (failed(tileAndFuseResult)) {
     return failure();
   }
+
   auto tiledConsumerOp = cast<TilingInterface>(tileAndFuseResult->tiledOps[0]);
-  rewriter.replaceAllUsesWith(tiledConsumerOp->getOperand(operandNumber),
-                              clonedInsertSliceOp.getSource());
+  for (auto [operandNum, clonedSliceOp] :
+       llvm::zip_equal(operandNumbers, clonedInsertSlices)) {
+    rewriter.replaceAllUsesWith(tiledConsumerOp->getOperand(operandNum),
+                                clonedSliceOp.getSource());
+  }
 
   // 7. Reconstruct [nested] loop with new inits.
   YieldTiledValuesFn newYieldValuesFn =
@@ -2185,14 +2258,20 @@ mlir::scf::tileAndFuseConsumerOfSlice(
     // 8. Set inner insertPoint right before tiled consumer op.
     innerRewriter.setInsertionPoint(tiledConsumerOp);
 
-    SmallVector<OpFoldResult> offsets = ossSliceOp.getMixedOffsets();
-    SmallVector<OpFoldResult> sizes = ossSliceOp.getMixedSizes();
-    SmallVector<OpFoldResult> strides = ossSliceOp.getMixedStrides();
+    SmallVector<SmallVector<OpFoldResult>> allOffsets, allSizes;
+    for (auto candidateSliceOp : clonedInsertSlices) {
+      SmallVector<OpFoldResult> offsets = candidateSliceOp.getMixedOffsets();
+      SmallVector<OpFoldResult> sizes = candidateSliceOp.getMixedSizes();
+      SmallVector<OpFoldResult> strides = candidateSliceOp.getMixedStrides();
 
-    // 9. Check all insert stride is 1.
-    if (!llvm::all_of(strides, isOneInteger)) {
-      return rewriter.notifyMatchFailure(
-          candidateSliceOp, "containingOp's result yield with stride");
+      // 9. Check all insert stride is 1.
+      if (!llvm::all_of(strides, isOneInteger)) {
+        return rewriter.notifyMatchFailure(
+            candidateSliceOp, "containingOp's result yield with stride");
+      }
+
+      allOffsets.emplace_back(std::move(offsets));
+      allSizes.emplace_back(std::move(sizes));
     }
 
     // 10. Try to get iter domain position from input position. Use
@@ -2202,8 +2281,8 @@ mlir::scf::tileAndFuseConsumerOfSlice(
     // tiledConsumerOp could lead to some chained unnecessary extra index
     // computation.
     SmallVector<OpFoldResult> iterDomainOffsets, iterDomainSizes;
-    if (failed(clonedConsumerOp.getIterationDomainTileFromOperandTile(
-            rewriter, operandNumber, offsets, sizes, iterDomainOffsets,
+    if (failed(clonedConsumerOp.getIterationDomainTileFromOperandTiles(
+            rewriter, operandNumbers, allOffsets, allSizes, iterDomainOffsets,
             iterDomainSizes))) {
       return rewriter.notifyMatchFailure(
           clonedConsumerOp,
@@ -2279,10 +2358,13 @@ mlir::scf::tileAndFuseConsumerOfSlice(
   // 16. Need to erase the old scf loop and the cloned consumer op.
   rewriter.eraseOp(clonedConsumerOp);
 
+  SmallVector<OpOperand *> tiledAndFusedOpOperands =
+      llvm::map_to_vector(operandNumbers, [&](unsigned operandNum) {
+        return &tileAndFuseResult->tiledOps[0]->getOpOperand(operandNum);
+      });
   return scf::SCFFuseConsumerOfSliceResult{
-      consumerOpOperand,
-      &(tileAndFuseResult->tiledOps[0]->getOpOperand(operandNumber)),
-      tileAndFuseResult->tiledOps};
+      std::move(consumerOpOperands), std::move(tiledAndFusedOpOperands),
+      std::move(tileAndFuseResult->tiledOps)};
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 8ab5bdc0c5dc..f4047be68ccf 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -301,19 +301,8 @@ static Value ceilDivPositive(OpBuilder &builder, Location loc, Value dividend,
 /// constants, or optional otherwise. Trip count is computed as
 /// ceilDiv(highBound - lowBound, step).
 static std::optional<int64_t> getConstantTripCount(scf::ForOp forOp) {
-  std::optional<int64_t> lbCstOp = getConstantIntValue(forOp.getLowerBound());
-  std::optional<int64_t> ubCstOp = getConstantIntValue(forOp.getUpperBound());
-  std::optional<int64_t> stepCstOp = getConstantIntValue(forOp.getStep());
-  if (!lbCstOp.has_value() || !ubCstOp.has_value() || !stepCstOp.has_value())
-    return {};
-
-  // Constant loop bounds computation.
-  int64_t lbCst = lbCstOp.value();
-  int64_t ubCst = ubCstOp.value();
-  int64_t stepCst = stepCstOp.value();
-  assert(lbCst >= 0 && ubCst >= 0 && stepCst > 0 &&
-         "expected positive loop bounds and step");
-  return llvm::divideCeilSigned(ubCst - lbCst, stepCst);
+  return constantTripCount(forOp.getLowerBound(), forOp.getUpperBound(),
+                           forOp.getStep());
 }
 
 /// Generates unrolled copies of scf::ForOp 'loopBodyBlock', with
diff --git a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
index 2ff3efdc96a7..fa20cc179f89 100644
--- a/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/CooperativeMatrixOps.cpp
@@ -23,7 +23,8 @@ namespace mlir::spirv {
 
 static LogicalResult
 verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
-                       spirv::MemoryAccessAttr memoryOperand) {
+                       spirv::MemoryAccessAttr memoryOperand,
+                       IntegerAttr alignment) {
   auto pointerType = cast<PointerType>(pointer);
   Type pointeeType = pointerType.getPointeeType();
   if (!isa<ScalarType, VectorType>(pointeeType)) {
@@ -49,13 +50,18 @@ verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
           "not compatible with memory operand 'MakePointerVisible'");
     }
 
-    // The 'Aligned' memory operand requires an alignment literal to follow,
-    // which needs to be implemented on the level of op parsing and
-    // (de-)serialization.
-    // TODO: Consider adding support for this attribute value.
-    if (spirv::bitEnumContainsAll(memoryOperand.getValue(),
-                                  spirv::MemoryAccess::Aligned)) {
-      return op->emitOpError("has unhandled memory operand 'Aligned'");
+    // TODO: Need to check that NonPrivatePointer is set for MakePointer*. See
+    // #145485.
+
+    if (spirv::bitEnumContainsAll(operandSet, spirv::MemoryAccess::Aligned) &&
+        !alignment) {
+      return op->emitOpError("missing value for the 'Aligned' memory operand");
+    }
+
+    if (!spirv::bitEnumContainsAll(operandSet, spirv::MemoryAccess::Aligned) &&
+        alignment) {
+      return op->emitOpError(
+          "found alignment attribute for non-'Aligned' memory operand");
     }
   }
 
@@ -72,7 +78,8 @@ verifyCoopMatrixAccess(Operation *op, Type pointer, Type coopMatrix,
 
 LogicalResult KHRCooperativeMatrixLoadOp::verify() {
   return verifyCoopMatrixAccess(*this, getPointer().getType(),
-                                getResult().getType(), getMemoryOperandAttr());
+                                getResult().getType(), getMemoryOperandAttr(),
+                                getAlignmentAttr());
 }
 
 //===----------------------------------------------------------------------===//
@@ -81,7 +88,8 @@ LogicalResult KHRCooperativeMatrixLoadOp::verify() {
 
 LogicalResult KHRCooperativeMatrixStoreOp::verify() {
   return verifyCoopMatrixAccess(*this, getPointer().getType(),
-                                getObject().getType(), getMemoryOperandAttr());
+                                getObject().getType(), getMemoryOperandAttr(),
+                                getAlignmentAttr());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
index a21acef1c4b4..88c7adf3dfcb 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVDialect.cpp
@@ -194,6 +194,13 @@ static Type parseAndVerifyType(SPIRVDialect const &dialect,
           << t.getNumElements();
       return Type();
     }
+  } else if (auto t = dyn_cast<TensorArmType>(type)) {
+    if (!isa<ScalarType>(t.getElementType())) {
+      parser.emitError(
+          typeLoc, "only scalar element type allowed in tensor type but found ")
+          << t.getElementType();
+      return Type();
+    }
   } else {
     parser.emitError(typeLoc, "cannot use ")
         << type << " to compose SPIR-V types";
@@ -363,6 +370,52 @@ static Type parseCooperativeMatrixType(SPIRVDialect const &dialect,
   return CooperativeMatrixType::get(elementTy, dims[0], dims[1], scope, use);
 }
 
+// tensor-arm-type ::=
+//   `!spirv.arm.tensor` `<` dim0 `x` dim1 `x` ... `x` dimN `x` element-type`>`
+static Type parseTensorArmType(SPIRVDialect const &dialect,
+                               DialectAsmParser &parser) {
+  if (parser.parseLess())
+    return {};
+
+  bool unranked = false;
+  SmallVector<int64_t, 4> dims;
+  SMLoc countLoc = parser.getCurrentLocation();
+
+  if (parser.parseOptionalStar().succeeded()) {
+    unranked = true;
+    if (parser.parseXInDimensionList())
+      return {};
+  } else if (parser.parseDimensionList(dims, /*allowDynamic=*/true)) {
+    return {};
+  }
+
+  if (!unranked && dims.empty()) {
+    parser.emitError(countLoc, "arm.tensors do not support rank zero");
+    return {};
+  }
+
+  if (llvm::is_contained(dims, 0)) {
+    parser.emitError(countLoc, "arm.tensors do not support zero dimensions");
+    return {};
+  }
+
+  if (llvm::any_of(dims, [](int64_t dim) { return dim < 0; }) &&
+      llvm::any_of(dims, [](int64_t dim) { return dim > 0; })) {
+    parser.emitError(countLoc, "arm.tensor shape dimensions must be either "
+                               "fully dynamic or completed shaped");
+    return {};
+  }
+
+  auto elementTy = parseAndVerifyType(dialect, parser);
+  if (!elementTy)
+    return {};
+
+  if (parser.parseGreater())
+    return {};
+
+  return TensorArmType::get(dims, elementTy);
+}
+
 // TODO: Reorder methods to be utilities first and parse*Type
 // methods in alphabetical order
 //
@@ -759,6 +812,8 @@ Type SPIRVDialect::parseType(DialectAsmParser &parser) const {
     return parseStructType(*this, parser);
   if (keyword == "matrix")
     return parseMatrixType(*this, parser);
+  if (keyword == "arm.tensor")
+    return parseTensorArmType(*this, parser);
   parser.emitError(parser.getNameLoc(), "unknown SPIR-V type: ") << keyword;
   return Type();
 }
@@ -855,10 +910,28 @@ static void print(MatrixType type, DialectAsmPrinter &os) {
   os << ">";
 }
 
+static void print(TensorArmType type, DialectAsmPrinter &os) {
+  os << "arm.tensor<";
+
+  llvm::interleave(
+      type.getShape(), os,
+      [&](int64_t dim) {
+        if (ShapedType::isDynamic(dim))
+          os << '?';
+        else
+          os << dim;
+      },
+      "x");
+  if (!type.hasRank()) {
+    os << "*";
+  }
+  os << "x" << type.getElementType() << ">";
+}
+
 void SPIRVDialect::printType(Type type, DialectAsmPrinter &os) const {
   TypeSwitch<Type>(type)
       .Case<ArrayType, CooperativeMatrixType, PointerType, RuntimeArrayType,
-            ImageType, SampledImageType, StructType, MatrixType>(
+            ImageType, SampledImageType, StructType, MatrixType, TensorArmType>(
           [&](auto type) { print(type, os); })
       .Default([](Type) { llvm_unreachable("unhandled SPIR-V type"); });
 }
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
index 7148027dae78..eb2974d62fdd 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
@@ -547,6 +547,12 @@ ParseResult spirv::ConstantOp::parse(OpAsmParser &parser,
       return failure();
   }
 
+  if (llvm::isa<TensorArmType>(type)) {
+    if (parser.parseOptionalColon().succeeded())
+      if (parser.parseType(type))
+        return failure();
+  }
+
   return parser.addTypeToList(type, result.types);
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
index 93e0c9b33c54..2b90df42af5c 100644
--- a/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVTypes.cpp
@@ -18,8 +18,9 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+#include <algorithm>
 #include <cstdint>
-#include <iterator>
+#include <numeric>
 
 using namespace mlir;
 using namespace mlir::spirv;
@@ -96,7 +97,7 @@ bool CompositeType::classof(Type type) {
     return isValid(vectorType);
   return llvm::isa<spirv::ArrayType, spirv::CooperativeMatrixType,
                    spirv::MatrixType, spirv::RuntimeArrayType,
-                   spirv::StructType>(type);
+                   spirv::StructType, spirv::TensorArmType>(type);
 }
 
 bool CompositeType::isValid(VectorType type) {
@@ -107,8 +108,8 @@ bool CompositeType::isValid(VectorType type) {
 
 Type CompositeType::getElementType(unsigned index) const {
   return TypeSwitch<Type, Type>(*this)
-      .Case<ArrayType, CooperativeMatrixType, RuntimeArrayType, VectorType>(
-          [](auto type) { return type.getElementType(); })
+      .Case<ArrayType, CooperativeMatrixType, RuntimeArrayType, VectorType,
+            TensorArmType>([](auto type) { return type.getElementType(); })
       .Case<MatrixType>([](MatrixType type) { return type.getColumnType(); })
       .Case<StructType>(
           [index](StructType type) { return type.getElementType(index); })
@@ -125,6 +126,8 @@ unsigned CompositeType::getNumElements() const {
     return structType.getNumElements();
   if (auto vectorType = llvm::dyn_cast<VectorType>(*this))
     return vectorType.getNumElements();
+  if (auto tensorArmType = dyn_cast<TensorArmType>(*this))
+    return tensorArmType.getNumElements();
   if (llvm::isa<CooperativeMatrixType>(*this)) {
     llvm_unreachable(
         "invalid to query number of elements of spirv Cooperative Matrix type");
@@ -151,6 +154,13 @@ void CompositeType::getExtensions(
         return llvm::cast<ScalarType>(type.getElementType())
             .getExtensions(extensions, storage);
       })
+      .Case<TensorArmType>([&](TensorArmType type) {
+        static constexpr Extension ext{Extension::SPV_ARM_tensors};
+        extensions.push_back(ext);
+        return llvm::cast<ScalarType>(type.getElementType())
+            .getExtensions(extensions, storage);
+      })
+
       .Default([](Type) { llvm_unreachable("invalid composite type"); });
 }
 
@@ -171,6 +181,12 @@ void CompositeType::getCapabilities(
         return llvm::cast<ScalarType>(type.getElementType())
             .getCapabilities(capabilities, storage);
       })
+      .Case<TensorArmType>([&](TensorArmType type) {
+        static constexpr Capability cap{Capability::TensorsARM};
+        capabilities.push_back(cap);
+        return llvm::cast<ScalarType>(type.getElementType())
+            .getCapabilities(capabilities, storage);
+      })
       .Default([](Type) { llvm_unreachable("invalid composite type"); });
 }
 
@@ -186,6 +202,13 @@ std::optional<int64_t> CompositeType::getSizeInBytes() {
       return std::nullopt;
     return *elementSize * vectorType.getNumElements();
   }
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    std::optional<int64_t> elementSize =
+        llvm::cast<ScalarType>(tensorArmType.getElementType()).getSizeInBytes();
+    if (!elementSize)
+      return std::nullopt;
+    return *elementSize * tensorArmType.getNumElements();
+  }
   return std::nullopt;
 }
 
@@ -691,6 +714,8 @@ bool SPIRVType::classof(Type type) {
     return true;
   if (auto vectorType = llvm::dyn_cast<VectorType>(type))
     return CompositeType::isValid(vectorType);
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(type))
+    return llvm::isa<ScalarType>(tensorArmType.getElementType());
   return false;
 }
 
@@ -712,6 +737,8 @@ void SPIRVType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
     matrixType.getExtensions(extensions, storage);
   } else if (auto ptrType = llvm::dyn_cast<PointerType>(*this)) {
     ptrType.getExtensions(extensions, storage);
+  } else if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    tensorArmType.getExtensions(extensions, storage);
   } else {
     llvm_unreachable("invalid SPIR-V Type to getExtensions");
   }
@@ -732,6 +759,8 @@ void SPIRVType::getCapabilities(
     matrixType.getCapabilities(capabilities, storage);
   } else if (auto ptrType = llvm::dyn_cast<PointerType>(*this)) {
     ptrType.getCapabilities(capabilities, storage);
+  } else if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(*this)) {
+    tensorArmType.getCapabilities(capabilities, storage);
   } else {
     llvm_unreachable("invalid SPIR-V Type to getCapabilities");
   }
@@ -1203,11 +1232,85 @@ void MatrixType::getCapabilities(
   llvm::cast<SPIRVType>(getColumnType()).getCapabilities(capabilities, storage);
 }
 
+//===----------------------------------------------------------------------===//
+// TensorArmType
+//===----------------------------------------------------------------------===//
+
+struct spirv::detail::TensorArmTypeStorage final : TypeStorage {
+  using KeyTy = std::tuple<ArrayRef<int64_t>, Type>;
+
+  static TensorArmTypeStorage *construct(TypeStorageAllocator &allocator,
+                                         const KeyTy &key) {
+    auto [shape, elementType] = key;
+    shape = allocator.copyInto(shape);
+    return new (allocator.allocate<TensorArmTypeStorage>())
+        TensorArmTypeStorage(shape, elementType);
+  }
+
+  static llvm::hash_code hashKey(const KeyTy &key) {
+    auto [shape, elementType] = key;
+    return llvm::hash_combine(shape, elementType);
+  }
+
+  bool operator==(const KeyTy &key) const {
+    return key == KeyTy(shape, elementType);
+  }
+
+  TensorArmTypeStorage(ArrayRef<int64_t> shape, Type elementType)
+      : shape(std::move(shape)), elementType(std::move(elementType)) {}
+
+  ArrayRef<int64_t> shape;
+  Type elementType;
+};
+
+TensorArmType TensorArmType::get(ArrayRef<int64_t> shape, Type elementType) {
+  return Base::get(elementType.getContext(), shape, elementType);
+}
+
+TensorArmType TensorArmType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
+                                       Type elementType) const {
+  return TensorArmType::get(shape.value_or(getShape()), elementType);
+}
+
+Type TensorArmType::getElementType() const { return getImpl()->elementType; }
+ArrayRef<int64_t> TensorArmType::getShape() const { return getImpl()->shape; }
+
+void TensorArmType::getExtensions(
+    SPIRVType::ExtensionArrayRefVector &extensions,
+    std::optional<StorageClass> storage) {
+
+  llvm::cast<SPIRVType>(getElementType()).getExtensions(extensions, storage);
+  static constexpr Extension ext{Extension::SPV_ARM_tensors};
+  extensions.push_back(ext);
+}
+
+void TensorArmType::getCapabilities(
+    SPIRVType::CapabilityArrayRefVector &capabilities,
+    std::optional<StorageClass> storage) {
+  llvm::cast<SPIRVType>(getElementType())
+      .getCapabilities(capabilities, storage);
+  static constexpr Capability cap{Capability::TensorsARM};
+  capabilities.push_back(cap);
+}
+
+LogicalResult
+TensorArmType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
+                                ArrayRef<int64_t> shape, Type elementType) {
+  if (llvm::is_contained(shape, 0))
+    return emitError() << "arm.tensor do not support dimensions = 0";
+  if (llvm::any_of(shape, [](int64_t dim) { return dim < 0; }) &&
+      llvm::any_of(shape, [](int64_t dim) { return dim > 0; }))
+    return emitError()
+           << "arm.tensor shape dimensions must be either fully dynamic or "
+              "completed shaped";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // SPIR-V Dialect
 //===----------------------------------------------------------------------===//
 
 void SPIRVDialect::registerTypes() {
   addTypes<ArrayType, CooperativeMatrixType, ImageType, MatrixType, PointerType,
-           RuntimeArrayType, SampledImageType, StructType>();
+           RuntimeArrayType, SampledImageType, StructType, TensorArmType>();
 }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 15ddcf58a3d7..6fd20466e36e 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -282,8 +282,8 @@ LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
 
   // Creates a new function with the update signature.
   rewriter.modifyOpInPlace(funcOp, [&] {
-    funcOp.setType(rewriter.getFunctionType(
-        signatureConverter.getConvertedTypes(), std::nullopt));
+    funcOp.setType(
+        rewriter.getFunctionType(signatureConverter.getConvertedTypes(), {}));
   });
   return success();
 }
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 31fecf61c5e4..5f395eef9d60 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -1303,7 +1303,7 @@ void FuncOp::build(OpBuilder &builder, OperationState &state, StringRef name,
     return;
   assert(type.getNumInputs() == argAttrs.size());
   call_interface_impl::addArgAndResultAttrs(
-      builder, state, argAttrs, /*resultAttrs=*/std::nullopt,
+      builder, state, argAttrs, /*resultAttrs=*/{},
       getArgAttrsAttrName(state.name), getResAttrsAttrName(state.name));
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 7f0b65768744..65adc37808fe 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -729,9 +729,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
     createFuncCall(rewriter, loc, name, {},
                    {tensor, lvlCoords, values, filled, added, count},
                    EmitCInterface::On);
+    Operation *parent = getTop(op);
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
-    Operation *parent = getTop(op);
     rewriter.setInsertionPointAfter(parent);
     rewriter.create<memref::DeallocOp>(loc, values);
     rewriter.create<memref::DeallocOp>(loc, filled);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
index 8981de58306d..347363f20d74 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
@@ -167,10 +167,10 @@ IterationGraphSorter::IterationGraphSorter(
 AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
   // Reset the adjacency matrix that represents the iteration graph.
   for (auto &row : itGraph)
-    std::fill(row.begin(), row.end(), false);
+    llvm::fill(row, false);
 
   // Reset in-degree.
-  std::fill(inDegree.begin(), inDegree.end(), 0);
+  llvm::fill(inDegree, 0);
 
   // Add the constraints for the loop to level map.
   for (auto [in, map] : llvm::zip(ins, loop2InsLvl)) {
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
index 84129edee375..9e41c8ec19bc 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
@@ -929,7 +929,7 @@ std::pair<Operation *, Value> sparse_tensor::genCoIteration(
     ivs.push_back(uniIdx);
 
   // Ensures all operands are valid.
-  assert(llvm::all_of(ivs, [](Value v) { return v != nullptr; }));
+  assert(!llvm::is_contained(ivs, nullptr));
   TypeRange types = ValueRange(ivs).getTypes();
   auto whileOp = builder.create<scf::WhileOp>(loc, types, ivs);
 
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
index 72144ec71c5d..0430e6fc6c63 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -1872,9 +1872,9 @@ OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
   if (!sourceTy || !resultTy || sourceTy != resultTy)
     return {};
 
-  // If the source and result are both 1D tensors and have the same type, the
-  // reshape has no effect, even if the tensor is dynamically shaped.
-  if (sourceTy.getRank() == 1)
+  // If the source and result are both 0D or 1D tensors and have the same type,
+  // the reshape has no effect, even if the tensor is dynamically shaped.
+  if (sourceTy.getRank() <= 1)
     return source;
 
   if (auto fromElements = getShape().getDefiningOp<tensor::FromElementsOp>()) {
@@ -3793,6 +3793,29 @@ struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
 
 } // namespace
 
+LogicalResult
+PadOp::reifyResultShapes(OpBuilder &b,
+                         ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
+  reifiedReturnShapes.resize(1, SmallVector<OpFoldResult>(getType().getRank()));
+  SmallVector<OpFoldResult> lp = getMixedLowPad();
+  SmallVector<OpFoldResult> hp = getMixedHighPad();
+  for (int64_t i = 0; i < getResultType().getRank(); ++i) {
+    if (!getType().isDynamicDim(i)) {
+      reifiedReturnShapes[0][i] = b.getIndexAttr(getType().getDimSize(i));
+      continue;
+    }
+    Location loc = getLoc();
+    Value dim = b.createOrFold<tensor::DimOp>(
+        loc, getSource(), b.create<arith::ConstantIndexOp>(loc, i));
+
+    AffineExpr d0, d1, d2;
+    bindDims(b.getContext(), d0, d1, d2);
+    reifiedReturnShapes[0][i] = affine::makeComposedFoldedAffineApply(
+        b, loc, {d0 + d1 + d2}, {dim, lp[i], hp[i]});
+  }
+  return success();
+}
+
 void PadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                         MLIRContext *context) {
   results.add<FoldStaticZeroPadding, FoldSourceTensorCast, FoldTargetTensorCast,
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 729c048db456..829b2ab92ac2 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -49,7 +49,7 @@ struct CastOpInterface
     return {{op->getResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -68,20 +68,22 @@ struct CastOpInterface
     if (isa<UnrankedTensorType>(castOp.getSource().getType())) {
       // When casting to a ranked tensor, we cannot infer any static offset or
       // strides from the source. Assume fully dynamic.
-      return getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace);
+      return cast<BufferLikeType>(
+          getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace));
     }
 
     // Case 2: Casting to an unranked tensor type
     if (isa<UnrankedTensorType>(castOp.getType())) {
-      return getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace);
+      return cast<BufferLikeType>(
+          getMemRefTypeWithFullyDynamicLayout(castOp.getType(), memorySpace));
     }
 
     // Case 3: Ranked tensor -> ranked tensor. The offsets and strides do not
     // change.
     auto rankedResultType = cast<RankedTensorType>(castOp.getType());
-    return MemRefType::get(
+    return cast<BufferLikeType>(MemRefType::get(
         rankedResultType.getShape(), rankedResultType.getElementType(),
-        llvm::cast<MemRefType>(*maybeSrcBufferType).getLayout(), memorySpace);
+        llvm::cast<MemRefType>(*maybeSrcBufferType).getLayout(), memorySpace));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -141,7 +143,7 @@ struct CollapseShapeOpInterface
     return {{op->getOpResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -157,12 +159,13 @@ struct CollapseShapeOpInterface
     if (!canBeCollapsed) {
       // If dims cannot be collapsed, this op bufferizes to a new allocation.
       RankedTensorType tensorResultType = collapseShapeOp.getResultType();
-      return bufferization::getMemRefTypeWithStaticIdentityLayout(
-          tensorResultType, srcBufferType.getMemorySpace());
+      return cast<BufferLikeType>(
+          bufferization::getMemRefTypeWithStaticIdentityLayout(
+              tensorResultType, srcBufferType.getMemorySpace()));
     }
 
-    return memref::CollapseShapeOp::computeCollapsedType(
-        srcBufferType, collapseShapeOp.getReassociationIndices());
+    return cast<BufferLikeType>(memref::CollapseShapeOp::computeCollapsedType(
+        srcBufferType, collapseShapeOp.getReassociationIndices()));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -319,7 +322,7 @@ struct ExpandShapeOpInterface
     return {{op->getOpResult(0), BufferRelation::Equivalent}};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -334,7 +337,7 @@ struct ExpandShapeOpInterface
         expandShapeOp.getReassociationIndices());
     if (failed(maybeResultType))
       return failure();
-    return *maybeResultType;
+    return cast<BufferLikeType>(*maybeResultType);
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -404,7 +407,7 @@ struct ExtractSliceOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -417,10 +420,10 @@ struct ExtractSliceOpInterface
     SmallVector<OpFoldResult> mixedOffsets = extractSliceOp.getMixedOffsets();
     SmallVector<OpFoldResult> mixedSizes = extractSliceOp.getMixedSizes();
     SmallVector<OpFoldResult> mixedStrides = extractSliceOp.getMixedStrides();
-    return memref::SubViewOp::inferRankReducedResultType(
+    return cast<BufferLikeType>(memref::SubViewOp::inferRankReducedResultType(
         extractSliceOp.getType().getShape(),
         llvm::cast<MemRefType>(*srcMemrefType), mixedOffsets, mixedSizes,
-        mixedStrides);
+        mixedStrides));
   }
 };
 
@@ -501,8 +504,8 @@ struct FromElementsOpInterface
         /*copy=*/false);
     if (failed(tensorAlloc))
       return failure();
-    FailureOr<BaseMemRefType> memrefType = bufferization::detail::asMemRefType(
-        bufferization::getBufferType(*tensorAlloc, options, state));
+    FailureOr<BufferLikeType> memrefType =
+        bufferization::getBufferType(*tensorAlloc, options, state);
     if (failed(memrefType))
       return failure();
     Value buffer = rewriter.create<bufferization::ToBufferOp>(
@@ -753,7 +756,7 @@ struct PadOpInterface
     return {};
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -765,9 +768,10 @@ struct PadOpInterface
     if (failed(maybeSrcBufferType))
       return failure();
     MemRefLayoutAttrInterface layout;
-    return MemRefType::get(padOp.getResultType().getShape(),
-                           padOp.getResultType().getElementType(), layout,
-                           maybeSrcBufferType->getMemorySpace());
+    return cast<BufferLikeType>(
+        MemRefType::get(padOp.getResultType().getShape(),
+                        padOp.getResultType().getElementType(), layout,
+                        maybeSrcBufferType->getMemorySpace()));
   }
 
   LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
@@ -927,7 +931,7 @@ struct ReshapeOpInterface
     return success();
   }
 
-  FailureOr<BaseMemRefType>
+  FailureOr<BufferLikeType>
   getBufferType(Operation *op, Value value, const BufferizationOptions &options,
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
@@ -937,9 +941,9 @@ struct ReshapeOpInterface
         reshapeOp.getSource(), options, state, invocationStack);
     if (failed(maybeSourceBufferType))
       return failure();
-    return getMemRefTypeWithStaticIdentityLayout(
+    return cast<BufferLikeType>(getMemRefTypeWithStaticIdentityLayout(
         reshapeOp.getResult().getType(),
-        cast<BaseMemRefType>(maybeSourceBufferType.value()).getMemorySpace());
+        cast<BaseMemRefType>(maybeSourceBufferType.value()).getMemorySpace()));
   }
 };
 
diff --git a/mlir/lib/Dialect/Tensor/Transforms/SwapExtractSliceWithProducerPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/SwapExtractSliceWithProducerPatterns.cpp
index 6f33f9b55ceb..4392a2c0eb83 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/SwapExtractSliceWithProducerPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/SwapExtractSliceWithProducerPatterns.cpp
@@ -17,6 +17,9 @@
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Interfaces/TilingInterface.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "tensor-swap-slices"
 
 using namespace mlir;
 
@@ -39,21 +42,55 @@ FailureOr<TilingResult> tensor::replaceExtractSliceWithTiledProducer(
   return *tiledResult;
 }
 
-FailureOr<TilingResult> tensor::replaceInsertSliceWithTiledConsumer(
-    OpBuilder &builder, OffsetSizeAndStrideOpInterface sliceOp,
-    OpOperand &consumer) {
-  auto consumerOp = dyn_cast<TilingInterface>(consumer.getOwner());
+FailureOr<TilingResult> tensor::replaceInsertSlicesWithTiledConsumer(
+    OpBuilder &builder, ArrayRef<tensor::InsertSliceOp> sliceOps,
+    ArrayRef<OpOperand *> consumerOperands) {
+  if (sliceOps.empty()) {
+    LLVM_DEBUG(
+        { llvm::dbgs() << "expected candidate slices list to be non-empty"; });
+    return failure();
+  }
+  if (sliceOps.size() != consumerOperands.size()) {
+    LLVM_DEBUG({
+      llvm::dbgs()
+          << "expected as many operands as the number of slices passed";
+    });
+    return failure();
+  }
+  auto consumerOp =
+      dyn_cast<TilingInterface>(consumerOperands.front()->getOwner());
   if (!consumerOp)
     return failure();
+  for (auto opOperand : consumerOperands.drop_front()) {
+    if (opOperand->getOwner() != consumerOp) {
+      LLVM_DEBUG({
+        llvm::dbgs()
+            << "expected all consumer operands to be from the same operation";
+      });
+      return failure();
+    }
+  }
 
-  // `TilingInterface` currently only supports strides being 1.
-  if (!llvm::all_of(sliceOp.getMixedStrides(), isOneInteger))
-    return failure();
+  auto consumerOperandNums = llvm::map_to_vector(
+      consumerOperands, [](OpOperand *opOperand) -> unsigned {
+        return opOperand->getOperandNumber();
+      });
+  SmallVector<SmallVector<OpFoldResult>> allOffsets;
+  SmallVector<SmallVector<OpFoldResult>> allSizes;
+  for (auto sliceOp : sliceOps) {
+
+    // `TilingInterface` currently only supports strides being 1.
+    if (!llvm::all_of(sliceOp.getMixedStrides(), isOneInteger))
+      return failure();
 
+    SmallVector<OpFoldResult> offsets = sliceOp.getMixedOffsets();
+    SmallVector<OpFoldResult> sizes = sliceOp.getMixedSizes();
+    allOffsets.emplace_back(std::move(offsets));
+    allSizes.emplace_back(std::move(sizes));
+  }
   FailureOr<TilingResult> tiledResult =
-      consumerOp.getTiledImplementationFromOperandTile(
-          builder, consumer.getOperandNumber(), sliceOp.getMixedOffsets(),
-          sliceOp.getMixedSizes());
+      consumerOp.getTiledImplementationFromOperandTiles(
+          builder, consumerOperandNums, allOffsets, allSizes);
   if (failed(tiledResult))
     return failure();
 
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index c19d3733769b..1d21096e8920 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -344,7 +344,7 @@ LogicalResult SelectOp::canonicalize(SelectOp op, PatternRewriter &rewriter) {
     return failure();
   rewriter.modifyOpInPlace(op, [&]() {
     op.getOperation()->setOperands(
-        {notOp.getInput1(), op.getInput3(), op.getInput2()});
+        {notOp.getInput1(), op.getOnFalse(), op.getOnTrue()});
   });
   return success();
 }
@@ -1510,8 +1510,8 @@ OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
 }
 
 OpFoldResult tosa::SelectOp::fold(FoldAdaptor adaptor) {
-  if (getInput2() == getInput3())
-    return getInput2();
+  if (getOnTrue() == getOnFalse())
+    return getOnTrue();
 
   auto predicate =
       llvm::dyn_cast_if_present<DenseIntElementsAttr>(adaptor.getInput1());
@@ -1520,8 +1520,8 @@ OpFoldResult tosa::SelectOp::fold(FoldAdaptor adaptor) {
 
   if (!predicate.isSplat())
     return {};
-  return predicate.getSplatValue<APInt>().getBoolValue() ? getInput2()
-                                                         : getInput3();
+  return predicate.getSplatValue<APInt>().getBoolValue() ? getOnTrue()
+                                                         : getOnFalse();
 }
 
 OpFoldResult TileOp::fold(FoldAdaptor adaptor) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 3135fbd49bfb..a0dc8964f7a0 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -3829,16 +3829,16 @@ LogicalResult ReverseOp::verify() {
 
 LogicalResult tosa::SelectOp::verify() {
   // verify input2 and input3 have same element type as output
-  if (verifySameElementTypes(*this, /* inType = */ getInput2().getType(),
+  if (verifySameElementTypes(*this, /* inType = */ getOnTrue().getType(),
                              /* outType = */ getOutput().getType())
           .failed() ||
-      verifySameElementTypes(*this, /* inType = */ getInput3().getType(),
+      verifySameElementTypes(*this, /* inType = */ getOnFalse().getType(),
                              /* outType = */ getOutput().getType())
           .failed()) {
     return failure();
   }
   // verify input1 has element type of bool
-  auto predicateType = llvm::dyn_cast<ShapedType>(getInput1().getType());
+  auto predicateType = llvm::dyn_cast<ShapedType>(getPred().getType());
   if (!predicateType) {
     return emitOpError("expect shaped tensor for input1, got ")
            << getInput1().getType();
diff --git a/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
index bbf079faea3d..803993bb1008 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tosa/Transforms/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_dialect_library(MLIRTosaTransforms
+  TosaConvertIntegerTypeToSignless.cpp
   TosaDecomposeTransposeConv.cpp
   TosaDecomposeDepthwise.cpp
   TosaFolders.cpp
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp
new file mode 100644
index 000000000000..7c51592dabe2
--- /dev/null
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaConvertIntegerTypeToSignless.cpp
@@ -0,0 +1,139 @@
+//===- TosaConvertIntegerTypeToSignless.cpp
+//-------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-------------------------------------------------------------------------------===//
+
+// -----------
+// Motivation:
+// -----------
+
+// The TOSA specification uses a signless type system, which means that
+// information about signedness must be encapsulated by the operations
+// themselves. For example, tosa.rescale provides the attributes
+// `input_unsigned` and `output_unsigned` to indicate whether the input/output
+// should be interpreted as unsigned or signed.
+
+// The TOSA dialect, on the other hand, allows the use of signed or unsigned
+// types in addition to signless. As such, when converting from TOSA dialect to
+// other formats, we need to ensure that we conform to the TOSA specification.
+
+// ---------
+// Overview:
+// ---------
+
+// This pass converts signed or unsigned integer types to signless. It currently
+// does this greedily for all operators and can also change the signature of the
+// function. Should the signature of the entrypoint function change, it will be
+// the responsibility of the user to carry signedness information of the inputs
+// and outputs independently.
+
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
+#include "mlir/Dialect/Tosa/IR/TosaOps.h"
+#include "mlir/Dialect/Tosa/Transforms/Passes.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+namespace tosa {
+
+#define GEN_PASS_DEF_TOSACONVERTINTEGERTYPETOSIGNLESS
+#include "mlir/Dialect/Tosa/Transforms/Passes.h.inc"
+
+namespace {
+class ToSignlessTensorTypeConverter : public TypeConverter {
+  static Type convertType(Type type) {
+    const auto tensorType = dyn_cast<TensorType>(type);
+    if (!tensorType)
+      return type;
+
+    const auto intType = dyn_cast<IntegerType>(tensorType.getElementType());
+    if (!intType ||
+        intType.getSignedness() == IntegerType::SignednessSemantics::Signless)
+      return type;
+
+    const auto signlessType = IntegerType::get(
+        intType.getContext(), intType.getWidth(), IntegerType::Signless);
+    return tensorType.cloneWith(std::nullopt, signlessType);
+  }
+
+public:
+  explicit ToSignlessTensorTypeConverter() { addConversion(convertType); }
+};
+
+class ConvertGenericOpWithIntegerTensorType : public ConversionPattern {
+public:
+  ConvertGenericOpWithIntegerTensorType(TypeConverter &typeConverter,
+                                        MLIRContext *context)
+      : ConversionPattern(typeConverter, MatchAnyOpTypeTag{}, 0, context) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    // Typically TOSA operators have a single result, but some have an
+    // arbitrary number. 4 seems like a good balance as an optimization
+    // hint for storing result types.
+    constexpr unsigned int numResults = 4;
+
+    // Convert integer types to signless
+    SmallVector<Type, numResults> resultTypes;
+    if (failed(typeConverter->convertTypes(op->getResultTypes(), resultTypes)))
+      return failure();
+
+    // Create new op with replaced operands and results
+    auto *newOp = Operation::create(
+        op->getLoc(), op->getName(), resultTypes, operands, op->getAttrs(),
+        op->getPropertiesStorage(), op->getSuccessors(), op->getNumRegions());
+
+    // Handle regions in e.g. tosa.cond_if and tosa.while_loop
+    for (auto regions : llvm::zip(op->getRegions(), newOp->getRegions())) {
+      Region &before = std::get<0>(regions);
+      Region &parent = std::get<1>(regions);
+      rewriter.inlineRegionBefore(before, parent, parent.end());
+      if (failed(rewriter.convertRegionTypes(&parent, *typeConverter)))
+        return failure();
+    }
+
+    // Replace with rewritten op
+    rewriter.insert(newOp);
+    rewriter.replaceOp(op, newOp->getResults());
+    return success();
+  }
+};
+
+class TosaConvertIntegerTypeToSignless
+    : public impl::TosaConvertIntegerTypeToSignlessBase<
+          TosaConvertIntegerTypeToSignless> {
+public:
+  void runOnOperation() override {
+    MLIRContext *context = &getContext();
+    ConversionTarget target(*context);
+    ToSignlessTensorTypeConverter typeConverter;
+
+    target.addDynamicallyLegalOp<func::FuncOp>([&](func::FuncOp op) {
+      return typeConverter.isSignatureLegal(op.getFunctionType()) &&
+             typeConverter.isLegal(&op.getBody());
+    });
+    target.markUnknownOpDynamicallyLegal([&](Operation *op) {
+      return typeConverter.isLegal(op->getOperandTypes()) &&
+             typeConverter.isLegal(op->getResultTypes());
+    });
+
+    RewritePatternSet patterns(context);
+    populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
+        patterns, typeConverter);
+    patterns.add<ConvertGenericOpWithIntegerTensorType>(typeConverter, context);
+
+    if (failed(
+            applyFullConversion(getOperation(), target, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+} // namespace tosa
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
index 02a3ad83bdef..799775346952 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaMakeBroadcastable.cpp
@@ -169,9 +169,9 @@ struct ConvertTosaOp<tosa::SelectOp> : public OpRewritePattern<tosa::SelectOp> {
   LogicalResult matchAndRewrite(tosa::SelectOp tosaOp,
                                 PatternRewriter &rewriter) const override {
 
-    Value input1 = tosaOp.getInput1();
-    Value input2 = tosaOp.getInput2();
-    Value input3 = tosaOp.getInput3();
+    Value input1 = tosaOp.getPred();
+    Value input2 = tosaOp.getOnTrue();
+    Value input3 = tosaOp.getOnFalse();
     Value output = tosaOp.getResult();
 
     auto outputType = dyn_cast<RankedTensorType>(output.getType());
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
index de08e7e9a439..a4edccfd4c9c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaProfileCompliance.cpp
@@ -188,8 +188,8 @@ LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::RFFT2dOp op) {
 
 template <>
 LogicalResult ProfileInfoDepot::populateProfileInfo(tosa::SelectOp op) {
-  addValue(op.getInput2());
-  addValue(op.getInput3());
+  addValue(op.getOnTrue());
+  addValue(op.getOnFalse());
   addValue(op.getOutput());
   return success();
 }
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
index 5a732ec78158..7f85cd52f6bd 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaReduceTransposes.cpp
@@ -602,8 +602,7 @@ void TosaReduceTransposes::runOnOperation() {
         !llvm::isa<RankedTensorType>(output.getType()))
       return;
 
-    llvm::for_each(transposeOp.getPerms(),
-                   [&perms](const auto i) { perms.emplace_back(i); });
+    llvm::append_range(perms, transposeOp.getPerms());
 
     // We let --canonicalize deal with identity transpose.
     if (llvm::equal(llvm::seq<int32_t>(0, perms.size()), perms))
diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
index 229f42d3178b..3f27849b8c90 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaValidation.cpp
@@ -1320,13 +1320,14 @@ void TosaValidation::runOnOperation() {
 
     // validate operator element types:
     // - rescale operator is allowed to have ui8/ui16/ui32
-    //   operands/results
+    //   operands/results when strictOpSpecAlignment is false
     // - perform valid element type check at the beginning to
     //   protect rest of code against quantized element types
-    const bool opIsRescale = isa<tosa::RescaleOp>(op);
+    const bool allowUnsigned =
+        !strictOpSpecAlignment && isa<tosa::RescaleOp>(op);
     for (Value operand : op->getOperands()) {
       auto elementTy = getElementTypeOrSelf(operand);
-      if (!isValidElementType(elementTy, opIsRescale)) {
+      if (!isValidElementType(elementTy, allowUnsigned)) {
         op->emitOpError() << "is not profile-aligned: element type "
                           << elementTy << " is not legal";
         return signalPassFailure();
@@ -1334,7 +1335,7 @@ void TosaValidation::runOnOperation() {
     }
     for (Type resultTy : op->getResultTypes()) {
       auto elementTy = getElementTypeOrSelf(resultTy);
-      if (!isValidElementType(elementTy, opIsRescale)) {
+      if (!isValidElementType(elementTy, allowUnsigned)) {
         op->emitOpError() << "is not profile-aligned: element type "
                           << elementTy << " is not legal";
         return signalPassFailure();
diff --git a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
index 7a9f8f4b1b52..12257da878a4 100644
--- a/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp
@@ -19,9 +19,9 @@ using namespace mlir;
 #include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.cpp.inc"
 
 DiagnosedSilenceableFailure
-transform::DebugEmitRemarkAtOp::apply(transform::TransformRewriter &rewriter,
-                                      transform::TransformResults &results,
-                                      transform::TransformState &state) {
+transform::EmitRemarkAtOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &results,
+                                 transform::TransformState &state) {
   if (isa<TransformHandleTypeInterface>(getAt().getType())) {
     auto payload = state.getPayloadOps(getAt());
     for (Operation *op : payload)
@@ -52,9 +52,10 @@ transform::DebugEmitRemarkAtOp::apply(transform::TransformRewriter &rewriter,
   return DiagnosedSilenceableFailure::success();
 }
 
-DiagnosedSilenceableFailure transform::DebugEmitParamAsRemarkOp::apply(
-    transform::TransformRewriter &rewriter,
-    transform::TransformResults &results, transform::TransformState &state) {
+DiagnosedSilenceableFailure
+transform::EmitParamAsRemarkOp::apply(transform::TransformRewriter &rewriter,
+                                      transform::TransformResults &results,
+                                      transform::TransformState &state) {
   std::string str;
   llvm::raw_string_ostream os(str);
   if (getMessage())
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index bb9bdd70625e..0db0317461c0 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -799,8 +799,8 @@ transform::ApplyRegisteredPassOp::apply(transform::TransformRewriter &rewriter,
     if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
       // The corresponding value attribute(s) is/are passed in via a param.
       // Obtain the param-operand via its specified index.
-      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
-      assert(dynamicOptionIdx < dynamicOptions.size() &&
+      int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      assert(dynamicOptionIdx < static_cast<int64_t>(dynamicOptions.size()) &&
              "the number of ParamOperandAttrs in the options DictionaryAttr"
              "should be the same as the number of options passed as params");
       ArrayRef<Attribute> attrsAssociatedToParam =
@@ -1015,8 +1015,9 @@ LogicalResult transform::ApplyRegisteredPassOp::verify() {
   std::function<LogicalResult(Attribute)> checkOptionValue =
       [&](Attribute valueAttr) -> LogicalResult {
     if (auto paramOperand = dyn_cast<transform::ParamOperandAttr>(valueAttr)) {
-      size_t dynamicOptionIdx = paramOperand.getIndex().getInt();
-      if (dynamicOptionIdx < 0 || dynamicOptionIdx >= dynamicOptions.size())
+      int64_t dynamicOptionIdx = paramOperand.getIndex().getInt();
+      if (dynamicOptionIdx < 0 ||
+          dynamicOptionIdx >= static_cast<int64_t>(dynamicOptions.size()))
         return emitOpError()
                << "dynamic option index " << dynamicOptionIdx
                << " is out of bounds for the number of dynamic options: "
diff --git a/mlir/lib/Dialect/Utils/CMakeLists.txt b/mlir/lib/Dialect/Utils/CMakeLists.txt
index a0096e5f299d..f5bb687ae071 100644
--- a/mlir/lib/Dialect/Utils/CMakeLists.txt
+++ b/mlir/lib/Dialect/Utils/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_library(MLIRDialectUtils
 
   DEPENDS
   MLIRDialectUtilsIncGen
+  MLIRArithDialect
 
   LINK_LIBS PUBLIC
   MLIRIR
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 5e0f36064be3..29f71b7cb924 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -398,6 +398,18 @@ std::optional<int64_t> vector::getConstantVscaleMultiplier(Value value) {
   return {};
 }
 
+/// Converts an IntegerAttr to have the specified type if needed.
+/// This handles cases where constant attributes have a different type than the
+/// target element type. If the input attribute is not an IntegerAttr or already
+/// has the correct type, returns it unchanged.
+static Attribute convertIntegerAttr(Attribute attr, Type expectedType) {
+  if (auto intAttr = mlir::dyn_cast<IntegerAttr>(attr)) {
+    if (intAttr.getType() != expectedType)
+      return IntegerAttr::get(expectedType, intAttr.getInt());
+  }
+  return attr;
+}
+
 //===----------------------------------------------------------------------===//
 // CombiningKindAttr
 //===----------------------------------------------------------------------===//
@@ -1384,6 +1396,11 @@ bool ExtractOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
 }
 
 LogicalResult vector::ExtractOp::verify() {
+  if (auto resTy = dyn_cast<VectorType>(getResult().getType()))
+    if (resTy.getRank() == 0)
+      return emitError(
+          "expected a scalar instead of a 0-d vector as the result type");
+
   // Note: This check must come before getMixedPosition() to prevent a crash.
   auto dynamicMarkersCount =
       llvm::count_if(getStaticPosition(), ShapedType::isDynamic);
@@ -2459,8 +2476,37 @@ static OpFoldResult foldFromElementsToElements(FromElementsOp fromElementsOp) {
   return {};
 }
 
+/// Fold vector.from_elements to a constant when all operands are constants.
+/// Example:
+///   %c1 = arith.constant 1 : i32
+///   %c2 = arith.constant 2 : i32
+///   %v = vector.from_elements %c1, %c2 : vector<2xi32>
+/// =>
+///   %v = arith.constant dense<[1, 2]> : vector<2xi32>
+///
+static OpFoldResult foldFromElementsToConstant(FromElementsOp fromElementsOp,
+                                               ArrayRef<Attribute> elements) {
+  if (llvm::any_of(elements, [](Attribute attr) { return !attr; }))
+    return {};
+
+  auto destVecType = fromElementsOp.getDest().getType();
+  auto destEltType = destVecType.getElementType();
+  // Constant attributes might have a different type than the return type.
+  // Convert them before creating the dense elements attribute.
+  auto convertedElements = llvm::map_to_vector(elements, [&](Attribute attr) {
+    return convertIntegerAttr(attr, destEltType);
+  });
+
+  return DenseElementsAttr::get(destVecType, convertedElements);
+}
+
 OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) {
-  return foldFromElementsToElements(*this);
+  if (auto res = foldFromElementsToElements(*this))
+    return res;
+  if (auto res = foldFromElementsToConstant(*this, adaptor.getElements()))
+    return res;
+
+  return {};
 }
 
 /// Rewrite a vector.from_elements into a vector.splat if all elements are the
@@ -3211,6 +3257,11 @@ void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
 }
 
 LogicalResult InsertOp::verify() {
+  if (auto srcTy = dyn_cast<VectorType>(getValueToStoreType()))
+    if (srcTy.getRank() == 0)
+      return emitError(
+          "expected a scalar instead of a 0-d vector as the source operand");
+
   SmallVector<OpFoldResult> position = getMixedPosition();
   auto destVectorType = getDestVectorType();
   if (position.size() > static_cast<unsigned>(destVectorType.getRank()))
@@ -3322,17 +3373,6 @@ foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr,
 
   /// Converts the expected type to an IntegerAttr if there's
   /// a mismatch.
-  auto convertIntegerAttr = [](Attribute attr, Type expectedType) -> Attribute {
-    if (auto intAttr = mlir::dyn_cast<IntegerAttr>(attr)) {
-      if (intAttr.getType() != expectedType)
-        return IntegerAttr::get(expectedType, intAttr.getInt());
-    }
-    return attr;
-  };
-
-  // The `convertIntegerAttr` method specifically handles the case
-  // for `llvm.mlir.constant` which can hold an attribute with a
-  // different type than the return type.
   if (auto denseSource = llvm::dyn_cast<DenseElementsAttr>(srcAttr)) {
     for (auto value : denseSource.getValues<Attribute>())
       insertedValues.push_back(convertIntegerAttr(value, destEltType));
@@ -4251,33 +4291,39 @@ void ExtractStridedSliceOp::getCanonicalizationPatterns(
 /// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, AffineMapAttr permutationMapAttr,
+                           ValueRange indices, std::optional<Value> padding,
+                           AffineMapAttr permutationMapAttr,
                            /*optional*/ ArrayAttr inBoundsAttr) {
+
   Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  Value padding = builder.create<arith::ConstantOp>(
-      result.location, elemType, builder.getZeroAttr(elemType));
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
-        padding, /*mask=*/Value(), inBoundsAttr);
+        *padding, /*mask=*/Value(), inBoundsAttr);
 }
 
 /// 2. Builder that sets padding to zero an empty mask (variant without attrs).
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, AffineMap permutationMap,
+                           ValueRange indices, std::optional<Value> padding,
+                           AffineMap permutationMap,
                            std::optional<ArrayRef<bool>> inBounds) {
   auto permutationMapAttr = AffineMapAttr::get(permutationMap);
   auto inBoundsAttr = (inBounds && !inBounds.value().empty())
                           ? builder.getBoolArrayAttr(inBounds.value())
                           : builder.getBoolArrayAttr(
                                 SmallVector<bool>(vectorType.getRank(), false));
-  build(builder, result, vectorType, source, indices, permutationMapAttr,
-        inBoundsAttr);
+  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
+  build(builder, result, vectorType, source, indices, *padding,
+        permutationMapAttr, inBoundsAttr);
 }
 
 /// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
 void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                            VectorType vectorType, Value source,
-                           ValueRange indices, Value padding,
+                           ValueRange indices, std::optional<Value> padding,
                            std::optional<ArrayRef<bool>> inBounds) {
   AffineMap permutationMap = getTransferMinorIdentityMap(
       llvm::cast<ShapedType>(source.getType()), vectorType);
@@ -4286,23 +4332,14 @@ void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                           ? builder.getBoolArrayAttr(inBounds.value())
                           : builder.getBoolArrayAttr(
                                 SmallVector<bool>(vectorType.getRank(), false));
+  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
+  if (!padding)
+    padding = builder.create<ub::PoisonOp>(result.location, elemType);
   build(builder, result, vectorType, source, indices, permutationMapAttr,
-        padding,
+        *padding,
         /*mask=*/Value(), inBoundsAttr);
 }
 
-/// 4. Builder that sets padding to zero and permutation map to
-/// 'getMinorIdentityMap'.
-void TransferReadOp::build(OpBuilder &builder, OperationState &result,
-                           VectorType vectorType, Value source,
-                           ValueRange indices,
-                           std::optional<ArrayRef<bool>> inBounds) {
-  Type elemType = llvm::cast<ShapedType>(source.getType()).getElementType();
-  Value padding = builder.create<arith::ConstantOp>(
-      result.location, elemType, builder.getZeroAttr(elemType));
-  build(builder, result, vectorType, source, indices, padding, inBounds);
-}
-
 template <typename EmitFun>
 static LogicalResult verifyPermutationMap(AffineMap permutationMap,
                                           EmitFun emitOpError) {
@@ -5856,18 +5893,7 @@ OpFoldResult ShapeCastOp::fold(FoldAdaptor adaptor) {
 
   // shape_cast(transpose(x)) -> shape_cast(x)
   if (auto transpose = getSource().getDefiningOp<TransposeOp>()) {
-    // This folder does
-    //    shape_cast(transpose) -> shape_cast
-    // But another pattern, ConvertIllegalShapeCastOpsToTransposes, does
-    //    shape_cast -> shape_cast(transpose)
-    // i.e. the complete opposite. When paired, these 2 patterns can cause
-    // infinite cycles in pattern rewriting.
-    // ConvertIllegalShapeCastOpsToTransposes only matches on scalable
-    // vectors, so by disabling this folder for scalable vectors the
-    // cycle is avoided.
-    // TODO: Check if ConvertIllegalShapeCastOpsToTransposes is
-    // still needed. If it's not, then we can fold here.
-    if (!transpose.getType().isScalable() && isOrderPreserving(transpose)) {
+    if (isOrderPreserving(transpose)) {
       setOperand(transpose.getVector());
       return getResult();
     }
@@ -6568,6 +6594,28 @@ bool ConstantMaskOp::isAllOnesMask() {
   return true;
 }
 
+OpFoldResult ConstantMaskOp::fold(FoldAdaptor adaptor) {
+  ArrayRef<int64_t> bounds = getMaskDimSizes();
+  ArrayRef<int64_t> vectorSizes = getVectorType().getShape();
+
+  auto createBoolSplat = [&](bool x) {
+    return SplatElementsAttr::get(getVectorType(),
+                                  BoolAttr::get(getContext(), x));
+  };
+
+  // Check the corner case of 0-D vectors first.
+  if (vectorSizes.empty()) {
+    assert(bounds.size() == 1 && "invalid sizes for zero rank mask");
+    return createBoolSplat(bounds[0] == 1);
+  }
+  // Fold vector.constant_mask to splat if possible.
+  if (bounds == vectorSizes)
+    return createBoolSplat(true);
+  if (llvm::all_of(bounds, [](int64_t x) { return x == 0; }))
+    return createBoolSplat(false);
+  return OpFoldResult();
+}
+
 //===----------------------------------------------------------------------===//
 // CreateMaskOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index 045c192787f1..fb99e22c77ea 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -15,9 +15,12 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorDistribution.h"
 #include "mlir/IR/AffineExpr.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <utility>
 
@@ -52,6 +55,25 @@ static AffineMap calculateImplicitMap(VectorType sequentialType,
   return map;
 }
 
+/// Given a sequential and distributed vector type, returns the distributed
+/// dimension. This function expects that only a single dimension is
+/// distributed.
+static int getDistributedDim(VectorType sequentialType,
+                             VectorType distributedType) {
+  assert(sequentialType.getRank() == distributedType.getRank() &&
+         "sequential and distributed vector types must have the same rank");
+  int64_t distributedDim = -1;
+  for (int64_t i = 0; i < sequentialType.getRank(); ++i) {
+    if (distributedType.getDimSize(i) != sequentialType.getDimSize(i)) {
+      // Keep this assert here in case WarpExecuteOnLane0Op gets extended to
+      // support distributing multiple dimensions in the future.
+      assert(distributedDim == -1 && "found multiple distributed dims");
+      distributedDim = i;
+    }
+  }
+  return distributedDim;
+}
+
 namespace {
 
 /// Helper struct to create the load / store operations that permit transit
@@ -151,7 +173,7 @@ struct DistributedLoadStoreHelper {
     }
     SmallVector<bool> inBounds(indices.size(), true);
     return b.create<vector::TransferReadOp>(
-        loc, cast<VectorType>(type), buffer, indices,
+        loc, cast<VectorType>(type), buffer, indices, /*padding=*/std::nullopt,
         ArrayRef<bool>(inBounds.begin(), inBounds.end()));
   }
 
@@ -1076,6 +1098,196 @@ struct WarpOpCreateMask : public WarpDistributionPattern {
   }
 };
 
+/// Sink out insert_strided_slice op feeding into a warp op yield.
+/// ```
+/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<8x1xf32>) {
+///   ...
+///   %src = ... : vector<4x32xf32>
+///   %dest = ... : vector<8x32xf32>
+///   %insert = vector.insert_strided_slice %src, %dest, offsets = [0, 0],
+///     strides = [1, 1] : vector<4x32xf32> into vector<8x32xf32>
+///   gpu.yield %insert : vector<8x32xf32>
+/// }
+/// ```
+/// To
+/// ```
+/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<4x1xf32>,
+/// vector<8x1xf32>) {
+///   ...
+///   %src = ... : vector<4x32xf32>
+///   %dest = ... : vector<8x32xf32>
+///   gpu.yield %src, %dest : vector<4x16xf32>, vector<8x16xf32>
+/// }
+/// %insert = vector.insert_strided_slice %0#0, %0#1,
+///   offsets = [0, 0], strides = [1, 1] : vector<4x1xf32> into vector<8x1xf32>
+/// ```
+/// NOTE: Current support assumes that both src and dest vectors are distributed
+/// to lanes and sinking the insert op does not require any cross lane
+/// communication.
+struct WarpOpInsertStridedSlice : public WarpDistributionPattern {
+  using Base::Base;
+  LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
+                                PatternRewriter &rewriter) const override {
+    OpOperand *operand =
+        getWarpResult(warpOp, llvm::IsaPred<vector::InsertStridedSliceOp>);
+    if (!operand)
+      return failure();
+    unsigned int operandNumber = operand->getOperandNumber();
+    auto insertOp =
+        operand->get().getDefiningOp<vector::InsertStridedSliceOp>();
+    auto distributedType =
+        cast<VectorType>(warpOp.getResult(operandNumber).getType());
+    // Distributed type must be 2D or higher.
+    // TODO: Support 1D distributed types.
+    if (distributedType.getRank() < 2)
+      return rewriter.notifyMatchFailure(
+          insertOp, "result vector type must be 2D or higher");
+    // Find the distributed dimension of the dest vector. There should be
+    // exactly one.
+    auto yieldedType = cast<VectorType>(operand->get().getType());
+    int64_t destDistributedDim =
+        getDistributedDim(yieldedType, distributedType);
+    assert(destDistributedDim != -1 && "could not find distributed dimension");
+
+    VectorType srcType = insertOp.getSourceVectorType();
+    VectorType destType = insertOp.getDestVectorType();
+    // Currently we require that both source (kD) and dest (nD) vectors are
+    // distributed. This requires that distributedDim (d) is contained in the
+    // last k dims of the dest vector (d >= n - k).
+    // TODO: Add support for case where source vector is not distributed.
+    int64_t sourceDistributedDim =
+        destDistributedDim - (destType.getRank() - srcType.getRank());
+    if (sourceDistributedDim < 0)
+      return rewriter.notifyMatchFailure(
+          insertOp,
+          "distributed dimension must be in the last k dims of dest vector");
+    // Distributed dimension must be fully inserted.
+    if (srcType.getDimSize(sourceDistributedDim) !=
+        destType.getDimSize(destDistributedDim))
+      return rewriter.notifyMatchFailure(
+          insertOp, "distributed dimension must be fully inserted");
+    SmallVector<int64_t> newSourceDistShape(
+        insertOp.getSourceVectorType().getShape());
+    newSourceDistShape[sourceDistributedDim] =
+        distributedType.getDimSize(destDistributedDim);
+    auto newSourceTy =
+        VectorType::get(newSourceDistShape, distributedType.getElementType());
+    VectorType newDestTy = distributedType;
+    SmallVector<size_t> newRetIndices;
+    WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
+        rewriter, warpOp, {insertOp.getValueToStore(), insertOp.getDest()},
+        {newSourceTy, newDestTy}, newRetIndices);
+    rewriter.setInsertionPointAfter(newWarpOp);
+    Value distributedSource = newWarpOp->getResult(newRetIndices[0]);
+    Value distributedDest = newWarpOp->getResult(newRetIndices[1]);
+    // Create a new insert strided slice op that inserts distributed source into
+    // distributed dest.
+    Value newInsert = rewriter.create<vector::InsertStridedSliceOp>(
+        insertOp.getLoc(), distributedDest.getType(), distributedSource,
+        distributedDest, insertOp.getOffsets(), insertOp.getStrides());
+    rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber), newInsert);
+    return success();
+  }
+};
+
+/// Sink out extract_strided_slice op feeding into a warp op yield.
+/// ```
+/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<16x1xf32>) {
+///   ...
+///   %src = ... : vector<64x32xf32>
+///   %extract = vector.extract_strided_slice %src, offsets = [0], sizes = [16],
+///     strides = [1] : vector<64x32xf32> to vector<16x32xf32>
+///   gpu.yield %extract : vector<16x32xf32>
+/// }
+/// ```
+/// To
+/// ```
+/// %0 = gpu.warp_execute_on_lane_0(%arg0) -> (vector<64x1xf32>) {
+///   ...
+///   %src = ... : vector<64x32xf32>
+///   gpu.yield %src : vector<64x32xf32>
+/// }
+/// %extract = vector.extract_strided_slice %0, offsets = [0], sizes = [16],
+///   strides = [1] : vector<64x1xf32> to vector<16x1xf32>
+/// ```
+/// NOTE: Current support assumes that the extraction happens only on non
+/// distributed dimensions (does not require cross lane communication).
+struct WarpOpExtractStridedSlice : public WarpDistributionPattern {
+  using Base::Base;
+  LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
+                                PatternRewriter &rewriter) const override {
+    OpOperand *operand =
+        getWarpResult(warpOp, llvm::IsaPred<vector::ExtractStridedSliceOp>);
+    if (!operand)
+      return failure();
+    unsigned int operandNumber = operand->getOperandNumber();
+    auto extractOp =
+        operand->get().getDefiningOp<vector::ExtractStridedSliceOp>();
+    auto distributedType =
+        cast<VectorType>(warpOp.getResult(operandNumber).getType());
+    // Distributed type must be 2D or higher.
+    // TODO: Support 1D distributed types.
+    if (distributedType.getRank() < 2)
+      return rewriter.notifyMatchFailure(
+          extractOp, "result vector type must be 2D or higher");
+
+    // Find the distributed dimension. There should be exactly one.
+    auto yieldedType = cast<VectorType>(operand->get().getType());
+    int64_t distributedDim = getDistributedDim(yieldedType, distributedType);
+    assert(distributedDim != -1 && "could not find distributed dimension");
+
+    int64_t numOfExtractedDims =
+        static_cast<int64_t>(extractOp.getSizes().size());
+    // If the distributed dim is included in the extracted dims,  then we make
+    // sure distributed dim is fully extracted. If distributed dim is not
+    // included in extracted dims, it is guaranteed to be fully extracted (i.e.
+    // distributed dim comes after all the extracted dims)
+    // TODO: Partial extraction from distributed dimension require cross lane
+    // communication.
+    if (distributedDim < numOfExtractedDims) {
+      int64_t distributedDimOffset =
+          llvm::cast<IntegerAttr>(extractOp.getOffsets()[distributedDim])
+              .getInt();
+      int64_t distributedDimSize =
+          llvm::cast<IntegerAttr>(extractOp.getSizes()[distributedDim])
+              .getInt();
+      if (distributedDimOffset != 0 ||
+          distributedDimSize != yieldedType.getDimSize(distributedDim))
+        return rewriter.notifyMatchFailure(
+            extractOp, "distributed dimension must be fully extracted");
+    }
+    SmallVector<int64_t> newDistributedShape(
+        extractOp.getSourceVectorType().getShape());
+    newDistributedShape[distributedDim] =
+        distributedType.getDimSize(distributedDim);
+    auto newDistributedType =
+        VectorType::get(newDistributedShape, distributedType.getElementType());
+    SmallVector<size_t> newRetIndices;
+    WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndAppendReturns(
+        rewriter, warpOp, {extractOp.getVector()}, {newDistributedType},
+        newRetIndices);
+    rewriter.setInsertionPointAfter(newWarpOp);
+    SmallVector<Attribute> distributedSizes = llvm::map_to_vector(
+        extractOp.getSizes(), [](Attribute attr) { return attr; });
+    // Update the distributed sizes to match the distributed type.
+    if (distributedDim < static_cast<int64_t>(distributedSizes.size()))
+      distributedSizes[distributedDim] = rewriter.getI64IntegerAttr(
+          distributedType.getDimSize(distributedDim));
+
+    // Create a new extract strided slice op that extracts from the
+    // distributed vector.
+    Value distributedVec = newWarpOp->getResult(newRetIndices[0]);
+    Value newExtract = rewriter.create<vector::ExtractStridedSliceOp>(
+        extractOp.getLoc(), distributedType, distributedVec,
+        extractOp.getOffsets(),
+        ArrayAttr::get(rewriter.getContext(), distributedSizes),
+        extractOp.getStrides());
+    rewriter.replaceAllUsesWith(newWarpOp->getResult(operandNumber),
+                                newExtract);
+    return success();
+  }
+};
+
 /// Pattern to move out vector.extract of single element vector. Those don't
 /// need to be distributed and can just be propagated outside of the region.
 struct WarpOpExtract : public WarpDistributionPattern {
@@ -1122,15 +1334,7 @@ struct WarpOpExtract : public WarpDistributionPattern {
     auto distributedType =
         cast<VectorType>(warpOp.getResult(operandNumber).getType());
     auto yieldedType = cast<VectorType>(operand->get().getType());
-    int64_t distributedDim = -1;
-    for (int64_t i = 0; i < yieldedType.getRank(); ++i) {
-      if (distributedType.getDimSize(i) != yieldedType.getDimSize(i)) {
-        // Keep this assert here in case WarpExecuteOnLane0Op gets extended to
-        // support distributing multiple dimensions in the future.
-        assert(distributedDim == -1 && "found multiple distributed dims");
-        distributedDim = i;
-      }
-    }
+    int64_t distributedDim = getDistributedDim(yieldedType, distributedType);
     assert(distributedDim != -1 && "could not find distributed dimension");
     (void)distributedDim;
 
@@ -1764,7 +1968,8 @@ void mlir::vector::populatePropagateWarpVectorDistributionPatterns(
   patterns.add<WarpOpElementwise, WarpOpDeadResult, WarpOpBroadcast,
                WarpOpShapeCast, WarpOpExtract, WarpOpForwardOperand,
                WarpOpConstant, WarpOpExtractElement, WarpOpInsertElement,
-               WarpOpInsertScalar, WarpOpInsert, WarpOpCreateMask>(
+               WarpOpInsertScalar, WarpOpInsert, WarpOpCreateMask,
+               WarpOpExtractStridedSlice, WarpOpInsertStridedSlice>(
       patterns.getContext(), benefit);
   patterns.add<WarpOpExtractScalar>(patterns.getContext(), warpShuffleFromIdxFn,
                                     benefit);
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index 785a8aaf3f0a..efdae93e730b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -660,7 +660,8 @@ class FlattenContiguousRowMajorTransferReadPattern
     VectorType flatVectorType = VectorType::get({vectorType.getNumElements()},
                                                 vectorType.getElementType());
     vector::TransferReadOp flatRead = rewriter.create<vector::TransferReadOp>(
-        loc, flatVectorType, collapsedSource, collapsedIndices, collapsedMap);
+        loc, flatVectorType, collapsedSource, collapsedIndices,
+        transferReadOp.getPadding(), collapsedMap);
     flatRead.setInBoundsAttr(rewriter.getBoolArrayAttr({true}));
 
     // 4. Replace the old transfer_read with the new one reading from the
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 0c7cfc13f87d..674a93d7c520 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -321,8 +321,7 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
                                      ArrayRef<int64_t> inputVectorSizes,
                                      Value padValue,
                                      bool useInBoundsInsteadOfMasking) {
-  assert(llvm::none_of(inputVectorSizes,
-                       [](int64_t s) { return s == ShapedType::kDynamic; }) &&
+  assert(!llvm::is_contained(inputVectorSizes, ShapedType::kDynamic) &&
          "invalid input vector sizes");
   auto sourceShapedType = cast<ShapedType>(source.getType());
   auto sourceShape = sourceShapedType.getShape();
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index 6790c5e3af2c..7ef61de190b4 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -309,11 +310,23 @@ LogicalResult TensorDescType::verify(
     llvm::ArrayRef<int64_t> shape, mlir::Type elementType,
     mlir::Attribute encoding, mlir::Attribute layout) {
   size_t rank = shape.size();
-  // Low-precision types are packed in 32-bit units.
-  int32_t packingFactor = 32 / elementType.getIntOrFloatBitWidth();
   if (rank != 1 && rank != 2)
     return emitError() << "expected 1D or 2D tensor";
 
+  auto blockAttr = mlir::dyn_cast_if_present<BlockTensorDescAttr>(encoding);
+  if (blockAttr) {
+    MemorySpaceAttr memorySpaceAttr = blockAttr.getMemorySpace();
+    if (rank == 2 && memorySpaceAttr &&
+        memorySpaceAttr.getValue() == MemorySpace::SLM)
+      return emitError() << "SLM is not supported for 2D block tensor";
+  }
+
+  // for gather and scatter ops, Low-precision types are packed in 32-bit units.
+  unsigned bitWidth = elementType.getIntOrFloatBitWidth();
+  int chunkAlignmentFactor =
+      bitWidth < targetinfo::packedSizeInBitsForGatherScatter
+          ? targetinfo::packedSizeInBitsForGatherScatter / bitWidth
+          : 1;
   auto scatterAttr = mlir::dyn_cast_if_present<ScatterTensorDescAttr>(encoding);
   if (scatterAttr) {
     // Expected tensor ranks for scattered data:
@@ -329,21 +342,13 @@ LogicalResult TensorDescType::verify(
     if (chunkSize > 1) {
       if (shape.back() != chunkSize)
         return emitError() << "expected tensor shape[1] to match chunk size";
-      if (shape.back() % packingFactor != 0)
-        return emitError()
-               << "expected tensor shape[1] to be a multiple of packing factor "
-               << packingFactor;
+      if (shape.back() % chunkAlignmentFactor != 0)
+        return emitError() << "expected tensor shape[1] to be a multiple of "
+                              "chunk alignment factor "
+                           << chunkAlignmentFactor;
     }
   }
 
-  auto blockAttr = mlir::dyn_cast_if_present<BlockTensorDescAttr>(encoding);
-  if (blockAttr) {
-    MemorySpaceAttr memorySpaceAttr = blockAttr.getMemorySpace();
-    if (rank == 2 && memorySpaceAttr &&
-        memorySpaceAttr.getValue() == MemorySpace::SLM)
-      return emitError() << "SLM is not supported for 2D block tensor";
-  }
-
   auto layoutAttr = llvm::dyn_cast_if_present<LayoutAttr>(layout);
   if (layoutAttr) {
     if (rank != (size_t)layoutAttr.getRank())
@@ -360,7 +365,7 @@ LogicalResult TensorDescType::verify(
       if (rank > 1 && laneData[0] != 1)
         return emitError()
                << "cannot map over non-contiguous scattered row elements";
-      if (laneData[rank - 1] != packingFactor)
+      if (laneData[rank - 1] != chunkAlignmentFactor)
         return emitError() << "work item data mapping must match the number of "
                               "contiguous elements";
     }
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 0afc502c026f..3bea6cb50ff7 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -20,13 +20,6 @@
 namespace mlir {
 namespace xegpu {
 
-static void transpose(llvm::ArrayRef<int64_t> trans,
-                      SmallVector<int64_t> &shape) {
-  SmallVector<int64_t> old = shape;
-  for (size_t i = 0; i < trans.size(); i++)
-    shape[i] = old[trans[i]];
-}
-
 template <typename T>
 static std::string makeString(T array, bool breakline = false) {
   std::string buf;
@@ -76,7 +69,7 @@ static bool isWriteHintOrNone(const CachePolicyAttr &attr) {
 
 static LogicalResult
 isValidGatherScatterParams(Type maskTy, VectorType valueTy,
-                           TensorDescType tdescTy, UnitAttr transposeAttr,
+                           TensorDescType tdescTy,
                            function_ref<InFlightDiagnostic()> emitError) {
 
   if (!tdescTy.isScattered())
@@ -102,17 +95,9 @@ isValidGatherScatterParams(Type maskTy, VectorType valueTy,
   if (valueTy.getRank() == 1 && valueTy.getNumElements() == chunkSize) {
     if (tdescTy.getLayoutAttr())
       return emitError() << "TensorDesc doesn't need LayoutAttr for SIMT code";
-    if (transposeAttr)
-      return emitError() << "doesn't need TransposeAttr for SIMT code";
     return success();
   }
 
-  if (tdescTy.getRank() == 2 && valueTy.getRank() == 2) {
-    if (!transposeAttr)
-      return emitError() << "rank-2 tensor has to be transposed.";
-    transpose({1, 0}, tdescShape);
-  }
-
   if (tdescShape != valueShape)
     return emitError() << "Value shape " << makeString(valueShape)
                        << " is neither a valid distribution for SIMT nor "
@@ -150,8 +135,8 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
          shape.size() == strides.size() && shape.size() == offsets.size());
 
   Type srcTy = source.getType();
-  assert(isa<IntegerType>(srcTy) ||
-         isa<MemRefType>(srcTy) && "Source has to be either int or memref.");
+  assert((isa<IntegerType, MemRefType>(srcTy)) &&
+         "Source has to be either int or memref.");
 
   llvm::SmallVector<Value> dynamicOffsets;
   llvm::SmallVector<Value> dynamicShape;
@@ -310,13 +295,9 @@ LogicalResult LoadNdOp::verify() {
 
   if (getTranspose()) {
     auto trans = getTranspose().value();
-
-    // Make sure the transpose value is valid.
-    bool valid = llvm::all_of(
-        trans, [&](int t) { return t >= 0 && t < tdescTy.getRank(); });
-
-    if (valid)
-      transpose(trans, tdescShape);
+    // Make sure the transpose value is valid, and apply it
+    if (llvm::all_of(trans, [&](size_t s) { return s < tdescShape.size(); }))
+      tdescShape = applyPermutation(tdescShape, trans);
     else
       mlir::emitWarning(getLoc()) << "Invalid transpose attr. It is ignored.";
   }
@@ -536,7 +517,6 @@ LogicalResult LoadGatherOp::verify() {
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
   return isValidGatherScatterParams(maskTy, valueTy, tdescTy,
-                                    getTransposeAttr(),
                                     [&]() { return emitOpError(); });
 }
 
@@ -558,7 +538,6 @@ LogicalResult StoreScatterOp::verify() {
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
   return isValidGatherScatterParams(maskTy, valueTy, tdescTy,
-                                    getTransposeAttr(),
                                     [&]() { return emitOpError(); });
 }
 
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
index cc22d2bbd8c3..9558c08c079f 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h"
 #include "mlir/Dialect/XeGPU/Transforms/Passes.h"
 #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
 #include "mlir/IR/Attributes.h"
@@ -213,6 +214,37 @@ static LayoutInfo getDefaultSIMTLayoutInfo(VectorType vectorTy) {
                     LaneData({1, packingFactor}));
 }
 
+/// Helper to get the default layout for a vector type.
+static LayoutInfo getDefaultSIMTLayoutInfo(xegpu::TensorDescType tdescTy) {
+  // Expecting a 1D or 2D vector.
+  assert((tdescTy.getRank() == 1 || tdescTy.getRank() == 2) &&
+         "Expected 1D or 2D TensorDesc.");
+  // Expecting int or float element type.
+  assert(tdescTy.getElementType().isIntOrFloat() &&
+         "Expected int or float element type.");
+  // If the rank is 1, then return default layout for 1D vector.
+  if (tdescTy.getRank() == 1)
+    return getDefaultSIMTLayoutInfo(1);
+  // Packing factor is determined by the element type bitwidth.
+  unsigned bitwidth = tdescTy.getElementType().getIntOrFloatBitWidth();
+
+  if (tdescTy.isScattered()) {
+    int packingFactor =
+        bitwidth < xegpu::targetinfo::packedSizeInBitsForGatherScatter
+            ? xegpu::targetinfo::packedSizeInBitsForGatherScatter / bitwidth
+            : 1;
+    return LayoutInfo(LaneLayout({xegpu::targetinfo::subgroupSize, 1}),
+                      LaneData({1, packingFactor}));
+  }
+
+  int packingFactor =
+      (bitwidth < xegpu::targetinfo::packedSizeInBitsForDefault)
+          ? xegpu::targetinfo::packedSizeInBitsForDefault / bitwidth
+          : 1;
+  return LayoutInfo(LaneLayout({1, xegpu::targetinfo::subgroupSize}),
+                    LaneData({1, packingFactor}));
+}
+
 /// Helper Function to get the expected layouts for DPAS operands. `lane_data`
 /// is set according to the following criteria:
 /// * For A operand, the data must be packed in minimum
@@ -379,8 +411,7 @@ void LayoutInfoPropagation::visitPrefetchNdOp(
   // Here we assign the default layout to the tensor descriptor operand of
   // prefetch.
   auto tdescTy = prefetch.getTensorDescType();
-  auto prefetchLayout = getDefaultSIMTLayoutInfo(
-      VectorType::get(tdescTy.getShape(), tdescTy.getElementType()));
+  auto prefetchLayout = getDefaultSIMTLayoutInfo(tdescTy);
   // Propagate the layout to the source tensor descriptor.
   propagateIfChanged(operands[0], operands[0]->meet(prefetchLayout));
 }
@@ -516,24 +547,14 @@ void LayoutInfoPropagation::visitVectorBitcastOp(
 void LayoutInfoPropagation::visitLoadGatherOp(
     xegpu::LoadGatherOp load, ArrayRef<LayoutInfoLattice *> operands,
     ArrayRef<const LayoutInfoLattice *> results) {
-  LayoutInfo valueLayout = results[0]->getValue();
-  // Need the layout of the value to propagate to the tensor descriptor.
-  if (!valueLayout.isAssigned())
-    return;
+  // The layout is strictly determined by the tensor descriptor type.
+  LayoutInfo layout = getDefaultSIMTLayoutInfo(load.getTensorDescType());
 
-  LayoutInfo tensorDescLayout = valueLayout;
-  if (load.getTranspose()) {
-    // LoadGatherOp has the transpose effect. However, at the stage of this
-    // analyis this effect is not expected and should be abstracted away. Emit
-    // a warning.
-    load.emitWarning("Transpose effect is not expected for LoadGatherOp at "
-                     "LayoutInfoPropagation stage.");
-    tensorDescLayout = valueLayout.getTransposedLayout({1, 0});
-  }
   // Mask operand should have 1D default layout.
   LayoutInfo maskLayout = getDefaultSIMTLayoutInfo(1);
+
   // Propagate the new layout to the tensor descriptor operand.
-  propagateIfChanged(operands[0], operands[0]->meet(tensorDescLayout));
+  propagateIfChanged(operands[0], operands[0]->meet(layout));
   // Propagate the new layout to the mask operand.
   propagateIfChanged(operands[1], operands[1]->meet(maskLayout));
 }
@@ -567,21 +588,13 @@ void LayoutInfoPropagation::visitStoreScatterOp(
         "Expected the first dimension of 2D tensor descriptor to be equal to "
         "subgroup size.");
 
-  LayoutInfo valueLayout =
-      getDefaultSIMTLayoutInfo(storeScatter.getValueType());
-  LayoutInfo storeScatterLayout = valueLayout;
-  if (storeScatter.getTranspose()) {
-    // StoreScatteOp allows transpose effect. However, at the stage of this
-    // analyis this effect is not expected and should be abstracted away. Emit
-    // a warning.
-    storeScatter.emitWarning("Transpose effect is not expected for "
-                             "StoreScatterOp at LayoutInfoPropagation stage.");
-    storeScatterLayout = valueLayout.getTransposedLayout({1, 0});
-  }
+  LayoutInfo layout =
+      getDefaultSIMTLayoutInfo(storeScatter.getTensorDescType());
+
   // Propagate the value layout.
-  propagateIfChanged(operands[0], operands[0]->meet(valueLayout));
+  propagateIfChanged(operands[0], operands[0]->meet(layout));
   // Propagate the tensor descriptor layout.
-  propagateIfChanged(operands[1], operands[1]->meet(storeScatterLayout));
+  propagateIfChanged(operands[1], operands[1]->meet(layout));
   // Use default 1D layout for mask operand.
   LayoutInfo maskLayout = getDefaultSIMTLayoutInfo(1);
   propagateIfChanged(operands[2], operands[2]->meet(maskLayout));
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index dabcae0bfe4b..c072557c2bd2 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/Vector/Transforms/VectorDistribution.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h"
 #include "mlir/Dialect/XeGPU/Transforms/Passes.h"
 #include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
 #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
@@ -455,6 +456,14 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
     if (!operand)
       return rewriter.notifyMatchFailure(
           subgroupOp, "warp result is not a xegpu::LoadNd op");
+    // Make sure the load op is the last operation in the warp op body. This
+    // ensure that load op is not sinked earlier violating any barrier
+    // synchronizations.
+    auto yield = cast<gpu::YieldOp>(
+        subgroupOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    Operation *lastNode = yield->getPrevNode();
+    if (!dyn_cast_or_null<xegpu::LoadNdOp>(lastNode))
+      return failure();
 
     auto loadOp = operand->get().getDefiningOp<xegpu::LoadNdOp>();
     xegpu::TensorDescType tensorDescTy = loadOp.getTensorDescType();
@@ -782,6 +791,29 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
   }
 };
 
+/// Sink a gpu::BarrierOp at the end of enclosing `gpu.warp_execute_on_lane_0`
+/// region. This will simply move the barrier op outside of the warp op.
+struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
+  using gpu::WarpDistributionPattern::WarpDistributionPattern;
+  LogicalResult matchAndRewrite(gpu::WarpExecuteOnLane0Op subgroupOp,
+                                PatternRewriter &rewriter) const override {
+    auto yield = cast<gpu::YieldOp>(
+        subgroupOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    Operation *lastNode = yield->getPrevNode();
+    // The last node must be a gpu::BarrierOp.
+    auto barrierOp = dyn_cast_or_null<gpu::BarrierOp>(lastNode);
+    if (!barrierOp)
+      return failure();
+    // Move the barrier op outside of the warp op.
+    rewriter.setInsertionPointAfter(subgroupOp);
+    rewriter.create<gpu::BarrierOp>(
+        barrierOp.getLoc(), barrierOp->getResultTypes(),
+        barrierOp->getOperands(), barrierOp->getAttrs());
+    rewriter.eraseOp(barrierOp);
+    return success();
+  }
+};
+
 } // namespace
 
 namespace {
@@ -796,7 +828,8 @@ void xegpu::populateXeGPUSubgroupDistributePatterns(
     RewritePatternSet &patterns) {
   patterns.add<CreateNdDescDistribution, StoreNdDistribution,
                LoadNdDistribution, DpasDistribution, PrefetchNdDistribution,
-               UpdateNdOffsetDistribution>(patterns.getContext());
+               UpdateNdOffsetDistribution, GpuBarrierDistribution>(
+      patterns.getContext());
 }
 
 void XeGPUSubgroupDistributePass::runOnOperation() {
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
index 0457f8128b90..2c48a735bf95 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUUnroll.cpp
@@ -498,17 +498,14 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
     if (originalChunkSize > 1) {
       targetMaskShape.pop_back();
       convertedMaskTypes = getUnrolledTypes(maskTy, targetMaskShape);
-      SmallVector<Value> convertedMasks1D = pack(
-          op.getMask(), convertedMaskTypes, targetMaskShape, loc, rewriter);
       int64_t blockedChunkSize = targetShape->back();
       int64_t numNewChunks = originalChunkSize / blockedChunkSize;
 
-      for (auto mask : convertedMasks1D) {
-        for (int64_t i = 0; i < numNewChunks; ++i)
-          convertedMasks.push_back(mask);
-      }
-      // This is to handle the transpose effect when chunkSize > 1.
-      std::swap((*targetShape)[0], (*targetShape)[1]);
+      // the mask is reused across the chunk_size dimension
+      for (auto mask : pack(op.getMask(), convertedMaskTypes, targetMaskShape,
+                            loc, rewriter))
+        convertedMasks.append(numNewChunks, mask);
+
       newValueTy = valueTy.cloneWith(*targetShape, elemTy);
     } else {
       convertedMaskTypes = getUnrolledTypes(maskTy, targetMaskShape);
@@ -519,8 +516,8 @@ struct UnrollLoadGatherOp : public UnrollPattern<xegpu::LoadGatherOp> {
     SmallVector<Value> newOps;
     for (auto [t, m] : llvm::zip(convertedTdescs, convertedMasks)) {
       auto newOp = rewriter.create<xegpu::LoadGatherOp>(
-          loc, newValueTy, t, m, op.getTransposeAttr(), op.getL1HintAttr(),
-          op.getL2HintAttr(), op.getL3HintAttr());
+          loc, newValueTy, t, m, op.getL1HintAttr(), op.getL2HintAttr(),
+          op.getL3HintAttr());
       newOps.push_back(newOp);
     }
 
@@ -573,7 +570,7 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
     if (!targetShape)
       return failure();
 
-    SmallVector<int64_t> targetIndiceShape(*targetShape);
+    SmallVector<int64_t> targetMaskShape(*targetShape);
     int64_t originalChunkSize = tdescTy.getChunkSize();
 
     VectorType maskTy = llvm::dyn_cast<VectorType>(op.getMask().getType());
@@ -587,24 +584,19 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
     SmallVector<Value> convertedMasks;
 
     if (originalChunkSize > 1) {
+      targetMaskShape.pop_back();
       int64_t blockedChunkSize = targetShape->back();
       int64_t numNewChunks = originalChunkSize / blockedChunkSize;
-      convertedMaskTypes = getUnrolledTypes(maskTy, (*targetShape)[0]);
-      SmallVector<Value> convertedMasks1D = pack(
-          op.getMask(), convertedMaskTypes, (*targetShape)[0], loc, rewriter);
-
-      for (auto mask : convertedMasks1D) {
-        for (int64_t i = 0; i < numNewChunks; ++i) {
-          convertedMasks.push_back(mask);
-        }
-      }
-      // This is to handle the transpose effect when chunkSize > 1.
-      std::swap((*targetShape)[0], (*targetShape)[1]);
+      convertedMaskTypes = getUnrolledTypes(maskTy, targetMaskShape);
 
+      // the mask is reused across the chunk_size dimension
+      for (auto mask : pack(op.getMask(), convertedMaskTypes, targetMaskShape,
+                            loc, rewriter))
+        convertedMasks.append(numNewChunks, mask);
     } else {
-      convertedMaskTypes = getUnrolledTypes(maskTy, *targetShape);
-      convertedMasks =
-          pack(op.getMask(), convertedMaskTypes, *targetShape, loc, rewriter);
+      convertedMaskTypes = getUnrolledTypes(maskTy, targetMaskShape);
+      convertedMasks = pack(op.getMask(), convertedMaskTypes, targetMaskShape,
+                            loc, rewriter);
     }
 
     SmallVector<Type> convertedValTypes =
@@ -616,9 +608,9 @@ struct UnrollStoreScatterOp : public UnrollPattern<xegpu::StoreScatterOp> {
       Value v = convertedValues[i];
       Value t = convertedTdescs[i];
       Value m = op.getMask() ? convertedMasks[i] : nullptr;
-      rewriter.create<xegpu::StoreScatterOp>(
-          loc, v, t, m, op.getTransposeAttr(), op.getL1HintAttr(),
-          op.getL2HintAttr(), op.getL3HintAttr());
+      rewriter.create<xegpu::StoreScatterOp>(loc, v, t, m, op.getL1HintAttr(),
+                                             op.getL2HintAttr(),
+                                             op.getL3HintAttr());
     }
 
     rewriter.eraseOp(op);
@@ -655,20 +647,15 @@ struct UnrollUpdateOffsetOp : public UnrollPattern<xegpu::UpdateOffsetOp> {
     SmallVector<Value> newOps;
     int64_t originalChunkSize = tdescTy.getChunkSize();
     if (originalChunkSize > 1) {
-      SmallVector<int64_t> shape1D(targetShape->begin(),
-                                   targetShape->end() - 1);
-      convertedOffsetTypes = getUnrolledTypes(offsetVecTy, shape1D);
-      SmallVector<Value> convertedOffsetVec1D =
-          pack(offsetVec, convertedOffsetTypes, shape1D, loc, rewriter);
+      auto targetOffsetShape = ArrayRef<int64_t>(*targetShape).drop_back();
+      convertedOffsetTypes = getUnrolledTypes(offsetVecTy, targetOffsetShape);
 
       int64_t blockedChunkSize = targetShape->back();
       int64_t numNewChunks = originalChunkSize / blockedChunkSize;
-
-      for (auto offset : convertedOffsetVec1D) {
-        for (int64_t i = 0; i < numNewChunks; ++i) {
-          convertedOffsetVec.push_back(offset);
-        }
-      }
+      // the offset is reused across the chunk_size dimension
+      for (auto offset : pack(offsetVec, convertedOffsetTypes,
+                              targetOffsetShape, loc, rewriter))
+        convertedOffsetVec.append(numNewChunks, offset);
 
     } else {
       convertedOffsetTypes = getUnrolledTypes(offsetVecTy, *targetShape);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index e3563d10bc6f..be7b860dd172 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -376,10 +376,12 @@ struct WgToSgElementwiseOp : public ConversionPattern {
       // Copy all attributes, but update "layout_result_0" to drop
       // sgLayout/sgData
       for (auto attr : op->getAttrs()) {
-        if (auto layout = dyn_cast<xegpu::LayoutAttr>(attr.getValue()))
-          state.addAttribute(attr.getName(), layout.dropSgLayoutAndData());
-        else
+        if (auto layout = dyn_cast<xegpu::LayoutAttr>(attr.getValue())) {
+          if (auto newLayout = layout.dropSgLayoutAndData())
+            state.addAttribute(attr.getName(), newLayout);
+        } else {
           state.addAttribute(attr.getName(), attr.getValue());
+        }
       }
       Operation *newOp = rewriter.create(state);
       newResults.push_back(newOp->getResult(0));
@@ -629,8 +631,10 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
       std::string name = xegpu::getLayoutName(result);
       if (auto layout = op->getAttrOfType<xegpu::LayoutAttr>(name)) {
         op->removeAttr(name);
-        if (!isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::ConditionOp>(op))
-          op->setAttr(name, layout.dropSgLayoutAndData());
+        if (!isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::ConditionOp>(op)) {
+          if (auto newLayout = layout.dropSgLayoutAndData())
+            op->setAttr(name, newLayout);
+        }
       }
     }
   });
diff --git a/mlir/lib/IR/AffineExpr.cpp b/mlir/lib/IR/AffineExpr.cpp
index cc81f9d19aca..0652202fe331 100644
--- a/mlir/lib/IR/AffineExpr.cpp
+++ b/mlir/lib/IR/AffineExpr.cpp
@@ -781,9 +781,45 @@ static AffineExpr simplifyAdd(AffineExpr lhs, AffineExpr rhs) {
   if (isPositiveRhs && lhs == llrhs && rlrhs == -rrhs) {
     return lhs % rlrhs;
   }
+
+  // Try simplify lhs's last operand with rhs. e.g:
+  // (s0 * 64 + s1) + (s1 // c * -c) --->
+  // s0 * 64 + (s1 + s1 // c * -c) -->
+  // s0 * 64 + s1 % c
+  if (lBinOpExpr && lBinOpExpr.getKind() == AffineExprKind::Add) {
+    if (auto simplified = simplifyAdd(lBinOpExpr.getRHS(), rhs))
+      return lBinOpExpr.getLHS() + simplified;
+  }
   return nullptr;
 }
 
+/// Get the canonical order of two commutative exprs arguments.
+static std::pair<AffineExpr, AffineExpr>
+orderCommutativeArgs(AffineExpr expr1, AffineExpr expr2) {
+  auto sym1 = dyn_cast<AffineSymbolExpr>(expr1);
+  auto sym2 = dyn_cast<AffineSymbolExpr>(expr2);
+  // Try to order by symbol/dim position first.
+  if (sym1 && sym2)
+    return sym1.getPosition() < sym2.getPosition() ? std::pair{expr1, expr2}
+                                                   : std::pair{expr2, expr1};
+
+  auto dim1 = dyn_cast<AffineDimExpr>(expr1);
+  auto dim2 = dyn_cast<AffineDimExpr>(expr2);
+  if (dim1 && dim2)
+    return dim1.getPosition() < dim2.getPosition() ? std::pair{expr1, expr2}
+                                                   : std::pair{expr2, expr1};
+
+  // Put dims before symbols.
+  if (dim1 && sym2)
+    return {dim1, sym2};
+
+  if (sym1 && dim2)
+    return {dim2, sym1};
+
+  // Otherwise, keep original order.
+  return {expr1, expr2};
+}
+
 AffineExpr AffineExpr::operator+(int64_t v) const {
   return *this + getAffineConstantExpr(v, getContext());
 }
@@ -791,9 +827,11 @@ AffineExpr AffineExpr::operator+(AffineExpr other) const {
   if (auto simplified = simplifyAdd(*this, other))
     return simplified;
 
+  auto [lhs, rhs] = orderCommutativeArgs(*this, other);
+
   StorageUniquer &uniquer = getContext()->getAffineUniquer();
   return uniquer.get<AffineBinaryOpExprStorage>(
-      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Add), *this, other);
+      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Add), lhs, rhs);
 }
 
 /// Simplify a multiply expression. Return nullptr if it can't be simplified.
@@ -856,9 +894,11 @@ AffineExpr AffineExpr::operator*(AffineExpr other) const {
   if (auto simplified = simplifyMul(*this, other))
     return simplified;
 
+  auto [lhs, rhs] = orderCommutativeArgs(*this, other);
+
   StorageUniquer &uniquer = getContext()->getAffineUniquer();
   return uniquer.get<AffineBinaryOpExprStorage>(
-      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Mul), *this, other);
+      /*initFn=*/{}, static_cast<unsigned>(AffineExprKind::Mul), lhs, rhs);
 }
 
 // Unary minus, delegate to operator*.
@@ -1362,7 +1402,7 @@ LogicalResult SimpleAffineExprFlattener::visitModExpr(AffineBinaryOpExpr expr) {
       break;
   // If yes, modulo expression here simplifies to zero.
   if (i == lhs.size()) {
-    std::fill(lhs.begin(), lhs.end(), 0);
+    llvm::fill(lhs, 0);
     return success();
   }
 
@@ -1442,7 +1482,7 @@ LogicalResult SimpleAffineExprFlattener::addLocalVariableSemiAffine(
     if (failed(addLocalIdSemiAffine(lhs, rhs, localExpr)))
       return failure();
   }
-  std::fill(result.begin(), result.end(), 0);
+  llvm::fill(result, 0);
   if (loc == -1)
     result[getLocalVarStartIndex() + numLocals - 1] = 1;
   else
@@ -1529,7 +1569,7 @@ LogicalResult SimpleAffineExprFlattener::visitDivExpr(AffineBinaryOpExpr expr,
   }
   // Set the expression on stack to the local var introduced to capture the
   // result of the division (floor or ceil).
-  std::fill(lhs.begin(), lhs.end(), 0);
+  llvm::fill(lhs, 0);
   if (loc == -1)
     lhs[getLocalVarStartIndex() + numLocals - 1] = 1;
   else
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 2838843e2eba..7235b442aa56 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -14,12 +14,8 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MathExtras.h"
-#include "llvm/Support/raw_ostream.h"
-#include <iterator>
 #include <numeric>
 #include <optional>
 #include <type_traits>
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index c7cc6a02ad20..f95ad290a198 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -36,7 +36,6 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -442,6 +441,7 @@ class AsmPrinter::Impl {
   /// Print the given attribute without considering an alias.
   void printAttributeImpl(Attribute attr,
                           AttrTypeElision typeElision = AttrTypeElision::Never);
+  void printNamedAttribute(NamedAttribute attr);
 
   /// Print the alias for the given attribute, return failure if no alias could
   /// be printed.
@@ -481,7 +481,6 @@ class AsmPrinter::Impl {
   void printOptionalAttrDict(ArrayRef<NamedAttribute> attrs,
                              ArrayRef<StringRef> elidedAttrs = {},
                              bool withKeyword = false);
-  void printNamedAttribute(NamedAttribute attr);
   void printTrailingLocation(Location loc, bool allowAlias = true);
   void printLocationInternal(LocationAttr loc, bool pretty = false,
                              bool isTopLevel = false);
@@ -801,6 +800,10 @@ class DummyAliasOperationPrinter : private OpAsmPrinter {
   void printAttributeWithoutType(Attribute attr) override {
     printAttribute(attr);
   }
+  void printNamedAttribute(NamedAttribute attr) override {
+    printAttribute(attr.getValue());
+  }
+
   LogicalResult printAlias(Attribute attr) override {
     initializer.visit(attr);
     return success();
@@ -975,6 +978,10 @@ class DummyAliasDialectAsmPrinter : public DialectAsmPrinter {
     recordAliasResult(
         initializer.visit(attr, canBeDeferred, /*elideType=*/true));
   }
+  void printNamedAttribute(NamedAttribute attr) override {
+    printAttribute(attr.getValue());
+  }
+
   LogicalResult printAlias(Attribute attr) override {
     printAttribute(attr);
     return success();
@@ -2381,7 +2388,6 @@ void AsmPrinter::Impl::printAttribute(Attribute attr,
     return;
   return printAttributeImpl(attr, typeElision);
 }
-
 void AsmPrinter::Impl::printAttributeImpl(Attribute attr,
                                           AttrTypeElision typeElision) {
   if (!isa<BuiltinDialect>(attr.getDialect())) {
@@ -2973,6 +2979,11 @@ void AsmPrinter::printAttributeWithoutType(Attribute attr) {
   impl->printAttribute(attr, Impl::AttrTypeElision::Must);
 }
 
+void AsmPrinter::printNamedAttribute(NamedAttribute attr) {
+  assert(impl && "expected AsmPrinter::printNamedAttribute to be overriden");
+  impl->printNamedAttribute(attr);
+}
+
 void AsmPrinter::printKeywordOrString(StringRef keyword) {
   assert(impl && "expected AsmPrinter::printKeywordOrString to be overriden");
   ::printKeywordOrString(keyword, impl->getStream());
diff --git a/mlir/lib/IR/Block.cpp b/mlir/lib/IR/Block.cpp
index 959a60c04450..57825d9b4217 100644
--- a/mlir/lib/IR/Block.cpp
+++ b/mlir/lib/IR/Block.cpp
@@ -10,8 +10,6 @@
 
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Operation.h"
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index 5f7bc50afc41..f657db142eeb 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -12,11 +12,8 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/IRMapping.h"
-#include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/SymbolTable.h"
 #include "llvm/ADT/SmallVectorExtras.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index 617dcc222cd6..f9cc4d4a27aa 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -18,8 +18,6 @@
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include <optional>
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 2867d9b4ef68..6d7e2aa0ece7 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -17,9 +17,7 @@
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeRange.h"
 
 using namespace mlir;
diff --git a/mlir/lib/IR/BuiltinTypeInterfaces.cpp b/mlir/lib/IR/BuiltinTypeInterfaces.cpp
index c663f6c90946..ce076a3b4ac5 100644
--- a/mlir/lib/IR/BuiltinTypeInterfaces.cpp
+++ b/mlir/lib/IR/BuiltinTypeInterfaces.cpp
@@ -7,9 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/Diagnostics.h"
 #include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/Sequence.h"
 
 using namespace mlir;
 using namespace mlir::detail;
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index 6661efa8907b..a89276106ac1 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -17,9 +17,7 @@
 #include "mlir/IR/TensorEncoding.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/Twine.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
diff --git a/mlir/lib/IR/Diagnostics.cpp b/mlir/lib/IR/Diagnostics.cpp
index 3a0f81d24033..3e337951bcd3 100644
--- a/mlir/lib/IR/Diagnostics.cpp
+++ b/mlir/lib/IR/Diagnostics.cpp
@@ -727,7 +727,7 @@ SourceMgrDiagnosticVerifierHandlerImpl::computeExpectedDiags(
     raw_ostream &os, llvm::SourceMgr &mgr, const llvm::MemoryBuffer *buf) {
   // If the buffer is invalid, return an empty list.
   if (!buf)
-    return std::nullopt;
+    return {};
   auto &expectedDiags = expectedDiagsPerFile[buf->getBufferIdentifier()];
 
   // The number of the last line that did not correlate to a designator.
diff --git a/mlir/lib/IR/Dialect.cpp b/mlir/lib/IR/Dialect.cpp
index 47568594947c..f84fe8915707 100644
--- a/mlir/lib/IR/Dialect.cpp
+++ b/mlir/lib/IR/Dialect.cpp
@@ -17,12 +17,9 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/TypeID.h"
 #include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Regex.h"
 #include <memory>
 
diff --git a/mlir/lib/IR/Dominance.cpp b/mlir/lib/IR/Dominance.cpp
index 1c54e09d29b9..0e53b431b5d3 100644
--- a/mlir/lib/IR/Dominance.cpp
+++ b/mlir/lib/IR/Dominance.cpp
@@ -14,7 +14,6 @@
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/RegionKindInterface.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/GenericDomTreeConstruction.h"
 
 using namespace mlir;
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index d6febc39f451..c66b5460c1ce 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -25,11 +25,7 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/Types.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/mlir/lib/IR/ODSSupport.cpp b/mlir/lib/IR/ODSSupport.cpp
index 5b0a3e22139e..69b4a563e6b7 100644
--- a/mlir/lib/IR/ODSSupport.cpp
+++ b/mlir/lib/IR/ODSSupport.cpp
@@ -173,3 +173,8 @@ Attribute mlir::convertToAttribute(MLIRContext *ctx,
                                    ArrayRef<int64_t> storage) {
   return DenseI64ArrayAttr::get(ctx, storage);
 }
+
+Attribute mlir::convertToAttribute(MLIRContext *ctx,
+                                   ArrayRef<int32_t> storage) {
+  return DenseI32ArrayAttr::get(ctx, storage);
+}
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index a479ff97f3f1..8bcfa465e4a2 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -19,7 +19,6 @@
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <numeric>
 #include <optional>
@@ -810,13 +809,16 @@ void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
     ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
     llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
                                                   elidedProps.end());
-    bool atLeastOneAttr = llvm::any_of(attrs, [&](NamedAttribute attr) {
-      return !elidedAttrsSet.contains(attr.getName().strref());
-    });
-    if (atLeastOneAttr) {
-      p << "<";
-      p.printOptionalAttrDict(dictAttr.getValue(), elidedProps);
-      p << ">";
+    auto filteredAttrs =
+        llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
+          return !elidedAttrsSet.contains(attr.getName().strref());
+        });
+    if (!filteredAttrs.empty()) {
+      p << "<{";
+      interleaveComma(filteredAttrs, p, [&](NamedAttribute attr) {
+        p.printNamedAttribute(attr);
+      });
+      p << "}>";
     }
   } else {
     p << "<" << properties << ">";
diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp
index d3e9aebc4dc2..394ac7765ed6 100644
--- a/mlir/lib/IR/OperationSupport.cpp
+++ b/mlir/lib/IR/OperationSupport.cpp
@@ -15,7 +15,6 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/Support/SHA1.h"
 #include <numeric>
 #include <optional>
diff --git a/mlir/lib/IR/TypeRange.cpp b/mlir/lib/IR/TypeRange.cpp
index 26ffc0d5485d..88e788aa1b2b 100644
--- a/mlir/lib/IR/TypeRange.cpp
+++ b/mlir/lib/IR/TypeRange.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/TypeRange.h"
-#include "mlir/IR/Operation.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/IR/Unit.cpp b/mlir/lib/IR/Unit.cpp
index c109d387e6b7..58c1d27db87f 100644
--- a/mlir/lib/IR/Unit.cpp
+++ b/mlir/lib/IR/Unit.cpp
@@ -12,7 +12,6 @@
 #include "mlir/IR/Region.h"
 #include "llvm/Support/raw_ostream.h"
 #include <iterator>
-#include <sstream>
 
 using namespace mlir;
 
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index d1371d2b508f..dafec39fd5eb 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -829,7 +829,7 @@ void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
 
   // An atomic failure variable for the async executors.
   std::vector<std::atomic<bool>> activePMs(asyncExecutors.size());
-  std::fill(activePMs.begin(), activePMs.end(), false);
+  llvm::fill(activePMs, false);
   std::atomic<bool> hasFailure = false;
   parallelForEach(context, opInfos, [&](OpPMInfo &opInfo) {
     // Find an executor for this operation.
diff --git a/mlir/lib/Query/Matcher/VariantValue.cpp b/mlir/lib/Query/Matcher/VariantValue.cpp
index 7bf4774dba83..98ed4cc5df31 100644
--- a/mlir/lib/Query/Matcher/VariantValue.cpp
+++ b/mlir/lib/Query/Matcher/VariantValue.cpp
@@ -172,11 +172,11 @@ void VariantValue::setSigned(int64_t newValue) {
 // Boolean
 bool VariantValue::isBoolean() const { return type == ValueType::Boolean; }
 
-bool VariantValue::getBoolean() const { return value.Signed; }
+bool VariantValue::getBoolean() const { return value.Boolean; }
 
 void VariantValue::setBoolean(bool newValue) {
   type = ValueType::Boolean;
-  value.Signed = newValue;
+  value.Boolean = newValue;
 }
 
 bool VariantValue::isString() const { return type == ValueType::String; }
diff --git a/mlir/lib/Reducer/Tester.cpp b/mlir/lib/Reducer/Tester.cpp
index 14760c0be70d..03f12af17483 100644
--- a/mlir/lib/Reducer/Tester.cpp
+++ b/mlir/lib/Reducer/Tester.cpp
@@ -68,7 +68,7 @@ Tester::Interestingness Tester::isInteresting(StringRef testCase) const {
 
   std::string errMsg;
   int result = llvm::sys::ExecuteAndWait(
-      testScript, testerArgs, /*Env=*/std::nullopt, /*Redirects=*/std::nullopt,
+      testScript, testerArgs, /*Env=*/std::nullopt, /*Redirects=*/{},
       /*SecondsToWait=*/0, /*MemoryLimit=*/0, &errMsg);
 
   if (result < 0)
diff --git a/mlir/lib/Support/CMakeLists.txt b/mlir/lib/Support/CMakeLists.txt
index 488decd52ae6..02b6c694a28f 100644
--- a/mlir/lib/Support/CMakeLists.txt
+++ b/mlir/lib/Support/CMakeLists.txt
@@ -11,6 +11,7 @@ add_mlir_library(MLIRSupport
   FileUtilities.cpp
   InterfaceSupport.cpp
   RawOstreamExtras.cpp
+  StateStack.cpp
   StorageUniquer.cpp
   Timing.cpp
   ToolUtilities.cpp
diff --git a/pstl/include/__pstl_memory b/mlir/lib/Support/StateStack.cpp
similarity index 61%
rename from pstl/include/__pstl_memory
rename to mlir/lib/Support/StateStack.cpp
index 12b7f5aa3c04..a9bb3ffb2e1b 100644
--- a/pstl/include/__pstl_memory
+++ b/mlir/lib/Support/StateStack.cpp
@@ -1,5 +1,4 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
+//===- StateStack.cpp - Utility for storing a stack of state --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __PSTL_MEMORY
-#define __PSTL_MEMORY
+#include "mlir/Support/StateStack.h"
+
+namespace mlir {
 
-#include <pstl/internal/glue_memory_impl.h>
+void StateStackFrame::anchor() {}
 
-#endif /* __PSTL_MEMORY */
+} // namespace mlir
diff --git a/mlir/lib/TableGen/Argument.cpp b/mlir/lib/TableGen/Argument.cpp
index c847760f8a46..8dec21d9a354 100644
--- a/mlir/lib/TableGen/Argument.cpp
+++ b/mlir/lib/TableGen/Argument.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Argument.h"
-#include "llvm/TableGen/Record.h"
 
 using namespace mlir;
 using namespace mlir::tblgen;
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index 142d19426094..d53ccea576b8 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -11,7 +11,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/Operator.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Class.cpp b/mlir/lib/TableGen/Class.cpp
index fedf64fd96b0..c65f67d50a47 100644
--- a/mlir/lib/TableGen/Class.cpp
+++ b/mlir/lib/TableGen/Class.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Class.h"
-#include "mlir/TableGen/Format.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Debug.h"
 
diff --git a/mlir/lib/TableGen/Interfaces.cpp b/mlir/lib/TableGen/Interfaces.cpp
index 29d8497aacff..ec7adf3b02c2 100644
--- a/mlir/lib/TableGen/Interfaces.cpp
+++ b/mlir/lib/TableGen/Interfaces.cpp
@@ -8,9 +8,7 @@
 
 #include "mlir/TableGen/Interfaces.h"
 #include "llvm/ADT/FunctionExtras.h"
-#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 07520a2f94d7..da86b00ebc30 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -15,7 +15,6 @@
 #include "mlir/TableGen/Predicate.h"
 #include "mlir/TableGen/Trait.h"
 #include "mlir/TableGen/Type.h"
-#include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -26,7 +25,6 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
-#include <list>
 
 #define DEBUG_TYPE "mlir-tblgen-operator"
 
diff --git a/mlir/lib/TableGen/Predicate.cpp b/mlir/lib/TableGen/Predicate.cpp
index 11afa27caf70..a48c7468251e 100644
--- a/mlir/lib/TableGen/Predicate.cpp
+++ b/mlir/lib/TableGen/Predicate.cpp
@@ -14,7 +14,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/TableGen/Property.cpp b/mlir/lib/TableGen/Property.cpp
index 47f43267cd19..b003d74c7bde 100644
--- a/mlir/lib/TableGen/Property.cpp
+++ b/mlir/lib/TableGen/Property.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/TableGen/Property.h"
-#include "mlir/TableGen/Format.h"
 #include "mlir/TableGen/Operator.h"
 #include "mlir/TableGen/Predicate.h"
 #include "llvm/TableGen/Record.h"
diff --git a/mlir/lib/TableGen/Trait.cpp b/mlir/lib/TableGen/Trait.cpp
index 6246ba959b00..8fb8c3472f43 100644
--- a/mlir/lib/TableGen/Trait.cpp
+++ b/mlir/lib/TableGen/Trait.cpp
@@ -13,8 +13,6 @@
 #include "mlir/TableGen/Trait.h"
 #include "mlir/TableGen/Interfaces.h"
 #include "mlir/TableGen/Predicate.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/FormatVariadic.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 067a0470b14e..ab91b32d7ef4 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -997,6 +997,49 @@ static LogicalResult printOperation(CppEmitter &emitter, ModuleOp moduleOp) {
   return success();
 }
 
+static LogicalResult printOperation(CppEmitter &emitter, ClassOp classOp) {
+  CppEmitter::Scope classScope(emitter);
+  raw_indented_ostream &os = emitter.ostream();
+  os << "class " << classOp.getSymName();
+  if (classOp.getFinalSpecifier())
+    os << " final";
+  os << " {\n public:\n";
+  os.indent();
+
+  for (Operation &op : classOp) {
+    if (failed(emitter.emitOperation(op, /*trailingSemicolon=*/false)))
+      return failure();
+  }
+
+  os.unindent();
+  os << "};";
+  return success();
+}
+
+static LogicalResult printOperation(CppEmitter &emitter, FieldOp fieldOp) {
+  raw_ostream &os = emitter.ostream();
+  if (failed(emitter.emitType(fieldOp->getLoc(), fieldOp.getType())))
+    return failure();
+  os << " " << fieldOp.getSymName() << ";";
+  return success();
+}
+
+static LogicalResult printOperation(CppEmitter &emitter,
+                                    GetFieldOp getFieldOp) {
+  raw_indented_ostream &os = emitter.ostream();
+
+  Value result = getFieldOp.getResult();
+  if (failed(emitter.emitType(getFieldOp->getLoc(), result.getType())))
+    return failure();
+  os << " ";
+  if (failed(emitter.emitOperand(result)))
+    return failure();
+  os << " = ";
+
+  os << getFieldOp.getFieldName().str();
+  return success();
+}
+
 static LogicalResult printOperation(CppEmitter &emitter, FileOp file) {
   if (!emitter.shouldEmitFile(file))
     return success();
@@ -1605,14 +1648,16 @@ LogicalResult CppEmitter::emitOperation(Operation &op, bool trailingSemicolon) {
                 emitc::BitwiseAndOp, emitc::BitwiseLeftShiftOp,
                 emitc::BitwiseNotOp, emitc::BitwiseOrOp,
                 emitc::BitwiseRightShiftOp, emitc::BitwiseXorOp, emitc::CallOp,
-                emitc::CallOpaqueOp, emitc::CastOp, emitc::CmpOp,
-                emitc::ConditionalOp, emitc::ConstantOp, emitc::DeclareFuncOp,
-                emitc::DivOp, emitc::ExpressionOp, emitc::FileOp, emitc::ForOp,
-                emitc::FuncOp, emitc::GlobalOp, emitc::IfOp, emitc::IncludeOp,
-                emitc::LoadOp, emitc::LogicalAndOp, emitc::LogicalNotOp,
-                emitc::LogicalOrOp, emitc::MulOp, emitc::RemOp, emitc::ReturnOp,
-                emitc::SubOp, emitc::SwitchOp, emitc::UnaryMinusOp,
-                emitc::UnaryPlusOp, emitc::VariableOp, emitc::VerbatimOp>(
+                emitc::CallOpaqueOp, emitc::CastOp, emitc::ClassOp,
+                emitc::CmpOp, emitc::ConditionalOp, emitc::ConstantOp,
+                emitc::DeclareFuncOp, emitc::DivOp, emitc::ExpressionOp,
+                emitc::FieldOp, emitc::FileOp, emitc::ForOp, emitc::FuncOp,
+                emitc::GetFieldOp, emitc::GlobalOp, emitc::IfOp,
+                emitc::IncludeOp, emitc::LoadOp, emitc::LogicalAndOp,
+                emitc::LogicalNotOp, emitc::LogicalOrOp, emitc::MulOp,
+                emitc::RemOp, emitc::ReturnOp, emitc::SubOp, emitc::SwitchOp,
+                emitc::UnaryMinusOp, emitc::UnaryPlusOp, emitc::VariableOp,
+                emitc::VerbatimOp>(
 
               [&](auto op) { return printOperation(*this, op); })
           // Func ops.
diff --git a/mlir/lib/Target/LLVM/ModuleToObject.cpp b/mlir/lib/Target/LLVM/ModuleToObject.cpp
index 04e870acfc5f..5055cd9938c7 100644
--- a/mlir/lib/Target/LLVM/ModuleToObject.cpp
+++ b/mlir/lib/Target/LLVM/ModuleToObject.cpp
@@ -16,8 +16,6 @@
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
@@ -28,7 +26,6 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 565f153b50c4..b4d53d403cb2 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -16,10 +16,7 @@
 #include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/IR/BuiltinAttributeInterfaces.h"
-#include "mlir/IR/BuiltinDialect.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/DialectResourceBlobManager.h"
 #include "mlir/Target/LLVM/NVVM/Utils.h"
 #include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
diff --git a/mlir/lib/Target/LLVM/ROCDL/Utils.cpp b/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
index 04b1b22279e5..abcc3ddcda46 100644
--- a/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
+++ b/mlir/lib/Target/LLVM/ROCDL/Utils.cpp
@@ -12,7 +12,6 @@
 
 #include "mlir/Target/LLVM/ROCDL/Utils.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
-#include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Frontend/Offloading/Utility.h"
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
index 5f5ca2af277c..fbad5c2fb78d 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
@@ -8,7 +8,6 @@
 
 #include "DataLayoutImporter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 779c1d12720c..69ae29f23ba2 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -12,15 +12,11 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Location.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/ErrorHandling.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index 3c289eede8a1..4e44c3eaf534 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -221,7 +221,9 @@ llvm::DIFile *DebugTranslation::translateImpl(DIFileAttr attr) {
 llvm::DILabel *DebugTranslation::translateImpl(DILabelAttr attr) {
   return llvm::DILabel::get(llvmCtx, translate(attr.getScope()),
                             getMDStringOrNull(attr.getName()),
-                            translate(attr.getFile()), attr.getLine());
+                            translate(attr.getFile()), attr.getLine(),
+                            /*Column=*/0, /*IsArtificial=*/false,
+                            /*CoroSuspendIdx=*/std::nullopt);
 }
 
 llvm::DILexicalBlock *DebugTranslation::translateImpl(DILexicalBlockAttr attr) {
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
index e67aa892afe0..8d5d7f9b649f 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMIRToLLVMTranslation.cpp
@@ -16,15 +16,11 @@
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
-#include "llvm/ADT/PostOrderIterator.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
-#include "llvm/Support/ModRef.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index ba87d1ad0dd8..70029d7e15a9 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -22,7 +22,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
-#include "llvm/IR/Operator.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
index 2c0b665ad0d8..ad01a64c6d3d 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/LLVMIRToNVVMTranslation.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
 
 using namespace mlir;
 using namespace mlir::NVVM;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 32b9144233f0..eecca64c4bf8 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -13,7 +13,6 @@
 
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
-#include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
index e9ef63aedaf4..87e77945b7ef 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenACC/OpenACCToLLVMIRTranslation.cpp
@@ -15,7 +15,6 @@
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"
@@ -23,7 +22,6 @@
 
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
-#include "llvm/Support/FormatVariadic.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 025a3d8b9ee1..5b5715a0b057 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -75,7 +75,7 @@ convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
 /// ModuleTranslation stack frame for OpenMP operations. This keeps track of the
 /// insertion points for allocas.
 class OpenMPAllocaStackFrame
-    : public LLVM::ModuleTranslation::StackFrameBase<OpenMPAllocaStackFrame> {
+    : public StateStackFrameBase<OpenMPAllocaStackFrame> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocaStackFrame)
 
@@ -88,7 +88,7 @@ class OpenMPAllocaStackFrame
 /// collapsed canonical loop information corresponding to an \c omp.loop_nest
 /// operation.
 class OpenMPLoopInfoStackFrame
-    : public LLVM::ModuleTranslation::StackFrameBase<OpenMPLoopInfoStackFrame> {
+    : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
   llvm::CanonicalLoopInfo *loopInfo = nullptr;
@@ -372,7 +372,13 @@ findAllocaInsertPoint(llvm::IRBuilderBase &builder,
         allocaInsertPoint = frame.allocaInsertPoint;
         return WalkResult::interrupt();
       });
-  if (walkResult.wasInterrupted())
+  // In cases with multiple levels of outlining, the tree walk might find an
+  // alloca insertion point that is inside the original function while the
+  // builder insertion point is inside the outlined function. We need to make
+  // sure that we do not use it in those cases.
+  if (walkResult.wasInterrupted() &&
+      allocaInsertPoint.getBlock()->getParent() ==
+          builder.GetInsertBlock()->getParent())
     return allocaInsertPoint;
 
   // Otherwise, insert to the entry block of the surrounding function.
@@ -4461,6 +4467,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   llvm::OpenMPIRBuilder::TargetDataInfo info(/*RequiresDevicePointerInfo=*/true,
                                              /*SeparateBeginEndCalls=*/true);
+  bool isTargetDevice = ompBuilder->Config.isTargetDevice();
+  bool isOffloadEntry =
+      isTargetDevice || !ompBuilder->Config.TargetTriples.empty();
 
   LogicalResult result =
       llvm::TypeSwitch<Operation *, LogicalResult>(op)
@@ -4550,6 +4559,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
 
   if (failed(result))
     return failure();
+  // Pretend we have IF(false) if we're not doing offload.
+  if (!isOffloadEntry)
+    ifCond = builder.getFalse();
 
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   MapInfoData mapData;
@@ -4602,6 +4614,8 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
   using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
   auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType)
       -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+    // We must always restoreIP regardless of doing anything the caller
+    // does not restore it, leading to incorrect (no) branch generation.
     builder.restoreIP(codeGenIP);
     assert(isa<omp::TargetDataOp>(op) &&
            "BodyGen requested for non TargetDataOp");
@@ -4638,9 +4652,18 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
       }
       break;
     case BodyGenTy::DupNoPriv:
-      // We must always restoreIP regardless of doing anything the caller
-      // does not restore it, leading to incorrect (no) branch generation.
-      builder.restoreIP(codeGenIP);
+      if (info.DevicePtrInfoMap.empty()) {
+        // For host device we still need to do the mapping for codegen,
+        // otherwise it may try to lookup a missing value.
+        if (!ompBuilder->Config.IsTargetDevice.value_or(false)) {
+          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,
+                       blockArgIface.getUseDeviceAddrBlockArgs(),
+                       useDeviceAddrVars, mapData);
+          mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,
+                       blockArgIface.getUseDevicePtrBlockArgs(),
+                       useDevicePtrVars, mapData);
+        }
+      }
       break;
     case BodyGenTy::NoPriv:
       // If device info is available then region has already been generated
@@ -4902,6 +4925,7 @@ handleDeclareTargetMapVar(MapInfoData &mapData,
                           llvm::IRBuilderBase &builder, llvm::Function *func) {
   assert(moduleTranslation.getOpenMPBuilder()->Config.isTargetDevice() &&
          "function only supported for target device codegen");
+  llvm::IRBuilderBase::InsertPointGuard guard(builder);
   for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
     // In the case of declare target mapped variables, the basePointer is
     // the reference pointer generated by the convertDeclareTargetAttr
@@ -4940,6 +4964,7 @@ handleDeclareTargetMapVar(MapInfoData &mapData,
             llvm::Value *substitute = mapData.BasePointers[i];
             if (isDeclareTargetLink(mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr()
                                                          : mapOp.getVarPtr())) {
+              builder.SetCurrentDebugLocation(insn->getDebugLoc());
               auto *load = builder.CreateLoad(
                   mapData.BasePointers[i]->getType(), mapData.BasePointers[i]);
               load->moveBefore(insn);
@@ -5399,9 +5424,27 @@ static LogicalResult
 convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
                  LLVM::ModuleTranslation &moduleTranslation) {
   auto targetOp = cast<omp::TargetOp>(opInst);
+  // The current debug location already has the DISubprogram for the outlined
+  // function that will be created for the target op. We save it here so that
+  // we can set it on the outlined function.
+  llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
   if (failed(checkImplementationStatus(opInst)))
     return failure();
 
+  // During the handling of target op, we will generate instructions in the
+  // parent function like call to the oulined function or branch to a new
+  // BasicBlock. We set the debug location here to parent function so that those
+  // get the correct debug locations. For outlined functions, the normal MLIR op
+  // conversion will automatically pick the correct location.
+  llvm::BasicBlock *parentBB = builder.GetInsertBlock();
+  assert(parentBB && "No insert block is set for the builder");
+  llvm::Function *parentLLVMFn = parentBB->getParent();
+  assert(parentLLVMFn && "Parent Function must be valid");
+  if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
+    builder.SetCurrentDebugLocation(llvm::DILocation::get(
+        parentLLVMFn->getContext(), outlinedFnLoc.getLine(),
+        outlinedFnLoc.getCol(), SP, outlinedFnLoc.getInlinedAt()));
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   bool isTargetDevice = ompBuilder->Config.isTargetDevice();
   bool isGPU = ompBuilder->Config.isGPU();
@@ -5496,6 +5539,9 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
     assert(llvmParentFn && llvmOutlinedFn &&
            "Both parent and outlined functions must exist at this point");
 
+    if (outlinedFnLoc && llvmParentFn->getSubprogram())
+      llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
+
     if (auto attr = llvmParentFn->getFnAttribute("target-cpu");
         attr.isStringAttribute())
       llvmOutlinedFn->addFnAttr(attr);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
index 638edca5efde..3cacbc4b3646 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.cpp
@@ -13,8 +13,6 @@
 
 #include "mlir/Target/LLVMIR/Dialect/SPIRV/SPIRVToLLVMIRTranslation.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
-#include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/Operation.h"
 #include "mlir/Target/LLVMIR/ModuleTranslation.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
index b78b002d3229..4bf787faef8c 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/VCIX/VCIXToLLVMIRTranslation.cpp
@@ -19,7 +19,6 @@
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsRISCV.h"
-#include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
 using namespace mlir::LLVM;
diff --git a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
index e353d11c0d0a..4e7f1d318512 100644
--- a/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
+++ b/mlir/lib/Target/LLVMIR/LLVMImportInterface.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Target/LLVMIR/LLVMImportInterface.h"
-#include "mlir/Target/LLVMIR/Import.h"
 #include "mlir/Target/LLVMIR/ModuleImport.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
index 54ae01c77c44..f3ec18c9074e 100644
--- a/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp
@@ -206,7 +206,7 @@ void LoopAnnotationConversion::convertLocation(FusedLoc location) {
 
 llvm::MDNode *LoopAnnotationConversion::convert() {
   // Reserve operand 0 for loop id self reference.
-  auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
+  auto dummy = llvm::MDNode::getTemporary(ctx, {});
   metadataNodes.push_back(dummy.get());
 
   if (FusedLoc startLoc = attr.getStartLoc())
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 803ecb39f4da..bfda223fe0f5 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -30,7 +30,6 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constants.h"
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 3eaa24eb5c95..8908703cc136 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -22,21 +22,16 @@
 #include "mlir/Dialect/LLVMIR/LLVMInterfaces.h"
 #include "mlir/Dialect/LLVMIR/Transforms/DIExpressionLegalization.h"
 #include "mlir/Dialect/LLVMIR/Transforms/LegalizeForExport.h"
-#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
-#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
 #include "mlir/IR/AttrTypeSubElements.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectResourceBlobManager.h"
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
-#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Analysis/TargetFolder.h"
@@ -47,7 +42,6 @@
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
@@ -727,10 +721,8 @@ llvm::Constant *mlir::LLVM::detail::getLLVMConstant(
   }
 
   if (auto stringAttr = dyn_cast<StringAttr>(attr)) {
-    return llvm::ConstantDataArray::get(
-        moduleTranslation.getLLVMContext(),
-        ArrayRef<char>{stringAttr.getValue().data(),
-                       stringAttr.getValue().size()});
+    return llvm::ConstantDataArray::get(moduleTranslation.getLLVMContext(),
+                                        ArrayRef<char>{stringAttr.getValue()});
   }
 
   // Handle arrays of structs that cannot be represented as DenseElementsAttr
@@ -777,7 +769,7 @@ ModuleTranslation::ModuleTranslation(Operation *module,
 }
 
 ModuleTranslation::~ModuleTranslation() {
-  if (ompBuilder)
+  if (ompBuilder && !ompBuilder->isFinalized())
     ompBuilder->finalize();
 }
 
@@ -1922,7 +1914,7 @@ ModuleTranslation::getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr) {
   if (!scopeInserted)
     return scopeIt->second;
   llvm::LLVMContext &ctx = llvmModule->getContext();
-  auto dummy = llvm::MDNode::getTemporary(ctx, std::nullopt);
+  auto dummy = llvm::MDNode::getTemporary(ctx, {});
   // Convert the domain metadata node if necessary.
   auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(
       aliasScopeAttr.getDomain(), nullptr);
@@ -2225,8 +2217,6 @@ ModuleTranslation::getOrInsertNamedModuleMetadata(StringRef name) {
   return llvmModule->getOrInsertNamedMetadata(name);
 }
 
-void ModuleTranslation::StackFrame::anchor() {}
-
 static std::unique_ptr<llvm::Module>
 prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext,
                   StringRef name) {
@@ -2333,6 +2323,10 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
   // beforehand.
   translator.debugTranslation->addModuleFlagsIfNotPresent();
 
+  // Call the OpenMP IR Builder callbacks prior to verifying the module
+  if (auto *ompBuilder = translator.getOpenMPBuilder())
+    ompBuilder->finalize();
+
   if (!disableVerification &&
       llvm::verifyModule(*translator.llvmModule, &llvm::errs()))
     return nullptr;
diff --git a/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp b/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
index af78dcd16f79..e4ba478f1d3b 100644
--- a/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
+++ b/mlir/lib/Target/LLVMIR/TypeToLLVM.cpp
@@ -9,7 +9,6 @@
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/MLIRContext.h"
 
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/IR/DataLayout.h"
diff --git a/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp b/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
index 47235c4a5a52..e6d8e34ea737 100644
--- a/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
+++ b/mlir/lib/Target/SMTLIB/ExportSMTLIB.cpp
@@ -21,7 +21,6 @@
 #include "mlir/Tools/mlir-translate/Translation.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index b30da773d489..55d6a380d0bf 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -164,6 +164,7 @@ LogicalResult spirv::Deserializer::processInstruction(
   case spirv::Opcode::OpTypeRuntimeArray:
   case spirv::Opcode::OpTypeStruct:
   case spirv::Opcode::OpTypePointer:
+  case spirv::Opcode::OpTypeTensorARM:
   case spirv::Opcode::OpTypeCooperativeMatrixKHR:
     return processType(opcode, operands);
   case spirv::Opcode::OpTypeForwardPointer:
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index b9d9a9015eb6..b1abd8b3dffe 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -935,6 +935,8 @@ LogicalResult spirv::Deserializer::processType(spirv::Opcode opcode,
     return processStructType(operands);
   case spirv::Opcode::OpTypeMatrix:
     return processMatrixType(operands);
+  case spirv::Opcode::OpTypeTensorARM:
+    return processTensorARMType(operands);
   default:
     return emitError(unknownLoc, "unhandled type instruction");
   }
@@ -1238,6 +1240,55 @@ spirv::Deserializer::processMatrixType(ArrayRef<uint32_t> operands) {
   return success();
 }
 
+LogicalResult
+spirv::Deserializer::processTensorARMType(ArrayRef<uint32_t> operands) {
+  unsigned size = operands.size();
+  if (size < 2 || size > 4)
+    return emitError(unknownLoc, "OpTypeTensorARM must have 2-4 operands "
+                                 "(result_id, element_type, (rank), (shape)) ")
+           << size;
+
+  Type elementTy = getType(operands[1]);
+  if (!elementTy)
+    return emitError(unknownLoc,
+                     "OpTypeTensorARM references undefined element type ")
+           << operands[1];
+
+  if (size == 2) {
+    typeMap[operands[0]] = TensorArmType::get({}, elementTy);
+    return success();
+  }
+
+  IntegerAttr rankAttr = getConstantInt(operands[2]);
+  if (!rankAttr)
+    return emitError(unknownLoc, "OpTypeTensorARM rank must come from a "
+                                 "scalar integer constant instruction");
+  unsigned rank = rankAttr.getValue().getZExtValue();
+  if (size == 3) {
+    SmallVector<int64_t, 4> shape(rank, ShapedType::kDynamic);
+    typeMap[operands[0]] = TensorArmType::get(shape, elementTy);
+    return success();
+  }
+
+  std::optional<std::pair<Attribute, Type>> shapeInfo =
+      getConstant(operands[3]);
+  if (!shapeInfo)
+    return emitError(unknownLoc, "OpTypeTensorARM shape must come from a "
+                                 "constant instruction of type OpTypeArray");
+
+  ArrayAttr shapeArrayAttr = llvm::dyn_cast<ArrayAttr>(shapeInfo->first);
+  SmallVector<int64_t, 1> shape;
+  for (auto dimAttr : shapeArrayAttr.getValue()) {
+    auto dimIntAttr = llvm::dyn_cast<IntegerAttr>(dimAttr);
+    if (!dimIntAttr)
+      return emitError(unknownLoc, "OpTypeTensorARM shape has an invalid "
+                                   "dimension size");
+    shape.push_back(dimIntAttr.getValue().getSExtValue());
+  }
+  typeMap[operands[0]] = TensorArmType::get(shape, elementTy);
+  return success();
+}
+
 LogicalResult
 spirv::Deserializer::processTypeForwardPointer(ArrayRef<uint32_t> operands) {
   if (operands.size() != 2)
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
index e4556e7652b1..1bc9e4a3c75d 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.h
@@ -291,6 +291,8 @@ class Deserializer {
 
   LogicalResult processMatrixType(ArrayRef<uint32_t> operands);
 
+  LogicalResult processTensorARMType(ArrayRef<uint32_t> operands);
+
   LogicalResult processTypeForwardPointer(ArrayRef<uint32_t> operands);
 
   //===--------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
index 0ec468d4c166..cbd1d4255e99 100644
--- a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
+++ b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Target/SPIRV/SPIRVBinaryUtils.h"
-#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_VERSION_MAJOR
 #include "llvm/Support/Debug.h"
 
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp b/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
index 80d1bb85611d..f8527908b83e 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serialization.cpp
@@ -15,7 +15,6 @@
 
 #include "mlir/Target/SPIRV/Serialization.h"
 
-#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "spirv-serialization"
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index d258bfd85296..ebebd2d283af 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -729,6 +729,54 @@ LogicalResult Serializer::prepareBasicType(
     return success();
   }
 
+  if (auto tensorArmType = llvm::dyn_cast<TensorArmType>(type)) {
+    uint32_t elementTypeID = 0;
+    uint32_t rank = 0;
+    uint32_t shapeID = 0;
+    uint32_t rankID = 0;
+    if (failed(processTypeImpl(loc, tensorArmType.getElementType(),
+                               elementTypeID, serializationCtx))) {
+      return failure();
+    }
+    if (tensorArmType.hasRank()) {
+      ArrayRef<int64_t> dims = tensorArmType.getShape();
+      rank = dims.size();
+      rankID = prepareConstantInt(loc, mlirBuilder.getI32IntegerAttr(rank));
+      if (rankID == 0) {
+        return failure();
+      }
+
+      bool shaped = llvm::all_of(dims, [](const auto &dim) { return dim > 0; });
+      if (rank > 0 && shaped) {
+        auto I32Type = IntegerType::get(type.getContext(), 32);
+        auto shapeType = ArrayType::get(I32Type, rank);
+        if (rank == 1) {
+          SmallVector<uint64_t, 1> index(rank);
+          shapeID = prepareDenseElementsConstant(
+              loc, shapeType,
+              mlirBuilder.getI32TensorAttr(SmallVector<int32_t>(dims)), 0,
+              index);
+        } else {
+          shapeID = prepareArrayConstant(
+              loc, shapeType,
+              mlirBuilder.getI32ArrayAttr(SmallVector<int32_t>(dims)));
+        }
+        if (shapeID == 0) {
+          return failure();
+        }
+      }
+    }
+    typeEnum = spirv::Opcode::OpTypeTensorARM;
+    operands.push_back(elementTypeID);
+    if (rankID == 0)
+      return success();
+    operands.push_back(rankID);
+    if (shapeID == 0)
+      return success();
+    operands.push_back(shapeID);
+    return success();
+  }
+
   // TODO: Handle other types.
   return emitError(loc, "unhandled type in serialization: ") << type;
 }
diff --git a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
index 682fff278477..ac338d555e32 100644
--- a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
+++ b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
@@ -14,8 +14,6 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/Dialect.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Parser/Parser.h"
 #include "mlir/Support/FileUtilities.h"
@@ -24,9 +22,7 @@
 #include "mlir/Tools/mlir-translate/Translation.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/ToolOutputFile.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
index 29bc49b78f15..824201d17b5a 100644
--- a/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
+++ b/mlir/lib/Tools/PDLL/CodeGen/MLIRGen.cpp
@@ -350,8 +350,9 @@ Value CodeGen::genNonInitializerVar(const ast::VariableDecl *varDecl,
     Value results = builder.create<pdl::TypesOp>(
         loc, pdl::RangeType::get(builder.getType<pdl::TypeType>()),
         /*types=*/ArrayAttr());
-    return builder.create<pdl::OperationOp>(
-        loc, opType.getName(), operands, std::nullopt, ValueRange(), results);
+    return builder.create<pdl::OperationOp>(loc, opType.getName(), operands,
+                                            ArrayRef<StringRef>(), ValueRange(),
+                                            results);
   }
 
   if (ast::RangeType rangeTy = dyn_cast<ast::RangeType>(type)) {
diff --git a/mlir/lib/Tools/PDLL/Parser/Parser.cpp b/mlir/lib/Tools/PDLL/Parser/Parser.cpp
index ad7d71b0bdd2..50e10447ee46 100644
--- a/mlir/lib/Tools/PDLL/Parser/Parser.cpp
+++ b/mlir/lib/Tools/PDLL/Parser/Parser.cpp
@@ -985,8 +985,7 @@ ast::Decl *Parser::createODSNativePDLLConstraintDecl(
   // Build the native constraint.
   auto *constraintDecl = ast::UserConstraintDecl::createNative(
       ctx, ast::Name::create(ctx, name, loc), paramVar,
-      /*results=*/std::nullopt, codeBlock, ast::TupleType::get(ctx),
-      nativeType);
+      /*results=*/{}, codeBlock, ast::TupleType::get(ctx), nativeType);
   constraintDecl->setDocComment(ctx, docString);
   curDeclScope->add(constraintDecl);
   return constraintDecl;
@@ -1782,7 +1781,7 @@ Parser::parseConstraint(std::optional<SMRange> &typeConstraint,
 
 FailureOr<ast::ConstraintRef> Parser::parseArgOrResultConstraint() {
   std::optional<SMRange> typeConstraint;
-  return parseConstraint(typeConstraint, /*existingConstraints=*/std::nullopt,
+  return parseConstraint(typeConstraint, /*existingConstraints=*/{},
                          /*allowInlineTypeConstraints=*/false);
 }
 
@@ -2995,8 +2994,8 @@ LogicalResult Parser::validateOperationOperandsOrResults(
       // Otherwise, create dummy values for each of the entries so that we
       // adhere to the ODS signature.
       for (unsigned i = 0, e = odsValues.size(); i < e; ++i) {
-        values.push_back(ast::RangeExpr::create(
-            ctx, loc, /*elements=*/std::nullopt, rangeTy));
+        values.push_back(
+            ast::RangeExpr::create(ctx, loc, /*elements=*/{}, rangeTy));
       }
       return success();
     }
diff --git a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
index 02c3ea1ce9b6..da29fab97964 100644
--- a/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Transforms/LoopInvariantCodeMotion.cpp
@@ -14,7 +14,6 @@
 
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
-#include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 
 namespace mlir {
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 0a02a01977ca..608bdcb94817 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -33,11 +33,9 @@
 
 #include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/LivenessAnalysis.h"
-#include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/IRMapping.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
diff --git a/mlir/lib/Transforms/StripDebugInfo.cpp b/mlir/lib/Transforms/StripDebugInfo.cpp
index 87f5f76d3d43..de6cc03c6b54 100644
--- a/mlir/lib/Transforms/StripDebugInfo.cpp
+++ b/mlir/lib/Transforms/StripDebugInfo.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Transforms/Passes.h"
 
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Pass/Pass.h"
 
diff --git a/mlir/lib/Transforms/SymbolDCE.cpp b/mlir/lib/Transforms/SymbolDCE.cpp
index 93d9a6547883..0a925c47a9e8 100644
--- a/mlir/lib/Transforms/SymbolDCE.cpp
+++ b/mlir/lib/Transforms/SymbolDCE.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Transforms/Passes.h"
 
 #include "mlir/IR/SymbolTable.h"
+#include "llvm/Support/Debug.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_SYMBOLDCE
@@ -22,6 +23,8 @@ namespace mlir {
 
 using namespace mlir;
 
+#define DEBUG_TYPE "symbol-dce"
+
 namespace {
 struct SymbolDCE : public impl::SymbolDCEBase<SymbolDCE> {
   void runOnOperation() override;
@@ -84,6 +87,8 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
                                          SymbolTableCollection &symbolTable,
                                          bool symbolTableIsHidden,
                                          DenseSet<Operation *> &liveSymbols) {
+  LLVM_DEBUG(llvm::dbgs() << "computeLiveness: " << symbolTableOp->getName()
+                          << "\n");
   // A worklist of live operations to propagate uses from.
   SmallVector<Operation *, 16> worklist;
 
@@ -105,9 +110,13 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
   }
 
   // Process the set of symbols that were known to be live, adding new symbols
-  // that are referenced within.
+  // that are referenced within. For operations that are not symbol tables, it
+  // considers the liveness with respect to the op itself rather than scope of
+  // nested symbol tables by enqueuing all the top level operations for
+  // consideration.
   while (!worklist.empty()) {
     Operation *op = worklist.pop_back_val();
+    LLVM_DEBUG(llvm::dbgs() << "processing: " << op->getName() << "\n");
 
     // If this is a symbol table, recursively compute its liveness.
     if (op->hasTrait<OpTrait::SymbolTable>()) {
@@ -115,26 +124,56 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
       // symbol, or if it is a private symbol.
       SymbolOpInterface symbol = dyn_cast<SymbolOpInterface>(op);
       bool symIsHidden = symbolTableIsHidden || !symbol || symbol.isPrivate();
+      LLVM_DEBUG(llvm::dbgs() << "\tsymbol table: " << op->getName()
+                              << " is hidden: " << symIsHidden << "\n");
       if (failed(computeLiveness(op, symbolTable, symIsHidden, liveSymbols)))
         return failure();
+    } else {
+      LLVM_DEBUG(llvm::dbgs()
+                 << "\tnon-symbol table: " << op->getName() << "\n");
+      // If the op is not a symbol table, then, unless op itself is dead which
+      // would be handled by DCE, we need to check all the regions and blocks
+      // within the op to find the uses (e.g., consider visibility within op as
+      // if top level rather than relying on pure symbol table visibility). This
+      // is more conservative than SymbolTable::walkSymbolTables in the case
+      // where there is again SymbolTable information to take advantage of.
+      for (auto &region : op->getRegions())
+        for (auto &block : region.getBlocks())
+          for (Operation &op : block)
+            if (op.getNumRegions())
+              worklist.push_back(&op);
     }
 
+    // Get the first parent symbol table op. Note: due to enqueueing of
+    // top-level ops, we may not have a symbol table parent here, but if we do
+    // not, then we also don't have a symbol.
+    Operation *parentOp = op->getParentOp();
+    if (!parentOp->hasTrait<OpTrait::SymbolTable>())
+      continue;
+
     // Collect the uses held by this operation.
     std::optional<SymbolTable::UseRange> uses = SymbolTable::getSymbolUses(op);
     if (!uses) {
       return op->emitError()
-             << "operation contains potentially unknown symbol table, "
-                "meaning that we can't reliable compute symbol uses";
+             << "operation contains potentially unknown symbol table, meaning "
+             << "that we can't reliable compute symbol uses";
     }
 
     SmallVector<Operation *, 4> resolvedSymbols;
+    LLVM_DEBUG(llvm::dbgs() << "uses of " << op->getName() << "\n");
     for (const SymbolTable::SymbolUse &use : *uses) {
+      LLVM_DEBUG(llvm::dbgs() << "\tuse: " << use.getUser() << "\n");
       // Lookup the symbols referenced by this use.
       resolvedSymbols.clear();
-      if (failed(symbolTable.lookupSymbolIn(
-              op->getParentOp(), use.getSymbolRef(), resolvedSymbols)))
+      if (failed(symbolTable.lookupSymbolIn(parentOp, use.getSymbolRef(),
+                                            resolvedSymbols)))
         // Ignore references to unknown symbols.
         continue;
+      LLVM_DEBUG({
+        llvm::dbgs() << "\t\tresolved symbols: ";
+        llvm::interleaveComma(resolvedSymbols, llvm::dbgs());
+        llvm::dbgs() << "\n";
+      });
 
       // Mark each of the resolved symbols as live.
       for (Operation *resolvedSymbol : resolvedSymbols)
diff --git a/mlir/lib/Transforms/Utils/CFGToSCF.cpp b/mlir/lib/Transforms/Utils/CFGToSCF.cpp
index 7c1781044d2a..dbde75e17b3b 100644
--- a/mlir/lib/Transforms/Utils/CFGToSCF.cpp
+++ b/mlir/lib/Transforms/Utils/CFGToSCF.cpp
@@ -115,14 +115,12 @@
 
 #include "mlir/Transforms/CFGToSCF.h"
 
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 7cfe7250d02c..d118fe422f2f 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -16,8 +16,6 @@
 #include "mlir/IR/Iterators.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "mlir/Rewrite/PatternApplicator.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -1080,6 +1078,16 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// to modify/access them is invalid rewriter API usage.
   SetVector<Operation *> replacedOps;
 
+  /// A set of operations that were created by the current pattern.
+  SetVector<Operation *> patternNewOps;
+
+  /// A set of operations that were modified by the current pattern.
+  SetVector<Operation *> patternModifiedOps;
+
+  /// A set of blocks that were inserted (newly-created blocks or moved blocks)
+  /// by the current pattern.
+  SetVector<Block *> patternInsertedBlocks;
+
   /// A mapping of all unresolved materializations (UnrealizedConversionCastOp)
   /// to the corresponding rewrite objects.
   DenseMap<UnrealizedConversionCastOp, UnresolvedMaterializationRewrite *>
@@ -1571,6 +1579,7 @@ void ConversionPatternRewriterImpl::notifyOperationInserted(
   if (!previous.isSet()) {
     // This is a newly created op.
     appendRewrite<CreateOperationRewrite>(op);
+    patternNewOps.insert(op);
     return;
   }
   Operation *prevOp = previous.getPoint() == previous.getBlock()->end()
@@ -1655,6 +1664,8 @@ void ConversionPatternRewriterImpl::notifyBlockInserted(
         }
       });
 
+  patternInsertedBlocks.insert(block);
+
   if (!previous) {
     // This is a newly created block.
     appendRewrite<CreateBlockRewrite>(block);
@@ -1852,6 +1863,8 @@ void ConversionPatternRewriter::finalizeOpModification(Operation *op) {
   assert(!impl->wasOpReplaced(op) &&
          "attempting to modify a replaced/erased op");
   PatternRewriter::finalizeOpModification(op);
+  impl->patternModifiedOps.insert(op);
+
   // There is nothing to do here, we only need to track the operation at the
   // start of the update.
 #ifndef NDEBUG
@@ -1964,21 +1977,25 @@ class OperationLegalizer {
   /// Legalize the resultant IR after successfully applying the given pattern.
   LogicalResult legalizePatternResult(Operation *op, const Pattern &pattern,
                                       ConversionPatternRewriter &rewriter,
-                                      RewriterState &curState);
+                                      const SetVector<Operation *> &newOps,
+                                      const SetVector<Operation *> &modifiedOps,
+                                      const SetVector<Block *> &insertedBlocks);
 
   /// Legalizes the actions registered during the execution of a pattern.
   LogicalResult
   legalizePatternBlockRewrites(Operation *op,
                                ConversionPatternRewriter &rewriter,
                                ConversionPatternRewriterImpl &impl,
-                               RewriterState &state, RewriterState &newState);
-  LogicalResult legalizePatternCreatedOperations(
-      ConversionPatternRewriter &rewriter, ConversionPatternRewriterImpl &impl,
-      RewriterState &state, RewriterState &newState);
-  LogicalResult legalizePatternRootUpdates(ConversionPatternRewriter &rewriter,
-                                           ConversionPatternRewriterImpl &impl,
-                                           RewriterState &state,
-                                           RewriterState &newState);
+                               const SetVector<Block *> &insertedBlocks,
+                               const SetVector<Operation *> &newOps);
+  LogicalResult
+  legalizePatternCreatedOperations(ConversionPatternRewriter &rewriter,
+                                   ConversionPatternRewriterImpl &impl,
+                                   const SetVector<Operation *> &newOps);
+  LogicalResult
+  legalizePatternRootUpdates(ConversionPatternRewriter &rewriter,
+                             ConversionPatternRewriterImpl &impl,
+                             const SetVector<Operation *> &modifiedOps);
 
   //===--------------------------------------------------------------------===//
   // Cost Model
@@ -2131,6 +2148,15 @@ OperationLegalizer::legalize(Operation *op,
   return failure();
 }
 
+/// Helper function that moves and returns the given object. Also resets the
+/// original object, so that it is in a valid, empty state again.
+template <typename T>
+static T moveAndReset(T &obj) {
+  T result = std::move(obj);
+  obj = T();
+  return result;
+}
+
 LogicalResult
 OperationLegalizer::legalizeWithFold(Operation *op,
                                      ConversionPatternRewriter &rewriter) {
@@ -2192,6 +2218,9 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
   RewriterState curState = rewriterImpl.getCurrentState();
   auto onFailure = [&](const Pattern &pattern) {
     assert(rewriterImpl.pendingRootUpdates.empty() && "dangling root updates");
+    rewriterImpl.patternNewOps.clear();
+    rewriterImpl.patternModifiedOps.clear();
+    rewriterImpl.patternInsertedBlocks.clear();
     LLVM_DEBUG({
       logFailure(rewriterImpl.logger, "pattern failed to match");
       if (rewriterImpl.config.notifyCallback) {
@@ -2212,7 +2241,13 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
   // successfully applied.
   auto onSuccess = [&](const Pattern &pattern) {
     assert(rewriterImpl.pendingRootUpdates.empty() && "dangling root updates");
-    auto result = legalizePatternResult(op, pattern, rewriter, curState);
+    SetVector<Operation *> newOps = moveAndReset(rewriterImpl.patternNewOps);
+    SetVector<Operation *> modifiedOps =
+        moveAndReset(rewriterImpl.patternModifiedOps);
+    SetVector<Block *> insertedBlocks =
+        moveAndReset(rewriterImpl.patternInsertedBlocks);
+    auto result = legalizePatternResult(op, pattern, rewriter, newOps,
+                                        modifiedOps, insertedBlocks);
     appliedPatterns.erase(&pattern);
     if (failed(result)) {
       if (!rewriterImpl.config.allowPatternRollback)
@@ -2253,10 +2288,11 @@ bool OperationLegalizer::canApplyPattern(Operation *op, const Pattern &pattern,
   return true;
 }
 
-LogicalResult
-OperationLegalizer::legalizePatternResult(Operation *op, const Pattern &pattern,
-                                          ConversionPatternRewriter &rewriter,
-                                          RewriterState &curState) {
+LogicalResult OperationLegalizer::legalizePatternResult(
+    Operation *op, const Pattern &pattern, ConversionPatternRewriter &rewriter,
+    const SetVector<Operation *> &newOps,
+    const SetVector<Operation *> &modifiedOps,
+    const SetVector<Block *> &insertedBlocks) {
   auto &impl = rewriter.getImpl();
   assert(impl.pendingRootUpdates.empty() && "dangling root updates");
 
@@ -2274,12 +2310,10 @@ OperationLegalizer::legalizePatternResult(Operation *op, const Pattern &pattern,
 #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
 
   // Legalize each of the actions registered during application.
-  RewriterState newState = impl.getCurrentState();
-  if (failed(legalizePatternBlockRewrites(op, rewriter, impl, curState,
-                                          newState)) ||
-      failed(legalizePatternRootUpdates(rewriter, impl, curState, newState)) ||
-      failed(legalizePatternCreatedOperations(rewriter, impl, curState,
-                                              newState))) {
+  if (failed(legalizePatternBlockRewrites(op, rewriter, impl, insertedBlocks,
+                                          newOps)) ||
+      failed(legalizePatternRootUpdates(rewriter, impl, modifiedOps)) ||
+      failed(legalizePatternCreatedOperations(rewriter, impl, newOps))) {
     return failure();
   }
 
@@ -2289,20 +2323,14 @@ OperationLegalizer::legalizePatternResult(Operation *op, const Pattern &pattern,
 
 LogicalResult OperationLegalizer::legalizePatternBlockRewrites(
     Operation *op, ConversionPatternRewriter &rewriter,
-    ConversionPatternRewriterImpl &impl, RewriterState &state,
-    RewriterState &newState) {
-  SmallPtrSet<Operation *, 16> operationsToIgnore;
+    ConversionPatternRewriterImpl &impl,
+    const SetVector<Block *> &insertedBlocks,
+    const SetVector<Operation *> &newOps) {
+  SmallPtrSet<Operation *, 16> alreadyLegalized;
 
   // If the pattern moved or created any blocks, make sure the types of block
   // arguments get legalized.
-  for (int i = state.numRewrites, e = newState.numRewrites; i != e; ++i) {
-    BlockRewrite *rewrite = dyn_cast<BlockRewrite>(impl.rewrites[i].get());
-    if (!rewrite)
-      continue;
-    Block *block = rewrite->getBlock();
-    if (isa<BlockTypeConversionRewrite, EraseBlockRewrite,
-            ReplaceBlockArgRewrite, InlineBlockRewrite>(rewrite))
-      continue;
+  for (Block *block : insertedBlocks) {
     // Only check blocks outside of the current operation.
     Operation *parentOp = block->getParentOp();
     if (!parentOp || parentOp == op || block->getNumArguments() == 0)
@@ -2322,41 +2350,26 @@ LogicalResult OperationLegalizer::legalizePatternBlockRewrites(
       continue;
     }
 
-    // Otherwise, check that this operation isn't one generated by this pattern.
-    // This is because we will attempt to legalize the parent operation, and
-    // blocks in regions created by this pattern will already be legalized later
-    // on. If we haven't built the set yet, build it now.
-    if (operationsToIgnore.empty()) {
-      for (unsigned i = state.numRewrites, e = impl.rewrites.size(); i != e;
-           ++i) {
-        auto *createOp =
-            dyn_cast<CreateOperationRewrite>(impl.rewrites[i].get());
-        if (!createOp)
-          continue;
-        operationsToIgnore.insert(createOp->getOperation());
+    // Otherwise, try to legalize the parent operation if it was not generated
+    // by this pattern. This is because we will attempt to legalize the parent
+    // operation, and blocks in regions created by this pattern will already be
+    // legalized later on.
+    if (!newOps.count(parentOp) && alreadyLegalized.insert(parentOp).second) {
+      if (failed(legalize(parentOp, rewriter))) {
+        LLVM_DEBUG(logFailure(
+            impl.logger, "operation '{0}'({1}) became illegal after rewrite",
+            parentOp->getName(), parentOp));
+        return failure();
       }
     }
-
-    // If this operation should be considered for re-legalization, try it.
-    if (operationsToIgnore.insert(parentOp).second &&
-        failed(legalize(parentOp, rewriter))) {
-      LLVM_DEBUG(logFailure(impl.logger,
-                            "operation '{0}'({1}) became illegal after rewrite",
-                            parentOp->getName(), parentOp));
-      return failure();
-    }
   }
   return success();
 }
 
 LogicalResult OperationLegalizer::legalizePatternCreatedOperations(
     ConversionPatternRewriter &rewriter, ConversionPatternRewriterImpl &impl,
-    RewriterState &state, RewriterState &newState) {
-  for (int i = state.numRewrites, e = newState.numRewrites; i != e; ++i) {
-    auto *createOp = dyn_cast<CreateOperationRewrite>(impl.rewrites[i].get());
-    if (!createOp)
-      continue;
-    Operation *op = createOp->getOperation();
+    const SetVector<Operation *> &newOps) {
+  for (Operation *op : newOps) {
     if (failed(legalize(op, rewriter))) {
       LLVM_DEBUG(logFailure(impl.logger,
                             "failed to legalize generated operation '{0}'({1})",
@@ -2369,12 +2382,8 @@ LogicalResult OperationLegalizer::legalizePatternCreatedOperations(
 
 LogicalResult OperationLegalizer::legalizePatternRootUpdates(
     ConversionPatternRewriter &rewriter, ConversionPatternRewriterImpl &impl,
-    RewriterState &state, RewriterState &newState) {
-  for (int i = state.numRewrites, e = newState.numRewrites; i != e; ++i) {
-    auto *rewrite = dyn_cast<ModifyOperationRewrite>(impl.rewrites[i].get());
-    if (!rewrite)
-      continue;
-    Operation *op = rewrite->getOperation();
+    const SetVector<Operation *> &modifiedOps) {
+  for (Operation *op : modifiedOps) {
     if (failed(legalize(op, rewriter))) {
       LLVM_DEBUG(logFailure(
           impl.logger, "failed to legalize operation updated in-place '{0}'",
@@ -2702,8 +2711,7 @@ legalizeUnresolvedMaterialization(RewriterBase &rewriter,
 }
 
 LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
-  if (ops.empty())
-    return success();
+  assert(!ops.empty() && "expected at least one operation");
   const ConversionTarget &target = opLegalizer.getTarget();
 
   // Compute the set of operations and blocks to convert.
@@ -3406,6 +3414,38 @@ void mlir::registerConversionPDLFunctions(RewritePatternSet &patterns) {
 // Op Conversion Entry Points
 //===----------------------------------------------------------------------===//
 
+/// This is the type of Action that is dispatched when a conversion is applied.
+class ApplyConversionAction
+    : public tracing::ActionImpl<ApplyConversionAction> {
+public:
+  using Base = tracing::ActionImpl<ApplyConversionAction>;
+  ApplyConversionAction(ArrayRef<IRUnit> irUnits) : Base(irUnits) {}
+  static constexpr StringLiteral tag = "apply-conversion";
+  static constexpr StringLiteral desc =
+      "Encapsulate the application of a dialect conversion";
+
+  void print(raw_ostream &os) const override { os << tag; }
+};
+
+static LogicalResult applyConversion(ArrayRef<Operation *> ops,
+                                     const ConversionTarget &target,
+                                     const FrozenRewritePatternSet &patterns,
+                                     ConversionConfig config,
+                                     OpConversionMode mode) {
+  if (ops.empty())
+    return success();
+  MLIRContext *ctx = ops.front()->getContext();
+  LogicalResult status = success();
+  SmallVector<IRUnit> irUnits(ops.begin(), ops.end());
+  ctx->executeAction<ApplyConversionAction>(
+      [&] {
+        OperationConverter opConverter(target, patterns, config, mode);
+        status = opConverter.convertOperations(ops);
+      },
+      irUnits);
+  return status;
+}
+
 //===----------------------------------------------------------------------===//
 // Partial Conversion
 //===----------------------------------------------------------------------===//
@@ -3413,9 +3453,8 @@ void mlir::registerConversionPDLFunctions(RewritePatternSet &patterns) {
 LogicalResult mlir::applyPartialConversion(
     ArrayRef<Operation *> ops, const ConversionTarget &target,
     const FrozenRewritePatternSet &patterns, ConversionConfig config) {
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Partial);
-  return opConverter.convertOperations(ops);
+  return applyConversion(ops, target, patterns, config,
+                         OpConversionMode::Partial);
 }
 LogicalResult
 mlir::applyPartialConversion(Operation *op, const ConversionTarget &target,
@@ -3432,9 +3471,7 @@ LogicalResult mlir::applyFullConversion(ArrayRef<Operation *> ops,
                                         const ConversionTarget &target,
                                         const FrozenRewritePatternSet &patterns,
                                         ConversionConfig config) {
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Full);
-  return opConverter.convertOperations(ops);
+  return applyConversion(ops, target, patterns, config, OpConversionMode::Full);
 }
 LogicalResult mlir::applyFullConversion(Operation *op,
                                         const ConversionTarget &target,
@@ -3501,9 +3538,8 @@ LogicalResult mlir::applyAnalysisConversion(
   // Convert the cloned operations. The original IR will remain unchanged.
   SmallVector<Operation *> opsToConvert = llvm::map_to_vector(
       ops, [&](Operation *op) { return mapping.lookup(op); });
-  OperationConverter opConverter(target, patterns, config,
-                                 OpConversionMode::Analysis);
-  LogicalResult status = opConverter.convertOperations(opsToConvert);
+  LogicalResult status = applyConversion(opsToConvert, target, patterns, config,
+                                         OpConversionMode::Analysis);
 
   // Remap `legalizableOps`, so that they point to the original ops and not the
   // cloned ops.
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 5a719200e002..b82d85041394 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -22,8 +22,6 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/mlir/lib/Transforms/Utils/Inliner.cpp b/mlir/lib/Transforms/Utils/Inliner.cpp
index ae34c4ad49e2..b639e87f5274 100644
--- a/mlir/lib/Transforms/Utils/Inliner.cpp
+++ b/mlir/lib/Transforms/Utils/Inliner.cpp
@@ -17,12 +17,10 @@
 #include "mlir/IR/Threading.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
-#include "mlir/Pass/Pass.h"
 #include "mlir/Support/DebugStringHelper.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "inlining"
@@ -541,7 +539,7 @@ Inliner::Impl::optimizeSCCAsync(MutableArrayRef<CallGraphNode *> nodesToVisit,
 
   // An atomic failure variable for the async executors.
   std::vector<std::atomic<bool>> activePMs(pipelines.size());
-  std::fill(activePMs.begin(), activePMs.end(), false);
+  llvm::fill(activePMs, false);
   return failableParallelForEach(ctx, nodesToVisit, [&](CallGraphNode *node) {
     // Find a pass manager for this operation.
     auto it = llvm::find_if(activePMs, [](std::atomic<bool> &isActive) {
diff --git a/mlir/lib/Transforms/Utils/InliningUtils.cpp b/mlir/lib/Transforms/Utils/InliningUtils.cpp
index 8594136570a7..eeb40529cc2f 100644
--- a/mlir/lib/Transforms/Utils/InliningUtils.cpp
+++ b/mlir/lib/Transforms/Utils/InliningUtils.cpp
@@ -16,7 +16,6 @@
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/CallInterfaces.h"
-#include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index 70a97f26b01f..353f8a58b3ac 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -11,12 +11,10 @@
 #include "mlir/Analysis/SliceAnalysis.h"
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/IR/Block.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
@@ -25,7 +23,6 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 
 #include <deque>
 #include <iterator>
diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index 28fe2a7501c0..08cac1fe3695 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -8,7 +8,6 @@
 
 #include "mlir/Transforms/ViewOpGraph.h"
 
-#include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/IR/Block.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Operation.h"
diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt
index ee07081246fc..b2daabb2a595 100644
--- a/mlir/python/CMakeLists.txt
+++ b/mlir/python/CMakeLists.txt
@@ -171,6 +171,15 @@ ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
   DIALECT_NAME transform
   EXTENSION_NAME transform_pdl_extension)
 
+declare_mlir_dialect_extension_python_bindings(
+ADD_TO_PARENT MLIRPythonSources.Dialects
+ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
+  TD_FILE dialects/TransformDebugExtensionOps.td
+  SOURCES
+    dialects/transform/debug.py
+  DIALECT_NAME transform
+  EXTENSION_NAME transform_debug_extension)
+
 declare_mlir_dialect_python_bindings(
   ADD_TO_PARENT MLIRPythonSources.Dialects
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir"
diff --git a/mlir/python/mlir/dialects/TransformDebugExtensionOps.td b/mlir/python/mlir/dialects/TransformDebugExtensionOps.td
new file mode 100644
index 000000000000..22a85d236699
--- /dev/null
+++ b/mlir/python/mlir/dialects/TransformDebugExtensionOps.td
@@ -0,0 +1,19 @@
+//===-- TransformDebugExtensionOps.td - Binding entry point *- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Entry point of the generated Python bindings for the Debug extension of the
+// Transform dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
+#define PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
+
+include "mlir/Dialect/Transform/DebugExtension/DebugExtensionOps.td"
+
+#endif // PYTHON_BINDINGS_TRANSFORM_DEBUG_EXTENSION_OPS
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index 48e724d80c92..1b359da40a29 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -21,21 +21,6 @@ def copy(
     O[None] = cast(U, I[None])
 
 
-@linalg_structured_op
-def elemwise_unary(
-    I=TensorDef(T1),
-    O=TensorDef(U, output=True),
-    fun=UnaryFnAttrDef(default=UnaryFn.exp),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Applies the unary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-    """
-    O[None] = fun(cast(U, I[None]))
-
-
 @linalg_structured_op
 def exp(
     I=TensorDef(T1),
@@ -192,22 +177,6 @@ def erf(
     O[None] = UnaryFn.erf(I[None])
 
 
-@linalg_structured_op
-def elemwise_binary(
-    lhs=TensorDef(T1),
-    rhs=TensorDef(T2),
-    O=TensorDef(U, output=True),
-    fun=BinaryFnAttrDef(default=BinaryFn.add),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Applies the binary function fun elementwise.
-
-    Numeric casting is performed on the input operand, promoting it to the same
-    data type as the accumulator/output.
-    """
-    O[None] = fun(cast(U, lhs[None]), cast(U, rhs[None]))
-
-
 @linalg_structured_op
 def add(
     lhs=TensorDef(T1),
diff --git a/mlir/python/mlir/dialects/transform/debug.py b/mlir/python/mlir/dialects/transform/debug.py
new file mode 100644
index 000000000000..f7c04268dc03
--- /dev/null
+++ b/mlir/python/mlir/dialects/transform/debug.py
@@ -0,0 +1,81 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from typing import Optional
+
+from ...ir import Attribute, Operation, Value, StringAttr
+from .._transform_debug_extension_ops_gen import *
+from .._transform_pdl_extension_ops_gen import _Dialect
+
+try:
+    from .._ods_common import _cext as _ods_cext
+except ImportError as e:
+    raise RuntimeError("Error loading imports from extension module") from e
+
+from typing import Union
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class EmitParamAsRemarkOp(EmitParamAsRemarkOp):
+    def __init__(
+        self,
+        param: Attribute,
+        *,
+        anchor: Optional[Operation] = None,
+        message: Optional[Union[StringAttr, str]] = None,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(message, str):
+            message = StringAttr.get(message)
+
+        super().__init__(
+            param,
+            anchor=anchor,
+            message=message,
+            loc=loc,
+            ip=ip,
+        )
+
+
+def emit_param_as_remark(
+    param: Attribute,
+    *,
+    anchor: Optional[Operation] = None,
+    message: Optional[Union[StringAttr, str]] = None,
+    loc=None,
+    ip=None,
+):
+    return EmitParamAsRemarkOp(param, anchor=anchor, message=message, loc=loc, ip=ip)
+
+
+@_ods_cext.register_operation(_Dialect, replace=True)
+class EmitRemarkAtOp(EmitRemarkAtOp):
+    def __init__(
+        self,
+        at: Union[Operation, Value],
+        message: Optional[Union[StringAttr, str]] = None,
+        *,
+        loc=None,
+        ip=None,
+    ):
+        if isinstance(message, str):
+            message = StringAttr.get(message)
+
+        super().__init__(
+            at,
+            message,
+            loc=loc,
+            ip=ip,
+        )
+
+
+def emit_remark_at(
+    at: Union[Operation, Value],
+    message: Optional[Union[StringAttr, str]] = None,
+    *,
+    loc=None,
+    ip=None,
+):
+    return EmitRemarkAtOp(at, message, loc=loc, ip=ip)
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
new file mode 100644
index 000000000000..68799098f1d3
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx950 | FileCheck %s
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx945 2>&1 | FileCheck %s --check-prefix=CHECK-OLD 
+
+// CHECK-LABEL: func @transpose_load_to_rocdl_4xf16
+func.func @transpose_load_to_rocdl_4xf16(%idx1 : index, %idx2 : index, %wgmem : memref<128x72xf16, 3>) -> vector<4xf16> {
+  // CHECK: rocdl.ds.read.tr16.b64
+  // CHECK-OLD: error: 'amdgpu.transpose_load' op Non-gfx950 chipset not supported
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x72xf16, 3> -> vector<4xf16>
+  return %0 : vector<4xf16>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_load_to_rocdl_8xi8
+func.func @transpose_load_to_rocdl_8xi8(%idx1 : index, %idx2 : index, %wgmem : memref<128x128xi8, 3>) -> vector<8xi8> {
+  // CHECK: %[[RES:.*]] = rocdl.ds.read.tr8.b64
+  // CHECK-SAME: -> vector<2xi32>
+  // CHECK-NEXT: llvm.bitcast %[[RES]] : vector<2xi32> to vector<8xi8>
+  // CHECK-OLD: error: 'amdgpu.transpose_load' op Non-gfx950 chipset not supported
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x128xi8, 3> -> vector<8xi8>
+  return %0 : vector<8xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_load_to_rocdl_i4_memrefxi8
+func.func @transpose_load_to_rocdl_i4_memrefxi8(%idx1 : index, %idx2 : index, %wgmem : memref<128x32xi8, 3>) -> vector<16xi4> {
+  // CHECK: %[[RES:.*]] = rocdl.ds.read.tr4.b64
+  // CHECK-SAME: -> vector<2xi32>
+  // CHECK-NEXT: llvm.bitcast %[[RES]] : vector<2xi32> to vector<16xi4>
+  // CHECK-OLD: error: 'amdgpu.transpose_load' op Non-gfx950 chipset not supported
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x32xi8, 3> -> vector<16xi4>
+  return %0 : vector<16xi4>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_load_to_rocdl_i6_memrefxi8
+func.func @transpose_load_to_rocdl_i6_memrefxi8(%idx1 : index, %idx2 : index, %wgmem : memref<128x32xi8, 3>) -> vector<16xi6> {
+  // CHECK: %[[RES:.*]] = rocdl.ds.read.tr6.b96
+  // CHECK-SAME: -> vector<3xi32>
+  // CHECK-NEXT: llvm.bitcast %[[RES]] : vector<3xi32> to vector<16xi6>
+  // CHECK-OLD: error: 'amdgpu.transpose_load' op Non-gfx950 chipset not supported
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x32xi8, 3> -> vector<16xi6>
+  return %0 : vector<16xi6>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_load_to_rocdl_i16_memrefxi8
+func.func @transpose_load_to_rocdl_i16_memrefxi8(%idx1 : index, %idx2 : index, %wgmem : memref<128x32xi8, 3>) -> vector<4xi16> {
+  // CHECK: rocdl.ds.read.tr16.b64
+  // CHECK-OLD: error: 'amdgpu.transpose_load' op Non-gfx950 chipset not supported
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x32xi8, 3> -> vector<4xi16>
+  return %0 : vector<4xi16>
+}
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
new file mode 100644
index 000000000000..a41051c904ed
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/transpose_load_reject.mlir
@@ -0,0 +1,17 @@
+// RUN: not mlir-opt %s --split-input-file -convert-amdgpu-to-rocdl=chipset=gfx950 2>&1 | FileCheck %s
+
+// -----
+
+func.func @transpose_load_to_rocdl_16xi4(%idx1 : index, %idx2 : index, %wgmem : memref<128x16xi4, 3>) -> vector<16xi4> {
+  // CHECK: memref to have at least 8 bits element size, got 4
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x16xi4, 3> -> vector<16xi4>
+  return %0 : vector<16xi4>
+}
+
+// -----
+
+func.func @transpose_load_to_rocdl_16xi6(%idx1 : index, %idx2 : index, %wgmem : memref<128x32xi6, 3>) -> vector<16xi6> {
+  // CHECK: memref to have at least 8 bits element size, got 6
+  %0 = amdgpu.transpose_load %wgmem[%idx1, %idx2] : memref<128x32xi6, 3> -> vector<16xi6>
+  return %0 : vector<16xi6>
+}
diff --git a/mlir/test/Conversion/GPUToSPIRV/rotate.mlir b/mlir/test/Conversion/GPUToSPIRV/rotate.mlir
new file mode 100644
index 000000000000..b96dd37219b4
--- /dev/null
+++ b/mlir/test/Conversion/GPUToSPIRV/rotate.mlir
@@ -0,0 +1,102 @@
+// RUN: mlir-opt -split-input-file -convert-gpu-to-spirv -verify-diagnostics %s -o - | FileCheck %s
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @rotate()
+  gpu.func @rotate() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 16 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 16 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformRotateKHR <Subgroup> %[[VAL]], %[[OFFSET]], cluster_size(%[[WIDTH]]) : f32, i32, i32 -> f32
+    // CHECK: %{{.+}} = spirv.Constant true
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @rotate_width_less_than_subgroup_size()
+  gpu.func @rotate_width_less_than_subgroup_size() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 8 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 8 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformRotateKHR <Subgroup> %[[VAL]], %[[OFFSET]], cluster_size(%[[WIDTH]]) : f32, i32, i32 -> f32
+    // CHECK: %[[INVOCATION_ID_ADDR:.+]] = spirv.mlir.addressof @__builtin__SubgroupLocalInvocationId__
+    // CHECK: %[[INVOCATION_ID:.+]] = spirv.Load "Input" %[[INVOCATION_ID_ADDR]]
+    // CHECK: %{{.+}} = spirv.ULessThan %[[INVOCATION_ID]], %[[WIDTH]]
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  gpu.func @rotate_with_bigger_than_subgroup_size() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 32 : i32
+    %val = arith.constant 42.0 : f32
+
+    // expected-error @+1 {{failed to legalize operation 'gpu.rotate'}}
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformRotateKHR], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  gpu.func @rotate_non_const_width(%width: i32) kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %val = arith.constant 42.0 : f32
+
+    // expected-error @+1 {{'gpu.rotate' op width is not a constant value}}
+    %result, %valid = gpu.rotate %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 3df14528bac3..c03d67fdc33f 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1875,7 +1875,7 @@ func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
 // CHECK: %[[CAST_MEMREF:.*]] = builtin.unrealized_conversion_cast %{{.*}} : memref<200x100xf32> to !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[CST:.*]] = arith.constant dense<1.100000e+01> : vector<f32>
 // CHECK: %[[VAL:.*]] = builtin.unrealized_conversion_cast %[[CST]] : vector<f32> to vector<1xf32>
-// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> 
+// CHECK: %[[REF:.*]] = llvm.extractvalue %[[CAST_MEMREF]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK: %[[C100:.*]] = llvm.mlir.constant(100 : index) : i64
 // CHECK: %[[MUL:.*]] = llvm.mul %[[I]], %[[C100]] : i64
 // CHECK: %[[ADD:.*]] = llvm.add %[[MUL]], %[[J]] : i64
@@ -2421,6 +2421,44 @@ func.func @from_elements_0d(%arg0: f32) -> vector<f32> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.to_elements
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: func.func @to_elements_no_dead_elements
+ // CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+ //      CHECK:   %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64
+ //      CHECK:   %[[ELEM0:.*]] = llvm.extractelement %[[A]][%[[C0]] : i64] : vector<4xf32>
+ //      CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64
+ //      CHECK:   %[[ELEM1:.*]] = llvm.extractelement %[[A]][%[[C1]] : i64] : vector<4xf32>
+ //      CHECK:   %[[C2:.*]] = llvm.mlir.constant(2 : i64) : i64
+ //      CHECK:   %[[ELEM2:.*]] = llvm.extractelement %[[A]][%[[C2]] : i64] : vector<4xf32>
+ //      CHECK:   %[[C3:.*]] = llvm.mlir.constant(3 : i64) : i64
+ //      CHECK:   %[[ELEM3:.*]] = llvm.extractelement %[[A]][%[[C3]] : i64] : vector<4xf32>
+ //      CHECK:   return %[[ELEM0]], %[[ELEM1]], %[[ELEM2]], %[[ELEM3]] : f32, f32, f32, f32
+func.func @to_elements_no_dead_elements(%a: vector<4xf32>) -> (f32, f32, f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#0, %0#1, %0#2, %0#3 : f32, f32, f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: func.func @to_elements_dead_elements
+ // CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+ //  CHECK-NOT:   llvm.mlir.constant(0 : i64) : i64
+ //      CHECK:   %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64
+ //      CHECK:   %[[ELEM1:.*]] = llvm.extractelement %[[A]][%[[C1]] : i64] : vector<4xf32>
+ //  CHECK-NOT:   llvm.mlir.constant(2 : i64) : i64
+ //      CHECK:   %[[C3:.*]] = llvm.mlir.constant(3 : i64) : i64
+ //      CHECK:   %[[ELEM3:.*]] = llvm.extractelement %[[A]][%[[C3]] : i64] : vector<4xf32>
+ //      CHECK:   return %[[ELEM1]], %[[ELEM3]] : f32, f32
+func.func @to_elements_dead_elements(%a: vector<4xf32>) -> (f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#1, %0#3 : f32, f32
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // vector.step
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
index 4701ac5d9600..99ab0e1dc4ee 100644
--- a/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
+++ b/mlir/test/Conversion/VectorToSPIRV/vector-to-spirv.mlir
@@ -246,6 +246,41 @@ func.func @extract_dynamic_cst(%arg0 : vector<4xf32>) -> f32 {
 
 // -----
 
+// CHECK-LABEL: func.func @to_elements_one_element 
+// CHECK-SAME:     %[[A:.*]]: vector<1xf32>)
+//      CHECK:   %[[ELEM0:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<1xf32> to f32
+//      CHECK:   return %[[ELEM0]] : f32
+func.func @to_elements_one_element(%a: vector<1xf32>) -> (f32) {
+  %0:1 = vector.to_elements %a : vector<1xf32>
+  return %0#0 : f32
+}
+
+// CHECK-LABEL: func.func @to_elements_no_dead_elements
+// CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+//      CHECK:   %[[ELEM0:.*]] = spirv.CompositeExtract %[[A]][0 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM1:.*]] = spirv.CompositeExtract %[[A]][1 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM2:.*]] = spirv.CompositeExtract %[[A]][2 : i32] : vector<4xf32>
+//      CHECK:   %[[ELEM3:.*]] = spirv.CompositeExtract %[[A]][3 : i32] : vector<4xf32>
+//      CHECK:   return %[[ELEM0]], %[[ELEM1]], %[[ELEM2]], %[[ELEM3]] : f32, f32, f32, f32
+func.func @to_elements_no_dead_elements(%a: vector<4xf32>) -> (f32, f32, f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#0, %0#1, %0#2, %0#3 : f32, f32, f32, f32
+}
+
+// CHECK-LABEL: func.func @to_elements_dead_elements
+// CHECK-SAME:     %[[A:.*]]: vector<4xf32>)
+//  CHECK-NOT:   spirv.CompositeExtract %[[A]][0 : i32]
+//      CHECK:   %[[ELEM1:.*]] = spirv.CompositeExtract %[[A]][1 : i32] : vector<4xf32>
+//  CHECK-NOT:   spirv.CompositeExtract %[[A]][2 : i32]
+//      CHECK:   %[[ELEM3:.*]] = spirv.CompositeExtract %[[A]][3 : i32] : vector<4xf32>
+//      CHECK:   return %[[ELEM1]], %[[ELEM3]] : f32, f32
+func.func @to_elements_dead_elements(%a: vector<4xf32>) -> (f32, f32) {
+  %0:4 = vector.to_elements %a : vector<4xf32>
+  return %0#1, %0#3 : f32, f32
+}
+
+// -----
+
 // CHECK-LABEL: @from_elements_0d
 //  CHECK-SAME: %[[ARG0:.+]]: f32
 //       CHECK:   %[[RETVAL:.+]] = builtin.unrealized_conversion_cast %[[ARG0]]
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 73306ba6b3f9..6d55583f8bc7 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -166,3 +166,59 @@ func.func @swizzle_scalable_vec(%arg0 : vector<[4]xf32>) -> vector<[4]xf32> {
   %0 = amdgpu.swizzle_bitmode %arg0 1 2 4 : vector<[4]xf32>
   func.return %0 : vector<[4]xf32>
 }
+
+// -----
+
+func.func @transpose_load_addrspace(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16, 1>) -> vector<4xf16> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op source memory address space must be Workgroup}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf16, 1> -> vector<4xf16>
+  func.return %0 : vector<4xf16>
+}
+
+// -----
+
+func.func @transpose_load_addrspace(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16, 1>) -> vector<4xf16> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op source memory address space must be Workgroup}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf16, 1> -> vector<4xf16>
+  func.return %0 : vector<4xf16>
+}
+
+// -----
+
+func.func @transpose_load_elem_f32(%idx1 : index, %idx2 : index, %mem : memref<128x32xf32, 3>) -> vector<4xf32> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op Unsupported element type size for transpose load: 32 bits}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf32, 3> -> vector<4xf32>
+  func.return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @transpose_load_vector_size_f16(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16, 3>) -> vector<2xf16> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op Transferring type size mismatch: expected num of elements: 4}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf16, 3> -> vector<2xf16>
+  func.return %0 : vector<2xf16>
+}
+
+// -----
+
+func.func @transpose_load_vector_size_i4(%idx1 : index, %idx2 : index, %mem : memref<128x32xi4, 3>) -> vector<20xi4> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op Transferring type size mismatch: expected num of elements: 16}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xi4, 3> -> vector<20xi4>
+  func.return %0 : vector<20xi4>
+}
+
+// -----
+
+func.func @transpose_load_vector_size_i8(%idx1 : index, %idx2 : index, %mem : memref<128x32xi8, 3>) -> vector<20xi8> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op Transferring type size mismatch: expected num of elements: 8}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xi8, 3> -> vector<20xi8>
+  func.return %0 : vector<20xi8>
+}
+
+// -----
+
+func.func @transpose_load_vector_size_i8(%idx1 : index, %idx2 : index, %mem : memref<128x32xi6, 3>) -> vector<8xi6> {
+  // expected-error@+1 {{'amdgpu.transpose_load' op Transferring type size mismatch: expected num of elements: 16}}
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xi6, 3> -> vector<8xi6>
+  func.return %0 : vector<8xi6>
+}
diff --git a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir b/mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
similarity index 56%
rename from mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
rename to mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
index 20999af10553..febe46bf7a75 100644
--- a/mlir/test/Dialect/AMDGPU/transfer-read-to-load.mlir
+++ b/mlir/test/Dialect/AMDGPU/maskedload-to-load.mlir
@@ -1,17 +1,17 @@
-// RUN: mlir-opt %s --amdgpu-transfer-read-to-load --split-input-file | FileCheck %s
+// RUN: mlir-opt %s --amdgpu-maskedload-to-load --split-input-file | FileCheck %s
 
 // CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer(
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
 
 // CHECK: %[[IF:.*]] = scf.if
-// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]]
+// CHECK: vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]]
 
 // CHECK: } else {
 // CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
@@ -25,10 +25,10 @@ func.func @transfer_to_maskedload_fatrawbuffer(%mem : memref<8x8xf32, #amdgpu.ad
 // CHECK-LABEL: func @transfer_to_maskedload_fatrawbuffer_f16(
 // CHECK-SAME: %[[ARG0:.+]]: memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>,
 // CHECK-SAME: %[[ARG1:.+]]: index, %[[ARG2:.+]]: index,
-// CHECK-SAME: %[[ARG3:.+]]: vector<4xi1>)
-func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xf16> {
-  %cf0 = arith.constant 0.0 : f16
-  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf16>
+// CHECK-SAME: %[[ARG3:.+]]: vector<4xi1>
+// CHECK-SAME: %[[ARG4:.+]]: vector<4xf16>
+func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>, %passthru : vector<4xf16>) -> vector<4xf16> {
+  %res = vector.maskedload %mem[%idx0, %idx1], %mask, %passthru : memref<8x8xf16, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xf16> into vector<4xf16>
   return %res : vector<4xf16>
 }
 // CHECK-DAG: %[[C0:.*]] = arith.constant 0
@@ -45,7 +45,7 @@ func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgp
 
 // CHECK: %[[COND:.*]] = arith.andi %[[COND1]], %[[COND2]]
 // CHECK: %[[IF:.*]] = scf.if %[[COND]] -> (vector<4xf16>) {
-// CHECK: vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG2]]]
+// CHECK: vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG2]]]
 // CHECK: } else {
 // CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG2]]]
 // CHECK: return %[[IF]] : vector<4xf16>
@@ -58,13 +58,11 @@ func.func @transfer_to_maskedload_fatrawbuffer_f16(%mem : memref<8x8xf16, #amdgp
 // CHECK-SAME: %[[ARG0:.*]]: memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
 // CHECK-SAME: %[[ARG3:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>) -> vector<4xi8> {
-  %cf0 = arith.constant 0 : i8
-  %res = vector.transfer_read %mem[%idx0, %idx1], %cf0, %mask {in_bounds = [true]} : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi8>
+// CHECK-SAME: %[[ARG4:.*]]: vector<4xi8>
+func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, %idx0 : index, %idx1 : index, %mask : vector<4xi1>, %passthru : vector<4xi8>) -> vector<4xi8> {
+  %res = vector.maskedload %mem[%idx0, %idx1], %mask, %passthru : memref<?x?xi8, #amdgpu.address_space<fat_raw_buffer>>, vector<4xi1>, vector<4xi8> into vector<4xi8>
   return %res : vector<4xi8>
 }
-
-// CHECK:     %[[CST:.*]] = arith.constant dense<0> : vector<4xi8>
 // CHECK:     %[[C0:.*]] = arith.constant 0 : index
 // CHECK:     %[[C4:.*]] = arith.constant 4 : index
 // CHECK:     %[[BASE:.*]], %[[OFFSET:.*]], %[[SIZES:.*]]:2, %[[STRIDES:.*]]:2 = memref.extract_strided_metadata %[[ARG0]]
@@ -79,13 +77,12 @@ func.func @transfer_to_maskedload_fatrawbuffer_dynamic_i8(%mem : memref<?x?xi8,
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
-// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
+// CHECK: %[[RES:.*]] = vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[ARG2]], %[[ARG3]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
@@ -94,49 +91,26 @@ func.func @transfer_to_maskedload_regular(%mem : memref<8x8xf32>, %idx : index,
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #gpu.address_space<workgroup>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<4xi1>
-func.func @transfer_to_maskedload_addrspace(%mem : memref<8x8xf32, #gpu.address_space<workgroup>>, %idx : index, %mask : vector<4xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask {in_bounds = [true]} : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<4xf32>
+func.func @transfer_to_maskedload_addrspace(%mem : memref<8x8xf32, #gpu.address_space<workgroup>>, %idx : index, %mask : vector<4xi1>, %passthru : vector<4xf32>) -> vector<4xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru : memref<8x8xf32, #gpu.address_space<workgroup>>, vector<4xi1>, vector<4xf32> into vector<4xf32>
   return %res : vector<4xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
-// CHECK: %[[RES:.*]] = vector.transfer_read %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[CST]], %[[ARG2]]
+// CHECK: %[[RES:.*]] = vector.maskedload %[[ARG0]][%[[ARG1]], %[[ARG1]]], %[[ARG2]], %[[ARG3]]
 // CHECK: return %[[RES]] : vector<4xf32>
 
 // -----
 
-// CHECK-LABEL: func @transfer_broadcasting(
-// CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
-// CHECK-SAME: %[[ARG1:.*]]: index
-// CHECK-SAME: %[[ARG2:.*]]: vector<1xi1>
-#broadcast_1d = affine_map<(d0, d1) -> (0)>
-func.func @transfer_broadcasting(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>) -> vector<4xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask
-    {in_bounds = [true], permutation_map = #broadcast_1d}
-      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<4xf32>
-  return %res : vector<4xf32>
-}
-// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
-// CHECK: %[[IF:.*]] = scf.if
-// CHECK: %[[LOAD:.*]] = vector.load %arg0[%arg1, %arg1]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
-// CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[SELECT]] : vector<1xf32> to vector<4xf32>
-
-// -----
-
 // CHECK-LABEL: func @transfer_scalar(
 // CHECK-SAME: %[[ARG0:.*]]: memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>
 // CHECK-SAME: %[[ARG1:.*]]: index
 // CHECK-SAME: %[[ARG2:.*]]: vector<1xi1>
-func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>) -> vector<1xf32> {
-  %cf0 = arith.constant 0.0 : f32
-  %res = vector.transfer_read %mem[%idx, %idx], %cf0, %mask
-    {in_bounds = [true]}
-      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<1xf32>
+// CHECK-SAME: %[[ARG3:.*]]: vector<1xf32>
+func.func @transfer_scalar(%mem : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, %idx : index, %mask : vector<1xi1>, %passthru : vector<1xf32>) -> vector<1xf32> {
+  %res = vector.maskedload %mem[%idx, %idx], %mask, %passthru
+      : memref<8x8xf32, #amdgpu.address_space<fat_raw_buffer>>, vector<1xi1>, vector<1xf32> into vector<1xf32>
   return %res : vector<1xf32>
 }
-// CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<1xf32>
 // CHECK: %[[IF:.*]] = scf.if
 // CHECK: %[[LOAD:.*]] = vector.load %[[ARG0]][%[[ARG1]], %[[ARG1]]]
-// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[CST]]
+// CHECK: %[[SELECT:.*]] = arith.select %arg2, %[[LOAD]], %[[ARG3]]
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index 6c3ffb575f7c..51f3bbd9ae45 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -486,3 +486,10 @@ func.func @scaled_mfma(%arg0 : f8E8M0FNU, %arg1 : vector<32xf6E2M3FN>, %arg2 : v
   %0 = amdgpu.scaled_mfma(%arg0[0] * %arg1) * (%arg0[1] * %arg1) + %arg2 { k = 64 : i32, m = 32 : i32, n = 32 : i32 } : f8E8M0FNU, vector<32xf6E2M3FN>, f8E8M0FNU, vector<32xf6E2M3FN>, vector<16xf32>
   func.return %0 : vector<16xf32>
 }
+
+// CHECK-LABEL: func @transpose_load
+func.func @transpose_load(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16, 3>) -> vector<4xf16> {
+  // CHECK: amdgpu.transpose_load
+  %0 = amdgpu.transpose_load %mem[%idx1, %idx2] : memref<128x32xf16, 3> -> vector<4xf16>
+  func.return %0 : vector<4xf16>
+}
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
index 81b04ccceaf2..72ced5b53879 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
@@ -21,7 +21,7 @@ func.func @vec1d_1(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK: for {{.*}} step 128
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%[[C0]])
-// CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %{{.*}} = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i0 = 0 to %M { // vectorized due to scalar -> vector
      %a0 = affine.load %A[%c0, %c0] : memref<?x?xf32>
@@ -47,7 +47,7 @@ func.func @vec1d_2(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
    %P = memref.dim %B, %c2 : memref<?x?x?xf32>
 
 // CHECK:for [[IV3:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
-// CHECK-NEXT: %[[CST:.*]] = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %[[CST]] : memref<?x?xf32>, vector<128xf32>
    affine.for %i3 = 0 to %M { // vectorized
      %a3 = affine.load %A[%c0, %i3] : memref<?x?xf32>
@@ -76,7 +76,7 @@ func.func @vec1d_3(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK-NEXT:   for [[IV9:%[0-9a-zA-Z_]*]] = 0 to [[ARG_N]] {
 // CHECK-NEXT:   %[[APP9_0:[0-9a-zA-Z_]+]] = affine.apply {{.*}}([[IV9]], [[IV8]])
 // CHECK-NEXT:   %[[APP9_1:[0-9a-zA-Z_]+]] = affine.apply {{.*}}([[IV9]], [[IV8]])
-// CHECK-NEXT:   %[[CST:.*]] = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT:   %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:   {{.*}} = vector.transfer_read %{{.*}}[%[[APP9_0]], %[[APP9_1]]], %[[CST]] : memref<?x?xf32>, vector<128xf32>
    affine.for %i8 = 0 to %M { // vectorized
      affine.for %i9 = 0 to %N {
@@ -280,7 +280,7 @@ func.func @vec_rejected_3(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 
 // CHECK:for [[IV4:%[0-9a-zA-Z_]+]] = 0 to [[ARG_M]] step 128 {
 // CHECK-NEXT:   for [[IV5:%[0-9a-zA-Z_]*]] = 0 to [[ARG_N]] {
-// CHECK-NEXT:     %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT:     %{{.*}} = ub.poison : f32
 // CHECK-NEXT:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{[a-zA-Z0-9_]*}} : memref<?x?xf32>, vector<128xf32>
    affine.for %i4 = 0 to %M { // vectorized
      affine.for %i5 = 0 to %N { // not vectorized, would vectorize with --test-fastest-varying=1
@@ -424,7 +424,7 @@ func.func @vec_rejected_8(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:   for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK:     %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
-// CHECK:     %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK:     %{{.*}} = ub.poison : f32
 // CHECK:     {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %{{.*}} in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
@@ -458,7 +458,7 @@ func.func @vec_rejected_9(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
 // CHECK:   for [[IV18:%[a-zA-Z0-9]+]] = 0 to [[ARG_M]] step 128
 // CHECK:      %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
 // CHECK-NEXT: %{{.*}} = affine.apply #[[$map_id1]](%{{.*}})
-// CHECK-NEXT: %{{.*}} = arith.constant 0.0{{.*}}: f32
+// CHECK-NEXT: %{{.*}} = ub.poison : f32
 // CHECK-NEXT: {{.*}} = vector.transfer_read %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}} {permutation_map = #[[$map_proj_d0d1_0]]} : memref<?x?xf32>, vector<128xf32>
    affine.for %i17 = 0 to %M { // not vectorized, the 1-D pattern that matched %i18 in DFS post-order prevents vectorizing %{{.*}}
      affine.for %i18 = 0 to %M { // vectorized due to scalar -> vector
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
index 15a7133cf0f6..7d4d111c0979 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_affine_apply.mlir
@@ -11,7 +11,7 @@ func.func @vec_affine_apply(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24x48xf3
 // CHECK-NEXT:      affine.for %[[ARG4:.*]] = 0 to 48 step 8 {
 // CHECK-NEXT:        %[[S0:.*]] = affine.apply #[[$MAP_ID0]](%[[ARG3]])
 // CHECK-NEXT:        %[[S1:.*]] = affine.apply #[[$MAP_ID1]](%[[ARG4]])
-// CHECK-NEXT:        %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:        %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:        %[[S2:.*]] = vector.transfer_read %[[ARG0]][%[[ARG2]], %[[S0]], %[[S1]]], %[[CST]] : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:        vector.transfer_write %[[S2]], %[[ARG1]][%[[ARG2]], %[[ARG3]], %[[ARG4]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:      }
@@ -42,7 +42,7 @@ func.func @vec_affine_apply_2(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24x48x
 // CHECK-NEXT:   affine.for %[[ARG3:.*]] = 0 to 12 {
 // CHECK-NEXT:     affine.for %[[ARG4:.*]] = 0 to 48 step 8 {
 // CHECK-NEXT:       %[[S0:.*]] = affine.apply #[[$MAP_ID2]](%[[ARG4]])
-// CHECK-NEXT:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:       %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:       %[[S1:.*]] = vector.transfer_read %[[ARG0]][%[[ARG2]], %[[ARG3]], %[[S0]]], %[[CST]] : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:       vector.transfer_write %[[S1]], %[[ARG1]][%[[ARG2]], %[[ARG3]], %[[ARG4]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:     }
@@ -140,7 +140,7 @@ func.func @affine_map_with_expr_2(%arg0: memref<8x12x16xf32>, %arg1: memref<8x24
 // CHECK-NEXT:       %[[S0:.*]] = affine.apply #[[$MAP_ID3]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[S1:.*]] = affine.apply #[[$MAP_ID4]](%[[ARG3]], %[[ARG4]], %[[I0]])
 // CHECK-NEXT:       %[[S2:.*]] = affine.apply #[[$MAP_ID5]](%[[ARG3]], %[[ARG4]], %[[I0]])
-// CHECK-NEXT:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-NEXT:       %[[CST:.*]] = ub.poison : f32
 // CHECK-NEXT:       %[[S3:.*]] = vector.transfer_read %[[ARG0]][%[[S0]], %[[S1]], %[[S2]]], %[[CST]] {permutation_map = #[[$MAP_ID6]]} : memref<8x12x16xf32>, vector<8xf32>
 // CHECK-NEXT:       vector.transfer_write %[[S3]], %[[ARG1]][%[[ARG3]], %[[ARG4]], %[[ARG5]]] : vector<8xf32>, memref<8x24x48xf32>
 // CHECK-NEXT:     }
diff --git a/mlir/test/Dialect/Affine/affine-data-copy.mlir b/mlir/test/Dialect/Affine/affine-data-copy.mlir
index a1f0d952e7c6..a745271eb9ca 100644
--- a/mlir/test/Dialect/Affine/affine-data-copy.mlir
+++ b/mlir/test/Dialect/Affine/affine-data-copy.mlir
@@ -447,3 +447,51 @@ func.func @memref_def_inside(%arg0: index) {
   // LIMITED-MEM-NEXT:   memref.dealloc %{{.*}} : memref<1xf32>
   return
 }
+
+// Test with uses across multiple blocks.
+
+memref.global "private" constant @__constant_1x2x1xi32_1 : memref<1x2x1xi32> = dense<0> {alignment = 64 : i64}
+
+// CHECK-LABEL: func @multiple_blocks
+func.func @multiple_blocks(%arg0: index) -> memref<1x2x1xi32> {
+  %c1_i32 = arith.constant 1 : i32
+  %c3_i32 = arith.constant 3 : i32
+  %0 = memref.get_global @__constant_1x2x1xi32_1 : memref<1x2x1xi32>
+  %alloc = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi32>
+  memref.copy %0, %alloc : memref<1x2x1xi32> to memref<1x2x1xi32>
+  cf.br ^bb1(%alloc : memref<1x2x1xi32>)
+^bb1(%1: memref<1x2x1xi32>):  // 2 preds: ^bb0, ^bb2
+// CHECK: ^bb1(%[[MEM:.*]]: memref<1x2x1xi32>):
+  %alloc_0 = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi1>
+  // CHECK: %[[BUF:.*]] = memref.alloc() : memref<1x2x1xi32>
+  affine.for %arg1 = 0 to 1 {
+    affine.for %arg2 = 0 to 2 {
+      affine.for %arg3 = 0 to 1 {
+        // CHECK: affine.load %[[BUF]]
+        %3 = affine.load %1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+        %4 = arith.cmpi slt, %3, %c3_i32 : i32
+        affine.store %4, %alloc_0[%arg1, %arg2, %arg3] : memref<1x2x1xi1>
+      }
+    }
+  }
+  // CHECK: memref.dealloc %[[BUF]]
+  %2 = memref.load %alloc_0[%arg0, %arg0, %arg0] : memref<1x2x1xi1>
+  cf.cond_br %2, ^bb2, ^bb3
+^bb2:  // pred: ^bb1
+// CHECK: ^bb2
+  %alloc_1 = memref.alloc() {alignment = 64 : i64} : memref<1x2x1xi32>
+  affine.for %arg1 = 0 to 1 {
+    affine.for %arg2 = 0 to 2 {
+      affine.for %arg3 = 0 to 1 {
+        // Ensure that this reference isn't replaced.
+        %3 = affine.load %1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+        // CHECK: affine.load %[[MEM]]
+        %4 = arith.addi %3, %c1_i32 : i32
+        affine.store %4, %alloc_1[%arg1, %arg2, %arg3] : memref<1x2x1xi32>
+      }
+    }
+  }
+  cf.br ^bb1(%alloc_1 : memref<1x2x1xi32>)
+^bb3:  // pred: ^bb1
+  return %1 : memref<1x2x1xi32>
+}
diff --git a/mlir/test/Dialect/Affine/decompose-affine-ops.mlir b/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
index 6acdc436fe67..e5db5cd9181d 100644
--- a/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
+++ b/mlir/test/Dialect/Affine/decompose-affine-ops.mlir
@@ -27,8 +27,7 @@ func.func @simple_test_1(%0: index, %1: index, %2: index, %lb: index, %ub: index
 // CHECK-DAG: #[[$c42:.*]] = affine_map<() -> (42)>
 // CHECK-DAG: #[[$id:.*]] = affine_map<()[s0] -> (s0)>
 // CHECK-DAG: #[[$add:.*]] = affine_map<()[s0, s1] -> (s0 + s1)>
-// CHECK-DAG: #[[$div32div4timesm4:.*]] = affine_map<()[s0] -> (((s0 floordiv 32) floordiv 4) * -4)>
-// CHECK-DAG: #[[$div32:.*]] = affine_map<()[s0] -> (s0 floordiv 32)>
+// CHECK-DAG: #[[$div32mod4:.*]] = affine_map<()[s0] -> ((s0 floordiv 32) mod 4)>
 
 // CHECK-LABEL:  func.func @simple_test_2
 //  CHECK-SAME:  %[[I0:[0-9a-zA-Z]+]]: index,
@@ -45,10 +44,8 @@ func.func @simple_test_2(%0: index, %1: index, %2: index, %lb: index, %ub: index
     // CHECK:   %[[R2:.*]] = affine.apply #[[$add]]()[%[[c42]], %[[R1]]]
     // CHECK:   scf.for %[[j:.*]] =
     scf.for %j = %lb to %ub step %step {
-      // CHECK:     %[[R3:.*]] = affine.apply #[[$div32div4timesm4]]()[%[[j]]]
-      // CHECK:     %[[R4:.*]] = affine.apply #[[$add]]()[%[[R2]], %[[R3]]]
-      // CHECK:     %[[R5:.*]] = affine.apply #[[$div32]]()[%[[j]]]
-      // CHECK:      %[[a:.*]] = affine.apply #[[$add]]()[%[[R4]], %[[R5]]]
+      // CHECK:     %[[R3:.*]] = affine.apply #[[$div32mod4]]()[%[[j]]]
+      // CHECK:     %[[a:.*]] = affine.apply #[[$add]]()[%[[R2]], %[[R3]]]
       %a = affine.apply affine_map<(d0)[s0] -> ((d0 floordiv 32) mod 4 + s0 + 42)>(%j)[%i]
 
       // CHECK:     "some_side_effecting_consumer"(%[[a]]) : (index) -> ()
@@ -67,8 +64,7 @@ func.func @simple_test_2(%0: index, %1: index, %2: index, %lb: index, %ub: index
 // CHECK-DAG: #[[$div4timesm32:.*]] = affine_map<()[s0] -> ((s0 floordiv 4) * -32)>
 // CHECK-DAG: #[[$times8:.*]] = affine_map<()[s0] -> (s0 * 8)>
 // CHECK-DAG: #[[$id:.*]] = affine_map<()[s0] -> (s0)>
-// CHECK-DAG: #[[$div32div4timesm4:.*]] = affine_map<()[s0] -> (((s0 floordiv 32) floordiv 4) * -4)>
-// CHECK-DAG: #[[$div32:.*]] = affine_map<()[s0] -> (s0 floordiv 32)>
+// CHECK-DAG: #[[$div32mod4:.*]] = affine_map<()[s0] -> ((s0 floordiv 32) mod 4)>
 
 // CHECK-LABEL:  func.func @larger_test
 //  CHECK-SAME:  %[[I0:[0-9a-zA-Z]+]]: index,
@@ -126,10 +122,8 @@ func.func @larger_test(%0: index, %1: index, %2: index, %lb: index, %ub: index,
           // CHECK-NEXT: %[[e:.*]] = affine.apply #[[$add]]()[%[[c]], %[[idk]]]
           %e = affine.apply affine_map<()[s0, s1] -> (s0 + s1 * 8 - (s1 floordiv 4) * 32)>()[%k, %0]
 
-          // CHECK-NEXT: %[[R15:.*]] = affine.apply #[[$div32div4timesm4]]()[%[[k]]]
-          // CHECK-NEXT: %[[R16:.*]] = affine.apply #[[$add]]()[%[[idj]], %[[R15]]]
-          // CHECK-NEXT: %[[R17:.*]] = affine.apply #[[$div32]]()[%[[k]]]
-          // CHECK-NEXT: %[[f:.*]] = affine.apply #[[$add]]()[%[[R16]], %[[R17]]]
+          // CHECK-NEXT: %[[R15:.*]] = affine.apply #[[$div32mod4]]()[%[[k]]]
+          // CHECK-NEXT: %[[f:.*]] = affine.apply #[[$add]]()[%[[idj]], %[[R15]]]
           %f = affine.apply affine_map<(d0)[s0] -> ((d0 floordiv 32) mod 4 + s0)>(%k)[%j]
 
           // CHECK-NEXT: %[[g:.*]] = affine.apply #[[$add]]()[%[[b]], %[[idk]]]
diff --git a/mlir/test/Dialect/Affine/loop-fusion-4.mlir b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
index ca8099b9bb51..b059b5a98405 100644
--- a/mlir/test/Dialect/Affine/loop-fusion-4.mlir
+++ b/mlir/test/Dialect/Affine/loop-fusion-4.mlir
@@ -666,3 +666,80 @@ func.func @unrolled(%arg0: memref<2x4xf32>, %arg1: memref<1x2x4xf32>) {
   // PRODUCER-CONSUMER-MAXIMAL:          affine.load %{{.*}}[0, %{{.*}}, %{{.*}}]
   return
 }
+
+// -----
+
+// Exercises fix for crash reported at https://github.com/llvm/llvm-project/issues/139231
+
+#map = affine_map<(d0, d1) -> (d0 + d1)>
+#map1 = affine_map<(d0, d1) -> (d0 * 2 + d1 * 2)>
+module {
+  func.func @zero_candidates() {
+    %cst = arith.constant 2.221140e+03 : f32
+    %cst_0 = arith.constant 2.606200e+03 : f32
+    %cst_1 = arith.constant 3.224000e+03 : f32
+    %cst_2 = arith.constant 0.000000e+00 : f32
+    %alloc = memref.alloc() {alignment = 64 : i64} : memref<3x7x5x6xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 7 {
+        affine.for %arg2 = 0 to 5 {
+          affine.for %arg3 = 0 to 6 {
+            affine.store %cst_1, %alloc[%arg0, %arg1, %arg2, %arg3] : memref<3x7x5x6xf32>
+          }
+        }
+      }
+    }
+    %alloc_3 = memref.alloc() {alignment = 64 : i64} : memref<3x10x7x6xf32>
+    %subview = memref.subview %alloc_3[0, 2, 1, 0] [3, 7, 5, 6] [1, 1, 1, 1] : memref<3x10x7x6xf32> to memref<3x7x5x6xf32, strided<[420, 42, 6, 1], offset: 90>>
+    memref.copy %alloc, %subview : memref<3x7x5x6xf32> to memref<3x7x5x6xf32, strided<[420, 42, 6, 1], offset: 90>>
+    %alloc_4 = memref.alloc() {alignment = 64 : i64} : memref<3x10x3x6x1xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              affine.store %cst_2, %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+            }
+          }
+        }
+      }
+    }
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              affine.for %arg5 = 0 to 1 {
+                affine.for %arg6 = 0 to 2 {
+                  %0 = affine.apply #map(%arg1, %arg5)
+                  %1 = affine.apply #map1(%arg2, %arg6)
+                  %2 = affine.load %alloc_3[%arg0, %0, %1, %arg3] : memref<3x10x7x6xf32>
+                  %3 = affine.load %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+                  %4 = arith.mulf %2, %cst_0 : f32
+                  %5 = arith.addf %3, %4 : f32
+                  affine.store %5, %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    %alloc_5 = memref.alloc() {alignment = 64 : i64} : memref<3x10x3x6xf32>
+    %expand_shape = memref.expand_shape %alloc_5 [[0], [1], [2], [3, 4]] output_shape [3, 10, 3, 6, 1] : memref<3x10x3x6xf32> into memref<3x10x3x6x1xf32>
+    affine.for %arg0 = 0 to 3 {
+      affine.for %arg1 = 0 to 10 {
+        affine.for %arg2 = 0 to 3 {
+          affine.for %arg3 = 0 to 6 {
+            affine.for %arg4 = 0 to 1 {
+              %0 = affine.load %alloc_4[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+              %1 = arith.addf %0, %cst : f32
+              affine.store %1, %expand_shape[%arg0, %arg1, %arg2, %arg3, %arg4] : memref<3x10x3x6x1xf32>
+            }
+          }
+        }
+      }
+    }
+    return
+  }
+}
diff --git a/mlir/test/Dialect/Affine/simplify-min-max-ops.mlir b/mlir/test/Dialect/Affine/simplify-min-max-ops.mlir
new file mode 100644
index 000000000000..300486eb1366
--- /dev/null
+++ b/mlir/test/Dialect/Affine/simplify-min-max-ops.mlir
@@ -0,0 +1,60 @@
+// RUN: mlir-opt -pass-pipeline="builtin.module(func.func(affine-simplify-min-max))" %s | FileCheck %s
+
+// CHECK-DAG: #[[MAP_0:.*]] = affine_map<()[s0] -> (32, s0)>
+// CHECK-DAG: #[[MAP_1:.*]] = affine_map<()[s0, s1] -> (s1, s0)>
+// CHECK-DAG: #[[MAP_2:.*]] = affine_map<()[s0] -> (256, s0)>
+
+// CHECK: @min_max_full_simplify
+func.func @min_max_full_simplify() -> (index, index) {
+  %0 = test.value_with_bounds {max = 128 : index, min = 0 : index}
+  %1 = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 128 : index, min = 0 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  // CHECK-NOT: affine.min
+  // CHECK-NOT: affine.max
+  // CHECK: return %[[V0]], %[[V1]]
+  %r0 = affine.min affine_map<()[s0, s1] -> (s0, 192, s1)>()[%0, %1]
+  %r1 = affine.max affine_map<()[s0, s1] -> (s0, 192, s1)>()[%0, %1]
+  return %r0, %r1 : index, index
+}
+
+// CHECK: @min_only_simplify
+func.func @min_only_simplify() -> (index, index) {
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 512 : index, min = 0 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  // CHECK: affine.min #[[MAP_0]]()[%[[V0]]]
+  // CHECK: affine.max #[[MAP_1]]()[%[[V0]], %[[V1]]]
+  %0 = test.value_with_bounds {max = 512 : index, min = 0 : index}
+  %1 = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  %r0 = affine.min affine_map<()[s0, s1] -> (s0, 32, s1)>()[%0, %1]
+  %r1 = affine.max affine_map<()[s0, s1] -> (s0, 32, s1)>()[%0, %1]
+  return %r0, %r1 : index, index
+}
+
+// CHECK: @max_only_simplify
+func.func @max_only_simplify() -> (index, index) {
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 128 : index, min = 0 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 512 : index, min = 0 : index}
+  // CHECK: affine.min #[[MAP_1]]()[%[[V0]], %[[V1]]]
+  // CHECK: affine.max #[[MAP_2]]()[%[[V1]]]
+  %0 = test.value_with_bounds {max = 128 : index, min = 0 : index}
+  %1 = test.value_with_bounds {max = 512 : index, min = 0 : index}
+  %r0 = affine.min affine_map<()[s0, s1] -> (s0, 256, s1)>()[%0, %1]
+  %r1 = affine.max affine_map<()[s0, s1] -> (s0, 256, s1)>()[%0, %1]
+  return %r0, %r1 : index, index
+}
+
+// CHECK: @overlapping_constraints
+func.func @overlapping_constraints() -> (index, index) {
+  %0 = test.value_with_bounds {max = 192 : index, min = 0 : index}
+  %1 = test.value_with_bounds {max = 384 : index, min = 128 : index}
+  %2 = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 192 : index, min = 0 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 384 : index, min = 128 : index}
+  // CHECK: %[[V2:.*]] = test.value_with_bounds {max = 512 : index, min = 256 : index}
+  // CHECK: affine.min #[[MAP_1]]()[%[[V0]], %[[V1]]]
+  // CHECK: affine.max #[[MAP_1]]()[%[[V1]], %[[V2]]]
+  %r0 = affine.min affine_map<()[s0, s1, s2] -> (s0, s1, s2)>()[%0, %1, %2]
+  %r1 = affine.max affine_map<()[s0, s1, s2] -> (s0, s1, s2)>()[%0, %1, %2]
+  return %r0, %r1 : index, index
+}
diff --git a/mlir/test/Dialect/Affine/simplify-structures.mlir b/mlir/test/Dialect/Affine/simplify-structures.mlir
index 6f2737a98275..653c2cb52163 100644
--- a/mlir/test/Dialect/Affine/simplify-structures.mlir
+++ b/mlir/test/Dialect/Affine/simplify-structures.mlir
@@ -508,7 +508,7 @@ func.func @test_not_trivially_true_or_false_returning_three_results() -> (index,
 // -----
 
 // Test simplification of mod expressions.
-// CHECK-DAG:   #[[$MOD:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s3 + s4 * s1 + (s0 - s1) mod s2)>
+// CHECK-DAG:   #[[$MOD:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s1 * s4 + s3 + (s0 - s1) mod s2)>
 // CHECK-DAG:   #[[$SIMPLIFIED_MOD_RHS:.*]] = affine_map<()[s0, s1, s2, s3] -> (s3 mod (s2 - s0 * s1))>
 // CHECK-DAG:   #[[$MODULO_AND_PRODUCT:.*]] = affine_map<()[s0, s1, s2, s3] -> (s0 * s1 + s3 - (-s0 + s3) mod s2)>
 // CHECK-LABEL: func @semiaffine_simplification_mod
@@ -547,7 +547,7 @@ func.func @semiaffine_simplification_floordiv_and_ceildiv(%arg0: index, %arg1: i
 
 // Test simplification of product expressions.
 // CHECK-DAG:   #[[$PRODUCT:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s3 + s4 + (s0 - s1) * s2)>
-// CHECK-DAG:   #[[$SUM_OF_PRODUCTS:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s2 + s2 * s0 + s3 + s3 * s0 + s3 * s1 + s4 + s4 * s1)>
+// CHECK-DAG:   #[[$SUM_OF_PRODUCTS:.*]] = affine_map<()[s0, s1, s2, s3, s4] -> (s0 * s2 + s0 * s3 + s1 * s3 + s1 * s4 + s2 + s3 + s4)>
 // CHECK-LABEL: func @semiaffine_simplification_product
 // CHECK-SAME:  (%[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index, %[[ARG5:.*]]: index)
 func.func @semiaffine_simplification_product(%arg0: index, %arg1: index, %arg2: index, %arg3: index, %arg4: index, %arg5: index) -> (index, index) {
diff --git a/mlir/test/Dialect/Affine/transform-op-simplify-min-max-ops.mlir b/mlir/test/Dialect/Affine/transform-op-simplify-min-max-ops.mlir
index 948f434f3fa5..aa9bdf2b34eb 100644
--- a/mlir/test/Dialect/Affine/transform-op-simplify-min-max-ops.mlir
+++ b/mlir/test/Dialect/Affine/transform-op-simplify-min-max-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt  %s  --transform-interpreter | FileCheck %s
+// RUN: mlir-opt  %s  --transform-interpreter --split-input-file | FileCheck %s
 
 // CHECK-DAG: #[[MAP_0:.*]] = affine_map<()[s0] -> (32, s0)>
 // CHECK-DAG: #[[MAP_1:.*]] = affine_map<()[s0, s1] -> (s1, s0)>
diff --git a/mlir/test/Dialect/ArmNeon/invalid.mlir b/mlir/test/Dialect/ArmNeon/invalid.mlir
index be49e0e4597b..00d489b63919 100644
--- a/mlir/test/Dialect/ArmNeon/invalid.mlir
+++ b/mlir/test/Dialect/ArmNeon/invalid.mlir
@@ -91,3 +91,63 @@ func.func @usmmla_invalid_dimensions(%a: vector<8xi32>,
   %0 = arm_neon.intr.usmmla %a, %b, %c : vector<32xi8> to vector<8xi32>
   return %0 : vector<8xi32>
 }
+
+// -----
+
+func.func @bfmmla_invalid_element_type_lhs_rhs(%acc: vector<4xf32>,
+                                               %lhs: vector<8xf16>,
+                                               %rhs: vector<8xf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #1 must be a vector with length 8 of bfloat16 type values, but got 'vector<8xf16>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_lhs_rhs(%acc: vector<4xf32>,
+                                            %lhs: vector<4xbf16>,
+                                            %rhs: vector<4xbf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #1 must be a vector with length 8 of bfloat16 type values, but got 'vector<4xbf16>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<4xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @bfmmla_scalable_dimension_lhs_rhs(%acc: vector<4xf32>,
+                                             %lhs: vector<[8]xbf16>,
+                                             %rhs: vector<[8]xbf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #1 must be a vector with length 8 of bfloat16 type values, but got 'vector<[8]xbf16>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_element_type_acc(%acc: vector<4xi32>,
+                                           %lhs: vector<8xbf16>,
+                                           %rhs: vector<8xbf16>) -> vector<4xi32> {
+  // expected-error@+1 {{op operand #0 must be a vector with length 4 of 32-bit float values, but got 'vector<4xi32>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<4xi32>
+  return %0 : vector<4xi32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_acc(%acc: vector<8xf32>,
+                                        %lhs: vector<8xbf16>,
+                                        %rhs: vector<8xbf16>) -> vector<8xf32> {
+  // expected-error@+1 {{op operand #0 must be a vector with length 4 of 32-bit float values, but got 'vector<8xf32>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// -----
+
+func.func @bfmmla_scalable_dimension_acc(%acc: vector<[4]xf32>,
+                                         %lhs: vector<8xbf16>,
+                                         %rhs: vector<8xbf16>) -> vector<[4]xf32> {
+  // expected-error@+1 {{op operand #0 must be a vector with length 4 of 32-bit float values, but got 'vector<[4]xf32>'}}
+  %0 = arm_neon.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
diff --git a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
index e4f7ea150c85..5fc29c644260 100644
--- a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
+++ b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir
@@ -17,14 +17,28 @@ func.func @vector_arm_neon_mixed_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi4
 
 // -----
 
-// CHECK-LABEL: vector_arm_neon_same_types
-// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi8>, %[[A1:.*]]: vector<2x8xi8>, %[[A2:.*]]: vector<2x2xi32>
-// CHECK-DAG: %[[D0:.*]] = vector.shape_cast %[[A0]] : vector<2x8xi8> to vector<16xi8>
-// CHECK-DAG: %[[D1:.*]] = vector.shape_cast %[[A1]] : vector<2x8xi8> to vector<16xi8>
-// CHECK-DAG: %[[D2:.*]] = vector.shape_cast %[[A2]] : vector<2x2xi32> to vector<4xi32>
-// CHECK-DAG: %[[D3:.*]] = arm_neon.intr.smmla %[[D2]], %[[D0]], %[[D1]] : vector<16xi8> to vector<4xi32>
-// CHECK-DAG: %[[D4:.*]] = vector.shape_cast %[[D3]] : vector<4xi32> to vector<2x2xi32>
-func.func @vector_arm_neon_same_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+// CHECK-LABEL: vector_arm_neon_implicit_extsi
+// CHECK-SAME:    %[[LHS:.+]]: vector<2x8xi8>, %[[RHS:.+]]: vector<2x8xi8>, %[[ACC:.+]]: vector<2x2xi32>
+// CHECK:       %[[L:.+]] = vector.shape_cast %[[LHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[R:.+]] = vector.shape_cast %[[RHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[A:.+]] = vector.shape_cast %[[ACC]] : vector<2x2xi32> to vector<4xi32>
+// CHECK:       %[[M:.+]] = arm_neon.intr.smmla %[[A]], %[[L]], %[[R]] : vector<16xi8> to vector<4xi32>
+// CHECK:       %{{.+}} = vector.shape_cast %[[M]] : vector<4xi32> to vector<2x2xi32>
+func.func @vector_arm_neon_implicit_extsi(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs, %rhs, %acc : vector<2x8xi8>, vector<2x8xi8> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: vector_arm_neon_signed_signed
+// CHECK-SAME:    %[[LHS:.+]]: vector<2x8xi8>, %[[RHS:.+]]: vector<2x8xi8>, %[[ACC:.+]]: vector<2x2xi32>
+// CHECK:       %[[L:.+]] = vector.shape_cast %[[LHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[R:.+]] = vector.shape_cast %[[RHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[A:.+]] = vector.shape_cast %[[ACC]] : vector<2x2xi32> to vector<4xi32>
+// CHECK:       %[[M:.+]] = arm_neon.intr.smmla %[[A]], %[[L]], %[[R]] : vector<16xi8> to vector<4xi32>
+// CHECK:       %{{.+}} = vector.shape_cast %[[M]] : vector<4xi32> to vector<2x2xi32>
+func.func @vector_arm_neon_signed_signed(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
   %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
   %rhs_extsi = arith.extsi %rhs : vector<2x8xi8> to vector<2x8xi32>
   %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
@@ -33,11 +47,51 @@ func.func @vector_arm_neon_same_types(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>
 
 // -----
 
-// CHECK-LABEL: vector_arm_neon_without_extsi
-// CHECK-SAME:    %[[A0:.*]]: vector<2x8xi32>, %[[A1:.*]]: vector<2x8xi32>, %[[A2:.*]]: vector<2x2xi32>
-// CHECK-DAG: %[[D0:.*]] = vector.contract
-func.func @vector_arm_neon_without_extsi(%lhs: vector<2x8xi32>, %rhs: vector<2x8xi32>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
-  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs, %rhs, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+// CHECK-LABEL: vector_arm_neon_unsigned_signed
+// CHECK-SAME:    %[[LHS:.+]]: vector<2x8xi8>, %[[RHS:.+]]: vector<2x8xi8>, %[[ACC:.+]]: vector<2x2xi32>
+// CHECK:       %[[L:.+]] = vector.shape_cast %[[LHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[R:.+]] = vector.shape_cast %[[RHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[A:.+]] = vector.shape_cast %[[ACC]] : vector<2x2xi32> to vector<4xi32>
+// CHECK:       %[[M:.+]] = arm_neon.intr.usmmla %[[A]], %[[L]], %[[R]] : vector<16xi8> to vector<4xi32>
+// CHECK:       %{{.+}} = vector.shape_cast %[[M]] : vector<4xi32> to vector<2x2xi32>
+func.func @vector_arm_neon_unsigned_signed(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extui %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extsi %rhs : vector<2x8xi8> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: vector_arm_neon_unsigned_unsigned
+// CHECK-SAME:    %[[LHS:.+]]: vector<2x8xi8>, %[[RHS:.+]]: vector<2x8xi8>, %[[ACC:.+]]: vector<2x2xi32>
+// CHECK:       %[[L:.+]] = vector.shape_cast %[[LHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[R:.+]] = vector.shape_cast %[[RHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[A:.+]] = vector.shape_cast %[[ACC]] : vector<2x2xi32> to vector<4xi32>
+// CHECK:       %[[M:.+]] = arm_neon.intr.ummla %[[A]], %[[L]], %[[R]] : vector<16xi8> to vector<4xi32>
+// CHECK:       %{{.+}} = vector.shape_cast %[[M]] : vector<4xi32> to vector<2x2xi32>
+func.func @vector_arm_neon_unsigned_unsigned(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extui %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extui %rhs : vector<2x8xi8> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: vector_arm_neon_signed_unsigned
+// CHECK-SAME:    %[[LHS:.+]]: vector<2x8xi8>, %[[RHS:.+]]: vector<2x8xi8>, %[[ACC:.+]]: vector<2x2xi32>
+// CHECK:       %[[ACC_T:.+]] = vector.transpose %[[ACC]], [1, 0] : vector<2x2xi32> to vector<2x2xi32>
+// CHECK:       %[[L:.+]] = vector.shape_cast %[[LHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[R:.+]] = vector.shape_cast %[[RHS]] : vector<2x8xi8> to vector<16xi8>
+// CHECK:       %[[A:.+]] = vector.shape_cast %[[ACC_T]] : vector<2x2xi32> to vector<4xi32>
+// CHECK:       %[[M:.+]] = arm_neon.intr.usmmla %[[A]], %[[R]], %[[L]] : vector<16xi8> to vector<4xi32>
+// CHECK:       %[[OUT_T:.+]] = vector.shape_cast %[[M]] : vector<4xi32> to vector<2x2xi32>
+// CHECK:       %{{.+}} = vector.transpose %[[OUT_T]], [1, 0] : vector<2x2xi32> to vector<2x2xi32>
+func.func @vector_arm_neon_signed_unsigned(%lhs: vector<2x8xi8>, %rhs: vector<2x8xi8>, %acc : vector<2x2xi32>) -> vector<2x2xi32> {
+  %lhs_extsi = arith.extsi %lhs : vector<2x8xi8> to vector<2x8xi32>
+  %rhs_extsi = arith.extui %rhs : vector<2x8xi8> to vector<2x8xi32>
+  %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<2x8xi32>, vector<2x8xi32> into vector<2x2xi32>
   return %res : vector<2x2xi32>
 }
 
diff --git a/mlir/test/Dialect/ArmNeon/roundtrip.mlir b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
index b5df0ffa8105..60133ce0fa6f 100644
--- a/mlir/test/Dialect/ArmNeon/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
@@ -60,3 +60,15 @@ func.func @arm_neon_usmmla(%a: vector<16xi8>,
   %0 = arm_neon.intr.usmmla %c, %a, %b : vector<16xi8> to vector<4xi32>
   return %0 : vector<4xi32>
 }
+
+
+// -----
+
+// CHECK-LABEL: arm_neon_bfmmla
+func.func @arm_neon_bfmmla(%a: vector<8xbf16>,
+                           %b: vector<8xbf16>,
+                           %c: vector<4xf32>) -> vector<4xf32> {
+  // CHECK: arm_neon.intr.bfmmla {{.*}}: vector<8xbf16> to vector<4xf32>
+  %0 = arm_neon.intr.bfmmla %c, %a, %b : vector<8xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
diff --git a/mlir/test/Dialect/ArmSME/vector-legalization.mlir b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
index d56df9814f17..6cdf576272eb 100644
--- a/mlir/test/Dialect/ArmSME/vector-legalization.mlir
+++ b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
@@ -491,51 +491,6 @@ func.func @illegal_transpose_no_defining_source_op(%vec: vector<[4]x1xf32>) -> v
 
 // -----
 
-// CHECK-LABEL: @illegal_shape_cast_to_transpose_2d(
-// CHECK-SAME:                                      %[[VEC:.*]]: vector<[4]x1xf32>)
-func.func @illegal_shape_cast_to_transpose_2d(%vec: vector<[4]x1xf32>) -> vector<1x[4]xf32> {
-  // CHECK: vector.transpose %[[VEC]], [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32>
-  %0 = vector.shape_cast %vec : vector<[4]x1xf32> to vector<1x[4]xf32>
-  return %0 : vector<1x[4]xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @illegal_shape_cast_to_transpose_1d(
-// CHECK-SAME:                                      %[[VEC:.*]]: vector<[4]x1xf32>)
-func.func @illegal_shape_cast_to_transpose_1d(%vec: vector<[4]x1xf32>) -> vector<[4]xf32> {
-  // CHECK: %[[TRANSPOSE:.*]] = vector.transpose %[[VEC]], [1, 0] : vector<[4]x1xf32> to vector<1x[4]xf32>
-  // CHECK: vector.shape_cast %[[TRANSPOSE]] : vector<1x[4]xf32> to vector<[4]xf32>
-  %0 = vector.shape_cast %vec : vector<[4]x1xf32> to vector<[4]xf32>
-  return %0 : vector<[4]xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @lift_illegal_2d_shape_cast_to_memory
-func.func @lift_illegal_2d_shape_cast_to_memory(%a: index, %b: index, %memref: memref<?x?xf32>) -> vector<1x[4]xf32> {
-  // CHECK: vector.transfer_read {{.*}} : memref<?x?xf32, {{.*}}>, vector<1x[4]xf32>
-  // CHECK-NOT: vector.shape_cast
-  %pad = arith.constant 0.0 : f32
-  %illegalRead = vector.transfer_read %memref[%a, %b], %pad {in_bounds = [false, true]}: memref<?x?xf32>, vector<[4]x1xf32>
-  %cast = vector.shape_cast %illegalRead : vector<[4]x1xf32> to vector<1x[4]xf32>
-  return %cast : vector<1x[4]xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @lift_illegal_1d_shape_cast_to_memory
-func.func @lift_illegal_1d_shape_cast_to_memory(%a: index, %b: index, %memref: memref<?x?xf32>) -> vector<[4]xf32> {
-  // CHECK: vector.transfer_read {{.*}} : memref<?x?xf32, {{.*}}>, vector<1x[4]xf32>
-  // CHECK-NOT: vector.shape_cast {{.*}} : vector<[4]x1xf32> to vector<[4]xf32>
-  %pad = arith.constant 0.0 : f32
-  %illegalRead = vector.transfer_read %memref[%a, %b], %pad {in_bounds = [false, true]}: memref<?x?xf32>, vector<[4]x1xf32>
-  %cast = vector.shape_cast %illegalRead : vector<[4]x1xf32> to vector<[4]xf32>
-  return %cast : vector<[4]xf32>
-}
-
-// -----
-
 // CHECK-LABEL: @multi_tile_splat
 func.func @multi_tile_splat() -> vector<[8]x[8]xi32>
 {
@@ -656,3 +611,59 @@ func.func @vector_mask_without_maskable_op(%mask: vector<16x2xi1>, %vec: vector<
   %0 = vector.mask %mask { vector.yield %vec : vector<16x16xf32> } : vector<16x2xi1> -> vector<16x16xf32>
   return %0 : vector<16x16xf32>
 }
+
+// -----
+
+//=============================================================================
+// 1D examples - to be moved to the SVE dialect
+//=============================================================================
+
+/// TODO: Handle in_bounds
+
+// CHECK-LABEL:   func.func @xfer_read_scalable_column(
+// CHECK-SAME:      %[[IDX_0:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[IDX_1:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[PAD:.*]]: f32,
+// CHECK-SAME:      %[[SRC:.*]]: memref<?x?xf32>) -> vector<[4]x1xf32> {
+func.func @xfer_read_scalable_column(%a: index, %b: index, %pad: f32, %src: memref<?x?xf32>) -> (vector<[4]x1xf32>) {
+  // CHECK:           %[[INIT:.*]] = arith.constant dense<0.000000e+00> : vector<[4]xf32>
+  // CHECK:           %[[STEP:.*]] = arith.constant 1 : index
+  // CHECK:           %[[C4:.*]] = arith.constant 4 : index
+  // CHECK:           %[[LB:.*]] = arith.constant 0 : index
+  // CHECK:           %[[VSCALE:.*]] = vector.vscale
+  // CHECK:           %[[C4_VSCALE:.*]] = arith.muli %[[VSCALE]], %[[C4]] : index
+
+  // <scf.for>
+  // CHECK:           %[[SCF:.*]] = scf.for %[[IND_VAR:.*]] = %[[LB]] to %[[C4_VSCALE]] step %[[STEP]] iter_args(%[[SCF_RES:.*]] = %[[INIT]]) -> (vector<[4]xf32>) {
+  // CHECK:             %[[IDX_0_UPDATED:.*]] = arith.addi %[[IND_VAR]], %[[IDX_0]] : index
+  // CHECK:             %[[VAL_10:.*]] = memref.load %[[SRC]][%[[IDX_0_UPDATED]], %[[IDX_1]]] : memref<?x?xf32>
+  // CHECK:             %[[RES_UPDATED:.*]] = vector.insert %[[VAL_10]], %[[SCF_RES]] [%[[IND_VAR]]] : f32 into vector<[4]xf32>
+  // CHECK:             scf.yield %[[RES_UPDATED]] : vector<[4]xf32>
+  // CHECK:           }
+
+  // <shape-cast>
+  // CHECK:           %[[SC:.*]] = vector.shape_cast %[[SCF]] : vector<[4]xf32> to vector<[4]x1xf32>
+  // CHECK:           return %[[SC]]
+  %read = vector.transfer_read %src[%a, %b], %pad : memref<?x?xf32>, vector<[4]x1xf32>
+  return %read : vector<[4]x1xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @negative_xfer_read_scalable_column_x2
+func.func @negative_xfer_read_scalable_column_x2(%a: index, %b: index, %pad: f32, %src: memref<?x?xf32>) -> (vector<[4]x2xf32>) {
+  // CHECK-NOT: scf.for
+  // CHECK-NOT: memref.load
+  %read = vector.transfer_read %src[%a, %b], %pad : memref<?x?xf32>, vector<[4]x2xf32>
+  return %read : vector<[4]x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @negative_xfer_read_scalable_column_scalable_trailing_dim
+func.func @negative_xfer_read_scalable_column_scalable_trailing_dim(%a: index, %b: index, %pad: f32, %src: memref<?x?xf32>) -> (vector<4x[1]xf32>) {
+  // CHECK-NOT: scf.for
+  // CHECK-NOT: memref.load
+  %read = vector.transfer_read %src[%a, %b], %pad : memref<?x?xf32>, vector<4x[1]xf32>
+  return %read : vector<4x[1]xf32>
+}
diff --git a/mlir/test/Dialect/ArmSVE/invalid.mlir b/mlir/test/Dialect/ArmSVE/invalid.mlir
index a021d4393107..1d2529be0560 100644
--- a/mlir/test/Dialect/ArmSVE/invalid.mlir
+++ b/mlir/test/Dialect/ArmSVE/invalid.mlir
@@ -72,3 +72,63 @@ func.func @arm_sve_psel_bad_vector_type(%a : vector<[7]xi1>, %index: index) {
   arm_sve.psel %a, %a[%index] : vector<[7]xi1>, vector<[7]xi1>
   return
 }
+
+// -----
+
+func.func @bfmmla_invalid_element_type_lhs_rhs(%acc: vector<[4]xf32>,
+                                               %lhs: vector<[8]xf16>,
+                                               %rhs: vector<[8]xf16>) -> vector<[4]xf32> {
+  // expected-error@+1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[8]xf16>'}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xf16> to vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_lhs_rhs(%acc: vector<[4]xf32>,
+                                            %lhs: vector<[4]xbf16>,
+                                            %rhs: vector<[4]xbf16>) -> vector<[4]xf32> {
+  // expected-error@+1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<[4]xbf16>}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[4]xbf16> to vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+
+// -----
+
+func.func @bfmmla_fixed_dimension_lhs_rhs(%acc: vector<[4]xf32>,
+                                          %lhs: vector<8xbf16>,
+                                          %rhs: vector<8xbf16>) -> vector<[4]xf32> {
+  // expected-error@+1 {{operand #1 must be scalable vector of bfloat16 type values of length 8, but got 'vector<8xbf16>}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<8xbf16> to vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_element_type_acc(%acc: vector<[4]xi32>,
+                                           %lhs: vector<[8]xbf16>,
+                                           %rhs: vector<[8]xbf16>) -> vector<[4]xi32> {
+  // expected-error@+1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[4]xi32>'}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[4]xi32>
+  return %0 : vector<[4]xi32>
+}
+
+// -----
+
+func.func @bfmmla_invalid_dimension_acc(%acc: vector<[8]xf32>,
+                                        %lhs: vector<[8]xbf16>,
+                                        %rhs: vector<[8]xbf16>) -> vector<[8]xf32> {
+  // expected-error@+1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<[8]xf32>'}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<[8]xf32>
+  return %0 : vector<[8]xf32>
+}
+
+// -----
+
+func.func @bfmmla_fixed_dimension_acc(%acc: vector<4xf32>,
+                                      %lhs: vector<[8]xbf16>,
+                                      %rhs: vector<[8]xbf16>) -> vector<4xf32> {
+  // expected-error@+1 {{operand #0 must be scalable vector of 32-bit float values of length 4, but got 'vector<4xf32>'}}
+  %0 = arm_sve.intr.bfmmla %acc, %lhs, %rhs : vector<[8]xbf16> to vector<4xf32>
+  return %0 : vector<4xf32>
+}
diff --git a/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir b/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
new file mode 100644
index 000000000000..49bd2eddbded
--- /dev/null
+++ b/mlir/test/Dialect/ArmSVE/legalize-transfer-read.mlir
@@ -0,0 +1,257 @@
+// RUN: mlir-opt --arm-sve-legalize-vector-storage --split-input-file %s | FileCheck %s
+
+
+// Test the `LegalizeTransferRead` pattern
+// (mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp)
+
+// -----
+
+// This is the base case, unremarkable in any way, except that it's our main
+// motivating example and use case.
+
+// CHECK-LABEL:       @base_case
+// CHECK-SAME:          %[[I:.+]]: index, %[[J:.+]]: index, %[[M:.+]]:
+// CHECK-DAG:           %[[PAD:.+]] = arith.constant 123 : i8
+// CHECK-DAG:           %[[C0:.+]] = arith.constant 0 : index
+// CHECK:               %[[COLLAPSE:.+]] = memref.collapse_shape %[[M]]
+// CHECK-SAME{LITERAL}:   [[0], [1], [2, 3]]
+// CHECK-SAME:            : memref<?x?x?x8xi8> into memref<?x?x?xi8>
+// CHECK-NEXT:          %[[T0:.+]] = vector.transfer_read %[[COLLAPSE]][%[[I]], %[[J]], %[[C0]]], %[[PAD]] {in_bounds = [true]}
+// CHECK-SAME:            : memref<?x?x?xi8>, vector<[32]xi8>
+// CHECK-NEXT:          %[[T1:.+]] = vector.shape_cast %[[T0]] : vector<[32]xi8> to vector<[4]x8xi8>
+// CHECK-NEXT:          return %[[T1]] : vector<[4]x8xi8>
+
+func.func @base_case(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<[4]x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true]} : memref<?x?x?x8xi8>, vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test the case where the scalable dimension is not the second-to-last.
+
+// CHECK-LABEL:       @with_3d_vector
+// CHECK-SAME:          %[[I:.+]]: index, %[[J:.+]]: index, %[[M:.+]]:
+// CHECK-DAG:           %[[PAD:.+]] = arith.constant 123 : i8
+// CHECK-DAG:           %[[COLLAPSED:.+]] = memref.collapse_shape %[[M]]
+// CHECK-SAME{LITERAL}:   [[0], [1, 2, 3]]
+// CHECK-SAME:            : memref<?x?x2x8xi8> into memref<?x?xi8>
+// CHECK-NEXT:          %[[T0:.+]] = vector.transfer_read %[[COLLAPSED]][%[[I]], %[[J]]], %[[PAD]] {in_bounds = [true]}
+// CHECK-SAME:            : memref<?x?xi8>, vector<[64]xi8>
+// CHECK-NEXT:          %[[T1:.+]] = vector.shape_cast %[[T0]] : vector<[64]xi8> to vector<[4]x2x8xi8>
+// CHECK-NEXT:          return %[[T1]] : vector<[4]x2x8xi8>
+
+func.func @with_3d_vector(%i : index, %j : index, %M : memref<?x?x2x8xi8>) -> vector<[4]x2x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true, true]} : memref<?x?x2x8xi8>, vector<[4]x2x8xi8>
+
+  return %A : vector<[4]x2x8xi8>
+}
+
+// -----
+
+// Test the case when the vector is already LLVM-legal (fixed).
+
+// CHECK-LABEL: @negative_vector_legal_fixed
+// CHECK-NOT: memref.collapse
+
+func.func @negative_vector_legal_fixed(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<8x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true]} : memref<?x?x?x8xi8>, vector<8x8xi8>
+
+  return %A : vector<8x8xi8>
+}
+
+// -----
+
+// Test the case when the vector is already LLVM-legal (single-dimension scalable).
+
+// CHECK-LABEL: @negative_vector_legal_1d_scalable
+// CHECK-NOT: memref.collapse
+
+func.func @negative_vector_legal_1d_scalable(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<[8]xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true]} : memref<?x?x?x8xi8>, vector<[8]xi8>
+
+  return %A : vector<[8]xi8>
+}
+
+// -----
+
+// Test the case when the vector is already LLVM-legal (single trailing
+// scalable dimension).
+
+// CHECK-LABEL: @negative_vector_legal_trailing_scalable_dim
+// CHECK-NOT: memref.collapse
+
+func.func @negative_vector_legal_trailing_scalable_dim(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<8x[8]xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true]} : memref<?x?x?x8xi8>, vector<8x[8]xi8>
+
+  return %A : vector<8x[8]xi8>
+}
+
+// -----
+
+// Test the case of unsupported vector type (more than one scalable dimension)
+
+// CHECK-LABEL: @negative_vector_type_two_scalable_dims
+// CHECK-NOT: memref.collapse
+
+func.func @negative_vector_type_two_scalable_dims(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<[8]x[8]x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true, true]} : memref<?x?x?x8xi8>, vector<[8]x[8]x8xi8>
+
+  return %A : vector<[8]x[8]x8xi8>
+}
+
+// -----
+
+// Test the case of reading from a tensor - not supported, since the
+// transform reasons about memory layouts.
+
+// CHECK-LABEL: @negative_tensor_transfer
+// CHECK-NOT: memref.collapse
+
+func.func @negative_tensor_transfer(%i : index, %j : index, %M : tensor<?x?x?x8xi8>) -> vector<[4]x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true]} : tensor<?x?x?x8xi8>, vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test the case when the transfer is discontiguous because the memref
+// is discontiguous.
+// There are other ways to make a memref discontiguous. The transformation
+// is not concerned with the particular reason a memref is discontiguous, but
+// only with the fact. Therefore there are no variations with the memref made
+// discontiguous by some other mechanism.
+
+// CHECK-LABEL: @negative_discontig_mem
+// CHECK-NOT: memref.collapse
+
+#strides = strided<[?, ?, 16, 1]>
+
+func.func @negative_discontig_mem(%i : index, %j : index, %M : memref<?x?x?x8xi8, #strides>) -> vector<[4]x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true]} : memref<?x?x?x8xi8, #strides>, vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test the case when the transformation is not applied because of
+// a non-trivial permutation map (broadcast).
+
+// CHECK-LABEL: @negative_broadcast
+// CHECK-NOT: memref.collapse
+
+#perm = affine_map<(i, j, k, p) -> (k, 0)>
+
+func.func @negative_broadcast(%i : index, %j : index, %M : memref<?x?x?x8xi8>) -> vector<[4]x8xi8> {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {permutation_map = #perm, in_bounds = [true, true] } : memref<?x?x?x8xi8>, vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test the case of a masked read - not supported right now.
+// (see mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp)
+
+// CHECK-LABEL: @negative_masked
+// CHECK-NOT: memref.collapse
+
+func.func @negative_masked(
+  %i : index, %j : index,
+  %M : memref<?x?x?x8xi8>, %mask : vector<[4]x8xi1>) -> vector<[4]x8xi8> {
+  
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.mask %mask {
+    vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true] } : memref<?x?x?x8xi8>, vector<[4]x8xi8>
+  } : vector<[4]x8xi1> -> vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test case with a mask operand - not supported right now.
+// (see mlir/lib/Dialect/ArmSVE/Transforms/LegalizeVectorStorage.cpp)
+
+// CHECK-LABEL: @negative_with_mask
+// CHECK-NOT: memref.collapse
+
+func.func @negative_with_mask(
+  %i : index, %j : index,
+  %M : memref<?x?x?x8xi8>, %mask : vector<[4]x8xi1>) -> vector<[4]x8xi8> {
+  
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad, %mask {in_bounds = [true, true] } : memref<?x?x?x8xi8>, vector<[4]x8xi8>
+
+  return %A : vector<[4]x8xi8>
+}
+
+// -----
+
+// Test the case when the dimensions to collapse (excluding the scalable one)
+// of the vector and the memref do not match (static non matching dimension).
+
+// CHECK-LABEL: @negative_non_matching_dim_static
+// CHECK-NOT: memref.collapse
+
+func.func @negative_non_matching_dim_static(%i : index, %j : index,  %M : memref<?x?x?x8xi8>) -> vector<[4]x4xi8> {
+  
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true] } : memref<?x?x?x8xi8>, vector<[4]x4xi8>
+
+  return %A : vector<[4]x4xi8>
+}
+
+// -----
+
+// Test the case when the dimensions to collapse (excluding the scalable one)
+// of the vector and the memref do not match (dynamic non matching dimension).
+
+// CHECK-LABEL: @negative_non_matching_dim_dynamic
+// CHECK-NOT: memref.collapse
+
+func.func @negative_non_matching_dim_dynamic(%i : index, %j : index,  %M : memref<?x?x?x?xi8>) -> vector<[4]x4xi8> {
+  
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 123 : i8
+
+  %A = vector.transfer_read %M[%i, %j, %c0, %c0], %pad {in_bounds = [true, true] } : memref<?x?x?x?xi8>, vector<[4]x4xi8>
+
+  return %A : vector<[4]x4xi8>
+}
diff --git a/mlir/test/Dialect/ArmSVE/roundtrip.mlir b/mlir/test/Dialect/ArmSVE/roundtrip.mlir
index 64e0cff39eb0..b7b9329f1cb5 100644
--- a/mlir/test/Dialect/ArmSVE/roundtrip.mlir
+++ b/mlir/test/Dialect/ArmSVE/roundtrip.mlir
@@ -55,6 +55,16 @@ func.func @arm_sve_usmmla(%a: vector<[16]xi8>,
 
 // -----
 
+func.func @arm_sve_bfmmla(%a: vector<[8]xbf16>,
+                          %b: vector<[8]xbf16>,
+                          %c: vector<[4]xf32>) -> vector<[4]xf32> {
+  // CHECK: arm_sve.intr.bfmmla {{.*}}: vector<[8]xbf16> to vector<[4]xf32>
+  %0 = arm_sve.intr.bfmmla %c, %a, %b : vector<[8]xbf16> to vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+
+// -----
+
 func.func @arm_sve_masked_arithi(%a: vector<[4]xi32>,
                             %b: vector<[4]xi32>,
                             %c: vector<[4]xi32>,
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
index da3c26ce36ba..803173201183 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
@@ -272,10 +272,10 @@ func.func @materialize_in_dest_raw(%f: f32, %f2: f32, %idx: index) -> (tensor<5x
 
 // -----
 
-// CHECK-LABEL: func.func @test_dialect_op(
+// CHECK:       func.func @custom_op(
 // CHECK-SAME:    %[[ARG:.*]]: !test.test_tensor<[32, 64], f64>
 // CHECK-SAME:  ) -> !test.test_tensor<[32, 128], f64> {
-func.func @test_dialect_op(%arg: !test.test_tensor<[32, 64], f64>)
+func.func @custom_op(%arg: !test.test_tensor<[32, 64], f64>)
     -> !test.test_tensor<[32, 128], f64> {
   // CHECK: %[[MEMREF:.*]] = bufferization.to_buffer %[[ARG]]
   // CHECK: %[[DUMMY:.*]] = "test.dummy_memref_op"(%[[MEMREF]])
@@ -288,3 +288,22 @@ func.func @test_dialect_op(%arg: !test.test_tensor<[32, 64], f64>)
   // CHECK: return %[[OUT]]
   return %out : !test.test_tensor<[32, 128], f64>
 }
+
+// -----
+
+// CHECK:       func.func @custom_origin_op()
+// CHECK-SAME:  -> !test.test_tensor<[42], f64> {
+func.func @custom_origin_op() -> !test.test_tensor<[42], f64> {
+  // CHECK: %[[MEMREF:.*]] = "test.create_memref_op"() : ()
+  // CHECK-SAME: -> !test.test_memref<[21], f64>
+  // CHECK: %[[DUMMY:.*]] = "test.dummy_memref_op"(%[[MEMREF]])
+  // CHECK-SAME: : (!test.test_memref<[21], f64>)
+  // CHECK-SAME: -> !test.test_memref<[42], f64>
+  %in = "test.create_tensor_op"() : () -> !test.test_tensor<[21], f64>
+  %out = "test.dummy_tensor_op"(%in) : (!test.test_tensor<[21], f64>)
+    -> !test.test_tensor<[42], f64>
+
+  // CHECK: %[[OUT:.*]] = bufferization.to_tensor %[[DUMMY]]
+  // CHECK: return %[[OUT]]
+  return %out : !test.test_tensor<[42], f64>
+}
diff --git a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir
new file mode 100644
index 000000000000..c67a0c197fcd
--- /dev/null
+++ b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir
@@ -0,0 +1,40 @@
+// RUN: mlir-opt --wrap-emitc-func-in-class='named-attribute=emitc.name_hint' %s | FileCheck %s
+
+module attributes { } {
+  emitc.func @model(%arg0: !emitc.array<1xf32> {emitc.name_hint = "another_feature"}, 
+   %arg1: !emitc.array<1xf32> {emitc.name_hint = "some_feature"},
+   %arg2: !emitc.array<1xf32> {emitc.name_hint = "output_0"}) attributes { } {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    %1 = subscript %arg1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    %2 = load %1 : <f32>
+    %3 = subscript %arg0[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    %4 = load %3 : <f32>
+    %5 = add %2, %4 : (f32, f32) -> f32
+    %6 = subscript %arg2[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    assign %5 : f32 to %6 : <f32>
+    return
+  }
+}
+
+
+// CHECK: module {
+// CHECK-NEXT:   emitc.class @modelClass {
+// CHECK-NEXT:     emitc.field @fieldName0 : !emitc.array<1xf32> {emitc.name_hint = "another_feature"}
+// CHECK-NEXT:     emitc.field @fieldName1 : !emitc.array<1xf32>  {emitc.name_hint = "some_feature"}
+// CHECK-NEXT:     emitc.field @fieldName2 : !emitc.array<1xf32>  {emitc.name_hint = "output_0"}
+// CHECK-NEXT:     emitc.func @execute() {
+// CHECK-NEXT:       get_field @fieldName0 : !emitc.array<1xf32>
+// CHECK-NEXT:       get_field @fieldName1 : !emitc.array<1xf32>
+// CHECK-NEXT:       get_field @fieldName2 : !emitc.array<1xf32>
+// CHECK-NEXT:       "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK-NEXT:       load {{.*}} : <f32>
+// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK-NEXT:       load {{.*}} : <f32>
+// CHECK-NEXT:       add {{.*}}, {{.*}} : (f32, f32) -> f32
+// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK-NEXT:       assign {{.*}} : f32 to {{.*}} : <f32>
+// CHECK-NEXT:       return
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
diff --git a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir
new file mode 100644
index 000000000000..92ed20c4b14e
--- /dev/null
+++ b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir
@@ -0,0 +1,17 @@
+// RUN: mlir-opt --wrap-emitc-func-in-class %s | FileCheck %s
+
+emitc.func @foo(%arg0 : !emitc.array<1xf32>) {
+  emitc.call_opaque "bar" (%arg0) : (!emitc.array<1xf32>) -> ()
+  emitc.return
+}
+
+// CHECK: module {
+// CHECK-NEXT:   emitc.class @fooClass {
+// CHECK-NEXT:     emitc.field @fieldName0 : !emitc.array<1xf32>
+// CHECK-NEXT:     emitc.func @execute() {
+// CHECK-NEXT:       %0 = get_field @fieldName0 : !emitc.array<1xf32>
+// CHECK-NEXT:       call_opaque "bar"(%0) : (!emitc.array<1xf32>) -> ()
+// CHECK-NEXT:       return
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
diff --git a/mlir/test/Dialect/GPU/invalid.mlir b/mlir/test/Dialect/GPU/invalid.mlir
index ce1be7b5618f..162ff0662e91 100644
--- a/mlir/test/Dialect/GPU/invalid.mlir
+++ b/mlir/test/Dialect/GPU/invalid.mlir
@@ -478,6 +478,84 @@ func.func @shuffle_unsupported_type_vec(%arg0 : vector<[4]xf32>, %arg1 : i32, %a
 
 // -----
 
+func.func @rotate_mismatching_type(%arg0 : f32) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op failed to verify that all of {value, rotateResult} have same type}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (i32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_type(%arg0 : index) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'index'}}
+  %rotate, %valid = gpu.rotate %arg0, %offset, %width : index
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_type_vec(%arg0 : vector<[4]xf32>) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op operand #0 must be Integer or Float or fixed-length vector of Integer or Float values of ranks 1, but got 'vector<[4]xf32>'}}
+  %rotate, %valid = gpu.rotate %arg0, %offset, %width : vector<[4]xf32>
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_width(%arg0 : f32) {
+  %offset = arith.constant 4 : i32
+  %width = arith.constant 15 : i32
+  // expected-error@+1 {{op width must be a power of two}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_offset(%arg0 : f32) {
+  %offset = arith.constant 16 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset must be in the range [0, 16)}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_unsupported_offset_minus(%arg0 : f32) {
+  %offset = arith.constant -1 : i32
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset must be in the range [0, 16)}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_offset_non_constant(%arg0 : f32, %offset : i32) {
+  %width = arith.constant 16 : i32
+  // expected-error@+1 {{op offset is not a constant value}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
+func.func @rotate_width_non_constant(%arg0 : f32, %width : i32) {
+  %offset = arith.constant 0 : i32
+  // expected-error@+1 {{op width is not a constant value}}
+  %rotate, %valid = "gpu.rotate"(%arg0, %offset, %width) : (f32, i32, i32) -> (f32, i1)
+  return
+}
+
+// -----
+
 module {
   gpu.module @gpu_funcs {
     // expected-error @+1 {{custom op 'gpu.func' gpu.func requires named arguments}}
diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir
index 9dbe16774f51..2aef80f73feb 100644
--- a/mlir/test/Dialect/GPU/ops.mlir
+++ b/mlir/test/Dialect/GPU/ops.mlir
@@ -140,6 +140,10 @@ module attributes {gpu.container_module} {
       // CHECK: gpu.shuffle idx %{{.*}}, %{{.*}}, %{{.*}} : f32
       %shfl3, %pred3 = gpu.shuffle idx %arg0, %offset, %width : f32
 
+      // CHECK: gpu.rotate %{{.*}}, %{{.*}}, %{{.*}} : f32
+      %rotate_width = arith.constant 16 : i32
+      %rotate, %pred4 = gpu.rotate %arg0, %offset, %rotate_width : f32
+
       "gpu.barrier"() : () -> ()
 
       "some_op"(%bIdX, %tIdX) : (index, index) -> ()
diff --git a/mlir/test/Dialect/GPU/transform-gpu.mlir b/mlir/test/Dialect/GPU/transform-gpu.mlir
index 09ae0f4af686..bfe3cd03cb20 100644
--- a/mlir/test/Dialect/GPU/transform-gpu.mlir
+++ b/mlir/test/Dialect/GPU/transform-gpu.mlir
@@ -691,3 +691,67 @@ module attributes {transform.with_named_sequence} {
       transform.yield
   }
 }
+
+// -----
+
+#map = affine_map<(d0) -> (d0 * 128)>
+#map1 = affine_map<(d0) -> (d0 * 32)>
+
+// CHECK-DAG: #[[$MAPB:.*]] = affine_map<()[s0] -> (s0 * 128)>
+// CHECK-DAG: #[[$MAPLANE:.*]] = affine_map<()[s0, s1] -> ((s0 + s1 * 73) mod 32)>
+// CHECK-DAG: #[[$MAPI:.*]] = affine_map<()[s0, s1] -> (s0 * 32 + s1 * 2336 - ((s0 + s1 * 73) floordiv 2) * 64)>
+// CHECK-DAG: #[[$MAPJ:.*]] = affine_map<()[s0, s1] -> ((((s0 + s1 * 73) mod 32) floordiv 2) * 32)>
+
+// CHECK-LABEL: func.func @simple_fill(
+func.func @simple_fill(%arg0: memref<128x256xf32>) -> memref<128x256xf32> {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<16x32xf32>
+    //   CHECK:   %[[C6:.*]] = arith.constant 6 : index
+    //   CHECK:   gpu.launch
+  scf.forall (%arg1) in (1) {
+    //   CHECK:     %[[BIDX:.*]] = gpu.block_id  x
+    //   CHECK:     %[[BLX:.*]] = affine.apply #[[$MAPB]]()[%[[BIDX]]]
+    %0 = affine.apply #map(%arg1)
+    %subview = memref.subview %arg0[%0, 0] [128, 256] [1, 1]
+      : memref<128x256xf32> to memref<128x256xf32, strided<[256, 1], offset: ?>>
+
+    // %arg2 and %arg3 map to lanes [0, 6) and are turned into epxressions
+    // involving threadIdx.x/y by the map_nested_forall_to_threads
+    // transformation. This results in a if (linear_thread_id < 6) conditional.
+    scf.forall (%arg2, %arg3) in (2, 3) {
+      //       CHECK:     %[[TIDX:.*]] = gpu.thread_id  x
+      //       CHECK:     %[[TIDY:.*]] = gpu.thread_id  y
+      //       CHECK:     %[[LID:.*]] = affine.apply #[[$MAPLANE]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:     %[[COND:.*]] = arith.cmpi ult, %[[LID]], %[[C6]]
+      //       CHECK:     scf.if %[[COND]]
+      //       CHECK:       %[[I:.*]] = affine.apply #[[$MAPI]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:       %[[J:.*]] = affine.apply #[[$MAPJ]]()[%[[TIDX]], %[[TIDY]]]
+      //       CHECK:       memref.subview %{{.*}}[%[[I]], %[[J]]]
+      %1 = affine.apply #map1(%arg2)
+      %2 = affine.apply #map1(%arg3)
+      %subview_0 = memref.subview %subview[%1, %2] [16, 32] [1, 1] 
+        : memref<128x256xf32, strided<[256, 1], offset: ?>> to memref<16x32xf32, strided<[256, 1], offset: ?>>
+      vector.transfer_write %cst, %subview_0[%c0, %c0] {in_bounds = [true, true]} 
+        : vector<16x32xf32>, memref<16x32xf32, strided<[256, 1], offset: ?>>
+
+    // This could be obtained e.g. if a previous transformation mapped this loop
+    // to lanes. This can aslo be written by hand as valid IR.
+    } {mapping = [#gpu.lane<linear_dim_0>, #gpu.lane<linear_dim_1>]}
+  } {mapping = [#gpu.block<x>]}
+  return %arg0 : memref<128x256xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
+    %func = transform.structured.match ops{["func.func"]} in %module_op
+      : (!transform.any_op) -> !transform.any_op
+    %gpu_launch = transform.gpu.map_forall_to_blocks %func generate_gpu_launch
+      : (!transform.any_op) -> !transform.any_op
+
+    // This transformation maps scf.forall ivs to a particular mapping of thread
+    // ids (laneid, threadid, warpid or warpgroupid).
+    transform.gpu.map_nested_forall_to_threads %gpu_launch block_dims = [73, 5, 1]
+      : (!transform.any_op) -> !transform.any_op
+      transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 251ca716c7a7..bd1106e304c6 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -1875,7 +1875,7 @@ llvm.mlir.global @bad_struct_array_init_elements() : !llvm.array<1x!llvm.struct<
   llvm.return %0 : !llvm.array<1x!llvm.struct<(i32, f32)>>
 }
 
-// ----
+// -----
 
 llvm.mlir.global internal constant @bad_array_attr_simple_type() : !llvm.array<2 x f64> {
   // expected-error@below {{'llvm.mlir.constant' op for array with an array attribute must have a struct element type}}
@@ -1883,10 +1883,51 @@ llvm.mlir.global internal constant @bad_array_attr_simple_type() : !llvm.array<2
   llvm.return %0 : !llvm.array<2 x f64>
 }
 
-// ----
+// -----
 
 llvm.func @inlineAsmMustTail(%arg0: i32, %arg1 : !llvm.ptr) {
   // expected-error@+1 {{op tail call kind 'musttail' is not supported}}
   %8 = llvm.inline_asm tail_call_kind = <musttail> "foo", "=r,=r,r" %arg0 : (i32) -> !llvm.struct<(i8, i8)>
   llvm.return
 }
+
+// -----
+
+llvm.func @invalid_xevm_prefetch(%arg0: !llvm.ptr) {
+  // expected-error@+1 {{op operand #0 must be LLVM pointer in address space 1 or LLVM pointer in address space 4}}
+  xevm.prefetch %arg0 <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_mma(%loaded_c_casted: vector<4xf32>, %loaded_a: vector<8xi16>, %loaded_b_casted: vector<8xi32>) -> vector<8xf32> {
+  // expected-error@+1 {{op type of C operand must match result type}}
+  %c_result = xevm.mma %loaded_a, %loaded_b_casted, %loaded_c_casted {shape = <m = 8, n = 16, k = 16>, types = <d = f32, a = f16, b = f16, c = f32>} : (vector<8xi16>, vector<8xi32>, vector<4xf32>) -> vector<8xf32>
+  llvm.return %c_result : vector<8xf32>
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_1(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32, %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // expected-error@+1 {{op expecting tile_width to be between 1 and 8}}
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted <{elem_size_in_bits=64 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_2(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32, %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // expected-error@+1 {{op expecting elem_size_in_bits to be 8, 16, 32, or 64}}
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted <{elem_size_in_bits=18 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  llvm.return
+}
+
+// -----
+
+llvm.func @invalid_xevm_matrix_3(%a: !llvm.ptr<1>, %base_width_a: i32, %base_height_a: i32, %base_pitch_a: i32, %x: i32, %y: i32) -> vector<8xi16> {
+  // expected-error@+1 {{op result size of 128 bits does not match the expected size of 208 bits}}
+  %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y <{elem_size_in_bits=16 : i32, tile_width=26 : i32, tile_height=8 : i32, v_blocks=1 : i32, transpose=false, pack_register=false, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  llvm.return %loaded_a : vector<8xi16>
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/xevm.mlir b/mlir/test/Dialect/LLVMIR/xevm.mlir
new file mode 100644
index 000000000000..3dd5f872f898
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/xevm.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics | FileCheck %s
+
+// CHECK-LABEL: func.func @blockload2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32)
+func.func @blockload2d(%a: !llvm.ptr<1>, %base_width_a: i32, %base_height_a: i32,
+  %base_pitch_a: i32, %x: i32, %y: i32) -> vector<8xi16> {
+  // CHECK: %[[VAR0:.*]] = xevm.blockload2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]]
+  // CHECK-DAG: elem_size_in_bits = 16 : i32
+  // CHECK-DAG: tile_width = 16 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK-DAG: v_blocks = 1 : i32
+  // CHECK-DAG: transpose = false
+  // CHECK-DAG: pack_register = false
+  // CHECK-DAG: cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>
+  // CHECK: (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y
+    <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+    transpose=false, pack_register=false, cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}>
+    : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+  return %loaded_a : vector<8xi16>
+}
+
+// -----
+// CHECK-LABEL: func.func @blockstore2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32,
+// CHECK-SAME: %[[ARG6:.*]]: vector<8xi32>)
+func.func @blockstore2d(%c: !llvm.ptr<1>, %base_width_c: i32, %base_height_c: i32,
+  %base_pitch_c: i32, %x: i32, %y: i32, %c_result_casted: vector<8xi32>) {
+  // CHECK: xevm.blockstore2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]], %[[ARG6]]
+  // CHECK-DAG: elem_size_in_bits = 32 : i32
+  // CHECK-DAG: tile_width = 16 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK: (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted
+    <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32}>
+    : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @blockprefetch2d(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>,
+// CHECK-SAME: %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[ARG3:.*]]: i32, %[[ARG4:.*]]: i32, %[[ARG5:.*]]: i32)
+func.func @blockprefetch2d(%ptr: !llvm.ptr<1>, %base_width: i32, %base_height: i32,
+  %base_pitch: i32, %x: i32, %y: i32) {
+  // CHECK: xevm.blockprefetch2d %[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]], %[[ARG4]], %[[ARG5]]
+  // CHECK-DAG: elem_size_in_bits = 8 : i32
+  // CHECK-DAG: tile_width = 32 : i32
+  // CHECK-DAG: tile_height = 8 : i32
+  // CHECK-DAG: v_blocks = 1 : i32
+  // CHECK-DAG: cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>
+  // CHECK:  (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+  xevm.blockprefetch2d %ptr, %base_width, %base_height, %base_pitch, %x, %y <{elem_size_in_bits=8 : i32,
+    tile_width=32 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+    cache_control=#xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32)
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @mma(
+// CHECK-SAME: %[[ARG0:.*]]: vector<8xf32>, %[[ARG1:.*]]: vector<8xi16>, %[[ARG2:.*]]: vector<8xi32>)
+func.func @mma(%loaded_c_casted: vector<8xf32>, %loaded_a: vector<8xi16>, %loaded_b_casted: vector<8xi32>) -> vector<8xf32> {
+  // CHECK: %0 = xevm.mma %[[ARG1]], %[[ARG2]], %[[ARG0]] {shape = <m = 8, n = 16, k = 16>,
+  // CHECK-SAME: types = <d = f32, a = f16, b = f16, c = f32>} : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+  %c_result = xevm.mma %loaded_a, %loaded_b_casted, %loaded_c_casted { shape=<m=8, n=16, k=16>,
+    types=<d=f32, a=f16, b=f16, c=f32> } : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+  return %c_result : vector<8xf32>
+}
+
+// -----
+// CHECK-LABEL: func.func @memfence()
+func.func @memfence() {
+  // CHECK: xevm.memfence
+  // CHECK-DAG: addrspace = #xevm.addr_space<global>
+  // CHECK-DAG: scope = #xevm.mem_scope<workgroup>
+  xevm.memfence <{addrspace=#xevm.addr_space<global>, scope=#xevm.mem_scope<workgroup>}>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @prefetch(
+// CHECK-SAME: %[[ARG0:.*]]: !llvm.ptr<1>)
+func.func @prefetch(%ptr: !llvm.ptr<1>) {
+  // CHECK: xevm.prefetch %[[ARG0]]
+  // CHECK-SAME: <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>)
+  xevm.prefetch %ptr <{cache_control = #xevm.load_cache_control<L1uc_L2uc_L3uc>}> : (!llvm.ptr<1>)
+  return
+}
+
+// -----
+// CHECK-LABEL: @xevm_module [#xevm.target<O = 3, chip = "pvc">] {
+gpu.module @xevm_module [#xevm.target<O = 3, chip = "pvc">]{
+}
diff --git a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
index 31c9e9ed3c50..6fc8d9f152f4 100644
--- a/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
+++ b/mlir/test/Dialect/Linalg/data-layout-propagation.mlir
@@ -435,6 +435,40 @@ func.func @elem_pack_transpose_outer_dims(%arg0: tensor<128x256xi32>, %init: ten
 
 // -----
 
+#map0 = affine_map<(d0, d1) -> (d0, d1)>
+func.func @elem_pack_transpose_outer_dims_unused_init(%arg0: tensor<128x256xi32>, %init: tensor<128x256xi32>) -> tensor<16x4x32x16xi32>{
+  %elem = linalg.generic {indexing_maps = [#map0, #map0], iterator_types = ["parallel", "parallel"]}
+      ins(%arg0 : tensor<128x256xi32>)
+      outs(%init : tensor<128x256xi32>) {
+    ^bb0(%arg3: i32, %arg4: i32):
+      %4 = arith.addi %arg3, %arg3 : i32
+      linalg.yield %4 : i32
+  } -> tensor<128x256xi32>
+  %empty = tensor.empty() : tensor<16x4x32x16xi32>
+  %pack = linalg.pack %elem
+    outer_dims_perm = [1, 0]
+    inner_dims_pos = [0, 1]
+    inner_tiles = [32, 16]
+    into %empty : tensor<128x256xi32> -> tensor<16x4x32x16xi32>
+  return %pack : tensor<16x4x32x16xi32>
+}
+
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+// CHECK-LABEL: func.func @elem_pack_transpose_outer_dims
+// CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]
+// CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]
+// CHECK:         %[[ARG1_EMPTY:.+]] = tensor.empty() : tensor<16x4x32x16xi32>
+// CHECK:         %[[ARG0_EMPTY:.+]] = tensor.empty() : tensor<16x4x32x16xi32>
+// CHECK:         %[[PACKED_ARG0:.+]] = linalg.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [32, 16]
+// CHECK-SAME:      into %[[ARG0_EMPTY]]
+// CHECK:         %[[RES:.+]] = linalg.generic
+// CHECK-SAME:      indexing_maps = [#[[$MAP]], #[[$MAP]]]
+// CHECK-SAME:      ins(%[[PACKED_ARG0]]
+// CHECK-SAME:      outs(%[[ARG1_EMPTY]]
+
+// -----
+
 #map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
 
 func.func @unpack_on_output(%arg0: tensor<12x2x56x56x32xf32>) -> tensor<12x56x56x64xf32> {
@@ -497,7 +531,7 @@ func.func @unpack_on_input(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56
 
 #map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
 
-func.func @unpack_element_type_change(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56x56x64xf16>) -> tensor<12x56x56x64xf16> {
+func.func @unpack_element_type_change_no_use(%arg0: tensor<12x2x56x56x32xf32>, %init: tensor<12x56x56x64xf16>) -> tensor<12x56x56x64xf16> {
   %0 = tensor.empty() : tensor<12x56x56x64xf32>
   %1 = linalg.unpack %arg0 outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32] into %0 : tensor<12x2x56x56x32xf32> -> tensor<12x56x56x64xf32>
   %2 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%1: tensor<12x56x56x64xf32>) outs(%init : tensor<12x56x56x64xf16>) {
@@ -509,17 +543,14 @@ func.func @unpack_element_type_change(%arg0: tensor<12x2x56x56x32xf32>, %init: t
 }
 
 // CHECK: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>
-// CHECK-LABEL: func.func @unpack_element_type_change
+// CHECK-LABEL: func.func @unpack_element_type_change_no_use
 // CHECK-SAME:     %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]
-// CHECK:         %[[ARG1_PACK_EMPTY:.+]] = tensor.empty() : tensor<12x2x56x56x32xf16>
-// CHECK:         %[[ARG1_PACK:.+]] = linalg.pack %[[ARG1]]
-// CHECK-SAME:      outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32]
-// CHECK-SAME:      into %[[ARG1_PACK_EMPTY]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<12x2x56x56x32xf16>
 // CHECK:         %[[RES:.+]] = linalg.generic
 // CHECK-SAME:      indexing_maps = [#[[$MAP]], #[[$MAP]]]
 // CHECK-SAME:      ins(%[[ARG0]]
-// CHECK-SAME:      outs(%[[ARG1_PACK]]
+// CHECK-SAME:      outs(%[[EMPTY]]
 // CHECK:         %[[UNPACK:.+]] = linalg.unpack %[[RES]]
 // CHECK-SAME:      outer_dims_perm = [0, 3, 1, 2] inner_dims_pos = [3] inner_tiles = [32]
 // CHECK-SAME:      into %[[ARG1]]
@@ -1402,13 +1433,10 @@ func.func @push_unpack_in_padded_domain_foldable(%arg0: tensor<8x8x4x8xf32>, %de
 // CHECK-SAME:    %[[ARG0:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[ARG1:[a-zA-Z0-9]+]]
 // CHECK-SAME:    %[[ARG2:[a-zA-Z0-9]+]]
-// CHECK:         %[[ARG2_PACK_EMPTY:.+]] = tensor.empty
-// CHECK:         %[[ARG2_PACK:.+]] = linalg.pack %[[ARG2]]
-// CHECK-SAME:      inner_dims_pos = [0, 1] inner_tiles = [4, 8]
-// CHECK-SAME:      into %[[ARG2_PACK_EMPTY]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty
 // CHECK:         %[[GENERIC:.+]] = linalg.generic
 // CHECK-SAME:    ins(%[[ARG0]] : tensor<8x8x4x8xf32>)
-// CHECK-SAME:    outs(%[[ARG2_PACK]] : tensor<?x8x4x8xbf16>)
+// CHECK-SAME:    outs(%[[EMPTY]] : tensor<?x8x4x8xbf16>)
 // CHECK:         %[[UNPACK:.+]] = linalg.unpack %[[GENERIC]]
 // CHECK-SAME:    into %[[ARG2]]
 // CHECK:         return %[[UNPACK]] : tensor<?x64xbf16>
diff --git a/mlir/test/Dialect/Linalg/decompose-pack.mlir b/mlir/test/Dialect/Linalg/decompose-pack.mlir
index 911b453f919c..17e6c29754f9 100644
--- a/mlir/test/Dialect/Linalg/decompose-pack.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-pack.mlir
@@ -229,3 +229,48 @@ func.func @simple_KCRS_to_KRSCsr(%arg0: tensor<1x1x32x8xf32>, %arg1: tensor<1x1x
 // CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
 // CHECK-SAME:      [0, 0, 0, 0, 0, 0] [1, 1, 1, 1, 8, 32] [1, 1, 1, 1, 1, 1]
 // CHECK:         return %[[INSERT]]
+
+// -----
+
+// The following example shows a pack operation that is defined with inner
+// dimension positions that are not adjacent, i.e. `[2, 0]`. And the outer
+// dimensions of the packed tensor are of unit values, i.e. `1x1x1`.
+func.func @pack_with_non_adjacent_inner_dims_pos_and_unit_outer(%arg0: tensor<1x1x4xf32>, %arg1: tensor<1x1x1x4x1xf32>) -> tensor<1x1x1x4x1xf32> {
+  %pack = linalg.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 0] inner_tiles = [4, 1] into %arg1 : tensor<1x1x4xf32> -> tensor<1x1x1x4x1xf32>
+  return %pack : tensor<1x1x1x4x1xf32>
+}
+// CHECK-LABEL: func.func @pack_with_non_adjacent_inner_dims_pos_and_unit_outer
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<1x4x1xf32>
+// CHECK:         %[[TRANSP:.+]] = linalg.transpose
+// CHECK-SAME:      ins(%[[SRC]] : tensor<1x1x4xf32>)
+// CHECK-SAME:      outs(%[[EMPTY]] : tensor<1x4x1xf32>)
+// CHECK-SAME:      permutation = [1, 2, 0]
+// CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
+// CHECK-SAME:      [0, 0, 0, 0, 0] [1, 1, 1, 4, 1] [1, 1, 1, 1, 1] : tensor<1x4x1xf32> into tensor<1x1x1x4x1xf32>
+// CHECK:         return %[[INSERT]]
+
+// -----
+
+// The following example shows a pack operation where the inner dimension
+// positions are specified as [2, 1] which are termed adjacent trailing
+// dimensions as they contain the last dimension of the source tensor with a
+// neighboring dimension. [1, 2] would also be considered trailing adjacent.
+// And the outer dimensions of the packed tensor are all set to unit values
+// of `1x1x1`.
+func.func @pack_with_adjacent_trailing_dimensions_inner_dims_pos_and_unit_outer(%arg0: tensor<1x1x4xf32>, %arg1: tensor<1x1x1x4x1xf32>) -> tensor<1x1x1x4x1xf32> {
+  %pack = linalg.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [4, 1] into %arg1 : tensor<1x1x4xf32> -> tensor<1x1x1x4x1xf32>
+  return %pack : tensor<1x1x1x4x1xf32>
+}
+// CHECK-LABEL: func.func @pack_with_adjacent_trailing_dimensions_inner_dims_pos_and_unit_outer
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:         %[[EMPTY:.+]] = tensor.empty() : tensor<1x4x1xf32>
+// CHECK:         %[[TRANSP:.+]] = linalg.transpose
+// CHECK-SAME:      ins(%[[SRC]] : tensor<1x1x4xf32>)
+// CHECK-SAME:      outs(%[[EMPTY]] : tensor<1x4x1xf32>)
+// CHECK-SAME:      permutation = [0, 2, 1]
+// CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[TRANSP]] into %[[DEST]]
+// CHECK-SAME:      [0, 0, 0, 0, 0] [1, 1, 1, 4, 1] [1, 1, 1, 1, 1] : tensor<1x4x1xf32> into tensor<1x1x1x4x1xf32>
+// CHECK:         return %[[INSERT]]
diff --git a/mlir/test/Dialect/Linalg/decompose-unpack.mlir b/mlir/test/Dialect/Linalg/decompose-unpack.mlir
index d460c506d6e1..e173d557c770 100644
--- a/mlir/test/Dialect/Linalg/decompose-unpack.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-unpack.mlir
@@ -169,3 +169,37 @@ func.func @unpack_with_dynamic_dims(%arg0: tensor<?x1x1x1x8x32xf32>, %arg1: tens
 // CHECK:         %[[INSERT:.+]] = tensor.insert_slice %[[EXTRACT_SLICE]] into %[[DEST]]
 // CHECK-SAME:      [0, 0, 0, 0] [%[[DIM0_DEST]], 1, 32, 8] [1, 1, 1, 1]
 // CHECK:         return %[[INSERT]]
+
+// -----
+
+func.func @unpack_with_non_adjacent_inner_dims_pos_and_unit_outer(%arg0: tensor<1x1x1x4x1xf32>, %arg1: tensor<1x1x4xf32>) -> tensor<1x1x4xf32> {
+  %0 = linalg.unpack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 0] inner_tiles = [4, 1] into %arg1 : tensor<1x1x1x4x1xf32> -> tensor<1x1x4xf32>
+  return %0 : tensor<1x1x4xf32>
+}
+// CHECK-LABEL: func.func @unpack_with_non_adjacent_inner_dims_pos_and_unit_outer
+// CHECK-SAME:     %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:     %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:        %[[SLICE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0, 0] [1, 1, 1, 4, 1] [1, 1, 1, 1, 1] : tensor<1x1x1x4x1xf32> to tensor<4x1xf32>
+// CHECK:        %[[EMPTY:.+]] = tensor.empty() : tensor<1x4xf32>
+// CHECK:        %[[TRANSP:.+]] = linalg.transpose
+// CHECK-SAME:                      ins(%[[SLICE]] : tensor<4x1xf32>)
+// CHECK-SAME:                      outs(%[[EMPTY]] : tensor<1x4xf32>) permutation = [1, 0]
+// CHECK:        %[[INSERT:.+]] = tensor.insert_slice %transposed into %[[DEST]][0, 0, 0] [1, 1, 4] [1, 1, 1] : tensor<1x4xf32> into tensor<1x1x4xf32>
+// CHECK:        return %[[INSERT]]
+
+// -----
+
+func.func @unpack_with_non_trailing_dimensions_in_inner_dims(%arg0: tensor<1x1x1x4x1xf32>, %arg1: tensor<1x1x4xf32>) -> tensor<1x1x4xf32> {
+  %pack = linalg.unpack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [4, 1] into %arg1 : tensor<1x1x1x4x1xf32> -> tensor<1x1x4xf32>
+  return %pack : tensor<1x1x4xf32>
+}
+// CHECK-LABEL: func.func @unpack_with_non_trailing_dimensions_in_inner_dims
+// CHECK-SAME:    %[[SRC:[a-zA-Z0-9]+]]
+// CHECK-SAME:    %[[DEST:[a-zA-Z0-9]+]]
+// CHECK:        %[[SLICE:.+]] = tensor.extract_slice %[[SRC]][0, 0, 0, 0, 0] [1, 1, 1, 4, 1] [1, 1, 1, 1, 1] : tensor<1x1x1x4x1xf32> to tensor<4x1xf32>
+// CHECK:        %[[EMPTY:.+]] = tensor.empty() : tensor<1x4xf32>
+// CHECK:        %[[TRANSP:.+]] = linalg.transpose
+// CHECK-SAME:                      ins(%[[SLICE]] : tensor<4x1xf32>)
+// CHECK-SAME:                      outs(%[[EMPTY]] : tensor<1x4xf32>) permutation = [1, 0]
+// CHECK:        %[[INSERT:.+]] = tensor.insert_slice %transposed into %[[DEST]][0, 0, 0] [1, 1, 4] [1, 1, 1] : tensor<1x4xf32> into tensor<1x1x4xf32>
+// CHECK:        return %[[INSERT]]
diff --git a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
index bbd6e0fc8e2c..290c6c7c36f7 100644
--- a/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
+++ b/mlir/test/Dialect/Linalg/generalize-named-polymorphic-ops.mlir
@@ -434,117 +434,6 @@ func.func @generalize_const(%min: f64, %max: f64, %seed: i32, %O: tensor<16x32xf
 
 // -----
 
-// Verifies the default value of the fun attribute is an exp op.
-func.func @generalize_elemwise_exp(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_exp
-// CHECK:        = math.exp
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_log(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<log>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_log
-// CHECK:        = math.log
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_abs(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<abs>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_abs
-// CHECK:        = math.absf
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_ceil(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<ceil>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_ceil
-// CHECK:        = math.ceil
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_floor(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<floor>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_floor
-// CHECK:        = math.floor
-
-// -----
-
-// Verifies the fun attribute controls the unary function used.
-func.func @generalize_elemwise_negf(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_negf
-// CHECK:        = arith.negf
-
-// -----
-
-// Verifies the default value of the fun attribute is an add op.
-func.func @generalize_elemwise_add(%lhs : tensor<4x8xf32>, %rhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary ins(%lhs, %rhs: tensor<4x8xf32>, tensor<4x8xf32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_add
-// CHECK:        = arith.addf
-
-// -----
-
-// Verifies the fun attribute controls the binary function used.
-func.func @generalize_elemwise_mul(%lhs : tensor<4x8xf32>, %rhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<mul>}
-                              ins(%lhs, %rhs: tensor<4x8xf32>, tensor<4x8xf32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_mul
-// CHECK:        = arith.mulf
-
-// -----
-
-// Verifies pointwise ops support rank zero input tensors
-func.func @generalize_elemwise_rank_zero(%lhs : tensor<f32>, %rhs : tensor<f32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<sub>}
-                              ins(%lhs, %rhs: tensor<f32>, tensor<f32>)
-                              outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
-  return %0: tensor<4x8xf32>
-}
-
-// CHECK-LABEL: @generalize_elemwise_rank_zero
-// CHECK:       linalg.generic
-// CHECK-SAME:  iterator_types = ["parallel", "parallel"]
-// CHECK:        = arith.subf
-
-// -----
-
 // Verifies the fun attribute controls the binary function used.
 func.func @generalize_copy(%lhs : tensor<4x8xf32>, %output : tensor<4x8xf32>) -> tensor<4x8xf32> {
   %0 = linalg.copy ins(%lhs: tensor<4x8xf32>) outs(%output: tensor<4x8xf32>) -> tensor<4x8xf32>
diff --git a/mlir/test/Dialect/Linalg/hoisting.mlir b/mlir/test/Dialect/Linalg/hoisting.mlir
index 318edca73cce..aa0b97a4787f 100644
--- a/mlir/test/Dialect/Linalg/hoisting.mlir
+++ b/mlir/test/Dialect/Linalg/hoisting.mlir
@@ -1,76 +1,439 @@
 // RUN: mlir-opt  -transform-interpreter -canonicalize --split-input-file --allow-unregistered-dialect %s | FileCheck %s
 
-// CHECK-LABEL: func @hoist_vector_transfer_pairs(
-//  CHECK-SAME:   %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[MEMREF2:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[MEMREF3:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[MEMREF4:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[MEMREF5:[a-zA-Z0-9]*]]: memref<?x?xf32>,
-//  CHECK-SAME:   %[[VAL:[a-zA-Z0-9]*]]: index,
-//  CHECK-SAME:   %[[LB:[a-zA-Z0-9]*]]: index,
-//  CHECK-SAME:   %[[UB:[a-zA-Z0-9]*]]: index,
-//  CHECK-SAME:   %[[STEP:[a-zA-Z0-9]*]]: index,
-//  CHECK-SAME:   %[[CMP:[a-zA-Z0-9]*]]: i1
-func.func @hoist_vector_transfer_pairs(
-    %memref0: memref<?x?xf32>, %memref1: memref<?x?xf32>, %memref2: memref<?x?xf32>,
-    %memref3: memref<?x?xf32>, %memref4: memref<?x?xf32>, %memref5: memref<?x?xf32>,
-    %val: index, %lb : index, %ub : index, %step: index, %cmp: i1) {
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested inside a single loop
+//  * Indices are constant
+///----------------------------------------------------------------------------------------
+
+// The most basic example - hoisting is safe.
+
+// CHECK-LABEL:   func.func @hoist_basic_vector_xfer_pair(
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index) {
+func.func @hoist_basic_vector_xfer_pair(
+    %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
   %c0 = arith.constant 0 : index
-  %cst = arith.constant 0.0 : f32
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:           %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK:             %[[VAL_6:.*]] = "val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:             scf.yield %[[VAL_6]] : vector<1xf32>
+// CHECK:           }
+// CHECK:           vector.transfer_write %[[SCF]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+  scf.for %i = %lb to %ub step %step {
+      %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+  }
+  return
+}
 
-// CHECK: vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>
-// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>) {
-// CHECK:   vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<2xf32>
-// CHECK:   scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args({{.*}}) -> (vector<1xf32>, vector<2xf32>) {
-// CHECK:     vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<3xf32>
-// CHECK:     vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<4xf32>
-// CHECK:     "some_crippling_use"(%[[MEMREF4]]) : (memref<?x?xf32>) -> ()
-// CHECK:     vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<5xf32>
-// CHECK:     "some_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
-// CHECK:     "some_use"(%{{.*}}) : (vector<2xf32>) -> vector<2xf32>
-// CHECK:     "some_use"(%[[MEMREF2]], %{{.*}}) : (memref<?x?xf32>, vector<3xf32>) -> vector<3xf32>
-// CHECK:     "some_use"(%{{.*}}) : (vector<4xf32>) -> vector<4xf32>
-// CHECK:     "some_use"(%{{.*}}) : (vector<5xf32>) -> vector<5xf32>
-// CHECK:     vector.transfer_write %{{.*}} : vector<3xf32>, memref<?x?xf32>
-// CHECK:     vector.transfer_write %{{.*}} : vector<4xf32>, memref<?x?xf32>
-// CHECK:     vector.transfer_write %{{.*}} : vector<5xf32>, memref<?x?xf32>
-// CHECK:     "some_crippling_use"(%[[MEMREF3]]) : (memref<?x?xf32>) -> ()
-// CHECK:     scf.yield {{.*}} : vector<1xf32>, vector<2xf32>
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is no longer safe. That's due to
+// an extra xfer_write inside the loop.
+
+// CHECK-LABEL:   func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_extra_write(
+    %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             vector.transfer_write %[[IN]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:             %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:             %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:             vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:           }
+
+  scf.for %i = %lb to %ub step %step {
+      vector.transfer_write %in, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+
+      %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is no longer safe. That's due to
+// an extra xfer_write into _an alias_ of the %mem Op that is used by the
+// original xfer pair.
+
+// CHECK-LABEL:   func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_extra_write_into_alias(
+    %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[SV:.*]] = memref.subview %[[MEM]][0, 0] [1, 1] [1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             vector.transfer_write %[[IN]], %[[SV]][%[[C0]], %[[C0]]] {{.*}} : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
+// CHECK:             %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:             %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:             vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:           }
+
+  %sv = memref.subview %mem[0, 0][1, 1][1, 1] : memref<?x?xf32> to memref<1x1xf32, strided<[?, 1]>>
+  scf.for %i = %lb to %ub step %step {
+      vector.transfer_write %in, %sv[%c0, %c0] : vector<1xf32>, memref<1x1xf32, strided<[?, 1]>>
+
+      %r0 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %u0, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Similar as the example above, but the memory access is done via
+// memref.assume_alignment. Hoisting is safe as the only users of the
+// "allignment" Op are the xfer Ops within the loop that we want to hoist.
+
+// CHECK-LABEL:   func.func @hoist_basic_vector_xfer_pair_with_assume_align(
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @hoist_basic_vector_xfer_pair_with_assume_align(
+    %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:           %[[SCF:.*]] = scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]]  iter_args(%[[INIT:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK:             %[[USE:.*]] = "val_use"(%[[INIT]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:           }
+// CHECK:           vector.transfer_write %[[SCF]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+
+  %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
+  scf.for %i = %lb to %ub step %step {
+      %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %u0 = "val_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %u0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Similar as the example above, but hoisting is not safe due to extra memory
+// access inside the loop via the original memref.
+
+// CHECK-LABEL:   func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[IN:[a-zA-Z0-9]+]]: vector<1xf32>) {
+func.func @negative_hoist_basic_vector_xfer_pair_with_assume_align(
+    %mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index, %in: vector<1xf32>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[AA:.*]] = memref.assume_alignment %[[MEM]], 4 : memref<?x?xf32>
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             %[[READ:.*]] = vector.transfer_read %[[AA]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:             "mem_use"(%[[MEM]])
+// CHECK:             vector.transfer_write %[[READ]], %[[AA]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:           }
+
+  %aa = memref.assume_alignment %mem, 4 : memref<?x?xf32>
+  scf.for %i = %lb to %ub step %step {
+      %r0 = vector.transfer_read %aa[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+      vector.transfer_write %r0, %aa[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested in double loops
+//  * Indices depend on induction variables
+///----------------------------------------------------------------------------------------
+
+// CHECK-LABEL: func @mem_use_outside
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @mem_use_outside(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:             %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK:               %[[USE:.*]] = "val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:               scf.yield %[[USE]] : vector<1xf32>
+// CHECK:             }
+// CHECK:             vector.transfer_write %[[SCF]], %[[MEM]][%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:             "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK:           }
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
+      %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
+    }
+  }
+  "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @mem_use_inside_outer_loop
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @mem_use_inside_outer_loop(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             %[[READ:.*]] = vector.transfer_read %[[MEM]]{{\[}}%[[I]], %[[I]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:             %[[SCF:.*]] = scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[VAL_5:.*]] = %[[READ]]) -> (vector<1xf32>) {
+// CHECK:               %[[USE:.*]] = "val_use"(%[[VAL_5]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:               scf.yield %[[USE]] : vector<1xf32>
+// CHECK:             }
+// CHECK:             vector.transfer_write %[[SCF]], %[[MEM]]{{\[}}%[[I]], %[[I]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:           "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK:           }
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %read = vector.transfer_read %mem[%i, %i], %pad: memref<?x?xf32>, vector<1xf32>
+      %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %use, %mem[%i, %i] : vector<1xf32>, memref<?x?xf32>
+    }
+    "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for vector.transfer_read + vector.transfer_write pairs
+///
+/// * Nested in double loops
+//  * Indices are constant
+///----------------------------------------------------------------------------------------
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_write
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_before_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:               %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:               %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:               "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK:               vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:             }
+// CHECK:           }
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+      "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+      vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+    }
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_after_write
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_after_write(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK:           %[[C0:.*]] = arith.constant 0 : index
+// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:             scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:               %[[READ:.*]] = vector.transfer_read %[[MEM]][%[[C0]], %[[C0]]], %[[PAD]] : memref<?x?xf32>, vector<1xf32>
+// CHECK:               %[[USE:.*]] = "val_use"(%[[READ]]) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:               vector.transfer_write %[[USE]], %[[MEM]][%[[C0]], %[[C0]]] : vector<1xf32>, memref<?x?xf32>
+// CHECK:               "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK:             }
+// CHECK:           }
+  scf.for %i = %lb to %ub step %step {
+    scf.for %j = %lb to %ub step %step {
+      %r3 = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %u3 = "val_use"(%r3) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %u3, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
+      "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+    }
+  }
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["func.func"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    transform.structured.hoist_redundant_vector_transfers %0
+      : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_mem_use_inside_inner_loop_before_read
+// CHECK-SAME:      %[[MEM:[a-zA-Z0-9]+]]: memref<?x?xf32>,
+// CHECK-SAME:      %[[LB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[UB:[a-zA-Z0-9]+]]: index,
+// CHECK-SAME:      %[[STEP:[a-zA-Z0-9]+]]: index)
+func.func @negative_mem_use_inside_inner_loop_before_read(%mem: memref<?x?xf32>, %lb : index, %ub : index, %step: index) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f32
+
+// CHECK: scf.for %[[I:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:   scf.for %[[J:.*]] = %[[LB]] to %[[UB]] step %[[STEP]] {
+// CHECK:     "mem_use"(%[[MEM]]) : (memref<?x?xf32>) -> ()
+// CHECK:     vector.transfer_read %{{.*}} : memref<?x?xf32>, vector<1xf32>
+// CHECK:     "val_use"(%{{.*}}) : (vector<1xf32>) -> vector<1xf32>
+// CHECK:     vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>
 // CHECK:   }
-// CHECK:   vector.transfer_write %{{.*}} : vector<2xf32>, memref<?x?xf32>
-// CHECK:   "unrelated_use"(%[[MEMREF0]]) : (memref<?x?xf32>) -> ()
-// CHECK:   scf.yield {{.*}} : vector<1xf32>
 // CHECK: }
-// CHECK: vector.transfer_write %{{.*}} : vector<1xf32>, memref<?x?xf32>
-// CHECK: "unrelated_use"(%[[MEMREF1]]) : (memref<?x?xf32>) -> ()
   scf.for %i = %lb to %ub step %step {
     scf.for %j = %lb to %ub step %step {
-      %r0 = vector.transfer_read %memref1[%c0, %c0], %cst: memref<?x?xf32>, vector<1xf32>
-      %r1 = vector.transfer_read %memref0[%i, %i], %cst: memref<?x?xf32>, vector<2xf32>
-      %r2 = vector.transfer_read %memref2[%c0, %c0], %cst: memref<?x?xf32>, vector<3xf32>
-      %r3 = vector.transfer_read %memref3[%c0, %c0], %cst: memref<?x?xf32>, vector<4xf32>
-      "some_crippling_use"(%memref4) : (memref<?x?xf32>) -> ()
-      %r4 = vector.transfer_read %memref4[%c0, %c0], %cst: memref<?x?xf32>, vector<5xf32>
-      %r5 = vector.transfer_read %memref5[%c0, %c0], %cst: memref<?x?xf32>, vector<6xf32>
-      "some_crippling_use"(%memref5) : (memref<?x?xf32>) -> ()
-      %u0 = "some_use"(%r0) : (vector<1xf32>) -> vector<1xf32>
-      %u1 = "some_use"(%r1) : (vector<2xf32>) -> vector<2xf32>
-      %u2 = "some_use"(%memref2, %r2) : (memref<?x?xf32>, vector<3xf32>) -> vector<3xf32>
-      %u3 = "some_use"(%r3) : (vector<4xf32>) -> vector<4xf32>
-      %u4 = "some_use"(%r4) : (vector<5xf32>) -> vector<5xf32>
-      %u5 = "some_use"(%r5) : (vector<6xf32>) -> vector<6xf32>
-      vector.transfer_write %u0, %memref1[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
-      vector.transfer_write %u1, %memref0[%i, %i] : vector<2xf32>, memref<?x?xf32>
-      vector.transfer_write %u2, %memref2[%c0, %c0] : vector<3xf32>, memref<?x?xf32>
-      vector.transfer_write %u3, %memref3[%c0, %c0] : vector<4xf32>, memref<?x?xf32>
-      vector.transfer_write %u4, %memref4[%c0, %c0] : vector<5xf32>, memref<?x?xf32>
-      vector.transfer_write %u5, %memref5[%c0, %c0] : vector<6xf32>, memref<?x?xf32>
-      "some_crippling_use"(%memref3) : (memref<?x?xf32>) -> ()
+      "mem_use"(%mem) : (memref<?x?xf32>) -> ()
+      %read = vector.transfer_read %mem[%c0, %c0], %pad: memref<?x?xf32>, vector<1xf32>
+      %use = "val_use"(%read) : (vector<1xf32>) -> vector<1xf32>
+      vector.transfer_write %use, %mem[%c0, %c0] : vector<1xf32>, memref<?x?xf32>
     }
-    "unrelated_use"(%memref0) : (memref<?x?xf32>) -> ()
   }
-  "unrelated_use"(%memref1) : (memref<?x?xf32>) -> ()
   return
 }
 
@@ -86,6 +449,12 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+///----------------------------------------------------------------------------------------
+/// Other tests
+///
+/// TODO: Document
+///----------------------------------------------------------------------------------------
+
 // CHECK-LABEL: func @hoist_vector_transfer_pairs_disjoint(
 //  CHECK-SAME:   %[[MEMREF0:[a-zA-Z0-9]*]]: memref<?x?xf32>,
 //  CHECK-SAME:   %[[MEMREF1:[a-zA-Z0-9]*]]: memref<?x?xf32>,
diff --git a/mlir/test/Dialect/Linalg/invalid.mlir b/mlir/test/Dialect/Linalg/invalid.mlir
index ca40301f04fa..da1dfc7b6a62 100644
--- a/mlir/test/Dialect/Linalg/invalid.mlir
+++ b/mlir/test/Dialect/Linalg/invalid.mlir
@@ -1165,7 +1165,7 @@ func.func @mixed_semantics(%a: tensor<?x?xf32>, %b: tensor<?x?xf32>, %c: memref<
 
 func.func @winograd_filter_transform_height(%arg0: tensor<2x4x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
   // expected-error @+1 {{expect filter height either equals to r or 1}}
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x4x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x4x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   return %0 : tensor<6x6x5x2xf32>
 }
 
@@ -1173,7 +1173,7 @@ func.func @winograd_filter_transform_height(%arg0: tensor<2x4x3x5xf32>, %arg1: t
 
 func.func @winograd_filter_transform_width(%arg0: tensor<2x3x4x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
   // expected-error @+1 {{expect filter width either equals to r or 1}}
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x4x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x3x4x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   return %0 : tensor<6x6x5x2xf32>
 }
 
@@ -1181,7 +1181,7 @@ func.func @winograd_filter_transform_width(%arg0: tensor<2x3x4x5xf32>, %arg1: te
 
 func.func @winograd_filter_transform(%arg0: tensor<2x1x1x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
   // expected-error @+1 {{expect either filter height or width equals to r}}
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x1x1x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x1x1x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   return %0 : tensor<6x6x5x2xf32>
 }
 
@@ -1189,7 +1189,7 @@ func.func @winograd_filter_transform(%arg0: tensor<2x1x1x5xf32>, %arg1: tensor<6
 
 func.func @winograd_filter_dyn(%arg0: tensor<?x3x3x?xf32>, %arg1: tensor<6x5x?x?xf32>) -> tensor<6x5x?x?xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x5x?x?xf32>) -> tensor<6x5x?x?xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x5x?x?xf32>) -> tensor<6x5x?x?xf32>
   return %0 : tensor<6x5x?x?xf32>
 }
 
@@ -1197,7 +1197,7 @@ func.func @winograd_filter_dyn(%arg0: tensor<?x3x3x?xf32>, %arg1: tensor<6x5x?x?
 
 func.func @winograd_input_transform_height(%arg0: tensor<2x13x14x5xf32>, %arg1: tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x13x14x5xf32>) outs(%arg1 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x13x14x5xf32>) outs(%arg1 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
   return %0 : tensor<6x6x3x3x2x5xf32>
 }
 
@@ -1205,7 +1205,7 @@ func.func @winograd_input_transform_height(%arg0: tensor<2x13x14x5xf32>, %arg1:
 
 func.func @winograd_input_transform_width(%arg0: tensor<2x14x13x5xf32>, %arg1: tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x14x13x5xf32>) outs(%arg1 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x14x13x5xf32>) outs(%arg1 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
   return %0 : tensor<6x6x3x3x2x5xf32>
 }
 
@@ -1213,7 +1213,7 @@ func.func @winograd_input_transform_width(%arg0: tensor<2x14x13x5xf32>, %arg1: t
 
 func.func @winograd_input_transform_output_tileH(%arg0: tensor<2x14x14x5xf32>, %arg1: tensor<6x6x2x3x2x5xf32>) -> tensor<6x6x2x3x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x6x2x3x2x5xf32>) -> tensor<6x6x2x3x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x6x2x3x2x5xf32>) -> tensor<6x6x2x3x2x5xf32>
   return %0 : tensor<6x6x2x3x2x5xf32>
 }
 
@@ -1221,7 +1221,7 @@ func.func @winograd_input_transform_output_tileH(%arg0: tensor<2x14x14x5xf32>, %
 
 func.func @winograd_input_transform_output_tileW(%arg0: tensor<2x14x14x5xf32>, %arg1: tensor<6x6x3x2x2x5xf32>) -> tensor<6x6x3x2x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x6x3x2x2x5xf32>) -> tensor<6x6x3x2x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x6x3x2x2x5xf32>) -> tensor<6x6x3x2x2x5xf32>
   return %0 : tensor<6x6x3x2x2x5xf32>
 }
 
@@ -1229,7 +1229,7 @@ func.func @winograd_input_transform_output_tileW(%arg0: tensor<2x14x14x5xf32>, %
 
 func.func @winograd_input_transform_output_height(%arg0: tensor<2x14x14x5xf32>, %arg1: tensor<5x6x3x3x2x5xf32>) -> tensor<5x6x3x3x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<5x6x3x3x2x5xf32>) -> tensor<5x6x3x3x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<5x6x3x3x2x5xf32>) -> tensor<5x6x3x3x2x5xf32>
   return %0 : tensor<5x6x3x3x2x5xf32>
 }
 
@@ -1237,7 +1237,7 @@ func.func @winograd_input_transform_output_height(%arg0: tensor<2x14x14x5xf32>,
 
 func.func @winograd_input_transform_output_width(%arg0: tensor<2x14x14x5xf32>, %arg1: tensor<6x5x3x3x2x5xf32>) -> tensor<6x5x3x3x2x5xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x5x3x3x2x5xf32>) -> tensor<6x5x3x3x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x14x14x5xf32>) outs(%arg1 : tensor<6x5x3x3x2x5xf32>) -> tensor<6x5x3x3x2x5xf32>
   return %0 : tensor<6x5x3x3x2x5xf32>
 }
 
@@ -1245,7 +1245,7 @@ func.func @winograd_input_transform_output_width(%arg0: tensor<2x14x14x5xf32>, %
 
 func.func @winograd_input_dyn(%arg0: tensor<?x?x?x?xf32>, %arg1: tensor<6x5x?x?x?x?xf32>) -> tensor<6x5x?x?x?x?xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x5x?x?x?x?xf32>) -> tensor<6x5x?x?x?x?xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x5x?x?x?x?xf32>) -> tensor<6x5x?x?x?x?xf32>
   return %0 : tensor<6x5x?x?x?x?xf32>
 }
 
@@ -1253,7 +1253,7 @@ func.func @winograd_input_dyn(%arg0: tensor<?x?x?x?xf32>, %arg1: tensor<6x5x?x?x
 
 func.func @winograd_output_transform_input_height(%arg0: tensor<5x6x3x3x2x2xf32>, %arg1: tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32> {
   // expected-error @+1 {{expect input height equals to input tile size}}
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<5x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<5x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
   return %0 : tensor<2x12x12x2xf32>
 }
 
@@ -1261,7 +1261,7 @@ func.func @winograd_output_transform_input_height(%arg0: tensor<5x6x3x3x2x2xf32>
 
 func.func @winograd_output_transform_input_width(%arg0: tensor<6x5x3x3x2x2xf32>, %arg1: tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32> {
   // expected-error @+1 {{expect input width equals to input tile size}}
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x5x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x5x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
   return %0 : tensor<2x12x12x2xf32>
 }
 
@@ -1269,7 +1269,7 @@ func.func @winograd_output_transform_input_width(%arg0: tensor<6x5x3x3x2x2xf32>,
 
 func.func @winograd_output_transform_output_height(%arg0: tensor<6x6x3x3x2x2xf32>, %arg1: tensor<2x11x12x2xf32>) -> tensor<2x11x12x2xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x11x12x2xf32>) -> tensor<2x11x12x2xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x11x12x2xf32>) -> tensor<2x11x12x2xf32>
   return %0 : tensor<2x11x12x2xf32>
 }
 
@@ -1277,7 +1277,7 @@ func.func @winograd_output_transform_output_height(%arg0: tensor<6x6x3x3x2x2xf32
 
 func.func @winograd_output_transform_output_width(%arg0: tensor<6x6x3x3x2x2xf32>, %arg1: tensor<2x12x11x2xf32>) -> tensor<2x12x11x2xf32> {
   // expected-error @+1 {{the output shape is not expected}}
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x11x2xf32>) -> tensor<2x12x11x2xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x3x3x2x2xf32>) outs(%arg1 : tensor<2x12x11x2xf32>) -> tensor<2x12x11x2xf32>
   return %0 : tensor<2x12x11x2xf32>
 }
 
@@ -1824,6 +1824,16 @@ func.func @unpack_invalid_outer_dims_perm(%source: tensor<128x256xf32>, %dest: t
 
 // -----
 
+// The outer dims in the output tensor are incorrectly/unexpectedly transposed.
+// This could be fixed by adding `outer_dims_perm = [1, 0]` (the default value assumes no transpose).
+func.func @pack_invalid_result_shape(%input: tensor<256x128xf32>, %output: tensor<4x16x32x16xf32>) -> tensor<4x16x32x16xf32> {
+  // expected-error@+1 {{the shape of output is not large enough to hold the packed data. Expected at least 'tensor<16x4x32x16xf32>', got 'tensor<4x16x32x16xf32>'}}
+  %0 = linalg.pack %input inner_dims_pos = [1, 0] inner_tiles = [32, 16] into %output : tensor<256x128xf32> -> tensor<4x16x32x16xf32>
+  return %0 : tensor<4x16x32x16xf32>
+}
+
+// -----
+
 func.func @pack_invalid(%input: tensor<256x128xf32>, %output: tensor<8x8x32x16xf32>) -> tensor<8x8x32x16xf32> {
   // expected-error@+1 {{the shape of output is not large enough to hold the packed data. Expected at least 'tensor<8x8x16x32xf32>', got 'tensor<8x8x32x16xf32>'}}
   %0 = linalg.pack %input inner_dims_pos = [1, 0] inner_tiles = [16, 32] into %output : tensor<256x128xf32> -> tensor<8x8x32x16xf32>
@@ -1868,9 +1878,43 @@ func.func @unpack_static_inner_tile_size_and_dynamic_output_shape(
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// linalg.reduce
+//===----------------------------------------------------------------------===//
+
+
 func.func @reduce_non_operation_name(%arg0: tensor<4xf32>, %arg1: tensor<f32>) -> tensor<f32> {
   // expected-error @below {{expected bare identifier or keyword}}
   %0 = linalg.reduce {@reduce_fusion_elementwise} ins(
     %arg0: tensor<4xf32>) outs(%arg1: tensor<f32>) dimensions = [0]
   return %0 : tensor<f32>
 }
+
+// -----
+
+
+//===----------------------------------------------------------------------===//
+// Tests for generic infrastructure for named Ops. The actual Ops used are
+// secondary - we merely want to ensure that the diagnostic infra triggers
+// correctly.
+//===----------------------------------------------------------------------===//
+
+module {
+  func.func @add_invalid_mixed_types(%in_f32: memref<3xf32>, %in_i32 : memref< 3xi32>, %out_f32: memref<3xf32>, %arg3: memref<3xf32>) {
+    // expected-error @below {{Cannot build binary Linalg operation: expects allComplex, allFloatingPoint, or allInteger, got 'f32' and 'i32'}}
+    linalg.add ins(%in_f32, %in_i32 : memref<3xf32>, memref< 3xi32>) outs(%out_f32 : memref<3xf32>)
+    return
+  }
+}
+
+// -----
+
+func.func @matmul_invalid_mixed_types(%t: tensor<?xf16>, %f: vector<4xf16>)
+  -> (tensor<?xf16>, vector<4xf16>)
+{
+  // expected-warning @unknown {{could not cast operand of type 'f16' to 'vector<4xf16>'}}
+  // expected-error @below {{Cannot build binary Linalg operation: expects allComplex, allFloatingPoint, or allInteger, got 'vector<4xf16>' and 'f16'}}
+  %0 = linalg.matmul ins(%t, %t : tensor<?xf16>, tensor<?xf16>)
+                                outs(%f : vector<4xf16>) -> tensor<?xf16>
+  func.return %0, %f : tensor<?xf16>, vector<4xf16>
+}
diff --git a/mlir/test/Dialect/Linalg/library-calls.mlir b/mlir/test/Dialect/Linalg/library-calls.mlir
index 1fa675d8b4b6..77c9d4a91144 100644
--- a/mlir/test/Dialect/Linalg/library-calls.mlir
+++ b/mlir/test/Dialect/Linalg/library-calls.mlir
@@ -59,43 +59,3 @@ module {
     return
   }
 }
-
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_unary_negf_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_unary_negf_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_neg(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16xf32>, %D: memref<16xf32>) {
-  linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%A: memref<16x8xf32>) outs(%B: memref<16x8xf32>)
-  linalg.elemwise_unary {fun = #linalg.unary_fn<negf>}
-                              ins(%C: memref<16xf32>) outs(%D: memref<16xf32>)
-  return
-}
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_unary_exp_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_unary_exp_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_exp(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16xf32>, %D: memref<16xf32>) {
-  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>}
-                              ins(%A: memref<16x8xf32>) outs(%B: memref<16x8xf32>)
-  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>}
-                              ins(%C: memref<16xf32>) outs(%D: memref<16xf32>)
-  return
-}
-
-// -----
-
-// CHECK: func.func private @linalg_elemwise_binary_add_view16x8xf32_view16x8xf32_view16x8xf32(memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>, memref<16x8xf32, strided<[?, ?], offset: ?>>) attributes {llvm.emit_c_interface}
-// CHECK: func.func private @linalg_elemwise_binary_add_view16xf32_view16xf32_view16xf32(memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>, memref<16xf32, strided<[?], offset: ?>>) attributes {llvm.emit_c_interface}
-
-func.func @test_add(%A : memref<16x8xf32>, %B: memref<16x8xf32>, %C: memref<16x8xf32>, %D: memref<16xf32>, %E: memref<16xf32>, %F: memref<16xf32>) {
-  linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-                              ins(%A, %B: memref<16x8xf32>, memref<16x8xf32>) outs(%C: memref<16x8xf32>)
-  linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-                              ins(%D, %E: memref<16xf32>, memref<16xf32>) outs(%F: memref<16xf32>)
-  return
-}
diff --git a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
index bfe7a07cb38a..618ba3402ff5 100644
--- a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
+++ b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
@@ -842,15 +842,15 @@ module attributes { transform.with_named_sequence } {
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
     // expected-remark @below {{single user}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<negf>} ins(%2 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.negf ins(%2 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     // expected-remark @below {{matched result value}}
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.exp ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     // expected-remark @below {{matched result value}}
     // expected-remark @below {{op result}}
     // expected-note @below {{value handle points to an op result #0}}
-    linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
+    linalg.exp ins(%3 : tensor<42x42xf32>) outs(%0 : tensor<42x42xf32>) -> tensor<42x42xf32>
     return
   }
 }
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index 470bc1c78640..412f40d50115 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -2771,6 +2771,101 @@ func.func @pad_and_pack_partially_dynamic(%source: tensor<?x?xf32>, %dest: tenso
 
 // -----
 
+func.func @pack_transposed_inner_dims_with_padding(%source: tensor<1x5x7xf32>, %dest: tensor<1x3x2x4x2xf32>, %pad: f32) -> tensor<1x3x2x4x2xf32> {
+  %0 = linalg.pack %source padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [4, 2] into %dest : tensor<1x5x7xf32> -> tensor<1x3x2x4x2xf32>
+  return %0 : tensor<1x3x2x4x2xf32>
+}
+
+// CHECK-LABEL: func.func @pack_transposed_inner_dims_with_padding(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<1x5x7xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<1x3x2x4x2xf32>,
+// CHECK-SAME:  %[[PAD:.*]]: f32)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      inner_dims_pos = [2, 1]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<1x5x7xf32> -> tensor<1x3x2x4x2xf32>
+
+// -----
+
+// The function suffix "with_padding" refers to the padding that was introduced by the pack operation. But here
+// we are dropping the padding. Creating a tensor with less elements than what we started with.
+func.func @unpack_descending_inner_dims_with_padding(%source: tensor<1x3x2x4x2xf32>, %dest: tensor<1x5x7xf32>) -> tensor<1x5x7xf32> {
+  %0 = linalg.unpack %source inner_dims_pos = [2, 1] inner_tiles = [4, 2] into %dest : tensor<1x3x2x4x2xf32> -> tensor<1x5x7xf32>
+  return %0 : tensor<1x5x7xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_descending_inner_dims_with_padding(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<1x3x2x4x2xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<1x5x7xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      inner_dims_pos = [2, 1]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<1x3x2x4x2xf32> -> tensor<1x5x7xf32>
+
+// -----
+
+func.func @pack_non_adjacent_inner_dims(%source: tensor<20x1x12xf32>, %dest: tensor<10x1x3x4x2xf32>) -> tensor<10x1x3x4x2xf32> {
+  %0 = linalg.pack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2] into %dest : tensor<20x1x12xf32> -> tensor<10x1x3x4x2xf32>
+  return %0 : tensor<10x1x3x4x2xf32>
+}
+
+// CHECK-LABEL: func.func @pack_non_adjacent_inner_dims(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<20x1x12xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<10x1x3x4x2xf32>)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      inner_dims_pos = [2, 0]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<20x1x12xf32> -> tensor<10x1x3x4x2xf32>
+
+// -----
+
+func.func @unpack_non_adjacent_inner_dims(%source: tensor<10x1x3x4x2xf32>, %dest: tensor<20x1x12xf32>) -> tensor<20x1x12xf32> {
+  %0 = linalg.unpack %source inner_dims_pos = [2, 0] inner_tiles = [4, 2] into %dest : tensor<10x1x3x4x2xf32> -> tensor<20x1x12xf32>
+  return %0 : tensor<20x1x12xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_non_adjacent_inner_dims(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<10x1x3x4x2xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<20x1x12xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      inner_dims_pos = [2, 0]
+// CHECK-SAME:      inner_tiles = [4, 2]
+// CHECK-SAME:      into %[[DEST]] : tensor<10x1x3x4x2xf32> -> tensor<20x1x12xf32>
+
+// -----
+
+func.func @pack_implementing_transpose(%source: tensor<3x5x7xf32>, %dest: tensor<3x7x5xf32>) -> tensor<3x7x5xf32> {
+  %0 = linalg.pack %source outer_dims_perm = [0, 2, 1] inner_dims_pos = [] inner_tiles = [] into %dest : tensor<3x5x7xf32> -> tensor<3x7x5xf32>
+  return %0 : tensor<3x7x5xf32>
+}
+
+// CHECK-LABEL: func.func @pack_implementing_transpose(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<3x5x7xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<3x7x5xf32>)
+// CHECK:       %{{.*}} = linalg.pack
+// CHECK-SAME:      outer_dims_perm = [0, 2, 1]
+// CHECK-SAME:      inner_dims_pos = []
+// CHECK-SAME:      inner_tiles = []
+// CHECK-SAME:      into %[[DEST]] : tensor<3x5x7xf32> -> tensor<3x7x5xf32>
+
+// -----
+
+func.func @unpack_implementing_transpose(%source: tensor<3x7x5xf32>, %dest: tensor<3x5x7xf32>) -> tensor<3x5x7xf32> {
+  %0 = linalg.unpack %source outer_dims_perm = [0, 2, 1] inner_dims_pos = [] inner_tiles = [] into %dest : tensor<3x7x5xf32> -> tensor<3x5x7xf32>
+  return %0 : tensor<3x5x7xf32>
+}
+
+// CHECK-LABEL: func.func @unpack_implementing_transpose(
+// CHECK-SAME:  %[[SOURCE:.*]]: tensor<3x7x5xf32>,
+// CHECK-SAME:  %[[DEST:.*]]: tensor<3x5x7xf32>)
+// CHECK:       %{{.*}} = linalg.unpack
+// CHECK-SAME:      outer_dims_perm = [0, 2, 1]
+// CHECK-SAME:      inner_dims_pos = []
+// CHECK-SAME:      inner_tiles = []
+// CHECK-SAME:      into %[[DEST]] : tensor<3x7x5xf32> -> tensor<3x5x7xf32>
+
+// -----
+
 func.func @unpack_fully_dynamic(%source: tensor<?x?x?x?xf32>, %dest: tensor<?x?xf32>, %tile_n : index, %tile_m : index) -> tensor<?x?xf32> {
   %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [%tile_n, %tile_m] into %dest : tensor<?x?x?x?xf32> -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
diff --git a/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir b/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
index 5b7c2baf9d84..a0922bdfcfbe 100644
--- a/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
+++ b/mlir/test/Dialect/Linalg/one-shot-bufferize-analysis.mlir
@@ -1,43 +1,5 @@
 // RUN: mlir-opt %s -one-shot-bufferize="bufferize-function-boundaries test-analysis-only" -split-input-file | FileCheck %s
 
-// CHECK-LABEL: @elementwise_no_conflict
-func.func @elementwise_no_conflict(%a: tensor<5xf32>,
-                                   %b: tensor<5xf32>) -> tensor<5xf32> {
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "true", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %b : tensor<5xf32>, tensor<5xf32>)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @elementwise_no_conflict_2
-func.func @elementwise_no_conflict_2(%a: tensor<5xf32>) -> tensor<5xf32> {
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "true", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %a : tensor<5xf32>, tensor<5xf32>)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
-// CHECK-LABEL: @elementwise_no_conflict_3
-func.func @elementwise_no_conflict_3(%a: tensor<5xf32>) -> tensor<5xf32> {
-  %c0f = arith.constant 1.0 : f32
-  // CHECK: linalg.elemwise_binary
-  // CHECK-SAME: {__inplace_operands_attr__ = ["true", "none", "true"], fun = #linalg.binary_fn<add>}
-  %0 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
-      ins(%a, %c0f : tensor<5xf32>, f32)
-      outs(%a : tensor<5xf32>) -> tensor<5xf32>
-  return %0 : tensor<5xf32>
-}
-
-// -----
-
 func.func @not_elementwise(%a: tensor<5x6xf32>) -> tensor<5x6xf32> {
   %cst = arith.constant 5.0 : f32
   // CHECK: tensor.extract_slice
diff --git a/mlir/test/Dialect/Linalg/pad_fusion.mlir b/mlir/test/Dialect/Linalg/pad_fusion.mlir
index a0d9a6ded34c..56951b365b29 100644
--- a/mlir/test/Dialect/Linalg/pad_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/pad_fusion.mlir
@@ -34,9 +34,9 @@ func.func @dynamic_pad_fusion(%arg0 : tensor<?x?xf32>, %arg1 : index, %arg2 : in
 //  CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
 //  CHECK-DAG:   %[[SOURCE:.+]] = linalg.generic
 //  CHECK-DAG:   %[[SOURCE_D0:.+]] = tensor.dim %[[SOURCE]], %[[C0]]
-//  CHECK-DAG:   %[[TARGET_D0:.+]] = affine.apply #[[MAP]]()[%[[ARG1]], %[[ARG3]], %[[SOURCE_D0]]]
+//  CHECK-DAG:   %[[TARGET_D0:.+]] = affine.apply #[[MAP]]()[%[[SOURCE_D0]], %[[ARG1]], %[[ARG3]]]
 //  CHECK-DAG:   %[[SOURCE_D1:.+]] = tensor.dim %[[SOURCE]], %[[C1]]
-//  CHECK-DAG:   %[[TARGET_D1:.+]] = affine.apply #[[MAP]]()[%[[ARG2]], %[[ARG4]], %[[SOURCE_D1]]]
+//  CHECK-DAG:   %[[TARGET_D1:.+]] = affine.apply #[[MAP]]()[%[[SOURCE_D1]], %[[ARG2]], %[[ARG4]]]
 //      CHECK:   %[[INIT:.+]] = tensor.empty(%[[TARGET_D0]], %[[TARGET_D1]])
 //      CHECK:   %[[FILL:.+]] = linalg.fill ins(%[[ARG5]]{{.*}}outs(%[[INIT]]
 //  CHECK-DAG:   %[[SIZE_D0:.+]] = tensor.dim %[[SOURCE]], %[[C0]]
@@ -80,7 +80,7 @@ func.func @mixed_pad_fusion(%arg0 : tensor<?x42xf32>, %arg1 : index, %arg2 : ind
 //  CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
 //  CHECK-DAG:   %[[SOURCE:.+]] = linalg.generic
 //  CHECK-DAG:   %[[SOURCE_D1:.+]] = tensor.dim %[[SOURCE]], %[[C1]]
-//  CHECK-DAG:   %[[TARGET_D1:.+]] = affine.apply #[[MAP]]()[%[[ARG1]], %[[ARG2]], %[[SOURCE_D1]]]
+//  CHECK-DAG:   %[[TARGET_D1:.+]] = affine.apply #[[MAP]]()[%[[SOURCE_D1]], %[[ARG1]], %[[ARG2]]]
 //      CHECK:   %[[INIT:.+]] = tensor.empty(%[[TARGET_D1]])
 //      CHECK:   %[[FILL:.+]] = linalg.fill ins(%[[ARG3]]{{.*}}outs(%[[INIT]]
 //  CHECK-DAG:   %[[SIZE_D1:.+]] = tensor.dim %[[SOURCE]], %[[C1]]
diff --git a/mlir/test/Dialect/Linalg/promotion_options.mlir b/mlir/test/Dialect/Linalg/promotion_options.mlir
index caa72ba24316..dbc073c2665f 100644
--- a/mlir/test/Dialect/Linalg/promotion_options.mlir
+++ b/mlir/test/Dialect/Linalg/promotion_options.mlir
@@ -42,3 +42,59 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+func.func @matmul_f32(%A: memref<512x256xf32>, %B: memref<256x512xf32>, %C: memref<256x256xf32>, %s0: index, %s1: index, %s2: index) {
+  %c0 = arith.constant 0 : index
+  %c256 = arith.constant 256 : index
+  %c512 = arith.constant 512 : index
+  scf.for %arg4 = %c0 to %c512 step %s0 {
+    scf.for %arg5 = %c0 to %c512 step %s1 {
+      scf.for %arg6 = %c0 to %c256 step %s2 {
+        %i0 = affine.min affine_map<(d0)[s0] -> (-d0 + 512, s0)>(%arg4)[%s0]
+        %i1 = affine.min affine_map<(d0)[s0] -> (-d0 + 512, s0)>(%arg5)[%s1]
+        %i2 = affine.min affine_map<(d0)[s0] -> (-d0 + 256, s0)>(%arg6)[%s2]
+        %0 = memref.subview %A[%arg4, %arg6][%i0, %i2][1, 1] : memref<512x256xf32> to memref<?x?xf32, strided<[256, 1], offset: ?>>
+        %1 = memref.subview %B[%arg6, %arg5][%i2, %i1][1, 1] : memref<256x512xf32> to memref<?x?xf32, strided<[512, 1], offset: ?>>
+        %2 = memref.subview %C[%arg4, %arg5][%i0, %i1][1, 1] : memref<256x256xf32> to memref<?x?xf32, strided<[256, 1], offset: ?>>
+        linalg.matmul
+          ins(%0, %1: memref<?x?xf32, strided<[256, 1], offset: ?>>,
+                      memref<?x?xf32, strided<[512, 1], offset: ?>>)
+          outs(%2: memref<?x?xf32, strided<[256, 1], offset: ?>>)
+      }
+    }
+  }
+  return
+}
+
+// CHECK-LABEL:   func.func @matmul_f32(
+// CHECK-SAME:      %[[ARG0:.*]]: memref<512x256xf32>
+// CHECK-SAME:      %[[ARG1:.*]]: memref<256x512xf32>
+// CHECK-SAME:      %[[ARG2:.*]]: memref<256x256xf32>
+// CHECK-SAME:      %[[ARG3:.*]]: index, %[[ARG4:.*]]: index, %[[ARG5:.*]]: index
+// CHECK:           %[[C4:.*]] = arith.constant 4 : index
+
+// CHECK:                 %[[i0:.*]] = affine.min
+// CHECK:                 %[[i1:.*]] = affine.min
+// CHECK:                 %[[i2:.*]] = affine.min
+
+// CHECK:                 %[[VAL_13:.*]] = arith.muli %[[i0]], %[[i2]] : index
+// CHECK:                 %[[VAL_14:.*]] = arith.muli %[[VAL_13]], %[[C4]] : index
+// CHECK:                 %[[VAL_15:.*]] = memref.alloc(%[[VAL_14]]) : memref<?xi8>
+
+// CHECK:                 %[[VAL_18:.*]] = arith.muli %[[i2]], %[[i1]] : index
+// CHECK:                 %[[VAL_19:.*]] = arith.muli %[[VAL_18]], %[[C4]] : index
+// CHECK:                 %[[VAL_20:.*]] = memref.alloc(%[[VAL_19]]) : memref<?xi8>
+
+// CHECK:                 %[[VAL_23:.*]] = arith.muli %[[i0]], %[[i1]] : index
+// CHECK:                 %[[VAL_24:.*]] = arith.muli %[[VAL_23]], %[[C4]] : index
+// CHECK:                 %[[VAL_25:.*]] = memref.alloc(%[[VAL_24]]) : memref<?xi8>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.structured.promote %0 { use_original_subview_size } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
diff --git a/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir b/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
index 3bc1f56d816d..7fbb123d4e16 100644
--- a/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
+++ b/mlir/test/Dialect/Linalg/resolve-shaped-type-result-dims.mlir
@@ -268,9 +268,9 @@ func.func @dim_of_pad_op(%arg0 : tensor<2x?x?xf32>, %arg1 : index, %arg2 : index
 //  CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
 //  CHECK-DAG:   %[[C12:.+]] = arith.constant 12 : index
 //      CHECK:   %[[IN_DIM1:.+]] = tensor.dim %[[ARG0]], %[[C1]]
-//      CHECK:   %[[OUT_DIM1:.+]] = affine.apply #[[MAP0]]()[%[[ARG1]], %[[IN_DIM1]]]
+//      CHECK:   %[[OUT_DIM1:.+]] = affine.apply #[[MAP0]]()[%[[IN_DIM1]], %[[ARG1]]]
 //      CHECK:   %[[IN_DIM2:.+]] = tensor.dim %[[ARG0]], %[[C2]]
-//      CHECK:   %[[OUT_DIM2:.+]] = affine.apply #[[MAP1]]()[%[[ARG2]], %[[IN_DIM2]]]
+//      CHECK:   %[[OUT_DIM2:.+]] = affine.apply #[[MAP1]]()[%[[IN_DIM2]], %[[ARG2]]]
 //      CHECK:   return %[[C12]], %[[OUT_DIM1]], %[[OUT_DIM2]]
 
 // -----
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index dc556761b09e..4edbc6eda3ea 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -630,52 +630,52 @@ func.func @softmax(%arg0: tensor<2x16x32xf32>) -> tensor<2x16x32xf32> {
 
 func.func @winograd(%arg0: tensor<2x6x6x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<1xf32>, %arg3: tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32> {
   %0 = tensor.empty() : tensor<6x6x5x2xf32>
-  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   %2 = tensor.empty() : tensor<6x6x1x1x2x5xf32>
-  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x6x5xf32>) outs(%2 : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+  %3 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x6x6x5xf32>) outs(%2 : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
   %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x1x1x2x5xf32> into tensor<36x2x5xf32>
   %4 = tensor.empty() : tensor<36x2x2xf32>
   %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x2x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
   %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 1, 1, 2, 2] : tensor<36x2x2xf32> into tensor<6x6x1x1x2x2xf32>
-  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x1x1x2x2xf32>) outs(%arg3 : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
+  %6 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x6x1x1x2x2xf32>) outs(%arg3 : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
   return %6 : tensor<2x4x4x2xf32>
 }
 
 // CHECK-LABEL: func @winograd
-// CHECK:         linalg.winograd_filter_transform m(4) r(3)
-// CHECK:         linalg.winograd_input_transform m(4) r(3)
-// CHECK:         linalg.winograd_output_transform m(4) r(3)
+// CHECK:         linalg.winograd_filter_transform fmr(F_4_3)
+// CHECK:         linalg.winograd_input_transform fmr(F_4_3)
+// CHECK:         linalg.winograd_output_transform fmr(F_4_3)
 
 // -----
 
 func.func @winograd_filter_dyn(%arg0: tensor<?x3x3x?xf32>, %arg1: tensor<6x6x?x?xf32>) -> tensor<6x6x?x?xf32> {
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x6x?x?xf32>) -> tensor<6x6x?x?xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x6x?x?xf32>) -> tensor<6x6x?x?xf32>
   return %0 : tensor<6x6x?x?xf32>
 }
 
 // CHECK-LABEL: func @winograd_filter_dyn
-// CHECK:         linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x6x?x?xf32>) -> tensor<6x6x?x?xf32>
+// CHECK:         linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<?x3x3x?xf32>) outs(%arg1 : tensor<6x6x?x?xf32>) -> tensor<6x6x?x?xf32>
 
 // -----
 
 func.func @winograd_input_dyn(%arg0: tensor<?x?x?x?xf32>, %arg1: tensor<6x6x?x?x?x?xf32>) -> tensor<6x6x?x?x?x?xf32> {
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x6x?x?x?x?xf32>) -> tensor<6x6x?x?x?x?xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x6x?x?x?x?xf32>) -> tensor<6x6x?x?x?x?xf32>
   return %0 : tensor<6x6x?x?x?x?xf32>
 }
 
 // CHECK-LABEL: func @winograd_input_dyn
-// CHECK:         linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x6x?x?x?x?xf32>) -> tensor<6x6x?x?x?x?xf32>
+// CHECK:         linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<?x?x?x?xf32>) outs(%arg1 : tensor<6x6x?x?x?x?xf32>) -> tensor<6x6x?x?x?x?xf32>
 
 // -----
 
 func.func @winograd_output_dyn(%arg0: tensor<6x6x?x?x?x?xf32>, %arg1: tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32> {
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x?x?x?x?xf32>) outs(%arg1 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x?x?x?x?xf32>) outs(%arg1 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
   return %0 : tensor<?x?x?x?xf32>
 }
 
 // CHECK-LABEL: func @winograd_output_dyn
-// CHECK:         linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x?x?x?x?xf32>) outs(%arg1 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+// CHECK:         linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x?x?x?x?xf32>) outs(%arg1 : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
 
 // -----
 
diff --git a/mlir/test/Dialect/Linalg/transform-op-bufferize-to-allocation.mlir b/mlir/test/Dialect/Linalg/transform-op-bufferize-to-allocation.mlir
index 4d7ddc8a513c..15a0a94079b2 100644
--- a/mlir/test/Dialect/Linalg/transform-op-bufferize-to-allocation.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-bufferize-to-allocation.mlir
@@ -9,7 +9,7 @@
 //   CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 //   CHECK-DAG:   %[[c50:.*]] = arith.constant 50 : index
 //   CHECK-DAG:   %[[dim0:.*]] = tensor.dim %[[t]], %[[c0]]
-//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[h1]], %[[dim0]]]
+//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[dim0]], %[[h1]]]
 //   CHECK-DAG:   %[[size1:.*]] = affine.apply #[[$map1]]()[%[[l2]], %[[h2]]]
 //       CHECK:   %[[alloc:.*]] = memref.alloc(%[[size0]], %[[size1]]) : memref<?x?xindex>
 //       CHECK:   linalg.fill ins(%[[c50]] : index) outs(%[[alloc]] : memref<?x?xindex>)
diff --git a/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir b/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
index 572a2ae70e0a..e5216089692b 100644
--- a/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-fuse-into-containing.mlir
@@ -26,8 +26,8 @@ module {
       // CHECK: %[[T1:.*]] = linalg.fill {{.*}} outs(%[[T0]]
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[T1]]
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%[[T1]]
+      %7 = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -76,8 +76,8 @@ module {
       %4 = affine.min #map2(%arg3)[%arg0]
       %5 = tensor.extract_slice %o[%3] [%4] [1] : tensor<64xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[INIT_TENSOR]]
-      %7 = linalg.elemwise_unary ins(%0 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp ins(%[[INIT_TENSOR]]
+      %7 = linalg.exp ins(%0 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<64xf32>
       }
@@ -177,8 +177,8 @@ module {
       // CHECK: %[[T1:.*]] = linalg.fill {{.*}} outs(%[[T0]]
       %6 = tensor.extract_slice %arg1[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary {{.*}} outs(%[[T1]]
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp {{.*}} outs(%[[T1]]
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -228,8 +228,8 @@ module {
       // CHECK: %[[T2:.*]] = linalg.fill {{.*}} outs(%[[T1]]
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T3:.*]] = linalg.elemwise_unary ins(%[[T2]] : tensor<?xf32>) outs(%[[T0]] : tensor<?xf32>)
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T3:.*]] = linalg.exp ins(%[[T2]] : tensor<?xf32>) outs(%[[T0]] : tensor<?xf32>)
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -261,7 +261,7 @@ module {
     %c2 = arith.constant 2 : index
     %c1 = arith.constant 1 : index
     %c0 = arith.constant 0 : index
-    %0 = linalg.elemwise_unary {fun = #linalg.unary_fn<abs>} ins(%arg0 : tensor<?x?x?xf32>) outs(%arg1 : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+    %0 = linalg.elementwise kind=#linalg.elementwise_kind<abs> ins(%arg0 : tensor<?x?x?xf32>) outs(%arg1 : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
     %dim = tensor.dim %arg1, %c0 : tensor<?x?x?xf32>
     %dim_0 = tensor.dim %arg1, %c1 : tensor<?x?x?xf32>
     %dim_1 = tensor.dim %arg1, %c2 : tensor<?x?x?xf32>
@@ -272,12 +272,12 @@ module {
       %2 = scf.for %arg4 = %c0 to %dim_0 step %c1 iter_args(%arg5 = %arg3) -> (tensor<?x?x?xf32>) {
         %3 = scf.for %arg6 = %c0 to %dim_1 step %c1 iter_args(%arg7 = %arg5) -> (tensor<?x?x?xf32>) {
           // CHECK:  %[[EX1:.*]] = tensor.extract_slice %[[BBARG2]]{{.*}}: tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          // CHECK:  linalg.elemwise_unary {fun = #linalg.unary_fn<abs>} ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX1]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          // CHECK:  linalg.elementwise kind=#linalg.elementwise_kind<abs> ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX1]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           // CHECK:  %[[EX2:.*]] = tensor.extract_slice %[[BBARG2]]{{.*}} : tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          // CHECK:  linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX2]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          // CHECK:  linalg.elementwise kind=#linalg.elementwise_kind<exp> ins({{.*}} : tensor<1x1x1xf32>) outs(%[[EX2]] : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           %extracted_slice = tensor.extract_slice %0[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<?x?x?xf32> to tensor<1x1x1xf32>
           %extracted_slice_2 = tensor.extract_slice %arg7[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<?x?x?xf32> to tensor<1x1x1xf32>
-          %4 = linalg.elemwise_unary {fun = #linalg.unary_fn<exp>} ins(%extracted_slice : tensor<1x1x1xf32>) outs(%extracted_slice_2 : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
+          %4 = linalg.elementwise kind=#linalg.elementwise_kind<exp> ins(%extracted_slice : tensor<1x1x1xf32>) outs(%extracted_slice_2 : tensor<1x1x1xf32>) -> tensor<1x1x1xf32>
           %inserted_slice = tensor.insert_slice %4 into %arg7[%arg2, %arg4, %arg6] [1, 1, 1] [1, 1, 1] : tensor<1x1x1xf32> into tensor<?x?x?xf32>
           scf.yield %inserted_slice : tensor<?x?x?xf32>
         }
@@ -290,7 +290,7 @@ module {
 
   module attributes {transform.with_named_sequence} {
     transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-      %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %0 = transform.structured.match ops{["linalg.elementwise"]} in %arg0 : (!transform.any_op) -> !transform.any_op
       %1 = transform.structured.match ops{["scf.for"]} in %arg0 : (!transform.any_op) -> !transform.any_op
       %2:2 = transform.split_handle %0 : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
       %3:3 = transform.split_handle %1 : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
@@ -340,8 +340,8 @@ module {
       // CHECK: %[[T1:.*]]:2 = linalg.generic {{.*}} ins(%[[T0]]
       %6 = tensor.extract_slice %0#0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      // CHECK: %[[T2:.*]] = linalg.elemwise_unary ins(%[[T1]]#0
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      // CHECK: %[[T2:.*]] = linalg.exp ins(%[[T1]]#0
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -376,8 +376,8 @@ module {
       %2 = tensor.extract_slice %0[%i][1][1] : tensor<2xf32> to tensor<1xf32>
       %3 = tensor.extract_slice %arg1[%i][1][1] : tensor<2xf32> to tensor<1xf32>
       // CHECK: %[[FUSED:.+]] = linalg.fill
-      // CHECK: elemwise_unary ins(%[[FUSED]]
-      %4 = linalg.elemwise_unary ins(%2 : tensor<1xf32>) outs(%3 : tensor<1xf32>) -> tensor<1xf32>
+      // CHECK: exp ins(%[[FUSED]]
+      %4 = linalg.exp ins(%2 : tensor<1xf32>) outs(%3 : tensor<1xf32>) -> tensor<1xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %4 into %arg1[%i][1][1] : tensor<1xf32> into tensor<2xf32>
       }
@@ -446,7 +446,7 @@ module {
       // CHECK: %[[T1:.*]]:2 = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0#0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]]#0 into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -515,7 +515,7 @@ module {
       // CHECK: %[[T1:.*]] = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -582,7 +582,7 @@ module {
       // CHECK: %[[T1:.*]] = linalg.generic {{.*}}
       %6 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %7 = linalg.elemwise_unary ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
+      %7 = linalg.exp ins(%6 : tensor<?xf32>) outs(%5 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T1]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %7 into %o[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -653,11 +653,12 @@ module {
       %5 = affine.min #map2(%i)[%d0, %idx]
       %6 = tensor.extract_slice %o[%4] [%5] [1] : tensor<?xf32> to tensor<?xf32>
 
+      // CHECK: linalg.generic
       // CHECK: %[[T1:.*]] = linalg.generic {{.*}}
       // CHECK: %[[T2:.*]] = linalg.generic {{.*}}
       %7 = tensor.extract_slice %1[%4] [%5] [1] : tensor<?xf32> to tensor<?xf32>
 
-      %8 = linalg.elemwise_unary ins(%7 : tensor<?xf32>) outs(%6 : tensor<?xf32>) -> tensor<?xf32>
+      %8 = linalg.exp ins(%7 : tensor<?xf32>) outs(%6 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         // CHECK: tensor.parallel_insert_slice %[[T2]] into %[[ARG7]][%[[I0]]] [%[[I1]]] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %8 into %o[%2] [%5] [1] : tensor<?xf32> into tensor<?xf32>
diff --git a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
index 962858076db9..9a44f95afb58 100644
--- a/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-fuse.mlir
@@ -5,19 +5,19 @@ func.func @fuse_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:    scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
-  %1 = linalg.elemwise_binary ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %1 = linalg.add ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -31,23 +31,23 @@ func.func @fuse_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<
 
   //     CHECK: %[[PARTIAL_RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: %[[RES:.*]] = scf.for {{.*}}%[[PARTIAL_RES]]
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
-  %1 = linalg.elemwise_binary ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %1 = linalg.add ins(%0, %arg0 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %1 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.loop.peel %loops#0 : (!transform.op<"scf.for">) -> (!transform.any_op, !transform.any_op)
@@ -107,20 +107,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
 // CHECK:             linalg.unpack
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @unpack_elemwise(%arg0: tensor<16x48x8x8xf32>, %arg1: tensor<128x384xf32>) -> tensor<128x384xf32> {
   %0 = tensor.empty() : tensor<128x384xf32>
   %1 = linalg.unpack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<16x48x8x8xf32> -> tensor<128x384xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<128x384xf32>)
+  %2 = linalg.exp ins(%1: tensor<128x384xf32>)
                              outs(%arg1: tensor<128x384xf32>) -> tensor<128x384xf32>
   return %2 : tensor<128x384xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [16, 32], tile_interchange = [0, 1]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -133,20 +133,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
 // CHECK:             linalg.pack
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @pack_elemwise(%arg0: tensor<128x384xf32>, %arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32> {
   %0 = tensor.empty() : tensor<16x48x8x8xf32>
   %1 = linalg.pack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<128x384xf32> -> tensor<16x48x8x8xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<16x48x8x8xf32>)
+  %2 = linalg.exp ins(%1: tensor<16x48x8x8xf32>)
                              outs(%arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32>
   return %2 : tensor<16x48x8x8xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [3, 5, 0, 0]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -159,20 +159,20 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         linalg.pack
 // CHECK:         %[[RES:.*]] = scf.for
 // CHECK:           scf.for
-// CHECK:             linalg.elemwise_unary
+// CHECK:             linalg.exp
 // CHECK:         return %[[RES]]
 func.func @nofuse_pack_elemwise(%arg0: tensor<128x384xf32>, %arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32> {
   %0 = tensor.empty() : tensor<16x48x8x8xf32>
   %1 = linalg.pack %arg0 inner_dims_pos = [0, 1] inner_tiles = [8, 8] into %0
       : tensor<128x384xf32> -> tensor<16x48x8x8xf32>
-  %2 = linalg.elemwise_unary ins(%1: tensor<16x48x8x8xf32>)
+  %2 = linalg.exp ins(%1: tensor<16x48x8x8xf32>)
                              outs(%arg1: tensor<16x48x8x8xf32>) -> tensor<16x48x8x8xf32>
   return %2 : tensor<16x48x8x8xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_unary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.exp"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:3 = transform.structured.fuse %0 {tile_sizes = [3, 5, 2, 0]}
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
       transform.yield
@@ -186,24 +186,24 @@ func.func @fuse_through_slice(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) ->
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32>
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   %dim0 = tensor.dim %arg1, %c0 : tensor<?x?xf32>
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %1 = tensor.extract_slice %0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %2 = linalg.elemwise_binary ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %2 = linalg.add ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %2 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
@@ -217,10 +217,10 @@ func.func @fuse_through_slice_and_cast_chain(%arg0: tensor<100x100xf32>, %arg1:
 
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]]
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<100x100xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<100x100xf32>)
                              outs(%arg0: tensor<100x100xf32>) -> tensor<100x100xf32>
   %1 = tensor.cast %0 : tensor<100x100xf32> to tensor<100x?xf32>
   %2 = tensor.extract_slice %1 [1, 1] [98, 98] [1, 1] : tensor<100x?xf32> to tensor<98x98xf32>
@@ -230,14 +230,14 @@ func.func @fuse_through_slice_and_cast_chain(%arg0: tensor<100x100xf32>, %arg1:
   %dim0 = tensor.dim %arg1, %c0 : tensor<?x?xf32>
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %4 = tensor.extract_slice %3 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %5 = linalg.elemwise_binary ins(%4, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %5 = linalg.add ins(%4, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %5 : tensor<?x?xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
@@ -253,8 +253,8 @@ func.func @fuse_unrelated_slices(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>)
   //     CHECK: %[[SLICE2:.+]] = tensor.extract_slice %[[SLICE1]]
   //     CHECK: %[[RES:.*]] = scf.for
   //     CHECK:     scf.for
-  //     CHECK:       linalg.elemwise_unary
-  //     CHECK:       linalg.elemwise_binary
+  //     CHECK:       linalg.exp
+  //     CHECK:       linalg.add
   //     CHECK: return %[[RES]], %[[SLICE2]]
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
@@ -262,17 +262,17 @@ func.func @fuse_unrelated_slices(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>)
   %dim1 = tensor.dim %arg1, %c1 : tensor<?x?xf32>
   %slice1 = tensor.extract_slice %arg0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
   %slice2 = tensor.extract_slice %slice1 [1, 1] [10, 10] [1, 1] : tensor<?x?xf32> to tensor<10x10xf32>
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32>
   %1 = tensor.extract_slice %0 [1, 1] [%dim0, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
-  %2 = linalg.elemwise_binary ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
+  %2 = linalg.add ins(%1, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %2, %slice2 : tensor<?x?xf32>, tensor<10x10xf32>
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.elemwise_binary"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.add"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1, %loops:2 = transform.structured.fuse %0 {tile_sizes = [32, 32], tile_interchange = [0, 1], apply_cleanup = true}
       : (!transform.any_op) -> (!transform.any_op, !transform.op<"scf.for">, !transform.any_op)
     transform.yield
diff --git a/mlir/test/Dialect/Linalg/transform-op-generalize.mlir b/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
index a0aa33c072dd..331c9c0fbbfd 100644
--- a/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-generalize.mlir
@@ -3,9 +3,9 @@
 // CHECK-LABEL: func.func @generalize_unary
 func.func @generalize_unary(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
 
-  // CHECK-NOT:   linalg.elemwise_unary
+  // CHECK-NOT:   linalg.exp
   //     CHECK:   linalg.generic
-  %0 = linalg.elemwise_unary ins(%arg0 : tensor<?x?xf32>)
+  %0 = linalg.exp ins(%arg0 : tensor<?x?xf32>)
                              outs(%arg1: tensor<?x?xf32>) -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
 }
diff --git a/mlir/test/Dialect/Linalg/transform-op-rewrite-in-destination-passing-style.mlir b/mlir/test/Dialect/Linalg/transform-op-rewrite-in-destination-passing-style.mlir
index fb37d78b50ce..63c9f1f27517 100644
--- a/mlir/test/Dialect/Linalg/transform-op-rewrite-in-destination-passing-style.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-rewrite-in-destination-passing-style.mlir
@@ -119,7 +119,7 @@ module attributes {transform.with_named_sequence} {
 //  CHECK-SAME:   %[[t1:.*]]: tensor<?x10xindex>, %[[l2:.*]]: index, %[[h1:.*]]: index, %[[h2:.*]]: index
 //   CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 //   CHECK-DAG:   %[[dim0:.*]] = tensor.dim %[[t1]], %[[c0]]
-//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[h1]], %[[dim0]]]
+//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[dim0]], %[[h1]]]
 //   CHECK-DAG:   %[[size1:.*]] = affine.apply #[[$map1]]()[%[[l2]], %[[h2]]]
 //       CHECK:   %[[empty:.*]] = tensor.empty(%[[size0]], %[[size1]]) : tensor<?x?xindex>
 //       CHECK:   %[[generic:.*]] = linalg.generic
@@ -162,7 +162,7 @@ module attributes {transform.with_named_sequence} {
 //   CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 //   CHECK-DAG:   %[[c50:.*]] = arith.constant 50 : index
 //   CHECK-DAG:   %[[dim0:.*]] = tensor.dim %[[t1]], %[[c0]]
-//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[h1]], %[[dim0]]]
+//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[dim0]], %[[h1]]]
 //   CHECK-DAG:   %[[size1:.*]] = affine.apply #[[$map1]]()[%[[l2]], %[[h2]]]
 //       CHECK:   %[[empty:.*]] = tensor.empty(%[[size0]], %[[size1]]) : tensor<?x?xindex>
 //       CHECK:   %[[filled:.*]] = linalg.fill ins(%[[c50]] : index) outs(%[[empty]] : tensor<?x?xindex>)
@@ -197,7 +197,7 @@ module attributes {transform.with_named_sequence} {
 //  CHECK-SAME:   %[[t1:.*]]: tensor<?x10xindex>, %[[l2:.*]]: index, %[[h1:.*]]: index, %[[h2:.*]]: index, %[[padding:.*]]: index
 //   CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 //   CHECK-DAG:   %[[dim0:.*]] = tensor.dim %[[t1]], %[[c0]]
-//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[h1]], %[[dim0]]]
+//   CHECK-DAG:   %[[size0:.*]] = affine.apply #[[$map]]()[%[[dim0]], %[[h1]]]
 //   CHECK-DAG:   %[[size1:.*]] = affine.apply #[[$map1]]()[%[[l2]], %[[h2]]]
 //       CHECK:   %[[empty:.*]] = tensor.empty(%[[size0]], %[[size1]]) : tensor<?x?xindex>
 //       CHECK:   %[[filled:.*]] = linalg.fill ins(%[[padding]] : index) outs(%[[empty]] : tensor<?x?xindex>)
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
index cdc4b8a72a27..445ded4bfcaf 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
@@ -2,15 +2,15 @@
 
 func.func @conv2d(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
   %0 = tensor.empty() : tensor<6x6x5x2xf32>
-  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   %2 = tensor.empty() : tensor<6x6x2x2x2x5xf32>
-  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%2 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %3 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%2 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
   %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x2x2x2x5xf32> into tensor<36x8x5xf32>
   %4 = tensor.empty() : tensor<36x8x2xf32>
   %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x8x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x8x2xf32>) -> tensor<36x8x2xf32>
   %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2] : tensor<36x8x2xf32> into tensor<6x6x2x2x2x2xf32>
-  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x2x2x2x2xf32>) outs(%arg2 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  %6 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x6x2x2x2x2xf32>) outs(%arg2 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
   return %6 : tensor<2x8x8x2xf32>
 }
 
@@ -123,13 +123,13 @@ module attributes {transform.with_named_sequence} {
 func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
   %cst = arith.constant 0.000000e+00 : f32
   %0 = tensor.empty() : tensor<6x6x5x2xf32>
-  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   %padded = tensor.pad %arg0 low[0, 0, 0, 0] high[0, 3, 3, 0] {
   ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
     tensor.yield %cst : f32
   } : tensor<2x11x11x5xf32> to tensor<2x14x14x5xf32>
   %2 = tensor.empty() : tensor<6x6x3x3x2x5xf32>
-  %3 = linalg.winograd_input_transform m(4) r(3) ins(%padded : tensor<2x14x14x5xf32>) outs(%2 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %3 = linalg.winograd_input_transform fmr(F_4_3) ins(%padded : tensor<2x14x14x5xf32>) outs(%2 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
   %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x3x3x2x5xf32> into tensor<36x18x5xf32>
   %4 = tensor.empty() : tensor<36x18x2xf32>
@@ -140,7 +140,7 @@ func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5x
   ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
     tensor.yield %cst : f32
   } : tensor<2x9x9x2xf32> to tensor<2x12x12x2xf32>
-  %7 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %7 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
   %extracted_slice = tensor.extract_slice %7[0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<2x9x9x2xf32>
   return %extracted_slice : tensor<2x9x9x2xf32>
 }
@@ -259,16 +259,16 @@ module attributes {transform.with_named_sequence} {
 func.func @conv2d_mx1_rx1(%arg0: tensor<2x6x1x5xf32>, %arg1: tensor<2x3x1x5xf32>, %arg2: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
   %cst = arith.constant 0.000000e+00 : f32
   %0 = tensor.empty() : tensor<6x1x5x2xf32>
-  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %1 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
   %2 = tensor.empty() : tensor<6x1x1x1x2x5xf32>
-  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x1x5xf32>) outs(%2 : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
+  %3 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x6x1x5xf32>) outs(%2 : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
   %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x1x2x5xf32> into tensor<6x2x5xf32>
   %4 = tensor.empty() : tensor<6x2x2xf32>
   %5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
   %6 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<6x2x5xf32>, tensor<6x5x2xf32>) outs(%5 : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
   %expanded = tensor.expand_shape %6 [[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2] : tensor<6x2x2xf32> into tensor<6x1x1x1x2x2xf32>
-  %7 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x1x1x1x2x2xf32>) outs(%arg2 : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
+  %7 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x1x1x1x2x2xf32>) outs(%arg2 : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
   return %7 : tensor<2x4x1x2xf32>
 }
 
@@ -350,16 +350,16 @@ module attributes {transform.with_named_sequence} {
 func.func @conv2d_mx1_rx1_2(%arg0: tensor<2x6x2x5xf32>, %arg1: tensor<2x3x1x5xf32>, %arg2: tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32> {
   %cst = arith.constant 0.000000e+00 : f32
   %0 = tensor.empty() : tensor<6x1x5x2xf32>
-  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %1 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
   %2 = tensor.empty() : tensor<6x1x1x2x2x5xf32>
-  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x2x5xf32>) outs(%2 : tensor<6x1x1x2x2x5xf32>) -> tensor<6x1x1x2x2x5xf32>
+  %3 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x6x2x5xf32>) outs(%2 : tensor<6x1x1x2x2x5xf32>) -> tensor<6x1x1x2x2x5xf32>
   %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x2x2x5xf32> into tensor<6x4x5xf32>
   %4 = tensor.empty() : tensor<6x4x2xf32>
   %5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
   %6 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<6x4x5xf32>, tensor<6x5x2xf32>) outs(%5 : tensor<6x4x2xf32>) -> tensor<6x4x2xf32>
   %expanded = tensor.expand_shape %6 [[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 2, 2, 2] : tensor<6x4x2xf32> into tensor<6x1x1x2x2x2xf32>
-  %7 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x1x1x2x2x2xf32>) outs(%arg2 : tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32>
+  %7 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x1x1x2x2x2xf32>) outs(%arg2 : tensor<2x4x2x2xf32>) -> tensor<2x4x2x2xf32>
   return %7 : tensor<2x4x2x2xf32>
 }
 
diff --git a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
index fc6424fd4c81..beb8d0b12573 100644
--- a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
+++ b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
@@ -1,7 +1,7 @@
 // RUN: mlir-opt %s -transform-interpreter --split-input-file | FileCheck %s
 
 func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   return %0 : tensor<6x6x5x2xf32>
 }
 
@@ -25,13 +25,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:    %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, 1] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x1xf32>
 // CHECK:      %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x1x1xf32>
-// CHECK:      %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x6x1x1xf32>) -> tensor<6x6x1x1xf32>
+// CHECK:      %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x6x1x1xf32>) -> tensor<6x6x1x1xf32>
 // CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S3]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x1x1xf32> into tensor<6x6x5x2xf32>
 
 // -----
 
 func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   return %0 : tensor<6x6x5x2xf32>
 }
 
@@ -58,12 +58,12 @@ module attributes {transform.with_named_sequence} {
 // CHECK:       %[[S3:.*]] = affine.min #[[$MAP0]](%[[ARG4]])
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, %[[S3]]] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x?xf32>
 // CHECK:       %[[EXTRACTED_SLICE_3:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x?x1xf32>
-// CHECK:       %[[S4:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x?xf32>) outs(%[[EXTRACTED_SLICE_3]] : tensor<6x6x?x1xf32>) -> tensor<6x6x?x1xf32>
+// CHECK:       %[[S4:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x?xf32>) outs(%[[EXTRACTED_SLICE_3]] : tensor<6x6x?x1xf32>) -> tensor<6x6x?x1xf32>
 // CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S4]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x?x1xf32> into tensor<6x6x5x2xf32>
 // -----
 
 func.func @tile_winograd_filter(%arg0: tensor<2x3x1x5xf32>, %arg1: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
-  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x1x5xf32>) outs(%arg1 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %0 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg0 : tensor<2x3x1x5xf32>) outs(%arg1 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
   return %0 : tensor<6x1x5x2xf32>
 }
 
@@ -87,13 +87,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]] iter_args(%[[ARG5:.*]] = %[[ARG3]])
 // CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 1, 1] [1, 1, 1, 1] : tensor<2x3x1x5xf32> to tensor<1x3x1x1xf32>
 // CHECK:       %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x5x2xf32> to tensor<6x1x1x1xf32>
-// CHECK:       %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x1x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x1x1x1xf32>) -> tensor<6x1x1x1xf32>
+// CHECK:       %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x1x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x1x1x1xf32>) -> tensor<6x1x1x1xf32>
 // CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S3]] into %[[ARG5]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x1x1xf32> into tensor<6x1x5x2xf32>
 
 // -----
 
 func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
   return %0 : tensor<6x6x2x2x2x5xf32>
 }
 
@@ -123,13 +123,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:   %[[S6:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:   %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 5] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x5xf32>
 // CHECK:   %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG5]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x2x5xf32>
-// CHECK:   %[[S7:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x5xf32>) outs(%[[EXTRACTED_SLICE_5]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+// CHECK:   %[[S7:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x5xf32>) outs(%[[EXTRACTED_SLICE_5]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
 // CHECK:   %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S7]] into %[[ARG5]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x1x1x2x5xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
 func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
   return %0 : tensor<6x6x2x2x2x5xf32>
 }
 
@@ -167,13 +167,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[S8:.*]] = affine.apply #[[$MAP1]]()
 // CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], %[[S8]], 1] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<1x?x?x1xf32>
 // CHECK:         %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x1x1xf32>
-// CHECK:         %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x?x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<6x6x1x1x1x1xf32>) -> tensor<6x6x1x1x1x1xf32>
+// CHECK:         %[[S9:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<1x?x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<6x6x1x1x1x1xf32>) -> tensor<6x6x1x1x1x1xf32>
 // CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x1x1x1x1xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
 func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
   return %0 : tensor<6x6x2x2x2x5xf32>
 }
 
@@ -213,13 +213,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:         %[[S9:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S6]], %[[S7]], %[[ARG8]]] [2, %[[S8]], %[[S9]], %[[S5]]] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x?xf32>
 // CHECK:         %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x2x2x2x?xf32>
-// CHECK:         %[[S10:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x?xf32>) outs(%[[EXTRACTED_SLICE_12]] : tensor<6x6x2x2x2x?xf32>) -> tensor<6x6x2x2x2x?xf32>
+// CHECK:         %[[S10:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x?xf32>) outs(%[[EXTRACTED_SLICE_12]] : tensor<6x6x2x2x2x?xf32>) -> tensor<6x6x2x2x2x?xf32>
 // CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S10]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x?xf32> into tensor<6x6x2x2x2x5xf32>
 
 // -----
 
 func.func @tile_winograd_input(%arg0: tensor<2x1x10x5xf32>, %arg1: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
-  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x1x10x5xf32>) outs(%arg1 : tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32>
+  %0 = linalg.winograd_input_transform fmr(F_4_3) ins(%arg0 : tensor<2x1x10x5xf32>) outs(%arg1 : tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32>
   return %0 : tensor<1x6x1x2x2x5xf32>
 }
 
@@ -258,13 +258,13 @@ module attributes {transform.with_named_sequence} {
 // CHECK:           %[[S8:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, 1, %[[S8]], 1] [1, 1, 1, 1] : tensor<2x1x10x5xf32> to tensor<1x1x?x1xf32>
 // CHECK:           %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x2x2x5xf32> to tensor<1x6x1x1x1x1xf32>
-// CHECK:           %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x1x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<1x6x1x1x1x1xf32>) -> tensor<1x6x1x1x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<1x1x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<1x6x1x1x1x1xf32>) -> tensor<1x6x1x1x1x1xf32>
 // CHECK:         %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG9]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x1x1x1xf32> into tensor<1x6x1x2x2x5xf32>
 
 // -----
 
 func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x2x2xf32>, %arg1: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x2x2xf32>) outs(%arg1 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x2x2x2x2xf32>) outs(%arg1 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
   return %0 : tensor<2x8x8x2xf32>
 }
 
@@ -298,7 +298,7 @@ module attributes {transform.with_named_sequence} {
 // -----
 
 func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x3x5xf32>, %arg1: tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32> {
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x3x5xf32>) outs(%arg1 : tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x6x2x2x3x5xf32>) outs(%arg1 : tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32>
   return %0 : tensor<3x8x8x5xf32>
 }
 
@@ -346,7 +346,7 @@ module attributes {transform.with_named_sequence} {
 // -----
 
 func.func @tile_winograd_output(%arg0 : tensor<6x1x2x1x3x5xf32>, %arg1: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
-  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x1x2x1x3x5xf32>) outs(%arg1 : tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32>
+  %0 = linalg.winograd_output_transform fmr(F_4_3) ins(%arg0 : tensor<6x1x2x1x3x5xf32>) outs(%arg1 : tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32>
   return %0 : tensor<3x8x1x5xf32>
 }
 
@@ -385,4 +385,4 @@ module attributes {transform.with_named_sequence} {
 // CHECK:           %[[S7:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:           %[[S8:.*]] = affine.apply #[[$MAP2]]()
 // CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG12]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
-// CHECK:           %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>
diff --git a/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir b/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
index 1de861e65300..e0ead54c956f 100644
--- a/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
+++ b/mlir/test/Dialect/Linalg/transform-winograd-conv2d.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -transform-interpreter -canonicalize --split-input-file -verify-diagnostics| FileCheck %s
+// RUN: mlir-opt %s -allow-unregistered-dialect -transform-interpreter -canonicalize --split-input-file -verify-diagnostics| FileCheck %s
 
 func.func @conv2d(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<1xf32>, %arg3: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
   %0 = linalg.conv_2d_nhwc_fhwc {dilations = dense<1> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} ins(%arg0, %arg1 : tensor<2x10x10x5xf32>, tensor<2x3x3x5xf32>) outs(%arg3 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
@@ -8,16 +8,16 @@ func.func @conv2d(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.winograd_conv2d %0 { fmr = 1: i32 } : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
 
 // CHECK-LABEL: func.func @conv2d
-// CHECK: linalg.winograd_filter_transform m(4) r(3)
-// CHECK: linalg.winograd_input_transform m(4) r(3)
+// CHECK: linalg.winograd_filter_transform fmr(F_4_3)
+// CHECK: linalg.winograd_input_transform fmr(F_4_3)
 // CHECK: linalg.batch_matmul
-// CHECK: linalg.winograd_output_transform m(4) r(3)
+// CHECK: linalg.winograd_output_transform fmr(F_4_3)
 
 // -----
 
@@ -29,19 +29,19 @@ func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5x
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.winograd_conv2d %0 { fmr = 1: i32 } : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
 
 // CHECK-LABEL: func.func @conv2d_unaligned
-// CHECK:       linalg.winograd_filter_transform m(4) r(3)
+// CHECK:       linalg.winograd_filter_transform fmr(F_4_3)
 // CHECK:       tensor.pad
 // CHECK-SAME:  low[0, 0, 0, 0] high[0, 3, 3, 0]
-// CHECK:       linalg.winograd_input_transform m(4) r(3)
+// CHECK:       linalg.winograd_input_transform fmr(F_4_3)
 // CHECK:       tensor.pad
 // CHECK-SAME:  low[0, 0, 0, 0] high[0, 3, 3, 0]
-// CHECK:       linalg.winograd_output_transform m(4) r(3)
+// CHECK:       linalg.winograd_output_transform fmr(F_4_3)
 
 // -----
 
@@ -54,7 +54,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_hwcf"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @+1 {{this operation is not supported to convert to Winograd Conv2D}}
-    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.winograd_conv2d %0 { fmr = 1: i32 } : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
@@ -70,7 +70,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @+1 {{apply Winograd Conv2D failed}}
-    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.winograd_conv2d %0 { fmr = 1: i32 } : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
@@ -86,7 +86,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.conv_2d_nhwc_fhwc"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     // expected-error @+1 {{apply Winograd Conv2D failed}}
-    %1 = transform.structured.winograd_conv2d %0 { m = 4, r = 3 } : (!transform.any_op) -> (!transform.any_op)
+    %1 = transform.structured.winograd_conv2d %0 { fmr = 1: i32 } : (!transform.any_op) -> (!transform.any_op)
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
index f62e257f8001..c3ee8929dc3f 100644
--- a/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir
@@ -31,9 +31,9 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic(
 // CHECK-SAME: %[[ARG1:.*]]: tensor<1x1x3xf32>
 
 // CHECK-DAG:  %[[C0:.+]] = arith.constant 0 : index
-// CHECK-DAG:  %[[CST:.+]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:  %[[PV:.+]] = ub.poison : f32
 
-// CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32>
+// CHECK:   %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32>
 // CHECK:   vector.transfer_write %[[READ]], %[[ARG1]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
 
  // -----
@@ -64,12 +64,12 @@ func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16
 // CHECK-SAME:      %[[ARG5:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[C79:.*]] = arith.constant 79 : index
 // CHECK:           %[[ADD1:.*]] = arith.addi %[[ARG1]], %[[ARG2]] : index
 // CHECK:           %[[ADD2:.*]] = arith.addi %[[ARG3]], %[[ARG4]] : index
 
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[CST]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[PV]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG5]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[WRITE]] : tensor<1x4xf32>
 // CHECK:         }
@@ -97,11 +97,11 @@ func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<8
 // CHECK-SAME:      %[[ARG1:.*]]: index,
 // CHECK-SAME:      %[[ARG2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C79:.*]] = arith.constant 79 : index
 
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[CST]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG2]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[WRITE]] : tensor<1x4xf32>
 // CHECK:         }
@@ -164,9 +164,9 @@ func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16
 // CHECK-SAME:                                                                 %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C16:.*]] = arith.constant 16 : index
-// CHECK-DAG:       %[[CST:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
 
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[CST]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
 // CHECK:           %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_1]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
 // CHECK:           return %[[VAL_9]] : tensor<1x4xf32>
 // CHECK:         }
@@ -229,12 +229,12 @@ func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>,
 // CHECK-SAME: %[[ARG3:.*]]: tensor<4x7x2xf32>
 // CHECK-SAME: %[[ARG4:.*]]: tensor<4x7x3x2xf32>
 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG: %[[C0_i32:.*]] = arith.constant 0 : i32
+// CHECK-DAG: %[[PV:.*]] = ub.poison : i32
 // CHECK-DAG: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex>
 // CHECK-DAG: %[[CST_1:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1>
 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32>
-// CHECK:    %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
-// CHECK:    %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[C0_i32]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
+// CHECK:    %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
+// CHECK:    %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
 // CHECK:    %[[CAST:.*]] = arith.index_cast %[[V0]] : vector<4x3xi32> to vector<4x3xindex>
 // CHECK:    %[[B1:.*]] = vector.broadcast %[[CAST]] : vector<4x3xindex> to vector<7x2x4x3xindex>
 // CHECK:    %[[CAST_1:.*]] = arith.index_cast %[[V1]] : vector<4x3xi32> to vector<4x3xindex>
@@ -382,7 +382,7 @@ func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32
 // CHECK-SAME:                    %[[VAL_0:.*]]: tensor<6xf32>
 // CHECK-SAME:                    %[[VAL_1:.*]]: tensor<5xi32>
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[VAL_3:.*]] = ub.poison : i32
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant dense<true> : vector<5xi1>
@@ -480,13 +480,14 @@ func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1:
 // CHECK-LABEL:   func.func @vectorize_nd_tensor_extract_block_arg(
 // CHECK-SAME:                                                     %[[VAL_0:.*]]: tensor<5x6xf32>,
 // CHECK-SAME:                                                     %[[VAL_1:.*]]: tensor<5xindex>) -> tensor<5xf32> {
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : index
 // CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex>
 // CHECK-DAG:       %[[VAL_4:.*]] = arith.constant dense<true> : vector<5xi1>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32>
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex>
 // CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<5xf32>
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[VAL_2]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[PAD]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex>
 // CHECK:           %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_6]] : vector<5xindex>
 // CHECK:           %[[VAL_10:.*]] = arith.addi %[[VAL_9]], %[[VAL_3]] : vector<5xindex>
 // CHECK:           %[[VAL_11:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]]] {{\[}}%[[VAL_10]]], %[[VAL_4]], %[[VAL_5]] : tensor<5x6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32>
@@ -559,7 +560,7 @@ func.func @vectorize_nd_tensor_extract_scalar_broadcast(%src: tensor<3x3xf32>, %
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[C1:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[C2:.*]] = arith.constant 2 : index
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C1]], %[[C2]]], %[[PAD]] : tensor<3x3xf32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
@@ -583,7 +584,7 @@ func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>)
 // CHECK-LABEL:   func.func @extract_scalar_from_0d_into_0d(
 // CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<f32>) -> tensor<f32> {
-// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
 
@@ -606,7 +607,7 @@ func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32
 // CHECK-SAME:      %[[SRC:.*]]: tensor<f32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<1xf32>) -> tensor<1xf32> {
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
@@ -654,7 +655,7 @@ func.func @scalar_read_with_broadcast_from_column_tensor(%init: tensor<1x1x4xi32
 
 // CHECK-LABEL:   func.func @scalar_read_with_broadcast_from_column_tensor
 // CHECK-SAME:      %[[INIT:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
-// CHECK-DAG:       %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[PAD:.*]] = ub.poison : i32
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
 // CHECK-DAG:       %[[SRC:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
 // CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
@@ -688,7 +689,7 @@ func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
 // CHECK-SAME:      %[[SRC:.*]]: tensor<3x3x3xf32>,
 // CHECK-SAME:      %[[INIT:.*]]: tensor<3x1x1xf32>)
 // CHECK-DAG:       %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[CST_0:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[CST_0]] : tensor<3x3x3xf32>, vector<f32>
+// CHECK-DAG:       %[[PV:.*]] = ub.poison : f32
+// CHECK:           %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] : tensor<3x3x3xf32>, vector<f32>
 // CHECK:           %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<3x1x1xf32>
 // CHECK:           vector.transfer_write %[[READ_BCAST]], %[[INIT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization/extract.mlir b/mlir/test/Dialect/Linalg/vectorization/extract.mlir
index d0d3b58a0570..76ac4b839806 100644
--- a/mlir/test/Dialect/Linalg/vectorization/extract.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/extract.mlir
@@ -299,7 +299,7 @@ func.func @masked_dynamic_vectorize_nd_tensor_extract_with_affine_apply_gather(%
 // CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_7:.*]] = tensor.dim %[[VAL_2]], %[[VAL_6]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_7]] : vector<1x4xi1>
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_2]]{{\[}}%[[VAL_8]], %[[VAL_8]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<1x4xf32> } : vector<1x4xi1> -> vector<1x4xf32>
 // CHECK:           %[[VAL_12:.*]] = vector.step : vector<4xindex>
@@ -356,7 +356,7 @@ func.func @extract_masked_vectorize(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf3
 // CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_7:.*]] = tensor.dim %[[VAL_1]], %[[VAL_6]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_8:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_7]] : vector<3x3xi1>
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_8]], %[[VAL_8]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<3x3xf32> } : vector<3x3xi1> -> vector<3x3xf32>
 // CHECK:           %[[VAL_12:.*]] = arith.constant dense<true> : vector<3x3xi1>
@@ -458,7 +458,7 @@ func.func @scalar_broadcast(%init : tensor<1x1x3xi32>, %src: tensor<1x3x2x4xi32>
 // CHECK:           %[[MASK_RES:.*]] = vector.create_mask %[[C1]], %[[C1_2]], %[[C3]] : vector<1x1x4xi1>
 
 /// Read and broadcast the scalar
-// CHECK:           %[[PAD:.*]] = arith.constant 0 : i32
+// CHECK:           %[[PAD:.*]] = ub.poison : i32
 // CHECK:           %[[MASK_READ:.*]] = vector.constant_mask [1] : vector<1xi1>
 // CHECK:           %[[READ:.*]] = vector.mask %[[MASK_READ]] {
 // CHECK-SAME:          vector.transfer_read %[[SRC]]{{\[}}%[[IDX]], %[[IDX]], %[[IDX]], %[[IDX]]],  %[[PAD]]
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
index b282c57e3e4c..4eeae4c06451 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
@@ -161,7 +161,7 @@ module attributes {transform.with_named_sequence} {
 // CHECK-SAME:     %[[ARG_0:.*]]: tensor<f32>, %[[ARG_1:.*]]: tensor<f32>, %[[ARG_2:.*]]: tensor<f32>)
 func.func @generic_0d(%arg0: tensor<f32>, %arg1: tensor<f32>,
                       %arg2: tensor<f32>) -> tensor<f32> {
-// CHECK:           %[[PAD:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[PAD:.*]] = ub.poison : f32
 // CHECK:           %[[READ_0:.*]] = vector.transfer_read %[[ARG_0]][], %[[PAD]] : tensor<f32>, vector<f32>
 // CHECK:           %[[ARG_0_AS_SCALAR:.*]] = vector.extract %[[READ_0]][] : f32 from vector<f32>
 // CHECK:           %[[READ_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PAD]] : tensor<f32>, vector<f32>
@@ -770,11 +770,11 @@ module attributes {transform.with_named_sequence} {
 // CHECK-DAG: #[[$MAP3:.*]] = affine_map<(d0, d1) -> (d1, 0, d0, 0)>
 //     CHECK: func @generic_vectorize_broadcast_transpose
 // CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
-// CHECK-DAG:   %[[CF:.*]] = arith.constant 0.000000e+00 : f32
-//     CHECK:   %[[V0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP0]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V1:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP1]]} : memref<4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V2:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP2]]} : memref<4xf32>, vector<4x4x4x4xf32>
-//     CHECK:   %[[V3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[CF]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP3]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
+// CHECK-DAG:   %[[PV:.*]] = ub.poison : f32
+//     CHECK:   %[[V0:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP0]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V1:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP1]]} : memref<4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V2:.*]] = vector.transfer_read %{{.*}}[%[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP2]]} : memref<4xf32>, vector<4x4x4x4xf32>
+//     CHECK:   %[[V3:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true, true], permutation_map = #[[$MAP3]]} : memref<4x4xf32>, vector<4x4x4x4xf32>
 //     CHECK:   %[[SUB:.*]] = arith.subf %[[V0]], %[[V1]] : vector<4x4x4x4xf32>
 //     CHECK:   %[[ADD0:.*]] = arith.addf %[[V2]], %[[SUB]] : vector<4x4x4x4xf32>
 //     CHECK:   %[[ADD1:.*]] = arith.addf %[[V3]], %[[ADD0]] : vector<4x4x4x4xf32>
@@ -1702,7 +1702,7 @@ module attributes {transform.with_named_sequence} {
 
 // CHECK-LABEL:   func.func @generic_with_reduction_and_broadcast(
 // CHECK-SAME:                                                    %[[VAL_0:.*]]: tensor<1x12x197x197xf32>) -> tensor<1x12x197x1xf32> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_3:.*]] = tensor.empty() : tensor<1x12x197x1xf32>
 // CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]], %[[VAL_1]] {in_bounds = [true, true, true, true]} : tensor<1x12x197x197xf32>, vector<1x12x197x197xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
index 6722de817f6b..9e501affdd2a 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
@@ -211,10 +211,10 @@ func.func @dynamic_generic_with_reduction_and_broadcast(%arg0: tensor<?x?xf32>,
 // CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
 // CHECK:           %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf32>
 // CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_7:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_7:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]] : vector<4x4xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_7]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x4xf32> } : vector<4x4xi1> -> vector<4x4xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.create_mask %[[VAL_3]] : vector<4xi1>
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_11]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_10]] {in_bounds = [true], permutation_map = #[[$MAP]]} : tensor<?x?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.multi_reduction <add>, %[[VAL_9]], %[[VAL_12]] [1] : vector<4x4xf32> to vector<4xf32> } : vector<4x4xi1> -> vector<4xf32>
@@ -257,7 +257,7 @@ func.func @vectorize_dynamic_2d_transpose(%arg0: tensor<?x?xf32>,
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<4x8xf32> } : vector<8x4xi1> -> vector<4x8xf32>
 // CHECK:           %[[VAL_12:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<4x8xi1>
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
-// CHECK:           %[[VAL_14:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_14:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_15:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
 // CHECK:           %[[VAL_16:.*]] = arith.addf %[[VAL_10]], %[[VAL_13]] : vector<4x8xf32>
 // CHECK:           %[[VAL_17:.*]] = vector.mask %[[VAL_12]] { vector.transfer_write %[[VAL_16]], %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, tensor<?x?xf32> } : vector<4x8xi1> -> tensor<?x?xf32>
@@ -372,12 +372,12 @@ func.func @vectorize_dynamic_reduction_2d_scalable(%arg0: tensor<?x?xf32>,
 // CHECK:    %[[C1_IDX:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_IDX]] : tensor<?x?xf32>
 // CHECK:    %[[C0_IDX:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2D:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[8]xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2D]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]], %[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[8]xf32> } : vector<4x[8]xi1> -> vector<4x[8]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2D]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]], %[[C0_IDX]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[8]xf32> } : vector<4x[8]xi1> -> vector<4x[8]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_1D:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_1D]] { vector.transfer_read %[[ARG_1]][%[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_1D]] { vector.transfer_read %[[ARG_1]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2D]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[VEC_RD_1]] [1] : vector<4x[8]xf32> to vector<4xf32> } : vector<4x[8]xi1> -> vector<4xf32>
 // CHECK:    %[[C0_IDX:.*]] = arith.constant 0 : index
 // CHECK:    %{{.*}} = vector.mask %[[MASK_1D]] { vector.transfer_write %[[REDUCE]], %[[ARG_1]][%[[C0_IDX]]] {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
@@ -408,11 +408,11 @@ func.func @vectorize_dynamic_reduction_scalable_1d(%arg0: tensor<?xf32>,
 // CHECK:          %[[C0_IDX:.*]] = arith.constant 0 : index
 // CHECK:          %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_IDX]] : tensor<?xf32>
 // CHECK:          %[[C0_IDX:.*]] = arith.constant 0 : index
-// CHECK:          %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:          %[[PV:.*]] = ub.poison : f32
 // CHECK:          %[[MASK:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
-// CHECK:          %[[VEC_RD_0:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
-// CHECK:          %[[C0_F32:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:          %[[VEC_RD_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[C0_F32]] : tensor<f32>, vector<f32>
+// CHECK:          %[[VEC_RD_0:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
+// CHECK:          %[[PV:.*]] = ub.poison : f32
+// CHECK:          %[[VEC_RD_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PV]] : tensor<f32>, vector<f32>
 // CHECK:          %[[ACC_f32:.*]] = vector.extract %[[VEC_RD_1]][] : f32 from vector<f32>
 // CHECK:          %[[REDUCE:.*]] = vector.mask %[[MASK]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[ACC_f32]] [0] : vector<[4]xf32> to f32 } : vector<[4]xi1> -> f32
 // CHECK:          %[[VEC_f32:.*]] = vector.broadcast %[[REDUCE]] : f32 to vector<f32>
@@ -532,13 +532,13 @@ func.func @vectorize_partial_dynamic_identity(%arg0: tensor<8x?xf32>,
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_6:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 8 : index
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x32xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_12:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x32xf32>
 // CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
@@ -576,13 +576,13 @@ func.func @vectorize_partial_dynamic_identity_scalable(%arg0: tensor<8x?xf32>,
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_6:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_7:.*]] = arith.constant 8 : index
 // CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x[32]xi1>
 // CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_10:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_12:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_12:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x[32]xf32>
 // CHECK:           %[[VAL_15:.*]] = arith.constant 0 : index
@@ -647,14 +647,14 @@ func.func @vectorize_static_shape_with_mask(%arg0: tensor<8x30xf32>,
 // CHECK-LABEL:   func.func @vectorize_static_shape_with_mask(
 // CHECK-SAME:      %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_4:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 8 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 30 : index
 // CHECK:           %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x32xi1>
 // CHECK:           %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
-// CHECK:           %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_11:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
 // CHECK:           %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x32xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.constant 0 : index
@@ -689,14 +689,14 @@ func.func @vectorize_static_shape_with_mask_scalable(%arg0: tensor<8x30xf32>,
 // CHECK-LABEL:   func.func @vectorize_static_shape_with_mask_scalable(
 // CHECK-SAME:      %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
 // CHECK-DAG:       %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK-DAG:       %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK-DAG:       %[[VAL_4:.*]] = ub.poison : f32
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 8 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant 30 : index
 // CHECK:           %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x[32]xi1>
 // CHECK:           %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_9:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
-// CHECK:           %[[VAL_11:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_11:.*]] = ub.poison : f32
 // CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
 // CHECK:           %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x[32]xf32>
 // CHECK:           %[[VAL_14:.*]] = arith.constant 0 : index
@@ -733,15 +733,15 @@ func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32
 // CHECK:    %[[C1_idx:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[4]xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[4]xf32> } : vector<4x[4]xi1> -> vector<4x[4]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[4]xf32> } : vector<4x[4]xi1> -> vector<4x[4]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<[4]xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<4x[4]xf32> } : vector<[4]xi1> -> vector<4x[4]xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<4x[4]xf32> } : vector<[4]xi1> -> vector<4x[4]xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
 // CHECK:    %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<4x[4]xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<4x[4]xf32> to vector<4xf32> } : vector<4x[4]xi1> -> vector<4xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
@@ -776,15 +776,15 @@ func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32
 // CHECK:    %[[C1_idx:.*]] = arith.constant 1 : index
 // CHECK:    %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<[4]x4xi1>
-// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<[4]x4xf32> } : vector<[4]x4xi1> -> vector<[4]x4xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<[4]x4xf32> } : vector<[4]x4xi1> -> vector<[4]x4xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<4xi1>
-// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<[4]x4xf32> } : vector<4xi1> -> vector<[4]x4xf32>
-// CHECK:    %[[C0_f32:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:    %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<[4]x4xf32> } : vector<4xi1> -> vector<[4]x4xf32>
+// CHECK:    %[[PV:.*]] = ub.poison : f32
 // CHECK:    %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
-// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[C0_f32]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
+// CHECK:    %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
 // CHECK:    %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<[4]x4xf32>
 // CHECK:    %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<[4]x4xf32> to vector<[4]xf32> } : vector<[4]x4xi1> -> vector<[4]xf32>
 // CHECK:    %[[C0_idx:.*]] = arith.constant 0 : index
@@ -822,10 +822,10 @@ func.func @vectorize_linalg_index_scalable(%dest: tensor<?xindex>) -> tensor<?xi
 // CHECK:           %[[C0:.*]] = arith.constant 0 : index
 // CHECK:           %[[D0:.*]] = tensor.dim %[[DEST]], %[[C0]] : tensor<?xindex>
 // CHECK:           %[[C0_1:.*]] = arith.constant 0 : index
-// CHECK:           %[[C0_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[PV:.*]] = ub.poison : index
 // CHECK:           %[[MASK:.*]] = vector.create_mask %[[D0]] : vector<[4]xi1>
 // TODO: This xfer_read is not used - avoid creating it.
-// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{\[}}%[[C0_1]]], %[[C0_2]] {in_bounds = [true]} : tensor<?xindex>, vector<[4]xindex> } : vector<[4]xi1> -> vector<[4]xindex>
+// CHECK:           %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{\[}}%[[C0_1]]], %[[PV]] {in_bounds = [true]} : tensor<?xindex>, vector<[4]xindex> } : vector<[4]xi1> -> vector<[4]xindex>
 // CHECK:           %[[STEP:.*]] = vector.step : vector<[4]xindex>
 // CHECK:           %[[C0_3:.*]] = arith.constant 0 : index
 // CHECK:           %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[STEP]], %[[DEST]]{{\[}}%[[C0_3]]] {in_bounds = [true]} : vector<[4]xindex>, tensor<?xindex> } : vector<[4]xi1> -> tensor<?xindex>
diff --git a/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir b/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
index 16d06a747327..c7b0bd51308b 100644
--- a/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
+++ b/mlir/test/Dialect/Linalg/winograd-conv2d-rewrite.mlir
@@ -3,13 +3,13 @@
 func.func @conv2d(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
   %cst = arith.constant 0.000000e+00 : f32
   %2 = tensor.empty() : tensor<6x6x5x2xf32>
-  %3 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%2 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %3 = linalg.winograd_filter_transform fmr(F_4_3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%2 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
   %padded = tensor.pad %arg0 low[0, 0, 0, 0] high[0, 3, 3, 0] {
   ^bb0(%arg3: index, %arg4: index, %arg5: index, %arg6: index):
     tensor.yield %cst : f32
   } : tensor<2x11x11x5xf32> to tensor<2x14x14x5xf32>
   %4 = tensor.empty() : tensor<6x6x3x3x2x5xf32>
-  %5 = linalg.winograd_input_transform m(4) r(3) ins(%padded : tensor<2x14x14x5xf32>) outs(%4 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %5 = linalg.winograd_input_transform fmr(F_4_3) ins(%padded : tensor<2x14x14x5xf32>) outs(%4 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
   %collapsed = tensor.collapse_shape %3 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
   %collapsed_0 = tensor.collapse_shape %5 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x3x3x2x5xf32> into tensor<36x18x5xf32>
   %6 = tensor.empty() : tensor<36x18x2xf32>
@@ -20,7 +20,7 @@ func.func @conv2d(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg
   ^bb0(%arg3: index, %arg4: index, %arg5: index, %arg6: index):
     tensor.yield %cst : f32
   } : tensor<2x9x9x2xf32> to tensor<2x12x12x2xf32>
-  %9 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %9 = linalg.winograd_output_transform fmr(F_4_3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
   %extracted_slice = tensor.extract_slice %9[0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<2x9x9x2xf32>
   return %extracted_slice : tensor<2x9x9x2xf32>
 }
diff --git a/mlir/test/Dialect/Linalg/winograd-conv2d.mlir b/mlir/test/Dialect/Linalg/winograd-conv2d.mlir
index 0040d81a2d24..e80fa6b4af94 100644
--- a/mlir/test/Dialect/Linalg/winograd-conv2d.mlir
+++ b/mlir/test/Dialect/Linalg/winograd-conv2d.mlir
@@ -9,16 +9,16 @@ func.func @conv2d_4x4_3x3(%arg0: tensor<2x6x6x5xf32>, %arg1: tensor<2x3x3x5xf32>
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x6x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32> {
 // CHECK-NEXT:  %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:  %[[S2:.*]] = tensor.empty() : tensor<6x6x5x2xf32>
-// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
 // CHECK-NEXT:  %[[S4:.*]] = tensor.empty() : tensor<6x6x1x1x2x5xf32>
-// CHECK-NEXT:  %[[S5:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[ARG0]] : tensor<2x6x6x5xf32>) outs(%[[S4]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+// CHECK-NEXT:  %[[S5:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[ARG0]] : tensor<2x6x6x5xf32>) outs(%[[S4]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
 // CHECK-NEXT:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
 // CHECK-NEXT:  %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x6x1x1x2x5xf32> into tensor<36x2x5xf32>
 // CHECK-NEXT:  %[[S6:.*]] = tensor.empty() : tensor<36x2x2xf32>
 // CHECK-NEXT:  %[[S7:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S6]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:  %[[S8:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<36x2x5xf32>, tensor<36x5x2xf32>) outs(%[[S7]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S8]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 1, 1, 2, 2] : tensor<36x2x2xf32> into tensor<6x6x1x1x2x2xf32>
-// CHECK-NEXT:  %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
+// CHECK-NEXT:  %[[S9:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
 // CHECK-NEXT:  return %[[S9]] : tensor<2x4x4x2xf32>
 // CHECK-NEXT: }
 
@@ -33,16 +33,16 @@ func.func @conv2d_2x2_5x5(%arg0: tensor<2x6x6x5xf32>, %arg1: tensor<2x5x5x5xf32>
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x6x5xf32>, %[[ARG1:.*]]: tensor<2x5x5x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x2x2x2xf32>) -> tensor<2x2x2x2xf32> {
 // CHECK-NEXT:   %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:   %[[S2:.*]] = tensor.empty() : tensor<6x6x5x2xf32>
-// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform m(2) r(5) ins(%[[ARG1]] : tensor<2x5x5x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_2_5) ins(%[[ARG1]] : tensor<2x5x5x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
 // CHECK-NEXT:   %[[S4:.*]] = tensor.empty() : tensor<6x6x1x1x2x5xf32>
-// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform m(2) r(5) ins(%[[ARG0]] : tensor<2x6x6x5xf32>) outs(%[[S4]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform fmr(F_2_5) ins(%[[ARG0]] : tensor<2x6x6x5xf32>) outs(%[[S4]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
 // CHECK-NEXT:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
 // CHECK-NEXT:   %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x6x1x1x2x5xf32> into tensor<36x2x5xf32>
 // CHECK-NEXT:   %[[S6:.*]] = tensor.empty() : tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[S7:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S6]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[S8:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<36x2x5xf32>, tensor<36x5x2xf32>) outs(%[[S7]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S8]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 1, 1, 2, 2] : tensor<36x2x2xf32> into tensor<6x6x1x1x2x2xf32>
-// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform m(2) r(5) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x2x2x2xf32>) -> tensor<2x2x2x2xf32>
+// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform fmr(F_2_5) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x2x2x2xf32>) -> tensor<2x2x2x2xf32>
 // CHECK-NEXT:   return %[[S9]] : tensor<2x2x2x2xf32>
 // CHECK-NEXT: }
 
@@ -57,16 +57,16 @@ func.func @conv2d_1x4_1x3(%arg0: tensor<2x1x6x5xf32>, %arg1: tensor<2x1x3x5xf32>
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x1x6x5xf32>, %[[ARG1:.*]]: tensor<2x1x3x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x1x4x2xf32>) -> tensor<2x1x4x2xf32> {
 // CHECK-NEXT:   %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:   %[[S2:.*]] = tensor.empty() : tensor<1x6x5x2xf32>
-// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x1x3x5xf32>) outs(%[[S2]] : tensor<1x6x5x2xf32>) -> tensor<1x6x5x2xf32>
+// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x1x3x5xf32>) outs(%[[S2]] : tensor<1x6x5x2xf32>) -> tensor<1x6x5x2xf32>
 // CHECK-NEXT:   %[[S4:.*]] = tensor.empty() : tensor<1x6x1x1x2x5xf32>
-// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[ARG0]] : tensor<2x1x6x5xf32>) outs(%[[S4]] : tensor<1x6x1x1x2x5xf32>) -> tensor<1x6x1x1x2x5xf32>
+// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[ARG0]] : tensor<2x1x6x5xf32>) outs(%[[S4]] : tensor<1x6x1x1x2x5xf32>) -> tensor<1x6x1x1x2x5xf32>
 // CHECK-NEXT:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2], [3]] : tensor<1x6x5x2xf32> into tensor<6x5x2xf32>
 // CHECK-NEXT:   %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<1x6x1x1x2x5xf32> into tensor<6x2x5xf32>
 // CHECK-NEXT:   %[[S6:.*]] = tensor.empty() : tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[S7:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S6]] : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[S8:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<6x2x5xf32>, tensor<6x5x2xf32>) outs(%[[S7]] : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S8]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [1, 6, 1, 1, 2, 2] : tensor<6x2x2xf32> into tensor<1x6x1x1x2x2xf32>
-// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<1x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x1x4x2xf32>) -> tensor<2x1x4x2xf32>
+// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<1x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x1x4x2xf32>) -> tensor<2x1x4x2xf32>
 // CHECK-NEXT:   return %[[S9]] : tensor<2x1x4x2xf32>
 // CHECK-NEXT: }
 
@@ -81,16 +81,16 @@ func.func @conv2d_4x1_3x1(%arg0: tensor<2x6x1x5xf32>, %arg1: tensor<2x3x1x5xf32>
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x1x5xf32>, %[[ARG1:.*]]: tensor<2x3x1x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
 // CHECK-NEXT:   %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:   %[[S2:.*]] = tensor.empty() : tensor<6x1x5x2xf32>
-// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x3x1x5xf32>) outs(%[[S2]] : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x3x1x5xf32>) outs(%[[S2]] : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
 // CHECK-NEXT:   %[[S4:.*]] = tensor.empty() : tensor<6x1x1x1x2x5xf32>
-// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[ARG0]] : tensor<2x6x1x5xf32>) outs(%[[S4]] : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
+// CHECK-NEXT:   %[[S5:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[ARG0]] : tensor<2x6x1x5xf32>) outs(%[[S4]] : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
 // CHECK-NEXT:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
 // CHECK-NEXT:   %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x1x2x5xf32> into tensor<6x2x5xf32>
 // CHECK-NEXT:   %[[S6:.*]] = tensor.empty() : tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[S7:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S6]] : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[S8:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<6x2x5xf32>, tensor<6x5x2xf32>) outs(%[[S7]] : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
 // CHECK-NEXT:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S8]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2] : tensor<6x2x2xf32> into tensor<6x1x1x1x2x2xf32>
-// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<6x1x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
+// CHECK-NEXT:   %[[S9:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<6x1x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
 // CHECK-NEXT:   return %[[S9]] : tensor<2x4x1x2xf32>
 // CHECK-NEXT: }
 
@@ -105,16 +105,16 @@ func.func @conv2d_aligned(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf3
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
 // CHECK-NEXT:  %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:  %[[S2:.*]] = tensor.empty() : tensor<6x6x5x2xf32>
-// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S2]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
 // CHECK-NEXT:  %[[S4:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
-// CHECK-NEXT:  %[[S5:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[ARG0]] : tensor<2x10x10x5xf32>) outs(%[[S4]] : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+// CHECK-NEXT:  %[[S5:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[ARG0]] : tensor<2x10x10x5xf32>) outs(%[[S4]] : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
 // CHECK-NEXT:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
 // CHECK-NEXT:  %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x6x2x2x2x5xf32> into tensor<36x8x5xf32>
 // CHECK-NEXT:  %[[S6:.*]] = tensor.empty() : tensor<36x8x2xf32>
 // CHECK-NEXT:  %[[S7:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S6]] : tensor<36x8x2xf32>) -> tensor<36x8x2xf32>
 // CHECK-NEXT:  %[[S8:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<36x8x5xf32>, tensor<36x5x2xf32>) outs(%[[S7]] : tensor<36x8x2xf32>) -> tensor<36x8x2xf32>
 // CHECK-NEXT:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S8]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2] : tensor<36x8x2xf32> into tensor<6x6x2x2x2x2xf32>
-// CHECK-NEXT:  %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<6x6x2x2x2x2xf32>) outs(%[[ARG3]] : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+// CHECK-NEXT:  %[[S9:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<6x6x2x2x2x2xf32>) outs(%[[ARG3]] : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
 // CHECK-NEXT:  return %[[S9]] : tensor<2x8x8x2xf32>
 // CHECK-NEXT: }
 
@@ -129,13 +129,13 @@ func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5x
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x11x11x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
 // CHECK-DAG:   %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK:       %[[S0:.*]] = tensor.empty() : tensor<6x6x5x2xf32>
-// CHECK-NEXT:  %[[S1:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S0]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+// CHECK-NEXT:  %[[S1:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x3x3x5xf32>) outs(%[[S0]] : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
 // CHECK-NEXT:  %[[PADDED:.*]] = tensor.pad %[[ARG0]] low[0, 0, 0, 0] high[0, 3, 3, 0] {
 // CHECK-NEXT:  ^bb0
 // CHECK-NEXT:    tensor.yield %[[CST]] : f32
 // CHECK-NEXT:  } : tensor<2x11x11x5xf32> to tensor<2x14x14x5xf32>
 // CHECK-NEXT:  %[[S2:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
-// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[PADDED]] : tensor<2x14x14x5xf32>) outs(%[[S2]] : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+// CHECK-NEXT:  %[[S3:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[PADDED]] : tensor<2x14x14x5xf32>) outs(%[[S2]] : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
 // CHECK-NEXT:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
 // CHECK-NEXT:  %[[COLLAPSED_0:.*]] = tensor.collapse_shape %3 {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x6x3x3x2x5xf32> into tensor<36x18x5xf32>
 // CHECK-NEXT:  %[[S4:.*]] = tensor.empty() : tensor<36x18x2xf32>
@@ -146,7 +146,7 @@ func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5x
 // CHECK-NEXT:  ^bb0
 // CHECK-NEXT:    tensor.yield %[[CST]] : f32
 // CHECK-NEXT:  } : tensor<2x9x9x2xf32> to tensor<2x12x12x2xf32>
-// CHECK-NEXT:  %[[S7:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<6x6x3x3x2x2xf32>) outs(%[[PADDED_1]] : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+// CHECK-NEXT:  %[[S7:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<6x6x3x3x2x2xf32>) outs(%[[PADDED_1]] : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
 // CHECK-NEXT:  %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[S7]][0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<2x9x9x2xf32>
 // CHECK-NEXT:  return %[[EXTRACTED_SLICE]] : tensor<2x9x9x2xf32>
 // CHECK-NEXT: }
@@ -162,16 +162,16 @@ func.func @conv2d_type_promotion(%arg0: tensor<2x6x6x5xf16>, %arg1: tensor<2x3x3
 // CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x6x5xf16>, %[[ARG1:.*]]: tensor<2x3x3x5xf16>, %[[ARG2:.*]]: tensor<1xf32>, %[[ARG3:.*]]: tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32> {
 // CHECK:        %[[CST:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK-NEXT:   %[[S0:.*]] = tensor.empty() : tensor<6x6x5x2xf16>
-// CHECK-NEXT:   %[[S1:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[ARG1]] : tensor<2x3x3x5xf16>) outs(%[[S0]] : tensor<6x6x5x2xf16>) -> tensor<6x6x5x2xf16>
+// CHECK-NEXT:   %[[S1:.*]] = linalg.winograd_filter_transform fmr(F_4_3) ins(%[[ARG1]] : tensor<2x3x3x5xf16>) outs(%[[S0]] : tensor<6x6x5x2xf16>) -> tensor<6x6x5x2xf16>
 // CHECK-NEXT:   %[[S2:.*]] = tensor.empty() : tensor<6x6x1x1x2x5xf16>
-// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[ARG0]] : tensor<2x6x6x5xf16>) outs(%[[S2]] : tensor<6x6x1x1x2x5xf16>) -> tensor<6x6x1x1x2x5xf16>
+// CHECK-NEXT:   %[[S3:.*]] = linalg.winograd_input_transform fmr(F_4_3) ins(%[[ARG0]] : tensor<2x6x6x5xf16>) outs(%[[S2]] : tensor<6x6x1x1x2x5xf16>) -> tensor<6x6x1x1x2x5xf16>
 // CHECK-NEXT:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]] : tensor<6x6x5x2xf16> into tensor<36x5x2xf16>
 // CHECK-NEXT:   %[[COLLAPSED_0:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2, 3, 4], [5]] : tensor<6x6x1x1x2x5xf16> into tensor<36x2x5xf16>
 // CHECK-NEXT:   %[[S4:.*]] = tensor.empty() : tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[S5:.*]] = linalg.fill ins(%[[CST]] : f32) outs(%[[S4]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[S6:.*]] = linalg.batch_matmul ins(%[[COLLAPSED_0]], %[[COLLAPSED]] : tensor<36x2x5xf16>, tensor<36x5x2xf16>) outs(%[[S5]] : tensor<36x2x2xf32>) -> tensor<36x2x2xf32>
 // CHECK-NEXT:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 1, 1, 2, 2] : tensor<36x2x2xf32> into tensor<6x6x1x1x2x2xf32>
-// CHECK-NEXT:   %[[S7:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
+// CHECK-NEXT:   %[[S7:.*]] = linalg.winograd_output_transform fmr(F_4_3) ins(%[[EXPANDED]] : tensor<6x6x1x1x2x2xf32>) outs(%[[ARG3]] : tensor<2x4x4x2xf32>) -> tensor<2x4x4x2xf32>
 // CHECK-NEXT:   return %[[S7]] : tensor<2x4x4x2xf32>
 // CHECK-NEXT: }
 
diff --git a/mlir/test/Dialect/MemRef/canonicalize.mlir b/mlir/test/Dialect/MemRef/canonicalize.mlir
index 7a267ae8a2c9..16b7a5c8bcb0 100644
--- a/mlir/test/Dialect/MemRef/canonicalize.mlir
+++ b/mlir/test/Dialect/MemRef/canonicalize.mlir
@@ -466,6 +466,24 @@ func.func @compose_collapse_of_collapse(%arg0 : memref<?x?x?x?x?xf32>)
 
 // -----
 
+func.func @compose_collapse_of_expand_partially_dynamic(%arg0: memref<?xf16>, %arg1: index, %arg2: index) -> memref<8x?x?xf16> {
+  %expanded = memref.expand_shape %arg0 [[0, 1, 2, 3, 4]] output_shape [4, 2, %arg1, %arg2, 32] : memref<?xf16> into memref<4x2x?x?x32xf16>
+  %collapsed = memref.collapse_shape %expanded [[0, 1], [2], [3, 4]] : memref<4x2x?x?x32xf16> into memref<8x?x?xf16>
+  return %collapsed : memref<8x?x?xf16>
+}
+//       CHECK: func @compose_collapse_of_expand_partially_dynamic
+//  CHECK-SAME:   %[[SRC:.[a-zA-Z0-9]+]]
+//  CHECK-SAME:   %[[ORIG_D2:.[a-zA-Z0-9]+]]
+//  CHECK-SAME:   %[[ORIG_D3:.[a-zA-Z0-9]+]]
+//   CHECK-DAG:   %[[C32:.+]] = arith.constant 32
+//       CHECK:   %[[COLLAPSED_D2:.+]] = arith.muli %[[ORIG_D3]], %[[C32]]
+//       CHECK:   %[[RESULT:.+]] = memref.expand_shape %[[SRC]]
+//  CHECK-SAME:     [0, 1, 2]
+//  CHECK-SAME:     output_shape [8, %[[ORIG_D2]], %[[COLLAPSED_D2]]]
+//       CHECK:   return %[[RESULT]]
+
+// -----
+
 func.func @do_not_compose_collapse_of_expand_non_identity_layout(
     %arg0: memref<?x?xf32, strided<[?, 1], offset: 0>>, %sz0: index, %sz1: index)
     -> memref<?xf32, strided<[?], offset: 0>> {
@@ -1190,3 +1208,24 @@ func.func @fold_assume_alignment_chain(%0: memref<128xf32>) -> memref<128xf32> {
   // CHECK: return %[[ALIGN]]
   return %2 : memref<128xf32>
 }
+
+// -----
+
+// CHECK-LABEL: func @fold_view_same_source_result_types
+func.func @fold_view_same_source_result_types(%0: memref<128xi8>) -> memref<128xi8> {
+  %c0 = arith.constant 0: index
+  // CHECK-NOT: memref.view
+  %res = memref.view %0[%c0][] : memref<128xi8> to memref<128xi8>
+  return %res : memref<128xi8>
+}
+
+// -----
+
+// CHECK-LABEL: func @non_fold_view_same_source_res_types
+//  CHECK-SAME:   %[[ARG0:[a-zA-Z0-9]+]]
+func.func @non_fold_view_same_source_res_types(%0: memref<?xi8>, %arg0 : index) -> memref<?xi8> {
+  %c0 = arith.constant 0: index
+  // CHECK: memref.view
+  %res = memref.view %0[%c0][%arg0] : memref<?xi8> to memref<?xi8>
+  return %res : memref<?xi8>
+}
diff --git a/mlir/test/Dialect/MemRef/invalid.mlir b/mlir/test/Dialect/MemRef/invalid.mlir
index f908efb63844..704cdaf838f4 100644
--- a/mlir/test/Dialect/MemRef/invalid.mlir
+++ b/mlir/test/Dialect/MemRef/invalid.mlir
@@ -342,6 +342,16 @@ memref.global "priate" constant @memref5 : memref<2xf32>  = uninitialized
 
 // -----
 
+// expected-error @+1 {{op initial value element expected to be of type 'f16', but was of type 'f32'}}
+"memref.global"() <{constant, initial_value = dense<1.000000e+00> : tensor<1xf32>, sym_name = "memref6", sym_visibility = "private", type = memref<1xf16>}> : () -> ()
+
+// -----
+
+// expected-error @+1 {{op initial value shape expected to be 1, 2 but was 2, 2}}
+"memref.global"() <{constant, initial_value = dense<1.000000e+00> : tensor<2x2xf16>, sym_name = "memref7", sym_visibility = "private", type = memref<1x2xf16>}> : () -> ()
+
+// -----
+
 func.func @nonexistent_global_memref() {
   // expected-error @+1 {{'gv' does not reference a valid global memref}}
   %0 = memref.get_global @gv : memref<3xf32>
@@ -703,7 +713,7 @@ func.func @invalid_subview(%arg0 : index, %arg1 : index, %arg2 : index) {
 
 func.func @invalid_subview(%arg0 : index, %arg1 : index, %arg2 : index) {
   %0 = memref.alloc() : memref<8x16x4xf32>
-  // expected-error@+1 {{expected result element type to be 'f32'}}
+  // expected-error@+1 {{expected result element type to be 'f32', but got 'memref<8x16x4xi32>'}}
   %1 = memref.subview %0[0, 0, 0][8, 16, 4][1, 1, 1]
     : memref<8x16x4xf32> to
       memref<8x16x4xi32>
@@ -714,10 +724,10 @@ func.func @invalid_subview(%arg0 : index, %arg1 : index, %arg2 : index) {
 
 func.func @invalid_subview(%arg0 : index, %arg1 : index, %arg2 : index) {
   %0 = memref.alloc() : memref<8x16x4xf32>
-  // expected-error@+1 {{expected result rank to be smaller or equal to the source rank.}}
+  // expected-error@+1 {{expected result rank to be smaller or equal to the source rank, but got 'memref<8x16x4x3xf32>'}}
   %1 = memref.subview %0[0, 0, 0][8, 16, 4][1, 1, 1]
     : memref<8x16x4xf32> to
-      memref<8x16x4x3xi32>
+      memref<8x16x4x3xf32>
   return
 }
 
diff --git a/mlir/test/Dialect/MemRef/ops.mlir b/mlir/test/Dialect/MemRef/ops.mlir
index 13fdf3cf1351..e11de7bec2d0 100644
--- a/mlir/test/Dialect/MemRef/ops.mlir
+++ b/mlir/test/Dialect/MemRef/ops.mlir
@@ -174,6 +174,9 @@ memref.global "private" @memref3 : memref<2xf32>  = uninitialized
 // CHECK-LABEL: memref.global "private" constant @memref4 : memref<2xf32> = uninitialized
 memref.global "private" constant @memref4 : memref<2xf32>  = uninitialized
 
+// CHECK-LABEL: memref.global "private" constant @memref5 : memref<1xf16, 42 : i32> = dense<1.000000e+00>
+"memref.global"() <{constant, initial_value = dense<1.000000e+00> : tensor<1xf16>, sym_name = "memref5", sym_visibility = "private", type = memref<1xf16, 42 : i32>}> : () -> ()
+
 // CHECK-LABEL: func @read_global_memref
 func.func @read_global_memref() {
   %0 = memref.get_global @memref0 : memref<2xf32>
diff --git a/mlir/test/Dialect/OpenACC/invalid.mlir b/mlir/test/Dialect/OpenACC/invalid.mlir
index 8f6e961a0616..68afd9fccba7 100644
--- a/mlir/test/Dialect/OpenACC/invalid.mlir
+++ b/mlir/test/Dialect/OpenACC/invalid.mlir
@@ -819,3 +819,15 @@ func.func @acc_loop_container() {
   } attributes { collapse = [3], collapseDeviceType = [#acc.device_type<none>], independent = [#acc.device_type<none>]}
   return
 }
+
+// -----
+
+%value = memref.alloc() : memref<f32>
+// expected-error @below {{no data clause modifiers are allowed}}
+%0 = acc.private varPtr(%value : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier zero>}
+
+// -----
+
+%value = memref.alloc() : memref<f32>
+// expected-error @below {{invalid data clause modifiers: readonly}}
+%0 = acc.create varPtr(%value : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier readonly,zero,capture,always>}
diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir
index 97278f869534..7bb6cf43e49a 100644
--- a/mlir/test/Dialect/OpenACC/ops.mlir
+++ b/mlir/test/Dialect/OpenACC/ops.mlir
@@ -921,6 +921,31 @@ func.func @testdataop(%a: memref<f32>, %b: memref<f32>, %c: memref<f32>) -> () {
 
 // -----
 
+func.func @testdataopmodifiers(%a: memref<f32>, %b: memref<f32>, %c: memref<f32>) -> () {
+  %0 = acc.create varPtr(%a : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier capture,zero>}
+  %1 = acc.copyin varPtr(%b : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier readonly,capture,always>}
+  %2 = acc.copyin varPtr(%c : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier always>}
+  %3 = acc.create varPtr(%c : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier always>}
+  acc.data dataOperands(%0, %1, %2, %3 : memref<f32>, memref<f32>, memref<f32>, memref<f32>) {
+  }
+  acc.copyout accPtr(%0 : memref<f32>) to varPtr(%a : memref<f32>) {modifiers = #acc<data_clause_modifier zero,capture,always>}
+  acc.delete accPtr(%2 : memref<f32>) {modifiers = #acc<data_clause_modifier always>}
+  acc.copyout accPtr(%3 : memref<f32>) to varPtr(%c : memref<f32>) {modifiers = #acc<data_clause_modifier always>}
+
+  func.return
+}
+
+// CHECK:      func @testdataopmodifiers(%[[ARGA:.*]]: memref<f32>, %[[ARGB:.*]]: memref<f32>, %[[ARGC:.*]]: memref<f32>) {
+// CHECK:      %[[CREATEA:.*]] = acc.create varPtr(%[[ARGA]] : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier zero,capture>}
+// CHECK:      %[[COPYINB:.*]] = acc.copyin varPtr(%[[ARGB]] : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier always,readonly,capture>}
+// CHECK:      %[[COPYINC:.*]] = acc.copyin varPtr(%[[ARGC]] : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier always>}
+// CHECK:      %[[CREATEC:.*]] = acc.create varPtr(%[[ARGC]] : memref<f32>) -> memref<f32> {modifiers = #acc<data_clause_modifier always>}
+// CHECK:      acc.copyout accPtr(%[[CREATEA]] : memref<f32>) to varPtr(%[[ARGA]] : memref<f32>) {modifiers = #acc<data_clause_modifier always,zero,capture>}
+// CHECK:      acc.delete accPtr(%[[COPYINC]] : memref<f32>) {modifiers = #acc<data_clause_modifier always>}
+// CHECK:      acc.copyout accPtr(%[[CREATEC]] : memref<f32>) to varPtr(%[[ARGC]] : memref<f32>) {modifiers = #acc<data_clause_modifier always>}
+
+// -----
+
 func.func @testupdateop(%a: memref<f32>, %b: memref<f32>, %c: memref<f32>) -> () {
   %i64Value = arith.constant 1 : i64
   %i32Value = arith.constant 1 : i32
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 51e5ecf4607e..294b2cba5cf7 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -480,6 +480,39 @@ func.func @omp_simd_pretty_simdlen_safelen(%lb : index, %ub : index, %step : ind
 
 // -----
 
+func.func @omp_simd_bad_privatizer(%lb : index, %ub : index, %step : index) {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+  // expected-error @below {{Cannot find privatizer '@not_defined'}}
+  omp.simd private(@not_defined %1 -> %arg0 : !llvm.ptr) {
+    omp.loop_nest (%arg2) : index = (%lb) to (%ub) inclusive step (%step) {
+      omp.yield
+    }
+  }
+}
+
+// -----
+
+omp.private {type = firstprivate} @_QFEp_firstprivate_i32 : i32 copy {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+  %0 = llvm.load %arg0 : !llvm.ptr -> i32
+  llvm.store %0, %arg1 : i32, !llvm.ptr
+  omp.yield(%arg1 : !llvm.ptr)
+}
+func.func @omp_simd_firstprivate(%lb : index, %ub : index, %step : index) {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+  // expected-error @below {{FIRSTPRIVATE cannot be used with SIMD}}
+  omp.simd private(@_QFEp_firstprivate_i32 %1 -> %arg0 : !llvm.ptr) {
+    omp.loop_nest (%arg2) : index = (%lb) to (%ub) inclusive step (%step) {
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 // expected-error @below {{op expects alloc region to yield a value of the reduction type}}
 omp.declare_reduction @add_f32 : f32
 alloc {
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index 5e32a3a78c03..8ba8013d008a 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -1811,7 +1811,7 @@ module {
       %4 = affine.min #map2(%arg3)[%dim, %arg0]
       %extracted_slice0 = tensor.extract_slice %arg4[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice1 = tensor.extract_slice %arg5[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
-      %5 = linalg.elemwise_unary ins(%extracted_slice0 : tensor<?xf32>) outs(%extracted_slice1 : tensor<?xf32>) -> tensor<?xf32>
+      %5 = linalg.exp ins(%extracted_slice0 : tensor<?xf32>) outs(%extracted_slice1 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %5 into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
       }
@@ -1825,7 +1825,7 @@ module {
 //  CHECK-SAME:                       shared_outs(%[[ITER_ARG_5:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
 //       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
 //       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ITER_ARG_5]]
-//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      %[[ELEM:.*]] = linalg.exp ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
 //       CHECK:      scf.forall.in_parallel {
 //  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_5]]
 //  CHECK-NEXT:      }
@@ -1851,7 +1851,7 @@ module {
       %extracted_slice_0 = tensor.extract_slice %arg6[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice_1 = tensor.extract_slice %arg7[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
       %extracted_slice_2 = tensor.extract_slice %0[%3] [%4] [1] : tensor<?xf32> to tensor<?xf32>
-      %5 = linalg.elemwise_unary ins(%extracted_slice : tensor<?xf32>) outs(%extracted_slice_1 : tensor<?xf32>) -> tensor<?xf32>
+      %5 = linalg.exp ins(%extracted_slice : tensor<?xf32>) outs(%extracted_slice_1 : tensor<?xf32>) -> tensor<?xf32>
       scf.forall.in_parallel {
         tensor.parallel_insert_slice %5 into %arg6[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
         tensor.parallel_insert_slice %extracted_slice into %arg5[%3] [%4] [1] : tensor<?xf32> into tensor<?xf32>
@@ -1868,7 +1868,7 @@ module {
 //  CHECK-SAME:                       shared_outs(%[[ITER_ARG_6:.*]] = %[[ARG2]]) -> (tensor<?xf32>) {
 //       CHECK:      %[[OPERAND0:.*]] = tensor.extract_slice %[[ARG1]]
 //       CHECK:      %[[OPERAND1:.*]] = tensor.extract_slice %[[ARG3]]
-//       CHECK:      %[[ELEM:.*]] = linalg.elemwise_unary ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
+//       CHECK:      %[[ELEM:.*]] = linalg.exp ins(%[[OPERAND0]] : tensor<?xf32>) outs(%[[OPERAND1]] : tensor<?xf32>) -> tensor<?xf32>
 //       CHECK:      scf.forall.in_parallel {
 //  CHECK-NEXT:         tensor.parallel_insert_slice %[[ELEM]] into %[[ITER_ARG_6]]
 //  CHECK-NEXT:      }
diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir
index 25e5703cea5c..86af637fc05d 100644
--- a/mlir/test/Dialect/SCF/loop-pipelining.mlir
+++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir
@@ -34,6 +34,42 @@ func.func @simple_pipeline(%A: memref<?xf32>, %result: memref<?xf32>) {
   return
 }
 
+// -----
+
+// A static loop does not satisfy `numIteration >= maxStage`
+
+// CHECK-LABEL: func.func @iteration_lt_stage(
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+//   CHECK-DAG:   %[[C_NEG1:.*]] = arith.constant -1 : index
+//   CHECK-DAG:   %[[TRUE:.*]] = arith.constant true
+//   CHECK-DAG:   %[[FALSE:.*]] = arith.constant false
+// Prologue:
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[FALSE]]
+// Kernel:
+//       CHECK:   scf.for %[[IV:.*]] = %[[C0:.*]] to %[[C_NEG1:.*]] step %[[C1:.*]]
+// Epilogue:
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[TRUE]]
+//       CHECK:   scf.if %[[FALSE]]
+//       CHECK:   scf.if %[[FALSE]]
+func.func @iteration_lt_stage(%A: memref<?xf32>, %result: memref<?xf32>) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %cf = arith.constant 1.0 : f32
+  scf.for %i0 = %c0 to %c2 step %c1 {
+    %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>
+    %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 0 } : f32
+    memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 3, __test_pipelining_op_order__ = 1 } : memref<?xf32>
+  }  { __test_pipelining_loop__ }
+  return
+}
+
 
 // -----
 
diff --git a/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir b/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
index 8733ff93768a..56d477cca97b 100644
--- a/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/khr-cooperative-matrix-ops.mlir
@@ -58,6 +58,15 @@ spirv.func @cooperative_matrix_load_stride_i16(%ptr : !spirv.ptr<i32, StorageBuf
   spirv.Return
 }
 
+// CHECK-LABEL: @cooperative_matrix_load_aligned
+spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+  // CHECK:      {{%.*}} = spirv.KHR.CooperativeMatrixLoad {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16 :
+  // CHECK-SAME:   !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <Aligned>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  spirv.Return
+}
+
 // CHECK-LABEL: @cooperative_matrix_store
 spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                      %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
@@ -90,6 +99,16 @@ spirv.func @cooperative_matrix_store_stride_i16(%ptr : !spirv.ptr<i32, StorageBu
   spirv.Return
 }
 
+// CHECK-LABEL: @cooperative_matrix_store_aligned
+spirv.func @cooperative_matrix_store_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
+                                     %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
+  // CHECK:      spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16 :
+  // CHECK-SAME:   !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
+  spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
+  spirv.Return
+}
+
 // -----
 
 spirv.func @cooperative_matrix_load_bad_ptr(%ptr : !spirv.ptr<!spirv.struct<(f32 [0])>, StorageBuffer>, %stride : i32) "None" {
@@ -120,7 +139,7 @@ spirv.func @cooperative_matrix_load_bad_operad(%ptr : !spirv.ptr<i32, StorageBuf
 // -----
 
 spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <ColumnMajor>, <Aligned> :
     !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>
   spirv.Return
@@ -129,7 +148,7 @@ spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer
 // -----
 
 spirv.func @cooperative_matrix_load_aligned(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <ColumnMajor>, <Volatile|Aligned> :
     !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>
   spirv.Return
@@ -179,7 +198,7 @@ spirv.func @cooperative_matrix_store_bad_operand(%ptr : !spirv.ptr<i32, StorageB
 
 spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                      %m : !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>) "None" {
-  // expected-error @+1 {{op has unhandled memory operand 'Aligned'}}
+  // expected-error @+1 {{missing value for the 'Aligned' memory operand}}
   spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned> :
     !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<8x16xi32, Workgroup, MatrixA>, i32
   spirv.Return
@@ -187,6 +206,15 @@ spirv.func @cooperative_matrix_store(%ptr : !spirv.ptr<i32, StorageBuffer>, %str
 
 // -----
 
+spirv.func @cooperative_matrix_store_bad_operand_arg(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+  // expected-error @+1 {{found alignment attribute for non-'Aligned' memory operand}}
+  %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <MakePointerVisible>, 16 :
+    !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+  spirv.Return
+}
+
+// -----
+
 spirv.func @cooperative_matrix_muladd(%a : !spirv.coopmatrix<8x16xi32, Subgroup, MatrixA>,
                                       %b : !spirv.coopmatrix<16x4xi32, Subgroup, MatrixB>,
                                       %c : !spirv.coopmatrix<8x4xi32, Subgroup, MatrixAcc>) "None" {
diff --git a/mlir/test/Dialect/SPIRV/IR/types.mlir b/mlir/test/Dialect/SPIRV/IR/types.mlir
index c23894c62826..7d45b5ea8264 100644
--- a/mlir/test/Dialect/SPIRV/IR/types.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/types.mlir
@@ -564,3 +564,54 @@ func.func private @matrix_size_type(!spirv.matrix< x vector<3xi32>>) -> ()
 func.func private @matrix_size_type(!spirv.matrix<2.0 x vector<3xi32>>) -> ()
 
 // -----
+
+//===----------------------------------------------------------------------===//
+// TensorArm
+//===----------------------------------------------------------------------===//
+
+// CHECK: func private @arm_tensor_type_single_dim_i32(!spirv.arm.tensor<1xi32>)
+func.func private @arm_tensor_type_single_dim_i32(!spirv.arm.tensor<1xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_multi_dim_i32(!spirv.arm.tensor<1x2x3xi32>)
+func.func private @arm_tensor_type_multi_dim_i32(!spirv.arm.tensor<1x2x3xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_single_dim_f16(!spirv.arm.tensor<1xf16>)
+func.func private @arm_tensor_type_single_dim_f16(!spirv.arm.tensor<1xf16>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_multi_dim_f16(!spirv.arm.tensor<1x2x3xf16>)
+func.func private @arm_tensor_type_multi_dim_f16(!spirv.arm.tensor<1x2x3xf16>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<?xi32>)
+func.func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<?xi32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_dynamic_dim_2(!spirv.arm.tensor<?x?xi32>)
+func.func private @arm_tensor_type_dynamic_dim_2(!spirv.arm.tensor<?x?xi32>) -> ()
+// -----
+
+// expected-error @+1 {{arm.tensor shape dimensions must be either fully dynamic or completed shaped}}
+func.func private @arm_tensor_type_dynamic_dim(!spirv.arm.tensor<1x?xi32>) -> ()
+
+// -----
+
+// expected-error @+1 {{arm.tensors do not support rank zero}}
+func.func private @arm_tensor_rank_zero(!spirv.arm.tensor<i32>) -> ()
+
+// -----
+
+// CHECK: func private @arm_tensor_type_unranked(!spirv.arm.tensor<*xi32>)
+func.func private @arm_tensor_type_unranked(!spirv.arm.tensor<*xi32>) -> ()
+
+// -----
+
+// expected-error @+1 {{arm.tensors do not support zero dimensions}}
+func.func private @arm_tensor_type_zero_dim(!spirv.arm.tensor<0xi32>) -> ()
diff --git a/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir b/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
index 99d6a3dc390e..35fd7c33e4cf 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_vector_peeled.mlir
@@ -23,6 +23,7 @@
 // CHECK-DAG:   %[[c0:.*]] = arith.constant 0 : index
 // CHECK-DAG:   %[[c1:.*]] = arith.constant 1 : index
 // CHECK-DAG:   %[[c16:.*]] = arith.constant 16 : index
+// CHECK-DAG:   %[[mask:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK:       %[[p:.*]] = memref.load %{{.*}}[%[[c0]]] : memref<?xi32>
 // CHECK:       %[[a:.*]] = arith.extui %[[p]] : i32 to i64
 // CHECK:       %[[q:.*]] = arith.index_cast %[[a]] : i64 to index
@@ -31,7 +32,6 @@
 // CHECK:       %[[s:.*]] = arith.index_cast %[[b]] : i64 to index
 // CHECK:       %[[boundary:.*]] = affine.apply #[[$map0]]()[%[[q]], %[[s]]]
 // CHECK:       scf.for %[[i:.*]] = %[[q]] to %[[boundary]] step %[[c16]] {
-// CHECK:         %[[mask:.*]] = vector.constant_mask [16] : vector<16xi1>
 // CHECK:         %[[li:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xi32>, vector<16xi32>
 // CHECK:         %[[zi:.*]] = arith.extui %[[li]] : vector<16xi32> to vector<16xi64>
 // CHECK:         %[[la:.*]] = vector.load %{{.*}}[%[[i]]] : memref<?xf32>, vector<16xf32>
diff --git a/mlir/test/Dialect/Tensor/bufferize.mlir b/mlir/test/Dialect/Tensor/bufferize.mlir
index e9c3ba7e3b97..296ca02564e3 100644
--- a/mlir/test/Dialect/Tensor/bufferize.mlir
+++ b/mlir/test/Dialect/Tensor/bufferize.mlir
@@ -571,7 +571,7 @@ func.func @tensor.pad(%t1: tensor<?x10xindex>, %l2: index, %h1: index,
   // CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index
   // CHECK-DAG: %[[dim0:.*]] = memref.dim %[[m1]], %[[c0]]
   // CHECK-DAG: %[[dim1:.*]] = memref.dim %[[m1]], %[[c1]]
-  // CHECK-DAG: %[[size0:.*]] = affine.apply #[[$sum_map_1]]()[%[[h1]], %[[dim0]]]
+  // CHECK-DAG: %[[size0:.*]] = affine.apply #[[$sum_map_1]]()[%[[dim0]], %[[h1]]]
   // CHECK-DAG: %[[size1:.*]] = affine.apply #[[$sum_map_2]]()[%[[l2]], %[[h2]]]
   // CHECK:     %[[alloc:.*]] = memref.alloc(%[[size0]], %[[size1]]) {{.*}} : memref<?x?xindex>
   // CHECK:     %[[alloc_t:.*]] = bufferization.to_tensor %[[alloc]]
@@ -615,6 +615,21 @@ func.func @tensor.splat(%f: f32) -> tensor<10x2x4xf32> {
 
 // -----
 
+// CHECK-LABEL:   func @tensor.splat_other(
+// CHECK-SAME:        %[[F:.*]]: !test.memref_element)
+// CHECK-DAG:       %[[ALLOC:.*]] = memref.alloc() {{.*}} : memref<10x2x4x!test.memref_element>
+// CHECK:           %[[ALLOC_T:.*]] = bufferization.to_tensor %[[ALLOC]]
+// CHECK:           %[[MAPPED:.*]] = linalg.map
+// CHECK:                 outs(%[[ALLOC_T]] : tensor<10x2x4x!test.memref_element>)
+// CHECK:             linalg.yield %[[F]]
+// CHECK:           return %[[MAPPED]] : tensor<10x2x4x!test.memref_element>
+func.func @tensor.splat_other(%f: !test.memref_element) -> tensor<10x2x4x!test.memref_element> {
+  %t = tensor.splat %f : tensor<10x2x4x!test.memref_element>
+  return %t : tensor<10x2x4x!test.memref_element>
+}
+
+// -----
+
 // CHECK-LABEL:   func @tensor.concat(
 // CHECK-SAME:        %[[F:.*]]: tensor<8xf32>)
 // CHECK:           %[[F_MEMREF:.*]] = bufferization.to_buffer %[[F]]
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
index 3251c5a4a2bf..95c5b8c91edf 100644
--- a/mlir/test/Dialect/Tensor/canonicalize.mlir
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -971,6 +971,17 @@ func.func @fold_reshape_1d(%input: tensor<?xf32>, %shape: tensor<1xindex>) -> te
 
 // -----
 
+// CHECK-LABEL: func @fold_reshape_0d
+//  CHECK-SAME: %[[INPUT:[a-zA-Z0-9_]+]]: tensor<f32>
+//  CHECK-SAME: %[[SHAPE:[a-zA-Z0-9_]+]]: tensor<0xindex>
+//       CHECK: return %[[INPUT]]
+func.func @fold_reshape_0d(%input: tensor<f32>, %shape: tensor<0xindex>) -> tensor<f32> {
+  %0 = tensor.reshape %input(%shape) : (tensor<f32>, tensor<0xindex>) -> tensor<f32>
+  return %0 : tensor<f32>
+}
+
+// -----
+
 // CHECK-LABEL: func @fold_extract_constant_splat
 //   CHECK-NOT: tensor.extract_slice
 //       CHECK: arith.constant dense<42> : tensor<4x4xi32>
@@ -1243,6 +1254,24 @@ func.func @compose_collapse_of_expand_1D(%arg0 : tensor<2048xf32>)
 
 // -----
 
+func.func @compose_collapse_of_expand_partially_dynamic(%arg0: tensor<?xf16>, %arg1: index, %arg2: index) -> tensor<8x?x?xf16> {
+  %expanded = tensor.expand_shape %arg0 [[0, 1, 2, 3, 4]] output_shape [4, 2, %arg1, %arg2, 32] : tensor<?xf16> into tensor<4x2x?x?x32xf16>
+  %collapsed = tensor.collapse_shape %expanded [[0, 1], [2], [3, 4]] : tensor<4x2x?x?x32xf16> into tensor<8x?x?xf16>
+  return %collapsed : tensor<8x?x?xf16>
+}
+//       CHECK: func @compose_collapse_of_expand_partially_dynamic
+//  CHECK-SAME:   %[[SRC:.[a-zA-Z0-9]+]]
+//  CHECK-SAME:   %[[ORIG_D2:.[a-zA-Z0-9]+]]
+//  CHECK-SAME:   %[[ORIG_D3:.[a-zA-Z0-9]+]]
+//   CHECK-DAG:   %[[C32:.+]] = arith.constant 32
+//       CHECK:   %[[COLLAPSED_D2:.+]] = arith.muli %[[ORIG_D3]], %[[C32]]
+//       CHECK:   %[[RESULT:.+]] = tensor.expand_shape %[[SRC]]
+//  CHECK-SAME:     [0, 1, 2]
+//  CHECK-SAME:     output_shape [8, %[[ORIG_D2]], %[[COLLAPSED_D2]]]
+//       CHECK:   return %[[RESULT]]
+
+// -----
+
 func.func @compose_expand_of_collapse_0_rank_to_expand(%arg0 : tensor<1x1x1xf32>)
     -> tensor<1x1x1x1xf32> {
   %0 = tensor.collapse_shape %arg0 []
diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
index 84eb60248b8b..16efa73f87a2 100644
--- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
+++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt -split-input-file -test-linalg-transform-patterns=test-fold-into-pack-and-unpack  %s | FileCheck %s
+// RUN: mlir-opt -split-input-file -test-linalg-transform-patterns=test-fold-into-pack-and-unpack-control  %s | FileCheck %s --check-prefix=CONTROL
 
 func.func @fold_unpack_slice(%arg0 : tensor<?x?x8x4xf32>, %arg1 : tensor<?x?xf32>,
     %arg2 : index, %arg3 : index) -> tensor<?x?xf32> {
@@ -373,6 +374,36 @@ func.func @linalg_transpose_linalg.pack_fold(%arg0: tensor<56x57x1x64xf32>) -> t
 
 // -----
 
+func.func @linalg_transpose_linalg.pack_fold_multi_result(%arg0: tensor<56x57x1x64xf32>) -> (tensor<1x56x57x64xf32>, tensor<1x57x56x2x32xf32>) {
+  %0 = tensor.empty() : tensor<1x56x57x64xf32>
+  %transposed = linalg.transpose
+    ins(%arg0 : tensor<56x57x1x64xf32>)
+    outs(%0 : tensor<1x56x57x64xf32>)
+    permutation = [2, 0, 1, 3]
+
+  %1 = tensor.empty() : tensor<1x57x56x2x32xf32>
+  %pack = linalg.pack %transposed
+    outer_dims_perm = [0, 2, 1, 3]
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %1 : tensor<1x56x57x64xf32> -> tensor<1x57x56x2x32xf32>
+  return %transposed, %pack : tensor<1x56x57x64xf32>, tensor<1x57x56x2x32xf32>
+}
+// CHECK-LABEL: func @linalg_transpose_linalg.pack_fold_multi_result(
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<56x57x1x64xf32>)
+//       CHECK:   %[[TRANSPOSE:.+]] = linalg.transpose
+//       CHECK:   %[[PACK:.+]] = linalg.pack %[[ARG0]]
+//  CHECK-SAME:      outer_dims_perm = [2, 1, 0, 3]
+//       CHECK:   return %[[TRANSPOSE]], %[[PACK]]
+
+// CONTROL-LABEL: func @linalg_transpose_linalg.pack_fold_multi_result(
+//       CONTROL:   %[[TRANSPOSE:.+]] = linalg.transpose
+//       CONTROL:   %[[PACK:.+]] = linalg.pack %[[TRANSPOSE]]
+//  CONTROL-SAME:      outer_dims_perm = [0, 2, 1, 3]
+//       CONTROL:   return %[[TRANSPOSE]], %[[PACK]]
+
+// -----
+
 func.func @linalg_transpose_linalg.pack_fold_with_padding(%arg0: tensor<56x57x1x55xf32>, %padding: f32) -> tensor<1x57x56x2x32xf32> {
   %0 = tensor.empty() : tensor<1x56x57x55xf32>
   %transpose = linalg.transpose
@@ -550,6 +581,36 @@ func.func @linalg_transpose_linalg.unpack_fold(%arg0: tensor<1x1x4x16xi32>) -> t
 
 // -----
 
+func.func @linalg_transpose_linalg.unpack_fold_multi_result(%arg0: tensor<1x1x4x16xi32>) -> (tensor<1x1x16x4xi32>, tensor<16x4xi32>) {
+  %0 = tensor.empty() : tensor<1x1x16x4xi32>
+  %transposed = linalg.transpose ins(%arg0 : tensor<1x1x4x16xi32>)
+                outs(%0 : tensor<1x1x16x4xi32>)
+                permutation = [1, 0, 3, 2]
+  %1 = tensor.empty() : tensor<16x4xi32>
+  %unpack = linalg.unpack %transposed
+            outer_dims_perm = [0, 1]
+            inner_dims_pos = [0, 1]
+            inner_tiles = [16, 4] into
+            %1 : tensor<1x1x16x4xi32> -> tensor<16x4xi32>
+  return %transposed, %unpack : tensor<1x1x16x4xi32>, tensor<16x4xi32>
+}
+//CHECK-LABEL:  func.func @linalg_transpose_linalg.unpack_fold_multi_result(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<1x1x4x16xi32>) 
+//      CHECK:     %[[TRANSPOSE:.+]] = linalg.transpose
+//      CHECK:     %[[UNPACK:.+]] = linalg.unpack %[[ARG0]]
+// CHECK-SAME:        outer_dims_perm = [1, 0]
+//      CHECK:     return %[[TRANSPOSE]], %[[UNPACK]]
+//      CHECK:   }
+
+//CONTROL-LABEL:  func.func @linalg_transpose_linalg.unpack_fold_multi_result(
+//      CONTROL:     %[[TRANSPOSE:.+]] = linalg.transpose
+//      CONTROL:     %[[UNPACK:.+]] = linalg.unpack %[[TRANSPOSE]]
+// CONTROL-SAME:        outer_dims_perm = [0, 1]
+//      CONTROL:     return %[[TRANSPOSE]], %[[UNPACK]]
+//      CONTROL:   }
+
+// -----
+
 func.func @linalg_transpose_linalg.unpack_fold_partial_tile(%arg0: tensor<1x1x4x16xi32>) -> tensor<15x3xi32> {
   %0 = tensor.empty() : tensor<1x1x16x4xi32>
   %transposed = linalg.transpose ins(%arg0 : tensor<1x1x4x16xi32>)
diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir
index f35d52e70008..665657a67dc6 100644
--- a/mlir/test/Dialect/Tensor/invalid.mlir
+++ b/mlir/test/Dialect/Tensor/invalid.mlir
@@ -466,9 +466,10 @@ func.func @invalid_splat(%v : f32) {
 
 // -----
 
-func.func @invalid_splat(%v : vector<8xf32>) {
-  // expected-error@+1 {{must be integer/index/float type}}
-  %w = tensor.splat %v : tensor<8xvector<8xf32>>
+// expected-note@+1 {{prior use here}}
+func.func @invalid_splat(%v : f32) {
+  // expected-error@+1 {{expects different type than prior uses: 'i32' vs 'f32'}}
+  %w = tensor.splat %v : tensor<1xi32>
   return
 }
 
diff --git a/mlir/test/Dialect/Tensor/ops.mlir b/mlir/test/Dialect/Tensor/ops.mlir
index 930986211cb6..681a934ba069 100644
--- a/mlir/test/Dialect/Tensor/ops.mlir
+++ b/mlir/test/Dialect/Tensor/ops.mlir
@@ -313,13 +313,17 @@ func.func @pad_to_static_size(%arg0: tensor<?x?xf32>, %ub0: index, %ub1: index,
 // -----
 
 // CHECK-LABEL: func @test_splat_op
-// CHECK-SAME: [[S:%arg[0-9]+]]: f32
-func.func @test_splat_op(%s : f32) {
-  // CHECK: tensor.splat [[S]] : tensor<8xf32>
+// CHECK-SAME: %[[S:.*]]: f32
+// CHECK-SAME: %[[P:.*]]: !llvm.ptr
+func.func @test_splat_op(%s : f32, %p : !llvm.ptr) {
+  // CHECK: tensor.splat %[[S]] : tensor<8xf32>
   %v = tensor.splat %s : tensor<8xf32>
 
-  // CHECK: tensor.splat [[S]] : tensor<4xf32>
+  // CHECK: tensor.splat %[[S]] : tensor<4xf32>
   %u = "tensor.splat"(%s) : (f32) -> tensor<4xf32>
+
+  // CHECK: tensor.splat %[[P]] : tensor<8x!llvm.ptr>
+  %w = tensor.splat %p : tensor<8x!llvm.ptr>
   return
 }
 
diff --git a/mlir/test/Dialect/Tensor/reify-shapes.mlir b/mlir/test/Dialect/Tensor/reify-shapes.mlir
new file mode 100644
index 000000000000..5569d90f8b73
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/reify-shapes.mlir
@@ -0,0 +1,31 @@
+// RUN: mlir-opt -reify-result-shapes  %s | FileCheck %s
+
+// The test below checks concat op reification. In the first case, no cast is inserted while on the second a cast gets inserted.
+// CHECK-LABEL:  func.func @concat_reification
+func.func @concat_reification(%arg0: tensor<4x7x3xf32>, %arg1 : tensor<4x4x3xf32>, %arg2: tensor<?x?x?xf32>)
+  -> (tensor<4x11x3xf32>, tensor<?x?x?xf32>) {
+  // CHECK: %[[RES0:.*]] = tensor.concat dim(1) %{{.*}} : (tensor<4x7x3xf32>, tensor<4x4x3xf32>) -> tensor<4x11x3xf32>
+  %1 = tensor.concat dim(1) %arg0, %arg1 : (tensor<4x7x3xf32>, tensor<4x4x3xf32>) -> tensor<4x11x3xf32>
+  // CHECK: %[[V0:.*]] = tensor.concat dim(2) %{{.*}} : (tensor<4x7x3xf32>, tensor<?x?x?xf32>) -> tensor<4x7x?xf32>
+  // CHECK: %[[RES1:.*]] = tensor.cast %[[V0]] : tensor<4x7x?xf32> to tensor<?x?x?xf32>
+  %2 = tensor.concat dim(2) %arg0, %arg2 : (tensor<4x7x3xf32>, tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+  // CHECK: return %[[RES0]], %[[RES1]] : tensor<4x11x3xf32>, tensor<?x?x?xf32>
+  return %1, %2 : tensor<4x11x3xf32>, tensor<?x?x?xf32>
+}
+
+// CHECK-LABEL:  func.func @pad_reification
+func.func @pad_reification(%cst : f32, %idx : index, %t: tensor<64x?x64xf32>) -> tensor<1x?x64xf32> {
+  %pad_amt = affine.apply affine_map<(d0) -> (-d0 + 256)>(%idx)
+  %es = tensor.extract_slice %t[0, 0, 0] [1, %idx, 64] [1, 1, 1] 
+    : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+
+  // CHECK: tensor.pad
+  // CHECK: : tensor<1x?x64xf32> to tensor<1x256x64xf32>
+  // CHECK: tensor.cast %{{.*}} : tensor<1x256x64xf32> to tensor<1x?x64xf32>
+  %padded = tensor.pad %es low[0, 0, 0] high[0, %pad_amt, 0] {
+    ^bb0(%a: index, %b: index, %c: index):
+    tensor.yield %cst : f32
+  } : tensor<1x?x64xf32> to tensor<1x?x64xf32>
+
+  return %padded : tensor<1x?x64xf32>
+}
diff --git a/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
index 6610d3180cf0..e526adb18cf1 100644
--- a/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
@@ -213,3 +213,20 @@ func.func @dynamic_dims_are_maybe_equal_2(%t: tensor<?x?xf32>) {
   "test.compare"(%dim0, %dim1) : (index, index) -> ()
   return
 }
+
+// -----
+
+// CHECK-LABEL:  func.func @pad_reification
+func.func @pad_reification(%cst : f32, %idx : index, %t: tensor<64x?x64xf32>) {
+  %pad_amt = affine.apply affine_map<(d0) -> (-d0 + 256)>(%idx)
+  %es = tensor.extract_slice %t[0, 0, 0] [1, %idx, 64] [1, 1, 1] : tensor<64x?x64xf32> to tensor<1x?x64xf32>
+
+  %padded = tensor.pad %es low[0, 0, 0] high[0, %pad_amt, 0] {
+  ^bb0(%a: index, %b: index, %c: index):
+    tensor.yield %cst : f32
+  } : tensor<1x?x64xf32> to tensor<1x?x64xf32>
+
+  // CHECK: arith.constant 256 : index
+  %1 = "test.reify_bound"(%padded) {dim = 1, constant} : (tensor<1x?x64xf32>) -> (index)
+  return
+}
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index 6fa6756822ac..5a424c41775c 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -2000,6 +2000,7 @@ func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui8>) -> (tensor<1x1xi8
   %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
   %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
   %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op is not profile-aligned: element type 'ui8' is not legal}}
   %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
   return %r : tensor<1x1xi8>
 }
@@ -2012,6 +2013,7 @@ func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui
   %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
   %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
   %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // expected-error@+1 {{'tosa.rescale' op is not profile-aligned: element type 'ui8' is not legal}}
   %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
   return %r : tensor<1x1xui8>
 }
diff --git a/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir b/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir
new file mode 100644
index 000000000000..38ac8d8fb66d
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/tosa-convert-integer-type-to-signless.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-opt --split-input-file --tosa-convert-integer-type-to-signless %s | FileCheck %s
+
+// -----
+
+// CHECK-LABEL: test_rescale_output_unsigned
+// CHECK: %arg0: tensor<1x1xi8>
+func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  // CHECK: %[[RESCALE:.*]] = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
+  // CHECK: return %[[RESCALE]] : tensor<1x1xi8>
+  return %r : tensor<1x1xui8>
+}
+
+// -----
+
+// CHECK-LABEL: test_rescale_input_unsigned
+// CHECK: %arg0: tensor<1x1xi16>
+func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui16>) -> (tensor<1x1xi8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<32768> : tensor<1xi16>}> : () -> tensor<1xi16>
+  // CHECK: %[[RESCALE:.*]] = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi8>) -> tensor<1x1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui16>, tensor<1xi32>, tensor<1xi8>, tensor<1xi16>, tensor<1xi8>) -> tensor<1x1xi8>
+  // CHECK: return %[[RESCALE]] : tensor<1x1xi8>
+  return %r : tensor<1x1xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_unsigned_function_signature
+// CHECK: %arg0: tensor<1xi8>, %arg1: tensor<1xi8>
+func.func @test_unsigned_function_signature(%arg0: tensor<1xui8>, %arg1: tensor<1xui8>) -> (tensor<1xui8>, tensor<1xui8>) {
+  // CHECK: return %arg0, %arg1 : tensor<1xi8>, tensor<1xi8>
+  return %arg0, %arg1 : tensor<1xui8>, tensor<1xui8>
+}
+
+// -----
+
+// CHECK-LABEL: test_no_change
+// CHECK: %arg0: tensor<13x21x3xi8>
+func.func @test_no_change(%arg0: tensor<13x21x3xi8>) -> tensor<13x21x3xi8> {
+  %0 = tosa.reverse %arg0 {axis = 0 : i32} : (tensor<13x21x3xi8>) -> tensor<13x21x3xi8>
+  // CHECK: return %0 : tensor<13x21x3xi8>
+  return %0 : tensor<13x21x3xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_regions
+// CHECK: %arg0: tensor<i8>, %arg1: tensor<i8>
+func.func @test_regions(%arg0: tensor<ui8>, %arg1: tensor<ui8>, %arg2: tensor<i1>) -> tensor<ui8> {
+  // CHECK: tosa.cond_if %arg2 -> (tensor<i8>)
+  %0 = "tosa.cond_if"(%arg2, %arg0, %arg1) ({
+  ^bb0(%arg3: tensor<ui8>, %arg4: tensor<ui8>):
+    // CHECK: %1 = tosa.add %arg0, %arg1 : (tensor<i8>, tensor<i8>) -> tensor<i8>
+    %1 = tosa.add %arg0, %arg1 : (tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+    // CHECK: tosa.yield %1 : tensor<i8>
+    tosa.yield %1 : tensor<ui8>
+  },  {
+  ^bb0(%arg3: tensor<ui8>, %arg4: tensor<ui8>):
+    // CHECK: %1 = tosa.sub %arg0, %arg1 : (tensor<i8>, tensor<i8>) -> tensor<i8>
+    %1 = tosa.sub %arg0, %arg1 : (tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+    // CHECK: tosa.yield %1 : tensor<i8>
+    tosa.yield %1 : tensor<ui8>
+  }) : (tensor<i1>, tensor<ui8>, tensor<ui8>) -> tensor<ui8>
+  // CHECK: return %0 : tensor<i8>
+  return %0 : tensor<ui8>
+}
diff --git a/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir b/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir
new file mode 100644
index 000000000000..cab14201dc0c
--- /dev/null
+++ b/mlir/test/Dialect/Tosa/tosa-validation-valid.mlir
@@ -0,0 +1,31 @@
+//--------------------------------------------------------------------------------------------------
+// Test valid IR in terms of the shape and type of tensor, and the argument type of
+// operation. Excludes the profile compilance checking since it is performed earlier in the
+// validation flow.
+//--------------------------------------------------------------------------------------------------
+
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics --tosa-validate="profile=pro_int,pro_fp extension=int16,int4,bf16,fp8e4m3,fp8e5m2,fft,variable,controlflow,doubleround,inexactround" | FileCheck %s
+
+// -----
+
+// CHECK-LABEL: test_rescale_input_unsigned
+func.func @test_rescale_input_unsigned(%arg0: tensor<1x1xui8>) -> (tensor<1x1xi8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = true, output_unsigned = false, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xui8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xi8>
+  return %r : tensor<1x1xi8>
+}
+
+// -----
+
+// CHECK-LABEL: test_rescale_output_unsigned
+func.func @test_rescale_output_unsigned(%arg0: tensor<1x1xi8>) -> (tensor<1x1xui8>) {
+  %0 = "tosa.const"() <{values = dense<1> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %1 = "tosa.const"() <{values = dense<2> : tensor<1xi32>}> : () -> tensor<1xi32>
+  %2 = "tosa.const"() <{values = dense<3> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %3 = "tosa.const"() <{values = dense<-128> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %r = tosa.rescale %arg0, %1, %0, %3, %2 {input_unsigned = false, output_unsigned = true, per_channel = false, rounding_mode = "SINGLE_ROUND", scale32 = true} : (tensor<1x1xi8>, tensor<1xi32>, tensor<1xi8>, tensor<1xi8>, tensor<1xi8>) -> tensor<1x1xui8>
+  return %r : tensor<1x1xui8>
+}
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 65b73375831d..8cda8d47cb90 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -14,7 +14,7 @@ func.func @create_vector_mask_to_constant_mask() -> (vector<4x3xi1>) {
 // CHECK-LABEL: create_scalable_vector_mask_to_constant_mask
 func.func @create_scalable_vector_mask_to_constant_mask() -> (vector<[8]xi1>) {
   %c-1 = arith.constant -1 : index
-  // CHECK: vector.constant_mask [0] : vector<[8]xi1>
+  // CHECK: arith.constant dense<false> : vector<[8]xi1>
   %0 = vector.create_mask %c-1 : vector<[8]xi1>
   return %0 : vector<[8]xi1>
 }
@@ -36,7 +36,7 @@ func.func @create_vector_mask_to_constant_mask_truncation() -> (vector<4x3xi1>)
 func.func @create_vector_mask_to_constant_mask_truncation_neg() -> (vector<4x3xi1>) {
   %cneg2 = arith.constant -2 : index
   %c5 = arith.constant 5 : index
-  // CHECK: vector.constant_mask [0, 0] : vector<4x3xi1>
+  // CHECK: arith.constant dense<false> : vector<4x3xi1>
   %0 = vector.create_mask %c5, %cneg2 : vector<4x3xi1>
   return %0 : vector<4x3xi1>
 }
@@ -47,7 +47,7 @@ func.func @create_vector_mask_to_constant_mask_truncation_neg() -> (vector<4x3xi
 func.func @create_vector_mask_to_constant_mask_truncation_zero() -> (vector<4x3xi1>) {
   %c2 = arith.constant 2 : index
   %c0 = arith.constant 0 : index
-  // CHECK: vector.constant_mask [0, 0] : vector<4x3xi1>
+  // CHECK: arith.constant dense<false> : vector<4x3xi1>
   %0 = vector.create_mask %c0, %c2 : vector<4x3xi1>
   return %0 : vector<4x3xi1>
 }
@@ -60,7 +60,7 @@ func.func @create_vector_mask_to_constant_mask_scalable_all_true() -> (vector<8x
   %c16 = arith.constant 16 : index
   %0 = vector.vscale
   %1 = arith.muli %0, %c16 : index
-  // CHECK: vector.constant_mask [8, 16] : vector<8x[16]xi1>
+  // CHECK: arith.constant dense<true> : vector<8x[16]xi1>
   %10 = vector.create_mask %c8, %1 : vector<8x[16]xi1>
   return %10 : vector<8x[16]xi1>
 }
@@ -272,6 +272,30 @@ func.func @extract_from_non_constant_create_mask(%dim0: index) -> vector<[2]xi1>
 
 // -----
 
+// CHECK-LABEL: constant_mask_to_true_splat
+func.func @constant_mask_to_true_splat() -> vector<2x4xi1> {
+  // CHECK: arith.constant dense<true>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [2, 4] : vector<2x4xi1>
+  return %0 : vector<2x4xi1>
+}
+
+// CHECK-LABEL: constant_mask_to_false_splat
+func.func @constant_mask_to_false_splat() -> vector<2x4xi1> {
+  // CHECK: arith.constant dense<false>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [0, 0] : vector<2x4xi1>
+  return %0 : vector<2x4xi1>
+}
+
+// CHECK-LABEL: constant_mask_to_true_splat_0d
+func.func @constant_mask_to_true_splat_0d() -> vector<i1> {
+  // CHECK: arith.constant dense<true>
+  // CHECK-NOT: vector.constant_mask
+  %0 = vector.constant_mask [1] : vector<i1>
+  return %0 : vector<i1>
+}
+
 // CHECK-LABEL: constant_mask_transpose_to_transposed_constant_mask
 func.func @constant_mask_transpose_to_transposed_constant_mask() -> (vector<2x3x4xi1>, vector<4x2x3xi1>) {
   //     CHECK: vector.constant_mask [1, 2, 3] : vector<2x3x4xi1>
@@ -289,7 +313,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x2xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x2xi1>
-  // CHECK: vector.constant_mask [2, 2] : vector<2x2xi1>
+  // CHECK: arith.constant dense<true> : vector<2x2xi1>
   return %1 : vector<2x2xi1>
 }
 
@@ -322,7 +346,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x2xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x2xi1>
-  // CHECK: vector.constant_mask [0, 0] : vector<2x2xi1>
+  // CHECK: arith.constant dense<false> : vector<2x2xi1>
   return %1 : vector<2x2xi1>
 }
 
@@ -333,7 +357,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x1xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 2], sizes = [2, 1], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x1xi1>
-  // CHECK: vector.constant_mask [0, 0] : vector<2x1xi1>
+  // CHECK: arith.constant dense<false> : vector<2x1xi1>
   return %1 : vector<2x1xi1>
 }
 
@@ -344,7 +368,7 @@ func.func @extract_strided_slice_of_constant_mask() -> (vector<2x1xi1>) {
   %1 = vector.extract_strided_slice %0
     {offsets = [0, 1], sizes = [2, 1], strides = [1, 1]}
       : vector<4x3xi1> to vector<2x1xi1>
-  // CHECK: vector.constant_mask [2, 1] : vector<2x1xi1>
+  // CHECK: arith.constant dense<true> : vector<2x1xi1>
   return %1 : vector<2x1xi1>
 }
 
@@ -3075,6 +3099,33 @@ func.func @from_elements_to_elements_shuffle(%a: vector<4x2xf32>) -> vector<4x2x
 
 // -----
 
+// CHECK-LABEL: func @from_elements_all_elements_constant(
+func.func @from_elements_all_elements_constant() -> vector<2x2xi32> {
+  %c0_i32 = arith.constant 0 : i32
+  %c1_i32 = arith.constant 1 : i32
+  %c2_i32 = arith.constant 2 : i32
+  %c3_i32 = arith.constant 3 : i32
+  // CHECK: %[[RES:.*]] = arith.constant dense<{{\[\[0, 1\], \[2, 3\]\]}}> : vector<2x2xi32>
+  %res = vector.from_elements %c0_i32, %c1_i32, %c2_i32, %c3_i32 : vector<2x2xi32>
+  // CHECK: return %[[RES]]
+  return %res : vector<2x2xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_partial_elements_constant(
+// CHECK-SAME:     %[[A:.*]]: f32
+func.func @from_elements_partial_elements_constant(%arg0: f32) -> vector<2xf32> {
+  // CHECK: %[[C:.*]] = arith.constant 1.000000e+00 : f32
+  %c = arith.constant 1.0 : f32
+  // CHECK: %[[RES:.*]] = vector.from_elements %[[A]], %[[C]] : vector<2xf32>
+  %res = vector.from_elements %arg0, %c : vector<2xf32>
+  // CHECK: return %[[RES]]
+  return %res : vector<2xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @vector_insert_const_regression(
 //       CHECK:   llvm.mlir.undef
 //       CHECK:   vector.insert
diff --git a/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir b/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir
index c84aea660966..f1e1c5e896c6 100644
--- a/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize/vector-transpose.mlir
@@ -165,6 +165,25 @@ func.func @shape_cast_of_transpose(%arg : vector<1x4x4x1x1xi8>) -> vector<4x4xi8
 
 // -----
 
+// In this test, the permutation maps the non-unit dimensions (1 and 2) as follows:
+// 1 -> 0
+// 2 -> 4
+// Because 0 < 4, this permutation is order preserving and effectively a shape_cast.
+// (same as the example above, but one of the dims is scalable)
+// CHECK-LABEL: @shape_cast_of_transpose_scalable
+//  CHECK-SAME:   %[[ARG:.*]]: vector<1x[4]x4x1x1xi8>) -> vector<[4]x4xi8> {
+//       CHECK:   %[[SHAPE_CAST:.*]] = vector.shape_cast %[[ARG]] :
+//  CHECK-SAME:   vector<1x[4]x4x1x1xi8> to vector<[4]x4xi8>
+//       CHECK:   return %[[SHAPE_CAST]] : vector<[4]x4xi8>
+func.func @shape_cast_of_transpose_scalable(%arg : vector<1x[4]x4x1x1xi8>) -> vector<[4]x4xi8> {
+  %0 = vector.transpose %arg, [1, 0, 3, 4, 2]
+     : vector<1x[4]x4x1x1xi8> to vector<[4]x1x1x1x4xi8>
+  %1 = vector.shape_cast %0 : vector<[4]x1x1x1x4xi8> to vector<[4]x4xi8>
+  return %1 : vector<[4]x4xi8>
+}
+
+// -----
+
 // In this test, the mapping of non-unit dimensions (1 and 2) is as follows:
 // 1 -> 2
 // 2 -> 1
@@ -184,36 +203,10 @@ func.func @negative_shape_cast_of_transpose(%arg : vector<1x4x4x1xi8>) -> vector
 
 // -----
 
-// Currently the conversion shape_cast(transpose) -> shape_cast is disabled for
-// scalable vectors because of bad interaction with ConvertIllegalShapeCastOpsToTransposes
-// CHECK-LABEL: @negative_shape_cast_of_transpose_scalable
-//       CHECK:  vector.transpose
-//       CHECK:  vector.shape_cast
-func.func @negative_shape_cast_of_transpose_scalable(%arg : vector<[4]x1xi8>) -> vector<[4]xi8> {
-  %0 = vector.transpose %arg, [1, 0] : vector<[4]x1xi8> to vector<1x[4]xi8>
-  %1 = vector.shape_cast %0 : vector<1x[4]xi8> to vector<[4]xi8>
-  return %1 : vector<[4]xi8>
-}
-
-// -----
-
 /// +--------------------------------------------------------------------------
 /// Tests of FoldTransposeShapeCast:  transpose(shape_cast) -> shape_cast
 /// +--------------------------------------------------------------------------
 
-// The conversion transpose(shape_cast) -> shape_cast is not disabled for scalable
-// vectors.
-// CHECK-LABEL: @transpose_of_shape_cast_scalable
-//       CHECK: vector.shape_cast
-//  CHECK-SAME: vector<[4]xi8> to vector<[4]x1xi8>
-func.func @transpose_of_shape_cast_scalable(%arg : vector<[4]xi8>) -> vector<[4]x1xi8> {
-  %0 = vector.shape_cast %arg : vector<[4]xi8> to vector<1x[4]xi8>
-  %1 = vector.transpose %0, [1, 0] : vector<1x[4]xi8> to vector<[4]x1xi8>
-  return %1 : vector<[4]x1xi8>
-}
-
-// -----
-
 // A transpose that is 'order preserving' can be treated like a shape_cast. 
 // CHECK-LABEL: @transpose_of_shape_cast
 //  CHECK-SAME:   %[[ARG:.*]]: vector<2x3x1x1xi8>) -> vector<6x1x1xi8> {
@@ -229,11 +222,26 @@ func.func @transpose_of_shape_cast(%arg : vector<2x3x1x1xi8>) ->  vector<6x1x1xi
 
 // -----
 
-// Scalable dimensions should be treated as non-unit dimensions.
 // CHECK-LABEL: @transpose_of_shape_cast_scalable
+//  CHECK-SAME:   %[[ARG:.*]]: vector<[2]x3x1x1xi8>) -> vector<[6]x1x1xi8> {
+//       CHECK:   %[[SHAPE_CAST:.*]] = vector.shape_cast %[[ARG]] :
+//  CHECK-SAME:   vector<[2]x3x1x1xi8> to vector<[6]x1x1xi8>
+//       CHECK:   return %[[SHAPE_CAST]] : vector<[6]x1x1xi8>
+func.func @transpose_of_shape_cast_scalable(%arg : vector<[2]x3x1x1xi8>) ->  vector<[6]x1x1xi8> {
+  %0 = vector.shape_cast %arg : vector<[2]x3x1x1xi8> to vector<[6]x1x1xi8>
+  %1 = vector.transpose %0, [0, 2, 1]
+     : vector<[6]x1x1xi8> to vector<[6]x1x1xi8>
+  return %1 : vector<[6]x1x1xi8>
+}
+
+// -----
+
+// Scalable 1 dimensions (i.e. [1]) should be treated as non-unit dimensions
+// (hence no folding).
+// CHECK-LABEL: @negative_transpose_of_shape_cast_scalable_unit
 //       CHECK: vector.shape_cast
 //       CHECK: vector.transpose
-func.func @transpose_of_shape_cast_scalable_unit(%arg : vector<[1]x4x1xi8>) -> vector<4x[1]xi8> {
+func.func @negative_transpose_of_shape_cast_scalable_unit(%arg : vector<[1]x4x1xi8>) -> vector<4x[1]xi8> {
   %0 = vector.shape_cast %arg : vector<[1]x4x1xi8> to vector<[1]x4xi8>
   %1 = vector.transpose %0, [1, 0] : vector<[1]x4xi8> to vector<4x[1]xi8>
   return %1 : vector<4x[1]xi8>
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index ec7cee7b2c64..5038646e1f02 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -178,9 +178,9 @@ func.func @extract_precise_position_overflow(%arg0: vector<4x8x16xf32>) {
 
 // -----
 
-func.func @extract_0d(%arg0: vector<f32>) {
-  // expected-error@+1 {{expected position attribute of rank no greater than vector rank}}
-  %1 = vector.extract %arg0[0] : f32 from vector<f32>
+func.func @extract_0d_result(%arg0: vector<f32>) {
+  // expected-error@+1 {{expected a scalar instead of a 0-d vector as the result type}}
+  %1 = vector.extract %arg0[] : vector<f32> from vector<f32>
 }
 
 // -----
@@ -259,16 +259,9 @@ func.func @insert_precise_position_overflow(%a: f32, %b: vector<4x8x16xf32>) {
 
 // -----
 
-func.func @insert_0d(%a: vector<f32>, %b: vector<4x8x16xf32>) {
-  // expected-error@+1 {{expected position attribute rank + source rank to match dest vector rank}}
-  %1 = vector.insert %a, %b[2, 6] : vector<f32> into vector<4x8x16xf32>
-}
-
-// -----
-
-func.func @insert_0d(%a: f32, %b: vector<f32>) {
-  // expected-error@+1 {{expected position attribute of rank no greater than dest vector rank}}
-  %1 = vector.insert %a, %b[0] : f32 into vector<f32>
+func.func @insert_0d_value_to_store(%a: vector<f32>, %b: vector<4x8x16xf32>) {
+  // expected-error@+1 {{expected a scalar instead of a 0-d vector as the source operand}}
+  %1 = vector.insert %a, %b[0, 0, 0] : vector<f32> into vector<4x8x16xf32>
 }
 
 // -----
@@ -1667,13 +1660,6 @@ func.func @scan_unsupported_kind(%arg0: vector<2x3xf32>, %arg1: vector<3xf32>) -
   return %0#0 : vector<2x3xf32>
 }
 
-// -----
-
-func.func @invalid_splat(%v : f32) {
-  // expected-error@+1 {{invalid kind of type specified: expected builtin.vector, but found 'memref<8xf32>'}}
-  vector.splat %v : memref<8xf32>
-  return
-}
 
 // -----
 
@@ -1973,6 +1959,27 @@ func.func @flat_transpose_scalable(%arg0: vector<[16]xf32>) -> vector<[16]xf32>
   return %0 : vector<[16]xf32>
 }
 
+//===----------------------------------------------------------------------===//
+// vector.splat
+//===----------------------------------------------------------------------===//
+
+// -----
+
+func.func @vector_splat_invalid_result(%v : f32) {
+  // expected-error@+1 {{invalid kind of type specified: expected builtin.vector, but found 'memref<8xf32>'}}
+  vector.splat %v : memref<8xf32>
+  return
+}
+
+// -----
+
+// expected-note @+1 {{prior use here}}
+func.func @vector_splat_type_mismatch(%a: f32) {
+  // expected-error @+1 {{expects different type than prior uses: 'i32' vs 'f32'}}
+  %0 = vector.splat %a : vector<1xi32>
+  return
+}
+
 // -----
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index c59f7bd00190..10bf0f162056 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -149,7 +149,7 @@ func.func @vector_transfer_ops_tensor(%arg0: tensor<?x?xf32>,
 }
 
 // CHECK-LABEL: @vector_broadcast
-func.func @vector_broadcast(%a: f32, %b: vector<f32>, %c: vector<16xf32>, %d: vector<1x16xf32>, %e: vector<8x1xf32>) -> vector<8x16xf32> {
+func.func @vector_broadcast(%a: f32, %b: vector<f32>, %c: vector<16xf32>, %d: vector<1x16xf32>, %e: vector<8x1xf32>, %f: vector<8x1x!llvm.ptr<1>>) {
   // CHECK: vector.broadcast %{{.*}} : f32 to vector<f32>
   %0 = vector.broadcast %a : f32 to vector<f32>
   // CHECK: vector.broadcast %{{.*}} : vector<f32> to vector<4xf32>
@@ -162,7 +162,9 @@ func.func @vector_broadcast(%a: f32, %b: vector<f32>, %c: vector<16xf32>, %d: ve
   %4 = vector.broadcast %d : vector<1x16xf32> to vector<8x16xf32>
   // CHECK-NEXT: vector.broadcast %{{.*}} : vector<8x1xf32> to vector<8x16xf32>
   %5 = vector.broadcast %e : vector<8x1xf32> to vector<8x16xf32>
-  return %4 : vector<8x16xf32>
+  // CHECK-NEXT: vector.broadcast %{{.*}} : vector<8x1x!llvm.ptr<1>> to vector<8x16x!llvm.ptr<1>>
+  %6 = vector.broadcast %f : vector<8x1x!llvm.ptr<1>> to vector<8x16x!llvm.ptr<1>>
+  return
 }
 
 // CHECK-LABEL: @shuffle0D
@@ -298,12 +300,10 @@ func.func @insert_val_idx(%a: f32, %b: vector<16xf32>, %c: vector<8x16xf32>,
 }
 
 // CHECK-LABEL: @insert_0d
-func.func @insert_0d(%a: f32, %b: vector<f32>, %c: vector<2x3xf32>) -> (vector<f32>, vector<2x3xf32>) {
+func.func @insert_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
   // CHECK-NEXT: vector.insert %{{.*}}, %{{.*}}[] : f32 into vector<f32>
   %1 = vector.insert %a,  %b[] : f32 into vector<f32>
-  // CHECK-NEXT: vector.insert %{{.*}}, %{{.*}}[0, 1] : vector<f32> into vector<2x3xf32>
-  %2 = vector.insert %b,  %c[0, 1] : vector<f32> into vector<2x3xf32>
-  return %1, %2 : vector<f32>, vector<2x3xf32>
+  return %1 : vector<f32>
 }
 
 // CHECK-LABEL: @insert_poison_idx
@@ -959,13 +959,16 @@ func.func @vector_scan(%0: vector<4x8x16x32xf32>) -> vector<4x8x16x32xf32> {
 }
 
 // CHECK-LABEL: func @test_splat_op
-// CHECK-SAME: [[S:%arg[0-9]+]]: f32
-func.func @test_splat_op(%s : f32) {
-  // CHECK: vector.splat [[S]] : vector<8xf32>
+// CHECK-SAME: %[[s:.*]]: f32, %[[s2:.*]]: !llvm.ptr<1>
+func.func @test_splat_op(%s : f32, %s2 : !llvm.ptr<1>) {
+  // CHECK: vector.splat %[[s]] : vector<8xf32>
   %v = vector.splat %s : vector<8xf32>
 
-  // CHECK: vector.splat [[S]] : vector<4xf32>
+  // CHECK: vector.splat %[[s]] : vector<4xf32>
   %u = "vector.splat"(%s) : (f32) -> vector<4xf32>
+
+  // CHECK: vector.splat %[[s2]] : vector<16x!llvm.ptr<1>>
+  %w = vector.splat %s2 : vector<16x!llvm.ptr<1>>
   return
 }
 
diff --git a/mlir/test/Dialect/Vector/vector-mem-transforms.mlir b/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
index 8cb25c757849..e6593320f1bd 100644
--- a/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
+++ b/mlir/test/Dialect/Vector/vector-mem-transforms.mlir
@@ -83,7 +83,7 @@ func.func @maskedstore2(%base: memref<16xf32>, %value: vector<16xf32>)  {
 // CHECK-SAME:                  %[[A1:.*]]: vector<16xi32>,
 // CHECK-SAME:                  %[[A2:.*]]: vector<16xf32>) -> vector<16xf32> {
 // CHECK-NEXT:      %[[C:.*]] = arith.constant 0 : index
-// CHECK-NEXT:      %[[M:.*]] = vector.constant_mask [16] : vector<16xi1>
+// CHECK-NEXT:      %[[M:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK-NEXT:      %[[G:.*]] = vector.gather %[[A0]][%[[C]]] [%[[A1]]], %[[M]], %[[A2]] : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
 // CHECK-NEXT:      return %[[G]] : vector<16xf32>
 func.func @gather1(%base: memref<16xf32>, %indices: vector<16xi32>, %pass_thru: vector<16xf32>) -> vector<16xf32> {
@@ -112,7 +112,7 @@ func.func @gather2(%base: memref<16xf32>, %indices: vector<16xi32>, %pass_thru:
 // CHECK-SAME:                   %[[A1:.*]]: vector<16xi32>,
 // CHECK-SAME:                   %[[A2:.*]]: vector<16xf32>) {
 // CHECK-NEXT:      %[[C:.*]] = arith.constant 0 : index
-// CHECK-NEXT:      %[[M:.*]] = vector.constant_mask [16] : vector<16xi1>
+// CHECK-NEXT:      %[[M:.*]] = arith.constant dense<true> : vector<16xi1>
 // CHECK-NEXT:      vector.scatter %[[A0]][%[[C]]] [%[[A1]]], %[[M]], %[[A2]] : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>
 // CHECK-NEXT:      return
 func.func @scatter1(%base: memref<16xf32>, %indices: vector<16xi32>, %value: vector<16xf32>) {
diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
index 0f04d3b79b53..d18edd0ac556 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
@@ -85,8 +85,8 @@ func.func @transfer_read_dims_mismatch_contiguous(
 
 // CHECK-LABEL: func.func @transfer_read_dims_mismatch_contiguous(
 // CHECK-SAME:    %[[MEM:.+]]: memref<5x4x3x2xi8, {{.+}}>) -> vector<2x3x2xi8> {
-// CHECK:         %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:     %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:     %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[COLLAPSED_MEM:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1, 2, 3]]
 // CHECK-SAME:          : memref<5x4x3x2xi8, {{.+}}> into memref<5x24xi8, {{.+}}>
@@ -116,8 +116,8 @@ func.func @transfer_read_dims_mismatch_contiguous_unit_dims(
 // CHECK-LABEL: func.func @transfer_read_dims_mismatch_contiguous_unit_dims(
 // CHECK-SAME:    %[[MEM:.+]]: memref<6x5x4x3x2xi8, strided<[120, 24, 6, 2, 1], offset: ?>>)
 // CHECK-SAME:    -> vector<1x1x4x3x2xi8>
-// CHECK:       %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:       %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
 // CHECK:       %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3, 4]]
 // CHECK-SAME:    : memref<6x5x4x3x2xi8, strided<[120, 24, 6, 2, 1], offset: ?>>
@@ -149,8 +149,8 @@ func.func @transfer_read_non_contiguous_unit_dims(
 
 // CHECK-LABEL:   func.func @transfer_read_non_contiguous_unit_dims(
 // CHECK-SAME:      %[[MEM:.*]]: memref<5x4x3x2xi8, strided<[48, 6, 2, 1], offset: ?>>) -> vector<1x1x3x2xi8> {
-// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i8
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK-DAG:       %[[VAL_1:.*]] = arith.constant 0 : i8
+// CHECK-DAG:       %[[VAL_2:.*]] = arith.constant 0 : index
 // CHECK:           %[[VAL_3:.*]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:        : memref<5x4x3x2xi8, strided<[48, 6, 2, 1], offset: ?>> into memref<5x4x6xi8, strided<[48, 6, 1], offset: ?>>
@@ -182,8 +182,8 @@ func.func @transfer_read_dims_mismatch_non_zero_indices(
 // CHECK-LABEL:   func.func @transfer_read_dims_mismatch_non_zero_indices(
 // CHECK-SAME:      %[[IDX_1:.+]]: index, %[[IDX_2:.+]]: index,
 // CHECK-SAME:      %[[MEM:.+]]: memref<1x43x4x6xi32>
-// CHECK:           %[[C0_I32:.+]] = arith.constant 0 : i32
-// CHECK:           %[[C_0:.+]] = arith.constant 0 : index
+// CHECK-DAG:       %[[C0_I32:.+]] = arith.constant 0 : i32
+// CHECK-DAG:       %[[C_0:.+]] = arith.constant 0 : index
 // CHECK:           %[[COLLAPSED_IN:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:         : memref<1x43x4x6xi32> into memref<1x43x24xi32>
@@ -241,8 +241,8 @@ func.func @transfer_read_leading_dynamic_dims(
 
 // CHECK-LABEL: func @transfer_read_leading_dynamic_dims
 // CHECK-SAME:    %[[MEM:.+]]: memref<?x?x8x4xi8, {{.+}}>, %[[IDX_1:.+]]: index, %[[IDX_2:.+]]: index
-// CHECK:         %[[C0_I8:.+]] = arith.constant 0 : i8
-// CHECK:         %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:     %[[C0_I8:.+]] = arith.constant 0 : i8
+// CHECK-DAG:     %[[C0:.+]] = arith.constant 0 : index
 // CHECK:         %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}: [[0], [1], [2, 3]]
 // CHECK-SAME:      : memref<?x?x8x4xi8, {{.+}}> into memref<?x?x32xi8, {{.+}}>
@@ -304,8 +304,8 @@ func.func @transfer_read_dynamic_dim_to_flatten(
 // CHECK-SAME:    %[[IDX_1:arg0]]
 // CHECK-SAME:    %[[IDX_2:arg1]]
 // CHECK-SAME:    %[[MEM:arg2]]
-// CHECK:              %[[C0_I32:.+]] = arith.constant 0 : i32
-// CHECK:              %[[C0:.+]] = arith.constant 0 : index
+// CHECK-DAG:          %[[C0_I32:.+]] = arith.constant 0 : i32
+// CHECK-DAG:          %[[C0:.+]] = arith.constant 0 : index
 // CHECK:              %[[COLLAPSED:.+]] = memref.collapse_shape %[[MEM]]
 // CHECK-SAME{LITERAL}:  [[0], [1], [2, 3]]
 // CHECK-SAME:           memref<1x?x4x6xi32> into memref<1x?x24xi32>
diff --git a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
index 38771f259344..1161dbd4b216 100644
--- a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
+++ b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
@@ -1132,8 +1132,8 @@ func.func @warp_execute_nd_distribute(%laneid: index, %v0: vector<1x64x1xf32>, %
 
   //     CHECK-SCF-IF: gpu.barrier
   //     CHECK-SCF-IF: %[[WID:.*]] = affine.apply #[[$TIMES2]]()[%[[LANEID]]]
-  // CHECK-SCF-IF-DAG: %[[R0:.*]] = vector.transfer_read %{{.*}}[%[[LANEID]], %[[C0]], %[[C0]]], %cst {in_bounds = [true, true, true]} : memref<32x64x1xf32, 3>, vector<1x64x1xf32>
-  // CHECK-SCF-IF-DAG: %[[R1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[WID]], %[[C0]]], %cst {in_bounds = [true, true, true]} : memref<1x64x128xf32, 3>, vector<1x2x128xf32>
+  // CHECK-SCF-IF-DAG: %[[R0:.*]] = vector.transfer_read %{{.*}}[%[[LANEID]], %[[C0]], %[[C0]]], %{{.*}} {in_bounds = [true, true, true]} : memref<32x64x1xf32, 3>, vector<1x64x1xf32>
+  // CHECK-SCF-IF-DAG: %[[R1:.*]] = vector.transfer_read %{{.*}}[%[[C0]], %[[WID]], %[[C0]]], %{{.*}} {in_bounds = [true, true, true]} : memref<1x64x128xf32, 3>, vector<1x2x128xf32>
   //     CHECK-SCF-IF: return %[[R0]], %[[R1]] : vector<1x64x1xf32>, vector<1x2x128xf32>
   return %r#0, %r#1 : vector<1x64x1xf32>, vector<1x2x128xf32>
 }
@@ -1296,6 +1296,86 @@ func.func @vector_insert_2d_broadcast(%laneid: index) -> (vector<4x96xf32>) {
   return %r : vector<4x96xf32>
 }
 
+// -----
+// CHECK-PROP-LABEL: func.func @vector_extract_strided_slice_2d_distr_inner(
+//  CHECK-RPOP-SAME: %[[LANEID:.*]]: index
+//       CHECK-PROP: %[[W:.*]] = gpu.warp_execute_on_lane_0{{.*}} -> (vector<64x1xf32>) {
+//       CHECK-PROP: %[[VEC:.*]] = "some_def"() : () -> vector<64x32xf32>
+//       CHECK-PROP: gpu.yield %[[VEC]] : vector<64x32xf32>
+//       CHECK-PROP: %[[EXTRACT:.*]] = vector.extract_strided_slice %[[W]]
+//  CHECK-PROP-SAME: {offsets = [8], sizes = [24], strides = [1]} : vector<64x1xf32> to vector<24x1xf32>
+//       CHECK-PROP: return %[[EXTRACT]] : vector<24x1xf32>
+func.func @vector_extract_strided_slice_2d_distr_inner(%laneid: index) -> (vector<24x1xf32>) {
+  %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<24x1xf32>) {
+    %0 = "some_def"() : () -> (vector<64x32xf32>)
+    %1 = vector.extract_strided_slice %0 { offsets = [8], sizes = [24], strides = [1]}
+      : vector<64x32xf32> to vector<24x32xf32>
+    gpu.yield %1 : vector<24x32xf32>
+  }
+  return %r : vector<24x1xf32>
+}
+
+// -----
+// CHECK-PROP-LABEL: func.func @vector_extract_strided_slice_2d_distr_outer(
+//  CHECK-PROP-SAME: %[[LANEID:.*]]: index
+//       CHECK-PROP: %[[W:.*]] = gpu.warp_execute_on_lane_0{{.*}} -> (vector<1x64xf32>) {
+//       CHECK-PROP: %[[VEC:.*]] = "some_def"() : () -> vector<32x64xf32>
+//       CHECK-PROP: gpu.yield %[[VEC]] : vector<32x64xf32>
+//       CHECK-PROP: %[[EXTRACT:.*]] = vector.extract_strided_slice %[[W]]
+//  CHECK-PROP-SAME: {offsets = [0, 12], sizes = [1, 8], strides = [1, 1]} : vector<1x64xf32> to vector<1x8xf32>
+//       CHECK-PROP: return %[[EXTRACT]] : vector<1x8xf32>
+func.func @vector_extract_strided_slice_2d_distr_outer(%laneid: index) -> (vector<1x8xf32>) {
+  %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<1x8xf32>) {
+    %0 = "some_def"() : () -> (vector<32x64xf32>)
+    %1 = vector.extract_strided_slice %0 { offsets = [0, 12], sizes = [32, 8], strides = [1, 1]}
+      : vector<32x64xf32> to vector<32x8xf32>
+    gpu.yield %1 : vector<32x8xf32>
+  }
+  return %r : vector<1x8xf32>
+}
+
+// -----
+// CHECK-PROP-LABEL: func.func @vector_insert_strided_slice_1d_to_2d(
+//  CHECK-PROP-SAME: %[[LANEID:.*]]: index)
+//       CHECK-PROP: %[[W:.*]]:2 = gpu.warp_execute_on_lane_0({{.*}} -> (vector<1xf32>, vector<64x1xf32>) {
+//       CHECK-PROP: %[[SRC:.*]] = "some_def"() : () -> vector<32xf32>
+//       CHECK-PROP: %[[DEST:.*]] = "some_def"() : () -> vector<64x32xf32>
+//       CHECK-PROP: gpu.yield %[[SRC]], %[[DEST]] : vector<32xf32>, vector<64x32xf32>
+//       CHECK-PROP: %[[INSERT:.*]] = vector.insert_strided_slice %[[W]]#0, %[[W]]#1
+//  CHECK-PROP-SAME: {offsets = [18, 0], strides = [1]} : vector<1xf32> into vector<64x1xf32>
+//       CHECK-PROP: return %[[INSERT]] : vector<64x1xf32>
+func.func @vector_insert_strided_slice_1d_to_2d(%laneid: index) -> (vector<64x1xf32>) {
+  %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<64x1xf32>) {
+    %0 = "some_def"() : () -> (vector<32xf32>)
+    %1 = "some_def"() : () -> (vector<64x32xf32>)
+    %2 = vector.insert_strided_slice %0, %1 { offsets = [18, 0], strides = [1]}
+      : vector<32xf32> into vector<64x32xf32>
+    gpu.yield %2 : vector<64x32xf32>
+  }
+  return %r : vector<64x1xf32>
+}
+
+// -----
+// CHECK-PROP-LABEL: func.func @vector_insert_strided_slice_2d_to_2d(
+//  CHECK-PROP-SAME: %[[LANEID:.*]]: index)
+//       CHECK-PROP: %[[W:.*]]:2 = gpu.warp_execute_on_lane_0{{.*}} -> (vector<16x1xf32>, vector<64x1xf32>) {
+//       CHECK-PROP: %[[SRC:.*]] = "some_def"() : () -> vector<16x32xf32>
+//       CHECK-PROP: %[[DEST:.*]] = "some_def"() : () -> vector<64x32xf32>
+//       CHECK-PROP: gpu.yield %[[SRC]], %[[DEST]] : vector<16x32xf32>, vector<64x32xf32>
+//       CHECK-PROP: %[[INSERT:.*]] = vector.insert_strided_slice %[[W]]#0, %[[W]]#1 {offsets = [36, 0], strides = [1, 1]} :
+//  CHECK-PROP-SAME: vector<16x1xf32> into vector<64x1xf32>
+//       CHECK-PROP: return %[[INSERT]] : vector<64x1xf32>
+func.func @vector_insert_strided_slice_2d_to_2d(%laneid: index) -> (vector<64x1xf32>) {
+  %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<64x1xf32>) {
+    %0 = "some_def"() : () -> (vector<16x32xf32>)
+    %1 = "some_def"() : () -> (vector<64x32xf32>)
+    %2 = vector.insert_strided_slice %0, %1 { offsets = [36, 0],  strides = [1, 1]}
+      : vector<16x32xf32> into vector<64x32xf32>
+    gpu.yield %2 : vector<64x32xf32>
+  }
+  return %r : vector<64x1xf32>
+}
+
 // -----
 
 // Make sure that all operands of the transfer_read op are properly propagated.
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
index 0a37ae70b5d9..a2778cd94d96 100644
--- a/mlir/test/Dialect/XeGPU/invalid.mlir
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -239,7 +239,7 @@ func.func @create_tdesc_vc_5(%src: memref<?xf32>) {
 func.func @create_tdesc_vc_6(%src: memref<?xf16>) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   %1 = xegpu.create_tdesc %src, %0 : memref<?xf16>, vector<4xindex>
-  // expected-error@+1 {{tensor shape[1] to be a multiple of packing factor 2}}
+  // expected-error@+1 {{tensor shape[1] to be a multiple of chunk alignment factor 2}}
           -> !xegpu.tensor_desc<4x3xf16, #xegpu.scatter_tdesc_attr<chunk_size = 3>>
   return
 }
diff --git a/mlir/test/Dialect/XeGPU/ops.mlir b/mlir/test/Dialect/XeGPU/ops.mlir
index 054c4d12fdb2..aff8f63adc05 100644
--- a/mlir/test/Dialect/XeGPU/ops.mlir
+++ b/mlir/test/Dialect/XeGPU/ops.mlir
@@ -235,8 +235,6 @@ gpu.func @simt_store_nd(%src: memref<24x32xf16>) {
   gpu.return
 }
 
-
-
 // CHECK: func @subgroup_store_nd_2(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @subgroup_store_nd_2(%dst: memref<24x32xf16>) {
   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<32xf16>
@@ -248,7 +246,6 @@ gpu.func @subgroup_store_nd_2(%dst: memref<24x32xf16>) {
   gpu.return
 }
 
-
 // CHECK: func @simt_store_nd_2(%[[arg0:.*]]: memref<24x32xf16>) {
 gpu.func @simt_store_nd_2(%src: memref<24x32xf16>) {
   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<2xf16>
@@ -318,8 +315,8 @@ gpu.func @subgroup_load(%src: ui64) {
   %1 = arith.constant dense<1>: vector<4xi1>
   //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
-  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1> -> vector<2x4xf32>
-  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1> -> vector<2x4xf32>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1> -> vector<4x2xf32>
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1> -> vector<4x2xf32>
   gpu.return
 }
 
@@ -370,8 +367,8 @@ gpu.func @subgroup_load_3(%src: ui64) {
   %1 = arith.constant dense<1>: vector<4xi1>
   //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
   %2 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>>
-  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<4xi1> -> vector<8x4xf16>
-  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>>, vector<4xi1> -> vector<8x4xf16>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<4xi1> -> vector<4x8xf16>
+  %3 = xegpu.load %2, %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x8xf16, #xegpu.scatter_tdesc_attr<chunk_size = 8>>, vector<4xi1> -> vector<4x8xf16>
   gpu.return
 }
 
@@ -394,17 +391,15 @@ gpu.func @subgroup_store(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<2x4xf32>
-  %2 = arith.constant dense<2.9>: vector<2x4xf32>
+  //CHECK: %[[cst2:.*]] = arith.constant dense<2.900000e+00> : vector<4x2xf32>
+  %2 = arith.constant dense<2.9>: vector<4x2xf32>
   //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
-  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x4xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
-  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x4xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<4x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<4x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
   gpu.return
 }
 
-
-
 // CHECK: gpu.func @simt_store(%[[arg0:.*]]: ui64) {
 gpu.func @simt_store(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
@@ -426,17 +421,15 @@ gpu.func @subgroup_store_2(%src: ui64) {
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   //CHECK: %[[cst1:.*]] = arith.constant dense<true> : vector<4xi1>
   %1 = arith.constant dense<1>: vector<4xi1>
-  //CHECK: %[[cst2:.*]] = arith.constant {{.*}} : vector<2x4xf16>
-  %2 = arith.constant dense<2.9>: vector<2x4xf16>
+  //CHECK: %[[cst2:.*]] = arith.constant {{.*}} : vector<4x2xf16>
+  %2 = arith.constant dense<2.9>: vector<4x2xf16>
   //CHECK: %[[R0:.*]] = xegpu.create_tdesc %[[arg0]], %[[cst]] : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   %3 = xegpu.create_tdesc %src, %0 : ui64, vector<4xindex> -> !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>
-  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x4xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
-  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x4xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
+  //CHECK: xegpu.store %[[cst2]], %[[R0]], %[[cst1]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<4x2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<4xi1>
+  xegpu.store %2, %3, %1 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<4x2xf16>, !xegpu.tensor_desc<4x2xf16, #xegpu.scatter_tdesc_attr<chunk_size = 2>>, vector<4xi1>
   gpu.return
 }
 
-
-
 // CHECK: gpu.func @simt_store_2(%[[arg0:.*]]: ui64) {
 gpu.func @simt_store_2(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
@@ -467,7 +460,6 @@ gpu.func @subgroup_store_3(%src: ui64) {
   gpu.return
 }
 
-
 // CHECK: gpu.func @simt_store_3(%[[arg0:.*]]: ui64) {
 gpu.func @simt_store_3(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
diff --git a/mlir/test/Dialect/XeGPU/propagate-layout.mlir b/mlir/test/Dialect/XeGPU/propagate-layout.mlir
index 429081079de1..0214d84f2c16 100644
--- a/mlir/test/Dialect/XeGPU/propagate-layout.mlir
+++ b/mlir/test/Dialect/XeGPU/propagate-layout.mlir
@@ -90,26 +90,27 @@ func.func @extf_truncf(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor
 }
 
 // -----
-// CHECK-LABEL: func.func @load_gather_with_transpose_effect(
+// CHECK-LABEL: func.func @load_gather_with_chunksize(
 // CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<256xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
 // CHECK: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>}
 // CHECK-SAME:  dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
 // CHECK-NEXT: %[[CST0:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} dense<true> : vector<16xi1>
 // CHECK-NEXT: %[[T2:.*]] = xegpu.create_tdesc %[[ARG1]], %[[CST]] : memref<256xf16>, vector<16xindex> ->
 // CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>>
-// CHECK-NEXT: %{{.*}} = xegpu.load %[[T2]], %[[CST0]] <{transpose}> {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-NEXT: %{{.*}} = xegpu.load %[[T2]], %[[CST0]]  {layout_result_0 = #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>}
 // CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>>, vector<16xi1> -> vector<16x16xf16>
-func.func @load_gather_with_transpose_effect(%arg0: memref<8x16xf16>, %arg1: memref<256xf16>, %arg2: memref<8x16xf32>) {
+func.func @load_gather_with_chunksize(%arg0: memref<8x16xf16>, %arg1: memref<256xf16>, %arg2: memref<8x16xf32>) {
   %c0 = arith.constant 0 : index
   %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
   %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
   %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
   %cst_0 = arith.constant dense<true> : vector<16xi1>
   %2 = xegpu.create_tdesc %arg1, %cst : memref<256xf16>, vector<16xindex> -> !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>
-  %3 = xegpu.load %2, %cst_0 <{transpose}> : !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>, vector<16xi1> -> vector<16x16xf16>
-  %4 = xegpu.dpas %1, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  %3 = xegpu.load %2, %cst_0 : !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>, vector<16xi1> -> vector<16x16xf16>
+  %4 = vector.transpose %3, [1, 0] : vector<16x16xf16> to vector<16x16xf16>
+  %5 = xegpu.dpas %1, %4 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  %6 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %5, %6  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
   return
 }
 
@@ -127,24 +128,24 @@ func.func @load_gather_1d(%arg0: memref<256xf32>, %arg1: !xegpu.tensor_desc<16xf
   %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
   %cst_0 = arith.constant dense<true> : vector<16xi1>
   %0 = xegpu.create_tdesc %arg0, %cst : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-  %1 = xegpu.load %0, %cst_0  : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
+  %1 = xegpu.load %0, %cst_0 : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
   xegpu.store_nd %1, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
   return
 }
 
 // -----
-// CHECK-LABEL: func.func @store_scatter_with_transpose_effect(
+// CHECK-LABEL: func.func @store_scatter_with_chunksize(
 // CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<128xf32>) {
 // CHECK: %[[T0:.*]] = xegpu.create_tdesc %[[ARG0]], %{{.*}} : memref<128xf32>, vector<16xindex> ->
 // CHECK-SAME: !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>
-// CHECK-NEXT: xegpu.store %{{.*}}, %[[T0]], %{{.*}} <{transpose}> : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>,
+// CHECK-NEXT: xegpu.store %{{.*}}, %[[T0]], %{{.*}} : vector<16x8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>,
 // CHECK-SAME: #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>, vector<16xi1>
-func.func @store_scatter_with_transpose_effect(%arg0: memref<128xf32>) {
-  %cst = arith.constant dense<1.000000e+00> : vector<8x16xf32>
+func.func @store_scatter_with_chunksize(%arg0: memref<128xf32>) {
+  %cst = arith.constant dense<1.000000e+00> : vector<16x8xf32>
   %cst_0 = arith.constant dense<true> : vector<16xi1>
   %cst_1 = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
   %0 = xegpu.create_tdesc %arg0, %cst_1 : memref<128xf32>, vector<16xindex> -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
-  xegpu.store %cst, %0, %cst_0 <{transpose}> : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<16xi1>
+  xegpu.store %cst, %0, %cst_0 : vector<16x8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<16xi1>
   return
 }
 
diff --git a/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir b/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
index a59633b0cbd9..3d91b2269bc4 100644
--- a/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
+++ b/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
@@ -278,3 +278,22 @@ gpu.module @test {
     gpu.return
   }
 }
+
+// -----
+// CHECK-LABEL: gpu.func @gpu_barrier({{.*}}) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
+// CHECK-NEXT: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16xf16> -> vector<1xf16>
+// CHECK-NEXT: gpu.barrier
+// CHECK-NEXT: %[[T2:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
+// CHECK-NEXT: xegpu.store_nd %[[T1]], %[[T2]] : vector<1xf16>, !xegpu.tensor_desc<16xf16>
+gpu.module @test {
+  gpu.func @gpu_barrier(%arg0: memref<256xf16>, %arg1: memref<256xf16>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} : !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<16xf16>
+    gpu.barrier
+    %2 = xegpu.create_nd_tdesc %arg1[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    xegpu.store_nd %1, %2 : vector<16xf16>, !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    gpu.return
+  }
+}
diff --git a/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir b/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
index f977ba3c11bc..ac5fe89a67f9 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
@@ -368,35 +368,35 @@ gpu.module @test_kernel {
     0,   8,  16,  24,  32,  40,  48,  56,
     64,  72,  80,  88,  96, 104, 112, 120,
     128, 136, 144, 152, 160, 168, 176, 184,
-    192, 200, 208, 216, 224, 232, 240, 248 
+    192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
 
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
     xegpu.prefetch %tdesc: !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
-   
+
     %delta = arith.constant dense<[
     32,   32,  32,  32,  32,  32,  32,  32,
     32,   32,  32,  32,  32,  32,  32,  64,
     128, 128, 128, 128, 128, 128, 128, 128,
-    128, 128, 128, 128, 128, 128, 128, 256 
+    128, 128, 128, 128, 128, 128, 128, 256
     ]> : vector<32xindex>
     %new_tdesc = xegpu.update_offset %tdesc, %delta
-              : !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xindex>     
- 
+              : !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xindex>
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
 
     %ld_vec = xegpu.load %new_tdesc, %mask: !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xi1> -> vector<32xf32>
 
     %st_vec = arith.addf %ld_vec, %ld_vec : vector<32xf32>
-    xegpu.store %st_vec, %tdesc, %mask: 
-                 vector<32xf32>, 
-                 !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, 
+    xegpu.store %st_vec, %tdesc, %mask:
+                 vector<32xf32>,
+                 !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>,
                  vector<32xi1>
-  
+
     gpu.return
   }
-  
+
 }
 
 // -----
@@ -407,8 +407,8 @@ gpu.module @test_kernel   {
   // CHECK-COUNT-4: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   // CHECK-COUNT-4: xegpu.prefetch {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
    // CHECK-COUNT-4: xegpu.update_offset {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xindex>
-   // CHECK-COUNT-4: xegpu.load  {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1> -> vector<2x16xf32>
-  // CHECK-COUNT-4: xegpu.store  {{.*}} : vector<2x16xf32>, !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1>
+   // CHECK-COUNT-4: xegpu.load  {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1> -> vector<16x2xf32>
+  // CHECK-COUNT-4: xegpu.store  {{.*}} : vector<16x2xf32>, !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1>
 
   gpu.func @test_prefetch_load_store_update_chunk(%src: ui64)  {
 
@@ -416,32 +416,32 @@ gpu.module @test_kernel   {
       0,   8,  16,  24,  32,  40,  48,  56,
       64,  72,  80,  88,  96, 104, 112, 120,
       128, 136, 144, 152, 160, 168, 176, 184,
-      192, 200, 208, 216, 224, 232, 240, 248 
+      192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
 
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
     xegpu.prefetch %tdesc: !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
-   
+
     %delta = arith.constant dense<[
       32,   32,  32,  32,  32,  32,  32,  32,
       32,   32,  32,  32,  32,  32,  32,  64,
       128, 128, 128, 128, 128, 128, 128, 128,
-      128, 128, 128, 128, 128, 128, 128, 256 
+      128, 128, 128, 128, 128, 128, 128, 256
     ]> : vector<32xindex>
     %new_tdesc = xegpu.update_offset %tdesc, %delta
-              : !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xindex>     
- 
+              : !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xindex>
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
 
-    %ld_vec = xegpu.load %new_tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xi1> -> vector<4x32xf32>
+    %ld_vec = xegpu.load %new_tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xi1> -> vector<32x4xf32>
 
-    %st_vec = arith.addf %ld_vec, %ld_vec : vector<4x32xf32>
-    xegpu.store %st_vec, %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: 
-                 vector<4x32xf32>, 
-                 !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, 
+    %st_vec = arith.addf %ld_vec, %ld_vec : vector<32x4xf32>
+    xegpu.store %st_vec, %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>:
+                 vector<32x4xf32>,
+                 !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>,
                  vector<32xi1>
-  
+
     gpu.return
   }
 }
diff --git a/mlir/test/Dialect/XeGPU/xegpu-unroll-patterns.mlir b/mlir/test/Dialect/XeGPU/xegpu-unroll-patterns.mlir
index 41414d802f21..6999da5d222f 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-unroll-patterns.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-unroll-patterns.mlir
@@ -169,7 +169,7 @@ gpu.module @test {
     0,   8,  16,  24,  32,  40,  48,  56,
     64,  72,  80,  88,  96, 104, 112, 120,
     128, 136, 144, 152, 160, 168, 176, 184,
-    192, 200, 208, 216, 224, 232, 240, 248 
+    192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
     gpu.return %tdesc : !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>,  #xegpu.layout<inst_data = [16]>>
@@ -199,15 +199,15 @@ gpu.module @test {
     0,   8,  16,  24,  32,  40,  48,  56,
     64,  72,  80,  88,  96, 104, 112, 120,
     128, 136, 144, 152, 160, 168, 176, 184,
-    192, 200, 208, 216, 224, 232, 240, 248 
+    192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
-      
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
     %ld = xegpu.load %tdesc, %mask: !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xi1> -> vector<32xf32>
-      
-    gpu.return %ld : vector<32xf32> 
+
+    gpu.return %ld : vector<32xf32>
   }
 
 //-----
@@ -222,7 +222,7 @@ gpu.module @test {
     0,   8,  16,  24,  32,  40,  48,  56,
     64,  72,  80,  88,  96, 104, 112, 120,
     128, 136, 144, 152, 160, 168, 176, 184,
-    192, 200, 208, 216, 224, 232, 240, 248 
+    192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
 
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
@@ -242,16 +242,16 @@ gpu.module @test {
     0,   8,  16,  24,  32,  40,  48,  56,
     64,  72,  80,  88,  96, 104, 112, 120,
     128, 136, 144, 152, 160, 168, 176, 184,
-    192, 200, 208, 216, 224, 232, 240, 248 
+    192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
-    
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
 
     %st_vec = arith.constant dense<1023.0>: vector<32xf32>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
     xegpu.store %st_vec, %tdesc, %mask: vector<32xf32>, !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xi1>
-    
+
     gpu.return
   }
 
@@ -280,7 +280,7 @@ gpu.module @test {
   }
 
 // CHECK-LABEL: create_tdesc_step_chunk3
-  // CHECK-SAME: [[arg0:%.+]]: ui64  
+  // CHECK-SAME: [[arg0:%.+]]: ui64
   // CHECK: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
   // CHECK: arith.addi %{{.*}}, %{{.*}} : vector<16xindex>
   // CHECK: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
@@ -300,45 +300,45 @@ gpu.module @test {
   // CHECK-LABEL: load_chunk
   // CHECK-SAME: [[arg0:%.+]]: ui64
   // CHECK-COUNT-4: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  // CHECK-COUNT-4: xegpu.load  {{.*}} <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1> -> vector<2x16xf32>
+  // CHECK-COUNT-4: xegpu.load  {{.*}} <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1> -> vector<16x2xf32>
 
-  gpu.func @load_chunk(%src: ui64) -> vector<4x32xf32> {
+  gpu.func @load_chunk(%src: ui64) -> vector<32x4xf32> {
     %cst = arith.constant dense<[
         0,   8,  16,  24,  32,  40,  48,  56,
         64,  72,  80,  88,  96, 104, 112, 120,
         128, 136, 144, 152, 160, 168, 176, 184,
-        192, 200, 208, 216, 224, 232, 240, 248 
+        192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
-    
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
 
-    %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>> 
-    %ld = xegpu.load %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xi1> -> vector<4x32xf32>
-    
-    gpu.return %ld : vector<4x32xf32> 
+    %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
+    %ld = xegpu.load %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xi1> -> vector<32x4xf32>
+
+    gpu.return %ld : vector<32x4xf32>
    }
 
 //-----
   // CHECK-LABEL: store_chunk
   // CHECK-SAME: [[arg0:%.+]]: ui64
   // CHECK-COUNT-4: xegpu.create_tdesc [[arg0]], {{.*}} :  ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
-  // CHECK-COUNT-4: xegpu.store  {{.*}} <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}> : vector<2x16xf32>, !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1>
+  // CHECK-COUNT-4: xegpu.store  {{.*}} <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<16x2xf32>, !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1>
   gpu.func @store_chunk(%src: ui64) {
     %cst = arith.constant dense<[
       0,   8,  16,  24,  32,  40,  48,  56,
       64,  72,  80,  88,  96, 104, 112, 120,
       128, 136, 144, 152, 160, 168, 176, 184,
-      192, 200, 208, 216, 224, 232, 240, 248 
+      192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
-    
+
     %c17 = arith.constant 17: index
     %mask = vector.create_mask %c17: vector<32xi1>
 
-    %st_vec = arith.constant dense<1023.>: vector<4x32xf32>
+    %st_vec = arith.constant dense<1023.>: vector<32x4xf32>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
-    xegpu.store %st_vec, %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: vector<4x32xf32>, !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16,2]>>, vector<32xi1>
-    
+    xegpu.store %st_vec, %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<32x4xf32>, !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16,2]>>, vector<32xi1>
+
     gpu.return
   }
 
@@ -352,11 +352,11 @@ gpu.module @test {
       0,   8,  16,  24,  32,  40,  48,  56,
       64,  72,  80,  88,  96, 104, 112, 120,
       128, 136, 144, 152, 160, 168, 176, 184,
-      192, 200, 208, 216, 224, 232, 240, 248 
+      192, 200, 208, 216, 224, 232, 240, 248
       ]> : vector<32xindex>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
     xegpu.prefetch %tdesc: !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
-    
+
     gpu.return
   }
 
@@ -370,7 +370,7 @@ gpu.module @test {
       0,   8,  16,  24,  32,  40,  48,  56,
       64,  72,  80,  88,  96, 104, 112, 120,
       128, 136, 144, 152, 160, 168, 176, 184,
-      192, 200, 208, 216, 224, 232, 240, 248 
+      192, 200, 208, 216, 224, 232, 240, 248
     ]> : vector<32xindex>
     %delta = arith.constant dense<32>: vector<32xindex>
     %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
@@ -379,6 +379,6 @@ gpu.module @test {
         : !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xindex>
 
     gpu.return %new_tdesc : !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
-  }  
+  }
 }
 
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
index 64f01d61d6e8..09df1e4da43e 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-elemwise.mlir
@@ -1,6 +1,25 @@
 // RUN: mlir-opt --xegpu-wg-to-sg-distribute -split-input-file %s | FileCheck %s
 
 gpu.module @test_elementwise_ops {
+
+  // CHECK-LABEL: unary_ops_sg_layout_only
+  gpu.func @unary_ops_sg_layout_only(%a: memref<24x32xf32>) {
+    %tdesc_a = xegpu.create_nd_tdesc %a[0, 0] : memref<24x32xf32>
+      -> !xegpu.tensor_desc<24x32xf32, #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>>
+    %load_a = xegpu.load_nd %tdesc_a
+      : !xegpu.tensor_desc<24x32xf32, #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>>
+      -> vector<24x32xf32>
+    // CHECK: math.exp {{.*}} : vector<12x8xf32>
+    %exp = math.exp %load_a
+      {layout_result_0 = #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>}
+      : vector<24x32xf32>
+    // CHECK: arith.negf {{.*}} : vector<12x8xf32>
+    %negf = arith.negf %load_a
+      {layout_result_0 = #xegpu.layout<sg_layout = [2, 4], sg_data = [12, 8]>}
+      : vector<24x32xf32>
+    gpu.return
+  }
+
   // CHECK-LABEL: unary_ops
   gpu.func @unary_ops(%a: memref<24x32xf32>) {
     %tdesc_a = xegpu.create_nd_tdesc %a[0, 0] : memref<24x32xf32>
diff --git a/mlir/test/Examples/transform/Ch1/invalidation-1.mlir b/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
index 2264ade7f9b7..d8b8039631de 100644
--- a/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
+++ b/mlir/test/Examples/transform/Ch1/invalidation-1.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" \
 // RUN:   --split-input-file --verify-diagnostics
 
@@ -16,7 +16,7 @@ module attributes {transform.with_named_sequence} {
       %arg0: !transform.any_op,
       // expected-note @below {{handle to invalidated ops}}
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // The actual tiling transformation takes tile sizes as attributes.
     // expected-note @below {{invalidated by this transform op that consumes its operand #0 and invalidates all handles to payload IR entities associated with this operand and entities nested in them}}
     %tiled, %loop = transform.structured.tile_using_forall %arg1 tile_sizes [4, 32]
@@ -39,14 +39,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -57,7 +57,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // We can cast one type to another as long as operations are compatible
     // with both types. This creates "aliasing" handles.
     // expected-note @below {{handle to invalidated ops}}
@@ -88,14 +88,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
diff --git a/mlir/test/Examples/transform/Ch1/invalidation-2.mlir b/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
index 0a84a5cb6c68..e09aca31a90d 100644
--- a/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
+++ b/mlir/test/Examples/transform/Ch1/invalidation-2.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" \
 // RUN:   --split-input-file --verify-diagnostics
 // ****************************** IMPORTANT NOTE ******************************
@@ -23,14 +23,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
   // Elementwise addition.
 
   // expected-note @below {{ancestor payload op}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -47,11 +47,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
   // Since the %arg2 handle is associated with both elementwise operations,
   // we need to split it into two handles so we can target only the second
   // elementwise operation.
-  %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+  %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
       -> (!transform.any_op, !transform.any_op)
 
   // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch1/sequence.mlir b/mlir/test/Examples/transform/Ch1/sequence.mlir
index 3107adccf78f..f8dfd2de01d9 100644
--- a/mlir/test/Examples/transform/Ch1/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch1/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,21 +20,21 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
 
 // CHECK: func @outlined
 // CHECK:   linalg.matmul
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<add>
 
 // CHECK-LABEL: func @fc_relu
 // CHECK: scf.forall
@@ -46,9 +46,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @outlined(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -63,11 +63,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
   
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch2/sequence.mlir b/mlir/test/Examples/transform/Ch2/sequence.mlir
index 976df1d55503..8d6e129c25f7 100644
--- a/mlir/test/Examples/transform/Ch2/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch2/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: transform-opt-ch2 %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,14 +20,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -42,9 +42,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @microkernel(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -59,11 +59,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
       %arg0: !transform.any_op,
       %arg1: !transform.op<"linalg.matmul">,
-      %arg2: !transform.op<"linalg.elemwise_binary">) {
+      %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
 
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch3/sequence.mlir b/mlir/test/Examples/transform/Ch3/sequence.mlir
index 8dc33c3560c2..4d28518ca3d9 100644
--- a/mlir/test/Examples/transform/Ch3/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch3/sequence.mlir
@@ -1,6 +1,6 @@
 // RUN: transform-opt-ch3 %s \
 // RUN:   --pass-pipeline="builtin.module(transform-interpreter{ \
-// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elemwise_binary},\
+// RUN:        debug-bind-trailing-args=linalg.matmul,linalg.elementwise},\
 // RUN:        canonicalize,cse,symbol-dce)" |\
 // RUN: FileCheck %s
 
@@ -20,14 +20,14 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
                           outs(%output: tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise addition.
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -42,9 +42,9 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 // CHECK:     %[[SLICE8:.+]] = tensor.extract_slice
 // CHECK:     func.call @microkernel(%[[SLICE4]], %[[SLICE5]], %[[SLICE6]], %[[SLICE7]], %[[SLICE8]])
 // CHECK-NOT: linalg.matmul
-// CHECK-NOT: linalg.elemwise_binary
+// CHECK-NOT: linalg.elementwise
 // CHECK:     scf.forall.in_parallel
-// CHECK:   linalg.elemwise_binary {fun = #linalg.binary_fn<max_signed>}
+// CHECK:   linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
 // CHECK:   scf.forall.in_parallel
 
 // Declaration of the "microkernel" function that we will be targeting.
@@ -59,11 +59,11 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(
        %arg0: !transform.any_op,
        %arg1: !transform.op<"linalg.matmul">,
-       %arg2: !transform.op<"linalg.elemwise_binary">) {
+       %arg2: !transform.op<"linalg.elementwise">) {
     // Since the %arg2 handle is associated with both elementwise operations,
     // we need to split it into two handles so we can target only the second
     // elementwise operation.
-    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elemwise_binary">)
+    %add, %max = transform.split_handle %arg2 : (!transform.op<"linalg.elementwise">)
         -> (!transform.any_op, !transform.any_op)
   
     // The actual tiling transformation takes tile sizes as attributes. It produces a
diff --git a/mlir/test/Examples/transform/Ch4/multiple.mlir b/mlir/test/Examples/transform/Ch4/multiple.mlir
index 2c2c70590552..e6ab5ca65235 100644
--- a/mlir/test/Examples/transform/Ch4/multiple.mlir
+++ b/mlir/test/Examples/transform/Ch4/multiple.mlir
@@ -12,15 +12,15 @@ func.func @fc_relu_operands_00(
 
   // Elementwise addition.
   // expected-remark @below {{add # 0}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{max # 0}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -37,15 +37,15 @@ func.func @fc_relu_operands_01(
 
   // Elementwise addition.
   // expected-remark @below {{add # 1}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{max # 1}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%c0f, %biased : f32, tensor<512x512xf32>)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%c0f, %biased : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -97,7 +97,7 @@ module @transforms attributes { transform.with_named_sequence } {
       -> (!transform.any_op, !transform.any_op, !transform.any_op,
           !transform.param<i32>) {
     // The last operation must be an elementwise binary.
-    transform.match.operation_name %last ["linalg.elemwise_binary"]
+    transform.match.operation_name %last ["linalg.elementwise"]
       : !transform.any_op
 
     // One of its operands must be defined by another operation, to which we
@@ -111,7 +111,7 @@ module @transforms attributes { transform.with_named_sequence } {
       %def = transform.get_defining_op %operand 
         : (!transform.any_value) -> !transform.any_op
       // The defining operation must itself be an elementwise binary.
-      transform.match.operation_name %def ["linalg.elemwise_binary"]
+      transform.match.operation_name %def ["linalg.elementwise"]
         : !transform.any_op
       transform.yield %def : !transform.any_op
     }
diff --git a/mlir/test/Examples/transform/Ch4/sequence.mlir b/mlir/test/Examples/transform/Ch4/sequence.mlir
index 304533f70641..7dd725dfe688 100644
--- a/mlir/test/Examples/transform/Ch4/sequence.mlir
+++ b/mlir/test/Examples/transform/Ch4/sequence.mlir
@@ -22,15 +22,15 @@ func.func @fc_relu(%lhs: tensor<512x512xf32>, %rhs: tensor<512x512xf32>,
 
   // Elementwise addition.
   // expected-remark @below {{elementwise binary}}
-  %biased = linalg.elemwise_binary { fun = #linalg.binary_fn<add> }
+  %biased = linalg.elementwise kind=#linalg.elementwise_kind<add>
     ins(%matmul, %bias : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
 
   // Elementwise max with 0 (ReLU).
-  %c0f = arith.constant 0.0 : f32
+  %c0f = arith.constant dense<0.0> : tensor<512x512xf32>
   // expected-remark @below {{elementwise binary}}
-  %relued = linalg.elemwise_binary { fun = #linalg.binary_fn<max_signed> }
-    ins(%biased, %c0f : tensor<512x512xf32>, f32)
+  %relued = linalg.elementwise kind=#linalg.elementwise_kind<max_signed>
+    ins(%biased, %c0f : tensor<512x512xf32>, tensor<512x512xf32>)
     outs(%output : tensor<512x512xf32>) -> tensor<512x512xf32>
   func.return %relued : tensor<512x512xf32>
 }
@@ -82,7 +82,7 @@ module @transforms attributes { transform.with_named_sequence } {
   // rewriter sequence on success.
   transform.named_sequence @match_elemwise(
       %entry: !transform.any_op {transform.readonly}) -> !transform.any_op {
-    transform.match.operation_name %entry ["linalg.elemwise_binary"] 
+    transform.match.operation_name %entry ["linalg.elementwise"] 
       : !transform.any_op
     transform.yield %entry : !transform.any_op
   }
@@ -113,7 +113,7 @@ module @transforms attributes { transform.with_named_sequence } {
       %last: !transform.any_op {transform.readonly}) 
       -> (!transform.any_op, !transform.any_op, !transform.any_op) {
     // The last operation must be an elementwise binary.
-    transform.match.operation_name %last ["linalg.elemwise_binary"]
+    transform.match.operation_name %last ["linalg.elementwise"]
       : !transform.any_op
     // Its first operand must be defined by another operation, to which we
     // will get a handle here. We are guaranteed that the first operand exists
@@ -123,7 +123,7 @@ module @transforms attributes { transform.with_named_sequence } {
     %middle = transform.get_producer_of_operand %last[0]
       : (!transform.any_op) -> !transform.any_op
     // The defining operation must itself be an elementwise binary.
-    transform.match.operation_name %middle ["linalg.elemwise_binary"]
+    transform.match.operation_name %middle ["linalg.elementwise"]
       : !transform.any_op
     // And the first operand of that operation must be defined by yet another
     // operation.
diff --git a/mlir/test/IR/affine-map.mlir b/mlir/test/IR/affine-map.mlir
index 977aec2536b1..6277b28561f3 100644
--- a/mlir/test/IR/affine-map.mlir
+++ b/mlir/test/IR/affine-map.mlir
@@ -139,7 +139,7 @@
 #map44 = affine_map<(i, j) -> (i - 2*j, j * 6 floordiv 4)>
 
 // Simplifications
-// CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2)[s0] -> (d0 + d1 + d2 + 1, d2 + d1, (d0 * s0) * 8)>
+// CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2)[s0] -> (d0 + d1 + d2 + 1, d1 + d2, (d0 * s0) * 8)>
 #map45 = affine_map<(i, j, k) [N] -> (1 + i + 3 + j - 3 + k, k + 5 + j - 5, 2*i*4*N)>
 
 // CHECK: #map{{[0-9]*}} = affine_map<(d0, d1, d2) -> (0, d1, d0 * 2, 0)>
diff --git a/mlir/test/IR/attribute.mlir b/mlir/test/IR/attribute.mlir
index 4f280bde1aec..edb7357e4e04 100644
--- a/mlir/test/IR/attribute.mlir
+++ b/mlir/test/IR/attribute.mlir
@@ -454,6 +454,10 @@ func.func @allowed_cases_pass() {
   %0 = "test.i32_enum_attr"() {attr = 5: i32} : () -> i32
   // CHECK: test.i32_enum_attr
   %1 = "test.i32_enum_attr"() {attr = 10: i32} : () -> i32
+  // CHECK: test.i32_enum_attr
+  %2 = "test.i32_enum_attr"() {attr = 2147483648: i32} : () -> i32
+  // CHECK: test.i32_enum_attr
+  %3 = "test.i32_enum_attr"() {attr = 4294967295: i32} : () -> i32
   return
 }
 
diff --git a/mlir/test/IR/properties.mlir b/mlir/test/IR/properties.mlir
index dde9100cde14..b541447e85c7 100644
--- a/mlir/test/IR/properties.mlir
+++ b/mlir/test/IR/properties.mlir
@@ -2,10 +2,10 @@
 // # RUN: mlir-opt %s -mlir-print-op-generic -split-input-file  | mlir-opt -mlir-print-op-generic | FileCheck %s --check-prefix=GENERIC
 
 // CHECK:   test.with_properties
-// CHECK-SAME: a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]{{$}}
+// CHECK-SAME: a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]{{$}}
 // GENERIC:   "test.with_properties"()
-// GENERIC-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, b = "foo", c = "bar", flag = true}> : () -> ()
-test.with_properties a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]
+// GENERIC-SAME: <{a = 32 : i64, array = array<i64: 1, 2, 3, 4>, array32 = array<i32: 5, 6>, b = "foo", c = "bar", flag = true}> : () -> ()
+test.with_properties a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]
 
 // CHECK:   test.with_nice_properties
 // CHECK-SAME:    "foo bar" is -3{{$}}
diff --git a/mlir/test/IR/region.mlir b/mlir/test/IR/region.mlir
index 0b959915d6bb..e2088817c520 100644
--- a/mlir/test/IR/region.mlir
+++ b/mlir/test/IR/region.mlir
@@ -106,3 +106,10 @@ func.func @named_region_has_wrong_number_of_blocks() {
 test.single_no_terminator_custom_asm_op {
   "important_dont_drop"() : () -> ()
 }
+
+// -----
+
+// CHECK: test.dummy_op_with_region_ref
+test.dummy_op_with_region_ref {
+  ^bb0:
+}
diff --git a/mlir/test/Integration/Dialect/Transform/match_matmul.mlir b/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
index 7b695cb02725..a374d9a61125 100644
--- a/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
+++ b/mlir/test/Integration/Dialect/Transform/match_matmul.mlir
@@ -69,7 +69,7 @@ func.func @matmul_with_extra_ops_in_func(%lhs: tensor<10x20xf32>, %rhs: tensor<2
   // expected-remark @below {{fill}}
   %fill = linalg.fill ins(%cst : f64) outs(%empty : tensor<10x15xf32>) -> tensor<10x15xf32>
 
-  %real_lhs = linalg.elemwise_binary { fun = #linalg.binary_fn<mul> } 
+  %real_lhs = linalg.mul
     ins(%lhs, %lhs : tensor<10x20xf32>, tensor<10x20xf32>) outs(%lhs : tensor<10x20xf32>) -> tensor<10x20xf32>
 
   // expected-remark @below {{matmul}}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmNeon/vector-contract-i8mm.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmNeon/vector-contract-i8mm.mlir
new file mode 100644
index 000000000000..1ce55ca05c90
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmNeon/vector-contract-i8mm.mlir
@@ -0,0 +1,336 @@
+// REQUIRES: arm-emulator
+
+// DEFINE: %{compile} = mlir-opt %s \
+// DEFINE:   --convert-vector-to-scf --convert-scf-to-cf  --convert-vector-to-llvm='enable-arm-neon enable-arm-i8mm' \
+// DEFINE:   --expand-strided-metadata --convert-to-llvm --finalize-memref-to-llvm  \
+// DEFINE:   --lower-affine --convert-arith-to-llvm --reconcile-unrealized-casts \
+// DEFINE: -o %t
+
+// DEFINE: %{entry_point} = main
+
+// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void  --march=aarch64 --mattr="+neon,+i8mm" \
+// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%native_mlir_arm_runner_utils
+
+// RUN: rm -f %t && %{compile} && FileCheck %s --input-file=%t -check-prefix CHECK-IR && %{run} | FileCheck %s
+
+#packed_maps = [
+  affine_map<(m, n, k) -> (m, k)>,
+  affine_map<(m, n, k) -> (n, k)>,
+  affine_map<(m, n, k) -> (m, n)>
+]
+
+//
+// Test the lowering of `vector.contract` using the `LowerContractionToNeonI8MMPattern`
+//
+// The operation that the `vector.contract` in this test performs is matrix
+// multiplication with accumulate
+//     OUT = ACC + LHS * RHS
+// of two 8-bit integer matrices LHS and RHS, and a 32-bit integer matrix ACC
+// into a 32-bit integer matrix OUT. The LHS and RHS can be sign- or zero- extended,
+// this test covers all the possible variants.
+//
+// Tested are calculations as well as that the relevant `ArmNeon` dialect
+// operations ('arm_neon.smmla`, arm_neon.ummla`, etc) are emitted.
+//
+// That pattern above handles (therefore this test prepares) input/output vectors with
+// specific shapes:
+//   * LHS:      vector<MxKxi8>
+//   * RHS:      vector<NxKxi8>
+//   * ACC, OUT: vector<MxNxi32>
+// where the M and N are even and K is divisible by 8.
+// Note that the RHS is transposed.
+// This data layout makes it efficient to load data into SIMD
+// registers in the layout expected by FEAT_I8MM instructions.
+// Such a `vector.contract` is representative of the code we aim to generate
+// by vectorisation of `linalg.mmt4d`.
+//
+// In this specific test we use M == 4, N == 4, and K == 8.
+//
+
+// Test the operation where both LHS and RHS are interpreted as signed, hence
+// we ultimately emit and execute the `smmla` instruction.
+
+// CHECK-IR-LABEL: llvm.func @test_smmla
+// CHECK-IR-COUNT-4: arm_neon.intr.smmla
+func.func @test_smmla() {
+
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %c0_i8 = arith.constant 0 : i8
+
+  // Accumulator test data
+  %acc_cst = arith.constant dense<[[-44,  20,  44, -46],
+                                   [ -8,  25, -34,  26],
+                                   [-20, -36,  -3,  39],
+                                   [-48, -31, -25, -21]]> : vector<4x4xi32>
+
+  %acc_mem = memref.alloca() : memref<4x4xi32>
+  vector.transfer_write %acc_cst, %acc_mem[%c0, %c0] : vector<4x4xi32>, memref<4x4xi32>
+  %acc = vector.transfer_read %acc_mem[%c0, %c0], %c0_i32 {in_bounds = [true, true]} : memref<4x4xi32>, vector<4x4xi32>
+
+  // LHS test data
+  %lhs_cst = arith.constant dense<[[-35, -27, -36, -31,  23, -34,  -8, -33],
+                                   [-20,  17, -32, -47,  37,  22,  -7, -21],
+                                   [ -7, -35,  20,  -4,  39,  46, -23,  40],
+                                   [ 40,  27,  37,  43,  38,  -6,  37,  49]]> : vector<4x8xi8>
+
+  %lhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %lhs_cst, %lhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %lhs = vector.transfer_read %lhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // RHS test data
+  %rhs_cst = arith.constant dense<[[-17, -50,  -1,  48, -13,  22,  39,  33],
+                                   [-35, -24,  37, -32,  33,  30, -11, -17],
+                                   [-28,  31,   3, -44, -15, -27,  22,  35],
+                                   [-23,  39,  48,  26, -23,  32, -39, -38]]> : vector<4x8xi8>
+
+  %rhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %rhs_cst, %rhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %rhs = vector.transfer_read %rhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+
+  // Matrix multiplication and accumulate with transposed RHS.
+  %0 = arith.extsi %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extsi %rhs : vector<4x8xi8> to vector<4x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<4x8xi32> into vector<4x4xi32>
+
+  // Display the result of the multiplication
+  vector.print str "Result(SMMLA):\n"
+  %u0 = vector.extract %2[0] : vector<4xi32> from vector<4x4xi32>
+  %u1 = vector.extract %2[1] : vector<4xi32> from vector<4x4xi32>
+  %u2 = vector.extract %2[2] : vector<4xi32> from vector<4x4xi32>
+  %u3 = vector.extract %2[3] : vector<4xi32> from vector<4x4xi32>
+  vector.print %u0 : vector<4xi32>
+  vector.print %u1 : vector<4xi32>
+  vector.print %u2 : vector<4xi32>
+  vector.print %u3 : vector<4xi32>
+
+  return
+}
+
+// Test the operation where both LHS and RHS are interpreted as unsigned, hence
+// we ultimately emit and execute the `ummla` instruction.
+
+// CHECK-IR-LABEL: llvm.func @test_ummla
+// CHECK-IR-COUNT-4: arm_neon.intr.ummla
+func.func @test_ummla() {
+
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %c0_i8 = arith.constant 0 : i8
+
+  // Accumulator test data
+  %acc_cst = arith.constant dense<[[16, 16, 48, 40],
+                                   [40, 24, 35, 12],
+                                   [33, 24, 29, 19],
+                                   [28, 13, 33, 18]]> : vector<4x4xi32>
+
+  %acc_mem = memref.alloca() : memref<4x4xi32>
+  vector.transfer_write %acc_cst, %acc_mem[%c0, %c0] : vector<4x4xi32>, memref<4x4xi32>
+  %acc = vector.transfer_read %acc_mem[%c0, %c0], %c0_i32 {in_bounds = [true, true]} : memref<4x4xi32>, vector<4x4xi32>
+
+  // LHS test data
+  %lhs_cst = arith.constant dense<[[35, 42, 37, 49, 36, 36, 23, 33],
+                                   [39, 34, 33, 45, 43, 10, 44, 47],
+                                   [18, 35, 29, 25, 36, 33, 28, 29],
+                                   [26, 49, 43, 32, 27, 16, 45, 33]]> : vector<4x8xi8>
+
+  %lhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %lhs_cst, %lhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %lhs = vector.transfer_read %lhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // RHS test data
+  %rhs_cst = arith.constant dense<[[18, 31, 37, 35, 44, 22, 37, 28],
+                                   [21, 22, 49, 39, 30, 28, 35, 37],
+                                   [21, 47, 39, 35, 23, 43, 24, 49],
+                                   [49, 49, 40, 32, 37, 20, 47, 40]]> : vector<4x8xi8>
+
+  %rhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %rhs_cst, %rhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %rhs = vector.transfer_read %rhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // Matrix multiplication and accumulate with transposed RHS.
+  %0 = arith.extui %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extui %rhs : vector<4x8xi8> to vector<4x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<4x8xi32> into vector<4x4xi32>
+
+  // Display the result of the multiplication
+  vector.print str "Result(UMMLA):\n"
+  %u0 = vector.extract %2[0] : vector<4xi32> from vector<4x4xi32>
+  %u1 = vector.extract %2[1] : vector<4xi32> from vector<4x4xi32>
+  %u2 = vector.extract %2[2] : vector<4xi32> from vector<4x4xi32>
+  %u3 = vector.extract %2[3] : vector<4xi32> from vector<4x4xi32>
+  vector.print %u0 : vector<4xi32>
+  vector.print %u1 : vector<4xi32>
+  vector.print %u2 : vector<4xi32>
+  vector.print %u3 : vector<4xi32>
+
+  return
+}
+
+// Test the operation where LHS is interpreted as unsigned and RHS is
+// interpreted as signed, hence we ultimately emit and execute the `usmmla`
+// instruction.
+
+// CHECK-IR-LABEL: llvm.func @test_usmmla
+// CHECK-IR-COUNT-4: arm_neon.intr.usmmla
+func.func @test_usmmla() {
+
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %c0_i8 = arith.constant 0 : i8
+
+  // Accumulator test data
+  %acc_cst = arith.constant dense<[[-44,  20,  44, -46],
+                                   [ -8,  25, -34,  26],
+                                   [-20, -36,  -3,  39],
+                                   [-48, -31, -25, -21]]> : vector<4x4xi32>
+
+  %acc_mem = memref.alloca() : memref<4x4xi32>
+  vector.transfer_write %acc_cst, %acc_mem[%c0, %c0] : vector<4x4xi32>, memref<4x4xi32>
+  %acc = vector.transfer_read %acc_mem[%c0, %c0], %c0_i32 {in_bounds = [true, true]} : memref<4x4xi32>, vector<4x4xi32>
+
+  // LHS test data
+  %lhs_cst = arith.constant dense<[[153, 161,  24, 157, 211, 154,  52,  27],
+                                   [168,  77, 136, 124, 249,  28,  13, 122],
+                                   [ 97,  82, 181,  39,  53,  25,  80, 240],
+                                   [184, 227, 106, 165, 126, 113, 121, 228]]> : vector<4x8xi8>
+
+  %lhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %lhs_cst, %lhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %lhs = vector.transfer_read %lhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // RHS test data
+  %rhs_cst = arith.constant dense<[[ 40,  27,  37,  43,  38,  -6,  37,  49],
+                                   [-17, -50,  -1,  48, -13,  22,  39,  33],
+                                   [-35, -24,  37, -32,  33,  30, -11, -17],
+                                   [-28,  31,   3, -44, -15, -27,  22,  35]]> : vector<4x8xi8>
+
+  %rhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %rhs_cst, %rhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %rhs = vector.transfer_read %rhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // Matrix multiplication and accumulate with transposed RHS.
+  %0 = arith.extui %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extsi %rhs : vector<4x8xi8> to vector<4x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<4x8xi32> into vector<4x4xi32>
+
+  // Display the result of the multiplication
+  vector.print str "Result(USMMLA):\n"
+  %u0 = vector.extract %2[0] : vector<4xi32> from vector<4x4xi32>
+  %u1 = vector.extract %2[1] : vector<4xi32> from vector<4x4xi32>
+  %u2 = vector.extract %2[2] : vector<4xi32> from vector<4x4xi32>
+  %u3 = vector.extract %2[3] : vector<4xi32> from vector<4x4xi32>
+  vector.print %u0 : vector<4xi32>
+  vector.print %u1 : vector<4xi32>
+  vector.print %u2 : vector<4xi32>
+  vector.print %u3 : vector<4xi32>
+
+  return
+}
+
+// Test the operation where LHS is interpreted as signed and RHS is interpreted
+// as unsigned. In this test we ultimately emit end execute the `usmmla`
+// instruction with reversed operands, see `LowerContractionToNeonI8MMPattern.cpp`
+// for more details.
+
+// CHECK-IR-LABEL: llvm.func @test_summla
+// CHECK-IR-COUNT-4: arm_neon.intr.usmmla
+func.func @test_summla() {
+
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %c0_i8 = arith.constant 0 : i8
+
+  // Accumulator test data
+  %acc_cst = arith.constant dense<[[-44,  20,  44, -46],
+                                   [ -8,  25, -34,  26],
+                                   [-20, -36,  -3,  39],
+                                   [-48, -31, -25, -21]]> : vector<4x4xi32>
+
+  %acc_mem = memref.alloca() : memref<4x4xi32>
+  vector.transfer_write %acc_cst, %acc_mem[%c0, %c0] : vector<4x4xi32>, memref<4x4xi32>
+  %acc = vector.transfer_read %acc_mem[%c0, %c0], %c0_i32 {in_bounds = [true, true]} : memref<4x4xi32>, vector<4x4xi32>
+
+  // LHS test data
+  %lhs_cst = arith.constant dense<[[-35, -27, -36, -31,  23, -34,  -8, -33],
+                                   [-20,  17, -32, -47,  37,  22,  -7, -21],
+                                   [ -7, -35,  20,  -4,  39,  46, -23,  40],
+                                   [ 40,  27,  37,  43,  38,  -6,  37,  49]]> : vector<4x8xi8>
+
+  %lhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %lhs_cst, %lhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %lhs = vector.transfer_read %lhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // RHS test data
+  %rhs_cst = arith.constant dense<[[125, 171, 138, 187, 108, 175,  82,  99],
+                                   [221,  25, 164,  97, 156, 221, 218, 177],
+                                   [171, 160, 219, 191, 144,  45, 161, 210],
+                                   [223, 165, 123,  99, 108,  86,  37,  92]]> : vector<4x8xi8>
+
+  %rhs_mem = memref.alloca() : memref<4x8xi8>
+  vector.transfer_write %rhs_cst, %rhs_mem[%c0, %c0] : vector<4x8xi8>, memref<4x8xi8>
+  %rhs = vector.transfer_read %rhs_mem[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<4x8xi8>, vector<4x8xi8>
+
+  // Matrix multiplication and accumulate with transposed RHS.
+  %0 = arith.extsi %lhs : vector<4x8xi8> to vector<4x8xi32>
+  %1 = arith.extui %rhs : vector<4x8xi8> to vector<4x8xi32>
+  %2 = vector.contract {indexing_maps = #packed_maps,
+                        iterator_types = ["parallel", "parallel", "reduction"],
+                        kind = #vector.kind<add>} %0, %1, %acc
+    : vector<4x8xi32>, vector<4x8xi32> into vector<4x4xi32>
+
+  // Display the result of the multiplication
+  vector.print str "Result(SUMMLA (i.e. USMMLA transposed)):\n"
+  %u0 = vector.extract %2[0] : vector<4xi32> from vector<4x4xi32>
+  %u1 = vector.extract %2[1] : vector<4xi32> from vector<4x4xi32>
+  %u2 = vector.extract %2[2] : vector<4xi32> from vector<4x4xi32>
+  %u3 = vector.extract %2[3] : vector<4xi32> from vector<4x4xi32>
+  vector.print %u0 : vector<4xi32>
+  vector.print %u1 : vector<4xi32>
+  vector.print %u2 : vector<4xi32>
+  vector.print %u3 : vector<4xi32>
+
+  return
+}
+
+func.func @main() {
+// CHECK-LABEL: Result(SMMLA):
+// CHECK: ( -1999,  1941,   685, -2879 )
+// CHECK: ( -3705,  2952,   987,  -685 )
+// CHECK: (  2565,  4157, -1589,  -357 )
+// CHECK: (  2383, -2252,    32, -1365 )
+  func.call @test_smmla() : () -> ()
+
+// CHECK-LABEL: Result(UMMLA):
+// CHECK: ( 9183,  9513, 10460, 11314 )
+// CHECK: ( 9648,  9812, 10092, 12088 )
+// CHECK: ( 7548,  7625,  8398,  9044 )
+// CHECK: ( 8855,  9046,  9685, 11191 )
+  func.call @test_ummla() : () -> ()
+
+// CHECK-LABEL: Result(USMMLA):
+// CHECK: ( 28403,    445,  -2759, -11409 )
+// CHECK: ( 34908,   1047,    142,  -7274 )
+// CHECK: ( 31032,   6807,  -2378,   7382 )
+// CHECK: ( 44217,   6396, -10930,    623 )
+  func.call @test_usmmla() : () -> ()
+
+// CHECK-LABEL: Result(SUMMLA (i.e. USMMLA transposed)):
+// CHECK: ( -27190, -28812, -30502, -23575 )
+// CHECK: (  -7613,  -8386, -15938,  -6521 )
+// CHECK: (   9468,  18750,   9199,   5764 )
+// CHECK: (  33655,  41064,  48900,  31627 )
+  func.call @test_summla() : () -> ()
+
+  return
+}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/transfer-read-scalable-non-trailing.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/transfer-read-scalable-non-trailing.mlir
new file mode 100644
index 000000000000..36fdb60d3e7b
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/transfer-read-scalable-non-trailing.mlir
@@ -0,0 +1,79 @@
+// REQUIRES: arm-emulator
+
+// DEFINE: %{compile} = mlir-opt %s \
+// DEFINE:   --arm-sve-legalize-vector-storage --convert-vector-to-scf --convert-scf-to-cf  --convert-vector-to-llvm='enable-arm-sve' \
+// DEFINE:   --expand-strided-metadata    --lower-affine --convert-to-llvm --finalize-memref-to-llvm  --reconcile-unrealized-casts \
+// DEFINE: -o %t
+
+// DEFINE: %{entry_point} = main
+
+// DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void  --march=aarch64 --mattr="+sve" \
+// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%native_mlir_arm_runner_utils
+
+// RUN: rm -f %t && %{compile} && %{run} | FileCheck %s
+
+// Test the transfer_read with vector type with a non-trailing scalable
+// dimension as transformed by the pattern LegalizeTransferRead.
+
+func.func @transfer_read_scalable_non_trailing(%vs : i32, %M : memref<?x8xi8>) {
+  func.call @setArmVLBits(%vs) : (i32) -> ()
+
+  // Read an LLVM-illegal vector
+  %c0 = arith.constant 0 : index
+  %c0_i8 = arith.constant 0 : i8
+  %A = vector.transfer_read %M[%c0, %c0], %c0_i8 {in_bounds = [true, true]} : memref<?x8xi8>, vector<[4]x8xi8>
+
+  // Print the vector, for verification.
+  %B = vector.shape_cast %A : vector<[4]x8xi8> to vector<[32]xi8>
+  func.call @printVec(%B) : (vector<[32]xi8>) -> ()
+
+  return
+}
+
+func.func @main() {
+
+  %c0 = arith.constant 0 : index
+
+// Prepare an 8x8 buffer with test data. The test performs two reads
+// of a [4]x8 vector from the buffer. One read, with vector length 128 bits,
+// reads the first half the buffer. The other read, with vector length
+// 256 bits, reads the entire buffer.
+  %T = arith.constant dense<[[11, 12, 13, 14, 15, 16, 17, 18],
+                             [21, 22, 23, 24, 25, 26, 27, 28],
+                             [31, 32, 33, 34, 35, 36, 37, 38],
+                             [41, 42, 43, 44, 45, 46, 47, 48],
+                             [51, 52, 53, 54, 55, 56, 57, 58],
+                             [61, 62, 63, 64, 65, 66, 67, 68],
+                             [71, 72, 73, 74, 75, 76, 77, 78],
+                             [81, 82, 83, 84, 85, 86, 87, 88]]> : vector<8x8xi8>
+
+  %M = memref.alloca() : memref<8x8xi8>
+  vector.transfer_write %T, %M[%c0, %c0] : vector<8x8xi8>, memref<8x8xi8>
+  %MM = memref.cast %M : memref<8x8xi8> to memref<?x8xi8>
+
+// CHECK-LABEL: Result(VL128):
+// CHECK:( 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28 )
+// CHECK:( 31, 32, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46, 47, 48 )
+  vector.print str "Result(VL128):\n"
+  %c128 = arith.constant 128 : i32
+  func.call @transfer_read_scalable_non_trailing(%c128, %MM) : (i32, memref<?x8xi8>) -> ()
+
+// CHECK-LABEL: Result(VL256):
+// CHECK: ( 11, 12, 13, 14, 15, 16, 17, 18, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46, 47, 48 )
+// CHECK: ( 51, 52, 53, 54, 55, 56, 57, 58, 61, 62, 63, 64, 65, 66, 67, 68, 71, 72, 73, 74, 75, 76, 77, 78, 81, 82, 83, 84, 85, 86, 87, 88 )
+  vector.print str "Result(VL256):\n"
+  %c256 = arith.constant 256 : i32
+  func.call @transfer_read_scalable_non_trailing(%c256, %MM) : (i32, memref<?x8xi8>) -> ()
+
+  return
+}
+
+func.func private @printVec(%v : vector<[32]xi8>) {
+  %v0 = vector.scalable.extract %v[0] : vector<[16]xi8> from vector<[32]xi8>
+  %v1 = vector.scalable.extract %v[16] : vector<[16]xi8> from vector<[32]xi8>
+  vector.print %v0 : vector<[16]xi8>
+  vector.print %v1 : vector<[16]xi8>
+  return
+}
+
+func.func private @setArmVLBits(%bits : i32)
diff --git a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
index 77e52946b830..d09373bdb3f1 100644
--- a/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
+++ b/mlir/test/Interfaces/TilingInterface/tile-and-fuse-consumer.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --transform-interpreter --cse --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --transform-interpreter --cse --split-input-file --verify-diagnostics %s | FileCheck %s
 
 #map = affine_map<(d0) -> (d0)>
 module {
@@ -19,7 +19,7 @@ module {
     }
     %in_operand_2 = tensor.empty() : tensor<64xf32>
     %out_operand_3 = tensor.empty() : tensor<64xf32>
-    %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%1#1, %in_operand_2 : tensor<64xf32>, tensor<64xf32>) outs(%out_operand_3 : tensor<64xf32>) -> tensor<64xf32>
+    %2 = linalg.add ins(%1#1, %in_operand_2 : tensor<64xf32>, tensor<64xf32>) outs(%out_operand_3 : tensor<64xf32>) -> tensor<64xf32>
     return %2 : tensor<64xf32>
   }
 }
@@ -50,7 +50,7 @@ module attributes {transform.with_named_sequence} {
 //      CHECK:      %[[INSERT_MAT:.*]] = tensor.insert_slice %[[MAT_OUT]] into %[[FIRST_OUT_ARG]][%[[IV]]] [32] [1]
 //      CHECK:      %[[SLICE_OPERAND2:.*]] = tensor.extract_slice %0[%[[IV]]] [32] [1]
 //      CHECK:      %[[SLICE_OUT:.*]] = tensor.extract_slice %[[ELEM_OUT_ARG]][%[[IV]]] [32] [1]
-//      CHECK:      %[[ELEM_OUT:.*]] = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+//      CHECK:      %[[ELEM_OUT:.*]] = linalg.add
 // CHECK-SAME:              ins(%[[MAT_OUT]], %[[SLICE_OPERAND2]] :
 // CHECK-SAME:              outs(%[[SLICE_OUT]] :
 //      CHECK:      %[[INSERT_ELEM:.*]] = tensor.insert_slice %[[ELEM_OUT]] into %[[ELEM_OUT_ARG]][%[[IV]]] [32] [1]
@@ -76,7 +76,7 @@ module {
     }
     %in_operand_2 = tensor.empty() : tensor<64x64xf32>
     %out_operand_3 = tensor.empty() : tensor<64x64xf32>
-    %2 = linalg.elemwise_binary {fun = #linalg.binary_fn<add>} ins(%1#1, %in_operand_2 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%out_operand_3 : tensor<64x64xf32>) -> tensor<64x64xf32>
+    %2 = linalg.add ins(%1#1, %in_operand_2 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%out_operand_3 : tensor<64x64xf32>) -> tensor<64x64xf32>
     return %2 : tensor<64x64xf32>
   }
 }
@@ -109,7 +109,7 @@ module attributes {transform.with_named_sequence} {
 // CHECK-SAME:              outs(%[[MAT_OUT_SLICE]] :
 //      CHECK:      %[[SLICE_OPERAND2:.*]] = tensor.extract_slice %[[OUT_INIT]][%[[IV1]], %[[IV2]]] [32, 32] [1, 1]
 //      CHECK:      %[[SLICE_OUT:.*]] = tensor.extract_slice %[[ELEM_OUT_ARG]][%[[IV1]], %[[IV2]]] [32, 32] [1, 1]
-//      CHECK:      %[[ELEM_OUT:.*]] = linalg.elemwise_binary {fun = #linalg.binary_fn<add>}
+//      CHECK:      %[[ELEM_OUT:.*]] = linalg.add
 // CHECK-SAME:              ins(%[[MAT_OUT]], %[[SLICE_OPERAND2]] :
 // CHECK-SAME:              outs(%[[SLICE_OUT]] :
 //      CHECK:      scf.forall.in_parallel {
@@ -620,3 +620,294 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
+
+// -----
+
+func.func @multi_slice_fusion1(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?xf32>, %arg2 : tensor<?xf32>, %arg3 : index) -> tensor<?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %dim0 = tensor.dim %arg0, %c0 : tensor<?x?xf32>
+  %dim1 = tensor.dim %arg0, %c1 : tensor<?x?xf32>
+  %loop:2 = scf.forall (%iv0) =  (%c0) to (%dim0) step (%arg3) shared_outs(%init0 = %arg1, %init1 = %arg2) -> (tensor<?xf32>, tensor<?xf32>) {
+    %tilesize = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv0)[%dim0, %arg3]
+    %arg0_slice = tensor.extract_slice %arg0[%iv0, 0] [%tilesize, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+    %init0_slice = tensor.extract_slice %init0[%iv0] [%tilesize] [1] : tensor<?xf32> to tensor<?xf32>
+    %init1_slice = tensor.extract_slice %init1[%iv0] [%tilesize] [1] : tensor<?xf32> to tensor<?xf32>
+    %generic:2 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>, affine_map<(d0, d1) -> (d0)>],
+	iterator_types = ["parallel", "reduction"]}
+	ins(%arg0_slice : tensor<?x?xf32>) outs(%init0_slice, %init1_slice : tensor<?xf32>, tensor<?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+        %0 = arith.mulf %b0, %b1 : f32
+	%1 = arith.addf %b0, %b2 : f32
+	linalg.yield %0, %1 : f32, f32
+    } -> (tensor<?xf32>, tensor<?xf32>)
+    scf.forall.in_parallel {
+      tensor.parallel_insert_slice %generic#0 into %init0[%iv0] [%tilesize] [1] : tensor<?xf32> into tensor<?xf32>
+      tensor.parallel_insert_slice %generic#1 into %init1[%iv0] [%tilesize] [1] : tensor<?xf32> into tensor<?xf32>
+    }	
+  }
+  %empty = tensor.empty(%dim0) : tensor<?xf32>
+  %result = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]}
+      ins(%loop#0, %loop#1 : tensor<?xf32>, tensor<?xf32>) outs(%empty : tensor<?xf32>) {
+    ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+      %0 = arith.addf %b0, %b1 : f32
+      linalg.yield %0 : f32
+  } -> tensor<?xf32>
+  return %result : tensor<?xf32>
+}
+// CHECK-LABEL: func @multi_slice_fusion1(
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<?x?xf32>
+//       CHECK:   %[[C0:.+]] = arith.constant 0
+//       CHECK:   %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]]
+//       CHECK:   %[[EMPTY:.+]] = tensor.empty(%[[DIM0]])
+//       CHECK:   %[[RESULT:.+]]:3 = scf.forall (%[[IV:.+]]) =
+//  CHECK-SAME:       , %[[INIT:[a-zA-Z0-9]+]] = %[[EMPTY]])
+//       CHECK:     %[[TILESIZE:.+]] = affine.min
+//   CHECK-DAG:     %[[GENERIC:.+]]:2 = linalg.generic
+//   CHECK-DAG:     %[[INIT_SLICE:.+]] = tensor.extract_slice %[[INIT]][%[[IV]]] [%[[TILESIZE]]]
+//       CHECK:     %[[FUSED:.+]] = linalg.generic
+//  CHECK-SAME:         ins(%[[GENERIC]]#0, %[[GENERIC]]#1 :
+//       CHECK:     tensor.parallel_insert_slice %[[FUSED]] into %[[INIT]][%[[IV]]] [%[[TILESIZE]]]
+//       CHECK:   return %[[RESULT]]#2
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %loop = transform.structured.match ops{["scf.forall"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield0, %yield1 = transform.split_handle %yield : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %a, %b = transform.test.fuse_consumer %yield0, %yield1 in (%loop)
+      : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// -----
+
+// Check that when the given operand tiles are inconsistent, tiling fails.
+
+func.func @multi_slice_fusion2(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?xf32>, %arg2 : tensor<?xf32>, %arg3 : index) -> tensor<?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %dim0 = tensor.dim %arg0, %c0 : tensor<?x?xf32>
+  %dim1 = tensor.dim %arg0, %c1 : tensor<?x?xf32>
+  %loop:2 = scf.forall (%iv0) =  (%c0) to (%dim0) step (%arg3) shared_outs(%init0 = %arg1, %init1 = %arg2) -> (tensor<?xf32>, tensor<?xf32>) {
+    %tilesize = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv0)[%dim0, %arg3]
+    %arg0_slice = tensor.extract_slice %arg0[%iv0, 0] [%tilesize, %dim1] [1, 1] : tensor<?x?xf32> to tensor<?x?xf32>
+    %init0_slice = tensor.extract_slice %init0[%iv0] [%tilesize] [1] : tensor<?xf32> to tensor<?xf32>
+    %generic0 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
+	iterator_types = ["parallel", "reduction"]}
+	ins(%arg0_slice : tensor<?x?xf32>) outs(%init0_slice : tensor<?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+        %0 = arith.mulf %b0, %b1 : f32
+	linalg.yield %0 : f32
+    } -> tensor<?xf32>
+    %init1_slice = tensor.extract_slice %init1[%iv0] [%tilesize] [1] : tensor<?xf32> to tensor<?xf32>
+    %generic1 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
+	iterator_types = ["parallel", "reduction"]}
+	ins(%arg0_slice : tensor<?x?xf32>) outs(%init1_slice: tensor<?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+	%0 = arith.addf %b0, %b1 : f32
+	linalg.yield %0: f32
+    } -> tensor<?xf32>
+    scf.forall.in_parallel {
+      tensor.parallel_insert_slice %generic0 into %init0[%iv0] [%tilesize] [1] : tensor<?xf32> into tensor<?xf32>
+      tensor.parallel_insert_slice %generic1 into %init1[%iv0] [%tilesize] [1] : tensor<?xf32> into tensor<?xf32>
+    }	
+  }
+  %empty = tensor.empty(%dim0) : tensor<?xf32>
+  %result = linalg.generic {
+      indexing_maps = [affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>, affine_map<(d0) -> (d0)>],
+      iterator_types = ["parallel"]}
+      ins(%loop#0, %loop#1 : tensor<?xf32>, tensor<?xf32>) outs(%empty : tensor<?xf32>) {
+    ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+      %0 = arith.addf %b0, %b1 : f32
+      linalg.yield %0 : f32
+  } -> tensor<?xf32>
+  return %result : tensor<?xf32>
+}
+// CHECK-LABEL: func @multi_slice_fusion2(
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<?x?xf32>
+//       CHECK:   %[[C0:.+]] = arith.constant 0
+//       CHECK:   %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]]
+//       CHECK:   %[[EMPTY:.+]] = tensor.empty(%[[DIM0]])
+//       CHECK:   %[[RESULT:.+]]:3 = scf.forall (%[[IV:.+]]) =
+//  CHECK-SAME:       , %[[INIT:[a-zA-Z0-9]+]] = %[[EMPTY]])
+//       CHECK:     %[[TILESIZE:.+]] = affine.min
+//       CHECK:     %[[GENERIC0:.+]] = linalg.generic
+//       CHECK:     %[[GENERIC1:.+]] = linalg.generic
+//   CHECK-DAG:     %[[INIT_SLICE:.+]] = tensor.extract_slice %[[INIT]][%[[IV]]] [%[[TILESIZE]]]
+//       CHECK:     %[[FUSED:.+]] = linalg.generic
+//  CHECK-SAME:         ins(%[[GENERIC0]], %[[GENERIC1]] :
+//       CHECK:     tensor.parallel_insert_slice %[[FUSED]] into %[[INIT]][%[[IV]]] [%[[TILESIZE]]]
+//       CHECK:   return %[[RESULT]]#2
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %loop = transform.structured.match ops{["scf.forall"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield0, %yield1 = transform.split_handle %yield : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %a, %b = transform.test.fuse_consumer %yield0, %yield1 in (%loop)
+      : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @multi_slice_fusion_with_broadcast(%arg0 : tensor<?x?x?xf32>, %arg1 : tensor<?x?xf32>, %arg2 : tensor<?xf32>,
+    %arg3 : index, %arg4 : index) -> tensor<?x?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %dim0 = tensor.dim %arg0, %c0 : tensor<?x?x?xf32>
+  %dim1 = tensor.dim %arg0, %c1 : tensor<?x?x?xf32>
+  %dim2 = tensor.dim %arg0, %c2 : tensor<?x?x?xf32>
+  %loop:2 = scf.forall (%iv0, %iv1) =  (%c0, %c0) to (%dim0, %dim1) step (%arg3, %arg4)
+      shared_outs(%init0 = %arg1, %init1 = %arg2) -> (tensor<?x?xf32>, tensor<?xf32>) {
+    %tilesize0 = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv0)[%dim0, %arg3]
+    %tilesize1 = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv1)[%dim1, %arg4]
+    %arg0_slice = tensor.extract_slice %arg0[%iv0, %iv1, 0] [%tilesize0, %tilesize1, %dim2] [1, 1, 1]
+        : tensor<?x?x?xf32> to tensor<?x?x?xf32>
+    %init0_slice = tensor.extract_slice %init0[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+        : tensor<?x?xf32> to tensor<?x?xf32>
+    %generic0 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>],
+	      iterator_types = ["parallel", "parallel", "reduction"]}
+	      ins(%arg0_slice : tensor<?x?x?xf32>) outs(%init0_slice : tensor<?x?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+        %0 = arith.mulf %b0, %b1 : f32
+	      linalg.yield %0 : f32
+    } -> tensor<?x?xf32>
+    %init1_slice = tensor.extract_slice %init1[%iv0] [%tilesize0] [1] : tensor<?xf32> to tensor<?xf32>
+    %generic1 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>],
+	      iterator_types = ["parallel", "reduction"]}
+	      ins(%generic0 : tensor<?x?xf32>) outs(%init1_slice: tensor<?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+      	%0 = arith.addf %b0, %b1 : f32
+	      linalg.yield %0: f32
+    } -> tensor<?xf32>
+    scf.forall.in_parallel {
+      tensor.parallel_insert_slice %generic0 into %init0[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+          : tensor<?x?xf32> into tensor<?x?xf32>
+      tensor.parallel_insert_slice %generic1 into %init1[%iv0] [%tilesize0] [1] : tensor<?xf32> into tensor<?xf32>
+    }
+  }
+  %empty = tensor.empty(%dim0, %dim1) : tensor<?x?xf32>
+  %result = linalg.generic {
+      indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>, affine_map<(d0, d1) -> (d0, d1)>],
+      iterator_types = ["parallel", "parallel"]}
+      ins(%loop#0, %loop#1 : tensor<?x?xf32>, tensor<?xf32>) outs(%empty : tensor<?x?xf32>) {
+    ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+      %0 = arith.addf %b0, %b1 : f32
+      linalg.yield %0 : f32
+  } -> tensor<?x?xf32>
+  return %result : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %loop = transform.structured.match ops{["scf.forall"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield0, %yield1 = transform.split_handle %yield : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %a, %b = transform.test.fuse_consumer %yield0, %yield1 in (%loop)
+      : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+// CHECK-LABEL: func @multi_slice_fusion_with_broadcast(
+//  CHECK-SAME:     %[[ARG0:.+]]: tensor<?x?x?xf32>
+//   CHECK-DAG:   %[[C0:.+]] = arith.constant 0
+//   CHECK-DAG:   %[[C1:.+]] = arith.constant 1
+//   CHECK-DAG:   %[[DIM0:.+]] = tensor.dim %[[ARG0]], %[[C0]]
+//   CHECK-DAG:   %[[DIM1:.+]] = tensor.dim %[[ARG0]], %[[C1]]
+//       CHECK:   %[[EMPTY:.+]] = tensor.empty(%[[DIM0]], %[[DIM1]])
+//       CHECK:   %[[RESULT:.+]]:3 = scf.forall (%[[IV0:[a-zA-Z0-9]+]], %[[IV1:[a-zA-Z0-9]+]]) =
+//  CHECK-SAME:       , %[[INIT:[a-zA-Z0-9]+]] = %[[EMPTY]])
+//   CHECK-DAG:     %[[TILESIZE0:.+]] = affine.min {{.+}}(%[[IV0]])
+//   CHECK-DAG:     %[[TILESIZE1:.+]] = affine.min {{.+}}(%[[IV1]])
+//       CHECK:     %[[GENERIC0:.+]] = linalg.generic
+//       CHECK:     %[[GENERIC1:.+]] = linalg.generic
+//   CHECK-DAG:     %[[INIT_SLICE:.+]] = tensor.extract_slice %[[INIT]][%[[IV0]], %[[IV1]]] [%[[TILESIZE0]], %[[TILESIZE1]]]
+//       CHECK:     %[[FUSED:.+]] = linalg.generic
+//  CHECK-SAME:         ins(%[[GENERIC0]], %[[GENERIC1]] :
+//       CHECK:     tensor.parallel_insert_slice %[[FUSED]] into %[[INIT]][%[[IV0]], %[[IV1]]] [%[[TILESIZE0]], %[[TILESIZE1]]]
+//       CHECK:   return %[[RESULT]]#2
+
+// -----
+
+func.func @multi_slice_fusion_invalid(%arg0 : tensor<?x?x?xf32>, %arg1 : tensor<?x?xf32>, %arg2 : tensor<?x?xf32>,
+    %arg3 : index, %arg4 : index) -> tensor<?x?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2 : index
+  %dim0 = tensor.dim %arg0, %c0 : tensor<?x?x?xf32>
+  %dim1 = tensor.dim %arg0, %c1 : tensor<?x?x?xf32>
+  %dim2 = tensor.dim %arg0, %c2 : tensor<?x?x?xf32>
+  %loop:2 = scf.forall (%iv0, %iv1) =  (%c0, %c0) to (%dim0, %dim1) step (%arg3, %arg4)
+      shared_outs(%init0 = %arg1, %init1 = %arg2) -> (tensor<?x?xf32>, tensor<?x?xf32>) {
+    %tilesize0 = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv0)[%dim0, %arg3]
+    %tilesize1 = affine.min affine_map<(d0)[s0, s1] -> (s1, s0 - d0)>(%iv1)[%dim1, %arg4]
+    %arg0_slice = tensor.extract_slice %arg0[%iv0, %iv1, 0] [%tilesize0, %tilesize1, %dim2] [1, 1, 1]
+        : tensor<?x?x?xf32> to tensor<?x?x?xf32>
+    %init0_slice = tensor.extract_slice %init0[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+        : tensor<?x?xf32> to tensor<?x?xf32>
+    %generic0 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>],
+	      iterator_types = ["parallel", "parallel", "reduction"]}
+	      ins(%arg0_slice : tensor<?x?x?xf32>) outs(%init0_slice : tensor<?x?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+        %0 = arith.mulf %b0, %b1 : f32
+	      linalg.yield %0 : f32
+    } -> tensor<?x?xf32>
+    %init1_slice = tensor.extract_slice %init1[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+        : tensor<?x?xf32> to tensor<?x?xf32>
+    %generic1 = linalg.generic {
+        indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>],
+	      iterator_types = ["parallel", "parallel", "reduction"]}
+	      ins(%arg0_slice : tensor<?x?x?xf32>) outs(%init1_slice: tensor<?x?xf32>) {
+      ^bb0(%b0 : f32, %b1 : f32):
+      	%0 = arith.addf %b0, %b1 : f32
+	      linalg.yield %0: f32
+    } -> tensor<?x?xf32>
+    scf.forall.in_parallel {
+      // expected-error @below {{failed to fuse consumer of slice}}
+      tensor.parallel_insert_slice %generic0 into %init0[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+          : tensor<?x?xf32> into tensor<?x?xf32>
+      tensor.parallel_insert_slice %generic1 into %init1[%iv0, %iv1] [%tilesize0, %tilesize1] [1, 1]
+          : tensor<?x?xf32> into tensor<?x?xf32>
+    }
+  }
+  %empty = tensor.empty(%dim0, %dim1) : tensor<?x?xf32>
+  %result = linalg.generic {
+      indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d1, d0)>, affine_map<(d0, d1) -> (d0, d1)>],
+      iterator_types = ["parallel", "parallel"]}
+      ins(%loop#0, %loop#1 : tensor<?x?xf32>, tensor<?x?xf32>) outs(%empty : tensor<?x?xf32>) {
+    ^bb0(%b0 : f32, %b1 : f32, %b2 : f32):
+      %0 = arith.addf %b0, %b1 : f32
+      linalg.yield %0 : f32
+  } -> tensor<?x?xf32>
+  return %result : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1 : !transform.any_op {transform.readonly}) {
+    %loop = transform.structured.match ops{["scf.forall"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield = transform.structured.match ops{["tensor.parallel_insert_slice"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+    %yield0, %yield1 = transform.split_handle %yield : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %a, %b = transform.test.fuse_consumer %yield0, %yield1 in (%loop)
+      : (!transform.any_op, !transform.any_op, !transform.any_op) -> (!transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll
index a05a2b4bd450..d48be66f2063 100644
--- a/mlir/test/Target/LLVMIR/Import/import-failure.ll
+++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll
@@ -258,22 +258,6 @@ end:
 
 ; // -----
 
-; CHECK:      <unknown>
-; CHECK-SAME: warning: expected function_entry_count to be attached to a function
-; CHECK:      warning: unhandled metadata: !0 = !{!"function_entry_count", i64 42}
-define void @cond_br(i1 %arg) {
-entry:
-  br i1 %arg, label %bb1, label %bb2, !prof !0
-bb1:
-  ret void
-bb2:
-  ret void
-}
-
-!0 = !{!"function_entry_count", i64 42}
-
-; // -----
-
 ; CHECK:      <unknown>
 ; CHECK-SAME: warning: dropped instruction: call void @llvm.experimental.noalias.scope.decl(metadata !0)
 define void @unused_scope() {
diff --git a/mlir/test/Target/LLVMIR/arm-neon.mlir b/mlir/test/Target/LLVMIR/arm-neon.mlir
index e096172667c9..e1328ad448f0 100644
--- a/mlir/test/Target/LLVMIR/arm-neon.mlir
+++ b/mlir/test/Target/LLVMIR/arm-neon.mlir
@@ -82,3 +82,16 @@ llvm.func @arm_neon_usmmla(%arg0: vector<16xi8>,
         -> vector<4xi32>
   llvm.return %0 : vector<4xi32>
 }
+
+// -----
+
+// CHECK-LABEL: arm_neon_bfmmla
+llvm.func @arm_neon_bfmmla(%arg0: vector<8xbf16>,
+                           %arg1: vector<8xbf16>,
+                           %arg2: vector<4xf32>) -> vector<4xf32> {
+  // CHECK: <4 x float> @llvm.aarch64.neon.bfmmla(<4 x float
+  %0 = "arm_neon.intr.bfmmla"(%arg2, %arg0, %arg1) :
+    (vector<4xf32>, vector<8xbf16>, vector<8xbf16>)
+        -> vector<4xf32>
+  llvm.return %0 : vector<4xf32>
+}
diff --git a/mlir/test/Target/LLVMIR/arm-sve.mlir b/mlir/test/Target/LLVMIR/arm-sve.mlir
index da71cb5a63bd..737145c74e33 100644
--- a/mlir/test/Target/LLVMIR/arm-sve.mlir
+++ b/mlir/test/Target/LLVMIR/arm-sve.mlir
@@ -60,6 +60,18 @@ llvm.func @arm_sve_usmmla(%arg0: vector<[16]xi8>,
   llvm.return %0 : vector<[4]xi32>
 }
 
+// CHECK-LABEL: define <vscale x 4 x float> @arm_sve_bfmmla
+llvm.func @arm_sve_bfmmla(%arg0: vector<[8]xbf16>,
+                          %arg1: vector<[8]xbf16>,
+                          %arg2: vector<[4]xf32>)
+                          -> vector<[4]xf32> {
+  // CHECK: call <vscale x 4 x float> @llvm.aarch64.sve.bfmmla(<vscale x 4 x float>
+  %0 = "arm_sve.intr.bfmmla"(%arg2, %arg0, %arg1) :
+    (vector<[4]xf32>, vector<[8]xbf16>, vector<[8]xbf16>)
+        -> vector<[4]xf32>
+  llvm.return %0 : vector<[4]xf32>
+}
+
 // CHECK-LABEL: define <vscale x 4 x i32> @arm_sve_arithi
 llvm.func @arm_sve_arithi(%arg0: vector<[4]xi32>,
                           %arg1: vector<[4]xi32>,
diff --git a/mlir/test/Target/LLVMIR/nvvm/elect.mlir b/mlir/test/Target/LLVMIR/nvvm/elect.mlir
new file mode 100644
index 000000000000..3c5cac4b650b
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/elect.mlir
@@ -0,0 +1,20 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK-LABEL: @test_nvvm_elect_sync
+llvm.func @test_nvvm_elect_sync() -> i1 {
+  // CHECK: %[[RES:.*]] = call { i32, i1 } @llvm.nvvm.elect.sync(i32 -1)
+  // CHECK-NEXT: %[[PRED:.*]] = extractvalue { i32, i1 } %[[RES]], 1
+  // CHECK-NEXT: ret i1 %[[PRED]]
+  %0 = nvvm.elect.sync -> i1
+  llvm.return %0 : i1
+}
+
+// CHECK-LABEL: @test_nvvm_elect_sync_mask
+llvm.func @test_nvvm_elect_sync_mask(%mask : i32) -> i1 {
+  // CHECK: %[[RES:.*]] = call { i32, i1 } @llvm.nvvm.elect.sync(i32 %0)
+  // CHECK-NEXT: %[[PRED:.*]] = extractvalue { i32, i1 } %[[RES]], 1
+  // CHECK-NEXT: ret i1 %[[PRED]]
+  %0 = nvvm.elect.sync %mask -> i1
+  llvm.return %0 : i1
+}
+
diff --git a/mlir/test/Target/LLVMIR/nvvmir.mlir b/mlir/test/Target/LLVMIR/nvvmir.mlir
index 660d0a22dce9..f86a04186f51 100644
--- a/mlir/test/Target/LLVMIR/nvvmir.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir.mlir
@@ -265,15 +265,6 @@ llvm.func @nvvm_vote(%0 : i32, %1 : i1) -> i32 {
   llvm.return %3 : i32
 }
 
-// CHECK-LABEL: @nvvm_elect_sync
-llvm.func @nvvm_elect_sync() -> i1 {
-  // CHECK: %[[RES:.*]] = call { i32, i1 } @llvm.nvvm.elect.sync(i32 -1)
-  // CHECK-NEXT: %[[PRED:.*]] = extractvalue { i32, i1 } %[[RES]], 1
-  // CHECK-NEXT: ret i1 %[[PRED]]
-  %0 = nvvm.elect.sync -> i1
-  llvm.return %0 : i1
-}
-
 // CHECK-LABEL: @nvvm_mma_mn8n8k4_row_col_f32_f32
 llvm.func @nvvm_mma_mn8n8k4_row_col_f32_f32(%a0 : vector<2xf16>, %a1 : vector<2xf16>,
                     %b0 : vector<2xf16>, %b1 : vector<2xf16>,
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
new file mode 100644
index 000000000000..45e5d2612e2c
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-empty.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+module attributes {omp.is_target_device = false} {
+  llvm.func @test() {
+    omp.target {
+      omp.terminator
+    }  loc(#loc4)
+    llvm.return
+  }  loc(#loc3)
+}
+#file = #llvm.di_file<"target.f90" in "">
+#cu = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #file, isOptimized = false,
+ emissionKind = Full>
+#sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
+#sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
+ name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
+#loc1 = loc("target.f90":1:1)
+#loc2 = loc("target.f90":46:3)
+#loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc2])
+
+// CHECK: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
+
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
index f17fd96df00f..bbb125c65645 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-loop-loc.mlir
@@ -42,7 +42,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
         omp.terminator loc(#loc9)
       } loc(#loc9)
       omp.terminator loc(#loc9)
-    } loc(#loc9)
+    } loc(#loc15)
     llvm.return loc(#loc9)
   } loc(#loc14)
   llvm.mlir.global internal @_QFEarray() {addr_space = 0 : i32} : !llvm.array<16384 x i32> {
@@ -60,7 +60,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = true, emissionKind = LineTablesOnly>
 #di_subroutine_type = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #di_subprogram = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #di_compile_unit, scope = #di_file, name = "main", file = #di_file, subprogramFlags = "Definition|Optimized|MainSubprogram", type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit, name = "target", file = #di_file, subprogramFlags = "Definition", type = #di_subroutine_type>
 #loc14 = loc(fused<#di_subprogram>[#loc3])
+#loc15 = loc(fused<#di_subprogram1>[#loc9])
 
 
 // CHECK: call void @__kmpc_distribute_static{{.*}}!dbg
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
new file mode 100644
index 000000000000..492610251769
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-map-link-loc.mlir
@@ -0,0 +1,43 @@
+// RUN: mlir-translate -mlir-to-llvmir %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
+  llvm.mlir.global external @_QMtest_0Esp() {addr_space = 0 : i32, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : i32 {
+    %0 = llvm.mlir.constant(1 : i32) : i32 loc(#loc1)
+    llvm.return %0 : i32 loc(#loc1)
+  } loc(#loc1)
+  llvm.func @_QQmain() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr<5> loc(#loc2)
+    %2 = llvm.addrspacecast %1 : !llvm.ptr<5> to !llvm.ptr loc(#loc2)
+    %6 = llvm.mlir.addressof @_QMtest_0Esp : !llvm.ptr loc(#loc1)
+    %7 = omp.map.info var_ptr(%2 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr loc(#loc3)
+    %8 = omp.map.info var_ptr(%6 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr loc(#loc3)
+    omp.target map_entries(%7 -> %arg0, %8 -> %arg1 : !llvm.ptr, !llvm.ptr) {
+      %16 = llvm.load %arg1 : !llvm.ptr -> i32 loc(#loc5)
+      llvm.store %16, %arg0 : i32, !llvm.ptr loc(#loc5)
+      omp.terminator loc(#loc5)
+    } loc(#loc16)
+    llvm.return loc(#loc6)
+  } loc(#loc15)
+}
+#di_file = #llvm.di_file<"target7.f90" in "">
+#di_null_type = #llvm.di_null_type
+#di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang",
+ isOptimized = false, emissionKind = LineTablesOnly>
+#di_subroutine_type = #llvm.di_subroutine_type<
+  callingConvention = DW_CC_program, types = #di_null_type>
+#di_subprogram = #llvm.di_subprogram<id = distinct[1]<>,
+  compileUnit = #di_compile_unit, scope = #di_file, name = "main",
+  file = #di_file, subprogramFlags = "Definition|MainSubprogram",
+  type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit,
+  name = "target", file = #di_file, subprogramFlags = "Definition",
+  type = #di_subroutine_type>
+#loc1 = loc("test.f90":3:18)
+#loc2 = loc("test.f90":7:7)
+#loc3 = loc("test.f90":9:18)
+#loc5 = loc("test.f90":11:7)
+#loc6 = loc("test.f90":12:7)
+#loc15 = loc(fused<#di_subprogram>[#loc2])
+#loc16 = loc(fused<#di_subprogram1>[#loc5])
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
index eaa88d9dd605..3bd724f42e8c 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-nowait.mlir
@@ -20,7 +20,7 @@ module attributes {omp.is_target_device = false} {
     ^bb3:  // pred: ^bb1
       llvm.store %13, %arg1 : i32, !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc3)
     llvm.return
   } loc(#loc2)
 }
@@ -34,7 +34,10 @@ module attributes {omp.is_target_device = false} {
  types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, name = "main", file=#file,
  subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, name = "target", file=#file,
+ subprogramFlags = "Definition", type = #sp_ty>
 
 #loc1 = loc("test.f90":6:7)
 #loc2 = loc(fused<#sp>[#loc1])
+#loc3 = loc(fused<#sp1>[#loc1])
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
index ea92589bbd03..8f42995af23a 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-1.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui32>>, llvm.target_triple = "amdgcn-amd-amdhsa", omp.is_target_device = true} {
@@ -47,7 +49,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    } loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -57,8 +59,9 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "dyn_ptr", arg: 1, scope: ![[SP]]{{.*}}flags: DIFlagArtificial)
 // CHECK: !DILocalVariable(name: "x", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
index 22db86fd85e2..11a07dfd9a18 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-var-2.mlir
@@ -19,11 +19,13 @@
 #g_var_expr = #llvm.di_global_variable_expression<var = #g_var>
 #sp = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
   name = "test", file = #file, subprogramFlags = "Definition", type = #sp_ty>
-#var_arr = #llvm.di_local_variable<scope = #sp,
+#sp1 = #llvm.di_subprogram<id = distinct[3]<>, compileUnit = #cu, scope = #file,
+  name = "target", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#var_arr = #llvm.di_local_variable<scope = #sp1,
   name = "arr", file = #file, line = 4, type = #array_ty>
-#var_i = #llvm.di_local_variable<scope = #sp,
+#var_i = #llvm.di_local_variable<scope = #sp1,
   name = "i", file = #file, line = 13, type = #int_ty>
-#var_x = #llvm.di_local_variable<scope = #sp,
+#var_x = #llvm.di_local_variable<scope = #sp1,
  name = "x", file = #file, line = 12, type = #real_ty>
 
 module attributes {omp.is_target_device = false} {
@@ -45,7 +47,7 @@ module attributes {omp.is_target_device = false} {
       llvm.intr.dbg.declare #var_arr = %arg1 : !llvm.ptr
       llvm.intr.dbg.declare #var_i = %arg2 : !llvm.ptr
       omp.terminator
-    }
+    }  loc(#loc5)
     llvm.return
   } loc(#loc3)
   llvm.mlir.global internal @_QFEarr() {addr_space = 0 : i32, dbg_exprs = [#g_var_expr]} : !llvm.array<10 x i32> {
@@ -55,8 +57,9 @@ module attributes {omp.is_target_device = false} {
 #loc2 = loc("target.f90":11:7)
 #loc3 = loc(fused<#sp>[#loc2])
 #loc4 = loc(fused<#g_var>[#loc1])
+#loc5 = loc(fused<#sp1>[#loc2])
 
-// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "__omp_offloading{{.*}}test{{.*}})
+// CHECK: ![[SP:[0-9]+]] = distinct !DISubprogram(name: "target"{{.*}})
 // CHECK: !DILocalVariable(name: "x", arg: 1, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "arr", arg: 2, scope: ![[SP]]{{.*}})
 // CHECK: !DILocalVariable(name: "i", arg: 3, scope: ![[SP]]{{.*}})
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
index 9c8344d69dc7..ab687f198b9b 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug.mlir
@@ -10,7 +10,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -21,9 +21,13 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
index 78dc6e18a40a..6cf75af38f91 100644
--- a/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-debug2.mlir
@@ -11,7 +11,7 @@ module attributes {omp.is_target_device = false} {
       %13 = llvm.mlir.constant(1 : i32) : i32
       llvm.store %13, %arg0 : i32, !llvm.ptr loc(#loc2)
       omp.terminator
-    }
+    }  loc(#loc4)
     llvm.return
   } loc(#loc3)
 }
@@ -22,9 +22,13 @@ module attributes {omp.is_target_device = false} {
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_normal>
 #sp = #llvm.di_subprogram<id = distinct[1]<>, compileUnit = #cu, scope = #file,
  name = "_QQmain", file = #file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<id = distinct[2]<>, compileUnit = #cu, scope = #file,
+ name = "__omp_offloading_target", file = #file, subprogramFlags = "Definition",
+ type = #sp_ty>
 #loc1 = loc("target.f90":1:1)
 #loc2 = loc("target.f90":46:3)
 #loc3 = loc(fused<#sp>[#loc1])
+#loc4 = loc(fused<#sp1>[#loc1])
 
-// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_{{.*}}"{{.*}})
+// CHECK-DAG: ![[SP:.*]] = {{.*}}!DISubprogram(name: "__omp_offloading_target"{{.*}})
 // CHECK-DAG: !DILocation(line: 46, column: 3, scope: ![[SP]])
diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
index 971bea206854..e6ea3aaeec65 100644
--- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
@@ -1,15 +1,17 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
 
-llvm.func @_QPopenmp_target_data() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(99 : i32) : i32
-    llvm.store %3, %1 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(99 : i32) : i32
+      llvm.store %3, %1 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
@@ -38,23 +40,25 @@ llvm.func @_QPopenmp_target_data() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
-  %1 = llvm.mlir.constant(1023 : index) : i64
-  %2 = llvm.mlir.constant(0 : index) : i64
-  %3 = llvm.mlir.constant(1024 : index) : i64
-  %4 = llvm.mlir.constant(1 : index) : i64
-  %5 = omp.map.bounds   lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%3 : i64) stride(%4 : i64) start_idx(%4 : i64)
-  %6 = omp.map.info var_ptr(%0 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%5)  -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%6 : !llvm.ptr) {
-    %7 = llvm.mlir.constant(99 : i32) : i32
-    %8 = llvm.mlir.constant(1 : i64) : i64
-    %9 = llvm.mlir.constant(1 : i64) : i64
-    %10 = llvm.mlir.constant(0 : i64) : i64
-    %11 = llvm.getelementptr %0[0, %10] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<1024 x i32>
-    llvm.store %7, %11 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
+    %1 = llvm.mlir.constant(1023 : index) : i64
+    %2 = llvm.mlir.constant(0 : index) : i64
+    %3 = llvm.mlir.constant(1024 : index) : i64
+    %4 = llvm.mlir.constant(1 : index) : i64
+    %5 = omp.map.bounds   lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%3 : i64) stride(%4 : i64) start_idx(%4 : i64)
+    %6 = omp.map.info var_ptr(%0 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%5)  -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%6 : !llvm.ptr) {
+      %7 = llvm.mlir.constant(99 : i32) : i32
+      %8 = llvm.mlir.constant(1 : i64) : i64
+      %9 = llvm.mlir.constant(1 : i64) : i64
+      %10 = llvm.mlir.constant(0 : i64) : i64
+      %11 = llvm.getelementptr %0[0, %10] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<1024 x i32>
+      llvm.store %7, %11 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4096]
@@ -85,50 +89,52 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
 
 // -----
 
-llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
-  %4 = llvm.mlir.constant(1 : i64) : i64
-  %5 = llvm.alloca %4 x i32 {bindc_name = "dvc", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEdvc"} : (i64) -> !llvm.ptr
-  %6 = llvm.mlir.constant(1 : i64) : i64
-  %7 = llvm.alloca %6 x i32 {bindc_name = "i", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEi"} : (i64) -> !llvm.ptr
-  %8 = llvm.mlir.constant(5 : i32) : i32
-  llvm.store %8, %7 : i32, !llvm.ptr
-  %9 = llvm.mlir.constant(2 : i32) : i32
-  llvm.store %9, %5 : i32, !llvm.ptr
-  %10 = llvm.load %7 : !llvm.ptr -> i32
-  %11 = llvm.mlir.constant(10 : i32) : i32
-  %12 = llvm.icmp "slt" %10, %11 : i32
-  %13 = llvm.load %5 : !llvm.ptr -> i32
-  %14 = llvm.mlir.constant(1023 : index) : i64
-  %15 = llvm.mlir.constant(0 : index) : i64
-  %16 = llvm.mlir.constant(1024 : index) : i64
-  %17 = llvm.mlir.constant(1 : index) : i64
-  %18 = omp.map.bounds   lower_bound(%15 : i64) upper_bound(%14 : i64) extent(%16 : i64) stride(%17 : i64) start_idx(%17 : i64)
-  %map1 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(to) capture(ByRef) bounds(%18) -> !llvm.ptr {name = ""}
-  %19 = llvm.mlir.constant(511 : index) : i64
-  %20 = llvm.mlir.constant(0 : index) : i64
-  %21 = llvm.mlir.constant(512 : index) : i64
-  %22 = llvm.mlir.constant(1 : index) : i64
-  %23 = omp.map.bounds   lower_bound(%20 : i64) upper_bound(%19 : i64) extent(%21 : i64) stride(%22 : i64) start_idx(%22 : i64)
-  %map2 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%23) -> !llvm.ptr {name = ""}
-  omp.target_enter_data   if(%12) device(%13 : i32) map_entries(%map1, %map2 : !llvm.ptr, !llvm.ptr)
-  %24 = llvm.load %7 : !llvm.ptr -> i32
-  %25 = llvm.mlir.constant(10 : i32) : i32
-  %26 = llvm.icmp "sgt" %24, %25 : i32
-  %27 = llvm.load %5 : !llvm.ptr -> i32
-  %28 = llvm.mlir.constant(1023 : index) : i64
-  %29 = llvm.mlir.constant(0 : index) : i64
-  %30 = llvm.mlir.constant(1024 : index) : i64
-  %31 = llvm.mlir.constant(1 : index) : i64
-  %32 = omp.map.bounds   lower_bound(%29 : i64) upper_bound(%28 : i64) extent(%30 : i64) stride(%31 : i64) start_idx(%31 : i64)
-  %map3 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%32) -> !llvm.ptr {name = ""}
-  %33 = llvm.mlir.constant(511 : index) : i64
-  %34 = llvm.mlir.constant(0 : index) : i64
-  %35 = llvm.mlir.constant(512 : index) : i64
-  %36 = llvm.mlir.constant(1 : index) : i64
-  %37 = omp.map.bounds   lower_bound(%34 : i64) upper_bound(%33 : i64) extent(%35 : i64) stride(%36 : i64) start_idx(%36 : i64)
-  %map4 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%37) -> !llvm.ptr {name = ""}
-  omp.target_exit_data   if(%26) device(%27 : i32) map_entries(%map3, %map4 : !llvm.ptr, !llvm.ptr)
-  llvm.return
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
+    %4 = llvm.mlir.constant(1 : i64) : i64
+    %5 = llvm.alloca %4 x i32 {bindc_name = "dvc", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEdvc"} : (i64) -> !llvm.ptr
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "i", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEi"} : (i64) -> !llvm.ptr
+    %8 = llvm.mlir.constant(5 : i32) : i32
+    llvm.store %8, %7 : i32, !llvm.ptr
+    %9 = llvm.mlir.constant(2 : i32) : i32
+    llvm.store %9, %5 : i32, !llvm.ptr
+    %10 = llvm.load %7 : !llvm.ptr -> i32
+    %11 = llvm.mlir.constant(10 : i32) : i32
+    %12 = llvm.icmp "slt" %10, %11 : i32
+    %13 = llvm.load %5 : !llvm.ptr -> i32
+    %14 = llvm.mlir.constant(1023 : index) : i64
+    %15 = llvm.mlir.constant(0 : index) : i64
+    %16 = llvm.mlir.constant(1024 : index) : i64
+    %17 = llvm.mlir.constant(1 : index) : i64
+    %18 = omp.map.bounds   lower_bound(%15 : i64) upper_bound(%14 : i64) extent(%16 : i64) stride(%17 : i64) start_idx(%17 : i64)
+    %map1 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(to) capture(ByRef) bounds(%18) -> !llvm.ptr {name = ""}
+    %19 = llvm.mlir.constant(511 : index) : i64
+    %20 = llvm.mlir.constant(0 : index) : i64
+    %21 = llvm.mlir.constant(512 : index) : i64
+    %22 = llvm.mlir.constant(1 : index) : i64
+    %23 = omp.map.bounds   lower_bound(%20 : i64) upper_bound(%19 : i64) extent(%21 : i64) stride(%22 : i64) start_idx(%22 : i64)
+    %map2 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%23) -> !llvm.ptr {name = ""}
+    omp.target_enter_data   if(%12) device(%13 : i32) map_entries(%map1, %map2 : !llvm.ptr, !llvm.ptr)
+    %24 = llvm.load %7 : !llvm.ptr -> i32
+    %25 = llvm.mlir.constant(10 : i32) : i32
+    %26 = llvm.icmp "sgt" %24, %25 : i32
+    %27 = llvm.load %5 : !llvm.ptr -> i32
+    %28 = llvm.mlir.constant(1023 : index) : i64
+    %29 = llvm.mlir.constant(0 : index) : i64
+    %30 = llvm.mlir.constant(1024 : index) : i64
+    %31 = llvm.mlir.constant(1 : index) : i64
+    %32 = omp.map.bounds   lower_bound(%29 : i64) upper_bound(%28 : i64) extent(%30 : i64) stride(%31 : i64) start_idx(%31 : i64)
+    %map3 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%32) -> !llvm.ptr {name = ""}
+    %33 = llvm.mlir.constant(511 : index) : i64
+    %34 = llvm.mlir.constant(0 : index) : i64
+    %35 = llvm.mlir.constant(512 : index) : i64
+    %36 = llvm.mlir.constant(1 : index) : i64
+    %37 = omp.map.bounds   lower_bound(%34 : i64) upper_bound(%33 : i64) extent(%35 : i64) stride(%36 : i64) start_idx(%36 : i64)
+    %map4 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%37) -> !llvm.ptr {name = ""}
+    omp.target_exit_data   if(%26) device(%27 : i32) map_entries(%map3, %map4 : !llvm.ptr, !llvm.ptr)
+    llvm.return
+  }
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 4096, i64 2048]
@@ -205,18 +211,20 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_ptr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map1 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map1 : !llvm.ptr) use_device_ptr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %1, %2 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_ptr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map1 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map1 : !llvm.ptr) use_device_ptr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %1, %2 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
@@ -249,18 +257,20 @@ llvm.func @_QPopenmp_target_use_dev_ptr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %1, %2 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %1, %2 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
@@ -291,17 +301,19 @@ llvm.func @_QPopenmp_target_use_dev_addr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    llvm.store %1, %arg0 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      llvm.store %1, %arg0 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
@@ -331,23 +343,25 @@ llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %1 = llvm.mlir.constant(1 : i64) : i64
-  %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %2 = llvm.mlir.constant(10 : i32) : i32
-    %3 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %2, %3 : i32, !llvm.ptr
-    %4 = llvm.mlir.constant(20 : i32) : i32
-    %5 = llvm.load %b : !llvm.ptr -> !llvm.ptr
-    llvm.store %4, %5 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %1 = llvm.mlir.constant(1 : i64) : i64
+    %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %2 = llvm.mlir.constant(10 : i32) : i32
+      %3 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %2, %3 : i32, !llvm.ptr
+      %4 = llvm.mlir.constant(20 : i32) : i32
+      %5 = llvm.load %b : !llvm.ptr -> !llvm.ptr
+      llvm.store %4, %5 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 8, i64 0]
@@ -387,25 +401,27 @@ llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_both() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %1 = llvm.mlir.constant(1 : i64) : i64
-  %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map1 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map3 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map, %map1 : !llvm.ptr, !llvm.ptr) use_device_addr(%map3 -> %arg0 : !llvm.ptr) use_device_ptr(%map2 -> %arg1 : !llvm.ptr)  {
-    %2 = llvm.mlir.constant(10 : i32) : i32
-    %3 = llvm.load %arg1 : !llvm.ptr -> !llvm.ptr
-    llvm.store %2, %3 : i32, !llvm.ptr
-    %4 = llvm.mlir.constant(20 : i32) : i32
-    %5 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %4, %5 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_both() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %1 = llvm.mlir.constant(1 : i64) : i64
+    %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map1 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map3 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map, %map1 : !llvm.ptr, !llvm.ptr) use_device_addr(%map3 -> %arg0 : !llvm.ptr) use_device_ptr(%map2 -> %arg1 : !llvm.ptr)  {
+      %2 = llvm.mlir.constant(10 : i32) : i32
+      %3 = llvm.load %arg1 : !llvm.ptr -> !llvm.ptr
+      llvm.store %2, %3 : i32, !llvm.ptr
+      %4 = llvm.mlir.constant(20 : i32) : i32
+      %5 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %4, %5 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 8, i64 8]
@@ -448,19 +464,21 @@ llvm.func @_QPopenmp_target_use_dev_both() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_update() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(99 : i32) : i32
-    llvm.store %3, %1 : i32, !llvm.ptr
-    omp.terminator
-  }
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_update() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(99 : i32) : i32
+      llvm.store %3, %1 : i32, !llvm.ptr
+      omp.terminator
+    }
 
-  omp.target_update map_entries(%2 : !llvm.ptr)
+    omp.target_update map_entries(%2 : !llvm.ptr)
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK-LABEL: define void @_QPopenmp_target_data_update
@@ -488,26 +506,28 @@ llvm.func @_QPopenmp_target_data_update() {
 
 // -----
 
-omp.declare_mapper @_QQFmy_testmy_mapper : !llvm.struct<"_QFmy_testTmy_type", (i32)> {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(0 : i32) : i32
-  %1 = llvm.getelementptr %arg0[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "var%data"}
-  %3 = omp.map.info var_ptr(%arg0 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) members(%2 : [0] : !llvm.ptr) -> !llvm.ptr {name = "var", partial_map = true}
-  omp.declare_mapper.info map_entries(%3, %2 : !llvm.ptr, !llvm.ptr)
-}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  omp.declare_mapper @_QQFmy_testmy_mapper : !llvm.struct<"_QFmy_testTmy_type", (i32)> {
+  ^bb0(%arg0: !llvm.ptr):
+    %0 = llvm.mlir.constant(0 : i32) : i32
+    %1 = llvm.getelementptr %arg0[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "var%data"}
+    %3 = omp.map.info var_ptr(%arg0 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) members(%2 : [0] : !llvm.ptr) -> !llvm.ptr {name = "var", partial_map = true}
+    omp.declare_mapper.info map_entries(%3, %2 : !llvm.ptr, !llvm.ptr)
+  }
 
-llvm.func @_QPopenmp_target_data_mapper() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x !llvm.struct<"_QFmy_testTmy_type", (i32)> {bindc_name = "a"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) mapper(@_QQFmy_testmy_mapper) -> !llvm.ptr {name = "a"}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(10 : i32) : i32
-    %4 = llvm.getelementptr %1[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
-    llvm.store %3, %4 : i32, !llvm.ptr
-    omp.terminator
+  llvm.func @_QPopenmp_target_data_mapper() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x !llvm.struct<"_QFmy_testTmy_type", (i32)> {bindc_name = "a"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) mapper(@_QQFmy_testmy_mapper) -> !llvm.ptr {name = "a"}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(10 : i32) : i32
+      %4 = llvm.getelementptr %1[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
+      llvm.store %3, %4 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
index 3c45f1f1c76f..b18338ea35cc 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm-debug.mlir
@@ -6,8 +6,10 @@
 #cu = #llvm.di_compile_unit<id = distinct[0]<>, sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang", isOptimized = false, emissionKind = Full>
 #sp_ty = #llvm.di_subroutine_type<callingConvention = DW_CC_program, types = #di_null_type>
 #sp = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "test", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
+#sp1 = #llvm.di_subprogram<compileUnit = #cu, scope = #di_file, name = "kernel", file = #di_file, subprogramFlags = "Definition", type = #sp_ty>
 #int_ty = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "integer", sizeInBits = 32, encoding = DW_ATE_signed>
 #var_x = #llvm.di_local_variable<scope = #sp, name = "x", file = #di_file, type = #int_ty>
+#var_x1 = #llvm.di_local_variable<scope = #sp1, name = "x", file = #di_file, type = #int_ty>
 module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, f80 = dense<128> : vector<2xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, "dlti.endianness" = "little", "dlti.stack_alignment" = 128 : i64, "dlti.mangling_mode" = "e">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "x86-64", llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 21.0.0 (/home/haqadeer/work/src/aomp-llvm-project/flang 793f9220ab32f92fc3b253efec2e332c18090e53)", llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_gpu = false, omp.is_target_device = false, omp.requires = #omp<clause_requires none>, omp.target_triples = ["amdgcn-amd-amdhsa"], omp.version = #omp.version<version = 52>} {
   llvm.func @_QQmain() attributes {fir.bindc_name = "test", frame_pointer = #llvm.framePointerKind<all>, target_cpu = "x86-64"} {
     %0 = llvm.mlir.constant(1 : i64) : i64
@@ -16,7 +18,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
     %5 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "x"}
     omp.target map_entries(%5 -> %arg0 : !llvm.ptr) {
       %6 = llvm.mlir.constant(1 : i32) : i32
-      llvm.intr.dbg.declare #var_x = %arg0 : !llvm.ptr loc(#loc2)
+      llvm.intr.dbg.declare #var_x1 = %arg0 : !llvm.ptr loc(#loc3)
       omp.parallel {
         %7 = llvm.load %arg0 : !llvm.ptr -> i32
         %8 = llvm.add %7, %6 : i32
@@ -24,12 +26,14 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<i32 = dense<32> : vector<2xi64>,
         omp.terminator
       }
       omp.terminator
-    }
+    } loc(#loc4)
     llvm.return
   } loc(#loc10)
 }
 #loc1 = loc("target.f90":1:7)
 #loc2 = loc("target.f90":3:18)
+#loc3 = loc("target.f90":6:18)
+#loc4 = loc(fused<#sp1>[#loc3])
 #loc10 = loc(fused<#sp>[#loc1])
 
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
index 1a0d2cdb8138..424e948fac75 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-llvm.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
-
+// XFAIL: *
 // The aim of the test is to check the LLVM IR codegen for the device
 // for omp target parallel construct
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
index 3bc4be1fe0cc..5660f2698d06 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
-
+// XFAIL: *
 // The aim of the test is to check the GPU LLVM IR codegen
 // for nested omp do loop inside omp target region
 
diff --git a/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir b/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
index dba8c553aaca..f5c620a8942d 100644
--- a/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
@@ -1,13 +1,15 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s 2>&1 | FileCheck %s
 
-llvm.func @_QPopenmp_target_data_enter() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_enter() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
+    omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_enter() {
@@ -32,14 +34,16 @@ llvm.func @_QPopenmp_target_data_enter() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_update() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_update() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_update map_entries(%2 : !llvm.ptr) nowait
+    omp.target_update map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_update() {
@@ -64,14 +68,16 @@ llvm.func @_QPopenmp_target_data_update() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_exit() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_exit() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
+    omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_exit() {
diff --git a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
index 717a77e61b9a..53c9b4f55964 100644
--- a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
@@ -3,7 +3,7 @@
 // This tests checks that a target op inside a data op
 // We are only interested in ensuring that the -mlir-to-llmvir pass doesn't crash.
 // CHECK: {{.*}} = add i32 {{.*}}, 1
-module attributes { } {
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
   llvm.mlir.global weak_odr hidden local_unnamed_addr constant @__oclc_ABI_version(400 : i32) {addr_space = 4 : i32} : i32
   llvm.func @_QQmain() attributes {fir.bindc_name = "main", omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>} {
     %0 = llvm.mlir.constant(99 : index) : i64
diff --git a/mlir/test/Target/LLVMIR/openmp-nested-task-target-parallel.mlir b/mlir/test/Target/LLVMIR/openmp-nested-task-target-parallel.mlir
new file mode 100644
index 000000000000..1589778e0627
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-nested-task-target-parallel.mlir
@@ -0,0 +1,62 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+// This tests the fix for https://github.com/llvm/llvm-project/issues/138102
+// We are only interested in ensuring that the -mlir-to-llvmir pass doesn't crash
+
+// CHECK-LABEL: define internal void @_QQmain..omp_par
+
+omp.private {type = private} @_QFEi_private_i32 : i32
+omp.private {type = firstprivate} @_QFEc_firstprivate_i32 : i32 copy {
+^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr):
+%0 = llvm.load %arg0 : !llvm.ptr -> i32
+llvm.store %0, %arg1 : i32, !llvm.ptr
+omp.yield(%arg1 : !llvm.ptr)
+}
+llvm.func @_QQmain() {
+%0 = llvm.mlir.constant(1 : i64) : i64
+%1 = llvm.alloca %0 x i32 {bindc_name = "i"} : (i64) -> !llvm.ptr
+%2 = llvm.mlir.constant(1 : i64) : i64
+%3 = llvm.alloca %2 x i32 {bindc_name = "c"} : (i64) -> !llvm.ptr
+%4 = llvm.mlir.constant(10 : index) : i64
+%5 = llvm.mlir.constant(0 : index) : i64
+%6 = llvm.mlir.constant(10000 : index) : i64
+%7 = llvm.mlir.constant(1 : index) : i64
+%8 = llvm.mlir.constant(1 : i64) : i64
+%9 = llvm.mlir.addressof @_QFECchunksz : !llvm.ptr
+%10 = llvm.mlir.constant(1 : i64) : i64
+%11 = llvm.trunc %7 : i64 to i32
+llvm.br ^bb1(%11, %4 : i32, i64)
+^bb1(%12: i32, %13: i64):  // 2 preds: ^bb0, ^bb2
+%14 = llvm.icmp "sgt" %13, %5 : i64
+llvm.store %12, %3 : i32, !llvm.ptr
+omp.task private(@_QFEc_firstprivate_i32 %3 -> %arg0 : !llvm.ptr) {
+  %19 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "i"}
+  %20 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "c"}
+  %21 = omp.map.info var_ptr(%9 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "chunksz"}
+  omp.target map_entries(%19 -> %arg1, %20 -> %arg2, %21 -> %arg3 : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+    %22 = llvm.mlir.constant(9999 : i32) : i32
+    %23 = llvm.mlir.constant(1 : i32) : i32
+    omp.parallel {
+      %24 = llvm.load %arg2 : !llvm.ptr -> i32
+      %25 = llvm.add %24, %22 : i32
+      omp.wsloop private(@_QFEi_private_i32 %arg1 -> %arg4 : !llvm.ptr) {
+        omp.loop_nest (%arg5) : i32 = (%24) to (%25) inclusive step (%23) {
+          llvm.store %arg5, %arg4 : i32, !llvm.ptr
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  omp.terminator
+}
+llvm.return
+}
+llvm.mlir.global internal constant @_QFECchunksz() {addr_space = 0 : i32} : i32 {
+%0 = llvm.mlir.constant(10000 : i32) : i32
+llvm.return %0 : i32
+}
+llvm.mlir.global internal constant @_QFECn() {addr_space = 0 : i32} : i32 {
+%0 = llvm.mlir.constant(100000 : i32) : i32
+llvm.return %0 : i32
+}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index d6baa4db0047..7fef486fd511 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file -verify-diagnostics %s
+// XFAIL: *
 
 llvm.func @atomic_hint(%v : !llvm.ptr, %x : !llvm.ptr, %expr : i32) {
   // expected-warning@below {{hint clause discarded}}
@@ -158,7 +159,8 @@ atomic {
   omp.yield
 }
 llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-warning@below {{ignored clause: reduction in omp.simd operation}}
+  // expected-error@below {{not yet implemented: Unhandled clause reduction in omp.simd operation}}
+  // expected-error@below {{LLVM Translation failed for operation: omp.simd}}
   omp.simd reduction(@add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.yield
diff --git a/mlir/test/Target/SPIRV/image-ops.mlir b/mlir/test/Target/SPIRV/image-ops.mlir
index b8d19f0f9a7d..1ea66af6404d 100644
--- a/mlir/test/Target/SPIRV/image-ops.mlir
+++ b/mlir/test/Target/SPIRV/image-ops.mlir
@@ -42,7 +42,7 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, ImageQuery, Link
 
 // -----
 
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat, Linkage], []> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, StorageImageWriteWithoutFormat, StorageImageReadWithoutFormat, Linkage], []> {
   spirv.func @image_read_write(%arg0 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, %arg1 : vector<2xsi32>) "None" {
     // CHECK: spirv.ImageRead {{%.*}}, {{%.*}} : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
     %0 = spirv.ImageRead %arg0, %arg1 : !spirv.image<f32, Dim2D, NoDepth, NonArrayed, SingleSampled, NoSampler, Unknown>, vector<2xsi32> -> vector<4xf32>
diff --git a/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir b/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
index 153ff4793797..77949908e888 100644
--- a/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
+++ b/mlir/test/Target/SPIRV/khr-cooperative-matrix-ops.mlir
@@ -30,6 +30,15 @@ spirv.module Logical GLSL450 requires
     spirv.Return
   }
 
+  // CHECK-LABEL: @cooperative_matrix_load_3
+  spirv.func @cooperative_matrix_load_3(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32) "None" {
+    // CHECK:      {{%.+}} = spirv.KHR.CooperativeMatrixLoad {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16
+    // CHECK-SAME:   : !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+    %0 = spirv.KHR.CooperativeMatrixLoad %ptr, %stride, <RowMajor>, <Aligned>, 16 :
+      !spirv.ptr<i32, StorageBuffer>, i32 -> !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>
+    spirv.Return
+  }
+
   // CHECK-LABEL: @cooperative_matrix_store_1
   spirv.func @cooperative_matrix_store_1(%ptr : !spirv.ptr<i32, StorageBuffer>, %stride : i32,
                                          %m : !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>) "None" {
@@ -38,6 +47,11 @@ spirv.module Logical GLSL450 requires
     spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor> :
       !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
 
+    // CHECK:      spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, <RowMajor>, <Aligned>, 16
+    // CHECK-SAME:   : !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
+    spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Aligned>, 16 :
+      !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
+
     // CHECK-NEXT:  spirv.KHR.CooperativeMatrixStore {{%.*}}, {{%.*}}, <RowMajor>, <Volatile|Nontemporal>
     spirv.KHR.CooperativeMatrixStore %ptr, %m, %stride, <RowMajor>, <Volatile|Nontemporal> :
       !spirv.ptr<i32, StorageBuffer>, !spirv.coopmatrix<16x8xi32, Workgroup, MatrixA>, i32
diff --git a/mlir/test/Target/SPIRV/tensorARM.mlir b/mlir/test/Target/SPIRV/tensorARM.mlir
new file mode 100644
index 000000000000..75b648ebfd00
--- /dev/null
+++ b/mlir/test/Target/SPIRV/tensorARM.mlir
@@ -0,0 +1,66 @@
+// RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip %s | FileCheck %s
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, TensorsARM], [SPV_ARM_tensors]> {
+  // CHECK: spirv.func @shaped_int_arm_tensor(%arg0: !spirv.arm.tensor<2xi32>) "None" {
+  spirv.func @shaped_int_arm_tensor(%arg0 : !spirv.arm.tensor<2xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @shaped_rank2_int_arm_tensor(%arg0: !spirv.arm.tensor<2x3xi32>) "None" {
+  spirv.func @shaped_rank2_int_arm_tensor(%arg0 : !spirv.arm.tensor<2x3xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @ui64_arm_tensor_const() -> !spirv.arm.tensor<3xi64> "None" {
+  spirv.func @ui64_arm_tensor_const() -> !spirv.arm.tensor<3xui64> "None" {
+    // CHECK: spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xi64>
+    %0 = spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xui64>
+
+    spirv.ReturnValue %0: !spirv.arm.tensor<3xui64>
+  }
+
+// -----
+
+  // CHECK: spirv.func @si32_arm_tensor_const() -> !spirv.arm.tensor<3xsi32> "None" {
+  spirv.func @si32_arm_tensor_const() -> !spirv.arm.tensor<3xsi32> "None" {
+    // CHECK: spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xsi32>
+    %0 = spirv.Constant dense<[5, 6, 7]> : !spirv.arm.tensor<3xsi32>
+
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xsi32>
+  }
+
+// -----
+
+  // CHECK: spirv.func @float_arm_tensor_const() -> !spirv.arm.tensor<3xf32> "None" {
+  spirv.func @float_arm_tensor_const() -> !spirv.arm.tensor<3xf32> "None" {
+    // CHECK: spirv.Constant dense<[3.000000e+00, 4.000000e+00, 5.000000e+00]> : !spirv.arm.tensor<3xf32>
+    %0 = spirv.Constant dense<[3., 4., 5.]> : !spirv.arm.tensor<3xf32>
+
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xf32>
+  }
+
+// -----
+
+  // CHECK: spirv.func @unranked_int_arm_tensor(%arg0: !spirv.arm.tensor<*xi32>) "None" {
+  spirv.func @unranked_int_arm_tensor(%arg0 : !spirv.arm.tensor<*xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @unshaped_int_arm_tensor(%arg0: !spirv.arm.tensor<?xi32>) "None" {
+  spirv.func @unshaped_int_arm_tensor(%arg0 : !spirv.arm.tensor<?xi32>) "None" {
+    spirv.Return
+  }
+
+// -----
+
+  // CHECK: spirv.func @unshaped_int_arm_tensor_2(%arg0: !spirv.arm.tensor<?x?xi32>) "None" {
+  spirv.func @unshaped_int_arm_tensor_2(%arg0 : !spirv.arm.tensor<?x?xi32>) "None" {
+    spirv.Return
+  }
+}
diff --git a/mlir/test/Transforms/make-composed-folded-affine-apply.mlir b/mlir/test/Transforms/make-composed-folded-affine-apply.mlir
new file mode 100644
index 000000000000..138426827fa1
--- /dev/null
+++ b/mlir/test/Transforms/make-composed-folded-affine-apply.mlir
@@ -0,0 +1,77 @@
+// RUN: mlir-opt --transform-interpreter %s | FileCheck %s
+
+#map = affine_map<()[s0, s1] -> (s0, s1, 128)>
+#map1 = affine_map<()[s0, s1] -> (s0 ceildiv 128 + s0 ceildiv s1)>
+#map2 = affine_map<()[s0, s1, s2] -> (s0, s1 + s2)>
+#map3 = affine_map<()[s0, s1, s2, s3] -> (3 * (s0 ceildiv s3) + s0 ceildiv (s1 + s2))>
+#map4 = affine_map<()[s0, s1] -> (s1)>
+#map5 = affine_map<()[s0, s1] -> (s0 ceildiv s1)>
+#map6 = affine_map<()[s0, s1] -> (s0, s1, -128)>
+// CHECK-DAG: #[[MAP1:.*]] = affine_map<()[s0, s1] -> (s0 ceildiv 128 + s0 ceildiv s1)>
+// CHECK-DAG: #[[MAP5:.*]] = affine_map<()[s0, s1] -> (s0 ceildiv s1)>
+
+// These test checks the `affine::makeComposedFoldedAffineApply` function when
+// `composeAffineMin == true`.
+
+// Check the apply gets simplified.
+// CHECK: @apply_simplification
+func.func @apply_simplification_1() -> index {
+  %0 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %1 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %2 = affine.min #map()[%0, %1]
+  // CHECK-NOT: affine.apply
+  // CHECK: arith.constant 2 : index
+  %3 = affine.apply #map1()[%2, %1]
+  return %3 : index
+}
+
+// Check the simplification can match non-trivial affine expressions like s1 + s2.
+func.func @apply_simplification_2() -> index {
+  %0 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %1 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %2 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %3 = affine.min #map2()[%0, %1, %2]
+  // CHECK-NOT: affine.apply
+  // CHECK: arith.constant 4 : index
+  %4 = affine.apply #map3()[%3, %1, %2, %0]
+  return %4 : index
+}
+
+// Check there's no simplification.
+// The apply cannot be simplified because `s1 = %0` doesn't appear in the input min.
+// CHECK: @no_simplification_0
+func.func @no_simplification_0() -> index {
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 64 : index, min = 16 : index}
+  // CHECK: %[[V2:.*]] = affine.min #{{.*}}()[%[[V0]], %[[V1]]]
+  // CHECK: %[[V3:.*]] = affine.apply #[[MAP5]]()[%[[V2]], %[[V0]]]
+  // CHECK: return %[[V3]] : index
+  %0 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %1 = test.value_with_bounds {max = 64 : index, min = 16 : index}
+  %2 = affine.min #map4()[%0, %1]
+  %3 = affine.apply #map5()[%2, %0]
+  return %3 : index
+}
+
+// The apply cannot be simplified because the min cannot be proven to be greater than 0.
+// CHECK: @no_simplification_1
+func.func @no_simplification_1() -> index {
+  // CHECK: %[[V0:.*]] = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  // CHECK: %[[V1:.*]] = test.value_with_bounds {max = 64 : index, min = 16 : index}
+  // CHECK: %[[V2:.*]] = affine.min #{{.*}}()[%[[V0]], %[[V1]]]
+  // CHECK: %[[V3:.*]] = affine.apply #[[MAP1]]()[%[[V2]], %[[V1]]]
+  // CHECK: return %[[V3]] : index
+  %0 = test.value_with_bounds {max = 64 : index, min = 32 : index}
+  %1 = test.value_with_bounds {max = 64 : index, min = 16 : index}
+  %2 = affine.min #map6()[%0, %1]
+  %3 = affine.apply #map1()[%2, %1]
+  return %3 : index
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["affine.apply"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.test.make_composed_folded_affine_apply %0 : (!transform.any_op) -> !transform.any_op
+    transform.yield 
+  }
+}
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index 79518b04e715..68c863cff69b 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -1,10 +1,18 @@
-// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics %s | FileCheck %s
-
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics -profile-actions-to=- %s | FileCheck %s
+
+//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "B"
+//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "E"
+// Note: Listener notifications appear after the pattern application because
+// the conversion driver sends all notifications at the end of the conversion
+// in bulk.
 //      CHECK: notifyOperationInserted: test.legal_op_a, was unlinked
 // CHECK-NEXT: notifyOperationReplaced: test.illegal_op_a
 // CHECK-NEXT: notifyOperationModified: func.return
 // CHECK-NEXT: notifyOperationErased: test.illegal_op_a
-
+//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "E"
+//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "E"
 // CHECK-LABEL: verifyDirectPattern
 func.func @verifyDirectPattern() -> i32 {
   // CHECK-NEXT:  "test.legal_op_a"() <{status = "Success"}
@@ -434,7 +442,7 @@ func.func @test_properties_rollback() {
   // CHECK: test.with_properties a = 32,
   // expected-remark @below{{op 'test.with_properties' is not legalizable}}
   test.with_properties
-      a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4]
+      a = 32, b = "foo", c = "bar", flag = true, array = [1, 2, 3, 4], array32 = [5, 6]
       {modify_inplace}
   "test.return"() : () -> ()
 }
diff --git a/mlir/test/Transforms/test-symbol-dce.mlir b/mlir/test/Transforms/test-symbol-dce.mlir
index 7bd784928e6f..90fe93f806d6 100644
--- a/mlir/test/Transforms/test-symbol-dce.mlir
+++ b/mlir/test/Transforms/test-symbol-dce.mlir
@@ -98,3 +98,38 @@ module {
   // CHECK: "live.user"() {uses = [@unknown_symbol]} : () -> ()
   "live.user"() {uses = [@unknown_symbol]} : () -> ()
 }
+
+// -----
+
+// Check that we don't DCE nested symbols if they are nested inside region
+// without SymbolTable.
+
+// CHECK-LABEL: module attributes {test.nested_nosymboltable_region}
+module attributes { test.nested_nosymboltable_region } {
+  "test.one_region_op"() ({
+    "test.symbol_scope"() ({
+      // CHECK: func nested @nfunction
+      func.func nested @nfunction() {
+        return
+      }
+      func.call @nfunction() : () -> ()
+      "test.finish"() : () -> ()
+    }) : () -> ()
+    "test.finish"() : () -> ()
+  }) : () -> ()
+}
+
+// -----
+
+// CHECK-LABEL: module attributes {test.nested_nosymboltable_region_notcalled}
+// CHECK-NOT: @nested
+// CHECK: @main
+module attributes { test.nested_nosymboltable_region_notcalled } {
+  "test.one_region_op"() ({
+    module {
+      func.func nested @nested() { return }
+      func.func @main() { return }
+    }
+    "test.finish"() : () -> ()
+  }) : () -> ()
+}
diff --git a/mlir/test/lib/Conversion/FuncToLLVM/TestConvertCallOp.cpp b/mlir/test/lib/Conversion/FuncToLLVM/TestConvertCallOp.cpp
index f878a262512e..3b29867cc326 100644
--- a/mlir/test/lib/Conversion/FuncToLLVM/TestConvertCallOp.cpp
+++ b/mlir/test/lib/Conversion/FuncToLLVM/TestConvertCallOp.cpp
@@ -28,7 +28,7 @@ class TestTypeProducerOpConverter
   LogicalResult
   matchAndRewrite(test::TestTypeProducerOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.replaceOpWithNewOp<LLVM::ZeroOp>(op, getVoidPtrType());
+    rewriter.replaceOpWithNewOp<LLVM::ZeroOp>(op, getPtrType());
     return success();
   }
 };
diff --git a/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp b/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
index 738648b8ccdc..727c84cc9c07 100644
--- a/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Dialect/Linalg/TestLinalgTransforms.cpp
@@ -130,6 +130,11 @@ struct TestLinalgTransforms
       *this, "test-fold-into-pack-and-unpack",
       llvm::cl::desc("Test folding ops into linalg.pack and linalg.unpack"),
       llvm::cl::init(false)};
+  Option<bool> testFoldIntoPackAndUnpackWithControlFn{
+      *this, "test-fold-into-pack-and-unpack-control",
+      llvm::cl::desc(
+          "Test controlling folding ops into linalg.pack and linalg.unpack"),
+      llvm::cl::init(false)};
   Option<bool> testSimplifyPackUnpackPatterns{
       *this, "test-simplify-pack-unpack-patterns",
       llvm::cl::desc("Test patterns to simplify linalg.pack and linalg.unpack"),
@@ -211,8 +216,8 @@ static void applyEraseUnnecessaryInputs(func::FuncOp funcOp) {
 
 static void applyWinogradConv2D(func::FuncOp funcOp) {
   RewritePatternSet patterns(funcOp.getContext());
-  populateWinogradConv2DPatterns(patterns, /*m=*/4, /*r=*/3);
-  populateWinogradConv2DPatterns(patterns, /*m=*/2, /*r=*/5);
+  populateWinogradConv2DPatterns(patterns, WinogradConv2DFmr::F_4_3);
+  populateWinogradConv2DPatterns(patterns, WinogradConv2DFmr::F_2_5);
   (void)applyPatternsGreedily(funcOp, std::move(patterns));
 }
 
@@ -222,9 +227,11 @@ static void applyDecomposeWinogradOps(func::FuncOp funcOp) {
   (void)applyPatternsGreedily(funcOp, std::move(patterns));
 }
 
-static void applyFoldIntoPackAndUnpackPatterns(Operation *rootOp) {
+static void applyFoldIntoPackAndUnpackPatterns(
+    Operation *rootOp,
+    linalg::ControlFoldIntoPackUnpackFn controlFn = nullptr) {
   RewritePatternSet patterns(rootOp->getContext());
-  linalg::populateFoldIntoPackAndUnpackPatterns(patterns);
+  linalg::populateFoldIntoPackAndUnpackPatterns(patterns, controlFn);
   (void)applyPatternsGreedily(rootOp, std::move(patterns));
 }
 
@@ -263,6 +270,19 @@ void TestLinalgTransforms::runOnOperation() {
   Operation *rootOp = getOperation();
   if (testFoldIntoPackAndUnpack)
     applyFoldIntoPackAndUnpackPatterns(rootOp);
+  if (testFoldIntoPackAndUnpackWithControlFn) {
+    linalg::ControlFoldIntoPackUnpackFn controlFn = [](OpOperand *opOperand) {
+      Operation *producer = opOperand->get().getDefiningOp();
+      Operation *consumer = opOperand->getOwner();
+      // If we have a pack/unpack consumer and a producer that has multiple
+      // uses, do not apply the folding patterns.
+      if (isa<linalg::PackOp, linalg::UnPackOp>(consumer) &&
+          isa<TilingInterface>(producer) && !producer->hasOneUse())
+        return false;
+      return true;
+    };
+    applyFoldIntoPackAndUnpackPatterns(rootOp, controlFn);
+  }
   if (testSimplifyPackUnpackPatterns)
     applySimplifyPackUnpackPatterns(rootOp);
 }
diff --git a/mlir/test/lib/Dialect/Test/TestEnumDefs.td b/mlir/test/lib/Dialect/Test/TestEnumDefs.td
index 5b785a600aad..10e424a0f252 100644
--- a/mlir/test/lib/Dialect/Test/TestEnumDefs.td
+++ b/mlir/test/lib/Dialect/Test/TestEnumDefs.td
@@ -17,9 +17,13 @@ include "mlir/IR/EnumAttr.td"
 
 def I32Case5:  I32EnumAttrCase<"case5", 5>;
 def I32Case10: I32EnumAttrCase<"case10", 10>;
+def I32CaseSignedMaxPlusOne
+    : I32EnumAttrCase<"caseSignedMaxPlusOne", 2147483648>;
+def I32CaseUnsignedMax : I32EnumAttrCase<"caseUnsignedMax", 4294967295>;
 
-def SomeI32Enum: I32EnumAttr<
-  "SomeI32Enum", "", [I32Case5, I32Case10]>;
+def SomeI32Enum : I32EnumAttr<"SomeI32Enum", "",
+                              [I32Case5, I32Case10, I32CaseSignedMaxPlusOne,
+                               I32CaseUnsignedMax]>;
 
 def I64Case5:  I64EnumAttrCase<"case5", 5>;
 def I64Case10: I64EnumAttrCase<"case10", 10>;
diff --git a/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp b/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
index 9ed1b3a47be3..70bab21b8325 100644
--- a/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
+++ b/mlir/test/lib/Dialect/Test/TestFormatUtils.cpp
@@ -381,3 +381,26 @@ void test::printAttrElideType(AsmPrinter &printer, Operation *op, TypeAttr type,
                               Attribute attr) {
   printer.printAttributeWithoutType(attr);
 }
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummyRegionRef
+//===----------------------------------------------------------------------===//
+
+ParseResult test::parseDummyRegionRef(OpAsmParser &parser, Region &region) {
+  return success();
+}
+
+void test::printDummyRegionRef(OpAsmPrinter &printer, Operation *op,
+                               Region &region) { /* do nothing */ }
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummySuccessorRef
+//===----------------------------------------------------------------------===//
+
+ParseResult test::parseDummySuccessorRef(OpAsmParser &parser,
+                                         Block *successor) {
+  return success();
+}
+
+void test::printDummySuccessorRef(OpAsmPrinter &printer, Operation *op,
+                                  Block *successor) { /* do nothing */ }
diff --git a/mlir/test/lib/Dialect/Test/TestFormatUtils.h b/mlir/test/lib/Dialect/Test/TestFormatUtils.h
index 6d4df7d82ffa..e914f9a27b79 100644
--- a/mlir/test/lib/Dialect/Test/TestFormatUtils.h
+++ b/mlir/test/lib/Dialect/Test/TestFormatUtils.h
@@ -207,6 +207,24 @@ mlir::ParseResult parseAttrElideType(mlir::AsmParser &parser,
 void printAttrElideType(mlir::AsmPrinter &printer, mlir::Operation *op,
                         mlir::TypeAttr type, mlir::Attribute attr);
 
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummyRegionRef
+//===----------------------------------------------------------------------===//
+
+mlir::ParseResult parseDummyRegionRef(mlir::OpAsmParser &parser,
+                                      mlir::Region &region);
+void printDummyRegionRef(mlir::OpAsmPrinter &printer, mlir::Operation *op,
+                         mlir::Region &region);
+
+//===----------------------------------------------------------------------===//
+// CustomDirectiveDummySuccessorRef
+//===----------------------------------------------------------------------===//
+
+mlir::ParseResult parseDummySuccessorRef(mlir::OpAsmParser &parser,
+                                         mlir::Block *successor);
+void printDummySuccessorRef(mlir::OpAsmPrinter &printer, mlir::Operation *op,
+                            mlir::Block *successor);
+
 } // end namespace test
 
 #endif // MLIR_TESTFORMATUTILS_H
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index 6c1a5d344153..3ab4ef268097 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -1420,3 +1420,37 @@ ::mlir::LogicalResult test::TestDummyTensorOp::bufferize(
 
   return mlir::success();
 }
+
+::mlir::LogicalResult test::TestCreateTensorOp::bufferize(
+    ::mlir::RewriterBase &rewriter,
+    const ::mlir::bufferization::BufferizationOptions &options,
+    ::mlir::bufferization::BufferizationState &state) {
+  // Note: mlir::bufferization::getBufferType() would internally call
+  // TestCreateTensorOp::getBufferType()
+  const auto bufferizedOutType =
+      mlir::bufferization::getBufferType(getOutput(), options, state);
+  if (mlir::failed(bufferizedOutType))
+    return failure();
+
+  // replace op with memref analogy
+  auto createMemrefOp =
+      rewriter.create<test::TestCreateMemrefOp>(getLoc(), *bufferizedOutType);
+
+  mlir::bufferization::replaceOpWithBufferizedValues(
+      rewriter, getOperation(), createMemrefOp.getResult());
+
+  return mlir::success();
+}
+
+mlir::FailureOr<mlir::bufferization::BufferLikeType>
+test::TestCreateTensorOp::getBufferType(
+    mlir::Value value, const mlir::bufferization::BufferizationOptions &,
+    const mlir::bufferization::BufferizationState &,
+    llvm::SmallVector<::mlir::Value> &) {
+  const auto type = dyn_cast<test::TestTensorType>(value.getType());
+  if (type == nullptr)
+    return failure();
+
+  return cast<mlir::bufferization::BufferLikeType>(test::TestMemrefType::get(
+      getContext(), type.getShape(), type.getElementType(), nullptr));
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 5fc7c33f4fb2..535f5e9b4a15 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2737,12 +2737,14 @@ def TestLinalgConvOp :
     bool hasIndexSemantics() { return false; }
 
     static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
-                              mlir::ArrayRef<mlir::NamedAttribute> attrs) {
+                              mlir::ArrayRef<mlir::NamedAttribute> attrs,
+                              llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
       b.create<mlir::linalg::YieldOp>(block.getArguments().back());
     }
 
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-                              mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              llvm::function_ref<mlir::InFlightDiagnostic()>)>
     getRegionBuilder() {
       return &regionBuilder;
     }
@@ -2798,12 +2800,14 @@ def TestLinalgFillOp :
     bool hasIndexSemantics() { return false; }
 
     static void regionBuilder(mlir::ImplicitLocOpBuilder &b, mlir::Block &block,
-                              mlir::ArrayRef<mlir::NamedAttribute> attrs) {
+                              mlir::ArrayRef<mlir::NamedAttribute> attrs,
+                              llvm::function_ref<mlir::InFlightDiagnostic()> emitError) {
       b.create<mlir::linalg::YieldOp>(block.getArguments().back());
     }
 
     static std::function<void(mlir::ImplicitLocOpBuilder &, mlir::Block &,
-                              mlir::ArrayRef<mlir::NamedAttribute>)>
+                              mlir::ArrayRef<mlir::NamedAttribute>,
+                              llvm::function_ref<mlir::InFlightDiagnostic()>)>
     getRegionBuilder() {
       return &regionBuilder;
     }
@@ -3175,13 +3179,15 @@ def TestOpWithProperties : TEST_Op<"with_properties"> {
     `b` `=` $b `,`
     `c` `=` $c `,`
     `flag` `=` $flag `,`
-    `array` `=` $array attr-dict}];
+    `array` `=` $array `,`
+    `array32` `=` $array32 attr-dict}];
   let arguments = (ins
     I64Prop:$a,
     StrAttr:$b, // Attributes can directly be used here.
     StringProp:$c,
     BoolProp:$flag,
-    IntArrayProp<I64Prop>:$array // example of an array
+    IntArrayProp<I64Prop>:$array, // Example of an array.
+    IntArrayProp<I32Prop>:$array32 // Example of an array.
   );
 }
 
@@ -3611,29 +3617,16 @@ def TestAllocWithMultipleResults : TEST_Op<"alloc_with_multiple_results"> {
 // Test Ops bufferization
 //===----------------------------------------------------------------------===//
 
-def TestDummyTensorOp : TEST_Op<"dummy_tensor_op", [BufferizableOpInterface]> {
+def TestDummyTensorOp : TEST_Op<"dummy_tensor_op",
+    [DeclareOpInterfaceMethods<BufferizableOpInterface,
+        ["bufferize", "bufferizesToMemoryRead",
+         "bufferizesToMemoryWrite", "getAliasingValues"]>]> {
   let arguments = (ins
     Arg<TestTensorType>:$input
   );
   let results = (outs
     Arg<TestTensorType>:$output
   );
-  let extraClassDeclaration = [{
-    // BufferizableOpInterface
-    bool bufferizesToMemoryRead(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    bool bufferizesToMemoryWrite(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    mlir::bufferization::AliasingValueList getAliasingValues(mlir::OpOperand&,
-      const mlir::bufferization::AnalysisState&);
-
-    mlir::LogicalResult bufferize(
-      mlir::RewriterBase& rewriter,
-      const mlir::bufferization::BufferizationOptions& options,
-      mlir::bufferization::BufferizationState &state);
-  }];
 
   let extraClassDefinition = [{
     bool test::TestDummyTensorOp::bufferizesToMemoryRead(::mlir::OpOperand&,
@@ -3661,4 +3654,55 @@ def TestDummyMemrefOp : TEST_Op<"dummy_memref_op", []> {
   );
 }
 
+def TestCreateTensorOp : TEST_Op<"create_tensor_op",
+    [DeclareOpInterfaceMethods<BufferizableOpInterface,
+        ["bufferize", "getBufferType", "bufferizesToMemoryRead",
+         "bufferizesToMemoryWrite", "getAliasingValues",
+         "bufferizesToAllocation"]>]> {
+  let arguments = (ins);
+  let results = (outs Arg<TestTensorType>:$output);
+  let extraClassDefinition = [{
+    bool test::TestCreateTensorOp::bufferizesToMemoryRead(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    bool test::TestCreateTensorOp::bufferizesToMemoryWrite(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    bool test::TestCreateTensorOp::bufferizesToAllocation(mlir::Value value) {
+      return false;
+    }
+
+    ::mlir::bufferization::AliasingValueList
+    test::TestCreateTensorOp::getAliasingValues(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return {};
+    }
+  }];
+}
+
+def TestCreateMemrefOp : TEST_Op<"create_memref_op"> {
+  let arguments = (ins);
+  let results = (outs Arg<TestMemrefType>:$output);
+}
+
+//===----------------------------------------------------------------------===//
+// Test assembly format references
+//===----------------------------------------------------------------------===//
+
+def TestOpWithRegionRef : TEST_Op<"dummy_op_with_region_ref", [NoTerminator]> {
+  let regions = (region AnyRegion:$body);
+  let assemblyFormat = [{
+    $body attr-dict custom<DummyRegionRef>(ref($body))
+  }];
+}
+
+def TestOpWithSuccessorRef : TEST_Op<"dummy_op_with_successor_ref"> {
+  let successors = (successor AnySuccessor:$successor);
+  let assemblyFormat = [{
+    $successor attr-dict custom<DummySuccessorRef>(ref($successor))
+  }];
+}
+
 #endif // TEST_OPS
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 9126736d1d17..6b22b171822a 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -1014,7 +1014,7 @@ struct TestPassthroughInvalidOp : public ConversionPattern {
               .getResult());
     }
     rewriter.replaceOpWithNewOp<TestValidOp>(op, TypeRange(), flattened,
-                                             std::nullopt);
+                                             ArrayRef<NamedAttribute>());
     return success();
   }
 };
@@ -1030,7 +1030,7 @@ struct TestDropAndReplaceInvalidOp : public ConversionPattern {
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
     rewriter.replaceOpWithNewOp<TestValidOp>(op, TypeRange(), ValueRange(),
-                                             std::nullopt);
+                                             ArrayRef<NamedAttribute>());
     return success();
   }
 };
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
index 9971f0cde4ed..ee3eb9522db7 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.cpp
@@ -21,6 +21,9 @@
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/Interfaces/TilingInterface.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "test-tiling-interface"
 
 #define GET_OP_CLASSES
 #include "TestTilingInterfaceTransformOps.h.inc"
@@ -168,29 +171,30 @@ transform::TestFuseAndYieldOp::apply(TransformRewriter &rewriter,
 
 /// Apply fusing of consumer transformation to all payload ops and store both
 /// the original consumer operation as well as the fused consumer operation.
-template <typename Range>
 static LogicalResult applyFuseConsumer(
-    RewriterBase &rewriter, Operation *transformOp, Range &&payloadOps,
-    MutableArrayRef<LoopLikeOpInterface> loops, uint32_t numConsumerToFuse,
-    TransformResults &transformResults) {
+    RewriterBase &rewriter, Operation *transformOp,
+    ArrayRef<Operation *> slices, MutableArrayRef<LoopLikeOpInterface> loops,
+    uint32_t numConsumerToFuse, TransformResults &transformResults) {
   SmallVector<Operation *> originalConsumerOps;
   SmallVector<Operation *> fusedConsumerOps;
 
-  for (Operation *target : payloadOps) {
-    rewriter.setInsertionPoint(target);
+  rewriter.setInsertionPoint(slices.front());
 
-    while (numConsumerToFuse--) {
-      FailureOr<scf::SCFFuseConsumerOfSliceResult> fuseConsumerResults =
-          scf::tileAndFuseConsumerOfSlice(rewriter, target, loops);
+  while (numConsumerToFuse--) {
+    FailureOr<scf::SCFFuseConsumerOfSliceResult> fuseConsumerResults =
+        scf::tileAndFuseConsumerOfSlices(rewriter, slices, loops);
 
-      if (failed(fuseConsumerResults))
-        return failure();
+    if (failed(fuseConsumerResults))
+      return slices.front()->emitOpError("failed to fuse consumer of slice");
 
-      // Report back the relevant handles to the transform op.
-      originalConsumerOps.push_back(
-          fuseConsumerResults->origConsumerOperand->getOwner());
-      fusedConsumerOps.push_back(
-          fuseConsumerResults->tiledAndFusedConsumerOperand->getOwner());
+    // Report back the relevant handles to the transform op.
+    for (OpOperand *origConsumerOperand :
+         fuseConsumerResults->origConsumerOperands) {
+      originalConsumerOps.push_back(origConsumerOperand->getOwner());
+    }
+    for (OpOperand *tiledAndFusedConsumerOperand :
+         fuseConsumerResults->tiledAndFusedConsumerOperands) {
+      fusedConsumerOps.push_back(tiledAndFusedConsumerOperand->getOwner());
     }
   }
 
@@ -203,6 +207,12 @@ DiagnosedSilenceableFailure
 transform::TestFuseConsumerOp::apply(TransformRewriter &rewriter,
                                      TransformResults &transformResults,
                                      TransformState &state) {
+  SmallVector<Operation *> slices;
+  for (auto op : getTargets()) {
+    auto sliceOp = *state.getPayloadOps(op).begin();
+    slices.push_back(sliceOp);
+  }
+
   SmallVector<LoopLikeOpInterface> loops;
   for (auto op : llvm::reverse(getLoops())) {
     auto loopLikeOp =
@@ -212,16 +222,16 @@ transform::TestFuseConsumerOp::apply(TransformRewriter &rewriter,
     }
     loops.push_back(loopLikeOp);
   }
-  LogicalResult result = applyFuseConsumer(
-      rewriter, getOperation(), state.getPayloadOps(getTarget()), loops,
-      getNumConsumerToFuse(), transformResults);
+  LogicalResult result =
+      applyFuseConsumer(rewriter, getOperation(), slices, loops,
+                        getNumConsumerToFuse(), transformResults);
   return failed(result) ? DiagnosedSilenceableFailure::definiteFailure()
                         : DiagnosedSilenceableFailure::success();
 }
 
 void transform::TestFuseConsumerOp::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
-  consumesHandle(getTargetMutable(), effects);
+  consumesHandle(getTargetsMutable(), effects);
   consumesHandle(getLoopsMutable(), effects);
   producesHandle(getOperation()->getOpResults(), effects);
   modifiesPayload(effects);
diff --git a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td
index 98f7145c99cb..3c09082e192e 100644
--- a/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td
+++ b/mlir/test/lib/Interfaces/TilingInterface/TestTilingInterfaceTransformOps.td
@@ -50,7 +50,8 @@ def TestFuseAndYieldOp : Op<Transform_Dialect, "test.fuse_and_yield",
 }
 
 def TestFuseConsumerOp : Op<Transform_Dialect, "test.fuse_consumer",
-       [DeclareOpInterfaceMethods<TransformOpInterface>,
+       [AttrSizedOperandSegments,
+        DeclareOpInterfaceMethods<TransformOpInterface>,
         DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
         ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
@@ -59,14 +60,14 @@ def TestFuseConsumerOp : Op<Transform_Dialect, "test.fuse_consumer",
   }];
 
   let arguments = (ins 
-      TransformHandleTypeInterface:$target,
+      Variadic<TransformHandleTypeInterface>:$targets,
       Variadic<TransformHandleTypeInterface>:$loops,
       DefaultValuedAttr<I32Attr, "1">:$num_consumer_to_fuse);
   let results = (outs TransformHandleTypeInterface:$consumer,
                       TransformHandleTypeInterface:$fused_consumer);
 
   let assemblyFormat = [{
-    $target `in` `(` $loops `)`
+    $targets `in` `(` $loops `)`
     (`num_consumer_to_fuse` `=` $num_consumer_to_fuse^)? 
     attr-dict `:` functional-type(operands, results)
   }];
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.cpp b/mlir/test/lib/Transforms/TestTransformsOps.cpp
index c05b32bed9b9..9a5632bb99c0 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.cpp
+++ b/mlir/test/lib/Transforms/TestTransformsOps.cpp
@@ -11,9 +11,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Transform/IR/TransformAttrs.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/OpDefinition.h"
 #include "mlir/Transforms/RegionUtils.h"
 
 #define GET_OP_CLASSES
@@ -56,6 +58,33 @@ transform::TestMoveValueDefns::apply(TransformRewriter &rewriter,
   return DiagnosedSilenceableFailure::success();
 }
 
+//===----------------------------------------------------------------------===//
+// Test affine functionality.
+//===----------------------------------------------------------------------===//
+DiagnosedSilenceableFailure
+transform::TestMakeComposedFoldedAffineApply::applyToOne(
+    TransformRewriter &rewriter, affine::AffineApplyOp affineApplyOp,
+    ApplyToEachResultList &results, TransformState &state) {
+  Location loc = affineApplyOp.getLoc();
+  OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
+      rewriter, loc, affineApplyOp.getAffineMap(),
+      getAsOpFoldResult(affineApplyOp.getOperands()),
+      /*composeAffineMin=*/true);
+  Value result;
+  if (auto v = dyn_cast<Value>(ofr)) {
+    result = v;
+  } else {
+    result = rewriter.create<arith::ConstantIndexOp>(
+        loc, getConstantIntValue(ofr).value());
+  }
+  results.push_back(result.getDefiningOp());
+  rewriter.replaceOp(affineApplyOp, result);
+  return DiagnosedSilenceableFailure::success();
+}
+
+//===----------------------------------------------------------------------===//
+// Extension
+//===----------------------------------------------------------------------===//
 namespace {
 
 class TestTransformsDialectExtension
diff --git a/mlir/test/lib/Transforms/TestTransformsOps.td b/mlir/test/lib/Transforms/TestTransformsOps.td
index 495579b452df..9b0a26082490 100644
--- a/mlir/test/lib/Transforms/TestTransformsOps.td
+++ b/mlir/test/lib/Transforms/TestTransformsOps.td
@@ -59,5 +59,32 @@ def TestMoveValueDefns :
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Test affine functionality.
+//===----------------------------------------------------------------------===//
+
+def TestMakeComposedFoldedAffineApply :
+    Op<Transform_Dialect, "test.make_composed_folded_affine_apply",
+        [FunctionalStyleTransformOpTrait, 
+         MemoryEffectsOpInterface,
+         TransformOpInterface,
+         TransformEachOpTrait,
+         ReportTrackingListenerFailuresOpTrait]> {
+  let description = [{
+    Rewrite an affine_apply by using the makeComposedFoldedAffineApply API.
+  }];
+  let arguments = (ins TransformHandleTypeInterface:$op);
+  let results = (outs TransformHandleTypeInterface:$composed);
+  let assemblyFormat = [{
+    $op attr-dict `:` functional-type(operands, results)
+  }];
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
+        ::mlir::affine::AffineApplyOp affineApplyOp,
+        ::mlir::transform::ApplyToEachResultList &results,
+        ::mlir::transform::TransformState &state);
+  }];
+}
 
 #endif // TEST_TRANSFORM_OPS
diff --git a/mlir/test/mlir-linalg-ods-gen/test-linalg-ods-yaml-gen.yaml b/mlir/test/mlir-linalg-ods-gen/test-linalg-ods-yaml-gen.yaml
index ab7b86125f69..00c70705cbb3 100644
--- a/mlir/test/mlir-linalg-ods-gen/test-linalg-ods-yaml-gen.yaml
+++ b/mlir/test/mlir-linalg-ods-gen/test-linalg-ods-yaml-gen.yaml
@@ -87,7 +87,8 @@ structured_op: !LinalgStructuredOpConfig
 #  ODS-NEXT:    }
 
 # IMPL-LABEL:  void Test1Op::regionBuilder(ImplicitLocOpBuilder &b,
-#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs)
+#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs,
+#  IMPL-NEXT:    function_ref<InFlightDiagnostic()> emitError)
 #       IMPL:  TypeFn castVal = TypeFn::cast_signed;
 #  IMPL-NEXT:  auto castIter = llvm::find_if(attrs, [&](const NamedAttribute &attr) {
 #  IMPL-NEXT:                                return attr.getName() == "cast"; });
@@ -97,10 +98,10 @@ structured_op: !LinalgStructuredOpConfig
 #  IMPL-NEXT:  }
 
 #       IMPL:  Value [[VAL0:[a-z0-9]+]] = helper.constant("42 : i64");
-#   IMPL-DAG:  Value [[VAL1:[a-z0-9]+]] = helper.buildTypeFn(castVal, block.getArgument(0).getType(), [[VAL0]]);
+#   IMPL-DAG:  Value [[VAL1:[a-z0-9]+]] = helper.buildTypeFn(castVal, block.getArgument(0).getType(), [[VAL0]], emitError);
 #   IMPL-DAG:  Value [[VAL2:[a-z0-9]+]] = helper.index(1);
-#   IMPL-DAG:  Value [[VAL3:[a-z0-9]+]] = helper.buildTypeFn(castVal, block.getArgument(0).getType(), [[VAL2]]);
-#   IMPL-DAG:  Value [[VAL4:[a-z0-9]+]] = helper.buildBinaryFn(BinaryFn::add, [[VAL1]], [[VAL3]]);
+#   IMPL-DAG:  Value [[VAL3:[a-z0-9]+]] = helper.buildTypeFn(castVal, block.getArgument(0).getType(), [[VAL2]], emitError);
+#   IMPL-DAG:  Value [[VAL4:[a-z0-9]+]] = helper.buildBinaryFn(BinaryFn::add, [[VAL1]], [[VAL3]], emitError);
 
 
 # @linalg_structured_op
@@ -186,7 +187,8 @@ structured_op: !LinalgStructuredOpConfig
 #       IMPL:  "incorrect element type for index attribute 'strides'"
 #       IMPL:  "incorrect shape for index attribute 'strides'"
 #       IMPL:  void Test2Op::regionBuilder(ImplicitLocOpBuilder &b,
-#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs)
+#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs,
+#  IMPL-NEXT:    function_ref<InFlightDiagnostic()> emitError)
 #  IMPL-NEXT:    assert(2 > 0 && block.getNumArguments() == 2 &&
 
 #       IMPL:   yields.push_back(block.getArgument(0));
@@ -315,13 +317,18 @@ structured_op: !LinalgStructuredOpConfig
 #  ODS-NEXT:    $_state.addAttribute("binary_fun", binary_fun)
 
 # IMPL-LABEL:  void Test4Op::regionBuilder(ImplicitLocOpBuilder &b,
-#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs)
+#  IMPL-NEXT:    Block &block, ArrayRef<NamedAttribute> attrs,
+#  IMPL-NEXT:    function_ref<InFlightDiagnostic()> emitError)
 #       IMPL:  UnaryFn unary_funVal = UnaryFn::exp
 #       IMPL:  BinaryFn binary_funVal = BinaryFn::add
 
-#       IMPL:  Value [[VAL0:[a-z0-9]+]] = helper.buildUnaryFn(unary_funVal, block.getArgument(0))
-#  IMPL-NEXT:  Value [[VAL1:[a-z0-9]+]] = helper.buildBinaryFn(binary_funVal, [[VAL0]], block.getArgument(0))
-#  IMPL-NEXT:  yields.push_back([[VAL1]])
+#       IMPL:  Value [[VAL0:[a-z0-9]+]] = helper.buildUnaryFn(unary_funVal, block.getArgument(0), emitError);
+#  IMPL-NEXT:  if (![[VAL0]])
+#  IMPL-NEXT:    return;
+#  IMPL:  Value [[VAL1:[a-z0-9]+]] = helper.buildBinaryFn(binary_funVal, [[VAL0]], block.getArgument(0), emitError);
+#  IMPL-NEXT:  if (![[VAL1]])
+#  IMPL-NEXT:    return;
+#  IMPL:  yields.push_back([[VAL1]])
 
 # @linalg_structured_op
 # def test5(value=ScalarDef(T1), O=TensorDef(U, output=True)):
diff --git a/mlir/test/mlir-tblgen/op-decl-and-defs.td b/mlir/test/mlir-tblgen/op-decl-and-defs.td
index a3dce9b31f8d..3ccefd4d8236 100644
--- a/mlir/test/mlir-tblgen/op-decl-and-defs.td
+++ b/mlir/test/mlir-tblgen/op-decl-and-defs.td
@@ -125,6 +125,7 @@ def NS_AOp : NS_Op<"a_op", [IsolatedFromAbove, IsolatedFromAbove]> {
 // CHECK:   ::llvm::LogicalResult verifyInvariants();
 // CHECK:   static void getCanonicalizationPatterns(::mlir::RewritePatternSet &results, ::mlir::MLIRContext *context);
 // CHECK:   ::llvm::LogicalResult fold(FoldAdaptor adaptor, ::llvm::SmallVectorImpl<::mlir::OpFoldResult> &results);
+// CHECK:   static ::llvm::LogicalResult setPropertiesFromParsedAttr(Properties &prop, ::mlir::Attribute attr, ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError);
 // CHECK:   // Display a graph for debugging purposes.
 // CHECK:   void displayGraph();
 // CHECK: };
diff --git a/mlir/test/mlir-tblgen/op-format-spec.td b/mlir/test/mlir-tblgen/op-format-spec.td
index 02bf65609b21..1541cd09f53e 100644
--- a/mlir/test/mlir-tblgen/op-format-spec.td
+++ b/mlir/test/mlir-tblgen/op-format-spec.td
@@ -49,6 +49,19 @@ def DirectiveCustomValidD : TestFormat_Op<[{
 def DirectiveCustomValidE : TestFormat_Op<[{
   custom<MyDirective>(prop-dict) attr-dict
 }]>, Arguments<(ins UnitAttr:$flag)>;
+def DirectiveCustomValidF : TestFormat_Op<[{
+  $operand custom<MyDirective>(ref($operand)) attr-dict
+}]>, Arguments<(ins Optional<I64>:$operand)>;
+def DirectiveCustomValidG : TestFormat_Op<[{
+  $body custom<MyDirective>(ref($body)) attr-dict
+}]> {
+  let regions = (region AnyRegion:$body);
+}
+def DirectiveCustomValidH : TestFormat_Op<[{
+  $successor custom<MyDirective>(ref($successor)) attr-dict
+}]> {
+  let successors = (successor AnySuccessor:$successor);
+}
 
 //===----------------------------------------------------------------------===//
 // functional-type
@@ -82,7 +95,7 @@ def DirectiveRegionsValid : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error:
-def DirectiveResultsInvalidA : TestFormat_Op<[{
+def DirectiveResultsValid : TestFormat_Op<[{
   type(results) attr-dict
 }]>;
 
@@ -91,7 +104,7 @@ def DirectiveResultsInvalidA : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error:
-def DirectiveSuccessorsInvalidA : TestFormat_Op<[{
+def DirectiveSuccessorsValid : TestFormat_Op<[{
   successors attr-dict
 }]>;
 
@@ -154,7 +167,7 @@ def OptionalValidA : TestFormat_Op<[{
 //===----------------------------------------------------------------------===//
 
 // CHECK-NOT: error
-def StringInvalidA : TestFormat_Op<[{ custom<Foo>("foo") attr-dict }]>;
+def StringValid : TestFormat_Op<[{ custom<Foo>("foo") attr-dict }]>;
 
 //===----------------------------------------------------------------------===//
 // Variables
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 981fb5aff2aa..fa929c2df9d1 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -290,10 +290,10 @@ test.format_optional_prop_dict <{a = [], b = 1 : i32}>
 // CHECK: test.format_optional_prop_dict {{$}}
 test.format_optional_prop_dict <{}>
 
-// CHECK: test.format_optional_prop_dict < {a = ["foo"]}>
+// CHECK: test.format_optional_prop_dict <{a = ["foo"]}>
 test.format_optional_prop_dict <{a = ["foo"]}>
 
-// CHECK: test.format_optional_prop_dict < {b = 2 : i32}>
+// CHECK: test.format_optional_prop_dict <{b = 2 : i32}>
 test.format_optional_prop_dict <{b = 2 : i32}>
 
 // CHECK: test.format_optional_prop_dict <{a = ["foo"], b = 2 : i32}>
@@ -513,15 +513,15 @@ test.format_infer_variadic_type_from_non_variadic %i64, %i64 : i64
 // CHECK: test.format_infer_type_variadic_operands(%[[I32]], %[[I32]] : i32, i32) (%[[I64]], %[[I64]] : i64, i64)
 %ignored_res13:4 = test.format_infer_type_variadic_operands(%i32, %i32 : i32, i32) (%i64, %i64 : i64, i64)
 
-// CHECK: test.with_properties_and_attr 16 < {rhs = 16 : i64}>
+// CHECK: test.with_properties_and_attr 16 <{rhs = 16 : i64}>
 test.with_properties_and_attr 16 <{rhs = 16 : i64}>
 
-// CHECK: test.with_properties_and_inferred_type 16 < {packed, rhs = 16 : i64}>
+// CHECK: test.with_properties_and_inferred_type 16 <{packed, rhs = 16 : i64}>
 %should_be_i32 = test.with_properties_and_inferred_type 16 <{packed, rhs = 16 : i64}>
 // Assert through the verifier that its inferred as i32.
 test.format_all_types_match_var %should_be_i32, %i32 : i32
 
-// CHECK: test.using_property_in_custom_and_other [1, 4, 20] < {other = 16 : i64}>
+// CHECK: test.using_property_in_custom_and_other [1, 4, 20] <{other = 16 : i64}>
 test.using_property_in_custom_and_other [1, 4, 20] <{other = 16 : i64}>
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/mlir-tblgen/op-format.td b/mlir/test/mlir-tblgen/op-format.td
index 09e068b91a40..1790737a3a34 100644
--- a/mlir/test/mlir-tblgen/op-format.td
+++ b/mlir/test/mlir-tblgen/op-format.td
@@ -109,3 +109,23 @@ def OptionalGroupC : TestFormat_Op<[{
 def OptionalGroupD : TestFormat_Op<[{
   (custom<Custom>($a, $b)^)? attr-dict
 }], [AttrSizedOperandSegments]>, Arguments<(ins Optional<I64>:$a, Optional<I64>:$b)>;
+
+// CHECK-LABEL: RegionRef::parse
+// CHECK:   auto odsResult = parseCustom(parser, *bodyRegion);
+// CHECK-LABEL: RegionRef::print
+// CHECK:   printCustom(_odsPrinter, *this, getBody());
+def RegionRef : TestFormat_Op<[{
+  $body custom<Custom>(ref($body)) attr-dict
+}]> {
+  let regions = (region AnyRegion:$body);
+}
+
+// CHECK-LABEL: SuccessorRef::parse
+// CHECK:   auto odsResult = parseCustom(parser, successorSuccessor);
+// CHECK-LABEL: SuccessorRef::print
+// CHECK:   printCustom(_odsPrinter, *this, getSuccessor());
+def SuccessorRef : TestFormat_Op<[{
+  $successor custom<Custom>(ref($successor)) attr-dict
+}]> {
+  let successors = (successor AnySuccessor:$successor);
+}
diff --git a/mlir/test/mlir-tblgen/op-operand.td b/mlir/test/mlir-tblgen/op-operand.td
index a2fa1f7046a9..ab8d721ed542 100644
--- a/mlir/test/mlir-tblgen/op-operand.td
+++ b/mlir/test/mlir-tblgen/op-operand.td
@@ -13,6 +13,9 @@ def OpA : NS_Op<"one_normal_operand_op", []> {
   let arguments = (ins I32:$input);
 }
 
+// DECL-LABEL: class OpA : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpA definitions
 
 // CHECK:      void OpA::build
@@ -28,6 +31,9 @@ def OpB : NS_Op<"one_variadic_operand_op", []> {
   let arguments = (ins Variadic<I32>:$input);
 }
 
+// DECL-LABEL: class OpB : {{.*}} {
+// DECL: static constexpr int odsIndex_input = 0;
+
 // CHECK-LABEL: OpB::build
 // CHECK:         ::mlir::ValueRange input
 // CHECK-NOT:     assert
@@ -37,6 +43,11 @@ def OpD : NS_Op<"mix_variadic_and_normal_inputs_op", [SameVariadicOperandSize]>
   let arguments = (ins Variadic<AnyTensor>:$input1, AnyTensor:$input2, Variadic<AnyTensor>:$input3);
 }
 
+// DECL-LABEL: class OpD : {{.*}} {
+// DECL: static constexpr int odsIndex_input1 = 0;
+// DECL: static constexpr int odsIndex_input2 = 1;
+// DECL: static constexpr int odsIndex_input3 = 2;
+
 // DECL-LABEL: ::mlir::Operation::operand_range getInput1
 // DECL-NEXT: return getODSOperands(0);
 
diff --git a/mlir/test/mlir-translate/emitc_classops.mlir b/mlir/test/mlir-translate/emitc_classops.mlir
new file mode 100644
index 000000000000..4b7ddf4630d5
--- /dev/null
+++ b/mlir/test/mlir-translate/emitc_classops.mlir
@@ -0,0 +1,51 @@
+// RUN: mlir-translate --mlir-to-cpp %s | FileCheck %s
+
+emitc.class @modelClass {
+    emitc.field @fieldName0 : !emitc.array<1xf32> 
+    emitc.field @fieldName1 : !emitc.array<1xf32> 
+    emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = get_field @fieldName1 : !emitc.array<1xf32>
+        %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+    }
+}
+
+// CHECK-LABEL: class modelClass {
+// CHECK-NEXT: public:
+// CHECK-NEXT:  float[1] fieldName0;
+// CHECK-NEXT:  float[1] fieldName1;
+// CHECK-NEXT:  void execute() {
+// CHECK-NEXT:    size_t v1 = 0;
+// CHECK-NEXT:    float[1] v2 = fieldName0;
+// CHECK-NEXT:    float[1] v3 = fieldName1;
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT: };
+
+emitc.class final @finalClass {
+    emitc.field @fieldName0 : !emitc.array<1xf32> 
+    emitc.field @fieldName1 : !emitc.array<1xf32> 
+    emitc.func @execute() {
+        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+        %1 = get_field @fieldName0 : !emitc.array<1xf32>
+        %2 = get_field @fieldName1 : !emitc.array<1xf32>
+        %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+        return
+    }
+}
+
+// CHECK-LABEL: class finalClass final {
+// CHECK-NEXT: public:
+// CHECK-NEXT:  float[1] fieldName0;
+// CHECK-NEXT:  float[1] fieldName1;
+// CHECK-NEXT:  void execute() {
+// CHECK-NEXT:    size_t v1 = 0;
+// CHECK-NEXT:    float[1] v2 = fieldName0;
+// CHECK-NEXT:    float[1] v3 = fieldName1;
+// CHECK-NEXT:    return;
+// CHECK-NEXT:  }
+// CHECK-EMPTY:
+// CHECK-NEXT: };
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index f1e2afa0f240..709a1d2424f3 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -49,42 +49,6 @@ def fill_buffer(out):
     print(module)
 
 
-# CHECK-LABEL: TEST: testNamedStructuredOpCustomForm
-@run
-def testNamedStructuredOpCustomForm():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                RankedTensorType.get((4, 8), f32), RankedTensorType.get((4, 8), f32)
-            )
-            def named_form(lhs, rhs):
-                init_result = tensor.EmptyOp([4, 8], f32)
-                # Check for the named form with custom format
-                #      CHECK: linalg.elemwise_unary
-                # CHECK-SAME:    cast = #linalg.type_fn<cast_signed>
-                # CHECK-SAME:    fun = #linalg.unary_fn<exp>
-                # CHECK-SAME:    ins(%{{.*}} : tensor<4x8xf32>) outs(%{{.*}} : tensor<4x8xf32>)
-                unary_result = linalg.elemwise_unary(lhs, outs=[init_result.result])
-                #      CHECK: linalg.elemwise_binary
-                # CHECK-SAME:    cast = #linalg.type_fn<cast_unsigned>
-                # CHECK-SAME:    fun = #linalg.binary_fn<mul>
-                # CHECK-SAME:    ins(%{{.*}}, %{{.*}} : tensor<4x8xf32>, tensor<4x8xf32>) outs(%{{.*}} : tensor<4x8xf32>)
-                #      CHECK: return
-                binary_result = linalg.elemwise_binary(
-                    lhs,
-                    rhs,
-                    outs=[init_result.result],
-                    fun=BinaryFn.mul,
-                    cast=TypeFn.cast_unsigned,
-                )
-                return unary_result, binary_result
-
-    print(module)
-
-
 # CHECK-LABEL: TEST: testIdentityRegionOps
 @run
 def testIdentityRegionOps():
diff --git a/mlir/test/python/dialects/transform_debug_ext.py b/mlir/test/python/dialects/transform_debug_ext.py
new file mode 100644
index 000000000000..2dfdaed34386
--- /dev/null
+++ b/mlir/test/python/dialects/transform_debug_ext.py
@@ -0,0 +1,45 @@
+# RUN: %PYTHON %s | FileCheck %s
+
+from mlir.ir import *
+from mlir.dialects import transform
+from mlir.dialects.transform import debug
+
+
+def run(f):
+    print("\nTEST:", f.__name__)
+    with Context(), Location.unknown():
+        module = Module.create()
+        with InsertionPoint(module.body):
+            sequence = transform.SequenceOp(
+                transform.FailurePropagationMode.Propagate,
+                [],
+                transform.AnyOpType.get(),
+            )
+            with InsertionPoint(sequence.body):
+                f(sequence.bodyTarget)
+                transform.YieldOp()
+        print(module)
+    return f
+
+
+@run
+def testDebugEmitParamAsRemark(target):
+    i0 = IntegerAttr.get(IntegerType.get_signless(32), 0)
+    i0_param = transform.ParamConstantOp(transform.AnyParamType.get(), i0)
+    debug.emit_param_as_remark(i0_param)
+    debug.emit_param_as_remark(i0_param, anchor=target, message="some text")
+    # CHECK-LABEL: TEST: testDebugEmitParamAsRemark
+    # CHECK: ^{{.*}}(%[[ARG0:.+]]: !transform.any_op):
+    # CHECK: %[[PARAM:.*]] = transform.param.constant
+    # CHECK: transform.debug.emit_param_as_remark %[[PARAM]]
+    # CHECK: transform.debug.emit_param_as_remark %[[PARAM]]
+    # CHECK-SAME: "some text"
+    # CHECK-SAME: at %[[ARG0]]
+
+
+@run
+def testDebugEmitRemarkAtOp(target):
+    debug.emit_remark_at(target, "some text")
+    # CHECK-LABEL: TEST: testDebugEmitRemarkAtOp
+    # CHECK: ^{{.*}}(%[[ARG0:.+]]: !transform.any_op):
+    # CHECK: transform.debug.emit_remark_at %[[ARG0]], "some text"
diff --git a/mlir/test/python/integration/dialects/linalg/opsrun.py b/mlir/test/python/integration/dialects/linalg/opsrun.py
index 69f97e15e139..8f202318146e 100644
--- a/mlir/test/python/integration/dialects/linalg/opsrun.py
+++ b/mlir/test/python/integration/dialects/linalg/opsrun.py
@@ -19,37 +19,6 @@ def log(*args):
     sys.stderr.flush()
 
 
-elemwise_boiler = """
-func.func @main() -> f32 attributes {llvm.emit_c_interface} {
-  %v0 = arith.constant 0.0 : f32
-  %v1 = arith.constant 1.0 : f32
-  %v2 = arith.constant 2.0 : f32
-
-  %lhs = memref.alloc() : memref<f32>
-  %rhs = memref.alloc() : memref<4x8xf32>
-  %O0 = memref.alloc() : memref<4x8xf32>
-  %O1 = memref.alloc() : memref<4x8xf32>
-  linalg.fill ins(%v1 : f32) outs(%lhs : memref<f32>)
-  linalg.fill ins(%v2 : f32) outs(%rhs : memref<4x8xf32>)
-  linalg.fill ins(%v0 : f32) outs(%O0 : memref<4x8xf32>)
-  linalg.fill ins(%v0 : f32) outs(%O1 : memref<4x8xf32>)
-
-  call @elemwise_exp_add_on_buffers(%lhs, %rhs, %O0) :
-    (memref<f32>, memref<4x8xf32>, memref<4x8xf32>) -> ()
-  call @elemwise_log_mul_on_buffers(%lhs, %rhs, %O1) :
-    (memref<f32>, memref<4x8xf32>, memref<4x8xf32>) -> ()
-
-  %c0 = arith.constant 0 : index
-  %res0 = memref.load %O0[%c0, %c0] : memref<4x8xf32>
-  %res1 = memref.load %O1[%c0, %c0] : memref<4x8xf32>
-
-  %0 = arith.addf %res0, %res1 : f32
-
-  // TODO: FFI-based solution to allow testing and printing with python code.
-  return %0 : f32
-}
-"""
-
 fill_boiler = """
 func.func @main() -> i32 attributes {llvm.emit_c_interface} {
   %O0 = memref.alloc() : memref<i32>
@@ -177,96 +146,6 @@ def transform(module, boilerplate):
     return mod
 
 
-def test_elemwise_builtin():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        i8 = IntegerType.get_signless(8)
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_exp_add_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out])
-                linalg.elemwise_binary(out, rhs, outs=[out])
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_log_mul_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out], fun=UnaryFn.log)
-                linalg.elemwise_binary(out, rhs, outs=[out], fun=BinaryFn.mul)
-
-        execution_engine = ExecutionEngine(transform(module, elemwise_boiler))
-
-        # TODO: FFI-based solution to allow testing and printing with python code.
-        # Prepare arguments: one result f32.
-        # Arguments must be passed as pointers.
-        c_float_p = ctypes.c_float * 1
-        res = c_float_p(-1.0)
-        execution_engine.invoke("main", res)
-
-        log("RESULT: ", res[0])
-        # elemwise_exp_add_on_buffers: exp(1.0) + 2.0 = 4.71828182846
-        # elemwise_log_mul_on_buffers: log(1.0) * 2.0 = 0.0
-        # CHECK: RESULT: 4.71828
-
-
-test_elemwise_builtin()
-
-
-def test_elemwise_generic():
-    with Context() as ctx, Location.unknown():
-        module = Module.create()
-        f32 = F32Type.get()
-        i8 = IntegerType.get_signless(8)
-        with InsertionPoint(module.body):
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_exp_add_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(lhs, outs=[out], emit_generic=True)
-                linalg.elemwise_binary(out, rhs, outs=[out], emit_generic=True)
-
-            @func.FuncOp.from_py_func(
-                MemRefType.get((), f32),
-                MemRefType.get((4, 8), f32),
-                MemRefType.get((4, 8), f32),
-            )
-            def elemwise_log_mul_on_buffers(lhs, rhs, out):
-                linalg.elemwise_unary(
-                    lhs, outs=[out], fun=UnaryFn.log, emit_generic=True
-                )
-                linalg.elemwise_binary(
-                    out, rhs, outs=[out], fun=BinaryFn.mul, emit_generic=True
-                )
-
-        execution_engine = ExecutionEngine(transform(module, elemwise_boiler))
-
-        # TODO: FFI-based solution to allow testing and printing with python code.
-        # Prepare arguments: one result f32.
-        # Arguments must be passed as pointers.
-        c_float_p = ctypes.c_float * 1
-        res = c_float_p(-1.0)
-        execution_engine.invoke("main", res)
-
-        log("RESULT: ", res[0])
-        # elemwise_exp_add_on_buffers: exp(1.0) + 2.0 = 4.71828182846
-        # elemwise_log_mul_on_buffers: log(1.0) * 2.0 = 0.0
-        # CHECK: RESULT: 4.71828
-
-
-test_elemwise_generic()
-
-
 def test_fill_builtin():
     with Context() as ctx, Location.unknown():
         module = Module.create()
diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
index 93a300e0b24a..0a1693cff1d3 100644
--- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
+++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp
@@ -559,9 +559,10 @@ def {0} : LinalgStructuredBase_Op<"{1}", !listconcat([AttrSizedOperandSegments],
       SmallVector<utils::IteratorType> getIteratorTypesArray();
       ArrayAttr getIndexingMaps();
       static void regionBuilder(ImplicitLocOpBuilder &b,
-                                Block &block, ArrayRef<NamedAttribute> attrs);
+                                Block &block, ArrayRef<NamedAttribute> attrs,
+                                function_ref<InFlightDiagnostic()> emitError);
       static std::function<void(ImplicitLocOpBuilder &,
-                                Block &, ArrayRef<NamedAttribute>)>
+                                Block &, ArrayRef<NamedAttribute>, function_ref<InFlightDiagnostic()> emitError)>
       getRegionBuilder() {{
         return regionBuilder;
       }
@@ -1010,7 +1011,8 @@ LogicalResult {0}::verifyIndexingMapRequiredAttributes() {{
     // {3}: Statements
     static const char structuredOpRegionBuilderFormat[] = R"FMT(
 void {0}::regionBuilder(ImplicitLocOpBuilder &b,
-                        Block &block, ArrayRef<NamedAttribute> attrs) {{
+                        Block &block, ArrayRef<NamedAttribute> attrs,
+                        function_ref<InFlightDiagnostic()> emitError) {{
   assert({1} > 0 && block.getNumArguments() == {1} &&
          "{0} regionBuilder expects {1} (>=0) args");
   RegionBuilderHelper helper(b, block);
@@ -1137,8 +1139,13 @@ void {0}::regionBuilder(ImplicitLocOpBuilder &b,
           // Call the function builder.
           std::string cppIdent = llvm::formatv("value{0}", ++localCounter);
           stmts.push_back(llvm::formatv(
-              "Value {0} = helper.build{1}({2}, {3});", cppIdent, enumName,
-              funcType, interleaveToString(operandCppValues, ", ")));
+              R"mlir(
+              Value {0} = helper.build{1}({2}, {3}, emitError);
+              if (!{0})
+                return;
+              )mlir",
+              cppIdent, enumName, funcType,
+              interleaveToString(operandCppValues, ", ")));
           return cppIdent;
         }
         emitError(genContext.getLoc()) << "unknown ScalarExpression type";
diff --git a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
index defd1fa12ca1..d9aa901ee2b2 100644
--- a/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
+++ b/mlir/tools/mlir-tblgen/AttrOrTypeDefGen.cpp
@@ -245,17 +245,15 @@ void DefGen::createParentWithTraits() {
                    ? cast<NativeTrait>(&trait)->getFullyQualifiedTraitName()
                    : cast<InterfaceTrait>(&trait)->getFullyQualifiedTraitName();
       }));
-  llvm::for_each(traitNames, [&](auto &traitName) {
+  for (auto &traitName : traitNames)
     defParent.addTemplateParam(traitName);
-  });
 
   // Add OpAsmInterface::Trait if we automatically generate mnemonic alias
   // method.
   std::string opAsmInterfaceTraitName =
       strfmt("::mlir::OpAsm{0}Interface::Trait", defType);
-  if (def.genMnemonicAlias() && llvm::none_of(traitNames, [&](auto &traitName) {
-        return traitName == opAsmInterfaceTraitName;
-      })) {
+  if (def.genMnemonicAlias() &&
+      !llvm::is_contained(traitNames, opAsmInterfaceTraitName)) {
     defParent.addTemplateParam(opAsmInterfaceTraitName);
   }
   defCls.addParent(std::move(defParent));
diff --git a/mlir/tools/mlir-tblgen/EnumsGen.cpp b/mlir/tools/mlir-tblgen/EnumsGen.cpp
index 9941a203bc5c..06dc588f9020 100644
--- a/mlir/tools/mlir-tblgen/EnumsGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumsGen.cpp
@@ -648,8 +648,10 @@ static void emitSpecializedAttrDef(const Record &enumDef, raw_ostream &os) {
 
   os << formatv("{0} {1}::getValue() const {{\n", enumName, attrClassName);
 
-  os << formatv("  return static_cast<{0}>(::mlir::IntegerAttr::getInt());\n",
-                enumName);
+  os << formatv(
+      "  return "
+      "static_cast<{0}>(::mlir::IntegerAttr::getValue().getZExtValue());\n",
+      enumName);
 
   os << "}\n";
 }
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 6008ed4673d1..cbb4030f3adb 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -2223,6 +2223,17 @@ generateNamedOperandGetters(const Operator &op, Class &opClass,
                     "'SameVariadicOperandSize' traits");
   }
 
+  // Print the ods names so they don't need to be hardcoded in the source.
+  for (int i = 0; i != numOperands; ++i) {
+    const auto &operand = op.getOperand(i);
+    if (operand.name.empty())
+      continue;
+
+    opClass.declare<Field>("static constexpr int", Twine("odsIndex_") +
+                                                       operand.name + " = " +
+                                                       Twine(i));
+  }
+
   // First emit a few "sink" getter methods upon which we layer all nicer named
   // getter methods.
   // If generating for an adaptor, the method is put into the non-templated
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 11edf2523f1a..67fc7636af60 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1302,8 +1302,8 @@ if (!dict) {
 }
 // keep track of used keys in the input dictionary to be able to error out
 // if there are some unknown ones.
-DenseSet<StringAttr> usedKeys;
-MLIRContext *ctx = dict.getContext();
+::mlir::DenseSet<::mlir::StringAttr> usedKeys;
+::mlir::MLIRContext *ctx = dict.getContext();
 (void)ctx;
 )decl";
 
@@ -1315,7 +1315,7 @@ auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
          ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) -> ::mlir::LogicalResult {{
   {0};
 };
-auto {1}AttrName = StringAttr::get(ctx, "{1}");
+auto {1}AttrName = ::mlir::StringAttr::get(ctx, "{1}");
 usedKeys.insert({1}AttrName);
 auto attr = dict.get({1}AttrName);
 if (!attr && {2}) {{
@@ -1363,7 +1363,7 @@ if (attr && ::mlir::failed(setFromAttr(prop.{1}, attr, emitError)))
     bool isRequired = !attr.isOptional() && !attr.hasDefaultValue();
     body << formatv(R"decl(
 auto &propStorage = prop.{0};
-auto {0}AttrName = StringAttr::get(ctx, "{0}");
+auto {0}AttrName = ::mlir::StringAttr::get(ctx, "{0}");
 auto attr = dict.get({0}AttrName);
 usedKeys.insert({0}AttrName);
 if (attr || /*isRequired=*/{1}) {{
@@ -1384,7 +1384,7 @@ if (attr || /*isRequired=*/{1}) {{
                     namedAttr.name, isRequired);
   }
   body << R"decl(
-for (NamedAttribute attr : dict) {
+for (::mlir::NamedAttribute attr : dict) {
   if (!usedKeys.contains(attr.getName()))
     return emitError() << "unknown key '" << attr.getName() <<
         "' when parsing properties dictionary";
@@ -3376,11 +3376,13 @@ OpFormatParser::parseVariableImpl(SMLoc loc, StringRef name, Context ctx) {
     if (ctx == TopLevelContext || ctx == CustomDirectiveContext) {
       if (hasAllRegions || !seenRegions.insert(region).second)
         return emitError(loc, "region '" + name + "' is already bound");
-    } else if (ctx == RefDirectiveContext && !seenRegions.count(region)) {
-      return emitError(loc, "region '" + name +
-                                "' must be bound before it is referenced");
+    } else if (ctx == RefDirectiveContext) {
+      if (!seenRegions.count(region))
+        return emitError(loc, "region '" + name +
+                                  "' must be bound before it is referenced");
     } else {
-      return emitError(loc, "regions can only be used at the top level");
+      return emitError(loc, "regions can only be used at the top level "
+                            "or in a ref directive");
     }
     return create<RegionVariable>(region);
   }
@@ -3396,11 +3398,13 @@ OpFormatParser::parseVariableImpl(SMLoc loc, StringRef name, Context ctx) {
     if (ctx == TopLevelContext || ctx == CustomDirectiveContext) {
       if (hasAllSuccessors || !seenSuccessors.insert(successor).second)
         return emitError(loc, "successor '" + name + "' is already bound");
-    } else if (ctx == RefDirectiveContext && !seenSuccessors.count(successor)) {
-      return emitError(loc, "successor '" + name +
-                                "' must be bound before it is referenced");
+    } else if (ctx == RefDirectiveContext) {
+      if (!seenSuccessors.count(successor))
+        return emitError(loc, "successor '" + name +
+                                  "' must be bound before it is referenced");
     } else {
-      return emitError(loc, "successors can only be used at the top level");
+      return emitError(loc, "successors can only be used at the top level "
+                            "or in a ref directive");
     }
 
     return create<SuccessorVariable>(successor);
@@ -3775,7 +3779,8 @@ LogicalResult OpFormatParser::verifyOptionalGroupElement(SMLoc loc,
         Property prop = propEle->getVar()->prop;
         if (isAnchor && !(prop.hasDefaultValue() && prop.hasOptionalParser()))
           return emitError(loc, "only properties with default values "
-                                "that can be optionally parsed "
+                                "that can be optionally parsed (have the `let "
+                                "optionalParser = ...` field defined) "
                                 "can be used to anchor an optional group");
         return success();
       })
@@ -3848,8 +3853,14 @@ void mlir::tblgen::generateOpFormat(const Operator &constOp, OpClass &opClass,
   // TODO: Operator doesn't expose all necessary functionality via
   // the const interface.
   Operator &op = const_cast<Operator &>(constOp);
-  if (!op.hasAssemblyFormat())
+  if (!op.hasAssemblyFormat()) {
+    // We still need to generate the parsed attribute properties setter for
+    // allowing it to be reused in custom assembly implementations.
+    OperationFormat format(op, hasProperties);
+    format.hasPropDict = true;
+    genParsedAttrPropertiesSetter(format, op, opClass);
     return;
+  }
 
   // Parse the format description.
   llvm::SourceMgr mgr;
diff --git a/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp b/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
index 5b48e80749c8..f688fa97e840 100644
--- a/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
+++ b/mlir/unittests/Debug/FileLineColLocBreakpointManagerTest.cpp
@@ -23,9 +23,9 @@ static Operation *createOp(MLIRContext *context, Location loc,
                            StringRef operationName,
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
-  return Operation::create(loc, OperationName(operationName, context),
-                           std::nullopt, std::nullopt, std::nullopt,
-                           OpaqueProperties(nullptr), std::nullopt, numRegions);
+  return Operation::create(loc, OperationName(operationName, context), {}, {},
+                           std::nullopt, OpaqueProperties(nullptr), {},
+                           numRegions);
 }
 
 namespace {
diff --git a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
index 13c83c00d152..ffe815b9be5e 100644
--- a/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
+++ b/mlir/unittests/Dialect/SPIRV/DeserializationTest.cpp
@@ -188,11 +188,11 @@ TEST_F(DeserializationTest, OpMemberNameSuccess) {
   std::swap(typeDecl, binary);
 
   SmallVector<uint32_t, 5> operands1 = {structType, 0};
-  (void)spirv::encodeStringLiteralInto(operands1, "i1");
+  spirv::encodeStringLiteralInto(operands1, "i1");
   addInstruction(spirv::Opcode::OpMemberName, operands1);
 
   SmallVector<uint32_t, 5> operands2 = {structType, 1};
-  (void)spirv::encodeStringLiteralInto(operands2, "i2");
+  spirv::encodeStringLiteralInto(operands2, "i2");
   addInstruction(spirv::Opcode::OpMemberName, operands2);
 
   binary.append(typeDecl.begin(), typeDecl.end());
@@ -227,7 +227,7 @@ TEST_F(DeserializationTest, OpMemberNameExcessOperands) {
   std::swap(typeDecl, binary);
 
   SmallVector<uint32_t, 5> operands = {structType, 0};
-  (void)spirv::encodeStringLiteralInto(operands, "int32");
+  spirv::encodeStringLiteralInto(operands, "int32");
   operands.push_back(42);
   addInstruction(spirv::Opcode::OpMemberName, operands);
 
diff --git a/mlir/unittests/IR/AffineExprTest.cpp b/mlir/unittests/IR/AffineExprTest.cpp
index 8a2d697540d5..f8494ecb971c 100644
--- a/mlir/unittests/IR/AffineExprTest.cpp
+++ b/mlir/unittests/IR/AffineExprTest.cpp
@@ -84,6 +84,20 @@ TEST(AffineExprTest, constantFolding) {
   ASSERT_EQ(cminfloordivcn1.getKind(), AffineExprKind::FloorDiv);
 }
 
+TEST(AffineExprTest, commutative) {
+  MLIRContext ctx;
+  OpBuilder b(&ctx);
+  auto c2 = b.getAffineConstantExpr(1);
+  auto d0 = b.getAffineDimExpr(0);
+  auto d1 = b.getAffineDimExpr(1);
+  auto s0 = b.getAffineSymbolExpr(0);
+  auto s1 = b.getAffineSymbolExpr(1);
+
+  ASSERT_EQ(d0 * d1, d1 * d0);
+  ASSERT_EQ(s0 + s1, s1 + s0);
+  ASSERT_EQ(s0 * c2, c2 * s0);
+}
+
 TEST(AffineExprTest, divisionSimplification) {
   MLIRContext ctx;
   OpBuilder b(&ctx);
@@ -147,3 +161,12 @@ TEST(AffineExprTest, simpleAffineExprFlattenerRegression) {
   ASSERT_TRUE(isa<AffineConstantExpr>(result));
   ASSERT_EQ(cast<AffineConstantExpr>(result).getValue(), 7);
 }
+
+TEST(AffineExprTest, simplifyCommutative) {
+  MLIRContext ctx;
+  OpBuilder b(&ctx);
+  auto s0 = b.getAffineSymbolExpr(0);
+  auto s1 = b.getAffineSymbolExpr(1);
+
+  ASSERT_EQ(s0 * s1 - s1 * s0 + 1, 1);
+}
diff --git a/mlir/unittests/IR/OperationSupportTest.cpp b/mlir/unittests/IR/OperationSupportTest.cpp
index 6ea74a988ca8..4b800fa36a37 100644
--- a/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/mlir/unittests/IR/OperationSupportTest.cpp
@@ -22,9 +22,9 @@ static Operation *createOp(MLIRContext *context, ArrayRef<Value> operands = {},
                            ArrayRef<Type> resultTypes = {},
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
-  return Operation::create(
-      UnknownLoc::get(context), OperationName("foo.bar", context), resultTypes,
-      operands, std::nullopt, nullptr, std::nullopt, numRegions);
+  return Operation::create(UnknownLoc::get(context),
+                           OperationName("foo.bar", context), resultTypes,
+                           operands, std::nullopt, nullptr, {}, numRegions);
 }
 
 namespace {
@@ -33,7 +33,7 @@ TEST(OperandStorageTest, NonResizable) {
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a non-resizable operation with one operand.
@@ -44,7 +44,7 @@ TEST(OperandStorageTest, NonResizable) {
   EXPECT_EQ(user->getNumOperands(), 1u);
 
   // Removing is okay.
-  user->setOperands(std::nullopt);
+  user->setOperands({});
   EXPECT_EQ(user->getNumOperands(), 0u);
 
   // Destroy the operations.
@@ -57,7 +57,7 @@ TEST(OperandStorageTest, Resizable) {
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a resizable operation with one operand.
@@ -68,7 +68,7 @@ TEST(OperandStorageTest, Resizable) {
   EXPECT_EQ(user->getNumOperands(), 1u);
 
   // Removing is okay.
-  user->setOperands(std::nullopt);
+  user->setOperands({});
   EXPECT_EQ(user->getNumOperands(), 0u);
 
   // Adding more operands is okay.
@@ -85,7 +85,7 @@ TEST(OperandStorageTest, RangeReplace) {
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a resizable operation with one operand.
@@ -121,7 +121,7 @@ TEST(OperandStorageTest, MutableRange) {
   Builder builder(&context);
 
   Operation *useOp =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value operand = useOp->getResult(0);
 
   // Create a resizable operation with one operand.
@@ -158,8 +158,7 @@ TEST(OperandStorageTest, RangeErase) {
   Builder builder(&context);
 
   Type type = builder.getNoneType();
-  Operation *useOp =
-      createOp(&context, /*operands=*/std::nullopt, {type, type});
+  Operation *useOp = createOp(&context, /*operands=*/{}, {type, type});
   Value operand1 = useOp->getResult(0);
   Value operand2 = useOp->getResult(1);
 
@@ -189,8 +188,8 @@ TEST(OperationOrderTest, OrderIsAlwaysValid) {
   MLIRContext context;
   Builder builder(&context);
 
-  Operation *containerOp = createOp(&context, /*operands=*/std::nullopt,
-                                    /*resultTypes=*/std::nullopt,
+  Operation *containerOp = createOp(&context, /*operands=*/{},
+                                    /*resultTypes=*/{},
                                     /*numRegions=*/1);
   Region &region = containerOp->getRegion(0);
   Block *block = new Block();
@@ -236,8 +235,8 @@ TEST(OperationFormatPrintTest, CanPrintNameAsPrefix) {
   context.allowUnregisteredDialects();
   Operation *op = Operation::create(
       NameLoc::get(StringAttr::get(&context, "my_named_loc")),
-      OperationName("t.op", &context), builder.getIntegerType(16), std::nullopt,
-      std::nullopt, nullptr, std::nullopt, 0);
+      OperationName("t.op", &context), builder.getIntegerType(16), {},
+      std::nullopt, nullptr, {}, 0);
 
   std::string str;
   OpPrintingFlags flags;
diff --git a/mlir/unittests/IR/ValueTest.cpp b/mlir/unittests/IR/ValueTest.cpp
index 58678224780b..fc671be39f1e 100644
--- a/mlir/unittests/IR/ValueTest.cpp
+++ b/mlir/unittests/IR/ValueTest.cpp
@@ -20,9 +20,9 @@ static Operation *createOp(MLIRContext *context, ArrayRef<Value> operands = {},
                            ArrayRef<Type> resultTypes = {},
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
-  return Operation::create(
-      UnknownLoc::get(context), OperationName("foo.bar", context), resultTypes,
-      operands, std::nullopt, nullptr, std::nullopt, numRegions);
+  return Operation::create(UnknownLoc::get(context),
+                           OperationName("foo.bar", context), resultTypes,
+                           operands, std::nullopt, nullptr, {}, numRegions);
 }
 
 namespace {
@@ -32,7 +32,7 @@ TEST(ValueTest, getNumUses) {
   Builder builder(&context);
 
   Operation *op0 =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
 
   Value v0 = op0->getResult(0);
   EXPECT_EQ(v0.getNumUses(), (unsigned)0);
@@ -53,7 +53,7 @@ TEST(ValueTest, hasNUses) {
   Builder builder(&context);
 
   Operation *op0 =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value v0 = op0->getResult(0);
   EXPECT_TRUE(v0.hasNUses(0));
   EXPECT_FALSE(v0.hasNUses(1));
@@ -77,7 +77,7 @@ TEST(ValueTest, hasNUsesOrMore) {
   Builder builder(&context);
 
   Operation *op0 =
-      createOp(&context, /*operands=*/std::nullopt, builder.getIntegerType(16));
+      createOp(&context, /*operands=*/{}, builder.getIntegerType(16));
   Value v0 = op0->getResult(0);
   EXPECT_TRUE(v0.hasNUsesOrMore(0));
   EXPECT_FALSE(v0.hasNUsesOrMore(1));
diff --git a/mlir/unittests/Pass/AnalysisManagerTest.cpp b/mlir/unittests/Pass/AnalysisManagerTest.cpp
index d494db86a9de..1a28659f47e6 100644
--- a/mlir/unittests/Pass/AnalysisManagerTest.cpp
+++ b/mlir/unittests/Pass/AnalysisManagerTest.cpp
@@ -63,9 +63,8 @@ TEST(AnalysisManagerTest, FineGrainFunctionAnalysisPreservation) {
 
   // Create a function and a module.
   OwningOpRef<ModuleOp> module(ModuleOp::create(UnknownLoc::get(&context)));
-  func::FuncOp func1 =
-      func::FuncOp::create(builder.getUnknownLoc(), "foo",
-                           builder.getFunctionType(std::nullopt, std::nullopt));
+  func::FuncOp func1 = func::FuncOp::create(builder.getUnknownLoc(), "foo",
+                                            builder.getFunctionType({}, {}));
   func1.setPrivate();
   module->push_back(func1);
 
@@ -94,9 +93,8 @@ TEST(AnalysisManagerTest, FineGrainChildFunctionAnalysisPreservation) {
 
   // Create a function and a module.
   OwningOpRef<ModuleOp> module(ModuleOp::create(UnknownLoc::get(&context)));
-  func::FuncOp func1 =
-      func::FuncOp::create(builder.getUnknownLoc(), "foo",
-                           builder.getFunctionType(std::nullopt, std::nullopt));
+  func::FuncOp func1 = func::FuncOp::create(builder.getUnknownLoc(), "foo",
+                                            builder.getFunctionType({}, {}));
   func1.setPrivate();
   module->push_back(func1);
 
diff --git a/mlir/unittests/Pass/PassManagerTest.cpp b/mlir/unittests/Pass/PassManagerTest.cpp
index 7ceed3bb3bc3..7e618811eabf 100644
--- a/mlir/unittests/Pass/PassManagerTest.cpp
+++ b/mlir/unittests/Pass/PassManagerTest.cpp
@@ -63,9 +63,8 @@ TEST(PassManagerTest, OpSpecificAnalysis) {
   // Create a module with 2 functions.
   OwningOpRef<ModuleOp> module(ModuleOp::create(UnknownLoc::get(&context)));
   for (StringRef name : {"secret", "not_secret"}) {
-    auto func = func::FuncOp::create(
-        builder.getUnknownLoc(), name,
-        builder.getFunctionType(std::nullopt, std::nullopt));
+    auto func = func::FuncOp::create(builder.getUnknownLoc(), name,
+                                     builder.getFunctionType({}, {}));
     func.setPrivate();
     module->push_back(func);
   }
@@ -125,9 +124,8 @@ TEST(PassManagerTest, ExecutionAction) {
 
   // Create a module with 2 functions.
   OwningOpRef<ModuleOp> module(ModuleOp::create(UnknownLoc::get(&context)));
-  auto f =
-      func::FuncOp::create(builder.getUnknownLoc(), "process_me_once",
-                           builder.getFunctionType(std::nullopt, std::nullopt));
+  auto f = func::FuncOp::create(builder.getUnknownLoc(), "process_me_once",
+                                builder.getFunctionType({}, {}));
   f.setPrivate();
   module->push_back(f);
 
diff --git a/mlir/unittests/Transforms/DialectConversion.cpp b/mlir/unittests/Transforms/DialectConversion.cpp
index 10d7fb041278..7bb27f721414 100644
--- a/mlir/unittests/Transforms/DialectConversion.cpp
+++ b/mlir/unittests/Transforms/DialectConversion.cpp
@@ -13,9 +13,9 @@ using namespace mlir;
 
 static Operation *createOp(MLIRContext *context) {
   context->allowUnregisteredDialects();
-  return Operation::create(
-      UnknownLoc::get(context), OperationName("foo.bar", context), std::nullopt,
-      std::nullopt, std::nullopt, /*properties=*/nullptr, std::nullopt, 0);
+  return Operation::create(UnknownLoc::get(context),
+                           OperationName("foo.bar", context), {}, {},
+                           std::nullopt, /*properties=*/nullptr, {}, 0);
 }
 
 namespace {
diff --git a/offload/DeviceRTL/CMakeLists.txt b/offload/DeviceRTL/CMakeLists.txt
index 50aae7c7ecac..3901fbee4d6d 100644
--- a/offload/DeviceRTL/CMakeLists.txt
+++ b/offload/DeviceRTL/CMakeLists.txt
@@ -205,7 +205,7 @@ function(compileDeviceRTLLibrary target_name target_triple)
   add_dependencies(omptarget.${target_name} libomptarget-${target_name})
   target_link_libraries(omptarget.${target_name} PRIVATE omptarget.${target_name}.all_objs)
   target_link_options(omptarget.${target_name} PRIVATE "--target=${target_triple}"
-                      "-Wno-unused-command-line-argument""-r" "-nostdlib" "-flto"
+                      "-Wno-unused-command-line-argument" "-r" "-nostdlib" "-flto"
                        "-Wl,--lto-emit-llvm" "-fuse-ld=lld" "-march=" "-mcpu=")
 
   install(TARGETS omptarget.${target_name}
diff --git a/offload/cmake/caches/AMDGPULibcBot.cmake b/offload/cmake/caches/AMDGPULibcBot.cmake
index 728dfe3f0a3f..a772043c7966 100644
--- a/offload/cmake/caches/AMDGPULibcBot.cmake
+++ b/offload/cmake/caches/AMDGPULibcBot.cmake
@@ -18,3 +18,4 @@ set(CLANG_DEFAULT_RTLIB "compiler-rt" STRING "")
 
 set(LLVM_RUNTIME_TARGETS default;amdgcn-amd-amdhsa CACHE STRING "")
 set(RUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES "compiler-rt;libc" CACHE STRING "")
+set(RUNTIMES_amdgcn-amd-amdhsa_LIBC_GPU_TEST_JOBS 4 CACHE STRING "")
diff --git a/offload/liboffload/API/Common.td b/offload/liboffload/API/Common.td
index cd8c3c63fde8..850a01d06759 100644
--- a/offload/liboffload/API/Common.td
+++ b/offload/liboffload/API/Common.td
@@ -1,3 +1,15 @@
+//===-- Common.td - Common definitions for Offload ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains shared Offload API definitions
+//
+//===----------------------------------------------------------------------===//
+
 def : Macro {
   let name = "OL_VERSION_MAJOR";
   let desc = "Major version of the Offload API";
@@ -104,6 +116,7 @@ def ErrorCode : Enum {
     Etor<"UNIMPLEMENTED", "generic error code for features currently unimplemented by the device/backend">,
     Etor<"UNSUPPORTED", "generic error code for features unsupported by the device/backend">,
     Etor<"ASSEMBLE_FAILURE", "assembler failure while processing binary image">,
+    Etor<"COMPILE_FAILURE", "jit compile failure while processing binary image">,
     Etor<"LINK_FAILURE", "linker failure while processing binary image">,
     Etor<"BACKEND_FAILURE", "the plugin backend is in an invalid or unsupported state">,
     Etor<"UNINITIALIZED", "not initialized">,
@@ -175,7 +188,7 @@ def : Function {
   let desc = "Release the resources in use by Offload";
   let details = [
     "This decrements an internal reference count. When this reaches 0, all resources will be released",
-    "Subsequent API calls made after this are not valid"
+    "Subsequent API calls to methods other than `olInit` made after resources are released will return OL_ERRC_UNINITIALIZED"
   ];
   let params = [];
   let returns = [];
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
index 4abc24f3ba27..94bd6cbf0e5b 100644
--- a/offload/liboffload/API/Device.td
+++ b/offload/liboffload/API/Device.td
@@ -31,7 +31,8 @@ def : Enum {
     TaggedEtor<"PLATFORM", "ol_platform_handle_t", "the platform associated with the device">,
     TaggedEtor<"NAME", "char[]", "Device name">,
     TaggedEtor<"VENDOR", "char[]", "Device vendor">,
-    TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">
+    TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">,
+    TaggedEtor<"MAX_WORK_GROUP_SIZE", "ol_dimensions_t", "Maximum work group size in each dimension">,
   ];
 }
 
diff --git a/offload/liboffload/API/Program.td b/offload/liboffload/API/Program.td
index 8c88fe6e21e6..0476fa1f7c27 100644
--- a/offload/liboffload/API/Program.td
+++ b/offload/liboffload/API/Program.td
@@ -13,7 +13,9 @@
 def : Function {
     let name = "olCreateProgram";
     let desc = "Create a program for the device from the binary image pointed to by `ProgData`.";
-    let details = [];
+    let details = [
+        "The provided `ProgData` will be copied and need not outlive the returned handle",
+    ];
     let params = [
         Param<"ol_device_handle_t", "Device", "handle of the device", PARAM_IN>,
         Param<"const void*", "ProgData", "pointer to the program binary data", PARAM_IN>,
diff --git a/offload/liboffload/API/README.md b/offload/liboffload/API/README.md
index fda1ad39fa93..dad88df6b876 100644
--- a/offload/liboffload/API/README.md
+++ b/offload/liboffload/API/README.md
@@ -1,8 +1,5 @@
 # Offload API definitions
 
-**Note**: This is a work-in-progress. It is loosely based on equivalent
-tooling in Unified Runtime.
-
 The Tablegen files in this directory are used to define the Offload API. They
 are used with the `offload-tblgen` tool to generate API headers, print headers,
 and other implementation details.
@@ -10,17 +7,48 @@ and other implementation details.
 The root file is `OffloadAPI.td` - additional `.td` files can be included in
 this file to add them to the API.
 
+## Modifying the API
+
+API modifications, including additions, can be made by modifying the existing
+`.td` files. It is also possible to add a new tablegen file to the API by adding
+it to the includes in `OffloadAPI.td`. When Offload is rebuilt the new
+definition will be included in the generated files.
+
+Most API changes and additions do not require any additional work beyond this,
+other than new functions which are described below.
+
+### Adding a new function to the API
+
+When a new function is added (e.g. `offloadDeviceFoo`), the actual entry
+point is automatically generated, which contains validation and tracing code.
+It expects an implementation function (`offloadDeviceFoo_impl`) to be defined,
+which it will call into. The definition of this implementation function should
+be added to `liboffload/src/OffloadImpl.cpp`
+
+In short, the steps to add a new function are:
+* Add the new function definition to the `.td` files.
+* Build the `LLVMOffload` target. The relevant files will be regenerated, but
+  the library will fail to link because it is missing the implementation
+  function.
+* Add the new implementation function to `liboffload/src/OffloadImpl.cpp`. You
+  can copy the new function declaration from the generated
+  `OffloadImplFuncDecls.inc` file.
+* Rebuild `LLVMOffload`
+
 ## API Objects
+
 The API consists of a number of objects, which always have a *name* field and
 *description* field, and are one of the following types:
 
 ### Function
+
 Represents an API entry point function. Has a list of returns and parameters.
-Also has fields for details (representing a bullet-point list of
-information about the function that would otherwise be too detailed for the
-description), and analogues (equivalent functions in other APIs).
+Also has fields for details (representing a bullet-point list of information
+about the function that would otherwise be too detailed for the description),
+and analogues (equivalent functions in other APIs).
 
 #### Parameter
+
 Represents a parameter to a function, has *type*, *name*, and *desc* fields.
 Also has a *flags* field containing flags representing whether the parameter is
 in, out, or optional.
@@ -30,17 +58,24 @@ A *handle* type is a pointer to an opaque struct, used to abstract over
 plugin-specific implementation details.
 
 There are two special variants of a *parameter*:
-* **RangedParameter** - Represents a parameter that has a range described by other parameters. Generally these are pointers to an arbitrary number of objects. The range is used for generating validation and printing code. E.g, a range might be between `(0, NumDevices)`
-* **TypeTaggedParameter** - Represents a parameter (usually of `void*` type) that has the type and size of its pointee data described by other function parameters. The type is usually described by a type-tagged enum. This allows functions (e.g. `olGetDeviceInfo`) to return data of an arbitrary type.
+* **RangedParameter** - Represents a parameter that has a range described by
+  other parameters. Generally these are pointers to an arbitrary number of
+  objects. The range is used for generating validation and printing code. E.g,
+  a range might be between `(0, NumDevices)`
+* **TypeTaggedParameter** - Represents a parameter (usually of `void*` type)
+  that has the type and size of its pointee data described by other function
+  parameters. The type is usually described by a type-tagged enum. This allows
+  functions (e.g. `olGetDeviceInfo`) to return data of an arbitrary type.
 
 #### Return
+
 A return represents a possible return code from the function, and optionally a
 list of conditions in which this value may be returned. The conditions list is
-not expected to be exhaustive. A condition is considered free-form text, but
-if it is wrapped in \`backticks\` then it is treated as literal code
-representing an error condition (e.g. `someParam < 1`). These conditions are
-used to automatically create validation checks by the `offload-tblgen`
-validation generator.
+not expected to be exhaustive. A condition is considered free-form text, but if
+it is wrapped in \`backticks\` then it is treated as literal code representing
+an error condition (e.g. `someParam < 1`). These conditions are used to
+automatically create validation checks by the `offload-tblgen` validation
+generator.
 
 Returns are automatically generated for functions with pointer or handle
 parameters, so API authors do not need to exhaustively add null checks for
@@ -49,6 +84,7 @@ values automatically.
 
 
 ### Struct
+
 Represents a struct. Contains a list of members, which each have a *type*,
 *name*, and *desc*.
 
@@ -59,24 +95,28 @@ actual C++ inheritance, but instead explicitly has those members copied in,
 which preserves ABI compatibility with C.
 
 ### Enum
+
 Represents a C-style enum. Contains a list of `etor` values, which have a name
 and description.
 
-A `TaggedEtor` record type also exists which additionally takes a type. This type
-is used when the enum is used as a parameter to a function with a type-tagged
-function parameter (e.g. `olGetDeviceInfo`).
+A `TaggedEtor` record type also exists which additionally takes a type. This
+type is used when the enum is used as a parameter to a function with a
+type-tagged function parameter (e.g. `olGetDeviceInfo`).
 
 All enums automatically get a `<enum_name>_FORCE_UINT32 = 0x7fffffff` value,
 which forces the underlying type to be uint32.
 
 ### Handle
+
 Represents a pointer to an opaque struct, as described in the Parameter section.
 It does not take any extra fields.
 
 ### Typedef
+
 Represents a typedef, contains only a *value* field.
 
 ### Macro
+
 Represents a C preprocessor `#define`. Contains a *value* field. Optionally
 takes a *condition* field, which allows the macro to be conditionally defined,
 and an *alt_value* field, which represents the value if the condition is false.
@@ -90,15 +130,20 @@ files, rather than requiring a mix of C source and tablegen.
 ## Generation
 
 ### API header
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-api
 ```
 The comments in the generated header are in Doxygen format, although
 generating documentation from them hasn't been implemented yet.
 
-The entirety of this header is generated by Tablegen, rather than having a predefined header file that includes one or more `.inc` files. This is because this header is expected to be part of the installation and distributed to end-users, so should be self-contained.
+The entirety of this header is generated by Tablegen, rather than having a
+predefined header file that includes one or more `.inc` files. This is because
+this header is expected to be part of the installation and distributed to
+end-users, so should be self-contained.
 
 ### Entry Points
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-entry-points
 ```
@@ -111,6 +156,7 @@ function calls with arguments and results. The tracing can be enabled with the
 `OFFLOAD_TRACE` environment variable.
 
 ### Implementation function declarations
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-impl-func-decls
 ```
@@ -118,11 +164,12 @@ Generates declarations of the implementation of functions of every entry point
 in the API, e.g. `offloadDeviceFoo_impl` for `offloadDeviceFoo`.
 
 ### Print header
+
 ```
 ./offload-tblgen -I <path-to-llvm>/offload/API  <path-to-llvm>/offload/API/OffloadAPI.td --gen-print-header
 ```
-This header contains `std::ostream &operator<<(std::ostream&)` definitions for
-various API objects, including function parameters.
+This header contains `llvm::raw_ostream &operator<<(llvm::raw_ostream &)`
+definitions for various API objects, including function parameters.
 
 As with the API header, it is expected that this header is part of the installed
 package, so it is entirely generated by Tablegen.
@@ -130,21 +177,7 @@ package, so it is entirely generated by Tablegen.
 For ease of implementation, and since it is not strictly part of the API, this
 is a C++ header file. If a C version is desirable it could be added.
 
-### Future Tablegen backends
-`RecordTypes.hpp` contains wrappers for all of the API object types, which will
-allow more backends to be easily added in future.
-
-## Adding to the API
-
-A new object can be added to the API by adding to one of the existing `.td`
-files. It is also possible to add a new tablegen file to the API by adding it
-to the includes in `OffloadAPI.td`. When the `OffloadGenerate` target is
-rebuilt, the new definition will be included in the generated files.
-
-### Adding a new entry point
+### Additional Tablegen backends
 
-When a new entry point is added (e.g. `offloadDeviceFoo`), the actual entry
-point is automatically generated, which contains validation and tracing code.
-It expects an implementation function (`offloadDeviceFoo_impl`) to be defined,
-which it will call into. The definition of this implementation function should
-be added to `src/OffloadImpl.cpp`
+`RecordTypes.hpp` contains wrappers for all of the API object types, which
+allows new backends to be easily added if needed.
diff --git a/offload/liboffload/README.md b/offload/liboffload/README.md
index 95c9bf54d7ba..267d27172828 100644
--- a/offload/liboffload/README.md
+++ b/offload/liboffload/README.md
@@ -1,8 +1,34 @@
-# Offload New API
+# Liboffload
 
-This directory contains the implementation of the experimental work-in-progress
-new API for Offload. It builds on top of the existing plugin implementations but
-provides a single level of abstraction suitable for runtimes for languages other
-than OpenMP to be built on top of.
+This directory contains the implementation of the work-in-progress new API for
+Offload. It builds on top of the existing plugin implementations but provides a
+single level of abstraction suitable for implementation of many offloading
+language runtimes, rather than just OpenMP.
 
-See the [API definition readme](API/README.md) for implementation details.
\ No newline at end of file
+## Testing liboffload
+
+The main test suite for liboffload can be run with the `check-offload-unit`
+target, which runs the `offload.unittests` executable. The test suite will
+automatically run on every available device, but can be restricted to a single
+platform (CUDA, AMDGPU) with a command line argument:
+
+```sh
+$ ./offload.unittests --platform=CUDA
+```
+
+Tracing of Offload API calls can be enabled by setting the `OFFLOAD_TRACE`
+environment variable. This works with any program that uses liboffload.
+
+```sh
+$ OFFLOAD_TRACE=1 ./offload.unittests
+---> olInit()-> OL_SUCCESS
+# etc
+```
+
+The host plugin is not currently supported.
+
+## Modifying liboffload
+
+The main header (`OffloadAPI.h`) and some implementation details are
+autogenerated with tablegen. See the [API definition README](API/README.md)
+for implementation details.
diff --git a/offload/liboffload/src/Helpers.hpp b/offload/liboffload/src/Helpers.hpp
index 425934b6760d..8b85945508b9 100644
--- a/offload/liboffload/src/Helpers.hpp
+++ b/offload/liboffload/src/Helpers.hpp
@@ -61,39 +61,41 @@ llvm::Error getInfoArray(size_t array_length, size_t ParamValueSize,
                      array_length * sizeof(T), memcpy);
 }
 
-template <>
-inline llvm::Error
-getInfo<const char *>(size_t ParamValueSize, void *ParamValue,
-                      size_t *ParamValueSizeRet, const char *Value) {
-  return getInfoArray(strlen(Value) + 1, ParamValueSize, ParamValue,
-                      ParamValueSizeRet, Value);
+llvm::Error getInfoString(size_t ParamValueSize, void *ParamValue,
+                          size_t *ParamValueSizeRet, llvm::StringRef Value) {
+  return getInfoArray(Value.size() + 1, ParamValueSize, ParamValue,
+                      ParamValueSizeRet, Value.data());
 }
 
-class ReturnHelper {
+class InfoWriter {
 public:
-  ReturnHelper(size_t ParamValueSize, void *ParamValue,
-               size_t *ParamValueSizeRet)
-      : ParamValueSize(ParamValueSize), ParamValue(ParamValue),
-        ParamValueSizeRet(ParamValueSizeRet) {}
+  InfoWriter(size_t Size, void *Target, size_t *SizeRet)
+      : Size(Size), Target(Target), SizeRet(SizeRet) {};
+  InfoWriter() = delete;
+  InfoWriter(InfoWriter &) = delete;
+  ~InfoWriter() = default;
 
-  // A version where in/out info size is represented by a single pointer
-  // to a value which is updated on return
-  ReturnHelper(size_t *ParamValueSize, void *ParamValue)
-      : ParamValueSize(*ParamValueSize), ParamValue(ParamValue),
-        ParamValueSizeRet(ParamValueSize) {}
+  template <typename T> llvm::Error write(llvm::Expected<T> &&Val) {
+    if (Val)
+      return getInfo(Size, Target, SizeRet, *Val);
+    return Val.takeError();
+  }
 
-  // Scalar return Value
-  template <class T> llvm::Error operator()(const T &t) {
-    return getInfo(ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  template <typename T>
+  llvm::Error writeArray(llvm::Expected<T> &&Val, size_t Elems) {
+    if (Val)
+      return getInfoArray(Elems, Size, Target, SizeRet, *Val);
+    return Val.takeError();
   }
 
-  // Array return Value
-  template <class T> llvm::Error operator()(const T *t, size_t s) {
-    return getInfoArray(s, ParamValueSize, ParamValue, ParamValueSizeRet, t);
+  llvm::Error writeString(llvm::Expected<llvm::StringRef> &&Val) {
+    if (Val)
+      return getInfoString(Size, Target, SizeRet, *Val);
+    return Val.takeError();
   }
 
-protected:
-  size_t ParamValueSize;
-  void *ParamValue;
-  size_t *ParamValueSizeRet;
+private:
+  size_t Size;
+  void *Target;
+  size_t *SizeRet;
 };
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index eba8e91ed688..f9da63843670 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -13,6 +13,7 @@
 
 #include "OffloadImpl.hpp"
 #include "Helpers.hpp"
+#include "OffloadPrint.hpp"
 #include "PluginManager.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <OffloadAPI.h>
@@ -43,18 +44,19 @@ using namespace error;
 // interface.
 struct ol_device_impl_t {
   ol_device_impl_t(int DeviceNum, GenericDeviceTy *Device,
-                   ol_platform_handle_t Platform)
-      : DeviceNum(DeviceNum), Device(Device), Platform(Platform) {}
+                   ol_platform_handle_t Platform, InfoTreeNode &&DevInfo)
+      : DeviceNum(DeviceNum), Device(Device), Platform(Platform),
+        Info(std::forward<InfoTreeNode>(DevInfo)) {}
   int DeviceNum;
   GenericDeviceTy *Device;
   ol_platform_handle_t Platform;
+  InfoTreeNode Info;
 };
 
 struct ol_platform_impl_t {
   ol_platform_impl_t(std::unique_ptr<GenericPluginTy> Plugin,
-                     std::vector<ol_device_impl_t> Devices,
                      ol_platform_backend_t BackendType)
-      : Plugin(std::move(Plugin)), Devices(Devices), BackendType(BackendType) {}
+      : Plugin(std::move(Plugin)), BackendType(BackendType) {}
   std::unique_ptr<GenericPluginTy> Plugin;
   std::vector<ol_device_impl_t> Devices;
   ol_platform_backend_t BackendType;
@@ -95,7 +97,10 @@ struct AllocInfo {
 
 // Global shared state for liboffload
 struct OffloadContext;
-static OffloadContext *OffloadContextVal;
+// This pointer is non-null if and only if the context is valid and fully
+// initialized
+static std::atomic<OffloadContext *> OffloadContextVal;
+std::mutex OffloadContextValMutex;
 struct OffloadContext {
   OffloadContext(OffloadContext &) = delete;
   OffloadContext(OffloadContext &&) = delete;
@@ -106,6 +111,7 @@ struct OffloadContext {
   bool ValidationEnabled = true;
   DenseMap<void *, AllocInfo> AllocInfoMap{};
   SmallVector<ol_platform_impl_t, 4> Platforms{};
+  size_t RefCount;
 
   ol_device_handle_t HostDevice() {
     // The host platform is always inserted last
@@ -144,21 +150,18 @@ constexpr ol_platform_backend_t pluginNameToBackend(StringRef Name) {
 #define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();
 #include "Shared/Targets.def"
 
-void initPlugins() {
-  auto *Context = new OffloadContext{};
-
+Error initPlugins(OffloadContext &Context) {
   // Attempt to create an instance of each supported plugin.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    Context->Platforms.emplace_back(ol_platform_impl_t{                        \
+    Context.Platforms.emplace_back(ol_platform_impl_t{                         \
         std::unique_ptr<GenericPluginTy>(createPlugin_##Name()),               \
-        {},                                                                    \
         pluginNameToBackend(#Name)});                                          \
   } while (false);
 #include "Shared/Targets.def"
 
   // Preemptively initialize all devices in the plugin
-  for (auto &Platform : Context->Platforms) {
+  for (auto &Platform : Context.Platforms) {
     // Do not use the host plugin - it isn't supported.
     if (Platform.BackendType == OL_PLATFORM_BACKEND_UNKNOWN)
       continue;
@@ -167,55 +170,87 @@ void initPlugins() {
     for (auto DevNum = 0; DevNum < Platform.Plugin->number_of_devices();
          DevNum++) {
       if (Platform.Plugin->init_device(DevNum) == OFFLOAD_SUCCESS) {
-        Platform.Devices.emplace_back(ol_device_impl_t{
-            DevNum, &Platform.Plugin->getDevice(DevNum), &Platform});
+        auto Device = &Platform.Plugin->getDevice(DevNum);
+        auto Info = Device->obtainInfoImpl();
+        if (auto Err = Info.takeError())
+          return Err;
+        Platform.Devices.emplace_back(DevNum, Device, &Platform,
+                                      std::move(*Info));
       }
     }
   }
 
   // Add the special host device
-  auto &HostPlatform = Context->Platforms.emplace_back(
-      ol_platform_impl_t{nullptr,
-                         {ol_device_impl_t{-1, nullptr, nullptr}},
-                         OL_PLATFORM_BACKEND_HOST});
-  Context->HostDevice()->Platform = &HostPlatform;
+  auto &HostPlatform = Context.Platforms.emplace_back(
+      ol_platform_impl_t{nullptr, OL_PLATFORM_BACKEND_HOST});
+  HostPlatform.Devices.emplace_back(-1, nullptr, nullptr, InfoTreeNode{});
+  Context.HostDevice()->Platform = &HostPlatform;
 
-  Context->TracingEnabled = std::getenv("OFFLOAD_TRACE");
-  Context->ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");
+  Context.TracingEnabled = std::getenv("OFFLOAD_TRACE");
+  Context.ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");
 
-  OffloadContextVal = Context;
+  return Plugin::success();
 }
 
-// TODO: We can properly reference count here and manage the resources in a more
-// clever way
 Error olInit_impl() {
-  static std::once_flag InitFlag;
-  std::call_once(InitFlag, initPlugins);
+  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
 
-  return Error::success();
+  if (isOffloadInitialized()) {
+    OffloadContext::get().RefCount++;
+    return Plugin::success();
+  }
+
+  // Use a temporary to ensure that entry points querying OffloadContextVal do
+  // not get a partially initialized context
+  auto *NewContext = new OffloadContext{};
+  Error InitResult = initPlugins(*NewContext);
+  OffloadContextVal.store(NewContext);
+  OffloadContext::get().RefCount++;
+
+  return InitResult;
+}
+
+Error olShutDown_impl() {
+  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
+
+  if (--OffloadContext::get().RefCount != 0)
+    return Error::success();
+
+  llvm::Error Result = Error::success();
+  auto *OldContext = OffloadContextVal.exchange(nullptr);
+
+  for (auto &P : OldContext->Platforms) {
+    // Host plugin is nullptr and has no deinit
+    if (!P.Plugin)
+      continue;
+
+    if (auto Res = P.Plugin->deinit())
+      Result = llvm::joinErrors(std::move(Result), std::move(Res));
+  }
+
+  delete OldContext;
+  return Result;
 }
-Error olShutDown_impl() { return Error::success(); }
 
 Error olGetPlatformInfoImplDetail(ol_platform_handle_t Platform,
                                   ol_platform_info_t PropName, size_t PropSize,
                                   void *PropValue, size_t *PropSizeRet) {
-  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
   bool IsHost = Platform->BackendType == OL_PLATFORM_BACKEND_HOST;
 
   switch (PropName) {
   case OL_PLATFORM_INFO_NAME:
-    return ReturnValue(IsHost ? "Host" : Platform->Plugin->getName());
+    return Info.writeString(IsHost ? "Host" : Platform->Plugin->getName());
   case OL_PLATFORM_INFO_VENDOR_NAME:
     // TODO: Implement this
-    return ReturnValue("Unknown platform vendor");
+    return Info.writeString("Unknown platform vendor");
   case OL_PLATFORM_INFO_VERSION: {
-    return ReturnValue(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,
-                               OL_VERSION_MINOR, OL_VERSION_PATCH)
-                           .str()
-                           .c_str());
+    return Info.writeString(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,
+                                    OL_VERSION_MINOR, OL_VERSION_PATCH)
+                                .str());
   }
   case OL_PLATFORM_INFO_BACKEND: {
-    return ReturnValue(Platform->BackendType);
+    return Info.write<ol_platform_backend_t>(Platform->BackendType);
   }
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
@@ -242,43 +277,108 @@ Error olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,
 Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
                                 ol_device_info_t PropName, size_t PropSize,
                                 void *PropValue, size_t *PropSizeRet) {
+  assert(Device != OffloadContext::get().HostDevice());
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
 
-  ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
+  auto makeError = [&](ErrorCode Code, StringRef Err) {
+    std::string ErrBuffer;
+    llvm::raw_string_ostream(ErrBuffer) << PropName << ": " << Err;
+    return Plugin::error(ErrorCode::UNIMPLEMENTED, ErrBuffer.c_str());
+  };
 
   // Find the info if it exists under any of the given names
-  auto GetInfoString = [&](std::vector<std::string> Names) {
-    if (Device == OffloadContext::get().HostDevice())
-      return "Host";
-
-    if (!Device->Device)
-      return "";
+  auto getInfoString =
+      [&](std::vector<std::string> Names) -> llvm::Expected<const char *> {
+    for (auto &Name : Names) {
+      if (auto Entry = Device->Info.get(Name)) {
+        if (!std::holds_alternative<std::string>((*Entry)->Value))
+          return makeError(ErrorCode::BACKEND_FAILURE,
+                           "plugin returned incorrect type");
+        return std::get<std::string>((*Entry)->Value).c_str();
+      }
+    }
 
-    auto Info = Device->Device->obtainInfoImpl();
-    if (auto Err = Info.takeError())
-      return "";
+    return makeError(ErrorCode::UNIMPLEMENTED,
+                     "plugin did not provide a response for this information");
+  };
 
-    for (auto Name : Names) {
-      if (auto Entry = Info->get(Name))
-        return std::get<std::string>((*Entry)->Value).c_str();
+  auto getInfoXyz =
+      [&](std::vector<std::string> Names) -> llvm::Expected<ol_dimensions_t> {
+    for (auto &Name : Names) {
+      if (auto Entry = Device->Info.get(Name)) {
+        auto Node = *Entry;
+        ol_dimensions_t Out{0, 0, 0};
+
+        auto getField = [&](StringRef Name, uint32_t &Dest) {
+          if (auto F = Node->get(Name)) {
+            if (!std::holds_alternative<size_t>((*F)->Value))
+              return makeError(
+                  ErrorCode::BACKEND_FAILURE,
+                  "plugin returned incorrect type for dimensions element");
+            Dest = std::get<size_t>((*F)->Value);
+          } else
+            return makeError(ErrorCode::BACKEND_FAILURE,
+                             "plugin didn't provide all values for dimensions");
+          return Plugin::success();
+        };
+
+        if (auto Res = getField("x", Out.x))
+          return Res;
+        if (auto Res = getField("y", Out.y))
+          return Res;
+        if (auto Res = getField("z", Out.z))
+          return Res;
+
+        return Out;
+      }
     }
 
-    return "";
+    return makeError(ErrorCode::UNIMPLEMENTED,
+                     "plugin did not provide a response for this information");
   };
 
   switch (PropName) {
   case OL_DEVICE_INFO_PLATFORM:
-    return ReturnValue(Device->Platform);
+    return Info.write<void *>(Device->Platform);
+  case OL_DEVICE_INFO_TYPE:
+    return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_GPU);
+  case OL_DEVICE_INFO_NAME:
+    return Info.writeString(getInfoString({"Device Name"}));
+  case OL_DEVICE_INFO_VENDOR:
+    return Info.writeString(getInfoString({"Vendor Name"}));
+  case OL_DEVICE_INFO_DRIVER_VERSION:
+    return Info.writeString(
+        getInfoString({"CUDA Driver Version", "HSA Runtime Version"}));
+  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
+    return Info.write(getInfoXyz({"Workgroup Max Size per Dimension" /*AMD*/,
+                                  "Maximum Block Dimensions" /*CUDA*/}));
+  default:
+    return createOffloadError(ErrorCode::INVALID_ENUMERATION,
+                              "getDeviceInfo enum '%i' is invalid", PropName);
+  }
+
+  return Error::success();
+}
+
+Error olGetDeviceInfoImplDetailHost(ol_device_handle_t Device,
+                                    ol_device_info_t PropName, size_t PropSize,
+                                    void *PropValue, size_t *PropSizeRet) {
+  assert(Device == OffloadContext::get().HostDevice());
+  InfoWriter Info(PropSize, PropValue, PropSizeRet);
+
+  switch (PropName) {
+  case OL_DEVICE_INFO_PLATFORM:
+    return Info.write<void *>(Device->Platform);
   case OL_DEVICE_INFO_TYPE:
-    return Device == OffloadContext::get().HostDevice()
-               ? ReturnValue(OL_DEVICE_TYPE_HOST)
-               : ReturnValue(OL_DEVICE_TYPE_GPU);
+    return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_HOST);
   case OL_DEVICE_INFO_NAME:
-    return ReturnValue(GetInfoString({"Device Name"}));
+    return Info.writeString("Virtual Host Device");
   case OL_DEVICE_INFO_VENDOR:
-    return ReturnValue(GetInfoString({"Vendor Name"}));
+    return Info.writeString("Liboffload");
   case OL_DEVICE_INFO_DRIVER_VERSION:
-    return ReturnValue(
-        GetInfoString({"CUDA Driver Version", "HSA Runtime Version"}));
+    return Info.writeString(LLVM_VERSION_STRING);
+  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
+    return Info.write<ol_dimensions_t>(ol_dimensions_t{1, 1, 1});
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "getDeviceInfo enum '%i' is invalid", PropName);
@@ -289,12 +389,18 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
 
 Error olGetDeviceInfo_impl(ol_device_handle_t Device, ol_device_info_t PropName,
                            size_t PropSize, void *PropValue) {
+  if (Device == OffloadContext::get().HostDevice())
+    return olGetDeviceInfoImplDetailHost(Device, PropName, PropSize, PropValue,
+                                         nullptr);
   return olGetDeviceInfoImplDetail(Device, PropName, PropSize, PropValue,
                                    nullptr);
 }
 
 Error olGetDeviceInfoSize_impl(ol_device_handle_t Device,
                                ol_device_info_t PropName, size_t *PropSizeRet) {
+  if (Device == OffloadContext::get().HostDevice())
+    return olGetDeviceInfoImplDetailHost(Device, PropName, 0, nullptr,
+                                         PropSizeRet);
   return olGetDeviceInfoImplDetail(Device, PropName, 0, nullptr, PropSizeRet);
 }
 
@@ -471,6 +577,7 @@ Error olCreateProgram_impl(ol_device_handle_t Device, const void *ProgData,
     delete Prog;
     return Res.takeError();
   }
+  assert(*Res != nullptr && "loadBinary returned nullptr");
 
   Prog->Image = *Res;
   *Program = Prog;
@@ -479,6 +586,14 @@ Error olCreateProgram_impl(ol_device_handle_t Device, const void *ProgData,
 }
 
 Error olDestroyProgram_impl(ol_program_handle_t Program) {
+  auto &Device = Program->Image->getDevice();
+  if (auto Err = Device.unloadBinary(Program->Image))
+    return Err;
+
+  auto &LoadedImages = Device.LoadedImages;
+  LoadedImages.erase(
+      std::find(LoadedImages.begin(), LoadedImages.end(), Program->Image));
+
   return olDestroy(Program);
 }
 
diff --git a/offload/libomptarget/OpenMP/Mapping.cpp b/offload/libomptarget/OpenMP/Mapping.cpp
index 2cbd260112fa..8281a3b30f4b 100644
--- a/offload/libomptarget/OpenMP/Mapping.cpp
+++ b/offload/libomptarget/OpenMP/Mapping.cpp
@@ -362,6 +362,30 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
   if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
       (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
 
+    // If we have something like:
+    //   #pragma omp target map(to: s.myarr[0:10]) map(to: s.myarr[0:10])
+    // then we see two "new" mappings of the struct member s.myarr here --
+    // and both have the "IsNewEntry" flag set, so trigger the copy to device
+    // below.  But, the shadow pointer is only initialised on the target for
+    // the first copy, and the second copy clobbers it.  So, this condition
+    // avoids the (second) copy here if we have already set shadow pointer info.
+    auto FailOnPtrFound = [HstPtrBegin, Size](ShadowPtrInfoTy &SP) {
+      if (SP.HstPtrAddr >= HstPtrBegin &&
+          SP.HstPtrAddr < (void *)((char *)HstPtrBegin + Size))
+        return OFFLOAD_FAIL;
+      return OFFLOAD_SUCCESS;
+    };
+    if (LR.TPR.getEntry()->foreachShadowPointerInfo(FailOnPtrFound) ==
+        OFFLOAD_FAIL) {
+      DP("Multiple new mappings of %" PRId64 " bytes detected (hst:" DPxMOD
+         ") -> (tgt:" DPxMOD ")\n",
+         Size, DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
+      return std::move(LR.TPR);
+    }
+
+    DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
+       DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
+
     if (LR.TPR.Flags.IsNewEntry ||
         LR.TPR.getEntry()->AllocKind != TARGET_ALLOC_SHARED) {
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index a33f00f7bbf8..409d9708b3e2 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -3312,6 +3312,13 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  Error unloadBinaryImpl(DeviceImageTy *Image) override {
+    AMDGPUDeviceImageTy &AMDImage = static_cast<AMDGPUDeviceImageTy &>(*Image);
+
+    // Unload the executable of the image.
+    return AMDImage.unloadExecutable();
+  }
+
   /// Deinitialize the device and release its resources.
   Error deinitImpl() override {
     // Deinitialize the stream and event pools.
@@ -3324,19 +3331,6 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = AMDGPUSignalManager.deinit())
       return Err;
 
-    // Close modules if necessary.
-    if (!LoadedImages.empty()) {
-      // Each image has its own module.
-      for (DeviceImageTy *Image : LoadedImages) {
-        AMDGPUDeviceImageTy &AMDImage =
-            static_cast<AMDGPUDeviceImageTy &>(*Image);
-
-        // Unload the executable of the image.
-        if (auto Err = AMDImage.unloadExecutable())
-          return Err;
-      }
-    }
-
     // Invalidate agent reference.
     Agent = {0};
 
@@ -4013,6 +4007,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       case HSA_DEVICE_TYPE_DSP:
         TmpCharPtr = "DSP";
         break;
+      default:
+        TmpCharPtr = "Unknown";
+        break;
       }
       Info.add("Device Type", TmpCharPtr);
     }
@@ -5431,6 +5428,9 @@ static Error Plugin::check(int32_t Code, const char *ErrFmt, ArgsTy... Args) {
   case HSA_STATUS_ERROR_INVALID_SYMBOL_NAME:
     OffloadErrCode = ErrorCode::NOT_FOUND;
     break;
+  case HSA_STATUS_ERROR_INVALID_CODE_OBJECT:
+    OffloadErrCode = ErrorCode::INVALID_BINARY;
+    break;
   default:
     OffloadErrCode = ErrorCode::UNKNOWN;
   }
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index edb444cf29c1..791574e08496 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -42,7 +42,7 @@ endif()
 # Include the RPC server from the `libc` project.
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
-if(OFFLOAD_ENABLE_EMISSARY_APIS)
+if(OFFLOAD_ENABLE_EMISSARY_APIS AND LIBOMPTARGET_BUILD_DEVICE_FORTRT)
   target_link_libraries(PluginCommon PRIVATE flang_rt.runtime
   -L${CMAKE_BINARY_DIR}/../../lib  -L${CMAKE_INSTALL_PREFIX}/lib)
 endif()
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index be3c74eb1a22..9609d3f29d2e 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -319,7 +319,7 @@ struct GenericKernelTy {
                            AsyncInfoWrapperTy &AsyncInfoWrapper) const = 0;
 
   /// Get the kernel name.
-  const char *getName() const { return Name; }
+  const char *getName() const { return Name.c_str(); }
 
   /// Get the kernel image.
   DeviceImageTy &getImage() const {
@@ -482,7 +482,7 @@ struct GenericKernelTy {
                                 bool IsNumThreadsFromUser) const;
 
   /// The kernel name.
-  const char *Name;
+  std::string Name;
 
   /// The execution flags of the kernel.
   OMPTgtExecModeFlags ExecutionMode;
@@ -835,6 +835,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   virtual Expected<DeviceImageTy *>
   loadBinaryImpl(const __tgt_device_image *TgtImage, int32_t ImageId) = 0;
 
+  /// Unload a previously loaded Image from the device
+  Error unloadBinary(DeviceImageTy *Image);
+  virtual Error unloadBinaryImpl(DeviceImageTy *Image) = 0;
+
   /// Setup the device environment if needed. Notice this setup may not be run
   /// on some plugins. By default, it will be executed, but plugins can change
   /// this behavior by overriding the shouldSetupDeviceEnvironment function.
@@ -1238,6 +1242,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
 
   uint32_t getAndIncrementLaunchId() { return LaunchId.fetch_add(1); }
 
+  /// Array of images loaded into the device. Images are automatically
+  /// deallocated by the allocator.
+  llvm::SmallVector<DeviceImageTy *> LoadedImages;
+
 private:
   /// Get and set the stack size and heap size for the device. If not used, the
   /// plugin can implement the setters as no-op and setting the output
@@ -1300,10 +1308,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Envar to enable runtime tuning.
   BoolEnvar OMPX_EnableRuntimeAutotuning;
 
-  /// Array of images loaded into the device. Images are automatically
-  /// deallocated by the allocator.
-  llvm::SmallVector<DeviceImageTy *> LoadedImages;
-
   /// The identifier of the device within the plugin. Notice this is not a
   /// global device id and is not the device id visible to the OpenMP user.
   const int32_t DeviceId;
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 3dad6a845096..1809a7f7d605 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -504,11 +504,11 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
     KernelEnvironment = KernelEnvironmentTy{};
     DP("Failed to read kernel environment for '%s' Using default Bare (0) "
        "execution mode\n",
-       Name);
+       getName());
   }
 
   // Create a metadata object for the exec mode global (auto-generated).
-  StaticGlobalTy<llvm::omp::OMPTgtExecModeFlags> ExecModeGlobal(Name,
+  StaticGlobalTy<llvm::omp::OMPTgtExecModeFlags> ExecModeGlobal(getName(),
                                                                 "_exec_mode");
 
   // Retrieve execution mode for the kernel. This may fail since some kernels
@@ -519,7 +519,7 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
     [[maybe_unused]] std::string ErrStr = toString(std::move(Err));
     DP("Failed to read execution mode for '%s': %s\n"
        "Using default Bare (0) execution mode\n",
-       Name, ErrStr.data());
+       getName(), ErrStr.data());
 
     ExecutionMode = OMP_TGT_EXEC_MODE_BARE;
   } else {
@@ -527,12 +527,12 @@ Error GenericKernelTy::init(GenericDeviceTy &GenericDevice,
     if (!GenericKernelTy::isValidExecutionMode(ExecModeGlobal.getValue()))
       return Plugin::error(ErrorCode::UNKNOWN, 
                            "Invalid execution mode %d for '%s'",
-                           ExecModeGlobal.getValue(), Name);
+                           ExecModeGlobal.getValue(), getName());
     ExecutionMode = ExecModeGlobal.getValue();
   }
 
   // Create a metadata object for the multi-device global (auto-generated).
-  StaticGlobalTy<int8_t> MultiDeviceGlobal(Name, "_multi_device");
+  StaticGlobalTy<int8_t> MultiDeviceGlobal(getName(), "_multi_device");
   if (auto Err = GHandler.readGlobalFromImage(GenericDevice, Image,
                                               MultiDeviceGlobal)) {
     DP("Missing symbol %s, continue execution anyway.\n",
@@ -1039,27 +1039,50 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   return Plugin::success();
 }
 
-Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
+Error GenericDeviceTy::unloadBinary(DeviceImageTy *Image) {
   clear_ArgBufs();
-  for (DeviceImageTy *Image : LoadedImages)
-    if (auto Err = callGlobalDestructors(Plugin, *Image))
-      return Err;
+  if (auto Err = callGlobalDestructors(Plugin, *Image))
+    return Err;
 
   if (OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::AllocationTracker)) {
     GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler();
-    for (auto *Image : LoadedImages) {
-      DeviceMemoryPoolTrackingTy ImageDeviceMemoryPoolTracking = {0, 0, ~0U, 0};
-      GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker",
-                             sizeof(DeviceMemoryPoolTrackingTy),
-                             &ImageDeviceMemoryPoolTracking);
-      if (auto Err =
-              GHandler.readGlobalFromDevice(*this, *Image, TrackerGlobal)) {
-        consumeError(std::move(Err));
-        continue;
-      }
-      DeviceMemoryPoolTracking.combine(ImageDeviceMemoryPoolTracking);
+    DeviceMemoryPoolTrackingTy ImageDeviceMemoryPoolTracking = {0, 0, ~0U, 0};
+    GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker",
+                           sizeof(DeviceMemoryPoolTrackingTy),
+                           &ImageDeviceMemoryPoolTracking);
+    if (auto Err =
+            GHandler.readGlobalFromDevice(*this, *Image, TrackerGlobal)) {
+      consumeError(std::move(Err));
     }
+    DeviceMemoryPoolTracking.combine(ImageDeviceMemoryPoolTracking);
+  }
+
+  GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
+  auto ProfOrErr = Handler.readProfilingGlobals(*this, *Image);
+  if (!ProfOrErr)
+    return ProfOrErr.takeError();
+
+  if (!ProfOrErr->empty()) {
+    // Dump out profdata
+    if ((OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::PGODump)) ==
+        uint32_t(DeviceDebugKind::PGODump))
+      ProfOrErr->dump();
+
+    // Write data to profiling file
+    if (auto Err = ProfOrErr->write())
+      return Err;
+  }
 
+  return unloadBinaryImpl(Image);
+}
+
+Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
+  for (auto &I : LoadedImages)
+    if (auto Err = unloadBinary(I))
+      return Err;
+  LoadedImages.clear();
+
+  if (OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::AllocationTracker)) {
     // TODO: Write this by default into a file.
     printf("\n\n|-----------------------\n"
            "| Device memory tracker:\n"
@@ -1075,25 +1098,6 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
            DeviceMemoryPoolTracking.AllocationMax);
   }
 
-  for (auto *Image : LoadedImages) {
-    GenericGlobalHandlerTy &Handler = Plugin.getGlobalHandler();
-    auto ProfOrErr = Handler.readProfilingGlobals(*this, *Image);
-    if (!ProfOrErr)
-      return ProfOrErr.takeError();
-
-    if (ProfOrErr->empty())
-      continue;
-
-    // Dump out profdata
-    if ((OMPX_DebugKind.get() & uint32_t(DeviceDebugKind::PGODump)) ==
-        uint32_t(DeviceDebugKind::PGODump))
-      ProfOrErr->dump();
-
-    // Write data to profiling file
-    if (auto Err = ProfOrErr->write())
-      return Err;
-  }
-
   // Delete the memory manager before deinitializing the device. Otherwise,
   // we may delete device allocations after the device is deinitialized.
   if (MemoryManager)
@@ -1137,8 +1141,9 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (!PostJITImageOrErr) {
     auto Err = PostJITImageOrErr.takeError();
     REPORT("Failure to jit IR image %p on device %d: %s\n", InputTgtImage,
-           DeviceId, toString(std::move(Err)).data());
-    return nullptr;
+           DeviceId, toStringWithoutConsuming(Err).data());
+    return Plugin::error(ErrorCode::COMPILE_FAILURE, std::move(Err),
+                         "failure to jit IR image");
   }
 
   // Load the binary and allocate the image object. Use the next available id
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index af8d323aa585..14e7f41a2136 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -358,6 +358,19 @@ struct CUDADeviceTy : public GenericDeviceTy {
     return Plugin::success();
   }
 
+  Error unloadBinaryImpl(DeviceImageTy *Image) override {
+    assert(Context && "Invalid CUDA context");
+
+    // Each image has its own module.
+    CUDADeviceImageTy &CUDAImage = static_cast<CUDADeviceImageTy &>(*Image);
+
+    // Unload the module of the image.
+    if (auto Err = CUDAImage.unloadModule())
+      return Err;
+
+    return Plugin::success();
+  }
+
   /// Deinitialize the device and release its resources.
   Error deinitImpl() override {
     if (Context) {
@@ -372,20 +385,6 @@ struct CUDADeviceTy : public GenericDeviceTy {
     if (auto Err = CUDAEventManager.deinit())
       return Err;
 
-    // Close modules if necessary.
-    if (!LoadedImages.empty()) {
-      assert(Context && "Invalid CUDA context");
-
-      // Each image has its own module.
-      for (DeviceImageTy *Image : LoadedImages) {
-        CUDADeviceImageTy &CUDAImage = static_cast<CUDADeviceImageTy &>(*Image);
-
-        // Unload the module of the image.
-        if (auto Err = CUDAImage.unloadModule())
-          return Err;
-      }
-    }
-
     if (Context) {
       CUresult Res = cuDevicePrimaryCtxRelease(Device);
       if (auto Err =
@@ -931,7 +930,9 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
     CUresult Res = cuDriverGetVersion(&TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add("CUDA Driver Version", TmpInt);
+      // For consistency with other drivers, store the version as a string
+      // rather than an integer
+      Info.add("CUDA Driver Version", std::to_string(TmpInt));
 
     Info.add("CUDA OpenMP Device Number", DeviceId);
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index fe8b2b64b8e2..3cd48d6d659b 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -157,6 +157,12 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
   /// Initialize the device, which is a no-op
   Error initImpl(GenericPluginTy &Plugin) override { return Plugin::success(); }
 
+  /// Unload the binary image
+  ///
+  /// TODO: This currently does nothing, and should be implemented as part of
+  /// broader memory handling logic for this plugin
+  Error unloadBinaryImpl(DeviceImageTy *) override { return Plugin::success(); }
+
   /// Deinitialize the device, which is a no-op
   Error deinitImpl() override { return Plugin::success(); }
 
diff --git a/offload/test/mapping/duplicate_mappings_1.cpp b/offload/test/mapping/duplicate_mappings_1.cpp
new file mode 100644
index 000000000000..5bed3f161b4f
--- /dev/null
+++ b/offload/test/mapping/duplicate_mappings_1.cpp
@@ -0,0 +1,27 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic -Wno-openmp-mapping && %libomptarget-run-generic
+
+// clang-format on
+
+#include <assert.h>
+
+struct Inner {
+  int *data;
+  Inner(int size) { data = new int[size](); }
+  ~Inner() { delete[] data; }
+};
+
+struct Outer {
+  Inner i;
+  Outer() : i(10) {}
+};
+
+int main() {
+  Outer o;
+#pragma omp target map(tofrom : o.i.data[0 : 10]) map(tofrom : o.i.data[0 : 10])
+  {
+    o.i.data[0] = 42;
+  }
+  assert(o.i.data[0] == 42);
+  return 0;
+}
diff --git a/offload/test/mapping/duplicate_mappings_2.cpp b/offload/test/mapping/duplicate_mappings_2.cpp
new file mode 100644
index 000000000000..ca8112ee7254
--- /dev/null
+++ b/offload/test/mapping/duplicate_mappings_2.cpp
@@ -0,0 +1,29 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic -Wno-openmp-mapping && %libomptarget-run-generic
+
+#include <assert.h>
+
+// clang-format on
+
+struct Inner {
+  int *data;
+  Inner(int size) { data = new int[size](); }
+  ~Inner() { delete[] data; }
+};
+#pragma omp declare mapper(Inner i) map(i, i.data[0 : 10])
+
+struct Outer {
+  Inner i;
+  Outer() : i(10) {}
+};
+#pragma omp declare mapper(Outer o) map(o, o.i)
+
+int main() {
+  Outer o;
+#pragma omp target map(tofrom : o)
+  {
+    o.i.data[0] = 42;
+  }
+  assert(o.i.data[0] == 42);
+  return 0;
+}
diff --git a/offload/test/offloading/disable_default_device.c b/offload/test/offloading/disable_default_device.c
new file mode 100644
index 000000000000..77ea9d4669f1
--- /dev/null
+++ b/offload/test/offloading/disable_default_device.c
@@ -0,0 +1,22 @@
+// RUN: %libomptarget-compile-generic
+// RUN:   env OMP_TARGET_OFFLOAD=disabled %libomptarget-run-generic 2>&1 |
+// %fcheck-generic
+
+#include <omp.h>
+#include <stdio.h>
+
+// Sanity checks to make sure that this works and is thread safe.
+int main() {
+  // CHECK: number of devices 0
+  printf("number of devices %d\n", omp_get_num_devices());
+  // CHECK:initial device 0
+  printf("initial device %d\n", omp_get_initial_device());
+  // CHECK:default device 0
+  printf("default device %d\n", omp_get_default_device());
+  // CHECK: PASS
+  if (omp_get_initial_device() == omp_get_default_device()) {
+    printf("PASS\n");
+    return 0;
+  }
+  return 1;
+}
diff --git a/offload/test/ompTest/CMakeLists.txt b/offload/test/ompTest/CMakeLists.txt
index 29061b3743f0..dac81a15738c 100644
--- a/offload/test/ompTest/CMakeLists.txt
+++ b/offload/test/ompTest/CMakeLists.txt
@@ -64,10 +64,10 @@ target_include_directories(omptest
 if(LIBOMPTEST_BUILD_UNITTESTS)
   include(GoogleTest)
   add_executable(omptest-unittests
-
                  test/main-test.cpp
                  test/internal-event-test.cpp
                  test/internal-util-test.cpp
+                 test/asserter-seq-test.cpp
   )
   target_link_libraries(omptest-unittests omptest)
   target_link_libraries(omptest-unittests gtest gtest_main)
diff --git a/offload/test/ompTest/include/AssertMacros.h b/offload/test/ompTest/include/AssertMacros.h
index 2ee41c36b89b..e76a1715cb76 100644
--- a/offload/test/ompTest/include/AssertMacros.h
+++ b/offload/test/ompTest/include/AssertMacros.h
@@ -107,6 +107,19 @@
   OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy);                        \
   OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy);
 
+// Set logging level for asserters
+#define OMPT_ASSERTER_LOG_LEVEL(Asserter, LogLevel)                            \
+  Asserter->getLog()->setLoggingLevel(LogLevel);
+
+// Set colored logging for asserters
+#define OMPT_ASSERTER_LOG_COLORED(Asserter, ColoredLog)                        \
+  Asserter->getLog()->setLoggingLevel(ColoredLog);
+
+#define OMPT_LOG_COLORED(ColoredLog)                                           \
+  OMPT_ASSERTER_LOGLEVEL(SetAsserter, ColoredLog);                             \
+  OMPT_ASSERTER_LOGLEVEL(EventReporter, ColoredLog);                           \
+  OMPT_ASSERTER_LOGLEVEL(SequenceAsserter, ColoredLog);
+
 #define OMPT_ASSERT_SYNC_POINT(SyncPointName)                                  \
   flush_traced_devices();                                                      \
   OmptCallbackHandler::get().handleAssertionSyncPoint(SyncPointName);
diff --git a/offload/test/ompTest/include/Logging.h b/offload/test/ompTest/include/Logging.h
index ab0085c339f0..a597ba5305d4 100644
--- a/offload/test/ompTest/include/Logging.h
+++ b/offload/test/ompTest/include/Logging.h
@@ -12,6 +12,7 @@
 
 #include <iostream>
 #include <map>
+#include <mutex>
 #include <set>
 #include <sstream>
 #include <string>
@@ -31,7 +32,10 @@ enum class Level : uint32_t {
   Default,
   ExpectedEvent,
   ObservedEvent,
-  OffendingEvent
+  OffendingEvent,
+
+  // Suppress all prints
+  SILENT = 0xFFFFFFFF
 };
 
 enum class FormatOption : uint32_t {
@@ -98,13 +102,10 @@ std::string format(const std::string &Message, std::set<FormatOption> Options);
 
 class Logger {
 public:
+  Logger(Level LogLevel = Level::WARNING, std::ostream &OutStream = std::cerr,
+         bool FormatOutput = true);
   ~Logger();
 
-  /// Retrieve the singleton logger (and initialize, if not done already)
-  static Logger &get(Level LogLevel = Level::WARNING,
-                     std::ostream &OutStream = std::cerr,
-                     bool FormatOutput = true);
-
   /// Log the given message to the output.
   void log(Level LogLevel, const std::string &Message) const;
 
@@ -119,7 +120,7 @@ class Logger {
                      const std::string &Message,
                      Level LogLevel = Level::ERROR) const;
 
-  /// Set if output is being formatted.
+  /// Set if output is being formatted (e.g. colored).
   void setFormatOutput(bool Enabled);
 
   /// Return the current (minimum) Logging Level.
@@ -129,9 +130,6 @@ class Logger {
   void setLoggingLevel(Level LogLevel);
 
 private:
-  Logger(Level LogLevel = Level::WARNING, std::ostream &OutStream = std::cerr,
-         bool FormatOutput = true);
-
   /// The minimum logging level that is considered by the logger instance.
   Level LoggingLevel;
 
@@ -140,12 +138,12 @@ class Logger {
 
   /// Determine if log messages are formatted using control sequences.
   bool FormatOutput;
+
+  /// Mutex to ensure serialized logging
+  mutable std::mutex LogMutex;
 };
 
 } // namespace logging
 } // namespace omptest
 
-// Pointer to global logger
-extern omptest::logging::Logger *Log;
-
 #endif
\ No newline at end of file
diff --git a/offload/test/ompTest/include/OmptAssertEvent.h b/offload/test/ompTest/include/OmptAssertEvent.h
index 7e4d21b25a9d..fc84498dcfe4 100644
--- a/offload/test/ompTest/include/OmptAssertEvent.h
+++ b/offload/test/ompTest/include/OmptAssertEvent.h
@@ -16,7 +16,6 @@ enum class ObserveState { generated, always, never };
 const char *to_string(ObserveState State);
 
 struct OmptAssertEvent {
-
   static OmptAssertEvent AssertionSyncPoint(const std::string &Name,
                                             const std::string &Group,
                                             const ObserveState &Expected,
@@ -340,6 +339,7 @@ struct OmptAssertEvent {
   std::unique_ptr<internal::InternalEvent> TheEvent;
 };
 
+/// POD type, which holds the target region id, corresponding to an event group.
 struct AssertEventGroup {
   AssertEventGroup(uint64_t TargetRegion) : TargetRegion(TargetRegion) {}
   uint64_t TargetRegion;
diff --git a/offload/test/ompTest/include/OmptAsserter.h b/offload/test/ompTest/include/OmptAsserter.h
index 0f1e84d37f9b..36504bb1c0d2 100644
--- a/offload/test/ompTest/include/OmptAsserter.h
+++ b/offload/test/ompTest/include/OmptAsserter.h
@@ -1,6 +1,7 @@
 #ifndef OFFLOAD_TEST_OMPTEST_INCLUDE_OMPTASSERTER_H
 #define OFFLOAD_TEST_OMPTEST_INCLUDE_OMPTASSERTER_H
 
+#include "Logging.h"
 #include "OmptAssertEvent.h"
 
 #include <cassert>
@@ -70,7 +71,8 @@ class OmptListener {
 /// Base class for asserting on OMPT events
 class OmptAsserter : public OmptListener {
 public:
-  OmptAsserter(OmptEventGroupInterface *ParentTC) : TC(ParentTC) {}
+  OmptAsserter();
+  virtual ~OmptAsserter() = default;
 
   /// Add an event to the asserter's internal data structure.
   virtual void insert(omptest::OmptAssertEvent &&AE);
@@ -86,10 +88,30 @@ class OmptAsserter : public OmptListener {
   /// Get the number of currently remaining events, with: ObserveState::always.
   virtual size_t getRemainingEventCount() = 0;
 
-  /// Determine and return the asserter's current state.
-  virtual omptest::AssertState getState();
+  /// Get the total number of received, effective notifications.
+  int getNotificationCount() { return NumNotifications; }
 
-  /// Check the given events' group association. If the event indicates the
+  /// Get the total number of successful assertion checks.
+  int getSuccessfulAssertionCount() { return NumSuccessfulAsserts; }
+
+  /// Get the asserter's current operationmode: e.g.: strict or relaxed.
+  AssertMode getOperationMode() { return OperationMode; }
+
+  /// Return the asserter's current state.
+  omptest::AssertState getState() { return State; }
+
+  /// Determine and return the asserter's state.
+  virtual omptest::AssertState checkState();
+
+  /// Accessor for the event group interface.
+  std::shared_ptr<OmptEventGroupInterface> getEventGroups() const {
+    return EventGroups;
+  }
+
+  /// Accessor for the event group interface.
+  std::shared_ptr<logging::Logger> getLog() const { return Log; }
+
+  /// Check the observed events' group association. If the event indicates the
   /// begin/end of an OpenMP target region, we will create/deprecate the
   /// expected event's group. Return true if the expected event group exists
   /// (and is active), otherwise: false. Note: BufferRecords may also match with
@@ -100,6 +122,13 @@ class OmptAsserter : public OmptListener {
   /// Set the asserter's mode of operation w.r.t. assertion.
   void setOperationMode(AssertMode Mode);
 
+  /// The total number of effective notifications. For example, if specific
+  /// notifications are to be ignored, they will not count towards this total.
+  int NumNotifications{0};
+
+  /// The number of successful assertion checks.
+  int NumSuccessfulAsserts{0};
+
 protected:
   /// The asserter's current state.
   omptest::AssertState State{omptest::AssertState::pass};
@@ -107,18 +136,31 @@ class OmptAsserter : public OmptListener {
   /// Mutex to avoid data races w.r.t. event notifications and/or insertions.
   std::mutex AssertMutex;
 
-  /// Pointer to the parent TestCase.
-  OmptEventGroupInterface *TC{nullptr};
+  /// Pointer to the OmptEventGroupInterface.
+  std::shared_ptr<OmptEventGroupInterface> EventGroups{nullptr};
+
+  /// Pointer to the logging instance.
+  std::shared_ptr<logging::Logger> Log{nullptr};
 
   /// Operation mode during assertion / notification.
   AssertMode OperationMode{AssertMode::strict};
+
+private:
+  /// Mutex for creating/accessing the singleton members
+  static std::mutex StaticMemberAccessMutex;
+
+  /// Static member to manage the singleton event group interface instance
+  static std::weak_ptr<OmptEventGroupInterface> EventGroupInterfaceInstance;
+
+  /// Static member to manage the singleton logging instance
+  static std::weak_ptr<logging::Logger> LoggingInstance;
 };
 
 /// Class that can assert in a sequenced fashion, i.e., events have to occur in
 /// the order they were registered
-struct OmptSequencedAsserter : public OmptAsserter {
-  OmptSequencedAsserter(OmptEventGroupInterface *ParentTC = nullptr)
-      : OmptAsserter(ParentTC), NextEvent(0) {}
+class OmptSequencedAsserter : public OmptAsserter {
+public:
+  OmptSequencedAsserter() : OmptAsserter(), NextEvent(0) {}
 
   /// Add the event to the in-sequence set of events that the asserter should
   /// check for.
@@ -129,19 +171,38 @@ struct OmptSequencedAsserter : public OmptAsserter {
 
   size_t getRemainingEventCount() override;
 
-  omptest::AssertState getState() override;
+  omptest::AssertState checkState() override;
 
   bool AssertionSuspended{false};
-  int NumAssertSuccesses{0};
-  int NumNotifications{0};
+
+protected:
+  /// Notification helper function, implementing SyncPoint logic. Returns true
+  /// in case of consumed event, indicating early exit of notification.
+  bool consumeSyncPoint(const omptest::OmptAssertEvent &AE);
+
+  /// Notification helper function, implementing excess event notification
+  /// logic. Returns true when no more events were expected, indicating early
+  /// exit of notification.
+  bool checkExcessNotify(const omptest::OmptAssertEvent &AE);
+
+  /// Notification helper function, implementing Suspend logic. Returns true
+  /// in case of consumed event, indicating early exit of notification.
+  bool consumeSuspend();
+
+  /// Notification helper function, implementing regular event notification
+  /// logic. Returns true when a matching event was encountered, indicating
+  /// early exit of notification.
+  bool consumeRegularEvent(const omptest::OmptAssertEvent &AE);
+
+public:
+  /// Index of the next, expected event.
   size_t NextEvent{0};
   std::vector<omptest::OmptAssertEvent> Events{};
 };
 
 /// Class that asserts with set semantics, i.e., unordered
 struct OmptEventAsserter : public OmptAsserter {
-  OmptEventAsserter(OmptEventGroupInterface *ParentTC = nullptr)
-      : OmptAsserter(ParentTC), NumEvents(0), Events() {}
+  OmptEventAsserter() : OmptAsserter(), NumEvents(0), Events() {}
 
   /// Add the event to the set of events that the asserter should check for.
   void insert(omptest::OmptAssertEvent &&AE) override;
@@ -151,10 +212,8 @@ struct OmptEventAsserter : public OmptAsserter {
 
   size_t getRemainingEventCount() override;
 
-  omptest::AssertState getState() override;
+  omptest::AssertState checkState() override;
 
-  int NumAssertSuccesses{0};
-  int NumNotifications{0};
   size_t NumEvents{0};
 
   /// For now use vector (but do set semantics)
@@ -168,8 +227,8 @@ class OmptEventReporter : public OmptListener {
   OmptEventReporter(std::ostream &OutStream = std::cout)
       : OutStream(OutStream) {}
 
-  // Called from the CallbackHandler with a corresponding AssertEvent to which
-  // callback was handled.
+  /// Called from the CallbackHandler with a corresponding AssertEvent to which
+  /// callback was handled.
   void notify(omptest::OmptAssertEvent &&AE) override;
 
 private:
@@ -181,33 +240,34 @@ class OmptEventReporter : public OmptListener {
 /// coherent view of active and past events or SyncPoints.
 class OmptEventGroupInterface {
 public:
-  OmptEventGroupInterface()
-      : SequenceAsserter{std::make_unique<OmptSequencedAsserter>(this)},
-        SetAsserter{std::make_unique<OmptEventAsserter>(this)},
-        EventReporter{std::make_unique<OmptEventReporter>()} {}
+  OmptEventGroupInterface() = default;
+  ~OmptEventGroupInterface() = default;
+
+  /// Non-copyable and non-movable
+  OmptEventGroupInterface(const OmptEventGroupInterface &) = delete;
+  OmptEventGroupInterface &operator=(const OmptEventGroupInterface &) = delete;
+  OmptEventGroupInterface(OmptEventGroupInterface &&) = delete;
+  OmptEventGroupInterface &operator=(OmptEventGroupInterface &&) = delete;
 
-  // Add given group to the set of active event groups.
+  /// Add given group to the set of active event groups. Effectively connecting
+  /// the given groupname (expected) with a target region id (observed).
   bool addActiveEventGroup(const std::string &GroupName,
                            omptest::AssertEventGroup Group);
 
-  // Move given group from the set of active event groups to the set of
-  // previously active event groups.
+  /// Move given group from the set of active event groups to the set of
+  /// previously active event groups.
   bool deprecateActiveEventGroup(const std::string &GroupName);
 
-  // Check if given group is currently part of the active event groups.
+  /// Check if given group is currently part of the active event groups.
   bool checkActiveEventGroups(const std::string &GroupName,
                               omptest::AssertEventGroup Group);
 
-  // Check if given group is currently part of the deprecated event groups.
+  /// Check if given group is currently part of the deprecated event groups.
   bool checkDeprecatedEventGroups(const std::string &GroupName,
                                   omptest::AssertEventGroup Group);
 
-  std::unique_ptr<OmptSequencedAsserter> SequenceAsserter;
-  std::unique_ptr<OmptEventAsserter> SetAsserter;
-  std::unique_ptr<OmptEventReporter> EventReporter;
-
 private:
-  std::mutex GroupMutex;
+  mutable std::mutex GroupMutex;
   std::map<std::string, omptest::AssertEventGroup> ActiveEventGroups{};
   std::map<std::string, omptest::AssertEventGroup> DeprecatedEventGroups{};
   std::set<std::string> EncounteredSyncPoints{};
diff --git a/offload/test/ompTest/include/OmptTesterGoogleTest.h b/offload/test/ompTest/include/OmptTesterGoogleTest.h
index 5a80a5c8efe2..2c9b7dbcc0da 100644
--- a/offload/test/ompTest/include/OmptTesterGoogleTest.h
+++ b/offload/test/ompTest/include/OmptTesterGoogleTest.h
@@ -16,6 +16,13 @@ namespace testing {
 class GTEST_API_ OmptTestCase : public testing::Test,
                                 public omptest::OmptEventGroupInterface {
 public:
+  std::unique_ptr<omptest::OmptSequencedAsserter> SequenceAsserter =
+      std::make_unique<omptest::OmptSequencedAsserter>();
+  std::unique_ptr<omptest::OmptEventAsserter> SetAsserter =
+      std::make_unique<omptest::OmptEventAsserter>();
+  std::unique_ptr<omptest::OmptEventReporter> EventReporter =
+      std::make_unique<omptest::OmptEventReporter>();
+
 protected:
   void SetUp() override {
     omptest::OmptCallbackHandler::get().subscribe(SequenceAsserter.get());
@@ -31,8 +38,8 @@ class GTEST_API_ OmptTestCase : public testing::Test,
     omptest::OmptCallbackHandler::get().clearSubscribers();
 
     // This common testcase must not encounter any failures.
-    if (SequenceAsserter->getState() == omptest::AssertState::fail ||
-        SetAsserter->getState() == omptest::AssertState::fail)
+    if (SequenceAsserter->checkState() == omptest::AssertState::fail ||
+        SetAsserter->checkState() == omptest::AssertState::fail)
       ADD_FAILURE();
   }
 };
@@ -47,8 +54,8 @@ class GTEST_API_ OmptTestCaseXFail : public testing::OmptTestCase {
     omptest::OmptCallbackHandler::get().clearSubscribers();
 
     // This eXpectedly failing testcase has to encounter at least one failure.
-    if (SequenceAsserter->getState() == omptest::AssertState::pass &&
-        SetAsserter->getState() == omptest::AssertState::pass)
+    if (SequenceAsserter->checkState() == omptest::AssertState::pass &&
+        SetAsserter->checkState() == omptest::AssertState::pass)
       ADD_FAILURE();
   }
 };
diff --git a/offload/test/ompTest/include/OmptTesterStandalone.h b/offload/test/ompTest/include/OmptTesterStandalone.h
index bc6f18b5c893..1e2a780dc7a0 100644
--- a/offload/test/ompTest/include/OmptTesterStandalone.h
+++ b/offload/test/ompTest/include/OmptTesterStandalone.h
@@ -12,7 +12,7 @@
 namespace omptest {
 struct OmptEventAsserter;
 class OmptEventReporter;
-struct OmptSequencedAsserter;
+class OmptSequencedAsserter;
 } // namespace omptest
 
 struct Error {
@@ -21,8 +21,7 @@ struct Error {
 };
 
 /// A pretty crude test case abstraction
-class TestCase : public omptest::OmptEventGroupInterface {
-public:
+struct TestCase {
   TestCase(const std::string &name)
       : IsDisabled(name.rfind("DISABLED_", 0) == 0), Name(name) {}
   TestCase(const std::string &name, const omptest::AssertState &expected)
@@ -36,6 +35,13 @@ class TestCase : public omptest::OmptEventGroupInterface {
   std::string Name;
   omptest::AssertState ExpectedState{omptest::AssertState::pass};
   omptest::AssertState ResultState{omptest::AssertState::pass};
+
+  std::unique_ptr<omptest::OmptSequencedAsserter> SequenceAsserter =
+      std::make_unique<omptest::OmptSequencedAsserter>();
+  std::unique_ptr<omptest::OmptEventAsserter> SetAsserter =
+      std::make_unique<omptest::OmptEventAsserter>();
+  std::unique_ptr<omptest::OmptEventReporter> EventReporter =
+      std::make_unique<omptest::OmptEventReporter>();
 };
 /// A pretty crude test suite abstraction
 struct TestSuite {
@@ -51,7 +57,8 @@ struct TestSuite {
   TestCaseVec TestCases;
 };
 /// Static class used to register all test cases and provide them to the driver
-struct TestRegistrar {
+class TestRegistrar {
+public:
   static TestRegistrar &get();
   static std::vector<TestSuite> getTestSuites();
   static void addCaseToSuite(TestCase *TC, std::string TSName);
diff --git a/offload/test/ompTest/src/InternalEventOperators.cpp b/offload/test/ompTest/src/InternalEventOperators.cpp
index 371d935e19d9..d0425b4a7a9e 100644
--- a/offload/test/ompTest/src/InternalEventOperators.cpp
+++ b/offload/test/ompTest/src/InternalEventOperators.cpp
@@ -129,7 +129,11 @@ event_class_operator_w_body(DeviceInitialize,                                  \
        true : (Expected.Device == Observed.Device);                            \
   return isSameDeviceNum && isSameType && isSameDevice;                        \
 )
-event_class_operator_stub(DeviceFinalize)
+event_class_operator_w_body(DeviceFinalize,                                    \
+  bool isSameDeviceNum = (Expected.DeviceNum == expectedDefault(int)) ?        \
+                            true : (Expected.DeviceNum == Observed.DeviceNum); \
+  return isSameDeviceNum;
+)
 event_class_operator_w_body(DeviceLoad,                                        \
   bool isSameDeviceNum = (Expected.DeviceNum == expectedDefault(int)) ?        \
                             true : (Expected.DeviceNum == Observed.DeviceNum); \
diff --git a/offload/test/ompTest/src/Logging.cpp b/offload/test/ompTest/src/Logging.cpp
index 0382477eee78..280fa3c25e81 100644
--- a/offload/test/ompTest/src/Logging.cpp
+++ b/offload/test/ompTest/src/Logging.cpp
@@ -11,7 +11,16 @@
 using namespace omptest;
 using namespace logging;
 
-Logger *Log = nullptr;
+Logger::Logger(Level LogLevel, std::ostream &OutStream, bool FormatOutput)
+    : LoggingLevel(LogLevel), OutStream(OutStream), FormatOutput(FormatOutput) {
+  // Flush any buffered output
+  OutStream << std::flush;
+}
+
+Logger::~Logger() {
+  // Flush any buffered output
+  OutStream << std::flush;
+}
 
 std::map<Level, std::set<FormatOption>> AggregatedFormatOptions{
     {Level::DIAGNOSTIC, {FormatOption::COLOR_LightBlue}},
@@ -76,26 +85,10 @@ std::string logging::format(const std::string &Message,
   return SS.str();
 }
 
-Logger::Logger(Level LogLevel, std::ostream &OutStream, bool FormatOutput)
-    : LoggingLevel(LogLevel), OutStream(OutStream), FormatOutput(FormatOutput) {
-  // Flush any buffered output
-  OutStream << std::flush;
-}
-
-Logger::~Logger() {
-  // Flush any buffered output
-  OutStream << std::flush;
-}
-
-Logger &Logger::get(Level LogLevel, std::ostream &OutStream,
-                    bool FormatOutput) {
-  if (Log == nullptr)
-    Log = new Logger(LogLevel, OutStream, FormatOutput);
-
-  return *Log;
-}
-
 void Logger::log(Level LogLevel, const std::string &Message) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
+
   if (LoggingLevel > LogLevel)
     return;
 
@@ -109,6 +102,8 @@ void Logger::log(Level LogLevel, const std::string &Message) const {
 
 void Logger::eventMismatch(const omptest::OmptAssertEvent &OffendingEvent,
                            const std::string &Message, Level LogLevel) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
   if (LoggingLevel > LogLevel)
     return;
 
@@ -134,6 +129,8 @@ void Logger::eventMismatch(const omptest::OmptAssertEvent &OffendingEvent,
 void Logger::eventMismatch(const omptest::OmptAssertEvent &ExpectedEvent,
                            const omptest::OmptAssertEvent &ObservedEvent,
                            const std::string &Message, Level LogLevel) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
   if (LoggingLevel > LogLevel)
     return;
 
diff --git a/offload/test/ompTest/src/OmptAsserter.cpp b/offload/test/ompTest/src/OmptAsserter.cpp
index d8528bd5d3f9..e10cf21043b1 100644
--- a/offload/test/ompTest/src/OmptAsserter.cpp
+++ b/offload/test/ompTest/src/OmptAsserter.cpp
@@ -6,6 +6,44 @@
 using namespace omptest;
 using namespace internal;
 
+// Initialize static members
+std::mutex OmptAsserter::StaticMemberAccessMutex;
+std::weak_ptr<OmptEventGroupInterface>
+    OmptAsserter::EventGroupInterfaceInstance;
+std::weak_ptr<logging::Logger> OmptAsserter::LoggingInstance;
+
+OmptAsserter::OmptAsserter() {
+  // Protect static members access
+  std::lock_guard<std::mutex> Lock(StaticMemberAccessMutex);
+
+  // Upgrade OmptEventGroupInterface weak_ptr to shared_ptr
+  {
+    EventGroups = EventGroupInterfaceInstance.lock();
+    if (!EventGroups) {
+      // Coordinator doesn't exist or was previously destroyed, create a new
+      // one.
+      EventGroups = std::make_shared<OmptEventGroupInterface>();
+      // Store a weak reference to it
+      EventGroupInterfaceInstance = EventGroups;
+    }
+    // EventGroups is now a valid shared_ptr, either to a new or existing
+    // instance.
+  }
+
+  // Upgrade logging::Logger weak_ptr to shared_ptr
+  {
+    Log = LoggingInstance.lock();
+    if (!Log) {
+      // Coordinator doesn't exist or was previously destroyed, create a new
+      // one.
+      Log = std::make_shared<logging::Logger>();
+      // Store a weak reference to it
+      LoggingInstance = Log;
+    }
+    // Log is now a valid shared_ptr, either to a new or existing instance.
+  }
+}
+
 void OmptListener::setActive(bool Enabled) { Active = Enabled; }
 
 bool OmptListener::isActive() { return Active; }
@@ -32,16 +70,17 @@ void OmptAsserter::notify(OmptAssertEvent &&AE) {
   this->notifyImpl(std::move(AE));
 }
 
-AssertState OmptAsserter::getState() { return State; }
+AssertState OmptAsserter::checkState() { return State; }
 
 bool OmptAsserter::verifyEventGroups(const OmptAssertEvent &ExpectedEvent,
                                      const OmptAssertEvent &ObservedEvent) {
   assert(ExpectedEvent.getEventType() == ObservedEvent.getEventType() &&
          "Type mismatch: Expected != Observed event type");
-  assert(TC && "Missing parent TestCase");
+  assert(EventGroups && "Missing EventGroups interface");
 
   // Ignore all events within "default" group
   auto GroupName = ExpectedEvent.getEventGroup();
+
   if (GroupName == "default")
     return true;
 
@@ -53,10 +92,11 @@ bool OmptAsserter::verifyEventGroups(const OmptAssertEvent &ExpectedEvent,
     if (auto E = static_cast<const internal::Target *>(Event)) {
       if (E->Endpoint == ompt_scope_begin) {
         // Add new group since we entered a Target Region
-        TC->addActiveEventGroup(GroupName, AssertEventGroup{E->TargetId});
+        EventGroups->addActiveEventGroup(GroupName,
+                                         AssertEventGroup{E->TargetId});
       } else if (E->Endpoint == ompt_scope_end) {
         // Deprecate group since we return from a Target Region
-        TC->deprecateActiveEventGroup(GroupName);
+        EventGroups->deprecateActiveEventGroup(GroupName);
       }
       return true;
     }
@@ -65,45 +105,45 @@ bool OmptAsserter::verifyEventGroups(const OmptAssertEvent &ExpectedEvent,
     if (auto E = static_cast<const internal::TargetEmi *>(Event)) {
       if (E->Endpoint == ompt_scope_begin) {
         // Add new group since we entered a Target Region
-        TC->addActiveEventGroup(GroupName,
-                                AssertEventGroup{E->TargetData->value});
+        EventGroups->addActiveEventGroup(
+            GroupName, AssertEventGroup{E->TargetData->value});
       } else if (E->Endpoint == ompt_scope_end) {
         // Deprecate group since we return from a Target Region
-        TC->deprecateActiveEventGroup(GroupName);
+        EventGroups->deprecateActiveEventGroup(GroupName);
       }
       return true;
     }
     return false;
   case EventTy::TargetDataOp:
     if (auto E = static_cast<const internal::TargetDataOp *>(Event))
-      return TC->checkActiveEventGroups(GroupName,
-                                        AssertEventGroup{E->TargetId});
+      return EventGroups->checkActiveEventGroups(GroupName,
+                                                 AssertEventGroup{E->TargetId});
 
     return false;
   case EventTy::TargetDataOpEmi:
     if (auto E = static_cast<const internal::TargetDataOpEmi *>(Event))
-      return TC->checkActiveEventGroups(GroupName,
-                                        AssertEventGroup{E->TargetData->value});
+      return EventGroups->checkActiveEventGroups(
+          GroupName, AssertEventGroup{E->TargetData->value});
 
     return false;
   case EventTy::TargetSubmit:
     if (auto E = static_cast<const internal::TargetSubmit *>(Event))
-      return TC->checkActiveEventGroups(GroupName,
-                                        AssertEventGroup{E->TargetId});
+      return EventGroups->checkActiveEventGroups(GroupName,
+                                                 AssertEventGroup{E->TargetId});
 
     return false;
   case EventTy::TargetSubmitEmi:
     if (auto E = static_cast<const internal::TargetSubmitEmi *>(Event))
-      return TC->checkActiveEventGroups(GroupName,
-                                        AssertEventGroup{E->TargetData->value});
+      return EventGroups->checkActiveEventGroups(
+          GroupName, AssertEventGroup{E->TargetData->value});
 
     return false;
   case EventTy::BufferRecord:
     // BufferRecords are delivered asynchronously: also check deprecated groups.
     if (auto E = static_cast<const internal::BufferRecord *>(Event))
-      return (TC->checkActiveEventGroups(
+      return (EventGroups->checkActiveEventGroups(
                   GroupName, AssertEventGroup{E->Record.target_id}) ||
-              TC->checkDeprecatedEventGroups(
+              EventGroups->checkDeprecatedEventGroups(
                   GroupName, AssertEventGroup{E->Record.target_id}));
     return false;
   // Some event types do not need any handling
@@ -151,58 +191,98 @@ void OmptSequencedAsserter::insert(OmptAssertEvent &&AE) {
 
 void OmptSequencedAsserter::notifyImpl(OmptAssertEvent &&AE) {
   std::lock_guard<std::mutex> Lock(AssertMutex);
-  // Ignore notifications while inactive
+  // Ignore notifications while inactive, or for suppressed events
   if (Events.empty() || !isActive() || isSuppressedEventType(AE.getEventType()))
     return;
 
   ++NumNotifications;
 
+  // Note: Order of these checks has semantic meaning.
+  // (1) Synchronization points should fail if there are remaining events,
+  // otherwise pass. (2) Regular notification while no further events are
+  // expected: fail. (3) Assertion suspension relies on a next expected event
+  // being available. (4) All other cases are considered 'regular' and match the
+  // next expected against the observed event. (5+6) Depending on the state /
+  // mode we signal failure if no other check has done already, or signaled pass
+  // by early-exit.
+  if (consumeSyncPoint(AE) ||               // Handle observed SyncPoint event
+      checkExcessNotify(AE) ||              // Check for remaining expected
+      consumeSuspend() ||                   // Handle requested suspend
+      consumeRegularEvent(AE) ||            // Handle regular event
+      AssertionSuspended ||                 // Ignore fail, if suspended
+      OperationMode == AssertMode::relaxed) // Ignore fail, if relaxed op-mode
+    return;
+
+  Log->eventMismatch(Events[NextEvent], AE,
+                     "[OmptSequencedAsserter] The events are not equal");
+  State = AssertState::fail;
+}
+
+bool OmptSequencedAsserter::consumeSyncPoint(
+    const omptest::OmptAssertEvent &AE) {
   if (AE.getEventType() == EventTy::AssertionSyncPoint) {
     auto NumRemainingEvents = getRemainingEventCount();
     // Upon encountering a SyncPoint, all events should have been processed
     if (NumRemainingEvents == 0)
-      return;
+      return true;
 
     Log->eventMismatch(
         AE,
         "[OmptSequencedAsserter] Encountered SyncPoint while still awaiting " +
             std::to_string(NumRemainingEvents) + " events. Asserted " +
-            std::to_string(NumAssertSuccesses) + "/" +
+            std::to_string(NumSuccessfulAsserts) + "/" +
             std::to_string(Events.size()) + " events successfully.");
     State = AssertState::fail;
-    return;
+    return true;
   }
 
+  // Nothing to process: continue.
+  return false;
+}
+
+bool OmptSequencedAsserter::checkExcessNotify(
+    const omptest::OmptAssertEvent &AE) {
   if (NextEvent >= Events.size()) {
     // If we are not expecting any more events and passively asserting: return
     if (AssertionSuspended)
-      return;
+      return true;
 
     Log->eventMismatch(
         AE, "[OmptSequencedAsserter] Too many events to check (" +
                 std::to_string(NumNotifications) + "). Asserted " +
-                std::to_string(NumAssertSuccesses) + "/" +
+                std::to_string(NumSuccessfulAsserts) + "/" +
                 std::to_string(Events.size()) + " events successfully.");
     State = AssertState::fail;
-    return;
+    return true;
   }
 
+  // Remaining expected events present: continue.
+  return false;
+}
+
+bool OmptSequencedAsserter::consumeSuspend() {
   // On AssertionSuspend -- enter 'passive' assertion.
   // Since we may encounter multiple, successive AssertionSuspend events, loop
   // until we hit the next non-AssertionSuspend event.
   while (Events[NextEvent].getEventType() == EventTy::AssertionSuspend) {
     AssertionSuspended = true;
-    // We just hit the very last event: return
+    // We just hit the very last event: indicate early exit.
     if (++NextEvent >= Events.size())
-      return;
+      return true;
   }
 
+  // Continue with remaining notification logic.
+  return false;
+}
+
+bool OmptSequencedAsserter::consumeRegularEvent(
+    const omptest::OmptAssertEvent &AE) {
   // If we are actively asserting, increment the event counter.
   // Otherwise: If passively asserting, we will keep waiting for a match.
   auto &E = Events[NextEvent];
   if (E == AE && verifyEventGroups(E, AE)) {
     if (E.getEventExpectedState() == ObserveState::always) {
-      ++NumAssertSuccesses;
+      ++NumSuccessfulAsserts;
     } else if (E.getEventExpectedState() == ObserveState::never) {
       Log->eventMismatch(E, AE,
                          "[OmptSequencedAsserter] Encountered forbidden event");
@@ -213,16 +293,13 @@ void OmptSequencedAsserter::notifyImpl(OmptAssertEvent &&AE) {
     if (AssertionSuspended)
       AssertionSuspended = false;
 
-    // Match found, increment index
+    // Match found, increment index and indicate early exit (success).
     ++NextEvent;
-    return;
+    return true;
   }
 
-  if (AssertionSuspended || OperationMode == AssertMode::relaxed)
-    return;
-
-  Log->eventMismatch(E, AE, "[OmptSequencedAsserter] The events are not equal");
-  State = AssertState::fail;
+  // Continue with remaining notification logic.
+  return false;
 }
 
 size_t OmptSequencedAsserter::getRemainingEventCount() {
@@ -231,10 +308,10 @@ size_t OmptSequencedAsserter::getRemainingEventCount() {
                          return E.getEventExpectedState() ==
                                 ObserveState::always;
                        }) -
-         NumAssertSuccesses;
+         NumSuccessfulAsserts;
 }
 
-AssertState OmptSequencedAsserter::getState() {
+AssertState OmptSequencedAsserter::checkState() {
   // This is called after the testcase executed.
   // Once reached the number of successful notifications should be equal to the
   // number of expected events. However, there may still be excluded as well as
@@ -277,7 +354,7 @@ void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) {
     Log->eventMismatch(
         AE, "[OmptEventAsserter] Encountered SyncPoint while still awaiting " +
                 std::to_string(NumRemainingEvents) + " events. Asserted " +
-                std::to_string(NumAssertSuccesses) + " events successfully.");
+                std::to_string(NumSuccessfulAsserts) + " events successfully.");
     State = AssertState::fail;
     return;
   }
@@ -287,7 +364,7 @@ void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) {
     if (E == AE && verifyEventGroups(E, AE)) {
       if (E.getEventExpectedState() == ObserveState::always) {
         Events.erase(Events.begin() + i);
-        ++NumAssertSuccesses;
+        ++NumSuccessfulAsserts;
       } else if (E.getEventExpectedState() == ObserveState::never) {
         Log->eventMismatch(E, AE,
                            "[OmptEventAsserter] Encountered forbidden event");
@@ -301,7 +378,7 @@ void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) {
     Log->eventMismatch(AE, "[OmptEventAsserter] Too many events to check (" +
                                std::to_string(NumNotifications) +
                                "). Asserted " +
-                               std::to_string(NumAssertSuccesses) +
+                               std::to_string(NumSuccessfulAsserts) +
                                " events successfully. (Remaining events: " +
                                std::to_string(getRemainingEventCount()) + ")");
     State = AssertState::fail;
@@ -319,7 +396,7 @@ size_t OmptEventAsserter::getRemainingEventCount() {
       });
 }
 
-AssertState OmptEventAsserter::getState() {
+AssertState OmptEventAsserter::checkState() {
   // This is called after the testcase executed.
   // Once reached no more expected events should be in the queue
   for (const auto &E : Events) {
diff --git a/offload/test/ompTest/src/OmptTester.cpp b/offload/test/ompTest/src/OmptTester.cpp
index e4d75d56cd73..3ca8c0f9f9f8 100644
--- a/offload/test/ompTest/src/OmptTester.cpp
+++ b/offload/test/ompTest/src/OmptTester.cpp
@@ -7,9 +7,6 @@
 
 using namespace omptest;
 
-// Global Logger instance
-extern logging::Logger *Log;
-
 // Callback handler, which receives and relays OMPT callbacks
 extern OmptCallbackHandler *Handler;
 
@@ -407,9 +404,6 @@ int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
     register_ompt_callback(ompt_callback_target_map);
   }
 
-  // Construct global logger instance
-  logging::Logger::get(logging::Level::WARNING, std::cerr, ColoredLog);
-
   // Construct & subscribe the reporter, so it will be notified of events
   EventReporter = new OmptEventReporter();
   OmptCallbackHandler::get().subscribe(EventReporter);
@@ -423,10 +417,8 @@ int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
 void ompt_finalize(ompt_data_t *tool_data) {
   assert(Handler && "Callback handler should be present at this point");
   assert(EventReporter && "EventReporter should be present at this point");
-  assert(Log && "Logger should be present at this point");
   delete Handler;
   delete EventReporter;
-  delete Log;
 }
 
 #ifdef __cplusplus
diff --git a/offload/test/ompTest/src/OmptTesterStandalone.cpp b/offload/test/ompTest/src/OmptTesterStandalone.cpp
index 68943efd4cb0..2cea11f2a0dc 100644
--- a/offload/test/ompTest/src/OmptTesterStandalone.cpp
+++ b/offload/test/ompTest/src/OmptTesterStandalone.cpp
@@ -30,8 +30,8 @@ Error TestCase::exec() {
   // We remove subscribers to not be notified of events after our test case
   // finished.
   OmptCallbackHandler::get().clearSubscribers();
-  omptest::AssertState SequenceResultState = SequenceAsserter->getState();
-  omptest::AssertState SetResultState = SetAsserter->getState();
+  omptest::AssertState SequenceResultState = SequenceAsserter->checkState();
+  omptest::AssertState SetResultState = SetAsserter->checkState();
   bool AnyFail = SequenceResultState == omptest::AssertState::fail ||
                  SetResultState == omptest::AssertState::fail;
   bool AllPass = SequenceResultState == omptest::AssertState::pass &&
diff --git a/offload/test/ompTest/test/asserter-seq-test.cpp b/offload/test/ompTest/test/asserter-seq-test.cpp
new file mode 100644
index 000000000000..34ceb7597b79
--- /dev/null
+++ b/offload/test/ompTest/test/asserter-seq-test.cpp
@@ -0,0 +1,358 @@
+#include "OmptAliases.h"
+#include "OmptAsserter.h"
+#include <omp-tools.h>
+#include <sstream>
+
+#include "gtest/gtest.h"
+
+using namespace omptest;
+using OAE = omptest::OmptAssertEvent;
+using OS = omptest::ObserveState;
+
+/// SequencedAsserter test-fixture class to avoid code duplication among tests.
+class OmptSequencedAsserterTest : public testing::Test {
+protected:
+  OmptSequencedAsserterTest() {
+    // Construct default sequenced asserter
+    SeqAsserter = std::make_unique<omptest::OmptSequencedAsserter>();
+
+    // Silence all potential log prints
+    SeqAsserter->getLog()->setLoggingLevel(logging::Level::SILENT);
+  }
+
+  std::unique_ptr<omptest::OmptSequencedAsserter> SeqAsserter;
+};
+
+TEST_F(OmptSequencedAsserterTest, DefaultState) {
+  // Assertion should neither start as 'deactivated' nor 'suspended'
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+
+  // Assertion should begin with event ID zero
+  ASSERT_EQ(SeqAsserter->NextEvent, 0);
+
+  // Assertion should begin without previous notifications or assertions
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+
+  // There should be no expected events
+  ASSERT_EQ(SeqAsserter->Events.empty(), true);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+
+  // Default mode should be strict
+  ASSERT_NE(SeqAsserter->getOperationMode(), AssertMode::relaxed);
+  ASSERT_EQ(SeqAsserter->getOperationMode(), AssertMode::strict);
+
+  // Default state should be passing
+  ASSERT_NE(SeqAsserter->getState(), AssertState::fail);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::pass);
+  ASSERT_NE(SeqAsserter->checkState(), AssertState::fail);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, IgnoreNotificationsWhenEmpty) {
+  // ParallelBegin events are suppressed by default
+  auto SuppressedEvent = OAE::ParallelBegin(
+      /*Name=*/"ParBegin", /*Group=*/"", /*Expected=*/OS::always,
+      /*NumThreads=*/3);
+
+  // DeviceFinalize events are not ignored by default
+  auto IgnoredEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+
+  // Situation: There is nothing to assert.
+  // Result: All notifications are ignored.
+  // Hence, check that the perceived count of notifications remains unchanged
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  SeqAsserter->notify(std::move(SuppressedEvent));
+
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  SeqAsserter->notify(std::move(IgnoredEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, IgnoreNotificationsWhileDeactivated) {
+  auto ExpectedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevUnload", /*Group=*/"", /*Expected=*/OS::always);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Deactivate asserter, effectively ignoring notifications
+  SeqAsserter->setActive(false);
+  ASSERT_EQ(SeqAsserter->isActive(), false);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // DeviceFinalize events are not ignored by default
+  auto IgnoredEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->notify(std::move(IgnoredEvent));
+
+  // Assertion was deactivated: No change
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+
+  SeqAsserter->setActive(true);
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+
+  auto ObservedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always);
+  SeqAsserter->notify(std::move(ObservedEvent));
+
+  // Assertion was activated, one notification expected
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEvent) {
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // Sanity check: Notifications should not be triggered
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  // Adding an expected event must change the event count but not the state
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventIgnoreSuppressed) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // ParallelBegin events are suppressed by default
+  auto SuppressedEvent = OAE::ParallelBegin(
+      /*Name=*/"ParBegin", /*Group=*/"", /*Expected=*/OS::always,
+      /*NumThreads=*/3);
+  // Situation: There is one expected event and ParallelBegins are suppressed.
+  // Notification count remains unchanged for suppressed events
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  SeqAsserter->notify(std::move(SuppressedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObservePass) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObserveFail) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  // Provide wrong DeviceNum
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/23);
+
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  // Observed and expected event do not match: Fail
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObserveDifferentType) {
+  auto ExpectedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevUnload", /*Group=*/"", /*Expected=*/OS::always);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7);
+
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  // Observed and expected event do not match: Fail
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckTargetGroupNoEffect) {
+  // Situation: Groups are designed to be used as an indicator -WITHIN- target
+  // regions. Hence, comparing two target regions w.r.t. their groups has no
+  // effect on pass or fail.
+
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Deactivate asserter, effectively ignoring notifications
+  SeqAsserter->setActive(false);
+  ASSERT_EQ(SeqAsserter->isActive(), false);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+
+  // Assertion was deactivated: No change
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Re-activate asserter
+  SeqAsserter->setActive(true);
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // Actually observe a target event from "AnotherGroup"
+  auto AnotherObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"AnotherGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(AnotherObservedEvent));
+
+  // Observed all expected events; groups of target regions do not affect pass
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckSyncPoint) {
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  SeqAsserter->notify(OAE::AssertionSyncPoint(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::always,
+      /*SyncPointName=*/"SyncPoint 1"));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+
+  // All events processed: SyncPoint "passes"
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+
+  auto AnotherExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+  SeqAsserter->insert(std::move(AnotherExpectedEvent));
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Remaining events present: SyncPoint "fails"
+  SeqAsserter->notify(OAE::AssertionSyncPoint(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::always,
+      /*SyncPointName=*/"SyncPoint 2"));
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckExcessNotify) {
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  // All events processed: pass
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+
+  // Target events are not ignored by default
+  auto AnotherObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+
+  // No more events expected: notify "fails"
+  SeqAsserter->notify(std::move(AnotherObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckSuspend) {
+  SeqAsserter->insert(OAE::AssertionSuspend(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::never));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Being notified while the next expected event is a "suspend" should change
+  // the asserter's state
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+  SeqAsserter->notify(OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::always,
+      /*DeviceNum=*/7));
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, true);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+
+  // Being notified with an observed event, which matches the next expected
+  // event, resumes assertion (suspended = false)
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, true);
+  SeqAsserter->notify(OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr));
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::pass);
+}
diff --git a/offload/tools/offload-tblgen/APIGen.cpp b/offload/tools/offload-tblgen/APIGen.cpp
index 800c9cadfe38..35b080e3c801 100644
--- a/offload/tools/offload-tblgen/APIGen.cpp
+++ b/offload/tools/offload-tblgen/APIGen.cpp
@@ -34,7 +34,8 @@ static std::string MakeComment(StringRef in) {
     }
     out += std::string("/// ") +
            in.substr(LineStart, LineBreak - LineStart).str() + "\n";
-    LineStart = LineBreak + 1;
+    if (LineBreak != std::string::npos)
+      LineStart = LineBreak + 1;
   }
 
   return out;
diff --git a/offload/tools/offload-tblgen/MiscGen.cpp b/offload/tools/offload-tblgen/MiscGen.cpp
index bc3d204f78d8..14787729503e 100644
--- a/offload/tools/offload-tblgen/MiscGen.cpp
+++ b/offload/tools/offload-tblgen/MiscGen.cpp
@@ -82,7 +82,7 @@ void EmitOffloadErrcodes(const RecordKeeper &Records, raw_ostream &OS) {
 #endif
 
 // Error codes are shared between PluginInterface and liboffload.
-// To add new error codes, add them to offload/liboffload/API/Common.td and run the GenerateOffload target.
+// To add new error codes, add them to offload/liboffload/API/Common.td.
 
 )";
 
diff --git a/offload/tools/offload-tblgen/PrintGen.cpp b/offload/tools/offload-tblgen/PrintGen.cpp
index a964ff09d0f6..d1189688a90a 100644
--- a/offload/tools/offload-tblgen/PrintGen.cpp
+++ b/offload/tools/offload-tblgen/PrintGen.cpp
@@ -213,6 +213,11 @@ template <typename T> inline void printTagged(llvm::raw_ostream &os, const void
                   "enum {0} value);\n",
                   EnumRec{R}.getName());
   }
+  for (auto *R : Records.getAllDerivedDefinitions("Struct")) {
+    OS << formatv("inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os, "
+                  "const struct {0} param);\n",
+                  StructRec{R}.getName());
+  }
   OS << "\n";
 
   // Create definitions
diff --git a/offload/tools/offload-tblgen/RecordTypes.hpp b/offload/tools/offload-tblgen/RecordTypes.hpp
index bb7c06c643da..65c0a4ce4a2c 100644
--- a/offload/tools/offload-tblgen/RecordTypes.hpp
+++ b/offload/tools/offload-tblgen/RecordTypes.hpp
@@ -157,9 +157,15 @@ class ParamRec {
   bool isHandleType() const { return getType().ends_with("_handle_t"); }
   bool isFptrType() const { return getType().ends_with("_cb_t"); }
   StringRef getDesc() const { return rec->getValueAsString("desc"); }
-  bool isIn() const { return dyn_cast<BitInit>(flags->getBit(0))->getValue(); }
-  bool isOut() const { return dyn_cast<BitInit>(flags->getBit(1))->getValue(); }
-  bool isOpt() const { return dyn_cast<BitInit>(flags->getBit(2))->getValue(); }
+  bool getFlagBit(unsigned int Bit) const {
+    if (auto *BitValue = dyn_cast<BitInit>(flags->getBit(Bit)))
+      return BitValue->getValue();
+    assert(false && "Parameter flags has no default or set value");
+    return false;
+  }
+  bool isIn() const { return getFlagBit(0); }
+  bool isOut() const { return getFlagBit(1); }
+  bool isOpt() const { return getFlagBit(2); }
 
   const Record *getRec() const { return rec; }
   std::optional<std::pair<StringRef, StringRef>> getRange() const {
diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index fc1b5894b245..b113875b24ef 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -81,11 +81,36 @@ function(add_offload_unittest test_dirname)
   target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
 endfunction()
 
-# FIXME: Re-enable once OMPT design allows
-# add_subdirectory(Plugins)
-# add_subdirectory(OffloadAPI)
+function(add_conformance_test test_name)
+  set(target_name "${test_name}.conformance")
+
+  list(TRANSFORM ARGN PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" OUTPUT_VARIABLE files)
+
+  if(NOT TARGET libc)
+    message(WARNING "Cannot run conformance tests without the LLVM C library")
+    return()
+  endif()
+
+  add_executable(${target_name} ${files})
+  add_dependencies(${target_name} ${PLUGINS_TEST_COMMON} ${test_name}.bin)
+  target_compile_definitions(${target_name} PRIVATE DEVICE_CODE_PATH="${CONFORMANCE_TEST_DEVICE_CODE_PATH}")
+  target_link_libraries(${target_name} PRIVATE ${PLUGINS_TEST_COMMON} libc)
+  target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
+  set_target_properties(${target_name} PROPERTIES EXCLUDE_FROM_ALL TRUE)
+
+  add_custom_target(offload.conformance.${test_name}
+    COMMAND $<TARGET_FILE:${target_name}>
+    DEPENDS ${target_name}
+    COMMENT "Running conformance test ${test_name}")
+  add_dependencies(offload.conformance offload.conformance.${test_name})
+endfunction()
+
 set(OFFLOAD_TESTS_FORCE_NVPTX_ARCH "" CACHE STRING
   "Force building of NVPTX device code for Offload unit tests with the given arch, e.g. sm_61")
 set(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH "" CACHE STRING
   "Force building of AMDGPU device code for Offload unit tests with the given arch, e.g. gfx1030")
 
+# FIXME: Re-enable once OMPT design allows
+# add_subdirectory(Plugins)
+# add_subdirectory(OffloadAPI)
+add_subdirectory(Conformance)
diff --git a/offload/unittests/Conformance/CMakeLists.txt b/offload/unittests/Conformance/CMakeLists.txt
new file mode 100644
index 000000000000..bc3141757372
--- /dev/null
+++ b/offload/unittests/Conformance/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_custom_target(offload.conformance)
+
+set(PLUGINS_TEST_COMMON LLVMOffload LLVMSupport)
+set(PLUGINS_TEST_INCLUDE ${LIBOMPTARGET_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/common)
+
+add_subdirectory(device_code)
+
+add_conformance_test(sin sin.cpp)
diff --git a/offload/unittests/Conformance/device_code/CMakeLists.txt b/offload/unittests/Conformance/device_code/CMakeLists.txt
new file mode 100644
index 000000000000..223f04ccfb69
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/CMakeLists.txt
@@ -0,0 +1,4 @@
+# FIXME: Currently missing dependencies to build GPU portion automatically.
+add_offload_test_device_code(sin.c sin)
+
+set(OFFLOAD_TEST_DEVICE_CODE_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
diff --git a/offload/unittests/Conformance/device_code/sin.c b/offload/unittests/Conformance/device_code/sin.c
new file mode 100644
index 000000000000..e969e60f352a
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/sin.c
@@ -0,0 +1,4 @@
+#include <gpuintrin.h>
+#include <math.h>
+
+__gpu_kernel void kernel(double *out) { *out = sin(*out); }
diff --git a/offload/unittests/Conformance/sin.cpp b/offload/unittests/Conformance/sin.cpp
new file mode 100644
index 000000000000..9e15690a9e9d
--- /dev/null
+++ b/offload/unittests/Conformance/sin.cpp
@@ -0,0 +1,8 @@
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <OffloadAPI.h>
+#include <math.h>
+
+llvm::StringRef DeviceBinsDirectory = DEVICE_CODE_PATH;
+
+int main() { llvm::errs() << sin(0.0) << "\n"; }
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
index 0247744911ea..c534c4520599 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
@@ -77,6 +77,15 @@ TEST_P(olGetDeviceInfoTest, SuccessDriverVersion) {
   ASSERT_EQ(std::strlen(DriverVersion.data()), Size - 1);
 }
 
+TEST_P(olGetDeviceInfoTest, SuccessMaxWorkGroupSize) {
+  ol_dimensions_t Value{0, 0, 0};
+  ASSERT_SUCCESS(olGetDeviceInfo(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE,
+                                 sizeof(Value), &Value));
+  ASSERT_GT(Value.x, 0u);
+  ASSERT_GT(Value.y, 0u);
+  ASSERT_GT(Value.z, 0u);
+}
+
 TEST_P(olGetDeviceInfoTest, InvalidNullHandleDevice) {
   ol_device_type_t DeviceType;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
index edd2704a722d..a908078a2521 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
@@ -44,6 +44,14 @@ TEST_P(olGetDeviceInfoSizeTest, SuccessDriverVersion) {
   ASSERT_NE(Size, 0ul);
 }
 
+TEST_P(olGetDeviceInfoSizeTest, SuccessMaxWorkGroupSize) {
+  size_t Size = 0;
+  ASSERT_SUCCESS(
+      olGetDeviceInfoSize(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE, &Size));
+  ASSERT_EQ(Size, sizeof(ol_dimensions_t));
+  ASSERT_EQ(Size, sizeof(uint32_t) * 3);
+}
+
 TEST_P(olGetDeviceInfoSizeTest, InvalidNullHandle) {
   size_t Size = 0;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
index 132c7a7c51fb..d867e2aae131 100644
--- a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
+++ b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
@@ -2,6 +2,16 @@ add_offload_test_device_code(foo.c foo)
 add_offload_test_device_code(bar.c bar)
 # Compile with optimizations to eliminate AMDGPU implicit arguments.
 add_offload_test_device_code(noargs.c noargs -O3)
+add_offload_test_device_code(localmem.c localmem)
+add_offload_test_device_code(localmem_reduction.c localmem_reduction)
+add_offload_test_device_code(localmem_static.c localmem_static)
 
-add_custom_target(offload_device_binaries DEPENDS foo.bin bar.bin noargs.bin)
+add_custom_target(offload_device_binaries DEPENDS
+    foo.bin
+    bar.bin
+    noargs.bin
+    localmem.bin
+    localmem_reduction.bin
+    localmem_static.bin
+)
 set(OFFLOAD_TEST_DEVICE_CODE_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
diff --git a/offload/unittests/OffloadAPI/device_code/localmem.c b/offload/unittests/OffloadAPI/device_code/localmem.c
new file mode 100644
index 000000000000..d70847900bc4
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem.c
@@ -0,0 +1,11 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+extern __gpu_local uint32_t shared_mem[];
+
+__gpu_kernel void localmem(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = __gpu_thread_id(0);
+  shared_mem[__gpu_thread_id(0)] *= 2;
+  out[__gpu_thread_id(0) + (__gpu_num_threads(0) * __gpu_block_id(0))] =
+      shared_mem[__gpu_thread_id(0)];
+}
diff --git a/offload/unittests/OffloadAPI/device_code/localmem_reduction.c b/offload/unittests/OffloadAPI/device_code/localmem_reduction.c
new file mode 100644
index 000000000000..8a9a46cfb6a1
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem_reduction.c
@@ -0,0 +1,16 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+extern __gpu_local uint32_t shared_mem[];
+
+__gpu_kernel void localmem_reduction(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = 2;
+
+  __gpu_sync_threads();
+
+  if (__gpu_thread_id(0) == 0) {
+    out[__gpu_block_id(0)] = 0;
+    for (uint32_t i = 0; i < __gpu_num_threads(0); i++)
+      out[__gpu_block_id(0)] += shared_mem[i];
+  }
+}
diff --git a/offload/unittests/OffloadAPI/device_code/localmem_static.c b/offload/unittests/OffloadAPI/device_code/localmem_static.c
new file mode 100644
index 000000000000..928b48422a0d
--- /dev/null
+++ b/offload/unittests/OffloadAPI/device_code/localmem_static.c
@@ -0,0 +1,17 @@
+#include <gpuintrin.h>
+#include <stdint.h>
+
+[[clang::loader_uninitialized]]
+__gpu_local uint32_t shared_mem[64];
+
+__gpu_kernel void localmem_static(uint32_t *out) {
+  shared_mem[__gpu_thread_id(0)] = 2;
+
+  __gpu_sync_threads();
+
+  if (__gpu_thread_id(0) == 0) {
+    out[__gpu_block_id(0)] = 0;
+    for (uint32_t i = 0; i < __gpu_num_threads(0); i++)
+      out[__gpu_block_id(0)] += shared_mem[i];
+  }
+}
diff --git a/offload/unittests/OffloadAPI/init/olInit.cpp b/offload/unittests/OffloadAPI/init/olInit.cpp
index 8e27e77cd0fb..508615152b4f 100644
--- a/offload/unittests/OffloadAPI/init/olInit.cpp
+++ b/offload/unittests/OffloadAPI/init/olInit.cpp
@@ -15,8 +15,20 @@
 
 struct olInitTest : ::testing::Test {};
 
+TEST_F(olInitTest, Success) {
+  ASSERT_SUCCESS(olInit());
+  ASSERT_SUCCESS(olShutDown());
+}
+
 TEST_F(olInitTest, Uninitialized) {
   ASSERT_ERROR(OL_ERRC_UNINITIALIZED,
                olIterateDevices(
                    [](ol_device_handle_t, void *) { return false; }, nullptr));
 }
+
+TEST_F(olInitTest, RepeatedInit) {
+  for (size_t I = 0; I < 10; I++) {
+    ASSERT_SUCCESS(olInit());
+    ASSERT_SUCCESS(olShutDown());
+  }
+}
diff --git a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
index a3da334afcca..b444aed9a6be 100644
--- a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
+++ b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
@@ -38,21 +38,19 @@ struct LaunchKernelTestBase : OffloadQueueTest {
   ol_kernel_launch_size_args_t LaunchArgs{};
 };
 
-struct olLaunchKernelTest : LaunchKernelTestBase {
-  void SetUp() override {
-    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("foo"));
-  }
-};
-OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelTest);
-
-struct olLaunchKernelNoArgsTest : LaunchKernelTestBase {
-  void SetUp() override {
-    RETURN_ON_FATAL_FAILURE(LaunchKernelTestBase::SetUpKernel("noargs"));
-  }
-};
-OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernelNoArgsTest);
-
-TEST_P(olLaunchKernelTest, Success) {
+#define KERNEL_TEST(NAME, KERNEL)                                              \
+  struct olLaunchKernel##NAME##Test : LaunchKernelTestBase {                   \
+    void SetUp() override { LaunchKernelTestBase::SetUpKernel(#KERNEL); }      \
+  };                                                                           \
+  OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchKernel##NAME##Test);
+
+KERNEL_TEST(Foo, foo)
+KERNEL_TEST(NoArgs, noargs)
+KERNEL_TEST(LocalMem, localmem)
+KERNEL_TEST(LocalMemReduction, localmem_reduction)
+KERNEL_TEST(LocalMemStatic, localmem_static)
+
+TEST_P(olLaunchKernelFooTest, Success) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
@@ -80,7 +78,7 @@ TEST_P(olLaunchKernelNoArgsTest, Success) {
   ASSERT_SUCCESS(olWaitQueue(Queue));
 }
 
-TEST_P(olLaunchKernelTest, SuccessSynchronous) {
+TEST_P(olLaunchKernelFooTest, SuccessSynchronous) {
   void *Mem;
   ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
                             LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
@@ -99,3 +97,74 @@ TEST_P(olLaunchKernelTest, SuccessSynchronous) {
 
   ASSERT_SUCCESS(olMemFree(Mem));
 }
+
+TEST_P(olLaunchKernelLocalMemTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x *
+                                sizeof(uint32_t),
+                            &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.GroupSize.x * LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], (i % 64) * 2);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
+
+TEST_P(olLaunchKernelLocalMemReductionTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 64 * sizeof(uint32_t);
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
+
+TEST_P(olLaunchKernelLocalMemStaticTest, Success) {
+  LaunchArgs.NumGroups.x = 4;
+  LaunchArgs.DynSharedMemory = 0;
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.NumGroups.x * sizeof(uint32_t), &Mem));
+  struct {
+    void *Mem;
+  } Args{Mem};
+
+  ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                &LaunchArgs, nullptr));
+
+  ASSERT_SUCCESS(olWaitQueue(Queue));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < LaunchArgs.NumGroups.x; i++)
+    ASSERT_EQ(Data[i], 2 * LaunchArgs.GroupSize.x);
+
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 9e93276c61cd..b5d01fb061ff 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -68,6 +68,8 @@ else()
     set(OPENMP_INSTALL_LIBDIR "lib${LLVM_LIBDIR_SUFFIX}" CACHE STRING
         "Path where built OpenMP libraries should be installed.")
   endif()
+  set(OPENMP_INSTALL_CFGDIR "lib/cmake" CACHE STRING
+        "Path where OpenMP config should be installed")
 
   if (NOT MSVC)
     set(OPENMP_TEST_C_COMPILER ${LLVM_RUNTIME_OUTPUT_INTDIR}/clang)
@@ -123,6 +125,8 @@ if (OPENMP_STANDALONE_BUILD)
     set(OPENMP_LIBDIR_SUFFIX "/asan")
     set(OPENMP_INSTALL_LIBDIR "lib${OPENMP_LIBDIR_SUFFIX}")
   endif()
+  set(OPENMP_INSTALL_CFGDIR "${OPENMP_INSTALL_LIBDIR}/cmake" CACHE STRING
+        "Path where OpenMP config should be installed")
 endif()
 
 # Check and set up common compiler flags.
diff --git a/openmp/runtime/openmp-config.cmake.in b/openmp/runtime/openmp-config.cmake.in
index 73b832080a6c..20eb531c97db 100644
--- a/openmp/runtime/openmp-config.cmake.in
+++ b/openmp/runtime/openmp-config.cmake.in
@@ -1,12 +1,13 @@
 @PACKAGE_INIT@
-get_filename_component(openmp_PACKAGE_PREFIX_DIR "${PACKAGE_PREFIX_DIR}" REALPATH)
 
+# Partial path copied from build variable OPENMP_INSTALL_LIBDIR
 set( openmp_LIB_DIR "@OPENMP_INSTALL_LIBDIR@" )
-set_and_check( openmp_LIB_INSTALL_DIR "@CMAKE_INSTALL_PREFIX@/@OPENMP_INSTALL_LIBDIR@" )
-set_and_check( openmp_LIB_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@")
 
-string(REPLACE "${openmp_PACKAGE_PREFIX_DIR}/" "" openmp_LIB_NO_PACKAGE_PREFIX_PATH_TEMP ${openmp_LIB_INSTALL_DIR})
-set(openmp_LIB_NO_PACKAGE_PREFIX_PATH ${openmp_LIB_NO_PACKAGE_PREFIX_PATH_TEMP})
+# Full path to libomp.so using PACKAGE_PREFIX_DIR and OPENMP_INSTALL_LIBDIR partial path.
+set_and_check( openmp_LIB_INSTALL_DIR "@PACKAGE_OPENMP_INSTALL_LIBDIR@" )
 
+# Full path to omp.h using PACKAGE_PREFIX and LIBOMP_HEADERS_INSTALL_PATH partial path.
+set_and_check( openmp_INCLUDE_DIR "@PACKAGE_LIBOMP_HEADERS_INSTALL_PATH@" )
+set_and_check( openmp_INCLUDE_DIRS "${openmp_INCLUDE_DIR}" )
 
 include( "${CMAKE_CURRENT_LIST_DIR}/openmpTargets.cmake" )
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 09f5a586c907..72c1ba25d05f 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -463,17 +463,21 @@ else()
     install(TARGETS omp ${export_to_llvmexports} EXPORT openmpTargets ${LIBOMP_INSTALL_KIND}
             DESTINATION "${OPENMP_INSTALL_LIBDIR}")
     install(EXPORT openmpTargets FILE openmpTargets.cmake NAMESPACE OpenMP::
-            DESTINATION ${OPENMP_INSTALL_LIBDIR}/cmake/openmp)
+            DESTINATION ${OPENMP_INSTALL_CFGDIR}/openmp)
+    target_include_directories(omp PUBLIC $<INSTALL_INTERFACE:${LIBOMP_HEADERS_INSTALL_PATH}>)
   endif()
+
   # Create cmake configuration files
   include(CMakePackageConfigHelpers)
 
   configure_package_config_file(
    ../openmp-config.cmake.in
    openmp-config.cmake
-   INSTALL_DESTINATION ${OPENMP_INSTALL_LIBDIR}/cmake/openmp )
+   INSTALL_DESTINATION ${OPENMP_INSTALL_CFGDIR}/openmp
+   PATH_VARS LIBOMP_HEADERS_INSTALL_PATH OPENMP_INSTALL_LIBDIR)
+
   install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openmp-config.cmake
-          DESTINATION ${OPENMP_INSTALL_LIBDIR}/cmake/openmp )
+          DESTINATION ${OPENMP_INSTALL_CFGDIR}/openmp)
 
   if(${LIBOMP_INSTALL_ALIASES})
     # Create aliases (symlinks) of the library for backwards compatibility
diff --git a/openmp/runtime/src/kmp_alloc.cpp b/openmp/runtime/src/kmp_alloc.cpp
index cc79ea4a5255..3efb80681a39 100644
--- a/openmp/runtime/src/kmp_alloc.cpp
+++ b/openmp/runtime/src/kmp_alloc.cpp
@@ -70,10 +70,10 @@ static void bectl(kmp_info_t *th, bget_compact_t compact,
 /* Buffer allocation size quantum: all buffers allocated are a
    multiple of this size.  This MUST be a power of two. */
 
-/* On IA-32 architecture with  Linux* OS, malloc() does not
-   ensure 16 byte alignment */
+/* On some architectures, malloc() does not ensure 16 byte alignment,
+   Solaris/sparc and x86 among them. */
 
-#if KMP_ARCH_X86 || !KMP_HAVE_QUAD
+#if KMP_ARCH_X86 || KMP_ARCH_SPARC || !KMP_HAVE_QUAD
 
 #define SizeQuant 8
 #define AlignType double
@@ -1923,7 +1923,7 @@ typedef struct kmp_mem_desc { // Memory block descriptor
   void *ptr_align; // Pointer to aligned memory, returned
   kmp_allocator_t *allocator; // allocator
 } kmp_mem_desc_t;
-static int alignment = sizeof(void *); // align to pointer size by default
+constexpr size_t alignment = SizeQuant;
 
 // external interfaces are wrappers over internal implementation
 void *__kmpc_alloc(int gtid, size_t size, omp_allocator_handle_t allocator) {
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index fdbf9ff45e35..3ca32ba583fe 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -570,7 +570,7 @@ void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
 
   this_thr->th.th_teams_microtask = NULL;
   this_thr->th.th_teams_level = 0;
-  *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
+  memset(&this_thr->th.th_teams_size, 0, sizeof(kmp_teams_size_t));
   va_end(ap);
 #if KMP_STATS_ENABLED
   if (previous_state == stats_state_e::SERIAL_REGION) {
diff --git a/openmp/runtime/src/kmp_lock.cpp b/openmp/runtime/src/kmp_lock.cpp
index 0ad14f862bcb..11fa233c4bd2 100644
--- a/openmp/runtime/src/kmp_lock.cpp
+++ b/openmp/runtime/src/kmp_lock.cpp
@@ -3242,6 +3242,8 @@ static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
   kmp_uint32 gtid = __kmp_entry_gtid();
   kmp_indirect_lock_t *l =
       __kmp_lookup_indirect_lock((void **)lock, "omp_destroy_lock");
+  if (l == nullptr)
+    return; // avoid segv if lock already destroyed
   KMP_I_LOCK_FUNC(l, destroy)(l->lock);
   kmp_indirect_locktag_t tag = l->type;
 
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index e6d509bec8b2..36d27b4218e2 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -1528,7 +1528,7 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
   // Calculate shared structure offset including padding after kmp_task_t struct
   // to align pointers in shared struct
   shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
-  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *));
+  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
 
   // Allocate a kmp_taskdata_t block and a kmp_task_t block.
   KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,
diff --git a/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp b/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
new file mode 100644
index 000000000000..4d57f069a4b1
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
@@ -0,0 +1,43 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s
+
+// disabled until fixed, see: https://github.com/llvm/llvm-project/pull/145625#issuecomment-3007625680
+// remove "needs-fix", after fixing the issue in the runtime
+// REQUIRES: ompt, needs-fix
+#include "callback.h"
+#include "omp_testsuite.h"
+
+// tests that the destructor doesn't segv even though
+// ompt_finalize_tool() destroys the lock
+struct myLock {
+  omp_lock_t lock;
+  myLock() { omp_init_lock(&lock); }
+  ~myLock() { omp_destroy_lock(&lock); }
+};
+
+myLock lock;
+
+int main() {
+  go_parallel_nthreads(2);
+
+  printf("Before ompt_finalize_tool\n");
+  ompt_finalize_tool();
+  printf("After ompt_finalize_tool\n");
+
+  return get_exit_value();
+}
+
+// CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+// CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin:
+// CHECK-SAME: thread_type=ompt_thread_initial=1
+
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_init_lock
+
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_end
+
+// CHECK: {{^}}Before ompt_finalize_tool
+
+// CHECK: {{^}}[[THREAD_ID]]: ompt_event_thread_end: thread_id=[[THREAD_ID]]
+// CHECK: 0: ompt_event_runtime_shutdown
+
+// CHECK: {{^}}After ompt_finalize_tool
diff --git a/openmp/runtime/tools/message-converter.py b/openmp/runtime/tools/message-converter.py
index a493d64c1692..5e2aeba6c99d 100644
--- a/openmp/runtime/tools/message-converter.py
+++ b/openmp/runtime/tools/message-converter.py
@@ -11,7 +11,6 @@
 #
 
 import argparse
-import datetime
 import os
 import platform
 import re
@@ -188,11 +187,10 @@ def insert_header(f, data, commentChar="//"):
     f.write(
         "{0} Do not edit this file! {0}\n"
         "{0} The file was generated from"
-        " {1} by {2} on {3}. {0}\n\n".format(
+        " {1} by {2}. {0}\n\n".format(
             commentChar,
             os.path.basename(data.filename),
             os.path.basename(__file__),
-            datetime.datetime.now().ctime(),
         )
     )
 
diff --git a/pstl/.clang-format b/pstl/.clang-format
deleted file mode 100644
index 8a110c1cec8b..000000000000
--- a/pstl/.clang-format
+++ /dev/null
@@ -1,20 +0,0 @@
-BasedOnStyle: LLVM
-
-Language: Cpp
-Standard: Cpp11
-
-IndentWidth: 4
-ColumnLimit: 120
-
-AlwaysBreakTemplateDeclarations: true
-AlwaysBreakAfterReturnType: All
-PointerAlignment: Left
-AllowShortIfStatementsOnASingleLine: false
-BreakBeforeBraces: Allman
-
-# Disable formatting options which may break tests.
-SortIncludes: false
-ReflowComments: false
-
-# Indent preprocessor directives
-IndentPPDirectives: AfterHash
diff --git a/pstl/CMakeLists.txt b/pstl/CMakeLists.txt
deleted file mode 100644
index 592e11d35647..000000000000
--- a/pstl/CMakeLists.txt
+++ /dev/null
@@ -1,103 +0,0 @@
-#===-- CMakeLists.txt ----------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-cmake_minimum_required(VERSION 3.20.0)
-set(LLVM_SUBPROJECT_TITLE "Parallel STL")
-
-set(PARALLELSTL_VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/include/pstl/internal/pstl_config.h")
-file(STRINGS "${PARALLELSTL_VERSION_FILE}" PARALLELSTL_VERSION_SOURCE REGEX "#define _PSTL_VERSION .*$")
-string(REGEX REPLACE "#define _PSTL_VERSION (.*)$" "\\1" PARALLELSTL_VERSION_SOURCE "${PARALLELSTL_VERSION_SOURCE}")
-math(EXPR VERSION_MAJOR "(${PARALLELSTL_VERSION_SOURCE} / 1000)")
-math(EXPR VERSION_MINOR "((${PARALLELSTL_VERSION_SOURCE} % 1000) / 10)")
-math(EXPR VERSION_PATCH "(${PARALLELSTL_VERSION_SOURCE} % 10)")
-
-project(ParallelSTL VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH} LANGUAGES CXX)
-
-# Must go below project(..)
-include(GNUInstallDirs)
-
-set(PSTL_PARALLEL_BACKEND "serial" CACHE STRING "Threading backend to use. Valid choices are 'serial', 'omp', and 'tbb'. The default is 'serial'.")
-set(PSTL_HIDE_FROM_ABI_PER_TU OFF CACHE BOOL "Whether to constrain ABI-unstable symbols to each translation unit (basically, mark them with C's static keyword).")
-set(_PSTL_HIDE_FROM_ABI_PER_TU ${PSTL_HIDE_FROM_ABI_PER_TU}) # For __pstl_config_site
-
-if (NOT TBB_DIR)
-    get_filename_component(PSTL_DIR_NAME ${CMAKE_CURRENT_SOURCE_DIR} NAME)
-    string(REPLACE pstl tbb TBB_DIR_NAME ${PSTL_DIR_NAME})
-    if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../${TBB_DIR_NAME}/cmake")
-        get_filename_component(TBB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${TBB_DIR_NAME}/cmake" ABSOLUTE)
-    endif()
-endif()
-
-###############################################################################
-# Setup the ParallelSTL library target
-###############################################################################
-add_library(ParallelSTL INTERFACE)
-add_library(pstl::ParallelSTL ALIAS ParallelSTL)
-target_compile_features(ParallelSTL INTERFACE cxx_std_17)
-
-if (PSTL_PARALLEL_BACKEND STREQUAL "serial")
-    message(STATUS "Parallel STL uses the serial backend")
-    set(_PSTL_PAR_BACKEND_SERIAL ON)
-elseif (PSTL_PARALLEL_BACKEND STREQUAL "tbb")
-    find_package(TBB 2018 REQUIRED tbb OPTIONAL_COMPONENTS tbbmalloc)
-    message(STATUS "Parallel STL uses TBB ${TBB_VERSION} (interface version: ${TBB_INTERFACE_VERSION})")
-    target_link_libraries(ParallelSTL INTERFACE TBB::tbb)
-    set(_PSTL_PAR_BACKEND_TBB ON)
-elseif (PSTL_PARALLEL_BACKEND STREQUAL "omp")
-    message(STATUS "Parallel STL uses the omp backend")
-    target_compile_options(ParallelSTL INTERFACE "-fopenmp=libomp")
-    set(_PSTL_PAR_BACKEND_OPENMP ON)
-else()
-    message(FATAL_ERROR "Requested unknown Parallel STL backend '${PSTL_PARALLEL_BACKEND}'.")
-endif()
-
-set(PSTL_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_headers")
-set(PSTL_CONFIG_SITE_PATH "${PSTL_GENERATED_HEADERS_DIR}/__pstl_config_site")
-configure_file("include/__pstl_config_site.in"
-               "${PSTL_CONFIG_SITE_PATH}"
-               @ONLY)
-
-target_include_directories(ParallelSTL
-    INTERFACE
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<BUILD_INTERFACE:${PSTL_GENERATED_HEADERS_DIR}>
-    $<INSTALL_INTERFACE:include>)
-
-###############################################################################
-# Setup tests
-###############################################################################
-enable_testing()
-add_subdirectory(test)
-
-###############################################################################
-# Install the target and the associated CMake files
-###############################################################################
-include(CMakePackageConfigHelpers)
-write_basic_package_version_file("${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfigVersion.cmake"
-                                 COMPATIBILITY ExactVersion)
-
-configure_file(cmake/ParallelSTLConfig.cmake.in
-               "${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfig.cmake"
-               @ONLY)
-
-install(TARGETS ParallelSTL
-        EXPORT ParallelSTLTargets)
-install(EXPORT ParallelSTLTargets
-        FILE ParallelSTLTargets.cmake
-        NAMESPACE pstl::
-        DESTINATION lib/cmake/ParallelSTL)
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfig.cmake"
-              "${CMAKE_CURRENT_BINARY_DIR}/ParallelSTLConfigVersion.cmake"
-        DESTINATION lib/cmake/ParallelSTL)
-install(DIRECTORY include/
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-        PATTERN "*.in" EXCLUDE)
-install(FILES "${PSTL_CONFIG_SITE_PATH}"
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-
-add_custom_target(install-pstl
-                  COMMAND "${CMAKE_COMMAND}" -P "${PROJECT_BINARY_DIR}/cmake_install.cmake" -DCOMPONENT=ParallelSTL)
diff --git a/pstl/CREDITS.txt b/pstl/CREDITS.txt
deleted file mode 100644
index 4945fd5ad308..000000000000
--- a/pstl/CREDITS.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-This file is a partial list of people who have contributed to the LLVM/pstl
-(Parallel STL) project.  If you have contributed a patch or made some other
-contribution to LLVM/pstl, please submit a patch to this file to add yourself,
-and it will be done!
-
-The list is sorted by surname and formatted to allow easy grepping and
-beautification by scripts.  The fields are: name (N), email (E), web-address 
-(W), PGP key ID and fingerprint (P), description (D), and snail-mail address
-(S).
-
-N: Intel Corporation
-W: http://www.intel.com
-D: Created the initial implementation.
-
-N: Thomas Rodgers
-E: trodgers@redhat.com
-D: Identifier name transformation for inclusion in a Standard C++ library.
-
-N: Christopher Nelson
-E: nadiasvertex@gmail.com
-D: Add support for an OpenMP backend.
diff --git a/pstl/LICENSE.TXT b/pstl/LICENSE.TXT
deleted file mode 100644
index 9eda544c8a81..000000000000
--- a/pstl/LICENSE.TXT
+++ /dev/null
@@ -1,307 +0,0 @@
-==============================================================================
-The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
-==============================================================================
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-    1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-    2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-    3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-    4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-    5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-    6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-    7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-    8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-    9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-    END OF TERMS AND CONDITIONS
-
-    APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-    Copyright [yyyy] [name of copyright owner]
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-
----- LLVM Exceptions to the Apache 2.0 License ----
-
-As an exception, if, as a result of your compiling your source code, portions
-of this Software are embedded into an Object form of such source code, you
-may redistribute such embedded portions in such Object form without complying
-with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
-
-In addition, if you combine or link compiled forms of this Software with
-software that is licensed under the GPLv2 ("Combined Software") and if a
-court of competent jurisdiction determines that the patent provision (Section
-3), the indemnity provision (Section 9) or other Section of the License
-conflicts with the conditions of the GPLv2, you may retroactively and
-prospectively choose to deem waived or otherwise exclude such Section(s) of
-the License, but only in their entirety and only with respect to the Combined
-Software.
-
-==============================================================================
-Software from third parties included in the LLVM Project:
-==============================================================================
-The LLVM Project contains third party software which is under different license
-terms. All such code will be identified clearly using at least one of two
-mechanisms:
-1) It will be in a separate directory tree with its own `LICENSE.txt` or
-   `LICENSE` file at the top containing the specific license and restrictions
-   which apply to that software, or
-2) It will contain specific license and restriction terms at the top of every
-   file.
-
-==============================================================================
-Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
-==============================================================================
-
-The software contained in this directory tree is dual licensed under both the
-University of Illinois "BSD-Like" license and the MIT license.  As a user of
-this code you may choose to use it under either license.  As a contributor,
-you agree to allow your code to be used under both.
-
-Full text of the relevant licenses is included below.
-
-==============================================================================
-
-University of Illinois/NCSA
-Open Source License
-
-Copyright (c) 2017-2019 by the contributors listed in CREDITS.TXT
-
-All rights reserved.
-
-Developed by:
-    Threading Runtimes Team
-    Intel Corporation
-    http://www.intel.com
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal with
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-    * Redistributions of source code must retain the above copyright notice,
-      this list of conditions and the following disclaimers.
-
-    * Redistributions in binary form must reproduce the above copyright notice,
-      this list of conditions and the following disclaimers in the
-      documentation and/or other materials provided with the distribution.
-
-    * Neither the names of Intel Corporation Threading Runtimes Team nor the 
-      names of its contributors may be used to endorse or promote products 
-      derived from this Software without specific prior written permission.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
-SOFTWARE.
-
-==============================================================================
-
-Copyright (c) 2017-2019 by the contributors listed in CREDITS.TXT
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff --git a/pstl/README.md b/pstl/README.md
deleted file mode 100644
index 5c862cbde4a7..000000000000
--- a/pstl/README.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# Parallel STL 
-
-Parallel STL is an implementation of the C++ standard library algorithms with support for execution policies,
-as specified in ISO/IEC 14882:2017 standard, commonly called C++17. The implementation also supports the unsequenced 
-execution policy specified in Parallelism TS version 2 and proposed for the next version of the C++ standard in the 
-C++ working group paper [P1001](https://wg21.link/p1001).
-Parallel STL offers efficient support for both parallel and vectorized execution of algorithms. For sequential 
-execution, it relies on an available implementation of the C++ standard library.
-
-## Prerequisites
-
-To use Parallel STL, you must have the following software installed:
-* C++ compiler with:
-  * Support for C++11
-  * Support for OpenMP* 4.0 SIMD constructs
-* Threading Building Blocks (TBB) which is available for download at https://github.com/01org/tbb/
-
-## Known issues and limitations
-
-* `unseq` and `par_unseq` policies only have effect with compilers that support `#pragma omp simd` or `#pragma simd`.
-* Parallel and vector execution is only supported for the algorithms if random access iterators are provided,
-  while for other iterator types the execution will remain serial.
-* The following algorithms do not allow efficient SIMD execution: `includes`, `inplace_merge`, `merge`, `nth_element`,
-  `partial_sort`, `partial_sort_copy`, `set_difference`, `set_intersection`, `set_symmetric_difference`, `set_union`,
-  `sort`, `stable_partition`, `stable_sort`, `unique`.
-* The initial value type for `exclusive_scan`, `inclusive_scan`, `transform_exclusive_scan`, `transform_inclusive_scan`
-  shall be DefaultConstructible. A default constructed-instance of the initial value type shall be the identity element
-  for the specified binary operation.
-* For `max_element`, `min_element`, `minmax_element`, `partial_sort`, `partial_sort_copy`, `sort`, `stable_sort`
-  the dereferenced value type of the provided iterators shall be DefaultConstructible.
-* For `remove`, `remove_if`, `unique` the dereferenced value type of the provided iterators shall be MoveConstructible.
-* The following algorithms require additional O(n) memory space for parallel execution: `copy_if`, `inplace_merge`,
-  `partial_sort`, `partial_sort_copy`, `partition_copy`, `remove`, `remove_if`, `rotate`, `sort`, `stable_sort`,
-  `unique`, `unique_copy`.
-
diff --git a/pstl/cmake/ParallelSTLConfig.cmake.in b/pstl/cmake/ParallelSTLConfig.cmake.in
deleted file mode 100644
index 445ebafce921..000000000000
--- a/pstl/cmake/ParallelSTLConfig.cmake.in
+++ /dev/null
@@ -1,19 +0,0 @@
-#===-- ParallelSTLConfig.cmake.in ----------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-include(CMakeFindDependencyMacro)
-
-set(PSTL_PARALLEL_BACKEND "@PSTL_PARALLEL_BACKEND@")
-
-if (PSTL_PARALLEL_BACKEND STREQUAL "tbb")
-    find_dependency(TBB REQUIRED tbb)
-endif()
-
-if (NOT TARGET pstl::ParallelSTL)
-    include("${CMAKE_CURRENT_LIST_DIR}/ParallelSTLTargets.cmake")
-endif()
diff --git a/pstl/docs/ReleaseNotes.rst b/pstl/docs/ReleaseNotes.rst
deleted file mode 100644
index a0b375134f78..000000000000
--- a/pstl/docs/ReleaseNotes.rst
+++ /dev/null
@@ -1,40 +0,0 @@
-=======================================
-PSTL 21.0.0 (In-Progress) Release Notes
-=======================================
-
-.. contents::
-   :local:
-   :depth: 2
-
-Written by the `PSTL Team <https://pstl.llvm.org>`_
-
-.. warning::
-
-   These are in-progress notes for the upcoming pstl 19 release.
-   Release notes for previous releases can be found on
-   `the Download Page <https://releases.llvm.org/download.html>`_.
-
-Introduction
-============
-
-This document contains the release notes for the PSTL parallel algorithms
-library, part of the LLVM Compiler Infrastructure, release 17.0.0. Here we
-describe the status of the library in some detail, including major improvements
-from the previous release and new feature work. For the general LLVM release
-notes, see `the LLVM documentation <https://llvm.org/docs/ReleaseNotes.html>`_.
-All LLVM releases may be downloaded from the `LLVM releases web site
-<https://llvm.org/releases/>`_.
-
-Note that if you are reading this file from a source checkout or the main PSTL
-web page, this document applies to the *next* release, not the current one.
-To see the release notes for a specific release, please see the `releases
-page <https://llvm.org/releases/>`_.
-
-What's New in PSTL 19.0.0?
-==========================
-
-New Features
-------------
-
-API Changes
------------
diff --git a/pstl/include/__pstl_algorithm b/pstl/include/__pstl_algorithm
deleted file mode 100644
index 79c18385c5a5..000000000000
--- a/pstl/include/__pstl_algorithm
+++ /dev/null
@@ -1,15 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef __PSTL_ALGORITHM
-#define __PSTL_ALGORITHM
-
-#include <pstl/internal/glue_algorithm_impl.h>
-
-#endif /* __PSTL_ALGORITHM */
diff --git a/pstl/include/__pstl_config_site.in b/pstl/include/__pstl_config_site.in
deleted file mode 100644
index a41a1c383a45..000000000000
--- a/pstl/include/__pstl_config_site.in
+++ /dev/null
@@ -1,17 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef __PSTL_CONFIG_SITE
-#define __PSTL_CONFIG_SITE
-
-#cmakedefine _PSTL_PAR_BACKEND_SERIAL
-#cmakedefine _PSTL_PAR_BACKEND_TBB
-#cmakedefine _PSTL_PAR_BACKEND_OPENMP
-#cmakedefine _PSTL_HIDE_FROM_ABI_PER_TU
-
-#endif // __PSTL_CONFIG_SITE
diff --git a/pstl/include/__pstl_execution b/pstl/include/__pstl_execution
deleted file mode 100644
index 0e2cd44561cd..000000000000
--- a/pstl/include/__pstl_execution
+++ /dev/null
@@ -1,15 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef __PSTL_EXECUTION
-#define __PSTL_EXECUTION
-
-#include <pstl/internal/glue_execution_defs.h>
-
-#endif /* __PSTL_EXECUTION */
diff --git a/pstl/include/__pstl_numeric b/pstl/include/__pstl_numeric
deleted file mode 100644
index cf168ef7053b..000000000000
--- a/pstl/include/__pstl_numeric
+++ /dev/null
@@ -1,15 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef __PSTL_NUMERIC
-#define __PSTL_NUMERIC
-
-#include <pstl/internal/glue_numeric_impl.h>
-
-#endif /* __PSTL_NUMERIC */
diff --git a/pstl/include/pstl/internal/algorithm_fwd.h b/pstl/include/pstl/internal/algorithm_fwd.h
deleted file mode 100644
index 3dcf9e9f5e9e..000000000000
--- a/pstl/include/pstl/internal/algorithm_fwd.h
+++ /dev/null
@@ -1,1202 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_ALGORITHM_FWD_H
-#define _PSTL_ALGORITHM_FWD_H
-
-#include <iterator>
-#include <type_traits>
-#include <utility>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// any_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Pred>
-bool
-__brick_any_of(const _ForwardIterator, const _ForwardIterator, _Pred,
-               /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Pred>
-bool
-__brick_any_of(const _RandomAccessIterator, const _RandomAccessIterator, _Pred,
-               /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-bool
-__pattern_any_of(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Pred) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Pred>
-bool
-__pattern_any_of(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Pred);
-
-//------------------------------------------------------------------------
-// walk1 (pseudo)
-//
-// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Function>
-void __brick_walk1(_ForwardIterator, _ForwardIterator, _Function,
-                   /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Function>
-void __brick_walk1(_RandomAccessIterator, _RandomAccessIterator, _Function,
-                   /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
-void
-__pattern_walk1(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
-void
-__pattern_walk1(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
-void
-__pattern_walk_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
-void
-__pattern_walk_brick(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                     _Brick);
-
-//------------------------------------------------------------------------
-// walk1_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator __brick_walk1_n(_ForwardIterator, _Size, _Function,
-                                 /*_IsVectorTag=*/std::false_type);
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Function>
-_RandomAccessIterator __brick_walk1_n(_RandomAccessIterator, _DifferenceType, _Function,
-                                      /*vectorTag=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
-_RandomAccessIterator
-__pattern_walk1_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
-_ForwardIterator
-__pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _Size, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
-_RandomAccessIterator
-__pattern_walk_brick_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, _Brick);
-
-//------------------------------------------------------------------------
-// walk2 (pseudo)
-//
-// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2 __brick_walk2(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Function,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2 __brick_walk2(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _Function,
-                                     /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
-_ForwardIterator2 __brick_walk2_n(_ForwardIterator1, _Size, _ForwardIterator2, _Function,
-                                  /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2 __brick_walk2_n(_RandomAccessIterator1, _Size, _RandomAccessIterator2, _Function,
-                                       /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__pattern_walk2(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Function>
-_RandomAccessIterator2
-__pattern_walk2(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Function>
-_ForwardIterator2
-__pattern_walk2_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__pattern_walk2_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size, _RandomAccessIterator2,
-                  _Function);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                      _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                      _RandomAccessIterator2, _Brick);
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _Size, _ForwardIterator2, _Brick) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _Size,
-                        _RandomAccessIterator2, _Brick);
-
-//------------------------------------------------------------------------
-// walk3 (pseudo)
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
-_ForwardIterator3 __brick_walk3(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator3, _Function,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3 __brick_walk3(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                     _RandomAccessIterator3, _Function,
-                                     /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
-          class _Function>
-_ForwardIterator3
-__pattern_walk3(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator3,
-                _Function) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__pattern_walk3(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator3, _Function);
-
-//------------------------------------------------------------------------
-// equal
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool __brick_equal(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool __brick_equal(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _BinaryPredicate);
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool __brick_equal(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2, _BinaryPredicate,
-                   /* is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool __brick_equal(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                   _BinaryPredicate, /* is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate);
-
-//------------------------------------------------------------------------
-// find_if
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Predicate>
-_ForwardIterator __brick_find_if(_ForwardIterator, _ForwardIterator, _Predicate,
-                                 /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator __brick_find_if(_RandomAccessIterator, _RandomAccessIterator, _Predicate,
-                                      /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__pattern_find_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__pattern_find_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                  _Predicate);
-
-//------------------------------------------------------------------------
-// find_end
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_find_end(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                   _BinaryPredicate,
-                                   /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_find_end(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                        _RandomAccessIterator2, _BinaryPredicate,
-                                        /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_end(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_end(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// find_first_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_find_first_of(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                        _BinaryPredicate,
-                                        /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_find_first_of(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                             _RandomAccessIterator2, _BinaryPredicate,
-                                             /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_first_of(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                        _ForwardIterator2, _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_first_of(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                        _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// search
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1 __brick_search(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                 _BinaryPredicate,
-                                 /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1 __brick_search(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                      _RandomAccessIterator2, _BinaryPredicate,
-                                      /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_search(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                 _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_search(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                 _RandomAccessIterator2, _RandomAccessIterator2, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// search_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__brick_search_n(_ForwardIterator, _ForwardIterator, _Size, const _Tp&, _BinaryPredicate,
-                 /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_search_n(_RandomAccessIterator, _RandomAccessIterator, _Size, const _Tp&, _BinaryPredicate,
-                 /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__pattern_search_n(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Size, const _Tp&,
-                   _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp,
-          class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_search_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Size,
-                   const _Tp&, _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// copy_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _OutputIterator>
-_OutputIterator __brick_copy_n(_ForwardIterator, _Size, _OutputIterator,
-                               /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Size, class _OutputIterator>
-_OutputIterator __brick_copy_n(_RandomAccessIterator, _Size, _OutputIterator,
-                               /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_copy(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                             /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                             /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// move
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_move(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                             /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_move(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                             /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// swap_ranges
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_swap_ranges(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                                    /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_swap_ranges(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                    /*vector=*/std::true_type) noexcept;
-
-//------------------------------------------------------------------------
-// copy_if
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator __brick_copy_if(_ForwardIterator, _ForwardIterator, _OutputIterator, _UnaryPredicate,
-                                /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator __brick_copy_if(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, _UnaryPredicate,
-                                /*vector=*/std::true_type) noexcept;
-
-template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_ForwardIterator, _ForwardIterator, bool* __restrict, _UnaryPredicate,
-                    /*vector=*/std::false_type) noexcept;
-template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _UnaryPredicate,
-                    /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _OutputIterator>
-void
-__brick_copy_by_mask(_ForwardIterator, _ForwardIterator, _OutputIterator, bool*,
-                     /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-void
-__brick_copy_by_mask(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, bool* __restrict,
-                     /*vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_ForwardIterator, _ForwardIterator, _OutputIterator1, _OutputIterator2, bool*,
-                          /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator1, _OutputIterator2, bool*,
-                          /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                  _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                  _OutputIterator, _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// count
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-    __brick_count(_RandomAccessIterator, _RandomAccessIterator, _Predicate,
-                  /* is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-    __brick_count(_ForwardIterator, _ForwardIterator, _Predicate,
-                  /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__pattern_count(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__pattern_count(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                _Predicate);
-
-//------------------------------------------------------------------------
-// unique
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator __brick_unique(_ForwardIterator, _ForwardIterator, _BinaryPredicate,
-                                /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator __brick_unique(_RandomAccessIterator, _RandomAccessIterator, _BinaryPredicate,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_unique(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_unique(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                 _BinaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// unique_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
-OutputIterator __brick_unique_copy(_ForwardIterator, _ForwardIterator, OutputIterator, _BinaryPredicate,
-                                   /*vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator __brick_unique_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator, _BinaryPredicate,
-                                    /*vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                      _BinaryPredicate) noexcept;
-
-template <class _ExecutionPolicy, class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _BinaryPredicate,
-                    /*vector=*/std::false_type) noexcept;
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator, _RandomAccessIterator, bool* __restrict, _BinaryPredicate,
-                    /*vector=*/std::true_type) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _OutputIterator, _BinaryPredicate);
-
-//------------------------------------------------------------------------
-// reverse
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator>
-void __brick_reverse(_BidirectionalIterator, _BidirectionalIterator,
-                     /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-void __brick_reverse(_RandomAccessIterator, _RandomAccessIterator,
-                     /*__is_vector=*/std::true_type) noexcept;
-
-template <class _BidirectionalIterator>
-void __brick_reverse(_BidirectionalIterator, _BidirectionalIterator, _BidirectionalIterator,
-                     /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-void __brick_reverse(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
-void
-__pattern_reverse(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-void
-__pattern_reverse(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator);
-
-//------------------------------------------------------------------------
-// reverse_copy
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator __brick_reverse_copy(_BidirectionalIterator, _BidirectionalIterator, _OutputIterator,
-                                     /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_reverse_copy(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                       _OutputIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                       _OutputIterator);
-
-//------------------------------------------------------------------------
-// rotate
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator>
-_ForwardIterator __brick_rotate(_ForwardIterator, _ForwardIterator, _ForwardIterator,
-                                /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator>
-_RandomAccessIterator __brick_rotate(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
-_ForwardIterator
-__pattern_rotate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-_RandomAccessIterator
-__pattern_rotate(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                 _RandomAccessIterator);
-
-//------------------------------------------------------------------------
-// rotate_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator __brick_rotate_copy(_ForwardIterator, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                                    /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator __brick_rotate_copy(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                                    _OutputIterator,
-                                    /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _ForwardIterator,
-                      _OutputIterator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _RandomAccessIterator, _OutputIterator);
-
-//------------------------------------------------------------------------
-// is_partitioned
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-bool __brick_is_partitioned(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                            /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-bool __brick_is_partitioned(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                            /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// partition
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator __brick_partition(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                                   /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_partition(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                        /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_partition(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_partition(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                    _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// stable_partition
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator __brick_stable_partition(_BidirectionalIterator, _BidirectionalIterator, _UnaryPredicate,
-                                                /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_stable_partition(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                               /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__pattern_stable_partition(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                           _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_stable_partition(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                           _UnaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// partition_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-    __brick_partition_copy(_ForwardIterator, _ForwardIterator, _OutputIterator1, _OutputIterator2, _UnaryPredicate,
-                           /*is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2> __brick_partition_copy(_RandomAccessIterator, _RandomAccessIterator,
-                                                                     _OutputIterator1, _OutputIterator2,
-                                                                     _UnaryPredicate,
-                                                                     /*is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
-          class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator1,
-                         _OutputIterator2, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator1,
-          class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _OutputIterator1, _OutputIterator2, _UnaryPredicate);
-
-//------------------------------------------------------------------------
-// sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
-void
-__pattern_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-               _IsMoveConstructible) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-               /*is_move_constructible=*/std::true_type);
-
-//------------------------------------------------------------------------
-// stable_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare);
-
-//------------------------------------------------------------------------
-// partial_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                       _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                       _RandomAccessIterator, _Compare);
-
-//------------------------------------------------------------------------
-// partial_sort_copy
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(_Tag, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                            _RandomAccessIterator2, _RandomAccessIterator2, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                            _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2, _Compare);
-
-//------------------------------------------------------------------------
-// adjacent_find
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_adjacent_find(_RandomAccessIterator, _RandomAccessIterator, _BinaryPredicate,
-                      /* IsVector = */ std::true_type, bool) noexcept;
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__brick_adjacent_find(_ForwardIterator, _ForwardIterator, _BinaryPredicate,
-                      /* IsVector = */ std::false_type, bool) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_adjacent_find(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _BinaryPredicate, bool) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_adjacent_find(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _BinaryPredicate, bool);
-
-//------------------------------------------------------------------------
-// nth_element
-//------------------------------------------------------------------------
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _RandomAccessIterator, _Compare) noexcept;
-
-//------------------------------------------------------------------------
-// fill, fill_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Tp>
-void
-__brick_fill(_RandomAccessIterator, _RandomAccessIterator, const _Tp&,
-             /* __is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Tp>
-void
-__brick_fill(_ForwardIterator, _ForwardIterator, const _Tp&,
-             /* __is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-void
-__pattern_fill(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, const _Tp&) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp>
-_RandomAccessIterator
-__pattern_fill(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, const _Tp&);
-
-template <class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__brick_fill_n(_RandomAccessIterator, _Size, const _Tp&,
-               /* __is_vector = */ std::true_type) noexcept;
-
-template <class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__brick_fill_n(_OutputIterator, _Size, const _Tp&,
-               /* __is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__pattern_fill_n(_Tag, _ExecutionPolicy&&, _OutputIterator, _Size, const _Tp&) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__pattern_fill_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _Size, const _Tp&);
-
-//------------------------------------------------------------------------
-// generate, generate_n
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Generator>
-void __brick_generate(_RandomAccessIterator, _RandomAccessIterator, _Generator,
-                      /* is_vector = */ std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Generator>
-void __brick_generate(_ForwardIterator, _ForwardIterator, _Generator,
-                      /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-void
-__pattern_generate(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Generator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
-_RandomAccessIterator
-__pattern_generate(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                   _Generator);
-
-template <class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator __brick_generate_n(_RandomAccessIterator, Size, _Generator,
-                                         /* is_vector = */ std::true_type) noexcept;
-
-template <class OutputIterator, class Size, class _Generator>
-OutputIterator __brick_generate_n(OutputIterator, Size, _Generator,
-                                  /* is_vector = */ std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class OutputIterator, class Size, class _Generator>
-OutputIterator
-__pattern_generate_n(_Tag, _ExecutionPolicy&&, OutputIterator, Size, _Generator) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator
-__pattern_generate_n(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, Size, _Generator);
-
-//------------------------------------------------------------------------
-// remove
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator __brick_remove_if(_ForwardIterator, _ForwardIterator, _UnaryPredicate,
-                                   /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator __brick_remove_if(_RandomAccessIterator, _RandomAccessIterator, _UnaryPredicate,
-                                        /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_remove_if(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _UnaryPredicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_remove_if(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                    _UnaryPredicate) noexcept;
-
-//------------------------------------------------------------------------
-// merge
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_merge(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                              _OutputIterator, _Compare,
-                              /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_merge(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                              _RandomAccessIterator2, _OutputIterator, _Compare,
-                              /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_merge(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_merge(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// inplace_merge
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _Compare>
-void __brick_inplace_merge(_BidirectionalIterator, _BidirectionalIterator, _BidirectionalIterator, _Compare,
-                           /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Compare>
-void __brick_inplace_merge(_RandomAccessIterator, _RandomAccessIterator, _RandomAccessIterator, _Compare,
-                           /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-void
-__pattern_inplace_merge(_Tag, _ExecutionPolicy&&, _BidirectionalIterator, _BidirectionalIterator,
-                        _BidirectionalIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_inplace_merge(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _RandomAccessIterator, _Compare);
-
-//------------------------------------------------------------------------
-// includes
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_includes(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_includes(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _Compare);
-
-//------------------------------------------------------------------------
-// set_union
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_union(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                  _OutputIterator, _Compare,
-                                  /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_union(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                  _RandomAccessIterator2, _OutputIterator, _Compare,
-                                  /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_union(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                    _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_union(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                    _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// set_intersection
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_intersection(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                         _OutputIterator, _Compare,
-                                         /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_intersection(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                         _RandomAccessIterator2, _OutputIterator, _Compare,
-                                         /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_intersection(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                           _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_intersection(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                           _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator,
-                           _Compare);
-
-//------------------------------------------------------------------------
-// set_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_difference(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                       _OutputIterator, _Compare,
-                                       /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_difference(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                       _RandomAccessIterator2, _OutputIterator, _Compare,
-                                       /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                         _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                         _RandomAccessIterator2, _RandomAccessIterator2, _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// set_symmetric_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_symmetric_difference(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                                 _ForwardIterator2, _OutputIterator, _Compare,
-                                                 /*__is_vector=*/std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator __brick_set_symmetric_difference(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                                 _RandomAccessIterator2, _OutputIterator, _Compare,
-                                                 /*__is_vector=*/std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                   _ForwardIterator2, _OutputIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                                   _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                                   _OutputIterator, _Compare);
-
-//------------------------------------------------------------------------
-// is_heap_until
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator __brick_is_heap_until(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                            /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator __brick_is_heap_until(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                            /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(_Tag, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                        _Compare) noexcept;
-
-//------------------------------------------------------------------------
-// min_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator __brick_min_element(_ForwardIterator, _ForwardIterator, _Compare,
-                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator __brick_min_element(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                                          /* __is_vector = */ std::true_type) noexcept;
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__pattern_min_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__pattern_min_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                      _Compare);
-
-//------------------------------------------------------------------------
-// minmax_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator> __brick_minmax_element(_ForwardIterator, _ForwardIterator, _Compare,
-                                                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-    __brick_minmax_element(_RandomAccessIterator, _RandomAccessIterator, _Compare,
-                           /* __is_vector = */ std::true_type) noexcept;
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__pattern_minmax_element(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Compare) noexcept;
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__pattern_minmax_element(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _Compare);
-
-//------------------------------------------------------------------------
-// mismatch
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2> __brick_mismatch(_ForwardIterator1, _ForwardIterator1,
-                                                                 _ForwardIterator2, _ForwardIterator2, _Predicate,
-                                                                 /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-    __brick_mismatch(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                     _Predicate,
-                     /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__pattern_mismatch(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                   _Predicate) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__pattern_mismatch(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1, _RandomAccessIterator1,
-                   _RandomAccessIterator2, _RandomAccessIterator2, _Predicate) noexcept;
-
-//------------------------------------------------------------------------
-// lexicographical_compare
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool __brick_lexicographical_compare(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _ForwardIterator2,
-                                     _Compare,
-                                     /* __is_vector = */ std::false_type) noexcept;
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
-bool __brick_lexicographical_compare(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2,
-                                     _RandomAccessIterator2, _Compare,
-                                     /* __is_vector = */ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_lexicographical_compare(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2,
-                                  _ForwardIterator2, _Compare) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_lexicographical_compare(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                                  _RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator2,
-                                  _Compare) noexcept;
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_ALGORITHM_FWD_H */
diff --git a/pstl/include/pstl/internal/algorithm_impl.h b/pstl/include/pstl/internal/algorithm_impl.h
deleted file mode 100644
index 2b505d94611f..000000000000
--- a/pstl/include/pstl/internal/algorithm_impl.h
+++ /dev/null
@@ -1,3819 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_ALGORITHM_IMPL_H
-#define _PSTL_ALGORITHM_IMPL_H
-
-#include <iterator>
-#include <type_traits>
-#include <utility>
-#include <functional>
-#include <algorithm>
-
-#include "execution_impl.h"
-#include "memory_impl.h"
-#include "parallel_backend.h"
-#include "parallel_backend_utils.h"
-#include "parallel_impl.h"
-#include "pstl_config.h"
-#include "unseq_backend_simd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// any_of
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Pred>
-bool
-__brick_any_of(const _ForwardIterator __first, const _ForwardIterator __last, _Pred __pred,
-               /*__is_vector=*/std::false_type) noexcept
-{
-    return std::any_of(__first, __last, __pred);
-};
-
-template <class _RandomAccessIterator, class _Pred>
-bool
-__brick_any_of(const _RandomAccessIterator __first, const _RandomAccessIterator __last, _Pred __pred,
-               /*__is_vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_or(__first, __last - __first, __pred);
-};
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-bool
-__pattern_any_of(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred) noexcept
-{
-    return __internal::__brick_any_of(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Pred>
-bool
-__pattern_any_of(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __last, _Pred __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_or(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                             [__pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                             { return __internal::__brick_any_of(__i, __j, __pred, _IsVector{}); });
-        });
-}
-
-// [alg.foreach]
-// for_each_n with no policy
-
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__for_each_n_it_serial(_ForwardIterator __first, _Size __n, _Function __f)
-{
-    for (; __n > 0; ++__first, --__n)
-        __f(__first);
-    return __first;
-}
-
-//------------------------------------------------------------------------
-// walk1 (pseudo)
-//
-// walk1 evaluates f(x) for each dereferenced value x drawn from [first,last)
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Function>
-void
-__brick_walk1(_ForwardIterator __first, _ForwardIterator __last, _Function __f, /*vector=*/std::false_type) noexcept
-{
-    std::for_each(__first, __last, __f);
-}
-
-template <class _RandomAccessIterator, class _Function>
-void
-__brick_walk1(_RandomAccessIterator __first, _RandomAccessIterator __last, _Function __f,
-              /*vector=*/std::true_type) noexcept
-{
-    __unseq_backend::__simd_walk_1(__first, __last - __first, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Function>
-void
-__pattern_walk1(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Function __f) noexcept
-{
-    __internal::__brick_walk1(__first, __last, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Function>
-void
-__pattern_walk1(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                _RandomAccessIterator __last, _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__f](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                          { __internal::__brick_walk1(__i, __j, __f, _IsVector{}); });
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Brick>
-void
-__pattern_walk_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                     _Brick __brick) noexcept
-{
-    __brick(__first, __last);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Brick>
-void
-__pattern_walk_brick(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                     _RandomAccessIterator __last, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                                          { __brick(__i, __j); });
-        });
-}
-
-//------------------------------------------------------------------------
-// walk1_n
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__brick_walk1_n(_ForwardIterator __first, _Size __n, _Function __f, /*_IsVectorTag=*/std::false_type)
-{
-    return __internal::__for_each_n_it_serial(__first, __n,
-                                              [&__f](_ForwardIterator __it) { __f(*__it); }); // calling serial version
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Function>
-_RandomAccessIterator
-__brick_walk1_n(_RandomAccessIterator __first, _DifferenceType __n, _Function __f,
-                /*vectorTag=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_1(__first, __n, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-_ForwardIterator
-__pattern_walk1_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Function __f) noexcept
-{
-    return __internal::__brick_walk1_n(__first, __n, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Function>
-_RandomAccessIterator
-__pattern_walk1_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first, _Size __n,
-                  _Function __f)
-{
-    __internal::__pattern_walk1(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __n, __f);
-
-    return __first + __n;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Brick>
-_ForwardIterator
-__pattern_walk_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _Size __n, _Brick __brick) noexcept
-{
-    return __brick(__first, __n);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Brick>
-_RandomAccessIterator
-__pattern_walk_brick_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                       _Size __n, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __first + __n,
-                [__brick](_RandomAccessIterator __i, _RandomAccessIterator __j) { __brick(__i, __j - __i); });
-            return __first + __n;
-        });
-}
-
-//------------------------------------------------------------------------
-// walk2 (pseudo)
-//
-// walk2 evaluates f(x,y) for deferenced values (x,y) drawn from [first1,last1) and [first2,...)
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__brick_walk2(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Function __f,
-              /*vector=*/std::false_type) noexcept
-{
-    for (; __first1 != __last1; ++__first1, ++__first2)
-        __f(*__first1, *__first2);
-    return __first2;
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__brick_walk2(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _Function __f,
-              /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_2(__first1, __last1 - __first1, __first2, __f);
-}
-
-template <class _ForwardIterator1, class _Size, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__brick_walk2_n(_ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2, _Function __f,
-                /*vector=*/std::false_type) noexcept
-{
-    for (; __n > 0; --__n, ++__first1, ++__first2)
-        __f(*__first1, *__first2);
-    return __first2;
-}
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__brick_walk2_n(_RandomAccessIterator1 __first1, _Size __n, _RandomAccessIterator2 __first2, _Function __f,
-                /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_2(__first1, __n, __first2, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Function>
-_ForwardIterator2
-__pattern_walk2(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _Function __f) noexcept
-{
-    return __internal::__brick_walk2(__first1, __last1, __first2, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Function>
-_RandomAccessIterator2
-__pattern_walk2(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__f, __first1, __first2](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __internal::__brick_walk2(__i, __j, __first2 + (__i - __first1), __f, _IsVector{}); });
-            return __first2 + (__last1 - __first1);
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Function>
-_ForwardIterator2
-__pattern_walk2_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
-                  _Function __f) noexcept
-{
-    return __internal::__brick_walk2_n(__first1, __n, __first2, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Function>
-_RandomAccessIterator2
-__pattern_walk2_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                  _Size __n, _RandomAccessIterator2 __first2, _Function __f)
-{
-    return __internal::__pattern_walk2(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-                                       __first2, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                      _ForwardIterator2 __first2, _Brick __brick) noexcept
-{
-    return __brick(__first1, __last1, __first2);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                      _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __brick(__i, __j, __first2 + (__i - __first1)); });
-            return __first2 + (__last1 - __first1);
-        });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2,
-          class _Brick>
-_ForwardIterator2
-__pattern_walk2_brick_n(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _Size __n, _ForwardIterator2 __first2,
-                        _Brick __brick) noexcept
-{
-    return __brick(__first1, __n, __first2);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _Size,
-          class _RandomAccessIterator2, class _Brick>
-_RandomAccessIterator2
-__pattern_walk2_brick_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                        _Size __n, _RandomAccessIterator2 __first2, _Brick __brick)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-                [__first1, __first2, __brick](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { __brick(__i, __j - __i, __first2 + (__i - __first1)); });
-            return __first2 + __n;
-        });
-}
-
-//------------------------------------------------------------------------
-// walk3 (pseudo)
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3, class _Function>
-_ForwardIterator3
-__brick_walk3(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator3 __first3, _Function __f, /*vector=*/std::false_type) noexcept
-{
-    for (; __first1 != __last1; ++__first1, ++__first2, ++__first3)
-        __f(*__first1, *__first2, *__first3);
-    return __first3;
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__brick_walk3(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator3 __first3, _Function __f, /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_walk_3(__first1, __last1 - __first1, __first2, __first3, __f);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator3,
-          class _Function>
-_ForwardIterator3
-__pattern_walk3(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator3 __first3, _Function __f) noexcept
-{
-    return __internal::__brick_walk3(__first1, __last1, __first2, __first3, __f, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Function>
-_RandomAccessIterator3
-__pattern_walk3(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator3 __first3,
-                _Function __f)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__f, __first1, __first2, __first3](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                    __internal::__brick_walk3(__i, __j, __first2 + (__i - __first1), __first3 + (__i - __first1), __f,
-                                              _IsVector{});
-                });
-            return __first3 + (__last1 - __first1);
-        });
-}
-
-//------------------------------------------------------------------------
-// equal
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator2 __last2, _BinaryPredicate __p, /* IsVector = */ std::false_type) noexcept
-{
-    return std::equal(__first1, __last1, __first2, __last2, __p);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator2 __last2, _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
-{
-    if (__last1 - __first1 != __last2 - __first2)
-        return false;
-
-    return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator2 __last2, _BinaryPredicate __p) noexcept
-{
-    return __internal::__brick_equal(__first1, __last1, __first2, __last2, __p, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                _BinaryPredicate __p)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last1 - __first1 != __last2 - __first2)
-        return false;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                {
-                    return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1),
-                                                      __first2 + (__j - __first1), __p, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// equal version for sequences with equal length
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _BinaryPredicate __p,
-              /* IsVector = */ std::false_type) noexcept
-{
-    return std::equal(__first1, __last1, __first2, __p);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-bool
-__brick_equal(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _BinaryPredicate __p, /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_first(__first1, __last1 - __first1, __first2, std::not_fn(__p)).first == __last1;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-bool
-__pattern_equal(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _BinaryPredicate __p) noexcept
-{
-    return __internal::__brick_equal(__first1, __last1, __first2, __p, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-bool
-__pattern_equal(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _BinaryPredicate __p)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __p](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { return !__internal::__brick_equal(__i, __j, __first2 + (__i - __first1), __p, _IsVector{}); });
-        });
-}
-
-//------------------------------------------------------------------------
-// find_if
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__brick_find_if(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
-                /*is_vector=*/std::false_type) noexcept
-{
-    return std::find_if(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__brick_find_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
-                /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __unseq_backend::__simd_first(
-        __first, _SizeType(0), __last - __first,
-        [&__pred](_RandomAccessIterator __it, _SizeType __i) { return __pred(__it[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-_ForwardIterator
-__pattern_find_if(_Tag __tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                  _Predicate __pred) noexcept
-{
-    return __internal::__brick_find_if(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-_RandomAccessIterator
-__pattern_find_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                  _RandomAccessIterator __last, _Predicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                { return __internal::__brick_find_if(__i, __j, __pred, _IsVector{}); },
-                std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
-                /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// find_end
-//------------------------------------------------------------------------
-
-// find the first occurrence of the subsequence [s_first, s_last)
-//   or the  last occurrence of the subsequence in the range [first, last)
-// b_first determines what occurrence we want to find (first or last)
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate, class _IsVector>
-_RandomAccessIterator1
-__find_subrange(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator1 __global_last,
-                _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last, _BinaryPredicate __pred,
-                bool __b_first, _IsVector __is_vector) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _ValueType;
-    auto __n2 = __s_last - __s_first;
-    if (__n2 < 1)
-    {
-        return __b_first ? __first : __last;
-    }
-
-    auto __n1 = __global_last - __first;
-    if (__n1 < __n2)
-    {
-        return __last;
-    }
-
-    auto __cur = __last;
-    while (__first != __last && (__global_last - __first >= __n2))
-    {
-        // find position of *s_first in [first, last) (it can be start of subsequence)
-        __first = __internal::__brick_find_if(
-            __first, __last, __equal_value_by_pred<_ValueType, _BinaryPredicate>(*__s_first, __pred), __is_vector);
-
-        // if position that was found previously is the start of subsequence
-        // then we can exit the loop (b_first == true) or keep the position
-        // (b_first == false)
-        if (__first != __last && (__global_last - __first >= __n2) &&
-            __internal::__brick_equal(__s_first + 1, __s_last, __first + 1, __pred, __is_vector))
-        {
-            if (__b_first)
-            {
-                return __first;
-            }
-            else
-            {
-                __cur = __first;
-            }
-        }
-        else if (__first == __last)
-        {
-            break;
-        }
-        else
-        {
-        }
-
-        // in case of b_first == false we try to find new start position
-        // for the next subsequence
-        ++__first;
-    }
-    return __cur;
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate, class _IsVector>
-_RandomAccessIterator
-__find_subrange(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __global_last,
-                _Size __count, const _Tp& __value, _BinaryPredicate __pred, _IsVector __is_vector) noexcept
-{
-    if (static_cast<_Size>(__global_last - __first) < __count || __count < 1)
-    {
-        return __last; // According to the standard last shall be returned when count < 1
-    }
-
-    auto __unary_pred = __equal_value_by_pred<_Tp, _BinaryPredicate>(__value, __pred);
-    while (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count))
-    {
-        __first = __internal::__brick_find_if(__first, __last, __unary_pred, __is_vector);
-
-        // check that all of elements in [first+1, first+count) equal to value
-        if (__first != __last && (static_cast<_Size>(__global_last - __first) >= __count) &&
-            !__internal::__brick_any_of(__first + 1, __first + __count, std::not_fn(__unary_pred), __is_vector))
-        {
-            return __first;
-        }
-        else if (__first == __last)
-        {
-            break;
-        }
-        else
-        {
-            ++__first;
-        }
-    }
-    return __last;
-}
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__brick_find_end(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-                 _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::find_end(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_find_end(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                 _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
-{
-    return __find_subrange(__first, __last, __last, __s_first, __s_last, __pred, false, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_end(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                   _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_find_end(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_end(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                   _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last,
-                   _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first == __s_last - __s_first)
-    {
-        const bool __res = __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                       __s_first, __pred);
-        return __res ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                        return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, false,
-                                                           _IsVector{});
-                    },
-                    std::greater<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
-                    /*is_first=*/false);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// find_first_of
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__brick_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-                      _ForwardIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::find_first_of(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_find_first_of(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                      _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*__is_vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_find_first_of(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_find_first_of(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                        _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_find_first_of(__first, __last, __s_first, __s_last, __pred,
-                                             typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_find_first_of(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                        _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-                        _RandomAccessIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __internal::__parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-                { return __internal::__brick_find_first_of(__i, __j, __s_first, __s_last, __pred, _IsVector{}); },
-                std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// search
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_search(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-               _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    return std::search(__first, __last, __s_first, __s_last, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator1
-__brick_search(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first,
-               _RandomAccessIterator2 __s_last, _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __internal::__find_subrange(__first, __last, __last, __s_first, __s_last, __pred, true, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__pattern_search(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                 _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_search(__first, __last, __s_first, __s_last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator1
-__pattern_search(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                 _RandomAccessIterator1 __last, _RandomAccessIterator2 __s_first, _RandomAccessIterator2 __s_last,
-                 _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first == __s_last - __s_first)
-    {
-        const bool __res = __internal::__pattern_equal(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                       __s_first, __pred);
-        return __res ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __s_first, __s_last, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j) {
-                        return __internal::__find_subrange(__i, __j, __last, __s_first, __s_last, __pred, true,
-                                                           _IsVector{});
-                    },
-                    std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(),
-                    /*is_first=*/true);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// search_n
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__brick_search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value,
-                 _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    return std::search_n(__first, __last, __count, __value, __pred);
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_search_n(_RandomAccessIterator __first, _RandomAccessIterator __last, _Size __count, const _Tp& __value,
-                 _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __internal::__find_subrange(__first, __last, __last, __count, __value, __pred, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-_ForwardIterator
-__pattern_search_n(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-                   const _Tp& __value, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_search_n(__first, __last, __count, __value, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp,
-          class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_search_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                   _RandomAccessIterator __last, _Size __count, const _Tp& __value, _BinaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (static_cast<_Size>(__last - __first) == __count)
-    {
-        const bool __result =
-            !__internal::__pattern_any_of(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [&__value, &__pred](const _Tp& __val) { return !__pred(__val, __value); });
-        return __result ? __first : __last;
-    }
-    else
-    {
-        return __internal::__except_handler(
-            [&__exec, __first, __last, __count, &__value, __pred]()
-            {
-                return __internal::__parallel_find(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    [__last, __count, &__value, __pred](_RandomAccessIterator __i, _RandomAccessIterator __j)
-                    { return __internal::__find_subrange(__i, __j, __last, __count, __value, __pred, _IsVector{}); },
-                    std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(),
-                    /*is_first=*/true);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// copy_n
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Size, class _OutputIterator>
-_OutputIterator
-__brick_copy_n(_ForwardIterator __first, _Size __n, _OutputIterator __result, /*vector=*/std::false_type) noexcept
-{
-    return std::copy_n(__first, __n, __result);
-}
-
-template <class _RandomAccessIterator1, class _Size, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_copy_n(_RandomAccessIterator1 __first, _Size __n, _RandomAccessIterator2 __result,
-               /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __n, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = *__first; });
-}
-
-//------------------------------------------------------------------------
-// copy
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-             /*vector=*/std::false_type) noexcept
-{
-    return std::copy(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-             /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __last - __first, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = *__first; });
-}
-
-//------------------------------------------------------------------------
-// move
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-             /*vector=*/std::false_type) noexcept
-{
-    return std::move(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_move(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-             /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_assign(
-        __first, __last - __first, __result,
-        [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) { *__result = std::move(*__first); });
-}
-
-struct __brick_move_destroy
-{
-    template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
-    _RandomAccessIterator2
-    operator()(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-               /*vec*/ std::true_type) const
-    {
-        using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
-
-        return __unseq_backend::__simd_assign(__first, __last - __first, __result,
-                                              [](_RandomAccessIterator1 __first, _RandomAccessIterator2 __result) {
-                                                  *__result = std::move(*__first);
-                                                  (*__first).~_IteratorValueType();
-                                              });
-    }
-
-    template <typename _RandomAccessIterator1, typename _RandomAccessIterator2>
-    _RandomAccessIterator2
-    operator()(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-               /*vec*/ std::false_type) const
-    {
-        using _IteratorValueType = typename std::iterator_traits<_RandomAccessIterator1>::value_type;
-
-        for (; __first != __last; ++__first, ++__result)
-        {
-            *__result = std::move(*__first);
-            (*__first).~_IteratorValueType();
-        }
-        return __result;
-    }
-};
-
-//------------------------------------------------------------------------
-// swap_ranges
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_swap_ranges(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                    /*vector=*/std::false_type) noexcept
-{
-    return std::swap_ranges(__first, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_swap_ranges(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                    /*vector=*/std::true_type) noexcept
-{
-    using std::iter_swap;
-    return __unseq_backend::__simd_assign(__first, __last - __first, __result,
-                                          iter_swap<_RandomAccessIterator1, _RandomAccessIterator2>);
-}
-
-//------------------------------------------------------------------------
-// copy_if
-//------------------------------------------------------------------------
-template <class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__brick_copy_if(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, _UnaryPredicate __pred,
-                /*vector=*/std::false_type) noexcept
-{
-    return std::copy_if(__first, __last, __result, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _UnaryPredicate>
-_RandomAccessIterator2
-__brick_copy_if(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                _UnaryPredicate __pred,
-                /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_copy_if(__first, __last - __first, __result, __pred);
-#else
-    return std::copy_if(__first, __last, __result, __pred);
-#endif
-}
-
-// TODO: Try to use transform_reduce for combining __brick_copy_if_phase1 on IsVector.
-template <class _DifferenceType, class _ForwardIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_ForwardIterator __first, _ForwardIterator __last, bool* __restrict __mask, _UnaryPredicate __pred,
-                    /*vector=*/std::false_type) noexcept
-{
-    auto __count_true = _DifferenceType(0);
-    auto __size = __last - __first;
-
-    static_assert(__are_random_access_iterators<_ForwardIterator>::value,
-                  "Pattern-brick error. Should be a random access iterator.");
-
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        *__mask = __pred(*__first);
-        if (*__mask)
-        {
-            ++__count_true;
-        }
-    }
-    return std::make_pair(__count_true, __size - __count_true);
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _UnaryPredicate>
-std::pair<_DifferenceType, _DifferenceType>
-__brick_calc_mask_1(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __mask, _UnaryPredicate __pred,
-                    /*vector=*/std::true_type) noexcept
-{
-    auto __result = __unseq_backend::__simd_calc_mask_1(__first, __last - __first, __mask, __pred);
-    return std::make_pair(__result, (__last - __first) - __result);
-}
-
-template <class _ForwardIterator, class _OutputIterator, class _Assigner>
-void
-__brick_copy_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result, bool* __mask,
-                     _Assigner __assigner, /*vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        if (*__mask)
-        {
-            __assigner(__first, __result);
-            ++__result;
-        }
-    }
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Assigner>
-void
-__brick_copy_by_mask(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                     bool* __restrict __mask, _Assigner __assigner, /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    __unseq_backend::__simd_copy_by_mask(__first, __last - __first, __result, __mask, __assigner);
-#else
-    __internal::__brick_copy_by_mask(__first, __last, __result, __mask, __assigner, std::false_type());
-#endif
-}
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2>
-void
-__brick_partition_by_mask(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
-                          _OutputIterator2 __out_false, bool* __mask, /*vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        if (*__mask)
-        {
-            *__out_true = *__first;
-            ++__out_true;
-        }
-        else
-        {
-            *__out_false = *__first;
-            ++__out_false;
-        }
-    }
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3>
-void
-__brick_partition_by_mask(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                          _RandomAccessIterator2 __out_true, _RandomAccessIterator3 __out_false, bool* __mask,
-                          /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    __unseq_backend::__simd_partition_by_mask(__first, __last - __first, __out_true, __out_false, __mask);
-#else
-    __internal::__brick_partition_by_mask(__first, __last, __out_true, __out_false, __mask, std::false_type());
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__pattern_copy_if(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                  _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_copy_if(__first, __last, __result, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _UnaryPredicate>
-_RandomAccessIterator2
-__pattern_copy_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                  _RandomAccessIterator1 __last, _RandomAccessIterator2 __result, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(1) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        return __internal::__except_handler(
-            [&__exec, __n, __first, __result, __pred, &__mask_buf]()
-            {
-                bool* __mask = __mask_buf.get();
-                _DifferenceType __m{};
-                __par_backend::__parallel_strict_scan(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-                    [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
-                        return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
-                                                                                __mask + __i, __pred, _IsVector{})
-                            .first;
-                    },
-                    std::plus<_DifferenceType>(),                                                // Combine
-                    [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
-                        __internal::__brick_copy_by_mask(
-                            __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
-                            [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; }, _IsVector{});
-                    },
-                    [&__m](_DifferenceType __total) { __m = __total; });
-                return __result + __m;
-            });
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_copy_if(__first, __last, __result, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// count
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__brick_count(_RandomAccessIterator __first, _RandomAccessIterator __last, _Predicate __pred,
-              /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_count(__first, __last - __first, __pred);
-}
-
-template <class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__brick_count(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred,
-              /* is_vector = */ std::false_type) noexcept
-{
-    return std::count_if(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-typename std::iterator_traits<_ForwardIterator>::difference_type
-__pattern_count(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) noexcept
-{
-    return __internal::__brick_count(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Predicate>
-typename std::iterator_traits<_RandomAccessIterator>::difference_type
-__pattern_count(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                _RandomAccessIterator __last, _Predicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, _SizeType(0),
-                [__pred](_RandomAccessIterator __begin, _RandomAccessIterator __end, _SizeType __value) -> _SizeType
-                { return __value + __internal::__brick_count(__begin, __end, __pred, _IsVector{}); },
-                std::plus<_SizeType>());
-        });
-}
-
-//------------------------------------------------------------------------
-// unique
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_unique(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-               /*is_vector=*/std::false_type) noexcept
-{
-    return std::unique(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_unique(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-               /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::unique(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_unique(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                 _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_unique(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-// That function is shared between two algorithms - remove_if (__pattern_remove_if) and unique (pattern unique). But a mask calculation is different.
-// So, a caller passes _CalcMask brick into remove_elements.
-template <class _IsVector, class _ExecutionPolicy, class _ForwardIterator, class _CalcMask>
-_ForwardIterator
-__remove_elements(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _ForwardIterator __first,
-                  _ForwardIterator __last, _CalcMask __calc_mask)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator>::difference_type _DifferenceType;
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _Tp;
-    _DifferenceType __n = __last - __first;
-    __par_backend::__buffer<bool> __mask_buf(__n);
-    // 1. find a first iterator that should be removed
-    return __internal::__except_handler([&]() {
-        bool* __mask = __mask_buf.get();
-        _DifferenceType __min = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), _DifferenceType(0), __n, __n,
-            [__first, __mask, &__calc_mask](_DifferenceType __i, _DifferenceType __j,
-                                            _DifferenceType __local_min) -> _DifferenceType
-            {
-                // Create mask
-                __calc_mask(__mask + __i, __mask + __j, __first + __i);
-
-                // if minimum was found in a previous range we shouldn't do anymore
-                if (__local_min < __i)
-                {
-                    return __local_min;
-                }
-                // find first iterator that should be removed
-                bool* __result = __internal::__brick_find_if(
-                    __mask + __i, __mask + __j, [](bool __val) { return !__val; }, _IsVector{});
-                if (__result - __mask == __j)
-                {
-                    return __local_min;
-                }
-                return std::min(__local_min, _DifferenceType(__result - __mask));
-            },
-            [](_DifferenceType __local_min1, _DifferenceType __local_min2) -> _DifferenceType
-            { return std::min(__local_min1, __local_min2); });
-
-        // No elements to remove - exit
-        if (__min == __n)
-        {
-            return __last;
-        }
-        __n -= __min;
-        __first += __min;
-
-        __par_backend::__buffer<_Tp> __buf(__n);
-        _Tp* __result = __buf.get();
-        __mask += __min;
-        _DifferenceType __m{};
-        // 2. Elements that doesn't satisfy pred are moved to result
-        __par_backend::__parallel_strict_scan(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-            [__mask](_DifferenceType __i, _DifferenceType __len)
-            {
-                return __internal::__brick_count(
-                    __mask + __i, __mask + __i + __len, [](bool __val) { return __val; }, _IsVector{});
-            },
-            std::plus<_DifferenceType>(),
-            [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial)
-            {
-                __internal::__brick_copy_by_mask(
-                    __first + __i, __first + __i + __len, __result + __initial, __mask + __i,
-                    [](_ForwardIterator __x, _Tp* __z)
-                    {
-                        __internal::__invoke_if_else(
-                            std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
-                            [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
-                    },
-                    _IsVector{});
-            },
-            [&__m](_DifferenceType __total) { __m = __total; });
-
-        // 3. Elements from result are moved to [first, last)
-        __par_backend::__parallel_for(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
-            [__result, __first](_Tp* __i, _Tp* __j)
-            {
-                __invoke_if_else(
-                    std::is_trivial<_Tp>(), [&]() { __brick_move(__i, __j, __first + (__i - __result), _IsVector{}); },
-                    [&]() { __brick_move_destroy()(__i, __j, __first + (__i - __result), _IsVector{}); });
-            });
-        return __first + __m;
-    });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_unique(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __last, _BinaryPredicate __pred) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    if (__first == __last)
-    {
-        return __last;
-    }
-    if (__first + 1 == __last || __first + 2 == __last)
-    {
-        // Trivial sequence - use serial algorithm
-        return __internal::__brick_unique(__first, __last, __pred, _IsVector{});
-    }
-    return __internal::__remove_elements(
-        __tag, std::forward<_ExecutionPolicy>(__exec), ++__first, __last,
-        [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it)
-        {
-            __internal::__brick_walk3(
-                __b, __e, __it - 1, __it,
-                [&__pred](bool& __x, _ReferenceType __y, _ReferenceType __z) { __x = !__pred(__y, __z); }, _IsVector{});
-        });
-}
-
-//------------------------------------------------------------------------
-// unique_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class OutputIterator, class _BinaryPredicate>
-OutputIterator
-__brick_unique_copy(_ForwardIterator __first, _ForwardIterator __last, OutputIterator __result, _BinaryPredicate __pred,
-                    /*vector=*/std::false_type) noexcept
-{
-    return std::unique_copy(__first, __last, __result, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _BinaryPredicate>
-_RandomAccessIterator2
-__brick_unique_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result,
-                    _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_unique_copy(__first, __last - __first, __result, __pred);
-#else
-    return std::unique_copy(__first, __last, __result, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
-__pattern_unique_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                      _OutputIterator __result, _BinaryPredicate __pred) noexcept
-{
-    return __internal::__brick_unique_copy(__first, __last, __result, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
-                    _BinaryPredicate __pred, /*vector=*/std::false_type) noexcept
-{
-    _DifferenceType __count = 0;
-    for (; __first != __last; ++__first, ++__mask)
-    {
-        *__mask = !__pred(*__first, *(__first - 1));
-        __count += *__mask;
-    }
-    return __count;
-}
-
-template <class _DifferenceType, class _RandomAccessIterator, class _BinaryPredicate>
-_DifferenceType
-__brick_calc_mask_2(_RandomAccessIterator __first, _RandomAccessIterator __last, bool* __restrict __mask,
-                    _BinaryPredicate __pred, /*vector=*/std::true_type) noexcept
-{
-    return __unseq_backend::__simd_calc_mask_2(__first, __last - __first, __mask, __pred);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryPredicate>
-_RandomAccessIterator2
-__pattern_unique_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                      _RandomAccessIterator1 __last, _RandomAccessIterator2 __result, _BinaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(2) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        if (_DifferenceType(2) < __n)
-        {
-            return __internal::__except_handler(
-                [&__exec, __n, __first, __result, __pred, &__mask_buf]()
-                {
-                    bool* __mask = __mask_buf.get();
-                    _DifferenceType __m{};
-                    __par_backend::__parallel_strict_scan(
-                        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, _DifferenceType(0),
-                        [=](_DifferenceType __i, _DifferenceType __len) -> _DifferenceType { // Reduce
-                            _DifferenceType __extra = 0;
-                            if (__i == 0)
-                            {
-                                // Special boundary case
-                                __mask[__i] = true;
-                                if (--__len == 0)
-                                    return 1;
-                                ++__i;
-                                ++__extra;
-                            }
-                            return __internal::__brick_calc_mask_2<_DifferenceType>(
-                                       __first + __i, __first + (__i + __len), __mask + __i, __pred, _IsVector{}) +
-                                   __extra;
-                        },
-                        std::plus<_DifferenceType>(),                                                // Combine
-                        [=](_DifferenceType __i, _DifferenceType __len, _DifferenceType __initial) { // Scan
-                            // Phase 2 is same as for __pattern_copy_if
-                            __internal::__brick_copy_by_mask(
-                                __first + __i, __first + (__i + __len), __result + __initial, __mask + __i,
-                                [](_RandomAccessIterator1 __x, _RandomAccessIterator2 __z) { *__z = *__x; },
-                                _IsVector{});
-                        },
-                        [&__m](_DifferenceType __total) { __m = __total; });
-                    return __result + __m;
-                });
-        }
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_unique_copy(__first, __last, __result, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// reverse
-//------------------------------------------------------------------------
-template <class _BidirectionalIterator>
-void
-__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, /*__is_vector=*/std::false_type) noexcept
-{
-    std::reverse(__first, __last);
-}
-
-template <class _RandomAccessIterator>
-void
-__brick_reverse(_RandomAccessIterator __first, _RandomAccessIterator __last, /*__is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    const auto __n = (__last - __first) / 2;
-    __unseq_backend::__simd_walk_2(__first, __n, std::reverse_iterator<_RandomAccessIterator>(__last),
-                                   [](_ReferenceType __x, _ReferenceType __y) {
-                                       using std::swap;
-                                       swap(__x, __y);
-                                   });
-}
-
-// this brick is called in parallel version, so we can use iterator arithmetic
-template <class _BidirectionalIterator>
-void
-__brick_reverse(_BidirectionalIterator __first, _BidirectionalIterator __last, _BidirectionalIterator __d_last,
-                /*is_vector=*/std::false_type) noexcept
-{
-    for (--__d_last; __first != __last; ++__first, --__d_last)
-    {
-        using std::iter_swap;
-        iter_swap(__first, __d_last);
-    }
-}
-
-// this brick is called in parallel version, so we can use iterator arithmetic
-template <class _RandomAccessIterator>
-void
-__brick_reverse(_RandomAccessIterator __first, _RandomAccessIterator __last, _RandomAccessIterator __d_last,
-                /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    __unseq_backend::__simd_walk_2(__first, __last - __first, std::reverse_iterator<_RandomAccessIterator>(__d_last),
-                                   [](_ReferenceType __x, _ReferenceType __y) {
-                                       using std::swap;
-                                       swap(__x, __y);
-                                   });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator>
-void
-__pattern_reverse(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last) noexcept
-{
-    __internal::__brick_reverse(__first, __last, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-void
-__pattern_reverse(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                  _RandomAccessIterator __last)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __first + (__last - __first) / 2,
-        [__first, __last](_RandomAccessIterator __inner_first, _RandomAccessIterator __inner_last)
-        { __internal::__brick_reverse(__inner_first, __inner_last, __last - (__inner_first - __first), _IsVector{}); });
-}
-
-//------------------------------------------------------------------------
-// reverse_copy
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__brick_reverse_copy(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __d_first,
-                     /*is_vector=*/std::false_type) noexcept
-{
-    return std::reverse_copy(__first, __last, __d_first);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_reverse_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first,
-                     /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    return __unseq_backend::__simd_walk_2(std::reverse_iterator<_RandomAccessIterator1>(__last), __last - __first,
-                                          __d_first, [](_ReferenceType1 __x, _ReferenceType2 __y) { __y = __x; });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _OutputIterator>
-_OutputIterator
-__pattern_reverse_copy(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                       _OutputIterator __d_first) noexcept
-{
-    return __internal::__brick_reverse_copy(__first, __last, __d_first, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__pattern_reverse_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                       _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    auto __len = __last - __first;
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [__first, __len, __d_first](_RandomAccessIterator1 __inner_first, _RandomAccessIterator1 __inner_last)
-        {
-            __internal::__brick_reverse_copy(__inner_first, __inner_last,
-                                             __d_first + (__len - (__inner_last - __first)), _IsVector{});
-        });
-    return __d_first + __len;
-}
-
-//------------------------------------------------------------------------
-// rotate
-//------------------------------------------------------------------------
-template <class _ForwardIterator>
-_ForwardIterator
-__brick_rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
-               /*is_vector=*/std::false_type) noexcept
-{
-#if defined(_PSTL_CPP11_STD_ROTATE_BROKEN)
-    std::rotate(__first, __middle, __last);
-    return std::next(__first, std::distance(__middle, __last));
-#else
-    return std::rotate(__first, __middle, __last);
-#endif
-}
-
-template <class _RandomAccessIterator>
-_RandomAccessIterator
-__brick_rotate(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last,
-               /*is_vector=*/std::true_type) noexcept
-{
-    auto __n = __last - __first;
-    auto __m = __middle - __first;
-    const _RandomAccessIterator __ret = __first + (__last - __middle);
-
-    bool __is_left = (__m <= __n / 2);
-    if (!__is_left)
-        __m = __n - __m;
-
-    while (__n > 1 && __m > 0)
-    {
-        using std::iter_swap;
-        const auto __m_2 = __m * 2;
-        if (__is_left)
-        {
-            for (; __last - __first >= __m_2; __first += __m)
-            {
-                __unseq_backend::__simd_assign(__first, __m, __first + __m,
-                                               iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
-            }
-        }
-        else
-        {
-            for (; __last - __first >= __m_2; __last -= __m)
-            {
-                __unseq_backend::__simd_assign(__last - __m, __m, __last - __m_2,
-                                               iter_swap<_RandomAccessIterator, _RandomAccessIterator>);
-            }
-        }
-        __is_left = !__is_left;
-        __m = __n % __m;
-        __n = __last - __first;
-    }
-
-    return __ret;
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator>
-_ForwardIterator
-__pattern_rotate(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle,
-                 _ForwardIterator __last) noexcept
-{
-    return __internal::__brick_rotate(__first, __middle, __last, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator>
-_RandomAccessIterator
-__pattern_rotate(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _RandomAccessIterator __middle, _RandomAccessIterator __last)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    auto __n = __last - __first;
-    auto __m = __middle - __first;
-    if (__m <= __n / 2)
-    {
-        __par_backend::__buffer<_Tp> __buf(__n - __m);
-        return __internal::__except_handler(
-            [&__exec, __n, __m, __first, __middle, __last, &__buf]()
-            {
-                _Tp* __result = __buf.get();
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __middle, __last,
-                    [__middle, __result](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle,
-                    [__last, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_move(__b, __e, __b + (__last - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + (__n - __m),
-                    [__first, __result](_Tp* __b, _Tp* __e)
-                    { __brick_move_destroy()(__b, __e, __first + (__b - __result), _IsVector{}); });
-
-                return __first + (__last - __middle);
-            });
-    }
-    else
-    {
-        __par_backend::__buffer<_Tp> __buf(__m);
-        return __internal::__except_handler(
-            [&__exec, __n, __m, __first, __middle, __last, &__buf]()
-            {
-                _Tp* __result = __buf.get();
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle,
-                    [__first, __result](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_uninitialized_move(__b, __e, __result + (__b - __first), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __middle, __last,
-                    [__first, __middle](_RandomAccessIterator __b, _RandomAccessIterator __e)
-                    { __internal::__brick_move(__b, __e, __first + (__b - __middle), _IsVector{}); });
-
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __result, __result + __m,
-                    [__n, __m, __first, __result](_Tp* __b, _Tp* __e)
-                    { __brick_move_destroy()(__b, __e, __first + ((__n - __m) + (__b - __result)), _IsVector{}); });
-
-                return __first + (__last - __middle);
-            });
-    }
-}
-
-//------------------------------------------------------------------------
-// rotate_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__brick_rotate_copy(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last,
-                    _OutputIterator __result, /*__is_vector=*/std::false_type) noexcept
-{
-    return std::rotate_copy(__first, __middle, __last, __result);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__brick_rotate_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __middle, _RandomAccessIterator1 __last,
-                    _RandomAccessIterator2 __result, /*__is_vector=*/std::true_type) noexcept
-{
-    _RandomAccessIterator2 __res = __internal::__brick_copy(__middle, __last, __result, std::true_type());
-    return __internal::__brick_copy(__first, __middle, __res, std::true_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator>
-_OutputIterator
-__pattern_rotate_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __middle,
-                      _ForwardIterator __last, _OutputIterator __result) noexcept
-{
-    return __internal::__brick_rotate_copy(__first, __middle, __last, __result, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2>
-_RandomAccessIterator2
-__pattern_rotate_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                      _RandomAccessIterator1 __middle, _RandomAccessIterator1 __last, _RandomAccessIterator2 __result)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_for(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [__first, __last, __middle, __result](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e)
-        {
-            if (__b > __middle)
-            {
-                __internal::__brick_copy(__b, __e, __result + (__b - __middle), _IsVector{});
-            }
-            else
-            {
-                _RandomAccessIterator2 __new_result = __result + ((__last - __middle) + (__b - __first));
-                if (__e < __middle)
-                {
-                    __internal::__brick_copy(__b, __e, __new_result, _IsVector{});
-                }
-                else
-                {
-                    __internal::__brick_copy(__b, __middle, __new_result, _IsVector{});
-                    __internal::__brick_copy(__middle, __e, __result, _IsVector{});
-                }
-            }
-        });
-    return __result + (__last - __first);
-}
-
-//------------------------------------------------------------------------
-// is_partitioned
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-bool
-__brick_is_partitioned(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                       /*is_vector=*/std::false_type) noexcept
-{
-    return std::is_partitioned(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__brick_is_partitioned(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                       /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    if (__first == __last)
-    {
-        return true;
-    }
-    else
-    {
-        _RandomAccessIterator __result = __unseq_backend::__simd_first(
-            __first, _SizeType(0), __last - __first,
-            [&__pred](_RandomAccessIterator __it, _SizeType __i) { return !__pred(__it[__i]); });
-        if (__result == __last)
-        {
-            return true;
-        }
-        else
-        {
-            ++__result;
-            return !__unseq_backend::__simd_or(__result, __last - __result, __pred);
-        }
-    }
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_is_partitioned(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-bool
-__pattern_is_partitioned(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _UnaryPredicate __pred)
-{
-    if (__first == __last)
-    {
-        return true;
-    }
-    else
-    {
-        using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-        return __internal::__except_handler([&]() {
-            // State of current range:
-            // broken     - current range is not partitioned by pred
-            // all_true   - all elements in current range satisfy pred
-            // all_false  - all elements in current range don't satisfy pred
-            // true_false - elements satisfy pred are placed before elements that don't satisfy pred
-            enum _ReduceType
-            {
-                __not_init = -1,
-                __broken,
-                __all_true,
-                __all_false,
-                __true_false
-            };
-            _ReduceType __init = __not_init;
-
-            // Array with states that we'll have when state from the left branch is merged with state from the right branch.
-            // State is calculated by formula: new_state = table[left_state * 4 + right_state]
-            _ReduceType __table[] = {__broken,     __broken,     __broken,     __broken, __broken,    __all_true,
-                                     __true_false, __true_false, __broken,     __broken, __all_false, __broken,
-                                     __broken,     __broken,     __true_false, __broken};
-
-            __init = __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-                [&__pred, &__table](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                    _ReduceType __value) -> _ReduceType
-                {
-                    if (__value == __broken)
-                    {
-                        return __broken;
-                    }
-                    _ReduceType __res = __not_init;
-                    // if first element satisfy pred
-                    if (__pred(*__i))
-                    {
-                        // find first element that don't satisfy pred
-                        _RandomAccessIterator __x =
-                            __internal::__brick_find_if(__i + 1, __j, std::not_fn(__pred), _IsVector{});
-                        if (__x != __j)
-                        {
-                            // find first element after "x" that satisfy pred
-                            _RandomAccessIterator __y = __internal::__brick_find_if(__x + 1, __j, __pred, _IsVector{});
-                            // if it was found then range isn't partitioned by pred
-                            if (__y != __j)
-                            {
-                                return __broken;
-                            }
-                            else
-                            {
-                                __res = __true_false;
-                            }
-                        }
-                        else
-                        {
-                            __res = __all_true;
-                        }
-                    }
-                    else
-                    { // if first element doesn't satisfy pred
-                        // then we should find the first element that satisfy pred.
-                        // If we found it then range isn't partitioned by pred
-                        if (__internal::__brick_find_if(__i + 1, __j, __pred, _IsVector{}) != __j)
-                        {
-                            return __broken;
-                        }
-                        else
-                        {
-                            __res = __all_false;
-                        }
-                    }
-                    // if we have value from left range then we should calculate the result
-                    return (__value == -1) ? __res : __table[__value * 4 + __res];
-                },
-
-                [&__table](_ReduceType __val1, _ReduceType __val2) -> _ReduceType
-                {
-                    if (__val1 == __broken || __val2 == __broken)
-                    {
-                        return __broken;
-                    }
-                    // calculate the result for new big range
-                    return __table[__val1 * 4 + __val2];
-                });
-            return __init != __broken;
-        });
-    }
-}
-
-//------------------------------------------------------------------------
-// partition
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__brick_partition(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                  /*is_vector=*/std::false_type) noexcept
-{
-    return std::partition(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                  /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::partition(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_partition(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                    _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_partition(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_partition(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                    _RandomAccessIterator __last, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    // partitioned range: elements before pivot satisfy pred (true part),
-    //                    elements after pivot don't satisfy pred (false part)
-    struct _PartitionRange
-    {
-        _RandomAccessIterator __begin;
-        _RandomAccessIterator __pivot;
-        _RandomAccessIterator __end;
-    };
-
-    return __internal::__except_handler([&]() {
-        _PartitionRange __init{__last, __last, __last};
-
-        // lambda for merging two partitioned ranges to one partitioned range
-        auto __reductor = [&__exec](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange
-        {
-            auto __size1 = __val1.__end - __val1.__pivot;
-            auto __size2 = __val2.__pivot - __val2.__begin;
-            auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
-
-            // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
-            if (__val1.__end == __val1.__pivot)
-            {
-                return {__new_begin, __val2.__pivot, __val2.__end};
-            }
-            // if true part of right range greater than false part of left range
-            // then we should swap the false part of left range and last part of true part of right range
-            else if (__size2 > __size1)
-            {
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size1,
-                    [__val1, __val2, __size1](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                        __internal::__brick_swap_ranges(__i, __j, (__val2.__pivot - __size1) + (__i - __val1.__pivot),
-                                                        _IsVector{});
-                    });
-                return {__new_begin, __val2.__pivot - __size1, __val2.__end};
-            }
-            // else we should swap the first part of false part of left range and true part of right range
-            else
-            {
-                __par_backend::__parallel_for(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __val1.__pivot, __val1.__pivot + __size2,
-                    [__val1, __val2](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                        __internal::__brick_swap_ranges(__i, __j, __val2.__begin + (__i - __val1.__pivot), _IsVector{});
-                    });
-                return {__new_begin, __val1.__pivot + __size2, __val2.__end};
-            }
-        };
-
-        _PartitionRange __result = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-            [__pred, __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                 _PartitionRange __value) -> _PartitionRange
-            {
-                //1. serial partition
-                _RandomAccessIterator __pivot = __internal::__brick_partition(__i, __j, __pred, _IsVector{});
-
-                // 2. merging of two ranges (left and right respectively)
-                return __reductor(__value, {__i, __pivot, __j});
-            },
-            __reductor);
-        return __result.__pivot;
-    });
-}
-
-//------------------------------------------------------------------------
-// stable_partition
-//------------------------------------------------------------------------
-
-template <class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__brick_stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last, _UnaryPredicate __pred,
-                         /*__is_vector=*/std::false_type) noexcept
-{
-    return std::stable_partition(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_stable_partition(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                         /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::stable_partition(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-_BidirectionalIterator
-__pattern_stable_partition(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                           _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_stable_partition(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_stable_partition(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                           _RandomAccessIterator __last, _UnaryPredicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    // partitioned range: elements before pivot satisfy pred (true part),
-    //                    elements after pivot don't satisfy pred (false part)
-    struct _PartitionRange
-    {
-        _RandomAccessIterator __begin;
-        _RandomAccessIterator __pivot;
-        _RandomAccessIterator __end;
-    };
-
-    return __internal::__except_handler([&]() {
-        _PartitionRange __init{__last, __last, __last};
-
-        // lambda for merging two partitioned ranges to one partitioned range
-        auto __reductor = [](_PartitionRange __val1, _PartitionRange __val2) -> _PartitionRange
-        {
-            auto __size1 = __val1.__end - __val1.__pivot;
-            auto __new_begin = __val2.__begin - (__val1.__end - __val1.__begin);
-
-            // if all elements in left range satisfy pred then we can move new pivot to pivot of right range
-            if (__val1.__end == __val1.__pivot)
-            {
-                return {__new_begin, __val2.__pivot, __val2.__end};
-            }
-            // if true part of right range greater than false part of left range
-            // then we should swap the false part of left range and last part of true part of right range
-            else
-            {
-                __internal::__brick_rotate(__val1.__pivot, __val2.__begin, __val2.__pivot, _IsVector{});
-                return {__new_begin, __val2.__pivot - __size1, __val2.__end};
-            }
-        };
-
-        _PartitionRange __result = __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __init,
-            [&__pred, __reductor](_RandomAccessIterator __i, _RandomAccessIterator __j,
-                                  _PartitionRange __value) -> _PartitionRange
-            {
-                //1. serial stable_partition
-                _RandomAccessIterator __pivot = __internal::__brick_stable_partition(__i, __j, __pred, _IsVector{});
-
-                // 2. merging of two ranges (left and right respectively)
-                return __reductor(__value, {__i, __pivot, __j});
-            },
-            __reductor);
-        return __result.__pivot;
-    });
-}
-
-//------------------------------------------------------------------------
-// partition_copy
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator1, class _OutputIterator2, class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__brick_partition_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator1 __out_true,
-                       _OutputIterator2 __out_false, _UnaryPredicate __pred, /*is_vector=*/std::false_type) noexcept
-{
-    return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3,
-          class _UnaryPredicate>
-std::pair<_RandomAccessIterator2, _RandomAccessIterator3>
-__brick_partition_copy(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last, _RandomAccessIterator2 __out_true,
-                       _RandomAccessIterator3 __out_false, _UnaryPredicate __pred,
-                       /*is_vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_partition_copy(__first, __last - __first, __out_true, __out_false, __pred);
-#else
-    return std::partition_copy(__first, __last, __out_true, __out_false, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator1, class _OutputIterator2,
-          class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__pattern_partition_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _OutputIterator1 __out_true, _OutputIterator2 __out_false, _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred,
-                                              typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _UnaryPredicate>
-std::pair<_RandomAccessIterator2, _RandomAccessIterator3>
-__pattern_partition_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                         _RandomAccessIterator1 __last, _RandomAccessIterator2 __out_true,
-                         _RandomAccessIterator3 __out_false, _UnaryPredicate __pred)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    typedef std::pair<_DifferenceType, _DifferenceType> _ReturnType;
-    const _DifferenceType __n = __last - __first;
-    if (_DifferenceType(1) < __n)
-    {
-        __par_backend::__buffer<bool> __mask_buf(__n);
-        return __internal::__except_handler(
-            [&__exec, __n, __first, __out_true, __out_false, __pred, &__mask_buf]()
-            {
-                bool* __mask = __mask_buf.get();
-                _ReturnType __m{};
-                __par_backend::__parallel_strict_scan(
-                    __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n,
-                    std::make_pair(_DifferenceType(0), _DifferenceType(0)),
-                    [=](_DifferenceType __i, _DifferenceType __len) { // Reduce
-                        return __internal::__brick_calc_mask_1<_DifferenceType>(__first + __i, __first + (__i + __len),
-                                                                                __mask + __i, __pred, _IsVector{});
-                    },
-                    [](const _ReturnType& __x, const _ReturnType& __y) -> _ReturnType
-                    { return std::make_pair(__x.first + __y.first, __x.second + __y.second); }, // Combine
-                    [=](_DifferenceType __i, _DifferenceType __len, _ReturnType __initial) {    // Scan
-                        __internal::__brick_partition_by_mask(
-                            __first + __i, __first + (__i + __len), __out_true + __initial.first,
-                            __out_false + __initial.second, __mask + __i, _IsVector{});
-                    },
-                    [&__m](_ReturnType __total) { __m = __total; });
-                return std::make_pair(__out_true + __m.first, __out_false + __m.second);
-            });
-    }
-    // trivial sequence - use serial algorithm
-    return __internal::__brick_partition_copy(__first, __last, __out_true, __out_false, __pred, _IsVector{});
-}
-
-//------------------------------------------------------------------------
-// sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare, class _IsMoveConstructible>
-void
-__pattern_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-               _IsMoveConstructible) noexcept
-{
-    std::sort(__first, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-               _RandomAccessIterator __last, _Compare __comp, /*is_move_constructible=*/std::true_type)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-                { std::sort(__first, __last, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// stable_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
-                      _Compare __comp) noexcept
-{
-    std::stable_sort(__first, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_stable_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [](_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-                { std::stable_sort(__first, __last, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// partial_sort
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-                       _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    std::partial_sort(__first, __middle, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_partial_sort(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                       _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    const auto __n = __middle - __first;
-    if (__n == 0)
-        return;
-
-    __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp,
-                [__n](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Compare __comp)
-                {
-                    if (__n < __end - __begin)
-                        std::partial_sort(__begin, __begin + __n, __end, __comp);
-                    else
-                        std::sort(__begin, __end, __comp);
-                },
-                __n);
-        });
-}
-
-//------------------------------------------------------------------------
-// partial_sort_copy
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_partial_sort_copy(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                            _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp) noexcept
-{
-    return std::partial_sort_copy(__first, __last, __d_first, __d_last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-_RandomAccessIterator2
-__pattern_partial_sort_copy(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first,
-                            _RandomAccessIterator1 __last, _RandomAccessIterator2 __d_first,
-                            _RandomAccessIterator2 __d_last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last == __first || __d_last == __d_first)
-    {
-        return __d_first;
-    }
-    auto __n1 = __last - __first;
-    auto __n2 = __d_last - __d_first;
-    return __internal::__except_handler([&]() {
-        if (__n2 >= __n1)
-        {
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __d_first, __d_first + __n1, __comp,
-                [__first, __d_first](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j, _Compare __comp)
-                {
-                    _RandomAccessIterator1 __i1 = __first + (__i - __d_first);
-                    _RandomAccessIterator1 __j1 = __first + (__j - __d_first);
-
-                // 1. Copy elements from input to output
-#if !defined(_PSTL_ICC_18_OMP_SIMD_BROKEN)
-                    __internal::__brick_copy(__i1, __j1, __i, _IsVector{});
-#else
-                    std::copy(__i1, __j1, __i);
-#endif
-                    // 2. Sort elements in output sequence
-                    std::sort(__i, __j, __comp);
-                },
-                __n1);
-            return __d_first + __n1;
-        }
-        else
-        {
-            typedef typename std::iterator_traits<_RandomAccessIterator1>::value_type _T1;
-            typedef typename std::iterator_traits<_RandomAccessIterator2>::value_type _T2;
-            __par_backend::__buffer<_T1> __buf(__n1);
-            _T1* __r = __buf.get();
-
-            __par_backend::__parallel_stable_sort(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n1, __comp,
-                [__n2, __first, __r](_T1* __i, _T1* __j, _Compare __comp)
-                {
-                    _RandomAccessIterator1 __it = __first + (__i - __r);
-
-                    // 1. Copy elements from input to raw memory
-                    for (_T1* __k = __i; __k != __j; ++__k, ++__it)
-                    {
-                        ::new (__k) _T2(*__it);
-                    }
-
-                    // 2. Sort elements in temporary __buffer
-                    if (__n2 < __j - __i)
-                        std::partial_sort(__i, __i + __n2, __j, __comp);
-                    else
-                        std::sort(__i, __j, __comp);
-                },
-                __n2);
-
-            // 3. Move elements from temporary __buffer to output
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n2,
-                                          [__r, __d_first](_T1* __i, _T1* __j)
-                                          { __brick_move_destroy()(__i, __j, __d_first + (__i - __r), _IsVector{}); });
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r + __n2,
-                                          __r + __n1,
-                                          [](_T1* __i, _T1* __j) { __brick_destroy(__i, __j, _IsVector{}); });
-
-            return __d_first + __n2;
-        }
-    });
-}
-
-//------------------------------------------------------------------------
-// adjacent_find
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__brick_adjacent_find(_RandomAccessIterator __first, _RandomAccessIterator __last, _BinaryPredicate __pred,
-                      /* IsVector = */ std::true_type, bool __or_semantic) noexcept
-{
-    return __unseq_backend::__simd_adjacent_find(__first, __last, __pred, __or_semantic);
-}
-
-template <class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__brick_adjacent_find(_ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred,
-                      /* IsVector = */ std::false_type, bool) noexcept
-{
-    return std::adjacent_find(__first, __last, __pred);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-_ForwardIterator
-__pattern_adjacent_find(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                        _BinaryPredicate __pred, bool __or_semantic) noexcept
-{
-    return __internal::__brick_adjacent_find(__first, __last, __pred, typename _Tag::__is_vector{}, __or_semantic);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _BinaryPredicate>
-_RandomAccessIterator
-__pattern_adjacent_find(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __last, _BinaryPredicate __pred, bool __or_semantic)
-{
-    if (__last - __first < 2)
-        return __last;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last, __last,
-                [__last, __pred, __or_semantic](_RandomAccessIterator __begin, _RandomAccessIterator __end,
-                                                _RandomAccessIterator __value) -> _RandomAccessIterator
-                {
-                    // TODO: investigate performance benefits from the use of shared variable for the result,
-                    // checking (compare_and_swap idiom) its __value at __first.
-                    if (__or_semantic && __value < __last)
-                    { //found
-                        __par_backend::__cancel_execution();
-                        return __value;
-                    }
-
-                    if (__value > __begin)
-                    {
-                        // modify __end to check the predicate on the boundary __values;
-                        // TODO: to use a custom range with boundaries overlapping
-                        // TODO: investigate what if we remove "if" below and run algorithm on range [__first, __last-1)
-                        // then check the pair [__last-1, __last)
-                        if (__end != __last)
-                            ++__end;
-
-                        //correct the global result iterator if the "brick" returns a local "__last"
-                        const _RandomAccessIterator __res =
-                            __internal::__brick_adjacent_find(__begin, __end, __pred, _IsVector{}, __or_semantic);
-                        if (__res < __end)
-                            __value = __res;
-                    }
-                    return __value;
-                },
-                [](_RandomAccessIterator __x, _RandomAccessIterator __y) -> _RandomAccessIterator
-                { return __x < __y ? __x : __y; } //reduce a __value
-            );
-        });
-}
-
-//------------------------------------------------------------------------
-// nth_element
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-                      _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    std::nth_element(__first, __nth, __last, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_nth_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    if (__first == __last || __nth == __last)
-    {
-        return;
-    }
-
-    using std::iter_swap;
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    _RandomAccessIterator __x;
-    do
-    {
-        __x = __internal::__pattern_partition(__tag, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
-                                              [&__comp, __first](const _Tp& __x) { return __comp(__x, *__first); });
-        --__x;
-        if (__x != __first)
-        {
-            iter_swap(__first, __x);
-        }
-        // if x > nth then our new range for partition is [first, x)
-        if (__x - __nth > 0)
-        {
-            __last = __x;
-        }
-        // if x < nth then our new range for partition is [x, last)
-        else if (__x - __nth < 0)
-        {
-            // if *x == *nth then we can start new partition with x+1
-            if (!__comp(*__nth, *__x) && !__comp(*__x, *__nth))
-            {
-                ++__x;
-            }
-            else
-            {
-                iter_swap(__nth, __x);
-            }
-            __first = __x;
-        }
-    } while (__x != __nth);
-}
-
-//------------------------------------------------------------------------
-// fill, fill_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Tp>
-void
-__brick_fill(_RandomAccessIterator __first, _RandomAccessIterator __last, const _Tp& __value,
-             /* __is_vector = */ std::true_type) noexcept
-{
-    __unseq_backend::__simd_fill_n(__first, __last - __first, __value);
-}
-
-template <class _ForwardIterator, class _Tp>
-void
-__brick_fill(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value,
-             /* __is_vector = */ std::false_type) noexcept
-{
-    std::fill(__first, __last, __value);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-void
-__pattern_fill(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) noexcept
-{
-    __internal::__brick_fill(__first, __last, __value, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp>
-_RandomAccessIterator
-__pattern_fill(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-               _RandomAccessIterator __last, const _Tp& __value)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&__exec, __first, __last, &__value]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [&__value](_RandomAccessIterator __begin, _RandomAccessIterator __end)
-                                          { __internal::__brick_fill(__begin, __end, __value, _IsVector{}); });
-            return __last;
-        });
-}
-
-template <class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__brick_fill_n(_RandomAccessIterator __first, _Size __count, const _Tp& __value,
-               /* __is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_fill_n(__first, __count, __value);
-}
-
-template <class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__brick_fill_n(_OutputIterator __first, _Size __count, const _Tp& __value, /* __is_vector = */ std::false_type) noexcept
-{
-    return std::fill_n(__first, __count, __value);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Tp>
-_OutputIterator
-__pattern_fill_n(_Tag, _ExecutionPolicy&&, _OutputIterator __first, _Size __count, const _Tp& __value) noexcept
-{
-    return __internal::__brick_fill_n(__first, __count, __value, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Tp>
-_RandomAccessIterator
-__pattern_fill_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                 _Size __count, const _Tp& __value)
-{
-    return __internal::__pattern_fill(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __count,
-                                      __value);
-}
-
-//------------------------------------------------------------------------
-// generate, generate_n
-//------------------------------------------------------------------------
-template <class _RandomAccessIterator, class _Generator>
-void
-__brick_generate(_RandomAccessIterator __first, _RandomAccessIterator __last, _Generator __g,
-                 /* is_vector = */ std::true_type) noexcept
-{
-    __unseq_backend::__simd_generate_n(__first, __last - __first, __g);
-}
-
-template <class _ForwardIterator, class _Generator>
-void
-__brick_generate(_ForwardIterator __first, _ForwardIterator __last, _Generator __g,
-                 /* is_vector = */ std::false_type) noexcept
-{
-    std::generate(__first, __last, __g);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-void
-__pattern_generate(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Generator __g) noexcept
-{
-    __internal::__brick_generate(__first, __last, __g, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Generator>
-_RandomAccessIterator
-__pattern_generate(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                   _RandomAccessIterator __last, _Generator __g)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                          [__g](_RandomAccessIterator __begin, _RandomAccessIterator __end)
-                                          { __internal::__brick_generate(__begin, __end, __g, _IsVector{}); });
-            return __last;
-        });
-}
-
-template <class _RandomAccessIterator, class Size, class _Generator>
-_RandomAccessIterator
-__brick_generate_n(_RandomAccessIterator __first, Size __count, _Generator __g,
-                   /* is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_generate_n(__first, __count, __g);
-}
-
-template <class OutputIterator, class Size, class _Generator>
-OutputIterator
-__brick_generate_n(OutputIterator __first, Size __count, _Generator __g, /* is_vector = */ std::false_type) noexcept
-{
-    return std::generate_n(__first, __count, __g);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _OutputIterator, class _Size, class _Generator>
-_OutputIterator
-__pattern_generate_n(_Tag, _ExecutionPolicy&&, _OutputIterator __first, _Size __count, _Generator __g) noexcept
-{
-    return __internal::__brick_generate_n(__first, __count, __g, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Size, class _Generator>
-_RandomAccessIterator
-__pattern_generate_n(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                     _Size __count, _Generator __g)
-{
-    static_assert(__are_random_access_iterators<_RandomAccessIterator>::value,
-                  "Pattern-brick error. Should be a random access iterator.");
-    return __internal::__pattern_generate(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __first + __count,
-                                          __g);
-}
-
-//------------------------------------------------------------------------
-// remove
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__brick_remove_if(_ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-                  /* __is_vector = */ std::false_type) noexcept
-{
-    return std::remove_if(__first, __last, __pred);
-}
-
-template <class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__brick_remove_if(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryPredicate __pred,
-                  /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_MONOTONIC_PRESENT)
-    return __unseq_backend::__simd_remove_if(__first, __last - __first, __pred);
-#else
-    return std::remove_if(__first, __last, __pred);
-#endif
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-_ForwardIterator
-__pattern_remove_if(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                    _UnaryPredicate __pred) noexcept
-{
-    return __internal::__brick_remove_if(__first, __last, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryPredicate>
-_RandomAccessIterator
-__pattern_remove_if(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                    _RandomAccessIterator __last, _UnaryPredicate __pred) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::reference _ReferenceType;
-
-    if (__first == __last || __first + 1 == __last)
-    {
-        // Trivial sequence - use serial algorithm
-        return __internal::__brick_remove_if(__first, __last, __pred, _IsVector{});
-    }
-
-    return __internal::__remove_elements(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-        [&__pred](bool* __b, bool* __e, _RandomAccessIterator __it)
-        {
-            __internal::__brick_walk2(
-                __b, __e, __it, [&__pred](bool& __x, _ReferenceType __y) { __x = !__pred(__y); }, _IsVector{});
-        });
-}
-
-//------------------------------------------------------------------------
-// merge
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_merge(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-              _ForwardIterator2 __last2, _OutputIterator __d_first, _Compare __comp,
-              /* __is_vector = */ std::false_type) noexcept
-{
-    return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_merge(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-              _RandomAccessIterator2 __last2, _RandomAccessIterator3 __d_first, _Compare __comp,
-              /* __is_vector = */ std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::merge(__first1, __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_merge(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __d_first,
-                _Compare __comp) noexcept
-{
-    return __internal::__brick_merge(__first1, __last1, __first2, __last2, __d_first, __comp,
-                                     typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_merge(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                _RandomAccessIterator3 __d_first, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    __par_backend::__parallel_merge(
-        __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first,
-        __comp,
-        [](_RandomAccessIterator1 __f1, _RandomAccessIterator1 __l1, _RandomAccessIterator2 __f2,
-           _RandomAccessIterator2 __l2, _RandomAccessIterator3 __f3, _Compare __comp)
-        { return __internal::__brick_merge(__f1, __l1, __f2, __l2, __f3, __comp, _IsVector{}); });
-    return __d_first + (__last1 - __first1) + (__last2 - __first2);
-}
-
-//------------------------------------------------------------------------
-// inplace_merge
-//------------------------------------------------------------------------
-template <class _BidirectionalIterator, class _Compare>
-void
-__brick_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle, _BidirectionalIterator __last,
-                      _Compare __comp, /* __is_vector = */ std::false_type) noexcept
-{
-    std::inplace_merge(__first, __middle, __last, __comp);
-}
-
-template <class _RandomAccessIterator, class _Compare>
-void
-__brick_inplace_merge(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last,
-                      _Compare __comp, /* __is_vector = */ std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial")
-    std::inplace_merge(__first, __middle, __last, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-void
-__pattern_inplace_merge(_Tag, _ExecutionPolicy&&, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-                        _BidirectionalIterator __last, _Compare __comp) noexcept
-{
-    __internal::__brick_inplace_merge(__first, __middle, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-void
-__pattern_inplace_merge(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __middle, _RandomAccessIterator __last, _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first == __last || __first == __middle || __middle == __last)
-    {
-        return;
-    }
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _Tp;
-    auto __n = __last - __first;
-    __par_backend::__buffer<_Tp> __buf(__n);
-    _Tp* __r = __buf.get();
-    __internal::__except_handler(
-        [&]()
-        {
-            auto __move_values = [](_RandomAccessIterator __x, _Tp* __z)
-            {
-                __internal::__invoke_if_else(
-                    std::is_trivial<_Tp>(), [&]() { *__z = std::move(*__x); },
-                    [&]() { ::new (std::addressof(*__z)) _Tp(std::move(*__x)); });
-            };
-
-            auto __move_sequences = [](_RandomAccessIterator __first1, _RandomAccessIterator __last1, _Tp* __first2)
-            { return __internal::__brick_uninitialized_move(__first1, __last1, __first2, _IsVector()); };
-
-            __par_backend::__parallel_merge(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __middle, __middle, __last, __r,
-                __comp,
-                [__n, __move_values, __move_sequences](_RandomAccessIterator __f1, _RandomAccessIterator __l1,
-                                                       _RandomAccessIterator __f2, _RandomAccessIterator __l2,
-                                                       _Tp* __f3, _Compare __comp)
-                {
-                    (__utils::__serial_move_merge(__n))(__f1, __l1, __f2, __l2, __f3, __comp, __move_values,
-                                                        __move_values, __move_sequences, __move_sequences);
-                    return __f3 + (__l1 - __f1) + (__l2 - __f2);
-                });
-            __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __r, __r + __n,
-                                          [__r, __first](_Tp* __i, _Tp* __j)
-                                          { __brick_move_destroy()(__i, __j, __first + (__i - __r), _IsVector{}); });
-        });
-}
-
-//------------------------------------------------------------------------
-// includes
-//------------------------------------------------------------------------
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_includes(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp) noexcept
-{
-    return std::includes(__first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_includes(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                   _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                   _Compare __comp)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first2 >= __last2)
-        return true;
-
-    if (__first1 >= __last1 || __comp(*__first2, *__first1) || __comp(*(__last1 - 1), *(__last2 - 1)))
-        return false;
-
-    __first1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    if (__first1 == __last1)
-        return false;
-
-    if (__last2 - __first2 == 1)
-        return !__comp(*__first1, *__first2) && !__comp(*__first2, *__first1);
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return !__internal::__parallel_or(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                [__first1, __last1, __first2, __last2, &__comp](_RandomAccessIterator2 __i, _RandomAccessIterator2 __j)
-                {
-                    _PSTL_ASSERT(__j > __i);
-                    //_PSTL_ASSERT(__j - __i > 1);
-
-                    //1. moving boundaries to "consume" subsequence of equal elements
-                    auto __is_equal = [&__comp](_RandomAccessIterator2 __a, _RandomAccessIterator2 __b) -> bool
-                    { return !__comp(*__a, *__b) && !__comp(*__b, *__a); };
-
-                    //1.1 left bound, case "aaa[aaaxyz...]" - searching "x"
-                    if (__i > __first2 && __is_equal(__i, __i - 1))
-                    {
-                        //whole subrange continues to content equal elements - return "no op"
-                        if (__is_equal(__i, __j - 1))
-                            return false;
-
-                        __i = std::upper_bound(__i, __last2, *__i, __comp);
-                    }
-
-                    //1.2 right bound, case "[...aaa]aaaxyz" - searching "x"
-                    if (__j < __last2 && __is_equal(__j - 1, __j))
-                        __j = std::upper_bound(__j, __last2, *__j, __comp);
-
-                    //2. testing is __a subsequence of the second range included into the first range
-                    auto __b = std::lower_bound(__first1, __last1, *__i, __comp);
-
-                    _PSTL_ASSERT(!__comp(*(__last1 - 1), *__b));
-                    _PSTL_ASSERT(!__comp(*(__j - 1), *__i));
-                    return !std::includes(__b, __last1, __i, __j, __comp);
-                });
-        });
-}
-
-constexpr auto __set_algo_cut_off = 1000;
-
-template <class _IsVector, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2,
-          class _OutputIterator, class _Compare, class _SizeFunction, class _SetOP>
-_OutputIterator
-__parallel_set_op(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _ForwardIterator1 __first1,
-                  _ForwardIterator1 __last1, _ForwardIterator2 __first2, _ForwardIterator2 __last2,
-                  _OutputIterator __result, _Compare __comp, _SizeFunction __size_func, _SetOP __set_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
-    typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
-
-    struct _SetRange
-    {
-        _DifferenceType __pos, __len, __buf_pos;
-        bool
-        empty() const
-        {
-            return __len == 0;
-        }
-    };
-
-    const _DifferenceType __n1 = __last1 - __first1;
-    const _DifferenceType __n2 = __last2 - __first2;
-
-    __par_backend::__buffer<_Tp> __buf(__size_func(__n1, __n2));
-
-    return __internal::__except_handler(
-        [&__exec, __n1, __first1, __last1, __first2, __last2, __result, __comp, __size_func, __set_op, &__buf]()
-        {
-            auto __buffer = __buf.get();
-            _DifferenceType __m{};
-            auto __scan = [=](_DifferenceType, _DifferenceType, const _SetRange& __s) { // Scan
-                if (!__s.empty())
-                    __brick_move_destroy()(__buffer + __s.__buf_pos, __buffer + (__s.__buf_pos + __s.__len),
-                                           __result + __s.__pos, _IsVector{});
-            };
-            __par_backend::__parallel_strict_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n1, _SetRange{0, 0, 0}, //-1, 0},
-                [=](_DifferenceType __i, _DifferenceType __len) {                                  // Reduce
-                    //[__b; __e) - a subrange of the first sequence, to reduce
-                    _ForwardIterator1 __b = __first1 + __i, __e = __first1 + (__i + __len);
-
-                    //try searching for the first element which not equal to *__b
-                    if (__b != __first1)
-                        __b = std::upper_bound(__b, __last1, *__b, __comp);
-
-                    //try searching for the first element which not equal to *__e
-                    if (__e != __last1)
-                        __e = std::upper_bound(__e, __last1, *__e, __comp);
-
-                    //check is [__b; __e) empty
-                    if (__e - __b < 1)
-                    {
-                        _ForwardIterator2 __bb = __last2;
-                        if (__b != __last1)
-                            __bb = std::lower_bound(__first2, __last2, *__b, __comp);
-
-                        const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
-                        return _SetRange{0, 0, __buf_pos};
-                    }
-
-                    //try searching for "corresponding" subrange [__bb; __ee) in the second sequence
-                    _ForwardIterator2 __bb = __first2;
-                    if (__b != __first1)
-                        __bb = std::lower_bound(__first2, __last2, *__b, __comp);
-
-                    _ForwardIterator2 __ee = __last2;
-                    if (__e != __last1)
-                        __ee = std::lower_bound(__bb, __last2, *__e, __comp);
-
-                    const _DifferenceType __buf_pos = __size_func((__b - __first1), (__bb - __first2));
-                    auto __buffer_b = __buffer + __buf_pos;
-                    auto __res = __set_op(__b, __e, __bb, __ee, __buffer_b, __comp);
-
-                    return _SetRange{0, __res - __buffer_b, __buf_pos};
-                },
-                [](const _SetRange& __a, const _SetRange& __b) { // Combine
-                    if (__b.__buf_pos > __a.__buf_pos || ((__b.__buf_pos == __a.__buf_pos) && !__b.empty()))
-                        return _SetRange{__a.__pos + __a.__len + __b.__pos, __b.__len, __b.__buf_pos};
-                    return _SetRange{__b.__pos + __b.__len + __a.__pos, __a.__len, __a.__buf_pos};
-                },
-                __scan,                                     // Scan
-                [&__m, &__scan](const _SetRange& __total) { // Apex
-                    //final scan
-                    __scan(0, 0, __total);
-                    __m = __total.__pos + __total.__len;
-                });
-            return __result + __m;
-        });
-}
-
-//a shared parallel pattern for '__pattern_set_union' and '__pattern_set_symmetric_difference'
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare, class _SetUnionOp>
-_OutputIterator
-__parallel_set_union_op(_Tag __tag, _ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                        _Compare __comp, _SetUnionOp __set_union_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    auto copy_range1 = [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _OutputIterator __res)
-    { return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{}); };
-    auto copy_range2 = [](_ForwardIterator2 __begin, _ForwardIterator2 __end, _OutputIterator __res)
-    { return __internal::__brick_copy(__begin, __end, __res, typename _Tag::__is_vector{}); };
-
-    // {1} {}: parallel copying just first sequence
-    if (__n2 == 0)
-        return __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                 __result, copy_range1);
-
-    // {} {2}: parallel copying justmake  second sequence
-    if (__n1 == 0)
-        return __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                 __result, copy_range2);
-
-    // testing  whether the sequences are intersected
-    _ForwardIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-
-    if (__left_bound_seq_1 == __last1)
-    {
-        //{1} < {2}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                  __result, copy_range1);
-            },
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                  __result + __n1, copy_range2);
-            });
-        return __result + __n1 + __n2;
-    }
-
-    // testing  whether the sequences are intersected
-    _ForwardIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-
-    if (__left_bound_seq_2 == __last2)
-    {
-        //{2} < {1}: seq2 is wholly greater than seq1, so, do parallel copying seq1 and seq2
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2, __last2,
-                                                  __result, copy_range2);
-            },
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                                                  __result + __n2, copy_range1);
-            });
-        return __result + __n1 + __n2;
-    }
-
-    const auto __m1 = __left_bound_seq_1 - __first1;
-    if (__m1 > __set_algo_cut_off)
-    {
-        auto __res_or = __result;
-        __result += __m1; //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            //do parallel copying of [first1; left_bound_seq_1)
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __left_bound_seq_1, __res_or, copy_range1);
-            },
-            [=, &__result]
-            {
-                __result = __internal::__parallel_set_op(
-                    __tag, std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2,
-                    __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
-                    __set_union_op);
-            });
-        return __result;
-    }
-
-    const auto __m2 = __left_bound_seq_2 - __first2;
-    _PSTL_ASSERT(__m1 == 0 || __m2 == 0);
-    if (__m2 > __set_algo_cut_off)
-    {
-        auto __res_or = __result;
-        __result += __m2; //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
-        __par_backend::__parallel_invoke(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec),
-            //do parallel copying of [first2; left_bound_seq_2)
-            [=]
-            {
-                __internal::__pattern_walk2_brick(__tag, std::forward<_ExecutionPolicy>(__exec), __first2,
-                                                  __left_bound_seq_2, __res_or, copy_range2);
-            },
-            [=, &__result]
-            {
-                __result = __internal::__parallel_set_op(
-                    __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2,
-                    __result, __comp, [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; },
-                    __set_union_op);
-            });
-        return __result;
-    }
-
-    return __internal::__parallel_set_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_DifferenceType __n, _DifferenceType __m) { return __n + __m; }, __set_union_op);
-}
-
-//------------------------------------------------------------------------
-// set_union
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_union(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                  _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                  /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <typename _IsVector>
-struct __BrickCopyConstruct
-{
-    template <typename _ForwardIterator, typename _OutputIterator>
-    _OutputIterator
-    operator()(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result)
-    {
-        return __brick_uninitialized_copy(__first, __last, __result, _IsVector());
-    }
-};
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_union(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                  _RandomAccessIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                  /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_union(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                    _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                    _Compare __comp) noexcept
-{
-    return __internal::__brick_set_union(__first1, __last1, __first2, __last2, __result, __comp,
-                                         typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _OutputIterator, class _Compare>
-_OutputIterator
-__pattern_set_union(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                    _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                    _OutputIterator __result, _Compare __comp)
-{
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // use serial algorithm
-    if (__n1 + __n2 <= __set_algo_cut_off)
-        return std::set_union(__first1, __last1, __first2, __last2, __result, __comp);
-
-    typedef typename std::iterator_traits<_OutputIterator>::value_type _Tp;
-    return __parallel_set_union_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-           _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-        {
-            return __pstl::__utils::__set_union_construct(__first1, __last1, __first2, __last2, __result, __comp,
-                                                          __BrickCopyConstruct<_IsVector>());
-        });
-}
-
-//------------------------------------------------------------------------
-// set_intersection
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_intersection(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                         _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                         /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_intersection(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                         _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                         _RandomAccessIterator3 __result, _Compare __comp,
-                         /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_intersection(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_intersection(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                           _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                           _Compare __comp) noexcept
-{
-    return __internal::__brick_set_intersection(__first1, __last1, __first2, __last2, __result, __comp,
-                                                typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_intersection(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                           _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                           _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // intersection is empty
-    if (__n1 == 0 || __n2 == 0)
-        return __result;
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    //{1} < {2}: seq 2 is wholly greater than seq 1, so, the intersection is empty
-    if (__left_bound_seq_1 == __last1)
-        return __result;
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-    //{2} < {1}: seq 1 is wholly greater than seq 2, so, the intersection is empty
-    if (__left_bound_seq_2 == __last2)
-        return __result;
-
-    const auto __m1 = __last1 - __left_bound_seq_1 + __n2;
-    if (__m1 > __set_algo_cut_off)
-    {
-        //we know proper offset due to [first1; left_bound_seq_1) < [first2; last2)
-        return __internal::__parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __left_bound_seq_1, __last1, __first2, __last2, __result,
-            __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp) {
-                return __pstl::__utils::__set_intersection_construct(__first1, __last1, __first2, __last2, __result,
-                                                                     __comp);
-            });
-    }
-
-    const auto __m2 = __last2 - __left_bound_seq_2 + __n1;
-    if (__m2 > __set_algo_cut_off)
-    {
-        //we know proper offset due to [first2; left_bound_seq_2) < [first1; last1)
-        __result = __internal::__parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __left_bound_seq_2, __last2, __result,
-            __comp, [](_DifferenceType __n, _DifferenceType __m) { return std::min(__n, __m); },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp) {
-                return __pstl::__utils::__set_intersection_construct(__first2, __last2, __first1, __last1, __result,
-                                                                     __comp);
-            });
-        return __result;
-    }
-
-    // [left_bound_seq_1; last1) and [left_bound_seq_2; last2) - use serial algorithm
-    return std::set_intersection(__left_bound_seq_1, __last1, __left_bound_seq_2, __last2, __result, __comp);
-}
-
-//------------------------------------------------------------------------
-// set_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                       _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                       /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_difference(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                       _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp,
-                       /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                         _Compare __comp) noexcept
-{
-    return __internal::__brick_set_difference(__first1, __last1, __first2, __last2, __result, __comp,
-                                              typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                         _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                         _RandomAccessIterator2 __last2, _RandomAccessIterator3 __result, _Compare __comp)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // {} \ {2}: the difference is empty
-    if (__n1 == 0)
-        return __result;
-
-    // {1} \ {}: parallel copying just first sequence
-    if (__n2 == 0)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator1 __left_bound_seq_1 = std::lower_bound(__first1, __last1, *__first2, __comp);
-    //{1} < {2}: seq 2 is wholly greater than seq 1, so, parallel copying just first sequence
-    if (__left_bound_seq_1 == __last1)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    // testing  whether the sequences are intersected
-    _RandomAccessIterator2 __left_bound_seq_2 = std::lower_bound(__first2, __last2, *__first1, __comp);
-    //{2} < {1}: seq 1 is wholly greater than seq 2, so, parallel copying just first sequence
-    if (__left_bound_seq_2 == __last2)
-        return __internal::__pattern_walk2_brick(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __result,
-            [](_RandomAccessIterator1 __begin, _RandomAccessIterator1 __end, _RandomAccessIterator3 __res)
-            { return __internal::__brick_copy(__begin, __end, __res, _IsVector{}); });
-
-    if (__n1 + __n2 > __set_algo_cut_off)
-        return __parallel_set_op(
-            __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-            [](_DifferenceType __n, _DifferenceType) { return __n; },
-            [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-               _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-            {
-                return __pstl::__utils::__set_difference_construct(__first1, __last1, __first2, __last2, __result,
-                                                                   __comp, __BrickCopyConstruct<_IsVector>());
-            });
-
-    // use serial algorithm
-    return std::set_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-//------------------------------------------------------------------------
-// set_symmetric_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator, class _Compare>
-_OutputIterator
-__brick_set_symmetric_difference(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                                 _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                                 /*__is_vector=*/std::false_type) noexcept
-{
-    return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__brick_set_symmetric_difference(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                 _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                 _RandomAccessIterator3 __result, _Compare __comp,
-                                 /*__is_vector=*/std::true_type) noexcept
-{
-    _PSTL_PRAGMA_MESSAGE("Vectorized algorithm unimplemented, redirected to serial");
-    return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _OutputIterator,
-          class _Compare>
-_OutputIterator
-__pattern_set_symmetric_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _OutputIterator __result,
-                                   _Compare __comp) noexcept
-{
-    return __internal::__brick_set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp,
-                                                        typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _RandomAccessIterator3, class _Compare>
-_RandomAccessIterator3
-__pattern_set_symmetric_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                                   _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                   _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                   _RandomAccessIterator3 __result, _Compare __comp)
-{
-
-    const auto __n1 = __last1 - __first1;
-    const auto __n2 = __last2 - __first2;
-
-    // use serial algorithm
-    if (__n1 + __n2 <= __set_algo_cut_off)
-        return std::set_symmetric_difference(__first1, __last1, __first2, __last2, __result, __comp);
-
-    typedef typename std::iterator_traits<_RandomAccessIterator3>::value_type _Tp;
-    return __internal::__parallel_set_union_op(
-        __tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp,
-        [](_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-           _RandomAccessIterator2 __last2, _Tp* __result, _Compare __comp)
-        {
-            return __pstl::__utils::__set_symmetric_difference_construct(__first1, __last1, __first2, __last2, __result,
-                                                                         __comp, __BrickCopyConstruct<_IsVector>());
-        });
-}
-
-//------------------------------------------------------------------------
-// is_heap_until
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                      /* __is_vector = */ std::false_type) noexcept
-{
-    return std::is_heap_until(__first, __last, __comp);
-}
-
-template <class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__brick_is_heap_until(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                      /* __is_vector = */ std::true_type) noexcept
-{
-    if (__last - __first < 2)
-        return __last;
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _SizeType;
-    return __unseq_backend::__simd_first(
-        __first, _SizeType(0), __last - __first,
-        [&__comp](_RandomAccessIterator __it, _SizeType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(_Tag, _ExecutionPolicy&&, _RandomAccessIterator __first, _RandomAccessIterator __last,
-                        _Compare __comp) noexcept
-{
-    return __internal::__brick_is_heap_until(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
-_RandomAccessIterator
-__is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
-                      /* __is_vector = */ std::false_type) noexcept
-{
-    _DifferenceType __i = __begin;
-    for (; __i < __end; ++__i)
-    {
-        if (__comp(__first[(__i - 1) / 2], __first[__i]))
-        {
-            break;
-        }
-    }
-    return __first + __i;
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _Compare>
-_RandomAccessIterator
-__is_heap_until_local(_RandomAccessIterator __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp,
-                      /* __is_vector = */ std::true_type) noexcept
-{
-    return __unseq_backend::__simd_first(
-        __first, __begin, __end,
-        [&__comp](_RandomAccessIterator __it, _DifferenceType __i) { return __comp(__it[(__i - 1) / 2], __it[__i]); });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-_RandomAccessIterator
-__pattern_is_heap_until(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                        _RandomAccessIterator __last, _Compare __comp) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__last - __first < 2)
-        return __last;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __parallel_find(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__first, __comp](_RandomAccessIterator __i, _RandomAccessIterator __j) {
-                    return __internal::__is_heap_until_local(__first, __i - __first, __j - __first, __comp,
-                                                             _IsVector{});
-                },
-                std::less<typename std::iterator_traits<_RandomAccessIterator>::difference_type>(), /*is_first=*/true);
-        });
-}
-
-//------------------------------------------------------------------------
-// min_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__brick_min_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
-                    /* __is_vector = */ std::false_type) noexcept
-{
-    return std::min_element(__first, __last, __comp);
-}
-
-template <typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__brick_min_element(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                    /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_UDR_PRESENT)
-    return __unseq_backend::__simd_min_element(__first, __last - __first, __comp);
-#else
-    return std::min_element(__first, __last, __comp);
-#endif
-}
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__pattern_min_element(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                      _Compare __comp) noexcept
-{
-    return __internal::__brick_min_element(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-_RandomAccessIterator
-__pattern_min_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                      _RandomAccessIterator __last, _Compare __comp)
-{
-    if (__first == __last)
-        return __last;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last, __first,
-                [=](_RandomAccessIterator __begin, _RandomAccessIterator __end,
-                    _RandomAccessIterator __init) -> _RandomAccessIterator
-                {
-                    const _RandomAccessIterator subresult =
-                        __internal::__brick_min_element(__begin, __end, __comp, _IsVector{});
-                    return __internal::__cmp_iterators_by_values(__init, subresult, __comp);
-                },
-                [=](_RandomAccessIterator __it1, _RandomAccessIterator __it2) -> _RandomAccessIterator
-                { return __internal::__cmp_iterators_by_values(__it1, __it2, __comp); });
-        });
-}
-
-//------------------------------------------------------------------------
-// minmax_element
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__brick_minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp,
-                       /* __is_vector = */ std::false_type) noexcept
-{
-    return std::minmax_element(__first, __last, __comp);
-}
-
-template <typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__brick_minmax_element(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp,
-                       /* __is_vector = */ std::true_type) noexcept
-{
-#if defined(_PSTL_UDR_PRESENT)
-    return __unseq_backend::__simd_minmax_element(__first, __last - __first, __comp);
-#else
-    return std::minmax_element(__first, __last, __comp);
-#endif
-}
-
-template <typename _Tag, typename _ExecutionPolicy, typename _ForwardIterator, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__pattern_minmax_element(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _Compare __comp) noexcept
-{
-    return __internal::__brick_minmax_element(__first, __last, __comp, typename _Tag::__is_vector{});
-}
-
-template <typename _IsVector, typename _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare>
-std::pair<_RandomAccessIterator, _RandomAccessIterator>
-__pattern_minmax_element(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _Compare __comp)
-{
-    if (__first == __last)
-        return std::make_pair(__first, __first);
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler([&]() {
-        typedef std::pair<_RandomAccessIterator, _RandomAccessIterator> _Result;
-
-        return __par_backend::__parallel_reduce(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first + 1, __last,
-            std::make_pair(__first, __first),
-            [=](_RandomAccessIterator __begin, _RandomAccessIterator __end, _Result __init) -> _Result
-            {
-                const _Result __subresult = __internal::__brick_minmax_element(__begin, __end, __comp, _IsVector{});
-                return std::make_pair(
-                    __internal::__cmp_iterators_by_values(__subresult.first, __init.first, __comp),
-                    __internal::__cmp_iterators_by_values(__init.second, __subresult.second, std::not_fn(__comp)));
-            },
-            [=](_Result __p1, _Result __p2) -> _Result
-            {
-                return std::make_pair(
-                    __internal::__cmp_iterators_by_values(__p1.first, __p2.first, __comp),
-                    __internal::__cmp_iterators_by_values(__p2.second, __p1.second, std::not_fn(__comp)));
-            });
-    });
-}
-
-//------------------------------------------------------------------------
-// mismatch
-//------------------------------------------------------------------------
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__mismatch_serial(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                  _ForwardIterator2 __last2, _BinaryPredicate __pred)
-{
-#if defined(_PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT)
-    return std::mismatch(__first1, __last1, __first2, __last2, __pred);
-#else
-    for (; __first1 != __last1 && __first2 != __last2 && __pred(*__first1, *__first2); ++__first1, ++__first2)
-    {
-    }
-    return std::make_pair(__first1, __first2);
-#endif
-}
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__brick_mismatch(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                 _ForwardIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::false_type) noexcept
-{
-    return __mismatch_serial(__first1, __last1, __first2, __last2, __pred);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__brick_mismatch(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2,
-                 _RandomAccessIterator2 __last2, _Predicate __pred, /* __is_vector = */ std::true_type) noexcept
-{
-    auto __n = std::min(__last1 - __first1, __last2 - __first2);
-    return __unseq_backend::__simd_first(__first1, __n, __first2, std::not_fn(__pred));
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-std::pair<_ForwardIterator1, _ForwardIterator2>
-__pattern_mismatch(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                   _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Predicate __pred) noexcept
-{
-    return __internal::__brick_mismatch(__first1, __last1, __first2, __last2, __pred, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Predicate>
-std::pair<_RandomAccessIterator1, _RandomAccessIterator2>
-__pattern_mismatch(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                   _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                   _Predicate __pred) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler([&]() {
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        auto __result = __internal::__parallel_find(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-            [__first1, __first2, __pred](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-            {
-                return __internal::__brick_mismatch(__i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
-                                                    __pred, _IsVector{})
-                    .first;
-            },
-            std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-        return std::make_pair(__result, __first2 + (__result - __first1));
-    });
-}
-
-//------------------------------------------------------------------------
-// lexicographical_compare
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__brick_lexicographical_compare(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                                _ForwardIterator2 __last2, _Compare __comp,
-                                /* __is_vector = */ std::false_type) noexcept
-{
-    return std::lexicographical_compare(__first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Compare>
-bool
-__brick_lexicographical_compare(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2, _Compare __comp,
-                                /* __is_vector = */ std::true_type) noexcept
-{
-    if (__first2 == __last2)
-    { // if second sequence is empty
-        return false;
-    }
-    else if (__first1 == __last1)
-    { // if first sequence is empty
-        return true;
-    }
-    else
-    {
-        typedef typename std::iterator_traits<_RandomAccessIterator1>::reference ref_type1;
-        typedef typename std::iterator_traits<_RandomAccessIterator2>::reference ref_type2;
-        --__last1;
-        --__last2;
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        std::pair<_RandomAccessIterator1, _RandomAccessIterator2> __result = __unseq_backend::__simd_first(
-            __first1, __n, __first2, [__comp](const ref_type1 __x, const ref_type2 __y) mutable {
-                return __comp(__x, __y) || __comp(__y, __x);
-            });
-
-        if (__result.first == __last1 && __result.second != __last2)
-        { // if first sequence shorter than second
-            return !__comp(*__result.second, *__result.first);
-        }
-        else
-        { // if second sequence shorter than first or both have the same number of elements
-            return __comp(*__result.first, *__result.second);
-        }
-    }
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-bool
-__pattern_lexicographical_compare(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                                  _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp) noexcept
-{
-    return __internal::__brick_lexicographical_compare(__first1, __last1, __first2, __last2, __comp,
-                                                       typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Compare>
-bool
-__pattern_lexicographical_compare(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                                  _RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                                  _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                                  _Compare __comp) noexcept
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    if (__first2 == __last2)
-    { // if second sequence is empty
-        return false;
-    }
-    else if (__first1 == __last1)
-    { // if first sequence is empty
-        return true;
-    }
-    else
-    {
-        typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _RefType1;
-        typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _RefType2;
-        --__last1;
-        --__last2;
-        auto __n = std::min(__last1 - __first1, __last2 - __first2);
-        auto __result = __internal::__parallel_find(
-            __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __first1 + __n,
-            [__first1, __first2, &__comp](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j)
-            {
-                return __internal::__brick_mismatch(
-                           __i, __j, __first2 + (__i - __first1), __first2 + (__j - __first1),
-                           [&__comp](const _RefType1 __x, const _RefType2 __y)
-                           { return !__comp(__x, __y) && !__comp(__y, __x); },
-                           _IsVector{})
-                    .first;
-            },
-            std::less<typename std::iterator_traits<_RandomAccessIterator1>::difference_type>(), /*is_first=*/true);
-
-        if (__result == __last1 && __first2 + (__result - __first1) != __last2)
-        { // if first sequence shorter than second
-            return !__comp(*(__first2 + (__result - __first1)), *__result);
-        }
-        else
-        { // if second sequence shorter than first or both have the same number of elements
-            return __comp(*__result, *(__first2 + (__result - __first1)));
-        }
-    }
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_ALGORITHM_IMPL_H */
diff --git a/pstl/include/pstl/internal/execution_defs.h b/pstl/include/pstl/internal/execution_defs.h
deleted file mode 100644
index d7c412697233..000000000000
--- a/pstl/include/pstl/internal/execution_defs.h
+++ /dev/null
@@ -1,100 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_EXECUTION_POLICY_DEFS_H
-#define _PSTL_EXECUTION_POLICY_DEFS_H
-
-#include <type_traits>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace execution
-{
-inline namespace v1
-{
-
-// 2.4, Sequential execution policy
-class sequenced_policy
-{
-};
-
-// 2.5, Parallel execution policy
-class parallel_policy
-{
-};
-
-// 2.6, Parallel+Vector execution policy
-class parallel_unsequenced_policy
-{
-};
-
-class unsequenced_policy
-{
-};
-
-// 2.8, Execution policy objects
-constexpr sequenced_policy seq{};
-constexpr parallel_policy par{};
-constexpr parallel_unsequenced_policy par_unseq{};
-constexpr unsequenced_policy unseq{};
-
-// 2.3, Execution policy type trait
-template <class T>
-struct is_execution_policy : std::false_type
-{
-};
-
-template <>
-struct is_execution_policy<__pstl::execution::sequenced_policy> : std::true_type
-{
-};
-template <>
-struct is_execution_policy<__pstl::execution::parallel_policy> : std::true_type
-{
-};
-template <>
-struct is_execution_policy<__pstl::execution::parallel_unsequenced_policy> : std::true_type
-{
-};
-template <>
-struct is_execution_policy<__pstl::execution::unsequenced_policy> : std::true_type
-{
-};
-
-#if defined(_PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT)
-template <class T>
-constexpr bool is_execution_policy_v = __pstl::execution::is_execution_policy<T>::value;
-#endif
-
-} // namespace v1
-} // namespace execution
-
-namespace __internal
-{
-template <class ExecPolicy, class T>
-using __enable_if_execution_policy =
-    typename std::enable_if<__pstl::execution::is_execution_policy<typename std::decay<ExecPolicy>::type>::value,
-                            T>::type;
-
-template <class _IsVector>
-struct __serial_tag;
-template <class _IsVector>
-struct __parallel_tag;
-
-} // namespace __internal
-
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_EXECUTION_POLICY_DEFS_H */
diff --git a/pstl/include/pstl/internal/execution_impl.h b/pstl/include/pstl/internal/execution_impl.h
deleted file mode 100644
index 5dc622b44122..000000000000
--- a/pstl/include/pstl/internal/execution_impl.h
+++ /dev/null
@@ -1,105 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_EXECUTION_IMPL_H
-#define _PSTL_EXECUTION_IMPL_H
-
-#include <iterator>
-#include <type_traits>
-
-#include "pstl_config.h"
-#include "execution_defs.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-template <typename _IteratorTag, typename... _IteratorTypes>
-using __are_iterators_of = std::conjunction<
-    std::is_base_of<_IteratorTag, typename std::iterator_traits<std::decay_t<_IteratorTypes>>::iterator_category>...>;
-
-template <typename... _IteratorTypes>
-using __are_random_access_iterators = __are_iterators_of<std::random_access_iterator_tag, _IteratorTypes...>;
-
-struct __serial_backend_tag
-{
-};
-struct __tbb_backend_tag
-{
-};
-struct __openmp_backend_tag
-{
-};
-
-#if defined(_PSTL_PAR_BACKEND_TBB)
-using __par_backend_tag = __tbb_backend_tag;
-#elif defined(_PSTL_PAR_BACKEND_OPENMP)
-using __par_backend_tag = __openmp_backend_tag;
-#elif defined(_PSTL_PAR_BACKEND_SERIAL)
-using __par_backend_tag = __serial_backend_tag;
-#else
-#    error "A parallel backend must be specified";
-#endif
-
-template <class _IsVector>
-struct __serial_tag
-{
-    using __is_vector = _IsVector;
-};
-
-template <class _IsVector>
-struct __parallel_tag
-{
-    using __is_vector = _IsVector;
-    // backend tag can be change depending on
-    // TBB availability in the environment
-    using __backend_tag = __par_backend_tag;
-};
-
-template <class _IsVector, class... _IteratorTypes>
-using __tag_type = typename std::conditional<__internal::__are_random_access_iterators<_IteratorTypes...>::value,
-                                             __parallel_tag<_IsVector>, __serial_tag<_IsVector>>::type;
-
-template <class... _IteratorTypes>
-__serial_tag</*_IsVector = */ std::false_type>
-__select_backend(__pstl::execution::sequenced_policy, _IteratorTypes&&...)
-{
-    return {};
-}
-
-template <class... _IteratorTypes>
-__serial_tag<__internal::__are_random_access_iterators<_IteratorTypes...>>
-__select_backend(__pstl::execution::unsequenced_policy, _IteratorTypes&&...)
-{
-    return {};
-}
-
-template <class... _IteratorTypes>
-__tag_type</*_IsVector = */ std::false_type, _IteratorTypes...>
-__select_backend(__pstl::execution::parallel_policy, _IteratorTypes&&...)
-{
-    return {};
-}
-
-template <class... _IteratorTypes>
-__tag_type<__internal::__are_random_access_iterators<_IteratorTypes...>, _IteratorTypes...>
-__select_backend(__pstl::execution::parallel_unsequenced_policy, _IteratorTypes&&...)
-{
-    return {};
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_EXECUTION_IMPL_H */
diff --git a/pstl/include/pstl/internal/glue_algorithm_defs.h b/pstl/include/pstl/internal/glue_algorithm_defs.h
deleted file mode 100644
index 28a7f92163b1..000000000000
--- a/pstl/include/pstl/internal/glue_algorithm_defs.h
+++ /dev/null
@@ -1,558 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_ALGORITHM_DEFS_H
-#define _PSTL_GLUE_ALGORITHM_DEFS_H
-
-#include <functional>
-#include <iterator>
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [alg.any_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.all_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-all_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.none_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-none_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.foreach]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-for_each(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-for_each_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, _Function __f);
-
-// [alg.find]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if_not(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-// [alg.find.end]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last);
-
-// [alg.find_first_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last);
-
-// [alg.adjacent_find]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred);
-
-// [alg.count]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred);
-
-// [alg.search]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value);
-
-// [alg.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_n(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _Size __n, _ForwardIterator2 __result);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 result,
-        _Predicate __pred);
-
-// [alg.swap]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-swap_ranges(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-            _ForwardIterator2 __first2);
-
-// [alg.transform]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-          _UnaryOperation __op);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator __result, _BinaryOperation __op);
-
-// [alg.replace]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-           const _Tp& __new_value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value,
-        const _Tp& __new_value);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                _ForwardIterator2 __result, _UnaryPredicate __pred, const _Tp& __new_value);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-             const _Tp& __old_value, const _Tp& __new_value);
-
-// [alg.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, const _Tp& __value);
-
-// [alg.generate]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size count, _Generator __g);
-
-// [alg.remove]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Predicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            const _Tp& __value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-// [alg.unique]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result);
-
-// [alg.reverse]
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last);
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-             _ForwardIterator __d_first);
-
-// [alg.rotate]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __middle, _ForwardIterator1 __last,
-            _ForwardIterator2 __result);
-
-// [alg.partitions]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator>
-stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                 _UnaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _ForwardIterator1, class _ForwardIterator2,
-          class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-partition_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-               _ForwardIterator1 __out_true, _ForwardIterator2 __out_false, _UnaryPredicate __pred);
-
-// [alg.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
-
-// [stable.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
-
-// [mismatch]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _BinaryPredicate __pred);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
-
-// [alg.equal]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _BinaryPredicate __p);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _BinaryPredicate __p);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2);
-
-// [alg.move]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first);
-
-// [partial.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last);
-
-// [partial.sort.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last);
-
-// [is.sorted]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-// [alg.nth.element]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last);
-
-// [alg.merge]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first);
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last);
-
-// [includes]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2);
-
-// [set.union]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result);
-
-// [set.intersection]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
-
-// [set.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
-
-// [set.symmetric.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator result,
-                         _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result);
-
-// [is.heap]
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last);
-
-// [alg.min.max]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-// [alg.lex.comparison]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2);
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_ALGORITHM_DEFS_H */
diff --git a/pstl/include/pstl/internal/glue_algorithm_impl.h b/pstl/include/pstl/internal/glue_algorithm_impl.h
deleted file mode 100644
index fb0c19d77665..000000000000
--- a/pstl/include/pstl/internal/glue_algorithm_impl.h
+++ /dev/null
@@ -1,1108 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_ALGORITHM_IMPL_H
-#define _PSTL_GLUE_ALGORITHM_IMPL_H
-
-#include <functional>
-
-#include "pstl_config.h"
-
-#include "execution_defs.h"
-#include "utils.h"
-#include "algorithm_fwd.h"
-#include "numeric_fwd.h" /* count and count_if use __pattern_transform_reduce */
-
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [alg.any_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-any_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_any_of(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __pred);
-}
-
-// [alg.all_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Pred>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-all_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Pred __pred)
-{
-    return !std::any_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::not_fn(__pred));
-}
-
-// [alg.none_of]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-none_of(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    return !std::any_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, __pred);
-}
-
-// [alg.foreach]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-for_each(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Function __f)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __f);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Function>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-for_each_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, _Function __f)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_walk1_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                                                 __f);
-}
-
-// [alg.find]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_find_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                 __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find_if_not(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    return std::find_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::not_fn(__pred));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    return std::find_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                        __pstl::__internal::__equal_value<_Tp>(__value));
-}
-
-// [alg.find.end]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_find_end(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                  __last, __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_end(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-         _ForwardIterator2 __s_last)
-{
-    return std::find_end(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last,
-                         std::equal_to<>());
-}
-
-// [alg.find_first_of]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_find_first_of(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-find_first_of(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-              _ForwardIterator2 __s_first, _ForwardIterator2 __s_last)
-{
-    return std::find_first_of(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last,
-                              std::equal_to<>());
-}
-
-// [alg.adjacent_find]
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, std::equal_to<_ValueType>(), /*first_semantic*/ false);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-adjacent_find(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __pred, /*first_semantic*/ false);
-}
-
-// [alg.count]
-
-// Implementation note: count and count_if call the pattern directly instead of calling std::transform_reduce
-// so that we do not have to include <numeric>.
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return __pstl::__internal::__pattern_count(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               [&__value](const _ValueType& __x) { return __value == __x; });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::difference_type>
-count_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_count(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               __pred);
-}
-
-// [alg.search]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __s_first);
-
-    return __pstl::__internal::__pattern_search(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __s_first, __s_last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator1>
-search(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-       _ForwardIterator2 __s_last)
-{
-    return std::search(std::forward<_ExecutionPolicy>(__exec), __first, __last, __s_first, __s_last, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_search_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                  __last, __count, __value, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-search_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Size __count,
-         const _Tp& __value)
-{
-    return std::search_n(std::forward<_ExecutionPolicy>(__exec), __first, __last, __count, __value,
-                         std::equal_to<typename iterator_traits<_ForwardIterator>::value_type>());
-}
-
-// [alg.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-        [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _Size, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_n(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _Size __n, _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick_n(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-        [](_ForwardIterator1 __begin, _Size __sz, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-        _Predicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_copy_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                 __last, __result, __pred);
-}
-
-// [alg.swap]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-swap_ranges(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-            _ForwardIterator2 __first2)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2,
-                                               [](_ReferenceType1 __x, _ReferenceType2 __y)
-                                               {
-                                                   using std::swap;
-                                                   swap(__x, __y);
-                                               });
-}
-
-// [alg.transform]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-          _UnaryOperation __op)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                               __result,
-                                               [__op](_InputType __x, _OutputType __y) mutable { __y = __op(__x); });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-transform(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator __result, _BinaryOperation __op)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _Input1Type;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _Input2Type;
-    typedef typename iterator_traits<_ForwardIterator>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_walk3(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __result,
-        [__op](_Input1Type x, _Input2Type y, _OutputType z) mutable { z = __op(x, y); });
-}
-
-// [alg.replace]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred,
-           const _Tp& __new_value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::reference _ElementType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                        [&__pred, &__new_value](_ElementType __elem)
-                                        {
-                                            if (__pred(__elem))
-                                            {
-                                                __elem = __new_value;
-                                            }
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-replace(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __old_value,
-        const _Tp& __new_value)
-{
-    std::replace_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                    __pstl::__internal::__equal_value<_Tp>(__old_value), __new_value);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryPredicate, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                _ForwardIterator2 __result, _UnaryPredicate __pred, const _Tp& __new_value)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::reference _InputType;
-    typedef typename iterator_traits<_ForwardIterator2>::reference _OutputType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_walk2(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-        [__pred, &__new_value](_InputType __x, _OutputType __y) mutable { __y = __pred(__x) ? __new_value : __x; });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-replace_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-             const _Tp& __old_value, const _Tp& __new_value)
-{
-    return std::replace_copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                                __pstl::__internal::__equal_value<_Tp>(__old_value), __new_value);
-}
-
-// [alg.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_fill(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                       __value);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, const _Tp& __value)
-{
-    if (__count <= 0)
-        return __first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_fill_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __count, __value);
-}
-
-// [alg.generate]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-generate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Generator __g)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_generate(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                           __g);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Generator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-generate_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __count, _Generator __g)
-{
-    if (__count <= 0)
-        return __first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_generate_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                    __count, __g);
-}
-
-// [alg.remove]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Predicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy_if(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Predicate __pred)
-{
-    return std::copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, std::not_fn(__pred));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-remove_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            const _Tp& __value)
-{
-    return std::copy_if(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                        __pstl::__internal::__not_equal_value<_Tp>(__value));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove_if(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_remove_if(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                   __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-remove(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    return std::remove_if(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                          __pstl::__internal::__equal_value<_Tp>(__value));
-}
-
-// [alg.unique]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_unique(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-unique(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    return std::unique(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result,
-            _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_unique_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __result, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-unique_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __result)
-{
-    return std::unique_copy(__exec, __first, __last, __result, std::equal_to<>());
-}
-
-// [alg.reverse]
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-reverse(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_reverse(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-reverse_copy(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-             _ForwardIterator __d_first)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_reverse_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                      __last, __d_first);
-}
-
-// [alg.rotate]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-rotate(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_rotate(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __middle, __last);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-rotate_copy(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __middle, _ForwardIterator1 __last,
-            _ForwardIterator2 __result)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_rotate_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __middle, __last, __result);
-}
-
-// [alg.partitions]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_partitioned(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_is_partitioned(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-partition(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_partition(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                   __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _BidirectionalIterator>
-stable_partition(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __last,
-                 _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_stable_partition(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first, __last, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _ForwardIterator1, class _ForwardIterator2,
-          class _UnaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-partition_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-               _ForwardIterator1 __out_true, _ForwardIterator2 __out_false, _UnaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __out_true, __out_false);
-
-    return __pstl::__internal::__pattern_partition_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __out_true, __out_false, __pred);
-}
-
-// [alg.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return __pstl::__internal::__pattern_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                              __comp, typename std::is_move_constructible<_InputType>::type());
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::sort(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [stable.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_stable_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-stable_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::stable_sort(__exec, __first, __last, std::less<_InputType>());
-}
-
-// [mismatch]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _BinaryPredicate __pred)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_mismatch(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __last1, __first2, __last2, __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _BinaryPredicate __pred)
-{
-    return std::mismatch(__exec, __first1, __last1, __first2, std::next(__first2, std::distance(__first1, __last1)),
-                         __pred);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2)
-{
-    return std::mismatch(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                         std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator1, _ForwardIterator2>>
-mismatch(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2)
-{
-    //TODO: to get rid of "distance"
-    return std::mismatch(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2,
-                         std::next(__first2, std::distance(__first1, __last1)));
-}
-
-// [alg.equal]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _BinaryPredicate __p)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_equal(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __p);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2)
-{
-    return std::equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, std::equal_to<>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _BinaryPredicate __p)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_equal(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __last2, __p);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-equal(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2)
-{
-    return equal(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::equal_to<>());
-}
-
-// [alg.move]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-move(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __d_first)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__pattern_walk2_brick(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first,
-        [](_ForwardIterator1 __begin, _ForwardIterator1 __end, _ForwardIterator2 __res)
-        { return __pstl::__internal::__brick_move(__begin, __end, __res, __is_vector{}); });
-}
-
-// [partial.sort]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_partial_sort(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                               __middle, __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-partial_sort(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __middle,
-             _RandomAccessIterator __last)
-{
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::partial_sort(__exec, __first, __middle, __last, std::less<_InputType>());
-}
-
-// [partial.sort.copy]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_partial_sort_copy(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                           __first, __last, __d_first, __d_last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-partial_sort_copy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last,
-                  _RandomAccessIterator __d_first, _RandomAccessIterator __d_last)
-{
-    return std::partial_sort_copy(std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first, __d_last,
-                                  std::less<>());
-}
-
-// [is.sorted]
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    const _ForwardIterator __res =
-        __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                    __last, __pstl::__internal::__reorder_pred<_Compare>(__comp),
-                                                    /*first_semantic*/ false);
-    return __res == __last ? __last : std::next(__res);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-is_sorted_until(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return is_sorted_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_adjacent_find(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __pstl::__internal::__reorder_pred<_Compare>(__comp),
-                                                       /*or_semantic*/ true) == __last;
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_sorted(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::is_sorted(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [alg.merge]
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __d_first);
-
-    return __pstl::__internal::__pattern_merge(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                               __last1, __first2, __last2, __d_first, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-merge(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-      _ForwardIterator2 __last2, _ForwardIterator __d_first)
-{
-    return std::merge(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __d_first,
-                      std::less<>());
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_inplace_merge(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                __middle, __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _BidirectionalIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-inplace_merge(_ExecutionPolicy&& __exec, _BidirectionalIterator __first, _BidirectionalIterator __middle,
-              _BidirectionalIterator __last)
-{
-    typedef typename std::iterator_traits<_BidirectionalIterator>::value_type _InputType;
-    std::inplace_merge(__exec, __first, __middle, __last, std::less<_InputType>());
-}
-
-// [includes]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_includes(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                  __last1, __first2, __last2, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-includes(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-         _ForwardIterator2 __last2)
-{
-    return std::includes(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, std::less<>());
-}
-
-// [set.union]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_union(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1,
-                                                   __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_union(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-          _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_union(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                          std::less<>());
-}
-
-// [set.intersection]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_intersection(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_intersection(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_intersection(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                                 std::less<>());
-}
-
-// [set.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_difference(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                        __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-               _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_difference(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result,
-                               std::less<>());
-}
-
-// [set.symmetric.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator,
-          class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result,
-                         _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2, __result);
-
-    return __pstl::__internal::__pattern_set_symmetric_difference(
-        __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2, __result, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-set_symmetric_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                         _ForwardIterator2 __first2, _ForwardIterator2 __last2, _ForwardIterator __result)
-{
-    return std::set_symmetric_difference(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                                         __result, std::less<>());
-}
-
-// [is.heap]
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__pattern_is_heap_until(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _RandomAccessIterator>
-is_heap_until(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
-{
-    return std::is_heap_until(std::forward<_ExecutionPolicy>(__exec), __first, __last, __comp) == __last;
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-is_heap(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __last)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    return std::is_heap(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-// [alg.min.max]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_min_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                     __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-min_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    return min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                       __pstl::__internal::__reorder_pred<_Compare>(__comp));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-max_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _InputType;
-    return std::min_element(std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                            __pstl::__internal::__reorder_pred<std::less<_InputType>>(std::less<_InputType>()));
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_minmax_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, std::pair<_ForwardIterator, _ForwardIterator>>
-minmax_element(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return std::minmax_element(std::forward<_ExecutionPolicy>(__exec), __first, __last, std::less<_ValueType>());
-}
-
-// [alg.nth.element]
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__pattern_nth_element(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __nth,
-                                              __last, __comp);
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-nth_element(_ExecutionPolicy&& __exec, _RandomAccessIterator __first, _RandomAccessIterator __nth,
-            _RandomAccessIterator __last)
-{
-    typedef typename iterator_traits<_RandomAccessIterator>::value_type _InputType;
-    std::nth_element(std::forward<_ExecutionPolicy>(__exec), __first, __nth, __last, std::less<_InputType>());
-}
-
-// [alg.lex.comparison]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Compare>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2, _Compare __comp)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    return __pstl::__internal::__pattern_lexicographical_compare(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                                 __first1, __last1, __first2, __last2, __comp);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, bool>
-lexicographical_compare(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                        _ForwardIterator2 __first2, _ForwardIterator2 __last2)
-{
-    return std::lexicographical_compare(std::forward<_ExecutionPolicy>(__exec), __first1, __last1, __first2, __last2,
-                                        std::less<>());
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_ALGORITHM_IMPL_H */
diff --git a/pstl/include/pstl/internal/glue_execution_defs.h b/pstl/include/pstl/internal/glue_execution_defs.h
deleted file mode 100644
index df9a477f721e..000000000000
--- a/pstl/include/pstl/internal/glue_execution_defs.h
+++ /dev/null
@@ -1,55 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_EXECUTION_DEFS_H
-#define _PSTL_GLUE_EXECUTION_DEFS_H
-
-#include <type_traits>
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-namespace std
-{
-// Type trait
-using __pstl::execution::is_execution_policy;
-#if defined(_PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT)
-#    if defined(__INTEL_COMPILER)
-template <class T>
-constexpr bool is_execution_policy_v = is_execution_policy<T>::value;
-#    else
-using __pstl::execution::is_execution_policy_v;
-#    endif
-#endif
-
-namespace execution
-{
-// Standard C++ policy classes
-using __pstl::execution::parallel_policy;
-using __pstl::execution::parallel_unsequenced_policy;
-using __pstl::execution::sequenced_policy;
-
-// Standard predefined policy instances
-using __pstl::execution::par;
-using __pstl::execution::par_unseq;
-using __pstl::execution::seq;
-
-// Implementation-defined names
-// Unsequenced policy is not yet standard, but for consistency
-// we include it into namespace std::execution as well
-using __pstl::execution::unseq;
-using __pstl::execution::unsequenced_policy;
-} // namespace execution
-} // namespace std
-
-#include "algorithm_impl.h"
-#include "numeric_impl.h"
-#include "parallel_backend.h"
-
-#endif /* _PSTL_GLUE_EXECUTION_DEFS_H */
diff --git a/pstl/include/pstl/internal/glue_memory_defs.h b/pstl/include/pstl/internal/glue_memory_defs.h
deleted file mode 100644
index ae523339d52d..000000000000
--- a/pstl/include/pstl/internal/glue_memory_defs.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_MEMORY_DEFS_H
-#define _PSTL_GLUE_MEMORY_DEFS_H
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [uninitialized.copy]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
-
-// [uninitialized.move]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result);
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result);
-
-// [uninitialized.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value);
-
-// [specialized.destroy]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
-
-// [uninitialized.construct.default]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
-
-// [uninitialized.construct.value]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n);
-
-} //  namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_MEMORY_DEFS_H */
diff --git a/pstl/include/pstl/internal/glue_memory_impl.h b/pstl/include/pstl/internal/glue_memory_impl.h
deleted file mode 100644
index 39c595d697a2..000000000000
--- a/pstl/include/pstl/internal/glue_memory_impl.h
+++ /dev/null
@@ -1,352 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_MEMORY_IMPL_H
-#define _PSTL_GLUE_MEMORY_IMPL_H
-
-#include "pstl_config.h"
-
-#include "execution_defs.h"
-#include "utils.h"
-#include "algorithm_fwd.h"
-
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [uninitialized.copy]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                       __last, __result,
-                                                       [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                                                       { ::new (std::addressof(__val2)) _ValueType2(__val1); });
-        });
-}
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_copy_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_InputIterator __begin, _Size __sz, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                         __first, __n, __result,
-                                                         [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                                                         { ::new (std::addressof(__val2)) _ValueType2(__val1); });
-        });
-}
-
-// [uninitialized.move]
-
-template <class _ExecutionPolicy, class _InputIterator, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move(_ExecutionPolicy&& __exec, _InputIterator __first, _InputIterator __last, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_InputIterator __begin, _InputIterator __end, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy(__begin, __end, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                { ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); });
-        });
-}
-
-template <class _ExecutionPolicy, class _InputIterator, class _Size, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_move_n(_ExecutionPolicy&& __exec, _InputIterator __first, _Size __n, _ForwardIterator __result)
-{
-    typedef typename iterator_traits<_InputIterator>::value_type _ValueType1;
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType2;
-    typedef typename iterator_traits<_InputIterator>::reference _ReferenceType1;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType2;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::integral_constant < bool, std::is_trivial<_ValueType1>::value&& std::is_trivial<_ValueType2>::value > (),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_InputIterator __begin, _Size __sz, _ForwardIterator __res)
-                { return __pstl::__internal::__brick_copy_n(__begin, __sz, __res, __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk2_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n, __result,
-                [](_ReferenceType1 __val1, _ReferenceType2 __val2)
-                { ::new (std::addressof(__val2)) _ValueType2(std::move(__val1)); });
-        });
-}
-
-// [uninitialized.fill]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_fill(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    __pstl::__internal::__invoke_if_else(
-        std::is_arithmetic<_ValueType>(),
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [&__value](_ForwardIterator __begin, _ForwardIterator __end)
-                { __pstl::__internal::__brick_fill(__begin, __end, _ValueType(__value), __is_vector{}); });
-        },
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [&__value](_ReferenceType __val)
-                                                { ::new (std::addressof(__val)) _ValueType(__value); });
-        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_fill_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n, const _Tp& __value)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_arithmetic<_ValueType>(),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [&__value](_ForwardIterator __begin, _Size __count)
-                { return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(__value), __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [&__value](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(__value); });
-        });
-}
-
-// [specialized.destroy]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-destroy(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__invoke_if_not(std::is_trivially_destructible<_ValueType>(),
-                                        [&]()
-                                        {
-                                            __pstl::__internal::__pattern_walk1(
-                                                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val) { __val.~_ValueType(); });
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-destroy_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivially_destructible<_ValueType>(), [&]() { return std::next(__first, __n); },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                         __first, __n,
-                                                         [](_ReferenceType __val) { __val.~_ValueType(); });
-        });
-}
-
-// [uninitialized.construct.default]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_default_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    __pstl::__internal::__invoke_if_not(std::is_trivial<_ValueType>(),
-                                        [&]()
-                                        {
-                                            __pstl::__internal::__pattern_walk1(
-                                                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; });
-                                        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_default_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(), [&]() { return std::next(__first, __n); },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType; });
-        });
-}
-
-// [uninitialized.construct.value]
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, void>
-uninitialized_value_construct(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(),
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk_brick(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [](_ForwardIterator __begin, _ForwardIterator __end)
-                { __pstl::__internal::__brick_fill(__begin, __end, _ValueType(), __is_vector{}); });
-        },
-        [&]()
-        {
-            __pstl::__internal::__pattern_walk1(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                [](_ReferenceType __val)
-                                                { ::new (std::addressof(__val)) _ValueType(); });
-        });
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Size>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator>
-uninitialized_value_construct_n(_ExecutionPolicy&& __exec, _ForwardIterator __first, _Size __n)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    typedef typename iterator_traits<_ForwardIterator>::reference _ReferenceType;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-
-    using __is_vector = typename decltype(__dispatch_tag)::__is_vector;
-
-    return __pstl::__internal::__invoke_if_else(
-        std::is_trivial<_ValueType>(),
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk_brick_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ForwardIterator __begin, _Size __count)
-                { return __pstl::__internal::__brick_fill_n(__begin, __count, _ValueType(), __is_vector{}); });
-        },
-        [&]()
-        {
-            return __pstl::__internal::__pattern_walk1_n(
-                __dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __n,
-                [](_ReferenceType __val) { ::new (std::addressof(__val)) _ValueType(); });
-        });
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_MEMORY_IMPL_H */
diff --git a/pstl/include/pstl/internal/glue_numeric_defs.h b/pstl/include/pstl/internal/glue_numeric_defs.h
deleted file mode 100644
index 86cd38b34b13..000000000000
--- a/pstl/include/pstl/internal/glue_numeric_defs.h
+++ /dev/null
@@ -1,124 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_NUMERIC_DEFS_H
-#define _PSTL_GLUE_NUMERIC_DEFS_H
-
-#include <iterator>
-
-#include "execution_defs.h"
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-// [reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-       _BinaryOperation __binary_op);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::value_type>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                 _BinaryOperation2 __binary_op2);
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                 _BinaryOperation __binary_op, _UnaryOperation __unary_op);
-
-// [exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op);
-
-// [inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op, _Tp __init);
-
-// [transform.exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op);
-
-// [transform.inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation,
-          class _UnaryOperation, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         _Tp __init);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op);
-
-// [adjacent.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first, _BinaryOperation op);
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first);
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_NUMERIC_DEFS_H */
diff --git a/pstl/include/pstl/internal/glue_numeric_impl.h b/pstl/include/pstl/internal/glue_numeric_impl.h
deleted file mode 100644
index ad268b51a16c..000000000000
--- a/pstl/include/pstl/internal/glue_numeric_impl.h
+++ /dev/null
@@ -1,232 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_GLUE_NUMERIC_IMPL_H
-#define _PSTL_GLUE_NUMERIC_IMPL_H
-
-#include <functional>
-
-#include "pstl_config.h"
-
-#include "utils.h"
-#include "numeric_fwd.h"
-#include "execution_impl.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace std
-{
-
-// [reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-       _BinaryOperation __binary_op)
-{
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, __binary_op,
-                            __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init)
-{
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, __init, std::plus<_Tp>(),
-                            __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy,
-                                                 typename iterator_traits<_ForwardIterator>::value_type>
-reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last)
-{
-    typedef typename iterator_traits<_ForwardIterator>::value_type _ValueType;
-    return transform_reduce(std::forward<_ExecutionPolicy>(__exec), __first, __last, _ValueType{},
-                            std::plus<_ValueType>(), __pstl::__internal::__no_op());
-}
-
-// [transform.reduce]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __init, std::plus<_InputType>(),
-                                                          std::multiplies<_InputType>());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                 _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first1, __first2);
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first1, __last1, __first2, __init, __binary_op1,
-                                                          __binary_op2);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _Tp>
-transform_reduce(_ExecutionPolicy&& __exec, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                 _BinaryOperation __binary_op, _UnaryOperation __unary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first);
-    return __pstl::__internal::__pattern_transform_reduce(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                          __first, __last, __init, __binary_op, __unary_op);
-}
-
-// [exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using namespace __pstl;
-    return __internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __result, __pstl::__internal::__no_op(), __init, std::plus<_Tp>(),
-                                                /*inclusive=*/std::false_type());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    using namespace __pstl;
-    return __internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                                __result, __pstl::__internal::__no_op(), __init, __binary_op,
-                                                /*inclusive=*/std::false_type());
-}
-
-// [inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _InputType;
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result,
-                                    std::plus<_InputType>(), __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op)
-{
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op,
-                                    __pstl::__internal::__no_op());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-               _ForwardIterator2 __result, _BinaryOperation __binary_op, _Tp __init)
-{
-    return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), __first, __last, __result, __binary_op,
-                                    __pstl::__internal::__no_op(), __init);
-}
-
-// [transform.exclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_exclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __result, __unary_op, __init, __binary_op,
-                                                        /*inclusive=*/std::false_type());
-}
-
-// [transform.inclusive.scan]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation,
-          class _UnaryOperation, class _Tp>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         _Tp __init)
-{
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __result);
-
-    return __pstl::__internal::__pattern_transform_scan(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec), __first,
-                                                        __last, __result, __unary_op, __init, __binary_op,
-                                                        /*inclusive=*/std::true_type());
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _UnaryOperation,
-          class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-transform_inclusive_scan(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                         _ForwardIterator2 __result, _BinaryOperation __binary_op, _UnaryOperation __unary_op)
-{
-    if (__first != __last)
-    {
-        auto __tmp = __unary_op(*__first);
-        *__result = __tmp;
-        return transform_inclusive_scan(std::forward<_ExecutionPolicy>(__exec), ++__first, __last, ++__result,
-                                        __binary_op, __unary_op, __tmp);
-    }
-    else
-    {
-        return __result;
-    }
-}
-
-// [adjacent.difference]
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _BinaryOperation>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first, _BinaryOperation __op)
-{
-
-    if (__first == __last)
-        return __d_first;
-
-    auto __dispatch_tag = __pstl::__internal::__select_backend(__exec, __first, __d_first);
-
-    return __pstl::__internal::__pattern_adjacent_difference(__dispatch_tag, std::forward<_ExecutionPolicy>(__exec),
-                                                             __first, __last, __d_first, __op);
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2>
-__pstl::__internal::__enable_if_execution_policy<_ExecutionPolicy, _ForwardIterator2>
-adjacent_difference(_ExecutionPolicy&& __exec, _ForwardIterator1 __first, _ForwardIterator1 __last,
-                    _ForwardIterator2 __d_first)
-{
-    typedef typename iterator_traits<_ForwardIterator1>::value_type _ValueType;
-    return adjacent_difference(std::forward<_ExecutionPolicy>(__exec), __first, __last, __d_first,
-                               std::minus<_ValueType>());
-}
-
-} // namespace std
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_GLUE_NUMERIC_IMPL_H_ */
diff --git a/pstl/include/pstl/internal/memory_impl.h b/pstl/include/pstl/internal/memory_impl.h
deleted file mode 100644
index 942a30ec1a65..000000000000
--- a/pstl/include/pstl/internal/memory_impl.h
+++ /dev/null
@@ -1,112 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_MEMORY_IMPL_H
-#define _PSTL_MEMORY_IMPL_H
-
-#include <iterator>
-
-#include "pstl_config.h"
-#include "unseq_backend_simd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// uninitialized_move
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_move(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                           /*vector=*/std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
-    {
-        ::new (std::addressof(*__result)) _ValueType(std::move(*__first));
-    }
-    return __result;
-}
-
-template <typename _RandomAccessIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_move(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                           /*vector=*/std::true_type) noexcept
-{
-    using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
-    using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
-
-    return __unseq_backend::__simd_walk_2(
-        __first, __last - __first, __result,
-        [](_ReferenceType1 __x, _ReferenceType2 __y) { ::new (std::addressof(__y)) __ValueType(std::move(__x)); });
-}
-
-template <typename _Iterator>
-void
-__brick_destroy(_Iterator __first, _Iterator __last, /*vector*/ std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_Iterator>::value_type;
-
-    for (; __first != __last; ++__first)
-        __first->~_ValueType();
-}
-
-template <typename _RandomAccessIterator>
-void
-__brick_destroy(_RandomAccessIterator __first, _RandomAccessIterator __last, /*vector*/ std::true_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_RandomAccessIterator>::value_type;
-    using _ReferenceType = typename std::iterator_traits<_RandomAccessIterator>::reference;
-
-    __unseq_backend::__simd_walk_1(__first, __last - __first, [](_ReferenceType __x) { __x.~_ValueType(); });
-}
-
-//------------------------------------------------------------------------
-// uninitialized copy
-//------------------------------------------------------------------------
-
-template <typename _ForwardIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_copy(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                           /*vector=*/std::false_type) noexcept
-{
-    using _ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    for (; __first != __last; ++__first, ++__result)
-    {
-        ::new (std::addressof(*__result)) _ValueType(*__first);
-    }
-    return __result;
-}
-
-template <typename _RandomAccessIterator, typename _OutputIterator>
-_OutputIterator
-__brick_uninitialized_copy(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                           /*vector=*/std::true_type) noexcept
-{
-    using __ValueType = typename std::iterator_traits<_OutputIterator>::value_type;
-    using _ReferenceType1 = typename std::iterator_traits<_RandomAccessIterator>::reference;
-    using _ReferenceType2 = typename std::iterator_traits<_OutputIterator>::reference;
-
-    return __unseq_backend::__simd_walk_2(
-        __first, __last - __first, __result,
-        [](_ReferenceType1 __x, _ReferenceType2 __y) { ::new (std::addressof(__y)) __ValueType(__x); });
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_MEMORY_IMPL_H */
diff --git a/pstl/include/pstl/internal/numeric_fwd.h b/pstl/include/pstl/internal/numeric_fwd.h
deleted file mode 100644
index 7f7845bd3e26..000000000000
--- a/pstl/include/pstl/internal/numeric_fwd.h
+++ /dev/null
@@ -1,139 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_NUMERIC_FWD_H
-#define _PSTL_NUMERIC_FWD_H
-
-#include <type_traits>
-#include <utility>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// transform_reduce (version with two binary functions, according to draft N4659)
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-_Tp __brick_transform_reduce(_RandomAccessIterator1, _RandomAccessIterator1, _RandomAccessIterator2, _Tp,
-                             _BinaryOperation1, _BinaryOperation2,
-                             /*__is_vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp __brick_transform_reduce(_ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Tp, _BinaryOperation1,
-                             _BinaryOperation2,
-                             /*__is_vector=*/std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp,
-          class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator1, _ForwardIterator1, _ForwardIterator2, _Tp,
-                           _BinaryOperation1, _BinaryOperation2) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator1,
-                           _RandomAccessIterator1, _RandomAccessIterator2, _Tp, _BinaryOperation1, _BinaryOperation2);
-
-//------------------------------------------------------------------------
-// transform_reduce (version with unary and binary functions)
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
-_Tp __brick_transform_reduce(_RandomAccessIterator, _RandomAccessIterator, _Tp, _BinaryOperation, _UnaryOperation,
-                             /*is_vector=*/std::true_type) noexcept;
-
-template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-_Tp __brick_transform_reduce(_ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation, _UnaryOperation,
-                             /*is_vector=*/std::false_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _Tp, _BinaryOperation,
-                           _UnaryOperation) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                           _Tp, _BinaryOperation, _UnaryOperation);
-
-//------------------------------------------------------------------------
-// transform_exclusive_scan
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp> __brick_transform_scan(_ForwardIterator, _ForwardIterator, _OutputIterator,
-                                                       _UnaryOperation, _Tp, _BinaryOperation,
-                                                       /*Inclusive*/ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp> __brick_transform_scan(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                                       _UnaryOperation, _Tp, _BinaryOperation,
-                                                       /*Inclusive*/ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryOperation,
-          class _Tp, class _BinaryOperation, class _Inclusive>
-_OutputIterator
-__pattern_transform_scan(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator, _UnaryOperation,
-                         _Tp, _BinaryOperation, _Inclusive) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&&, _RandomAccessIterator,
-                         _RandomAccessIterator, _OutputIterator, _UnaryOperation, _Tp, _BinaryOperation, _Inclusive);
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator, _RandomAccessIterator,
-                         _OutputIterator, _UnaryOperation, _Tp, _BinaryOperation, _Inclusive);
-
-//------------------------------------------------------------------------
-// adjacent_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator __brick_adjacent_difference(_ForwardIterator, _ForwardIterator, _OutputIterator, _BinaryOperation,
-                                            /*is_vector*/ std::false_type) noexcept;
-
-template <class _RandomAccessIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator __brick_adjacent_difference(_RandomAccessIterator, _RandomAccessIterator, _OutputIterator,
-                                            _BinaryOperation,
-                                            /*is_vector*/ std::true_type) noexcept;
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator, _ForwardIterator, _OutputIterator,
-                              _BinaryOperation) noexcept;
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(__parallel_tag<_IsVector>, _ExecutionPolicy&&, _RandomAccessIterator,
-                              _RandomAccessIterator, _OutputIterator, _BinaryOperation);
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_NUMERIC_FWD_H */
diff --git a/pstl/include/pstl/internal/numeric_impl.h b/pstl/include/pstl/internal/numeric_impl.h
deleted file mode 100644
index a0387aef50b0..000000000000
--- a/pstl/include/pstl/internal/numeric_impl.h
+++ /dev/null
@@ -1,383 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_NUMERIC_IMPL_H
-#define _PSTL_NUMERIC_IMPL_H
-
-#include <iterator>
-#include <type_traits>
-#include <numeric>
-
-#include "parallel_backend.h"
-#include "pstl_config.h"
-#include "execution_impl.h"
-#include "unseq_backend_simd.h"
-#include "algorithm_fwd.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// transform_reduce (version with two binary functions, according to draft N4659)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__brick_transform_reduce(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2, _Tp __init,
-                         _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2,
-                         /*is_vector=*/std::false_type) noexcept
-{
-    return std::inner_product(__first1, __last1, __first2, __init, __binary_op1, __binary_op2);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _Tp, class _BinaryOperation1,
-          class _BinaryOperation2>
-_Tp
-__brick_transform_reduce(_RandomAccessIterator1 __first1, _RandomAccessIterator1 __last1,
-                         _RandomAccessIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                         _BinaryOperation2 __binary_op2,
-                         /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType;
-    return __unseq_backend::__simd_transform_reduce(
-        __last1 - __first1, __init, __binary_op1,
-        [=, &__binary_op2](_DifferenceType __i) { return __binary_op2(__first1[__i], __first2[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator1, class _ForwardIterator2, class _Tp,
-          class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator1 __first1, _ForwardIterator1 __last1,
-                           _ForwardIterator2 __first2, _Tp __init, _BinaryOperation1 __binary_op1,
-                           _BinaryOperation2 __binary_op2) noexcept
-{
-    return __brick_transform_reduce(__first1, __last1, __first2, __init, __binary_op1, __binary_op2,
-                                    typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _Tp, class _BinaryOperation1, class _BinaryOperation2>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator1 __first1,
-                           _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _Tp __init,
-                           _BinaryOperation1 __binary_op1, _BinaryOperation2 __binary_op2)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_transform_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first1, __last1,
-                [__first1, __first2, __binary_op2](_RandomAccessIterator1 __i) mutable
-                { return __binary_op2(*__i, *(__first2 + (__i - __first1))); },
-                __init,
-                __binary_op1, // Combine
-                [__first1, __first2, __binary_op1, __binary_op2](_RandomAccessIterator1 __i, _RandomAccessIterator1 __j,
-                                                                 _Tp __init) -> _Tp
-                {
-                    return __internal::__brick_transform_reduce(__i, __j, __first2 + (__i - __first1), __init,
-                                                                __binary_op1, __binary_op2, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// transform_reduce (version with unary and binary functions)
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _Tp, class _BinaryOperation, class _UnaryOperation>
-_Tp
-__brick_transform_reduce(_ForwardIterator __first, _ForwardIterator __last, _Tp __init, _BinaryOperation __binary_op,
-                         _UnaryOperation __unary_op, /*is_vector=*/std::false_type) noexcept
-{
-    return std::transform_reduce(__first, __last, __init, __binary_op, __unary_op);
-}
-
-template <class _RandomAccessIterator, class _Tp, class _UnaryOperation, class _BinaryOperation>
-_Tp
-__brick_transform_reduce(_RandomAccessIterator __first, _RandomAccessIterator __last, _Tp __init,
-                         _BinaryOperation __binary_op, _UnaryOperation __unary_op,
-                         /*is_vector=*/std::true_type) noexcept
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-    return __unseq_backend::__simd_transform_reduce(
-        __last - __first, __init, __binary_op,
-        [=, &__unary_op](_DifferenceType __i) { return __unary_op(__first[__i]); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Tp __init,
-                           _BinaryOperation __binary_op, _UnaryOperation __unary_op) noexcept
-{
-    return __internal::__brick_transform_reduce(__first, __last, __init, __binary_op, __unary_op,
-                                                typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _Tp, class _BinaryOperation,
-          class _UnaryOperation>
-_Tp
-__pattern_transform_reduce(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                           _RandomAccessIterator __last, _Tp __init, _BinaryOperation __binary_op,
-                           _UnaryOperation __unary_op)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            return __par_backend::__parallel_transform_reduce(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                [__unary_op](_RandomAccessIterator __i) mutable { return __unary_op(*__i); }, __init, __binary_op,
-                [__unary_op, __binary_op](_RandomAccessIterator __i, _RandomAccessIterator __j, _Tp __init) {
-                    return __internal::__brick_transform_reduce(__i, __j, __init, __binary_op, __unary_op, _IsVector{});
-                });
-        });
-}
-
-//------------------------------------------------------------------------
-// transform_exclusive_scan
-//
-// walk3 evaluates f(x,y,z) for (x,y,z) drawn from [first1,last1), [first2,...), [first3,...)
-//------------------------------------------------------------------------
-
-// Exclusive form
-template <class _ForwardIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp>
-__brick_transform_scan(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                       /*Inclusive*/ std::false_type, /*is_vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__result)
-    {
-        *__result = __init;
-        _PSTL_PRAGMA_FORCEINLINE
-        __init = __binary_op(__init, __unary_op(*__first));
-    }
-    return std::make_pair(__result, __init);
-}
-
-// Inclusive form
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation>
-std::pair<_OutputIterator, _Tp>
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                       /*Inclusive*/ std::true_type, /*is_vector=*/std::false_type) noexcept
-{
-    for (; __first != __last; ++__first, ++__result)
-    {
-        _PSTL_PRAGMA_FORCEINLINE
-        __init = __binary_op(__init, __unary_op(*__first));
-        *__result = __init;
-    }
-    return std::make_pair(__result, __init);
-}
-
-// type is arithmetic and binary operation is a user defined operation.
-template <typename _Tp, typename _BinaryOperation>
-using is_arithmetic_udop = std::integral_constant<bool, std::is_arithmetic<_Tp>::value &&
-                                                            !std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
-
-// [restriction] - T shall be DefaultConstructible.
-// [violation] - default ctor of T shall set the identity value for binary_op.
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation,
-          class _Inclusive>
-typename std::enable_if<!is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op, _Inclusive,
-                       /*is_vector=*/std::true_type) noexcept
-{
-#if defined(_PSTL_UDS_PRESENT)
-    return __unseq_backend::__simd_scan(__first, __last - __first, __result, __unary_op, __init, __binary_op,
-                                        _Inclusive());
-#else
-    // We need to call serial brick here to call function for inclusive and exclusive scan that depends on _Inclusive() value
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              /*is_vector=*/std::false_type());
-#endif
-}
-
-template <class _RandomAccessIterator, class _OutputIterator, class _UnaryOperation, class _Tp, class _BinaryOperation,
-          class _Inclusive>
-typename std::enable_if<is_arithmetic_udop<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__brick_transform_scan(_RandomAccessIterator __first, _RandomAccessIterator __last, _OutputIterator __result,
-                       _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op, _Inclusive,
-                       /*is_vector=*/std::true_type) noexcept
-{
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              /*is_vector=*/std::false_type());
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _UnaryOperation,
-          class _Tp, class _BinaryOperation, class _Inclusive>
-_OutputIterator
-__pattern_transform_scan(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                         _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init, _BinaryOperation __binary_op,
-                         _Inclusive) noexcept
-{
-    return __internal::__brick_transform_scan(__first, __last, __result, __unary_op, __init, __binary_op, _Inclusive(),
-                                              typename _Tag::__is_vector{})
-        .first;
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<!std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-                         _BinaryOperation __binary_op, _Inclusive)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_transform_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __last - __first,
-                [__first, __unary_op](_DifferenceType __i) mutable { return __unary_op(__first[__i]); }, __init,
-                __binary_op,
-                [__first, __unary_op, __binary_op](_DifferenceType __i, _DifferenceType __j, _Tp __init)
-                {
-                    // Execute serial __brick_transform_reduce, due to the explicit SIMD vectorization (reduction) requires a commutative operation for the guarantee of correct scan.
-                    return __internal::__brick_transform_reduce(__first + __i, __first + __j, __init, __binary_op,
-                                                                __unary_op,
-                                                                /*__is_vector*/ std::false_type());
-                },
-                [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __j, _Tp __init)
-                {
-                    return __internal::__brick_transform_scan(__first + __i, __first + __j, __result + __i, __unary_op,
-                                                              __init, __binary_op, _Inclusive(), _IsVector{})
-                        .second;
-                });
-            return __result + (__last - __first);
-        });
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator, class _OutputIterator,
-          class _UnaryOperation, class _Tp, class _BinaryOperation, class _Inclusive>
-typename std::enable_if<std::is_floating_point<_Tp>::value, _OutputIterator>::type
-__pattern_transform_scan(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec, _RandomAccessIterator __first,
-                         _RandomAccessIterator __last, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-                         _BinaryOperation __binary_op, _Inclusive)
-{
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-    _DifferenceType __n = __last - __first;
-
-    if (__n <= 0)
-    {
-        return __result;
-    }
-    return __internal::__except_handler(
-        [&]()
-        {
-            __par_backend::__parallel_strict_scan(
-                __backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __n, __init,
-                [__first, __unary_op, __binary_op, __result](_DifferenceType __i, _DifferenceType __len)
-                {
-                    return __internal::__brick_transform_scan(__first + __i, __first + (__i + __len), __result + __i,
-                                                              __unary_op, _Tp{}, __binary_op, _Inclusive(), _IsVector{})
-                        .second;
-                },
-                __binary_op,
-                [__result, &__binary_op](_DifferenceType __i, _DifferenceType __len, _Tp __initial)
-                {
-                    return *(std::transform(__result + __i, __result + __i + __len, __result + __i,
-                                            [&__initial, &__binary_op](const _Tp& __x)
-                                            {
-                                                _PSTL_PRAGMA_FORCEINLINE
-                                                return __binary_op(__initial, __x);
-                                            }) -
-                             1);
-                },
-                [](_Tp) {});
-            return __result + (__last - __first);
-        });
-}
-
-//------------------------------------------------------------------------
-// adjacent_difference
-//------------------------------------------------------------------------
-
-template <class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__brick_adjacent_difference(_ForwardIterator __first, _ForwardIterator __last, _OutputIterator __d_first,
-                            _BinaryOperation __op, /*is_vector*/ std::false_type) noexcept
-{
-    return std::adjacent_difference(__first, __last, __d_first, __op);
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class BinaryOperation>
-_RandomAccessIterator2
-__brick_adjacent_difference(_RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                            _RandomAccessIterator2 __d_first, BinaryOperation __op,
-                            /*is_vector=*/std::true_type) noexcept
-{
-    _PSTL_ASSERT(__first != __last);
-
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    auto __n = __last - __first;
-    *__d_first = *__first;
-    return __unseq_backend::__simd_walk_3(
-        __first + 1, __n - 1, __first, __d_first + 1,
-        [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z) { __z = __op(__x, __y); });
-}
-
-template <class _Tag, class _ExecutionPolicy, class _ForwardIterator, class _OutputIterator, class _BinaryOperation>
-_OutputIterator
-__pattern_adjacent_difference(_Tag, _ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last,
-                              _OutputIterator __d_first, _BinaryOperation __op) noexcept
-{
-    return __internal::__brick_adjacent_difference(__first, __last, __d_first, __op, typename _Tag::__is_vector{});
-}
-
-template <class _IsVector, class _ExecutionPolicy, class _RandomAccessIterator1, class _RandomAccessIterator2,
-          class _BinaryOperation>
-_RandomAccessIterator2
-__pattern_adjacent_difference(__parallel_tag<_IsVector> __tag, _ExecutionPolicy&& __exec,
-                              _RandomAccessIterator1 __first, _RandomAccessIterator1 __last,
-                              _RandomAccessIterator2 __d_first, _BinaryOperation __op)
-{
-    _PSTL_ASSERT(__first != __last);
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::reference _ReferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::reference _ReferenceType2;
-
-    using __backend_tag = typename decltype(__tag)::__backend_tag;
-
-    *__d_first = *__first;
-    __par_backend::__parallel_for(__backend_tag{}, std::forward<_ExecutionPolicy>(__exec), __first, __last - 1,
-                                  [&__op, __d_first, __first](_RandomAccessIterator1 __b, _RandomAccessIterator1 __e)
-                                  {
-                                      _RandomAccessIterator2 __d_b = __d_first + (__b - __first);
-                                      __internal::__brick_walk3(
-                                          __b, __e, __b + 1, __d_b + 1,
-                                          [&__op](_ReferenceType1 __x, _ReferenceType1 __y, _ReferenceType2 __z)
-                                          { __z = __op(__y, __x); },
-                                          _IsVector{});
-                                  });
-    return __d_first + (__last - __first);
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_NUMERIC_IMPL_H */
diff --git a/pstl/include/pstl/internal/omp/parallel_for.h b/pstl/include/pstl/internal/omp/parallel_for.h
deleted file mode 100644
index 0f841e531181..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_for.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_FOR_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_FOR_H
-
-#include <cstddef>
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <class _Index, class _Fp>
-void
-__parallel_for_body(_Index __first, _Index __last, _Fp __f)
-{
-    // initial partition of the iteration space into chunks
-    auto __policy = __omp_backend::__chunk_partitioner(__first, __last);
-
-    // To avoid over-subscription we use taskloop for the nested parallelism
-    _PSTL_PRAGMA(omp taskloop untied mergeable)
-    for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
-    {
-        __pstl::__omp_backend::__process_chunk(__policy, __first, __chunk, __f);
-    }
-}
-
-//------------------------------------------------------------------------
-// Notation:
-// Evaluation of brick f[i,j) for each subrange [i,j) of [first, last)
-//------------------------------------------------------------------------
-
-template <class _ExecutionPolicy, class _Index, class _Fp>
-void
-__parallel_for(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last, _Fp __f)
-{
-    if (omp_in_parallel())
-    {
-        // we don't create a nested parallel region in an existing parallel
-        // region: just create tasks
-        __pstl::__omp_backend::__parallel_for_body(__first, __last, __f);
-    }
-    else
-    {
-        // in any case (nested or non-nested) one parallel region is created and
-        // only one thread creates a set of tasks
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait) { __pstl::__omp_backend::__parallel_for_body(__first, __last, __f); }
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_FOR_H
diff --git a/pstl/include/pstl/internal/omp/parallel_for_each.h b/pstl/include/pstl/internal/omp/parallel_for_each.h
deleted file mode 100644
index b9bfb05930a1..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_for_each.h
+++ /dev/null
@@ -1,59 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <class _ForwardIterator, class _Fp>
-void
-__parallel_for_each_body(_ForwardIterator __first, _ForwardIterator __last, _Fp __f)
-{
-    using DifferenceType = typename std::iterator_traits<_ForwardIterator>::difference_type;
-    // TODO: Think of an approach to remove the std::distance call
-    auto __size = std::distance(__first, __last);
-
-    _PSTL_PRAGMA(omp taskloop untied mergeable)
-    for (DifferenceType __index = 0; __index < __size; ++__index)
-    {
-        // TODO: Think of an approach to remove the increment here each time.
-        auto __iter = std::next(__first, __index);
-        __f(*__iter);
-    }
-}
-
-template <class _ExecutionPolicy, class _ForwardIterator, class _Fp>
-void
-__parallel_for_each(_ExecutionPolicy&&, _ForwardIterator __first, _ForwardIterator __last, _Fp __f)
-{
-    if (omp_in_parallel())
-    {
-        // we don't create a nested parallel region in an existing parallel
-        // region: just create tasks
-        __pstl::__omp_backend::__parallel_for_each_body(__first, __last, __f);
-    }
-    else
-    {
-        // in any case (nested or non-nested) one parallel region is created and
-        // only one thread creates a set of tasks
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait) { __pstl::__omp_backend::__parallel_for_each_body(__first, __last, __f); }
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_FOR_EACH_H
diff --git a/pstl/include/pstl/internal/omp/parallel_invoke.h b/pstl/include/pstl/internal/omp/parallel_invoke.h
deleted file mode 100644
index 045ccbe9a18c..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_invoke.h
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <typename _F1, typename _F2>
-void
-__parallel_invoke_body(_F1&& __f1, _F2&& __f2)
-{
-    _PSTL_PRAGMA(omp taskgroup)
-    {
-        _PSTL_PRAGMA(omp task untied mergeable) { std::forward<_F1>(__f1)(); }
-        _PSTL_PRAGMA(omp task untied mergeable) { std::forward<_F2>(__f2)(); }
-    }
-}
-
-template <class _ExecutionPolicy, typename _F1, typename _F2>
-void
-__parallel_invoke(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
-{
-    if (omp_in_parallel())
-    {
-        __pstl::__omp_backend::__parallel_invoke_body(std::forward<_F1>(__f1), std::forward<_F2>(__f2));
-    }
-    else
-    {
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait)
-        __pstl::__omp_backend::__parallel_invoke_body(std::forward<_F1>(__f1), std::forward<_F2>(__f2));
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_INVOKE_H
diff --git a/pstl/include/pstl/internal/omp/parallel_merge.h b/pstl/include/pstl/internal/omp/parallel_merge.h
deleted file mode 100644
index e6f82c5e1866..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_merge.h
+++ /dev/null
@@ -1,98 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _RandomAccessIterator3,
-          typename _Compare, typename _LeafMerge>
-void
-__parallel_merge_body(std::size_t __size_x, std::size_t __size_y, _RandomAccessIterator1 __xs,
-                      _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys, _RandomAccessIterator2 __ye,
-                      _RandomAccessIterator3 __zs, _Compare __comp, _LeafMerge __leaf_merge)
-{
-
-    if (__size_x + __size_y <= __omp_backend::__default_chunk_size)
-    {
-        __leaf_merge(__xs, __xe, __ys, __ye, __zs, __comp);
-        return;
-    }
-
-    _RandomAccessIterator1 __xm;
-    _RandomAccessIterator2 __ym;
-
-    if (__size_x < __size_y)
-    {
-        __ym = __ys + (__size_y / 2);
-        __xm = std::upper_bound(__xs, __xe, *__ym, __comp);
-    }
-    else
-    {
-        __xm = __xs + (__size_x / 2);
-        __ym = std::lower_bound(__ys, __ye, *__xm, __comp);
-    }
-
-    auto __zm = __zs + (__xm - __xs) + (__ym - __ys);
-
-    _PSTL_PRAGMA(omp task untied mergeable default(none)
-                     firstprivate(__xs, __xm, __ys, __ym, __zs, __comp, __leaf_merge))
-    __pstl::__omp_backend::__parallel_merge_body(__xm - __xs, __ym - __ys, __xs, __xm, __ys, __ym, __zs, __comp,
-                                                      __leaf_merge);
-
-    _PSTL_PRAGMA(omp task untied mergeable default(none)
-                     firstprivate(__xm, __xe, __ym, __ye, __zm, __comp, __leaf_merge))
-    __pstl::__omp_backend::__parallel_merge_body(__xe - __xm, __ye - __ym, __xm, __xe, __ym, __ye, __zm, __comp,
-                                                      __leaf_merge);
-
-    _PSTL_PRAGMA(omp taskwait)
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator1, typename _RandomAccessIterator2,
-          typename _RandomAccessIterator3, typename _Compare, typename _LeafMerge>
-void
-__parallel_merge(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&& /*__exec*/, _RandomAccessIterator1 __xs,
-                 _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys, _RandomAccessIterator2 __ye,
-                 _RandomAccessIterator3 __zs, _Compare __comp, _LeafMerge __leaf_merge)
-
-{
-    std::size_t __size_x = __xe - __xs;
-    std::size_t __size_y = __ye - __ys;
-
-    /*
-     * Run the merge in parallel by chunking it up. Use the smaller range (if any) as the iteration range, and the
-     * larger range as the search range.
-     */
-
-    if (omp_in_parallel())
-    {
-        __pstl::__omp_backend::__parallel_merge_body(__size_x, __size_y, __xs, __xe, __ys, __ye, __zs, __comp,
-                                                          __leaf_merge);
-    }
-    else
-    {
-        _PSTL_PRAGMA(omp parallel)
-        {
-            _PSTL_PRAGMA(omp single nowait)
-            __pstl::__omp_backend::__parallel_merge_body(__size_x, __size_y, __xs, __xe, __ys, __ye, __zs, __comp,
-                                                              __leaf_merge);
-        }
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_MERGE_H
diff --git a/pstl/include/pstl/internal/omp/parallel_reduce.h b/pstl/include/pstl/internal/omp/parallel_reduce.h
deleted file mode 100644
index 841d48fc3051..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_reduce.h
+++ /dev/null
@@ -1,73 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <class _RandomAccessIterator, class _Value, typename _RealBody, typename _Reduction>
-_Value
-__parallel_reduce_body(_RandomAccessIterator __first, _RandomAccessIterator __last, _Value __identity,
-                       _RealBody __real_body, _Reduction __reduce)
-{
-    if (__should_run_serial(__first, __last))
-    {
-        return __real_body(__first, __last, __identity);
-    }
-
-    auto __middle = __first + ((__last - __first) / 2);
-    _Value __v1(__identity), __v2(__identity);
-    __parallel_invoke_body(
-        [&]() { __v1 = __parallel_reduce_body(__first, __middle, __identity, __real_body, __reduce); },
-        [&]() { __v2 = __parallel_reduce_body(__middle, __last, __identity, __real_body, __reduce); });
-
-    return __reduce(__v1, __v2);
-}
-
-//------------------------------------------------------------------------
-// Notation:
-//      r(i,j,init) returns reduction of init with reduction over [i,j)
-//      c(x,y) combines values x and y that were the result of r
-//------------------------------------------------------------------------
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _Value, typename _RealBody, typename _Reduction>
-_Value
-__parallel_reduce(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
-                  _RandomAccessIterator __last, _Value __identity, _RealBody __real_body, _Reduction __reduction)
-{
-    // We don't create a nested parallel region in an existing parallel region:
-    // just create tasks.
-    if (omp_in_parallel())
-    {
-        return __pstl::__omp_backend::__parallel_reduce_body(__first, __last, __identity, __real_body, __reduction);
-    }
-
-    // In any case (nested or non-nested) one parallel region is created and only
-    // one thread creates a set of tasks.
-    _Value __res = __identity;
-
-    _PSTL_PRAGMA(omp parallel)
-    _PSTL_PRAGMA(omp single nowait)
-    {
-        __res = __pstl::__omp_backend::__parallel_reduce_body(__first, __last, __identity, __real_body, __reduction);
-    }
-
-    return __res;
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_REDUCE_H
diff --git a/pstl/include/pstl/internal/omp/parallel_scan.h b/pstl/include/pstl/internal/omp/parallel_scan.h
deleted file mode 100644
index f3eb967ca4fb..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_scan.h
+++ /dev/null
@@ -1,136 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H
-
-#include "parallel_invoke.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <typename _Index>
-_Index
-__split(_Index __m)
-{
-    _Index __k = 1;
-    while (2 * __k < __m)
-        __k *= 2;
-    return __k;
-}
-
-template <typename _Index, typename _Tp, typename _Rp, typename _Cp>
-void
-__upsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Rp __reduce, _Cp __combine)
-{
-    if (__m == 1)
-        __r[0] = __reduce(__i * __tilesize, __lastsize);
-    else
-    {
-        _Index __k = __split(__m);
-        __omp_backend::__parallel_invoke_body(
-            [=] { __omp_backend::__upsweep(__i, __k, __tilesize, __r, __tilesize, __reduce, __combine); },
-            [=] {
-                __omp_backend::__upsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, __reduce, __combine);
-            });
-        if (__m == 2 * __k)
-            __r[__m - 1] = __combine(__r[__k - 1], __r[__m - 1]);
-    }
-}
-
-template <typename _Index, typename _Tp, typename _Cp, typename _Sp>
-void
-__downsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Tp __initial, _Cp __combine,
-            _Sp __scan)
-{
-    if (__m == 1)
-        __scan(__i * __tilesize, __lastsize, __initial);
-    else
-    {
-        const _Index __k = __split(__m);
-        __omp_backend::__parallel_invoke_body(
-            [=] { __omp_backend::__downsweep(__i, __k, __tilesize, __r, __tilesize, __initial, __combine, __scan); },
-            // Assumes that __combine never throws.
-            // TODO: Consider adding a requirement for user functors to be constant.
-            [=, &__combine]
-            {
-                __omp_backend::__downsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize,
-                                           __combine(__initial, __r[__k - 1]), __combine, __scan);
-            });
-    }
-}
-
-template <typename _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp,
-          typename _Ap>
-void
-__parallel_strict_scan_body(_Index __n, _Tp __initial, _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
-{
-    _Index __p = omp_get_num_threads();
-    const _Index __slack = 4;
-    _Index __tilesize = (__n - 1) / (__slack * __p) + 1;
-    _Index __m = (__n - 1) / __tilesize;
-    __buffer<_Tp> __buf(__m + 1);
-    _Tp* __r = __buf.get();
-
-    __omp_backend::__upsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __reduce, __combine);
-
-    std::size_t __k = __m + 1;
-    _Tp __t = __r[__k - 1];
-    while ((__k &= __k - 1))
-    {
-        __t = __combine(__r[__k - 1], __t);
-    }
-
-    __apex(__combine(__initial, __t));
-    __omp_backend::__downsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __initial,
-                               __combine, __scan);
-}
-
-template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
-void
-__parallel_strict_scan(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
-                       _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
-{
-    if (__n <= __default_chunk_size)
-    {
-        _Tp __sum = __initial;
-        if (__n)
-        {
-            __sum = __combine(__sum, __reduce(_Index(0), __n));
-        }
-        __apex(__sum);
-        if (__n)
-        {
-            __scan(_Index(0), __n, __initial);
-        }
-        return;
-    }
-
-    if (omp_in_parallel())
-    {
-        __pstl::__omp_backend::__parallel_strict_scan_body<_ExecutionPolicy>(__n, __initial, __reduce, __combine,
-                                                                             __scan, __apex);
-    }
-    else
-    {
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait)
-        {
-            __pstl::__omp_backend::__parallel_strict_scan_body<_ExecutionPolicy>(__n, __initial, __reduce, __combine,
-                                                                                 __scan, __apex);
-        }
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_SCAN_H
diff --git a/pstl/include/pstl/internal/omp/parallel_stable_partial_sort.h b/pstl/include/pstl/internal/omp/parallel_stable_partial_sort.h
deleted file mode 100644
index 06cd55bb4899..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_stable_partial_sort.h
+++ /dev/null
@@ -1,33 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
-__parallel_stable_partial_sort(__pstl::__internal::__openmp_backend_tag, _RandomAccessIterator __xs,
-                               _RandomAccessIterator __xe, _Compare __comp, _LeafSort __leaf_sort,
-                               std::size_t /* __nsort */)
-{
-    // TODO: "Parallel partial sort needs to be implemented.");
-    __leaf_sort(__xs, __xe, __comp);
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_STABLE_PARTIAL_SORT_H
diff --git a/pstl/include/pstl/internal/omp/parallel_stable_sort.h b/pstl/include/pstl/internal/omp/parallel_stable_sort.h
deleted file mode 100644
index e4d067662726..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_stable_sort.h
+++ /dev/null
@@ -1,160 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H
-
-#include "util.h"
-#include "parallel_merge.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-namespace __sort_details
-{
-struct __move_value
-{
-    template <typename _Iterator, typename _OutputIterator>
-    void
-    operator()(_Iterator __x, _OutputIterator __z) const
-    {
-        *__z = std::move(*__x);
-    }
-};
-
-template <typename _RandomAccessIterator, typename _OutputIterator>
-_OutputIterator
-__parallel_move_range(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _OutputIterator __d_first)
-{
-    std::size_t __size = __last1 - __first1;
-
-    // Perform serial moving of small chunks
-
-    if (__size <= __default_chunk_size)
-    {
-        return std::move(__first1, __last1, __d_first);
-    }
-
-    // Perform parallel moving of larger chunks
-    auto __policy = __pstl::__omp_backend::__chunk_partitioner(__first1, __last1);
-
-    _PSTL_PRAGMA(omp taskloop)
-    for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
-    {
-        __pstl::__omp_backend::__process_chunk(__policy, __first1, __chunk,
-                                       [&](auto __chunk_first, auto __chunk_last)
-                                       {
-                                           auto __chunk_offset = __chunk_first - __first1;
-                                           auto __output_it = __d_first + __chunk_offset;
-                                           std::move(__chunk_first, __chunk_last, __output_it);
-                                       });
-    }
-
-    return __d_first + __size;
-}
-
-struct __move_range
-{
-    template <typename _RandomAccessIterator, typename _OutputIterator>
-    _OutputIterator
-    operator()(_RandomAccessIterator __first1, _RandomAccessIterator __last1, _OutputIterator __d_first) const
-    {
-        return __pstl::__omp_backend::__sort_details::__parallel_move_range(__first1, __last1, __d_first);
-    }
-};
-} // namespace __sort_details
-
-template <typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
-__parallel_stable_sort_body(_RandomAccessIterator __xs, _RandomAccessIterator __xe, _Compare __comp,
-                            _LeafSort __leaf_sort)
-{
-    using _ValueType = typename std::iterator_traits<_RandomAccessIterator>::value_type;
-    using _VecType = typename std::vector<_ValueType>;
-    using _OutputIterator = typename _VecType::iterator;
-    using _MoveValue = typename __omp_backend::__sort_details::__move_value;
-    using _MoveRange = __omp_backend::__sort_details::__move_range;
-
-    if (__should_run_serial(__xs, __xe))
-    {
-        __leaf_sort(__xs, __xe, __comp);
-    }
-    else
-    {
-        std::size_t __size = __xe - __xs;
-        auto __mid = __xs + (__size / 2);
-        __pstl::__omp_backend::__parallel_invoke_body(
-            [&]() { __parallel_stable_sort_body(__xs, __mid, __comp, __leaf_sort); },
-            [&]() { __parallel_stable_sort_body(__mid, __xe, __comp, __leaf_sort); });
-
-        // Perform a parallel merge of the sorted ranges into __output_data.
-        _VecType __output_data(__size);
-        _MoveValue __move_value;
-        _MoveRange __move_range;
-        __utils::__serial_move_merge __merge(__size);
-        __pstl::__omp_backend::__parallel_merge_body(
-            __mid - __xs, __xe - __mid, __xs, __mid, __mid, __xe, __output_data.begin(), __comp,
-            [&__merge, &__move_value, &__move_range](_RandomAccessIterator __as, _RandomAccessIterator __ae,
-                                                     _RandomAccessIterator __bs, _RandomAccessIterator __be,
-                                                     _OutputIterator __cs, _Compare __comp)
-            { __merge(__as, __ae, __bs, __be, __cs, __comp, __move_value, __move_value, __move_range, __move_range); });
-
-        // Move the values from __output_data back in the original source range.
-        __pstl::__omp_backend::__sort_details::__parallel_move_range(__output_data.begin(), __output_data.end(), __xs);
-    }
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
-__parallel_stable_sort(__pstl::__internal::__openmp_backend_tag __tag, _ExecutionPolicy&& /*__exec*/,
-                       _RandomAccessIterator __xs, _RandomAccessIterator __xe, _Compare __comp, _LeafSort __leaf_sort,
-                       std::size_t __nsort = 0)
-{
-    auto __count = static_cast<std::size_t>(__xe - __xs);
-    if (__count <= __default_chunk_size || __nsort < __count)
-    {
-        __leaf_sort(__xs, __xe, __comp);
-        return;
-    }
-
-    // TODO: the partial sort implementation should
-    // be shared with the other backends.
-
-    if (omp_in_parallel())
-    {
-        if (__count <= __nsort)
-        {
-            __pstl::__omp_backend::__parallel_stable_sort_body(__xs, __xe, __comp, __leaf_sort);
-        }
-        else
-        {
-            __pstl::__omp_backend::__parallel_stable_partial_sort(__tag, __xs, __xe, __comp, __leaf_sort, __nsort);
-        }
-    }
-    else
-    {
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait)
-        if (__count <= __nsort)
-        {
-            __pstl::__omp_backend::__parallel_stable_sort_body(__xs, __xe, __comp, __leaf_sort);
-        }
-        else
-        {
-            __pstl::__omp_backend::__parallel_stable_partial_sort(__tag, __xs, __xe, __comp, __leaf_sort, __nsort);
-        }
-    }
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_STABLE_SORT_H
diff --git a/pstl/include/pstl/internal/omp/parallel_transform_reduce.h b/pstl/include/pstl/internal/omp/parallel_transform_reduce.h
deleted file mode 100644
index 1d4cc0a5974d..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_transform_reduce.h
+++ /dev/null
@@ -1,113 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-//------------------------------------------------------------------------
-// parallel_transform_reduce
-//
-// Notation:
-//      r(i,j,init) returns reduction of init with reduction over [i,j)
-//      u(i) returns f(i,i+1,identity) for a hypothetical left identity element
-//      of r c(x,y) combines values x and y that were the result of r or u
-//------------------------------------------------------------------------
-
-template <class _RandomAccessIterator, class _UnaryOp, class _Value, class _Combiner, class _Reduction>
-auto
-__transform_reduce_body(_RandomAccessIterator __first, _RandomAccessIterator __last, _UnaryOp __unary_op, _Value __init,
-                        _Combiner __combiner, _Reduction __reduction)
-{
-    const std::size_t __num_threads = omp_get_num_threads();
-    const std::size_t __size = __last - __first;
-
-    // Initial partition of the iteration space into chunks. If the range is too small,
-    // this will result in a nonsense policy, so we check on the size as well below.
-    auto __policy = __omp_backend::__chunk_partitioner(__first + __num_threads, __last);
-
-    if (__size <= __num_threads || __policy.__n_chunks < 2)
-    {
-        return __reduction(__first, __last, __init);
-    }
-
-    // Here, we cannot use OpenMP UDR because we must store the init value in
-    // the combiner and it will be used several times. Although there should be
-    // the only one; we manually generate the identity elements for each thread.
-    std::vector<_Value> __accums;
-    __accums.reserve(__num_threads);
-
-    // initialize accumulators for all threads
-    for (std::size_t __i = 0; __i < __num_threads; ++__i)
-    {
-        __accums.emplace_back(__unary_op(__first + __i));
-    }
-
-    // main loop
-    _PSTL_PRAGMA(omp taskloop shared(__accums))
-    for (std::size_t __chunk = 0; __chunk < __policy.__n_chunks; ++__chunk)
-    {
-        __pstl::__omp_backend::__process_chunk(__policy, __first + __num_threads, __chunk,
-                                       [&](auto __chunk_first, auto __chunk_last)
-                                       {
-                                           auto __thread_num = omp_get_thread_num();
-                                           __accums[__thread_num] =
-                                               __reduction(__chunk_first, __chunk_last, __accums[__thread_num]);
-                                       });
-    }
-
-    // combine by accumulators
-    for (std::size_t __i = 0; __i < __num_threads; ++__i)
-    {
-        __init = __combiner(__init, __accums[__i]);
-    }
-
-    return __init;
-}
-
-template <class _ExecutionPolicy, class _RandomAccessIterator, class _UnaryOp, class _Value, class _Combiner,
-          class _Reduction>
-_Value
-__parallel_transform_reduce(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
-                            _RandomAccessIterator __last, _UnaryOp __unary_op, _Value __init, _Combiner __combiner,
-                            _Reduction __reduction)
-{
-    _Value __result = __init;
-    if (omp_in_parallel())
-    {
-        // We don't create a nested parallel region in an existing parallel
-        // region: just create tasks
-        __result = __pstl::__omp_backend::__transform_reduce_body(__first, __last, __unary_op, __init, __combiner,
-                                                                  __reduction);
-    }
-    else
-    {
-        // Create a parallel region, and a single thread will create tasks
-        // for the region.
-        _PSTL_PRAGMA(omp parallel)
-        _PSTL_PRAGMA(omp single nowait)
-        {
-            __result = __pstl::__omp_backend::__transform_reduce_body(__first, __last, __unary_op, __init, __combiner,
-                                                                      __reduction);
-        }
-    }
-
-    return __result;
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_REDUCE_H
diff --git a/pstl/include/pstl/internal/omp/parallel_transform_scan.h b/pstl/include/pstl/internal/omp/parallel_transform_scan.h
deleted file mode 100644
index f83628994bc2..000000000000
--- a/pstl/include/pstl/internal/omp/parallel_transform_scan.h
+++ /dev/null
@@ -1,32 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H
-#define _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H
-
-#include "util.h"
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-template <class _ExecutionPolicy, class _Index, class _Up, class _Tp, class _Cp, class _Rp, class _Sp>
-_Tp
-__parallel_transform_scan(__pstl::__internal::__openmp_backend_tag, _ExecutionPolicy&&, _Index __n, _Up /* __u */,
-                          _Tp __init, _Cp /* __combine */, _Rp /* __brick_reduce */, _Sp __scan)
-{
-    // TODO: parallelize this function.
-    return __scan(_Index(0), __n, __init);
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-#endif // _PSTL_INTERNAL_OMP_PARALLEL_TRANSFORM_SCAN_H
diff --git a/pstl/include/pstl/internal/omp/util.h b/pstl/include/pstl/internal/omp/util.h
deleted file mode 100644
index c88d9808379c..000000000000
--- a/pstl/include/pstl/internal/omp/util.h
+++ /dev/null
@@ -1,173 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_INTERNAL_OMP_UTIL_H
-#define _PSTL_INTERNAL_OMP_UTIL_H
-
-#include <algorithm>
-#include <atomic>
-#include <iterator>
-#include <cstddef>
-#include <cstdio>
-#include <memory>
-#include <vector>
-#include <omp.h>
-
-#include "../parallel_backend_utils.h"
-#include "../unseq_backend_simd.h"
-#include "../utils.h"
-
-// Portability "#pragma" definition
-#ifdef _MSC_VER
-#    define _PSTL_PRAGMA(x) __pragma(x)
-#else
-#    define _PSTL_PRAGMA(x) _Pragma(#    x)
-#endif
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __omp_backend
-{
-
-//------------------------------------------------------------------------
-// use to cancel execution
-//------------------------------------------------------------------------
-inline void
-__cancel_execution()
-{
-    // TODO: Figure out how to make cancelation work.
-}
-
-//------------------------------------------------------------------------
-// raw buffer
-//------------------------------------------------------------------------
-
-template <typename _Tp>
-class __buffer
-{
-    std::allocator<_Tp> __allocator_;
-    _Tp* __ptr_;
-    const std::size_t __buf_size_;
-    __buffer(const __buffer&) = delete;
-    void
-    operator=(const __buffer&) = delete;
-
-  public:
-    __buffer(std::size_t __n) : __allocator_(), __ptr_(__allocator_.allocate(__n)), __buf_size_(__n) {}
-
-    operator bool() const { return __ptr_ != nullptr; }
-
-    _Tp*
-    get() const
-    {
-        return __ptr_;
-    }
-    ~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
-};
-
-// Preliminary size of each chunk: requires further discussion
-inline constexpr std::size_t __default_chunk_size = 2048;
-
-// Convenience function to determine when we should run serial.
-template <typename _Iterator, std::enable_if_t<!std::is_integral<_Iterator>::value, bool> = true>
-constexpr auto
-__should_run_serial(_Iterator __first, _Iterator __last) -> bool
-{
-    using _difference_type = typename std::iterator_traits<_Iterator>::difference_type;
-    auto __size = std::distance(__first, __last);
-    return __size <= static_cast<_difference_type>(__default_chunk_size);
-}
-
-template <typename _Index, std::enable_if_t<std::is_integral<_Index>::value, bool> = true>
-constexpr auto
-__should_run_serial(_Index __first, _Index __last) -> bool
-{
-    using _difference_type = _Index;
-    auto __size = __last - __first;
-    return __size <= static_cast<_difference_type>(__default_chunk_size);
-}
-
-struct __chunk_metrics
-{
-    std::size_t __n_chunks;
-    std::size_t __chunk_size;
-    std::size_t __first_chunk_size;
-};
-
-// The iteration space partitioner according to __requested_chunk_size
-template <class _RandomAccessIterator, class _Size = std::size_t>
-auto
-__chunk_partitioner(_RandomAccessIterator __first, _RandomAccessIterator __last,
-                    _Size __requested_chunk_size = __default_chunk_size) -> __chunk_metrics
-{
-    /*
-     * This algorithm improves distribution of elements in chunks by avoiding
-     * small tail chunks. The leftover elements that do not fit neatly into
-     * the chunk size are redistributed to early chunks. This improves
-     * utilization of the processor's prefetch and reduces the number of
-     * tasks needed by 1.
-     */
-
-    const _Size __n = __last - __first;
-    _Size __n_chunks = 0;
-    _Size __chunk_size = 0;
-    _Size __first_chunk_size = 0;
-    if (__n < __requested_chunk_size)
-    {
-        __chunk_size = __n;
-        __first_chunk_size = __n;
-        __n_chunks = 1;
-        return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
-    }
-
-    __n_chunks = (__n / __requested_chunk_size) + 1;
-    __chunk_size = __n / __n_chunks;
-    __first_chunk_size = __chunk_size;
-    const _Size __n_leftover_items = __n - (__n_chunks * __chunk_size);
-
-    if (__n_leftover_items == __chunk_size)
-    {
-        __n_chunks += 1;
-        return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
-    }
-    else if (__n_leftover_items == 0)
-    {
-        __first_chunk_size = __chunk_size;
-        return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
-    }
-
-    const _Size __n_extra_items_per_chunk = __n_leftover_items / __n_chunks;
-    const _Size __n_final_leftover_items = __n_leftover_items - (__n_extra_items_per_chunk * __n_chunks);
-
-    __chunk_size += __n_extra_items_per_chunk;
-    __first_chunk_size = __chunk_size + __n_final_leftover_items;
-
-    return __chunk_metrics{__n_chunks, __chunk_size, __first_chunk_size};
-}
-
-template <typename _Iterator, typename _Index, typename _Func>
-void
-__process_chunk(const __chunk_metrics& __metrics, _Iterator __base, _Index __chunk_index, _Func __f)
-{
-    auto __this_chunk_size = __chunk_index == 0 ? __metrics.__first_chunk_size : __metrics.__chunk_size;
-    auto __index = __chunk_index == 0 ? 0
-                                      : (__chunk_index * __metrics.__chunk_size) +
-                                            (__metrics.__first_chunk_size - __metrics.__chunk_size);
-    auto __first = __base + __index;
-    auto __last = __first + __this_chunk_size;
-    __f(__first, __last);
-}
-
-} // namespace __omp_backend
-} // namespace __pstl
-
-#endif // _PSTL_INTERNAL_OMP_UTIL_H
diff --git a/pstl/include/pstl/internal/parallel_backend.h b/pstl/include/pstl/internal/parallel_backend.h
deleted file mode 100644
index 4da871bfe74f..000000000000
--- a/pstl/include/pstl/internal/parallel_backend.h
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_BACKEND_H
-#define _PSTL_PARALLEL_BACKEND_H
-
-#include "pstl_config.h"
-
-#if defined(_PSTL_PAR_BACKEND_SERIAL)
-#    include "parallel_backend_serial.h"
-namespace __pstl
-{
-namespace __par_backend = __serial_backend;
-}
-#elif defined(_PSTL_PAR_BACKEND_TBB)
-#    include "parallel_backend_tbb.h"
-namespace __pstl
-{
-namespace __par_backend = __tbb_backend;
-}
-#elif defined(_PSTL_PAR_BACKEND_OPENMP)
-#    include "parallel_backend_omp.h"
-namespace __pstl
-{
-namespace __par_backend = __omp_backend;
-}
-#else
-_PSTL_PRAGMA_MESSAGE("Parallel backend was not specified");
-#endif
-
-#endif /* _PSTL_PARALLEL_BACKEND_H */
diff --git a/pstl/include/pstl/internal/parallel_backend_omp.h b/pstl/include/pstl/internal/parallel_backend_omp.h
deleted file mode 100644
index 7398cfe55929..000000000000
--- a/pstl/include/pstl/internal/parallel_backend_omp.h
+++ /dev/null
@@ -1,58 +0,0 @@
-// -*- C++ -*-
-// -*-===----------------------------------------------------------------------===//
-//
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_BACKEND_OMP_H
-#define _PSTL_PARALLEL_BACKEND_OMP_H
-
-//------------------------------------------------------------------------
-// parallel_invoke
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_invoke.h"
-
-//------------------------------------------------------------------------
-// parallel_for
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_for.h"
-
-//------------------------------------------------------------------------
-// parallel_for_each
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_for_each.h"
-
-//------------------------------------------------------------------------
-// parallel_reduce
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_reduce.h"
-#include "./omp/parallel_transform_reduce.h"
-
-//------------------------------------------------------------------------
-// parallel_scan
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_scan.h"
-#include "./omp/parallel_transform_scan.h"
-
-//------------------------------------------------------------------------
-// parallel_stable_sort
-//------------------------------------------------------------------------
-
-#include "./omp/parallel_stable_partial_sort.h"
-#include "./omp/parallel_stable_sort.h"
-
-//------------------------------------------------------------------------
-// parallel_merge
-//------------------------------------------------------------------------
-#include "./omp/parallel_merge.h"
-
-#endif //_PSTL_PARALLEL_BACKEND_OMP_H
diff --git a/pstl/include/pstl/internal/parallel_backend_serial.h b/pstl/include/pstl/internal/parallel_backend_serial.h
deleted file mode 100644
index ad2c5fc60ec3..000000000000
--- a/pstl/include/pstl/internal/parallel_backend_serial.h
+++ /dev/null
@@ -1,137 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_BACKEND_SERIAL_H
-#define _PSTL_PARALLEL_BACKEND_SERIAL_H
-
-#include <algorithm>
-#include <cstddef>
-#include <memory>
-#include <numeric>
-#include <utility>
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __serial_backend
-{
-
-template <typename _Tp>
-class __buffer
-{
-    std::allocator<_Tp> __allocator_;
-    _Tp* __ptr_;
-    const std::size_t __buf_size_;
-    __buffer(const __buffer&) = delete;
-    void
-    operator=(const __buffer&) = delete;
-
-  public:
-    __buffer(std::size_t __n) : __allocator_(), __ptr_(__allocator_.allocate(__n)), __buf_size_(__n) {}
-
-    operator bool() const { return __ptr_ != nullptr; }
-    _Tp*
-    get() const
-    {
-        return __ptr_;
-    }
-    ~__buffer() { __allocator_.deallocate(__ptr_, __buf_size_); }
-};
-
-inline void
-__cancel_execution()
-{
-}
-
-template <class _ExecutionPolicy, class _Index, class _Fp>
-void
-__parallel_for(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last, _Fp __f)
-{
-    __f(__first, __last);
-}
-
-template <class _ExecutionPolicy, class _Value, class _Index, typename _RealBody, typename _Reduction>
-_Value
-__parallel_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
-                  const _Value& __identity, const _RealBody& __real_body, const _Reduction&)
-{
-    if (__first == __last)
-    {
-        return __identity;
-    }
-    else
-    {
-        return __real_body(__first, __last, __identity);
-    }
-}
-
-template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
-_Tp
-__parallel_transform_reduce(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
-                            _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce)
-{
-    return __reduce(__first, __last, __init);
-}
-
-template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
-void
-__parallel_strict_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
-                       _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
-{
-    _Tp __sum = __initial;
-    if (__n)
-        __sum = __combine(__sum, __reduce(_Index(0), __n));
-    __apex(__sum);
-    if (__n)
-        __scan(_Index(0), __n, __initial);
-}
-
-template <class _ExecutionPolicy, class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce, class _Scan>
-_Tp
-__parallel_transform_scan(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _Index __n, _UnaryOp,
-                          _Tp __init, _BinaryOp, _Reduce, _Scan __scan)
-{
-    return __scan(_Index(0), __n, __init);
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
-__parallel_stable_sort(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __first,
-                       _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort, std::size_t = 0)
-{
-    __leaf_sort(__first, __last, __comp);
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator1, typename _RandomAccessIterator2,
-          typename _RandomAccessIterator3, typename _Compare, typename _LeafMerge>
-void
-__parallel_merge(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator1 __first1,
-                 _RandomAccessIterator1 __last1, _RandomAccessIterator2 __first2, _RandomAccessIterator2 __last2,
-                 _RandomAccessIterator3 __outit, _Compare __comp, _LeafMerge __leaf_merge)
-{
-    __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
-}
-
-template <class _ExecutionPolicy, typename _F1, typename _F2>
-void
-__parallel_invoke(__pstl::__internal::__serial_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
-{
-    std::forward<_F1>(__f1)();
-    std::forward<_F2>(__f2)();
-}
-
-} // namespace __serial_backend
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_PARALLEL_BACKEND_SERIAL_H */
diff --git a/pstl/include/pstl/internal/parallel_backend_tbb.h b/pstl/include/pstl/internal/parallel_backend_tbb.h
deleted file mode 100644
index e336f69799eb..000000000000
--- a/pstl/include/pstl/internal/parallel_backend_tbb.h
+++ /dev/null
@@ -1,1296 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_BACKEND_TBB_H
-#define _PSTL_PARALLEL_BACKEND_TBB_H
-
-#include <algorithm>
-#include <type_traits>
-
-#include "pstl_config.h"
-#include "parallel_backend_utils.h"
-
-// Bring in minimal required subset of Intel TBB
-#include <tbb/blocked_range.h>
-#include <tbb/parallel_for.h>
-#include <tbb/parallel_reduce.h>
-#include <tbb/parallel_scan.h>
-#include <tbb/parallel_invoke.h>
-#include <tbb/task_arena.h>
-#include <tbb/tbb_allocator.h>
-#include <tbb/task.h>
-
-#if TBB_INTERFACE_VERSION < 10000
-#    error Intel(R) Threading Building Blocks 2018 is required; older versions are not supported.
-#endif
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __tbb_backend
-{
-
-//! Raw memory buffer with automatic freeing and no exceptions.
-/** Some of our algorithms need to start with raw memory buffer,
-not an initialize array, because initialization/destruction
-would make the span be at least O(N). */
-// tbb::allocator can improve performance in some cases.
-template <typename _Tp>
-class __buffer
-{
-    tbb::tbb_allocator<_Tp> _M_allocator;
-    _Tp* _M_ptr;
-    const std::size_t _M_buf_size;
-    __buffer(const __buffer&) = delete;
-    void
-    operator=(const __buffer&) = delete;
-
-  public:
-    //! Try to obtain buffer of given size to store objects of _Tp type
-    __buffer(std::size_t n) : _M_allocator(), _M_ptr(_M_allocator.allocate(n)), _M_buf_size(n) {}
-    //! True if buffer was successfully obtained, zero otherwise.
-    operator bool() const { return _M_ptr != NULL; }
-    //! Return pointer to buffer, or  NULL if buffer could not be obtained.
-    _Tp*
-    get() const
-    {
-        return _M_ptr;
-    }
-    //! Destroy buffer
-    ~__buffer() { _M_allocator.deallocate(_M_ptr, _M_buf_size); }
-};
-
-// Wrapper for tbb::task
-inline void
-__cancel_execution()
-{
-#if TBB_INTERFACE_VERSION <= 12000
-    tbb::task::self().group()->cancel_group_execution();
-#else
-    tbb::task::current_context()->cancel_group_execution();
-#endif
-}
-
-//------------------------------------------------------------------------
-// parallel_for
-//------------------------------------------------------------------------
-
-template <class _Index, class _RealBody>
-class __parallel_for_body
-{
-  public:
-    __parallel_for_body(const _RealBody& __body) : _M_body(__body) {}
-    __parallel_for_body(const __parallel_for_body& __body) : _M_body(__body._M_body) {}
-    void
-    operator()(const tbb::blocked_range<_Index>& __range) const
-    {
-        _M_body(__range.begin(), __range.end());
-    }
-
-  private:
-    _RealBody _M_body;
-};
-
-//! Evaluation of brick f[i,j) for each subrange [i,j) of [first,last)
-// wrapper over tbb::parallel_for
-template <class _ExecutionPolicy, class _Index, class _Fp>
-void
-__parallel_for(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last, _Fp __f)
-{
-    tbb::this_task_arena::isolate([=]() {
-        tbb::parallel_for(tbb::blocked_range<_Index>(__first, __last), __parallel_for_body<_Index, _Fp>(__f));
-    });
-}
-
-//! Evaluation of brick f[i,j) for each subrange [i,j) of [first,last)
-// wrapper over tbb::parallel_reduce
-template <class _ExecutionPolicy, class _Value, class _Index, typename _RealBody, typename _Reduction>
-_Value
-__parallel_reduce(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
-                  const _Value& __identity, const _RealBody& __real_body, const _Reduction& __reduction)
-{
-    return tbb::this_task_arena::isolate([__first, __last, &__identity, &__real_body, &__reduction]() -> _Value {
-        return tbb::parallel_reduce(
-            tbb::blocked_range<_Index>(__first, __last), __identity,
-            [__real_body](const tbb::blocked_range<_Index>& __r, const _Value& __value) -> _Value {
-                return __real_body(__r.begin(), __r.end(), __value);
-            },
-            __reduction);
-    });
-}
-
-//------------------------------------------------------------------------
-// parallel_transform_reduce
-//
-// Notation:
-//      r(i,j,init) returns reduction of init with reduction over [i,j)
-//      u(i) returns f(i,i+1,identity) for a hypothetical left identity element of r
-//      c(x,y) combines values x and y that were the result of r or u
-//------------------------------------------------------------------------
-
-template <class _Index, class _Up, class _Tp, class _Cp, class _Rp>
-struct __par_trans_red_body
-{
-    alignas(_Tp) char _M_sum_storage[sizeof(_Tp)]; // Holds generalized non-commutative sum when has_sum==true
-    _Rp _M_brick_reduce;                           // Most likely to have non-empty layout
-    _Up _M_u;
-    _Cp _M_combine;
-    bool _M_has_sum; // Put last to minimize size of class
-    _Tp&
-    sum()
-    {
-        __TBB_ASSERT(_M_has_sum, "sum expected");
-        return *(_Tp*)_M_sum_storage;
-    }
-    __par_trans_red_body(_Up __u, _Tp __init, _Cp __c, _Rp __r)
-        : _M_brick_reduce(__r), _M_u(__u), _M_combine(__c), _M_has_sum(true)
-    {
-        new (_M_sum_storage) _Tp(__init);
-    }
-
-    __par_trans_red_body(__par_trans_red_body& __left, tbb::split)
-        : _M_brick_reduce(__left._M_brick_reduce), _M_u(__left._M_u), _M_combine(__left._M_combine), _M_has_sum(false)
-    {
-    }
-
-    ~__par_trans_red_body()
-    {
-        // 17.6.5.12 tells us to not worry about catching exceptions from destructors.
-        if (_M_has_sum)
-            sum().~_Tp();
-    }
-
-    void
-    join(__par_trans_red_body& __rhs)
-    {
-        sum() = _M_combine(sum(), __rhs.sum());
-    }
-
-    void
-    operator()(const tbb::blocked_range<_Index>& __range)
-    {
-        _Index __i = __range.begin();
-        _Index __j = __range.end();
-        if (!_M_has_sum)
-        {
-            __TBB_ASSERT(__range.size() > 1, "there should be at least 2 elements");
-            new (&_M_sum_storage)
-                _Tp(_M_combine(_M_u(__i), _M_u(__i + 1))); // The condition i+1 < j is provided by the grain size of 3
-            _M_has_sum = true;
-            std::advance(__i, 2);
-            if (__i == __j)
-                return;
-        }
-        sum() = _M_brick_reduce(__i, __j, sum());
-    }
-};
-
-template <class _ExecutionPolicy, class _Index, class _Up, class _Tp, class _Cp, class _Rp>
-_Tp
-__parallel_transform_reduce(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _Index __first, _Index __last,
-                            _Up __u, _Tp __init, _Cp __combine, _Rp __brick_reduce)
-{
-    __tbb_backend::__par_trans_red_body<_Index, _Up, _Tp, _Cp, _Rp> __body(__u, __init, __combine, __brick_reduce);
-    // The grain size of 3 is used in order to provide mininum 2 elements for each body
-    tbb::this_task_arena::isolate(
-        [__first, __last, &__body]() { tbb::parallel_reduce(tbb::blocked_range<_Index>(__first, __last, 3), __body); });
-    return __body.sum();
-}
-
-//------------------------------------------------------------------------
-// parallel_scan
-//------------------------------------------------------------------------
-
-template <class _Index, class _Up, class _Tp, class _Cp, class _Rp, class _Sp>
-class __trans_scan_body
-{
-    alignas(_Tp) char _M_sum_storage[sizeof(_Tp)]; // Holds generalized non-commutative sum when has_sum==true
-    _Rp _M_brick_reduce;                           // Most likely to have non-empty layout
-    _Up _M_u;
-    _Cp _M_combine;
-    _Sp _M_scan;
-    bool _M_has_sum; // Put last to minimize size of class
-  public:
-    __trans_scan_body(_Up __u, _Tp __init, _Cp __combine, _Rp __reduce, _Sp __scan)
-        : _M_brick_reduce(__reduce), _M_u(__u), _M_combine(__combine), _M_scan(__scan), _M_has_sum(true)
-    {
-        new (_M_sum_storage) _Tp(__init);
-    }
-
-    __trans_scan_body(__trans_scan_body& __b, tbb::split)
-        : _M_brick_reduce(__b._M_brick_reduce), _M_u(__b._M_u), _M_combine(__b._M_combine), _M_scan(__b._M_scan),
-          _M_has_sum(false)
-    {
-    }
-
-    ~__trans_scan_body()
-    {
-        // 17.6.5.12 tells us to not worry about catching exceptions from destructors.
-        if (_M_has_sum)
-            sum().~_Tp();
-    }
-
-    _Tp&
-    sum() const
-    {
-        __TBB_ASSERT(_M_has_sum, "sum expected");
-        return *const_cast<_Tp*>(reinterpret_cast<_Tp const*>(_M_sum_storage));
-    }
-
-    void
-    operator()(const tbb::blocked_range<_Index>& __range, tbb::pre_scan_tag)
-    {
-        _Index __i = __range.begin();
-        _Index __j = __range.end();
-        if (!_M_has_sum)
-        {
-            new (&_M_sum_storage) _Tp(_M_u(__i));
-            _M_has_sum = true;
-            ++__i;
-            if (__i == __j)
-                return;
-        }
-        sum() = _M_brick_reduce(__i, __j, sum());
-    }
-
-    void
-    operator()(const tbb::blocked_range<_Index>& __range, tbb::final_scan_tag)
-    {
-        sum() = _M_scan(__range.begin(), __range.end(), sum());
-    }
-
-    void
-    reverse_join(__trans_scan_body& __a)
-    {
-        if (_M_has_sum)
-        {
-            sum() = _M_combine(__a.sum(), sum());
-        }
-        else
-        {
-            new (&_M_sum_storage) _Tp(__a.sum());
-            _M_has_sum = true;
-        }
-    }
-
-    void
-    assign(__trans_scan_body& __b)
-    {
-        sum() = __b.sum();
-    }
-};
-
-template <typename _Index>
-_Index
-__split(_Index __m)
-{
-    _Index __k = 1;
-    while (2 * __k < __m)
-        __k *= 2;
-    return __k;
-}
-
-//------------------------------------------------------------------------
-// __parallel_strict_scan
-//------------------------------------------------------------------------
-
-template <typename _Index, typename _Tp, typename _Rp, typename _Cp>
-void
-__upsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Rp __reduce, _Cp __combine)
-{
-    if (__m == 1)
-        __r[0] = __reduce(__i * __tilesize, __lastsize);
-    else
-    {
-        _Index __k = __split(__m);
-        tbb::parallel_invoke(
-            [=] { __tbb_backend::__upsweep(__i, __k, __tilesize, __r, __tilesize, __reduce, __combine); },
-            [=] {
-                __tbb_backend::__upsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize, __reduce, __combine);
-            });
-        if (__m == 2 * __k)
-            __r[__m - 1] = __combine(__r[__k - 1], __r[__m - 1]);
-    }
-}
-
-template <typename _Index, typename _Tp, typename _Cp, typename _Sp>
-void
-__downsweep(_Index __i, _Index __m, _Index __tilesize, _Tp* __r, _Index __lastsize, _Tp __initial, _Cp __combine,
-            _Sp __scan)
-{
-    if (__m == 1)
-        __scan(__i * __tilesize, __lastsize, __initial);
-    else
-    {
-        const _Index __k = __split(__m);
-        tbb::parallel_invoke(
-            [=] { __tbb_backend::__downsweep(__i, __k, __tilesize, __r, __tilesize, __initial, __combine, __scan); },
-            // Assumes that __combine never throws.
-            //TODO: Consider adding a requirement for user functors to be constant.
-            [=, &__combine] {
-                __tbb_backend::__downsweep(__i + __k, __m - __k, __tilesize, __r + __k, __lastsize,
-                                           __combine(__initial, __r[__k - 1]), __combine, __scan);
-            });
-    }
-}
-
-// Adapted from Intel(R) Cilk(TM) version from cilkpub.
-// Let i:len denote a counted interval of length n starting at i.  s denotes a generalized-sum value.
-// Expected actions of the functors are:
-//     reduce(i,len) -> s  -- return reduction value of i:len.
-//     combine(s1,s2) -> s -- return merged sum
-//     apex(s) -- do any processing necessary between reduce and scan.
-//     scan(i,len,initial) -- perform scan over i:len starting with initial.
-// The initial range 0:n is partitioned into consecutive subranges.
-// reduce and scan are each called exactly once per subrange.
-// Thus callers can rely upon side effects in reduce.
-// combine must not throw an exception.
-// apex is called exactly once, after all calls to reduce and before all calls to scan.
-// For example, it's useful for allocating a __buffer used by scan but whose size is the sum of all reduction values.
-// T must have a trivial constructor and destructor.
-template <class _ExecutionPolicy, typename _Index, typename _Tp, typename _Rp, typename _Cp, typename _Sp, typename _Ap>
-void
-__parallel_strict_scan(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _Index __n, _Tp __initial,
-                       _Rp __reduce, _Cp __combine, _Sp __scan, _Ap __apex)
-{
-    tbb::this_task_arena::isolate([=, &__combine]() {
-        if (__n > 1)
-        {
-            _Index __p = tbb::this_task_arena::max_concurrency();
-            const _Index __slack = 4;
-            _Index __tilesize = (__n - 1) / (__slack * __p) + 1;
-            _Index __m = (__n - 1) / __tilesize;
-            __buffer<_Tp> __buf(__m + 1);
-            _Tp* __r = __buf.get();
-            __tbb_backend::__upsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __reduce,
-                                     __combine);
-
-            // When __apex is a no-op and __combine has no side effects, a good optimizer
-            // should be able to eliminate all code between here and __apex.
-            // Alternatively, provide a default value for __apex that can be
-            // recognized by metaprogramming that conditionlly executes the following.
-            size_t __k = __m + 1;
-            _Tp __t = __r[__k - 1];
-            while ((__k &= __k - 1))
-                __t = __combine(__r[__k - 1], __t);
-            __apex(__combine(__initial, __t));
-            __tbb_backend::__downsweep(_Index(0), _Index(__m + 1), __tilesize, __r, __n - __m * __tilesize, __initial,
-                                       __combine, __scan);
-            return;
-        }
-        // Fewer than 2 elements in sequence, or out of memory.  Handle has single block.
-        _Tp __sum = __initial;
-        if (__n)
-            __sum = __combine(__sum, __reduce(_Index(0), __n));
-        __apex(__sum);
-        if (__n)
-            __scan(_Index(0), __n, __initial);
-    });
-}
-
-template <class _ExecutionPolicy, class _Index, class _Up, class _Tp, class _Cp, class _Rp, class _Sp>
-_Tp
-__parallel_transform_scan(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _Index __n, _Up __u, _Tp __init,
-                          _Cp __combine, _Rp __brick_reduce, _Sp __scan)
-{
-    __trans_scan_body<_Index, _Up, _Tp, _Cp, _Rp, _Sp> __body(__u, __init, __combine, __brick_reduce, __scan);
-    auto __range = tbb::blocked_range<_Index>(0, __n);
-    tbb::this_task_arena::isolate([__range, &__body]() { tbb::parallel_scan(__range, __body); });
-    return __body.sum();
-}
-
-//------------------------------------------------------------------------
-// parallel_stable_sort
-//------------------------------------------------------------------------
-
-//------------------------------------------------------------------------
-// stable_sort utilities
-//
-// These are used by parallel implementations but do not depend on them.
-//------------------------------------------------------------------------
-#define _PSTL_MERGE_CUT_OFF 2000
-
-template <typename _Func>
-class __func_task;
-template <typename _Func>
-class __root_task;
-
-#if TBB_INTERFACE_VERSION <= 12000
-class __task : public tbb::task
-{
-  public:
-    template <typename _Fn>
-    __task*
-    make_continuation(_Fn&& __f)
-    {
-        return new (allocate_continuation()) __func_task<typename std::decay<_Fn>::type>(std::forward<_Fn>(__f));
-    }
-
-    template <typename _Fn>
-    __task*
-    make_child_of(__task* parent, _Fn&& __f)
-    {
-        return new (parent->allocate_child()) __func_task<typename std::decay<_Fn>::type>(std::forward<_Fn>(__f));
-    }
-
-    template <typename _Fn>
-    __task*
-    make_additional_child_of(tbb::task* parent, _Fn&& __f)
-    {
-        return new (tbb::task::allocate_additional_child_of(*parent))
-            __func_task<typename std::decay<_Fn>::type>(std::forward<_Fn>(__f));
-    }
-
-    inline void
-    recycle_as_continuation()
-    {
-        tbb::task::recycle_as_continuation();
-    }
-
-    inline void
-    recycle_as_child_of(__task* parent)
-    {
-        tbb::task::recycle_as_child_of(*parent);
-    }
-
-    inline void
-    spawn(__task* __t)
-    {
-        tbb::task::spawn(*__t);
-    }
-
-    template <typename _Fn>
-    static inline void
-    spawn_root_and_wait(__root_task<_Fn>& __root)
-    {
-        tbb::task::spawn_root_and_wait(*__root._M_task);
-    }
-};
-
-template <typename _Func>
-class __func_task : public __task
-{
-    _Func _M_func;
-
-    tbb::task*
-    execute()
-    {
-        return _M_func(this);
-    };
-
-  public:
-    template <typename _Fn>
-    __func_task(_Fn&& __f) : _M_func{std::forward<_Fn>(__f)}
-    {
-    }
-
-    _Func&
-    body()
-    {
-        return _M_func;
-    }
-};
-
-template <typename _Func>
-class __root_task
-{
-    tbb::task* _M_task;
-
-  public:
-    template <typename... Args>
-    __root_task(Args&&... args)
-        : _M_task{new (tbb::task::allocate_root()) __func_task<_Func>{_Func(std::forward<Args>(args)...)}}
-    {
-    }
-
-    friend class __task;
-    friend class __func_task<_Func>;
-};
-
-#else  // TBB_INTERFACE_VERSION <= 12000
-class __task : public tbb::detail::d1::task
-{
-  protected:
-    tbb::detail::d1::small_object_allocator _M_allocator{};
-    tbb::detail::d1::execution_data* _M_execute_data{};
-    __task* _M_parent{};
-    std::atomic<int> _M_refcount{};
-    bool _M_recycle{};
-
-    template <typename _Fn>
-    __task*
-    allocate_func_task(_Fn&& __f)
-    {
-        _PSTL_ASSERT(_M_execute_data != nullptr);
-        tbb::detail::d1::small_object_allocator __alloc{};
-        auto __t =
-            __alloc.new_object<__func_task<typename std::decay<_Fn>::type>>(*_M_execute_data, std::forward<_Fn>(__f));
-        __t->_M_allocator = __alloc;
-        return __t;
-    }
-
-  public:
-    __task*
-    parent()
-    {
-        return _M_parent;
-    }
-
-    void
-    set_ref_count(int __n)
-    {
-        _M_refcount.store(__n, std::memory_order_release);
-    }
-
-    template <typename _Fn>
-    __task*
-    make_continuation(_Fn&& __f)
-    {
-        auto __t = allocate_func_task(std::forward<_Fn&&>(__f));
-        __t->_M_parent = _M_parent;
-        _M_parent = nullptr;
-        return __t;
-    }
-
-    template <typename _Fn>
-    __task*
-    make_child_of(__task* __parent, _Fn&& __f)
-    {
-        auto __t = allocate_func_task(std::forward<_Fn&&>(__f));
-        __t->_M_parent = __parent;
-        return __t;
-    }
-
-    template <typename _Fn>
-    __task*
-    make_additional_child_of(__task* __parent, _Fn&& __f)
-    {
-        auto __t = make_child_of(__parent, std::forward<_Fn>(__f));
-        _PSTL_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0);
-        ++__parent->_M_refcount;
-        return __t;
-    }
-
-    inline void
-    recycle_as_continuation()
-    {
-        _M_recycle = true;
-    }
-
-    inline void
-    recycle_as_child_of(__task* parent)
-    {
-        _M_recycle = true;
-        _M_parent = parent;
-    }
-
-    inline void
-    spawn(__task* __t)
-    {
-        _PSTL_ASSERT(_M_execute_data != nullptr);
-        tbb::detail::d1::spawn(*__t, *_M_execute_data->context);
-    }
-
-    template <typename _Fn>
-    static inline void
-    spawn_root_and_wait(__root_task<_Fn>& __root)
-    {
-        tbb::detail::d1::execute_and_wait(*__root._M_func_task, __root._M_context, __root._M_wait_object,
-                                          __root._M_context);
-    }
-
-    template <typename _Func>
-    friend class __func_task;
-};
-
-template <typename _Func>
-class __func_task : public __task
-{
-    _Func _M_func;
-
-    __task*
-    execute(tbb::detail::d1::execution_data& __ed) override
-    {
-        _M_execute_data = &__ed;
-        _M_recycle = false;
-        __task* __next = _M_func(this);
-        return finalize(__next);
-    };
-
-    __task*
-    cancel(tbb::detail::d1::execution_data& __ed) override
-    {
-        return finalize(nullptr);
-    }
-
-    __task*
-    finalize(__task* __next)
-    {
-        bool __recycle = _M_recycle;
-        _M_recycle = false;
-
-        if (__recycle)
-        {
-            return __next;
-        }
-
-        auto __parent = _M_parent;
-        auto __alloc = _M_allocator;
-        auto __ed = _M_execute_data;
-
-        this->~__func_task();
-
-        _PSTL_ASSERT(__parent != nullptr);
-        _PSTL_ASSERT(__parent->_M_refcount.load(std::memory_order_relaxed) > 0);
-        if (--__parent->_M_refcount == 0)
-        {
-            _PSTL_ASSERT(__next == nullptr);
-            __alloc.deallocate(this, *__ed);
-            return __parent;
-        }
-
-        return __next;
-    }
-
-    friend class __root_task<_Func>;
-
-  public:
-    template <typename _Fn>
-    __func_task(_Fn&& __f) : _M_func(std::forward<_Fn>(__f))
-    {
-    }
-
-    _Func&
-    body()
-    {
-        return _M_func;
-    }
-};
-
-template <typename _Func>
-class __root_task : public __task
-{
-    __task*
-    execute(tbb::detail::d1::execution_data& __ed) override
-    {
-        _M_wait_object.release();
-        return nullptr;
-    };
-
-    __task*
-    cancel(tbb::detail::d1::execution_data& __ed) override
-    {
-        _M_wait_object.release();
-        return nullptr;
-    }
-
-    __func_task<_Func>* _M_func_task{};
-    tbb::detail::d1::wait_context _M_wait_object{0};
-    tbb::task_group_context _M_context{};
-
-  public:
-    template <typename... Args>
-    __root_task(Args&&... args) : _M_wait_object{1}
-    {
-        tbb::detail::d1::small_object_allocator __alloc{};
-        _M_func_task = __alloc.new_object<__func_task<_Func>>(_Func(std::forward<Args>(args)...));
-        _M_func_task->_M_allocator = __alloc;
-        _M_func_task->_M_parent = this;
-        _M_refcount.store(1, std::memory_order_relaxed);
-    }
-
-    friend class __task;
-};
-#endif // TBB_INTERFACE_VERSION <= 12000
-
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _Compare, typename _Cleanup,
-          typename _LeafMerge>
-class __merge_func
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::difference_type _DifferenceType2;
-    typedef typename std::common_type<_DifferenceType1, _DifferenceType2>::type _SizeType;
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::value_type _ValueType;
-
-    _RandomAccessIterator1 _M_x_beg;
-    _RandomAccessIterator2 _M_z_beg;
-
-    _SizeType _M_xs, _M_xe;
-    _SizeType _M_ys, _M_ye;
-    _SizeType _M_zs;
-    _Compare _M_comp;
-    _LeafMerge _M_leaf_merge;
-    _SizeType _M_nsort; //number of elements to be sorted for partial_sort alforithm
-
-    static const _SizeType __merge_cut_off = _PSTL_MERGE_CUT_OFF;
-
-    bool _root;   //means a task is merging root task
-    bool _x_orig; //"true" means X(or left ) subrange is in the original container; false - in the buffer
-    bool _y_orig; //"true" means Y(or right) subrange is in the original container; false - in the buffer
-    bool _split; //"true" means a merge task is a split task for parallel merging, the execution logic differs
-
-    bool
-    is_partial() const
-    {
-        return _M_nsort > 0;
-    }
-
-    struct __move_value
-    {
-        template <typename Iterator1, typename Iterator2>
-        void
-        operator()(Iterator1 __x, Iterator2 __z)
-        {
-            *__z = std::move(*__x);
-        }
-    };
-
-    struct __move_value_construct
-    {
-        template <typename Iterator1, typename Iterator2>
-        void
-        operator()(Iterator1 __x, Iterator2 __z)
-        {
-            ::new (std::addressof(*__z)) _ValueType(std::move(*__x));
-        }
-    };
-
-    struct __move_range
-    {
-        template <typename Iterator1, typename Iterator2>
-        Iterator2
-        operator()(Iterator1 __first1, Iterator1 __last1, Iterator2 __first2)
-        {
-            if (__last1 - __first1 < __merge_cut_off)
-                return std::move(__first1, __last1, __first2);
-
-            auto __n = __last1 - __first1;
-            tbb::parallel_for(tbb::blocked_range<_SizeType>(0, __n, __merge_cut_off),
-                              [__first1, __first2](const tbb::blocked_range<_SizeType>& __range) {
-                                  std::move(__first1 + __range.begin(), __first1 + __range.end(),
-                                            __first2 + __range.begin());
-                              });
-            return __first2 + __n;
-        }
-    };
-
-    struct __move_range_construct
-    {
-        template <typename Iterator1, typename Iterator2>
-        Iterator2
-        operator()(Iterator1 __first1, Iterator1 __last1, Iterator2 __first2)
-        {
-            if (__last1 - __first1 < __merge_cut_off)
-            {
-                for (; __first1 != __last1; ++__first1, ++__first2)
-                    __move_value_construct()(__first1, __first2);
-                return __first2;
-            }
-
-            auto __n = __last1 - __first1;
-            tbb::parallel_for(tbb::blocked_range<_SizeType>(0, __n, __merge_cut_off),
-                              [__first1, __first2](const tbb::blocked_range<_SizeType>& __range) {
-                                  for (auto i = __range.begin(); i != __range.end(); ++i)
-                                      __move_value_construct()(__first1 + i, __first2 + i);
-                              });
-            return __first2 + __n;
-        }
-    };
-
-    struct __cleanup_range
-    {
-        template <typename Iterator>
-        void
-        operator()(Iterator __first, Iterator __last)
-        {
-            if (__last - __first < __merge_cut_off)
-                _Cleanup()(__first, __last);
-            else
-            {
-                auto __n = __last - __first;
-                tbb::parallel_for(tbb::blocked_range<_SizeType>(0, __n, __merge_cut_off),
-                                  [__first](const tbb::blocked_range<_SizeType>& __range) {
-                                      _Cleanup()(__first + __range.begin(), __first + __range.end());
-                                  });
-            }
-        }
-    };
-
-  public:
-    __merge_func(_SizeType __xs, _SizeType __xe, _SizeType __ys, _SizeType __ye, _SizeType __zs, _Compare __comp,
-                 _Cleanup, _LeafMerge __leaf_merge, _SizeType __nsort, _RandomAccessIterator1 __x_beg,
-                 _RandomAccessIterator2 __z_beg, bool __x_orig, bool __y_orig, bool __root)
-        : _M_xs(__xs), _M_xe(__xe), _M_ys(__ys), _M_ye(__ye), _M_zs(__zs), _M_x_beg(__x_beg), _M_z_beg(__z_beg),
-          _M_comp(__comp), _M_leaf_merge(__leaf_merge), _M_nsort(__nsort), _root(__root),
-          _x_orig(__x_orig), _y_orig(__y_orig), _split(false)
-    {
-    }
-
-    bool
-    is_left(_SizeType __idx) const
-    {
-        return _M_xs == __idx;
-    }
-
-    template <typename IndexType>
-    void
-    set_odd(IndexType __idx, bool __on_off)
-    {
-        if (is_left(__idx))
-            _x_orig = __on_off;
-        else
-            _y_orig = __on_off;
-    }
-
-    __task*
-    operator()(__task* __self);
-
-  private:
-    __merge_func*
-    parent_merge(__task* __self) const
-    {
-        return _root ? nullptr : &static_cast<__func_task<__merge_func>*>(__self->parent())->body();
-    }
-    bool
-    x_less_y()
-    {
-        const auto __nx = (_M_xe - _M_xs);
-        const auto __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0 && __ny > 0);
-
-        _PSTL_ASSERT(_x_orig == _y_orig);
-        _PSTL_ASSERT(!is_partial());
-
-        if (_x_orig)
-        {
-            _PSTL_ASSERT(std::is_sorted(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_comp));
-            _PSTL_ASSERT(std::is_sorted(_M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_comp));
-            return !_M_comp(*(_M_x_beg + _M_ys), *(_M_x_beg + _M_xe - 1));
-        }
-
-        _PSTL_ASSERT(std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp));
-        _PSTL_ASSERT(std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp));
-        return !_M_comp(*(_M_z_beg + _M_zs + __nx), *(_M_z_beg + _M_zs + __nx - 1));
-    }
-    void
-    move_x_range()
-    {
-        const auto __nx = (_M_xe - _M_xs);
-        const auto __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0 && __ny > 0);
-
-        if (_x_orig)
-            __move_range_construct()(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_z_beg + _M_zs);
-        else
-        {
-            __move_range()(_M_z_beg + _M_zs, _M_z_beg + _M_zs + __nx, _M_x_beg + _M_xs);
-            __cleanup_range()(_M_z_beg + _M_zs, _M_z_beg + _M_zs + __nx);
-        }
-
-        _x_orig = !_x_orig;
-    }
-    void
-    move_y_range()
-    {
-        const auto __nx = (_M_xe - _M_xs);
-        const auto __ny = (_M_ye - _M_ys);
-
-        if (_y_orig)
-            __move_range_construct()(_M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_z_beg + _M_zs + __nx);
-        else
-        {
-            __move_range()(_M_z_beg + _M_zs + __nx, _M_z_beg + _M_zs + __nx + __ny, _M_x_beg + _M_ys);
-            __cleanup_range()(_M_z_beg + _M_zs + __nx, _M_z_beg + _M_zs + __nx + __ny);
-        }
-
-        _y_orig = !_y_orig;
-    }
-    __task*
-    merge_ranges(__task* __self)
-    {
-        _PSTL_ASSERT(_x_orig == _y_orig); //two merged subrange must be lie into the same buffer
-
-        const auto __nx = (_M_xe - _M_xs);
-        const auto __ny = (_M_ye - _M_ys);
-        const auto __n = __nx + __ny;
-
-        // need to merge {x} and {y}
-        if (__n > __merge_cut_off)
-            return split_merging(__self);
-
-        //merge to buffer
-        if (_x_orig)
-        {
-            _M_leaf_merge(_M_x_beg + _M_xs, _M_x_beg + _M_xe, _M_x_beg + _M_ys, _M_x_beg + _M_ye, _M_z_beg + _M_zs,
-                          _M_comp, __move_value_construct(), __move_value_construct(), __move_range_construct(),
-                          __move_range_construct());
-            _PSTL_ASSERT(parent_merge(__self)); //not root merging task
-        }
-        //merge to "origin"
-        else
-        {
-            _PSTL_ASSERT(_x_orig == _y_orig);
-
-            _PSTL_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_comp));
-            _PSTL_ASSERT(is_partial() || std::is_sorted(_M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_comp));
-
-            const auto __nx = (_M_xe - _M_xs);
-            const auto __ny = (_M_ye - _M_ys);
-
-            _M_leaf_merge(_M_z_beg + _M_xs, _M_z_beg + _M_xe, _M_z_beg + _M_ys, _M_z_beg + _M_ye, _M_x_beg + _M_zs,
-                          _M_comp, __move_value(), __move_value(), __move_range(), __move_range());
-
-            __cleanup_range()(_M_z_beg + _M_xs, _M_z_beg + _M_xe);
-            __cleanup_range()(_M_z_beg + _M_ys, _M_z_beg + _M_ye);
-        }
-        return nullptr;
-    }
-
-    __task*
-    process_ranges(__task* __self)
-    {
-        _PSTL_ASSERT(_x_orig == _y_orig);
-        _PSTL_ASSERT(!_split);
-
-        auto p = parent_merge(__self);
-
-        if (!p)
-        { //root merging task
-
-            //optimization, just for sort algorithm, //{x} <= {y}
-            if (!is_partial() && x_less_y()) //we have a solution
-            {
-                if (!_x_orig)
-                {                   //we have to move the solution to the origin
-                    move_x_range(); //parallel moving
-                    move_y_range(); //parallel moving
-                }
-                return nullptr;
-            }
-            //else: if we have data in the origin,
-            //we have to move data to the buffer for final merging into the origin.
-            if (_x_orig)
-            {
-                move_x_range(); //parallel moving
-                move_y_range(); //parallel moving
-            }
-            // need to merge {x} and {y}.
-            return merge_ranges(__self);
-        }
-        //else: not root merging task (parent_merge() == NULL)
-        //optimization, just for sort algorithm, //{x} <= {y}
-        if (!is_partial() && x_less_y())
-        {
-            const auto id_range = _M_zs;
-            p->set_odd(id_range, _x_orig);
-            return nullptr;
-        }
-        //else: we have to revert "_x(y)_orig" flag of the parent merging task
-        const auto id_range = _M_zs;
-        p->set_odd(id_range, !_x_orig);
-
-        return merge_ranges(__self);
-    }
-
-    //splitting as merge task into 2 of the same level
-    __task*
-    split_merging(__task* __self)
-    {
-        _PSTL_ASSERT(_x_orig == _y_orig);
-        const auto __nx = (_M_xe - _M_xs);
-        const auto __ny = (_M_ye - _M_ys);
-
-        _SizeType __xm{};
-        _SizeType __ym{};
-        if (__nx < __ny)
-        {
-            __ym = _M_ys + __ny / 2;
-
-            if (_x_orig)
-                __xm = std::upper_bound(_M_x_beg + _M_xs, _M_x_beg + _M_xe, *(_M_x_beg + __ym), _M_comp) - _M_x_beg;
-            else
-                __xm = std::upper_bound(_M_z_beg + _M_xs, _M_z_beg + _M_xe, *(_M_z_beg + __ym), _M_comp) - _M_z_beg;
-        }
-        else
-        {
-            __xm = _M_xs + __nx / 2;
-
-            if (_y_orig)
-                __ym = std::lower_bound(_M_x_beg + _M_ys, _M_x_beg + _M_ye, *(_M_x_beg + __xm), _M_comp) - _M_x_beg;
-            else
-                __ym = std::lower_bound(_M_z_beg + _M_ys, _M_z_beg + _M_ye, *(_M_z_beg + __xm), _M_comp) - _M_z_beg;
-        }
-
-        auto __zm = _M_zs + ((__xm - _M_xs) + (__ym - _M_ys));
-        __merge_func __right_func(__xm, _M_xe, __ym, _M_ye, __zm, _M_comp, _Cleanup(), _M_leaf_merge, _M_nsort,
-                                  _M_x_beg, _M_z_beg, _x_orig, _y_orig, _root);
-        __right_func._split = true;
-        auto __merge_task = __self->make_additional_child_of(__self->parent(), std::move(__right_func));
-        __self->spawn(__merge_task);
-        __self->recycle_as_continuation();
-
-        _M_xe = __xm;
-        _M_ye = __ym;
-        _split = true;
-
-        return __self;
-    }
-};
-
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename __M_Compare, typename _Cleanup,
-          typename _LeafMerge>
-__task*
-__merge_func<_RandomAccessIterator1, _RandomAccessIterator2, __M_Compare, _Cleanup, _LeafMerge>::
-operator()(__task* __self)
-{
-    //a. split merge task into 2 of the same level; the special logic,
-    //without processing(process_ranges) adjacent sub-ranges x and y
-    if (_split)
-        return merge_ranges(__self);
-
-    //b. General merging of adjacent sub-ranges x and y (with optimization in case of {x} <= {y} )
-
-    //1. x and y are in the even buffer
-    //2. x and y are in the odd buffer
-    if (_x_orig == _y_orig)
-        return process_ranges(__self);
-
-    //3. x is in even buffer, y is in the odd buffer
-    //4. x is in odd buffer, y is in the even buffer
-    if (!parent_merge(__self))
-    { //root merge task
-        if (_x_orig)
-            move_x_range();
-        else
-            move_y_range();
-    }
-    else
-    {
-        const _SizeType __nx = (_M_xe - _M_xs);
-        const _SizeType __ny = (_M_ye - _M_ys);
-        _PSTL_ASSERT(__nx > 0);
-        _PSTL_ASSERT(__nx > 0);
-
-        if (__nx < __ny)
-            move_x_range();
-        else
-            move_y_range();
-    }
-
-    return process_ranges(__self);
-}
-
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _Compare, typename _LeafSort>
-class __stable_sort_func
-{
-  public:
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::difference_type _DifferenceType2;
-    typedef typename std::common_type<_DifferenceType1, _DifferenceType2>::type _SizeType;
-
-  private:
-    _RandomAccessIterator1 _M_xs, _M_xe, _M_x_beg;
-    _RandomAccessIterator2 _M_zs, _M_z_beg;
-    _Compare _M_comp;
-    _LeafSort _M_leaf_sort;
-    bool _M_root;
-    _SizeType _M_nsort; //zero or number of elements to be sorted for partial_sort alforithm
-
-  public:
-    __stable_sort_func(_RandomAccessIterator1 __xs, _RandomAccessIterator1 __xe, _RandomAccessIterator2 __zs,
-                       bool __root, _Compare __comp, _LeafSort __leaf_sort, _SizeType __nsort,
-                       _RandomAccessIterator1 __x_beg, _RandomAccessIterator2 __z_beg)
-        : _M_xs(__xs), _M_xe(__xe), _M_x_beg(__x_beg), _M_zs(__zs), _M_z_beg(__z_beg), _M_comp(__comp),
-          _M_leaf_sort(__leaf_sort), _M_root(__root), _M_nsort(__nsort)
-    {
-    }
-
-    __task*
-    operator()(__task* __self);
-};
-
-#define _PSTL_STABLE_SORT_CUT_OFF 500
-
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _Compare, typename _LeafSort>
-__task*
-__stable_sort_func<_RandomAccessIterator1, _RandomAccessIterator2, _Compare, _LeafSort>::operator()(__task* __self)
-{
-    typedef __merge_func<_RandomAccessIterator1, _RandomAccessIterator2, _Compare, __utils::__serial_destroy,
-                         __utils::__serial_move_merge>
-        _MergeTaskType;
-
-    const _SizeType __n = _M_xe - _M_xs;
-    const _SizeType __nmerge = _M_nsort > 0 ? _M_nsort : __n;
-    const _SizeType __sort_cut_off = _PSTL_STABLE_SORT_CUT_OFF;
-    if (__n <= __sort_cut_off)
-    {
-        _M_leaf_sort(_M_xs, _M_xe, _M_comp);
-        _PSTL_ASSERT(!_M_root);
-        return nullptr;
-    }
-
-    const _RandomAccessIterator1 __xm = _M_xs + __n / 2;
-    const _RandomAccessIterator2 __zm = _M_zs + (__xm - _M_xs);
-    const _RandomAccessIterator2 __ze = _M_zs + __n;
-    _MergeTaskType __m(_MergeTaskType(_M_xs - _M_x_beg, __xm - _M_x_beg, __xm - _M_x_beg, _M_xe - _M_x_beg,
-                                      _M_zs - _M_z_beg, _M_comp, __utils::__serial_destroy(),
-                                      __utils::__serial_move_merge(__nmerge), _M_nsort, _M_x_beg, _M_z_beg,
-                                      /*x_orig*/ true, /*y_orig*/ true, /*root*/ _M_root));
-    auto __parent = __self->make_continuation(std::move(__m));
-    __parent->set_ref_count(2);
-    auto __right = __self->make_child_of(
-        __parent, __stable_sort_func(__xm, _M_xe, __zm, false, _M_comp, _M_leaf_sort, _M_nsort, _M_x_beg, _M_z_beg));
-    __self->spawn(__right);
-    __self->recycle_as_child_of(__parent);
-    _M_root = false;
-    _M_xe = __xm;
-
-    return __self;
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator, typename _Compare, typename _LeafSort>
-void
-__parallel_stable_sort(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator __xs,
-                       _RandomAccessIterator __xe, _Compare __comp, _LeafSort __leaf_sort, std::size_t __nsort = 0)
-{
-    tbb::this_task_arena::isolate([=, &__nsort]() {
-        //sorting based on task tree and parallel merge
-        typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _ValueType;
-        typedef typename std::iterator_traits<_RandomAccessIterator>::difference_type _DifferenceType;
-        const _DifferenceType __n = __xe - __xs;
-        if (__nsort == __n)
-            __nsort = 0; // 'partial_sort' becames 'sort'
-
-        const _DifferenceType __sort_cut_off = _PSTL_STABLE_SORT_CUT_OFF;
-        if (__n > __sort_cut_off)
-        {
-            __buffer<_ValueType> __buf(__n);
-            __root_task<__stable_sort_func<_RandomAccessIterator, _ValueType*, _Compare, _LeafSort>> __root{
-                __xs, __xe, __buf.get(), true, __comp, __leaf_sort, __nsort, __xs, __buf.get()};
-            __task::spawn_root_and_wait(__root);
-            return;
-        }
-        //serial sort
-        __leaf_sort(__xs, __xe, __comp);
-    });
-}
-
-//------------------------------------------------------------------------
-// parallel_merge
-//------------------------------------------------------------------------
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _RandomAccessIterator3,
-          typename _Compare, typename _LeafMerge>
-class __merge_func_static
-{
-    _RandomAccessIterator1 _M_xs, _M_xe;
-    _RandomAccessIterator2 _M_ys, _M_ye;
-    _RandomAccessIterator3 _M_zs;
-    _Compare _M_comp;
-    _LeafMerge _M_leaf_merge;
-
-  public:
-    __merge_func_static(_RandomAccessIterator1 __xs, _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys,
-                        _RandomAccessIterator2 __ye, _RandomAccessIterator3 __zs, _Compare __comp,
-                        _LeafMerge __leaf_merge)
-        : _M_xs(__xs), _M_xe(__xe), _M_ys(__ys), _M_ye(__ye), _M_zs(__zs), _M_comp(__comp), _M_leaf_merge(__leaf_merge)
-    {
-    }
-
-    __task*
-    operator()(__task* __self);
-};
-
-//TODO: consider usage of parallel_for with a custom blocked_range
-template <typename _RandomAccessIterator1, typename _RandomAccessIterator2, typename _RandomAccessIterator3,
-          typename __M_Compare, typename _LeafMerge>
-__task*
-__merge_func_static<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3, __M_Compare, _LeafMerge>::
-operator()(__task* __self)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::difference_type _DifferenceType2;
-    typedef typename std::common_type<_DifferenceType1, _DifferenceType2>::type _SizeType;
-    const _SizeType __n = (_M_xe - _M_xs) + (_M_ye - _M_ys);
-    const _SizeType __merge_cut_off = _PSTL_MERGE_CUT_OFF;
-    if (__n <= __merge_cut_off)
-    {
-        _M_leaf_merge(_M_xs, _M_xe, _M_ys, _M_ye, _M_zs, _M_comp);
-        return nullptr;
-    }
-
-    _RandomAccessIterator1 __xm;
-    _RandomAccessIterator2 __ym;
-    if (_M_xe - _M_xs < _M_ye - _M_ys)
-    {
-        __ym = _M_ys + (_M_ye - _M_ys) / 2;
-        __xm = std::upper_bound(_M_xs, _M_xe, *__ym, _M_comp);
-    }
-    else
-    {
-        __xm = _M_xs + (_M_xe - _M_xs) / 2;
-        __ym = std::lower_bound(_M_ys, _M_ye, *__xm, _M_comp);
-    }
-    const _RandomAccessIterator3 __zm = _M_zs + ((__xm - _M_xs) + (__ym - _M_ys));
-    auto __right = __self->make_additional_child_of(
-        __self->parent(), __merge_func_static(__xm, _M_xe, __ym, _M_ye, __zm, _M_comp, _M_leaf_merge));
-    __self->spawn(__right);
-    __self->recycle_as_continuation();
-    _M_xe = __xm;
-    _M_ye = __ym;
-
-    return __self;
-}
-
-template <class _ExecutionPolicy, typename _RandomAccessIterator1, typename _RandomAccessIterator2,
-          typename _RandomAccessIterator3, typename _Compare, typename _LeafMerge>
-void
-__parallel_merge(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _RandomAccessIterator1 __xs,
-                 _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys, _RandomAccessIterator2 __ye,
-                 _RandomAccessIterator3 __zs, _Compare __comp, _LeafMerge __leaf_merge)
-{
-    typedef typename std::iterator_traits<_RandomAccessIterator1>::difference_type _DifferenceType1;
-    typedef typename std::iterator_traits<_RandomAccessIterator2>::difference_type _DifferenceType2;
-    typedef typename std::common_type<_DifferenceType1, _DifferenceType2>::type _SizeType;
-    const _SizeType __n = (__xe - __xs) + (__ye - __ys);
-    const _SizeType __merge_cut_off = _PSTL_MERGE_CUT_OFF;
-    if (__n <= __merge_cut_off)
-    {
-        // Fall back on serial merge
-        __leaf_merge(__xs, __xe, __ys, __ye, __zs, __comp);
-    }
-    else
-    {
-        tbb::this_task_arena::isolate([=]() {
-            typedef __merge_func_static<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3,
-                                        _Compare, _LeafMerge>
-                _TaskType;
-            __root_task<_TaskType> __root{__xs, __xe, __ys, __ye, __zs, __comp, __leaf_merge};
-            __task::spawn_root_and_wait(__root);
-        });
-    }
-}
-
-//------------------------------------------------------------------------
-// parallel_invoke
-//------------------------------------------------------------------------
-template <class _ExecutionPolicy, typename _F1, typename _F2>
-void
-__parallel_invoke(__pstl::__internal::__tbb_backend_tag, _ExecutionPolicy&&, _F1&& __f1, _F2&& __f2)
-{
-    //TODO: a version of tbb::this_task_arena::isolate with variadic arguments pack should be added in the future
-    tbb::this_task_arena::isolate([&]() { tbb::parallel_invoke(std::forward<_F1>(__f1), std::forward<_F2>(__f2)); });
-}
-
-} // namespace __tbb_backend
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_PARALLEL_BACKEND_TBB_H */
diff --git a/pstl/include/pstl/internal/parallel_backend_utils.h b/pstl/include/pstl/internal/parallel_backend_utils.h
deleted file mode 100644
index e176d7e93518..000000000000
--- a/pstl/include/pstl/internal/parallel_backend_utils.h
+++ /dev/null
@@ -1,263 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_BACKEND_UTILS_H
-#define _PSTL_PARALLEL_BACKEND_UTILS_H
-
-#include <iterator>
-#include <utility>
-#include "utils.h"
-
-#include "pstl_config.h"
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-
-namespace __utils
-{
-
-//! Destroy sequence [xs,xe)
-struct __serial_destroy
-{
-    template <typename _RandomAccessIterator>
-    void
-    operator()(_RandomAccessIterator __zs, _RandomAccessIterator __ze)
-    {
-        typedef typename std::iterator_traits<_RandomAccessIterator>::value_type _ValueType;
-        while (__zs != __ze)
-        {
-            --__ze;
-            (*__ze).~_ValueType();
-        }
-    }
-};
-
-//! Merge sequences [__xs,__xe) and [__ys,__ye) to output sequence [__zs,(__xe-__xs)+(__ye-__ys)), using std::move
-struct __serial_move_merge
-{
-    const std::size_t _M_nmerge;
-
-    explicit __serial_move_merge(std::size_t __nmerge) : _M_nmerge(__nmerge) {}
-    template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIterator3, class _Compare,
-              class _MoveValueX, class _MoveValueY, class _MoveSequenceX, class _MoveSequenceY>
-    void
-    operator()(_RandomAccessIterator1 __xs, _RandomAccessIterator1 __xe, _RandomAccessIterator2 __ys,
-               _RandomAccessIterator2 __ye, _RandomAccessIterator3 __zs, _Compare __comp, _MoveValueX __move_value_x,
-               _MoveValueY __move_value_y, _MoveSequenceX __move_sequence_x, _MoveSequenceY __move_sequence_y)
-    {
-        constexpr bool __same_move_val = std::is_same<_MoveValueX, _MoveValueY>::value;
-        constexpr bool __same_move_seq = std::is_same<_MoveSequenceX, _MoveSequenceY>::value;
-
-        auto __n = _M_nmerge;
-        _PSTL_ASSERT(__n > 0);
-
-        auto __nx = __xe - __xs;
-        //auto __ny = __ye - __ys;
-        _RandomAccessIterator3 __zs_beg = __zs;
-
-        if (__xs != __xe)
-        {
-            if (__ys != __ye)
-            {
-                for (;;)
-                {
-                    if (__comp(*__ys, *__xs))
-                    {
-                        const auto __i = __zs - __zs_beg;
-                        if (__i < __nx)
-                            __move_value_x(__ys, __zs);
-                        else
-                            __move_value_y(__ys, __zs);
-                        ++__zs, --__n;
-                        if (++__ys == __ye)
-                        {
-                            break;
-                        }
-                        else if (__n == 0)
-                        {
-                            const auto __j = __zs - __zs_beg;
-                            if (__same_move_seq || __j < __nx)
-                                __zs = __move_sequence_x(__ys, __ye, __zs);
-                            else
-                                __zs = __move_sequence_y(__ys, __ye, __zs);
-                            break;
-                        }
-                    }
-                    else
-                    {
-                        const auto __i = __zs - __zs_beg;
-                        if (__same_move_val || __i < __nx)
-                            __move_value_x(__xs, __zs);
-                        else
-                            __move_value_y(__xs, __zs);
-                        ++__zs, --__n;
-                        if (++__xs == __xe)
-                        {
-                            const auto __j = __zs - __zs_beg;
-                            if (__same_move_seq || __j < __nx)
-                                __move_sequence_x(__ys, __ye, __zs);
-                            else
-                                __move_sequence_y(__ys, __ye, __zs);
-                            return;
-                        }
-                        else if (__n == 0)
-                        {
-                            const auto __j = __zs - __zs_beg;
-                            if (__same_move_seq || __j < __nx)
-                            {
-                                __zs = __move_sequence_x(__xs, __xe, __zs);
-                                __move_sequence_x(__ys, __ye, __zs);
-                            }
-                            else
-                            {
-                                __zs = __move_sequence_y(__xs, __xe, __zs);
-                                __move_sequence_y(__ys, __ye, __zs);
-                            }
-                            return;
-                        }
-                    }
-                }
-            }
-            __ys = __xs;
-            __ye = __xe;
-        }
-        const auto __i = __zs - __zs_beg;
-        if (__same_move_seq || __i < __nx)
-            __move_sequence_x(__ys, __ye, __zs);
-        else
-            __move_sequence_y(__ys, __ye, __zs);
-    }
-};
-
-template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
-          typename _CopyConstructRange>
-_OutputIterator
-__set_union_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                      _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                      _CopyConstructRange __cc_range)
-{
-    using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
-
-    for (; __first1 != __last1; ++__result)
-    {
-        if (__first2 == __last2)
-            return __cc_range(__first1, __last1, __result);
-        if (__comp(*__first2, *__first1))
-        {
-            ::new (std::addressof(*__result)) _Tp(*__first2);
-            ++__first2;
-        }
-        else
-        {
-            ::new (std::addressof(*__result)) _Tp(*__first1);
-            if (!__comp(*__first1, *__first2))
-                ++__first2;
-            ++__first1;
-        }
-    }
-    return __cc_range(__first2, __last2, __result);
-}
-
-template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare>
-_OutputIterator
-__set_intersection_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                             _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp)
-{
-    using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
-
-    for (; __first1 != __last1 && __first2 != __last2;)
-    {
-        if (__comp(*__first1, *__first2))
-            ++__first1;
-        else
-        {
-            if (!__comp(*__first2, *__first1))
-            {
-                ::new (std::addressof(*__result)) _Tp(*__first1);
-                ++__result;
-                ++__first1;
-            }
-            ++__first2;
-        }
-    }
-    return __result;
-}
-
-template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
-          typename _CopyConstructRange>
-_OutputIterator
-__set_difference_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                           _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                           _CopyConstructRange __cc_range)
-{
-    using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
-
-    for (; __first1 != __last1;)
-    {
-        if (__first2 == __last2)
-            return __cc_range(__first1, __last1, __result);
-
-        if (__comp(*__first1, *__first2))
-        {
-            ::new (std::addressof(*__result)) _Tp(*__first1);
-            ++__result;
-            ++__first1;
-        }
-        else
-        {
-            if (!__comp(*__first2, *__first1))
-                ++__first1;
-            ++__first2;
-        }
-    }
-    return __result;
-}
-template <typename _ForwardIterator1, typename _ForwardIterator2, typename _OutputIterator, typename _Compare,
-          typename _CopyConstructRange>
-_OutputIterator
-__set_symmetric_difference_construct(_ForwardIterator1 __first1, _ForwardIterator1 __last1, _ForwardIterator2 __first2,
-                                     _ForwardIterator2 __last2, _OutputIterator __result, _Compare __comp,
-                                     _CopyConstructRange __cc_range)
-{
-    using _Tp = typename std::iterator_traits<_OutputIterator>::value_type;
-
-    for (; __first1 != __last1;)
-    {
-        if (__first2 == __last2)
-            return __cc_range(__first1, __last1, __result);
-
-        if (__comp(*__first1, *__first2))
-        {
-            ::new (std::addressof(*__result)) _Tp(*__first1);
-            ++__result;
-            ++__first1;
-        }
-        else
-        {
-            if (__comp(*__first2, *__first1))
-            {
-                ::new (std::addressof(*__result)) _Tp(*__first2);
-                ++__result;
-            }
-            else
-                ++__first1;
-            ++__first2;
-        }
-    }
-    return __cc_range(__first2, __last2, __result);
-}
-
-} // namespace __utils
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_PARALLEL_BACKEND_UTILS_H */
diff --git a/pstl/include/pstl/internal/parallel_impl.h b/pstl/include/pstl/internal/parallel_impl.h
deleted file mode 100644
index 76b3f43e0d52..000000000000
--- a/pstl/include/pstl/internal/parallel_impl.h
+++ /dev/null
@@ -1,90 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_PARALLEL_IMPL_H
-#define _PSTL_PARALLEL_IMPL_H
-
-#include "pstl_config.h"
-
-#include <atomic>
-// This header defines the minimum set of parallel routines required to support Parallel STL,
-// implemented on top of Intel(R) Threading Building Blocks (Intel(R) TBB) library
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-//------------------------------------------------------------------------
-// parallel_find
-//-----------------------------------------------------------------------
-/** Return extremum value returned by brick f[i,j) for subranges [i,j) of [first,last)
-Each f[i,j) must return a value in [i,j). */
-template <class _BackendTag, class _ExecutionPolicy, class _Index, class _Brick, class _Compare>
-_Index
-__parallel_find(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Index __last, _Brick __f,
-                _Compare __comp, bool __b_first)
-{
-    typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
-    const _DifferenceType __n = __last - __first;
-    _DifferenceType __initial_dist = __b_first ? __n : -1;
-    std::atomic<_DifferenceType> __extremum(__initial_dist);
-    // TODO: find out what is better here: parallel_for or parallel_reduce
-    __par_backend::__parallel_for(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                  [__comp, __f, __first, &__extremum](_Index __i, _Index __j)
-                                  {
-                                      // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of
-                                      // why using a shared variable scales fairly well in this situation.
-                                      if (__comp(__i - __first, __extremum))
-                                      {
-                                          _Index __res = __f(__i, __j);
-                                          // If not '__last' returned then we found what we want so put this to extremum
-                                          if (__res != __j)
-                                          {
-                                              const _DifferenceType __k = __res - __first;
-                                              for (_DifferenceType __old = __extremum; __comp(__k, __old);
-                                                   __old = __extremum)
-                                              {
-                                                  __extremum.compare_exchange_weak(__old, __k);
-                                              }
-                                          }
-                                      }
-                                  });
-    return __extremum != __initial_dist ? __first + __extremum : __last;
-}
-
-//------------------------------------------------------------------------
-// parallel_or
-//------------------------------------------------------------------------
-//! Return true if brick f[i,j) returns true for some subrange [i,j) of [first,last)
-template <class _BackendTag, class _ExecutionPolicy, class _Index, class _Brick>
-bool
-__parallel_or(_BackendTag __tag, _ExecutionPolicy&& __exec, _Index __first, _Index __last, _Brick __f)
-{
-    std::atomic<bool> __found(false);
-    __par_backend::__parallel_for(__tag, std::forward<_ExecutionPolicy>(__exec), __first, __last,
-                                  [__f, &__found](_Index __i, _Index __j)
-                                  {
-                                      if (!__found.load(std::memory_order_relaxed) && __f(__i, __j))
-                                      {
-                                          __found.store(true, std::memory_order_relaxed);
-                                          __par_backend::__cancel_execution();
-                                      }
-                                  });
-    return __found;
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_PARALLEL_IMPL_H */
diff --git a/pstl/include/pstl/internal/pstl_config.h b/pstl/include/pstl/internal/pstl_config.h
deleted file mode 100644
index 830322695c74..000000000000
--- a/pstl/include/pstl/internal/pstl_config.h
+++ /dev/null
@@ -1,204 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_CONFIG_H
-#define _PSTL_CONFIG_H
-
-#include <__pstl_config_site>
-
-// The version is XYYZ, where X is major, YY is minor, and Z is patch (i.e. X.YY.Z)
-#define _PSTL_VERSION 17000
-#define _PSTL_VERSION_MAJOR (_PSTL_VERSION / 1000)
-#define _PSTL_VERSION_MINOR ((_PSTL_VERSION % 1000) / 10)
-#define _PSTL_VERSION_PATCH (_PSTL_VERSION % 10)
-
-#if !defined(_PSTL_PAR_BACKEND_SERIAL) && !defined(_PSTL_PAR_BACKEND_TBB) && !defined(_PSTL_PAR_BACKEND_OPENMP)
-#    error "A parallel backend must be specified"
-#endif
-
-// Check the user-defined macro for warnings
-#if defined(PSTL_USAGE_WARNINGS)
-#    define _PSTL_USAGE_WARNINGS
-#endif
-
-#if defined(_LIBCPP_VERSION)
-#    include <__assert>
-#    define _PSTL_ASSERT(pred) _LIBCPP_ASSERT(pred, "")
-#elif defined(__GLIBCXX__)
-#    define _PSTL_ASSERT(pred) __glibcxx_assert(pred)
-#else
-#    include <cassert>
-#    define _PSTL_ASSERT(pred) (assert((pred)))
-#endif
-
-// Portability "#pragma" definition
-#ifdef _MSC_VER
-#    define _PSTL_PRAGMA(x) __pragma(x)
-#else
-#    define _PSTL_PRAGMA(x) _Pragma(#    x)
-#endif
-
-#define _PSTL_STRING_AUX(x) #x
-#define _PSTL_STRING(x) _PSTL_STRING_AUX(x)
-#define _PSTL_STRING_CONCAT(x, y) x #y
-
-#ifdef _PSTL_HIDE_FROM_ABI_PER_TU
-#    define _PSTL_HIDE_FROM_ABI_PUSH                                                                                   \
-        _Pragma("clang attribute push(__attribute__((internal_linkage)), apply_to=any(function,record))")
-#    define _PSTL_HIDE_FROM_ABI_POP _Pragma("clang attribute pop")
-#else
-#    define _PSTL_HIDE_FROM_ABI_PUSH /* nothing */
-#    define _PSTL_HIDE_FROM_ABI_POP  /* nothing */
-#endif
-
-// note that when ICC or Clang is in use, _PSTL_GCC_VERSION might not fully match
-// the actual GCC version on the system.
-#define _PSTL_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
-
-#if defined(__clang__)
-// according to clang documentation, version can be vendor specific
-#    define _PSTL_CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
-#endif
-
-// Enable SIMD for compilers that support OpenMP 4.0
-#if (defined(_OPENMP) && _OPENMP >= 201307) || \
-    (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1600) || \
-    (!defined(__INTEL_COMPILER) && _PSTL_GCC_VERSION >= 40900) || \
-    defined(__clang__)
-#    define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(omp simd)
-#    define _PSTL_PRAGMA_DECLARE_SIMD _PSTL_PRAGMA(omp declare simd)
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(omp simd reduction(PRM))
-#elif !defined(_MSC_VER) //#pragma simd
-#    define _PSTL_PRAGMA_SIMD _PSTL_PRAGMA(simd)
-#    define _PSTL_PRAGMA_DECLARE_SIMD
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM) _PSTL_PRAGMA(simd reduction(PRM))
-#else //no simd
-#    define _PSTL_PRAGMA_SIMD
-#    define _PSTL_PRAGMA_DECLARE_SIMD
-#    define _PSTL_PRAGMA_SIMD_REDUCTION(PRM)
-#endif //Enable SIMD
-
-#if defined(__INTEL_COMPILER)
-#    define _PSTL_PRAGMA_FORCEINLINE _PSTL_PRAGMA(forceinline)
-#else
-#    define _PSTL_PRAGMA_FORCEINLINE
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900
-#    define _PSTL_PRAGMA_SIMD_SCAN(PRM) _PSTL_PRAGMA(omp simd reduction(inscan, PRM))
-#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan inclusive(PRM))
-#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM) _PSTL_PRAGMA(omp scan exclusive(PRM))
-#else
-#    define _PSTL_PRAGMA_SIMD_SCAN(PRM)
-#    define _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(PRM)
-#    define _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(PRM)
-#endif
-
-// Should be defined to 1 for environments with a vendor implementation of C++17 execution policies
-#define _PSTL_CPP17_EXECUTION_POLICIES_PRESENT (_MSC_VER >= 1912 && _MSVC_LANG >= 201703L) ||                          \
-    (_GLIBCXX_RELEASE >= 9 && __GLIBCXX__ >= 20190503 && __cplusplus >= 201703L)
-
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || \
-    __cplusplus >= 201300L || \
-    __cpp_lib_robust_nonmodifying_seq_ops == 201304
-#   define _PSTL_CPP14_2RANGE_MISMATCH_EQUAL_PRESENT
-#endif
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || \
-    __cplusplus >= 201402L || \
-    __cpp_lib_make_reverse_iterator == 201402
-#   define _PSTL_CPP14_MAKE_REVERSE_ITERATOR_PRESENT
-#endif
-#if (defined(_MSC_VER) && _MSC_VER >= 1900) || __cplusplus >= 201402L
-#   define _PSTL_CPP14_INTEGER_SEQUENCE_PRESENT
-#endif
-#if (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1700) || \
-    (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 190023918) || \
-    __cplusplus >= 201402L
-#   define _PSTL_CPP14_VARIABLE_TEMPLATES_PRESENT
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1800
-#   define _PSTL_EARLYEXIT_PRESENT
-#   define _PSTL_MONOTONIC_PRESENT
-#endif
-
-#if (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900) || \
-    (!defined(__INTEL_COMPILER) && _PSTL_GCC_VERSION >= 40900) || \
-    (defined(_OPENMP) && _OPENMP >= 201307)
-#    define _PSTL_UDR_PRESENT
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1900 && __INTEL_COMPILER_BUILD_DATE >= 20180626
-#   define _PSTL_UDS_PRESENT
-#endif
-
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-#    define _PSTL_PRAGMA_SIMD_EARLYEXIT _PSTL_PRAGMA(omp simd early_exit)
-#else
-#    define _PSTL_PRAGMA_SIMD_EARLYEXIT
-#endif
-
-#if defined(_PSTL_MONOTONIC_PRESENT)
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(PRM) _PSTL_PRAGMA(omp ordered simd monotonic(PRM))
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(PRM1, PRM2) _PSTL_PRAGMA(omp ordered simd monotonic(PRM1, PRM2))
-#else
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(PRM)
-#    define _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(PRM1, PRM2)
-#endif
-
-// Declaration of reduction functor, where
-// NAME - the name of the functor
-// OP - type of the callable object with the reduction operation
-// omp_in - refers to the local partial result
-// omp_out - refers to the final value of the combiner operator
-// omp_priv - refers to the private copy of the initial value
-// omp_orig - refers to the original variable to be reduced
-#define _PSTL_PRAGMA_DECLARE_REDUCTION(NAME, OP)                                                                       \
-    _PSTL_PRAGMA(omp declare reduction(NAME:OP : omp_out(omp_in)) initializer(omp_priv = omp_orig))
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 1600
-#    define _PSTL_PRAGMA_VECTOR_UNALIGNED _PSTL_PRAGMA(vector unaligned)
-#else
-#    define _PSTL_PRAGMA_VECTOR_UNALIGNED
-#endif
-
-// Check the user-defined macro to use non-temporal stores
-#if defined(PSTL_USE_NONTEMPORAL_STORES) && (__INTEL_COMPILER >= 1600)
-#    define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED _PSTL_PRAGMA(vector nontemporal)
-#else
-#    define _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
-#endif
-
-#if defined(_MSC_VER) || defined(__INTEL_COMPILER) // the preprocessors don't type a message location
-#    define _PSTL_PRAGMA_LOCATION __FILE__ ":" _PSTL_STRING(__LINE__) ": [Parallel STL message]: "
-#else
-#    define _PSTL_PRAGMA_LOCATION " [Parallel STL message]: "
-#endif
-
-#define _PSTL_PRAGMA_MESSAGE_IMPL(x) _PSTL_PRAGMA(message(_PSTL_STRING_CONCAT(_PSTL_PRAGMA_LOCATION, x)))
-
-#if defined(_PSTL_USAGE_WARNINGS)
-#    define _PSTL_PRAGMA_MESSAGE(x) _PSTL_PRAGMA_MESSAGE_IMPL(x)
-#    define _PSTL_PRAGMA_MESSAGE_POLICIES(x) _PSTL_PRAGMA_MESSAGE_IMPL(x)
-#else
-#    define _PSTL_PRAGMA_MESSAGE(x)
-#    define _PSTL_PRAGMA_MESSAGE_POLICIES(x)
-#endif
-
-// broken macros
-#if (defined(__GLIBCXX__) && __GLIBCXX__ < 20150716) || \
-    (defined(_MSC_VER) && _MSC_VER < 1800)
-#   define _PSTL_CPP11_STD_ROTATE_BROKEN
-#endif
-
-#if defined(__INTEL_COMPILER) && __INTEL_COMPILER == 1800
-#   define _PSTL_ICC_18_OMP_SIMD_BROKEN
-#endif
-
-#endif /* _PSTL_CONFIG_H */
diff --git a/pstl/include/pstl/internal/unseq_backend_simd.h b/pstl/include/pstl/internal/unseq_backend_simd.h
deleted file mode 100644
index 9521a420609a..000000000000
--- a/pstl/include/pstl/internal/unseq_backend_simd.h
+++ /dev/null
@@ -1,862 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_UNSEQ_BACKEND_SIMD_H
-#define _PSTL_UNSEQ_BACKEND_SIMD_H
-
-#include <type_traits>
-
-#include "pstl_config.h"
-#include "utils.h"
-
-// This header defines the minimum set of vector routines required
-// to support parallel STL.
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __unseq_backend
-{
-
-// Expect vector width up to 64 (or 512 bit)
-const std::size_t __lane_size = 64;
-
-template <class _Iterator, class _DifferenceType, class _Function>
-_Iterator
-__simd_walk_1(_Iterator __first, _DifferenceType __n, _Function __f) noexcept
-{
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __f(__first[__i]);
-
-    return __first + __n;
-}
-
-template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Function>
-_Iterator2
-__simd_walk_2(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Function __f) noexcept
-{
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __f(__first1[__i], __first2[__i]);
-    return __first2 + __n;
-}
-
-template <class _Iterator1, class _DifferenceType, class _Iterator2, class _Iterator3, class _Function>
-_Iterator3
-__simd_walk_3(_Iterator1 __first1, _DifferenceType __n, _Iterator2 __first2, _Iterator3 __first3,
-              _Function __f) noexcept
-{
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __f(__first1[__i], __first2[__i], __first3[__i]);
-    return __first3 + __n;
-}
-
-// TODO: check whether __simd_first() can be used here
-template <class _Index, class _DifferenceType, class _Pred>
-bool
-__simd_or(_Index __first, _DifferenceType __n, _Pred __pred) noexcept
-{
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (__i = 0; __i < __n; ++__i)
-        if (__pred(__first[__i]))
-            break;
-    return __i < __n;
-#else
-    _DifferenceType __block_size = 4 < __n ? 4 : __n;
-    const _Index __last = __first + __n;
-    while (__last != __first)
-    {
-        int32_t __flag = 1;
-        _PSTL_PRAGMA_SIMD_REDUCTION(& : __flag)
-        for (_DifferenceType __i = 0; __i < __block_size; ++__i)
-            if (__pred(*(__first + __i)))
-                __flag = 0;
-        if (!__flag)
-            return true;
-
-        __first += __block_size;
-        if (__last - __first >= __block_size << 1)
-        {
-            // Double the block _Size.  Any unnecessary iterations can be amortized against work done so far.
-            __block_size <<= 1;
-        }
-        else
-        {
-            __block_size = __last - __first;
-        }
-    }
-    return false;
-#endif
-}
-
-template <class _Index, class _DifferenceType, class _Compare>
-_Index
-__simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept
-{
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i = __begin;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
-        _PSTL_PRAGMA_SIMD_EARLYEXIT for (; __i < __end; ++__i)
-    {
-        if (__comp(__first, __i))
-        {
-            break;
-        }
-    }
-    return __first + __i;
-#else
-    // Experiments show good block sizes like this
-    const _DifferenceType __block_size = 8;
-    alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
-    while (__end - __begin >= __block_size)
-    {
-        _DifferenceType __found = 0;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
-            _PSTL_PRAGMA_SIMD_REDUCTION(|
-                                        : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size;
-                                                        ++__i)
-        {
-            const _DifferenceType __t = __comp(__first, __i);
-            __lane[__i - __begin] = __t;
-            __found |= __t;
-        }
-        if (__found)
-        {
-            _DifferenceType __i;
-            // This will vectorize
-            for (__i = 0; __i < __block_size; ++__i)
-            {
-                if (__lane[__i])
-                {
-                    break;
-                }
-            }
-            return __first + __begin + __i;
-        }
-        __begin += __block_size;
-    }
-
-    //Keep remainder scalar
-    while (__begin != __end)
-    {
-        if (__comp(__first, __begin))
-        {
-            return __first + __begin;
-        }
-        ++__begin;
-    }
-    return __first + __end;
-#endif //_PSTL_EARLYEXIT_PRESENT
-}
-
-template <class _Index1, class _DifferenceType, class _Index2, class _Pred>
-std::pair<_Index1, _Index2>
-__simd_first(_Index1 __first1, _DifferenceType __n, _Index2 __first2, _Pred __pred) noexcept
-{
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    _DifferenceType __i = 0;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (; __i < __n; ++__i)
-        if (__pred(__first1[__i], __first2[__i]))
-            break;
-    return std::make_pair(__first1 + __i, __first2 + __i);
-#else
-    const _Index1 __last1 = __first1 + __n;
-    const _Index2 __last2 = __first2 + __n;
-    // Experiments show good block sizes like this
-    const _DifferenceType __block_size = 8;
-    alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
-    while (__last1 - __first1 >= __block_size)
-    {
-        _DifferenceType __found = 0;
-        _DifferenceType __i;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
-            _PSTL_PRAGMA_SIMD_REDUCTION(|
-                                        : __found) for (__i = 0; __i < __block_size; ++__i)
-        {
-            const _DifferenceType __t = __pred(__first1[__i], __first2[__i]);
-            __lane[__i] = __t;
-            __found |= __t;
-        }
-        if (__found)
-        {
-            _DifferenceType __i2;
-            // This will vectorize
-            for (__i2 = 0; __i2 < __block_size; ++__i2)
-            {
-                if (__lane[__i2])
-                    break;
-            }
-            return std::make_pair(__first1 + __i2, __first2 + __i2);
-        }
-        __first1 += __block_size;
-        __first2 += __block_size;
-    }
-
-    //Keep remainder scalar
-    for (; __last1 != __first1; ++__first1, ++__first2)
-        if (__pred(*(__first1), *(__first2)))
-            return std::make_pair(__first1, __first2);
-
-    return std::make_pair(__last1, __last2);
-#endif //_PSTL_EARLYEXIT_PRESENT
-}
-
-template <class _Index, class _DifferenceType, class _Pred>
-_DifferenceType
-__simd_count(_Index __index, _DifferenceType __n, _Pred __pred) noexcept
-{
-    _DifferenceType __count = 0;
-    _PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        if (__pred(*(__index + __i)))
-            ++__count;
-
-    return __count;
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _BinaryPredicate>
-_OutputIterator
-__simd_unique_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator __result,
-                   _BinaryPredicate __pred) noexcept
-{
-    if (__n == 0)
-        return __result;
-
-    _DifferenceType __cnt = 1;
-    __result[0] = __first[0];
-
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 1; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
-        if (!__pred(__first[__i], __first[__i - 1]))
-        {
-            __result[__cnt] = __first[__i];
-            ++__cnt;
-        }
-    }
-    return __result + __cnt;
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
-_OutputIterator
-__simd_assign(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _Assigner __assigner) noexcept
-{
-    _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __assigner(__first + __i, __result + __i);
-    return __result + __n;
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _UnaryPredicate>
-_OutputIterator
-__simd_copy_if(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, _UnaryPredicate __pred) noexcept
-{
-    _DifferenceType __cnt = 0;
-
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
-        if (__pred(__first[__i]))
-        {
-            __result[__cnt] = __first[__i];
-            ++__cnt;
-        }
-    }
-    return __result + __cnt;
-}
-
-template <class _InputIterator, class _DifferenceType, class _BinaryPredicate>
-_DifferenceType
-__simd_calc_mask_2(_InputIterator __first, _DifferenceType __n, bool* __mask, _BinaryPredicate __pred) noexcept
-{
-    _DifferenceType __count = 0;
-
-    _PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        __mask[__i] = !__pred(__first[__i], __first[__i - 1]);
-        __count += __mask[__i];
-    }
-    return __count;
-}
-
-template <class _InputIterator, class _DifferenceType, class _UnaryPredicate>
-_DifferenceType
-__simd_calc_mask_1(_InputIterator __first, _DifferenceType __n, bool* __mask, _UnaryPredicate __pred) noexcept
-{
-    _DifferenceType __count = 0;
-
-    _PSTL_PRAGMA_SIMD_REDUCTION(+ : __count)
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        __mask[__i] = __pred(__first[__i]);
-        __count += __mask[__i];
-    }
-    return __count;
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator, class _Assigner>
-void
-__simd_copy_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator __result, bool* __mask,
-                    _Assigner __assigner) noexcept
-{
-    _DifferenceType __cnt = 0;
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        if (__mask[__i])
-        {
-            _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
-            {
-                __assigner(__first + __i, __result + __cnt);
-                ++__cnt;
-            }
-        }
-    }
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2>
-void
-__simd_partition_by_mask(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
-                         _OutputIterator2 __out_false, bool* __mask) noexcept
-{
-    _DifferenceType __cnt_true = 0, __cnt_false = 0;
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(__cnt_true : 1, __cnt_false : 1)
-        if (__mask[__i])
-        {
-            __out_true[__cnt_true] = __first[__i];
-            ++__cnt_true;
-        }
-        else
-        {
-            __out_false[__cnt_false] = __first[__i];
-            ++__cnt_false;
-        }
-    }
-}
-
-template <class _Index, class _DifferenceType, class _Tp>
-_Index
-__simd_fill_n(_Index __first, _DifferenceType __n, const _Tp& __value) noexcept
-{
-    _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __first[__i] = __value;
-    return __first + __n;
-}
-
-template <class _Index, class _DifferenceType, class _Generator>
-_Index
-__simd_generate_n(_Index __first, _DifferenceType __size, _Generator __g) noexcept
-{
-    _PSTL_USE_NONTEMPORAL_STORES_IF_ALLOWED
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __size; ++__i)
-        __first[__i] = __g();
-    return __first + __size;
-}
-
-template <class _Index, class _BinaryPredicate>
-_Index
-__simd_adjacent_find(_Index __first, _Index __last, _BinaryPredicate __pred, bool __or_semantic) noexcept
-{
-    if (__last - __first < 2)
-        return __last;
-
-    typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
-    _DifferenceType __i = 0;
-
-#if defined(_PSTL_EARLYEXIT_PRESENT)
-    //Some compiler versions fail to compile the following loop when iterators are used. Indices are used instead
-    const _DifferenceType __n = __last - __first - 1;
-    _PSTL_PRAGMA_VECTOR_UNALIGNED
-    _PSTL_PRAGMA_SIMD_EARLYEXIT
-    for (; __i < __n; ++__i)
-        if (__pred(__first[__i], __first[__i + 1]))
-            break;
-
-    return __i < __n ? __first + __i : __last;
-#else
-    // Experiments show good block sizes like this
-    //TODO: to consider tuning block_size for various data types
-    const _DifferenceType __block_size = 8;
-    alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
-    while (__last - __first >= __block_size)
-    {
-        _DifferenceType __found = 0;
-        _PSTL_PRAGMA_VECTOR_UNALIGNED // Do not generate peel loop part
-            _PSTL_PRAGMA_SIMD_REDUCTION(|
-                                        : __found) for (__i = 0; __i < __block_size - 1; ++__i)
-        {
-            //TODO: to improve SIMD vectorization
-            const _DifferenceType __t = __pred(*(__first + __i), *(__first + __i + 1));
-            __lane[__i] = __t;
-            __found |= __t;
-        }
-
-        //Process a pair of elements on a boundary of a data block
-        if (__first + __block_size < __last && __pred(*(__first + __i), *(__first + __i + 1)))
-            __lane[__i] = __found = 1;
-
-        if (__found)
-        {
-            if (__or_semantic)
-                return __first;
-
-            // This will vectorize
-            for (__i = 0; __i < __block_size; ++__i)
-                if (__lane[__i])
-                    break;
-            return __first + __i; //As far as found is true a __result (__lane[__i] is true) is guaranteed
-        }
-        __first += __block_size;
-    }
-    //Process the rest elements
-    for (; __last - __first > 1; ++__first)
-        if (__pred(*__first, *(__first + 1)))
-            return __first;
-
-    return __last;
-#endif
-}
-
-// It was created to reduce the code inside std::enable_if
-template <typename _Tp, typename _BinaryOperation>
-using is_arithmetic_plus = std::integral_constant<bool, std::is_arithmetic<_Tp>::value &&
-                                                            std::is_same<_BinaryOperation, std::plus<_Tp>>::value>;
-
-template <typename _DifferenceType, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
-typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
-__simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _UnaryOperation __f) noexcept
-{
-    _PSTL_PRAGMA_SIMD_REDUCTION(+ : __init)
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-        __init += __f(__i);
-    return __init;
-}
-
-template <typename _Size, typename _Tp, typename _BinaryOperation, typename _UnaryOperation>
-typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, _Tp>::type
-__simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept
-{
-    const _Size __block_size = __lane_size / sizeof(_Tp);
-    if (__n > 2 * __block_size && __block_size > 1)
-    {
-        alignas(__lane_size) char __lane_[__lane_size];
-        _Tp* __lane = reinterpret_cast<_Tp*>(__lane_);
-
-        // initializer
-        _PSTL_PRAGMA_SIMD
-        for (_Size __i = 0; __i < __block_size; ++__i)
-        {
-            ::new (__lane + __i) _Tp(__binary_op(__f(__i), __f(__block_size + __i)));
-        }
-        // main loop
-        _Size __i = 2 * __block_size;
-        const _Size last_iteration = __block_size * (__n / __block_size);
-        for (; __i < last_iteration; __i += __block_size)
-        {
-            _PSTL_PRAGMA_SIMD
-            for (_Size __j = 0; __j < __block_size; ++__j)
-            {
-                __lane[__j] = __binary_op(__lane[__j], __f(__i + __j));
-            }
-        }
-        // remainder
-        _PSTL_PRAGMA_SIMD
-        for (_Size __j = 0; __j < __n - last_iteration; ++__j)
-        {
-            __lane[__j] = __binary_op(__lane[__j], __f(last_iteration + __j));
-        }
-        // combiner
-        for (_Size __j = 0; __j < __block_size; ++__j)
-        {
-            __init = __binary_op(__init, __lane[__j]);
-        }
-        // destroyer
-        _PSTL_PRAGMA_SIMD
-        for (_Size __j = 0; __j < __block_size; ++__j)
-        {
-            __lane[__j].~_Tp();
-        }
-    }
-    else
-    {
-        for (_Size __i = 0; __i < __n; ++__i)
-        {
-            __init = __binary_op(__init, __f(__i));
-        }
-    }
-    return __init;
-}
-
-// Exclusive scan for "+" and arithmetic types
-template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
-          class _BinaryOperation>
-typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-            _BinaryOperation, /*Inclusive*/ std::false_type)
-{
-    _PSTL_PRAGMA_SIMD_SCAN(+ : __init)
-    for (_Size __i = 0; __i < __n; ++__i)
-    {
-        __result[__i] = __init;
-        _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__init)
-        __init += __unary_op(__first[__i]);
-    }
-    return std::make_pair(__result + __n, __init);
-}
-
-// As soon as we cannot call __binary_op in "combiner" we create a wrapper over _Tp to encapsulate __binary_op
-template <typename _Tp, typename _BinaryOp>
-struct _Combiner
-{
-    _Tp __value_;
-    _BinaryOp* __bin_op; // Here is a pointer to function because of default ctor
-
-    _Combiner() : __value_{}, __bin_op(nullptr) {}
-    _Combiner(const _Tp& value, const _BinaryOp* bin_op) : __value_(value), __bin_op(const_cast<_BinaryOp*>(bin_op)) {}
-    _Combiner(const _Combiner& __obj) : __value_{}, __bin_op(__obj.__bin_op) {}
-
-    void
-    operator()(const _Combiner& __obj)
-    {
-        __value_ = (*__bin_op)(__value_, __obj.__value);
-    }
-};
-
-// Exclusive scan for other binary operations and types
-template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
-          class _BinaryOperation>
-typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-            _BinaryOperation __binary_op, /*Inclusive*/ std::false_type)
-{
-    typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
-    _CombinerType __init_{__init, &__binary_op};
-
-    _PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
-
-    _PSTL_PRAGMA_SIMD_SCAN(__bin_op : __init_)
-    for (_Size __i = 0; __i < __n; ++__i)
-    {
-        __result[__i] = __init_.__value_;
-        _PSTL_PRAGMA_SIMD_EXCLUSIVE_SCAN(__init_)
-        _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value_ = __binary_op(__init_.__value_, __unary_op(__first[__i]));
-    }
-    return std::make_pair(__result + __n, __init_.__value_);
-}
-
-// Inclusive scan for "+" and arithmetic types
-template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
-          class _BinaryOperation>
-typename std::enable_if<is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-            _BinaryOperation, /*Inclusive*/ std::true_type)
-{
-    _PSTL_PRAGMA_SIMD_SCAN(+ : __init)
-    for (_Size __i = 0; __i < __n; ++__i)
-    {
-        __init += __unary_op(__first[__i]);
-        _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__init)
-        __result[__i] = __init;
-    }
-    return std::make_pair(__result + __n, __init);
-}
-
-// Inclusive scan for other binary operations and types
-template <class _InputIterator, class _Size, class _OutputIterator, class _UnaryOperation, class _Tp,
-          class _BinaryOperation>
-typename std::enable_if<!is_arithmetic_plus<_Tp, _BinaryOperation>::value, std::pair<_OutputIterator, _Tp>>::type
-__simd_scan(_InputIterator __first, _Size __n, _OutputIterator __result, _UnaryOperation __unary_op, _Tp __init,
-            _BinaryOperation __binary_op, std::true_type)
-{
-    typedef _Combiner<_Tp, _BinaryOperation> _CombinerType;
-    _CombinerType __init_{__init, &__binary_op};
-
-    _PSTL_PRAGMA_DECLARE_REDUCTION(__bin_op, _CombinerType)
-
-    _PSTL_PRAGMA_SIMD_SCAN(__bin_op : __init_)
-    for (_Size __i = 0; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_FORCEINLINE
-        __init_.__value_ = __binary_op(__init_.__value_, __unary_op(__first[__i]));
-        _PSTL_PRAGMA_SIMD_INCLUSIVE_SCAN(__init_)
-        __result[__i] = __init_.__value_;
-    }
-    return std::make_pair(__result + __n, __init_.__value_);
-}
-
-// [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
-// complexity [violation] - We will have at most (__n-1 + number_of_lanes) comparisons instead of at most __n-1.
-template <typename _ForwardIterator, typename _Size, typename _Compare>
-_ForwardIterator
-__simd_min_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
-{
-    if (__n == 0)
-    {
-        return __first;
-    }
-
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _ValueType;
-    struct _ComplexType
-    {
-        _ValueType __min_val;
-        _Size __min_ind;
-        _Compare* __min_comp;
-
-        _ComplexType() : __min_val{}, __min_ind{}, __min_comp(nullptr) {}
-        _ComplexType(const _ValueType& val, const _Compare* comp)
-            : __min_val(val), __min_ind(0), __min_comp(const_cast<_Compare*>(comp))
-        {
-        }
-        _ComplexType(const _ComplexType& __obj)
-            : __min_val(__obj.__min_val), __min_ind(__obj.__min_ind), __min_comp(__obj.__min_comp)
-        {
-        }
-
-        _PSTL_PRAGMA_DECLARE_SIMD
-        void
-        operator()(const _ComplexType& __obj)
-        {
-            if (!(*__min_comp)(__min_val, __obj.__min_val) &&
-                ((*__min_comp)(__obj.__min_val, __min_val) || __obj.__min_ind - __min_ind < 0))
-            {
-                __min_val = __obj.__min_val;
-                __min_ind = __obj.__min_ind;
-            }
-        }
-    };
-
-    _ComplexType __init{*__first, &__comp};
-
-    _PSTL_PRAGMA_DECLARE_REDUCTION(__min_func, _ComplexType)
-
-    _PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
-    for (_Size __i = 1; __i < __n; ++__i)
-    {
-        const _ValueType __min_val = __init.__min_val;
-        const _ValueType __current = __first[__i];
-        if (__comp(__current, __min_val))
-        {
-            __init.__min_val = __current;
-            __init.__min_ind = __i;
-        }
-    }
-    return __first + __init.__min_ind;
-}
-
-// [restriction] - std::iterator_traits<_ForwardIterator>::value_type should be DefaultConstructible.
-// complexity [violation] - We will have at most (2*(__n-1) + 4*number_of_lanes) comparisons instead of at most [1.5*(__n-1)].
-template <typename _ForwardIterator, typename _Size, typename _Compare>
-std::pair<_ForwardIterator, _ForwardIterator>
-__simd_minmax_element(_ForwardIterator __first, _Size __n, _Compare __comp) noexcept
-{
-    if (__n == 0)
-    {
-        return std::make_pair(__first, __first);
-    }
-    typedef typename std::iterator_traits<_ForwardIterator>::value_type _ValueType;
-
-    struct _ComplexType
-    {
-        _ValueType __min_val;
-        _ValueType __max_val;
-        _Size __min_ind;
-        _Size __max_ind;
-        _Compare* __minmax_comp;
-
-        _ComplexType() : __min_val{}, __max_val{}, __min_ind{}, __max_ind{}, __minmax_comp(nullptr) {}
-        _ComplexType(const _ValueType& min_val, const _ValueType& max_val, const _Compare* comp)
-            : __min_val(min_val), __max_val(max_val), __min_ind(0), __max_ind(0),
-              __minmax_comp(const_cast<_Compare*>(comp))
-        {
-        }
-        _ComplexType(const _ComplexType& __obj)
-            : __min_val(__obj.__min_val), __max_val(__obj.__max_val), __min_ind(__obj.__min_ind),
-              __max_ind(__obj.__max_ind), __minmax_comp(__obj.__minmax_comp)
-        {
-        }
-
-        void
-        operator()(const _ComplexType& __obj)
-        {
-            // min
-            if ((*__minmax_comp)(__obj.__min_val, __min_val))
-            {
-                __min_val = __obj.__min_val;
-                __min_ind = __obj.__min_ind;
-            }
-            else if (!(*__minmax_comp)(__min_val, __obj.__min_val))
-            {
-                __min_val = __obj.__min_val;
-                __min_ind = (__min_ind - __obj.__min_ind < 0) ? __min_ind : __obj.__min_ind;
-            }
-
-            // max
-            if ((*__minmax_comp)(__max_val, __obj.__max_val))
-            {
-                __max_val = __obj.__max_val;
-                __max_ind = __obj.__max_ind;
-            }
-            else if (!(*__minmax_comp)(__obj.__max_val, __max_val))
-            {
-                __max_val = __obj.__max_val;
-                __max_ind = (__max_ind - __obj.__max_ind < 0) ? __obj.__max_ind : __max_ind;
-            }
-        }
-    };
-
-    _ComplexType __init{*__first, *__first, &__comp};
-
-    _PSTL_PRAGMA_DECLARE_REDUCTION(__min_func, _ComplexType);
-
-    _PSTL_PRAGMA_SIMD_REDUCTION(__min_func : __init)
-    for (_Size __i = 1; __i < __n; ++__i)
-    {
-        auto __min_val = __init.__min_val;
-        auto __max_val = __init.__max_val;
-        auto __current = __first + __i;
-        if (__comp(*__current, __min_val))
-        {
-            __init.__min_val = *__current;
-            __init.__min_ind = __i;
-        }
-        else if (!__comp(*__current, __max_val))
-        {
-            __init.__max_val = *__current;
-            __init.__max_ind = __i;
-        }
-    }
-    return std::make_pair(__first + __init.__min_ind, __first + __init.__max_ind);
-}
-
-template <class _InputIterator, class _DifferenceType, class _OutputIterator1, class _OutputIterator2,
-          class _UnaryPredicate>
-std::pair<_OutputIterator1, _OutputIterator2>
-__simd_partition_copy(_InputIterator __first, _DifferenceType __n, _OutputIterator1 __out_true,
-                      _OutputIterator2 __out_false, _UnaryPredicate __pred) noexcept
-{
-    _DifferenceType __cnt_true = 0, __cnt_false = 0;
-
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 0; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC_2ARGS(__cnt_true : 1, __cnt_false : 1)
-        if (__pred(__first[__i]))
-        {
-            __out_true[__cnt_true] = __first[__i];
-            ++__cnt_true;
-        }
-        else
-        {
-            __out_false[__cnt_false] = __first[__i];
-            ++__cnt_false;
-        }
-    }
-    return std::make_pair(__out_true + __cnt_true, __out_false + __cnt_false);
-}
-
-template <class _ForwardIterator1, class _ForwardIterator2, class _BinaryPredicate>
-_ForwardIterator1
-__simd_find_first_of(_ForwardIterator1 __first, _ForwardIterator1 __last, _ForwardIterator2 __s_first,
-                     _ForwardIterator2 __s_last, _BinaryPredicate __pred) noexcept
-{
-    typedef typename std::iterator_traits<_ForwardIterator1>::difference_type _DifferencType;
-
-    const _DifferencType __n1 = __last - __first;
-    const _DifferencType __n2 = __s_last - __s_first;
-    if (__n1 == 0 || __n2 == 0)
-    {
-        return __last; // according to the standard
-    }
-
-    // Common case
-    // If first sequence larger than second then we'll run simd_first with parameters of first sequence.
-    // Otherwise, vice versa.
-    if (__n1 < __n2)
-    {
-        for (; __first != __last; ++__first)
-        {
-            if (__unseq_backend::__simd_or(
-                    __s_first, __n2,
-                    __internal::__equal_value_by_pred<decltype(*__first), _BinaryPredicate>(*__first, __pred)))
-            {
-                return __first;
-            }
-        }
-    }
-    else
-    {
-        for (; __s_first != __s_last; ++__s_first)
-        {
-            const auto __result = __unseq_backend::__simd_first(
-                __first, _DifferencType(0), __n1, [__s_first, &__pred](_ForwardIterator1 __it, _DifferencType __i) {
-                    return __pred(__it[__i], *__s_first);
-                });
-            if (__result != __last)
-            {
-                return __result;
-            }
-        }
-    }
-    return __last;
-}
-
-template <class _RandomAccessIterator, class _DifferenceType, class _UnaryPredicate>
-_RandomAccessIterator
-__simd_remove_if(_RandomAccessIterator __first, _DifferenceType __n, _UnaryPredicate __pred) noexcept
-{
-    // find first element we need to remove
-    auto __current = __unseq_backend::__simd_first(
-        __first, _DifferenceType(0), __n,
-        [&__pred](_RandomAccessIterator __it, _DifferenceType __i) { return __pred(__it[__i]); });
-    __n -= __current - __first;
-
-    // if we have in sequence only one element that pred(__current[1]) != false we can exit the function
-    if (__n < 2)
-    {
-        return __current;
-    }
-
-    _DifferenceType __cnt = 0;
-    _PSTL_PRAGMA_SIMD
-    for (_DifferenceType __i = 1; __i < __n; ++__i)
-    {
-        _PSTL_PRAGMA_SIMD_ORDERED_MONOTONIC(__cnt : 1)
-        if (!__pred(__current[__i]))
-        {
-            __current[__cnt] = std::move(__current[__i]);
-            ++__cnt;
-        }
-    }
-    return __current + __cnt;
-}
-} // namespace __unseq_backend
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_UNSEQ_BACKEND_SIMD_H */
diff --git a/pstl/include/pstl/internal/utils.h b/pstl/include/pstl/internal/utils.h
deleted file mode 100644
index ec9d46713a68..000000000000
--- a/pstl/include/pstl/internal/utils.h
+++ /dev/null
@@ -1,177 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_UTILS_H
-#define _PSTL_UTILS_H
-
-#include <new>
-#include <iterator>
-
-_PSTL_HIDE_FROM_ABI_PUSH
-
-namespace __pstl
-{
-namespace __internal
-{
-
-template <typename _Fp>
-auto
-__except_handler(_Fp __f) -> decltype(__f())
-{
-    try
-    {
-        return __f();
-    }
-    catch (const std::bad_alloc&)
-    {
-        throw; // re-throw bad_alloc according to the standard [algorithms.parallel.exceptions]
-    }
-    catch (...)
-    {
-        std::terminate(); // Good bye according to the standard [algorithms.parallel.exceptions]
-    }
-}
-
-template <typename _Fp>
-void
-__invoke_if(std::true_type, _Fp __f)
-{
-    __f();
-}
-
-template <typename _Fp>
-void __invoke_if(std::false_type, _Fp)
-{
-}
-
-template <typename _Fp>
-void
-__invoke_if_not(std::false_type, _Fp __f)
-{
-    __f();
-}
-
-template <typename _Fp>
-void __invoke_if_not(std::true_type, _Fp)
-{
-}
-
-template <typename _F1, typename _F2>
-auto
-__invoke_if_else(std::true_type, _F1 __f1, _F2) -> decltype(__f1())
-{
-    return __f1();
-}
-
-template <typename _F1, typename _F2>
-auto
-__invoke_if_else(std::false_type, _F1, _F2 __f2) -> decltype(__f2())
-{
-    return __f2();
-}
-
-//! Unary operator that returns reference to its argument.
-struct __no_op
-{
-    template <typename _Tp>
-    _Tp&&
-    operator()(_Tp&& __a) const
-    {
-        return std::forward<_Tp>(__a);
-    }
-};
-
-template <typename _Pred>
-class __reorder_pred
-{
-    _Pred _M_pred;
-
-  public:
-    explicit __reorder_pred(_Pred __pred) : _M_pred(__pred) {}
-
-    template <typename _FTp, typename _STp>
-    bool
-    operator()(_FTp&& __a, _STp&& __b)
-    {
-        return _M_pred(std::forward<_STp>(__b), std::forward<_FTp>(__a));
-    }
-};
-
-//! Like a polymorphic lambda for pred(...,value)
-template <typename _Tp, typename _Predicate>
-class __equal_value_by_pred
-{
-    const _Tp& _M_value;
-    _Predicate _M_pred;
-
-  public:
-    __equal_value_by_pred(const _Tp& __value, _Predicate __pred) : _M_value(__value), _M_pred(__pred) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg)
-    {
-        return _M_pred(std::forward<_Arg>(__arg), _M_value);
-    }
-};
-
-//! Like a polymorphic lambda for ==value
-template <typename _Tp>
-class __equal_value
-{
-    const _Tp& _M_value;
-
-  public:
-    explicit __equal_value(const _Tp& __value) : _M_value(__value) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg) const
-    {
-        return std::forward<_Arg>(__arg) == _M_value;
-    }
-};
-
-//! Logical negation of ==value
-template <typename _Tp>
-class __not_equal_value
-{
-    const _Tp& _M_value;
-
-  public:
-    explicit __not_equal_value(const _Tp& __value) : _M_value(__value) {}
-
-    template <typename _Arg>
-    bool
-    operator()(_Arg&& __arg) const
-    {
-        return !(std::forward<_Arg>(__arg) == _M_value);
-    }
-};
-
-template <typename _ForwardIterator, typename _Compare>
-_ForwardIterator
-__cmp_iterators_by_values(_ForwardIterator __a, _ForwardIterator __b, _Compare __comp)
-{
-    if (__a < __b)
-    { // we should return closer iterator
-        return __comp(*__b, *__a) ? __b : __a;
-    }
-    else
-    {
-        return __comp(*__a, *__b) ? __a : __b;
-    }
-}
-
-} // namespace __internal
-} // namespace __pstl
-
-_PSTL_HIDE_FROM_ABI_POP
-
-#endif /* _PSTL_UTILS_H */
diff --git a/pstl/test/CMakeLists.txt b/pstl/test/CMakeLists.txt
deleted file mode 100644
index bc027c30d085..000000000000
--- a/pstl/test/CMakeLists.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-#===-- CMakeLists.txt ----------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===----------------------------------------------------------------------===##
-
-# TODO(ldionne): This CMake testing infrastructure should be replaced with a
-#                llvm-lit test suite.
-
-add_custom_target(pstl-build-tests
-    COMMENT "Build all the pstl tests.")
-
-add_custom_target(check-pstl
-    COMMAND "${CMAKE_CTEST_COMMAND}" --output-on-failure
-    USES_TERMINAL
-    DEPENDS pstl-build-tests
-    COMMENT "Build and run all the unit tests.")
-
-add_library(test_stdlib INTERFACE)
-target_include_directories(test_stdlib INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/support/stdlib")
-target_link_libraries(test_stdlib INTERFACE pstl::ParallelSTL)
-target_compile_options(test_stdlib INTERFACE -Wno-gnu-include-next)
-
-file(GLOB_RECURSE UNIT_TESTS "*.pass.cpp")
-foreach(_file IN LISTS UNIT_TESTS)
-    file(RELATIVE_PATH _target "${CMAKE_CURRENT_SOURCE_DIR}" "${_file}")
-    string(REPLACE ".cpp" "" _target "${_target}")
-    string(REPLACE "/" "-" _target "${_target}")
-    set(_target "pstl-${_target}")
-
-    add_executable(${_target} EXCLUDE_FROM_ALL "${_file}")
-    target_include_directories(${_target} PRIVATE "${CMAKE_CURRENT_LIST_DIR}")
-    target_compile_options(${_target} PRIVATE -Wno-unused-local-typedef -Wno-unused-variable)
-    target_link_libraries(${_target} PRIVATE test_stdlib)
-    set_target_properties(${_target} PROPERTIES CXX_EXTENSIONS NO
-                                                RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
-    add_test(${_target} "${CMAKE_CURRENT_BINARY_DIR}/${_target}")
-    add_dependencies(pstl-build-tests ${_target})
-endforeach()
diff --git a/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
deleted file mode 100644
index 8cdeb18d6d75..000000000000
--- a/pstl/test/pstl/header_inclusion_order_algorithm_0.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_algorithm_0.pass.cpp -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
deleted file mode 100644
index ea6c4f49a0ae..000000000000
--- a/pstl/test/pstl/header_inclusion_order_algorithm_1.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_algorithm_1.pass.cpp -----------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <algorithm>
-#include <execution>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
deleted file mode 100644
index dbb70c5566bd..000000000000
--- a/pstl/test/pstl/header_inclusion_order_memory_0.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_memory_0.pass.cpp --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <memory>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
deleted file mode 100644
index 99f059c2e736..000000000000
--- a/pstl/test/pstl/header_inclusion_order_memory_1.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_memory_1.pass.cpp --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <memory>
-#include <execution>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp b/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
deleted file mode 100644
index 4b1865729bff..000000000000
--- a/pstl/test/pstl/header_inclusion_order_numeric_0.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_numeric_0.pass.cpp -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <numeric>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp b/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
deleted file mode 100644
index a0bb24e554ae..000000000000
--- a/pstl/test/pstl/header_inclusion_order_numeric_1.pass.cpp
+++ /dev/null
@@ -1,21 +0,0 @@
-// -*- C++ -*-
-//===-- header_inclusion_order_numeric_0.pass.cpp -------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <numeric>
-#include <execution>
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/pstl/version.pass.cpp b/pstl/test/pstl/version.pass.cpp
deleted file mode 100644
index 820dabcc243e..000000000000
--- a/pstl/test/pstl/version.pass.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include <pstl/internal/pstl_config.h>
-
-static_assert(_PSTL_VERSION == 17000);
-static_assert(_PSTL_VERSION_MAJOR == 17);
-static_assert(_PSTL_VERSION_MINOR == 00);
-static_assert(_PSTL_VERSION_PATCH == 0);
-
-int
-main()
-{
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp b/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
deleted file mode 100644
index 3446d955093a..000000000000
--- a/pstl/test/std/algorithms/alg.merge/inplace_merge.pass.cpp
+++ /dev/null
@@ -1,159 +0,0 @@
-// -*- C++ -*-
-//===-- inplace_merge.pass.cpp --------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename BiDirIt1, typename Size, typename Generator1, typename Generator2, typename Compare>
-    void
-    operator()(pstl::execution::unsequenced_policy, BiDirIt1 first1, BiDirIt1 last1, BiDirIt1 first2, BiDirIt1 last2,
-               Size n, Size m, Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-    }
-
-    template <typename BiDirIt1, typename Size, typename Generator1, typename Generator2, typename Compare>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, BiDirIt1 first1, BiDirIt1 last1, BiDirIt1 first2,
-               BiDirIt1 last2, Size n, Size m, Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-    }
-#endif
-
-    // inplace_merge works with bidirectional iterators at least
-    template <typename Policy, typename BiDirIt1, typename Size, typename Generator1, typename Generator2,
-              typename Compare>
-    typename std::enable_if<!is_same_iterator_category<BiDirIt1, std::forward_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, BiDirIt1 first1, BiDirIt1 last1, BiDirIt1 first2, BiDirIt1 last2, Size n, Size m,
-               Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-        const BiDirIt1 mid1 = std::next(first1, m);
-        fill_data(first1, mid1, generator1);
-        fill_data(mid1, last1, generator2);
-
-        const BiDirIt1 mid2 = std::next(first2, m);
-        fill_data(first2, mid2, generator1);
-        fill_data(mid2, last2, generator2);
-
-        std::inplace_merge(first1, mid1, last1, comp);
-        std::inplace_merge(exec, first2, mid2, last2, comp);
-        EXPECT_EQ_N(first1, first2, n, "wrong effect from inplace_merge with predicate");
-    }
-
-    template <typename Policy, typename BiDirIt1, typename Size, typename Generator1, typename Generator2,
-              typename Compare>
-    typename std::enable_if<is_same_iterator_category<BiDirIt1, std::forward_iterator_tag>::value, void>::type
-    operator()(Policy&&, BiDirIt1, BiDirIt1, BiDirIt1, BiDirIt1, Size, Size, Generator1, Generator2, Compare)
-    {
-    }
-};
-
-template <typename T, typename Generator1, typename Generator2, typename Compare>
-void
-test_by_type(Generator1 generator1, Generator2 generator2, Compare comp)
-{
-    using namespace std;
-    size_t max_size = 100000;
-    Sequence<T> in1(max_size, [](size_t v) { return T(v); });
-    Sequence<T> exp(max_size, [](size_t v) { return T(v); });
-    size_t m;
-
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        m = 0;
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n, exp.begin(), exp.begin() + n, n, m,
-                               generator1, generator2, comp);
-
-        m = n / 3;
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n, exp.begin(), exp.begin() + n, n, m,
-                               generator1, generator2, comp);
-
-        m = 2 * n / 3;
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n, exp.begin(), exp.begin() + n, n, m,
-                               generator1, generator2, comp);
-    }
-}
-
-template <typename T>
-struct LocalWrapper
-{
-    explicit LocalWrapper(int32_t k) : my_val(k) {}
-    LocalWrapper(LocalWrapper&& input) { my_val = std::move(input.my_val); }
-    LocalWrapper&
-    operator=(LocalWrapper&& input)
-    {
-        my_val = std::move(input.my_val);
-        return *this;
-    }
-    bool
-    operator<(const LocalWrapper<T>& w) const
-    {
-        return my_val < w.my_val;
-    }
-    friend bool
-    operator==(const LocalWrapper<T>& x, const LocalWrapper<T>& y)
-    {
-        return x.my_val == y.my_val;
-    }
-    friend std::ostream&
-    operator<<(std::ostream& stream, const LocalWrapper<T>& input)
-    {
-        return stream << input.my_val;
-    }
-
-  private:
-    T my_val;
-};
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() { inplace_merge(exec, iter, iter, iter, non_const(std::less<T>())); });
-    }
-};
-
-int
-main()
-{
-    test_by_type<float64_t>([](int32_t i) { return -2 * i; }, [](int32_t i) { return -(2 * i + 1); },
-                            [](const float64_t x, const float64_t y) { return x > y; });
-
-    test_by_type<int32_t>([](int32_t i) { return 10 * i; }, [](int32_t i) { return i + 1; }, std::less<int32_t>());
-
-    test_by_type<LocalWrapper<float32_t>>([](int32_t i) { return LocalWrapper<float32_t>(2 * i + 1); },
-                                          [](int32_t i) { return LocalWrapper<float32_t>(2 * i); },
-                                          std::less<LocalWrapper<float32_t>>());
-
-    test_algo_basic_single<int32_t>(run_for_rnd_bi<test_non_const<int32_t>>());
-
-    test_by_type<MemoryChecker>(
-        [](std::size_t idx){ return MemoryChecker{std::int32_t(idx * 2)}; },
-        [](std::size_t idx){ return MemoryChecker{std::int32_t(idx * 2 + 1)}; },
-        [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() == val2.value(); });
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from inplace_merge: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from inplace_merge: number of ctors calls > num of dtors calls");
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.merge/merge.pass.cpp b/pstl/test/std/algorithms/alg.merge/merge.pass.cpp
deleted file mode 100644
index a09ef4223bf7..000000000000
--- a/pstl/test/std/algorithms/alg.merge/merge.pass.cpp
+++ /dev/null
@@ -1,113 +0,0 @@
-// -*- C++ -*-
-//===-- merge.pass.cpp ----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-#include <functional>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_merge
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename OutputIterator,
-              typename Compare>
-    void
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               OutputIterator out_first, OutputIterator out_last, Compare comp)
-    {
-        using namespace std;
-        {
-            const auto res = merge(exec, first1, last1, first2, last2, out_first, comp);
-            EXPECT_TRUE(res == out_last, "wrong return result from merge with predicate");
-            EXPECT_TRUE(is_sorted(out_first, res, comp), "wrong result from merge with predicate");
-            EXPECT_TRUE(includes(out_first, res, first1, last1, comp), "first sequence is not a part of result");
-            EXPECT_TRUE(includes(out_first, res, first2, last2, comp), "second sequence is not a part of result");
-        }
-        {
-            const auto res = merge(exec, first1, last1, first2, last2, out_first);
-            EXPECT_TRUE(res == out_last, "wrong return result from merge");
-            EXPECT_TRUE(is_sorted(out_first, res), "wrong result from merge");
-        }
-    }
-
-    // for reverse iterators
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename OutputIterator,
-              typename Compare>
-    void
-    operator()(Policy&& exec, std::reverse_iterator<InputIterator1> first1, std::reverse_iterator<InputIterator1> last1,
-               std::reverse_iterator<InputIterator2> first2, std::reverse_iterator<InputIterator2> last2,
-               std::reverse_iterator<OutputIterator> out_first, std::reverse_iterator<OutputIterator> out_last, Compare)
-    {
-        using namespace std;
-        typedef typename std::iterator_traits<std::reverse_iterator<InputIterator1>>::value_type T;
-        const auto res = merge(exec, first1, last1, first2, last2, out_first, std::greater<T>());
-
-        EXPECT_TRUE(res == out_last, "wrong return result from merge with predicate");
-        EXPECT_TRUE(is_sorted(out_first, res, std::greater<T>()), "wrong result from merge with predicate");
-        EXPECT_TRUE(includes(out_first, res, first1, last1, std::greater<T>()),
-                    "first sequence is not a part of result");
-        EXPECT_TRUE(includes(out_first, res, first2, last2, std::greater<T>()),
-                    "second sequence is not a part of result");
-    }
-};
-
-template <typename T, typename Generator1, typename Generator2>
-void
-test_merge_by_type(Generator1 generator1, Generator2 generator2)
-{
-    using namespace std;
-    size_t max_size = 100000;
-    Sequence<T> in1(max_size, generator1);
-    Sequence<T> in2(max_size / 2, generator2);
-    Sequence<T> out(in1.size() + in2.size());
-    std::sort(in1.begin(), in1.end());
-    std::sort(in2.begin(), in2.end());
-
-    for (size_t size = 0; size <= max_size; size = size <= 16 ? size + 1 : size_t(3.1415 * size))
-    {
-        invoke_on_all_policies(test_merge(), in1.cbegin(), in1.cbegin() + size, in2.data(), in2.data() + size / 2,
-                               out.begin(), out.begin() + 1.5 * size, std::less<T>());
-        invoke_on_all_policies(test_merge(), in1.data(), in1.data() + size, in2.cbegin(), in2.cbegin() + size / 2,
-                               out.begin(), out.begin() + 3 * size / 2, std::less<T>());
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputIterator out_iter)
-    {
-        merge(exec, input_iter, input_iter, input_iter, input_iter, out_iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-    test_merge_by_type<int32_t>([](size_t v) { return (v % 2 == 0 ? v : -v) * 3; }, [](size_t v) { return v * 2; });
-    test_merge_by_type<float64_t>([](size_t v) { return float64_t(v); }, [](size_t v) { return float64_t(v - 100); });
-
-#if !defined(_PSTL_ICC_16_17_TEST_64_TIMEOUT)
-    test_merge_by_type<Wrapper<int16_t>>([](size_t v) { return Wrapper<int16_t>(v % 100); },
-                                         [](size_t v) { return Wrapper<int16_t>(v % 10); });
-#endif
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
deleted file mode 100644
index 692907e0f92e..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.copy/copy_if.pass.cpp
+++ /dev/null
@@ -1,147 +0,0 @@
-// -*- C++ -*-
-//===-- copy_if.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for copy_if and remove_copy_if
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_copy_if
-{
-#if defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN) // dummy specializations to skip testing in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(pstl::execution::parallel_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2 expected_last, Size n,
-               Predicate pred, T trash)
-    {
-    }
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, Size n, Predicate pred, T trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size n, Predicate pred,
-               T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-
-        // Run copy_if
-        auto i = copy_if(first, last, expected_first, pred);
-        auto k = copy_if(exec, first, last, out_first, pred);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong copy_if effect");
-        for (size_t j = 0; j < GuardSize; ++j)
-        {
-            ++k;
-        }
-        EXPECT_TRUE(out_last == k, "wrong return value from copy_if");
-
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-        // Run remove_copy_if
-        i = remove_copy_if(first, last, expected_first, [=](const T& x) { return !pred(x); });
-        k = remove_copy_if(exec, first, last, out_first, [=](const T& x) { return !pred(x); });
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong remove_copy_if effect");
-        for (size_t j = 0; j < GuardSize; ++j)
-        {
-            ++k;
-        }
-        EXPECT_TRUE(out_last == k, "wrong return value from remove_copy_if");
-    }
-};
-
-template <typename T, typename Predicate, typename Convert>
-void
-test(T trash, Predicate pred, Convert convert, bool check_weakness = true)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        // count is number of output elements, plus a handful
-        // more for sake of detecting buffer overruns.
-        size_t count = GuardSize;
-        Sequence<T> in(n, [&](size_t k) -> T {
-            T val = convert(n ^ k);
-            count += pred(val) ? 1 : 0;
-            return val;
-        });
-
-        Sequence<T> out(count, [=](size_t) { return trash; });
-        Sequence<T> expected(count, [=](size_t) { return trash; });
-        if (check_weakness)
-        {
-            auto expected_result = copy_if(in.cfbegin(), in.cfend(), expected.begin(), pred);
-            size_t m = expected_result - expected.begin();
-            EXPECT_TRUE(n / 4 <= m && m <= 3 * (n + 1) / 4, "weak test for copy_if");
-        }
-        invoke_on_all_policies(run_copy_if(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), count, pred, trash);
-        invoke_on_all_policies(run_copy_if(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), count, pred, trash);
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        copy_if(exec, input_iter, input_iter, out_iter, non_const(is_even));
-
-        invoke_if(exec, [&]() { remove_copy_if(exec, input_iter, input_iter, out_iter, non_const(is_even)); });
-    }
-};
-
-int
-main()
-{
-    test<float64_t>(-666.0, [](const float64_t& x) { return x * x <= 1024; },
-                    [](size_t j) { return ((j + 1) % 7 & 2) != 0 ? float64_t(j % 32) : float64_t(j % 33 + 34); });
-
-    test<int32_t>(-666, [](const int32_t& x) { return x != 42; },
-                  [](size_t j) { return ((j + 1) % 5 & 2) != 0 ? int32_t(j + 1) : 42; });
-
-#if !defined(_PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN)
-    test<Number>(Number(42, OddTag()), IsMultiple(3, OddTag()), [](int32_t j) { return Number(j, OddTag()); });
-#endif
-
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN)
-    test<int32_t>(-666, [](const int32_t&) { return true; }, [](size_t j) { return j; }, false);
-#endif
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
deleted file mode 100644
index bd6b259c4d0a..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/is_partitioned.pass.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-// -*- C++ -*-
-//===-- is_partitioned.pass.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-    //dummy specialization by policy type, in case of broken configuration
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) || defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN)
-
-    template <typename Iterator1, typename Predicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator1 begin1, Iterator1 end1, Predicate pred)
-    {
-    }
-    template <typename Iterator1, typename Predicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 begin1, Iterator1 end1, Predicate pred)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 begin1, Iterator1 end1, Predicate pred)
-    {
-        const bool expected = std::is_partitioned(begin1, end1, pred);
-        const bool actual = std::is_partitioned(exec, begin1, end1, pred);
-        EXPECT_TRUE(actual == expected, "wrong return result from is_partitioned");
-    }
-};
-
-template <typename T, typename Predicate>
-void
-test(Predicate pred)
-{
-
-    const std::size_t max_n = 1000000;
-    Sequence<T> in(max_n, [](std::size_t k) { return T(k); });
-
-    for (std::size_t n1 = 0; n1 <= max_n; n1 = n1 <= 16 ? n1 + 1 : std::size_t(3.1415 * n1))
-    {
-        invoke_on_all_policies(test_one_policy(), in.begin(), in.begin() + n1, pred);
-        std::partition(in.begin(), in.begin() + n1, pred);
-        invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cbegin() + n1, pred);
-    }
-}
-
-template <typename T>
-struct LocalWrapper
-{
-    explicit LocalWrapper(std::size_t k) : my_val(k) {}
-
-  private:
-    T my_val;
-};
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        invoke_if(exec, [&]() { is_partitioned(exec, iter, iter, non_const(is_even)); });
-    }
-};
-
-int
-main()
-{
-    test<float64_t>([](const float64_t x) { return x < 0; });
-    test<int32_t>([](const int32_t x) { return x > 1000; });
-    test<uint16_t>([](const uint16_t x) { return x % 5 < 3; });
-#if !defined(_PSTL_ICC_18_TEST_EARLY_EXIT_MONOTONIC_RELEASE_BROKEN) && !defined(_PSTL_ICC_19_TEST_IS_PARTITIONED_RELEASE_BROKEN)
-    test<LocalWrapper<float64_t>>([](const LocalWrapper<float64_t>&) { return true; });
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
deleted file mode 100644
index 724f0ba6d3f4..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition.pass.cpp
+++ /dev/null
@@ -1,178 +0,0 @@
-// -*- C++ -*-
-//===-- partition.pass.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for stable_partition and partition
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-#include <iterator>
-#include <type_traits>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct DataType
-{
-    explicit DataType(int32_t k) : my_val(k) {}
-    DataType(DataType&& input) { my_val = std::move(input.my_val); }
-    DataType&
-    operator=(DataType&& input)
-    {
-        my_val = std::move(input.my_val);
-        return *this;
-    }
-    T
-    get_val() const
-    {
-        return my_val;
-    }
-
-    friend std::ostream&
-    operator<<(std::ostream& stream, const DataType<T>& input)
-    {
-        return stream << input.my_val;
-    }
-
-  private:
-    T my_val;
-};
-
-template <typename Iterator>
-typename std::enable_if<std::is_trivial<typename std::iterator_traits<Iterator>::value_type>::value, bool>::type
-is_equal(Iterator first, Iterator last, Iterator d_first)
-{
-    return std::equal(first, last, d_first);
-}
-
-template <typename Iterator>
-typename std::enable_if<!std::is_trivial<typename std::iterator_traits<Iterator>::value_type>::value, bool>::type
-    is_equal(Iterator, Iterator, Iterator)
-{
-    return true;
-}
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specializations to skip testing in case of broken configuration
-    template <typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    void
-    operator()(pstl::execution::unsequenced_policy, BiDirIt first, BiDirIt last, BiDirIt exp_first, BiDirIt exp_last,
-               Size n, UnaryOp unary_op, Generator generator)
-    {
-    }
-
-    template <typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, BiDirIt first, BiDirIt last, BiDirIt exp_first,
-               BiDirIt exp_last, Size n, UnaryOp unary_op, Generator generator)
-    {
-    }
-#elif defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN) //dummy specializations to skip testing in case of broken configuration
-    template <typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    void
-    operator()(pstl::execution::parallel_policy, BiDirIt first, BiDirIt last, BiDirIt exp_first, BiDirIt exp_last,
-               Size n, UnaryOp unary_op, Generator generator)
-    {
-    }
-
-    template <typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, BiDirIt first, BiDirIt last, BiDirIt exp_first,
-               BiDirIt exp_last, Size n, UnaryOp unary_op, Generator generator)
-    {
-    }
-#endif
-
-    template <typename Policy, typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    typename std::enable_if<!is_same_iterator_category<BiDirIt, std::forward_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, BiDirIt first, BiDirIt last, BiDirIt exp_first, BiDirIt exp_last, Size, UnaryOp unary_op,
-               Generator generator)
-    {
-        // partition
-        {
-            fill_data(first, last, generator);
-            BiDirIt actual_ret = std::partition(exec, first, last, unary_op);
-            EXPECT_TRUE(std::all_of(first, actual_ret, unary_op) && !std::any_of(actual_ret, last, unary_op),
-                        "wrong effect from partition");
-        }
-        // stable_partition
-        {
-            fill_data(exp_first, exp_last, generator);
-            BiDirIt exp_ret = std::stable_partition(exp_first, exp_last, unary_op);
-            fill_data(first, last, generator);
-            BiDirIt actual_ret = std::stable_partition(exec, first, last, unary_op);
-
-            EXPECT_TRUE(std::distance(first, actual_ret) == std::distance(exp_first, exp_ret),
-                        "wrong result from stable_partition");
-            EXPECT_TRUE((is_equal<BiDirIt>(exp_first, exp_last, first)), "wrong effect from stable_partition");
-        }
-    }
-    template <typename Policy, typename BiDirIt, typename Size, typename UnaryOp, typename Generator>
-    typename std::enable_if<is_same_iterator_category<BiDirIt, std::forward_iterator_tag>::value, void>::type
-    operator()(Policy&&, BiDirIt, BiDirIt, BiDirIt, BiDirIt, Size, UnaryOp, Generator)
-    {
-    }
-};
-
-template <typename T, typename Generator, typename UnaryPred>
-void
-test_by_type(Generator generator, UnaryPred pred)
-{
-
-    using namespace std;
-    size_t max_size = 100000;
-    Sequence<T> in(max_size, [](size_t v) { return T(v); });
-    Sequence<T> exp(max_size, [](size_t v) { return T(v); });
-
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        invoke_on_all_policies(test_one_policy(), in.begin(), in.begin() + n, exp.begin(), exp.begin() + n, n, pred,
-                               generator);
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        invoke_if(exec, [&]() {
-            partition(exec, iter, iter, non_const(is_even));
-            stable_partition(exec, iter, iter, non_const(is_even));
-        });
-    }
-};
-
-int
-main()
-{
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN)
-    test_by_type<int32_t>([](int32_t i) { return i; }, [](int32_t) { return true; });
-#endif
-    test_by_type<float64_t>([](int32_t i) { return -i; }, [](const float64_t x) { return x < 0; });
-    test_by_type<int64_t>([](int32_t i) { return i + 1; }, [](int64_t x) { return x % 3 == 0; });
-    test_by_type<DataType<float32_t>>([](int32_t i) { return DataType<float32_t>(2 * i + 1); },
-                                      [](const DataType<float32_t>& x) { return x.get_val() < 0; });
-
-    test_algo_basic_single<int32_t>(run_for_rnd_bi<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
deleted file mode 100644
index c621f8352f1d..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.partitions/partition_copy.pass.cpp
+++ /dev/null
@@ -1,116 +0,0 @@
-// -*- C++ -*-
-//===-- partition_copy.pass.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for stable_partition and partition_copy
-#include "support/pstl_test_config.h"
-
-#include <algorithm>
-#include <cstdlib>
-#include <execution>
-#include <functional>
-#include <iterator>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_partition_copy
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2,
-              typename UnaryOp>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator true_first, OutputIterator,
-               OutputIterator2 false_first, OutputIterator2, UnaryOp unary_op)
-    {
-
-        auto actual_ret = std::partition_copy(exec, first, last, true_first, false_first, unary_op);
-
-        EXPECT_TRUE(std::distance(true_first, actual_ret.first) == std::count_if(first, last, unary_op),
-                    "partition_copy has wrong effect from true sequence");
-        EXPECT_TRUE(std::distance(false_first, actual_ret.second) == std::count_if(first, last, std::not_fn(unary_op)),
-                    "partition_copy has wrong effect from false sequence");
-    }
-
-    //dummy specialization by iterator type and policy type, in case of broken configuration
-#if defined(_PSTL_ICC_1800_TEST_MONOTONIC_RELEASE_64_BROKEN)
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename UnaryOp>
-    void
-    operator()(pstl::execution::unsequenced_policy, std::reverse_iterator<InputIterator> first,
-               std::reverse_iterator<InputIterator> last, std::reverse_iterator<OutputIterator> true_first,
-               std::reverse_iterator<OutputIterator> true_last, std::reverse_iterator<OutputIterator2> false_first,
-               OutputIterator2 false_last, UnaryOp unary_op)
-    {
-    }
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename UnaryOp>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, std::reverse_iterator<InputIterator> first,
-               std::reverse_iterator<InputIterator> last, std::reverse_iterator<OutputIterator> true_first,
-               std::reverse_iterator<OutputIterator> true_last, std::reverse_iterator<OutputIterator2> false_first,
-               OutputIterator2 false_last, UnaryOp unary_op)
-    {
-    }
-#endif
-};
-
-template <typename T, typename UnaryPred>
-void
-test(UnaryPred pred)
-{
-
-    const std::size_t max_size = 100000;
-    Sequence<T> in(max_size, [](std::size_t v) -> T { return T(v); });
-    Sequence<T> actual_true(max_size);
-    Sequence<T> actual_false(max_size);
-    for (std::size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : std::size_t(3.1415 * n))
-    {
-
-        // for non-const input iterators
-        invoke_on_all_policies(test_partition_copy(), in.begin(), in.begin() + n, actual_true.begin(),
-                               actual_true.begin() + n, actual_false.begin(), actual_false.begin() + n, pred);
-
-        // for const input iterators
-        invoke_on_all_policies(test_partition_copy(), in.cbegin(), in.cbegin() + n, actual_true.begin(),
-                               actual_true.begin() + n, actual_false.begin(), actual_false.begin() + n, pred);
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-
-        partition_copy(exec, input_iter, input_iter, out_iter, out_iter, non_const(is_even));
-    }
-};
-
-int
-main()
-{
-    test<int32_t>([](const int32_t value) { return value % 2; });
-
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN)
-    test<int32_t>([](const int32_t) { return true; });
-#endif
-
-    test<float64_t>([](const float64_t value) { return value > 2 << 6; });
-    test<Wrapper<float64_t>>([](const Wrapper<float64_t>& value) -> bool { return value.get_my_field() != nullptr; });
-
-    test_algo_basic_double<int32_t>(run_for_rnd_bi<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
deleted file mode 100644
index 126454fd7272..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse.pass.cpp
+++ /dev/null
@@ -1,104 +0,0 @@
-// -*- C++ -*-
-//===-- reverse.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_18_VC141_TEST_SIMD_LAMBDA_RELEASE_BROKEN) || defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||       \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Iterator2>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e)
-    {
-    }
-    template <typename Iterator1, typename Iterator2>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    typename std::enable_if<!is_same_iterator_category<Iterator1, std::forward_iterator_tag>::value>::type
-    operator()(ExecutionPolicy&& exec, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b, Iterator2 actual_e)
-    {
-        using namespace std;
-
-        copy(data_b, data_e, actual_b);
-
-        reverse(exec, actual_b, actual_e);
-
-        bool check = equal(data_b, data_e, reverse_iterator<Iterator2>(actual_e));
-
-        EXPECT_TRUE(check, "wrong result of reverse");
-    }
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::forward_iterator_tag>::value>::type
-    operator()(ExecutionPolicy&&, Iterator1, Iterator1, Iterator2, Iterator2)
-    {
-    }
-};
-
-template <typename T>
-void
-test()
-{
-    const std::size_t max_len = 100000;
-
-    Sequence<T> actual(max_len);
-
-    Sequence<T> data(max_len, [](std::size_t i) { return T(i); });
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        invoke_on_all_policies(test_one_policy(), data.begin(), data.begin() + len, actual.begin(),
-                               actual.begin() + len);
-    }
-}
-
-template <typename T>
-struct wrapper
-{
-    T t;
-    wrapper() {}
-    explicit wrapper(T t_) : t(t_) {}
-    bool
-    operator==(const wrapper<T>& a) const
-    {
-        return t == a.t;
-    }
-};
-
-int
-main()
-{
-    test<int32_t>();
-    test<uint16_t>();
-    test<float64_t>();
-#if !defined(_PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN)
-    test<wrapper<float64_t>>();
-#endif
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
deleted file mode 100644
index 533443896c35..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/alg.reverse/reverse_copy.pass.cpp
+++ /dev/null
@@ -1,130 +0,0 @@
-// -*- C++ -*-
-//===-- reverse_copy.pass.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct wrapper
-{
-    T t;
-    wrapper() {}
-    explicit wrapper(T t_) : t(t_) {}
-    wrapper&
-    operator=(const T& t_)
-    {
-        t = t_;
-        return *this;
-    }
-    bool
-    operator==(const wrapper& t_) const
-    {
-        return t == t_.t;
-    }
-};
-
-template <typename T1, typename T2>
-bool
-eq(const wrapper<T1>& a, const wrapper<T2>& b)
-{
-    return a.t == b.t;
-}
-
-template <typename T1, typename T2>
-bool
-eq(const T1& a, const T2& b)
-{
-    return a == b;
-}
-
-// we need to save state here, because we need to test with different types of iterators
-// due to the caller invoke_on_all_policies does forcing modification passed iterator type to cover additional usage cases.
-template <typename Iterator>
-struct test_one_policy
-{
-    Iterator data_b;
-    Iterator data_e;
-    test_one_policy(Iterator b, Iterator e) : data_b(b), data_e(e) {}
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator1 actual_b, Iterator1 actual_e)
-    {
-    }
-    template <typename Iterator1>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 actual_b, Iterator1 actual_e)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 actual_b, Iterator1 actual_e)
-    {
-        using namespace std;
-        using T = typename iterator_traits<Iterator1>::value_type;
-
-        fill(actual_b, actual_e, T(-123));
-        Iterator1 actual_return = reverse_copy(exec, data_b, data_e, actual_b);
-
-        EXPECT_TRUE(actual_return == actual_e, "wrong result of reverse_copy");
-
-        const auto n = std::distance(data_b, data_e);
-        Sequence<T> res(n);
-        std::copy(std::reverse_iterator<Iterator>(data_e), std::reverse_iterator<Iterator>(data_b), res.begin());
-
-        EXPECT_EQ_N(res.begin(), actual_b, n, "wrong effect of reverse_copy");
-    }
-};
-
-template <typename T1, typename T2>
-void
-test()
-{
-    typedef typename Sequence<T1>::iterator iterator_type;
-    typedef typename Sequence<T1>::const_bidirectional_iterator cbi_iterator_type;
-
-    const std::size_t max_len = 100000;
-
-    Sequence<T2> actual(max_len);
-
-    Sequence<T1> data(max_len, [](std::size_t i) { return T1(i); });
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        invoke_on_all_policies(test_one_policy<iterator_type>(data.begin(), data.begin() + len), actual.begin(),
-                               actual.begin() + len);
-        invoke_on_all_policies(test_one_policy<cbi_iterator_type>(data.cbibegin(), std::next(data.cbibegin(), len)),
-                               actual.begin(), actual.begin() + len);
-    }
-}
-
-int
-main()
-{
-    test<int16_t, int8_t>();
-    test<uint16_t, float32_t>();
-    test<float64_t, int64_t>();
-    test<wrapper<float64_t>, wrapper<float64_t>>();
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
deleted file mode 100644
index 24b784383916..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/copy_move.pass.cpp
+++ /dev/null
@@ -1,197 +0,0 @@
-// -*- C++ -*-
-//===-- copy_move.pass.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for copy, move and copy_n
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_copy
-{
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size, typename T>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2 expected_last, Size size,
-               Size n, T trash)
-    {
-    }
-
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size, typename T>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, Size size, Size n, T trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size size, Size n, T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, size, trash);
-        std::fill_n(out_first, size, trash);
-
-        // Run copy
-        copy(first, last, expected_first);
-        auto k = copy(exec, first, last, out_first);
-        for (size_t j = 0; j < GuardSize; ++j)
-            ++k;
-        EXPECT_EQ_N(expected_first, out_first, size, "wrong effect from copy");
-        EXPECT_TRUE(out_last == k, "wrong return value from copy");
-
-        // Cleaning
-        std::fill_n(out_first, size, trash);
-        // Run copy_n
-        k = copy_n(exec, first, n, out_first);
-        for (size_t j = 0; j < GuardSize; ++j)
-            ++k;
-        EXPECT_EQ_N(expected_first, out_first, size, "wrong effect from copy_n");
-        EXPECT_TRUE(out_last == k, "wrong return value from copy_n");
-    }
-};
-
-template <typename T>
-struct run_move
-{
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2 expected_last, Size size,
-               Size n, T trash)
-    {
-    }
-
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, Size size, Size n, T trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size size, Size, T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, size, trash);
-        std::fill_n(out_first, size, trash);
-
-        // Run move
-        move(first, last, expected_first);
-        auto k = move(exec, first, last, out_first);
-        for (size_t j = 0; j < GuardSize; ++j)
-            ++k;
-        EXPECT_EQ_N(expected_first, out_first, size, "wrong effect from move");
-        EXPECT_TRUE(out_last == k, "wrong return value from move");
-    }
-};
-
-template <typename T>
-struct run_move<Wrapper<T>>
-{
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2 expected_last, Size size,
-               Size n, Wrapper<T> trash)
-    {
-    }
-
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, Size size, Size n, Wrapper<T> trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2, OutputIterator2, Size size, Size, Wrapper<T> trash)
-    {
-        // Cleaning
-        std::fill_n(out_first, size, trash);
-        Wrapper<T>::SetMoveCount(0);
-
-        // Run move
-        auto k = move(exec, first, last, out_first);
-        for (size_t j = 0; j < GuardSize; ++j)
-            ++k;
-        EXPECT_TRUE(Wrapper<T>::MoveCount() == size, "wrong effect from move");
-        EXPECT_TRUE(out_last == k, "wrong return value from move");
-    }
-};
-
-template <typename T, typename Convert>
-void
-test(T trash, Convert convert)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        // count is number of output elements, plus a handful
-        // more for sake of detecting buffer overruns.
-        Sequence<T> in(n, [&](size_t k) -> T {
-            T val = convert(n ^ k);
-            return val;
-        });
-
-        const size_t outN = n + GuardSize;
-        Sequence<T> out(outN, [=](size_t) { return trash; });
-        Sequence<T> expected(outN, [=](size_t) { return trash; });
-        invoke_on_all_policies(run_copy(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), outN, n, trash);
-        invoke_on_all_policies(run_copy(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), outN, n, trash);
-        invoke_on_all_policies(run_move<T>(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), n, n, trash);
-
-        // For this test const iterator isn't suitable
-        // because const rvalue-reference call copy assignment operator
-    }
-}
-
-int
-main()
-{
-    test<int32_t>(-666, [](size_t j) { return int32_t(j); });
-    test<Wrapper<float64_t>>(Wrapper<float64_t>(-666.0), [](int32_t j) { return Wrapper<float64_t>(j); });
-
-#if !defined(_PSTL_ICC_16_17_TEST_64_TIMEOUT)
-    test<float64_t>(-666.0, [](size_t j) { return float64_t(j); });
-    test<Number>(Number(42, OddTag()), [](int32_t j) { return Number(j, OddTag()); });
-#endif
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
deleted file mode 100644
index d44a1a483c19..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/fill.pass.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// -*- C++ -*-
-//===-- fill.pass.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_fill
-{
-    template <typename It, typename T>
-    bool
-    check(It first, It last, const T& value)
-    {
-        for (; first != last; ++first)
-            if (*first != value)
-                return false;
-        return true;
-    }
-
-    template <typename Policy, typename Iterator, typename T>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, const T& value)
-    {
-        fill(first, last, T(value + 1)); // initialize memory with different value
-
-        fill(exec, first, last, value);
-        EXPECT_TRUE(check(first, last, value), "fill wrong result");
-    }
-};
-
-struct test_fill_n
-{
-    template <typename It, typename Size, typename T>
-    bool
-    check(It first, Size n, const T& value)
-    {
-        for (Size i = 0; i < n; ++i, ++first)
-            if (*first != value)
-                return false;
-        return true;
-    }
-
-    template <typename Policy, typename Iterator, typename Size, typename T>
-    void
-    operator()(Policy&& exec, Iterator first, Size n, const T& value)
-    {
-        fill_n(first, n, T(value + 1)); // initialize memory with different value
-
-        const Iterator one_past_last = fill_n(exec, first, n, value);
-        const Iterator expected_return = std::next(first, n);
-
-        EXPECT_TRUE(expected_return == one_past_last, "fill_n should return Iterator to one past the element assigned");
-        EXPECT_TRUE(check(first, n, value), "fill_n wrong result");
-
-        //n == -1
-        const Iterator res = fill_n(exec, first, -1, value);
-        EXPECT_TRUE(res == first, "fill_n wrong result for n == -1");
-    }
-};
-
-template <typename T>
-void
-test_fill_by_type(std::size_t n)
-{
-    Sequence<T> in(n, [](std::size_t) -> T { return T(0); }); //fill with zeros
-    T value = -1;
-
-    invoke_on_all_policies(test_fill(), in.begin(), in.end(), value);
-    invoke_on_all_policies(test_fill_n(), in.begin(), n, value);
-}
-
-int
-main()
-{
-
-    const std::size_t N = 100000;
-
-    for (std::size_t n = 0; n < N; n = n < 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        test_fill_by_type<int32_t>(n);
-        test_fill_by_type<float64_t>(n);
-    }
-
-    std::cout << done() << std::endl;
-
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
deleted file mode 100644
index 92d87ea26e54..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/generate.pass.cpp
+++ /dev/null
@@ -1,104 +0,0 @@
-// -*- C++ -*-
-//===-- generate.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <atomic>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct Generator_count
-{
-    const T def_val = T(-1);
-    T
-    operator()()
-    {
-        return def_val;
-    }
-    T
-    default_value() const
-    {
-        return def_val;
-    }
-};
-
-struct test_generate
-{
-    template <typename Policy, typename Iterator, typename Size>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Size n)
-    {
-        using namespace std;
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-
-        // Try random-access iterator
-        {
-            Generator_count<T> g;
-            generate(exec, first, last, g);
-            Size count = std::count(first, last, g.default_value());
-            EXPECT_TRUE(count == n, "generate wrong result for generate");
-            std::fill(first, last, T(0));
-        }
-
-        {
-            Generator_count<T> g;
-            const auto m = n / 2;
-            auto actual_last = generate_n(exec, first, m, g);
-            Size count = std::count(first, actual_last, g.default_value());
-            EXPECT_TRUE(count == m && actual_last == std::next(first, m), "generate_n wrong result for generate_n");
-            std::fill(first, actual_last, T(0));
-        }
-    }
-};
-
-template <typename T>
-void
-test_generate_by_type()
-{
-    for (size_t n = 0; n <= 100000; n = n < 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [](size_t) -> T { return T(0); }); //fill by zero
-
-        invoke_on_all_policies(test_generate(), in.begin(), in.end(), in.size());
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto gen = []() { return T(0); };
-
-        generate(exec, iter, iter, non_const(gen));
-        generate_n(exec, iter, 0, non_const(gen));
-    }
-};
-
-int
-main()
-{
-
-    test_generate_by_type<int32_t>();
-    test_generate_by_type<float64_t>();
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
deleted file mode 100644
index 872b0d292caa..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/remove.pass.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-// -*- C++ -*-
-//===-- remove.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Test for remove, remove_if
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_remove
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename Size, typename T>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator expected_first, OutputIterator expected_last, Size n,
-               const T& value)
-    {
-    }
-    template <typename InputIterator, typename OutputIterator, typename Size, typename T>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator expected_first,
-               OutputIterator expected_last, Size n, const T& value)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename Size, typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator expected_first, OutputIterator expected_last, Size,
-               const T& value)
-    {
-        // Cleaning
-        std::copy(first, last, expected_first);
-        std::copy(first, last, out_first);
-
-        // Run remove
-        OutputIterator i = remove(expected_first, expected_last, value);
-        OutputIterator k = remove(exec, out_first, out_last, value);
-        EXPECT_TRUE(std::distance(expected_first, i) == std::distance(out_first, k), "wrong return value from remove");
-        EXPECT_EQ_N(expected_first, out_first, std::distance(expected_first, i), "wrong remove effect");
-    }
-};
-
-struct run_remove_if
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename Size, typename Predicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator expected_first, OutputIterator expected_last, Size n,
-               Predicate pred)
-    {
-    }
-    template <typename InputIterator, typename OutputIterator, typename Size, typename Predicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator expected_first,
-               OutputIterator expected_last, Size n, Predicate pred)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename Size, typename Predicate>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator expected_first, OutputIterator expected_last, Size,
-               Predicate pred)
-    {
-        // Cleaning
-        std::copy(first, last, expected_first);
-        std::copy(first, last, out_first);
-
-        // Run remove_if
-        OutputIterator i = remove_if(expected_first, expected_last, pred);
-        OutputIterator k = remove_if(exec, out_first, out_last, pred);
-        EXPECT_TRUE(std::distance(expected_first, i) == std::distance(out_first, k),
-                    "wrong return value from remove_if");
-        EXPECT_EQ_N(expected_first, out_first, std::distance(expected_first, i), "wrong remove_if effect");
-    }
-};
-
-template <typename T, typename Predicate, typename Convert>
-void
-test(T trash, const T& value, Predicate pred, Convert convert)
-{
-    const std::size_t max_size = 100000;
-    Sequence<T> out(max_size, [trash](size_t) { return trash; });
-    Sequence<T> expected(max_size, [trash](size_t) { return trash; });
-
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> data(n, [&](size_t k) -> T { return convert(k); });
-
-        invoke_on_all_policies(run_remove(), data.begin(), data.end(), out.begin(), out.begin() + n, expected.begin(),
-                               expected.begin() + n, n, value);
-        invoke_on_all_policies(run_remove_if(), data.begin(), data.end(), out.begin(), out.begin() + n,
-                               expected.begin(), expected.begin() + n, n, pred);
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-
-        invoke_if(exec, [&]() { remove_if(exec, iter, iter, non_const(is_even)); });
-    }
-};
-
-int
-main()
-{
-#if !defined(_PSTL_ICC_18_TEST_EARLY_EXIT_MONOTONIC_RELEASE_BROKEN)
-    test<int32_t>(666, 42, [](int32_t) { return true; }, [](size_t j) { return j; });
-#endif
-
-    test<int32_t>(666, 2001, [](const int32_t& val) { return val != 2001; },
-                  [](size_t j) { return ((j + 1) % 5 & 2) != 0 ? 2001 : -1 - int32_t(j); });
-    test<float64_t>(-666.0, 8.5, [](const float64_t& val) { return val != 8.5; },
-                    [](size_t j) { return ((j + 1) % 7 & 2) != 0 ? 8.5 : float64_t(j % 32 + j); });
-
-#if !defined(_PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN)
-    test<Number>(Number(-666, OddTag()), Number(42, OddTag()), IsMultiple(3, OddTag()),
-                 [](int32_t j) { return Number(j, OddTag()); });
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    test<MemoryChecker>(MemoryChecker{0}, MemoryChecker{1},
-        [](const MemoryChecker& val){ return val.value() == 1; },
-        [](std::size_t idx){ return MemoryChecker{std::int32_t(idx % 3 == 0)}; }
-    );
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from remove,remove_if: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from remove,remove_if: number of ctors calls > num of dtors calls");
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
deleted file mode 100644
index 9cdd7d62a4f2..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/remove_copy.pass.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-// -*- C++ -*-
-//===-- remove_copy.pass.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_remove_copy
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size n, const T& value,
-               T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-
-        // Run copy_if
-        auto i = std::remove_copy(first, last, expected_first, value);
-        (void)i;
-        auto k = std::remove_copy(exec, first, last, out_first, value);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong remove_copy effect");
-        for (size_t j = 0; j < GuardSize; ++j)
-        {
-            ++k;
-        }
-        EXPECT_TRUE(out_last == k, "wrong return value from remove_copy");
-    }
-};
-
-template <typename T, typename Convert>
-void
-test(T trash, const T& value, Convert convert, bool check_weakness = true)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        // count is number of output elements, plus a handful
-        // more for sake of detecting buffer overruns.
-        size_t count = GuardSize;
-        Sequence<T> in(n, [&](size_t k) -> T {
-            T x = convert(n ^ k);
-            count += !(x == value) ? 1 : 0;
-            return x;
-        });
-        using namespace std;
-
-        Sequence<T> out(count, [=](size_t) { return trash; });
-        Sequence<T> expected(count, [=](size_t) { return trash; });
-        if (check_weakness)
-        {
-            auto expected_result = remove_copy(in.cfbegin(), in.cfend(), expected.begin(), value);
-            size_t m = expected_result - expected.begin();
-            EXPECT_TRUE(n / 4 <= m && m <= 3 * (n + 1) / 4, "weak test for remove_copy");
-        }
-        invoke_on_all_policies(run_remove_copy(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), count, value, trash);
-        invoke_on_all_policies(run_remove_copy(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), count, value, trash);
-    }
-}
-
-int
-main()
-{
-
-    test<float64_t>(-666.0, 8.5, [](size_t j) { return ((j + 1) % 7 & 2) != 0 ? 8.5 : float64_t(j % 32 + j); });
-
-    test<int32_t>(-666, 42, [](size_t j) { return ((j + 1) % 5 & 2) != 0 ? 42 : -1 - int32_t(j); });
-
-    test<Number>(Number(42, OddTag()), Number(2001, OddTag()),
-                 [](int32_t j) { return ((j + 1) % 3 & 2) != 0 ? Number(2001, OddTag()) : Number(j, OddTag()); });
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
deleted file mode 100644
index 9c6ca1ea8c0f..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/replace.pass.cpp
+++ /dev/null
@@ -1,160 +0,0 @@
-// -*- C++ -*-
-//===-- replace.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// This class is needed to check the self-copying
-struct copy_int
-{
-    int32_t value;
-    int32_t copied_times = 0;
-    constexpr explicit copy_int(int32_t val = 0) : value(val) {}
-    constexpr copy_int(copy_int const& other) : value(other.value), copied_times(other.copied_times) { }
-
-    constexpr copy_int&
-    operator=(const copy_int& other)
-    {
-        if (&other == this)
-            copied_times++;
-        else
-        {
-            value = other.value;
-            copied_times = other.copied_times;
-        }
-        return *this;
-    }
-
-    constexpr bool
-    operator==(const copy_int& other) const
-    {
-        return (value == other.value);
-    }
-};
-
-template <typename Iterator>
-struct test_one_policy
-{
-    std::size_t len;
-    Iterator data_b;
-    Iterator data_e;
-    test_one_policy(Iterator data_, std::size_t len_)
-    {
-        len = len_;
-        data_b = data_;
-        data_e = std::next(data_b, len);
-    }
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 expected_b, Iterator1 expected_e, Iterator2 actual_b,
-               Iterator2 actual_e, Predicate pred, const T& value, const T& old_value)
-    {
-        using namespace std;
-
-        copy(data_b, data_e, expected_b);
-        copy(data_b, data_e, actual_b);
-
-        replace(expected_b, expected_e, old_value, value);
-        replace(exec, actual_b, actual_e, old_value, value);
-
-        EXPECT_TRUE((check<T, Iterator2>(actual_b, actual_e)), "wrong result of self assignment check");
-        EXPECT_TRUE(equal(expected_b, expected_e, actual_b), "wrong result of replace");
-
-        copy(data_b, data_e, expected_b);
-        copy(data_b, data_e, actual_b);
-
-        replace_if(expected_b, expected_e, pred, value);
-        replace_if(exec, actual_b, actual_e, pred, value);
-        EXPECT_TRUE(equal(expected_b, expected_e, actual_b), "wrong result of replace_if");
-    }
-
-    template <typename T, typename Iterator1>
-    bool check(Iterator1, Iterator1)
-    {
-        return true;
-    }
-
-    template <typename T, typename Iterator1>
-    typename std::enable_if<std::is_same<T, copy_int>::value, bool>::type_t
-    check(Iterator1 b, Iterator1 e)
-    {
-        return std::all_of(b, e, [](const copy_int& elem) { return elem.copied_times == 0; });
-    }
-};
-
-template <typename T1, typename T2, typename Pred>
-void
-test(Pred pred)
-{
-    typedef typename Sequence<T2>::iterator iterator_type;
-
-    const std::size_t max_len = 100000;
-
-    static constexpr T1 value = T1(0);
-    static constexpr T1 new_value = T1(666);
-
-    Sequence<T2> expected(max_len);
-    Sequence<T2> actual(max_len);
-
-    Sequence<T2> data(max_len, [](std::size_t i) {
-        if (i % 3 == 2)
-        {
-            return T1(i);
-        }
-        else
-        {
-            return value;
-        }
-    });
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        test_one_policy<iterator_type> temp(data.begin(), len);
-
-        invoke_on_all_policies(temp, expected.begin(), expected.begin() + len, actual.begin(), actual.begin() + len,
-                               pred, new_value, value);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        invoke_if(exec, [&]() { replace_if(exec, iter, iter, non_const(is_even), T(0)); });
-    }
-};
-
-int
-main()
-{
-    test<int32_t, float32_t>(__pstl::__internal::__equal_value<int32_t>(666));
-    test<uint16_t, uint8_t>([](const uint16_t& elem) { return elem % 3 < 2; });
-    test<float64_t, int64_t>([](const float64_t& elem) { return elem * elem - 3.5 * elem > 10; });
-    test<copy_int, copy_int>([](const copy_int& val) { return val.value / 5 > 2; });
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
deleted file mode 100644
index 94d725fb1c50..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/replace_copy.pass.cpp
+++ /dev/null
@@ -1,105 +0,0 @@
-// -*- C++ -*-
-//===-- replace_copy.pass.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for replace_copy and replace_copy_if
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_replace_copy
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size n, Predicate pred,
-               const T& old_value, const T& new_value, T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-        // Run replace_copy
-        auto i = std::replace_copy(first, last, expected_first, old_value, new_value);
-        auto k = std::replace_copy(exec, first, last, out_first, old_value, new_value);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong replace_copy effect");
-        EXPECT_TRUE(out_last == k, "wrong return value from replace_copy");
-
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-        // Run replace_copy_if
-        i = replace_copy_if(first, last, expected_first, pred, new_value);
-        k = replace_copy_if(exec, first, last, out_first, pred, new_value);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong replace_copy_if effect");
-        EXPECT_TRUE(out_last == k, "wrong return value from replace_copy_if");
-    }
-};
-
-template <typename T, typename Convert, typename Predicate>
-void
-test(T trash, const T& old_value, const T& new_value, Predicate pred, Convert convert)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [&](size_t k) -> T { return convert(n ^ k); });
-        Sequence<T> out(n, [=](size_t) { return trash; });
-        Sequence<T> expected(n, [=](size_t) { return trash; });
-
-        invoke_on_all_policies(test_replace_copy(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), out.size(), pred, old_value, new_value, trash);
-        invoke_on_all_policies(test_replace_copy(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), out.size(), pred, old_value, new_value, trash);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-
-        invoke_if(exec, [&]() { replace_copy_if(exec, input_iter, input_iter, out_iter, non_const(is_even), T(0)); });
-    }
-};
-
-int
-main()
-{
-
-    test<float64_t>(-666.0, 8.5, 0.33, [](const float64_t& x) { return x * x <= 1024; },
-                    [](size_t j) { return ((j + 1) % 7 & 2) != 0 ? 8.5 : float64_t(j % 32 + j); });
-
-    test<int32_t>(-666, 42, 99, [](const int32_t& x) { return x != 42; },
-                  [](size_t j) { return ((j + 1) % 5 & 2) != 0 ? 42 : -1 - int32_t(j); });
-
-#if !defined(_PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN)
-    test<Number>(Number(42, OddTag()), Number(2001, OddTag()), Number(2017, OddTag()), IsMultiple(3, OddTag()),
-                 [](int32_t j) { return ((j + 1) % 3 & 2) != 0 ? Number(2001, OddTag()) : Number(j, OddTag()); });
-#endif
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
deleted file mode 100644
index 0d1cfeb4ae8e..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/rotate.pass.cpp
+++ /dev/null
@@ -1,176 +0,0 @@
-// -*- C++ -*-
-//===-- rotate.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct wrapper
-{
-    T t;
-    int move_count;
-    explicit wrapper(T t_) : t(t_), move_count(0) {}
-    wrapper&
-    operator=(const T& t_)
-    {
-        t = t_;
-        return *this;
-    }
-
-    wrapper(const wrapper<T>& a) : move_count(0) { t = a.t; }
-
-    wrapper<T>&
-    operator=(wrapper<T>& a)
-    {
-        t = a.t;
-        return *this;
-    }
-
-    wrapper<T>&
-    operator=(wrapper<T>&& a)
-    {
-        t = a.t;
-        move_count += 1;
-        return *this;
-    }
-};
-
-template <typename T>
-struct compare
-{
-    bool
-    operator()(const T& a, const T& b)
-    {
-        return a == b;
-    }
-};
-
-template <typename T>
-struct compare<wrapper<T>>
-{
-    bool
-    operator()(const wrapper<T>& a, const wrapper<T>& b)
-    {
-        return a.t == b.t;
-    }
-};
-#include <typeinfo>
-
-struct test_one_policy
-{
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specializations to skip testing in case of broken configuration
-    template <typename Iterator, typename Size>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator data_b, Iterator data_e, Iterator actual_b,
-               Iterator actual_e, Size shift)
-    {
-    }
-    template <typename Iterator, typename Size>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator data_b, Iterator data_e, Iterator actual_b,
-               Iterator actual_e, Size shift)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator, typename Size>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator data_b, Iterator data_e, Iterator actual_b, Iterator actual_e,
-               Size shift)
-    {
-        using namespace std;
-        using T = typename iterator_traits<Iterator>::value_type;
-        Iterator actual_m = std::next(actual_b, shift);
-
-        copy(data_b, data_e, actual_b);
-        Iterator actual_return = rotate(exec, actual_b, actual_m, actual_e);
-
-        EXPECT_TRUE(actual_return == std::next(actual_b, std::distance(actual_m, actual_e)), "wrong result of rotate");
-        auto comparator = compare<T>();
-        bool check = std::equal(actual_return, actual_e, data_b, comparator);
-        check = check && std::equal(actual_b, actual_return, std::next(data_b, shift), comparator);
-
-        EXPECT_TRUE(check, "wrong effect of rotate");
-        EXPECT_TRUE(check_move(exec, actual_b, actual_e, shift), "wrong move test of rotate");
-    }
-
-    template <typename ExecutionPolicy, typename Iterator, typename Size>
-    typename std::enable_if<
-        is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-            !std::is_same<ExecutionPolicy, std::execution::sequenced_policy>::value &&
-            std::is_same<typename std::iterator_traits<Iterator>::value_type, wrapper<float32_t>>::value,
-        bool>::type
-    check_move(ExecutionPolicy&&, Iterator b, Iterator e, Size shift)
-    {
-        bool result = all_of(b, e, [](wrapper<float32_t>& a) {
-            bool temp = a.move_count > 0;
-            a.move_count = 0;
-            return temp;
-        });
-        return shift == 0 || result;
-    }
-
-    template <typename ExecutionPolicy, typename Iterator, typename Size>
-    typename std::enable_if<
-        !(is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-          !std::is_same<ExecutionPolicy, std::execution::sequenced_policy>::value &&
-          std::is_same<typename std::iterator_traits<Iterator>::value_type, wrapper<float32_t>>::value),
-        bool>::type
-    check_move(ExecutionPolicy&&, Iterator, Iterator, Size)
-    {
-        return true;
-    }
-};
-
-template <typename T>
-void
-test()
-{
-    const int32_t max_len = 100000;
-
-    Sequence<T> actual(max_len, [](std::size_t i) { return T(i); });
-    Sequence<T> data(max_len, [](std::size_t i) { return T(i); });
-
-    for (int32_t len = 0; len < max_len; len = len <= 16 ? len + 1 : int32_t(3.1415 * len))
-    {
-        int32_t shifts[] = {0, 1, 2, len / 3, (2 * len) / 3, len - 1};
-        for (auto shift : shifts)
-        {
-            if (shift >= 0 && shift < len)
-            {
-                invoke_on_all_policies(test_one_policy(), data.begin(), data.begin() + len, actual.begin(),
-                                       actual.begin() + len, shift);
-            }
-        }
-    }
-}
-
-int
-main()
-{
-    test<int32_t>();
-    test<wrapper<float64_t>>();
-    test<MemoryChecker>();
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from rotate: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from rotate: number of ctors calls > num of dtors calls");
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
deleted file mode 100644
index 539cefc929d4..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/rotate_copy.pass.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-// -*- C++ -*-
-//===-- rotate_copy.pass.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct wrapper;
-
-template <typename T>
-bool
-compare(const wrapper<T>& a, const wrapper<T>& b)
-{
-    return a.t == b.t;
-}
-
-template <typename T>
-bool
-compare(const T& a, const T& b)
-{
-    return a == b;
-}
-
-template <typename T>
-struct wrapper
-{
-    explicit wrapper(T t_) : t(t_) {}
-    wrapper&
-    operator=(const T& t_)
-    {
-        t = t_;
-        return *this;
-    }
-    friend bool
-    compare<T>(const wrapper<T>& a, const wrapper<T>& b);
-
-  private:
-    T t;
-};
-
-template <typename T, typename It1, typename It2>
-struct comparator
-{
-    using T1 = typename std::iterator_traits<It1>::value_type;
-    using T2 = typename std::iterator_traits<It2>::value_type;
-    bool
-    operator()(T1 a, T2 b)
-    {
-        T temp = a;
-        return compare(temp, b);
-    }
-};
-
-struct test_one_policy
-{
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Iterator2>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e, std::size_t shift)
-    {
-    }
-    template <typename Iterator1, typename Iterator2>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e, std::size_t shift)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b, Iterator2 actual_e,
-               std::size_t shift)
-    {
-        using namespace std;
-        using T = typename iterator_traits<Iterator2>::value_type;
-        Iterator1 data_m = std::next(data_b, shift);
-
-        fill(actual_b, actual_e, T(-123));
-        Iterator2 actual_return = rotate_copy(exec, data_b, data_m, data_e, actual_b);
-
-        EXPECT_TRUE(actual_return == actual_e, "wrong result of rotate_copy");
-        auto comparer = comparator<T, Iterator1, Iterator2>();
-        bool check = std::equal(data_m, data_e, actual_b, comparer);
-        check = check && std::equal(data_b, data_m, std::next(actual_b, std::distance(data_m, data_e)), comparer);
-
-        EXPECT_TRUE(check, "wrong effect of rotate_copy");
-    }
-};
-
-template <typename T1, typename T2>
-void
-test()
-{
-
-    const std::size_t max_len = 100000;
-
-    Sequence<T2> actual(max_len, [](std::size_t i) { return T1(i); });
-
-    Sequence<T1> data(max_len, [](std::size_t i) { return T1(i); });
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        std::size_t shifts[] = {0, 1, 2, len / 3, (2 * len) / 3, len - 1};
-        for (std::size_t shift : shifts)
-        {
-            if (shift > 0 && shift < len)
-            {
-                invoke_on_all_policies(test_one_policy(), data.begin(), data.begin() + len, actual.begin(),
-                                       actual.begin() + len, shift);
-                invoke_on_all_policies(test_one_policy(), data.cbegin(), data.cbegin() + len, actual.begin(),
-                                       actual.begin() + len, shift);
-            }
-        }
-    }
-}
-
-int
-main()
-{
-    test<int32_t, int8_t>();
-    test<uint16_t, float32_t>();
-    test<float64_t, int64_t>();
-    test<wrapper<float64_t>, wrapper<float64_t>>();
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
deleted file mode 100644
index c3c3693af594..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/swap_ranges.pass.cpp
+++ /dev/null
@@ -1,133 +0,0 @@
-// -*- C++ -*-
-//===-- swap_ranges.pass.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct wrapper
-{
-    T t;
-    std::size_t number_of_swaps = 0;
-    wrapper() {}
-    explicit wrapper(T t_) : t(t_) {}
-    template <typename U>
-    void
-    operator=(const U& b)
-    {
-        t = b;
-    }
-    bool
-    operator==(const wrapper<T>& a) const
-    {
-        return t == a.t;
-    }
-};
-
-template <typename T>
-void
-swap(wrapper<T>& a, wrapper<T>& b)
-{
-    std::swap(a.t, b.t);
-    a.number_of_swaps++;
-    b.number_of_swaps++;
-}
-
-template <typename T>
-struct check_swap
-{
-    bool
-    operator()(T&)
-    {
-        return true;
-    }
-};
-
-template <typename T>
-struct check_swap<wrapper<T>>
-{
-    bool
-    operator()(wrapper<T>& a)
-    {
-        bool temp = (a.number_of_swaps == 1);
-        a.number_of_swaps = 0;
-        return temp;
-    }
-};
-
-struct test_one_policy
-{
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b, Iterator2 actual_e)
-    {
-        using namespace std;
-        using T_ref = typename iterator_traits<Iterator1>::reference;
-        using T = typename iterator_traits<Iterator1>::value_type;
-
-        iota(data_b, data_e, 0);
-        iota(actual_b, actual_e, std::distance(data_b, data_e));
-
-        Iterator2 actual_return = swap_ranges(exec, data_b, data_e, actual_b);
-        bool check_return = (actual_return == actual_e);
-        EXPECT_TRUE(check_return, "wrong result of swap_ranges");
-        if (check_return)
-        {
-            std::size_t i = 0;
-            bool check = all_of(actual_b, actual_e, [&i](T_ref a) { return a == T(i++); }) &&
-                         all_of(data_b, data_e, [&i](T_ref a) { return a == T(i++); });
-
-            EXPECT_TRUE(check, "wrong effect of swap_ranges");
-
-            if (check)
-            {
-                bool swap_check =
-                    all_of(data_b, data_e, check_swap<T>()) && all_of(actual_b, actual_e, check_swap<T>());
-                EXPECT_TRUE(swap_check, "wrong effect of swap_ranges swap check");
-            }
-        }
-    }
-};
-
-template <typename T>
-void
-test()
-{
-    const std::size_t max_len = 100000;
-
-    Sequence<T> data(max_len);
-    Sequence<T> actual(max_len);
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        invoke_on_all_policies(test_one_policy(), data.begin(), data.begin() + len, actual.begin(),
-                               actual.begin() + len);
-    }
-}
-
-int
-main()
-{
-    test<wrapper<uint16_t>>();
-    test<wrapper<float64_t>>();
-    test<int32_t>();
-    test<float32_t>();
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
deleted file mode 100644
index 949e6f54ffb4..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/transform_binary.pass.cpp
+++ /dev/null
@@ -1,122 +0,0 @@
-// -*- C++ -*-
-//===-- transform_binary.pass.cpp -----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename In1, typename In2, typename Out>
-class TheOperation
-{
-    Out val;
-
-  public:
-    TheOperation(Out v) : val(v) {}
-    Out
-    operator()(const In1& x, const In2& y) const
-    {
-        return Out(val + x - y);
-    }
-};
-
-template <typename InputIterator1, typename InputIterator2, typename OutputIterator>
-void
-check_and_reset(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, OutputIterator out_first)
-{
-    typedef typename std::iterator_traits<OutputIterator>::value_type Out;
-    typename std::iterator_traits<OutputIterator>::difference_type k = 0;
-    for (; first1 != last1; ++first1, ++first2, ++out_first, ++k)
-    {
-        // check
-        Out expected = Out(1.5) + *first1 - *first2;
-        Out actual = *out_first;
-        if (std::is_floating_point<Out>::value)
-        {
-            EXPECT_TRUE((expected > actual ? expected - actual : actual - expected) < 1e7,
-                        "wrong value in output sequence");
-        }
-        else
-        {
-            EXPECT_EQ(expected, actual, "wrong value in output sequence");
-        }
-        // reset
-        *out_first = k % 7 != 4 ? 7 * k - 5 : 0;
-    }
-}
-
-struct test_one_policy
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename OutputIterator,
-              typename BinaryOp>
-    void
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2,
-               OutputIterator out_first, OutputIterator, BinaryOp op)
-    {
-        auto result = std::transform(exec, first1, last1, first2, out_first, op);
-        (void)result;
-        check_and_reset(first1, last1, first2, out_first);
-    }
-};
-
-template <typename In1, typename In2, typename Out, typename Predicate>
-void
-test(Predicate pred)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<In1> in1(n, [](size_t k) { return k % 5 != 1 ? 3 * k - 7 : 0; });
-        Sequence<In2> in2(n, [](size_t k) { return k % 7 != 2 ? 5 * k - 5 : 0; });
-
-        Sequence<Out> out(n, [](size_t) { return -1; });
-
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.end(), in2.begin(), in2.end(), out.begin(),
-                               out.end(), pred);
-        invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cend(), in2.cbegin(), in2.cend(), out.begin(),
-                               out.end(), pred);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        invoke_if(exec, [&]() {
-            InputIterator input_iter2 = input_iter;
-            transform(exec, input_iter, input_iter, input_iter2, out_iter, non_const(std::plus<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    //const operator()
-    test<int32_t, int32_t, int32_t>(TheOperation<int32_t, int32_t, int32_t>(1));
-    test<float32_t, float32_t, float32_t>(TheOperation<float32_t, float32_t, float32_t>(1.5));
-    //non-const operator()
-    test<int32_t, float32_t, float32_t>(non_const(TheOperation<int32_t, float32_t, float32_t>(1.5)));
-    test<int64_t, float64_t, float32_t>(non_const(TheOperation<int64_t, float64_t, float32_t>(1.5)));
-    //lambda
-    test<int8_t, float64_t, int8_t>([](const int8_t& x, const float64_t& y) { return int8_t(int8_t(1.5) + x - y); });
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
deleted file mode 100644
index 69a410c4cd6e..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/transform_unary.pass.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-// -*- C++ -*-
-//===-- transform_unary.pass.cpp ------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename InputIterator, typename OutputIterator>
-void
-check_and_reset(InputIterator first, InputIterator last, OutputIterator out_first)
-{
-    typedef typename std::iterator_traits<OutputIterator>::value_type Out;
-    typename std::iterator_traits<OutputIterator>::difference_type k = 0;
-    for (; first != last; ++first, ++out_first, ++k)
-    {
-        // check
-        Out expected = 1 - *first;
-        Out actual = *out_first;
-        EXPECT_EQ(expected, actual, "wrong value in output sequence");
-        // reset
-        *out_first = k % 7 != 4 ? 7 * k - 5 : 0;
-    }
-}
-
-struct test_one_policy
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename UnaryOp>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, UnaryOp op)
-    {
-        auto orr = std::transform(exec, first, last, out_first, op);
-        EXPECT_TRUE(out_last == orr, "transform returned wrong iterator");
-        check_and_reset(first, last, out_first);
-    }
-};
-
-template <typename Tin, typename Tout>
-void
-test()
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<Tin> in(n, [](int32_t k) { return k % 5 != 1 ? 3 * k - 7 : 0; });
-
-        Sequence<Tout> out(n);
-
-        const auto flip = Complement<Tin, Tout>(1);
-        invoke_on_all_policies(test_one_policy(), in.begin(), in.end(), out.begin(), out.end(), flip);
-        invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cend(), out.begin(), out.end(), flip);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        invoke_if(exec, [&]() { transform(exec, input_iter, input_iter, out_iter, non_const(std::negate<T>())); });
-    }
-};
-
-int
-main()
-{
-    test<int32_t, int32_t>();
-    test<int32_t, float32_t>();
-    test<uint16_t, float32_t>();
-    test<float32_t, float64_t>();
-    test<float64_t, float64_t>();
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
deleted file mode 100644
index fbd4742b7e45..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/unique.pass.cpp
+++ /dev/null
@@ -1,163 +0,0 @@
-// -*- C++ -*-
-//===-- unique.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Test for unique
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_unique
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename ForwardIt, typename Generator>
-    void
-    operator()(pstl::execution::unsequenced_policy, ForwardIt first1, ForwardIt last1, ForwardIt first2,
-               ForwardIt last2, Generator generator)
-    {
-    }
-
-    template <typename ForwardIt, typename Generator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, ForwardIt first1, ForwardIt last1, ForwardIt first2,
-               ForwardIt last2, Generator generator)
-    {
-    }
-
-    template <typename ForwardIt, typename BinaryPred, typename Generator>
-    void
-    operator()(pstl::execution::unsequenced_policy, ForwardIt first1, ForwardIt last1, ForwardIt first2,
-               ForwardIt last2, BinaryPred pred, Generator generator)
-    {
-    }
-
-    template <typename ForwardIt, typename BinaryPred, typename Generator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, ForwardIt first1, ForwardIt last1, ForwardIt first2,
-               ForwardIt last2, BinaryPred pred, Generator generator)
-    {
-    }
-#endif
-
-    template <typename Policy, typename ForwardIt, typename Generator>
-    void
-    operator()(Policy&& exec, ForwardIt first1, ForwardIt last1, ForwardIt first2, ForwardIt last2, Generator generator)
-    {
-        using namespace std;
-
-        // Preparation
-        fill_data(first1, last1, generator);
-        fill_data(first2, last2, generator);
-
-        ForwardIt i = unique(first1, last1);
-        ForwardIt k = unique(exec, first2, last2);
-
-        auto n = std::distance(first1, i);
-        EXPECT_TRUE(std::distance(first2, k) == n, "wrong return value from unique without predicate");
-        EXPECT_EQ_N(first1, first2, n, "wrong effect from unique without predicate");
-    }
-
-    template <typename Policy, typename ForwardIt, typename BinaryPred, typename Generator>
-    void
-    operator()(Policy&& exec, ForwardIt first1, ForwardIt last1, ForwardIt first2, ForwardIt last2, BinaryPred pred,
-               Generator generator)
-    {
-        using namespace std;
-
-        // Preparation
-        fill_data(first1, last1, generator);
-        fill_data(first2, last2, generator);
-
-        ForwardIt i = unique(first1, last1, pred);
-        ForwardIt k = unique(exec, first2, last2, pred);
-
-        auto n = std::distance(first1, i);
-        EXPECT_TRUE(std::distance(first2, k) == n, "wrong return value from unique with predicate");
-        EXPECT_EQ_N(first1, first2, n, "wrong effect from unique with predicate");
-    }
-};
-
-template <typename T, typename Generator, typename Predicate>
-void
-test(Generator generator, Predicate pred)
-{
-    const std::size_t max_size = 1000000;
-    Sequence<T> in(max_size, [](size_t v) { return T(v); });
-    Sequence<T> exp(max_size, [](size_t v) { return T(v); });
-
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        invoke_on_all_policies(run_unique(), exp.begin(), exp.begin() + n, in.begin(), in.begin() + n, generator);
-        invoke_on_all_policies(run_unique(), exp.begin(), exp.begin() + n, in.begin(), in.begin() + n, pred, generator);
-    }
-}
-
-template <typename T>
-struct LocalWrapper
-{
-    T my_val;
-
-    explicit LocalWrapper(T k) : my_val(k) {}
-    LocalWrapper(LocalWrapper&& input) : my_val(std::move(input.my_val)) {}
-    LocalWrapper&
-    operator=(LocalWrapper&& input)
-    {
-        my_val = std::move(input.my_val);
-        return *this;
-    }
-    friend bool
-    operator==(const LocalWrapper<T>& x, const LocalWrapper<T>& y)
-    {
-        return x.my_val == y.my_val;
-    }
-};
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() { unique(exec, iter, iter, non_const(std::equal_to<T>())); });
-    }
-};
-
-int
-main()
-{
-#if !defined(_PSTL_ICC_16_17_18_TEST_UNIQUE_MASK_RELEASE_BROKEN)
-    test<int32_t>([](size_t j) { return j / 3; },
-                  [](const int32_t& val1, const int32_t& val2) { return val1 * val1 == val2 * val2; });
-    test<float64_t>([](size_t) { return float64_t(1); },
-                    [](const float64_t& val1, const float64_t& val2) { return val1 != val2; });
-#endif
-    test<LocalWrapper<uint32_t>>([](size_t j) { return LocalWrapper<uint32_t>(j); },
-                                 [](const LocalWrapper<uint32_t>& val1, const LocalWrapper<uint32_t>& val2) {
-                                     return val1.my_val != val2.my_val;
-                                 });
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    test<MemoryChecker>(
-        [](std::size_t idx){ return MemoryChecker{std::int32_t(idx / 3)}; },
-        [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() == val2.value(); });
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from unique: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from unique: number of ctors calls > num of dtors calls");
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp b/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
deleted file mode 100644
index 356d4d24aba6..000000000000
--- a/pstl/test/std/algorithms/alg.modifying.operations/unique_copy_equal.pass.cpp
+++ /dev/null
@@ -1,135 +0,0 @@
-// -*- C++ -*-
-//===-- unique_copy_equal.pass.cpp ----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for unique_copy
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct run_unique_copy
-{
-#if defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN) // dummy specializations to skip testing in case of broken configuration
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(pstl::execution::parallel_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2 expected_last, Size n,
-               Predicate pred, T trash)
-    {
-    }
-
-    template <typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, OutputIterator out_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, Size n, Predicate pred, T trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Predicate, typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator2 expected_first, OutputIterator2, Size n, Predicate pred,
-               T trash)
-    {
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-
-        // Run unique_copy
-        auto i = unique_copy(first, last, expected_first);
-        auto k = unique_copy(exec, first, last, out_first);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong unique_copy effect");
-        for (size_t j = 0; j < GuardSize; ++j)
-        {
-            ++k;
-        }
-        EXPECT_TRUE(out_last == k, "wrong return value from unique_copy");
-
-        // Cleaning
-        std::fill_n(expected_first, n, trash);
-        std::fill_n(out_first, n, trash);
-        // Run unique_copy with predicate
-        i = unique_copy(first, last, expected_first, pred);
-        k = unique_copy(exec, first, last, out_first, pred);
-        EXPECT_EQ_N(expected_first, out_first, n, "wrong unique_copy with predicate effect");
-        for (size_t j = 0; j < GuardSize; ++j)
-        {
-            ++k;
-        }
-        EXPECT_TRUE(out_last == k, "wrong return value from unique_copy with predicate");
-    }
-};
-
-template <typename T, typename BinaryPredicate, typename Convert>
-void
-test(T trash, BinaryPredicate pred, Convert convert, bool check_weakness = true)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        // count is number of output elements, plus a handful
-        // more for sake of detecting buffer overruns.
-        Sequence<T> in(n, [&](size_t k) -> T { return convert(k ^ n); });
-        using namespace std;
-        size_t count = GuardSize;
-        for (size_t k = 0; k < in.size(); ++k)
-            count += k == 0 || !pred(in[k], in[k - 1]) ? 1 : 0;
-        Sequence<T> out(count, [=](size_t) { return trash; });
-        Sequence<T> expected(count, [=](size_t) { return trash; });
-        if (check_weakness)
-        {
-            auto expected_result = unique_copy(in.begin(), in.end(), expected.begin(), pred);
-            size_t m = expected_result - expected.begin();
-            EXPECT_TRUE(n / (n < 10000 ? 4 : 6) <= m && m <= (3 * n + 1) / 4, "weak test for unique_copy");
-        }
-        invoke_on_all_policies(run_unique_copy(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), count, pred, trash);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        unique_copy(exec, input_iter, input_iter, out_iter, non_const(std::equal_to<T>()));
-    }
-};
-
-int
-main()
-{
-    test<Number>(Number(42, OddTag()), std::equal_to<Number>(),
-                 [](int32_t j) { return Number(3 * j / 13 ^ (j & 8), OddTag()); });
-
-    test<float32_t>(float32_t(42), std::equal_to<float32_t>(),
-                    [](int32_t j) { return float32_t(5 * j / 23 ^ (j / 7)); });
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN)
-    test<float32_t>(float32_t(42), [](float32_t, float32_t) { return false; }, [](int32_t j) { return float32_t(j); },
-                    false);
-#endif
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
deleted file mode 100644
index 36a623bac42a..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/adjacent_find.pass.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-// -*- C++ -*-
-//===-- adjacent_find.pass.cpp --------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_adjacent_find
-{
-    template <typename Policy, typename Iterator, typename Pred>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Pred pred)
-    {
-        using namespace std;
-
-        auto k = std::adjacent_find(first, last, pred);
-        auto i = adjacent_find(exec, first, last, pred);
-        EXPECT_TRUE(i == k, "wrong return value from adjacent_find with predicate");
-
-        i = adjacent_find(exec, first, last);
-        EXPECT_TRUE(i == k, "wrong return value from adjacent_find without predicate");
-    }
-};
-
-template <typename T>
-void
-test_adjacent_find_by_type()
-{
-
-    size_t counts[] = {2, 3, 500};
-    for (size_t c = 0; c < const_size(counts); ++c)
-    {
-
-        for (size_t e = 0; e < (counts[c] >= 64 ? 64 : (counts[c] == 2 ? 1 : 2)); ++e)
-        {
-            Sequence<T> in(counts[c], [](size_t v) -> T { return T(v); }); //fill 0...n
-            in[e] = in[e + 1] = -1;                                        //make an adjacent pair
-
-            auto i = std::adjacent_find(in.cbegin(), in.cend(), std::equal_to<T>());
-            EXPECT_TRUE(i == in.cbegin() + e, "std::adjacent_find returned wrong result");
-
-            invoke_on_all_policies(test_adjacent_find(), in.begin(), in.end(), std::equal_to<T>());
-            invoke_on_all_policies(test_adjacent_find(), in.cbegin(), in.cend(), std::equal_to<T>());
-        }
-    }
-
-    //special cases: size=0, size=1;
-    for (size_t expect = 0; expect < 1; ++expect)
-    {
-        Sequence<T> in(expect, [](size_t v) -> T { return T(v); }); //fill 0...n
-        auto i = std::adjacent_find(in.cbegin(), in.cend(), std::equal_to<T>());
-        EXPECT_TRUE(i == in.cbegin() + expect, "std::adjacent_find returned wrong result");
-
-        invoke_on_all_policies(test_adjacent_find(), in.begin(), in.end(), std::equal_to<T>());
-        invoke_on_all_policies(test_adjacent_find(), in.cbegin(), in.cend(), std::equal_to<T>());
-    }
-
-    //special cases:
-    Sequence<T> a1 = {5, 5, 5, 6, 7, 8, 9};
-    invoke_on_all_policies(test_adjacent_find(), a1.begin(), a1.end(), std::equal_to<T>());
-    invoke_on_all_policies(test_adjacent_find(), a1.begin() + 1, a1.end(), std::equal_to<T>());
-
-    invoke_on_all_policies(test_adjacent_find(), a1.cbegin(), a1.cend(), std::equal_to<T>());
-    invoke_on_all_policies(test_adjacent_find(), a1.cbegin() + 1, a1.cend(), std::equal_to<T>());
-
-    Sequence<T> a2 = {5, 6, 7, 8, 9, 9};
-    invoke_on_all_policies(test_adjacent_find(), a2.begin(), a2.end(), std::equal_to<T>());
-    invoke_on_all_policies(test_adjacent_find(), a2.begin(), a2.end() - 1, std::equal_to<T>());
-
-    invoke_on_all_policies(test_adjacent_find(), a2.cbegin(), a2.cend(), std::equal_to<T>());
-    invoke_on_all_policies(test_adjacent_find(), a2.cbegin(), a2.cend() - 1, std::equal_to<T>());
-
-    Sequence<T> a3 = {5, 6, 6, 6, 7, 9, 9, 9, 9};
-    invoke_on_all_policies(test_adjacent_find(), a3.begin(), a3.end(), std::equal_to<T>());
-
-    invoke_on_all_policies(test_adjacent_find(), a3.cbegin(), a3.cend(), std::equal_to<T>());
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        adjacent_find(exec, iter, iter, non_const(std::equal_to<T>()));
-    }
-};
-
-int
-main()
-{
-
-    test_adjacent_find_by_type<int32_t>();
-    test_adjacent_find_by_type<float64_t>();
-
-    test_algo_basic_single<int32_t>(run_for_rnd_bi<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
deleted file mode 100644
index ec23dc47e2f9..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/all_of.pass.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-// -*- C++ -*-
-//===-- all_of.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-/*
-  TODO: consider implementing the following tests for a better code coverage
-  - correctness
-  - bad input argument (if applicable)
-  - data corruption around/of input and output
-  - correctly work with nested parallelism
-  - check that algorithm does not require anything more than is described in its requirements section
-*/
-
-using namespace TestUtils;
-
-struct test_all_of
-{
-    template <typename ExecutionPolicy, typename Iterator, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator begin, Iterator end, Predicate pred, bool expected)
-    {
-
-        auto actualr = std::all_of(exec, begin, end, pred);
-        EXPECT_EQ(expected, actualr, "result for all_of");
-    }
-};
-
-template <typename T>
-struct Parity
-{
-    bool parity;
-
-  public:
-    Parity(bool parity_) : parity(parity_) {}
-    bool
-    operator()(T value) const
-    {
-        return (size_t(value) ^ parity) % 2 == 0;
-    }
-};
-
-template <typename T>
-void
-test(size_t bits)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-
-        // Sequence of odd values
-        Sequence<T> in(n, [n, bits](size_t) { return T(2 * HashBits(n, bits - 1) ^ 1); });
-
-        // Even value, or false when T is bool.
-        T spike(2 * HashBits(n, bits - 1));
-        Sequence<T> inCopy(in);
-
-        invoke_on_all_policies(test_all_of(), in.begin(), in.end(), Parity<T>(1), true);
-        invoke_on_all_policies(test_all_of(), in.cbegin(), in.cend(), Parity<T>(1), true);
-        EXPECT_EQ(in, inCopy, "all_of modified input sequence");
-        if (n > 0)
-        {
-            // Sprinkle in a miss
-            in[2 * n / 3] = spike;
-            invoke_on_all_policies(test_all_of(), in.begin(), in.end(), Parity<T>(1), false);
-            invoke_on_all_policies(test_all_of(), in.cbegin(), in.cend(), Parity<T>(1), false);
-
-            // Sprinkle in a few more misses
-            in[n / 2] = spike;
-            in[n / 3] = spike;
-            invoke_on_all_policies(test_all_of(), in.begin(), in.end(), Parity<T>(1), false);
-            invoke_on_all_policies(test_all_of(), in.cbegin(), in.cend(), Parity<T>(1), false);
-        }
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        all_of(exec, iter, iter, non_const(is_even));
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(8 * sizeof(int32_t));
-    test<uint16_t>(8 * sizeof(uint16_t));
-    test<float64_t>(53);
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>(1);
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
deleted file mode 100644
index 97d1691fa34e..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/any_of.pass.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-// -*- C++ -*-
-//===-- any_of.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-/*
-  TODO: consider implementing the following tests for a better code coverage
-  - correctness
-  - bad input argument (if applicable)
-  - data corruption around/of input and output
-  - correctly work with nested parallelism
-  - check that algorithm does not require anything more than is described in its requirements section
-*/
-
-using namespace TestUtils;
-
-struct test_any_of
-{
-    template <typename ExecutionPolicy, typename Iterator, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator begin, Iterator end, Predicate pred, bool expected)
-    {
-
-        auto actualr = std::any_of(exec, begin, end, pred);
-        EXPECT_EQ(expected, actualr, "result for any_of");
-    }
-};
-
-template <typename T>
-void
-test(size_t bits)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-
-        // Sequence of odd values
-        Sequence<T> in(n, [n, bits](size_t) { return T(2 * HashBits(n, bits - 1) ^ 1); });
-
-        // Even value, or false when T is bool.
-        T spike(2 * HashBits(n, bits - 1));
-        Sequence<T> inCopy(in);
-
-        invoke_on_all_policies(test_any_of(), in.begin(), in.end(), is_equal_to<T>(spike), false);
-        invoke_on_all_policies(test_any_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), false);
-        EXPECT_EQ(in, inCopy, "any_of modified input sequence");
-        if (n > 0)
-        {
-            // Sprinkle in a hit
-            in[2 * n / 3] = spike;
-            invoke_on_all_policies(test_any_of(), in.begin(), in.end(), is_equal_to<T>(spike), true);
-            invoke_on_all_policies(test_any_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), true);
-
-            // Sprinkle in a few more hits
-            in[n / 2] = spike;
-            in[n / 3] = spike;
-            invoke_on_all_policies(test_any_of(), in.begin(), in.end(), is_equal_to<T>(spike), true);
-            invoke_on_all_policies(test_any_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), true);
-        }
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        any_of(exec, iter, iter, non_const(is_even));
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(8 * sizeof(int32_t));
-    test<uint16_t>(8 * sizeof(uint16_t));
-    test<float64_t>(53);
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>(1);
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
deleted file mode 100644
index e8eca023d15a..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/count.pass.cpp
+++ /dev/null
@@ -1,108 +0,0 @@
-// -*- C++ -*-
-//===-- count.pass.cpp ----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for count and count_if
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_count
-{
-    template <typename Policy, typename Iterator, typename T>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, T needle)
-    {
-        auto expected = std::count(first, last, needle);
-        auto result = std::count(exec, first, last, needle);
-        EXPECT_EQ(expected, result, "wrong count result");
-    }
-};
-
-struct test_count_if
-{
-    template <typename Policy, typename Iterator, typename Predicate>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Predicate pred)
-    {
-        auto expected = std::count_if(first, last, pred);
-        auto result = std::count_if(exec, first, last, pred);
-        EXPECT_EQ(expected, result, "wrong count_if result");
-    }
-};
-
-template <typename T>
-class IsEqual
-{
-    T value;
-
-  public:
-    IsEqual(T value_, OddTag) : value(value_) {}
-    bool
-    operator()(const T& x) const
-    {
-        return x == value;
-    }
-};
-
-template <typename In, typename T, typename Predicate, typename Convert>
-void
-test(T needle, Predicate pred, Convert convert)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<In> in(n, [=](size_t k) -> In {
-            // Sprinkle "42" and "50" early, so that short sequences have non-zero count.
-            return convert((n - k - 1) % 3 == 0 ? 42 : (n - k - 2) % 5 == 0 ? 50 : 3 * (int(k) % 1000 - 500));
-        });
-        invoke_on_all_policies(test_count(), in.begin(), in.end(), needle);
-        invoke_on_all_policies(test_count_if(), in.begin(), in.end(), pred);
-
-        invoke_on_all_policies(test_count(), in.cbegin(), in.cend(), needle);
-        invoke_on_all_policies(test_count_if(), in.cbegin(), in.cend(), pred);
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        count_if(exec, iter, iter, non_const(is_even));
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(42, IsEqual<int32_t>(50, OddTag()), [](int32_t j) { return j; });
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN)
-    test<int32_t>(42, [](const int32_t&) { return true; }, [](int32_t j) { return j; });
-#endif
-    test<float64_t>(42, IsEqual<float64_t>(50, OddTag()), [](int32_t j) { return float64_t(j); });
-    test<Number>(Number(42, OddTag()), IsEqual<Number>(Number(50, OddTag()), OddTag()),
-                 [](int32_t j) { return Number(j, OddTag()); });
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
deleted file mode 100644
index a6983eae2ff1..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/equal.pass.cpp
+++ /dev/null
@@ -1,168 +0,0 @@
-// -*- C++ -*-
-//===-- equal.pass.cpp ----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-#define CPP14_ENABLED 0
-
-struct UserType
-{
-    size_t key;
-    float32_t f;
-    float64_t d;
-    int32_t i;
-
-    bool
-    operator()(UserType a, UserType b)
-    {
-        return a.key < b.key;
-    }
-    bool
-    operator<(UserType a)
-    {
-        return a.key < key;
-    }
-    bool
-    operator>=(UserType a)
-    {
-        return a.key <= key;
-    }
-    bool
-    operator<=(UserType a)
-    {
-        return a.key >= key;
-    }
-    bool
-    operator==(UserType a)
-    {
-        return a.key == key;
-    }
-    bool
-    operator==(UserType a) const
-    {
-        return a.key == key;
-    }
-    bool
-    operator!=(UserType a)
-    {
-        return a.key != key;
-    }
-    UserType operator!()
-    {
-        UserType tmp;
-        tmp.key = !key;
-        return tmp;
-    }
-    friend std::ostream&
-    operator<<(std::ostream& stream, const UserType a)
-    {
-        stream << a.key;
-        return stream;
-    }
-
-    UserType() : key(-1), f(0.0f), d(0.0), i(0) {}
-    UserType(size_t Number) : key(Number), f(0.0f), d(0.0), i(0) {}
-    UserType&
-    operator=(const UserType& other)
-    {
-        key = other.key;
-        return *this;
-    }
-    UserType(const UserType& other) : key(other.key), f(other.f), d(other.d), i(other.i) {}
-    UserType(UserType&& other) : key(other.key), f(other.f), d(other.d), i(other.i)
-    {
-        other.key = -1;
-        other.f = 0.0f;
-        other.d = 0.0;
-        other.i = 0;
-    }
-};
-
-struct test_one_policy
-{
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, bool is_true_equal)
-    {
-        using namespace std;
-
-        auto expected = equal(first1, last1, first2);
-        auto actual = equal(exec, first1, last1, first2);
-        EXPECT_EQ(expected, actual, "result for equal for random-access iterator, checking against std::equal()");
-
-        // testing bool
-        EXPECT_TRUE(is_true_equal == actual, "result for equal for random-access iterator, bool");
-
-//add C++14 equal symantics tests
-//add more cases for inCopy size less than in
-#if CPP14_ENABLED
-        auto actualr14 = std::equal(in.cbegin(), in.cend(), inCopy.cbegin(), inCopy.cend());
-        EXPECT_EQ(expected, actualr14, "result for equal for random-access iterator");
-#endif
-    }
-};
-
-template <typename T>
-void
-test(size_t bits)
-{
-    for (size_t n = 1; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-
-        // Sequence of odd values
-        Sequence<T> in(n, [bits](size_t k) { return T(2 * HashBits(k, bits - 1) ^ 1); });
-        Sequence<T> inCopy(in);
-
-        invoke_on_all_policies(test_one_policy(), in.begin(), in.end(), inCopy.begin(), true);
-        invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cend(), inCopy.cbegin(), true);
-
-        // testing bool !equal()
-        inCopy[0] = !inCopy[0];
-        invoke_on_all_policies(test_one_policy(), in.begin(), in.end(), inCopy.begin(), false);
-        invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cend(), inCopy.cbegin(), false);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename FirstIterator, typename SecondInterator>
-    void
-    operator()(Policy&& exec, FirstIterator first_iter, SecondInterator second_iter)
-    {
-        equal(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::equal_to<T>()));
-    }
-};
-
-int
-main()
-{
-
-    test<int32_t>(8 * sizeof(int32_t));
-    test<uint16_t>(8 * sizeof(uint16_t));
-    test<float64_t>(53);
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>(1);
-#endif
-    test<UserType>(256);
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
deleted file mode 100644
index 54b25c27ea2f..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/find.pass.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-// -*- C++ -*-
-//===-- find.pass.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for find
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_find
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator, typename Value>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator first, Iterator last, Value value)
-    {
-    }
-    template <typename Iterator, typename Value>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator first, Iterator last, Value value)
-    {
-    }
-#endif
-
-    template <typename Policy, typename Iterator, typename Value>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Value value)
-    {
-        auto i = std::find(first, last, value);
-        auto j = find(exec, first, last, value);
-        EXPECT_TRUE(i == j, "wrong return value from find");
-    }
-};
-
-template <typename T, typename Value, typename Hit, typename Miss>
-void
-test(Value value, Hit hit, Miss miss)
-{
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [&](size_t k) -> T { return miss(n ^ k); });
-        // Try different find positions, including not found.
-        // By going backwards, we can add extra matches that are *not* supposed to be found.
-        // The decreasing exponential gives us O(n) total work for the loop since each find takes O(m) time.
-        for (size_t m = n; m > 0; m *= 0.6)
-        {
-            if (m < n)
-                in[m] = hit(n ^ m);
-            invoke_on_all_policies(test_find(), in.begin(), in.end(), value);
-            invoke_on_all_policies(test_find(), in.cbegin(), in.cend(), value);
-        }
-    }
-}
-
-// Type defined for sake of checking that std::find works with asymmetric ==.
-class Weird
-{
-    Number value;
-
-  public:
-    friend bool
-    operator==(Number x, Weird y)
-    {
-        return x == y.value;
-    }
-    Weird(int32_t val, OddTag) : value(val, OddTag()) {}
-};
-
-int
-main()
-{
-    // Note that the "hit" and "miss" functions here avoid overflow issues.
-    test<Number>(Weird(42, OddTag()), [](int32_t) { return Number(42, OddTag()); }, // hit
-                 [](int32_t j) { return Number(j == 42 ? 0 : j, OddTag()); });      // miss
-
-    // Test with value that is equal to two different bit patterns (-0.0 and 0.0)
-    test<float32_t>(-0.0, [](int32_t j) { return j & 1 ? 0.0 : -0.0; }, // hit
-                    [](int32_t j) { return j == 0 ? ~j : j; });         // miss
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
deleted file mode 100644
index ed0185f7b0c5..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_end.pass.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-// -*- C++ -*-
-//===-- find_end.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub,
-               Predicate pred)
-    {
-    }
-    template <typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub,
-               Predicate pred)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub, Predicate pred)
-    {
-        using namespace std;
-        // For find_end
-        {
-            auto expected = find_end(b, e, bsub, esub, pred);
-            auto actual = find_end(exec, b, e, bsub, esub);
-            EXPECT_TRUE(actual == expected, "wrong return result from find_end");
-
-            actual = find_end(exec, b, e, bsub, esub, pred);
-            EXPECT_TRUE(actual == expected, "wrong return result from find_end with a predicate");
-        }
-
-        // For search
-        {
-            auto expected = search(b, e, bsub, esub, pred);
-            auto actual = search(exec, b, e, bsub, esub);
-            EXPECT_TRUE(actual == expected, "wrong return result from search");
-
-            actual = search(exec, b, e, bsub, esub, pred);
-            EXPECT_TRUE(actual == expected, "wrong return result from search with a predicate");
-        }
-    }
-};
-
-template <typename T>
-void
-test(const std::size_t bits)
-{
-
-    const std::size_t max_n1 = 1000;
-    const std::size_t max_n2 = (max_n1 * 10) / 8;
-    Sequence<T> in(max_n1, [bits](std::size_t) { return T(2 * HashBits(max_n1, bits - 1) ^ 1); });
-    Sequence<T> sub(max_n2, [bits](std::size_t) { return T(2 * HashBits(max_n1, bits - 1)); });
-    for (std::size_t n1 = 0; n1 <= max_n1; n1 = n1 <= 16 ? n1 + 1 : size_t(3.1415 * n1))
-    {
-        std::size_t sub_n[] = {0, 1, 3, n1, (n1 * 10) / 8};
-        std::size_t res[] = {0, 1, n1 / 2, n1};
-        for (auto n2 : sub_n)
-        {
-            for (auto r : res)
-            {
-                std::size_t i = r, isub = 0;
-                for (; i < n1 && isub < n2; ++i, ++isub)
-                    in[i] = sub[isub];
-                invoke_on_all_policies(test_one_policy(), in.begin(), in.begin() + n1, sub.begin(), sub.begin() + n2,
-                                       std::equal_to<T>());
-                invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cbegin() + n1, sub.cbegin(),
-                                       sub.cbegin() + n2, std::equal_to<T>());
-            }
-        }
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename FirstIterator, typename SecondInterator>
-    void
-    operator()(Policy&& exec, FirstIterator first_iter, SecondInterator second_iter)
-    {
-        invoke_if(exec, [&]() {
-            find_end(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::equal_to<T>()));
-            search(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::equal_to<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(8 * sizeof(int32_t));
-    test<uint16_t>(8 * sizeof(uint16_t));
-    test<float64_t>(53);
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>(1);
-#endif
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
deleted file mode 100644
index 5b4801e9b3a8..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_first_of.pass.cpp
+++ /dev/null
@@ -1,112 +0,0 @@
-// -*- C++ -*-
-//===-- find_first_of.pass.cpp --------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub,
-               Predicate pred)
-    {
-    }
-    template <typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub,
-               Predicate pred)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 b, Iterator1 e, Iterator2 bsub, Iterator2 esub, Predicate pred)
-    {
-        using namespace std;
-        Iterator1 expected = find_first_of(b, e, bsub, esub, pred);
-        Iterator1 actual = find_first_of(exec, b, e, bsub, esub, pred);
-        EXPECT_TRUE(actual == expected, "wrong return result from find_first_of with a predicate");
-
-        expected = find_first_of(b, e, bsub, esub);
-        actual = find_first_of(exec, b, e, bsub, esub);
-        EXPECT_TRUE(actual == expected, "wrong return result from find_first_of");
-    }
-};
-
-template <typename T, typename Predicate>
-void
-test(Predicate pred)
-{
-
-    const std::size_t max_n1 = 1000;
-    const std::size_t max_n2 = (max_n1 * 10) / 8;
-    Sequence<T> in1(max_n1, [](std::size_t) { return T(1); });
-    Sequence<T> in2(max_n2, [](std::size_t) { return T(0); });
-    for (std::size_t n1 = 0; n1 <= max_n1; n1 = n1 <= 16 ? n1 + 1 : size_t(3.1415 * n1))
-    {
-        std::size_t sub_n[] = {0, 1, n1 / 3, n1, (n1 * 10) / 8};
-        for (const auto n2 : sub_n)
-        {
-            invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.data(), in2.data() + n2, pred);
-
-            in2[n2 / 2] = T(1);
-            invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cbegin() + n1, in2.data(), in2.data() + n2,
-                                   pred);
-
-            if (n2 >= 3)
-            {
-                in2[2 * n2 / 3] = T(1);
-                invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cbegin() + n1, in2.begin(),
-                                       in2.begin() + n2, pred);
-                in2[2 * n2 / 3] = T(0);
-            }
-            in2[n2 / 2] = T(0);
-        }
-    }
-    invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + max_n1 / 10, in1.data(),
-                           in1.data() + max_n1 / 10, pred);
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename FirstIterator, typename SecondInterator>
-    void
-    operator()(Policy&& exec, FirstIterator first_iter, SecondInterator second_iter)
-    {
-        invoke_if(exec, [&]() {
-            find_first_of(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::equal_to<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(std::equal_to<int32_t>());
-    test<uint16_t>(std::not_equal_to<uint16_t>());
-    test<float64_t>([](const float64_t x, const float64_t y) { return x * x == y * y; });
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
deleted file mode 100644
index 180d003996dd..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/find_if.pass.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-// -*- C++ -*-
-//===-- find_if.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for find_if and find_if_not
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_find_if
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator, typename Predicate, typename NotPredicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator first, Iterator last, Predicate pred,
-               NotPredicate not_pred)
-    {
-    }
-    template <typename Iterator, typename Predicate, typename NotPredicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator first, Iterator last, Predicate pred,
-               NotPredicate not_pred)
-    {
-    }
-#endif
-
-    template <typename Policy, typename Iterator, typename Predicate, typename NotPredicate>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Predicate pred, NotPredicate not_pred)
-    {
-        auto i = std::find_if(first, last, pred);
-        auto j = find_if(exec, first, last, pred);
-        EXPECT_TRUE(i == j, "wrong return value from find_if");
-        auto i_not = find_if_not(exec, first, last, not_pred);
-        EXPECT_TRUE(i_not == i, "wrong return value from find_if_not");
-    }
-};
-
-template <typename T, typename Predicate, typename Hit, typename Miss>
-void
-test(Predicate pred, Hit hit, Miss miss)
-{
-    auto not_pred = [pred](T x) { return !pred(x); };
-    // Try sequences of various lengths.
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [&](size_t k) -> T { return miss(n ^ k); });
-        // Try different find positions, including not found.
-        // By going backwards, we can add extra matches that are *not* supposed to be found.
-        // The decreasing exponential gives us O(n) total work for the loop since each find takes O(m) time.
-        for (size_t m = n; m > 0; m *= 0.6)
-        {
-            if (m < n)
-                in[m] = hit(n ^ m);
-            invoke_on_all_policies(test_find_if(), in.begin(), in.end(), pred, not_pred);
-            invoke_on_all_policies(test_find_if(), in.cbegin(), in.cend(), pred, not_pred);
-        }
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-
-        invoke_if(exec, [&]() {
-            find_if(exec, iter, iter, non_const(is_even));
-            find_if_not(exec, iter, iter, non_const(is_even));
-        });
-    }
-};
-
-int
-main()
-{
-#if !defined(_PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN)
-    // Note that the "hit" and "miss" functions here avoid overflow issues.
-    test<Number>(IsMultiple(5, OddTag()), [](int32_t j) { return Number(j - j % 5, OddTag()); }, // hit
-                 [](int32_t j) { return Number(j % 5 == 0 ? j ^ 1 : j, OddTag()); });            // miss
-#endif
-
-    // Try type for which algorithm can really be vectorized.
-    test<float32_t>([](float32_t x) { return x >= 0; }, [](float32_t j) { return j * j; },
-                    [](float32_t j) { return -1 - j * j; });
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
deleted file mode 100644
index 14a0ad4f32b1..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/for_each.pass.cpp
+++ /dev/null
@@ -1,102 +0,0 @@
-// -*- C++ -*-
-//===-- for_each.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename Type>
-struct Gen
-{
-    Type
-    operator()(std::size_t k)
-    {
-        return Type(k % 5 != 1 ? 3 * k - 7 : 0);
-    };
-};
-
-template <typename T>
-struct Flip
-{
-    int32_t val;
-    Flip(int32_t y) : val(y) {}
-    T
-    operator()(T& x) const
-    {
-        return x = val - x;
-    }
-};
-
-struct test_one_policy
-{
-    template <typename Policy, typename Iterator, typename Size>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Iterator expected_first, Iterator expected_last, Size n)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-
-        // Try for_each
-        std::for_each(expected_first, expected_last, Flip<T>(1));
-        for_each(exec, first, last, Flip<T>(1));
-        EXPECT_EQ_N(expected_first, first, n, "wrong effect from for_each");
-
-        // Try for_each_n
-        std::for_each_n(std::execution::seq, expected_first, n, Flip<T>(1));
-        for_each_n(exec, first, n, Flip<T>(1));
-        EXPECT_EQ_N(expected_first, first, n, "wrong effect from for_each_n");
-    }
-};
-
-template <typename T>
-void
-test()
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> inout(n, Gen<T>());
-        Sequence<T> expected(n, Gen<T>());
-        invoke_on_all_policies(test_one_policy(), inout.begin(), inout.end(), expected.begin(), expected.end(),
-                               inout.size());
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() {
-            auto f = [](typename std::iterator_traits<Iterator>::reference x) { x = x + 1; };
-
-            for_each(exec, iter, iter, non_const(f));
-            for_each_n(exec, iter, 0, non_const(f));
-        });
-    }
-};
-
-int
-main()
-{
-    test<int32_t>();
-    test<uint16_t>();
-    test<float64_t>();
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
deleted file mode 100644
index 4d83ad6de782..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/mismatch.pass.cpp
+++ /dev/null
@@ -1,132 +0,0 @@
-// -*- C++ -*-
-//===-- mismatch.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_mismatch
-{
-    template <typename Policy, typename Iterator1, typename Iterator2>
-    void
-    operator()(Policy&& exec, Iterator1 first1, Iterator1 last1, Iterator2 first2)
-    {
-        using namespace std;
-        typedef typename iterator_traits<Iterator1>::value_type T;
-        {
-            const auto expected = std::mismatch(first1, last1, first2, std::equal_to<T>());
-            const auto res3 = mismatch(exec, first1, last1, first2, std::equal_to<T>());
-            EXPECT_TRUE(expected == res3, "wrong return result from mismatch");
-            const auto res4 = mismatch(exec, first1, last1, first2);
-            EXPECT_TRUE(expected == res4, "wrong return result from mismatch");
-        }
-    }
-    template <typename Policy, typename Iterator1, typename Iterator2>
-    void
-    operator()(Policy&& exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, Iterator2 last2)
-    {
-        using namespace std;
-        typedef typename iterator_traits<Iterator1>::value_type T;
-        {
-            const auto expected = mismatch(std::execution::seq, first1, last1, first2, last2, std::equal_to<T>());
-            const auto res1 = mismatch(exec, first1, last1, first2, last2, std::equal_to<T>());
-            EXPECT_TRUE(expected == res1, "wrong return result from mismatch");
-            const auto res2 = mismatch(exec, first1, last1, first2, last2);
-            EXPECT_TRUE(expected == res2, "wrong return result from mismatch");
-        }
-    }
-};
-
-template <typename T>
-void
-test_mismatch_by_type()
-{
-    using namespace std;
-    for (size_t size = 0; size <= 100000; size = size <= 16 ? size + 1 : size_t(3.1415 * size))
-    {
-        const T val = T(-1);
-        Sequence<T> in(size, [](size_t v) -> T { return T(v % 100); });
-        {
-            Sequence<T> in2(in);
-            invoke_on_all_policies(test_mismatch(), in.begin(), in.end(), in2.begin(), in2.end());
-            invoke_on_all_policies(test_mismatch(), in.begin(), in.end(), in2.begin());
-
-            const size_t min_size = 3;
-            if (size > min_size)
-            {
-                const size_t idx_for_1 = size / min_size;
-                in[idx_for_1] = val, in[idx_for_1 + 1] = val, in[idx_for_1 + 2] = val;
-                invoke_on_all_policies(test_mismatch(), in.begin(), in.end(), in2.begin(), in2.end());
-                invoke_on_all_policies(test_mismatch(), in.begin(), in.end(), in2.begin());
-            }
-
-            const size_t idx_for_2 = 500;
-            if (size >= idx_for_2 - 1)
-            {
-                in2[size / idx_for_2] = val;
-                invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin(), in2.cend());
-                invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin());
-            }
-        }
-        {
-            Sequence<T> in2(100, [](size_t v) -> T { return T(v); });
-            invoke_on_all_policies(test_mismatch(), in2.begin(), in2.end(), in.begin(), in.end());
-            //  We can't call std::mismatch with semantic below when size of second sequence less than size of first sequence
-            if (in2.size() <= in.size())
-                invoke_on_all_policies(test_mismatch(), in2.begin(), in2.end(), in.begin());
-
-            const size_t idx = 97;
-            in2[idx] = val;
-            in2[idx + 1] = val;
-            invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin(), in2.cend());
-            if (in.size() <= in2.size())
-                invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin());
-        }
-        {
-            Sequence<T> in2({});
-            invoke_on_all_policies(test_mismatch(), in2.begin(), in2.end(), in.begin(), in.end());
-
-            invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin(), in2.cend());
-            if (in.size() == 0)
-                invoke_on_all_policies(test_mismatch(), in.cbegin(), in.cend(), in2.cbegin());
-        }
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename FirstIterator, typename SecondInterator>
-    void
-    operator()(Policy&& exec, FirstIterator first_iter, SecondInterator second_iter)
-    {
-        mismatch(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-
-    test_mismatch_by_type<int32_t>();
-    test_mismatch_by_type<float64_t>();
-    test_mismatch_by_type<Wrapper<int32_t>>();
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
deleted file mode 100644
index dbdcd54438c4..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/none_of.pass.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-// -*- C++ -*-
-//===-- none_of.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-/*
-  TODO: consider implementing the following tests for a better code coverage
-  - correctness
-  - bad input argument (if applicable)
-  - data corruption around/of input and output
-  - correctly work with nested parallelism
-  - check that algorithm does not require anything more than is described in its requirements section
-*/
-
-using namespace TestUtils;
-
-struct test_none_of
-{
-    template <typename ExecutionPolicy, typename Iterator, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator begin, Iterator end, Predicate pred, bool expected)
-    {
-
-        auto actualr = std::none_of(exec, begin, end, pred);
-        EXPECT_EQ(expected, actualr, "result for none_of");
-    }
-};
-
-template <typename T>
-void
-test(size_t bits)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-
-        // Sequence of odd values
-        Sequence<T> in(n, [n, bits](size_t) { return T(2 * HashBits(n, bits - 1) ^ 1); });
-
-        // Even value, or false when T is bool.
-        T spike(2 * HashBits(n, bits - 1));
-
-        invoke_on_all_policies(test_none_of(), in.begin(), in.end(), is_equal_to<T>(spike), true);
-        invoke_on_all_policies(test_none_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), true);
-        if (n > 0)
-        {
-            // Sprinkle in a hit
-            in[2 * n / 3] = spike;
-            invoke_on_all_policies(test_none_of(), in.begin(), in.end(), is_equal_to<T>(spike), false);
-            invoke_on_all_policies(test_none_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), false);
-
-            // Sprinkle in a few more hits
-            in[n / 3] = spike;
-            in[n / 2] = spike;
-            invoke_on_all_policies(test_none_of(), in.begin(), in.end(), is_equal_to<T>(spike), false);
-            invoke_on_all_policies(test_none_of(), in.cbegin(), in.cend(), is_equal_to<T>(spike), false);
-        }
-    }
-}
-
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        auto is_even = [&](float64_t v) {
-            uint32_t i = (uint32_t)v;
-            return i % 2 == 0;
-        };
-        none_of(exec, iter, iter, non_const(is_even));
-    }
-};
-
-int
-main()
-{
-    test<int32_t>(8 * sizeof(int32_t));
-    test<uint16_t>(8 * sizeof(uint16_t));
-    test<float64_t>(53);
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>(1);
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
deleted file mode 100644
index f3e43da39a3a..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/nth_element.pass.cpp
+++ /dev/null
@@ -1,175 +0,0 @@
-// -*- C++ -*-
-//===-- nth_element.pass.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iostream>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// User defined type with minimal requirements
-template <typename T>
-struct DataType
-{
-    explicit DataType(int32_t k) : my_val(k) {}
-    DataType(DataType&& input)
-    {
-        my_val = std::move(input.my_val);
-        input.my_val = T(0);
-    }
-    DataType&
-    operator=(DataType&& input)
-    {
-        my_val = std::move(input.my_val);
-        input.my_val = T(0);
-        return *this;
-    }
-    T
-    get_val() const
-    {
-        return my_val;
-    }
-
-    friend std::ostream&
-    operator<<(std::ostream& stream, const DataType<T>& input)
-    {
-        return stream << input.my_val;
-    }
-
-  private:
-    T my_val;
-};
-
-template <typename T>
-bool
-is_equal(const DataType<T>& x, const DataType<T>& y)
-{
-    return x.get_val() == y.get_val();
-}
-
-template <typename T>
-bool
-is_equal(const T& x, const T& y)
-{
-    return x == y;
-}
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Size, typename Generator1, typename Generator2, typename Compare>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator1 first1, Iterator1 last1, Iterator1 first2,
-               Iterator1 last2, Size n, Size m, Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-    }
-    template <typename Iterator1, typename Size, typename Generator1, typename Generator2, typename Compare>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 first1, Iterator1 last1, Iterator1 first2,
-               Iterator1 last2, Size n, Size m, Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-    }
-#endif
-
-    // nth_element works only with random access iterators
-    template <typename Policy, typename Iterator1, typename Size, typename Generator1, typename Generator2,
-              typename Compare>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, Iterator1 first1, Iterator1 last1, Iterator1 first2, Iterator1 last2, Size n, Size m,
-               Generator1 generator1, Generator2 generator2, Compare comp)
-    {
-
-        using T = typename std::iterator_traits<Iterator1>::value_type;
-        const Iterator1 mid1 = std::next(first1, m);
-        const Iterator1 mid2 = std::next(first2, m);
-
-        fill_data(first1, mid1, generator1);
-        fill_data(mid1, last1, generator2);
-        fill_data(first2, mid2, generator1);
-        fill_data(mid2, last2, generator2);
-        std::nth_element(first1, mid1, last1, comp);
-        std::nth_element(exec, first2, mid2, last2, comp);
-        if (m > 0 && m < n)
-        {
-            EXPECT_TRUE(is_equal(*mid1, *mid2), "wrong result from nth_element with predicate");
-        }
-        EXPECT_TRUE(std::find_first_of(first2, mid2, mid2, last2, [comp](T& x, T& y) { return comp(y, x); }) == mid2,
-                    "wrong effect from nth_element with predicate");
-    }
-
-    template <typename Policy, typename Iterator1, typename Size, typename Generator1, typename Generator2,
-              typename Compare>
-    typename std::enable_if<!is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&&, Iterator1, Iterator1, Iterator1, Iterator1, Size, Size, Generator1, Generator2, Compare)
-    {
-    }
-};
-
-template <typename T, typename Generator1, typename Generator2, typename Compare>
-void
-test_by_type(Generator1 generator1, Generator2 generator2, Compare comp)
-{
-    using namespace std;
-    size_t max_size = 10000;
-    Sequence<T> in1(max_size, [](size_t v) { return T(v); });
-    Sequence<T> exp(max_size, [](size_t v) { return T(v); });
-    size_t m;
-
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        m = 0;
-        invoke_on_all_policies(test_one_policy(), exp.begin(), exp.begin() + n, in1.begin(), in1.begin() + n, n, m,
-                               generator1, generator2, comp);
-        m = n / 7;
-        invoke_on_all_policies(test_one_policy(), exp.begin(), exp.begin() + n, in1.begin(), in1.begin() + n, n, m,
-                               generator1, generator2, comp);
-        m = 3 * n / 5;
-        invoke_on_all_policies(test_one_policy(), exp.begin(), exp.begin() + n, in1.begin(), in1.begin() + n, n, m,
-                               generator1, generator2, comp);
-    }
-    invoke_on_all_policies(test_one_policy(), exp.begin(), exp.begin() + max_size, in1.begin(), in1.begin() + max_size,
-                           max_size, max_size, generator1, generator2, comp);
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() { nth_element(exec, iter, iter, iter, non_const(std::less<T>())); });
-    }
-};
-
-int
-main()
-{
-    test_by_type<int32_t>([](int32_t i) { return 10 * i; }, [](int32_t i) { return i + 1; }, std::less<int32_t>());
-    test_by_type<int32_t>([](int32_t) { return 0; }, [](int32_t) { return 0; }, std::less<int32_t>());
-
-    test_by_type<float64_t>([](int32_t i) { return -2 * i; }, [](int32_t i) { return -(2 * i + 1); },
-                            [](const float64_t x, const float64_t y) { return x > y; });
-
-    test_by_type<DataType<float32_t>>(
-        [](int32_t i) { return DataType<float32_t>(2 * i + 1); }, [](int32_t i) { return DataType<float32_t>(2 * i); },
-        [](const DataType<float32_t>& x, const DataType<float32_t>& y) { return x.get_val() < y.get_val(); });
-
-    test_algo_basic_single<int32_t>(run_for_rnd<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp b/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
deleted file mode 100644
index 573f364edef2..000000000000
--- a/pstl/test/std/algorithms/alg.nonmodifying/search_n.pass.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-// -*- C++ -*-
-//===-- search_n.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator, typename Size, typename T, typename Predicate>
-    void
-    operator()(pstl::execution::unsequenced_policy, Iterator b, Iterator e, Size count, const T& value, Predicate pred)
-    {
-    }
-    template <typename Iterator, typename Size, typename T, typename Predicate>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator b, Iterator e, Size count, const T& value,
-               Predicate pred)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator, typename Size, typename T, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator b, Iterator e, Size count, const T& value, Predicate pred)
-    {
-        using namespace std;
-        auto expected = search_n(b, e, count, value, pred);
-        auto actual = search_n(exec, b, e, count, value);
-        EXPECT_TRUE(actual == expected, "wrong return result from search_n");
-
-        actual = search_n(exec, b, e, count, value, pred);
-        EXPECT_TRUE(actual == expected, "wrong return result from search_n with a predicate");
-    }
-};
-
-template <typename T>
-void
-test()
-{
-
-    const std::size_t max_n1 = 100000;
-    const T value = T(1);
-    for (std::size_t n1 = 0; n1 <= max_n1; n1 = n1 <= 16 ? n1 + 1 : size_t(3.1415 * n1))
-    {
-        std::size_t sub_n[] = {0, 1, 3, n1, (n1 * 10) / 8};
-        std::size_t res[] = {0, 1, n1 / 2, n1};
-        for (auto n2 : sub_n)
-        {
-            // Some of standard libraries return "first" in this case. We return "last" according to the standard
-            if (n2 == 0)
-            {
-                continue;
-            }
-            for (auto r : res)
-            {
-                Sequence<T> in(n1, [](std::size_t) { return T(0); });
-                std::size_t i = r, isub = 0;
-                for (; i < n1 && isub < n2; ++i, ++isub)
-                    in[i] = value;
-
-                invoke_on_all_policies(test_one_policy(), in.begin(), in.begin() + n1, n2, value, std::equal_to<T>());
-                invoke_on_all_policies(test_one_policy(), in.cbegin(), in.cbegin() + n1, n2, value, std::equal_to<T>());
-            }
-        }
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() { search_n(exec, iter, iter, 0, T(0), non_const(std::equal_to<T>())); });
-    }
-};
-
-int
-main()
-{
-    test<int32_t>();
-    test<uint16_t>();
-    test<float64_t>();
-#if !defined(_PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN)
-    test<bool>();
-#endif
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
deleted file mode 100644
index 08eca8eed1c5..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/alg.heap.operations/is_heap.pass.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-// -*- C++ -*-
-//===-- is_heap.pass.cpp --------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for is_heap, is_heap_until
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-#include <iostream>
-
-using namespace TestUtils;
-
-struct WithCmpOp
-{
-    int32_t _first;
-    int32_t _second;
-    WithCmpOp() : _first(0), _second(0){};
-    explicit WithCmpOp(int32_t x) : _first(x), _second(x){};
-    bool
-    operator<(const WithCmpOp& rhs) const
-    {
-        return this->_first < rhs._first;
-    }
-};
-
-struct test_is_heap
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator, typename Predicate>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator first, Iterator last, Predicate pred)
-    {
-    }
-    template <typename Iterator, typename Predicate>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator first, Iterator last, Predicate pred)
-    {
-    }
-#endif
-
-    template <typename Policy, typename Iterator, typename Predicate>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, Iterator first, Iterator last, Predicate pred)
-    {
-        using namespace std;
-        // is_heap
-        {
-            bool expected = is_heap(first, last);
-            bool actual = is_heap(exec, first, last);
-            EXPECT_TRUE(expected == actual, "wrong return value from is_heap");
-        }
-        // is_heap with predicate
-        {
-            bool expected = is_heap(first, last, pred);
-            bool actual = is_heap(exec, first, last, pred);
-            EXPECT_TRUE(expected == actual, "wrong return value from is_heap with predicate");
-        }
-        // is_heap_until
-        {
-            Iterator expected = is_heap_until(first, last);
-            Iterator actual = is_heap_until(exec, first, last);
-            EXPECT_TRUE(expected == actual, "wrong return value from is_heap_until");
-        }
-        // is_heap_until with predicate
-        {
-            const Iterator expected = is_heap_until(first, last, pred);
-            const auto y = std::distance(first, expected);
-            const Iterator actual = is_heap_until(exec, first, last, pred);
-            const auto x = std::distance(first, actual);
-            EXPECT_TRUE(expected == actual, "wrong return value from is_heap_until with predicate");
-            EXPECT_EQ(x, y, "both iterators should be the same distance away from 'first'");
-        }
-    }
-
-    // is_heap, is_heap_until works only with random access iterators
-    template <typename Policy, typename Iterator, typename Predicate>
-    typename std::enable_if<!is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&&, Iterator, Iterator, Predicate)
-    {
-    }
-};
-
-template <typename T, typename Comp>
-void
-test_is_heap_by_type(Comp comp)
-{
-    using namespace std;
-
-    const size_t max_size = 100000;
-    for (size_t n = 0; n <= max_size; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [](size_t v) -> T { return T(v); });
-
-        invoke_on_all_policies(test_is_heap(), in.begin(), in.end(), comp);
-
-        std::make_heap(in.begin(), in.begin() + n / 4, comp);
-        invoke_on_all_policies(test_is_heap(), in.cbegin(), in.cend(), comp);
-
-        std::make_heap(in.begin(), in.begin() + n / 3, comp);
-        invoke_on_all_policies(test_is_heap(), in.begin(), in.end(), comp);
-
-        std::make_heap(in.begin(), in.end(), comp);
-        invoke_on_all_policies(test_is_heap(), in.cbegin(), in.cend(), comp);
-    }
-
-    Sequence<T> in(max_size / 10, [](size_t) -> T { return T(1); });
-    invoke_on_all_policies(test_is_heap(), in.begin(), in.end(), comp);
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        invoke_if(exec, [&]() {
-            is_heap(exec, iter, iter, non_const(std::less<T>()));
-            is_heap_until(exec, iter, iter, non_const(std::less<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    test_is_heap_by_type<float32_t>(std::greater<float32_t>());
-    test_is_heap_by_type<WithCmpOp>(std::less<WithCmpOp>());
-    test_is_heap_by_type<uint64_t>([](uint64_t x, uint64_t y) { return x % 100 < y % 100; });
-
-    test_algo_basic_single<int32_t>(run_for_rnd<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
deleted file mode 100644
index cb920574ba35..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/alg.lex.comparison/lexicographical_compare.pass.cpp
+++ /dev/null
@@ -1,175 +0,0 @@
-// -*- C++ -*-
-//===-- lexicographical_compare.pass.cpp ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iostream>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_one_policy
-{
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 begin1, Iterator1 end1, Iterator2 begin2, Iterator2 end2,
-               Predicate pred)
-    {
-        const bool expected = std::lexicographical_compare(begin1, end1, begin2, end2, pred);
-        const bool actual = std::lexicographical_compare(exec, begin1, end1, begin2, end2, pred);
-        EXPECT_TRUE(actual == expected, "wrong return result from lexicographical compare with predicate");
-    }
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 begin1, Iterator1 end1, Iterator2 begin2, Iterator2 end2)
-    {
-        const bool expected = std::lexicographical_compare(begin1, end1, begin2, end2);
-        const bool actual = std::lexicographical_compare(exec, begin1, end1, begin2, end2);
-        EXPECT_TRUE(actual == expected, "wrong return result from lexicographical compare without predicate");
-    }
-};
-
-template <typename T1, typename T2, typename Predicate>
-void
-test(Predicate pred)
-{
-
-    const std::size_t max_n = 1000000;
-    Sequence<T1> in1(max_n, [](std::size_t k) { return T1(k); });
-    Sequence<T2> in2(2 * max_n, [](std::size_t k) { return T2(k); });
-
-    std::size_t n2;
-
-    // Test case: Call algorithm's version without predicate.
-    invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cbegin() + max_n, in2.cbegin() + 3 * max_n / 10,
-                           in2.cbegin() + 5 * max_n / 10);
-
-    // Test case: If one range is a prefix of another, the shorter range is lexicographically less than the other.
-    std::size_t max_n2 = max_n / 10;
-    invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + max_n, in2.cbegin(), in2.cbegin() + max_n2,
-                           pred);
-    invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + max_n, in2.begin() + max_n2,
-                           in2.begin() + 3 * max_n2, pred);
-
-    // Test case: If one range is a prefix of another, the shorter range is lexicographically less than the other.
-    max_n2 = 2 * max_n;
-    invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cbegin() + max_n, in2.begin(), in2.begin() + max_n2,
-                           pred);
-
-    for (std::size_t n1 = 0; n1 <= max_n; n1 = n1 <= 16 ? n1 + 1 : std::size_t(3.1415 * n1))
-    {
-        // Test case: If two ranges have equivalent elements and are of the same length, then the ranges are lexicographically equal.
-        n2 = n1;
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.begin(), in2.begin() + n2, pred);
-
-        n2 = n1;
-        // Test case: two ranges have different elements and are of the same length (second sequence less than first)
-        std::size_t ind = n1 / 2;
-        in2[ind] = T2(-1);
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.begin(), in2.begin() + n2, pred);
-        in2[ind] = T2(ind);
-
-        // Test case: two ranges have different elements and are of the same length (first sequence less than second)
-        ind = n1 / 5;
-        in1[ind] = T1(-1);
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.cbegin(), in2.cbegin() + n2, pred);
-        in1[ind] = T1(ind);
-    }
-}
-
-template <typename Predicate>
-void
-test_string(Predicate pred)
-{
-
-    const std::size_t max_n = 1000000;
-    std::string in1 = "";
-    std::string in2 = "";
-    for (std::size_t n1 = 0; n1 <= max_n; ++n1)
-    {
-        in1 += n1;
-    }
-
-    for (std::size_t n1 = 0; n1 <= 2 * max_n; ++n1)
-    {
-        in2 += n1;
-    }
-
-    std::size_t n2;
-
-    for (std::size_t n1 = 0; n1 < in1.size(); n1 = n1 <= 16 ? n1 + 1 : std::size_t(3.1415 * n1))
-    {
-        // Test case: If two ranges have equivalent elements and are of the same length, then the ranges are lexicographically equal.
-        n2 = n1;
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.begin(), in2.begin() + n2, pred);
-
-        n2 = n1;
-        // Test case: two ranges have different elements and are of the same length (second sequence less than first)
-        in2[n1 / 2] = 'a';
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.begin(), in2.begin() + n2, pred);
-
-        // Test case: two ranges have different elements and are of the same length (first sequence less than second)
-        in1[n1 / 5] = 'a';
-        invoke_on_all_policies(test_one_policy(), in1.begin(), in1.begin() + n1, in2.cbegin(), in2.cbegin() + n2, pred);
-    }
-    invoke_on_all_policies(test_one_policy(), in1.cbegin(), in1.cbegin() + max_n, in2.cbegin() + 3 * max_n / 10,
-                           in2.cbegin() + 5 * max_n / 10);
-}
-
-template <typename T>
-struct LocalWrapper
-{
-    explicit LocalWrapper(std::size_t k) : my_val(k) {}
-    bool
-    operator<(const LocalWrapper<T>& w) const
-    {
-        return my_val < w.my_val;
-    }
-
-  private:
-    T my_val;
-};
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename FirstIterator, typename SecondInterator>
-    void
-    operator()(Policy&& exec, FirstIterator first_iter, SecondInterator second_iter)
-    {
-        invoke_if(exec, [&]() {
-            lexicographical_compare(exec, first_iter, first_iter, second_iter, second_iter, non_const(std::less<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    test<uint16_t, float64_t>(std::less<float64_t>());
-    test<float32_t, int32_t>(std::greater<float32_t>());
-#if !defined(_PSTL_ICC_18_TEST_EARLY_EXIT_AVX_RELEASE_BROKEN)
-    test<float64_t, int32_t>([](const float64_t x, const int32_t y) { return x * x < y * y; });
-#endif
-    test<LocalWrapper<int32_t>, LocalWrapper<int32_t>>(
-        [](const LocalWrapper<int32_t>& x, const LocalWrapper<int32_t>& y) { return x < y; });
-    test_string([](const char x, const char y) { return x < y; });
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
deleted file mode 100644
index 715b25014992..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/alg.min.max/minmax_element.pass.cpp
+++ /dev/null
@@ -1,192 +0,0 @@
-// -*- C++ -*-
-//===-- minmax_element.pass.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-#include <set>
-#include <cmath>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct check_minelement
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator begin, Iterator end)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-        const Iterator expect = std::min_element(begin, end);
-        const Iterator result = std::min_element(exec, begin, end);
-        const Iterator result_pred = std::min_element(exec, begin, end, std::less<T>());
-        EXPECT_TRUE(expect == result, "wrong return result from min_element");
-        EXPECT_TRUE(expect == result_pred, "wrong return result from min_element");
-    }
-};
-
-struct check_maxelement
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator begin, Iterator end)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-        const Iterator expect = std::max_element(begin, end);
-        const Iterator result = std::max_element(exec, begin, end);
-        const Iterator result_pred = std::max_element(exec, begin, end, std::less<T>());
-        EXPECT_TRUE(expect == result, "wrong return result from max_element");
-        EXPECT_TRUE(expect == result_pred, "wrong return result from max_element");
-    }
-};
-
-struct check_minmaxelement
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator begin, Iterator end)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-        const std::pair<Iterator, Iterator> expect = std::minmax_element(begin, end);
-        const std::pair<Iterator, Iterator> got = std::minmax_element(exec, begin, end);
-        const std::pair<Iterator, Iterator> got_pred = std::minmax_element(exec, begin, end, std::less<T>());
-        EXPECT_TRUE(expect.first == got.first, "wrong return result from minmax_element (min part)");
-        EXPECT_TRUE(expect.second == got.second, "wrong return result from minmax_element (max part)");
-        EXPECT_TRUE(expect == got_pred, "wrong return result from minmax_element");
-    }
-};
-
-template <typename T>
-struct sequence_wrapper
-{
-    TestUtils::Sequence<T> seq;
-    const T min_value;
-    const T max_value;
-    static const std::size_t bits = 30; // We assume that T can handle signed 2^bits+1 value
-
-    // TestUtils::HashBits returns value between 0 and (1<<bits)-1,
-    // therefore we could threat 1<<bits as maximum and -(1<<bits) as a minimum
-    sequence_wrapper(std::size_t n) : seq(n), min_value(-(1 << bits)), max_value(1 << bits) {}
-
-    void
-    pattern_fill()
-    {
-        seq.fill([](std::size_t i) -> T { return T(TestUtils::HashBits(i, bits)); });
-    }
-
-    // sets first one at position `at` and bunch of them farther
-    void
-    set_desired_value(std::size_t at, T value)
-    {
-        if (seq.size() == 0)
-            return;
-        seq[at] = value;
-
-        //Producing serveral red herrings
-        for (std::size_t i = at + 1; i < seq.size(); i += 1 + TestUtils::HashBits(i, 5))
-            seq[i] = value;
-    }
-};
-
-template <typename T>
-void
-test_by_type(std::size_t n)
-{
-    sequence_wrapper<T> wseq(n);
-
-    // to avoid overtesing we use std::set to leave only unique indexes
-    std::set<std::size_t> targets{0};
-    if (n > 1)
-    {
-        targets.insert(1);
-        targets.insert(2.718282 * n / 3);
-        targets.insert(n / 2);
-        targets.insert(n / 7.389056);
-        targets.insert(n - 1); // last
-    }
-
-    for (std::set<std::size_t>::iterator it = targets.begin(); it != targets.end(); ++it)
-    {
-        wseq.pattern_fill();
-        wseq.set_desired_value(*it, wseq.min_value);
-        TestUtils::invoke_on_all_policies(check_minelement(), wseq.seq.cbegin(), wseq.seq.cend());
-        TestUtils::invoke_on_all_policies(check_minelement(), wseq.seq.begin(), wseq.seq.end());
-
-        wseq.set_desired_value(*it, wseq.max_value);
-        TestUtils::invoke_on_all_policies(check_maxelement(), wseq.seq.cbegin(), wseq.seq.cend());
-        TestUtils::invoke_on_all_policies(check_maxelement(), wseq.seq.begin(), wseq.seq.end());
-
-        if (targets.size() > 1)
-        {
-            for (std::set<std::size_t>::reverse_iterator rit = targets.rbegin(); rit != targets.rend(); ++rit)
-            {
-                if (*rit == *it) // we requires at least 2 unique indexes in targets
-                    break;
-                wseq.pattern_fill();
-                wseq.set_desired_value(*it, wseq.min_value);  // setting minimum element
-                wseq.set_desired_value(*rit, wseq.max_value); // setting maximum element
-                TestUtils::invoke_on_all_policies(check_minmaxelement(), wseq.seq.cbegin(), wseq.seq.cend());
-                TestUtils::invoke_on_all_policies(check_minmaxelement(), wseq.seq.begin(), wseq.seq.end());
-            }
-        }
-        else
-        { // we must check this corner case; it can not be tested in loop above
-            TestUtils::invoke_on_all_policies(check_minmaxelement(), wseq.seq.cbegin(), wseq.seq.cend());
-            TestUtils::invoke_on_all_policies(check_minmaxelement(), wseq.seq.begin(), wseq.seq.end());
-        }
-    }
-}
-
-// should provide minimal requirements only
-struct OnlyLessCompare
-{
-    int32_t val;
-    OnlyLessCompare() : val(0) {}
-    OnlyLessCompare(int32_t val_) : val(val_) {}
-    bool
-    operator<(const OnlyLessCompare& other) const
-    {
-        return val < other.val;
-    }
-};
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        max_element(exec, iter, iter, non_const(std::less<T>()));
-        min_element(exec, iter, iter, non_const(std::less<T>()));
-        minmax_element(exec, iter, iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-    using TestUtils::float64_t;
-    const std::size_t N = 100000;
-
-    for (std::size_t n = 0; n < N; n = n < 16 ? n + 1 : size_t(3.14159 * n))
-    {
-        test_by_type<float64_t>(n);
-        test_by_type<OnlyLessCompare>(n);
-    }
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << TestUtils::done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
deleted file mode 100644
index 778d265f9ca7..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/includes.pass.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-// -*- C++ -*-
-//===-- includes.pass.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <algorithm>
-#include <cmath>
-#include <execution>
-#include <functional>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct Num
-{
-    T val;
-    explicit Num(const T& v) : val(v) {}
-
-    //for "includes" checks
-    template <typename T1>
-    bool
-    operator<(const Num<T1>& v1) const
-    {
-        return val < v1.val;
-    }
-
-    //The types Type1 and Type2 must be such that an object of type InputIt can be dereferenced and then implicitly converted to both of them
-    template <typename T1>
-    operator Num<T1>() const
-    {
-        return Num<T1>((T1)val);
-    }
-};
-
-struct test_one_policy
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
-    {
-
-        auto expect_res = std::includes(first1, last1, first2, last2, comp);
-        auto res = std::includes(exec, first1, last1, first2, last2, comp);
-
-        EXPECT_TRUE(expect_res == res, "wrong result for includes");
-    }
-
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
-    {
-    }
-};
-
-template <typename T1, typename T2, typename Compare>
-void
-test_includes(Compare compare)
-{
-
-    const std::size_t n_max = 1000000;
-
-    // The rand()%(2*n+1) encourages generation of some duplicates.
-    std::srand(42);
-
-    for (std::size_t n = 0; n < n_max; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        for (std::size_t m = 0; m < n_max; m = m <= 16 ? m + 1 : size_t(2.71828 * m))
-        {
-            //prepare the input ranges
-            Sequence<T1> in1(n, [](std::size_t k) { return rand() % (2 * k + 1); });
-            Sequence<T2> in2(m, [](std::size_t k) { return rand() % (k + 1); });
-
-            std::sort(in1.begin(), in1.end(), compare);
-            std::sort(in2.begin(), in2.end(), compare);
-
-            invoke_on_all_policies(test_one_policy(), in1.begin(), in1.end(), in2.cbegin(), in2.cend(), compare);
-
-            //test w/ non constant predicate
-            if (n < 5 && m < 5)
-                invoke_on_all_policies(test_one_policy(), in1.begin(), in1.end(), in2.cbegin(), in2.cend(),
-                                       non_const(compare));
-        }
-    }
-}
-
-int
-main()
-{
-
-    test_includes<float64_t, float64_t>(std::less<>());
-    test_includes<Num<int64_t>, Num<int32_t>>([](const Num<int64_t>& x, const Num<int32_t>& y) { return x < y; });
-    std::cout << done() << std::endl;
-
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp b/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
deleted file mode 100644
index 40ae3d92c8a0..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/alg.set.operations/set.pass.cpp
+++ /dev/null
@@ -1,280 +0,0 @@
-// -*- C++ -*-
-//===-- set.pass.cpp ------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <algorithm>
-#include <chrono>
-#include <cmath>
-#include <execution>
-#include <functional>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct Num
-{
-    T val;
-
-    Num() : val{} {}
-    Num(const T& v) : val(v) {}
-
-    //for "includes" checks
-    template <typename T1>
-    bool
-    operator<(const Num<T1>& v1) const
-    {
-        return val < v1.val;
-    }
-
-    //The types Type1 and Type2 must be such that an object of type InputIt can be dereferenced and then implicitly converted to both of them
-    template <typename T1>
-    operator Num<T1>() const
-    {
-        return Num<T1>((T1)val);
-    }
-
-    friend bool
-    operator==(const Num& v1, const Num& v2)
-    {
-        return v1.val == v2.val;
-    }
-};
-
-template <typename Type>
-struct test_set_union
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
-    {
-        using T1 = typename std::iterator_traits<InputIterator1>::value_type;
-
-        auto n1 = std::distance(first1, last1);
-        auto n2 = std::distance(first2, last2);
-        auto n = n1 + n2;
-        Sequence<T1> expect(n);
-        Sequence<T1> out(n);
-
-        auto expect_res = std::set_union(first1, last1, first2, last2, expect.begin(), comp);
-        auto res = std::set_union(exec, first1, last1, first2, last2, out.begin(), comp);
-
-        EXPECT_TRUE(expect_res - expect.begin() == res - out.begin(), "wrong result for set_union");
-        EXPECT_EQ_N(expect.begin(), out.begin(), std::distance(out.begin(), res), "wrong set_union effect");
-    }
-
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
-    {
-    }
-};
-
-template <typename Type>
-struct test_set_intersection
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
-    {
-        using T1 = typename std::iterator_traits<InputIterator1>::value_type;
-
-        auto n1 = std::distance(first1, last1);
-        auto n2 = std::distance(first2, last2);
-        auto n = n1 + n2;
-        Sequence<T1> expect(n);
-        Sequence<T1> out(n);
-
-        auto expect_res = std::set_intersection(first1, last1, first2, last2, expect.begin(), comp);
-        auto res = std::set_intersection(exec, first1, last1, first2, last2, out.begin(), comp);
-
-        EXPECT_TRUE(expect_res - expect.begin() == res - out.begin(), "wrong result for set_intersection");
-        EXPECT_EQ_N(expect.begin(), out.begin(), std::distance(out.begin(), res), "wrong set_intersection effect");
-    }
-
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
-    {
-    }
-};
-
-template <typename Type>
-struct test_set_difference
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
-    {
-        using T1 = typename std::iterator_traits<InputIterator1>::value_type;
-
-        auto n1 = std::distance(first1, last1);
-        auto n2 = std::distance(first2, last2);
-        auto n = n1 + n2;
-        Sequence<T1> expect(n);
-        Sequence<T1> out(n);
-
-        auto expect_res = std::set_difference(first1, last1, first2, last2, expect.begin(), comp);
-        auto res = std::set_difference(exec, first1, last1, first2, last2, out.begin(), comp);
-
-        EXPECT_TRUE(expect_res - expect.begin() == res - out.begin(), "wrong result for set_difference");
-        EXPECT_EQ_N(expect.begin(), out.begin(), std::distance(out.begin(), res), "wrong set_difference effect");
-    }
-
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
-    {
-    }
-};
-
-template <typename Type>
-struct test_set_symmetric_difference
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2,
-               Compare comp)
-    {
-        using T1 = typename std::iterator_traits<InputIterator1>::value_type;
-
-        auto n1 = std::distance(first1, last1);
-        auto n2 = std::distance(first2, last2);
-        auto n = n1 + n2;
-        Sequence<T1> expect(n);
-        Sequence<T1> out(n);
-
-        auto expect_res = std::set_symmetric_difference(first1, last1, first2, last2, expect.begin(), comp);
-        auto res = std::set_symmetric_difference(exec, first1, last1, first2, last2, out.begin(), comp);
-
-        EXPECT_TRUE(expect_res - expect.begin() == res - out.begin(), "wrong result for set_symmetric_difference");
-        EXPECT_EQ_N(expect.begin(), out.begin(), std::distance(out.begin(), res),
-                    "wrong set_symmetric_difference effect");
-    }
-
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename Compare>
-    typename std::enable_if<TestUtils::isReverse<InputIterator1>::value, void>::type
-    operator()(Policy&&, InputIterator1, InputIterator1, InputIterator2, InputIterator2, Compare)
-    {
-    }
-};
-
-template <typename T1, typename T2, typename Compare>
-void
-test_set(Compare compare)
-{
-
-    const std::size_t n_max = 100000;
-
-    // The rand()%(2*n+1) encourages generation of some duplicates.
-    std::srand(4200);
-
-    for (std::size_t n = 0; n < n_max; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        for (std::size_t m = 0; m < n_max; m = m <= 16 ? m + 1 : size_t(2.71828 * m))
-        {
-            //prepare the input ranges
-            Sequence<T1> in1(n, [](std::size_t k) { return rand() % (2 * k + 1); });
-            Sequence<T2> in2(m, [m](std::size_t k) { return (m % 2) * rand() + rand() % (k + 1); });
-
-            std::sort(in1.begin(), in1.end(), compare);
-            std::sort(in2.begin(), in2.end(), compare);
-
-            invoke_on_all_policies(test_set_union<T1>(), in1.begin(), in1.end(), in2.cbegin(), in2.cend(),
-                                        compare);
-
-            invoke_on_all_policies(test_set_intersection<T1>(), in1.begin(), in1.end(), in2.cbegin(), in2.cend(),
-                                        compare);
-
-            invoke_on_all_policies(test_set_difference<T1>(), in1.begin(), in1.end(), in2.cbegin(), in2.cend(),
-                                        compare);
-
-            invoke_on_all_policies(test_set_symmetric_difference<T1>(), in1.begin(), in1.end(), in2.cbegin(),
-                                        in2.cend(), compare);
-        }
-    }
-}
-
-template <typename T>
-struct test_non_const_set_difference
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        set_difference(exec, input_iter, input_iter, input_iter, input_iter, out_iter, non_const(std::less<T>()));
-    }
-};
-
-template <typename T>
-struct test_non_const_set_intersection
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        set_intersection(exec, input_iter, input_iter, input_iter, input_iter, out_iter, non_const(std::less<T>()));
-    }
-};
-
-template <typename T>
-struct test_non_const_set_symmetric_difference
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        set_symmetric_difference(exec, input_iter, input_iter, input_iter, input_iter, out_iter,
-                                 non_const(std::less<T>()));
-    }
-};
-
-template <typename T>
-struct test_non_const_set_union
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        set_union(exec, input_iter, input_iter, input_iter, input_iter, out_iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-
-    test_set<float64_t, float64_t>(std::less<>());
-    test_set<Num<int64_t>, Num<int32_t>>([](const Num<int64_t>& x, const Num<int32_t>& y) { return x < y; });
-
-    test_set<MemoryChecker, MemoryChecker>([](const MemoryChecker& val1, const MemoryChecker& val2) -> bool {
-        return val1.value() < val2.value();
-    });
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from set algorithms: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from set algorithms: number of ctors calls > num of dtors calls");
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const_set_difference<int32_t>>());
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const_set_intersection<int32_t>>());
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const_set_symmetric_difference<int32_t>>());
-
-    test_algo_basic_double<int32_t>(run_for_rnd_fw<test_non_const_set_union<int32_t>>());
-
-    std::cout << done() << std::endl;
-
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp b/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
deleted file mode 100644
index d4c6fe291ed8..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/is_sorted.pass.cpp
+++ /dev/null
@@ -1,100 +0,0 @@
-// -*- C++ -*-
-//===-- is_sorted.pass.cpp ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_is_sorted
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, bool exam)
-    {
-        using namespace std;
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-
-        //try random-access iterator
-        bool res = is_sorted(exec, first, last);
-        EXPECT_TRUE(exam == res, "is_sorted wrong result for random-access iterator");
-        auto iexam = is_sorted_until(first, last);
-        auto ires = is_sorted_until(exec, first, last);
-        EXPECT_TRUE(iexam == ires, "is_sorted_until wrong result for random-access iterator");
-
-        //try random-access iterator with a predicate
-        res = is_sorted(exec, first, last, std::less<T>());
-        EXPECT_TRUE(exam == res, "is_sorted wrong result for random-access iterator");
-        iexam = is_sorted_until(first, last, std::less<T>());
-        ires = is_sorted_until(exec, first, last, std::less<T>());
-        EXPECT_TRUE(iexam == ires, "is_sorted_until wrong result for random-access iterator");
-    }
-};
-
-template <typename T>
-void
-test_is_sorted_by_type()
-{
-
-    Sequence<T> in(99999, [](size_t v) -> T { return T(v); }); //fill 0..n
-
-    invoke_on_all_policies(test_is_sorted(), in.begin(), in.end(), std::is_sorted(in.begin(), in.end()));
-    invoke_on_all_policies(test_is_sorted(), in.cbegin(), in.cend(), std::is_sorted(in.begin(), in.end()));
-
-    in[in.size() / 2] = -1;
-    invoke_on_all_policies(test_is_sorted(), in.begin(), in.end(), std::is_sorted(in.begin(), in.end()));
-    invoke_on_all_policies(test_is_sorted(), in.cbegin(), in.cend(), std::is_sorted(in.begin(), in.end()));
-
-    in[1] = -1;
-    invoke_on_all_policies(test_is_sorted(), in.begin(), in.end(), std::is_sorted(in.begin(), in.end()));
-    invoke_on_all_policies(test_is_sorted(), in.cbegin(), in.cend(), std::is_sorted(in.begin(), in.end()));
-
-    //an empty container
-    Sequence<T> in0(0);
-    invoke_on_all_policies(test_is_sorted(), in0.begin(), in0.end(), std::is_sorted(in0.begin(), in0.end()));
-    invoke_on_all_policies(test_is_sorted(), in0.cbegin(), in0.cend(), std::is_sorted(in0.begin(), in0.end()));
-
-    //non-descending order
-    Sequence<T> in1(9, [](size_t) -> T { return T(0); });
-    invoke_on_all_policies(test_is_sorted(), in1.begin(), in1.end(), std::is_sorted(in1.begin(), in1.end()));
-    invoke_on_all_policies(test_is_sorted(), in1.cbegin(), in1.cend(), std::is_sorted(in1.begin(), in1.end()));
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        is_sorted(exec, iter, iter, std::less<T>());
-        is_sorted_until(exec, iter, iter, std::less<T>());
-    }
-};
-
-int
-main()
-{
-
-    test_is_sorted_by_type<int32_t>();
-    test_is_sorted_by_type<float64_t>();
-
-    test_is_sorted_by_type<Wrapper<int32_t>>();
-
-    test_algo_basic_single<int32_t>(run_for_rnd_fw<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp b/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
deleted file mode 100644
index bf6e68be8c07..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/partial_sort.pass.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-// -*- C++ -*-
-//===-- partial_sort.pass.cpp ---------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <cmath>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-static std::atomic<int32_t> count_val;
-static std::atomic<int32_t> count_comp;
-
-template <typename T>
-struct Num
-{
-    T val;
-
-    Num() { ++count_val; }
-    Num(T v) : val(v) { ++count_val; }
-    Num(const Num<T>& v) : val(v.val) { ++count_val; }
-    Num(Num<T>&& v) : val(v.val) { ++count_val; }
-    ~Num() { --count_val; }
-    Num<T>&
-    operator=(const Num<T>& v)
-    {
-        val = v.val;
-        return *this;
-    }
-    operator T() const { return val; }
-    bool
-    operator<(const Num<T>& v) const
-    {
-        ++count_comp;
-        return val < v.val;
-    }
-};
-
-struct test_brick_partial_sort
-{
-    template <typename Policy, typename InputIterator, typename Compare>
-    typename std::enable_if<is_same_iterator_category<InputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, InputIterator first, InputIterator last, InputIterator exp_first, InputIterator exp_last,
-               Compare compare)
-    {
-
-        typedef typename std::iterator_traits<InputIterator>::value_type T;
-
-        // The rand()%(2*n+1) encourages generation of some duplicates.
-        std::srand(42);
-        const std::size_t n = last - first;
-        for (std::size_t k = 0; k < n; ++k)
-        {
-            first[k] = T(rand() % (2 * n + 1));
-        }
-        std::copy(first, last, exp_first);
-
-        for (std::size_t p = 0; p < n; p = p <= 16 ? p + 1 : std::size_t(31.415 * p))
-        {
-            auto m1 = first + p;
-            auto m2 = exp_first + p;
-
-            std::partial_sort(exp_first, m2, exp_last, compare);
-            count_comp = 0;
-            std::partial_sort(exec, first, m1, last, compare);
-            EXPECT_EQ_N(exp_first, first, p, "wrong effect from partial_sort");
-
-            //checking upper bound number of comparisons; O(p*(last-first)log(middle-first)); where p - number of threads;
-            if (m1 - first > 1)
-            {
-#ifdef _DEBUG
-#    if defined(_PSTL_PAR_BACKEND_TBB)
-                auto p = tbb::this_task_arena::max_concurrency();
-#    else
-                auto p = 1;
-#    endif
-                auto complex = std::ceil(n * std::log(float32_t(m1 - first)));
-                if (count_comp > complex * p)
-                {
-                    std::cout << "complexity exceeded" << std::endl;
-                }
-#endif // _DEBUG
-            }
-        }
-    }
-
-    template <typename Policy, typename InputIterator, typename Compare>
-    typename std::enable_if<!is_same_iterator_category<InputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&&, InputIterator, InputIterator, InputIterator, InputIterator, Compare)
-    {
-    }
-};
-
-template <typename T, typename Compare>
-void
-test_partial_sort(Compare compare)
-{
-
-    const std::size_t n_max = 100000;
-    Sequence<T> in(n_max);
-    Sequence<T> exp(n_max);
-    for (std::size_t n = 0; n < n_max; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        invoke_on_all_policies(test_brick_partial_sort(), in.begin(), in.begin() + n, exp.begin(), exp.begin() + n,
-                               compare);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        partial_sort(exec, iter, iter, iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-    count_val = 0;
-
-    test_partial_sort<Num<float32_t>>([](Num<float32_t> x, Num<float32_t> y) { return x < y; });
-
-    EXPECT_TRUE(count_val == 0, "cleanup error");
-
-    test_partial_sort<int32_t>(
-        [](int32_t x, int32_t y) { return x > y; }); // Reversed so accidental use of < will be detected.
-
-    test_algo_basic_single<int32_t>(run_for_rnd<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp b/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
deleted file mode 100644
index 9090f89e4361..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/partial_sort_copy.pass.cpp
+++ /dev/null
@@ -1,196 +0,0 @@
-// -*- C++ -*-
-//===-- partial_sort_copy.pass.cpp ----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for partial_sort_copy
-#include "support/pstl_test_config.h"
-
-#include <cmath>
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct Num
-{
-    T val;
-
-    Num() : val(0) {}
-    Num(T v) : val(v) {}
-    Num(const Num<T>& v) : val(v.val) {}
-    Num(Num<T>&& v) : val(v.val) {}
-    Num<T>&
-    operator=(const Num<T>& v)
-    {
-        val = v.val;
-        return *this;
-    }
-    operator T() const { return val; }
-    bool
-    operator<(const Num<T>& v) const
-    {
-        return val < v.val;
-    }
-};
-
-template <typename RandomAccessIterator>
-struct test_one_policy
-{
-    RandomAccessIterator d_first;
-    RandomAccessIterator d_last;
-    RandomAccessIterator exp_first;
-    RandomAccessIterator exp_last;
-    // This ctor is needed because output shouldn't be transformed to any iterator type (only random access iterators are allowed)
-    test_one_policy(RandomAccessIterator b1, RandomAccessIterator e1, RandomAccessIterator b2, RandomAccessIterator e2)
-        : d_first(b1), d_last(e1), exp_first(b2), exp_last(e2)
-    {
-    }
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename InputIterator, typename Size, typename T, typename Compare>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, Size n1, Size n2,
-               const T& trash, Compare compare)
-    {
-    }
-
-    template <typename InputIterator, typename Size, typename T, typename Compare>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last, Size n1, Size n2,
-               const T& trash, Compare compare)
-    {
-    }
-
-    template <typename InputIterator, typename Size, typename T>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, Size n1, Size n2,
-               const T& trash)
-    {
-    }
-
-    template <typename InputIterator, typename Size, typename T>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last, Size n1, Size n2,
-               const T& trash)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename Size, typename T, typename Compare>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, Size n1, Size n2, const T& trash,
-               Compare compare)
-    {
-        prepare_data(first, last, n1, trash);
-        RandomAccessIterator exp = std::partial_sort_copy(first, last, exp_first, exp_last, compare);
-        RandomAccessIterator res = std::partial_sort_copy(exec, first, last, d_first, d_last, compare);
-
-        EXPECT_TRUE((exp - exp_first) == (res - d_first), "wrong result from partial_sort_copy with predicate");
-        EXPECT_EQ_N(exp_first, d_first, n2, "wrong effect from partial_sort_copy with predicate");
-    }
-
-    template <typename Policy, typename InputIterator, typename Size, typename T>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, Size n1, Size n2, const T& trash)
-    {
-        prepare_data(first, last, n1, trash);
-        RandomAccessIterator exp = std::partial_sort_copy(first, last, exp_first, exp_last);
-        RandomAccessIterator res = std::partial_sort_copy(exec, first, last, d_first, d_last);
-
-        EXPECT_TRUE((exp - exp_first) == (res - d_first), "wrong result from partial_sort_copy without predicate");
-        EXPECT_EQ_N(exp_first, d_first, n2, "wrong effect from partial_sort_copy without predicate");
-    }
-
-  private:
-    template <typename InputIterator, typename Size, typename T>
-    void
-    prepare_data(InputIterator first, InputIterator last, Size n1, const T& trash)
-    {
-        // The rand()%(2*n+1) encourages generation of some duplicates.
-        std::srand(42);
-        std::generate(first, last, [n1]() { return T(rand() % (2 * n1 + 1)); });
-
-        std::fill(exp_first, exp_last, trash);
-        std::fill(d_first, d_last, trash);
-    }
-};
-
-template <typename T, typename Compare>
-void
-test_partial_sort_copy(Compare compare)
-{
-
-    typedef typename Sequence<T>::iterator iterator_type;
-    const std::size_t n_max = 100000;
-    Sequence<T> in(n_max);
-    Sequence<T> out(2 * n_max);
-    Sequence<T> exp(2 * n_max);
-    std::size_t n1 = 0;
-    std::size_t n2;
-    T trash = T(-666);
-    for (; n1 < n_max; n1 = n1 <= 16 ? n1 + 1 : size_t(3.1415 * n1))
-    {
-        // If both sequences are equal
-        n2 = n1;
-        invoke_on_all_policies(
-            test_one_policy<iterator_type>(out.begin(), out.begin() + n2, exp.begin(), exp.begin() + n2), in.begin(),
-            in.begin() + n1, n1, n2, trash, compare);
-
-        // If first sequence is greater than second
-        n2 = n1 / 3;
-        invoke_on_all_policies(
-            test_one_policy<iterator_type>(out.begin(), out.begin() + n2, exp.begin(), exp.begin() + n2), in.begin(),
-            in.begin() + n1, n1, n2, trash, compare);
-
-        // If first sequence is less than second
-        n2 = 2 * n1;
-        invoke_on_all_policies(
-            test_one_policy<iterator_type>(out.begin(), out.begin() + n2, exp.begin(), exp.begin() + n2), in.begin(),
-            in.begin() + n1, n1, n2, trash, compare);
-    }
-    // Test partial_sort_copy without predicate
-    n1 = n_max;
-    n2 = 2 * n1;
-    invoke_on_all_policies(test_one_policy<iterator_type>(out.begin(), out.begin() + n2, exp.begin(), exp.begin() + n2),
-                           in.begin(), in.begin() + n1, n1, n2, trash);
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename InputIterator, typename OutputInterator>
-    void
-    operator()(Policy&& exec, InputIterator input_iter, OutputInterator out_iter)
-    {
-        invoke_if(exec, [&]() {
-            partial_sort_copy(exec, input_iter, input_iter, out_iter, out_iter, non_const(std::less<T>()));
-        });
-    }
-};
-
-int
-main()
-{
-    test_partial_sort_copy<Num<float32_t>>([](Num<float32_t> x, Num<float32_t> y) { return x < y; });
-    test_partial_sort_copy<int32_t>([](int32_t x, int32_t y) { return x > y; });
-
-    test_algo_basic_double<int32_t>(run_for_rnd<test_non_const<int32_t>>());
-
-    test_partial_sort_copy<MemoryChecker>(
-        [](const MemoryChecker& val1, const MemoryChecker& val2){ return val1.value() < val2.value(); });
-    EXPECT_FALSE(MemoryChecker::alive_objects() < 0, "wrong effect from partial_sort_copy: number of ctors calls < num of dtors calls");
-    EXPECT_FALSE(MemoryChecker::alive_objects() > 0, "wrong effect from partial_sort_copy: number of ctors calls > num of dtors calls");
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp b/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
deleted file mode 100644
index 5617a6019f80..000000000000
--- a/pstl/test/std/algorithms/alg.sorting/sort.pass.cpp
+++ /dev/null
@@ -1,247 +0,0 @@
-// -*- C++ -*-
-//===-- sort.pass.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <algorithm>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-#define _CRT_SECURE_NO_WARNINGS
-
-#include <atomic>
-
-static bool Stable;
-
-//! Number of extant keys
-static std::atomic<int32_t> KeyCount;
-
-//! One more than highest index in array to be sorted.
-static uint32_t LastIndex;
-
-//! Keeping Equal() static and a friend of ParanoidKey class (C++, paragraphs 3.5/7.1.1)
-class ParanoidKey;
-static bool
-Equal(const ParanoidKey& x, const ParanoidKey& y);
-
-//! A key to be sorted, with lots of checking.
-class ParanoidKey
-{
-    //! Value used by comparator
-    int32_t value;
-    //! Original position or special value (Empty or Dead)
-    int32_t index;
-    //! Special value used to mark object without a comparable value, e.g. after being moved from.
-    static const int32_t Empty = -1;
-    //! Special value used to mark destroyed objects.
-    static const int32_t Dead = -2;
-    // True if key object has comparable value
-    bool
-    isLive() const
-    {
-        return (uint32_t)(index) < LastIndex;
-    }
-    // True if key object has been constructed.
-    bool
-    isConstructed() const
-    {
-        return isLive() || index == Empty;
-    }
-
-  public:
-    ParanoidKey()
-    {
-        ++KeyCount;
-        index = Empty;
-        value = Empty;
-    }
-    ParanoidKey(const ParanoidKey& k) : value(k.value), index(k.index)
-    {
-        EXPECT_TRUE(k.isLive(), "source for copy-constructor is dead");
-        ++KeyCount;
-    }
-    ~ParanoidKey()
-    {
-        EXPECT_TRUE(isConstructed(), "double destruction");
-        index = Dead;
-        --KeyCount;
-    }
-    ParanoidKey&
-    operator=(const ParanoidKey& k)
-    {
-        EXPECT_TRUE(k.isLive(), "source for copy-assignment is dead");
-        EXPECT_TRUE(isConstructed(), "destination for copy-assignment is dead");
-        value = k.value;
-        index = k.index;
-        return *this;
-    }
-    ParanoidKey(int32_t index, int32_t value, OddTag) : value(value), index(index) {}
-    ParanoidKey(ParanoidKey&& k) : value(k.value), index(k.index)
-    {
-        EXPECT_TRUE(k.isConstructed(), "source for move-construction is dead");
-// std::stable_sort() fails in move semantics on paranoid test before VS2015
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-        k.index = Empty;
-#endif
-        ++KeyCount;
-    }
-    ParanoidKey&
-    operator=(ParanoidKey&& k)
-    {
-        EXPECT_TRUE(k.isConstructed(), "source for move-assignment is dead");
-        EXPECT_TRUE(isConstructed(), "destination for move-assignment is dead");
-        value = k.value;
-        index = k.index;
-// std::stable_sort() fails in move semantics on paranoid test before VS2015
-#if !defined(_MSC_VER) || _MSC_VER >= 1900
-        k.index = Empty;
-#endif
-        return *this;
-    }
-    friend class KeyCompare;
-    friend bool
-    Equal(const ParanoidKey& x, const ParanoidKey& y);
-};
-
-class KeyCompare
-{
-    enum statusType
-    {
-        //! Special value used to mark defined object.
-        Live = 0xabcd,
-        //! Special value used to mark destroyed objects.
-        Dead = -1
-    } status;
-
-  public:
-    KeyCompare(OddTag) : status(Live) {}
-    ~KeyCompare() { status = Dead; }
-    bool
-    operator()(const ParanoidKey& j, const ParanoidKey& k) const
-    {
-        EXPECT_TRUE(status == Live, "key comparison object not defined");
-        EXPECT_TRUE(j.isLive(), "first key to operator() is not live");
-        EXPECT_TRUE(k.isLive(), "second key to operator() is not live");
-        return j.value < k.value;
-    }
-};
-
-// Equal is equality comparison used for checking result of sort against expected result.
-static bool
-Equal(const ParanoidKey& x, const ParanoidKey& y)
-{
-    return (x.value == y.value && !Stable) || (x.index == y.index);
-}
-
-static bool
-Equal(float32_t x, float32_t y)
-{
-    return x == y;
-}
-
-static bool
-Equal(int32_t x, int32_t y)
-{
-    return x == y;
-}
-
-struct test_sort_with_compare
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Compare>
-    typename std::enable_if<is_same_iterator_category<InputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, OutputIterator tmp_first, OutputIterator tmp_last, OutputIterator2 expected_first,
-               OutputIterator2 expected_last, InputIterator first, InputIterator, Size n, Compare compare)
-    {
-        using namespace std;
-        copy_n(first, n, expected_first);
-        copy_n(first, n, tmp_first);
-        if (Stable)
-            std::stable_sort(expected_first + 1, expected_last - 1, compare);
-        else
-            std::sort(expected_first + 1, expected_last - 1, compare);
-        int32_t count0 = KeyCount;
-        if (Stable)
-            stable_sort(exec, tmp_first + 1, tmp_last - 1, compare);
-        else
-            sort(exec, tmp_first + 1, tmp_last - 1, compare);
-
-        for (size_t i = 0; i < n; ++i, ++expected_first, ++tmp_first)
-        {
-            // Check that expected[i] is equal to tmp[i]
-            EXPECT_TRUE(Equal(*expected_first, *tmp_first), "bad sort");
-        }
-        int32_t count1 = KeyCount;
-        EXPECT_EQ(count0, count1, "key cleanup error");
-    }
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename OutputIterator2, typename Size,
-              typename Compare>
-    typename std::enable_if<!is_same_iterator_category<InputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&&, OutputIterator, OutputIterator, OutputIterator2, OutputIterator2, InputIterator, InputIterator,
-               Size, Compare)
-    {
-    }
-};
-
-template <typename T, typename Compare, typename Convert>
-void
-test_sort(Compare compare, Convert convert)
-{
-    for (size_t n = 0; n < 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        LastIndex = n + 2;
-        // The rand()%(2*n+1) encourages generation of some duplicates.
-        // Sequence is padded with an extra element at front and back, to detect overwrite bugs.
-        Sequence<T> in(n + 2, [=](size_t k) { return convert(k, rand() % (2 * n + 1)); });
-        Sequence<T> expected(in);
-        Sequence<T> tmp(in);
-        invoke_on_all_policies(test_sort_with_compare(), tmp.begin(), tmp.end(), expected.begin(), expected.end(),
-                               in.begin(), in.end(), in.size(), compare);
-    }
-}
-
-template <typename T>
-struct test_non_const
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator iter)
-    {
-        sort(exec, iter, iter, non_const(std::less<T>()));
-        stable_sort(exec, iter, iter, non_const(std::less<T>()));
-    }
-};
-
-int
-main()
-{
-    std::srand(42);
-    for (int32_t kind = 0; kind < 2; ++kind)
-    {
-        Stable = kind != 0;
-        test_sort<ParanoidKey>(KeyCompare(OddTag()),
-                               [](size_t k, size_t val) { return ParanoidKey(k, val, OddTag()); });
-        test_sort<float32_t>([](float32_t x, float32_t y) { return x < y; },
-                             [](size_t, size_t val) { return float32_t(val); });
-        test_sort<int32_t>(
-            [](int32_t x, int32_t y) { return x > y; }, // Reversed so accidental use of < will be detected.
-            [](size_t, size_t val) { return int32_t(val); });
-    }
-
-    test_algo_basic_single<int32_t>(run_for_rnd<test_non_const<int32_t>>());
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/lit.local.cfg b/pstl/test/std/lit.local.cfg
deleted file mode 100644
index f259d8f62c93..000000000000
--- a/pstl/test/std/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if "parallel-algorithms" not in config.available_features:
-    config.unsupported = True
diff --git a/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp b/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
deleted file mode 100644
index 8e56f61c0ccb..000000000000
--- a/pstl/test/std/numerics/numeric.ops/adjacent_difference.pass.cpp
+++ /dev/null
@@ -1,170 +0,0 @@
-// -*- C++ -*-
-//===-- adjacent_difference.pass.cpp --------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <iterator>
-#include <execution>
-#include <numeric>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-template <typename T>
-struct wrapper
-{
-    T t;
-    constexpr explicit wrapper(T t_) : t(t_) {}
-    template <typename T2>
-    constexpr wrapper(const wrapper<T2>& a)
-    {
-        t = a.t;
-    }
-    template <typename T2>
-    constexpr void
-    operator=(const wrapper<T2>& a)
-    {
-        t = a.t;
-    }
-    constexpr wrapper<T>
-    operator-(const wrapper<T>& a) const
-    {
-        return wrapper<T>(t - a.t);
-    }
-};
-
-template <typename T>
-bool
-compare(const T& a, const T& b)
-{
-    return a == b;
-}
-
-template <typename T>
-bool
-compare(const wrapper<T>& a, const wrapper<T>& b)
-{
-    return a.t == b.t;
-}
-
-template <typename Iterator1, typename Iterator2, typename T, typename Function>
-typename std::enable_if<!std::is_floating_point<T>::value, bool>::type
-compute_and_check(Iterator1 first, Iterator1 last, Iterator2 d_first, T, Function f)
-{
-    using T2 = typename std::iterator_traits<Iterator2>::value_type;
-
-    if (first == last)
-        return true;
-
-    {
-        T2 temp(*first);
-        if (!compare(temp, *d_first))
-            return false;
-    }
-    Iterator1 second = std::next(first);
-
-    ++d_first;
-    for (; second != last; ++first, ++second, ++d_first)
-    {
-        T2 temp(f(*second, *first));
-        if (!compare(temp, *d_first))
-            return false;
-    }
-
-    return true;
-}
-
-// we don't want to check equality here
-// because we can't be sure it will be strictly equal for floating point types
-template <typename Iterator1, typename Iterator2, typename T, typename Function>
-typename std::enable_if<std::is_floating_point<T>::value, bool>::type compute_and_check(Iterator1, Iterator1, Iterator2,
-                                                                                        T, Function)
-{
-    return true;
-}
-
-struct test_one_policy
-{
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) ||                                                             \
-    defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) // dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator1, typename Iterator2, typename T, typename Function>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e, T trash, Function f)
-    {
-    }
-    template <typename Iterator1, typename Iterator2, typename T, typename Function>
-    typename std::enable_if<is_same_iterator_category<Iterator1, std::random_access_iterator_tag>::value, void>::type
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b,
-               Iterator2 actual_e, T trash, Function f)
-    {
-    }
-#endif
-
-    template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T, typename Function>
-    void
-    operator()(ExecutionPolicy&& exec, Iterator1 data_b, Iterator1 data_e, Iterator2 actual_b, Iterator2 actual_e,
-               T trash, Function f)
-    {
-        using namespace std;
-        using T2 = typename std::iterator_traits<Iterator1>::value_type;
-
-        fill(actual_b, actual_e, trash);
-
-        Iterator2 actual_return = adjacent_difference(exec, data_b, data_e, actual_b);
-        EXPECT_TRUE(compute_and_check(data_b, data_e, actual_b, T2(0), std::minus<T2>()),
-                    "wrong effect of adjacent_difference");
-        EXPECT_TRUE(actual_return == actual_e, "wrong result of adjacent_difference");
-
-        fill(actual_b, actual_e, trash);
-
-        actual_return = adjacent_difference(exec, data_b, data_e, actual_b, f);
-        EXPECT_TRUE(compute_and_check(data_b, data_e, actual_b, T2(0), f),
-                    "wrong effect of adjacent_difference with functor");
-        EXPECT_TRUE(actual_return == actual_e, "wrong result of adjacent_difference with functor");
-    }
-};
-
-template <typename T1, typename T2, typename Pred>
-void
-test(Pred pred)
-{
-    const std::size_t max_len = 100000;
-
-    static constexpr T2 value = T2(77);
-    static constexpr T1 trash = T1(31);
-
-    Sequence<T1> actual(max_len, [](std::size_t i) { return T1(i); });
-
-    Sequence<T2> data(max_len, [](std::size_t i) { return i % 3 == 2 ? T2(i * i) : value; });
-
-    for (std::size_t len = 0; len < max_len; len = len <= 16 ? len + 1 : std::size_t(3.1415 * len))
-    {
-        invoke_on_all_policies(test_one_policy(), data.begin(), data.begin() + len, actual.begin(),
-                               actual.begin() + len, trash, pred);
-        invoke_on_all_policies(test_one_policy(), data.cbegin(), data.cbegin() + len, actual.begin(),
-                               actual.begin() + len, trash, pred);
-    }
-}
-
-int
-main()
-{
-    test<uint8_t, uint32_t>([](uint32_t a, uint32_t b) { return a - b; });
-    test<int32_t, int64_t>([](int64_t a, int64_t b) { return a / (b + 1); });
-    test<int64_t, float32_t>([](float32_t a, float32_t b) { return (a + b) / 2; });
-    test<wrapper<int32_t>, wrapper<int64_t>>(
-        [](const wrapper<int64_t>& a, const wrapper<int64_t>& b) { return a - b; });
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp b/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
deleted file mode 100644
index b2144b96f5dd..000000000000
--- a/pstl/test/std/numerics/numeric.ops/reduce.pass.cpp
+++ /dev/null
@@ -1,114 +0,0 @@
-// -*- C++ -*-
-//===-- reduce.pass.cpp ---------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <numeric>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_long_forms_for_one_policy
-{
-    template <typename Policy, typename Iterator, typename T, typename BinaryOp>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, T init, BinaryOp binary, T expected)
-    {
-        T result_r = std::reduce(exec, first, last, init, binary);
-        EXPECT_EQ(expected, result_r, "bad result from reduce(exec, first, last, init, binary_op)");
-    }
-};
-
-template <typename T, typename BinaryOp, typename F>
-void
-test_long_form(T init, BinaryOp binary_op, F f)
-{
-    // Try sequences of various lengths
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        T expected(init);
-        Sequence<T> in(n, [n, f](size_t k) { return f((int32_t(k ^ n) % 1000 - 500)); });
-        for (size_t k = 0; k < n; ++k)
-            expected = binary_op(expected, in[k]);
-
-        using namespace std;
-
-        T result = transform_reduce_serial(in.cfbegin(), in.cfend(), init, binary_op, [](const T& t) { return t; });
-        EXPECT_EQ(expected, result, "bad result from reduce(first, last, init, binary_op_op)");
-
-        invoke_on_all_policies(test_long_forms_for_one_policy(), in.begin(), in.end(), init, binary_op, expected);
-        invoke_on_all_policies(test_long_forms_for_one_policy(), in.cbegin(), in.cend(), init, binary_op, expected);
-    }
-}
-
-struct test_two_short_forms
-{
-
-#if defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN) //dummy specialization by policy type, in case of broken configuration
-    template <typename Iterator>
-    void
-    operator()(pstl::execution::parallel_policy, Iterator first, Iterator last, Sum init, Sum expected)
-    {
-    }
-    template <typename Iterator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, Iterator first, Iterator last, Sum init, Sum expected)
-    {
-    }
-#endif
-
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, Sum init, Sum expected)
-    {
-        using namespace std;
-
-        Sum r0 = init + reduce(exec, first, last);
-        EXPECT_EQ(expected, r0, "bad result from reduce(exec, first, last)");
-
-        Sum r1 = reduce(exec, first, last, init);
-        EXPECT_EQ(expected, r1, "bad result from reduce(exec, first, last, init)");
-    }
-};
-
-// Test forms of reduce(...) that omit the binary_op or init operands.
-void
-test_short_forms()
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sum init(42, OddTag());
-        Sum expected(init);
-        Sequence<Sum> in(n, [n](size_t k) { return Sum((int32_t(k ^ n) % 1000 - 500), OddTag()); });
-        for (size_t k = 0; k < n; ++k)
-            expected = expected + in[k];
-        invoke_on_all_policies(test_two_short_forms(), in.begin(), in.end(), init, expected);
-        invoke_on_all_policies(test_two_short_forms(), in.cbegin(), in.cend(), init, expected);
-    }
-}
-
-int
-main()
-{
-    // Test for popular types
-    test_long_form(42, std::plus<int32_t>(), [](int32_t x) { return x; });
-    test_long_form(42.0, std::plus<float64_t>(), [](float64_t x) { return x; });
-
-    // Test for strict types
-    test_long_form<Number>(Number(42, OddTag()), Add(OddTag()), [](int32_t x) { return Number(x, OddTag()); });
-
-    // Short forms are just facade for long forms, so just test with a single type.
-    test_short_forms();
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/numerics/numeric.ops/scan.fail.cpp b/pstl/test/std/numerics/numeric.ops/scan.fail.cpp
deleted file mode 100644
index b17944ae0846..000000000000
--- a/pstl/test/std/numerics/numeric.ops/scan.fail.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// -*- C++ -*-
-//===-- scan.fail.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include <execution>
-#include <numeric>
-
-struct CustomPolicy
-{
-} policy;
-
-int32_t
-main()
-{
-    int *first = nullptr, *last = nullptr, *result = nullptr;
-
-    std::exclusive_scan(policy, first, last, result, 0); // expected-error {{no matching function for call to 'exclusive_scan'}}
-    std::exclusive_scan(policy, first, last, result, 0, std::plus<int>()); // expected-error {{no matching function for call to 'exclusive_scan'}}
-
-    return 0;
-}
diff --git a/pstl/test/std/numerics/numeric.ops/scan.pass.cpp b/pstl/test/std/numerics/numeric.ops/scan.pass.cpp
deleted file mode 100644
index e89edc71a2dd..000000000000
--- a/pstl/test/std/numerics/numeric.ops/scan.pass.cpp
+++ /dev/null
@@ -1,201 +0,0 @@
-// -*- C++ -*-
-//===-- scan.pass.cpp -----------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <numeric>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// We provide the no execution policy versions of the exclusive_scan and inclusive_scan due checking correctness result of the versions with execution policies.
-//TODO: to add a macro for availability of ver implementations
-template <class InputIterator, class OutputIterator, class T>
-OutputIterator
-exclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, T init)
-{
-    for (; first != last; ++first, ++result)
-    {
-        *result = init;
-        init = init + *first;
-    }
-    return result;
-}
-
-template <class InputIterator, class OutputIterator, class T, class BinaryOperation>
-OutputIterator
-exclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, T init, BinaryOperation binary_op)
-{
-    for (; first != last; ++first, ++result)
-    {
-        *result = init;
-        init = binary_op(init, *first);
-    }
-    return result;
-}
-
-// Note: N4582 is missing the ", class T".  Issue was reported 2016-Apr-11 to cxxeditor@gmail.com
-template <class InputIterator, class OutputIterator, class BinaryOperation, class T>
-OutputIterator
-inclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op, T init)
-{
-    for (; first != last; ++first, ++result)
-    {
-        init = binary_op(init, *first);
-        *result = init;
-    }
-    return result;
-}
-
-template <class InputIterator, class OutputIterator, class BinaryOperation>
-OutputIterator
-inclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, BinaryOperation binary_op)
-{
-    if (first != last)
-    {
-        auto tmp = *first;
-        *result = tmp;
-        return inclusive_scan_serial(++first, last, ++result, binary_op, tmp);
-    }
-    else
-    {
-        return result;
-    }
-}
-
-template <class InputIterator, class OutputIterator>
-OutputIterator
-inclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result)
-{
-    typedef typename std::iterator_traits<InputIterator>::value_type input_type;
-    return inclusive_scan_serial(first, last, result, std::plus<input_type>());
-}
-
-// Most of the framework required for testing inclusive and exclusive scan is identical,
-// so the tests for both are in this file.  Which is being tested is controlled by the global
-// flag inclusive, which is set to each alternative by main().
-static bool inclusive;
-
-template <typename Iterator, typename Size, typename T>
-void
-check_and_reset(Iterator expected_first, Iterator out_first, Size n, T trash)
-{
-    EXPECT_EQ_N(expected_first, out_first, n,
-                inclusive ? "wrong result from inclusive_scan" : "wrong result from exclusive_scan");
-    std::fill_n(out_first, n, trash);
-}
-
-struct test_scan_with_plus
-{
-    template <typename Policy, typename Iterator1, typename Iterator2, typename Iterator3, typename Size, typename T>
-    void
-    operator()(Policy&& exec, Iterator1 in_first, Iterator1 in_last, Iterator2 out_first, Iterator2 out_last,
-               Iterator3 expected_first, Iterator3, Size n, T init, T trash)
-    {
-        using namespace std;
-
-        auto orr1 = inclusive ? inclusive_scan_serial(in_first, in_last, expected_first)
-                              : exclusive_scan_serial(in_first, in_last, expected_first, init);
-        (void)orr1;
-        auto orr = inclusive ? inclusive_scan(exec, in_first, in_last, out_first)
-                             : exclusive_scan(exec, in_first, in_last, out_first, init);
-        EXPECT_TRUE(out_last == orr,
-                    inclusive ? "inclusive_scan returned wrong iterator" : "exclusive_scan returned wrong iterator");
-
-        check_and_reset(expected_first, out_first, n, trash);
-        fill(out_first, out_last, trash);
-    }
-};
-
-template <typename T, typename Convert>
-void
-test_with_plus(T init, T trash, Convert convert)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, convert);
-        Sequence<T> expected(in);
-        Sequence<T> out(n, [&](int32_t) { return trash; });
-
-        invoke_on_all_policies(test_scan_with_plus(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), init, trash);
-        invoke_on_all_policies(test_scan_with_plus(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), init, trash);
-    }
-}
-struct test_scan_with_binary_op
-{
-    template <typename Policy, typename Iterator1, typename Iterator2, typename Iterator3, typename Size, typename T,
-              typename BinaryOp>
-    typename std::enable_if<!TestUtils::isReverse<Iterator1>::value, void>::type
-    operator()(Policy&& exec, Iterator1 in_first, Iterator1 in_last, Iterator2 out_first, Iterator2 out_last,
-               Iterator3 expected_first, Iterator3, Size n, T init, BinaryOp binary_op, T trash)
-    {
-        using namespace std;
-
-        auto orr1 = inclusive ? inclusive_scan_serial(in_first, in_last, expected_first, binary_op, init)
-                              : exclusive_scan_serial(in_first, in_last, expected_first, init, binary_op);
-        (void)orr1;
-        auto orr = inclusive ? inclusive_scan(exec, in_first, in_last, out_first, binary_op, init)
-                             : exclusive_scan(exec, in_first, in_last, out_first, init, binary_op);
-
-        EXPECT_TRUE(out_last == orr, "scan returned wrong iterator");
-        check_and_reset(expected_first, out_first, n, trash);
-    }
-
-    template <typename Policy, typename Iterator1, typename Iterator2, typename Iterator3, typename Size, typename T,
-              typename BinaryOp>
-    typename std::enable_if<TestUtils::isReverse<Iterator1>::value, void>::type
-    operator()(Policy&&, Iterator1, Iterator1, Iterator2, Iterator2, Iterator3, Iterator3, Size, T, BinaryOp, T)
-    {
-    }
-};
-
-template <typename In, typename Out, typename BinaryOp>
-void
-test_matrix(Out init, BinaryOp binary_op, Out trash)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<In> in(n, [](size_t k) { return In(k, k + 1); });
-
-        Sequence<Out> out(n, [&](size_t) { return trash; });
-        Sequence<Out> expected(n, [&](size_t) { return trash; });
-
-        invoke_on_all_policies(test_scan_with_binary_op(), in.begin(), in.end(), out.begin(), out.end(),
-                               expected.begin(), expected.end(), in.size(), init, binary_op, trash);
-        invoke_on_all_policies(test_scan_with_binary_op(), in.cbegin(), in.cend(), out.begin(), out.end(),
-                               expected.begin(), expected.end(), in.size(), init, binary_op, trash);
-    }
-}
-
-int
-main()
-{
-    for (int32_t mode = 0; mode < 2; ++mode)
-    {
-        inclusive = mode != 0;
-#if !defined(_PSTL_ICC_19_TEST_SIMD_UDS_WINDOWS_RELEASE_BROKEN)
-        // Test with highly restricted type and associative but not commutative operation
-        test_matrix<Matrix2x2<int32_t>, Matrix2x2<int32_t>>(Matrix2x2<int32_t>(), multiply_matrix<int32_t>,
-                                                            Matrix2x2<int32_t>(-666, 666));
-#endif
-
-        // Since the implict "+" forms of the scan delegate to the generic forms,
-        // there's little point in using a highly restricted type, so just use double.
-        test_with_plus<float64_t>(inclusive ? 0.0 : -1.0, -666.0,
-                                  [](uint32_t k) { return float64_t((k % 991 + 1) ^ (k % 997 + 2)); });
-    }
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp b/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
deleted file mode 100644
index f9c4538bf793..000000000000
--- a/pstl/test/std/numerics/numeric.ops/transform_reduce.pass.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-// -*- C++ -*-
-//===-- transform_reduce.pass.cpp -----------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <numeric>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// Functor for xor-operation for modeling binary operations in inner_product
-class XOR
-{
-  public:
-    template <typename T>
-    T
-    operator()(const T& left, const T& right) const
-    {
-        return left ^ right;
-    }
-};
-
-// Model of User-defined class
-class MyClass
-{
-  public:
-    int32_t my_field;
-    MyClass() { my_field = 0; }
-    MyClass(int32_t in) { my_field = in; }
-    MyClass(const MyClass& in) = default;
-
-    friend MyClass
-    operator+(const MyClass& x, const MyClass& y)
-    {
-        return MyClass(x.my_field + y.my_field);
-    }
-    friend MyClass
-    operator-(const MyClass& x)
-    {
-        return MyClass(-x.my_field);
-    }
-    friend MyClass operator*(const MyClass& x, const MyClass& y)
-    {
-        return MyClass(x.my_field * y.my_field);
-    }
-    friend bool operator==(const MyClass& x, const MyClass& y)
-    {
-        return x.my_field == y.my_field;
-    }
-};
-
-template <typename T>
-void
-CheckResults(const T& expected, const T& in)
-{
-    EXPECT_TRUE(expected == in, "wrong result of transform_reduce");
-}
-
-// We need to check correctness only for "int" (for example) except cases
-// if we have "floating-point type"-specialization
-void
-CheckResults(const float32_t&, const float32_t&)
-{
-}
-
-// Test for different types and operations with different iterators
-struct test_transform_reduce
-{
-    template <typename Policy, typename InputIterator1, typename InputIterator2, typename T, typename BinaryOperation1,
-              typename BinaryOperation2, typename UnaryOp>
-    void
-    operator()(Policy&& exec, InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2,
-               T init, BinaryOperation1 opB1, BinaryOperation2 opB2, UnaryOp opU)
-    {
-
-        auto expectedB = std::inner_product(first1, last1, first2, init, opB1, opB2);
-        auto expectedU = transform_reduce_serial(first1, last1, init, opB1, opU);
-        T resRA = std::transform_reduce(exec, first1, last1, first2, init, opB1, opB2);
-        CheckResults(expectedB, resRA);
-        resRA = std::transform_reduce(exec, first1, last1, init, opB1, opU);
-        CheckResults(expectedU, resRA);
-    }
-};
-
-template <typename T, typename BinaryOperation1, typename BinaryOperation2, typename UnaryOp, typename Initializer>
-void
-test_by_type(T init, BinaryOperation1 opB1, BinaryOperation2 opB2, UnaryOp opU, Initializer initObj)
-{
-
-    std::size_t maxSize = 100000;
-    Sequence<T> in1(maxSize, initObj);
-    Sequence<T> in2(maxSize, initObj);
-
-    for (std::size_t n = 0; n < maxSize; n = n < 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        invoke_on_all_policies(test_transform_reduce(), in1.begin(), in1.begin() + n, in2.begin(), in2.begin() + n,
-                               init, opB1, opB2, opU);
-        invoke_on_all_policies(test_transform_reduce(), in1.cbegin(), in1.cbegin() + n, in2.cbegin(), in2.cbegin() + n,
-                               init, opB1, opB2, opU);
-    }
-}
-
-int
-main()
-{
-    test_by_type<int32_t>(42, std::plus<int32_t>(), std::multiplies<int32_t>(), std::negate<int32_t>(),
-                          [](std::size_t) -> int32_t { return int32_t(rand() % 1000); });
-    test_by_type<int64_t>(0, [](const int64_t& a, const int64_t& b) -> int64_t { return a | b; }, XOR(),
-                          [](const int64_t& x) -> int64_t { return x * 2; },
-                          [](std::size_t) -> int64_t { return int64_t(rand() % 1000); });
-    test_by_type<float32_t>(
-        1.0f, std::multiplies<float32_t>(), [](const float32_t& a, const float32_t& b) -> float32_t { return a + b; },
-        [](const float32_t& x) -> float32_t { return x + 2; }, [](std::size_t) -> float32_t { return rand() % 1000; });
-    test_by_type<MyClass>(MyClass(), std::plus<MyClass>(), std::multiplies<MyClass>(), std::negate<MyClass>(),
-                          [](std::size_t) -> MyClass { return MyClass(rand() % 1000); });
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp b/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
deleted file mode 100644
index 95294e43c4c1..000000000000
--- a/pstl/test/std/numerics/numeric.ops/transform_scan.pass.cpp
+++ /dev/null
@@ -1,177 +0,0 @@
-// -*- C++ -*-
-//===-- transform_scan.pass.cpp -------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <numeric>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// Most of the framework required for testing inclusive and exclusive transform-scans is identical,
-// so the tests for both are in this file.  Which is being tested is controlled by the global
-// flag inclusive, which is set to each alternative by main().
-static bool inclusive;
-
-template <typename Iterator, typename Size, typename T>
-void
-check_and_reset(Iterator expected_first, Iterator out_first, Size n, T trash)
-{
-    EXPECT_EQ_N(expected_first, out_first, n,
-                inclusive ? "wrong result from transform_inclusive_scan"
-                          : "wrong result from transform_exclusive_scan");
-    std::fill_n(out_first, n, trash);
-}
-
-struct test_transform_scan
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename Size, typename UnaryOp,
-              typename T, typename BinaryOp>
-    typename std::enable_if<!TestUtils::isReverse<InputIterator>::value, void>::type
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first,
-               OutputIterator out_last, OutputIterator expected_first, OutputIterator, Size n, UnaryOp unary_op, T init,
-               BinaryOp binary_op, T trash)
-    {
-        using namespace std;
-
-        auto orr1 =
-            inclusive
-                ? transform_inclusive_scan(std::execution::seq, first, last, expected_first, binary_op, unary_op, init)
-                : transform_exclusive_scan(std::execution::seq, first, last, expected_first, init, binary_op, unary_op);
-        auto orr2 = inclusive ? transform_inclusive_scan(exec, first, last, out_first, binary_op, unary_op, init)
-                              : transform_exclusive_scan(exec, first, last, out_first, init, binary_op, unary_op);
-        EXPECT_TRUE(out_last == orr2, "transform...scan returned wrong iterator");
-        check_and_reset(expected_first, out_first, n, trash);
-
-        // Checks inclusive scan if init is not provided
-        if (inclusive && n > 0)
-        {
-            orr1 = transform_inclusive_scan(std::execution::seq, first, last, expected_first, binary_op, unary_op);
-            orr2 = transform_inclusive_scan(exec, first, last, out_first, binary_op, unary_op);
-            EXPECT_TRUE(out_last == orr2, "transform...scan returned wrong iterator");
-            check_and_reset(expected_first, out_first, n, trash);
-        }
-    }
-
-    template <typename Policy, typename InputIterator, typename OutputIterator, typename Size, typename UnaryOp,
-              typename T, typename BinaryOp>
-    typename std::enable_if<TestUtils::isReverse<InputIterator>::value, void>::type
-    operator()(Policy&&, InputIterator, InputIterator, OutputIterator, OutputIterator, OutputIterator, OutputIterator,
-               Size, UnaryOp, T, BinaryOp, T)
-    {
-    }
-};
-
-const uint32_t encryption_mask = 0x314;
-
-template <typename InputIterator, typename OutputIterator, typename UnaryOperation, typename T,
-          typename BinaryOperation>
-std::pair<OutputIterator, T>
-transform_inclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, UnaryOperation unary_op,
-                                T init, BinaryOperation binary_op) noexcept
-{
-    for (; first != last; ++first, ++result)
-    {
-        init = binary_op(init, unary_op(*first));
-        *result = init;
-    }
-    return std::make_pair(result, init);
-}
-
-template <typename InputIterator, typename OutputIterator, typename UnaryOperation, typename T,
-          typename BinaryOperation>
-std::pair<OutputIterator, T>
-transform_exclusive_scan_serial(InputIterator first, InputIterator last, OutputIterator result, UnaryOperation unary_op,
-                                T init, BinaryOperation binary_op) noexcept
-{
-    for (; first != last; ++first, ++result)
-    {
-        *result = init;
-        init = binary_op(init, unary_op(*first));
-    }
-    return std::make_pair(result, init);
-}
-
-template <typename In, typename Out, typename UnaryOp, typename BinaryOp>
-void
-test(UnaryOp unary_op, Out init, BinaryOp binary_op, Out trash)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<In> in(n, [](size_t k) { return In(k ^ encryption_mask); });
-
-        Out tmp = init;
-        Sequence<Out> expected(n, [&](size_t k) -> Out {
-            if (inclusive)
-            {
-                tmp = binary_op(tmp, unary_op(in[k]));
-                return tmp;
-            }
-            else
-            {
-                Out val = tmp;
-                tmp = binary_op(tmp, unary_op(in[k]));
-                return val;
-            }
-        });
-
-        Sequence<Out> out(n, [&](size_t) { return trash; });
-
-        auto result =
-            inclusive
-                ? transform_inclusive_scan_serial(in.cbegin(), in.cend(), out.fbegin(), unary_op, init, binary_op)
-                : transform_exclusive_scan_serial(in.cbegin(), in.cend(), out.fbegin(), unary_op, init, binary_op);
-        (void)result;
-        check_and_reset(expected.begin(), out.begin(), out.size(), trash);
-
-        invoke_on_all_policies(test_transform_scan(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), unary_op, init, binary_op, trash);
-        invoke_on_all_policies(test_transform_scan(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), unary_op, init, binary_op, trash);
-    }
-}
-
-template <typename In, typename Out, typename UnaryOp, typename BinaryOp>
-void
-test_matrix(UnaryOp unary_op, Out init, BinaryOp binary_op, Out trash)
-{
-    for (size_t n = 0; n <= 100000; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<In> in(n, [](size_t k) { return In(k, k + 1); });
-
-        Sequence<Out> out(n, [&](size_t) { return trash; });
-        Sequence<Out> expected(n, [&](size_t) { return trash; });
-
-        invoke_on_all_policies(test_transform_scan(), in.begin(), in.end(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), unary_op, init, binary_op, trash);
-        invoke_on_all_policies(test_transform_scan(), in.cbegin(), in.cend(), out.begin(), out.end(), expected.begin(),
-                               expected.end(), in.size(), unary_op, init, binary_op, trash);
-    }
-}
-
-int
-main()
-{
-    for (int32_t mode = 0; mode < 2; ++mode)
-    {
-        inclusive = mode != 0;
-#if !defined(_PSTL_ICC_19_TEST_SIMD_UDS_WINDOWS_RELEASE_BROKEN)
-        test_matrix<Matrix2x2<int32_t>, Matrix2x2<int32_t>>([](const Matrix2x2<int32_t> x) { return x; },
-                                                            Matrix2x2<int32_t>(), multiply_matrix<int32_t>,
-                                                            Matrix2x2<int32_t>(-666, 666));
-#endif
-        test<int32_t, uint32_t>([](int32_t x) { return x++; }, -123, [](int32_t x, int32_t y) { return x + y; }, 666);
-    }
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
deleted file mode 100644
index a5bdbb3a9c7e..000000000000
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_construct.pass.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-// -*- C++ -*-
-//===-- uninitialized_construct.pass.cpp ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for uninitialized_default_construct, uninitialized_default_construct_n,
-//           uninitialized_value_construct,   uninitialized_value_construct_n
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <memory>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// function of checking correctness for uninitialized.construct.value
-template <typename T, typename Iterator>
-bool
-IsCheckValueCorrectness(Iterator begin, Iterator end)
-{
-    for (; begin != end; ++begin)
-    {
-        if (*begin != T())
-        {
-            return false;
-        }
-    }
-    return true;
-}
-
-struct test_uninit_construct
-{
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator begin, Iterator end, size_t n, /*is_trivial<T>=*/std::false_type)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-        // it needs for cleaning memory that was filled by default constructors in unique_ptr<T[]> p(new T[n])
-        // and for cleaning memory after last calling of uninitialized_value_construct_n.
-        // It is important for non-trivial types
-        std::destroy_n(exec, begin, n);
-
-        // reset counter of constructors
-        T::SetCount(0);
-        // run algorithm
-        std::uninitialized_default_construct(exec, begin, end);
-        // compare counter of constructors to length of container
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_default_construct");
-        // destroy objects for testing new algorithms on same memory
-        std::destroy(exec, begin, end);
-
-        std::uninitialized_default_construct_n(exec, begin, n);
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_default_construct_n");
-        std::destroy_n(exec, begin, n);
-
-        std::uninitialized_value_construct(exec, begin, end);
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_value_construct");
-        std::destroy(exec, begin, end);
-
-        std::uninitialized_value_construct_n(exec, begin, n);
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_value_construct_n");
-    }
-
-    template <typename Policy, typename Iterator>
-    void
-    operator()(Policy&& exec, Iterator begin, Iterator end, size_t n, /*is_trivial<T>=*/std::true_type)
-    {
-        typedef typename std::iterator_traits<Iterator>::value_type T;
-
-        std::uninitialized_default_construct(exec, begin, end);
-        std::destroy(exec, begin, end);
-
-        std::uninitialized_default_construct_n(exec, begin, n);
-        std::destroy_n(exec, begin, n);
-
-        std::uninitialized_value_construct(exec, begin, end);
-        // check correctness for uninitialized.construct.value
-        EXPECT_TRUE(IsCheckValueCorrectness<T>(begin, end), "wrong uninitialized_value_construct");
-        std::destroy(exec, begin, end);
-
-        std::uninitialized_value_construct_n(exec, begin, n);
-        EXPECT_TRUE(IsCheckValueCorrectness<T>(begin, end), "wrong uninitialized_value_construct_n");
-        std::destroy_n(exec, begin, n);
-    }
-};
-
-template <typename T>
-void
-test_uninit_construct_by_type()
-{
-    std::size_t N = 100000;
-    for (size_t n = 0; n <= N; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        std::unique_ptr<T[]> p(new T[n]);
-        invoke_on_all_policies(test_uninit_construct(), p.get(), std::next(p.get(), n), n, std::is_trivial<T>());
-    }
-}
-
-int
-main()
-{
-
-    // for user-defined types
-#if !defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN)
-    test_uninit_construct_by_type<Wrapper<int32_t>>();
-    test_uninit_construct_by_type<Wrapper<std::vector<std::string>>>();
-#endif
-
-    // for trivial types
-    test_uninit_construct_by_type<int8_t>();
-    test_uninit_construct_by_type<float64_t>();
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
deleted file mode 100644
index 21186b41ee13..000000000000
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_copy_move.pass.cpp
+++ /dev/null
@@ -1,143 +0,0 @@
-// -*- C++ -*-
-//===-- uninitialized_copy_move.pass.cpp ----------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-// Tests for uninitialized_copy, uninitialized_copy_n, uninitialized_move, uninitialized_move_n
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <memory>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-// function of checking correctness for uninitialized.construct.value
-template <typename InputIterator, typename OutputIterator, typename Size>
-bool
-IsCheckValueCorrectness(InputIterator first1, OutputIterator first2, Size n)
-{
-    for (Size i = 0; i < n; ++i, ++first1, ++first2)
-    {
-        if (*first1 != *first2)
-        {
-            return false;
-        }
-    }
-    return true;
-}
-
-struct test_uninitialized_copy_move
-{
-    template <typename Policy, typename InputIterator, typename OutputIterator>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first, size_t n,
-               /*is_trivial<T>=*/std::false_type)
-    {
-        typedef typename std::iterator_traits<InputIterator>::value_type T;
-        // it needs for cleaning memory that was filled by default constructors in unique_ptr<T[]> p(new T[n])
-        // and for cleaning memory after last calling of uninitialized_value_construct_n.
-        // It is important for non-trivial types
-        std::destroy_n(exec, out_first, n);
-
-        // reset counter of constructors
-        T::SetCount(0);
-        // run algorithm
-        std::uninitialized_copy(exec, first, last, out_first);
-        // compare counter of constructors to length of container
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_copy");
-        // destroy objects for testing new algorithms on same memory
-        std::destroy_n(exec, out_first, n);
-
-        std::uninitialized_copy_n(exec, first, n, out_first);
-        EXPECT_TRUE(T::Count() == n, "wrong uninitialized_copy_n");
-        std::destroy_n(exec, out_first, n);
-
-        // For move
-        std::uninitialized_move(exec, first, last, out_first);
-        // compare counter of constructors to length of container
-        EXPECT_TRUE(T::MoveCount() == n, "wrong uninitialized_move");
-        // destroy objects for testing new algorithms on same memory
-        std::destroy_n(exec, out_first, n);
-
-        std::uninitialized_move_n(exec, first, n, out_first);
-        EXPECT_TRUE(T::MoveCount() == n, "wrong uninitialized_move_n");
-        std::destroy_n(exec, out_first, n);
-    }
-
-#if defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) || defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN)
-    template <typename InputIterator, typename OutputIterator>
-    void
-    operator()(pstl::execution::unsequenced_policy, InputIterator first, InputIterator last, OutputIterator out_first,
-               size_t n, /*is_trivial<T>=*/std::true_type)
-    {
-    }
-    template <typename InputIterator, typename OutputIterator>
-    void
-    operator()(pstl::execution::parallel_unsequenced_policy, InputIterator first, InputIterator last,
-               OutputIterator out_first, size_t n, /*is_trivial<T>=*/std::true_type)
-    {
-    }
-#endif
-
-    template <typename Policy, typename InputIterator, typename OutputIterator>
-    void
-    operator()(Policy&& exec, InputIterator first, InputIterator last, OutputIterator out_first, size_t n,
-               /*is_trivial<T>=*/std::true_type)
-    {
-        std::uninitialized_copy(exec, first, last, out_first);
-        EXPECT_TRUE(IsCheckValueCorrectness(first, out_first, n), "wrong uninitialized_copy");
-        std::destroy_n(exec, out_first, n);
-
-        std::uninitialized_copy_n(exec, first, n, out_first);
-        EXPECT_TRUE(IsCheckValueCorrectness(first, out_first, n), "wrong uninitialized_copy_n");
-        std::destroy_n(exec, out_first, n);
-
-        std::uninitialized_move(exec, first, last, out_first);
-        EXPECT_TRUE(IsCheckValueCorrectness(first, out_first, n), "wrong uninitialized_move");
-        std::destroy_n(exec, out_first, n);
-
-        std::uninitialized_move_n(exec, first, n, out_first);
-        EXPECT_TRUE(IsCheckValueCorrectness(first, out_first, n), "wrong uninitialized_move_n");
-        std::destroy_n(exec, out_first, n);
-    }
-};
-
-template <typename T>
-void
-test_uninitialized_copy_move_by_type()
-{
-    std::size_t N = 100000;
-    for (size_t n = 0; n <= N; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        Sequence<T> in(n, [=](size_t k) -> T { return T(k); });
-        std::unique_ptr<T[]> p(new T[n]);
-        invoke_on_all_policies(test_uninitialized_copy_move(), in.begin(), in.end(), p.get(), n, std::is_trivial<T>());
-    }
-}
-
-int
-main()
-{
-
-    // for trivial types
-    test_uninitialized_copy_move_by_type<int16_t>();
-    test_uninitialized_copy_move_by_type<float64_t>();
-
-    // for user-defined types
-#if !defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) && !defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) &&     \
-    !defined(_PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN)
-    test_uninitialized_copy_move_by_type<Wrapper<int8_t>>();
-#endif
-
-    std::cout << done() << std::endl;
-    return 0;
-}
diff --git a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp b/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
deleted file mode 100644
index d579ea558839..000000000000
--- a/pstl/test/std/utilities/memory/specialized.algorithms/uninitialized_fill_destroy.pass.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-// -*- C++ -*-
-//===-- uninitialized_fill_destroy.pass.cpp -------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14
-
-#include "support/pstl_test_config.h"
-
-#include <execution>
-#include <memory>
-
-#include "support/utils.h"
-
-using namespace TestUtils;
-
-struct test_uninitialized_fill_destroy
-{
-    template <typename Policy, typename Iterator, typename T>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, const T& in, std::size_t n, std::false_type)
-    {
-        using namespace std;
-        {
-            T::SetCount(0);
-            uninitialized_fill(exec, first, last, in);
-            size_t count = count_if(first, last, [&in](T& x) -> bool { return x == in; });
-            EXPECT_TRUE(n == count, "wrong work of uninitialized_fill");
-            destroy(exec, first, last);
-            EXPECT_TRUE(T::Count() == 0, "wrong work of destroy");
-        }
-
-        {
-            auto res = uninitialized_fill_n(exec, first, n, in);
-            EXPECT_TRUE(res == last, "wrong result of uninitialized_fill_n");
-            size_t count = count_if(first, last, [&in](T& x) -> bool { return x == in; });
-            EXPECT_TRUE(n == count, "wrong work of uninitialized_fill_n");
-            destroy_n(exec, first, n);
-            EXPECT_TRUE(T::Count() == 0, "wrong work of destroy_n");
-        }
-    }
-    template <typename Policy, typename Iterator, typename T>
-    void
-    operator()(Policy&& exec, Iterator first, Iterator last, const T& in, std::size_t n, std::true_type)
-    {
-        using namespace std;
-        {
-            destroy(exec, first, last);
-            uninitialized_fill(exec, first, last, in);
-            size_t count = count_if(first, last, [&in](T& x) -> bool { return x == in; });
-            EXPECT_EQ(n, count, "wrong work of uninitialized:_fill");
-        }
-        {
-            destroy_n(exec, first, n);
-            auto res = uninitialized_fill_n(exec, first, n, in);
-            size_t count = count_if(first, last, [&in](T& x) -> bool { return x == in; });
-            EXPECT_EQ(n, count, "wrong work of uninitialized_fill_n");
-            EXPECT_TRUE(res == last, "wrong result of uninitialized_fill_n");
-        }
-    }
-};
-
-template <typename T>
-void
-test_uninitialized_fill_destroy_by_type()
-{
-    std::size_t N = 100000;
-    for (size_t n = 0; n <= N; n = n <= 16 ? n + 1 : size_t(3.1415 * n))
-    {
-        std::unique_ptr<T[]> p(new T[n]);
-        invoke_on_all_policies(test_uninitialized_fill_destroy(), p.get(), std::next(p.get(), n), T(), n,
-                               std::is_trivial<T>());
-    }
-}
-
-int
-main()
-{
-    // for trivial types
-    test_uninitialized_fill_destroy_by_type<int32_t>();
-    test_uninitialized_fill_destroy_by_type<float64_t>();
-
-    // for user-defined types
-    test_uninitialized_fill_destroy_by_type<Wrapper<std::string>>();
-    test_uninitialized_fill_destroy_by_type<Wrapper<int8_t*>>();
-    std::cout << done() << std::endl;
-
-    return 0;
-}
diff --git a/pstl/test/support/pstl_test_config.h b/pstl/test/support/pstl_test_config.h
deleted file mode 100644
index daf5625a0554..000000000000
--- a/pstl/test/support/pstl_test_config.h
+++ /dev/null
@@ -1,52 +0,0 @@
-// -*- C++ -*-
-//===-- pstl_test_config.h ------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _PSTL_TEST_config_H
-#define _PSTL_TEST_config_H
-
-#if defined(_MSC_VER) && defined(_DEBUG)
-#    define _SCL_SECURE_NO_WARNINGS //to prevent the compilation warning. Microsoft STL implementation has specific checking of an iterator range in DEBUG mode for the containers from the standard library.
-#endif
-
-#ifndef __clang__
-#    define _PSTL_ICC_16_17_TEST_REDUCTION_BOOL_TYPE_RELEASE_64_BROKEN                                                 \
-        (__x86_64 && !_DEBUG && __INTEL_COMPILER && __INTEL_COMPILER <= 1700 && !__APPLE__)
-#    define _PSTL_ICC_16_17_TEST_REDUCTION_RELEASE_BROKEN                                                              \
-        (!_DEBUG && __INTEL_COMPILER &&                                                                                \
-         (__INTEL_COMPILER < 1800 || (__INTEL_COMPILER == 1800 && __INTEL_COMPILER_UPDATE < 1)))
-#    define _PSTL_ICC_1800_TEST_MONOTONIC_RELEASE_64_BROKEN                                                            \
-        (__x86_64 && !_DEBUG && __INTEL_COMPILER && __INTEL_COMPILER == 1800 && __INTEL_COMPILER_UPDATE < 1)
-#    define _PSTL_ICC_17_TEST_MAC_RELEASE_32_BROKEN                                                                    \
-        (__i386__ && !_DEBUG && __INTEL_COMPILER >= 1700 && __INTEL_COMPILER < 1800 && __APPLE__)
-#    define _PSTL_ICC_18_VC141_TEST_SIMD_LAMBDA_RELEASE_BROKEN                                                         \
-        (!_DEBUG && __INTEL_COMPILER >= 1800 && __INTEL_COMPILER < 1900 && _MSC_VER == 1910)
-#    define _PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN                                                        \
-        (_M_IX86 && _DEBUG && __INTEL_COMPILER >= 1700 && __INTEL_COMPILER < 1800 && _MSC_VER >= 1900)
-#    define _PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN                                                         \
-        (_M_IX86 && _DEBUG && __INTEL_COMPILER >= 1600 && __INTEL_COMPILER < 1700 && _MSC_VER == 1900)
-#    define _PSTL_ICC_16_VC14_TEST_PAR_TBB_RT_RELEASE_64_BROKEN                                                        \
-        (((_M_X64 && _MSC_VER == 1900) || __x86_64) && !_DEBUG && __INTEL_COMPILER < 1700)
-#    define _PSTL_ICC_16_17_TEST_64_TIMEOUT (__x86_64 && __INTEL_COMPILER && __INTEL_COMPILER < 1800 && !__APPLE__)
-#    define _PSTL_ICC_18_TEST_EARLY_EXIT_MONOTONIC_RELEASE_BROKEN                                                      \
-        (!_DEBUG && __INTEL_COMPILER && __INTEL_COMPILER == 1800)
-#    define _PSTL_CLANG_TEST_BIG_OBJ_DEBUG_32_BROKEN                                                                   \
-        (__i386__ && PSTL_USE_DEBUG && __clang__ && _PSTL_CLANG_VERSION <= 90000)
-#    define _PSTL_ICC_16_17_18_TEST_UNIQUE_MASK_RELEASE_BROKEN                                                         \
-        (!_DEBUG && __INTEL_COMPILER &&                                                                                \
-         (__INTEL_COMPILER < 1800 || (__INTEL_COMPILER == 1800 && __INTEL_COMPILER_UPDATE < 3)))
-#    define _PSTL_ICC_18_TEST_EARLY_EXIT_AVX_RELEASE_BROKEN                                                            \
-        (!_DEBUG && __INTEL_COMPILER == 1800 && __AVX__ && !__AVX2__ && !__AVX512__)
-#    define _PSTL_ICC_19_TEST_IS_PARTITIONED_RELEASE_BROKEN                                                            \
-        (!PSTL_USE_DEBUG && (__linux__ || __APPLE__) && __INTEL_COMPILER == 1900)
-#    define _PSTL_ICL_19_VC14_VC141_TEST_SCAN_RELEASE_BROKEN                                                           \
-        (__INTEL_COMPILER == 1900 && _MSC_VER >= 1900 && _MSC_VER <= 1910)
-#    define _PSTL_ICC_19_TEST_SIMD_UDS_WINDOWS_RELEASE_BROKEN (__INTEL_COMPILER == 1900 && _MSC_VER && !_DEBUG)
-#endif // !__clang__
-
-#endif /* _PSTL_TEST_config_H */
diff --git a/pstl/test/support/stdlib/algorithm b/pstl/test/support/stdlib/algorithm
deleted file mode 100644
index 39e6f7471a6d..000000000000
--- a/pstl/test/support/stdlib/algorithm
+++ /dev/null
@@ -1,26 +0,0 @@
-// -*- C++ -*-
-//===-- algorithm ---------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _TEST_SUPPORT_STDLIB_ALGORITHM
-#define _TEST_SUPPORT_STDLIB_ALGORITHM
-
-#include_next <algorithm>
-
-#include <pstl/internal/pstl_config.h>
-
-#if _PSTL_EXECUTION_POLICIES_DEFINED
-// If <execution> has already been included, pull in implementations
-#    include <pstl/internal/glue_algorithm_impl.h>
-#else
-// Otherwise just pull in forward declarations
-#    include <pstl/internal/glue_algorithm_defs.h>
-#    define _PSTL_ALGORITHM_FORWARD_DECLARED 1
-#endif
-
-#endif /* _TEST_SUPPORT_STDLIB_ALGORITHM */
diff --git a/pstl/test/support/stdlib/execution b/pstl/test/support/stdlib/execution
deleted file mode 100644
index ad18c450115a..000000000000
--- a/pstl/test/support/stdlib/execution
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- C++ -*-
-//===-- execution ---------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _TEST_SUPPORT_STDLIB_EXECUTION
-#define _TEST_SUPPORT_STDLIB_EXECUTION
-
-// #include_next <execution> // None of the standard libraries PSTL is built on top have the <execution> header yet.
-
-#include <pstl/internal/pstl_config.h>
-#include <pstl/internal/execution_defs.h>
-
-#define _PSTL_EXECUTION_POLICIES_DEFINED 1
-
-#if _PSTL_ALGORITHM_FORWARD_DECLARED
-#    include <pstl/internal/glue_algorithm_impl.h>
-#endif
-
-#if _PSTL_MEMORY_FORWARD_DECLARED
-#    include <pstl/internal/glue_memory_impl.h>
-#endif
-
-#if _PSTL_NUMERIC_FORWARD_DECLARED
-#    include <pstl/internal/glue_numeric_impl.h>
-#endif
-
-#if _PSTL_CPP17_EXECUTION_POLICIES_PRESENT
-_PSTL_PRAGMA_MESSAGE_POLICIES("The <Parallel STL> execution policies are defined in the namespace __pstl::execution")
-#else
-#    include <pstl/internal/glue_execution_defs.h>
-_PSTL_PRAGMA_MESSAGE_POLICIES(
-    "The <Parallel STL> execution policies are injected into the standard namespace std::execution")
-#endif
-
-//TODO: __pstl::execution namespace is injected into the pstl::execution namespace when the implementation is not a part of
-// standard C++ library
-namespace pstl
-{
-namespace execution
-{
-using namespace __pstl::execution;
-}
-} // namespace pstl
-
-#endif /* _TEST_SUPPORT_STDLIB_EXECUTION */
diff --git a/pstl/test/support/stdlib/memory b/pstl/test/support/stdlib/memory
deleted file mode 100644
index 7d1109122967..000000000000
--- a/pstl/test/support/stdlib/memory
+++ /dev/null
@@ -1,26 +0,0 @@
-// -*- C++ -*-
-//===-- memory ------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _TEST_SUPPORT_STDLIB_MEMORY
-#define _TEST_SUPPORT_STDLIB_MEMORY
-
-#include_next <memory>
-
-#include <pstl/internal/pstl_config.h>
-
-#if _PSTL_EXECUTION_POLICIES_DEFINED
-// If <execution> has already been included, pull in implementations
-#    include <pstl/internal/glue_memory_impl.h>
-#else
-// Otherwise just pull in forward declarations
-#    include <pstl/internal/glue_memory_defs.h>
-#    define _PSTL_MEMORY_FORWARD_DECLARED 1
-#endif
-
-#endif /* _TEST_SUPPORT_STDLIB_MEMORY */
diff --git a/pstl/test/support/stdlib/numeric b/pstl/test/support/stdlib/numeric
deleted file mode 100644
index 8f98bc754643..000000000000
--- a/pstl/test/support/stdlib/numeric
+++ /dev/null
@@ -1,26 +0,0 @@
-// -*- C++ -*-
-//===-- numeric -----------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _TEST_SUPPORT_STDLIB_NUMERIC
-#define _TEST_SUPPORT_STDLIB_NUMERIC
-
-#include_next <numeric>
-
-#include <pstl/internal/pstl_config.h>
-
-#if _PSTL_EXECUTION_POLICIES_DEFINED
-// If <execution> has already been included, pull in implementations
-#    include <pstl/internal/glue_numeric_impl.h>
-#else
-// Otherwise just pull in forward declarations
-#    include <pstl/internal/glue_numeric_defs.h>
-#    define _PSTL_NUMERIC_FORWARD_DECLARED 1
-#endif
-
-#endif /* _TEST_SUPPORT_STDLIB_NUMERIC */
diff --git a/pstl/test/support/utils.h b/pstl/test/support/utils.h
deleted file mode 100644
index ed6d48b9471a..000000000000
--- a/pstl/test/support/utils.h
+++ /dev/null
@@ -1,1318 +0,0 @@
-// -*- C++ -*-
-//===-- utils.h -----------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// File contains common utilities that tests rely on
-
-// Do not #include <algorithm>, because if we do we will not detect accidental dependencies.
-#include <atomic>
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-#include <iostream>
-#include <iterator>
-#include <memory>
-#include <sstream>
-#include <vector>
-
-#include "pstl_test_config.h"
-
-namespace TestUtils
-{
-
-typedef double float64_t;
-typedef float float32_t;
-
-template <class T, std::size_t N>
-constexpr size_t
-const_size(const T (&)[N]) noexcept
-{
-    return N;
-}
-
-template <typename T>
-class Sequence;
-
-// Handy macros for error reporting
-#define EXPECT_TRUE(condition, message) ::TestUtils::expect(true, condition, __FILE__, __LINE__, message)
-#define EXPECT_FALSE(condition, message) ::TestUtils::expect(false, condition, __FILE__, __LINE__, message)
-
-// Check that expected and actual are equal and have the same type.
-#define EXPECT_EQ(expected, actual, message) ::TestUtils::expect_equal(expected, actual, __FILE__, __LINE__, message)
-
-// Check that sequences started with expected and actual and have had size n are equal and have the same type.
-#define EXPECT_EQ_N(expected, actual, n, message)                                                                      \
-    ::TestUtils::expect_equal(expected, actual, n, __FILE__, __LINE__, message)
-
-// Issue error message from outstr, adding a newline.
-// Real purpose of this routine is to have a place to hang a breakpoint.
-inline void
-issue_error_message(std::stringstream& outstr)
-{
-    outstr << std::endl;
-    std::cerr << outstr.str();
-    std::exit(EXIT_FAILURE);
-}
-
-inline void
-expect(bool expected, bool condition, const char* file, int32_t line, const char* message)
-{
-    if (condition != expected)
-    {
-        std::stringstream outstr;
-        outstr << "error at " << file << ":" << line << " - " << message;
-        issue_error_message(outstr);
-    }
-}
-
-// Do not change signature to const T&.
-// Function must be able to detect const differences between expected and actual.
-template <typename T>
-void
-expect_equal(T& expected, T& actual, const char* file, int32_t line, const char* message)
-{
-    if (!(expected == actual))
-    {
-        std::stringstream outstr;
-        outstr << "error at " << file << ":" << line << " - " << message << ", expected " << expected << " got "
-               << actual;
-        issue_error_message(outstr);
-    }
-}
-
-template <typename T>
-void
-expect_equal(Sequence<T>& expected, Sequence<T>& actual, const char* file, int32_t line, const char* message)
-{
-    size_t n = expected.size();
-    size_t m = actual.size();
-    if (n != m)
-    {
-        std::stringstream outstr;
-        outstr << "error at " << file << ":" << line << " - " << message << ", expected sequence of size " << n
-               << " got sequence of size " << m;
-        issue_error_message(outstr);
-        return;
-    }
-    size_t error_count = 0;
-    for (size_t k = 0; k < n && error_count < 10; ++k)
-    {
-        if (!(expected[k] == actual[k]))
-        {
-            std::stringstream outstr;
-            outstr << "error at " << file << ":" << line << " - " << message << ", at index " << k << " expected "
-                   << expected[k] << " got " << actual[k];
-            issue_error_message(outstr);
-            ++error_count;
-        }
-    }
-}
-
-template <typename Iterator1, typename Iterator2, typename Size>
-void
-expect_equal(Iterator1 expected_first, Iterator2 actual_first, Size n, const char* file, int32_t line,
-             const char* message)
-{
-    size_t error_count = 0;
-    for (Size k = 0; k < n && error_count < 10; ++k, ++expected_first, ++actual_first)
-    {
-        if (!(*expected_first == *actual_first))
-        {
-            std::stringstream outstr;
-            outstr << "error at " << file << ":" << line << " - " << message << ", at index " << k;
-            issue_error_message(outstr);
-            ++error_count;
-        }
-    }
-}
-
-// ForwardIterator is like type Iterator, but restricted to be a forward iterator.
-// Only the forward iterator signatures that are necessary for tests are present.
-// Post-increment in particular is deliberatly omitted since our templates should avoid using it
-// because of efficiency considerations.
-template <typename Iterator, typename IteratorTag>
-class ForwardIterator
-{
-  public:
-    typedef IteratorTag iterator_category;
-    typedef typename std::iterator_traits<Iterator>::value_type value_type;
-    typedef typename std::iterator_traits<Iterator>::difference_type difference_type;
-    typedef typename std::iterator_traits<Iterator>::pointer pointer;
-    typedef typename std::iterator_traits<Iterator>::reference reference;
-
-  protected:
-    Iterator my_iterator;
-    typedef value_type element_type;
-
-  public:
-    ForwardIterator() = default;
-    explicit ForwardIterator(Iterator i) : my_iterator(i) {}
-    reference operator*() const { return *my_iterator; }
-    Iterator operator->() const { return my_iterator; }
-    ForwardIterator
-    operator++()
-    {
-        ++my_iterator;
-        return *this;
-    }
-    ForwardIterator operator++(int32_t)
-    {
-        auto retval = *this;
-        my_iterator++;
-        return retval;
-    }
-    friend bool
-    operator==(const ForwardIterator& i, const ForwardIterator& j)
-    {
-        return i.my_iterator == j.my_iterator;
-    }
-    friend bool
-    operator!=(const ForwardIterator& i, const ForwardIterator& j)
-    {
-        return i.my_iterator != j.my_iterator;
-    }
-
-    Iterator
-    iterator() const
-    {
-        return my_iterator;
-    }
-};
-
-template <typename Iterator, typename IteratorTag>
-class BidirectionalIterator : public ForwardIterator<Iterator, IteratorTag>
-{
-    typedef ForwardIterator<Iterator, IteratorTag> base_type;
-
-  public:
-    BidirectionalIterator() = default;
-    explicit BidirectionalIterator(Iterator i) : base_type(i) {}
-    BidirectionalIterator(const base_type& i) : base_type(i.iterator()) {}
-
-    BidirectionalIterator
-    operator++()
-    {
-        ++base_type::my_iterator;
-        return *this;
-    }
-    BidirectionalIterator
-    operator--()
-    {
-        --base_type::my_iterator;
-        return *this;
-    }
-    BidirectionalIterator operator++(int32_t)
-    {
-        auto retval = *this;
-        base_type::my_iterator++;
-        return retval;
-    }
-    BidirectionalIterator operator--(int32_t)
-    {
-        auto retval = *this;
-        base_type::my_iterator--;
-        return retval;
-    }
-};
-
-template <typename Iterator, typename F>
-void
-fill_data(Iterator first, Iterator last, F f)
-{
-    typedef typename std::iterator_traits<Iterator>::value_type T;
-    for (std::size_t i = 0; first != last; ++first, ++i)
-    {
-        *first = T(f(i));
-    }
-}
-
-struct MemoryChecker {
-    // static counters and state tags
-    static std::atomic<std::int64_t> alive_object_counter; // initialized outside
-    static constexpr std::int64_t alive_state = 0xAAAAAAAAAAAAAAAA;
-    static constexpr std::int32_t dead_state = 0; // only used as a set value to cancel alive_state
-
-    std::int32_t _value; // object value used for algorithms
-    std::int64_t _state; // state tag used for checks
-
-    // ctors, dtors, assign ops
-    explicit MemoryChecker(std::int32_t value = 0) : _value(value) {
-        // check for EXPECT_TRUE(state() != alive_state, ...) has not been done since we cannot guarantee that
-        // raw memory for object being constructed does not have a bit sequence being equal to alive_state
-
-        // set constructed state and increment counter for living object
-        inc_alive_objects();
-        _state = alive_state;
-    }
-    MemoryChecker(MemoryChecker&& other) : _value(other.value()) {
-        // check for EXPECT_TRUE(state() != alive_state, ...) has not been done since
-        // compiler can optimize out the move ctor call that results in false positive failure
-        EXPECT_TRUE(other.state() == alive_state, "wrong effect from MemoryChecker(MemoryChecker&&): attemp to construct an object from non-existing object");
-        // set constructed state and increment counter for living object
-        inc_alive_objects();
-        _state = alive_state;
-    }
-    MemoryChecker(const MemoryChecker& other) : _value(other.value()) {
-        // check for EXPECT_TRUE(state() != alive_state, ...) has not been done since
-        // compiler can optimize out the copy ctor call that results in false positive failure
-        EXPECT_TRUE(other.state() == alive_state, "wrong effect from MemoryChecker(const MemoryChecker&): attemp to construct an object from non-existing object");
-        // set constructed state and increment counter for living object
-        inc_alive_objects();
-        _state = alive_state;
-    }
-    MemoryChecker& operator=(MemoryChecker&& other) {
-        // check if we do not assign over uninitialized memory
-        EXPECT_TRUE(state() == alive_state, "wrong effect from MemoryChecker::operator=(MemoryChecker&& other): attemp to assign to non-existing object");
-        EXPECT_TRUE(other.state() == alive_state, "wrong effect from MemoryChecker::operator=(MemoryChecker&& other): attemp to assign from non-existing object");
-        // just assign new value, counter is the same, state is the same
-        _value = other.value();
-
-        return *this;
-    }
-    MemoryChecker& operator=(const MemoryChecker& other) {
-        // check if we do not assign over uninitialized memory
-        EXPECT_TRUE(state() == alive_state, "wrong effect from MemoryChecker::operator=(const MemoryChecker& other): attemp to assign to non-existing object");
-        EXPECT_TRUE(other.state() == alive_state, "wrong effect from MemoryChecker::operator=(const MemoryChecker& other): attemp to assign from non-existing object");
-        // just assign new value, counter is the same, state is the same
-        _value = other.value();
-
-        return *this;
-    }
-    ~MemoryChecker() {
-        // check if we do not double destruct the object
-        EXPECT_TRUE(state() == alive_state, "wrong effect from ~MemoryChecker(): attemp to destroy non-existing object");
-        // set destructed state and decrement counter for living object
-        static_cast<volatile std::int64_t&>(_state) = dead_state;
-        dec_alive_objects();
-    }
-
-    // getters
-    std::int32_t value() const { return _value; }
-    std::int64_t state() const { return _state; }
-    static std::int32_t alive_objects() { return alive_object_counter.load(); }
-private:
-    // setters
-    void inc_alive_objects() { alive_object_counter.fetch_add(1); }
-    void dec_alive_objects() { alive_object_counter.fetch_sub(1); }
-};
-
-std::atomic<std::int64_t> MemoryChecker::alive_object_counter{0};
-
-std::ostream& operator<<(std::ostream& os, const MemoryChecker& val) { return (os << val.value()); }
-bool operator==(const MemoryChecker& v1, const MemoryChecker& v2) { return v1.value() == v2.value(); }
-bool operator<(const MemoryChecker& v1, const MemoryChecker& v2) { return v1.value() < v2.value(); }
-
-// Sequence<T> is a container of a sequence of T with lots of kinds of iterators.
-// Prefixes on begin/end mean:
-//      c = "const"
-//      f = "forward"
-// No prefix indicates non-const random-access iterator.
-template <typename T>
-class Sequence
-{
-    std::vector<T> m_storage;
-
-  public:
-    typedef typename std::vector<T>::iterator iterator;
-    typedef typename std::vector<T>::const_iterator const_iterator;
-    typedef ForwardIterator<iterator, std::forward_iterator_tag> forward_iterator;
-    typedef ForwardIterator<const_iterator, std::forward_iterator_tag> const_forward_iterator;
-
-    typedef BidirectionalIterator<iterator, std::bidirectional_iterator_tag> bidirectional_iterator;
-    typedef BidirectionalIterator<const_iterator, std::bidirectional_iterator_tag> const_bidirectional_iterator;
-
-    typedef T value_type;
-    explicit Sequence(size_t size) : m_storage(size) {}
-
-    // Construct sequence [f(0), f(1), ... f(size-1)]
-    // f can rely on its invocations being sequential from 0 to size-1.
-    template <typename Func>
-    Sequence(size_t size, Func f)
-    {
-        m_storage.reserve(size);
-        // Use push_back because T might not have a default constructor
-        for (size_t k = 0; k < size; ++k)
-            m_storage.push_back(T(f(k)));
-    }
-    Sequence(const std::initializer_list<T>& data) : m_storage(data) {}
-
-    const_iterator
-    begin() const
-    {
-        return m_storage.begin();
-    }
-    const_iterator
-    end() const
-    {
-        return m_storage.end();
-    }
-    iterator
-    begin()
-    {
-        return m_storage.begin();
-    }
-    iterator
-    end()
-    {
-        return m_storage.end();
-    }
-    const_iterator
-    cbegin() const
-    {
-        return m_storage.cbegin();
-    }
-    const_iterator
-    cend() const
-    {
-        return m_storage.cend();
-    }
-    forward_iterator
-    fbegin()
-    {
-        return forward_iterator(m_storage.begin());
-    }
-    forward_iterator
-    fend()
-    {
-        return forward_iterator(m_storage.end());
-    }
-    const_forward_iterator
-    cfbegin() const
-    {
-        return const_forward_iterator(m_storage.cbegin());
-    }
-    const_forward_iterator
-    cfend() const
-    {
-        return const_forward_iterator(m_storage.cend());
-    }
-    const_forward_iterator
-    fbegin() const
-    {
-        return const_forward_iterator(m_storage.cbegin());
-    }
-    const_forward_iterator
-    fend() const
-    {
-        return const_forward_iterator(m_storage.cend());
-    }
-
-    const_bidirectional_iterator
-    cbibegin() const
-    {
-        return const_bidirectional_iterator(m_storage.cbegin());
-    }
-    const_bidirectional_iterator
-    cbiend() const
-    {
-        return const_bidirectional_iterator(m_storage.cend());
-    }
-
-    bidirectional_iterator
-    bibegin()
-    {
-        return bidirectional_iterator(m_storage.begin());
-    }
-    bidirectional_iterator
-    biend()
-    {
-        return bidirectional_iterator(m_storage.end());
-    }
-
-    std::size_t
-    size() const
-    {
-        return m_storage.size();
-    }
-    const T*
-    data() const
-    {
-        return m_storage.data();
-    }
-    typename std::vector<T>::reference operator[](size_t j) { return m_storage[j]; }
-    const T& operator[](size_t j) const { return m_storage[j]; }
-
-    // Fill with given value
-    void
-    fill(const T& value)
-    {
-        for (size_t i = 0; i < m_storage.size(); i++)
-            m_storage[i] = value;
-    }
-
-    void
-    print() const;
-
-    template <typename Func>
-    void
-    fill(Func f)
-    {
-        fill_data(m_storage.begin(), m_storage.end(), f);
-    }
-};
-
-template <typename T>
-void
-Sequence<T>::print() const
-{
-    std::cout << "size = " << size() << ": { ";
-    std::copy(begin(), end(), std::ostream_iterator<T>(std::cout, " "));
-    std::cout << " } " << std::endl;
-}
-
-// Predicates for algorithms
-template <typename DataType>
-struct is_equal_to
-{
-    is_equal_to(const DataType& expected) : m_expected(expected) {}
-    bool
-    operator()(const DataType& actual) const
-    {
-        return actual == m_expected;
-    }
-
-  private:
-    DataType m_expected;
-};
-
-// Low-quality hash function, returns value between 0 and (1<<bits)-1
-// Warning: low-order bits are quite predictable.
-inline size_t
-HashBits(size_t i, size_t bits)
-{
-    size_t mask = bits >= 8 * sizeof(size_t) ? ~size_t(0) : (size_t(1) << bits) - 1;
-    return (424157 * i ^ 0x24aFa) & mask;
-}
-
-// Stateful unary op
-template <typename T, typename U>
-class Complement
-{
-    int32_t val;
-
-  public:
-    Complement(T v) : val(v) {}
-    U
-    operator()(const T& x) const
-    {
-        return U(val - x);
-    }
-};
-
-// Tag used to prevent accidental use of converting constructor, even if use is explicit.
-struct OddTag
-{
-};
-
-class Sum;
-
-// Type with limited set of operations.  Not default-constructible.
-// Only available operator is "==".
-// Typically used as value type in tests.
-class Number
-{
-    int32_t value;
-    friend class Add;
-    friend class Sum;
-    friend class IsMultiple;
-    friend class Congruent;
-    friend Sum
-    operator+(const Sum& x, const Sum& y);
-
-  public:
-    Number(int32_t val, OddTag) : value(val) {}
-    friend bool
-    operator==(const Number& x, const Number& y)
-    {
-        return x.value == y.value;
-    }
-    friend std::ostream&
-    operator<<(std::ostream& o, const Number& d)
-    {
-        return o << d.value;
-    }
-};
-
-// Stateful predicate for Number.  Not default-constructible.
-class IsMultiple
-{
-    long modulus;
-
-  public:
-    // True if x is multiple of modulus
-    bool
-    operator()(Number x) const
-    {
-        return x.value % modulus == 0;
-    }
-    IsMultiple(long modulus_, OddTag) : modulus(modulus_) {}
-};
-
-// Stateful equivalence-class predicate for Number.  Not default-constructible.
-class Congruent
-{
-    long modulus;
-
-  public:
-    // True if x and y have same remainder for the given modulus.
-    // Note: this is not quite the same as "equivalent modulo modulus" when x and y have different
-    // sign, but nonetheless AreCongruent is still an equivalence relationship, which is all
-    // we need for testing.
-    bool
-    operator()(Number x, Number y) const
-    {
-        return x.value % modulus == y.value % modulus;
-    }
-    Congruent(long modulus_, OddTag) : modulus(modulus_) {}
-};
-
-// Stateful reduction operation for Number
-class Add
-{
-    long bias;
-
-  public:
-    explicit Add(OddTag) : bias(1) {}
-    Number
-    operator()(Number x, const Number& y)
-    {
-        return Number(x.value + y.value + (bias - 1), OddTag());
-    }
-};
-
-// Class similar to Number, but has default constructor and +.
-class Sum : public Number
-{
-  public:
-    Sum() : Number(0, OddTag()) {}
-    Sum(long x, OddTag) : Number(x, OddTag()) {}
-    friend Sum
-    operator+(const Sum& x, const Sum& y)
-    {
-        return Sum(x.value + y.value, OddTag());
-    }
-};
-
-// Type with limited set of operations, which includes an associative but not commutative operation.
-// Not default-constructible.
-// Typically used as value type in tests involving "GENERALIZED_NONCOMMUTATIVE_SUM".
-class MonoidElement
-{
-    size_t a, b;
-
-  public:
-    MonoidElement(size_t a_, size_t b_, OddTag) : a(a_), b(b_) {}
-    friend bool
-    operator==(const MonoidElement& x, const MonoidElement& y)
-    {
-        return x.a == y.a && x.b == y.b;
-    }
-    friend std::ostream&
-    operator<<(std::ostream& o, const MonoidElement& x)
-    {
-        return o << "[" << x.a << ".." << x.b << ")";
-    }
-    friend class AssocOp;
-};
-
-// Stateful associative op for MonoidElement
-// It's not really a monoid since the operation is not allowed for any two elements.
-// But it's good enough for testing.
-class AssocOp
-{
-    unsigned c;
-
-  public:
-    explicit AssocOp(OddTag) : c(5) {}
-    MonoidElement
-    operator()(const MonoidElement& x, const MonoidElement& y)
-    {
-        unsigned d = 5;
-        EXPECT_EQ(d, c, "state lost");
-        EXPECT_EQ(x.b, y.a, "commuted?");
-
-        return MonoidElement(x.a, y.b, OddTag());
-    }
-};
-
-// Multiplication of matrix is an associative but not commutative operation
-// Typically used as value type in tests involving "GENERALIZED_NONCOMMUTATIVE_SUM".
-template <typename T>
-struct Matrix2x2
-{
-    T a[2][2];
-    Matrix2x2() : a{{1, 0}, {0, 1}} {}
-    Matrix2x2(T x, T y) : a{{0, x}, {x, y}} {}
-#if !defined(_PSTL_ICL_19_VC14_VC141_TEST_SCAN_RELEASE_BROKEN)
-    Matrix2x2(const Matrix2x2& m) : a{{m.a[0][0], m.a[0][1]}, {m.a[1][0], m.a[1][1]}} {}
-    Matrix2x2&
-    operator=(const Matrix2x2& m)
-    {
-        a[0][0] = m.a[0][0], a[0][1] = m.a[0][1], a[1][0] = m.a[1][0], a[1][1] = m.a[1][1];
-        return *this;
-    }
-#endif
-};
-
-template <typename T>
-bool
-operator==(const Matrix2x2<T>& left, const Matrix2x2<T>& right)
-{
-    return left.a[0][0] == right.a[0][0] && left.a[0][1] == right.a[0][1] && left.a[1][0] == right.a[1][0] &&
-           left.a[1][1] == right.a[1][1];
-}
-
-template <typename T>
-Matrix2x2<T>
-multiply_matrix(const Matrix2x2<T>& left, const Matrix2x2<T>& right)
-{
-    Matrix2x2<T> result;
-    for (int32_t i = 0; i < 2; ++i)
-    {
-        for (int32_t j = 0; j < 2; ++j)
-        {
-            result.a[i][j] = left.a[i][0] * right.a[0][j] + left.a[i][1] * right.a[1][j];
-        }
-    }
-    return result;
-}
-
-//============================================================================
-// Adapters for creating different types of iterators.
-//
-// In this block we implemented some adapters for creating differnet types of iterators.
-// It's needed for extending the unit testing of Parallel STL algorithms.
-// We have adapters for iterators with different tags (forward_iterator_tag, bidirectional_iterator_tag), reverse iterators.
-// The input iterator should be const or non-const, non-reverse random access iterator.
-// Iterator creates in "MakeIterator":
-// firstly, iterator is "packed" by "IteratorTypeAdapter" (creating forward or bidirectional iterator)
-// then iterator is "packed" by "ReverseAdapter" (if it's possible)
-// So, from input iterator we may create, for example, reverse bidirectional iterator.
-// "Main" functor for testing iterators is named "invoke_on_all_iterator_types".
-
-// Base adapter
-template <typename Iterator>
-struct BaseAdapter
-{
-    typedef Iterator iterator_type;
-    iterator_type
-    operator()(Iterator it)
-    {
-        return it;
-    }
-};
-
-// Check if the iterator is reverse iterator
-// Note: it works only for iterators that created by std::reverse_iterator
-template <typename NotReverseIterator>
-struct isReverse : std::false_type
-{
-};
-
-template <typename Iterator>
-struct isReverse<std::reverse_iterator<Iterator>> : std::true_type
-{
-};
-
-// Reverse adapter
-template <typename Iterator, typename IsReverse>
-struct ReverseAdapter
-{
-    typedef std::reverse_iterator<Iterator> iterator_type;
-    iterator_type
-    operator()(Iterator it)
-    {
-#if defined(_PSTL_CPP14_MAKE_REVERSE_ITERATOR_PRESENT)
-        return std::make_reverse_iterator(it);
-#else
-        return iterator_type(it);
-#endif
-    }
-};
-
-// Non-reverse adapter
-template <typename Iterator>
-struct ReverseAdapter<Iterator, std::false_type> : BaseAdapter<Iterator>
-{
-};
-
-// Iterator adapter by type (by default std::random_access_iterator_tag)
-template <typename Iterator, typename IteratorTag>
-struct IteratorTypeAdapter : BaseAdapter<Iterator>
-{
-};
-
-// Iterator adapter for forward iterator
-template <typename Iterator>
-struct IteratorTypeAdapter<Iterator, std::forward_iterator_tag>
-{
-    typedef ForwardIterator<Iterator, std::forward_iterator_tag> iterator_type;
-    iterator_type
-    operator()(Iterator it)
-    {
-        return iterator_type(it);
-    }
-};
-
-// Iterator adapter for bidirectional iterator
-template <typename Iterator>
-struct IteratorTypeAdapter<Iterator, std::bidirectional_iterator_tag>
-{
-    typedef BidirectionalIterator<Iterator, std::bidirectional_iterator_tag> iterator_type;
-    iterator_type
-    operator()(Iterator it)
-    {
-        return iterator_type(it);
-    }
-};
-
-//For creating iterator with new type
-template <typename InputIterator, typename IteratorTag, typename IsReverse>
-struct MakeIterator
-{
-    typedef IteratorTypeAdapter<InputIterator, IteratorTag> IterByType;
-    typedef ReverseAdapter<typename IterByType::iterator_type, IsReverse> ReverseIter;
-
-    typename ReverseIter::iterator_type
-    operator()(InputIterator it)
-    {
-        return ReverseIter()(IterByType()(it));
-    }
-};
-
-// Useful constant variables
-constexpr std::size_t GuardSize = 5;
-constexpr std::ptrdiff_t sizeLimit = 1000;
-
-template <typename Iter, typename Void = void> // local iterator_traits for non-iterators
-struct iterator_traits_
-{
-};
-
-template <typename Iter> // For iterators
-struct iterator_traits_<Iter,
-                        typename std::enable_if<!std::is_void<typename Iter::iterator_category>::value, void>::type>
-{
-    typedef typename Iter::iterator_category iterator_category;
-};
-
-template <typename T> // For pointers
-struct iterator_traits_<T*>
-{
-    typedef std::random_access_iterator_tag iterator_category;
-};
-
-// is iterator Iter has tag Tag
-template <typename Iter, typename Tag>
-using is_same_iterator_category = std::is_same<typename iterator_traits_<Iter>::iterator_category, Tag>;
-
-// if we run with reverse or const iterators we shouldn't test the large range
-template <typename IsReverse, typename IsConst>
-struct invoke_if_
-{
-    template <typename Op, typename... Rest>
-    void
-    operator()(bool is_allow, Op op, Rest&&... rest)
-    {
-        if (is_allow)
-            op(std::forward<Rest>(rest)...);
-    }
-};
-template <>
-struct invoke_if_<std::false_type, std::false_type>
-{
-    template <typename Op, typename... Rest>
-    void
-    operator()(bool, Op op, Rest&&... rest)
-    {
-        op(std::forward<Rest>(rest)...);
-    }
-};
-
-// Base non_const_wrapper struct. It is used to distinguish non_const testcases
-// from a regular one. For non_const testcases only compilation is checked.
-struct non_const_wrapper
-{
-};
-
-// Generic wrapper to specify iterator type to execute callable Op on.
-// The condition can be either positive(Op is executed only with IteratorTag)
-// or negative(Op is executed with every type of iterators except IteratorTag)
-template <typename Op, typename IteratorTag, bool IsPositiveCondition = true>
-struct non_const_wrapper_tagged : non_const_wrapper
-{
-    template <typename Policy, typename Iterator>
-    typename std::enable_if<IsPositiveCondition == is_same_iterator_category<Iterator, IteratorTag>::value, void>::type
-    operator()(Policy&& exec, Iterator iter)
-    {
-        Op()(exec, iter);
-    }
-
-    template <typename Policy, typename InputIterator, typename OutputIterator>
-    typename std::enable_if<IsPositiveCondition == is_same_iterator_category<OutputIterator, IteratorTag>::value,
-                            void>::type
-    operator()(Policy&& exec, InputIterator input_iter, OutputIterator out_iter)
-    {
-        Op()(exec, input_iter, out_iter);
-    }
-
-    template <typename Policy, typename Iterator>
-    typename std::enable_if<IsPositiveCondition != is_same_iterator_category<Iterator, IteratorTag>::value, void>::type
-    operator()(Policy&&, Iterator)
-    {
-    }
-
-    template <typename Policy, typename InputIterator, typename OutputIterator>
-    typename std::enable_if<IsPositiveCondition != is_same_iterator_category<OutputIterator, IteratorTag>::value,
-                            void>::type
-    operator()(Policy&&, InputIterator, OutputIterator)
-    {
-    }
-};
-
-// These run_for_* structures specify with which types of iterators callable object Op
-// should be executed.
-template <typename Op>
-struct run_for_rnd : non_const_wrapper_tagged<Op, std::random_access_iterator_tag>
-{
-};
-
-template <typename Op>
-struct run_for_rnd_bi : non_const_wrapper_tagged<Op, std::forward_iterator_tag, false>
-{
-};
-
-template <typename Op>
-struct run_for_rnd_fw : non_const_wrapper_tagged<Op, std::bidirectional_iterator_tag, false>
-{
-};
-
-// Invoker for different types of iterators.
-template <typename IteratorTag, typename IsReverse>
-struct iterator_invoker
-{
-    template <typename Iterator>
-    using make_iterator = MakeIterator<Iterator, IteratorTag, IsReverse>;
-    template <typename Iterator>
-    using IsConst = typename std::is_const<
-        typename std::remove_pointer<typename std::iterator_traits<Iterator>::pointer>::type>::type;
-    template <typename Iterator>
-    using invoke_if = invoke_if_<IsReverse, IsConst<Iterator>>;
-
-    // A single iterator version which is used for non_const testcases
-    template <typename Policy, typename Op, typename Iterator>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-                                std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, Iterator iter)
-    {
-        op(std::forward<Policy>(exec), make_iterator<Iterator>()(iter));
-    }
-
-    // A version with 2 iterators which is used for non_const testcases
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value &&
-                                std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator input_iter, OutputIterator out_iter)
-    {
-        op(std::forward<Policy>(exec), make_iterator<InputIterator>()(input_iter),
-           make_iterator<OutputIterator>()(out_iter));
-    }
-
-    template <typename Policy, typename Op, typename Iterator, typename Size, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, Op op, Iterator begin, Size n, Rest&&... rest)
-    {
-        invoke_if<Iterator>()(n <= sizeLimit, op, exec, make_iterator<Iterator>()(begin), n,
-                              std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename Iterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-                                !std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, Iterator inputBegin, Iterator inputEnd, Rest&&... rest)
-    {
-        invoke_if<Iterator>()(std::distance(inputBegin, inputEnd) <= sizeLimit, op, exec,
-                              make_iterator<Iterator>()(inputBegin), make_iterator<Iterator>()(inputEnd),
-                              std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator inputBegin, InputIterator inputEnd, OutputIterator outputBegin,
-               Rest&&... rest)
-    {
-        invoke_if<InputIterator>()(std::distance(inputBegin, inputEnd) <= sizeLimit, op, exec,
-                                   make_iterator<InputIterator>()(inputBegin), make_iterator<InputIterator>()(inputEnd),
-                                   make_iterator<OutputIterator>()(outputBegin), std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator inputBegin, InputIterator inputEnd, OutputIterator outputBegin,
-               OutputIterator outputEnd, Rest&&... rest)
-    {
-        invoke_if<InputIterator>()(std::distance(inputBegin, inputEnd) <= sizeLimit, op, exec,
-                                   make_iterator<InputIterator>()(inputBegin), make_iterator<InputIterator>()(inputEnd),
-                                   make_iterator<OutputIterator>()(outputBegin),
-                                   make_iterator<OutputIterator>()(outputEnd), std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator1, typename InputIterator2, typename OutputIterator,
-              typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator1 inputBegin1, InputIterator1 inputEnd1, InputIterator2 inputBegin2,
-               InputIterator2 inputEnd2, OutputIterator outputBegin, OutputIterator outputEnd, Rest&&... rest)
-    {
-        invoke_if<InputIterator1>()(
-            std::distance(inputBegin1, inputEnd1) <= sizeLimit, op, exec, make_iterator<InputIterator1>()(inputBegin1),
-            make_iterator<InputIterator1>()(inputEnd1), make_iterator<InputIterator2>()(inputBegin2),
-            make_iterator<InputIterator2>()(inputEnd2), make_iterator<OutputIterator>()(outputBegin),
-            make_iterator<OutputIterator>()(outputEnd), std::forward<Rest>(rest)...);
-    }
-};
-
-// Invoker for reverse iterators only
-// Note: if we run with reverse iterators we shouldn't test the large range
-template <typename IteratorTag>
-struct iterator_invoker<IteratorTag, /* IsReverse = */ std::true_type>
-{
-
-    template <typename Iterator>
-    using make_iterator = MakeIterator<Iterator, IteratorTag, std::true_type>;
-
-    // A single iterator version which is used for non_const testcases
-    template <typename Policy, typename Op, typename Iterator>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-                                std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, Iterator iter)
-    {
-        op(std::forward<Policy>(exec), make_iterator<Iterator>()(iter));
-    }
-
-    // A version with 2 iterators which is used for non_const testcases
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value &&
-                                std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator input_iter, OutputIterator out_iter)
-    {
-        op(std::forward<Policy>(exec), make_iterator<InputIterator>()(input_iter),
-           make_iterator<OutputIterator>()(out_iter));
-    }
-
-    template <typename Policy, typename Op, typename Iterator, typename Size, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value, void>::type
-    operator()(Policy&& exec, Op op, Iterator begin, Size n, Rest&&... rest)
-    {
-        if (n <= sizeLimit)
-            op(exec, make_iterator<Iterator>()(begin + n), n, std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename Iterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<Iterator, std::random_access_iterator_tag>::value &&
-                                !std::is_base_of<non_const_wrapper, Op>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, Iterator inputBegin, Iterator inputEnd, Rest&&... rest)
-    {
-        if (std::distance(inputBegin, inputEnd) <= sizeLimit)
-            op(exec, make_iterator<Iterator>()(inputEnd), make_iterator<Iterator>()(inputBegin),
-               std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator inputBegin, InputIterator inputEnd, OutputIterator outputBegin,
-               Rest&&... rest)
-    {
-        if (std::distance(inputBegin, inputEnd) <= sizeLimit)
-            op(exec, make_iterator<InputIterator>()(inputEnd), make_iterator<InputIterator>()(inputBegin),
-               make_iterator<OutputIterator>()(outputBegin + (inputEnd - inputBegin)), std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator, typename OutputIterator, typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator inputBegin, InputIterator inputEnd, OutputIterator outputBegin,
-               OutputIterator outputEnd, Rest&&... rest)
-    {
-        if (std::distance(inputBegin, inputEnd) <= sizeLimit)
-            op(exec, make_iterator<InputIterator>()(inputEnd), make_iterator<InputIterator>()(inputBegin),
-               make_iterator<OutputIterator>()(outputEnd), make_iterator<OutputIterator>()(outputBegin),
-               std::forward<Rest>(rest)...);
-    }
-
-    template <typename Policy, typename Op, typename InputIterator1, typename InputIterator2, typename OutputIterator,
-              typename... Rest>
-    typename std::enable_if<is_same_iterator_category<OutputIterator, std::random_access_iterator_tag>::value,
-                            void>::type
-    operator()(Policy&& exec, Op op, InputIterator1 inputBegin1, InputIterator1 inputEnd1, InputIterator2 inputBegin2,
-               InputIterator2 inputEnd2, OutputIterator outputBegin, OutputIterator outputEnd, Rest&&... rest)
-    {
-        if (std::distance(inputBegin1, inputEnd1) <= sizeLimit)
-            op(exec, make_iterator<InputIterator1>()(inputEnd1), make_iterator<InputIterator1>()(inputBegin1),
-               make_iterator<InputIterator2>()(inputEnd2), make_iterator<InputIterator2>()(inputBegin2),
-               make_iterator<OutputIterator>()(outputEnd), make_iterator<OutputIterator>()(outputBegin),
-               std::forward<Rest>(rest)...);
-    }
-};
-
-// We can't create reverse iterator from forward iterator
-template <>
-struct iterator_invoker<std::forward_iterator_tag, /*isReverse=*/std::true_type>
-{
-    template <typename... Rest>
-    void
-    operator()(Rest&&...)
-    {
-    }
-};
-
-template <typename IsReverse>
-struct reverse_invoker
-{
-    template <typename... Rest>
-    void
-    operator()(Rest&&... rest)
-    {
-        // Random-access iterator
-        iterator_invoker<std::random_access_iterator_tag, IsReverse>()(std::forward<Rest>(rest)...);
-
-        // Forward iterator
-        iterator_invoker<std::forward_iterator_tag, IsReverse>()(std::forward<Rest>(rest)...);
-
-        // Bidirectional iterator
-        iterator_invoker<std::bidirectional_iterator_tag, IsReverse>()(std::forward<Rest>(rest)...);
-    }
-};
-
-struct invoke_on_all_iterator_types
-{
-    template <typename... Rest>
-    void
-    operator()(Rest&&... rest)
-    {
-        reverse_invoker</* IsReverse = */ std::false_type>()(std::forward<Rest>(rest)...);
-        reverse_invoker</* IsReverse = */ std::true_type>()(std::forward<Rest>(rest)...);
-    }
-};
-//============================================================================
-
-// Invoke op(policy,rest...) for each possible policy.
-template <typename Op, typename... T>
-void
-invoke_on_all_policies(Op op, T&&... rest)
-{
-    using namespace __pstl::execution;
-
-    // Try static execution policies
-    invoke_on_all_iterator_types()(seq, op, std::forward<T>(rest)...);
-    invoke_on_all_iterator_types()(unseq, op, std::forward<T>(rest)...);
-    invoke_on_all_iterator_types()(par, op, std::forward<T>(rest)...);
-    invoke_on_all_iterator_types()(par_unseq, op, std::forward<T>(rest)...);
-}
-
-template <typename F>
-struct NonConstAdapter
-{
-    F my_f;
-    NonConstAdapter(const F& f) : my_f(f) {}
-
-    template <typename... Types>
-    auto
-    operator()(Types&&... args) -> decltype(std::declval<F>().
-                                            operator()(std::forward<Types>(args)...))
-    {
-        return my_f(std::forward<Types>(args)...);
-    }
-};
-
-template <typename F>
-NonConstAdapter<F>
-non_const(const F& f)
-{
-    return NonConstAdapter<F>(f);
-}
-
-// Wrapper for types. It's need for counting of constructing and destructing objects
-template <typename T>
-class Wrapper
-{
-  public:
-    Wrapper()
-    {
-        my_field = std::shared_ptr<T>(new T());
-        ++my_count;
-    }
-    Wrapper(const T& input)
-    {
-        my_field = std::shared_ptr<T>(new T(input));
-        ++my_count;
-    }
-    Wrapper(const Wrapper& input)
-    {
-        my_field = input.my_field;
-        ++my_count;
-    }
-    Wrapper(Wrapper&& input)
-    {
-        my_field = input.my_field;
-        input.my_field = nullptr;
-        ++move_count;
-    }
-    Wrapper&
-    operator=(const Wrapper& input)
-    {
-        my_field = input.my_field;
-        return *this;
-    }
-    Wrapper&
-    operator=(Wrapper&& input)
-    {
-        my_field = input.my_field;
-        input.my_field = nullptr;
-        ++move_count;
-        return *this;
-    }
-    bool
-    operator==(const Wrapper& input) const
-    {
-        return my_field == input.my_field;
-    }
-    bool
-    operator<(const Wrapper& input) const
-    {
-        return *my_field < *input.my_field;
-    }
-    bool
-    operator>(const Wrapper& input) const
-    {
-        return *my_field > *input.my_field;
-    }
-    friend std::ostream&
-    operator<<(std::ostream& stream, const Wrapper& input)
-    {
-        return stream << *(input.my_field);
-    }
-    ~Wrapper()
-    {
-        --my_count;
-        if (move_count > 0)
-        {
-            --move_count;
-        }
-    }
-    T*
-    get_my_field() const
-    {
-        return my_field.get();
-    };
-    static size_t
-    Count()
-    {
-        return my_count;
-    }
-    static size_t
-    MoveCount()
-    {
-        return move_count;
-    }
-    static void
-    SetCount(const size_t& n)
-    {
-        my_count = n;
-    }
-    static void
-    SetMoveCount(const size_t& n)
-    {
-        move_count = n;
-    }
-
-  private:
-    static std::atomic<size_t> my_count;
-    static std::atomic<size_t> move_count;
-    std::shared_ptr<T> my_field;
-};
-
-template <typename T>
-std::atomic<size_t> Wrapper<T>::my_count = {0};
-
-template <typename T>
-std::atomic<size_t> Wrapper<T>::move_count = {0};
-
-template <typename InputIterator, typename T, typename BinaryOperation, typename UnaryOperation>
-T
-transform_reduce_serial(InputIterator first, InputIterator last, T init, BinaryOperation binary_op,
-                        UnaryOperation unary_op) noexcept
-{
-    for (; first != last; ++first)
-    {
-        init = binary_op(init, unary_op(*first));
-    }
-    return init;
-}
-
-static const char*
-done()
-{
-#if defined(_PSTL_TEST_SUCCESSFUL_KEYWORD)
-    return "done";
-#else
-    return "passed";
-#endif
-}
-
-// test_algo_basic_* functions are used to execute
-// f on a very basic sequence of elements of type T.
-
-// Should be used with unary predicate
-template <typename T, typename F>
-static void
-test_algo_basic_single(F&& f)
-{
-    size_t N = 10;
-    Sequence<T> in(N, [](size_t v) -> T { return T(v); });
-
-    invoke_on_all_policies(f, in.begin());
-}
-
-// Should be used with binary predicate
-template <typename T, typename F>
-static void
-test_algo_basic_double(F&& f)
-{
-    size_t N = 10;
-    Sequence<T> in(N, [](size_t v) -> T { return T(v); });
-    Sequence<T> out(N, [](size_t v) -> T { return T(v); });
-
-    invoke_on_all_policies(f, in.begin(), out.begin());
-}
-
-template <typename Policy, typename F>
-static void
-invoke_if(Policy&&, F f)
-{
-#if defined(_PSTL_ICC_16_VC14_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN) || defined(_PSTL_ICC_17_VC141_TEST_SIMD_LAMBDA_DEBUG_32_BROKEN)
-    using decay_policy = typename std::decay<Policy>::type;
-    using allow_unsequenced =
-        std::integral_constant<bool, (std::is_same<decay_policy, std::execution::unsequenced_policy>::value ||
-                                      std::is_same<decay_policy, std::execution::parallel_unsequenced_policy>::value)>;
-    __pstl::__internal::__invoke_if_not(allow_unsequenced{}, f);
-#else
-    f();
-#endif
-}
-
-} /* namespace TestUtils */
diff --git a/third-party/unittest/googletest/src/gtest.cc b/third-party/unittest/googletest/src/gtest.cc
index 30a5cc3f83a7..37d380a78983 100644
--- a/third-party/unittest/googletest/src/gtest.cc
+++ b/third-party/unittest/googletest/src/gtest.cc
@@ -43,6 +43,7 @@
 #include <algorithm>
 #include <chrono>  // NOLINT
 #include <cmath>
+#include <csignal>
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
diff --git a/utils/bazel/WORKSPACE b/utils/bazel/WORKSPACE
index 84e65e6f67b5..5e28c3b67fc3 100644
--- a/utils/bazel/WORKSPACE
+++ b/utils/bazel/WORKSPACE
@@ -164,6 +164,15 @@ maybe(
     url = "https://github.com/pybind/pybind11/archive/v2.10.3.zip",
 )
 
+maybe(
+    http_archive,
+    name = "pyyaml",
+    build_file = "@llvm-raw//utils/bazel/third_party_build:pyyaml.BUILD",
+    sha256 = "f0a35d7f282a6d6b1a4f3f3965ef5c124e30ed27a0088efb97c0977268fd671f",
+    strip_prefix = "pyyaml-5.1/lib3",
+    url = "https://github.com/yaml/pyyaml/archive/refs/tags/5.1.zip",
+)
+
 maybe(
     http_archive,
     name = "robin_map",
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 02cc04fa4f7b..ac4e3813fbba 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -700,6 +700,7 @@ cc_library(
         ":diagnostic_defs_gen",
         ":sema_attr_gen",
         ":support",
+        "//llvm:BinaryFormat",
         "//llvm:Core",
         "//llvm:FrontendDebug",
         "//llvm:FrontendDriver",
@@ -1828,6 +1829,7 @@ cc_library(
         "//llvm:Target",
         "//llvm:TargetParser",
         "//llvm:TransformUtils",
+        "//llvm:config",
     ],
 )
 
@@ -2343,6 +2345,7 @@ cc_binary(
     stamp = 0,
     deps = [
         ":basic",
+        ":config",
         ":frontend",
         ":interpreter",
         ":lex",
@@ -2351,8 +2354,10 @@ cc_binary(
         "//llvm:Core",
         "//llvm:LineEditor",
         "//llvm:Option",
+        "//llvm:OrcDebugging",
         "//llvm:OrcJIT",
         "//llvm:Support",
+        "//llvm:TargetParser",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 8e629270c89d..d3fc6912cd4e 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -9,6 +9,7 @@ load("@rules_python//python:defs.bzl", "py_binary")
 load(
     ":libc_build_rules.bzl",
     "libc_function",
+    "libc_generated_header",
     "libc_header_library",
     "libc_math_function",
     "libc_support_library",
@@ -88,6 +89,14 @@ py_binary(
     srcs = glob(["utils/hdrgen/hdrgen/**/*.py"]),
     imports = ["utils/hdrgen"],
     main = "utils/hdrgen/hdrgen/main.py",
+    deps = ["@pyyaml//:yaml"],
+)
+
+libc_generated_header(
+    name = "include_stdbit_h",
+    hdr = "staging/include/stdbit.h",
+    other_srcs = ["include/stdbit.h.def"],
+    yaml_template = "include/stdbit.yaml",
 )
 
 # Library containing all headers that can be transitively included by generated llvm-libc
@@ -259,6 +268,19 @@ libc_support_library(
     hdrs = ["hdr/unistd_overlay.h"],
 )
 
+libc_support_library(
+    name = "hdr_wchar_macros",
+    hdrs = ["hdr/wchar_macros.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
+libc_support_library(
+    name = "hdr_wchar_overlay",
+    hdrs = ["hdr/wchar_overlay.h"],
+)
+
 ############################ Type Proxy Header Files ###########################
 
 libc_support_library(
@@ -423,6 +445,22 @@ libc_support_library(
     hdrs = ["hdr/types/struct_msghdr.h"],
 )
 
+libc_support_library(
+    name = "types_wchar_t",
+    hdrs = ["hdr/types/wchar_t.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
+libc_support_library(
+    name = "types_wint_t",
+    hdrs = ["hdr/types/wint_t.h"],
+    deps = [
+        ":hdr_wchar_overlay",
+    ],
+)
+
 ############################### Support libraries ##############################
 
 libc_support_library(
@@ -714,6 +752,7 @@ libc_support_library(
     name = "__support_blockstore",
     hdrs = ["src/__support/blockstore.h"],
     deps = [
+        ":__support_cpp_array",
         ":__support_cpp_new",
         ":__support_libc_assert",
     ],
@@ -1636,6 +1675,17 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_wctype_utils",
+    hdrs = ["src/__support/wctype_utils.h"],
+    deps = [
+        ":__support_cpp_optional",
+        ":__support_macros_attributes",
+        ":__support_macros_config",
+        ":types_wint_t",
+    ],
+)
+
 ########################## externally shared targets ###########################
 
 libc_header_library(
@@ -4218,6 +4268,7 @@ libc_support_library(
         "src/string/memory_utils/aarch64/inline_memcpy.h",
         "src/string/memory_utils/aarch64/inline_memmove.h",
         "src/string/memory_utils/aarch64/inline_memset.h",
+        "src/string/memory_utils/arm/inline_memcpy.h",
         "src/string/memory_utils/generic/aligned_access.h",
         "src/string/memory_utils/generic/byte_per_byte.h",
         "src/string/memory_utils/inline_bcmp.h",
@@ -5675,3 +5726,288 @@ libc_function(
         ":types_struct_timespec",
     ],
 )
+
+############################## wchar targets ###############################
+
+libc_function(
+    name = "btowc",
+    srcs = ["src/wchar/btowc.cpp"],
+    hdrs = ["src/wchar/btowc.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_wctype_utils",
+        ":hdr_wchar_macros",
+        ":types_wint_t",
+    ],
+)
+
+libc_function(
+    name = "wcpcpy",
+    srcs = ["src/wchar/wcpcpy.cpp"],
+    hdrs = ["src/wchar/wcpcpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcpncpy",
+    srcs = ["src/wchar/wcpncpy.cpp"],
+    hdrs = ["src/wchar/wcpncpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscat",
+    srcs = ["src/wchar/wcscat.cpp"],
+    hdrs = ["src/wchar/wcscat.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcschr",
+    srcs = ["src/wchar/wcschr.cpp"],
+    hdrs = ["src/wchar/wcschr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscmp",
+    srcs = ["src/wchar/wcscmp.cpp"],
+    hdrs = ["src/wchar/wcscmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscpy",
+    srcs = ["src/wchar/wcscpy.cpp"],
+    hdrs = ["src/wchar/wcscpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcscspn",
+    srcs = ["src/wchar/wcscspn.cpp"],
+    hdrs = ["src/wchar/wcscspn.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcslen",
+    srcs = ["src/wchar/wcslen.cpp"],
+    hdrs = ["src/wchar/wcslen.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncat",
+    srcs = ["src/wchar/wcsncat.cpp"],
+    hdrs = ["src/wchar/wcsncat.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncmp",
+    srcs = ["src/wchar/wcsncmp.cpp"],
+    hdrs = ["src/wchar/wcsncmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsncpy",
+    srcs = ["src/wchar/wcsncpy.cpp"],
+    hdrs = ["src/wchar/wcsncpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcspbrk",
+    srcs = ["src/wchar/wcspbrk.cpp"],
+    hdrs = ["src/wchar/wcspbrk.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsrchr",
+    srcs = ["src/wchar/wcsrchr.cpp"],
+    hdrs = ["src/wchar/wcsrchr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsspn",
+    srcs = ["src/wchar/wcsspn.cpp"],
+    hdrs = ["src/wchar/wcsspn.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wcsstr",
+    srcs = ["src/wchar/wcsstr.cpp"],
+    hdrs = ["src/wchar/wcsstr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":string_utils",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wctob",
+    srcs = ["src/wchar/wctob.cpp"],
+    hdrs = ["src/wchar/wctob.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":__support_wctype_utils",
+        ":hdr_stdio_macros",
+        ":types_wint_t",
+    ],
+)
+
+libc_function(
+    name = "wmemchr",
+    srcs = ["src/wchar/wmemchr.cpp"],
+    hdrs = ["src/wchar/wmemchr.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemcmp",
+    srcs = ["src/wchar/wmemcmp.cpp"],
+    hdrs = ["src/wchar/wmemcmp.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemcpy",
+    srcs = ["src/wchar/wmemcpy.cpp"],
+    hdrs = ["src/wchar/wmemcpy.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_config",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemmove",
+    srcs = ["src/wchar/wmemmove.cpp"],
+    hdrs = ["src/wchar/wmemmove.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmempcpy",
+    srcs = ["src/wchar/wmempcpy.cpp"],
+    hdrs = ["src/wchar/wmempcpy.h"],
+    deps = [
+        ":__support_common",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
+
+libc_function(
+    name = "wmemset",
+    srcs = ["src/wchar/wmemset.cpp"],
+    hdrs = ["src/wchar/wmemset.h"],
+    deps = [
+        ":__support_common",
+        ":types_size_t",
+        ":types_wchar_t",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
index 7fe263ab08f7..e123e4d32b78 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_build_rules.bzl
@@ -248,6 +248,36 @@ def libc_header_library(name, hdrs, deps = [], **kwargs):
         **kwargs
     )
 
+def libc_generated_header(name, hdr, yaml_template, other_srcs = []):
+    """Generates a libc header file from YAML template.
+
+    Args:
+      name: Name of the genrule target.
+      hdr: Path of the header file to generate.
+      yaml_template: Path of the YAML template file.
+      other_srcs: Other files required to generate the header, if any.
+    """
+    hdrgen = "//libc:hdrgen"
+    cmd = "$(location {hdrgen}) $(location {yaml}) -o $@".format(
+        hdrgen = hdrgen,
+        yaml = yaml_template,
+    )
+
+    if not hdr.startswith("staging/"):
+        fail(
+            "Generated headers should be placed in a 'staging/' directory " +
+            "so that they can be treated differently from constant source files " +
+            "when bootstrapping builds.",
+        )
+
+    native.genrule(
+        name = name,
+        outs = [hdr],
+        srcs = [yaml_template] + other_srcs,
+        cmd = cmd,
+        tools = [hdrgen],
+    )
+
 def libc_math_function(
         name,
         additional_deps = None):
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 2c5ad7d27ce8..090bd1b1ba7a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -72,6 +72,9 @@ libc_test_library(
         "//libc:llvm_libc_macros_stdfix_macros",
         "//llvm:Support",
     ],
+    # Force linking in this library's `main()` to surface
+    # a duplicate symbol error if a test defines its own main.
+    alwayslink = True,
 )
 
 libc_test_library(
diff --git a/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel
index 51e8424282af..9be937e37a2a 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/include/BUILD.bazel
@@ -10,13 +10,375 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
+libc_test(
+    name = "assert_test",
+    srcs = ["assert_test.cpp"],
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "complex_test",
+    srcs = ["complex_test.cpp"],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "fpclassify_test",
+    srcs = [
+        "FpClassifyTest.h",
+        "fpclassify_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "fpclassifyf_test",
+    srcs = [
+        "FpClassifyTest.h",
+        "fpclassifyf_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "fpclassifyl_test",
+    srcs = [
+        "FpClassifyTest.h",
+        "fpclassifyl_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "fpclassify_c_test",
+    srcs = ["fpclassify_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "isfinite_test",
+    srcs = [
+        "IsFiniteTest.h",
+        "isfinite_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isfinitef_test",
+    srcs = [
+        "IsFiniteTest.h",
+        "isfinitef_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isfinitel_test",
+    srcs = [
+        "IsFiniteTest.h",
+        "isfinitel_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isfinite_c_test",
+    srcs = ["isfinite_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "isinf_test",
+    srcs = [
+        "IsInfTest.h",
+        "isinf_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isinff_test",
+    srcs = [
+        "IsInfTest.h",
+        "isinff_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isinfl_test",
+    srcs = [
+        "IsInfTest.h",
+        "isinfl_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isinf_c_test",
+    srcs = ["isinf_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "isnan_test",
+    srcs = [
+        "IsNanTest.h",
+        "isnan_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnanf_test",
+    srcs = [
+        "IsNanTest.h",
+        "isnanf_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnanl_test",
+    srcs = [
+        "IsNanTest.h",
+        "isnanl_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnan_c_test",
+    srcs = ["isnan_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "isnormal_test",
+    srcs = [
+        "IsNormalTest.h",
+        "isnormal_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnormalf_test",
+    srcs = [
+        "IsNormalTest.h",
+        "isnormalf_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnormall_test",
+    srcs = [
+        "IsNormalTest.h",
+        "isnormall_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "isnormal_c_test",
+    srcs = ["isnormal_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "issubnormal_c_test",
+    srcs = ["issubnormal_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "iszero_test",
+    srcs = [
+        "IsZeroTest.h",
+        "iszero_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "iszerof_test",
+    srcs = [
+        "IsZeroTest.h",
+        "iszerof_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "iszerol_test",
+    srcs = [
+        "IsZeroTest.h",
+        "iszerol_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "iszero_c_test",
+    srcs = ["iszero_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "signbit_test",
+    srcs = [
+        "SignbitTest.h",
+        "signbit_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "signbitf_test",
+    srcs = [
+        "SignbitTest.h",
+        "signbitf_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
+libc_test(
+    name = "signbitl_test",
+    srcs = [
+        "SignbitTest.h",
+        "signbitl_test.cpp",
+    ],
+    full_build = True,
+    deps = [
+        "//libc:public_headers_deps",
+        "//libc/test/UnitTest:fp_test_helpers",
+    ],
+)
+
 libc_test(
     name = "stdbit_test",
     srcs = [
         "stdbit_stub.h",
         "stdbit_test.cpp",
     ],
+    full_build = True,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "signbit_c_test",
+    srcs = ["signbit_test.c"],
+    use_test_framework = False,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "stdckdint_test",
+    srcs = ["stdckdint_test.cpp"],
+    full_build = True,
+    deps = ["//libc:public_headers_deps"],
+)
+
+libc_test(
+    name = "sys_queue_test",
+    srcs = ["sys/queue_test.cpp"],
+    full_build = True,
     deps = [
+        "//libc:__support_char_vector",
+        "//libc:__support_cpp_string",
         "//libc:public_headers_deps",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
index ba8753a17a85..41a706d2d92f 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/test/libc_test_rules.bzl
@@ -15,7 +15,21 @@ When performing tests we make sure to always use the internal version.
 load("//libc:libc_build_rules.bzl", "libc_common_copts")
 load("//libc:libc_configure_options.bzl", "LIBC_CONFIGURE_OPTIONS")
 
-def libc_test(name, copts = [], deps = [], local_defines = [], **kwargs):
+_FULL_BUILD_COPTS = [
+    "-nostdlib++",
+    "-nostdlib",
+    "-DLIBC_FULL_BUILD",
+    "-DLIBC_COPT_USE_C_ASSERT",
+]
+
+def libc_test(
+        name,
+        copts = [],
+        deps = [],
+        local_defines = [],
+        use_test_framework = True,
+        full_build = False,
+        **kwargs):
     """Add target for a libc test.
 
     Args:
@@ -23,21 +37,31 @@ def libc_test(name, copts = [], deps = [], local_defines = [], **kwargs):
       copts: The list of options to add to the C++ compilation command.
       deps: The list of libc functions and libraries to be linked in.
       local_defines: The list of target local_defines if any.
+      use_test_framework: Whether to use the libc unit test `main` function.
+      full_build: Whether to compile with LIBC_FULL_BUILD and disallow
+          use of system headers. This is useful for tests that include both
+          LLVM libc headers and proxy headers to avoid conflicting definitions.
       **kwargs: Attributes relevant for a cc_test.
     """
+    deps = deps + [
+        "//libc:__support_macros_config",
+        "//libc:__support_libc_errno",
+        "//libc:errno",
+        "//libc:func_aligned_alloc",
+        "//libc:func_free",
+        "//libc:func_malloc",
+        "//libc:func_realloc",
+    ]
+    if use_test_framework:
+        deps = deps + ["//libc/test/UnitTest:LibcUnitTest"]
+
+    if full_build:
+        copts = copts + _FULL_BUILD_COPTS
+
     native.cc_test(
         name = name,
         local_defines = local_defines + LIBC_CONFIGURE_OPTIONS,
-        deps = [
-            "//libc/test/UnitTest:LibcUnitTest",
-            "//libc:__support_macros_config",
-            "//libc:__support_libc_errno",
-            "//libc:errno",
-            "//libc:func_aligned_alloc",
-            "//libc:func_free",
-            "//libc:func_malloc",
-            "//libc:func_realloc",
-        ] + deps,
+        deps = deps,
         copts = copts + libc_common_copts(),
         linkstatic = 1,
         **kwargs
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel
new file mode 100644
index 000000000000..f94065237ef4
--- /dev/null
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/wchar/BUILD.bazel
@@ -0,0 +1,224 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Tests for LLVM libc wchar.h functions.
+
+load("//libc/test:libc_test_rules.bzl", "libc_test")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+libc_test(
+    name = "btowc_test",
+    srcs = ["btowc_test.cpp"],
+    deps = [
+        "//libc:btowc",
+        "//libc:hdr_wchar_macros",
+    ],
+)
+
+libc_test(
+    name = "wcpcpy_test",
+    srcs = ["wcpcpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcpcpy",
+    ],
+)
+
+libc_test(
+    name = "wcpncpy_test",
+    srcs = ["wcpncpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcpncpy",
+    ],
+)
+
+libc_test(
+    name = "wcscat_test",
+    srcs = ["wcscat_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscat",
+    ],
+)
+
+libc_test(
+    name = "wcschr_test",
+    srcs = ["wcschr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcschr",
+    ],
+)
+
+libc_test(
+    name = "wcscmp_test",
+    srcs = ["wcscmp_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscmp",
+    ],
+)
+
+libc_test(
+    name = "wcscpy_test",
+    srcs = ["wcscpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcscpy",
+    ],
+)
+
+libc_test(
+    name = "wcscspn_test",
+    srcs = ["wcscspn_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcscspn",
+    ],
+)
+
+libc_test(
+    name = "wcslen_test",
+    srcs = ["wcslen_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcslen",
+    ],
+)
+
+libc_test(
+    name = "wcsncat_test",
+    srcs = ["wcsncat_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncat",
+    ],
+)
+
+libc_test(
+    name = "wcsncmp_test",
+    srcs = ["wcsncmp_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncmp",
+    ],
+)
+
+libc_test(
+    name = "wcsncpy_test",
+    srcs = ["wcsncpy_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsncpy",
+    ],
+)
+
+libc_test(
+    name = "wcsrchr_test",
+    srcs = ["wcsrchr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsrchr",
+    ],
+)
+
+libc_test(
+    name = "wcspbrk_test",
+    srcs = ["wcspbrk_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcspbrk",
+    ],
+)
+
+libc_test(
+    name = "wcsspn_test",
+    srcs = ["wcsspn_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wcsspn",
+    ],
+)
+
+libc_test(
+    name = "wcsstr_test",
+    srcs = ["wcsstr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wcsstr",
+    ],
+)
+
+libc_test(
+    name = "wctob_test",
+    srcs = ["wctob_test.cpp"],
+    deps = [
+        "//libc:hdr_stdio_macros",
+        "//libc:wctob",
+    ],
+)
+
+libc_test(
+    name = "wmemchr_test",
+    srcs = ["wmemchr_test.cpp"],
+    deps = [
+        "//libc:types_wchar_t",
+        "//libc:wmemchr",
+    ],
+)
+
+libc_test(
+    name = "wmemcmp_test",
+    srcs = ["wmemcmp_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemcmp",
+    ],
+)
+
+libc_test(
+    name = "wmemcpy_test",
+    srcs = ["wmemcpy_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemcpy",
+    ],
+)
+
+libc_test(
+    name = "wmemmove_test",
+    srcs = ["wmemmove_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemmove",
+    ],
+)
+
+libc_test(
+    name = "wmempcpy_test",
+    srcs = ["wmempcpy_test.cpp"],
+    deps = [
+        "//libc:wmempcpy",
+    ],
+)
+
+libc_test(
+    name = "wmemset_test",
+    srcs = ["wmemset_test.cpp"],
+    deps = [
+        "//libc:types_size_t",
+        "//libc:types_wchar_t",
+        "//libc:wmemset",
+    ],
+)
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 6ec7cb5f2377..202f224b0798 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -307,6 +307,7 @@ cc_library(
         "//llvm:BinaryFormat",
         "//llvm:Core",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:ExecutionEngine",
         "//llvm:Support",
     ],
@@ -454,6 +455,7 @@ cc_library(
         ":UtilityPrivateHeaders",
         "//llvm:DebugInfo",
         "//llvm:DebugInfoDWARF",
+        "//llvm:DebugInfoDWARFLowLevel",
         "//llvm:Support",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index a2108e24d908..caf609fd172e 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -2127,6 +2127,7 @@ cc_library(
         ":PluginProcessUtility",
         "//lldb:Breakpoint",
         "//lldb:Core",
+        "//lldb:Headers",
         "//lldb:Host",
         "//lldb:Symbol",
         "//lldb:SymbolHeaders",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index d5a7fdb747f8..e3489a15db36 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -655,6 +655,7 @@ cc_binary(
         "utils/TableGen/Basic/DirectiveEmitter.cpp",
         "utils/TableGen/Basic/IntrinsicEmitter.cpp",
         "utils/TableGen/Basic/RISCVTargetDefEmitter.cpp",
+        "utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp",
         "utils/TableGen/Basic/SDNodeProperties.cpp",
         "utils/TableGen/Basic/SDNodeProperties.h",
         "utils/TableGen/Basic/SequenceToOffsetTable.h",
@@ -758,6 +759,14 @@ gentbl_cc_library(
     deps = [":CommonTargetTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "runtime_libcalls_gen",
+    tbl_outs = {"include/llvm/IR/RuntimeLibcalls.inc": ["-gen-runtime-libcalls"]},
+    tblgen = ":llvm-min-tblgen",
+    td_file = "include/llvm/IR/RuntimeLibcalls.td",
+    deps = [":IRTdFiles"],
+)
+
 gentbl_cc_library(
     name = "vt_gen",
     tbl_outs = {"include/llvm/CodeGen/GenVT.inc": ["-gen-vt"]},
@@ -960,11 +969,12 @@ cc_library(
         "include/llvm/Analysis/SimplifyQuery.h",
         "include/llvm/Analysis/ValueTracking.h",
         "include/llvm/Analysis/WithCache.h",
+        "include/llvm/ProfileData/InstrProf.h",
+        "include/llvm/ProfileData/InstrProfData.inc",
     ] + [":llvm_intrinsics_headers"],
     copts = llvm_copts,
     textual_hdrs = glob([
         "include/llvm/IR/*.def",
-        "lib/IR/*.def",
     ]),
     deps = [
         ":BinaryFormat",
@@ -976,6 +986,7 @@ cc_library(
         ":config",
         ":intrinsic_enums_gen",
         ":intrinsics_impl_gen",
+        ":runtime_libcalls_gen",
     ],
 )
 
@@ -1682,6 +1693,7 @@ cc_library(
     ]),
     copts = llvm_copts,
     deps = [
+        ":BinaryFormat",
         ":Core",
         ":Support",
     ],
@@ -5127,6 +5139,11 @@ cc_binary(
     ],
 )
 
+filegroup(
+    name = "gdb-scripts",
+    srcs = glob(["utils/gdb-scripts/*"]),
+)
+
 ################################################################################
 # Begin testonly libraries
 
diff --git a/utils/bazel/llvm-project-overlay/llvm/config.bzl b/utils/bazel/llvm-project-overlay/llvm/config.bzl
index 7cb4b7e9ffe7..ba9db05c651a 100644
--- a/utils/bazel/llvm-project-overlay/llvm/config.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/config.bzl
@@ -32,6 +32,7 @@ posix_defines = [
     "BACKTRACE_HEADER=<execinfo.h>",
     r'LTDL_SHLIB_EXT=\".so\"',
     r'LLVM_PLUGIN_EXT=\".so\"',
+    "LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS=1",
     "LLVM_ENABLE_PLUGINS=1",
     "LLVM_ENABLE_THREADS=1",
     "HAVE_DEREGISTER_FRAME=1",
diff --git a/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h b/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
index 8a9c74d67b12..31d0dc57a718 100644
--- a/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
+++ b/utils/bazel/llvm-project-overlay/llvm/include/llvm/Config/llvm-config.h
@@ -114,6 +114,9 @@
 /* Define if building LLVM with BUILD_SHARED_LIBS */
 /* #undef LLVM_BUILD_SHARED_LIBS */
 
+/* Define if exporting LLVM public interface for shared library */
+/* LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS defined in Bazel */
+
 /* Define if building LLVM with LLVM_FORCE_USE_OLD_TOOLCHAIN_LIBS */
 /* #undef LLVM_FORCE_USE_OLD_TOOLCHAIN ${LLVM_FORCE_USE_OLD_TOOLCHAIN} */
 
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index ec691bc8dd28..633776398649 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -30,6 +30,11 @@ exports_files([
     "utils/textmate/mlir.json",
 ])
 
+filegroup(
+    name = "gdb-scripts",
+    srcs = glob(["utils/gdb-scripts/*"]),
+)
+
 bool_flag(
     name = "enable_cuda",
     build_setting_default = False,
@@ -3519,11 +3524,8 @@ gentbl_cc_library(
 
 cc_library(
     name = "XeGPUDialect",
-    srcs = [
-        "lib/Dialect/XeGPU/IR/XeGPUDialect.cpp",
-        "lib/Dialect/XeGPU/IR/XeGPUOps.cpp",
-    ],
-    hdrs = ["include/mlir/Dialect/XeGPU/IR/XeGPU.h"],
+    srcs = glob(["lib/Dialect/XeGPU/IR/*.cpp"]),
+    hdrs = glob(["include/mlir/Dialect/XeGPU/IR/*.h"]),
     includes = ["include"],
     deps = [
         ":ArithDialect",
@@ -3686,6 +3688,7 @@ cc_library(
     ]),
     includes = ["include"],
     deps = [
+        ":ArithDialect",
         ":DialectUtilsIncGen",
         ":IR",
         ":Support",
@@ -5073,6 +5076,8 @@ cc_library(
             "lib/Dialect/LLVMIR/IR/VCIX*.h",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.cpp",
             "lib/Dialect/LLVMIR/IR/*X86Vector*.h",
+            "lib/Dialect/LLVMIR/IR/XeVM*.cpp",
+            "lib/Dialect/LLVMIR/IR/XeVM*.h",
         ],
     ),
     hdrs = glob(
@@ -5085,6 +5090,7 @@ cc_library(
             "include/mlir/Dialect/LLVMIR/ROCDL*.h",
             "include/mlir/Dialect/LLVMIR/VCIX*.h",
             "include/mlir/Dialect/LLVMIR/*X86Vector*.h",
+            "include/mlir/Dialect/LLVMIR/XeVM*.h",
         ],
     ),
     includes = ["include"],
@@ -9067,6 +9073,7 @@ cc_library(
         ":X86VectorTransforms",
         ":XeGPUDialect",
         ":XeGPUTransforms",
+        ":XeVMDialect",
     ],
 )
 
@@ -10169,6 +10176,7 @@ td_library(
     ]),
     includes = ["include"],
     deps = [
+        ":LinalgOpsTdFiles",
         ":SCFTdFiles",
         ":TransformDialectTdFiles",
     ],
@@ -10478,6 +10486,7 @@ cc_library(
         ":DestinationStyleOpInterface",
         ":DialectUtils",
         ":IR",
+        ":IndexingMapOpInterface",
         ":InferTypeOpInterface",
         ":LinalgInterfacesIncGen",
         ":LinalgStructuredOpsIncGen",
@@ -11171,6 +11180,7 @@ cc_library(
         ":Dialect",
         ":DialectUtils",
         ":FuncDialect",
+        ":FuncTransforms",
         ":IR",
         ":InferTypeOpInterface",
         ":InliningUtils",
@@ -12361,6 +12371,7 @@ cc_library(
         ":BufferizationDialect",
         ":BufferizationInterfaces",
         ":ControlFlowDialect",
+        ":DestinationStyleOpInterface",
         ":DialectUtils",
         ":FuncDialect",
         ":GPUDialect",
@@ -12874,6 +12885,7 @@ cc_library(
         ":AffineDialect",
         ":AllocationOpInterface",
         ":ArithDialect",
+        ":ArithUtils",
         ":BufferDeallocationOpInterfaceIncGen",
         ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
@@ -13528,3 +13540,60 @@ cc_library(
         "//llvm:Support",
     ],
 )
+
+cc_library(
+    name = "XeVMDialect",
+    srcs = ["lib/Dialect/LLVMIR/IR/XeVMDialect.cpp"],
+    hdrs = ["include/mlir/Dialect/LLVMIR/XeVMDialect.h"],
+    includes = ["include"],
+    deps = [
+        ":BytecodeOpInterface",
+        ":DialectUtils",
+        ":GPUDialect",
+        ":IR",
+        ":LLVMDialect",
+        ":ToLLVMIRTranslation",
+        ":XeVMOpsIncGen",
+        "//llvm:Support",
+    ],
+)
+
+td_library(
+    name = "XeVMTdFiles",
+    srcs = ["include/mlir/Dialect/LLVMIR/XeVMOps.td"],
+    includes = ["include"],
+    deps = [
+        ":GPUOpsTdFiles",
+        ":LLVMOpsTdFiles",
+        ":OpBaseTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "XeVMOpsIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/LLVMIR/XeVMOps.h.inc": ["-gen-op-decls"],
+        "include/mlir/Dialect/LLVMIR/XeVMOps.cpp.inc": ["-gen-op-defs"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsDialect.h.inc": [
+            "-gen-dialect-decls",
+            "-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsDialect.cpp.inc": [
+            "-gen-dialect-defs",
+            "-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsEnums.h.inc": ["-gen-enum-decls"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsEnums.cpp.inc": ["-gen-enum-defs"],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsAttributes.h.inc": [
+            "-gen-attrdef-decls",
+            "-attrdefs-dialect=xevm",
+        ],
+        "include/mlir/Dialect/LLVMIR/XeVMOpsAttributes.cpp.inc": [
+            "-gen-attrdef-defs",
+            "-attrdefs-dialect=xevm",
+        ],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/LLVMIR/XeVMOps.td",
+    deps = [":XeVMTdFiles"],
+)
diff --git a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
index be5bbb043412..1d00afb4e8a2 100644
--- a/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/python/BUILD.bazel
@@ -674,6 +674,32 @@ filegroup(
     ],
 )
 
+##---------------------------------------------------------------------------##
+# Debug dialect.
+##---------------------------------------------------------------------------##
+
+gentbl_filegroup(
+    name = "DebugTransformOpsPyGen",
+    tbl_outs = {"mlir/dialects/_transform_debug_extension_ops_gen.py": [
+        "-gen-python-op-bindings",
+        "-bind-dialect=transform",
+        "-dialect-extension=transform_debug_extension",
+    ]},
+    tblgen = "//mlir:mlir-tblgen",
+    td_file = "mlir/dialects/TransformDebugExtensionOps.td",
+    deps = [
+        "//mlir:TransformDebugExtensionTdFiles",
+    ],
+)
+
+filegroup(
+    name = "DebugPyFiles",
+    srcs = [
+        "mlir/dialects/transform/debug.py",
+        ":DebugTransformOpsPyGen",
+    ],
+)
+
 ##---------------------------------------------------------------------------##
 # PythonTest dialect.
 ##---------------------------------------------------------------------------##
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 90e0f1b8ca74..95e3ee4df7bc 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -613,6 +613,7 @@ cc_library(
         "//mlir:AffineDialect",
         "//mlir:Analysis",
         "//mlir:ArithDialect",
+        "//mlir:DialectUtils",
         "//mlir:FuncDialect",
         "//mlir:IR",
         "//mlir:InliningUtils",
diff --git a/utils/bazel/llvm_configs/llvm-config.h.cmake b/utils/bazel/llvm_configs/llvm-config.h.cmake
index a0ad517a6ecf..0d8db0b6b52f 100644
--- a/utils/bazel/llvm_configs/llvm-config.h.cmake
+++ b/utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -110,6 +110,9 @@
 /* Define if building LLVM with BUILD_SHARED_LIBS */
 #cmakedefine LLVM_BUILD_SHARED_LIBS
 
+/* Define if exporting LLVM public interface for shared library */
+#cmakedefine LLVM_ENABLE_LLVM_EXPORT_ANNOTATIONS
+
 /* Define if building LLVM with LLVM_FORCE_USE_OLD_TOOLCHAIN_LIBS */
 #cmakedefine LLVM_FORCE_USE_OLD_TOOLCHAIN ${LLVM_FORCE_USE_OLD_TOOLCHAIN}
 
diff --git a/utils/bazel/third_party_build/pyyaml.BUILD b/utils/bazel/third_party_build/pyyaml.BUILD
new file mode 100644
index 000000000000..78261a7f60d9
--- /dev/null
+++ b/utils/bazel/third_party_build/pyyaml.BUILD
@@ -0,0 +1,16 @@
+# This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@rules_python//python:defs.bzl", "py_library")
+
+package(
+    default_visibility = ["//visibility:public"],
+    # BSD/MIT-like license (for PyYAML)
+    licenses = ["notice"],
+)
+
+py_library(
+    name = "yaml",
+    srcs = glob(["yaml/*.py"]),
+)

From 9590740aa78bf7d51021c46e2bbb1b8330e37307 Mon Sep 17 00:00:00 2001
From: Djordje Antic <djoantic@amd.com>
Date: Wed, 30 Jul 2025 14:58:48 +0000
Subject: [PATCH 2/6] Fix build issues

---
 mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp   |  3 ++-
 .../Rock/Transforms/ThreadwiseGemmLowering.cpp      | 13 +++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp b/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
index 29c64e4e3d18..c13a0922cade 100644
--- a/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
+++ b/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
@@ -1500,7 +1500,8 @@ struct InBoundsLoadRewritePattern : public OpRewritePattern<InBoundsLoadOp> {
     if (auto destType = dyn_cast<VectorType>(op.getResult().getType())) {
       b.replaceOpWithNewOp<vector::TransferReadOp>(
           op, destType, op.getSource(), op.getCoords(),
-          /*inbounds=*/ArrayRef<bool>(true));
+          std::nullopt,
+          /*inbounds=*/std::optional{ArrayRef<bool>{true}});
     } else {
       b.replaceOpWithNewOp<memref::LoadOp>(op, op.getSource(), op.getCoords());
     }
diff --git a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
index 225d642af246..f0e74f6cfa3b 100644
--- a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
+++ b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
@@ -44,6 +44,7 @@
 #include <iterator>
 #include <memory>
 #include <numeric>
+#include <optional>
 
 namespace mlir {
 namespace rock {
@@ -167,10 +168,10 @@ struct ThreadwiseGemmRewritePattern
       // These are vector::TransferRead ops so they always return a vector
       // result so that FMA doesn't complain
       Value aVal = b.create<vector::TransferReadOp>(
-          loc, abType, bufferA, gemmLoop.getLowerCoords(/*domain=*/0),
+          loc, abType, bufferA, gemmLoop.getLowerCoords(/*domain=*/0), std::nullopt,
           /*inBounds=*/ArrayRef<bool>(true));
       Value bVal = b.create<vector::TransferReadOp>(
-          loc, abType, bufferB, gemmLoop.getLowerCoords(/*domain=*/1),
+          loc, abType, bufferB, gemmLoop.getLowerCoords(/*domain=*/1), std::nullopt,
           /*inBounds=*/ArrayRef<bool>(true));
       ValueRange cCoords = gemmLoop.getLowerCoords(/*domain=*/2);
       Value cVal = b.create<InBoundsLoadOp>(loc, dataType, bufferC, cCoords);
@@ -196,6 +197,9 @@ struct ThreadwiseGemmRewritePattern
 
       b.create<InBoundsStoreOp>(loc, result, bufferC, cCoords);
     }
+    LLVM_DEBUG(llvm::dbgs() << "debug before\n");
+    b.eraseOp(op);
+    LLVM_DEBUG(llvm::dbgs() << "debug 202\n");
     return success();
   }
 };
@@ -840,10 +844,11 @@ void RockThreadwiseGemmLoweringPass::runOnOperation() {
   }
 
   ConversionTarget target(*ctx);
-  target.addIllegalOp<rock::ThreadwiseGemmOp, rock::ThreadwiseAccelGemmOp>();
+  target.addIllegalOp<rock::ThreadwiseAccelGemmOp>();
   target.addLegalDialect<amdgpu::AMDGPUDialect, arith::ArithDialect,
                          rock::RockDialect, affine::AffineDialect,
-                         memref::MemRefDialect, vector::VectorDialect>();
+                         memref::MemRefDialect, vector::VectorDialect,
+                         scf::SCFDialect>();
   target.addLegalOp<gpu::PrintfOp>();
 
   RewritePatternSet patterns(ctx);

From b33257defd61bdc1ded57530918f4cc3ef647163 Mon Sep 17 00:00:00 2001
From: Daniel Hernandez <danherna@amd.com>
Date: Mon, 11 Aug 2025 07:34:34 +0000
Subject: [PATCH 3/6] Fix for ThreadwiseGemmLowering: vector::TransferReadOp
 constructor uses PoisonOp and changes needed for test_rock_pipeline.mlir test

---
 .../Transforms/ThreadwiseGemmLowering.cpp     | 12 ++--
 .../test/Dialect/Rock/test_rock_pipeline.mlir | 71 +++++++------------
 2 files changed, 31 insertions(+), 52 deletions(-)

diff --git a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
index f0e74f6cfa3b..d3f5fb5b7532 100644
--- a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
+++ b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
@@ -34,6 +34,7 @@
 
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/Passes.h"
@@ -197,9 +198,6 @@ struct ThreadwiseGemmRewritePattern
 
       b.create<InBoundsStoreOp>(loc, result, bufferC, cCoords);
     }
-    LLVM_DEBUG(llvm::dbgs() << "debug before\n");
-    b.eraseOp(op);
-    LLVM_DEBUG(llvm::dbgs() << "debug 202\n");
     return success();
   }
 };
@@ -844,12 +842,12 @@ void RockThreadwiseGemmLoweringPass::runOnOperation() {
   }
 
   ConversionTarget target(*ctx);
-  target.addIllegalOp<rock::ThreadwiseAccelGemmOp>();
+  target.addIllegalOp<rock::ThreadwiseGemmOp, rock::ThreadwiseAccelGemmOp>();
   target.addLegalDialect<amdgpu::AMDGPUDialect, arith::ArithDialect,
                          rock::RockDialect, affine::AffineDialect,
-                         memref::MemRefDialect, vector::VectorDialect,
-                         scf::SCFDialect>();
-  target.addLegalOp<gpu::PrintfOp>();
+                         memref::MemRefDialect, vector::VectorDialect>();
+  // vector::TransferReadOp constructor uses poison
+  target.addLegalOp<gpu::PrintfOp, ub::PoisonOp>();
 
   RewritePatternSet patterns(ctx);
   patterns.add<ThreadwiseGemmRewritePattern, ThreadwiseAccelGemmRewritePattern>(
diff --git a/mlir/test/Dialect/Rock/test_rock_pipeline.mlir b/mlir/test/Dialect/Rock/test_rock_pipeline.mlir
index 0aeecfab8577..c4f791144e03 100644
--- a/mlir/test/Dialect/Rock/test_rock_pipeline.mlir
+++ b/mlir/test/Dialect/Rock/test_rock_pipeline.mlir
@@ -19,20 +19,15 @@ func.func @rock_pipeline_3_stages_ii_1(%input : memref<16xi8, #gpu.address_space
     // CHECK: %[[rawRegA:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawRegB:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
 
-    // CHECK: %[[lds0View:.*]] = memref.view {{.*}}
-    // CHECK: %[[lds1View:.*]] = memref.view {{.*}}
-    // CHECK: %[[rawRegAView:.*]] = memref.view {{.*}}
-    // CHECK: %[[rawRegBView:.*]] = memref.view {{.*}}
-
     // CHECK: name = "S0"
     // CHECK: name = "S1"
     // CHECK: name = "S0"
     // CHECK: scf.for
       // CHECK: name = "__fwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[lds0View]], %[[lds1View]])
+      // CHECK: rock.extract_multibuffer(%[[lds0]], %[[lds1]])
       // CHECK: name = "S1"
       // CHECK: name = "S0"
-      // CHECK: rock.extract_multibuffer(%[[lds0View]], %[[lds1View]])
+      // CHECK: rock.extract_multibuffer(%[[lds0]], %[[lds1]])
       // CHECK: name = "S2"
     // CHECK: name = "__fwd_barrier__"
     // CHECK: name = "S1"
@@ -81,22 +76,19 @@ func.func @rock_pipeline_3_stages_ii_2(%input : memref<16xi8, #gpu.address_space
     // CHECK: %[[rawLds:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<workgroup>>
     // CHECK: %[[rawRegA:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawRegB:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
-    // CHECK: %[[ldsView:.*]] = memref.view %[[rawLds]]
-    // CHECK: %[[regAView:.*]] = memref.view %[[rawRegA]]
-    // CHECK: %[[regBView:.*]] = memref.view %[[rawRegB]]
 
     // CHECK: name = "S0"
     // CHECK: name = "__bwd_barrier__"
     // CHECK: name = "S1"
     // CHECK: scf.for
       // CHECK: name = "__fwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[regAView]])
+      // CHECK: rock.extract_multibuffer(%[[rawRegA]])
       // CHECK: name = "S0"
-      // CHECK: rock.extract_multibuffer(%[[ldsView]])
+      // CHECK: rock.extract_multibuffer(%[[rawLds]])
       // CHECK: name = "S2"
       // CHECK: name = "__bwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[regAView]])
-      // CHECK: rock.extract_multibuffer(%[[ldsView]])
+      // CHECK: rock.extract_multibuffer(%[[rawRegA]])
+      // CHECK: rock.extract_multibuffer(%[[rawLds]])
       // CHECK: name = "S1"
     // CHECK: name = "__fwd_barrier__"
     // CHECK: name = "S2"
@@ -143,16 +135,14 @@ func.func @rock_pipeline_3_stages_ii_2_less_iterations(%input : memref<16xi8, #g
     // CHECK: %[[rawLds:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<workgroup>>
     // CHECK: %[[rawRegA:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawRegB:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
-    // CHECK: %[[ldsView:.*]] = memref.view %[[rawLds]]
-    // CHECK: %[[regAView:.*]] = memref.view %[[rawRegA]]
-    // CHECK: %[[regBView:.*]] = memref.view %[[rawRegB]]
-    // CHECK: rock.extract_multibuffer(%[[regAView]])
+    
+    // CHECK: rock.extract_multibuffer(%[[rawRegA]])
     // CHECK: name = "S0"
-    // CHECK: rock.extract_multibuffer(%[[regAView]])
-    // CHECK: rock.extract_multibuffer(%[[ldsView]])
+    // CHECK: rock.extract_multibuffer(%[[rawRegA]])
+    // CHECK: rock.extract_multibuffer(%[[rawLds]])
     // CHECK: name = "S1"
     // CHECK: name = "__fwd_barrier__"
-    // CHECK: rock.extract_multibuffer(%[[ldsView]])
+    // CHECK: rock.extract_multibuffer(%[[rawLds]])
     // CHECK: name = "S2"
     scf.for %arg3 = %c0 to %c1_0 step %c1 {
       rock.stage {
@@ -195,19 +185,16 @@ func.func @rock_pipeline_3_stages_ii_3(%input : memref<16xi8, #gpu.address_space
     // CHECK: %[[rawLds:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<workgroup>>
     // CHECK: %[[rawRegA:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawRegB:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
-    // CHECK: %[[ldsView:.*]] = memref.view %[[rawLds]]
-    // CHECK: %[[regAView:.*]] = memref.view %[[rawRegA]]
-    // CHECK: %[[regBView:.*]] = memref.view %[[rawRegB]]
 
     // CHECK: scf.for
       // CHECK: name = "__bwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[regAView]])
+      // CHECK: rock.extract_multibuffer(%[[rawRegA]])
       // CHECK: name = "S0"
-      // CKECK: rock.extract_multibuffer(%[[ldsView]])
-      // CHECK: rock.extract_multibuffer(%[[regAView]])
+      // CKECK: rock.extract_multibuffer(%[[rawLds]])
+      // CHECK: rock.extract_multibuffer(%[[rawRegA]])
       // CHECK: name = "S1"
       // CHECK: name = "__fwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[ldsView]])
+      // CHECK: rock.extract_multibuffer(%[[rawLds]])
       // CHECK: name = "S2"
     scf.for %arg3 = %c0 to %c16 step %c1 {
       rock.stage {
@@ -292,9 +279,7 @@ func.func @rock_pipeline_4_stages_ii_2(%input : memref<16xi8, #gpu.address_space
     // CHECK: %[[rawLds0:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<workgroup>>
     // CHECK: %[[rawLds1:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<workgroup>>
     // CHECK: %[[rawReg:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
-    // CHECK: %[[lds0View:.*]] = memref.view %[[rawLds0]]
-    // CHECK: %[[lds1View:.*]] = memref.view %[[rawLds1]]
-    // CHECK: memref.view %[[rawReg]]
+    
     // CHECK: name = "__bwd_barrier__"
     // CHECK: name = "S0"
     // CHECK: name = "__fwd_barrier__"
@@ -302,14 +287,14 @@ func.func @rock_pipeline_4_stages_ii_2(%input : memref<16xi8, #gpu.address_space
     // CHECK: scf.for
       // CHECK: name = "__bwd_barrier__"
       // CHECK: name = "__fwd_barrier__"
-      // CHECK:  rock.extract_multibuffer(%[[lds0View]], %[[lds1View]])
+      // CHECK:  rock.extract_multibuffer(%[[rawLds0]], %[[rawLds1]])
       // CHECK: name = "S0"
-      // CHECK:  rock.extract_multibuffer(%[[lds0View]], %[[lds1View]])
+      // CHECK:  rock.extract_multibuffer(%[[rawLds0]], %[[rawLds1]])
       // CHECK: name = "S2"
       // CHECK: name = "__fwd_barrier__"
       // CHECK: name = "__fwd_barrier__"
       // CHECK: name = "S1"
-      // CHECK:  rock.extract_multibuffer(%[[lds0View]], %[[lds1View]])
+      // CHECK:  rock.extract_multibuffer(%[[rawLds0]], %[[rawLds1]])
       // CHECK: name = "S3"
     // CHECK: name = "__fwd_barrier__"
     // CHECK: name = "S2"
@@ -367,11 +352,6 @@ func.func @rock_pipeline_4_stages_ii_1_i8(%input : memref<16xi8, #gpu.address_sp
     // CHECK: %[[rawReg0:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawReg1:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
     // CHECK: %[[rawReg2:.*]] = rock.alloc() : memref<16xi8, #gpu.address_space<private>>
-    // CHECK: %[[ldsView0:.*]] = memref.view %[[rawLds0]]
-    // CHECK: %[[ldsView1:.*]] = memref.view %[[rawLds1]]
-    // CHECK: %[[regView0:.*]] = memref.view %[[rawReg0]]
-    // CHECK: %[[regView1:.*]] = memref.view %[[rawReg1]]
-    // CHECK: %[[regView2:.*]] = memref.view %[[rawReg2]]
 
     // Please note how we swap S0/S1 and S2/S3 to avoid private multi-buffers
     // CHECK: name = "S0"
@@ -383,15 +363,15 @@ func.func @rock_pipeline_4_stages_ii_1_i8(%input : memref<16xi8, #gpu.address_sp
     // CHECK: name = "S2"
     // CHECK: scf.for
       // CHECK: name = "__fwd_barrier__"
-      // CHECK: rock.extract_multibuffer(%[[regView0]])
-      // CHECK: rock.extract_multibuffer(%[[ldsView0]], %[[ldsView1]])
+      // CHECK: rock.extract_multibuffer(%[[rawReg0]])
+      // CHECK: rock.extract_multibuffer(%[[rawLds0]], %[[rawLds1]])
       // CHECK: name = "S1"
-      // CHECK: rock.extract_multibuffer(%[[regView0]])
+      // CHECK: rock.extract_multibuffer(%[[rawReg0]])
       // CHECK: name = "S0"
-      // CHECK: rock.extract_multibuffer(%[[regView1]])
+      // CHECK: rock.extract_multibuffer(%[[rawReg1]])
       // CHECK: name = "S3"
-      // CHECK: rock.extract_multibuffer(%[[ldsView0]], %[[ldsView1]])
-      // CHECK: rock.extract_multibuffer(%[[regView1]])
+      // CHECK: rock.extract_multibuffer(%[[rawLds0]], %[[rawLds1]])
+      // CHECK: rock.extract_multibuffer(%[[rawReg1]])
       // CHECK: name = "S2"
     // CHECK: name = "__fwd_barrier__"
     // CHECK: name = "S1"
@@ -527,6 +507,7 @@ func.func @rock_pipeline_4_stages_ii_1_f16_less_iterations(%input : memref<16xf1
     // CHECK: %[[reg1:.*]] = rock.alloc() : memref<16xf16, #gpu.address_space<private>>
     // CHECK: %[[reg2:.*]] = rock.alloc() : memref<16xf16, #gpu.address_space<private>>
     // CHECK: %[[ldsView:.*]] = memref.view %[[rawLds]]
+
     // CHECK: name = "S0" 
     // CHECK: name = "__bwd_barrier__"
     // CHECK: name = "S1"

From 5bd8b03e2c6554f6e7ec00bd414909e84d21b459 Mon Sep 17 00:00:00 2001
From: Djordje Antic <djoantic@amd.com>
Date: Mon, 11 Aug 2025 10:56:29 +0000
Subject: [PATCH 4/6] Apply clang-format to files in the MLIR directory

---
 mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp           | 3 +--
 mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp | 6 ++++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp b/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
index c13a0922cade..3b8a8ab0c34b 100644
--- a/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
+++ b/mlir/lib/Dialect/Rock/Transforms/SugarToLoops.cpp
@@ -1499,8 +1499,7 @@ struct InBoundsLoadRewritePattern : public OpRewritePattern<InBoundsLoadOp> {
                                 PatternRewriter &b) const override {
     if (auto destType = dyn_cast<VectorType>(op.getResult().getType())) {
       b.replaceOpWithNewOp<vector::TransferReadOp>(
-          op, destType, op.getSource(), op.getCoords(),
-          std::nullopt,
+          op, destType, op.getSource(), op.getCoords(), std::nullopt,
           /*inbounds=*/std::optional{ArrayRef<bool>{true}});
     } else {
       b.replaceOpWithNewOp<memref::LoadOp>(op, op.getSource(), op.getCoords());
diff --git a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
index 69f43995db8e..b2163e23ad6c 100644
--- a/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
+++ b/mlir/lib/Dialect/Rock/Transforms/ThreadwiseGemmLowering.cpp
@@ -170,10 +170,12 @@ struct ThreadwiseGemmRewritePattern
       // These are vector::TransferRead ops so they always return a vector
       // result so that FMA doesn't complain
       Value aVal = b.create<vector::TransferReadOp>(
-          loc, abType, bufferA, gemmLoop.getLowerCoords(/*domain=*/0), std::nullopt,
+          loc, abType, bufferA, gemmLoop.getLowerCoords(/*domain=*/0),
+          std::nullopt,
           /*inBounds=*/ArrayRef<bool>(true));
       Value bVal = b.create<vector::TransferReadOp>(
-          loc, abType, bufferB, gemmLoop.getLowerCoords(/*domain=*/1), std::nullopt,
+          loc, abType, bufferB, gemmLoop.getLowerCoords(/*domain=*/1),
+          std::nullopt,
           /*inBounds=*/ArrayRef<bool>(true));
       ValueRange cCoords = gemmLoop.getLowerCoords(/*domain=*/2);
       Value cVal = b.create<InBoundsLoadOp>(loc, dataType, bufferC, cCoords);

From 4a1e4dfc7e8e2e4370603e50c429d32bcb6fe987 Mon Sep 17 00:00:00 2001
From: Djordje Antic <djoantic@amd.com>
Date: Tue, 12 Aug 2025 13:59:42 +0000
Subject: [PATCH 5/6] Add TransposeLoadOpLowering pattern

---
 .../mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 96320b4a6b8e..91dd9b4e244b 100644
--- a/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/external/llvm-project/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -1850,7 +1850,7 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
            ScaledMFMAOpLowering, WMMAOpLowering, ExtPackedFp8OpLowering,
            ScaledExtPackedOpLowering, PackedScaledTruncOpLowering,
            PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering,
-           GatherToLDSOpLowering>(converter, chipset);
+           GatherToLDSOpLowering, TransposeLoadOpLowering>(converter, chipset);
   patterns.add<LDSBarrierOpLowering>(converter, chipset, hackForDirectToLDS);
   patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
 }

From a2734cf324323cd53c6f242c7dfe59e7a1061bf8 Mon Sep 17 00:00:00 2001
From: Umang Yadav <umayadav@amd.com>
Date: Thu, 14 Aug 2025 12:48:43 +0000
Subject: [PATCH 6/6] change seqlen from 1 to 384 for accuracy checker

---
 mlir/utils/jenkins/Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/utils/jenkins/Jenkinsfile b/mlir/utils/jenkins/Jenkinsfile
index 30400b1d6d93..912ca32457c5 100644
--- a/mlir/utils/jenkins/Jenkinsfile
+++ b/mlir/utils/jenkins/Jenkinsfile
@@ -1551,8 +1551,8 @@ pipeline {
                                         dir('MIGraphX/tools/accuracy') {
                                             withEnv(['MIGRAPHX_MLIR_USE_SPECIFIC_OPS=convolution,fused,dot,attention', 'MIGRAPHX_ENABLE_MLIR_INPUT_FUSION=1', 'MIGRAPHX_ENABLE_MLIR_REDUCE_FUSION=1', 'MIGRAPHX_MLIR_ENABLE_SPLITK=1']) {
                                                 sh 'python3 accuracy_checker.py --onnx /MIGraphXDeps/resnet50-v1-7.onnx'
-                                                sh 'python3 accuracy_checker.py --fill1 --onnx /MIGraphXDeps/bert_base_cased_1.onnx'
-                                                sh 'python3 accuracy_checker.py --fill1 --onnx /MIGraphXDeps/distilgpt2_1.onnx'
+                                                sh 'python3 accuracy_checker.py --fill1 --onnx /MIGraphXDeps/bert_base_cased_1.onnx --input-dim input_ids:1,384'
+                                                sh 'python3 accuracy_checker.py --fill1 --onnx /MIGraphXDeps/distilgpt2_1.onnx --input-dim input_ids:1,384'
                                             }
                                         }
                                     }